diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index b5b2765e43844f..7490dd947e1504 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -1,11 +1,11 @@ -trigger: ['main', '3.12', '3.11', '3.10', '3.9', '3.8', '3.7'] +trigger: ['main', '3.13', '3.12', '3.11', '3.10', '3.9', '3.8'] jobs: - job: Prebuild displayName: Pre-build checks pool: - vmImage: ubuntu-22.04 + vmImage: ubuntu-24.04 steps: - template: ./prebuild-checks.yml diff --git a/.azure-pipelines/posix-deps-apt.sh b/.azure-pipelines/posix-deps-apt.sh deleted file mode 100755 index e0f4ca5d8d8e88..00000000000000 --- a/.azure-pipelines/posix-deps-apt.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -apt-get update - -apt-get -yq install \ - build-essential \ - zlib1g-dev \ - libbz2-dev \ - liblzma-dev \ - libncurses5-dev \ - libreadline6-dev \ - libsqlite3-dev \ - libssl-dev \ - libgdbm-dev \ - tk-dev \ - lzma \ - lzma-dev \ - liblzma-dev \ - libffi-dev \ - uuid-dev \ - xvfb - -if [ ! -z "$1" ] -then - echo ##vso[task.prependpath]$PWD/multissl/openssl/$1 - echo ##vso[task.setvariable variable=OPENSSL_DIR]$PWD/multissl/openssl/$1 - python3 Tools/ssl/multissltests.py --steps=library --base-directory $PWD/multissl --openssl $1 --system Linux -fi diff --git a/.azure-pipelines/posix-steps.yml b/.azure-pipelines/posix-steps.yml deleted file mode 100644 index e23c7b1dcb55c1..00000000000000 --- a/.azure-pipelines/posix-steps.yml +++ /dev/null @@ -1,26 +0,0 @@ -steps: -- checkout: self - clean: true - fetchDepth: 5 - -# Work around a known issue affecting Ubuntu VMs on Pipelines -- script: sudo setfacl -Rb /home/vsts - displayName: 'Workaround ACL issue' - -- script: sudo ./.azure-pipelines/posix-deps-apt.sh $(openssl_version) - displayName: 'Install dependencies' - -- script: ./configure --with-pydebug - displayName: 'Configure CPython (debug)' - -- script: make -j4 - displayName: 'Build CPython' - -- script: make pythoninfo - displayName: 'Display build info' - -- script: | - git fetch origin - ./python Tools/patchcheck/patchcheck.py --ci true - displayName: 'Run patchcheck.py' - condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml deleted file mode 100644 index 335a4b407cb83c..00000000000000 --- a/.azure-pipelines/pr.yml +++ /dev/null @@ -1,28 +0,0 @@ -pr: ['main', '3.12', '3.11', '3.10', '3.9', '3.8', '3.7'] - -jobs: -- job: Prebuild - displayName: Pre-build checks - - pool: - vmImage: ubuntu-22.04 - - steps: - - template: ./prebuild-checks.yml - - -- job: Ubuntu_Patchcheck - displayName: Ubuntu patchcheck - dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) - - pool: - vmImage: ubuntu-22.04 - - variables: - testRunTitle: '$(system.pullRequest.TargetBranch)-linux' - testRunPlatform: linux - openssl_version: 1.1.1u - - steps: - - template: ./posix-steps.yml diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index 9f808af38e69df..00000000000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM docker.io/library/fedora:37 - -ENV CC=clang - -ENV WASI_SDK_VERSION=20 -ENV WASI_SDK_PATH=/opt/wasi-sdk - -ENV WASMTIME_HOME=/opt/wasmtime -ENV WASMTIME_VERSION=14.0.4 -ENV WASMTIME_CPU_ARCH=x86_64 - -RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \ - dnf -y --nodocs --setopt=install_weak_deps=False builddep python3 && \ - dnf -y clean all - -RUN mkdir ${WASI_SDK_PATH} && \ - curl --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-linux.tar.gz | \ - tar --strip-components 1 --directory ${WASI_SDK_PATH} --extract --gunzip - -RUN mkdir --parents ${WASMTIME_HOME} && \ - curl --location "https://github.com/bytecodealliance/wasmtime/releases/download/v${WASMTIME_VERSION}/wasmtime-v${WASMTIME_VERSION}-${WASMTIME_CPU_ARCH}-linux.tar.xz" | \ - xz --decompress | \ - tar --strip-components 1 --directory ${WASMTIME_HOME} -x && \ - ln -s ${WASMTIME_HOME}/wasmtime /usr/local/bin diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 0dc303015df5c7..64c85c1101e6e6 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,5 @@ { - "build": { - "dockerfile": "Dockerfile" - }, + "image": "ghcr.io/python/devcontainer:2024.09.25.11038928730", "onCreateCommand": [ // Install common tooling. "dnf", diff --git a/.gitattributes b/.gitattributes index 4f82cea7480cfe..2f5a030981fb94 100644 --- a/.gitattributes +++ b/.gitattributes @@ -27,8 +27,6 @@ Lib/test/cjkencodings/* noeol Lib/test/tokenizedata/coding20731.py noeol Lib/test/decimaltestdata/*.decTest noeol Lib/test/test_email/data/*.txt noeol -Lib/test/test_importlib/resources/data01/* noeol -Lib/test/test_importlib/resources/namespacedata01/* noeol Lib/test/xmltestdata/* noeol # Shell scripts should have LF even on Windows because of Cygwin @@ -83,6 +81,8 @@ Include/opcode_ids.h generated Include/token.h generated Lib/_opcode_metadata.py generated Lib/keyword.py generated +Lib/test/certdata/*.pem generated +Lib/test/certdata/*.0 generated Lib/test/levenshtein_examples.json generated Lib/test/test_stable_abi_ctypes.py generated Lib/token.py generated diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e8eed400d961fc..0add48e777866c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,23 +5,28 @@ # https://git-scm.com/docs/gitignore#_pattern_format # GitHub -.github/** @ezio-melotti @hugovk +.github/** @ezio-melotti @hugovk @AA-Turner # pre-commit .pre-commit-config.yaml @hugovk @AlexWaygood -.ruff.toml @hugovk @AlexWaygood +.ruff.toml @hugovk @AlexWaygood @AA-Turner # Build system configure* @erlend-aasland @corona10 +Makefile.pre.in @erlend-aasland +Modules/Setup* @erlend-aasland + +# argparse +**/*argparse* @savannahostrowski # asyncio -**/*asyncio* @1st1 @asvetlov @gvanrossum @kumaraditya303 @willingc +**/*asyncio* @1st1 @asvetlov @kumaraditya303 @willingc # Core **/*context* @1st1 **/*genobject* @markshannon **/*hamt* @1st1 -**/*jit* @brandtbucher +**/*jit* @brandtbucher @savannahostrowski Objects/set* @rhettinger Objects/dict* @methane @markshannon Objects/typevarobject.c @JelleZijlstra @@ -29,24 +34,36 @@ Objects/type* @markshannon Objects/codeobject.c @markshannon Objects/frameobject.c @markshannon Objects/call.c @markshannon -Python/ceval*.c @markshannon @gvanrossum -Python/ceval*.h @markshannon @gvanrossum +Python/ceval*.c @markshannon +Python/ceval*.h @markshannon +Python/codegen.c @markshannon @iritkatriel Python/compile.c @markshannon @iritkatriel Python/assemble.c @markshannon @iritkatriel Python/flowgraph.c @markshannon @iritkatriel -Python/ast_opt.c @isidentical -Python/bytecodes.c @markshannon @gvanrossum -Python/optimizer*.c @markshannon @gvanrossum +Python/instruction_sequence.c @iritkatriel +Python/bytecodes.c @markshannon +Python/optimizer*.c @markshannon Python/optimizer_analysis.c @Fidget-Spinner Python/optimizer_bytecodes.c @Fidget-Spinner +Python/symtable.c @JelleZijlstra @carljm +Lib/_pyrepl/* @pablogsal @lysnikolaou @ambv Lib/test/test_patma.py @brandtbucher Lib/test/test_type_*.py @JelleZijlstra -Lib/test/test_capi/test_misc.py @markshannon @gvanrossum +Lib/test/test_capi/test_misc.py @markshannon +Lib/test/test_pyrepl/* @pablogsal @lysnikolaou @ambv Tools/c-analyzer/ @ericsnowcurrently # dbm **/*dbm* @corona10 @erlend-aasland @serhiy-storchaka +# Doc/ tools +Doc/conf.py @AA-Turner @hugovk +Doc/Makefile @AA-Turner @hugovk +Doc/make.bat @AA-Turner @hugovk +Doc/requirements.txt @AA-Turner @hugovk +Doc/_static/** @AA-Turner @hugovk +Doc/tools/** @AA-Turner @hugovk + # runtime state/lifecycle **/*pylifecycle* @ericsnowcurrently **/*pystate* @ericsnowcurrently @@ -66,32 +83,41 @@ Include/internal/pycore_freelist.h @ericsnowcurrently Include/internal/pycore_global_objects.h @ericsnowcurrently Include/internal/pycore_obmalloc.h @ericsnowcurrently Include/internal/pycore_pymem.h @ericsnowcurrently +Include/internal/pycore_stackref.h @Fidget-Spinner Modules/main.c @ericsnowcurrently Programs/_bootstrap_python.c @ericsnowcurrently Programs/python.c @ericsnowcurrently Tools/build/generate_global_objects.py @ericsnowcurrently +# Initialization +Doc/library/sys_path_init.rst @FFY00 +Doc/c-api/init_config.rst @FFY00 + +# getpath +**/*getpath* @FFY00 + +# site +**/*site.py @FFY00 +Doc/library/site.rst @FFY00 + # Exceptions -Lib/traceback.py @iritkatriel Lib/test/test_except*.py @iritkatriel -Lib/test/test_traceback.py @iritkatriel Objects/exceptions.c @iritkatriel -Python/traceback.c @iritkatriel -# Hashing -**/*hashlib* @gpshead @tiran +# Hashing & cryptographic primitives +**/*hashlib* @gpshead @tiran @picnixz **/*pyhash* @gpshead @tiran -**/sha* @gpshead @tiran -Modules/md5* @gpshead @tiran -**/*blake* @gpshead @tiran -Modules/_blake2/** @gpshead @tiran +**/sha* @gpshead @tiran @picnixz +Modules/md5* @gpshead @tiran @picnixz +**/*blake* @gpshead @tiran @picnixz Modules/_hacl/** @gpshead +**/*hmac* @gpshead @picnixz # logging **/*logging* @vsajip # venv -**/*venv* @vsajip +**/*venv* @vsajip @FFY00 # Launcher /PC/launcher.c @vsajip @@ -119,7 +145,7 @@ Python/dynload_*.c @ericsnowcurrently Lib/test/test_module/ @ericsnowcurrently Doc/c-api/module.rst @ericsnowcurrently **/*importlib/resources/* @jaraco @warsaw @FFY00 -**/importlib/metadata/* @jaraco @warsaw +**/*importlib/metadata/* @jaraco @warsaw # Dates and times **/*datetime* @pganssle @abalkin @@ -150,13 +176,15 @@ Include/internal/pycore_time.h @pganssle @abalkin /Lib/test/test_tokenize.py @pablogsal @lysnikolaou # Code generator -/Tools/cases_generator/ @gvanrossum +/Tools/cases_generator/ @markshannon # AST -Python/ast.c @isidentical -Parser/asdl.py @isidentical -Parser/asdl_c.py @isidentical -Lib/ast.py @isidentical +Python/ast.c @isidentical @JelleZijlstra @eclips4 +Python/ast_opt.c @isidentical @eclips4 +Parser/asdl.py @isidentical @JelleZijlstra @eclips4 +Parser/asdl_c.py @isidentical @JelleZijlstra @eclips4 +Lib/ast.py @isidentical @JelleZijlstra @eclips4 +Lib/test/test_ast/ @eclips4 # Mock /Lib/unittest/mock.py @cjw296 @@ -173,6 +201,10 @@ Lib/ast.py @isidentical /Lib/test/test_subprocess.py @gpshead /Modules/*subprocess* @gpshead +# debugger +**/*pdb* @gaogaotiantian +**/*bdb* @gaogaotiantian + # Limited C API & stable ABI Tools/build/stable_abi.py @encukou Misc/stable_abi.toml @encukou @@ -194,18 +226,19 @@ Doc/c-api/stable.rst @encukou **/*itertools* @rhettinger **/*collections* @rhettinger **/*random* @rhettinger -**/*queue* @rhettinger **/*bisect* @rhettinger **/*heapq* @rhettinger **/*functools* @rhettinger -**/*decimal* @rhettinger **/*dataclasses* @ericvsmith **/*ensurepip* @pfmoore @pradyunsg +/Doc/library/idle.rst @terryjreedy **/*idlelib* @terryjreedy +**/*turtledemo* @terryjreedy +**/*annotationlib* @JelleZijlstra **/*typing* @JelleZijlstra @AlexWaygood **/*ftplib @giampaolo @@ -240,11 +273,23 @@ Doc/howto/clinic.rst @erlend-aasland **/*interpreteridobject.* @ericsnowcurrently **/*crossinterp* @ericsnowcurrently Lib/test/support/interpreters/ @ericsnowcurrently -Modules/_xx*interp*module.c @ericsnowcurrently +Modules/_interp*module.c @ericsnowcurrently Lib/test/test_interpreters/ @ericsnowcurrently +# Android +**/*Android* @mhsmith @freakboy3742 +**/*android* @mhsmith @freakboy3742 + +# iOS (but not termios) +**/iOS* @freakboy3742 +**/ios* @freakboy3742 +**/*_iOS* @freakboy3742 +**/*_ios* @freakboy3742 +**/*-iOS* @freakboy3742 +**/*-ios* @freakboy3742 + # WebAssembly -/Tools/wasm/ @brettcannon +/Tools/wasm/ @brettcannon @freakboy3742 # SBOM /Misc/externals.spdx.json @sethmlarson @@ -254,3 +299,12 @@ Lib/test/test_interpreters/ @ericsnowcurrently # Config Parser Lib/configparser.py @jaraco Lib/test/test_configparser.py @jaraco + +# Doc sections +Doc/reference/ @willingc @AA-Turner + +**/*weakref* @kumaraditya303 + +# Colorize +Lib/_colorize.py @hugovk +Lib/test/test__colorize.py @hugovk diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 395516f29b037c..7b7810cf6965fd 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -34,12 +34,12 @@ body: label: "CPython versions tested on:" multiple: true options: - - "3.8" - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" + - "3.14" - "CPython main branch" validations: required: true diff --git a/.github/ISSUE_TEMPLATE/crash.yml b/.github/ISSUE_TEMPLATE/crash.yml index c14d7cf2599d4c..58da2dfe0c7354 100644 --- a/.github/ISSUE_TEMPLATE/crash.yml +++ b/.github/ISSUE_TEMPLATE/crash.yml @@ -27,11 +27,12 @@ body: label: "CPython versions tested on:" multiple: true options: - - "3.8" - "3.9" - "3.10" - "3.11" - "3.12" + - "3.13" + - "3.14" - "CPython main branch" validations: required: true diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000000000..af125266ae7813 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,10 @@ +self-hosted-runner: + labels: ["windows-aarch64"] + +config-variables: null + +paths: + .github/workflows/**/*.yml: + ignore: + - 1st argument of function call is not assignable + - SC2(015|038|086|091|097|098|129|155) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 20d1fad40ecafe..c10c5b4aa46ffb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,26 +1,15 @@ name: Tests -# gh-84728: "paths-ignore" is not used to skip documentation-only PRs, because -# it prevents to mark a job as mandatory. A PR cannot be merged if a job is -# mandatory but not scheduled because of "paths-ignore". on: workflow_dispatch: push: branches: - 'main' - - '3.12' - - '3.11' - - '3.10' - - '3.9' - - '3.8' + - '3.*' pull_request: branches: - 'main' - - '3.12' - - '3.11' - - '3.10' - - '3.9' - - '3.8' + - '3.*' permissions: contents: read @@ -29,88 +18,24 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-reusable cancel-in-progress: true +env: + FORCE_COLOR: 1 + jobs: check_source: - name: 'Check for source changes' - runs-on: ubuntu-latest - timeout-minutes: 10 - outputs: - run-docs: ${{ steps.docs-changes.outputs.run-docs || false }} - run_tests: ${{ steps.check.outputs.run_tests }} - run_hypothesis: ${{ steps.check.outputs.run_hypothesis }} - run_cifuzz: ${{ steps.check.outputs.run_cifuzz }} - config_hash: ${{ steps.config_hash.outputs.hash }} - steps: - - uses: actions/checkout@v4 - - name: Check for source changes - id: check - run: | - if [ -z "$GITHUB_BASE_REF" ]; then - echo "run_tests=true" >> $GITHUB_OUTPUT - else - git fetch origin $GITHUB_BASE_REF --depth=1 - # git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more - # reliable than git diff "origin/$GITHUB_BASE_REF.." (2 dots), - # but it requires to download more commits (this job uses - # "git fetch --depth=1"). - # - # git diff "origin/$GITHUB_BASE_REF..." (3 dots) works with Git - # 2.26, but Git 2.28 is stricter and fails with "no merge base". - # - # git diff "origin/$GITHUB_BASE_REF.." (2 dots) should be enough on - # GitHub, since GitHub starts by merging origin/$GITHUB_BASE_REF - # into the PR branch anyway. - # - # https://github.com/python/core-workflow/issues/373 - git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true - fi - - # Check if we should run hypothesis tests - GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} - echo $GIT_BRANCH - if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then - echo "Branch too old for hypothesis tests" - echo "run_hypothesis=false" >> $GITHUB_OUTPUT - else - echo "Run hypothesis tests" - echo "run_hypothesis=true" >> $GITHUB_OUTPUT - fi - - # oss-fuzz maintains a configuration for fuzzing the main branch of - # CPython, so CIFuzz should be run only for code that is likely to be - # merged into the main branch; compatibility with older branches may - # be broken. - FUZZ_RELEVANT_FILES='(\.c$|\.h$|\.cpp$|^configure$|^\.github/workflows/build\.yml$|^Modules/_xxtestfuzz)' - if [ "$GITHUB_BASE_REF" = "main" ] && [ "$(git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qE $FUZZ_RELEVANT_FILES; echo $?)" -eq 0 ]; then - # The tests are pretty slow so they are executed only for PRs - # changing relevant files. - echo "Run CIFuzz tests" - echo "run_cifuzz=true" >> $GITHUB_OUTPUT - else - echo "Branch too old for CIFuzz tests; or no C files were changed" - echo "run_cifuzz=false" >> $GITHUB_OUTPUT - fi - - name: Compute hash for config cache key - id: config_hash - run: | - echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> $GITHUB_OUTPUT - - name: Get a list of the changed documentation-related files - if: github.event_name == 'pull_request' - id: changed-docs-files - uses: Ana06/get-changed-files@v2.2.0 - with: - filter: | - Doc/** - Misc/** - .github/workflows/reusable-docs.yml - format: csv # works for paths with spaces - - name: Check for docs changes - if: >- - github.event_name == 'pull_request' - && steps.changed-docs-files.outputs.added_modified_renamed != '' - id: docs-changes - run: | - echo "run-docs=true" >> "${GITHUB_OUTPUT}" + name: Change detection + # To use boolean outputs from this job, parse them as JSON. + # Here's some examples: + # + # if: fromJSON(needs.check_source.outputs.run-docs) + # + # ${{ + # fromJSON(needs.check_source.outputs.run_tests) + # && 'truthy-branch' + # || 'falsy-branch' + # }} + # + uses: ./.github/workflows/reusable-change-detection.yml check-docs: name: Docs @@ -118,51 +43,90 @@ jobs: if: fromJSON(needs.check_source.outputs.run-docs) uses: ./.github/workflows/reusable-docs.yml + check_autoconf_regen: + name: 'Check if Autoconf files are up to date' + # Don't use ubuntu-latest but a specific version to make the job + # reproducible: to get the same tools versions (autoconf, aclocal, ...) + runs-on: ubuntu-24.04 + container: + image: ghcr.io/python/autoconf:2025.01.02.12581854023 + timeout-minutes: 60 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + steps: + - name: Install Git + run: | + apt update && apt install git -yq + git config --global --add safe.directory "$GITHUB_WORKSPACE" + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" + - name: Check Autoconf and aclocal versions + run: | + grep "Generated by GNU Autoconf 2.72" configure + grep "aclocal 1.16.5" aclocal.m4 + grep -q "runstatedir" configure + grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4 + - name: Regenerate autoconf files + # Same command used by Tools/build/regen-configure.sh ($AUTORECONF) + run: autoreconf -ivf -Werror + - name: Check for changes + run: | + git add -u + changes=$(git status --porcelain) + # Check for changes in regenerated files + if test -n "$changes"; then + echo "Generated files not up to date." + echo "Perhaps you forgot to run make regen-configure ;)" + echo "configure files must be regenerated with a specific version of autoconf." + echo "$changes" + echo "" + git diff --staged || true + exit 1 + fi + check_generated_files: name: 'Check if generated files are up to date' # Don't use ubuntu-latest but a specific version to make the job # reproducible: to get the same tools versions (autoconf, aclocal, ...) - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: '3.x' - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: path: config.cache + # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }} - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Add ccache to PATH - run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: save: false - - name: Check Autoconf and aclocal versions - run: | - grep "Generated by GNU Autoconf 2.71" configure - grep "aclocal 1.16.5" aclocal.m4 - grep -q "runstatedir" configure - grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4 - name: Configure CPython run: | # Build Python with the libpython dynamic library ./configure --config-cache --with-pydebug --enable-shared - - name: Regenerate autoconf files - # Same command used by Tools/build/regen-configure.sh ($AUTORECONF) - run: autoreconf -ivf -Werror - name: Build CPython run: | make -j4 regen-all - make regen-stdlib-module-names regen-sbom + make regen-stdlib-module-names regen-sbom regen-unicodedata - name: Check for changes run: | git add -u @@ -186,73 +150,130 @@ jobs: run: make check-c-globals build_windows: - name: 'Windows' + name: >- + Windows + ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - uses: ./.github/workflows/reusable-windows.yml - - build_windows_free_threading: - name: 'Windows (free-threading)' - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + if: fromJSON(needs.check_source.outputs.run_tests) + strategy: + fail-fast: false + matrix: + os: + - windows-latest + arch: + - x64 + free-threading: + - false + - true + include: + - os: windows-latest # FIXME(diegorusso): change to os: windows-aarch64 + arch: arm64 + free-threading: false + - os: windows-latest # FIXME(diegorusso): change to os: windows-aarch64 + arch: arm64 + free-threading: true + - os: windows-latest + arch: Win32 + free-threading: false uses: ./.github/workflows/reusable-windows.yml with: - free-threading: true + os: ${{ matrix.os }} + arch: ${{ matrix.arch }} + free-threading: ${{ matrix.free-threading }} - build_macos: - name: 'macOS' + build_windows_msi: + name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category + Windows MSI${{ '' }} needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - uses: ./.github/workflows/reusable-macos.yml + if: fromJSON(needs.check_source.outputs.run-win-msi) + strategy: + matrix: + arch: + - x86 + - x64 + - arm64 + uses: ./.github/workflows/reusable-windows-msi.yml with: - config_hash: ${{ needs.check_source.outputs.config_hash }} + arch: ${{ matrix.arch }} - build_macos_free_threading: - name: 'macOS (free-threading)' + build_macos: + name: >- + macOS + ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} needs: check_source if: needs.check_source.outputs.run_tests == 'true' + strategy: + fail-fast: false + matrix: + # Cirrus and macos-14 are M1, macos-13 is default GHA Intel. + # macOS 13 only runs tests against the GIL-enabled CPython. + # Cirrus used for upstream, macos-14 for forks. + os: + - ghcr.io/cirruslabs/macos-runner:sonoma + - macos-14 + - macos-13 + is-fork: # only used for the exclusion trick + - ${{ github.repository_owner != 'python' }} + free-threading: + - false + - true + exclude: + - os: ghcr.io/cirruslabs/macos-runner:sonoma + is-fork: true + - os: macos-14 + is-fork: false + - os: macos-13 + free-threading: true uses: ./.github/workflows/reusable-macos.yml with: config_hash: ${{ needs.check_source.outputs.config_hash }} - free-threading: true + free-threading: ${{ matrix.free-threading }} + os: ${{ matrix.os }} build_ubuntu: - name: 'Ubuntu' - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - uses: ./.github/workflows/reusable-ubuntu.yml - with: - config_hash: ${{ needs.check_source.outputs.config_hash }} - options: | - ../cpython-ro-srcdir/configure \ - --config-cache \ - --with-pydebug \ - --with-openssl=$OPENSSL_DIR - - build_ubuntu_free_threading: - name: 'Ubuntu (free-threading)' + name: >- + Ubuntu + ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} + ${{ fromJSON(matrix.bolt) && '(bolt)' || '' }} needs: check_source if: needs.check_source.outputs.run_tests == 'true' + strategy: + matrix: + bolt: + - false + - true + free-threading: + - false + - true + os: + - ubuntu-24.04 + - ubuntu-24.04-arm + exclude: + # Do not test BOLT with free-threading, to conserve resources + - bolt: true + free-threading: true + # BOLT currently crashes during instrumentation on aarch64 + - os: ubuntu-24.04-arm + bolt: true uses: ./.github/workflows/reusable-ubuntu.yml with: config_hash: ${{ needs.check_source.outputs.config_hash }} - options: | - ../cpython-ro-srcdir/configure \ - --config-cache \ - --with-pydebug \ - --with-openssl=$OPENSSL_DIR \ - --disable-gil + bolt-optimizations: ${{ matrix.bolt }} + free-threading: ${{ matrix.free-threading }} + os: ${{ matrix.os }} build_ubuntu_ssltests: name: 'Ubuntu SSL tests with OpenSSL' - runs-on: ubuntu-20.04 + runs-on: ${{ matrix.os }} timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' strategy: fail-fast: false matrix: - openssl_ver: [1.1.1w, 3.0.13, 3.1.5, 3.2.1] + os: [ubuntu-24.04] + openssl_ver: [3.0.15, 3.1.7, 3.2.3, 3.3.2, 3.4.0] + # See Tools/ssl/make_ssl_data.py for notes on adding a new version env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl @@ -260,8 +281,10 @@ jobs: LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: @@ -273,27 +296,27 @@ jobs: run: sudo ./.github/workflows/posix-deps-apt.sh - name: Configure OpenSSL env vars run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: save: false - name: Configure CPython - run: ./configure --config-cache --with-pydebug --with-openssl=$OPENSSL_DIR + run: ./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$OPENSSL_DIR" - name: Build CPython run: make -j4 - name: Display build info @@ -301,26 +324,36 @@ jobs: - name: SSL tests run: ./python Lib/test/ssltests.py + build_wasi: + name: 'WASI' + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + uses: ./.github/workflows/reusable-wasi.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + test_hypothesis: name: "Hypothesis tests on Ubuntu" - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' env: - OPENSSL_VER: 3.0.13 + OPENSSL_VER: 3.0.15 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Configure OpenSSL env vars run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl uses: actions/cache@v4 @@ -329,24 +362,24 @@ jobs: key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: save: false - name: Setup directory envs for out-of-tree builds run: | - echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" + echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - name: Create directories for read-only out-of-tree builds - run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR + run: mkdir -p "$CPYTHON_RO_SRCDIR" "$CPYTHON_BUILDDIR" - name: Bind mount sources read-only - run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: @@ -358,7 +391,8 @@ jobs: ../cpython-ro-srcdir/configure \ --config-cache \ --with-pydebug \ - --with-openssl=$OPENSSL_DIR + --enable-slower-safety \ + --with-openssl="$OPENSSL_DIR" - name: Build CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make -j4 @@ -367,26 +401,26 @@ jobs: run: make pythoninfo - name: Remount sources writable for tests # some tests write to srcdir, lack of pyc files slows down testing - run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw + run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw - name: Setup directory envs for out-of-tree builds run: | - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - name: "Create hypothesis venv" working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | VENV_LOC=$(realpath -m .)/hypovenv VENV_PYTHON=$VENV_LOC/bin/python - echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV - echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV - ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -r ${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt + echo "HYPOVENV=${VENV_LOC}" >> "$GITHUB_ENV" + echo "VENV_PYTHON=${VENV_PYTHON}" >> "$GITHUB_ENV" + ./python -m venv "$VENV_LOC" && "$VENV_PYTHON" -m pip install -r "${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt" - name: 'Restore Hypothesis database' id: cache-hypothesis-database uses: actions/cache@v4 with: - path: ./hypothesis + path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/ key: hypothesis-database-${{ github.head_ref || github.run_id }} restore-keys: | - - hypothesis-database- + hypothesis-database- - name: "Run tests" working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -394,7 +428,7 @@ jobs: # # (GH-104097) test_sysconfig is skipped because it has tests that are # failing when executed from inside a virtual environment. - ${{ env.VENV_PYTHON }} -m test \ + "${VENV_PYTHON}" -m test \ -W \ -o \ -j4 \ @@ -411,23 +445,28 @@ jobs: if: always() with: name: hypothesis-example-db - path: .hypothesis/examples/ + path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/examples/ build_asan: name: 'Address sanitizer' - runs-on: ubuntu-20.04 + runs-on: ${{ matrix.os }} timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' + strategy: + matrix: + os: [ubuntu-24.04] env: - OPENSSL_VER: 3.0.13 + OPENSSL_VER: 3.0.15 PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: @@ -443,21 +482,21 @@ jobs: version: 10 - name: Configure OpenSSL env vars run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: @@ -470,7 +509,62 @@ jobs: - name: Display build info run: make pythoninfo - name: Tests - run: xvfb-run make test + run: xvfb-run make ci + + build_tsan: + name: >- + Thread sanitizer + ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + strategy: + matrix: + free-threading: + - false + - true + uses: ./.github/workflows/reusable-tsan.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + free-threading: ${{ matrix.free-threading }} + + cross-build-linux: + name: Cross build Linux + runs-on: ubuntu-latest + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Set build dir + run: + # an absolute path outside of the working directoy + echo "BUILD_DIR=$(realpath ${{ github.workspace }}/../build)" >> "$GITHUB_ENV" + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure host build + run: ./configure --prefix="$BUILD_DIR/host-python" + - name: Install host Python + run: make -j8 install + - name: Run test subset with host build + run: | + "$BUILD_DIR/host-python/bin/python3" -m test test_sysconfig test_site test_embed + - name: Configure cross build + run: ./configure --prefix="$BUILD_DIR/cross-python" --with-build-python="$BUILD_DIR/host-python/bin/python3" + - name: Install cross Python + run: make -j8 install + - name: Run test subset with host build + run: | + "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ cifuzz: @@ -519,16 +613,17 @@ jobs: needs: - check_source # Transitive dependency, needed to access `run_tests` value - check-docs + - check_autoconf_regen - check_generated_files - build_macos - - build_macos_free_threading - build_ubuntu - - build_ubuntu_free_threading - build_ubuntu_ssltests + - build_wasi - build_windows - - build_windows_free_threading + - build_windows_msi - test_hypothesis - build_asan + - build_tsan - cifuzz runs-on: ubuntu-latest @@ -539,6 +634,7 @@ jobs: with: allowed-failures: >- build_ubuntu_ssltests, + build_windows_msi, cifuzz, test_hypothesis, allowed-skips: >- @@ -552,15 +648,15 @@ jobs: ${{ needs.check_source.outputs.run_tests != 'true' && ' + check_autoconf_regen, check_generated_files, build_macos, - build_macos_free_threading, build_ubuntu, - build_ubuntu_free_threading, build_ubuntu_ssltests, + build_wasi, build_windows, - build_windows_free_threading, build_asan, + build_tsan, ' || '' }} diff --git a/.github/workflows/build_msi.yml b/.github/workflows/build_msi.yml deleted file mode 100644 index 65d32c734e7745..00000000000000 --- a/.github/workflows/build_msi.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: TestsMSI - -on: - workflow_dispatch: - push: - branches: - - 'main' - - '3.*' - paths: - - 'Tools/msi/**' - - '.github/workflows/build_msi.yml' - pull_request: - branches: - - 'main' - - '3.*' - paths: - - 'Tools/msi/**' - - '.github/workflows/build_msi.yml' - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - build: - name: Windows Installer - runs-on: windows-latest - timeout-minutes: 60 - strategy: - matrix: - type: [x86, x64, arm64] - env: - IncludeFreethreaded: true - steps: - - uses: actions/checkout@v4 - - name: Build CPython installer - run: .\Tools\msi\build.bat --doc -${{ matrix.type }} diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml index 43a7afec73884e..fdb4b9aa29a7c8 100644 --- a/.github/workflows/documentation-links.yml +++ b/.github/workflows/documentation-links.yml @@ -10,9 +10,6 @@ on: - 'Doc/**' - '.github/workflows/doc.yml' -permissions: - pull-requests: write - concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true @@ -20,6 +17,9 @@ concurrency: jobs: documentation-links: runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: - uses: readthedocs/actions/preview@v1 with: diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 809ac45919fe74..806a8524112d76 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -5,24 +5,50 @@ on: - '**jit**' - 'Python/bytecodes.c' - 'Python/optimizer*.c' - - 'Python/optimizer_bytecodes.c' + - '!Python/perf_jit_trampoline.c' + - '!**/*.md' + - '!**/*.ini' push: paths: - '**jit**' - 'Python/bytecodes.c' - 'Python/optimizer*.c' - - 'Python/optimizer_bytecodes.c' + - '!Python/perf_jit_trampoline.c' + - '!**/*.md' + - '!**/*.ini' workflow_dispatch: +permissions: + contents: read + concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +env: + FORCE_COLOR: 1 + jobs: + interpreter: + name: Interpreter (Debug) + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Build tier two interpreter + run: | + ./configure --enable-experimental-jit=interpreter --with-pydebug + make all --jobs 4 + - name: Test tier two interpreter + run: | + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 jit: name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) + needs: interpreter runs-on: ${{ matrix.runner }} - timeout-minutes: 60 + timeout-minutes: 90 strategy: fail-fast: false matrix: @@ -33,59 +59,38 @@ jobs: - x86_64-apple-darwin/clang - aarch64-apple-darwin/clang - x86_64-unknown-linux-gnu/gcc - - x86_64-unknown-linux-gnu/clang - aarch64-unknown-linux-gnu/gcc - - aarch64-unknown-linux-gnu/clang debug: - true - false llvm: - - 16 + - 19 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 runner: windows-latest - compiler: msvc - target: x86_64-pc-windows-msvc/msvc architecture: x64 runner: windows-latest - compiler: msvc - target: aarch64-pc-windows-msvc/msvc architecture: ARM64 runner: windows-latest - compiler: msvc - target: x86_64-apple-darwin/clang architecture: x86_64 runner: macos-13 - compiler: clang - target: aarch64-apple-darwin/clang architecture: aarch64 runner: macos-14 - compiler: clang - target: x86_64-unknown-linux-gnu/gcc architecture: x86_64 - runner: ubuntu-latest - compiler: gcc - - target: x86_64-unknown-linux-gnu/clang - architecture: x86_64 - runner: ubuntu-latest - compiler: clang + runner: ubuntu-24.04 - target: aarch64-unknown-linux-gnu/gcc architecture: aarch64 - runner: ubuntu-latest - compiler: gcc - # These fail because of emulation, not because of the JIT: - exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection - - target: aarch64-unknown-linux-gnu/clang - architecture: aarch64 - runner: ubuntu-latest - compiler: clang - # These fail because of emulation, not because of the JIT: - exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection - env: - CC: ${{ matrix.compiler }} + runner: ubuntu-24.04-arm steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: '3.11' @@ -93,51 +98,62 @@ jobs: - name: Native Windows if: runner.os == 'Windows' && matrix.architecture != 'ARM64' run: | - choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} - ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }} - ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 + choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 + ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} + ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - # No PGO or tests (yet): + # No tests (yet): - name: Emulated Windows if: runner.os == 'Windows' && matrix.architecture == 'ARM64' run: | - choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} + choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} + # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. + # This is a bug in the macOS runner image where the pre-installed Python is installed in the same + # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes + # the symlink to the pre-installed Python so that the Homebrew Python is used instead. - name: Native macOS if: runner.os == 'macOS' run: | + brew update + find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete brew install llvm@${{ matrix.llvm }} export SDKROOT="$(xcrun --show-sdk-path)" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 - ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 + ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - name: Native Linux - if: runner.os == 'Linux' && matrix.architecture == 'x86_64' + if: runner.os == 'Linux' run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 - ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - name: Emulated Linux - if: runner.os == 'Linux' && matrix.architecture != 'x86_64' + jit-with-disabled-gil: + name: Free-Threaded (Debug) + needs: interpreter + runs-on: ubuntu-24.04 + strategy: + matrix: + llvm: + - 19 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Build with JIT enabled and GIL disabled run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - ./configure --prefix="$(pwd)/../build" - make install --jobs 4 - make clean --jobs 4 - export HOST=${{ matrix.architecture }}-linux-gnu - sudo apt install --yes "gcc-$HOST" qemu-user - ${{ !matrix.debug && matrix.compiler == 'clang' && './configure --enable-optimizations' || '' }} - ${{ !matrix.debug && matrix.compiler == 'clang' && 'make profile-run-stamp --jobs 4' || '' }} - export CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" - export CPP="$CC --preprocess" - export HOSTRUNNER=qemu-${{ matrix.architecture }} - export QEMU_LD_PREFIX="/usr/$HOST" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes + ./configure --enable-experimental-jit --with-pydebug --disable-gil make all --jobs 4 - ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 + - name: Run tests + run: | + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4a70ec6205a05b..d74ce8fcc256dc 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -20,7 +20,9 @@ jobs: steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: "3.x" - - uses: pre-commit/action@v3.0.0 + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index b766785de405d2..5dfa8d7bcafd78 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -8,6 +8,7 @@ on: pull_request: paths: - ".github/workflows/mypy.yml" + - "Lib/_pyrepl/**" - "Lib/test/libregrtest/**" - "Tools/build/generate_sbom.py" - "Tools/cases_generator/**" @@ -33,10 +34,12 @@ concurrency: jobs: mypy: strategy: + fail-fast: false matrix: target: [ + "Lib/_pyrepl", "Lib/test/libregrtest", - "Tools/build/", + "Tools/build", "Tools/cases_generator", "Tools/clinic", "Tools/jit", @@ -48,9 +51,11 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.13" cache: pip cache-dependency-path: Tools/requirements-dev.txt - run: pip install -r Tools/requirements-dev.txt diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index 0800401f4cd113..d5538cd9367ec6 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -1,11 +1,9 @@ #!/bin/sh apt-get update -# autoconf-archive is needed by autoreconf (check_generated_files job) apt-get -yq install \ build-essential \ pkg-config \ - autoconf-archive \ ccache \ gdb \ lcov \ diff --git a/.github/workflows/project-updater.yml b/.github/workflows/project-updater.yml index 7574bfc208ff76..066d8593a70cf6 100644 --- a/.github/workflows/project-updater.yml +++ b/.github/workflows/project-updater.yml @@ -23,7 +23,7 @@ jobs: - { project: 32, label: sprint } steps: - - uses: actions/add-to-project@v0.1.0 + - uses: actions/add-to-project@v1.0.0 with: project-url: https://github.com/orgs/python/projects/${{ matrix.project }} github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index ff5cbdf3eda749..0a6277c779ff67 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -4,20 +4,60 @@ on: pull_request: types: [opened, reopened, labeled, unlabeled, synchronize] -permissions: - issues: write - pull-requests: write - jobs: - label: - name: DO-NOT-MERGE / unresolved review + label-dnm: + name: DO-NOT-MERGE + if: github.repository_owner == 'python' + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + timeout-minutes: 10 + + steps: + - name: Check there's no DO-NOT-MERGE + uses: mheap/github-action-required-labels@v5 + with: + mode: exactly + count: 0 + labels: | + DO-NOT-MERGE + + label-reviews: + name: Unresolved review if: github.repository_owner == 'python' runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write timeout-minutes: 10 steps: - - uses: mheap/github-action-required-labels@v5 + # Check that the PR is not awaiting changes from the author due to previous review. + - name: Check there's no required changes + uses: mheap/github-action-required-labels@v5 with: mode: exactly count: 0 - labels: "DO-NOT-MERGE, awaiting changes, awaiting change review" + labels: | + awaiting changes + awaiting change review + - id: is-feature + name: Check whether this PR is a feature (contains a "type-feature" label) + uses: mheap/github-action-required-labels@v5 + with: + mode: exactly + count: 1 + labels: | + type-feature + exit_type: success # don't fail the check if the PR is not a feature, just record the result + # In case of a feature PR, check for a complete review (contains an "awaiting merge" label). + - id: awaiting-merge + if: steps.is-feature.outputs.status == 'success' + name: Check for complete review + uses: mheap/github-action-required-labels@v5 + with: + mode: exactly + count: 1 + labels: | + awaiting merge diff --git a/.github/workflows/reusable-change-detection.yml b/.github/workflows/reusable-change-detection.yml new file mode 100644 index 00000000000000..c08c0cb8873f12 --- /dev/null +++ b/.github/workflows/reusable-change-detection.yml @@ -0,0 +1,173 @@ +name: Reusable change detection + +on: # yamllint disable-line rule:truthy + workflow_call: + outputs: + # Some of the referenced steps set outputs conditionally and there may be + # cases when referencing them evaluates to empty strings. It is nice to + # work with proper booleans so they have to be evaluated through JSON + # conversion in the expressions. However, empty strings used like that + # may trigger all sorts of undefined and hard-to-debug behaviors in + # GitHub Actions CI/CD. To help with this, all of the outputs set here + # that are meant to be used as boolean flags (and not arbitrary strings), + # MUST have fallbacks with default values set. A common pattern would be + # to add ` || false` to all such expressions here, in the output + # definitions. They can then later be safely used through the following + # idiom in job conditionals and other expressions. Here's some examples: + # + # if: fromJSON(needs.change-detection.outputs.run-docs) + # + # ${{ + # fromJSON(needs.change-detection.outputs.run-tests) + # && 'truthy-branch' + # || 'falsy-branch' + # }} + # + config_hash: + description: Config hash value for use in cache keys + value: ${{ jobs.compute-changes.outputs.config-hash }} # str + run-docs: + description: Whether to build the docs + value: ${{ jobs.compute-changes.outputs.run-docs || false }} # bool + run_tests: + description: Whether to run the regular tests + value: ${{ jobs.compute-changes.outputs.run-tests || false }} # bool + run-win-msi: + description: Whether to run the MSI installer smoke tests + value: >- # bool + ${{ jobs.compute-changes.outputs.run-win-msi || false }} + run_hypothesis: + description: Whether to run the Hypothesis tests + value: >- # bool + ${{ jobs.compute-changes.outputs.run-hypothesis || false }} + run_cifuzz: + description: Whether to run the CIFuzz job + value: >- # bool + ${{ jobs.compute-changes.outputs.run-cifuzz || false }} + +jobs: + compute-changes: + name: Compute changed files + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + config-hash: ${{ steps.config-hash.outputs.hash }} + run-cifuzz: ${{ steps.check.outputs.run-cifuzz }} + run-docs: ${{ steps.docs-changes.outputs.run-docs }} + run-hypothesis: ${{ steps.check.outputs.run-hypothesis }} + run-tests: ${{ steps.check.outputs.run-tests }} + run-win-msi: ${{ steps.win-msi-changes.outputs.run-win-msi }} + steps: + - run: >- + echo '${{ github.event_name }}' + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Check for source changes + id: check + run: | + if [ -z "$GITHUB_BASE_REF" ]; then + echo "run-tests=true" >> "$GITHUB_OUTPUT" + else + git fetch origin "$GITHUB_BASE_REF" --depth=1 + # git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more + # reliable than git diff "origin/$GITHUB_BASE_REF.." (2 dots), + # but it requires to download more commits (this job uses + # "git fetch --depth=1"). + # + # git diff "origin/$GITHUB_BASE_REF..." (3 dots) works with Git + # 2.26, but Git 2.28 is stricter and fails with "no merge base". + # + # git diff "origin/$GITHUB_BASE_REF.." (2 dots) should be enough on + # GitHub, since GitHub starts by merging origin/$GITHUB_BASE_REF + # into the PR branch anyway. + # + # https://github.com/python/core-workflow/issues/373 + grep_ignore_args=( + # file extensions + -e '\.md$' + -e '\.rst$' + # top-level folders + -e '^Doc/' + -e '^Misc/' + # configuration files + -e '^\.github/CODEOWNERS$' + -e '^\.pre-commit-config\.yaml$' + -e '\.ruff\.toml$' + -e 'mypy\.ini$' + ) + git diff --name-only "origin/$GITHUB_BASE_REF.." \ + | grep -qvE "${grep_ignore_args[@]}" \ + && echo "run-tests=true" >> "$GITHUB_OUTPUT" || true + fi + + # Check if we should run hypothesis tests + GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} + echo "$GIT_BRANCH" + if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then + echo "Branch too old for hypothesis tests" + echo "run-hypothesis=false" >> "$GITHUB_OUTPUT" + else + echo "Run hypothesis tests" + echo "run-hypothesis=true" >> "$GITHUB_OUTPUT" + fi + + # oss-fuzz maintains a configuration for fuzzing the main branch of + # CPython, so CIFuzz should be run only for code that is likely to be + # merged into the main branch; compatibility with older branches may + # be broken. + FUZZ_RELEVANT_FILES='(\.c$|\.h$|\.cpp$|^configure$|^\.github/workflows/build\.yml$|^Modules/_xxtestfuzz)' + if [ "$GITHUB_BASE_REF" = "main" ] && [ "$(git diff --name-only "origin/$GITHUB_BASE_REF.." | grep -qE $FUZZ_RELEVANT_FILES; echo $?)" -eq 0 ]; then + # The tests are pretty slow so they are executed only for PRs + # changing relevant files. + echo "Run CIFuzz tests" + echo "run-cifuzz=true" >> "$GITHUB_OUTPUT" + else + echo "Branch too old for CIFuzz tests; or no C files were changed" + echo "run-cifuzz=false" >> "$GITHUB_OUTPUT" + fi + - name: Compute hash for config cache key + id: config-hash + run: | + echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> "$GITHUB_OUTPUT" + - name: Get a list of the changed documentation-related files + if: github.event_name == 'pull_request' + id: changed-docs-files + uses: Ana06/get-changed-files@v2.3.0 + with: + filter: | + Doc/** + Misc/** + .github/workflows/reusable-docs.yml + format: csv # works for paths with spaces + - name: Check for docs changes + # We only want to run this on PRs when related files are changed, + # or when user triggers manual workflow run. + if: >- + ( + github.event_name == 'pull_request' + && steps.changed-docs-files.outputs.added_modified_renamed != '' + ) || github.event_name == 'workflow_dispatch' + id: docs-changes + run: | + echo "run-docs=true" >> "${GITHUB_OUTPUT}" + - name: Get a list of the MSI installer-related files + if: github.event_name == 'pull_request' + id: changed-win-msi-files + uses: Ana06/get-changed-files@v2.3.0 + with: + filter: | + Tools/msi/** + .github/workflows/reusable-windows-msi.yml + format: csv # works for paths with spaces + - name: Check for changes in MSI installer-related files + # We only want to run this on PRs when related files are changed, + # or when user triggers manual workflow run. + if: >- + ( + github.event_name == 'pull_request' + && steps.changed-win-msi-files.outputs.added_modified_renamed != '' + ) || github.event_name == 'workflow_dispatch' + id: win-msi-changes + run: | + echo "run-win-msi=true" >> "${GITHUB_OUTPUT}" diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index cea8f93d67b29c..6738acc98c6565 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -1,4 +1,4 @@ -name: Docs +name: Reusable Docs on: workflow_call: @@ -11,6 +11,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +env: + FORCE_COLOR: 1 + jobs: build_doc: name: 'Docs' @@ -19,26 +22,34 @@ jobs: env: branch_base: 'origin/${{ github.event.pull_request.base.ref }}' branch_pr: 'origin/${{ github.event.pull_request.head.ref }}' + commits: ${{ github.event.pull_request.commits }} refspec_base: '+${{ github.event.pull_request.base.sha }}:remotes/origin/${{ github.event.pull_request.base.ref }}' refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}' steps: - name: 'Check out latest PR branch commit' uses: actions/checkout@v4 with: - ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + ref: >- + ${{ + github.event_name == 'pull_request' + && github.event.pull_request.head.sha + || '' + }} # Adapted from https://github.com/actions/checkout/issues/520#issuecomment-1167205721 - name: 'Fetch commits to get branch diff' + if: github.event_name == 'pull_request' run: | # Fetch enough history to find a common ancestor commit (aka merge-base): - git fetch origin ${{ env.refspec_pr }} --depth=$(( ${{ github.event.pull_request.commits }} + 1 )) \ + git fetch origin "${refspec_pr}" --depth=$(( commits + 1 )) \ --no-tags --prune --no-recurse-submodules # This should get the oldest commit in the local fetched history (which may not be the commit the PR branched from): - COMMON_ANCESTOR=$( git rev-list --first-parent --max-parents=0 --max-count=1 ${{ env.branch_pr }} ) + COMMON_ANCESTOR=$( git rev-list --first-parent --max-parents=0 --max-count=1 "${branch_pr}" ) DATE=$( git log --date=iso8601 --format=%cd "${COMMON_ANCESTOR}" ) # Get all commits since that commit date from the base branch (eg: master or main): - git fetch origin ${{ env.refspec_base }} --shallow-since="${DATE}" \ + git fetch origin "${refspec_base}" --shallow-since="${DATE}" \ --no-tags --prune --no-recurse-submodules - name: 'Set up Python' uses: actions/setup-python@v5 @@ -54,41 +65,26 @@ jobs: continue-on-error: true run: | set -Eeuo pipefail - # Build docs with the '-n' (nit-picky) option; write warnings to file - make -C Doc/ PYTHON=../python SPHINXOPTS="-q -n -W --keep-going -w sphinx-warnings.txt" html + # Build docs with the nit-picky option; write warnings to file + make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet --nitpicky --fail-on-warning --keep-going --warning-file sphinx-warnings.txt" html - name: 'Check warnings' if: github.event_name == 'pull_request' run: | python Doc/tools/check-warnings.py \ - --annotate-diff '${{ env.branch_base }}' '${{ env.branch_pr }}' \ + --annotate-diff "${branch_base}" "${branch_pr}" \ --fail-if-regression \ - --fail-if-improved - - # This build doesn't use problem matchers or check annotations - build_doc_oldest_supported_sphinx: - name: 'Docs (Oldest Sphinx)' - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - - name: 'Set up Python' - uses: actions/setup-python@v5 - with: - python-version: '3.11' # known to work with Sphinx 4.2 - cache: 'pip' - cache-dependency-path: 'Doc/requirements-oldest-sphinx.txt' - - name: 'Install build dependencies' - run: make -C Doc/ venv REQUIREMENTS="requirements-oldest-sphinx.txt" - - name: 'Build HTML documentation' - run: make -C Doc/ SPHINXOPTS="-q" SPHINXERRORHANDLING="-W --keep-going" html + --fail-if-improved \ + --fail-if-new-news-nit # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release doctest: name: 'Doctest' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/cache@v4 with: path: ~/.cache/pip @@ -105,4 +101,4 @@ jobs: run: make -C Doc/ PYTHON=../python venv # Use "xvfb-run" since some doctest tests open GUI windows - name: 'Run documentation doctest' - run: xvfb-run make -C Doc/ PYTHON=../python SPHINXERRORHANDLING="-W --keep-going" doctest + run: xvfb-run make -C Doc/ PYTHON=../python SPHINXERRORHANDLING="--fail-on-warning --keep-going" doctest diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 65b73cd791c75c..4e7aced94a321c 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -1,3 +1,5 @@ +name: Reusable macOS + on: workflow_call: inputs: @@ -8,10 +10,17 @@ on: required: false type: boolean default: false + os: + description: OS to run the job + required: true + type: string + +env: + FORCE_COLOR: 1 jobs: build_macos: - name: 'build and test' + name: build and test (${{ inputs.os }}) timeout-minutes: 60 env: HOMEBREW_NO_ANALYTICS: 1 @@ -19,38 +28,54 @@ jobs: HOMEBREW_NO_INSTALL_CLEANUP: 1 HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 PYTHONSTRICTEXTENSIONBUILD: 1 - strategy: - fail-fast: false - matrix: - os: [ - "macos-14", # M1 - "macos-13", # Intel - ] - runs-on: ${{ matrix.os }} + TERM: linux + runs-on: ${{ inputs.os }} steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ matrix.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} + key: ${{ github.job }}-${{ inputs.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} - name: Install Homebrew dependencies - run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk + run: | + brew install pkg-config openssl@3.0 xz gdbm tcl-tk@8 make + # Because alternate versions are not symlinked into place by default: + brew link --overwrite tcl-tk@8 - name: Configure CPython run: | + MACOSX_DEPLOYMENT_TARGET=10.15 \ GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \ ./configure \ --config-cache \ --with-pydebug \ + --enable-slower-safety \ + --enable-safety \ ${{ inputs.free-threading && '--disable-gil' || '' }} \ --prefix=/opt/python-dev \ --with-openssl="$(brew --prefix openssl@3.0)" - name: Build CPython - run: make -j4 + if : ${{ inputs.free-threading || inputs.os != 'macos-13' }} + run: gmake -j8 + - name: Build CPython for compiler warning check + if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt - name: Display build info run: make pythoninfo + - name: Check compiler warnings + if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + run: >- + python3 Tools/build/check_warnings.py + --compiler-output-file-path=compiler_output_macos.txt + --warning-ignore-file-path=Tools/build/.warningignore_macos + --compiler-output-type=clang + --fail-on-regression + --fail-on-improvement + --path-prefix="./" - name: Tests - run: make test + run: make ci diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml new file mode 100644 index 00000000000000..1d2548565d50ef --- /dev/null +++ b/.github/workflows/reusable-tsan.yml @@ -0,0 +1,91 @@ +name: Reusable Thread Sanitizer + +on: + workflow_call: + inputs: + config_hash: + required: true + type: string + free-threading: + description: Whether to use free-threaded mode + required: false + type: boolean + default: false + +env: + FORCE_COLOR: 1 + +jobs: + build_tsan_reusable: + name: 'Thread sanitizer' + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} + - name: Install Dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + # Install clang-18 + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 17 # gh-121946: llvm-18 package is temporarily broken + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 + sudo update-alternatives --set clang /usr/bin/clang-17 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 + sudo update-alternatives --set clang++ /usr/bin/clang++-17 + # Reduce ASLR to avoid TSAN crashing + sudo sysctl -w vm.mmap_rnd_bits=28 + - name: TSAN Option Setup + run: | + echo "TSAN_OPTIONS=log_path=${GITHUB_WORKSPACE}/tsan_log suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${{ + fromJSON(inputs.free-threading) + && '_free_threading' + || '' + }}.txt handle_segv=0" >> "$GITHUB_ENV" + echo "CC=clang" >> "$GITHUB_ENV" + echo "CXX=clang++" >> "$GITHUB_ENV" + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" + - name: Configure CPython + run: >- + ./configure + --config-cache + --with-thread-sanitizer + --with-pydebug + ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: ./python -m test --tsan -j4 + - name: Display TSAN logs + if: always() + run: find "${GITHUB_WORKSPACE}" -name 'tsan_log.*' | xargs head -n 1000 + - name: Archive TSAN logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: >- + tsan-logs-${{ + fromJSON(inputs.free-threading) + && 'free-threading' + || 'default' + }} + path: tsan_log.* + if-no-files-found: ignore diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index ee64fe62a0bd0a..aa8ba00f19d8ca 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -1,44 +1,69 @@ +name: Reusable Ubuntu + on: workflow_call: inputs: config_hash: required: true type: string - options: - required: true - type: string + bolt-optimizations: + description: Whether to enable BOLT optimizations + required: false + type: boolean + default: false + free-threading: + description: Whether to use free-threaded mode + required: false + type: boolean + default: false + os: + description: OS to run the job + required: true + type: string + +env: + FORCE_COLOR: 1 jobs: build_ubuntu_reusable: - name: 'build and test' + name: build and test (${{ inputs.os }}) timeout-minutes: 60 - runs-on: ubuntu-20.04 + runs-on: ${{ inputs.os }} env: - OPENSSL_VER: 3.0.13 + OPENSSL_VER: 3.0.15 PYTHONSTRICTEXTENSIONBUILD: 1 + TERM: linux steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Install Clang and BOLT + if: ${{ fromJSON(inputs.bolt-optimizations) }} + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh 19 + sudo apt-get install bolt-19 + echo PATH="$(llvm-config-19 --bindir):$PATH" >> $GITHUB_ENV - name: Configure OpenSSL env vars run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + key: ${{ inputs.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: @@ -46,14 +71,14 @@ jobs: max-size: "200M" - name: Setup directory envs for out-of-tree builds run: | - echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" + echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - name: Create directories for read-only out-of-tree builds - run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR + run: mkdir -p "$CPYTHON_RO_SRCDIR" "$CPYTHON_BUILDDIR" - name: Bind mount sources read-only - run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: @@ -61,16 +86,42 @@ jobs: key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: ${{ inputs.options }} + # `test_unpickle_module_race` writes to the source directory, which is + # read-only during builds — so we exclude it from profiling with BOLT. + run: >- + PROFILE_TASK='-m test --pgo --ignore test_unpickle_module_race' + ../cpython-ro-srcdir/configure + --config-cache + --with-pydebug + --enable-slower-safety + --enable-safety + --with-openssl="$OPENSSL_DIR" + ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + ${{ fromJSON(inputs.bolt-optimizations) && '--enable-bolt' || '' }} - name: Build CPython out-of-tree + if: ${{ inputs.free-threading }} + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make -j + - name: Build CPython out-of-tree (for compiler warning check) + if: ${{ !inputs.free-threading }} working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make -j4 + run: set -o pipefail; make -j --output-sync 2>&1 | tee compiler_output_ubuntu.txt - name: Display build info working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make pythoninfo + - name: Check compiler warnings + if: ${{ !inputs.free-threading }} + run: >- + python Tools/build/check_warnings.py + --compiler-output-file-path="${CPYTHON_BUILDDIR}/compiler_output_ubuntu.txt" + --warning-ignore-file-path "${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu" + --compiler-output-type=gcc + --fail-on-regression + --fail-on-improvement + --path-prefix="../cpython-ro-srcdir/" - name: Remount sources writable for tests # some tests write to srcdir, lack of pyc files slows down testing - run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw + run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw - name: Tests working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: xvfb-run make test + run: xvfb-run make ci diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml new file mode 100644 index 00000000000000..4456b83c8e0032 --- /dev/null +++ b/.github/workflows/reusable-wasi.yml @@ -0,0 +1,82 @@ +name: Reusable WASI + +on: + workflow_call: + inputs: + config_hash: + required: true + type: string + +env: + FORCE_COLOR: 1 + +jobs: + build_wasi_reusable: + name: 'build and test' + timeout-minutes: 60 + runs-on: ubuntu-24.04 + env: + WASMTIME_VERSION: 22.0.0 + WASI_SDK_VERSION: 24 + WASI_SDK_PATH: /opt/wasi-sdk + CROSS_BUILD_PYTHON: cross-build/build + CROSS_BUILD_WASI: cross-build/wasm32-wasip1 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + # No problem resolver registered as one doesn't currently exist for Clang. + - name: "Install wasmtime" + uses: bytecodealliance/actions/wasmtime/setup@v1 + with: + version: ${{ env.WASMTIME_VERSION }} + - name: "Restore WASI SDK" + id: cache-wasi-sdk + uses: actions/cache@v4 + with: + path: ${{ env.WASI_SDK_PATH }} + key: ${{ runner.os }}-wasi-sdk-${{ env.WASI_SDK_VERSION }} + - name: "Install WASI SDK" # Hard-coded to x64. + if: steps.cache-wasi-sdk.outputs.cache-hit != 'true' + run: | + mkdir "${WASI_SDK_PATH}" && \ + curl -s -S --location "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz" | \ + tar --strip-components 1 --directory "${WASI_SDK_PATH}" --extract --gunzip + - name: "Configure ccache action" + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" + - name: "Add ccache to PATH" + run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: "Install Python" + uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: "Restore Python build config.cache" + uses: actions/cache@v4 + with: + path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache + # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python. + # Include the hash of `Tools/wasm/wasi.py` as it may change the environment variables. + # (Make sure to keep the key in sync with the other config.cache step below.) + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} + - name: "Configure build Python" + run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug + - name: "Make build Python" + run: python3 Tools/wasm/wasi.py make-build-python + - name: "Restore host config.cache" + uses: actions/cache@v4 + with: + path: ${{ env.CROSS_BUILD_WASI }}/config.cache + # Should be kept in sync with the other config.cache step above. + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} + - name: "Configure host" + # `--with-pydebug` inferred from configure-build-python + run: python3 Tools/wasm/wasi.py configure-host -- --config-cache + - name: "Make host" + run: python3 Tools/wasm/wasi.py make-host + - name: "Display build info" + run: make --directory "${CROSS_BUILD_WASI}" pythoninfo + - name: "Test" + run: make --directory "${CROSS_BUILD_WASI}" test diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml new file mode 100644 index 00000000000000..bc0414d1bbcd8f --- /dev/null +++ b/.github/workflows/reusable-windows-msi.yml @@ -0,0 +1,31 @@ +name: Reusable Windows MSI + +on: + workflow_call: + inputs: + arch: + description: CPU architecture + required: true + type: string + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + +jobs: + build: + name: installer for ${{ inputs.arch }} + runs-on: windows-latest + timeout-minutes: 60 + env: + ARCH: ${{ inputs.arch }} + IncludeFreethreaded: true + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Build CPython installer + run: ./Tools/msi/build.bat --doc -"${ARCH}" + shell: bash diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index c0209e0e1c92e9..5485a0169130b0 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -1,53 +1,56 @@ +name: Reusable Windows + on: workflow_call: inputs: + os: + description: OS to run on + required: true + type: string + arch: + description: CPU architecture + required: true + type: string free-threading: + description: Whether to compile CPython in free-threading mode required: false type: boolean default: false -jobs: - build_win32: - name: 'build and test (x86)' - runs-on: windows-latest - timeout-minutes: 60 - env: - IncludeUwp: 'true' - steps: - - uses: actions/checkout@v4 - - name: Build CPython - run: .\PCbuild\build.bat -e -d -v -p Win32 ${{ inputs.free-threading && '--disable-gil' || '' }} - - name: Display build info - run: .\python.bat -m test.pythoninfo - - name: Tests - run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci ${{ inputs.free-threading && '--disable-gil' || '' }} - - build_win_amd64: - name: 'build and test (x64)' - runs-on: windows-latest - timeout-minutes: 60 - env: - IncludeUwp: 'true' - steps: - - uses: actions/checkout@v4 - - name: Register MSVC problem matcher - run: echo "::add-matcher::.github/problem-matchers/msvc.json" - - name: Build CPython - run: .\PCbuild\build.bat -e -d -v -p x64 ${{ inputs.free-threading && '--disable-gil' || '' }} - - name: Display build info - run: .\python.bat -m test.pythoninfo - - name: Tests - run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci ${{ inputs.free-threading && '--disable-gil' || '' }} +env: + FORCE_COLOR: 1 + IncludeUwp: >- + true - build_win_arm64: - name: 'build (arm64)' - runs-on: windows-latest +jobs: + build: + name: ${{ inputs.arch == 'arm64' && 'build' || 'build and test' }} (${{ inputs.arch }}) + runs-on: ${{ inputs.os }} timeout-minutes: 60 env: - IncludeUwp: 'true' + ARCH: ${{ inputs.arch }} steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Register MSVC problem matcher + if: inputs.arch != 'Win32' run: echo "::add-matcher::.github/problem-matchers/msvc.json" - name: Build CPython - run: .\PCbuild\build.bat -e -d -v -p arm64 ${{ inputs.free-threading && '--disable-gil' || '' }} + run: >- + .\\PCbuild\\build.bat + -e -d -v + -p "${ARCH}" + ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + shell: bash + - name: Display build info # FIXME(diegorusso): remove the `if` + if: inputs.arch != 'arm64' + run: .\\python.bat -m test.pythoninfo + - name: Tests # FIXME(diegorusso): remove the `if` + if: inputs.arch != 'arm64' + run: >- + .\\PCbuild\\rt.bat + -p "${ARCH}" + -d -q --fast-ci + ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + shell: bash diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 07608fe91b4dbe..7578189f5d4d67 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,16 +2,15 @@ name: Mark stale pull requests on: schedule: - - cron: "0 0 * * *" - -permissions: - pull-requests: write + - cron: "0 */6 * * *" jobs: stale: if: github.repository_owner == 'python' runs-on: ubuntu-latest + permissions: + pull-requests: write timeout-minutes: 10 steps: diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 83b007f1c9c2ef..463e7bf3355cc3 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -26,6 +26,8 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: '3' diff --git a/.github/zizmor.yml b/.github/zizmor.yml new file mode 100644 index 00000000000000..eeda8d9eaaf484 --- /dev/null +++ b/.github/zizmor.yml @@ -0,0 +1,6 @@ +# Configuration for the zizmor static analysis tool, run via pre-commit in CI +# https://woodruffw.github.io/zizmor/configuration/ +rules: + dangerous-triggers: + ignore: + - documentation-links.yml diff --git a/.gitignore b/.gitignore index 2194b393aa4821..8872e9d5508ff1 100644 --- a/.gitignore +++ b/.gitignore @@ -69,7 +69,17 @@ Lib/test/data/* /_bootstrap_python /Makefile /Makefile.pre +/iOSTestbed.* +iOS/Frameworks/ iOS/Resources/Info.plist +iOS/testbed/build +iOS/testbed/Python.xcframework/ios-*/bin +iOS/testbed/Python.xcframework/ios-*/include +iOS/testbed/Python.xcframework/ios-*/lib +iOS/testbed/Python.xcframework/ios-*/Python.framework +iOS/testbed/iOSTestbed.xcodeproj/project.xcworkspace +iOS/testbed/iOSTestbed.xcodeproj/xcuserdata +iOS/testbed/iOSTestbed.xcodeproj/xcshareddata Mac/Makefile Mac/PythonLauncher/Info.plist Mac/PythonLauncher/Makefile @@ -132,7 +142,6 @@ Tools/unicode/data/ /profile-clean-stamp /profile-run-stamp /profile-bolt-stamp -/Python/deepfreeze/*.c /pybuilddir.txt /pyconfig.h /python-config diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 69d85238985150..fb44c27704d455 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,19 +1,44 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.2.0 + rev: v0.9.1 hooks: - id: ruff - name: Run Ruff on Lib/test/ + name: Run Ruff (lint) on Doc/ + args: [--exit-non-zero-on-fix] + files: ^Doc/ + - id: ruff + name: Run Ruff (lint) on Lib/test/ args: [--exit-non-zero-on-fix] files: ^Lib/test/ - id: ruff - name: Run Ruff on Argument Clinic + name: Run Ruff (lint) on Tools/build/check_warnings.py + args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml] + files: ^Tools/build/check_warnings.py + - id: ruff + name: Run Ruff (lint) on Argument Clinic args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml] files: ^Tools/clinic/|Lib/test/test_clinic.py + - id: ruff-format + name: Run Ruff (format) on Doc/ + args: [--check] + files: ^Doc/ + + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.10.0 + hooks: + - id: black + name: Run Black on Tools/build/check_warnings.py + files: ^Tools/build/check_warnings.py + args: [--line-length=79] + - id: black + name: Run Black on Tools/jit/ + files: ^Tools/jit/ - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v5.0.0 hooks: + - id: check-case-conflict + - id: check-merge-conflict - id: check-toml exclude: ^Lib/test/test_tomllib/ - id: check-yaml @@ -23,13 +48,46 @@ repos: - id: trailing-whitespace types_or: [c, inc, python, rst] + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.31.0 + hooks: + - id: check-dependabot + - id: check-github-workflows + - id: check-readthedocs + + - repo: https://github.com/rhysd/actionlint + rev: v1.7.7 + hooks: + - id: actionlint + + - repo: https://github.com/woodruffw/zizmor-pre-commit + rev: v1.1.1 + hooks: + - id: zizmor + - repo: https://github.com/sphinx-contrib/sphinx-lint - rev: v0.9.1 + rev: v1.0.0 hooks: - id: sphinx-lint args: [--enable=default-role] files: ^Doc/|^Misc/NEWS.d/ + - repo: local + hooks: + - id: blurb-no-space-c-api + name: Check C API news entries + language: fail + entry: Space found in path, move to Misc/NEWS.d/next/C_API/ + files: Misc/NEWS.d/next/C API/20.*.rst + + - repo: local + hooks: + - id: blurb-no-space-core-and-builtins + name: Check Core and Builtins news entries + language: fail + entry: Space found in path, move to Misc/NEWS.d/next/Core_and_Builtins/ + files: Misc/NEWS.d/next/Core and Builtins/20.*.rst + - repo: meta hooks: - id: check-hooks-apply diff --git a/.readthedocs.yml b/.readthedocs.yml index 59830c79a404e0..a57de00544e4e3 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -8,7 +8,7 @@ sphinx: configuration: Doc/conf.py build: - os: ubuntu-22.04 + os: ubuntu-24.04 tools: python: "3" @@ -26,6 +26,9 @@ build: exit 183; fi + - asdf plugin add uv + - asdf install uv latest + - asdf global uv latest - make -C Doc venv html - mkdir _readthedocs - mv Doc/build/html _readthedocs/html diff --git a/Android/README.md b/Android/README.md new file mode 100644 index 00000000000000..3daa545cc93746 --- /dev/null +++ b/Android/README.md @@ -0,0 +1,136 @@ +# Python for Android + +These instructions are only needed if you're planning to compile Python for +Android yourself. Most users should *not* need to do this. Instead, use one of +the tools listed in `Doc/using/android.rst`, which will provide a much easier +experience. + + +## Prerequisites + +First, make sure you have all the usual tools and libraries needed to build +Python for your development machine. + +Second, you'll need an Android SDK. If you already have the SDK installed, +export the `ANDROID_HOME` environment variable to point at its location. +Otherwise, here's how to install it: + +* Download the "Command line tools" from . +* Create a directory `android-sdk/cmdline-tools`, and unzip the command line + tools package into it. +* Rename `android-sdk/cmdline-tools/cmdline-tools` to + `android-sdk/cmdline-tools/latest`. +* `export ANDROID_HOME=/path/to/android-sdk` + +The `android.py` script also requires the following commands to be on the `PATH`: + +* `curl` +* `java` (or set the `JAVA_HOME` environment variable) +* `tar` +* `unzip` + + +## Building + +Python can be built for Android on any POSIX platform supported by the Android +development tools, which currently means Linux or macOS. This involves doing a +cross-build where you use a "build" Python (for your development machine) to +help produce a "host" Python for Android. + +The easiest way to do a build is to use the `android.py` script. You can either +have it perform the entire build process from start to finish in one step, or +you can do it in discrete steps that mirror running `configure` and `make` for +each of the two builds of Python you end up producing. + +The discrete steps for building via `android.py` are: + +```sh +./android.py configure-build +./android.py make-build +./android.py configure-host HOST +./android.py make-host HOST +``` + +`HOST` identifies which architecture to build. To see the possible values, run +`./android.py configure-host --help`. + +To do all steps in a single command, run: + +```sh +./android.py build HOST +``` + +In the end you should have a build Python in `cross-build/build`, and an Android +build in `cross-build/HOST`. + +You can use `--` as a separator for any of the `configure`-related commands – +including `build` itself – to pass arguments to the underlying `configure` +call. For example, if you want a pydebug build that also caches the results from +`configure`, you can do: + +```sh +./android.py build HOST -- -C --with-pydebug +``` + + +## Testing + +The test suite can be run on Linux, macOS, or Windows: + +* On Linux, the emulator needs access to the KVM virtualization interface, and + a DISPLAY environment variable pointing at an X server. +* On Windows, you won't be able to do the build on the same machine, so you'll + have to copy the `cross-build/HOST` directory from somewhere else. + +The test suite can usually be run on a device with 2 GB of RAM, but this is +borderline, so you may need to increase it to 4 GB. As of Android +Studio Koala, 2 GB is the default for all emulators, although the user interface +may indicate otherwise. Locate the emulator's directory under `~/.android/avd`, +and find `hw.ramSize` in both config.ini and hardware-qemu.ini. Either set these +manually to the same value, or use the Android Studio Device Manager, which will +update both files. + +Before running the test suite, follow the instructions in the previous section +to build the architecture you want to test. Then run the test script in one of +the following modes: + +* In `--connected` mode, it runs on a device or emulator you have already + connected to the build machine. List the available devices with + `$ANDROID_HOME/platform-tools/adb devices -l`, then pass a device ID to the + script like this: + + ```sh + ./android.py test --connected emulator-5554 + ``` + +* In `--managed` mode, it uses a temporary headless emulator defined in the + `managedDevices` section of testbed/app/build.gradle.kts. This mode is slower, + but more reproducible. + + We currently define two devices: `minVersion` and `maxVersion`, corresponding + to our minimum and maximum supported Android versions. For example: + + ```sh + ./android.py test --managed maxVersion + ``` + +By default, the only messages the script will show are Python's own stdout and +stderr. Add the `-v` option to also show Gradle output, and non-Python logcat +messages. + +Any other arguments on the `android.py test` command line will be passed through +to `python -m test` – use `--` to separate them from android.py's own options. +See the [Python Developer's +Guide](https://devguide.python.org/testing/run-write-tests/) for common options +– most of them will work on Android, except for those that involve subprocesses, +such as `-j`. + +Every time you run `android.py test`, changes in pure-Python files in the +repository's `Lib` directory will be picked up immediately. Changes in C files, +and architecture-specific files such as sysconfigdata, will not take effect +until you re-run `android.py make-host` or `build`. + + +## Using in your own app + +See `Doc/using/android.rst`. diff --git a/Android/android-env.sh b/Android/android-env.sh new file mode 100644 index 00000000000000..bab4130c9e92d0 --- /dev/null +++ b/Android/android-env.sh @@ -0,0 +1,99 @@ +# This script must be sourced with the following variables already set: +: "${ANDROID_HOME:?}" # Path to Android SDK +: "${HOST:?}" # GNU target triplet + +# You may also override the following: +: "${api_level:=24}" # Minimum Android API level the build will run on +: "${PREFIX:-}" # Path in which to find required libraries + + +# Print all messages on stderr so they're visible when running within build-wheel. +log() { + echo "$1" >&2 +} + +fail() { + log "$1" + exit 1 +} + +# When moving to a new version of the NDK, carefully review the following: +# +# * https://developer.android.com/ndk/downloads/revision_history +# +# * https://android.googlesource.com/platform/ndk/+/ndk-rXX-release/docs/BuildSystemMaintainers.md +# where XX is the NDK version. Do a diff against the version you're upgrading from, e.g.: +# https://android.googlesource.com/platform/ndk/+/ndk-r25-release..ndk-r26-release/docs/BuildSystemMaintainers.md +ndk_version=27.1.12297006 + +ndk=$ANDROID_HOME/ndk/$ndk_version +if ! [ -e "$ndk" ]; then + log "Installing NDK - this may take several minutes" + yes | "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" "ndk;$ndk_version" +fi + +if [ "$HOST" = "arm-linux-androideabi" ]; then + clang_triplet=armv7a-linux-androideabi +else + clang_triplet="$HOST" +fi + +# These variables are based on BuildSystemMaintainers.md above, and +# $ndk/build/cmake/android.toolchain.cmake. +toolchain=$(echo "$ndk"/toolchains/llvm/prebuilt/*) +export AR="$toolchain/bin/llvm-ar" +export AS="$toolchain/bin/llvm-as" +export CC="$toolchain/bin/${clang_triplet}${api_level}-clang" +export CXX="${CC}++" +export LD="$toolchain/bin/ld" +export NM="$toolchain/bin/llvm-nm" +export RANLIB="$toolchain/bin/llvm-ranlib" +export READELF="$toolchain/bin/llvm-readelf" +export STRIP="$toolchain/bin/llvm-strip" + +# The quotes make sure the wildcard in the `toolchain` assignment has been expanded. +for path in "$AR" "$AS" "$CC" "$CXX" "$LD" "$NM" "$RANLIB" "$READELF" "$STRIP"; do + if ! [ -e "$path" ]; then + fail "$path does not exist" + fi +done + +export CFLAGS="-D__BIONIC_NO_PAGE_SIZE_MACRO" +export LDFLAGS="-Wl,--build-id=sha1 -Wl,--no-rosegment -Wl,-z,max-page-size=16384" + +# Unlike Linux, Android does not implicitly use a dlopened library to resolve +# relocations in subsequently-loaded libraries, even if RTLD_GLOBAL is used +# (https://github.com/android/ndk/issues/1244). So any library that fails to +# build with this flag, would also fail to load at runtime. +LDFLAGS="$LDFLAGS -Wl,--no-undefined" + +# Many packages get away with omitting -lm on Linux, but Android is stricter. +LDFLAGS="$LDFLAGS -lm" + +# -mstackrealign is included where necessary in the clang launcher scripts which are +# pointed to by $CC, so we don't need to include it here. +if [ "$HOST" = "arm-linux-androideabi" ]; then + CFLAGS="$CFLAGS -march=armv7-a -mthumb" +fi + +if [ -n "${PREFIX:-}" ]; then + abs_prefix="$(realpath "$PREFIX")" + CFLAGS="$CFLAGS -I$abs_prefix/include" + LDFLAGS="$LDFLAGS -L$abs_prefix/lib" + + export PKG_CONFIG="pkg-config --define-prefix" + export PKG_CONFIG_LIBDIR="$abs_prefix/lib/pkgconfig" +fi + +# When compiling C++, some build systems will combine CFLAGS and CXXFLAGS, and some will +# use CXXFLAGS alone. +export CXXFLAGS="$CFLAGS" + +# Use the same variable name as conda-build +if [ "$(uname)" = "Darwin" ]; then + CPU_COUNT="$(sysctl -n hw.ncpu)" + export CPU_COUNT +else + CPU_COUNT="$(nproc)" + export CPU_COUNT +fi diff --git a/Android/android.py b/Android/android.py new file mode 100755 index 00000000000000..ae630aa8f4427c --- /dev/null +++ b/Android/android.py @@ -0,0 +1,650 @@ +#!/usr/bin/env python3 + +import asyncio +import argparse +from glob import glob +import os +import re +import shlex +import shutil +import signal +import subprocess +import sys +import sysconfig +from asyncio import wait_for +from contextlib import asynccontextmanager +from os.path import basename, relpath +from pathlib import Path +from subprocess import CalledProcessError +from tempfile import TemporaryDirectory + + +SCRIPT_NAME = Path(__file__).name +CHECKOUT = Path(__file__).resolve().parent.parent +ANDROID_DIR = CHECKOUT / "Android" +TESTBED_DIR = ANDROID_DIR / "testbed" +CROSS_BUILD_DIR = CHECKOUT / "cross-build" + +APP_ID = "org.python.testbed" +DECODE_ARGS = ("UTF-8", "backslashreplace") + + +try: + android_home = Path(os.environ['ANDROID_HOME']) +except KeyError: + sys.exit("The ANDROID_HOME environment variable is required.") + +adb = Path( + f"{android_home}/platform-tools/adb" + + (".exe" if os.name == "nt" else "") +) + +gradlew = Path( + f"{TESTBED_DIR}/gradlew" + + (".bat" if os.name == "nt" else "") +) + +logcat_started = False + + +def delete_glob(pattern): + # Path.glob doesn't accept non-relative patterns. + for path in glob(str(pattern)): + path = Path(path) + print(f"Deleting {path} ...") + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + else: + path.unlink() + + +def subdir(name, *, clean=None): + path = CROSS_BUILD_DIR / name + if clean: + delete_glob(path) + if not path.exists(): + if clean is None: + sys.exit( + f"{path} does not exist. Create it by running the appropriate " + f"`configure` subcommand of {SCRIPT_NAME}.") + else: + path.mkdir(parents=True) + return path + + +def run(command, *, host=None, env=None, log=True, **kwargs): + kwargs.setdefault("check", True) + if env is None: + env = os.environ.copy() + original_env = env.copy() + + if host: + env_script = ANDROID_DIR / "android-env.sh" + env_output = subprocess.run( + f"set -eu; " + f"HOST={host}; " + f"PREFIX={subdir(host)}/prefix; " + f". {env_script}; " + f"export", + check=True, shell=True, text=True, stdout=subprocess.PIPE + ).stdout + + for line in env_output.splitlines(): + # We don't require every line to match, as there may be some other + # output from installing the NDK. + if match := re.search( + "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line + ): + key, value = match[2], match[3] + if env.get(key) != value: + print(line) + env[key] = value + + if env == original_env: + raise ValueError(f"Found no variables in {env_script.name} output:\n" + + env_output) + + if log: + print(">", " ".join(map(str, command))) + return subprocess.run(command, env=env, **kwargs) + + +def build_python_path(): + """The path to the build Python binary.""" + build_dir = subdir("build") + binary = build_dir / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError("Unable to find `python(.exe)` in " + f"{build_dir}") + + return binary + + +def configure_build_python(context): + os.chdir(subdir("build", clean=context.clean)) + + command = [relpath(CHECKOUT / "configure")] + if context.args: + command.extend(context.args) + run(command) + + +def make_build_python(context): + os.chdir(subdir("build")) + run(["make", "-j", str(os.cpu_count())]) + + +def unpack_deps(host): + deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" + for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-4", + "sqlite-3.45.3-3", "xz-5.4.6-1"]: + filename = f"{name_ver}-{host}.tar.gz" + download(f"{deps_url}/{name_ver}/{filename}") + run(["tar", "-xf", filename]) + os.remove(filename) + + +def download(url, target_dir="."): + out_path = f"{target_dir}/{basename(url)}" + run(["curl", "-Lf", "-o", out_path, url]) + return out_path + + +def configure_host_python(context): + host_dir = subdir(context.host, clean=context.clean) + + prefix_dir = host_dir / "prefix" + if not prefix_dir.exists(): + prefix_dir.mkdir() + os.chdir(prefix_dir) + unpack_deps(context.host) + + build_dir = host_dir / "build" + build_dir.mkdir(exist_ok=True) + os.chdir(build_dir) + + command = [ + # Basic cross-compiling configuration + relpath(CHECKOUT / "configure"), + f"--host={context.host}", + f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", + f"--with-build-python={build_python_path()}", + "--without-ensurepip", + + # Android always uses a shared libpython. + "--enable-shared", + "--without-static-libpython", + + # Dependent libraries. The others are found using pkg-config: see + # android-env.sh. + f"--with-openssl={prefix_dir}", + ] + + if context.args: + command.extend(context.args) + run(command, host=context.host) + + +def make_host_python(context): + # The CFLAGS and LDFLAGS set in android-env include the prefix dir, so + # delete any previous Python installation to prevent it being used during + # the build. + host_dir = subdir(context.host) + prefix_dir = host_dir / "prefix" + delete_glob(f"{prefix_dir}/include/python*") + delete_glob(f"{prefix_dir}/lib/libpython*") + delete_glob(f"{prefix_dir}/lib/python*") + + os.chdir(host_dir / "build") + run(["make", "-j", str(os.cpu_count())], host=context.host) + run(["make", "install", f"prefix={prefix_dir}"], host=context.host) + + +def build_all(context): + steps = [configure_build_python, make_build_python, configure_host_python, + make_host_python] + for step in steps: + step(context) + + +def clean_all(context): + delete_glob(CROSS_BUILD_DIR) + + +def setup_sdk(): + sdkmanager = android_home / ( + "cmdline-tools/latest/bin/sdkmanager" + + (".bat" if os.name == "nt" else "") + ) + + # Gradle will fail if it needs to install an SDK package whose license + # hasn't been accepted, so pre-accept all licenses. + if not all((android_home / "licenses" / path).exists() for path in [ + "android-sdk-arm-dbt-license", "android-sdk-license" + ]): + run([sdkmanager, "--licenses"], text=True, input="y\n" * 100) + + # Gradle may install this automatically, but we can't rely on that because + # we need to run adb within the logcat task. + if not adb.exists(): + run([sdkmanager, "platform-tools"]) + + +# To avoid distributing compiled artifacts without corresponding source code, +# the Gradle wrapper is not included in the CPython repository. Instead, we +# extract it from the Gradle release. +def setup_testbed(): + if all((TESTBED_DIR / path).exists() for path in [ + "gradlew", "gradlew.bat", "gradle/wrapper/gradle-wrapper.jar", + ]): + return + + ver_long = "8.7.0" + ver_short = ver_long.removesuffix(".0") + + for filename in ["gradlew", "gradlew.bat"]: + out_path = download( + f"https://raw.githubusercontent.com/gradle/gradle/v{ver_long}/{filename}", + TESTBED_DIR) + os.chmod(out_path, 0o755) + + with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: + bin_zip = download( + f"https://services.gradle.org/distributions/gradle-{ver_short}-bin.zip", + temp_dir) + outer_jar = f"gradle-{ver_short}/lib/plugins/gradle-wrapper-{ver_short}.jar" + run(["unzip", "-d", temp_dir, bin_zip, outer_jar]) + run(["unzip", "-o", "-d", f"{TESTBED_DIR}/gradle/wrapper", + f"{temp_dir}/{outer_jar}", "gradle-wrapper.jar"]) + + +# run_testbed will build the app automatically, but it's useful to have this as +# a separate command to allow running the app outside of this script. +def build_testbed(context): + setup_sdk() + setup_testbed() + run( + [gradlew, "--console", "plain", "packageDebug", "packageDebugAndroidTest"], + cwd=TESTBED_DIR, + ) + + +# Work around a bug involving sys.exit and TaskGroups +# (https://github.com/python/cpython/issues/101515). +def exit(*args): + raise MySystemExit(*args) + + +class MySystemExit(Exception): + pass + + +# The `test` subcommand runs all subprocesses through this context manager so +# that no matter what happens, they can always be cancelled from another task, +# and they will always be cleaned up on exit. +@asynccontextmanager +async def async_process(*args, **kwargs): + process = await asyncio.create_subprocess_exec(*args, **kwargs) + try: + yield process + finally: + if process.returncode is None: + # Allow a reasonably long time for Gradle to clean itself up, + # because we don't want stale emulators left behind. + timeout = 10 + process.terminate() + try: + await wait_for(process.wait(), timeout) + except TimeoutError: + print( + f"Command {args} did not terminate after {timeout} seconds " + f" - sending SIGKILL" + ) + process.kill() + + # Even after killing the process we must still wait for it, + # otherwise we'll get the warning "Exception ignored in __del__". + await wait_for(process.wait(), timeout=1) + + +async def async_check_output(*args, **kwargs): + async with async_process( + *args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs + ) as process: + stdout, stderr = await process.communicate() + if process.returncode == 0: + return stdout.decode(*DECODE_ARGS) + else: + raise CalledProcessError( + process.returncode, args, + stdout.decode(*DECODE_ARGS), stderr.decode(*DECODE_ARGS) + ) + + +# Return a list of the serial numbers of connected devices. Emulators will have +# serials of the form "emulator-5678". +async def list_devices(): + serials = [] + header_found = False + + lines = (await async_check_output(adb, "devices")).splitlines() + for line in lines: + # Ignore blank lines, and all lines before the header. + line = line.strip() + if line == "List of devices attached": + header_found = True + elif header_found and line: + try: + serial, status = line.split() + except ValueError: + raise ValueError(f"failed to parse {line!r}") + if status == "device": + serials.append(serial) + + if not header_found: + raise ValueError(f"failed to parse {lines}") + return serials + + +async def find_device(context, initial_devices): + if context.managed: + print("Waiting for managed device - this may take several minutes") + while True: + new_devices = set(await list_devices()).difference(initial_devices) + if len(new_devices) == 0: + await asyncio.sleep(1) + elif len(new_devices) == 1: + serial = new_devices.pop() + print(f"Serial: {serial}") + return serial + else: + exit(f"Found more than one new device: {new_devices}") + else: + return context.connected + + +# An older version of this script in #121595 filtered the logs by UID instead. +# But logcat can't filter by UID until API level 31. If we ever switch back to +# filtering by UID, we'll also have to filter by time so we only show messages +# produced after the initial call to `stop_app`. +# +# We're more likely to miss the PID because it's shorter-lived, so there's a +# workaround in PythonSuite.kt to stop it being *too* short-lived. +async def find_pid(serial): + print("Waiting for app to start - this may take several minutes") + shown_error = False + while True: + try: + # `pidof` requires API level 24 or higher. The level 23 emulator + # includes it, but it doesn't work (it returns all processes). + pid = (await async_check_output( + adb, "-s", serial, "shell", "pidof", "-s", APP_ID + )).strip() + except CalledProcessError as e: + # If the app isn't running yet, pidof gives no output. So if there + # is output, there must have been some other error. However, this + # sometimes happens transiently, especially when running a managed + # emulator for the first time, so don't make it fatal. + if (e.stdout or e.stderr) and not shown_error: + print_called_process_error(e) + print("This may be transient, so continuing to wait") + shown_error = True + else: + # Some older devices (e.g. Nexus 4) return zero even when no process + # was found, so check whether we actually got any output. + if pid: + print(f"PID: {pid}") + return pid + + # Loop fairly rapidly to avoid missing a short-lived process. + await asyncio.sleep(0.2) + + +async def logcat_task(context, initial_devices): + # Gradle may need to do some large downloads of libraries and emulator + # images. This will happen during find_device in --managed mode, or find_pid + # in --connected mode. + startup_timeout = 600 + serial = await wait_for(find_device(context, initial_devices), startup_timeout) + pid = await wait_for(find_pid(serial), startup_timeout) + + # `--pid` requires API level 24 or higher. + args = [adb, "-s", serial, "logcat", "--pid", pid, "--format", "tag"] + hidden_output = [] + async with async_process( + *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) as process: + while line := (await process.stdout.readline()).decode(*DECODE_ARGS): + if match := re.fullmatch(r"([A-Z])/(.*)", line, re.DOTALL): + level, message = match.groups() + else: + # If the regex doesn't match, this is probably the second or + # subsequent line of a multi-line message. Python won't produce + # such messages, but other components might. + level, message = None, line + + # Exclude high-volume messages which are rarely useful. + if context.verbose < 2 and "from python test_syslog" in message: + continue + + # Put high-level messages on stderr so they're highlighted in the + # buildbot logs. This will include Python's own stderr. + stream = ( + sys.stderr + if level in ["W", "E", "F"] # WARNING, ERROR, FATAL (aka ASSERT) + else sys.stdout + ) + + # To simplify automated processing of the output, e.g. a buildbot + # posting a failure notice on a GitHub PR, we strip the level and + # tag indicators from Python's stdout and stderr. + for prefix in ["python.stdout: ", "python.stderr: "]: + if message.startswith(prefix): + global logcat_started + logcat_started = True + stream.write(message.removeprefix(prefix)) + break + else: + if context.verbose: + # Non-Python messages add a lot of noise, but they may + # sometimes help explain a failure. + stream.write(line) + else: + hidden_output.append(line) + + # If the device disconnects while logcat is running, which always + # happens in --managed mode, some versions of adb return non-zero. + # Distinguish this from a logcat startup error by checking whether we've + # received a message from Python yet. + status = await wait_for(process.wait(), timeout=1) + if status != 0 and not logcat_started: + raise CalledProcessError(status, args, "".join(hidden_output)) + + +def stop_app(serial): + run([adb, "-s", serial, "shell", "am", "force-stop", APP_ID], log=False) + + +async def gradle_task(context): + env = os.environ.copy() + if context.managed: + task_prefix = context.managed + else: + task_prefix = "connected" + env["ANDROID_SERIAL"] = context.connected + + args = [ + gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest", + "-Pandroid.testInstrumentationRunnerArguments.pythonArgs=" + + shlex.join(context.args), + ] + hidden_output = [] + try: + async with async_process( + *args, cwd=TESTBED_DIR, env=env, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) as process: + while line := (await process.stdout.readline()).decode(*DECODE_ARGS): + # Gradle may take several minutes to install SDK packages, so + # it's worth showing those messages even in non-verbose mode. + if context.verbose or line.startswith('Preparing "Install'): + sys.stdout.write(line) + else: + hidden_output.append(line) + + status = await wait_for(process.wait(), timeout=1) + if status == 0: + exit(0) + else: + raise CalledProcessError(status, args) + finally: + # If logcat never started, then something has gone badly wrong, so the + # user probably wants to see the Gradle output even in non-verbose mode. + if hidden_output and not logcat_started: + sys.stdout.write("".join(hidden_output)) + + # Gradle does not stop the tests when interrupted. + if context.connected: + stop_app(context.connected) + + +async def run_testbed(context): + setup_sdk() + setup_testbed() + + if context.managed: + # In this mode, Gradle will create a device with an unpredictable name. + # So we save a list of the running devices before starting Gradle, and + # find_device then waits for a new device to appear. + initial_devices = await list_devices() + else: + # In case the previous shutdown was unclean, make sure the app isn't + # running, otherwise we might show logs from a previous run. This is + # unnecessary in --managed mode, because Gradle creates a new emulator + # every time. + stop_app(context.connected) + initial_devices = None + + try: + async with asyncio.TaskGroup() as tg: + tg.create_task(logcat_task(context, initial_devices)) + tg.create_task(gradle_task(context)) + except* MySystemExit as e: + raise SystemExit(*e.exceptions[0].args) from None + except* CalledProcessError as e: + # Extract it from the ExceptionGroup so it can be handled by `main`. + raise e.exceptions[0] + + +# Handle SIGTERM the same way as SIGINT. This ensures that if we're terminated +# by the buildbot worker, we'll make an attempt to clean up our subprocesses. +def install_signal_handler(): + def signal_handler(*args): + os.kill(os.getpid(), signal.SIGINT) + + signal.signal(signal.SIGTERM, signal_handler) + + +def parse_args(): + parser = argparse.ArgumentParser() + subcommands = parser.add_subparsers(dest="subcommand") + build = subcommands.add_parser("build", help="Build everything") + configure_build = subcommands.add_parser("configure-build", + help="Run `configure` for the " + "build Python") + make_build = subcommands.add_parser("make-build", + help="Run `make` for the build Python") + configure_host = subcommands.add_parser("configure-host", + help="Run `configure` for Android") + make_host = subcommands.add_parser("make-host", + help="Run `make` for Android") + subcommands.add_parser( + "clean", help="Delete the cross-build directory") + + for subcommand in build, configure_build, configure_host: + subcommand.add_argument( + "--clean", action="store_true", default=False, dest="clean", + help="Delete any relevant directories before building") + for subcommand in build, configure_host, make_host: + subcommand.add_argument( + "host", metavar="HOST", + choices=["aarch64-linux-android", "x86_64-linux-android"], + help="Host triplet: choices=[%(choices)s]") + for subcommand in build, configure_build, configure_host: + subcommand.add_argument("args", nargs="*", + help="Extra arguments to pass to `configure`") + + subcommands.add_parser( + "build-testbed", help="Build the testbed app") + test = subcommands.add_parser( + "test", help="Run the test suite") + test.add_argument( + "-v", "--verbose", action="count", default=0, + help="Show Gradle output, and non-Python logcat messages. " + "Use twice to include high-volume messages which are rarely useful.") + device_group = test.add_mutually_exclusive_group(required=True) + device_group.add_argument( + "--connected", metavar="SERIAL", help="Run on a connected device. " + "Connect it yourself, then get its serial from `adb devices`.") + device_group.add_argument( + "--managed", metavar="NAME", help="Run on a Gradle-managed device. " + "These are defined in `managedDevices` in testbed/app/build.gradle.kts.") + test.add_argument( + "args", nargs="*", help=f"Arguments for `python -m test`. " + f"Separate them from {SCRIPT_NAME}'s own arguments with `--`.") + + return parser.parse_args() + + +def main(): + install_signal_handler() + + # Under the buildbot, stdout is not a TTY, but we must still flush after + # every line to make sure our output appears in the correct order relative + # to the output of our subprocesses. + for stream in [sys.stdout, sys.stderr]: + stream.reconfigure(line_buffering=True) + + context = parse_args() + dispatch = {"configure-build": configure_build_python, + "make-build": make_build_python, + "configure-host": configure_host_python, + "make-host": make_host_python, + "build": build_all, + "clean": clean_all, + "build-testbed": build_testbed, + "test": run_testbed} + + try: + result = dispatch[context.subcommand](context) + if asyncio.iscoroutine(result): + asyncio.run(result) + except CalledProcessError as e: + print_called_process_error(e) + sys.exit(1) + + +def print_called_process_error(e): + for stream_name in ["stdout", "stderr"]: + content = getattr(e, stream_name) + stream = getattr(sys, stream_name) + if content: + stream.write(content) + if not content.endswith("\n"): + stream.write("\n") + + # Format the command so it can be copied into a shell. shlex uses single + # quotes, so we surround the whole command with double quotes. + args_joined = ( + e.cmd if isinstance(e.cmd, str) + else " ".join(shlex.quote(str(arg)) for arg in e.cmd) + ) + print( + f'Command "{args_joined}" returned exit status {e.returncode}' + ) + + +if __name__ == "__main__": + main() diff --git a/Android/testbed/.gitignore b/Android/testbed/.gitignore new file mode 100644 index 00000000000000..b9a7d611c943cf --- /dev/null +++ b/Android/testbed/.gitignore @@ -0,0 +1,21 @@ +# The Gradle wrapper should be downloaded by running `../android.py setup-testbed`. +/gradlew +/gradlew.bat +/gradle/wrapper/gradle-wrapper.jar + +*.iml +.gradle +/local.properties +/.idea/caches +/.idea/deploymentTargetDropdown.xml +/.idea/libraries +/.idea/modules.xml +/.idea/workspace.xml +/.idea/navEditor.xml +/.idea/assetWizardSettings.xml +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties diff --git a/Android/testbed/app/.gitignore b/Android/testbed/app/.gitignore new file mode 100644 index 00000000000000..42afabfd2abebf --- /dev/null +++ b/Android/testbed/app/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts new file mode 100644 index 00000000000000..211b5bbfadf64d --- /dev/null +++ b/Android/testbed/app/build.gradle.kts @@ -0,0 +1,200 @@ +import com.android.build.api.variant.* +import kotlin.math.max + +plugins { + id("com.android.application") + id("org.jetbrains.kotlin.android") +} + +val PYTHON_DIR = file("../../..").canonicalPath +val PYTHON_CROSS_DIR = "$PYTHON_DIR/cross-build" + +val ABIS = mapOf( + "arm64-v8a" to "aarch64-linux-android", + "x86_64" to "x86_64-linux-android", +).filter { file("$PYTHON_CROSS_DIR/${it.value}").exists() } +if (ABIS.isEmpty()) { + throw GradleException( + "No Android ABIs found in $PYTHON_CROSS_DIR: see Android/README.md " + + "for building instructions." + ) +} + +val PYTHON_VERSION = file("$PYTHON_DIR/Include/patchlevel.h").useLines { + for (line in it) { + val match = """#define PY_VERSION\s+"(\d+\.\d+)""".toRegex().find(line) + if (match != null) { + return@useLines match.groupValues[1] + } + } + throw GradleException("Failed to find Python version") +} + + +android { + val androidEnvFile = file("../../android-env.sh").absoluteFile + + namespace = "org.python.testbed" + compileSdk = 34 + + defaultConfig { + applicationId = "org.python.testbed" + + minSdk = androidEnvFile.useLines { + for (line in it) { + """api_level:=(\d+)""".toRegex().find(line)?.let { + return@useLines it.groupValues[1].toInt() + } + } + throw GradleException("Failed to find API level in $androidEnvFile") + } + targetSdk = 34 + + versionCode = 1 + versionName = "1.0" + + ndk.abiFilters.addAll(ABIS.keys) + externalNativeBuild.cmake.arguments( + "-DPYTHON_CROSS_DIR=$PYTHON_CROSS_DIR", + "-DPYTHON_VERSION=$PYTHON_VERSION", + "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON", + ) + + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + } + + ndkVersion = androidEnvFile.useLines { + for (line in it) { + """ndk_version=(\S+)""".toRegex().find(line)?.let { + return@useLines it.groupValues[1] + } + } + throw GradleException("Failed to find NDK version in $androidEnvFile") + } + externalNativeBuild.cmake { + path("src/main/c/CMakeLists.txt") + } + + // Set this property to something non-empty, otherwise it'll use the default + // list, which ignores asset directories beginning with an underscore. + aaptOptions.ignoreAssetsPattern = ".git" + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + } + kotlinOptions { + jvmTarget = "1.8" + } + + testOptions { + managedDevices { + localDevices { + create("minVersion") { + device = "Small Phone" + + // Managed devices have a minimum API level of 27. + apiLevel = max(27, defaultConfig.minSdk!!) + + // ATD devices are smaller and faster, but have a minimum + // API level of 30. + systemImageSource = if (apiLevel >= 30) "aosp-atd" else "aosp" + } + + create("maxVersion") { + device = "Small Phone" + apiLevel = defaultConfig.targetSdk!! + systemImageSource = "aosp-atd" + } + } + + // If the previous test run succeeded and nothing has changed, + // Gradle thinks there's no need to run it again. Override that. + afterEvaluate { + (localDevices.names + listOf("connected")).forEach { + tasks.named("${it}DebugAndroidTest") { + outputs.upToDateWhen { false } + } + } + } + } + } +} + +dependencies { + implementation("androidx.appcompat:appcompat:1.6.1") + implementation("com.google.android.material:material:1.11.0") + implementation("androidx.constraintlayout:constraintlayout:2.1.4") + androidTestImplementation("androidx.test.ext:junit:1.1.5") + androidTestImplementation("androidx.test:rules:1.5.0") +} + + +// Create some custom tasks to copy Python and its standard library from +// elsewhere in the repository. +androidComponents.onVariants { variant -> + val pyPlusVer = "python$PYTHON_VERSION" + generateTask(variant, variant.sources.assets!!) { + into("python") { + into("include/$pyPlusVer") { + for (triplet in ABIS.values) { + from("$PYTHON_CROSS_DIR/$triplet/prefix/include/$pyPlusVer") + } + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + into("lib/$pyPlusVer") { + // To aid debugging, the source directory takes priority. + from("$PYTHON_DIR/Lib") + + // The cross-build directory provides ABI-specific files such as + // sysconfigdata. + for (triplet in ABIS.values) { + from("$PYTHON_CROSS_DIR/$triplet/prefix/lib/$pyPlusVer") + } + + into("site-packages") { + from("$projectDir/src/main/python") + } + + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + exclude("**/__pycache__") + } + } + } + + generateTask(variant, variant.sources.jniLibs!!) { + for ((abi, triplet) in ABIS.entries) { + into(abi) { + from("$PYTHON_CROSS_DIR/$triplet/prefix/lib") + include("libpython*.*.so") + include("lib*_python.so") + } + } + } +} + + +fun generateTask( + variant: ApplicationVariant, directories: SourceDirectories, + configure: GenerateTask.() -> Unit +) { + val taskName = "generate" + + listOf(variant.name, "Python", directories.name) + .map { it.replaceFirstChar(Char::uppercase) } + .joinToString("") + + directories.addGeneratedSourceDirectory( + tasks.register(taskName) { + into(outputDir) + configure() + }, + GenerateTask::outputDir) +} + + +// addGeneratedSourceDirectory requires the task to have a DirectoryProperty. +abstract class GenerateTask: Sync() { + @get:OutputDirectory + abstract val outputDir: DirectoryProperty +} diff --git a/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt new file mode 100644 index 00000000000000..0e888ab71d87da --- /dev/null +++ b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt @@ -0,0 +1,35 @@ +package org.python.testbed + +import androidx.test.annotation.UiThreadTest +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.ext.junit.runners.AndroidJUnit4 + +import org.junit.Test +import org.junit.runner.RunWith + +import org.junit.Assert.* + + +@RunWith(AndroidJUnit4::class) +class PythonSuite { + @Test + @UiThreadTest + fun testPython() { + val start = System.currentTimeMillis() + try { + val context = + InstrumentationRegistry.getInstrumentation().targetContext + val args = + InstrumentationRegistry.getArguments().getString("pythonArgs", "") + val status = PythonTestRunner(context).run(args) + assertEquals(0, status) + } finally { + // Make sure the process lives long enough for the test script to + // detect it (see `find_pid` in android.py). + val delay = 2000 - (System.currentTimeMillis() - start) + if (delay > 0) { + Thread.sleep(delay) + } + } + } +} diff --git a/Android/testbed/app/src/main/AndroidManifest.xml b/Android/testbed/app/src/main/AndroidManifest.xml new file mode 100644 index 00000000000000..2be8a82d426099 --- /dev/null +++ b/Android/testbed/app/src/main/AndroidManifest.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Android/testbed/app/src/main/c/CMakeLists.txt b/Android/testbed/app/src/main/c/CMakeLists.txt new file mode 100644 index 00000000000000..1d5df9a73465b6 --- /dev/null +++ b/Android/testbed/app/src/main/c/CMakeLists.txt @@ -0,0 +1,9 @@ +cmake_minimum_required(VERSION 3.4.1) +project(testbed) + +set(PREFIX_DIR ${PYTHON_CROSS_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/prefix) +include_directories(${PREFIX_DIR}/include/python${PYTHON_VERSION}) +link_directories(${PREFIX_DIR}/lib) +link_libraries(log python${PYTHON_VERSION}) + +add_library(main_activity SHARED main_activity.c) diff --git a/Android/testbed/app/src/main/c/main_activity.c b/Android/testbed/app/src/main/c/main_activity.c new file mode 100644 index 00000000000000..ec7f93a3e5ee13 --- /dev/null +++ b/Android/testbed/app/src/main/c/main_activity.c @@ -0,0 +1,152 @@ +#include +#include +#include +#include +#include +#include +#include +#include + + +static void throw_runtime_exception(JNIEnv *env, const char *message) { + (*env)->ThrowNew( + env, + (*env)->FindClass(env, "java/lang/RuntimeException"), + message); +} + + +// --- Stdio redirection ------------------------------------------------------ + +// Most apps won't need this, because the Python-level sys.stdout and sys.stderr +// are redirected to the Android logcat by Python itself. However, in the +// testbed it's useful to redirect the native streams as well, to debug problems +// in the Python startup or redirection process. +// +// Based on +// https://github.com/beeware/briefcase-android-gradle-template/blob/v0.3.11/%7B%7B%20cookiecutter.safe_formal_name%20%7D%7D/app/src/main/cpp/native-lib.cpp + +typedef struct { + FILE *file; + int fd; + android_LogPriority priority; + char *tag; + int pipe[2]; +} StreamInfo; + +// The FILE member can't be initialized here because stdout and stderr are not +// compile-time constants. Instead, it's initialized immediately before the +// redirection. +static StreamInfo STREAMS[] = { + {NULL, STDOUT_FILENO, ANDROID_LOG_INFO, "native.stdout", {-1, -1}}, + {NULL, STDERR_FILENO, ANDROID_LOG_WARN, "native.stderr", {-1, -1}}, + {NULL, -1, ANDROID_LOG_UNKNOWN, NULL, {-1, -1}}, +}; + +// The maximum length of a log message in bytes, including the level marker and +// tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD in +// platform/system/logging/liblog/include/log/log.h. As of API level 30, messages +// longer than this will be be truncated by logcat. This limit has already been +// reduced at least once in the history of Android (from 4076 to 4068 between API +// level 23 and 26), so leave some headroom. +static const int MAX_BYTES_PER_WRITE = 4000; + +static void *redirection_thread(void *arg) { + StreamInfo *si = (StreamInfo*)arg; + ssize_t read_size; + char buf[MAX_BYTES_PER_WRITE]; + while ((read_size = read(si->pipe[0], buf, sizeof buf - 1)) > 0) { + buf[read_size] = '\0'; /* add null-terminator */ + __android_log_write(si->priority, si->tag, buf); + } + return 0; +} + +static char *redirect_stream(StreamInfo *si) { + /* make the FILE unbuffered, to ensure messages are never lost */ + if (setvbuf(si->file, 0, _IONBF, 0)) { + return "setvbuf"; + } + + /* create the pipe and redirect the file descriptor */ + if (pipe(si->pipe)) { + return "pipe"; + } + if (dup2(si->pipe[1], si->fd) == -1) { + return "dup2"; + } + + /* start the logging thread */ + pthread_t thr; + if ((errno = pthread_create(&thr, 0, redirection_thread, si))) { + return "pthread_create"; + } + if ((errno = pthread_detach(thr))) { + return "pthread_detach"; + } + return 0; +} + +JNIEXPORT void JNICALL Java_org_python_testbed_PythonTestRunner_redirectStdioToLogcat( + JNIEnv *env, jobject obj +) { + STREAMS[0].file = stdout; + STREAMS[1].file = stderr; + for (StreamInfo *si = STREAMS; si->file; si++) { + char *error_prefix; + if ((error_prefix = redirect_stream(si))) { + char error_message[1024]; + snprintf(error_message, sizeof(error_message), + "%s: %s", error_prefix, strerror(errno)); + throw_runtime_exception(env, error_message); + return; + } + } +} + + +// --- Python initialization --------------------------------------------------- + +static PyStatus set_config_string( + JNIEnv *env, PyConfig *config, wchar_t **config_str, jstring value +) { + const char *value_utf8 = (*env)->GetStringUTFChars(env, value, NULL); + PyStatus status = PyConfig_SetBytesString(config, config_str, value_utf8); + (*env)->ReleaseStringUTFChars(env, value, value_utf8); + return status; +} + +static void throw_status(JNIEnv *env, PyStatus status) { + throw_runtime_exception(env, status.err_msg ? status.err_msg : ""); +} + +JNIEXPORT int JNICALL Java_org_python_testbed_PythonTestRunner_runPython( + JNIEnv *env, jobject obj, jstring home, jstring runModule +) { + PyConfig config; + PyStatus status; + PyConfig_InitIsolatedConfig(&config); + + status = set_config_string(env, &config, &config.home, home); + if (PyStatus_Exception(status)) { + throw_status(env, status); + return 1; + } + + status = set_config_string(env, &config, &config.run_module, runModule); + if (PyStatus_Exception(status)) { + throw_status(env, status); + return 1; + } + + // Some tests generate SIGPIPE and SIGXFSZ, which should be ignored. + config.install_signal_handlers = 1; + + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + throw_status(env, status); + return 1; + } + + return Py_RunMain(); +} diff --git a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt new file mode 100644 index 00000000000000..c4bf6cbe83d8cd --- /dev/null +++ b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt @@ -0,0 +1,79 @@ +package org.python.testbed + +import android.content.Context +import android.os.* +import android.system.Os +import android.widget.TextView +import androidx.appcompat.app.* +import java.io.* + + +// Launching the tests from an activity is OK for a quick check, but for +// anything more complicated it'll be more convenient to use `android.py test` +// to launch the tests via PythonSuite. +class MainActivity : AppCompatActivity() { + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setContentView(R.layout.activity_main) + val status = PythonTestRunner(this).run("-W -uall") + findViewById(R.id.tvHello).text = "Exit status $status" + } +} + + +class PythonTestRunner(val context: Context) { + /** @param args Extra arguments for `python -m test`. + * @return The Python exit status: zero if the tests passed, nonzero if + * they failed. */ + fun run(args: String = "") : Int { + Os.setenv("PYTHON_ARGS", args, true) + + // Python needs this variable to help it find the temporary directory, + // but Android only sets it on API level 33 and later. + Os.setenv("TMPDIR", context.cacheDir.toString(), false) + + val pythonHome = extractAssets() + System.loadLibrary("main_activity") + redirectStdioToLogcat() + + // The main module is in src/main/python/main.py. + return runPython(pythonHome.toString(), "main") + } + + private fun extractAssets() : File { + val pythonHome = File(context.filesDir, "python") + if (pythonHome.exists() && !pythonHome.deleteRecursively()) { + throw RuntimeException("Failed to delete $pythonHome") + } + extractAssetDir("python", context.filesDir) + return pythonHome + } + + private fun extractAssetDir(path: String, targetDir: File) { + val names = context.assets.list(path) + ?: throw RuntimeException("Failed to list $path") + val targetSubdir = File(targetDir, path) + if (!targetSubdir.mkdirs()) { + throw RuntimeException("Failed to create $targetSubdir") + } + + for (name in names) { + val subPath = "$path/$name" + val input: InputStream + try { + input = context.assets.open(subPath) + } catch (e: FileNotFoundException) { + extractAssetDir(subPath, targetDir) + continue + } + input.use { + File(targetSubdir, name).outputStream().use { output -> + input.copyTo(output) + } + } + } + } + + private external fun redirectStdioToLogcat() + private external fun runPython(home: String, runModule: String) : Int +} diff --git a/Android/testbed/app/src/main/python/main.py b/Android/testbed/app/src/main/python/main.py new file mode 100644 index 00000000000000..d6941b14412fcc --- /dev/null +++ b/Android/testbed/app/src/main/python/main.py @@ -0,0 +1,32 @@ +import os +import runpy +import shlex +import signal +import sys + +# Some tests use SIGUSR1, but that's blocked by default in an Android app in +# order to make it available to `sigwait` in the Signal Catcher thread. +# (https://cs.android.com/android/platform/superproject/+/android14-qpr3-release:art/runtime/signal_catcher.cc). +# That thread's functionality is only useful for debugging the JVM, so disabling +# it should not weaken the tests. +# +# There's no safe way of stopping the thread completely (#123982), but simply +# unblocking SIGUSR1 is enough to fix most tests. +# +# However, in tests that generate multiple different signals in quick +# succession, it's possible for SIGUSR1 to arrive while the main thread is busy +# running the C-level handler for a different signal. In that case, the SIGUSR1 +# may be sent to the Signal Catcher thread instead, which will generate a log +# message containing the text "reacting to signal". +# +# Such tests may need to be changed in one of the following ways: +# * Use a signal other than SIGUSR1 (e.g. test_stress_delivery_simultaneous in +# test_signal.py). +# * Send the signal to a specific thread rather than the whole process (e.g. +# test_signals in test_threadsignals.py. +signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGUSR1]) + +sys.argv[1:] = shlex.split(os.environ["PYTHON_ARGS"]) + +# The test module will call sys.exit to indicate whether the tests passed. +runpy.run_module("test") diff --git a/Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png b/Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png new file mode 100644 index 00000000000000..741d6580d60e05 Binary files /dev/null and b/Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png differ diff --git a/Android/testbed/app/src/main/res/layout/activity_main.xml b/Android/testbed/app/src/main/res/layout/activity_main.xml new file mode 100644 index 00000000000000..21398609ec9c78 --- /dev/null +++ b/Android/testbed/app/src/main/res/layout/activity_main.xml @@ -0,0 +1,19 @@ + + + + + + \ No newline at end of file diff --git a/Android/testbed/app/src/main/res/values/strings.xml b/Android/testbed/app/src/main/res/values/strings.xml new file mode 100644 index 00000000000000..352d2f9e885a2a --- /dev/null +++ b/Android/testbed/app/src/main/res/values/strings.xml @@ -0,0 +1,3 @@ + + Python testbed + \ No newline at end of file diff --git a/Android/testbed/build.gradle.kts b/Android/testbed/build.gradle.kts new file mode 100644 index 00000000000000..4d1d6f87594da3 --- /dev/null +++ b/Android/testbed/build.gradle.kts @@ -0,0 +1,5 @@ +// Top-level build file where you can add configuration options common to all sub-projects/modules. +plugins { + id("com.android.application") version "8.6.1" apply false + id("org.jetbrains.kotlin.android") version "1.9.22" apply false +} diff --git a/Android/testbed/gradle.properties b/Android/testbed/gradle.properties new file mode 100644 index 00000000000000..e9f345c8c26250 --- /dev/null +++ b/Android/testbed/gradle.properties @@ -0,0 +1,28 @@ +# Project-wide Gradle settings. +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +# org.gradle.parallel=true +# AndroidX package structure to make it clearer which packages are bundled with the +# Android operating system, and which are packaged with your app's APK +# https://developer.android.com/topic/libraries/support-library/androidx-rn +android.useAndroidX=true +# Kotlin code style for this project: "official" or "obsolete": +kotlin.code.style=official +# Enables namespacing of each library's R class so that its R class includes only the +# resources declared in the library itself and none from the library's dependencies, +# thereby reducing the size of the R class for that library +android.nonTransitiveRClass=true + +# By default, the app will be uninstalled after the tests finish (apparently +# after 10 seconds in case of an unclean shutdown). We disable this, because +# when using android.py it can conflict with the installation of the next run. +android.injected.androidTest.leaveApksInstalledAfterRun=true diff --git a/Android/testbed/gradle/wrapper/gradle-wrapper.properties b/Android/testbed/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000000000..36529c896426b0 --- /dev/null +++ b/Android/testbed/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Mon Feb 19 20:29:06 GMT 2024 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/Android/testbed/settings.gradle.kts b/Android/testbed/settings.gradle.kts new file mode 100644 index 00000000000000..5e08773e02450f --- /dev/null +++ b/Android/testbed/settings.gradle.kts @@ -0,0 +1,18 @@ +pluginManagement { + repositories { + google() + mavenCentral() + gradlePluginPortal() + } +} +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + google() + mavenCentral() + } +} + +rootProject.name = "Python testbed" +include(":app") + \ No newline at end of file diff --git a/Doc/.ruff.toml b/Doc/.ruff.toml new file mode 100644 index 00000000000000..111ce03b91df38 --- /dev/null +++ b/Doc/.ruff.toml @@ -0,0 +1,42 @@ +target-version = "py312" # Align with the version in oldest_supported_sphinx +fix = true +output-format = "full" +line-length = 79 +extend-exclude = [ + "includes/*", + # Temporary exclusions: + "tools/extensions/pyspecific.py", +] + +[lint] +preview = true +select = [ + "C4", # flake8-comprehensions + "B", # flake8-bugbear + "E", # pycodestyle + "F", # pyflakes + "FA", # flake8-future-annotations + "FLY", # flynt + "FURB", # refurb + "G", # flake8-logging-format + "I", # isort + "LOG", # flake8-logging + "N", # pep8-naming + "PERF", # perflint + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "TCH", # flake8-type-checking + "UP", # pyupgrade + "W", # pycodestyle +] +ignore = [ + "E501", # Ignore line length errors (we use auto-formatting) +] + +[format] +preview = true +quote-style = "preserve" +docstring-code-format = true +exclude = [ + "tools/extensions/lexers/*", +] diff --git a/Doc/Makefile b/Doc/Makefile index 38fd60f2ae01d1..b8896da4a91869 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -6,6 +6,7 @@ # You can set these variables from the command line. PYTHON = python3 VENVDIR = ./venv +UV = uv SPHINXBUILD = PATH=$(VENVDIR)/bin:$$PATH sphinx-build BLURB = PATH=$(VENVDIR)/bin:$$PATH blurb JOBS = auto @@ -13,15 +14,15 @@ PAPER = SOURCES = DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py) REQUIREMENTS = requirements.txt -SPHINXERRORHANDLING = -W +SPHINXERRORHANDLING = --fail-on-warning # Internal variables. -PAPEROPT_a4 = -D latex_elements.papersize=a4paper -PAPEROPT_letter = -D latex_elements.papersize=letterpaper +PAPEROPT_a4 = --define latex_elements.papersize=a4paper +PAPEROPT_letter = --define latex_elements.papersize=letterpaper -ALLSPHINXOPTS = -b $(BUILDER) \ - -d build/doctrees \ - -j $(JOBS) \ +ALLSPHINXOPTS = --builder $(BUILDER) \ + --doctree-dir build/doctrees \ + --jobs $(JOBS) \ $(PAPEROPT_$(PAPER)) \ $(SPHINXOPTS) $(SPHINXERRORHANDLING) \ . build/$(BUILDER) $(SOURCES) @@ -32,6 +33,7 @@ help: @echo " clean to remove build files" @echo " venv to create a venv with necessary tools" @echo " html to make standalone HTML files" + @echo " gettext to generate POT files" @echo " htmlview to open the index page built by the html target in your browser" @echo " htmllive to rebuild and reload HTML files in your browser" @echo " htmlhelp to make HTML files and a HTML help project" @@ -140,14 +142,19 @@ pydoc-topics: build @echo "Building finished; now run this:" \ "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py" +.PHONY: gettext +gettext: BUILDER = gettext +gettext: override SPHINXOPTS := --doctree-dir build/doctrees-gettext $(SPHINXOPTS) +gettext: build + .PHONY: htmlview htmlview: html $(PYTHON) -c "import os, webbrowser; webbrowser.open('file://' + os.path.realpath('build/html/index.html'))" .PHONY: htmllive -htmllive: SPHINXBUILD = $(VENVDIR)/bin/sphinx-autobuild +htmllive: SPHINXBUILD = PATH=$(VENVDIR)/bin:$$PATH sphinx-autobuild htmllive: SPHINXOPTS = --re-ignore="/venv/" --open-browser --delay 0 -htmllive: html +htmllive: _ensure-sphinx-autobuild html .PHONY: clean clean: clean-venv @@ -163,92 +170,150 @@ venv: echo "venv already exists."; \ echo "To recreate it, remove it first with \`make clean-venv'."; \ else \ - $(PYTHON) -m venv $(VENVDIR); \ - $(VENVDIR)/bin/python3 -m pip install --upgrade pip; \ - $(VENVDIR)/bin/python3 -m pip install -r $(REQUIREMENTS); \ + echo "Creating venv in $(VENVDIR)"; \ + if $(UV) --version >/dev/null 2>&1; then \ + $(UV) venv --python=$(PYTHON) $(VENVDIR); \ + VIRTUAL_ENV=$(VENVDIR) $(UV) pip install -r $(REQUIREMENTS); \ + else \ + $(PYTHON) -m venv $(VENVDIR); \ + $(VENVDIR)/bin/python3 -m pip install --upgrade pip; \ + $(VENVDIR)/bin/python3 -m pip install -r $(REQUIREMENTS); \ + fi; \ echo "The venv has been created in the $(VENVDIR) directory"; \ fi +.PHONY: dist-no-html +dist-no-html: dist-text dist-pdf dist-epub dist-texinfo + .PHONY: dist dist: rm -rf dist mkdir -p dist - + $(MAKE) dist-html + $(MAKE) dist-text + $(MAKE) dist-pdf + $(MAKE) dist-epub + $(MAKE) dist-texinfo + +.PHONY: dist-html +dist-html: # archive the HTML - make html + @echo "Building HTML..." + mkdir -p dist + rm -rf build/html + find dist -name 'python-$(DISTVERSION)-docs-html*' -exec rm -rf {} \; + $(MAKE) html cp -pPR build/html dist/python-$(DISTVERSION)-docs-html tar -C dist -cf dist/python-$(DISTVERSION)-docs-html.tar python-$(DISTVERSION)-docs-html bzip2 -9 -k dist/python-$(DISTVERSION)-docs-html.tar (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-html.zip python-$(DISTVERSION)-docs-html) rm -r dist/python-$(DISTVERSION)-docs-html rm dist/python-$(DISTVERSION)-docs-html.tar + @echo "Build finished and archived!" +.PHONY: dist-text +dist-text: # archive the text build - make text + @echo "Building text..." + mkdir -p dist + rm -rf build/text + find dist -name 'python-$(DISTVERSION)-docs-text*' -exec rm -rf {} \; + $(MAKE) text cp -pPR build/text dist/python-$(DISTVERSION)-docs-text tar -C dist -cf dist/python-$(DISTVERSION)-docs-text.tar python-$(DISTVERSION)-docs-text bzip2 -9 -k dist/python-$(DISTVERSION)-docs-text.tar (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-text.zip python-$(DISTVERSION)-docs-text) rm -r dist/python-$(DISTVERSION)-docs-text rm dist/python-$(DISTVERSION)-docs-text.tar + @echo "Build finished and archived!" +.PHONY: dist-pdf +dist-pdf: # archive the A4 latex + @echo "Building LaTeX (A4 paper)..." + mkdir -p dist rm -rf build/latex - make latex PAPER=a4 - -sed -i 's/makeindex/makeindex -q/' build/latex/Makefile - (cd build/latex; make clean && make all-pdf && make FMT=pdf zip bz2) + find dist -name 'python-$(DISTVERSION)-docs-pdf*' -exec rm -rf {} \; + $(MAKE) latex PAPER=a4 + # remove zip & bz2 dependency on all-pdf, + # as otherwise the full latexmk process is run twice. + # ($$ is needed to escape the $; https://www.gnu.org/software/make/manual/make.html#Basics-of-Variable-References) + -sed -i 's/: all-$$(FMT)/:/' build/latex/Makefile + (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`nproc`+1)) --output-sync LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-a4.zip cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-a4.tar.bz2 + @echo "Build finished and archived!" - # archive the letter latex - rm -rf build/latex - make latex PAPER=letter - -sed -i 's/makeindex/makeindex -q/' build/latex/Makefile - (cd build/latex; make clean && make all-pdf && make FMT=pdf zip bz2) - cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-letter.zip - cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-letter.tar.bz2 - +.PHONY: dist-epub +dist-epub: # copy the epub build + @echo "Building EPUB..." + mkdir -p dist rm -rf build/epub - make epub + rm -f dist/python-$(DISTVERSION)-docs.epub + $(MAKE) epub cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub + @echo "Build finished and archived!" +.PHONY: dist-texinfo +dist-texinfo: # archive the texinfo build + @echo "Building Texinfo..." + mkdir -p dist rm -rf build/texinfo - make texinfo - make info --directory=build/texinfo + find dist -name 'python-$(DISTVERSION)-docs-texinfo*' -exec rm -rf {} \; + $(MAKE) texinfo + $(MAKE) info --directory=build/texinfo cp -pPR build/texinfo dist/python-$(DISTVERSION)-docs-texinfo tar -C dist -cf dist/python-$(DISTVERSION)-docs-texinfo.tar python-$(DISTVERSION)-docs-texinfo bzip2 -9 -k dist/python-$(DISTVERSION)-docs-texinfo.tar (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-texinfo.zip python-$(DISTVERSION)-docs-texinfo) rm -r dist/python-$(DISTVERSION)-docs-texinfo rm dist/python-$(DISTVERSION)-docs-texinfo.tar + @echo "Build finished and archived!" + +.PHONY: _ensure-package +_ensure-package: venv + if $(UV) --version >/dev/null 2>&1; then \ + VIRTUAL_ENV=$(VENVDIR) $(UV) pip install $(PACKAGE); \ + else \ + $(VENVDIR)/bin/python3 -m pip install $(PACKAGE); \ + fi + +.PHONY: _ensure-pre-commit +_ensure-pre-commit: + $(MAKE) _ensure-package PACKAGE=pre-commit + +.PHONY: _ensure-sphinx-autobuild +_ensure-sphinx-autobuild: + $(MAKE) _ensure-package PACKAGE=sphinx-autobuild .PHONY: check -check: venv - $(VENVDIR)/bin/python3 -m pre_commit --version > /dev/null || $(VENVDIR)/bin/python3 -m pip install pre-commit +check: _ensure-pre-commit $(VENVDIR)/bin/python3 -m pre_commit run --all-files .PHONY: serve serve: - @echo "The serve target was removed, use htmlview instead (see bpo-36329)" + @echo "The serve target was removed, use htmllive instead (see gh-80510)" # Targets for daily automated doc build # By default, Sphinx only rebuilds pages where the page content has changed. # This means it doesn't always pick up changes to preferred link targets, etc # To ensure such changes are picked up, we build the published docs with -# `-E` (to ignore the cached environment) and `-a` (to ignore already existing -# output files) +# ``--fresh-env`` (to ignore the cached environment) and ``--write-all`` +# (to ignore already existing output files) # for development releases: always build .PHONY: autobuild-dev +autobuild-dev: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short) autobuild-dev: - make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' + $(MAKE) dist-no-html SPHINXOPTS='$(SPHINXOPTS) --fresh-env --write-all --html-define daily=1' DISTVERSION=$(DISTVERSION) -# for quick rebuilds (HTML only) +# for HTML-only rebuilds .PHONY: autobuild-dev-html +autobuild-dev-html: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short) autobuild-dev-html: - make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' + $(MAKE) dist-html SPHINXOPTS='$(SPHINXOPTS) --fresh-env --write-all --html-define daily=1' DISTVERSION=$(DISTVERSION) # for stable releases: only build if not in pre-release stage (alpha, beta) # release candidate downloads are okay, since the stable tree can be in that stage @@ -258,7 +323,7 @@ autobuild-stable: echo "Not building; $(DISTVERSION) is not a release version."; \ exit 1;; \ esac - @make autobuild-dev + @$(MAKE) autobuild-dev .PHONY: autobuild-stable-html autobuild-stable-html: @@ -266,4 +331,4 @@ autobuild-stable-html: echo "Not building; $(DISTVERSION) is not a release version."; \ exit 1;; \ esac - @make autobuild-dev-html + @$(MAKE) autobuild-dev-html diff --git a/Doc/README.rst b/Doc/README.rst index a3bb5fa5445c23..2d1148753e0c6b 100644 --- a/Doc/README.rst +++ b/Doc/README.rst @@ -28,7 +28,7 @@ install the tools into there. Using make ---------- -To get started on UNIX, you can create a virtual environment and build +To get started on Unix, you can create a virtual environment and build documentation with the commands:: make venv @@ -40,13 +40,13 @@ If you'd like to create the virtual environment in a different location, you can specify it using the ``VENVDIR`` variable. You can also skip creating the virtual environment altogether, in which case -the Makefile will look for instances of ``sphinx-build`` and ``blurb`` +the ``Makefile`` will look for instances of ``sphinx-build`` and ``blurb`` installed on your process ``PATH`` (configurable with the ``SPHINXBUILD`` and ``BLURB`` variables). -On Windows, we try to emulate the Makefile as closely as possible with a +On Windows, we try to emulate the ``Makefile`` as closely as possible with a ``make.bat`` file. If you need to specify the Python interpreter to use, -set the PYTHON environment variable. +set the ``PYTHON`` environment variable. Available make targets are: @@ -62,15 +62,19 @@ Available make targets are: * "htmlview", which re-uses the "html" builder, but then opens the main page in your default web browser. +* "htmllive", which re-uses the "html" builder, rebuilds the docs, + starts a local server, and automatically reloads the page in your browser + when you make changes to reST files (Unix only). + * "htmlhelp", which builds HTML files and a HTML Help project file usable to convert them into a single Compiled HTML (.chm) file -- these are popular under Microsoft Windows, but very handy on every platform. To create the CHM file, you need to run the Microsoft HTML Help Workshop - over the generated project (.hhp) file. The make.bat script does this for + over the generated project (.hhp) file. The ``make.bat`` script does this for you on Windows. -* "latex", which builds LaTeX source files as input to "pdflatex" to produce +* "latex", which builds LaTeX source files as input to ``pdflatex`` to produce PDF documents. * "text", which builds a plain text file for each source file. @@ -95,8 +99,6 @@ Available make targets are: * "check", which checks for frequent markup errors. -* "serve", which serves the build/html directory on port 8000. - * "dist", (Unix only) which creates distributable archives of HTML, text, PDF, and EPUB builds. @@ -131,8 +133,5 @@ Bugs in the content should be reported to the Bugs in the toolset should be reported to the tools themselves. -You can also send a mail to the Python Documentation Team at docs@python.org, -and we will process your request as soon as possible. - -If you want to help the Documentation Team, you are always welcome. Just send -a mail to docs@python.org. +To help with the documentation, or report any problems, please leave a message +on `discuss.python.org `_. diff --git a/Doc/about.rst b/Doc/about.rst index 5e6160ff2700ed..8f635d7f743a98 100644 --- a/Doc/about.rst +++ b/Doc/about.rst @@ -1,10 +1,11 @@ -===================== -About these documents -===================== +======================== +About this documentation +======================== -These documents are generated from `reStructuredText`_ sources by `Sphinx`_, a -document processor specifically written for the Python documentation. +Python's documentation is generated from `reStructuredText`_ sources +using `Sphinx`_, a documentation generator originally created for Python +and now maintained as an independent project. .. _reStructuredText: https://docutils.sourceforge.io/rst.html .. _Sphinx: https://www.sphinx-doc.org/ @@ -20,14 +21,14 @@ volunteers are always welcome! Many thanks go to: * Fred L. Drake, Jr., the creator of the original Python documentation toolset - and writer of much of the content; + and author of much of the content; * the `Docutils `_ project for creating reStructuredText and the Docutils suite; * Fredrik Lundh for his Alternative Python Reference project from which Sphinx got many good ideas. -Contributors to the Python Documentation +Contributors to the Python documentation ---------------------------------------- Many people have contributed to the Python language, the Python standard diff --git a/Doc/bugs.rst b/Doc/bugs.rst index 908987cf41ff6e..5d0f68ca69675e 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -16,16 +16,15 @@ Documentation bugs ================== If you find a bug in this documentation or would like to propose an improvement, -please submit a bug report on the :ref:`tracker `. If you +please submit a bug report on the :ref:`issue tracker `. If you have a suggestion on how to fix it, include that as well. You can also open a discussion item on our `Documentation Discourse forum `_. -If you're short on time, you can also email documentation bug reports to -docs@python.org (behavioral bugs can be sent to python-list@python.org). -'docs@' is a mailing list run by volunteers; your request will be noticed, -though it may take a while to be processed. +If you find a bug in the theme (HTML / CSS / JavaScript) of the +documentation, please submit a bug report on the `python-doc-theme issue +tracker `_. .. seealso:: diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index b3609c233156b6..b7e0f22b52c57e 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -15,10 +15,8 @@ Allocating Objects on the Heap .. c:function:: PyObject* PyObject_Init(PyObject *op, PyTypeObject *type) Initialize a newly allocated object *op* with its type and initial - reference. Returns the initialized object. If *type* indicates that the - object participates in the cyclic garbage detector, it is added to the - detector's set of observed objects. Other fields of the object are not - affected. + reference. Returns the initialized object. Other fields of the object are + not affected. .. c:function:: PyVarObject* PyObject_InitVar(PyVarObject *op, PyTypeObject *type, Py_ssize_t size) @@ -54,12 +52,7 @@ Allocating Objects on the Heap .. c:function:: void PyObject_Del(void *op) - Releases memory allocated to an object using :c:macro:`PyObject_New` or - :c:macro:`PyObject_NewVar`. This is normally called from the - :c:member:`~PyTypeObject.tp_dealloc` handler specified in the object's type. The fields of - the object should not be accessed after this call as the memory is no - longer a valid Python object. - + Same as :c:func:`PyObject_Free`. .. c:var:: PyObject _Py_NoneStruct diff --git a/Doc/c-api/apiabiversion.rst b/Doc/c-api/apiabiversion.rst index f6c8284daeacb0..96050f59bd5250 100644 --- a/Doc/c-api/apiabiversion.rst +++ b/Doc/c-api/apiabiversion.rst @@ -6,9 +6,13 @@ API and ABI Versioning *********************** + +Build-time version constants +---------------------------- + CPython exposes its version number in the following macros. -Note that these correspond to the version code is **built** with, -not necessarily the version used at **run time**. +Note that these correspond to the version code is **built** with. +See :c:var:`Py_Version` for the version used at **run time**. See :ref:`stable` for a discussion of API and ABI stability across versions. @@ -37,37 +41,83 @@ See :ref:`stable` for a discussion of API and ABI stability across versions. .. c:macro:: PY_VERSION_HEX The Python version number encoded in a single integer. + See :c:func:`Py_PACK_FULL_VERSION` for the encoding details. - The underlying version information can be found by treating it as a 32 bit - number in the following manner: - - +-------+-------------------------+-------------------------+--------------------------+ - | Bytes | Bits (big endian order) | Meaning | Value for ``3.4.1a2`` | - +=======+=========================+=========================+==========================+ - | 1 | 1-8 | ``PY_MAJOR_VERSION`` | ``0x03`` | - +-------+-------------------------+-------------------------+--------------------------+ - | 2 | 9-16 | ``PY_MINOR_VERSION`` | ``0x04`` | - +-------+-------------------------+-------------------------+--------------------------+ - | 3 | 17-24 | ``PY_MICRO_VERSION`` | ``0x01`` | - +-------+-------------------------+-------------------------+--------------------------+ - | 4 | 25-28 | ``PY_RELEASE_LEVEL`` | ``0xA`` | - + +-------------------------+-------------------------+--------------------------+ - | | 29-32 | ``PY_RELEASE_SERIAL`` | ``0x2`` | - +-------+-------------------------+-------------------------+--------------------------+ + Use this for numeric comparisons, for example, + ``#if PY_VERSION_HEX >= ...``. - Thus ``3.4.1a2`` is hexversion ``0x030401a2`` and ``3.10.0`` is - hexversion ``0x030a00f0``. - Use this for numeric comparisons, e.g. ``#if PY_VERSION_HEX >= ...``. - - This version is also available via the symbol :c:var:`Py_Version`. +Run-time version +---------------- .. c:var:: const unsigned long Py_Version - The Python runtime version number encoded in a single constant integer, with - the same format as the :c:macro:`PY_VERSION_HEX` macro. + The Python runtime version number encoded in a single constant integer. + See :c:func:`Py_PACK_FULL_VERSION` for the encoding details. This contains the Python version used at run time. + Use this for numeric comparisons, for example, ``if (Py_Version >= ...)``. + .. versionadded:: 3.11 -All the given macros are defined in :source:`Include/patchlevel.h`. + +Bit-packing macros +------------------ + +.. c:function:: uint32_t Py_PACK_FULL_VERSION(int major, int minor, int micro, int release_level, int release_serial) + + Return the given version, encoded as a single 32-bit integer with + the following structure: + + +------------------+-------+----------------+-----------+--------------------------+ + | | No. | | | Example values | + | | of | | +-------------+------------+ + | Argument | bits | Bit mask | Bit shift | ``3.4.1a2`` | ``3.10.0`` | + +==================+=======+================+===========+=============+============+ + | *major* | 8 | ``0xFF000000`` | 24 | ``0x03`` | ``0x03`` | + +------------------+-------+----------------+-----------+-------------+------------+ + | *minor* | 8 | ``0x00FF0000`` | 16 | ``0x04`` | ``0x0A`` | + +------------------+-------+----------------+-----------+-------------+------------+ + | *micro* | 8 | ``0x0000FF00`` | 8 | ``0x01`` | ``0x00`` | + +------------------+-------+----------------+-----------+-------------+------------+ + | *release_level* | 4 | ``0x000000F0`` | 4 | ``0xA`` | ``0xF`` | + +------------------+-------+----------------+-----------+-------------+------------+ + | *release_serial* | 4 | ``0x0000000F`` | 0 | ``0x2`` | ``0x0`` | + +------------------+-------+----------------+-----------+-------------+------------+ + + For example: + + +-------------+------------------------------------+-----------------+ + | Version | ``Py_PACK_FULL_VERSION`` arguments | Encoded version | + +=============+====================================+=================+ + | ``3.4.1a2`` | ``(3, 4, 1, 0xA, 2)`` | ``0x030401a2`` | + +-------------+------------------------------------+-----------------+ + | ``3.10.0`` | ``(3, 10, 0, 0xF, 0)`` | ``0x030a00f0`` | + +-------------+------------------------------------+-----------------+ + + Out-of range bits in the arguments are ignored. + That is, the macro can be defined as: + + .. code-block:: c + + #ifndef Py_PACK_FULL_VERSION + #define Py_PACK_FULL_VERSION(X, Y, Z, LEVEL, SERIAL) ( \ + (((X) & 0xff) << 24) | \ + (((Y) & 0xff) << 16) | \ + (((Z) & 0xff) << 8) | \ + (((LEVEL) & 0xf) << 4) | \ + (((SERIAL) & 0xf) << 0)) + #endif + + ``Py_PACK_FULL_VERSION`` is primarily a macro, intended for use in + ``#if`` directives, but it is also available as an exported function. + + .. versionadded:: 3.14 + +.. c:function:: uint32_t Py_PACK_VERSION(int major, int minor) + + Equivalent to ``Py_PACK_FULL_VERSION(major, minor, 0, 0, 0)``. + The result does not correspond to any Python release, but is useful + in numeric comparisons. + + .. versionadded:: 3.14 diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 834aae9372fe3b..209056ef2f8bce 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -229,12 +229,24 @@ There are three ways strings and buffers can be converted to C: Numbers ------- +These formats allow representing Python numbers or single characters as C numbers. +Formats that require :class:`int`, :class:`float` or :class:`complex` can +also use the corresponding special methods :meth:`~object.__index__`, +:meth:`~object.__float__` or :meth:`~object.__complex__` to convert +the Python object to the required type. + +For signed integer formats, :exc:`OverflowError` is raised if the value +is out of range for the C type. +For unsigned integer formats, no range checking is done --- the +most significant bits are silently truncated when the receiving field is too +small to receive the value. + ``b`` (:class:`int`) [unsigned char] - Convert a nonnegative Python integer to an unsigned tiny int, stored in a C + Convert a nonnegative Python integer to an unsigned tiny integer, stored in a C :c:expr:`unsigned char`. ``B`` (:class:`int`) [unsigned char] - Convert a Python integer to a tiny int without overflow checking, stored in a C + Convert a Python integer to a tiny integer without overflow checking, stored in a C :c:expr:`unsigned char`. ``h`` (:class:`int`) [short int] @@ -280,10 +292,10 @@ Numbers length 1, to a C :c:expr:`int`. ``f`` (:class:`float`) [float] - Convert a Python floating point number to a C :c:expr:`float`. + Convert a Python floating-point number to a C :c:expr:`float`. ``d`` (:class:`float`) [double] - Convert a Python floating point number to a C :c:expr:`double`. + Convert a Python floating-point number to a C :c:expr:`double`. ``D`` (:class:`complex`) [Py_complex] Convert a Python complex number to a C :c:type:`Py_complex` structure. @@ -307,7 +319,7 @@ Other objects .. _o_ampersand: -``O&`` (object) [*converter*, *anything*] +``O&`` (object) [*converter*, *address*] Convert a Python object to a C variable through a *converter* function. This takes two arguments: the first is a function, the second is the address of a C variable (of arbitrary type), converted to :c:expr:`void *`. The *converter* @@ -321,14 +333,20 @@ Other objects the conversion has failed. When the conversion fails, the *converter* function should raise an exception and leave the content of *address* unmodified. - If the *converter* returns ``Py_CLEANUP_SUPPORTED``, it may get called a + .. c:macro:: Py_CLEANUP_SUPPORTED + :no-typesetting: + + If the *converter* returns :c:macro:`!Py_CLEANUP_SUPPORTED`, it may get called a second time if the argument parsing eventually fails, giving the converter a chance to release any memory that it had already allocated. In this second call, the *object* parameter will be ``NULL``; *address* will have the same value as in the original call. + Examples of converters: :c:func:`PyUnicode_FSConverter` and + :c:func:`PyUnicode_FSDecoder`. + .. versionchanged:: 3.1 - ``Py_CLEANUP_SUPPORTED`` was added. + :c:macro:`!Py_CLEANUP_SUPPORTED` was added. ``p`` (:class:`bool`) [int] Tests the value passed in for truth (a boolean **p**\ redicate) and converts @@ -344,12 +362,6 @@ Other objects in *items*. The C arguments must correspond to the individual format units in *items*. Format units for sequences may be nested. -It is possible to pass "long" integers (integers whose value exceeds the -platform's :c:macro:`LONG_MAX`) however no proper range checking is done --- the -most significant bits are silently truncated when the receiving field is too -small to receive the value (actually, the semantics are inherited from downcasts -in C --- your mileage may vary). - A few other characters have a meaning in a format string. These may not occur inside nested parentheses. They are: @@ -642,10 +654,10 @@ Building values object of length 1. ``d`` (:class:`float`) [double] - Convert a C :c:expr:`double` to a Python floating point number. + Convert a C :c:expr:`double` to a Python floating-point number. ``f`` (:class:`float`) [float] - Convert a C :c:expr:`float` to a Python floating point number. + Convert a C :c:expr:`float` to a Python floating-point number. ``D`` (:class:`complex`) [Py_complex \*] Convert a C :c:type:`Py_complex` structure to a Python complex number. diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 1e1cabdf242bd1..dc43a3d5fcb094 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -147,9 +147,9 @@ a buffer, see :c:func:`PyObject_GetBuffer`. or a :c:macro:`PyBUF_WRITABLE` request, the consumer must disregard :c:member:`~Py_buffer.itemsize` and assume ``itemsize == 1``. - .. c:member:: const char *format + .. c:member:: char *format - A *NUL* terminated string in :mod:`struct` module style syntax describing + A *NULL* terminated string in :mod:`struct` module style syntax describing the contents of a single item. If this is ``NULL``, ``"B"`` (unsigned bytes) is assumed. @@ -244,7 +244,6 @@ The following fields are not influenced by *flags* and must always be filled in with the correct values: :c:member:`~Py_buffer.obj`, :c:member:`~Py_buffer.buf`, :c:member:`~Py_buffer.len`, :c:member:`~Py_buffer.itemsize`, :c:member:`~Py_buffer.ndim`. - readonly, format ~~~~~~~~~~~~~~~~ @@ -253,7 +252,8 @@ readonly, format Controls the :c:member:`~Py_buffer.readonly` field. If set, the exporter MUST provide a writable buffer or else report failure. Otherwise, the exporter MAY provide either a read-only or writable buffer, but the choice - MUST be consistent for all consumers. + MUST be consistent for all consumers. For example, :c:expr:`PyBUF_SIMPLE | PyBUF_WRITABLE` + can be used to request a simple writable buffer. .. c:macro:: PyBUF_FORMAT @@ -265,8 +265,9 @@ readonly, format Since :c:macro:`PyBUF_SIMPLE` is defined as 0, :c:macro:`PyBUF_WRITABLE` can be used as a stand-alone flag to request a simple writable buffer. -:c:macro:`PyBUF_FORMAT` can be \|'d to any of the flags except :c:macro:`PyBUF_SIMPLE`. -The latter already implies format ``B`` (unsigned bytes). +:c:macro:`PyBUF_FORMAT` must be \|'d to any of the flags except :c:macro:`PyBUF_SIMPLE`, because +the latter already implies format ``B`` (unsigned bytes). :c:macro:`!PyBUF_FORMAT` cannot be +used on its own. shape, strides, suboffsets diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index 456f7d89bca03c..9045689a6be567 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -42,17 +42,22 @@ Direct API functions Return a new bytearray object from any object, *o*, that implements the :ref:`buffer protocol `. + On failure, return ``NULL`` with an exception set. + .. c:function:: PyObject* PyByteArray_FromStringAndSize(const char *string, Py_ssize_t len) - Create a new bytearray object from *string* and its length, *len*. On - failure, ``NULL`` is returned. + Create a new bytearray object from *string* and its length, *len*. + + On failure, return ``NULL`` with an exception set. .. c:function:: PyObject* PyByteArray_Concat(PyObject *a, PyObject *b) Concat bytearrays *a* and *b* and return a new bytearray with the result. + On failure, return ``NULL`` with an exception set. + .. c:function:: Py_ssize_t PyByteArray_Size(PyObject *bytearray) diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 4790d3b2da4375..d47beee68eaa33 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -189,12 +189,30 @@ called with a non-bytes parameter. to *newpart* (i.e. decrements its reference count). +.. c:function:: PyObject* PyBytes_Join(PyObject *sep, PyObject *iterable) + + Similar to ``sep.join(iterable)`` in Python. + + *sep* must be Python :class:`bytes` object. + (Note that :c:func:`PyUnicode_Join` accepts ``NULL`` separator and treats + it as a space, whereas :c:func:`PyBytes_Join` doesn't accept ``NULL`` + separator.) + + *iterable* must be an iterable object yielding objects that implement the + :ref:`buffer protocol `. + + On success, return a new :class:`bytes` object. + On error, set an exception and return ``NULL``. + + .. versionadded:: 3.14 + + .. c:function:: int _PyBytes_Resize(PyObject **bytes, Py_ssize_t newsize) - A way to resize a bytes object even though it is "immutable". Only use this - to build up a brand new bytes object; don't use this if the bytes may already - be known in other parts of the code. It is an error to call this function if - the refcount on the input bytes object is not one. Pass the address of an + Resize a bytes object. *newsize* will be the new length of the bytes object. + You can think of it as creating a new bytes object and destroying the old + one, only more efficiently. + Pass the address of an existing bytes object as an lvalue (it may be written into), and the new size desired. On success, *\*bytes* holds the resized bytes object and ``0`` is returned; the address in *\*bytes* may differ from its input value. If the diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst index f8cd0344fdd1c0..61eb994c370946 100644 --- a/Doc/c-api/cell.rst +++ b/Doc/c-api/cell.rst @@ -39,7 +39,8 @@ Cell objects are not likely to be useful elsewhere. .. c:function:: PyObject* PyCell_Get(PyObject *cell) - Return the contents of the cell *cell*. + Return the contents of the cell *cell*, which can be ``NULL``. + If *cell* is not a cell object, returns ``NULL`` with an exception set. .. c:function:: PyObject* PyCell_GET(PyObject *cell) @@ -52,8 +53,10 @@ Cell objects are not likely to be useful elsewhere. Set the contents of the cell object *cell* to *value*. This releases the reference to any current content of the cell. *value* may be ``NULL``. *cell* - must be non-``NULL``; if it is not a cell object, ``-1`` will be returned. On - success, ``0`` will be returned. + must be non-``NULL``. + + On success, return ``0``. + If *cell* is not a cell object, set an exception and return ``-1``. .. c:function:: void PyCell_SET(PyObject *cell, PyObject *value) diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index f6fdd7574323c7..6eae24b38fae48 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -32,11 +32,19 @@ bound into a function. .. c:function:: Py_ssize_t PyCode_GetNumFree(PyCodeObject *co) - Return the number of free variables in a code object. + Return the number of :term:`free (closure) variables ` + in a code object. -.. c:function:: int PyCode_GetFirstFree(PyCodeObject *co) +.. c:function:: int PyUnstable_Code_GetFirstFree(PyCodeObject *co) - Return the position of the first free variable in a code object. + Return the position of the first :term:`free (closure) variable ` + in a code object. + + .. versionchanged:: 3.13 + + Renamed from ``PyCode_GetFirstFree`` as part of :ref:`unstable-c-api`. + The old name is deprecated, but will remain available until the + signature changes again. .. c:function:: PyCodeObject* PyUnstable_Code_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable) @@ -90,8 +98,8 @@ bound into a function. Return the line number of the instruction that occurs on or before ``byte_offset`` and ends after it. If you just need the line number of a frame, use :c:func:`PyFrame_GetLineNumber` instead. - For efficiently iterating over the line numbers in a code object, use `the API described in PEP 626 - `_. + For efficiently iterating over the line numbers in a code object, use :pep:`the API described in PEP 626 + <0626#out-of-process-debuggers-and-profilers>`. .. c:function:: int PyCode_Addr2Location(PyObject *co, int byte_offset, int *start_line, int *start_column, int *end_line, int *end_column) @@ -138,7 +146,8 @@ bound into a function. Equivalent to the Python code ``getattr(co, 'co_freevars')``. Returns a new reference to a :c:type:`PyTupleObject` containing the names of - the free variables. On error, ``NULL`` is returned and an exception is raised. + the :term:`free (closure) variables `. On error, ``NULL`` is returned + and an exception is raised. .. versionadded:: 3.11 diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 5a0474869071d9..d1f5d8eda676ef 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -25,12 +25,16 @@ pointers. This is consistent throughout the API. The C structure which corresponds to the value portion of a Python complex number object. Most of the functions for dealing with complex number objects - use structures of this type as input or output values, as appropriate. It is - defined as:: + use structures of this type as input or output values, as appropriate. + + .. c:member:: double real + double imag + + The structure is defined as:: typedef struct { - double real; - double imag; + double real; + double imag; } Py_complex; @@ -40,12 +44,36 @@ pointers. This is consistent throughout the API. representation. +.. c:function:: Py_complex _Py_cr_sum(Py_complex left, double right) + + Return the sum of a complex number and a real number, using the C :c:type:`Py_complex` + representation. + + .. versionadded:: 3.14 + + .. c:function:: Py_complex _Py_c_diff(Py_complex left, Py_complex right) Return the difference between two complex numbers, using the C :c:type:`Py_complex` representation. +.. c:function:: Py_complex _Py_cr_diff(Py_complex left, double right) + + Return the difference between a complex number and a real number, using the C + :c:type:`Py_complex` representation. + + .. versionadded:: 3.14 + + +.. c:function:: Py_complex _Py_rc_diff(double left, Py_complex right) + + Return the difference between a real number and a complex number, using the C + :c:type:`Py_complex` representation. + + .. versionadded:: 3.14 + + .. c:function:: Py_complex _Py_c_neg(Py_complex num) Return the negation of the complex number *num*, using the C @@ -58,6 +86,14 @@ pointers. This is consistent throughout the API. representation. +.. c:function:: Py_complex _Py_cr_prod(Py_complex left, double right) + + Return the product of a complex number and a real number, using the C + :c:type:`Py_complex` representation. + + .. versionadded:: 3.14 + + .. c:function:: Py_complex _Py_c_quot(Py_complex dividend, Py_complex divisor) Return the quotient of two complex numbers, using the C :c:type:`Py_complex` @@ -67,6 +103,28 @@ pointers. This is consistent throughout the API. :c:data:`errno` to :c:macro:`!EDOM`. +.. c:function:: Py_complex _Py_cr_quot(Py_complex dividend, double divisor) + + Return the quotient of a complex number and a real number, using the C + :c:type:`Py_complex` representation. + + If *divisor* is zero, this method returns zero and sets + :c:data:`errno` to :c:macro:`!EDOM`. + + .. versionadded:: 3.14 + + +.. c:function:: Py_complex _Py_rc_quot(double dividend, Py_complex divisor) + + Return the quotient of a real number and a complex number, using the C + :c:type:`Py_complex` representation. + + If *divisor* is zero, this method returns zero and sets + :c:data:`errno` to :c:macro:`!EDOM`. + + .. versionadded:: 3.14 + + .. c:function:: Py_complex _Py_c_pow(Py_complex num, Py_complex exp) Return the exponentiation of *num* by *exp*, using the C :c:type:`Py_complex` @@ -75,6 +133,8 @@ pointers. This is consistent throughout the API. If *num* is null and *exp* is not a positive real number, this method returns zero and sets :c:data:`errno` to :c:macro:`!EDOM`. + Set :c:data:`errno` to :c:macro:`!ERANGE` on overflows. + Complex Numbers as Python Objects ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -106,11 +166,13 @@ Complex Numbers as Python Objects .. c:function:: PyObject* PyComplex_FromCComplex(Py_complex v) Create a new Python complex number object from a C :c:type:`Py_complex` value. + Return ``NULL`` with an exception set on error. .. c:function:: PyObject* PyComplex_FromDoubles(double real, double imag) Return a new :c:type:`PyComplexObject` object from *real* and *imag*. + Return ``NULL`` with an exception set on error. .. c:function:: double PyComplex_RealAsDouble(PyObject *op) @@ -121,7 +183,9 @@ Complex Numbers as Python Objects :meth:`~object.__complex__` method, this method will first be called to convert *op* to a Python complex number object. If :meth:`!__complex__` is not defined then it falls back to call :c:func:`PyFloat_AsDouble` and - returns its result. Upon failure, this method returns ``-1.0``, so one + returns its result. + + Upon failure, this method returns ``-1.0`` with an exception set, so one should call :c:func:`PyErr_Occurred` to check for errors. .. versionchanged:: 3.13 @@ -135,8 +199,10 @@ Complex Numbers as Python Objects :meth:`~object.__complex__` method, this method will first be called to convert *op* to a Python complex number object. If :meth:`!__complex__` is not defined then it falls back to call :c:func:`PyFloat_AsDouble` and - returns ``0.0`` on success. Upon failure, this method returns ``-1.0``, so - one should call :c:func:`PyErr_Occurred` to check for errors. + returns ``0.0`` on success. + + Upon failure, this method returns ``-1.0`` with an exception set, so one + should call :c:func:`PyErr_Occurred` to check for errors. .. versionchanged:: 3.13 Use :meth:`~object.__complex__` if available. @@ -149,8 +215,11 @@ Complex Numbers as Python Objects method, this method will first be called to convert *op* to a Python complex number object. If :meth:`!__complex__` is not defined then it falls back to :meth:`~object.__float__`. If :meth:`!__float__` is not defined then it falls back - to :meth:`~object.__index__`. Upon failure, this method returns ``-1.0`` as a real - value. + to :meth:`~object.__index__`. + + Upon failure, this method returns :c:type:`Py_complex` + with :c:member:`~Py_complex.real` set to ``-1.0`` and with an exception set, so one + should call :c:func:`PyErr_Occurred` to check for errors. .. versionchanged:: 3.8 Use :meth:`~object.__index__` if available. diff --git a/Doc/c-api/contextvars.rst b/Doc/c-api/contextvars.rst index d970f5443b1df5..b7c6550ff34aac 100644 --- a/Doc/c-api/contextvars.rst +++ b/Doc/c-api/contextvars.rst @@ -6,6 +6,8 @@ Context Variables Objects ------------------------- .. _contextvarsobjects_pointertype_change: +.. versionadded:: 3.7 + .. versionchanged:: 3.7.1 .. note:: @@ -24,8 +26,6 @@ Context Variables Objects See :issue:`34762` for more details. -.. versionadded:: 3.7 - This section details the public C API for the :mod:`contextvars` module. .. c:type:: PyContext @@ -101,6 +101,52 @@ Context object management functions: current context for the current thread. Returns ``0`` on success, and ``-1`` on error. +.. c:function:: int PyContext_AddWatcher(PyContext_WatchCallback callback) + + Register *callback* as a context object watcher for the current interpreter. + Return an ID which may be passed to :c:func:`PyContext_ClearWatcher`. + In case of error (e.g. no more watcher IDs available), + return ``-1`` and set an exception. + + .. versionadded:: 3.14 + +.. c:function:: int PyContext_ClearWatcher(int watcher_id) + + Clear watcher identified by *watcher_id* previously returned from + :c:func:`PyContext_AddWatcher` for the current interpreter. + Return ``0`` on success, or ``-1`` and set an exception on error + (e.g. if the given *watcher_id* was never registered.) + + .. versionadded:: 3.14 + +.. c:type:: PyContextEvent + + Enumeration of possible context object watcher events: + + - ``Py_CONTEXT_SWITCHED``: The :term:`current context` has switched to a + different context. The object passed to the watch callback is the + now-current :class:`contextvars.Context` object, or None if no context is + current. + + .. versionadded:: 3.14 + +.. c:type:: int (*PyContext_WatchCallback)(PyContextEvent event, PyObject *obj) + + Context object watcher callback function. The object passed to the callback + is event-specific; see :c:type:`PyContextEvent` for details. + + If the callback returns with an exception set, it must return ``-1``; this + exception will be printed as an unraisable exception using + :c:func:`PyErr_FormatUnraisable`. Otherwise it should return ``0``. + + There may already be a pending exception set on entry to the callback. In + this case, the callback should return ``0`` with the same exception still + set. This means the callback may not call any other API that can set an + exception unless it saves and clears the exception state first, and restores + it before returning. + + .. versionadded:: 3.14 + Context variable functions: diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index 4aaf3905e81c8a..c92ef4c653a675 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -105,7 +105,7 @@ The following functions provide locale-independent string to number conversions. If ``s`` represents a value that is too large to store in a float (for example, ``"1e500"`` is such a string on many platforms) then - if ``overflow_exception`` is ``NULL`` return ``Py_HUGE_VAL`` (with + if ``overflow_exception`` is ``NULL`` return ``Py_INFINITY`` (with an appropriate sign) and don't set any exception. Otherwise, ``overflow_exception`` must point to a Python exception object; raise that exception and return ``-1.0``. In both cases, set diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index 97522da773477e..d2d4d5309c7098 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -318,10 +318,10 @@ Macros for the convenience of modules implementing the DB API: .. c:function:: PyObject* PyDateTime_FromTimestamp(PyObject *args) Create and return a new :class:`datetime.datetime` object given an argument - tuple suitable for passing to :meth:`datetime.datetime.fromtimestamp()`. + tuple suitable for passing to :meth:`datetime.datetime.fromtimestamp`. .. c:function:: PyObject* PyDate_FromTimestamp(PyObject *args) Create and return a new :class:`datetime.date` object given an argument - tuple suitable for passing to :meth:`datetime.date.fromtimestamp()`. + tuple suitable for passing to :meth:`datetime.date.fromtimestamp`. diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 03f3d28187bfe9..ce73fa0cc60ebb 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -156,7 +156,7 @@ Dictionary Objects .. c:function:: int PyDict_GetItemStringRef(PyObject *p, const char *key, PyObject **result) - Similar than :c:func:`PyDict_GetItemRef`, but *key* is specified as a + Similar to :c:func:`PyDict_GetItemRef`, but *key* is specified as a :c:expr:`const char*` UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. @@ -191,6 +191,7 @@ Dictionary Objects to both *default_value* and *\*result* (if it's not ``NULL``). These may refer to the same object: in that case you hold two separate references to it. + .. versionadded:: 3.13 @@ -205,7 +206,7 @@ Dictionary Objects ``NULL``, and return ``0``. - On error, raise an exception and return ``-1``. - This is similar to :meth:`dict.pop`, but without the default value and + Similar to :meth:`dict.pop`, but without the default value and not raising :exc:`KeyError` if the key missing. .. versionadded:: 3.13 @@ -289,6 +290,17 @@ Dictionary Objects Py_DECREF(o); } + The function is not thread-safe in the :term:`free-threaded ` + build without external synchronization. You can use + :c:macro:`Py_BEGIN_CRITICAL_SECTION` to lock the dictionary while iterating + over it:: + + Py_BEGIN_CRITICAL_SECTION(self->dict); + while (PyDict_Next(self->dict, &pos, &key, &value)) { + ... + } + Py_END_CRITICAL_SECTION(); + .. c:function:: int PyDict_Merge(PyObject *a, PyObject *b, int override) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index ba13fd1b9973e0..c1f0bd750361d6 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -34,7 +34,7 @@ propagated, additional calls into the Python/C API may not behave as intended and may fail in mysterious ways. .. note:: - The error indicator is **not** the result of :func:`sys.exc_info()`. + The error indicator is **not** the result of :func:`sys.exc_info`. The former corresponds to an exception that is not yet caught (and is therefore still propagating), while the latter returns an exception after it is caught (and has therefore stopped propagating). @@ -104,7 +104,7 @@ Printing and clearing Similar to :c:func:`PyErr_WriteUnraisable`, but the *format* and subsequent parameters help format the warning message; they have the same meaning and values as in :c:func:`PyUnicode_FromFormat`. - ``PyErr_WriteUnraisable(obj)`` is roughtly equivalent to + ``PyErr_WriteUnraisable(obj)`` is roughly equivalent to ``PyErr_FormatUnraisable("Exception ignored in: %R", obj)``. If *format* is ``NULL``, only the traceback is printed. @@ -221,13 +221,14 @@ For convenience, some of these functions will always return a .. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr) - This is a convenience function to raise :exc:`WindowsError`. If called with + This is a convenience function to raise :exc:`OSError`. If called with *ierr* of ``0``, the error code returned by a call to :c:func:`!GetLastError` is used instead. It calls the Win32 function :c:func:`!FormatMessage` to retrieve the Windows description of error code given by *ierr* or :c:func:`!GetLastError`, - then it constructs a tuple object whose first item is the *ierr* value and whose - second item is the corresponding error message (gotten from - :c:func:`!FormatMessage`), and then calls ``PyErr_SetObject(PyExc_WindowsError, + then it constructs a :exc:`OSError` object with the :attr:`~OSError.winerror` + attribute set to the error code, the :attr:`~OSError.strerror` attribute + set to the corresponding error message (gotten from + :c:func:`!FormatMessage`), and then calls ``PyErr_SetObject(PyExc_OSError, object)``. This function always returns ``NULL``. .. availability:: Windows. @@ -732,7 +733,7 @@ Exception Classes This creates a class object derived from :exc:`Exception` (accessible in C as :c:data:`PyExc_Exception`). - The :attr:`!__module__` attribute of the new class is set to the first part (up + The :attr:`~type.__module__` attribute of the new class is set to the first part (up to the last dot) of the *name* argument, and the class name is set to the last part (after the last dot). The *base* argument can be used to specify alternate base classes; it can either be only one class or a tuple of classes. The *dict* @@ -852,12 +853,23 @@ The following functions are used to create and modify Unicode exceptions from C. *\*start*. *start* must not be ``NULL``. Return ``0`` on success, ``-1`` on failure. + If the :attr:`UnicodeError.object` is an empty sequence, the resulting + *start* is ``0``. Otherwise, it is clipped to ``[0, len(object) - 1]``. + + .. seealso:: :attr:`UnicodeError.start` + .. c:function:: int PyUnicodeDecodeError_SetStart(PyObject *exc, Py_ssize_t start) int PyUnicodeEncodeError_SetStart(PyObject *exc, Py_ssize_t start) int PyUnicodeTranslateError_SetStart(PyObject *exc, Py_ssize_t start) - Set the *start* attribute of the given exception object to *start*. Return - ``0`` on success, ``-1`` on failure. + Set the *start* attribute of the given exception object to *start*. + Return ``0`` on success, ``-1`` on failure. + + .. note:: + + While passing a negative *start* does not raise an exception, + the corresponding getters will not consider it as a relative + offset. .. c:function:: int PyUnicodeDecodeError_GetEnd(PyObject *exc, Py_ssize_t *end) int PyUnicodeEncodeError_GetEnd(PyObject *exc, Py_ssize_t *end) @@ -867,6 +879,9 @@ The following functions are used to create and modify Unicode exceptions from C. *\*end*. *end* must not be ``NULL``. Return ``0`` on success, ``-1`` on failure. + If the :attr:`UnicodeError.object` is an empty sequence, the resulting + *end* is ``0``. Otherwise, it is clipped to ``[1, len(object)]``. + .. c:function:: int PyUnicodeDecodeError_SetEnd(PyObject *exc, Py_ssize_t end) int PyUnicodeEncodeError_SetEnd(PyObject *exc, Py_ssize_t end) int PyUnicodeTranslateError_SetEnd(PyObject *exc, Py_ssize_t end) @@ -874,6 +889,8 @@ The following functions are used to create and modify Unicode exceptions from C. Set the *end* attribute of the given exception object to *end*. Return ``0`` on success, ``-1`` on failure. + .. seealso:: :attr:`UnicodeError.end` + .. c:function:: PyObject* PyUnicodeDecodeError_GetReason(PyObject *exc) PyObject* PyUnicodeEncodeError_GetReason(PyObject *exc) PyObject* PyUnicodeTranslateError_GetReason(PyObject *exc) @@ -1003,6 +1020,7 @@ the variables: single: PyExc_OverflowError (C var) single: PyExc_PermissionError (C var) single: PyExc_ProcessLookupError (C var) + single: PyExc_PythonFinalizationError (C var) single: PyExc_RecursionError (C var) single: PyExc_ReferenceError (C var) single: PyExc_RuntimeError (C var) @@ -1095,6 +1113,8 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_PythonFinalizationError` | :exc:`PythonFinalizationError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_RecursionError` | :exc:`RecursionError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | | diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 4f6ac0d8175c6b..1da37a5bcaeef9 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -2,20 +2,20 @@ .. _floatobjects: -Floating Point Objects +Floating-Point Objects ====================== -.. index:: pair: object; floating point +.. index:: pair: object; floating-point .. c:type:: PyFloatObject - This subtype of :c:type:`PyObject` represents a Python floating point object. + This subtype of :c:type:`PyObject` represents a Python floating-point object. .. c:var:: PyTypeObject PyFloat_Type - This instance of :c:type:`PyTypeObject` represents the Python floating point + This instance of :c:type:`PyTypeObject` represents the Python floating-point type. This is the same object as :class:`float` in the Python layer. @@ -45,7 +45,7 @@ Floating Point Objects .. c:function:: double PyFloat_AsDouble(PyObject *pyfloat) Return a C :c:expr:`double` representation of the contents of *pyfloat*. If - *pyfloat* is not a Python floating point object but has a :meth:`~object.__float__` + *pyfloat* is not a Python floating-point object but has a :meth:`~object.__float__` method, this method will first be called to convert *pyfloat* into a float. If :meth:`!__float__` is not defined then it falls back to :meth:`~object.__index__`. This method returns ``-1.0`` upon failure, so one should call diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 6bb1e9b5803b58..1a52e146a69751 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -120,18 +120,46 @@ See also :ref:`Reflection `. .. c:function:: PyObject* PyFrame_GetLocals(PyFrameObject *frame) - Get the *frame*'s :attr:`~frame.f_locals` attribute (:class:`dict`). + Get the *frame*'s :attr:`~frame.f_locals` attribute. + If the frame refers to an :term:`optimized scope`, this returns a + write-through proxy object that allows modifying the locals. + In all other cases (classes, modules, :func:`exec`, :func:`eval`) it returns + the mapping representing the frame locals directly (as described for + :func:`locals`). Return a :term:`strong reference`. .. versionadded:: 3.11 + .. versionchanged:: 3.13 + As part of :pep:`667`, return an instance of :c:var:`PyFrameLocalsProxy_Type`. + .. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame) Return the line number that *frame* is currently executing. +Frame Locals Proxies +^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 3.13 + +The :attr:`~frame.f_locals` attribute on a :ref:`frame object ` +is an instance of a "frame-locals proxy". The proxy object exposes a +write-through view of the underlying locals dictionary for the frame. This +ensures that the variables exposed by ``f_locals`` are always up to date with +the live local variables in the frame itself. + +See :pep:`667` for more information. + +.. c:var:: PyTypeObject PyFrameLocalsProxy_Type + + The type of frame :func:`locals` proxy objects. + +.. c:function:: int PyFrameLocalsProxy_Check(PyObject *obj) + + Return non-zero if *obj* is a frame :func:`locals` proxy. Internal Frames ^^^^^^^^^^^^^^^ diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst index 91d88ae27bc9f4..00f8cb887dc7eb 100644 --- a/Doc/c-api/hash.rst +++ b/Doc/c-api/hash.rst @@ -3,7 +3,7 @@ PyHash API ---------- -See also the :c:member:`PyTypeObject.tp_hash` member. +See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. .. c:type:: Py_hash_t @@ -17,6 +17,35 @@ See also the :c:member:`PyTypeObject.tp_hash` member. .. versionadded:: 3.2 +.. c:macro:: PyHASH_MODULUS + + The `Mersenne prime `_ ``P = 2**n -1``, used for numeric hash scheme. + + .. versionadded:: 3.13 + +.. c:macro:: PyHASH_BITS + + The exponent ``n`` of ``P`` in :c:macro:`PyHASH_MODULUS`. + + .. versionadded:: 3.13 + +.. c:macro:: PyHASH_MULTIPLIER + + Prime multiplier used in string and various other hashes. + + .. versionadded:: 3.13 + +.. c:macro:: PyHASH_INF + + The hash value returned for a positive infinity. + + .. versionadded:: 3.13 + +.. c:macro:: PyHASH_IMAG + + The multiplier used for the imaginary part of a complex number. + + .. versionadded:: 3.13 .. c:type:: PyHash_FuncDef @@ -59,3 +88,33 @@ See also the :c:member:`PyTypeObject.tp_hash` member. The function cannot fail: it cannot return ``-1``. .. versionadded:: 3.13 + + +.. c:function:: Py_hash_t Py_HashBuffer(const void *ptr, Py_ssize_t len) + + Compute and return the hash value of a buffer of *len* bytes + starting at address *ptr*. The hash is guaranteed to match that of + :class:`bytes`, :class:`memoryview`, and other built-in objects + that implement the :ref:`buffer protocol `. + + Use this function to implement hashing for immutable objects whose + :c:member:`~PyTypeObject.tp_richcompare` function compares to another + object's buffer. + + *len* must be greater than or equal to ``0``. + + This function always succeeds. + + .. versionadded:: 3.14 + + +.. c:function:: Py_hash_t PyObject_GenericHash(PyObject *obj) + + Generic hashing function that is meant to be put into a type + object's ``tp_hash`` slot. + Its result only depends on the object's identity. + + .. impl-detail:: + In CPython, it is equivalent to :c:func:`Py_HashPointer`. + + .. versionadded:: 3.13 diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 7c74e9e88678dc..6e48644c8fef8b 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -136,14 +136,14 @@ Importing Modules such modules have no way to know that the module object is an unknown (and probably damaged with respect to the module author's intents) state. - The module's :attr:`__spec__` and :attr:`__loader__` will be set, if - not set already, with the appropriate values. The spec's loader will - be set to the module's ``__loader__`` (if set) and to an instance of - :class:`~importlib.machinery.SourceFileLoader` otherwise. + The module's :attr:`~module.__spec__` and :attr:`~module.__loader__` will be + set, if not set already, with the appropriate values. The spec's loader + will be set to the module's :attr:`!__loader__` (if set) and to an instance + of :class:`~importlib.machinery.SourceFileLoader` otherwise. - The module's :attr:`__file__` attribute will be set to the code object's - :attr:`~codeobject.co_filename`. If applicable, :attr:`__cached__` will also - be set. + The module's :attr:`~module.__file__` attribute will be set to the code + object's :attr:`~codeobject.co_filename`. If applicable, + :attr:`~module.__cached__` will also be set. This function will reload the module if it was already imported. See :c:func:`PyImport_ReloadModule` for the intended way to reload a module. @@ -155,29 +155,29 @@ Importing Modules :c:func:`PyImport_ExecCodeModuleWithPathnames`. .. versionchanged:: 3.12 - The setting of :attr:`__cached__` and :attr:`__loader__` is - deprecated. See :class:`~importlib.machinery.ModuleSpec` for + The setting of :attr:`~module.__cached__` and :attr:`~module.__loader__` + is deprecated. See :class:`~importlib.machinery.ModuleSpec` for alternatives. .. c:function:: PyObject* PyImport_ExecCodeModuleEx(const char *name, PyObject *co, const char *pathname) - Like :c:func:`PyImport_ExecCodeModule`, but the :attr:`__file__` attribute of - the module object is set to *pathname* if it is non-``NULL``. + Like :c:func:`PyImport_ExecCodeModule`, but the :attr:`~module.__file__` + attribute of the module object is set to *pathname* if it is non-``NULL``. See also :c:func:`PyImport_ExecCodeModuleWithPathnames`. .. c:function:: PyObject* PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, PyObject *cpathname) - Like :c:func:`PyImport_ExecCodeModuleEx`, but the :attr:`__cached__` + Like :c:func:`PyImport_ExecCodeModuleEx`, but the :attr:`~module.__cached__` attribute of the module object is set to *cpathname* if it is non-``NULL``. Of the three functions, this is the preferred one to use. .. versionadded:: 3.3 .. versionchanged:: 3.12 - Setting :attr:`__cached__` is deprecated. See + Setting :attr:`~module.__cached__` is deprecated. See :class:`~importlib.machinery.ModuleSpec` for alternatives. @@ -190,7 +190,7 @@ Importing Modules .. versionadded:: 3.2 .. versionchanged:: 3.3 - Uses :func:`!imp.source_from_cache()` in calculating the source path if + Uses :func:`!imp.source_from_cache` in calculating the source path if only the bytecode path is provided. .. versionchanged:: 3.12 No longer uses the removed :mod:`!imp` module. @@ -308,7 +308,7 @@ Importing Modules The module name, as an ASCII encoded string. - .. c: member:: PyObject* (*initfunc)(void) + .. c:member:: PyObject* (*initfunc)(void) Initialization function for a module built into the interpreter. diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst index 9a8f1507b3f4cc..ba56b03c6ac8e7 100644 --- a/Doc/c-api/index.rst +++ b/Doc/c-api/index.rst @@ -25,3 +25,4 @@ document the API functions in detail. memory.rst objimpl.rst apiabiversion.rst + monitoring.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index e7199ad5e0c1b1..dc44f3eaf87765 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -7,7 +7,8 @@ Initialization, Finalization, and Threads ***************************************** -See also :ref:`Python Initialization Configuration `. +See :ref:`Python Initialization Configuration ` for details +on how to configure the interpreter prior to initialization. .. _pre-init-safe: @@ -21,6 +22,15 @@ a few functions and the :ref:`global configuration variables The following functions can be safely called before Python is initialized: +* Functions that initialize the interpreter: + + * :c:func:`Py_Initialize` + * :c:func:`Py_InitializeEx` + * :c:func:`Py_InitializeFromConfig` + * :c:func:`Py_BytesMain` + * :c:func:`Py_Main` + * the runtime pre-initialization functions covered in :ref:`init-config` + * Configuration functions: * :c:func:`PyImport_AppendInittab` @@ -29,7 +39,10 @@ The following functions can be safely called before Python is initialized: * :c:func:`PyMem_SetAllocator` * :c:func:`PyMem_SetupDebugHooks` * :c:func:`PyObject_SetArenaAllocator` + * :c:func:`Py_SetProgramName` + * :c:func:`Py_SetPythonHome` * :c:func:`PySys_ResetWarnOptions` + * the configuration functions covered in :ref:`init-config` * Informative functions: @@ -41,10 +54,12 @@ The following functions can be safely called before Python is initialized: * :c:func:`Py_GetCopyright` * :c:func:`Py_GetPlatform` * :c:func:`Py_GetVersion` + * :c:func:`Py_IsInitialized` * Utilities: * :c:func:`Py_DecodeLocale` + * the status reporting and utility functions covered in :ref:`init-config` * Memory allocators: @@ -53,13 +68,20 @@ The following functions can be safely called before Python is initialized: * :c:func:`PyMem_RawCalloc` * :c:func:`PyMem_RawFree` +* Synchronization: + + * :c:func:`PyMutex_Lock` + * :c:func:`PyMutex_Unlock` + .. note:: - The following functions **should not be called** before - :c:func:`Py_Initialize`: :c:func:`Py_EncodeLocale`, :c:func:`Py_GetPath`, + Despite their apparent similarity to some of the functions listed above, + the following functions **should not be called** before the interpreter has + been initialized: :c:func:`Py_EncodeLocale`, :c:func:`Py_GetPath`, :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome`, - and :c:func:`Py_GetProgramName`. + :c:func:`Py_GetProgramName`, :c:func:`PyEval_InitThreads`, and + :c:func:`Py_RunMain`. .. _global-conf-vars: @@ -87,7 +109,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-b` option. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_DebugFlag @@ -101,7 +123,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment variable. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_DontWriteBytecodeFlag @@ -115,7 +137,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` environment variable. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_FrozenFlag @@ -128,7 +150,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Private flag used by ``_freeze_module`` and ``frozenmain`` programs. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_HashRandomizationFlag @@ -143,7 +165,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment variable to initialize the secret hash seed. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_IgnoreEnvironmentFlag @@ -156,7 +178,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-E` and :option:`-I` options. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_InspectFlag @@ -171,7 +193,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment variable. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_InteractiveFlag @@ -196,7 +218,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. versionadded:: 3.4 - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_LegacyWindowsFSEncodingFlag @@ -215,7 +237,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. availability:: Windows. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_LegacyWindowsStdioFlag @@ -233,7 +255,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. availability:: Windows. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_NoSiteFlag @@ -248,7 +270,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-S` option. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_NoUserSiteDirectory @@ -262,7 +284,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-s` and :option:`-I` options, and the :envvar:`PYTHONNOUSERSITE` environment variable. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_OptimizeFlag @@ -273,7 +295,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment variable. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_QuietFlag @@ -287,7 +309,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. versionadded:: 3.2 - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_UnbufferedStdioFlag @@ -300,7 +322,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` environment variable. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_VerboseFlag @@ -316,7 +338,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment variable. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 Initializing and finalizing the interpreter @@ -326,6 +348,7 @@ Initializing and finalizing the interpreter .. c:function:: void Py_Initialize() .. index:: + single: PyEval_InitThreads() single: modules (in module sys) single: path (in module sys) pair: module; builtins @@ -338,34 +361,42 @@ Initializing and finalizing the interpreter this should be called before using any other Python/C API functions; see :ref:`Before Python Initialization ` for the few exceptions. - This initializes - the table of loaded modules (``sys.modules``), and creates the fundamental - modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. It also initializes - the module search path (``sys.path``). It does not set ``sys.argv``; use - the new :c:type:`PyConfig` API of the :ref:`Python Initialization - Configuration ` for that. This is a no-op when called for a - second time - (without calling :c:func:`Py_FinalizeEx` first). There is no return value; it is a - fatal error if the initialization fails. - - Use the :c:func:`Py_InitializeFromConfig` function to customize the + This initializes the table of loaded modules (``sys.modules``), and creates + the fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. + It also initializes the module search path (``sys.path``). It does not set + ``sys.argv``; use the :ref:`Python Initialization Configuration ` + API for that. This is a no-op when called for a second time (without calling + :c:func:`Py_FinalizeEx` first). There is no return value; it is a fatal + error if the initialization fails. + + Use :c:func:`Py_InitializeFromConfig` to customize the :ref:`Python Initialization Configuration `. .. note:: - On Windows, changes the console mode from ``O_TEXT`` to ``O_BINARY``, which will - also affect non-Python uses of the console using the C Runtime. + On Windows, changes the console mode from ``O_TEXT`` to ``O_BINARY``, + which will also affect non-Python uses of the console using the C Runtime. .. c:function:: void Py_InitializeEx(int initsigs) This function works like :c:func:`Py_Initialize` if *initsigs* is ``1``. If - *initsigs* is ``0``, it skips initialization registration of signal handlers, which - might be useful when Python is embedded. + *initsigs* is ``0``, it skips initialization registration of signal handlers, + which may be useful when CPython is embedded as part of a larger application. - Use the :c:func:`Py_InitializeFromConfig` function to customize the + Use :c:func:`Py_InitializeFromConfig` to customize the :ref:`Python Initialization Configuration `. +.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) + + Initialize Python from *config* configuration, as described in + :ref:`init-from-config`. + + See the :ref:`init-config` section for details on pre-initializing the + interpreter, populating the runtime configuration structure, and querying + the returned status structure. + + .. c:function:: int Py_IsInitialized() Return true (nonzero) when the Python interpreter has been initialized, false @@ -386,11 +417,23 @@ Initializing and finalizing the interpreter Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of Python/C API functions, and destroy all sub-interpreters (see :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since - the last call to :c:func:`Py_Initialize`. Ideally, this frees all memory - allocated by the Python interpreter. This is a no-op when called for a second - time (without calling :c:func:`Py_Initialize` again first). Normally the - return value is ``0``. If there were errors during finalization - (flushing buffered data), ``-1`` is returned. + the last call to :c:func:`Py_Initialize`. This is a no-op when called for a second + time (without calling :c:func:`Py_Initialize` again first). + + Since this is the reverse of :c:func:`Py_Initialize`, it should be called + in the same thread with the same interpreter active. That means + the main thread and the main interpreter. + This should never be called while :c:func:`Py_RunMain` is running. + + Normally the return value is ``0``. + If there were errors during finalization (flushing buffered data), + ``-1`` is returned. + + Note that Python will do a best effort at freeing all memory allocated by the Python + interpreter. Therefore, any C-Extension should make sure to correctly clean up all + of the preveiously allocated PyObjects before using them in subsequent calls to + :c:func:`Py_Initialize`. Otherwise it could introduce vulnerabilities and incorrect + behavior. This function is provided for a number of reasons. An embedding application might want to restart Python without having to restart the application itself. @@ -406,25 +449,166 @@ Initializing and finalizing the interpreter loaded extension modules loaded by Python are not unloaded. Small amounts of memory allocated by the Python interpreter may not be freed (if you find a leak, please report it). Memory tied up in circular references between objects is not - freed. Some memory allocated by extension modules may not be freed. Some - extensions may not work properly if their initialization routine is called more - than once; this can happen if an application calls :c:func:`Py_Initialize` and - :c:func:`Py_FinalizeEx` more than once. + freed. Interned strings will all be deallocated regardless of their reference count. + Some memory allocated by extension modules may not be freed. Some extensions may not + work properly if their initialization routine is called more than once; this can + happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx` + more than once. :c:func:`Py_FinalizeEx` must not be called recursively from + within itself. Therefore, it must not be called by any code that may be run + as part of the interpreter shutdown process, such as :py:mod:`atexit` + handlers, object finalizers, or any code that may be run while flushing the + stdout and stderr files. .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx .. versionadded:: 3.6 + .. c:function:: void Py_Finalize() This is a backwards-compatible version of :c:func:`Py_FinalizeEx` that disregards the return value. +.. c:function:: int Py_BytesMain(int argc, char **argv) + + Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings, + allowing the calling application to delegate the text decoding step to + the CPython runtime. + + .. versionadded:: 3.8 + + +.. c:function:: int Py_Main(int argc, wchar_t **argv) + + The main program for the standard interpreter, encapsulating a full + initialization/finalization cycle, as well as additional + behaviour to implement reading configurations settings from the environment + and command line, and then executing ``__main__`` in accordance with + :ref:`using-on-cmdline`. + + This is made available for programs which wish to support the full CPython + command line interface, rather than just embedding a Python runtime in a + larger application. + + The *argc* and *argv* parameters are similar to those which are passed to a + C program's :c:func:`main` function, except that the *argv* entries are first + converted to ``wchar_t`` using :c:func:`Py_DecodeLocale`. It is also + important to note that the argument list entries may be modified to point to + strings other than those passed in (however, the contents of the strings + pointed to by the argument list are not modified). + + The return value will be ``0`` if the interpreter exits normally (i.e., + without an exception), ``1`` if the interpreter exits due to an exception, + or ``2`` if the argument list does not represent a valid Python command + line. + + Note that if an otherwise unhandled :exc:`SystemExit` is raised, this + function will not return ``1``, but exit the process, as long as + ``Py_InspectFlag`` is not set. If ``Py_InspectFlag`` is set, execution will + drop into the interactive Python prompt, at which point a second otherwise + unhandled :exc:`SystemExit` will still exit the process, while any other + means of exiting will set the return value as described above. + + In terms of the CPython runtime configuration APIs documented in the + :ref:`runtime configuration ` section (and without accounting + for error handling), ``Py_Main`` is approximately equivalent to:: + + PyConfig config; + PyConfig_InitPythonConfig(&config); + PyConfig_SetArgv(&config, argc, argv); + Py_InitializeFromConfig(&config); + PyConfig_Clear(&config); + + Py_RunMain(); + + In normal usage, an embedding application will call this function + *instead* of calling :c:func:`Py_Initialize`, :c:func:`Py_InitializeEx` or + :c:func:`Py_InitializeFromConfig` directly, and all settings will be applied + as described elsewhere in this documentation. If this function is instead + called *after* a preceding runtime initialization API call, then exactly + which environmental and command line configuration settings will be updated + is version dependent (as it depends on which settings correctly support + being modified after they have already been set once when the runtime was + first initialized). + + +.. c:function:: int Py_RunMain(void) + + Executes the main module in a fully configured CPython runtime. + + Executes the command (:c:member:`PyConfig.run_command`), the script + (:c:member:`PyConfig.run_filename`) or the module + (:c:member:`PyConfig.run_module`) specified on the command line or in the + configuration. If none of these values are set, runs the interactive Python + prompt (REPL) using the ``__main__`` module's global namespace. + + If :c:member:`PyConfig.inspect` is not set (the default), the return value + will be ``0`` if the interpreter exits normally (that is, without raising + an exception), or ``1`` if the interpreter exits due to an exception. If an + otherwise unhandled :exc:`SystemExit` is raised, the function will immediately + exit the process instead of returning ``1``. + + If :c:member:`PyConfig.inspect` is set (such as when the :option:`-i` option + is used), rather than returning when the interpreter exits, execution will + instead resume in an interactive Python prompt (REPL) using the ``__main__`` + module's global namespace. If the interpreter exited with an exception, it + is immediately raised in the REPL session. The function return value is + then determined by the way the *REPL session* terminates: returning ``0`` + if the session terminates without raising an unhandled exception, exiting + immediately for an unhandled :exc:`SystemExit`, and returning ``1`` for + any other unhandled exception. + + This function always finalizes the Python interpreter regardless of whether + it returns a value or immediately exits the process due to an unhandled + :exc:`SystemExit` exception. + + See :ref:`Python Configuration ` for an example of a + customized Python that always runs in isolated mode using + :c:func:`Py_RunMain`. + +.. c:function:: int PyUnstable_AtExit(PyInterpreterState *interp, void (*func)(void *), void *data) + + Register an :mod:`atexit` callback for the target interpreter *interp*. + This is similar to :c:func:`Py_AtExit`, but takes an explicit interpreter and + data pointer for the callback. + + The :term:`GIL` must be held for *interp*. + + .. versionadded:: 3.13 + Process-wide parameters ======================= +.. c:function:: void Py_SetProgramName(const wchar_t *name) + + .. index:: + single: Py_Initialize() + single: main() + single: Py_GetPath() + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.program_name` should be used instead, see :ref:`Python + Initialization Configuration `. + + This function should be called before :c:func:`Py_Initialize` is called for + the first time, if it is called at all. It tells the interpreter the value + of the ``argv[0]`` argument to the :c:func:`main` function of the program + (converted to wide characters). + This is used by :c:func:`Py_GetPath` and some other functions below to find + the Python run-time libraries relative to the interpreter executable. The + default value is ``'python'``. The argument should point to a + zero-terminated wide character string in static storage whose contents will not + change for the duration of the program's execution. No code in the Python + interpreter will change the contents of this storage. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_*` string. + + .. deprecated-removed:: 3.11 3.15 + + .. c:function:: wchar_t* Py_GetProgramName() Return the program name set with :c:member:`PyConfig.program_name`, or the default. @@ -450,7 +634,7 @@ Process-wide parameters returned string points into static storage; the caller should not modify its value. This corresponds to the :makevar:`prefix` variable in the top-level :file:`Makefile` and the :option:`--prefix` argument to the :program:`configure` - script at build time. The value is available to Python code as ``sys.prefix``. + script at build time. The value is available to Python code as ``sys.base_prefix``. It is only useful on Unix. See also the next function. This function should not be called before :c:func:`Py_Initialize`, otherwise @@ -460,7 +644,8 @@ Process-wide parameters It now returns ``NULL`` if called before :c:func:`Py_Initialize`. .. deprecated-removed:: 3.13 3.15 - Get :data:`sys.prefix` instead. + Get :data:`sys.base_prefix` instead, or :data:`sys.prefix` if + :ref:`virtual environments ` need to be handled. .. c:function:: wchar_t* Py_GetExecPrefix() @@ -473,7 +658,8 @@ Process-wide parameters should not modify its value. This corresponds to the :makevar:`exec_prefix` variable in the top-level :file:`Makefile` and the ``--exec-prefix`` argument to the :program:`configure` script at build time. The value is - available to Python code as ``sys.exec_prefix``. It is only useful on Unix. + available to Python code as ``sys.base_exec_prefix``. It is only useful on + Unix. Background: The exec-prefix differs from the prefix when platform dependent files (such as executables and shared libraries) are installed in a different @@ -504,7 +690,8 @@ Process-wide parameters It now returns ``NULL`` if called before :c:func:`Py_Initialize`. .. deprecated-removed:: 3.13 3.15 - Get :data:`sys.exec_prefix` instead. + Get :data:`sys.base_exec_prefix` instead, or :data:`sys.exec_prefix` if + :ref:`virtual environments ` need to be handled. .. c:function:: wchar_t* Py_GetProgramFullPath() @@ -626,6 +813,106 @@ Process-wide parameters ``sys.version``. +.. c:function:: void PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) + + .. index:: + single: main() + single: Py_FatalError() + single: argv (in module sys) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.argv`, :c:member:`PyConfig.parse_argv` and + :c:member:`PyConfig.safe_path` should be used instead, see :ref:`Python + Initialization Configuration `. + + Set :data:`sys.argv` based on *argc* and *argv*. These parameters are + similar to those passed to the program's :c:func:`main` function with the + difference that the first entry should refer to the script file to be + executed rather than the executable hosting the Python interpreter. If there + isn't a script that will be run, the first entry in *argv* can be an empty + string. If this function fails to initialize :data:`sys.argv`, a fatal + condition is signalled using :c:func:`Py_FatalError`. + + If *updatepath* is zero, this is all the function does. If *updatepath* + is non-zero, the function also modifies :data:`sys.path` according to the + following algorithm: + + - If the name of an existing script is passed in ``argv[0]``, the absolute + path of the directory where the script is located is prepended to + :data:`sys.path`. + - Otherwise (that is, if *argc* is ``0`` or ``argv[0]`` doesn't point + to an existing file name), an empty string is prepended to + :data:`sys.path`, which is the same as prepending the current working + directory (``"."``). + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_*` string. + + See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` + members of the :ref:`Python Initialization Configuration `. + + .. note:: + It is recommended that applications embedding the Python interpreter + for purposes other than executing a single script pass ``0`` as *updatepath*, + and update :data:`sys.path` themselves if desired. + See :cve:`2008-5983`. + + On versions before 3.1.3, you can achieve the same effect by manually + popping the first :data:`sys.path` element after having called + :c:func:`PySys_SetArgv`, for example using:: + + PyRun_SimpleString("import sys; sys.path.pop(0)\n"); + + .. versionadded:: 3.1.3 + + .. XXX impl. doesn't seem consistent in allowing ``0``/``NULL`` for the params; + check w/ Guido. + + .. deprecated-removed:: 3.11 3.15 + + +.. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.argv` and :c:member:`PyConfig.parse_argv` should be used + instead, see :ref:`Python Initialization Configuration `. + + This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set + to ``1`` unless the :program:`python` interpreter was started with the + :option:`-I`. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_*` string. + + See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` + members of the :ref:`Python Initialization Configuration `. + + .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. + + .. deprecated-removed:: 3.11 3.15 + + +.. c:function:: void Py_SetPythonHome(const wchar_t *home) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.home` should be used instead, see :ref:`Python + Initialization Configuration `. + + Set the default "home" directory, that is, the location of the standard + Python libraries. See :envvar:`PYTHONHOME` for the meaning of the + argument string. + + The argument should point to a zero-terminated character string in static + storage whose contents will not change for the duration of the program's + execution. No code in the Python interpreter will change the contents of + this storage. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_*` string. + + .. deprecated-removed:: 3.11 3.15 + + .. c:function:: wchar_t* Py_GetPythonHome() Return the default "home", that is, the value set by @@ -811,6 +1098,37 @@ thread, where the CPython global runtime was originally initialized. The only exception is if :c:func:`exec` will be called immediately after. +.. _cautions-regarding-runtime-finalization: + +Cautions regarding runtime finalization +--------------------------------------- + +In the late stage of :term:`interpreter shutdown`, after attempting to wait for +non-daemon threads to exit (though this can be interrupted by +:class:`KeyboardInterrupt`) and running the :mod:`atexit` functions, the runtime +is marked as *finalizing*: :c:func:`_Py_IsFinalizing` and +:func:`sys.is_finalizing` return true. At this point, only the *finalization +thread* that initiated finalization (typically the main thread) is allowed to +acquire the :term:`GIL`. + +If any thread, other than the finalization thread, attempts to acquire the GIL +during finalization, either explicitly via :c:func:`PyGILState_Ensure`, +:c:macro:`Py_END_ALLOW_THREADS`, :c:func:`PyEval_AcquireThread`, or +:c:func:`PyEval_AcquireLock`, or implicitly when the interpreter attempts to +reacquire it after having yielded it, the thread enters **a permanently blocked +state** where it remains until the program exits. In most cases this is +harmless, but this can result in deadlock if a later stage of finalization +attempts to acquire a lock owned by the blocked thread, or otherwise waits on +the blocked thread. + +Gross? Yes. This prevents random crashes and/or unexpectedly skipped C++ +finalizations further up the call stack when such threads were forcibly exited +here in CPython 3.13 and earlier. The CPython runtime GIL acquiring C APIs +have never had any error reporting or handling expectations at GIL acquisition +time that would've allowed for graceful exit from this situation. Changing that +would require new stable C APIs and rewriting the majority of C code in the +CPython ecosystem to use those with error handling. + High-level API -------------- @@ -841,6 +1159,33 @@ code, or when embedding the Python interpreter: This thread's interpreter state. +.. c:function:: void PyEval_InitThreads() + + .. index:: + single: PyEval_AcquireThread() + single: PyEval_ReleaseThread() + single: PyEval_SaveThread() + single: PyEval_RestoreThread() + + Deprecated function which does nothing. + + In Python 3.6 and older, this function created the GIL if it didn't exist. + + .. versionchanged:: 3.9 + The function now does nothing. + + .. versionchanged:: 3.7 + This function is now called by :c:func:`Py_Initialize()`, so you don't + have to call it yourself anymore. + + .. versionchanged:: 3.2 + This function cannot be called before :c:func:`Py_Initialize()` anymore. + + .. deprecated:: 3.9 + + .. index:: pair: module; _thread + + .. c:function:: PyThreadState* PyEval_SaveThread() Release the global interpreter lock (if it has been created) and reset the @@ -857,11 +1202,14 @@ code, or when embedding the Python interpreter: ensues. .. note:: - Calling this function from a thread when the runtime is finalizing - will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to - check if the interpreter is in process of being finalized before calling - this function to avoid unwanted termination. + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. .. c:function:: PyThreadState* PyThreadState_Get() @@ -916,11 +1264,14 @@ with sub-interpreters: to call arbitrary Python code. Failure is a fatal error. .. note:: - Calling this function from a thread when the runtime is finalizing - will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to - check if the interpreter is in process of being finalized before calling - this function to avoid unwanted termination. + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. .. c:function:: void PyGILState_Release(PyGILState_STATE) @@ -1037,6 +1388,9 @@ All of the following functions must be called after :c:func:`Py_Initialize`. This function now calls the :c:member:`PyThreadState.on_delete` callback. Previously, that happened in :c:func:`PyThreadState_Delete`. + .. versionchanged:: 3.13 + The :c:member:`PyThreadState.on_delete` callback was removed. + .. c:function:: void PyThreadState_Delete(PyThreadState *tstate) @@ -1048,7 +1402,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. c:function:: void PyThreadState_DeleteCurrent(void) Destroy the current thread state and release the global interpreter lock. - Like :c:func:`PyThreadState_Delete`, the global interpreter lock need not + Like :c:func:`PyThreadState_Delete`, the global interpreter lock must be held. The thread state must have been reset with a previous call to :c:func:`PyThreadState_Clear`. @@ -1138,6 +1492,17 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. versionadded:: 3.8 + +.. c:function:: PyObject* PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *interp) + + Return a :term:`strong reference` to the ``__main__`` `module object `_ + for the given interpreter. + + The caller must hold the GIL. + + .. versionadded:: 3.13 + + .. c:type:: PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) Type of a frame evaluation function. @@ -1198,17 +1563,20 @@ All of the following functions must be called after :c:func:`Py_Initialize`. If this thread already has the lock, deadlock ensues. .. note:: - Calling this function from a thread when the runtime is finalizing - will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to - check if the interpreter is in process of being finalized before calling - this function to avoid unwanted termination. + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. .. versionchanged:: 3.8 Updated to be consistent with :c:func:`PyEval_RestoreThread`, :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, and terminate the current thread if called while the interpreter is finalizing. + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + :c:func:`PyEval_RestoreThread` is a higher-level function which is always available (even when threads have not been initialized). @@ -1390,7 +1758,11 @@ function. You can create and destroy them using the following functions: .check_multi_interp_extensions = 1, .gil = PyInterpreterConfig_OWN_GIL, }; - PyThreadState *tstate = Py_NewInterpreterFromConfig(&config); + PyThreadState *tstate = NULL; + PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } Note that the config is used only briefly and does not get modified. During initialization the config's values are converted into various @@ -1575,14 +1947,14 @@ pointer and a void pointer argument. function is generally **not** suitable for calling Python code from arbitrary C threads. Instead, use the :ref:`PyGILState API`. + .. versionadded:: 3.1 + .. versionchanged:: 3.9 If this function is called in a subinterpreter, the function *func* is now scheduled to be called from the subinterpreter, rather than being called from the main interpreter. Each subinterpreter now has its own list of scheduled calls. - .. versionadded:: 3.1 - .. _profiling: Profiling and Tracing @@ -1746,6 +2118,58 @@ Python-level trace functions in previous versions. .. versionadded:: 3.12 +Reference tracing +================= + +.. versionadded:: 3.13 + +.. c:type:: int (*PyRefTracer)(PyObject *, int event, void* data) + + The type of the trace function registered using :c:func:`PyRefTracer_SetTracer`. + The first parameter is a Python object that has been just created (when **event** + is set to :c:data:`PyRefTracer_CREATE`) or about to be destroyed (when **event** + is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer + that was provided when :c:func:`PyRefTracer_SetTracer` was called. + +.. versionadded:: 3.13 + +.. c:var:: int PyRefTracer_CREATE + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been created. + +.. c:var:: int PyRefTracer_DESTROY + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been destroyed. + +.. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) + + Register a reference tracer function. The function will be called when a new + Python has been created or when an object is going to be destroyed. If + **data** is provided it must be an opaque pointer that will be provided when + the tracer function is called. Return ``0`` on success. Set an exception and + return ``-1`` on error. + + Not that tracer functions **must not** create Python objects inside or + otherwise the call will be re-entrant. The tracer also **must not** clear + any existing exception or set an exception. The GIL will be held every time + the tracer function is called. + + The GIL must be held when calling this function. + +.. versionadded:: 3.13 + +.. c:function:: PyRefTracer PyRefTracer_GetTracer(void** data) + + Get the registered reference tracer function and the value of the opaque data + pointer that was registered when :c:func:`PyRefTracer_SetTracer` was called. + If no tracer was registered this function will return NULL and will set the + **data** pointer to NULL. + + The GIL must be held when calling this function. + +.. versionadded:: 3.13 .. _advanced-debugging: @@ -1942,3 +2366,145 @@ be used in new code. .. c:function:: void PyThread_delete_key_value(int key) .. c:function:: void PyThread_ReInitTLS() +Synchronization Primitives +========================== + +The C-API provides a basic mutual exclusion lock. + +.. c:type:: PyMutex + + A mutual exclusion lock. The :c:type:`!PyMutex` should be initialized to + zero to represent the unlocked state. For example:: + + PyMutex mutex = {0}; + + Instances of :c:type:`!PyMutex` should not be copied or moved. Both the + contents and address of a :c:type:`!PyMutex` are meaningful, and it must + remain at a fixed, writable location in memory. + + .. note:: + + A :c:type:`!PyMutex` currently occupies one byte, but the size should be + considered unstable. The size may change in future Python releases + without a deprecation period. + + .. versionadded:: 3.13 + +.. c:function:: void PyMutex_Lock(PyMutex *m) + + Lock mutex *m*. If another thread has already locked it, the calling + thread will block until the mutex is unlocked. While blocked, the thread + will temporarily release the :term:`GIL` if it is held. + + .. versionadded:: 3.13 + +.. c:function:: void PyMutex_Unlock(PyMutex *m) + + Unlock mutex *m*. The mutex must be locked --- otherwise, the function will + issue a fatal error. + + .. versionadded:: 3.13 + +.. _python-critical-section-api: + +Python Critical Section API +--------------------------- + +The critical section API provides a deadlock avoidance layer on top of +per-object locks for :term:`free-threaded ` CPython. They are +intended to replace reliance on the :term:`global interpreter lock`, and are +no-ops in versions of Python with the global interpreter lock. + +Critical sections avoid deadlocks by implicitly suspending active critical +sections and releasing the locks during calls to :c:func:`PyEval_SaveThread`. +When :c:func:`PyEval_RestoreThread` is called, the most recent critical section +is resumed, and its locks reacquired. This means the critical section API +provides weaker guarantees than traditional locks -- they are useful because +their behavior is similar to the :term:`GIL`. + +The functions and structs used by the macros are exposed for cases +where C macros are not available. They should only be used as in the +given macro expansions. Note that the sizes and contents of the structures may +change in future Python versions. + +.. note:: + + Operations that need to lock two objects at once must use + :c:macro:`Py_BEGIN_CRITICAL_SECTION2`. You *cannot* use nested critical + sections to lock more than one object at once, because the inner critical + section may suspend the outer critical sections. This API does not provide + a way to lock more than two objects at once. + +Example usage:: + + static PyObject * + set_field(MyObject *self, PyObject *value) + { + Py_BEGIN_CRITICAL_SECTION(self); + Py_SETREF(self->field, Py_XNewRef(value)); + Py_END_CRITICAL_SECTION(); + Py_RETURN_NONE; + } + +In the above example, :c:macro:`Py_SETREF` calls :c:macro:`Py_DECREF`, which +can call arbitrary code through an object's deallocation function. The critical +section API avoids potential deadlocks due to reentrancy and lock ordering +by allowing the runtime to temporarily suspend the critical section if the +code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION(op) + + Acquires the per-object lock for the object *op* and begins a + critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection _py_cs; + PyCriticalSection_Begin(&_py_cs, (PyObject*)(op)) + + In the default build, this macro expands to ``{``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_END_CRITICAL_SECTION() + + Ends the critical section and releases the per-object lock. + + In the free-threaded build, this macro expands to:: + + PyCriticalSection_End(&_py_cs); + } + + In the default build, this macro expands to ``}``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION2(a, b) + + Acquires the per-objects locks for the objects *a* and *b* and begins a + critical section. The locks are acquired in a consistent order (lowest + address first) to avoid lock ordering deadlocks. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection2 _py_cs2; + PyCriticalSection2_Begin(&_py_cs2, (PyObject*)(a), (PyObject*)(b)) + + In the default build, this macro expands to ``{``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_END_CRITICAL_SECTION2() + + Ends the critical section and releases the per-object locks. + + In the free-threaded build, this macro expands to:: + + PyCriticalSection2_End(&_py_cs2); + } + + In the default build, this macro expands to ``}``. + + .. versionadded:: 3.13 diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 47a8fbb2cd9c97..b791d3cdc5d95c 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -6,6 +6,623 @@ Python Initialization Configuration *********************************** + +.. _pyinitconfig_api: + +PyInitConfig C API +================== + +.. versionadded:: 3.14 + +Python can be initialized with :c:func:`Py_InitializeFromInitConfig`. + +The :c:func:`Py_RunMain` function can be used to write a customized Python +program. + +See also :ref:`Initialization, Finalization, and Threads `. + +.. seealso:: + :pep:`741` "Python Configuration C API". + + +Example +------- + +Example of customized Python always running with the :ref:`Python Development +Mode ` enabled; return ``-1`` on error: + +.. code-block:: c + + int init_python(void) + { + PyInitConfig *config = PyInitConfig_Create(); + if (config == NULL) { + printf("PYTHON INIT ERROR: memory allocation failed\n"); + return -1; + } + + // Enable the Python Development Mode + if (PyInitConfig_SetInt(config, "dev_mode", 1) < 0) { + goto error; + } + + // Initialize Python with the configuration + if (Py_InitializeFromInitConfig(config) < 0) { + goto error; + } + PyInitConfig_Free(config); + return 0; + + error: + { + // Display the error message. + // + // This uncommon braces style is used, because you cannot make + // goto targets point to variable declarations. + const char *err_msg; + (void)PyInitConfig_GetError(config, &err_msg); + printf("PYTHON INIT ERROR: %s\n", err_msg); + PyInitConfig_Free(config); + return -1; + } + } + +Create Config +------------- + +.. c:struct:: PyInitConfig + + Opaque structure to configure the Python initialization. + + +.. c:function:: PyInitConfig* PyInitConfig_Create(void) + + Create a new initialization configuration using :ref:`Isolated Configuration + ` default values. + + It must be freed by :c:func:`PyInitConfig_Free`. + + Return ``NULL`` on memory allocation failure. + + +.. c:function:: void PyInitConfig_Free(PyInitConfig *config) + + Free memory of the initialization configuration *config*. + + If *config* is ``NULL``, no operation is performed. + + +Error Handling +-------------- + +.. c:function:: int PyInitConfig_GetError(PyInitConfig* config, const char **err_msg) + + Get the *config* error message. + + * Set *\*err_msg* and return ``1`` if an error is set. + * Set *\*err_msg* to ``NULL`` and return ``0`` otherwise. + + An error message is an UTF-8 encoded string. + + If *config* has an exit code, format the exit code as an error + message. + + The error message remains valid until another ``PyInitConfig`` + function is called with *config*. The caller doesn't have to free the + error message. + + +.. c:function:: int PyInitConfig_GetExitCode(PyInitConfig* config, int *exitcode) + + Get the *config* exit code. + + * Set *\*exitcode* and return ``1`` if *config* has an exit code set. + * Return ``0`` if *config* has no exit code set. + + Only the ``Py_InitializeFromInitConfig()`` function can set an exit + code if the ``parse_argv`` option is non-zero. + + An exit code can be set when parsing the command line failed (exit + code ``2``) or when a command line option asks to display the command + line help (exit code ``0``). + + +Get Options +----------- + +The configuration option *name* parameter must be a non-NULL null-terminated +UTF-8 encoded string. See :ref:`Configuration Options `. + +.. c:function:: int PyInitConfig_HasOption(PyInitConfig *config, const char *name) + + Test if the configuration has an option called *name*. + + Return ``1`` if the option exists, or return ``0`` otherwise. + + +.. c:function:: int PyInitConfig_GetInt(PyInitConfig *config, const char *name, int64_t *value) + + Get an integer configuration option. + + * Set *\*value*, and return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + +.. c:function:: int PyInitConfig_GetStr(PyInitConfig *config, const char *name, char **value) + + Get a string configuration option as a null-terminated UTF-8 + encoded string. + + * Set *\*value*, and return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + *\*value* can be set to ``NULL`` if the option is an optional string and the + option is unset. + + On success, the string must be released with ``free(value)`` if it's not + ``NULL``. + + +.. c:function:: int PyInitConfig_GetStrList(PyInitConfig *config, const char *name, size_t *length, char ***items) + + Get a string list configuration option as an array of + null-terminated UTF-8 encoded strings. + + * Set *\*length* and *\*value*, and return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + On success, the string list must be released with + ``PyInitConfig_FreeStrList(length, items)``. + + +.. c:function:: void PyInitConfig_FreeStrList(size_t length, char **items) + + Free memory of a string list created by + ``PyInitConfig_GetStrList()``. + + +Set Options +----------- + +The configuration option *name* parameter must be a non-NULL null-terminated +UTF-8 encoded string. See :ref:`Configuration Options `. + +Some configuration options have side effects on other options. This logic is +only implemented when ``Py_InitializeFromInitConfig()`` is called, not by the +"Set" functions below. For example, setting ``dev_mode`` to ``1`` does not set +``faulthandler`` to ``1``. + +.. c:function:: int PyInitConfig_SetInt(PyInitConfig *config, const char *name, int64_t value) + + Set an integer configuration option. + + * Return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + +.. c:function:: int PyInitConfig_SetStr(PyInitConfig *config, const char *name, const char *value) + + Set a string configuration option from a null-terminated UTF-8 + encoded string. The string is copied. + + * Return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + +.. c:function:: int PyInitConfig_SetStrList(PyInitConfig *config, const char *name, size_t length, char * const *items) + + Set a string list configuration option from an array of + null-terminated UTF-8 encoded strings. The string list is copied. + + * Return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + +Module +------ + +.. c:function:: int PyInitConfig_AddModule(PyInitConfig *config, const char *name, PyObject* (*initfunc)(void)) + + Add a built-in extension module to the table of built-in modules. + + The new module can be imported by the name *name*, and uses the function + *initfunc* as the initialization function called on the first attempted + import. + + * Return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + If Python is initialized multiple times, ``PyInitConfig_AddModule()`` must + be called at each Python initialization. + + Similar to the :c:func:`PyImport_AppendInittab` function. + + +Initialize Python +----------------- + +.. c:function:: int Py_InitializeFromInitConfig(PyInitConfig *config) + + Initialize Python from the initialization configuration. + + * Return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + * Set an exit code in *config* and return ``-1`` if Python wants to + exit. + + See ``PyInitConfig_GetExitcode()`` for the exit code case. + + +.. _pyinitconfig-opts: + +Configuration Options +===================== + +.. list-table:: + :header-rows: 1 + + * - Option + - PyConfig/PyPreConfig member + - Type + - Visibility + * - ``"allocator"`` + - :c:member:`allocator ` + - ``int`` + - Read-only + * - ``"argv"`` + - :c:member:`argv ` + - ``list[str]`` + - Public + * - ``"base_exec_prefix"`` + - :c:member:`base_exec_prefix ` + - ``str`` + - Public + * - ``"base_executable"`` + - :c:member:`base_executable ` + - ``str`` + - Public + * - ``"base_prefix"`` + - :c:member:`base_prefix ` + - ``str`` + - Public + * - ``"buffered_stdio"`` + - :c:member:`buffered_stdio ` + - ``bool`` + - Read-only + * - ``"bytes_warning"`` + - :c:member:`bytes_warning ` + - ``int`` + - Public + * - ``"check_hash_pycs_mode"`` + - :c:member:`check_hash_pycs_mode ` + - ``str`` + - Read-only + * - ``"code_debug_ranges"`` + - :c:member:`code_debug_ranges ` + - ``bool`` + - Read-only + * - ``"coerce_c_locale"`` + - :c:member:`coerce_c_locale ` + - ``bool`` + - Read-only + * - ``"coerce_c_locale_warn"`` + - :c:member:`coerce_c_locale_warn ` + - ``bool`` + - Read-only + * - ``"configure_c_stdio"`` + - :c:member:`configure_c_stdio ` + - ``bool`` + - Read-only + * - ``"configure_locale"`` + - :c:member:`configure_locale ` + - ``bool`` + - Read-only + * - ``"cpu_count"`` + - :c:member:`cpu_count ` + - ``int`` + - Read-only + * - ``"dev_mode"`` + - :c:member:`dev_mode ` + - ``bool`` + - Read-only + * - ``"dump_refs"`` + - :c:member:`dump_refs ` + - ``bool`` + - Read-only + * - ``"dump_refs_file"`` + - :c:member:`dump_refs_file ` + - ``str`` + - Read-only + * - ``"exec_prefix"`` + - :c:member:`exec_prefix ` + - ``str`` + - Public + * - ``"executable"`` + - :c:member:`executable ` + - ``str`` + - Public + * - ``"faulthandler"`` + - :c:member:`faulthandler ` + - ``bool`` + - Read-only + * - ``"filesystem_encoding"`` + - :c:member:`filesystem_encoding ` + - ``str`` + - Read-only + * - ``"filesystem_errors"`` + - :c:member:`filesystem_errors ` + - ``str`` + - Read-only + * - ``"hash_seed"`` + - :c:member:`hash_seed ` + - ``int`` + - Read-only + * - ``"home"`` + - :c:member:`home ` + - ``str`` + - Read-only + * - ``"import_time"`` + - :c:member:`import_time ` + - ``bool`` + - Read-only + * - ``"inspect"`` + - :c:member:`inspect ` + - ``bool`` + - Public + * - ``"install_signal_handlers"`` + - :c:member:`install_signal_handlers ` + - ``bool`` + - Read-only + * - ``"int_max_str_digits"`` + - :c:member:`int_max_str_digits ` + - ``int`` + - Public + * - ``"interactive"`` + - :c:member:`interactive ` + - ``bool`` + - Public + * - ``"isolated"`` + - :c:member:`isolated ` + - ``bool`` + - Read-only + * - ``"legacy_windows_fs_encoding"`` + - :c:member:`legacy_windows_fs_encoding ` + - ``bool`` + - Read-only + * - ``"legacy_windows_stdio"`` + - :c:member:`legacy_windows_stdio ` + - ``bool`` + - Read-only + * - ``"malloc_stats"`` + - :c:member:`malloc_stats ` + - ``bool`` + - Read-only + * - ``"module_search_paths"`` + - :c:member:`module_search_paths ` + - ``list[str]`` + - Public + * - ``"optimization_level"`` + - :c:member:`optimization_level ` + - ``int`` + - Public + * - ``"orig_argv"`` + - :c:member:`orig_argv ` + - ``list[str]`` + - Read-only + * - ``"parse_argv"`` + - :c:member:`parse_argv ` + - ``bool`` + - Read-only + * - ``"parser_debug"`` + - :c:member:`parser_debug ` + - ``bool`` + - Public + * - ``"pathconfig_warnings"`` + - :c:member:`pathconfig_warnings ` + - ``bool`` + - Read-only + * - ``"perf_profiling"`` + - :c:member:`perf_profiling ` + - ``bool`` + - Read-only + * - ``"platlibdir"`` + - :c:member:`platlibdir ` + - ``str`` + - Public + * - ``"prefix"`` + - :c:member:`prefix ` + - ``str`` + - Public + * - ``"program_name"`` + - :c:member:`program_name ` + - ``str`` + - Read-only + * - ``"pycache_prefix"`` + - :c:member:`pycache_prefix ` + - ``str`` + - Public + * - ``"quiet"`` + - :c:member:`quiet ` + - ``bool`` + - Public + * - ``"run_command"`` + - :c:member:`run_command ` + - ``str`` + - Read-only + * - ``"run_filename"`` + - :c:member:`run_filename ` + - ``str`` + - Read-only + * - ``"run_module"`` + - :c:member:`run_module ` + - ``str`` + - Read-only + * - ``"run_presite"`` + - :c:member:`run_presite ` + - ``str`` + - Read-only + * - ``"safe_path"`` + - :c:member:`safe_path ` + - ``bool`` + - Read-only + * - ``"show_ref_count"`` + - :c:member:`show_ref_count ` + - ``bool`` + - Read-only + * - ``"site_import"`` + - :c:member:`site_import ` + - ``bool`` + - Read-only + * - ``"skip_source_first_line"`` + - :c:member:`skip_source_first_line ` + - ``bool`` + - Read-only + * - ``"stdio_encoding"`` + - :c:member:`stdio_encoding ` + - ``str`` + - Read-only + * - ``"stdio_errors"`` + - :c:member:`stdio_errors ` + - ``str`` + - Read-only + * - ``"stdlib_dir"`` + - :c:member:`stdlib_dir ` + - ``str`` + - Public + * - ``"tracemalloc"`` + - :c:member:`tracemalloc ` + - ``int`` + - Read-only + * - ``"use_environment"`` + - :c:member:`use_environment ` + - ``bool`` + - Public + * - ``"use_frozen_modules"`` + - :c:member:`use_frozen_modules ` + - ``bool`` + - Read-only + * - ``"use_hash_seed"`` + - :c:member:`use_hash_seed ` + - ``bool`` + - Read-only + * - ``"user_site_directory"`` + - :c:member:`user_site_directory ` + - ``bool`` + - Read-only + * - ``"utf8_mode"`` + - :c:member:`utf8_mode ` + - ``bool`` + - Read-only + * - ``"verbose"`` + - :c:member:`verbose ` + - ``int`` + - Public + * - ``"warn_default_encoding"`` + - :c:member:`warn_default_encoding ` + - ``bool`` + - Read-only + * - ``"warnoptions"`` + - :c:member:`warnoptions ` + - ``list[str]`` + - Public + * - ``"write_bytecode"`` + - :c:member:`write_bytecode ` + - ``bool`` + - Public + * - ``"xoptions"`` + - :c:member:`xoptions ` + - ``dict[str, str]`` + - Public + * - ``"_pystats"`` + - :c:member:`_pystats ` + - ``bool`` + - Read-only + +Visibility: + +* Public: Can by get by :c:func:`PyConfig_Get` and set by + :c:func:`PyConfig_Set`. +* Read-only: Can by get by :c:func:`PyConfig_Get`, but cannot be set by + :c:func:`PyConfig_Set`. + + +Runtime Python configuration API +================================ + +At runtime, it's possible to get and set configuration options using +:c:func:`PyConfig_Get` and :c:func:`PyConfig_Set` functions. + +The configuration option *name* parameter must be a non-NULL null-terminated +UTF-8 encoded string. See :ref:`Configuration Options `. + +Some options are read from the :mod:`sys` attributes. For example, the option +``"argv"`` is read from :data:`sys.argv`. + + +.. c:function:: PyObject* PyConfig_Get(const char *name) + + Get the current runtime value of a configuration option as a Python object. + + * Return a new reference on success. + * Set an exception and return ``NULL`` on error. + + The object type depends on the configuration option. It can be: + + * ``bool`` + * ``int`` + * ``str`` + * ``list[str]`` + * ``dict[str, str]`` + + The caller must hold the GIL. The function cannot be called before + Python initialization nor after Python finalization. + + .. versionadded:: 3.14 + + +.. c:function:: int PyConfig_GetInt(const char *name, int *value) + + Similar to :c:func:`PyConfig_Get`, but get the value as a C int. + + * Return ``0`` on success. + * Set an exception and return ``-1`` on error. + + .. versionadded:: 3.14 + + +.. c:function:: PyObject* PyConfig_Names(void) + + Get all configuration option names as a ``frozenset``. + + * Return a new reference on success. + * Set an exception and return ``NULL`` on error. + + The caller must hold the GIL. The function cannot be called before + Python initialization nor after Python finalization. + + .. versionadded:: 3.14 + + +.. c:function:: int PyConfig_Set(const char *name, PyObject *value) + + Set the current runtime value of a configuration option. + + * Raise a :exc:`ValueError` if there is no option *name*. + * Raise a :exc:`ValueError` if *value* is an invalid value. + * Raise a :exc:`ValueError` if the option is read-only (cannot be set). + * Raise a :exc:`TypeError` if *value* has not the proper type. + + The caller must hold the GIL. The function cannot be called before + Python initialization nor after Python finalization. + + .. versionadded:: 3.14 + + +.. _pyconfig_api: + +PyConfig C API +============== + .. versionadded:: 3.8 Python can be initialized with :c:func:`Py_InitializeFromConfig` and the @@ -34,7 +651,7 @@ See also :ref:`Initialization, Finalization, and Threads `. Example -======= +------- Example of customized Python always running in isolated mode:: @@ -73,7 +690,7 @@ Example of customized Python always running in isolated mode:: PyWideStringList -================ +---------------- .. c:type:: PyWideStringList @@ -116,7 +733,7 @@ PyWideStringList List items. PyStatus -======== +-------- .. c:type:: PyStatus @@ -210,7 +827,7 @@ Example:: PyPreConfig -=========== +----------- .. c:type:: PyPreConfig @@ -321,7 +938,7 @@ PyPreConfig * Set :c:member:`PyConfig.filesystem_encoding` to ``"mbcs"``, * Set :c:member:`PyConfig.filesystem_errors` to ``"replace"``. - Initialized the from :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment + Initialized from the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable value. Only available on Windows. ``#ifdef MS_WINDOWS`` macro can be used for @@ -360,7 +977,7 @@ PyPreConfig .. _c-preinit: Preinitialize Python with PyPreConfig -===================================== +------------------------------------- The preinitialization of Python: @@ -440,7 +1057,7 @@ the :ref:`Python UTF-8 Mode `:: PyConfig -======== +-------- .. c:type:: PyConfig @@ -509,7 +1126,7 @@ PyConfig The :c:func:`PyConfig_Read` function only parses :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` is set to ``2`` after arguments are parsed. Since Python arguments are - strippped from :c:member:`PyConfig.argv`, parsing arguments twice would + stripped from :c:member:`PyConfig.argv`, parsing arguments twice would parse the application options as Python options. :ref:`Preinitialize Python ` if needed. @@ -731,6 +1348,16 @@ PyConfig Default: ``0``. + .. c:member:: wchar_t* dump_refs_file + + Filename where to dump Python references. + + Set by the :envvar:`PYTHONDUMPREFSFILE` environment variable. + + Default: ``NULL``. + + .. versionadded:: 3.11 + .. c:member:: wchar_t* exec_prefix The site-specific directory prefix where the platform-dependent Python @@ -809,6 +1436,15 @@ PyConfig See also the :c:member:`~PyConfig.filesystem_encoding` member. + .. c:member:: int use_frozen_modules + + If non-zero, use frozen modules. + + Set by the :envvar:`PYTHON_FROZEN_MODULES` environment variable. + + Default: ``1`` in a release build, or ``0`` in a :ref:`debug build + `. + .. c:member:: unsigned long hash_seed .. c:member:: int use_hash_seed @@ -1041,7 +1677,7 @@ PyConfig The :c:func:`PyConfig_Read` function only parses :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` is set to ``2`` after arguments are parsed. Since Python arguments are - strippped from :c:member:`PyConfig.argv`, parsing arguments twice would + stripped from :c:member:`PyConfig.argv`, parsing arguments twice would parse the application options as Python options. Default: ``1`` in Python mode, ``0`` in isolated mode. @@ -1245,18 +1881,34 @@ PyConfig .. c:member:: int perf_profiling - Enable compatibility mode with the perf profiler? + Enable the Linux ``perf`` profiler support? + + If equals to ``1``, enable support for the Linux ``perf`` profiler. - If non-zero, initialize the perf trampoline. See :ref:`perf_profiling` - for more information. + If equals to ``2``, enable support for the Linux ``perf`` profiler with + DWARF JIT support. - Set by :option:`-X perf <-X>` command line option and by the + Set to ``1`` by :option:`-X perf <-X>` command-line option and the :envvar:`PYTHONPERFSUPPORT` environment variable. + Set to ``2`` by the :option:`-X perf_jit <-X>` command-line option and + the :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable. + Default: ``-1``. + .. seealso:: + See :ref:`perf_profiling` for more information. + .. versionadded:: 3.12 + .. c:member:: wchar_t* stdlib_dir + + Directory of the Python standard library. + + Default: ``NULL``. + + .. versionadded:: 3.11 + .. c:member:: int use_environment Use :ref:`environment variables `? @@ -1268,6 +1920,17 @@ PyConfig Default: ``1`` in Python config and ``0`` in isolated config. + .. c:member:: int use_system_logger + + If non-zero, ``stdout`` and ``stderr`` will be redirected to the system + log. + + Only available on macOS 10.12 and later, and on iOS. + + Default: ``0`` (don't use system log). + + .. versionadded:: 3.13.2 + .. c:member:: int user_site_directory If non-zero, add the user site directory to :data:`sys.path`. @@ -1332,6 +1995,15 @@ PyConfig Default: empty list. + .. c:member:: int _pystats + + If non-zero, write performance statistics at Python exit. + + Need a special build with the ``Py_STATS`` macro: + see :option:`--enable-pystats`. + + Default: ``0``. + If :c:member:`~PyConfig.parse_argv` is non-zero, :c:member:`~PyConfig.argv` arguments are parsed the same way the regular Python parses :ref:`command line arguments `, and Python arguments are stripped from @@ -1345,14 +2017,13 @@ the :option:`-X` command line option. The ``show_alloc_count`` field has been removed. -Initialization with PyConfig -============================ +.. _init-from-config: -Function to initialize Python: - -.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) +Initialization with PyConfig +---------------------------- - Initialize Python from *config* configuration. +Initializing the interpreter from a populated configuration struct is handled +by calling :c:func:`Py_InitializeFromConfig`. The caller is responsible to handle exceptions (error or exit) using :c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. @@ -1458,7 +2129,7 @@ initialization:: .. _init-isolated-conf: Isolated Configuration -====================== +---------------------- :c:func:`PyPreConfig_InitIsolatedConfig` and :c:func:`PyConfig_InitIsolatedConfig` functions create a configuration to @@ -1478,7 +2149,7 @@ to avoid computing the default path configuration. .. _init-python-config: Python Configuration -==================== +-------------------- :c:func:`PyPreConfig_InitPythonConfig` and :c:func:`PyConfig_InitPythonConfig` functions create a configuration to build a customized Python which behaves as @@ -1496,7 +2167,7 @@ and :ref:`Python UTF-8 Mode ` .. _init-path-config: Python Path Configuration -========================= +------------------------- :c:type:`PyConfig` contains multiple fields for the path configuration: @@ -1578,28 +2249,22 @@ If a ``._pth`` file is present: * Set :c:member:`~PyConfig.site_import` to ``0``. * Set :c:member:`~PyConfig.safe_path` to ``1``. +If :c:member:`~PyConfig.home` is not set and a ``pyvenv.cfg`` file is present in +the same directory as :c:member:`~PyConfig.executable`, or its parent, +:c:member:`~PyConfig.prefix` and :c:member:`~PyConfig.exec_prefix` are set that +location. When this happens, :c:member:`~PyConfig.base_prefix` and +:c:member:`~PyConfig.base_exec_prefix` still keep their value, pointing to the +base installation. See :ref:`sys-path-init-virtual-environments` for more +information. + The ``__PYVENV_LAUNCHER__`` environment variable is used to set :c:member:`PyConfig.base_executable`. +.. versionchanged:: 3.14 -Py_RunMain() -============ - -.. c:function:: int Py_RunMain(void) - - Execute the command (:c:member:`PyConfig.run_command`), the script - (:c:member:`PyConfig.run_filename`) or the module - (:c:member:`PyConfig.run_module`) specified on the command line or in the - configuration. - - By default and when if :option:`-i` option is used, run the REPL. - - Finally, finalizes Python and returns an exit status that can be passed to - the ``exit()`` function. - -See :ref:`Python Configuration ` for an example of -customized Python always running in isolated mode using -:c:func:`Py_RunMain`. + :c:member:`~PyConfig.prefix`, and :c:member:`~PyConfig.exec_prefix`, are now + set to the ``pyvenv.cfg`` directory. This was previously done by :mod:`site`, + therefore affected by :option:`-S`. Py_GetArgcArgv() @@ -1611,89 +2276,13 @@ Py_GetArgcArgv() See also :c:member:`PyConfig.orig_argv` member. +Delaying main module execution +============================== -Multi-Phase Initialization Private Provisional API -================================================== - -This section is a private provisional API introducing multi-phase -initialization, the core feature of :pep:`432`: - -* "Core" initialization phase, "bare minimum Python": - - * Builtin types; - * Builtin exceptions; - * Builtin and frozen modules; - * The :mod:`sys` module is only partially initialized - (ex: :data:`sys.path` doesn't exist yet). - -* "Main" initialization phase, Python is fully initialized: - - * Install and configure :mod:`importlib`; - * Apply the :ref:`Path Configuration `; - * Install signal handlers; - * Finish :mod:`sys` module initialization (ex: create :data:`sys.stdout` - and :data:`sys.path`); - * Enable optional features like :mod:`faulthandler` and :mod:`tracemalloc`; - * Import the :mod:`site` module; - * etc. - -Private provisional API: - -* :c:member:`PyConfig._init_main`: if set to ``0``, - :c:func:`Py_InitializeFromConfig` stops at the "Core" initialization phase. - -.. c:function:: PyStatus _Py_InitializeMain(void) - - Move to the "Main" initialization phase, finish the Python initialization. - -No module is imported during the "Core" phase and the ``importlib`` module is -not configured: the :ref:`Path Configuration ` is only -applied during the "Main" phase. It may allow to customize Python in Python to -override or tune the :ref:`Path Configuration `, maybe -install a custom :data:`sys.meta_path` importer or an import hook, etc. +In some embedding use cases, it may be desirable to separate interpreter initialization +from the execution of the main module. -It may become possible to calculate the :ref:`Path Configuration -` in Python, after the Core phase and before the Main phase, -which is one of the :pep:`432` motivation. - -The "Core" phase is not properly defined: what should be and what should -not be available at this phase is not specified yet. The API is marked -as private and provisional: the API can be modified or even be removed -anytime until a proper public API is designed. - -Example running Python code between "Core" and "Main" initialization -phases:: - - void init_python(void) - { - PyStatus status; - - PyConfig config; - PyConfig_InitPythonConfig(&config); - config._init_main = 0; - - /* ... customize 'config' configuration ... */ - - status = Py_InitializeFromConfig(&config); - PyConfig_Clear(&config); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); - } - - /* Use sys.stderr because sys.stdout is only created - by _Py_InitializeMain() */ - int res = PyRun_SimpleString( - "import sys; " - "print('Run Python code before _Py_InitializeMain', " - "file=sys.stderr)"); - if (res < 0) { - exit(1); - } - - /* ... put more configuration code here ... */ - - status = _Py_InitializeMain(); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); - } - } +This separation can be achieved by setting ``PyConfig.run_command`` to the empty +string during initialization (to prevent the interpreter from dropping into the +interactive prompt), and then subsequently executing the desired main module +code using ``__main__.__dict__`` as the global namespace. diff --git a/Doc/c-api/iter.rst b/Doc/c-api/iter.rst index 434d2021cea8e6..bf9df62c6f1706 100644 --- a/Doc/c-api/iter.rst +++ b/Doc/c-api/iter.rst @@ -10,7 +10,8 @@ There are two functions specifically for working with iterators. .. c:function:: int PyIter_Check(PyObject *o) Return non-zero if the object *o* can be safely passed to - :c:func:`PyIter_Next`, and ``0`` otherwise. This function always succeeds. + :c:func:`PyIter_NextItem` and ``0`` otherwise. + This function always succeeds. .. c:function:: int PyAIter_Check(PyObject *o) @@ -19,41 +20,27 @@ There are two functions specifically for working with iterators. .. versionadded:: 3.10 +.. c:function:: int PyIter_NextItem(PyObject *iter, PyObject **item) + + Return ``1`` and set *item* to a :term:`strong reference` of the + next value of the iterator *iter* on success. + Return ``0`` and set *item* to ``NULL`` if there are no remaining values. + Return ``-1``, set *item* to ``NULL`` and set an exception on error. + + .. versionadded:: 3.14 + .. c:function:: PyObject* PyIter_Next(PyObject *o) + This is an older version of :c:func:`!PyIter_NextItem`, + which is retained for backwards compatibility. + Prefer :c:func:`PyIter_NextItem`. + Return the next value from the iterator *o*. The object must be an iterator according to :c:func:`PyIter_Check` (it is up to the caller to check this). If there are no remaining values, returns ``NULL`` with no exception set. If an error occurs while retrieving the item, returns ``NULL`` and passes along the exception. -To write a loop which iterates over an iterator, the C code should look -something like this:: - - PyObject *iterator = PyObject_GetIter(obj); - PyObject *item; - - if (iterator == NULL) { - /* propagate error */ - } - - while ((item = PyIter_Next(iterator))) { - /* do something with item */ - ... - /* release reference when done */ - Py_DECREF(item); - } - - Py_DECREF(iterator); - - if (PyErr_Occurred()) { - /* propagate error */ - } - else { - /* continue doing useful work */ - } - - .. c:type:: PySendResult The enum value used to represent different results of :c:func:`PyIter_Send`. diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index 53eb54d3e1021a..758415a76e5cb4 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -38,9 +38,12 @@ List Objects .. note:: If *len* is greater than zero, the returned list object's items are - set to ``NULL``. Thus you cannot use abstract API functions such as - :c:func:`PySequence_SetItem` or expose the object to Python code before - setting all items to a real object with :c:func:`PyList_SetItem`. + set to ``NULL``. Thus you cannot use abstract API functions such as + :c:func:`PySequence_SetItem` or expose the object to Python code before + setting all items to a real object with :c:func:`PyList_SetItem` or + :c:func:`PyList_SET_ITEM()`. The following APIs are safe APIs before + the list is fully initialized: :c:func:`PyList_SetItem()` and :c:func:`PyList_SET_ITEM()`. + .. c:function:: Py_ssize_t PyList_Size(PyObject *list) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 582f5c7bf05471..25d9e62e387279 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -69,12 +69,32 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. on failure. +.. c:function:: PyObject* PyLong_FromInt32(int32_t value) + PyObject* PyLong_FromInt64(int64_t value) + + Return a new :c:type:`PyLongObject` object from a signed C + :c:expr:`int32_t` or :c:expr:`int64_t`, or ``NULL`` + with an exception set on failure. + + .. versionadded:: 3.14 + + .. c:function:: PyObject* PyLong_FromUnsignedLongLong(unsigned long long v) Return a new :c:type:`PyLongObject` object from a C :c:expr:`unsigned long long`, or ``NULL`` on failure. +.. c:function:: PyObject* PyLong_FromUInt32(uint32_t value) + PyObject* PyLong_FromUInt64(uint64_t value) + + Return a new :c:type:`PyLongObject` object from an unsigned C + :c:expr:`uint32_t` or :c:expr:`uint64_t`, or ``NULL`` + with an exception set on failure. + + .. versionadded:: 3.14 + + .. c:function:: PyObject* PyLong_FromDouble(double v) Return a new :c:type:`PyLongObject` object from the integer part of *v*, or @@ -94,9 +114,9 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. ignored. If there are no digits or *str* is not NULL-terminated following the digits and trailing whitespace, :exc:`ValueError` will be raised. - .. seealso:: Python methods :meth:`int.to_bytes` and :meth:`int.from_bytes` - to convert a :c:type:`PyLongObject` to/from an array of bytes in base - ``256``. You can call those from C using :c:func:`PyObject_CallMethod`. + .. seealso:: :c:func:`PyLong_AsNativeBytes()` and + :c:func:`PyLong_FromNativeBytes()` functions can be used to convert + a :c:type:`PyLongObject` to/from an array of bytes in base ``256``. .. c:function:: PyObject* PyLong_FromUnicodeObject(PyObject *u, int base) @@ -113,29 +133,32 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. retrieved from the resulting value using :c:func:`PyLong_AsVoidPtr`. -.. c:function:: PyObject* PyLong_FromNativeBytes(const void* buffer, size_t n_bytes, int endianness) +.. c:function:: PyObject* PyLong_FromNativeBytes(const void* buffer, size_t n_bytes, int flags) Create a Python integer from the value contained in the first *n_bytes* of *buffer*, interpreted as a two's-complement signed number. - *endianness* may be passed ``-1`` for the native endian that CPython was - compiled with, or else ``0`` for big endian and ``1`` for little. + *flags* are as for :c:func:`PyLong_AsNativeBytes`. Passing ``-1`` will select + the native endian that CPython was compiled with and assume that the + most-significant bit is a sign bit. Passing + ``Py_ASNATIVEBYTES_UNSIGNED_BUFFER`` will produce the same result as calling + :c:func:`PyLong_FromUnsignedNativeBytes`. Other flags are ignored. .. versionadded:: 3.13 -.. c:function:: PyObject* PyLong_FromUnsignedNativeBytes(const void* buffer, size_t n_bytes, int endianness) +.. c:function:: PyObject* PyLong_FromUnsignedNativeBytes(const void* buffer, size_t n_bytes, int flags) Create a Python integer from the value contained in the first *n_bytes* of *buffer*, interpreted as an unsigned number. - *endianness* may be passed ``-1`` for the native endian that CPython was - compiled with, or else ``0`` for big endian and ``1`` for little. + *flags* are as for :c:func:`PyLong_AsNativeBytes`. Passing ``-1`` will select + the native endian that CPython was compiled with and assume that the + most-significant bit is not a sign bit. Flags other than endian are ignored. .. versionadded:: 3.13 -.. XXX alias PyLong_AS_LONG (for now) .. c:function:: long PyLong_AsLong(PyObject *obj) .. index:: @@ -157,6 +180,16 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionchanged:: 3.10 This function will no longer use :meth:`~object.__int__`. + .. c:namespace:: NULL + + .. c:function:: long PyLong_AS_LONG(PyObject *obj) + + A :term:`soft deprecated` alias. + Exactly equivalent to the preferred ``PyLong_AsLong``. In particular, + it can fail with :exc:`OverflowError` or another exception. + + .. deprecated:: 3.14 + The function is soft deprecated. .. c:function:: int PyLong_AsInt(PyObject *obj) @@ -333,6 +366,43 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. This function will no longer use :meth:`~object.__int__`. +.. c:function:: int PyLong_AsInt32(PyObject *obj, int32_t *value) + int PyLong_AsInt64(PyObject *obj, int64_t *value) + + Set *\*value* to a signed C :c:expr:`int32_t` or :c:expr:`int64_t` + representation of *obj*. + + If the *obj* value is out of range, raise an :exc:`OverflowError`. + + Set *\*value* and return ``0`` on success. + Set an exception and return ``-1`` on error. + + *value* must not be ``NULL``. + + .. versionadded:: 3.14 + + +.. c:function:: int PyLong_AsUInt32(PyObject *obj, uint32_t *value) + int PyLong_AsUInt64(PyObject *obj, uint64_t *value) + + Set *\*value* to an unsigned C :c:expr:`uint32_t` or :c:expr:`uint64_t` + representation of *obj*. + + If *obj* is not an instance of :c:type:`PyLongObject`, first call its + :meth:`~object.__index__` method (if present) to convert it to a + :c:type:`PyLongObject`. + + * If *obj* is negative, raise a :exc:`ValueError`. + * If the *obj* value is out of range, raise an :exc:`OverflowError`. + + Set *\*value* and return ``0`` on success. + Set an exception and return ``-1`` on error. + + *value* must not be ``NULL``. + + .. versionadded:: 3.14 + + .. c:function:: double PyLong_AsDouble(PyObject *pylong) Return a C :c:expr:`double` representation of *pylong*. *pylong* must be @@ -354,14 +424,41 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Returns ``NULL`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. -.. c:function:: Py_ssize_t PyLong_AsNativeBytes(PyObject *pylong, void* buffer, Py_ssize_t n_bytes, int endianness) +.. c:function:: Py_ssize_t PyLong_AsNativeBytes(PyObject *pylong, void* buffer, Py_ssize_t n_bytes, int flags) + + Copy the Python integer value *pylong* to a native *buffer* of size + *n_bytes*. The *flags* can be set to ``-1`` to behave similarly to a C cast, + or to values documented below to control the behavior. + + Returns ``-1`` with an exception raised on error. This may happen if + *pylong* cannot be interpreted as an integer, or if *pylong* was negative + and the ``Py_ASNATIVEBYTES_REJECT_NEGATIVE`` flag was set. + + Otherwise, returns the number of bytes required to store the value. + If this is equal to or less than *n_bytes*, the entire value was copied. + All *n_bytes* of the buffer are written: large buffers are padded with + zeroes. + + If the returned value is greater than than *n_bytes*, the value was + truncated: as many of the lowest bits of the value as could fit are written, + and the higher bits are ignored. This matches the typical behavior + of a C-style downcast. + + .. note:: + + Overflow is not considered an error. If the returned value + is larger than *n_bytes*, most significant bits were discarded. + + ``0`` will never be returned. + + Values are always copied as two's-complement. - Copy the Python integer value to a native *buffer* of size *n_bytes*:: + Usage example:: int32_t value; - Py_ssize_t bytes = PyLong_AsNativeBits(pylong, &value, sizeof(value), -1); + Py_ssize_t bytes = PyLong_AsNativeBytes(pylong, &value, sizeof(value), -1); if (bytes < 0) { - // A Python exception was set with the reason. + // Failed. A Python exception was set with the reason. return NULL; } else if (bytes <= (Py_ssize_t)sizeof(value)) { @@ -372,19 +469,23 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. // lowest bits of pylong. } - The above example may look *similar* to - :c:func:`PyLong_As* ` - but instead fills in a specific caller defined type and never raises an - error about of the :class:`int` *pylong*'s value regardless of *n_bytes* - or the returned byte count. + Passing zero to *n_bytes* will return the size of a buffer that would + be large enough to hold the value. This may be larger than technically + necessary, but not unreasonably so. If *n_bytes=0*, *buffer* may be + ``NULL``. + + .. note:: + + Passing *n_bytes=0* to this function is not an accurate way to determine + the bit length of the value. - To get at the entire potentially big Python value, this can be used to - reserve enough space and copy it:: + To get at the entire Python value of an unknown size, the function can be + called twice: first to determine the buffer size, then to fill it:: // Ask how much space we need. - Py_ssize_t expected = PyLong_AsNativeBits(pylong, NULL, 0, -1); + Py_ssize_t expected = PyLong_AsNativeBytes(pylong, NULL, 0, -1); if (expected < 0) { - // A Python exception was set with the reason. + // Failed. A Python exception was set with the reason. return NULL; } assert(expected != 0); // Impossible per the API definition. @@ -394,12 +495,12 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. return NULL; } // Safely get the entire value. - Py_ssize_t bytes = PyLong_AsNativeBits(pylong, bignum, expected, -1); - if (bytes < 0) { // Exception set. + Py_ssize_t bytes = PyLong_AsNativeBytes(pylong, bignum, expected, -1); + if (bytes < 0) { // Exception has been set. free(bignum); return NULL; } - else if (bytes > expected) { // Be safe, should not be possible. + else if (bytes > expected) { // This should not be possible. PyErr_SetString(PyExc_RuntimeError, "Unexpected bignum truncation after a size check."); free(bignum); @@ -409,37 +510,120 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. // ... use bignum ... free(bignum); - *endianness* may be passed ``-1`` for the native endian that CPython was - compiled with, or ``0`` for big endian and ``1`` for little. + *flags* is either ``-1`` (``Py_ASNATIVEBYTES_DEFAULTS``) to select defaults + that behave most like a C cast, or a combination of the other flags in + the table below. + Note that ``-1`` cannot be combined with other flags. + + Currently, ``-1`` corresponds to + ``Py_ASNATIVEBYTES_NATIVE_ENDIAN | Py_ASNATIVEBYTES_UNSIGNED_BUFFER``. + + .. c:namespace:: NULL + + ============================================= ====== + Flag Value + ============================================= ====== + .. c:macro:: Py_ASNATIVEBYTES_DEFAULTS ``-1`` + .. c:macro:: Py_ASNATIVEBYTES_BIG_ENDIAN ``0`` + .. c:macro:: Py_ASNATIVEBYTES_LITTLE_ENDIAN ``1`` + .. c:macro:: Py_ASNATIVEBYTES_NATIVE_ENDIAN ``3`` + .. c:macro:: Py_ASNATIVEBYTES_UNSIGNED_BUFFER ``4`` + .. c:macro:: Py_ASNATIVEBYTES_REJECT_NEGATIVE ``8`` + .. c:macro:: Py_ASNATIVEBYTES_ALLOW_INDEX ``16`` + ============================================= ====== + + Specifying ``Py_ASNATIVEBYTES_NATIVE_ENDIAN`` will override any other endian + flags. Passing ``2`` is reserved. + + By default, sufficient buffer will be requested to include a sign bit. + For example, when converting 128 with *n_bytes=1*, the function will return + 2 (or more) in order to store a zero sign bit. + + If ``Py_ASNATIVEBYTES_UNSIGNED_BUFFER`` is specified, a zero sign bit + will be omitted from size calculations. This allows, for example, 128 to fit + in a single-byte buffer. If the destination buffer is later treated as + signed, a positive input value may become negative. + Note that the flag does not affect handling of negative values: for those, + space for a sign bit is always requested. + + Specifying ``Py_ASNATIVEBYTES_REJECT_NEGATIVE`` causes an exception to be set + if *pylong* is negative. Without this flag, negative values will be copied + provided there is enough space for at least one sign bit, regardless of + whether ``Py_ASNATIVEBYTES_UNSIGNED_BUFFER`` was specified. + + If ``Py_ASNATIVEBYTES_ALLOW_INDEX`` is specified and a non-integer value is + passed, its :meth:`~object.__index__` method will be called first. This may + result in Python code executing and other threads being allowed to run, which + could cause changes to other objects or values in use. When *flags* is + ``-1``, this option is not set, and non-integer values will raise + :exc:`TypeError`. - Returns ``-1`` with an exception raised if *pylong* cannot be interpreted as - an integer. Otherwise, return the size of the buffer required to store the - value. If this is equal to or less than *n_bytes*, the entire value was - copied. ``0`` will never be returned. + .. note:: - Unless an exception is raised, all *n_bytes* of the buffer will always be - written. In the case of truncation, as many of the lowest bits of the value - as could fit are written. This allows the caller to ignore all non-negative - results if the intent is to match the typical behavior of a C-style - downcast. No exception is set on truncation. + With the default *flags* (``-1``, or *UNSIGNED_BUFFER* without + *REJECT_NEGATIVE*), multiple Python integers can map to a single value + without overflow. For example, both ``255`` and ``-1`` fit a single-byte + buffer and set all its bits. + This matches typical C cast behavior. - Values are always copied as two's-complement and sufficient buffer will be - requested to include a sign bit. For example, this may cause an value that - fits into 8 bytes when treated as unsigned to request 9 bytes, even though - all eight bytes were copied into the buffer. What has been omitted is the - zero sign bit -- redundant if the caller's intention is to treat the value - as unsigned. + .. versionadded:: 3.13 - Passing zero to *n_bytes* will return the size of a buffer that would - be large enough to hold the value. This may be larger than technically - necessary, but not unreasonably so. - .. note:: +.. c:function:: int PyLong_GetSign(PyObject *obj, int *sign) - Passing *n_bytes=0* to this function is not an accurate way to determine - the bit length of a value. + Get the sign of the integer object *obj*. - .. versionadded:: 3.13 + On success, set *\*sign* to the integer sign (0, -1 or +1 for zero, negative or + positive integer, respectively) and return 0. + + On failure, return -1 with an exception set. This function always succeeds + if *obj* is a :c:type:`PyLongObject` or its subtype. + + .. versionadded:: 3.14 + + +.. c:function:: int PyLong_IsPositive(PyObject *obj) + + Check if the integer object *obj* is positive (``obj > 0``). + + If *obj* is an instance of :c:type:`PyLongObject` or its subtype, + return ``1`` when it's positive and ``0`` otherwise. Else set an + exception and return ``-1``. + + .. versionadded:: 3.14 + + +.. c:function:: int PyLong_IsNegative(PyObject *obj) + + Check if the integer object *obj* is negative (``obj < 0``). + + If *obj* is an instance of :c:type:`PyLongObject` or its subtype, + return ``1`` when it's negative and ``0`` otherwise. Else set an + exception and return ``-1``. + + .. versionadded:: 3.14 + + +.. c:function:: int PyLong_IsZero(PyObject *obj) + + Check if the integer object *obj* is zero. + + If *obj* is an instance of :c:type:`PyLongObject` or its subtype, + return ``1`` when it's zero and ``0`` otherwise. Else set an + exception and return ``-1``. + + .. versionadded:: 3.14 + + +.. c:function:: PyObject* PyLong_GetInfo(void) + + On success, return a read only :term:`named tuple`, that holds + information about Python's internal representation of integers. + See :data:`sys.int_info` for description of individual fields. + + On failure, return ``NULL`` with an exception set. + + .. versionadded:: 3.1 .. c:function:: int PyUnstable_Long_IsCompact(const PyLongObject* op) @@ -450,13 +634,16 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. a “fast path” for small integers. For compact values use :c:func:`PyUnstable_Long_CompactValue`; for others fall back to a :c:func:`PyLong_As* ` function or - :c:func:`calling ` :meth:`int.to_bytes`. + :c:func:`PyLong_AsNativeBytes`. The speedup is expected to be negligible for most users. Exactly what values are considered compact is an implementation detail and is subject to change. + .. versionadded:: 3.12 + + .. c:function:: Py_ssize_t PyUnstable_Long_CompactValue(const PyLongObject* op) If *op* is compact, as determined by :c:func:`PyUnstable_Long_IsCompact`, @@ -464,3 +651,179 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Otherwise, the return value is undefined. + .. versionadded:: 3.12 + + +Export API +^^^^^^^^^^ + +.. versionadded:: 3.14 + +.. c:struct:: PyLongLayout + + Layout of an array of "digits" ("limbs" in the GMP terminology), used to + represent absolute value for arbitrary precision integers. + + Use :c:func:`PyLong_GetNativeLayout` to get the native layout of Python + :class:`int` objects, used internally for integers with "big enough" + absolute value. + + See also :data:`sys.int_info` which exposes similar information in Python. + + .. c:member:: uint8_t bits_per_digit + + Bits per digit. For example, a 15 bit digit means that bits 0-14 contain + meaningful information. + + .. c:member:: uint8_t digit_size + + Digit size in bytes. For example, a 15 bit digit will require at least 2 + bytes. + + .. c:member:: int8_t digits_order + + Digits order: + + - ``1`` for most significant digit first + - ``-1`` for least significant digit first + + .. c:member:: int8_t digit_endianness + + Digit endianness: + + - ``1`` for most significant byte first (big endian) + - ``-1`` for least significant byte first (little endian) + + +.. c:function:: const PyLongLayout* PyLong_GetNativeLayout(void) + + Get the native layout of Python :class:`int` objects. + + See the :c:struct:`PyLongLayout` structure. + + The function must not be called before Python initialization nor after + Python finalization. The returned layout is valid until Python is + finalized. The layout is the same for all Python sub-interpreters + in a process, and so it can be cached. + + +.. c:struct:: PyLongExport + + Export of a Python :class:`int` object. + + There are two cases: + + * If :c:member:`digits` is ``NULL``, only use the :c:member:`value` member. + * If :c:member:`digits` is not ``NULL``, use :c:member:`negative`, + :c:member:`ndigits` and :c:member:`digits` members. + + .. c:member:: int64_t value + + The native integer value of the exported :class:`int` object. + Only valid if :c:member:`digits` is ``NULL``. + + .. c:member:: uint8_t negative + + ``1`` if the number is negative, ``0`` otherwise. + Only valid if :c:member:`digits` is not ``NULL``. + + .. c:member:: Py_ssize_t ndigits + + Number of digits in :c:member:`digits` array. + Only valid if :c:member:`digits` is not ``NULL``. + + .. c:member:: const void *digits + + Read-only array of unsigned digits. Can be ``NULL``. + + +.. c:function:: int PyLong_Export(PyObject *obj, PyLongExport *export_long) + + Export a Python :class:`int` object. + + *export_long* must point to a :c:struct:`PyLongExport` structure allocated + by the caller. It must not be ``NULL``. + + On success, fill in *\*export_long* and return ``0``. + On error, set an exception and return ``-1``. + + :c:func:`PyLong_FreeExport` must be called when the export is no longer + needed. + + .. impl-detail:: + This function always succeeds if *obj* is a Python :class:`int` object + or a subclass. + + +.. c:function:: void PyLong_FreeExport(PyLongExport *export_long) + + Release the export *export_long* created by :c:func:`PyLong_Export`. + + .. impl-detail:: + Calling :c:func:`PyLong_FreeExport` is optional if *export_long->digits* + is ``NULL``. + + +PyLongWriter API +^^^^^^^^^^^^^^^^ + +The :c:type:`PyLongWriter` API can be used to import an integer. + +.. versionadded:: 3.14 + +.. c:struct:: PyLongWriter + + A Python :class:`int` writer instance. + + The instance must be destroyed by :c:func:`PyLongWriter_Finish` or + :c:func:`PyLongWriter_Discard`. + + +.. c:function:: PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits) + + Create a :c:type:`PyLongWriter`. + + On success, allocate *\*digits* and return a writer. + On error, set an exception and return ``NULL``. + + *negative* is ``1`` if the number is negative, or ``0`` otherwise. + + *ndigits* is the number of digits in the *digits* array. It must be + greater than 0. + + *digits* must not be NULL. + + After a successful call to this function, the caller should fill in the + array of digits *digits* and then call :c:func:`PyLongWriter_Finish` to get + a Python :class:`int`. + The layout of *digits* is described by :c:func:`PyLong_GetNativeLayout`. + + Digits must be in the range [``0``; ``(1 << bits_per_digit) - 1``] + (where the :c:struct:`~PyLongLayout.bits_per_digit` is the number of bits + per digit). + Any unused most significant digits must be set to ``0``. + + Alternately, call :c:func:`PyLongWriter_Discard` to destroy the writer + instance without creating an :class:`~int` object. + + +.. c:function:: PyObject* PyLongWriter_Finish(PyLongWriter *writer) + + Finish a :c:type:`PyLongWriter` created by :c:func:`PyLongWriter_Create`. + + On success, return a Python :class:`int` object. + On error, set an exception and return ``NULL``. + + The function takes care of normalizing the digits and converts the object + to a compact integer if needed. + + The writer instance and the *digits* array are invalid after the call. + + +.. c:function:: void PyLongWriter_Discard(PyLongWriter *writer) + + Discard a :c:type:`PyLongWriter` created by :c:func:`PyLongWriter_Create`. + + If *writer* is ``NULL``, no operation is performed. + + The writer instance and the *digits* array are invalid after the call. diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index 489f1580a414b2..61218a1bf6f171 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -13,11 +13,12 @@ binary mode. Numeric values are stored with the least significant byte first. -The module supports two versions of the data format: version 0 is the -historical version, version 1 shares interned strings in the file, and upon -unmarshalling. Version 2 uses a binary format for floating point numbers. -``Py_MARSHAL_VERSION`` indicates the current file format (currently 2). +The module supports several versions of the data format; see +the :py:mod:`Python module documentation ` for details. +.. c:macro:: Py_MARSHAL_VERSION + + The current format version. See :py:data:`marshal.version`. .. c:function:: void PyMarshal_WriteLongToFile(long value, FILE *file, int version) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 9da09a21607f61..aa2ef499bddaf3 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -102,30 +102,38 @@ All allocating functions belong to one of three different "domains" (see also strategies and are optimized for different purposes. The specific details on how every domain allocates memory or what internal functions each domain calls is considered an implementation detail, but for debugging purposes a simplified -table can be found at :ref:`here `. There is no hard -requirement to use the memory returned by the allocation functions belonging to -a given domain for only the purposes hinted by that domain (although this is the -recommended practice). For example, one could use the memory returned by -:c:func:`PyMem_RawMalloc` for allocating Python objects or the memory returned -by :c:func:`PyObject_Malloc` for allocating memory for buffers. +table can be found at :ref:`here `. +The APIs used to allocate and free a block of memory must be from the same domain. +For example, :c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. The three allocation domains are: * Raw domain: intended for allocating memory for general-purpose memory buffers where the allocation *must* go to the system allocator or where the allocator can operate without the :term:`GIL`. The memory is requested directly - to the system. + from the system. See :ref:`Raw Memory Interface `. * "Mem" domain: intended for allocating memory for Python buffers and general-purpose memory buffers where the allocation must be performed with the :term:`GIL` held. The memory is taken from the Python private heap. + See :ref:`Memory Interface `. -* Object domain: intended for allocating memory belonging to Python objects. The - memory is taken from the Python private heap. +* Object domain: intended for allocating memory for Python objects. The + memory is taken from the Python private heap. See :ref:`Object allocators `. -When freeing memory previously allocated by the allocating functions belonging to a -given domain,the matching specific deallocating functions must be used. For example, -:c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. +.. note:: + + The :term:`free-threaded ` build requires that only Python objects are allocated using the "object" domain + and that all Python objects are allocated using that domain. This differs from the prior Python versions, + where this was only a best practice and not a hard requirement. + + For example, buffers (non-Python objects) should be allocated using :c:func:`PyMem_Malloc`, + :c:func:`PyMem_RawMalloc`, or :c:func:`malloc`, but not :c:func:`PyObject_Malloc`. + + See :ref:`Memory Allocation APIs `. + + +.. _raw-memoryinterface: Raw Memory Interface ==================== @@ -299,6 +307,8 @@ versions and is therefore deprecated in extension modules. * ``PyMem_DEL(ptr)`` +.. _objectinterface: + Object allocators ================= @@ -734,7 +744,7 @@ The same code using the type-oriented function set:: return PyErr_NoMemory(); /* ...Do some I/O operation involving buf... */ res = PyBytes_FromString(buf); - PyMem_Del(buf); /* allocated with PyMem_New */ + PyMem_Free(buf); /* allocated with PyMem_New */ return res; Note that in the two examples above, the buffer is always manipulated via @@ -750,11 +760,11 @@ allocators operating on different heaps. :: ... PyMem_Del(buf3); /* Wrong -- should be PyMem_Free() */ free(buf2); /* Right -- allocated via malloc() */ - free(buf1); /* Fatal -- should be PyMem_Del() */ + free(buf1); /* Fatal -- should be PyMem_Free() */ In addition to the functions aimed at handling raw memory blocks from the Python heap, objects in Python are allocated and released with :c:macro:`PyObject_New`, -:c:macro:`PyObject_NewVar` and :c:func:`PyObject_Del`. +:c:macro:`PyObject_NewVar` and :c:func:`PyObject_Free`. These will be explained in the next chapter on defining and implementing new object types in C. diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 979b22261efa3b..f82a050ab75de0 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -37,16 +37,19 @@ Module Objects single: __package__ (module attribute) single: __loader__ (module attribute) - Return a new module object with the :attr:`__name__` attribute set to *name*. - The module's :attr:`__name__`, :attr:`__doc__`, :attr:`__package__`, and - :attr:`__loader__` attributes are filled in (all but :attr:`__name__` are set - to ``None``); the caller is responsible for providing a :attr:`__file__` - attribute. + Return a new module object with :attr:`module.__name__` set to *name*. + The module's :attr:`!__name__`, :attr:`~module.__doc__`, + :attr:`~module.__package__` and :attr:`~module.__loader__` attributes are + filled in (all but :attr:`!__name__` are set to ``None``). The caller is + responsible for setting a :attr:`~module.__file__` attribute. + + Return ``NULL`` with an exception set on error. .. versionadded:: 3.3 .. versionchanged:: 3.4 - :attr:`__package__` and :attr:`__loader__` are set to ``None``. + :attr:`~module.__package__` and :attr:`~module.__loader__` are now set to + ``None``. .. c:function:: PyObject* PyModule_New(const char *name) @@ -75,8 +78,9 @@ Module Objects single: __name__ (module attribute) single: SystemError (built-in exception) - Return *module*'s :attr:`__name__` value. If the module does not provide one, - or if it is not a string, :exc:`SystemError` is raised and ``NULL`` is returned. + Return *module*'s :attr:`~module.__name__` value. If the module does not + provide one, or if it is not a string, :exc:`SystemError` is raised and + ``NULL`` is returned. .. versionadded:: 3.3 @@ -106,8 +110,8 @@ Module Objects single: SystemError (built-in exception) Return the name of the file from which *module* was loaded using *module*'s - :attr:`__file__` attribute. If this is not defined, or if it is not a - unicode string, raise :exc:`SystemError` and return ``NULL``; otherwise return + :attr:`~module.__file__` attribute. If this is not defined, or if it is not a + string, raise :exc:`SystemError` and return ``NULL``; otherwise return a reference to a Unicode object. .. versionadded:: 3.2 @@ -265,6 +269,8 @@ of the following two module creation functions: API version *module_api_version*. If that version does not match the version of the running interpreter, a :exc:`RuntimeWarning` is emitted. + Return ``NULL`` with an exception set on error. + .. note:: Most uses of this function should be using :c:func:`PyModule_Create` @@ -338,7 +344,8 @@ The available slot types are: The *value* pointer of this slot must point to a function of the signature: .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def) - :noindex: + :no-index-entry: + :no-contents-entry: The function receives a :py:class:`~importlib.machinery.ModuleSpec` instance, as defined in :PEP:`451`, and the module definition. @@ -373,7 +380,8 @@ The available slot types are: The signature of the function is: .. c:function:: int exec_module(PyObject* module) - :noindex: + :no-index-entry: + :no-contents-entry: If multiple ``Py_mod_exec`` slots are specified, they are processed in the order they appear in the *m_slots* array. @@ -411,6 +419,33 @@ The available slot types are: .. versionadded:: 3.12 +.. c:macro:: Py_mod_gil + + Specifies one of the following values: + + .. c:namespace:: NULL + + .. c:macro:: Py_MOD_GIL_USED + + The module depends on the presence of the global interpreter lock (GIL), + and may access global state without synchronization. + + .. c:macro:: Py_MOD_GIL_NOT_USED + + The module is safe to run without an active GIL. + + This slot is ignored by Python builds not configured with + :option:`--disable-gil`. Otherwise, it determines whether or not importing + this module will cause the GIL to be automatically enabled. See + :ref:`whatsnew313-free-threaded-cpython` for more detail. + + Multiple ``Py_mod_gil`` slots may not be specified in one module definition. + + If ``Py_mod_gil`` is not specified, the import machinery defaults to + ``Py_MOD_GIL_USED``. + + .. versionadded:: 3.13 + See :PEP:`489` for more details on multi-phase initialization. Low-level module creation functions @@ -436,6 +471,8 @@ objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and If that version does not match the version of the running interpreter, a :exc:`RuntimeWarning` is emitted. + Return ``NULL`` with an exception set on error. + .. note:: Most uses of this function should be using :c:func:`PyModule_FromDefAndSpec` @@ -486,7 +523,7 @@ state: On success, return ``0``. On error, raise an exception and return ``-1``. - Return ``NULL`` if *value* is ``NULL``. It must be called with an exception + Return ``-1`` if *value* is ``NULL``. It must be called with an exception raised in this case. Example usage:: @@ -518,6 +555,14 @@ state: Note that ``Py_XDECREF()`` should be used instead of ``Py_DECREF()`` in this case, since *obj* can be ``NULL``. + The number of different *name* strings passed to this function + should be kept small, usually by only using statically allocated strings + as *name*. + For names that aren't known at compile time, prefer calling + :c:func:`PyUnicode_FromString` and :c:func:`PyObject_SetAttr` directly. + For more details, see :c:func:`PyUnicode_InternFromString`, which may be + used internally to create a key object. + .. versionadded:: 3.10 @@ -576,15 +621,23 @@ state: .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) Add an integer constant to *module* as *name*. This convenience function can be - used from the module's initialization function. Return ``-1`` on error, ``0`` on - success. + used from the module's initialization function. + Return ``-1`` with an exception set on error, ``0`` on success. + + This is a convenience function that calls :c:func:`PyLong_FromLong` and + :c:func:`PyModule_AddObjectRef`; see their documentation for details. .. c:function:: int PyModule_AddStringConstant(PyObject *module, const char *name, const char *value) Add a string constant to *module* as *name*. This convenience function can be used from the module's initialization function. The string *value* must be - ``NULL``-terminated. Return ``-1`` on error, ``0`` on success. + ``NULL``-terminated. + Return ``-1`` with an exception set on error, ``0`` on success. + + This is a convenience function that calls + :c:func:`PyUnicode_InternFromString` and :c:func:`PyModule_AddObjectRef`; + see their documentation for details. .. c:macro:: PyModule_AddIntMacro(module, macro) @@ -592,7 +645,7 @@ state: Add an int constant to *module*. The name and the value are taken from *macro*. For example ``PyModule_AddIntMacro(module, AF_INET)`` adds the int constant *AF_INET* with the value of *AF_INET* to *module*. - Return ``-1`` on error, ``0`` on success. + Return ``-1`` with an exception set on error, ``0`` on success. .. c:macro:: PyModule_AddStringMacro(module, macro) @@ -605,10 +658,23 @@ state: The type object is finalized by calling internally :c:func:`PyType_Ready`. The name of the type object is taken from the last component of :c:member:`~PyTypeObject.tp_name` after dot. - Return ``-1`` on error, ``0`` on success. + Return ``-1`` with an exception set on error, ``0`` on success. .. versionadded:: 3.9 +.. c:function:: int PyUnstable_Module_SetGIL(PyObject *module, void *gil) + + Indicate that *module* does or does not support running without the global + interpreter lock (GIL), using one of the values from + :c:macro:`Py_mod_gil`. It must be called during *module*'s initialization + function. If this function is not called during module initialization, the + import machinery assumes the module does not support running without the + GIL. This function is only available in Python builds configured with + :option:`--disable-gil`. + Return ``-1`` with an exception set on error, ``0`` on success. + + .. versionadded:: 3.13 + Module lookup ^^^^^^^^^^^^^ @@ -644,14 +710,14 @@ since multiple such modules can be created from a single definition. The caller must hold the GIL. - Return 0 on success or -1 on failure. + Return ``-1`` with an exception set on error, ``0`` on success. .. versionadded:: 3.3 .. c:function:: int PyState_RemoveModule(PyModuleDef *def) Removes the module object created from *def* from the interpreter state. - Return 0 on success or -1 on failure. + Return ``-1`` with an exception set on error, ``0`` on success. The caller must hold the GIL. diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst new file mode 100644 index 00000000000000..bda6cd271197d0 --- /dev/null +++ b/Doc/c-api/monitoring.rst @@ -0,0 +1,198 @@ +.. highlight:: c + +.. _c-api-monitoring: + +Monitoring C API +================ + +Added in version 3.13. + +An extension may need to interact with the event monitoring system. Subscribing +to events and registering callbacks can be done via the Python API exposed in +:mod:`sys.monitoring`. + +Generating Execution Events +=========================== + +The functions below make it possible for an extension to fire monitoring +events as it emulates the execution of Python code. Each of these functions +accepts a ``PyMonitoringState`` struct which contains concise information +about the activation state of events, as well as the event arguments, which +include a ``PyObject*`` representing the code object, the instruction offset +and sometimes additional, event-specific arguments (see :mod:`sys.monitoring` +for details about the signatures of the different event callbacks). +The ``codelike`` argument should be an instance of :class:`types.CodeType` +or of a type that emulates it. + +The VM disables tracing when firing an event, so there is no need for user +code to do that. + +Monitoring functions should not be called with an exception set, +except those listed below as working with the current exception. + +.. c:type:: PyMonitoringState + + Representation of the state of an event type. It is allocated by the user + while its contents are maintained by the monitoring API functions described below. + + +All of the functions below return 0 on success and -1 (with an exception set) on error. + +See :mod:`sys.monitoring` for descriptions of the events. + +.. c:function:: int PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_START`` event. + + +.. c:function:: int PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_RESUME`` event. + + +.. c:function:: int PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject* retval) + + Fire a ``PY_RETURN`` event. + + +.. c:function:: int PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject* retval) + + Fire a ``PY_YIELD`` event. + + +.. c:function:: int PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject* callable, PyObject *arg0) + + Fire a ``CALL`` event. + + +.. c:function:: int PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, int lineno) + + Fire a ``LINE`` event. + + +.. c:function:: int PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) + + Fire a ``JUMP`` event. + + +.. c:function:: int PyMonitoring_FireBranchLeftEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) + + Fire a ``BRANCH_LEFT`` event. + + +.. c:function:: int PyMonitoring_FireBranchRightEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) + + Fire a ``BRANCH_RIGHT`` event. + + +.. c:function:: int PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *retval) + + Fire a ``C_RETURN`` event. + + +.. c:function:: int PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_THROW`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``RAISE`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``C_RAISE`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``RERAISE`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire an ``EXCEPTION_HANDLED`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_UNWIND`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value) + + Fire a ``STOP_ITERATION`` event. If ``value`` is an instance of :exc:`StopIteration`, it is used. Otherwise, + a new :exc:`StopIteration` instance is created with ``value`` as its argument. + + +Managing the Monitoring State +----------------------------- + +Monitoring states can be managed with the help of monitoring scopes. A scope +would typically correspond to a python function. + +.. c:function:: int PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, const uint8_t *event_types, Py_ssize_t length) + + Enter a monitored scope. ``event_types`` is an array of the event IDs for + events that may be fired from the scope. For example, the ID of a ``PY_START`` + event is the value ``PY_MONITORING_EVENT_PY_START``, which is numerically equal + to the base-2 logarithm of ``sys.monitoring.events.PY_START``. + ``state_array`` is an array with a monitoring state entry for each event in + ``event_types``, it is allocated by the user but populated by + :c:func:`!PyMonitoring_EnterScope` with information about the activation state of + the event. The size of ``event_types`` (and hence also of ``state_array``) + is given in ``length``. + + The ``version`` argument is a pointer to a value which should be allocated + by the user together with ``state_array`` and initialized to 0, + and then set only by :c:func:`!PyMonitoring_EnterScope` itself. It allows this + function to determine whether event states have changed since the previous call, + and to return quickly if they have not. + + The scopes referred to here are lexical scopes: a function, class or method. + :c:func:`!PyMonitoring_EnterScope` should be called whenever the lexical scope is + entered. Scopes can be reentered, reusing the same *state_array* and *version*, + in situations like when emulating a recursive Python function. When a code-like's + execution is paused, such as when emulating a generator, the scope needs to + be exited and re-entered. + + The macros for *event_types* are: + + .. c:namespace:: NULL + + .. The table is here to make the docs searchable, and to allow automatic + links to the identifiers. + + ================================================== ===================================== + Macro Event + ================================================== ===================================== + .. c:macro:: PY_MONITORING_EVENT_BRANCH_LEFT :monitoring-event:`BRANCH_LEFT` + .. c:macro:: PY_MONITORING_EVENT_BRANCH_RIGHT :monitoring-event:`BRANCH_RIGHT` + .. c:macro:: PY_MONITORING_EVENT_CALL :monitoring-event:`CALL` + .. c:macro:: PY_MONITORING_EVENT_C_RAISE :monitoring-event:`C_RAISE` + .. c:macro:: PY_MONITORING_EVENT_C_RETURN :monitoring-event:`C_RETURN` + .. c:macro:: PY_MONITORING_EVENT_EXCEPTION_HANDLED :monitoring-event:`EXCEPTION_HANDLED` + .. c:macro:: PY_MONITORING_EVENT_INSTRUCTION :monitoring-event:`INSTRUCTION` + .. c:macro:: PY_MONITORING_EVENT_JUMP :monitoring-event:`JUMP` + .. c:macro:: PY_MONITORING_EVENT_LINE :monitoring-event:`LINE` + .. c:macro:: PY_MONITORING_EVENT_PY_RESUME :monitoring-event:`PY_RESUME` + .. c:macro:: PY_MONITORING_EVENT_PY_RETURN :monitoring-event:`PY_RETURN` + .. c:macro:: PY_MONITORING_EVENT_PY_START :monitoring-event:`PY_START` + .. c:macro:: PY_MONITORING_EVENT_PY_THROW :monitoring-event:`PY_THROW` + .. c:macro:: PY_MONITORING_EVENT_PY_UNWIND :monitoring-event:`PY_UNWIND` + .. c:macro:: PY_MONITORING_EVENT_PY_YIELD :monitoring-event:`PY_YIELD` + .. c:macro:: PY_MONITORING_EVENT_RAISE :monitoring-event:`RAISE` + .. c:macro:: PY_MONITORING_EVENT_RERAISE :monitoring-event:`RERAISE` + .. c:macro:: PY_MONITORING_EVENT_STOP_ITERATION :monitoring-event:`STOP_ITERATION` + ================================================== ===================================== + +.. c:function:: int PyMonitoring_ExitScope(void) + + Exit the last scope that was entered with :c:func:`!PyMonitoring_EnterScope`. diff --git a/Doc/c-api/number.rst b/Doc/c-api/number.rst index 13d3c5af956905..ad8b5935258fa7 100644 --- a/Doc/c-api/number.rst +++ b/Doc/c-api/number.rst @@ -51,8 +51,8 @@ Number Protocol Return a reasonable approximation for the mathematical value of *o1* divided by *o2*, or ``NULL`` on failure. The return value is "approximate" because binary - floating point numbers are approximate; it is not possible to represent all real - numbers in base two. This function can return a floating point value when + floating-point numbers are approximate; it is not possible to represent all real + numbers in base two. This function can return a floating-point value when passed two integers. This is the equivalent of the Python expression ``o1 / o2``. @@ -177,8 +177,8 @@ Number Protocol Return a reasonable approximation for the mathematical value of *o1* divided by *o2*, or ``NULL`` on failure. The return value is "approximate" because binary - floating point numbers are approximate; it is not possible to represent all real - numbers in base two. This function can return a floating point value when + floating-point numbers are approximate; it is not possible to represent all real + numbers in base two. This function can return a floating-point value when passed two integers. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 /= o2``. diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 12476412799a4f..1ba5942c63601d 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -6,6 +6,56 @@ Object Protocol =============== +.. c:function:: PyObject* Py_GetConstant(unsigned int constant_id) + + Get a :term:`strong reference` to a constant. + + Set an exception and return ``NULL`` if *constant_id* is invalid. + + *constant_id* must be one of these constant identifiers: + + .. c:namespace:: NULL + + ======================================== ===== ========================= + Constant Identifier Value Returned object + ======================================== ===== ========================= + .. c:macro:: Py_CONSTANT_NONE ``0`` :py:data:`None` + .. c:macro:: Py_CONSTANT_FALSE ``1`` :py:data:`False` + .. c:macro:: Py_CONSTANT_TRUE ``2`` :py:data:`True` + .. c:macro:: Py_CONSTANT_ELLIPSIS ``3`` :py:data:`Ellipsis` + .. c:macro:: Py_CONSTANT_NOT_IMPLEMENTED ``4`` :py:data:`NotImplemented` + .. c:macro:: Py_CONSTANT_ZERO ``5`` ``0`` + .. c:macro:: Py_CONSTANT_ONE ``6`` ``1`` + .. c:macro:: Py_CONSTANT_EMPTY_STR ``7`` ``''`` + .. c:macro:: Py_CONSTANT_EMPTY_BYTES ``8`` ``b''`` + .. c:macro:: Py_CONSTANT_EMPTY_TUPLE ``9`` ``()`` + ======================================== ===== ========================= + + Numeric values are only given for projects which cannot use the constant + identifiers. + + + .. versionadded:: 3.13 + + .. impl-detail:: + + In CPython, all of these constants are :term:`immortal`. + + +.. c:function:: PyObject* Py_GetConstantBorrowed(unsigned int constant_id) + + Similar to :c:func:`Py_GetConstant`, but return a :term:`borrowed + reference`. + + This function is primarily intended for backwards compatibility: + using :c:func:`Py_GetConstant` is recommended for new code. + + The reference is borrowed from the interpreter, and is valid until the + interpreter finalization. + + .. versionadded:: 3.13 + + .. c:var:: PyObject* Py_NotImplemented The ``NotImplemented`` singleton, used to signal that an operation is @@ -16,7 +66,7 @@ Object Protocol Properly handle returning :c:data:`Py_NotImplemented` from within a C function (that is, create a new :term:`strong reference` - to NotImplemented and return it). + to :const:`NotImplemented` and return it). .. c:macro:: Py_PRINT_RAW @@ -35,7 +85,7 @@ Object Protocol instead of the :func:`repr`. -.. c:function:: int PyObject_HasAttrWithError(PyObject *o, const char *attr_name) +.. c:function:: int PyObject_HasAttrWithError(PyObject *o, PyObject *attr_name) Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. This is equivalent to the Python expression ``hasattr(o, attr_name)``. @@ -61,7 +111,8 @@ Object Protocol .. note:: Exceptions that occur when this calls :meth:`~object.__getattr__` and - :meth:`~object.__getattribute__` methods are silently ignored. + :meth:`~object.__getattribute__` methods aren't propagated, + but instead given to :func:`sys.unraisablehook`. For proper error handling, use :c:func:`PyObject_HasAttrWithError`, :c:func:`PyObject_GetOptionalAttr` or :c:func:`PyObject_GetAttr` instead. @@ -156,6 +207,13 @@ Object Protocol If *v* is ``NULL``, the attribute is deleted, but this feature is deprecated in favour of using :c:func:`PyObject_DelAttrString`. + The number of different attribute names passed to this function + should be kept small, usually by using a statically allocated string + as *attr_name*. + For attribute names that aren't known at compile time, prefer calling + :c:func:`PyUnicode_FromString` and :c:func:`PyObject_SetAttr` directly. + For more details, see :c:func:`PyUnicode_InternFromString`, which may be + used internally to create a key object. .. c:function:: int PyObject_GenericSetAttr(PyObject *o, PyObject *name, PyObject *value) @@ -181,6 +239,14 @@ Object Protocol specified as a :c:expr:`const char*` UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. + The number of different attribute names passed to this function + should be kept small, usually by using a statically allocated string + as *attr_name*. + For attribute names that aren't known at compile time, prefer calling + :c:func:`PyUnicode_FromString` and :c:func:`PyObject_DelAttr` directly. + For more details, see :c:func:`PyUnicode_InternFromString`, which may be + used internally to create a key object for lookup. + .. c:function:: PyObject* PyObject_GenericGetDict(PyObject *o, void *context) @@ -302,14 +368,14 @@ Object Protocol The result will be ``1`` when at least one of the checks returns ``1``, otherwise it will be ``0``. - If *cls* has a :meth:`~class.__subclasscheck__` method, it will be called to + If *cls* has a :meth:`~type.__subclasscheck__` method, it will be called to determine the subclass status as described in :pep:`3119`. Otherwise, *derived* is a subclass of *cls* if it is a direct or indirect subclass, - i.e. contained in ``cls.__mro__``. + i.e. contained in :attr:`cls.__mro__ `. Normally only class objects, i.e. instances of :class:`type` or a derived class, are considered classes. However, objects can override this by having - a :attr:`~class.__bases__` attribute (which must be a tuple of base classes). + a :attr:`~type.__bases__` attribute (which must be a tuple of base classes). .. c:function:: int PyObject_IsInstance(PyObject *inst, PyObject *cls) @@ -321,15 +387,15 @@ Object Protocol The result will be ``1`` when at least one of the checks returns ``1``, otherwise it will be ``0``. - If *cls* has a :meth:`~class.__instancecheck__` method, it will be called to + If *cls* has a :meth:`~type.__instancecheck__` method, it will be called to determine the subclass status as described in :pep:`3119`. Otherwise, *inst* is an instance of *cls* if its class is a subclass of *cls*. An instance *inst* can override what is considered its class by having a - :attr:`~instance.__class__` attribute. + :attr:`~object.__class__` attribute. An object *cls* can override if it is considered a class, and what its base - classes are, by having a :attr:`~class.__bases__` attribute (which must be a tuple + classes are, by having a :attr:`~type.__bases__` attribute (which must be a tuple of base classes). @@ -427,6 +493,13 @@ Object Protocol on failure. This is equivalent to the Python statement ``del o[key]``. +.. c:function:: int PyObject_DelItemString(PyObject *o, const char *key) + + This is the same as :c:func:`PyObject_DelItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + .. c:function:: PyObject* PyObject_Dir(PyObject *o) This is equivalent to the Python expression ``dir(o)``, returning a (possibly @@ -444,6 +517,12 @@ Object Protocol iterated. +.. c:function:: PyObject* PyObject_SelfIter(PyObject *obj) + + This is equivalent to the Python ``__iter__(self): return self`` method. + It is intended for :term:`iterator` types, to be used in the :c:member:`PyTypeObject.tp_iter` slot. + + .. c:function:: PyObject* PyObject_GetAIter(PyObject *o) This is the equivalent to the Python expression ``aiter(o)``. Takes an @@ -510,3 +589,119 @@ Object Protocol has the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag set. .. versionadded:: 3.13 + +.. c:function:: int PyUnstable_Object_EnableDeferredRefcount(PyObject *obj) + + Enable `deferred reference counting `_ on *obj*, + if supported by the runtime. In the :term:`free-threaded ` build, + this allows the interpreter to avoid reference count adjustments to *obj*, + which may improve multi-threaded performance. The tradeoff is + that *obj* will only be deallocated by the tracing garbage collector. + + This function returns ``1`` if deferred reference counting is enabled on *obj* + (including when it was enabled before the call), + and ``0`` if deferred reference counting is not supported or if the hint was + ignored by the runtime. This function is thread-safe, and cannot fail. + + This function does nothing on builds with the :term:`GIL` enabled, which do + not support deferred reference counting. This also does nothing if *obj* is not + an object tracked by the garbage collector (see :func:`gc.is_tracked` and + :c:func:`PyObject_GC_IsTracked`). + + This function is intended to be used soon after *obj* is created, + by the code that creates it. + + .. versionadded:: 3.14 + +.. c:function:: int PyUnstable_IsImmortal(PyObject *obj) + + This function returns non-zero if *obj* is :term:`immortal`, and zero + otherwise. This function cannot fail. + + .. note:: + + Objects that are immortal in one CPython version are not guaranteed to + be immortal in another. + + .. versionadded:: next + +.. c:function:: int PyUnstable_TryIncRef(PyObject *obj) + + Increments the reference count of *obj* if it is not zero. Returns ``1`` + if the object's reference count was successfully incremented. Otherwise, + this function returns ``0``. + + :c:func:`PyUnstable_EnableTryIncRef` must have been called + earlier on *obj* or this function may spuriously return ``0`` in the + :term:`free threading` build. + + This function is logically equivalent to the following C code, except that + it behaves atomically in the :term:`free threading` build:: + + if (Py_REFCNT(op) > 0) { + Py_INCREF(op); + return 1; + } + return 0; + + This is intended as a building block for managing weak references + without the overhead of a Python :ref:`weak reference object `. + + Typically, correct use of this function requires support from *obj*'s + deallocator (:c:member:`~PyTypeObject.tp_dealloc`). + For example, the following sketch could be adapted to implement a + "weakmap" that works like a :py:class:`~weakref.WeakValueDictionary` + for a specific type: + + .. code-block:: c + + PyMutex mutex; + + PyObject * + add_entry(weakmap_key_type *key, PyObject *value) + { + PyUnstable_EnableTryIncRef(value); + weakmap_type weakmap = ...; + PyMutex_Lock(&mutex); + weakmap_add_entry(weakmap, key, value); + PyMutex_Unlock(&mutex); + Py_RETURN_NONE; + } + + PyObject * + get_value(weakmap_key_type *key) + { + weakmap_type weakmap = ...; + PyMutex_Lock(&mutex); + PyObject *result = weakmap_find(weakmap, key); + if (PyUnstable_TryIncRef(result)) { + // `result` is safe to use + PyMutex_Unlock(&mutex); + return result; + } + // if we get here, `result` is starting to be garbage-collected, + // but has not been removed from the weakmap yet + PyMutex_Unlock(&mutex); + return NULL; + } + + // tp_dealloc function for weakmap values + void + value_dealloc(PyObject *value) + { + weakmap_type weakmap = ...; + PyMutex_Lock(&mutex); + weakmap_remove_value(weakmap, value); + + ... + PyMutex_Unlock(&mutex); + } + + .. versionadded:: 3.14 + +.. c:function:: void PyUnstable_EnableTryIncRef(PyObject *obj) + + Enables subsequent uses of :c:func:`PyUnstable_TryIncRef` on *obj*. The + caller must hold a :term:`strong reference` to *obj* when calling this. + + .. versionadded:: 3.14 diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 75e1d46474f1e7..d75dad737bc992 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -23,12 +23,12 @@ of Python objects. Use the :c:func:`Py_SET_REFCNT()` function to set an object reference count. - .. versionchanged:: 3.11 - The parameter type is no longer :c:expr:`const PyObject*`. - .. versionchanged:: 3.10 :c:func:`Py_REFCNT()` is changed to the inline static function. + .. versionchanged:: 3.11 + The parameter type is no longer :c:expr:`const PyObject*`. + .. c:function:: void Py_SET_REFCNT(PyObject *o, Py_ssize_t refcnt) @@ -62,7 +62,7 @@ of Python objects. ``NULL``, use :c:func:`Py_XINCREF`. Do not expect this function to actually modify *o* in any way. - For at least `some objects `_, + For at least :pep:`some objects <0683>`, this function has no effect. .. versionchanged:: 3.12 @@ -130,7 +130,7 @@ of Python objects. use :c:func:`Py_XDECREF`. Do not expect this function to actually modify *o* in any way. - For at least `some objects `_, + For at least :pep:`some objects <683>`, this function has no effect. .. warning:: diff --git a/Doc/c-api/reflection.rst b/Doc/c-api/reflection.rst index 4b1c4770848a30..038e6977104560 100644 --- a/Doc/c-api/reflection.rst +++ b/Doc/c-api/reflection.rst @@ -7,18 +7,48 @@ Reflection .. c:function:: PyObject* PyEval_GetBuiltins(void) + .. deprecated:: 3.13 + + Use :c:func:`PyEval_GetFrameBuiltins` instead. + Return a dictionary of the builtins in the current execution frame, or the interpreter of the thread state if no frame is currently executing. .. c:function:: PyObject* PyEval_GetLocals(void) - Return a dictionary of the local variables in the current execution frame, + .. deprecated:: 3.13 + + Use either :c:func:`PyEval_GetFrameLocals` to obtain the same behaviour as calling + :func:`locals` in Python code, or else call :c:func:`PyFrame_GetLocals` on the result + of :c:func:`PyEval_GetFrame` to access the :attr:`~frame.f_locals` attribute of the + currently executing frame. + + Return a mapping providing access to the local variables in the current execution frame, or ``NULL`` if no frame is currently executing. + Refer to :func:`locals` for details of the mapping returned at different scopes. + + As this function returns a :term:`borrowed reference`, the dictionary returned for + :term:`optimized scopes ` is cached on the frame object and will remain + alive as long as the frame object does. Unlike :c:func:`PyEval_GetFrameLocals` and + :func:`locals`, subsequent calls to this function in the same frame will update the + contents of the cached dictionary to reflect changes in the state of the local variables + rather than returning a new snapshot. + + .. versionchanged:: 3.13 + As part of :pep:`667`, :c:func:`PyFrame_GetLocals`, :func:`locals`, and + :attr:`FrameType.f_locals ` no longer make use of the shared cache + dictionary. Refer to the :ref:`What's New entry ` for + additional details. + .. c:function:: PyObject* PyEval_GetGlobals(void) + .. deprecated:: 3.13 + + Use :c:func:`PyEval_GetFrameGlobals` instead. + Return a dictionary of the global variables in the current execution frame, or ``NULL`` if no frame is currently executing. @@ -31,6 +61,36 @@ Reflection See also :c:func:`PyThreadState_GetFrame`. +.. c:function:: PyObject* PyEval_GetFrameBuiltins(void) + + Return a dictionary of the builtins in the current execution frame, + or the interpreter of the thread state if no frame is currently executing. + + .. versionadded:: 3.13 + + +.. c:function:: PyObject* PyEval_GetFrameLocals(void) + + Return a dictionary of the local variables in the current execution frame, + or ``NULL`` if no frame is currently executing. Equivalent to calling + :func:`locals` in Python code. + + To access :attr:`~frame.f_locals` on the current frame without making an independent + snapshot in :term:`optimized scopes `, call :c:func:`PyFrame_GetLocals` + on the result of :c:func:`PyEval_GetFrame`. + + .. versionadded:: 3.13 + + +.. c:function:: PyObject* PyEval_GetFrameGlobals(void) + + Return a dictionary of the global variables in the current execution frame, + or ``NULL`` if no frame is currently executing. Equivalent to calling + :func:`globals` in Python code. + + .. versionadded:: 3.13 + + .. c:function:: const char* PyEval_GetFuncName(PyObject *func) Return the name of *func* if it is a function, class or instance object, else the diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index ce28839f5ba739..df5bf6b64a93a0 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -105,6 +105,15 @@ Sequence Protocol equivalent to the Python expression ``value in o``. +.. c:function:: int PySequence_In(PyObject *o, PyObject *value) + + Alias for :c:func:`PySequence_Contains`. + + .. deprecated:: 3.14 + The function is :term:`soft deprecated` and should no longer be used to + write new code. + + .. c:function:: Py_ssize_t PySequence_Index(PyObject *o, PyObject *value) Return the first index *i* for which ``o[i] == value``. On error, return diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst index 27a1757c745d8b..8adf6a961378a3 100644 --- a/Doc/c-api/slice.rst +++ b/Doc/c-api/slice.rst @@ -23,7 +23,9 @@ Slice Objects Return a new slice object with the given values. The *start*, *stop*, and *step* parameters are used as the values of the slice object attributes of the same names. Any of the values may be ``NULL``, in which case the - ``None`` will be used for the corresponding attribute. Return ``NULL`` if + ``None`` will be used for the corresponding attribute. + + Return ``NULL`` with an exception set if the new object could not be allocated. @@ -52,7 +54,7 @@ Slice Objects of bounds indices are clipped in a manner consistent with the handling of normal slices. - Returns ``0`` on success and ``-1`` on error with exception set. + Return ``0`` on success and ``-1`` on error with an exception set. .. note:: This function is considered not safe for resizable sequences. @@ -95,7 +97,7 @@ Slice Objects ``PY_SSIZE_T_MIN`` to ``PY_SSIZE_T_MIN``, and silently boost the step values less than ``-PY_SSIZE_T_MAX`` to ``-PY_SSIZE_T_MAX``. - Return ``-1`` on error, ``0`` on success. + Return ``-1`` with an exception set on error, ``0`` on success. .. versionadded:: 3.6.1 diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 5b9e43874c7f2b..124e58cf950b7a 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -66,7 +66,7 @@ Limited C API Python 3.2 introduced the *Limited API*, a subset of Python's C API. Extensions that only use the Limited API can be -compiled once and work with multiple versions of Python. +compiled once and be loaded on multiple versions of Python. Contents of the Limited API are :ref:`listed below `. .. c:macro:: Py_LIMITED_API @@ -76,7 +76,7 @@ Contents of the Limited API are :ref:`listed below `. Define ``Py_LIMITED_API`` to the value of :c:macro:`PY_VERSION_HEX` corresponding to the lowest Python version your extension supports. - The extension will work without recompilation with all Python 3 releases + The extension will be ABI-compatible with all Python 3 releases from the specified one onward, and can use Limited API introduced up to that version. @@ -94,7 +94,15 @@ Stable ABI ---------- To enable this, Python provides a *Stable ABI*: a set of symbols that will -remain compatible across Python 3.x versions. +remain ABI-compatible across Python 3.x versions. + +.. note:: + + The Stable ABI prevents ABI issues, like linker errors due to missing + symbols or data corruption due to changes in structure layouts or function + signatures. + However, other changes in Python can change the *behavior* of extensions. + See Python's Backwards Compatibility Policy (:pep:`387`) for details. The Stable ABI contains symbols exposed in the :ref:`Limited API `, but also other ones – for example, functions necessary to diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index e2943f18ddc601..d333df397782e0 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -485,7 +485,8 @@ Accessing attributes of extension types ``PyMemberDef`` may contain a definition for the special member ``"__vectorcalloffset__"``, corresponding to :c:member:`~PyTypeObject.tp_vectorcall_offset` in type objects. - These must be defined with ``Py_T_PYSSIZET`` and ``Py_READONLY``, for example:: + This member must be defined with ``Py_T_PYSSIZET``, and either + ``Py_READONLY`` or ``Py_READONLY | Py_RELATIVE_OFFSET``. For example:: static PyMemberDef spam_type_members[] = { {"__vectorcalloffset__", Py_T_PYSSIZET, @@ -506,6 +507,12 @@ Accessing attributes of extension types ``PyMemberDef`` is always available. Previously, it required including ``"structmember.h"``. + .. versionchanged:: 3.14 + + :c:macro:`Py_RELATIVE_OFFSET` is now allowed for + ``"__vectorcalloffset__"``, ``"__dictoffset__"`` and + ``"__weaklistoffset__"``. + .. c:function:: PyObject* PyMember_GetOne(const char *obj_addr, struct PyMemberDef *m) Get an attribute belonging to the object at address *obj_addr*. The @@ -702,12 +709,12 @@ Defining Getters and Setters .. c:member:: void* closure - Optional function pointer, providing additional data for getter and setter. + Optional user data pointer, providing additional data for getter and setter. .. c:type:: PyObject *(*getter)(PyObject *, void *) The ``get`` function takes one :c:expr:`PyObject*` parameter (the - instance) and a function pointer (the associated ``closure``): + instance) and a user data pointer (the associated ``closure``): It should return a new reference on success or ``NULL`` with a set exception on failure. @@ -715,7 +722,7 @@ Defining Getters and Setters .. c:type:: int (*setter)(PyObject *, PyObject *, void *) ``set`` functions take two :c:expr:`PyObject*` parameters (the instance and - the value to be set) and a function pointer (the associated ``closure``): + the value to be set) and a user data pointer (the associated ``closure``): In case the attribute should be deleted the second parameter is ``NULL``. Should return ``0`` on success or ``-1`` with a set exception on failure. diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index d6fca1a0b0a219..4ab5df4ccccdbb 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -216,6 +216,38 @@ Operating System Utilities The function now uses the UTF-8 encoding on Windows if :c:member:`PyPreConfig.legacy_windows_fs_encoding` is zero. +.. c:function:: FILE* Py_fopen(PyObject *path, const char *mode) + + Similar to :c:func:`!fopen`, but *path* is a Python object and + an exception is set on error. + + *path* must be a :class:`str` object, a :class:`bytes` object, + or a :term:`path-like object`. + + On success, return the new file pointer. + On error, set an exception and return ``NULL``. + + The file must be closed by :c:func:`Py_fclose` rather than calling directly + :c:func:`!fclose`. + + The file descriptor is created non-inheritable (:pep:`446`). + + The caller must hold the GIL. + + .. versionadded:: 3.14 + + +.. c:function:: int Py_fclose(FILE *file) + + Close a file that was opened by :c:func:`Py_fopen`. + + On success, return ``0``. + On error, return ``EOF`` and ``errno`` is set to indicate the error. + In either case, any further access (including another call to + :c:func:`Py_fclose`) to the stream results in undefined behavior. + + .. versionadded:: 3.14 + .. _systemfunctions: @@ -426,3 +458,7 @@ Process Control function registered last is called first. Each cleanup function will be called at most once. Since Python's internal finalization will have completed before the cleanup function, no Python APIs should be called by *func*. + + .. seealso:: + + :c:func:`PyUnstable_AtExit` for passing a ``void *data`` argument. diff --git a/Doc/c-api/time.rst b/Doc/c-api/time.rst index 7791cdb1781055..7032cc48aa6913 100644 --- a/Doc/c-api/time.rst +++ b/Doc/c-api/time.rst @@ -1,5 +1,7 @@ .. highlight:: c +.. _c-api-time: + PyTime C API ============ @@ -72,6 +74,35 @@ with the :term:`GIL` held. See :func:`time.time` for details important on this clock. +Raw Clock Functions +------------------- + +Similar to clock functions, but don't set an exception on error and don't +require the caller to hold the GIL. + +On success, the functions return ``0``. + +On failure, they set ``*result`` to ``0`` and return ``-1``, *without* setting +an exception. To get the cause of the error, acquire the GIL and call the +regular (non-``Raw``) function. Note that the regular function may succeed after +the ``Raw`` one failed. + +.. c:function:: int PyTime_MonotonicRaw(PyTime_t *result) + + Similar to :c:func:`PyTime_Monotonic`, + but don't set an exception on error and don't require holding the GIL. + +.. c:function:: int PyTime_PerfCounterRaw(PyTime_t *result) + + Similar to :c:func:`PyTime_PerfCounter`, + but don't set an exception on error and don't require holding the GIL. + +.. c:function:: int PyTime_TimeRaw(PyTime_t *result) + + Similar to :c:func:`PyTime_Time`, + but don't set an exception on error and don't require holding the GIL. + + Conversion functions -------------------- diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index b3710560ebe7ac..815afddad19df1 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -33,12 +33,14 @@ Tuple Objects .. c:function:: PyObject* PyTuple_New(Py_ssize_t len) - Return a new tuple object of size *len*, or ``NULL`` on failure. + Return a new tuple object of size *len*, + or ``NULL`` with an exception set on failure. .. c:function:: PyObject* PyTuple_Pack(Py_ssize_t n, ...) - Return a new tuple object of size *n*, or ``NULL`` on failure. The tuple values + Return a new tuple object of size *n*, + or ``NULL`` with an exception set on failure. The tuple values are initialized to the subsequent *n* C arguments pointing to Python objects. ``PyTuple_Pack(2, a, b)`` is equivalent to ``Py_BuildValue("(OO)", a, b)``. @@ -46,12 +48,12 @@ Tuple Objects .. c:function:: Py_ssize_t PyTuple_Size(PyObject *p) Take a pointer to a tuple object, and return the size of that tuple. + On error, return ``-1`` and with an exception set. .. c:function:: Py_ssize_t PyTuple_GET_SIZE(PyObject *p) - Return the size of the tuple *p*, which must be non-``NULL`` and point to a tuple; - no error checking is performed. + Like :c:func:`PyTuple_Size`, but without error checking. .. c:function:: PyObject* PyTuple_GetItem(PyObject *p, Py_ssize_t pos) @@ -59,6 +61,12 @@ Tuple Objects Return the object at position *pos* in the tuple pointed to by *p*. If *pos* is negative or out of bounds, return ``NULL`` and set an :exc:`IndexError` exception. + The returned reference is borrowed from the tuple *p* + (that is: it is only valid as long as you hold a reference to *p*). + To get a :term:`strong reference`, use + :c:func:`Py_NewRef(PyTuple_GetItem(...)) ` + or :c:func:`PySequence_GetItem`. + .. c:function:: PyObject* PyTuple_GET_ITEM(PyObject *p, Py_ssize_t pos) @@ -68,8 +76,10 @@ Tuple Objects .. c:function:: PyObject* PyTuple_GetSlice(PyObject *p, Py_ssize_t low, Py_ssize_t high) Return the slice of the tuple pointed to by *p* between *low* and *high*, - or ``NULL`` on failure. This is the equivalent of the Python expression - ``p[low:high]``. Indexing from the end of the tuple is not supported. + or ``NULL`` with an exception set on failure. + + This is the equivalent of the Python expression ``p[low:high]``. + Indexing from the end of the tuple is not supported. .. c:function:: int PyTuple_SetItem(PyObject *p, Py_ssize_t pos, PyObject *o) @@ -99,6 +109,12 @@ Tuple Objects is being replaced; any reference in the tuple at position *pos* will be leaked. + .. warning:: + + This macro should *only* be used on tuples that are newly created. + Using this macro on a tuple that is already in use (or in other words, has + a refcount > 1) could lead to undefined behavior. + .. c:function:: int _PyTuple_Resize(PyObject **p, Py_ssize_t newsize) @@ -129,6 +145,8 @@ type. Create a new struct sequence type from the data in *desc*, described below. Instances of the resulting type can be created with :c:func:`PyStructSequence_New`. + Return ``NULL`` with an exception set on failure. + .. c:function:: void PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc) @@ -137,8 +155,8 @@ type. .. c:function:: int PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) - The same as ``PyStructSequence_InitType``, but returns ``0`` on success and ``-1`` on - failure. + Like :c:func:`PyStructSequence_InitType`, but returns ``0`` on success + and ``-1`` with an exception set on failure. .. versionadded:: 3.4 @@ -149,7 +167,8 @@ type. .. c:member:: const char *name - Name of the struct sequence type. + Fully qualified name of the type; null-terminated UTF-8 encoded. + The name must contain the module name. .. c:member:: const char *doc @@ -195,6 +214,8 @@ type. Creates an instance of *type*, which must have been created with :c:func:`PyStructSequence_NewType`. + Return ``NULL`` with an exception set on failure. + .. c:function:: PyObject* PyStructSequence_GetItem(PyObject *p, Py_ssize_t pos) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 5aaa8147dd3176..444b3456f051d8 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -53,7 +53,8 @@ Type Objects .. c:function:: PyObject* PyType_GetDict(PyTypeObject* type) Return the type object's internal namespace, which is otherwise only - exposed via a read-only proxy (``cls.__dict__``). This is a + exposed via a read-only proxy (:attr:`cls.__dict__ `). + This is a replacement for accessing :c:member:`~PyTypeObject.tp_dict` directly. The returned dictionary must be treated as read-only. @@ -140,7 +141,7 @@ Type Objects Return true if *a* is a subtype of *b*. This function only checks for actual subtypes, which means that - :meth:`~class.__subclasscheck__` is not called on *b*. Call + :meth:`~type.__subclasscheck__` is not called on *b*. Call :c:func:`PyObject_IsSubclass` to do the same check that :func:`issubclass` would do. @@ -174,17 +175,33 @@ Type Objects .. c:function:: PyObject* PyType_GetName(PyTypeObject *type) - Return the type's name. Equivalent to getting the type's ``__name__`` attribute. + Return the type's name. Equivalent to getting the type's + :attr:`~type.__name__` attribute. .. versionadded:: 3.11 .. c:function:: PyObject* PyType_GetQualName(PyTypeObject *type) Return the type's qualified name. Equivalent to getting the - type's ``__qualname__`` attribute. + type's :attr:`~type.__qualname__` attribute. .. versionadded:: 3.11 +.. c:function:: PyObject* PyType_GetFullyQualifiedName(PyTypeObject *type) + + Return the type's fully qualified name. Equivalent to + ``f"{type.__module__}.{type.__qualname__}"``, or :attr:`type.__qualname__` + if :attr:`type.__module__` is not a string or is equal to ``"builtins"``. + + .. versionadded:: 3.13 + +.. c:function:: PyObject* PyType_GetModuleName(PyTypeObject *type) + + Return the type's module name. Equivalent to getting the + :attr:`type.__module__` attribute. + + .. versionadded:: 3.13 + .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) Return the function pointer stored in the given slot. If the @@ -249,6 +266,24 @@ Type Objects .. versionadded:: 3.11 +.. c:function:: int PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) + + Find the first superclass in *type*'s :term:`method resolution order` whose + :c:macro:`Py_tp_token` token is equal to the given one. + + * If found, set *\*result* to a new :term:`strong reference` + to it and return ``1``. + * If not found, set *\*result* to ``NULL`` and return ``0``. + * On error, set *\*result* to ``NULL`` and return ``-1`` with an + exception set. + + The *result* argument may be ``NULL``, in which case *\*result* is not set. + Use this if you need only the return value. + + The *token* argument may not be ``NULL``. + + .. versionadded:: 3.14 + .. c:function:: int PyUnstable_Type_AssignVersionTag(PyTypeObject *type) Attempt to assign a version tag to the given type. @@ -330,8 +365,12 @@ The following functions and structs are used to create The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. which may result in incomplete initialization. Creating classes whose metaclass overrides - :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it - will be no longer allowed. + :c:member:`~PyTypeObject.tp_new` is deprecated. + + .. versionchanged:: 3.14 + + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is no longer allowed. .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) @@ -347,8 +386,12 @@ The following functions and structs are used to create The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. which may result in incomplete initialization. Creating classes whose metaclass overrides - :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it - will be no longer allowed. + :c:member:`~PyTypeObject.tp_new` is deprecated. + + .. versionchanged:: 3.14 + + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is no longer allowed. .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) @@ -363,8 +406,26 @@ The following functions and structs are used to create The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. which may result in incomplete initialization. Creating classes whose metaclass overrides - :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it - will be no longer allowed. + :c:member:`~PyTypeObject.tp_new` is deprecated. + + .. versionchanged:: 3.14 + + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is no longer allowed. + +.. c:function:: int PyType_Freeze(PyTypeObject *type) + + Make a type immutable: set the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag. + + All base classes of *type* must be immutable. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + The type must not be used before it's made immutable. For example, type + instances must not be created before the type is made immutable. + + .. versionadded:: 3.14 .. raw:: html @@ -461,30 +522,31 @@ The following functions and structs are used to create * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add` * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length` + An additional slot is supported that does not correspond to a + :c:type:`!PyTypeObject` struct field: + + * :c:data:`Py_tp_token` + The following “offset” fields cannot be set using :c:type:`PyType_Slot`: - * :c:member:`~PyTypeObject.tp_weaklistoffset` - (use :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead if possible) - * :c:member:`~PyTypeObject.tp_dictoffset` - (use :c:macro:`Py_TPFLAGS_MANAGED_DICT` instead if possible) - * :c:member:`~PyTypeObject.tp_vectorcall_offset` - (use ``"__vectorcalloffset__"`` in - :ref:`PyMemberDef `) - - If it is not possible to switch to a ``MANAGED`` flag (for example, - for vectorcall or to support Python older than 3.12), specify the - offset in :c:member:`Py_tp_members `. - See :ref:`PyMemberDef documentation ` - for details. + * :c:member:`~PyTypeObject.tp_weaklistoffset` + (use :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead if possible) + * :c:member:`~PyTypeObject.tp_dictoffset` + (use :c:macro:`Py_TPFLAGS_MANAGED_DICT` instead if possible) + * :c:member:`~PyTypeObject.tp_vectorcall_offset` + (use ``"__vectorcalloffset__"`` in + :ref:`PyMemberDef `) - The following fields cannot be set at all when creating a heap type: + If it is not possible to switch to a ``MANAGED`` flag (for example, + for vectorcall or to support Python older than 3.12), specify the + offset in :c:member:`Py_tp_members `. + See :ref:`PyMemberDef documentation ` + for details. - * :c:member:`~PyTypeObject.tp_vectorcall` - (use :c:member:`~PyTypeObject.tp_new` and/or - :c:member:`~PyTypeObject.tp_init`) + The following internal fields cannot be set at all when creating a heap + type: - * Internal fields: - :c:member:`~PyTypeObject.tp_dict`, + * :c:member:`~PyTypeObject.tp_dict`, :c:member:`~PyTypeObject.tp_mro`, :c:member:`~PyTypeObject.tp_cache`, :c:member:`~PyTypeObject.tp_subclasses`, and @@ -495,18 +557,65 @@ The following functions and structs are used to create To avoid issues, use the *bases* argument of :c:func:`PyType_FromSpecWithBases` instead. - .. versionchanged:: 3.9 + .. versionchanged:: 3.9 + Slots in :c:type:`PyBufferProcs` may be set in the unlimited API. - Slots in :c:type:`PyBufferProcs` may be set in the unlimited API. + .. versionchanged:: 3.11 + :c:member:`~PyBufferProcs.bf_getbuffer` and + :c:member:`~PyBufferProcs.bf_releasebuffer` are now available + under the :ref:`limited API `. - .. versionchanged:: 3.11 - :c:member:`~PyBufferProcs.bf_getbuffer` and - :c:member:`~PyBufferProcs.bf_releasebuffer` are now available - under the :ref:`limited API `. + .. versionchanged:: 3.14 + The field :c:member:`~PyTypeObject.tp_vectorcall` can now set + using ``Py_tp_vectorcall``. See the field's documentation + for details. .. c:member:: void *pfunc The desired value of the slot. In most cases, this is a pointer to a function. - Slots other than ``Py_tp_doc`` may not be ``NULL``. + *pfunc* values may not be ``NULL``, except for the following slots: + + * ``Py_tp_doc`` + * :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` + rather than ``NULL``) + +.. c:macro:: Py_tp_token + + A :c:member:`~PyType_Slot.slot` that records a static memory layout ID + for a class. + + If the :c:type:`PyType_Spec` of the class is statically + allocated, the token can be set to the spec using the special value + :c:data:`Py_TP_USE_SPEC`: + + .. code-block:: c + + static PyType_Slot foo_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, + + It can also be set to an arbitrary pointer, but you must ensure that: + + * The pointer outlives the class, so it's not reused for something else + while the class exists. + * It "belongs" to the extension module where the class lives, so it will not + clash with other extensions. + + Use :c:func:`PyType_GetBaseByToken` to check if a class's superclass has + a given token -- that is, check whether the memory layout is compatible. + + To get the token for a given class (without considering superclasses), + use :c:func:`PyType_GetSlot` with ``Py_tp_token``. + + .. versionadded:: 3.14 + + .. c:namespace:: NULL + + .. c:macro:: Py_TP_USE_SPEC + + Used as a value with :c:data:`Py_tp_token` to set the token to the + class's :c:type:`PyType_Spec`. + Expands to ``NULL``. + + .. versionadded:: 3.14 diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 8a26f237652d12..69c518d8e64cbc 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -355,7 +355,7 @@ slot typedefs +-----------------------------+-----------------------------+----------------------+ | :c:type:`newfunc` | .. line-block:: | :c:type:`PyObject` * | | | | | -| | :c:type:`PyObject` * | | +| | :c:type:`PyTypeObject` * | | | | :c:type:`PyObject` * | | | | :c:type:`PyObject` * | | +-----------------------------+-----------------------------+----------------------+ @@ -567,12 +567,12 @@ and :c:data:`PyType_Type` effectively act as defaults.) For :ref:`statically allocated type objects `, the *tp_name* field should contain a dot. - Everything before the last dot is made accessible as the :attr:`__module__` + Everything before the last dot is made accessible as the :attr:`~type.__module__` attribute, and everything after the last dot is made accessible as the - :attr:`~definition.__name__` attribute. + :attr:`~type.__name__` attribute. If no dot is present, the entire :c:member:`~PyTypeObject.tp_name` field is made accessible as the - :attr:`~definition.__name__` attribute, and the :attr:`__module__` attribute is undefined + :attr:`~type.__name__` attribute, and the :attr:`~type.__module__` attribute is undefined (unless explicitly set in the dictionary, as explained above). This means your type will be impossible to pickle. Additionally, it will not be listed in module documentations created with pydoc. @@ -650,7 +650,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) (doesn't have the :c:macro:`Py_TPFLAGS_BASETYPE` flag bit set), it is permissible to call the object deallocator directly instead of via :c:member:`~PyTypeObject.tp_free`. The object deallocator should be the one used to allocate the - instance; this is normally :c:func:`PyObject_Del` if the instance was allocated + instance; this is normally :c:func:`PyObject_Free` if the instance was allocated using :c:macro:`PyObject_New` or :c:macro:`PyObject_NewVar`, or :c:func:`PyObject_GC_Del` if the instance was allocated using :c:macro:`PyObject_GC_New` or :c:macro:`PyObject_GC_NewVar`. @@ -682,6 +682,19 @@ and :c:data:`PyType_Type` effectively act as defaults.) Py_DECREF(tp); } + .. warning:: + + In a garbage collected Python, :c:member:`!tp_dealloc` may be called from + any Python thread, not just the thread which created the object (if the + object becomes part of a refcount cycle, that cycle might be collected by + a garbage collection on any thread). This is not a problem for Python + API calls, since the thread on which :c:member:`!tp_dealloc` is called + will own the Global Interpreter Lock (GIL). However, if the object being + destroyed in turn destroys objects from some other C or C++ library, care + should be taken to ensure that destroying those objects on the thread + which called :c:member:`!tp_dealloc` will not violate any assumptions of + the library. + **Inheritance:** @@ -883,6 +896,10 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash`, when the subtype's :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` are both ``NULL``. + **Default:** + + :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_GenericHash`. + .. c:member:: ternaryfunc PyTypeObject.tp_call @@ -1006,6 +1023,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is clear in the subtype and the :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields in the subtype exist and have ``NULL`` values. + .. XXX are most flag bits *really* inherited individually? **Default:** @@ -1030,7 +1048,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) the type, and the type object is INCREF'ed when a new instance is created, and DECREF'ed when an instance is destroyed (this does not apply to instances of subtypes; only the type referenced by the instance's ob_type gets INCREF'ed or - DECREF'ed). + DECREF'ed). Heap types should also :ref:`support garbage collection ` + as they can form a reference cycle with their own module object. **Inheritance:** @@ -1126,7 +1145,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_MANAGED_DICT - This bit indicates that instances of the class have a ``__dict__`` + This bit indicates that instances of the class have a `~object.__dict__` attribute, and that the space for the dictionary is managed by the VM. If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` should also be set. @@ -1323,15 +1342,15 @@ and :c:data:`PyType_Type` effectively act as defaults.) To indicate that a class has changed call :c:func:`PyType_Modified` .. warning:: - This flag is present in header files, but is an internal feature and should - not be used. It will be removed in a future version of CPython + This flag is present in header files, but is not be used. + It will be removed in a future version of CPython .. c:member:: const char* PyTypeObject.tp_doc An optional pointer to a NUL-terminated C string giving the docstring for this - type object. This is exposed as the :attr:`__doc__` attribute on the type and - instances of the type. + type object. This is exposed as the :attr:`~type.__doc__` attribute on the + type and instances of the type. **Inheritance:** @@ -1376,7 +1395,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) Py_VISIT(Py_TYPE(self)); It is only needed since Python 3.9. To support Python 3.8 and older, this - line must be conditionnal:: + line must be conditional:: #if PY_VERSION_HEX >= 0x03090000 Py_VISIT(Py_TYPE(self)); @@ -1587,7 +1606,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) weak references to the type object itself. It is an error to set both the :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` bit and - :c:member:`~PyTypeObject.tp_weaklist`. + :c:member:`~PyTypeObject.tp_weaklistoffset`. **Inheritance:** @@ -1599,7 +1618,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) **Default:** If the :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` bit is set in the - :c:member:`~PyTypeObject.tp_dict` field, then + :c:member:`~PyTypeObject.tp_flags` field, then :c:member:`~PyTypeObject.tp_weaklistoffset` will be set to a negative value, to indicate that it is unsafe to use this field. @@ -1949,7 +1968,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) match :c:func:`PyType_GenericAlloc` and the value of the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit. - For static subtypes, :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_Del`. + For static subtypes, :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_Free`. .. c:member:: inquiry PyTypeObject.tp_is_gc @@ -2031,7 +2050,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) A collection of subclasses. Internal use only. May be an invalid pointer. To get a list of subclasses, call the Python method - :py:meth:`~class.__subclasses__`. + :py:meth:`~type.__subclasses__`. .. versionchanged:: 3.12 @@ -2104,17 +2123,6 @@ and :c:data:`PyType_Type` effectively act as defaults.) PyErr_Restore(error_type, error_value, error_traceback); } - Also, note that, in a garbage collected Python, - :c:member:`~PyTypeObject.tp_dealloc` may be called from - any Python thread, not just the thread which created the object (if the object - becomes part of a refcount cycle, that cycle might be collected by a garbage - collection on any thread). This is not a problem for Python API calls, since - the thread on which tp_dealloc is called will own the Global Interpreter Lock - (GIL). However, if the object being destroyed in turn destroys objects from some - other C or C++ library, care should be taken to ensure that destroying those - objects on the thread which called tp_dealloc will not violate any assumptions - of the library. - **Inheritance:** This field is inherited by subtypes. @@ -2132,11 +2140,40 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: vectorcallfunc PyTypeObject.tp_vectorcall - Vectorcall function to use for calls of this type object. - In other words, it is used to implement - :ref:`vectorcall ` for ``type.__call__``. - If ``tp_vectorcall`` is ``NULL``, the default call implementation - using :meth:`~object.__new__` and :meth:`~object.__init__` is used. + A :ref:`vectorcall function ` to use for calls of this type + object (rather than instances). + In other words, ``tp_vectorcall`` can be used to optimize ``type.__call__``, + which typically returns a new instance of *type*. + + As with any vectorcall function, if ``tp_vectorcall`` is ``NULL``, + the *tp_call* protocol (``Py_TYPE(type)->tp_call``) is used instead. + + .. note:: + + The :ref:`vectorcall protocol ` requires that the vectorcall + function has the same behavior as the corresponding ``tp_call``. + This means that ``type->tp_vectorcall`` must match the behavior of + ``Py_TYPE(type)->tp_call``. + + Specifically, if *type* uses the default metaclass, + ``type->tp_vectorcall`` must behave the same as + :c:expr:`PyType_Type->tp_call`, which: + + - calls ``type->tp_new``, + + - if the result is a subclass of *type*, calls ``type->tp_init`` + on the result of ``tp_new``, and + + - returns the result of ``tp_new``. + + Typically, ``tp_vectorcall`` is overridden to optimize this process + for specific :c:member:`~PyTypeObject.tp_new` and + :c:member:`~PyTypeObject.tp_init`. + When doing this for user-subclassable types, note that both can be + overridden (using :py:func:`~object.__new__` and + :py:func:`~object.__init__`, respectively). + + **Inheritance:** @@ -2194,7 +2231,7 @@ This is done by filling a :c:type:`PyType_Spec` structure and calling .. _number-structs: Number Object Structures -======================== +------------------------ .. sectionauthor:: Amaury Forgeot d'Arc @@ -2308,7 +2345,7 @@ Number Object Structures .. _mapping-structs: Mapping Object Structures -========================= +------------------------- .. sectionauthor:: Amaury Forgeot d'Arc @@ -2345,7 +2382,7 @@ Mapping Object Structures .. _sequence-structs: Sequence Object Structures -========================== +-------------------------- .. sectionauthor:: Amaury Forgeot d'Arc @@ -2425,7 +2462,7 @@ Sequence Object Structures .. _buffer-structs: Buffer Object Structures -======================== +------------------------ .. sectionauthor:: Greg J. Stein .. sectionauthor:: Benjamin Peterson @@ -2520,7 +2557,7 @@ Buffer Object Structures Async Object Structures -======================= +----------------------- .. sectionauthor:: Yury Selivanov @@ -2588,7 +2625,7 @@ Async Object Structures .. _slot-typedefs: Slot Type typedefs -================== +------------------ .. c:type:: PyObject *(*allocfunc)(PyTypeObject *cls, Py_ssize_t nitems) @@ -2611,7 +2648,7 @@ Slot Type typedefs See :c:member:`~PyTypeObject.tp_free`. -.. c:type:: PyObject *(*newfunc)(PyObject *, PyObject *, PyObject *) +.. c:type:: PyObject *(*newfunc)(PyTypeObject *, PyObject *, PyObject *) See :c:member:`~PyTypeObject.tp_new`. @@ -2697,7 +2734,7 @@ Slot Type typedefs .. _typedef-examples: Examples -======== +-------- The following are simple examples of Python type definitions. They include common usage you may encounter. Some demonstrate tricky corner diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 666ffe89605c56..94110d48ed7d85 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -317,7 +317,7 @@ These APIs can be used to work with surrogates: .. c:function:: Py_UCS4 Py_UNICODE_JOIN_SURROGATES(Py_UCS4 high, Py_UCS4 low) - Join two surrogate characters and return a single :c:type:`Py_UCS4` value. + Join two surrogate code points and return a single :c:type:`Py_UCS4` value. *high* and *low* are respectively the leading and trailing surrogates in a surrogate pair. *high* must be in the range [0xD800; 0xDBFF] and *low* must be in the range [0xDC00; 0xDFFF]. @@ -338,6 +338,8 @@ APIs: This is the recommended way to allocate a new Unicode object. Objects created using this function are not resizable. + On error, set an exception and return ``NULL``. + .. versionadded:: 3.3 @@ -518,6 +520,26 @@ APIs: - :c:expr:`PyObject*` - The result of calling :c:func:`PyObject_Repr`. + * - ``T`` + - :c:expr:`PyObject*` + - Get the fully qualified name of an object type; + call :c:func:`PyType_GetFullyQualifiedName`. + + * - ``#T`` + - :c:expr:`PyObject*` + - Similar to ``T`` format, but use a colon (``:``) as separator between + the module name and the qualified name. + + * - ``N`` + - :c:expr:`PyTypeObject*` + - Get the fully qualified name of a type; + call :c:func:`PyType_GetFullyQualifiedName`. + + * - ``#N`` + - :c:expr:`PyTypeObject*` + - Similar to ``N`` format, but use a colon (``:``) as separator between + the module name and the qualified name. + .. note:: The width formatter unit is number of characters rather than bytes. The precision formatter unit is number of bytes or :c:type:`wchar_t` @@ -553,6 +575,9 @@ APIs: In previous versions it caused all the rest of the format string to be copied as-is to the result string, and any extra arguments discarded. + .. versionchanged:: 3.13 + Support for ``%T``, ``%#T``, ``%N`` and ``%#N`` formats added. + .. c:function:: PyObject* PyUnicode_FromFormatV(const char *format, va_list vargs) @@ -591,6 +616,8 @@ APIs: Return the length of the Unicode object, in code points. + On error, set an exception and return ``-1``. + .. versionadded:: 3.3 @@ -634,6 +661,8 @@ APIs: not out of bounds, and that the object can be modified safely (i.e. that it its reference count is one). + Return ``0`` on success, ``-1`` on error with an exception set. + .. versionadded:: 3.3 @@ -643,6 +672,8 @@ APIs: Unicode object and the index is not out of bounds, in contrast to :c:func:`PyUnicode_READ_CHAR`, which performs no error checking. + Return character on success, ``-1`` on error with an exception set. + .. versionadded:: 3.3 @@ -651,6 +682,7 @@ APIs: Return a substring of *unicode*, from character index *start* (included) to character index *end* (excluded). Negative indices are not supported. + On error, set an exception and return ``NULL``. .. versionadded:: 3.3 @@ -754,16 +786,25 @@ Functions encoding to and decoding from the :term:`filesystem encoding and error handler` (:pep:`383` and :pep:`529`). To encode file names to :class:`bytes` during argument parsing, the ``"O&"`` -converter should be used, passing :c:func:`PyUnicode_FSConverter` as the +converter should be used, passing :c:func:`!PyUnicode_FSConverter` as the conversion function: .. c:function:: int PyUnicode_FSConverter(PyObject* obj, void* result) - ParseTuple converter: encode :class:`str` objects -- obtained directly or + :ref:`PyArg_Parse\* converter `: encode :class:`str` objects -- obtained directly or through the :class:`os.PathLike` interface -- to :class:`bytes` using :c:func:`PyUnicode_EncodeFSDefault`; :class:`bytes` objects are output as-is. - *result* must be a :c:expr:`PyBytesObject*` which must be released when it is - no longer used. + *result* must be an address of a C variable of type :c:expr:`PyObject*` + (or :c:expr:`PyBytesObject*`). + On success, set the variable to a new :term:`strong reference` to + a :ref:`bytes object ` which must be released + when it is no longer used and return a non-zero value + (:c:macro:`Py_CLEANUP_SUPPORTED`). + Embedded null bytes are not allowed in the result. + On failure, return ``0`` with an exception set. + + If *obj* is ``NULL``, the function releases a strong reference + stored in the variable referred by *result* and returns ``1``. .. versionadded:: 3.1 @@ -771,16 +812,26 @@ conversion function: Accepts a :term:`path-like object`. To decode file names to :class:`str` during argument parsing, the ``"O&"`` -converter should be used, passing :c:func:`PyUnicode_FSDecoder` as the +converter should be used, passing :c:func:`!PyUnicode_FSDecoder` as the conversion function: .. c:function:: int PyUnicode_FSDecoder(PyObject* obj, void* result) - ParseTuple converter: decode :class:`bytes` objects -- obtained either + :ref:`PyArg_Parse\* converter `: decode :class:`bytes` objects -- obtained either directly or indirectly through the :class:`os.PathLike` interface -- to :class:`str` using :c:func:`PyUnicode_DecodeFSDefaultAndSize`; :class:`str` - objects are output as-is. *result* must be a :c:expr:`PyUnicodeObject*` which - must be released when it is no longer used. + objects are output as-is. + *result* must be an address of a C variable of type :c:expr:`PyObject*` + (or :c:expr:`PyUnicodeObject*`). + On success, set the variable to a new :term:`strong reference` to + a :ref:`Unicode object ` which must be released + when it is no longer used and return a non-zero value + (:c:macro:`Py_CLEANUP_SUPPORTED`). + Embedded null characters are not allowed in the result. + On failure, return ``0`` with an exception set. + + If *obj* is ``NULL``, release the strong reference + to the object referred to by *result* and return ``1``. .. versionadded:: 3.2 @@ -967,6 +1018,9 @@ These are the UTF-8 codec APIs: object. Error handling is "strict". Return ``NULL`` if an exception was raised by the codec. + The function fails if the string contains surrogate code points + (``U+D800`` - ``U+DFFF``). + .. c:function:: const char* PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *size) @@ -979,6 +1033,9 @@ These are the UTF-8 codec APIs: On error, set an exception, set *size* to ``-1`` (if it's not NULL) and return ``NULL``. + The function fails if the string contains surrogate code points + (``U+D800`` - ``U+DFFF``). + This caches the UTF-8 representation of the string in the Unicode object, and subsequent calls will return a pointer to the same buffer. The caller is not responsible for deallocating the buffer. The buffer is deallocated and @@ -997,6 +1054,15 @@ These are the UTF-8 codec APIs: As :c:func:`PyUnicode_AsUTF8AndSize`, but does not store the size. + .. warning:: + + This function does not have any special behavior for + `null characters `_ embedded within + *unicode*. As a result, strings containing null characters will remain in the returned + string, which some C functions might interpret as the end of the string, leading to + truncation. If truncation is an issue, it is recommended to use :c:func:`PyUnicode_AsUTF8AndSize` + instead. + .. versionadded:: 3.3 .. versionchanged:: 3.7 @@ -1400,14 +1466,40 @@ They all return ``NULL`` or ``-1`` if an exception occurs. This function returns ``-1`` upon failure, so one should call :c:func:`PyErr_Occurred` to check for errors. + .. seealso:: + + The :c:func:`PyUnicode_Equal` function. + + +.. c:function:: int PyUnicode_Equal(PyObject *a, PyObject *b) + + Test if two strings are equal: + + * Return ``1`` if *a* is equal to *b*. + * Return ``0`` if *a* is not equal to *b*. + * Set a :exc:`TypeError` exception and return ``-1`` if *a* or *b* is not a + :class:`str` object. + + The function always succeeds if *a* and *b* are :class:`str` objects. + + The function works for :class:`str` subclasses, but does not honor custom + ``__eq__()`` method. + + .. seealso:: + + The :c:func:`PyUnicode_Compare` function. + + .. versionadded:: 3.14 + .. c:function:: int PyUnicode_EqualToUTF8AndSize(PyObject *unicode, const char *string, Py_ssize_t size) Compare a Unicode object with a char buffer which is interpreted as being UTF-8 or ASCII encoded and return true (``1``) if they are equal, or false (``0``) otherwise. - If the Unicode object contains surrogate characters or - the C string is not valid UTF-8, false (``0``) is returned. + If the Unicode object contains surrogate code points + (``U+D800`` - ``U+DFFF``) or the C string is not valid UTF-8, + false (``0``) is returned. This function does not raise exceptions. @@ -1467,15 +1559,173 @@ They all return ``NULL`` or ``-1`` if an exception occurs. existing interned string that is the same as :c:expr:`*p_unicode`, it sets :c:expr:`*p_unicode` to it (releasing the reference to the old string object and creating a new :term:`strong reference` to the interned string object), otherwise it leaves - :c:expr:`*p_unicode` alone and interns it (creating a new :term:`strong reference`). + :c:expr:`*p_unicode` alone and interns it. + (Clarification: even though there is a lot of talk about references, think - of this function as reference-neutral; you own the object after the call - if and only if you owned it before the call.) + of this function as reference-neutral. You must own the object you pass in; + after the call you no longer own the passed-in reference, but you newly own + the result.) + + This function never raises an exception. + On error, it leaves its argument unchanged without interning it. + + Instances of subclasses of :py:class:`str` may not be interned, that is, + :c:expr:`PyUnicode_CheckExact(*p_unicode)` must be true. If it is not, + then -- as with any other error -- the argument is left unchanged. + + Note that interned strings are not “immortal”. + You must keep a reference to the result to benefit from interning. .. c:function:: PyObject* PyUnicode_InternFromString(const char *str) A combination of :c:func:`PyUnicode_FromString` and - :c:func:`PyUnicode_InternInPlace`, returning either a new Unicode string - object that has been interned, or a new ("owned") reference to an earlier - interned string object with the same value. + :c:func:`PyUnicode_InternInPlace`, meant for statically allocated strings. + + Return a new ("owned") reference to either a new Unicode string object + that has been interned, or an earlier interned string object with the + same value. + + Python may keep a reference to the result, or make it :term:`immortal`, + preventing it from being garbage-collected promptly. + For interning an unbounded number of different strings, such as ones coming + from user input, prefer calling :c:func:`PyUnicode_FromString` and + :c:func:`PyUnicode_InternInPlace` directly. + + .. impl-detail:: + + Strings interned this way are made :term:`immortal`. + + +PyUnicodeWriter +^^^^^^^^^^^^^^^ + +The :c:type:`PyUnicodeWriter` API can be used to create a Python :class:`str` +object. + +.. versionadded:: 3.14 + +.. c:type:: PyUnicodeWriter + + A Unicode writer instance. + + The instance must be destroyed by :c:func:`PyUnicodeWriter_Finish` on + success, or :c:func:`PyUnicodeWriter_Discard` on error. + +.. c:function:: PyUnicodeWriter* PyUnicodeWriter_Create(Py_ssize_t length) + + Create a Unicode writer instance. + + *length* must be greater than or equal to ``0``. + + If *length* is greater than ``0``, preallocate an internal buffer of + *length* characters. + + Set an exception and return ``NULL`` on error. + +.. c:function:: PyObject* PyUnicodeWriter_Finish(PyUnicodeWriter *writer) + + Return the final Python :class:`str` object and destroy the writer instance. + + Set an exception and return ``NULL`` on error. + + The writer instance is invalid after this call. + +.. c:function:: void PyUnicodeWriter_Discard(PyUnicodeWriter *writer) + + Discard the internal Unicode buffer and destroy the writer instance. + + If *writer* is ``NULL``, no operation is performed. + + The writer instance is invalid after this call. + +.. c:function:: int PyUnicodeWriter_WriteChar(PyUnicodeWriter *writer, Py_UCS4 ch) + + Write the single Unicode character *ch* into *writer*. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_WriteUTF8(PyUnicodeWriter *writer, const char *str, Py_ssize_t size) + + Decode the string *str* from UTF-8 in strict mode and write the output into *writer*. + + *size* is the string length in bytes. If *size* is equal to ``-1``, call + ``strlen(str)`` to get the string length. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + + See also :c:func:`PyUnicodeWriter_DecodeUTF8Stateful`. + +.. c:function:: int PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *writer, const wchar_t *str, Py_ssize_t size) + + Writer the wide string *str* into *writer*. + + *size* is a number of wide characters. If *size* is equal to ``-1``, call + ``wcslen(str)`` to get the string length. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *writer, Py_UCS4 *str, Py_ssize_t size) + + Writer the UCS4 string *str* into *writer*. + + *size* is a number of UCS4 characters. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_WriteStr(PyUnicodeWriter *writer, PyObject *obj) + + Call :c:func:`PyObject_Str` on *obj* and write the output into *writer*. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) + + Call :c:func:`PyObject_Repr` on *obj* and write the output into *writer*. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_WriteSubstring(PyUnicodeWriter *writer, PyObject *str, Py_ssize_t start, Py_ssize_t end) + + Write the substring ``str[start:end]`` into *writer*. + + *str* must be Python :class:`str` object. *start* must be greater than or + equal to 0, and less than or equal to *end*. *end* must be less than or + equal to *str* length. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_Format(PyUnicodeWriter *writer, const char *format, ...) + + Similar to :c:func:`PyUnicode_FromFormat`, but write the output directly into *writer*. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_DecodeUTF8Stateful(PyUnicodeWriter *writer, const char *string, Py_ssize_t length, const char *errors, Py_ssize_t *consumed) + + Decode the string *str* from UTF-8 with *errors* error handler and write the + output into *writer*. + + *size* is the string length in bytes. If *size* is equal to ``-1``, call + ``strlen(str)`` to get the string length. + + *errors* is an error handler name, such as ``"replace"``. If *errors* is + ``NULL``, use the strict error handler. + + If *consumed* is not ``NULL``, set *\*consumed* to the number of decoded + bytes on success. + If *consumed* is ``NULL``, treat trailing incomplete UTF-8 byte sequences + as an error. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + + See also :c:func:`PyUnicodeWriter_WriteUTF8`. diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 67167444d0a685..1ef4181d52eb10 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -25,30 +25,6 @@ are only passed to these functions if it is certain that they were created by the same library that the Python runtime is using. -.. c:function:: int Py_Main(int argc, wchar_t **argv) - - The main program for the standard interpreter. This is made available for - programs which embed Python. The *argc* and *argv* parameters should be - prepared exactly as those which are passed to a C program's :c:func:`main` - function (converted to wchar_t according to the user's locale). It is - important to note that the argument list may be modified (but the contents of - the strings pointed to by the argument list are not). The return value will - be ``0`` if the interpreter exits normally (i.e., without an exception), - ``1`` if the interpreter exits due to an exception, or ``2`` if the parameter - list does not represent a valid Python command line. - - Note that if an otherwise unhandled :exc:`SystemExit` is raised, this - function will not return ``1``, but exit the process, as long as - :c:member:`PyConfig.inspect` is zero. - - -.. c:function:: int Py_BytesMain(int argc, char **argv) - - Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings. - - .. versionadded:: 3.8 - - .. c:function:: int PyRun_AnyFile(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_AnyFileExFlags` below, leaving @@ -372,8 +348,20 @@ the same library that the Python runtime is using. .. versionchanged:: 3.8 Added *cf_feature_version* field. + The available compiler flags are accessible as macros: + + .. c:namespace:: NULL + + .. c:macro:: PyCF_ALLOW_TOP_LEVEL_AWAIT + PyCF_ONLY_AST + PyCF_OPTIMIZED_AST + PyCF_TYPE_COMMENTS + + See :ref:`compiler flags ` in documentation of the + :py:mod:`!ast` Python module, which exports these constants under + the same names. -.. c:var:: int CO_FUTURE_DIVISION + .. c:var:: int CO_FUTURE_DIVISION - This bit can be set in *flags* to cause division operator ``/`` to be - interpreted as "true division" according to :pep:`238`. + This bit can be set in *flags* to cause division operator ``/`` to be + interpreted as "true division" according to :pep:`238`. diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index 038f54a9751fd1..c3c6cf413dcef5 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -35,7 +35,7 @@ as much as it can. callable object that receives notification when *ob* is garbage collected; it should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* is not a - weakly referencable object, or if *callback* is not callable, ``None``, or + weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. @@ -47,7 +47,7 @@ as much as it can. be a callable object that receives notification when *ob* is garbage collected; it should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* - is not a weakly referencable object, or if *callback* is not callable, + is not a weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. @@ -88,6 +88,15 @@ as much as it can. Use :c:func:`PyWeakref_GetRef` instead. +.. c:function:: int PyWeakref_IsDead(PyObject *ref) + + Test if the weak reference *ref* is dead. Returns 1 if the reference is + dead, 0 if it is alive, and -1 with an error set if *ref* is not a weak + reference object. + + .. versionadded:: 3.14 + + .. c:function:: void PyObject_ClearWeakRefs(PyObject *object) This function is called by the :c:member:`~PyTypeObject.tp_dealloc` handler @@ -96,3 +105,19 @@ as much as it can. This iterates through the weak references for *object* and calls callbacks for those references which have one. It returns when all callbacks have been attempted. + + +.. c:function:: void PyUnstable_Object_ClearWeakRefsNoCallbacks(PyObject *object) + + Clears the weakrefs for *object* without calling the callbacks. + + This function is called by the :c:member:`~PyTypeObject.tp_dealloc` handler + for types with finalizers (i.e., :meth:`~object.__del__`). The handler for + those objects first calls :c:func:`PyObject_ClearWeakRefs` to clear weakrefs + and call their callbacks, then the finalizer, and finally this function to + clear any weakrefs that may have been created by the finalizer. + + In most circumstances, it's more appropriate to use + :c:func:`PyObject_ClearWeakRefs` to clear weakrefs instead of this function. + + .. versionadded:: 3.13 diff --git a/Doc/conf.py b/Doc/conf.py index f4c75c5758cb28..1aeecaeb3073f5 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -6,35 +6,44 @@ # The contents of this file are pickled, so don't put values in the namespace # that aren't pickleable (module imports are okay, they're removed automatically). +import importlib import os import sys -import time + +# Make our custom extensions available to Sphinx sys.path.append(os.path.abspath('tools/extensions')) sys.path.append(os.path.abspath('includes')) +# Python specific content from Doc/Tools/extensions/pyspecific.py +from pyspecific import SOURCE_URI + # General configuration # --------------------- +# Our custom Sphinx extensions are found in Doc/Tools/extensions/ extensions = [ - 'asdl_highlight', + 'audit_events', + 'availability', 'c_annotations', - 'escape4chm', + 'changes', 'glossary_search', - 'peg_highlight', + 'lexers', + 'pydoc_topics', 'pyspecific', 'sphinx.ext.coverage', 'sphinx.ext.doctest', + 'sphinx.ext.extlinks', ] # Skip if downstream redistributors haven't installed them try: - import notfound.extension + import notfound.extension # noqa: F401 except ImportError: pass else: extensions.append('notfound.extension') try: - import sphinxext.opengraph + import sphinxext.opengraph # noqa: F401 except ImportError: pass else: @@ -47,7 +56,7 @@ except ImportError: _tkinter = None # Treat warnings as errors, done here to prevent warnings in Sphinx code from -# causing spurious test failures. +# causing spurious CPython test failures. import warnings warnings.simplefilter('error') del warnings @@ -57,37 +66,45 @@ # General substitutions. project = 'Python' -copyright = f"2001-{time.strftime('%Y')}, Python Software Foundation" +copyright = "2001 Python Software Foundation" # We look for the Include/patchlevel.h file in the current Python source tree # and replace the values accordingly. -import patchlevel -version, release = patchlevel.get_version_info() +# See Doc/tools/extensions/patchlevel.py +version, release = importlib.import_module('patchlevel').get_version_info() rst_epilog = f""" .. |python_version_literal| replace:: ``Python {version}`` .. |python_x_dot_y_literal| replace:: ``python{version}`` .. |usr_local_bin_python_x_dot_y_literal| replace:: ``/usr/local/bin/python{version}`` + +.. Apparently this how you hack together a formatted link: + (https://www.docutils.org/docs/ref/rst/directives.html#replacement-text) +.. |FORCE_COLOR| replace:: ``FORCE_COLOR`` +.. _FORCE_COLOR: https://force-color.org/ +.. |NO_COLOR| replace:: ``NO_COLOR`` +.. _NO_COLOR: https://no-color.org/ """ -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: +# There are two options for replacing |today|. Either, you set today to some +# non-false value and use it. today = '' -# Else, today_fmt is used as the format for a strftime call. +# Or else, today_fmt is used as the format for a strftime call. today_fmt = '%B %d, %Y' # By default, highlight as Python 3. highlight_language = 'python3' # Minimum version of sphinx required -needs_sphinx = '4.2' +# Keep this version in sync with ``Doc/requirements.txt``. +needs_sphinx = '8.1.3' # Create table of contents entries for domain objects (e.g. functions, classes, # attributes, etc.). Default is True. toc_object_entries = False # Ignore any .rst files in the includes/ directory; -# they're embedded in pages but not rendered individually. +# they're embedded in pages but not rendered as individual pages. # Ignore any .rst files in the venv/ directory. exclude_patterns = ['includes/*.rst', 'venv/*', 'README.rst'] venvdir = os.getenv('VENVDIR') @@ -128,6 +145,9 @@ ('c:func', 'vsnprintf'), # Standard C types ('c:type', 'FILE'), + ('c:type', 'int8_t'), + ('c:type', 'int16_t'), + ('c:type', 'int32_t'), ('c:type', 'int64_t'), ('c:type', 'intmax_t'), ('c:type', 'off_t'), @@ -136,6 +156,9 @@ ('c:type', 'size_t'), ('c:type', 'ssize_t'), ('c:type', 'time_t'), + ('c:type', 'uint8_t'), + ('c:type', 'uint16_t'), + ('c:type', 'uint32_t'), ('c:type', 'uint64_t'), ('c:type', 'uintmax_t'), ('c:type', 'uintptr_t'), @@ -178,6 +201,7 @@ ('envvar', 'LC_TIME'), ('envvar', 'LINES'), ('envvar', 'LOGNAME'), + ('envvar', 'MANPAGER'), ('envvar', 'PAGER'), ('envvar', 'PATH'), ('envvar', 'PATHEXT'), @@ -238,6 +262,7 @@ ('c:data', 'PyExc_OverflowError'), ('c:data', 'PyExc_PermissionError'), ('c:data', 'PyExc_ProcessLookupError'), + ('c:data', 'PyExc_PythonFinalizationError'), ('c:data', 'PyExc_RecursionError'), ('c:data', 'PyExc_ReferenceError'), ('c:data', 'PyExc_RuntimeError'), @@ -268,6 +293,9 @@ ('c:data', 'PyExc_UnicodeWarning'), ('c:data', 'PyExc_UserWarning'), ('c:data', 'PyExc_Warning'), + # Undocumented public C macros + ('c:macro', 'Py_BUILD_ASSERT'), + ('c:macro', 'Py_BUILD_ASSERT_EXPR'), # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot # be resolved, as the method is currently undocumented. For context, see # https://github.com/python/cpython/pull/103289. @@ -292,32 +320,37 @@ # Disable Docutils smartquotes for several translations smartquotes_excludes = { - 'languages': ['ja', 'fr', 'zh_TW', 'zh_CN'], 'builders': ['man', 'text'], + 'languages': ['ja', 'fr', 'zh_TW', 'zh_CN'], + 'builders': ['man', 'text'], } -# Avoid a warning with Sphinx >= 2.0 -master_doc = 'contents' +# Avoid a warning with Sphinx >= 4.0 +root_doc = 'contents' # Allow translation of index directives gettext_additional_targets = [ 'index', + 'literal-block', ] # Options for HTML output # ----------------------- -# Use our custom theme. +# Use our custom theme: https://github.com/python/python-docs-theme html_theme = 'python_docs_theme' +# Location of overrides for theme templates and static files html_theme_path = ['tools'] html_theme_options = { 'collapsiblesidebar': True, 'issues_url': '/bugs.html', 'license_url': '/license.html', - 'root_include_title': False # We use the version switcher instead. + 'root_include_title': False, # We use the version switcher instead. } if os.getenv("READTHEDOCS"): - html_theme_options["hosted_on"] = 'Read the Docs' + html_theme_options["hosted_on"] = ( + 'Read the Docs' + ) # Override stylesheet fingerprinting for Windows CHM htmlhelp to fix GH-91207 # https://github.com/python/cpython/issues/91207 @@ -331,17 +364,21 @@ # Deployment preview information # (See .readthedocs.yml and https://docs.readthedocs.io/en/stable/reference/environment-variables.html) -repository_url = os.getenv("READTHEDOCS_GIT_CLONE_URL") +is_deployment_preview = os.getenv("READTHEDOCS_VERSION_TYPE") == "external" +repository_url = os.getenv("READTHEDOCS_GIT_CLONE_URL", "") +repository_url = repository_url.removesuffix(".git") html_context = { - "is_deployment_preview": os.getenv("READTHEDOCS_VERSION_TYPE") == "external", - "repository_url": repository_url.removesuffix(".git") if repository_url else None, - "pr_id": os.getenv("READTHEDOCS_VERSION") + "is_deployment_preview": is_deployment_preview, + "repository_url": repository_url or None, + "pr_id": os.getenv("READTHEDOCS_VERSION"), + "enable_analytics": os.getenv("PYTHON_DOCS_ENABLE_ANALYTICS"), } # This 'Last updated on:' timestamp is inserted at the bottom of every page. -html_last_updated_fmt = time.strftime('%b %d, %Y (%H:%M UTC)', time.gmtime()) +html_last_updated_fmt = '%b %d, %Y (%H:%M UTC)' +html_last_updated_use_utc = True -# Path to find HTML templates. +# Path to find HTML templates to override theme templates_path = ['tools/templates'] # Custom sidebar templates, filenames relative to this file. @@ -370,6 +407,8 @@ # Split the index html_split_index = True +# Split pot files one per reST file +gettext_compact = False # Options for LaTeX output # ------------------------ @@ -387,8 +426,8 @@ \let\endVerbatim=\endOriginalVerbatim \setcounter{tocdepth}{2} ''', - # The paper size ('letter' or 'a4'). - 'papersize': 'a4', + # The paper size ('letterpaper' or 'a4paper'). + 'papersize': 'a4paper', # The font size ('10pt', '11pt' or '12pt'). 'pointsize': '10pt', } @@ -397,30 +436,70 @@ # (source start file, target name, title, author, document class [howto/manual]). _stdauthor = 'Guido van Rossum and the Python development team' latex_documents = [ - ('c-api/index', 'c-api.tex', - 'The Python/C API', _stdauthor, 'manual'), - ('extending/index', 'extending.tex', - 'Extending and Embedding Python', _stdauthor, 'manual'), - ('installing/index', 'installing.tex', - 'Installing Python Modules', _stdauthor, 'manual'), - ('library/index', 'library.tex', - 'The Python Library Reference', _stdauthor, 'manual'), - ('reference/index', 'reference.tex', - 'The Python Language Reference', _stdauthor, 'manual'), - ('tutorial/index', 'tutorial.tex', - 'Python Tutorial', _stdauthor, 'manual'), - ('using/index', 'using.tex', - 'Python Setup and Usage', _stdauthor, 'manual'), - ('faq/index', 'faq.tex', - 'Python Frequently Asked Questions', _stdauthor, 'manual'), - ('whatsnew/' + version, 'whatsnew.tex', - 'What\'s New in Python', 'A. M. Kuchling', 'howto'), + ('c-api/index', 'c-api.tex', 'The Python/C API', _stdauthor, 'manual'), + ( + 'extending/index', + 'extending.tex', + 'Extending and Embedding Python', + _stdauthor, + 'manual', + ), + ( + 'installing/index', + 'installing.tex', + 'Installing Python Modules', + _stdauthor, + 'manual', + ), + ( + 'library/index', + 'library.tex', + 'The Python Library Reference', + _stdauthor, + 'manual', + ), + ( + 'reference/index', + 'reference.tex', + 'The Python Language Reference', + _stdauthor, + 'manual', + ), + ( + 'tutorial/index', + 'tutorial.tex', + 'Python Tutorial', + _stdauthor, + 'manual', + ), + ( + 'using/index', + 'using.tex', + 'Python Setup and Usage', + _stdauthor, + 'manual', + ), + ( + 'faq/index', + 'faq.tex', + 'Python Frequently Asked Questions', + _stdauthor, + 'manual', + ), + ( + 'whatsnew/' + version, + 'whatsnew.tex', + 'What\'s New in Python', + 'A. M. Kuchling', + 'howto', + ), ] # Collect all HOWTOs individually -latex_documents.extend(('howto/' + fn[:-4], 'howto-' + fn[:-4] + '.tex', - '', _stdauthor, 'howto') - for fn in os.listdir('howto') - if fn.endswith('.rst') and fn != 'index.rst') +latex_documents.extend( + ('howto/' + fn[:-4], 'howto-' + fn[:-4] + '.tex', '', _stdauthor, 'howto') + for fn in os.listdir('howto') + if fn.endswith('.rst') and fn != 'index.rst' +) # Documents to append as an appendix to all manuals. latex_appendices = ['glossary', 'about', 'license', 'copyright'] @@ -431,6 +510,10 @@ epub_author = 'Python Documentation Authors' epub_publisher = 'Python Software Foundation' +# index pages are not valid xhtml +# https://github.com/sphinx-doc/sphinx/issues/12359 +epub_use_index = False + # Options for the coverage checker # -------------------------------- @@ -444,8 +527,7 @@ 'test($|_)', ] -coverage_ignore_classes = [ -] +coverage_ignore_classes = [] # Glob patterns for C source files for C API coverage, relative to this directory. coverage_c_path = [ @@ -462,7 +544,7 @@ # The coverage checker will ignore all C items whose names match these regexes # (using re.match) -- the keys must be the same as in coverage_c_regexes. coverage_ignore_c_items = { -# 'cfunction': [...] + # 'cfunction': [...] } @@ -478,15 +560,19 @@ r'https://github.com/python/cpython/tree/.*': 'https://github.com/python/cpython/blob/.*', # Intentional HTTP use at Misc/NEWS.d/3.5.0a1.rst r'http://www.python.org/$': 'https://www.python.org/$', - # Used in license page, keep as is - r'https://www.zope.org/': r'https://www.zope.dev/', # Microsoft's redirects to learn.microsoft.com r'https://msdn.microsoft.com/.*': 'https://learn.microsoft.com/.*', r'https://docs.microsoft.com/.*': 'https://learn.microsoft.com/.*', r'https://go.microsoft.com/fwlink/\?LinkID=\d+': 'https://learn.microsoft.com/.*', + # Debian's man page redirects to its current stable version + r'https://manpages.debian.org/\w+\(\d(\w+)?\)': r'https://manpages.debian.org/\w+/[\w/\-\.]*\.\d(\w+)?\.en\.html', # Language redirects r'https://toml.io': 'https://toml.io/en/', r'https://www.redhat.com': 'https://www.redhat.com/en', + # pypi.org project name normalization (upper to lowercase, underscore to hyphen) + r'https://pypi.org/project/[A-Za-z\d_\-\.]+/': r'https://pypi.org/project/[a-z\d\-\.]+/', + # Discourse title name expansion (text changes when title is edited) + r'https://discuss\.python\.org/t/\d+': r'https://discuss\.python\.org/t/.*/\d+', # Other redirects r'https://www.boost.org/libs/.+': r'https://www.boost.org/doc/libs/\d_\d+_\d/.+', r'https://support.microsoft.com/en-us/help/\d+': 'https://support.microsoft.com/en-us/topic/.+', @@ -513,15 +599,28 @@ r'https://unix.org/version2/whatsnew/lp64_wp.html', ] +# Options for sphinx.ext.extlinks +# ------------------------------- + +# This config is a dictionary of external sites, +# mapping unique short aliases to a base URL and a prefix. +# https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html +extlinks = { + "pypi": ("https://pypi.org/project/%s/", "%s"), + "source": (SOURCE_URI, "%s"), +} +extlinks_detect_hardcoded_links = True -# Options for extensions -# ---------------------- +# Options for c_annotations extension +# ----------------------------------- # Relative filename of the data files refcount_file = 'data/refcounts.dat' stable_abi_file = 'data/stable_abi.dat' -# sphinxext-opengraph config +# Options for sphinxext-opengraph +# ------------------------------- + ogp_site_url = 'https://docs.python.org/3/' ogp_site_name = 'Python documentation' ogp_image = '_static/og-image.png' diff --git a/Doc/constraints.txt b/Doc/constraints.txt index 147de1271eb2b7..29cd4be1d3c8db 100644 --- a/Doc/constraints.txt +++ b/Doc/constraints.txt @@ -7,18 +7,18 @@ # Direct dependencies of Sphinx babel<3 colorama<0.5 -imagesize<1.5 -Jinja2<3.2 -packaging<24 -Pygments>=2.16.1,<3 +imagesize<2 +Jinja2<4 +packaging<25 +Pygments<3 requests<3 snowballstemmer<3 -sphinxcontrib-applehelp<1.1 -sphinxcontrib-devhelp<1.1 -sphinxcontrib-htmlhelp<2.1 -sphinxcontrib-jsmath<1.1 -sphinxcontrib-qthelp<1.1 -sphinxcontrib-serializinghtml<1.2 +sphinxcontrib-applehelp<3 +sphinxcontrib-devhelp<3 +sphinxcontrib-htmlhelp<3 +sphinxcontrib-jsmath<2 +sphinxcontrib-qthelp<3 +sphinxcontrib-serializinghtml<3 # Direct dependencies of Jinja2 (Jinja is a dependency of Sphinx, see above) -MarkupSafe<2.2 +MarkupSafe<3 diff --git a/Doc/contents.rst b/Doc/contents.rst index 24ceacb0076b5e..b57f4b09a5dcb6 100644 --- a/Doc/contents.rst +++ b/Doc/contents.rst @@ -14,6 +14,7 @@ installing/index.rst howto/index.rst faq/index.rst + deprecations/index.rst glossary.rst about.rst diff --git a/Doc/copyright.rst b/Doc/copyright.rst index 8629ed1fc38009..9210d5f50ed841 100644 --- a/Doc/copyright.rst +++ b/Doc/copyright.rst @@ -4,7 +4,7 @@ Copyright Python and this documentation is: -Copyright © 2001-2024 Python Software Foundation. All rights reserved. +Copyright © 2001 Python Software Foundation. All rights reserved. Copyright © 2000 BeOpen.com. All rights reserved. diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 62a96146d605ff..e78754e24e23d8 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -180,7 +180,7 @@ PyCapsule_IsValid:const char*:name:: PyCapsule_New:PyObject*::+1: PyCapsule_New:void*:pointer:: PyCapsule_New:const char *:name:: -PyCapsule_New::void (* destructor)(PyObject* ):: +PyCapsule_New:void (*)(PyObject *):destructor:: PyCapsule_SetContext:int::: PyCapsule_SetContext:PyObject*:self:0: @@ -349,11 +349,11 @@ PyComplex_CheckExact:int::: PyComplex_CheckExact:PyObject*:p:0: PyComplex_FromCComplex:PyObject*::+1: -PyComplex_FromCComplex::Py_complex v:: +PyComplex_FromCComplex:Py_complex:v:: PyComplex_FromDoubles:PyObject*::+1: -PyComplex_FromDoubles::double real:: -PyComplex_FromDoubles::double imag:: +PyComplex_FromDoubles:double:real:: +PyComplex_FromDoubles:double:imag:: PyComplex_ImagAsDouble:double::: PyComplex_ImagAsDouble:PyObject*:op:0: @@ -364,8 +364,6 @@ PyComplex_RealAsDouble:PyObject*:op:0: PyContext_CheckExact:int::: PyContext_CheckExact:PyObject*:o:0: -PyContext_ClearFreeList:int::: - PyContext_Copy:PyObject*::+1: PyContext_Copy:PyObject*:ctx:0: @@ -622,7 +620,9 @@ PyErr_GetExcInfo:PyObject**:pvalue:+1: PyErr_GetExcInfo:PyObject**:ptraceback:+1: PyErr_GetRaisedException:PyObject*::+1: -PyErr_SetRaisedException:::: + +PyErr_SetRaisedException:void::: +PyErr_SetRaisedException:PyObject *:exc:0:stolen PyErr_GivenExceptionMatches:int::: PyErr_GivenExceptionMatches:PyObject*:given:0: @@ -642,9 +642,9 @@ PyErr_NewExceptionWithDoc:PyObject*:dict:0: PyErr_NoMemory:PyObject*::null: PyErr_NormalizeException:void::: -PyErr_NormalizeException:PyObject**:exc::??? -PyErr_NormalizeException:PyObject**:val::??? -PyErr_NormalizeException:PyObject**:tb::??? +PyErr_NormalizeException:PyObject**:exc:+1:??? +PyErr_NormalizeException:PyObject**:val:+1:??? +PyErr_NormalizeException:PyObject**:tb:+1:??? PyErr_Occurred:PyObject*::0: @@ -790,6 +790,12 @@ PyEval_GetGlobals:PyObject*::0: PyEval_GetFrame:PyObject*::0: +PyEval_GetFrameBuiltins:PyObject*::+1: + +PyEval_GetFrameLocals:PyObject*::+1: + +PyEval_GetFrameGlobals:PyObject*::+1: + PyEval_GetFuncDesc:const char*::: PyEval_GetFuncDesc:PyObject*:func:0: @@ -916,6 +922,32 @@ PyFloat_FromString:PyObject*:str:0: PyFloat_GetInfo:PyObject*::+1: PyFloat_GetInfo::void:: +PyFrame_GetBack:PyObject*::+1: +PyFrame_GetBack:PyFrameObject*:frame:0: + +PyFrame_GetBuiltins:PyObject*::+1: +PyFrame_GetBuiltins:PyFrameObject*:frame:0: + +PyFrame_GetCode:PyObject*::+1: +PyFrame_GetCode:PyFrameObject*:frame:0: + +PyFrame_GetGenerator:PyObject*::+1: +PyFrame_GetGenerator:PyFrameObject*:frame:0: + +PyFrame_GetGlobals:PyObject*::+1: +PyFrame_GetGlobals:PyFrameObject*:frame:0: + +PyFrame_GetLocals:PyObject*::+1: +PyFrame_GetLocals:PyFrameObject*:frame:0: + +PyFrame_GetVar:PyObject*::+1: +PyFrame_GetVar:PyFrameObject*:frame:0: +PyFrame_GetVar:PyObject*:name:0: + +PyFrame_GetVarString:PyObject*::+1: +PyFrame_GetVarString:PyFrameObject*:frame:0: +PyFrame_GetVarString:const char*:name:: + PyFrozenSet_Check:int::: PyFrozenSet_Check:PyObject*:p:0: @@ -998,8 +1030,6 @@ PyImport_AddModule:const char*:name:: PyImport_AddModuleObject:PyObject*::0:reference borrowed from sys.modules PyImport_AddModuleObject:PyObject*:name:0: -PyImport_Cleanup:void::: - PyImport_ExecCodeModule:PyObject*::+1: PyImport_ExecCodeModule:const char*:name:: PyImport_ExecCodeModule:PyObject*:co:0: @@ -1104,6 +1134,10 @@ PyAIter_Check:PyObject*:o:0: PyIter_Next:PyObject*::+1: PyIter_Next:PyObject*:o:0: +PyIter_NextItem:int::: +PyIter_NextItem:PyObject*:iter:0: +PyIter_NextItem:PyObject**:item:+1: + PyIter_Send:int::: PyIter_Send:PyObject*:iter:0: PyIter_Send:PyObject*:arg:0: @@ -1252,6 +1286,26 @@ PyLong_FromUnsignedLong:unsignedlong:v:: PyLong_FromVoidPtr:PyObject*::+1: PyLong_FromVoidPtr:void*:p:: +PyLong_IsPositive:int::: +PyLong_IsPositive:PyObject*:obj:0: + +PyLong_IsNegative:int::: +PyLong_IsNegative:PyObject*:obj:0: + +PyLong_IsZero:int::: +PyLong_IsZero:PyObject*:obj:0: + +PyLong_GetSign:int::: +PyLong_GetSign:PyObject*:v:0: +PyLong_GetSign:int*:sign:: + +PyLong_Export:int::: +PyLong_Export:PyObject*:obj:0: +PyLong_Export:PyLongExport*:export_long:: + +PyLongWriter_Finish:PyObject*::+1: +PyLongWriter_Finish:PyLongWriter*:writer:: + PyMapping_Check:int::: PyMapping_Check:PyObject*:o:0: @@ -1269,7 +1323,7 @@ PyMapping_GetItemString:const char*:key:: PyMapping_HasKey:int::: PyMapping_HasKey:PyObject*:o:0: -PyMapping_HasKey:PyObject*:key:: +PyMapping_HasKey:PyObject*:key:0: PyMapping_HasKeyString:int::: PyMapping_HasKeyString:PyObject*:o:0: @@ -1429,7 +1483,7 @@ PyModule_GetState:void*::: PyModule_GetState:PyObject*:module:0: PyModule_New:PyObject*::+1: -PyModule_New::char* name:: +PyModule_New:char*:name:: PyModule_NewObject:PyObject*::+1: PyModule_NewObject:PyObject*:name:+1: @@ -1439,7 +1493,7 @@ PyModule_SetDocString:PyObject*:module:0: PyModule_SetDocString:const char*:docstring:: PyModuleDef_Init:PyObject*::0: -PyModuleDef_Init:PyModuleDef*:def:0: +PyModuleDef_Init:PyModuleDef*:def:: PyNumber_Absolute:PyObject*::+1: PyNumber_Absolute:PyObject*:o:0: @@ -1802,6 +1856,9 @@ PyObject_RichCompareBool:PyObject*:o1:0: PyObject_RichCompareBool:PyObject*:o2:0: PyObject_RichCompareBool:int:opid:: +PyObject_SelfIter:PyObject*::+1: +PyObject_SelfIter:PyObject*:obj:0: + PyObject_SetAttr:int::: PyObject_SetAttr:PyObject*:o:0: PyObject_SetAttr:PyObject*:attr_name:0: @@ -1939,10 +1996,10 @@ PyRun_StringFlags:PyObject*:locals:0: PyRun_StringFlags:PyCompilerFlags*:flags:: PySeqIter_Check:int::: -PySeqIter_Check::op:: +PySeqIter_Check:PyObject *:op:0: PySeqIter_New:PyObject*::+1: -PySeqIter_New:PyObject*:seq:: +PySeqIter_New:PyObject*:seq:0: PySequence_Check:int::: PySequence_Check:PyObject*:o:0: @@ -2373,16 +2430,10 @@ PyUnicode_DATA:PyObject*:o:0: PyUnicode_GET_LENGTH:Py_ssize_t::: PyUnicode_GET_LENGTH:PyObject*:o:0: -PyUnicode_GET_SIZE:Py_ssize_t::: -PyUnicode_GET_SIZE:PyObject*:o:0: - -PyUnicode_GET_DATA_SIZE:Py_ssize_t::: -PyUnicode_GET_DATA_SIZE:PyObject*:o:0: - PyUnicode_KIND:int::: PyUnicode_KIND:PyObject*:o:0: -PyUnicode_MAX_CHAR_VALUE:::: +PyUnicode_MAX_CHAR_VALUE:Py_UCS4::: PyUnicode_MAX_CHAR_VALUE:PyObject*:o:0: Py_UNICODE_ISALNUM:int::: @@ -2449,7 +2500,7 @@ PyUnicode_FromWideChar:const wchar_t*:w:: PyUnicode_FromWideChar:Py_ssize_t:size:: PyUnicode_AsWideChar:Py_ssize_t::: -PyUnicode_AsWideChar:PyObject*:*unicode:0: +PyUnicode_AsWideChar:PyObject*:unicode:0: PyUnicode_AsWideChar:wchar_t*:w:: PyUnicode_AsWideChar:Py_ssize_t:size:: @@ -2502,7 +2553,7 @@ PyUnicode_AsUTF8String:PyObject*:unicode:0: PyUnicode_AsUTF8AndSize:const char*::: PyUnicode_AsUTF8AndSize:PyObject*:unicode:0: -PyUnicode_AsUTF8AndSize:Py_ssize_t*:size:0: +PyUnicode_AsUTF8AndSize:Py_ssize_t*:size:: PyUnicode_AsUTF8:const char*::: PyUnicode_AsUTF8:PyObject*:unicode:0: @@ -2825,13 +2876,13 @@ PyUnicodeDecodeError_SetStart:PyObject*:exc:0: PyUnicodeDecodeError_SetStart:Py_ssize_t:start:: PyWeakref_Check:int::: -PyWeakref_Check:PyObject*:ob:: +PyWeakref_Check:PyObject*:ob:0: PyWeakref_CheckProxy:int::: -PyWeakref_CheckProxy:PyObject*:ob:: +PyWeakref_CheckProxy:PyObject*:ob:0: PyWeakref_CheckRef:int::: -PyWeakref_CheckRef:PyObject*:ob:: +PyWeakref_CheckRef:PyObject*:ob:0: PyWeakref_GET_OBJECT:PyObject*::0: PyWeakref_GET_OBJECT:PyObject*:ref:0: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 25629b4da053da..c15f82603aa944 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,875 +1,903 @@ role,name,added,ifdef_note,struct_abi_kind macro,PY_VECTORCALL_ARGUMENTS_OFFSET,3.12,, -function,PyAIter_Check,3.10,, -function,PyArg_Parse,3.2,, -function,PyArg_ParseTuple,3.2,, -function,PyArg_ParseTupleAndKeywords,3.2,, -function,PyArg_UnpackTuple,3.2,, -function,PyArg_VaParse,3.2,, -function,PyArg_VaParseTupleAndKeywords,3.2,, -function,PyArg_ValidateKeywordArguments,3.2,, -var,PyBaseObject_Type,3.2,, -function,PyBool_FromLong,3.2,, -var,PyBool_Type,3.2,, -function,PyBuffer_FillContiguousStrides,3.11,, -function,PyBuffer_FillInfo,3.11,, -function,PyBuffer_FromContiguous,3.11,, -function,PyBuffer_GetPointer,3.11,, -function,PyBuffer_IsContiguous,3.11,, -function,PyBuffer_Release,3.11,, -function,PyBuffer_SizeFromFormat,3.11,, -function,PyBuffer_ToContiguous,3.11,, -var,PyByteArrayIter_Type,3.2,, -function,PyByteArray_AsString,3.2,, -function,PyByteArray_Concat,3.2,, -function,PyByteArray_FromObject,3.2,, -function,PyByteArray_FromStringAndSize,3.2,, -function,PyByteArray_Resize,3.2,, -function,PyByteArray_Size,3.2,, -var,PyByteArray_Type,3.2,, -var,PyBytesIter_Type,3.2,, -function,PyBytes_AsString,3.2,, -function,PyBytes_AsStringAndSize,3.2,, -function,PyBytes_Concat,3.2,, -function,PyBytes_ConcatAndDel,3.2,, -function,PyBytes_DecodeEscape,3.2,, -function,PyBytes_FromFormat,3.2,, -function,PyBytes_FromFormatV,3.2,, -function,PyBytes_FromObject,3.2,, -function,PyBytes_FromString,3.2,, -function,PyBytes_FromStringAndSize,3.2,, -function,PyBytes_Repr,3.2,, -function,PyBytes_Size,3.2,, -var,PyBytes_Type,3.2,, +func,PyAIter_Check,3.10,, +func,PyArg_Parse,3.2,, +func,PyArg_ParseTuple,3.2,, +func,PyArg_ParseTupleAndKeywords,3.2,, +func,PyArg_UnpackTuple,3.2,, +func,PyArg_VaParse,3.2,, +func,PyArg_VaParseTupleAndKeywords,3.2,, +func,PyArg_ValidateKeywordArguments,3.2,, +data,PyBaseObject_Type,3.2,, +func,PyBool_FromLong,3.2,, +data,PyBool_Type,3.2,, +func,PyBuffer_FillContiguousStrides,3.11,, +func,PyBuffer_FillInfo,3.11,, +func,PyBuffer_FromContiguous,3.11,, +func,PyBuffer_GetPointer,3.11,, +func,PyBuffer_IsContiguous,3.11,, +func,PyBuffer_Release,3.11,, +func,PyBuffer_SizeFromFormat,3.11,, +func,PyBuffer_ToContiguous,3.11,, +data,PyByteArrayIter_Type,3.2,, +func,PyByteArray_AsString,3.2,, +func,PyByteArray_Concat,3.2,, +func,PyByteArray_FromObject,3.2,, +func,PyByteArray_FromStringAndSize,3.2,, +func,PyByteArray_Resize,3.2,, +func,PyByteArray_Size,3.2,, +data,PyByteArray_Type,3.2,, +data,PyBytesIter_Type,3.2,, +func,PyBytes_AsString,3.2,, +func,PyBytes_AsStringAndSize,3.2,, +func,PyBytes_Concat,3.2,, +func,PyBytes_ConcatAndDel,3.2,, +func,PyBytes_DecodeEscape,3.2,, +func,PyBytes_FromFormat,3.2,, +func,PyBytes_FromFormatV,3.2,, +func,PyBytes_FromObject,3.2,, +func,PyBytes_FromString,3.2,, +func,PyBytes_FromStringAndSize,3.2,, +func,PyBytes_Repr,3.2,, +func,PyBytes_Size,3.2,, +data,PyBytes_Type,3.2,, type,PyCFunction,3.2,, type,PyCFunctionFast,3.13,, type,PyCFunctionFastWithKeywords,3.13,, type,PyCFunctionWithKeywords,3.2,, -function,PyCFunction_GetFlags,3.2,, -function,PyCFunction_GetFunction,3.2,, -function,PyCFunction_GetSelf,3.2,, -function,PyCFunction_New,3.4,, -function,PyCFunction_NewEx,3.2,, -var,PyCFunction_Type,3.2,, -function,PyCMethod_New,3.9,, -function,PyCallIter_New,3.2,, -var,PyCallIter_Type,3.2,, -function,PyCallable_Check,3.2,, +func,PyCFunction_GetFlags,3.2,, +func,PyCFunction_GetFunction,3.2,, +func,PyCFunction_GetSelf,3.2,, +func,PyCFunction_New,3.4,, +func,PyCFunction_NewEx,3.2,, +data,PyCFunction_Type,3.2,, +func,PyCMethod_New,3.9,, +func,PyCallIter_New,3.2,, +data,PyCallIter_Type,3.2,, +func,PyCallable_Check,3.2,, type,PyCapsule_Destructor,3.2,, -function,PyCapsule_GetContext,3.2,, -function,PyCapsule_GetDestructor,3.2,, -function,PyCapsule_GetName,3.2,, -function,PyCapsule_GetPointer,3.2,, -function,PyCapsule_Import,3.2,, -function,PyCapsule_IsValid,3.2,, -function,PyCapsule_New,3.2,, -function,PyCapsule_SetContext,3.2,, -function,PyCapsule_SetDestructor,3.2,, -function,PyCapsule_SetName,3.2,, -function,PyCapsule_SetPointer,3.2,, -var,PyCapsule_Type,3.2,, -var,PyClassMethodDescr_Type,3.2,, -function,PyCodec_BackslashReplaceErrors,3.2,, -function,PyCodec_Decode,3.2,, -function,PyCodec_Decoder,3.2,, -function,PyCodec_Encode,3.2,, -function,PyCodec_Encoder,3.2,, -function,PyCodec_IgnoreErrors,3.2,, -function,PyCodec_IncrementalDecoder,3.2,, -function,PyCodec_IncrementalEncoder,3.2,, -function,PyCodec_KnownEncoding,3.2,, -function,PyCodec_LookupError,3.2,, -function,PyCodec_NameReplaceErrors,3.7,, -function,PyCodec_Register,3.2,, -function,PyCodec_RegisterError,3.2,, -function,PyCodec_ReplaceErrors,3.2,, -function,PyCodec_StreamReader,3.2,, -function,PyCodec_StreamWriter,3.2,, -function,PyCodec_StrictErrors,3.2,, -function,PyCodec_Unregister,3.10,, -function,PyCodec_XMLCharRefReplaceErrors,3.2,, -function,PyComplex_FromDoubles,3.2,, -function,PyComplex_ImagAsDouble,3.2,, -function,PyComplex_RealAsDouble,3.2,, -var,PyComplex_Type,3.2,, -function,PyDescr_NewClassMethod,3.2,, -function,PyDescr_NewGetSet,3.2,, -function,PyDescr_NewMember,3.2,, -function,PyDescr_NewMethod,3.2,, -var,PyDictItems_Type,3.2,, -var,PyDictIterItem_Type,3.2,, -var,PyDictIterKey_Type,3.2,, -var,PyDictIterValue_Type,3.2,, -var,PyDictKeys_Type,3.2,, -function,PyDictProxy_New,3.2,, -var,PyDictProxy_Type,3.2,, -var,PyDictRevIterItem_Type,3.8,, -var,PyDictRevIterKey_Type,3.8,, -var,PyDictRevIterValue_Type,3.8,, -var,PyDictValues_Type,3.2,, -function,PyDict_Clear,3.2,, -function,PyDict_Contains,3.2,, -function,PyDict_Copy,3.2,, -function,PyDict_DelItem,3.2,, -function,PyDict_DelItemString,3.2,, -function,PyDict_GetItem,3.2,, -function,PyDict_GetItemRef,3.13,, -function,PyDict_GetItemString,3.2,, -function,PyDict_GetItemStringRef,3.13,, -function,PyDict_GetItemWithError,3.2,, -function,PyDict_Items,3.2,, -function,PyDict_Keys,3.2,, -function,PyDict_Merge,3.2,, -function,PyDict_MergeFromSeq2,3.2,, -function,PyDict_New,3.2,, -function,PyDict_Next,3.2,, -function,PyDict_SetItem,3.2,, -function,PyDict_SetItemString,3.2,, -function,PyDict_Size,3.2,, -var,PyDict_Type,3.2,, -function,PyDict_Update,3.2,, -function,PyDict_Values,3.2,, -var,PyEllipsis_Type,3.2,, -var,PyEnum_Type,3.2,, -function,PyErr_BadArgument,3.2,, -function,PyErr_BadInternalCall,3.2,, -function,PyErr_CheckSignals,3.2,, -function,PyErr_Clear,3.2,, -function,PyErr_Display,3.2,, -function,PyErr_DisplayException,3.12,, -function,PyErr_ExceptionMatches,3.2,, -function,PyErr_Fetch,3.2,, -function,PyErr_Format,3.2,, -function,PyErr_FormatV,3.5,, -function,PyErr_GetExcInfo,3.7,, -function,PyErr_GetHandledException,3.11,, -function,PyErr_GetRaisedException,3.12,, -function,PyErr_GivenExceptionMatches,3.2,, -function,PyErr_NewException,3.2,, -function,PyErr_NewExceptionWithDoc,3.2,, -function,PyErr_NoMemory,3.2,, -function,PyErr_NormalizeException,3.2,, -function,PyErr_Occurred,3.2,, -function,PyErr_Print,3.2,, -function,PyErr_PrintEx,3.2,, -function,PyErr_ProgramText,3.2,, -function,PyErr_ResourceWarning,3.6,, -function,PyErr_Restore,3.2,, -function,PyErr_SetExcFromWindowsErr,3.7,on Windows, -function,PyErr_SetExcFromWindowsErrWithFilename,3.7,on Windows, -function,PyErr_SetExcFromWindowsErrWithFilenameObject,3.7,on Windows, -function,PyErr_SetExcFromWindowsErrWithFilenameObjects,3.7,on Windows, -function,PyErr_SetExcInfo,3.7,, -function,PyErr_SetFromErrno,3.2,, -function,PyErr_SetFromErrnoWithFilename,3.2,, -function,PyErr_SetFromErrnoWithFilenameObject,3.2,, -function,PyErr_SetFromErrnoWithFilenameObjects,3.7,, -function,PyErr_SetFromWindowsErr,3.7,on Windows, -function,PyErr_SetFromWindowsErrWithFilename,3.7,on Windows, -function,PyErr_SetHandledException,3.11,, -function,PyErr_SetImportError,3.7,, -function,PyErr_SetImportErrorSubclass,3.6,, -function,PyErr_SetInterrupt,3.2,, -function,PyErr_SetInterruptEx,3.10,, -function,PyErr_SetNone,3.2,, -function,PyErr_SetObject,3.2,, -function,PyErr_SetRaisedException,3.12,, -function,PyErr_SetString,3.2,, -function,PyErr_SyntaxLocation,3.2,, -function,PyErr_SyntaxLocationEx,3.7,, -function,PyErr_WarnEx,3.2,, -function,PyErr_WarnExplicit,3.2,, -function,PyErr_WarnFormat,3.2,, -function,PyErr_WriteUnraisable,3.2,, -function,PyEval_AcquireThread,3.2,, -function,PyEval_EvalCode,3.2,, -function,PyEval_EvalCodeEx,3.2,, -function,PyEval_EvalFrame,3.2,, -function,PyEval_EvalFrameEx,3.2,, -function,PyEval_GetBuiltins,3.2,, -function,PyEval_GetFrame,3.2,, -function,PyEval_GetFuncDesc,3.2,, -function,PyEval_GetFuncName,3.2,, -function,PyEval_GetGlobals,3.2,, -function,PyEval_GetLocals,3.2,, -function,PyEval_ReleaseThread,3.2,, -function,PyEval_RestoreThread,3.2,, -function,PyEval_SaveThread,3.2,, -var,PyExc_ArithmeticError,3.2,, -var,PyExc_AssertionError,3.2,, -var,PyExc_AttributeError,3.2,, -var,PyExc_BaseException,3.2,, -var,PyExc_BaseExceptionGroup,3.11,, -var,PyExc_BlockingIOError,3.7,, -var,PyExc_BrokenPipeError,3.7,, -var,PyExc_BufferError,3.2,, -var,PyExc_BytesWarning,3.2,, -var,PyExc_ChildProcessError,3.7,, -var,PyExc_ConnectionAbortedError,3.7,, -var,PyExc_ConnectionError,3.7,, -var,PyExc_ConnectionRefusedError,3.7,, -var,PyExc_ConnectionResetError,3.7,, -var,PyExc_DeprecationWarning,3.2,, -var,PyExc_EOFError,3.2,, -var,PyExc_EncodingWarning,3.10,, -var,PyExc_EnvironmentError,3.2,, -var,PyExc_Exception,3.2,, -var,PyExc_FileExistsError,3.7,, -var,PyExc_FileNotFoundError,3.7,, -var,PyExc_FloatingPointError,3.2,, -var,PyExc_FutureWarning,3.2,, -var,PyExc_GeneratorExit,3.2,, -var,PyExc_IOError,3.2,, -var,PyExc_ImportError,3.2,, -var,PyExc_ImportWarning,3.2,, -var,PyExc_IncompleteInputError,3.13,, -var,PyExc_IndentationError,3.2,, -var,PyExc_IndexError,3.2,, -var,PyExc_InterruptedError,3.7,, -var,PyExc_IsADirectoryError,3.7,, -var,PyExc_KeyError,3.2,, -var,PyExc_KeyboardInterrupt,3.2,, -var,PyExc_LookupError,3.2,, -var,PyExc_MemoryError,3.2,, -var,PyExc_ModuleNotFoundError,3.6,, -var,PyExc_NameError,3.2,, -var,PyExc_NotADirectoryError,3.7,, -var,PyExc_NotImplementedError,3.2,, -var,PyExc_OSError,3.2,, -var,PyExc_OverflowError,3.2,, -var,PyExc_PendingDeprecationWarning,3.2,, -var,PyExc_PermissionError,3.7,, -var,PyExc_ProcessLookupError,3.7,, -var,PyExc_RecursionError,3.7,, -var,PyExc_ReferenceError,3.2,, -var,PyExc_ResourceWarning,3.7,, -var,PyExc_RuntimeError,3.2,, -var,PyExc_RuntimeWarning,3.2,, -var,PyExc_StopAsyncIteration,3.7,, -var,PyExc_StopIteration,3.2,, -var,PyExc_SyntaxError,3.2,, -var,PyExc_SyntaxWarning,3.2,, -var,PyExc_SystemError,3.2,, -var,PyExc_SystemExit,3.2,, -var,PyExc_TabError,3.2,, -var,PyExc_TimeoutError,3.7,, -var,PyExc_TypeError,3.2,, -var,PyExc_UnboundLocalError,3.2,, -var,PyExc_UnicodeDecodeError,3.2,, -var,PyExc_UnicodeEncodeError,3.2,, -var,PyExc_UnicodeError,3.2,, -var,PyExc_UnicodeTranslateError,3.2,, -var,PyExc_UnicodeWarning,3.2,, -var,PyExc_UserWarning,3.2,, -var,PyExc_ValueError,3.2,, -var,PyExc_Warning,3.2,, -var,PyExc_WindowsError,3.7,on Windows, -var,PyExc_ZeroDivisionError,3.2,, -function,PyExceptionClass_Name,3.8,, -function,PyException_GetArgs,3.12,, -function,PyException_GetCause,3.2,, -function,PyException_GetContext,3.2,, -function,PyException_GetTraceback,3.2,, -function,PyException_SetArgs,3.12,, -function,PyException_SetCause,3.2,, -function,PyException_SetContext,3.2,, -function,PyException_SetTraceback,3.2,, -function,PyFile_FromFd,3.2,, -function,PyFile_GetLine,3.2,, -function,PyFile_WriteObject,3.2,, -function,PyFile_WriteString,3.2,, -var,PyFilter_Type,3.2,, -function,PyFloat_AsDouble,3.2,, -function,PyFloat_FromDouble,3.2,, -function,PyFloat_FromString,3.2,, -function,PyFloat_GetInfo,3.2,, -function,PyFloat_GetMax,3.2,, -function,PyFloat_GetMin,3.2,, -var,PyFloat_Type,3.2,, +func,PyCapsule_GetContext,3.2,, +func,PyCapsule_GetDestructor,3.2,, +func,PyCapsule_GetName,3.2,, +func,PyCapsule_GetPointer,3.2,, +func,PyCapsule_Import,3.2,, +func,PyCapsule_IsValid,3.2,, +func,PyCapsule_New,3.2,, +func,PyCapsule_SetContext,3.2,, +func,PyCapsule_SetDestructor,3.2,, +func,PyCapsule_SetName,3.2,, +func,PyCapsule_SetPointer,3.2,, +data,PyCapsule_Type,3.2,, +data,PyClassMethodDescr_Type,3.2,, +func,PyCodec_BackslashReplaceErrors,3.2,, +func,PyCodec_Decode,3.2,, +func,PyCodec_Decoder,3.2,, +func,PyCodec_Encode,3.2,, +func,PyCodec_Encoder,3.2,, +func,PyCodec_IgnoreErrors,3.2,, +func,PyCodec_IncrementalDecoder,3.2,, +func,PyCodec_IncrementalEncoder,3.2,, +func,PyCodec_KnownEncoding,3.2,, +func,PyCodec_LookupError,3.2,, +func,PyCodec_NameReplaceErrors,3.7,, +func,PyCodec_Register,3.2,, +func,PyCodec_RegisterError,3.2,, +func,PyCodec_ReplaceErrors,3.2,, +func,PyCodec_StreamReader,3.2,, +func,PyCodec_StreamWriter,3.2,, +func,PyCodec_StrictErrors,3.2,, +func,PyCodec_Unregister,3.10,, +func,PyCodec_XMLCharRefReplaceErrors,3.2,, +func,PyComplex_FromDoubles,3.2,, +func,PyComplex_ImagAsDouble,3.2,, +func,PyComplex_RealAsDouble,3.2,, +data,PyComplex_Type,3.2,, +func,PyDescr_NewClassMethod,3.2,, +func,PyDescr_NewGetSet,3.2,, +func,PyDescr_NewMember,3.2,, +func,PyDescr_NewMethod,3.2,, +data,PyDictItems_Type,3.2,, +data,PyDictIterItem_Type,3.2,, +data,PyDictIterKey_Type,3.2,, +data,PyDictIterValue_Type,3.2,, +data,PyDictKeys_Type,3.2,, +func,PyDictProxy_New,3.2,, +data,PyDictProxy_Type,3.2,, +data,PyDictRevIterItem_Type,3.8,, +data,PyDictRevIterKey_Type,3.8,, +data,PyDictRevIterValue_Type,3.8,, +data,PyDictValues_Type,3.2,, +func,PyDict_Clear,3.2,, +func,PyDict_Contains,3.2,, +func,PyDict_Copy,3.2,, +func,PyDict_DelItem,3.2,, +func,PyDict_DelItemString,3.2,, +func,PyDict_GetItem,3.2,, +func,PyDict_GetItemRef,3.13,, +func,PyDict_GetItemString,3.2,, +func,PyDict_GetItemStringRef,3.13,, +func,PyDict_GetItemWithError,3.2,, +func,PyDict_Items,3.2,, +func,PyDict_Keys,3.2,, +func,PyDict_Merge,3.2,, +func,PyDict_MergeFromSeq2,3.2,, +func,PyDict_New,3.2,, +func,PyDict_Next,3.2,, +func,PyDict_SetItem,3.2,, +func,PyDict_SetItemString,3.2,, +func,PyDict_Size,3.2,, +data,PyDict_Type,3.2,, +func,PyDict_Update,3.2,, +func,PyDict_Values,3.2,, +data,PyEllipsis_Type,3.2,, +data,PyEnum_Type,3.2,, +func,PyErr_BadArgument,3.2,, +func,PyErr_BadInternalCall,3.2,, +func,PyErr_CheckSignals,3.2,, +func,PyErr_Clear,3.2,, +func,PyErr_Display,3.2,, +func,PyErr_DisplayException,3.12,, +func,PyErr_ExceptionMatches,3.2,, +func,PyErr_Fetch,3.2,, +func,PyErr_Format,3.2,, +func,PyErr_FormatV,3.5,, +func,PyErr_GetExcInfo,3.7,, +func,PyErr_GetHandledException,3.11,, +func,PyErr_GetRaisedException,3.12,, +func,PyErr_GivenExceptionMatches,3.2,, +func,PyErr_NewException,3.2,, +func,PyErr_NewExceptionWithDoc,3.2,, +func,PyErr_NoMemory,3.2,, +func,PyErr_NormalizeException,3.2,, +func,PyErr_Occurred,3.2,, +func,PyErr_Print,3.2,, +func,PyErr_PrintEx,3.2,, +func,PyErr_ProgramText,3.2,, +func,PyErr_ResourceWarning,3.6,, +func,PyErr_Restore,3.2,, +func,PyErr_SetExcFromWindowsErr,3.7,on Windows, +func,PyErr_SetExcFromWindowsErrWithFilename,3.7,on Windows, +func,PyErr_SetExcFromWindowsErrWithFilenameObject,3.7,on Windows, +func,PyErr_SetExcFromWindowsErrWithFilenameObjects,3.7,on Windows, +func,PyErr_SetExcInfo,3.7,, +func,PyErr_SetFromErrno,3.2,, +func,PyErr_SetFromErrnoWithFilename,3.2,, +func,PyErr_SetFromErrnoWithFilenameObject,3.2,, +func,PyErr_SetFromErrnoWithFilenameObjects,3.7,, +func,PyErr_SetFromWindowsErr,3.7,on Windows, +func,PyErr_SetFromWindowsErrWithFilename,3.7,on Windows, +func,PyErr_SetHandledException,3.11,, +func,PyErr_SetImportError,3.7,, +func,PyErr_SetImportErrorSubclass,3.6,, +func,PyErr_SetInterrupt,3.2,, +func,PyErr_SetInterruptEx,3.10,, +func,PyErr_SetNone,3.2,, +func,PyErr_SetObject,3.2,, +func,PyErr_SetRaisedException,3.12,, +func,PyErr_SetString,3.2,, +func,PyErr_SyntaxLocation,3.2,, +func,PyErr_SyntaxLocationEx,3.7,, +func,PyErr_WarnEx,3.2,, +func,PyErr_WarnExplicit,3.2,, +func,PyErr_WarnFormat,3.2,, +func,PyErr_WriteUnraisable,3.2,, +func,PyEval_AcquireThread,3.2,, +func,PyEval_EvalCode,3.2,, +func,PyEval_EvalCodeEx,3.2,, +func,PyEval_EvalFrame,3.2,, +func,PyEval_EvalFrameEx,3.2,, +func,PyEval_GetBuiltins,3.2,, +func,PyEval_GetFrame,3.2,, +func,PyEval_GetFrameBuiltins,3.13,, +func,PyEval_GetFrameGlobals,3.13,, +func,PyEval_GetFrameLocals,3.13,, +func,PyEval_GetFuncDesc,3.2,, +func,PyEval_GetFuncName,3.2,, +func,PyEval_GetGlobals,3.2,, +func,PyEval_GetLocals,3.2,, +func,PyEval_InitThreads,3.2,, +func,PyEval_ReleaseThread,3.2,, +func,PyEval_RestoreThread,3.2,, +func,PyEval_SaveThread,3.2,, +data,PyExc_ArithmeticError,3.2,, +data,PyExc_AssertionError,3.2,, +data,PyExc_AttributeError,3.2,, +data,PyExc_BaseException,3.2,, +data,PyExc_BaseExceptionGroup,3.11,, +data,PyExc_BlockingIOError,3.7,, +data,PyExc_BrokenPipeError,3.7,, +data,PyExc_BufferError,3.2,, +data,PyExc_BytesWarning,3.2,, +data,PyExc_ChildProcessError,3.7,, +data,PyExc_ConnectionAbortedError,3.7,, +data,PyExc_ConnectionError,3.7,, +data,PyExc_ConnectionRefusedError,3.7,, +data,PyExc_ConnectionResetError,3.7,, +data,PyExc_DeprecationWarning,3.2,, +data,PyExc_EOFError,3.2,, +data,PyExc_EncodingWarning,3.10,, +data,PyExc_EnvironmentError,3.2,, +data,PyExc_Exception,3.2,, +data,PyExc_FileExistsError,3.7,, +data,PyExc_FileNotFoundError,3.7,, +data,PyExc_FloatingPointError,3.2,, +data,PyExc_FutureWarning,3.2,, +data,PyExc_GeneratorExit,3.2,, +data,PyExc_IOError,3.2,, +data,PyExc_ImportError,3.2,, +data,PyExc_ImportWarning,3.2,, +data,PyExc_IndentationError,3.2,, +data,PyExc_IndexError,3.2,, +data,PyExc_InterruptedError,3.7,, +data,PyExc_IsADirectoryError,3.7,, +data,PyExc_KeyError,3.2,, +data,PyExc_KeyboardInterrupt,3.2,, +data,PyExc_LookupError,3.2,, +data,PyExc_MemoryError,3.2,, +data,PyExc_ModuleNotFoundError,3.6,, +data,PyExc_NameError,3.2,, +data,PyExc_NotADirectoryError,3.7,, +data,PyExc_NotImplementedError,3.2,, +data,PyExc_OSError,3.2,, +data,PyExc_OverflowError,3.2,, +data,PyExc_PendingDeprecationWarning,3.2,, +data,PyExc_PermissionError,3.7,, +data,PyExc_ProcessLookupError,3.7,, +data,PyExc_RecursionError,3.7,, +data,PyExc_ReferenceError,3.2,, +data,PyExc_ResourceWarning,3.7,, +data,PyExc_RuntimeError,3.2,, +data,PyExc_RuntimeWarning,3.2,, +data,PyExc_StopAsyncIteration,3.7,, +data,PyExc_StopIteration,3.2,, +data,PyExc_SyntaxError,3.2,, +data,PyExc_SyntaxWarning,3.2,, +data,PyExc_SystemError,3.2,, +data,PyExc_SystemExit,3.2,, +data,PyExc_TabError,3.2,, +data,PyExc_TimeoutError,3.7,, +data,PyExc_TypeError,3.2,, +data,PyExc_UnboundLocalError,3.2,, +data,PyExc_UnicodeDecodeError,3.2,, +data,PyExc_UnicodeEncodeError,3.2,, +data,PyExc_UnicodeError,3.2,, +data,PyExc_UnicodeTranslateError,3.2,, +data,PyExc_UnicodeWarning,3.2,, +data,PyExc_UserWarning,3.2,, +data,PyExc_ValueError,3.2,, +data,PyExc_Warning,3.2,, +data,PyExc_WindowsError,3.7,on Windows, +data,PyExc_ZeroDivisionError,3.2,, +func,PyExceptionClass_Name,3.8,, +func,PyException_GetArgs,3.12,, +func,PyException_GetCause,3.2,, +func,PyException_GetContext,3.2,, +func,PyException_GetTraceback,3.2,, +func,PyException_SetArgs,3.12,, +func,PyException_SetCause,3.2,, +func,PyException_SetContext,3.2,, +func,PyException_SetTraceback,3.2,, +func,PyFile_FromFd,3.2,, +func,PyFile_GetLine,3.2,, +func,PyFile_WriteObject,3.2,, +func,PyFile_WriteString,3.2,, +data,PyFilter_Type,3.2,, +func,PyFloat_AsDouble,3.2,, +func,PyFloat_FromDouble,3.2,, +func,PyFloat_FromString,3.2,, +func,PyFloat_GetInfo,3.2,, +func,PyFloat_GetMax,3.2,, +func,PyFloat_GetMin,3.2,, +data,PyFloat_Type,3.2,, type,PyFrameObject,3.2,,opaque -function,PyFrame_GetCode,3.10,, -function,PyFrame_GetLineNumber,3.10,, -function,PyFrozenSet_New,3.2,, -var,PyFrozenSet_Type,3.2,, -function,PyGC_Collect,3.2,, -function,PyGC_Disable,3.10,, -function,PyGC_Enable,3.10,, -function,PyGC_IsEnabled,3.10,, -function,PyGILState_Ensure,3.2,, -function,PyGILState_GetThisThreadState,3.2,, -function,PyGILState_Release,3.2,, +func,PyFrame_GetCode,3.10,, +func,PyFrame_GetLineNumber,3.10,, +func,PyFrozenSet_New,3.2,, +data,PyFrozenSet_Type,3.2,, +func,PyGC_Collect,3.2,, +func,PyGC_Disable,3.10,, +func,PyGC_Enable,3.10,, +func,PyGC_IsEnabled,3.10,, +func,PyGILState_Ensure,3.2,, +func,PyGILState_GetThisThreadState,3.2,, +func,PyGILState_Release,3.2,, type,PyGILState_STATE,3.2,, type,PyGetSetDef,3.2,,full-abi -var,PyGetSetDescr_Type,3.2,, -function,PyImport_AddModule,3.2,, -function,PyImport_AddModuleObject,3.7,, -function,PyImport_AddModuleRef,3.13,, -function,PyImport_AppendInittab,3.2,, -function,PyImport_ExecCodeModule,3.2,, -function,PyImport_ExecCodeModuleEx,3.2,, -function,PyImport_ExecCodeModuleObject,3.7,, -function,PyImport_ExecCodeModuleWithPathnames,3.2,, -function,PyImport_GetImporter,3.2,, -function,PyImport_GetMagicNumber,3.2,, -function,PyImport_GetMagicTag,3.2,, -function,PyImport_GetModule,3.8,, -function,PyImport_GetModuleDict,3.2,, -function,PyImport_Import,3.2,, -function,PyImport_ImportFrozenModule,3.2,, -function,PyImport_ImportFrozenModuleObject,3.7,, -function,PyImport_ImportModule,3.2,, -function,PyImport_ImportModuleLevel,3.2,, -function,PyImport_ImportModuleLevelObject,3.7,, -function,PyImport_ImportModuleNoBlock,3.2,, -function,PyImport_ReloadModule,3.2,, -function,PyIndex_Check,3.8,, +data,PyGetSetDescr_Type,3.2,, +func,PyImport_AddModule,3.2,, +func,PyImport_AddModuleObject,3.7,, +func,PyImport_AddModuleRef,3.13,, +func,PyImport_AppendInittab,3.2,, +func,PyImport_ExecCodeModule,3.2,, +func,PyImport_ExecCodeModuleEx,3.2,, +func,PyImport_ExecCodeModuleObject,3.7,, +func,PyImport_ExecCodeModuleWithPathnames,3.2,, +func,PyImport_GetImporter,3.2,, +func,PyImport_GetMagicNumber,3.2,, +func,PyImport_GetMagicTag,3.2,, +func,PyImport_GetModule,3.8,, +func,PyImport_GetModuleDict,3.2,, +func,PyImport_Import,3.2,, +func,PyImport_ImportFrozenModule,3.2,, +func,PyImport_ImportFrozenModuleObject,3.7,, +func,PyImport_ImportModule,3.2,, +func,PyImport_ImportModuleLevel,3.2,, +func,PyImport_ImportModuleLevelObject,3.7,, +func,PyImport_ImportModuleNoBlock,3.2,, +func,PyImport_ReloadModule,3.2,, +func,PyIndex_Check,3.8,, type,PyInterpreterState,3.2,,opaque -function,PyInterpreterState_Clear,3.2,, -function,PyInterpreterState_Delete,3.2,, -function,PyInterpreterState_Get,3.9,, -function,PyInterpreterState_GetDict,3.8,, -function,PyInterpreterState_GetID,3.7,, -function,PyInterpreterState_New,3.2,, -function,PyIter_Check,3.8,, -function,PyIter_Next,3.2,, -function,PyIter_Send,3.10,, -var,PyListIter_Type,3.2,, -var,PyListRevIter_Type,3.2,, -function,PyList_Append,3.2,, -function,PyList_AsTuple,3.2,, -function,PyList_GetItem,3.2,, -function,PyList_GetItemRef,3.13,, -function,PyList_GetSlice,3.2,, -function,PyList_Insert,3.2,, -function,PyList_New,3.2,, -function,PyList_Reverse,3.2,, -function,PyList_SetItem,3.2,, -function,PyList_SetSlice,3.2,, -function,PyList_Size,3.2,, -function,PyList_Sort,3.2,, -var,PyList_Type,3.2,, +func,PyInterpreterState_Clear,3.2,, +func,PyInterpreterState_Delete,3.2,, +func,PyInterpreterState_Get,3.9,, +func,PyInterpreterState_GetDict,3.8,, +func,PyInterpreterState_GetID,3.7,, +func,PyInterpreterState_New,3.2,, +func,PyIter_Check,3.8,, +func,PyIter_Next,3.2,, +func,PyIter_NextItem,3.14,, +func,PyIter_Send,3.10,, +data,PyListIter_Type,3.2,, +data,PyListRevIter_Type,3.2,, +func,PyList_Append,3.2,, +func,PyList_AsTuple,3.2,, +func,PyList_GetItem,3.2,, +func,PyList_GetItemRef,3.13,, +func,PyList_GetSlice,3.2,, +func,PyList_Insert,3.2,, +func,PyList_New,3.2,, +func,PyList_Reverse,3.2,, +func,PyList_SetItem,3.2,, +func,PyList_SetSlice,3.2,, +func,PyList_Size,3.2,, +func,PyList_Sort,3.2,, +data,PyList_Type,3.2,, type,PyLongObject,3.2,,opaque -var,PyLongRangeIter_Type,3.2,, -function,PyLong_AsDouble,3.2,, -function,PyLong_AsInt,3.13,, -function,PyLong_AsLong,3.2,, -function,PyLong_AsLongAndOverflow,3.2,, -function,PyLong_AsLongLong,3.2,, -function,PyLong_AsLongLongAndOverflow,3.2,, -function,PyLong_AsSize_t,3.2,, -function,PyLong_AsSsize_t,3.2,, -function,PyLong_AsUnsignedLong,3.2,, -function,PyLong_AsUnsignedLongLong,3.2,, -function,PyLong_AsUnsignedLongLongMask,3.2,, -function,PyLong_AsUnsignedLongMask,3.2,, -function,PyLong_AsVoidPtr,3.2,, -function,PyLong_FromDouble,3.2,, -function,PyLong_FromLong,3.2,, -function,PyLong_FromLongLong,3.2,, -function,PyLong_FromSize_t,3.2,, -function,PyLong_FromSsize_t,3.2,, -function,PyLong_FromString,3.2,, -function,PyLong_FromUnsignedLong,3.2,, -function,PyLong_FromUnsignedLongLong,3.2,, -function,PyLong_FromVoidPtr,3.2,, -function,PyLong_GetInfo,3.2,, -var,PyLong_Type,3.2,, -var,PyMap_Type,3.2,, -function,PyMapping_Check,3.2,, -function,PyMapping_GetItemString,3.2,, -function,PyMapping_GetOptionalItem,3.13,, -function,PyMapping_GetOptionalItemString,3.13,, -function,PyMapping_HasKey,3.2,, -function,PyMapping_HasKeyString,3.2,, -function,PyMapping_HasKeyStringWithError,3.13,, -function,PyMapping_HasKeyWithError,3.13,, -function,PyMapping_Items,3.2,, -function,PyMapping_Keys,3.2,, -function,PyMapping_Length,3.2,, -function,PyMapping_SetItemString,3.2,, -function,PyMapping_Size,3.2,, -function,PyMapping_Values,3.2,, -function,PyMem_Calloc,3.7,, -function,PyMem_Free,3.2,, -function,PyMem_Malloc,3.2,, -function,PyMem_RawCalloc,3.13,, -function,PyMem_RawFree,3.13,, -function,PyMem_RawMalloc,3.13,, -function,PyMem_RawRealloc,3.13,, -function,PyMem_Realloc,3.2,, +data,PyLongRangeIter_Type,3.2,, +func,PyLong_AsDouble,3.2,, +func,PyLong_AsInt,3.13,, +func,PyLong_AsInt32,3.14,, +func,PyLong_AsInt64,3.14,, +func,PyLong_AsLong,3.2,, +func,PyLong_AsLongAndOverflow,3.2,, +func,PyLong_AsLongLong,3.2,, +func,PyLong_AsLongLongAndOverflow,3.2,, +func,PyLong_AsSize_t,3.2,, +func,PyLong_AsSsize_t,3.2,, +func,PyLong_AsUInt32,3.14,, +func,PyLong_AsUInt64,3.14,, +func,PyLong_AsUnsignedLong,3.2,, +func,PyLong_AsUnsignedLongLong,3.2,, +func,PyLong_AsUnsignedLongLongMask,3.2,, +func,PyLong_AsUnsignedLongMask,3.2,, +func,PyLong_AsVoidPtr,3.2,, +func,PyLong_FromDouble,3.2,, +func,PyLong_FromInt32,3.14,, +func,PyLong_FromInt64,3.14,, +func,PyLong_FromLong,3.2,, +func,PyLong_FromLongLong,3.2,, +func,PyLong_FromSize_t,3.2,, +func,PyLong_FromSsize_t,3.2,, +func,PyLong_FromString,3.2,, +func,PyLong_FromUInt32,3.14,, +func,PyLong_FromUInt64,3.14,, +func,PyLong_FromUnsignedLong,3.2,, +func,PyLong_FromUnsignedLongLong,3.2,, +func,PyLong_FromVoidPtr,3.2,, +func,PyLong_GetInfo,3.2,, +data,PyLong_Type,3.2,, +data,PyMap_Type,3.2,, +func,PyMapping_Check,3.2,, +func,PyMapping_GetItemString,3.2,, +func,PyMapping_GetOptionalItem,3.13,, +func,PyMapping_GetOptionalItemString,3.13,, +func,PyMapping_HasKey,3.2,, +func,PyMapping_HasKeyString,3.2,, +func,PyMapping_HasKeyStringWithError,3.13,, +func,PyMapping_HasKeyWithError,3.13,, +func,PyMapping_Items,3.2,, +func,PyMapping_Keys,3.2,, +func,PyMapping_Length,3.2,, +func,PyMapping_SetItemString,3.2,, +func,PyMapping_Size,3.2,, +func,PyMapping_Values,3.2,, +func,PyMem_Calloc,3.7,, +func,PyMem_Free,3.2,, +func,PyMem_Malloc,3.2,, +func,PyMem_RawCalloc,3.13,, +func,PyMem_RawFree,3.13,, +func,PyMem_RawMalloc,3.13,, +func,PyMem_RawRealloc,3.13,, +func,PyMem_Realloc,3.2,, type,PyMemberDef,3.2,,full-abi -var,PyMemberDescr_Type,3.2,, -function,PyMember_GetOne,3.2,, -function,PyMember_SetOne,3.2,, -function,PyMemoryView_FromBuffer,3.11,, -function,PyMemoryView_FromMemory,3.7,, -function,PyMemoryView_FromObject,3.2,, -function,PyMemoryView_GetContiguous,3.2,, -var,PyMemoryView_Type,3.2,, +data,PyMemberDescr_Type,3.2,, +func,PyMember_GetOne,3.2,, +func,PyMember_SetOne,3.2,, +func,PyMemoryView_FromBuffer,3.11,, +func,PyMemoryView_FromMemory,3.7,, +func,PyMemoryView_FromObject,3.2,, +func,PyMemoryView_GetContiguous,3.2,, +data,PyMemoryView_Type,3.2,, type,PyMethodDef,3.2,,full-abi -var,PyMethodDescr_Type,3.2,, +data,PyMethodDescr_Type,3.2,, type,PyModuleDef,3.2,,full-abi type,PyModuleDef_Base,3.2,,full-abi -function,PyModuleDef_Init,3.5,, -var,PyModuleDef_Type,3.5,, -function,PyModule_Add,3.13,, -function,PyModule_AddFunctions,3.7,, -function,PyModule_AddIntConstant,3.2,, -function,PyModule_AddObject,3.2,, -function,PyModule_AddObjectRef,3.10,, -function,PyModule_AddStringConstant,3.2,, -function,PyModule_AddType,3.10,, -function,PyModule_Create2,3.2,, -function,PyModule_ExecDef,3.7,, -function,PyModule_FromDefAndSpec2,3.7,, -function,PyModule_GetDef,3.2,, -function,PyModule_GetDict,3.2,, -function,PyModule_GetFilename,3.2,, -function,PyModule_GetFilenameObject,3.2,, -function,PyModule_GetName,3.2,, -function,PyModule_GetNameObject,3.7,, -function,PyModule_GetState,3.2,, -function,PyModule_New,3.2,, -function,PyModule_NewObject,3.7,, -function,PyModule_SetDocString,3.7,, -var,PyModule_Type,3.2,, -function,PyNumber_Absolute,3.2,, -function,PyNumber_Add,3.2,, -function,PyNumber_And,3.2,, -function,PyNumber_AsSsize_t,3.2,, -function,PyNumber_Check,3.2,, -function,PyNumber_Divmod,3.2,, -function,PyNumber_Float,3.2,, -function,PyNumber_FloorDivide,3.2,, -function,PyNumber_InPlaceAdd,3.2,, -function,PyNumber_InPlaceAnd,3.2,, -function,PyNumber_InPlaceFloorDivide,3.2,, -function,PyNumber_InPlaceLshift,3.2,, -function,PyNumber_InPlaceMatrixMultiply,3.7,, -function,PyNumber_InPlaceMultiply,3.2,, -function,PyNumber_InPlaceOr,3.2,, -function,PyNumber_InPlacePower,3.2,, -function,PyNumber_InPlaceRemainder,3.2,, -function,PyNumber_InPlaceRshift,3.2,, -function,PyNumber_InPlaceSubtract,3.2,, -function,PyNumber_InPlaceTrueDivide,3.2,, -function,PyNumber_InPlaceXor,3.2,, -function,PyNumber_Index,3.2,, -function,PyNumber_Invert,3.2,, -function,PyNumber_Long,3.2,, -function,PyNumber_Lshift,3.2,, -function,PyNumber_MatrixMultiply,3.7,, -function,PyNumber_Multiply,3.2,, -function,PyNumber_Negative,3.2,, -function,PyNumber_Or,3.2,, -function,PyNumber_Positive,3.2,, -function,PyNumber_Power,3.2,, -function,PyNumber_Remainder,3.2,, -function,PyNumber_Rshift,3.2,, -function,PyNumber_Subtract,3.2,, -function,PyNumber_ToBase,3.2,, -function,PyNumber_TrueDivide,3.2,, -function,PyNumber_Xor,3.2,, -function,PyOS_AfterFork,3.2,on platforms with fork(), -function,PyOS_AfterFork_Child,3.7,on platforms with fork(), -function,PyOS_AfterFork_Parent,3.7,on platforms with fork(), -function,PyOS_BeforeFork,3.7,on platforms with fork(), -function,PyOS_CheckStack,3.7,on platforms with USE_STACKCHECK, -function,PyOS_FSPath,3.6,, -var,PyOS_InputHook,3.2,, -function,PyOS_InterruptOccurred,3.2,, -function,PyOS_double_to_string,3.2,, -function,PyOS_getsig,3.2,, -function,PyOS_mystricmp,3.2,, -function,PyOS_mystrnicmp,3.2,, -function,PyOS_setsig,3.2,, +func,PyModuleDef_Init,3.5,, +data,PyModuleDef_Type,3.5,, +func,PyModule_Add,3.13,, +func,PyModule_AddFunctions,3.7,, +func,PyModule_AddIntConstant,3.2,, +func,PyModule_AddObject,3.2,, +func,PyModule_AddObjectRef,3.10,, +func,PyModule_AddStringConstant,3.2,, +func,PyModule_AddType,3.10,, +func,PyModule_Create2,3.2,, +func,PyModule_ExecDef,3.7,, +func,PyModule_FromDefAndSpec2,3.7,, +func,PyModule_GetDef,3.2,, +func,PyModule_GetDict,3.2,, +func,PyModule_GetFilename,3.2,, +func,PyModule_GetFilenameObject,3.2,, +func,PyModule_GetName,3.2,, +func,PyModule_GetNameObject,3.7,, +func,PyModule_GetState,3.2,, +func,PyModule_New,3.2,, +func,PyModule_NewObject,3.7,, +func,PyModule_SetDocString,3.7,, +data,PyModule_Type,3.2,, +func,PyNumber_Absolute,3.2,, +func,PyNumber_Add,3.2,, +func,PyNumber_And,3.2,, +func,PyNumber_AsSsize_t,3.2,, +func,PyNumber_Check,3.2,, +func,PyNumber_Divmod,3.2,, +func,PyNumber_Float,3.2,, +func,PyNumber_FloorDivide,3.2,, +func,PyNumber_InPlaceAdd,3.2,, +func,PyNumber_InPlaceAnd,3.2,, +func,PyNumber_InPlaceFloorDivide,3.2,, +func,PyNumber_InPlaceLshift,3.2,, +func,PyNumber_InPlaceMatrixMultiply,3.7,, +func,PyNumber_InPlaceMultiply,3.2,, +func,PyNumber_InPlaceOr,3.2,, +func,PyNumber_InPlacePower,3.2,, +func,PyNumber_InPlaceRemainder,3.2,, +func,PyNumber_InPlaceRshift,3.2,, +func,PyNumber_InPlaceSubtract,3.2,, +func,PyNumber_InPlaceTrueDivide,3.2,, +func,PyNumber_InPlaceXor,3.2,, +func,PyNumber_Index,3.2,, +func,PyNumber_Invert,3.2,, +func,PyNumber_Long,3.2,, +func,PyNumber_Lshift,3.2,, +func,PyNumber_MatrixMultiply,3.7,, +func,PyNumber_Multiply,3.2,, +func,PyNumber_Negative,3.2,, +func,PyNumber_Or,3.2,, +func,PyNumber_Positive,3.2,, +func,PyNumber_Power,3.2,, +func,PyNumber_Remainder,3.2,, +func,PyNumber_Rshift,3.2,, +func,PyNumber_Subtract,3.2,, +func,PyNumber_ToBase,3.2,, +func,PyNumber_TrueDivide,3.2,, +func,PyNumber_Xor,3.2,, +func,PyOS_AfterFork,3.2,on platforms with fork(), +func,PyOS_AfterFork_Child,3.7,on platforms with fork(), +func,PyOS_AfterFork_Parent,3.7,on platforms with fork(), +func,PyOS_BeforeFork,3.7,on platforms with fork(), +func,PyOS_CheckStack,3.7,on platforms with USE_STACKCHECK, +func,PyOS_FSPath,3.6,, +data,PyOS_InputHook,3.2,, +func,PyOS_InterruptOccurred,3.2,, +func,PyOS_double_to_string,3.2,, +func,PyOS_getsig,3.2,, +func,PyOS_mystricmp,3.2,, +func,PyOS_mystrnicmp,3.2,, +func,PyOS_setsig,3.2,, type,PyOS_sighandler_t,3.2,, -function,PyOS_snprintf,3.2,, -function,PyOS_string_to_double,3.2,, -function,PyOS_strtol,3.2,, -function,PyOS_strtoul,3.2,, -function,PyOS_vsnprintf,3.2,, +func,PyOS_snprintf,3.2,, +func,PyOS_string_to_double,3.2,, +func,PyOS_strtol,3.2,, +func,PyOS_strtoul,3.2,, +func,PyOS_vsnprintf,3.2,, type,PyObject,3.2,,members member,PyObject.ob_refcnt,3.2,, member,PyObject.ob_type,3.2,, -function,PyObject_ASCII,3.2,, -function,PyObject_AsFileDescriptor,3.2,, -function,PyObject_Bytes,3.2,, -function,PyObject_Call,3.2,, -function,PyObject_CallFunction,3.2,, -function,PyObject_CallFunctionObjArgs,3.2,, -function,PyObject_CallMethod,3.2,, -function,PyObject_CallMethodObjArgs,3.2,, -function,PyObject_CallNoArgs,3.10,, -function,PyObject_CallObject,3.2,, -function,PyObject_Calloc,3.7,, -function,PyObject_CheckBuffer,3.11,, -function,PyObject_ClearWeakRefs,3.2,, -function,PyObject_CopyData,3.11,, -function,PyObject_DelAttr,3.13,, -function,PyObject_DelAttrString,3.13,, -function,PyObject_DelItem,3.2,, -function,PyObject_DelItemString,3.2,, -function,PyObject_Dir,3.2,, -function,PyObject_Format,3.2,, -function,PyObject_Free,3.2,, -function,PyObject_GC_Del,3.2,, -function,PyObject_GC_IsFinalized,3.9,, -function,PyObject_GC_IsTracked,3.9,, -function,PyObject_GC_Track,3.2,, -function,PyObject_GC_UnTrack,3.2,, -function,PyObject_GenericGetAttr,3.2,, -function,PyObject_GenericGetDict,3.10,, -function,PyObject_GenericSetAttr,3.2,, -function,PyObject_GenericSetDict,3.7,, -function,PyObject_GetAIter,3.10,, -function,PyObject_GetAttr,3.2,, -function,PyObject_GetAttrString,3.2,, -function,PyObject_GetBuffer,3.11,, -function,PyObject_GetItem,3.2,, -function,PyObject_GetIter,3.2,, -function,PyObject_GetOptionalAttr,3.13,, -function,PyObject_GetOptionalAttrString,3.13,, -function,PyObject_GetTypeData,3.12,, -function,PyObject_HasAttr,3.2,, -function,PyObject_HasAttrString,3.2,, -function,PyObject_HasAttrStringWithError,3.13,, -function,PyObject_HasAttrWithError,3.13,, -function,PyObject_Hash,3.2,, -function,PyObject_HashNotImplemented,3.2,, -function,PyObject_Init,3.2,, -function,PyObject_InitVar,3.2,, -function,PyObject_IsInstance,3.2,, -function,PyObject_IsSubclass,3.2,, -function,PyObject_IsTrue,3.2,, -function,PyObject_Length,3.2,, -function,PyObject_Malloc,3.2,, -function,PyObject_Not,3.2,, -function,PyObject_Realloc,3.2,, -function,PyObject_Repr,3.2,, -function,PyObject_RichCompare,3.2,, -function,PyObject_RichCompareBool,3.2,, -function,PyObject_SelfIter,3.2,, -function,PyObject_SetAttr,3.2,, -function,PyObject_SetAttrString,3.2,, -function,PyObject_SetItem,3.2,, -function,PyObject_Size,3.2,, -function,PyObject_Str,3.2,, -function,PyObject_Type,3.2,, -function,PyObject_Vectorcall,3.12,, -function,PyObject_VectorcallMethod,3.12,, -var,PyProperty_Type,3.2,, -var,PyRangeIter_Type,3.2,, -var,PyRange_Type,3.2,, -var,PyReversed_Type,3.2,, -function,PySeqIter_New,3.2,, -var,PySeqIter_Type,3.2,, -function,PySequence_Check,3.2,, -function,PySequence_Concat,3.2,, -function,PySequence_Contains,3.2,, -function,PySequence_Count,3.2,, -function,PySequence_DelItem,3.2,, -function,PySequence_DelSlice,3.2,, -function,PySequence_Fast,3.2,, -function,PySequence_GetItem,3.2,, -function,PySequence_GetSlice,3.2,, -function,PySequence_In,3.2,, -function,PySequence_InPlaceConcat,3.2,, -function,PySequence_InPlaceRepeat,3.2,, -function,PySequence_Index,3.2,, -function,PySequence_Length,3.2,, -function,PySequence_List,3.2,, -function,PySequence_Repeat,3.2,, -function,PySequence_SetItem,3.2,, -function,PySequence_SetSlice,3.2,, -function,PySequence_Size,3.2,, -function,PySequence_Tuple,3.2,, -var,PySetIter_Type,3.2,, -function,PySet_Add,3.2,, -function,PySet_Clear,3.2,, -function,PySet_Contains,3.2,, -function,PySet_Discard,3.2,, -function,PySet_New,3.2,, -function,PySet_Pop,3.2,, -function,PySet_Size,3.2,, -var,PySet_Type,3.2,, -function,PySlice_AdjustIndices,3.7,, -function,PySlice_GetIndices,3.2,, -function,PySlice_GetIndicesEx,3.2,, -function,PySlice_New,3.2,, -var,PySlice_Type,3.2,, -function,PySlice_Unpack,3.7,, -function,PyState_AddModule,3.3,, -function,PyState_FindModule,3.2,, -function,PyState_RemoveModule,3.3,, +func,PyObject_ASCII,3.2,, +func,PyObject_AsFileDescriptor,3.2,, +func,PyObject_Bytes,3.2,, +func,PyObject_Call,3.2,, +func,PyObject_CallFunction,3.2,, +func,PyObject_CallFunctionObjArgs,3.2,, +func,PyObject_CallMethod,3.2,, +func,PyObject_CallMethodObjArgs,3.2,, +func,PyObject_CallNoArgs,3.10,, +func,PyObject_CallObject,3.2,, +func,PyObject_Calloc,3.7,, +func,PyObject_CheckBuffer,3.11,, +func,PyObject_ClearWeakRefs,3.2,, +func,PyObject_CopyData,3.11,, +func,PyObject_DelAttr,3.13,, +func,PyObject_DelAttrString,3.13,, +func,PyObject_DelItem,3.2,, +func,PyObject_DelItemString,3.2,, +func,PyObject_Dir,3.2,, +func,PyObject_Format,3.2,, +func,PyObject_Free,3.2,, +func,PyObject_GC_Del,3.2,, +func,PyObject_GC_IsFinalized,3.9,, +func,PyObject_GC_IsTracked,3.9,, +func,PyObject_GC_Track,3.2,, +func,PyObject_GC_UnTrack,3.2,, +func,PyObject_GenericGetAttr,3.2,, +func,PyObject_GenericGetDict,3.10,, +func,PyObject_GenericSetAttr,3.2,, +func,PyObject_GenericSetDict,3.7,, +func,PyObject_GetAIter,3.10,, +func,PyObject_GetAttr,3.2,, +func,PyObject_GetAttrString,3.2,, +func,PyObject_GetBuffer,3.11,, +func,PyObject_GetItem,3.2,, +func,PyObject_GetIter,3.2,, +func,PyObject_GetOptionalAttr,3.13,, +func,PyObject_GetOptionalAttrString,3.13,, +func,PyObject_GetTypeData,3.12,, +func,PyObject_HasAttr,3.2,, +func,PyObject_HasAttrString,3.2,, +func,PyObject_HasAttrStringWithError,3.13,, +func,PyObject_HasAttrWithError,3.13,, +func,PyObject_Hash,3.2,, +func,PyObject_HashNotImplemented,3.2,, +func,PyObject_Init,3.2,, +func,PyObject_InitVar,3.2,, +func,PyObject_IsInstance,3.2,, +func,PyObject_IsSubclass,3.2,, +func,PyObject_IsTrue,3.2,, +func,PyObject_Length,3.2,, +func,PyObject_Malloc,3.2,, +func,PyObject_Not,3.2,, +func,PyObject_Realloc,3.2,, +func,PyObject_Repr,3.2,, +func,PyObject_RichCompare,3.2,, +func,PyObject_RichCompareBool,3.2,, +func,PyObject_SelfIter,3.2,, +func,PyObject_SetAttr,3.2,, +func,PyObject_SetAttrString,3.2,, +func,PyObject_SetItem,3.2,, +func,PyObject_Size,3.2,, +func,PyObject_Str,3.2,, +func,PyObject_Type,3.2,, +func,PyObject_Vectorcall,3.12,, +func,PyObject_VectorcallMethod,3.12,, +data,PyProperty_Type,3.2,, +data,PyRangeIter_Type,3.2,, +data,PyRange_Type,3.2,, +data,PyReversed_Type,3.2,, +func,PySeqIter_New,3.2,, +data,PySeqIter_Type,3.2,, +func,PySequence_Check,3.2,, +func,PySequence_Concat,3.2,, +func,PySequence_Contains,3.2,, +func,PySequence_Count,3.2,, +func,PySequence_DelItem,3.2,, +func,PySequence_DelSlice,3.2,, +func,PySequence_Fast,3.2,, +func,PySequence_GetItem,3.2,, +func,PySequence_GetSlice,3.2,, +func,PySequence_In,3.2,, +func,PySequence_InPlaceConcat,3.2,, +func,PySequence_InPlaceRepeat,3.2,, +func,PySequence_Index,3.2,, +func,PySequence_Length,3.2,, +func,PySequence_List,3.2,, +func,PySequence_Repeat,3.2,, +func,PySequence_SetItem,3.2,, +func,PySequence_SetSlice,3.2,, +func,PySequence_Size,3.2,, +func,PySequence_Tuple,3.2,, +data,PySetIter_Type,3.2,, +func,PySet_Add,3.2,, +func,PySet_Clear,3.2,, +func,PySet_Contains,3.2,, +func,PySet_Discard,3.2,, +func,PySet_New,3.2,, +func,PySet_Pop,3.2,, +func,PySet_Size,3.2,, +data,PySet_Type,3.2,, +func,PySlice_AdjustIndices,3.7,, +func,PySlice_GetIndices,3.2,, +func,PySlice_GetIndicesEx,3.2,, +func,PySlice_New,3.2,, +data,PySlice_Type,3.2,, +func,PySlice_Unpack,3.7,, +func,PyState_AddModule,3.3,, +func,PyState_FindModule,3.2,, +func,PyState_RemoveModule,3.3,, type,PyStructSequence_Desc,3.2,,full-abi type,PyStructSequence_Field,3.2,,full-abi -function,PyStructSequence_GetItem,3.2,, -function,PyStructSequence_New,3.2,, -function,PyStructSequence_NewType,3.2,, -function,PyStructSequence_SetItem,3.2,, -var,PyStructSequence_UnnamedField,3.11,, -var,PySuper_Type,3.2,, -function,PySys_Audit,3.13,, -function,PySys_AuditTuple,3.13,, -function,PySys_FormatStderr,3.2,, -function,PySys_FormatStdout,3.2,, -function,PySys_GetObject,3.2,, -function,PySys_GetXOptions,3.7,, -function,PySys_ResetWarnOptions,3.2,, -function,PySys_SetObject,3.2,, -function,PySys_WriteStderr,3.2,, -function,PySys_WriteStdout,3.2,, +func,PyStructSequence_GetItem,3.2,, +func,PyStructSequence_New,3.2,, +func,PyStructSequence_NewType,3.2,, +func,PyStructSequence_SetItem,3.2,, +data,PyStructSequence_UnnamedField,3.11,, +data,PySuper_Type,3.2,, +func,PySys_Audit,3.13,, +func,PySys_AuditTuple,3.13,, +func,PySys_FormatStderr,3.2,, +func,PySys_FormatStdout,3.2,, +func,PySys_GetObject,3.2,, +func,PySys_GetXOptions,3.7,, +func,PySys_ResetWarnOptions,3.2,, +func,PySys_SetArgv,3.2,, +func,PySys_SetArgvEx,3.2,, +func,PySys_SetObject,3.2,, +func,PySys_WriteStderr,3.2,, +func,PySys_WriteStdout,3.2,, type,PyThreadState,3.2,,opaque -function,PyThreadState_Clear,3.2,, -function,PyThreadState_Delete,3.2,, -function,PyThreadState_Get,3.2,, -function,PyThreadState_GetDict,3.2,, -function,PyThreadState_GetFrame,3.10,, -function,PyThreadState_GetID,3.10,, -function,PyThreadState_GetInterpreter,3.10,, -function,PyThreadState_New,3.2,, -function,PyThreadState_SetAsyncExc,3.2,, -function,PyThreadState_Swap,3.2,, -function,PyThread_GetInfo,3.3,, -function,PyThread_ReInitTLS,3.2,, -function,PyThread_acquire_lock,3.2,, -function,PyThread_acquire_lock_timed,3.2,, -function,PyThread_allocate_lock,3.2,, -function,PyThread_create_key,3.2,, -function,PyThread_delete_key,3.2,, -function,PyThread_delete_key_value,3.2,, -function,PyThread_exit_thread,3.2,, -function,PyThread_free_lock,3.2,, -function,PyThread_get_key_value,3.2,, -function,PyThread_get_stacksize,3.2,, -function,PyThread_get_thread_ident,3.2,, -function,PyThread_get_thread_native_id,3.2,on platforms with native thread IDs, -function,PyThread_init_thread,3.2,, -function,PyThread_release_lock,3.2,, -function,PyThread_set_key_value,3.2,, -function,PyThread_set_stacksize,3.2,, -function,PyThread_start_new_thread,3.2,, -function,PyThread_tss_alloc,3.7,, -function,PyThread_tss_create,3.7,, -function,PyThread_tss_delete,3.7,, -function,PyThread_tss_free,3.7,, -function,PyThread_tss_get,3.7,, -function,PyThread_tss_is_created,3.7,, -function,PyThread_tss_set,3.7,, -function,PyTraceBack_Here,3.2,, -function,PyTraceBack_Print,3.2,, -var,PyTraceBack_Type,3.2,, -var,PyTupleIter_Type,3.2,, -function,PyTuple_GetItem,3.2,, -function,PyTuple_GetSlice,3.2,, -function,PyTuple_New,3.2,, -function,PyTuple_Pack,3.2,, -function,PyTuple_SetItem,3.2,, -function,PyTuple_Size,3.2,, -var,PyTuple_Type,3.2,, +func,PyThreadState_Clear,3.2,, +func,PyThreadState_Delete,3.2,, +func,PyThreadState_Get,3.2,, +func,PyThreadState_GetDict,3.2,, +func,PyThreadState_GetFrame,3.10,, +func,PyThreadState_GetID,3.10,, +func,PyThreadState_GetInterpreter,3.10,, +func,PyThreadState_New,3.2,, +func,PyThreadState_SetAsyncExc,3.2,, +func,PyThreadState_Swap,3.2,, +func,PyThread_GetInfo,3.3,, +func,PyThread_ReInitTLS,3.2,, +func,PyThread_acquire_lock,3.2,, +func,PyThread_acquire_lock_timed,3.2,, +func,PyThread_allocate_lock,3.2,, +func,PyThread_create_key,3.2,, +func,PyThread_delete_key,3.2,, +func,PyThread_delete_key_value,3.2,, +func,PyThread_exit_thread,3.2,, +func,PyThread_free_lock,3.2,, +func,PyThread_get_key_value,3.2,, +func,PyThread_get_stacksize,3.2,, +func,PyThread_get_thread_ident,3.2,, +func,PyThread_get_thread_native_id,3.2,on platforms with native thread IDs, +func,PyThread_init_thread,3.2,, +func,PyThread_release_lock,3.2,, +func,PyThread_set_key_value,3.2,, +func,PyThread_set_stacksize,3.2,, +func,PyThread_start_new_thread,3.2,, +func,PyThread_tss_alloc,3.7,, +func,PyThread_tss_create,3.7,, +func,PyThread_tss_delete,3.7,, +func,PyThread_tss_free,3.7,, +func,PyThread_tss_get,3.7,, +func,PyThread_tss_is_created,3.7,, +func,PyThread_tss_set,3.7,, +func,PyTraceBack_Here,3.2,, +func,PyTraceBack_Print,3.2,, +data,PyTraceBack_Type,3.2,, +data,PyTupleIter_Type,3.2,, +func,PyTuple_GetItem,3.2,, +func,PyTuple_GetSlice,3.2,, +func,PyTuple_New,3.2,, +func,PyTuple_Pack,3.2,, +func,PyTuple_SetItem,3.2,, +func,PyTuple_Size,3.2,, +data,PyTuple_Type,3.2,, type,PyTypeObject,3.2,,opaque -function,PyType_ClearCache,3.2,, -function,PyType_FromMetaclass,3.12,, -function,PyType_FromModuleAndSpec,3.10,, -function,PyType_FromSpec,3.2,, -function,PyType_FromSpecWithBases,3.3,, -function,PyType_GenericAlloc,3.2,, -function,PyType_GenericNew,3.2,, -function,PyType_GetFlags,3.2,, -function,PyType_GetModule,3.10,, -function,PyType_GetModuleState,3.10,, -function,PyType_GetName,3.11,, -function,PyType_GetQualName,3.11,, -function,PyType_GetSlot,3.4,, -function,PyType_GetTypeDataSize,3.12,, -function,PyType_IsSubtype,3.2,, -function,PyType_Modified,3.2,, -function,PyType_Ready,3.2,, +func,PyType_ClearCache,3.2,, +func,PyType_Freeze,3.14,, +func,PyType_FromMetaclass,3.12,, +func,PyType_FromModuleAndSpec,3.10,, +func,PyType_FromSpec,3.2,, +func,PyType_FromSpecWithBases,3.3,, +func,PyType_GenericAlloc,3.2,, +func,PyType_GenericNew,3.2,, +func,PyType_GetBaseByToken,3.14,, +func,PyType_GetFlags,3.2,, +func,PyType_GetFullyQualifiedName,3.13,, +func,PyType_GetModule,3.10,, +func,PyType_GetModuleByDef,3.13,, +func,PyType_GetModuleName,3.13,, +func,PyType_GetModuleState,3.10,, +func,PyType_GetName,3.11,, +func,PyType_GetQualName,3.11,, +func,PyType_GetSlot,3.4,, +func,PyType_GetTypeDataSize,3.12,, +func,PyType_IsSubtype,3.2,, +func,PyType_Modified,3.2,, +func,PyType_Ready,3.2,, type,PyType_Slot,3.2,,full-abi type,PyType_Spec,3.2,,full-abi -var,PyType_Type,3.2,, -function,PyUnicodeDecodeError_Create,3.2,, -function,PyUnicodeDecodeError_GetEncoding,3.2,, -function,PyUnicodeDecodeError_GetEnd,3.2,, -function,PyUnicodeDecodeError_GetObject,3.2,, -function,PyUnicodeDecodeError_GetReason,3.2,, -function,PyUnicodeDecodeError_GetStart,3.2,, -function,PyUnicodeDecodeError_SetEnd,3.2,, -function,PyUnicodeDecodeError_SetReason,3.2,, -function,PyUnicodeDecodeError_SetStart,3.2,, -function,PyUnicodeEncodeError_GetEncoding,3.2,, -function,PyUnicodeEncodeError_GetEnd,3.2,, -function,PyUnicodeEncodeError_GetObject,3.2,, -function,PyUnicodeEncodeError_GetReason,3.2,, -function,PyUnicodeEncodeError_GetStart,3.2,, -function,PyUnicodeEncodeError_SetEnd,3.2,, -function,PyUnicodeEncodeError_SetReason,3.2,, -function,PyUnicodeEncodeError_SetStart,3.2,, -var,PyUnicodeIter_Type,3.2,, -function,PyUnicodeTranslateError_GetEnd,3.2,, -function,PyUnicodeTranslateError_GetObject,3.2,, -function,PyUnicodeTranslateError_GetReason,3.2,, -function,PyUnicodeTranslateError_GetStart,3.2,, -function,PyUnicodeTranslateError_SetEnd,3.2,, -function,PyUnicodeTranslateError_SetReason,3.2,, -function,PyUnicodeTranslateError_SetStart,3.2,, -function,PyUnicode_Append,3.2,, -function,PyUnicode_AppendAndDel,3.2,, -function,PyUnicode_AsASCIIString,3.2,, -function,PyUnicode_AsCharmapString,3.2,, -function,PyUnicode_AsDecodedObject,3.2,, -function,PyUnicode_AsDecodedUnicode,3.2,, -function,PyUnicode_AsEncodedObject,3.2,, -function,PyUnicode_AsEncodedString,3.2,, -function,PyUnicode_AsEncodedUnicode,3.2,, -function,PyUnicode_AsLatin1String,3.2,, -function,PyUnicode_AsMBCSString,3.7,on Windows, -function,PyUnicode_AsRawUnicodeEscapeString,3.2,, -function,PyUnicode_AsUCS4,3.7,, -function,PyUnicode_AsUCS4Copy,3.7,, -function,PyUnicode_AsUTF16String,3.2,, -function,PyUnicode_AsUTF32String,3.2,, -function,PyUnicode_AsUTF8AndSize,3.10,, -function,PyUnicode_AsUTF8String,3.2,, -function,PyUnicode_AsUnicodeEscapeString,3.2,, -function,PyUnicode_AsWideChar,3.2,, -function,PyUnicode_AsWideCharString,3.7,, -function,PyUnicode_BuildEncodingMap,3.2,, -function,PyUnicode_Compare,3.2,, -function,PyUnicode_CompareWithASCIIString,3.2,, -function,PyUnicode_Concat,3.2,, -function,PyUnicode_Contains,3.2,, -function,PyUnicode_Count,3.2,, -function,PyUnicode_Decode,3.2,, -function,PyUnicode_DecodeASCII,3.2,, -function,PyUnicode_DecodeCharmap,3.2,, -function,PyUnicode_DecodeCodePageStateful,3.7,on Windows, -function,PyUnicode_DecodeFSDefault,3.2,, -function,PyUnicode_DecodeFSDefaultAndSize,3.2,, -function,PyUnicode_DecodeLatin1,3.2,, -function,PyUnicode_DecodeLocale,3.7,, -function,PyUnicode_DecodeLocaleAndSize,3.7,, -function,PyUnicode_DecodeMBCS,3.7,on Windows, -function,PyUnicode_DecodeMBCSStateful,3.7,on Windows, -function,PyUnicode_DecodeRawUnicodeEscape,3.2,, -function,PyUnicode_DecodeUTF16,3.2,, -function,PyUnicode_DecodeUTF16Stateful,3.2,, -function,PyUnicode_DecodeUTF32,3.2,, -function,PyUnicode_DecodeUTF32Stateful,3.2,, -function,PyUnicode_DecodeUTF7,3.2,, -function,PyUnicode_DecodeUTF7Stateful,3.2,, -function,PyUnicode_DecodeUTF8,3.2,, -function,PyUnicode_DecodeUTF8Stateful,3.2,, -function,PyUnicode_DecodeUnicodeEscape,3.2,, -function,PyUnicode_EncodeCodePage,3.7,on Windows, -function,PyUnicode_EncodeFSDefault,3.2,, -function,PyUnicode_EncodeLocale,3.7,, -function,PyUnicode_EqualToUTF8,3.13,, -function,PyUnicode_EqualToUTF8AndSize,3.13,, -function,PyUnicode_FSConverter,3.2,, -function,PyUnicode_FSDecoder,3.2,, -function,PyUnicode_Find,3.2,, -function,PyUnicode_FindChar,3.7,, -function,PyUnicode_Format,3.2,, -function,PyUnicode_FromEncodedObject,3.2,, -function,PyUnicode_FromFormat,3.2,, -function,PyUnicode_FromFormatV,3.2,, -function,PyUnicode_FromObject,3.2,, -function,PyUnicode_FromOrdinal,3.2,, -function,PyUnicode_FromString,3.2,, -function,PyUnicode_FromStringAndSize,3.2,, -function,PyUnicode_FromWideChar,3.2,, -function,PyUnicode_GetDefaultEncoding,3.2,, -function,PyUnicode_GetLength,3.7,, -function,PyUnicode_InternFromString,3.2,, -function,PyUnicode_InternInPlace,3.2,, -function,PyUnicode_IsIdentifier,3.2,, -function,PyUnicode_Join,3.2,, -function,PyUnicode_Partition,3.2,, -function,PyUnicode_RPartition,3.2,, -function,PyUnicode_RSplit,3.2,, -function,PyUnicode_ReadChar,3.7,, -function,PyUnicode_Replace,3.2,, -function,PyUnicode_Resize,3.2,, -function,PyUnicode_RichCompare,3.2,, -function,PyUnicode_Split,3.2,, -function,PyUnicode_Splitlines,3.2,, -function,PyUnicode_Substring,3.7,, -function,PyUnicode_Tailmatch,3.2,, -function,PyUnicode_Translate,3.2,, -var,PyUnicode_Type,3.2,, -function,PyUnicode_WriteChar,3.7,, +data,PyType_Type,3.2,, +func,PyUnicodeDecodeError_Create,3.2,, +func,PyUnicodeDecodeError_GetEncoding,3.2,, +func,PyUnicodeDecodeError_GetEnd,3.2,, +func,PyUnicodeDecodeError_GetObject,3.2,, +func,PyUnicodeDecodeError_GetReason,3.2,, +func,PyUnicodeDecodeError_GetStart,3.2,, +func,PyUnicodeDecodeError_SetEnd,3.2,, +func,PyUnicodeDecodeError_SetReason,3.2,, +func,PyUnicodeDecodeError_SetStart,3.2,, +func,PyUnicodeEncodeError_GetEncoding,3.2,, +func,PyUnicodeEncodeError_GetEnd,3.2,, +func,PyUnicodeEncodeError_GetObject,3.2,, +func,PyUnicodeEncodeError_GetReason,3.2,, +func,PyUnicodeEncodeError_GetStart,3.2,, +func,PyUnicodeEncodeError_SetEnd,3.2,, +func,PyUnicodeEncodeError_SetReason,3.2,, +func,PyUnicodeEncodeError_SetStart,3.2,, +data,PyUnicodeIter_Type,3.2,, +func,PyUnicodeTranslateError_GetEnd,3.2,, +func,PyUnicodeTranslateError_GetObject,3.2,, +func,PyUnicodeTranslateError_GetReason,3.2,, +func,PyUnicodeTranslateError_GetStart,3.2,, +func,PyUnicodeTranslateError_SetEnd,3.2,, +func,PyUnicodeTranslateError_SetReason,3.2,, +func,PyUnicodeTranslateError_SetStart,3.2,, +func,PyUnicode_Append,3.2,, +func,PyUnicode_AppendAndDel,3.2,, +func,PyUnicode_AsASCIIString,3.2,, +func,PyUnicode_AsCharmapString,3.2,, +func,PyUnicode_AsDecodedObject,3.2,, +func,PyUnicode_AsDecodedUnicode,3.2,, +func,PyUnicode_AsEncodedObject,3.2,, +func,PyUnicode_AsEncodedString,3.2,, +func,PyUnicode_AsEncodedUnicode,3.2,, +func,PyUnicode_AsLatin1String,3.2,, +func,PyUnicode_AsMBCSString,3.7,on Windows, +func,PyUnicode_AsRawUnicodeEscapeString,3.2,, +func,PyUnicode_AsUCS4,3.7,, +func,PyUnicode_AsUCS4Copy,3.7,, +func,PyUnicode_AsUTF16String,3.2,, +func,PyUnicode_AsUTF32String,3.2,, +func,PyUnicode_AsUTF8AndSize,3.10,, +func,PyUnicode_AsUTF8String,3.2,, +func,PyUnicode_AsUnicodeEscapeString,3.2,, +func,PyUnicode_AsWideChar,3.2,, +func,PyUnicode_AsWideCharString,3.7,, +func,PyUnicode_BuildEncodingMap,3.2,, +func,PyUnicode_Compare,3.2,, +func,PyUnicode_CompareWithASCIIString,3.2,, +func,PyUnicode_Concat,3.2,, +func,PyUnicode_Contains,3.2,, +func,PyUnicode_Count,3.2,, +func,PyUnicode_Decode,3.2,, +func,PyUnicode_DecodeASCII,3.2,, +func,PyUnicode_DecodeCharmap,3.2,, +func,PyUnicode_DecodeCodePageStateful,3.7,on Windows, +func,PyUnicode_DecodeFSDefault,3.2,, +func,PyUnicode_DecodeFSDefaultAndSize,3.2,, +func,PyUnicode_DecodeLatin1,3.2,, +func,PyUnicode_DecodeLocale,3.7,, +func,PyUnicode_DecodeLocaleAndSize,3.7,, +func,PyUnicode_DecodeMBCS,3.7,on Windows, +func,PyUnicode_DecodeMBCSStateful,3.7,on Windows, +func,PyUnicode_DecodeRawUnicodeEscape,3.2,, +func,PyUnicode_DecodeUTF16,3.2,, +func,PyUnicode_DecodeUTF16Stateful,3.2,, +func,PyUnicode_DecodeUTF32,3.2,, +func,PyUnicode_DecodeUTF32Stateful,3.2,, +func,PyUnicode_DecodeUTF7,3.2,, +func,PyUnicode_DecodeUTF7Stateful,3.2,, +func,PyUnicode_DecodeUTF8,3.2,, +func,PyUnicode_DecodeUTF8Stateful,3.2,, +func,PyUnicode_DecodeUnicodeEscape,3.2,, +func,PyUnicode_EncodeCodePage,3.7,on Windows, +func,PyUnicode_EncodeFSDefault,3.2,, +func,PyUnicode_EncodeLocale,3.7,, +func,PyUnicode_Equal,3.14,, +func,PyUnicode_EqualToUTF8,3.13,, +func,PyUnicode_EqualToUTF8AndSize,3.13,, +func,PyUnicode_FSConverter,3.2,, +func,PyUnicode_FSDecoder,3.2,, +func,PyUnicode_Find,3.2,, +func,PyUnicode_FindChar,3.7,, +func,PyUnicode_Format,3.2,, +func,PyUnicode_FromEncodedObject,3.2,, +func,PyUnicode_FromFormat,3.2,, +func,PyUnicode_FromFormatV,3.2,, +func,PyUnicode_FromObject,3.2,, +func,PyUnicode_FromOrdinal,3.2,, +func,PyUnicode_FromString,3.2,, +func,PyUnicode_FromStringAndSize,3.2,, +func,PyUnicode_FromWideChar,3.2,, +func,PyUnicode_GetDefaultEncoding,3.2,, +func,PyUnicode_GetLength,3.7,, +func,PyUnicode_InternFromString,3.2,, +func,PyUnicode_InternInPlace,3.2,, +func,PyUnicode_IsIdentifier,3.2,, +func,PyUnicode_Join,3.2,, +func,PyUnicode_Partition,3.2,, +func,PyUnicode_RPartition,3.2,, +func,PyUnicode_RSplit,3.2,, +func,PyUnicode_ReadChar,3.7,, +func,PyUnicode_Replace,3.2,, +func,PyUnicode_Resize,3.2,, +func,PyUnicode_RichCompare,3.2,, +func,PyUnicode_Split,3.2,, +func,PyUnicode_Splitlines,3.2,, +func,PyUnicode_Substring,3.7,, +func,PyUnicode_Tailmatch,3.2,, +func,PyUnicode_Translate,3.2,, +data,PyUnicode_Type,3.2,, +func,PyUnicode_WriteChar,3.7,, type,PyVarObject,3.2,,members member,PyVarObject.ob_base,3.2,, member,PyVarObject.ob_size,3.2,, -function,PyVectorcall_Call,3.12,, -function,PyVectorcall_NARGS,3.12,, +func,PyVectorcall_Call,3.12,, +func,PyVectorcall_NARGS,3.12,, type,PyWeakReference,3.2,,opaque -function,PyWeakref_GetObject,3.2,, -function,PyWeakref_GetRef,3.13,, -function,PyWeakref_NewProxy,3.2,, -function,PyWeakref_NewRef,3.2,, -var,PyWrapperDescr_Type,3.2,, -function,PyWrapper_New,3.2,, -var,PyZip_Type,3.2,, -function,Py_AddPendingCall,3.2,, -function,Py_AtExit,3.2,, +func,PyWeakref_GetObject,3.2,, +func,PyWeakref_GetRef,3.13,, +func,PyWeakref_NewProxy,3.2,, +func,PyWeakref_NewRef,3.2,, +data,PyWrapperDescr_Type,3.2,, +func,PyWrapper_New,3.2,, +data,PyZip_Type,3.2,, +func,Py_AddPendingCall,3.2,, +func,Py_AtExit,3.2,, macro,Py_BEGIN_ALLOW_THREADS,3.2,, macro,Py_BLOCK_THREADS,3.2,, -function,Py_BuildValue,3.2,, -function,Py_BytesMain,3.8,, -function,Py_CompileString,3.2,, -function,Py_DecRef,3.2,, -function,Py_DecodeLocale,3.7,, +func,Py_BuildValue,3.2,, +func,Py_BytesMain,3.8,, +func,Py_CompileString,3.2,, +func,Py_DecRef,3.2,, +func,Py_DecodeLocale,3.7,, macro,Py_END_ALLOW_THREADS,3.2,, -function,Py_EncodeLocale,3.7,, -function,Py_EndInterpreter,3.2,, -function,Py_EnterRecursiveCall,3.9,, -function,Py_Exit,3.2,, -function,Py_FatalError,3.2,, -var,Py_FileSystemDefaultEncodeErrors,3.10,, -var,Py_FileSystemDefaultEncoding,3.2,, -function,Py_Finalize,3.2,, -function,Py_FinalizeEx,3.6,, -function,Py_GenericAlias,3.9,, -var,Py_GenericAliasType,3.9,, -function,Py_GetBuildInfo,3.2,, -function,Py_GetCompiler,3.2,, -function,Py_GetCopyright,3.2,, -function,Py_GetExecPrefix,3.2,, -function,Py_GetPath,3.2,, -function,Py_GetPlatform,3.2,, -function,Py_GetPrefix,3.2,, -function,Py_GetProgramFullPath,3.2,, -function,Py_GetProgramName,3.2,, -function,Py_GetPythonHome,3.2,, -function,Py_GetRecursionLimit,3.2,, -function,Py_GetVersion,3.2,, -var,Py_HasFileSystemDefaultEncoding,3.2,, -function,Py_IncRef,3.2,, -function,Py_Initialize,3.2,, -function,Py_InitializeEx,3.2,, -function,Py_Is,3.10,, -function,Py_IsFalse,3.10,, -function,Py_IsFinalizing,3.13,, -function,Py_IsInitialized,3.2,, -function,Py_IsNone,3.10,, -function,Py_IsTrue,3.10,, -function,Py_LeaveRecursiveCall,3.9,, -function,Py_Main,3.2,, -function,Py_MakePendingCalls,3.2,, -function,Py_NewInterpreter,3.2,, -function,Py_NewRef,3.10,, -function,Py_ReprEnter,3.2,, -function,Py_ReprLeave,3.2,, -function,Py_SetRecursionLimit,3.2,, +func,Py_EncodeLocale,3.7,, +func,Py_EndInterpreter,3.2,, +func,Py_EnterRecursiveCall,3.9,, +func,Py_Exit,3.2,, +func,Py_FatalError,3.2,, +data,Py_FileSystemDefaultEncodeErrors,3.10,, +data,Py_FileSystemDefaultEncoding,3.2,, +func,Py_Finalize,3.2,, +func,Py_FinalizeEx,3.6,, +func,Py_GenericAlias,3.9,, +data,Py_GenericAliasType,3.9,, +func,Py_GetBuildInfo,3.2,, +func,Py_GetCompiler,3.2,, +func,Py_GetConstant,3.13,, +func,Py_GetConstantBorrowed,3.13,, +func,Py_GetCopyright,3.2,, +func,Py_GetExecPrefix,3.2,, +func,Py_GetPath,3.2,, +func,Py_GetPlatform,3.2,, +func,Py_GetPrefix,3.2,, +func,Py_GetProgramFullPath,3.2,, +func,Py_GetProgramName,3.2,, +func,Py_GetPythonHome,3.2,, +func,Py_GetRecursionLimit,3.2,, +func,Py_GetVersion,3.2,, +data,Py_HasFileSystemDefaultEncoding,3.2,, +func,Py_IncRef,3.2,, +func,Py_Initialize,3.2,, +func,Py_InitializeEx,3.2,, +func,Py_Is,3.10,, +func,Py_IsFalse,3.10,, +func,Py_IsFinalizing,3.13,, +func,Py_IsInitialized,3.2,, +func,Py_IsNone,3.10,, +func,Py_IsTrue,3.10,, +func,Py_LeaveRecursiveCall,3.9,, +func,Py_Main,3.2,, +func,Py_MakePendingCalls,3.2,, +func,Py_NewInterpreter,3.2,, +func,Py_NewRef,3.10,, +func,Py_PACK_FULL_VERSION,3.14,, +func,Py_PACK_VERSION,3.14,, +func,Py_REFCNT,3.14,, +func,Py_ReprEnter,3.2,, +func,Py_ReprLeave,3.2,, +func,Py_SetProgramName,3.2,, +func,Py_SetPythonHome,3.2,, +func,Py_SetRecursionLimit,3.2,, +func,Py_TYPE,3.14,, type,Py_UCS4,3.2,, macro,Py_UNBLOCK_THREADS,3.2,, -var,Py_UTF8Mode,3.8,, -function,Py_VaBuildValue,3.2,, -var,Py_Version,3.11,, -function,Py_XNewRef,3.10,, +data,Py_UTF8Mode,3.8,, +func,Py_VaBuildValue,3.2,, +data,Py_Version,3.11,, +func,Py_XNewRef,3.10,, type,Py_buffer,3.11,,full-abi type,Py_intptr_t,3.2,, type,Py_ssize_t,3.2,, diff --git a/Doc/deprecations/c-api-pending-removal-in-3.14.rst b/Doc/deprecations/c-api-pending-removal-in-3.14.rst new file mode 100644 index 00000000000000..c805074669811a --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-3.14.rst @@ -0,0 +1,8 @@ +Pending removal in Python 3.14 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* The ``ma_version_tag`` field in :c:type:`PyDictObject` for extension modules + (:pep:`699`; :gh:`101193`). + +* Creating :c:data:`immutable types ` with mutable + bases (:gh:`95388`). diff --git a/Doc/deprecations/c-api-pending-removal-in-3.15.rst b/Doc/deprecations/c-api-pending-removal-in-3.15.rst new file mode 100644 index 00000000000000..ac31b3cc8cd451 --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-3.15.rst @@ -0,0 +1,114 @@ +Pending removal in Python 3.15 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* The bundled copy of ``libmpdecimal``. +* The :c:func:`PyImport_ImportModuleNoBlock`: + Use :c:func:`PyImport_ImportModule` instead. +* :c:func:`PyWeakref_GetObject` and :c:func:`PyWeakref_GET_OBJECT`: + Use :c:func:`PyWeakref_GetRef` instead. The `pythoncapi-compat project + `__ can be used to get + :c:func:`PyWeakref_GetRef` on Python 3.12 and older. +* :c:type:`Py_UNICODE` type and the :c:macro:`!Py_UNICODE_WIDE` macro: + Use :c:type:`wchar_t` instead. +* Python initialization functions: + + * :c:func:`PySys_ResetWarnOptions`: + Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. + * :c:func:`Py_GetExecPrefix`: + Get :data:`sys.base_exec_prefix` and :data:`sys.exec_prefix` instead. + * :c:func:`Py_GetPath`: + Get :data:`sys.path` instead. + * :c:func:`Py_GetPrefix`: + Get :data:`sys.base_prefix` and :data:`sys.prefix` instead. + * :c:func:`Py_GetProgramFullPath`: + Get :data:`sys.executable` instead. + * :c:func:`Py_GetProgramName`: + Get :data:`sys.executable` instead. + * :c:func:`Py_GetPythonHome`: + Get :c:func:`PyConfig_Get("home") ` + or the :envvar:`PYTHONHOME` environment variable instead. + + See also the :c:func:`PyConfig_Get` function. + +* Functions to configure Python's initialization, deprecated in Python 3.11: + + * :c:func:`!PySys_SetArgvEx()`: + Set :c:member:`PyConfig.argv` instead. + * :c:func:`!PySys_SetArgv()`: + Set :c:member:`PyConfig.argv` instead. + * :c:func:`!Py_SetProgramName()`: + Set :c:member:`PyConfig.program_name` instead. + * :c:func:`!Py_SetPythonHome()`: + Set :c:member:`PyConfig.home` instead. + + The :c:func:`Py_InitializeFromConfig` API should be used with + :c:type:`PyConfig` instead. + +* Global configuration variables: + + * :c:var:`Py_DebugFlag`: + Use :c:member:`PyConfig.parser_debug` or + :c:func:`PyConfig_Get("parser_debug") ` instead. + * :c:var:`Py_VerboseFlag`: + Use :c:member:`PyConfig.verbose` or + :c:func:`PyConfig_Get("verbose") ` instead. + * :c:var:`Py_QuietFlag`: + Use :c:member:`PyConfig.quiet` or + :c:func:`PyConfig_Get("quiet") ` instead. + * :c:var:`Py_InteractiveFlag`: + Use :c:member:`PyConfig.interactive` or + :c:func:`PyConfig_Get("interactive") ` instead. + * :c:var:`Py_InspectFlag`: + Use :c:member:`PyConfig.inspect` or + :c:func:`PyConfig_Get("inspect") ` instead. + * :c:var:`Py_OptimizeFlag`: + Use :c:member:`PyConfig.optimization_level` or + :c:func:`PyConfig_Get("optimization_level") ` instead. + * :c:var:`Py_NoSiteFlag`: + Use :c:member:`PyConfig.site_import` or + :c:func:`PyConfig_Get("site_import") ` instead. + * :c:var:`Py_BytesWarningFlag`: + Use :c:member:`PyConfig.bytes_warning` or + :c:func:`PyConfig_Get("bytes_warning") ` instead. + * :c:var:`Py_FrozenFlag`: + Use :c:member:`PyConfig.pathconfig_warnings` or + :c:func:`PyConfig_Get("pathconfig_warnings") ` instead. + * :c:var:`Py_IgnoreEnvironmentFlag`: + Use :c:member:`PyConfig.use_environment` or + :c:func:`PyConfig_Get("use_environment") ` instead. + * :c:var:`Py_DontWriteBytecodeFlag`: + Use :c:member:`PyConfig.write_bytecode` or + :c:func:`PyConfig_Get("write_bytecode") ` instead. + * :c:var:`Py_NoUserSiteDirectory`: + Use :c:member:`PyConfig.user_site_directory` or + :c:func:`PyConfig_Get("user_site_directory") ` instead. + * :c:var:`Py_UnbufferedStdioFlag`: + Use :c:member:`PyConfig.buffered_stdio` or + :c:func:`PyConfig_Get("buffered_stdio") ` instead. + * :c:var:`Py_HashRandomizationFlag`: + Use :c:member:`PyConfig.use_hash_seed` + and :c:member:`PyConfig.hash_seed` or + :c:func:`PyConfig_Get("hash_seed") ` instead. + * :c:var:`Py_IsolatedFlag`: + Use :c:member:`PyConfig.isolated` or + :c:func:`PyConfig_Get("isolated") ` instead. + * :c:var:`Py_LegacyWindowsFSEncodingFlag`: + Use :c:member:`PyPreConfig.legacy_windows_fs_encoding` or + :c:func:`PyConfig_Get("legacy_windows_fs_encoding") ` instead. + * :c:var:`Py_LegacyWindowsStdioFlag`: + Use :c:member:`PyConfig.legacy_windows_stdio` or + :c:func:`PyConfig_Get("legacy_windows_stdio") ` instead. + * :c:var:`!Py_FileSystemDefaultEncoding`, :c:var:`!Py_HasFileSystemDefaultEncoding`: + Use :c:member:`PyConfig.filesystem_encoding` or + :c:func:`PyConfig_Get("filesystem_encoding") ` instead. + * :c:var:`!Py_FileSystemDefaultEncodeErrors`: + Use :c:member:`PyConfig.filesystem_errors` or + :c:func:`PyConfig_Get("filesystem_errors") ` instead. + * :c:var:`!Py_UTF8Mode`: + Use :c:member:`PyPreConfig.utf8_mode` or + :c:func:`PyConfig_Get("utf8_mode") ` instead. + (see :c:func:`Py_PreInitialize`) + + The :c:func:`Py_InitializeFromConfig` API should be used with + :c:type:`PyConfig` to set these options. Or :c:func:`PyConfig_Get` can be + used to get these options at runtime. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.18.rst b/Doc/deprecations/c-api-pending-removal-in-3.18.rst new file mode 100644 index 00000000000000..0689d8b4f9e959 --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-3.18.rst @@ -0,0 +1,19 @@ +Pending removal in Python 3.18 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* Deprecated private functions (:gh:`128863`): + + * :c:func:`!_PyBytes_Join`: use :c:func:`PyBytes_Join`. + * :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`. + * :c:func:`!_PyDict_Pop()`: :c:func:`PyDict_Pop`. + * :c:func:`!_PyLong_Sign()`: use :c:func:`PyLong_GetSign`. + * :c:func:`!_PyLong_FromDigits` and :c:func:`!_PyLong_New`: + use :c:func:`PyLongWriter_Create`. + * :c:func:`!_PyThreadState_UncheckedGet`: use :c:func:`PyThreadState_GetUnchecked`. + * :c:func:`!_PyUnicode_AsString`: use :c:func:`PyUnicode_AsUTF8`. + * :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`. + * :c:func:`!_Py_fopen_obj`: use :c:func:`Py_fopen`. + + The `pythoncapi-compat project + `__ can be used to get these + new public functions on Python 3.13 and older. diff --git a/Doc/deprecations/c-api-pending-removal-in-future.rst b/Doc/deprecations/c-api-pending-removal-in-future.rst new file mode 100644 index 00000000000000..1003047344a3cc --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-future.rst @@ -0,0 +1,50 @@ +Pending removal in future versions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The following APIs are deprecated and will be removed, +although there is currently no date scheduled for their removal. + +* :c:macro:`Py_TPFLAGS_HAVE_FINALIZE`: + Unneeded since Python 3.8. +* :c:func:`PyErr_Fetch`: + Use :c:func:`PyErr_GetRaisedException` instead. +* :c:func:`PyErr_NormalizeException`: + Use :c:func:`PyErr_GetRaisedException` instead. +* :c:func:`PyErr_Restore`: + Use :c:func:`PyErr_SetRaisedException` instead. +* :c:func:`PyModule_GetFilename`: + Use :c:func:`PyModule_GetFilenameObject` instead. +* :c:func:`PyOS_AfterFork`: + Use :c:func:`PyOS_AfterFork_Child` instead. +* :c:func:`PySlice_GetIndicesEx`: + Use :c:func:`PySlice_Unpack` and :c:func:`PySlice_AdjustIndices` instead. +* :c:func:`!PyUnicode_AsDecodedObject`: + Use :c:func:`PyCodec_Decode` instead. +* :c:func:`!PyUnicode_AsDecodedUnicode`: + Use :c:func:`PyCodec_Decode` instead. +* :c:func:`!PyUnicode_AsEncodedObject`: + Use :c:func:`PyCodec_Encode` instead. +* :c:func:`!PyUnicode_AsEncodedUnicode`: + Use :c:func:`PyCodec_Encode` instead. +* :c:func:`PyUnicode_READY`: + Unneeded since Python 3.12 +* :c:func:`!PyErr_Display`: + Use :c:func:`PyErr_DisplayException` instead. +* :c:func:`!_PyErr_ChainExceptions`: + Use :c:func:`!_PyErr_ChainExceptions1` instead. +* :c:member:`!PyBytesObject.ob_shash` member: + call :c:func:`PyObject_Hash` instead. +* Thread Local Storage (TLS) API: + + * :c:func:`PyThread_create_key`: + Use :c:func:`PyThread_tss_alloc` instead. + * :c:func:`PyThread_delete_key`: + Use :c:func:`PyThread_tss_free` instead. + * :c:func:`PyThread_set_key_value`: + Use :c:func:`PyThread_tss_set` instead. + * :c:func:`PyThread_get_key_value`: + Use :c:func:`PyThread_tss_get` instead. + * :c:func:`PyThread_delete_key_value`: + Use :c:func:`PyThread_tss_delete` instead. + * :c:func:`PyThread_ReInitTLS`: + Unneeded since Python 3.7. diff --git a/Doc/deprecations/index.rst b/Doc/deprecations/index.rst new file mode 100644 index 00000000000000..bac6e3f18d4594 --- /dev/null +++ b/Doc/deprecations/index.rst @@ -0,0 +1,15 @@ +Deprecations +============ + +.. include:: pending-removal-in-3.15.rst + +.. include:: pending-removal-in-3.16.rst + +.. include:: pending-removal-in-future.rst + +C API deprecations +------------------ + +.. include:: c-api-pending-removal-in-3.15.rst + +.. include:: c-api-pending-removal-in-future.rst diff --git a/Doc/deprecations/pending-removal-in-3.13.rst b/Doc/deprecations/pending-removal-in-3.13.rst new file mode 100644 index 00000000000000..2fd2f12cc6a2c4 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.13.rst @@ -0,0 +1,52 @@ +Pending removal in Python 3.13 +------------------------------ + +Modules (see :pep:`594`): + +* :mod:`!aifc` +* :mod:`!audioop` +* :mod:`!cgi` +* :mod:`!cgitb` +* :mod:`!chunk` +* :mod:`!crypt` +* :mod:`!imghdr` +* :mod:`!mailcap` +* :mod:`!msilib` +* :mod:`!nis` +* :mod:`!nntplib` +* :mod:`!ossaudiodev` +* :mod:`!pipes` +* :mod:`!sndhdr` +* :mod:`!spwd` +* :mod:`!sunau` +* :mod:`!telnetlib` +* :mod:`!uu` +* :mod:`!xdrlib` + +Other modules: + +* :mod:`!lib2to3`, and the :program:`2to3` program (:gh:`84540`) + +APIs: + +* :class:`!configparser.LegacyInterpolation` (:gh:`90765`) +* ``locale.resetlocale()`` (:gh:`90817`) +* :meth:`!turtle.RawTurtle.settiltangle` (:gh:`50096`) +* :func:`!unittest.findTestCases` (:gh:`50096`) +* :func:`!unittest.getTestCaseNames` (:gh:`50096`) +* :func:`!unittest.makeSuite` (:gh:`50096`) +* :meth:`!unittest.TestProgram.usageExit` (:gh:`67048`) +* :class:`!webbrowser.MacOSX` (:gh:`86421`) +* :class:`classmethod` descriptor chaining (:gh:`89519`) +* :mod:`importlib.resources` deprecated methods: + + * ``contents()`` + * ``is_resource()`` + * ``open_binary()`` + * ``open_text()`` + * ``path()`` + * ``read_binary()`` + * ``read_text()`` + + Use :func:`importlib.resources.files` instead. Refer to `importlib-resources: Migrating from Legacy + `_ (:gh:`106531`) diff --git a/Doc/deprecations/pending-removal-in-3.14.rst b/Doc/deprecations/pending-removal-in-3.14.rst new file mode 100644 index 00000000000000..6159fa48848285 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.14.rst @@ -0,0 +1,105 @@ +Pending removal in Python 3.14 +------------------------------ + +* :mod:`argparse`: The *type*, *choices*, and *metavar* parameters + of :class:`!argparse.BooleanOptionalAction` are deprecated + and will be removed in 3.14. + (Contributed by Nikita Sobolev in :gh:`92248`.) + +* :mod:`ast`: The following features have been deprecated in documentation + since Python 3.8, now cause a :exc:`DeprecationWarning` to be emitted at + runtime when they are accessed or used, and will be removed in Python 3.14: + + * :class:`!ast.Num` + * :class:`!ast.Str` + * :class:`!ast.Bytes` + * :class:`!ast.NameConstant` + * :class:`!ast.Ellipsis` + + Use :class:`ast.Constant` instead. + (Contributed by Serhiy Storchaka in :gh:`90953`.) + +* :mod:`asyncio`: + + * The child watcher classes :class:`!asyncio.MultiLoopChildWatcher`, + :class:`!asyncio.FastChildWatcher`, :class:`!asyncio.AbstractChildWatcher` + and :class:`!asyncio.SafeChildWatcher` are deprecated and + will be removed in Python 3.14. + (Contributed by Kumar Aditya in :gh:`94597`.) + + * :func:`!asyncio.set_child_watcher`, :func:`!asyncio.get_child_watcher`, + :meth:`!asyncio.AbstractEventLoopPolicy.set_child_watcher` and + :meth:`!asyncio.AbstractEventLoopPolicy.get_child_watcher` are deprecated + and will be removed in Python 3.14. + (Contributed by Kumar Aditya in :gh:`94597`.) + + * The :meth:`~asyncio.get_event_loop` method of the + default event loop policy now emits a :exc:`DeprecationWarning` if there + is no current event loop set and it decides to create one. + (Contributed by Serhiy Storchaka and Guido van Rossum in :gh:`100160`.) + +* :mod:`collections.abc`: Deprecated :class:`!collections.abc.ByteString`. + Prefer :class:`!Sequence` or :class:`~collections.abc.Buffer`. + For use in typing, prefer a union, like ``bytes | bytearray``, + or :class:`collections.abc.Buffer`. + (Contributed by Shantanu Jain in :gh:`91896`.) + +* :mod:`email`: Deprecated the *isdst* parameter in :func:`email.utils.localtime`. + (Contributed by Alan Williams in :gh:`72346`.) + +* :mod:`importlib.abc` deprecated classes: + + * :class:`!importlib.abc.ResourceReader` + * :class:`!importlib.abc.Traversable` + * :class:`!importlib.abc.TraversableResources` + + Use :mod:`importlib.resources.abc` classes instead: + + * :class:`importlib.resources.abc.Traversable` + * :class:`importlib.resources.abc.TraversableResources` + + (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) + +* :mod:`itertools` had undocumented, inefficient, historically buggy, + and inconsistent support for copy, deepcopy, and pickle operations. + This will be removed in 3.14 for a significant reduction in code + volume and maintenance burden. + (Contributed by Raymond Hettinger in :gh:`101588`.) + +* :mod:`multiprocessing`: The default start method will change to a safer one on + Linux, BSDs, and other non-macOS POSIX platforms where ``'fork'`` is currently + the default (:gh:`84559`). Adding a runtime warning about this was deemed too + disruptive as the majority of code is not expected to care. Use the + :func:`~multiprocessing.get_context` or + :func:`~multiprocessing.set_start_method` APIs to explicitly specify when + your code *requires* ``'fork'``. See :ref:`multiprocessing-start-methods`. + +* :mod:`pathlib`: :meth:`~pathlib.PurePath.is_relative_to` and + :meth:`~pathlib.PurePath.relative_to`: passing additional arguments is + deprecated. + +* :mod:`pkgutil`: :func:`!pkgutil.find_loader` and :func:!pkgutil.get_loader` + now raise :exc:`DeprecationWarning`; + use :func:`importlib.util.find_spec` instead. + (Contributed by Nikita Sobolev in :gh:`97850`.) + +* :mod:`pty`: + + * ``master_open()``: use :func:`pty.openpty`. + * ``slave_open()``: use :func:`pty.openpty`. + +* :mod:`sqlite3`: + + * :data:`!version` and :data:`!version_info`. + + * :meth:`~sqlite3.Cursor.execute` and :meth:`~sqlite3.Cursor.executemany` + if :ref:`named placeholders ` are used and + *parameters* is a sequence instead of a :class:`dict`. + +* :mod:`typing`: :class:`!typing.ByteString`, deprecated since Python 3.9, + now causes a :exc:`DeprecationWarning` to be emitted when it is used. + +* :mod:`urllib`: + :class:`!urllib.parse.Quoter` is deprecated: it was not intended to be a + public API. + (Contributed by Gregory P. Smith in :gh:`88168`.) diff --git a/Doc/deprecations/pending-removal-in-3.15.rst b/Doc/deprecations/pending-removal-in-3.15.rst new file mode 100644 index 00000000000000..390bbff2835cf8 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.15.rst @@ -0,0 +1,94 @@ +Pending removal in Python 3.15 +------------------------------ + +* The import system: + + * Setting :attr:`~module.__cached__` on a module while + failing to set :attr:`__spec__.cached ` + is deprecated. In Python 3.15, :attr:`!__cached__` will cease to be set or + take into consideration by the import system or standard library. (:gh:`97879`) + + * Setting :attr:`~module.__package__` on a module while + failing to set :attr:`__spec__.parent ` + is deprecated. In Python 3.15, :attr:`!__package__` will cease to be set or + take into consideration by the import system or standard library. (:gh:`97879`) + +* :mod:`ctypes`: + + * The undocumented :func:`!ctypes.SetPointerType` function + has been deprecated since Python 3.13. + +* :mod:`http.server`: + + * The obsolete and rarely used :class:`~http.server.CGIHTTPRequestHandler` + has been deprecated since Python 3.13. + No direct replacement exists. + *Anything* is better than CGI to interface + a web server with a request handler. + + * The :option:`!--cgi` flag to the :program:`python -m http.server` + command-line interface has been deprecated since Python 3.13. + +* :class:`locale`: + + * The :func:`~locale.getdefaultlocale` function + has been deprecated since Python 3.11. + Its removal was originally planned for Python 3.13 (:gh:`90817`), + but has been postponed to Python 3.15. + Use :func:`~locale.getlocale`, :func:`~locale.setlocale`, + and :func:`~locale.getencoding` instead. + (Contributed by Hugo van Kemenade in :gh:`111187`.) + +* :mod:`pathlib`: + + * :meth:`.PurePath.is_reserved` + has been deprecated since Python 3.13. + Use :func:`os.path.isreserved` to detect reserved paths on Windows. + +* :mod:`platform`: + + * :func:`~platform.java_ver` has been deprecated since Python 3.13. + This function is only useful for Jython support, has a confusing API, + and is largely untested. + +* :mod:`sysconfig`: + + * The ``check_home`` argument of :func:`sysconfig.is_python_build` has been + deprecated since Python 3.12. + +* :mod:`threading`: + + * :func:`~threading.RLock` will take no arguments in Python 3.15. + Passing any arguments has been deprecated since Python 3.14, + as the Python version does not permit any arguments, + but the C version allows any number of positional or keyword arguments, + ignoring every argument. + +* :mod:`types`: + + * :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was + deprecated in :pep:`626` + since 3.10 and was planned to be removed in 3.12, + but it only got a proper :exc:`DeprecationWarning` in 3.12. + May be removed in 3.15. + (Contributed by Nikita Sobolev in :gh:`101866`.) + +* :mod:`typing`: + + * The undocumented keyword argument syntax for creating + :class:`~typing.NamedTuple` classes + (for example, ``Point = NamedTuple("Point", x=int, y=int)``) + has been deprecated since Python 3.13. + Use the class-based syntax or the functional syntax instead. + + * The :func:`typing.no_type_check_decorator` decorator function + has been deprecated since Python 3.13. + After eight years in the :mod:`typing` module, + it has yet to be supported by any major type checker. + +* :mod:`wave`: + + * The :meth:`~wave.Wave_read.getmark`, :meth:`!setmark`, + and :meth:`~wave.Wave_read.getmarkers` methods of + the :class:`~wave.Wave_read` and :class:`~wave.Wave_write` classes + have been deprecated since Python 3.13. diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst new file mode 100644 index 00000000000000..b408a6d72febe0 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -0,0 +1,92 @@ +Pending removal in Python 3.16 +------------------------------ + +* The import system: + + * Setting :attr:`~module.__loader__` on a module while + failing to set :attr:`__spec__.loader ` + is deprecated. In Python 3.16, :attr:`!__loader__` will cease to be set or + taken into consideration by the import system or the standard library. + +* :mod:`array`: + + * The ``'u'`` format code (:c:type:`wchar_t`) + has been deprecated in documentation since Python 3.3 + and at runtime since Python 3.13. + Use the ``'w'`` format code (:c:type:`Py_UCS4`) + for Unicode characters instead. + +* :mod:`asyncio`: + + * :func:`!asyncio.iscoroutinefunction` is deprecated + and will be removed in Python 3.16; + use :func:`inspect.iscoroutinefunction` instead. + (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) + + * :mod:`asyncio` policy system is deprecated and will be removed in Python 3.16. + In particular, the following classes and functions are deprecated: + + * :class:`asyncio.AbstractEventLoopPolicy` + * :class:`asyncio.DefaultEventLoopPolicy` + * :class:`asyncio.WindowsSelectorEventLoopPolicy` + * :class:`asyncio.WindowsProactorEventLoopPolicy` + * :func:`asyncio.get_event_loop_policy` + * :func:`asyncio.set_event_loop_policy` + * :func:`asyncio.set_event_loop` + + Users should use :func:`asyncio.run` or :class:`asyncio.Runner` with + *loop_factory* to use the desired event loop implementation. + + For example, to use :class:`asyncio.SelectorEventLoop` on Windows:: + + import asyncio + + async def main(): + ... + + asyncio.run(main(), loop_factory=asyncio.SelectorEventLoop) + + (Contributed by Kumar Aditya in :gh:`127949`.) + +* :mod:`builtins`: + + * Bitwise inversion on boolean types, ``~True`` or ``~False`` + has been deprecated since Python 3.12, + as it produces surprising and unintuitive results (``-2`` and ``-1``). + Use ``not x`` instead for the logical negation of a Boolean. + In the rare case that you need the bitwise inversion of + the underlying integer, convert to ``int`` explicitly (``~int(x)``). + +* :mod:`functools`: + + * Calling the Python implementation of :func:`functools.reduce` with *function* + or *sequence* as keyword arguments has been deprecated since Python 3.14. + +* :mod:`shutil`: + + * The :class:`!ExecError` exception + has been deprecated since Python 3.14. + It has not been used by any function in :mod:`!shutil` since Python 3.4, + and is now an alias of :exc:`RuntimeError`. + +* :mod:`symtable`: + + * The :meth:`Class.get_methods ` method + has been deprecated since Python 3.14. + +* :mod:`sys`: + + * The :func:`~sys._enablelegacywindowsfsencoding` function + has been deprecated since Python 3.13. + Use the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable instead. + +* :mod:`sysconfig`: + + * The :func:`!sysconfig.expand_makefile_vars` function + has been deprecated since Python 3.14. + Use the ``vars`` argument of :func:`sysconfig.get_paths` instead. + +* :mod:`tarfile`: + + * The undocumented and unused :attr:`!TarFile.tarfile` attribute + has been deprecated since Python 3.13. diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst new file mode 100644 index 00000000000000..ea7581bf16bd34 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-future.rst @@ -0,0 +1,157 @@ +Pending removal in future versions +---------------------------------- + +The following APIs will be removed in the future, +although there is currently no date scheduled for their removal. + +* :mod:`argparse`: + + * Nesting argument groups and nesting mutually exclusive + groups are deprecated. + * Passing the undocumented keyword argument *prefix_chars* to + :meth:`~argparse.ArgumentParser.add_argument_group` is now + deprecated. + * The :class:`argparse.FileType` type converter is deprecated. + +* :mod:`array`'s ``'u'`` format code (:gh:`57281`) + +* :mod:`builtins`: + + * ``bool(NotImplemented)``. + * Generators: ``throw(type, exc, tb)`` and ``athrow(type, exc, tb)`` + signature is deprecated: use ``throw(exc)`` and ``athrow(exc)`` instead, + the single argument signature. + * Currently Python accepts numeric literals immediately followed by keywords, + for example ``0in x``, ``1or x``, ``0if 1else 2``. It allows confusing and + ambiguous expressions like ``[0x1for x in y]`` (which can be interpreted as + ``[0x1 for x in y]`` or ``[0x1f or x in y]``). A syntax warning is raised + if the numeric literal is immediately followed by one of keywords + :keyword:`and`, :keyword:`else`, :keyword:`for`, :keyword:`if`, + :keyword:`in`, :keyword:`is` and :keyword:`or`. In a future release it + will be changed to a syntax error. (:gh:`87999`) + * Support for ``__index__()`` and ``__int__()`` method returning non-int type: + these methods will be required to return an instance of a strict subclass of + :class:`int`. + * Support for ``__float__()`` method returning a strict subclass of + :class:`float`: these methods will be required to return an instance of + :class:`float`. + * Support for ``__complex__()`` method returning a strict subclass of + :class:`complex`: these methods will be required to return an instance of + :class:`complex`. + * Delegation of ``int()`` to ``__trunc__()`` method. + * Passing a complex number as the *real* or *imag* argument in the + :func:`complex` constructor is now deprecated; it should only be passed + as a single positional argument. + (Contributed by Serhiy Storchaka in :gh:`109218`.) + +* :mod:`calendar`: ``calendar.January`` and ``calendar.February`` constants are + deprecated and replaced by :data:`calendar.JANUARY` and + :data:`calendar.FEBRUARY`. + (Contributed by Prince Roshan in :gh:`103636`.) + +* :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method + instead. + +* :mod:`datetime`: + + * :meth:`~datetime.datetime.utcnow`: + use ``datetime.datetime.now(tz=datetime.UTC)``. + * :meth:`~datetime.datetime.utcfromtimestamp`: + use ``datetime.datetime.fromtimestamp(timestamp, tz=datetime.UTC)``. + +* :mod:`gettext`: Plural value must be an integer. + +* :mod:`importlib`: + + * ``load_module()`` method: use ``exec_module()`` instead. + * :func:`~importlib.util.cache_from_source` *debug_override* parameter is + deprecated: use the *optimization* parameter instead. + +* :mod:`importlib.metadata`: + + * ``EntryPoints`` tuple interface. + * Implicit ``None`` on return values. + +* :mod:`logging`: the ``warn()`` method has been deprecated + since Python 3.3, use :meth:`~logging.warning` instead. + +* :mod:`mailbox`: Use of StringIO input and text mode is deprecated, use + BytesIO and binary mode instead. + +* :mod:`os`: Calling :func:`os.register_at_fork` in multi-threaded process. + +* :class:`!pydoc.ErrorDuringImport`: A tuple value for *exc_info* parameter is + deprecated, use an exception instance. + +* :mod:`re`: More strict rules are now applied for numerical group references + and group names in regular expressions. Only sequence of ASCII digits is now + accepted as a numerical reference. The group name in bytes patterns and + replacement strings can now only contain ASCII letters and digits and + underscore. + (Contributed by Serhiy Storchaka in :gh:`91760`.) + +* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. + +* :mod:`shutil`: :func:`~shutil.rmtree`'s *onerror* parameter is deprecated in + Python 3.12; use the *onexc* parameter instead. + +* :mod:`ssl` options and protocols: + + * :class:`ssl.SSLContext` without protocol argument is deprecated. + * :class:`ssl.SSLContext`: :meth:`~ssl.SSLContext.set_npn_protocols` and + :meth:`!selected_npn_protocol` are deprecated: use ALPN + instead. + * ``ssl.OP_NO_SSL*`` options + * ``ssl.OP_NO_TLS*`` options + * ``ssl.PROTOCOL_SSLv3`` + * ``ssl.PROTOCOL_TLS`` + * ``ssl.PROTOCOL_TLSv1`` + * ``ssl.PROTOCOL_TLSv1_1`` + * ``ssl.PROTOCOL_TLSv1_2`` + * ``ssl.TLSVersion.SSLv3`` + * ``ssl.TLSVersion.TLSv1`` + * ``ssl.TLSVersion.TLSv1_1`` + +* :func:`sysconfig.is_python_build` *check_home* parameter is deprecated and + ignored. + +* :mod:`threading` methods: + + * :meth:`!threading.Condition.notifyAll`: use :meth:`~threading.Condition.notify_all`. + * :meth:`!threading.Event.isSet`: use :meth:`~threading.Event.is_set`. + * :meth:`!threading.Thread.isDaemon`, :meth:`threading.Thread.setDaemon`: + use :attr:`threading.Thread.daemon` attribute. + * :meth:`!threading.Thread.getName`, :meth:`threading.Thread.setName`: + use :attr:`threading.Thread.name` attribute. + * :meth:`!threading.currentThread`: use :meth:`threading.current_thread`. + * :meth:`!threading.activeCount`: use :meth:`threading.active_count`. + +* :class:`typing.Text` (:gh:`92332`). + +* :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value + that is not ``None`` from a test case. + +* :mod:`urllib.parse` deprecated functions: :func:`~urllib.parse.urlparse` instead + + * ``splitattr()`` + * ``splithost()`` + * ``splitnport()`` + * ``splitpasswd()`` + * ``splitport()`` + * ``splitquery()`` + * ``splittag()`` + * ``splittype()`` + * ``splituser()`` + * ``splitvalue()`` + * ``to_bytes()`` + +* :mod:`wsgiref`: ``SimpleHandler.stdout.write()`` should not do partial + writes. + +* :mod:`xml.etree.ElementTree`: Testing the truth value of an + :class:`~xml.etree.ElementTree.Element` is deprecated. In a future release it + will always return ``True``. Prefer explicit ``len(elem)`` or + ``elem is not None`` tests instead. + +* :meth:`zipimport.zipimporter.load_module` is deprecated: + use :meth:`~zipimport.zipimporter.exec_module` instead. diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index b70e1b1fe57e67..b0493bed75b151 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -868,7 +868,7 @@ It is important to call :c:func:`free` at the right time. If a block's address is forgotten but :c:func:`free` is not called for it, the memory it occupies cannot be reused until the program terminates. This is called a :dfn:`memory leak`. On the other hand, if a program calls :c:func:`free` for a block and then -continues to use the block, it creates a conflict with re-use of the block +continues to use the block, it creates a conflict with reuse of the block through another :c:func:`malloc` call. This is called :dfn:`using freed memory`. It has the same bad consequences as referencing uninitialized data --- core dumps, wrong results, mysterious crashes. diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 473a418809cff1..7f57a3a6aac0ed 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -296,7 +296,7 @@ An interesting advantage of using the :c:member:`~PyTypeObject.tp_members` table descriptors that are used at runtime is that any attribute defined this way can have an associated doc string simply by providing the text in the table. An application can use the introspection API to retrieve the descriptor from the -class object, and get the doc string using its :attr:`!__doc__` attribute. +class object, and get the doc string using its :attr:`~type.__doc__` attribute. As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :c:member:`~PyMethodDef.ml_name` value of ``NULL`` is required. @@ -545,7 +545,7 @@ performance-critical objects (such as numbers). .. seealso:: Documentation for the :mod:`weakref` module. -For an object to be weakly referencable, the extension type must set the +For an object to be weakly referenceable, the extension type must set the ``Py_TPFLAGS_MANAGED_WEAKREF`` bit of the :c:member:`~PyTypeObject.tp_flags` field. The legacy :c:member:`~PyTypeObject.tp_weaklistoffset` field should be left as zero. diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst index 7eba9759119b3b..bcf938f117d148 100644 --- a/Doc/extending/newtypes_tutorial.rst +++ b/Doc/extending/newtypes_tutorial.rst @@ -144,7 +144,7 @@ only used for variable-sized objects and should otherwise be zero. If you want your type to be subclassable from Python, and your type has the same :c:member:`~PyTypeObject.tp_basicsize` as its base type, you may have problems with multiple inheritance. A Python subclass of your type will have to list your type first - in its :attr:`~class.__bases__`, or else it will not be able to call your type's + in its :attr:`~type.__bases__`, or else it will not be able to call your type's :meth:`~object.__new__` method without getting an error. You can avoid this problem by ensuring that your type has a larger value for :c:member:`~PyTypeObject.tp_basicsize` than its base type does. Most of the time, this will be true anyway, because either your @@ -447,7 +447,7 @@ Further, the attributes can be deleted, setting the C pointers to ``NULL``. Eve though we can make sure the members are initialized to non-``NULL`` values, the members can be set to ``NULL`` if the attributes are deleted. -We define a single method, :meth:`!Custom.name()`, that outputs the objects name as the +We define a single method, :meth:`!Custom.name`, that outputs the objects name as the concatenation of the first and last names. :: static PyObject * diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index 300e1b6cc40a58..e2710fab9cf800 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -70,7 +70,7 @@ operations. This means that as far as floating-point operations are concerned, Python behaves like many popular languages including C and Java. Many numbers that can be written easily in decimal notation cannot be expressed -exactly in binary floating-point. For example, after:: +exactly in binary floating point. For example, after:: >>> x = 1.2 @@ -87,7 +87,7 @@ which is exactly:: The typical precision of 53 bits provides Python floats with 15--16 decimal digits of accuracy. -For a fuller explanation, please see the :ref:`floating point arithmetic +For a fuller explanation, please see the :ref:`floating-point arithmetic ` chapter in the Python tutorial. @@ -259,9 +259,11 @@ is evaluated in all cases. Why isn't there a switch or case statement in Python? ----------------------------------------------------- -You can do this easily enough with a sequence of ``if... elif... elif... else``. -For literal values, or constants within a namespace, you can also use a -``match ... case`` statement. +In general, structured switch statements execute one block of code +when an expression has a particular value or set of values. +Since Python 3.10 one can easily match literal values, or constants +within a namespace, with a ``match ... case`` statement. +An older alternative is a sequence of ``if... elif... elif... else``. For cases where you need to choose from a very large number of possibilities, you can create a dictionary mapping case values to functions to call. For @@ -290,6 +292,9 @@ It's suggested that you use a prefix for the method names, such as ``visit_`` in this example. Without such a prefix, if values are coming from an untrusted source, an attacker would be able to call any method on your object. +Imitating switch with fallthrough, as with C's switch-case-default, +is possible, much harder, and less needed. + Can't you emulate threads in the interpreter instead of relying on an OS-specific thread implementation? -------------------------------------------------------------------------------------------------------- @@ -323,7 +328,7 @@ Can Python be compiled to machine code, C or some other language? ----------------------------------------------------------------- `Cython `_ compiles a modified version of Python with -optional annotations into C extensions. `Nuitka `_ is +optional annotations into C extensions. `Nuitka `_ is an up-and-coming compiler of Python into C++ code, aiming to support the full Python language. @@ -340,7 +345,7 @@ to perform a garbage collection, obtain debugging statistics, and tune the collector's parameters. Other implementations (such as `Jython `_ or -`PyPy `_), however, can rely on a different mechanism +`PyPy `_), however, can rely on a different mechanism such as a full-blown garbage collector. This difference can cause some subtle porting problems if your Python code depends on the behavior of the reference counting implementation. diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 2a8b976925d042..3147fda7c37124 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -50,7 +50,7 @@ to learn Python's C API. If you need to interface to some C or C++ library for which no Python extension currently exists, you can try wrapping the library's data types and functions with a tool such as `SWIG `_. `SIP -`__, `CXX +`__, `CXX `_ `Boost `_, or `Weave `_ are also @@ -246,13 +246,12 @@ Then, when you run GDB: I want to compile a Python module on my Linux system, but some files are missing. Why? -------------------------------------------------------------------------------------- -Most packaged versions of Python don't include the -:file:`/usr/lib/python2.{x}/config/` directory, which contains various files +Most packaged versions of Python omit some files required for compiling Python extensions. -For Red Hat, install the python-devel RPM to get the necessary files. +For Red Hat, install the python3-devel RPM to get the necessary files. -For Debian, run ``apt-get install python-dev``. +For Debian, run ``apt-get install python3-dev``. How do I tell "incomplete input" from "invalid input"? ------------------------------------------------------ diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index ec7c2897594999..578777d7f23621 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -122,6 +122,8 @@ available. Consult `the Python Package Index `_ to find packages of interest to you. +.. _faq-version-numbering-scheme: + How does the Python version numbering scheme work? -------------------------------------------------- @@ -183,8 +185,6 @@ information on getting the source code and compiling it. How do I get documentation on Python? ------------------------------------- -.. XXX mention py3k - The standard documentation for the current stable version of Python is available at https://docs.python.org/3/. PDF, plain text, and downloadable HTML versions are also available at https://docs.python.org/3/download.html. @@ -309,10 +309,9 @@ guaranteed that interfaces will remain the same throughout a series of bugfix releases. The latest stable releases can always be found on the `Python download page -`_. There are two production-ready versions -of Python: 2.x and 3.x. The recommended version is 3.x, which is supported by -most widely used libraries. Although 2.x is still widely used, `it is not -maintained anymore `_. +`_. +Python 3.x is the recommended version and supported by most widely used libraries. +Python 2.x :pep:`is not maintained anymore <373>`. How many people are using Python? --------------------------------- diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index e2f8004c7e3aea..d8d75ca6f2ec96 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -541,91 +541,12 @@ Thus, to read *n* bytes from a pipe *p* created with :func:`os.popen`, you need use ``p.read(n)``. -.. XXX update to use subprocess. See the :ref:`subprocess-replacements` section. - - How do I run a subprocess with pipes connected to both input and output? - ------------------------------------------------------------------------ - - Use the :mod:`popen2` module. For example:: - - import popen2 - fromchild, tochild = popen2.popen2("command") - tochild.write("input\n") - tochild.flush() - output = fromchild.readline() - - Warning: in general it is unwise to do this because you can easily cause a - deadlock where your process is blocked waiting for output from the child - while the child is blocked waiting for input from you. This can be caused - by the parent expecting the child to output more text than it does or - by data being stuck in stdio buffers due to lack of flushing. - The Python parent can of course explicitly flush the data it sends to the - child before it reads any output, but if the child is a naive C program it - may have been written to never explicitly flush its output, even if it is - interactive, since flushing is normally automatic. - - Note that a deadlock is also possible if you use :func:`popen3` to read - stdout and stderr. If one of the two is too large for the internal buffer - (increasing the buffer size does not help) and you ``read()`` the other one - first, there is a deadlock, too. - - Note on a bug in popen2: unless your program calls ``wait()`` or - ``waitpid()``, finished child processes are never removed, and eventually - calls to popen2 will fail because of a limit on the number of child - processes. Calling :func:`os.waitpid` with the :const:`os.WNOHANG` option can - prevent this; a good place to insert such a call would be before calling - ``popen2`` again. - - In many cases, all you really need is to run some data through a command and - get the result back. Unless the amount of data is very large, the easiest - way to do this is to write it to a temporary file and run the command with - that temporary file as input. The standard module :mod:`tempfile` exports a - :func:`~tempfile.mktemp` function to generate unique temporary file names. :: - - import tempfile - import os - - class Popen3: - """ - This is a deadlock-safe version of popen that returns - an object with errorlevel, out (a string) and err (a string). - (capturestderr may not work under windows.) - Example: print(Popen3('grep spam','\n\nhere spam\n\n').out) - """ - def __init__(self,command,input=None,capturestderr=None): - outfile=tempfile.mktemp() - command="( %s ) > %s" % (command,outfile) - if input: - infile=tempfile.mktemp() - open(infile,"w").write(input) - command=command+" <"+infile - if capturestderr: - errfile=tempfile.mktemp() - command=command+" 2>"+errfile - self.errorlevel=os.system(command) >> 8 - self.out=open(outfile,"r").read() - os.remove(outfile) - if input: - os.remove(infile) - if capturestderr: - self.err=open(errfile,"r").read() - os.remove(errfile) - - Note that many interactive programs (e.g. vi) don't work well with pipes - substituted for standard input and output. You will have to use pseudo ttys - ("ptys") instead of pipes. Or you can use a Python interface to Don Libes' - "expect" library. A Python extension that interfaces to expect is called - "expy" and available from https://expectpy.sourceforge.net. A pure Python - solution that works like expect is `pexpect - `_. - - How do I access the serial (RS232) port? ---------------------------------------- For Win32, OSX, Linux, BSD, Jython, IronPython: - https://pypi.org/project/pyserial/ + :pypi:`pyserial` For Unix, see a Usenet post by Mitch Chapman: @@ -797,12 +718,12 @@ is simple:: import random random.random() -This returns a random floating point number in the range [0, 1). +This returns a random floating-point number in the range [0, 1). There are also many other specialized generators in this module, such as: * ``randrange(a, b)`` chooses an integer in the range [a, b). -* ``uniform(a, b)`` chooses a floating point number in the range [a, b). +* ``uniform(a, b)`` chooses a floating-point number in the range [a, b). * ``normalvariate(mean, sdev)`` samples the normal (Gaussian) distribution. Some higher-level functions operate on sequences directly, such as: diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 0a88c5f6384f2b..776bab1ed5b779 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -869,7 +869,7 @@ How do I convert a string to a number? -------------------------------------- For integers, use the built-in :func:`int` type constructor, e.g. ``int('144') -== 144``. Similarly, :func:`float` converts to floating-point, +== 144``. Similarly, :func:`float` converts to a floating-point number, e.g. ``float('144') == 144.0``. By default, these interpret the number as decimal, so that ``int('0144') == @@ -1013,7 +1013,7 @@ Not as such. For simple input parsing, the easiest approach is usually to split the line into whitespace-delimited words using the :meth:`~str.split` method of string objects and then convert decimal strings to numeric values using :func:`int` or -:func:`float`. :meth:`!split()` supports an optional "sep" parameter which is useful +:func:`float`. :meth:`!split` supports an optional "sep" parameter which is useful if the line uses something other than whitespace as a separator. For more complicated input parsing, regular expressions are more powerful @@ -1613,9 +1613,16 @@ method too, and it must do so carefully. The basic implementation of self.__dict__[name] = value ... -Most :meth:`!__setattr__` implementations must modify -:meth:`self.__dict__ ` to store -local state for self without causing an infinite recursion. +Many :meth:`~object.__setattr__` implementations call :meth:`!object.__setattr__` to set +an attribute on self without causing infinite recursion:: + + class X: + def __setattr__(self, name, value): + # Custom logic here... + object.__setattr__(self, name, value) + +Alternatively, it is possible to set attributes by inserting +entries into :attr:`self.__dict__ ` directly. How do I call a method defined in a base class from a derived class that extends it? @@ -1741,11 +1748,31 @@ but effective way to define class private variables. Any identifier of the form is textually replaced with ``_classname__spam``, where ``classname`` is the current class name with any leading underscores stripped. -This doesn't guarantee privacy: an outside user can still deliberately access -the "_classname__spam" attribute, and private values are visible in the object's -``__dict__``. Many Python programmers never bother to use private variable -names at all. +The identifier can be used unchanged within the class, but to access it outside +the class, the mangled name must be used: + +.. code-block:: python + + class A: + def __one(self): + return 1 + def two(self): + return 2 * self.__one() + class B(A): + def three(self): + return 3 * self._A__one() + + four = 4 * A()._A__one() + +In particular, this does not guarantee privacy since an outside user can still +deliberately access the private attribute; many Python programmers never bother +to use private variable names at all. + +.. seealso:: + + The :ref:`private name mangling specifications ` + for details and special cases. My class defines __del__ but it is not called when I delete the object. ----------------------------------------------------------------------- @@ -1879,28 +1906,30 @@ In the standard library code, you will see several common patterns for correctly using identity tests: 1) As recommended by :pep:`8`, an identity test is the preferred way to check -for ``None``. This reads like plain English in code and avoids confusion with -other objects that may have boolean values that evaluate to false. + for ``None``. This reads like plain English in code and avoids confusion + with other objects that may have boolean values that evaluate to false. 2) Detecting optional arguments can be tricky when ``None`` is a valid input -value. In those situations, you can create a singleton sentinel object -guaranteed to be distinct from other objects. For example, here is how -to implement a method that behaves like :meth:`dict.pop`:: + value. In those situations, you can create a singleton sentinel object + guaranteed to be distinct from other objects. For example, here is how + to implement a method that behaves like :meth:`dict.pop`: + + .. code-block:: python - _sentinel = object() + _sentinel = object() - def pop(self, key, default=_sentinel): - if key in self: - value = self[key] - del self[key] - return value - if default is _sentinel: - raise KeyError(key) - return default + def pop(self, key, default=_sentinel): + if key in self: + value = self[key] + del self[key] + return value + if default is _sentinel: + raise KeyError(key) + return default 3) Container implementations sometimes need to augment equality tests with -identity tests. This prevents the code from being confused by objects such as -``float('NaN')`` that are not equal to themselves. + identity tests. This prevents the code from being confused by objects + such as ``float('NaN')`` that are not equal to themselves. For example, here is the implementation of :meth:`!collections.abc.Sequence.__contains__`:: diff --git a/Doc/glossary.rst b/Doc/glossary.rst index f656e32514c717..e3a14601398e89 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -9,13 +9,14 @@ Glossary .. glossary:: ``>>>`` - The default Python prompt of the interactive shell. Often seen for code - examples which can be executed interactively in the interpreter. + The default Python prompt of the :term:`interactive` shell. Often + seen for code examples which can be executed interactively in the + interpreter. ``...`` Can refer to: - * The default Python prompt of the interactive shell when entering the + * The default Python prompt of the :term:`interactive` shell when entering the code for an indented code block, when within a pair of matching left and right delimiters (parentheses, square brackets, curly braces or triple quotes), or after specifying a decorator. @@ -35,6 +36,12 @@ Glossary and loaders (in the :mod:`importlib.abc` module). You can create your own ABCs with the :mod:`abc` module. + annotate function + A function that can be called to retrieve the :term:`annotations ` + of an object. This function is accessible as the :attr:`~object.__annotate__` + attribute of functions, classes, and modules. Annotate functions are a + subset of :term:`evaluate functions `. + annotation A label associated with a variable, a class attribute or a function parameter or return value, @@ -42,12 +49,11 @@ Glossary Annotations of local variables cannot be accessed at runtime, but annotations of global variables, class attributes, and functions - are stored in the :attr:`__annotations__` - special attribute of modules, classes, and functions, - respectively. + can be retrieved by calling :func:`annotationlib.get_annotations` + on modules, classes, and functions, respectively. - See :term:`variable annotation`, :term:`function annotation`, :pep:`484` - and :pep:`526`, which describe this functionality. + See :term:`variable annotation`, :term:`function annotation`, :pep:`484`, + :pep:`526`, and :pep:`649`, which describe this functionality. Also see :ref:`annotations-howto` for best practices on working with annotations. @@ -109,7 +115,7 @@ Glossary :keyword:`yield` expression. Each :keyword:`yield` temporarily suspends processing, remembering the - location execution state (including local variables and pending + execution state (including local variables and pending try-statements). When the *asynchronous generator iterator* effectively resumes with another awaitable returned by :meth:`~object.__anext__`, it picks up where it left off. See :pep:`492` and :pep:`525`. @@ -225,6 +231,28 @@ Glossary A variable defined in a class and intended to be modified only at class level (i.e., not in an instance of the class). + closure variable + A :term:`free variable` referenced from a :term:`nested scope` that is defined in an outer + scope rather than being resolved at runtime from the globals or builtin namespaces. + May be explicitly defined with the :keyword:`nonlocal` keyword to allow write access, + or implicitly defined if the variable is only being read. + + For example, in the ``inner`` function in the following code, both ``x`` and ``print`` are + :term:`free variables `, but only ``x`` is a *closure variable*:: + + def outer(): + x = 0 + def inner(): + nonlocal x + x += 1 + print(x) + return inner + + Due to the :attr:`codeobject.co_freevars` attribute (which, despite its name, only + includes the names of closure variables rather than listing all referenced free + variables), the more general :term:`free variable` term is sometimes used even + when the intended meaning is to refer specifically to closure variables. + complex number An extension of the familiar real number system in which all numbers are expressed as a sum of a real part and an imaginary part. Imaginary @@ -237,19 +265,33 @@ Glossary advanced mathematical feature. If you're not aware of a need for them, it's almost certain you can safely ignore them. + context + This term has different meanings depending on where and how it is used. + Some common meanings: + + * The temporary state or environment established by a :term:`context + manager` via a :keyword:`with` statement. + * The collection of key­value bindings associated with a particular + :class:`contextvars.Context` object and accessed via + :class:`~contextvars.ContextVar` objects. Also see :term:`context + variable`. + * A :class:`contextvars.Context` object. Also see :term:`current + context`. + + context management protocol + The :meth:`~object.__enter__` and :meth:`~object.__exit__` methods called + by the :keyword:`with` statement. See :pep:`343`. + context manager - An object which controls the environment seen in a :keyword:`with` - statement by defining :meth:`~object.__enter__` and :meth:`~object.__exit__` methods. - See :pep:`343`. + An object which implements the :term:`context management protocol` and + controls the environment seen in a :keyword:`with` statement. See + :pep:`343`. context variable - A variable which can have different values depending on its context. - This is similar to Thread-Local Storage in which each execution - thread may have a different value for a variable. However, with context - variables, there may be several contexts in one execution thread and the - main usage for context variables is to keep track of variables in + A variable whose value depends on which context is the :term:`current + context`. Values are accessed via :class:`contextvars.ContextVar` + objects. Context variables are primarily used to isolate state between concurrent asynchronous tasks. - See :mod:`contextvars`. contiguous .. index:: C-contiguous, Fortran contiguous @@ -283,6 +325,14 @@ Glossary is used when necessary to distinguish this implementation from others such as Jython or IronPython. + current context + The :term:`context` (:class:`contextvars.Context` object) that is + currently used by :class:`~contextvars.ContextVar` objects to access (get + or set) the values of :term:`context variables `. Each + thread has its own current context. Frameworks for executing asynchronous + tasks (see :mod:`asyncio`) associate each task with a context which + becomes the current context whenever the task starts or resumes execution. + decorator A function returning another function, usually applied as a function transformation using the ``@wrapper`` syntax. Common examples for @@ -341,7 +391,7 @@ Glossary docstring A string literal which appears as the first expression in a class, function or module. While ignored when the suite is executed, it is - recognized by the compiler and put into the :attr:`!__doc__` attribute + recognized by the compiler and put into the :attr:`~definition.__doc__` attribute of the enclosing class, function or module. Since it is available via introspection, it is the canonical place for documentation of the object. @@ -365,6 +415,11 @@ Glossary statements. The technique contrasts with the :term:`LBYL` style common to many other languages such as C. + evaluate function + A function that can be called to evaluate a lazily evaluated attribute + of an object, such as the value of type aliases created with the :keyword:`type` + statement. + expression A piece of syntax which can be evaluated to some value. In other words, an expression is an accumulation of expression elements like literals, @@ -424,11 +479,11 @@ Glossary An object that tries to find the :term:`loader` for a module that is being imported. - Since Python 3.3, there are two types of finder: :term:`meta path finders + There are two types of finder: :term:`meta path finders ` for use with :data:`sys.meta_path`, and :term:`path entry finders ` for use with :data:`sys.path_hooks`. - See :pep:`302`, :pep:`420` and :pep:`451` for much more detail. + See :ref:`finders-and-loaders` and :mod:`importlib` for much more detail. floor division Mathematical division that rounds down to nearest integer. The floor @@ -437,6 +492,19 @@ Glossary division. Note that ``(-11) // 4`` is ``-3`` because that is ``-2.75`` rounded *downward*. See :pep:`238`. + free threading + A threading model where multiple threads can run Python bytecode + simultaneously within the same interpreter. This is in contrast to + the :term:`global interpreter lock` which allows only one thread to + execute Python bytecode at a time. See :pep:`703`. + + free variable + Formally, as defined in the :ref:`language execution model `, a free + variable is any variable used in a namespace which is not a local variable in that + namespace. See :term:`closure variable` for an example. + Pragmatically, due to the name of the :attr:`codeobject.co_freevars` attribute, + the term is also sometimes used as a synonym for :term:`closure variable`. + function A series of statements which returns some value to a caller. It can also be passed zero or more :term:`arguments ` which may be used in @@ -496,7 +564,7 @@ Glossary An object created by a :term:`generator` function. Each :keyword:`yield` temporarily suspends processing, remembering the - location execution state (including local variables and pending + execution state (including local variables and pending try-statements). When the *generator iterator* resumes, it picks up where it left off (in contrast to functions which start fresh on every invocation). @@ -547,12 +615,12 @@ Glossary tasks such as compression or hashing. Also, the GIL is always released when doing I/O. - Past efforts to create a "free-threaded" interpreter (one which locks - shared data at a much finer granularity) have not been successful - because performance suffered in the common single-processor case. It - is believed that overcoming this performance issue would make the - implementation much more complicated and therefore costlier to maintain. - + As of Python 3.13, the GIL can be disabled using the :option:`--disable-gil` + build configuration. After building Python with this option, code must be + run with :option:`-X gil=0 <-X>` or after setting the :envvar:`PYTHON_GIL=0 ` + environment variable. This feature enables improved performance for + multi-threaded applications and makes it easier to use multi-core CPUs + efficiently. For more details, see :pep:`703`. hash-based pyc A bytecode cache file that uses the hash rather than the last-modified @@ -583,14 +651,12 @@ Glossary which ships with the standard distribution of Python. immortal - If an object is immortal, its reference count is never modified, and - therefore it is never deallocated. + *Immortal objects* are a CPython implementation detail introduced + in :pep:`683`. - Built-in strings and singletons are immortal objects. For example, - :const:`True` and :const:`None` singletons are immmortal. - - See `PEP 683 – Immortal Objects, Using a Fixed Refcount - `_ for more information. + If an object is immortal, its :term:`reference count` is never modified, + and therefore it is never deallocated while the interpreter is running. + For example, :const:`True` and :const:`None` are immortal in CPython. immutable An object with a fixed value. Immutable objects include numbers, strings and @@ -620,7 +686,8 @@ Glossary execute them and see their results. Just launch ``python`` with no arguments (possibly by selecting it from your computer's main menu). It is a very powerful way to test out new ideas or inspect - modules and packages (remember ``help(x)``). + modules and packages (remember ``help(x)``). For more on interactive + mode, see :ref:`tut-interac`. interpreted Python is an interpreted language, as opposed to a compiled one, @@ -686,6 +753,9 @@ Glossary CPython does not consistently apply the requirement that an iterator define :meth:`~iterator.__iter__`. + And also please note that the free-threading CPython does not guarantee + the thread-safety of iterator operations. + key function A key function or collation function is a callable that returns a value @@ -727,18 +797,6 @@ Glossary thread removes *key* from *mapping* after the test, but before the lookup. This issue can be solved with locks or by using the EAFP approach. - locale encoding - On Unix, it is the encoding of the LC_CTYPE locale. It can be set with - :func:`locale.setlocale(locale.LC_CTYPE, new_locale) `. - - On Windows, it is the ANSI code page (ex: ``"cp1252"``). - - On Android and VxWorks, Python uses ``"utf-8"`` as the locale encoding. - - ``locale.getencoding()`` can be used to get the locale encoding. - - See also the :term:`filesystem encoding and error handler`. - list A built-in Python :term:`sequence`. Despite its name it is more akin to an array in other languages than to a linked list since access to @@ -753,10 +811,27 @@ Glossary processed. loader - An object that loads a module. It must define a method named - :meth:`load_module`. A loader is typically returned by a - :term:`finder`. See :pep:`302` for details and - :class:`importlib.abc.Loader` for an :term:`abstract base class`. + An object that loads a module. + It must define the :meth:`!exec_module` and :meth:`!create_module` methods + to implement the :class:`~importlib.abc.Loader` interface. + A loader is typically returned by a :term:`finder`. + See also: + + * :ref:`finders-and-loaders` + * :class:`importlib.abc.Loader` + * :pep:`302` + + locale encoding + On Unix, it is the encoding of the LC_CTYPE locale. It can be set with + :func:`locale.setlocale(locale.LC_CTYPE, new_locale) `. + + On Windows, it is the ANSI code page (ex: ``"cp1252"``). + + On Android and VxWorks, Python uses ``"utf-8"`` as the locale encoding. + + :func:`locale.getencoding` can be used to get the locale encoding. + + See also the :term:`filesystem encoding and error handler`. magic method .. index:: pair: magic; method @@ -800,8 +875,7 @@ Glossary method resolution order Method Resolution Order is the order in which base classes are searched - for a member during lookup. See `The Python 2.3 Method Resolution Order - `_ for details of the + for a member during lookup. See :ref:`python_2.3_mro` for details of the algorithm used by the Python interpreter since the 2.3 release. module @@ -815,6 +889,8 @@ Glossary A namespace containing the import-related information used to load a module. An instance of :class:`importlib.machinery.ModuleSpec`. + See also :ref:`module-specs`. + MRO See :term:`method resolution order`. @@ -841,10 +917,11 @@ Glossary Some named tuples are built-in types (such as the above examples). Alternatively, a named tuple can be created from a regular class definition that inherits from :class:`tuple` and that defines named - fields. Such a class can be written by hand or it can be created with - the factory function :func:`collections.namedtuple`. The latter - technique also adds some extra methods that may not be found in - hand-written or built-in named tuples. + fields. Such a class can be written by hand, or it can be created by + inheriting :class:`typing.NamedTuple`, or with the factory function + :func:`collections.namedtuple`. The latter techniques also add some + extra methods that may not be found in hand-written or built-in named + tuples. namespace The place where a variable is stored. Namespaces are implemented as @@ -887,6 +964,15 @@ Glossary (methods). Also the ultimate base class of any :term:`new-style class`. + optimized scope + A scope where target local variable names are reliably known to the + compiler when the code is compiled, allowing optimization of read and + write access to these names. The local namespaces for functions, + generators, coroutines, comprehensions, and generator expressions are + optimized in this fashion. Note: most interpreter optimizations are + applied to all scopes, only those relying on a known set of local + and nonlocal variable names are restricted to optimized scopes. + package A Python :term:`module` which can contain submodules or recursively, subpackages. Technically, a package is a Python module with a @@ -1084,6 +1170,10 @@ Glossary See also :term:`namespace package`. + REPL + An acronym for the "read–eval–print loop", another name for the + :term:`interactive` interpreter shell. + __slots__ A declaration inside a class that saves memory by pre-declaring space for instance attributes and eliminating instance dictionaries. Though @@ -1099,7 +1189,7 @@ Glossary :class:`tuple`, and :class:`bytes`. Note that :class:`dict` also supports :meth:`~object.__getitem__` and :meth:`!__len__`, but is considered a mapping rather than a sequence because the lookups use arbitrary - :term:`immutable` keys rather than integers. + :term:`hashable` keys rather than integers. The :class:`collections.abc.Sequence` abstract base class defines a much richer interface that goes beyond just @@ -1128,16 +1218,12 @@ Glossary (subscript) notation uses :class:`slice` objects internally. soft deprecated - A soft deprecation can be used when using an API which should no longer - be used to write new code, but it remains safe to continue using it in - existing code. The API remains documented and tested, but will not be - developed further (no enhancement). - - The main difference between a "soft" and a (regular) "hard" deprecation - is that the soft deprecation does not imply scheduling the removal of the - deprecated API. + A soft deprecated API should not be used in new code, + but it is safe for already existing code to use it. + The API remains documented and tested, but will not be enhanced further. - Another difference is that a soft deprecation does not issue a warning. + Soft deprecation, unlike normal deprecation, does not plan on removing the API + and will not emit warnings. See `PEP 387: Soft Deprecation `_. @@ -1209,7 +1295,7 @@ Glossary type The type of a Python object determines what kind of object it is; every object has a type. An object's type is accessible as its - :attr:`~instance.__class__` attribute or can be retrieved with + :attr:`~object.__class__` attribute or can be retrieved with ``type(obj)``. type alias diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index be8c7e6c827f57..78f3704ba5d000 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -34,11 +34,16 @@ Accessing The Annotations Dict Of An Object In Python 3.10 And Newer Python 3.10 adds a new function to the standard library: :func:`inspect.get_annotations`. In Python versions 3.10 -and newer, calling this function is the best practice for +through 3.13, calling this function is the best practice for accessing the annotations dict of any object that supports annotations. This function can also "un-stringize" stringized annotations for you. +In Python 3.14, there is a new :mod:`annotationlib` module +with functionality for working with annotations. This +includes a :func:`annotationlib.get_annotations` function, +which supersedes :func:`inspect.get_annotations`. + If for some reason :func:`inspect.get_annotations` isn't viable for your use case, you may access the ``__annotations__`` data member manually. Best practice @@ -102,9 +107,9 @@ Your code will have to have a separate code path if the object you're examining is a class (``isinstance(o, type)``). In that case, best practice relies on an implementation detail of Python 3.9 and before: if a class has annotations defined, -they are stored in the class's ``__dict__`` dictionary. Since +they are stored in the class's :attr:`~type.__dict__` dictionary. Since the class may or may not have annotations defined, best practice -is to call the ``get`` method on the class dict. +is to call the :meth:`~dict.get` method on the class dict. To put it all together, here is some sample code that safely accesses the ``__annotations__`` attribute on an arbitrary @@ -121,8 +126,8 @@ the type of ``ann`` using :func:`isinstance` before further examination. Note that some exotic or malformed type objects may not have -a ``__dict__`` attribute, so for extra safety you may also wish -to use :func:`getattr` to access ``__dict__``. +a :attr:`~type.__dict__` attribute, so for extra safety you may also wish +to use :func:`getattr` to access :attr:`!__dict__`. Manually Un-Stringizing Stringized Annotations @@ -184,7 +189,11 @@ Best Practices For ``__annotations__`` In Any Python Version * If you do assign directly to the ``__annotations__`` member of an object, you should always set it to a ``dict`` object. -* If you directly access the ``__annotations__`` member +* You should avoid accessing ``__annotations__`` directly on any object. + Instead, use :func:`annotationlib.get_annotations` (Python 3.14+) + or :func:`inspect.get_annotations` (Python 3.10+). + +* If you do directly access the ``__annotations__`` member of an object, you should ensure that it's a dictionary before attempting to examine its contents. @@ -231,3 +240,12 @@ itself be quoted. In effect the annotation is quoted This prints ``{'a': "'str'"}``. This shouldn't really be considered a "quirk"; it's mentioned here simply because it might be surprising. + +If you use a class with a custom metaclass and access ``__annotations__`` +on the class, you may observe unexpected behavior; see +:pep:`749 <749#pep749-metaclasses>` for some examples. You can avoid these +quirks by using :func:`annotationlib.get_annotations` on Python 3.14+ or +:func:`inspect.get_annotations` on Python 3.10+. On earlier versions of +Python, you can avoid these bugs by accessing the annotations from the +class's :attr:`~type.__dict__` +(e.g., ``cls.__dict__.get('__annotations__', None)``). diff --git a/Doc/howto/argparse-optparse.rst b/Doc/howto/argparse-optparse.rst new file mode 100644 index 00000000000000..b684619885b4c7 --- /dev/null +++ b/Doc/howto/argparse-optparse.rst @@ -0,0 +1,65 @@ +.. currentmodule:: argparse + +.. _upgrading-optparse-code: +.. _migrating-optparse-code: + +============================================ +Migrating ``optparse`` code to ``argparse`` +============================================ + +The :mod:`argparse` module offers several higher level features not natively +provided by the :mod:`optparse` module, including: + +* Handling positional arguments. +* Supporting subcommands. +* Allowing alternative option prefixes like ``+`` and ``/``. +* Handling zero-or-more and one-or-more style arguments. +* Producing more informative usage messages. +* Providing a much simpler interface for custom ``type`` and ``action``. + +Originally, the :mod:`argparse` module attempted to maintain compatibility +with :mod:`optparse`. However, the fundamental design differences between +supporting declarative command line option processing (while leaving positional +argument processing to application code), and supporting both named options +and positional arguments in the declarative interface mean that the +API has diverged from that of ``optparse`` over time. + +As described in :ref:`choosing-an-argument-parser`, applications that are +currently using :mod:`optparse` and are happy with the way it works can +just continue to use ``optparse``. + +Application developers that are considering migrating should also review +the list of intrinsic behavioural differences described in that section +before deciding whether or not migration is desirable. + +For applications that do choose to migrate from :mod:`optparse` to :mod:`argparse`, +the following suggestions should be helpful: + +* Replace all :meth:`optparse.OptionParser.add_option` calls with + :meth:`ArgumentParser.add_argument` calls. + +* Replace ``(options, args) = parser.parse_args()`` with ``args = + parser.parse_args()`` and add additional :meth:`ArgumentParser.add_argument` + calls for the positional arguments. Keep in mind that what was previously + called ``options``, now in the :mod:`argparse` context is called ``args``. + +* Replace :meth:`optparse.OptionParser.disable_interspersed_args` + by using :meth:`~ArgumentParser.parse_intermixed_args` instead of + :meth:`~ArgumentParser.parse_args`. + +* Replace callback actions and the ``callback_*`` keyword arguments with + ``type`` or ``action`` arguments. + +* Replace string names for ``type`` keyword arguments with the corresponding + type objects (e.g. int, float, complex, etc). + +* Replace :class:`optparse.Values` with :class:`Namespace` and + :exc:`optparse.OptionError` and :exc:`optparse.OptionValueError` with + :exc:`ArgumentError`. + +* Replace strings with implicit arguments such as ``%default`` or ``%prog`` with + the standard Python syntax to use dictionaries to format strings, that is, + ``%(default)s`` and ``%(prog)s``. + +* Replace the OptionParser constructor ``version`` argument with a call to + ``parser.add_argument('--version', action='version', version='')``. diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index ae5bab90bf8131..902c50de00803c 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -13,11 +13,16 @@ recommended command-line parsing module in the Python standard library. .. note:: - There are two other modules that fulfill the same task, namely - :mod:`getopt` (an equivalent for ``getopt()`` from the C - language) and the deprecated :mod:`optparse`. - Note also that :mod:`argparse` is based on :mod:`optparse`, - and therefore very similar in terms of usage. + The standard library includes two other libraries directly related + to command-line parameter processing: the lower level :mod:`optparse` + module (which may require more code to configure for a given application, + but also allows an application to request behaviors that ``argparse`` + doesn't support), and the very low level :mod:`getopt` (which specifically + serves as an equivalent to the :c:func:`!getopt` family of functions + available to C programmers). + While neither of those modules is covered directly in this guide, many of + the core concepts in ``argparse`` first originated in ``optparse``, so + some aspects of this tutorial will also be relevant to ``optparse`` users. Concepts @@ -444,7 +449,7 @@ And the output: options: -h, --help show this help message and exit - -v {0,1,2}, --verbosity {0,1,2} + -v, --verbosity {0,1,2} increase output verbosity Note that the change also reflects both in the error message as well as the @@ -841,6 +846,53 @@ translated messages. To translate your own strings in the :mod:`argparse` output, use :mod:`gettext`. +Custom type converters +====================== + +The :mod:`argparse` module allows you to specify custom type converters for +your command-line arguments. This allows you to modify user input before it's +stored in the :class:`argparse.Namespace`. This can be useful when you need to +pre-process the input before it is used in your program. + +When using a custom type converter, you can use any callable that takes a +single string argument (the argument value) and returns the converted value. +However, if you need to handle more complex scenarios, you can use a custom +action class with the **action** parameter instead. + +For example, let's say you want to handle arguments with different prefixes and +process them accordingly:: + + import argparse + + parser = argparse.ArgumentParser(prefix_chars='-+') + + parser.add_argument('-a', metavar='', action='append', + type=lambda x: ('-', x)) + parser.add_argument('+a', metavar='', action='append', + type=lambda x: ('+', x)) + + args = parser.parse_args() + print(args) + +Output: + +.. code-block:: shell-session + + $ python prog.py -a value1 +a value2 + Namespace(a=[('-', 'value1'), ('+', 'value2')]) + +In this example, we: + +* Created a parser with custom prefix characters using the ``prefix_chars`` + parameter. + +* Defined two arguments, ``-a`` and ``+a``, which used the ``type`` parameter to + create custom type converters to store the value in a tuple with the prefix. + +Without the custom type converters, the arguments would have treated the ``-a`` +and ``+a`` as the same argument, which would have been undesirable. By using custom +type converters, we were able to differentiate between the two arguments. + Conclusion ========== diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst index 4828e2fa29bd24..f9ad81e38f8dc3 100644 --- a/Doc/howto/curses.rst +++ b/Doc/howto/curses.rst @@ -43,7 +43,7 @@ appearance---and the curses library will figure out what control codes need to be sent to the terminal to produce the right output. curses doesn't provide many user-interface concepts such as buttons, checkboxes, or dialogs; if you need such features, consider a user interface library such as -`Urwid `_. +:pypi:`Urwid`. The curses library was originally written for BSD Unix; the later System V versions of Unix from AT&T added many enhancements and new functions. BSD curses @@ -56,8 +56,7 @@ versions of curses carried by some proprietary Unixes may not support everything, though. The Windows version of Python doesn't include the :mod:`curses` -module. A ported version called `UniCurses -`_ is available. +module. A ported version called :pypi:`UniCurses` is available. The Python curses module @@ -429,8 +428,7 @@ User Input The C curses library offers only very simple input mechanisms. Python's :mod:`curses` module adds a basic text-input widget. (Other libraries -such as `Urwid `_ have more extensive -collections of widgets.) +such as :pypi:`Urwid` have more extensive collections of widgets.) There are two methods for getting input from a window: diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 51f9f4a6556e57..f6c3e473f1c36d 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -42,7 +42,7 @@ add new capabilities one by one. Simple example: A descriptor that returns a constant ---------------------------------------------------- -The :class:`Ten` class is a descriptor whose :meth:`__get__` method always +The :class:`!Ten` class is a descriptor whose :meth:`~object.__get__` method always returns the constant ``10``: .. testcode:: @@ -120,10 +120,10 @@ different, updated answers each time:: 2 Besides showing how descriptors can run computations, this example also -reveals the purpose of the parameters to :meth:`__get__`. The *self* +reveals the purpose of the parameters to :meth:`~object.__get__`. The *self* parameter is *size*, an instance of *DirectorySize*. The *obj* parameter is either *g* or *s*, an instance of *Directory*. It is the *obj* parameter that -lets the :meth:`__get__` method learn the target directory. The *objtype* +lets the :meth:`~object.__get__` method learn the target directory. The *objtype* parameter is the class *Directory*. @@ -133,7 +133,7 @@ Managed attributes A popular use for descriptors is managing access to instance data. The descriptor is assigned to a public attribute in the class dictionary while the actual data is stored as a private attribute in the instance dictionary. The -descriptor's :meth:`__get__` and :meth:`__set__` methods are triggered when +descriptor's :meth:`~object.__get__` and :meth:`~object.__set__` methods are triggered when the public attribute is accessed. In the following example, *age* is the public attribute and *_age* is the @@ -215,9 +215,9 @@ Customized names When a class uses descriptors, it can inform each descriptor about which variable name was used. -In this example, the :class:`Person` class has two descriptor instances, -*name* and *age*. When the :class:`Person` class is defined, it makes a -callback to :meth:`__set_name__` in *LoggedAccess* so that the field names can +In this example, the :class:`!Person` class has two descriptor instances, +*name* and *age*. When the :class:`!Person` class is defined, it makes a +callback to :meth:`~object.__set_name__` in *LoggedAccess* so that the field names can be recorded, giving each descriptor its own *public_name* and *private_name*: .. testcode:: @@ -253,8 +253,8 @@ be recorded, giving each descriptor its own *public_name* and *private_name*: def birthday(self): self.age += 1 -An interactive session shows that the :class:`Person` class has called -:meth:`__set_name__` so that the field names would be recorded. Here +An interactive session shows that the :class:`!Person` class has called +:meth:`~object.__set_name__` so that the field names would be recorded. Here we call :func:`vars` to look up the descriptor without triggering it: .. doctest:: @@ -294,10 +294,10 @@ The two *Person* instances contain only the private names: Closing thoughts ---------------- -A :term:`descriptor` is what we call any object that defines :meth:`__get__`, -:meth:`__set__`, or :meth:`__delete__`. +A :term:`descriptor` is what we call any object that defines :meth:`~object.__get__`, +:meth:`~object.__set__`, or :meth:`~object.__delete__`. -Optionally, descriptors can have a :meth:`__set_name__` method. This is only +Optionally, descriptors can have a :meth:`~object.__set_name__` method. This is only used in cases where a descriptor needs to know either the class where it was created or the name of class variable it was assigned to. (This method, if present, is called even if the class is not a descriptor.) @@ -337,7 +337,7 @@ any data, it verifies that the new value meets various type and range restrictions. If those restrictions aren't met, it raises an exception to prevent data corruption at its source. -This :class:`Validator` class is both an :term:`abstract base class` and a +This :class:`!Validator` class is both an :term:`abstract base class` and a managed attribute descriptor: .. testcode:: @@ -360,8 +360,8 @@ managed attribute descriptor: def validate(self, value): pass -Custom validators need to inherit from :class:`Validator` and must supply a -:meth:`validate` method to test various restrictions as needed. +Custom validators need to inherit from :class:`!Validator` and must supply a +:meth:`!validate` method to test various restrictions as needed. Custom validators @@ -369,13 +369,13 @@ Custom validators Here are three practical data validation utilities: -1) :class:`OneOf` verifies that a value is one of a restricted set of options. +1) :class:`!OneOf` verifies that a value is one of a restricted set of options. -2) :class:`Number` verifies that a value is either an :class:`int` or +2) :class:`!Number` verifies that a value is either an :class:`int` or :class:`float`. Optionally, it verifies that a value is between a given minimum or maximum. -3) :class:`String` verifies that a value is a :class:`str`. Optionally, it +3) :class:`!String` verifies that a value is a :class:`str`. Optionally, it validates a given minimum or maximum length. It can validate a user-defined `predicate `_ as well. @@ -389,7 +389,9 @@ Here are three practical data validation utilities: def validate(self, value): if value not in self.options: - raise ValueError(f'Expected {value!r} to be one of {self.options!r}') + raise ValueError( + f'Expected {value!r} to be one of {self.options!r}' + ) class Number(Validator): @@ -469,6 +471,7 @@ The descriptors prevent invalid instances from being created: Traceback (most recent call last): ... ValueError: Expected -5 to be at least 0 + >>> Component('WIDGET', 'metal', 'V') # Blocked: 'V' isn't a number Traceback (most recent call last): ... @@ -498,8 +501,8 @@ Definition and introduction --------------------------- In general, a descriptor is an attribute value that has one of the methods in -the descriptor protocol. Those methods are :meth:`__get__`, :meth:`__set__`, -and :meth:`__delete__`. If any of those methods are defined for an +the descriptor protocol. Those methods are :meth:`~object.__get__`, :meth:`~object.__set__`, +and :meth:`~object.__delete__`. If any of those methods are defined for an attribute, it is said to be a :term:`descriptor`. The default behavior for attribute access is to get, set, or delete the @@ -513,7 +516,7 @@ were defined. Descriptors are a powerful, general purpose protocol. They are the mechanism behind properties, methods, static methods, class methods, and -:func:`super()`. They are used throughout Python itself. Descriptors +:func:`super`. They are used throughout Python itself. Descriptors simplify the underlying C code and offer a flexible set of new tools for everyday Python programs. @@ -531,8 +534,8 @@ That is all there is to it. Define any of these methods and an object is considered a descriptor and can override default behavior upon being looked up as an attribute. -If an object defines :meth:`__set__` or :meth:`__delete__`, it is considered -a data descriptor. Descriptors that only define :meth:`__get__` are called +If an object defines :meth:`~object.__set__` or :meth:`~object.__delete__`, it is considered +a data descriptor. Descriptors that only define :meth:`~object.__get__` are called non-data descriptors (they are often used for methods but other uses are possible). @@ -542,9 +545,9 @@ has an entry with the same name as a data descriptor, the data descriptor takes precedence. If an instance's dictionary has an entry with the same name as a non-data descriptor, the dictionary entry takes precedence. -To make a read-only data descriptor, define both :meth:`__get__` and -:meth:`__set__` with the :meth:`__set__` raising an :exc:`AttributeError` when -called. Defining the :meth:`__set__` method with an exception raising +To make a read-only data descriptor, define both :meth:`~object.__get__` and +:meth:`~object.__set__` with the :meth:`~object.__set__` raising an :exc:`AttributeError` when +called. Defining the :meth:`~object.__set__` method with an exception raising placeholder is enough to make it a data descriptor. @@ -559,8 +562,8 @@ attribute access. The expression ``obj.x`` looks up the attribute ``x`` in the chain of namespaces for ``obj``. If the search finds a descriptor outside of the -instance ``__dict__``, its :meth:`__get__` method is invoked according to the -precedence rules listed below. +instance :attr:`~object.__dict__`, its :meth:`~object.__get__` method is +invoked according to the precedence rules listed below. The details of invocation depend on whether ``obj`` is an object, class, or instance of super. @@ -571,7 +574,7 @@ Invocation from an instance Instance lookup scans through a chain of namespaces giving data descriptors the highest priority, followed by instance variables, then non-data -descriptors, then class variables, and lastly :meth:`__getattr__` if it is +descriptors, then class variables, and lastly :meth:`~object.__getattr__` if it is provided. If a descriptor is found for ``a.x``, then it is invoked with: @@ -716,12 +719,12 @@ a pure Python equivalent: >>> object_getattribute(u2, 'x') == u2.x == (D1, u2, U2) True -Note, there is no :meth:`__getattr__` hook in the :meth:`__getattribute__` -code. That is why calling :meth:`__getattribute__` directly or with -``super().__getattribute__`` will bypass :meth:`__getattr__` entirely. +Note, there is no :meth:`~object.__getattr__` hook in the :meth:`~object.__getattribute__` +code. That is why calling :meth:`~object.__getattribute__` directly or with +``super().__getattribute__`` will bypass :meth:`~object.__getattr__` entirely. Instead, it is the dot operator and the :func:`getattr` function that are -responsible for invoking :meth:`__getattr__` whenever :meth:`__getattribute__` +responsible for invoking :meth:`~object.__getattr__` whenever :meth:`~object.__getattribute__` raises an :exc:`AttributeError`. Their logic is encapsulated in a helper function: @@ -773,8 +776,8 @@ Invocation from a class ----------------------- The logic for a dotted lookup such as ``A.x`` is in -:meth:`type.__getattribute__`. The steps are similar to those for -:meth:`object.__getattribute__` but the instance dictionary lookup is replaced +:meth:`!type.__getattribute__`. The steps are similar to those for +:meth:`!object.__getattribute__` but the instance dictionary lookup is replaced by a search through the class's :term:`method resolution order`. If a descriptor is found, it is invoked with ``desc.__get__(None, A)``. @@ -786,8 +789,8 @@ The full C implementation can be found in :c:func:`!type_getattro` and Invocation from super --------------------- -The logic for super's dotted lookup is in the :meth:`__getattribute__` method for -object returned by :class:`super()`. +The logic for super's dotted lookup is in the :meth:`~object.__getattribute__` method for +object returned by :func:`super`. A dotted lookup such as ``super(A, obj).m`` searches ``obj.__class__.__mro__`` for the base class ``B`` immediately following ``A`` and then returns @@ -803,21 +806,21 @@ The full C implementation can be found in :c:func:`!super_getattro` in Summary of invocation logic --------------------------- -The mechanism for descriptors is embedded in the :meth:`__getattribute__()` +The mechanism for descriptors is embedded in the :meth:`~object.__getattribute__` methods for :class:`object`, :class:`type`, and :func:`super`. The important points to remember are: -* Descriptors are invoked by the :meth:`__getattribute__` method. +* Descriptors are invoked by the :meth:`~object.__getattribute__` method. * Classes inherit this machinery from :class:`object`, :class:`type`, or :func:`super`. -* Overriding :meth:`__getattribute__` prevents automatic descriptor calls +* Overriding :meth:`~object.__getattribute__` prevents automatic descriptor calls because all the descriptor logic is in that method. -* :meth:`object.__getattribute__` and :meth:`type.__getattribute__` make - different calls to :meth:`__get__`. The first includes the instance and may +* :meth:`!object.__getattribute__` and :meth:`!type.__getattribute__` make + different calls to :meth:`~object.__get__`. The first includes the instance and may include the class. The second puts in ``None`` for the instance and always includes the class. @@ -832,16 +835,16 @@ Automatic name notification Sometimes it is desirable for a descriptor to know what class variable name it was assigned to. When a new class is created, the :class:`type` metaclass scans the dictionary of the new class. If any of the entries are descriptors -and if they define :meth:`__set_name__`, that method is called with two +and if they define :meth:`~object.__set_name__`, that method is called with two arguments. The *owner* is the class where the descriptor is used, and the *name* is the class variable the descriptor was assigned to. The implementation details are in :c:func:`!type_new` and :c:func:`!set_names` in :source:`Objects/typeobject.c`. -Since the update logic is in :meth:`type.__new__`, notifications only take +Since the update logic is in :meth:`!type.__new__`, notifications only take place at the time of class creation. If descriptors are added to the class -afterwards, :meth:`__set_name__` will need to be called manually. +afterwards, :meth:`~object.__set_name__` will need to be called manually. ORM example @@ -870,7 +873,7 @@ care of lookups or updates: conn.execute(self.store, [value, obj.key]) conn.commit() -We can use the :class:`Field` class to define `models +We can use the :class:`!Field` class to define `models `_ that describe the schema for each table in a database: @@ -990,7 +993,7 @@ The documentation shows a typical use to define a managed attribute ``x``: AttributeError: 'C' object has no attribute '_C__x' To see how :func:`property` is implemented in terms of the descriptor protocol, -here is a pure Python equivalent: +here is a pure Python equivalent that implements most of the core functionality: .. testcode:: @@ -1004,59 +1007,35 @@ here is a pure Python equivalent: if doc is None and fget is not None: doc = fget.__doc__ self.__doc__ = doc - self._name = None def __set_name__(self, owner, name): - self._name = name - - @property - def __name__(self): - return self._name if self._name is not None else self.fget.__name__ - - @__name__.setter - def __name__(self, value): - self._name = value + self.__name__ = name def __get__(self, obj, objtype=None): if obj is None: return self if self.fget is None: - raise AttributeError( - f'property {self.__name__!r} of {type(obj).__name__!r} ' - 'object has no getter' - ) + raise AttributeError return self.fget(obj) def __set__(self, obj, value): if self.fset is None: - raise AttributeError( - f'property {self.__name__!r} of {type(obj).__name__!r} ' - 'object has no setter' - ) + raise AttributeError self.fset(obj, value) def __delete__(self, obj): if self.fdel is None: - raise AttributeError( - f'property {self.__name__!r} of {type(obj).__name__!r} ' - 'object has no deleter' - ) + raise AttributeError self.fdel(obj) def getter(self, fget): - prop = type(self)(fget, self.fset, self.fdel, self.__doc__) - prop._name = self._name - return prop + return type(self)(fget, self.fset, self.fdel, self.__doc__) def setter(self, fset): - prop = type(self)(self.fget, fset, self.fdel, self.__doc__) - prop._name = self._name - return prop + return type(self)(self.fget, fset, self.fdel, self.__doc__) def deleter(self, fdel): - prop = type(self)(self.fget, self.fset, fdel, self.__doc__) - prop._name = self._name - return prop + return type(self)(self.fget, self.fset, fdel, self.__doc__) .. testcode:: :hide: @@ -1119,23 +1098,23 @@ here is a pure Python equivalent: >>> try: ... cc.no_getter ... except AttributeError as e: - ... e.args[0] + ... type(e).__name__ ... - "property 'no_getter' of 'CC' object has no getter" + 'AttributeError' >>> try: ... cc.no_setter = 33 ... except AttributeError as e: - ... e.args[0] + ... type(e).__name__ ... - "property 'no_setter' of 'CC' object has no setter" + 'AttributeError' >>> try: ... del cc.no_deleter ... except AttributeError as e: - ... e.args[0] + ... type(e).__name__ ... - "property 'no_deleter' of 'CC' object has no deleter" + 'AttributeError' >>> CC.no_doc.__doc__ is None True @@ -1161,7 +1140,7 @@ to wrap access to the value attribute in a property data descriptor: self.recalc() return self._value -Either the built-in :func:`property` or our :func:`Property` equivalent would +Either the built-in :func:`property` or our :func:`!Property` equivalent would work in this example. @@ -1208,7 +1187,7 @@ roughly equivalent to: return self To support automatic creation of methods, functions include the -:meth:`__get__` method for binding methods during attribute access. This +:meth:`~object.__get__` method for binding methods during attribute access. This means that functions are non-data descriptors that return bound methods during dotted lookup from an instance. Here's how it works: @@ -1252,19 +1231,19 @@ The function has a :term:`qualified name` attribute to support introspection: 'D.f' Accessing the function through the class dictionary does not invoke -:meth:`__get__`. Instead, it just returns the underlying function object:: +:meth:`~object.__get__`. Instead, it just returns the underlying function object:: >>> D.__dict__['f'] -Dotted access from a class calls :meth:`__get__` which just returns the +Dotted access from a class calls :meth:`~object.__get__` which just returns the underlying function unchanged:: >>> D.f The interesting behavior occurs during dotted access from an instance. The -dotted lookup calls :meth:`__get__` which returns a bound method object:: +dotted lookup calls :meth:`~object.__get__` which returns a bound method object:: >>> d = D() >>> d.f @@ -1289,7 +1268,7 @@ Kinds of methods Non-data descriptors provide a simple mechanism for variations on the usual patterns of binding functions into methods. -To recap, functions have a :meth:`__get__` method so that they can be converted +To recap, functions have a :meth:`~object.__get__` method so that they can be converted to a method when accessed as attributes. The non-data descriptor transforms an ``obj.f(*args)`` call into ``f(obj, *args)``. Calling ``cls.f(*args)`` becomes ``f(*args)``. @@ -1326,8 +1305,8 @@ mean, median, and other descriptive statistics that depend on the data. However, there may be useful functions which are conceptually related but do not depend on the data. For instance, ``erf(x)`` is handy conversion routine that comes up in statistical work but does not directly depend on a particular dataset. -It can be called either from an object or the class: ``s.erf(1.5) --> .9332`` or -``Sample.erf(1.5) --> .9332``. +It can be called either from an object or the class: ``s.erf(1.5) --> 0.9332`` +or ``Sample.erf(1.5) --> 0.9332``. Since static methods return the underlying function with no changes, the example calls are unexciting: @@ -1366,11 +1345,15 @@ Using the non-data descriptor protocol, a pure Python version of def __call__(self, *args, **kwds): return self.f(*args, **kwds) + @property + def __annotations__(self): + return self.f.__annotations__ + The :func:`functools.update_wrapper` call adds a ``__wrapped__`` attribute that refers to the underlying function. Also it carries forward the attributes necessary to make the wrapper look like the wrapped -function: :attr:`~function.__name__`, :attr:`~function.__qualname__`, -:attr:`~function.__doc__`, and :attr:`~function.__annotations__`. +function, including :attr:`~function.__name__`, :attr:`~function.__qualname__`, +and :attr:`~function.__doc__`. .. testcode:: :hide: @@ -1688,7 +1671,7 @@ by member descriptors: 'Emulate member_repr() in Objects/descrobject.c' return f'' -The :meth:`type.__new__` method takes care of adding member objects to class +The :meth:`!type.__new__` method takes care of adding member objects to class variables: .. testcode:: @@ -1739,7 +1722,7 @@ Python: ) super().__delattr__(name) -To use the simulation in a real class, just inherit from :class:`Object` and +To use the simulation in a real class, just inherit from :class:`!Object` and set the :term:`metaclass` to :class:`Type`: .. testcode:: diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 30be15230fc088..6441b7aed1eda8 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -1,3 +1,5 @@ +.. _enum-howto: + ========== Enum HOWTO ========== @@ -7,7 +9,7 @@ Enum HOWTO .. currentmodule:: enum An :class:`Enum` is a set of symbolic names bound to unique values. They are -similar to global variables, but they offer a more useful :func:`repr()`, +similar to global variables, but they offer a more useful :func:`repr`, grouping, type-safety, and a few other features. They are most useful when you have a variable that can take one of a limited @@ -62,12 +64,12 @@ The *type* of an enumeration member is the enum it belongs to:: >>> isinstance(Weekday.FRIDAY, Weekday) True -Enum members have an attribute that contains just their :attr:`name`:: +Enum members have an attribute that contains just their :attr:`!name`:: >>> print(Weekday.TUESDAY.name) TUESDAY -Likewise, they have an attribute for their :attr:`value`:: +Likewise, they have an attribute for their :attr:`!value`:: >>> Weekday.WEDNESDAY.value @@ -75,17 +77,18 @@ Likewise, they have an attribute for their :attr:`value`:: Unlike many languages that treat enumerations solely as name/value pairs, Python Enums can have behavior added. For example, :class:`datetime.date` -has two methods for returning the weekday: :meth:`weekday` and :meth:`isoweekday`. +has two methods for returning the weekday: +:meth:`~datetime.date.weekday` and :meth:`~datetime.date.isoweekday`. The difference is that one of them counts from 0-6 and the other from 1-7. -Rather than keep track of that ourselves we can add a method to the :class:`Weekday` -enum to extract the day from the :class:`date` instance and return the matching +Rather than keep track of that ourselves we can add a method to the :class:`!Weekday` +enum to extract the day from the :class:`~datetime.date` instance and return the matching enum member:: @classmethod def from_date(cls, date): return cls(date.isoweekday()) -The complete :class:`Weekday` enum now looks like this:: +The complete :class:`!Weekday` enum now looks like this:: >>> class Weekday(Enum): ... MONDAY = 1 @@ -108,7 +111,7 @@ Now we can find out what today is! Observe:: Of course, if you're reading this on some other day, you'll see that day instead. -This :class:`Weekday` enum is great if our variable only needs one day, but +This :class:`!Weekday` enum is great if our variable only needs one day, but what if we need several? Maybe we're writing a function to plot chores during a week, and don't want to use a :class:`list` -- we could use a different type of :class:`Enum`:: @@ -126,7 +129,7 @@ of :class:`Enum`:: We've changed two things: we're inherited from :class:`Flag`, and the values are all powers of 2. -Just like the original :class:`Weekday` enum above, we can have a single selection:: +Just like the original :class:`!Weekday` enum above, we can have a single selection:: >>> first_week_day = Weekday.MONDAY >>> first_week_day @@ -165,7 +168,7 @@ And a function to display the chores for a given day:: answer SO questions In cases where the actual values of the members do not matter, you can save -yourself some work and use :func:`auto()` for the values:: +yourself some work and use :func:`auto` for the values:: >>> from enum import auto >>> class Weekday(Flag): @@ -201,7 +204,7 @@ If you want to access enum members by *name*, use item access:: >>> Color['GREEN'] -If you have an enum member and need its :attr:`name` or :attr:`value`:: +If you have an enum member and need its :attr:`!name` or :attr:`!value`:: >>> member = Color.RED >>> member.name @@ -282,7 +285,7 @@ If the exact value is unimportant you can use :class:`auto`:: >>> [member.value for member in Color] [1, 2, 3] -The values are chosen by :func:`_generate_next_value_`, which can be +The values are chosen by :func:`~Enum._generate_next_value_`, which can be overridden:: >>> class AutoName(Enum): @@ -301,7 +304,7 @@ overridden:: .. note:: - The :meth:`_generate_next_value_` method must be defined before any members. + The :meth:`~Enum._generate_next_value_` method must be defined before any members. Iteration --------- @@ -422,18 +425,18 @@ Then:: The rules for what is allowed are as follows: names that start and end with a single underscore are reserved by enum and cannot be used; all other attributes defined within an enumeration will become members of this -enumeration, with the exception of special methods (:meth:`__str__`, -:meth:`__add__`, etc.), descriptors (methods are also descriptors), and -variable names listed in :attr:`_ignore_`. +enumeration, with the exception of special methods (:meth:`~object.__str__`, +:meth:`~object.__add__`, etc.), descriptors (methods are also descriptors), and +variable names listed in :attr:`~Enum._ignore_`. -Note: if your enumeration defines :meth:`__new__` and/or :meth:`__init__`, +Note: if your enumeration defines :meth:`~object.__new__` and/or :meth:`~object.__init__`, any value(s) given to the enum member will be passed into those methods. See `Planet`_ for an example. .. note:: - The :meth:`__new__` method, if defined, is used during creation of the Enum - members; it is then replaced by Enum's :meth:`__new__` which is used after + The :meth:`~object.__new__` method, if defined, is used during creation of the Enum + members; it is then replaced by Enum's :meth:`~object.__new__` which is used after class creation for lookup of existing members. See :ref:`new-vs-init` for more details. @@ -542,7 +545,7 @@ from that module. nested in other classes. It is possible to modify how enum members are pickled/unpickled by defining -:meth:`__reduce_ex__` in the enumeration class. The default method is by-value, +:meth:`~object.__reduce_ex__` in the enumeration class. The default method is by-value, but enums with complicated values may want to use by-name:: >>> import enum @@ -578,7 +581,7 @@ values. The last two options enable assigning arbitrary values to enumerations; the others auto-assign increasing integers starting with 1 (use the ``start`` parameter to specify a different starting value). A new class derived from :class:`Enum` is returned. In other words, the above -assignment to :class:`Animal` is equivalent to:: +assignment to :class:`!Animal` is equivalent to:: >>> class Animal(Enum): ... ANT = 1 @@ -606,7 +609,7 @@ The solution is to specify the module name explicitly as follows:: the source, pickling will be disabled. The new pickle protocol 4 also, in some circumstances, relies on -:attr:`~definition.__qualname__` being set to the location where pickle will be able +:attr:`~type.__qualname__` being set to the location where pickle will be able to find the class. For example, if the class was made available in class SomeData in the global scope:: @@ -889,7 +892,7 @@ simple to implement independently:: pass This demonstrates how similar derived enumerations can be defined; for example -a :class:`FloatEnum` that mixes in :class:`float` instead of :class:`int`. +a :class:`!FloatEnum` that mixes in :class:`float` instead of :class:`int`. Some rules: @@ -903,32 +906,32 @@ Some rules: additional type, all the members must have values of that type, e.g. :class:`int` above. This restriction does not apply to mix-ins which only add methods and don't specify another type. -4. When another data type is mixed in, the :attr:`value` attribute is *not the +4. When another data type is mixed in, the :attr:`~Enum.value` attribute is *not the same* as the enum member itself, although it is equivalent and will compare equal. -5. A ``data type`` is a mixin that defines :meth:`__new__`, or a +5. A ``data type`` is a mixin that defines :meth:`~object.__new__`, or a :class:`~dataclasses.dataclass` 6. %-style formatting: ``%s`` and ``%r`` call the :class:`Enum` class's - :meth:`__str__` and :meth:`__repr__` respectively; other codes (such as + :meth:`~object.__str__` and :meth:`~object.__repr__` respectively; other codes (such as ``%i`` or ``%h`` for IntEnum) treat the enum member as its mixed-in type. 7. :ref:`Formatted string literals `, :meth:`str.format`, - and :func:`format` will use the enum's :meth:`__str__` method. + and :func:`format` will use the enum's :meth:`~object.__str__` method. .. note:: Because :class:`IntEnum`, :class:`IntFlag`, and :class:`StrEnum` are designed to be drop-in replacements for existing constants, their - :meth:`__str__` method has been reset to their data types' - :meth:`__str__` method. + :meth:`~object.__str__` method has been reset to their data types' + :meth:`~object.__str__` method. .. _new-vs-init: -When to use :meth:`__new__` vs. :meth:`__init__` ------------------------------------------------- +When to use :meth:`~object.__new__` vs. :meth:`~object.__init__` +---------------------------------------------------------------- -:meth:`__new__` must be used whenever you want to customize the actual value of +:meth:`~object.__new__` must be used whenever you want to customize the actual value of the :class:`Enum` member. Any other modifications may go in either -:meth:`__new__` or :meth:`__init__`, with :meth:`__init__` being preferred. +:meth:`~object.__new__` or :meth:`~object.__init__`, with :meth:`~object.__init__` being preferred. For example, if you want to pass several items to the constructor, but only want one of them to be the value:: @@ -967,11 +970,11 @@ Finer Points Supported ``__dunder__`` names """""""""""""""""""""""""""""" -:attr:`__members__` is a read-only ordered mapping of ``member_name``:``member`` +:attr:`~enum.EnumType.__members__` is a read-only ordered mapping of ``member_name``:``member`` items. It is only available on the class. -:meth:`__new__`, if specified, must create and return the enum members; it is -also a very good idea to set the member's :attr:`_value_` appropriately. Once +:meth:`~object.__new__`, if specified, must create and return the enum members; it is +also a very good idea to set the member's :attr:`~Enum._value_` appropriately. Once all the members are created it is no longer used. @@ -987,9 +990,9 @@ Supported ``_sunder_`` names from the final class - :meth:`~Enum._generate_next_value_` -- used to get an appropriate value for an enum member; may be overridden -- :meth:`~Enum._add_alias_` -- adds a new name as an alias to an existing +- :meth:`~EnumType._add_alias_` -- adds a new name as an alias to an existing member. -- :meth:`~Enum._add_value_alias_` -- adds a new value as an alias to an +- :meth:`~EnumType._add_value_alias_` -- adds a new value as an alias to an existing member. See `MultiValueEnum`_ for an example. .. note:: @@ -1007,7 +1010,7 @@ Supported ``_sunder_`` names .. versionadded:: 3.7 ``_ignore_`` .. versionadded:: 3.13 ``_add_alias_``, ``_add_value_alias_`` -To help keep Python 2 / Python 3 code in sync an :attr:`_order_` attribute can +To help keep Python 2 / Python 3 code in sync an :attr:`~Enum._order_` attribute can be provided. It will be checked against the actual order of the enumeration and raise an error if the two do not match:: @@ -1025,7 +1028,7 @@ and raise an error if the two do not match:: .. note:: - In Python 2 code the :attr:`_order_` attribute is necessary as definition + In Python 2 code the :attr:`~Enum._order_` attribute is necessary as definition order is lost before it can be recorded. @@ -1150,6 +1153,14 @@ the following are true: >>> (Color.RED | Color.GREEN).name 'RED|GREEN' + >>> class Perm(IntFlag): + ... R = 4 + ... W = 2 + ... X = 1 + ... + >>> (Perm.R & Perm.W).name is None # effectively Perm(0) + True + - multi-bit flags, aka aliases, can be returned from operations:: >>> Color.RED | Color.BLUE @@ -1206,12 +1217,12 @@ Enum Classes ^^^^^^^^^^^^ The :class:`EnumType` metaclass is responsible for providing the -:meth:`__contains__`, :meth:`__dir__`, :meth:`__iter__` and other methods that +:meth:`~object.__contains__`, :meth:`~object.__dir__`, :meth:`~object.__iter__` and other methods that allow one to do things with an :class:`Enum` class that fail on a typical class, such as ``list(Color)`` or ``some_enum_var in Color``. :class:`EnumType` is responsible for ensuring that various other methods on the final :class:`Enum` -class are correct (such as :meth:`__new__`, :meth:`__getnewargs__`, -:meth:`__str__` and :meth:`__repr__`). +class are correct (such as :meth:`~object.__new__`, :meth:`~object.__getnewargs__`, +:meth:`~object.__str__` and :meth:`~object.__repr__`). Flag Classes ^^^^^^^^^^^^ @@ -1226,7 +1237,7 @@ Enum Members (aka instances) The most interesting thing about enum members is that they are singletons. :class:`EnumType` creates them all while it is creating the enum class itself, -and then puts a custom :meth:`__new__` in place to ensure that no new ones are +and then puts a custom :meth:`~object.__new__` in place to ensure that no new ones are ever instantiated by returning only the existing member instances. Flag Members @@ -1274,7 +1285,7 @@ is. There are several ways to define this type of simple enumeration: - use instances of :class:`auto` for the value - use instances of :class:`object` as the value - use a descriptive string as the value -- use a tuple as the value and a custom :meth:`__new__` to replace the +- use a tuple as the value and a custom :meth:`~object.__new__` to replace the tuple with an :class:`int` value Using any of these methods signifies to the user that these values are not @@ -1310,7 +1321,7 @@ Using :class:`object` would look like:: > This is also a good example of why you might want to write your own -:meth:`__repr__`:: +:meth:`~object.__repr__`:: >>> class Color(Enum): ... RED = object() @@ -1338,10 +1349,10 @@ Using a string as the value would look like:: -Using a custom :meth:`__new__` -"""""""""""""""""""""""""""""" +Using a custom :meth:`~object.__new__` +"""""""""""""""""""""""""""""""""""""" -Using an auto-numbering :meth:`__new__` would look like:: +Using an auto-numbering :meth:`~object.__new__` would look like:: >>> class AutoNumber(Enum): ... def __new__(cls): @@ -1387,8 +1398,8 @@ to handle any extra arguments:: .. note:: - The :meth:`__new__` method, if defined, is used during creation of the Enum - members; it is then replaced by Enum's :meth:`__new__` which is used after + The :meth:`~object.__new__` method, if defined, is used during creation of the Enum + members; it is then replaced by Enum's :meth:`~object.__new__` which is used after class creation for lookup of existing members. .. warning:: @@ -1494,7 +1505,7 @@ Supports having more than one value per member:: Planet ^^^^^^ -If :meth:`__new__` or :meth:`__init__` is defined, the value of the enum member +If :meth:`~object.__new__` or :meth:`~object.__init__` is defined, the value of the enum member will be passed to those methods:: >>> class Planet(Enum): @@ -1525,7 +1536,7 @@ will be passed to those methods:: TimePeriod ^^^^^^^^^^ -An example to show the :attr:`_ignore_` attribute in use:: +An example to show the :attr:`~Enum._ignore_` attribute in use:: >>> from datetime import timedelta >>> class Period(timedelta, Enum): diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst new file mode 100644 index 00000000000000..95f214179bfb0e --- /dev/null +++ b/Doc/howto/free-threading-extensions.rst @@ -0,0 +1,284 @@ +.. highlight:: c + +.. _freethreading-extensions-howto: + +****************************************** +C API Extension Support for Free Threading +****************************************** + +Starting with the 3.13 release, CPython has experimental support for running +with the :term:`global interpreter lock` (GIL) disabled in a configuration +called :term:`free threading`. This document describes how to adapt C API +extensions to support free threading. + + +Identifying the Free-Threaded Build in C +======================================== + +The CPython C API exposes the ``Py_GIL_DISABLED`` macro: in the free-threaded +build it's defined to ``1``, and in the regular build it's not defined. +You can use it to enable code that only runs under the free-threaded build:: + + #ifdef Py_GIL_DISABLED + /* code that only runs in the free-threaded build */ + #endif + +Module Initialization +===================== + +Extension modules need to explicitly indicate that they support running with +the GIL disabled; otherwise importing the extension will raise a warning and +enable the GIL at runtime. + +There are two ways to indicate that an extension module supports running with +the GIL disabled depending on whether the extension uses multi-phase or +single-phase initialization. + +Multi-Phase Initialization +.......................... + +Extensions that use multi-phase initialization (i.e., +:c:func:`PyModuleDef_Init`) should add a :c:data:`Py_mod_gil` slot in the +module definition. If your extension supports older versions of CPython, +you should guard the slot with a :c:data:`PY_VERSION_HEX` check. + +:: + + static struct PyModuleDef_Slot module_slots[] = { + ... + #if PY_VERSION_HEX >= 0x030D0000 + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + #endif + {0, NULL} + }; + + static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + .m_slots = module_slots, + ... + }; + + +Single-Phase Initialization +........................... + +Extensions that use single-phase initialization (i.e., +:c:func:`PyModule_Create`) should call :c:func:`PyUnstable_Module_SetGIL` to +indicate that they support running with the GIL disabled. The function is +only defined in the free-threaded build, so you should guard the call with +``#ifdef Py_GIL_DISABLED`` to avoid compilation errors in the regular build. + +:: + + static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + ... + }; + + PyMODINIT_FUNC + PyInit_mymodule(void) + { + PyObject *m = PyModule_Create(&moduledef); + if (m == NULL) { + return NULL; + } + #ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); + #endif + return m; + } + + +General API Guidelines +====================== + +Most of the C API is thread-safe, but there are some exceptions. + +* **Struct Fields**: Accessing fields in Python C API objects or structs + directly is not thread-safe if the field may be concurrently modified. +* **Macros**: Accessor macros like :c:macro:`PyList_GET_ITEM`, + :c:macro:`PyList_SET_ITEM`, and macros like + :c:macro:`PySequence_Fast_GET_SIZE` that use the object returned by + :c:func:`PySequence_Fast` do not perform any error checking or locking. + These macros are not thread-safe if the container object may be modified + concurrently. +* **Borrowed References**: C API functions that return + :term:`borrowed references ` may not be thread-safe if + the containing object is modified concurrently. See the section on + :ref:`borrowed references ` for more information. + + +Container Thread Safety +....................... + +Containers like :c:struct:`PyListObject`, +:c:struct:`PyDictObject`, and :c:struct:`PySetObject` perform internal locking +in the free-threaded build. For example, the :c:func:`PyList_Append` will +lock the list before appending an item. + +.. _PyDict_Next: + +``PyDict_Next`` +''''''''''''''' + +A notable exception is :c:func:`PyDict_Next`, which does not lock the +dictionary. You should use :c:macro:`Py_BEGIN_CRITICAL_SECTION` to protect +the dictionary while iterating over it if the dictionary may be concurrently +modified:: + + Py_BEGIN_CRITICAL_SECTION(dict); + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(dict, &pos, &key, &value)) { + ... + } + Py_END_CRITICAL_SECTION(); + + +Borrowed References +=================== + +.. _borrowed-references: + +Some C API functions return :term:`borrowed references `. +These APIs are not thread-safe if the containing object is modified +concurrently. For example, it's not safe to use :c:func:`PyList_GetItem` +if the list may be modified concurrently. + +The following table lists some borrowed reference APIs and their replacements +that return :term:`strong references `. + ++-----------------------------------+-----------------------------------+ +| Borrowed reference API | Strong reference API | ++===================================+===================================+ +| :c:func:`PyList_GetItem` | :c:func:`PyList_GetItemRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyDict_GetItem` | :c:func:`PyDict_GetItemRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyDict_GetItemWithError` | :c:func:`PyDict_GetItemRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyDict_GetItemString` | :c:func:`PyDict_GetItemStringRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyDict_SetDefault` | :c:func:`PyDict_SetDefaultRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyDict_Next` | none (see :ref:`PyDict_Next`) | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyWeakref_GetObject` | :c:func:`PyWeakref_GetRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyWeakref_GET_OBJECT` | :c:func:`PyWeakref_GetRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyImport_AddModule` | :c:func:`PyImport_AddModuleRef` | ++-----------------------------------+-----------------------------------+ +| :c:func:`PyCell_GET` | :c:func:`PyCell_Get` | ++-----------------------------------+-----------------------------------+ + +Not all APIs that return borrowed references are problematic. For +example, :c:func:`PyTuple_GetItem` is safe because tuples are immutable. +Similarly, not all uses of the above APIs are problematic. For example, +:c:func:`PyDict_GetItem` is often used for parsing keyword argument +dictionaries in function calls; those keyword argument dictionaries are +effectively private (not accessible by other threads), so using borrowed +references in that context is safe. + +Some of these functions were added in Python 3.13. You can use the +`pythoncapi-compat `_ package +to provide implementations of these functions for older Python versions. + + +.. _free-threaded-memory-allocation: + +Memory Allocation APIs +====================== + +Python's memory management C API provides functions in three different +:ref:`allocation domains `: "raw", "mem", and "object". +For thread-safety, the free-threaded build requires that only Python objects +are allocated using the object domain, and that all Python object are +allocated using that domain. This differs from the prior Python versions, +where this was only a best practice and not a hard requirement. + +.. note:: + + Search for uses of :c:func:`PyObject_Malloc` in your + extension and check that the allocated memory is used for Python objects. + Use :c:func:`PyMem_Malloc` to allocate buffers instead of + :c:func:`PyObject_Malloc`. + + +Thread State and GIL APIs +========================= + +Python provides a set of functions and macros to manage thread state and the +GIL, such as: + +* :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` +* :c:func:`PyEval_SaveThread` and :c:func:`PyEval_RestoreThread` +* :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` + +These functions should still be used in the free-threaded build to manage +thread state even when the :term:`GIL` is disabled. For example, if you +create a thread outside of Python, you must call :c:func:`PyGILState_Ensure` +before calling into the Python API to ensure that the thread has a valid +Python thread state. + +You should continue to call :c:func:`PyEval_SaveThread` or +:c:macro:`Py_BEGIN_ALLOW_THREADS` around blocking operations, such as I/O or +lock acquisitions, to allow other threads to run the +:term:`cyclic garbage collector `. + + +Protecting Internal Extension State +=================================== + +Your extension may have internal state that was previously protected by the +GIL. You may need to add locking to protect this state. The approach will +depend on your extension, but some common patterns include: + +* **Caches**: global caches are a common source of shared state. Consider + using a lock to protect the cache or disabling it in the free-threaded build + if the cache is not critical for performance. +* **Global State**: global state may need to be protected by a lock or moved + to thread local storage. C11 and C++11 provide the ``thread_local`` or + ``_Thread_local`` for + `thread-local storage `_. + + +Building Extensions for the Free-Threaded Build +=============================================== + +C API extensions need to be built specifically for the free-threaded build. +The wheels, shared libraries, and binaries are indicated by a ``t`` suffix. + +* `pypa/manylinux `_ supports the + free-threaded build, with the ``t`` suffix, such as ``python3.13t``. +* `pypa/cibuildwheel `_ supports the + free-threaded build if you set + `CIBW_FREE_THREADED_SUPPORT `_. + +Limited C API and Stable ABI +............................ + +The free-threaded build does not currently support the +:ref:`Limited C API ` or the stable ABI. If you use +`setuptools `_ to build +your extension and currently set ``py_limited_api=True`` you can use +``py_limited_api=not sysconfig.get_config_var("Py_GIL_DISABLED")`` to opt out +of the limited API when building with the free-threaded build. + +.. note:: + You will need to build separate wheels specifically for the free-threaded + build. If you currently use the stable ABI, you can continue to build a + single wheel for multiple non-free-threaded Python versions. + + +Windows +....... + +Due to a limitation of the official Windows installer, you will need to +manually define ``Py_GIL_DISABLED=1`` when building extensions from source. + +.. seealso:: + + `Porting Extension Modules to Support Free-Threading + `_: + A community-maintained porting guide for extension authors. diff --git a/Doc/howto/free-threading-python.rst b/Doc/howto/free-threading-python.rst new file mode 100644 index 00000000000000..b21e3287ecaa3f --- /dev/null +++ b/Doc/howto/free-threading-python.rst @@ -0,0 +1,154 @@ +.. _freethreading-python-howto: + +********************************************** +Python experimental support for free threading +********************************************** + +Starting with the 3.13 release, CPython has experimental support for a build of +Python called :term:`free threading` where the :term:`global interpreter lock` +(GIL) is disabled. Free-threaded execution allows for full utilization of the +available processing power by running threads in parallel on available CPU cores. +While not all software will benefit from this automatically, programs +designed with threading in mind will run faster on multi-core hardware. + +**The free-threaded mode is experimental** and work is ongoing to improve it: +expect some bugs and a substantial single-threaded performance hit. + +This document describes the implications of free threading +for Python code. See :ref:`freethreading-extensions-howto` for information on +how to write C extensions that support the free-threaded build. + +.. seealso:: + + :pep:`703` – Making the Global Interpreter Lock Optional in CPython for an + overall description of free-threaded Python. + + +Installation +============ + +Starting with Python 3.13, the official macOS and Windows installers +optionally support installing free-threaded Python binaries. The installers +are available at https://www.python.org/downloads/. + +For information on other platforms, see the `Installing a Free-Threaded Python +`_, a +community-maintained installation guide for installing free-threaded Python. + +When building CPython from source, the :option:`--disable-gil` configure option +should be used to build a free-threaded Python interpreter. + + +Identifying free-threaded Python +================================ + +To check if the current interpreter supports free-threading, :option:`python -VV <-V>` +and :attr:`sys.version` contain "experimental free-threading build". +The new :func:`sys._is_gil_enabled` function can be used to check whether +the GIL is actually disabled in the running process. + +The ``sysconfig.get_config_var("Py_GIL_DISABLED")`` configuration variable can +be used to determine whether the build supports free threading. If the variable +is set to ``1``, then the build supports free threading. This is the recommended +mechanism for decisions related to the build configuration. + + +The global interpreter lock in free-threaded Python +=================================================== + +Free-threaded builds of CPython support optionally running with the GIL enabled +at runtime using the environment variable :envvar:`PYTHON_GIL` or +the command-line option :option:`-X gil`. + +The GIL may also automatically be enabled when importing a C-API extension +module that is not explicitly marked as supporting free threading. A warning +will be printed in this case. + +In addition to individual package documentation, the following websites track +the status of popular packages support for free threading: + +* https://py-free-threading.github.io/tracking/ +* https://hugovk.github.io/free-threaded-wheels/ + + +Thread safety +============= + +The free-threaded build of CPython aims to provide similar thread-safety +behavior at the Python level to the default GIL-enabled build. Built-in +types like :class:`dict`, :class:`list`, and :class:`set` use internal locks +to protect against concurrent modifications in ways that behave similarly to +the GIL. However, Python has not historically guaranteed specific behavior for +concurrent modifications to these built-in types, so this should be treated +as a description of the current implementation, not a guarantee of current or +future behavior. + +.. note:: + + It's recommended to use the :class:`threading.Lock` or other synchronization + primitives instead of relying on the internal locks of built-in types, when + possible. + + +Known limitations +================= + +This section describes known limitations of the free-threaded CPython build. + +Immortalization +--------------- + +The free-threaded build of the 3.13 release makes some objects :term:`immortal`. +Immortal objects are not deallocated and have reference counts that are +never modified. This is done to avoid reference count contention that would +prevent efficient multi-threaded scaling. + +An object will be made immortal when a new thread is started for the first time +after the main thread is running. The following objects are immortalized: + +* :ref:`function ` objects declared at the module level +* :ref:`method ` descriptors +* :ref:`code ` objects +* :term:`module` objects and their dictionaries +* :ref:`classes ` (type objects) + +Because immortal objects are never deallocated, applications that create many +objects of these types may see increased memory usage. This is expected to be +addressed in the 3.14 release. + +Additionally, numeric and string literals in the code as well as strings +returned by :func:`sys.intern` are also immortalized. This behavior is +expected to remain in the 3.14 free-threaded build. + + +Frame objects +------------- + +It is not safe to access :ref:`frame ` objects from other +threads and doing so may cause your program to crash . This means that +:func:`sys._current_frames` is generally not safe to use in a free-threaded +build. Functions like :func:`inspect.currentframe` and :func:`sys._getframe` +are generally safe as long as the resulting frame object is not passed to +another thread. + +Iterators +--------- + +Sharing the same iterator object between multiple threads is generally not +safe and threads may see duplicate or missing elements when iterating or crash +the interpreter. + + +Single-threaded performance +--------------------------- + +The free-threaded build has additional overhead when executing Python code +compared to the default GIL-enabled build. In 3.13, this overhead is about +40% on the `pyperformance `_ suite. +Programs that spend most of their time in C extensions or I/O will see +less of an impact. The largest impact is because the specializing adaptive +interpreter (:pep:`659`) is disabled in the free-threaded build. We expect +to re-enable it in a thread-safe way in the 3.14 release. This overhead is +expected to be reduced in upcoming Python release. We are aiming for an +overhead of 10% or less on the pyperformance suite compared to the default +GIL-enabled build. diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index b0f9d22d74f0e3..1f0608fb0fc53f 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -1,3 +1,5 @@ +.. _functional-howto: + ******************************** Functional Programming HOWTO ******************************** diff --git a/Doc/howto/gdb_helpers.rst b/Doc/howto/gdb_helpers.rst index 53bbf7ddaa2ab9..98ce813ca4ab02 100644 --- a/Doc/howto/gdb_helpers.rst +++ b/Doc/howto/gdb_helpers.rst @@ -180,7 +180,7 @@ regular machine-level integer:: (gdb) p some_python_integer $4 = 42 -The internal structure can be revealed with a cast to :c:expr:`PyLongObject *`: +The internal structure can be revealed with a cast to :c:expr:`PyLongObject *`:: (gdb) p *(PyLongObject*)some_python_integer $5 = {ob_base = {ob_base = {ob_refcnt = 8, ob_type = 0x3dad39f5e0}, ob_size = 1}, diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index 8b334555ab6463..c09f92c9528ee1 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -2,16 +2,14 @@ Python HOWTOs *************** -Python HOWTOs are documents that cover a single, specific topic, -and attempt to cover it fairly completely. Modelled on the Linux -Documentation Project's HOWTO collection, this collection is an +Python HOWTOs are documents that cover a specific topic in-depth. +Modeled on the Linux Documentation Project's HOWTO collection, this collection is an effort to foster documentation that's more detailed than the Python Library Reference. -Currently, the HOWTOs are: - .. toctree:: :maxdepth: 1 + :hidden: cporting.rst curses.rst @@ -33,4 +31,38 @@ Currently, the HOWTOs are: annotations.rst isolating-extensions.rst timerfd.rst + mro.rst + free-threading-python.rst + free-threading-extensions.rst + +General: + +* :ref:`annotations-howto` +* :ref:`argparse-tutorial` +* :ref:`descriptorhowto` +* :ref:`enum-howto` +* :ref:`functional-howto` +* :ref:`ipaddress-howto` +* :ref:`logging-howto` +* :ref:`logging-cookbook` +* :ref:`regex-howto` +* :ref:`sortinghowto` +* :ref:`unicode-howto` +* :ref:`urllib-howto` + +Advanced development: + +* :ref:`curses-howto` +* :ref:`freethreading-python-howto` +* :ref:`freethreading-extensions-howto` +* :ref:`isolating-extensions-howto` +* :ref:`python_2.3_mro` +* :ref:`socket-howto` +* :ref:`timerfd-howto` +* :ref:`cporting-howto` + +Debugging and profiling: +* :ref:`gdb` +* :ref:`instrumentation` +* :ref:`perf_profiling` diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst index 9c99fcecce1fcb..6e03ef20a21fa3 100644 --- a/Doc/howto/instrumentation.rst +++ b/Doc/howto/instrumentation.rst @@ -307,7 +307,7 @@ Available static markers .. object:: gc__start(int generation) Fires when the Python interpreter starts a garbage collection cycle. - ``arg0`` is the generation to scan, like :func:`gc.collect()`. + ``arg0`` is the generation to scan, like :func:`gc.collect`. .. object:: gc__done(long collected) diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index e35855deedbe5f..a636e06bda8344 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -339,7 +339,7 @@ That is, heap types should: - Define a traverse function using ``Py_tp_traverse``, which visits the type (e.g. using ``Py_VISIT(Py_TYPE(self))``). -Please refer to the the documentation of +Please refer to the documentation of :c:macro:`Py_TPFLAGS_HAVE_GC` and :c:member:`~PyTypeObject.tp_traverse` for additional considerations. diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index f7d885ec88483d..3cd2f1d96a7b34 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -1267,11 +1267,8 @@ to adapt in your own applications. You could also write your own handler which uses the :class:`~multiprocessing.Lock` class from the :mod:`multiprocessing` module to serialize access to the -file from your processes. The existing :class:`FileHandler` and subclasses do -not make use of :mod:`multiprocessing` at present, though they may do so in the -future. Note that at present, the :mod:`multiprocessing` module does not provide -working lock functionality on all platforms (see -https://bugs.python.org/issue3770). +file from your processes. The stdlib :class:`FileHandler` and subclasses do +not make use of :mod:`multiprocessing`. .. currentmodule:: logging.handlers @@ -1846,8 +1843,11 @@ the use of a :class:`Filter` does not provide the desired result. .. _zeromq-handlers: -Subclassing QueueHandler - a ZeroMQ example -------------------------------------------- +Subclassing QueueHandler and QueueListener- a ZeroMQ example +------------------------------------------------------------ + +Subclass ``QueueHandler`` +^^^^^^^^^^^^^^^^^^^^^^^^^ You can use a :class:`QueueHandler` subclass to send messages to other kinds of queues, for example a ZeroMQ 'publish' socket. In the example below,the @@ -1885,8 +1885,8 @@ data needed by the handler to create the socket:: self.queue.close() -Subclassing QueueListener - a ZeroMQ example --------------------------------------------- +Subclass ``QueueListener`` +^^^^^^^^^^^^^^^^^^^^^^^^^^ You can also subclass :class:`QueueListener` to get messages from other kinds of queues, for example a ZeroMQ 'subscribe' socket. Here's an example:: @@ -1903,25 +1903,194 @@ of queues, for example a ZeroMQ 'subscribe' socket. Here's an example:: msg = self.queue.recv_json() return logging.makeLogRecord(msg) +.. _pynng-handlers: -.. seealso:: +Subclassing QueueHandler and QueueListener- a ``pynng`` example +--------------------------------------------------------------- - Module :mod:`logging` - API reference for the logging module. +In a similar way to the above section, we can implement a listener and handler +using :pypi:`pynng`, which is a Python binding to +`NNG `_, billed as a spiritual successor to ZeroMQ. +The following snippets illustrate -- you can test them in an environment which has +``pynng`` installed. Just for variety, we present the listener first. - Module :mod:`logging.config` - Configuration API for the logging module. - Module :mod:`logging.handlers` - Useful handlers included with the logging module. +Subclass ``QueueListener`` +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + + # listener.py + import json + import logging + import logging.handlers + + import pynng + + DEFAULT_ADDR = "tcp://localhost:13232" + + interrupted = False + + class NNGSocketListener(logging.handlers.QueueListener): + + def __init__(self, uri, /, *handlers, **kwargs): + # Have a timeout for interruptability, and open a + # subscriber socket + socket = pynng.Sub0(listen=uri, recv_timeout=500) + # The b'' subscription matches all topics + topics = kwargs.pop('topics', None) or b'' + socket.subscribe(topics) + # We treat the socket as a queue + super().__init__(socket, *handlers, **kwargs) + + def dequeue(self, block): + data = None + # Keep looping while not interrupted and no data received over the + # socket + while not interrupted: + try: + data = self.queue.recv(block=block) + break + except pynng.Timeout: + pass + except pynng.Closed: # sometimes happens when you hit Ctrl-C + break + if data is None: + return None + # Get the logging event sent from a publisher + event = json.loads(data.decode('utf-8')) + return logging.makeLogRecord(event) + + def enqueue_sentinel(self): + # Not used in this implementation, as the socket isn't really a + # queue + pass - :ref:`A basic logging tutorial ` + logging.getLogger('pynng').propagate = False + listener = NNGSocketListener(DEFAULT_ADDR, logging.StreamHandler(), topics=b'') + listener.start() + print('Press Ctrl-C to stop.') + try: + while True: + pass + except KeyboardInterrupt: + interrupted = True + finally: + listener.stop() - :ref:`A more advanced logging tutorial ` +Subclass ``QueueHandler`` +^^^^^^^^^^^^^^^^^^^^^^^^^ .. currentmodule:: logging +.. code-block:: python + + # sender.py + import json + import logging + import logging.handlers + import time + import random + + import pynng + + DEFAULT_ADDR = "tcp://localhost:13232" + + class NNGSocketHandler(logging.handlers.QueueHandler): + + def __init__(self, uri): + socket = pynng.Pub0(dial=uri, send_timeout=500) + super().__init__(socket) + + def enqueue(self, record): + # Send the record as UTF-8 encoded JSON + d = dict(record.__dict__) + data = json.dumps(d) + self.queue.send(data.encode('utf-8')) + + def close(self): + self.queue.close() + + logging.getLogger('pynng').propagate = False + handler = NNGSocketHandler(DEFAULT_ADDR) + # Make sure the process ID is in the output + logging.basicConfig(level=logging.DEBUG, + handlers=[logging.StreamHandler(), handler], + format='%(levelname)-8s %(name)10s %(process)6s %(message)s') + levels = (logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, + logging.CRITICAL) + logger_names = ('myapp', 'myapp.lib1', 'myapp.lib2') + msgno = 1 + while True: + # Just randomly select some loggers and levels and log away + level = random.choice(levels) + logger = logging.getLogger(random.choice(logger_names)) + logger.log(level, 'Message no. %5d' % msgno) + msgno += 1 + delay = random.random() * 2 + 0.5 + time.sleep(delay) + +You can run the above two snippets in separate command shells. If we run the +listener in one shell and run the sender in two separate shells, we should see +something like the following. In the first sender shell: + +.. code-block:: console + + $ python sender.py + DEBUG myapp 613 Message no. 1 + WARNING myapp.lib2 613 Message no. 2 + CRITICAL myapp.lib2 613 Message no. 3 + WARNING myapp.lib2 613 Message no. 4 + CRITICAL myapp.lib1 613 Message no. 5 + DEBUG myapp 613 Message no. 6 + CRITICAL myapp.lib1 613 Message no. 7 + INFO myapp.lib1 613 Message no. 8 + (and so on) + +In the second sender shell: + +.. code-block:: console + + $ python sender.py + INFO myapp.lib2 657 Message no. 1 + CRITICAL myapp.lib2 657 Message no. 2 + CRITICAL myapp 657 Message no. 3 + CRITICAL myapp.lib1 657 Message no. 4 + INFO myapp.lib1 657 Message no. 5 + WARNING myapp.lib2 657 Message no. 6 + CRITICAL myapp 657 Message no. 7 + DEBUG myapp.lib1 657 Message no. 8 + (and so on) + +In the listener shell: + +.. code-block:: console + + $ python listener.py + Press Ctrl-C to stop. + DEBUG myapp 613 Message no. 1 + WARNING myapp.lib2 613 Message no. 2 + INFO myapp.lib2 657 Message no. 1 + CRITICAL myapp.lib2 613 Message no. 3 + CRITICAL myapp.lib2 657 Message no. 2 + CRITICAL myapp 657 Message no. 3 + WARNING myapp.lib2 613 Message no. 4 + CRITICAL myapp.lib1 613 Message no. 5 + CRITICAL myapp.lib1 657 Message no. 4 + INFO myapp.lib1 657 Message no. 5 + DEBUG myapp 613 Message no. 6 + WARNING myapp.lib2 657 Message no. 6 + CRITICAL myapp 657 Message no. 7 + CRITICAL myapp.lib1 613 Message no. 7 + INFO myapp.lib1 613 Message no. 8 + DEBUG myapp.lib1 657 Message no. 8 + (and so on) + +As you can see, the logging from the two sender processes is interleaved in the +listener's output. + + An example dictionary-based configuration ----------------------------------------- @@ -2778,7 +2947,7 @@ When run, this produces a file with exactly two lines: .. code-block:: none 28/01/2015 07:21:23|INFO|Sample message| - 28/01/2015 07:21:23|ERROR|ZeroDivisionError: integer division or modulo by zero|'Traceback (most recent call last):\n File "logtest7.py", line 30, in main\n x = 1 / 0\nZeroDivisionError: integer division or modulo by zero'| + 28/01/2015 07:21:23|ERROR|ZeroDivisionError: division by zero|'Traceback (most recent call last):\n File "logtest7.py", line 30, in main\n x = 1 / 0\nZeroDivisionError: division by zero'| While the above treatment is simplistic, it points the way to how exception information can be formatted to your liking. The :mod:`traceback` module may be @@ -3403,9 +3572,8 @@ A Qt GUI for logging A question that comes up from time to time is about how to log to a GUI application. The `Qt `_ framework is a popular -cross-platform UI framework with Python bindings using `PySide2 -`_ or `PyQt5 -`_ libraries. +cross-platform UI framework with Python bindings using :pypi:`PySide2` +or :pypi:`PyQt5` libraries. The following example shows how to log to a Qt GUI. This introduces a simple ``QtHandler`` class which takes a callable, which should be a slot in the main @@ -3418,9 +3586,10 @@ The worker thread is implemented using Qt's ``QThread`` class rather than the :mod:`threading` module, as there are circumstances where one has to use ``QThread``, which offers better integration with other ``Qt`` components. -The code should work with recent releases of either ``PySide2`` or ``PyQt5``. -You should be able to adapt the approach to earlier versions of Qt. Please -refer to the comments in the code snippet for more detailed information. +The code should work with recent releases of any of ``PySide6``, ``PyQt6``, +``PySide2`` or ``PyQt5``. You should be able to adapt the approach to earlier +versions of Qt. Please refer to the comments in the code snippet for more +detailed information. .. code-block:: python3 @@ -3430,16 +3599,25 @@ refer to the comments in the code snippet for more detailed information. import sys import time - # Deal with minor differences between PySide2 and PyQt5 + # Deal with minor differences between different Qt packages try: - from PySide2 import QtCore, QtGui, QtWidgets + from PySide6 import QtCore, QtGui, QtWidgets Signal = QtCore.Signal Slot = QtCore.Slot except ImportError: - from PyQt5 import QtCore, QtGui, QtWidgets - Signal = QtCore.pyqtSignal - Slot = QtCore.pyqtSlot - + try: + from PyQt6 import QtCore, QtGui, QtWidgets + Signal = QtCore.pyqtSignal + Slot = QtCore.pyqtSlot + except ImportError: + try: + from PySide2 import QtCore, QtGui, QtWidgets + Signal = QtCore.Signal + Slot = QtCore.Slot + except ImportError: + from PyQt5 import QtCore, QtGui, QtWidgets + Signal = QtCore.pyqtSignal + Slot = QtCore.pyqtSlot logger = logging.getLogger(__name__) @@ -3511,8 +3689,14 @@ refer to the comments in the code snippet for more detailed information. while not QtCore.QThread.currentThread().isInterruptionRequested(): delay = 0.5 + random.random() * 2 time.sleep(delay) - level = random.choice(LEVELS) - logger.log(level, 'Message after delay of %3.1f: %d', delay, i, extra=extra) + try: + if random.random() < 0.1: + raise ValueError('Exception raised: %d' % i) + else: + level = random.choice(LEVELS) + logger.log(level, 'Message after delay of %3.1f: %d', delay, i, extra=extra) + except ValueError as e: + logger.exception('Failed: %s', e, extra=extra) i += 1 # @@ -3539,7 +3723,10 @@ refer to the comments in the code snippet for more detailed information. self.textedit = te = QtWidgets.QPlainTextEdit(self) # Set whatever the default monospace font is for the platform f = QtGui.QFont('nosuchfont') - f.setStyleHint(f.Monospace) + if hasattr(f, 'Monospace'): + f.setStyleHint(f.Monospace) + else: + f.setStyleHint(f.StyleHint.Monospace) # for Qt6 te.setFont(f) te.setReadOnly(True) PB = QtWidgets.QPushButton @@ -3626,7 +3813,11 @@ refer to the comments in the code snippet for more detailed information. app = QtWidgets.QApplication(sys.argv) example = Window(app) example.show() - sys.exit(app.exec_()) + if hasattr(app, 'exec'): + rc = app.exec() + else: + rc = app.exec_() + sys.exit(rc) if __name__=='__main__': main() @@ -3828,7 +4019,7 @@ As you can see, this output isn't ideal. That's because the underlying code which writes to ``sys.stderr`` makes multiple writes, each of which results in a separate logged line (for example, the last three lines above). To get around this problem, you need to buffer things and only output log lines when newlines -are seen. Let's use a slghtly better implementation of ``LoggerWriter``: +are seen. Let's use a slightly better implementation of ``LoggerWriter``: .. code-block:: python diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 347330e98dd00c..2982cf88bf97b4 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -1,3 +1,5 @@ +.. _logging-howto: + ============= Logging HOWTO ============= @@ -25,10 +27,12 @@ or *severity*. When to use logging ^^^^^^^^^^^^^^^^^^^ -Logging provides a set of convenience functions for simple logging usage. These -are :func:`debug`, :func:`info`, :func:`warning`, :func:`error` and -:func:`critical`. To determine when to use logging, see the table below, which -states, for each of a set of common tasks, the best tool to use for it. +You can access logging functionality by creating a logger via ``logger = +getLogger(__name__)``, and then calling the logger's :meth:`~Logger.debug`, +:meth:`~Logger.info`, :meth:`~Logger.warning`, :meth:`~Logger.error` and +:meth:`~Logger.critical` methods. To determine when to use logging, and to see +which logger methods to use when, see the table below. It states, for each of a +set of common tasks, the best tool to use for that task. +-------------------------------------+--------------------------------------+ | Task you want to perform | The best tool for the task | @@ -37,8 +41,8 @@ states, for each of a set of common tasks, the best tool to use for it. | usage of a command line script or | | | program | | +-------------------------------------+--------------------------------------+ -| Report events that occur during | :func:`logging.info` (or | -| normal operation of a program (e.g. | :func:`logging.debug` for very | +| Report events that occur during | A logger's :meth:`~Logger.info` (or | +| normal operation of a program (e.g. | :meth:`~Logger.debug` method for very| | for status monitoring or fault | detailed output for diagnostic | | investigation) | purposes) | +-------------------------------------+--------------------------------------+ @@ -47,22 +51,23 @@ states, for each of a set of common tasks, the best tool to use for it. | | the client application should be | | | modified to eliminate the warning | | | | -| | :func:`logging.warning` if there is | -| | nothing the client application can do| -| | about the situation, but the event | -| | should still be noted | +| | A logger's :meth:`~Logger.warning` | +| | method if there is nothing the client| +| | application can do about the | +| | situation, but the event should still| +| | be noted | +-------------------------------------+--------------------------------------+ | Report an error regarding a | Raise an exception | | particular runtime event | | +-------------------------------------+--------------------------------------+ -| Report suppression of an error | :func:`logging.error`, | -| without raising an exception (e.g. | :func:`logging.exception` or | -| error handler in a long-running | :func:`logging.critical` as | +| Report suppression of an error | A logger's :meth:`~Logger.error`, | +| without raising an exception (e.g. | :meth:`~Logger.exception` or | +| error handler in a long-running | :meth:`~Logger.critical` method as | | server process) | appropriate for the specific error | | | and application domain | +-------------------------------------+--------------------------------------+ -The logging functions are named after the level or severity of the events +The logger methods are named after the level or severity of the events they are used to track. The standard levels and their applicability are described below (in increasing order of severity): @@ -115,12 +120,18 @@ If you type these lines into a script and run it, you'll see: WARNING:root:Watch out! printed out on the console. The ``INFO`` message doesn't appear because the -default level is ``WARNING``. The printed message includes the indication of -the level and the description of the event provided in the logging call, i.e. -'Watch out!'. Don't worry about the 'root' part for now: it will be explained -later. The actual output can be formatted quite flexibly if you need that; -formatting options will also be explained later. - +default level is ``WARNING``. The printed message includes the indication of the +level and the description of the event provided in the logging call, i.e. +'Watch out!'. The actual output can be formatted quite flexibly if you need +that; formatting options will also be explained later. + +Notice that in this example, we use functions directly on the ``logging`` +module, like ``logging.debug``, rather than creating a logger and calling +functions on it. These functions operate on the root logger, but can be useful +as they will call :func:`~logging.basicConfig` for you if it has not been called yet, like in +this example. In larger programs you'll usually want to control the logging +configuration explicitly however - so for that reason as well as others, it's +better to create loggers and call their methods. Logging to a file ^^^^^^^^^^^^^^^^^ @@ -130,11 +141,12 @@ look at that next. Be sure to try the following in a newly started Python interpreter, and don't just continue from the session described above:: import logging + logger = logging.getLogger(__name__) logging.basicConfig(filename='example.log', encoding='utf-8', level=logging.DEBUG) - logging.debug('This message should go to the log file') - logging.info('So should this') - logging.warning('And this, too') - logging.error('And non-ASCII stuff, too, like Øresund and Malmö') + logger.debug('This message should go to the log file') + logger.info('So should this') + logger.warning('And this, too') + logger.error('And non-ASCII stuff, too, like Øresund and Malmö') .. versionchanged:: 3.9 The *encoding* argument was added. In earlier Python versions, or if not @@ -148,10 +160,10 @@ messages: .. code-block:: none - DEBUG:root:This message should go to the log file - INFO:root:So should this - WARNING:root:And this, too - ERROR:root:And non-ASCII stuff, too, like Øresund and Malmö + DEBUG:__main__:This message should go to the log file + INFO:__main__:So should this + WARNING:__main__:And this, too + ERROR:__main__:And non-ASCII stuff, too, like Øresund and Malmö This example also shows how you can set the logging level which acts as the threshold for tracking. In this case, because we set the threshold to @@ -180,11 +192,9 @@ following example:: raise ValueError('Invalid log level: %s' % loglevel) logging.basicConfig(level=numeric_level, ...) -The call to :func:`basicConfig` should come *before* any calls to -:func:`debug`, :func:`info`, etc. Otherwise, those functions will call -:func:`basicConfig` for you with the default options. As it's intended as a -one-off simple configuration facility, only the first call will actually do -anything: subsequent calls are effectively no-ops. +The call to :func:`basicConfig` should come *before* any calls to a logger's +methods such as :meth:`~Logger.debug`, :meth:`~Logger.info`, etc. Otherwise, +that logging event may not be handled in the desired manner. If you run the above script several times, the messages from successive runs are appended to the file *example.log*. If you want each run to start afresh, @@ -197,50 +207,6 @@ The output will be the same as before, but the log file is no longer appended to, so the messages from earlier runs are lost. -Logging from multiple modules -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If your program consists of multiple modules, here's an example of how you -could organize logging in it:: - - # myapp.py - import logging - import mylib - - def main(): - logging.basicConfig(filename='myapp.log', level=logging.INFO) - logging.info('Started') - mylib.do_something() - logging.info('Finished') - - if __name__ == '__main__': - main() - -:: - - # mylib.py - import logging - - def do_something(): - logging.info('Doing something') - -If you run *myapp.py*, you should see this in *myapp.log*: - -.. code-block:: none - - INFO:root:Started - INFO:root:Doing something - INFO:root:Finished - -which is hopefully what you were expecting to see. You can generalize this to -multiple modules, using the pattern in *mylib.py*. Note that for this simple -usage pattern, you won't know, by looking in the log file, *where* in your -application your messages came from, apart from looking at the event -description. If you want to track the location of your messages, you'll need -to refer to the documentation beyond the tutorial level -- see -:ref:`logging-advanced-tutorial`. - - Logging variable data ^^^^^^^^^^^^^^^^^^^^^ @@ -416,8 +382,52 @@ Logging Flow The flow of log event information in loggers and handlers is illustrated in the following diagram. -.. image:: logging_flow.png - :class: invert-in-dark-mode +.. only:: not html + + .. image:: logging_flow.* + +.. raw:: html + :file: logging_flow.svg + +.. raw:: html + + Loggers ^^^^^^^ diff --git a/Doc/howto/logging_flow.png b/Doc/howto/logging_flow.png index d65e597f811db5..d60ed7c031585a 100644 Binary files a/Doc/howto/logging_flow.png and b/Doc/howto/logging_flow.png differ diff --git a/Doc/howto/logging_flow.svg b/Doc/howto/logging_flow.svg new file mode 100644 index 00000000000000..4974994ac6b400 --- /dev/null +++ b/Doc/howto/logging_flow.svg @@ -0,0 +1,327 @@ + + + + + + + + + + + Logger flow + + + + + Create + LogRecord + + + + + + + + + + + + Logging call in user + code, e.g. + + + logger.info(...) + + + + + + + + + Stop + + + + + + Does a filter attached + to logger reject the + record? + + + + + + + + + + Pass record to + handlers of + current logger + + + + + + Is propagate true for + current logger? + + + + + + Is there a parent + logger? + + + + + + Set current + logger to parent + + + + + + At least one handler + in hierarchy? + + + + + + Use + lastResort + handler + + + + + + Handler enabled for + level of record? + + + + + + Does a filter attached + to handler reject the + record? + + + + + + Stop + + + + + + Emit (includes formatting) + + + + Handler flow + + + + + Logger enabled for + level of call? + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + No + + + Yes + + + Yes + + + No + + + No + + + Yes + + + Yes + + + No + + + No + + + Yes + + + + + + No + + + + + + + + + Yes + + + No + + + + + + Yes + + + Record passed + to handler + + + + + + + + diff --git a/Doc/howto/mro.rst b/Doc/howto/mro.rst new file mode 100644 index 00000000000000..0872bedcd3a2d3 --- /dev/null +++ b/Doc/howto/mro.rst @@ -0,0 +1,671 @@ +.. _python_2.3_mro: + +The Python 2.3 Method Resolution Order +====================================== + +.. note:: + + This is a historical document, provided as an appendix to the official + documentation. + The Method Resolution Order discussed here was *introduced* in Python 2.3, + but it is still used in later versions -- including Python 3. + +By `Michele Simionato `__. + +:Abstract: + + *This document is intended for Python programmers who want to + understand the C3 Method Resolution Order used in Python 2.3. + Although it is not intended for newbies, it is quite pedagogical with + many worked out examples. I am not aware of other publicly available + documents with the same scope, therefore it should be useful.* + +Disclaimer: + + *I donate this document to the Python Software Foundation, under the + Python 2.3 license. As usual in these circumstances, I warn the + reader that what follows* should *be correct, but I don't give any + warranty. Use it at your own risk and peril!* + +Acknowledgments: + + *All the people of the Python mailing list who sent me their support. + Paul Foley who pointed out various imprecisions and made me to add the + part on local precedence ordering. David Goodger for help with the + formatting in reStructuredText. David Mertz for help with the editing. + Finally, Guido van Rossum who enthusiastically added this document to + the official Python 2.3 home-page.* + +The beginning +------------- + + *Felix qui potuit rerum cognoscere causas* -- Virgilius + +Everything started with a post by Samuele Pedroni to the Python +development mailing list [#]_. In his post, Samuele showed that the +Python 2.2 method resolution order is not monotonic and he proposed to +replace it with the C3 method resolution order. Guido agreed with his +arguments and therefore now Python 2.3 uses C3. The C3 method itself +has nothing to do with Python, since it was invented by people working +on Dylan and it is described in a paper intended for lispers [#]_. The +present paper gives a (hopefully) readable discussion of the C3 +algorithm for Pythonistas who want to understand the reasons for the +change. + +First of all, let me point out that what I am going to say only applies +to the *new style classes* introduced in Python 2.2: *classic classes* +maintain their old method resolution order, depth first and then left to +right. Therefore, there is no breaking of old code for classic classes; +and even if in principle there could be breaking of code for Python 2.2 +new style classes, in practice the cases in which the C3 resolution +order differs from the Python 2.2 method resolution order are so rare +that no real breaking of code is expected. Therefore: + + *Don't be scared!* + +Moreover, unless you make strong use of multiple inheritance and you +have non-trivial hierarchies, you don't need to understand the C3 +algorithm, and you can easily skip this paper. On the other hand, if +you really want to know how multiple inheritance works, then this paper +is for you. The good news is that things are not as complicated as you +might expect. + +Let me begin with some basic definitions. + +1) Given a class C in a complicated multiple inheritance hierarchy, it + is a non-trivial task to specify the order in which methods are + overridden, i.e. to specify the order of the ancestors of C. + +2) The list of the ancestors of a class C, including the class itself, + ordered from the nearest ancestor to the furthest, is called the + class precedence list or the *linearization* of C. + +3) The *Method Resolution Order* (MRO) is the set of rules that + construct the linearization. In the Python literature, the idiom + "the MRO of C" is also used as a synonymous for the linearization of + the class C. + +4) For instance, in the case of single inheritance hierarchy, if C is a + subclass of C1, and C1 is a subclass of C2, then the linearization of + C is simply the list [C, C1 , C2]. However, with multiple + inheritance hierarchies, the construction of the linearization is + more cumbersome, since it is more difficult to construct a + linearization that respects *local precedence ordering* and + *monotonicity*. + +5) I will discuss the local precedence ordering later, but I can give + the definition of monotonicity here. A MRO is monotonic when the + following is true: *if C1 precedes C2 in the linearization of C, + then C1 precedes C2 in the linearization of any subclass of C*. + Otherwise, the innocuous operation of deriving a new class could + change the resolution order of methods, potentially introducing very + subtle bugs. Examples where this happens will be shown later. + +6) Not all classes admit a linearization. There are cases, in + complicated hierarchies, where it is not possible to derive a class + such that its linearization respects all the desired properties. + +Here I give an example of this situation. Consider the hierarchy + + >>> O = object + >>> class X(O): pass + >>> class Y(O): pass + >>> class A(X,Y): pass + >>> class B(Y,X): pass + +which can be represented with the following inheritance graph, where I +have denoted with O the ``object`` class, which is the beginning of any +hierarchy for new style classes: + + .. code-block:: text + + ----------- + | | + | O | + | / \ | + - X Y / + | / | / + | / |/ + A B + \ / + ? + +In this case, it is not possible to derive a new class C from A and B, +since X precedes Y in A, but Y precedes X in B, therefore the method +resolution order would be ambiguous in C. + +Python 2.3 raises an exception in this situation (TypeError: MRO +conflict among bases Y, X) forbidding the naive programmer from creating +ambiguous hierarchies. Python 2.2 instead does not raise an exception, +but chooses an *ad hoc* ordering (CABXYO in this case). + +The C3 Method Resolution Order +------------------------------ + +Let me introduce a few simple notations which will be useful for the +following discussion. I will use the shortcut notation:: + + C1 C2 ... CN + +to indicate the list of classes [C1, C2, ... , CN]. + +The *head* of the list is its first element:: + + head = C1 + +whereas the *tail* is the rest of the list:: + + tail = C2 ... CN. + +I shall also use the notation:: + + C + (C1 C2 ... CN) = C C1 C2 ... CN + +to denote the sum of the lists [C] + [C1, C2, ... ,CN]. + +Now I can explain how the MRO works in Python 2.3. + +Consider a class C in a multiple inheritance hierarchy, with C +inheriting from the base classes B1, B2, ... , BN. We want to +compute the linearization L[C] of the class C. The rule is the +following: + + *the linearization of C is the sum of C plus the merge of the + linearizations of the parents and the list of the parents.* + +In symbolic notation:: + + L[C(B1 ... BN)] = C + merge(L[B1] ... L[BN], B1 ... BN) + +In particular, if C is the ``object`` class, which has no parents, the +linearization is trivial:: + + L[object] = object. + +However, in general one has to compute the merge according to the following +prescription: + + *take the head of the first list, i.e L[B1][0]; if this head is not in + the tail of any of the other lists, then add it to the linearization + of C and remove it from the lists in the merge, otherwise look at the + head of the next list and take it, if it is a good head. Then repeat + the operation until all the class are removed or it is impossible to + find good heads. In this case, it is impossible to construct the + merge, Python 2.3 will refuse to create the class C and will raise an + exception.* + +This prescription ensures that the merge operation *preserves* the +ordering, if the ordering can be preserved. On the other hand, if the +order cannot be preserved (as in the example of serious order +disagreement discussed above) then the merge cannot be computed. + +The computation of the merge is trivial if C has only one parent +(single inheritance); in this case:: + + L[C(B)] = C + merge(L[B],B) = C + L[B] + +However, in the case of multiple inheritance things are more cumbersome +and I don't expect you can understand the rule without a couple of +examples ;-) + +Examples +-------- + +First example. Consider the following hierarchy: + + >>> O = object + >>> class F(O): pass + >>> class E(O): pass + >>> class D(O): pass + >>> class C(D,F): pass + >>> class B(D,E): pass + >>> class A(B,C): pass + +In this case the inheritance graph can be drawn as: + + .. code-block:: text + + 6 + --- + Level 3 | O | (more general) + / --- \ + / | \ | + / | \ | + / | \ | + --- --- --- | + Level 2 3 | D | 4| E | | F | 5 | + --- --- --- | + \ \ _ / | | + \ / \ _ | | + \ / \ | | + --- --- | + Level 1 1 | B | | C | 2 | + --- --- | + \ / | + \ / \ / + --- + Level 0 0 | A | (more specialized) + --- + + +The linearizations of O,D,E and F are trivial:: + + L[O] = O + L[D] = D O + L[E] = E O + L[F] = F O + +The linearization of B can be computed as:: + + L[B] = B + merge(DO, EO, DE) + +We see that D is a good head, therefore we take it and we are reduced to +compute ``merge(O,EO,E)``. Now O is not a good head, since it is in the +tail of the sequence EO. In this case the rule says that we have to +skip to the next sequence. Then we see that E is a good head; we take +it and we are reduced to compute ``merge(O,O)`` which gives O. Therefore:: + + L[B] = B D E O + +Using the same procedure one finds:: + + L[C] = C + merge(DO,FO,DF) + = C + D + merge(O,FO,F) + = C + D + F + merge(O,O) + = C D F O + +Now we can compute:: + + L[A] = A + merge(BDEO,CDFO,BC) + = A + B + merge(DEO,CDFO,C) + = A + B + C + merge(DEO,DFO) + = A + B + C + D + merge(EO,FO) + = A + B + C + D + E + merge(O,FO) + = A + B + C + D + E + F + merge(O,O) + = A B C D E F O + +In this example, the linearization is ordered in a pretty nice way +according to the inheritance level, in the sense that lower levels (i.e. +more specialized classes) have higher precedence (see the inheritance +graph). However, this is not the general case. + +I leave as an exercise for the reader to compute the linearization for +my second example: + + >>> O = object + >>> class F(O): pass + >>> class E(O): pass + >>> class D(O): pass + >>> class C(D,F): pass + >>> class B(E,D): pass + >>> class A(B,C): pass + +The only difference with the previous example is the change B(D,E) --> +B(E,D); however even such a little modification completely changes the +ordering of the hierarchy: + + .. code-block:: text + + 6 + --- + Level 3 | O | + / --- \ + / | \ + / | \ + / | \ + --- --- --- + Level 2 2 | E | 4 | D | | F | 5 + --- --- --- + \ / \ / + \ / \ / + \ / \ / + --- --- + Level 1 1 | B | | C | 3 + --- --- + \ / + \ / + --- + Level 0 0 | A | + --- + + +Notice that the class E, which is in the second level of the hierarchy, +precedes the class C, which is in the first level of the hierarchy, i.e. +E is more specialized than C, even if it is in a higher level. + +A lazy programmer can obtain the MRO directly from Python 2.2, since in +this case it coincides with the Python 2.3 linearization. It is enough +to invoke the :meth:`~type.mro` method of class A: + + >>> A.mro() # doctest: +NORMALIZE_WHITESPACE + [, , , + , , , + ] + +Finally, let me consider the example discussed in the first section, +involving a serious order disagreement. In this case, it is +straightforward to compute the linearizations of O, X, Y, A and B: + + .. code-block:: text + + L[O] = 0 + L[X] = X O + L[Y] = Y O + L[A] = A X Y O + L[B] = B Y X O + +However, it is impossible to compute the linearization for a class C +that inherits from A and B:: + + L[C] = C + merge(AXYO, BYXO, AB) + = C + A + merge(XYO, BYXO, B) + = C + A + B + merge(XYO, YXO) + +At this point we cannot merge the lists XYO and YXO, since X is in the +tail of YXO whereas Y is in the tail of XYO: therefore there are no +good heads and the C3 algorithm stops. Python 2.3 raises an error and +refuses to create the class C. + +Bad Method Resolution Orders +---------------------------- + +A MRO is *bad* when it breaks such fundamental properties as local +precedence ordering and monotonicity. In this section, I will show +that both the MRO for classic classes and the MRO for new style classes +in Python 2.2 are bad. + +It is easier to start with the local precedence ordering. Consider the +following example: + + >>> F=type('Food',(),{'remember2buy':'spam'}) + >>> E=type('Eggs',(F,),{'remember2buy':'eggs'}) + >>> G=type('GoodFood',(F,E),{}) # under Python 2.3 this is an error! # doctest: +SKIP + +with inheritance diagram + + .. code-block:: text + + O + | + (buy spam) F + | \ + | E (buy eggs) + | / + G + + (buy eggs or spam ?) + + +We see that class G inherits from F and E, with F *before* E: therefore +we would expect the attribute *G.remember2buy* to be inherited by +*F.remember2buy* and not by *E.remember2buy*: nevertheless Python 2.2 +gives + + >>> G.remember2buy # doctest: +SKIP + 'eggs' + +This is a breaking of local precedence ordering since the order in the +local precedence list, i.e. the list of the parents of G, is not +preserved in the Python 2.2 linearization of G:: + + L[G,P22]= G E F object # F *follows* E + +One could argue that the reason why F follows E in the Python 2.2 +linearization is that F is less specialized than E, since F is the +superclass of E; nevertheless the breaking of local precedence ordering +is quite non-intuitive and error prone. This is particularly true since +it is a different from old style classes: + + >>> class F: remember2buy='spam' + >>> class E(F): remember2buy='eggs' + >>> class G(F,E): pass # doctest: +SKIP + >>> G.remember2buy # doctest: +SKIP + 'spam' + +In this case the MRO is GFEF and the local precedence ordering is +preserved. + +As a general rule, hierarchies such as the previous one should be +avoided, since it is unclear if F should override E or vice-versa. +Python 2.3 solves the ambiguity by raising an exception in the creation +of class G, effectively stopping the programmer from generating +ambiguous hierarchies. The reason for that is that the C3 algorithm +fails when the merge:: + + merge(FO,EFO,FE) + +cannot be computed, because F is in the tail of EFO and E is in the tail +of FE. + +The real solution is to design a non-ambiguous hierarchy, i.e. to derive +G from E and F (the more specific first) and not from F and E; in this +case the MRO is GEF without any doubt. + + .. code-block:: text + + O + | + F (spam) + / | + (eggs) E | + \ | + G + (eggs, no doubt) + + +Python 2.3 forces the programmer to write good hierarchies (or, at +least, less error-prone ones). + +On a related note, let me point out that the Python 2.3 algorithm is +smart enough to recognize obvious mistakes, as the duplication of +classes in the list of parents: + + >>> class A(object): pass + >>> class C(A,A): pass # error + Traceback (most recent call last): + File "", line 1, in ? + TypeError: duplicate base class A + +Python 2.2 (both for classic classes and new style classes) in this +situation, would not raise any exception. + +Finally, I would like to point out two lessons we have learned from this +example: + +1. despite the name, the MRO determines the resolution order of + attributes, not only of methods; + +2. the default food for Pythonistas is spam ! (but you already knew + that ;-) + +Having discussed the issue of local precedence ordering, let me now +consider the issue of monotonicity. My goal is to show that neither the +MRO for classic classes nor that for Python 2.2 new style classes is +monotonic. + +To prove that the MRO for classic classes is non-monotonic is rather +trivial, it is enough to look at the diamond diagram: + + .. code-block:: text + + + C + / \ + / \ + A B + \ / + \ / + D + +One easily discerns the inconsistency:: + + L[B,P21] = B C # B precedes C : B's methods win + L[D,P21] = D A C B C # B follows C : C's methods win! + +On the other hand, there are no problems with the Python 2.2 and 2.3 +MROs, they give both:: + + L[D] = D A B C + +Guido points out in his essay [#]_ that the classic MRO is not so bad in +practice, since one can typically avoids diamonds for classic classes. +But all new style classes inherit from ``object``, therefore diamonds are +unavoidable and inconsistencies shows up in every multiple inheritance +graph. + +The MRO of Python 2.2 makes breaking monotonicity difficult, but not +impossible. The following example, originally provided by Samuele +Pedroni, shows that the MRO of Python 2.2 is non-monotonic: + + >>> class A(object): pass + >>> class B(object): pass + >>> class C(object): pass + >>> class D(object): pass + >>> class E(object): pass + >>> class K1(A,B,C): pass + >>> class K2(D,B,E): pass + >>> class K3(D,A): pass + >>> class Z(K1,K2,K3): pass + +Here are the linearizations according to the C3 MRO (the reader should +verify these linearizations as an exercise and draw the inheritance +diagram ;-) :: + + L[A] = A O + L[B] = B O + L[C] = C O + L[D] = D O + L[E] = E O + L[K1]= K1 A B C O + L[K2]= K2 D B E O + L[K3]= K3 D A O + L[Z] = Z K1 K2 K3 D A B C E O + +Python 2.2 gives exactly the same linearizations for A, B, C, D, E, K1, +K2 and K3, but a different linearization for Z:: + + L[Z,P22] = Z K1 K3 A K2 D B C E O + +It is clear that this linearization is *wrong*, since A comes before D +whereas in the linearization of K3 A comes *after* D. In other words, in +K3 methods derived by D override methods derived by A, but in Z, which +still is a subclass of K3, methods derived by A override methods derived +by D! This is a violation of monotonicity. Moreover, the Python 2.2 +linearization of Z is also inconsistent with local precedence ordering, +since the local precedence list of the class Z is [K1, K2, K3] (K2 +precedes K3), whereas in the linearization of Z K2 *follows* K3. These +problems explain why the 2.2 rule has been dismissed in favor of the C3 +rule. + +The end +------- + +This section is for the impatient reader, who skipped all the previous +sections and jumped immediately to the end. This section is for the +lazy programmer too, who didn't want to exercise her/his brain. +Finally, it is for the programmer with some hubris, otherwise s/he would +not be reading a paper on the C3 method resolution order in multiple +inheritance hierarchies ;-) These three virtues taken all together (and +*not* separately) deserve a prize: the prize is a short Python 2.2 +script that allows you to compute the 2.3 MRO without risk to your +brain. Simply change the last line to play with the various examples I +have discussed in this paper.:: + + # + + """C3 algorithm by Samuele Pedroni (with readability enhanced by me).""" + + class __metaclass__(type): + "All classes are metamagically modified to be nicely printed" + __repr__ = lambda cls: cls.__name__ + + class ex_2: + "Serious order disagreement" #From Guido + class O: pass + class X(O): pass + class Y(O): pass + class A(X,Y): pass + class B(Y,X): pass + try: + class Z(A,B): pass #creates Z(A,B) in Python 2.2 + except TypeError: + pass # Z(A,B) cannot be created in Python 2.3 + + class ex_5: + "My first example" + class O: pass + class F(O): pass + class E(O): pass + class D(O): pass + class C(D,F): pass + class B(D,E): pass + class A(B,C): pass + + class ex_6: + "My second example" + class O: pass + class F(O): pass + class E(O): pass + class D(O): pass + class C(D,F): pass + class B(E,D): pass + class A(B,C): pass + + class ex_9: + "Difference between Python 2.2 MRO and C3" #From Samuele + class O: pass + class A(O): pass + class B(O): pass + class C(O): pass + class D(O): pass + class E(O): pass + class K1(A,B,C): pass + class K2(D,B,E): pass + class K3(D,A): pass + class Z(K1,K2,K3): pass + + def merge(seqs): + print '\n\nCPL[%s]=%s' % (seqs[0][0],seqs), + res = []; i=0 + while 1: + nonemptyseqs=[seq for seq in seqs if seq] + if not nonemptyseqs: return res + i+=1; print '\n',i,'round: candidates...', + for seq in nonemptyseqs: # find merge candidates among seq heads + cand = seq[0]; print ' ',cand, + nothead=[s for s in nonemptyseqs if cand in s[1:]] + if nothead: cand=None #reject candidate + else: break + if not cand: raise "Inconsistent hierarchy" + res.append(cand) + for seq in nonemptyseqs: # remove cand + if seq[0] == cand: del seq[0] + + def mro(C): + "Compute the class precedence list (mro) according to C3" + return merge([[C]]+map(mro,C.__bases__)+[list(C.__bases__)]) + + def print_mro(C): + print '\nMRO[%s]=%s' % (C,mro(C)) + print '\nP22 MRO[%s]=%s' % (C,C.mro()) + + print_mro(ex_9.Z) + + # + +That's all folks, + + enjoy ! + + +Resources +--------- + +.. [#] The thread on python-dev started by Samuele Pedroni: + https://mail.python.org/pipermail/python-dev/2002-October/029035.html + +.. [#] The paper *A Monotonic Superclass Linearization for Dylan*: + https://doi.org/10.1145/236337.236343 + +.. [#] Guido van Rossum's essay, *Unifying types and classes in Python 2.2*: + https://web.archive.org/web/20140210194412/http://www.python.org/download/releases/2.2.2/descrintro diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index bb1c00e0aa51d5..06459d1b222964 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -162,12 +162,12 @@ the :option:`!-X` option takes precedence over the environment variable. Example, using the environment variable:: - $ PYTHONPERFSUPPORT=1 python script.py + $ PYTHONPERFSUPPORT=1 perf record -F 9999 -g -o perf.data python script.py $ perf report -g -i perf.data Example, using the :option:`!-X` option:: - $ python -X perf script.py + $ perf record -F 9999 -g -o perf.data python -X perf script.py $ perf report -g -i perf.data Example, using the :mod:`sys` APIs in file :file:`example.py`: @@ -184,7 +184,7 @@ Example, using the :mod:`sys` APIs in file :file:`example.py`: ...then:: - $ python ./example.py + $ perf record -F 9999 -g -o perf.data python ./example.py $ perf report -g -i perf.data @@ -205,3 +205,62 @@ You can check if your system has been compiled with this flag by running:: If you don't see any output it means that your interpreter has not been compiled with frame pointers and therefore it may not be able to show Python functions in the output of ``perf``. + + +How to work without frame pointers +---------------------------------- + +If you are working with a Python interpreter that has been compiled without +frame pointers, you can still use the ``perf`` profiler, but the overhead will be +a bit higher because Python needs to generate unwinding information for every +Python function call on the fly. Additionally, ``perf`` will take more time to +process the data because it will need to use the DWARF debugging information to +unwind the stack and this is a slow process. + +To enable this mode, you can use the environment variable +:envvar:`PYTHON_PERF_JIT_SUPPORT` or the :option:`-X perf_jit <-X>` option, +which will enable the JIT mode for the ``perf`` profiler. + +.. note:: + + Due to a bug in the ``perf`` tool, only ``perf`` versions higher than v6.8 + will work with the JIT mode. The fix was also backported to the v6.7.2 + version of the tool. + + Note that when checking the version of the ``perf`` tool (which can be done + by running ``perf version``) you must take into account that some distros + add some custom version numbers including a ``-`` character. This means + that ``perf 6.7-3`` is not necessarily ``perf 6.7.3``. + +When using the perf JIT mode, you need an extra step before you can run ``perf +report``. You need to call the ``perf inject`` command to inject the JIT +information into the ``perf.data`` file.:: + + $ perf record -F 9999 -g --call-graph dwarf -o perf.data python -Xperf_jit my_script.py + $ perf inject -i perf.data --jit --output perf.jit.data + $ perf report -g -i perf.jit.data + +or using the environment variable:: + + $ PYTHON_PERF_JIT_SUPPORT=1 perf record -F 9999 -g --call-graph dwarf -o perf.data python my_script.py + $ perf inject -i perf.data --jit --output perf.jit.data + $ perf report -g -i perf.jit.data + +``perf inject --jit`` command will read ``perf.data``, +automatically pick up the perf dump file that Python creates (in +``/tmp/perf-$PID.dump``), and then create ``perf.jit.data`` which merges all the +JIT information together. It should also create a lot of ``jitted-XXXX-N.so`` +files in the current directory which are ELF images for all the JIT trampolines +that were created by Python. + +.. warning:: + Notice that when using ``--call-graph dwarf`` the ``perf`` tool will take + snapshots of the stack of the process being profiled and save the + information in the ``perf.data`` file. By default the size of the stack dump + is 8192 bytes but the user can change the size by passing the size after + comma like ``--call-graph dwarf,4096``. The size of the stack dump is + important because if the size is too small ``perf`` will not be able to + unwind the stack and the output will be incomplete. On the other hand, if + the size is too big, then ``perf`` won't be able to sample the process as + frequently as it would like as the overhead will be higher. + diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst index d560364107bd12..9f73c811cfcbc0 100644 --- a/Doc/howto/pyporting.rst +++ b/Doc/howto/pyporting.rst @@ -18,9 +18,9 @@ please see :ref:`cporting-howto`. The archived python-porting_ mailing list may contain some useful guidance. -Since Python 3.13 the original porting guide was discontinued. +Since Python 3.11 the original porting guide was discontinued. You can find the old guide in the -`archive `_. +`archive `_. Third-party guides diff --git a/Doc/howto/sockets.rst b/Doc/howto/sockets.rst index 0bbf97da39768d..cbc49d15a0771b 100644 --- a/Doc/howto/sockets.rst +++ b/Doc/howto/sockets.rst @@ -100,8 +100,8 @@ mainloop of the web server:: (clientsocket, address) = serversocket.accept() # now do something with the clientsocket # in this case, we'll pretend this is a threaded server - ct = client_thread(clientsocket) - ct.run() + ct = make_client_thread(clientsocket) + ct.start() There's actually 3 general ways in which this loop could work - dispatching a thread to handle ``clientsocket``, create a new process to handle diff --git a/Doc/howto/sorting.rst b/Doc/howto/sorting.rst index b98f91e023bdfc..70c34cde8a0659 100644 --- a/Doc/howto/sorting.rst +++ b/Doc/howto/sorting.rst @@ -47,11 +47,14 @@ lists. In contrast, the :func:`sorted` function accepts any iterable. Key Functions ============= -Both :meth:`list.sort` and :func:`sorted` have a *key* parameter to specify a -function (or other callable) to be called on each list element prior to making +The :meth:`list.sort` method and the functions :func:`sorted`, +:func:`min`, :func:`max`, :func:`heapq.nsmallest`, and +:func:`heapq.nlargest` have a *key* parameter to specify a function (or +other callable) to be called on each list element prior to making comparisons. -For example, here's a case-insensitive string comparison: +For example, here's a case-insensitive string comparison using +:meth:`str.casefold`: .. doctest:: @@ -272,6 +275,70 @@ to make it usable as a key function:: sorted(words, key=cmp_to_key(strcoll)) # locale-aware sort order +Strategies For Unorderable Types and Values +=========================================== + +A number of type and value issues can arise when sorting. +Here are some strategies that can help: + +* Convert non-comparable input types to strings prior to sorting: + +.. doctest:: + + >>> data = ['twelve', '11', 10] + >>> sorted(map(str, data)) + ['10', '11', 'twelve'] + +This is needed because most cross-type comparisons raise a +:exc:`TypeError`. + +* Remove special values prior to sorting: + +.. doctest:: + + >>> from math import isnan + >>> from itertools import filterfalse + >>> data = [3.3, float('nan'), 1.1, 2.2] + >>> sorted(filterfalse(isnan, data)) + [1.1, 2.2, 3.3] + +This is needed because the `IEEE-754 standard +`_ specifies that, "Every NaN +shall compare unordered with everything, including itself." + +Likewise, ``None`` can be stripped from datasets as well: + +.. doctest:: + + >>> data = [3.3, None, 1.1, 2.2] + >>> sorted(x for x in data if x is not None) + [1.1, 2.2, 3.3] + +This is needed because ``None`` is not comparable to other types. + +* Convert mapping types into sorted item lists before sorting: + +.. doctest:: + + >>> data = [{'a': 1}, {'b': 2}] + >>> sorted(data, key=lambda d: sorted(d.items())) + [{'a': 1}, {'b': 2}] + +This is needed because dict-to-dict comparisons raise a +:exc:`TypeError`. + +* Convert set types into sorted lists before sorting: + +.. doctest:: + + >>> data = [{'a', 'b', 'c'}, {'b', 'c', 'd'}] + >>> sorted(map(sorted, data)) + [['a', 'b', 'c'], ['b', 'c', 'd']] + +This is needed because the elements contained in set types do not have a +deterministic order. For example, ``list({'a', 'b'})`` may produce +either ``['a', 'b']`` or ``['b', 'a']``. + Odds and Ends ============= diff --git a/Doc/howto/timerfd.rst b/Doc/howto/timerfd.rst index 98f0294f9d082d..b5fc06ae8c6810 100644 --- a/Doc/howto/timerfd.rst +++ b/Doc/howto/timerfd.rst @@ -108,7 +108,7 @@ descriptors to wait until the file descriptor is ready for reading: # In 1.5 seconds, 1st timer, 2nd timer and 3rd timer fires at once. # # If a timer file descriptor is signaled more than once since - # the last os.read() call, os.read() returns the nubmer of signaled + # the last os.read() call, os.read() returns the number of signaled # as host order of class bytes. print(f"Signaled events={events}") for fd, event in events: diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 7f54a410881514..33a2a7ea89ea07 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -594,5 +594,5 @@ This document was reviewed and revised by John Lee. scripts with a localhost server, I have to prevent urllib from using the proxy. .. [#] urllib opener for SSL proxy (CONNECT method): `ASPN Cookbook Recipe - `_. + `_. diff --git a/Doc/includes/email-dir.py b/Doc/includes/email-dir.py index 2fc1570e654db6..aa2a5c7cda52aa 100644 --- a/Doc/includes/email-dir.py +++ b/Doc/includes/email-dir.py @@ -53,7 +53,7 @@ def main(): # Guess the content type based on the file's extension. Encoding # will be ignored, although we should check for simple things like # gzip'd or compressed files. - ctype, encoding = mimetypes.guess_type(path) + ctype, encoding = mimetypes.guess_file_type(path) if ctype is None or encoding is not None: # No guess could be made, or the file is encoded (compressed), so # use a generic bag-of-bits type. diff --git a/Doc/includes/newtypes/custom2.c b/Doc/includes/newtypes/custom2.c index a0222b1795209b..768ce29fab9ff0 100644 --- a/Doc/includes/newtypes/custom2.c +++ b/Doc/includes/newtypes/custom2.c @@ -23,12 +23,12 @@ Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) CustomObject *self; self = (CustomObject *) type->tp_alloc(type, 0); if (self != NULL) { - self->first = PyUnicode_FromString(""); + self->first = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (self->first == NULL) { Py_DECREF(self); return NULL; } - self->last = PyUnicode_FromString(""); + self->last = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (self->last == NULL) { Py_DECREF(self); return NULL; diff --git a/Doc/includes/newtypes/custom3.c b/Doc/includes/newtypes/custom3.c index 4aeebe0a7507d1..7d969adfa7c9cc 100644 --- a/Doc/includes/newtypes/custom3.c +++ b/Doc/includes/newtypes/custom3.c @@ -23,12 +23,12 @@ Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) CustomObject *self; self = (CustomObject *) type->tp_alloc(type, 0); if (self != NULL) { - self->first = PyUnicode_FromString(""); + self->first = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (self->first == NULL) { Py_DECREF(self); return NULL; } - self->last = PyUnicode_FromString(""); + self->last = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (self->last == NULL) { Py_DECREF(self); return NULL; diff --git a/Doc/includes/newtypes/custom4.c b/Doc/includes/newtypes/custom4.c index 3998918f68301e..a7b8de44a57c90 100644 --- a/Doc/includes/newtypes/custom4.c +++ b/Doc/includes/newtypes/custom4.c @@ -39,12 +39,12 @@ Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) CustomObject *self; self = (CustomObject *) type->tp_alloc(type, 0); if (self != NULL) { - self->first = PyUnicode_FromString(""); + self->first = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (self->first == NULL) { Py_DECREF(self); return NULL; } - self->last = PyUnicode_FromString(""); + self->last = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (self->last == NULL) { Py_DECREF(self); return NULL; diff --git a/Doc/includes/wasm-mobile-notavail.rst b/Doc/includes/wasm-mobile-notavail.rst new file mode 100644 index 00000000000000..725b0f7a86d17d --- /dev/null +++ b/Doc/includes/wasm-mobile-notavail.rst @@ -0,0 +1,6 @@ +.. include for modules that don't work on WASM or mobile platforms + +.. availability:: not Android, not iOS, not WASI. + + This module is not supported on :ref:`mobile platforms ` + or :ref:`WebAssembly platforms `. diff --git a/Doc/includes/wasm-notavail.rst b/Doc/includes/wasm-notavail.rst index e680e1f9b43807..c1b79d2a4a0508 100644 --- a/Doc/includes/wasm-notavail.rst +++ b/Doc/includes/wasm-notavail.rst @@ -1,7 +1,6 @@ .. include for modules that don't work on WASM -.. availability:: not Emscripten, not WASI. +.. availability:: not WASI. - This module does not work or is not available on WebAssembly platforms - ``wasm32-emscripten`` and ``wasm32-wasi``. See + This module does not work or is not available on WebAssembly. See :ref:`wasm-availability` for more information. diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index 762f8b4695b3dd..4f3b663006fb28 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -1,5 +1,5 @@ -:mod:`__future__` --- Future statement definitions -================================================== +:mod:`!__future__` --- Future statement definitions +=================================================== .. module:: __future__ :synopsis: Future statement definitions @@ -64,8 +64,10 @@ language using this mechanism: | generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | | | | | *StopIteration handling inside generators* | +------------------+-------------+--------------+---------------------------------------------+ -| annotations | 3.7.0b1 | TBD [1]_ | :pep:`563`: | -| | | | *Postponed evaluation of annotations* | +| annotations | 3.7.0b1 | Never [1]_ | :pep:`563`: | +| | | | *Postponed evaluation of annotations*, | +| | | | :pep:`649`: *Deferred evaluation of | +| | | | annotations using descriptors* | +------------------+-------------+--------------+---------------------------------------------+ .. XXX Adding a new entry? Remember to update simple_stmts.rst, too. @@ -115,11 +117,9 @@ language using this mechanism: .. [1] ``from __future__ import annotations`` was previously scheduled to - become mandatory in Python 3.10, but the Python Steering Council - twice decided to delay the change - (`announcement for Python 3.10 `__; - `announcement for Python 3.11 `__). - No final decision has been made yet. See also :pep:`563` and :pep:`649`. + become mandatory in Python 3.10, but the change was delayed and ultimately + canceled. This feature will eventually be deprecated and removed. See + :pep:`649` and :pep:`749`. .. seealso:: diff --git a/Doc/library/__main__.rst b/Doc/library/__main__.rst index c999253f781b10..4407ba2f7714dd 100644 --- a/Doc/library/__main__.rst +++ b/Doc/library/__main__.rst @@ -1,5 +1,5 @@ -:mod:`__main__` --- Top-level code environment -============================================== +:mod:`!__main__` --- Top-level code environment +=============================================== .. module:: __main__ :synopsis: The environment where top-level code is run. Covers command-line @@ -251,9 +251,9 @@ attribute will include the package's path if imported:: >>> asyncio.__main__.__name__ 'asyncio.__main__' -This won't work for ``__main__.py`` files in the root directory of a .zip file -though. Hence, for consistency, minimal ``__main__.py`` like the :mod:`venv` -one mentioned below are preferred. +This won't work for ``__main__.py`` files in the root directory of a +``.zip`` file though. Hence, for consistency, a minimal ``__main__.py`` +without a ``__name__`` check is preferred. .. seealso:: @@ -292,10 +292,7 @@ Here is an example module that consumes the ``__main__`` namespace:: if not did_user_define_their_name(): raise ValueError('Define the variable `my_name`!') - if '__file__' in dir(__main__): - print(__main__.my_name, "found in file", __main__.__file__) - else: - print(__main__.my_name) + print(__main__.my_name) Example usage of this module could be as follows:: @@ -330,7 +327,7 @@ status code 0, indicating success: .. code-block:: shell-session $ python start.py - Dinsdale found in file /path/to/start.py + Dinsdale Note that importing ``__main__`` doesn't cause any issues with unintentionally running top-level code meant for script use which is put in the @@ -361,8 +358,5 @@ defined in the REPL becomes part of the ``__main__`` scope:: >>> namely.print_user_name() Jabberwocky -Note that in this case the ``__main__`` scope doesn't contain a ``__file__`` -attribute as it's interactive. - The ``__main__`` scope is used in the implementation of :mod:`pdb` and :mod:`rlcompleter`. diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index 297f50a46e0692..ed29ac70035597 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -1,5 +1,5 @@ -:mod:`_thread` --- Low-level threading API -========================================== +:mod:`!_thread` --- Low-level threading API +=========================================== .. module:: _thread :synopsis: Low-level threading API. @@ -169,14 +169,14 @@ Lock objects have the following methods: time can acquire a lock --- that's their reason for existence). If the *blocking* argument is present, the action depends on its - value: if it is False, the lock is only acquired if it can be acquired - immediately without waiting, while if it is True, the lock is acquired + value: if it is false, the lock is only acquired if it can be acquired + immediately without waiting, while if it is true, the lock is acquired unconditionally as above. If the floating-point *timeout* argument is present and positive, it specifies the maximum wait time in seconds before returning. A negative *timeout* argument specifies an unbounded wait. You cannot specify - a *timeout* if *blocking* is False. + a *timeout* if *blocking* is false. The return value is ``True`` if the lock is acquired successfully, ``False`` if not. @@ -187,6 +187,9 @@ Lock objects have the following methods: .. versionchanged:: 3.2 Lock acquires can now be interrupted by signals on POSIX. + .. versionchanged:: 3.14 + Lock acquires can now be interrupted by signals on Windows. + .. method:: lock.release() @@ -213,23 +216,14 @@ In addition to these methods, lock objects can also be used via the .. index:: pair: module; signal -* Threads interact strangely with interrupts: the :exc:`KeyboardInterrupt` - exception will be received by an arbitrary thread. (When the :mod:`signal` - module is available, interrupts always go to the main thread.) +* Interrupts always go to the main thread (the :exc:`KeyboardInterrupt` + exception will be received by that thread.) * Calling :func:`sys.exit` or raising the :exc:`SystemExit` exception is equivalent to calling :func:`_thread.exit`. -* It is not possible to interrupt the :meth:`~threading.Lock.acquire` method on - a lock --- the :exc:`KeyboardInterrupt` exception will happen after the lock - has been acquired. - * When the main thread exits, it is system defined whether the other threads survive. On most systems, they are killed without executing :keyword:`try` ... :keyword:`finally` clauses or executing object destructors. -* When the main thread exits, it does not do any of its usual cleanup (except - that :keyword:`try` ... :keyword:`finally` clauses are honored), and the - standard I/O files are not flushed. - diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index 10e2cba50e49b0..38d744e97d087d 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -1,5 +1,5 @@ -:mod:`abc` --- Abstract Base Classes -==================================== +:mod:`!abc` --- Abstract Base Classes +===================================== .. module:: abc :synopsis: Abstract base classes according to :pep:`3119`. @@ -99,7 +99,7 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: that you can customize the behavior of :func:`issubclass` further without the need to call :meth:`register` on every class you want to consider a subclass of the ABC. (This class method is called from the - :meth:`~class.__subclasscheck__` method of the ABC.) + :meth:`~type.__subclasscheck__` method of the ABC.) This method should return ``True``, ``False`` or :data:`NotImplemented`. If it returns ``True``, the *subclass* is considered a subclass of this ABC. @@ -149,7 +149,7 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: The :meth:`__subclasshook__` class method defined here says that any class that has an :meth:`~iterator.__iter__` method in its :attr:`~object.__dict__` (or in that of one of its base classes, accessed - via the :attr:`~class.__mro__` list) is considered a ``MyIterable`` too. + via the :attr:`~type.__mro__` list) is considered a ``MyIterable`` too. Finally, the last line makes ``Foo`` a virtual subclass of ``MyIterable``, even though it does not define an :meth:`~iterator.__iter__` method (it uses diff --git a/Doc/library/aifc.rst b/Doc/library/aifc.rst new file mode 100644 index 00000000000000..a756d679036ecb --- /dev/null +++ b/Doc/library/aifc.rst @@ -0,0 +1,15 @@ +:mod:`!aifc` --- Read and write AIFF and AIFC files +=================================================== + +.. module:: aifc + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!aifc` module was +`Python 3.12 `_. diff --git a/Doc/library/allos.rst b/Doc/library/allos.rst index f7105d8af8e28b..1aed340b2527ac 100644 --- a/Doc/library/allos.rst +++ b/Doc/library/allos.rst @@ -15,15 +15,9 @@ but they are available on most other systems as well. Here's an overview: os.rst io.rst time.rst - argparse.rst - getopt.rst logging.rst logging.config.rst logging.handlers.rst - getpass.rst - curses.rst - curses.ascii.rst - curses.panel.rst platform.rst errno.rst ctypes.rst diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst new file mode 100644 index 00000000000000..dcaff3d7fdbec5 --- /dev/null +++ b/Doc/library/annotationlib.rst @@ -0,0 +1,396 @@ +:mod:`!annotationlib` --- Functionality for introspecting annotations +===================================================================== + +.. module:: annotationlib + :synopsis: Functionality for introspecting annotations + + +**Source code:** :source:`Lib/annotationlib.py` + +.. testsetup:: default + + import annotationlib + from annotationlib import * + +-------------- + +The :mod:`!annotationlib` module provides tools for introspecting +:term:`annotations ` on modules, classes, and functions. + +Annotations are :ref:`lazily evaluated ` and often contain +forward references to objects that are not yet defined when the annotation +is created. This module provides a set of low-level tools that can be used to retrieve annotations in a reliable way, even +in the presence of forward references and other edge cases. + +This module supports retrieving annotations in three main formats +(see :class:`Format`), each of which works best for different use cases: + +* :attr:`~Format.VALUE` evaluates the annotations and returns their value. + This is most straightforward to work with, but it may raise errors, + for example if the annotations contain references to undefined names. +* :attr:`~Format.FORWARDREF` returns :class:`ForwardRef` objects + for annotations that cannot be resolved, allowing you to inspect the + annotations without evaluating them. This is useful when you need to + work with annotations that may contain unresolved forward references. +* :attr:`~Format.STRING` returns the annotations as a string, similar + to how it would appear in the source file. This is useful for documentation + generators that want to display annotations in a readable way. + +The :func:`get_annotations` function is the main entry point for +retrieving annotations. Given a function, class, or module, it returns +an annotations dictionary in the requested format. This module also provides +functionality for working directly with the :term:`annotate function` +that is used to evaluate annotations, such as :func:`get_annotate_function` +and :func:`call_annotate_function`, as well as the +:func:`call_evaluate_function` function for working with +:term:`evaluate functions `. + + +.. seealso:: + + :pep:`649` proposed the current model for how annotations work in Python. + + :pep:`749` expanded on various aspects of :pep:`649` and introduced the + :mod:`!annotationlib` module. + + :ref:`annotations-howto` provides best practices for working with + annotations. + + :pypi:`typing-extensions` provides a backport of :func:`get_annotations` + that works on earlier versions of Python. + +Annotation semantics +-------------------- + +The way annotations are evaluated has changed over the history of Python 3, +and currently still depends on a :ref:`future import `. +There have been execution models for annotations: + +* *Stock semantics* (default in Python 3.0 through 3.13; see :pep:`3107` + and :pep:`526`): Annotations are evaluated eagerly, as they are + encountered in the source code. +* *Stringified annotations* (used with ``from __future__ import annotations`` + in Python 3.7 and newer; see :pep:`563`): Annotations are stored as + strings only. +* *Deferred evaluation* (default in Python 3.14 and newer; see :pep:`649` and + :pep:`749`): Annotations are evaluated lazily, only when they are accessed. + +As an example, consider the following program:: + + def func(a: Cls) -> None: + print(a) + + class Cls: pass + + print(func.__annotations__) + +This will behave as follows: + +* Under stock semantics (Python 3.13 and earlier), it will throw a + :exc:`NameError` at the line where ``func`` is defined, + because ``Cls`` is an undefined name at that point. +* Under stringified annotations (if ``from __future__ import annotations`` + is used), it will print ``{'a': 'Cls', 'return': 'None'}``. +* Under deferred evaluation (Python 3.14 and later), it will print + ``{'a': , 'return': None}``. + +Stock semantics were used when function annotations were first introduced +in Python 3.0 (by :pep:`3107`) because this was the simplest, most obvious +way to implement annotations. The same execution model was used when variable +annotations were introduced in Python 3.6 (by :pep:`526`). However, +stock semantics caused problems when using annotations as type hints, +such as a need to refer to names that are not yet defined when the +annotation is encountered. In addition, there were performance problems +with executing annotations at module import time. Therefore, in Python 3.7, +:pep:`563` introduced the ability to store annotations as strings using the +``from __future__ import annotations`` syntax. The plan at the time was to +eventually make this behavior the default, but a problem appeared: +stringified annotations are more difficult to process for those who +introspect annotations at runtime. An alternative proposal, :pep:`649`, +introduced the third execution model, deferred evaluation, and was implemented +in Python 3.14. Stringified annotations are still used if +``from __future__ import annotations`` is present, but this behavior will +eventually be removed. + +Classes +------- + +.. class:: Format + + An :class:`~enum.IntEnum` describing the formats in which annotations + can be returned. Members of the enum, or their equivalent integer values, + can be passed to :func:`get_annotations` and other functions in this + module, as well as to :attr:`~object.__annotate__` functions. + + .. attribute:: VALUE + :value: 1 + + Values are the result of evaluating the annotation expressions. + + .. attribute:: FORWARDREF + :value: 2 + + Values are real annotation values (as per :attr:`Format.VALUE` format) + for defined values, and :class:`ForwardRef` proxies for undefined + values. Real objects may contain references to, :class:`ForwardRef` + proxy objects. + + .. attribute:: STRING + :value: 3 + + Values are the text string of the annotation as it appears in the + source code, up to modifications including, but not restricted to, + whitespace normalizations and constant values optimizations. + + The exact values of these strings may change in future versions of Python. + + .. attribute:: VALUE_WITH_FAKE_GLOBALS + :value: 4 + + Special value used to signal that an annotate function is being + evaluated in a special environment with fake globals. When passed this + value, annotate functions should either return the same value as for + the :attr:`Format.VALUE` format, or raise :exc:`NotImplementedError` + to signal that they do not support execution in this environment. + This format is only used internally and should not be passed to + the functions in this module. + + .. versionadded:: 3.14 + +.. class:: ForwardRef + + A proxy object for forward references in annotations. + + Instances of this class are returned when the :attr:`~Format.FORWARDREF` + format is used and annotations contain a name that cannot be resolved. + This can happen when a forward reference is used in an annotation, such as + when a class is referenced before it is defined. + + .. attribute:: __forward_arg__ + + A string containing the code that was evaluated to produce the + :class:`~ForwardRef`. The string may not be exactly equivalent + to the original source. + + .. method:: evaluate(*, globals=None, locals=None, type_params=None, owner=None) + + Evaluate the forward reference, returning its value. + + This may throw an exception, such as :exc:`NameError`, if the forward + reference refers to names that do not exist. The arguments to this + method can be used to provide bindings for names that would otherwise + be undefined. + + :class:`~ForwardRef` instances returned by :func:`get_annotations` + retain references to information about the scope they originated from, + so calling this method with no further arguments may be sufficient to + evaluate such objects. :class:`~ForwardRef` instances created by other + means may not have any information about their scope, so passing + arguments to this method may be necessary to evaluate them successfully. + + *globals* and *locals* are passed to :func:`eval`, representing + the global and local namespaces in which the name is evaluated. + *type_params*, if given, must be a tuple of + :ref:`type parameters ` that are in scope while the forward + reference is being evaluated. *owner* is the object that owns the + annotation from which the forward reference derives, usually a function, + class, or module. + + .. important:: + + Once a :class:`~ForwardRef` instance has been evaluated, it caches + the evaluated value, and future calls to :meth:`evaluate` will return + the cached value, regardless of the parameters passed in. + + .. versionadded:: 3.14 + + +Functions +--------- + +.. function:: annotations_to_string(annotations) + + Convert an annotations dict containing runtime values to a + dict containing only strings. If the values are not already strings, + they are converted using :func:`value_to_string`. + This is meant as a helper for user-provided + annotate functions that support the :attr:`~Format.STRING` format but + do not have access to the code creating the annotations. + + For example, this is used to implement the :attr:`~Format.STRING` for + :class:`typing.TypedDict` classes created through the functional syntax: + + .. doctest:: + + >>> from typing import TypedDict + >>> Movie = TypedDict("movie", {"name": str, "year": int}) + >>> get_annotations(Movie, format=Format.STRING) + {'name': 'str', 'year': 'int'} + + .. versionadded:: 3.14 + +.. function:: call_annotate_function(annotate, format, *, owner=None) + + Call the :term:`annotate function` *annotate* with the given *format*, + a member of the :class:`Format` enum, and return the annotations + dictionary produced by the function. + + This helper function is required because annotate functions generated by + the compiler for functions, classes, and modules only support + the :attr:`~Format.VALUE` format when called directly. + To support other formats, this function calls the annotate function + in a special environment that allows it to produce annotations in the + other formats. This is a useful building block when implementing + functionality that needs to partially evaluate annotations while a class + is being constructed. + + *owner* is the object that owns the annotation function, usually + a function, class, or module. If provided, it is used in the + :attr:`~Format.FORWARDREF` format to produce a :class:`ForwardRef` + object that carries more information. + + .. seealso:: + + :PEP:`PEP 649 <649#the-stringizer-and-the-fake-globals-environment>` + contains an explanation of the implementation technique used by this + function. + + .. versionadded:: 3.14 + +.. function:: call_evaluate_function(evaluate, format, *, owner=None) + + Call the :term:`evaluate function` *evaluate* with the given *format*, + a member of the :class:`Format` enum, and return the value produced by + the function. This is similar to :func:`call_annotate_function`, + but the latter always returns a dictionary mapping strings to annotations, + while this function returns a single value. + + This is intended for use with the evaluate functions generated for lazily + evaluated elements related to type aliases and type parameters: + + * :meth:`typing.TypeAliasType.evaluate_value`, the value of type aliases + * :meth:`typing.TypeVar.evaluate_bound`, the bound of type variables + * :meth:`typing.TypeVar.evaluate_constraints`, the constraints of + type variables + * :meth:`typing.TypeVar.evaluate_default`, the default value of + type variables + * :meth:`typing.ParamSpec.evaluate_default`, the default value of + parameter specifications + * :meth:`typing.TypeVarTuple.evaluate_default`, the default value of + type variable tuples + + *owner* is the object that owns the evaluate function, such as the type + alias or type variable object. + + *format* can be used to control the format in which the value is returned: + + .. doctest:: + + >>> type Alias = undefined + >>> call_evaluate_function(Alias.evaluate_value, Format.VALUE) + Traceback (most recent call last): + ... + NameError: name 'undefined' is not defined + >>> call_evaluate_function(Alias.evaluate_value, Format.FORWARDREF) + ForwardRef('undefined') + >>> call_evaluate_function(Alias.evaluate_value, Format.STRING) + 'undefined' + + .. versionadded:: 3.14 + +.. function:: get_annotate_function(obj) + + Retrieve the :term:`annotate function` for *obj*. Return :const:`!None` + if *obj* does not have an annotate function. + + This is usually equivalent to accessing the :attr:`~object.__annotate__` + attribute of *obj*, but direct access to the attribute may return the wrong + object in certain situations involving metaclasses. This function should be + used instead of accessing the attribute directly. + + .. versionadded:: 3.14 + +.. function:: get_annotations(obj, *, globals=None, locals=None, eval_str=False, format=Format.VALUE) + + Compute the annotations dict for an object. + + *obj* may be a callable, class, module, or other object with + :attr:`~object.__annotate__` and :attr:`~object.__annotations__` attributes. + Passing in an object of any other type raises :exc:`TypeError`. + + The *format* parameter controls the format in which annotations are returned, + and must be a member of the :class:`Format` enum or its integer equivalent. + + Returns a dict. :func:`!get_annotations` returns a new dict every time + it's called; calling it twice on the same object will return two + different but equivalent dicts. + + This function handles several details for you: + + * If *eval_str* is true, values of type :class:`!str` will + be un-stringized using :func:`eval`. This is intended + for use with stringized annotations + (``from __future__ import annotations``). It is an error + to set *eval_str* to true with formats other than :attr:`Format.VALUE`. + * If *obj* doesn't have an annotations dict, returns an + empty dict. (Functions and methods always have an + annotations dict; classes, modules, and other types of + callables may not.) + * Ignores inherited annotations on classes, as well as annotations + on metaclasses. If a class + doesn't have its own annotations dict, returns an empty dict. + * All accesses to object members and dict values are done + using ``getattr()`` and ``dict.get()`` for safety. + + *eval_str* controls whether or not values of type :class:`!str` are + replaced with the result of calling :func:`eval` on those values: + + * If eval_str is true, :func:`eval` is called on values of type + :class:`!str`. (Note that :func:`!get_annotations` doesn't catch + exceptions; if :func:`eval` raises an exception, it will unwind + the stack past the :func:`!get_annotations` call.) + * If *eval_str* is false (the default), values of type :class:`!str` are + unchanged. + + *globals* and *locals* are passed in to :func:`eval`; see the documentation + for :func:`eval` for more information. If *globals* or *locals* + is :const:`!None`, this function may replace that value with a + context-specific default, contingent on ``type(obj)``: + + * If *obj* is a module, *globals* defaults to ``obj.__dict__``. + * If *obj* is a class, *globals* defaults to + ``sys.modules[obj.__module__].__dict__`` and *locals* defaults + to the *obj* class namespace. + * If *obj* is a callable, *globals* defaults to + :attr:`obj.__globals__ `, + although if *obj* is a wrapped function (using + :func:`functools.update_wrapper`) or a :class:`functools.partial` object, + it is unwrapped until a non-wrapped function is found. + + Calling :func:`!get_annotations` is best practice for accessing the + annotations dict of any object. See :ref:`annotations-howto` for + more information on annotations best practices. + + .. doctest:: + + >>> def f(a: int, b: str) -> float: + ... pass + >>> get_annotations(f) + {'a': , 'b': , 'return': } + + .. versionadded:: 3.14 + +.. function:: value_to_string(value) + + Convert an arbitrary Python value to a format suitable for use by the + :attr:`~Format.STRING` format. This calls :func:`repr` for most + objects, but has special handling for some objects, such as type objects. + + This is meant as a helper for user-provided + annotate functions that support the :attr:`~Format.STRING` format but + do not have access to the code creating the annotations. It can also + be used to provide a user-friendly string representation for other + objects that contain values that are commonly encountered in annotations. + + .. versionadded:: 3.14 + diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index eaddd44e2defd7..8d0116d8c060b8 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -1,5 +1,5 @@ -:mod:`argparse` --- Parser for command-line options, arguments and sub-commands -=============================================================================== +:mod:`!argparse` --- Parser for command-line options, arguments and subcommands +================================================================================ .. module:: argparse :synopsis: Command-line option and argument parsing library. @@ -11,6 +11,18 @@ **Source code:** :source:`Lib/argparse.py` +.. note:: + + While :mod:`argparse` is the default recommended standard library module + for implementing basic command line applications, authors with more + exacting requirements for exactly how their command line applications + behave may find it doesn't provide the necessary level of control. + Refer to :ref:`choosing-an-argument-parser` for alternatives to + consider when ``argparse`` doesn't support behaviors that the application + requires (such as entirely disabling support for interspersed options and + positional arguments, or accepting option parameter values that start + with ``-`` even when they correspond to another defined option). + -------------- .. sidebar:: Tutorial @@ -19,17 +31,13 @@ introduction to Python command-line parsing, have a look at the :ref:`argparse tutorial `. -The :mod:`argparse` module makes it easy to write user-friendly command-line -interfaces. The program defines what arguments it requires, and :mod:`argparse` -will figure out how to parse those out of :data:`sys.argv`. The :mod:`argparse` +The :mod:`!argparse` module makes it easy to write user-friendly command-line +interfaces. The program defines what arguments it requires, and :mod:`!argparse` +will figure out how to parse those out of :data:`sys.argv`. The :mod:`!argparse` module also automatically generates help and usage messages. The module will also issue errors when users give the program invalid arguments. - -Core Functionality ------------------- - -The :mod:`argparse` module's support for command-line interfaces is built +The :mod:`!argparse` module's support for command-line interfaces is built around an instance of :class:`argparse.ArgumentParser`. It is a container for argument specifications and has options that apply to the parser as whole:: @@ -53,133 +61,9 @@ the extracted data in a :class:`argparse.Namespace` object:: args = parser.parse_args() print(args.filename, args.count, args.verbose) - -Quick Links for add_argument() ------------------------------- - -============================ =========================================================== ========================================================================================================================== -Name Description Values -============================ =========================================================== ========================================================================================================================== -action_ Specify how an argument should be handled ``'store'``, ``'store_const'``, ``'store_true'``, ``'append'``, ``'append_const'``, ``'count'``, ``'help'``, ``'version'`` -choices_ Limit values to a specific set of choices ``['foo', 'bar']``, ``range(1, 10)``, or :class:`~collections.abc.Container` instance -const_ Store a constant value -default_ Default value used when an argument is not provided Defaults to ``None`` -dest_ Specify the attribute name used in the result namespace -help_ Help message for an argument -metavar_ Alternate display name for the argument as shown in help -nargs_ Number of times the argument can be used :class:`int`, ``'?'``, ``'*'``, or ``'+'`` -required_ Indicate whether an argument is required or optional ``True`` or ``False`` -:ref:`type ` Automatically convert an argument to the given type :class:`int`, :class:`float`, ``argparse.FileType('w')``, or callable function -============================ =========================================================== ========================================================================================================================== - - -Example -------- - -The following code is a Python program that takes a list of integers and -produces either the sum or the max:: - - import argparse - - parser = argparse.ArgumentParser(description='Process some integers.') - parser.add_argument('integers', metavar='N', type=int, nargs='+', - help='an integer for the accumulator') - parser.add_argument('--sum', dest='accumulate', action='store_const', - const=sum, default=max, - help='sum the integers (default: find the max)') - - args = parser.parse_args() - print(args.accumulate(args.integers)) - -Assuming the above Python code is saved into a file called ``prog.py``, it can -be run at the command line and it provides useful help messages: - -.. code-block:: shell-session - - $ python prog.py -h - usage: prog.py [-h] [--sum] N [N ...] - - Process some integers. - - positional arguments: - N an integer for the accumulator - - options: - -h, --help show this help message and exit - --sum sum the integers (default: find the max) - -When run with the appropriate arguments, it prints either the sum or the max of -the command-line integers: - -.. code-block:: shell-session - - $ python prog.py 1 2 3 4 - 4 - - $ python prog.py 1 2 3 4 --sum - 10 - -If invalid arguments are passed in, an error will be displayed: - -.. code-block:: shell-session - - $ python prog.py a b c - usage: prog.py [-h] [--sum] N [N ...] - prog.py: error: argument N: invalid int value: 'a' - -The following sections walk you through this example. - - -Creating a parser -^^^^^^^^^^^^^^^^^ - -The first step in using the :mod:`argparse` is creating an -:class:`ArgumentParser` object:: - - >>> parser = argparse.ArgumentParser(description='Process some integers.') - -The :class:`ArgumentParser` object will hold all the information necessary to -parse the command line into Python data types. - - -Adding arguments -^^^^^^^^^^^^^^^^ - -Filling an :class:`ArgumentParser` with information about program arguments is -done by making calls to the :meth:`~ArgumentParser.add_argument` method. -Generally, these calls tell the :class:`ArgumentParser` how to take the strings -on the command line and turn them into objects. This information is stored and -used when :meth:`~ArgumentParser.parse_args` is called. For example:: - - >>> parser.add_argument('integers', metavar='N', type=int, nargs='+', - ... help='an integer for the accumulator') - >>> parser.add_argument('--sum', dest='accumulate', action='store_const', - ... const=sum, default=max, - ... help='sum the integers (default: find the max)') - -Later, calling :meth:`~ArgumentParser.parse_args` will return an object with -two attributes, ``integers`` and ``accumulate``. The ``integers`` attribute -will be a list of one or more integers, and the ``accumulate`` attribute will be -either the :func:`sum` function, if ``--sum`` was specified at the command line, -or the :func:`max` function if it was not. - - -Parsing arguments -^^^^^^^^^^^^^^^^^ - -:class:`ArgumentParser` parses arguments through the -:meth:`~ArgumentParser.parse_args` method. This will inspect the command line, -convert each argument to the appropriate type and then invoke the appropriate action. -In most cases, this means a simple :class:`Namespace` object will be built up from -attributes parsed out of the command line:: - - >>> parser.parse_args(['--sum', '7', '-1', '42']) - Namespace(accumulate=, integers=[7, -1, 42]) - -In a script, :meth:`~ArgumentParser.parse_args` will typically be called with no -arguments, and the :class:`ArgumentParser` will automatically determine the -command-line arguments from :data:`sys.argv`. - +.. note:: + If you're looking for a guide about how to upgrade :mod:`optparse` code + to :mod:`!argparse`, see :ref:`Upgrading Optparse Code `. ArgumentParser objects ---------------------- @@ -189,14 +73,15 @@ ArgumentParser objects formatter_class=argparse.HelpFormatter, \ prefix_chars='-', fromfile_prefix_chars=None, \ argument_default=None, conflict_handler='error', \ - add_help=True, allow_abbrev=True, exit_on_error=True) + add_help=True, allow_abbrev=True, exit_on_error=True, \ + suggest_on_error=False) Create a new :class:`ArgumentParser` object. All parameters should be passed as keyword arguments. Each parameter has its own more detailed description below, but in short they are: - * prog_ - The name of the program (default: - ``os.path.basename(sys.argv[0])``) + * prog_ - The name of the program (default: generated from the ``__main__`` + module attributes and ``sys.argv[0]``) * usage_ - The string describing the program usage (default: generated from arguments added to parser) @@ -228,9 +113,13 @@ ArgumentParser objects * allow_abbrev_ - Allows long options to be abbreviated if the abbreviation is unambiguous. (default: ``True``) - * exit_on_error_ - Determines whether or not ArgumentParser exits with + * exit_on_error_ - Determines whether or not :class:`!ArgumentParser` exits with error info when an error occurs. (default: ``True``) + * suggest_on_error_ - Enables suggestions for mistyped argument choices + and subparser names (default: ``False``) + + .. versionchanged:: 3.5 *allow_abbrev* parameter was added. @@ -249,38 +138,21 @@ The following sections describe how each of these are used. prog ^^^^ -By default, :class:`ArgumentParser` objects use ``sys.argv[0]`` to determine -how to display the name of the program in help messages. This default is almost -always desirable because it will make the help messages match how the program was -invoked on the command line. For example, consider a file named -``myprogram.py`` with the following code:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument('--foo', help='foo help') - args = parser.parse_args() - -The help for this program will display ``myprogram.py`` as the program name -(regardless of where the program was invoked from): -.. code-block:: shell-session - - $ python myprogram.py --help - usage: myprogram.py [-h] [--foo FOO] - - options: - -h, --help show this help message and exit - --foo FOO foo help - $ cd .. - $ python subdir/myprogram.py --help - usage: myprogram.py [-h] [--foo FOO] +By default, :class:`ArgumentParser` calculates the name of the program +to display in help messages depending on the way the Python interpreter was run: - options: - -h, --help show this help message and exit - --foo FOO foo help +* The :func:`base name ` of ``sys.argv[0]`` if a file was + passed as argument. +* The Python interpreter name followed by ``sys.argv[0]`` if a directory or + a zipfile was passed as argument. +* The Python interpreter name followed by ``-m`` followed by the + module or package name if the :option:`-m` option was used. -To change this default behavior, another value can be supplied using the -``prog=`` argument to :class:`ArgumentParser`:: +This default is almost always desirable because it will make the help messages +match the string that was used to invoke the program on the command line. +However, to change this default behavior, another value can be supplied using +the ``prog=`` argument to :class:`ArgumentParser`:: >>> parser = argparse.ArgumentParser(prog='myprogram') >>> parser.print_help() @@ -289,7 +161,8 @@ To change this default behavior, another value can be supplied using the options: -h, --help show this help message and exit -Note that the program name, whether determined from ``sys.argv[0]`` or from the +Note that the program name, whether determined from ``sys.argv[0]``, +from the ``__main__`` module attributes or from the ``prog=`` argument, is available to help messages using the ``%(prog)s`` format specifier. @@ -304,27 +177,16 @@ specifier. -h, --help show this help message and exit --foo FOO foo of the myprogram program +.. versionchanged:: 3.14 + The default ``prog`` value now reflects how ``__main__`` was actually executed, + rather than always being ``os.path.basename(sys.argv[0])``. usage ^^^^^ By default, :class:`ArgumentParser` calculates the usage message from the -arguments it contains:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('--foo', nargs='?', help='foo help') - >>> parser.add_argument('bar', nargs='+', help='bar help') - >>> parser.print_help() - usage: PROG [-h] [--foo [FOO]] bar [bar ...] - - positional arguments: - bar bar help - - options: - -h, --help show this help message and exit - --foo [FOO] foo help - -The default message can be overridden with the ``usage=`` keyword argument:: +arguments it contains. The default message can be overridden with the +``usage=`` keyword argument:: >>> parser = argparse.ArgumentParser(prog='PROG', usage='%(prog)s [options]') >>> parser.add_argument('--foo', nargs='?', help='foo help') @@ -342,6 +204,12 @@ The default message can be overridden with the ``usage=`` keyword argument:: The ``%(prog)s`` format specifier is available to fill in the program name in your usage messages. +When a custom usage message is specified for the main parser, you may also want to +consider passing the ``prog`` argument to :meth:`~ArgumentParser.add_subparsers` +or the ``prog`` and the ``usage`` arguments to +:meth:`~_SubParsersAction.add_parser`, to ensure consistent command prefixes and +usage information across subparsers. + .. _description: @@ -352,16 +220,7 @@ Most calls to the :class:`ArgumentParser` constructor will use the ``description=`` keyword argument. This argument gives a brief description of what the program does and how it works. In help messages, the description is displayed between the command-line usage string and the help messages for the -various arguments:: - - >>> parser = argparse.ArgumentParser(description='A foo that bars') - >>> parser.print_help() - usage: argparse.py [-h] - - A foo that bars - - options: - -h, --help show this help message and exit +various arguments. By default, the description will be line-wrapped so that it fits within the given space. To change this behavior, see the formatter_class_ argument. @@ -491,7 +350,7 @@ should not be line-wrapped:: -h, --help show this help message and exit :class:`RawTextHelpFormatter` maintains whitespace for all sorts of help text, -including argument descriptions. However, multiple new lines are replaced with +including argument descriptions. However, multiple newlines are replaced with one. If you wish to preserve multiple blank lines, add spaces between the newlines. @@ -540,7 +399,7 @@ Most command-line options will use ``-`` as the prefix, e.g. ``-f/--foo``. Parsers that need to support different or additional prefix characters, e.g. for options like ``+f`` or ``/foo``, may specify them using the ``prefix_chars=`` argument -to the ArgumentParser constructor:: +to the :class:`ArgumentParser` constructor:: >>> parser = argparse.ArgumentParser(prog='PROG', prefix_chars='-+') >>> parser.add_argument('+f') @@ -585,8 +444,8 @@ arguments will never be treated as file references. .. versionchanged:: 3.12 :class:`ArgumentParser` changed encoding and errors to read arguments files - from default (e.g. :func:`locale.getpreferredencoding(False) ` and - ``"strict"``) to :term:`filesystem encoding and error handler`. + from default (e.g. :func:`locale.getpreferredencoding(False) ` + and ``"strict"``) to the :term:`filesystem encoding and error handler`. Arguments file should be encoded in UTF-8 instead of ANSI Codepage on Windows. @@ -671,26 +530,9 @@ string was overridden. add_help ^^^^^^^^ -By default, ArgumentParser objects add an option which simply displays -the parser's help message. For example, consider a file named -``myprogram.py`` containing the following code:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument('--foo', help='foo help') - args = parser.parse_args() - -If ``-h`` or ``--help`` is supplied at the command line, the ArgumentParser -help will be printed: - -.. code-block:: shell-session - - $ python myprogram.py --help - usage: myprogram.py [-h] [--foo FOO] - - options: - -h, --help show this help message and exit - --foo FOO foo help +By default, :class:`ArgumentParser` objects add an option which simply displays +the parser's help message. If ``-h`` or ``--help`` is supplied at the command +line, the :class:`!ArgumentParser` help will be printed. Occasionally, it may be useful to disable the addition of this help option. This can be achieved by passing ``False`` as the ``add_help=`` argument to @@ -722,7 +564,8 @@ exit_on_error ^^^^^^^^^^^^^ Normally, when you pass an invalid argument list to the :meth:`~ArgumentParser.parse_args` -method of an :class:`ArgumentParser`, it will exit with error info. +method of an :class:`ArgumentParser`, it will print a *message* to :data:`sys.stderr` and exit with a status +code of 2. If the user would like to catch errors manually, the feature can be enabled by setting ``exit_on_error`` to ``False``:: @@ -739,19 +582,48 @@ If the user would like to catch errors manually, the feature can be enabled by s .. versionadded:: 3.9 +suggest_on_error +^^^^^^^^^^^^^^^^ + +By default, when a user passes an invalid argument choice or subparser name, +:class:`ArgumentParser` will exit with error info and list the permissible +argument choices (if specified) or subparser names as part of the error message. + +If the user would like to enable suggestions for mistyped argument choices and +subparser names, the feature can be enabled by setting ``suggest_on_error`` to +``True``. Note that this only applies for arguments when the choices specified +are strings:: + + >>> parser = argparse.ArgumentParser(description='Process some integers.', suggest_on_error=True) + >>> parser.add_argument('--action', choices=['sum', 'max']) + >>> parser.add_argument('integers', metavar='N', type=int, nargs='+', + ... help='an integer for the accumulator') + >>> parser.parse_args(['--action', 'sumn', 1, 2, 3]) + tester.py: error: argument --action: invalid choice: 'sumn', maybe you meant 'sum'? (choose from 'sum', 'max') + +If you're writing code that needs to be compatible with older Python versions +and want to opportunistically use ``suggest_on_error`` when it's available, you +can set it as an attribute after initializing the parser instead of using the +keyword argument:: + + >>> parser = argparse.ArgumentParser(description='Process some integers.') + >>> parser.suggest_on_error = True + +.. versionadded:: 3.14 + The add_argument() method ------------------------- -.. method:: ArgumentParser.add_argument(name or flags..., [action], [nargs], \ +.. method:: ArgumentParser.add_argument(name or flags..., *, [action], [nargs], \ [const], [default], [type], [choices], [required], \ [help], [metavar], [dest], [deprecated]) Define how a single command-line argument should be parsed. Each parameter has its own more detailed description below, but in short they are: - * `name or flags`_ - Either a name or a list of option strings, e.g. ``foo`` - or ``-f, --foo``. + * `name or flags`_ - Either a name or a list of option strings, e.g. ``'foo'`` + or ``'-f', '--foo'``. * action_ - The basic type of action to be taken when this argument is encountered at the command line. @@ -782,7 +654,7 @@ The add_argument() method The following sections describe how each of these are used. -.. _name_or_flags: +.. _`name or flags`: name or flags ^^^^^^^^^^^^^ @@ -816,6 +688,25 @@ be positional:: usage: PROG [-h] [-f FOO] bar PROG: error: the following arguments are required: bar +By default, :mod:`!argparse` automatically handles the internal naming and +display names of arguments, simplifying the process without requiring +additional configuration. +As such, you do not need to specify the dest_ and metavar_ parameters. +The dest_ parameter defaults to the argument name with underscores ``_`` +replacing hyphens ``-`` . The metavar_ parameter defaults to the +upper-cased name. For example:: + + >>> parser = argparse.ArgumentParser(prog='PROG') + >>> parser.add_argument('--foo-bar') + >>> parser.parse_args(['--foo-bar', 'FOO-BAR'] + Namespace(foo_bar='FOO-BAR') + >>> parser.print_help() + usage: [-h] [--foo-bar FOO-BAR] + + optional arguments: + -h, --help show this help message and exit + --foo-bar FOO-BAR + .. _action: @@ -829,12 +720,7 @@ them, though most actions simply add an attribute to the object returned by how the command-line arguments should be handled. The supplied actions are: * ``'store'`` - This just stores the argument's value. This is the default - action. For example:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo') - >>> parser.parse_args('--foo 1'.split()) - Namespace(foo='1') + action. * ``'store_const'`` - This stores the value specified by the const_ keyword argument; note that the const_ keyword argument defaults to ``None``. The @@ -849,7 +735,7 @@ how the command-line arguments should be handled. The supplied actions are: * ``'store_true'`` and ``'store_false'`` - These are special cases of ``'store_const'`` used for storing the values ``True`` and ``False`` respectively. In addition, they create default values of ``False`` and - ``True`` respectively. For example:: + ``True`` respectively:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', action='store_true') @@ -881,6 +767,21 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args('--str --int'.split()) Namespace(types=[, ]) +* ``'extend'`` - This stores a list and appends each item from the multi-value + argument list to it. + The ``'extend'`` action is typically used with the nargs_ keyword argument + value ``'+'`` or ``'*'``. + Note that when nargs_ is ``None`` (the default) or ``'?'``, each + character of the argument string will be appended to the list. + Example usage:: + + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument("--foo", action="extend", nargs="+", type=str) + >>> parser.parse_args(["--foo", "f1", "--foo", "f2", "f3", "f4"]) + Namespace(foo=['f1', 'f2', 'f3', 'f4']) + + .. versionadded:: 3.8 + * ``'count'`` - This counts the number of times a keyword argument occurs. For example, this is useful for increasing verbosity levels:: @@ -906,33 +807,28 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args(['--version']) PROG 2.0 -* ``'extend'`` - This stores a list, and extends each argument value to the - list. - Example usage:: +Only actions that consume command-line arguments (e.g. ``'store'``, +``'append'`` or ``'extend'``) can be used with positional arguments. - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument("--foo", action="extend", nargs="+", type=str) - >>> parser.parse_args(["--foo", "f1", "--foo", "f2", "f3", "f4"]) - Namespace(foo=['f1', 'f2', 'f3', 'f4']) +.. class:: BooleanOptionalAction - .. versionadded:: 3.8 + You may also specify an arbitrary action by passing an :class:`Action` subclass or + other object that implements the same interface. The :class:`!BooleanOptionalAction` + is available in :mod:`!argparse` and adds support for boolean actions such as + ``--foo`` and ``--no-foo``:: -You may also specify an arbitrary action by passing an Action subclass or -other object that implements the same interface. The ``BooleanOptionalAction`` -is available in ``argparse`` and adds support for boolean actions such as -``--foo`` and ``--no-foo``:: + >>> import argparse + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo', action=argparse.BooleanOptionalAction) + >>> parser.parse_args(['--no-foo']) + Namespace(foo=False) - >>> import argparse - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action=argparse.BooleanOptionalAction) - >>> parser.parse_args(['--no-foo']) - Namespace(foo=False) - -.. versionadded:: 3.9 + .. versionadded:: 3.9 The recommended way to create a custom action is to extend :class:`Action`, -overriding the ``__call__`` method and optionally the ``__init__`` and -``format_usage`` methods. +overriding the :meth:`!__call__` method and optionally the :meth:`!__init__` and +:meth:`!format_usage` methods. You can also register custom actions using the +:meth:`~ArgumentParser.register` method and reference them by their registered name. An example of a custom action:: @@ -962,7 +858,7 @@ For more details, see :class:`Action`. nargs ^^^^^ -ArgumentParser objects usually associate a single command-line argument with a +:class:`ArgumentParser` objects usually associate a single command-line argument with a single action to be taken. The ``nargs`` keyword argument associates a different number of command-line arguments with a single action. See also :ref:`specifying-ambiguous-arguments`. The supported values are: @@ -1002,16 +898,14 @@ See also :ref:`specifying-ambiguous-arguments`. The supported values are: output files:: >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('infile', nargs='?', type=argparse.FileType('r'), - ... default=sys.stdin) - >>> parser.add_argument('outfile', nargs='?', type=argparse.FileType('w'), - ... default=sys.stdout) + >>> parser.add_argument('infile', nargs='?') + >>> parser.add_argument('outfile', nargs='?') >>> parser.parse_args(['input.txt', 'output.txt']) - Namespace(infile=<_io.TextIOWrapper name='input.txt' encoding='UTF-8'>, - outfile=<_io.TextIOWrapper name='output.txt' encoding='UTF-8'>) + Namespace(infile='input.txt', outfile='output.txt') + >>> parser.parse_args(['input.txt']) + Namespace(infile='input.txt', outfile=None) >>> parser.parse_args([]) - Namespace(infile=<_io.TextIOWrapper name='' encoding='UTF-8'>, - outfile=<_io.TextIOWrapper name='' encoding='UTF-8'>) + Namespace(infile=None, outfile=None) .. index:: single: * (asterisk); in argparse module @@ -1044,6 +938,8 @@ See also :ref:`specifying-ambiguous-arguments`. The supported values are: If the ``nargs`` keyword argument is not provided, the number of arguments consumed is determined by the action_. Generally this means a single command-line argument will be consumed and a single item (not a list) will be produced. +Actions that do not consume command-line arguments (e.g. +``'store_const'``) set ``nargs=0``. .. _const: @@ -1094,7 +990,7 @@ was not present at the command line:: Namespace(foo=42) If the target namespace already has an attribute set, the action *default* -will not over write it:: +will not overwrite it:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', default=42) @@ -1122,6 +1018,9 @@ is used when no command-line argument was present:: >>> parser.parse_args([]) Namespace(foo=42) +For required_ arguments, the ``default`` value is ignored. For example, this +applies to positional arguments with nargs_ values other than ``?`` or ``*``, +or optional arguments marked as ``required=True``. Providing ``default=argparse.SUPPRESS`` causes no attribute to be added if the command-line argument was not present:: @@ -1148,10 +1047,11 @@ necessary type-checking and type conversions to be performed. If the type_ keyword is used with the default_ keyword, the type converter is only applied if the default is a string. -The argument to ``type`` can be any callable that accepts a single string. +The argument to ``type`` can be a callable that accepts a single string or +the name of a registered type (see :meth:`~ArgumentParser.register`) If the function raises :exc:`ArgumentTypeError`, :exc:`TypeError`, or :exc:`ValueError`, the exception is caught and a nicely formatted error -message is displayed. No other exception types are handled. +message is displayed. Other exception types are not handled. Common built-in types and functions can be used as type converters: @@ -1165,8 +1065,6 @@ Common built-in types and functions can be used as type converters: parser.add_argument('distance', type=float) parser.add_argument('street', type=ascii) parser.add_argument('code_point', type=ord) - parser.add_argument('source_file', type=open) - parser.add_argument('dest_file', type=argparse.FileType('w', encoding='latin-1')) parser.add_argument('datapath', type=pathlib.Path) User defined functions can be used as well: @@ -1196,10 +1094,11 @@ better reporting than can be given by the ``type`` keyword. A :exc:`FileNotFoundError` exception would not be handled at all. Even :class:`~argparse.FileType` has its limitations for use with the ``type`` -keyword. If one argument uses *FileType* and then a subsequent argument fails, -an error is reported but the file is not automatically closed. In this case, it -would be better to wait until after the parser has run and then use the -:keyword:`with`-statement to manage the files. +keyword. If one argument uses :class:`~argparse.FileType` and then a +subsequent argument fails, an error is reported but the file is not +automatically closed. In this case, it would be better to wait until after +the parser has run and then use the :keyword:`with`-statement to manage the +files. For type checkers that simply check against a fixed set of values, consider using the choices_ keyword instead. @@ -1227,15 +1126,7 @@ if the argument was not one of the acceptable values:: Note that inclusion in the *choices* sequence is checked after any type_ conversions have been performed, so the type of the objects in the *choices* -sequence should match the type_ specified:: - - >>> parser = argparse.ArgumentParser(prog='doors.py') - >>> parser.add_argument('door', type=int, choices=range(1, 4)) - >>> print(parser.parse_args(['3'])) - Namespace(door=3) - >>> parser.parse_args(['4']) - usage: doors.py [-h] {1,2,3} - doors.py: error: argument door: invalid choice: 4 (choose from 1, 2, 3) +sequence should match the type_ specified. Any sequence can be passed as the *choices* value, so :class:`list` objects, :class:`tuple` objects, and custom sequences are all supported. @@ -1254,7 +1145,7 @@ many choices), just specify an explicit metavar_. required ^^^^^^^^ -In general, the :mod:`argparse` module assumes that flags like ``-f`` and ``--bar`` +In general, the :mod:`!argparse` module assumes that flags like ``-f`` and ``--bar`` indicate *optional* arguments, which can always be omitted at the command line. To make an option *required*, ``True`` can be specified for the ``required=`` keyword argument to :meth:`~ArgumentParser.add_argument`:: @@ -1285,22 +1176,7 @@ help The ``help`` value is a string containing a brief description of the argument. When a user requests help (usually by using ``-h`` or ``--help`` at the command line), these ``help`` descriptions will be displayed with each -argument:: - - >>> parser = argparse.ArgumentParser(prog='frobble') - >>> parser.add_argument('--foo', action='store_true', - ... help='foo the bars before frobbling') - >>> parser.add_argument('bar', nargs='+', - ... help='one of the bars to be frobbled') - >>> parser.parse_args(['-h']) - usage: frobble [-h] [--foo] bar [bar ...] - - positional arguments: - bar one of the bars to be frobbled - - options: - -h, --help show this help message and exit - --foo foo the bars before frobbling +argument. The ``help`` strings can include various format specifiers to avoid repetition of things like the program name or the argument default_. The available @@ -1322,7 +1198,7 @@ specifiers include the program name, ``%(prog)s`` and most keyword arguments to As the help string supports %-formatting, if you want a literal ``%`` to appear in the help string, you must escape it as ``%%``. -:mod:`argparse` supports silencing the help entry for certain options, by +:mod:`!argparse` supports silencing the help entry for certain options, by setting the ``help`` value to ``argparse.SUPPRESS``:: >>> parser = argparse.ArgumentParser(prog='frobble') @@ -1340,7 +1216,7 @@ metavar ^^^^^^^ When :class:`ArgumentParser` generates help messages, it needs some way to refer -to each expected argument. By default, ArgumentParser objects use the dest_ +to each expected argument. By default, :class:`!ArgumentParser` objects use the dest_ value as the "name" of each object. By default, for positional argument actions, the dest_ value is used directly, and for optional argument actions, the dest_ value is uppercased. So, a single positional argument with @@ -1455,7 +1331,7 @@ The ``deprecated`` keyword argument of specifies if the argument is deprecated and will be removed in the future. For arguments, if ``deprecated`` is ``True``, then a warning will be -printed to standard error when the argument is used:: +printed to :data:`sys.stderr` when the argument is used:: >>> import argparse >>> parser = argparse.ArgumentParser(prog='snake.py') @@ -1466,13 +1342,13 @@ printed to standard error when the argument is used:: snake.py: warning: option '--legs' is deprecated Namespace(legs=4) -.. versionchanged:: 3.13 +.. versionadded:: 3.13 Action classes ^^^^^^^^^^^^^^ -Action classes implement the Action API, a callable which returns a callable +:class:`!Action` classes implement the Action API, a callable which returns a callable which processes arguments from the command-line. Any object which follows this API may be passed as the ``action`` parameter to :meth:`~ArgumentParser.add_argument`. @@ -1481,40 +1357,46 @@ this API may be passed as the ``action`` parameter to type=None, choices=None, required=False, help=None, \ metavar=None) -Action objects are used by an ArgumentParser to represent the information -needed to parse a single argument from one or more strings from the -command line. The Action class must accept the two positional arguments -plus any keyword arguments passed to :meth:`ArgumentParser.add_argument` -except for the ``action`` itself. + :class:`!Action` objects are used by an :class:`ArgumentParser` to represent the information + needed to parse a single argument from one or more strings from the + command line. The :class:`!Action` class must accept the two positional arguments + plus any keyword arguments passed to :meth:`ArgumentParser.add_argument` + except for the ``action`` itself. -Instances of Action (or return value of any callable to the ``action`` -parameter) should have attributes "dest", "option_strings", "default", "type", -"required", "help", etc. defined. The easiest way to ensure these attributes -are defined is to call ``Action.__init__``. + Instances of :class:`!Action` (or return value of any callable to the + ``action`` parameter) should have attributes :attr:`!dest`, + :attr:`!option_strings`, :attr:`!default`, :attr:`!type`, :attr:`!required`, + :attr:`!help`, etc. defined. The easiest way to ensure these attributes + are defined is to call :meth:`!Action.__init__`. -Action instances should be callable, so subclasses must override the -``__call__`` method, which should accept four parameters: + .. method:: __call__(parser, namespace, values, option_string=None) -* ``parser`` - The ArgumentParser object which contains this action. + :class:`!Action` instances should be callable, so subclasses must override the + :meth:`!__call__` method, which should accept four parameters: -* ``namespace`` - The :class:`Namespace` object that will be returned by - :meth:`~ArgumentParser.parse_args`. Most actions add an attribute to this - object using :func:`setattr`. + * *parser* - The :class:`ArgumentParser` object which contains this action. -* ``values`` - The associated command-line arguments, with any type conversions - applied. Type conversions are specified with the type_ keyword argument to - :meth:`~ArgumentParser.add_argument`. + * *namespace* - The :class:`Namespace` object that will be returned by + :meth:`~ArgumentParser.parse_args`. Most actions add an attribute to this + object using :func:`setattr`. -* ``option_string`` - The option string that was used to invoke this action. - The ``option_string`` argument is optional, and will be absent if the action - is associated with a positional argument. + * *values* - The associated command-line arguments, with any type conversions + applied. Type conversions are specified with the type_ keyword argument to + :meth:`~ArgumentParser.add_argument`. -The ``__call__`` method may perform arbitrary actions, but will typically set -attributes on the ``namespace`` based on ``dest`` and ``values``. + * *option_string* - The option string that was used to invoke this action. + The ``option_string`` argument is optional, and will be absent if the action + is associated with a positional argument. + + The :meth:`!__call__` method may perform arbitrary actions, but will typically set + attributes on the ``namespace`` based on ``dest`` and ``values``. + + .. method:: format_usage() + + :class:`!Action` subclasses can define a :meth:`!format_usage` method that takes no argument + and return a string which will be used when printing the usage of the program. + If such method is not provided, a sensible default will be used. -Action subclasses can define a ``format_usage`` method that takes no argument -and return a string which will be used when printing the usage of the program. -If such method is not provided, a sensible default will be used. The parse_args() method ----------------------- @@ -1526,7 +1408,7 @@ The parse_args() method Previous calls to :meth:`add_argument` determine exactly what objects are created and how they are assigned. See the documentation for - :meth:`add_argument` for details. + :meth:`!add_argument` for details. * args_ - List of strings to parse. The default is taken from :data:`sys.argv`. @@ -1682,7 +1564,7 @@ This feature can be disabled by setting :ref:`allow_abbrev` to ``False``. Beyond ``sys.argv`` ^^^^^^^^^^^^^^^^^^^ -Sometimes it may be useful to have an ArgumentParser parse arguments other than those +Sometimes it may be useful to have an :class:`ArgumentParser` parse arguments other than those of :data:`sys.argv`. This can be accomplished by passing a list of strings to :meth:`~ArgumentParser.parse_args`. This is useful for testing at the interactive prompt:: @@ -1709,29 +1591,29 @@ The Namespace object Simple class used by default by :meth:`~ArgumentParser.parse_args` to create an object holding attributes and return it. -This class is deliberately simple, just an :class:`object` subclass with a -readable string representation. If you prefer to have dict-like view of the -attributes, you can use the standard Python idiom, :func:`vars`:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo') - >>> args = parser.parse_args(['--foo', 'BAR']) - >>> vars(args) - {'foo': 'BAR'} + This class is deliberately simple, just an :class:`object` subclass with a + readable string representation. If you prefer to have dict-like view of the + attributes, you can use the standard Python idiom, :func:`vars`:: -It may also be useful to have an :class:`ArgumentParser` assign attributes to an -already existing object, rather than a new :class:`Namespace` object. This can -be achieved by specifying the ``namespace=`` keyword argument:: - - >>> class C: - ... pass - ... - >>> c = C() - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo') - >>> parser.parse_args(args=['--foo', 'BAR'], namespace=c) - >>> c.foo - 'BAR' + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo') + >>> args = parser.parse_args(['--foo', 'BAR']) + >>> vars(args) + {'foo': 'BAR'} + + It may also be useful to have an :class:`ArgumentParser` assign attributes to an + already existing object, rather than a new :class:`Namespace` object. This can + be achieved by specifying the ``namespace=`` keyword argument:: + + >>> class C: + ... pass + ... + >>> c = C() + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo') + >>> parser.parse_args(args=['--foo', 'BAR'], namespace=c) + >>> c.foo + 'BAR' Other utilities @@ -1740,38 +1622,38 @@ Other utilities Sub-commands ^^^^^^^^^^^^ -.. method:: ArgumentParser.add_subparsers([title], [description], [prog], \ +.. method:: ArgumentParser.add_subparsers(*, [title], [description], [prog], \ [parser_class], [action], \ - [option_strings], [dest], [required], \ + [dest], [required], \ [help], [metavar]) - Many programs split up their functionality into a number of sub-commands, - for example, the ``svn`` program can invoke sub-commands like ``svn + Many programs split up their functionality into a number of subcommands, + for example, the ``svn`` program can invoke subcommands like ``svn checkout``, ``svn update``, and ``svn commit``. Splitting up functionality this way can be a particularly good idea when a program performs several different functions which require different kinds of command-line arguments. - :class:`ArgumentParser` supports the creation of such sub-commands with the - :meth:`add_subparsers` method. The :meth:`add_subparsers` method is normally + :class:`ArgumentParser` supports the creation of such subcommands with the + :meth:`!add_subparsers` method. The :meth:`!add_subparsers` method is normally called with no arguments and returns a special action object. This object has a single method, :meth:`~_SubParsersAction.add_parser`, which takes a - command name and any :class:`ArgumentParser` constructor arguments, and - returns an :class:`ArgumentParser` object that can be modified as usual. + command name and any :class:`!ArgumentParser` constructor arguments, and + returns an :class:`!ArgumentParser` object that can be modified as usual. Description of parameters: - * title - title for the sub-parser group in help output; by default + * *title* - title for the sub-parser group in help output; by default "subcommands" if description is provided, otherwise uses title for positional arguments - * description - description for the sub-parser group in help output, by + * *description* - description for the sub-parser group in help output, by default ``None`` - * prog - usage information that will be displayed with sub-command help, + * *prog* - usage information that will be displayed with sub-command help, by default the name of the program and any positional arguments before the subparser argument - * parser_class - class which will be used to create sub-parser instances, by - default the class of the current parser (e.g. ArgumentParser) + * *parser_class* - class which will be used to create sub-parser instances, by + default the class of the current parser (e.g. :class:`ArgumentParser`) * action_ - the basic type of action to be taken when this argument is encountered at the command line @@ -1784,15 +1666,15 @@ Sub-commands * help_ - help for sub-parser group in help output, by default ``None`` - * metavar_ - string presenting available sub-commands in help; by default it - is ``None`` and presents sub-commands in form {cmd1, cmd2, ..} + * metavar_ - string presenting available subcommands in help; by default it + is ``None`` and presents subcommands in form {cmd1, cmd2, ..} Some example usage:: >>> # create the top-level parser >>> parser = argparse.ArgumentParser(prog='PROG') >>> parser.add_argument('--foo', action='store_true', help='foo help') - >>> subparsers = parser.add_subparsers(help='sub-command help') + >>> subparsers = parser.add_subparsers(help='subcommand help') >>> >>> # create the parser for the "a" command >>> parser_a = subparsers.add_parser('a', help='a help') @@ -1800,7 +1682,7 @@ Sub-commands >>> >>> # create the parser for the "b" command >>> parser_b = subparsers.add_parser('b', help='b help') - >>> parser_b.add_argument('--baz', choices='XYZ', help='baz help') + >>> parser_b.add_argument('--baz', choices=('X', 'Y', 'Z'), help='baz help') >>> >>> # parse some argument lists >>> parser.parse_args(['a', '12']) @@ -1827,7 +1709,7 @@ Sub-commands usage: PROG [-h] [--foo] {a,b} ... positional arguments: - {a,b} sub-command help + {a,b} subcommand help a a help b b help @@ -1898,12 +1780,12 @@ Sub-commands .. versionadded:: 3.13 - One particularly effective way of handling sub-commands is to combine the use + One particularly effective way of handling subcommands is to combine the use of the :meth:`add_subparsers` method with calls to :meth:`set_defaults` so that each subparser knows which Python function it should execute. For example:: - >>> # sub-command functions + >>> # subcommand functions >>> def foo(args): ... print(args.x * args.y) ... @@ -1952,7 +1834,11 @@ Sub-commands Namespace(subparser_name='2', y='frobble') .. versionchanged:: 3.7 - New *required* keyword argument. + New *required* keyword-only parameter. + + .. versionchanged:: 3.14 + Subparser's *prog* is no longer affected by a custom usage message in + the main parser. FileType objects @@ -1981,20 +1867,31 @@ FileType objects >>> parser.parse_args(['-']) Namespace(infile=<_io.TextIOWrapper name='' encoding='UTF-8'>) + .. note:: + + If one argument uses *FileType* and then a subsequent argument fails, + an error is reported but the file is not automatically closed. + This can also clobber the output files. + In this case, it would be better to wait until after the parser has + run and then use the :keyword:`with`-statement to manage the files. + .. versionchanged:: 3.4 Added the *encodings* and *errors* parameters. + .. deprecated:: 3.14 + Argument groups ^^^^^^^^^^^^^^^ -.. method:: ArgumentParser.add_argument_group(title=None, description=None) +.. method:: ArgumentParser.add_argument_group(title=None, description=None, *, \ + [argument_default], [conflict_handler]) By default, :class:`ArgumentParser` groups command-line arguments into "positional arguments" and "options" when displaying help messages. When there is a better conceptual grouping of arguments than this default one, appropriate groups can be created using the - :meth:`add_argument_group` method:: + :meth:`!add_argument_group` method:: >>> parser = argparse.ArgumentParser(prog='PROG', add_help=False) >>> group = parser.add_argument_group('group') @@ -2011,7 +1908,7 @@ Argument groups has an :meth:`~ArgumentParser.add_argument` method just like a regular :class:`ArgumentParser`. When an argument is added to the group, the parser treats it just like a normal argument, but displays the argument in a - separate group for help messages. The :meth:`add_argument_group` method + separate group for help messages. The :meth:`!add_argument_group` method accepts *title* and *description* arguments which can be used to customize this display:: @@ -2033,14 +1930,22 @@ Argument groups --bar BAR bar help + The optional, keyword-only parameters argument_default_ and conflict_handler_ + allow for finer-grained control of the behavior of the argument group. These + parameters have the same meaning as in the :class:`ArgumentParser` constructor, + but apply specifically to the argument group rather than the entire parser. + Note that any arguments not in your user-defined groups will end up back in the usual "positional arguments" and "optional arguments" sections. - .. versionchanged:: 3.11 - Calling :meth:`add_argument_group` on an argument group is deprecated. - This feature was never supported and does not always work correctly. - The function exists on the API by accident through inheritance and - will be removed in the future. + .. deprecated-removed:: 3.11 3.14 + Calling :meth:`add_argument_group` on an argument group now raises an + exception. This nesting was never supported, often failed to work + correctly, and was unintentionally exposed through inheritance. + + .. deprecated:: 3.14 + Passing prefix_chars_ to :meth:`add_argument_group` + is now deprecated. Mutual exclusion @@ -2048,7 +1953,7 @@ Mutual exclusion .. method:: ArgumentParser.add_mutually_exclusive_group(required=False) - Create a mutually exclusive group. :mod:`argparse` will make sure that only + Create a mutually exclusive group. :mod:`!argparse` will make sure that only one of the arguments in the mutually exclusive group was present on the command line:: @@ -2099,11 +2004,11 @@ Mutual exclusion --foo FOO foo help --bar BAR bar help - .. versionchanged:: 3.11 - Calling :meth:`add_argument_group` or :meth:`add_mutually_exclusive_group` - on a mutually exclusive group is deprecated. These features were never - supported and do not always work correctly. The functions exist on the - API by accident through inheritance and will be removed in the future. + .. deprecated-removed:: 3.11 3.14 + Calling :meth:`add_argument_group` or :meth:`add_mutually_exclusive_group` + on a mutually exclusive group now raises an exception. This nesting was + never supported, often failed to work correctly, and was unintentionally + exposed through inheritance. Parser defaults @@ -2185,20 +2090,20 @@ Partial parsing .. method:: ArgumentParser.parse_known_args(args=None, namespace=None) -Sometimes a script may only parse a few of the command-line arguments, passing -the remaining arguments on to another script or program. In these cases, the -:meth:`~ArgumentParser.parse_known_args` method can be useful. It works much like -:meth:`~ArgumentParser.parse_args` except that it does not produce an error when -extra arguments are present. Instead, it returns a two item tuple containing -the populated namespace and the list of remaining argument strings. + Sometimes a script may only parse a few of the command-line arguments, passing + the remaining arguments on to another script or program. In these cases, the + :meth:`~ArgumentParser.parse_known_args` method can be useful. It works much like + :meth:`~ArgumentParser.parse_args` except that it does not produce an error when + extra arguments are present. Instead, it returns a two item tuple containing + the populated namespace and the list of remaining argument strings. -:: + :: - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action='store_true') - >>> parser.add_argument('bar') - >>> parser.parse_known_args(['--foo', '--badger', 'BAR', 'spam']) - (Namespace(bar='BAR', foo=True), ['--badger', 'spam']) + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo', action='store_true') + >>> parser.add_argument('bar') + >>> parser.parse_known_args(['--foo', '--badger', 'BAR', 'spam']) + (Namespace(bar='BAR', foo=True), ['--badger', 'spam']) .. warning:: :ref:`Prefix matching ` rules apply to @@ -2235,8 +2140,8 @@ Exiting methods .. method:: ArgumentParser.exit(status=0, message=None) This method terminates the program, exiting with the specified *status* - and, if given, it prints a *message* before that. The user can override - this method to handle these steps differently:: + and, if given, it prints a *message* to :data:`sys.stderr` before that. + The user can override this method to handle these steps differently:: class ErrorCatchingArgumentParser(argparse.ArgumentParser): def exit(self, status=0, message=None): @@ -2246,8 +2151,8 @@ Exiting methods .. method:: ArgumentParser.error(message) - This method prints a usage message including the *message* to the - standard error and terminates the program with a status code of 2. + This method prints a usage message, including the *message*, to + :data:`sys.stderr` and terminates the program with a status code of 2. Intermixed parsing @@ -2256,90 +2161,66 @@ Intermixed parsing .. method:: ArgumentParser.parse_intermixed_args(args=None, namespace=None) .. method:: ArgumentParser.parse_known_intermixed_args(args=None, namespace=None) -A number of Unix commands allow the user to intermix optional arguments with -positional arguments. The :meth:`~ArgumentParser.parse_intermixed_args` -and :meth:`~ArgumentParser.parse_known_intermixed_args` methods -support this parsing style. - -These parsers do not support all the argparse features, and will raise -exceptions if unsupported features are used. In particular, subparsers, -and mutually exclusive groups that include both -optionals and positionals are not supported. - -The following example shows the difference between -:meth:`~ArgumentParser.parse_known_args` and -:meth:`~ArgumentParser.parse_intermixed_args`: the former returns ``['2', -'3']`` as unparsed arguments, while the latter collects all the positionals -into ``rest``. :: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo') - >>> parser.add_argument('cmd') - >>> parser.add_argument('rest', nargs='*', type=int) - >>> parser.parse_known_args('doit 1 --foo bar 2 3'.split()) - (Namespace(cmd='doit', foo='bar', rest=[1]), ['2', '3']) - >>> parser.parse_intermixed_args('doit 1 --foo bar 2 3'.split()) - Namespace(cmd='doit', foo='bar', rest=[1, 2, 3]) - -:meth:`~ArgumentParser.parse_known_intermixed_args` returns a two item tuple -containing the populated namespace and the list of remaining argument strings. -:meth:`~ArgumentParser.parse_intermixed_args` raises an error if there are any -remaining unparsed argument strings. + A number of Unix commands allow the user to intermix optional arguments with + positional arguments. The :meth:`~ArgumentParser.parse_intermixed_args` + and :meth:`~ArgumentParser.parse_known_intermixed_args` methods + support this parsing style. -.. versionadded:: 3.7 + These parsers do not support all the :mod:`!argparse` features, and will raise + exceptions if unsupported features are used. In particular, subparsers, + and mutually exclusive groups that include both + optionals and positionals are not supported. -.. _upgrading-optparse-code: + The following example shows the difference between + :meth:`~ArgumentParser.parse_known_args` and + :meth:`~ArgumentParser.parse_intermixed_args`: the former returns ``['2', + '3']`` as unparsed arguments, while the latter collects all the positionals + into ``rest``. :: -Upgrading optparse code ------------------------ - -Originally, the :mod:`argparse` module had attempted to maintain compatibility -with :mod:`optparse`. However, :mod:`optparse` was difficult to extend -transparently, particularly with the changes required to support the new -``nargs=`` specifiers and better usage messages. When most everything in -:mod:`optparse` had either been copy-pasted over or monkey-patched, it no -longer seemed practical to try to maintain the backwards compatibility. - -The :mod:`argparse` module improves on the standard library :mod:`optparse` -module in a number of ways including: - -* Handling positional arguments. -* Supporting sub-commands. -* Allowing alternative option prefixes like ``+`` and ``/``. -* Handling zero-or-more and one-or-more style arguments. -* Producing more informative usage messages. -* Providing a much simpler interface for custom ``type`` and ``action``. + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo') + >>> parser.add_argument('cmd') + >>> parser.add_argument('rest', nargs='*', type=int) + >>> parser.parse_known_args('doit 1 --foo bar 2 3'.split()) + (Namespace(cmd='doit', foo='bar', rest=[1]), ['2', '3']) + >>> parser.parse_intermixed_args('doit 1 --foo bar 2 3'.split()) + Namespace(cmd='doit', foo='bar', rest=[1, 2, 3]) -A partial upgrade path from :mod:`optparse` to :mod:`argparse`: + :meth:`~ArgumentParser.parse_known_intermixed_args` returns a two item tuple + containing the populated namespace and the list of remaining argument strings. + :meth:`~ArgumentParser.parse_intermixed_args` raises an error if there are any + remaining unparsed argument strings. -* Replace all :meth:`optparse.OptionParser.add_option` calls with - :meth:`ArgumentParser.add_argument` calls. + .. versionadded:: 3.7 -* Replace ``(options, args) = parser.parse_args()`` with ``args = - parser.parse_args()`` and add additional :meth:`ArgumentParser.add_argument` - calls for the positional arguments. Keep in mind that what was previously - called ``options``, now in the :mod:`argparse` context is called ``args``. -* Replace :meth:`optparse.OptionParser.disable_interspersed_args` - by using :meth:`~ArgumentParser.parse_intermixed_args` instead of - :meth:`~ArgumentParser.parse_args`. +Registering custom types or actions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* Replace callback actions and the ``callback_*`` keyword arguments with - ``type`` or ``action`` arguments. +.. method:: ArgumentParser.register(registry_name, value, object) -* Replace string names for ``type`` keyword arguments with the corresponding - type objects (e.g. int, float, complex, etc). + Sometimes it's desirable to use a custom string in error messages to provide + more user-friendly output. In these cases, :meth:`!register` can be used to + register custom actions or types with a parser and allow you to reference the + type by their registered name instead of their callable name. -* Replace :class:`optparse.Values` with :class:`Namespace` and - :exc:`optparse.OptionError` and :exc:`optparse.OptionValueError` with - :exc:`ArgumentError`. + The :meth:`!register` method accepts three arguments - a *registry_name*, + specifying the internal registry where the object will be stored (e.g., + ``action``, ``type``), *value*, which is the key under which the object will + be registered, and object, the callable to be registered. -* Replace strings with implicit arguments such as ``%default`` or ``%prog`` with - the standard Python syntax to use dictionaries to format strings, that is, - ``%(default)s`` and ``%(prog)s``. + The following example shows how to register a custom type with a parser:: -* Replace the OptionParser constructor ``version`` argument with a call to - ``parser.add_argument('--version', action='version', version='')``. + >>> import argparse + >>> parser = argparse.ArgumentParser() + >>> parser.register('type', 'hexadecimal integer', lambda s: int(s, 16)) + >>> parser.add_argument('--foo', type='hexadecimal integer') + _StoreAction(option_strings=['--foo'], dest='foo', nargs=None, const=None, default=None, type='hexadecimal integer', choices=None, required=False, help=None, metavar=None, deprecated=False) + >>> parser.parse_args(['--foo', '0xFA']) + Namespace(foo=250) + >>> parser.parse_args(['--foo', '1.2']) + usage: PROG [-h] [--foo FOO] + PROG: error: argument --foo: invalid 'hexadecimal integer' value: '1.2' Exceptions ---------- @@ -2354,3 +2235,12 @@ Exceptions .. exception:: ArgumentTypeError Raised when something goes wrong converting a command line string to a type. + + +.. rubric:: Guides and Tutorials + +.. toctree:: + :maxdepth: 1 + + ../howto/argparse.rst + ../howto/argparse-optparse.rst diff --git a/Doc/library/array.rst b/Doc/library/array.rst index cdf21db8779fe8..e0b1eb89cf6c05 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -1,5 +1,5 @@ -:mod:`array` --- Efficient arrays of numeric values -=================================================== +:mod:`!array` --- Efficient arrays of numeric values +==================================================== .. module:: array :synopsis: Space efficient arrays of uniformly typed numeric values. @@ -9,7 +9,7 @@ -------------- This module defines an object type which can compactly represent an array of -basic values: characters, integers, floating point numbers. Arrays are sequence +basic values: characters, integers, floating-point numbers. Arrays are sequence types and behave very much like lists, except that the type of objects stored in them is constrained. The type is specified at object creation time by using a :dfn:`type code`, which is a single character. The following type codes are @@ -263,7 +263,7 @@ The string representation is guaranteed to be able to be converted back to an array with the same type and value using :func:`eval`, so long as the :class:`~array.array` class has been imported using ``from array import array``. Variables ``inf`` and ``nan`` must also be defined if it contains -corresponding floating point values. +corresponding floating-point values. Examples:: array('l') diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index d10f3f275c0eaf..fd901e232855b5 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -1,5 +1,5 @@ -:mod:`ast` --- Abstract Syntax Trees -==================================== +:mod:`!ast` --- Abstract Syntax Trees +===================================== .. module:: ast :synopsis: Abstract Syntax Tree classes and manipulation. @@ -61,7 +61,7 @@ Node classes .. attribute:: _fields - Each concrete class has an attribute :attr:`_fields` which gives the names + Each concrete class has an attribute :attr:`!_fields` which gives the names of all child nodes. Each instance of a concrete class has one attribute for each child node, @@ -74,6 +74,18 @@ Node classes as Python lists. All possible attributes must be present and have valid values when compiling an AST with :func:`compile`. + .. attribute:: _field_types + + The :attr:`!_field_types` attribute on each concrete class is a dictionary + mapping field names (as also listed in :attr:`_fields`) to their types. + + .. doctest:: + + >>> ast.TypeVar._field_types + {'name': , 'bound': ast.expr | None, 'default_value': ast.expr | None} + + .. versionadded:: 3.13 + .. attribute:: lineno col_offset end_lineno @@ -108,7 +120,8 @@ Node classes If a field that is optional in the grammar is omitted from the constructor, it defaults to ``None``. If a list field is omitted, it defaults to the empty - list. If any other field is omitted, a :exc:`DeprecationWarning` is raised + list. If a field of type :class:`!ast.expr_context` is omitted, it defaults to + :class:`Load() `. If any other field is omitted, a :exc:`DeprecationWarning` is raised and the AST node will not have this field. In Python 3.15, this condition will raise an error. @@ -121,6 +134,11 @@ Node classes Simple indices are represented by their value, extended slices are represented as tuples. +.. versionchanged:: 3.14 + + The :meth:`~object.__repr__` output of :class:`~ast.AST` nodes includes + the values of the node fields. + .. deprecated:: 3.8 Old classes :class:`!ast.Num`, :class:`!ast.Str`, :class:`!ast.Bytes`, @@ -160,9 +178,9 @@ Root nodes A Python module, as with :ref:`file input `. Node type generated by :func:`ast.parse` in the default ``"exec"`` *mode*. - *body* is a :class:`list` of the module's :ref:`ast-statements`. + ``body`` is a :class:`list` of the module's :ref:`ast-statements`. - *type_ignores* is a :class:`list` of the module's type ignore comments; + ``type_ignores`` is a :class:`list` of the module's type ignore comments; see :func:`ast.parse` for more details. .. doctest:: @@ -173,8 +191,7 @@ Root nodes Assign( targets=[ Name(id='x', ctx=Store())], - value=Constant(value=1))], - type_ignores=[]) + value=Constant(value=1))]) .. class:: Expression(body) @@ -182,7 +199,7 @@ Root nodes A single Python :ref:`expression input `. Node type generated by :func:`ast.parse` when *mode* is ``"eval"``. - *body* is a single node, + ``body`` is a single node, one of the :ref:`expression types `. .. doctest:: @@ -197,7 +214,7 @@ Root nodes A single :ref:`interactive input `, like in :ref:`tut-interac`. Node type generated by :func:`ast.parse` when *mode* is ``"single"``. - *body* is a :class:`list` of :ref:`statement nodes `. + ``body`` is a :class:`list` of :ref:`statement nodes `. .. doctest:: @@ -226,9 +243,9 @@ Root nodes # type: (int, int) -> int return a + b - *argtypes* is a :class:`list` of :ref:`expression nodes `. + ``argtypes`` is a :class:`list` of :ref:`expression nodes `. - *returns* is a single :ref:`expression node `. + ``returns`` is a single :ref:`expression node `. .. doctest:: @@ -302,8 +319,7 @@ Literals value=Call( func=Name(id='sin', ctx=Load()), args=[ - Name(id='a', ctx=Load())], - keywords=[]), + Name(id='a', ctx=Load())]), conversion=-1, format_spec=JoinedStr( values=[ @@ -398,8 +414,7 @@ Variables Module( body=[ Expr( - value=Name(id='a', ctx=Load()))], - type_ignores=[]) + value=Name(id='a', ctx=Load()))]) >>> print(ast.dump(ast.parse('a = 1'), indent=4)) Module( @@ -407,16 +422,14 @@ Variables Assign( targets=[ Name(id='a', ctx=Store())], - value=Constant(value=1))], - type_ignores=[]) + value=Constant(value=1))]) >>> print(ast.dump(ast.parse('del a'), indent=4)) Module( body=[ Delete( targets=[ - Name(id='a', ctx=Del())])], - type_ignores=[]) + Name(id='a', ctx=Del())])]) .. class:: Starred(value, ctx) @@ -439,8 +452,7 @@ Variables value=Name(id='b', ctx=Store()), ctx=Store())], ctx=Store())], - value=Name(id='it', ctx=Load()))], - type_ignores=[]) + value=Name(id='it', ctx=Load()))]) .. _ast-expressions: @@ -463,8 +475,7 @@ Expressions Expr( value=UnaryOp( op=USub(), - operand=Name(id='a', ctx=Load())))], - type_ignores=[]) + operand=Name(id='a', ctx=Load())))]) .. class:: UnaryOp(op, operand) @@ -591,8 +602,7 @@ Expressions * ``keywords`` holds a list of :class:`.keyword` objects representing arguments passed by keyword. - When creating a ``Call`` node, ``args`` and ``keywords`` are required, but - they can be empty lists. + The ``args`` and ``keywords`` arguments are optional and default to empty lists. .. doctest:: @@ -729,7 +739,10 @@ Comprehensions .. doctest:: - >>> print(ast.dump(ast.parse('[x for x in numbers]', mode='eval'), indent=4)) + >>> print(ast.dump( + ... ast.parse('[x for x in numbers]', mode='eval'), + ... indent=4, + ... )) Expression( body=ListComp( elt=Name(id='x', ctx=Load()), @@ -737,9 +750,11 @@ Comprehensions comprehension( target=Name(id='x', ctx=Store()), iter=Name(id='numbers', ctx=Load()), - ifs=[], is_async=0)])) - >>> print(ast.dump(ast.parse('{x: x**2 for x in numbers}', mode='eval'), indent=4)) + >>> print(ast.dump( + ... ast.parse('{x: x**2 for x in numbers}', mode='eval'), + ... indent=4, + ... )) Expression( body=DictComp( key=Name(id='x', ctx=Load()), @@ -751,9 +766,11 @@ Comprehensions comprehension( target=Name(id='x', ctx=Store()), iter=Name(id='numbers', ctx=Load()), - ifs=[], is_async=0)])) - >>> print(ast.dump(ast.parse('{x for x in numbers}', mode='eval'), indent=4)) + >>> print(ast.dump( + ... ast.parse('{x for x in numbers}', mode='eval'), + ... indent=4, + ... )) Expression( body=SetComp( elt=Name(id='x', ctx=Load()), @@ -761,7 +778,6 @@ Comprehensions comprehension( target=Name(id='x', ctx=Store()), iter=Name(id='numbers', ctx=Load()), - ifs=[], is_async=0)])) @@ -784,18 +800,15 @@ Comprehensions elt=Call( func=Name(id='ord', ctx=Load()), args=[ - Name(id='c', ctx=Load())], - keywords=[]), + Name(id='c', ctx=Load())]), generators=[ comprehension( target=Name(id='line', ctx=Store()), iter=Name(id='file', ctx=Load()), - ifs=[], is_async=0), comprehension( target=Name(id='c', ctx=Store()), iter=Name(id='line', ctx=Load()), - ifs=[], is_async=0)])) >>> print(ast.dump(ast.parse('(n**2 for n in it if n>5 if n<10)', mode='eval'), @@ -834,7 +847,6 @@ Comprehensions comprehension( target=Name(id='i', ctx=Store()), iter=Name(id='soc', ctx=Load()), - ifs=[], is_async=1)])) @@ -864,8 +876,7 @@ Statements targets=[ Name(id='a', ctx=Store()), Name(id='b', ctx=Store())], - value=Constant(value=1))], - type_ignores=[]) + value=Constant(value=1))]) >>> print(ast.dump(ast.parse('a,b = c'), indent=4)) # Unpacking Module( @@ -877,18 +888,21 @@ Statements Name(id='a', ctx=Store()), Name(id='b', ctx=Store())], ctx=Store())], - value=Name(id='c', ctx=Load()))], - type_ignores=[]) + value=Name(id='c', ctx=Load()))]) .. class:: AnnAssign(target, annotation, value, simple) An assignment with a type annotation. ``target`` is a single node and can - be a :class:`Name`, a :class:`Attribute` or a :class:`Subscript`. + be a :class:`Name`, an :class:`Attribute` or a :class:`Subscript`. ``annotation`` is the annotation, such as a :class:`Constant` or :class:`Name` - node. ``value`` is a single optional node. ``simple`` is a boolean integer - set to True for a :class:`Name` node in ``target`` that do not appear in - between parenthesis and are hence pure names and not expressions. + node. ``value`` is a single optional node. + + ``simple`` is always either 0 (indicating a "complex" target) or 1 + (indicating a "simple" target). A "simple" target consists solely of a + :class:`Name` node that does not appear between parentheses; all other + targets are considered complex. Only simple targets appear in + the :attr:`~object.__annotations__` dictionary of modules and classes. .. doctest:: @@ -898,8 +912,7 @@ Statements AnnAssign( target=Name(id='c', ctx=Store()), annotation=Name(id='int', ctx=Load()), - simple=1)], - type_ignores=[]) + simple=1)]) >>> print(ast.dump(ast.parse('(a): int = 1'), indent=4)) # Annotation with parenthesis Module( @@ -908,8 +921,7 @@ Statements target=Name(id='a', ctx=Store()), annotation=Name(id='int', ctx=Load()), value=Constant(value=1), - simple=0)], - type_ignores=[]) + simple=0)]) >>> print(ast.dump(ast.parse('a.b: int'), indent=4)) # Attribute annotation Module( @@ -920,8 +932,7 @@ Statements attr='b', ctx=Store()), annotation=Name(id='int', ctx=Load()), - simple=0)], - type_ignores=[]) + simple=0)]) >>> print(ast.dump(ast.parse('a[1]: int'), indent=4)) # Subscript annotation Module( @@ -932,8 +943,7 @@ Statements slice=Constant(value=1), ctx=Store()), annotation=Name(id='int', ctx=Load()), - simple=0)], - type_ignores=[]) + simple=0)]) .. class:: AugAssign(target, op, value) @@ -954,8 +964,7 @@ Statements AugAssign( target=Name(id='x', ctx=Store()), op=Add(), - value=Constant(value=2))], - type_ignores=[]) + value=Constant(value=2))]) .. class:: Raise(exc, cause) @@ -971,8 +980,7 @@ Statements body=[ Raise( exc=Name(id='x', ctx=Load()), - cause=Name(id='y', ctx=Load()))], - type_ignores=[]) + cause=Name(id='y', ctx=Load()))]) .. class:: Assert(test, msg) @@ -987,8 +995,7 @@ Statements body=[ Assert( test=Name(id='x', ctx=Load()), - msg=Name(id='y', ctx=Load()))], - type_ignores=[]) + msg=Name(id='y', ctx=Load()))]) .. class:: Delete(targets) @@ -1005,8 +1012,7 @@ Statements targets=[ Name(id='x', ctx=Del()), Name(id='y', ctx=Del()), - Name(id='z', ctx=Del())])], - type_ignores=[]) + Name(id='z', ctx=Del())])]) .. class:: Pass() @@ -1018,8 +1024,7 @@ Statements >>> print(ast.dump(ast.parse('pass'), indent=4)) Module( body=[ - Pass()], - type_ignores=[]) + Pass()]) .. class:: TypeAlias(name, type_params, value) @@ -1036,9 +1041,7 @@ Statements body=[ TypeAlias( name=Name(id='Alias', ctx=Store()), - type_params=[], - value=Name(id='int', ctx=Load()))], - type_ignores=[]) + value=Name(id='int', ctx=Load()))]) .. versionadded:: 3.12 @@ -1061,8 +1064,7 @@ Imports names=[ alias(name='x'), alias(name='y'), - alias(name='z')])], - type_ignores=[]) + alias(name='z')])]) .. class:: ImportFrom(module, names, level) @@ -1083,8 +1085,7 @@ Imports alias(name='x'), alias(name='y'), alias(name='z')], - level=0)], - type_ignores=[]) + level=0)]) .. class:: alias(name, asname) @@ -1102,8 +1103,7 @@ Imports names=[ alias(name='a', asname='b'), alias(name='c')], - level=2)], - type_ignores=[]) + level=2)]) Control flow ^^^^^^^^^^^^ @@ -1146,8 +1146,7 @@ Control flow value=Constant(value=Ellipsis))], orelse=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. class:: For(target, iter, body, orelse, type_comment) @@ -1181,8 +1180,7 @@ Control flow value=Constant(value=Ellipsis))], orelse=[ Expr( - value=Constant(value=Ellipsis))])], - type_ignores=[]) + value=Constant(value=Ellipsis))])]) .. class:: While(test, body, orelse) @@ -1207,8 +1205,7 @@ Control flow value=Constant(value=Ellipsis))], orelse=[ Expr( - value=Constant(value=Ellipsis))])], - type_ignores=[]) + value=Constant(value=Ellipsis))])]) .. class:: Break @@ -1242,9 +1239,7 @@ Control flow body=[ Break()], orelse=[ - Continue()])], - orelse=[])], - type_ignores=[]) + Continue()])])]) .. class:: Try(body, handlers, orelse, finalbody) @@ -1289,8 +1284,7 @@ Control flow value=Constant(value=Ellipsis))], finalbody=[ Expr( - value=Constant(value=Ellipsis))])], - type_ignores=[]) + value=Constant(value=Ellipsis))])]) .. class:: TryStar(body, handlers, orelse, finalbody) @@ -1318,10 +1312,7 @@ Control flow type=Name(id='Exception', ctx=Load()), body=[ Expr( - value=Constant(value=Ellipsis))])], - orelse=[], - finalbody=[])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.11 @@ -1353,10 +1344,7 @@ Control flow ExceptHandler( type=Name(id='TypeError', ctx=Load()), body=[ - Pass()])], - orelse=[], - finalbody=[])], - type_ignores=[]) + Pass()])])]) .. class:: With(items, body, type_comment) @@ -1398,9 +1386,7 @@ Control flow func=Name(id='something', ctx=Load()), args=[ Name(id='b', ctx=Load()), - Name(id='d', ctx=Load())], - keywords=[]))])], - type_ignores=[]) + Name(id='d', ctx=Load())]))])]) Pattern matching @@ -1457,14 +1443,10 @@ Pattern matching value=Constant(value=Ellipsis))]), match_case( pattern=MatchClass( - cls=Name(id='tuple', ctx=Load()), - patterns=[], - kwd_attrs=[], - kwd_patterns=[]), + cls=Name(id='tuple', ctx=Load())), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1492,8 +1474,7 @@ Pattern matching value=Constant(value='Relevant')), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1519,8 +1500,7 @@ Pattern matching pattern=MatchSingleton(value=None), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1552,8 +1532,7 @@ Pattern matching value=Constant(value=2))]), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1594,8 +1573,7 @@ Pattern matching MatchStar()]), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1639,11 +1617,10 @@ Pattern matching Expr( value=Constant(value=Ellipsis))]), match_case( - pattern=MatchMapping(keys=[], patterns=[], rest='rest'), + pattern=MatchMapping(rest='rest'), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1685,16 +1662,13 @@ Pattern matching MatchValue( value=Constant(value=0)), MatchValue( - value=Constant(value=0))], - kwd_attrs=[], - kwd_patterns=[]), + value=Constant(value=0))]), body=[ Expr( value=Constant(value=Ellipsis))]), match_case( pattern=MatchClass( cls=Name(id='Point3D', ctx=Load()), - patterns=[], kwd_attrs=[ 'x', 'y', @@ -1708,8 +1682,7 @@ Pattern matching value=Constant(value=0))]), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1751,8 +1724,7 @@ Pattern matching pattern=MatchAs(), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1785,8 +1757,7 @@ Pattern matching MatchAs(name='y')]), body=[ Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) + value=Constant(value=Ellipsis))])])]) .. versionadded:: 3.10 @@ -1798,15 +1769,17 @@ Type parameters :ref:`Type parameters ` can exist on classes, functions, and type aliases. -.. class:: TypeVar(name, bound) +.. class:: TypeVar(name, bound, default_value) A :class:`typing.TypeVar`. ``name`` is the name of the type variable. ``bound`` is the bound or constraints, if any. If ``bound`` is a :class:`Tuple`, - it represents constraints; otherwise it represents the bound. + it represents constraints; otherwise it represents the bound. ``default_value`` + is the default value; if the :class:`!TypeVar` has no default, this + attribute will be set to ``None``. .. doctest:: - >>> print(ast.dump(ast.parse("type Alias[T: int] = list[T]"), indent=4)) + >>> print(ast.dump(ast.parse("type Alias[T: int = bool] = list[T]"), indent=4)) Module( body=[ TypeAlias( @@ -1814,28 +1787,39 @@ aliases. type_params=[ TypeVar( name='T', - bound=Name(id='int', ctx=Load()))], + bound=Name(id='int', ctx=Load()), + default_value=Name(id='bool', ctx=Load()))], value=Subscript( value=Name(id='list', ctx=Load()), slice=Name(id='T', ctx=Load()), - ctx=Load()))], - type_ignores=[]) + ctx=Load()))]) .. versionadded:: 3.12 -.. class:: ParamSpec(name) + .. versionchanged:: 3.13 + Added the *default_value* parameter. + +.. class:: ParamSpec(name, default_value) A :class:`typing.ParamSpec`. ``name`` is the name of the parameter specification. + ``default_value`` is the default value; if the :class:`!ParamSpec` has no default, + this attribute will be set to ``None``. .. doctest:: - >>> print(ast.dump(ast.parse("type Alias[**P] = Callable[P, int]"), indent=4)) + >>> print(ast.dump(ast.parse("type Alias[**P = [int, str]] = Callable[P, int]"), indent=4)) Module( body=[ TypeAlias( name=Name(id='Alias', ctx=Store()), type_params=[ - ParamSpec(name='P')], + ParamSpec( + name='P', + default_value=List( + elts=[ + Name(id='int', ctx=Load()), + Name(id='str', ctx=Load())], + ctx=Load()))], value=Subscript( value=Name(id='Callable', ctx=Load()), slice=Tuple( @@ -1843,24 +1827,30 @@ aliases. Name(id='P', ctx=Load()), Name(id='int', ctx=Load())], ctx=Load()), - ctx=Load()))], - type_ignores=[]) + ctx=Load()))]) .. versionadded:: 3.12 -.. class:: TypeVarTuple(name) + .. versionchanged:: 3.13 + Added the *default_value* parameter. + +.. class:: TypeVarTuple(name, default_value) A :class:`typing.TypeVarTuple`. ``name`` is the name of the type variable tuple. + ``default_value`` is the default value; if the :class:`!TypeVarTuple` has no + default, this attribute will be set to ``None``. .. doctest:: - >>> print(ast.dump(ast.parse("type Alias[*Ts] = tuple[*Ts]"), indent=4)) + >>> print(ast.dump(ast.parse("type Alias[*Ts = ()] = tuple[*Ts]"), indent=4)) Module( body=[ TypeAlias( name=Name(id='Alias', ctx=Store()), type_params=[ - TypeVarTuple(name='Ts')], + TypeVarTuple( + name='Ts', + default_value=Tuple(ctx=Load()))], value=Subscript( value=Name(id='tuple', ctx=Load()), slice=Tuple( @@ -1869,11 +1859,13 @@ aliases. value=Name(id='Ts', ctx=Load()), ctx=Load())], ctx=Load()), - ctx=Load()))], - type_ignores=[]) + ctx=Load()))]) .. versionadded:: 3.12 + .. versionchanged:: 3.13 + Added the *default_value* parameter. + Function and class definitions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1910,15 +1902,10 @@ Function and class definitions Expr( value=Lambda( args=arguments( - posonlyargs=[], args=[ arg(arg='x'), - arg(arg='y')], - kwonlyargs=[], - kw_defaults=[], - defaults=[]), - body=Constant(value=Ellipsis)))], - type_ignores=[]) + arg(arg='y')]), + body=Constant(value=Ellipsis)))]) .. class:: arguments(posonlyargs, args, vararg, kwonlyargs, kw_defaults, kwarg, defaults) @@ -1957,7 +1944,6 @@ Function and class definitions FunctionDef( name='f', args=arguments( - posonlyargs=[], args=[ arg( arg='a', @@ -1980,9 +1966,7 @@ Function and class definitions decorator_list=[ Name(id='decorator1', ctx=Load()), Name(id='decorator2', ctx=Load())], - returns=Constant(value='return annotation'), - type_params=[])], - type_ignores=[]) + returns=Constant(value='return annotation'))]) .. class:: Return(value) @@ -1995,15 +1979,14 @@ Function and class definitions Module( body=[ Return( - value=Constant(value=4))], - type_ignores=[]) + value=Constant(value=4))]) .. class:: Yield(value) YieldFrom(value) A ``yield`` or ``yield from`` expression. Because these are expressions, they - must be wrapped in a :class:`Expr` node if the value sent back is not used. + must be wrapped in an :class:`Expr` node if the value sent back is not used. .. doctest:: @@ -2012,16 +1995,14 @@ Function and class definitions body=[ Expr( value=Yield( - value=Name(id='x', ctx=Load())))], - type_ignores=[]) + value=Name(id='x', ctx=Load())))]) >>> print(ast.dump(ast.parse('yield from x'), indent=4)) Module( body=[ Expr( value=YieldFrom( - value=Name(id='x', ctx=Load())))], - type_ignores=[]) + value=Name(id='x', ctx=Load())))]) .. class:: Global(names) @@ -2038,8 +2019,7 @@ Function and class definitions names=[ 'x', 'y', - 'z'])], - type_ignores=[]) + 'z'])]) >>> print(ast.dump(ast.parse('nonlocal x,y,z'), indent=4)) Module( @@ -2048,8 +2028,7 @@ Function and class definitions names=[ 'x', 'y', - 'z'])], - type_ignores=[]) + 'z'])]) .. class:: ClassDef(name, bases, keywords, body, decorator_list, type_params) @@ -2059,8 +2038,7 @@ Function and class definitions * ``name`` is a raw string for the class name * ``bases`` is a list of nodes for explicitly specified base classes. * ``keywords`` is a list of :class:`.keyword` nodes, principally for 'metaclass'. - Other keywords will be passed to the metaclass, as per `PEP-3115 - `_. + Other keywords will be passed to the metaclass, as per :pep:`3115`. * ``body`` is a list of nodes representing the code within the class definition. * ``decorator_list`` is a list of nodes, as in :class:`FunctionDef`. @@ -2089,9 +2067,7 @@ Function and class definitions Pass()], decorator_list=[ Name(id='decorator1', ctx=Load()), - Name(id='decorator2', ctx=Load())], - type_params=[])], - type_ignores=[]) + Name(id='decorator2', ctx=Load())])]) .. versionchanged:: 3.12 Added ``type_params``. @@ -2123,22 +2099,12 @@ Async and await body=[ AsyncFunctionDef( name='f', - args=arguments( - posonlyargs=[], - args=[], - kwonlyargs=[], - kw_defaults=[], - defaults=[]), + args=arguments(), body=[ Expr( value=Await( value=Call( - func=Name(id='other_func', ctx=Load()), - args=[], - keywords=[])))], - decorator_list=[], - type_params=[])], - type_ignores=[]) + func=Name(id='other_func', ctx=Load()))))])]) .. class:: AsyncFor(target, iter, body, orelse, type_comment) @@ -2172,7 +2138,7 @@ and classes for traversing abstract syntax trees: If ``type_comments=True`` is given, the parser is modified to check and return type comments as specified by :pep:`484` and :pep:`526`. This is equivalent to adding :data:`ast.PyCF_TYPE_COMMENTS` to the - flags passed to :func:`compile()`. This will report syntax errors + flags passed to :func:`compile`. This will report syntax errors for misplaced type comments. Without this flag, type comments will be ignored, and the ``type_comment`` field on selected AST nodes will always be ``None``. In addition, the locations of ``# type: @@ -2425,7 +2391,7 @@ and classes for traversing abstract syntax trees: node = YourTransformer().visit(node) -.. function:: dump(node, annotate_fields=True, include_attributes=False, *, indent=None) +.. function:: dump(node, annotate_fields=True, include_attributes=False, *, indent=None, show_empty=False) Return a formatted dump of the tree in *node*. This is mainly useful for debugging purposes. If *annotate_fields* is true (by default), @@ -2442,9 +2408,42 @@ and classes for traversing abstract syntax trees: indents that many spaces per level. If *indent* is a string (such as ``"\t"``), that string is used to indent each level. + If *show_empty* is ``False`` (the default), empty lists and fields that are ``None`` + will be omitted from the output. + .. versionchanged:: 3.9 Added the *indent* option. + .. versionchanged:: 3.13 + Added the *show_empty* option. + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... async def f(): + ... await other_func() + ... """), indent=4, show_empty=True)) + Module( + body=[ + AsyncFunctionDef( + name='f', + args=arguments( + posonlyargs=[], + args=[], + kwonlyargs=[], + kw_defaults=[], + defaults=[]), + body=[ + Expr( + value=Await( + value=Call( + func=Name(id='other_func', ctx=Load()), + args=[], + keywords=[])))], + decorator_list=[], + type_params=[])], + type_ignores=[]) + .. _ast-compiler-flags: @@ -2481,6 +2480,20 @@ effects on the compilation of a program: .. versionadded:: 3.8 +.. function:: compare(a, b, /, *, compare_attributes=False) + + Recursively compares two ASTs. + + *compare_attributes* affects whether AST attributes are considered + in the comparison. If *compare_attributes* is ``False`` (default), then + attributes are ignored. Otherwise they must all be equal. This + option is useful to check whether the ASTs are structurally equal but + differ in whitespace or similar details. Attributes include line numbers + and column offsets. + + .. versionadded:: 3.14 + + .. _ast-cli: Command-Line Usage @@ -2536,7 +2549,8 @@ to stdout. Otherwise, the content is read from stdin. code that generated them. This is helpful for tools that make source code transformations. - `leoAst.py `_ unifies the + `leoAst.py `_ + unifies the token-based and parse-tree-based views of python programs by inserting two-way links between tokens and ast nodes. @@ -2548,4 +2562,4 @@ to stdout. Otherwise, the content is read from stdin. `Parso `_ is a Python parser that supports error recovery and round-trip parsing for different Python versions (in multiple Python versions). Parso is also able to list multiple syntax errors - in your python file. + in your Python file. diff --git a/Doc/library/asynchat.rst b/Doc/library/asynchat.rst new file mode 100644 index 00000000000000..5e5c3a99fe66f1 --- /dev/null +++ b/Doc/library/asynchat.rst @@ -0,0 +1,17 @@ +:mod:`!asynchat` --- Asynchronous socket command/response handler +================================================================= + +.. module:: asynchat + :synopsis: Removed in 3.12. + :deprecated: + +.. deprecated-removed:: 3.6 3.12 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.12 ` after +being deprecated in Python 3.6. The removal was decided in :pep:`594`. + +Applications should use the :mod:`asyncio` module instead. + +The last version of Python that provided the :mod:`!asynchat` module was +`Python 3.11 `_. diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index a9c3a0183bb72d..44b507a9811116 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -103,7 +103,8 @@ To handle signals the event loop must be run in the main thread. The :meth:`loop.run_in_executor` method can be used with a -:class:`concurrent.futures.ThreadPoolExecutor` to execute +:class:`concurrent.futures.ThreadPoolExecutor` or +:class:`~concurrent.futures.InterpreterPoolExecutor` to execute blocking code in a different OS thread without blocking the OS thread that the event loop runs in. @@ -128,7 +129,8 @@ if a function performs a CPU-intensive calculation for 1 second, all concurrent asyncio Tasks and IO operations would be delayed by 1 second. -An executor can be used to run a task in a different thread or even in +An executor can be used to run a task in a different thread, +including in a different interpreter, or even in a different process to avoid blocking the OS thread with the event loop. See the :meth:`loop.run_in_executor` method for more details. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 828e506a72c937..15ef33e195904d 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -59,14 +59,24 @@ an event loop: instead of using these lower level functions to manually create and close an event loop. - .. deprecated:: 3.12 - Deprecation warning is emitted if there is no current event loop. - In some future Python release this will become an error. + .. versionchanged:: 3.14 + Raises a :exc:`RuntimeError` if there is no current event loop. + + .. note:: + + The :mod:`!asyncio` policy system is deprecated and will be removed + in Python 3.16; from there on, this function will always return the + running event loop. + .. function:: set_event_loop(loop) Set *loop* as the current event loop for the current OS thread. + .. deprecated:: 3.14 + The :func:`set_event_loop` function is deprecated and will be removed + in Python 3.16. + .. function:: new_event_loop() Create and return a new event loop object. @@ -126,7 +136,7 @@ Running and stopping the loop Run the event loop until :meth:`stop` is called. - If :meth:`stop` is called before :meth:`run_forever()` is called, + If :meth:`stop` is called before :meth:`run_forever` is called, the loop will poll the I/O selector once with a timeout of zero, run all callbacks scheduled in response to I/O events (and those that were already scheduled), and then exit. @@ -165,7 +175,7 @@ Running and stopping the loop .. coroutinemethod:: loop.shutdown_asyncgens() Schedule all currently open :term:`asynchronous generator` objects to - close with an :meth:`~agen.aclose()` call. After calling this method, + close with an :meth:`~agen.aclose` call. After calling this method, the event loop will issue a warning if a new asynchronous generator is iterated. This should be used to reliably finalize all scheduled asynchronous generators. @@ -236,6 +246,9 @@ Scheduling callbacks another thread, this function *must* be used, since :meth:`call_soon` is not thread-safe. + This function is safe to be called from a reentrant context or signal handler, + however, it is not safe or fruitful to use the returned handle in such contexts. + Raises :exc:`RuntimeError` if called on a loop that's been closed. This can happen on a secondary thread when the main application is shutting down. @@ -379,9 +392,9 @@ Creating Futures and Tasks If *factory* is ``None`` the default task factory will be set. Otherwise, *factory* must be a *callable* with the signature matching - ``(loop, coro, context=None)``, where *loop* is a reference to the active + ``(loop, coro, **kwargs)``, where *loop* is a reference to the active event loop, and *coro* is a coroutine object. The callable - must return a :class:`asyncio.Future`-compatible object. + must pass on all *kwargs*, and return a :class:`asyncio.Task`-compatible object. .. method:: loop.get_task_factory() @@ -605,6 +618,9 @@ Opening network connections The *family*, *proto*, *flags*, *reuse_address*, *reuse_port*, *allow_broadcast*, and *sock* parameters were added. + .. versionchanged:: 3.8 + Added support for Windows. + .. versionchanged:: 3.8.1 The *reuse_address* parameter is no longer supported, as using :ref:`socket.SO_REUSEADDR ` @@ -622,11 +638,8 @@ Opening network connections prevents processes with differing UIDs from assigning sockets to the same socket address. - .. versionchanged:: 3.8 - Added support for Windows. - .. versionchanged:: 3.11 - The *reuse_address* parameter, disabled since Python 3.9.0, 3.8.1, + The *reuse_address* parameter, disabled since Python 3.8.1, 3.7.6 and 3.6.10, has been entirely removed. .. coroutinemethod:: loop.create_unix_connection(protocol_factory, \ @@ -796,7 +809,7 @@ Creating network servers :class:`str`, :class:`bytes`, and :class:`~pathlib.Path` paths are supported. - If *cleanup_socket* is True then the Unix socket will automatically + If *cleanup_socket* is true then the Unix socket will automatically be removed from the filesystem when the server is closed, unless the socket has been replaced after the server has been created. @@ -957,6 +970,9 @@ Watching file descriptors invoke *callback* with the specified arguments once *fd* is available for reading. + Any preexisting callback registered for *fd* is cancelled and replaced by + *callback*. + .. method:: loop.remove_reader(fd) Stop monitoring the *fd* file descriptor for read availability. Returns @@ -968,6 +984,9 @@ Watching file descriptors invoke *callback* with the specified arguments once *fd* is available for writing. + Any preexisting callback registered for *fd* is cancelled and replaced by + *callback*. + Use :func:`functools.partial` :ref:`to pass keyword arguments ` to *callback*. @@ -1155,6 +1174,14 @@ DNS Asynchronous version of :meth:`socket.getnameinfo`. +.. note:: + Both *getaddrinfo* and *getnameinfo* internally utilize their synchronous + versions through the loop's default thread pool executor. + When this executor is saturated, these methods may experience delays, + which higher-level networking libraries may report as increased timeouts. + To mitigate this, consider using a custom executor for other user tasks, + or setting a default executor with a larger number of workers. + .. versionchanged:: 3.7 Both *getaddrinfo* and *getnameinfo* methods were always documented to return a coroutine, but prior to Python 3.7 they were, in fact, @@ -1254,6 +1281,9 @@ Executing code in thread or process pools The *executor* argument should be an :class:`concurrent.futures.Executor` instance. The default executor is used if *executor* is ``None``. + The default executor can be set by :meth:`loop.set_default_executor`, + otherwise, a :class:`concurrent.futures.ThreadPoolExecutor` will be + lazy-initialized and used by :func:`run_in_executor` if needed. Example:: @@ -1294,6 +1324,12 @@ Executing code in thread or process pools pool, cpu_bound) print('custom process pool', result) + # 4. Run in a custom interpreter pool: + with concurrent.futures.InterpreterPoolExecutor() as pool: + result = await loop.run_in_executor( + pool, cpu_bound) + print('custom interpreter pool', result) + if __name__ == '__main__': asyncio.run(main()) @@ -1318,7 +1354,8 @@ Executing code in thread or process pools Set *executor* as the default executor used by :meth:`run_in_executor`. *executor* must be an instance of - :class:`~concurrent.futures.ThreadPoolExecutor`. + :class:`~concurrent.futures.ThreadPoolExecutor`, which includes + :class:`~concurrent.futures.InterpreterPoolExecutor`. .. versionchanged:: 3.11 *executor* must be an instance of @@ -1391,7 +1428,7 @@ Allows customizing how exceptions are handled in the event loop. This method should not be overloaded in subclassed event loops. For custom exception handling, use - the :meth:`set_exception_handler()` method. + the :meth:`set_exception_handler` method. Enabling debug mode ^^^^^^^^^^^^^^^^^^^ @@ -1474,7 +1511,7 @@ async/await code consider using the high-level * *stdin* can be any of these: * a file-like object - * an existing file descriptor (a positive integer), for example those created with :meth:`os.pipe()` + * an existing file descriptor (a positive integer), for example those created with :meth:`os.pipe` * the :const:`subprocess.PIPE` constant (default) which will create a new pipe and connect it, * the value ``None`` which will make the subprocess inherit the file @@ -1641,6 +1678,31 @@ Do not instantiate the :class:`Server` class directly. coroutine to wait until the server is closed (and no more connections are active). + .. method:: close_clients() + + Close all existing incoming client connections. + + Calls :meth:`~asyncio.BaseTransport.close` on all associated + transports. + + :meth:`close` should be called before :meth:`close_clients` when + closing the server to avoid races with new clients connecting. + + .. versionadded:: 3.13 + + .. method:: abort_clients() + + Close all existing incoming client connections immediately, + without waiting for pending operations to complete. + + Calls :meth:`~asyncio.WriteTransport.abort` on all associated + transports. + + :meth:`close` should be called before :meth:`abort_clients` when + closing the server to avoid races with new clients connecting. + + .. versionadded:: 3.13 + .. method:: get_loop() Return the event loop associated with the server object. @@ -1735,12 +1797,11 @@ By default asyncio is configured to use :class:`EventLoop`. import asyncio import selectors - class MyPolicy(asyncio.DefaultEventLoopPolicy): - def new_event_loop(self): - selector = selectors.SelectSelector() - return asyncio.SelectorEventLoop(selector) + async def main(): + ... - asyncio.set_event_loop_policy(MyPolicy()) + loop_factory = lambda: asyncio.SelectorEventLoop(selectors.SelectSelector()) + asyncio.run(main(), loop_factory=loop_factory) .. availability:: Unix, Windows. @@ -1755,7 +1816,7 @@ By default asyncio is configured to use :class:`EventLoop`. .. seealso:: `MSDN documentation on I/O Completion Ports - `_. + `_. .. class:: EventLoop diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index 893ae5518f757d..9dce0731411940 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -120,20 +120,20 @@ Future Object a :exc:`CancelledError` exception. If the Future's result isn't yet available, this method raises - a :exc:`InvalidStateError` exception. + an :exc:`InvalidStateError` exception. .. method:: set_result(result) Mark the Future as *done* and set its result. - Raises a :exc:`InvalidStateError` error if the Future is + Raises an :exc:`InvalidStateError` error if the Future is already *done*. .. method:: set_exception(exception) Mark the Future as *done* and set an exception. - Raises a :exc:`InvalidStateError` error if the Future is + Raises an :exc:`InvalidStateError` error if the Future is already *done*. .. method:: done() diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst new file mode 100644 index 00000000000000..814bfe6e269bf9 --- /dev/null +++ b/Doc/library/asyncio-graph.rst @@ -0,0 +1,145 @@ +.. currentmodule:: asyncio + + +.. _asyncio-graph: + +======================== +Call Graph Introspection +======================== + +**Source code:** :source:`Lib/asyncio/graph.py` + +------------------------------------- + +asyncio has powerful runtime call graph introspection utilities +to trace the entire call graph of a running *coroutine* or *task*, or +a suspended *future*. These utilities and the underlying machinery +can be used from within a Python program or by external profilers +and debuggers. + +.. versionadded:: next + + +.. function:: print_call_graph(future=None, /, *, file=None, depth=1, limit=None) + + Print the async call graph for the current task or the provided + :class:`Task` or :class:`Future`. + + This function prints entries starting from the top frame and going + down towards the invocation point. + + The function receives an optional *future* argument. + If not passed, the current running task will be used. + + If the function is called on *the current task*, the optional + keyword-only *depth* argument can be used to skip the specified + number of frames from top of the stack. + + If the optional keyword-only *limit* argument is provided, each call stack + in the resulting graph is truncated to include at most ``abs(limit)`` + entries. If *limit* is positive, the entries left are the closest to + the invocation point. If *limit* is negative, the topmost entries are + left. If *limit* is omitted or ``None``, all entries are present. + If *limit* is ``0``, the call stack is not printed at all, only + "awaited by" information is printed. + + If *file* is omitted or ``None``, the function will print + to :data:`sys.stdout`. + + **Example:** + + The following Python code: + + .. code-block:: python + + import asyncio + + async def test(): + asyncio.print_call_graph() + + async def main(): + async with asyncio.TaskGroup() as g: + g.create_task(test(), name='test') + + asyncio.run(main()) + + will print:: + + * Task(name='test', id=0x1039f0fe0) + + Call stack: + | File 't2.py', line 4, in async test() + + Awaited by: + * Task(name='Task-1', id=0x103a5e060) + + Call stack: + | File 'taskgroups.py', line 107, in async TaskGroup.__aexit__() + | File 't2.py', line 7, in async main() + +.. function:: format_call_graph(future=None, /, *, depth=1, limit=None) + + Like :func:`print_call_graph`, but returns a string. + If *future* is ``None`` and there's no current task, + the function returns an empty string. + + +.. function:: capture_call_graph(future=None, /, *, depth=1, limit=None) + + Capture the async call graph for the current task or the provided + :class:`Task` or :class:`Future`. + + The function receives an optional *future* argument. + If not passed, the current running task will be used. If there's no + current task, the function returns ``None``. + + If the function is called on *the current task*, the optional + keyword-only *depth* argument can be used to skip the specified + number of frames from top of the stack. + + Returns a ``FutureCallGraph`` data class object: + + * ``FutureCallGraph(future, call_stack, awaited_by)`` + + Where *future* is a reference to a :class:`Future` or + a :class:`Task` (or their subclasses.) + + ``call_stack`` is a tuple of ``FrameCallGraphEntry`` objects. + + ``awaited_by`` is a tuple of ``FutureCallGraph`` objects. + + * ``FrameCallGraphEntry(frame)`` + + Where *frame* is a frame object of a regular Python function + in the call stack. + + +Low level utility functions +=========================== + +To introspect an async call graph asyncio requires cooperation from +control flow structures, such as :func:`shield` or :class:`TaskGroup`. +Any time an intermediate :class:`Future` object with low-level APIs like +:meth:`Future.add_done_callback() ` is +involved, the following two functions should be used to inform asyncio +about how exactly such intermediate future objects are connected with +the tasks they wrap or control. + + +.. function:: future_add_to_awaited_by(future, waiter, /) + + Record that *future* is awaited on by *waiter*. + + Both *future* and *waiter* must be instances of + :class:`Future` or :class:`Task` or their subclasses, + otherwise the call would have no effect. + + A call to ``future_add_to_awaited_by()`` must be followed by an + eventual call to the :func:`future_discard_from_awaited_by` function + with the same arguments. + + +.. function:: future_discard_from_awaited_by(future, waiter, /) + + Record that *future* is no longer awaited on by *waiter*. + + Both *future* and *waiter* must be instances of + :class:`Future` or :class:`Task` or their subclasses, otherwise + the call would have no effect. diff --git a/Doc/library/asyncio-llapi-index.rst b/Doc/library/asyncio-llapi-index.rst index 67136ba69ec875..f5af888f31f186 100644 --- a/Doc/library/asyncio-llapi-index.rst +++ b/Doc/library/asyncio-llapi-index.rst @@ -56,10 +56,10 @@ See also the main documentation section about the * - :meth:`loop.close` - Close the event loop. - * - :meth:`loop.is_running()` + * - :meth:`loop.is_running` - Return ``True`` if the event loop is running. - * - :meth:`loop.is_closed()` + * - :meth:`loop.is_closed` - Return ``True`` if the event loop is closed. * - ``await`` :meth:`loop.shutdown_asyncgens` @@ -96,7 +96,7 @@ See also the main documentation section about the - Invoke a callback *at* the given time. -.. rubric:: Thread/Process Pool +.. rubric:: Thread/Interpreter/Process Pool .. list-table:: :widths: 50 50 :class: full-width-table diff --git a/Doc/library/asyncio-platforms.rst b/Doc/library/asyncio-platforms.rst index 19ec726c1be060..a2a3114ad6e4c5 100644 --- a/Doc/library/asyncio-platforms.rst +++ b/Doc/library/asyncio-platforms.rst @@ -77,11 +77,6 @@ Subprocess Support on Windows On Windows, the default event loop :class:`ProactorEventLoop` supports subprocesses, whereas :class:`SelectorEventLoop` does not. -The :meth:`policy.set_child_watcher() -` function is also -not supported, as :class:`ProactorEventLoop` has a different mechanism -to watch child processes. - macOS ===== diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index 346b740a8f757a..57f964912dd6ea 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -7,6 +7,14 @@ Policies ======== +.. warning:: + + Policies are deprecated and will be removed in Python 3.16. + Users are encouraged to use the :func:`asyncio.run` function + or the :class:`asyncio.Runner` with *loop_factory* to use + the desired loop implementation. + + An event loop policy is a global object used to get and set the current :ref:`event loop `, as well as create new event loops. @@ -40,12 +48,20 @@ for the current process: Return the current process-wide policy. + .. deprecated:: 3.14 + The :func:`get_event_loop_policy` function is deprecated and + will be removed in Python 3.16. + .. function:: set_event_loop_policy(policy) Set the current process-wide policy to *policy*. If *policy* is set to ``None``, the default policy is restored. + .. deprecated:: 3.14 + The :func:`set_event_loop_policy` function is deprecated and + will be removed in Python 3.16. + .. _asyncio-policy-objects: @@ -79,24 +95,9 @@ The abstract event loop policy base class is defined as follows: This method should never return ``None``. - .. method:: get_child_watcher() - - Get a child process watcher object. - - Return a watcher object implementing the - :class:`AbstractChildWatcher` interface. - - This function is Unix specific. - - .. deprecated:: 3.12 - - .. method:: set_child_watcher(watcher) - - Set the current child process watcher to *watcher*. - - This function is Unix specific. - - .. deprecated:: 3.12 + .. deprecated:: 3.14 + The :class:`AbstractEventLoopPolicy` class is deprecated and + will be removed in Python 3.16. .. _asyncio-policy-builtin: @@ -116,11 +117,13 @@ asyncio ships with the following built-in policies: On Windows, :class:`ProactorEventLoop` is now used by default. - .. deprecated:: 3.12 - The :meth:`get_event_loop` method of the default asyncio policy now emits - a :exc:`DeprecationWarning` if there is no current event loop set and it - decides to create one. - In some future Python release this will become an error. + .. versionchanged:: 3.14 + The :meth:`get_event_loop` method of the default asyncio policy now + raises a :exc:`RuntimeError` if there is no set event loop. + + .. deprecated:: 3.14 + The :class:`DefaultEventLoopPolicy` class is deprecated and + will be removed in Python 3.16. .. class:: WindowsSelectorEventLoopPolicy @@ -130,6 +133,10 @@ asyncio ships with the following built-in policies: .. availability:: Windows. + .. deprecated:: 3.14 + The :class:`WindowsSelectorEventLoopPolicy` class is deprecated and + will be removed in Python 3.16. + .. class:: WindowsProactorEventLoopPolicy @@ -138,171 +145,9 @@ asyncio ships with the following built-in policies: .. availability:: Windows. - -.. _asyncio-watchers: - -Process Watchers -================ - -A process watcher allows customization of how an event loop monitors -child processes on Unix. Specifically, the event loop needs to know -when a child process has exited. - -In asyncio, child processes are created with -:func:`create_subprocess_exec` and :meth:`loop.subprocess_exec` -functions. - -asyncio defines the :class:`AbstractChildWatcher` abstract base class, which child -watchers should implement, and has four different implementations: -:class:`ThreadedChildWatcher` (configured to be used by default), -:class:`MultiLoopChildWatcher`, :class:`SafeChildWatcher`, and -:class:`FastChildWatcher`. - -See also the :ref:`Subprocess and Threads ` -section. - -The following two functions can be used to customize the child process watcher -implementation used by the asyncio event loop: - -.. function:: get_child_watcher() - - Return the current child watcher for the current policy. - - .. deprecated:: 3.12 - -.. function:: set_child_watcher(watcher) - - Set the current child watcher to *watcher* for the current - policy. *watcher* must implement methods defined in the - :class:`AbstractChildWatcher` base class. - - .. deprecated:: 3.12 - -.. note:: - Third-party event loops implementations might not support - custom child watchers. For such event loops, using - :func:`set_child_watcher` might be prohibited or have no effect. - -.. class:: AbstractChildWatcher - - .. method:: add_child_handler(pid, callback, *args) - - Register a new child handler. - - Arrange for ``callback(pid, returncode, *args)`` to be called - when a process with PID equal to *pid* terminates. Specifying - another callback for the same process replaces the previous - handler. - - The *callback* callable must be thread-safe. - - .. method:: remove_child_handler(pid) - - Removes the handler for process with PID equal to *pid*. - - The function returns ``True`` if the handler was successfully - removed, ``False`` if there was nothing to remove. - - .. method:: attach_loop(loop) - - Attach the watcher to an event loop. - - If the watcher was previously attached to an event loop, then - it is first detached before attaching to the new loop. - - Note: loop may be ``None``. - - .. method:: is_active() - - Return ``True`` if the watcher is ready to use. - - Spawning a subprocess with *inactive* current child watcher raises - :exc:`RuntimeError`. - - .. versionadded:: 3.8 - - .. method:: close() - - Close the watcher. - - This method has to be called to ensure that underlying - resources are cleaned-up. - - .. deprecated:: 3.12 - - -.. class:: ThreadedChildWatcher - - This implementation starts a new waiting thread for every subprocess spawn. - - It works reliably even when the asyncio event loop is run in a non-main OS thread. - - There is no noticeable overhead when handling a big number of children (*O*\ (1) each - time a child terminates), but starting a thread per process requires extra memory. - - This watcher is used by default. - - .. versionadded:: 3.8 - -.. class:: MultiLoopChildWatcher - - This implementation registers a :py:data:`SIGCHLD` signal handler on - instantiation. That can break third-party code that installs a custom handler for - :py:data:`SIGCHLD` signal. - - The watcher avoids disrupting other code spawning processes - by polling every process explicitly on a :py:data:`SIGCHLD` signal. - - There is no limitation for running subprocesses from different threads once the - watcher is installed. - - The solution is safe but it has a significant overhead when - handling a big number of processes (*O*\ (*n*) each time a - :py:data:`SIGCHLD` is received). - - .. versionadded:: 3.8 - - .. deprecated:: 3.12 - -.. class:: SafeChildWatcher - - This implementation uses active event loop from the main thread to handle - :py:data:`SIGCHLD` signal. If the main thread has no running event loop another - thread cannot spawn a subprocess (:exc:`RuntimeError` is raised). - - The watcher avoids disrupting other code spawning processes - by polling every process explicitly on a :py:data:`SIGCHLD` signal. - - This solution is as safe as :class:`MultiLoopChildWatcher` and has the same *O*\ (*n*) - complexity but requires a running event loop in the main thread to work. - - .. deprecated:: 3.12 - -.. class:: FastChildWatcher - - This implementation reaps every terminated processes by calling - ``os.waitpid(-1)`` directly, possibly breaking other code spawning - processes and waiting for their termination. - - There is no noticeable overhead when handling a big number of - children (*O*\ (1) each time a child terminates). - - This solution requires a running event loop in the main thread to work, as - :class:`SafeChildWatcher`. - - .. deprecated:: 3.12 - -.. class:: PidfdChildWatcher - - This implementation polls process file descriptors (pidfds) to await child - process termination. In some respects, :class:`PidfdChildWatcher` is a - "Goldilocks" child watcher implementation. It doesn't require signals or - threads, doesn't interfere with any processes launched outside the event - loop, and scales linearly with the number of subprocesses launched by the - event loop. The main disadvantage is that pidfds are specific to Linux, and - only work on recent (5.3+) kernels. - - .. versionadded:: 3.9 + .. deprecated:: 3.14 + The :class:`WindowsProactorEventLoopPolicy` class is deprecated and + will be removed in Python 3.16. .. _asyncio-custom-policies: diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index d86fbc21351e2d..066edd424d150e 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -55,13 +55,16 @@ Queue Return ``True`` if there are :attr:`maxsize` items in the queue. If the queue was initialized with ``maxsize=0`` (the default), - then :meth:`full()` never returns ``True``. + then :meth:`full` never returns ``True``. .. coroutinemethod:: get() Remove and return an item from the queue. If queue is empty, wait until an item is available. + Raises :exc:`QueueShutDown` if the queue has been shut down and + is empty, or if the queue has been shut down immediately. + .. method:: get_nowait() Return an item if one is immediately available, else raise @@ -82,6 +85,8 @@ Queue Put an item into the queue. If the queue is full, wait until a free slot is available before adding the item. + Raises :exc:`QueueShutDown` if the queue has been shut down. + .. method:: put_nowait(item) Put an item into the queue without blocking. @@ -92,19 +97,38 @@ Queue Return the number of items in the queue. + .. method:: shutdown(immediate=False) + + Shut down the queue, making :meth:`~Queue.get` and :meth:`~Queue.put` + raise :exc:`QueueShutDown`. + + By default, :meth:`~Queue.get` on a shut down queue will only + raise once the queue is empty. Set *immediate* to true to make + :meth:`~Queue.get` raise immediately instead. + + All blocked callers of :meth:`~Queue.put` and :meth:`~Queue.get` + will be unblocked. If *immediate* is true, a task will be marked + as done for each remaining item in the queue, which may unblock + callers of :meth:`~Queue.join`. + + .. versionadded:: 3.13 + .. method:: task_done() - Indicate that a formerly enqueued task is complete. + Indicate that a formerly enqueued work item is complete. Used by queue consumers. For each :meth:`~Queue.get` used to - fetch a task, a subsequent call to :meth:`task_done` tells the - queue that the processing on the task is complete. + fetch a work item, a subsequent call to :meth:`task_done` tells the + queue that the processing on the work item is complete. If a :meth:`join` is currently blocking, it will resume when all items have been processed (meaning that a :meth:`task_done` call was received for every item that had been :meth:`~Queue.put` into the queue). + ``shutdown(immediate=True)`` calls :meth:`task_done` for each + remaining item in the queue. + Raises :exc:`ValueError` if called more times than there were items placed in the queue. @@ -145,6 +169,14 @@ Exceptions on a queue that has reached its *maxsize*. +.. exception:: QueueShutDown + + Exception raised when :meth:`~Queue.put` or :meth:`~Queue.get` is + called on a queue which has been shut down. + + .. versionadded:: 3.13 + + Examples ======== diff --git a/Doc/library/asyncio-runner.rst b/Doc/library/asyncio-runner.rst index ec170dfde9e9aa..48d78099fd3ce7 100644 --- a/Doc/library/asyncio-runner.rst +++ b/Doc/library/asyncio-runner.rst @@ -24,11 +24,13 @@ Running an asyncio Program .. function:: run(coro, *, debug=None, loop_factory=None) - Execute the :term:`coroutine` *coro* and return the result. + Execute *coro* in an asyncio event loop and return the result. - This function runs the passed coroutine, taking care of - managing the asyncio event loop, *finalizing asynchronous - generators*, and closing the executor. + The argument can be any awaitable object. + + This function runs the awaitable, taking care of managing the + asyncio event loop, *finalizing asynchronous generators*, and + closing the executor. This function cannot be called when another asyncio event loop is running in the same thread. @@ -70,6 +72,16 @@ Running an asyncio Program Added *loop_factory* parameter. + .. versionchanged:: 3.14 + + *coro* can be any awaitable object. + + .. note:: + + The :mod:`!asyncio` policy system is deprecated and will be removed + in Python 3.16; from there on, an explicit *loop_factory* is needed + to configure the event loop. + Runner context manager ====================== @@ -91,7 +103,7 @@ Runner context manager current one. By default :func:`asyncio.new_event_loop` is used and set as current event loop with :func:`asyncio.set_event_loop` if *loop_factory* is ``None``. - Basically, :func:`asyncio.run()` example can be rewritten with the runner usage:: + Basically, :func:`asyncio.run` example can be rewritten with the runner usage:: async def main(): await asyncio.sleep(1) @@ -104,17 +116,25 @@ Runner context manager .. method:: run(coro, *, context=None) - Run a :term:`coroutine ` *coro* in the embedded loop. + Execute *coro* in the embedded event loop. + + The argument can be any awaitable object. - Return the coroutine's result or raise its exception. + If the argument is a coroutine, it is wrapped in a Task. An optional keyword-only *context* argument allows specifying a - custom :class:`contextvars.Context` for the *coro* to run in. - The runner's default context is used if ``None``. + custom :class:`contextvars.Context` for the code to run in. + The runner's default context is used if context is ``None``. + + Returns the awaitable's result or raises an exception. This function cannot be called when another asyncio event loop is running in the same thread. + .. versionchanged:: 3.14 + + *coro* can be any awaitable object. + .. method:: close() Close the runner. diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index 68b1dff20213e1..48f2890c5eef8c 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -92,7 +92,8 @@ and work with streams: family=socket.AF_UNSPEC, \ flags=socket.AI_PASSIVE, sock=None, \ backlog=100, ssl=None, reuse_address=None, \ - reuse_port=None, ssl_handshake_timeout=None, \ + reuse_port=None, keep_alive=None, \ + ssl_handshake_timeout=None, \ ssl_shutdown_timeout=None, start_serving=True) Start a socket server. @@ -128,6 +129,9 @@ and work with streams: .. versionchanged:: 3.11 Added the *ssl_shutdown_timeout* parameter. + .. versionchanged:: 3.13 + Added the *keep_alive* parameter. + .. rubric:: Unix Sockets @@ -260,8 +264,19 @@ StreamReader buffer is reset. The :attr:`IncompleteReadError.partial` attribute may contain a portion of the separator. + The *separator* may also be a tuple of separators. In this + case the return value will be the shortest possible that has any + separator as the suffix. For the purposes of :exc:`LimitOverrunError`, + the shortest possible separator is considered to be the one that + matched. + .. versionadded:: 3.5.2 + .. versionchanged:: 3.13 + + The *separator* parameter may now be a :class:`tuple` of + separators. + .. method:: at_eof() Return ``True`` if the buffer is empty and :meth:`feed_eof` diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index 817a6ff3052f4a..b477a7fa2d37ef 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -316,18 +316,6 @@ default. On Windows subprocesses are provided by :class:`ProactorEventLoop` only (default), :class:`SelectorEventLoop` has no subprocess support. -On UNIX *child watchers* are used for subprocess finish waiting, see -:ref:`asyncio-watchers` for more info. - - -.. versionchanged:: 3.8 - - UNIX switched to use :class:`ThreadedChildWatcher` for spawning subprocesses from - different threads without any limitation. - - Spawning a subprocess with *inactive* current child watcher raises - :exc:`RuntimeError`. - Note that alternative event loop implementations might have own limitations; please refer to their documentation. diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 3cf8e2737e85dc..77c2e97da11990 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -259,8 +259,8 @@ Condition Note that a task *may* return from this call spuriously, which is why the caller should always re-check the state - and be prepared to :meth:`wait` again. For this reason, you may - prefer to use :meth:`wait_for` instead. + and be prepared to :meth:`~Condition.wait` again. For this reason, you may + prefer to use :meth:`~Condition.wait_for` instead. .. coroutinemethod:: wait_for(predicate) @@ -268,7 +268,7 @@ Condition The predicate must be a callable which result will be interpreted as a boolean value. The method will repeatedly - :meth:`wait` until the predicate evaluates to *true*. The final value is the + :meth:`~Condition.wait` until the predicate evaluates to *true*. The final value is the return value. @@ -434,7 +434,7 @@ Barrier .. coroutinemethod:: abort() Put the barrier into a broken state. This causes any active or future - calls to :meth:`wait` to fail with the :class:`BrokenBarrierError`. + calls to :meth:`~Barrier.wait` to fail with the :class:`BrokenBarrierError`. Use this for example if one of the tasks needs to abort, to avoid infinite waiting tasks. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 24bd36e6431b4f..4541cf28de0605 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -158,7 +158,7 @@ other coroutines:: # Nothing happens if we just call "nested()". # A coroutine object is created but not awaited, # so it *won't run at all*. - nested() + nested() # will raise a "RuntimeWarning". # Let's do it differently now and await it: print(await nested()) # will print "42". @@ -334,6 +334,13 @@ and reliable way to wait for all tasks in the group to finish. Create a task in this task group. The signature matches that of :func:`asyncio.create_task`. + If the task group is inactive (e.g. not yet entered, + already finished, or in the process of shutting down), + we will close the given ``coro``. + + .. versionchanged:: 3.13 + + Close the given coroutine if the task group is not active. Example:: @@ -385,6 +392,74 @@ is also included in the exception group. The same special case is made for :exc:`KeyboardInterrupt` and :exc:`SystemExit` as in the previous paragraph. +Task groups are careful not to mix up the internal cancellation used to +"wake up" their :meth:`~object.__aexit__` with cancellation requests +for the task in which they are running made by other parties. +In particular, when one task group is syntactically nested in another, +and both experience an exception in one of their child tasks simultaneously, +the inner task group will process its exceptions, and then the outer task group +will receive another cancellation and process its own exceptions. + +In the case where a task group is cancelled externally and also must +raise an :exc:`ExceptionGroup`, it will call the parent task's +:meth:`~asyncio.Task.cancel` method. This ensures that a +:exc:`asyncio.CancelledError` will be raised at the next +:keyword:`await`, so the cancellation is not lost. + +Task groups preserve the cancellation count +reported by :meth:`asyncio.Task.cancelling`. + +.. versionchanged:: 3.13 + + Improved handling of simultaneous internal and external cancellations + and correct preservation of cancellation counts. + +Terminating a Task Group +------------------------ + +While terminating a task group is not natively supported by the standard +library, termination can be achieved by adding an exception-raising task +to the task group and ignoring the raised exception: + +.. code-block:: python + + import asyncio + from asyncio import TaskGroup + + class TerminateTaskGroup(Exception): + """Exception raised to terminate a task group.""" + + async def force_terminate_task_group(): + """Used to force termination of a task group.""" + raise TerminateTaskGroup() + + async def job(task_id, sleep_time): + print(f'Task {task_id}: start') + await asyncio.sleep(sleep_time) + print(f'Task {task_id}: done') + + async def main(): + try: + async with TaskGroup() as group: + # spawn some tasks + group.create_task(job(1, 0.5)) + group.create_task(job(2, 1.5)) + # sleep for 1 second + await asyncio.sleep(1) + # add an exception-raising task to force the group to terminate + group.create_task(force_terminate_task_group()) + except* TerminateTaskGroup: + pass + + asyncio.run(main()) + +Expected output: + +.. code-block:: text + + Task 1: start + Task 2: start + Task 1: done Sleeping ======== @@ -510,7 +585,7 @@ Running Tasks Concurrently # [2, 6, 24] .. note:: - If *return_exceptions* is False, cancelling gather() after it + If *return_exceptions* is false, cancelling gather() after it has been marked done won't cancel any submitted awaitables. For instance, gather can be marked done after propagating an exception to the caller, therefore, calling ``gather.cancel()`` @@ -860,19 +935,50 @@ Waiting Primitives .. function:: as_completed(aws, *, timeout=None) - Run :ref:`awaitable objects ` in the *aws* - iterable concurrently. Return an iterator of coroutines. - Each coroutine returned can be awaited to get the earliest next - result from the iterable of the remaining awaitables. + Run :ref:`awaitable objects ` in the *aws* iterable + concurrently. The returned object can be iterated to obtain the results + of the awaitables as they finish. - Raises :exc:`TimeoutError` if the timeout occurs before - all Futures are done. + The object returned by ``as_completed()`` can be iterated as an + :term:`asynchronous iterator` or a plain :term:`iterator`. When asynchronous + iteration is used, the originally-supplied awaitables are yielded if they + are tasks or futures. This makes it easy to correlate previously-scheduled + tasks with their results. Example:: - Example:: + ipv4_connect = create_task(open_connection("127.0.0.1", 80)) + ipv6_connect = create_task(open_connection("::1", 80)) + tasks = [ipv4_connect, ipv6_connect] - for coro in as_completed(aws): - earliest_result = await coro - # ... + async for earliest_connect in as_completed(tasks): + # earliest_connect is done. The result can be obtained by + # awaiting it or calling earliest_connect.result() + reader, writer = await earliest_connect + + if earliest_connect is ipv6_connect: + print("IPv6 connection established.") + else: + print("IPv4 connection established.") + + During asynchronous iteration, implicitly-created tasks will be yielded for + supplied awaitables that aren't tasks or futures. + + When used as a plain iterator, each iteration yields a new coroutine that + returns the result or raises the exception of the next completed awaitable. + This pattern is compatible with Python versions older than 3.13:: + + ipv4_connect = create_task(open_connection("127.0.0.1", 80)) + ipv6_connect = create_task(open_connection("::1", 80)) + tasks = [ipv4_connect, ipv6_connect] + + for next_connect in as_completed(tasks): + # next_connect is not one of the original task objects. It must be + # awaited to obtain the result value or raise the exception of the + # awaitable that finishes next. + reader, writer = await next_connect + + A :exc:`TimeoutError` is raised if the timeout occurs before all awaitables + are done. This is raised by the ``async for`` loop during asynchronous + iteration or by the coroutines yielded during plain iteration. .. versionchanged:: 3.10 Removed the *loop* parameter. @@ -884,6 +990,10 @@ Waiting Primitives .. versionchanged:: 3.12 Added support for generators yielding tasks. + .. versionchanged:: 3.13 + The result can now be used as either an :term:`asynchronous iterator` + or as a plain :term:`iterator` (previously it was only a plain iterator). + Running in Threads ================== @@ -957,14 +1067,59 @@ Scheduling From Other Threads This function is meant to be called from a different OS thread than the one where the event loop is running. Example:: - # Create a coroutine - coro = asyncio.sleep(1, result=3) + def in_thread(loop: asyncio.AbstractEventLoop) -> None: + # Run some blocking IO + pathlib.Path("example.txt").write_text("hello world", encoding="utf8") + + # Create a coroutine + coro = asyncio.sleep(1, result=3) + + # Submit the coroutine to a given loop + future = asyncio.run_coroutine_threadsafe(coro, loop) + + # Wait for the result with an optional timeout argument + assert future.result(timeout=2) == 3 - # Submit the coroutine to a given loop - future = asyncio.run_coroutine_threadsafe(coro, loop) + async def amain() -> None: + # Get the running loop + loop = asyncio.get_running_loop() - # Wait for the result with an optional timeout argument - assert future.result(timeout) == 3 + # Run something in a thread + await asyncio.to_thread(in_thread, loop) + + It's also possible to run the other way around. Example:: + + @contextlib.contextmanager + def loop_in_thread() -> Generator[asyncio.AbstractEventLoop]: + loop_fut = concurrent.futures.Future[asyncio.AbstractEventLoop]() + stop_event = asyncio.Event() + + async def main() -> None: + loop_fut.set_result(asyncio.get_running_loop()) + await stop_event.wait() + + with concurrent.futures.ThreadPoolExecutor(1) as tpe: + complete_fut = tpe.submit(asyncio.run, main()) + for fut in concurrent.futures.as_completed((loop_fut, complete_fut)): + if fut is loop_fut: + loop = loop_fut.result() + try: + yield loop + finally: + loop.call_soon_threadsafe(stop_event.set) + else: + fut.result() + + # Create a loop in another thread + with loop_in_thread() as loop: + # Create a coroutine + coro = asyncio.sleep(1, result=3) + + # Submit the coroutine to a given loop + future = asyncio.run_coroutine_threadsafe(coro, loop) + + # Wait for the result with an optional timeout argument + assert future.result(timeout=2) == 3 If an exception is raised in the coroutine, the returned Future will be notified. It can also be used to cancel the task in @@ -1107,7 +1262,7 @@ Task Object a :exc:`CancelledError` exception. If the Task's result isn't yet available, this method raises - a :exc:`InvalidStateError` exception. + an :exc:`InvalidStateError` exception. .. method:: exception() @@ -1323,10 +1478,19 @@ Task Object with :meth:`uncancel`. :class:`TaskGroup` context managers use :func:`uncancel` in a similar fashion. - If end-user code is, for some reason, suppresing cancellation by + If end-user code is, for some reason, suppressing cancellation by catching :exc:`CancelledError`, it needs to call this method to remove the cancellation state. + When this method decrements the cancellation count to zero, + the method checks if a previous :meth:`cancel` call had arranged + for :exc:`CancelledError` to be thrown into the task. + If it hasn't been thrown yet, that arrangement will be + rescinded (by resetting the internal ``_must_cancel`` flag). + + .. versionchanged:: 3.13 + Changed to rescind pending cancellation requests upon reaching zero. + .. method:: cancelling() Return the number of pending cancellation requests to this Task, i.e., diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 5f33c6813e74c0..7d368dae49dc1d 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -1,5 +1,5 @@ -:mod:`asyncio` --- Asynchronous I/O -=================================== +:mod:`!asyncio` --- Asynchronous I/O +==================================== .. module:: asyncio :synopsis: Asynchronous I/O. @@ -56,9 +56,13 @@ Additionally, there are **low-level** APIs for * :ref:`bridge ` callback-based libraries and code with async/await syntax. +.. include:: ../includes/wasm-notavail.rst + .. _asyncio-cli: -You can experiment with an ``asyncio`` concurrent context in the REPL: +.. rubric:: asyncio REPL + +You can experiment with an ``asyncio`` concurrent context in the :term:`REPL`: .. code-block:: pycon @@ -70,7 +74,14 @@ You can experiment with an ``asyncio`` concurrent context in the REPL: >>> await asyncio.sleep(10, result='hello') 'hello' -.. include:: ../includes/wasm-notavail.rst +.. audit-event:: cpython.run_stdin "" "" + +.. versionchanged:: 3.12.5 (also 3.11.10, 3.10.15, 3.9.20, and 3.8.20) + Emits audit events. + +.. versionchanged:: 3.13 + Uses PyREPL if possible, in which case :envvar:`PYTHONSTARTUP` is + also executed. Emits audit events. .. We use the "rubric" directive here to avoid creating the "Reference" subsection in the TOC. @@ -88,6 +99,7 @@ You can experiment with an ``asyncio`` concurrent context in the REPL: asyncio-subprocess.rst asyncio-queue.rst asyncio-exceptions.rst + asyncio-graph.rst .. toctree:: :caption: Low-level APIs diff --git a/Doc/library/asyncore.rst b/Doc/library/asyncore.rst new file mode 100644 index 00000000000000..22c9881c3cca36 --- /dev/null +++ b/Doc/library/asyncore.rst @@ -0,0 +1,17 @@ +:mod:`!asyncore` --- Asynchronous socket handler +================================================ + +.. module:: asyncore + :synopsis: Removed in 3.12. + :deprecated: + +.. deprecated-removed:: 3.6 3.12 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.12 ` after +being deprecated in Python 3.6. The removal was decided in :pep:`594`. + +Applications should use the :mod:`asyncio` module instead. + +The last version of Python that provided the :mod:`!asyncore` module was +`Python 3.11 `_. diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index 43a8bd2d7cd133..02d2f0807df8f6 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -1,5 +1,5 @@ -:mod:`atexit` --- Exit handlers -=============================== +:mod:`!atexit` --- Exit handlers +================================ .. module:: atexit :synopsis: Register and execute cleanup functions. diff --git a/Doc/library/audioop.rst b/Doc/library/audioop.rst new file mode 100644 index 00000000000000..3bc580b0bd3433 --- /dev/null +++ b/Doc/library/audioop.rst @@ -0,0 +1,15 @@ +:mod:`!audioop` --- Manipulate raw audio data +============================================= + +.. module:: audioop + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!audioop` module was +`Python 3.12 `_. diff --git a/Doc/library/audit_events.rst b/Doc/library/audit_events.rst index 8227a7955bef81..a2a90a00d0cfca 100644 --- a/Doc/library/audit_events.rst +++ b/Doc/library/audit_events.rst @@ -7,7 +7,7 @@ Audit events table This table contains all events raised by :func:`sys.audit` or :c:func:`PySys_Audit` calls throughout the CPython runtime and the -standard library. These calls were added in 3.8.0 or later (see :pep:`578`). +standard library. These calls were added in 3.8 or later (see :pep:`578`). See :func:`sys.addaudithook` and :c:func:`PySys_AddAuditHook` for information on handling these events. diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index e596893358f3fb..834ab2536e6c14 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -1,5 +1,5 @@ -:mod:`base64` --- Base16, Base32, Base64, Base85 Data Encodings -=============================================================== +:mod:`!base64` --- Base16, Base32, Base64, Base85 Data Encodings +================================================================ .. module:: base64 :synopsis: RFC 4648: Base16, Base32, Base64 Data Encodings; @@ -193,7 +193,7 @@ The modern interface provides: *wrapcol* controls whether the output should have newline (``b'\n'``) characters added to it. If this is non-zero, each output line will be - at most this many characters long. + at most this many characters long, excluding the trailing newline. *pad* controls whether the input is padded to a multiple of 4 before encoding. Note that the ``btoa`` implementation always pads. diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index 7bf4308a96d0f5..85df7914a9a014 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -1,5 +1,5 @@ -:mod:`bdb` --- Debugger framework -================================= +:mod:`!bdb` --- Debugger framework +================================== .. module:: bdb :synopsis: Debugger framework. @@ -86,7 +86,7 @@ The :mod:`bdb` module also defines two classes: .. attribute:: temporary - True if a :class:`Breakpoint` at (file, line) is temporary. + ``True`` if a :class:`Breakpoint` at (file, line) is temporary. .. attribute:: cond @@ -99,7 +99,7 @@ The :mod:`bdb` module also defines two classes: .. attribute:: enabled - True if :class:`Breakpoint` is enabled. + ``True`` if :class:`Breakpoint` is enabled. .. attribute:: bpbynumber @@ -215,22 +215,22 @@ The :mod:`bdb` module also defines two classes: .. method:: is_skipped_line(module_name) - Return True if *module_name* matches any skip pattern. + Return ``True`` if *module_name* matches any skip pattern. .. method:: stop_here(frame) - Return True if *frame* is below the starting frame in the stack. + Return ``True`` if *frame* is below the starting frame in the stack. .. method:: break_here(frame) - Return True if there is an effective breakpoint for this line. + Return ``True`` if there is an effective breakpoint for this line. Check whether a line or function breakpoint exists and is in effect. Delete temporary breakpoints based on information from :func:`effective`. .. method:: break_anywhere(frame) - Return True if any breakpoint exists for *frame*'s filename. + Return ``True`` if any breakpoint exists for *frame*'s filename. Derived classes should override these methods to gain control over debugger operation. @@ -240,6 +240,9 @@ The :mod:`bdb` module also defines two classes: Called from :meth:`dispatch_call` if a break might stop inside the called function. + *argument_list* is not used anymore and will always be ``None``. + The argument is kept for backwards compatibility. + .. method:: user_line(frame) Called from :meth:`dispatch_line` when either :meth:`stop_here` or @@ -286,6 +289,10 @@ The :mod:`bdb` module also defines two classes: Start debugging from *frame*. If *frame* is not specified, debugging starts from caller's frame. + .. versionchanged:: 3.13 + :func:`set_trace` will enter the debugger immediately, rather than + on the next line of code to be executed. + .. method:: set_continue() Stop only at breakpoints or when finished. If there are no breakpoints, @@ -341,7 +348,7 @@ The :mod:`bdb` module also defines two classes: .. method:: get_break(filename, lineno) - Return True if there is a breakpoint for *lineno* in *filename*. + Return ``True`` if there is a breakpoint for *lineno* in *filename*. .. method:: get_breaks(filename, lineno) @@ -405,7 +412,7 @@ Finally, the module defines the following functions: .. function:: checkfuncname(b, frame) - Return True if we should break here, depending on the way the + Return ``True`` if we should break here, depending on the way the :class:`Breakpoint` *b* was set. If it was set via line number, it checks if @@ -424,14 +431,14 @@ Finally, the module defines the following functions: :attr:`bplist ` for the (:attr:`file `, :attr:`line `) (which must exist) that is :attr:`enabled `, for - which :func:`checkfuncname` is True, and that has neither a False + which :func:`checkfuncname` is true, and that has neither a false :attr:`condition ` nor positive :attr:`ignore ` count. The *flag*, meaning that a - temporary breakpoint should be deleted, is False only when the + temporary breakpoint should be deleted, is ``False`` only when the :attr:`cond ` cannot be evaluated (in which case, :attr:`ignore ` count is ignored). - If no such entry exists, then (None, None) is returned. + If no such entry exists, then ``(None, None)`` is returned. .. function:: set_trace() diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 39fabb59bb1984..1bab785684bbab 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -1,5 +1,5 @@ -:mod:`binascii` --- Convert between binary and ASCII -==================================================== +:mod:`!binascii` --- Convert between binary and ASCII +===================================================== .. module:: binascii :synopsis: Tools for converting between binary and various ASCII-encoded binary diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index 31c79b91061591..78da563397b625 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -1,5 +1,5 @@ -:mod:`bisect` --- Array bisection algorithm -=========================================== +:mod:`!bisect` --- Array bisection algorithm +============================================ .. module:: bisect :synopsis: Array bisection algorithms for binary searching. diff --git a/Doc/library/builtins.rst b/Doc/library/builtins.rst index 7e4f8fe0531567..c4979db52d2aed 100644 --- a/Doc/library/builtins.rst +++ b/Doc/library/builtins.rst @@ -1,5 +1,5 @@ -:mod:`builtins` --- Built-in objects -==================================== +:mod:`!builtins` --- Built-in objects +===================================== .. module:: builtins :synopsis: The module that provides the built-in namespace. @@ -7,10 +7,7 @@ -------------- This module provides direct access to all 'built-in' identifiers of Python; for -example, ``builtins.open`` is the full name for the built-in function -:func:`open`. See :ref:`built-in-funcs` and :ref:`built-in-consts` for -documentation. - +example, ``builtins.open`` is the full name for the built-in function :func:`open`. This module is not normally accessed explicitly by most applications, but can be useful in modules that provide objects with the same name as a built-in value, @@ -40,3 +37,10 @@ available as part of their globals. The value of ``__builtins__`` is normally either this module or the value of this module's :attr:`~object.__dict__` attribute. Since this is an implementation detail, it may not be used by alternate implementations of Python. + +.. seealso:: + + * :ref:`built-in-consts` + * :ref:`bltin-exceptions` + * :ref:`built-in-funcs` + * :ref:`bltin-types` diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index 6a95a4a6e8d183..ebe2e43febaefa 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -1,5 +1,5 @@ -:mod:`bz2` --- Support for :program:`bzip2` compression -======================================================= +:mod:`!bz2` --- Support for :program:`bzip2` compression +======================================================== .. module:: bz2 :synopsis: Interfaces for bzip2 compression and decompression. @@ -91,7 +91,7 @@ The :mod:`bz2` module contains: and :meth:`~io.IOBase.truncate`. Iteration and the :keyword:`with` statement are supported. - :class:`BZ2File` also provides the following methods: + :class:`BZ2File` also provides the following methods and attributes: .. method:: peek([n]) @@ -148,6 +148,19 @@ The :mod:`bz2` module contains: .. versionadded:: 3.3 + .. attribute:: mode + + ``'rb'`` for reading and ``'wb'`` for writing. + + .. versionadded:: 3.13 + + .. attribute:: name + + The bzip2 file name. Equivalent to the :attr:`~io.FileIO.name` + attribute of the underlying :term:`file object`. + + .. versionadded:: 3.13 + .. versionchanged:: 3.1 Support for the :keyword:`with` statement was added. @@ -156,7 +169,6 @@ The :mod:`bz2` module contains: Support was added for *filename* being a :term:`file object` instead of an actual filename. - .. versionchanged:: 3.3 The ``'a'`` (append) mode was added, along with support for reading multi-stream files. diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index e699a7284ac802..1c6b5e03af3560 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -1,5 +1,5 @@ -:mod:`calendar` --- General calendar-related functions -====================================================== +:mod:`!calendar` --- General calendar-related functions +======================================================= .. module:: calendar :synopsis: Functions for working with calendars, including some emulation @@ -38,13 +38,33 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is itself. This is the job of subclasses. - :class:`Calendar` instances have the following methods: + :class:`Calendar` instances have the following methods and attributes: + + .. attribute:: firstweekday + + The first weekday as an integer (0--6). + + This property can also be set and read using + :meth:`~Calendar.setfirstweekday` and + :meth:`~Calendar.getfirstweekday` respectively. + + .. method:: getfirstweekday() + + Return an :class:`int` for the current first weekday (0--6). + + Identical to reading the :attr:`~Calendar.firstweekday` property. + + .. method:: setfirstweekday(firstweekday) + + Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6) + + Identical to setting the :attr:`~Calendar.firstweekday` property. .. method:: iterweekdays() Return an iterator for the week day numbers that will be used for one week. The first value from the iterator will be the same as the value of - the :attr:`firstweekday` property. + the :attr:`~Calendar.firstweekday` property. .. method:: itermonthdates(year, month) @@ -138,7 +158,42 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is :class:`TextCalendar` instances have the following methods: - .. method:: formatmonth(theyear, themonth, w=0, l=0) + + .. method:: formatday(theday, weekday, width) + + Return a string representing a single day formatted with the given *width*. + If *theday* is ``0``, return a string of spaces of + the specified width, representing an empty day. The *weekday* parameter + is unused. + + .. method:: formatweek(theweek, w=0, highlight_day=None) + + Return a single week in a string with no newline. If *w* is provided, it + specifies the width of the date columns, which are centered. Depends + on the first weekday as specified in the constructor or set by the + :meth:`setfirstweekday` method. + + .. versionchanged:: 3.14 + If *highlight_day* is given, this date is highlighted in color. + This can be :ref:`controlled using environment variables + `. + + + .. method:: formatweekday(weekday, width) + + Return a string representing the name of a single weekday formatted to + the specified *width*. The *weekday* parameter is an integer representing + the day of the week, where ``0`` is Monday and ``6`` is Sunday. + + + .. method:: formatweekheader(width) + + Return a string containing the header row of weekday names, formatted + with the given *width* for each column. The names depend on the locale + settings and are padded to the specified width. + + + .. method:: formatmonth(theyear, themonth, w=0, l=0, highlight_day=None) Return a month's calendar in a multi-line string. If *w* is provided, it specifies the width of the date columns, which are centered. If *l* is @@ -146,13 +201,26 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is on the first weekday as specified in the constructor or set by the :meth:`setfirstweekday` method. + .. versionchanged:: 3.14 + If *highlight_day* is given, this date is highlighted in color. + This can be :ref:`controlled using environment variables + `. + + + .. method:: formatmonthname(theyear, themonth, width=0, withyear=True) + + Return a string representing the month's name centered within the + specified *width*. If *withyear* is ``True``, include the year in the + output. The *theyear* and *themonth* parameters specify the year + and month for the name to be formatted respectively. + .. method:: prmonth(theyear, themonth, w=0, l=0) Print a month's calendar as returned by :meth:`formatmonth`. - .. method:: formatyear(theyear, w=2, l=1, c=6, m=3) + .. method:: formatyear(theyear, w=2, l=1, c=6, m=3, highlight_day=None) Return a *m*-column calendar for an entire year as a multi-line string. Optional parameters *w*, *l*, and *c* are for date column width, lines per @@ -161,6 +229,11 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is :meth:`setfirstweekday` method. The earliest year for which a calendar can be generated is platform-dependent. + .. versionchanged:: 3.14 + If *highlight_day* is given, this date is highlighted in color. + This can be :ref:`controlled using environment variables + `. + .. method:: pryear(theyear, w=2, l=1, c=6, m=3) @@ -393,13 +466,22 @@ The :mod:`calendar` module exports the following data attributes: .. data:: day_name - An array that represents the days of the week in the current locale. + A sequence that represents the days of the week in the current locale, + where Monday is day number 0. + + >>> import calendar + >>> list(calendar.day_name) + ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'] .. data:: day_abbr - An array that represents the abbreviated days of the week in the current locale. + A sequence that represents the abbreviated days of the week in the current locale, + where Mon is day number 0. + >>> import calendar + >>> list(calendar.day_abbr) + ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] .. data:: MONDAY TUESDAY @@ -426,17 +508,24 @@ The :mod:`calendar` module exports the following data attributes: .. data:: month_name - An array that represents the months of the year in the current locale. This + A sequence that represents the months of the year in the current locale. This follows normal convention of January being month number 1, so it has a length of - 13 and ``month_name[0]`` is the empty string. + 13 and ``month_name[0]`` is the empty string. + + >>> import calendar + >>> list(calendar.month_name) + ['', 'January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'] .. data:: month_abbr - An array that represents the abbreviated months of the year in the current + A sequence that represents the abbreviated months of the year in the current locale. This follows normal convention of January being month number 1, so it has a length of 13 and ``month_abbr[0]`` is the empty string. + >>> import calendar + >>> list(calendar.month_abbr) + ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] .. data:: JANUARY FEBRUARY @@ -500,7 +589,7 @@ The :mod:`calendar` module defines the following exceptions: .. _calendar-cli: -Command-Line Usage +Command-line usage ------------------ .. versionadded:: 2.5 @@ -638,6 +727,9 @@ The following options are accepted: The number of months printed per row. Defaults to 3. +.. versionchanged:: 3.14 + By default, today's date is highlighted in color and can be + :ref:`controlled using environment variables `. *HTML-mode options:* diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst new file mode 100644 index 00000000000000..f9108fa954a906 --- /dev/null +++ b/Doc/library/cgi.rst @@ -0,0 +1,19 @@ +:mod:`!cgi` --- Common Gateway Interface support +================================================ + +.. module:: cgi + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +A fork of the module on PyPI can be used instead: :pypi:`legacy-cgi`. +This is a copy of the cgi module, no longer maintained or supported by the core +Python team. + +The last version of Python that provided the :mod:`!cgi` module was +`Python 3.12 `_. diff --git a/Doc/library/cgitb.rst b/Doc/library/cgitb.rst new file mode 100644 index 00000000000000..fc646aa4c48acd --- /dev/null +++ b/Doc/library/cgitb.rst @@ -0,0 +1,19 @@ +:mod:`!cgitb` --- Traceback manager for CGI scripts +=================================================== + +.. module:: cgitb + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +A fork of the module on PyPI can now be used instead: :pypi:`legacy-cgi`. +This is a copy of the cgi module, no longer maintained or supported by the core +Python team. + +The last version of Python that provided the :mod:`!cgitb` module was +`Python 3.12 `_. diff --git a/Doc/library/chunk.rst b/Doc/library/chunk.rst new file mode 100644 index 00000000000000..9950a0ea70649a --- /dev/null +++ b/Doc/library/chunk.rst @@ -0,0 +1,15 @@ +:mod:`!chunk` --- Read IFF chunked data +======================================= + +.. module:: chunk + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!chunk` module was +`Python 3.12 `_. diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index fdac51d9603ceb..e7c027dd4d0c22 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -1,5 +1,5 @@ -:mod:`cmath` --- Mathematical functions for complex numbers -=========================================================== +:mod:`!cmath` --- Mathematical functions for complex numbers +============================================================ .. module:: cmath :synopsis: Mathematical functions for complex numbers. @@ -24,17 +24,17 @@ the function is then applied to the result of the conversion. imaginary axis we look at the sign of the real part. For example, the :func:`cmath.sqrt` function has a branch cut along the - negative real axis. An argument of ``complex(-2.0, -0.0)`` is treated as + negative real axis. An argument of ``-2-0j`` is treated as though it lies *below* the branch cut, and so gives a result on the negative imaginary axis:: - >>> cmath.sqrt(complex(-2.0, -0.0)) + >>> cmath.sqrt(-2-0j) -1.4142135623730951j - But an argument of ``complex(-2.0, 0.0)`` is treated as though it lies above + But an argument of ``-2+0j`` is treated as though it lies above the branch cut:: - >>> cmath.sqrt(complex(-2.0, 0.0)) + >>> cmath.sqrt(-2+0j) 1.4142135623730951j @@ -43,10 +43,7 @@ Conversions to and from polar coordinates A Python complex number ``z`` is stored internally using *rectangular* or *Cartesian* coordinates. It is completely determined by its *real -part* ``z.real`` and its *imaginary part* ``z.imag``. In other -words:: - - z == z.real + z.imag*1j +part* ``z.real`` and its *imaginary part* ``z.imag``. *Polar coordinates* give an alternative way to represent a complex number. In polar coordinates, a complex number *z* is defined by the @@ -66,9 +63,9 @@ rectangular coordinates to polar coordinates and back. along the negative real axis. The sign of the result is the same as the sign of ``x.imag``, even when ``x.imag`` is zero:: - >>> phase(complex(-1.0, 0.0)) + >>> phase(-1+0j) 3.141592653589793 - >>> phase(complex(-1.0, -0.0)) + >>> phase(-1-0j) -3.141592653589793 @@ -90,7 +87,7 @@ rectangular coordinates to polar coordinates and back. .. function:: rect(r, phi) Return the complex number *x* with polar coordinates *r* and *phi*. - Equivalent to ``r * (math.cos(phi) + math.sin(phi)*1j)``. + Equivalent to ``complex(r * math.cos(phi), r * math.sin(phi))``. Power and logarithmic functions @@ -224,19 +221,21 @@ Classification functions ``False`` otherwise. Whether or not two values are considered close is determined according to - given absolute and relative tolerances. + given absolute and relative tolerances. If no errors occur, the result will + be: ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``. *rel_tol* is the relative tolerance -- it is the maximum allowed difference between *a* and *b*, relative to the larger absolute value of *a* or *b*. For example, to set a tolerance of 5%, pass ``rel_tol=0.05``. The default tolerance is ``1e-09``, which assures that the two values are the same - within about 9 decimal digits. *rel_tol* must be greater than zero. - - *abs_tol* is the minimum absolute tolerance -- useful for comparisons near - zero. *abs_tol* must be at least zero. - - If no errors occur, the result will be: - ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``. + within about 9 decimal digits. *rel_tol* must be nonnegative and less + than ``1.0``. + + *abs_tol* is the absolute tolerance; it defaults to ``0.0`` and it must be + nonnegative. When comparing ``x`` to ``0.0``, ``isclose(x, 0)`` is computed + as ``abs(x) <= rel_tol * abs(x)``, which is ``False`` for any ``x`` and + rel_tol less than ``1.0``. So add an appropriate positive abs_tol argument + to the call. The IEEE 754 special values of ``NaN``, ``inf``, and ``-inf`` will be handled according to IEEE rules. Specifically, ``NaN`` is not considered diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst index 39ef4b481478d1..66544f82f6ff3f 100644 --- a/Doc/library/cmd.rst +++ b/Doc/library/cmd.rst @@ -1,5 +1,5 @@ -:mod:`cmd` --- Support for line-oriented command interpreters -============================================================= +:mod:`!cmd` --- Support for line-oriented command interpreters +============================================================== .. module:: cmd :synopsis: Build line-oriented command interpreters. diff --git a/Doc/library/cmdline.rst b/Doc/library/cmdline.rst index b2379befeffcba..78fe95a014ff7c 100644 --- a/Doc/library/cmdline.rst +++ b/Doc/library/cmdline.rst @@ -23,7 +23,7 @@ The following modules have a command-line interface. * :ref:`http.server ` * :mod:`!idlelib` * :ref:`inspect ` -* :ref:`json.tool ` +* :ref:`json ` * :mod:`mimetypes` * :mod:`pdb` * :mod:`pickle` @@ -36,9 +36,11 @@ The following modules have a command-line interface. * :mod:`pyclbr` * :mod:`pydoc` * :mod:`quopri` +* :ref:`random ` * :mod:`runpy` * :ref:`site ` * :ref:`sqlite3 ` +* :ref:`symtable ` * :ref:`sysconfig ` * :mod:`tabnanny` * :ref:`tarfile ` diff --git a/Doc/library/cmdlinelibs.rst b/Doc/library/cmdlinelibs.rst new file mode 100644 index 00000000000000..085d31af7bca1f --- /dev/null +++ b/Doc/library/cmdlinelibs.rst @@ -0,0 +1,21 @@ +.. _cmdlinelibs: + +******************************** +Command Line Interface Libraries +******************************** + +The modules described in this chapter assist with implementing +command line and terminal interfaces for applications. + +Here's an overview: + +.. toctree:: + :maxdepth: 1 + + argparse.rst + optparse.rst + getpass.rst + fileinput.rst + curses.rst + curses.ascii.rst + curses.panel.rst diff --git a/Doc/library/code.rst b/Doc/library/code.rst index 091840781bd235..8f7692df9fb22d 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -1,5 +1,5 @@ -:mod:`code` --- Interpreter base classes -======================================== +:mod:`!code` --- Interpreter base classes +========================================= .. module:: code :synopsis: Facilities to implement read-eval-print loops. @@ -18,16 +18,16 @@ build applications which provide an interactive interpreter prompt. This class deals with parsing and interpreter state (the user's namespace); it does not deal with input buffering or prompting or input file naming (the filename is always passed in explicitly). The optional *locals* argument - specifies the dictionary in which code will be executed; it defaults to a newly - created dictionary with key ``'__name__'`` set to ``'__console__'`` and key - ``'__doc__'`` set to ``None``. + specifies a mapping to use as the namespace in which code will be executed; + it defaults to a newly created dictionary with key ``'__name__'`` set to + ``'__console__'`` and key ``'__doc__'`` set to ``None``. .. class:: InteractiveConsole(locals=None, filename="", local_exit=False) Closely emulate the behavior of the interactive Python interpreter. This class builds on :class:`InteractiveInterpreter` and adds prompting using the familiar - ``sys.ps1`` and ``sys.ps2``, and input buffering. If *local_exit* is True, + ``sys.ps1`` and ``sys.ps2``, and input buffering. If *local_exit* is true, ``exit()`` and ``quit()`` in the console will not raise :exc:`SystemExit`, but instead return to the calling code. @@ -41,7 +41,7 @@ build applications which provide an interactive interpreter prompt. the :meth:`InteractiveConsole.raw_input` method, if provided. If *local* is provided, it is passed to the :class:`InteractiveConsole` constructor for use as the default namespace for the interpreter loop. If *local_exit* is provided, - it is passed to the :class:`InteractiveConsole` constructor. The :meth:`interact` + it is passed to the :class:`InteractiveConsole` constructor. The :meth:`~InteractiveConsole.interact` method of the instance is then run with *banner* and *exitmsg* passed as the banner and exit message to use, if provided. The console object is discarded after use. diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index a757f19b99448c..a129a26190ba99 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1,5 +1,5 @@ -:mod:`codecs` --- Codec registry and base classes -================================================= +:mod:`!codecs` --- Codec registry and base classes +================================================== .. module:: codecs :synopsis: Encode and decode data and streams. @@ -1042,6 +1042,10 @@ is meant to be exhaustive. Notice that spelling alternatives that only differ in case or use a hyphen instead of an underscore are also valid aliases; therefore, e.g. ``'utf-8'`` is a valid alias for the ``'utf_8'`` codec. +On Windows, ``cpXXX`` codecs are available for all code pages. +But only codecs listed in the following table are guarantead to exist on +other platforms. + .. impl-detail:: Some common encodings can bypass the codecs lookup machinery to @@ -1307,6 +1311,9 @@ particular, the following variants typically exist: .. versionchanged:: 3.8 ``cp65001`` is now an alias to ``utf_8``. +.. versionchanged:: 3.14 + On Windows, ``cpXXX`` codecs are now available for all code pages. + Python Specific Encodings ------------------------- @@ -1478,7 +1485,7 @@ Internationalized Domain Names (IDN)). It builds upon the ``punycode`` encoding and :mod:`stringprep`. If you need the IDNA 2008 standard from :rfc:`5891` and :rfc:`5895`, use the -third-party `idna module `_. +third-party :pypi:`idna` module. These RFCs together define a protocol to support non-ASCII characters in domain names. A domain name containing non-ASCII characters (such as diff --git a/Doc/library/codeop.rst b/Doc/library/codeop.rst index 55606e1c5f09ac..16f674adb4b22b 100644 --- a/Doc/library/codeop.rst +++ b/Doc/library/codeop.rst @@ -1,5 +1,5 @@ -:mod:`codeop` --- Compile Python code -===================================== +:mod:`!codeop` --- Compile Python code +====================================== .. module:: codeop :synopsis: Compile (possibly incomplete) Python code. diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 7bcaba60c6ddbd..42e40de152148c 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -1,5 +1,5 @@ -:mod:`collections.abc` --- Abstract Base Classes for Containers -=============================================================== +:mod:`!collections.abc` --- Abstract Base Classes for Containers +================================================================ .. module:: collections.abc :synopsis: Abstract base classes for containers @@ -141,12 +141,10 @@ ABC Inherits from Abstract Methods Mi ``__len__``, ``insert`` -:class:`ByteString` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods - ``__len__`` - :class:`Set` :class:`Collection` ``__contains__``, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``, ``__iter__``, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``, - ``__len__`` ``__sub__``, ``__xor__``, and ``isdisjoint`` + ``__len__`` ``__sub__``, ``__rsub__``, ``__xor__``, ``__rxor__`` + and ``isdisjoint`` :class:`MutableSet` :class:`Set` ``__contains__``, Inherited :class:`Set` methods and ``__iter__``, ``clear``, ``pop``, ``remove``, ``__ior__``, @@ -165,7 +163,7 @@ ABC Inherits from Abstract Methods Mi ``__len__`` -:class:`MappingView` :class:`Sized` ``__len__`` +:class:`MappingView` :class:`Sized` ``__init__``, ``__len__`` and ``__repr__`` :class:`ItemsView` :class:`MappingView`, ``__contains__``, :class:`Set` ``__iter__`` :class:`KeysView` :class:`MappingView`, ``__contains__``, @@ -216,6 +214,9 @@ Collections Abstract Base Classes -- Detailed Descriptions ABC for classes that provide the :meth:`~object.__call__` method. + See :ref:`annotating-callables` for details on how to use + :class:`!Callable` in type annotations. + .. class:: Iterable ABC for classes that provide the :meth:`~container.__iter__` method. @@ -253,11 +254,13 @@ Collections Abstract Base Classes -- Detailed Descriptions :meth:`~generator.send`, :meth:`~generator.throw` and :meth:`~generator.close` methods. + See :ref:`annotating-generators-and-coroutines` + for details on using :class:`!Generator` in type annotations. + .. versionadded:: 3.5 .. class:: Sequence MutableSequence - ByteString ABCs for read-only and mutable :term:`sequences `. @@ -274,12 +277,6 @@ Collections Abstract Base Classes -- Detailed Descriptions The index() method added support for *stop* and *start* arguments. - .. deprecated-removed:: 3.12 3.14 - The :class:`ByteString` ABC has been deprecated. - For use in typing, prefer a union, like ``bytes | bytearray``, or - :class:`collections.abc.Buffer`. - For use as an ABC, prefer :class:`Sequence` or :class:`collections.abc.Buffer`. - .. class:: Set MutableSet @@ -331,6 +328,11 @@ Collections Abstract Base Classes -- Detailed Descriptions Using ``isinstance(gencoro, Coroutine)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. + See :ref:`annotating-generators-and-coroutines` + for details on using :class:`!Coroutine` in type annotations. + The variance and order of type parameters correspond to those of + :class:`Generator`. + .. versionadded:: 3.5 .. class:: AsyncIterable @@ -352,6 +354,9 @@ Collections Abstract Base Classes -- Detailed Descriptions ABC for :term:`asynchronous generator` classes that implement the protocol defined in :pep:`525` and :pep:`492`. + See :ref:`annotating-generators-and-coroutines` + for details on using :class:`!AsyncGenerator` in type annotations. + .. versionadded:: 3.6 .. class:: Buffer diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index c246173c1bbf53..5b4e445762e076 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -1,5 +1,5 @@ -:mod:`collections` --- Container datatypes -========================================== +:mod:`!collections` --- Container datatypes +=========================================== .. module:: collections :synopsis: Container datatypes @@ -99,7 +99,7 @@ The class can be used to simulate nested scopes and is useful in templating. :func:`super` function. A reference to ``d.parents`` is equivalent to: ``ChainMap(*d.maps[1:])``. - Note, the iteration order of a :class:`ChainMap()` is determined by + Note, the iteration order of a :class:`ChainMap` is determined by scanning the mappings last to first:: >>> baseline = {'music': 'bach', 'art': 'rembrandt'} @@ -134,7 +134,7 @@ The class can be used to simulate nested scopes and is useful in templating. :attr:`~collections.ChainMap.parents` property. * The `Nested Contexts recipe - `_ has options to control + `_ has options to control whether writes and other mutations apply only to the first mapping or to any mapping in the chain. @@ -343,7 +343,7 @@ superset relationships: ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``. All of those tests treat missing elements as having zero counts so that ``Counter(a=1) == Counter(a=1, b=0)`` returns true. -.. versionadded:: 3.10 +.. versionchanged:: 3.10 Rich comparison operations were added. .. versionchanged:: 3.10 @@ -783,10 +783,10 @@ sequence of key-value pairs into a dictionary of lists: When each key is encountered for the first time, it is not already in the mapping; so an entry is automatically created using the :attr:`~defaultdict.default_factory` -function which returns an empty :class:`list`. The :meth:`list.append` +function which returns an empty :class:`list`. The :meth:`!list.append` operation then attaches the value to the new list. When keys are encountered again, the look-up proceeds normally (returning the list for that key) and the -:meth:`list.append` operation adds another value to the list. This technique is +:meth:`!list.append` operation adds another value to the list. This technique is simpler and faster than an equivalent technique using :meth:`dict.setdefault`: >>> d = {} @@ -874,8 +874,8 @@ they add the ability to access fields by name instead of position index. ``(1, 2)``, then ``x`` will be a required argument, ``y`` will default to ``1``, and ``z`` will default to ``2``. - If *module* is defined, the ``__module__`` attribute of the named tuple is - set to that value. + If *module* is defined, the :attr:`~type.__module__` attribute of the + named tuple is set to that value. Named tuple instances do not have per-instance dictionaries, so they are lightweight and require no more memory than regular tuples. @@ -1169,8 +1169,11 @@ Some differences from :class:`dict` still remain: In addition to the usual mapping methods, ordered dictionaries also support reverse iteration using :func:`reversed`. +.. _collections_OrderedDict__eq__: + Equality tests between :class:`OrderedDict` objects are order-sensitive -and are implemented as ``list(od1.items())==list(od2.items())``. +and are roughly equivalent to ``list(od1.items())==list(od2.items())``. + Equality tests between :class:`OrderedDict` objects and other :class:`~collections.abc.Mapping` objects are order-insensitive like regular dictionaries. This allows :class:`OrderedDict` objects to be substituted @@ -1186,7 +1189,7 @@ anywhere a regular dictionary is used. method. .. versionchanged:: 3.9 - Added merge (``|``) and update (``|=``) operators, specified in :pep:`584`. + Added merge (``|``) and update (``|=``) operators, specified in :pep:`584`. :class:`OrderedDict` Examples and Recipes diff --git a/Doc/library/colorsys.rst b/Doc/library/colorsys.rst index b672a05b39145d..ffebf4e40dd609 100644 --- a/Doc/library/colorsys.rst +++ b/Doc/library/colorsys.rst @@ -1,5 +1,5 @@ -:mod:`colorsys` --- Conversions between color systems -===================================================== +:mod:`!colorsys` --- Conversions between color systems +====================================================== .. module:: colorsys :synopsis: Conversion functions between RGB and other color systems. @@ -14,7 +14,7 @@ The :mod:`colorsys` module defines bidirectional conversions of color values between colors expressed in the RGB (Red Green Blue) color space used in computer monitors and three other coordinate systems: YIQ, HLS (Hue Lightness Saturation) and HSV (Hue Saturation Value). Coordinates in all of these color -spaces are floating point values. In the YIQ space, the Y coordinate is between +spaces are floating-point values. In the YIQ space, the Y coordinate is between 0 and 1, but the I and Q coordinates can be positive or negative. In all other spaces, the coordinates are all between 0 and 1. diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index df1eefab839cc1..c42288419c4d2d 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -1,5 +1,5 @@ -:mod:`compileall` --- Byte-compile Python libraries -=================================================== +:mod:`!compileall` --- Byte-compile Python libraries +==================================================== .. module:: compileall :synopsis: Tools for byte-compiling all Python source files in a directory tree. @@ -90,7 +90,7 @@ compile Python sources. .. option:: -j N Use *N* workers to compile the files within the given directory. - If ``0`` is used, then the result of :func:`os.process_cpu_count()` + If ``0`` is used, then the result of :func:`os.process_cpu_count` will be used. .. option:: --invalidation-mode [timestamp|checked-hash|unchecked-hash] @@ -226,7 +226,7 @@ Public functions The *invalidation_mode* parameter was added. .. versionchanged:: 3.7.2 - The *invalidation_mode* parameter's default value is updated to None. + The *invalidation_mode* parameter's default value is updated to ``None``. .. versionchanged:: 3.8 Setting *workers* to 0 now chooses the optimal number of cores. @@ -289,7 +289,7 @@ Public functions The *invalidation_mode* parameter was added. .. versionchanged:: 3.7.2 - The *invalidation_mode* parameter's default value is updated to None. + The *invalidation_mode* parameter's default value is updated to ``None``. .. versionchanged:: 3.9 Added *stripdir*, *prependdir*, *limit_sl_dest* and *hardlink_dupes* arguments. @@ -318,7 +318,7 @@ Public functions The *invalidation_mode* parameter was added. .. versionchanged:: 3.7.2 - The *invalidation_mode* parameter's default value is updated to None. + The *invalidation_mode* parameter's default value is updated to ``None``. To force a recompile of all the :file:`.py` files in the :file:`Lib/` subdirectory and all its subdirectories:: diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index d3c7a40aa9d390..5a950081a1c98d 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -1,5 +1,5 @@ -:mod:`concurrent.futures` --- Launching parallel tasks -====================================================== +:mod:`!concurrent.futures` --- Launching parallel tasks +======================================================= .. module:: concurrent.futures :synopsis: Execute computations concurrently using threads or processes. @@ -15,9 +15,10 @@ The :mod:`concurrent.futures` module provides a high-level interface for asynchronously executing callables. The asynchronous execution can be performed with threads, using -:class:`ThreadPoolExecutor`, or separate processes, using -:class:`ProcessPoolExecutor`. Both implement the same interface, which is -defined by the abstract :class:`Executor` class. +:class:`ThreadPoolExecutor` or :class:`InterpreterPoolExecutor`, +or separate processes, using :class:`ProcessPoolExecutor`. +Each implements the same interface, which is defined +by the abstract :class:`Executor` class. .. include:: ../includes/wasm-notavail.rst @@ -63,7 +64,8 @@ Executor Objects setting *chunksize* to a positive integer. For very long iterables, using a large value for *chunksize* can significantly improve performance compared to the default size of 1. With - :class:`ThreadPoolExecutor`, *chunksize* has no effect. + :class:`ThreadPoolExecutor` and :class:`InterpreterPoolExecutor`, + *chunksize* has no effect. .. versionchanged:: 3.5 Added the *chunksize* argument. @@ -206,7 +208,7 @@ ThreadPoolExecutor Example 'http://www.cnn.com/', 'http://europe.wsj.com/', 'http://www.bbc.co.uk/', - 'http://nonexistant-subdomain.python.org/'] + 'http://nonexistent-subdomain.python.org/'] # Retrieve a single page and report the URL and contents def load_url(url, timeout): @@ -227,6 +229,111 @@ ThreadPoolExecutor Example print('%r page is %d bytes' % (url, len(data))) +InterpreterPoolExecutor +----------------------- + +The :class:`InterpreterPoolExecutor` class uses a pool of interpreters +to execute calls asynchronously. It is a :class:`ThreadPoolExecutor` +subclass, which means each worker is running in its own thread. +The difference here is that each worker has its own interpreter, +and runs each task using that interpreter. + +The biggest benefit to using interpreters instead of only threads +is true multi-core parallelism. Each interpreter has its own +:term:`Global Interpreter Lock `, so code +running in one interpreter can run on one CPU core, while code in +another interpreter runs unblocked on a different core. + +The tradeoff is that writing concurrent code for use with multiple +interpreters can take extra effort. However, this is because it +forces you to be deliberate about how and when interpreters interact, +and to be explicit about what data is shared between interpreters. +This results in several benefits that help balance the extra effort, +including true multi-core parallelism, For example, code written +this way can make it easier to reason about concurrency. Another +major benefit is that you don't have to deal with several of the +big pain points of using threads, like race conditions. + +Each worker's interpreter is isolated from all the other interpreters. +"Isolated" means each interpreter has its own runtime state and +operates completely independently. For example, if you redirect +:data:`sys.stdout` in one interpreter, it will not be automatically +redirected any other interpreter. If you import a module in one +interpreter, it is not automatically imported in any other. You +would need to import the module separately in interpreter where +you need it. In fact, each module imported in an interpreter is +a completely separate object from the same module in a different +interpreter, including :mod:`sys`, :mod:`builtins`, +and even ``__main__``. + +Isolation means a mutable object, or other data, cannot be used +by more than one interpreter at the same time. That effectively means +interpreters cannot actually share such objects or data. Instead, +each interpreter must have its own copy, and you will have to +synchronize any changes between the copies manually. Immutable +objects and data, like the builtin singletons, strings, and tuples +of immutable objects, don't have these limitations. + +Communicating and synchronizing between interpreters is most effectively +done using dedicated tools, like those proposed in :pep:`734`. One less +efficient alternative is to serialize with :mod:`pickle` and then send +the bytes over a shared :mod:`socket ` or +:func:`pipe `. + +.. class:: InterpreterPoolExecutor(max_workers=None, thread_name_prefix='', initializer=None, initargs=(), shared=None) + + A :class:`ThreadPoolExecutor` subclass that executes calls asynchronously + using a pool of at most *max_workers* threads. Each thread runs + tasks in its own interpreter. The worker interpreters are isolated + from each other, which means each has its own runtime state and that + they can't share any mutable objects or other data. Each interpreter + has its own :term:`Global Interpreter Lock `, + which means code run with this executor has true multi-core parallelism. + + The optional *initializer* and *initargs* arguments have the same + meaning as for :class:`!ThreadPoolExecutor`: the initializer is run + when each worker is created, though in this case it is run.in + the worker's interpreter. The executor serializes the *initializer* + and *initargs* using :mod:`pickle` when sending them to the worker's + interpreter. + + .. note:: + Functions defined in the ``__main__`` module cannot be pickled + and thus cannot be used. + + .. note:: + The executor may replace uncaught exceptions from *initializer* + with :class:`~concurrent.futures.interpreter.ExecutionFailed`. + + The optional *shared* argument is a :class:`dict` of objects that all + interpreters in the pool share. The *shared* items are added to each + interpreter's ``__main__`` module. Not all objects are shareable. + Shareable objects include the builtin singletons, :class:`str` + and :class:`bytes`, and :class:`memoryview`. See :pep:`734` + for more info. + + Other caveats from parent :class:`ThreadPoolExecutor` apply here. + +:meth:`~Executor.submit` and :meth:`~Executor.map` work like normal, +except the worker serializes the callable and arguments using +:mod:`pickle` when sending them to its interpreter. The worker +likewise serializes the return value when sending it back. + +.. note:: + Functions defined in the ``__main__`` module cannot be pickled + and thus cannot be used. + +When a worker's current task raises an uncaught exception, the worker +always tries to preserve the exception as-is. If that is successful +then it also sets the ``__cause__`` to a corresponding +:class:`~concurrent.futures.interpreter.ExecutionFailed` +instance, which contains a summary of the original exception. +In the uncommon case that the worker is not able to preserve the +original as-is then it directly preserves the corresponding +:class:`~concurrent.futures.interpreter.ExecutionFailed` +instance instead. + + ProcessPoolExecutor ------------------- @@ -286,14 +393,6 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. Added the *initializer* and *initargs* arguments. - .. note:: - The default :mod:`multiprocessing` start method - (see :ref:`multiprocessing-start-methods`) will change away from - *fork* in Python 3.14. Code that requires *fork* be used for their - :class:`ProcessPoolExecutor` should explicitly specify that by - passing a ``mp_context=multiprocessing.get_context("fork")`` - parameter. - .. versionchanged:: 3.11 The *max_tasks_per_child* argument was added to allow users to control the lifetime of workers in the pool. @@ -310,6 +409,12 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. *max_workers* uses :func:`os.process_cpu_count` by default, instead of :func:`os.cpu_count`. + .. versionchanged:: 3.14 + The default process start method (see + :ref:`multiprocessing-start-methods`) changed away from *fork*. If you + require the *fork* start method for :class:`ProcessPoolExecutor` you must + explicitly pass ``mp_context=multiprocessing.get_context("fork")``. + .. _processpoolexecutor-example: ProcessPoolExecutor Example @@ -576,6 +681,26 @@ Exception classes .. versionadded:: 3.7 +.. currentmodule:: concurrent.futures.interpreter + +.. exception:: BrokenInterpreterPool + + Derived from :exc:`~concurrent.futures.thread.BrokenThreadPool`, + this exception class is raised when one of the workers + of a :class:`~concurrent.futures.InterpreterPoolExecutor` + has failed initializing. + + .. versionadded:: 3.14 + +.. exception:: ExecutionFailed + + Raised from :class:`~concurrent.futures.InterpreterPoolExecutor` when + the given initializer fails or from + :meth:`~concurrent.futures.Executor.submit` when there's an uncaught + exception from the submitted task. + + .. versionadded:: 3.14 + .. currentmodule:: concurrent.futures.process .. exception:: BrokenProcessPool diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index 18e5bc20f3f690..ac0f3fca3d72fd 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -1,5 +1,5 @@ -:mod:`configparser` --- Configuration file parser -================================================= +:mod:`!configparser` --- Configuration file parser +================================================== .. module:: configparser :synopsis: Configuration file parser. @@ -54,6 +54,7 @@ can be customized by end users easily. import os os.remove("example.ini") + os.remove("override.ini") Quick Start @@ -147,23 +148,28 @@ case-insensitive and stored in lowercase [1]_. It is possible to read several configurations into a single :class:`ConfigParser`, where the most recently added configuration has the highest priority. Any conflicting keys are taken from the more recent -configuration while the previously existing keys are retained. +configuration while the previously existing keys are retained. The example +below reads in an ``override.ini`` file, which will override any conflicting +keys from the ``example.ini`` file. + +.. code-block:: ini + + [DEFAULT] + ServerAliveInterval = -1 .. doctest:: - >>> another_config = configparser.ConfigParser() - >>> another_config.read('example.ini') - ['example.ini'] - >>> another_config['topsecret.server.example']['Port'] - '50022' - >>> another_config.read_string("[topsecret.server.example]\nPort=48484") - >>> another_config['topsecret.server.example']['Port'] - '48484' - >>> another_config.read_dict({"topsecret.server.example": {"Port": 21212}}) - >>> another_config['topsecret.server.example']['Port'] - '21212' - >>> another_config['topsecret.server.example']['ForwardX11'] - 'no' + >>> config_override = configparser.ConfigParser() + >>> config_override['DEFAULT'] = {'ServerAliveInterval': '-1'} + >>> with open('override.ini', 'w') as configfile: + ... config_override.write(configfile) + ... + >>> config_override = configparser.ConfigParser() + >>> config_override.read(['example.ini', 'override.ini']) + ['example.ini', 'override.ini'] + >>> print(config_override.get('DEFAULT', 'ServerAliveInterval')) + -1 + This behaviour is equivalent to a :meth:`ConfigParser.read` call with several files passed to the *filenames* parameter. @@ -274,6 +280,11 @@ may be treated as parts of multiline values or ignored. By default, a valid section name can be any string that does not contain '\\n'. To change this, see :attr:`ConfigParser.SECTCRE`. +The first section name may be omitted if the parser is configured to allow an +unnamed top level section with ``allow_unnamed_section=True``. In this case, +the keys/values may be retrieved by :const:`UNNAMED_SECTION` as in +``config[UNNAMED_SECTION]``. + Configuration files may include comments, prefixed by specific characters (``#`` and ``;`` by default [1]_). Comments may appear on their own on an otherwise empty line, possibly indented. [1]_ @@ -325,6 +336,27 @@ For example: # Did I mention we can indent comments, too? +.. _unnamed-sections: + +Unnamed Sections +---------------- + +The name of the first section (or unique) may be omitted and values +retrieved by the :const:`UNNAMED_SECTION` attribute. + +.. doctest:: + + >>> config = """ + ... option = value + ... + ... [ Section 2 ] + ... another = val + ... """ + >>> unnamed = configparser.ConfigParser(allow_unnamed_section=True) + >>> unnamed.read_string(config) + >>> unnamed.get(configparser.UNNAMED_SECTION, 'option') + 'value' + Interpolation of values ----------------------- @@ -910,7 +942,13 @@ interpolation if an option used is not defined elsewhere. :: ConfigParser Objects -------------------- -.. class:: ConfigParser(defaults=None, dict_type=dict, allow_no_value=False, delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=configparser.DEFAULTSECT, interpolation=BasicInterpolation(), converters={}) +.. class:: ConfigParser(defaults=None, dict_type=dict, allow_no_value=False, *, \ + delimiters=('=', ':'), comment_prefixes=('#', ';'), \ + inline_comment_prefixes=None, strict=True, \ + empty_lines_in_values=True, \ + default_section=configparser.DEFAULTSECT, \ + interpolation=BasicInterpolation(), converters={}, \ + allow_unnamed_section=False) The main configuration parser. When *defaults* is given, it is initialized into the dictionary of intrinsic defaults. When *dict_type* is given, it @@ -955,9 +993,38 @@ ConfigParser Objects When *converters* is given, it should be a dictionary where each key represents the name of a type converter and each value is a callable implementing the conversion from string to the desired datatype. Every - converter gets its own corresponding :meth:`!get*()` method on the parser + converter gets its own corresponding :meth:`!get*` method on the parser object and section proxies. + When *allow_unnamed_section* is ``True`` (default: ``False``), + the first section name can be omitted. See the + `"Unnamed Sections" section <#unnamed-sections>`_. + + It is possible to read several configurations into a single + :class:`ConfigParser`, where the most recently added configuration has the + highest priority. Any conflicting keys are taken from the more recent + configuration while the previously existing keys are retained. The example + below reads in an ``override.ini`` file, which will override any conflicting + keys from the ``example.ini`` file. + + .. code-block:: ini + + [DEFAULT] + ServerAliveInterval = -1 + + .. doctest:: + + >>> config_override = configparser.ConfigParser() + >>> config_override['DEFAULT'] = {'ServerAliveInterval': '-1'} + >>> with open('override.ini', 'w') as configfile: + ... config_override.write(configfile) + ... + >>> config_override = configparser.ConfigParser() + >>> config_override.read(['example.ini', 'override.ini']) + ['example.ini', 'override.ini'] + >>> print(config_override.get('DEFAULT', 'ServerAliveInterval')) + -1 + .. versionchanged:: 3.1 The default *dict_type* is :class:`collections.OrderedDict`. @@ -970,7 +1037,7 @@ ConfigParser Objects The *converters* argument was added. .. versionchanged:: 3.7 - The *defaults* argument is read with :meth:`read_dict()`, + The *defaults* argument is read with :meth:`read_dict`, providing consistent behavior across the parser: non-string keys and values are implicitly converted to strings. @@ -978,6 +1045,13 @@ ConfigParser Objects The default *dict_type* is :class:`dict`, since it now preserves insertion order. + .. versionchanged:: 3.13 + Raise a :exc:`MultilineContinuationError` when *allow_no_value* is + ``True``, and a key without a value is continued with an indented line. + + .. versionchanged:: 3.13 + The *allow_unnamed_section* argument was added. + .. method:: defaults() Return a dictionary containing the instance-wide defaults. @@ -1123,7 +1197,7 @@ ConfigParser Objects .. method:: getfloat(section, option, *, raw=False, vars=None[, fallback]) A convenience method which coerces the *option* in the specified *section* - to a floating point number. See :meth:`get` for explanation of *raw*, + to a floating-point number. See :meth:`get` for explanation of *raw*, *vars* and *fallback*. @@ -1212,6 +1286,11 @@ ConfigParser Objects names is stripped before :meth:`optionxform` is called. +.. data:: UNNAMED_SECTION + + A special object representing a section name used to reference the unnamed section (see :ref:`unnamed-sections`). + + .. data:: MAX_INTERPOLATION_DEPTH The maximum depth for recursive interpolation for :meth:`~configparser.ConfigParser.get` when the *raw* @@ -1229,18 +1308,30 @@ RawConfigParser Objects comment_prefixes=('#', ';'), \ inline_comment_prefixes=None, strict=True, \ empty_lines_in_values=True, \ - default_section=configparser.DEFAULTSECT[, \ - interpolation]) + default_section=configparser.DEFAULTSECT, \ + interpolation=BasicInterpolation(), converters={}, \ + allow_unnamed_section=False) Legacy variant of the :class:`ConfigParser`. It has interpolation disabled by default and allows for non-string section names, option names, and values via its unsafe ``add_section`` and ``set`` methods, as well as the legacy ``defaults=`` keyword argument handling. + .. versionchanged:: 3.2 + *allow_no_value*, *delimiters*, *comment_prefixes*, *strict*, + *empty_lines_in_values*, *default_section* and *interpolation* were + added. + + .. versionchanged:: 3.5 + The *converters* argument was added. + .. versionchanged:: 3.8 The default *dict_type* is :class:`dict`, since it now preserves insertion order. + .. versionchanged:: 3.13 + The *allow_unnamed_section* argument was added. + .. note:: Consider using :class:`ConfigParser` instead which checks types of the values to be stored internally. If you don't want interpolation, you @@ -1249,13 +1340,19 @@ RawConfigParser Objects .. method:: add_section(section) - Add a section named *section* to the instance. If a section by the given - name already exists, :exc:`DuplicateSectionError` is raised. If the - *default section* name is passed, :exc:`ValueError` is raised. + Add a section named *section* or :const:`UNNAMED_SECTION` to the instance. + + If the given section already exists, :exc:`DuplicateSectionError` is + raised. If the *default section* name is passed, :exc:`ValueError` is + raised. If :const:`UNNAMED_SECTION` is passed and support is disabled, + :exc:`UnnamedSectionDisabledError` is raised. Type of *section* is not checked which lets users create non-string named sections. This behaviour is unsupported and may cause internal errors. + .. versionchanged:: 3.14 + Added support for :const:`UNNAMED_SECTION`. + .. method:: set(section, option, value) @@ -1340,7 +1437,6 @@ Exceptions Exception raised when attempting to parse a file which has no section headers. - .. exception:: ParsingError Exception raised when errors occur attempting to parse a file. @@ -1349,6 +1445,20 @@ Exceptions The ``filename`` attribute and :meth:`!__init__` constructor argument were removed. They have been available using the name ``source`` since 3.2. +.. exception:: MultilineContinuationError + + Exception raised when a key without a corresponding value is continued with + an indented line. + + .. versionadded:: 3.13 + +.. exception:: UnnamedSectionDisabledError + + Exception raised when attempting to use the + :const:`UNNAMED_SECTION` without enabling it. + + .. versionadded:: 3.14 + .. rubric:: Footnotes .. [1] Config parsers allow for heavy customization. If you are interested in diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index 93a7244f87de6b..04bb8c51a3b197 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -53,9 +53,12 @@ A small number of constants live in the built-in namespace. They are: See :exc:`NotImplementedError` for details on when to use it. .. versionchanged:: 3.9 - Evaluating :data:`!NotImplemented` in a boolean context is deprecated. While - it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. - It will raise a :exc:`TypeError` in a future version of Python. + Evaluating :data:`!NotImplemented` in a boolean context was deprecated. + + .. versionchanged:: 3.14 + Evaluating :data:`!NotImplemented` in a boolean context now raises a :exc:`TypeError`. + It previously evaluated to :const:`True` and emitted a :exc:`DeprecationWarning` + since Python 3.9. .. index:: single: ...; ellipsis literal @@ -79,6 +82,8 @@ A small number of constants live in the built-in namespace. They are: :exc:`SyntaxError`), so they can be considered "true" constants. +.. _site-consts: + Constants added by the :mod:`site` module ----------------------------------------- @@ -94,6 +99,13 @@ should not be used in programs. (i.e. EOF) to exit", and when called, raise :exc:`SystemExit` with the specified exit code. +.. data:: help + :noindex: + + Object that when printed, prints the message "Type help() for interactive + help, or help(object) for help about object.", and when called, + acts as described :func:`elsewhere `. + .. data:: copyright credits diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 73e53aec9cbf1c..e8f264f949807d 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -151,9 +151,9 @@ Functions and classes provided: created by :func:`asynccontextmanager` to meet the requirement that context managers support multiple invocations in order to be used as decorators. - .. versionchanged:: 3.10 - Async context managers created with :func:`asynccontextmanager` can - be used as decorators. + .. versionchanged:: 3.10 + Async context managers created with :func:`asynccontextmanager` can + be used as decorators. .. function:: closing(thing) @@ -314,13 +314,15 @@ Functions and classes provided: If the code within the :keyword:`!with` block raises a :exc:`BaseExceptionGroup`, suppressed exceptions are removed from the - group. If any exceptions in the group are not suppressed, a group containing them is re-raised. + group. Any exceptions of the group which are not suppressed are re-raised in + a new group which is created using the original group's :meth:`~BaseExceptionGroup.derive` + method. .. versionadded:: 3.4 .. versionchanged:: 3.12 ``suppress`` now supports suppressing exceptions raised as - part of an :exc:`BaseExceptionGroup`. + part of a :exc:`BaseExceptionGroup`. .. function:: redirect_stdout(new_target) @@ -796,7 +798,7 @@ executing that callback:: if result: stack.pop_all() -This allows the intended cleanup up behaviour to be made explicit up front, +This allows the intended cleanup behaviour to be made explicit up front, rather than requiring a separate flag variable. If a particular application uses this pattern a lot, it can be simplified diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 647832447de946..2b1fb9fdd29cd8 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -1,5 +1,5 @@ -:mod:`contextvars` --- Context Variables -======================================== +:mod:`!contextvars` --- Context Variables +========================================= .. module:: contextvars :synopsis: Context Variables @@ -15,7 +15,7 @@ function and the :class:`~contextvars.Context` class should be used to manage the current context in asynchronous frameworks. Context managers that have state should use Context Variables -instead of :func:`threading.local()` to prevent their state from +instead of :func:`threading.local` to prevent their state from bleeding to other code unexpectedly, when used in concurrent code. See also :pep:`567` for additional details. @@ -144,51 +144,89 @@ Manual Context Management To get a copy of the current context use the :func:`~contextvars.copy_context` function. - Every thread will have a different top-level :class:`~contextvars.Context` - object. This means that a :class:`ContextVar` object behaves in a similar - fashion to :func:`threading.local()` when values are assigned in different - threads. + Each thread has its own effective stack of :class:`!Context` objects. The + :term:`current context` is the :class:`!Context` object at the top of the + current thread's stack. All :class:`!Context` objects in the stacks are + considered to be *entered*. + + *Entering* a context, which can be done by calling its :meth:`~Context.run` + method, makes the context the current context by pushing it onto the top of + the current thread's context stack. + + *Exiting* from the current context, which can be done by returning from the + callback passed to the :meth:`~Context.run` method, restores the current + context to what it was before the context was entered by popping the context + off the top of the context stack. + + Since each thread has its own context stack, :class:`ContextVar` objects + behave in a similar fashion to :func:`threading.local` when values are + assigned in different threads. + + Attempting to enter an already entered context, including contexts entered in + other threads, raises a :exc:`RuntimeError`. + + After exiting a context, it can later be re-entered (from any thread). + + Any changes to :class:`ContextVar` values via the :meth:`ContextVar.set` + method are recorded in the current context. The :meth:`ContextVar.get` + method returns the value associated with the current context. Exiting a + context effectively reverts any changes made to context variables while the + context was entered (if needed, the values can be restored by re-entering the + context). Context implements the :class:`collections.abc.Mapping` interface. .. method:: run(callable, *args, **kwargs) - Execute ``callable(*args, **kwargs)`` code in the context object - the *run* method is called on. Return the result of the execution - or propagate an exception if one occurred. + Enters the Context, executes ``callable(*args, **kwargs)``, then exits the + Context. Returns *callable*'s return value, or propagates an exception if + one occurred. + + Example: + + .. testcode:: + + import contextvars - Any changes to any context variables that *callable* makes will - be contained in the context object:: + var = contextvars.ContextVar('var') + var.set('spam') + print(var.get()) # 'spam' - var = ContextVar('var') - var.set('spam') + ctx = contextvars.copy_context() - def main(): - # 'var' was set to 'spam' before - # calling 'copy_context()' and 'ctx.run(main)', so: - # var.get() == ctx[var] == 'spam' + def main(): + # 'var' was set to 'spam' before + # calling 'copy_context()' and 'ctx.run(main)', so: + print(var.get()) # 'spam' + print(ctx[var]) # 'spam' - var.set('ham') + var.set('ham') - # Now, after setting 'var' to 'ham': - # var.get() == ctx[var] == 'ham' + # Now, after setting 'var' to 'ham': + print(var.get()) # 'ham' + print(ctx[var]) # 'ham' - ctx = copy_context() + # Any changes that the 'main' function makes to 'var' + # will be contained in 'ctx'. + ctx.run(main) - # Any changes that the 'main' function makes to 'var' - # will be contained in 'ctx'. - ctx.run(main) + # The 'main()' function was run in the 'ctx' context, + # so changes to 'var' are contained in it: + print(ctx[var]) # 'ham' - # The 'main()' function was run in the 'ctx' context, - # so changes to 'var' are contained in it: - # ctx[var] == 'ham' + # However, outside of 'ctx', 'var' is still set to 'spam': + print(var.get()) # 'spam' - # However, outside of 'ctx', 'var' is still set to 'spam': - # var.get() == 'spam' + .. testoutput:: + :hide: - The method raises a :exc:`RuntimeError` when called on the same - context object from more than one OS thread, or when called - recursively. + spam + spam + spam + ham + ham + ham + spam .. method:: copy() @@ -254,7 +292,7 @@ client:: # without passing it explicitly to this function. client_addr = client_addr_var.get() - return f'Good bye, client @ {client_addr}\n'.encode() + return f'Good bye, client @ {client_addr}\r\n'.encode() async def handle_request(reader, writer): addr = writer.transport.get_extra_info('socket').getpeername() @@ -268,9 +306,10 @@ client:: print(line) if not line.strip(): break - writer.write(line) - writer.write(render_goodbye()) + writer.write(b'HTTP/1.1 200 OK\r\n') # status line + writer.write(b'\r\n') # headers + writer.write(render_goodbye()) # body writer.close() async def main(): @@ -282,5 +321,6 @@ client:: asyncio.run(main()) - # To test it you can use telnet: + # To test it you can use telnet or curl: # telnet 127.0.0.1 8081 + # curl 127.0.0.1:8081 diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index 74333b2e934814..95b41f988a035b 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -1,5 +1,5 @@ -:mod:`copy` --- Shallow and deep copy operations -================================================ +:mod:`!copy` --- Shallow and deep copy operations +================================================= .. module:: copy :synopsis: Shallow and deep copy operations. diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index 2a28c043f80723..6e3144824ebe91 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -1,5 +1,5 @@ -:mod:`copyreg` --- Register :mod:`pickle` support functions -=========================================================== +:mod:`!copyreg` --- Register :mod:`!pickle` support functions +============================================================= .. module:: copyreg :synopsis: Register pickle support functions. diff --git a/Doc/library/crypt.rst b/Doc/library/crypt.rst new file mode 100644 index 00000000000000..9ff37196ccf69f --- /dev/null +++ b/Doc/library/crypt.rst @@ -0,0 +1,20 @@ +:mod:`!crypt` --- Function to check Unix passwords +================================================== + +.. module:: crypt + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +Applications can use the :mod:`hashlib` module from the standard library. +Other possible replacements are third-party libraries from PyPI: +:pypi:`legacycrypt`, :pypi:`bcrypt`, :pypi:`argon2-cffi`, or :pypi:`passlib`. +These are not supported or maintained by the Python core team. + +The last version of Python that provided the :mod:`!crypt` module was +`Python 3.12 `_. diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 4ee7820585d3a2..533cdf13974be6 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -1,5 +1,5 @@ -:mod:`csv` --- CSV File Reading and Writing -=========================================== +:mod:`!csv` --- CSV File Reading and Writing +============================================ .. module:: csv :synopsis: Write and read tabular data to and from delimited files. @@ -156,8 +156,10 @@ The :mod:`csv` module defines the following classes: The *fieldnames* parameter is a :term:`sequence`. If *fieldnames* is omitted, the values in the first row of file *f* will be used as the - fieldnames. Regardless of how the fieldnames are determined, the - dictionary preserves their original ordering. + fieldnames and will be omitted from the results. If + *fieldnames* is provided, they will be used and the first row will be + included in the results. Regardless of how the fieldnames are determined, + the dictionary preserves their original ordering. If a row has more fields than fieldnames, the remaining data is put in a list and stored with the fieldname specified by *restkey* (which defaults @@ -347,8 +349,8 @@ The :mod:`csv` module defines the following constants: ``None``. This is similar to :data:`QUOTE_ALL`, except that if a field value is ``None`` an empty (unquoted) string is written. - Instructs :class:`reader` objects to interpret an empty (unquoted) field as None and - to otherwise behave as :data:`QUOTE_ALL`. + Instructs :class:`reader` objects to interpret an empty (unquoted) field + as ``None`` and to otherwise behave as :data:`QUOTE_ALL`. .. versionadded:: 3.12 diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index eed18201e3ede0..615138302e1379 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1,5 +1,5 @@ -:mod:`ctypes` --- A foreign function library for Python -======================================================= +:mod:`!ctypes` --- A foreign function library for Python +======================================================== .. module:: ctypes :synopsis: A foreign function library for Python. @@ -51,7 +51,7 @@ function call fails. Here are some examples for Windows. Note that ``msvcrt`` is the MS standard C -library containing most standard C functions, and uses the cdecl calling +library containing most standard C functions, and uses the ``cdecl`` calling convention:: >>> from ctypes import * @@ -93,7 +93,6 @@ Accessing functions from loaded dlls Functions are accessed as attributes of dll objects:: - >>> from ctypes import * >>> libc.printf <_FuncPtr object at 0x...> >>> print(windll.kernel32.GetModuleHandleA) # doctest: +WINDOWS @@ -108,7 +107,7 @@ Functions are accessed as attributes of dll objects:: Note that win32 system dlls like ``kernel32`` and ``user32`` often export ANSI as well as UNICODE versions of a function. The UNICODE version is exported with -an ``W`` appended to the name, while the ANSI version is exported with an ``A`` +a ``W`` appended to the name, while the ANSI version is exported with an ``A`` appended to the name. The win32 ``GetModuleHandle`` function, which returns a *module handle* for a given module name, has the following C prototype, and a macro is used to expose one of them as ``GetModuleHandle`` depending on whether @@ -267,6 +266,20 @@ Fundamental data types (1) The constructor accepts any object with a truth value. +Additionally, if IEC 60559 compatible complex arithmetic (Annex G) is supported +in both C and ``libffi``, the following complex types are available: + ++----------------------------------+---------------------------------+-----------------+ +| ctypes type | C type | Python type | ++==================================+=================================+=================+ +| :class:`c_float_complex` | :c:expr:`float complex` | complex | ++----------------------------------+---------------------------------+-----------------+ +| :class:`c_double_complex` | :c:expr:`double complex` | complex | ++----------------------------------+---------------------------------+-----------------+ +| :class:`c_longdouble_complex` | :c:expr:`long double complex` | complex | ++----------------------------------+---------------------------------+-----------------+ + + All these types can be created by calling them with an optional initializer of the correct type and value:: @@ -384,7 +397,7 @@ as calling functions with a fixed number of parameters. On some platforms, and i particular ARM64 for Apple Platforms, the calling convention for variadic functions is different than that for regular functions. -On those platforms it is required to specify the :attr:`~_FuncPtr.argtypes` +On those platforms it is required to specify the :attr:`~_CFuncPtr.argtypes` attribute for the regular, non-variadic, function arguments: .. code-block:: python3 @@ -392,7 +405,7 @@ attribute for the regular, non-variadic, function arguments: libc.printf.argtypes = [ctypes.c_char_p] Because specifying the attribute does not inhibit portability it is advised to always -specify :attr:`~_FuncPtr.argtypes` for all variadic functions. +specify :attr:`~_CFuncPtr.argtypes` for all variadic functions. .. _ctypes-calling-functions-with-own-custom-data-types: @@ -427,9 +440,9 @@ Specifying the required argument types (function prototypes) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ It is possible to specify the required argument types of functions exported from -DLLs by setting the :attr:`~_FuncPtr.argtypes` attribute. +DLLs by setting the :attr:`~_CFuncPtr.argtypes` attribute. -:attr:`~_FuncPtr.argtypes` must be a sequence of C data types (the :func:`!printf` function is +:attr:`~_CFuncPtr.argtypes` must be a sequence of C data types (the :func:`!printf` function is probably not a good example here, because it takes a variable number and different types of parameters depending on the format string, on the other hand this is quite handy to experiment with this feature):: @@ -454,7 +467,7 @@ prototype for a C function), and tries to convert the arguments to valid types:: If you have defined your own classes which you pass to function calls, you have to implement a :meth:`~_CData.from_param` class method for them to be able to use them -in the :attr:`~_FuncPtr.argtypes` sequence. The :meth:`~_CData.from_param` class method receives +in the :attr:`~_CFuncPtr.argtypes` sequence. The :meth:`~_CData.from_param` class method receives the Python object passed to the function call, it should do a typecheck or whatever is needed to make sure this object is acceptable, and then return the object itself, its :attr:`!_as_parameter_` attribute, or whatever you want to @@ -477,7 +490,7 @@ Return types By default functions are assumed to return the C :c:expr:`int` type. Other -return types can be specified by setting the :attr:`~_FuncPtr.restype` attribute of the +return types can be specified by setting the :attr:`~_CFuncPtr.restype` attribute of the function object. The C prototype of :c:func:`time` is ``time_t time(time_t *)``. Because :c:type:`time_t` @@ -486,7 +499,7 @@ specify the :attr:`!restype` attribute:: >>> libc.time.restype = c_time_t -The argument types can be specified using :attr:`~_FuncPtr.argtypes`:: +The argument types can be specified using :attr:`~_CFuncPtr.argtypes`:: >>> libc.time.argtypes = (POINTER(c_time_t),) @@ -509,7 +522,7 @@ a string pointer and a char, and returns a pointer to a string:: >>> If you want to avoid the :func:`ord("x") ` calls above, you can set the -:attr:`~_FuncPtr.argtypes` attribute, and the second argument will be converted from a +:attr:`~_CFuncPtr.argtypes` attribute, and the second argument will be converted from a single character Python bytes object into a C char: .. doctest:: @@ -528,7 +541,7 @@ single character Python bytes object into a C char: >>> You can also use a callable Python object (a function or a class for example) as -the :attr:`~_FuncPtr.restype` attribute, if the foreign function returns an integer. The +the :attr:`~_CFuncPtr.restype` attribute, if the foreign function returns an integer. The callable will be called with the *integer* the C function returns, and the result of this call will be used as the result of your function call. This is useful to check for error return values and automatically raise an exception:: @@ -556,7 +569,7 @@ get the string representation of an error code, and *returns* an exception. :func:`GetLastError` to retrieve it. Please note that a much more powerful error checking mechanism is available -through the :attr:`~_FuncPtr.errcheck` attribute; +through the :attr:`~_CFuncPtr.errcheck` attribute; see the reference manual for details. @@ -662,14 +675,18 @@ for debugging because they can provide useful information:: guaranteed by the library to work in the general case. Unions and structures with bit-fields should always be passed to functions by pointer. -Structure/union alignment and byte order -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Structure/union layout, alignment and byte order +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, Structure and Union fields are laid out in the same way the C +compiler does it. It is possible to override this behavior entirely by specifying a +:attr:`~Structure._layout_` class attribute in the subclass definition; see +the attribute documentation for details. + +It is possible to specify the maximum alignment for the fields by setting +the :attr:`~Structure._pack_` class attribute to a positive integer. +This matches what ``#pragma pack(n)`` does in MSVC. -By default, Structure and Union fields are aligned in the same way the C -compiler does it. It is possible to override this behavior by specifying a -:attr:`~Structure._pack_` class attribute in the subclass definition. -This must be set to a positive integer and specifies the maximum alignment for the fields. -This is what ``#pragma pack(n)`` also does in MSVC. It is also possible to set a minimum alignment for how the subclass itself is packed in the same way ``#pragma align(n)`` works in MSVC. This can be achieved by specifying a ::attr:`~Structure._align_` class attribute @@ -853,6 +870,36 @@ invalid non-\ ``NULL`` pointers would crash Python):: ValueError: NULL pointer access >>> +.. _ctypes-thread-safety: + +Thread safety without the GIL +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In Python 3.13, the :term:`GIL` may be disabled on :term:`experimental free threaded ` builds. +In ctypes, reads and writes to a single object concurrently is safe, but not across multiple objects: + + .. code-block:: pycon + + >>> number = c_int(42) + >>> pointer_a = pointer(number) + >>> pointer_b = pointer(number) + +In the above, it's only safe for one object to read and write to the address at once if the GIL is disabled. +So, ``pointer_a`` can be shared and written to across multiple threads, but only if ``pointer_b`` +is not also attempting to do the same. If this is an issue, consider using a :class:`threading.Lock` +to synchronize access to memory: + + .. code-block:: pycon + + >>> import threading + >>> lock = threading.Lock() + >>> # Thread 1 + >>> with lock: + ... pointer_a.contents = 24 + >>> # Thread 2 + >>> with lock: + ... pointer_b.contents = 42 + .. _ctypes-type-conversions: @@ -860,7 +907,7 @@ Type conversions ^^^^^^^^^^^^^^^^ Usually, ctypes does strict type checking. This means, if you have -``POINTER(c_int)`` in the :attr:`~_FuncPtr.argtypes` list of a function or as the type of +``POINTER(c_int)`` in the :attr:`~_CFuncPtr.argtypes` list of a function or as the type of a member field in a structure definition, only instances of exactly the same type are accepted. There are some exceptions to this rule, where ctypes accepts other objects. For example, you can pass compatible array instances instead of @@ -881,7 +928,7 @@ pointer types. So, for ``POINTER(c_int)``, ctypes accepts an array of c_int:: >>> In addition, if a function argument is explicitly declared to be a pointer type -(such as ``POINTER(c_int)``) in :attr:`~_FuncPtr.argtypes`, an object of the pointed +(such as ``POINTER(c_int)``) in :attr:`~_CFuncPtr.argtypes`, an object of the pointed type (``c_int`` in this case) can be passed to the function. ctypes will apply the required :func:`byref` conversion in this case automatically. @@ -1335,8 +1382,9 @@ Here are some examples:: 'libbz2.so.1.0' >>> -On macOS, :func:`~ctypes.util.find_library` tries several predefined naming schemes and paths -to locate the library, and returns a full pathname if successful:: +On macOS and Android, :func:`~ctypes.util.find_library` uses the system's +standard naming schemes and paths to locate the library, and returns a full +pathname if successful:: >>> from ctypes.util import find_library >>> find_library("c") @@ -1395,13 +1443,15 @@ way is to instantiate one of the following classes: .. class:: OleDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) - Windows only: Instances of this class represent loaded shared libraries, + Instances of this class represent loaded shared libraries, functions in these libraries use the ``stdcall`` calling convention, and are assumed to return the windows specific :class:`HRESULT` code. :class:`HRESULT` values contain information specifying whether the function call failed or succeeded, together with additional error code. If the return value signals a failure, an :class:`OSError` is automatically raised. + .. availability:: Windows + .. versionchanged:: 3.3 :exc:`WindowsError` used to be raised, which is now an alias of :exc:`OSError`. @@ -1413,14 +1463,17 @@ way is to instantiate one of the following classes: .. class:: WinDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) - Windows only: Instances of this class represent loaded shared libraries, + Instances of this class represent loaded shared libraries, functions in these libraries use the ``stdcall`` calling convention, and are assumed to return :c:expr:`int` by default. + .. availability:: Windows + .. versionchanged:: 3.12 The *name* parameter can now be a :term:`path-like object`. + The Python :term:`global interpreter lock` is released before calling any function exported by these libraries, and reacquired afterwards. @@ -1556,13 +1609,17 @@ These prefabricated library loaders are available: .. data:: windll :noindex: - Windows only: Creates :class:`WinDLL` instances. + Creates :class:`WinDLL` instances. + + .. availability:: Windows .. data:: oledll :noindex: - Windows only: Creates :class:`OleDLL` instances. + Creates :class:`OleDLL` instances. + + .. availability:: Windows .. data:: pydll @@ -1609,10 +1666,20 @@ As explained in the previous section, foreign functions can be accessed as attributes of loaded shared libraries. The function objects created in this way by default accept any number of arguments, accept any ctypes data instances as arguments, and return the default result type specified by the library loader. -They are instances of a private class: +They are instances of a private local class :class:`!_FuncPtr` (not exposed +in :mod:`!ctypes`) which inherits from the private :class:`_CFuncPtr` class: -.. class:: _FuncPtr +.. doctest:: + + >>> import ctypes + >>> lib = ctypes.CDLL(None) + >>> issubclass(lib._FuncPtr, ctypes._CFuncPtr) + True + >>> lib._FuncPtr is ctypes._CFuncPtr + False + +.. class:: _CFuncPtr Base class for C callable foreign functions. @@ -1679,12 +1746,6 @@ They are instances of a private class: and raise an exception if the foreign function call failed. -.. exception:: ArgumentError - - This exception is raised when a foreign function call cannot convert one of the - passed arguments. - - .. audit-event:: ctypes.set_exception code foreign-functions On Windows, when a foreign function call raises a system exception (for @@ -1724,11 +1785,13 @@ See :ref:`ctypes-callback-functions` for examples. .. function:: WINFUNCTYPE(restype, *argtypes, use_errno=False, use_last_error=False) - Windows only: The returned function prototype creates functions that use the + The returned function prototype creates functions that use the ``stdcall`` calling convention. The function will release the GIL during the call. *use_errno* and *use_last_error* have the same meaning as above. + .. availability:: Windows + .. function:: PYFUNCTYPE(restype, *argtypes) @@ -1771,14 +1834,21 @@ different ways, depending on the type and number of the parameters in the call: integer. *name* is name of the COM method. *iid* is an optional pointer to the interface identifier which is used in extended error reporting. + If *iid* is not specified, an :exc:`OSError` is raised if the COM method + call fails. If *iid* is specified, a :exc:`~ctypes.COMError` is raised + instead. + COM methods use a special calling convention: They require a pointer to the COM interface as first argument, in addition to those parameters that are specified in the :attr:`!argtypes` tuple. + .. availability:: Windows + + The optional *paramflags* parameter creates foreign function wrappers with much more functionality than the features described above. -*paramflags* must be a tuple of the same length as :attr:`~_FuncPtr.argtypes`. +*paramflags* must be a tuple of the same length as :attr:`~_CFuncPtr.argtypes`. Each item in this tuple contains further information about a parameter, it must be a tuple containing one, two, or three items. @@ -1849,7 +1919,7 @@ value if there is a single one, or a tuple containing the output parameter values when there are more than one, so the GetWindowRect function now returns a RECT instance, when called. -Output parameters can be combined with the :attr:`~_FuncPtr.errcheck` protocol to do +Output parameters can be combined with the :attr:`~_CFuncPtr.errcheck` protocol to do further output processing and error checking. The win32 ``GetWindowRect`` api function returns a ``BOOL`` to signal success or failure, so this function could do the error checking, and raises an exception when the api call failed:: @@ -1862,7 +1932,7 @@ do the error checking, and raises an exception when the api call failed:: >>> GetWindowRect.errcheck = errcheck >>> -If the :attr:`~_FuncPtr.errcheck` function returns the argument tuple it receives +If the :attr:`~_CFuncPtr.errcheck` function returns the argument tuple it receives unchanged, :mod:`ctypes` continues the normal processing it does on the output parameters. If you want to return a tuple of window coordinates instead of a ``RECT`` instance, you can retrieve the fields in the function and return them @@ -1911,6 +1981,24 @@ Utility functions It behaves similar to ``pointer(obj)``, but the construction is a lot faster. +.. function:: CopyComPointer(src, dst) + + Copies a COM pointer from *src* to *dst* and returns the Windows specific + :c:type:`!HRESULT` value. + + If *src* is not ``NULL``, its ``AddRef`` method is called, incrementing the + reference count. + + In contrast, the reference count of *dst* will not be decremented before + assigning the new value. Unless *dst* is ``NULL``, the caller is responsible + for decrementing the reference count by calling its ``Release`` method when + necessary. + + .. availability:: Windows + + .. versionadded:: 3.14 + + .. function:: cast(obj, type) This function is similar to the cast operator in C. It returns a new instance @@ -1954,17 +2042,21 @@ Utility functions .. function:: DllCanUnloadNow() - Windows only: This function is a hook which allows implementing in-process + This function is a hook which allows implementing in-process COM servers with ctypes. It is called from the DllCanUnloadNow function that the _ctypes extension dll exports. + .. availability:: Windows + .. function:: DllGetClassObject() - Windows only: This function is a hook which allows implementing in-process + This function is a hook which allows implementing in-process COM servers with ctypes. It is called from the DllGetClassObject function that the ``_ctypes`` extension dll exports. + .. availability:: Windows + .. function:: find_library(name) :module: ctypes.util @@ -1980,7 +2072,7 @@ Utility functions .. function:: find_msvcrt() :module: ctypes.util - Windows only: return the filename of the VC runtime library used by Python, + Returns the filename of the VC runtime library used by Python, and by the extension modules. If the name of the library cannot be determined, ``None`` is returned. @@ -1988,20 +2080,27 @@ Utility functions with a call to the ``free(void *)``, it is important that you use the function in the same library that allocated the memory. + .. availability:: Windows + .. function:: FormatError([code]) - Windows only: Returns a textual description of the error code *code*. If no - error code is specified, the last error code is used by calling the Windows - api function GetLastError. + Returns a textual description of the error code *code*. If no error code is + specified, the last error code is used by calling the Windows API function + :func:`GetLastError`. + + .. availability:: Windows .. function:: GetLastError() - Windows only: Returns the last error code set by Windows in the calling thread. + Returns the last error code set by Windows in the calling thread. This function calls the Windows ``GetLastError()`` function directly, it does not return the ctypes-private copy of the error code. + .. availability:: Windows + + .. function:: get_errno() Returns the current value of the ctypes-private copy of the system @@ -2011,11 +2110,14 @@ Utility functions .. function:: get_last_error() - Windows only: returns the current value of the ctypes-private copy of the system + Returns the current value of the ctypes-private copy of the system :data:`!LastError` variable in the calling thread. + .. availability:: Windows + .. audit-event:: ctypes.get_last_error "" ctypes.get_last_error + .. function:: memmove(dst, src, count) Same as the standard C memmove library function: copies *count* bytes from @@ -2064,10 +2166,12 @@ Utility functions .. function:: set_last_error(value) - Windows only: set the current value of the ctypes-private copy of the system + Sets the current value of the ctypes-private copy of the system :data:`!LastError` variable in the calling thread to *value* and return the previous value. + .. availability:: Windows + .. audit-event:: ctypes.set_last_error error ctypes.set_last_error @@ -2077,36 +2181,59 @@ Utility functions Does the same as the C ``sizeof`` operator. -.. function:: string_at(address, size=-1) +.. function:: string_at(ptr, size=-1) - This function returns the C string starting at memory address *address* as a bytes - object. If size is specified, it is used as size, otherwise the string is assumed + Return the byte string at *void \*ptr*. + If *size* is specified, it is used as size, otherwise the string is assumed to be zero-terminated. - .. audit-event:: ctypes.string_at address,size ctypes.string_at + .. audit-event:: ctypes.string_at ptr,size ctypes.string_at .. function:: WinError(code=None, descr=None) - Windows only: this function is probably the worst-named thing in ctypes. It - creates an instance of :exc:`OSError`. If *code* is not specified, - ``GetLastError`` is called to determine the error code. If *descr* is not + Creates an instance of :exc:`OSError`. If *code* is not specified, + :func:`GetLastError` is called to determine the error code. If *descr* is not specified, :func:`FormatError` is called to get a textual description of the error. + .. availability:: Windows + .. versionchanged:: 3.3 An instance of :exc:`WindowsError` used to be created, which is now an alias of :exc:`OSError`. -.. function:: wstring_at(address, size=-1) +.. function:: wstring_at(ptr, size=-1) - This function returns the wide character string starting at memory address - *address* as a string. If *size* is specified, it is used as the number of + Return the wide-character string at *void \*ptr*. + If *size* is specified, it is used as the number of characters of the string, otherwise the string is assumed to be zero-terminated. - .. audit-event:: ctypes.wstring_at address,size ctypes.wstring_at + .. audit-event:: ctypes.wstring_at ptr,size ctypes.wstring_at + + +.. function:: memoryview_at(ptr, size, readonly=False) + + Return a :class:`memoryview` object of length *size* that references memory + starting at *void \*ptr*. + + If *readonly* is true, the returned :class:`!memoryview` object can + not be used to modify the underlying memory. + (Changes made by other means will still be reflected in the returned + object.) + + This function is similar to :func:`string_at` with the key + difference of not making a copy of the specified memory. + It is a semantically equivalent (but more efficient) alternative to + ``memoryview((c_byte * size).from_address(ptr))``. + (While :meth:`~_CData.from_address` only takes integers, *ptr* can also + be given as a :class:`ctypes.POINTER` or a :func:`~ctypes.byref` object.) + + .. audit-event:: ctypes.memoryview_at address,size,readonly + + .. versionadded:: 3.14 .. _ctypes-data-types: @@ -2162,7 +2289,7 @@ Data types This method adapts *obj* to a ctypes type. It is called with the actual object used in a foreign function call when the type is present in the - foreign function's :attr:`~_FuncPtr.argtypes` tuple; + foreign function's :attr:`~_CFuncPtr.argtypes` tuple; it must return an object that can be used as a function call parameter. All ctypes data types have a default implementation of this classmethod @@ -2228,7 +2355,7 @@ Fundamental data types Fundamental data types, when returned as foreign function call results, or, for example, by retrieving structure field members or array items, are transparently converted to native Python types. In other words, if a foreign function has a -:attr:`~_FuncPtr.restype` of :class:`c_char_p`, you will always receive a Python bytes +:attr:`~_CFuncPtr.restype` of :class:`c_char_p`, you will always receive a Python bytes object, *not* a :class:`c_char_p` instance. .. XXX above is false, it actually returns a Unicode string @@ -2280,6 +2407,30 @@ These are the fundamental ctypes data types: optional float initializer. +.. class:: c_double_complex + + Represents the C :c:expr:`double complex` datatype, if available. The + constructor accepts an optional :class:`complex` initializer. + + .. versionadded:: 3.14 + + +.. class:: c_float_complex + + Represents the C :c:expr:`float complex` datatype, if available. The + constructor accepts an optional :class:`complex` initializer. + + .. versionadded:: 3.14 + + +.. class:: c_longdouble_complex + + Represents the C :c:expr:`long double complex` datatype, if available. The + constructor accepts an optional :class:`complex` initializer. + + .. versionadded:: 3.14 + + .. class:: c_int Represents the C :c:expr:`signed int` datatype. The constructor accepts an @@ -2433,9 +2584,11 @@ These are the fundamental ctypes data types: .. class:: HRESULT - Windows only: Represents a :c:type:`!HRESULT` value, which contains success or + Represents a :c:type:`!HRESULT` value, which contains success or error information for a function or method call. + .. availability:: Windows + .. class:: py_object @@ -2457,6 +2610,8 @@ Structured data types Abstract base class for unions in native byte order. + Unions share common attributes and behavior with structures; + see :class:`Structure` documentation for details. .. class:: BigEndianUnion(*args, **kw) @@ -2516,14 +2671,19 @@ fields, or any other data types containing pointer type fields. ... ] - The :attr:`_fields_` class variable must, however, be defined before the - type is first used (an instance is created, :func:`sizeof` is called on it, - and so on). Later assignments to the :attr:`_fields_` class variable will - raise an AttributeError. + The :attr:`!_fields_` class variable can only be set once. + Later assignments will raise an :exc:`AttributeError`. - It is possible to define sub-subclasses of structure types, they inherit - the fields of the base class plus the :attr:`_fields_` defined in the - sub-subclass, if any. + Additionally, the :attr:`!_fields_` class variable must be defined before + the structure or union type is first used: an instance or subclass is + created, :func:`sizeof` is called on it, and so on. + Later assignments to :attr:`!_fields_` will raise an :exc:`AttributeError`. + If :attr:`!_fields_` has not been set before such use, + the structure or union will have no own fields, as if :attr:`!_fields_` + was empty. + + Sub-subclasses of structure types inherit the fields of the base class + plus the :attr:`_fields_` defined in the sub-subclass, if any. .. attribute:: _pack_ @@ -2540,6 +2700,33 @@ fields, or any other data types containing pointer type fields. the structure when being packed or unpacked to/from memory. Setting this attribute to 0 is the same as not setting it at all. + .. versionadded:: 3.13 + + .. attribute:: _layout_ + + An optional string naming the struct/union layout. It can currently + be set to: + + - ``"ms"``: the layout used by the Microsoft compiler (MSVC). + On GCC and Clang, this layout can be selected with + ``__attribute__((ms_struct))``. + - ``"gcc-sysv"``: the layout used by GCC with the System V or “SysV-like” + data model, as used on Linux and macOS. + With this layout, :attr:`~Structure._pack_` must be unset or zero. + + If not set explicitly, ``ctypes`` will use a default that + matches the platform conventions. This default may change in future + Python releases (for example, when a new platform gains official support, + or when a difference between similar platforms is found). + Currently the default will be: + + - On Windows: ``"ms"`` + - When :attr:`~Structure._pack_` is specified: ``"ms"`` + - Otherwise: ``"gcc-sysv"`` + + :attr:`!_layout_` must already be defined when + :attr:`~Structure._fields_` is assigned, otherwise it will have no effect. + .. attribute:: _anonymous_ An optional sequence that lists the names of unnamed (anonymous) fields. @@ -2621,6 +2808,15 @@ Arrays and pointers Array subclass constructors accept positional arguments, used to initialize the elements in order. +.. function:: ARRAY(type, length) + + Create an array. + Equivalent to ``type * length``, where *type* is a + :mod:`ctypes` data type and *length* an integer. + + This function is :term:`soft deprecated` in favor of multiplication. + There are no plans to remove it. + .. class:: _Pointer @@ -2646,3 +2842,41 @@ Arrays and pointers Returns the object to which to pointer points. Assigning to this attribute changes the pointer to point to the assigned object. + + +.. _ctypes-exceptions: + +Exceptions +^^^^^^^^^^ + +.. exception:: ArgumentError + + This exception is raised when a foreign function call cannot convert one of the + passed arguments. + + +.. exception:: COMError(hresult, text, details) + + This exception is raised when a COM method call failed. + + .. attribute:: hresult + + The integer value representing the error code. + + .. attribute:: text + + The error message. + + .. attribute:: details + + The 5-tuple ``(descr, source, helpfile, helpcontext, progid)``. + + *descr* is the textual description. *source* is the language-dependent + ``ProgID`` for the class or application that raised the error. *helpfile* + is the path of the help file. *helpcontext* is the help context + identifier. *progid* is the ``ProgID`` of the interface that defined the + error. + + .. availability:: Windows + + .. versionadded:: 3.14 diff --git a/Doc/library/curses.ascii.rst b/Doc/library/curses.ascii.rst index 410b76e77c025b..cb895664ff1b11 100644 --- a/Doc/library/curses.ascii.rst +++ b/Doc/library/curses.ascii.rst @@ -1,5 +1,5 @@ -:mod:`curses.ascii` --- Utilities for ASCII characters -====================================================== +:mod:`!curses.ascii` --- Utilities for ASCII characters +======================================================= .. module:: curses.ascii :synopsis: Constants and set-membership functions for ASCII characters. diff --git a/Doc/library/curses.panel.rst b/Doc/library/curses.panel.rst index d770c03c8375f4..11fd841d381f69 100644 --- a/Doc/library/curses.panel.rst +++ b/Doc/library/curses.panel.rst @@ -1,5 +1,5 @@ -:mod:`curses.panel` --- A panel stack extension for curses -========================================================== +:mod:`!curses.panel` --- A panel stack extension for curses +=========================================================== .. module:: curses.panel :synopsis: A panel stack extension that adds depth to curses windows. diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 9b8a98f05f7cbb..6c7fc721a3e0fb 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -1,5 +1,5 @@ -:mod:`curses` --- Terminal handling for character-cell displays -=============================================================== +:mod:`!curses` --- Terminal handling for character-cell displays +================================================================ .. module:: curses :synopsis: An interface to the curses library, providing portable @@ -21,6 +21,8 @@ for Windows, DOS, and possibly other systems as well. This extension module is designed to match the API of ncurses, an open-source curses library hosted on Linux and the BSD variants of Unix. +.. include:: ../includes/wasm-mobile-notavail.rst + .. note:: Whenever the documentation mentions a *character* it can be specified @@ -922,7 +924,7 @@ the following methods and attributes: .. method:: window.getbegyx() - Return a tuple ``(y, x)`` of co-ordinates of upper-left corner. + Return a tuple ``(y, x)`` of coordinates of upper-left corner. .. method:: window.getbkgd() diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 4ada69d63abada..e34b2db0210960 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -1,5 +1,5 @@ -:mod:`dataclasses` --- Data Classes -=================================== +:mod:`!dataclasses` --- Data Classes +==================================== .. module:: dataclasses :synopsis: Generate special methods on user-defined classes. @@ -12,7 +12,7 @@ -------------- This module provides a decorator and functions for automatically -adding generated :term:`special method`\s such as :meth:`~object.__init__` and +adding generated :term:`special methods ` such as :meth:`~object.__init__` and :meth:`~object.__repr__` to user-defined classes. It was originally described in :pep:`557`. @@ -31,7 +31,7 @@ using :pep:`526` type annotations. For example, this code:: def total_cost(self) -> float: return self.unit_price * self.quantity_on_hand -will add, among other things, a :meth:`~object.__init__` that looks like:: +will add, among other things, a :meth:`!__init__` that looks like:: def __init__(self, name: str, unit_price: float, quantity_on_hand: int = 0): self.name = name @@ -39,7 +39,7 @@ will add, among other things, a :meth:`~object.__init__` that looks like:: self.quantity_on_hand = quantity_on_hand Note that this method is automatically added to the class: it is not -directly specified in the ``InventoryItem`` definition shown above. +directly specified in the :class:`!InventoryItem` definition shown above. .. versionadded:: 3.7 @@ -49,26 +49,26 @@ Module contents .. decorator:: dataclass(*, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False) This function is a :term:`decorator` that is used to add generated - :term:`special method`\s to classes, as described below. + :term:`special methods ` to classes, as described below. - The :func:`dataclass` decorator examines the class to find + The ``@dataclass`` decorator examines the class to find ``field``\s. A ``field`` is defined as a class variable that has a :term:`type annotation `. With two - exceptions described below, nothing in :func:`dataclass` + exceptions described below, nothing in ``@dataclass`` examines the type specified in the variable annotation. The order of the fields in all of the generated methods is the order in which they appear in the class definition. - The :func:`dataclass` decorator will add various "dunder" methods to + The ``@dataclass`` decorator will add various "dunder" methods to the class, described below. If any of the added methods already exist in the class, the behavior depends on the parameter, as documented below. The decorator returns the same class that it is called on; no new class is created. - If :func:`dataclass` is used just as a simple decorator with no parameters, + If ``@dataclass`` is used just as a simple decorator with no parameters, it acts as if it has the default values documented in this - signature. That is, these three uses of :func:`dataclass` are + signature. That is, these three uses of ``@dataclass`` are equivalent:: @dataclass @@ -84,124 +84,132 @@ Module contents class C: ... - The parameters to :func:`dataclass` are: + The parameters to ``@dataclass`` are: - - ``init``: If true (the default), a :meth:`~object.__init__` method will be + - *init*: If true (the default), a :meth:`~object.__init__` method will be generated. - If the class already defines :meth:`~object.__init__`, this parameter is + If the class already defines :meth:`!__init__`, this parameter is ignored. - - ``repr``: If true (the default), a :meth:`~object.__repr__` method will be + - *repr*: If true (the default), a :meth:`~object.__repr__` method will be generated. The generated repr string will have the class name and the name and repr of each field, in the order they are defined in the class. Fields that are marked as being excluded from the repr are not included. For example: ``InventoryItem(name='widget', unit_price=3.0, quantity_on_hand=10)``. - If the class already defines :meth:`~object.__repr__`, this parameter is + If the class already defines :meth:`!__repr__`, this parameter is ignored. - - ``eq``: If true (the default), an :meth:`~object.__eq__` method will be + - *eq*: If true (the default), an :meth:`~object.__eq__` method will be generated. This method compares the class as if it were a tuple of its fields, in order. Both instances in the comparison must be of the identical type. - If the class already defines :meth:`~object.__eq__`, this parameter is + If the class already defines :meth:`!__eq__`, this parameter is ignored. - - ``order``: If true (the default is ``False``), :meth:`~object.__lt__`, + - *order*: If true (the default is ``False``), :meth:`~object.__lt__`, :meth:`~object.__le__`, :meth:`~object.__gt__`, and :meth:`~object.__ge__` methods will be generated. These compare the class as if it were a tuple of its fields, in order. Both instances in the comparison must be of the - identical type. If ``order`` is true and ``eq`` is false, a + identical type. If *order* is true and *eq* is false, a :exc:`ValueError` is raised. - If the class already defines any of :meth:`~object.__lt__`, - :meth:`~object.__le__`, :meth:`~object.__gt__`, or :meth:`~object.__ge__`, then + If the class already defines any of :meth:`!__lt__`, + :meth:`!__le__`, :meth:`!__gt__`, or :meth:`!__ge__`, then :exc:`TypeError` is raised. - - ``unsafe_hash``: If ``False`` (the default), a :meth:`~object.__hash__` method - is generated according to how ``eq`` and ``frozen`` are set. + - *unsafe_hash*: If ``False`` (the default), a :meth:`~object.__hash__` method + is generated according to how *eq* and *frozen* are set. - :meth:`~object.__hash__` is used by built-in :meth:`hash()`, and when objects are + :meth:`!__hash__` is used by built-in :meth:`hash`, and when objects are added to hashed collections such as dictionaries and sets. Having a - :meth:`~object.__hash__` implies that instances of the class are immutable. + :meth:`!__hash__` implies that instances of the class are immutable. Mutability is a complicated property that depends on the programmer's - intent, the existence and behavior of :meth:`~object.__eq__`, and the values of - the ``eq`` and ``frozen`` flags in the :func:`dataclass` decorator. + intent, the existence and behavior of :meth:`!__eq__`, and the values of + the *eq* and *frozen* flags in the ``@dataclass`` decorator. - By default, :func:`dataclass` will not implicitly add a :meth:`~object.__hash__` + By default, ``@dataclass`` will not implicitly add a :meth:`~object.__hash__` method unless it is safe to do so. Neither will it add or change an - existing explicitly defined :meth:`~object.__hash__` method. Setting the class + existing explicitly defined :meth:`!__hash__` method. Setting the class attribute ``__hash__ = None`` has a specific meaning to Python, as - described in the :meth:`~object.__hash__` documentation. + described in the :meth:`!__hash__` documentation. - If :meth:`~object.__hash__` is not explicitly defined, or if it is set to ``None``, - then :func:`dataclass` *may* add an implicit :meth:`~object.__hash__` method. - Although not recommended, you can force :func:`dataclass` to create a - :meth:`~object.__hash__` method with ``unsafe_hash=True``. This might be the case + If :meth:`!__hash__` is not explicitly defined, or if it is set to ``None``, + then ``@dataclass`` *may* add an implicit :meth:`!__hash__` method. + Although not recommended, you can force ``@dataclass`` to create a + :meth:`!__hash__` method with ``unsafe_hash=True``. This might be the case if your class is logically immutable but can still be mutated. This is a specialized use case and should be considered carefully. - Here are the rules governing implicit creation of a :meth:`~object.__hash__` - method. Note that you cannot both have an explicit :meth:`~object.__hash__` + Here are the rules governing implicit creation of a :meth:`!__hash__` + method. Note that you cannot both have an explicit :meth:`!__hash__` method in your dataclass and set ``unsafe_hash=True``; this will result in a :exc:`TypeError`. - If ``eq`` and ``frozen`` are both true, by default :func:`dataclass` will - generate a :meth:`~object.__hash__` method for you. If ``eq`` is true and - ``frozen`` is false, :meth:`~object.__hash__` will be set to ``None``, marking it - unhashable (which it is, since it is mutable). If ``eq`` is false, - :meth:`~object.__hash__` will be left untouched meaning the :meth:`~object.__hash__` + If *eq* and *frozen* are both true, by default ``@dataclass`` will + generate a :meth:`!__hash__` method for you. If *eq* is true and + *frozen* is false, :meth:`!__hash__` will be set to ``None``, marking it + unhashable (which it is, since it is mutable). If *eq* is false, + :meth:`!__hash__` will be left untouched meaning the :meth:`!__hash__` method of the superclass will be used (if the superclass is :class:`object`, this means it will fall back to id-based hashing). - - ``frozen``: If true (the default is ``False``), assigning to fields will + - *frozen*: If true (the default is ``False``), assigning to fields will generate an exception. This emulates read-only frozen instances. If :meth:`~object.__setattr__` or :meth:`~object.__delattr__` is defined in the class, then :exc:`TypeError` is raised. See the discussion below. - - ``match_args``: If true (the default is ``True``), the - ``__match_args__`` tuple will be created from the list of + - *match_args*: If true (the default is ``True``), the + :attr:`~object.__match_args__` tuple will be created from the list of parameters to the generated :meth:`~object.__init__` method (even if - :meth:`~object.__init__` is not generated, see above). If false, or if - ``__match_args__`` is already defined in the class, then - ``__match_args__`` will not be generated. + :meth:`!__init__` is not generated, see above). If false, or if + :attr:`!__match_args__` is already defined in the class, then + :attr:`!__match_args__` will not be generated. .. versionadded:: 3.10 - - ``kw_only``: If true (the default value is ``False``), then all + - *kw_only*: If true (the default value is ``False``), then all fields will be marked as keyword-only. If a field is marked as keyword-only, then the only effect is that the :meth:`~object.__init__` parameter generated from a keyword-only field must be specified - with a keyword when :meth:`~object.__init__` is called. There is no + with a keyword when :meth:`!__init__` is called. There is no effect on any other aspect of dataclasses. See the :term:`parameter` glossary entry for details. Also see the :const:`KW_ONLY` section. .. versionadded:: 3.10 - - ``slots``: If true (the default is ``False``), :attr:`~object.__slots__` attribute + - *slots*: If true (the default is ``False``), :attr:`~object.__slots__` attribute will be generated and new class will be returned instead of the original one. - If :attr:`~object.__slots__` is already defined in the class, then :exc:`TypeError` + If :attr:`!__slots__` is already defined in the class, then :exc:`TypeError` is raised. + .. warning:: + Passing parameters to a base class :meth:`~object.__init_subclass__` + when using ``slots=True`` will result in a :exc:`TypeError`. + Either use ``__init_subclass__`` with no parameters + or use default values as a workaround. + See :gh:`91126` for full details. + .. versionadded:: 3.10 .. versionchanged:: 3.11 - If a field name is already included in the ``__slots__`` - of a base class, it will not be included in the generated ``__slots__`` + If a field name is already included in the :attr:`!__slots__` + of a base class, it will not be included in the generated :attr:`!__slots__` to prevent :ref:`overriding them `. - Therefore, do not use ``__slots__`` to retrieve the field names of a + Therefore, do not use :attr:`!__slots__` to retrieve the field names of a dataclass. Use :func:`fields` instead. To be able to determine inherited slots, - base class ``__slots__`` may be any iterable, but *not* an iterator. + base class :attr:`!__slots__` may be any iterable, but *not* an iterator. - - ``weakref_slot``: If true (the default is ``False``), add a slot + - *weakref_slot*: If true (the default is ``False``), add a slot named "__weakref__", which is required to make an instance - weakref-able. It is an error to specify ``weakref_slot=True`` + :func:`weakref-able `. + It is an error to specify ``weakref_slot=True`` without also specifying ``slots=True``. .. versionadded:: 3.11 @@ -214,7 +222,7 @@ Module contents a: int # 'a' has no default value b: int = 0 # assign a default value for 'b' - In this example, both ``a`` and ``b`` will be included in the added + In this example, both :attr:`!a` and :attr:`!b` will be included in the added :meth:`~object.__init__` method, which will be defined as:: def __init__(self, a: int, b: int = 0): @@ -223,13 +231,13 @@ Module contents follows a field with a default value. This is true whether this occurs in a single class, or as a result of class inheritance. -.. function:: field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, hash=None, compare=True, metadata=None, kw_only=MISSING) +.. function:: field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, hash=None, compare=True, metadata=None, kw_only=MISSING, doc=None) For common and simple use cases, no other functionality is required. There are, however, some dataclass features that require additional per-field information. To satisfy this need for additional information, you can replace the default field value - with a call to the provided :func:`field` function. For example:: + with a call to the provided :func:`!field` function. For example:: @dataclass class C: @@ -243,27 +251,27 @@ Module contents used because ``None`` is a valid value for some parameters with a distinct meaning. No code should directly use the :const:`MISSING` value. - The parameters to :func:`field` are: + The parameters to :func:`!field` are: - - ``default``: If provided, this will be the default value for this - field. This is needed because the :meth:`field` call itself + - *default*: If provided, this will be the default value for this + field. This is needed because the :func:`!field` call itself replaces the normal position of the default value. - - ``default_factory``: If provided, it must be a zero-argument + - *default_factory*: If provided, it must be a zero-argument callable that will be called when a default value is needed for this field. Among other purposes, this can be used to specify fields with mutable default values, as discussed below. It is an - error to specify both ``default`` and ``default_factory``. + error to specify both *default* and *default_factory*. - - ``init``: If true (the default), this field is included as a + - *init*: If true (the default), this field is included as a parameter to the generated :meth:`~object.__init__` method. - - ``repr``: If true (the default), this field is included in the + - *repr*: If true (the default), this field is included in the string returned by the generated :meth:`~object.__repr__` method. - - ``hash``: This can be a bool or ``None``. If true, this field is + - *hash*: This can be a bool or ``None``. If true, this field is included in the generated :meth:`~object.__hash__` method. If ``None`` (the - default), use the value of ``compare``: this would normally be + default), use the value of *compare*: this would normally be the expected behavior. A field should be considered in the hash if it's used for comparisons. Setting this value to anything other than ``None`` is discouraged. @@ -274,11 +282,11 @@ Module contents fields that contribute to the type's hash value. Even if a field is excluded from the hash, it will still be used for comparisons. - - ``compare``: If true (the default), this field is included in the + - *compare*: If true (the default), this field is included in the generated equality and comparison methods (:meth:`~object.__eq__`, :meth:`~object.__gt__`, et al.). - - ``metadata``: This can be a mapping or None. None is treated as + - *metadata*: This can be a mapping or ``None``. ``None`` is treated as an empty dict. This value is wrapped in :func:`~types.MappingProxyType` to make it read-only, and exposed on the :class:`Field` object. It is not used at all by Data @@ -286,17 +294,21 @@ Module contents Multiple third-parties can each have their own key, to use as a namespace in the metadata. - - ``kw_only``: If true, this field will be marked as keyword-only. + - *kw_only*: If true, this field will be marked as keyword-only. This is used when the generated :meth:`~object.__init__` method's parameters are computed. .. versionadded:: 3.10 + - ``doc``: optional docstring for this field. + + .. versionadded:: 3.13 + If the default value of a field is specified by a call to - :func:`field()`, then the class attribute for this field will be - replaced by the specified ``default`` value. If no ``default`` is + :func:`!field`, then the class attribute for this field will be + replaced by the specified *default* value. If *default* is not provided, then the class attribute will be deleted. The intent is - that after the :func:`dataclass` decorator runs, the class + that after the :func:`@dataclass ` decorator runs, the class attributes will all contain the default values for the fields, just as if the default value itself were specified. For example, after:: @@ -308,21 +320,21 @@ Module contents z: int = field(repr=False, default=10) t: int = 20 - The class attribute ``C.z`` will be ``10``, the class attribute - ``C.t`` will be ``20``, and the class attributes ``C.x`` and - ``C.y`` will not be set. + The class attribute :attr:`!C.z` will be ``10``, the class attribute + :attr:`!C.t` will be ``20``, and the class attributes :attr:`!C.x` and + :attr:`!C.y` will not be set. .. class:: Field - :class:`Field` objects describe each defined field. These objects + :class:`!Field` objects describe each defined field. These objects are created internally, and are returned by the :func:`fields` module-level method (see below). Users should never instantiate a - :class:`Field` object directly. Its documented attributes are: + :class:`!Field` object directly. Its documented attributes are: - - ``name``: The name of the field. - - ``type``: The type of the field. - - ``default``, ``default_factory``, ``init``, ``repr``, ``hash``, - ``compare``, ``metadata``, and ``kw_only`` have the identical + - :attr:`!name`: The name of the field. + - :attr:`!type`: The type of the field. + - :attr:`!default`, :attr:`!default_factory`, :attr:`!init`, :attr:`!repr`, :attr:`!hash`, + :attr:`!compare`, :attr:`!metadata`, and :attr:`!kw_only` have the identical meaning and values as they do in the :func:`field` function. Other attributes may exist, but they are private and must not be @@ -337,13 +349,13 @@ Module contents .. function:: asdict(obj, *, dict_factory=dict) - Converts the dataclass ``obj`` to a dict (by using the - factory function ``dict_factory``). Each dataclass is converted + Converts the dataclass *obj* to a dict (by using the + factory function *dict_factory*). Each dataclass is converted to a dict of its fields, as ``name: value`` pairs. dataclasses, dicts, lists, and tuples are recursed into. Other objects are copied with :func:`copy.deepcopy`. - Example of using :func:`asdict` on nested dataclasses:: + Example of using :func:`!asdict` on nested dataclasses:: @dataclass class Point: @@ -362,15 +374,15 @@ Module contents To create a shallow copy, the following workaround may be used:: - dict((field.name, getattr(obj, field.name)) for field in fields(obj)) + {field.name: getattr(obj, field.name) for field in fields(obj)} - :func:`asdict` raises :exc:`TypeError` if ``obj`` is not a dataclass + :func:`!asdict` raises :exc:`TypeError` if *obj* is not a dataclass instance. .. function:: astuple(obj, *, tuple_factory=tuple) - Converts the dataclass ``obj`` to a tuple (by using the - factory function ``tuple_factory``). Each dataclass is converted + Converts the dataclass *obj* to a tuple (by using the + factory function *tuple_factory*). Each dataclass is converted to a tuple of its field values. dataclasses, dicts, lists, and tuples are recursed into. Other objects are copied with :func:`copy.deepcopy`. @@ -384,28 +396,33 @@ Module contents tuple(getattr(obj, field.name) for field in dataclasses.fields(obj)) - :func:`astuple` raises :exc:`TypeError` if ``obj`` is not a dataclass + :func:`!astuple` raises :exc:`TypeError` if *obj* is not a dataclass instance. -.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None) +.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None, decorator=dataclass) - Creates a new dataclass with name ``cls_name``, fields as defined - in ``fields``, base classes as given in ``bases``, and initialized - with a namespace as given in ``namespace``. ``fields`` is an + Creates a new dataclass with name *cls_name*, fields as defined + in *fields*, base classes as given in *bases*, and initialized + with a namespace as given in *namespace*. *fields* is an iterable whose elements are each either ``name``, ``(name, type)``, or ``(name, type, Field)``. If just ``name`` is supplied, - ``typing.Any`` is used for ``type``. The values of ``init``, - ``repr``, ``eq``, ``order``, ``unsafe_hash``, ``frozen``, - ``match_args``, ``kw_only``, ``slots``, and ``weakref_slot`` have - the same meaning as they do in :func:`dataclass`. + :data:`typing.Any` is used for ``type``. The values of *init*, + *repr*, *eq*, *order*, *unsafe_hash*, *frozen*, + *match_args*, *kw_only*, *slots*, and *weakref_slot* have + the same meaning as they do in :func:`@dataclass `. - If ``module`` is defined, the ``__module__`` attribute + If *module* is defined, the :attr:`!__module__` attribute of the dataclass is set to that value. By default, it is set to the module name of the caller. + The *decorator* parameter is a callable that will be used to create the dataclass. + It should take the class object as a first argument and the same keyword arguments + as :func:`@dataclass `. By default, the :func:`@dataclass ` + function is used. + This function is not strictly required, because any Python - mechanism for creating a new class with ``__annotations__`` can - then apply the :func:`dataclass` function to convert that class to + mechanism for creating a new class with :attr:`!__annotations__` can + then apply the :func:`@dataclass ` function to convert that class to a dataclass. This function is provided as a convenience. For example:: @@ -426,40 +443,43 @@ Module contents def add_one(self): return self.x + 1 + .. versionadded:: 3.14 + Added the *decorator* parameter. + .. function:: replace(obj, /, **changes) - Creates a new object of the same type as ``obj``, replacing - fields with values from ``changes``. If ``obj`` is not a Data - Class, raises :exc:`TypeError`. If values in ``changes`` do not - specify fields, raises :exc:`TypeError`. + Creates a new object of the same type as *obj*, replacing + fields with values from *changes*. If *obj* is not a Data + Class, raises :exc:`TypeError`. If keys in *changes* are not + field names of the given dataclass, raises :exc:`TypeError`. The newly returned object is created by calling the :meth:`~object.__init__` method of the dataclass. This ensures that :meth:`__post_init__`, if present, is also called. Init-only variables without default values, if any exist, must be - specified on the call to :func:`replace` so that they can be passed to - :meth:`~object.__init__` and :meth:`__post_init__`. + specified on the call to :func:`!replace` so that they can be passed to + :meth:`!__init__` and :meth:`__post_init__`. - It is an error for ``changes`` to contain any fields that are + It is an error for *changes* to contain any fields that are defined as having ``init=False``. A :exc:`ValueError` will be raised in this case. Be forewarned about how ``init=False`` fields work during a call to - :func:`replace`. They are not copied from the source object, but + :func:`!replace`. They are not copied from the source object, but rather are initialized in :meth:`__post_init__`, if they're initialized at all. It is expected that ``init=False`` fields will be rarely and judiciously used. If they are used, it might be wise to have alternate class constructors, or perhaps a custom - ``replace()`` (or similarly named) method which handles instance + :func:`!replace` (or similarly named) method which handles instance copying. Dataclass instances are also supported by generic function :func:`copy.replace`. .. function:: is_dataclass(obj) - Return ``True`` if its parameter is a dataclass or an instance of one, - otherwise return ``False``. + Return ``True`` if its parameter is a dataclass (including subclasses of a + dataclass) or an instance of one, otherwise return ``False``. If you need to know if a class is an instance of a dataclass (and not a dataclass itself), then add a further check for ``not @@ -475,11 +495,11 @@ Module contents .. data:: KW_ONLY A sentinel value used as a type annotation. Any fields after a - pseudo-field with the type of :const:`KW_ONLY` are marked as + pseudo-field with the type of :const:`!KW_ONLY` are marked as keyword-only fields. Note that a pseudo-field of type - :const:`KW_ONLY` is otherwise completely ignored. This includes the + :const:`!KW_ONLY` is otherwise completely ignored. This includes the name of such a field. By convention, a name of ``_`` is used for a - :const:`KW_ONLY` field. Keyword-only fields signify + :const:`!KW_ONLY` field. Keyword-only fields signify :meth:`~object.__init__` parameters that must be specified as keywords when the class is instantiated. @@ -495,7 +515,7 @@ Module contents p = Point(0, y=1.5, z=2.0) In a single dataclass, it is an error to specify more than one - field whose type is :const:`KW_ONLY`. + field whose type is :const:`!KW_ONLY`. .. versionadded:: 3.10 @@ -513,11 +533,11 @@ Post-init processing .. function:: __post_init__() When defined on the class, it will be called by the generated - :meth:`~object.__init__`, normally as ``self.__post_init__()``. + :meth:`~object.__init__`, normally as :meth:`!self.__post_init__`. However, if any ``InitVar`` fields are defined, they will also be - passed to :meth:`__post_init__` in the order they were defined in the - class. If no :meth:`~object.__init__` method is generated, then - :meth:`__post_init__` will not automatically be called. + passed to :meth:`!__post_init__` in the order they were defined in the + class. If no :meth:`!__init__` method is generated, then + :meth:`!__post_init__` will not automatically be called. Among other uses, this allows for initializing field values that depend on one or more other fields. For example:: @@ -531,8 +551,8 @@ Post-init processing def __post_init__(self): self.c = self.a + self.b -The :meth:`~object.__init__` method generated by :func:`dataclass` does not call base -class :meth:`~object.__init__` methods. If the base class has an :meth:`~object.__init__` method +The :meth:`~object.__init__` method generated by :func:`@dataclass ` does not call base +class :meth:`!__init__` methods. If the base class has an :meth:`!__init__` method that has to be called, it is common to call this method in a :meth:`__post_init__` method:: @@ -548,29 +568,33 @@ that has to be called, it is common to call this method in a def __post_init__(self): super().__init__(self.side, self.side) -Note, however, that in general the dataclass-generated :meth:`~object.__init__` methods +Note, however, that in general the dataclass-generated :meth:`!__init__` methods don't need to be called, since the derived dataclass will take care of initializing all fields of any base class that is a dataclass itself. See the section below on init-only variables for ways to pass -parameters to :meth:`__post_init__`. Also see the warning about how +parameters to :meth:`!__post_init__`. Also see the warning about how :func:`replace` handles ``init=False`` fields. +.. _dataclasses-class-variables: + Class variables --------------- -One of the few places where :func:`dataclass` actually inspects the type +One of the few places where :func:`@dataclass ` actually inspects the type of a field is to determine if a field is a class variable as defined in :pep:`526`. It does this by checking if the type of the field is -``typing.ClassVar``. If a field is a ``ClassVar``, it is excluded +:data:`typing.ClassVar`. If a field is a ``ClassVar``, it is excluded from consideration as a field and is ignored by the dataclass mechanisms. Such ``ClassVar`` pseudo-fields are not returned by the module-level :func:`fields` function. +.. _dataclasses-init-only-variables: + Init-only variables ------------------- -Another place where :func:`dataclass` inspects a type annotation is to +Another place where :func:`@dataclass ` inspects a type annotation is to determine if a field is an init-only variable. It does this by seeing if the type of a field is of type ``dataclasses.InitVar``. If a field is an ``InitVar``, it is considered a pseudo-field called an init-only @@ -595,14 +619,16 @@ value is not provided when creating the class:: c = C(10, database=my_database) -In this case, :func:`fields` will return :class:`Field` objects for ``i`` and -``j``, but not for ``database``. +In this case, :func:`fields` will return :class:`Field` objects for :attr:`!i` and +:attr:`!j`, but not for :attr:`!database`. + +.. _dataclasses-frozen: Frozen instances ---------------- It is not possible to create truly immutable Python objects. However, -by passing ``frozen=True`` to the :meth:`dataclass` decorator you can +by passing ``frozen=True`` to the :func:`@dataclass ` decorator you can emulate immutability. In that case, dataclasses will add :meth:`~object.__setattr__` and :meth:`~object.__delattr__` methods to the class. These methods will raise a :exc:`FrozenInstanceError` when invoked. @@ -611,10 +637,14 @@ There is a tiny performance penalty when using ``frozen=True``: :meth:`~object.__init__` cannot use simple assignment to initialize fields, and must use :meth:`!object.__setattr__`. +.. Make sure to not remove "object" from "object.__setattr__" in the above markup! + +.. _dataclasses-inheritance: + Inheritance ----------- -When the dataclass is being created by the :meth:`dataclass` decorator, +When the dataclass is being created by the :func:`@dataclass ` decorator, it looks through all of the class's base classes in reverse MRO (that is, starting at :class:`object`) and, for each dataclass that it finds, adds the fields from that base class to an ordered mapping of fields. @@ -634,15 +664,15 @@ example:: z: int = 10 x: int = 15 -The final list of fields is, in order, ``x``, ``y``, ``z``. The final -type of ``x`` is ``int``, as specified in class ``C``. +The final list of fields is, in order, :attr:`!x`, :attr:`!y`, :attr:`!z`. The final +type of :attr:`!x` is :class:`int`, as specified in class :class:`!C`. -The generated :meth:`~object.__init__` method for ``C`` will look like:: +The generated :meth:`~object.__init__` method for :class:`!C` will look like:: def __init__(self, x: int = 15, y: int = 0, z: int = 10): -Re-ordering of keyword-only parameters in :meth:`~object.__init__` ------------------------------------------------------------------- +Re-ordering of keyword-only parameters in :meth:`!__init__` +----------------------------------------------------------- After the parameters needed for :meth:`~object.__init__` are computed, any keyword-only parameters are moved to come after all regular @@ -650,8 +680,8 @@ keyword-only parameters are moved to come after all regular keyword-only parameters are implemented in Python: they must come after non-keyword-only parameters. -In this example, ``Base.y``, ``Base.w``, and ``D.t`` are keyword-only -fields, and ``Base.x`` and ``D.z`` are regular fields:: +In this example, :attr:`!Base.y`, :attr:`!Base.w`, and :attr:`!D.t` are keyword-only +fields, and :attr:`!Base.x` and :attr:`!D.z` are regular fields:: @dataclass class Base: @@ -665,7 +695,7 @@ fields, and ``Base.x`` and ``D.z`` are regular fields:: z: int = 10 t: int = field(kw_only=True, default=0) -The generated :meth:`~object.__init__` method for ``D`` will look like:: +The generated :meth:`!__init__` method for :class:`!D` will look like:: def __init__(self, x: Any = 15.0, z: int = 10, *, y: int = 0, w: int = 1, t: int = 0): @@ -674,22 +704,22 @@ the list of fields: parameters derived from regular fields are followed by parameters derived from keyword-only fields. The relative ordering of keyword-only parameters is maintained in the -re-ordered :meth:`~object.__init__` parameter list. +re-ordered :meth:`!__init__` parameter list. Default factory functions ------------------------- -If a :func:`field` specifies a ``default_factory``, it is called with +If a :func:`field` specifies a *default_factory*, it is called with zero arguments when a default value for the field is needed. For example, to create a new instance of a list, use:: mylist: list = field(default_factory=list) If a field is excluded from :meth:`~object.__init__` (using ``init=False``) -and the field also specifies ``default_factory``, then the default +and the field also specifies *default_factory*, then the default factory function will always be called from the generated -:meth:`~object.__init__` function. This happens because there is no other +:meth:`!__init__` function. This happens because there is no other way to give the field an initial value. Mutable default values @@ -710,8 +740,8 @@ Consider this example, not using dataclasses:: assert o1.x == [1, 2] assert o1.x is o2.x -Note that the two instances of class ``C`` share the same class -variable ``x``, as expected. +Note that the two instances of class :class:`!C` share the same class +variable :attr:`!x`, as expected. Using dataclasses, *if* this code was valid:: @@ -719,7 +749,7 @@ Using dataclasses, *if* this code was valid:: class D: x: list = [] # This code raises ValueError def add(self, element): - self.x += element + self.x.append(element) it would generate code similar to:: @@ -728,17 +758,17 @@ it would generate code similar to:: def __init__(self, x=x): self.x = x def add(self, element): - self.x += element + self.x.append(element) assert D().x is D().x -This has the same issue as the original example using class ``C``. -That is, two instances of class ``D`` that do not specify a value -for ``x`` when creating a class instance will share the same copy -of ``x``. Because dataclasses just use normal Python class +This has the same issue as the original example using class :class:`!C`. +That is, two instances of class :class:`!D` that do not specify a value +for :attr:`!x` when creating a class instance will share the same copy +of :attr:`!x`. Because dataclasses just use normal Python class creation they also share this behavior. There is no general way for Data Classes to detect this condition. Instead, the -:func:`dataclass` decorator will raise a :exc:`ValueError` if it +:func:`@dataclass ` decorator will raise a :exc:`ValueError` if it detects an unhashable default parameter. The assumption is that if a value is unhashable, it is mutable. This is a partial solution, but it does protect against many common errors. @@ -753,8 +783,8 @@ mutable types as default values for fields:: assert D().x is not D().x .. versionchanged:: 3.11 - Instead of looking for and disallowing objects of type ``list``, - ``dict``, or ``set``, unhashable objects are now not allowed as + Instead of looking for and disallowing objects of type :class:`list`, + :class:`dict`, or :class:`set`, unhashable objects are now not allowed as default values. Unhashability is used to approximate mutability. @@ -764,15 +794,17 @@ Descriptor-typed fields Fields that are assigned :ref:`descriptor objects ` as their default value have the following special behaviors: -* The value for the field passed to the dataclass's ``__init__`` method is - passed to the descriptor's ``__set__`` method rather than overwriting the +* The value for the field passed to the dataclass's :meth:`~object.__init__` method is + passed to the descriptor's :meth:`~object.__set__` method rather than overwriting the descriptor object. + * Similarly, when getting or setting the field, the descriptor's - ``__get__`` or ``__set__`` method is called rather than returning or + :meth:`~object.__get__` or :meth:`!__set__` method is called rather than returning or overwriting the descriptor object. -* To determine whether a field contains a default value, ``dataclasses`` - will call the descriptor's ``__get__`` method using its class access - form (i.e. ``descriptor.__get__(obj=None, type=cls)``. If the + +* To determine whether a field contains a default value, :func:`@dataclass ` + will call the descriptor's :meth:`!__get__` method using its class access + form: ``descriptor.__get__(obj=None, type=cls)``. If the descriptor returns a value in this case, it will be used as the field's default. On the other hand, if the descriptor raises :exc:`AttributeError` in this situation, no default value will be diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 1905c9e1ca755d..2f81080d525f86 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1,5 +1,5 @@ -:mod:`datetime` --- Basic date and time types -============================================= +:mod:`!datetime` --- Basic date and time types +============================================== .. module:: datetime :synopsis: Basic date and time types. @@ -37,7 +37,7 @@ on efficient attribute extraction for output formatting and manipulation. Package `dateutil `_ Third-party library with expanded time zone and parsing support. - Package `DateType `_ + Package :pypi:`DateType` Third-party library that introduces distinct static types to e.g. allow :term:`static type checkers ` to differentiate between naive and aware datetimes. @@ -48,7 +48,7 @@ Aware and Naive Objects ----------------------- Date and time objects may be categorized as "aware" or "naive" depending on -whether or not they include timezone information. +whether or not they include time zone information. With sufficient knowledge of applicable algorithmic and political time adjustments, such as time zone and daylight saving time information, @@ -58,7 +58,7 @@ interpretation. [#]_ A **naive** object does not contain enough information to unambiguously locate itself relative to other date/time objects. Whether a naive object represents -Coordinated Universal Time (UTC), local time, or time in some other timezone is +Coordinated Universal Time (UTC), local time, or time in some other time zone is purely up to the program, just like it is up to the program whether a particular number represents metres, miles, or mass. Naive objects are easy to understand and to work with, at the cost of ignoring some aspects of reality. @@ -70,9 +70,9 @@ These :class:`tzinfo` objects capture information about the offset from UTC time, the time zone name, and whether daylight saving time is in effect. Only one concrete :class:`tzinfo` class, the :class:`timezone` class, is -supplied by the :mod:`!datetime` module. The :class:`timezone` class can -represent simple timezones with fixed offsets from UTC, such as UTC itself or -North American EST and EDT timezones. Supporting timezones at deeper levels of +supplied by the :mod:`!datetime` module. The :class:`!timezone` class can +represent simple time zones with fixed offsets from UTC, such as UTC itself or +North American EST and EDT time zones. Supporting time zones at deeper levels of detail is up to the application. The rules for time adjustment across the world are more political than rational, change frequently, and there is no standard suitable for every application aside from UTC. @@ -85,17 +85,17 @@ The :mod:`!datetime` module exports the following constants: .. data:: MINYEAR The smallest year number allowed in a :class:`date` or :class:`.datetime` object. - :const:`MINYEAR` is ``1``. + :const:`MINYEAR` is 1. .. data:: MAXYEAR The largest year number allowed in a :class:`date` or :class:`.datetime` object. - :const:`MAXYEAR` is ``9999``. + :const:`MAXYEAR` is 9999. .. attribute:: UTC - Alias for the UTC timezone singleton :attr:`datetime.timezone.utc`. + Alias for the UTC time zone singleton :attr:`datetime.timezone.utc`. .. versionadded:: 3.11 @@ -180,19 +180,19 @@ Objects of the :class:`date` type are always naive. An object of type :class:`.time` or :class:`.datetime` may be aware or naive. -A :class:`.datetime` object *d* is aware if both of the following hold: +A :class:`.datetime` object ``d`` is aware if both of the following hold: 1. ``d.tzinfo`` is not ``None`` 2. ``d.tzinfo.utcoffset(d)`` does not return ``None`` -Otherwise, *d* is naive. +Otherwise, ``d`` is naive. -A :class:`.time` object *t* is aware if both of the following hold: +A :class:`.time` object ``t`` is aware if both of the following hold: 1. ``t.tzinfo`` is not ``None`` 2. ``t.tzinfo.utcoffset(None)`` does not return ``None``. -Otherwise, *t* is naive. +Otherwise, ``t`` is naive. The distinction between aware and naive doesn't apply to :class:`timedelta` objects. @@ -207,7 +207,7 @@ A :class:`timedelta` object represents a duration, the difference between two .. class:: timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0) - All arguments are optional and default to ``0``. Arguments may be integers + All arguments are optional and default to 0. Arguments may be integers or floats, and may be positive or negative. Only *days*, *seconds* and *microseconds* are stored internally. @@ -280,20 +280,40 @@ Class attributes: The smallest possible difference between non-equal :class:`timedelta` objects, ``timedelta(microseconds=1)``. -Note that, because of normalization, ``timedelta.max`` > ``-timedelta.min``. +Note that, because of normalization, ``timedelta.max`` is greater than ``-timedelta.min``. ``-timedelta.max`` is not representable as a :class:`timedelta` object. + Instance attributes (read-only): -+------------------+--------------------------------------------+ -| Attribute | Value | -+==================+============================================+ -| ``days`` | Between -999999999 and 999999999 inclusive | -+------------------+--------------------------------------------+ -| ``seconds`` | Between 0 and 86399 inclusive | -+------------------+--------------------------------------------+ -| ``microseconds`` | Between 0 and 999999 inclusive | -+------------------+--------------------------------------------+ +.. attribute:: timedelta.days + + Between -999,999,999 and 999,999,999 inclusive. + + +.. attribute:: timedelta.seconds + + Between 0 and 86,399 inclusive. + + .. caution:: + + It is a somewhat common bug for code to unintentionally use this attribute + when it is actually intended to get a :meth:`~timedelta.total_seconds` + value instead: + + .. doctest:: + + >>> from datetime import timedelta + >>> duration = timedelta(seconds=11235813) + >>> duration.days, duration.seconds + (130, 3813) + >>> duration.total_seconds() + 11235813.0 + +.. attribute:: timedelta.microseconds + + Between 0 and 999,999 inclusive. + Supported operations: @@ -302,26 +322,27 @@ Supported operations: +--------------------------------+-----------------------------------------------+ | Operation | Result | +================================+===============================================+ -| ``t1 = t2 + t3`` | Sum of *t2* and *t3*. Afterwards *t1*-*t2* == | -| | *t3* and *t1*-*t3* == *t2* are true. (1) | +| ``t1 = t2 + t3`` | Sum of ``t2`` and ``t3``. | +| | Afterwards ``t1 - t2 == t3`` and | +| | ``t1 - t3 == t2`` are true. (1) | +--------------------------------+-----------------------------------------------+ -| ``t1 = t2 - t3`` | Difference of *t2* and *t3*. Afterwards *t1* | -| | == *t2* - *t3* and *t2* == *t1* + *t3* are | +| ``t1 = t2 - t3`` | Difference of ``t2`` and ``t3``. Afterwards | +| | ``t1 == t2 - t3`` and ``t2 == t1 + t3`` are | | | true. (1)(6) | +--------------------------------+-----------------------------------------------+ | ``t1 = t2 * i or t1 = i * t2`` | Delta multiplied by an integer. | -| | Afterwards *t1* // i == *t2* is true, | +| | Afterwards ``t1 // i == t2`` is true, | | | provided ``i != 0``. | +--------------------------------+-----------------------------------------------+ -| | In general, *t1* \* i == *t1* \* (i-1) + *t1* | +| | In general, ``t1 * i == t1 * (i-1) + t1`` | | | is true. (1) | +--------------------------------+-----------------------------------------------+ | ``t1 = t2 * f or t1 = f * t2`` | Delta multiplied by a float. The result is | | | rounded to the nearest multiple of | | | timedelta.resolution using round-half-to-even.| +--------------------------------+-----------------------------------------------+ -| ``f = t2 / t3`` | Division (3) of overall duration *t2* by | -| | interval unit *t3*. Returns a :class:`float` | +| ``f = t2 / t3`` | Division (3) of overall duration ``t2`` by | +| | interval unit ``t3``. Returns a :class:`float`| | | object. | +--------------------------------+-----------------------------------------------+ | ``t1 = t2 / f or t1 = t2 / i`` | Delta divided by a float or an int. The result| @@ -337,19 +358,18 @@ Supported operations: +--------------------------------+-----------------------------------------------+ | ``q, r = divmod(t1, t2)`` | Computes the quotient and the remainder: | | | ``q = t1 // t2`` (3) and ``r = t1 % t2``. | -| | q is an integer and r is a :class:`timedelta` | -| | object. | +| | ``q`` is an integer and ``r`` is a | +| | :class:`timedelta` object. | +--------------------------------+-----------------------------------------------+ | ``+t1`` | Returns a :class:`timedelta` object with the | | | same value. (2) | +--------------------------------+-----------------------------------------------+ -| ``-t1`` | equivalent to | -| | :class:`timedelta`\ (-*t1.days*, | -| | -*t1.seconds*, -*t1.microseconds*), | -| | and to *t1*\* -1. (1)(4) | +| ``-t1`` | Equivalent to ``timedelta(-t1.days, | +| | -t1.seconds, -t1.microseconds)``, | +| | and to ``t1 * -1``. (1)(4) | +--------------------------------+-----------------------------------------------+ -| ``abs(t)`` | equivalent to +\ *t* when ``t.days >= 0``, | -| | and to -*t* when ``t.days < 0``. (2) | +| ``abs(t)`` | Equivalent to ``+t`` when ``t.days >= 0``, | +| | and to ``-t`` when ``t.days < 0``. (2) | +--------------------------------+-----------------------------------------------+ | ``str(t)`` | Returns a string in the form | | | ``[D day[s], ][H]H:MM:SS[.UUUUUU]``, where D | @@ -370,10 +390,10 @@ Notes: This is exact and cannot overflow. (3) - Division by 0 raises :exc:`ZeroDivisionError`. + Division by zero raises :exc:`ZeroDivisionError`. (4) - -*timedelta.max* is not representable as a :class:`timedelta` object. + ``-timedelta.max`` is not representable as a :class:`timedelta` object. (5) String representations of :class:`timedelta` objects are normalized @@ -506,7 +526,7 @@ Other constructors, all class methods: January 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless ``1 <= ordinal <= - date.max.toordinal()``. For any date *d*, + date.max.toordinal()``. For any date ``d``, ``date.fromordinal(d.toordinal()) == d``. @@ -542,6 +562,39 @@ Other constructors, all class methods: .. versionadded:: 3.8 +.. classmethod:: date.strptime(date_string, format) + + Return a :class:`.date` corresponding to *date_string*, parsed according to + *format*. This is equivalent to:: + + date(*(time.strptime(date_string, format)[0:3])) + + :exc:`ValueError` is raised if the date_string and format + can't be parsed by :func:`time.strptime` or if it returns a value which isn't a + time tuple. See also :ref:`strftime-strptime-behavior` and + :meth:`date.fromisoformat`. + + .. note:: + + If *format* specifies a day of month without a year a + :exc:`DeprecationWarning` is emitted. This is to avoid a quadrennial + leap year bug in code seeking to parse only a month and day as the + default year used in absence of one in the format is not a leap year. + Such *format* values may raise an error as of Python 3.15. The + workaround is to always include a year in your *format*. If parsing + *date_string* values that do not have a year, explicitly add a year that + is a leap year before parsing: + + .. doctest:: + + >>> from datetime import date + >>> date_string = "02/29" + >>> when = date.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. + >>> when.strftime("%B %d") # doctest: +SKIP + 'February 29' + + .. versionadded:: 3.14 + Class attributes: @@ -583,10 +636,10 @@ Supported operations: +-------------------------------+----------------------------------------------+ | Operation | Result | +===============================+==============================================+ -| ``date2 = date1 + timedelta`` | *date2* will be ``timedelta.days`` days | -| | after *date1*. (1) | +| ``date2 = date1 + timedelta`` | ``date2`` will be ``timedelta.days`` days | +| | after ``date1``. (1) | +-------------------------------+----------------------------------------------+ -| ``date2 = date1 - timedelta`` | Computes *date2* such that ``date2 + | +| ``date2 = date1 - timedelta`` | Computes ``date2`` such that ``date2 + | | | timedelta == date1``. (2) | +-------------------------------+----------------------------------------------+ | ``timedelta = date1 - date2`` | \(3) | @@ -613,8 +666,8 @@ Notes: ``timedelta.seconds`` and ``timedelta.microseconds`` are ignored. (3) - This is exact, and cannot overflow. timedelta.seconds and - timedelta.microseconds are 0, and date2 + timedelta == date1 after. + This is exact, and cannot overflow. ``timedelta.seconds`` and + ``timedelta.microseconds`` are 0, and ``date2 + timedelta == date1`` after. (4) :class:`date` objects are equal if they represent the same date. @@ -635,7 +688,7 @@ Notes: .. versionchanged:: 3.13 Comparison between :class:`.datetime` object and an instance of the :class:`date` subclass that is not a :class:`!datetime` subclass - no longer coverts the latter to :class:`!date`, ignoring the time part + no longer converts the latter to :class:`!date`, ignoring the time part and the time zone. The default behavior can be changed by overriding the special comparison methods in subclasses. @@ -671,13 +724,13 @@ Instance methods: time.struct_time((d.year, d.month, d.day, 0, 0, 0, d.weekday(), yday, -1)) where ``yday = d.toordinal() - date(d.year, 1, 1).toordinal() + 1`` - is the day number within the current year starting with ``1`` for January 1st. + is the day number within the current year starting with 1 for January 1st. .. method:: date.toordinal() Return the proleptic Gregorian ordinal of the date, where January 1 of year 1 - has ordinal 1. For any :class:`date` object *d*, + has ordinal 1. For any :class:`date` object ``d``, ``date.fromordinal(d.toordinal()) == d``. @@ -729,7 +782,7 @@ Instance methods: .. method:: date.__str__() - For a date *d*, ``str(d)`` is equivalent to ``d.isoformat()``. + For a date ``d``, ``str(d)`` is equivalent to ``d.isoformat()``. .. method:: date.ctime() @@ -869,7 +922,7 @@ Other constructors, all class methods: .. classmethod:: datetime.today() - Return the current local datetime, with :attr:`.tzinfo` ``None``. + Return the current local date and time, with :attr:`.tzinfo` ``None``. Equivalent to:: @@ -895,6 +948,10 @@ Other constructors, all class methods: This function is preferred over :meth:`today` and :meth:`utcnow`. + .. note:: + + Subsequent calls to :meth:`!datetime.now` may return the same + instant depending on the precision of the underlying clock. .. classmethod:: datetime.utcnow() @@ -991,8 +1048,8 @@ Other constructors, all class methods: .. classmethod:: datetime.fromordinal(ordinal) Return the :class:`.datetime` corresponding to the proleptic Gregorian ordinal, - where January 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless ``1 - <= ordinal <= datetime.max.toordinal()``. The hour, minute, second and + where January 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless + ``1 <= ordinal <= datetime.max.toordinal()``. The hour, minute, second and microsecond of the result are all 0, and :attr:`.tzinfo` is ``None``. @@ -1006,7 +1063,7 @@ Other constructors, all class methods: is used. If the *date* argument is a :class:`.datetime` object, its time components and :attr:`.tzinfo` attributes are ignored. - For any :class:`.datetime` object *d*, + For any :class:`.datetime` object ``d``, ``d == datetime.combine(d.date(), d.time(), d.tzinfo)``. .. versionchanged:: 3.6 @@ -1053,7 +1110,7 @@ Other constructors, all class methods: .. versionadded:: 3.7 .. versionchanged:: 3.11 Previously, this method only supported formats that could be emitted by - :meth:`date.isoformat()` or :meth:`datetime.isoformat()`. + :meth:`date.isoformat` or :meth:`datetime.isoformat`. .. classmethod:: datetime.fromisocalendar(year, week, day) @@ -1070,7 +1127,7 @@ Other constructors, all class methods: Return a :class:`.datetime` corresponding to *date_string*, parsed according to *format*. - If *format* does not contain microseconds or timezone information, this is equivalent to:: + If *format* does not contain microseconds or time zone information, this is equivalent to:: datetime(*(time.strptime(date_string, format)[0:6])) @@ -1079,6 +1136,24 @@ Other constructors, all class methods: time tuple. See also :ref:`strftime-strptime-behavior` and :meth:`datetime.fromisoformat`. + .. versionchanged:: 3.13 + + If *format* specifies a day of month without a year a + :exc:`DeprecationWarning` is now emitted. This is to avoid a quadrennial + leap year bug in code seeking to parse only a month and day as the + default year used in absence of one in the format is not a leap year. + Such *format* values may raise an error as of Python 3.15. The + workaround is to always include a year in your *format*. If parsing + *date_string* values that do not have a year, explicitly add a year that + is a leap year before parsing: + + .. doctest:: + + >>> from datetime import datetime + >>> date_string = "02/29" + >>> when = datetime.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. + >>> when.strftime("%B %d") # doctest: +SKIP + 'February 29' Class attributes: @@ -1149,8 +1224,8 @@ Instance attributes (read-only): In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A repeated interval occurs when clocks are rolled back at the end of daylight saving time or when the UTC offset for the current zone is decreased for political reasons.) - The value 0 (1) represents the earlier (later) of the two moments with the same wall - time representation. + The values 0 and 1 represent, respectively, the earlier and later of the two + moments with the same wall time representation. .. versionadded:: 3.6 @@ -1175,16 +1250,16 @@ Supported operations: +---------------------------------------+--------------------------------+ (1) - datetime2 is a duration of timedelta removed from datetime1, moving forward in - time if ``timedelta.days`` > 0, or backward if ``timedelta.days`` < 0. The + ``datetime2`` is a duration of ``timedelta`` removed from ``datetime1``, moving forward in + time if ``timedelta.days > 0``, or backward if ``timedelta.days < 0``. The result has the same :attr:`~.datetime.tzinfo` attribute as the input datetime, and - datetime2 - datetime1 == timedelta after. :exc:`OverflowError` is raised if - datetime2.year would be smaller than :const:`MINYEAR` or larger than + ``datetime2 - datetime1 == timedelta`` after. :exc:`OverflowError` is raised if + ``datetime2.year`` would be smaller than :const:`MINYEAR` or larger than :const:`MAXYEAR`. Note that no time zone adjustments are done even if the input is an aware object. (2) - Computes the datetime2 such that datetime2 + timedelta == datetime1. As for + Computes the ``datetime2`` such that ``datetime2 + timedelta == datetime1``. As for addition, the result has the same :attr:`~.datetime.tzinfo` attribute as the input datetime, and no time zone adjustments are done even if the input is aware. @@ -1195,11 +1270,11 @@ Supported operations: If both are naive, or both are aware and have the same :attr:`~.datetime.tzinfo` attribute, the :attr:`~.datetime.tzinfo` attributes are ignored, and the result is a :class:`timedelta` - object *t* such that ``datetime2 + t == datetime1``. No time zone adjustments + object ``t`` such that ``datetime2 + t == datetime1``. No time zone adjustments are done in this case. If both are aware and have different :attr:`~.datetime.tzinfo` attributes, ``a-b`` acts - as if *a* and *b* were first converted to naive UTC datetimes. The + as if ``a`` and ``b`` were first converted to naive UTC datetimes. The result is ``(a.replace(tzinfo=None) - a.utcoffset()) - (b.replace(tzinfo=None) - b.utcoffset())`` except that the implementation never overflows. @@ -1239,7 +1314,7 @@ Supported operations: .. versionchanged:: 3.13 Comparison between :class:`.datetime` object and an instance of the :class:`date` subclass that is not a :class:`!datetime` subclass - no longer coverts the latter to :class:`!date`, ignoring the time part + no longer converts the latter to :class:`!date`, ignoring the time part and the time zone. The default behavior can be changed by overriding the special comparison methods in subclasses. @@ -1293,22 +1368,22 @@ Instance methods: If provided, *tz* must be an instance of a :class:`tzinfo` subclass, and its :meth:`utcoffset` and :meth:`dst` methods must not return ``None``. If *self* - is naive, it is presumed to represent time in the system timezone. + is naive, it is presumed to represent time in the system time zone. If called without arguments (or with ``tz=None``) the system local - timezone is assumed for the target timezone. The ``.tzinfo`` attribute of the converted + time zone is assumed for the target time zone. The ``.tzinfo`` attribute of the converted datetime instance will be set to an instance of :class:`timezone` with the zone name and offset obtained from the OS. If ``self.tzinfo`` is *tz*, ``self.astimezone(tz)`` is equal to *self*: no adjustment of date or time data is performed. Else the result is local - time in the timezone *tz*, representing the same UTC time as *self*: after + time in the time zone *tz*, representing the same UTC time as *self*: after ``astz = dt.astimezone(tz)``, ``astz - astz.utcoffset()`` will have the same date and time data as ``dt - dt.utcoffset()``. - If you merely want to attach a time zone object *tz* to a datetime *dt* without + If you merely want to attach a :class:`timezone` object *tz* to a datetime *dt* without adjustment of date and time data, use ``dt.replace(tzinfo=tz)``. If you - merely want to remove the time zone object from an aware datetime *dt* without + merely want to remove the :class:`!timezone` object from an aware datetime *dt* without conversion of date and time data, use ``dt.replace(tzinfo=None)``. Note that the default :meth:`tzinfo.fromutc` method can be overridden in a @@ -1318,7 +1393,7 @@ Instance methods: def astimezone(self, tz): if self.tzinfo is tz: return self - # Convert self to UTC, and attach the new time zone object. + # Convert self to UTC, and attach the new timezone object. utc = (self - self.utcoffset()).replace(tzinfo=tz) # Convert from UTC to tz's local time. return tz.fromutc(utc) @@ -1369,24 +1444,24 @@ Instance methods: d.weekday(), yday, dst)) where ``yday = d.toordinal() - date(d.year, 1, 1).toordinal() + 1`` - is the day number within the current year starting with ``1`` for January + is the day number within the current year starting with 1 for January 1st. The :attr:`~time.struct_time.tm_isdst` flag of the result is set according to the :meth:`dst` method: :attr:`.tzinfo` is ``None`` or :meth:`dst` returns ``None``, :attr:`!tm_isdst` is set to ``-1``; else if :meth:`dst` returns a - non-zero value, :attr:`!tm_isdst` is set to ``1``; else :attr:`!tm_isdst` is - set to ``0``. + non-zero value, :attr:`!tm_isdst` is set to 1; else :attr:`!tm_isdst` is + set to 0. .. method:: datetime.utctimetuple() - If :class:`.datetime` instance *d* is naive, this is the same as + If :class:`.datetime` instance ``d`` is naive, this is the same as ``d.timetuple()`` except that :attr:`~.time.struct_time.tm_isdst` is forced to 0 regardless of what ``d.dst()`` returns. DST is never in effect for a UTC time. - If *d* is aware, *d* is normalized to UTC time, by subtracting + If ``d`` is aware, ``d`` is normalized to UTC time, by subtracting ``d.utcoffset()``, and a :class:`time.struct_time` for the normalized time is returned. :attr:`!tm_isdst` is forced to 0. Note - that an :exc:`OverflowError` may be raised if *d*.year was + that an :exc:`OverflowError` may be raised if ``d.year`` was ``MINYEAR`` or ``MAXYEAR`` and UTC adjustment spills over a year boundary. @@ -1432,7 +1507,7 @@ Instance methods: There is no method to obtain the POSIX timestamp directly from a naive :class:`.datetime` instance representing UTC time. If your - application uses this convention and your system timezone is not + application uses this convention and your system time zone is not set to UTC, you can obtain the POSIX timestamp by supplying ``tzinfo=timezone.utc``:: @@ -1531,7 +1606,7 @@ Instance methods: .. method:: datetime.__str__() - For a :class:`.datetime` instance *d*, ``str(d)`` is equivalent to + For a :class:`.datetime` instance ``d``, ``str(d)`` is equivalent to ``d.isoformat(' ')``. @@ -1717,7 +1792,7 @@ day, and subject to adjustment via a :class:`tzinfo` object. * ``fold in [0, 1]``. If an argument outside those ranges is given, :exc:`ValueError` is raised. All - default to ``0`` except *tzinfo*, which defaults to :const:`None`. + default to 0 except *tzinfo*, which defaults to ``None``. Class attributes: @@ -1772,13 +1847,13 @@ Instance attributes (read-only): In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A repeated interval occurs when clocks are rolled back at the end of daylight saving time or when the UTC offset for the current zone is decreased for political reasons.) - The value 0 (1) represents the earlier (later) of the two moments with the same wall - time representation. + The values 0 and 1 represent, respectively, the earlier and later of the two + moments with the same wall time representation. .. versionadded:: 3.6 :class:`.time` objects support equality and order comparisons, -where *a* is considered less than *b* when *a* precedes *b* in time. +where ``a`` is considered less than ``b`` when ``a`` precedes ``b`` in time. Naive and aware :class:`!time` objects are never equal. Order comparison between naive and aware :class:`!time` objects raises @@ -1803,7 +1878,7 @@ In Boolean contexts, a :class:`.time` object is always considered to be true. details. -Other constructor: +Other constructors: .. classmethod:: time.fromisoformat(time_string) @@ -1843,7 +1918,23 @@ Other constructor: .. versionadded:: 3.7 .. versionchanged:: 3.11 Previously, this method only supported formats that could be emitted by - :meth:`time.isoformat()`. + :meth:`time.isoformat`. + +.. classmethod:: time.strptime(date_string, format) + + Return a :class:`.time` corresponding to *date_string*, parsed according to + *format*. + + If *format* does not contain microseconds or timezone information, this is equivalent to:: + + time(*(time.strptime(date_string, format)[3:6])) + + :exc:`ValueError` is raised if the *date_string* and *format* + cannot be parsed by :func:`time.strptime` or if it returns a value which is not a + time tuple. See also :ref:`strftime-strptime-behavior` and + :meth:`time.fromisoformat`. + + .. versionadded:: 3.14 Instance methods: @@ -1909,7 +2000,7 @@ Instance methods: .. method:: time.__str__() - For a time *t*, ``str(t)`` is equivalent to ``t.isoformat()``. + For a time ``t``, ``str(t)`` is equivalent to ``t.isoformat()``. .. method:: time.strftime(format) @@ -2003,7 +2094,7 @@ Examples of working with a :class:`.time` object:: supply implementations of the standard :class:`tzinfo` methods needed by the :class:`.datetime` methods you use. The :mod:`!datetime` module provides :class:`timezone`, a simple concrete subclass of :class:`tzinfo` which can - represent timezones with fixed offset from UTC such as UTC itself or North + represent time zones with fixed offset from UTC such as UTC itself or North American EST and EDT. Special requirement for pickling: A :class:`tzinfo` subclass must have an @@ -2065,7 +2156,7 @@ Examples of working with a :class:`.time` object:: ``tz.utcoffset(dt) - tz.dst(dt)`` must return the same result for every :class:`.datetime` *dt* with ``dt.tzinfo == - tz`` For sane :class:`tzinfo` subclasses, this expression yields the time + tz``. For sane :class:`tzinfo` subclasses, this expression yields the time zone's "standard offset", which should not depend on the date or the time, but only on geographic location. The implementation of :meth:`datetime.astimezone` relies on this, but cannot detect violations; it's the programmer's @@ -2102,7 +2193,7 @@ Examples of working with a :class:`.time` object:: Return the time zone name corresponding to the :class:`.datetime` object *dt*, as a string. Nothing about string names is defined by the :mod:`!datetime` module, and there's no requirement that it mean anything in particular. For example, - "GMT", "UTC", "-500", "-5:00", "EDT", "US/Eastern", "America/New York" are all + ``"GMT"``, ``"UTC"``, ``"-500"``, ``"-5:00"``, ``"EDT"``, ``"US/Eastern"``, ``"America/New York"`` are all valid replies. Return ``None`` if a string name isn't known. Note that this is a method rather than a fixed string primarily because some :class:`tzinfo` subclasses will wish to return different names depending on the specific value @@ -2128,14 +2219,14 @@ When a :class:`.datetime` object is passed in response to a :class:`.datetime` method, ``dt.tzinfo`` is the same object as *self*. :class:`tzinfo` methods can rely on this, unless user code calls :class:`tzinfo` methods directly. The intent is that the :class:`tzinfo` methods interpret *dt* as being in local -time, and not need worry about objects in other timezones. +time, and not need worry about objects in other time zones. There is one more :class:`tzinfo` method that a subclass may wish to override: .. method:: tzinfo.fromutc(dt) - This is called from the default :class:`datetime.astimezone()` + This is called from the default :meth:`datetime.astimezone` implementation. When called from that, ``dt.tzinfo`` is *self*, and *dt*'s date and time data are to be viewed as expressing a UTC time. The purpose of :meth:`fromutc` is to adjust the date and time data, returning an @@ -2245,12 +2336,12 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). :mod:`zoneinfo` The :mod:`!datetime` module has a basic :class:`timezone` class (for handling arbitrary fixed offsets from UTC) and its :attr:`timezone.utc` - attribute (a UTC timezone instance). + attribute (a UTC :class:`!timezone` instance). - ``zoneinfo`` brings the *IANA timezone database* (also known as the Olson + ``zoneinfo`` brings the *IANA time zone database* (also known as the Olson database) to Python, and its usage is recommended. - `IANA timezone database `_ + `IANA time zone database `_ The Time Zone Database (often called tz, tzdata or zoneinfo) contains code and data that represent the history of local time for many representative locations around the globe. It is updated periodically to reflect changes @@ -2264,10 +2355,10 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). ------------------------- The :class:`timezone` class is a subclass of :class:`tzinfo`, each -instance of which represents a timezone defined by a fixed offset from +instance of which represents a time zone defined by a fixed offset from UTC. -Objects of this class cannot be used to represent timezone information in the +Objects of this class cannot be used to represent time zone information in the locations where different offsets are used in different days of the year or where historical changes have been made to civil time. @@ -2328,7 +2419,7 @@ Class attributes: .. attribute:: timezone.utc - The UTC timezone, ``timezone(timedelta(0))``. + The UTC time zone, ``timezone(timedelta(0))``. .. index:: @@ -2343,24 +2434,22 @@ Class attributes: ``strftime(format)`` method, to create a string representing the time under the control of an explicit format string. -Conversely, the :meth:`datetime.strptime` class method creates a -:class:`.datetime` object from a string representing a date and time and a -corresponding format string. +Conversely, the :meth:`date.strptime`, :meth:`datetime.strptime` and +:meth:`time.strptime` class methods create an object from a string +representing the time and a corresponding format string. The table below provides a high-level comparison of :meth:`~.datetime.strftime` versus :meth:`~.datetime.strptime`: -+----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ -| | ``strftime`` | ``strptime`` | -+================+========================================================+==============================================================================+ -| Usage | Convert object to a string according to a given format | Parse a string into a :class:`.datetime` object given a corresponding format | -+----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ -| Type of method | Instance method | Class method | -+----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ -| Method of | :class:`date`; :class:`.datetime`; :class:`.time` | :class:`.datetime` | -+----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ -| Signature | ``strftime(format)`` | ``strptime(date_string, format)`` | -+----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ ++----------------+--------------------------------------------------------+------------------------------------------------------------+ +| | ``strftime`` | ``strptime`` | ++================+========================================================+============================================================+ +| Usage | Convert object to a string according to a given format | Parse a string into an object given a corresponding format | ++----------------+--------------------------------------------------------+------------------------------------------------------------+ +| Type of method | Instance method | Class method | ++----------------+--------------------------------------------------------+------------------------------------------------------------+ +| Signature | ``strftime(format)`` | ``strptime(date_string, format)`` | ++----------------+--------------------------------------------------------+------------------------------------------------------------+ .. _format-codes: @@ -2537,17 +2626,17 @@ Using ``datetime.strptime(date_string, format)`` is equivalent to:: datetime(*(time.strptime(date_string, format)[0:6])) -except when the format includes sub-second components or timezone offset +except when the format includes sub-second components or time zone offset information, which are supported in ``datetime.strptime`` but are discarded by ``time.strptime``. For :class:`.time` objects, the format codes for year, month, and day should not be used, as :class:`!time` objects have no such values. If they're used anyway, -``1900`` is substituted for the year, and ``1`` for the month and day. +1900 is substituted for the year, and 1 for the month and day. For :class:`date` objects, the format codes for hours, minutes, seconds, and microseconds should not be used, as :class:`date` objects have no such -values. If they're used anyway, ``0`` is substituted for them. +values. If they're used anyway, 0 is substituted for them. For the same reason, handling of format strings containing Unicode code points that can't be represented in the charset of the current locale is also @@ -2657,6 +2746,25 @@ Notes: for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%j``, ``%U``, ``%W``, and ``%V``. Format ``%y`` does require a leading zero. +(10) + When parsing a month and day using :meth:`~.datetime.strptime`, always + include a year in the format. If the value you need to parse lacks a year, + append an explicit dummy leap year. Otherwise your code will raise an + exception when it encounters leap day because the default year used by the + parser is not a leap year. Users run into this bug every four years... + + .. doctest:: + + >>> month_day = "02/29" + >>> datetime.strptime(f"{month_day};1984", "%m/%d;%Y") # No leap year bug. + datetime.datetime(1984, 2, 29, 0, 0) + + .. deprecated-removed:: 3.13 3.15 + :meth:`~.datetime.strptime` calls using a format string containing + a day of month without a year now emit a + :exc:`DeprecationWarning`. In 3.15 or later we may change this into + an error or change the default year to a leap year. See :gh:`70647`. + .. rubric:: Footnotes .. [#] If, that is, we ignore the effects of Relativity @@ -2671,4 +2779,4 @@ Notes: `_ for a good explanation. -.. [#] Passing ``datetime.strptime('Feb 29', '%b %d')`` will fail since ``1900`` is not a leap year. +.. [#] Passing ``datetime.strptime('Feb 29', '%b %d')`` will fail since 1900 is not a leap year. diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 227b55c4315419..6c659ea52ad821 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -1,5 +1,5 @@ -:mod:`dbm` --- Interfaces to Unix "databases" -============================================= +:mod:`!dbm` --- Interfaces to Unix "databases" +============================================== .. module:: dbm :synopsis: Interfaces to various Unix "database" formats. @@ -19,7 +19,6 @@ slow-but-simple implementation in module :mod:`dbm.dumb` will be used. There is a `third party interface `_ to the Oracle Berkeley DB. - .. exception:: error A tuple containing the exceptions that can be raised by each of the supported @@ -163,6 +162,8 @@ SQLite backend for the :mod:`dbm` module. The files created by :mod:`dbm.sqlite3` can thus be opened by :mod:`sqlite3`, or any other SQLite browser, including the SQLite CLI. +.. include:: ../includes/wasm-notavail.rst + .. function:: open(filename, /, flag="r", mode=0o666) Open an SQLite database. @@ -206,6 +207,8 @@ functionality like crash tolerance. The file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible and can not be used interchangeably. +.. include:: ../includes/wasm-mobile-notavail.rst + .. exception:: error Raised on :mod:`dbm.gnu`-specific errors, such as I/O errors. :exc:`KeyError` is @@ -325,6 +328,8 @@ This module can be used with the "classic" NDBM interface or the when storing values larger than this limit. Reading such corrupted files can result in a hard crash (segmentation fault). +.. include:: ../includes/wasm-mobile-notavail.rst + .. exception:: error Raised on :mod:`dbm.ndbm`-specific errors, such as I/O errors. :exc:`KeyError` is raised @@ -455,4 +460,3 @@ The :mod:`!dbm.dumb` module defines the following: .. method:: dumbdbm.close() Close the database. - diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index fb8bbf72adf268..185eaf3f721c72 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -1,5 +1,5 @@ -:mod:`decimal` --- Decimal fixed point and floating point arithmetic -==================================================================== +:mod:`!decimal` --- Decimal fixed-point and floating-point arithmetic +===================================================================== .. module:: decimal :synopsis: Implementation of the General Decimal Arithmetic Specification. @@ -31,7 +31,7 @@ -------------- The :mod:`decimal` module provides support for fast correctly rounded -decimal floating point arithmetic. It offers several advantages over the +decimal floating-point arithmetic. It offers several advantages over the :class:`float` datatype: * Decimal "is based on a floating-point model which was designed with people @@ -207,7 +207,7 @@ a decimal raises :class:`InvalidOperation`:: .. versionchanged:: 3.3 Decimals interact well with much of the rest of Python. Here is a small decimal -floating point flying circus: +floating-point flying circus: .. doctest:: :options: +NORMALIZE_WHITESPACE @@ -373,7 +373,7 @@ Decimal objects digits, and an integer exponent. For example, ``Decimal((0, (1, 4, 1, 4), -3))`` returns ``Decimal('1.414')``. - If *value* is a :class:`float`, the binary floating point value is losslessly + If *value* is a :class:`float`, the binary floating-point value is losslessly converted to its exact decimal equivalent. This conversion can often require 53 or more digits of precision. For example, ``Decimal(float('1.1'))`` converts to @@ -403,7 +403,7 @@ Decimal objects Underscores are allowed for grouping, as with integral and floating-point literals in code. - Decimal floating point objects share many properties with the other built-in + Decimal floating-point objects share many properties with the other built-in numeric types such as :class:`float` and :class:`int`. All of the usual math operations and special methods apply. Likewise, decimal objects can be copied, pickled, printed, used as dictionary keys, used as set elements, @@ -445,7 +445,7 @@ Decimal objects Mixed-type comparisons between :class:`Decimal` instances and other numeric types are now fully supported. - In addition to the standard numeric properties, decimal floating point + In addition to the standard numeric properties, decimal floating-point objects also have a number of specialized methods: @@ -598,6 +598,23 @@ Decimal objects .. versionadded:: 3.1 + .. classmethod:: from_number(number) + + Alternative constructor that only accepts instances of + :class:`float`, :class:`int` or :class:`Decimal`, but not strings + or tuples. + + .. doctest:: + + >>> Decimal.from_number(314) + Decimal('314') + >>> Decimal.from_number(0.1) + Decimal('0.1000000000000000055511151231257827021181583404541015625') + >>> Decimal.from_number(Decimal('3.14')) + Decimal('3.14') + + .. versionadded:: 3.14 + .. method:: fma(other, third, context=None) Fused multiply-add. Return self*other+third with no rounding of the @@ -897,6 +914,48 @@ Decimal objects :const:`Rounded`. If given, applies *rounding*; otherwise, uses the rounding method in either the supplied *context* or the current context. + Decimal numbers can be rounded using the :func:`.round` function: + + .. describe:: round(number) + .. describe:: round(number, ndigits) + + If *ndigits* is not given or ``None``, + returns the nearest :class:`int` to *number*, + rounding ties to even, and ignoring the rounding mode of the + :class:`Decimal` context. Raises :exc:`OverflowError` if *number* is an + infinity or :exc:`ValueError` if it is a (quiet or signaling) NaN. + + If *ndigits* is an :class:`int`, the context's rounding mode is respected + and a :class:`Decimal` representing *number* rounded to the nearest + multiple of ``Decimal('1E-ndigits')`` is returned; in this case, + ``round(number, ndigits)`` is equivalent to + ``self.quantize(Decimal('1E-ndigits'))``. Returns ``Decimal('NaN')`` if + *number* is a quiet NaN. Raises :class:`InvalidOperation` if *number* + is an infinity, a signaling NaN, or if the length of the coefficient after + the quantize operation would be greater than the current context's + precision. In other words, for the non-corner cases: + + * if *ndigits* is positive, return *number* rounded to *ndigits* decimal + places; + * if *ndigits* is zero, return *number* rounded to the nearest integer; + * if *ndigits* is negative, return *number* rounded to the nearest + multiple of ``10**abs(ndigits)``. + + For example:: + + >>> from decimal import Decimal, getcontext, ROUND_DOWN + >>> getcontext().rounding = ROUND_DOWN + >>> round(Decimal('3.75')) # context rounding ignored + 4 + >>> round(Decimal('3.5')) # round-ties-to-even + 4 + >>> round(Decimal('3.75'), 0) # uses the context rounding + Decimal('3') + >>> round(Decimal('3.75'), 1) + Decimal('3.7') + >>> round(Decimal('3.75'), -1) + Decimal('0E+1') + .. _logical_operands_label: @@ -974,7 +1033,7 @@ New contexts can also be created using the :class:`Context` constructor described below. In addition, the module provides three pre-made contexts: -.. class:: BasicContext +.. data:: BasicContext This is a standard context defined by the General Decimal Arithmetic Specification. Precision is set to nine. Rounding is set to @@ -985,7 +1044,7 @@ described below. In addition, the module provides three pre-made contexts: Because many of the traps are enabled, this context is useful for debugging. -.. class:: ExtendedContext +.. data:: ExtendedContext This is a standard context defined by the General Decimal Arithmetic Specification. Precision is set to nine. Rounding is set to @@ -998,7 +1057,7 @@ described below. In addition, the module provides three pre-made contexts: presence of conditions that would otherwise halt the program. -.. class:: DefaultContext +.. data:: DefaultContext This context is used by the :class:`Context` constructor as a prototype for new contexts. Changing a field (such a precision) has the effect of changing the @@ -1517,7 +1576,7 @@ are also included in the pure Python version for compatibility. the C version uses a thread-local rather than a coroutine-local context and the value is ``False``. This is slightly faster in some nested context scenarios. -.. versionadded:: 3.9 backported to 3.7 and 3.8. + .. versionadded:: 3.8.3 Rounding modes @@ -1699,7 +1758,7 @@ The following table summarizes the hierarchy of signals:: .. _decimal-notes: -Floating Point Notes +Floating-Point Notes -------------------- @@ -1712,7 +1771,7 @@ can still incur round-off error when non-zero digits exceed the fixed precision. The effects of round-off error can be amplified by the addition or subtraction of nearly offsetting quantities resulting in loss of significance. Knuth -provides two instructive examples where rounded floating point arithmetic with +provides two instructive examples where rounded floating-point arithmetic with insufficient precision causes the breakdown of the associative and distributive properties of addition: @@ -1802,7 +1861,7 @@ treated as equal and their sign is informational. In addition to the two signed zeros which are distinct yet equal, there are various representations of zero with differing precisions yet equivalent in value. This takes a bit of getting used to. For an eye accustomed to -normalized floating point representations, it is not immediately obvious that +normalized floating-point representations, it is not immediately obvious that the following calculation returns a value equal to zero: >>> 1 / Decimal('Infinity') @@ -2129,7 +2188,7 @@ value unchanged: Q. Is there a way to convert a regular float to a :class:`Decimal`? -A. Yes, any binary floating point number can be exactly expressed as a +A. Yes, any binary floating-point number can be exactly expressed as a Decimal though an exact conversion may take more precision than intuition would suggest: @@ -2183,7 +2242,7 @@ Q. Is the CPython implementation fast for large numbers? A. Yes. In the CPython and PyPy3 implementations, the C/CFFI versions of the decimal module integrate the high speed `libmpdec `_ library for -arbitrary precision correctly rounded decimal floating point arithmetic [#]_. +arbitrary precision correctly rounded decimal floating-point arithmetic [#]_. ``libmpdec`` uses `Karatsuba multiplication `_ for medium-sized numbers and the `Number Theoretic Transform diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index d45e46448207a4..ce948a6860f02c 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -1,5 +1,5 @@ -:mod:`difflib` --- Helpers for computing deltas -=============================================== +:mod:`!difflib` --- Helpers for computing deltas +================================================ .. module:: difflib :synopsis: Helpers for computing differences between objects. @@ -631,7 +631,7 @@ If you want to know how to change the first sequence into the second, use work. * `Simple version control recipe - `_ for a small application + `_ for a small application built with :class:`SequenceMatcher`. diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 190e994a12cc71..f8f4188d27b472 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1,5 +1,5 @@ -:mod:`dis` --- Disassembler for Python bytecode -=============================================== +:mod:`!dis` --- Disassembler for Python bytecode +================================================ .. module:: dis :synopsis: Disassembler for Python bytecode. @@ -56,6 +56,12 @@ interpreter. for jump targets and exception handlers. The ``-O`` command line option and the ``show_offsets`` argument were added. + .. versionchanged:: 3.14 + The :option:`-P ` command-line option + and the ``show_positions`` argument were added. + + The :option:`-S ` command-line option is added. + Example: Given the function :func:`!myfunc`:: def myfunc(alist): @@ -85,7 +91,7 @@ The :mod:`dis` module can be invoked as a script from the command line: .. code-block:: sh - python -m dis [-h] [-C] [-O] [infile] + python -m dis [-h] [-C] [-O] [-P] [-S] [infile] The following options are accepted: @@ -103,8 +109,16 @@ The following options are accepted: Show offsets of instructions. +.. cmdoption:: -P, --show-positions + + Show positions of instructions in the source code. + +.. cmdoption:: -S, --specialized + + Show specialized bytecode. + If :file:`infile` is specified, its disassembled code will be written to stdout. -Otherwise, disassembly is performed on compiled source code recieved from stdin. +Otherwise, disassembly is performed on compiled source code received from stdin. Bytecode analysis ----------------- @@ -116,7 +130,8 @@ The bytecode analysis API allows pieces of Python code to be wrapped in a code. .. class:: Bytecode(x, *, first_line=None, current_offset=None,\ - show_caches=False, adaptive=False, show_offsets=False) + show_caches=False, adaptive=False, show_offsets=False,\ + show_positions=False) Analyse the bytecode corresponding to a function, generator, asynchronous generator, coroutine, method, string of source code, or a code object (as @@ -144,6 +159,9 @@ code. If *show_offsets* is ``True``, :meth:`.dis` will include instruction offsets in the output. + If *show_positions* is ``True``, :meth:`.dis` will include instruction + source code positions in the output. + .. classmethod:: from_traceback(tb, *, show_caches=False) Construct a :class:`Bytecode` instance from the given traceback, setting @@ -173,6 +191,12 @@ code. .. versionchanged:: 3.11 Added the *show_caches* and *adaptive* parameters. + .. versionchanged:: 3.13 + Added the *show_offsets* parameter + + .. versionchanged:: 3.14 + Added the *show_positions* parameter. + Example: .. doctest:: @@ -226,7 +250,8 @@ operation is being performed, so the intermediate analysis object isn't useful: Added *file* parameter. -.. function:: dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False) +.. function:: dis(x=None, *, file=None, depth=None, show_caches=False,\ + adaptive=False, show_offsets=False, show_positions=False) Disassemble the *x* object. *x* can denote either a module, a class, a method, a function, a generator, an asynchronous generator, a coroutine, @@ -265,9 +290,14 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.11 Added the *show_caches* and *adaptive* parameters. + .. versionchanged:: 3.13 + Added the *show_offsets* parameter. + + .. versionchanged:: 3.14 + Added the *show_positions* parameter. -.. function:: distb(tb=None, *, file=None, show_caches=False, adaptive=False, - show_offset=False) +.. function:: distb(tb=None, *, file=None, show_caches=False, adaptive=False,\ + show_offset=False, show_positions=False) Disassemble the top-of-stack function of a traceback, using the last traceback if none was passed. The instruction causing the exception is @@ -285,14 +315,20 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.13 Added the *show_offsets* parameter. -.. function:: disassemble(code, lasti=-1, *, file=None, show_caches=False, adaptive=False) - disco(code, lasti=-1, *, file=None, show_caches=False, adaptive=False, - show_offsets=False) + .. versionchanged:: 3.14 + Added the *show_positions* parameter. + +.. function:: disassemble(code, lasti=-1, *, file=None, show_caches=False,\ + adaptive=False, show_offsets=False, show_positions=False) + disco(code, lasti=-1, *, file=None, show_caches=False, adaptive=False,\ + show_offsets=False, show_positions=False) Disassemble a code object, indicating the last instruction if *lasti* was provided. The output is divided in the following columns: - #. the line number, for the first instruction of each line + #. the source code location of the instruction. Complete location information + is shown if *show_positions* is true. Otherwise (the default) only the + line number is displayed. #. the current instruction, indicated as ``-->``, #. a labelled instruction, indicated with ``>>``, #. the address of the instruction, @@ -315,6 +351,9 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.13 Added the *show_offsets* parameter. + .. versionchanged:: 3.14 + Added the *show_positions* parameter. + .. function:: get_instructions(x, *, first_line=None, show_caches=False, adaptive=False) Return an iterator over the instructions in the supplied function, method, @@ -336,9 +375,10 @@ operation is being performed, so the intermediate analysis object isn't useful: Added the *show_caches* and *adaptive* parameters. .. versionchanged:: 3.13 - The *show_caches* parameter is deprecated and has no effect. The *cache_info* - field of each instruction is populated regardless of its value. - + The *show_caches* parameter is deprecated and has no effect. The iterator + generates the :class:`Instruction` instances with the *cache_info* + field populated (regardless of the value of *show_caches*) and it no longer + generates separate items for the cache entries. .. function:: findlinestarts(code) @@ -779,16 +819,6 @@ not have to be) the original ``STACK[-2]``. .. versionadded:: 3.12 -.. opcode:: BEFORE_ASYNC_WITH - - Resolves ``__aenter__`` and ``__aexit__`` from ``STACK[-1]``. - Pushes ``__aexit__`` and result of ``__aenter__()`` to the stack:: - - STACK.extend((__aexit__, __aenter__()) - - .. versionadded:: 3.5 - - **Miscellaneous opcodes** @@ -838,13 +868,6 @@ iterations of the loop. Returns with ``STACK[-1]`` to the caller of the function. -.. opcode:: RETURN_CONST (consti) - - Returns with ``co_consts[consti]`` to the caller of the function. - - .. versionadded:: 3.12 - - .. opcode:: YIELD_VALUE Yields ``STACK.pop()`` from a :term:`generator`. @@ -929,12 +952,13 @@ iterations of the loop. Exception representation on the stack now consist of one, not three, items. -.. opcode:: LOAD_ASSERTION_ERROR +.. opcode:: LOAD_COMMON_CONSTANT - Pushes :exc:`AssertionError` onto the stack. Used by the :keyword:`assert` - statement. + Pushes a common constant onto the stack. The interpreter contains a hardcoded + list of constants supported by this instruction. Used by the :keyword:`assert` + statement to load :exc:`AssertionError`. - .. versionadded:: 3.9 + .. versionadded:: 3.14 .. opcode:: LOAD_BUILD_CLASS @@ -942,21 +966,10 @@ iterations of the loop. Pushes :func:`!builtins.__build_class__` onto the stack. It is later called to construct a class. - -.. opcode:: BEFORE_WITH - - This opcode performs several operations before a with block starts. First, - it loads :meth:`~object.__exit__` from the context manager and pushes it onto - the stack for later use by :opcode:`WITH_EXCEPT_START`. Then, - :meth:`~object.__enter__` is called. Finally, the result of calling the - ``__enter__()`` method is pushed onto the stack. - - .. versionadded:: 3.11 - - .. opcode:: GET_LEN - Perform ``STACK.append(len(STACK[-1]))``. + Perform ``STACK.append(len(STACK[-1]))``. Used in :keyword:`match` statements where + comparison with structure of pattern is needed. .. versionadded:: 3.10 @@ -1072,6 +1085,22 @@ iterations of the loop. Pushes ``co_consts[consti]`` onto the stack. +.. opcode:: LOAD_SMALL_INT (i) + + Pushes the integer ``i`` onto the stack. + ``i`` must be in ``range(256)`` + + .. versionadded:: 3.14 + + +.. opcode:: LOAD_CONST_IMMORTAL (consti) + + Pushes ``co_consts[consti]`` onto the stack. + Can be used when the constant value is known to be immortal. + + .. versionadded:: 3.14 + + .. opcode:: LOAD_NAME (namei) Pushes the value associated with ``co_names[namei]`` onto the stack. @@ -1101,11 +1130,15 @@ iterations of the loop. .. opcode:: BUILD_TUPLE (count) Creates a tuple consuming *count* items from the stack, and pushes the - resulting tuple onto the stack.:: + resulting tuple onto the stack:: - assert count > 0 - STACK, values = STACK[:-count], STACK[-count:] - STACK.append(tuple(values)) + if count == 0: + value = () + else: + value = tuple(STACK[-count:]) + STACK = STACK[:-count] + + STACK.append(value) .. opcode:: BUILD_LIST (count) @@ -1129,15 +1162,6 @@ iterations of the loop. empty dictionary pre-sized to hold *count* items. -.. opcode:: BUILD_CONST_KEY_MAP (count) - - The version of :opcode:`BUILD_MAP` specialized for constant keys. Pops the - top element on the stack which contains a tuple of keys, then starting from - ``STACK[-2]``, pops *count* values to form values in the built dictionary. - - .. versionadded:: 3.6 - - .. opcode:: BUILD_STRING (count) Concatenates *count* strings from the stack and pushes the resulting string @@ -1224,7 +1248,7 @@ iterations of the loop. except that ``namei`` is shifted left by 2 bits instead of 1. The low bit of ``namei`` signals to attempt a method load, as with - :opcode:`LOAD_ATTR`, which results in pushing ``None`` and the loaded method. + :opcode:`LOAD_ATTR`, which results in pushing ``NULL`` and the loaded method. When it is unset a single value is pushed to the stack. The second-low bit of ``namei``, if set, means that this was a two-argument @@ -1377,6 +1401,13 @@ iterations of the loop. This opcode is now only used in situations where the local variable is guaranteed to be initialized. It cannot raise :exc:`UnboundLocalError`. +.. opcode:: LOAD_FAST_LOAD_FAST (var_nums) + + Pushes references to ``co_varnames[var_nums >> 4]`` and + ``co_varnames[var_nums & 15]`` onto the stack. + + .. versionadded:: 3.13 + .. opcode:: LOAD_FAST_CHECK (var_num) Pushes a reference to the local ``co_varnames[var_num]`` onto the stack, @@ -1397,6 +1428,20 @@ iterations of the loop. Stores ``STACK.pop()`` into the local ``co_varnames[var_num]``. +.. opcode:: STORE_FAST_STORE_FAST (var_nums) + + Stores ``STACK[-1]`` into ``co_varnames[var_nums >> 4]`` + and ``STACK[-2]`` into ``co_varnames[var_nums & 15]``. + + .. versionadded:: 3.13 + +.. opcode:: STORE_FAST_LOAD_FAST (var_nums) + + Stores ``STACK.pop()`` into the local ``co_varnames[var_nums >> 4]`` + and pushes a reference to the local ``co_varnames[var_nums & 15]`` + onto the stack. + + .. versionadded:: 3.13 .. opcode:: DELETE_FAST (var_num) @@ -1426,7 +1471,7 @@ iterations of the loop. slot ``i`` of the "fast locals" storage in this mapping. If the name is not found there, loads it from the cell contained in slot ``i``, similar to :opcode:`LOAD_DEREF`. This is used for loading - free variables in class bodies (which previously used + :term:`closure variables ` in class bodies (which previously used :opcode:`!LOAD_CLASSDEREF`) and in :ref:`annotation scopes ` within class bodies. @@ -1455,8 +1500,8 @@ iterations of the loop. .. opcode:: COPY_FREE_VARS (n) - Copies the ``n`` free variables from the closure into the frame. - Removes the need for special code on the caller's side when calling + Copies the ``n`` :term:`free (closure) variables ` from the closure + into the frame. Removes the need for special code on the caller's side when calling closures. .. versionadded:: 3.11 @@ -1544,7 +1589,7 @@ iterations of the loop. .. opcode:: MAKE_FUNCTION - Pushes a new function object on the stack built from the code object at ``STACK[1]``. + Pushes a new function object on the stack built from the code object at ``STACK[-1]``. .. versionchanged:: 3.10 Flag value ``0x04`` is a tuple of strings instead of dictionary @@ -1580,7 +1625,7 @@ iterations of the loop. end = STACK.pop() start = STACK.pop() - STACK.append(slice(start, stop)) + STACK.append(slice(start, end)) if it is 3, implements:: @@ -1629,7 +1674,7 @@ iterations of the loop. .. versionadded:: 3.13 -.. opcode:: FORMAT_SPEC +.. opcode:: FORMAT_WITH_SPEC Formats the given value with the given format spec:: @@ -1665,7 +1710,7 @@ iterations of the loop. A no-op. Performs internal tracing, debugging and optimization checks. - The ``context`` oparand consists of two parts. The lowest two bits + The ``context`` operand consists of two parts. The lowest two bits indicate where the ``RESUME`` occurs: * ``0`` The start of a function, which is neither a generator, coroutine @@ -1747,7 +1792,7 @@ iterations of the loop. | ``INTRINSIC_STOPITERATION_ERROR`` | Extracts the return value from a | | | ``StopIteration`` exception. | +-----------------------------------+-----------------------------------+ - | ``INTRINSIC_ASYNC_GEN_WRAP`` | Wraps an aync generator value | + | ``INTRINSIC_ASYNC_GEN_WRAP`` | Wraps an async generator value | +-----------------------------------+-----------------------------------+ | ``INTRINSIC_UNARY_POSITIVE`` | Performs the unary ``+`` | | | operation | @@ -1810,6 +1855,17 @@ iterations of the loop. .. versionadded:: 3.12 +.. opcode:: LOAD_SPECIAL + + Performs special method lookup on ``STACK[-1]``. + If ``type(STACK[-1]).__xxx__`` is a method, leave + ``type(STACK[-1]).__xxx__; STACK[-1]`` on the stack. + If ``type(STACK[-1]).__xxx__`` is not a method, leave + ``STACK[-1].__xxx__; NULL`` on the stack. + + .. versionadded:: 3.14 + + **Pseudo-instructions** These opcodes do not appear in Python bytecode. They are used by the compiler @@ -1853,6 +1909,12 @@ but are replaced by real opcodes or removed before bytecode is generated. Undirected relative jump instructions which are replaced by their directed (forward/backward) counterparts by the assembler. +.. opcode:: JUMP_IF_TRUE +.. opcode:: JUMP_IF_FALSE + + Conditional jumps which do not impact the stack. Replaced by the sequence + ``COPY 1``, ``TO_BOOL``, ``POP_JUMP_IF_TRUE/FALSE``. + .. opcode:: LOAD_CLOSURE (i) Pushes a reference to the cell contained in slot ``i`` of the "fast locals" @@ -1912,10 +1974,10 @@ instructions: .. data:: hasfree - Sequence of bytecodes that access a free variable. 'free' in this - context refers to names in the current scope that are referenced by inner - scopes or names in outer scopes that are referenced from this scope. It does - *not* include references to global or builtin scopes. + Sequence of bytecodes that access a :term:`free (closure) variable `. + 'free' in this context refers to names in the current scope that are + referenced by inner scopes or names in outer scopes that are referenced + from this scope. It does *not* include references to global or builtin scopes. .. data:: hasname diff --git a/Doc/library/distutils.rst b/Doc/library/distutils.rst new file mode 100644 index 00000000000000..af63e035bf3c4a --- /dev/null +++ b/Doc/library/distutils.rst @@ -0,0 +1,17 @@ +:mod:`!distutils` --- Building and installing Python modules +============================================================ + +.. module:: distutils + :synopsis: Removed in 3.12. + :deprecated: + +.. deprecated-removed:: 3.10 3.12 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.12 ` after +being deprecated in Python 3.10. The removal was decided in :pep:`632`, +which has `migration advice +`_. + +The last version of Python that provided the :mod:`!distutils` module was +`Python 3.11 `_. diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 1bfcd69f72df2e..106b0a6c95b7be 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -1,5 +1,5 @@ -:mod:`doctest` --- Test interactive Python examples -=================================================== +:mod:`!doctest` --- Test interactive Python examples +==================================================== .. module:: doctest :synopsis: Test pieces of code within docstrings. @@ -123,10 +123,10 @@ And so on, eventually ending with: OverflowError: n too large ok 2 items passed all tests: - 1 tests in __main__ - 8 tests in __main__.factorial - 9 tests in 2 items. - 9 passed and 0 failed. + 1 test in __main__ + 6 tests in __main__.factorial + 7 tests in 2 items. + 7 passed. Test passed. $ @@ -136,6 +136,10 @@ examples of doctests in the standard Python test suite and libraries. Especially useful examples can be found in the standard test file :file:`Lib/test/test_doctest/test_doctest.py`. +.. versionadded:: 3.13 + Output is colorized by default and can be + :ref:`controlled using environment variables `. + .. _doctest-simple-testmod: @@ -430,10 +434,10 @@ Simple example:: >>> [1, 2, 3].remove(42) Traceback (most recent call last): File "", line 1, in - ValueError: 42 is not in list + ValueError: list.remove(x): x not in list -That doctest succeeds if :exc:`ValueError` is raised, with the ``42 is not in list`` -detail as shown. +That doctest succeeds if :exc:`ValueError` is raised, with the ``list.remove(x): +x not in list`` detail as shown. The expected output for an exception must start with a traceback header, which may be either of the following two lines, indented the same as the first line of @@ -800,18 +804,18 @@ guarantee about output. For example, when printing a set, Python doesn't guarantee that the element is printed in any particular order, so a test like :: >>> foo() - {"Hermione", "Harry"} + {"spam", "eggs"} is vulnerable! One workaround is to do :: - >>> foo() == {"Hermione", "Harry"} + >>> foo() == {"spam", "eggs"} True instead. Another is to do :: >>> d = sorted(foo()) >>> d - ['Harry', 'Hermione'] + ['eggs', 'spam'] There are others, but you get the idea. @@ -1021,7 +1025,8 @@ from text files and modules with doctests: and runs the interactive examples in each file. If an example in any file fails, then the synthesized unit test fails, and a :exc:`failureException` exception is raised showing the name of the file containing the test and a - (sometimes approximate) line number. + (sometimes approximate) line number. If all the examples in a file are + skipped, then the synthesized unit test is also marked as skipped. Pass one or more paths (as strings) to text files to be examined. @@ -1087,7 +1092,8 @@ from text files and modules with doctests: and runs each doctest in the module. If any of the doctests fail, then the synthesized unit test fails, and a :exc:`failureException` exception is raised showing the name of the file containing the test and a (sometimes approximate) - line number. + line number. If all the examples in a docstring are skipped, then the + synthesized unit test is also marked as skipped. Optional argument *module* provides the module to be tested. It can be a module object or a (possibly dotted) module name. If not specified, the module calling @@ -1933,7 +1939,7 @@ such a test runner:: optionflags=flags) else: fail, total = doctest.testmod(optionflags=flags) - print("{} failures out of {} tests".format(fail, total)) + print(f"{fail} failures out of {total} tests") .. rubric:: Footnotes diff --git a/Doc/library/email.charset.rst b/Doc/library/email.charset.rst index aa0134412f3a60..6875af2be49d7a 100644 --- a/Doc/library/email.charset.rst +++ b/Doc/library/email.charset.rst @@ -1,5 +1,5 @@ -:mod:`email.charset`: Representing character sets -------------------------------------------------- +:mod:`!email.charset`: Representing character sets +-------------------------------------------------- .. module:: email.charset :synopsis: Character Sets diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index c4c322a82e1f44..4285c436e8da80 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -7,6 +7,7 @@ :synopsis: The base class representing email messages in a fashion backward compatible with Python 3.2 :noindex: + :no-index: The :class:`Message` class is very similar to the @@ -104,7 +105,7 @@ Here are the methods of the :class:`Message` class: .. method:: __str__() - Equivalent to :meth:`.as_string()`. Allows ``str(msg)`` to produce a + Equivalent to :meth:`.as_string`. Allows ``str(msg)`` to produce a string containing the formatted message. @@ -142,7 +143,7 @@ Here are the methods of the :class:`Message` class: .. method:: __bytes__() - Equivalent to :meth:`.as_bytes()`. Allows ``bytes(msg)`` to produce a + Equivalent to :meth:`.as_bytes`. Allows ``bytes(msg)`` to produce a bytes object containing the formatted message. .. versionadded:: 3.4 diff --git a/Doc/library/email.contentmanager.rst b/Doc/library/email.contentmanager.rst index 5b49339650f0e9..b33fe82a6e4c9f 100644 --- a/Doc/library/email.contentmanager.rst +++ b/Doc/library/email.contentmanager.rst @@ -1,5 +1,5 @@ -:mod:`email.contentmanager`: Managing MIME Content --------------------------------------------------- +:mod:`!email.contentmanager`: Managing MIME Content +--------------------------------------------------- .. module:: email.contentmanager :synopsis: Storing and Retrieving Content from MIME Parts @@ -58,11 +58,12 @@ * the type itself (``typ``) * the type's fully qualified name (``typ.__module__ + '.' + typ.__qualname__``). - * the type's qualname (``typ.__qualname__``) - * the type's name (``typ.__name__``). + * the type's :attr:`qualname ` (``typ.__qualname__``) + * the type's :attr:`name ` (``typ.__name__``). If none of the above match, repeat all of the checks above for each of - the types in the :term:`MRO` (``typ.__mro__``). Finally, if no other key + the types in the :term:`MRO` (:attr:`typ.__mro__ `). + Finally, if no other key yields a handler, check for a handler for the key ``None``. If there is no handler for ``None``, raise a :exc:`KeyError` for the fully qualified name of the type. @@ -156,7 +157,13 @@ Currently the email package provides only one concrete content manager, :exc:`ValueError`. * For ``str`` objects, if *cte* is not set use heuristics to - determine the most compact encoding. + determine the most compact encoding. Prior to encoding, + :meth:`str.splitlines` is used to normalize all line boundaries, + ensuring that each line of the payload is terminated by the + current policy's :data:`~email.policy.Policy.linesep` property + (even if the original string did not end with one). + * For ``bytes`` objects, *cte* is taken to be base64 if not set, + and the aforementioned newline translation is not performed. * For :class:`~email.message.EmailMessage`, per :rfc:`2046`, raise an error if a *cte* of ``quoted-printable`` or ``base64`` is requested for *subtype* ``rfc822``, and for any *cte* other than diff --git a/Doc/library/email.encoders.rst b/Doc/library/email.encoders.rst index 3bd377e33f6c15..9c8c8c9234ed7a 100644 --- a/Doc/library/email.encoders.rst +++ b/Doc/library/email.encoders.rst @@ -1,5 +1,5 @@ -:mod:`email.encoders`: Encoders -------------------------------- +:mod:`!email.encoders`: Encoders +-------------------------------- .. module:: email.encoders :synopsis: Encoders for email message payloads. diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst index 56aea6598b8615..f8f43d82a3df2e 100644 --- a/Doc/library/email.errors.rst +++ b/Doc/library/email.errors.rst @@ -1,5 +1,5 @@ -:mod:`email.errors`: Exception and Defect classes -------------------------------------------------- +:mod:`!email.errors`: Exception and Defect classes +-------------------------------------------------- .. module:: email.errors :synopsis: The exception classes used by the email package. @@ -58,6 +58,13 @@ The following exception classes are defined in the :mod:`email.errors` module: :class:`~email.mime.nonmultipart.MIMENonMultipart` (e.g. :class:`~email.mime.image.MIMEImage`). + +.. exception:: HeaderWriteError() + + Raised when an error occurs when the :mod:`~email.generator` outputs + headers. + + .. exception:: MessageDefect() This is the base class for all defects found when parsing email messages. diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst index afa0038ea2d6c4..a3132d02687bc9 100644 --- a/Doc/library/email.generator.rst +++ b/Doc/library/email.generator.rst @@ -1,5 +1,5 @@ -:mod:`email.generator`: Generating MIME documents -------------------------------------------------- +:mod:`!email.generator`: Generating MIME documents +-------------------------------------------------- .. module:: email.generator :synopsis: Generate flat text email messages from a message structure. diff --git a/Doc/library/email.header.rst b/Doc/library/email.header.rst index e093f138936b36..219fad0d2f6745 100644 --- a/Doc/library/email.header.rst +++ b/Doc/library/email.header.rst @@ -1,5 +1,5 @@ -:mod:`email.header`: Internationalized headers ----------------------------------------------- +:mod:`!email.header`: Internationalized headers +----------------------------------------------- .. module:: email.header :synopsis: Representing non-ASCII headers @@ -77,7 +77,7 @@ Here is the :class:`Header` class description: The maximum line length can be specified explicitly via *maxlinelen*. For splitting the first line to a shorter value (to account for the field header which isn't included in *s*, e.g. :mailheader:`Subject`) pass in the name of the - field in *header_name*. The default *maxlinelen* is 76, and the default value + field in *header_name*. The default *maxlinelen* is 78, and the default value for *header_name* is ``None``, meaning it is not taken into account for the first line of a long, split header. diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index 00a954e0307ea6..7f8044932fae99 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -1,5 +1,5 @@ -:mod:`email.headerregistry`: Custom Header Objects --------------------------------------------------- +:mod:`!email.headerregistry`: Custom Header Objects +--------------------------------------------------- .. module:: email.headerregistry :synopsis: Automatic Parsing of headers based on the field name @@ -317,7 +317,7 @@ variant, :attr:`~.BaseHeader.max_count` is set to 1. class. When *use_default_map* is ``True`` (the default), the standard mapping of header names to classes is copied in to the registry during initialization. *base_class* is always the last class in the generated - class's ``__bases__`` list. + class's :class:`~type.__bases__` list. The default mappings are: diff --git a/Doc/library/email.iterators.rst b/Doc/library/email.iterators.rst index d53ab33b8904a7..090981d84b4de3 100644 --- a/Doc/library/email.iterators.rst +++ b/Doc/library/email.iterators.rst @@ -1,5 +1,5 @@ -:mod:`email.iterators`: Iterators ---------------------------------- +:mod:`!email.iterators`: Iterators +---------------------------------- .. module:: email.iterators :synopsis: Iterate over a message object tree. diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index adea067e082615..71d6e321f387bc 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -1,5 +1,5 @@ -:mod:`email.message`: Representing an email message ---------------------------------------------------- +:mod:`!email.message`: Representing an email message +---------------------------------------------------- .. module:: email.message :synopsis: The base class representing email messages. @@ -41,7 +41,7 @@ The :class:`EmailMessage` dictionary-like interface is indexed by the header names, which must be ASCII values. The values of the dictionary are strings with some extra methods. Headers are stored and returned in case-preserving form, but field names are matched case-insensitively. The keys are ordered, -but unlike a real dict, there can be duplicates. Addtional methods are +but unlike a real dict, there can be duplicates. Additional methods are provided for working with headers that have duplicate keys. The *payload* is either a string or bytes object, in the case of simple message @@ -124,7 +124,7 @@ message objects. .. method:: __bytes__() - Equivalent to :meth:`.as_bytes()`. Allows ``bytes(msg)`` to produce a + Equivalent to :meth:`.as_bytes`. Allows ``bytes(msg)`` to produce a bytes object containing the serialized message. diff --git a/Doc/library/email.mime.rst b/Doc/library/email.mime.rst index dc0dd3b9eebde6..b85673a4acd0d0 100644 --- a/Doc/library/email.mime.rst +++ b/Doc/library/email.mime.rst @@ -1,5 +1,5 @@ -:mod:`email.mime`: Creating email and MIME objects from scratch ---------------------------------------------------------------- +:mod:`!email.mime`: Creating email and MIME objects from scratch +---------------------------------------------------------------- .. module:: email.mime :synopsis: Build MIME messages. diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index dda0466a6afa7d..439b5c8f34b65a 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -1,5 +1,5 @@ -:mod:`email.parser`: Parsing email messages -------------------------------------------- +:mod:`!email.parser`: Parsing email messages +-------------------------------------------- .. module:: email.parser :synopsis: Parse flat text email messages to produce a message object structure. diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index f4777bb2462138..6b997ee784f6e4 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -1,5 +1,5 @@ -:mod:`email.policy`: Policy Objects ------------------------------------ +:mod:`!email.policy`: Policy Objects +------------------------------------ .. module:: email.policy :synopsis: Controlling the parsing and generating of messages @@ -229,6 +229,24 @@ added matters. To illustrate:: .. versionadded:: 3.6 + + .. attribute:: verify_generated_headers + + If ``True`` (the default), the generator will raise + :exc:`~email.errors.HeaderWriteError` instead of writing a header + that is improperly folded or delimited, such that it would + be parsed as multiple headers or joined with adjacent data. + Such headers can be generated by custom header classes or bugs + in the ``email`` module. + + As it's a security feature, this defaults to ``True`` even in the + :class:`~email.policy.Compat32` policy. + For backwards compatible, but unsafe, behavior, it must be set to + ``False`` explicitly. + + .. versionadded:: 3.13 + + The following :class:`Policy` method is intended to be called by code using the email library to create policy instances with custom settings: @@ -249,7 +267,7 @@ added matters. To illustrate:: Handle a *defect* found on *obj*. When the email package calls this method, *defect* will always be a subclass of - :class:`~email.errors.Defect`. + :class:`~email.errors.MessageDefect`. The default implementation checks the :attr:`raise_on_defect` flag. If it is ``True``, *defect* is raised as an exception. If it is ``False`` @@ -259,7 +277,7 @@ added matters. To illustrate:: .. method:: register_defect(obj, defect) Register a *defect* on *obj*. In the email package, *defect* will always - be a subclass of :class:`~email.errors.Defect`. + be a subclass of :class:`~email.errors.MessageDefect`. The default implementation calls the ``append`` method of the ``defects`` attribute of *obj*. When the email package calls :attr:`handle_defect`, diff --git a/Doc/library/email.rst b/Doc/library/email.rst index 3a6039004fcaae..66c42e4a5008ee 100644 --- a/Doc/library/email.rst +++ b/Doc/library/email.rst @@ -1,5 +1,5 @@ -:mod:`email` --- An email and MIME handling package -=================================================== +:mod:`!email` --- An email and MIME handling package +==================================================== .. module:: email :synopsis: Package supporting the parsing, manipulating, and generating diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst index d693a9bc3933b5..611549604fda15 100644 --- a/Doc/library/email.utils.rst +++ b/Doc/library/email.utils.rst @@ -1,5 +1,5 @@ -:mod:`email.utils`: Miscellaneous utilities -------------------------------------------- +:mod:`!email.utils`: Miscellaneous utilities +-------------------------------------------- .. module:: email.utils :synopsis: Miscellaneous email package utilities. @@ -17,8 +17,7 @@ module: arguments, return current time. Otherwise *dt* argument should be a :class:`~datetime.datetime` instance, and it is converted to the local time zone according to the system time zone database. If *dt* is naive (that - is, ``dt.tzinfo`` is ``None``), it is assumed to be in local time. The - *isdst* parameter is ignored. + is, ``dt.tzinfo`` is ``None``), it is assumed to be in local time. .. versionadded:: 3.3 @@ -159,7 +158,7 @@ of the new API. Fri, 09 Nov 2001 01:08:47 -0000 - Optional *timeval* if given is a floating point time value as accepted by + Optional *timeval* if given is a floating-point time value as accepted by :func:`time.gmtime` and :func:`time.localtime`, otherwise the current time is used. diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index de3b93f5e61073..8dfb7ad9c95c3e 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -1,5 +1,5 @@ -:mod:`ensurepip` --- Bootstrapping the ``pip`` installer -======================================================== +:mod:`!ensurepip` --- Bootstrapping the ``pip`` installer +========================================================= .. module:: ensurepip :synopsis: Bootstrapping the "pip" installer into an existing Python @@ -38,7 +38,7 @@ when creating a virtual environment) or after explicitly uninstalling :pep:`453`: Explicit bootstrapping of pip in Python installations The original rationale and specification for this module. -.. include:: ../includes/wasm-notavail.rst +.. include:: ../includes/wasm-mobile-notavail.rst Command line interface ---------------------- diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 49bf40aa5a2869..8ca949368db4ff 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -1,5 +1,5 @@ -:mod:`enum` --- Support for enumerations -======================================== +:mod:`!enum` --- Support for enumerations +========================================= .. module:: enum :synopsis: Implementation of an enumeration class. @@ -44,7 +44,7 @@ using function-call syntax:: ... BLUE = 3 >>> # functional syntax - >>> Color = Enum('Color', ['RED', 'GREEN', 'BLUE']) + >>> Color = Enum('Color', [('RED', 1), ('GREEN', 2), ('BLUE', 3)]) Even though we can use :keyword:`class` syntax to create Enums, Enums are not normal Python classes. See @@ -110,6 +110,10 @@ Module Contents ``KEEP`` which allows for more fine-grained control over how invalid values are dealt with in an enumeration. + :class:`EnumDict` + + A subclass of :class:`dict` for use when subclassing :class:`EnumType`. + :class:`auto` Instances are replaced with an appropriate value for Enum members. @@ -152,6 +156,7 @@ Module Contents .. versionadded:: 3.6 ``Flag``, ``IntFlag``, ``auto`` .. versionadded:: 3.11 ``StrEnum``, ``EnumCheck``, ``ReprEnum``, ``FlagBoundary``, ``property``, ``member``, ``nonmember``, ``global_enum``, ``show_flag_values`` +.. versionadded:: 3.13 ``EnumDict`` --------------- @@ -279,6 +284,8 @@ Data Types >>> Color.RED.value 1 + Value of the member, can be set in :meth:`~Enum.__new__`. + .. note:: Enum member values Member values can be anything: :class:`int`, :class:`str`, etc. If @@ -286,13 +293,18 @@ Data Types appropriate value will be chosen for you. See :class:`auto` for the details. + While mutable/unhashable values, such as :class:`dict`, :class:`list` or + a mutable :class:`~dataclasses.dataclass`, can be used, they will have a + quadratic performance impact during creation relative to the + total number of mutable/unhashable values in the enum. + .. attribute:: Enum._name_ Name of the member. .. attribute:: Enum._value_ - Value of the member, can be set in :meth:`~object.__new__`. + Value of the member, can be set in :meth:`~Enum.__new__`. .. attribute:: Enum._order_ @@ -395,13 +407,15 @@ Data Types in the member assignment will be passed; e.g. >>> from enum import Enum - >>> class MyIntEnum(Enum): - ... SEVENTEEN = '1a', 16 + >>> class MyIntEnum(int, Enum): + ... TWENTYSIX = '1a', 16 - results in the call ``int('1a', 16)`` and a value of ``17`` for the member. + results in the call ``int('1a', 16)`` and a value of ``26`` for the member. - ..note:: When writing a custom ``__new__``, do not use ``super().__new__`` -- - call the appropriate ``__new__`` instead. + .. note:: + + When writing a custom ``__new__``, do not use ``super().__new__`` -- + call the appropriate ``__new__`` instead. .. method:: Enum.__repr__(self) @@ -518,7 +532,7 @@ Data Types ``Flag`` is the same as :class:`Enum`, but its members support the bitwise operators ``&`` (*AND*), ``|`` (*OR*), ``^`` (*XOR*), and ``~`` (*INVERT*); - the results of those operators are members of the enumeration. + the results of those operations are (aliases of) members of the enumeration. .. method:: __contains__(self, value) @@ -561,6 +575,8 @@ Data Types >>> len(white) 3 + .. versionadded:: 3.11 + .. method:: __bool__(self): Returns *True* if any members in flag, *False* otherwise:: @@ -620,7 +636,7 @@ Data Types of two, starting with ``1``. .. versionchanged:: 3.11 The *repr()* of zero-valued flags has changed. It - is now:: + is now: >>> Color(0) # doctest: +SKIP @@ -652,7 +668,7 @@ Data Types * the result is a valid *IntFlag*: an *IntFlag* is returned * the result is not a valid *IntFlag*: the result depends on the :class:`FlagBoundary` setting - The :func:`repr()` of unnamed zero-valued flags has changed. It is now: + The :func:`repr` of unnamed zero-valued flags has changed. It is now: >>> Color(0) @@ -810,7 +826,27 @@ Data Types >>> KeepFlag(2**2 + 2**4) -.. versionadded:: 3.11 + .. versionadded:: 3.11 + +.. class:: EnumDict + + *EnumDict* is a subclass of :class:`dict` that is used as the namespace + for defining enum classes (see :ref:`prepare`). + It is exposed to allow subclasses of :class:`EnumType` with advanced + behavior like having multiple values per member. + It should be called with the name of the enum class being created, otherwise + private names and internal classes will not be handled correctly. + + Note that only the :class:`~collections.abc.MutableMapping` interface + (:meth:`~object.__setitem__` and :meth:`~dict.update`) is overridden. + It may be possible to bypass the checks using other :class:`!dict` + operations like :meth:`|= `. + + .. attribute:: EnumDict.member_names + + A list of member names. + + .. versionadded:: 3.13 --------------- @@ -820,7 +856,7 @@ Supported ``__dunder__`` names :attr:`~EnumType.__members__` is a read-only ordered mapping of ``member_name``:``member`` items. It is only available on the class. -:meth:`~object.__new__`, if specified, must create and return the enum members; +:meth:`~Enum.__new__`, if specified, must create and return the enum members; it is also a very good idea to set the member's :attr:`!_value_` appropriately. Once all the members are created it is no longer used. @@ -852,9 +888,15 @@ Supported ``_sunder_`` names For :class:`Flag` classes the next value chosen will be the next highest power-of-two. +- While ``_sunder_`` names are generally reserved for the further development + of the :class:`Enum` class and can not be used, some are explicitly allowed: + + - ``_repr_*`` (e.g. ``_repr_html_``), as used in `IPython's rich display`_ + .. versionadded:: 3.6 ``_missing_``, ``_order_``, ``_generate_next_value_`` .. versionadded:: 3.7 ``_ignore_`` -.. versionadded:: 3.13 ``_add_alias_``, ``_add_value_alias_`` +.. versionadded:: 3.13 ``_add_alias_``, ``_add_value_alias_``, ``_repr_*`` +.. _`IPython's rich display`: https://ipython.readthedocs.io/en/stable/config/integrating.html#rich-display --------------- diff --git a/Doc/library/errno.rst b/Doc/library/errno.rst index 283e8b013265d9..48b9762d85c2e1 100644 --- a/Doc/library/errno.rst +++ b/Doc/library/errno.rst @@ -1,5 +1,5 @@ -:mod:`errno` --- Standard errno system symbols -============================================== +:mod:`!errno` --- Standard errno system symbols +=============================================== .. module:: errno :synopsis: Standard errno system symbols. @@ -613,6 +613,13 @@ defined by the module. The specific list of defined symbols is available as No route to host +.. data:: EHWPOISON + + Memory page has hardware error. + + .. versionadded:: 3.14 + + .. data:: EALREADY Operation already in progress. This error is mapped to the @@ -665,6 +672,171 @@ defined by the module. The specific list of defined symbols is available as .. versionadded:: 3.11 + +.. data:: ENOMEDIUM + + No medium found + + +.. data:: EMEDIUMTYPE + + Wrong medium type + + +.. data:: ENOKEY + + Required key not available + + +.. data:: EKEYEXPIRED + + Key has expired + + +.. data:: EKEYREVOKED + + Key has been revoked + + +.. data:: EKEYREJECTED + + Key was rejected by service + + +.. data:: ERFKILL + + Operation not possible due to RF-kill + + +.. data:: ELOCKUNMAPPED + + Locked lock was unmapped + + +.. data:: ENOTACTIVE + + Facility is not active + + +.. data:: EAUTH + + Authentication error + + .. versionadded:: 3.2 + + +.. data:: EBADARCH + + Bad CPU type in executable + + .. versionadded:: 3.2 + + +.. data:: EBADEXEC + + Bad executable (or shared library) + + .. versionadded:: 3.2 + + +.. data:: EBADMACHO + + Malformed Mach-o file + + .. versionadded:: 3.2 + + +.. data:: EDEVERR + + Device error + + .. versionadded:: 3.2 + + +.. data:: EFTYPE + + Inappropriate file type or format + + .. versionadded:: 3.2 + + +.. data:: ENEEDAUTH + + Need authenticator + + .. versionadded:: 3.2 + + +.. data:: ENOATTR + + Attribute not found + + .. versionadded:: 3.2 + + +.. data:: ENOPOLICY + + Policy not found + + .. versionadded:: 3.2 + + +.. data:: EPROCLIM + + Too many processes + + .. versionadded:: 3.2 + + +.. data:: EPROCUNAVAIL + + Bad procedure for program + + .. versionadded:: 3.2 + + +.. data:: EPROGMISMATCH + + Program version wrong + + .. versionadded:: 3.2 + + +.. data:: EPROGUNAVAIL + + RPC prog. not avail + + .. versionadded:: 3.2 + + +.. data:: EPWROFF + + Device power is off + + .. versionadded:: 3.2 + + +.. data:: EBADRPC + + RPC struct is bad + + .. versionadded:: 3.2 + + +.. data:: ERPCMISMATCH + + RPC version wrong + + .. versionadded:: 3.2 + + +.. data:: ESHLIBVERS + + Shared library version mismatch + + .. versionadded:: 3.2 + + .. data:: ENOTCAPABLE Capabilities insufficient. This error is mapped to the exception diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 7879fb015bddfa..f72b11e34c5c3d 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -412,8 +412,8 @@ The following exceptions are the exceptions that are usually raised. represented. This cannot occur for integers (which would rather raise :exc:`MemoryError` than give up). However, for historical reasons, OverflowError is sometimes raised for integers that are outside a required - range. Because of the lack of standardization of floating point exception - handling in C, most floating point operations are not checked. + range. Because of the lack of standardization of floating-point exception + handling in C, most floating-point operations are not checked. .. exception:: PythonFinalizationError @@ -644,10 +644,16 @@ The following exceptions are the exceptions that are usually raised. The first index of invalid data in :attr:`object`. + This value should not be negative as it is interpreted as an + absolute offset but this constraint is not enforced at runtime. + .. attribute:: end The index after the last invalid data in :attr:`object`. + This value should not be negative as it is interpreted as an + absolute offset but this constraint is not enforced at runtime. + .. exception:: UnicodeEncodeError @@ -989,7 +995,8 @@ their subgroups based on the types of the contained exceptions. Returns an exception group with the same :attr:`message`, but which wraps the exceptions in ``excs``. - This method is used by :meth:`subgroup` and :meth:`split`. A + This method is used by :meth:`subgroup` and :meth:`split`, which + are used in various contexts to break up an exception group. A subclass needs to override it in order to make :meth:`subgroup` and :meth:`split` return instances of the subclass rather than :exc:`ExceptionGroup`. diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index f64dfeb5e081c7..b7df9f6b9bcf96 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -1,5 +1,5 @@ -:mod:`faulthandler` --- Dump the Python traceback -================================================= +:mod:`!faulthandler` --- Dump the Python traceback +================================================== .. module:: faulthandler :synopsis: Dump the Python traceback. @@ -10,14 +10,15 @@ This module contains functions to dump Python tracebacks explicitly, on a fault, after a timeout, or on a user signal. Call :func:`faulthandler.enable` to -install fault handlers for the :const:`SIGSEGV`, :const:`SIGFPE`, -:const:`SIGABRT`, :const:`SIGBUS`, and :const:`SIGILL` signals. You can also +install fault handlers for the :const:`~signal.SIGSEGV`, +:const:`~signal.SIGFPE`, :const:`~signal.SIGABRT`, :const:`~signal.SIGBUS`, and +:const:`~signal.SIGILL` signals. You can also enable them at startup by setting the :envvar:`PYTHONFAULTHANDLER` environment variable or by using the :option:`-X` ``faulthandler`` command line option. The fault handler is compatible with system fault handlers like Apport or the Windows fault handler. The module uses an alternative stack for signal handlers -if the :c:func:`sigaltstack` function is available. This allows it to dump the +if the :c:func:`!sigaltstack` function is available. This allows it to dump the traceback even on a stack overflow. The fault handler is called on catastrophic cases and therefore can only use @@ -70,8 +71,9 @@ Fault handler state .. function:: enable(file=sys.stderr, all_threads=True) - Enable the fault handler: install handlers for the :const:`SIGSEGV`, - :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS` and :const:`SIGILL` + Enable the fault handler: install handlers for the :const:`~signal.SIGSEGV`, + :const:`~signal.SIGFPE`, :const:`~signal.SIGABRT`, :const:`~signal.SIGBUS` + and :const:`~signal.SIGILL` signals to dump the Python traceback. If *all_threads* is ``True``, produce tracebacks for every running thread. Otherwise, dump only the current thread. @@ -89,6 +91,10 @@ Fault handler state The dump now mentions if a garbage collector collection is running if *all_threads* is true. + .. versionchanged:: 3.14 + Only the current thread is dumped if the :term:`GIL` is disabled to + prevent the risk of data races. + .. function:: disable() Disable the fault handler: uninstall the signal handlers installed by @@ -106,8 +112,8 @@ Dumping the tracebacks after a timeout Dump the tracebacks of all threads, after a timeout of *timeout* seconds, or every *timeout* seconds if *repeat* is ``True``. If *exit* is ``True``, call - :c:func:`_exit` with status=1 after dumping the tracebacks. (Note - :c:func:`_exit` exits the process immediately, which means it doesn't do any + :c:func:`!_exit` with status=1 after dumping the tracebacks. (Note + :c:func:`!_exit` exits the process immediately, which means it doesn't do any cleanup like flushing file buffers.) If the function is called twice, the new call replaces previous parameters and resets the timeout. The timer has a sub-second resolution. @@ -118,12 +124,12 @@ Dumping the tracebacks after a timeout This function is implemented using a watchdog thread. - .. versionchanged:: 3.7 - This function is now always available. - .. versionchanged:: 3.5 Added support for passing file descriptor to this function. + .. versionchanged:: 3.7 + This function is now always available. + .. function:: cancel_dump_traceback_later() Cancel the last call to :func:`dump_traceback_later`. diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index b93d6ac7aab956..7bd64e43dd5bfe 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -1,5 +1,5 @@ -:mod:`fcntl` --- The ``fcntl`` and ``ioctl`` system calls -========================================================= +:mod:`!fcntl` --- The ``fcntl`` and ``ioctl`` system calls +========================================================== .. module:: fcntl :platform: Unix @@ -18,7 +18,7 @@ interface to the :c:func:`fcntl` and :c:func:`ioctl` Unix routines. See the :manpage:`fcntl(2)` and :manpage:`ioctl(2)` Unix manual pages for full details. -.. availability:: Unix, not Emscripten, not WASI. +.. availability:: Unix, not WASI. All functions in this module take a file descriptor *fd* as their first argument. This can be an integer file descriptor, such as returned by diff --git a/Doc/library/filecmp.rst b/Doc/library/filecmp.rst index 42d20b9c201783..282d0e0d8db5cf 100644 --- a/Doc/library/filecmp.rst +++ b/Doc/library/filecmp.rst @@ -1,5 +1,5 @@ -:mod:`filecmp` --- File and Directory Comparisons -================================================= +:mod:`!filecmp` --- File and Directory Comparisons +================================================== .. module:: filecmp :synopsis: Compare files efficiently. @@ -70,7 +70,7 @@ The :mod:`filecmp` module defines the following functions: The :class:`dircmp` class ------------------------- -.. class:: dircmp(a, b, ignore=None, hide=None, shallow=True) +.. class:: dircmp(a, b, ignore=None, hide=None, *, shallow=True) Construct a new directory comparison object, to compare the directories *a* and *b*. *ignore* is a list of names to ignore, and defaults to diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index f93e9a58791eeb..8f32b11e565365 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -1,5 +1,5 @@ -:mod:`fileinput` --- Iterate over lines from multiple input streams -=================================================================== +:mod:`!fileinput` --- Iterate over lines from multiple input streams +==================================================================== .. module:: fileinput :synopsis: Loop over standard input or a list of files. @@ -47,7 +47,7 @@ Lines are returned with any newlines intact, which means that the last line in a file may not have one. You can control how files are opened by providing an opening hook via the -*openhook* parameter to :func:`fileinput.input` or :class:`FileInput()`. The +*openhook* parameter to :func:`fileinput.input` or :func:`FileInput`. The hook must be a function that takes two arguments, *filename* and *mode*, and returns an accordingly opened file-like object. If *encoding* and/or *errors* are specified, they will be passed to the hook as additional keyword arguments. diff --git a/Doc/library/filesys.rst b/Doc/library/filesys.rst index 0ccf2b7bf59a0f..f1ea4761af7cb1 100644 --- a/Doc/library/filesys.rst +++ b/Doc/library/filesys.rst @@ -14,7 +14,6 @@ in this chapter is: pathlib.rst os.path.rst - fileinput.rst stat.rst filecmp.rst tempfile.rst diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index 7cddecd5e80887..5cb47777ae527d 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -1,5 +1,5 @@ -:mod:`fnmatch` --- Unix filename pattern matching -================================================= +:mod:`!fnmatch` --- Unix filename pattern matching +================================================== .. module:: fnmatch :synopsis: Unix shell style filename pattern matching. @@ -46,9 +46,15 @@ module. See module :mod:`glob` for pathname expansion (:mod:`glob` uses a period are not special for this module, and are matched by the ``*`` and ``?`` patterns. -Also note that :func:`functools.lru_cache` with the *maxsize* of 32768 is used to -cache the compiled regex patterns in the following functions: :func:`fnmatch`, -:func:`fnmatchcase`, :func:`.filter`. +Unless stated otherwise, "filename string" and "pattern string" either refer to +:class:`str` or ``ISO-8859-1`` encoded :class:`bytes` objects. Note that the +functions documented below do not allow to mix a :class:`!bytes` pattern with +a :class:`!str` filename, and vice-versa. + +Finally, note that :func:`functools.lru_cache` with a *maxsize* of 32768 +is used to cache the (typed) compiled regex patterns in the following +functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`. + .. function:: fnmatch(name, pat) @@ -78,8 +84,8 @@ cache the compiled regex patterns in the following functions: :func:`fnmatch`, .. function:: filter(names, pat) - Construct a list from those elements of the :term:`iterable` *names* - that match pattern *pat*. + Construct a list from those elements of the :term:`iterable` of filename + strings *names* that match the pattern string *pat*. It is the same as ``[n for n in names if fnmatch(n, pat)]``, but implemented more efficiently. @@ -87,7 +93,7 @@ cache the compiled regex patterns in the following functions: :func:`fnmatch`, .. function:: translate(pat) Return the shell-style pattern *pat* converted to a regular expression for - using with :func:`re.match`. + using with :func:`re.match`. The pattern is expected to be a :class:`str`. Example: diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index 887c3844d20faa..fc7f9a6301a915 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -1,5 +1,5 @@ -:mod:`fractions` --- Rational numbers -===================================== +:mod:`!fractions` --- Rational numbers +====================================== .. module:: fractions :synopsis: Rational numbers. @@ -17,25 +17,30 @@ The :mod:`fractions` module provides support for rational number arithmetic. A Fraction instance can be constructed from a pair of integers, from another rational number, or from a string. +.. index:: single: as_integer_ratio() + .. class:: Fraction(numerator=0, denominator=1) - Fraction(other_fraction) - Fraction(float) - Fraction(decimal) + Fraction(number) Fraction(string) The first version requires that *numerator* and *denominator* are instances of :class:`numbers.Rational` and returns a new :class:`Fraction` instance with value ``numerator/denominator``. If *denominator* is ``0``, it - raises a :exc:`ZeroDivisionError`. The second version requires that - *other_fraction* is an instance of :class:`numbers.Rational` and returns a - :class:`Fraction` instance with the same value. The next two versions accept - either a :class:`float` or a :class:`decimal.Decimal` instance, and return a - :class:`Fraction` instance with exactly the same value. Note that due to the - usual issues with binary floating-point (see :ref:`tut-fp-issues`), the + raises a :exc:`ZeroDivisionError`. + + The second version requires that *number* is an instance of + :class:`numbers.Rational` or has the :meth:`!as_integer_ratio` method + (this includes :class:`float` and :class:`decimal.Decimal`). + It returns a :class:`Fraction` instance with exactly the same value. + Assumed, that the :meth:`!as_integer_ratio` method returns a pair + of coprime integers and last one is positive. + Note that due to the + usual issues with binary point (see :ref:`tut-fp-issues`), the argument to ``Fraction(1.1)`` is not exactly equal to 11/10, and so ``Fraction(1.1)`` does *not* return ``Fraction(11, 10)`` as one might expect. (But see the documentation for the :meth:`limit_denominator` method below.) - The last version of the constructor expects a string or unicode instance. + + The last version of the constructor expects a string. The usual form for this instance is:: [sign] numerator ['/' denominator] @@ -87,7 +92,7 @@ another rational number, or from a string. .. versionchanged:: 3.9 The :func:`math.gcd` function is now used to normalize the *numerator* - and *denominator*. :func:`math.gcd` always return a :class:`int` type. + and *denominator*. :func:`math.gcd` always returns an :class:`int` type. Previously, the GCD type depended on *numerator* and *denominator*. .. versionchanged:: 3.11 @@ -110,6 +115,10 @@ another rational number, or from a string. Formatting of :class:`Fraction` instances without a presentation type now supports fill, alignment, sign handling, minimum width and grouping. + .. versionchanged:: 3.14 + The :class:`Fraction` constructor now accepts any objects with the + :meth:`!as_integer_ratio` method. + .. attribute:: numerator Numerator of the Fraction in lowest term. @@ -157,6 +166,16 @@ another rational number, or from a string. instance. + .. classmethod:: from_number(number) + + Alternative constructor which only accepts instances of + :class:`numbers.Integral`, :class:`numbers.Rational`, + :class:`float` or :class:`decimal.Decimal`, and objects with + the :meth:`!as_integer_ratio` method, but not strings. + + .. versionadded:: 3.14 + + .. method:: limit_denominator(max_denominator=1000000) Finds and returns the closest :class:`Fraction` to ``self`` that has diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index 8d1aae018ada12..bb15322067245e 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -1,5 +1,5 @@ -:mod:`ftplib` --- FTP protocol client -===================================== +:mod:`!ftplib` --- FTP protocol client +====================================== .. module:: ftplib :synopsis: FTP protocol client (requires sockets). @@ -243,7 +243,7 @@ FTP objects Retrieve a file in binary transfer mode. :param str cmd: - An appropriate ``STOR`` command: :samp:`"STOR {filename}"`. + An appropriate ``RETR`` command: :samp:`"RETR {filename}"`. :param callback: A single parameter callable that is called diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index e598ef423de497..a7549b9bce76e2 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -57,7 +57,7 @@ are always available. They are listed here in alphabetical order. .. function:: abs(x) Return the absolute value of a number. The argument may be an - integer, a floating point number, or an object implementing + integer, a floating-point number, or an object implementing :meth:`~object.__abs__`. If the argument is a complex number, its magnitude is returned. @@ -141,10 +141,11 @@ are always available. They are listed here in alphabetical order. See also :func:`format` for more information. -.. class:: bool(x=False) +.. class:: bool(object=False, /) - Return a Boolean value, i.e. one of ``True`` or ``False``. *x* is converted - using the standard :ref:`truth testing procedure `. If *x* is false + Return a Boolean value, i.e. one of ``True`` or ``False``. The argument + is converted using the standard :ref:`truth testing procedure `. + If the argument is false or omitted, this returns ``False``; otherwise, it returns ``True``. The :class:`bool` class is a subclass of :class:`int` (see :ref:`typesnumeric`). It cannot be subclassed further. Its only instances are ``False`` and @@ -153,14 +154,14 @@ are always available. They are listed here in alphabetical order. .. index:: pair: Boolean; type .. versionchanged:: 3.7 - *x* is now a positional-only parameter. + The parameter is now positional-only. .. function:: breakpoint(*args, **kws) This function drops you into the debugger at the call site. Specifically, it calls :func:`sys.breakpointhook`, passing ``args`` and ``kws`` straight through. By default, ``sys.breakpointhook()`` calls - :func:`pdb.set_trace()` expecting no arguments. In this case, it is + :func:`pdb.set_trace` expecting no arguments. In this case, it is purely a convenience function so you don't have to explicitly import :mod:`pdb` or type as much code to enter the debugger. However, :func:`sys.breakpointhook` can be set to some other function and @@ -282,9 +283,11 @@ are always available. They are listed here in alphabetical order. :func:`property`. .. versionchanged:: 3.10 - Class methods now inherit the method attributes (``__module__``, - ``__name__``, ``__qualname__``, ``__doc__`` and ``__annotations__``) and - have a new ``__wrapped__`` attribute. + Class methods now inherit the method attributes + (:attr:`~function.__module__`, :attr:`~function.__name__`, + :attr:`~function.__qualname__`, :attr:`~function.__doc__` and + :attr:`~function.__annotations__`) and have a new ``__wrapped__`` + attribute. .. deprecated-removed:: 3.11 3.13 Class methods can no longer wrap other :term:`descriptors ` such as @@ -371,29 +374,75 @@ are always available. They are listed here in alphabetical order. support for top-level ``await``, ``async for``, and ``async with``. -.. class:: complex(real=0, imag=0) - complex(string) +.. class:: complex(number=0, /) + complex(string, /) + complex(real=0, imag=0) + + Convert a single string or number to a complex number, or create a + complex number from real and imaginary parts. + + Examples: + + .. doctest:: + + >>> complex('+1.23') + (1.23+0j) + >>> complex('-4.5j') + -4.5j + >>> complex('-1.23+4.5j') + (-1.23+4.5j) + >>> complex('\t( -1.23+4.5J )\n') + (-1.23+4.5j) + >>> complex('-Infinity+NaNj') + (-inf+nanj) + >>> complex(1.23) + (1.23+0j) + >>> complex(imag=-4.5) + -4.5j + >>> complex(-1.23, 4.5) + (-1.23+4.5j) + + If the argument is a string, it must contain either a real part (in the + same format as for :func:`float`) or an imaginary part (in the same + format but with a ``'j'`` or ``'J'`` suffix), or both real and imaginary + parts (the sign of the imaginary part is mandatory in this case). + The string can optionally be surrounded by whitespaces and the round + parentheses ``'('`` and ``')'``, which are ignored. + The string must not contain whitespace between ``'+'``, ``'-'``, the + ``'j'`` or ``'J'`` suffix, and the decimal number. + For example, ``complex('1+2j')`` is fine, but ``complex('1 + 2j')`` raises + :exc:`ValueError`. + More precisely, the input must conform to the :token:`~float:complexvalue` + production rule in the following grammar, after parentheses and leading and + trailing whitespace characters are removed: - Return a complex number with the value *real* + *imag*\*1j or convert a string - or number to a complex number. If the first parameter is a string, it will - be interpreted as a complex number and the function must be called without a - second parameter. The second parameter can never be a string. Each argument - may be any numeric type (including complex). If *imag* is omitted, it - defaults to zero and the constructor serves as a numeric conversion like - :class:`int` and :class:`float`. If both arguments are omitted, returns - ``0j``. + .. productionlist:: float + complexvalue: `floatvalue` | + : `floatvalue` ("j" | "J") | + : `floatvalue` `sign` `absfloatvalue` ("j" | "J") + If the argument is a number, the constructor serves as a numeric + conversion like :class:`int` and :class:`float`. For a general Python object ``x``, ``complex(x)`` delegates to - ``x.__complex__()``. If :meth:`~object.__complex__` is not defined then it falls back - to :meth:`~object.__float__`. If :meth:`!__float__` is not defined then it falls back + ``x.__complex__()``. + If :meth:`~object.__complex__` is not defined then it falls back + to :meth:`~object.__float__`. + If :meth:`!__float__` is not defined then it falls back to :meth:`~object.__index__`. - .. note:: + If two arguments are provided or keyword arguments are used, each argument + may be any numeric type (including complex). + If both arguments are real numbers, return a complex number with the real + component *real* and the imaginary component *imag*. + If both arguments are complex numbers, return a complex number with the real + component ``real.real-imag.imag`` and the imaginary component + ``real.imag+imag.real``. + If one of arguments is a real number, only its real component is used in + the above expressions. + + See also :meth:`complex.from_number` which only accepts a single numeric argument. - When converting from a string, the string must not contain whitespace - around the central ``+`` or ``-`` operator. For example, - ``complex('1+2j')`` is fine, but ``complex('1 + 2j')`` raises - :exc:`ValueError`. + If all arguments are omitted, returns ``0j``. The complex type is described in :ref:`typesnumeric`. @@ -404,6 +453,10 @@ are always available. They are listed here in alphabetical order. Falls back to :meth:`~object.__index__` if :meth:`~object.__complex__` and :meth:`~object.__float__` are not defined. + .. deprecated:: 3.14 + Passing a complex number as the *real* or *imag* argument is now + deprecated; it should only be passed as a single positional argument. + .. function:: delattr(object, name) @@ -493,7 +546,7 @@ are always available. They are listed here in alphabetical order. Take two (non-complex) numbers as arguments and return a pair of numbers consisting of their quotient and remainder when using integer division. With mixed operand types, the rules for binary arithmetic operators apply. For - integers, the result is the same as ``(a // b, a % b)``. For floating point + integers, the result is the same as ``(a // b, a % b)``. For floating-point numbers the result is ``(q, a % b)``, where *q* is usually ``math.floor(a / b)`` but may be 1 less than that. In any case ``q * b + a % b`` is very close to *a*, if ``a % b`` is non-zero it has the same sign as *b*, and ``0 @@ -524,11 +577,11 @@ are always available. They are listed here in alphabetical order. .. _func-eval: -.. function:: eval(expression, globals=None, locals=None) +.. function:: eval(source, /, globals=None, locals=None) - :param expression: + :param source: A Python expression. - :type expression: :class:`str` | :ref:`code object ` + :type source: :class:`str` | :ref:`code object ` :param globals: The global namespace (default: ``None``). @@ -541,20 +594,26 @@ are always available. They are listed here in alphabetical order. :returns: The result of the evaluated expression. :raises: Syntax errors are reported as exceptions. + .. warning:: + + This function executes arbitrary code. Calling it with + user-supplied input may lead to security vulnerabilities. + The *expression* argument is parsed and evaluated as a Python expression (technically speaking, a condition list) using the *globals* and *locals* - dictionaries as global and local namespace. If the *globals* dictionary is + mappings as global and local namespace. If the *globals* dictionary is present and does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module :mod:`builtins` is inserted under that key before *expression* is parsed. That way you can control what builtins are available to the executed code by inserting your own ``__builtins__`` dictionary into *globals* before passing it to - :func:`eval`. If the *locals* dictionary is omitted it defaults to the - *globals* dictionary. If both dictionaries are omitted, the expression is + :func:`eval`. If the *locals* mapping is omitted it defaults to the + *globals* dictionary. If both mappings are omitted, the expression is executed with the *globals* and *locals* in the environment where - :func:`eval` is called. Note, *eval()* does not have access to the + :func:`eval` is called. Note, *eval()* will only have access to the :term:`nested scopes ` (non-locals) in the enclosing - environment. + environment if they are already referenced in the scope that is calling + :func:`eval` (e.g. via a :keyword:`nonlocal` statement). Example: @@ -583,11 +642,25 @@ are always available. They are listed here in alphabetical order. Raises an :ref:`auditing event ` ``exec`` with the code object as the argument. Code compilation events may also be raised. + .. versionchanged:: 3.13 + + The *globals* and *locals* arguments can now be passed as keywords. + + .. versionchanged:: 3.13 + + The semantics of the default *locals* namespace have been adjusted as + described for the :func:`locals` builtin. + .. index:: pair: built-in function; exec -.. function:: exec(object, globals=None, locals=None, /, *, closure=None) +.. function:: exec(source, /, globals=None, locals=None, *, closure=None) + + .. warning:: + + This function executes arbitrary code. Calling it with + user-supplied input may lead to security vulnerabilities. - This function supports dynamic execution of Python code. *object* must be + This function supports dynamic execution of Python code. *source* must be either a string or a code object. If it is a string, the string is parsed as a suite of Python statements which is then executed (unless a syntax error occurs). [#]_ If it is a code object, it is simply executed. In all cases, @@ -604,9 +677,15 @@ are always available. They are listed here in alphabetical order. will be used for both the global and the local variables. If *globals* and *locals* are given, they are used for the global and local variables, respectively. If provided, *locals* can be any mapping object. Remember - that at the module level, globals and locals are the same dictionary. If exec - gets two separate objects as *globals* and *locals*, the code will be - executed as if it were embedded in a class definition. + that at the module level, globals and locals are the same dictionary. + + .. note:: + + When ``exec`` gets two separate objects as *globals* and *locals*, the + code will be executed as if it were embedded in a class definition. This + means functions and classes defined in the executed code will not be able + to access variables assigned at the top level (as the "top level" + variables are treated as class variables in a class definition). If the *globals* dictionary does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module @@ -615,9 +694,10 @@ are always available. They are listed here in alphabetical order. ``__builtins__`` dictionary into *globals* before passing it to :func:`exec`. The *closure* argument specifies a closure--a tuple of cellvars. - It's only valid when the *object* is a code object containing free variables. - The length of the tuple must exactly match the number of free variables - referenced by the code object. + It's only valid when the *object* is a code object containing + :term:`free (closure) variables `. + The length of the tuple must exactly match the length of the code object's + :attr:`~codeobject.co_freevars` attribute. .. audit-event:: exec code_object exec @@ -627,19 +707,27 @@ are always available. They are listed here in alphabetical order. .. note:: The built-in functions :func:`globals` and :func:`locals` return the current - global and local dictionary, respectively, which may be useful to pass around + global and local namespace, respectively, which may be useful to pass around for use as the second and third argument to :func:`exec`. .. note:: - The default *locals* act as described for function :func:`locals` below: - modifications to the default *locals* dictionary should not be attempted. + The default *locals* act as described for function :func:`locals` below. Pass an explicit *locals* dictionary if you need to see effects of the code on *locals* after function :func:`exec` returns. .. versionchanged:: 3.11 Added the *closure* parameter. + .. versionchanged:: 3.13 + + The *globals* and *locals* arguments can now be passed as keywords. + + .. versionchanged:: 3.13 + + The semantics of the default *locals* namespace have been adjusted as + described for the :func:`locals` builtin. + .. function:: filter(function, iterable) @@ -658,21 +746,38 @@ are always available. They are listed here in alphabetical order. elements of *iterable* for which *function* is false. -.. class:: float(x=0.0) +.. class:: float(number=0.0, /) + float(string, /) .. index:: single: NaN single: Infinity - Return a floating point number constructed from a number or string *x*. + Return a floating-point number constructed from a number or a string. + + Examples: + + .. doctest:: + + >>> float('+1.23') + 1.23 + >>> float(' -12345\n') + -12345.0 + >>> float('1e-003') + 0.001 + >>> float('+1E6') + 1000000.0 + >>> float('-Infinity') + -inf If the argument is a string, it should contain a decimal number, optionally preceded by a sign, and optionally embedded in whitespace. The optional sign may be ``'+'`` or ``'-'``; a ``'+'`` sign has no effect on the value produced. The argument may also be a string representing a NaN - (not-a-number), or positive or negative infinity. More precisely, the - input must conform to the ``floatvalue`` production rule in the following - grammar, after leading and trailing whitespace characters are removed: + (not-a-number), or positive or negative infinity. + More precisely, the input must conform to the :token:`~float:floatvalue` + production rule in the following grammar, after leading and trailing + whitespace characters are removed: .. productionlist:: float sign: "+" | "-" @@ -681,15 +786,16 @@ are always available. They are listed here in alphabetical order. digit: digitpart: `digit` (["_"] `digit`)* number: [`digitpart`] "." `digitpart` | `digitpart` ["."] - exponent: ("e" | "E") ["+" | "-"] `digitpart` - floatnumber: number [`exponent`] - floatvalue: [`sign`] (`floatnumber` | `infinity` | `nan`) + exponent: ("e" | "E") [`sign`] `digitpart` + floatnumber: `number` [`exponent`] + absfloatvalue: `floatnumber` | `infinity` | `nan` + floatvalue: [`sign`] `absfloatvalue` Case is not significant, so, for example, "inf", "Inf", "INFINITY", and "iNfINity" are all acceptable spellings for positive infinity. - Otherwise, if the argument is an integer or a floating point number, a - floating point number with the same value (within Python's floating point + Otherwise, if the argument is an integer or a floating-point number, a + floating-point number with the same value (within Python's floating-point precision) is returned. If the argument is outside the range of a Python float, an :exc:`OverflowError` will be raised. @@ -697,20 +803,9 @@ are always available. They are listed here in alphabetical order. ``x.__float__()``. If :meth:`~object.__float__` is not defined then it falls back to :meth:`~object.__index__`. - If no argument is given, ``0.0`` is returned. + See also :meth:`float.from_number` which only accepts a numeric argument. - Examples:: - - >>> float('+1.23') - 1.23 - >>> float(' -12345\n') - -12345.0 - >>> float('1e-003') - 0.001 - >>> float('+1E6') - 1000000.0 - >>> float('-Infinity') - -inf + If no argument is given, ``0.0`` is returned. The float type is described in :ref:`typesnumeric`. @@ -718,7 +813,7 @@ are always available. They are listed here in alphabetical order. Grouping digits with underscores as in code literals is allowed. .. versionchanged:: 3.7 - *x* is now a positional-only parameter. + The parameter is now positional-only. .. versionchanged:: 3.8 Falls back to :meth:`~object.__index__` if :meth:`~object.__float__` is not defined. @@ -902,17 +997,35 @@ are always available. They are listed here in alphabetical order. with the result after successfully reading input. -.. class:: int(x=0) - int(x, base=10) +.. class:: int(number=0, /) + int(string, /, base=10) + + Return an integer object constructed from a number or a string, or return + ``0`` if no arguments are given. - Return an integer object constructed from a number or string *x*, or return - ``0`` if no arguments are given. If *x* defines :meth:`~object.__int__`, - ``int(x)`` returns ``x.__int__()``. If *x* defines :meth:`~object.__index__`, - it returns ``x.__index__()``. If *x* defines :meth:`~object.__trunc__`, - it returns ``x.__trunc__()``. - For floating point numbers, this truncates towards zero. + Examples: - If *x* is not a number or if *base* is given, then *x* must be a string, + .. doctest:: + + >>> int(123.45) + 123 + >>> int('123') + 123 + >>> int(' -12_345\n') + -12345 + >>> int('FACE', 16) + 64206 + >>> int('0xface', 0) + 64206 + >>> int('01110011', base=2) + 115 + + If the argument defines :meth:`~object.__int__`, + ``int(x)`` returns ``x.__int__()``. If the argument defines + :meth:`~object.__index__`, it returns ``x.__index__()``. + For floating-point numbers, this truncates towards zero. + + If the argument is not a number or if *base* is given, then it must be a string, :class:`bytes`, or :class:`bytearray` instance representing an integer in radix *base*. Optionally, the string can be preceded by ``+`` or ``-`` (with no space in between), have leading zeros, be surrounded by whitespace, @@ -942,22 +1055,22 @@ are always available. They are listed here in alphabetical order. Grouping digits with underscores as in code literals is allowed. .. versionchanged:: 3.7 - *x* is now a positional-only parameter. + The first parameter is now positional-only. .. versionchanged:: 3.8 Falls back to :meth:`~object.__index__` if :meth:`~object.__int__` is not defined. - .. versionchanged:: 3.11 - The delegation to :meth:`~object.__trunc__` is deprecated. - .. versionchanged:: 3.11 :class:`int` string inputs and string representations can be limited to help avoid denial of service attacks. A :exc:`ValueError` is raised when - the limit is exceeded while converting a string *x* to an :class:`int` or + the limit is exceeded while converting a string to an :class:`int` or when converting an :class:`int` into a string would exceed the limit. See the :ref:`integer string conversion length limitation ` documentation. + .. versionchanged:: 3.14 + :func:`int` no longer delegates to the :meth:`~object.__trunc__` method. + .. function:: isinstance(object, classinfo) Return ``True`` if the *object* argument is an instance of the *classinfo* @@ -1041,23 +1154,70 @@ are always available. They are listed here in alphabetical order. .. function:: locals() - Update and return a dictionary representing the current local symbol table. - Free variables are returned by :func:`locals` when it is called in function - blocks, but not in class blocks. Note that at the module level, :func:`locals` - and :func:`globals` are the same dictionary. + Return a mapping object representing the current local symbol table, with + variable names as the keys, and their currently bound references as the + values. + + At module scope, as well as when using :func:`exec` or :func:`eval` with + a single namespace, this function returns the same namespace as + :func:`globals`. + + At class scope, it returns the namespace that will be passed to the + metaclass constructor. + + When using ``exec()`` or ``eval()`` with separate local and global + arguments, it returns the local namespace passed in to the function call. + + In all of the above cases, each call to ``locals()`` in a given frame of + execution will return the *same* mapping object. Changes made through + the mapping object returned from ``locals()`` will be visible as assigned, + reassigned, or deleted local variables, and assigning, reassigning, or + deleting local variables will immediately affect the contents of the + returned mapping object. + + In an :term:`optimized scope` (including functions, generators, and + coroutines), each call to ``locals()`` instead returns a fresh dictionary + containing the current bindings of the function's local variables and any + nonlocal cell references. In this case, name binding changes made via the + returned dict are *not* written back to the corresponding local variables + or nonlocal cell references, and assigning, reassigning, or deleting local + variables and nonlocal cell references does *not* affect the contents + of previously returned dictionaries. + + Calling ``locals()`` as part of a comprehension in a function, generator, or + coroutine is equivalent to calling it in the containing scope, except that + the comprehension's initialised iteration variables will be included. In + other scopes, it behaves as if the comprehension were running as a nested + function. + + Calling ``locals()`` as part of a generator expression is equivalent to + calling it in a nested generator function. - .. note:: - The contents of this dictionary should not be modified; changes may not - affect the values of local and free variables used by the interpreter. + .. versionchanged:: 3.12 + The behaviour of ``locals()`` in a comprehension has been updated as + described in :pep:`709`. + + .. versionchanged:: 3.13 + As part of :pep:`667`, the semantics of mutating the mapping objects + returned from this function are now defined. The behavior in + :term:`optimized scopes ` is now as described above. + Aside from being defined, the behaviour in other scopes remains + unchanged from previous versions. -.. function:: map(function, iterable, *iterables) + +.. function:: map(function, iterable, /, *iterables, strict=False) Return an iterator that applies *function* to every item of *iterable*, yielding the results. If additional *iterables* arguments are passed, *function* must take that many arguments and is applied to the items from all iterables in parallel. With multiple iterables, the iterator stops when the - shortest iterable is exhausted. For cases where the function inputs are - already arranged into argument tuples, see :func:`itertools.starmap`\. + shortest iterable is exhausted. If *strict* is ``True`` and one of the + iterables is exhausted before the others, a :exc:`ValueError` is raised. For + cases where the function inputs are already arranged into argument tuples, + see :func:`itertools.starmap`. + + .. versionchanged:: 3.14 + Added the *strict* parameter. .. function:: max(iterable, *, key=None) @@ -1138,14 +1298,16 @@ are always available. They are listed here in alphabetical order. .. class:: object() - Return a new featureless object. :class:`object` is a base for all classes. - It has methods that are common to all instances of Python classes. This - function does not accept any arguments. + This is the ultimate base class of all other classes. It has methods + that are common to all instances of Python classes. When the constructor + is called, it returns a new featureless object. The constructor does not + accept any arguments. .. note:: - :class:`object` does *not* have a :attr:`~object.__dict__`, so you can't - assign arbitrary attributes to an instance of the :class:`object` class. + :class:`object` instances do *not* have :attr:`~object.__dict__` + attributes, so you can't assign arbitrary attributes to an instance of + :class:`object`. .. function:: oct(x) @@ -1194,7 +1356,7 @@ are always available. They are listed here in alphabetical order. (which on *some* Unix systems, means that *all* writes append to the end of the file regardless of the current seek position). In text mode, if *encoding* is not specified the encoding used is platform-dependent: - :func:`locale.getencoding()` is called to get the current locale encoding. + :func:`locale.getencoding` is called to get the current locale encoding. (For reading and writing raw bytes use binary mode and leave *encoding* unspecified.) The available modes are: @@ -1367,7 +1529,7 @@ are always available. They are listed here in alphabetical order. (where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`, and :mod:`shutil`. - .. audit-event:: open file,mode,flags open + .. audit-event:: open path,mode,flags open The ``mode`` and ``flags`` arguments may have been modified or inferred from the original call. @@ -1423,7 +1585,9 @@ are always available. They are listed here in alphabetical order. returns ``100``, but ``pow(10, -2)`` returns ``0.01``. For a negative base of type :class:`int` or :class:`float` and a non-integral exponent, a complex result is delivered. For example, ``pow(-9, 0.5)`` returns a value close - to ``3j``. + to ``3j``. Whereas, for a negative base of type :class:`int` or :class:`float` + with an integral exponent, a float result is delivered. For example, + ``pow(-9, 2.0)`` returns ``81.0``. For :class:`int` operands *base* and *exp*, if *mod* is present, *mod* must also be of integer type and *mod* must be nonzero. If *mod* is present and @@ -1556,6 +1720,13 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.5 The docstrings of property objects are now writeable. + .. attribute:: __name__ + + Attribute holding the name of the property. The name of the property + can be changed at runtime. + + .. versionadded:: 3.13 + .. _func-range: .. class:: range(stop) @@ -1733,8 +1904,9 @@ are always available. They are listed here in alphabetical order. :ref:`function` for details. A static method can be called either on the class (such as ``C.f()``) or on - an instance (such as ``C().f()``). Moreover, they can be called as regular - functions (such as ``f()``). + an instance (such as ``C().f()``). + Moreover, the static method :term:`descriptor` is also callable, so it can + be used in the class definition (such as ``f()``). Static methods in Python are similar to those found in Java or C++. Also, see :func:`classmethod` for a variant that is useful for creating alternate class @@ -1755,10 +1927,11 @@ are always available. They are listed here in alphabetical order. For more information on static methods, see :ref:`types`. .. versionchanged:: 3.10 - Static methods now inherit the method attributes (``__module__``, - ``__name__``, ``__qualname__``, ``__doc__`` and ``__annotations__``), - have a new ``__wrapped__`` attribute, and are now callable as regular - functions. + Static methods now inherit the method attributes + (:attr:`~function.__module__`, :attr:`~function.__name__`, + :attr:`~function.__qualname__`, :attr:`~function.__doc__` and + :attr:`~function.__annotations__`), have a new ``__wrapped__`` attribute, + and are now callable as regular functions. .. index:: @@ -1783,7 +1956,7 @@ are always available. They are listed here in alphabetical order. For some use cases, there are good alternatives to :func:`sum`. The preferred, fast way to concatenate a sequence of strings is by calling - ``''.join(sequence)``. To add floating point values with extended precision, + ``''.join(sequence)``. To add floating-point values with extended precision, see :func:`math.fsum`\. To concatenate a series of iterables, consider using :func:`itertools.chain`. @@ -1793,6 +1966,10 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.12 Summation of floats switched to an algorithm that gives higher accuracy and better commutativity on most builds. + .. versionchanged:: 3.14 + Added specialization for summation of complexes, + using same algorithm as for summation of floats. + .. class:: super() super(type, object_or_type=None) @@ -1805,14 +1982,14 @@ are always available. They are listed here in alphabetical order. to be searched. The search starts from the class right after the *type*. - For example, if :attr:`~class.__mro__` of *object_or_type* is + For example, if :attr:`~type.__mro__` of *object_or_type* is ``D -> B -> C -> A -> object`` and the value of *type* is ``B``, then :func:`super` searches ``C -> A -> object``. - The :attr:`~class.__mro__` attribute of the *object_or_type* lists the method - resolution search order used by both :func:`getattr` and :func:`super`. The - attribute is dynamic and can change whenever the inheritance hierarchy is - updated. + The :attr:`~type.__mro__` attribute of the class corresponding to + *object_or_type* lists the method resolution search order used by both + :func:`getattr` and :func:`super`. The attribute is dynamic and can change + whenever the inheritance hierarchy is updated. If the second argument is omitted, the super object returned is unbound. If the second argument is an object, ``isinstance(obj, type)`` must be true. If @@ -1871,6 +2048,10 @@ are always available. They are listed here in alphabetical order. :func:`super`, see `guide to using super() `_. + .. versionchanged:: 3.14 + :class:`super` objects are now :mod:`pickleable ` and + :mod:`copyable `. + .. _func-tuple: .. class:: tuple() @@ -1888,28 +2069,30 @@ are always available. They are listed here in alphabetical order. With one argument, return the type of an *object*. The return value is a type object and generally the same object as returned by - :attr:`object.__class__ `. + :attr:`object.__class__`. The :func:`isinstance` built-in function is recommended for testing the type of an object, because it takes subclasses into account. - With three arguments, return a new type object. This is essentially a dynamic form of the :keyword:`class` statement. The *name* string is - the class name and becomes the :attr:`~definition.__name__` attribute. + the class name and becomes the :attr:`~type.__name__` attribute. The *bases* tuple contains the base classes and becomes the - :attr:`~class.__bases__` attribute; if empty, :class:`object`, the + :attr:`~type.__bases__` attribute; if empty, :class:`object`, the ultimate base of all classes, is added. The *dict* dictionary contains attribute and method definitions for the class body; it may be copied - or wrapped before becoming the :attr:`~object.__dict__` attribute. - The following two statements create identical :class:`type` objects: + or wrapped before becoming the :attr:`~type.__dict__` attribute. + The following two statements create identical :class:`!type` objects: >>> class X: ... a = 1 ... >>> X = type('X', (), dict(a=1)) - See also :ref:`bltin-type-objects`. + See also: + + * :ref:`Documentation on attributes and methods on classes `. + * :ref:`bltin-type-objects` Keyword arguments provided to the three argument form are passed to the appropriate metaclass machinery (usually :meth:`~object.__init_subclass__`) @@ -1919,28 +2102,32 @@ are always available. They are listed here in alphabetical order. See also :ref:`class-customization`. .. versionchanged:: 3.6 - Subclasses of :class:`type` which don't override ``type.__new__`` may no + Subclasses of :class:`!type` which don't override ``type.__new__`` may no longer use the one-argument form to get the type of an object. .. function:: vars() vars(object) Return the :attr:`~object.__dict__` attribute for a module, class, instance, - or any other object with a :attr:`~object.__dict__` attribute. + or any other object with a :attr:`!__dict__` attribute. Objects such as modules and instances have an updateable :attr:`~object.__dict__` attribute; however, other objects may have write restrictions on their - :attr:`~object.__dict__` attributes (for example, classes use a + :attr:`!__dict__` attributes (for example, classes use a :class:`types.MappingProxyType` to prevent direct dictionary updates). - Without an argument, :func:`vars` acts like :func:`locals`. Note, the - locals dictionary is only useful for reads since updates to the locals - dictionary are ignored. + Without an argument, :func:`vars` acts like :func:`locals`. A :exc:`TypeError` exception is raised if an object is specified but it doesn't have a :attr:`~object.__dict__` attribute (for example, if its class defines the :attr:`~object.__slots__` attribute). + .. versionchanged:: 3.13 + + The result of calling this function without an argument has been + updated as described for the :func:`locals` builtin. + + .. function:: zip(*iterables, strict=False) Iterate over several iterables in parallel, producing tuples with an item diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 20fcbe76c36985..8ad5f48c9e5286 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -1,5 +1,5 @@ -:mod:`functools` --- Higher-order functions and operations on callable objects -============================================================================== +:mod:`!functools` --- Higher-order functions and operations on callable objects +=============================================================================== .. module:: functools :synopsis: Higher-order functions and operations on callable objects. @@ -34,7 +34,7 @@ The :mod:`functools` module defines the following functions: Returns the same as ``lru_cache(maxsize=None)``, creating a thin wrapper around a dictionary lookup for the function arguments. Because it never needs to evict old values, this is smaller and faster than - :func:`lru_cache()` with a size limit. + :func:`lru_cache` with a size limit. For example:: @@ -218,7 +218,7 @@ The :mod:`functools` module defines the following functions: cache. See :ref:`faq-cache-method-calls` An `LRU (least recently used) cache - `_ + `_ works best when the most recent calls are the best predictors of upcoming calls (for example, the most popular articles on a news server tend to change each day). The cache's size limit assures that the cache does not @@ -325,9 +325,17 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.2 .. versionchanged:: 3.4 - Returning NotImplemented from the underlying comparison function for + Returning ``NotImplemented`` from the underlying comparison function for unrecognised types is now supported. +.. data:: Placeholder + + A singleton object used as a sentinel to reserve a place + for positional arguments when calling :func:`partial` + and :func:`partialmethod`. + + .. versionadded:: 3.14 + .. function:: partial(func, /, *args, **keywords) Return a new :ref:`partial object` which when called @@ -338,26 +346,68 @@ The :mod:`functools` module defines the following functions: Roughly equivalent to:: def partial(func, /, *args, **keywords): - def newfunc(*fargs, **fkeywords): - newkeywords = {**keywords, **fkeywords} - return func(*args, *fargs, **newkeywords) + def newfunc(*more_args, **more_keywords): + return func(*args, *more_args, **(keywords | more_keywords)) newfunc.func = func newfunc.args = args newfunc.keywords = keywords return newfunc - The :func:`partial` is used for partial function application which "freezes" + The :func:`!partial` function is used for partial function application which "freezes" some portion of a function's arguments and/or keywords resulting in a new object with a simplified signature. For example, :func:`partial` can be used to create a callable that behaves like the :func:`int` function where the *base* argument - defaults to two: + defaults to ``2``: + + .. doctest:: - >>> from functools import partial >>> basetwo = partial(int, base=2) >>> basetwo.__doc__ = 'Convert base 2 string to an int.' >>> basetwo('10010') 18 + If :data:`Placeholder` sentinels are present in *args*, they will be filled first + when :func:`!partial` is called. This makes it possible to pre-fill any positional + argument with a call to :func:`!partial`; without :data:`!Placeholder`, + only the chosen number of leading positional arguments can be pre-filled. + + If any :data:`!Placeholder` sentinels are present, all must be filled at call time: + + .. doctest:: + + >>> say_to_world = partial(print, Placeholder, Placeholder, "world!") + >>> say_to_world('Hello', 'dear') + Hello dear world! + + Calling ``say_to_world('Hello')`` raises a :exc:`TypeError`, because + only one positional argument is provided, but there are two placeholders + that must be filled in. + + If :func:`!partial` is applied to an existing :func:`!partial` object, + :data:`!Placeholder` sentinels of the input object are filled in with + new positional arguments. + A placeholder can be retained by inserting a new + :data:`!Placeholder` sentinel to the place held by a previous :data:`!Placeholder`: + + .. doctest:: + + >>> from functools import partial, Placeholder as _ + >>> remove = partial(str.replace, _, _, '') + >>> message = 'Hello, dear dear world!' + >>> remove(message, ' dear') + 'Hello, world!' + >>> remove_dear = partial(remove, _, ' dear') + >>> remove_dear(message) + 'Hello, world!' + >>> remove_first_dear = partial(remove_dear, _, 1) + >>> remove_first_dear(message) + 'Hello, dear world!' + + :data:`!Placeholder` has no special treatment when used in a keyword + argument to :func:`!partial`. + + .. versionchanged:: 3.14 + Added support for :data:`Placeholder` in positional arguments. .. class:: partialmethod(func, /, *args, **keywords) @@ -403,7 +453,7 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.4 -.. function:: reduce(function, iterable[, initial], /) +.. function:: reduce(function, iterable, /[, initial]) Apply *function* of two arguments cumulatively to the items of *iterable*, from left to right, so as to reduce the iterable to a single value. For example, @@ -418,7 +468,7 @@ The :mod:`functools` module defines the following functions: initial_missing = object() - def reduce(function, iterable, initial=initial_missing, /): + def reduce(function, iterable, /, initial=initial_missing): it = iter(iterable) if initial is initial_missing: value = next(it) @@ -431,6 +481,9 @@ The :mod:`functools` module defines the following functions: See :func:`itertools.accumulate` for an iterator that yields all intermediate values. + .. versionchanged:: 3.14 + *initial* is now supported as a keyword argument. + .. decorator:: singledispatch Transform a function into a :term:`single-dispatch >> @fun.register(list) + ... def _(arg: list[int], verbose=False): + ... if verbose: + ... print("Enumerate this:") + ... for i, elem in enumerate(arg): + ... print(i, elem) + + .. note:: + + At runtime the function will dispatch on an instance of a list regardless + of the type contained within the list i.e. ``[1,2,3]`` will be + dispatched the same as ``["foo", "bar", "baz"]``. The annotation + provided in this example is for static type checkers only and has no + runtime impact. To enable registering :term:`lambdas` and pre-existing functions, the :func:`register` attribute can also be used in a functional form:: @@ -646,9 +718,11 @@ The :mod:`functools` module defines the following functions: attributes of the wrapper function are updated with the corresponding attributes from the original function. The default values for these arguments are the module level constants ``WRAPPER_ASSIGNMENTS`` (which assigns to the wrapper - function's ``__module__``, ``__name__``, ``__qualname__``, ``__annotations__`` - and ``__doc__``, the documentation string) and ``WRAPPER_UPDATES`` (which - updates the wrapper function's ``__dict__``, i.e. the instance dictionary). + function's :attr:`~function.__module__`, :attr:`~function.__name__`, + :attr:`~function.__qualname__`, :attr:`~function.__annotations__`, + :attr:`~function.__type_params__`, and :attr:`~function.__doc__`, the + documentation string) and ``WRAPPER_UPDATES`` (which updates the wrapper + function's :attr:`~function.__dict__`, i.e. the instance dictionary). To allow access to the original function for introspection and other purposes (e.g. bypassing a caching decorator such as :func:`lru_cache`), this function @@ -667,13 +741,9 @@ The :mod:`functools` module defines the following functions: on the wrapper function). :exc:`AttributeError` is still raised if the wrapper function itself is missing any attributes named in *updated*. - .. versionadded:: 3.2 - Automatic addition of the ``__wrapped__`` attribute. - - .. versionadded:: 3.2 - Copying of the ``__annotations__`` attribute by default. - .. versionchanged:: 3.2 + The ``__wrapped__`` attribute is now automatically added. + The :attr:`~function.__annotations__` attribute is now copied by default. Missing attributes no longer trigger an :exc:`AttributeError`. .. versionchanged:: 3.4 @@ -681,6 +751,9 @@ The :mod:`functools` module defines the following functions: function, even if that function defined a ``__wrapped__`` attribute. (see :issue:`17482`) + .. versionchanged:: 3.12 + The :attr:`~function.__type_params__` attribute is now copied by default. + .. decorator:: wraps(wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) @@ -741,9 +814,7 @@ have three read-only attributes: The keyword arguments that will be supplied when the :class:`partial` object is called. -:class:`partial` objects are like :class:`function` objects in that they are +:class:`partial` objects are like :ref:`function objects ` in that they are callable, weak referenceable, and can have attributes. There are some important -differences. For instance, the :attr:`~definition.__name__` and :attr:`__doc__` attributes -are not created automatically. Also, :class:`partial` objects defined in -classes behave like static methods and do not transform into bound methods -during instance attribute look-up. +differences. For instance, the :attr:`~definition.__name__` and :attr:`~definition.__doc__` attributes +are not created automatically. diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index e36a71af2b64ab..480a9dec7f133b 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -1,5 +1,5 @@ -:mod:`gc` --- Garbage Collector interface -========================================= +:mod:`!gc` --- Garbage Collector interface +========================================== .. module:: gc :synopsis: Interface to the cycle-detecting garbage collector. @@ -40,11 +40,18 @@ The :mod:`gc` module provides the following functions: .. function:: collect(generation=2) - With no arguments, run a full collection. The optional argument *generation* + Perform a collection. The optional argument *generation* may be an integer specifying which generation to collect (from 0 to 2). A - :exc:`ValueError` is raised if the generation number is invalid. The sum of + :exc:`ValueError` is raised if the generation number is invalid. The sum of collected objects and uncollectable objects is returned. + Calling ``gc.collect(0)`` will perform a GC collection on the young generation. + + Calling ``gc.collect(1)`` will perform a GC collection on the young generation + and an increment of the old generation. + + Calling ``gc.collect(2)`` or ``gc.collect()`` performs a full collection + The free lists maintained for a number of built-in types are cleared whenever a full collection or collection of the highest generation (2) is run. Not all items in some free lists may be freed due to the @@ -53,6 +60,9 @@ The :mod:`gc` module provides the following functions: The effect of calling ``gc.collect()`` while the interpreter is already performing a collection is undefined. + .. versionchanged:: 3.13 + ``generation=1`` performs an increment of collection. + .. function:: set_debug(flags) @@ -68,13 +78,20 @@ The :mod:`gc` module provides the following functions: .. function:: get_objects(generation=None) + Returns a list of all objects tracked by the collector, excluding the list - returned. If *generation* is not None, return only the objects tracked by - the collector that are in that generation. + returned. If *generation* is not ``None``, return only the objects as follows: + + * 0: All objects in the young generation + * 1: No objects, as there is no generation 1 (as of Python 3.13) + * 2: All objects in the old generation .. versionchanged:: 3.8 New *generation* parameter. + .. versionchanged:: 3.13 + Generation 1 is removed + .. audit-event:: gc.get_objects generation gc.get_objects .. function:: get_stats() @@ -101,19 +118,27 @@ The :mod:`gc` module provides the following functions: Set the garbage collection thresholds (the collection frequency). Setting *threshold0* to zero disables collection. - The GC classifies objects into three generations depending on how many - collection sweeps they have survived. New objects are placed in the youngest - generation (generation ``0``). If an object survives a collection it is moved - into the next older generation. Since generation ``2`` is the oldest - generation, objects in that generation remain there after a collection. In - order to decide when to run, the collector keeps track of the number object + The GC classifies objects into two generations depending on whether they have + survived a collection. New objects are placed in the young generation. If an + object survives a collection it is moved into the old generation. + + In order to decide when to run, the collector keeps track of the number of object allocations and deallocations since the last collection. When the number of allocations minus the number of deallocations exceeds *threshold0*, collection - starts. Initially only generation ``0`` is examined. If generation ``0`` has - been examined more than *threshold1* times since generation ``1`` has been - examined, then generation ``1`` is examined as well. - With the third generation, things are a bit more complicated, - see `Collecting the oldest generation `_ for more information. + starts. For each collection, all the objects in the young generation and some + fraction of the old generation is collected. + + The fraction of the old generation that is collected is **inversely** proportional + to *threshold1*. The larger *threshold1* is, the slower objects in the old generation + are collected. + For the default value of 10, 1% of the old generation is scanned during each collection. + + *threshold2* is ignored. + + See `Garbage collector design `_ for more information. + + .. versionchanged:: 3.13 + *threshold2* is ignored .. function:: get_count() @@ -179,8 +204,6 @@ The :mod:`gc` module provides the following functions: >>> gc.is_tracked({}) False >>> gc.is_tracked({"a": 1}) - False - >>> gc.is_tracked({"a": []}) True .. versionadded:: 3.1 diff --git a/Doc/library/getopt.rst b/Doc/library/getopt.rst index ada68b240143e8..5c63009e22d58c 100644 --- a/Doc/library/getopt.rst +++ b/Doc/library/getopt.rst @@ -1,5 +1,5 @@ -:mod:`getopt` --- C-style parser for command line options -========================================================= +:mod:`!getopt` --- C-style parser for command line options +========================================================== .. module:: getopt :synopsis: Portable parser for command line options; support both short and @@ -7,18 +7,13 @@ **Source code:** :source:`Lib/getopt.py` -.. deprecated:: 3.13 - The :mod:`getopt` module is :term:`soft deprecated` and will not be - developed further; development will continue with the :mod:`argparse` - module. - .. note:: - The :mod:`getopt` module is a parser for command line options whose API is - designed to be familiar to users of the C :c:func:`!getopt` function. Users who - are unfamiliar with the C :c:func:`!getopt` function or who would like to write - less code and get better help and error messages should consider using the - :mod:`argparse` module instead. + This module is considered feature complete. A more declarative and + extensible alternative to this API is provided in the :mod:`optparse` + module. Further functional enhancements for command line parameter + processing are provided either as third party modules on PyPI, + or else as features in the :mod:`argparse` module. -------------- @@ -28,6 +23,13 @@ the special meanings of arguments of the form '``-``' and '``--``'). Long options similar to those supported by GNU software may be used as well via an optional third argument. +Users who are unfamiliar with the Unix :c:func:`!getopt` function should consider +using the :mod:`argparse` module instead. Users who are familiar with the Unix +:c:func:`!getopt` function, but would like to get equivalent behavior while +writing less code and getting better help and error messages should consider +using the :mod:`optparse` module. See :ref:`choosing-an-argument-parser` for +additional details. + This module provides two functions and an exception: @@ -38,7 +40,8 @@ exception: be parsed, without the leading reference to the running program. Typically, this means ``sys.argv[1:]``. *shortopts* is the string of option letters that the script wants to recognize, with options that require an argument followed by a - colon (``':'``; i.e., the same format that Unix :c:func:`!getopt` uses). + colon (``':'``) and options that accept an optional argument followed by + two colons (``'::'``); i.e., the same format that Unix :c:func:`!getopt` uses. .. note:: @@ -49,8 +52,10 @@ exception: *longopts*, if specified, must be a list of strings with the names of the long options which should be supported. The leading ``'--'`` characters should not be included in the option name. Long options which require an - argument should be followed by an equal sign (``'='``). Optional arguments - are not supported. To accept only long options, *shortopts* should be an + argument should be followed by an equal sign (``'='``). + Long options which accept an optional argument should be followed by + an equal sign and question mark (``'=?'``). + To accept only long options, *shortopts* should be an empty string. Long options on the command line can be recognized so long as they provide a prefix of the option name that matches exactly one of the accepted options. For example, if *longopts* is ``['foo', 'frob']``, the @@ -67,6 +72,9 @@ exception: options occur in the list in the same order in which they were found, thus allowing multiple occurrences. Long and short options may be mixed. + .. versionchanged:: 3.14 + Optional arguments are supported. + .. function:: gnu_getopt(args, shortopts, longopts=[]) @@ -79,6 +87,16 @@ exception: variable :envvar:`!POSIXLY_CORRECT` is set, then option processing stops as soon as a non-option argument is encountered. + If the first character of the option string is ``'-'``, non-option arguments + that are followed by options are added to the list of option-and-value pairs + as a pair that has ``None`` as its first element and the list of non-option + arguments as its second element. + The second element of the :func:`!gnu_getopt` result is a list of + program arguments after the last option. + + .. versionchanged:: 3.14 + Support for returning intermixed options and non-option arguments in order. + .. exception:: GetoptError @@ -97,6 +115,8 @@ exception: An example using only Unix style options: +.. doctest:: + >>> import getopt >>> args = '-a -b -cfoo -d bar a1 a2'.split() >>> args @@ -109,6 +129,8 @@ An example using only Unix style options: Using long option names is equally easy: +.. doctest:: + >>> s = '--condition=foo --testing --output-file abc.def -x a1 a2' >>> args = s.split() >>> args @@ -120,7 +142,37 @@ Using long option names is equally easy: >>> args ['a1', 'a2'] -In a script, typical usage is something like this:: +Optional arguments should be specified explicitly: + +.. doctest:: + + >>> s = '-Con -C --color=off --color a1 a2' + >>> args = s.split() + >>> args + ['-Con', '-C', '--color=off', '--color', 'a1', 'a2'] + >>> optlist, args = getopt.getopt(args, 'C::', ['color=?']) + >>> optlist + [('-C', 'on'), ('-C', ''), ('--color', 'off'), ('--color', '')] + >>> args + ['a1', 'a2'] + +The order of options and non-option arguments can be preserved: + +.. doctest:: + + >>> s = 'a1 -x a2 a3 a4 --long a5 a6' + >>> args = s.split() + >>> args + ['a1', '-x', 'a2', 'a3', 'a4', '--long', 'a5', 'a6'] + >>> optlist, args = getopt.gnu_getopt(args, '-x:', ['long=']) + >>> optlist + [(None, ['a1']), ('-x', 'a2'), (None, ['a3', 'a4']), ('--long', 'a5')] + >>> args + ['a6'] + +In a script, typical usage is something like this: + +.. testcode:: import getopt, sys @@ -144,13 +196,29 @@ In a script, typical usage is something like this:: output = a else: assert False, "unhandled option" - # ... + process(args, output=output, verbose=verbose) if __name__ == "__main__": main() Note that an equivalent command line interface could be produced with less code -and more informative help and error messages by using the :mod:`argparse` module:: +and more informative help and error messages by using the :mod:`optparse` module: + +.. testcode:: + + import optparse + + if __name__ == '__main__': + parser = optparse.OptionParser() + parser.add_option('-o', '--output') + parser.add_option('-v', dest='verbose', action='store_true') + opts, args = parser.parse_args() + process(args, output=opts.output, verbose=opts.verbose) + +A roughly equivalent command line interface for this case can also be +produced by using the :mod:`argparse` module: + +.. testcode:: import argparse @@ -158,12 +226,18 @@ and more informative help and error messages by using the :mod:`argparse` module parser = argparse.ArgumentParser() parser.add_argument('-o', '--output') parser.add_argument('-v', dest='verbose', action='store_true') + parser.add_argument('rest', nargs='*') args = parser.parse_args() - # ... do something with args.output ... - # ... do something with args.verbose .. + process(args.rest, output=args.output, verbose=args.verbose) + +See :ref:`choosing-an-argument-parser` for details on how the ``argparse`` +version of this code differs in behaviour from the ``optparse`` (and +``getopt``) version. .. seealso:: - Module :mod:`argparse` - Alternative command line option and argument parsing library. + Module :mod:`optparse` + Declarative command line option parsing. + Module :mod:`argparse` + More opinionated command line option and argument parsing library. diff --git a/Doc/library/getpass.rst b/Doc/library/getpass.rst index 54c84d45a59856..3b5296f9ec66fa 100644 --- a/Doc/library/getpass.rst +++ b/Doc/library/getpass.rst @@ -1,5 +1,5 @@ -:mod:`getpass` --- Portable password input -========================================== +:mod:`!getpass` --- Portable password input +=========================================== .. module:: getpass :synopsis: Portable reading of passwords and retrieval of the userid. @@ -49,7 +49,7 @@ The :mod:`getpass` module provides two functions: systems which support the :mod:`pwd` module, otherwise, an :exc:`OSError` is raised. - In general, this function should be preferred over :func:`os.getlogin()`. + In general, this function should be preferred over :func:`os.getlogin`. .. versionchanged:: 3.13 Previously, various exceptions beyond just :exc:`OSError` were raised. diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index 41beac3e0c7396..d0de83907eb297 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -1,5 +1,5 @@ -:mod:`gettext` --- Multilingual internationalization services -============================================================= +:mod:`!gettext` --- Multilingual internationalization services +============================================================== .. module:: gettext :synopsis: Multilingual internationalization services. diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 19a0bbba8966ba..684466d354aef8 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -1,5 +1,5 @@ -:mod:`glob` --- Unix style pathname pattern expansion -===================================================== +:mod:`!glob` --- Unix style pathname pattern expansion +====================================================== .. module:: glob :synopsis: Unix shell style pathname pattern expansion. @@ -75,6 +75,10 @@ The :mod:`glob` module defines the following functions: Using the "``**``" pattern in large directory trees may consume an inordinate amount of time. + .. note:: + This function may return duplicate path names if *pathname* + contains multiple "``**``" patterns and *recursive* is true. + .. versionchanged:: 3.5 Support for recursive globs using "``**``". @@ -94,6 +98,10 @@ The :mod:`glob` module defines the following functions: .. audit-event:: glob.glob pathname,recursive glob.iglob .. audit-event:: glob.glob/2 pathname,recursive,root_dir,dir_fd glob.iglob + .. note:: + This function may return duplicate path names if *pathname* + contains multiple "``**``" patterns and *recursive* is true. + .. versionchanged:: 3.5 Support for recursive globs using "``**``". @@ -136,8 +144,7 @@ The :mod:`glob` module defines the following functions: separators, and ``*`` pattern segments match precisely one path segment. If *recursive* is true, the pattern segment "``**``" will match any number - of path segments. If "``**``" occurs in any position other than a full - pattern segment, :exc:`ValueError` is raised. + of path segments. If *include_hidden* is true, wildcards can match path segments that start with a dot (``.``). diff --git a/Doc/library/graphlib.rst b/Doc/library/graphlib.rst index 5414d6370b78ce..a0b16576fad219 100644 --- a/Doc/library/graphlib.rst +++ b/Doc/library/graphlib.rst @@ -1,5 +1,5 @@ -:mod:`graphlib` --- Functionality to operate with graph-like structures -========================================================================= +:mod:`!graphlib` --- Functionality to operate with graph-like structures +======================================================================== .. module:: graphlib :synopsis: Functionality to operate with graph-like structures diff --git a/Doc/library/grp.rst b/Doc/library/grp.rst index 274a353103b488..d1c7f22a209780 100644 --- a/Doc/library/grp.rst +++ b/Doc/library/grp.rst @@ -1,5 +1,5 @@ -:mod:`grp` --- The group database -================================= +:mod:`!grp` --- The group database +================================== .. module:: grp :platform: Unix @@ -10,7 +10,7 @@ This module provides access to the Unix group database. It is available on all Unix versions. -.. availability:: Unix, not Emscripten, not WASI. +.. availability:: Unix, not WASI, not Android, not iOS. Group database entries are reported as a tuple-like object, whose attributes correspond to the members of the ``group`` structure (Attribute field below, see diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index 79be215a766045..f24e73517e5767 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -1,5 +1,5 @@ -:mod:`gzip` --- Support for :program:`gzip` files -================================================= +:mod:`!gzip` --- Support for :program:`gzip` files +================================================== .. module:: gzip :synopsis: Interfaces for gzip compression and decompression using file objects. @@ -100,10 +100,12 @@ The module defines the following items: compression, and ``9`` is slowest and produces the most compression. ``0`` is no compression. The default is ``9``. - The *mtime* argument is an optional numeric timestamp to be written to - the last modification time field in the stream when compressing. It - should only be provided in compression mode. If omitted or ``None``, the - current time is used. See the :attr:`mtime` attribute for more details. + The optional *mtime* argument is the timestamp requested by gzip. The time + is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. + If *mtime* is omitted or ``None``, the current time is used. Use *mtime* = 0 + to generate a compressed stream that does not depend on creation time. + + See below for the :attr:`mtime` attribute that is set when decompressing. Calling a :class:`GzipFile` object's :meth:`!close` method does not close *fileobj*, since you might wish to append more material after the compressed @@ -131,17 +133,19 @@ The module defines the following items: .. versionadded:: 3.2 - .. attribute:: mtime + .. attribute:: mode - When decompressing, the value of the last modification time field in - the most recently read header may be read from this attribute, as an - integer. The initial value before reading any headers is ``None``. + ``'rb'`` for reading and ``'wb'`` for writing. - All :program:`gzip` compressed streams are required to contain this - timestamp field. Some programs, such as :program:`gunzip`\ , make use - of the timestamp. The format is the same as the return value of - :func:`time.time` and the :attr:`~os.stat_result.st_mtime` attribute of - the object returned by :func:`os.stat`. + .. versionchanged:: 3.13 + In previous versions it was an integer ``1`` or ``2``. + + .. attribute:: mtime + + When decompressing, this attribute is set to the last timestamp in the most + recently read header. It is an integer, holding the number of seconds + since the Unix epoch (00:00:00 UTC, January 1, 1970). + The initial value before reading any headers is ``None``. .. attribute:: name @@ -171,22 +175,21 @@ The module defines the following items: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. versionchanged:: 3.12 - Remove the ``filename`` attribute, use the :attr:`~GzipFile.name` - attribute instead. - .. deprecated:: 3.9 Opening :class:`GzipFile` for writing without specifying the *mode* argument is deprecated. + .. versionchanged:: 3.12 + Remove the ``filename`` attribute, use the :attr:`~GzipFile.name` + attribute instead. + -.. function:: compress(data, compresslevel=9, *, mtime=None) +.. function:: compress(data, compresslevel=9, *, mtime=0) Compress the *data*, returning a :class:`bytes` object containing the compressed data. *compresslevel* and *mtime* have the same meaning as in - the :class:`GzipFile` constructor above. When *mtime* is set to ``0``, this - function is equivalent to :func:`zlib.compress` with *wbits* set to ``31``. - The zlib function is faster. + the :class:`GzipFile` constructor above, + but *mtime* defaults to 0 for reproducible output. .. versionadded:: 3.2 .. versionchanged:: 3.8 @@ -194,7 +197,17 @@ The module defines the following items: .. versionchanged:: 3.11 Speed is improved by compressing all data at once instead of in a streamed fashion. Calls with *mtime* set to ``0`` are delegated to - :func:`zlib.compress` for better speed. + :func:`zlib.compress` for better speed. In this situation the + output may contain a gzip header "OS" byte value other than 255 + "unknown" as supplied by the underlying zlib implementation. + + .. versionchanged:: 3.13 + The gzip header OS byte is guaranteed to be set to 255 when this function + is used as was the case in 3.10 and earlier. + .. versionchanged:: 3.14 + The *mtime* parameter now defaults to 0 for reproducible output. + For the previous behaviour of using the current time, + pass ``None`` to *mtime*. .. function:: decompress(data) diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 0726be4590e399..dffb167c74771f 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -1,5 +1,5 @@ -:mod:`hashlib` --- Secure hashes and message digests -==================================================== +:mod:`!hashlib` --- Secure hashes and message digests +===================================================== .. module:: hashlib :synopsis: Secure hash and message digest algorithms. @@ -77,8 +77,6 @@ accessible by name via :func:`new`. See :data:`algorithms_available`. SHA3 (Keccak) and SHAKE constructors :func:`sha3_224`, :func:`sha3_256`, :func:`sha3_384`, :func:`sha3_512`, :func:`shake_128`, :func:`shake_256` were added. - -.. versionadded:: 3.6 :func:`blake2b` and :func:`blake2s` were added. .. _hashlib-usedforsecurity: @@ -330,7 +328,7 @@ include a `salt `_. your application, read *Appendix A.2.2* of NIST-SP-800-132_. The answers on the `stackexchange pbkdf2 iterations question`_ explain in detail. - *dklen* is the length of the derived key. If *dklen* is ``None`` then the + *dklen* is the length of the derived key in bytes. If *dklen* is ``None`` then the digest size of the hash algorithm *hash_name* is used, e.g. 64 for SHA-512. >>> from hashlib import pbkdf2_hmac @@ -359,7 +357,7 @@ include a `salt `_. *n* is the CPU/Memory cost factor, *r* the block size, *p* parallelization factor and *maxmem* limits memory (OpenSSL 1.1.0 defaults to 32 MiB). - *dklen* is the length of the derived key. + *dklen* is the length of the derived key in bytes. .. versionadded:: 3.6 @@ -657,7 +655,7 @@ on the hash function used in digital signatures. by the signer. (`NIST SP-800-106 "Randomized Hashing for Digital Signatures" - `_) + `_) In BLAKE2 the salt is processed as a one-time input to the hash function during initialization, rather than as an input to each compression function. @@ -811,8 +809,8 @@ Domain Dedication 1.0 Universal: .. _NIST-SP-800-132: https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-132.pdf .. _stackexchange pbkdf2 iterations question: https://security.stackexchange.com/questions/3959/recommended-of-iterations-when-using-pbkdf2-sha256/ .. _Attacks on cryptographic hash algorithms: https://en.wikipedia.org/wiki/Cryptographic_hash_function#Attacks_on_cryptographic_hash_algorithms -.. _the FIPS 180-4 standard: https://csrc.nist.gov/publications/detail/fips/180/4/final -.. _the FIPS 202 standard: https://csrc.nist.gov/publications/detail/fips/202/final +.. _the FIPS 180-4 standard: https://csrc.nist.gov/pubs/fips/180-4/upd1/final +.. _the FIPS 202 standard: https://csrc.nist.gov/pubs/fips/202/final .. _HACL\* project: https://github.com/hacl-star/hacl-star @@ -829,7 +827,7 @@ Domain Dedication 1.0 Universal: https://nvlpubs.nist.gov/nistpubs/fips/nist.fips.180-4.pdf The FIPS 180-4 publication on Secure Hash Algorithms. - https://csrc.nist.gov/publications/detail/fips/202/final + https://csrc.nist.gov/pubs/fips/202/final The FIPS 202 publication on the SHA-3 Standard. https://www.blake2.net/ diff --git a/Doc/library/heapq.rst b/Doc/library/heapq.rst index ddbada13bddf5b..d3c4b920ba500a 100644 --- a/Doc/library/heapq.rst +++ b/Doc/library/heapq.rst @@ -1,5 +1,5 @@ -:mod:`heapq` --- Heap queue algorithm -===================================== +:mod:`!heapq` --- Heap queue algorithm +====================================== .. module:: heapq :synopsis: Heap queue algorithm (a.k.a. priority queue). @@ -17,7 +17,9 @@ This module provides an implementation of the heap queue algorithm, also known as the priority queue algorithm. Heaps are binary trees for which every parent node has a value less than or -equal to any of its children. This implementation uses arrays for which +equal to any of its children. We refer to this condition as the heap invariant. + +This implementation uses arrays for which ``heap[k] <= heap[2*k+1]`` and ``heap[k] <= heap[2*k+2]`` for all *k*, counting elements from zero. For the sake of comparison, non-existing elements are considered to be infinite. The interesting property of a heap is that its @@ -319,4 +321,3 @@ applications, and I think it is good to keep a 'heap' module around. :-) backwards, and this was also used to avoid the rewinding time. Believe me, real good tape sorts were quite spectacular to watch! From all times, sorting has always been a Great Art! :-) - diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index 43012e03c580e8..d6692033b2d4c3 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -1,5 +1,5 @@ -:mod:`hmac` --- Keyed-Hashing for Message Authentication -======================================================== +:mod:`!hmac` --- Keyed-Hashing for Message Authentication +========================================================= .. module:: hmac :synopsis: Keyed-Hashing for Message Authentication (HMAC) implementation diff --git a/Doc/library/html.entities.rst b/Doc/library/html.entities.rst index 10529561a92cd0..add18e4c87d220 100644 --- a/Doc/library/html.entities.rst +++ b/Doc/library/html.entities.rst @@ -1,5 +1,5 @@ -:mod:`html.entities` --- Definitions of HTML general entities -============================================================= +:mod:`!html.entities` --- Definitions of HTML general entities +============================================================== .. module:: html.entities :synopsis: Definitions of HTML general entities. diff --git a/Doc/library/html.parser.rst b/Doc/library/html.parser.rst index d35090111e0822..6d433b5a04fc4a 100644 --- a/Doc/library/html.parser.rst +++ b/Doc/library/html.parser.rst @@ -1,5 +1,5 @@ -:mod:`html.parser` --- Simple HTML and XHTML parser -=================================================== +:mod:`!html.parser` --- Simple HTML and XHTML parser +==================================================== .. module:: html.parser :synopsis: A simple parser that can handle HTML and XHTML. diff --git a/Doc/library/html.rst b/Doc/library/html.rst index c2b01e14ea7555..9aa39ba9a42b0f 100644 --- a/Doc/library/html.rst +++ b/Doc/library/html.rst @@ -1,5 +1,5 @@ -:mod:`html` --- HyperText Markup Language support -================================================= +:mod:`!html` --- HyperText Markup Language support +================================================== .. module:: html :synopsis: Helpers for manipulating HTML. diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 7e4502064f22a1..2835c8d0eb711e 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -1,5 +1,5 @@ -:mod:`http.client` --- HTTP protocol client -=========================================== +:mod:`!http.client` --- HTTP protocol client +============================================ .. module:: http.client :synopsis: HTTP and HTTPS protocol client (requires sockets). diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index 2fe188be641c2d..23ddecf873876d 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -1,5 +1,5 @@ -:mod:`http.cookiejar` --- Cookie handling for HTTP clients -========================================================== +:mod:`!http.cookiejar` --- Cookie handling for HTTP clients +=========================================================== .. module:: http.cookiejar :synopsis: Classes for automatic handling of HTTP cookies. @@ -137,7 +137,7 @@ The following classes are provided: The Netscape protocol with the bugs fixed. Uses :mailheader:`Set-Cookie2` in place of :mailheader:`Set-Cookie`. Not widely used. - http://kristol.org/cookie/errata.html + https://kristol.org/cookie/errata.html Unfinished errata to :rfc:`2965`. :rfc:`2964` - Use of HTTP State Management diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst index e91972fe621a48..eb196320721194 100644 --- a/Doc/library/http.cookies.rst +++ b/Doc/library/http.cookies.rst @@ -1,5 +1,5 @@ -:mod:`http.cookies` --- HTTP state management -============================================= +:mod:`!http.cookies` --- HTTP state management +============================================== .. module:: http.cookies :synopsis: Support for HTTP state management (cookies). @@ -98,7 +98,7 @@ Cookie Objects .. method:: BaseCookie.output(attrs=None, header='Set-Cookie:', sep='\r\n') Return a string representation suitable to be sent as HTTP headers. *attrs* and - *header* are sent to each :class:`Morsel`'s :meth:`output` method. *sep* is used + *header* are sent to each :class:`Morsel`'s :meth:`~Morsel.output` method. *sep* is used to join the headers together, and is by default the combination ``'\r\n'`` (CRLF). @@ -142,6 +142,7 @@ Morsel Objects version httponly samesite + partitioned The attribute :attr:`httponly` specifies that the cookie is only transferred in HTTP requests, and is not accessible through JavaScript. This is intended @@ -151,6 +152,19 @@ Morsel Objects send the cookie along with cross-site requests. This helps to mitigate CSRF attacks. Valid values for this attribute are "Strict" and "Lax". + The attribute :attr:`partitioned` indicates to user agents that these + cross-site cookies *should* only be available in the same top-level context + that the cookie was first set in. For this to be accepted by the user agent, + you **must** also set ``Secure``. + + In addition, it is recommended to use the ``__Host`` prefix when setting + partitioned cookies to make them bound to the hostname and not the + registrable domain. Read + `CHIPS (Cookies Having Independent Partitioned State)`_ + for full details and examples. + + .. _CHIPS (Cookies Having Independent Partitioned State): https://github.com/privacycg/CHIPS/blob/main/README.md + The keys are case-insensitive and their default value is ``''``. .. versionchanged:: 3.5 @@ -165,6 +179,9 @@ Morsel Objects .. versionchanged:: 3.8 Added support for the :attr:`samesite` attribute. + .. versionchanged:: 3.14 + Added support for the :attr:`partitioned` attribute. + .. attribute:: Morsel.value diff --git a/Doc/library/http.rst b/Doc/library/http.rst index 5e1912716e5319..ce3fb9f8120502 100644 --- a/Doc/library/http.rst +++ b/Doc/library/http.rst @@ -1,5 +1,5 @@ -:mod:`http` --- HTTP modules -============================ +:mod:`!http` --- HTTP modules +============================= .. module:: http :synopsis: HTTP status codes and messages @@ -59,63 +59,63 @@ available in :class:`http.HTTPStatus` are: ======= =================================== ================================================================== Code Enum Name Details ======= =================================== ================================================================== -``100`` ``CONTINUE`` HTTP/1.1 :rfc:`7231`, Section 6.2.1 -``101`` ``SWITCHING_PROTOCOLS`` HTTP/1.1 :rfc:`7231`, Section 6.2.2 +``100`` ``CONTINUE`` HTTP Semantics :rfc:`9110`, Section 15.2.1 +``101`` ``SWITCHING_PROTOCOLS`` HTTP Semantics :rfc:`9110`, Section 15.2.2 ``102`` ``PROCESSING`` WebDAV :rfc:`2518`, Section 10.1 ``103`` ``EARLY_HINTS`` An HTTP Status Code for Indicating Hints :rfc:`8297` -``200`` ``OK`` HTTP/1.1 :rfc:`7231`, Section 6.3.1 -``201`` ``CREATED`` HTTP/1.1 :rfc:`7231`, Section 6.3.2 -``202`` ``ACCEPTED`` HTTP/1.1 :rfc:`7231`, Section 6.3.3 -``203`` ``NON_AUTHORITATIVE_INFORMATION`` HTTP/1.1 :rfc:`7231`, Section 6.3.4 -``204`` ``NO_CONTENT`` HTTP/1.1 :rfc:`7231`, Section 6.3.5 -``205`` ``RESET_CONTENT`` HTTP/1.1 :rfc:`7231`, Section 6.3.6 -``206`` ``PARTIAL_CONTENT`` HTTP/1.1 :rfc:`7233`, Section 4.1 +``200`` ``OK`` HTTP Semantics :rfc:`9110`, Section 15.3.1 +``201`` ``CREATED`` HTTP Semantics :rfc:`9110`, Section 15.3.2 +``202`` ``ACCEPTED`` HTTP Semantics :rfc:`9110`, Section 15.3.3 +``203`` ``NON_AUTHORITATIVE_INFORMATION`` HTTP Semantics :rfc:`9110`, Section 15.3.4 +``204`` ``NO_CONTENT`` HTTP Semantics :rfc:`9110`, Section 15.3.5 +``205`` ``RESET_CONTENT`` HTTP Semantics :rfc:`9110`, Section 15.3.6 +``206`` ``PARTIAL_CONTENT`` HTTP Semantics :rfc:`9110`, Section 15.3.7 ``207`` ``MULTI_STATUS`` WebDAV :rfc:`4918`, Section 11.1 ``208`` ``ALREADY_REPORTED`` WebDAV Binding Extensions :rfc:`5842`, Section 7.1 (Experimental) ``226`` ``IM_USED`` Delta Encoding in HTTP :rfc:`3229`, Section 10.4.1 -``300`` ``MULTIPLE_CHOICES`` HTTP/1.1 :rfc:`7231`, Section 6.4.1 -``301`` ``MOVED_PERMANENTLY`` HTTP/1.1 :rfc:`7231`, Section 6.4.2 -``302`` ``FOUND`` HTTP/1.1 :rfc:`7231`, Section 6.4.3 -``303`` ``SEE_OTHER`` HTTP/1.1 :rfc:`7231`, Section 6.4.4 -``304`` ``NOT_MODIFIED`` HTTP/1.1 :rfc:`7232`, Section 4.1 -``305`` ``USE_PROXY`` HTTP/1.1 :rfc:`7231`, Section 6.4.5 -``307`` ``TEMPORARY_REDIRECT`` HTTP/1.1 :rfc:`7231`, Section 6.4.7 -``308`` ``PERMANENT_REDIRECT`` Permanent Redirect :rfc:`7238`, Section 3 (Experimental) -``400`` ``BAD_REQUEST`` HTTP/1.1 :rfc:`7231`, Section 6.5.1 -``401`` ``UNAUTHORIZED`` HTTP/1.1 Authentication :rfc:`7235`, Section 3.1 -``402`` ``PAYMENT_REQUIRED`` HTTP/1.1 :rfc:`7231`, Section 6.5.2 -``403`` ``FORBIDDEN`` HTTP/1.1 :rfc:`7231`, Section 6.5.3 -``404`` ``NOT_FOUND`` HTTP/1.1 :rfc:`7231`, Section 6.5.4 -``405`` ``METHOD_NOT_ALLOWED`` HTTP/1.1 :rfc:`7231`, Section 6.5.5 -``406`` ``NOT_ACCEPTABLE`` HTTP/1.1 :rfc:`7231`, Section 6.5.6 -``407`` ``PROXY_AUTHENTICATION_REQUIRED`` HTTP/1.1 Authentication :rfc:`7235`, Section 3.2 -``408`` ``REQUEST_TIMEOUT`` HTTP/1.1 :rfc:`7231`, Section 6.5.7 -``409`` ``CONFLICT`` HTTP/1.1 :rfc:`7231`, Section 6.5.8 -``410`` ``GONE`` HTTP/1.1 :rfc:`7231`, Section 6.5.9 -``411`` ``LENGTH_REQUIRED`` HTTP/1.1 :rfc:`7231`, Section 6.5.10 -``412`` ``PRECONDITION_FAILED`` HTTP/1.1 :rfc:`7232`, Section 4.2 -``413`` ``REQUEST_ENTITY_TOO_LARGE`` HTTP/1.1 :rfc:`7231`, Section 6.5.11 -``414`` ``REQUEST_URI_TOO_LONG`` HTTP/1.1 :rfc:`7231`, Section 6.5.12 -``415`` ``UNSUPPORTED_MEDIA_TYPE`` HTTP/1.1 :rfc:`7231`, Section 6.5.13 -``416`` ``REQUESTED_RANGE_NOT_SATISFIABLE`` HTTP/1.1 Range Requests :rfc:`7233`, Section 4.4 -``417`` ``EXPECTATION_FAILED`` HTTP/1.1 :rfc:`7231`, Section 6.5.14 +``300`` ``MULTIPLE_CHOICES`` HTTP Semantics :rfc:`9110`, Section 15.4.1 +``301`` ``MOVED_PERMANENTLY`` HTTP Semantics :rfc:`9110`, Section 15.4.2 +``302`` ``FOUND`` HTTP Semantics :rfc:`9110`, Section 15.4.3 +``303`` ``SEE_OTHER`` HTTP Semantics :rfc:`9110`, Section 15.4.4 +``304`` ``NOT_MODIFIED`` HTTP Semantics :rfc:`9110`, Section 15.4.5 +``305`` ``USE_PROXY`` HTTP Semantics :rfc:`9110`, Section 15.4.6 +``307`` ``TEMPORARY_REDIRECT`` HTTP Semantics :rfc:`9110`, Section 15.4.8 +``308`` ``PERMANENT_REDIRECT`` HTTP Semantics :rfc:`9110`, Section 15.4.9 +``400`` ``BAD_REQUEST`` HTTP Semantics :rfc:`9110`, Section 15.5.1 +``401`` ``UNAUTHORIZED`` HTTP Semantics :rfc:`9110`, Section 15.5.2 +``402`` ``PAYMENT_REQUIRED`` HTTP Semantics :rfc:`9110`, Section 15.5.3 +``403`` ``FORBIDDEN`` HTTP Semantics :rfc:`9110`, Section 15.5.4 +``404`` ``NOT_FOUND`` HTTP Semantics :rfc:`9110`, Section 15.5.5 +``405`` ``METHOD_NOT_ALLOWED`` HTTP Semantics :rfc:`9110`, Section 15.5.6 +``406`` ``NOT_ACCEPTABLE`` HTTP Semantics :rfc:`9110`, Section 15.5.7 +``407`` ``PROXY_AUTHENTICATION_REQUIRED`` HTTP Semantics :rfc:`9110`, Section 15.5.8 +``408`` ``REQUEST_TIMEOUT`` HTTP Semantics :rfc:`9110`, Section 15.5.9 +``409`` ``CONFLICT`` HTTP Semantics :rfc:`9110`, Section 15.5.10 +``410`` ``GONE`` HTTP Semantics :rfc:`9110`, Section 15.5.11 +``411`` ``LENGTH_REQUIRED`` HTTP Semantics :rfc:`9110`, Section 15.5.12 +``412`` ``PRECONDITION_FAILED`` HTTP Semantics :rfc:`9110`, Section 15.5.13 +``413`` ``CONTENT_TOO_LARGE`` HTTP Semantics :rfc:`9110`, Section 15.5.14 +``414`` ``URI_TOO_LONG`` HTTP Semantics :rfc:`9110`, Section 15.5.15 +``415`` ``UNSUPPORTED_MEDIA_TYPE`` HTTP Semantics :rfc:`9110`, Section 15.5.16 +``416`` ``RANGE_NOT_SATISFIABLE`` HTTP Semantics :rfc:`9110`, Section 15.5.17 +``417`` ``EXPECTATION_FAILED`` HTTP Semantics :rfc:`9110`, Section 15.5.18 ``418`` ``IM_A_TEAPOT`` HTCPCP/1.0 :rfc:`2324`, Section 2.3.2 -``421`` ``MISDIRECTED_REQUEST`` HTTP/2 :rfc:`7540`, Section 9.1.2 -``422`` ``UNPROCESSABLE_ENTITY`` WebDAV :rfc:`4918`, Section 11.2 +``421`` ``MISDIRECTED_REQUEST`` HTTP Semantics :rfc:`9110`, Section 15.5.20 +``422`` ``UNPROCESSABLE_CONTENT`` HTTP Semantics :rfc:`9110`, Section 15.5.21 ``423`` ``LOCKED`` WebDAV :rfc:`4918`, Section 11.3 ``424`` ``FAILED_DEPENDENCY`` WebDAV :rfc:`4918`, Section 11.4 ``425`` ``TOO_EARLY`` Using Early Data in HTTP :rfc:`8470` -``426`` ``UPGRADE_REQUIRED`` HTTP/1.1 :rfc:`7231`, Section 6.5.15 +``426`` ``UPGRADE_REQUIRED`` HTTP Semantics :rfc:`9110`, Section 15.5.22 ``428`` ``PRECONDITION_REQUIRED`` Additional HTTP Status Codes :rfc:`6585` ``429`` ``TOO_MANY_REQUESTS`` Additional HTTP Status Codes :rfc:`6585` ``431`` ``REQUEST_HEADER_FIELDS_TOO_LARGE`` Additional HTTP Status Codes :rfc:`6585` ``451`` ``UNAVAILABLE_FOR_LEGAL_REASONS`` An HTTP Status Code to Report Legal Obstacles :rfc:`7725` -``500`` ``INTERNAL_SERVER_ERROR`` HTTP/1.1 :rfc:`7231`, Section 6.6.1 -``501`` ``NOT_IMPLEMENTED`` HTTP/1.1 :rfc:`7231`, Section 6.6.2 -``502`` ``BAD_GATEWAY`` HTTP/1.1 :rfc:`7231`, Section 6.6.3 -``503`` ``SERVICE_UNAVAILABLE`` HTTP/1.1 :rfc:`7231`, Section 6.6.4 -``504`` ``GATEWAY_TIMEOUT`` HTTP/1.1 :rfc:`7231`, Section 6.6.5 -``505`` ``HTTP_VERSION_NOT_SUPPORTED`` HTTP/1.1 :rfc:`7231`, Section 6.6.6 +``500`` ``INTERNAL_SERVER_ERROR`` HTTP Semantics :rfc:`9110`, Section 15.6.1 +``501`` ``NOT_IMPLEMENTED`` HTTP Semantics :rfc:`9110`, Section 15.6.2 +``502`` ``BAD_GATEWAY`` HTTP Semantics :rfc:`9110`, Section 15.6.3 +``503`` ``SERVICE_UNAVAILABLE`` HTTP Semantics :rfc:`9110`, Section 15.6.4 +``504`` ``GATEWAY_TIMEOUT`` HTTP Semantics :rfc:`9110`, Section 15.6.5 +``505`` ``HTTP_VERSION_NOT_SUPPORTED`` HTTP Semantics :rfc:`9110`, Section 15.6.6 ``506`` ``VARIANT_ALSO_NEGOTIATES`` Transparent Content Negotiation in HTTP :rfc:`2295`, Section 8.1 (Experimental) ``507`` ``INSUFFICIENT_STORAGE`` WebDAV :rfc:`4918`, Section 11.5 ``508`` ``LOOP_DETECTED`` WebDAV Binding Extensions :rfc:`5842`, Section 7.2 (Experimental) @@ -137,6 +137,10 @@ equal to the constant name (i.e. ``http.HTTPStatus.OK`` is also available as .. versionadded:: 3.9 Added ``103 EARLY_HINTS``, ``418 IM_A_TEAPOT`` and ``425 TOO_EARLY`` status codes. +.. versionchanged:: 3.13 + Implemented RFC9110 naming for status constants. Old constant names are preserved for + backwards compatibility. + HTTP status category -------------------- @@ -144,15 +148,15 @@ HTTP status category The enum values have several properties to indicate the HTTP status category: -==================== ======================== =============================== +==================== ======================== ====================================== Property Indicates that Details -==================== ======================== =============================== -``is_informational`` ``100 <= status <= 199`` HTTP/1.1 :rfc:`7231`, Section 6 -``is_success`` ``200 <= status <= 299`` HTTP/1.1 :rfc:`7231`, Section 6 -``is_redirection`` ``300 <= status <= 399`` HTTP/1.1 :rfc:`7231`, Section 6 -``is_client_error`` ``400 <= status <= 499`` HTTP/1.1 :rfc:`7231`, Section 6 -``is_server_error`` ``500 <= status <= 599`` HTTP/1.1 :rfc:`7231`, Section 6 -==================== ======================== =============================== +==================== ======================== ====================================== +``is_informational`` ``100 <= status <= 199`` HTTP Semantics :rfc:`9110`, Section 15 +``is_success`` ``200 <= status <= 299`` HTTP Semantics :rfc:`9110`, Section 15 +``is_redirection`` ``300 <= status <= 399`` HTTP Semantics :rfc:`9110`, Section 15 +``is_client_error`` ``400 <= status <= 499`` HTTP Semantics :rfc:`9110`, Section 15 +``is_server_error`` ``500 <= status <= 599`` HTTP Semantics :rfc:`9110`, Section 15 +==================== ======================== ====================================== Usage:: @@ -203,13 +207,13 @@ available in :class:`http.HTTPMethod` are: =========== =================================== ================================================================== Method Enum Name Details =========== =================================== ================================================================== -``GET`` ``GET`` HTTP/1.1 :rfc:`7231`, Section 4.3.1 -``HEAD`` ``HEAD`` HTTP/1.1 :rfc:`7231`, Section 4.3.2 -``POST`` ``POST`` HTTP/1.1 :rfc:`7231`, Section 4.3.3 -``PUT`` ``PUT`` HTTP/1.1 :rfc:`7231`, Section 4.3.4 -``DELETE`` ``DELETE`` HTTP/1.1 :rfc:`7231`, Section 4.3.5 -``CONNECT`` ``CONNECT`` HTTP/1.1 :rfc:`7231`, Section 4.3.6 -``OPTIONS`` ``OPTIONS`` HTTP/1.1 :rfc:`7231`, Section 4.3.7 -``TRACE`` ``TRACE`` HTTP/1.1 :rfc:`7231`, Section 4.3.8 +``GET`` ``GET`` HTTP Semantics :rfc:`9110`, Section 9.3.1 +``HEAD`` ``HEAD`` HTTP Semantics :rfc:`9110`, Section 9.3.2 +``POST`` ``POST`` HTTP Semantics :rfc:`9110`, Section 9.3.3 +``PUT`` ``PUT`` HTTP Semantics :rfc:`9110`, Section 9.3.4 +``DELETE`` ``DELETE`` HTTP Semantics :rfc:`9110`, Section 9.3.5 +``CONNECT`` ``CONNECT`` HTTP Semantics :rfc:`9110`, Section 9.3.6 +``OPTIONS`` ``OPTIONS`` HTTP Semantics :rfc:`9110`, Section 9.3.7 +``TRACE`` ``TRACE`` HTTP Semantics :rfc:`9110`, Section 9.3.8 ``PATCH`` ``PATCH`` HTTP/1.1 :rfc:`5789` =========== =================================== ================================================================== diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 886e359bd8cd62..1197b575c00455 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -1,5 +1,5 @@ -:mod:`http.server` --- HTTP servers -=================================== +:mod:`!http.server` --- HTTP servers +==================================== .. module:: http.server :synopsis: HTTP server and request handlers. @@ -263,7 +263,7 @@ provides three different variants: Adds a blank line (indicating the end of the HTTP headers in the response) - to the headers buffer and calls :meth:`flush_headers()`. + to the headers buffer and calls :meth:`flush_headers`. .. versionchanged:: 3.2 The buffered headers are written to the output stream. @@ -378,7 +378,7 @@ provides three different variants: If the request was mapped to a file, it is opened. Any :exc:`OSError` exception in opening the requested file is mapped to a ``404``, - ``'File not found'`` error. If there was a ``'If-Modified-Since'`` + ``'File not found'`` error. If there was an ``'If-Modified-Since'`` header in the request, and the file was not modified after this time, a ``304``, ``'Not Modified'`` response is sent. Otherwise, the content type is guessed by calling the :meth:`guess_type` method, which in turn diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 249dc0ea6ba735..59b181aab3e484 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -429,7 +429,7 @@ Several non-character keys move the cursor and possibly delete characters. Deletion does not puts text on the clipboard, but IDLE has an undo list. Wherever this doc discusses keys, 'C' refers to the :kbd:`Control` key on Windows and -Unix and the :kbd:`Command` key on macOS. (And all such dicussions +Unix and the :kbd:`Command` key on macOS. (And all such discussions assume that the keys have not been re-bound to something else.) * Arrow keys move the cursor one character or line. @@ -604,7 +604,7 @@ in an editor window. The editing features described in previous subsections work when entering code interactively. IDLE's Shell window also responds to the following: -* :kbd:`C-c` attemps to interrupt statement execution (but may fail). +* :kbd:`C-c` attempts to interrupt statement execution (but may fail). * :kbd:`C-d` closes Shell if typed at a ``>>>`` prompt. diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index d5c868def3b64f..a2dad58b00b9fa 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -1,5 +1,5 @@ -:mod:`imaplib` --- IMAP4 protocol client -======================================== +:mod:`!imaplib` --- IMAP4 protocol client +========================================= .. module:: imaplib :synopsis: IMAP4 protocol client (requires sockets). @@ -39,7 +39,7 @@ base class: initialized. If *host* is not specified, ``''`` (the local host) is used. If *port* is omitted, the standard IMAP4 port (143) is used. The optional *timeout* parameter specifies a timeout in seconds for the connection attempt. - If timeout is not given or is None, the global default socket timeout is used. + If timeout is not given or is ``None``, the global default socket timeout is used. The :class:`IMAP4` class supports the :keyword:`with` statement. When used like this, the IMAP4 ``LOGOUT`` command is issued automatically when the @@ -97,7 +97,7 @@ There's also a subclass for secure connections: best practices. The optional *timeout* parameter specifies a timeout in seconds for the - connection attempt. If timeout is not given or is None, the global default + connection attempt. If timeout is not given or is ``None``, the global default socket timeout is used. .. versionchanged:: 3.3 @@ -360,7 +360,7 @@ An :class:`IMAP4` instance has the following methods: Opens socket to *port* at *host*. The optional *timeout* parameter specifies a timeout in seconds for the connection attempt. - If timeout is not given or is None, the global default socket timeout + If timeout is not given or is ``None``, the global default socket timeout is used. Also note that if the *timeout* parameter is set to be zero, it will raise a :class:`ValueError` to reject creating a non-blocking socket. This method is implicitly called by the :class:`IMAP4` constructor. @@ -622,7 +622,7 @@ retrieves and prints all messages:: import getpass, imaplib - M = imaplib.IMAP4() + M = imaplib.IMAP4(host='example.org') M.login(getpass.getuser(), getpass.getpass()) M.select() typ, data = M.search(None, 'ALL') diff --git a/Doc/library/imghdr.rst b/Doc/library/imghdr.rst new file mode 100644 index 00000000000000..56f26355f42558 --- /dev/null +++ b/Doc/library/imghdr.rst @@ -0,0 +1,19 @@ +:mod:`!imghdr` --- Determine the type of an image +================================================= + +.. module:: imghdr + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +Possible replacements are third-party libraries from PyPI: +:pypi:`filetype`, :pypi:`puremagic`, or :pypi:`python-magic`. +These are not supported or maintained by the Python core team. + +The last version of Python that provided the :mod:`!imghdr` module was +`Python 3.12 `_. diff --git a/Doc/library/imp.rst b/Doc/library/imp.rst new file mode 100644 index 00000000000000..3dc4c568b1ae2f --- /dev/null +++ b/Doc/library/imp.rst @@ -0,0 +1,18 @@ +:mod:`!imp` --- Access the import internals +=========================================== + +.. module:: imp + :synopsis: Removed in 3.12. + :deprecated: + +.. deprecated-removed:: 3.4 3.12 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.12 ` after +being deprecated in Python 3.4. + +The :ref:`removal notice ` includes guidance for +migrating code from :mod:`!imp` to :mod:`importlib`. + +The last version of Python that provided the :mod:`!imp` module was +`Python 3.11 `_. diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index cc4a0da92da60a..d80255f5313061 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -19,14 +19,14 @@ such as its entry points or its top-level names (`Import Package `_\s, modules, if any). Built in part on Python's import system, this library intends to replace similar functionality in the `entry point -API`_ and `metadata API`_ of ``pkg_resources``. Along with +API`_ and `metadata API`_ of ``pkg_resources``. Along with :mod:`importlib.resources`, this package can eliminate the need to use the older and less efficient ``pkg_resources`` package. ``importlib.metadata`` operates on third-party *distribution packages* installed into Python's ``site-packages`` directory via tools such as -`pip `_. +:pypi:`pip`. Specifically, it works with distributions with discoverable ``dist-info`` or ``egg-info`` directories, and metadata defined by the `Core metadata specifications `_. @@ -46,7 +46,7 @@ and metadata defined by the `Core metadata specifications `_ you've installed -using ``pip``. We start by creating a virtual environment and installing +using ``pip``. We start by creating a virtual environment and installing something into it: .. code-block:: shell-session @@ -87,7 +87,7 @@ You can get the version string for ``wheel`` by running the following: '0.32.3' You can also get a collection of entry points selectable by properties of the EntryPoint (typically 'group' or 'name'), such as -``console_scripts``, ``distutils.commands`` and others. Each group contains a +``console_scripts``, ``distutils.commands`` and others. Each group contains a collection of :ref:`EntryPoint ` objects. You can get the :ref:`metadata for a distribution `:: @@ -100,6 +100,13 @@ You can also get a :ref:`distribution's version number `, list its :ref:`requirements`. +.. exception:: PackageNotFoundError + + Subclass of :class:`ModuleNotFoundError` raised by several functions in this + module when queried for a distribution package which is not installed in the + current Python environment. + + Functional API ============== @@ -111,31 +118,53 @@ This package provides the following functionality via its public API. Entry points ------------ -The ``entry_points()`` function returns a collection of entry points. -Entry points are represented by ``EntryPoint`` instances; -each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and -a ``.load()`` method to resolve the value. There are also ``.module``, -``.attr``, and ``.extras`` attributes for getting the components of the -``.value`` attribute. +.. function:: entry_points(**select_params) + + Returns a :class:`EntryPoints` instance describing entry points for the + current environment. Any given keyword parameters are passed to the + :meth:`!select` method for comparison to the attributes of + the individual entry point definitions. + + Note: it is not currently possible to query for entry points based on + their :attr:`!EntryPoint.dist` attribute (as different :class:`!Distribution` + instances do not currently compare equal, even if they have the same attributes) + +.. class:: EntryPoints + + Details of a collection of installed entry points. + + Also provides a ``.groups`` attribute that reports all identified entry + point groups, and a ``.names`` attribute that reports all identified entry + point names. + +.. class:: EntryPoint + + Details of an installed entry point. + + Each :class:`!EntryPoint` instance has ``.name``, ``.group``, and ``.value`` + attributes and a ``.load()`` method to resolve the value. There are also + ``.module``, ``.attr``, and ``.extras`` attributes for getting the + components of the ``.value`` attribute, and ``.dist`` for obtaining + information regarding the distribution package that provides the entry point. Query all entry points:: >>> eps = entry_points() # doctest: +SKIP -The ``entry_points()`` function returns an ``EntryPoints`` object, -a collection of all ``EntryPoint`` objects with ``names`` and ``groups`` +The :func:`!entry_points` function returns a :class:`!EntryPoints` object, +a collection of all :class:`!EntryPoint` objects with ``names`` and ``groups`` attributes for convenience:: >>> sorted(eps.groups) # doctest: +SKIP ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] -``EntryPoints`` has a ``select`` method to select entry points +:class:`!EntryPoints` has a :meth:`!select` method to select entry points matching specific properties. Select entry points in the ``console_scripts`` group:: >>> scripts = eps.select(group='console_scripts') # doctest: +SKIP -Equivalently, since ``entry_points`` passes keyword arguments +Equivalently, since :func:`!entry_points` passes keyword arguments through to select:: >>> scripts = entry_points(group='console_scripts') # doctest: +SKIP @@ -167,7 +196,7 @@ Inspect the resolved entry point:: The ``group`` and ``name`` are arbitrary values defined by the package author and usually a client will wish to resolve all entry points for a particular -group. Read `the setuptools docs +group. Read `the setuptools docs `_ for more information on entry points, their definition, and usage. @@ -177,7 +206,7 @@ for more information on entry points, their definition, and usage. no parameters and always returned a dictionary of entry points, keyed by group. With ``importlib_metadata`` 5.0 and Python 3.12, ``entry_points`` always returns an ``EntryPoints`` object. See - `backports.entry_points_selectable `_ + :pypi:`backports.entry_points_selectable` for compatibility options. .. versionchanged:: 3.13 @@ -189,37 +218,46 @@ for more information on entry points, their definition, and usage. Distribution metadata --------------------- -Every `Distribution Package `_ includes some metadata, -which you can extract using the -``metadata()`` function:: +.. function:: metadata(distribution_name) + + Return the distribution metadata corresponding to the named + distribution package as a :class:`PackageMetadata` instance. + + Raises :exc:`PackageNotFoundError` if the named distribution + package is not installed in the current Python environment. + +.. class:: PackageMetadata + + A concrete implementation of the + `PackageMetadata protocol `_. + + In addition to providing the defined protocol methods and attributes, subscripting + the instance is equivalent to calling the :meth:`!get` method. + +Every `Distribution Package `_ +includes some metadata, which you can extract using the :func:`!metadata` function:: >>> wheel_metadata = metadata('wheel') # doctest: +SKIP -The keys of the returned data structure, a ``PackageMetadata``, -name the metadata keywords, and +The keys of the returned data structure name the metadata keywords, and the values are returned unparsed from the distribution metadata:: >>> wheel_metadata['Requires-Python'] # doctest: +SKIP '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' -``PackageMetadata`` also presents a ``json`` attribute that returns +:class:`PackageMetadata` also presents a :attr:`!json` attribute that returns all the metadata in a JSON-compatible form per :PEP:`566`:: >>> wheel_metadata.json['requires_python'] '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' -.. note:: - - The actual type of the object returned by ``metadata()`` is an - implementation detail and should be accessed only through the interface - described by the - `PackageMetadata protocol `_. +The full set of available metadata is not described here. +See the PyPA `Core metadata specification `_ for additional details. .. versionchanged:: 3.10 The ``Description`` is now included in the metadata when presented through the payload. Line continuation characters have been removed. -.. versionadded:: 3.10 The ``json`` attribute was added. @@ -228,7 +266,15 @@ all the metadata in a JSON-compatible form per :PEP:`566`:: Distribution versions --------------------- -The ``version()`` function is the quickest way to get a +.. function:: version(distribution_name) + + Return the installed distribution package version for the named + distribution package. + + Raises :exc:`PackageNotFoundError` if the named distribution + package is not installed in the current Python environment. + +The :func:`!version` function is the quickest way to get a `Distribution Package `_'s version number, as a string:: @@ -241,12 +287,28 @@ number, as a string:: Distribution files ------------------ -You can also get the full set of files contained within a distribution. The -``files()`` function takes a `Distribution Package `_ name -and returns all of the -files installed by this distribution. Each file object returned is a -``PackagePath``, a :class:`pathlib.PurePath` derived object with additional ``dist``, -``size``, and ``hash`` properties as indicated by the metadata. For example:: +.. function:: files(distribution_name) + + Return the full set of files contained within the named + distribution package. + + Raises :exc:`PackageNotFoundError` if the named distribution + package is not installed in the current Python environment. + + Returns :const:`None` if the distribution is found but the installation + database records reporting the files associated with the distribuion package + are missing. + +.. class:: PackagePath + + A :class:`pathlib.PurePath` derived object with additional ``dist``, + ``size``, and ``hash`` properties corresponding to the distribution + package's installation metadata for that file. + +The :func:`!files` function takes a +`Distribution Package `_ +name and returns all of the files installed by this distribution. Each file is reported +as a :class:`PackagePath` instance. For example:: >>> util = [p for p in files('wheel') if 'util.py' in str(p)][0] # doctest: +SKIP >>> util # doctest: +SKIP @@ -269,16 +331,16 @@ Once you have the file, you can also read its contents:: return s.encode('utf-8') return s -You can also use the ``locate`` method to get a the absolute path to the -file:: +You can also use the :meth:`!locate` method to get the absolute +path to the file:: >>> util.locate() # doctest: +SKIP PosixPath('/home/gustav/example/lib/site-packages/wheel/util.py') In the case where the metadata file listing files -(RECORD or SOURCES.txt) is missing, ``files()`` will -return ``None``. The caller may wish to wrap calls to -``files()`` in `always_iterable +(``RECORD`` or ``SOURCES.txt``) is missing, :func:`!files` will +return :const:`None`. The caller may wish to wrap calls to +:func:`!files` in `always_iterable `_ or otherwise guard against this condition if the target distribution is not known to have the metadata present. @@ -288,8 +350,16 @@ distribution is not known to have the metadata present. Distribution requirements ------------------------- +.. function:: requires(distribution_name) + + Return the declared dependency specifiers for the named + distribution package. + + Raises :exc:`PackageNotFoundError` if the named distribution + package is not installed in the current Python environment. + To get the full set of requirements for a `Distribution Package `_, -use the ``requires()`` +use the :func:`!requires` function:: >>> requires('wheel') # doctest: +SKIP @@ -302,6 +372,16 @@ function:: Mapping import to distribution packages --------------------------------------- +.. function:: packages_distributions() + + Return a mapping from the top level module and import package + names found via :attr:`sys.meta_path` to the names of the distribution + packages (if any) that provide the corresponding files. + + To allow for namespace packages (which may have members provided by + multiple distribution packages), each top level import name maps to a + list of distribution names rather than mapping directly to a single name. + A convenience method to resolve the `Distribution Package `_ name (or names, in the case of a namespace package) that provide each importable top-level @@ -321,37 +401,56 @@ function is not reliable with such installs. Distributions ============= -While the above API is the most common and convenient usage, you can get all -of that information from the ``Distribution`` class. A ``Distribution`` is an -abstract object that represents the metadata for -a Python `Distribution Package `_. You can -get the ``Distribution`` instance:: +.. function:: distribution(distribution_name) + + Return a :class:`Distribution` instance describing the named + distribution package. + + Raises :exc:`PackageNotFoundError` if the named distribution + package is not installed in the current Python environment. + +.. class:: Distribution + + Details of an installed distribution package. + + Note: different :class:`!Distribution` instances do not currently compare + equal, even if they relate to the same installed distribution and + accordingly have the same attributes. + +While the module level API described above is the most common and convenient usage, +you can get all of that information from the :class:`!Distribution` class. +:class:`!Distribution` is an abstract object that represents the metadata for +a Python `Distribution Package `_. +You can get the concreate :class:`!Distribution` subclass instance for an installed +distribution package by calling the :func:`distribution` function:: >>> from importlib.metadata import distribution # doctest: +SKIP >>> dist = distribution('wheel') # doctest: +SKIP + >>> type(dist) # doctest: +SKIP + Thus, an alternative way to get the version number is through the -``Distribution`` instance:: +:class:`!Distribution` instance:: >>> dist.version # doctest: +SKIP '0.32.3' -There are all kinds of additional metadata available on the ``Distribution`` -instance:: +There are all kinds of additional metadata available on :class:`!Distribution` +instances:: >>> dist.metadata['Requires-Python'] # doctest: +SKIP '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' >>> dist.metadata['License'] # doctest: +SKIP 'MIT' -For editable packages, an origin property may present :pep:`610` +For editable packages, an ``origin`` property may present :pep:`610` metadata:: >>> dist.origin.url 'file:///path/to/wheel-0.32.3.editable-py3-none-any.whl' The full set of available metadata is not described here. -See the `Core metadata specifications `_ for additional details. +See the PyPA `Core metadata specification `_ for additional details. .. versionadded:: 3.13 The ``.origin`` property was added. @@ -367,21 +466,26 @@ This metadata finder search defaults to ``sys.path``, but varies slightly in how - ``importlib.metadata`` will incidentally honor :py:class:`pathlib.Path` objects on ``sys.path`` even though such values will be ignored for imports. -Extending the search algorithm -============================== +Implementing Custom Providers +============================= + +``importlib.metadata`` address two API surfaces, one for *consumers* +and another for *providers*. Most users are consumers, consuming +metadata provided by the packages. There are other use-cases, however, +where users wish to expose metadata through some other mechanism, +such as alongside a custom importer. Such a use case calls for a +*custom provider*. Because `Distribution Package `_ metadata is not available through :data:`sys.path` searches, or package loaders directly, the metadata for a distribution is found through import -system :ref:`finders `. To find a distribution package's metadata, +system :ref:`finders `. To find a distribution package's metadata, ``importlib.metadata`` queries the list of :term:`meta path finders ` on :data:`sys.meta_path`. -By default ``importlib.metadata`` installs a finder for distribution packages -found on the file system. -This finder doesn't actually find any *distributions*, -but it can find their metadata. +The implementation has hooks integrated into the ``PathFinder``, +serving metadata for distribution packages found on the file system. The abstract class :py:class:`importlib.abc.MetaPathFinder` defines the interface expected of finders by Python's import system. @@ -392,21 +496,98 @@ interface expected of finders by Python's import system. method:: @abc.abstractmethod - def find_distributions(context=DistributionFinder.Context()): + def find_distributions(context=DistributionFinder.Context()) -> Iterable[Distribution]: """Return an iterable of all Distribution instances capable of loading the metadata for packages for the indicated ``context``. """ The ``DistributionFinder.Context`` object provides ``.path`` and ``.name`` properties indicating the path to search and name to match and may -supply other relevant context. +supply other relevant context sought by the consumer. -What this means in practice is that to support finding distribution package +In practice, to support finding distribution package metadata in locations other than the file system, subclass ``Distribution`` and implement the abstract methods. Then from a custom finder, return instances of this derived ``Distribution`` in the ``find_distributions()`` method. +Example +------- + +Imagine a custom finder that loads Python modules from a database:: + + class DatabaseImporter(importlib.abc.MetaPathFinder): + def __init__(self, db): + self.db = db + + def find_spec(self, fullname, target=None) -> ModuleSpec: + return self.db.spec_from_name(fullname) + + sys.meta_path.append(DatabaseImporter(connect_db(...))) + +That importer now presumably provides importable modules from a +database, but it provides no metadata or entry points. For this +custom importer to provide metadata, it would also need to implement +``DistributionFinder``:: + + from importlib.metadata import DistributionFinder + + class DatabaseImporter(DistributionFinder): + ... + + def find_distributions(self, context=DistributionFinder.Context()): + query = dict(name=context.name) if context.name else {} + for dist_record in self.db.query_distributions(query): + yield DatabaseDistribution(dist_record) + +In this way, ``query_distributions`` would return records for +each distribution served by the database matching the query. For +example, if ``requests-1.0`` is in the database, ``find_distributions`` +would yield a ``DatabaseDistribution`` for ``Context(name='requests')`` +or ``Context(name=None)``. + +For the sake of simplicity, this example ignores ``context.path``\. The +``path`` attribute defaults to ``sys.path`` and is the set of import paths to +be considered in the search. A ``DatabaseImporter`` could potentially function +without any concern for a search path. Assuming the importer does no +partitioning, the "path" would be irrelevant. In order to illustrate the +purpose of ``path``, the example would need to illustrate a more complex +``DatabaseImporter`` whose behavior varied depending on +``sys.path``/``PYTHONPATH``. In that case, the ``find_distributions`` should +honor the ``context.path`` and only yield ``Distribution``\ s pertinent to that +path. + +``DatabaseDistribution``, then, would look something like:: + + class DatabaseDistribution(importlib.metadata.Distribution): + def __init__(self, record): + self.record = record + + def read_text(self, filename): + """ + Read a file like "METADATA" for the current distribution. + """ + if filename == "METADATA": + return f"""Name: {self.record.name} + Version: {self.record.version} + """ + if filename == "entry_points.txt": + return "\n".join( + f"""[{ep.group}]\n{ep.name}={ep.value}""" + for ep in self.record.entry_points) + + def locate_file(self, path): + raise RuntimeError("This distribution has no file system") + +This basic implementation should provide metadata and entry points for +packages served by the ``DatabaseImporter``, assuming that the +``record`` supplies suitable ``.name``, ``.version``, and +``.entry_points`` attributes. + +The ``DatabaseDistribution`` may also provide other metadata files, like +``RECORD`` (required for ``Distribution.files``) or override the +implementation of ``Distribution.files``. See the source for more inspiration. + .. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points .. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api diff --git a/Doc/library/importlib.resources.abc.rst b/Doc/library/importlib.resources.abc.rst index c508b6ba965cc0..4085bdf6598d98 100644 --- a/Doc/library/importlib.resources.abc.rst +++ b/Doc/library/importlib.resources.abc.rst @@ -1,5 +1,5 @@ -:mod:`importlib.resources.abc` -- Abstract base classes for resources ---------------------------------------------------------------------- +:mod:`!importlib.resources.abc` -- Abstract base classes for resources +---------------------------------------------------------------------- .. module:: importlib.resources.abc :synopsis: Abstract base classes for resources @@ -22,7 +22,7 @@ something like a data file that lives next to the ``__init__.py`` file of the package. The purpose of this class is to help abstract out the accessing of such data files so that it does not matter if - the package and its data file(s) are stored in a e.g. zip file + the package and its data file(s) are stored e.g. in a zip file versus on the file system. For any of methods of this class, a *resource* argument is @@ -43,7 +43,7 @@ :const:`None`. An object compatible with this ABC should only be returned when the specified module is a package. - .. deprecated-removed:: 3.12 3.14 + .. deprecated:: 3.12 Use :class:`importlib.resources.abc.TraversableResources` instead. .. abstractmethod:: open_resource(resource) @@ -103,19 +103,41 @@ .. abstractmethod:: is_dir() - Return True if self is a directory. + Return ``True`` if self is a directory. .. abstractmethod:: is_file() - Return True if self is a file. + Return ``True`` if self is a file. - .. abstractmethod:: joinpath(child) + .. abstractmethod:: joinpath(*pathsegments) - Return Traversable child in self. + Traverse directories according to *pathsegments* and return + the result as :class:`!Traversable`. + + Each *pathsegments* argument may contain multiple names separated by + forward slashes (``/``, ``posixpath.sep`` ). + For example, the following are equivalent:: + + files.joinpath('subdir', 'subsuddir', 'file.txt') + files.joinpath('subdir/subsuddir/file.txt') + + Note that some :class:`!Traversable` implementations + might not be updated to the latest version of the protocol. + For compatibility with such implementations, provide a single argument + without path separators to each call to ``joinpath``. For example:: + + files.joinpath('subdir').joinpath('subsubdir').joinpath('file.txt') + + .. versionchanged:: 3.11 + + ``joinpath`` accepts multiple *pathsegments*, and these segments + may contain forward slashes as path separators. + Previously, only a single *child* argument was accepted. .. abstractmethod:: __truediv__(child) Return Traversable child in self. + Equivalent to ``joinpath(child)``. .. abstractmethod:: open(mode='r', *args, **kwargs) diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index a5adf0b8546dbf..e002198899c8b8 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -1,5 +1,5 @@ -:mod:`importlib.resources` -- Package resource reading, opening and access --------------------------------------------------------------------------- +:mod:`!importlib.resources` -- Package resource reading, opening and access +--------------------------------------------------------------------------- .. module:: importlib.resources :synopsis: Package resource reading, opening, and access @@ -97,3 +97,181 @@ for example, a package and its resources can be imported from a zip file using .. versionchanged:: 3.12 Added support for *traversable* representing a directory. + + +.. _importlib_resources_functional: + +Functional API +^^^^^^^^^^^^^^ + +A set of simplified, backwards-compatible helpers is available. +These allow common operations in a single function call. + +For all the following functions: + +- *anchor* is an :class:`~importlib.resources.Anchor`, + as in :func:`~importlib.resources.files`. + Unlike in ``files``, it may not be omitted. + +- *path_names* are components of a resource's path name, relative to + the anchor. + For example, to get the text of resource named ``info.txt``, use:: + + importlib.resources.read_text(my_module, "info.txt") + + Like :meth:`Traversable.joinpath `, + The individual components should use forward slashes (``/``) + as path separators. + For example, the following are equivalent:: + + importlib.resources.read_binary(my_module, "pics/painting.png") + importlib.resources.read_binary(my_module, "pics", "painting.png") + + For backward compatibility reasons, functions that read text require + an explicit *encoding* argument if multiple *path_names* are given. + For example, to get the text of ``info/chapter1.txt``, use:: + + importlib.resources.read_text(my_module, "info", "chapter1.txt", + encoding='utf-8') + +.. function:: open_binary(anchor, *path_names) + + Open the named resource for binary reading. + + See :ref:`the introduction ` for + details on *anchor* and *path_names*. + + This function returns a :class:`~typing.BinaryIO` object, + that is, a binary stream open for reading. + + This function is roughly equivalent to:: + + files(anchor).joinpath(*path_names).open('rb') + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + + +.. function:: open_text(anchor, *path_names, encoding='utf-8', errors='strict') + + Open the named resource for text reading. + By default, the contents are read as strict UTF-8. + + See :ref:`the introduction ` for + details on *anchor* and *path_names*. + *encoding* and *errors* have the same meaning as in built-in :func:`open`. + + For backward compatibility reasons, the *encoding* argument must be given + explicitly if there are multiple *path_names*. + This limitation is scheduled to be removed in Python 3.15. + + This function returns a :class:`~typing.TextIO` object, + that is, a text stream open for reading. + + This function is roughly equivalent to:: + + files(anchor).joinpath(*path_names).open('r', encoding=encoding) + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + *encoding* and *errors* must be given as keyword arguments. + + +.. function:: read_binary(anchor, *path_names) + + Read and return the contents of the named resource as :class:`bytes`. + + See :ref:`the introduction ` for + details on *anchor* and *path_names*. + + This function is roughly equivalent to:: + + files(anchor).joinpath(*path_names).read_bytes() + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + + +.. function:: read_text(anchor, *path_names, encoding='utf-8', errors='strict') + + Read and return the contents of the named resource as :class:`str`. + By default, the contents are read as strict UTF-8. + + See :ref:`the introduction ` for + details on *anchor* and *path_names*. + *encoding* and *errors* have the same meaning as in built-in :func:`open`. + + For backward compatibility reasons, the *encoding* argument must be given + explicitly if there are multiple *path_names*. + This limitation is scheduled to be removed in Python 3.15. + + This function is roughly equivalent to:: + + files(anchor).joinpath(*path_names).read_text(encoding=encoding) + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + *encoding* and *errors* must be given as keyword arguments. + + +.. function:: path(anchor, *path_names) + + Provides the path to the *resource* as an actual file system path. This + function returns a context manager for use in a :keyword:`with` statement. + The context manager provides a :class:`pathlib.Path` object. + + Exiting the context manager cleans up any temporary files created, e.g. + when the resource needs to be extracted from a zip file. + + For example, the :meth:`~pathlib.Path.stat` method requires + an actual file system path; it can be used like this:: + + with importlib.resources.path(anchor, "resource.txt") as fspath: + result = fspath.stat() + + See :ref:`the introduction ` for + details on *anchor* and *path_names*. + + This function is roughly equivalent to:: + + as_file(files(anchor).joinpath(*path_names)) + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + *encoding* and *errors* must be given as keyword arguments. + + +.. function:: is_resource(anchor, *path_names) + + Return ``True`` if the named resource exists, otherwise ``False``. + This function does not consider directories to be resources. + + See :ref:`the introduction ` for + details on *anchor* and *path_names*. + + This function is roughly equivalent to:: + + files(anchor).joinpath(*path_names).is_file() + + .. versionchanged:: 3.13 + Multiple *path_names* are accepted. + + +.. function:: contents(anchor, *path_names) + + Return an iterable over the named items within the package or path. + The iterable returns names of resources (e.g. files) and non-resources + (e.g. directories) as :class:`str`. + The iterable does not recurse into subdirectories. + + See :ref:`the introduction ` for + details on *anchor* and *path_names*. + + This function is roughly equivalent to:: + + for resource in files(anchor).joinpath(*path_names).iterdir(): + yield resource.name + + .. deprecated:: 3.11 + Prefer ``iterdir()`` as above, which offers more control over the + results and richer functionality. diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index d92bb2f8e5cf83..b935fc0e42a4bd 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -249,7 +249,7 @@ ABC hierarchy:: An abstract method for finding a :term:`spec ` for the specified module. If this is a top-level import, *path* will be ``None``. Otherwise, this is a search for a subpackage or - module and *path* will be the value of :attr:`__path__` from the + module and *path* will be the value of :attr:`~module.__path__` from the parent package. If a spec cannot be found, ``None`` is returned. When passed in, ``target`` is a module object that the finder may use to make a more educated guess about what spec to return. @@ -355,34 +355,12 @@ ABC hierarchy:: (note that some of these attributes can change when a module is reloaded): - - :attr:`__name__` - The module's fully qualified name. - It is ``'__main__'`` for an executed module. - - - :attr:`__file__` - The location the :term:`loader` used to load the module. - For example, for modules loaded from a .py file this is the filename. - It is not set on all modules (e.g. built-in modules). - - - :attr:`__cached__` - The filename of a compiled version of the module's code. - It is not set on all modules (e.g. built-in modules). - - - :attr:`__path__` - The list of locations where the package's submodules will be found. - Most of the time this is a single directory. - The import system passes this attribute to ``__import__()`` and to finders - in the same way as :data:`sys.path` but just for the package. - It is not set on non-package modules so it can be used - as an indicator that the module is a package. - - - :attr:`__package__` - The fully qualified name of the package the module is in (or the - empty string for a top-level module). - If the module is a package then this is the same as :attr:`__name__`. - - - :attr:`__loader__` - The :term:`loader` used to load the module. + - :attr:`module.__name__` + - :attr:`module.__file__` + - :attr:`module.__cached__` *(deprecated)* + - :attr:`module.__path__` + - :attr:`module.__package__` *(deprecated)* + - :attr:`module.__loader__` *(deprecated)* When :meth:`exec_module` is available then backwards-compatible functionality is provided. @@ -402,13 +380,15 @@ ABC hierarchy:: .. class:: ResourceLoader + *Superseded by TraversableResources* + An abstract base class for a :term:`loader` which implements the optional :pep:`302` protocol for loading arbitrary resources from the storage back-end. .. deprecated:: 3.7 This ABC is deprecated in favour of supporting resource loading - through :class:`importlib.resources.abc.ResourceReader`. + through :class:`importlib.resources.abc.TraversableResources`. .. abstractmethod:: get_data(path) @@ -418,7 +398,8 @@ ABC hierarchy:: can implement this abstract method to give direct access to the data stored. :exc:`OSError` is to be raised if the *path* cannot be found. The *path* is expected to be constructed using a module's - :attr:`__file__` attribute or an item from a package's :attr:`__path__`. + :attr:`~module.__file__` attribute or an item from a package's + :attr:`~module.__path__`. .. versionchanged:: 3.4 Raises :exc:`OSError` instead of :exc:`NotImplementedError`. @@ -505,9 +486,9 @@ ABC hierarchy:: .. abstractmethod:: get_filename(fullname) - An abstract method that is to return the value of :attr:`__file__` for - the specified module. If no path is available, :exc:`ImportError` is - raised. + An abstract method that is to return the value of + :attr:`~module.__file__` for the specified module. If no path is + available, :exc:`ImportError` is raised. If source code is available, then the method should return the path to the source file, regardless of whether a bytecode was used to load the @@ -657,7 +638,7 @@ ABC hierarchy:: something like a data file that lives next to the ``__init__.py`` file of the package. The purpose of this class is to help abstract out the accessing of such data files so that it does not matter if - the package and its data file(s) are stored in a e.g. zip file + the package and its data file(s) are stored e.g. in a zip file versus on the file system. For any of methods of this class, a *resource* argument is @@ -1166,79 +1147,137 @@ find and load modules. .. class:: ModuleSpec(name, loader, *, origin=None, loader_state=None, is_package=None) A specification for a module's import-system-related state. This is - typically exposed as the module's :attr:`__spec__` attribute. In the - descriptions below, the names in parentheses give the corresponding - attribute available directly on the module object, - e.g. ``module.__spec__.origin == module.__file__``. Note, however, that + typically exposed as the module's :attr:`~module.__spec__` attribute. Many + of these attributes are also available directly on a module: for example, + ``module.__spec__.origin == module.__file__``. Note, however, that while the *values* are usually equivalent, they can differ since there is - no synchronization between the two objects. For example, it is possible to update - the module's :attr:`__file__` at runtime and this will not be automatically - reflected in the module's :attr:`__spec__.origin`, and vice versa. + no synchronization between the two objects. For example, it is possible to + update the module's :attr:`~module.__file__` at runtime and this will not be + automatically reflected in the module's + :attr:`__spec__.origin `, and vice versa. .. versionadded:: 3.4 .. attribute:: name - (:attr:`__name__`) - - The module's fully qualified name. - The :term:`finder` should always set this attribute to a non-empty string. + The module's fully qualified name (see :attr:`module.__name__`). + The :term:`finder` should always set this attribute to a non-empty string. .. attribute:: loader - (:attr:`__loader__`) - - The :term:`loader` used to load the module. - The :term:`finder` should always set this attribute. + The :term:`loader` used to load the module (see :attr:`module.__loader__`). + The :term:`finder` should always set this attribute. .. attribute:: origin - (:attr:`__file__`) - - The location the :term:`loader` should use to load the module. - For example, for modules loaded from a .py file this is the filename. - The :term:`finder` should always set this attribute to a meaningful value - for the :term:`loader` to use. In the uncommon case that there is not one - (like for namespace packages), it should be set to ``None``. + The location the :term:`loader` should use to load the module + (see :attr:`module.__file__`). + For example, for modules loaded from a ``.py`` file this is the filename. + The :term:`finder` should always set this attribute to a meaningful value + for the :term:`loader` to use. In the uncommon case that there is not one + (like for namespace packages), it should be set to ``None``. .. attribute:: submodule_search_locations - (:attr:`__path__`) + A (possibly empty) :term:`sequence` of strings enumerating the locations + in which a package's submodules will be found + (see :attr:`module.__path__`). Most of the time there will only be a + single directory in this list. - The list of locations where the package's submodules will be found. - Most of the time this is a single directory. - The :term:`finder` should set this attribute to a list, even an empty one, to indicate - to the import system that the module is a package. It should be set to ``None`` for - non-package modules. It is set automatically later to a special object for - namespace packages. + The :term:`finder` should set this attribute to a sequence, even an empty + one, to indicate + to the import system that the module is a package. It should be set to ``None`` for + non-package modules. It is set automatically later to a special object for + namespace packages. .. attribute:: loader_state - The :term:`finder` may set this attribute to an object containing additional, - module-specific data to use when loading the module. Otherwise it should be - set to ``None``. + The :term:`finder` may set this attribute to an object containing additional, + module-specific data to use when loading the module. Otherwise it should be + set to ``None``. .. attribute:: cached - (:attr:`__cached__`) - - The filename of a compiled version of the module's code. - The :term:`finder` should always set this attribute but it may be ``None`` - for modules that do not need compiled code stored. + The filename of a compiled version of the module's code + (see :attr:`module.__cached__`). + The :term:`finder` should always set this attribute but it may be ``None`` + for modules that do not need compiled code stored. .. attribute:: parent - (:attr:`__package__`) - - (Read-only) The fully qualified name of the package the module is in (or the - empty string for a top-level module). - If the module is a package then this is the same as :attr:`name`. + (Read-only) The fully qualified name of the package the module is in (or the + empty string for a top-level module). + See :attr:`module.__package__`. + If the module is a package then this is the same as :attr:`name`. .. attribute:: has_location - ``True`` if the spec's :attr:`origin` refers to a loadable location, - ``False`` otherwise. This value impacts how :attr:`origin` is interpreted - and how the module's :attr:`__file__` is populated. + ``True`` if the spec's :attr:`origin` refers to a loadable location, + ``False`` otherwise. This value impacts how :attr:`!origin` is interpreted + and how the module's :attr:`~module.__file__` is populated. + + +.. class:: AppleFrameworkLoader(name, path) + + A specialization of :class:`importlib.machinery.ExtensionFileLoader` that + is able to load extension modules in Framework format. + + For compatibility with the iOS App Store, *all* binary modules in an iOS app + must be dynamic libraries, contained in a framework with appropriate + metadata, stored in the ``Frameworks`` folder of the packaged app. There can + be only a single binary per framework, and there can be no executable binary + material outside the Frameworks folder. + + To accommodate this requirement, when running on iOS, extension module + binaries are *not* packaged as ``.so`` files on ``sys.path``, but as + individual standalone frameworks. To discover those frameworks, this loader + is be registered against the ``.fwork`` file extension, with a ``.fwork`` + file acting as a placeholder in the original location of the binary on + ``sys.path``. The ``.fwork`` file contains the path of the actual binary in + the ``Frameworks`` folder, relative to the app bundle. To allow for + resolving a framework-packaged binary back to the original location, the + framework is expected to contain a ``.origin`` file that contains the + location of the ``.fwork`` file, relative to the app bundle. + + For example, consider the case of an import ``from foo.bar import _whiz``, + where ``_whiz`` is implemented with the binary module + ``sources/foo/bar/_whiz.abi3.so``, with ``sources`` being the location + registered on ``sys.path``, relative to the application bundle. This module + *must* be distributed as + ``Frameworks/foo.bar._whiz.framework/foo.bar._whiz`` (creating the framework + name from the full import path of the module), with an ``Info.plist`` file + in the ``.framework`` directory identifying the binary as a framework. The + ``foo.bar._whiz`` module would be represented in the original location with + a ``sources/foo/bar/_whiz.abi3.fwork`` marker file, containing the path + ``Frameworks/foo.bar._whiz/foo.bar._whiz``. The framework would also contain + ``Frameworks/foo.bar._whiz.framework/foo.bar._whiz.origin``, containing the + path to the ``.fwork`` file. + + When a module is loaded with this loader, the ``__file__`` for the module + will report as the location of the ``.fwork`` file. This allows code to use + the ``__file__`` of a module as an anchor for file system traversal. + However, the spec origin will reference the location of the *actual* binary + in the ``.framework`` folder. + + The Xcode project building the app is responsible for converting any ``.so`` + files from wherever they exist in the ``PYTHONPATH`` into frameworks in the + ``Frameworks`` folder (including stripping extensions from the module file, + the addition of framework metadata, and signing the resulting framework), + and creating the ``.fwork`` and ``.origin`` files. This will usually be done + with a build step in the Xcode project; see the iOS documentation for + details on how to construct this build step. + + .. versionadded:: 3.13 + + .. availability:: iOS. + + .. attribute:: name + + Name of the module the loader supports. + + .. attribute:: path + + Path to the ``.fwork`` file for the extension module. :mod:`importlib.util` -- Utility code for importers @@ -1360,8 +1399,8 @@ an :term:`importer`. .. versionchanged:: 3.7 Raises :exc:`ModuleNotFoundError` instead of :exc:`AttributeError` if - **package** is in fact not a package (i.e. lacks a :attr:`__path__` - attribute). + **package** is in fact not a package (i.e. lacks a + :attr:`~module.__path__` attribute). .. function:: module_from_spec(spec) @@ -1521,20 +1560,34 @@ Note that if ``name`` is a submodule (contains a dot), Importing a source file directly '''''''''''''''''''''''''''''''' -To import a Python source file directly, use the following recipe:: +This recipe should be used with caution: it is an approximation of an import +statement where the file path is specified directly, rather than +:data:`sys.path` being searched. Alternatives should first be considered first, +such as modifying :data:`sys.path` when a proper module is required, or using +:func:`runpy.run_path` when the global namespace resulting from running a Python +file is appropriate. - import importlib.util - import sys +To import a Python source file directly from a path, use the following recipe:: - # For illustrative purposes. - import tokenize - file_path = tokenize.__file__ - module_name = tokenize.__name__ + import importlib.util + import sys + + + def import_from_path(module_name, file_path): + spec = importlib.util.spec_from_file_location(module_name, file_path) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module - spec = importlib.util.spec_from_file_location(module_name, file_path) - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - spec.loader.exec_module(module) + + # For illustrative purposes only (use of `json` is arbitrary). + import json + file_path = json.__file__ + module_name = json.__name__ + + # Similar outcome as `import json`. + json = import_from_path(module_name, file_path) Implementing lazy imports @@ -1560,7 +1613,6 @@ The example below shows how to implement lazy imports:: False - Setting up an importer '''''''''''''''''''''' diff --git a/Doc/library/index.rst b/Doc/library/index.rst index 0b348ae6f5c8c0..44b218948d07e1 100644 --- a/Doc/library/index.rst +++ b/Doc/library/index.rst @@ -55,6 +55,7 @@ the `Python Package Index `_. fileformats.rst crypto.rst allos.rst + cmdlinelibs.rst concurrency.rst ipc.rst netdata.rst @@ -75,4 +76,5 @@ the `Python Package Index `_. unix.rst cmdline.rst superseded.rst + removed.rst security_warnings.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index ed8d705da3b0b5..544efed1a76b96 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -1,5 +1,5 @@ -:mod:`inspect` --- Inspect live objects -======================================= +:mod:`!inspect` --- Inspect live objects +======================================== .. testsetup:: * @@ -42,210 +42,264 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. this function name is too big to fit in the ascii-art table below .. |coroutine-origin-link| replace:: :func:`sys.set_coroutine_origin_tracking_depth` -+-----------+-------------------+---------------------------+ -| Type | Attribute | Description | -+===========+===================+===========================+ -| class | __doc__ | documentation string | -+-----------+-------------------+---------------------------+ -| | __name__ | name with which this | -| | | class was defined | -+-----------+-------------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-------------------+---------------------------+ -| | __module__ | name of module in which | -| | | this class was defined | -+-----------+-------------------+---------------------------+ -| method | __doc__ | documentation string | -+-----------+-------------------+---------------------------+ -| | __name__ | name with which this | -| | | method was defined | -+-----------+-------------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-------------------+---------------------------+ -| | __func__ | function object | -| | | containing implementation | -| | | of method | -+-----------+-------------------+---------------------------+ -| | __self__ | instance to which this | -| | | method is bound, or | -| | | ``None`` | -+-----------+-------------------+---------------------------+ -| | __module__ | name of module in which | -| | | this method was defined | -+-----------+-------------------+---------------------------+ -| function | __doc__ | documentation string | -+-----------+-------------------+---------------------------+ -| | __name__ | name with which this | -| | | function was defined | -+-----------+-------------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-------------------+---------------------------+ -| | __code__ | code object containing | -| | | compiled function | -| | | :term:`bytecode` | -+-----------+-------------------+---------------------------+ -| | __defaults__ | tuple of any default | -| | | values for positional or | -| | | keyword parameters | -+-----------+-------------------+---------------------------+ -| | __kwdefaults__ | mapping of any default | -| | | values for keyword-only | -| | | parameters | -+-----------+-------------------+---------------------------+ -| | __globals__ | global namespace in which | -| | | this function was defined | -+-----------+-------------------+---------------------------+ -| | __builtins__ | builtins namespace | -+-----------+-------------------+---------------------------+ -| | __annotations__ | mapping of parameters | -| | | names to annotations; | -| | | ``"return"`` key is | -| | | reserved for return | -| | | annotations. | -+-----------+-------------------+---------------------------+ -| | __module__ | name of module in which | -| | | this function was defined | -+-----------+-------------------+---------------------------+ -| traceback | tb_frame | frame object at this | -| | | level | -+-----------+-------------------+---------------------------+ -| | tb_lasti | index of last attempted | -| | | instruction in bytecode | -+-----------+-------------------+---------------------------+ -| | tb_lineno | current line number in | -| | | Python source code | -+-----------+-------------------+---------------------------+ -| | tb_next | next inner traceback | -| | | object (called by this | -| | | level) | -+-----------+-------------------+---------------------------+ -| frame | f_back | next outer frame object | -| | | (this frame's caller) | -+-----------+-------------------+---------------------------+ -| | f_builtins | builtins namespace seen | -| | | by this frame | -+-----------+-------------------+---------------------------+ -| | f_code | code object being | -| | | executed in this frame | -+-----------+-------------------+---------------------------+ -| | f_globals | global namespace seen by | -| | | this frame | -+-----------+-------------------+---------------------------+ -| | f_lasti | index of last attempted | -| | | instruction in bytecode | -+-----------+-------------------+---------------------------+ -| | f_lineno | current line number in | -| | | Python source code | -+-----------+-------------------+---------------------------+ -| | f_locals | local namespace seen by | -| | | this frame | -+-----------+-------------------+---------------------------+ -| | f_trace | tracing function for this | -| | | frame, or ``None`` | -+-----------+-------------------+---------------------------+ -| code | co_argcount | number of arguments (not | -| | | including keyword only | -| | | arguments, \* or \*\* | -| | | args) | -+-----------+-------------------+---------------------------+ -| | co_code | string of raw compiled | -| | | bytecode | -+-----------+-------------------+---------------------------+ -| | co_cellvars | tuple of names of cell | -| | | variables (referenced by | -| | | containing scopes) | -+-----------+-------------------+---------------------------+ -| | co_consts | tuple of constants used | -| | | in the bytecode | -+-----------+-------------------+---------------------------+ -| | co_filename | name of file in which | -| | | this code object was | -| | | created | -+-----------+-------------------+---------------------------+ -| | co_firstlineno | number of first line in | -| | | Python source code | -+-----------+-------------------+---------------------------+ -| | co_flags | bitmap of ``CO_*`` flags, | -| | | read more :ref:`here | -| | | `| -+-----------+-------------------+---------------------------+ -| | co_lnotab | encoded mapping of line | -| | | numbers to bytecode | -| | | indices | -+-----------+-------------------+---------------------------+ -| | co_freevars | tuple of names of free | -| | | variables (referenced via | -| | | a function's closure) | -+-----------+-------------------+---------------------------+ -| | co_posonlyargcount| number of positional only | -| | | arguments | -+-----------+-------------------+---------------------------+ -| | co_kwonlyargcount | number of keyword only | -| | | arguments (not including | -| | | \*\* arg) | -+-----------+-------------------+---------------------------+ -| | co_name | name with which this code | -| | | object was defined | -+-----------+-------------------+---------------------------+ -| | co_qualname | fully qualified name with | -| | | which this code object | -| | | was defined | -+-----------+-------------------+---------------------------+ -| | co_names | tuple of names other | -| | | than arguments and | -| | | function locals | -+-----------+-------------------+---------------------------+ -| | co_nlocals | number of local variables | -+-----------+-------------------+---------------------------+ -| | co_stacksize | virtual machine stack | -| | | space required | -+-----------+-------------------+---------------------------+ -| | co_varnames | tuple of names of | -| | | arguments and local | -| | | variables | -+-----------+-------------------+---------------------------+ -| generator | __name__ | name | -+-----------+-------------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-------------------+---------------------------+ -| | gi_frame | frame | -+-----------+-------------------+---------------------------+ -| | gi_running | is the generator running? | -+-----------+-------------------+---------------------------+ -| | gi_code | code | -+-----------+-------------------+---------------------------+ -| | gi_yieldfrom | object being iterated by | -| | | ``yield from``, or | -| | | ``None`` | -+-----------+-------------------+---------------------------+ -| coroutine | __name__ | name | -+-----------+-------------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-------------------+---------------------------+ -| | cr_await | object being awaited on, | -| | | or ``None`` | -+-----------+-------------------+---------------------------+ -| | cr_frame | frame | -+-----------+-------------------+---------------------------+ -| | cr_running | is the coroutine running? | -+-----------+-------------------+---------------------------+ -| | cr_code | code | -+-----------+-------------------+---------------------------+ -| | cr_origin | where coroutine was | -| | | created, or ``None``. See | -| | | |coroutine-origin-link| | -+-----------+-------------------+---------------------------+ -| builtin | __doc__ | documentation string | -+-----------+-------------------+---------------------------+ -| | __name__ | original name of this | -| | | function or method | -+-----------+-------------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-------------------+---------------------------+ -| | __self__ | instance to which a | -| | | method is bound, or | -| | | ``None`` | -+-----------+-------------------+---------------------------+ ++-----------------+-------------------+---------------------------+ +| Type | Attribute | Description | ++=================+===================+===========================+ +| class | __doc__ | documentation string | ++-----------------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | class was defined | ++-----------------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------------+-------------------+---------------------------+ +| | __module__ | name of module in which | +| | | this class was defined | ++-----------------+-------------------+---------------------------+ +| | __type_params__ | A tuple containing the | +| | | :ref:`type parameters | +| | | ` of | +| | | a generic class | ++-----------------+-------------------+---------------------------+ +| method | __doc__ | documentation string | ++-----------------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | method was defined | ++-----------------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------------+-------------------+---------------------------+ +| | __func__ | function object | +| | | containing implementation | +| | | of method | ++-----------------+-------------------+---------------------------+ +| | __self__ | instance to which this | +| | | method is bound, or | +| | | ``None`` | ++-----------------+-------------------+---------------------------+ +| | __module__ | name of module in which | +| | | this method was defined | ++-----------------+-------------------+---------------------------+ +| function | __doc__ | documentation string | ++-----------------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | function was defined | ++-----------------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------------+-------------------+---------------------------+ +| | __code__ | code object containing | +| | | compiled function | +| | | :term:`bytecode` | ++-----------------+-------------------+---------------------------+ +| | __defaults__ | tuple of any default | +| | | values for positional or | +| | | keyword parameters | ++-----------------+-------------------+---------------------------+ +| | __kwdefaults__ | mapping of any default | +| | | values for keyword-only | +| | | parameters | ++-----------------+-------------------+---------------------------+ +| | __globals__ | global namespace in which | +| | | this function was defined | ++-----------------+-------------------+---------------------------+ +| | __builtins__ | builtins namespace | ++-----------------+-------------------+---------------------------+ +| | __annotations__ | mapping of parameters | +| | | names to annotations; | +| | | ``"return"`` key is | +| | | reserved for return | +| | | annotations. | ++-----------------+-------------------+---------------------------+ +| | __type_params__ | A tuple containing the | +| | | :ref:`type parameters | +| | | ` of | +| | | a generic function | ++-----------------+-------------------+---------------------------+ +| | __module__ | name of module in which | +| | | this function was defined | ++-----------------+-------------------+---------------------------+ +| traceback | tb_frame | frame object at this | +| | | level | ++-----------------+-------------------+---------------------------+ +| | tb_lasti | index of last attempted | +| | | instruction in bytecode | ++-----------------+-------------------+---------------------------+ +| | tb_lineno | current line number in | +| | | Python source code | ++-----------------+-------------------+---------------------------+ +| | tb_next | next inner traceback | +| | | object (called by this | +| | | level) | ++-----------------+-------------------+---------------------------+ +| frame | f_back | next outer frame object | +| | | (this frame's caller) | ++-----------------+-------------------+---------------------------+ +| | f_builtins | builtins namespace seen | +| | | by this frame | ++-----------------+-------------------+---------------------------+ +| | f_code | code object being | +| | | executed in this frame | ++-----------------+-------------------+---------------------------+ +| | f_globals | global namespace seen by | +| | | this frame | ++-----------------+-------------------+---------------------------+ +| | f_lasti | index of last attempted | +| | | instruction in bytecode | ++-----------------+-------------------+---------------------------+ +| | f_lineno | current line number in | +| | | Python source code | ++-----------------+-------------------+---------------------------+ +| | f_locals | local namespace seen by | +| | | this frame | ++-----------------+-------------------+---------------------------+ +| | f_generator | returns the generator or | +| | | coroutine object that | +| | | owns this frame, or | +| | | ``None`` if the frame is | +| | | of a regular function | ++-----------------+-------------------+---------------------------+ +| | f_trace | tracing function for this | +| | | frame, or ``None`` | ++-----------------+-------------------+---------------------------+ +| | f_trace_lines | indicate whether a | +| | | tracing event is | +| | | triggered for each source | +| | | source line | ++-----------------+-------------------+---------------------------+ +| | f_trace_opcodes | indicate whether | +| | | per-opcode events are | +| | | requested | ++-----------------+-------------------+---------------------------+ +| | clear() | used to clear all | +| | | references to local | +| | | variables | ++-----------------+-------------------+---------------------------+ +| code | co_argcount | number of arguments (not | +| | | including keyword only | +| | | arguments, \* or \*\* | +| | | args) | ++-----------------+-------------------+---------------------------+ +| | co_code | string of raw compiled | +| | | bytecode | ++-----------------+-------------------+---------------------------+ +| | co_cellvars | tuple of names of cell | +| | | variables (referenced by | +| | | containing scopes) | ++-----------------+-------------------+---------------------------+ +| | co_consts | tuple of constants used | +| | | in the bytecode | ++-----------------+-------------------+---------------------------+ +| | co_filename | name of file in which | +| | | this code object was | +| | | created | ++-----------------+-------------------+---------------------------+ +| | co_firstlineno | number of first line in | +| | | Python source code | ++-----------------+-------------------+---------------------------+ +| | co_flags | bitmap of ``CO_*`` flags, | +| | | read more :ref:`here | +| | | `| ++-----------------+-------------------+---------------------------+ +| | co_lnotab | encoded mapping of line | +| | | numbers to bytecode | +| | | indices | ++-----------------+-------------------+---------------------------+ +| | co_freevars | tuple of names of free | +| | | variables (referenced via | +| | | a function's closure) | ++-----------------+-------------------+---------------------------+ +| | co_posonlyargcount| number of positional only | +| | | arguments | ++-----------------+-------------------+---------------------------+ +| | co_kwonlyargcount | number of keyword only | +| | | arguments (not including | +| | | \*\* arg) | ++-----------------+-------------------+---------------------------+ +| | co_name | name with which this code | +| | | object was defined | ++-----------------+-------------------+---------------------------+ +| | co_qualname | fully qualified name with | +| | | which this code object | +| | | was defined | ++-----------------+-------------------+---------------------------+ +| | co_names | tuple of names other | +| | | than arguments and | +| | | function locals | ++-----------------+-------------------+---------------------------+ +| | co_nlocals | number of local variables | ++-----------------+-------------------+---------------------------+ +| | co_stacksize | virtual machine stack | +| | | space required | ++-----------------+-------------------+---------------------------+ +| | co_varnames | tuple of names of | +| | | arguments and local | +| | | variables | ++-----------------+-------------------+---------------------------+ +| | co_lines() | returns an iterator that | +| | | yields successive | +| | | bytecode ranges | ++-----------------+-------------------+---------------------------+ +| | co_positions() | returns an iterator of | +| | | source code positions for | +| | | each bytecode instruction | ++-----------------+-------------------+---------------------------+ +| | replace() | returns a copy of the | +| | | code object with new | +| | | values | ++-----------------+-------------------+---------------------------+ +| generator | __name__ | name | ++-----------------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------------+-------------------+---------------------------+ +| | gi_frame | frame | ++-----------------+-------------------+---------------------------+ +| | gi_running | is the generator running? | ++-----------------+-------------------+---------------------------+ +| | gi_code | code | ++-----------------+-------------------+---------------------------+ +| | gi_yieldfrom | object being iterated by | +| | | ``yield from``, or | +| | | ``None`` | ++-----------------+-------------------+---------------------------+ +| async generator | __name__ | name | ++-----------------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------------+-------------------+---------------------------+ +| | ag_await | object being awaited on, | +| | | or ``None`` | ++-----------------+-------------------+---------------------------+ +| | ag_frame | frame | ++-----------------+-------------------+---------------------------+ +| | ag_running | is the generator running? | ++-----------------+-------------------+---------------------------+ +| | ag_code | code | ++-----------------+-------------------+---------------------------+ +| coroutine | __name__ | name | ++-----------------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------------+-------------------+---------------------------+ +| | cr_await | object being awaited on, | +| | | or ``None`` | ++-----------------+-------------------+---------------------------+ +| | cr_frame | frame | ++-----------------+-------------------+---------------------------+ +| | cr_running | is the coroutine running? | ++-----------------+-------------------+---------------------------+ +| | cr_code | code | ++-----------------+-------------------+---------------------------+ +| | cr_origin | where coroutine was | +| | | created, or ``None``. See | +| | | |coroutine-origin-link| | ++-----------------+-------------------+---------------------------+ +| builtin | __doc__ | documentation string | ++-----------------+-------------------+---------------------------+ +| | __name__ | original name of this | +| | | function or method | ++-----------------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------------+-------------------+---------------------------+ +| | __self__ | instance to which a | +| | | method is bound, or | +| | | ``None`` | ++-----------------+-------------------+---------------------------+ .. versionchanged:: 3.5 @@ -262,6 +316,10 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Add ``__builtins__`` attribute to functions. +.. versionchanged:: next + + Add ``f_generator`` attribute to frames. + .. function:: getmembers(object[, predicate]) Return all the members of an object in a list of ``(name, value)`` @@ -326,6 +384,13 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object is a bound method written in Python. +.. function:: ispackage(object) + + Return ``True`` if the object is a :term:`package`. + + .. versionadded:: 3.14 + + .. function:: isfunction(object) Return ``True`` if the object is a Python function, which includes functions @@ -434,7 +499,7 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. versionchanged:: 3.8 Functions wrapped in :func:`functools.partial` now return ``True`` if the - wrapped function is a :term:`asynchronous generator` function. + wrapped function is an :term:`asynchronous generator` function. .. versionchanged:: 3.13 Functions wrapped in :func:`functools.partialmethod` now return ``True`` @@ -494,10 +559,10 @@ attributes (see :ref:`import-mod-attrs` for module attributes): are true. This, for example, is true of ``int.__add__``. An object passing this test - has a :meth:`~object.__get__` method but not a :meth:`~object.__set__` - method, but beyond that the set of attributes varies. A - :attr:`~definition.__name__` attribute is usually - sensible, and :attr:`!__doc__` often is. + has a :meth:`~object.__get__` method, but not a :meth:`~object.__set__` + method or a :meth:`~object.__delete__` method. Beyond that, the set of + attributes varies. A :attr:`~definition.__name__` attribute is usually + sensible, and :attr:`~definition.__doc__` often is. Methods implemented via descriptors that also pass one of the other tests return ``False`` from the :func:`ismethoddescriptor` test, simply because the @@ -505,6 +570,11 @@ attributes (see :ref:`import-mod-attrs` for module attributes): :attr:`~method.__func__` attribute (etc) when an object passes :func:`ismethod`. + .. versionchanged:: 3.13 + This function no longer incorrectly reports objects with :meth:`~object.__get__` + and :meth:`~object.__delete__`, but not :meth:`~object.__set__`, as being method + descriptors (such objects are data descriptors, not method descriptors). + .. function:: isdatadescriptor(object) @@ -641,7 +711,7 @@ and its return annotation. To retrieve a :class:`!Signature` object, use the :func:`!signature` function. -.. function:: signature(callable, *, follow_wrapped=True, globals=None, locals=None, eval_str=False) +.. function:: signature(callable, *, follow_wrapped=True, globals=None, locals=None, eval_str=False, annotation_format=Format.VALUE) Return a :class:`Signature` object for the given *callable*: @@ -665,22 +735,27 @@ function. Accepts a wide range of Python callables, from plain functions and classes to :func:`functools.partial` objects. - For objects defined in modules using stringized annotations - (``from __future__ import annotations``), :func:`signature` will + If some of the annotations are strings (e.g., because + ``from __future__ import annotations`` was used), :func:`signature` will attempt to automatically un-stringize the annotations using - :func:`get_annotations`. The + :func:`annotationlib.get_annotations`. The *globals*, *locals*, and *eval_str* parameters are passed - into :func:`get_annotations` when resolving the - annotations; see the documentation for :func:`get_annotations` - for instructions on how to use these parameters. + into :func:`!annotationlib.get_annotations` when resolving the + annotations; see the documentation for :func:`!annotationlib.get_annotations` + for instructions on how to use these parameters. A member of the + :class:`annotationlib.Format` enum can be passed to the + *annotation_format* parameter to control the format of the returned + annotations. For example, use + ``annotation_format=annotationlib.Format.STRING`` to return annotations in string + format. Raises :exc:`ValueError` if no signature can be provided, and :exc:`TypeError` if that type of object is not supported. Also, if the annotations are stringized, and *eval_str* is not false, - the ``eval()`` call(s) to un-stringize the annotations in :func:`get_annotations` + the ``eval()`` call(s) to un-stringize the annotations in :func:`annotationlib.get_annotations` could potentially raise any kind of exception. - A slash(/) in the signature of a function denotes that the parameters prior + A slash (/) in the signature of a function denotes that the parameters prior to it are positional-only. For more info, see :ref:`the FAQ entry on positional-only parameters `. @@ -693,6 +768,9 @@ function. .. versionchanged:: 3.10 The *globals*, *locals*, and *eval_str* parameters were added. + .. versionchanged:: 3.14 + The *annotation_format* parameter was added. + .. note:: Some callables may not be introspectable in certain implementations of @@ -785,7 +863,7 @@ function. :class:`Signature` objects are also supported by the generic function :func:`copy.replace`. - .. method:: format(*, max_width=None) + .. method:: format(*, max_width=None, quote_annotation_strings=True) Create a string representation of the :class:`Signature` object. @@ -794,8 +872,17 @@ function. If the signature is longer than *max_width*, all parameters will be on separate lines. + If *quote_annotation_strings* is False, :term:`annotations ` + in the signature are displayed without opening and closing quotation + marks if they are strings. This is useful if the signature was created with the + :attr:`~annotationlib.Format.STRING` format or if + ``from __future__ import annotations`` was used. + .. versionadded:: 3.13 + .. versionchanged:: 3.14 + The *unquote_annotations* parameter was added. + .. classmethod:: Signature.from_callable(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=False) Return a :class:`Signature` (or its subclass) object for a given callable @@ -910,7 +997,7 @@ function. .. attribute:: Parameter.kind.description - Describes a enum value of :attr:`Parameter.kind`. + Describes an enum value of :attr:`Parameter.kind`. .. versionadded:: 3.8 @@ -990,7 +1077,8 @@ function. .. attribute:: BoundArguments.kwargs A dict of keyword arguments values. Dynamically computed from the - :attr:`arguments` attribute. + :attr:`arguments` attribute. Arguments that can be passed positionally + are included in :attr:`args` instead. .. attribute:: BoundArguments.signature @@ -1194,62 +1282,19 @@ Classes and functions .. versionadded:: 3.4 -.. function:: get_annotations(obj, *, globals=None, locals=None, eval_str=False) +.. function:: get_annotations(obj, *, globals=None, locals=None, eval_str=False, format=annotationlib.Format.VALUE) Compute the annotations dict for an object. - ``obj`` may be a callable, class, or module. - Passing in an object of any other type raises :exc:`TypeError`. - - Returns a dict. ``get_annotations()`` returns a new dict every time - it's called; calling it twice on the same object will return two - different but equivalent dicts. - - This function handles several details for you: - - * If ``eval_str`` is true, values of type ``str`` will - be un-stringized using :func:`eval()`. This is intended - for use with stringized annotations - (``from __future__ import annotations``). - * If ``obj`` doesn't have an annotations dict, returns an - empty dict. (Functions and methods always have an - annotations dict; classes, modules, and other types of - callables may not.) - * Ignores inherited annotations on classes. If a class - doesn't have its own annotations dict, returns an empty dict. - * All accesses to object members and dict values are done - using ``getattr()`` and ``dict.get()`` for safety. - * Always, always, always returns a freshly created dict. - - ``eval_str`` controls whether or not values of type ``str`` are replaced - with the result of calling :func:`eval()` on those values: - - * If eval_str is true, :func:`eval()` is called on values of type ``str``. - (Note that ``get_annotations`` doesn't catch exceptions; if :func:`eval()` - raises an exception, it will unwind the stack past the ``get_annotations`` - call.) - * If eval_str is false (the default), values of type ``str`` are unchanged. - - ``globals`` and ``locals`` are passed in to :func:`eval()`; see the documentation - for :func:`eval()` for more information. If ``globals`` or ``locals`` - is ``None``, this function may replace that value with a context-specific - default, contingent on ``type(obj)``: - - * If ``obj`` is a module, ``globals`` defaults to ``obj.__dict__``. - * If ``obj`` is a class, ``globals`` defaults to - ``sys.modules[obj.__module__].__dict__`` and ``locals`` defaults - to the ``obj`` class namespace. - * If ``obj`` is a callable, ``globals`` defaults to - :attr:`obj.__globals__ `, - although if ``obj`` is a wrapped function (using - :func:`functools.update_wrapper`) it is first unwrapped. - - Calling ``get_annotations`` is best practice for accessing the - annotations dict of any object. See :ref:`annotations-howto` for - more information on annotations best practices. + This is an alias for :func:`annotationlib.get_annotations`; see the documentation + of that function for more information. .. versionadded:: 3.10 + .. versionchanged:: 3.14 + This function is now an alias for :func:`annotationlib.get_annotations`. + Calling it as ``inspect.get_annotations`` will continue to work. + .. _inspect-stack: @@ -1665,6 +1710,21 @@ which is a bitmap of the following flags: .. versionadded:: 3.6 +.. data:: CO_HAS_DOCSTRING + + The flag is set when there is a docstring for the code object in + the source code. If set, it will be the first item in + :attr:`~codeobject.co_consts`. + + .. versionadded:: 3.14 + +.. data:: CO_METHOD + + The flag is set when the code object is a function defined in class + scope. + + .. versionadded:: 3.14 + .. note:: The flags are specific to CPython, and may not be defined in other Python implementations. Furthermore, the flags are an implementation diff --git a/Doc/library/intro.rst b/Doc/library/intro.rst index 5a4c9b8b16ab3b..8f76044be488cd 100644 --- a/Doc/library/intro.rst +++ b/Doc/library/intro.rst @@ -58,7 +58,7 @@ Notes on availability operating system. * If not separately noted, all functions that claim "Availability: Unix" are - supported on macOS, which builds on a Unix core. + supported on macOS, iOS and Android, all of which build on a Unix core. * If an availability note contains both a minimum Kernel version and a minimum libc version, then both conditions must hold. For example a feature with note @@ -119,3 +119,58 @@ DOM APIs as well as limited networking capabilities with JavaScript's .. _wasmtime: https://wasmtime.dev/ .. _Pyodide: https://pyodide.org/ .. _PyScript: https://pyscript.net/ + +.. _mobile-availability: +.. _iOS-availability: + +Mobile platforms +---------------- + +Android and iOS are, in most respects, POSIX operating systems. File I/O, socket handling, +and threading all behave as they would on any POSIX operating system. However, +there are several major differences: + +* Mobile platforms can only use Python in "embedded" mode. There is no Python + REPL, and no ability to use separate executables such as :program:`python` or + :program:`pip`. To add Python code to your mobile app, you must use + the :ref:`Python embedding API `. For more details, see + :ref:`using-android` and :ref:`using-ios`. + +* Subprocesses: + + * On Android, creating subprocesses is possible but `officially unsupported + `__. + In particular, Android does not support any part of the System V IPC API, + so :mod:`multiprocessing` is not available. + + * An iOS app cannot use any form of subprocessing, multiprocessing, or + inter-process communication. If an iOS app attempts to create a subprocess, + the process creating the subprocess will either lock up, or crash. An iOS app + has no visibility of other applications that are running, nor any ability to + communicate with other running applications, outside of the iOS-specific APIs + that exist for this purpose. + +* Mobile apps have limited access to modify system resources (such as the system + clock). These resources will often be *readable*, but attempts to modify + those resources will usually fail. + +* Console input and output: + + * On Android, the native ``stdout`` and ``stderr`` are not connected to + anything, so Python installs its own streams which redirect messages to the + system log. These can be seen under the tags ``python.stdout`` and + ``python.stderr`` respectively. + + * iOS apps have a limited concept of console output. ``stdout`` and + ``stderr`` *exist*, and content written to ``stdout`` and ``stderr`` will be + visible in logs when running in Xcode, but this content *won't* be recorded + in the system log. If a user who has installed your app provides their app + logs as a diagnostic aid, they will not include any detail written to + ``stdout`` or ``stderr``. + + * Mobile apps have no usable ``stdin`` at all. While apps can display an on-screen + keyboard, this is a software feature, not something that is attached to + ``stdin``. + + As a result, Python modules that involve console manipulation (such as + :mod:`curses` and :mod:`readline`) are not available on mobile platforms. diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 8eb531aa4ea248..0d8cc5171d5476 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -1,5 +1,5 @@ -:mod:`io` --- Core tools for working with streams -================================================= +:mod:`!io` --- Core tools for working with streams +================================================== .. module:: io :synopsis: Core tools for working with streams. @@ -55,7 +55,7 @@ the backing store is natively made of bytes (such as in the case of a file), encoding and decoding of data is made transparently as well as optional translation of platform-specific newline characters. -The easiest way to create a text stream is with :meth:`open()`, optionally +The easiest way to create a text stream is with :meth:`open`, optionally specifying an encoding:: f = open("myfile.txt", "r", encoding="utf-8") @@ -64,6 +64,12 @@ In-memory text streams are also available as :class:`StringIO` objects:: f = io.StringIO("some initial text data") +.. note:: + + When working with a non-blocking stream, be aware that read operations on text I/O objects + might raise a :exc:`BlockingIOError` if the stream cannot perform the operation + immediately. + The text stream API is described in detail in the documentation of :class:`TextIOBase`. @@ -77,7 +83,7 @@ objects. No encoding, decoding, or newline translation is performed. This category of streams can be used for all kinds of non-text data, and also when manual control over the handling of text data is desired. -The easiest way to create a binary stream is with :meth:`open()` with ``'b'`` in +The easiest way to create a binary stream is with :meth:`open` with ``'b'`` in the mode string:: f = open("myfile.jpg", "rb") @@ -770,6 +776,11 @@ than raw I/O does. Read and return *size* bytes, or if *size* is not given or negative, until EOF or if the read call would block in non-blocking mode. + .. note:: + + When the underlying raw stream is non-blocking, a :exc:`BlockingIOError` + may be raised if a read operation cannot be completed immediately. + .. method:: read1(size=-1, /) Read and return up to *size* bytes with only one call on the raw stream. @@ -779,6 +790,10 @@ than raw I/O does. .. versionchanged:: 3.7 The *size* argument is now optional. + .. note:: + + When the underlying raw stream is non-blocking, a :exc:`BlockingIOError` + may be raised if a read operation cannot be completed immediately. .. class:: BufferedWriter(raw, buffer_size=DEFAULT_BUFFER_SIZE) @@ -950,7 +965,7 @@ Text I/O :class:`TextIOBase`. *encoding* gives the name of the encoding that the stream will be decoded or - encoded with. It defaults to :func:`locale.getencoding()`. + encoded with. It defaults to :func:`locale.getencoding`. ``encoding="locale"`` can be used to specify the current locale's encoding explicitly. See :ref:`io-text-encoding` for more information. @@ -1007,6 +1022,11 @@ Text I/O .. versionchanged:: 3.10 The *encoding* argument now supports the ``"locale"`` dummy encoding name. + .. note:: + + When the underlying raw stream is non-blocking, a :exc:`BlockingIOError` + may be raised if a read operation cannot be completed immediately. + :class:`TextIOWrapper` provides these data attributes and methods in addition to those from :class:`TextIOBase` and :class:`IOBase`: @@ -1182,7 +1202,7 @@ re-enter a buffered object which it is already accessing, a :exc:`RuntimeError` is raised. Note this doesn't prohibit a different thread from entering the buffered object. -The above implicitly extends to text files, since the :func:`open()` function +The above implicitly extends to text files, since the :func:`open` function will wrap a buffered object inside a :class:`TextIOWrapper`. This includes -standard streams and therefore affects the built-in :func:`print()` function as +standard streams and therefore affects the built-in :func:`print` function as well. diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 1de36b643c4dca..e5bdfbb144b65a 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -1,5 +1,5 @@ -:mod:`ipaddress` --- IPv4/IPv6 manipulation library -=================================================== +:mod:`!ipaddress` --- IPv4/IPv6 manipulation library +==================================================== .. module:: ipaddress :synopsis: IPv4/IPv6 manipulation library. @@ -121,26 +121,20 @@ write code that handles both IP versions correctly. Address objects are Leading zeros are tolerated, even in ambiguous cases that look like octal notation. - .. versionchanged:: 3.10 + .. versionchanged:: 3.9.5 Leading zeros are no longer tolerated and are treated as an error. IPv4 address strings are now parsed as strict as glibc :func:`~socket.inet_pton`. - .. versionchanged:: 3.9.5 - - The above change was also included in Python 3.9 starting with - version 3.9.5. - - .. versionchanged:: 3.8.12 - - The above change was also included in Python 3.8 starting with - version 3.8.12. - .. attribute:: version The appropriate version number: ``4`` for IPv4, ``6`` for IPv6. + .. versionchanged:: 3.14 + + Made available on the class. + .. attribute:: max_prefixlen The total number of bits in the address representation for this @@ -150,6 +144,10 @@ write code that handles both IP versions correctly. Address objects are are compared to determine whether or not an address is part of a network. + .. versionchanged:: 3.14 + + Made available on the class. + .. attribute:: compressed .. attribute:: exploded @@ -188,18 +186,53 @@ write code that handles both IP versions correctly. Address objects are .. attribute:: is_private - ``True`` if the address is allocated for private networks. See + ``True`` if the address is defined as not globally reachable by iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ - (for IPv6). + (for IPv6) with the following exceptions: + + * ``is_private`` is ``False`` for the shared address space (``100.64.0.0/10``) + * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_private == address.ipv4_mapped.is_private + + ``is_private`` has value opposite to :attr:`is_global`, except for the shared address space + (``100.64.0.0/10`` range) where they are both ``False``. + + .. versionchanged:: 3.13 + + Fixed some false positives and false negatives. + + * ``192.0.0.0/24`` is considered private with the exception of ``192.0.0.9/32`` and + ``192.0.0.10/32`` (previously: only the ``192.0.0.0/29`` sub-range was considered private). + * ``64:ff9b:1::/48`` is considered private. + * ``2002::/16`` is considered private. + * There are exceptions within ``2001::/23`` (otherwise considered private): ``2001:1::1/128``, + ``2001:1::2/128``, ``2001:3::/32``, ``2001:4:112::/48``, ``2001:20::/28``, ``2001:30::/28``. + The exceptions are not considered private. .. attribute:: is_global - ``True`` if the address is allocated for public networks. See + ``True`` if the address is defined as globally reachable by iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ - (for IPv6). + (for IPv6) with the following exception: + + For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_global == address.ipv4_mapped.is_global + + ``is_global`` has value opposite to :attr:`is_private`, except for the shared address space + (``100.64.0.0/10`` range) where they are both ``False``. .. versionadded:: 3.4 + .. versionchanged:: 3.13 + + Fixed some false positives and false negatives, see :attr:`is_private` for details. + .. attribute:: is_unspecified ``True`` if the address is unspecified. See :RFC:`5735` (for IPv4) @@ -309,14 +342,14 @@ write code that handles both IP versions correctly. Address objects are .. attribute:: is_multicast .. attribute:: is_private .. attribute:: is_global + + .. versionadded:: 3.4 + .. attribute:: is_unspecified .. attribute:: is_reserved .. attribute:: is_loopback .. attribute:: is_link_local - .. versionadded:: 3.4 - is_global - .. attribute:: is_site_local ``True`` if the address is reserved for site-local usage. Note that @@ -479,7 +512,7 @@ dictionaries. 4. A two-tuple of an address description and a netmask, where the address description is either a string, a 32-bits integer, a 4-bytes packed - integer, or an existing IPv4Address object; and the netmask is either + integer, or an existing :class:`IPv4Address` object; and the netmask is either an integer representing the prefix length (e.g. ``24``) or a string representing the prefix mask (e.g. ``255.255.255.0``). @@ -700,7 +733,7 @@ dictionaries. 4. A two-tuple of an address description and a netmask, where the address description is either a string, a 128-bits integer, a 16-bytes packed - integer, or an existing IPv6Address object; and the netmask is an + integer, or an existing :class:`IPv6Address` object; and the netmask is an integer representing the prefix length. An :exc:`AddressValueError` is raised if *address* is not a valid IPv6 @@ -756,7 +789,7 @@ dictionaries. .. attribute:: is_site_local - These attribute is true for the network as a whole if it is true + This attribute is true for the network as a whole if it is true for both the network address and the broadcast address. @@ -965,7 +998,7 @@ The module also provides the following module level functions: .. function:: collapse_addresses(addresses) Return an iterator of the collapsed :class:`IPv4Network` or - :class:`IPv6Network` objects. *addresses* is an iterator of + :class:`IPv6Network` objects. *addresses* is an :term:`iterable` of :class:`IPv4Network` or :class:`IPv6Network` objects. A :exc:`TypeError` is raised if *addresses* contains mixed version objects. @@ -985,7 +1018,7 @@ The module also provides the following module level functions: doesn't make sense. There are some times however, where you may wish to have :mod:`ipaddress` sort these anyway. If you need to do this, you can use - this function as the *key* argument to :func:`sorted()`. + this function as the *key* argument to :func:`sorted`. *obj* is either a network or address object. diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 2ee39fd9104df8..eb61453718bd3c 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -1,5 +1,5 @@ -:mod:`itertools` --- Functions creating iterators for efficient looping -======================================================================= +:mod:`!itertools` --- Functions creating iterators for efficient looping +======================================================================== .. module:: itertools :synopsis: Functions creating iterators for efficient looping. @@ -30,20 +30,15 @@ For instance, SML provides a tabulation tool: ``tabulate(f)`` which produces a sequence ``f(0), f(1), ...``. The same effect can be achieved in Python by combining :func:`map` and :func:`count` to form ``map(f, count())``. -These tools and their built-in counterparts also work well with the high-speed -functions in the :mod:`operator` module. For example, the multiplication -operator can be mapped across two vectors to form an efficient dot-product: -``sum(starmap(operator.mul, zip(vec1, vec2, strict=True)))``. - **Infinite iterators:** ================== ================= ================================================= ========================================= Iterator Arguments Results Example ================== ================= ================================================= ========================================= -:func:`count` [start[, step]] start, start+step, start+2*step, ... ``count(10) --> 10 11 12 13 14 ...`` -:func:`cycle` p p0, p1, ... plast, p0, p1, ... ``cycle('ABCD') --> A B C D A B C D ...`` -:func:`repeat` elem [,n] elem, elem, elem, ... endlessly or up to n times ``repeat(10, 3) --> 10 10 10`` +:func:`count` [start[, step]] start, start+step, start+2*step, ... ``count(10) → 10 11 12 13 14 ...`` +:func:`cycle` p p0, p1, ... plast, p0, p1, ... ``cycle('ABCD') → A B C D A B C D ...`` +:func:`repeat` elem [,n] elem, elem, elem, ... endlessly or up to n times ``repeat(10, 3) → 10 10 10`` ================== ================= ================================================= ========================================= **Iterators terminating on the shortest input sequence:** @@ -51,20 +46,20 @@ Iterator Arguments Results ============================ ============================ ================================================= ============================================================= Iterator Arguments Results Example ============================ ============================ ================================================= ============================================================= -:func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) --> 1 3 6 10 15`` -:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=3) --> ABC DEF G`` -:func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') --> A B C D E F`` -:func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) --> A B C D E F`` -:func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F`` -:func:`dropwhile` predicate, seq seq[n], seq[n+1], starting when predicate fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1`` -:func:`filterfalse` predicate, seq elements of seq where predicate(elem) fails ``filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8`` -:func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) -:func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) --> C D E F G`` -:func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') --> AB BC CD DE EF FG`` -:func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000`` -:func:`takewhile` predicate, seq seq[0], seq[1], until predicate fails ``takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4`` -:func:`tee` it, n it1, it2, ... itn splits one iterator into n -:func:`zip_longest` p, q, ... (p[0], q[0]), (p[1], q[1]), ... ``zip_longest('ABCD', 'xy', fillvalue='-') --> Ax By C- D-`` +:func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) → 1 3 6 10 15`` +:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=3) → ABC DEF G`` +:func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') → A B C D E F`` +:func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) → A B C D E F`` +:func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) → A C E F`` +:func:`dropwhile` predicate, seq seq[n], seq[n+1], starting when predicate fails ``dropwhile(lambda x: x<5, [1,4,6,3,8]) → 6 3 8`` +:func:`filterfalse` predicate, seq elements of seq where predicate(elem) fails ``filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8`` +:func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) ``groupby(['A','B','DEF'], len) → (1, A B) (3, DEF)`` +:func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) → C D E F G`` +:func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') → AB BC CD DE EF FG`` +:func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) → 32 9 1000`` +:func:`takewhile` predicate, seq seq[0], seq[1], until predicate fails ``takewhile(lambda x: x<5, [1,4,6,3,8]) → 1 4`` +:func:`tee` it, n it1, it2, ... itn splits one iterator into n ``tee('ABC', 2) → A B C, A B C`` +:func:`zip_longest` p, q, ... (p[0], q[0]), (p[1], q[1]), ... ``zip_longest('ABCD', 'xy', fillvalue='-') → Ax By C- D-`` ============================ ============================ ================================================= ============================================================= **Combinatoric iterators:** @@ -84,7 +79,7 @@ Examples Results ``product('ABCD', repeat=2)`` ``AA AB AC AD BA BB BC BD CA CB CC CD DA DB DC DD`` ``permutations('ABCD', 2)`` ``AB AC AD BA BC BD CA CB CD DA DB DC`` ``combinations('ABCD', 2)`` ``AB AC AD BC BD CD`` -``combinations_with_replacement('ABCD', 2)`` ``AA AB AC AD BB BC BD CC CD DD`` +``combinations_with_replacement('ABCD', 2)`` ``AA AB AC AD BB BC BD CC CD DD`` ============================================== ============================================================= @@ -93,63 +88,62 @@ Examples Results Itertool Functions ------------------ -The following module functions all construct and return iterators. Some provide +The following functions all construct and return iterators. Some provide streams of infinite length, so they should only be accessed by functions or loops that truncate the stream. -.. function:: accumulate(iterable[, func, *, initial=None]) - Make an iterator that returns accumulated sums, or accumulated - results of other binary functions (specified via the optional - *func* argument). +.. function:: accumulate(iterable[, function, *, initial=None]) + + Make an iterator that returns accumulated sums or accumulated + results from other binary functions. - If *func* is supplied, it should be a function - of two arguments. Elements of the input *iterable* may be any type - that can be accepted as arguments to *func*. (For example, with - the default operation of addition, elements may be any addable - type including :class:`~decimal.Decimal` or - :class:`~fractions.Fraction`.) + The *function* defaults to addition. The *function* should accept + two arguments, an accumulated total and a value from the *iterable*. - Usually, the number of elements output matches the input iterable. - However, if the keyword argument *initial* is provided, the - accumulation leads off with the *initial* value so that the output - has one more element than the input iterable. + If an *initial* value is provided, the accumulation will start with + that value and the output will have one more element than the input + iterable. Roughly equivalent to:: - def accumulate(iterable, func=operator.add, *, initial=None): + def accumulate(iterable, function=operator.add, *, initial=None): 'Return running totals' - # accumulate([1,2,3,4,5]) --> 1 3 6 10 15 - # accumulate([1,2,3,4,5], initial=100) --> 100 101 103 106 110 115 - # accumulate([1,2,3,4,5], operator.mul) --> 1 2 6 24 120 - it = iter(iterable) + # accumulate([1,2,3,4,5]) → 1 3 6 10 15 + # accumulate([1,2,3,4,5], initial=100) → 100 101 103 106 110 115 + # accumulate([1,2,3,4,5], operator.mul) → 1 2 6 24 120 + + iterator = iter(iterable) total = initial if initial is None: try: - total = next(it) + total = next(iterator) except StopIteration: return + yield total - for element in it: - total = func(total, element) + for element in iterator: + total = function(total, element) yield total - There are a number of uses for the *func* argument. It can be set to - :func:`min` for a running minimum, :func:`max` for a running maximum, or - :func:`operator.mul` for a running product. Amortization tables can be - built by accumulating interest and applying payments: + To compute a running minimum, set *function* to :func:`min`. + For a running maximum, set *function* to :func:`max`. + Or for a running product, set *function* to :func:`operator.mul`. + To build an `amortization table + `_, + accumulate the interest and apply payments: .. doctest:: >>> data = [3, 4, 6, 2, 1, 9, 0, 7, 5, 8] - >>> list(accumulate(data, operator.mul)) # running product - [3, 12, 72, 144, 144, 1296, 0, 0, 0, 0] >>> list(accumulate(data, max)) # running maximum [3, 4, 6, 6, 6, 9, 9, 9, 9, 9] + >>> list(accumulate(data, operator.mul)) # running product + [3, 12, 72, 144, 144, 1296, 0, 0, 0, 0] # Amortize a 5% loan of 1000 with 10 annual payments of 90 - >>> account_update = lambda bal, pmt: round(bal * 1.05) + pmt - >>> list(accumulate(repeat(-90, 10), account_update, initial=1_000)) + >>> update = lambda balance, payment: round(balance * 1.05) - payment + >>> list(accumulate(repeat(90, 10), update, initial=1_000)) [1000, 960, 918, 874, 828, 779, 728, 674, 618, 559, 497] See :func:`functools.reduce` for a similar function that returns only the @@ -158,7 +152,7 @@ loops that truncate the stream. .. versionadded:: 3.2 .. versionchanged:: 3.3 - Added the optional *func* parameter. + Added the optional *function* parameter. .. versionchanged:: 3.8 Added the optional *initial* parameter. @@ -184,21 +178,14 @@ loops that truncate the stream. >>> unflattened [('roses', 'red'), ('violets', 'blue'), ('sugar', 'sweet')] - >>> for batch in batched('ABCDEFG', 3): - ... print(batch) - ... - ('A', 'B', 'C') - ('D', 'E', 'F') - ('G',) - Roughly equivalent to:: def batched(iterable, n, *, strict=False): - # batched('ABCDEFG', 3) --> ABC DEF G + # batched('ABCDEFG', 3) → ABC DEF G if n < 1: raise ValueError('n must be at least one') - it = iter(iterable) - while batch := tuple(islice(it, n)): + iterator = iter(iterable) + while batch := tuple(islice(iterator, n)): if strict and len(batch) != n: raise ValueError('batched(): incomplete batch') yield batch @@ -211,16 +198,15 @@ loops that truncate the stream. .. function:: chain(*iterables) - Make an iterator that returns elements from the first iterable until it is - exhausted, then proceeds to the next iterable, until all of the iterables are - exhausted. Used for treating consecutive sequences as a single sequence. - Roughly equivalent to:: + Make an iterator that returns elements from the first iterable until + it is exhausted, then proceeds to the next iterable, until all of the + iterables are exhausted. This combines multiple data sources into a + single iterator. Roughly equivalent to:: def chain(*iterables): - # chain('ABC', 'DEF') --> A B C D E F - for it in iterables: - for element in it: - yield element + # chain('ABC', 'DEF') → A B C D E F + for iterable in iterables: + yield from iterable .. classmethod:: chain.from_iterable(iterable) @@ -229,34 +215,40 @@ loops that truncate the stream. single iterable argument that is evaluated lazily. Roughly equivalent to:: def from_iterable(iterables): - # chain.from_iterable(['ABC', 'DEF']) --> A B C D E F - for it in iterables: - for element in it: - yield element + # chain.from_iterable(['ABC', 'DEF']) → A B C D E F + for iterable in iterables: + yield from iterable .. function:: combinations(iterable, r) Return *r* length subsequences of elements from the input *iterable*. - The combination tuples are emitted in lexicographic ordering according to - the order of the input *iterable*. So, if the input *iterable* is sorted, + The output is a subsequence of :func:`product` keeping only entries that + are subsequences of the *iterable*. The length of the output is given + by :func:`math.comb` which computes ``n! / r! / (n - r)!`` when ``0 ≤ r + ≤ n`` or zero when ``r > n``. + + The combination tuples are emitted in lexicographic order according to + the order of the input *iterable*. If the input *iterable* is sorted, the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, there will be no repeated - values in each combination. + value. If the input elements are unique, there will be no repeated + values within each combination. Roughly equivalent to:: def combinations(iterable, r): - # combinations('ABCD', 2) --> AB AC AD BC BD CD - # combinations(range(4), 3) --> 012 013 023 123 + # combinations('ABCD', 2) → AB AC AD BC BD CD + # combinations(range(4), 3) → 012 013 023 123 + pool = tuple(iterable) n = len(pool) if r > n: return indices = list(range(r)) + yield tuple(pool[i] for i in indices) while True: for i in reversed(range(r)): @@ -269,42 +261,36 @@ loops that truncate the stream. indices[j] = indices[j-1] + 1 yield tuple(pool[i] for i in indices) - The code for :func:`combinations` can be also expressed as a subsequence - of :func:`permutations` after filtering entries where the elements are not - in sorted order (according to their position in the input pool):: - - def combinations(iterable, r): - pool = tuple(iterable) - n = len(pool) - for indices in permutations(range(n), r): - if sorted(indices) == list(indices): - yield tuple(pool[i] for i in indices) - - The number of items returned is ``n! / r! / (n-r)!`` when ``0 <= r <= n`` - or zero when ``r > n``. .. function:: combinations_with_replacement(iterable, r) Return *r* length subsequences of elements from the input *iterable* allowing individual elements to be repeated more than once. - The combination tuples are emitted in lexicographic ordering according to - the order of the input *iterable*. So, if the input *iterable* is sorted, + The output is a subsequence of :func:`product` that keeps only entries + that are subsequences (with possible repeated elements) of the + *iterable*. The number of subsequence returned is ``(n + r - 1)! / r! / + (n - 1)!`` when ``n > 0``. + + The combination tuples are emitted in lexicographic order according to + the order of the input *iterable*. if the input *iterable* is sorted, the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, the generated combinations + value. If the input elements are unique, the generated combinations will also be unique. Roughly equivalent to:: def combinations_with_replacement(iterable, r): - # combinations_with_replacement('ABC', 2) --> AA AB AC BB BC CC + # combinations_with_replacement('ABC', 2) → AA AB AC BB BC CC + pool = tuple(iterable) n = len(pool) if not n and r: return indices = [0] * r + yield tuple(pool[i] for i in indices) while True: for i in reversed(range(r)): @@ -315,104 +301,103 @@ loops that truncate the stream. indices[i:] = [indices[i] + 1] * (r - i) yield tuple(pool[i] for i in indices) - The code for :func:`combinations_with_replacement` can be also expressed as - a subsequence of :func:`product` after filtering entries where the elements - are not in sorted order (according to their position in the input pool):: - - def combinations_with_replacement(iterable, r): - pool = tuple(iterable) - n = len(pool) - for indices in product(range(n), repeat=r): - if sorted(indices) == list(indices): - yield tuple(pool[i] for i in indices) - - The number of items returned is ``(n+r-1)! / r! / (n-1)!`` when ``n > 0``. - .. versionadded:: 3.1 .. function:: compress(data, selectors) - Make an iterator that filters elements from *data* returning only those that - have a corresponding element in *selectors* that evaluates to ``True``. - Stops when either the *data* or *selectors* iterables has been exhausted. - Roughly equivalent to:: + Make an iterator that returns elements from *data* where the + corresponding element in *selectors* is true. Stops when either the + *data* or *selectors* iterables have been exhausted. Roughly + equivalent to:: def compress(data, selectors): - # compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F - return (d for d, s in zip(data, selectors) if s) + # compress('ABCDEF', [1,0,1,0,1,1]) → A C E F + return (datum for datum, selector in zip(data, selectors) if selector) .. versionadded:: 3.1 .. function:: count(start=0, step=1) - Make an iterator that returns evenly spaced values starting with number *start*. Often - used as an argument to :func:`map` to generate consecutive data points. - Also, used with :func:`zip` to add sequence numbers. Roughly equivalent to:: + Make an iterator that returns evenly spaced values beginning with + *start*. Can be used with :func:`map` to generate consecutive data + points or with :func:`zip` to add sequence numbers. Roughly + equivalent to:: def count(start=0, step=1): - # count(10) --> 10 11 12 13 14 ... - # count(2.5, 0.5) --> 2.5 3.0 3.5 ... + # count(10) → 10 11 12 13 14 ... + # count(2.5, 0.5) → 2.5 3.0 3.5 ... n = start while True: yield n n += step - When counting with floating point numbers, better accuracy can sometimes be + When counting with floating-point numbers, better accuracy can sometimes be achieved by substituting multiplicative code such as: ``(start + step * i for i in count())``. .. versionchanged:: 3.1 Added *step* argument and allowed non-integer arguments. + .. function:: cycle(iterable) - Make an iterator returning elements from the iterable and saving a copy of each. - When the iterable is exhausted, return elements from the saved copy. Repeats - indefinitely. Roughly equivalent to:: + Make an iterator returning elements from the *iterable* and saving a + copy of each. When the iterable is exhausted, return elements from + the saved copy. Repeats indefinitely. Roughly equivalent to:: def cycle(iterable): - # cycle('ABCD') --> A B C D A B C D A B C D ... + # cycle('ABCD') → A B C D A B C D A B C D ... + saved = [] for element in iterable: yield element saved.append(element) + while saved: for element in saved: - yield element + yield element - Note, this member of the toolkit may require significant auxiliary storage - (depending on the length of the iterable). + This itertool may require significant auxiliary storage (depending on + the length of the iterable). .. function:: dropwhile(predicate, iterable) - Make an iterator that drops elements from the iterable as long as the predicate - is true; afterwards, returns every element. Note, the iterator does not produce - *any* output until the predicate first becomes false, so it may have a lengthy - start-up time. Roughly equivalent to:: + Make an iterator that drops elements from the *iterable* while the + *predicate* is true and afterwards returns every element. Roughly + equivalent to:: def dropwhile(predicate, iterable): - # dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1 - iterable = iter(iterable) - for x in iterable: + # dropwhile(lambda x: x<5, [1,4,6,3,8]) → 6 3 8 + + iterator = iter(iterable) + for x in iterator: if not predicate(x): yield x break - for x in iterable: + + for x in iterator: yield x + Note this does not produce *any* output until the predicate first + becomes false, so this itertool may have a lengthy start-up time. + + .. function:: filterfalse(predicate, iterable) - Make an iterator that filters elements from iterable returning only those for - which the predicate is false. If *predicate* is ``None``, return the items - that are false. Roughly equivalent to:: + Make an iterator that filters elements from the *iterable* returning + only those for which the *predicate* returns a false value. If + *predicate* is ``None``, returns the items that are false. Roughly + equivalent to:: def filterfalse(predicate, iterable): - # filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8 + # filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8 + if predicate is None: predicate = bool + for x in iterable: if not predicate(x): yield x @@ -446,82 +431,81 @@ loops that truncate the stream. :func:`groupby` is roughly equivalent to:: - class groupby: - # [k for k, g in groupby('AAAABBBCCDAABBB')] --> A B C D A B - # [list(g) for k, g in groupby('AAAABBBCCD')] --> AAAA BBB CC D - - def __init__(self, iterable, key=None): - if key is None: - key = lambda x: x - self.keyfunc = key - self.it = iter(iterable) - self.tgtkey = self.currkey = self.currvalue = object() - - def __iter__(self): - return self - - def __next__(self): - self.id = object() - while self.currkey == self.tgtkey: - self.currvalue = next(self.it) # Exit on StopIteration - self.currkey = self.keyfunc(self.currvalue) - self.tgtkey = self.currkey - return (self.currkey, self._grouper(self.tgtkey, self.id)) - - def _grouper(self, tgtkey, id): - while self.id is id and self.currkey == tgtkey: - yield self.currvalue - try: - self.currvalue = next(self.it) - except StopIteration: + def groupby(iterable, key=None): + # [k for k, g in groupby('AAAABBBCCDAABBB')] → A B C D A B + # [list(g) for k, g in groupby('AAAABBBCCD')] → AAAA BBB CC D + + keyfunc = (lambda x: x) if key is None else key + iterator = iter(iterable) + exhausted = False + + def _grouper(target_key): + nonlocal curr_value, curr_key, exhausted + yield curr_value + for curr_value in iterator: + curr_key = keyfunc(curr_value) + if curr_key != target_key: return - self.currkey = self.keyfunc(self.currvalue) + yield curr_value + exhausted = True + + try: + curr_value = next(iterator) + except StopIteration: + return + curr_key = keyfunc(curr_value) + + while not exhausted: + target_key = curr_key + curr_group = _grouper(target_key) + yield curr_key, curr_group + if curr_key == target_key: + for _ in curr_group: + pass .. function:: islice(iterable, stop) islice(iterable, start, stop[, step]) - Make an iterator that returns selected elements from the iterable. If *start* is - non-zero, then elements from the iterable are skipped until start is reached. - Afterward, elements are returned consecutively unless *step* is set higher than - one which results in items being skipped. If *stop* is ``None``, then iteration - continues until the iterator is exhausted, if at all; otherwise, it stops at the - specified position. + Make an iterator that returns selected elements from the iterable. + Works like sequence slicing but does not support negative values for + *start*, *stop*, or *step*. - If *start* is ``None``, then iteration starts at zero. If *step* is ``None``, - then the step defaults to one. + If *start* is zero or ``None``, iteration starts at zero. Otherwise, + elements from the iterable are skipped until *start* is reached. - Unlike regular slicing, :func:`islice` does not support negative values for - *start*, *stop*, or *step*. Can be used to extract related fields from - data where the internal structure has been flattened (for example, a - multi-line report may list a name field on every third line). + If *stop* is ``None``, iteration continues until the input is + exhausted, if at all. Otherwise, it stops at the specified position. + + If *step* is ``None``, the step defaults to one. Elements are returned + consecutively unless *step* is set higher than one which results in + items being skipped. Roughly equivalent to:: def islice(iterable, *args): - # islice('ABCDEFG', 2) --> A B - # islice('ABCDEFG', 2, 4) --> C D - # islice('ABCDEFG', 2, None) --> C D E F G - # islice('ABCDEFG', 0, None, 2) --> A C E G + # islice('ABCDEFG', 2) → A B + # islice('ABCDEFG', 2, 4) → C D + # islice('ABCDEFG', 2, None) → C D E F G + # islice('ABCDEFG', 0, None, 2) → A C E G + s = slice(*args) - start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 - it = iter(range(start, stop, step)) - try: - nexti = next(it) - except StopIteration: - # Consume *iterable* up to the *start* position. - for i, element in zip(range(start), iterable): - pass - return - try: - for i, element in enumerate(iterable): - if i == nexti: - yield element - nexti = next(it) - except StopIteration: - # Consume to *stop*. - for i, element in zip(range(i + 1, stop), iterable): - pass + start = 0 if s.start is None else s.start + stop = s.stop + step = 1 if s.step is None else s.step + if start < 0 or (stop is not None and stop < 0) or step <= 0: + raise ValueError + + indices = count() if stop is None else range(max(start, stop)) + next_i = start + for i, element in zip(indices, iterable): + if i == next_i: + yield element + next_i += step + + If the input is an iterator, then fully consuming the *islice* + advances the input iterator by ``max(start, stop)`` steps regardless + of the *step* value. .. function:: pairwise(iterable) @@ -535,43 +519,56 @@ loops that truncate the stream. Roughly equivalent to:: def pairwise(iterable): - # pairwise('ABCDEFG') --> AB BC CD DE EF FG - a, b = tee(iterable) - next(b, None) - return zip(a, b) + # pairwise('ABCDEFG') → AB BC CD DE EF FG + + iterator = iter(iterable) + a = next(iterator, None) + + for b in iterator: + yield a, b + a = b .. versionadded:: 3.10 .. function:: permutations(iterable, r=None) - Return successive *r* length permutations of elements in the *iterable*. + Return successive *r* length `permutations of elements + `_ from the *iterable*. If *r* is not specified or is ``None``, then *r* defaults to the length of the *iterable* and all possible full-length permutations are generated. + The output is a subsequence of :func:`product` where entries with + repeated elements have been filtered out. The length of the output is + given by :func:`math.perm` which computes ``n! / (n - r)!`` when + ``0 ≤ r ≤ n`` or zero when ``r > n``. + The permutation tuples are emitted in lexicographic order according to - the order of the input *iterable*. So, if the input *iterable* is sorted, + the order of the input *iterable*. If the input *iterable* is sorted, the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, there will be no repeated + value. If the input elements are unique, there will be no repeated values within a permutation. Roughly equivalent to:: def permutations(iterable, r=None): - # permutations('ABCD', 2) --> AB AC AD BA BC BD CA CB CD DA DB DC - # permutations(range(3)) --> 012 021 102 120 201 210 + # permutations('ABCD', 2) → AB AC AD BA BC BD CA CB CD DA DB DC + # permutations(range(3)) → 012 021 102 120 201 210 + pool = tuple(iterable) n = len(pool) r = n if r is None else r if r > n: return + indices = list(range(n)) cycles = list(range(n, n-r, -1)) yield tuple(pool[i] for i in indices[:r]) + while n: for i in reversed(range(r)): cycles[i] -= 1 @@ -586,24 +583,11 @@ loops that truncate the stream. else: return - The code for :func:`permutations` can be also expressed as a subsequence of - :func:`product`, filtered to exclude entries with repeated elements (those - from the same position in the input pool):: - - def permutations(iterable, r=None): - pool = tuple(iterable) - n = len(pool) - r = n if r is None else r - for indices in product(range(n), repeat=r): - if len(set(indices)) == r: - yield tuple(pool[i] for i in indices) - - The number of items returned is ``n! / (n-r)!`` when ``0 <= r <= n`` - or zero when ``r > n``. .. function:: product(*iterables, repeat=1) - Cartesian product of input iterables. + `Cartesian product `_ + of the input iterables. Roughly equivalent to nested for-loops in a generator expression. For example, ``product(A, B)`` returns the same as ``((x,y) for x in A for y in B)``. @@ -620,13 +604,18 @@ loops that truncate the stream. This function is roughly equivalent to the following code, except that the actual implementation does not build up intermediate results in memory:: - def product(*args, repeat=1): - # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy - # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111 - pools = [tuple(pool) for pool in args] * repeat + def product(*iterables, repeat=1): + # product('ABCD', 'xy') → Ax Ay Bx By Cx Cy Dx Dy + # product(range(2), repeat=3) → 000 001 010 011 100 101 110 111 + + if repeat < 0: + raise ValueError('repeat argument cannot be negative') + pools = [tuple(pool) for pool in iterables] * repeat + result = [[]] for pool in pools: result = [x+[y] for x in result for y in pool] + for prod in result: yield tuple(prod) @@ -634,6 +623,7 @@ loops that truncate the stream. keeping pools of values in memory to generate the products. Accordingly, it is only useful with finite inputs. + .. function:: repeat(object[, times]) Make an iterator that returns *object* over and over again. Runs indefinitely @@ -642,7 +632,7 @@ loops that truncate the stream. Roughly equivalent to:: def repeat(object, times=None): - # repeat(10, 3) --> 10 10 10 + # repeat(10, 3) → 10 10 10 if times is None: while True: yield object @@ -658,41 +648,40 @@ loops that truncate the stream. >>> list(map(pow, range(10), repeat(2))) [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] + .. function:: starmap(function, iterable) - Make an iterator that computes the function using arguments obtained from - the iterable. Used instead of :func:`map` when argument parameters are already - grouped in tuples from a single iterable (when the data has been - "pre-zipped"). + Make an iterator that computes the *function* using arguments obtained + from the *iterable*. Used instead of :func:`map` when argument + parameters have already been "pre-zipped" into tuples. The difference between :func:`map` and :func:`starmap` parallels the distinction between ``function(a,b)`` and ``function(*c)``. Roughly equivalent to:: def starmap(function, iterable): - # starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000 + # starmap(pow, [(2,5), (3,2), (10,3)]) → 32 9 1000 for args in iterable: yield function(*args) .. function:: takewhile(predicate, iterable) - Make an iterator that returns elements from the iterable as long as the - predicate is true. Roughly equivalent to:: + Make an iterator that returns elements from the *iterable* as long as + the *predicate* is true. Roughly equivalent to:: def takewhile(predicate, iterable): - # takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4 + # takewhile(lambda x: x<5, [1,4,6,3,8]) → 1 4 for x in iterable: - if predicate(x): - yield x - else: + if not predicate(x): break + yield x Note, the element that first fails the predicate condition is consumed from the input iterator and there is no way to access it. This could be an issue if an application wants to further consume the - input iterator after takewhile has been run to exhaustion. To work - around this problem, consider using `more-iterools before_and_after() + input iterator after *takewhile* has been run to exhaustion. To work + around this problem, consider using `more-itertools before_and_after() `_ instead. @@ -701,28 +690,66 @@ loops that truncate the stream. Return *n* independent iterators from a single iterable. - The following Python code helps explain what *tee* does (although the actual - implementation is more complex and uses only a single underlying - :abbr:`FIFO (first-in, first-out)` queue):: + Roughly equivalent to:: def tee(iterable, n=2): - it = iter(iterable) - deques = [collections.deque() for i in range(n)] - def gen(mydeque): - while True: - if not mydeque: # when the local deque is empty - try: - newval = next(it) # fetch a new value and - except StopIteration: - return - for d in deques: # load it to all the deques - d.append(newval) - yield mydeque.popleft() - return tuple(gen(d) for d in deques) - - Once a :func:`tee` has been created, the original *iterable* should not be - used anywhere else; otherwise, the *iterable* could get advanced without - the tee objects being informed. + if n < 0: + raise ValueError + if n == 0: + return () + iterator = _tee(iterable) + result = [iterator] + for _ in range(n - 1): + result.append(_tee(iterator)) + return tuple(result) + + class _tee: + + def __init__(self, iterable): + it = iter(iterable) + if isinstance(it, _tee): + self.iterator = it.iterator + self.link = it.link + else: + self.iterator = it + self.link = [None, None] + + def __iter__(self): + return self + + def __next__(self): + link = self.link + if link[1] is None: + link[0] = next(self.iterator) + link[1] = [None, None] + value, self.link = link + return value + + When the input *iterable* is already a tee iterator object, all + members of the return tuple are constructed as if they had been + produced by the upstream :func:`tee` call. This "flattening step" + allows nested :func:`tee` calls to share the same underlying data + chain and to have a single update step rather than a chain of calls. + + The flattening property makes tee iterators efficiently peekable: + + .. testcode:: + + def lookahead(tee_iterator): + "Return the next value without moving the input forward" + [forked_iterator] = tee(tee_iterator, 1) + return next(forked_iterator) + + .. doctest:: + + >>> iterator = iter('abcdef') + >>> [iterator] = tee(iterator, 1) # Make the input peekable + >>> next(iterator) # Move the iterator forward + 'a' + >>> lookahead(iterator) # Check next value + 'b' + >>> next(iterator) # Continue moving forward + 'b' ``tee`` iterators are not threadsafe. A :exc:`RuntimeError` may be raised when simultaneously using iterators returned by the same :func:`tee` @@ -736,21 +763,29 @@ loops that truncate the stream. .. function:: zip_longest(*iterables, fillvalue=None) - Make an iterator that aggregates elements from each of the iterables. If the - iterables are of uneven length, missing values are filled-in with *fillvalue*. - Iteration continues until the longest iterable is exhausted. Roughly equivalent to:: + Make an iterator that aggregates elements from each of the + *iterables*. + + If the iterables are of uneven length, missing values are filled-in + with *fillvalue*. If not specified, *fillvalue* defaults to ``None``. - def zip_longest(*args, fillvalue=None): - # zip_longest('ABCD', 'xy', fillvalue='-') --> Ax By C- D- - iterators = [iter(it) for it in args] + Iteration continues until the longest iterable is exhausted. + + Roughly equivalent to:: + + def zip_longest(*iterables, fillvalue=None): + # zip_longest('ABCD', 'xy', fillvalue='-') → Ax By C- D- + + iterators = list(map(iter, iterables)) num_active = len(iterators) if not num_active: return + while True: values = [] - for i, it in enumerate(iterators): + for i, iterator in enumerate(iterators): try: - value = next(it) + value = next(iterator) except StopIteration: num_active -= 1 if not num_active: @@ -762,8 +797,7 @@ loops that truncate the stream. If one of the iterables is potentially infinite, then the :func:`zip_longest` function should be wrapped with something that limits the number of calls - (for example :func:`islice` or :func:`takewhile`). If not specified, - *fillvalue* defaults to ``None``. + (for example :func:`islice` or :func:`takewhile`). .. _itertools-recipes: @@ -785,30 +819,30 @@ well as with the built-in itertools such as ``map()``, ``filter()``, A secondary purpose of the recipes is to serve as an incubator. The ``accumulate()``, ``compress()``, and ``pairwise()`` itertools started out as -recipes. Currently, the ``sliding_window()`` and ``iter_index()`` recipes -are being tested to see whether they prove their worth. +recipes. Currently, the ``sliding_window()``, ``iter_index()``, and ``sieve()`` +recipes are being tested to see whether they prove their worth. Substantially all of these recipes and many, many others can be installed from -the `more-itertools project `_ found +the :pypi:`more-itertools` project found on the Python Package Index:: python -m pip install more-itertools Many of the recipes offer the same high performance as the underlying toolset. -Superior memory performance is kept by processing elements one at a time -rather than bringing the whole iterable into memory all at once. Code volume is -kept small by linking the tools together in a functional style which helps -eliminate temporary variables. High speed is retained by preferring -"vectorized" building blocks over the use of for-loops and :term:`generator`\s -which incur interpreter overhead. +Superior memory performance is kept by processing elements one at a time rather +than bringing the whole iterable into memory all at once. Code volume is kept +small by linking the tools together in a `functional style +`_. High speed +is retained by preferring "vectorized" building blocks over the use of for-loops +and :term:`generators ` which incur interpreter overhead. .. testcode:: - import collections - import functools - import math - import operator - import random + from collections import deque + from contextlib import suppress + from functools import reduce + from math import sumprod, isqrt + from operator import itemgetter, getitem, mul, neg def take(n, iterable): "Return first n items of the iterable as a list." @@ -816,21 +850,18 @@ which incur interpreter overhead. def prepend(value, iterable): "Prepend a single value in front of an iterable." - # prepend(1, [2, 3, 4]) --> 1 2 3 4 + # prepend(1, [2, 3, 4]) → 1 2 3 4 return chain([value], iterable) def tabulate(function, start=0): "Return function(0), function(1), ..." return map(function, count(start)) - def repeatfunc(func, times=None, *args): - """Repeat calls to func with specified arguments. - - Example: repeatfunc(random.random) - """ + def repeatfunc(function, times=None, *args): + "Repeat calls to a function with specified arguments." if times is None: - return starmap(func, repeat(args)) - return starmap(func, repeat(args, times)) + return starmap(function, repeat(args)) + return starmap(function, repeat(args, times)) def flatten(list_of_lists): "Flatten one level of nesting." @@ -840,19 +871,22 @@ which incur interpreter overhead. "Returns the sequence elements n times." return chain.from_iterable(repeat(tuple(iterable), n)) + def loops(n): + "Loop n times. Like range(n) but without creating integers." + # for _ in loops(100): ... + return repeat(None, n) + def tail(n, iterable): "Return an iterator over the last n items." - # tail(3, 'ABCDEFG') --> E F G - return iter(collections.deque(iterable, maxlen=n)) + # tail(3, 'ABCDEFG') → E F G + return iter(deque(iterable, maxlen=n)) def consume(iterator, n=None): "Advance the iterator n-steps ahead. If n is None, consume entirely." # Use functions that consume iterators at C speed. if n is None: - # feed the entire iterator into a zero-length deque - collections.deque(iterator, maxlen=0) + deque(iterator, maxlen=0) else: - # advance to the empty slice starting at position n next(islice(iterator, n, n), None) def nth(iterable, n, default=None): @@ -865,18 +899,27 @@ which incur interpreter overhead. def first_true(iterable, default=False, predicate=None): "Returns the first true value or the *default* if there is no true value." - # first_true([a,b,c], x) --> a or b or c or x - # first_true([a,b], x, f) --> a if f(a) else b if f(b) else x + # first_true([a,b,c], x) → a or b or c or x + # first_true([a,b], x, f) → a if f(a) else b if f(b) else x return next(filter(predicate, iterable), default) def all_equal(iterable, key=None): "Returns True if all the elements are equal to each other." + # all_equal('4٤௪౪໔', key=int) → True return len(take(2, groupby(iterable, key))) <= 1 + def unique_justseen(iterable, key=None): + "Yield unique elements, preserving order. Remember only the element just seen." + # unique_justseen('AAAABBBCCDAABBB') → A B C D A B + # unique_justseen('ABBcCAD', str.casefold) → A B c A D + if key is None: + return map(itemgetter(0), groupby(iterable)) + return map(next, map(itemgetter(1), groupby(iterable, key))) + def unique_everseen(iterable, key=None): - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBcCAD', str.casefold) --> A B c D + "Yield unique elements, preserving order. Remember all elements ever seen." + # unique_everseen('AAAABBBCCDAABBB') → A B C D + # unique_everseen('ABBcCAD', str.casefold) → A B c D seen = set() if key is None: for element in filterfalse(seen.__contains__, iterable): @@ -889,102 +932,77 @@ which incur interpreter overhead. seen.add(k) yield element - def unique_justseen(iterable, key=None): - "List unique elements, preserving order. Remember only the element just seen." - # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B - # unique_justseen('ABBcCAD', str.casefold) --> A B c A D - if key is None: - return map(operator.itemgetter(0), groupby(iterable)) - return map(next, map(operator.itemgetter(1), groupby(iterable, key))) - - def iter_index(iterable, value, start=0, stop=None): - "Return indices where a value occurs in a sequence or iterable." - # iter_index('AABCADEAF', 'A') --> 0 1 4 7 - seq_index = getattr(iterable, 'index', None) - if seq_index is None: - # Slow path for general iterables - it = islice(iterable, start, stop) - for i, element in enumerate(it, start): - if element is value or element == value: - yield i - else: - # Fast path for sequences - stop = len(iterable) if stop is None else stop - i = start - 1 - try: - while True: - yield (i := seq_index(value, i+1, stop)) - except ValueError: - pass + def unique(iterable, key=None, reverse=False): + "Yield unique elements in sorted order. Supports unhashable inputs." + # unique([[1, 2], [3, 4], [1, 2]]) → [1, 2] [3, 4] + sequenced = sorted(iterable, key=key, reverse=reverse) + return unique_justseen(sequenced, key=key) def sliding_window(iterable, n): "Collect data into overlapping fixed-length chunks or blocks." - # sliding_window('ABCDEFG', 4) --> ABCD BCDE CDEF DEFG - it = iter(iterable) - window = collections.deque(islice(it, n-1), maxlen=n) - for x in it: + # sliding_window('ABCDEFG', 4) → ABCD BCDE CDEF DEFG + iterator = iter(iterable) + window = deque(islice(iterator, n - 1), maxlen=n) + for x in iterator: window.append(x) yield tuple(window) def grouper(iterable, n, *, incomplete='fill', fillvalue=None): "Collect data into non-overlapping fixed-length chunks or blocks." - # grouper('ABCDEFG', 3, fillvalue='x') --> ABC DEF Gxx - # grouper('ABCDEFG', 3, incomplete='strict') --> ABC DEF ValueError - # grouper('ABCDEFG', 3, incomplete='ignore') --> ABC DEF - args = [iter(iterable)] * n + # grouper('ABCDEFG', 3, fillvalue='x') → ABC DEF Gxx + # grouper('ABCDEFG', 3, incomplete='strict') → ABC DEF ValueError + # grouper('ABCDEFG', 3, incomplete='ignore') → ABC DEF + iterators = [iter(iterable)] * n match incomplete: case 'fill': - return zip_longest(*args, fillvalue=fillvalue) + return zip_longest(*iterators, fillvalue=fillvalue) case 'strict': - return zip(*args, strict=True) + return zip(*iterators, strict=True) case 'ignore': - return zip(*args) + return zip(*iterators) case _: raise ValueError('Expected fill, strict, or ignore') def roundrobin(*iterables): "Visit input iterables in a cycle until each is exhausted." - # roundrobin('ABC', 'D', 'EF') --> A D E B F C - # Recipe credited to George Sakkis - num_active = len(iterables) - nexts = cycle(iter(it).__next__ for it in iterables) - while num_active: - try: - for next in nexts: - yield next() - except StopIteration: - # Remove the iterator we just exhausted from the cycle. - num_active -= 1 - nexts = cycle(islice(nexts, num_active)) - - def partition(predicate, iterable): - """Partition entries into false entries and true entries. - - If *predicate* is slow, consider wrapping it with functools.lru_cache(). - """ - # partition(is_odd, range(10)) --> 0 2 4 6 8 and 1 3 5 7 9 - t1, t2 = tee(iterable) - return filterfalse(predicate, t1), filter(predicate, t2) + # roundrobin('ABC', 'D', 'EF') → A D E B F C + # Algorithm credited to George Sakkis + iterators = map(iter, iterables) + for num_active in range(len(iterables), 0, -1): + iterators = cycle(islice(iterators, num_active)) + yield from map(next, iterators) def subslices(seq): "Return all contiguous non-empty subslices of a sequence." - # subslices('ABCD') --> A AB ABC ABCD B BC BCD C CD D + # subslices('ABCD') → A AB ABC ABCD B BC BCD C CD D slices = starmap(slice, combinations(range(len(seq) + 1), 2)) - return map(operator.getitem, repeat(seq), slices) + return map(getitem, repeat(seq), slices) - def iter_except(func, exception, first=None): - """ Call a function repeatedly until an exception is raised. + def iter_index(iterable, value, start=0, stop=None): + "Return indices where a value occurs in a sequence or iterable." + # iter_index('AABCADEAF', 'A') → 0 1 4 7 + seq_index = getattr(iterable, 'index', None) + if seq_index is None: + iterator = islice(iterable, start, stop) + for i, element in enumerate(iterator, start): + if element is value or element == value: + yield i + else: + stop = len(iterable) if stop is None else stop + i = start + with suppress(ValueError): + while True: + yield (i := seq_index(value, i, stop)) + i += 1 - Converts a call-until-exception interface to an iterator interface. - """ - # iter_except(d.popitem, KeyError) --> non-blocking dictionary iterator - try: + def iter_except(function, exception, first=None): + "Convert a call-until-exception interface to an iterator interface." + # iter_except(d.popitem, KeyError) → non-blocking dictionary iterator + with suppress(exception): if first is not None: yield first() while True: - yield func() - except exception: - pass + yield function() The following recipes have a more mathematical flavor: @@ -992,74 +1010,74 @@ The following recipes have a more mathematical flavor: .. testcode:: def powerset(iterable): - "powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)" + "Subsequences of the iterable from shortest to longest." + # powerset([1,2,3]) → () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3) s = list(iterable) return chain.from_iterable(combinations(s, r) for r in range(len(s)+1)) - def sum_of_squares(it): + def sum_of_squares(iterable): "Add up the squares of the input values." - # sum_of_squares([10, 20, 30]) -> 1400 - return math.sumprod(*tee(it)) + # sum_of_squares([10, 20, 30]) → 1400 + return sumprod(*tee(iterable)) - def reshape(matrix, cols): + def reshape(matrix, columns): "Reshape a 2-D matrix to have a given number of columns." - # reshape([(0, 1), (2, 3), (4, 5)], 3) --> (0, 1, 2), (3, 4, 5) - return batched(chain.from_iterable(matrix), cols, strict=True) + # reshape([(0, 1), (2, 3), (4, 5)], 3) → (0, 1, 2), (3, 4, 5) + return batched(chain.from_iterable(matrix), columns, strict=True) def transpose(matrix): "Swap the rows and columns of a 2-D matrix." - # transpose([(1, 2, 3), (11, 22, 33)]) --> (1, 11) (2, 22) (3, 33) + # transpose([(1, 2, 3), (11, 22, 33)]) → (1, 11) (2, 22) (3, 33) return zip(*matrix, strict=True) def matmul(m1, m2): "Multiply two matrices." - # matmul([(7, 5), (3, 5)], [(2, 5), (7, 9)]) --> (49, 80), (41, 60) + # matmul([(7, 5), (3, 5)], [(2, 5), (7, 9)]) → (49, 80), (41, 60) n = len(m2[0]) - return batched(starmap(math.sumprod, product(m1, transpose(m2))), n) + return batched(starmap(sumprod, product(m1, transpose(m2))), n) def convolve(signal, kernel): """Discrete linear convolution of two iterables. + Equivalent to polynomial multiplication. - The kernel is fully consumed before the calculations begin. - The signal is consumed lazily and can be infinite. - - Convolutions are mathematically commutative. - If the signal and kernel are swapped, - the output will be the same. + Convolutions are mathematically commutative; however, the inputs are + evaluated differently. The signal is consumed lazily and can be + infinite. The kernel is fully consumed before the calculations begin. Article: https://betterexplained.com/articles/intuitive-convolution/ Video: https://www.youtube.com/watch?v=KuXjwB4LzSA """ - # convolve(data, [0.25, 0.25, 0.25, 0.25]) --> Moving average (blur) - # convolve(data, [1/2, 0, -1/2]) --> 1st derivative estimate - # convolve(data, [1, -2, 1]) --> 2nd derivative estimate + # convolve([1, -1, -20], [1, -3]) → 1 -4 -17 60 + # convolve(data, [0.25, 0.25, 0.25, 0.25]) → Moving average (blur) + # convolve(data, [1/2, 0, -1/2]) → 1st derivative estimate + # convolve(data, [1, -2, 1]) → 2nd derivative estimate kernel = tuple(kernel)[::-1] n = len(kernel) padded_signal = chain(repeat(0, n-1), signal, repeat(0, n-1)) windowed_signal = sliding_window(padded_signal, n) - return map(math.sumprod, repeat(kernel), windowed_signal) + return map(sumprod, repeat(kernel), windowed_signal) def polynomial_from_roots(roots): """Compute a polynomial's coefficients from its roots. (x - 5) (x + 4) (x - 3) expands to: x³ -4x² -17x + 60 """ - # polynomial_from_roots([5, -4, 3]) --> [1, -4, -17, 60] - factors = zip(repeat(1), map(operator.neg, roots)) - return list(functools.reduce(convolve, factors, [1])) + # polynomial_from_roots([5, -4, 3]) → [1, -4, -17, 60] + factors = zip(repeat(1), map(neg, roots)) + return list(reduce(convolve, factors, [1])) def polynomial_eval(coefficients, x): """Evaluate a polynomial at a specific value. Computes with better numeric stability than Horner's method. """ - # Evaluate x³ -4x² -17x + 60 at x = 2.5 - # polynomial_eval([1, -4, -17, 60], x=2.5) --> 8.125 + # Evaluate x³ -4x² -17x + 60 at x = 5 + # polynomial_eval([1, -4, -17, 60], x=5) → 0 n = len(coefficients) if not n: return type(x)(0) powers = map(pow, repeat(x), reversed(range(n))) - return math.sumprod(coefficients, powers) + return sumprod(coefficients, powers) def polynomial_derivative(coefficients): """Compute the first derivative of a polynomial. @@ -1067,31 +1085,27 @@ The following recipes have a more mathematical flavor: f(x) = x³ -4x² -17x + 60 f'(x) = 3x² -8x -17 """ - # polynomial_derivative([1, -4, -17, 60]) -> [3, -8, -17] + # polynomial_derivative([1, -4, -17, 60]) → [3, -8, -17] n = len(coefficients) powers = reversed(range(1, n)) - return list(map(operator.mul, coefficients, powers)) + return list(map(mul, coefficients, powers)) def sieve(n): "Primes less than n." - # sieve(30) --> 2 3 5 7 11 13 17 19 23 29 + # sieve(30) → 2 3 5 7 11 13 17 19 23 29 if n > 2: yield 2 - start = 3 data = bytearray((0, 1)) * (n // 2) - limit = math.isqrt(n) + 1 - for p in iter_index(data, 1, start, limit): - yield from iter_index(data, 1, start, p*p) + for p in iter_index(data, 1, start=3, stop=isqrt(n) + 1): data[p*p : n : p+p] = bytes(len(range(p*p, n, p+p))) - start = p*p - yield from iter_index(data, 1, start) + yield from iter_index(data, 1, start=3) def factor(n): "Prime factors of n." - # factor(99) --> 3 3 11 - # factor(1_000_000_000_000_007) --> 47 59 360620266859 - # factor(1_000_000_000_000_403) --> 1000000000000403 - for prime in sieve(math.isqrt(n) + 1): + # factor(99) → 3 3 11 + # factor(1_000_000_000_000_007) → 47 59 360620266859 + # factor(1_000_000_000_000_403) → 1000000000000403 + for prime in sieve(isqrt(n) + 1): while not n % prime: yield prime n //= prime @@ -1100,12 +1114,17 @@ The following recipes have a more mathematical flavor: if n > 1: yield n + def is_prime(n): + "Return True if n is prime." + # is_prime(1_000_000_000_000_403) → True + return n > 1 and next(factor(n)) == n + def totient(n): "Count of natural numbers up to n that are coprime to n." # https://mathworld.wolfram.com/TotientFunction.html - # totient(12) --> 4 because len([1, 5, 7, 11]) == 4 - for p in unique_justseen(factor(n)): - n -= n // p + # totient(12) → 4 because len([1, 5, 7, 11]) == 4 + for prime in set(factor(n)): + n -= n // prime return n @@ -1169,63 +1188,108 @@ The following recipes have a more mathematical flavor: >>> take(10, count()) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + >>> # Verify that the input is consumed lazily + >>> it = iter('abcdef') + >>> take(3, it) + ['a', 'b', 'c'] + >>> list(it) + ['d', 'e', 'f'] + >>> list(prepend(1, [2, 3, 4])) [1, 2, 3, 4] + >>> list(enumerate('abc')) [(0, 'a'), (1, 'b'), (2, 'c')] + >>> list(islice(tabulate(lambda x: 2*x), 4)) [0, 2, 4, 6] + + >>> for _ in loops(5): + ... print('hi') + ... + hi + hi + hi + hi + hi + + >>> list(tail(3, 'ABCDEFG')) ['E', 'F', 'G'] + >>> # Verify the input is consumed greedily + >>> input_iterator = iter('ABCDEFG') + >>> output_iterator = tail(3, input_iterator) + >>> list(input_iterator) + [] + >>> it = iter(range(10)) >>> consume(it, 3) + >>> # Verify the input is consumed lazily >>> next(it) 3 + >>> # Verify the input is consumed completely >>> consume(it) >>> next(it, 'Done') 'Done' + >>> nth('abcde', 3) 'd' - >>> nth('abcde', 9) is None True + >>> # Verify that the input is consumed lazily + >>> it = iter('abcde') + >>> nth(it, 2) + 'c' + >>> list(it) + ['d', 'e'] + >>> [all_equal(s) for s in ('', 'A', 'AAAA', 'AAAB', 'AAABA')] [True, True, True, False, False] >>> [all_equal(s, key=str.casefold) for s in ('', 'A', 'AaAa', 'AAAB', 'AAABA')] [True, True, True, False, False] + >>> # Verify that the input is consumed lazily and that only + >>> # one element of a second equivalence class is used to disprove + >>> # the assertion that all elements are equal. + >>> it = iter('aaabbbccc') + >>> all_equal(it) + False + >>> ''.join(it) + 'bbccc' + >>> quantify(range(99), lambda x: x%2==0) 50 - >>> quantify([True, False, False, True, True]) 3 - >>> quantify(range(12), predicate=lambda x: x%2==1) 6 + >>> a = [[1, 2, 3], [4, 5, 6]] >>> list(flatten(a)) [1, 2, 3, 4, 5, 6] - >>> list(repeatfunc(pow, 5, 2, 3)) - [8, 8, 8, 8, 8] - - >>> take(5, map(int, repeatfunc(random.random))) - [0, 0, 0, 0, 0] >>> list(ncycles('abc', 3)) ['a', 'b', 'c', 'a', 'b', 'c', 'a', 'b', 'c'] + >>> # Verify greedy consumption of input iterator + >>> input_iterator = iter('abc') + >>> output_iterator = ncycles(input_iterator, 3) + >>> list(input_iterator) + [] + >>> sum_of_squares([10, 20, 30]) 1400 + >>> list(reshape([(0, 1), (2, 3), (4, 5)], 3)) [(0, 1, 2), (3, 4, 5)] >>> M = [(0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11)] @@ -1246,14 +1310,27 @@ The following recipes have a more mathematical flavor: >>> list(reshape(M, 12)) [(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)] + >>> list(transpose([(1, 2, 3), (11, 22, 33)])) [(1, 11), (2, 22), (3, 33)] + >>> # Verify that the inputs are consumed lazily + >>> input1 = iter([1, 2, 3]) + >>> input2 = iter([11, 22, 33]) + >>> output_iterator = transpose([input1, input2]) + >>> next(output_iterator) + (1, 11) + >>> list(zip(input1, input2)) + [(2, 22), (3, 33)] + >>> list(matmul([(7, 5), (3, 5)], [[2, 5], [7, 9]])) [(49, 80), (41, 60)] >>> list(matmul([[2, 5], [7, 9], [3, 4]], [[7, 11, 5, 4, 9], [3, 5, 2, 6, 3]])) [(29, 47, 20, 38, 33), (76, 122, 53, 82, 90), (33, 53, 23, 36, 39)] + + >>> list(convolve([1, -1, -20], [1, -3])) == [1, -4, -17, 60] + True >>> data = [20, 40, 24, 32, 20, 28, 16] >>> list(convolve(data, [0.25, 0.25, 0.25, 0.25])) [5.0, 15.0, 21.0, 29.0, 29.0, 26.0, 24.0, 16.0, 11.0, 4.0] @@ -1261,13 +1338,26 @@ The following recipes have a more mathematical flavor: [20, 20, -16, 8, -12, 8, -12, -16] >>> list(convolve(data, [1, -2, 1])) [20, 0, -36, 24, -20, 20, -20, -4, 16] + >>> # Verify signal is consumed lazily and the kernel greedily + >>> signal_iterator = iter([10, 20, 30, 40, 50]) + >>> kernel_iterator = iter([1, 2, 3]) + >>> output_iterator = convolve(signal_iterator, kernel_iterator) + >>> list(kernel_iterator) + [] + >>> next(output_iterator) + 10 + >>> next(output_iterator) + 40 + >>> list(signal_iterator) + [30, 40, 50] + >>> from fractions import Fraction >>> from decimal import Decimal - >>> polynomial_eval([1, -4, -17, 60], x=2) - 18 - >>> x = 2; x**3 - 4*x**2 -17*x + 60 - 18 + >>> polynomial_eval([1, -4, -17, 60], x=5) + 0 + >>> x = 5; x**3 - 4*x**2 -17*x + 60 + 0 >>> polynomial_eval([1, -4, -17, 60], x=2.5) 8.125 >>> x = 2.5; x**3 - 4*x**2 -17*x + 60 @@ -1293,6 +1383,7 @@ The following recipes have a more mathematical flavor: >>> polynomial_eval([11, 2], 7) == 11 * 7 + 2 True + >>> polynomial_from_roots([5, -4, 3]) [1, -4, -17, 60] >>> factored = lambda x: (x - 5) * (x + 4) * (x - 3) @@ -1300,9 +1391,11 @@ The following recipes have a more mathematical flavor: >>> all(factored(x) == expanded(x) for x in range(-10, 11)) True + >>> polynomial_derivative([1, -4, -17, 60]) [3, -8, -17] + >>> list(iter_index('AABCADEAF', 'A')) [0, 1, 4, 7] >>> list(iter_index('AABCADEAF', 'B')) @@ -1348,6 +1441,35 @@ The following recipes have a more mathematical flavor: >>> # Test list input. Lists do not support None for the stop argument >>> list(iter_index(list('AABCADEAF'), 'A')) [0, 1, 4, 7] + >>> # Verify that input is consumed lazily + >>> input_iterator = iter('AABCADEAF') + >>> output_iterator = iter_index(input_iterator, 'A') + >>> next(output_iterator) + 0 + >>> next(output_iterator) + 1 + >>> next(output_iterator) + 4 + >>> ''.join(input_iterator) + 'DEAF' + + + >>> # Verify that the target value can be a sequence. + >>> seq = [[10, 20], [30, 40], 30, 40, [30, 40], 50] + >>> target = [30, 40] + >>> list(iter_index(seq, target)) + [1, 4] + + + >>> # Verify faithfulness to type specific index() method behaviors. + >>> # For example, bytes and str perform continuous-subsequence searches + >>> # that do not match the general behavior specified + >>> # in collections.abc.Sequence.index(). + >>> seq = 'abracadabra' + >>> target = 'ab' + >>> list(iter_index(seq, target)) + [0, 7] + >>> list(sieve(30)) [2, 3, 5, 7, 11, 13, 17, 19, 23, 29] @@ -1368,6 +1490,24 @@ The following recipes have a more mathematical flavor: >>> set(sieve(10_000)).isdisjoint(carmichael) True + + >>> small_primes = [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97] + >>> list(filter(is_prime, range(-100, 100))) == small_primes + True + >>> carmichael = {561, 1105, 1729, 2465, 2821, 6601, 8911} # https://oeis.org/A002997 + >>> any(map(is_prime, carmichael)) + False + >>> # https://www.wolframalpha.com/input?i=is+128884753939+prime + >>> is_prime(128_884_753_939) # large prime + True + >>> is_prime(999953 * 999983) # large semiprime + False + >>> is_prime(1_000_000_000_000_007) # factor() example + False + >>> is_prime(1_000_000_000_000_403) # factor() example + True + + >>> list(factor(99)) # Code example 1 [3, 3, 11] >>> list(factor(1_000_000_000_000_007)) # Code example 2 @@ -1413,6 +1553,7 @@ The following recipes have a more mathematical flavor: >>> all(list(factor(n)) == sorted(factor(n)) for n in range(2_000)) True + >>> totient(0) # https://www.wolframalpha.com/input?i=totient+0 0 >>> first_totients = [1, 1, 2, 2, 4, 2, 6, 4, 6, 4, 10, 4, 12, 6, 8, 8, 16, 6, @@ -1432,9 +1573,15 @@ The following recipes have a more mathematical flavor: >>> totient(6 ** 20) == 1 * 2**19 * 2 * 3**19 # repeated primes True + >>> list(flatten([('a', 'b'), (), ('c', 'd', 'e'), ('f',), ('g', 'h', 'i')])) ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i'] + + >>> list(repeatfunc(pow, 5, 2, 3)) + [8, 8, 8, 8, 8] + >>> take(5, map(int, repeatfunc(random.random))) + [0, 0, 0, 0, 0] >>> random.seed(85753098575309) >>> list(repeatfunc(random.random, 3)) [0.16370491282496968, 0.45889608687313455, 0.3747076837820118] @@ -1443,9 +1590,11 @@ The following recipes have a more mathematical flavor: >>> list(repeatfunc(pow, 3, 2, 5)) [32, 32, 32] + >>> list(grouper('abcdefg', 3, fillvalue='x')) [('a', 'b', 'c'), ('d', 'e', 'f'), ('g', 'x', 'x')] + >>> it = grouper('abcdefg', 3, incomplete='strict') >>> next(it) ('a', 'b', 'c') @@ -1459,6 +1608,7 @@ The following recipes have a more mathematical flavor: >>> list(grouper('abcdefg', n=3, incomplete='ignore')) [('a', 'b', 'c'), ('d', 'e', 'f')] + >>> list(sliding_window('ABCDEFG', 1)) [('A',), ('B',), ('C',), ('D',), ('E',), ('F',), ('G',)] >>> list(sliding_window('ABCDEFG', 2)) @@ -1488,36 +1638,47 @@ The following recipes have a more mathematical flavor: ... 'zero or negative n not supported' + >>> list(roundrobin('abc', 'd', 'ef')) ['a', 'd', 'e', 'b', 'f', 'c'] + >>> ranges = [range(5, 1000), range(4, 3000), range(0), range(3, 2000), range(2, 5000), range(1, 3500)] + >>> collections.Counter(roundrobin(*ranges)) == collections.Counter(chain(*ranges)) + True + >>> # Verify that the inputs are consumed lazily + >>> input_iterators = list(map(iter, ['abcd', 'ef', '', 'ghijk', 'l', 'mnopqr'])) + >>> output_iterator = roundrobin(*input_iterators) + >>> ''.join(islice(output_iterator, 10)) + 'aeglmbfhnc' + >>> ''.join(chain(*input_iterators)) + 'dijkopqr' - >>> def is_odd(x): - ... return x % 2 == 1 - - >>> evens, odds = partition(is_odd, range(10)) - >>> list(evens) - [0, 2, 4, 6, 8] - >>> list(odds) - [1, 3, 5, 7, 9] >>> list(subslices('ABCD')) ['A', 'AB', 'ABC', 'ABCD', 'B', 'BC', 'BCD', 'C', 'CD', 'D'] + >>> list(powerset([1,2,3])) [(), (1,), (2,), (3,), (1, 2), (1, 3), (2, 3), (1, 2, 3)] - >>> all(len(list(powerset(range(n)))) == 2**n for n in range(18)) True - >>> list(powerset('abcde')) == sorted(sorted(set(powerset('abcde'))), key=len) True + >>> list(unique_everseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D'] >>> list(unique_everseen('ABBCcAD', str.casefold)) ['A', 'B', 'C', 'D'] >>> list(unique_everseen('ABBcCAD', str.casefold)) ['A', 'B', 'c', 'D'] + >>> # Verify that the input is consumed lazily + >>> input_iterator = iter('AAAABBBCCDAABBB') + >>> output_iterator = unique_everseen(input_iterator) + >>> next(output_iterator) + 'A' + >>> ''.join(input_iterator) + 'AAABBBCCDAABBB' + >>> list(unique_justseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D', 'A', 'B'] @@ -1525,6 +1686,22 @@ The following recipes have a more mathematical flavor: ['A', 'B', 'C', 'A', 'D'] >>> list(unique_justseen('ABBcCAD', str.casefold)) ['A', 'B', 'c', 'A', 'D'] + >>> # Verify that the input is consumed lazily + >>> input_iterator = iter('AAAABBBCCDAABBB') + >>> output_iterator = unique_justseen(input_iterator) + >>> next(output_iterator) + 'A' + >>> ''.join(input_iterator) + 'AAABBBCCDAABBB' + + + >>> list(unique([[1, 2], [3, 4], [1, 2]])) + [[1, 2], [3, 4]] + >>> list(unique('ABBcCAD', str.casefold)) + ['A', 'B', 'c', 'D'] + >>> list(unique('ABBcCAD', str.casefold, reverse=True)) + ['D', 'c', 'B', 'A'] + >>> d = dict(a=1, b=2, c=3) >>> it = iter_except(d.popitem, KeyError) @@ -1543,8 +1720,15 @@ The following recipes have a more mathematical flavor: >>> next(it, 'empty') 'empty' + >>> first_true('ABC0DEF1', '9', str.isdigit) '0' + >>> # Verify that inputs are consumed lazily + >>> it = iter('ABC0DEF1') + >>> first_true(it, predicate=str.isdigit) + '0' + >>> ''.join(it) + 'DEF1' .. testcode:: @@ -1552,7 +1736,7 @@ The following recipes have a more mathematical flavor: # Old recipes and their tests which are guaranteed to continue to work. - def sumprod(vec1, vec2): + def old_sumprod_recipe(vec1, vec2): "Compute a sum of products." return sum(starmap(operator.mul, zip(vec1, vec2, strict=True))) @@ -1568,7 +1752,7 @@ The following recipes have a more mathematical flavor: def triplewise(iterable): "Return overlapping triplets from an iterable" - # triplewise('ABCDEFG') --> ABC BCD CDE DEF EFG + # triplewise('ABCDEFG') → ABC BCD CDE DEF EFG for (a, _), (b, c) in pairwise(pairwise(iterable)): yield a, b, c @@ -1617,21 +1801,36 @@ The following recipes have a more mathematical flavor: return true_iterator(), chain(transition, it) + def partition(predicate, iterable): + """Partition entries into false entries and true entries. + + If *predicate* is slow, consider wrapping it with functools.lru_cache(). + """ + # partition(is_odd, range(10)) → 0 2 4 6 8 and 1 3 5 7 9 + t1, t2 = tee(iterable) + return filterfalse(predicate, t1), filter(predicate, t2) + + + .. doctest:: :hide: >>> dotproduct([1,2,3], [4,5,6]) 32 - >>> sumprod([1,2,3], [4,5,6]) + + >>> old_sumprod_recipe([1,2,3], [4,5,6]) 32 + >>> list(islice(pad_none('abc'), 0, 6)) ['a', 'b', 'c', None, None, None] + >>> list(triplewise('ABCDEFG')) [('A', 'B', 'C'), ('B', 'C', 'D'), ('C', 'D', 'E'), ('D', 'E', 'F'), ('E', 'F', 'G')] + >>> population = 'ABCDEFGH' >>> for r in range(len(population) + 1): ... seq = list(combinations(population, r)) @@ -1639,16 +1838,38 @@ The following recipes have a more mathematical flavor: ... assert nth_combination(population, r, i) == seq[i] ... for i in range(-len(seq), 0): ... assert nth_combination(population, r, i) == seq[i] - + ... >>> iterable = 'abcde' >>> r = 3 >>> combos = list(combinations(iterable, r)) >>> all(nth_combination(iterable, r, i) == comb for i, comb in enumerate(combos)) True + >>> it = iter('ABCdEfGhI') >>> all_upper, remainder = before_and_after(str.isupper, it) >>> ''.join(all_upper) 'ABC' >>> ''.join(remainder) 'dEfGhI' + + + >>> def is_odd(x): + ... return x % 2 == 1 + ... + >>> evens, odds = partition(is_odd, range(10)) + >>> list(evens) + [0, 2, 4, 6, 8] + >>> list(odds) + [1, 3, 5, 7, 9] + >>> # Verify that the input is consumed lazily + >>> input_iterator = iter(range(10)) + >>> evens, odds = partition(is_odd, input_iterator) + >>> next(odds) + 1 + >>> next(odds) + 3 + >>> next(evens) + 0 + >>> list(input_iterator) + [4, 5, 6, 7, 8, 9] diff --git a/Doc/library/json.rst b/Doc/library/json.rst index c82ff9dc325b4c..4e7046d6d8f6ac 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -1,5 +1,5 @@ -:mod:`json` --- JSON encoder and decoder -======================================== +:mod:`!json` --- JSON encoder and decoder +========================================= .. module:: json :synopsis: Encode and decode the JSON format. @@ -13,7 +13,7 @@ `JSON (JavaScript Object Notation) `_, specified by :rfc:`7159` (which obsoletes :rfc:`4627`) and by -`ECMA-404 `_, +`ECMA-404 `_, is a lightweight data interchange format inspired by `JavaScript `_ object literal syntax (although it is not a strict subset of JavaScript [#rfc-errata]_ ). @@ -116,15 +116,15 @@ Extending :class:`JSONEncoder`:: ['[2.0', ', 1.0', ']'] -Using :mod:`json.tool` from the shell to validate and pretty-print: +Using :mod:`json` from the shell to validate and pretty-print: .. code-block:: shell-session - $ echo '{"json":"obj"}' | python -m json.tool + $ echo '{"json":"obj"}' | python -m json { "json": "obj" } - $ echo '{1.2:3.4}' | python -m json.tool + $ echo '{1.2:3.4}' | python -m json Expecting property name enclosed in double quotes: line 1 column 2 (char 1) See :ref:`json-commandline` for detailed documentation. @@ -151,69 +151,94 @@ Basic Usage sort_keys=False, **kw) Serialize *obj* as a JSON formatted stream to *fp* (a ``.write()``-supporting - :term:`file-like object`) using this :ref:`conversion table + :term:`file-like object`) using this :ref:`Python-to-JSON conversion table `. - If *skipkeys* is true (default: ``False``), then dict keys that are not - of a basic type (:class:`str`, :class:`int`, :class:`float`, :class:`bool`, - ``None``) will be skipped instead of raising a :exc:`TypeError`. - - The :mod:`json` module always produces :class:`str` objects, not - :class:`bytes` objects. Therefore, ``fp.write()`` must support :class:`str` - input. - - If *ensure_ascii* is true (the default), the output is guaranteed to - have all incoming non-ASCII characters escaped. If *ensure_ascii* is - false, these characters will be output as-is. - - If *check_circular* is false (default: ``True``), then the circular - reference check for container types will be skipped and a circular reference - will result in a :exc:`RecursionError` (or worse). + .. note:: - If *allow_nan* is false (default: ``True``), then it will be a - :exc:`ValueError` to serialize out of range :class:`float` values (``nan``, - ``inf``, ``-inf``) in strict compliance of the JSON specification. - If *allow_nan* is true, their JavaScript equivalents (``NaN``, - ``Infinity``, ``-Infinity``) will be used. + Unlike :mod:`pickle` and :mod:`marshal`, JSON is not a framed protocol, + so trying to serialize multiple objects with repeated calls to + :func:`dump` using the same *fp* will result in an invalid JSON file. - If *indent* is a non-negative integer or string, then JSON array elements and - object members will be pretty-printed with that indent level. An indent level - of 0, negative, or ``""`` will only insert newlines. ``None`` (the default) - selects the most compact representation. Using a positive integer indent - indents that many spaces per level. If *indent* is a string (such as ``"\t"``), - that string is used to indent each level. + :param object obj: + The Python object to be serialized. + + :param fp: + The file-like object *obj* will be serialized to. + The :mod:`!json` module always produces :class:`str` objects, + not :class:`bytes` objects, + therefore ``fp.write()`` must support :class:`str` input. + :type fp: :term:`file-like object` + + :param bool skipkeys: + If ``True``, keys that are not of a basic type + (:class:`str`, :class:`int`, :class:`float`, :class:`bool`, ``None``) + will be skipped instead of raising a :exc:`TypeError`. + Default ``False``. + + :param bool ensure_ascii: + If ``True`` (the default), the output is guaranteed to + have all incoming non-ASCII characters escaped. + If ``False``, these characters will be outputted as-is. + + :param bool check_circular: + If ``False``, the circular reference check for container types is skipped + and a circular reference will result in a :exc:`RecursionError` (or worse). + Default ``True``. + + :param bool allow_nan: + If ``False``, serialization of out-of-range :class:`float` values + (``nan``, ``inf``, ``-inf``) will result in a :exc:`ValueError`, + in strict compliance with the JSON specification. + If ``True`` (the default), their JavaScript equivalents + (``NaN``, ``Infinity``, ``-Infinity``) are used. + + :param cls: + If set, a custom JSON encoder with the + :meth:`~JSONEncoder.default` method overridden, + for serializing into custom datatypes. + If ``None`` (the default), :class:`!JSONEncoder` is used. + :type cls: a :class:`JSONEncoder` subclass + + :param indent: + If a positive integer or string, JSON array elements and + object members will be pretty-printed with that indent level. + A positive integer indents that many spaces per level; + a string (such as ``"\t"``) is used to indent each level. + If zero, negative, or ``""`` (the empty string), + only newlines are inserted. + If ``None`` (the default), the most compact representation is used. + :type indent: int | str | None + + :param separators: + A two-tuple: ``(item_separator, key_separator)``. + If ``None`` (the default), *separators* defaults to + ``(', ', ': ')`` if *indent* is ``None``, + and ``(',', ': ')`` otherwise. + For the most compact JSON, + specify ``(',', ':')`` to eliminate whitespace. + :type separators: tuple | None + + :param default: + A function that is called for objects that can't otherwise be serialized. + It should return a JSON encodable version of the object + or raise a :exc:`TypeError`. + If ``None`` (the default), :exc:`!TypeError` is raised. + :type default: :term:`callable` | None + + :param bool sort_keys: + If ``True``, dictionaries will be outputted sorted by key. + Default ``False``. .. versionchanged:: 3.2 Allow strings for *indent* in addition to integers. - If specified, *separators* should be an ``(item_separator, key_separator)`` - tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and - ``(',', ': ')`` otherwise. To get the most compact JSON representation, - you should specify ``(',', ':')`` to eliminate whitespace. - .. versionchanged:: 3.4 Use ``(',', ': ')`` as default if *indent* is not ``None``. - If specified, *default* should be a function that gets called for objects that - can't otherwise be serialized. It should return a JSON encodable version of - the object or raise a :exc:`TypeError`. If not specified, :exc:`TypeError` - is raised. - - If *sort_keys* is true (default: ``False``), then the output of - dictionaries will be sorted by key. - - To use a custom :class:`JSONEncoder` subclass (e.g. one that overrides the - :meth:`~JSONEncoder.default` method to serialize additional types), specify it with the - *cls* kwarg; otherwise :class:`JSONEncoder` is used. - .. versionchanged:: 3.6 All optional parameters are now :ref:`keyword-only `. - .. note:: - - Unlike :mod:`pickle` and :mod:`marshal`, JSON is not a framed protocol, - so trying to serialize multiple objects with repeated calls to - :func:`dump` using the same *fp* will result in an invalid JSON file. .. function:: dumps(obj, *, skipkeys=False, ensure_ascii=True, \ check_circular=True, allow_nan=True, cls=None, \ @@ -233,36 +258,86 @@ Basic Usage the original one. That is, ``loads(dumps(x)) != x`` if x has non-string keys. -.. function:: load(fp, *, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw) - - Deserialize *fp* (a ``.read()``-supporting :term:`text file` or - :term:`binary file` containing a JSON document) to a Python object using - this :ref:`conversion table `. - - *object_hook* is an optional function that will be called with the result of - any object literal decoded (a :class:`dict`). The return value of - *object_hook* will be used instead of the :class:`dict`. This feature can be used - to implement custom decoders (e.g. `JSON-RPC `_ - class hinting). - - *object_pairs_hook* is an optional function that will be called with the - result of any object literal decoded with an ordered list of pairs. The - return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders. - If *object_hook* is also defined, the *object_pairs_hook* takes priority. +.. function:: load(fp, *, cls=None, object_hook=None, parse_float=None, \ + parse_int=None, parse_constant=None, \ + object_pairs_hook=None, **kw) + + Deserialize *fp* to a Python object + using the :ref:`JSON-to-Python conversion table `. + + :param fp: + A ``.read()``-supporting :term:`text file` or :term:`binary file` + containing the JSON document to be deserialized. + :type fp: :term:`file-like object` + + :param cls: + If set, a custom JSON decoder. + Additional keyword arguments to :func:`!load` + will be passed to the constructor of *cls*. + If ``None`` (the default), :class:`!JSONDecoder` is used. + :type cls: a :class:`JSONDecoder` subclass + + :param object_hook: + If set, a function that is called with the result of + any object literal decoded (a :class:`dict`). + The return value of this function will be used + instead of the :class:`dict`. + This feature can be used to implement custom decoders, + for example `JSON-RPC `_ class hinting. + Default ``None``. + :type object_hook: :term:`callable` | None + + :param object_pairs_hook: + If set, a function that is called with the result of + any object literal decoded with an ordered list of pairs. + The return value of this function will be used + instead of the :class:`dict`. + This feature can be used to implement custom decoders. + If *object_hook* is also set, *object_pairs_hook* takes priority. + Default ``None``. + :type object_pairs_hook: :term:`callable` | None + + :param parse_float: + If set, a function that is called with + the string of every JSON float to be decoded. + If ``None`` (the default), it is equivalent to ``float(num_str)``. + This can be used to parse JSON floats into custom datatypes, + for example :class:`decimal.Decimal`. + :type parse_float: :term:`callable` | None + + :param parse_int: + If set, a function that is called with + the string of every JSON int to be decoded. + If ``None`` (the default), it is equivalent to ``int(num_str)``. + This can be used to parse JSON integers into custom datatypes, + for example :class:`float`. + :type parse_int: :term:`callable` | None + + :param parse_constant: + If set, a function that is called with one of the following strings: + ``'-Infinity'``, ``'Infinity'``, or ``'NaN'``. + This can be used to raise an exception + if invalid JSON numbers are encountered. + Default ``None``. + :type parse_constant: :term:`callable` | None + + :raises JSONDecodeError: + When the data being deserialized is not a valid JSON document. + + :raises UnicodeDecodeError: + When the data being deserialized does not contain + UTF-8, UTF-16 or UTF-32 encoded data. .. versionchanged:: 3.1 - Added support for *object_pairs_hook*. - *parse_float*, if specified, will be called with the string of every JSON - float to be decoded. By default, this is equivalent to ``float(num_str)``. - This can be used to use another datatype or parser for JSON floats - (e.g. :class:`decimal.Decimal`). + * Added the optional *object_pairs_hook* parameter. + * *parse_constant* doesn't get called on 'null', 'true', 'false' anymore. + + .. versionchanged:: 3.6 - *parse_int*, if specified, will be called with the string of every JSON int - to be decoded. By default, this is equivalent to ``int(num_str)``. This can - be used to use another datatype or parser for JSON integers - (e.g. :class:`float`). + * All optional parameters are now :ref:`keyword-only `. + * *fp* can now be a :term:`binary file`. + The input encoding should be UTF-8, UTF-16 or UTF-32. .. versionchanged:: 3.11 The default *parse_int* of :func:`int` now limits the maximum length of @@ -270,39 +345,13 @@ Basic Usage conversion length limitation ` to help avoid denial of service attacks. - *parse_constant*, if specified, will be called with one of the following - strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. - This can be used to raise an exception if invalid JSON numbers - are encountered. - - .. versionchanged:: 3.1 - *parse_constant* doesn't get called on 'null', 'true', 'false' anymore. - - To use a custom :class:`JSONDecoder` subclass, specify it with the ``cls`` - kwarg; otherwise :class:`JSONDecoder` is used. Additional keyword arguments - will be passed to the constructor of the class. - - If the data being deserialized is not a valid JSON document, a - :exc:`JSONDecodeError` will be raised. - - .. versionchanged:: 3.6 - All optional parameters are now :ref:`keyword-only `. - - .. versionchanged:: 3.6 - *fp* can now be a :term:`binary file`. The input encoding should be - UTF-8, UTF-16 or UTF-32. - .. function:: loads(s, *, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw) - Deserialize *s* (a :class:`str`, :class:`bytes` or :class:`bytearray` + Identical to :func:`load`, but instead of a file-like object, + deserialize *s* (a :class:`str`, :class:`bytes` or :class:`bytearray` instance containing a JSON document) to a Python object using this :ref:`conversion table `. - The other arguments have the same meaning as in :func:`load`. - - If the data being deserialized is not a valid JSON document, a - :exc:`JSONDecodeError` will be raised. - .. versionchanged:: 3.6 *s* can now be of type :class:`bytes` or :class:`bytearray`. The input encoding should be UTF-8, UTF-16 or UTF-32. @@ -345,34 +394,33 @@ Encoders and Decoders It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as their corresponding ``float`` values, which is outside the JSON spec. - *object_hook*, if specified, will be called with the result of every JSON - object decoded and its return value will be used in place of the given - :class:`dict`. This can be used to provide custom deserializations (e.g. to - support `JSON-RPC `_ class hinting). + *object_hook* is an optional function that will be called with the result of + every JSON object decoded and its return value will be used in place of the + given :class:`dict`. This can be used to provide custom deserializations + (e.g. to support `JSON-RPC `_ class hinting). - *object_pairs_hook*, if specified will be called with the result of every - JSON object decoded with an ordered list of pairs. The return value of - *object_pairs_hook* will be used instead of the :class:`dict`. This - feature can be used to implement custom decoders. If *object_hook* is also - defined, the *object_pairs_hook* takes priority. + *object_pairs_hook* is an optional function that will be called with the + result of every JSON object decoded with an ordered list of pairs. The + return value of *object_pairs_hook* will be used instead of the + :class:`dict`. This feature can be used to implement custom decoders. If + *object_hook* is also defined, the *object_pairs_hook* takes priority. .. versionchanged:: 3.1 Added support for *object_pairs_hook*. - *parse_float*, if specified, will be called with the string of every JSON - float to be decoded. By default, this is equivalent to ``float(num_str)``. - This can be used to use another datatype or parser for JSON floats - (e.g. :class:`decimal.Decimal`). + *parse_float* is an optional function that will be called with the string of + every JSON float to be decoded. By default, this is equivalent to + ``float(num_str)``. This can be used to use another datatype or parser for + JSON floats (e.g. :class:`decimal.Decimal`). - *parse_int*, if specified, will be called with the string of every JSON int - to be decoded. By default, this is equivalent to ``int(num_str)``. This can - be used to use another datatype or parser for JSON integers - (e.g. :class:`float`). + *parse_int* is an optional function that will be called with the string of + every JSON int to be decoded. By default, this is equivalent to + ``int(num_str)``. This can be used to use another datatype or parser for + JSON integers (e.g. :class:`float`). - *parse_constant*, if specified, will be called with one of the following - strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. - This can be used to raise an exception if invalid JSON numbers - are encountered. + *parse_constant* is an optional function that will be called with one of the + following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This can be + used to raise an exception if invalid JSON numbers are encountered. If *strict* is false (``True`` is the default), then control characters will be allowed inside strings. Control characters in this context are @@ -559,7 +607,7 @@ Standard Compliance and Interoperability ---------------------------------------- The JSON format is specified by :rfc:`7159` and by -`ECMA-404 `_. +`ECMA-404 `_. This section details this module's level of compliance with the RFC. For simplicity, :class:`JSONEncoder` and :class:`JSONDecoder` subclasses, and parameters other than those explicitly mentioned, are not considered. @@ -678,31 +726,32 @@ when serializing instances of "exotic" numerical types such as .. _json-commandline: -.. program:: json.tool +.. program:: json -Command Line Interface +Command-line interface ---------------------- .. module:: json.tool - :synopsis: A command line to validate and pretty-print JSON. + :synopsis: A command-line interface to validate and pretty-print JSON. **Source code:** :source:`Lib/json/tool.py` -------------- -The :mod:`json.tool` module provides a simple command line interface to validate -and pretty-print JSON objects. +The :mod:`json` module can be invoked as a script via ``python -m json`` +to validate and pretty-print JSON objects. The :mod:`json.tool` submodule +implements this interface. If the optional ``infile`` and ``outfile`` arguments are not specified, :data:`sys.stdin` and :data:`sys.stdout` will be used respectively: .. code-block:: shell-session - $ echo '{"json": "obj"}' | python -m json.tool + $ echo '{"json": "obj"}' | python -m json { "json": "obj" } - $ echo '{1.2:3.4}' | python -m json.tool + $ echo '{1.2:3.4}' | python -m json Expecting property name enclosed in double quotes: line 1 column 2 (char 1) .. versionchanged:: 3.5 @@ -710,8 +759,13 @@ specified, :data:`sys.stdin` and :data:`sys.stdout` will be used respectively: :option:`--sort-keys` option to sort the output of dictionaries alphabetically by key. +.. versionchanged:: 3.14 + The :mod:`json` module may now be directly executed as + ``python -m json``. For backwards compatibility, invoking + the CLI as ``python -m json.tool`` remains supported. + -Command line options +Command-line options ^^^^^^^^^^^^^^^^^^^^ .. option:: infile @@ -720,7 +774,7 @@ Command line options .. code-block:: shell-session - $ python -m json.tool mp_films.json + $ python -m json mp_films.json [ { "title": "And Now for Something Completely Different", diff --git a/Doc/library/keyword.rst b/Doc/library/keyword.rst index c3b4699cb05af6..ac57140f888024 100644 --- a/Doc/library/keyword.rst +++ b/Doc/library/keyword.rst @@ -1,5 +1,5 @@ -:mod:`keyword` --- Testing for Python keywords -============================================== +:mod:`!keyword` --- Testing for Python keywords +=============================================== .. module:: keyword :synopsis: Test whether a string is a keyword in Python. diff --git a/Doc/library/linecache.rst b/Doc/library/linecache.rst index dd9f4ee45ba82e..88c6079a05b7fa 100644 --- a/Doc/library/linecache.rst +++ b/Doc/library/linecache.rst @@ -1,5 +1,5 @@ -:mod:`linecache` --- Random access to text lines -================================================ +:mod:`!linecache` --- Random access to text lines +================================================= .. module:: linecache :synopsis: Provides random access to individual lines from text files. diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index 414979524e57b6..426e3a06e1ef11 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -1,5 +1,5 @@ -:mod:`locale` --- Internationalization services -=============================================== +:mod:`!locale` --- Internationalization services +================================================ .. module:: locale :synopsis: Internationalization services. @@ -281,7 +281,8 @@ The :mod:`locale` module defines the following exception and functions: .. data:: ERA - Get a string that represents the era used in the current locale. + Get a string which describes how years are counted and displayed for + each era in a locale. Most locales do not define this value. An example of a locale which does define this value is the Japanese one. In Japan, the traditional @@ -290,9 +291,10 @@ The :mod:`locale` module defines the following exception and functions: Normally it should not be necessary to use this value directly. Specifying the ``E`` modifier in their format strings causes the :func:`time.strftime` - function to use this information. The format of the returned string is not - specified, and therefore you should not assume knowledge of it on different - systems. + function to use this information. + The format of the returned string is specified in *The Open Group Base + Specifications Issue 8*, paragraph `7.3.5.2 LC_TIME C-Language Access + `_. .. data:: ERA_D_T_FMT @@ -311,8 +313,18 @@ The :mod:`locale` module defines the following exception and functions: .. data:: ALT_DIGITS - Get a representation of up to 100 values used to represent the values - 0 to 99. + Get a string consisting of up to 100 semicolon-separated symbols used + to represent the values 0 to 99 in a locale-specific way. + In most locales this is an empty string. + + The function temporarily sets the ``LC_CTYPE`` locale to the locale + of the category that determines the requested value (``LC_TIME``, + ``LC_NUMERIC``, ``LC_MONETARY`` or ``LC_MESSAGES``) if locales are + different and the resulting string is non-ASCII. + This temporary change affects other threads. + + .. versionchanged:: 3.14 + The function now temporarily sets the ``LC_CTYPE`` locale in some cases. .. function:: getdefaultlocale([envvars]) @@ -424,7 +436,7 @@ The :mod:`locale` module defines the following exception and functions: .. function:: format_string(format, val, grouping=False, monetary=False) Formats a number *val* according to the current :const:`LC_NUMERIC` setting. - The format follows the conventions of the ``%`` operator. For floating point + The format follows the conventions of the ``%`` operator. For floating-point values, the decimal point is modified if appropriate. If *grouping* is ``True``, also takes the grouping into account. @@ -455,7 +467,7 @@ The :mod:`locale` module defines the following exception and functions: .. function:: str(float) - Formats a floating point number using the same format as the built-in function + Formats a floating-point number using the same format as the built-in function ``str(float)``, but takes the decimal point into account. diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 13850c91446da5..0e9dc33ae2123a 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -1,5 +1,5 @@ -:mod:`logging.config` --- Logging configuration -=============================================== +:mod:`!logging.config` --- Logging configuration +================================================ .. module:: logging.config :synopsis: Configuration of the logging module. @@ -69,7 +69,7 @@ in :mod:`logging` itself) and defining handlers which are declared either in dictConfigClass(config).configure() For example, a subclass of :class:`DictConfigurator` could call - ``DictConfigurator.__init__()`` in its own :meth:`__init__()`, then + ``DictConfigurator.__init__()`` in its own :meth:`__init__`, then set up custom prefixes which would be usable in the subsequent :meth:`configure` call. :attr:`dictConfigClass` would be bound to this new subclass, and then :func:`dictConfig` could be called exactly as @@ -753,13 +753,17 @@ The ``queue`` and ``listener`` keys are optional. If the ``queue`` key is present, the corresponding value can be one of the following: -* An actual instance of :class:`queue.Queue` or a subclass thereof. This is of course - only possible if you are constructing or modifying the configuration dictionary in - code. +* An object implementing the :meth:`Queue.put_nowait ` + and :meth:`Queue.get ` public API. For instance, this may be + an actual instance of :class:`queue.Queue` or a subclass thereof, or a proxy + obtained by :meth:`multiprocessing.managers.SyncManager.Queue`. + + This is of course only possible if you are constructing or modifying + the configuration dictionary in code. * A string that resolves to a callable which, when called with no arguments, returns - the :class:`queue.Queue` instance to use. That callable could be a - :class:`queue.Queue` subclass or a function which returns a suitable queue instance, + the queue instance to use. That callable could be a :class:`queue.Queue` subclass + or a function which returns a suitable queue instance, such as ``my.module.queue_factory()``. * A dict with a ``'()'`` key which is constructed in the usual way as discussed in diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 2fe9370333beaf..ffb54591b3563b 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -1,5 +1,5 @@ -:mod:`logging.handlers` --- Logging handlers -============================================ +:mod:`!logging.handlers` --- Logging handlers +============================================= .. module:: logging.handlers :synopsis: Handlers for the logging module. @@ -66,7 +66,7 @@ and :meth:`flush` methods). :param stream: The stream that the handler should use. - :return: the old stream, if the stream was changed, or *None* if it wasn't. + :return: the old stream, if the stream was changed, or ``None`` if it wasn't. .. versionadded:: 3.7 @@ -613,7 +613,7 @@ The :class:`SysLogHandler` class, located in the :mod:`logging.handlers` module, supports sending logging messages to a remote or local Unix syslog. -.. class:: SysLogHandler(address=('localhost', SYSLOG_UDP_PORT), facility=LOG_USER, socktype=socket.SOCK_DGRAM) +.. class:: SysLogHandler(address=('localhost', SYSLOG_UDP_PORT), facility=LOG_USER, socktype=socket.SOCK_DGRAM, timeout=None) Returns a new instance of the :class:`SysLogHandler` class intended to communicate with a remote Unix machine whose address is given by *address* in @@ -626,6 +626,11 @@ supports sending logging messages to a remote or local Unix syslog. *socktype* argument, which defaults to :const:`socket.SOCK_DGRAM` and thus opens a UDP socket. To open a TCP socket (for use with the newer syslog daemons such as rsyslog), specify a value of :const:`socket.SOCK_STREAM`. + If *timeout* is specified, it sets a timeout (in seconds) for the socket operations. + This can help prevent the program from hanging indefinitely if the syslog server is + unreachable. By default, *timeout* is ``None``, meaning no timeout is applied. + + Note that if your server is not listening on UDP port 514, :class:`SysLogHandler` may appear not to work. In that case, check what @@ -645,6 +650,8 @@ supports sending logging messages to a remote or local Unix syslog. .. versionchanged:: 3.2 *socktype* was added. + .. versionchanged:: 3.14 + *timeout* was added. .. method:: close() diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index e38b7435498507..235bcc281ac8f8 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -1,5 +1,5 @@ -:mod:`logging` --- Logging facility for Python -============================================== +:mod:`!logging` --- Logging facility for Python +=============================================== .. module:: logging :synopsis: Flexible event logging system for applications. @@ -30,20 +30,61 @@ is that all Python modules can participate in logging, so your application log can include your own messages integrated with messages from third-party modules. -The simplest example: +Here's a simple example of idiomatic usage: :: + + # myapp.py + import logging + import mylib + logger = logging.getLogger(__name__) + + def main(): + logging.basicConfig(filename='myapp.log', level=logging.INFO) + logger.info('Started') + mylib.do_something() + logger.info('Finished') + + if __name__ == '__main__': + main() + +:: + + # mylib.py + import logging + logger = logging.getLogger(__name__) + + def do_something(): + logger.info('Doing something') + +If you run *myapp.py*, you should see this in *myapp.log*: .. code-block:: none - >>> import logging - >>> logging.warning('Watch out!') - WARNING:root:Watch out! + INFO:__main__:Started + INFO:mylib:Doing something + INFO:__main__:Finished + +The key feature of this idiomatic usage is that the majority of code is simply +creating a module level logger with ``getLogger(__name__)``, and using that +logger to do any needed logging. This is concise, while allowing downstream +code fine-grained control if needed. Logged messages to the module-level logger +get forwarded to handlers of loggers in higher-level modules, all the way up to +the highest-level logger known as the root logger; this approach is known as +hierarchical logging. + +For logging to be useful, it needs to be configured: setting the levels and +destinations for each logger, potentially changing how specific modules log, +often based on command-line arguments or application configuration. In most +cases, like the one above, only the root logger needs to be so configured, since +all the lower level loggers at module level eventually forward their messages to +its handlers. :func:`~logging.basicConfig` provides a quick way to configure +the root logger that handles many use cases. The module provides a lot of functionality and flexibility. If you are unfamiliar with logging, the best way to get to grips with it is to view the tutorials (**see the links above and on the right**). -The basic classes defined by the module, together with their functions, are -listed below. +The basic classes defined by the module, together with their attributes and +methods, are listed in the sections below. * Loggers expose the interface that application code directly uses. * Handlers send the log records (created by loggers) to the appropriate @@ -68,11 +109,11 @@ The ``name`` is potentially a period-separated hierarchical value, like Loggers that are further down in the hierarchical list are children of loggers higher up in the list. For example, given a logger with a name of ``foo``, loggers with names of ``foo.bar``, ``foo.bar.baz``, and ``foo.bam`` are all -descendants of ``foo``. The logger name hierarchy is analogous to the Python -package hierarchy, and identical to it if you organise your loggers on a -per-module basis using the recommended construction -``logging.getLogger(__name__)``. That's because in a module, ``__name__`` -is the module's name in the Python package namespace. +descendants of ``foo``. In addition, all loggers are descendants of the root +logger. The logger name hierarchy is analogous to the Python package hierarchy, +and identical to it if you organise your loggers on a per-module basis using +the recommended construction ``logging.getLogger(__name__)``. That's because +in a module, ``__name__`` is the module's name in the Python package namespace. .. class:: Logger @@ -263,7 +304,8 @@ is the module's name in the Python package namespace. parameter mirrors the equivalent one in the :mod:`warnings` module. The fourth keyword argument is *extra* which can be used to pass a - dictionary which is used to populate the __dict__ of the :class:`LogRecord` + dictionary which is used to populate the :attr:`~object.__dict__` of the + :class:`LogRecord` created for the logging event with user-defined attributes. These custom attributes can then be used as you like. For example, they could be incorporated into logged messages. For example:: @@ -311,10 +353,6 @@ is the module's name in the Python package namespace. .. versionchanged:: 3.8 The *stacklevel* parameter was added. - .. versionchanged:: 3.13 - Remove the undocumented ``warn()`` method which was an alias to the - :meth:`warning` method. - .. method:: Logger.info(msg, *args, **kwargs) @@ -327,6 +365,10 @@ is the module's name in the Python package namespace. Logs a message with level :const:`WARNING` on this logger. The arguments are interpreted as for :meth:`debug`. + .. note:: There is an obsolete method ``warn`` which is functionally + identical to ``warning``. As ``warn`` is deprecated, please do not use + it - use ``warning`` instead. + .. method:: Logger.error(msg, *args, **kwargs) Logs a message with level :const:`ERROR` on this logger. The arguments are @@ -962,7 +1004,7 @@ the options available to you. | | | portion of the time). | +----------------+-------------------------+-----------------------------------------------+ | created | ``%(created)f`` | Time when the :class:`LogRecord` was created | -| | | (as returned by :func:`time.time`). | +| | | (as returned by :func:`time.time_ns` / 1e9). | +----------------+-------------------------+-----------------------------------------------+ | exc_info | You shouldn't need to | Exception tuple (à la ``sys.exc_info``) or, | | | format this yourself. | if no exception has occurred, ``None``. | @@ -1057,11 +1099,11 @@ information into logging calls. For a usage example, see the section on .. attribute:: manager - Delegates to the underlying :attr:`!manager`` on *logger*. + Delegates to the underlying :attr:`!manager` on *logger*. .. attribute:: _log - Delegates to the underlying :meth:`!_log`` method on *logger*. + Delegates to the underlying :meth:`!_log` method on *logger*. In addition to the above, :class:`LoggerAdapter` supports the following methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, @@ -1083,11 +1125,6 @@ information into logging calls. For a usage example, see the section on Attribute :attr:`!manager` and method :meth:`!_log` were added, which delegate to the underlying logger and allow adapters to be nested. - .. versionchanged:: 3.13 - - Remove the undocumented :meth:`!warn`` method which was an alias to the - :meth:`!warning` method. - .. versionchanged:: 3.13 The *merge_extra* argument was added. @@ -1116,10 +1153,12 @@ functions. .. function:: getLogger(name=None) - Return a logger with the specified name or, if name is ``None``, return a - logger which is the root logger of the hierarchy. If specified, the name is - typically a dot-separated hierarchical name like *'a'*, *'a.b'* or *'a.b.c.d'*. - Choice of these names is entirely up to the developer who is using logging. + Return a logger with the specified name or, if name is ``None``, return the + root logger of the hierarchy. If specified, the name is typically a + dot-separated hierarchical name like *'a'*, *'a.b'* or *'a.b.c.d'*. Choice + of these names is entirely up to the developer who is using logging, though + it is recommended that ``__name__`` be used unless you have a specific + reason for not doing that, as mentioned in :ref:`logger`. All calls to this function with a given name return the same logger instance. This means that logger instances never need to be passed between different parts @@ -1151,121 +1190,59 @@ functions. .. function:: debug(msg, *args, **kwargs) - Logs a message with level :const:`DEBUG` on the root logger. The *msg* is the - message format string, and the *args* are the arguments which are merged into - *msg* using the string formatting operator. (Note that this means that you can - use keywords in the format string, together with a single dictionary argument.) - - There are three keyword arguments in *kwargs* which are inspected: *exc_info* - which, if it does not evaluate as false, causes exception information to be - added to the logging message. If an exception tuple (in the format returned by - :func:`sys.exc_info`) or an exception instance is provided, it is used; - otherwise, :func:`sys.exc_info` is called to get the exception information. - - The second optional keyword argument is *stack_info*, which defaults to - ``False``. If true, stack information is added to the logging - message, including the actual logging call. Note that this is not the same - stack information as that displayed through specifying *exc_info*: The - former is stack frames from the bottom of the stack up to the logging call - in the current thread, whereas the latter is information about stack frames - which have been unwound, following an exception, while searching for - exception handlers. + This is a convenience function that calls :meth:`Logger.debug`, on the root + logger. The handling of the arguments is in every way identical + to what is described in that method. - You can specify *stack_info* independently of *exc_info*, e.g. to just show - how you got to a certain point in your code, even when no exceptions were - raised. The stack frames are printed following a header line which says: + The only difference is that if the root logger has no handlers, then + :func:`basicConfig` is called, prior to calling ``debug`` on the root logger. - .. code-block:: none + For very short scripts or quick demonstrations of ``logging`` facilities, + ``debug`` and the other module-level functions may be convenient. However, + most programs will want to carefully and explicitly control the logging + configuration, and should therefore prefer creating a module-level logger and + calling :meth:`Logger.debug` (or other level-specific methods) on it, as + described at the beginning of this documentation. - Stack (most recent call last): - - This mimics the ``Traceback (most recent call last):`` which is used when - displaying exception frames. - - The third optional keyword argument is *extra* which can be used to pass a - dictionary which is used to populate the __dict__ of the LogRecord created for - the logging event with user-defined attributes. These custom attributes can then - be used as you like. For example, they could be incorporated into logged - messages. For example:: - - FORMAT = '%(asctime)s %(clientip)-15s %(user)-8s %(message)s' - logging.basicConfig(format=FORMAT) - d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} - logging.warning('Protocol problem: %s', 'connection reset', extra=d) - - would print something like: - - .. code-block:: none - - 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset - - The keys in the dictionary passed in *extra* should not clash with the keys used - by the logging system. (See the :class:`Formatter` documentation for more - information on which keys are used by the logging system.) - - If you choose to use these attributes in logged messages, you need to exercise - some care. In the above example, for instance, the :class:`Formatter` has been - set up with a format string which expects 'clientip' and 'user' in the attribute - dictionary of the LogRecord. If these are missing, the message will not be - logged because a string formatting exception will occur. So in this case, you - always need to pass the *extra* dictionary with these keys. - - While this might be annoying, this feature is intended for use in specialized - circumstances, such as multi-threaded servers where the same code executes in - many contexts, and interesting conditions which arise are dependent on this - context (such as remote client IP address and authenticated user name, in the - above example). In such circumstances, it is likely that specialized - :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. - - This function (as well as :func:`info`, :func:`warning`, :func:`error` and - :func:`critical`) will call :func:`basicConfig` if the root logger doesn't - have any handler attached. - - .. versionchanged:: 3.2 - The *stack_info* parameter was added. .. function:: info(msg, *args, **kwargs) - Logs a message with level :const:`INFO` on the root logger. The arguments are - interpreted as for :func:`debug`. + Logs a message with level :const:`INFO` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. function:: warning(msg, *args, **kwargs) - Logs a message with level :const:`WARNING` on the root logger. The arguments - are interpreted as for :func:`debug`. + Logs a message with level :const:`WARNING` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. note:: There is an obsolete function ``warn`` which is functionally identical to ``warning``. As ``warn`` is deprecated, please do not use it - use ``warning`` instead. - .. versionchanged:: 3.13 - Remove the undocumented ``warn()`` function which was an alias to the - :func:`warning` function. - .. function:: error(msg, *args, **kwargs) - Logs a message with level :const:`ERROR` on the root logger. The arguments are - interpreted as for :func:`debug`. + Logs a message with level :const:`ERROR` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. function:: critical(msg, *args, **kwargs) - Logs a message with level :const:`CRITICAL` on the root logger. The arguments - are interpreted as for :func:`debug`. + Logs a message with level :const:`CRITICAL` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. function:: exception(msg, *args, **kwargs) - Logs a message with level :const:`ERROR` on the root logger. The arguments are - interpreted as for :func:`debug`. Exception info is added to the logging + Logs a message with level :const:`ERROR` on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. Exception info is added to the logging message. This function should only be called from an exception handler. .. function:: log(level, msg, *args, **kwargs) - Logs a message with level *level* on the root logger. The other arguments are - interpreted as for :func:`debug`. + Logs a message with level *level* on the root logger. The arguments and behavior + are otherwise the same as for :func:`debug`. .. function:: disable(level=CRITICAL) diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 0d69c3bc01d1e2..69f7cb8d48d7ae 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -1,5 +1,5 @@ -:mod:`lzma` --- Compression using the LZMA algorithm -==================================================== +:mod:`!lzma` --- Compression using the LZMA algorithm +===================================================== .. module:: lzma :synopsis: A Python wrapper for the liblzma compression library. @@ -104,7 +104,7 @@ Reading and writing compressed files and :meth:`~io.IOBase.truncate`. Iteration and the :keyword:`with` statement are supported. - The following method is also provided: + The following method and attributes are also provided: .. method:: peek(size=-1) @@ -117,6 +117,20 @@ Reading and writing compressed files file object (e.g. if the :class:`LZMAFile` was constructed by passing a file object for *filename*). + .. attribute:: mode + + ``'rb'`` for reading and ``'wb'`` for writing. + + .. versionadded:: 3.13 + + .. attribute:: name + + The lzma file name. Equivalent to the :attr:`~io.FileIO.name` + attribute of the underlying :term:`file object`. + + .. versionadded:: 3.13 + + .. versionchanged:: 3.4 Added support for the ``"x"`` and ``"xb"`` modes. diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index a613548c9e518e..abb32f9bf3457f 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -1,5 +1,5 @@ -:mod:`mailbox` --- Manipulate mailboxes in various formats -========================================================== +:mod:`!mailbox` --- Manipulate mailboxes in various formats +=========================================================== .. module:: mailbox :synopsis: Manipulate mailboxes in various formats @@ -1387,7 +1387,7 @@ When an :class:`!MHMessage` instance is created based upon a .. method:: get_visible() - Return an :class:`Message` instance whose headers are the message's + Return a :class:`Message` instance whose headers are the message's visible headers and whose body is empty. diff --git a/Doc/library/mailcap.rst b/Doc/library/mailcap.rst new file mode 100644 index 00000000000000..4467da146a5a05 --- /dev/null +++ b/Doc/library/mailcap.rst @@ -0,0 +1,15 @@ +:mod:`!mailcap` --- Mailcap file handling +========================================= + +.. module:: mailcap + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!mailcap` module was +`Python 3.12 `_. diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index c6a006b7b4028a..e8e9071a5c9ef4 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -1,5 +1,5 @@ -:mod:`marshal` --- Internal Python object serialization -======================================================= +:mod:`!marshal` --- Internal Python object serialization +======================================================== .. module:: marshal :synopsis: Convert Python objects to streams of bytes and back (with different @@ -10,7 +10,7 @@ This module contains functions that can read and write Python values in a binary format. The format is specific to Python, but independent of machine architecture issues (e.g., you can write a Python value to a file on a PC, -transport the file to a Sun, and read it back there). Details of the format are +transport the file to a Mac, and read it back there). Details of the format are undocumented on purpose; it may change between Python versions (although it rarely does). [#]_ @@ -38,23 +38,39 @@ supports a substantially wider range of objects than marshal. maliciously constructed data. Never unmarshal data received from an untrusted or unauthenticated source. +There are functions that read/write files as well as functions operating on +bytes-like objects. + .. index:: object; code, code object Not all Python object types are supported; in general, only objects whose value is independent from a particular invocation of Python can be written and read by -this module. The following types are supported: booleans, integers, floating -point numbers, complex numbers, strings, bytes, bytearrays, tuples, lists, sets, -frozensets, dictionaries, and code objects (if *allow_code* is true), -where it should be understood that -tuples, lists, sets, frozensets and dictionaries are only supported as long as -the values contained therein are themselves supported. The -singletons :const:`None`, :const:`Ellipsis` and :exc:`StopIteration` can also be -marshalled and unmarshalled. -For format *version* lower than 3, recursive lists, sets and dictionaries cannot -be written (see below). +this module. The following types are supported: + +* Numeric types: :class:`int`, :class:`bool`, :class:`float`, :class:`complex`. +* Strings (:class:`str`) and :class:`bytes`. + :term:`Bytes-like objects ` like :class:`bytearray` are + marshalled as :class:`!bytes`. +* Containers: :class:`tuple`, :class:`list`, :class:`set`, :class:`frozenset`, + and (since :data:`version` 5), :class:`slice`. + It should be understood that these are supported only if the values contained + therein are themselves supported. + Recursive containers are supported since :data:`version` 3. +* The singletons :const:`None`, :const:`Ellipsis` and :exc:`StopIteration`. +* :class:`code` objects, if *allow_code* is true. See note above about + version dependence. + +.. versionchanged:: 3.4 + + * Added format version 3, which supports marshalling recursive lists, sets + and dictionaries. + * Added format version 4, which supports efficient representations + of short strings. + +.. versionchanged:: 3.14 + + Added format version 5, which allows marshalling slices. -There are functions that read/write files as well as functions operating on -bytes-like objects. The module defines these functions: @@ -140,11 +156,24 @@ In addition, the following constants are defined: .. data:: version - Indicates the format that the module uses. Version 0 is the historical - format, version 1 shares interned strings and version 2 uses a binary format - for floating point numbers. - Version 3 adds support for object instancing and recursion. - The current version is 4. + Indicates the format that the module uses. + Version 0 is the historical first version; subsequent versions + add new features. + Generally, a new version becomes the default when it is introduced. + + ======= =============== ==================================================== + Version Available since New features + ======= =============== ==================================================== + 1 Python 2.4 Sharing interned strings + ------- --------------- ---------------------------------------------------- + 2 Python 2.5 Binary representation of floats + ------- --------------- ---------------------------------------------------- + 3 Python 3.4 Support for object instancing and recursion + ------- --------------- ---------------------------------------------------- + 4 Python 3.4 Efficient representation of short strings + ------- --------------- ---------------------------------------------------- + 5 Python 3.14 Support for :class:`slice` objects + ======= =============== ==================================================== .. rubric:: Footnotes @@ -154,4 +183,3 @@ In addition, the following constants are defined: around in a self-contained form. Strictly speaking, "to marshal" means to convert some data from internal to external form (in an RPC buffer for instance) and "unmarshalling" for the reverse process. - diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 3c850317f60858..c78b313db5152d 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -1,5 +1,5 @@ -:mod:`math` --- Mathematical functions -====================================== +:mod:`!math` --- Mathematical functions +======================================= .. module:: math :synopsis: Mathematical functions (sin() etc.). @@ -26,15 +26,103 @@ The following functions are provided by this module. Except when explicitly noted otherwise, all return values are floats. -Number-theoretic and representation functions ---------------------------------------------- - -.. function:: ceil(x) - - Return the ceiling of *x*, the smallest integer greater than or equal to *x*. - If *x* is not a float, delegates to :meth:`x.__ceil__ `, - which should return an :class:`~numbers.Integral` value. - +==================================================== ============================================ +**Number-theoretic functions** +-------------------------------------------------------------------------------------------------- +:func:`comb(n, k) ` Number of ways to choose *k* items from *n* items without repetition and without order +:func:`factorial(n) ` *n* factorial +:func:`gcd(*integers) ` Greatest common divisor of the integer arguments +:func:`isqrt(n) ` Integer square root of a nonnegative integer *n* +:func:`lcm(*integers) ` Least common multiple of the integer arguments +:func:`perm(n, k) ` Number of ways to choose *k* items from *n* items without repetition and with order + +**Floating point arithmetic** +-------------------------------------------------------------------------------------------------- +:func:`ceil(x) ` Ceiling of *x*, the smallest integer greater than or equal to *x* +:func:`fabs(x) ` Absolute value of *x* +:func:`floor(x) ` Floor of *x*, the largest integer less than or equal to *x* +:func:`fma(x, y, z) ` Fused multiply-add operation: ``(x * y) + z`` +:func:`fmod(x, y) ` Remainder of division ``x / y`` +:func:`modf(x) ` Fractional and integer parts of *x* +:func:`remainder(x, y) ` Remainder of *x* with respect to *y* +:func:`trunc(x) ` Integer part of *x* + +**Floating point manipulation functions** +-------------------------------------------------------------------------------------------------- +:func:`copysign(x, y) ` Magnitude (absolute value) of *x* with the sign of *y* +:func:`frexp(x) ` Mantissa and exponent of *x* +:func:`isclose(a, b, rel_tol, abs_tol) ` Check if the values *a* and *b* are close to each other +:func:`isfinite(x) ` Check if *x* is neither an infinity nor a NaN +:func:`isinf(x) ` Check if *x* is a positive or negative infinity +:func:`isnan(x) ` Check if *x* is a NaN (not a number) +:func:`ldexp(x, i) ` ``x * (2**i)``, inverse of function :func:`frexp` +:func:`nextafter(x, y, steps) ` Floating-point value *steps* steps after *x* towards *y* +:func:`ulp(x) ` Value of the least significant bit of *x* + +**Power, exponential and logarithmic functions** +-------------------------------------------------------------------------------------------------- +:func:`cbrt(x) ` Cube root of *x* +:func:`exp(x) ` *e* raised to the power *x* +:func:`exp2(x) ` *2* raised to the power *x* +:func:`expm1(x) ` *e* raised to the power *x*, minus 1 +:func:`log(x, base) ` Logarithm of *x* to the given base (*e* by default) +:func:`log1p(x) ` Natural logarithm of *1+x* (base *e*) +:func:`log2(x) ` Base-2 logarithm of *x* +:func:`log10(x) ` Base-10 logarithm of *x* +:func:`pow(x, y) ` *x* raised to the power *y* +:func:`sqrt(x) ` Square root of *x* + +**Summation and product functions** +-------------------------------------------------------------------------------------------------- +:func:`dist(p, q) ` Euclidean distance between two points *p* and *q* given as an iterable of coordinates +:func:`fsum(iterable) ` Sum of values in the input *iterable* +:func:`hypot(*coordinates) ` Euclidean norm of an iterable of coordinates +:func:`prod(iterable, start) ` Product of elements in the input *iterable* with a *start* value +:func:`sumprod(p, q) ` Sum of products from two iterables *p* and *q* + +**Angular conversion** +-------------------------------------------------------------------------------------------------- +:func:`degrees(x) ` Convert angle *x* from radians to degrees +:func:`radians(x) ` Convert angle *x* from degrees to radians + +**Trigonometric functions** +-------------------------------------------------------------------------------------------------- +:func:`acos(x) ` Arc cosine of *x* +:func:`asin(x) ` Arc sine of *x* +:func:`atan(x) ` Arc tangent of *x* +:func:`atan2(y, x) ` ``atan(y / x)`` +:func:`cos(x) ` Cosine of *x* +:func:`sin(x) ` Sine of *x* +:func:`tan(x) ` Tangent of *x* + +**Hyperbolic functions** +-------------------------------------------------------------------------------------------------- +:func:`acosh(x) ` Inverse hyperbolic cosine of *x* +:func:`asinh(x) ` Inverse hyperbolic sine of *x* +:func:`atanh(x) ` Inverse hyperbolic tangent of *x* +:func:`cosh(x) ` Hyperbolic cosine of *x* +:func:`sinh(x) ` Hyperbolic sine of *x* +:func:`tanh(x) ` Hyperbolic tangent of *x* + +**Special functions** +-------------------------------------------------------------------------------------------------- +:func:`erf(x) ` `Error function `_ at *x* +:func:`erfc(x) ` `Complementary error function `_ at *x* +:func:`gamma(x) ` `Gamma function `_ at *x* +:func:`lgamma(x) ` Natural logarithm of the absolute value of the `Gamma function `_ at *x* + +**Constants** +-------------------------------------------------------------------------------------------------- +:data:`pi` *π* = 3.141592... +:data:`e` *e* = 2.718281... +:data:`tau` *τ* = 2\ *π* = 6.283185... +:data:`inf` Positive infinity +:data:`nan` "Not a number" (NaN) +==================================================== ============================================ + + +Number-theoretic functions +-------------------------- .. function:: comb(n, k) @@ -54,25 +142,85 @@ Number-theoretic and representation functions .. versionadded:: 3.8 -.. function:: copysign(x, y) +.. function:: factorial(n) - Return a float with the magnitude (absolute value) of *x* but the sign of - *y*. On platforms that support signed zeros, ``copysign(1.0, -0.0)`` - returns *-1.0*. + Return *n* factorial as an integer. Raises :exc:`ValueError` if *n* is not integral or + is negative. + .. versionchanged:: 3.10 + Floats with integral values (like ``5.0``) are no longer accepted. -.. function:: fabs(x) - Return the absolute value of *x*. +.. function:: gcd(*integers) + Return the greatest common divisor of the specified integer arguments. + If any of the arguments is nonzero, then the returned value is the largest + positive integer that is a divisor of all arguments. If all arguments + are zero, then the returned value is ``0``. ``gcd()`` without arguments + returns ``0``. -.. function:: factorial(n) + .. versionadded:: 3.5 - Return *n* factorial as an integer. Raises :exc:`ValueError` if *n* is not integral or - is negative. + .. versionchanged:: 3.9 + Added support for an arbitrary number of arguments. Formerly, only two + arguments were supported. - .. versionchanged:: 3.10 - Floats with integral values (like ``5.0``) are no longer accepted. + +.. function:: isqrt(n) + + Return the integer square root of the nonnegative integer *n*. This is the + floor of the exact square root of *n*, or equivalently the greatest integer + *a* such that *a*\ ² |nbsp| ≤ |nbsp| *n*. + + For some applications, it may be more convenient to have the least integer + *a* such that *n* |nbsp| ≤ |nbsp| *a*\ ², or in other words the ceiling of + the exact square root of *n*. For positive *n*, this can be computed using + ``a = 1 + isqrt(n - 1)``. + + .. versionadded:: 3.8 + + +.. function:: lcm(*integers) + + Return the least common multiple of the specified integer arguments. + If all arguments are nonzero, then the returned value is the smallest + positive integer that is a multiple of all arguments. If any of the arguments + is zero, then the returned value is ``0``. ``lcm()`` without arguments + returns ``1``. + + .. versionadded:: 3.9 + + +.. function:: perm(n, k=None) + + Return the number of ways to choose *k* items from *n* items + without repetition and with order. + + Evaluates to ``n! / (n - k)!`` when ``k <= n`` and evaluates + to zero when ``k > n``. + + If *k* is not specified or is ``None``, then *k* defaults to *n* + and the function returns ``n!``. + + Raises :exc:`TypeError` if either of the arguments are not integers. + Raises :exc:`ValueError` if either of the arguments are negative. + + .. versionadded:: 3.8 + + +Floating point arithmetic +------------------------- + +.. function:: ceil(x) + + Return the ceiling of *x*, the smallest integer greater than or equal to *x*. + If *x* is not a float, delegates to :meth:`x.__ceil__ `, + which should return an :class:`~numbers.Integral` value. + + +.. function:: fabs(x) + + Return the absolute value of *x*. .. function:: floor(x) @@ -82,9 +230,26 @@ Number-theoretic and representation functions should return an :class:`~numbers.Integral` value. +.. function:: fma(x, y, z) + + Fused multiply-add operation. Return ``(x * y) + z``, computed as though with + infinite precision and range followed by a single round to the ``float`` + format. This operation often provides better accuracy than the direct + expression ``(x * y) + z``. + + This function follows the specification of the fusedMultiplyAdd operation + described in the IEEE 754 standard. The standard leaves one case + implementation-defined, namely the result of ``fma(0, inf, nan)`` + and ``fma(inf, 0, nan)``. In these cases, ``math.fma`` returns a NaN, + and does not raise any exception. + + .. versionadded:: 3.13 + + .. function:: fmod(x, y) - Return ``fmod(x, y)``, as defined by the platform C library. Note that the + Return the floating-point remainder of ``x / y``, + as defined by the platform C library function ``fmod(x, y)``. Note that the Python expression ``x % y`` may not return the same result. The intent of the C standard is that ``fmod(x, y)`` be exactly (mathematically; to infinite precision) equal to ``x - n*y`` for some integer *n* such that the result has @@ -97,44 +262,75 @@ Number-theoretic and representation functions floats, while Python's ``x % y`` is preferred when working with integers. -.. function:: frexp(x) +.. function:: modf(x) - Return the mantissa and exponent of *x* as the pair ``(m, e)``. *m* is a float - and *e* is an integer such that ``x == m * 2**e`` exactly. If *x* is zero, - returns ``(0.0, 0)``, otherwise ``0.5 <= abs(m) < 1``. This is used to "pick - apart" the internal representation of a float in a portable way. + Return the fractional and integer parts of *x*. Both results carry the sign + of *x* and are floats. + Note that :func:`modf` has a different call/return pattern + than its C equivalents: it takes a single argument and return a pair of + values, rather than returning its second return value through an 'output + parameter' (there is no such thing in Python). -.. function:: fsum(iterable) - Return an accurate floating point sum of values in the iterable. Avoids - loss of precision by tracking multiple intermediate partial sums. +.. function:: remainder(x, y) - The algorithm's accuracy depends on IEEE-754 arithmetic guarantees and the - typical case where the rounding mode is half-even. On some non-Windows - builds, the underlying C library uses extended precision addition and may - occasionally double-round an intermediate sum causing it to be off in its - least significant bit. + Return the IEEE 754-style remainder of *x* with respect to *y*. For + finite *x* and finite nonzero *y*, this is the difference ``x - n*y``, + where ``n`` is the closest integer to the exact value of the quotient ``x / + y``. If ``x / y`` is exactly halfway between two consecutive integers, the + nearest *even* integer is used for ``n``. The remainder ``r = remainder(x, + y)`` thus always satisfies ``abs(r) <= 0.5 * abs(y)``. - For further discussion and two alternative approaches, see the `ASPN cookbook - recipes for accurate floating point summation - `_\. + Special cases follow IEEE 754: in particular, ``remainder(x, math.inf)`` is + *x* for any finite *x*, and ``remainder(x, 0)`` and + ``remainder(math.inf, x)`` raise :exc:`ValueError` for any non-NaN *x*. + If the result of the remainder operation is zero, that zero will have + the same sign as *x*. + On platforms using IEEE 754 binary floating point, the result of this + operation is always exactly representable: no rounding error is introduced. -.. function:: gcd(*integers) + .. versionadded:: 3.7 - Return the greatest common divisor of the specified integer arguments. - If any of the arguments is nonzero, then the returned value is the largest - positive integer that is a divisor of all arguments. If all arguments - are zero, then the returned value is ``0``. ``gcd()`` without arguments - returns ``0``. - .. versionadded:: 3.5 +.. function:: trunc(x) - .. versionchanged:: 3.9 - Added support for an arbitrary number of arguments. Formerly, only two - arguments were supported. + Return *x* with the fractional part + removed, leaving the integer part. This rounds toward 0: ``trunc()`` is + equivalent to :func:`floor` for positive *x*, and equivalent to :func:`ceil` + for negative *x*. If *x* is not a float, delegates to :meth:`x.__trunc__ + `, which should return an :class:`~numbers.Integral` value. + + +For the :func:`ceil`, :func:`floor`, and :func:`modf` functions, note that *all* +floating-point numbers of sufficiently large magnitude are exact integers. +Python floats typically carry no more than 53 bits of precision (the same as the +platform C double type), in which case any float *x* with ``abs(x) >= 2**52`` +necessarily has no fractional bits. + + +Floating point manipulation functions +------------------------------------- + +.. function:: copysign(x, y) + + Return a float with the magnitude (absolute value) of *x* but the sign of + *y*. On platforms that support signed zeros, ``copysign(1.0, -0.0)`` + returns *-1.0*. + + +.. function:: frexp(x) + + Return the mantissa and exponent of *x* as the pair ``(m, e)``. *m* is a float + and *e* is an integer such that ``x == m * 2**e`` exactly. If *x* is zero, + returns ``(0.0, 0)``, otherwise ``0.5 <= abs(m) < 1``. This is used to "pick + apart" the internal representation of a float in a portable way. + Note that :func:`frexp` has a different call/return pattern + than its C equivalents: it takes a single argument and return a pair of + values, rather than returning its second return value through an 'output + parameter' (there is no such thing in Python). .. function:: isclose(a, b, *, rel_tol=1e-09, abs_tol=0.0) @@ -142,19 +338,21 @@ Number-theoretic and representation functions ``False`` otherwise. Whether or not two values are considered close is determined according to - given absolute and relative tolerances. + given absolute and relative tolerances. If no errors occur, the result will + be: ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``. *rel_tol* is the relative tolerance -- it is the maximum allowed difference between *a* and *b*, relative to the larger absolute value of *a* or *b*. For example, to set a tolerance of 5%, pass ``rel_tol=0.05``. The default tolerance is ``1e-09``, which assures that the two values are the same - within about 9 decimal digits. *rel_tol* must be greater than zero. + within about 9 decimal digits. *rel_tol* must be nonnegative and less + than ``1.0``. - *abs_tol* is the minimum absolute tolerance -- useful for comparisons near - zero. *abs_tol* must be at least zero. - - If no errors occur, the result will be: - ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``. + *abs_tol* is the absolute tolerance; it defaults to ``0.0`` and it must be + nonnegative. When comparing ``x`` to ``0.0``, ``isclose(x, 0)`` is computed + as ``abs(x) <= rel_tol * abs(x)``, which is ``False`` for any ``x`` and + rel_tol less than ``1.0``. So add an appropriate positive abs_tol argument + to the call. The IEEE 754 special values of ``NaN``, ``inf``, and ``-inf`` will be handled according to IEEE rules. Specifically, ``NaN`` is not considered @@ -187,43 +385,12 @@ Number-theoretic and representation functions Return ``True`` if *x* is a NaN (not a number), and ``False`` otherwise. -.. function:: isqrt(n) - - Return the integer square root of the nonnegative integer *n*. This is the - floor of the exact square root of *n*, or equivalently the greatest integer - *a* such that *a*\ ² |nbsp| ≤ |nbsp| *n*. - - For some applications, it may be more convenient to have the least integer - *a* such that *n* |nbsp| ≤ |nbsp| *a*\ ², or in other words the ceiling of - the exact square root of *n*. For positive *n*, this can be computed using - ``a = 1 + isqrt(n - 1)``. - - .. versionadded:: 3.8 - - -.. function:: lcm(*integers) - - Return the least common multiple of the specified integer arguments. - If all arguments are nonzero, then the returned value is the smallest - positive integer that is a multiple of all arguments. If any of the arguments - is zero, then the returned value is ``0``. ``lcm()`` without arguments - returns ``1``. - - .. versionadded:: 3.9 - - .. function:: ldexp(x, i) Return ``x * (2**i)``. This is essentially the inverse of function :func:`frexp`. -.. function:: modf(x) - - Return the fractional and integer parts of *x*. Both results carry the sign - of *x* and are floats. - - .. function:: nextafter(x, y, steps=1) Return the floating-point value *steps* steps after *x* towards *y*. @@ -244,79 +411,6 @@ Number-theoretic and representation functions .. versionchanged:: 3.12 Added the *steps* argument. -.. function:: perm(n, k=None) - - Return the number of ways to choose *k* items from *n* items - without repetition and with order. - - Evaluates to ``n! / (n - k)!`` when ``k <= n`` and evaluates - to zero when ``k > n``. - - If *k* is not specified or is None, then *k* defaults to *n* - and the function returns ``n!``. - - Raises :exc:`TypeError` if either of the arguments are not integers. - Raises :exc:`ValueError` if either of the arguments are negative. - - .. versionadded:: 3.8 - - -.. function:: prod(iterable, *, start=1) - - Calculate the product of all the elements in the input *iterable*. - The default *start* value for the product is ``1``. - - When the iterable is empty, return the start value. This function is - intended specifically for use with numeric values and may reject - non-numeric types. - - .. versionadded:: 3.8 - - -.. function:: remainder(x, y) - - Return the IEEE 754-style remainder of *x* with respect to *y*. For - finite *x* and finite nonzero *y*, this is the difference ``x - n*y``, - where ``n`` is the closest integer to the exact value of the quotient ``x / - y``. If ``x / y`` is exactly halfway between two consecutive integers, the - nearest *even* integer is used for ``n``. The remainder ``r = remainder(x, - y)`` thus always satisfies ``abs(r) <= 0.5 * abs(y)``. - - Special cases follow IEEE 754: in particular, ``remainder(x, math.inf)`` is - *x* for any finite *x*, and ``remainder(x, 0)`` and - ``remainder(math.inf, x)`` raise :exc:`ValueError` for any non-NaN *x*. - If the result of the remainder operation is zero, that zero will have - the same sign as *x*. - - On platforms using IEEE 754 binary floating-point, the result of this - operation is always exactly representable: no rounding error is introduced. - - .. versionadded:: 3.7 - - -.. function:: sumprod(p, q) - - Return the sum of products of values from two iterables *p* and *q*. - - Raises :exc:`ValueError` if the inputs do not have the same length. - - Roughly equivalent to:: - - sum(itertools.starmap(operator.mul, zip(p, q, strict=True))) - - For float and mixed int/float inputs, the intermediate products - and sums are computed with extended precision. - - .. versionadded:: 3.12 - - -.. function:: trunc(x) - - Return *x* with the fractional part - removed, leaving the integer part. This rounds toward 0: ``trunc()`` is - equivalent to :func:`floor` for positive *x*, and equivalent to :func:`ceil` - for negative *x*. If *x* is not a float, delegates to :meth:`x.__trunc__ - `, which should return an :class:`~numbers.Integral` value. .. function:: ulp(x) @@ -343,20 +437,8 @@ Number-theoretic and representation functions .. versionadded:: 3.9 -Note that :func:`frexp` and :func:`modf` have a different call/return pattern -than their C equivalents: they take a single argument and return a pair of -values, rather than returning their second return value through an 'output -parameter' (there is no such thing in Python). - -For the :func:`ceil`, :func:`floor`, and :func:`modf` functions, note that *all* -floating-point numbers of sufficiently large magnitude are exact integers. -Python floats typically carry no more than 53 bits of precision (the same as the -platform C double type), in which case any float *x* with ``abs(x) >= 2**52`` -necessarily has no fractional bits. - - -Power and logarithmic functions -------------------------------- +Power, exponential and logarithmic functions +-------------------------------------------- .. function:: cbrt(x) @@ -431,11 +513,11 @@ Power and logarithmic functions .. function:: pow(x, y) - Return ``x`` raised to the power ``y``. Exceptional cases follow + Return *x* raised to the power *y*. Exceptional cases follow the IEEE 754 standard as far as possible. In particular, ``pow(1.0, x)`` and ``pow(x, 0.0)`` always return ``1.0``, even - when ``x`` is a zero or a NaN. If both ``x`` and ``y`` are finite, - ``x`` is negative, and ``y`` is not an integer then ``pow(x, y)`` + when *x* is a zero or a NaN. If both *x* and *y* are finite, + *x* is negative, and *y* is not an integer then ``pow(x, y)`` is undefined, and raises :exc:`ValueError`. Unlike the built-in ``**`` operator, :func:`math.pow` converts both @@ -453,41 +535,8 @@ Power and logarithmic functions Return the square root of *x*. -Trigonometric functions ------------------------ - -.. function:: acos(x) - - Return the arc cosine of *x*, in radians. The result is between ``0`` and - ``pi``. - - -.. function:: asin(x) - - Return the arc sine of *x*, in radians. The result is between ``-pi/2`` and - ``pi/2``. - - -.. function:: atan(x) - - Return the arc tangent of *x*, in radians. The result is between ``-pi/2`` and - ``pi/2``. - - -.. function:: atan2(y, x) - - Return ``atan(y / x)``, in radians. The result is between ``-pi`` and ``pi``. - The vector in the plane from the origin to point ``(x, y)`` makes this angle - with the positive X axis. The point of :func:`atan2` is that the signs of both - inputs are known to it, so it can compute the correct quadrant for the angle. - For example, ``atan(1)`` and ``atan2(1, 1)`` are both ``pi/4``, but ``atan2(-1, - -1)`` is ``-3*pi/4``. - - -.. function:: cos(x) - - Return the cosine of *x* radians. - +Summation and product functions +------------------------------- .. function:: dist(p, q) @@ -502,6 +551,22 @@ Trigonometric functions .. versionadded:: 3.8 +.. function:: fsum(iterable) + + Return an accurate floating-point sum of values in the iterable. Avoids + loss of precision by tracking multiple intermediate partial sums. + + The algorithm's accuracy depends on IEEE-754 arithmetic guarantees and the + typical case where the rounding mode is half-even. On some non-Windows + builds, the underlying C library uses extended precision addition and may + occasionally double-round an intermediate sum causing it to be off in its + least significant bit. + + For further discussion and two alternative approaches, see the `ASPN cookbook + recipes for accurate floating-point summation + `_\. + + .. function:: hypot(*coordinates) Return the Euclidean norm, ``sqrt(sum(x**2 for x in coordinates))``. @@ -522,14 +587,32 @@ Trigonometric functions is almost always correctly rounded to within 1/2 ulp. -.. function:: sin(x) +.. function:: prod(iterable, *, start=1) - Return the sine of *x* radians. + Calculate the product of all the elements in the input *iterable*. + The default *start* value for the product is ``1``. + When the iterable is empty, return the start value. This function is + intended specifically for use with numeric values and may reject + non-numeric types. -.. function:: tan(x) + .. versionadded:: 3.8 - Return the tangent of *x* radians. + +.. function:: sumprod(p, q) + + Return the sum of products of values from two iterables *p* and *q*. + + Raises :exc:`ValueError` if the inputs do not have the same length. + + Roughly equivalent to:: + + sum(map(operator.mul, p, q, strict=True)) + + For float and mixed int/float inputs, the intermediate products + and sums are computed with extended precision. + + .. versionadded:: 3.12 Angular conversion @@ -545,6 +628,52 @@ Angular conversion Convert angle *x* from degrees to radians. +Trigonometric functions +----------------------- + +.. function:: acos(x) + + Return the arc cosine of *x*, in radians. The result is between ``0`` and + ``pi``. + + +.. function:: asin(x) + + Return the arc sine of *x*, in radians. The result is between ``-pi/2`` and + ``pi/2``. + + +.. function:: atan(x) + + Return the arc tangent of *x*, in radians. The result is between ``-pi/2`` and + ``pi/2``. + + +.. function:: atan2(y, x) + + Return ``atan(y / x)``, in radians. The result is between ``-pi`` and ``pi``. + The vector in the plane from the origin to point ``(x, y)`` makes this angle + with the positive X axis. The point of :func:`atan2` is that the signs of both + inputs are known to it, so it can compute the correct quadrant for the angle. + For example, ``atan(1)`` and ``atan2(1, 1)`` are both ``pi/4``, but ``atan2(-1, + -1)`` is ``-3*pi/4``. + + +.. function:: cos(x) + + Return the cosine of *x* radians. + + +.. function:: sin(x) + + Return the sine of *x* radians. + + +.. function:: tan(x) + + Return the tangent of *x* radians. + + Hyperbolic functions -------------------- @@ -592,7 +721,7 @@ Special functions The :func:`erf` function can be used to compute traditional statistical functions such as the `cumulative standard normal distribution - `_:: + `_:: def phi(x): 'Cumulative distribution function for the standard normal distribution' diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index f610032acbe417..8ad4850584a7e1 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -1,5 +1,5 @@ -:mod:`mimetypes` --- Map filenames to MIME types -================================================ +:mod:`!mimetypes` --- Map filenames to MIME types +================================================= .. module:: mimetypes :synopsis: Mapping of filename extensions to MIME types. @@ -52,7 +52,22 @@ the information :func:`init` sets up. are also recognized. .. versionchanged:: 3.8 - Added support for url being a :term:`path-like object`. + Added support for *url* being a :term:`path-like object`. + + .. deprecated:: 3.13 + Passing a file path instead of URL is :term:`soft deprecated`. + Use :func:`guess_file_type` for this. + + +.. function:: guess_file_type(path, *, strict=True) + + .. index:: pair: MIME; headers + + Guess the type of a file based on its path, given by *path*. + Similar to the :func:`guess_type` function, but accepts a path instead of URL. + Path can be a string, a bytes object or a :term:`path-like object`. + + .. versionadded:: 3.13 .. function:: guess_all_extensions(type, strict=True) @@ -61,7 +76,7 @@ the information :func:`init` sets up. return value is a list of strings giving all possible filename extensions, including the leading dot (``'.'``). The extensions are not guaranteed to have been associated with any particular data stream, but would be mapped to the MIME - type *type* by :func:`guess_type`. + type *type* by :func:`guess_type` and :func:`guess_file_type`. The optional *strict* argument has the same meaning as with the :func:`guess_type` function. @@ -72,8 +87,8 @@ the information :func:`init` sets up. return value is a string giving a filename extension, including the leading dot (``'.'``). The extension is not guaranteed to have been associated with any particular data stream, but would be mapped to the MIME type *type* by - :func:`guess_type`. If no extension can be guessed for *type*, ``None`` is - returned. + :func:`guess_type` and :func:`guess_file_type`. + If no extension can be guessed for *type*, ``None`` is returned. The optional *strict* argument has the same meaning as with the :func:`guess_type` function. @@ -238,6 +253,14 @@ than one MIME-type database; it provides an interface similar to the one of the the object. + .. method:: MimeTypes.guess_file_type(path, *, strict=True) + + Similar to the :func:`guess_file_type` function, using the tables stored + as part of the object. + + .. versionadded:: 3.13 + + .. method:: MimeTypes.guess_all_extensions(type, strict=True) Similar to the :func:`guess_all_extensions` function, using the tables stored @@ -272,3 +295,13 @@ than one MIME-type database; it provides an interface similar to the one of the types, else to the list of non-standard types. .. versionadded:: 3.2 + + + .. method:: MimeTypes.add_type(type, ext, strict=True) + + Add a mapping from the MIME type *type* to the extension *ext*. When the + extension is already known, the new type will replace the old one. When the type + is already known the extension will be added to the list of known extensions. + + When *strict* is ``True`` (the default), the mapping will be added to the + official MIME types, otherwise to the non-standard ones. diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 758721433f77de..4e20c07331a220 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -1,5 +1,5 @@ -:mod:`mmap` --- Memory-mapped file support -========================================== +:mod:`!mmap` --- Memory-mapped file support +=========================================== .. module:: mmap :synopsis: Interface to memory-mapped files for Unix and Windows. diff --git a/Doc/library/modulefinder.rst b/Doc/library/modulefinder.rst index 526f0ff868c2b7..823d853f1ed8eb 100644 --- a/Doc/library/modulefinder.rst +++ b/Doc/library/modulefinder.rst @@ -1,5 +1,5 @@ -:mod:`modulefinder` --- Find modules used by a script -===================================================== +:mod:`!modulefinder` --- Find modules used by a script +====================================================== .. module:: modulefinder :synopsis: Find modules used by a script. diff --git a/Doc/library/msilib.rst b/Doc/library/msilib.rst new file mode 100644 index 00000000000000..eb1ac551ded456 --- /dev/null +++ b/Doc/library/msilib.rst @@ -0,0 +1,15 @@ +:mod:`!msilib` --- Read and write Microsoft Installer files +=========================================================== + +.. module:: msilib + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!msilib` module was +`Python 3.12 `_. diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index ac3458c86fd4c4..327cc3602b1a77 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -1,5 +1,5 @@ -:mod:`msvcrt` --- Useful routines from the MS VC++ runtime -========================================================== +:mod:`!msvcrt` --- Useful routines from the MS VC++ runtime +=========================================================== .. module:: msvcrt :platform: Windows @@ -211,7 +211,7 @@ Other Functions After you use :func:`CrtSetReportMode` to specify :const:`CRTDBG_MODE_FILE`, you can specify the file handle to receive the message text. *type* must be - one of the :const:`!CRT_\*` constants listed below. *file* shuld be the file + one of the :const:`!CRT_\*` constants listed below. *file* should be the file handle your want specified. Only available in :ref:`debug build of Python `. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 0b87de4c61e6aa..783cb025826483 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1,5 +1,5 @@ -:mod:`multiprocessing` --- Process-based parallelism -==================================================== +:mod:`!multiprocessing` --- Process-based parallelism +===================================================== .. module:: multiprocessing :synopsis: Process-based parallelism. @@ -8,7 +8,7 @@ -------------- -.. include:: ../includes/wasm-notavail.rst +.. include:: ../includes/wasm-mobile-notavail.rst Introduction ------------ @@ -124,11 +124,11 @@ to start a process. These *start methods* are inherited by the child process. Note that safely forking a multithreaded process is problematic. - Available on POSIX systems. Currently the default on POSIX except macOS. + Available on POSIX systems. - .. note:: - The default start method will change away from *fork* in Python 3.14. - Code that requires *fork* should explicitly specify that via + .. versionchanged:: 3.14 + This is no longer the default start method on any platform. + Code that requires *fork* must explicitly specify that via :func:`get_context` or :func:`set_start_method`. .. versionchanged:: 3.12 @@ -146,9 +146,11 @@ to start a process. These *start methods* are side-effect so it is generally safe for it to use :func:`os.fork`. No unnecessary resources are inherited. - Available on POSIX platforms which support passing file descriptors - over Unix pipes such as Linux. + Available on POSIX platforms which support passing file descriptors over + Unix pipes such as Linux. The default on those. + .. versionchanged:: 3.14 + This became the default start method on POSIX platforms. .. versionchanged:: 3.4 *spawn* added on all POSIX platforms, and *forkserver* added for @@ -162,6 +164,13 @@ to start a process. These *start methods* are method should be considered unsafe as it can lead to crashes of the subprocess as macOS system libraries may start threads. See :issue:`33725`. +.. versionchanged:: 3.14 + + On POSIX platforms the default start method was changed from *fork* to + *forkserver* to retain the performance but avoid common multithreaded + process incompatibilities. See :gh:`84559`. + + On POSIX using the *spawn* or *forkserver* start methods will also start a *resource tracker* process which tracks the unlinked named system resources (such as named semaphores or @@ -254,6 +263,7 @@ processes: p.join() Queues are thread and process safe. + Any object put into a :mod:`~multiprocessing` queue will be serialized. **Pipes** @@ -281,6 +291,8 @@ processes: of corruption from processes using different ends of the pipe at the same time. + The :meth:`~Connection.send` method serializes the object and + :meth:`~Connection.recv` re-creates the object. Synchronization between processes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -502,7 +514,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the The constructor should always be called with keyword arguments. *group* should always be ``None``; it exists solely for compatibility with :class:`threading.Thread`. *target* is the callable object to be invoked by - the :meth:`run()` method. It defaults to ``None``, meaning nothing is + the :meth:`run` method. It defaults to ``None``, meaning nothing is called. *name* is the process name (see :attr:`name` for more details). *args* is the argument tuple for the target invocation. *kwargs* is a dictionary of keyword arguments for the target invocation. If provided, @@ -639,7 +651,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the You can use this value if you want to wait on several events at once using :func:`multiprocessing.connection.wait`. Otherwise - calling :meth:`join()` is simpler. + calling :meth:`join` is simpler. On Windows, this is an OS handle usable with the ``WaitForSingleObject`` and ``WaitForMultipleObjects`` family of API calls. On POSIX, this is @@ -666,7 +678,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the .. method:: kill() - Same as :meth:`terminate()` but using the ``SIGKILL`` signal on POSIX. + Same as :meth:`terminate` but using the ``SIGKILL`` signal on POSIX. .. versionadded:: 3.7 @@ -709,7 +721,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the .. exception:: BufferTooShort - Exception raised by :meth:`Connection.recv_bytes_into()` when the supplied + Exception raised by :meth:`Connection.recv_bytes_into` when the supplied buffer object is too small for the message read. If ``e`` is an instance of :exc:`BufferTooShort` then ``e.args[0]`` will give @@ -745,6 +757,11 @@ If you use :class:`JoinableQueue` then you **must** call semaphore used to count the number of unfinished tasks may eventually overflow, raising an exception. +One difference from other Python queue implementations, is that :mod:`multiprocessing` +queues serializes all objects that are put into them using :mod:`pickle`. +The object return by the get method is a re-created object that does not share memory +with the original object. + Note that one can also create a shared queue by using a manager object -- see :ref:`multiprocessing-managers`. @@ -811,6 +828,8 @@ For an example of the usage of queues for interprocess communication see used for receiving messages and ``conn2`` can only be used for sending messages. + The :meth:`~multiprocessing.Connection.send` method serializes the object using + :mod:`pickle` and the :meth:`~multiprocessing.Connection.recv` re-creates the object. .. class:: Queue([maxsize]) @@ -837,6 +856,8 @@ For an example of the usage of queues for interprocess communication see Return ``True`` if the queue is empty, ``False`` otherwise. Because of multithreading/multiprocessing semantics, this is not reliable. + May raise an :exc:`OSError` on closed queues. (not guaranteed) + .. method:: full() Return ``True`` if the queue is full, ``False`` otherwise. Because of @@ -940,6 +961,8 @@ For an example of the usage of queues for interprocess communication see Return ``True`` if the queue is empty, ``False`` otherwise. + Always raises an :exc:`OSError` if the SimpleQueue is closed. + .. method:: get() Remove and return an item from the queue. @@ -1459,17 +1482,6 @@ object -- see :ref:`multiprocessing-managers`. On macOS, ``sem_timedwait`` is unsupported, so calling ``acquire()`` with a timeout will emulate that function's behavior using a sleeping loop. -.. note:: - - If the SIGINT signal generated by :kbd:`Ctrl-C` arrives while the main thread is - blocked by a call to :meth:`BoundedSemaphore.acquire`, :meth:`Lock.acquire`, - :meth:`RLock.acquire`, :meth:`Semaphore.acquire`, :meth:`Condition.acquire` - or :meth:`Condition.wait` then the call will be immediately interrupted and - :exc:`KeyboardInterrupt` will be raised. - - This differs from the behaviour of :mod:`threading` where SIGINT will be - ignored while the equivalent blocking calls are in progress. - .. note:: Some of this package's functionality requires a functioning shared semaphore @@ -2483,9 +2495,9 @@ multiple connections at the same time. generally be omitted since it can usually be inferred from the format of *address*. (See :ref:`multiprocessing-address-formats`) - If *authkey* is given and not None, it should be a byte string and will be + If *authkey* is given and not ``None``, it should be a byte string and will be used as the secret key for an HMAC-based authentication challenge. No - authentication is done if *authkey* is None. + authentication is done if *authkey* is ``None``. :exc:`~multiprocessing.AuthenticationError` is raised if authentication fails. See :ref:`multiprocessing-auth-keys`. @@ -2518,9 +2530,9 @@ multiple connections at the same time. to the :meth:`~socket.socket.listen` method of the socket once it has been bound. - If *authkey* is given and not None, it should be a byte string and will be + If *authkey* is given and not ``None``, it should be a byte string and will be used as the secret key for an HMAC-based authentication challenge. No - authentication is done if *authkey* is None. + authentication is done if *authkey* is ``None``. :exc:`~multiprocessing.AuthenticationError` is raised if authentication fails. See :ref:`multiprocessing-auth-keys`. @@ -2958,7 +2970,7 @@ Beware of replacing :data:`sys.stdin` with a "file like object" resulting in a bad file descriptor error, but introduces a potential danger to applications which replace :func:`sys.stdin` with a "file-like object" with output buffering. This danger is that if multiple processes call - :meth:`~io.IOBase.close()` on this file-like object, it could result in the same + :meth:`~io.IOBase.close` on this file-like object, it could result in the same data being flushed to the object multiple times, resulting in corruption. If you write a file-like object and implement your own caching, you can diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index 933fd07d62418a..e8f04a6ac7b95d 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -1,5 +1,5 @@ -:mod:`multiprocessing.shared_memory` --- Shared memory for direct access across processes -========================================================================================= +:mod:`!multiprocessing.shared_memory` --- Shared memory for direct access across processes +========================================================================================== .. module:: multiprocessing.shared_memory :synopsis: Provides shared memory for direct access across processes. diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst index c36e5cfecfc6a8..f6260383b2b057 100644 --- a/Doc/library/netrc.rst +++ b/Doc/library/netrc.rst @@ -1,6 +1,5 @@ - -:mod:`netrc` --- netrc file processing -====================================== +:mod:`!netrc` --- netrc file processing +======================================= .. module:: netrc :synopsis: Loading of .netrc files. diff --git a/Doc/library/nis.rst b/Doc/library/nis.rst new file mode 100644 index 00000000000000..dcc36dd43fc313 --- /dev/null +++ b/Doc/library/nis.rst @@ -0,0 +1,15 @@ +:mod:`!nis` --- Interface to Sun’s NIS (Yellow Pages) +===================================================== + +.. module:: nis + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!nis` module was +`Python 3.12 `_. diff --git a/Doc/library/nntplib.rst b/Doc/library/nntplib.rst new file mode 100644 index 00000000000000..8053fe8cb8b9e1 --- /dev/null +++ b/Doc/library/nntplib.rst @@ -0,0 +1,15 @@ +:mod:`!nntplib` --- NNTP protocol client +======================================== + +.. module:: nntplib + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!nntplib` module was +`Python 3.12 `_. diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index 306bdd94aaca13..d0ae79c7a3df76 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -1,5 +1,5 @@ -:mod:`numbers` --- Numeric abstract base classes -================================================ +:mod:`!numbers` --- Numeric abstract base classes +================================================= .. module:: numbers :synopsis: Numeric abstract base classes (Complex, Real, Integral, etc.). @@ -84,10 +84,10 @@ The numeric tower ``~``. -Notes for type implementors +Notes for type implementers --------------------------- -Implementors should be careful to make equal numbers equal and hash +Implementers should be careful to make equal numbers equal and hash them to the same values. This may be subtle if there are two different extensions of the real numbers. For example, :class:`fractions.Fraction` implements :func:`hash` as follows:: diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 96f2c287875d41..e8e71068dd99eb 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -1,5 +1,5 @@ -:mod:`operator` --- Standard operators as functions -=================================================== +:mod:`!operator` --- Standard operators as functions +==================================================== .. module:: operator :synopsis: Functions corresponding to the standard operators. @@ -80,6 +80,20 @@ truth tests, identity tests, and boolean operations: Return ``a is not b``. Tests object identity. +.. function:: is_none(a) + + Return ``a is None``. Tests object identity. + + .. versionadded:: 3.14 + + +.. function:: is_not_none(a) + + Return ``a is not None``. Tests object identity. + + .. versionadded:: 3.14 + + The mathematical and bitwise operations are the most numerous: @@ -405,6 +419,10 @@ Python syntax and the functions in the :mod:`operator` module. +-----------------------+-------------------------+---------------------------------------+ | Identity | ``a is not b`` | ``is_not(a, b)`` | +-----------------------+-------------------------+---------------------------------------+ +| Identity | ``a is None`` | ``is_none(a)`` | ++-----------------------+-------------------------+---------------------------------------+ +| Identity | ``a is not None`` | ``is_not_none(a)`` | ++-----------------------+-------------------------+---------------------------------------+ | Indexed Assignment | ``obj[k] = v`` | ``setitem(obj, k, v)`` | +-----------------------+-------------------------+---------------------------------------+ | Indexed Deletion | ``del obj[k]`` | ``delitem(obj, k)`` | diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index 015e83ed2ce5f7..ff327cf9162a8c 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -1,27 +1,137 @@ -:mod:`optparse` --- Parser for command line options -=================================================== +:mod:`!optparse` --- Parser for command line options +==================================================== .. module:: optparse :synopsis: Command-line option parsing library. - :deprecated: .. moduleauthor:: Greg Ward .. sectionauthor:: Greg Ward **Source code:** :source:`Lib/optparse.py` -.. deprecated:: 3.2 - The :mod:`optparse` module is :term:`soft deprecated` and will not be - developed further; development will continue with the :mod:`argparse` - module. - -------------- +.. _choosing-an-argument-parser: + +Choosing an argument parsing library +------------------------------------ + +The standard library includes three argument parsing libraries: + +* :mod:`getopt`: a module that closely mirrors the procedural C ``getopt`` API. + Included in the standard library since before the initial Python 1.0 release. +* :mod:`optparse`: a declarative replacement for ``getopt`` that + provides equivalent functionality without requiring each application + to implement its own procedural option parsing logic. Included + in the standard library since the Python 2.3 release. +* :mod:`argparse`: a more opinionated alternative to ``optparse`` that + provides more functionality by default, at the expense of reduced application + flexibility in controlling exactly how arguments are processed. Included in + the standard library since the Python 2.7 and Python 3.2 releases. + +In the absence of more specific argument parsing design constraints, :mod:`argparse` +is the recommended choice for implementing command line applications, as it offers +the highest level of baseline functionality with the least application level code. + +:mod:`getopt` is retained almost entirely for backwards compatibility reasons. +However, it also serves a niche use case as a tool for prototyping and testing +command line argument handling in ``getopt``-based C applications. + +:mod:`optparse` should be considered as an alternative to :mod:`argparse` in the +following cases: + +* an application is already using :mod:`optparse` and doesn't want to risk the + subtle behavioural changes that may arise when migrating to :mod:`argparse` +* the application requires additional control over the way options and + positional parameters are interleaved on the command line (including + the ability to disable the interleaving feature completely) +* the application requires additional control over the incremental parsing + of command line elements (while ``argparse`` does support this, the + exact way it works in practice is undesirable for some use cases) +* the application requires additional control over the handling of options + which accept parameter values that may start with ``-`` (such as delegated + options to be passed to invoked subprocesses) +* the application requires some other command line parameter processing + behavior which ``argparse`` does not support, but which can be implemented + in terms of the lower level interface offered by ``optparse`` + +These considerations also mean that :mod:`optparse` is likely to provide a +better foundation for library authors writing third party command line +argument processing libraries. + +As a concrete example, consider the following two command line argument +parsing configurations, the first using ``optparse``, and the second +using ``argparse``: + +.. testcode:: + + import optparse + + if __name__ == '__main__': + parser = optparse.OptionParser() + parser.add_option('-o', '--output') + parser.add_option('-v', dest='verbose', action='store_true') + opts, args = parser.parse_args() + process(args, output=opts.output, verbose=opts.verbose) + +.. testcode:: + + import argparse + + if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-o', '--output') + parser.add_argument('-v', dest='verbose', action='store_true') + parser.add_argument('rest', nargs='*') + args = parser.parse_args() + process(args.rest, output=args.output, verbose=args.verbose) + +The most obvious difference is that in the ``optparse`` version, the non-option +arguments are processed separately by the application after the option processing +is complete. In the ``argparse`` version, positional arguments are declared and +processed in the same way as the named options. + +However, the ``argparse`` version will also handle some parameter combination +differently from the way the ``optparse`` version would handle them. +For example (amongst other differences): + +* supplying ``-o -v`` gives ``output="-v"`` and ``verbose=False`` + when using ``optparse``, but a usage error with ``argparse`` + (complaining that no value has been supplied for ``-o/--output``, + since ``-v`` is interpreted as meaning the verbosity flag) +* similarly, supplying ``-o --`` gives ``output="--"`` and ``args=()`` + when using ``optparse``, but a usage error with ``argparse`` + (also complaining that no value has been supplied for ``-o/--output``, + since ``--`` is interpreted as terminating the option processing + and treating all remaining values as positional arguments) +* supplying ``-o=foo`` gives ``output="=foo"`` when using ``optparse``, + but gives ``output="foo"`` with ``argparse`` (since ``=`` is special + cased as an alternative separator for option parameter values) + +Whether these differing behaviors in the ``argparse`` version are +considered desirable or a problem will depend on the specific command line +application use case. + +.. seealso:: + + :pypi:`click` is a third party argument processing library (originally + based on ``optparse``), which allows command line applications to be + developed as a set of decorated command implementation functions. + + Other third party libraries, such as :pypi:`typer` or :pypi:`msgspec-click`, + allow command line interfaces to be specified in ways that more effectively + integrate with static checking of Python type annotations. + + +Introduction +------------ + :mod:`optparse` is a more convenient, flexible, and powerful library for parsing -command-line options than the old :mod:`getopt` module. :mod:`optparse` uses a -more declarative style of command-line parsing: you create an instance of -:class:`OptionParser`, populate it with options, and parse the command -line. :mod:`optparse` allows users to specify options in the conventional +command-line options than the minimalist :mod:`getopt` module. +:mod:`optparse` uses a more declarative style of command-line parsing: +you create an instance of :class:`OptionParser`, +populate it with options, and parse the command line. +:mod:`optparse` allows users to specify options in the conventional GNU/POSIX syntax, and additionally generates usage and help messages for you. Here's an example of using :mod:`optparse` in a simple script:: @@ -82,10 +192,11 @@ Background ---------- :mod:`optparse` was explicitly designed to encourage the creation of programs -with straightforward, conventional command-line interfaces. To that end, it -supports only the most common command-line syntax and semantics conventionally -used under Unix. If you are unfamiliar with these conventions, read this -section to acquaint yourself with them. +with straightforward command-line interfaces that follow the conventions +established by the :c:func:`!getopt` family of functions available to C developers. +To that end, it supports only the most common command-line syntax and semantics +conventionally used under Unix. If you are unfamiliar with these conventions, +reading this section will allow you to acquaint yourself with them. .. _optparse-terminology: @@ -1352,7 +1463,7 @@ The whole point of creating and populating an OptionParser is to call its the list of arguments to process (default: ``sys.argv[1:]``) ``values`` - an :class:`Values` object to store option arguments in (default: a + a :class:`Values` object to store option arguments in (default: a new instance of :class:`Values`) -- if you give an existing object, the option defaults will not be initialized on it @@ -1739,7 +1850,7 @@ seen, but blow up if it comes after ``-b`` in the command-line. :: Callback example 3: check option order (generalized) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -If you want to re-use this callback for several similar options (set a flag, but +If you want to reuse this callback for several similar options (set a flag, but blow up if ``-b`` has already been seen), it needs a bit of work: the error message and the flag that it sets must be generalized. :: diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 3ee2b7db1e511b..ecbbc1d7605f9f 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -1,10 +1,10 @@ -:mod:`os.path` --- Common pathname manipulations -================================================ +:mod:`!os.path` --- Common pathname manipulations +================================================= .. module:: os.path :synopsis: Operations on pathnames. -**Source code:** :source:`Lib/posixpath.py` (for POSIX) and +**Source code:** :source:`Lib/genericpath.py`, :source:`Lib/posixpath.py` (for POSIX) and :source:`Lib/ntpath.py` (for Windows). .. index:: single: path; operations @@ -81,12 +81,10 @@ the :mod:`glob` module.) Return the longest common sub-path of each pathname in the iterable *paths*. Raise :exc:`ValueError` if *paths* contain both absolute - and relative pathnames, the *paths* are on the different drives or + and relative pathnames, if *paths* are on different drives, or if *paths* is empty. Unlike :func:`commonprefix`, this returns a valid path. - .. availability:: Unix, Windows. - .. versionadded:: 3.5 .. versionchanged:: 3.6 @@ -147,7 +145,7 @@ the :mod:`glob` module.) .. function:: lexists(path) - Return ``True`` if *path* refers to an existing path. Returns ``True`` for + Return ``True`` if *path* refers to an existing path, including broken symbolic links. Equivalent to :func:`exists` on platforms lacking :func:`os.lstat`. @@ -203,14 +201,14 @@ the :mod:`glob` module.) .. function:: getatime(path) - Return the time of last access of *path*. The return value is a floating point number giving + Return the time of last access of *path*. The return value is a floating-point number giving the number of seconds since the epoch (see the :mod:`time` module). Raise :exc:`OSError` if the file does not exist or is inaccessible. .. function:: getmtime(path) - Return the time of last modification of *path*. The return value is a floating point number + Return the time of last modification of *path*. The return value is a floating-point number giving the number of seconds since the epoch (see the :mod:`time` module). Raise :exc:`OSError` if the file does not exist or is inaccessible. @@ -304,8 +302,8 @@ the :mod:`glob` module.) always mount points, and for any other path ``GetVolumePathName`` is called to see if it is different from the input path. - .. versionadded:: 3.4 - Support for detecting non-root mount points on Windows. + .. versionchanged:: 3.4 + Added support for detecting non-root mount points on Windows. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -324,10 +322,11 @@ the :mod:`glob` module.) Dev Drives. See `the Windows documentation `_ for information on enabling and creating Dev Drives. - .. availability:: Windows. - .. versionadded:: 3.12 + .. versionchanged:: 3.13 + The function is now available on all platforms, and will always return ``False`` on those that have no support for Dev Drives + .. function:: isreserved(path) @@ -390,7 +389,7 @@ the :mod:`glob` module.) that contains symbolic links. On Windows, it converts forward slashes to backward slashes. To normalize case, use :func:`normcase`. - .. note:: + .. note:: On POSIX systems, in accordance with `IEEE Std 1003.1 2013 Edition; 4.13 Pathname Resolution `_, if a pathname begins with exactly two slashes, the first component @@ -410,9 +409,8 @@ the :mod:`glob` module.) style names such as ``C:\\PROGRA~1`` to ``C:\\Program Files``. If a path doesn't exist or a symlink loop is encountered, and *strict* is - ``True``, :exc:`OSError` is raised. If *strict* is ``False``, the path is - resolved as far as possible and any remainder is appended without checking - whether it exists. + ``True``, :exc:`OSError` is raised. If *strict* is ``False`` these errors + are ignored, and so the result might be missing or otherwise inaccessible. .. note:: This function emulates the operating system's procedure for making a path @@ -442,8 +440,6 @@ the :mod:`glob` module.) *start* defaults to :data:`os.curdir`. - .. availability:: Unix, Windows. - .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -454,8 +450,6 @@ the :mod:`glob` module.) This is determined by the device number and i-node number and raises an exception if an :func:`os.stat` call on either pathname fails. - .. availability:: Unix, Windows. - .. versionchanged:: 3.2 Added Windows support. @@ -470,8 +464,6 @@ the :mod:`glob` module.) Return ``True`` if the file descriptors *fp1* and *fp2* refer to the same file. - .. availability:: Unix, Windows. - .. versionchanged:: 3.2 Added Windows support. @@ -486,8 +478,6 @@ the :mod:`glob` module.) :func:`os.lstat`, or :func:`os.stat`. This function implements the underlying comparison used by :func:`samefile` and :func:`sameopenfile`. - .. availability:: Unix, Windows. - .. versionchanged:: 3.4 Added Windows support. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index cc9f3e75a80c51..7d3596622862ea 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1,5 +1,5 @@ -:mod:`os` --- Miscellaneous operating system interfaces -======================================================= +:mod:`!os` --- Miscellaneous operating system interfaces +======================================================== .. module:: os :synopsis: Miscellaneous operating system interfaces. @@ -34,12 +34,12 @@ Notes on the availability of these functions: * On VxWorks, os.popen, os.fork, os.execv and os.spawn*p* are not supported. -* On WebAssembly platforms ``wasm32-emscripten`` and ``wasm32-wasi``, large - parts of the :mod:`os` module are not available or behave differently. API - related to processes (e.g. :func:`~os.fork`, :func:`~os.execve`), signals - (e.g. :func:`~os.kill`, :func:`~os.wait`), and resources - (e.g. :func:`~os.nice`) are not available. Others like :func:`~os.getuid` - and :func:`~os.getpid` are emulated or stubs. +* On WebAssembly platforms, Android and iOS, large parts of the :mod:`os` module are + not available or behave differently. APIs related to processes (e.g. + :func:`~os.fork`, :func:`~os.execve`) and resources (e.g. :func:`~os.nice`) + are not available. Others like :func:`~os.getuid` and :func:`~os.getpid` are + emulated or stubs. WebAssembly platforms also lack support for signals (e.g. + :func:`~os.kill`, :func:`~os.wait`). .. note:: @@ -113,8 +113,8 @@ of the UTF-8 encoding: * Use UTF-8 as the :term:`filesystem encoding `. -* :func:`sys.getfilesystemencoding()` returns ``'utf-8'``. -* :func:`locale.getpreferredencoding()` returns ``'utf-8'`` (the *do_setlocale* +* :func:`sys.getfilesystemencoding` returns ``'utf-8'``. +* :func:`locale.getpreferredencoding` returns ``'utf-8'`` (the *do_setlocale* argument has no effect). * :data:`sys.stdin`, :data:`sys.stdout`, and :data:`sys.stderr` all use UTF-8 as their text encoding, with the ``surrogateescape`` @@ -133,8 +133,8 @@ level APIs also exhibit different default behaviours: * Command line arguments, environment variables and filenames are decoded to text using the UTF-8 encoding. -* :func:`os.fsdecode()` and :func:`os.fsencode()` use the UTF-8 encoding. -* :func:`open()`, :func:`io.open()`, and :func:`codecs.open()` use the UTF-8 +* :func:`os.fsdecode` and :func:`os.fsencode` use the UTF-8 encoding. +* :func:`open`, :func:`io.open`, and :func:`codecs.open` use the UTF-8 encoding by default. However, they still use the strict error handler by default so that attempting to open a binary file in text mode is likely to raise an exception rather than producing nonsense data. @@ -178,7 +178,7 @@ process and user. Return the filename corresponding to the controlling terminal of the process. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: environ @@ -222,6 +222,10 @@ process and user. :data:`os.environ`, and when one of the :meth:`pop` or :meth:`clear` methods is called. + .. seealso:: + + The :func:`os.reload_environ` function. + .. versionchanged:: 3.9 Updated to support :pep:`584`'s merge (``|``) and update (``|=``) operators. @@ -242,6 +246,24 @@ process and user. Updated to support :pep:`584`'s merge (``|``) and update (``|=``) operators. +.. function:: reload_environ() + + The :data:`os.environ` and :data:`os.environb` mappings are a cache of + environment variables at the time that Python started. + As such, changes to the current process environment are not reflected + if made outside Python, or by :func:`os.putenv` or :func:`os.unsetenv`. + Use :func:`!os.reload_environ` to update :data:`os.environ` and :data:`os.environb` + with any such changes to the current process environment. + + .. warning:: + This function is not thread-safe. Calling it while the environment is + being modified in an other thread is an undefined behavior. Reading from + :data:`os.environ` or :data:`os.environb`, or calling :func:`os.getenv` + while reloading, may return an empty result. + + .. versionadded:: 3.14 + + .. function:: chdir(path) fchdir(fd) getcwd() @@ -355,7 +377,7 @@ process and user. Return the effective group id of the current process. This corresponds to the "set id" bit on the file being executed in the current process. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: geteuid() @@ -364,7 +386,7 @@ process and user. Return the current process's effective user id. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: getgid() @@ -375,8 +397,8 @@ process and user. .. availability:: Unix. - The function is a stub on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is a stub on WASI, see :ref:`wasm-availability` for more + information. .. function:: getgrouplist(user, group, /) @@ -386,7 +408,7 @@ process and user. field from the password record for *user*, because that group ID will otherwise be potentially omitted. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.3 @@ -395,7 +417,7 @@ process and user. Return list of supplemental group ids associated with the current process. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. note:: @@ -423,7 +445,7 @@ process and user. falls back to ``pwd.getpwuid(os.getuid())[0]`` to get the login name of the current real user id. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI. .. function:: getpgid(pid) @@ -431,7 +453,7 @@ process and user. Return the process group id of the process with process id *pid*. If *pid* is 0, the process group id of the current process is returned. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: getpgrp() @@ -439,7 +461,7 @@ process and user. Return the id of the current process group. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: getpid() @@ -448,8 +470,8 @@ process and user. Return the current process id. - The function is a stub on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is a stub on WASI, see :ref:`wasm-availability` for more + information. .. function:: getppid() @@ -459,7 +481,7 @@ process and user. the id returned is the one of the init process (1), on Windows it is still the same id, which may be already reused by another process. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI. .. versionchanged:: 3.2 Added support for Windows. @@ -477,7 +499,7 @@ process and user. (respectively) the calling process, the process group of the calling process, or the real user ID of the calling process. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.3 @@ -488,7 +510,7 @@ process and user. Parameters for the :func:`getpriority` and :func:`setpriority` functions. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.3 @@ -509,7 +531,7 @@ process and user. Return a tuple (ruid, euid, suid) denoting the current process's real, effective, and saved user ids. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.2 @@ -519,7 +541,7 @@ process and user. Return a tuple (rgid, egid, sgid) denoting the current process's real, effective, and saved group ids. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.2 @@ -532,8 +554,8 @@ process and user. .. availability:: Unix. - The function is a stub on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is a stub on WASI, see :ref:`wasm-availability` for more + information. .. function:: initgroups(username, gid, /) @@ -542,7 +564,7 @@ process and user. the groups of which the specified username is a member, plus the specified group id. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android. .. versionadded:: 3.2 @@ -561,6 +583,8 @@ process and user. of :data:`os.environ`. This also applies to :func:`getenv` and :func:`getenvb`, which respectively use :data:`os.environ` and :data:`os.environb` in their implementations. + See also the :func:`os.reload_environ` function. + .. note:: On some platforms, including FreeBSD and macOS, setting ``environ`` may @@ -576,21 +600,21 @@ process and user. Set the current process's effective group id. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android. .. function:: seteuid(euid, /) Set the current process's effective user id. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android. .. function:: setgid(gid, /) Set the current process' group id. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android. .. function:: setgroups(groups, /) @@ -599,7 +623,7 @@ process and user. *groups*. *groups* must be a sequence, and each element must be an integer identifying a group. This operation is typically available only to the superuser. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. note:: On macOS, the length of *groups* may not exceed the system-defined maximum number of effective group ids, typically 16. @@ -649,7 +673,7 @@ process and user. Call the system call :c:func:`!setpgrp` or ``setpgrp(0, 0)`` depending on which version is implemented (if any). See the Unix manual for the semantics. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: setpgid(pid, pgrp, /) @@ -658,7 +682,7 @@ process and user. process with id *pid* to the process group with id *pgrp*. See the Unix manual for the semantics. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: setpriority(which, who, priority) @@ -675,7 +699,7 @@ process and user. *priority* is a value in the range -20 to 19. The default priority is 0; lower priorities cause more favorable scheduling. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.3 @@ -684,14 +708,14 @@ process and user. Set the current process's real and effective group ids. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android. .. function:: setresgid(rgid, egid, sgid, /) Set the current process's real, effective, and saved group ids. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android. .. versionadded:: 3.2 @@ -700,7 +724,7 @@ process and user. Set the current process's real, effective, and saved user ids. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android. .. versionadded:: 3.2 @@ -709,21 +733,21 @@ process and user. Set the current process's real and effective user ids. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android. .. function:: getsid(pid, /) Call the system call :c:func:`!getsid`. See the Unix manual for the semantics. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: setsid() Call the system call :c:func:`!setsid`. See the Unix manual for the semantics. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: setuid(uid, /) @@ -732,7 +756,7 @@ process and user. Set the current process's user id. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android. .. placed in this section since it relates to errno.... a little weak @@ -755,8 +779,8 @@ process and user. Set the current numeric umask and return the previous umask. - The function is a stub on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is a stub on WASI, see :ref:`wasm-availability` for more + information. .. function:: uname() @@ -784,6 +808,11 @@ process and user. :func:`socket.gethostname` or even ``socket.gethostbyaddr(socket.gethostname())``. + On macOS, iOS and Android, this returns the *kernel* name and version (i.e., + ``'Darwin'`` on macOS and iOS; ``'Linux'`` on Android). :func:`platform.uname` + can be used to get the user-facing operating system name and version on iOS and + Android. + .. availability:: Unix. .. versionchanged:: 3.3 @@ -804,6 +833,8 @@ process and user. don't update :data:`os.environ`, so it is actually preferable to delete items of :data:`os.environ`. + See also the :func:`os.reload_environ` function. + .. audit-event:: os.unsetenv key os.unsetenv .. versionchanged:: 3.9 @@ -918,10 +949,10 @@ as internal buffering of data. Copy *count* bytes from file descriptor *src*, starting from offset *offset_src*, to file descriptor *dst*, starting from offset *offset_dst*. - If *offset_src* is None, then *src* is read from the current position; + If *offset_src* is ``None``, then *src* is read from the current position; respectively for *offset_dst*. - In Linux kernel older than 5.3, the files pointed by *src* and *dst* + In Linux kernel older than 5.3, the files pointed to by *src* and *dst* must reside in the same filesystem, otherwise an :exc:`OSError` is raised with :attr:`~OSError.errno` set to :const:`errno.EXDEV`. @@ -1003,8 +1034,8 @@ as internal buffering of data. .. availability:: Unix, Windows. - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is limited on WASI, see :ref:`wasm-availability` for more + information. .. versionchanged:: 3.13 Added support on Windows. @@ -1021,8 +1052,8 @@ as internal buffering of data. .. availability:: Unix. - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is limited on WASI, see :ref:`wasm-availability` for more + information. .. function:: fdatasync(fd) @@ -1112,8 +1143,8 @@ as internal buffering of data. .. availability:: Unix, Windows. - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is limited on WASI, see :ref:`wasm-availability` for more + information. On Windows, this function is limited to pipes. @@ -1131,7 +1162,7 @@ as internal buffering of data. Calls the C standard library function :c:func:`grantpt`. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.13 @@ -1175,7 +1206,7 @@ as internal buffering of data. Make the calling process a session leader; make the tty the controlling tty, the stdin, the stdout, and the stderr of the calling process; close fd. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.11 @@ -1359,7 +1390,7 @@ or `the MSDN `_ on Windo descriptors are :ref:`non-inheritable `. For a (slightly) more portable approach, use the :mod:`pty` module. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionchanged:: 3.4 The new file descriptors are now non-inheritable. @@ -1385,7 +1416,7 @@ or `the MSDN `_ on Windo Return a pair of file descriptors ``(r, w)`` usable for reading and writing, respectively. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.3 @@ -1395,7 +1426,7 @@ or `the MSDN `_ on Windo Ensures that enough disk space is allocated for the file specified by *fd* starting from *offset* and continuing for *len* bytes. - .. availability:: Unix, not Emscripten. + .. availability:: Unix. .. versionadded:: 3.3 @@ -1455,7 +1486,7 @@ or `the MSDN `_ on Windo If the value :data:`O_CLOEXEC` is available on the system, it is added to *oflag*. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.13 @@ -1527,7 +1558,7 @@ or `the MSDN `_ on Windo it is available; otherwise, the C standard library function :c:func:`ptsname`, which is not guaranteed to be thread-safe, is called. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.13 @@ -1546,7 +1577,7 @@ or `the MSDN `_ on Windo .. function:: pwritev(fd, buffers, offset, flags=0, /) - Write the *buffers* contents to file descriptor *fd* at a offset *offset*, + Write the *buffers* contents to file descriptor *fd* at an offset *offset*, leaving the file offset unchanged. *buffers* must be a sequence of :term:`bytes-like objects `. Buffers are processed in array order. Entire contents of the first buffer is written before @@ -1628,6 +1659,33 @@ or `the MSDN `_ on Windo :exc:`InterruptedError` exception (see :pep:`475` for the rationale). +.. function:: readinto(fd, buffer, /) + + Read from a file descriptor *fd* into a mutable + :ref:`buffer object ` *buffer*. + + The *buffer* should be mutable and :term:`bytes-like `. On + success, returns the number of bytes read. Less bytes may be read than the + size of the buffer. The underlying system call will be retried when + interrupted by a signal, unless the signal handler raises an exception. + Other errors will not be retried and an error will be raised. + + Returns 0 if *fd* is at end of file or if the provided *buffer* has + length 0 (which can be used to check for errors without reading data). + Never returns negative. + + .. note:: + + This function is intended for low-level I/O and must be applied to a file + descriptor as returned by :func:`os.open` or :func:`os.pipe`. To read a + "file object" returned by the built-in function :func:`open`, or + :data:`sys.stdin`, use its member functions, for example + :meth:`io.BufferedIOBase.readinto`, :meth:`io.BufferedIOBase.read`, or + :meth:`io.TextIOBase.read` + + .. versionadded:: next + + .. function:: sendfile(out_fd, in_fd, offset, count) sendfile(out_fd, in_fd, offset, count, headers=(), trailers=(), flags=0) @@ -1654,7 +1712,7 @@ or `the MSDN `_ on Windo Cross-platform applications should not use *headers*, *trailers* and *flags* arguments. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. note:: @@ -1674,7 +1732,7 @@ or `the MSDN `_ on Windo Parameters to the :func:`sendfile` function, if the implementation supports them. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.3 @@ -1683,7 +1741,7 @@ or `the MSDN `_ on Windo Parameter to the :func:`sendfile` function, if the implementation supports it. The data won't be cached in the virtual memory and will be freed afterwards. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.11 @@ -1697,8 +1755,8 @@ or `the MSDN `_ on Windo .. availability:: Unix, Windows. - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is limited on WASI, see :ref:`wasm-availability` for more + information. On Windows, this function is limited to pipes. @@ -1708,14 +1766,31 @@ or `the MSDN `_ on Windo Added support for pipes on Windows. -.. function:: splice(src, dst, count, offset_src=None, offset_dst=None) +.. function:: splice(src, dst, count, offset_src=None, offset_dst=None, flags=0) Transfer *count* bytes from file descriptor *src*, starting from offset *offset_src*, to file descriptor *dst*, starting from offset *offset_dst*. + + The splicing behaviour can be modified by specifying a *flags* value. + Any of the following variables may used, combined using bitwise OR + (the ``|`` operator): + + * If :const:`SPLICE_F_MOVE` is specified, + the kernel is asked to move pages instead of copying, + but pages may still be copied if the kernel cannot move the pages from the pipe. + + * If :const:`SPLICE_F_NONBLOCK` is specified, + the kernel is asked to not block on I/O. + This makes the splice pipe operations nonblocking, + but splice may nevertheless block because the spliced file descriptors may block. + + * If :const:`SPLICE_F_MORE` is specified, + it hints to the kernel that more data will be coming in a subsequent splice. + At least one of the file descriptors must refer to a pipe. If *offset_src* - is None, then *src* is read from the current position; respectively for + is ``None``, then *src* is read from the current position; respectively for *offset_dst*. The offset associated to the file descriptor that refers to a - pipe must be ``None``. The files pointed by *src* and *dst* must reside in + pipe must be ``None``. The files pointed to by *src* and *dst* must reside in the same filesystem, otherwise an :exc:`OSError` is raised with :attr:`~OSError.errno` set to :const:`errno.EXDEV`. @@ -1730,6 +1805,8 @@ or `the MSDN `_ on Windo make sense to block because there are no writers connected to the write end of the pipe. + .. seealso:: The :manpage:`splice(2)` man page. + .. availability:: Linux >= 2.6.17 with glibc >= 2.5 .. versionadded:: 3.10 @@ -1792,7 +1869,7 @@ or `the MSDN `_ on Windo Calls the C standard library function :c:func:`unlockpt`. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionadded:: 3.13 @@ -1893,8 +1970,7 @@ Using the :mod:`subprocess` module, all file descriptors except standard streams are closed, and inheritable handles are only inherited if the *close_fds* parameter is ``False``. -On WebAssembly platforms ``wasm32-emscripten`` and ``wasm32-wasi``, the file -descriptor cannot be modified. +On WebAssembly platforms, the file descriptor cannot be modified. .. function:: get_inheritable(fd, /) @@ -2080,7 +2156,7 @@ features: .. audit-event:: os.chflags path,flags os.chflags - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionchanged:: 3.3 Added the *follow_symlinks* parameter. @@ -2126,12 +2202,12 @@ features: constants or a corresponding integer value). All other bits are ignored. The default value of *follow_symlinks* is ``False`` on Windows. - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is limited on WASI, see :ref:`wasm-availability` for more + information. .. audit-event:: os.chmod path,mode,dir_fd os.chmod - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor, and the *dir_fd* and *follow_symlinks* arguments. @@ -2159,10 +2235,10 @@ features: .. availability:: Unix. - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is limited on WASI, see :ref:`wasm-availability` for more + information. - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor, and the *dir_fd* and *follow_symlinks* arguments. @@ -2174,7 +2250,7 @@ features: Change the root directory of the current process to *path*. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -2214,7 +2290,7 @@ features: .. audit-event:: os.chflags path,flags os.lchflags - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -2264,7 +2340,7 @@ features: .. audit-event:: os.link src,dst,src_dir_fd,dst_dir_fd os.link - .. availability:: Unix, Windows, not Emscripten. + .. availability:: Unix, Windows. .. versionchanged:: 3.2 Added Windows support. @@ -2306,7 +2382,7 @@ features: .. versionchanged:: 3.2 The *path* parameter became optional. - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor. .. versionchanged:: 3.6 @@ -2426,6 +2502,10 @@ features: platform-dependent. On some platforms, they are ignored and you should call :func:`chmod` explicitly to set them. + On Windows, a *mode* of ``0o700`` is specifically handled to apply access + control to the new directory such that only the current user and + administrators have access. Other values of *mode* are ignored. + This function can also support :ref:`paths relative to directory descriptors `. @@ -2440,6 +2520,9 @@ features: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.13 + Windows now handles a *mode* of ``0o700``. + .. function:: makedirs(name, mode=0o777, exist_ok=False) @@ -2500,7 +2583,7 @@ features: FIFO for reading, and the client opens it for writing. Note that :func:`mkfifo` doesn't open the FIFO --- it just creates the rendezvous point. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionchanged:: 3.3 Added the *dir_fd* parameter. @@ -2522,7 +2605,7 @@ features: This function can also support :ref:`paths relative to directory descriptors `. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. versionchanged:: 3.3 Added the *dir_fd* parameter. @@ -2614,7 +2697,6 @@ features: .. versionchanged:: 3.8 Accepts a :term:`path-like object` and a bytes object on Windows. - .. versionchanged:: 3.8 Added support for directory junctions, and changed to return the substitution path (which typically includes ``\\?\`` prefix) rather than the optional "print name" field that was previously returned. @@ -2820,7 +2902,7 @@ features: .. versionchanged:: 3.6 Added support for the :term:`context manager` protocol and the - :func:`~scandir.close()` method. If a :func:`scandir` iterator is neither + :func:`~scandir.close` method. If a :func:`scandir` iterator is neither exhausted nor explicitly closed a :exc:`ResourceWarning` will be emitted in its destructor. @@ -3115,21 +3197,21 @@ features: Time of most recent access expressed in nanoseconds as an integer. - .. versionadded: 3.3 + .. versionadded:: 3.3 .. attribute:: st_mtime_ns Time of most recent content modification expressed in nanoseconds as an integer. - .. versionadded: 3.3 + .. versionadded:: 3.3 .. attribute:: st_ctime_ns Time of most recent metadata change expressed in nanoseconds as an integer. - .. versionadded: 3.3 + .. versionadded:: 3.3 .. versionchanged:: 3.12 ``st_ctime_ns`` is deprecated on Windows. Use ``st_birthtime_ns`` @@ -3257,10 +3339,10 @@ features: Windows now returns the file index as :attr:`st_ino` when available. - .. versionadded:: 3.7 + .. versionchanged:: 3.7 Added the :attr:`st_fstype` member to Solaris/derivatives. - .. versionadded:: 3.8 + .. versionchanged:: 3.8 Added the :attr:`st_reparse_tag` member on Windows. .. versionchanged:: 3.8 @@ -3274,16 +3356,13 @@ features: platforms, but for now still contains creation time. Use :attr:`st_birthtime` for the creation time. - .. versionchanged:: 3.12 On Windows, :attr:`st_ino` may now be up to 128 bits, depending on the file system. Previously it would not be above 64 bits, and larger file identifiers would be arbitrarily packed. - .. versionchanged:: 3.12 On Windows, :attr:`st_rdev` no longer returns a value. Previously it would contain the same as :attr:`st_dev`, which was incorrect. - .. versionadded:: 3.12 Added the :attr:`st_birthtime` member on Windows. @@ -3448,8 +3527,8 @@ features: .. availability:: Unix, Windows. - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. + The function is limited on WASI, see :ref:`wasm-availability` for more + information. .. versionchanged:: 3.2 Added support for Windows 6.0 (Vista) symbolic links. @@ -3638,6 +3717,7 @@ features: os.remove(os.path.join(root, name)) for name in dirs: os.rmdir(os.path.join(root, name)) + os.rmdir(top) .. audit-event:: os.walk top,topdown,onerror,followlinks os.walk @@ -3768,7 +3848,7 @@ features: new file descriptor is :ref:`non-inheritable `. *initval* is the initial value of the event counter. The initial value - must be an 32 bit unsigned integer. Please note that the initial value is + must be a 32 bit unsigned integer. Please note that the initial value is limited to a 32 bit unsigned int although the event counter is an unsigned 64 bit integer with a maximum value of 2\ :sup:`64`\ -\ 2. @@ -3847,7 +3927,7 @@ features: .. data:: EFD_SEMAPHORE - Provide semaphore-like semantics for reads from a :func:`eventfd` file + Provide semaphore-like semantics for reads from an :func:`eventfd` file descriptor. On read the internal counter is decremented by one. .. availability:: Linux >= 2.6.30 @@ -3855,6 +3935,8 @@ features: .. versionadded:: 3.10 +.. _os-timerfd: + Timer File Descriptors ~~~~~~~~~~~~~~~~~~~~~~ @@ -4275,7 +4357,7 @@ to be ignored. .. audit-event:: os.exec path,args,env os.execl - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI, not Android, not iOS. .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor @@ -4318,49 +4400,49 @@ written in Python, such as a mail server's external command delivery program. Exit code that means the command was used incorrectly, such as when the wrong number of arguments are given. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_DATAERR Exit code that means the input data was incorrect. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_NOINPUT Exit code that means an input file did not exist or was not readable. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_NOUSER Exit code that means a specified user did not exist. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_NOHOST Exit code that means a specified host did not exist. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_UNAVAILABLE Exit code that means that a required service is unavailable. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_SOFTWARE Exit code that means an internal software error was detected. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_OSERR @@ -4368,7 +4450,7 @@ written in Python, such as a mail server's external command delivery program. Exit code that means an operating system error was detected, such as the inability to fork or create a pipe. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_OSFILE @@ -4376,21 +4458,21 @@ written in Python, such as a mail server's external command delivery program. Exit code that means some system file did not exist, could not be opened, or had some other kind of error. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_CANTCREAT Exit code that means a user specified output file could not be created. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_IOERR Exit code that means that an error occurred while doing I/O on some file. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_TEMPFAIL @@ -4399,7 +4481,7 @@ written in Python, such as a mail server's external command delivery program. that may not really be an error, such as a network connection that couldn't be made during a retryable operation. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_PROTOCOL @@ -4407,7 +4489,7 @@ written in Python, such as a mail server's external command delivery program. Exit code that means that a protocol exchange was illegal, invalid, or not understood. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_NOPERM @@ -4415,21 +4497,21 @@ written in Python, such as a mail server's external command delivery program. Exit code that means that there were insufficient permissions to perform the operation (but not intended for file system problems). - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_CONFIG Exit code that means that some kind of configuration error occurred. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. data:: EX_NOTFOUND Exit code that means something like "an entry was not found". - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: fork() @@ -4478,7 +4560,7 @@ written in Python, such as a mail server's external command delivery program. for technical details of why we're surfacing this longstanding platform compatibility problem to developers. - .. availability:: POSIX, not Emscripten, not WASI. + .. availability:: POSIX, not WASI, not Android, not iOS. .. function:: forkpty() @@ -4496,16 +4578,16 @@ written in Python, such as a mail server's external command delivery program. On macOS the use of this function is unsafe when mixed with using higher-level system APIs, and that includes using :mod:`urllib.request`. + .. versionchanged:: 3.8 + Calling ``forkpty()`` in a subinterpreter is no longer supported + (:exc:`RuntimeError` is raised). + .. versionchanged:: 3.12 If Python is able to detect that your process has multiple threads, this now raises a :exc:`DeprecationWarning`. See the longer explanation on :func:`os.fork`. - .. versionchanged:: 3.8 - Calling ``forkpty()`` in a subinterpreter is no longer supported - (:exc:`RuntimeError` is raised). - - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. function:: kill(pid, sig, /) @@ -4522,14 +4604,13 @@ written in Python, such as a mail server's external command delivery program. only be sent to console processes which share a common console window, e.g., some subprocesses. Any other value for *sig* will cause the process to be unconditionally killed by the TerminateProcess API, and the exit code - will be set to *sig*. The Windows version of :func:`kill` additionally takes - process handles to be killed. + will be set to *sig*. See also :func:`signal.pthread_kill`. .. audit-event:: os.kill pid,sig os.kill - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI, not iOS. .. versionchanged:: 3.2 Added Windows support. @@ -4545,14 +4626,14 @@ written in Python, such as a mail server's external command delivery program. .. audit-event:: os.killpg pgid,sig os.killpg - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. function:: nice(increment, /) Add *increment* to the process's "niceness". Return the new niceness. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. .. function:: pidfd_open(pid, flags=0) @@ -4563,7 +4644,7 @@ written in Python, such as a mail server's external command delivery program. See the :manpage:`pidfd_open(2)` man page for more details. - .. availability:: Linux >= 5.3 + .. availability:: Linux >= 5.3, Android >= :func:`build-time ` API level 31 .. versionadded:: 3.9 .. data:: PIDFD_NONBLOCK @@ -4582,7 +4663,7 @@ written in Python, such as a mail server's external command delivery program. Lock program segments into memory. The value of *op* (defined in ````) determines which segments are locked. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not iOS. .. function:: popen(cmd, mode='r', buffering=-1) @@ -4614,7 +4695,7 @@ written in Python, such as a mail server's external command delivery program. documentation for more powerful ways to manage and communicate with subprocesses. - .. availability:: not Emscripten, not WASI. + .. availability:: not WASI, not Android, not iOS. .. note:: The :ref:`Python UTF-8 Mode ` affects encodings used @@ -4624,6 +4705,10 @@ written in Python, such as a mail server's external command delivery program. Use :class:`subprocess.Popen` or :func:`subprocess.run` to control options like encodings. + .. deprecated:: 3.14 + The function is :term:`soft deprecated` and should no longer be used to + write new code. The :mod:`subprocess` module is recommended instead. + .. function:: posix_spawn(path, argv, env, *, file_actions=None, \ setpgroup=None, resetids=False, setsid=False, setsigmask=(), \ @@ -4719,12 +4804,10 @@ written in Python, such as a mail server's external command delivery program. .. versionchanged:: 3.13 *env* parameter accepts ``None``. - - .. versionchanged:: 3.13 ``os.POSIX_SPAWN_CLOSEFROM`` is available on platforms where :c:func:`!posix_spawn_file_actions_addclosefrom_np` exists. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. function:: posix_spawnp(path, argv, env, *, file_actions=None, \ setpgroup=None, resetids=False, setsid=False, setsigmask=(), \ @@ -4740,7 +4823,7 @@ written in Python, such as a mail server's external command delivery program. .. versionadded:: 3.8 - .. availability:: POSIX, not Emscripten, not WASI. + .. availability:: POSIX, not WASI, not Android, not iOS. See :func:`posix_spawn` documentation. @@ -4773,7 +4856,7 @@ written in Python, such as a mail server's external command delivery program. There is no way to unregister a function. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. versionadded:: 3.7 @@ -4842,7 +4925,7 @@ written in Python, such as a mail server's external command delivery program. .. audit-event:: os.spawn mode,path,args,env os.spawnl - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI, not Android, not iOS. :func:`spawnlp`, :func:`spawnlpe`, :func:`spawnvp` and :func:`spawnvpe` are not available on Windows. :func:`spawnle` and @@ -4852,6 +4935,10 @@ written in Python, such as a mail server's external command delivery program. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. deprecated:: 3.14 + These functions are :term:`soft deprecated` and should no longer be used + to write new code. The :mod:`subprocess` module is recommended instead. + .. data:: P_NOWAIT P_NOWAITO @@ -4956,7 +5043,7 @@ written in Python, such as a mail server's external command delivery program. shell documentation. The :mod:`subprocess` module provides more powerful facilities for spawning - new processes and retrieving their results; using that module is preferable + new processes and retrieving their results; using that module is recommended to using this function. See the :ref:`subprocess-replacements` section in the :mod:`subprocess` documentation for some helpful recipes. @@ -4966,7 +5053,7 @@ written in Python, such as a mail server's external command delivery program. .. audit-event:: os.system command os.system - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI, not Android, not iOS. .. function:: times() @@ -5010,7 +5097,7 @@ written in Python, such as a mail server's external command delivery program. :func:`waitstatus_to_exitcode` can be used to convert the exit status into an exit code. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. seealso:: @@ -5044,7 +5131,7 @@ written in Python, such as a mail server's external command delivery program. Otherwise, if there are no matching children that could be waited for, :exc:`ChildProcessError` is raised. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. versionadded:: 3.3 @@ -5085,7 +5172,7 @@ written in Python, such as a mail server's external command delivery program. :func:`waitstatus_to_exitcode` can be used to convert the exit status into an exit code. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI, not Android, not iOS. .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise an @@ -5105,7 +5192,7 @@ written in Python, such as a mail server's external command delivery program. :func:`waitstatus_to_exitcode` can be used to convert the exit status into an exitcode. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. function:: wait4(pid, options) @@ -5119,7 +5206,7 @@ written in Python, such as a mail server's external command delivery program. :func:`waitstatus_to_exitcode` can be used to convert the exit status into an exitcode. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. data:: P_PID @@ -5136,7 +5223,7 @@ written in Python, such as a mail server's external command delivery program. * :data:`!P_PIDFD` - wait for the child identified by the file descriptor *id* (a process file descriptor created with :func:`pidfd_open`). - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. note:: :data:`!P_PIDFD` is only available on Linux >= 5.4. @@ -5151,7 +5238,7 @@ written in Python, such as a mail server's external command delivery program. :func:`waitid` causes child processes to be reported if they have been continued from a job control stop since they were last reported. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. data:: WEXITED @@ -5162,7 +5249,7 @@ written in Python, such as a mail server's external command delivery program. The other ``wait*`` functions always report children that have terminated, so this option is not available for them. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. versionadded:: 3.3 @@ -5174,7 +5261,7 @@ written in Python, such as a mail server's external command delivery program. This option is not available for the other ``wait*`` functions. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. versionadded:: 3.3 @@ -5187,7 +5274,7 @@ written in Python, such as a mail server's external command delivery program. This option is not available for :func:`waitid`. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. data:: WNOHANG @@ -5196,7 +5283,7 @@ written in Python, such as a mail server's external command delivery program. :func:`waitid` to return right away if no child process status is available immediately. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. data:: WNOWAIT @@ -5206,7 +5293,7 @@ written in Python, such as a mail server's external command delivery program. This option is not available for the other ``wait*`` functions. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. data:: CLD_EXITED @@ -5219,7 +5306,7 @@ written in Python, such as a mail server's external command delivery program. These are the possible values for :attr:`!si_code` in the result returned by :func:`waitid`. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. versionadded:: 3.3 @@ -5254,7 +5341,7 @@ written in Python, such as a mail server's external command delivery program. :func:`WIFEXITED`, :func:`WEXITSTATUS`, :func:`WIFSIGNALED`, :func:`WTERMSIG`, :func:`WIFSTOPPED`, :func:`WSTOPSIG` functions. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI, not Android, not iOS. .. versionadded:: 3.9 @@ -5270,7 +5357,7 @@ used to determine the disposition of a process. This function should be employed only if :func:`WIFSIGNALED` is true. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. function:: WIFCONTINUED(status) @@ -5281,7 +5368,7 @@ used to determine the disposition of a process. See :data:`WCONTINUED` option. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. function:: WIFSTOPPED(status) @@ -5293,14 +5380,14 @@ used to determine the disposition of a process. done using :data:`WUNTRACED` option or when the process is being traced (see :manpage:`ptrace(2)`). - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. function:: WIFSIGNALED(status) Return ``True`` if the process was terminated by a signal, otherwise return ``False``. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. function:: WIFEXITED(status) @@ -5309,7 +5396,7 @@ used to determine the disposition of a process. by calling ``exit()`` or ``_exit()``, or by returning from ``main()``; otherwise return ``False``. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. function:: WEXITSTATUS(status) @@ -5318,7 +5405,7 @@ used to determine the disposition of a process. This function should be employed only if :func:`WIFEXITED` is true. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. function:: WSTOPSIG(status) @@ -5327,7 +5414,7 @@ used to determine the disposition of a process. This function should be employed only if :func:`WIFSTOPPED` is true. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. .. function:: WTERMSIG(status) @@ -5336,7 +5423,7 @@ used to determine the disposition of a process. This function should be employed only if :func:`WIFSIGNALED` is true. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI, not Android, not iOS. Interface to the scheduler @@ -5351,6 +5438,8 @@ information, consult your Unix manpages. The following scheduling policies are exposed if they are supported by the operating system. +.. _os-scheduling-policy: + .. data:: SCHED_OTHER The default scheduling policy. @@ -5360,10 +5449,22 @@ operating system. Scheduling policy for CPU-intensive processes that tries to preserve interactivity on the rest of the computer. +.. data:: SCHED_DEADLINE + + Scheduling policy for tasks with deadline constraints. + + .. versionadded:: 3.14 + .. data:: SCHED_IDLE Scheduling policy for extremely low priority background tasks. +.. data:: SCHED_NORMAL + + Alias for :data:`SCHED_OTHER`. + + .. versionadded:: 3.14 + .. data:: SCHED_SPORADIC Scheduling policy for sporadic server programs. @@ -5442,7 +5543,7 @@ operating system. .. function:: sched_yield() - Voluntarily relinquish the CPU. + Voluntarily relinquish the CPU. See :manpage:`sched_yield(2)` for details. .. function:: sched_setaffinity(pid, mask, /) @@ -5534,7 +5635,7 @@ Miscellaneous System Information If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, :func:`process_cpu_count` returns the overridden value *n*. - See also the :func:`sched_getaffinity` functions. + See also the :func:`sched_getaffinity` function. .. versionadded:: 3.13 @@ -5708,20 +5809,20 @@ Random numbers easy-to-use interface to the random number generator provided by your platform, please see :class:`random.SystemRandom`. - .. versionchanged:: 3.6.0 - On Linux, ``getrandom()`` is now used in blocking mode to increase the - security. - - .. versionchanged:: 3.5.2 - On Linux, if the ``getrandom()`` syscall blocks (the urandom entropy pool - is not initialized yet), fall back on reading ``/dev/urandom``. - .. versionchanged:: 3.5 On Linux 3.17 and newer, the ``getrandom()`` syscall is now used when available. On OpenBSD 5.6 and newer, the C ``getentropy()`` function is now used. These functions avoid the usage of an internal file descriptor. + .. versionchanged:: 3.5.2 + On Linux, if the ``getrandom()`` syscall blocks (the urandom entropy pool + is not initialized yet), fall back on reading ``/dev/urandom``. + + .. versionchanged:: 3.6 + On Linux, ``getrandom()`` is now used in blocking mode to increase the + security. + .. versionchanged:: 3.11 On Windows, ``BCryptGenRandom()`` is used instead of ``CryptGenRandom()`` which is deprecated. diff --git a/Doc/library/ossaudiodev.rst b/Doc/library/ossaudiodev.rst new file mode 100644 index 00000000000000..320adbeff82539 --- /dev/null +++ b/Doc/library/ossaudiodev.rst @@ -0,0 +1,15 @@ +:mod:`!ossaudiodev` --- Access to OSS-compatible audio devices +============================================================== + +.. module:: ossaudiodev + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!ossaudiodev` module was +`Python 3.12 `_. diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 4b461a5d4a2949..4b48880d6d9a18 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1,13 +1,12 @@ - -:mod:`pathlib` --- Object-oriented filesystem paths -=================================================== +:mod:`!pathlib` --- Object-oriented filesystem paths +==================================================== .. module:: pathlib :synopsis: Object-oriented filesystem paths .. versionadded:: 3.4 -**Source code:** :source:`Lib/pathlib.py` +**Source code:** :source:`Lib/pathlib/` .. index:: single: path; operations @@ -22,6 +21,12 @@ inherit from pure paths but also provide I/O operations. .. image:: pathlib-inheritance.png :align: center :class: invert-in-dark-mode + :alt: Inheritance diagram showing the classes available in pathlib. The + most basic class is PurePath, which has three direct subclasses: + PurePosixPath, PureWindowsPath, and Path. Further to these four + classes, there are two classes that use multiple inheritance: + PosixPath subclasses PurePosixPath and Path, and WindowsPath + subclasses PureWindowsPath and Path. If you've never used this module before or just aren't sure which class is right for your task, :class:`Path` is most likely what you need. It instantiates @@ -173,8 +178,8 @@ we also call *flavours*: A subclass of :class:`PurePath`, this path flavour represents non-Windows filesystem paths:: - >>> PurePosixPath('/etc') - PurePosixPath('/etc') + >>> PurePosixPath('/etc/hosts') + PurePosixPath('/etc/hosts') *pathsegments* is specified similarly to :class:`PurePath`. @@ -183,8 +188,8 @@ we also call *flavours*: A subclass of :class:`PurePath`, this path flavour represents Windows filesystem paths, including `UNC paths`_:: - >>> PureWindowsPath('c:/Program Files/') - PureWindowsPath('c:/Program Files') + >>> PureWindowsPath('c:/', 'Users', 'Ximénez') + PureWindowsPath('c:/Users/Ximénez') >>> PureWindowsPath('//server/share/file') PureWindowsPath('//server/share/file') @@ -303,10 +308,10 @@ Methods and properties Pure paths provide the following methods and properties: -.. attribute:: PurePath.pathmod +.. attribute:: PurePath.parser The implementation of the :mod:`os.path` module used for low-level path - operations: either :mod:`posixpath` or :mod:`ntpath`. + parsing and joining: either :mod:`posixpath` or :mod:`ntpath`. .. versionadded:: 3.13 @@ -450,6 +455,10 @@ Pure paths provide the following methods and properties: This is commonly called the file extension. + .. versionchanged:: 3.14 + + A single dot ("``.``") is considered a valid suffix. + .. attribute:: PurePath.suffixes A list of the path's suffixes, often called file extensions:: @@ -461,6 +470,10 @@ Pure paths provide the following methods and properties: >>> PurePosixPath('my/library').suffixes [] + .. versionchanged:: 3.14 + + A single dot ("``.``") is considered a valid suffix. + .. attribute:: PurePath.stem @@ -628,8 +641,8 @@ Pure paths provide the following methods and properties: raise ValueError(error_message.format(str(self), str(formatted))) ValueError: '/etc/passwd' is not in the subpath of '/usr' OR one path is relative and the other is absolute. - When *walk_up* is False (the default), the path must start with *other*. - When the argument is True, ``..`` entries may be added to form the + When *walk_up* is false (the default), the path must start with *other*. + When the argument is true, ``..`` entries may be added to form the relative path. In all other cases, such as the paths referencing different drives, :exc:`ValueError` is raised.:: @@ -714,6 +727,11 @@ Pure paths provide the following methods and properties: >>> p.with_suffix('') PureWindowsPath('README') + .. versionchanged:: 3.14 + + A single dot ("``.``") is considered a valid suffix. In previous + versions, :exc:`ValueError` is raised if a single dot is supplied. + .. method:: PurePath.with_segments(*pathsegments) @@ -765,8 +783,8 @@ calls on path objects. There are three ways to instantiate concrete paths: A subclass of :class:`Path` and :class:`PurePosixPath`, this class represents concrete non-Windows filesystem paths:: - >>> PosixPath('/etc') - PosixPath('/etc') + >>> PosixPath('/etc/hosts') + PosixPath('/etc/hosts') *pathsegments* is specified similarly to :class:`PurePath`. @@ -780,8 +798,8 @@ calls on path objects. There are three ways to instantiate concrete paths: A subclass of :class:`Path` and :class:`PureWindowsPath`, this class represents concrete Windows filesystem paths:: - >>> WindowsPath('c:/Program Files/') - WindowsPath('c:/Program Files') + >>> WindowsPath('c:/', 'Users', 'Ximénez') + WindowsPath('c:/Users/Ximénez') *pathsegments* is specified similarly to :class:`PurePath`. @@ -808,9 +826,12 @@ bugs or failures in your application):: % (cls.__name__,)) UnsupportedOperation: cannot instantiate 'WindowsPath' on your system +Some concrete path methods can raise an :exc:`OSError` if a system call fails +(for example because the path doesn't exist). -File URIs -^^^^^^^^^ + +Parsing and generating URIs +^^^^^^^^^^^^^^^^^^^^^^^^^^^ Concrete path objects can be created from, and represented as, 'file' URIs conforming to :rfc:`8089`. @@ -870,21 +891,36 @@ conforming to :rfc:`8089`. it strictly impure. -Methods -^^^^^^^ +Expanding and resolving paths +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Concrete paths provide the following methods in addition to pure paths -methods. Many of these methods can raise an :exc:`OSError` if a system -call fails (for example because the path doesn't exist). +.. classmethod:: Path.home() -.. versionchanged:: 3.8 + Return a new path object representing the user's home directory (as + returned by :func:`os.path.expanduser` with ``~`` construct). If the home + directory can't be resolved, :exc:`RuntimeError` is raised. - :meth:`~Path.exists()`, :meth:`~Path.is_dir()`, :meth:`~Path.is_file()`, - :meth:`~Path.is_mount()`, :meth:`~Path.is_symlink()`, - :meth:`~Path.is_block_device()`, :meth:`~Path.is_char_device()`, - :meth:`~Path.is_fifo()`, :meth:`~Path.is_socket()` now return ``False`` - instead of raising an exception for paths that contain characters - unrepresentable at the OS level. + :: + + >>> Path.home() + PosixPath('/home/antoine') + + .. versionadded:: 3.5 + + +.. method:: Path.expanduser() + + Return a new path with expanded ``~`` and ``~user`` constructs, + as returned by :meth:`os.path.expanduser`. If a home directory can't be + resolved, :exc:`RuntimeError` is raised. + + :: + + >>> p = PosixPath('~/films/Monty Python') + >>> p.expanduser() + PosixPath('/home/eric/films/Monty Python') + + .. versionadded:: 3.5 .. classmethod:: Path.cwd() @@ -896,23 +932,91 @@ call fails (for example because the path doesn't exist). PosixPath('/home/antoine/pathlib') -.. classmethod:: Path.home() +.. method:: Path.absolute() - Return a new path object representing the user's home directory (as - returned by :func:`os.path.expanduser` with ``~`` construct). If the home - directory can't be resolved, :exc:`RuntimeError` is raised. + Make the path absolute, without normalization or resolving symlinks. + Returns a new path object:: - :: + >>> p = Path('tests') + >>> p + PosixPath('tests') + >>> p.absolute() + PosixPath('/home/antoine/pathlib/tests') - >>> Path.home() - PosixPath('/home/antoine') - .. versionadded:: 3.5 +.. method:: Path.resolve(strict=False) + + Make the path absolute, resolving any symlinks. A new path object is + returned:: + + >>> p = Path() + >>> p + PosixPath('.') + >>> p.resolve() + PosixPath('/home/antoine/pathlib') + + "``..``" components are also eliminated (this is the only method to do so):: + + >>> p = Path('docs/../setup.py') + >>> p.resolve() + PosixPath('/home/antoine/pathlib/setup.py') + + If a path doesn't exist or a symlink loop is encountered, and *strict* is + ``True``, :exc:`OSError` is raised. If *strict* is ``False``, the path is + resolved as far as possible and any remainder is appended without checking + whether it exists. + + .. versionchanged:: 3.6 + The *strict* parameter was added (pre-3.6 behavior is strict). + + .. versionchanged:: 3.13 + Symlink loops are treated like other errors: :exc:`OSError` is raised in + strict mode, and no exception is raised in non-strict mode. In previous + versions, :exc:`RuntimeError` is raised no matter the value of *strict*. + + +.. method:: Path.readlink() + + Return the path to which the symbolic link points (as returned by + :func:`os.readlink`):: + + >>> p = Path('mylink') + >>> p.symlink_to('setup.py') + >>> p.readlink() + PosixPath('setup.py') + + .. versionadded:: 3.9 + + .. versionchanged:: 3.13 + Raises :exc:`UnsupportedOperation` if :func:`os.readlink` is not + available. In previous versions, :exc:`NotImplementedError` was raised. + + +Querying file type and status +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionchanged:: 3.8 + + :meth:`~Path.exists`, :meth:`~Path.is_dir`, :meth:`~Path.is_file`, + :meth:`~Path.is_mount`, :meth:`~Path.is_symlink`, + :meth:`~Path.is_block_device`, :meth:`~Path.is_char_device`, + :meth:`~Path.is_fifo`, :meth:`~Path.is_socket` now return ``False`` + instead of raising an exception for paths that contain characters + unrepresentable at the OS level. + +.. versionchanged:: 3.14 + + The methods given above now return ``False`` instead of raising any + :exc:`OSError` exception from the operating system. In previous versions, + some kinds of :exc:`OSError` exception are raised, and others suppressed. + The new behaviour is consistent with :func:`os.path.exists`, + :func:`os.path.isdir`, etc. Use :meth:`~Path.stat` to retrieve the file + status without suppressing exceptions. .. method:: Path.stat(*, follow_symlinks=True) - Return a :class:`os.stat_result` object containing information about this path, like :func:`os.stat`. + Return an :class:`os.stat_result` object containing information about this path, like :func:`os.stat`. The result is looked up at each call to this method. This method normally follows symlinks; to stat a symlink add the argument @@ -929,29 +1033,18 @@ call fails (for example because the path doesn't exist). .. versionchanged:: 3.10 The *follow_symlinks* parameter was added. -.. method:: Path.chmod(mode, *, follow_symlinks=True) - - Change the file mode and permissions, like :func:`os.chmod`. - - This method normally follows symlinks. Some Unix flavours support changing - permissions on the symlink itself; on these platforms you may add the - argument ``follow_symlinks=False``, or use :meth:`~Path.lchmod`. - :: +.. method:: Path.lstat() - >>> p = Path('setup.py') - >>> p.stat().st_mode - 33277 - >>> p.chmod(0o444) - >>> p.stat().st_mode - 33060 + Like :meth:`Path.stat` but, if the path points to a symbolic link, return + the symbolic link's information rather than its target's. - .. versionchanged:: 3.10 - The *follow_symlinks* parameter was added. .. method:: Path.exists(*, follow_symlinks=True) Return ``True`` if the path points to an existing file or directory. + ``False`` will be returned if the path is invalid, inaccessible or missing. + Use :meth:`Path.stat` to distinguish between these cases. This method normally follows symlinks; to check if a symlink exists, add the argument ``follow_symlinks=False``. @@ -970,101 +1063,16 @@ call fails (for example because the path doesn't exist). .. versionchanged:: 3.12 The *follow_symlinks* parameter was added. -.. method:: Path.expanduser() - - Return a new path with expanded ``~`` and ``~user`` constructs, - as returned by :meth:`os.path.expanduser`. If a home directory can't be - resolved, :exc:`RuntimeError` is raised. - - :: - - >>> p = PosixPath('~/films/Monty Python') - >>> p.expanduser() - PosixPath('/home/eric/films/Monty Python') - - .. versionadded:: 3.5 - - -.. method:: Path.glob(pattern, *, case_sensitive=None, follow_symlinks=None) - - Glob the given relative *pattern* in the directory represented by this path, - yielding all matching files (of any kind):: - - >>> sorted(Path('.').glob('*.py')) - [PosixPath('pathlib.py'), PosixPath('setup.py'), PosixPath('test_pathlib.py')] - >>> sorted(Path('.').glob('*/*.py')) - [PosixPath('docs/conf.py')] - >>> sorted(Path('.').glob('**/*.py')) - [PosixPath('build/lib/pathlib.py'), - PosixPath('docs/conf.py'), - PosixPath('pathlib.py'), - PosixPath('setup.py'), - PosixPath('test_pathlib.py')] - - .. seealso:: - :ref:`pathlib-pattern-language` documentation. - - This method calls :meth:`Path.is_dir` on the top-level directory and - propagates any :exc:`OSError` exception that is raised. Subsequent - :exc:`OSError` exceptions from scanning directories are suppressed. - - By default, or when the *case_sensitive* keyword-only argument is set to - ``None``, this method matches paths using platform-specific casing rules: - typically, case-sensitive on POSIX, and case-insensitive on Windows. - Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. - - By default, or when the *follow_symlinks* keyword-only argument is set to - ``None``, this method follows symlinks except when expanding "``**``" - wildcards. Set *follow_symlinks* to ``True`` to always follow symlinks, or - ``False`` to treat all symlinks as files. - - .. tip:: - Set *follow_symlinks* to ``True`` or ``False`` to improve performance - of recursive globbing. - - .. audit-event:: pathlib.Path.glob self,pattern pathlib.Path.glob - - .. versionchanged:: 3.12 - The *case_sensitive* parameter was added. - - .. versionchanged:: 3.13 - The *follow_symlinks* parameter was added. - - .. versionchanged:: 3.13 - The *pattern* parameter accepts a :term:`path-like object`. - - -.. method:: Path.rglob(pattern, *, case_sensitive=None, follow_symlinks=None) - - Glob the given relative *pattern* recursively. This is like calling - :func:`Path.glob` with "``**/``" added in front of the *pattern*. - - .. seealso:: - :ref:`pathlib-pattern-language` and :meth:`Path.glob` documentation. - - .. audit-event:: pathlib.Path.rglob self,pattern pathlib.Path.rglob - - .. versionchanged:: 3.12 - The *case_sensitive* parameter was added. - - .. versionchanged:: 3.13 - The *follow_symlinks* parameter was added. - - .. versionchanged:: 3.13 - The *pattern* parameter accepts a :term:`path-like object`. - -.. method:: Path.group(*, follow_symlinks=True) - - Return the name of the group owning the file. :exc:`KeyError` is raised - if the file's gid isn't found in the system database. +.. method:: Path.is_file(*, follow_symlinks=True) - This method normally follows symlinks; to get the group of the symlink, add - the argument ``follow_symlinks=False``. + Return ``True`` if the path points to a regular file. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a regular file. Use :meth:`Path.stat` to + distinguish between these cases. - .. versionchanged:: 3.13 - Raises :exc:`UnsupportedOperation` if the :mod:`grp` module is not - available. In previous versions, :exc:`NotImplementedError` was raised. + This method normally follows symlinks; to exclude symlinks, add the + argument ``follow_symlinks=False``. .. versionchanged:: 3.13 The *follow_symlinks* parameter was added. @@ -1072,11 +1080,10 @@ call fails (for example because the path doesn't exist). .. method:: Path.is_dir(*, follow_symlinks=True) - Return ``True`` if the path points to a directory, ``False`` if it points - to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. + Return ``True`` if the path points to a directory. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a directory. Use :meth:`Path.stat` to distinguish + between these cases. This method normally follows symlinks; to exclude symlinks to directories, add the argument ``follow_symlinks=False``. @@ -1085,19 +1092,12 @@ call fails (for example because the path doesn't exist). The *follow_symlinks* parameter was added. -.. method:: Path.is_file(*, follow_symlinks=True) - - Return ``True`` if the path points to a regular file, ``False`` if it - points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. - - This method normally follows symlinks; to exclude symlinks, add the - argument ``follow_symlinks=False``. +.. method:: Path.is_symlink() - .. versionchanged:: 3.13 - The *follow_symlinks* parameter was added. + Return ``True`` if the path points to a symbolic link, even if that symlink + is broken. ``False`` will be returned if the path is invalid, inaccessible + or missing, or if it points to something other than a symbolic link. Use + :meth:`Path.stat` to distinguish between these cases. .. method:: Path.is_junction() @@ -1124,60 +1124,155 @@ call fails (for example because the path doesn't exist). .. versionchanged:: 3.12 Windows support was added. +.. method:: Path.is_socket() -.. method:: Path.is_symlink() + Return ``True`` if the path points to a Unix socket. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a Unix socket. Use :meth:`Path.stat` to + distinguish between these cases. - Return ``True`` if the path points to a symbolic link, ``False`` otherwise. - ``False`` is also returned if the path doesn't exist; other errors (such - as permission errors) are propagated. +.. method:: Path.is_fifo() + Return ``True`` if the path points to a FIFO. ``False`` will be returned if + the path is invalid, inaccessible or missing, or if it points to something + other than a FIFO. Use :meth:`Path.stat` to distinguish between these + cases. -.. method:: Path.is_socket() - Return ``True`` if the path points to a Unix socket (or a symbolic link - pointing to a Unix socket), ``False`` if it points to another kind of file. +.. method:: Path.is_block_device() - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. + Return ``True`` if the path points to a block device. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a block device. Use :meth:`Path.stat` to + distinguish between these cases. -.. method:: Path.is_fifo() +.. method:: Path.is_char_device() - Return ``True`` if the path points to a FIFO (or a symbolic link - pointing to a FIFO), ``False`` if it points to another kind of file. + Return ``True`` if the path points to a character device. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a character device. Use :meth:`Path.stat` to + distinguish between these cases. - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. +.. method:: Path.samefile(other_path) -.. method:: Path.is_block_device() + Return whether this path points to the same file as *other_path*, which + can be either a Path object, or a string. The semantics are similar + to :func:`os.path.samefile` and :func:`os.path.samestat`. - Return ``True`` if the path points to a block device (or a symbolic link - pointing to a block device), ``False`` if it points to another kind of file. + An :exc:`OSError` can be raised if either file cannot be accessed for some + reason. - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. + :: + >>> p = Path('spam') + >>> q = Path('eggs') + >>> p.samefile(q) + False + >>> p.samefile('spam') + True -.. method:: Path.is_char_device() + .. versionadded:: 3.5 - Return ``True`` if the path points to a character device (or a symbolic link - pointing to a character device), ``False`` if it points to another kind of file. - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. +Reading and writing files +^^^^^^^^^^^^^^^^^^^^^^^^^ -.. method:: Path.iterdir() +.. method:: Path.open(mode='r', buffering=-1, encoding=None, errors=None, newline=None) - When the path points to a directory, yield path objects of the directory - contents:: + Open the file pointed to by the path, like the built-in :func:`open` + function does:: - >>> p = Path('docs') - >>> for child in p.iterdir(): child + >>> p = Path('setup.py') + >>> with p.open() as f: + ... f.readline() ... - PosixPath('docs/conf.py') + '#!/usr/bin/env python3\n' + + +.. method:: Path.read_text(encoding=None, errors=None, newline=None) + + Return the decoded contents of the pointed-to file as a string:: + + >>> p = Path('my_text_file') + >>> p.write_text('Text file contents') + 18 + >>> p.read_text() + 'Text file contents' + + The file is opened and then closed. The optional parameters have the same + meaning as in :func:`open`. + + .. versionadded:: 3.5 + + .. versionchanged:: 3.13 + The *newline* parameter was added. + + +.. method:: Path.read_bytes() + + Return the binary contents of the pointed-to file as a bytes object:: + + >>> p = Path('my_binary_file') + >>> p.write_bytes(b'Binary file contents') + 20 + >>> p.read_bytes() + b'Binary file contents' + + .. versionadded:: 3.5 + + +.. method:: Path.write_text(data, encoding=None, errors=None, newline=None) + + Open the file pointed to in text mode, write *data* to it, and close the + file:: + + >>> p = Path('my_text_file') + >>> p.write_text('Text file contents') + 18 + >>> p.read_text() + 'Text file contents' + + An existing file of the same name is overwritten. The optional parameters + have the same meaning as in :func:`open`. + + .. versionadded:: 3.5 + + .. versionchanged:: 3.10 + The *newline* parameter was added. + + +.. method:: Path.write_bytes(data) + + Open the file pointed to in bytes mode, write *data* to it, and close the + file:: + + >>> p = Path('my_binary_file') + >>> p.write_bytes(b'Binary file contents') + 20 + >>> p.read_bytes() + b'Binary file contents' + + An existing file of the same name is overwritten. + + .. versionadded:: 3.5 + + +Reading directories +^^^^^^^^^^^^^^^^^^^ + +.. method:: Path.iterdir() + + When the path points to a directory, yield path objects of the directory + contents:: + + >>> p = Path('docs') + >>> for child in p.iterdir(): child + ... + PosixPath('docs/conf.py') PosixPath('docs/_templates') PosixPath('docs/make.bat') PosixPath('docs/index.rst') @@ -1187,8 +1282,77 @@ call fails (for example because the path doesn't exist). The children are yielded in arbitrary order, and the special entries ``'.'`` and ``'..'`` are not included. If a file is removed from or added - to the directory after creating the iterator, whether a path object for - that file be included is unspecified. + to the directory after creating the iterator, it is unspecified whether + a path object for that file is included. + + If the path is not a directory or otherwise inaccessible, :exc:`OSError` is + raised. + + +.. method:: Path.glob(pattern, *, case_sensitive=None, recurse_symlinks=False) + + Glob the given relative *pattern* in the directory represented by this path, + yielding all matching files (of any kind):: + + >>> sorted(Path('.').glob('*.py')) + [PosixPath('pathlib.py'), PosixPath('setup.py'), PosixPath('test_pathlib.py')] + >>> sorted(Path('.').glob('*/*.py')) + [PosixPath('docs/conf.py')] + >>> sorted(Path('.').glob('**/*.py')) + [PosixPath('build/lib/pathlib.py'), + PosixPath('docs/conf.py'), + PosixPath('pathlib.py'), + PosixPath('setup.py'), + PosixPath('test_pathlib.py')] + + .. seealso:: + :ref:`pathlib-pattern-language` documentation. + + By default, or when the *case_sensitive* keyword-only argument is set to + ``None``, this method matches paths using platform-specific casing rules: + typically, case-sensitive on POSIX, and case-insensitive on Windows. + Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. + + By default, or when the *recurse_symlinks* keyword-only argument is set to + ``False``, this method follows symlinks except when expanding "``**``" + wildcards. Set *recurse_symlinks* to ``True`` to always follow symlinks. + + .. audit-event:: pathlib.Path.glob self,pattern pathlib.Path.glob + + .. versionchanged:: 3.12 + The *case_sensitive* parameter was added. + + .. versionchanged:: 3.13 + The *recurse_symlinks* parameter was added. + + .. versionchanged:: 3.13 + The *pattern* parameter accepts a :term:`path-like object`. + + .. versionchanged:: 3.13 + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. In previous versions, such exceptions are suppressed in many + cases, but not all. + + +.. method:: Path.rglob(pattern, *, case_sensitive=None, recurse_symlinks=False) + + Glob the given relative *pattern* recursively. This is like calling + :func:`Path.glob` with "``**/``" added in front of the *pattern*. + + .. seealso:: + :ref:`pathlib-pattern-language` and :meth:`Path.glob` documentation. + + .. audit-event:: pathlib.Path.rglob self,pattern pathlib.Path.rglob + + .. versionchanged:: 3.12 + The *case_sensitive* parameter was added. + + .. versionchanged:: 3.13 + The *recurse_symlinks* parameter was added. + + .. versionchanged:: 3.13 + The *pattern* parameter accepts a :term:`path-like object`. + .. method:: Path.walk(top_down=True, on_error=None, follow_symlinks=False) @@ -1221,7 +1385,7 @@ call fails (for example because the path doesn't exist). This can be used to prune the search, or to impose a specific order of visiting, or even to inform :meth:`Path.walk` about directories the caller creates or renames before it resumes :meth:`Path.walk` again. Modifying *dirnames* when - *top_down* is false has no effect on the behavior of :meth:`Path.walk()` since the + *top_down* is false has no effect on the behavior of :meth:`Path.walk` since the directories in *dirnames* have already been generated by the time *dirnames* is yielded to the caller. @@ -1285,22 +1449,27 @@ call fails (for example because the path doesn't exist). .. versionadded:: 3.12 -.. method:: Path.lchmod(mode) - Like :meth:`Path.chmod` but, if the path points to a symbolic link, the - symbolic link's mode is changed rather than its target's. +Creating files and directories +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. method:: Path.touch(mode=0o666, exist_ok=True) -.. method:: Path.lstat() + Create a file at this given path. If *mode* is given, it is combined + with the process's ``umask`` value to determine the file mode and access + flags. If the file already exists, the function succeeds when *exist_ok* + is true (and its modification time is updated to the current time), + otherwise :exc:`FileExistsError` is raised. - Like :meth:`Path.stat` but, if the path points to a symbolic link, return - the symbolic link's information rather than its target's. + .. seealso:: + The :meth:`~Path.open`, :meth:`~Path.write_text` and + :meth:`~Path.write_bytes` methods are often used to create files. .. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False) Create a new directory at this given path. If *mode* is given, it is - combined with the process' ``umask`` value to determine the file mode + combined with the process's ``umask`` value to determine the file mode and access flags. If the path already exists, :exc:`FileExistsError` is raised. @@ -1322,87 +1491,102 @@ call fails (for example because the path doesn't exist). The *exist_ok* parameter was added. -.. method:: Path.open(mode='r', buffering=-1, encoding=None, errors=None, newline=None) - - Open the file pointed to by the path, like the built-in :func:`open` - function does:: +.. method:: Path.symlink_to(target, target_is_directory=False) - >>> p = Path('setup.py') - >>> with p.open() as f: - ... f.readline() - ... - '#!/usr/bin/env python3\n' + Make this path a symbolic link pointing to *target*. + On Windows, a symlink represents either a file or a directory, and does not + morph to the target dynamically. If the target is present, the type of the + symlink will be created to match. Otherwise, the symlink will be created + as a directory if *target_is_directory* is true or a file symlink (the + default) otherwise. On non-Windows platforms, *target_is_directory* is ignored. -.. method:: Path.owner(*, follow_symlinks=True) + :: - Return the name of the user owning the file. :exc:`KeyError` is raised - if the file's uid isn't found in the system database. + >>> p = Path('mylink') + >>> p.symlink_to('setup.py') + >>> p.resolve() + PosixPath('/home/antoine/pathlib/setup.py') + >>> p.stat().st_size + 956 + >>> p.lstat().st_size + 8 - This method normally follows symlinks; to get the owner of the symlink, add - the argument ``follow_symlinks=False``. + .. note:: + The order of arguments (link, target) is the reverse + of :func:`os.symlink`'s. .. versionchanged:: 3.13 - Raises :exc:`UnsupportedOperation` if the :mod:`pwd` module is not + Raises :exc:`UnsupportedOperation` if :func:`os.symlink` is not available. In previous versions, :exc:`NotImplementedError` was raised. - .. versionchanged:: 3.13 - The *follow_symlinks* parameter was added. +.. method:: Path.hardlink_to(target) -.. method:: Path.read_bytes() + Make this path a hard link to the same file as *target*. - Return the binary contents of the pointed-to file as a bytes object:: + .. note:: + The order of arguments (link, target) is the reverse + of :func:`os.link`'s. - >>> p = Path('my_binary_file') - >>> p.write_bytes(b'Binary file contents') - 20 - >>> p.read_bytes() - b'Binary file contents' + .. versionadded:: 3.10 - .. versionadded:: 3.5 + .. versionchanged:: 3.13 + Raises :exc:`UnsupportedOperation` if :func:`os.link` is not + available. In previous versions, :exc:`NotImplementedError` was raised. -.. method:: Path.read_text(encoding=None, errors=None, newline=None) +Copying, moving and deleting +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Return the decoded contents of the pointed-to file as a string:: +.. method:: Path.copy(target, *, follow_symlinks=True, dirs_exist_ok=False, \ + preserve_metadata=False) - >>> p = Path('my_text_file') - >>> p.write_text('Text file contents') - 18 - >>> p.read_text() - 'Text file contents' + Copy this file or directory tree to the given *target*, and return a new + :class:`!Path` instance pointing to *target*. - The file is opened and then closed. The optional parameters have the same - meaning as in :func:`open`. + If the source is a file, the target will be replaced if it is an existing + file. If the source is a symlink and *follow_symlinks* is true (the + default), the symlink's target is copied. Otherwise, the symlink is + recreated at the destination. - .. versionadded:: 3.5 + If the source is a directory and *dirs_exist_ok* is false (the default), a + :exc:`FileExistsError` is raised if the target is an existing directory. + If *dirs_exists_ok* is true, the copying operation will overwrite + existing files within the destination tree with corresponding files + from the source tree. - .. versionchanged:: 3.13 - The *newline* parameter was added. + If *preserve_metadata* is false (the default), only directory structures + and file data are guaranteed to be copied. Set *preserve_metadata* to true + to ensure that file and directory permissions, flags, last access and + modification times, and extended attributes are copied where supported. + This argument has no effect when copying files on Windows (where + metadata is always preserved). -.. method:: Path.readlink() + .. note:: + Where supported by the operating system and file system, this method + performs a lightweight copy, where data blocks are only copied when + modified. This is known as copy-on-write. - Return the path to which the symbolic link points (as returned by - :func:`os.readlink`):: + .. versionadded:: 3.14 - >>> p = Path('mylink') - >>> p.symlink_to('setup.py') - >>> p.readlink() - PosixPath('setup.py') - .. versionadded:: 3.9 +.. method:: Path.copy_into(target_dir, *, follow_symlinks=True, \ + dirs_exist_ok=False, preserve_metadata=False) - .. versionchanged:: 3.13 - Raises :exc:`UnsupportedOperation` if :func:`os.readlink` is not - available. In previous versions, :exc:`NotImplementedError` was raised. + Copy this file or directory tree into the given *target_dir*, which should + be an existing directory. Other arguments are handled identically to + :meth:`Path.copy`. Returns a new :class:`!Path` instance pointing to the + copy. + + .. versionadded:: 3.14 .. method:: Path.rename(target) - Rename this file or directory to the given *target*, and return a new Path - instance pointing to *target*. On Unix, if *target* exists and is a file, - it will be replaced silently if the user has permission. + Rename this file or directory to the given *target*, and return a new + :class:`!Path` instance pointing to *target*. On Unix, if *target* exists + and is a file, it will be replaced silently if the user has permission. On Windows, if *target* exists, :exc:`FileExistsError` will be raised. *target* can be either a string or another path object:: @@ -1416,200 +1600,135 @@ call fails (for example because the path doesn't exist). 'some text' The target path may be absolute or relative. Relative paths are interpreted - relative to the current working directory, *not* the directory of the Path - object. + relative to the current working directory, *not* the directory of the + :class:`!Path` object. It is implemented in terms of :func:`os.rename` and gives the same guarantees. .. versionchanged:: 3.8 - Added return value, return the new Path instance. + Added return value, return the new :class:`!Path` instance. .. method:: Path.replace(target) - Rename this file or directory to the given *target*, and return a new Path - instance pointing to *target*. If *target* points to an existing file or - empty directory, it will be unconditionally replaced. + Rename this file or directory to the given *target*, and return a new + :class:`!Path` instance pointing to *target*. If *target* points to an + existing file or empty directory, it will be unconditionally replaced. The target path may be absolute or relative. Relative paths are interpreted - relative to the current working directory, *not* the directory of the Path - object. + relative to the current working directory, *not* the directory of the + :class:`!Path` object. .. versionchanged:: 3.8 - Added return value, return the new Path instance. + Added return value, return the new :class:`!Path` instance. -.. method:: Path.absolute() +.. method:: Path.move(target) - Make the path absolute, without normalization or resolving symlinks. - Returns a new path object:: + Move this file or directory tree to the given *target*, and return a new + :class:`!Path` instance pointing to *target*. - >>> p = Path('tests') - >>> p - PosixPath('tests') - >>> p.absolute() - PosixPath('/home/antoine/pathlib/tests') + If the *target* doesn't exist it will be created. If both this path and the + *target* are existing files, then the target is overwritten. If both paths + point to the same file or directory, or the *target* is a non-empty + directory, then :exc:`OSError` is raised. + If both paths are on the same filesystem, the move is performed with + :func:`os.replace`. Otherwise, this path is copied (preserving metadata and + symlinks) and then deleted. -.. method:: Path.resolve(strict=False) + .. versionadded:: 3.14 - Make the path absolute, resolving any symlinks. A new path object is - returned:: - >>> p = Path() - >>> p - PosixPath('.') - >>> p.resolve() - PosixPath('/home/antoine/pathlib') +.. method:: Path.move_into(target_dir) - "``..``" components are also eliminated (this is the only method to do so):: - - >>> p = Path('docs/../setup.py') - >>> p.resolve() - PosixPath('/home/antoine/pathlib/setup.py') + Move this file or directory tree into the given *target_dir*, which should + be an existing directory. Returns a new :class:`!Path` instance pointing to + the moved path. - If a path doesn't exist or a symlink loop is encountered, and *strict* is - ``True``, :exc:`OSError` is raised. If *strict* is ``False``, the path is - resolved as far as possible and any remainder is appended without checking - whether it exists. + .. versionadded:: 3.14 - .. versionchanged:: 3.6 - The *strict* parameter was added (pre-3.6 behavior is strict). - .. versionchanged:: 3.13 - Symlink loops are treated like other errors: :exc:`OSError` is raised in - strict mode, and no exception is raised in non-strict mode. In previous - versions, :exc:`RuntimeError` is raised no matter the value of *strict*. - -.. method:: Path.rmdir() - - Remove this directory. The directory must be empty. - - -.. method:: Path.samefile(other_path) +.. method:: Path.unlink(missing_ok=False) - Return whether this path points to the same file as *other_path*, which - can be either a Path object, or a string. The semantics are similar - to :func:`os.path.samefile` and :func:`os.path.samestat`. + Remove this file or symbolic link. If the path points to a directory, + use :func:`Path.rmdir` instead. - An :exc:`OSError` can be raised if either file cannot be accessed for some - reason. + If *missing_ok* is false (the default), :exc:`FileNotFoundError` is + raised if the path does not exist. - :: + If *missing_ok* is true, :exc:`FileNotFoundError` exceptions will be + ignored (same behavior as the POSIX ``rm -f`` command). - >>> p = Path('spam') - >>> q = Path('eggs') - >>> p.samefile(q) - False - >>> p.samefile('spam') - True + .. versionchanged:: 3.8 + The *missing_ok* parameter was added. - .. versionadded:: 3.5 +.. method:: Path.rmdir() -.. method:: Path.symlink_to(target, target_is_directory=False) + Remove this directory. The directory must be empty. - Make this path a symbolic link pointing to *target*. - On Windows, a symlink represents either a file or a directory, and does not - morph to the target dynamically. If the target is present, the type of the - symlink will be created to match. Otherwise, the symlink will be created - as a directory if *target_is_directory* is ``True`` or a file symlink (the - default) otherwise. On non-Windows platforms, *target_is_directory* is ignored. +Permissions and ownership +^^^^^^^^^^^^^^^^^^^^^^^^^ - :: +.. method:: Path.owner(*, follow_symlinks=True) - >>> p = Path('mylink') - >>> p.symlink_to('setup.py') - >>> p.resolve() - PosixPath('/home/antoine/pathlib/setup.py') - >>> p.stat().st_size - 956 - >>> p.lstat().st_size - 8 + Return the name of the user owning the file. :exc:`KeyError` is raised + if the file's user identifier (UID) isn't found in the system database. - .. note:: - The order of arguments (link, target) is the reverse - of :func:`os.symlink`'s. + This method normally follows symlinks; to get the owner of the symlink, add + the argument ``follow_symlinks=False``. .. versionchanged:: 3.13 - Raises :exc:`UnsupportedOperation` if :func:`os.symlink` is not - available. In previous versions, :exc:`NotImplementedError` was raised. - - -.. method:: Path.hardlink_to(target) - - Make this path a hard link to the same file as *target*. - - .. note:: - The order of arguments (link, target) is the reverse - of :func:`os.link`'s. - - .. versionadded:: 3.10 + Raises :exc:`UnsupportedOperation` if the :mod:`pwd` module is not + available. In earlier versions, :exc:`NotImplementedError` was raised. .. versionchanged:: 3.13 - Raises :exc:`UnsupportedOperation` if :func:`os.link` is not - available. In previous versions, :exc:`NotImplementedError` was raised. - - -.. method:: Path.touch(mode=0o666, exist_ok=True) - - Create a file at this given path. If *mode* is given, it is combined - with the process' ``umask`` value to determine the file mode and access - flags. If the file already exists, the function succeeds if *exist_ok* - is true (and its modification time is updated to the current time), - otherwise :exc:`FileExistsError` is raised. - - -.. method:: Path.unlink(missing_ok=False) - - Remove this file or symbolic link. If the path points to a directory, - use :func:`Path.rmdir` instead. + The *follow_symlinks* parameter was added. - If *missing_ok* is false (the default), :exc:`FileNotFoundError` is - raised if the path does not exist. - If *missing_ok* is true, :exc:`FileNotFoundError` exceptions will be - ignored (same behavior as the POSIX ``rm -f`` command). +.. method:: Path.group(*, follow_symlinks=True) - .. versionchanged:: 3.8 - The *missing_ok* parameter was added. + Return the name of the group owning the file. :exc:`KeyError` is raised + if the file's group identifier (GID) isn't found in the system database. + This method normally follows symlinks; to get the group of the symlink, add + the argument ``follow_symlinks=False``. -.. method:: Path.write_bytes(data) + .. versionchanged:: 3.13 + Raises :exc:`UnsupportedOperation` if the :mod:`grp` module is not + available. In earlier versions, :exc:`NotImplementedError` was raised. - Open the file pointed to in bytes mode, write *data* to it, and close the - file:: + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. - >>> p = Path('my_binary_file') - >>> p.write_bytes(b'Binary file contents') - 20 - >>> p.read_bytes() - b'Binary file contents' - An existing file of the same name is overwritten. +.. method:: Path.chmod(mode, *, follow_symlinks=True) - .. versionadded:: 3.5 + Change the file mode and permissions, like :func:`os.chmod`. + This method normally follows symlinks. Some Unix flavours support changing + permissions on the symlink itself; on these platforms you may add the + argument ``follow_symlinks=False``, or use :meth:`~Path.lchmod`. -.. method:: Path.write_text(data, encoding=None, errors=None, newline=None) + :: - Open the file pointed to in text mode, write *data* to it, and close the - file:: + >>> p = Path('setup.py') + >>> p.stat().st_mode + 33277 + >>> p.chmod(0o444) + >>> p.stat().st_mode + 33060 - >>> p = Path('my_text_file') - >>> p.write_text('Text file contents') - 18 - >>> p.read_text() - 'Text file contents' + .. versionchanged:: 3.10 + The *follow_symlinks* parameter was added. - An existing file of the same name is overwritten. The optional parameters - have the same meaning as in :func:`open`. - .. versionadded:: 3.5 +.. method:: Path.lchmod(mode) - .. versionchanged:: 3.10 - The *newline* parameter was added. + Like :meth:`Path.chmod` but, if the path points to a symbolic link, the + symbolic link's mode is changed rather than its target's. .. _pathlib-pattern-language: @@ -1664,7 +1783,7 @@ the pattern to match only directories. Comparison to the :mod:`glob` module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------------ The patterns accepted and results generated by :meth:`Path.glob` and :meth:`Path.rglob` differ slightly from those by the :mod:`glob` module: @@ -1675,68 +1794,112 @@ The patterns accepted and results generated by :meth:`Path.glob` and passing ``recursive=True`` to :func:`glob.glob`. 3. "``**``" pattern components do not follow symlinks by default in pathlib. This behaviour has no equivalent in :func:`glob.glob`, but you can pass - ``follow_symlinks=True`` to :meth:`Path.glob` for compatible behaviour. + ``recurse_symlinks=True`` to :meth:`Path.glob` for compatible behaviour. 4. Like all :class:`PurePath` and :class:`Path` objects, the values returned from :meth:`Path.glob` and :meth:`Path.rglob` don't include trailing slashes. 5. The values returned from pathlib's ``path.glob()`` and ``path.rglob()`` include the *path* as a prefix, unlike the results of ``glob.glob(root_dir=path)``. -6. ``bytes``-based paths and :ref:`paths relative to directory descriptors - ` are not supported by pathlib. +6. The values returned from pathlib's ``path.glob()`` and ``path.rglob()`` + may include *path* itself, for example when globbing "``**``", whereas the + results of ``glob.glob(root_dir=path)`` never include an empty string that + would correspond to *path*. + + +Comparison to the :mod:`os` and :mod:`os.path` modules +------------------------------------------------------ + +pathlib implements path operations using :class:`PurePath` and :class:`Path` +objects, and so it's said to be *object-oriented*. On the other hand, the +:mod:`os` and :mod:`os.path` modules supply functions that work with low-level +``str`` and ``bytes`` objects, which is a more *procedural* approach. Some +users consider the object-oriented style to be more readable. + +Many functions in :mod:`os` and :mod:`os.path` support ``bytes`` paths and +:ref:`paths relative to directory descriptors `. These features aren't +available in pathlib. + +Python's ``str`` and ``bytes`` types, and portions of the :mod:`os` and +:mod:`os.path` modules, are written in C and are very speedy. pathlib is +written in pure Python and is often slower, but rarely slow enough to matter. +pathlib's path normalization is slightly more opinionated and consistent than +:mod:`os.path`. For example, whereas :func:`os.path.abspath` eliminates +"``..``" segments from a path, which may change its meaning if symlinks are +involved, :meth:`Path.absolute` preserves these segments for greater safety. -Correspondence to tools in the :mod:`os` module ------------------------------------------------ +pathlib's path normalization may render it unsuitable for some applications: + +1. pathlib normalizes ``Path("my_folder/")`` to ``Path("my_folder")``, which + changes a path's meaning when supplied to various operating system APIs and + command-line utilities. Specifically, the absence of a trailing separator + may allow the path to be resolved as either a file or directory, rather + than a directory only. +2. pathlib normalizes ``Path("./my_program")`` to ``Path("my_program")``, + which changes a path's meaning when used as an executable search path, such + as in a shell or when spawning a child process. Specifically, the absence + of a separator in the path may force it to be looked up in :envvar:`PATH` + rather than the current directory. + +As a consequence of these differences, pathlib is not a drop-in replacement +for :mod:`os.path`. + + +Corresponding tools +^^^^^^^^^^^^^^^^^^^ Below is a table mapping various :mod:`os` functions to their corresponding :class:`PurePath`/:class:`Path` equivalent. -.. note:: - - Not all pairs of functions/methods below are equivalent. Some of them, - despite having some overlapping use-cases, have different semantics. They - include :func:`os.path.abspath` and :meth:`Path.absolute`, - :func:`os.path.relpath` and :meth:`PurePath.relative_to`. - -==================================== ============================== -:mod:`os` and :mod:`os.path` :mod:`pathlib` -==================================== ============================== -:func:`os.path.abspath` :meth:`Path.absolute` [#]_ -:func:`os.path.realpath` :meth:`Path.resolve` -:func:`os.chmod` :meth:`Path.chmod` -:func:`os.mkdir` :meth:`Path.mkdir` -:func:`os.makedirs` :meth:`Path.mkdir` -:func:`os.rename` :meth:`Path.rename` -:func:`os.replace` :meth:`Path.replace` -:func:`os.rmdir` :meth:`Path.rmdir` -:func:`os.remove`, :func:`os.unlink` :meth:`Path.unlink` -:func:`os.getcwd` :func:`Path.cwd` -:func:`os.path.exists` :meth:`Path.exists` -:func:`os.path.expanduser` :meth:`Path.expanduser` and - :meth:`Path.home` -:func:`os.listdir` :meth:`Path.iterdir` -:func:`os.walk` :meth:`Path.walk` -:func:`os.path.isdir` :meth:`Path.is_dir` -:func:`os.path.isfile` :meth:`Path.is_file` -:func:`os.path.islink` :meth:`Path.is_symlink` -:func:`os.link` :meth:`Path.hardlink_to` -:func:`os.symlink` :meth:`Path.symlink_to` -:func:`os.readlink` :meth:`Path.readlink` -:func:`os.path.relpath` :meth:`PurePath.relative_to` [#]_ -:func:`os.stat` :meth:`Path.stat`, - :meth:`Path.owner`, - :meth:`Path.group` -:func:`os.path.isabs` :meth:`PurePath.is_absolute` -:func:`os.path.join` :func:`PurePath.joinpath` -:func:`os.path.basename` :attr:`PurePath.name` -:func:`os.path.dirname` :attr:`PurePath.parent` -:func:`os.path.samefile` :meth:`Path.samefile` -:func:`os.path.splitext` :attr:`PurePath.stem` and - :attr:`PurePath.suffix` -==================================== ============================== +===================================== ============================================== +:mod:`os` and :mod:`os.path` :mod:`pathlib` +===================================== ============================================== +:func:`os.path.dirname` :attr:`PurePath.parent` +:func:`os.path.basename` :attr:`PurePath.name` +:func:`os.path.splitext` :attr:`PurePath.stem`, :attr:`PurePath.suffix` +:func:`os.path.join` :meth:`PurePath.joinpath` +:func:`os.path.isabs` :meth:`PurePath.is_absolute` +:func:`os.path.relpath` :meth:`PurePath.relative_to` [1]_ +:func:`os.path.expanduser` :meth:`Path.expanduser` [2]_ +:func:`os.path.realpath` :meth:`Path.resolve` +:func:`os.path.abspath` :meth:`Path.absolute` [3]_ +:func:`os.path.exists` :meth:`Path.exists` +:func:`os.path.isfile` :meth:`Path.is_file` +:func:`os.path.isdir` :meth:`Path.is_dir` +:func:`os.path.islink` :meth:`Path.is_symlink` +:func:`os.path.isjunction` :meth:`Path.is_junction` +:func:`os.path.ismount` :meth:`Path.is_mount` +:func:`os.path.samefile` :meth:`Path.samefile` +:func:`os.getcwd` :meth:`Path.cwd` +:func:`os.stat` :meth:`Path.stat` +:func:`os.lstat` :meth:`Path.lstat` +:func:`os.listdir` :meth:`Path.iterdir` +:func:`os.walk` :meth:`Path.walk` [4]_ +:func:`os.mkdir`, :func:`os.makedirs` :meth:`Path.mkdir` +:func:`os.link` :meth:`Path.hardlink_to` +:func:`os.symlink` :meth:`Path.symlink_to` +:func:`os.readlink` :meth:`Path.readlink` +:func:`os.rename` :meth:`Path.rename` +:func:`os.replace` :meth:`Path.replace` +:func:`os.remove`, :func:`os.unlink` :meth:`Path.unlink` +:func:`os.rmdir` :meth:`Path.rmdir` +:func:`os.chmod` :meth:`Path.chmod` +:func:`os.lchmod` :meth:`Path.lchmod` +===================================== ============================================== .. rubric:: Footnotes -.. [#] :func:`os.path.abspath` normalizes the resulting path, which may change its meaning in the presence of symlinks, while :meth:`Path.absolute` does not. -.. [#] :meth:`PurePath.relative_to` requires ``self`` to be the subpath of the argument, but :func:`os.path.relpath` does not. +.. [1] :func:`os.path.relpath` calls :func:`~os.path.abspath` to make paths + absolute and remove "``..``" parts, whereas :meth:`PurePath.relative_to` + is a lexical operation that raises :exc:`ValueError` when its inputs' + anchors differ (e.g. if one path is absolute and the other relative.) +.. [2] :func:`os.path.expanduser` returns the path unchanged if the home + directory can't be resolved, whereas :meth:`Path.expanduser` raises + :exc:`RuntimeError`. +.. [3] :func:`os.path.abspath` removes "``..``" components without resolving + symlinks, which may change the meaning of the path, whereas + :meth:`Path.absolute` leaves any "``..``" components in the path. +.. [4] :func:`os.walk` always follows symlinks when categorizing paths into + *dirnames* and *filenames*, whereas :meth:`Path.walk` categorizes all + symlinks into *filenames* when *follow_symlinks* is false (the default.) diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index cb17acfb367619..f9d1213fb6d29d 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -49,7 +49,7 @@ You can then step through the code following this statement, and continue running without the debugger using the :pdbcmd:`continue` command. .. versionchanged:: 3.7 - The built-in :func:`breakpoint()`, when called with defaults, can be used + The built-in :func:`breakpoint`, when called with defaults, can be used instead of ``import pdb; pdb.set_trace()``. :: @@ -62,8 +62,8 @@ running without the debugger using the :pdbcmd:`continue` command. The debugger's prompt is ``(Pdb)``, which is the indicator that you are in debug mode:: - > ...(3)double() - -> return x * 2 + > ...(2)double() + -> breakpoint() (Pdb) p x 3 (Pdb) continue @@ -123,6 +123,11 @@ The typical usage to inspect a crashed program is:: 0 (Pdb) +.. versionchanged:: 3.13 + The implementation of :pep:`667` means that name assignments made via ``pdb`` + will immediately affect the active scope, even when running inside an + :term:`optimized scope`. + The module defines the following functions; each enters the debugger in a slightly different way: @@ -154,24 +159,35 @@ slightly different way: is entered. -.. function:: set_trace(*, header=None) +.. function:: set_trace(*, header=None, commands=None) Enter the debugger at the calling stack frame. This is useful to hard-code a breakpoint at a given point in a program, even if the code is not otherwise being debugged (e.g. when an assertion fails). If given, *header* is printed to the console just before debugging begins. + The *commands* argument, if given, is a list of commands to execute + when the debugger starts. + .. versionchanged:: 3.7 The keyword-only argument *header*. + .. versionchanged:: 3.13 + :func:`set_trace` will enter the debugger immediately, rather than + on the next line of code to be executed. + + .. versionadded:: 3.14 + The *commands* argument. -.. function:: post_mortem(traceback=None) +.. function:: post_mortem(t=None) - Enter post-mortem debugging of the given *traceback* object. If no - *traceback* is given, it uses the one of the exception that is currently - being handled (an exception must be being handled if the default is to be - used). + Enter post-mortem debugging of the given exception or + :ref:`traceback object `. If no value is given, it uses + the exception that is currently being handled, or raises ``ValueError`` if + there isn’t one. + .. versionchanged:: 3.13 + Support for exception objects was added. .. function:: pm() @@ -184,7 +200,7 @@ The ``run*`` functions and :func:`set_trace` are aliases for instantiating the access further features, you have to do this yourself: .. class:: Pdb(completekey='tab', stdin=None, stdout=None, skip=None, \ - nosigint=False, readrc=True) + nosigint=False, readrc=True, mode=None) :class:`Pdb` is the debugger class. @@ -203,6 +219,13 @@ access further features, you have to do this yourself: The *readrc* argument defaults to true and controls whether Pdb will load .pdbrc files from the filesystem. + The *mode* argument specifies how the debugger was invoked. + It impacts the workings of some debugger commands. + Valid values are ``'inline'`` (used by the breakpoint() builtin), + ``'cli'`` (used by the command line invocation) + or ``None`` (for backwards compatible behaviour, as before the *mode* + argument was added). + Example call to enable tracing with *skip*:: import pdb; pdb.Pdb(skip=['django.*']).set_trace() @@ -219,6 +242,9 @@ access further features, you have to do this yourself: .. versionchanged:: 3.6 The *readrc* argument. + .. versionadded:: 3.14 + Added the *mode* argument. + .. method:: run(statement, globals=None, locals=None) runeval(expression, globals=None, locals=None) runcall(function, *args, **kwds) @@ -282,25 +308,28 @@ There are three preset *convenience variables*: .. versionadded:: 3.12 + Added the *convenience variable* feature. + .. index:: pair: .pdbrc; file triple: debugger; configuration; file If a file :file:`.pdbrc` exists in the user's home directory or in the current directory, it is read with ``'utf-8'`` encoding and executed as if it had been -typed at the debugger prompt. This is particularly useful for aliases. If both +typed at the debugger prompt, with the exception that empty lines and lines +starting with ``#`` are ignored. This is particularly useful for aliases. If both files exist, the one in the home directory is read first and aliases defined there can be overridden by the local file. -.. versionchanged:: 3.11 - :file:`.pdbrc` is now read with ``'utf-8'`` encoding. Previously, it was read - with the system locale encoding. - .. versionchanged:: 3.2 :file:`.pdbrc` can now contain commands that continue debugging, such as :pdbcmd:`continue` or :pdbcmd:`next`. Previously, these commands had no effect. +.. versionchanged:: 3.11 + :file:`.pdbrc` is now read with ``'utf-8'`` encoding. Previously, it was read + with the system locale encoding. + .. pdbcommand:: h(elp) [command] @@ -310,11 +339,17 @@ can be overridden by the local file. argument must be an identifier, ``help exec`` must be entered to get help on the ``!`` command. -.. pdbcommand:: w(here) +.. pdbcommand:: w(here) [count] - Print a stack trace, with the most recent frame at the bottom. An arrow (``>``) + Print a stack trace, with the most recent frame at the bottom. if *count* + is 0, print the current frame entry. If *count* is negative, print the least + recent - *count* frames. If *count* is positive, print the most recent + *count* frames. An arrow (``>``) indicates the current frame, which determines the context of most commands. + .. versionchanged:: 3.14 + *count* argument is added. + .. pdbcommand:: d(own) [count] Move the current frame *count* (default one) levels down in the stack trace @@ -327,12 +362,16 @@ can be overridden by the local file. .. pdbcommand:: b(reak) [([filename:]lineno | function) [, condition]] - With a *lineno* argument, set a break there in the current file. With a - *function* argument, set a break at the first executable statement within - that function. The line number may be prefixed with a filename and a colon, - to specify a breakpoint in another file (probably one that hasn't been loaded - yet). The file is searched on :data:`sys.path`. Note that each breakpoint - is assigned a number to which all the other breakpoint commands refer. + With a *lineno* argument, set a break at line *lineno* in the current file. + The line number may be prefixed with a *filename* and a colon, + to specify a breakpoint in another file (possibly one that hasn't been loaded + yet). The file is searched on :data:`sys.path`. Acceptable forms of *filename* + are ``/abspath/to/file.py``, ``relpath/file.py``, ``module`` and + ``package.module``. + + With a *function* argument, set a break at the first executable statement within + that function. *function* can be any expression that evaluates to a function + in the current namespace. If a second argument is present, it is an expression which must evaluate to true before the breakpoint is honored. @@ -341,6 +380,9 @@ can be overridden by the local file. of times that breakpoint has been hit, the current ignore count, and the associated condition if any. + Each breakpoint is assigned a number to which all the other + breakpoint commands refer. + .. pdbcommand:: tbreak [([filename:]lineno | function) [, condition]] Temporary breakpoint, which is removed automatically when it is first hit. @@ -399,17 +441,20 @@ can be overridden by the local file. Specifying any command resuming execution (currently :pdbcmd:`continue`, :pdbcmd:`step`, :pdbcmd:`next`, - :pdbcmd:`return`, :pdbcmd:`jump`, :pdbcmd:`quit` and their abbreviations) + :pdbcmd:`return`, :pdbcmd:`until`, :pdbcmd:`jump`, :pdbcmd:`quit` and their abbreviations) terminates the command list (as if that command was immediately followed by end). This is because any time you resume execution (even with a simple next or step), you may encounter another breakpoint—which could have its own command list, leading to ambiguities about which list to execute. - If you use the ``silent`` command in the command list, the usual message about - stopping at a breakpoint is not printed. This may be desirable for breakpoints - that are to print a specific message and then continue. If none of the other - commands print anything, you see no sign that the breakpoint was reached. + If the list of commands contains the ``silent`` command, or a command that + resumes execution, then the breakpoint message containing information about + the frame is not displayed. + + .. versionchanged:: 3.14 + Frame information will not be displayed if a command that resumes execution + is present in the command list. .. pdbcommand:: s(tep) @@ -568,18 +613,17 @@ can be overridden by the local file. .. pdbcommand:: interact - Start an interactive interpreter (using the :mod:`code` module) whose global - namespace contains all the (global and local) names found in the current - scope. Use ``exit()`` or ``quit()`` to exit the interpreter and return to - the debugger. + Start an interactive interpreter (using the :mod:`code` module) in a new + global namespace initialised from the local and global namespaces for the + current scope. Use ``exit()`` or ``quit()`` to exit the interpreter and + return to the debugger. .. note:: - Because interact creates a new global namespace with the current global - and local namespace for execution, assignment to variables will not - affect the original namespaces. - However, modification to the mutable objects will be reflected in the - original namespaces. + As ``interact`` creates a new dedicated namespace for code execution, + assignments to variables will not affect the original namespaces. + However, modifications to any referenced mutable objects will be reflected + in the original namespaces as usual. .. versionadded:: 3.2 @@ -646,6 +690,10 @@ can be overridden by the local file. History, breakpoints, actions and debugger options are preserved. :pdbcmd:`restart` is an alias for :pdbcmd:`run`. + .. versionchanged:: 3.14 + :pdbcmd:`run` and :pdbcmd:`restart` commands are disabled when the + debugger is invoked in ``'inline'`` mode. + .. pdbcommand:: q(uit) Quit from the debugger. The program being executed is aborted. diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 223c27237e4d34..71fe3743c5968d 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -1,5 +1,5 @@ -:mod:`pickle` --- Python object serialization -============================================= +:mod:`!pickle` --- Python object serialization +============================================== .. module:: pickle :synopsis: Convert Python objects to streams of bytes and back. @@ -156,13 +156,14 @@ to read the pickle produced. * Protocol version 4 was added in Python 3.4. It adds support for very large objects, pickling more kinds of objects, and some data format - optimizations. It is the default protocol starting with Python 3.8. + optimizations. This was the default protocol in Python 3.8--3.13. Refer to :pep:`3154` for information about improvements brought by protocol 4. * Protocol version 5 was added in Python 3.8. It adds support for out-of-band - data and speedup for in-band data. Refer to :pep:`574` for information about - improvements brought by protocol 5. + data and speedup for in-band data. It is the default protocol starting with + Python 3.14. Refer to :pep:`574` for information about improvements brought + by protocol 5. .. note:: Serialization is a more primitive notion than persistence; although @@ -199,8 +200,10 @@ The :mod:`pickle` module provides the following constants: An integer, the default :ref:`protocol version ` used for pickling. May be less than :data:`HIGHEST_PROTOCOL`. Currently the - default protocol is 4, first introduced in Python 3.4 and incompatible - with previous versions. + default protocol is 5, introduced in Python 3.8 and incompatible + with previous versions. This version introduces support for out-of-band + buffers, where :pep:`3118`-compatible data can be transmitted separately + from the main pickle stream. .. versionchanged:: 3.0 @@ -210,6 +213,10 @@ The :mod:`pickle` module provides the following constants: The default protocol is 4. + .. versionchanged:: 3.14 + + The default protocol is 5. + The :mod:`pickle` module provides the following functions to make the pickling process more convenient: @@ -314,16 +321,16 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, map the new Python 3 names to the old module names used in Python 2, so that the pickle data stream is readable with Python 2. - If *buffer_callback* is None (the default), buffer views are + If *buffer_callback* is ``None`` (the default), buffer views are serialized into *file* as part of the pickle stream. - If *buffer_callback* is not None, then it can be called any number + If *buffer_callback* is not ``None``, then it can be called any number of times with a buffer view. If the callback returns a false value - (such as None), the given buffer is :ref:`out-of-band `; + (such as ``None``), the given buffer is :ref:`out-of-band `; otherwise the buffer is serialized in-band, i.e. inside the pickle stream. - It is an error if *buffer_callback* is not None and *protocol* is - None or smaller than 5. + It is an error if *buffer_callback* is not ``None`` and *protocol* is + ``None`` or smaller than 5. .. versionchanged:: 3.8 The *buffer_callback* argument was added. @@ -420,12 +427,12 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, instances of :class:`~datetime.datetime`, :class:`~datetime.date` and :class:`~datetime.time` pickled by Python 2. - If *buffers* is None (the default), then all data necessary for + If *buffers* is ``None`` (the default), then all data necessary for deserialization must be contained in the pickle stream. This means - that the *buffer_callback* argument was None when a :class:`Pickler` + that the *buffer_callback* argument was ``None`` when a :class:`Pickler` was instantiated (or when :func:`dump` or :func:`dumps` was called). - If *buffers* is not None, it should be an iterable of buffer-enabled + If *buffers* is not ``None``, it should be an iterable of buffer-enabled objects that is consumed each time the pickle stream references an :ref:`out-of-band ` buffer view. Such buffers have been given in order to the *buffer_callback* of a Pickler object. diff --git a/Doc/library/pickletools.rst b/Doc/library/pickletools.rst index 9739207a224431..e072605974f6c2 100644 --- a/Doc/library/pickletools.rst +++ b/Doc/library/pickletools.rst @@ -1,5 +1,5 @@ -:mod:`pickletools` --- Tools for pickle developers -================================================== +:mod:`!pickletools` --- Tools for pickle developers +=================================================== .. module:: pickletools :synopsis: Contains extensive comments about the pickle protocols and diff --git a/Doc/library/pipes.rst b/Doc/library/pipes.rst new file mode 100644 index 00000000000000..d9bcc3a5d99c9b --- /dev/null +++ b/Doc/library/pipes.rst @@ -0,0 +1,17 @@ +:mod:`!pipes` --- Interface to shell pipelines +============================================== + +.. module:: pipes + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +Applications should use the :mod:`subprocess` module instead. + +The last version of Python that provided the :mod:`!pipes` module was +`Python 3.12 `_. diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index 891a867d1ceb68..20b8f6bcf19117 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -1,5 +1,5 @@ -:mod:`pkgutil` --- Package extension utility -============================================ +:mod:`!pkgutil` --- Package extension utility +============================================= .. module:: pkgutil :synopsis: Utilities for the import system. @@ -26,7 +26,8 @@ support. __path__ = extend_path(__path__, __name__) For each directory on :data:`sys.path` that has a subdirectory that matches the - package name, add the subdirectory to the package's :attr:`__path__`. This is useful + package name, add the subdirectory to the package's + :attr:`~module.__path__`. This is useful if one wants to distribute different parts of a single logical package as multiple directories. @@ -34,9 +35,9 @@ support. *name* argument. This feature is similar to :file:`\*.pth` files (see the :mod:`site` module for more information), except that it doesn't special-case lines starting with ``import``. A :file:`\*.pkg` file is trusted at face - value: apart from checking for duplicates, all entries found in a - :file:`\*.pkg` file are added to the path, regardless of whether they exist - on the filesystem. (This is a feature.) + value: apart from skipping blank lines and ignoring comments, all entries + found in a :file:`\*.pkg` file are added to the path, regardless of whether + they exist on the filesystem (this is a feature). If the input path is not a list (as is the case for frozen packages) it is returned unchanged. The input path is not modified; an extended copy is @@ -48,25 +49,6 @@ support. this function to raise an exception (in line with :func:`os.path.isdir` behavior). -.. function:: find_loader(fullname) - - Retrieve a module :term:`loader` for the given *fullname*. - - This is a backwards compatibility wrapper around - :func:`importlib.util.find_spec` that converts most failures to - :exc:`ImportError` and only returns the loader rather than the full - :class:`importlib.machinery.ModuleSpec`. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal :pep:`302` import emulation. - - .. versionchanged:: 3.4 - Updated to be based on :pep:`451` - - .. deprecated-removed:: 3.12 3.14 - Use :func:`importlib.util.find_spec` instead. - .. function:: get_importer(path_item) @@ -83,27 +65,6 @@ support. on the package internal :pep:`302` import emulation. -.. function:: get_loader(module_or_name) - - Get a :term:`loader` object for *module_or_name*. - - If the module or package is accessible via the normal import mechanism, a - wrapper around the relevant part of that machinery is returned. Returns - ``None`` if the module cannot be found or imported. If the named module is - not already imported, its containing package (if any) is imported, in order - to establish the package ``__path__``. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal :pep:`302` import emulation. - - .. versionchanged:: 3.4 - Updated to be based on :pep:`451` - - .. deprecated-removed:: 3.12 3.14 - Use :func:`importlib.util.find_spec` instead. - - .. function:: iter_importers(fullname='') Yield :term:`finder` objects for the given module name. diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index ec2a7ebd5d6e0b..cfe1e7ba48da32 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -1,5 +1,5 @@ -:mod:`platform` --- Access to underlying platform's identifying data -===================================================================== +:mod:`!platform` --- Access to underlying platform's identifying data +====================================================================== .. module:: platform :synopsis: Retrieves as much platform identifying data as possible. @@ -148,6 +148,9 @@ Cross Platform Returns the system/OS name, such as ``'Linux'``, ``'Darwin'``, ``'Java'``, ``'Windows'``. An empty string is returned if the value cannot be determined. + On iOS and Android, this returns the user-facing OS name (i.e, ``'iOS``, + ``'iPadOS'`` or ``'Android'``). To obtain the kernel name (``'Darwin'`` or + ``'Linux'``), use :func:`os.uname`. .. function:: system_alias(system, release, version) @@ -161,6 +164,8 @@ Cross Platform Returns the system's release version, e.g. ``'#3 on degas'``. An empty string is returned if the value cannot be determined. + On iOS and Android, this is the user-facing OS version. To obtain the + Darwin or Linux kernel version, use :func:`os.uname`. .. function:: uname() @@ -196,6 +201,10 @@ Java Platform ``(os_name, os_version, os_arch)``. Values which cannot be determined are set to the defaults given as parameters (which all default to ``''``). + .. deprecated-removed:: 3.13 3.15 + It was largely untested, had a confusing API, + and was only useful for Jython support. + Windows Platform ---------------- @@ -210,8 +219,8 @@ Windows Platform default to an empty string). As a hint: *ptype* is ``'Uniprocessor Free'`` on single processor NT machines - and ``'Multiprocessor Free'`` on multi processor machines. The *'Free'* refers - to the OS version being free of debugging code. It could also state *'Checked'* + and ``'Multiprocessor Free'`` on multi processor machines. The ``'Free'`` refers + to the OS version being free of debugging code. It could also state ``'Checked'`` which means the OS version uses debugging code, i.e. code that checks arguments, ranges, etc. @@ -234,7 +243,6 @@ Windows Platform macOS Platform -------------- - .. function:: mac_ver(release='', versioninfo=('','',''), machine='') Get macOS version information and return it as tuple ``(release, versioninfo, @@ -244,6 +252,24 @@ macOS Platform Entries which cannot be determined are set to ``''``. All tuple entries are strings. +iOS Platform +------------ + +.. function:: ios_ver(system='', release='', model='', is_simulator=False) + + Get iOS version information and return it as a + :func:`~collections.namedtuple` with the following attributes: + + * ``system`` is the OS name; either ``'iOS'`` or ``'iPadOS'``. + * ``release`` is the iOS version number as a string (e.g., ``'17.2'``). + * ``model`` is the device model identifier; this will be a string like + ``'iPhone13,2'`` for a physical device, or ``'iPhone'`` on a simulator. + * ``is_simulator`` is a boolean describing if the app is running on a + simulator or a physical device. + + Entries which cannot be determined are set to the defaults given as + parameters. + Unix Platforms -------------- @@ -297,3 +323,51 @@ Linux Platforms return ids .. versionadded:: 3.10 + + +Android Platform +---------------- + +.. function:: android_ver(release="", api_level=0, manufacturer="", \ + model="", device="", is_emulator=False) + + Get Android device information. Returns a :func:`~collections.namedtuple` + with the following attributes. Values which cannot be determined are set to + the defaults given as parameters. + + * ``release`` - Android version, as a string (e.g. ``"14"``). + + * ``api_level`` - API level of the running device, as an integer (e.g. ``34`` + for Android 14). To get the API level which Python was built against, see + :func:`sys.getandroidapilevel`. + + * ``manufacturer`` - `Manufacturer name + `__. + + * ``model`` - `Model name + `__ – + typically the marketing name or model number. + + * ``device`` - `Device name + `__ – + typically the model number or a codename. + + * ``is_emulator`` - ``True`` if the device is an emulator; ``False`` if it's + a physical device. + + Google maintains a `list of known model and device names + `__. + + .. versionadded:: 3.13 + + +Miscellaneous +------------- + +.. function:: invalidate_caches() + + Clear out the internal cache of information, such as the :func:`uname`. + This is typically useful when the platform's :func:`node` is changed + by an external process and one needs to retrieve the updated value. + + .. versionadded:: 3.14 diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index 7416ca2650bab4..2906ebe7822f52 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -1,5 +1,5 @@ -:mod:`plistlib` --- Generate and parse Apple ``.plist`` files -============================================================= +:mod:`!plistlib` --- Generate and parse Apple ``.plist`` files +============================================================== .. module:: plistlib :synopsis: Generate and parse Apple plist files. @@ -117,7 +117,7 @@ This module defines the following functions: when a key of a dictionary is not a string, otherwise such keys are skipped. When *aware_datetime* is true and any field with type ``datetime.datetime`` - is set as a :ref:`aware object `, it will convert to + is set as an :ref:`aware object `, it will convert to UTC timezone before writing it. A :exc:`TypeError` will be raised if the object is of an unsupported type or diff --git a/Doc/library/poplib.rst b/Doc/library/poplib.rst index 943eb21f6eec02..23f20b00e6dc6d 100644 --- a/Doc/library/poplib.rst +++ b/Doc/library/poplib.rst @@ -1,5 +1,5 @@ -:mod:`poplib` --- POP3 protocol client -====================================== +:mod:`!poplib` --- POP3 protocol client +======================================= .. module:: poplib :synopsis: POP3 protocol client (requires sockets). diff --git a/Doc/library/posix.rst b/Doc/library/posix.rst index 5871574b442667..14ab3e91e8a8e4 100644 --- a/Doc/library/posix.rst +++ b/Doc/library/posix.rst @@ -1,5 +1,5 @@ -:mod:`posix` --- The most common POSIX system calls -=================================================== +:mod:`!posix` --- The most common POSIX system calls +==================================================== .. module:: posix :platform: Unix diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index 2a2eb098646364..2985f31bacb47a 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -1,5 +1,5 @@ -:mod:`pprint` --- Data pretty printer -===================================== +:mod:`!pprint` --- Data pretty printer +====================================== .. module:: pprint :synopsis: Data pretty printer. @@ -19,9 +19,8 @@ such as files, sockets or classes are included, as well as many other objects which are not representable as Python literals. The formatted representation keeps objects on a single line if it can, and -breaks them onto multiple lines if they don't fit within the allowed width. -Construct :class:`PrettyPrinter` objects explicitly if you need to adjust the -width constraint. +breaks them onto multiple lines if they don't fit within the allowed width, +adjustable by the *width* parameter defaulting to 80 characters. Dictionaries are sorted by key before the display is computed. @@ -36,13 +35,66 @@ Dictionaries are sorted by key before the display is computed. Functions --------- -.. function:: pp(object, *args, sort_dicts=False, **kwargs) +.. function:: pp(object, stream=None, indent=1, width=80, depth=None, *, \ + compact=False, sort_dicts=False, underscore_numbers=False) + + Prints the formatted representation of *object*, followed by a newline. + This function may be used in the interactive interpreter + instead of the :func:`print` function for inspecting values. + Tip: you can reassign ``print = pprint.pp`` for use within a scope. + + :param object: + The object to be printed. + + :param stream: + A file-like object to which the output will be written + by calling its :meth:`!write` method. + If ``None`` (the default), :data:`sys.stdout` is used. + :type stream: :term:`file-like object` | None + + :param int indent: + The amount of indentation added for each nesting level. + + :param int width: + The desired maximum number of characters per line in the output. + If a structure cannot be formatted within the width constraint, + a best effort will be made. + + :param depth: + The number of nesting levels which may be printed. + If the data structure being printed is too deep, + the next contained level is replaced by ``...``. + If ``None`` (the default), there is no constraint + on the depth of the objects being formatted. + :type depth: int | None + + :param bool compact: + Control the way long :term:`sequences ` are formatted. + If ``False`` (the default), + each item of a sequence will be formatted on a separate line, + otherwise as many items as will fit within the *width* + will be formatted on each output line. + + :param bool sort_dicts: + If ``True``, dictionaries will be formatted with + their keys sorted, otherwise + they will be displayed in insertion order (the default). + + :param bool underscore_numbers: + If ``True``, + integers will be formatted with the ``_`` character for a thousands separator, + otherwise underscores are not displayed (the default). - Prints the formatted representation of *object* followed by a newline. - If *sort_dicts* is false (the default), dictionaries will be displayed with - their keys in insertion order, otherwise the dict keys will be sorted. - *args* and *kwargs* will be passed to :func:`~pprint.pprint` as formatting - parameters. + >>> import pprint + >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] + >>> stuff.insert(0, stuff) + >>> pprint.pp(stuff) + [, + 'spam', + 'eggs', + 'lumberjack', + 'knights', + 'ni'] .. versionadded:: 3.8 @@ -50,27 +102,10 @@ Functions .. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ compact=False, sort_dicts=True, underscore_numbers=False) - Prints the formatted representation of *object* on *stream*, followed by a - newline. If *stream* is ``None``, :data:`sys.stdout` is used. This may be used - in the interactive interpreter instead of the :func:`print` function for - inspecting values (you can even reassign ``print = pprint.pprint`` for use - within a scope). - - The configuration parameters *stream*, *indent*, *width*, *depth*, - *compact*, *sort_dicts* and *underscore_numbers* are passed to the - :class:`PrettyPrinter` constructor and their meanings are as - described in its documentation above. - - >>> import pprint - >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] - >>> stuff.insert(0, stuff) - >>> pprint.pprint(stuff) - [, - 'spam', - 'eggs', - 'lumberjack', - 'knights', - 'ni'] + Alias for :func:`~pprint.pp` with *sort_dicts* set to ``True`` by default, + which would automatically sort the dictionaries' keys, + you might want to use :func:`~pprint.pp` instead where it is ``False`` by default. + .. function:: pformat(object, indent=1, width=80, depth=None, *, \ compact=False, sort_dicts=True, underscore_numbers=False) @@ -78,7 +113,7 @@ Functions Return the formatted representation of *object* as a string. *indent*, *width*, *depth*, *compact*, *sort_dicts* and *underscore_numbers* are passed to the :class:`PrettyPrinter` constructor as formatting parameters - and their meanings are as described in its documentation above. + and their meanings are as described in the documentation above. .. function:: isreadable(object) @@ -117,51 +152,39 @@ Functions PrettyPrinter Objects --------------------- -This module defines one class: - -.. First the implementation class: - - .. index:: single: ...; placeholder .. class:: PrettyPrinter(indent=1, width=80, depth=None, stream=None, *, \ compact=False, sort_dicts=True, underscore_numbers=False) - Construct a :class:`PrettyPrinter` instance. This constructor understands - several keyword parameters. - - *stream* (default :data:`!sys.stdout`) is a :term:`file-like object` to - which the output will be written by calling its :meth:`!write` method. - If both *stream* and :data:`!sys.stdout` are ``None``, then - :meth:`~PrettyPrinter.pprint` silently returns. - - Other values configure the manner in which nesting of complex data - structures is displayed. + Construct a :class:`PrettyPrinter` instance. - *indent* (default 1) specifies the amount of indentation added for - each nesting level. + Arguments have the same meaning as for :func:`~pprint.pp`. + Note that they are in a different order, and that *sort_dicts* defaults to ``True``. - *depth* controls the number of nesting levels which may be printed; if - the data structure being printed is too deep, the next contained level - is replaced by ``...``. By default, there is no constraint on the - depth of the objects being formatted. - - *width* (default 80) specifies the desired maximum number of characters per - line in the output. If a structure cannot be formatted within the width - constraint, a best effort will be made. - - *compact* impacts the way that long sequences (lists, tuples, sets, etc) - are formatted. If *compact* is false (the default) then each item of a - sequence will be formatted on a separate line. If *compact* is true, as - many items as will fit within the *width* will be formatted on each output - line. - - If *sort_dicts* is true (the default), dictionaries will be formatted with - their keys sorted, otherwise they will display in insertion order. + >>> import pprint + >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] + >>> stuff.insert(0, stuff[:]) + >>> pp = pprint.PrettyPrinter(indent=4) + >>> pp.pprint(stuff) + [ ['spam', 'eggs', 'lumberjack', 'knights', 'ni'], + 'spam', + 'eggs', + 'lumberjack', + 'knights', + 'ni'] + >>> pp = pprint.PrettyPrinter(width=41, compact=True) + >>> pp.pprint(stuff) + [['spam', 'eggs', 'lumberjack', + 'knights', 'ni'], + 'spam', 'eggs', 'lumberjack', 'knights', + 'ni'] + >>> tup = ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', + ... ('parrot', ('fresh fruit',)))))))) + >>> pp = pprint.PrettyPrinter(depth=6) + >>> pp.pprint(tup) + ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', (...))))))) - If *underscore_numbers* is true, integers will be formatted with the - ``_`` character for a thousands separator, otherwise underscores are not - displayed (the default). .. versionchanged:: 3.4 Added the *compact* parameter. @@ -175,29 +198,6 @@ This module defines one class: .. versionchanged:: 3.11 No longer attempts to write to :data:`!sys.stdout` if it is ``None``. - >>> import pprint - >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] - >>> stuff.insert(0, stuff[:]) - >>> pp = pprint.PrettyPrinter(indent=4) - >>> pp.pprint(stuff) - [ ['spam', 'eggs', 'lumberjack', 'knights', 'ni'], - 'spam', - 'eggs', - 'lumberjack', - 'knights', - 'ni'] - >>> pp = pprint.PrettyPrinter(width=41, compact=True) - >>> pp.pprint(stuff) - [['spam', 'eggs', 'lumberjack', - 'knights', 'ni'], - 'spam', 'eggs', 'lumberjack', 'knights', - 'ni'] - >>> tup = ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', - ... ('parrot', ('fresh fruit',)))))))) - >>> pp = pprint.PrettyPrinter(depth=6) - >>> pp.pprint(tup) - ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', (...))))))) - :class:`PrettyPrinter` instances have the following methods: @@ -261,18 +261,18 @@ are converted to strings. The default implementation uses the internals of the Example ------- -To demonstrate several uses of the :func:`~pprint.pprint` function and its parameters, +To demonstrate several uses of the :func:`~pprint.pp` function and its parameters, let's fetch information about a project from `PyPI `_:: >>> import json >>> import pprint >>> from urllib.request import urlopen - >>> with urlopen('https://pypi.org/pypi/sampleproject/json') as resp: + >>> with urlopen('https://pypi.org/pypi/sampleproject/1.2.0/json') as resp: ... project_info = json.load(resp)['info'] -In its basic form, :func:`~pprint.pprint` shows the whole object:: +In its basic form, :func:`~pprint.pp` shows the whole object:: - >>> pprint.pprint(project_info) + >>> pprint.pp(project_info) {'author': 'The Python Packaging Authority', 'author_email': 'pypa-dev@googlegroups.com', 'bugtrack_url': None, @@ -329,7 +329,7 @@ In its basic form, :func:`~pprint.pprint` shows the whole object:: The result can be limited to a certain *depth* (ellipsis is used for deeper contents):: - >>> pprint.pprint(project_info, depth=1) + >>> pprint.pp(project_info, depth=1) {'author': 'The Python Packaging Authority', 'author_email': 'pypa-dev@googlegroups.com', 'bugtrack_url': None, @@ -375,7 +375,7 @@ contents):: Additionally, maximum character *width* can be suggested. If a long object cannot be split, the specified width will be exceeded:: - >>> pprint.pprint(project_info, depth=1, width=60) + >>> pprint.pp(project_info, depth=1, width=60) {'author': 'The Python Packaging Authority', 'author_email': 'pypa-dev@googlegroups.com', 'bugtrack_url': None, diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 3ca802e024bc27..3334833eba6b8c 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -234,7 +234,7 @@ functions: .. function:: runctx(command, globals, locals, filename=None, sort=-1) This function is similar to :func:`run`, with added arguments to supply the - globals and locals dictionaries for the *command* string. This routine + globals and locals mappings for the *command* string. This routine executes:: exec(command, globals, locals) @@ -682,7 +682,7 @@ you are using :class:`profile.Profile` or :class:`cProfile.Profile`, that you choose (see :ref:`profile-calibration`). For most machines, a timer that returns a lone integer value will provide the best results in terms of low overhead during profiling. (:func:`os.times` is *pretty* bad, as it - returns a tuple of floating point values). If you want to substitute a + returns a tuple of floating-point values). If you want to substitute a better timer in the cleanest fashion, derive a class and hardwire a replacement dispatch method that best handles your timer call, along with the appropriate calibration constant. @@ -699,7 +699,7 @@ you are using :class:`profile.Profile` or :class:`cProfile.Profile`, As the :class:`cProfile.Profile` class cannot be calibrated, custom timer functions should be used with care and should be as fast as possible. For the best results with a custom timer, it might be necessary to hard-code it - in the C source of the internal :mod:`_lsprof` module. + in the C source of the internal :mod:`!_lsprof` module. Python 3.3 adds several new functions in :mod:`time` that can be used to make precise measurements of process or wall-clock time. For example, see diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst index bd2f5ed45cb8b4..1a44bb13a841de 100644 --- a/Doc/library/pty.rst +++ b/Doc/library/pty.rst @@ -1,5 +1,5 @@ -:mod:`pty` --- Pseudo-terminal utilities -======================================== +:mod:`!pty` --- Pseudo-terminal utilities +========================================= .. module:: pty :platform: Unix diff --git a/Doc/library/pwd.rst b/Doc/library/pwd.rst index dbe68cd14ec4d4..e1ff32912132f7 100644 --- a/Doc/library/pwd.rst +++ b/Doc/library/pwd.rst @@ -1,5 +1,5 @@ -:mod:`pwd` --- The password database -==================================== +:mod:`!pwd` --- The password database +===================================== .. module:: pwd :platform: Unix @@ -10,7 +10,7 @@ This module provides access to the Unix user account and password database. It is available on all Unix versions. -.. availability:: Unix, not Emscripten, not WASI. +.. availability:: Unix, not WASI, not iOS. Password database entries are reported as a tuple-like object, whose attributes correspond to the members of the ``passwd`` structure (Attribute field below, diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index 38c416f9ad0305..75aa739d1003b8 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -1,5 +1,5 @@ -:mod:`py_compile` --- Compile Python source files -================================================= +:mod:`!py_compile` --- Compile Python source files +================================================== .. module:: py_compile :synopsis: Generate byte-code files from Python source files. @@ -96,7 +96,7 @@ byte-code cache files in the directory containing the source code. .. class:: PycInvalidationMode - A enumeration of possible methods the interpreter can use to determine + An enumeration of possible methods the interpreter can use to determine whether a bytecode file is up to date with a source file. The ``.pyc`` file indicates the desired invalidation mode in its header. See :ref:`pyc-invalidation` for more information on how Python invalidates diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index 1e9876849b02f3..5efb11d89dd143 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -1,5 +1,5 @@ -:mod:`pyclbr` --- Python module browser support -=============================================== +:mod:`!pyclbr` --- Python module browser support +================================================ .. module:: pyclbr :synopsis: Supports information extraction for a Python module browser. @@ -142,7 +142,7 @@ Class Objects .. attribute:: parent - For top-level classes, None. For nested classes, the parent. + For top-level classes, ``None``. For nested classes, the parent. .. versionadded:: 3.7 diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst index 03e0915bf6d135..e8f153ee1b35ce 100644 --- a/Doc/library/pydoc.rst +++ b/Doc/library/pydoc.rst @@ -1,5 +1,5 @@ -:mod:`pydoc` --- Documentation generator and online help system -=============================================================== +:mod:`!pydoc` --- Documentation generator and online help system +================================================================ .. module:: pydoc :synopsis: Documentation generator and online help system. @@ -16,19 +16,19 @@ -------------- -The :mod:`pydoc` module automatically generates documentation from Python +The :mod:`!pydoc` module automatically generates documentation from Python modules. The documentation can be presented as pages of text on the console, served to a web browser, or saved to HTML files. For modules, classes, functions and methods, the displayed documentation is -derived from the docstring (i.e. the :attr:`__doc__` attribute) of the object, +derived from the docstring (i.e. the :attr:`~definition.__doc__` attribute) of the object, and recursively of its documentable members. If there is no docstring, -:mod:`pydoc` tries to obtain a description from the block of comment lines just +:mod:`!pydoc` tries to obtain a description from the block of comment lines just above the definition of the class, function or method in the source file, or at the top of the module (see :func:`inspect.getcomments`). The built-in function :func:`help` invokes the online help system in the -interactive interpreter, which uses :mod:`pydoc` to generate its documentation +interactive interpreter, which uses :mod:`!pydoc` to generate its documentation as text on the console. The same text documentation can also be viewed from outside the Python interpreter by running :program:`pydoc` as a script at the operating system's command prompt. For example, running :: @@ -46,14 +46,15 @@ produced for that file. .. note:: - In order to find objects and their documentation, :mod:`pydoc` imports the + In order to find objects and their documentation, :mod:`!pydoc` imports the module(s) to be documented. Therefore, any code on module level will be executed on that occasion. Use an ``if __name__ == '__main__':`` guard to only execute code when a file is invoked as a script and not just imported. When printing output to the console, :program:`pydoc` attempts to paginate the -output for easier reading. If the :envvar:`PAGER` environment variable is set, -:program:`pydoc` will use its value as a pagination program. +output for easier reading. If either the :envvar:`MANPAGER` or the +:envvar:`PAGER` environment variable is set, :program:`pydoc` will use its +value as a pagination program. When both are set, :envvar:`MANPAGER` is used. Specifying a ``-w`` flag before the argument will cause HTML documentation to be written out to a file in the current directory, instead of displaying text @@ -90,7 +91,7 @@ Python interpreter and typed ``import spam``. Module docs for core modules are assumed to reside in ``https://docs.python.org/X.Y/library/`` where ``X`` and ``Y`` are the major and minor version numbers of the Python interpreter. This can -be overridden by setting the :envvar:`PYTHONDOCS` environment variable +be overridden by setting the :envvar:`!PYTHONDOCS` environment variable to a different URL or to a local directory containing the Library Reference Manual pages. @@ -101,7 +102,7 @@ Reference Manual pages. The ``-g`` command line option was removed. .. versionchanged:: 3.4 - :mod:`pydoc` now uses :func:`inspect.signature` rather than + :mod:`!pydoc` now uses :func:`inspect.signature` rather than :func:`inspect.getfullargspec` to extract signature information from callables. diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index c897ec9e47b7ca..2d57cff10a9278 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -1,5 +1,5 @@ -:mod:`xml.parsers.expat` --- Fast XML parsing using Expat -========================================================= +:mod:`!xml.parsers.expat` --- Fast XML parsing using Expat +========================================================== .. module:: xml.parsers.expat :synopsis: An interface to the Expat non-validating XML parser. @@ -210,13 +210,18 @@ XMLParser Objects by default until a sufficient amount of input is reached. Due to this delay, registered handlers may — depending of the sizing of input chunks pushed to Expat — no longer be called right after pushing new - input to the parser. Where immediate feedback and taking over responsiblity + input to the parser. Where immediate feedback and taking over responsibility of protecting against denial of service from large tokens are both wanted, calling ``SetReparseDeferralEnabled(False)`` disables reparse deferral for the current Expat parser instance, temporarily or altogether. Calling ``SetReparseDeferralEnabled(True)`` allows re-enabling reparse deferral. + Note that :meth:`SetReparseDeferralEnabled` has been backported to some + prior releases of CPython as a security fix. Check for availability of + :meth:`SetReparseDeferralEnabled` using :func:`hasattr` if used in code + running across a variety of Python versions. + .. versionadded:: 3.13 .. method:: xmlparser.GetReparseDeferralEnabled() @@ -936,6 +941,13 @@ The ``errors`` module has the following attributes: has been breached. +.. data:: XML_ERROR_NOT_STARTED + + The parser was tried to be stopped or suspended before it started. + + .. versionadded:: 3.14 + + .. rubric:: Footnotes .. [1] The encoding string included in XML output should conform to the diff --git a/Doc/library/python.rst b/Doc/library/python.rst index 610435999d9f48..c2c231af7c3033 100644 --- a/Doc/library/python.rst +++ b/Doc/library/python.rst @@ -25,4 +25,5 @@ overview: __future__.rst gc.rst inspect.rst + annotationlib.rst site.rst diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index f2a6dbf589fd87..fbbebcf4ed8f92 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -1,5 +1,5 @@ -:mod:`queue` --- A synchronized queue class -=========================================== +:mod:`!queue` --- A synchronized queue class +============================================ .. module:: queue :synopsis: A synchronized queue class. @@ -245,8 +245,10 @@ them down. queue is empty. Set *immediate* to true to make :meth:`~Queue.get` raise immediately instead. - All blocked callers of :meth:`~Queue.put` will be unblocked. If *immediate* - is true, also unblock callers of :meth:`~Queue.get` and :meth:`~Queue.join`. + All blocked callers of :meth:`~Queue.put` and :meth:`~Queue.get` will be + unblocked. If *immediate* is true, a task will be marked as done for each + remaining item in the queue, which may unblock callers of + :meth:`~Queue.join`. .. versionadded:: 3.13 diff --git a/Doc/library/quopri.rst b/Doc/library/quopri.rst index 86717c00c3c136..977cb08d836afe 100644 --- a/Doc/library/quopri.rst +++ b/Doc/library/quopri.rst @@ -1,5 +1,5 @@ -:mod:`quopri` --- Encode and decode MIME quoted-printable data -============================================================== +:mod:`!quopri` --- Encode and decode MIME quoted-printable data +=============================================================== .. module:: quopri :synopsis: Encode and decode files using the MIME quoted-printable encoding. diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 8fbce18c56f17c..ef0cfb0e76cef6 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -1,5 +1,5 @@ -:mod:`random` --- Generate pseudo-random numbers -================================================ +:mod:`!random` --- Generate pseudo-random numbers +================================================= .. module:: random :synopsis: Generate pseudo-random numbers with various common distributions. @@ -55,10 +55,16 @@ from sources provided by the operating system. `Complementary-Multiply-with-Carry recipe - `_ for a compatible alternative + `_ for a compatible alternative random number generator with a long period and comparatively simple update operations. +.. note:: + The global random number generator and instances of :class:`Random` are thread-safe. + However, in the free-threaded build, concurrent calls to the global generator or + to the same instance of :class:`Random` may encounter contention and poor performance. + Consider using separate instances of :class:`Random` per thread instead. + Bookkeeping functions --------------------- @@ -194,8 +200,8 @@ Functions for sequences For a given seed, the :func:`choices` function with equal weighting typically produces a different sequence than repeated calls to - :func:`choice`. The algorithm used by :func:`choices` uses floating - point arithmetic for internal consistency and speed. The algorithm used + :func:`choice`. The algorithm used by :func:`choices` uses floating-point + arithmetic for internal consistency and speed. The algorithm used by :func:`choice` defaults to integer arithmetic with repeated selections to avoid small biases from round-off error. @@ -292,12 +298,12 @@ be found in any statistics text. .. function:: random() - Return the next random floating point number in the range ``0.0 <= X < 1.0`` + Return the next random floating-point number in the range ``0.0 <= X < 1.0`` .. function:: uniform(a, b) - Return a random floating point number *N* such that ``a <= N <= b`` for + Return a random floating-point number *N* such that ``a <= N <= b`` for ``a <= b`` and ``b <= N <= a`` for ``b < a``. The end-point value ``b`` may or may not be included in the range @@ -307,7 +313,7 @@ be found in any statistics text. .. function:: triangular(low, high, mode) - Return a random floating point number *N* such that ``low <= N <= high`` and + Return a random floating-point number *N* such that ``low <= N <= high`` and with the specified *mode* between those bounds. The *low* and *high* bounds default to zero and one. The *mode* argument defaults to the midpoint between the bounds, giving a symmetric distribution. @@ -700,3 +706,83 @@ positive unnormalized float and is equal to ``math.ulp(0.0)``.) `_ a paper by Allen B. Downey describing ways to generate more fine-grained floats than normally generated by :func:`.random`. + +.. _random-cli: + +Command-line usage +------------------ + +.. versionadded:: 3.13 + +The :mod:`!random` module can be executed from the command line. + +.. code-block:: sh + + python -m random [-h] [-c CHOICE [CHOICE ...] | -i N | -f N] [input ...] + +The following options are accepted: + +.. program:: random + +.. option:: -h, --help + + Show the help message and exit. + +.. option:: -c CHOICE [CHOICE ...] + --choice CHOICE [CHOICE ...] + + Print a random choice, using :meth:`choice`. + +.. option:: -i + --integer + + Print a random integer between 1 and N inclusive, using :meth:`randint`. + +.. option:: -f + --float + + Print a random floating-point number between 0 and N inclusive, + using :meth:`uniform`. + +If no options are given, the output depends on the input: + +* String or multiple: same as :option:`--choice`. +* Integer: same as :option:`--integer`. +* Float: same as :option:`--float`. + +.. _random-cli-example: + +Command-line example +-------------------- + +Here are some examples of the :mod:`!random` command-line interface: + +.. code-block:: console + + $ # Choose one at random + $ python -m random egg bacon sausage spam "Lobster Thermidor aux crevettes with a Mornay sauce" + Lobster Thermidor aux crevettes with a Mornay sauce + + $ # Random integer + $ python -m random 6 + 6 + + $ # Random floating-point number + $ python -m random 1.8 + 1.7080016272295635 + + $ # With explicit arguments + $ python -m random --choice egg bacon sausage spam "Lobster Thermidor aux crevettes with a Mornay sauce" + egg + + $ python -m random --integer 6 + 3 + + $ python -m random --float 1.8 + 1.5666339105010318 + + $ python -m random --integer 6 + 5 + + $ python -m random --float 6 + 3.1942323316565915 diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 0336121c2bc631..e2a78dc95d4ae1 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -1,5 +1,5 @@ -:mod:`re` --- Regular expression operations -=========================================== +:mod:`!re` --- Regular expression operations +============================================ .. module:: re :synopsis: Regular expression operations. @@ -48,7 +48,7 @@ fine-tuning parameters. .. seealso:: - The third-party `regex `_ module, + The third-party :pypi:`regex` module, which has an API compatible with the standard library :mod:`re` module, but offers additional functionality and a more thorough Unicode support. @@ -101,7 +101,7 @@ The special characters are: ``.`` (Dot.) In the default mode, this matches any character except a newline. If the :const:`DOTALL` flag has been specified, this matches any character - including a newline. + including a newline. ``(?s:.)`` matches any character regardless of flags. .. index:: single: ^ (caret); in regular expressions @@ -572,6 +572,9 @@ character ``'$'``. Word boundaries are determined by the current locale if the :py:const:`~re.LOCALE` flag is used. + .. versionchanged:: 3.14 + ``\B`` now matches empty input string. + .. index:: single: \d; in regular expressions ``\d`` @@ -600,10 +603,9 @@ character ``'$'``. ``\s`` For Unicode (str) patterns: - Matches Unicode whitespace characters (which includes - ``[ \t\n\r\f\v]``, and also many other characters, for example the - non-breaking spaces mandated by typography rules in many - languages). + Matches Unicode whitespace characters (as defined by :py:meth:`str.isspace`). + This includes ``[ \t\n\r\f\v]``, and also many other characters, for example the + non-breaking spaces mandated by typography rules in many languages. Matches ``[ \t\n\r\f\v]`` if the :py:const:`~re.ASCII` flag is used. @@ -911,6 +913,10 @@ Functions ``None`` if no position in the string matches the pattern; note that this is different from finding a zero-length match at some point in the string. + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. function:: match(pattern, string, flags=0) @@ -925,6 +931,10 @@ Functions If you want to locate a match anywhere in *string*, use :func:`search` instead (see also :ref:`search-vs-match`). + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. function:: fullmatch(pattern, string, flags=0) @@ -932,6 +942,10 @@ Functions corresponding :class:`~re.Match`. Return ``None`` if the string does not match the pattern; note that this is different from a zero-length match. + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. versionadded:: 3.4 @@ -974,6 +988,10 @@ Functions >>> re.split(r'(\W*)', '...words...') ['', '...', '', '', 'w', '', 'o', '', 'r', '', 'd', '', 's', '...', '', '', ''] + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. versionchanged:: 3.1 Added the optional flags argument. @@ -1004,6 +1022,10 @@ Functions >>> re.findall(r'(\w+)=(\d+)', 'set width=20 and height=10') [('width', '20'), ('height', '10')] + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. versionchanged:: 3.7 Non-empty matches can now start just after a previous empty match. @@ -1015,6 +1037,10 @@ Functions is scanned left-to-right, and matches are returned in the order found. Empty matches are included in the result. + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. versionchanged:: 3.7 Non-empty matches can now start just after a previous empty match. @@ -1070,6 +1096,10 @@ Functions character ``'0'``. The backreference ``\g<0>`` substitutes in the entire substring matched by the RE. + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. versionchanged:: 3.1 Added the optional flags argument. @@ -1102,6 +1132,10 @@ Functions Perform the same operation as :func:`sub`, but return a tuple ``(new_string, number_of_subs_made)``. + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the `flags`_ variables, combined using bitwise OR + (the ``|`` operator). + .. function:: escape(pattern) diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 54c6d9f3b32b1a..4a04205663258c 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -1,5 +1,5 @@ -:mod:`readline` --- GNU readline interface -========================================== +:mod:`!readline` --- GNU readline interface +=========================================== .. module:: readline :platform: Unix @@ -24,6 +24,8 @@ in the GNU Readline manual for information about the format and allowable constructs of that file, and the capabilities of the Readline library in general. +.. include:: ../includes/wasm-mobile-notavail.rst + .. note:: The underlying Readline library API may be implemented by @@ -43,6 +45,10 @@ Readline library in general. python:bind -v python:bind ^I rl_complete + Also note that different libraries may use different history file formats. + When switching the underlying library, existing history files may become + unusable. + .. data:: backend The name of the underlying Readline library being used, either diff --git a/Doc/library/removed.rst b/Doc/library/removed.rst new file mode 100644 index 00000000000000..4d75842eca1a03 --- /dev/null +++ b/Doc/library/removed.rst @@ -0,0 +1,39 @@ +:tocdepth: 1 + +.. _removed: + +*************** +Removed Modules +*************** + +The modules described in this chapter have been removed from the Python +standard library. They are documented here to help people find replacements. + + +.. toctree:: + :maxdepth: 1 + + aifc.rst + asynchat.rst + asyncore.rst + audioop.rst + cgi.rst + cgitb.rst + chunk.rst + crypt.rst + distutils.rst + imghdr.rst + imp.rst + mailcap.rst + msilib.rst + nis.rst + nntplib.rst + ossaudiodev.rst + pipes.rst + smtpd.rst + sndhdr.rst + spwd.rst + sunau.rst + telnetlib.rst + uu.rst + xdrlib.rst diff --git a/Doc/library/reprlib.rst b/Doc/library/reprlib.rst index 678a11c6f45490..28c7855dfeeef3 100644 --- a/Doc/library/reprlib.rst +++ b/Doc/library/reprlib.rst @@ -1,5 +1,5 @@ -:mod:`reprlib` --- Alternate :func:`repr` implementation -======================================================== +:mod:`!reprlib` --- Alternate :func:`repr` implementation +========================================================= .. module:: reprlib :synopsis: Alternate repr() implementation with size limits. diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index 4e58b043f1da31..0515d205bbca0b 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -1,5 +1,5 @@ -:mod:`resource` --- Resource usage information -============================================== +:mod:`!resource` --- Resource usage information +=============================================== .. module:: resource :platform: Unix @@ -13,7 +13,7 @@ This module provides basic mechanisms for measuring and controlling system resources utilized by a program. -.. availability:: Unix, not Emscripten, not WASI. +.. availability:: Unix, not WASI. Symbolic constants are used to specify particular system resources and to request usage information about either the current process or its children. @@ -177,6 +177,8 @@ platform. The largest area of mapped memory which the process may occupy. + .. availability:: FreeBSD >= 11. + .. data:: RLIMIT_AS @@ -303,7 +305,7 @@ These functions are used to retrieve resource usage information: elements. The fields :attr:`ru_utime` and :attr:`ru_stime` of the return value are - floating point values representing the amount of time spent executing in user + floating-point values representing the amount of time spent executing in user mode and the amount of time spent executing in system mode, respectively. The remaining values are integers. Consult the :manpage:`getrusage(2)` man page for detailed information about these values. A brief summary is presented here: diff --git a/Doc/library/rlcompleter.rst b/Doc/library/rlcompleter.rst index 8287699c5f013e..91779feb525013 100644 --- a/Doc/library/rlcompleter.rst +++ b/Doc/library/rlcompleter.rst @@ -1,5 +1,5 @@ -:mod:`rlcompleter` --- Completion function for GNU readline -=========================================================== +:mod:`!rlcompleter` --- Completion function for GNU readline +============================================================ .. module:: rlcompleter :synopsis: Python identifier completion, suitable for the GNU readline library. diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index f2cb595f495f6b..b07ec6e93f80ab 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -1,5 +1,5 @@ -:mod:`runpy` --- Locating and executing Python modules -====================================================== +:mod:`!runpy` --- Locating and executing Python modules +======================================================= .. module:: runpy :synopsis: Locate and run Python modules without importing them first. diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 4c980dd97f9394..517dbe8c321898 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -1,5 +1,5 @@ -:mod:`sched` --- Event scheduler -================================ +:mod:`!sched` --- Event scheduler +================================= .. module:: sched :synopsis: General purpose event scheduler. diff --git a/Doc/library/secrets.rst b/Doc/library/secrets.rst index 4405dfc0535973..75dafc54d40ca5 100644 --- a/Doc/library/secrets.rst +++ b/Doc/library/secrets.rst @@ -1,5 +1,5 @@ -:mod:`secrets` --- Generate secure random numbers for managing secrets -====================================================================== +:mod:`!secrets` --- Generate secure random numbers for managing secrets +======================================================================= .. module:: secrets :synopsis: Generate secure random numbers for managing secrets. @@ -42,17 +42,17 @@ randomness that your operating system provides. sources provided by the operating system. See :class:`random.SystemRandom` for additional details. -.. function:: choice(sequence) +.. function:: choice(seq) Return a randomly chosen element from a non-empty sequence. -.. function:: randbelow(n) +.. function:: randbelow(exclusive_upper_bound) - Return a random int in the range [0, *n*). + Return a random int in the range [0, *exclusive_upper_bound*). .. function:: randbits(k) - Return an int with *k* random bits. + Return a non-negative int with *k* random bits. Generating tokens @@ -155,7 +155,7 @@ Generate an eight-character alphanumeric password: .. note:: Applications should not - `store passwords in a recoverable format `_, + :cwe:`store passwords in a recoverable format <257>`, whether plain text or encrypted. They should be salted and hashed using a cryptographically strong one-way (irreversible) hash function. diff --git a/Doc/library/select.rst b/Doc/library/select.rst index a0058046d0ce4c..457970aed2dc73 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -1,5 +1,5 @@ -:mod:`select` --- Waiting for I/O completion -============================================ +:mod:`!select` --- Waiting for I/O completion +============================================= .. module:: select :synopsis: Wait for I/O completion on multiple streams. @@ -129,7 +129,7 @@ The module defines the following: Empty iterables are allowed, but acceptance of three empty iterables is platform-dependent. (It is known to work on Unix but not on Windows.) The - optional *timeout* argument specifies a time-out as a floating point number + optional *timeout* argument specifies a time-out as a floating-point number in seconds. When the *timeout* argument is omitted the function blocks until at least one file descriptor is ready. A time-out value of zero specifies a poll and never blocks. @@ -317,11 +317,17 @@ Edge and Level Trigger Polling (epoll) Objects +-------------------------+-----------------------------------------------+ | :const:`EPOLLMSG` | Ignored. | +-------------------------+-----------------------------------------------+ + | :const:`EPOLLWAKEUP` | Prevents sleep during event waiting. | + +-------------------------+-----------------------------------------------+ .. versionadded:: 3.6 :const:`EPOLLEXCLUSIVE` was added. It's only supported by Linux Kernel 4.5 or later. + .. versionadded:: 3.14 + :const:`EPOLLWAKEUP` was added. It's only supported by Linux Kernel 3.5 + or later. + .. method:: epoll.close() Close the control file descriptor of the epoll object. diff --git a/Doc/library/selectors.rst b/Doc/library/selectors.rst index 76cbf91412f763..de8c3ef0ea2275 100644 --- a/Doc/library/selectors.rst +++ b/Doc/library/selectors.rst @@ -1,5 +1,5 @@ -:mod:`selectors` --- High-level I/O multiplexing -================================================ +:mod:`!selectors` --- High-level I/O multiplexing +================================================= .. module:: selectors :synopsis: High-level I/O multiplexing. diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 95c54991887022..6e74a59b82b8ec 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -1,5 +1,5 @@ -:mod:`shelve` --- Python object persistence -=========================================== +:mod:`!shelve` --- Python object persistence +============================================ .. module:: shelve :synopsis: Python object persistence. @@ -86,7 +86,7 @@ Two additional methods are supported: .. seealso:: - `Persistent dictionary recipe `_ + `Persistent dictionary recipe `_ with widely supported storage formats and having the speed of native dictionaries. diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index f94833ad5331a9..a96f0864dc1260 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -1,5 +1,5 @@ -:mod:`shlex` --- Simple lexical analysis -======================================== +:mod:`!shlex` --- Simple lexical analysis +========================================= .. module:: shlex :synopsis: Simple lexical analysis for Unix shell-like languages. @@ -412,17 +412,17 @@ otherwise. To illustrate, you can see the difference in the following snippet: .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> import shlex - >>> text = "a && b; c && d || e; f >'abc'; (def \"ghi\")" - >>> s = shlex.shlex(text, posix=True) - >>> s.whitespace_split = True - >>> list(s) - ['a', '&&', 'b;', 'c', '&&', 'd', '||', 'e;', 'f', '>abc;', '(def', 'ghi)'] - >>> s = shlex.shlex(text, posix=True, punctuation_chars=True) - >>> s.whitespace_split = True - >>> list(s) - ['a', '&&', 'b', ';', 'c', '&&', 'd', '||', 'e', ';', 'f', '>', 'abc', ';', - '(', 'def', 'ghi', ')'] + >>> import shlex + >>> text = "a && b; c && d || e; f >'abc'; (def \"ghi\")" + >>> s = shlex.shlex(text, posix=True) + >>> s.whitespace_split = True + >>> list(s) + ['a', '&&', 'b;', 'c', '&&', 'd', '||', 'e;', 'f', '>abc;', '(def', 'ghi)'] + >>> s = shlex.shlex(text, posix=True, punctuation_chars=True) + >>> s.whitespace_split = True + >>> list(s) + ['a', '&&', 'b', ';', 'c', '&&', 'd', '||', 'e', ';', 'f', '>', 'abc', ';', + '(', 'def', 'ghi', ')'] Of course, tokens will be returned which are not valid for shells, and you'll need to implement your own error checks on the returned tokens. @@ -431,10 +431,10 @@ Instead of passing ``True`` as the value for the punctuation_chars parameter, you can pass a string with specific characters, which will be used to determine which characters constitute punctuation. For example:: - >>> import shlex - >>> s = shlex.shlex("a && b || c", punctuation_chars="|") - >>> list(s) - ['a', '&', '&', 'b', '||', 'c'] + >>> import shlex + >>> s = shlex.shlex("a && b || c", punctuation_chars="|") + >>> list(s) + ['a', '&', '&', 'b', '||', 'c'] .. note:: When ``punctuation_chars`` is specified, the :attr:`~shlex.wordchars` attribute is augmented with the characters ``~-./*?=``. That is because these diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 4f07b9f6040d24..2a8592f8bd69c1 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -1,5 +1,5 @@ -:mod:`shutil` --- High-level file operations -============================================ +:mod:`!shutil` --- High-level file operations +============================================= .. module:: shutil :synopsis: High-level file operations, including copying. @@ -242,7 +242,7 @@ Directory and files operations be copied as far as the platform allows; if false or omitted, the contents and metadata of the linked files are copied to the new tree. - When *symlinks* is false, if the file pointed by the symlink doesn't + When *symlinks* is false, if the file pointed to by the symlink doesn't exist, an exception will be added in the list of errors raised in an :exc:`Error` exception at the end of the copy process. You can set the optional *ignore_dangling_symlinks* flag to true if you @@ -338,7 +338,7 @@ Directory and files operations before removing the junction. .. versionchanged:: 3.11 - The *dir_fd* parameter. + Added the *dir_fd* parameter. .. versionchanged:: 3.12 Added the *onexc* parameter, deprecated *onerror*. @@ -421,7 +421,8 @@ Directory and files operations .. availability:: Unix, Windows. -.. function:: chown(path, user=None, group=None) +.. function:: chown(path, user=None, group=None, *, dir_fd=None, \ + follow_symlinks=True) Change owner *user* and/or *group* of the given *path*. @@ -436,6 +437,9 @@ Directory and files operations .. versionadded:: 3.3 + .. versionchanged:: 3.13 + Added *dir_fd* and *follow_symlinks* parameters. + .. function:: which(cmd, mode=os.F_OK | os.X_OK, path=None) @@ -443,10 +447,12 @@ Directory and files operations called. If no *cmd* would be called, return ``None``. *mode* is a permission mask passed to :func:`os.access`, by default - determining if the file exists and executable. + determining if the file exists and is executable. - When no *path* is specified, the results of :func:`os.environ` are used, - returning either the "PATH" value or a fallback of :data:`os.defpath`. + *path* is a "``PATH`` string" specifying the directories to look in, + delimited by :data:`os.pathsep`. When no *path* is specified, the + :envvar:`PATH` environment variable is read from :data:`os.environ`, + falling back to :data:`os.defpath` if it is not set. On Windows, the current directory is prepended to the *path* if *mode* does not include ``os.X_OK``. When the *mode* does include ``os.X_OK``, the @@ -455,9 +461,9 @@ Directory and files operations consulting the current working directory for executables: set the environment variable ``NoDefaultCurrentDirectoryInExePath``. - Also on Windows, the ``PATHEXT`` variable is used to resolve commands - that may not already include an extension. For example, if you call - ``shutil.which("python")``, :func:`which` will search ``PATHEXT`` + Also on Windows, the :envvar:`PATHEXT` environment variable is used to + resolve commands that may not already include an extension. For example, + if you call ``shutil.which("python")``, :func:`which` will search ``PATHEXT`` to know that it should look for ``python.exe`` within the *path* directories. For example, on Windows:: @@ -485,12 +491,6 @@ Directory and files operations or ends with an extension that is in ``PATHEXT``; and filenames that have no extension can now be found. - .. versionchanged:: 3.12.1 - On Windows, if *mode* includes ``os.X_OK``, executables with an - extension in ``PATHEXT`` will be preferred over executables without a - matching extension. - This brings behavior closer to that of Python 3.11. - .. exception:: Error This exception collects exceptions that are raised during a multi-file @@ -512,7 +512,7 @@ the use of userspace buffers in Python as in "``outfd.write(infd.read())``". On macOS `fcopyfile`_ is used to copy the file content (not metadata). -On Linux :func:`os.sendfile` is used. +On Linux and Solaris :func:`os.sendfile` is used. On Windows :func:`shutil.copyfile` uses a bigger default buffer size (1 MiB instead of 64 KiB) and a :func:`memoryview`-based variant of @@ -524,6 +524,9 @@ file then shutil will silently fallback on using less efficient .. versionchanged:: 3.8 +.. versionchanged:: 3.14 + Solaris now uses :func:`os.sendfile`. + .. _shutil-copytree-example: copytree example @@ -701,11 +704,9 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. The keyword-only *filter* argument is passed to the underlying unpacking function. For zip files, *filter* is not accepted. - For tar files, it is recommended to set it to ``'data'``, - unless using features specific to tar and UNIX-like filesystems. + For tar files, it is recommended to use ``'data'`` (default since Python + 3.14), unless using features specific to tar and UNIX-like filesystems. (See :ref:`tarfile-extraction-filter` for details.) - The ``'data'`` filter will become the default for tar files - in Python 3.14. .. audit-event:: shutil.unpack_archive filename,extract_dir,format shutil.unpack_archive @@ -716,6 +717,12 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. the *extract_dir* argument, e.g. members that have absolute filenames starting with "/" or filenames with two dots "..". + Since Python 3.14, the defaults for both built-in formats (zip and tar + files) will prevent the most dangerous of such security issues, + but will not prevent *all* unintended behavior. + Read the :ref:`tarfile-further-verification` + section for tar-specific details. + .. versionchanged:: 3.7 Accepts a :term:`path-like object` for *filename* and *extract_dir*. diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 85a073aad233ac..17fcb2b3707978 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -1,5 +1,5 @@ -:mod:`signal` --- Set handlers for asynchronous events -====================================================== +:mod:`!signal` --- Set handlers for asynchronous events +======================================================= .. module:: signal :synopsis: Set handlers for asynchronous events. @@ -26,9 +26,9 @@ explicitly reset (Python emulates the BSD style interface regardless of the underlying implementation), with the exception of the handler for :const:`SIGCHLD`, which follows the underlying implementation. -On WebAssembly platforms ``wasm32-emscripten`` and ``wasm32-wasi``, signals -are emulated and therefore behave differently. Several functions and signals -are not available on these platforms. +On WebAssembly platforms, signals are emulated and therefore behave +differently. Several functions and signals are not available on these +platforms. Execution of Python signal handlers ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -411,7 +411,7 @@ The :mod:`signal` module defines the following functions: See the :manpage:`pidfd_send_signal(2)` man page for more information. - .. availability:: Linux >= 5.1 + .. availability:: Linux >= 5.1, Android >= :func:`build-time ` API level 31 .. versionadded:: 3.9 @@ -425,7 +425,7 @@ The :mod:`signal` module defines the following functions: signal to a particular Python thread would be to force a running system call to fail with :exc:`InterruptedError`. - Use :func:`threading.get_ident()` or the :attr:`~threading.Thread.ident` + Use :func:`threading.get_ident` or the :attr:`~threading.Thread.ident` attribute of :class:`threading.Thread` objects to get a suitable value for *thread_id*. diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 2dc9fb09d727e2..5f2a0f610e1aa5 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -1,5 +1,5 @@ -:mod:`site` --- Site-specific configuration hook -================================================ +:mod:`!site` --- Site-specific configuration hook +================================================= .. module:: site :synopsis: Module responsible for site-specific configuration. @@ -15,8 +15,9 @@ import can be suppressed using the interpreter's :option:`-S` option. .. index:: triple: module; search; path -Importing this module will append site-specific paths to the module search path -and add a few builtins, unless :option:`-S` was used. In that case, this module +Importing this module normally appends site-specific paths to the module search path +and adds :ref:`callables `, including :func:`help` to the built-in +namespace. However, Python startup option :option:`-S` blocks this and this module can be safely imported with no automatic modifications to the module search path or additions to the builtins. To explicitly trigger the usual site-specific additions, call the :func:`main` function. @@ -32,7 +33,10 @@ It starts by constructing up to four directories from a head and a tail part. For the head part, it uses ``sys.prefix`` and ``sys.exec_prefix``; empty heads are skipped. For the tail part, it uses the empty string and then :file:`lib/site-packages` (on Windows) or -:file:`lib/python{X.Y}/site-packages` (on Unix and macOS). For each +:file:`lib/python{X.Y[t]}/site-packages` (on Unix and macOS). (The +optional suffix "t" indicates the :term:`free threading` build, and is +appended if ``"t"`` is present in the :attr:`sys.abiflags` constant.) +For each of the distinct head-tail combinations, it sees if it refers to an existing directory, and if so, adds it to ``sys.path`` and also inspects the newly added path for configuration files. @@ -40,14 +44,27 @@ added path for configuration files. .. versionchanged:: 3.5 Support for the "site-python" directory has been removed. -If a file named "pyvenv.cfg" exists one directory above sys.executable, -sys.prefix and sys.exec_prefix are set to that directory and -it is also checked for site-packages (sys.base_prefix and -sys.base_exec_prefix will always be the "real" prefixes of the Python -installation). If "pyvenv.cfg" (a bootstrap configuration file) contains -the key "include-system-site-packages" set to anything other than "true" -(case-insensitive), the system-level prefixes will not be -searched for site-packages; otherwise they will. +.. versionchanged:: 3.13 + On Unix, :term:`Free threading ` Python installations are + identified by the "t" suffix in the version-specific directory name, such as + :file:`lib/python3.13t/`. + +.. versionchanged:: 3.14 + + :mod:`site` is no longer responsible for updating :data:`sys.prefix` and + :data:`sys.exec_prefix` on :ref:`sys-path-init-virtual-environments`. This is + now done during the :ref:`path initialization `. As a result, + under :ref:`sys-path-init-virtual-environments`, :data:`sys.prefix` and + :data:`sys.exec_prefix` no longer depend on the :mod:`site` initialization, + and are therefore unaffected by :option:`-S`. + +.. _site-virtual-environments-configuration: + +When running under a :ref:`virtual environment `, +the ``pyvenv.cfg`` file in :data:`sys.prefix` is checked for site-specific +configurations. If the ``include-system-site-packages`` key exists and is set to +``true`` (case-insensitive), the system-level prefixes will be searched for +site-packages, otherwise they won't. .. index:: single: # (hash); comment @@ -74,6 +91,10 @@ with ``import`` (followed by space or tab) are executed. Limiting a code chunk to a single line is a deliberate measure to discourage putting anything more complex here. +.. versionchanged:: 3.13 + The :file:`.pth` files are now decoded by UTF-8 at first and then by the + :term:`locale encoding` if it fails. + .. index:: single: package triple: path; configuration; file @@ -184,11 +205,12 @@ Module contents Path to the user site-packages for the running Python. Can be ``None`` if :func:`getusersitepackages` hasn't been called yet. Default value is - :file:`~/.local/lib/python{X.Y}/site-packages` for UNIX and non-framework + :file:`~/.local/lib/python{X.Y}[t]/site-packages` for UNIX and non-framework macOS builds, :file:`~/Library/Python/{X.Y}/lib/python/site-packages` for macOS framework builds, and :file:`{%APPDATA%}\\Python\\Python{XY}\\site-packages` - on Windows. This directory is a site directory, which means that - :file:`.pth` files in it will be processed. + on Windows. The optional "t" indicates the free-threaded build. This + directory is a site directory, which means that :file:`.pth` files in it + will be processed. .. data:: USER_BASE diff --git a/Doc/library/smtpd.rst b/Doc/library/smtpd.rst new file mode 100644 index 00000000000000..c704f4a241b469 --- /dev/null +++ b/Doc/library/smtpd.rst @@ -0,0 +1,18 @@ +:mod:`!smtpd` --- SMTP Server +============================= + +.. module:: smtpd + :synopsis: Removed in 3.12. + :deprecated: + +.. deprecated-removed:: 3.6 3.12 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.12 ` after +being deprecated in Python 3.6. The removal was decided in :pep:`594`. + +A possible replacement is the third-party :pypi:`aiosmtpd` library. This +library is not maintained or supported by the Python core team. + +The last version of Python that provided the :mod:`!smtpd` module was +`Python 3.11 `_. diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index aaec2aa1ef1dbe..7cd530a5fd6438 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -1,5 +1,5 @@ -:mod:`smtplib` --- SMTP protocol client -======================================= +:mod:`!smtplib` --- SMTP protocol client +======================================== .. module:: smtplib :synopsis: SMTP protocol client (requires sockets). @@ -556,34 +556,33 @@ This example prompts the user for addresses needed in the message envelope ('To' and 'From' addresses), and the message to be delivered. Note that the headers to be included with the message must be included in the message as entered; this example doesn't do any processing of the :rfc:`822` headers. In particular, the -'To' and 'From' addresses must be included in the message headers explicitly. :: +'To' and 'From' addresses must be included in the message headers explicitly:: import smtplib - def prompt(prompt): - return input(prompt).strip() + def prompt(title): + return input(title).strip() - fromaddr = prompt("From: ") - toaddrs = prompt("To: ").split() + from_addr = prompt("From: ") + to_addrs = prompt("To: ").split() print("Enter message, end with ^D (Unix) or ^Z (Windows):") # Add the From: and To: headers at the start! - msg = ("From: %s\r\nTo: %s\r\n\r\n" - % (fromaddr, ", ".join(toaddrs))) + lines = [f"From: {from_addr}", f"To: {', '.join(to_addrs)}", ""] while True: try: line = input() except EOFError: break - if not line: - break - msg = msg + line + else: + lines.append(line) + msg = "\r\n".join(lines) print("Message length is", len(msg)) - server = smtplib.SMTP('localhost') + server = smtplib.SMTP("localhost") server.set_debuglevel(1) - server.sendmail(fromaddr, toaddrs, msg) + server.sendmail(from_addr, to_addrs, msg) server.quit() .. note:: diff --git a/Doc/library/sndhdr.rst b/Doc/library/sndhdr.rst new file mode 100644 index 00000000000000..6b71db4f6338a8 --- /dev/null +++ b/Doc/library/sndhdr.rst @@ -0,0 +1,19 @@ +:mod:`!sndhdr` --- Determine type of sound file +=============================================== + +.. module:: sndhdr + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +Possible replacements are third-party modules from PyPI: +:pypi:`filetype`, :pypi:`puremagic`, or :pypi:`python-magic`. +These are not supported or maintained by the Python core team. + +The last version of Python that provided the :mod:`!sndhdr` module was +`Python 3.12 `_. diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 3a931e25de91e5..b36acad29ecb00 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -1,5 +1,5 @@ -:mod:`socket` --- Low-level networking interface -================================================ +:mod:`!socket` --- Low-level networking interface +================================================= .. module:: socket :synopsis: Low-level networking interface. @@ -413,14 +413,14 @@ Constants ``TCP_USER_TIMEOUT``, ``TCP_CONGESTION`` were added. .. versionchanged:: 3.6.5 - On Windows, ``TCP_FASTOPEN``, ``TCP_KEEPCNT`` appear if run-time Windows - supports. + Added support for ``TCP_FASTOPEN``, ``TCP_KEEPCNT`` on Windows platforms + when available. .. versionchanged:: 3.7 ``TCP_NOTSENT_LOWAT`` was added. - On Windows, ``TCP_KEEPIDLE``, ``TCP_KEEPINTVL`` appear if run-time Windows - supports. + Added support for ``TCP_KEEPIDLE``, ``TCP_KEEPINTVL`` on Windows platforms + when available. .. versionchanged:: 3.10 ``IP_RECVTOS`` was added. @@ -450,6 +450,14 @@ Constants same way that ``SO_BINDTODEVICE`` is used, but with the index of a network interface instead of its name. + .. versionchanged:: 3.14 + Added missing ``IP_RECVERR``, ``IPV6_RECVERR``, ``IP_RECVTTL``, and + ``IP_RECVORIGDSTADDR`` on Linux. + + .. versionchanged:: 3.14 + Added support for ``TCP_QUICKACK`` on Windows platforms when available. + + .. data:: AF_CAN PF_CAN SOL_CAN_* @@ -666,6 +674,14 @@ Constants .. availability:: Linux >= 3.9 +.. data:: SO_REUSEPORT_LB + + Constant to enable duplicate address and port bindings with load balancing. + + .. versionadded:: 3.14 + + .. availability:: FreeBSD >= 12.0 + .. data:: AF_HYPERV HV_PROTOCOL_RAW HVSOCKET_CONNECT_TIMEOUT @@ -700,6 +716,13 @@ Constants .. versionadded:: 3.12 +.. data:: SHUT_RD + SHUT_WR + SHUT_RDWR + + These constants are used by the :meth:`~socket.socket.shutdown` method of socket objects. + + .. availability:: not WASI. Functions ^^^^^^^^^ @@ -729,7 +752,7 @@ The following functions all create :ref:`socket objects `. of :meth:`socket.getpeername` but not the actual OS resource. Unlike :func:`socket.fromfd`, *fileno* will return the same socket and not a duplicate. This may help close a detached socket using - :meth:`socket.close()`. + :meth:`socket.close`. The newly created socket is :ref:`non-inheritable `. @@ -913,7 +936,9 @@ The :mod:`socket` module also offers various network-related services: .. versionadded:: 3.7 -.. function:: getaddrinfo(host, port, family=0, type=0, proto=0, flags=0) +.. function:: getaddrinfo(host, port, family=AF_UNSPEC, type=0, proto=0, flags=0) + + This function wraps the C function ``getaddrinfo`` of the underlying system. Translate the *host*/*port* argument into a sequence of 5-tuples that contain all the necessary arguments for creating a socket connected to that service. @@ -923,8 +948,10 @@ The :mod:`socket` module also offers various network-related services: and *port*, you can pass ``NULL`` to the underlying C API. The *family*, *type* and *proto* arguments can be optionally specified - in order to narrow the list of addresses returned. Passing zero as a - value for each of these arguments selects the full range of results. + in order to provide options and limit the list of addresses returned. + Pass their default values (:data:`AF_UNSPEC`, 0, and 0, respectively) + to not limit the results. See the note below for details. + The *flags* argument can be one or several of the ``AI_*`` constants, and will influence how results are computed and returned. For example, :const:`AI_NUMERICHOST` will disable domain name resolution @@ -944,6 +971,29 @@ The :mod:`socket` module also offers various network-related services: :const:`AF_INET6`), and is meant to be passed to the :meth:`socket.connect` method. + .. note:: + + If you intend to use results from :func:`!getaddrinfo` to create a socket + (rather than, for example, retrieve *canonname*), + consider limiting the results by *type* (e.g. :data:`SOCK_STREAM` or + :data:`SOCK_DGRAM`) and/or *proto* (e.g. :data:`IPPROTO_TCP` or + :data:`IPPROTO_UDP`) that your application can handle. + + The behavior with default values of *family*, *type*, *proto* + and *flags* is system-specific. + + Many systems (for example, most Linux configurations) will return a sorted + list of all matching addresses. + These addresses should generally be tried in order until a connection succeeds + (possibly tried in parallel, for example, using a `Happy Eyeballs`_ algorithm). + In these cases, limiting the *type* and/or *proto* can help eliminate + unsuccessful or unusable connection attempts. + + Some systems will, however, only return a single address. + (For example, this was reported on Solaris and AIX configurations.) + On these systems, limiting the *type* and/or *proto* helps ensure that + this address is usable. + .. audit-event:: socket.getaddrinfo host,port,family,type,protocol socket.getaddrinfo The following example fetches address information for a hypothetical TCP @@ -963,6 +1013,8 @@ The :mod:`socket` module also offers various network-related services: for IPv6 multicast addresses, string representing an address will not contain ``%scope_id`` part. +.. _Happy Eyeballs: https://en.wikipedia.org/wiki/Happy_Eyeballs + .. function:: getfqdn([name]) Return a fully qualified domain name for *name*. If *name* is omitted or empty, @@ -1213,7 +1265,7 @@ The :mod:`socket` module also offers various network-related services: buffer. Raises :exc:`OverflowError` if *length* is outside the permissible range of values. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. Most Unix platforms. @@ -1236,7 +1288,7 @@ The :mod:`socket` module also offers various network-related services: amount of ancillary data that can be received, since additional data may be able to fit into the padding area. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, not WASI. most Unix platforms. @@ -1265,7 +1317,7 @@ The :mod:`socket` module also offers various network-related services: .. audit-event:: socket.sethostname name socket.sethostname - .. availability:: Unix. + .. availability:: Unix, not Android. .. versionadded:: 3.3 @@ -1276,7 +1328,7 @@ The :mod:`socket` module also offers various network-related services: (index int, name string) tuples. :exc:`OSError` if the system call fails. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI. .. versionadded:: 3.3 @@ -1303,7 +1355,7 @@ The :mod:`socket` module also offers various network-related services: interface name. :exc:`OSError` if no interface with the given name exists. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI. .. versionadded:: 3.3 @@ -1320,7 +1372,7 @@ The :mod:`socket` module also offers various network-related services: interface index number. :exc:`OSError` if no interface with the given index exists. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI. .. versionadded:: 3.3 @@ -1337,7 +1389,7 @@ The :mod:`socket` module also offers various network-related services: The *fds* parameter is a sequence of file descriptors. Consult :meth:`~socket.sendmsg` for the documentation of these parameters. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI. Unix platforms supporting :meth:`~socket.sendmsg` and :const:`SCM_RIGHTS` mechanism. @@ -1351,7 +1403,7 @@ The :mod:`socket` module also offers various network-related services: Return ``(msg, list(fds), flags, addr)``. Consult :meth:`~socket.recvmsg` for the documentation of these parameters. - .. availability:: Unix, Windows, not Emscripten, not WASI. + .. availability:: Unix, Windows, not WASI. Unix platforms supporting :meth:`~socket.sendmsg` and :const:`SCM_RIGHTS` mechanism. @@ -1408,7 +1460,7 @@ to sockets. .. method:: socket.close() Mark the socket closed. The underlying system resource (e.g. a file - descriptor) is also closed when all file objects from :meth:`makefile()` + descriptor) is also closed when all file objects from :meth:`makefile` are closed. Once that happens, all future operations on the socket object will fail. The remote end will receive no more data (after queued data is flushed). @@ -1423,10 +1475,10 @@ to sockets. .. note:: - :meth:`close()` releases the resource associated with a connection but + :meth:`close` releases the resource associated with a connection but does not necessarily close the connection immediately. If you want - to close the connection in a timely fashion, call :meth:`shutdown()` - before :meth:`close()`. + to close the connection in a timely fashion, call :meth:`shutdown` + before :meth:`close`. .. method:: socket.connect(address) @@ -1552,8 +1604,6 @@ to sockets. .. method:: socket.ioctl(control, option) - :platform: Windows - The :meth:`ioctl` method is a limited interface to the WSAIoctl system interface. Please refer to the `Win32 documentation `_ for more @@ -1565,9 +1615,12 @@ to sockets. Currently only the following control codes are supported: ``SIO_RCVALL``, ``SIO_KEEPALIVE_VALS``, and ``SIO_LOOPBACK_FAST_PATH``. + .. availability:: Windows + .. versionchanged:: 3.6 ``SIO_LOOPBACK_FAST_PATH`` was added. + .. method:: socket.listen([backlog]) Enable a server to accept connections. If *backlog* is specified, it must @@ -1589,7 +1642,8 @@ to sockets. Return a :term:`file object` associated with the socket. The exact returned type depends on the arguments given to :meth:`makefile`. These arguments are interpreted the same way as by the built-in :func:`open` function, except - the only supported *mode* values are ``'r'`` (default), ``'w'`` and ``'b'``. + the only supported *mode* values are ``'r'`` (default), ``'w'``, ``'b'``, or + a combination of those. The socket must be in blocking mode; it can have a timeout, but the file object's internal buffer may end up in an inconsistent state if a timeout @@ -1921,7 +1975,7 @@ to sockets. .. method:: socket.settimeout(value) Set a timeout on blocking socket operations. The *value* argument can be a - nonnegative floating point number expressing seconds, or ``None``. + nonnegative floating-point number expressing seconds, or ``None``. If a non-zero value is given, subsequent socket operations will raise a :exc:`timeout` exception if the timeout period *value* has elapsed before the operation has completed. If zero is given, the socket is put in @@ -2034,7 +2088,7 @@ can be changed by calling :func:`setdefaulttimeout`. in non-blocking mode. Also, the blocking and timeout modes are shared between file descriptors and socket objects that refer to the same network endpoint. This implementation detail can have visible consequences if e.g. you decide - to use the :meth:`~socket.fileno()` of a socket. + to use the :meth:`~socket.fileno` of a socket. Timeouts and the ``connect`` method ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 864b1dadb78562..69f06e6cf4d923 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -1,5 +1,5 @@ -:mod:`socketserver` --- A framework for network servers -======================================================= +:mod:`!socketserver` --- A framework for network servers +======================================================== .. module:: socketserver :synopsis: A framework for network servers. @@ -126,6 +126,12 @@ server is the address family. waits until all non-daemon threads complete, except if :attr:`block_on_close` attribute is ``False``. + .. attribute:: max_children + + Specify how many child processes will exist to handle requests at a time + for :class:`ForkingMixIn`. If the limit is reached, + new requests will wait until one child process has finished. + .. attribute:: daemon_threads For :class:`ThreadingMixIn` use daemonic threads by setting diff --git a/Doc/library/spwd.rst b/Doc/library/spwd.rst new file mode 100644 index 00000000000000..c16854bb380e52 --- /dev/null +++ b/Doc/library/spwd.rst @@ -0,0 +1,18 @@ +:mod:`!spwd` --- The shadow password database +============================================= + +.. module:: spwd + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +A possible replacement is the third-party library :pypi:`python-pam`. +This library is not supported or maintained by the Python core team. + +The last version of Python that provided the :mod:`!spwd` module was +`Python 3.12 `_. diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 87d5ef1e42ca3a..096892b605b99c 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1,5 +1,5 @@ -:mod:`sqlite3` --- DB-API 2.0 interface for SQLite databases -============================================================ +:mod:`!sqlite3` --- DB-API 2.0 interface for SQLite databases +============================================================= .. module:: sqlite3 :synopsis: A DB-API 2.0 implementation using SQLite 3.x. @@ -16,6 +16,8 @@ src = sqlite3.connect(":memory:", isolation_level=None) dst = sqlite3.connect("tutorial.db", isolation_level=None) src.backup(dst) + src.close() + dst.close() del src, dst .. _sqlite3-intro: @@ -125,7 +127,7 @@ and call :meth:`res.fetchone() ` to fetch the resulting row: We can see that the table has been created, as the query returns a :class:`tuple` containing the table's name. If we query ``sqlite_master`` for a non-existent table ``spam``, -:meth:`!res.fetchone()` will return ``None``: +:meth:`!res.fetchone` will return ``None``: .. doctest:: @@ -220,6 +222,7 @@ creating a new cursor, then querying the database: >>> title, year = res.fetchone() >>> print(f'The highest scoring Monty Python movie is {title!r}, released in {year}') The highest scoring Monty Python movie is 'Monty Python and the Holy Grail', released in 1975 + >>> new_con.close() You've now created an SQLite database using the :mod:`!sqlite3` module, inserted data and retrieved values from it in multiple ways. @@ -394,29 +397,11 @@ Module functions will get tracebacks from callbacks on :data:`sys.stderr`. Use ``False`` to disable the feature again. - Register an :func:`unraisable hook handler ` for an - improved debug experience: - - .. testsetup:: sqlite3.trace - - import sqlite3 + .. note:: - .. doctest:: sqlite3.trace - - >>> sqlite3.enable_callback_tracebacks(True) - >>> con = sqlite3.connect(":memory:") - >>> def evil_trace(stmt): - ... 5/0 - ... - >>> con.set_trace_callback(evil_trace) - >>> def debug(unraisable): - ... print(f"{unraisable.exc_value!r} in callback {unraisable.object.__name__}") - ... print(f"Error message: {unraisable.err_msg}") - >>> import sys - >>> sys.unraisablehook = debug - >>> cur = con.execute("SELECT 1") - ZeroDivisionError('division by zero') in callback evil_trace - Error message: None + Errors in user-defined function callbacks are logged as unraisable exceptions. + Use an :func:`unraisable hook handler ` for + introspection of the failed callback. .. function:: register_adapter(type, adapter, /) @@ -540,46 +525,25 @@ Module constants The mappings from SQLite threading modes to DB-API 2.0 threadsafety levels are as follows: - +------------------+-----------------+----------------------+-------------------------------+ - | SQLite threading | `threadsafety`_ | `SQLITE_THREADSAFE`_ | DB-API 2.0 meaning | - | mode | | | | - +==================+=================+======================+===============================+ - | single-thread | 0 | 0 | Threads may not share the | - | | | | module | - +------------------+-----------------+----------------------+-------------------------------+ - | multi-thread | 1 | 2 | Threads may share the module, | - | | | | but not connections | - +------------------+-----------------+----------------------+-------------------------------+ - | serialized | 3 | 1 | Threads may share the module, | - | | | | connections and cursors | - +------------------+-----------------+----------------------+-------------------------------+ - - .. _threadsafety: https://peps.python.org/pep-0249/#threadsafety + +------------------+----------------------+----------------------+-------------------------------+ + | SQLite threading | :pep:`threadsafety | `SQLITE_THREADSAFE`_ | DB-API 2.0 meaning | + | mode | <0249#threadsafety>` | | | + +==================+======================+======================+===============================+ + | single-thread | 0 | 0 | Threads may not share the | + | | | | module | + +------------------+----------------------+----------------------+-------------------------------+ + | multi-thread | 1 | 2 | Threads may share the module, | + | | | | but not connections | + +------------------+----------------------+----------------------+-------------------------------+ + | serialized | 3 | 1 | Threads may share the module, | + | | | | connections and cursors | + +------------------+----------------------+----------------------+-------------------------------+ + .. _SQLITE_THREADSAFE: https://sqlite.org/compile.html#threadsafe .. versionchanged:: 3.11 Set *threadsafety* dynamically instead of hard-coding it to ``1``. -.. data:: version - - Version number of this module as a :class:`string `. - This is not the version of the SQLite library. - - .. deprecated-removed:: 3.12 3.14 - This constant used to reflect the version number of the ``pysqlite`` - package, a third-party library which used to upstream changes to - :mod:`!sqlite3`. Today, it carries no meaning or practical value. - -.. data:: version_info - - Version number of this module as a :class:`tuple` of :class:`integers `. - This is not the version of the SQLite library. - - .. deprecated-removed:: 3.12 3.14 - This constant used to reflect the version number of the ``pysqlite`` - package, a third-party library which used to upstream changes to - :mod:`!sqlite3`. Today, it carries no meaning or practical value. - .. _sqlite3-dbconfig-constants: .. data:: SQLITE_DBCONFIG_DEFENSIVE @@ -612,6 +576,8 @@ Module constants https://www.sqlite.org/c3ref/c_dbconfig_defensive.html SQLite docs: Database Connection Configuration Options +.. deprecated-removed:: 3.12 3.14 + The :data:`!version` and :data:`!version_info` constants. .. _sqlite3-connection-objects: @@ -762,6 +728,7 @@ Connection objects >>> for row in con.execute("SELECT md5(?)", (b"foo",)): ... print(row) ('acbd18db4cc2f85cedef654fccc4a4d8',) + >>> con.close() .. versionchanged:: 3.13 @@ -908,6 +875,7 @@ Connection objects FROM test ORDER BY x """) print(cur.fetchall()) + con.close() .. testoutput:: :hide: @@ -1068,13 +1036,10 @@ Connection objects .. versionchanged:: 3.10 Added the ``sqlite3.enable_load_extension`` auditing event. - .. testsetup:: sqlite3.loadext - - import sqlite3 - con = sqlite3.connect(":memory:") + .. We cannot doctest the load extension API, since there is no convenient + way to skip it. - .. testcode:: sqlite3.loadext - :skipif: True # not testable at the moment + .. code-block:: con.enable_load_extension(True) @@ -1098,14 +1063,6 @@ Connection objects for row in con.execute("SELECT rowid, name, ingredients FROM recipe WHERE name MATCH 'pie'"): print(row) - con.close() - - .. testoutput:: sqlite3.loadext - :hide: - - (2, 'broccoli pie', 'broccoli cheese onions flour') - (3, 'pumpkin pie', 'pumpkin sugar flour butter') - .. method:: load_extension(path, /, *, entrypoint=None) Load an SQLite extension from a shared library. @@ -1135,7 +1092,7 @@ Connection objects .. versionchanged:: 3.12 Added the *entrypoint* parameter. - .. _Loading an Extension: https://www.sqlite.org/loadext.html#loading_an_extension_ + .. _Loading an Extension: https://www.sqlite.org/loadext.html#loading_an_extension .. method:: iterdump(*, filter=None) @@ -1230,6 +1187,8 @@ Connection objects src = sqlite3.connect('example.db') dst = sqlite3.connect(':memory:') src.backup(dst) + dst.close() + src.close() .. versionadded:: 3.7 @@ -1296,6 +1255,10 @@ Connection objects >>> con.getlimit(sqlite3.SQLITE_LIMIT_ATTACHED) 1 + .. testcleanup:: sqlite3.limits + + con.close() + .. versionadded:: 3.11 .. _SQLite limit category: https://www.sqlite.org/c3ref/c_limit_attached.html @@ -1577,6 +1540,10 @@ Cursor objects # cur is an sqlite3.Cursor object cur.executemany("INSERT INTO data VALUES(?)", rows) + .. testcleanup:: sqlite3.cursor + + con.close() + .. note:: Any resulting rows are discarded, @@ -1682,6 +1649,7 @@ Cursor objects >>> cur = con.cursor() >>> cur.connection == con True + >>> con.close() .. attribute:: description @@ -1802,6 +1770,7 @@ Blob objects greeting = blob.read() print(greeting) # outputs "b'Hello, world!'" + con.close() .. testoutput:: :hide: @@ -2114,6 +2083,7 @@ Here's an example of both styles: params = (1972,) cur.execute("SELECT * FROM lang WHERE first_appeared = ?", params) print(cur.fetchall()) + con.close() .. testoutput:: :hide: @@ -2172,6 +2142,7 @@ The object passed to *protocol* will be of type :class:`PrepareProtocol`. cur.execute("SELECT ?", (Point(4.0, -3.2),)) print(cur.fetchone()[0]) + con.close() .. testoutput:: :hide: @@ -2202,6 +2173,7 @@ This function can then be registered using :func:`register_adapter`. cur.execute("SELECT ?", (Point(1.0, 2.5),)) print(cur.fetchone()[0]) + con.close() .. testoutput:: :hide: @@ -2286,6 +2258,8 @@ The following example illustrates the implicit and explicit approaches: cur.execute("INSERT INTO test(p) VALUES(?)", (p,)) cur.execute('SELECT p AS "p [point]" FROM test') print("with column names:", cur.fetchone()[0]) + cur.close() + con.close() .. testoutput:: :hide: @@ -2468,6 +2442,7 @@ Some useful URI tricks include: >>> con.execute("CREATE TABLE readonly(data)") Traceback (most recent call last): OperationalError: attempt to write a readonly database + >>> con.close() * Do not implicitly create a new database file if it does not already exist; will raise :exc:`~sqlite3.OperationalError` if unable to create a new file: @@ -2492,6 +2467,8 @@ Some useful URI tricks include: res = con2.execute("SELECT data FROM shared") assert res.fetchone() == (28,) + con1.close() + con2.close() More information about this feature, including a list of parameters, can be found in the `SQLite URI documentation`_. @@ -2538,6 +2515,7 @@ Queries now return :class:`!Row` objects: 'Earth' >>> row["RADIUS"] # Column names are case-insensitive. 6378 + >>> con.close() .. note:: @@ -2564,6 +2542,7 @@ Using it, queries now return a :class:`!dict` instead of a :class:`!tuple`: >>> for row in con.execute("SELECT 1 AS a, 2 AS b"): ... print(row) {'a': 1, 'b': 2} + >>> con.close() The following row factory returns a :term:`named tuple`: @@ -2590,6 +2569,7 @@ The following row factory returns a :term:`named tuple`: 1 >>> row.b # Attribute access. 2 + >>> con.close() With some adjustments, the above recipe can be adapted to use a :class:`~dataclasses.dataclass`, or any other custom class, @@ -2747,3 +2727,11 @@ regardless of the value of :attr:`~Connection.isolation_level`. .. _SQLite transaction behaviour: https://www.sqlite.org/lang_transaction.html#deferred_immediate_and_exclusive_transactions + +.. testcleanup:: + + import os + os.remove("backup.db") + os.remove("dump.sql") + os.remove("example.db") + os.remove("tutorial.db") diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index ce8f3dd2d317c1..37ea32dc8a56e5 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1,5 +1,5 @@ -:mod:`ssl` --- TLS/SSL wrapper for socket objects -================================================= +:mod:`!ssl` --- TLS/SSL wrapper for socket objects +================================================== .. module:: ssl :synopsis: TLS/SSL wrapper for socket objects @@ -151,6 +151,12 @@ purposes. variable :envvar:`SSLKEYLOGFILE` is set, :func:`create_default_context` enables key logging. + The default settings for this context include + :data:`VERIFY_X509_PARTIAL_CHAIN` and :data:`VERIFY_X509_STRICT`. + These make the underlying OpenSSL implementation behave more like + a conforming implementation of :rfc:`5280`, in exchange for a small + amount of incompatibility with older X.509 certificates. + .. note:: The protocol, options, cipher and other settings may change to more restrictive values anytime without prior deprecation. The values @@ -172,6 +178,15 @@ purposes. ctx = ssl.create_default_context(Purpose.CLIENT_AUTH) ctx.options &= ~ssl.OP_NO_SSLv3 + .. note:: + This context enables :data:`VERIFY_X509_STRICT` by default, which + may reject pre-:rfc:`5280` or malformed certificates that the + underlying OpenSSL implementation otherwise would accept. While disabling + this is not recommended, you can do so using:: + + ctx = ssl.create_default_context() + ctx.verify_flags &= ~ssl.VERIFY_X509_STRICT + .. versionadded:: 3.4 .. versionchanged:: 3.4.4 @@ -194,6 +209,11 @@ purposes. :data:`PROTOCOL_TLS_SERVER` protocol instead of generic :data:`PROTOCOL_TLS`. + .. versionchanged:: 3.13 + + The context now uses :data:`VERIFY_X509_PARTIAL_CHAIN` and + :data:`VERIFY_X509_STRICT` in its default verify flags. + Exceptions ^^^^^^^^^^ @@ -737,11 +757,11 @@ Constants When Python has been compiled against an older version of OpenSSL, the flag defaults to *0*. - .. versionadded:: 3.7 + .. versionadded:: 3.6.3 .. deprecated:: 3.7 - The option is deprecated since OpenSSL 1.1.0. It was added to 2.7.15, - 3.6.3 and 3.7.0 for backwards compatibility with OpenSSL 1.0.2. + The option is deprecated since OpenSSL 1.1.0. It was added to 2.7.15 and + 3.6.3 for backwards compatibility with OpenSSL 1.0.2. .. data:: OP_NO_RENEGOTIATION @@ -761,7 +781,7 @@ Constants .. data:: OP_SINGLE_DH_USE - Prevents re-use of the same DH key for distinct SSL sessions. This + Prevents reuse of the same DH key for distinct SSL sessions. This improves forward secrecy but requires more computational resources. This option only applies to server sockets. @@ -769,7 +789,7 @@ Constants .. data:: OP_SINGLE_ECDH_USE - Prevents re-use of the same ECDH key for distinct SSL sessions. This + Prevents reuse of the same ECDH key for distinct SSL sessions. This improves forward secrecy but requires more computational resources. This option only applies to server sockets. @@ -914,6 +934,12 @@ Constants .. versionadded:: 3.13 +.. data:: HAS_PHA + + Whether the OpenSSL library has built-in support for TLS-PHA. + + .. versionadded:: 3.14 + .. data:: CHANNEL_BINDING_TYPES List of supported TLS channel binding types. Strings in this list @@ -1029,25 +1055,25 @@ SSL Sockets SSL sockets provide the following methods of :ref:`socket-objects`: - - :meth:`~socket.socket.accept()` - - :meth:`~socket.socket.bind()` - - :meth:`~socket.socket.close()` - - :meth:`~socket.socket.connect()` - - :meth:`~socket.socket.detach()` - - :meth:`~socket.socket.fileno()` - - :meth:`~socket.socket.getpeername()`, :meth:`~socket.socket.getsockname()` - - :meth:`~socket.socket.getsockopt()`, :meth:`~socket.socket.setsockopt()` - - :meth:`~socket.socket.gettimeout()`, :meth:`~socket.socket.settimeout()`, - :meth:`~socket.socket.setblocking()` - - :meth:`~socket.socket.listen()` - - :meth:`~socket.socket.makefile()` - - :meth:`~socket.socket.recv()`, :meth:`~socket.socket.recv_into()` + - :meth:`~socket.socket.accept` + - :meth:`~socket.socket.bind` + - :meth:`~socket.socket.close` + - :meth:`~socket.socket.connect` + - :meth:`~socket.socket.detach` + - :meth:`~socket.socket.fileno` + - :meth:`~socket.socket.getpeername`, :meth:`~socket.socket.getsockname` + - :meth:`~socket.socket.getsockopt`, :meth:`~socket.socket.setsockopt` + - :meth:`~socket.socket.gettimeout`, :meth:`~socket.socket.settimeout`, + :meth:`~socket.socket.setblocking` + - :meth:`~socket.socket.listen` + - :meth:`~socket.socket.makefile` + - :meth:`~socket.socket.recv`, :meth:`~socket.socket.recv_into` (but passing a non-zero ``flags`` argument is not allowed) - - :meth:`~socket.socket.send()`, :meth:`~socket.socket.sendall()` (with + - :meth:`~socket.socket.send`, :meth:`~socket.socket.sendall` (with the same limitation) - - :meth:`~socket.socket.sendfile()` (but :mod:`os.sendfile` will be used - for plain-text sockets only, else :meth:`~socket.socket.send()` will be used) - - :meth:`~socket.socket.shutdown()` + - :meth:`~socket.socket.sendfile` (but :mod:`os.sendfile` will be used + for plain-text sockets only, else :meth:`~socket.socket.send` will be used) + - :meth:`~socket.socket.shutdown` However, since the SSL (and TLS) protocol has its own framing atop of TCP, the SSL sockets abstraction can, in certain respects, diverge from @@ -1452,6 +1478,19 @@ to speed up repeated connections from the same clients. :data:`PROTOCOL_TLS`, :data:`PROTOCOL_TLS_CLIENT`, and :data:`PROTOCOL_TLS_SERVER` use TLS 1.2 as minimum TLS version. + .. note:: + + :class:`SSLContext` only supports limited mutation once it has been used + by a connection. Adding new certificates to the internal trust store is + allowed, but changing ciphers, verification settings, or mTLS + certificates may result in surprising behavior. + + .. note:: + + :class:`SSLContext` is designed to be shared and used by multiple + connections. + Thus, it is thread-safe as long as it is not reconfigured after being + used by a connection. :class:`SSLContext` objects have the following methods and attributes: @@ -1533,7 +1572,7 @@ to speed up repeated connections from the same clients. The *capath* string, if present, is the path to a directory containing several CA certificates in PEM format, following an `OpenSSL specific layout - `_. + `_. The *cadata* object, if present, is either an ASCII string of one or more PEM-encoded certificates or a :term:`bytes-like object` of DER-encoded @@ -1608,7 +1647,7 @@ to speed up repeated connections from the same clients. Set the available ciphers for sockets created with this context. It should be a string in the `OpenSSL cipher list format - `_. + `_. If no cipher can be selected (because compile-time options or other configuration forbids use of all the specified ciphers), an :class:`SSLError` will be raised. @@ -1709,7 +1748,7 @@ to speed up repeated connections from the same clients. IDN-encoded internationalized domain name, the *server_name_callback* receives a decoded U-label (``"pythön.org"``). - If there is an decoding error on the server name, the TLS connection will + If there is a decoding error on the server name, the TLS connection will terminate with an :const:`ALERT_DESCRIPTION_INTERNAL_ERROR` fatal TLS alert message to the client. @@ -1800,7 +1839,7 @@ to speed up repeated connections from the same clients. .. versionchanged:: 3.6 *session* argument was added. - .. versionchanged:: 3.7 + .. versionchanged:: 3.7 The method returns an instance of :attr:`SSLContext.sslsocket_class` instead of hard-coded :class:`SSLSocket`. @@ -1841,7 +1880,7 @@ to speed up repeated connections from the same clients. .. method:: SSLContext.session_stats() Get statistics about the SSL sessions created or managed by this context. - A dictionary is returned which maps the names of each `piece of information `_ to their + A dictionary is returned which maps the names of each `piece of information `_ to their numeric values. For example, here is the total number of hits and misses in the session cache since the context was created:: @@ -1984,7 +2023,7 @@ to speed up repeated connections from the same clients. .. attribute:: SSLContext.security_level An integer representing the `security level - `_ + `_ for the context. This attribute is read-only. .. versionadded:: 3.10 @@ -2475,8 +2514,8 @@ thus several things you need to be aware of: .. seealso:: The :mod:`asyncio` module supports :ref:`non-blocking SSL sockets - ` and provides a - higher level API. It polls for events using the :mod:`selectors` module and + ` and provides a higher level :ref:`Streams API `. + It polls for events using the :mod:`selectors` module and handles :exc:`SSLWantWriteError`, :exc:`SSLWantReadError` and :exc:`BlockingIOError` exceptions. It runs the SSL handshake asynchronously as well. @@ -2677,7 +2716,7 @@ Verifying certificates When calling the :class:`SSLContext` constructor directly, :const:`CERT_NONE` is the default. Since it does not authenticate the other -peer, it can be insecure, especially in client mode where most of time you +peer, it can be insecure, especially in client mode where most of the time you would like to ensure the authenticity of the server you're talking to. Therefore, when in client mode, it is highly recommended to use :const:`CERT_REQUIRED`. However, it is in itself not sufficient; you also @@ -2726,7 +2765,7 @@ enabled when negotiating a SSL session is possible through the :meth:`SSLContext.set_ciphers` method. Starting from Python 3.2.3, the ssl module disables certain weak ciphers by default, but you may want to further restrict the cipher choice. Be sure to read OpenSSL's documentation -about the `cipher list format `_. +about the `cipher list format `_. If you want to check which ciphers are enabled by a given cipher list, use :meth:`SSLContext.get_ciphers` or the ``openssl ciphers`` command on your system. diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst index c941d5557e31b5..8434b2e8c75cf4 100644 --- a/Doc/library/stat.rst +++ b/Doc/library/stat.rst @@ -1,5 +1,5 @@ -:mod:`stat` --- Interpreting :func:`~os.stat` results -===================================================== +:mod:`!stat` --- Interpreting :func:`~os.stat` results +====================================================== .. module:: stat :synopsis: Utilities for interpreting the results of os.stat(), @@ -354,7 +354,7 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: All user settable flags. - .. versionadded: 3.13 + .. versionadded:: 3.13 .. data:: UF_NODUMP @@ -384,13 +384,13 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: Used for handling document IDs (macOS) - .. versionadded: 3.13 + .. versionadded:: 3.13 .. data:: UF_DATAVAULT The file needs an entitlement for reading or writing (macOS 10.13+) - .. versionadded: 3.13 + .. versionadded:: 3.13 .. data:: UF_HIDDEN @@ -400,7 +400,7 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: All super-user changeable flags - .. versionadded: 3.13 + .. versionadded:: 3.13 .. data:: SF_SUPPORTED @@ -408,7 +408,7 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: .. availability:: macOS - .. versionadded: 3.13 + .. versionadded:: 3.13 .. data:: SF_SYNTHETIC @@ -416,7 +416,7 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: .. availability:: macOS - .. versionadded: 3.13 + .. versionadded:: 3.13 .. data:: SF_ARCHIVED @@ -434,7 +434,7 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: The file needs an entitlement to write to (macOS 10.13+) - .. versionadded: 3.13 + .. versionadded:: 3.13 .. data:: SF_NOUNLINK @@ -448,13 +448,13 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: The file is a firmlink (macOS 10.15+) - .. versionadded: 3.13 + .. versionadded:: 3.13 .. data:: SF_DATALESS The file is a dataless object (macOS 10.15+) - .. versionadded: 3.13 + .. versionadded:: 3.13 See the \*BSD or macOS systems man page :manpage:`chflags(2)` for more information. diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 1785c6bcc212b7..614f5b905a4a2e 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -1,5 +1,5 @@ -:mod:`statistics` --- Mathematical statistics functions -======================================================= +:mod:`!statistics` --- Mathematical statistics functions +======================================================== .. module:: statistics :synopsis: Mathematical statistics functions @@ -73,14 +73,15 @@ or sample. ======================= =============================================================== :func:`mean` Arithmetic mean ("average") of data. -:func:`fmean` Fast, floating point arithmetic mean, with optional weighting. +:func:`fmean` Fast, floating-point arithmetic mean, with optional weighting. :func:`geometric_mean` Geometric mean of data. :func:`harmonic_mean` Harmonic mean of data. :func:`kde` Estimate the probability density distribution of the data. +:func:`kde_random` Random sampling from the PDF generated by kde(). :func:`median` Median (middle value) of data. :func:`median_low` Low median of data. :func:`median_high` High median of data. -:func:`median_grouped` Median, or 50th percentile, of grouped data. +:func:`median_grouped` Median (50th percentile) of grouped data. :func:`mode` Single mode (most common value) of discrete or nominal data. :func:`multimode` List of modes (most common values) of discrete or nominal data. :func:`quantiles` Divide data into intervals with equal probability. @@ -219,7 +220,7 @@ However, for reading convenience, most of the examples show sorted sequences. .. function:: harmonic_mean(data, weights=None) Return the harmonic mean of *data*, a sequence or iterable of - real-valued numbers. If *weights* is omitted or *None*, then + real-valued numbers. If *weights* is omitted or ``None``, then equal weighting is assumed. The harmonic mean is the reciprocal of the arithmetic :func:`mean` of the @@ -261,11 +262,12 @@ However, for reading convenience, most of the examples show sorted sequences. Added support for *weights*. -.. function:: kde(data, h, kernel='normal') +.. function:: kde(data, h, kernel='normal', *, cumulative=False) `Kernel Density Estimation (KDE) `_: - Create a continuous probability density function from discrete samples. + Create a continuous probability density function or cumulative + distribution function from discrete samples. The basic idea is to smooth the data using `a kernel function `_. @@ -280,11 +282,13 @@ However, for reading convenience, most of the examples show sorted sequences. as much as the more influential bandwidth smoothing parameter. Kernels that give some weight to every sample point include - *normal* or *gauss*, *logistic*, and *sigmoid*. + *normal* (*gauss*), *logistic*, and *sigmoid*. Kernels that only give weight to sample points within the bandwidth - include *rectangular* or *uniform*, *triangular*, *parabolic* or - *epanechnikov*, *quartic* or *biweight*, *triweight*, and *cosine*. + include *rectangular* (*uniform*), *triangular*, *parabolic* + (*epanechnikov*), *quartic* (*biweight*), *triweight*, and *cosine*. + + If *cumulative* is true, will return a cumulative distribution function. A :exc:`StatisticsError` will be raised if the *data* sequence is empty. @@ -308,6 +312,30 @@ However, for reading convenience, most of the examples show sorted sequences. .. versionadded:: 3.13 +.. function:: kde_random(data, h, kernel='normal', *, seed=None) + + Return a function that makes a random selection from the estimated + probability density function produced by ``kde(data, h, kernel)``. + + Providing a *seed* allows reproducible selections. In the future, the + values may change slightly as more accurate kernel inverse CDF estimates + are implemented. The seed may be an integer, float, str, or bytes. + + A :exc:`StatisticsError` will be raised if the *data* sequence is empty. + + Continuing the example for :func:`kde`, we can use + :func:`kde_random` to generate new random selections from an + estimated probability density function: + + >>> data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> rand = kde_random(data, h=1.5, seed=8675309) + >>> new_selections = [rand() for i in range(10)] + >>> [round(x, 1) for x in new_selections] + [0.7, 6.2, 1.2, 6.9, 7.0, 1.8, 2.5, -0.5, -1.8, 5.6] + + .. versionadded:: 3.13 + + .. function:: median(data) Return the median (middle value) of numeric data, using the common "mean of @@ -378,55 +406,56 @@ However, for reading convenience, most of the examples show sorted sequences. be an actual data point rather than interpolated. -.. function:: median_grouped(data, interval=1) +.. function:: median_grouped(data, interval=1.0) - Return the median of grouped continuous data, calculated as the 50th - percentile, using interpolation. If *data* is empty, :exc:`StatisticsError` - is raised. *data* can be a sequence or iterable. + Estimates the median for numeric data that has been `grouped or binned + `_ around the midpoints + of consecutive, fixed-width intervals. - .. doctest:: + The *data* can be any iterable of numeric data with each value being + exactly the midpoint of a bin. At least one value must be present. - >>> median_grouped([52, 52, 53, 54]) - 52.5 + The *interval* is the width of each bin. - In the following example, the data are rounded, so that each value represents - the midpoint of data classes, e.g. 1 is the midpoint of the class 0.5--1.5, 2 - is the midpoint of 1.5--2.5, 3 is the midpoint of 2.5--3.5, etc. With the data - given, the middle value falls somewhere in the class 3.5--4.5, and - interpolation is used to estimate it: + For example, demographic information may have been summarized into + consecutive ten-year age groups with each group being represented + by the 5-year midpoints of the intervals: .. doctest:: - >>> median_grouped([1, 2, 2, 3, 4, 4, 4, 4, 4, 5]) - 3.7 - - Optional argument *interval* represents the class interval, and defaults - to 1. Changing the class interval naturally will change the interpolation: + >>> from collections import Counter + >>> demographics = Counter({ + ... 25: 172, # 20 to 30 years old + ... 35: 484, # 30 to 40 years old + ... 45: 387, # 40 to 50 years old + ... 55: 22, # 50 to 60 years old + ... 65: 6, # 60 to 70 years old + ... }) + ... + + The 50th percentile (median) is the 536th person out of the 1071 + member cohort. That person is in the 30 to 40 year old age group. + + The regular :func:`median` function would assume that everyone in the + tricenarian age group was exactly 35 years old. A more tenable + assumption is that the 484 members of that age group are evenly + distributed between 30 and 40. For that, we use + :func:`median_grouped`: .. doctest:: - >>> median_grouped([1, 3, 3, 5, 7], interval=1) - 3.25 - >>> median_grouped([1, 3, 3, 5, 7], interval=2) - 3.5 - - This function does not check whether the data points are at least - *interval* apart. + >>> data = list(demographics.elements()) + >>> median(data) + 35 + >>> round(median_grouped(data, interval=10), 1) + 37.5 - .. impl-detail:: + The caller is responsible for making sure the data points are separated + by exact multiples of *interval*. This is essential for getting a + correct result. The function does not check this precondition. - Under some circumstances, :func:`median_grouped` may coerce data points to - floats. This behaviour is likely to change in the future. - - .. seealso:: - - * "Statistics for the Behavioral Sciences", Frederick J Gravetter and - Larry B Wallnau (8th Edition). - - * The `SSMEDIAN - `_ - function in the Gnome Gnumeric spreadsheet, including `this discussion - `_. + Inputs may be any numeric type that can be coerced to a float during + the interpolation step. .. function:: mode(data) @@ -456,6 +485,12 @@ However, for reading convenience, most of the examples show sorted sequences. >>> mode(["red", "blue", "blue", "red", "green", "red", "red"]) 'red' + Only hashable inputs are supported. To handle type :class:`set`, + consider casting to :class:`frozenset`. To handle type :class:`list`, + consider casting to :class:`tuple`. For mixed or nested inputs, consider + using this slower quadratic algorithm that only depends on equality tests: + ``max(data, key=data.count)``. + .. versionchanged:: 3.8 Now handles multimodal datasets by returning the first mode encountered. Formerly, it raised :exc:`StatisticsError` when more than one mode was @@ -497,9 +532,9 @@ However, for reading convenience, most of the examples show sorted sequences. variance indicates that the data is spread out; a small variance indicates it is clustered closely around the mean. - If the optional second argument *mu* is given, it is typically the mean of - the *data*. It can also be used to compute the second moment around a - point that is not the mean. If it is missing or ``None`` (the default), + If the optional second argument *mu* is given, it should be the *population* + mean of the *data*. It can also be used to compute the second moment around + a point that is not the mean. If it is missing or ``None`` (the default), the arithmetic mean is automatically calculated. Use this function to calculate the variance from the entire population. To @@ -569,8 +604,8 @@ However, for reading convenience, most of the examples show sorted sequences. the data is spread out; a small variance indicates it is clustered closely around the mean. - If the optional second argument *xbar* is given, it should be the mean of - *data*. If it is missing or ``None`` (the default), the mean is + If the optional second argument *xbar* is given, it should be the *sample* + mean of *data*. If it is missing or ``None`` (the default), the mean is automatically calculated. Use this function when your data is a sample from a population. To calculate @@ -586,8 +621,8 @@ However, for reading convenience, most of the examples show sorted sequences. >>> variance(data) 1.3720238095238095 - If you have already calculated the mean of your data, you can pass it as the - optional second argument *xbar* to avoid recalculation: + If you have already calculated the sample mean of your data, you can pass it + as the optional second argument *xbar* to avoid recalculation: .. doctest:: @@ -997,8 +1032,8 @@ of applications in statistics. .. versionadded:: 3.8 -:class:`NormalDist` Examples and Recipes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Examples and Recipes +-------------------- Classic probability problems @@ -1033,7 +1068,7 @@ Find the `quartiles `_ and `deciles Monte Carlo inputs for simulations ********************************** -To estimate the distribution for a model than isn't easy to solve +To estimate the distribution for a model that isn't easy to solve analytically, :class:`NormalDist` can generate input samples for a `Monte Carlo simulation `_: diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index c81626a194e640..6050784264707b 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -209,18 +209,18 @@ Numeric Types --- :class:`int`, :class:`float`, :class:`complex` pair: object; numeric pair: object; Boolean pair: object; integer - pair: object; floating point + pair: object; floating-point pair: object; complex number pair: C; language -There are three distinct numeric types: :dfn:`integers`, :dfn:`floating -point numbers`, and :dfn:`complex numbers`. In addition, Booleans are a -subtype of integers. Integers have unlimited precision. Floating point +There are three distinct numeric types: :dfn:`integers`, :dfn:`floating-point +numbers`, and :dfn:`complex numbers`. In addition, Booleans are a +subtype of integers. Integers have unlimited precision. Floating-point numbers are usually implemented using :c:expr:`double` in C; information -about the precision and internal representation of floating point +about the precision and internal representation of floating-point numbers for the machine on which your program is running is available in :data:`sys.float_info`. Complex numbers have a real and imaginary -part, which are each a floating point number. To extract these parts +part, which are each a floating-point number. To extract these parts from a complex number *z*, use ``z.real`` and ``z.imag``. (The standard library includes the additional numeric types :mod:`fractions.Fraction`, for rationals, and :mod:`decimal.Decimal`, for floating-point numbers with @@ -229,7 +229,7 @@ user-definable precision.) .. index:: pair: numeric; literals pair: integer; literals - pair: floating point; literals + pair: floating-point; literals pair: complex number; literals pair: hexadecimal; literals pair: octal; literals @@ -238,11 +238,14 @@ user-definable precision.) Numbers are created by numeric literals or as the result of built-in functions and operators. Unadorned integer literals (including hex, octal and binary numbers) yield integers. Numeric literals containing a decimal point or an -exponent sign yield floating point numbers. Appending ``'j'`` or ``'J'`` to a +exponent sign yield floating-point numbers. Appending ``'j'`` or ``'J'`` to a numeric literal yields an imaginary number (a complex number with a zero real part) which you can add to an integer or float to get a complex number with real and imaginary parts. +The constructors :func:`int`, :func:`float`, and +:func:`complex` can be used to produce numbers of a specific type. + .. index:: single: arithmetic pair: built-in function; int @@ -262,12 +265,15 @@ and imaginary parts. Python fully supports mixed arithmetic: when a binary arithmetic operator has operands of different numeric types, the operand with the "narrower" type is -widened to that of the other, where integer is narrower than floating point, -which is narrower than complex. A comparison between numbers of different types -behaves as though the exact values of those numbers were being compared. [2]_ +widened to that of the other, where integer is narrower than floating point. +Arithmetic with complex and real operands is defined by the usual mathematical +formula, for example:: -The constructors :func:`int`, :func:`float`, and -:func:`complex` can be used to produce numbers of a specific type. + x + complex(u, v) = complex(x + u, v) + x * complex(u, v) = complex(x * u, x * v) + +A comparison between numbers of different types behaves as though the exact +values of those numbers were being compared. [2]_ All numeric types (except complex) support the following operations (for priorities of the operations, see :ref:`operator-summary`): @@ -625,6 +631,23 @@ Additional Methods on Float The float type implements the :class:`numbers.Real` :term:`abstract base class`. float also has the following additional methods. +.. classmethod:: float.from_number(x) + + Class method to return a floating-point number constructed from a number *x*. + + If the argument is an integer or a floating-point number, a + floating-point number with the same value (within Python's floating-point + precision) is returned. If the argument is outside the range of a Python + float, an :exc:`OverflowError` will be raised. + + For a general Python object ``x``, ``float.from_number(x)`` delegates to + ``x.__float__()``. + If :meth:`~object.__float__` is not defined then it falls back + to :meth:`~object.__index__`. + + .. versionadded:: 3.14 + + .. method:: float.as_integer_ratio() Return a pair of integers whose ratio is exactly equal to the @@ -703,6 +726,25 @@ hexadecimal string representing the same number:: '0x1.d380000000000p+11' +Additional Methods on Complex +----------------------------- + +The :class:`!complex` type implements the :class:`numbers.Complex` +:term:`abstract base class`. +:class:`!complex` also has the following additional methods. + +.. classmethod:: complex.from_number(x) + + Class method to convert a number to a complex number. + + For a general Python object ``x``, ``complex.from_number(x)`` delegates to + ``x.__complex__()``. If :meth:`~object.__complex__` is not defined then it falls back + to :meth:`~object.__float__`. If :meth:`!__float__` is not defined then it falls back + to :meth:`~object.__index__`. + + .. versionadded:: 3.14 + + .. _numeric-hash: Hashing of numeric types @@ -832,7 +874,7 @@ over ``&``, ``|`` and ``^``. .. deprecated:: 3.12 The use of the bitwise inversion operator ``~`` is deprecated and will - raise an error in Python 3.14. + raise an error in Python 3.16. :class:`bool` is a subclass of :class:`int` (see :ref:`typesnumeric`). In many numeric contexts, ``False`` and ``True`` behave like the integers 0 and 1, respectively. @@ -1209,8 +1251,9 @@ accepts integers that meet the value restriction ``0 <= x <= 255``). | ``s.pop()`` or ``s.pop(i)`` | retrieves the item at *i* and | \(2) | | | also removes it from *s* | | +------------------------------+--------------------------------+---------------------+ -| ``s.remove(x)`` | remove the first item from *s* | \(3) | -| | where ``s[i]`` is equal to *x* | | +| ``s.remove(x)`` | removes the first item from | \(3) | +| | *s* where ``s[i]`` is equal to | | +| | *x* | | +------------------------------+--------------------------------+---------------------+ | ``s.reverse()`` | reverses the items of *s* in | \(4) | | | place | | @@ -1220,7 +1263,7 @@ accepts integers that meet the value restriction ``0 <= x <= 255``). Notes: (1) - *t* must have the same length as the slice it is replacing. + If *k* is not equal to ``1``, *t* must have the same length as the slice it is replacing. (2) The optional argument *i* defaults to ``-1``, so that by default the last @@ -1491,21 +1534,114 @@ objects that compare equal might have different :attr:`~range.start`, sequence of values they define (instead of comparing based on object identity). -.. versionadded:: 3.3 - The :attr:`~range.start`, :attr:`~range.stop` and :attr:`~range.step` + Added the :attr:`~range.start`, :attr:`~range.stop` and :attr:`~range.step` attributes. .. seealso:: - * The `linspace recipe `_ - shows how to implement a lazy version of range suitable for floating - point applications. + * The `linspace recipe `_ + shows how to implement a lazy version of range suitable for floating-point + applications. .. index:: single: string; text sequence type single: str (built-in class); (see also string) pair: object; string +.. _text-methods-summary: + +Text and Binary Sequence Type Methods Summary +============================================= +The following table summarizes the text and binary sequence types methods by +category. + + ++--------------------------+-------------------------------------------+---------------------------------------------------+ +| Category | :class:`str` methods | :class:`bytes` and :class:`bytearray` methods | ++==========================+===========================================+===================================================+ +| Formatting | :meth:`str.format` | | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.format_map` | | +| +-------------------------------------------+---------------------------------------------------+ +| | :ref:`f-strings` | | +| +-------------------------------------------+---------------------------------------------------+ +| | :ref:`old-string-formatting` | :ref:`bytes-formatting` | ++--------------------------+------------------+------------------------+--------------------+------------------------------+ +| Searching and Replacing | :meth:`str.find` | :meth:`str.rfind` | :meth:`bytes.find` | :meth:`bytes.rfind` | +| +------------------+------------------------+--------------------+------------------------------+ +| | :meth:`str.index`| :meth:`str.rindex` | :meth:`bytes.index`| :meth:`bytes.rindex` | +| +------------------+------------------------+--------------------+------------------------------+ +| | :meth:`str.startswith` | :meth:`bytes.startswith` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.endswith` | :meth:`bytes.endswith` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.count` | :meth:`bytes.count` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.replace` | :meth:`bytes.replace` | ++--------------------------+-------------------+-----------------------+---------------------+-----------------------------+ +| Splitting and Joining | :meth:`str.split` | :meth:`str.rsplit` | :meth:`bytes.split` | :meth:`bytes.rsplit` | +| +-------------------+-----------------------+---------------------+-----------------------------+ +| | :meth:`str.splitlines` | :meth:`bytes.splitlines` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.partition` | :meth:`bytes.partition` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.rpartition` | :meth:`bytes.rpartition` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.join` | :meth:`bytes.join` | ++--------------------------+-------------------------------------------+---------------------------------------------------+ +| String Classification | :meth:`str.isalpha` | :meth:`bytes.isalpha` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.isdecimal` | | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.isdigit` | :meth:`bytes.isdigit` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.isnumeric` | | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.isalnum` | :meth:`bytes.isalnum` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.isidentifier` | | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.islower` | :meth:`bytes.islower` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.isupper` | :meth:`bytes.isupper` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.istitle` | :meth:`bytes.istitle` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.isspace` | :meth:`bytes.isspace` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.isprintable` | | ++--------------------------+-------------------------------------------+---------------------------------------------------+ +| Case Manipulation | :meth:`str.lower` | :meth:`bytes.lower` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.upper` | :meth:`bytes.upper` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.casefold` | | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.capitalize` | :meth:`bytes.capitalize` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.title` | :meth:`bytes.title` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.swapcase` | :meth:`bytes.swapcase` | ++--------------------------+-------------------+-----------------------+---------------------+-----------------------------+ +| Padding and Stripping | :meth:`str.ljust` | :meth:`str.rjust` | :meth:`bytes.ljust` | :meth:`bytes.rjust` | +| +-------------------+-----------------------+---------------------+-----------------------------+ +| | :meth:`str.center` | :meth:`bytes.center` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.expandtabs` | :meth:`bytes.expandtabs` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.strip` | :meth:`bytes.strip` | +| +--------------------+----------------------+----------------------+----------------------------+ +| | :meth:`str.lstrip` | :meth:`str.rstrip` | :meth:`bytes.lstrip` | :meth:`bytes.rstrip` | ++--------------------------+--------------------+----------------------+----------------------+----------------------------+ +| Translation and Encoding | :meth:`str.translate` | :meth:`bytes.translate` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.maketrans` | :meth:`bytes.maketrans` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.encode` | | +| +-------------------------------------------+---------------------------------------------------+ +| | | :meth:`bytes.decode` | ++--------------------------+-------------------------------------------+---------------------------------------------------+ + .. _textseq: Text Sequence Type --- :class:`str` @@ -1643,7 +1779,7 @@ expression support in the :mod:`re` module). The casefolding algorithm is `described in section 3.13 'Default Case Folding' of the Unicode Standard - `__. + `__. .. versionadded:: 3.3 @@ -1769,7 +1905,7 @@ expression support in the :mod:`re` module). cases. -.. method:: str.format_map(mapping) +.. method:: str.format_map(mapping, /) Similar to ``str.format(**mapping)``, except that ``mapping`` is used directly and not copied to a :class:`dict`. This is useful @@ -1807,7 +1943,7 @@ expression support in the :mod:`re` module). property being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is different from the `Alphabetic property defined in the section 4.10 'Letters, Alphabetic, and Ideographic' of the Unicode Standard - `_. + `_. .. method:: str.isascii() @@ -1943,7 +2079,7 @@ expression support in the :mod:`re` module). The lowercasing algorithm used is `described in section 3.13 'Default Case Folding' of the Unicode Standard - `__. + `__. .. method:: str.lstrip([chars]) @@ -2096,8 +2232,9 @@ expression support in the :mod:`re` module). If *sep* is given, consecutive delimiters are not grouped together and are deemed to delimit empty strings (for example, ``'1,,2'.split(',')`` returns ``['1', '', '2']``). The *sep* argument may consist of multiple characters - (for example, ``'1<>2<>3'.split('<>')`` returns ``['1', '2', '3']``). - Splitting an empty string with a specified separator returns ``['']``. + as a single delimiter (to split with multiple delimiters, use + :func:`re.split`). Splitting an empty string with a specified separator + returns ``['']``. For example:: @@ -2107,6 +2244,8 @@ expression support in the :mod:`re` module). ['1', '2,3'] >>> '1,2,,3,'.split(',') ['1', '2', '', '3', ''] + >>> '1<>2<>3<4'.split('<>') + ['1', '2', '3<4'] If *sep* is not specified or is ``None``, a different splitting algorithm is applied: runs of consecutive whitespace are regarded as a single separator, @@ -2292,7 +2431,7 @@ expression support in the :mod:`re` module). The uppercasing algorithm used is `described in section 3.13 'Default Case Folding' of the Unicode Standard - `__. + `__. .. method:: str.zfill(width) @@ -2340,7 +2479,13 @@ String objects have one unique built-in operation: the ``%`` operator (modulo). This is also known as the string *formatting* or *interpolation* operator. Given ``format % values`` (where *format* is a string), ``%`` conversion specifications in *format* are replaced with zero or more elements of *values*. -The effect is similar to using the :c:func:`sprintf` in the C language. +The effect is similar to using the :c:func:`sprintf` function in the C language. +For example: + +.. doctest:: + + >>> print('%s has %d quote types.' % ('Python', 2)) + Python has 2 quote types. If *format* requires a single argument, *values* may be a single non-tuple object. [5]_ Otherwise, *values* must be a tuple with exactly the number of @@ -2434,19 +2579,19 @@ The conversion types are: +------------+-----------------------------------------------------+-------+ | ``'X'`` | Signed hexadecimal (uppercase). | \(2) | +------------+-----------------------------------------------------+-------+ -| ``'e'`` | Floating point exponential format (lowercase). | \(3) | +| ``'e'`` | Floating-point exponential format (lowercase). | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'E'`` | Floating point exponential format (uppercase). | \(3) | +| ``'E'`` | Floating-point exponential format (uppercase). | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'f'`` | Floating point decimal format. | \(3) | +| ``'f'`` | Floating-point decimal format. | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'F'`` | Floating point decimal format. | \(3) | +| ``'F'`` | Floating-point decimal format. | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'g'`` | Floating point format. Uses lowercase exponential | \(4) | +| ``'g'`` | Floating-point format. Uses lowercase exponential | \(4) | | | format if exponent is less than -4 or not less than | | | | precision, decimal format otherwise. | | +------------+-----------------------------------------------------+-------+ -| ``'G'`` | Floating point format. Uses uppercase exponential | \(4) | +| ``'G'`` | Floating-point format. Uses uppercase exponential | \(4) | | | format if exponent is less than -4 or not less than | | | | precision, decimal format otherwise. | | +------------+-----------------------------------------------------+-------+ @@ -3144,10 +3289,9 @@ produce new objects. If *sep* is given, consecutive delimiters are not grouped together and are deemed to delimit empty subsequences (for example, ``b'1,,2'.split(b',')`` returns ``[b'1', b'', b'2']``). The *sep* argument may consist of a - multibyte sequence (for example, ``b'1<>2<>3'.split(b'<>')`` returns - ``[b'1', b'2', b'3']``). Splitting an empty sequence with a specified - separator returns ``[b'']`` or ``[bytearray(b'')]`` depending on the type - of object being split. The *sep* argument may be any + multibyte sequence as a single delimiter. Splitting an empty sequence with + a specified separator returns ``[b'']`` or ``[bytearray(b'')]`` depending + on the type of object being split. The *sep* argument may be any :term:`bytes-like object`. For example:: @@ -3158,6 +3302,8 @@ produce new objects. [b'1', b'2,3'] >>> b'1,2,,3,'.split(b',') [b'1', b'2', b'', b'3', b''] + >>> b'1<>2<>3<4'.split(b'<>') + [b'1', b'2', b'3<4'] If *sep* is not specified or is ``None``, a different splitting algorithm is applied: runs of consecutive ASCII whitespace are regarded as a single @@ -3431,7 +3577,7 @@ place, and instead produce new objects. ``b'abcdefghijklmnopqrstuvwxyz'``. Uppercase ASCII characters are those byte values in the sequence ``b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - Unlike :func:`str.swapcase()`, it is always the case that + Unlike :func:`str.swapcase`, it is always the case that ``bin.swapcase().swapcase() == bin`` for the binary versions. Case conversions are symmetrical in ASCII, even though that is not generally true for arbitrary Unicode code points. @@ -3652,19 +3798,19 @@ The conversion types are: +------------+-----------------------------------------------------+-------+ | ``'X'`` | Signed hexadecimal (uppercase). | \(2) | +------------+-----------------------------------------------------+-------+ -| ``'e'`` | Floating point exponential format (lowercase). | \(3) | +| ``'e'`` | Floating-point exponential format (lowercase). | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'E'`` | Floating point exponential format (uppercase). | \(3) | +| ``'E'`` | Floating-point exponential format (uppercase). | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'f'`` | Floating point decimal format. | \(3) | +| ``'f'`` | Floating-point decimal format. | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'F'`` | Floating point decimal format. | \(3) | +| ``'F'`` | Floating-point decimal format. | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'g'`` | Floating point format. Uses lowercase exponential | \(4) | +| ``'g'`` | Floating-point format. Uses lowercase exponential | \(4) | | | format if exponent is less than -4 or not less than | | | | precision, decimal format otherwise. | | +------------+-----------------------------------------------------+-------+ -| ``'G'`` | Floating point format. Uses uppercase exponential | \(4) | +| ``'G'`` | Floating-point format. Uses uppercase exponential | \(4) | | | format if exponent is less than -4 or not less than | | | | precision, decimal format otherwise. | | +------------+-----------------------------------------------------+-------+ @@ -3843,6 +3989,9 @@ copying. .. versionchanged:: 3.5 memoryviews can now be indexed with tuple of integers. + .. versionchanged:: 3.14 + memoryview is now a :term:`generic type`. + :class:`memoryview` has several methods: .. method:: __eq__(exporter) @@ -3886,7 +4035,7 @@ copying. >>> a == b False - Note that, as with floating point numbers, ``v is w`` does *not* imply + Note that, as with floating-point numbers, ``v is w`` does *not* imply ``v == w`` for memoryview objects. .. versionchanged:: 3.3 @@ -3977,7 +4126,7 @@ copying. dangling resources) as soon as possible. After this method has been called, any further operation on the view - raises a :class:`ValueError` (except :meth:`release()` itself which can + raises a :class:`ValueError` (except :meth:`release` itself which can be called multiple times):: >>> m = memoryview(b'abc') @@ -4094,6 +4243,21 @@ copying. .. versionchanged:: 3.5 The source format is no longer restricted when casting to a byte view. + .. method:: count(value, /) + + Count the number of occurrences of *value*. + + .. versionadded:: 3.14 + + .. method:: index(value, start=0, stop=sys.maxsize, /) + + Return the index of the first occurrence of *value* (at or after + index *start* and before index *stop*). + + Raises a :exc:`ValueError` if *value* cannot be found. + + .. versionadded:: 3.14 + There are also several readonly attributes available: .. attribute:: obj @@ -4459,14 +4623,14 @@ can be used interchangeably to index the same dictionary entry. ``dict([('foo', 100), ('bar', 200)])``, ``dict(foo=100, bar=200)`` If no positional argument is given, an empty dictionary is created. - If a positional argument is given and it is a mapping object, a dictionary - is created with the same key-value pairs as the mapping object. Otherwise, - the positional argument must be an :term:`iterable` object. Each item in - the iterable must itself be an iterable with exactly two objects. The - first object of each item becomes a key in the new dictionary, and the - second object the corresponding value. If a key occurs more than once, the - last value for that key becomes the corresponding value in the new - dictionary. + If a positional argument is given and it defines a ``keys()`` method, a + dictionary is created by calling :meth:`~object.__getitem__` on the argument with + each returned key from the method. Otherwise, the positional argument must be an + :term:`iterable` object. Each item in the iterable must itself be an iterable + with exactly two elements. The first element of each item becomes a key in the + new dictionary, and the second element the corresponding value. If a key occurs + more than once, the last value for that key becomes the corresponding value in + the new dictionary. If keyword arguments are given, the keyword arguments and their values are added to the dictionary created from the positional argument. If a key @@ -4560,7 +4724,7 @@ can be used interchangeably to index the same dictionary entry. Return a shallow copy of the dictionary. - .. classmethod:: fromkeys(iterable[, value]) + .. classmethod:: fromkeys(iterable, value=None, /) Create a new dictionary with keys from *iterable* and values set to *value*. @@ -4570,7 +4734,7 @@ can be used interchangeably to index the same dictionary entry. such as an empty list. To get distinct values, use a :ref:`dict comprehension ` instead. - .. method:: get(key[, default]) + .. method:: get(key, default=None) Return the value for *key* if *key* is in the dictionary, else *default*. If *default* is not given, it defaults to ``None``, so that this method @@ -4612,7 +4776,7 @@ can be used interchangeably to index the same dictionary entry. .. versionadded:: 3.8 - .. method:: setdefault(key[, default]) + .. method:: setdefault(key, default=None) If *key* is in the dictionary, return its value. If not, insert *key* with a value of *default* and return *default*. *default* defaults to @@ -4623,10 +4787,11 @@ can be used interchangeably to index the same dictionary entry. Update the dictionary with the key/value pairs from *other*, overwriting existing keys. Return ``None``. - :meth:`update` accepts either another dictionary object or an iterable of - key/value pairs (as tuples or other iterables of length two). If keyword - arguments are specified, the dictionary is then updated with those - key/value pairs: ``d.update(red=1, blue=2)``. + :meth:`update` accepts either another object with a ``keys()`` method (in + which case :meth:`~object.__getitem__` is called with every key returned from + the method) or an iterable of key/value pairs (as tuples or other iterables + of length two). If keyword arguments are specified, the dictionary is then + updated with those key/value pairs: ``d.update(red=1, blue=2)``. .. method:: values() @@ -5056,7 +5221,6 @@ list is non-exhaustive. * :class:`collections.abc.MutableMapping` * :class:`collections.abc.Sequence` * :class:`collections.abc.MutableSequence` -* :class:`collections.abc.ByteString` * :class:`collections.abc.MappingView` * :class:`collections.abc.KeysView` * :class:`collections.abc.ItemsView` @@ -5476,22 +5640,6 @@ types, where they are relevant. Some of these are not reported by the :func:`dir` built-in function. -.. attribute:: object.__dict__ - - A dictionary or other mapping object used to store an object's (writable) - attributes. - - -.. attribute:: instance.__class__ - - The class to which a class instance belongs. - - -.. attribute:: class.__bases__ - - The tuple of base classes of a class object. - - .. attribute:: definition.__name__ The name of the class, function, method, descriptor, or @@ -5506,35 +5654,23 @@ types, where they are relevant. Some of these are not reported by the .. versionadded:: 3.3 -.. attribute:: definition.__type_params__ +.. attribute:: definition.__module__ - The :ref:`type parameters ` of generic classes, functions, - and :ref:`type aliases `. - - .. versionadded:: 3.12 + The name of the module in which a class or function was defined. -.. attribute:: class.__mro__ +.. attribute:: definition.__doc__ - This attribute is a tuple of classes that are considered when looking for - base classes during method resolution. + The documentation string of a class or function, or ``None`` if undefined. -.. method:: class.mro() - - This method can be overridden by a metaclass to customize the method - resolution order for its instances. It is called at class instantiation, and - its result is stored in :attr:`~class.__mro__`. - - -.. method:: class.__subclasses__ +.. attribute:: definition.__type_params__ - Each class keeps a list of weak references to its immediate subclasses. This - method returns a list of all those references still alive. The list is in - definition order. Example:: + The :ref:`type parameters ` of generic classes, functions, + and :ref:`type aliases `. For classes and functions that + are not generic, this will be an empty tuple. - >>> int.__subclasses__() - [, , , ] + .. versionadded:: 3.12 .. _int_max_str_digits: @@ -5554,8 +5690,7 @@ a string to a binary integer or a binary integer to a string in linear time, have sub-quadratic complexity. Converting a large value such as ``int('1' * 500_000)`` can take over a second on a fast CPU. -Limiting conversion size offers a practical way to avoid `CVE-2020-10735 -`_. +Limiting conversion size offers a practical way to avoid :cve:`2020-10735`. The limit is applied to the number of digit characters in the input or output string when a non-linear conversion algorithm would be involved. Underscores diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 1867678b2077fc..09165c481b246e 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -1,5 +1,5 @@ -:mod:`string` --- Common string operations -========================================== +:mod:`!string` --- Common string operations +=========================================== .. module:: string :synopsis: Common string operations. @@ -59,11 +59,18 @@ The constants defined in this module are: String of ASCII characters which are considered punctuation characters in the ``C`` locale: ``!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~``. + .. data:: printable - String of ASCII characters which are considered printable. This is a - combination of :const:`digits`, :const:`ascii_letters`, :const:`punctuation`, - and :const:`whitespace`. + String of ASCII characters which are considered printable by Python. + This is a combination of :const:`digits`, :const:`ascii_letters`, + :const:`punctuation`, and :const:`whitespace`. + + .. note:: + + By design, :meth:`string.printable.isprintable() ` + returns :const:`False`. In particular, ``string.printable`` is not + printable in the POSIX sense (see :manpage:`LC_CTYPE `). .. data:: whitespace @@ -350,8 +357,9 @@ The meaning of the various alignment options is as follows: | ``'='`` | Forces the padding to be placed after the sign (if any) | | | but before the digits. This is used for printing fields | | | in the form '+000000120'. This alignment option is only | -| | valid for numeric types. It becomes the default for | -| | numbers when '0' immediately precedes the field width. | +| | valid for numeric types, excluding :class:`complex`. | +| | It becomes the default for numbers when '0' immediately | +| | precedes the field width. | +---------+----------------------------------------------------------+ | ``'^'`` | Forces the field to be centered within the available | | | space. | @@ -408,7 +416,9 @@ conversions, trailing zeros are not removed from the result. .. index:: single: , (comma); in string formatting -The ``','`` option signals the use of a comma for a thousands separator. +The ``','`` option signals the use of a comma for a thousands separator for +floating-point presentation types and for integer presentation type ``'d'``. +For other presentation types, this option is an error. For a locale aware separator, use the ``'n'`` integer presentation type instead. @@ -418,7 +428,7 @@ instead. .. index:: single: _ (underscore); in string formatting The ``'_'`` option signals the use of an underscore for a thousands -separator for floating point presentation types and for integer +separator for floating-point presentation types and for integer presentation type ``'d'``. For integer presentation types ``'b'``, ``'o'``, ``'x'``, and ``'X'``, underscores will be inserted every 4 digits. For other presentation types, specifying this option is an @@ -432,9 +442,9 @@ including any prefixes, separators, and other formatting characters. If not specified, then the field width will be determined by the content. When no explicit alignment is given, preceding the *width* field by a zero -(``'0'``) character enables -sign-aware zero-padding for numeric types. This is equivalent to a *fill* -character of ``'0'`` with an *alignment* type of ``'='``. +(``'0'``) character enables sign-aware zero-padding for numeric types, +excluding :class:`complex`. This is equivalent to a *fill* character of +``'0'`` with an *alignment* type of ``'='``. .. versionchanged:: 3.10 Preceding the *width* field by ``'0'`` no longer affects the default @@ -491,9 +501,9 @@ The available integer presentation types are: +---------+----------------------------------------------------------+ In addition to the above presentation types, integers can be formatted -with the floating point presentation types listed below (except +with the floating-point presentation types listed below (except ``'n'`` and ``None``). When doing so, :func:`float` is used to convert the -integer to a floating point number before formatting. +integer to a floating-point number before formatting. The available presentation types for :class:`float` and :class:`~decimal.Decimal` values are: @@ -509,9 +519,8 @@ The available presentation types for :class:`float` and | | significant digits. With no precision given, uses a | | | precision of ``6`` digits after the decimal point for | | | :class:`float`, and shows all coefficient digits | - | | for :class:`~decimal.Decimal`. If no digits follow the | - | | decimal point, the decimal point is also removed unless | - | | the ``#`` option is used. | + | | for :class:`~decimal.Decimal`. If ``p=0``, the decimal | + | | point is omitted unless the ``#`` option is used. | +---------+----------------------------------------------------------+ | ``'E'`` | Scientific notation. Same as ``'e'`` except it uses | | | an upper case 'E' as the separator character. | @@ -522,9 +531,8 @@ The available presentation types for :class:`float` and | | precision given, uses a precision of ``6`` digits after | | | the decimal point for :class:`float`, and uses a | | | precision large enough to show all coefficient digits | - | | for :class:`~decimal.Decimal`. If no digits follow the | - | | decimal point, the decimal point is also removed unless | - | | the ``#`` option is used. | + | | for :class:`~decimal.Decimal`. If ``p=0``, the decimal | + | | point is omitted unless the ``#`` option is used. | +---------+----------------------------------------------------------+ | ``'F'`` | Fixed-point notation. Same as ``'f'``, but converts | | | ``nan`` to ``NAN`` and ``inf`` to ``INF``. | @@ -574,11 +582,13 @@ The available presentation types for :class:`float` and | ``'%'`` | Percentage. Multiplies the number by 100 and displays | | | in fixed (``'f'``) format, followed by a percent sign. | +---------+----------------------------------------------------------+ - | None | For :class:`float` this is the same as ``'g'``, except | + | None | For :class:`float` this is like the ``'g'`` type, except | | | that when fixed-point notation is used to format the | | | result, it always includes at least one digit past the | - | | decimal point. The precision used is as large as needed | - | | to represent the given value faithfully. | + | | decimal point, and switches to the scientific notation | + | | when ``exp >= p - 1``. When the precision is not | + | | specified, the latter will be as large as needed to | + | | represent the given value faithfully. | | | | | | For :class:`~decimal.Decimal`, this is the same as | | | either ``'g'`` or ``'G'`` depending on the value of | @@ -588,6 +598,20 @@ The available presentation types for :class:`float` and | | as altered by the other format modifiers. | +---------+----------------------------------------------------------+ +The result should be correctly rounded to a given precision ``p`` of digits +after the decimal point. The rounding mode for :class:`float` matches that +of the :func:`round` builtin. For :class:`~decimal.Decimal`, the rounding +mode of the current :ref:`context ` will be used. + +The available presentation types for :class:`complex` are the same as those for +:class:`float` (``'%'`` is not allowed). Both the real and imaginary components +of a complex number are formatted as floating-point numbers, according to the +specified presentation type. They are separated by the mandatory sign of the +imaginary part, the latter being terminated by a ``j`` suffix. If the presentation +type is missing, the result will match the output of :func:`str` (complex numbers with +a non-zero real part are also surrounded by parentheses), possibly altered by +other format modifiers. + .. _formatexamples: diff --git a/Doc/library/stringprep.rst b/Doc/library/stringprep.rst index c6d78a356d97bc..37d5adf0fa9541 100644 --- a/Doc/library/stringprep.rst +++ b/Doc/library/stringprep.rst @@ -1,5 +1,5 @@ -:mod:`stringprep` --- Internet String Preparation -================================================= +:mod:`!stringprep` --- Internet String Preparation +================================================== .. module:: stringprep :synopsis: String preparation, as per RFC 3453 diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 3e507c1c7e7c85..3ea9e5ba071289 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -1,5 +1,5 @@ -:mod:`struct` --- Interpret bytes as packed binary data -======================================================= +:mod:`!struct` --- Interpret bytes as packed binary data +======================================================== .. testsetup:: * @@ -267,21 +267,35 @@ platform-dependent. | ``P`` | :c:expr:`void \*` | integer | | \(5) | +--------+--------------------------+--------------------+----------------+------------+ +Additionally, if IEC 60559 compatible complex arithmetic (Annex G of the +C11 standard) is supported, the following format characters are available: + ++--------+--------------------------+--------------------+----------------+------------+ +| Format | C Type | Python type | Standard size | Notes | ++========+==========================+====================+================+============+ +| ``E`` | :c:expr:`float complex` | complex | 8 | \(10) | ++--------+--------------------------+--------------------+----------------+------------+ +| ``C`` | :c:expr:`double complex` | complex | 16 | \(10) | ++--------+--------------------------+--------------------+----------------+------------+ + .. versionchanged:: 3.3 Added support for the ``'n'`` and ``'N'`` formats. .. versionchanged:: 3.6 Added support for the ``'e'`` format. +.. versionchanged:: 3.14 + Added support for the ``'E'`` and ``'C'`` formats. + Notes: (1) .. index:: single: ? (question mark); in struct format strings - The ``'?'`` conversion code corresponds to the :c:expr:`_Bool` type defined by - C99. If this type is not available, it is simulated using a :c:expr:`char`. In - standard mode, it is always represented by one byte. + The ``'?'`` conversion code corresponds to the :c:expr:`_Bool` type + defined by C standards since C99. In standard mode, it is + represented by one byte. (2) When attempting to pack a non-integer using any of the integer conversion @@ -349,6 +363,11 @@ Notes: of bytes. As a special case, ``'0s'`` means a single, empty string (while ``'0c'`` means 0 characters). +(10) + For the ``'E'`` and ``'C'`` format characters, the packed representation uses + the IEEE 754 binary32 and binary64 format for components of the complex + number, regardless of the floating-point format used by the platform. + A format character may be preceded by an integral repeat count. For example, the format string ``'4h'`` means exactly the same as ``'hhhh'``. diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 1dcfea58a8e89f..05d09e304b32bf 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -1,5 +1,5 @@ -:mod:`subprocess` --- Subprocess management -=========================================== +:mod:`!subprocess` --- Subprocess management +============================================ .. module:: subprocess :synopsis: Subprocess management. @@ -25,7 +25,7 @@ modules and functions can be found in the following sections. :pep:`324` -- PEP proposing the subprocess module -.. include:: ../includes/wasm-notavail.rst +.. include:: ../includes/wasm-mobile-notavail.rst Using the :mod:`subprocess` Module ---------------------------------- @@ -52,10 +52,12 @@ underlying :class:`Popen` interface can be used directly. If *capture_output* is true, stdout and stderr will be captured. When used, the internal :class:`Popen` object is automatically created with - ``stdout=PIPE`` and ``stderr=PIPE``. The *stdout* and *stderr* arguments may - not be supplied at the same time as *capture_output*. If you wish to capture - and combine both streams into one, use ``stdout=PIPE`` and ``stderr=STDOUT`` - instead of *capture_output*. + *stdout* and *stderr* both set to :data:`~subprocess.PIPE`. + The *stdout* and *stderr* arguments may not be supplied at the same time as *capture_output*. + If you wish to capture and combine both streams into one, + set *stdout* to :data:`~subprocess.PIPE` + and *stderr* to :data:`~subprocess.STDOUT`, + instead of using *capture_output*. A *timeout* may be specified in seconds, it is internally passed on to :meth:`Popen.communicate`. If the timeout expires, the child process will be @@ -69,7 +71,8 @@ underlying :class:`Popen` interface can be used directly. subprocess's stdin. If used it must be a byte sequence, or a string if *encoding* or *errors* is specified or *text* is true. When used, the internal :class:`Popen` object is automatically created with - ``stdin=PIPE``, and the *stdin* argument may not be used as well. + *stdin* set to :data:`~subprocess.PIPE`, + and the *stdin* argument may not be used as well. If *check* is true, and the process exits with a non-zero exit code, a :exc:`CalledProcessError` exception will be raised. Attributes of that @@ -605,7 +608,7 @@ functions. If *group* is not ``None``, the setregid() system call will be made in the child process prior to the execution of the subprocess. If the provided - value is a string, it will be looked up via :func:`grp.getgrnam()` and + value is a string, it will be looked up via :func:`grp.getgrnam` and the value in ``gr_gid`` will be used. If the value is an integer, it will be passed verbatim. (POSIX only) @@ -615,7 +618,7 @@ functions. If *extra_groups* is not ``None``, the setgroups() system call will be made in the child process prior to the execution of the subprocess. Strings provided in *extra_groups* will be looked up via - :func:`grp.getgrnam()` and the values in ``gr_gid`` will be used. + :func:`grp.getgrnam` and the values in ``gr_gid`` will be used. Integer values will be passed verbatim. (POSIX only) .. availability:: POSIX @@ -623,7 +626,7 @@ functions. If *user* is not ``None``, the setreuid() system call will be made in the child process prior to the execution of the subprocess. If the provided - value is a string, it will be looked up via :func:`pwd.getpwnam()` and + value is a string, it will be looked up via :func:`pwd.getpwnam` and the value in ``pw_uid`` will be used. If the value is an integer, it will be passed verbatim. (POSIX only) @@ -751,8 +754,8 @@ Exceptions defined in this module all inherit from :exc:`SubprocessError`. Security Considerations ----------------------- -Unlike some other popen functions, this implementation will never -implicitly call a system shell. This means that all characters, +Unlike some other popen functions, this library will not +implicitly choose to call a system shell. This means that all characters, including shell metacharacters, can safely be passed to child processes. If the shell is invoked explicitly, via ``shell=True``, it is the application's responsibility to ensure that all whitespace and metacharacters are @@ -761,6 +764,14 @@ quoted appropriately to avoid vulnerabilities. On :ref:`some platforms `, it is possible to use :func:`shlex.quote` for this escaping. +On Windows, batch files (:file:`*.bat` or :file:`*.cmd`) may be launched by the +operating system in a system shell regardless of the arguments passed to this +library. This could result in arguments being parsed according to shell rules, +but without any escaping added by Python. If you are intentionally launching a +batch file with arguments from untrusted sources, consider passing +``shell=True`` to allow Python to escape special characters. See :gh:`114539` +for additional discussion. + Popen Objects ------------- @@ -1055,6 +1066,22 @@ The :mod:`subprocess` module exposes the following constants. Specifies that the :attr:`STARTUPINFO.wShowWindow` attribute contains additional information. +.. data:: STARTF_FORCEONFEEDBACK + + A :attr:`STARTUPINFO.dwFlags` parameter to specify that the + *Working in Background* mouse cursor will be displayed while a + process is launching. This is the default behavior for GUI + processes. + + .. versionadded:: 3.13 + +.. data:: STARTF_FORCEOFFFEEDBACK + + A :attr:`STARTUPINFO.dwFlags` parameter to specify that the mouse + cursor will not be changed when launching a process. + + .. versionadded:: 3.13 + .. data:: CREATE_NEW_CONSOLE The new process has a new console, instead of inheriting its parent's @@ -1099,7 +1126,7 @@ The :mod:`subprocess` module exposes the following constants. .. data:: NORMAL_PRIORITY_CLASS A :class:`Popen` ``creationflags`` parameter to specify that a new process - will have an normal priority. (default) + will have a normal priority. (default) .. versionadded:: 3.7 @@ -1416,36 +1443,8 @@ Environment example:: -Replacing :func:`os.popen`, :func:`os.popen2`, :func:`os.popen3` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:: - - (child_stdin, child_stdout) = os.popen2(cmd, mode, bufsize) - ==> - p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) - (child_stdin, child_stdout) = (p.stdin, p.stdout) - -:: - - (child_stdin, - child_stdout, - child_stderr) = os.popen3(cmd, mode, bufsize) - ==> - p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) - (child_stdin, - child_stdout, - child_stderr) = (p.stdin, p.stdout, p.stderr) - -:: - - (child_stdin, child_stdout_and_stderr) = os.popen4(cmd, mode, bufsize) - ==> - p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) - (child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout) +Replacing :func:`os.popen` +^^^^^^^^^^^^^^^^^^^^^^^^^^ Return code handling translates as follows:: @@ -1462,44 +1461,6 @@ Return code handling translates as follows:: print("There were some errors") -Replacing functions from the :mod:`!popen2` module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. note:: - - If the cmd argument to popen2 functions is a string, the command is executed - through /bin/sh. If it is a list, the command is directly executed. - -:: - - (child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode) - ==> - p = Popen("somestring", shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) - (child_stdout, child_stdin) = (p.stdout, p.stdin) - -:: - - (child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, mode) - ==> - p = Popen(["mycmd", "myarg"], bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) - (child_stdout, child_stdin) = (p.stdout, p.stdin) - -:class:`popen2.Popen3` and :class:`popen2.Popen4` basically work as -:class:`subprocess.Popen`, except that: - -* :class:`Popen` raises an exception if the execution fails. - -* The *capturestderr* argument is replaced with the *stderr* argument. - -* ``stdin=PIPE`` and ``stdout=PIPE`` must be specified. - -* popen2 closes all file descriptors by default, but you have to specify - ``close_fds=True`` with :class:`Popen` to guarantee this behavior on - all platforms or past Python versions. - - Legacy Shell Invocation Functions --------------------------------- @@ -1600,36 +1561,22 @@ runtime): Module which provides function to parse and escape command lines. -.. _disable_vfork: .. _disable_posix_spawn: -Disabling use of ``vfork()`` or ``posix_spawn()`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Disable use of ``posix_spawn()`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ On Linux, :mod:`subprocess` defaults to using the ``vfork()`` system call internally when it is safe to do so rather than ``fork()``. This greatly improves performance. -If you ever encounter a presumed highly unusual situation where you need to -prevent ``vfork()`` from being used by Python, you can set the -:const:`subprocess._USE_VFORK` attribute to a false value. - -:: - - subprocess._USE_VFORK = False # See CPython issue gh-NNNNNN. - -Setting this has no impact on use of ``posix_spawn()`` which could use -``vfork()`` internally within its libc implementation. There is a similar -:const:`subprocess._USE_POSIX_SPAWN` attribute if you need to prevent use of -that. - :: subprocess._USE_POSIX_SPAWN = False # See CPython issue gh-NNNNNN. -It is safe to set these to false on any Python version. They will have no -effect on older versions when unsupported. Do not assume the attributes are -available to read. Despite their names, a true value does not indicate that the +It is safe to set this to false on any Python version. It will have no +effect on older or newer versions where unsupported. Do not assume the attribute +is available to read. Despite the name, a true value does not indicate the corresponding function will be used, only that it may be. Please file issues any time you have to use these private knobs with a way to @@ -1637,4 +1584,3 @@ reproduce the issue you were seeing. Link to that issue from a comment in your code. .. versionadded:: 3.8 ``_USE_POSIX_SPAWN`` -.. versionadded:: 3.11 ``_USE_VFORK`` diff --git a/Doc/library/sunau.rst b/Doc/library/sunau.rst new file mode 100644 index 00000000000000..feb7768f8bdd68 --- /dev/null +++ b/Doc/library/sunau.rst @@ -0,0 +1,15 @@ +:mod:`!sunau` --- Read and write Sun AU files +============================================= + +.. module:: sunau + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!sunau` module was +`Python 3.12 `_. diff --git a/Doc/library/superseded.rst b/Doc/library/superseded.rst index 17bfa66f043302..d120c6acf621e3 100644 --- a/Doc/library/superseded.rst +++ b/Doc/library/superseded.rst @@ -4,12 +4,23 @@ Superseded Modules ****************** -The modules described in this chapter are deprecated or :term:`soft deprecated` and only kept for -backwards compatibility. They have been superseded by other modules. +The modules described in this chapter have been superseded by other modules +for most use cases, and are retained primarily to preserve backwards compatibility. +Modules may appear in this chapter because they only cover a limited subset of +a problem space, and a more generally applicable solution is available elsewhere +in the standard library (for example, :mod:`getopt` covers the very specific +task of "mimic the C :c:func:`!getopt` API in Python", rather than the broader +command line option parsing and argument parsing capabilities offered by +:mod:`optparse` and :mod:`argparse`). + +Alternatively, modules may appear in this chapter because they are deprecated +outright, and awaiting removal in a future release, or they are +:term:`soft deprecated` and their use is actively discouraged in new projects. +With the removal of various obsolete modules through :pep:`594`, there are +currently no modules in this latter category. .. toctree:: :maxdepth: 1 getopt.rst - optparse.rst diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 47568387f9a7ce..54e19af4bd69a6 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -1,5 +1,5 @@ -:mod:`symtable` --- Access to the compiler's symbol tables -========================================================== +:mod:`!symtable` --- Access to the compiler's symbol tables +=========================================================== .. module:: symtable :synopsis: Interface to the compiler's internal symbol tables. @@ -31,21 +31,74 @@ Generating Symbol Tables Examining Symbol Tables ----------------------- +.. class:: SymbolTableType + + An enumeration indicating the type of a :class:`SymbolTable` object. + + .. attribute:: MODULE + :value: "module" + + Used for the symbol table of a module. + + .. attribute:: FUNCTION + :value: "function" + + Used for the symbol table of a function. + + .. attribute:: CLASS + :value: "class" + + Used for the symbol table of a class. + + The following members refer to different flavors of + :ref:`annotation scopes `. + + .. attribute:: ANNOTATION + :value: "annotation" + + Used for annotations if ``from __future__ import annotations`` is active. + + .. attribute:: TYPE_ALIAS + :value: "type alias" + + Used for the symbol table of :keyword:`type` constructions. + + .. attribute:: TYPE_PARAMETERS + :value: "type parameters" + + Used for the symbol table of :ref:`generic functions ` + or :ref:`generic classes `. + + .. attribute:: TYPE_VARIABLE + :value: "type variable" + + Used for the symbol table of the bound, the constraint tuple or the + default value of a single type variable in the formal sense, i.e., + a TypeVar, a TypeVarTuple or a ParamSpec object (the latter two do + not support a bound or a constraint tuple). + + .. versionadded:: 3.13 + .. class:: SymbolTable A namespace table for a block. The constructor is not public. .. method:: get_type() - Return the type of the symbol table. Possible values are ``'class'``, - ``'module'``, ``'function'``, ``'annotation'``, ``'TypeVar bound'``, - ``'type alias'``, and ``'type parameter'``. The latter four refer to - different flavors of :ref:`annotation scopes `. + Return the type of the symbol table. Possible values are members + of the :class:`SymbolTableType` enumeration. .. versionchanged:: 3.12 Added ``'annotation'``, ``'TypeVar bound'``, ``'type alias'``, and ``'type parameter'`` as possible return values. + .. versionchanged:: 3.13 + Return values are members of the :class:`SymbolTableType` enumeration. + + The exact values of the returned string may change in the future, + and thus, it is recommended to use :class:`SymbolTableType` members + instead of hard-coded strings. + .. method:: get_id() Return the table's identifier. @@ -114,11 +167,12 @@ Examining Symbol Tables .. method:: get_nonlocals() - Return a tuple containing names of nonlocals in this function. + Return a tuple containing names of explicitly declared nonlocals in this function. .. method:: get_frees() - Return a tuple containing names of free variables in this function. + Return a tuple containing names of :term:`free (closure) variables ` + in this function. .. class:: Class @@ -127,7 +181,54 @@ Examining Symbol Tables .. method:: get_methods() - Return a tuple containing the names of methods declared in the class. + Return a tuple containing the names of method-like functions declared + in the class. + + Here, the term 'method' designates *any* function defined in the class + body via :keyword:`def` or :keyword:`async def`. + + Functions defined in a deeper scope (e.g., in an inner class) are not + picked up by :meth:`get_methods`. + + For example: + + .. testsetup:: symtable.Class.get_methods + + import warnings + context = warnings.catch_warnings() + context.__enter__() + warnings.simplefilter("ignore", category=DeprecationWarning) + + .. testcleanup:: symtable.Class.get_methods + + context.__exit__() + + .. doctest:: symtable.Class.get_methods + + >>> import symtable + >>> st = symtable.symtable(''' + ... def outer(): pass + ... + ... class A: + ... def f(): + ... def w(): pass + ... + ... def g(self): pass + ... + ... @classmethod + ... async def h(cls): pass + ... + ... global outer + ... def outer(self): pass + ... ''', 'test', 'exec') + >>> class_A = st.get_children()[2] + >>> class_A.get_methods() + ('f', 'g', 'h') + + Although ``A().f()`` raises :exc:`TypeError` at runtime, ``A.f`` is still + considered as a method-like function. + + .. deprecated-removed:: 3.14 3.16 .. class:: Symbol @@ -151,6 +252,12 @@ Examining Symbol Tables Return ``True`` if the symbol is a parameter. + .. method:: is_type_parameter() + + Return ``True`` if the symbol is a type parameter. + + .. versionadded:: 3.14 + .. method:: is_global() Return ``True`` if the symbol is global. @@ -178,10 +285,42 @@ Examining Symbol Tables Return ``True`` if the symbol is referenced in its block, but not assigned to. + .. method:: is_free_class() + + Return *True* if a class-scoped symbol is free from + the perspective of a method. + + Consider the following example:: + + def f(): + x = 1 # function-scoped + class C: + x = 2 # class-scoped + def method(self): + return x + + In this example, the class-scoped symbol ``x`` is considered to + be free from the perspective of ``C.method``, thereby allowing + the latter to return *1* at runtime and not *2*. + + .. versionadded:: 3.14 + .. method:: is_assigned() Return ``True`` if the symbol is assigned to in its block. + .. method:: is_comp_iter() + + Return ``True`` if the symbol is a comprehension iteration variable. + + .. versionadded:: 3.14 + + .. method:: is_comp_cell() + + Return ``True`` if the symbol is a cell in an inlined comprehension. + + .. versionadded:: 3.14 + .. method:: is_namespace() Return ``True`` if name binding introduces new namespace. diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 4980227c60b21e..cfdcdf2e2df476 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -1,5 +1,5 @@ -:mod:`sys.monitoring` --- Execution event monitoring -==================================================== +:mod:`!sys.monitoring` --- Execution event monitoring +===================================================== .. module:: sys.monitoring :synopsis: Access and control event monitoring @@ -50,16 +50,14 @@ Registering and using tools *tool_id* must be in the range 0 to 5 inclusive. Raises a :exc:`ValueError` if *tool_id* is in use. -.. function:: free_tool_id(tool_id: int, /) -> None +.. function:: clear_tool_id(tool_id: int, /) -> None - Should be called once a tool no longer requires *tool_id*. + Unregister all events and callback functions associated with *tool_id*. -.. note:: +.. function:: free_tool_id(tool_id: int, /) -> None - :func:`free_tool_id` will not disable global or local events associated - with *tool_id*, nor will it unregister any callback functions. This - function is only intended to be used to notify the VM that the - particular *tool_id* is no longer in use. + Should be called once a tool no longer requires *tool_id*. + Will call :func:`clear_tool_id` before releasing *tool_id*. .. function:: get_tool(tool_id: int, /) -> str | None @@ -81,9 +79,17 @@ Events The following events are supported: -.. monitoring-event:: BRANCH +.. monitoring-event:: BRANCH_LEFT + + A conditional branch goes left. - A conditional branch is taken (or not). + It is up to the tool to determine how to present "left" and "right" branches. + There is no guarantee which branch is "left" and which is "right", except + that it will be consistent for the duration of the program. + +.. monitoring-event:: BRANCH_RIGHT + + A conditional branch goes right. .. monitoring-event:: CALL @@ -160,7 +166,7 @@ events, use the expression ``PY_RETURN | PY_START``. .. monitoring-event:: NO_EVENTS - An alias for ``0`` so users can do explict comparisions like:: + An alias for ``0`` so users can do explicit comparisons like:: if get_events(DEBUGGER_ID) == NO_EVENTS: ... @@ -182,9 +188,20 @@ The local events are: * :monitoring-event:`LINE` * :monitoring-event:`INSTRUCTION` * :monitoring-event:`JUMP` -* :monitoring-event:`BRANCH` +* :monitoring-event:`BRANCH_LEFT` +* :monitoring-event:`BRANCH_RIGHT` * :monitoring-event:`STOP_ITERATION` +Deprecated event +'''''''''''''''' + +* ``BRANCH`` + +The ``BRANCH`` event is deprecated in 3.14. +Using :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT` +events will give much better performance as they can be disabled +independently. + Ancillary events '''''''''''''''' @@ -226,6 +243,10 @@ To allow tools to monitor for real exceptions without slowing down generators and coroutines, the :monitoring-event:`STOP_ITERATION` event is provided. :monitoring-event:`STOP_ITERATION` can be locally disabled, unlike :monitoring-event:`RAISE`. +Note that the :monitoring-event:`STOP_ITERATION` event and the :monitoring-event:`RAISE` +event for a :exc:`StopIteration` exception are equivalent, and are treated as interchangeable +when generating events. Implementations will favor :monitoring-event:`STOP_ITERATION` for +performance reasons, but may generate a :monitoring-event:`RAISE` event with a :exc:`StopIteration`. Turning events on and off ------------------------- @@ -255,7 +276,10 @@ No events are active by default. Per code object events '''''''''''''''''''''' -Events can also be controlled on a per code object basis. +Events can also be controlled on a per code object basis. The functions +defined below which accept a :class:`types.CodeType` should be prepared +to accept a look-alike object from functions which are not defined +in Python (see :ref:`c-api-monitoring`). .. function:: get_local_events(tool_id: int, code: CodeType, /) -> int @@ -352,13 +376,11 @@ Different events will provide the callback function with different arguments, as func(code: CodeType, line_number: int) -> DISABLE | Any -* :monitoring-event:`BRANCH` and :monitoring-event:`JUMP`:: +* :monitoring-event:`BRANCH_LEFT`, :monitoring-event:`BRANCH_RIGHT` and :monitoring-event:`JUMP`:: func(code: CodeType, instruction_offset: int, destination_offset: int) -> DISABLE | Any Note that the *destination_offset* is where the code will next execute. - For an untaken branch this will be the offset of the instruction following - the branch. * :monitoring-event:`INSTRUCTION`:: diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 380ba1090b39b3..151fd60532048a 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1,5 +1,5 @@ -:mod:`sys` --- System-specific parameters and functions -======================================================= +:mod:`!sys` --- System-specific parameters and functions +======================================================== .. module:: sys :synopsis: Access system-specific parameters and functions. @@ -8,7 +8,7 @@ This module provides access to some variables used or maintained by the interpreter and to functions that interact strongly with the interpreter. It is -always available. +always available. Unless explicitly noted otherwise, all variables are read-only. .. data:: abiflags @@ -130,27 +130,26 @@ always available. .. data:: base_exec_prefix - Set during Python startup, before ``site.py`` is run, to the same value as - :data:`exec_prefix`. If not running in a - :ref:`virtual environment `, the values will stay the same; if - ``site.py`` finds that a virtual environment is in use, the values of - :data:`prefix` and :data:`exec_prefix` will be changed to point to the - virtual environment, whereas :data:`base_prefix` and - :data:`base_exec_prefix` will remain pointing to the base Python - installation (the one which the virtual environment was created from). + Equivalent to :data:`exec_prefix`, but refering to the base Python installation. + + When running under :ref:`sys-path-init-virtual-environments`, + :data:`exec_prefix` gets overwritten to the virtual environment prefix. + :data:`base_exec_prefix`, conversely, does not change, and always points to + the base Python installation. + Refer to :ref:`sys-path-init-virtual-environments` for more information. .. versionadded:: 3.3 .. data:: base_prefix - Set during Python startup, before ``site.py`` is run, to the same value as - :data:`prefix`. If not running in a :ref:`virtual environment `, the values - will stay the same; if ``site.py`` finds that a virtual environment is in - use, the values of :data:`prefix` and :data:`exec_prefix` will be changed to - point to the virtual environment, whereas :data:`base_prefix` and - :data:`base_exec_prefix` will remain pointing to the base Python - installation (the one which the virtual environment was created from). + Equivalent to :data:`prefix`, but refering to the base Python installation. + + When running under :ref:`virtual environment `, + :data:`prefix` gets overwritten to the virtual environment prefix. + :data:`base_prefix`, conversely, does not change, and always points to + the base Python installation. + Refer to :ref:`sys-path-init-virtual-environments` for more information. .. versionadded:: 3.3 @@ -483,11 +482,19 @@ always available. .. note:: - If a :ref:`virtual environment ` is in effect, this - value will be changed in ``site.py`` to point to the virtual environment. - The value for the Python installation will still be available, via - :data:`base_exec_prefix`. + If a :ref:`virtual environment ` is in effect, this :data:`exec_prefix` + will point to the virtual environment. The value for the Python installation + will still be available, via :data:`base_exec_prefix`. + Refer to :ref:`sys-path-init-virtual-environments` for more information. + + .. versionchanged:: 3.14 + When running under a :ref:`virtual environment `, + :data:`prefix` and :data:`exec_prefix` are now set to the virtual + environment prefix by the :ref:`path initialization `, + instead of :mod:`site`. This means that :data:`prefix` and + :data:`exec_prefix` always point to the virtual environment, even when + :mod:`site` is disabled (:option:`-S`). .. data:: executable @@ -735,11 +742,11 @@ always available. regardless of their size. This function is mainly useful for tracking and debugging memory leaks. Because of the interpreter's internal caches, the result can vary from call to call; you may have to call - :func:`_clear_internal_caches()` and :func:`gc.collect()` to get more + :func:`_clear_internal_caches` and :func:`gc.collect` to get more predictable results. If a Python build or implementation cannot reasonably compute this - information, :func:`getallocatedblocks()` is allowed to return 0 instead. + information, :func:`getallocatedblocks` is allowed to return 0 instead. .. versionadded:: 3.4 @@ -753,7 +760,9 @@ always available. .. function:: getandroidapilevel() - Return the build time API version of Android as an integer. + Return the build-time API level of Android as an integer. This represents the + minimum version of Android this build of Python can run on. For runtime + version information, see :func:`platform.android_ver`. .. availability:: Android. @@ -875,7 +884,7 @@ always available. additional garbage collector overhead if the object is managed by the garbage collector. - See `recursive sizeof recipe `_ + See `recursive sizeof recipe `_ for an example of using :func:`getsizeof` recursively to find the size of containers and all their contents. @@ -918,6 +927,35 @@ always available. It is not guaranteed to exist in all implementations of Python. +.. function:: getobjects(limit[, type]) + + This function only exists if CPython was built using the + specialized configure option :option:`--with-trace-refs`. + It is intended only for debugging garbage-collection issues. + + Return a list of up to *limit* dynamically allocated Python objects. + If *type* is given, only objects of that exact type (not subtypes) + are included. + + Objects from the list are not safe to use. + Specifically, the result will include objects from all interpreters that + share their object allocator state (that is, ones created with + :c:member:`PyInterpreterConfig.use_main_obmalloc` set to 1 + or using :c:func:`Py_NewInterpreter`, and the + :ref:`main interpreter `). + Mixing objects from different interpreters may lead to crashes + or other unexpected behavior. + + .. impl-detail:: + + This function should be used for specialized purposes only. + It is not guaranteed to exist in all implementations of Python. + + .. versionchanged:: 3.14 + + The result may include objects from other interpreters. + + .. function:: getprofile() .. index:: @@ -1197,6 +1235,14 @@ always available. return value of :func:`intern` around to benefit from it. +.. function:: _is_gil_enabled() + + Return :const:`True` if the :term:`GIL` is enabled and :const:`False` if + it is disabled. + + .. versionadded:: 3.13 + + .. function:: is_finalizing() Return :const:`True` if the main Python interpreter is @@ -1264,7 +1310,8 @@ always available. that implement Python's default import semantics. The :meth:`~importlib.abc.MetaPathFinder.find_spec` method is called with at least the absolute name of the module being imported. If the module to be - imported is contained in a package, then the parent package's :attr:`__path__` + imported is contained in a package, then the parent package's + :attr:`~module.__path__` attribute is passed in as a second argument. The method returns a :term:`module spec`, or ``None`` if the module cannot be found. @@ -1367,47 +1414,42 @@ always available. .. data:: platform - This string contains a platform identifier that can be used to append - platform-specific components to :data:`sys.path`, for instance. - - For Unix systems, except on Linux and AIX, this is the lowercased OS name as - returned by ``uname -s`` with the first part of the version as returned by - ``uname -r`` appended, e.g. ``'sunos5'`` or ``'freebsd8'``, *at the time - when Python was built*. Unless you want to test for a specific system - version, it is therefore recommended to use the following idiom:: - - if sys.platform.startswith('freebsd'): - # FreeBSD-specific code here... - elif sys.platform.startswith('linux'): - # Linux-specific code here... - elif sys.platform.startswith('aix'): - # AIX-specific code here... - - For other systems, the values are: + A string containing a platform identifier. Known values are: ================ =========================== System ``platform`` value ================ =========================== AIX ``'aix'`` + Android ``'android'`` Emscripten ``'emscripten'`` + iOS ``'ios'`` Linux ``'linux'`` - WASI ``'wasi'`` + macOS ``'darwin'`` Windows ``'win32'`` Windows/Cygwin ``'cygwin'`` - macOS ``'darwin'`` + WASI ``'wasi'`` ================ =========================== + On Unix systems not listed in the table, the value is the lowercased OS name + as returned by ``uname -s``, with the first part of the version as returned by + ``uname -r`` appended, e.g. ``'sunos5'`` or ``'freebsd8'``, *at the time + when Python was built*. Unless you want to test for a specific system + version, it is therefore recommended to use the following idiom:: + + if sys.platform.startswith('freebsd'): + # FreeBSD-specific code here... + .. versionchanged:: 3.3 On Linux, :data:`sys.platform` doesn't contain the major version anymore. - It is always ``'linux'``, instead of ``'linux2'`` or ``'linux3'``. Since - older Python versions include the version number, it is recommended to - always use the ``startswith`` idiom presented above. + It is always ``'linux'``, instead of ``'linux2'`` or ``'linux3'``. .. versionchanged:: 3.8 On AIX, :data:`sys.platform` doesn't contain the major version anymore. - It is always ``'aix'``, instead of ``'aix5'`` or ``'aix7'``. Since - older Python versions include the version number, it is recommended to - always use the ``startswith`` idiom presented above. + It is always ``'aix'``, instead of ``'aix5'`` or ``'aix7'``. + + .. versionchanged:: 3.13 + On Android, :data:`sys.platform` now returns ``'android'`` rather than + ``'linux'``. .. seealso:: @@ -1448,10 +1490,21 @@ always available. argument to the :program:`configure` script. See :ref:`installation_paths` for derived paths. - .. note:: If a :ref:`virtual environment ` is in effect, this - value will be changed in ``site.py`` to point to the virtual - environment. The value for the Python installation will still be - available, via :data:`base_prefix`. + .. note:: + + If a :ref:`virtual environment ` is in effect, this :data:`prefix` + will point to the virtual environment. The value for the Python installation + will still be available, via :data:`base_prefix`. + Refer to :ref:`sys-path-init-virtual-environments` for more information. + + .. versionchanged:: 3.14 + + When running under a :ref:`virtual environment `, + :data:`prefix` and :data:`exec_prefix` are now set to the virtual + environment prefix by the :ref:`path initialization `, + instead of :mod:`site`. This means that :data:`prefix` and + :data:`exec_prefix` always point to the virtual environment, even when + :mod:`site` is disabled (:option:`-S`). .. data:: ps1 @@ -1703,11 +1756,11 @@ always available. contain a tuple of (filename, line number, function name) tuples describing the traceback where the coroutine object was created, with the most recent call first. When disabled, ``cr_origin`` will - be None. + be ``None``. To enable, pass a *depth* value greater than zero; this sets the number of frames whose information will be captured. To disable, - pass set *depth* to zero. + set *depth* to zero. This setting is thread-specific. @@ -1722,6 +1775,8 @@ always available. Activate the stack profiler trampoline *backend*. The only supported backend is ``"perf"``. + Stack trampolines cannot be activated if the JIT is active. + .. availability:: Linux. .. versionadded:: 3.12 diff --git a/Doc/library/sys_path_init.rst b/Doc/library/sys_path_init.rst index a87a41cf829fa8..18fe32d9c7f10a 100644 --- a/Doc/library/sys_path_init.rst +++ b/Doc/library/sys_path_init.rst @@ -47,8 +47,15 @@ however on other platforms :file:`lib/python{majorversion}.{minorversion}/lib-dy ``exec_prefix``. On some platforms :file:`lib` may be :file:`lib64` or another value, see :data:`sys.platlibdir` and :envvar:`PYTHONPLATLIBDIR`. -Once found, ``prefix`` and ``exec_prefix`` are available at :data:`sys.prefix` and -:data:`sys.exec_prefix` respectively. +Once found, ``prefix`` and ``exec_prefix`` are available at +:data:`sys.base_prefix` and :data:`sys.base_exec_prefix` respectively. + +If :envvar:`PYTHONHOME` is not set, and a ``pyvenv.cfg`` file is found alongside +the main executable, or in its parent directory, :data:`sys.prefix` and +:data:`sys.exec_prefix` get set to the directory containing ``pyvenv.cfg``, +otherwise they are set to the same value as :data:`sys.base_prefix` and +:data:`sys.base_exec_prefix`, respectively. +This is used by :ref:`sys-path-init-virtual-environments`. Finally, the :mod:`site` module is processed and :file:`site-packages` directories are added to the module search path. A common way to customize the search path is @@ -60,18 +67,40 @@ the :mod:`site` module documentation. Certain command line options may further affect path calculations. See :option:`-E`, :option:`-I`, :option:`-s` and :option:`-S` for further details. -Virtual environments +.. versionchanged:: 3.14 + + :data:`sys.prefix` and :data:`sys.exec_prefix` are now set to the + ``pyvenv.cfg`` directory during the path initialization. This was previously + done by :mod:`site`, therefore affected by :option:`-S`. + +.. _sys-path-init-virtual-environments: + +Virtual Environments -------------------- -If Python is run in a virtual environment (as described at :ref:`tut-venv`) -then ``prefix`` and ``exec_prefix`` are specific to the virtual environment. +Virtual environments place a ``pyvenv.cfg`` file in their prefix, which causes +:data:`sys.prefix` and :data:`sys.exec_prefix` to point to them, instead of the +base installation. + +The ``prefix`` and ``exec_prefix`` values of the base installation are available +at :data:`sys.base_prefix` and :data:`sys.base_exec_prefix`. -If a ``pyvenv.cfg`` file is found alongside the main executable, or in the -directory one level above the executable, the following variations apply: +As well as being used as a marker to identify virtual environments, +``pyvenv.cfg`` may also be used to configure the :mod:`site` initialization. +Please refer to :mod:`site`'s +:ref:`virtual environments documentation `. + +.. note:: + + :envvar:`PYTHONHOME` overrides the ``pyvenv.cfg`` detection. + +.. note:: -* If ``home`` is an absolute path and :envvar:`PYTHONHOME` is not set, this - path is used instead of the path to the main executable when deducing ``prefix`` - and ``exec_prefix``. + There are other ways how "virtual environments" could be implemented, this + documentation referes implementations based on the ``pyvenv.cfg`` mechanism, + such as :mod:`venv`. Most virtual environment implementations follow the + model set by :mod:`venv`, but there may be exotic implementations that + diverge from it. _pth files ---------- diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 2faab212e46eff..9f018f9c8f0e50 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -1,5 +1,5 @@ -:mod:`sysconfig` --- Provide access to Python's configuration information -========================================================================= +:mod:`!sysconfig` --- Provide access to Python's configuration information +========================================================================== .. module:: sysconfig :synopsis: Python's configuration information @@ -305,7 +305,7 @@ Installation path functions mix with those by the other. End users should not use this function, but :func:`get_default_scheme` and - :func:`get_preferred_scheme()` instead. + :func:`get_preferred_scheme` instead. .. versionadded:: 3.10 @@ -376,7 +376,7 @@ Other functions This is used mainly to distinguish platform-specific build directories and platform-specific built distributions. Typically includes the OS name and - version and the architecture (as supplied by 'os.uname()'), although the + version and the architecture (as supplied by :func:`os.uname`), although the exact information included depends on the OS; e.g., on Linux, the kernel version isn't particularly important. @@ -388,7 +388,8 @@ Other functions Windows will return one of: - - win-amd64 (64bit Windows on AMD64, aka x86_64, Intel64, and EM64T) + - win-amd64 (64-bit Windows on AMD64, aka x86_64, Intel64, and EM64T) + - win-arm64 (64-bit Windows on ARM64, aka AArch64) - win32 (all others - specifically, sys.platform is returned) macOS can return: diff --git a/Doc/library/syslog.rst b/Doc/library/syslog.rst index 7b27fc7e85b62d..548898a37bc6ea 100644 --- a/Doc/library/syslog.rst +++ b/Doc/library/syslog.rst @@ -1,5 +1,5 @@ -:mod:`syslog` --- Unix syslog library routines -============================================== +:mod:`!syslog` --- Unix syslog library routines +=============================================== .. module:: syslog :platform: Unix @@ -11,7 +11,7 @@ This module provides an interface to the Unix ``syslog`` library routines. Refer to the Unix manual pages for a detailed description of the ``syslog`` facility. -.. availability:: Unix, not Emscripten, not WASI. +.. availability:: Unix, not WASI, not iOS. This module wraps the system ``syslog`` family of routines. A pure Python library that can speak to a syslog server is available in the diff --git a/Doc/library/tabnanny.rst b/Doc/library/tabnanny.rst index dfe688a2f93e0c..4f61b3dd761400 100644 --- a/Doc/library/tabnanny.rst +++ b/Doc/library/tabnanny.rst @@ -1,5 +1,5 @@ -:mod:`tabnanny` --- Detection of ambiguous indentation -====================================================== +:mod:`!tabnanny` --- Detection of ambiguous indentation +======================================================= .. module:: tabnanny :synopsis: Tool for detecting white space related problems in Python diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 2134293a0bb0de..c9d69cf5094095 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -1,5 +1,5 @@ -:mod:`tarfile` --- Read and write tar archive files -=================================================== +:mod:`!tarfile` --- Read and write tar archive files +==================================================== .. module:: tarfile :synopsis: Read and write tar-format archive files. @@ -40,9 +40,12 @@ Some facts and figures: Archives are extracted using a :ref:`filter `, which makes it possible to either limit surprising/dangerous features, or to acknowledge that they are expected and the archive is fully trusted. - By default, archives are fully trusted, but this default is deprecated - and slated to change in Python 3.14. +.. versionchanged:: 3.14 + Set the default extraction filter to :func:`data `, + which disallows some dangerous features such as links to absolute paths + or paths outside of the destination. Previously, the filter strategy + was equivalent to :func:`fully_trusted `. .. function:: open(name=None, mode='r', fileobj=None, bufsize=10240, **kwargs) @@ -495,18 +498,18 @@ be finalized; only the internally used file object will be closed. See the The *filter* argument specifies how ``members`` are modified or rejected before extraction. See :ref:`tarfile-extraction-filter` for details. - It is recommended to set this explicitly depending on which *tar* features - you need to support. + It is recommended to set this explicitly only if specific *tar* features + are required, or as ``filter='data'`` to support Python versions with a less + secure default (3.13 and lower). .. warning:: Never extract archives from untrusted sources without prior inspection. - It is possible that files are created outside of *path*, e.g. members - that have absolute filenames starting with ``"/"`` or filenames with two - dots ``".."``. - Set ``filter='data'`` to prevent the most dangerous security issues, - and read the :ref:`tarfile-extraction-filter` section for details. + Since Python 3.14, the default (:func:`data `) will prevent + the most dangerous security issues. + However, it will not prevent *all* unintended or insecure behavior. + Read the :ref:`tarfile-extraction-filter` section for details. .. versionchanged:: 3.5 Added the *numeric_owner* parameter. @@ -517,6 +520,9 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.12 Added the *filter* parameter. + .. versionchanged:: 3.14 + The *filter* parameter now defaults to ``'data'``. + .. method:: TarFile.extract(member, path="", set_attrs=True, *, numeric_owner=False, filter=None) @@ -536,10 +542,8 @@ be finalized; only the internally used file object will be closed. See the .. warning:: - See the warning for :meth:`extractall`. - - Set ``filter='data'`` to prevent the most dangerous security issues, - and read the :ref:`tarfile-extraction-filter` section for details. + Never extract archives from untrusted sources without prior inspection. + See the warning for :meth:`extractall` for details. .. versionchanged:: 3.2 Added the *set_attrs* parameter. @@ -565,6 +569,10 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.3 Return an :class:`io.BufferedReader` object. + .. versionchanged:: 3.13 + The returned :class:`io.BufferedReader` object has the :attr:`!mode` + attribute which is always equal to ``'rb'``. + .. attribute:: TarFile.errorlevel :type: int @@ -598,14 +606,8 @@ be finalized; only the internally used file object will be closed. See the String names are not allowed for this attribute, unlike the *filter* argument to :meth:`~TarFile.extract`. - If ``extraction_filter`` is ``None`` (the default), - calling an extraction method without a *filter* argument will raise a - ``DeprecationWarning``, - and fall back to the :func:`fully_trusted ` filter, - whose dangerous behavior matches previous versions of Python. - - In Python 3.14+, leaving ``extraction_filter=None`` will cause - extraction methods to use the :func:`data ` filter by default. + If ``extraction_filter`` is ``None`` (the default), extraction methods + will use the :func:`data ` filter by default. The attribute may be set on instances or overridden in subclasses. It also is possible to set it on the ``TarFile`` class itself to set a @@ -613,7 +615,15 @@ be finalized; only the internally used file object will be closed. See the it is best practice to only do so in top-level applications or :mod:`site configuration `. To set a global default this way, a filter function needs to be wrapped in - :func:`staticmethod()` to prevent injection of a ``self`` argument. + :func:`staticmethod` to prevent injection of a ``self`` argument. + + .. versionchanged:: 3.14 + + The default filter is set to :func:`data `, + which disallows some dangerous features such as links to absolute paths + or paths outside of the destination. + Previously, the default was equivalent to + :func:`fully_trusted `. .. method:: TarFile.add(name, arcname=None, recursive=True, *, filter=None) @@ -637,11 +647,15 @@ be finalized; only the internally used file object will be closed. See the .. method:: TarFile.addfile(tarinfo, fileobj=None) - Add the :class:`TarInfo` object *tarinfo* to the archive. If *fileobj* is given, - it should be a :term:`binary file`, and - ``tarinfo.size`` bytes are read from it and added to the archive. You can + Add the :class:`TarInfo` object *tarinfo* to the archive. If *tarinfo* represents + a non zero-size regular file, the *fileobj* argument should be a :term:`binary file`, + and ``tarinfo.size`` bytes are read from it and added to the archive. You can create :class:`TarInfo` objects directly, or by using :meth:`gettarinfo`. + .. versionchanged:: 3.13 + + *fileobj* must be given for non-zero-sized regular files. + .. method:: TarFile.gettarinfo(name=None, arcname=None, fileobj=None) @@ -961,6 +975,12 @@ In most cases, the full functionality is not needed. Therefore, *tarfile* supports extraction filters: a mechanism to limit functionality, and thus mitigate some of the security issues. +.. warning:: + + None of the available filters blocks *all* dangerous archive features. + Never extract archives from untrusted sources without prior inspection. + See also :ref:`tarfile-further-verification`. + .. seealso:: :pep:`706` @@ -984,12 +1004,13 @@ can be: * ``None`` (default): Use :attr:`TarFile.extraction_filter`. - If that is also ``None`` (the default), raise a ``DeprecationWarning``, - and fall back to the ``'fully_trusted'`` filter, whose dangerous behavior - matches previous versions of Python. + If that is also ``None`` (the default), the ``'data'`` filter will be used. + + .. versionchanged:: 3.14 - In Python 3.14, the ``'data'`` filter will become the default instead. - It's possible to switch earlier; see :attr:`TarFile.extraction_filter`. + The default filter is set to :func:`data `. + Previously, the default was equivalent to + :func:`fully_trusted `. * A callable which will be called for each extracted member with a :ref:`TarInfo ` describing the member and the destination @@ -1072,6 +1093,9 @@ reused in custom filters: Return the modified ``TarInfo`` member. + Note that this filter does not block *all* dangerous archive features. + See :ref:`tarfile-further-verification` for details. + .. _tarfile-extraction-refuse: @@ -1085,6 +1109,8 @@ With ``errorlevel=0`` the error will be logged and the member will be skipped, but extraction will continue. +.. _tarfile-further-verification: + Hints for further verification ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1102,9 +1128,10 @@ Here is an incomplete list of things to consider: disk, memory and CPU usage. * Check filenames against an allow-list of characters (to filter out control characters, confusables, foreign path separators, - etc.). + and so on). * Check that filenames have expected extensions (discouraging files that - execute when you “click on them”, or extension-less files like Windows special device names). + execute when you “click on them”, or extension-less files like Windows + special device names). * Limit the number of extracted files, total size of extracted data, filename length (including symlink length), and size of individual files. * Check for files that would be shadowed on case-insensitive filesystems. diff --git a/Doc/library/telnetlib.rst b/Doc/library/telnetlib.rst new file mode 100644 index 00000000000000..6971ad33ff9751 --- /dev/null +++ b/Doc/library/telnetlib.rst @@ -0,0 +1,19 @@ +:mod:`!telnetlib` --- Telnet client +=================================== + +.. module:: telnetlib + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +Possible replacements are third-party libraries from PyPI: :pypi:`telnetlib3` +or :pypi:`Exscript`. These are not supported or maintained by the Python core +team. + +The last version of Python that provided the :mod:`!telnetlib` module was +`Python 3.12 `_. diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index 9add8500c7788c..f0a81a093b435b 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -1,5 +1,5 @@ -:mod:`tempfile` --- Generate temporary files and directories -============================================================ +:mod:`!tempfile` --- Generate temporary files and directories +============================================================= .. module:: tempfile :synopsis: Generate temporary files and directories. diff --git a/Doc/library/termios.rst b/Doc/library/termios.rst index 57705ddc4e6470..0c6f3059fe71d1 100644 --- a/Doc/library/termios.rst +++ b/Doc/library/termios.rst @@ -1,5 +1,5 @@ -:mod:`termios` --- POSIX style tty control -========================================== +:mod:`!termios` --- POSIX style tty control +=========================================== .. module:: termios :platform: Unix diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 7d28f625345726..b5b6e442e218fd 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1,5 +1,5 @@ -:mod:`test` --- Regression tests package for Python -=================================================== +:mod:`!test` --- Regression tests package for Python +==================================================== .. module:: test :synopsis: Regression tests package containing the testing suite for Python. @@ -192,6 +192,10 @@ top-level directory where Python was built. On Windows, executing :program:`rt.bat` from your :file:`PCbuild` directory will run all regression tests. +.. versionadded:: 3.14 + Output is colorized by default and can be + :ref:`controlled using environment variables `. + :mod:`test.support` --- Utilities for the Python test suite =========================================================== @@ -324,7 +328,7 @@ The :mod:`test.support` module defines the following constants: .. data:: Py_DEBUG - True if Python was built with the :c:macro:`Py_DEBUG` macro + ``True`` if Python was built with the :c:macro:`Py_DEBUG` macro defined, that is, if Python was :ref:`built in debug mode `. @@ -731,6 +735,12 @@ The :mod:`test.support` module defines the following functions: macOS version is less than the minimum, the test is skipped. +.. decorator:: requires_gil_enabled + + Decorator for skipping tests on the free-threaded build. If the + :term:`GIL` is disabled, the test is skipped. + + .. decorator:: requires_IEEE_754 Decorator for skipping tests on non-IEEE 754 platforms. @@ -940,7 +950,7 @@ The :mod:`test.support` module defines the following functions: other modules, possibly a C backend (like ``csv`` and its ``_csv``). The *extra* argument can be a set of names that wouldn't otherwise be automatically - detected as "public", like objects without a proper ``__module__`` + detected as "public", like objects without a proper :attr:`~definition.__module__` attribute. If provided, it will be added to the automatically detected ones. The *not_exported* argument can be a set of names that must not be treated @@ -1695,7 +1705,7 @@ The :mod:`test.support.warnings_helper` module provides support for warnings tes .. function:: check_warnings(*filters, quiet=True) - A convenience wrapper for :func:`warnings.catch_warnings()` that makes it + A convenience wrapper for :func:`warnings.catch_warnings` that makes it easier to test that a warning was correctly raised. It is approximately equivalent to calling ``warnings.catch_warnings(record=True)`` with :meth:`warnings.simplefilter` set to ``always`` and with the option to diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index 7445410f91808c..a58b460fef409c 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -1,5 +1,5 @@ -:mod:`textwrap` --- Text wrapping and filling -============================================= +:mod:`!textwrap` --- Text wrapping and filling +============================================== .. module:: textwrap :synopsis: Text wrapping and filling @@ -154,7 +154,7 @@ hyphenated words; only then will long words be broken if necessary, unless wrapper = TextWrapper() wrapper.initial_indent = "* " - You can re-use the same :class:`TextWrapper` object many times, and you can + You can reuse the same :class:`TextWrapper` object many times, and you can change any of its options through direct assignment to instance attributes between uses. diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index cb511471579b69..00511df32e4388 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -1,5 +1,5 @@ -:mod:`threading` --- Thread-based parallelism -============================================= +:mod:`!threading` --- Thread-based parallelism +============================================== .. module:: threading :synopsis: Thread-based parallelism. @@ -363,12 +363,12 @@ since it is impossible to detect the termination of alien threads. base class constructor (``Thread.__init__()``) before doing anything else to the thread. + .. versionchanged:: 3.3 + Added the *daemon* parameter. + .. versionchanged:: 3.10 Use the *target* name if *name* argument is omitted. - .. versionchanged:: 3.3 - Added the *daemon* argument. - .. method:: start() Start the thread's activity. @@ -380,6 +380,13 @@ since it is impossible to detect the termination of alien threads. This method will raise a :exc:`RuntimeError` if called more than once on the same thread object. + If supported, set the operating system thread name to + :attr:`threading.Thread.name`. The name can be truncated depending on the + operating system thread name limits. + + .. versionchanged:: 3.14 + Set the operating system thread name. + .. method:: run() Method representing the thread's activity. @@ -412,7 +419,7 @@ since it is impossible to detect the termination of alien threads. timeout occurs. When the *timeout* argument is present and not ``None``, it should be a - floating point number specifying a timeout for the operation in seconds + floating-point number specifying a timeout for the operation in seconds (or fractions thereof). As :meth:`~Thread.join` always returns ``None``, you must call :meth:`~Thread.is_alive` after :meth:`~Thread.join` to decide whether a timeout happened -- if the thread is still alive, the @@ -434,6 +441,15 @@ since it is impossible to detect the termination of alien threads. Multiple threads may be given the same name. The initial name is set by the constructor. + On some platforms, the thread name is set at the operating system level + when the thread starts, so that it is visible in task managers. + This name may be truncated to fit in a system-specific limit (for example, + 15 bytes on Linux or 63 bytes on macOS). + + Changes to *name* are only reflected at the OS level when the currently + running thread is renamed. (Setting the *name* attribute of a + different thread only updates the Python Thread object.) + .. method:: getName() setName() @@ -567,6 +583,9 @@ All methods are executed atomically. Lock acquisition can now be interrupted by signals on POSIX if the underlying threading implementation supports it. + .. versionchanged:: 3.14 + Lock acquisition can now be interrupted by signals on Windows. + .. method:: release() @@ -598,14 +617,25 @@ and "recursion level" in addition to the locked/unlocked state used by primitive locks. In the locked state, some thread owns the lock; in the unlocked state, no thread owns it. -To lock the lock, a thread calls its :meth:`~RLock.acquire` method; this -returns once the thread owns the lock. To unlock the lock, a thread calls -its :meth:`~Lock.release` method. :meth:`~Lock.acquire`/:meth:`~Lock.release` -call pairs may be nested; only the final :meth:`~Lock.release` (the -:meth:`~Lock.release` of the outermost pair) resets the lock to unlocked and -allows another thread blocked in :meth:`~Lock.acquire` to proceed. +Threads call a lock's :meth:`~RLock.acquire` method to lock it, +and its :meth:`~Lock.release` method to unlock it. + +.. note:: + + Reentrant locks support the :ref:`context management protocol `, + so it is recommended to use :keyword:`with` instead of manually calling + :meth:`~RLock.acquire` and :meth:`~RLock.release` + to handle acquiring and releasing the lock for a block of code. -Reentrant locks also support the :ref:`context management protocol `. +RLock's :meth:`~RLock.acquire`/:meth:`~RLock.release` call pairs may be nested, +unlike Lock's :meth:`~Lock.acquire`/:meth:`~Lock.release`. Only the final +:meth:`~RLock.release` (the :meth:`~Lock.release` of the outermost pair) resets +the lock to an unlocked state and allows another thread blocked in +:meth:`~RLock.acquire` to proceed. + +:meth:`~RLock.acquire`/:meth:`~RLock.release` must be used in pairs: each acquire +must have a release in the thread that has acquired the lock. Failing to +call release as many times the lock has been acquired can lead to deadlock. .. class:: RLock() @@ -624,25 +654,41 @@ Reentrant locks also support the :ref:`context management protocol ` Acquire a lock, blocking or non-blocking. - When invoked without arguments: if this thread already owns the lock, increment - the recursion level by one, and return immediately. Otherwise, if another - thread owns the lock, block until the lock is unlocked. Once the lock is - unlocked (not owned by any thread), then grab ownership, set the recursion level - to one, and return. If more than one thread is blocked waiting until the lock - is unlocked, only one at a time will be able to grab ownership of the lock. - There is no return value in this case. + .. seealso:: - When invoked with the *blocking* argument set to ``True``, do the same thing as when - called without arguments, and return ``True``. + :ref:`Using RLock as a context manager ` + Recommended over manual :meth:`!acquire` and :meth:`release` calls + whenever practical. - When invoked with the *blocking* argument set to ``False``, do not block. If a call - without an argument would block, return ``False`` immediately; otherwise, do the - same thing as when called without arguments, and return ``True``. - When invoked with the floating-point *timeout* argument set to a positive - value, block for at most the number of seconds specified by *timeout* - and as long as the lock cannot be acquired. Return ``True`` if the lock has - been acquired, ``False`` if the timeout has elapsed. + When invoked with the *blocking* argument set to ``True`` (the default): + + * If no thread owns the lock, acquire the lock and return immediately. + + * If another thread owns the lock, block until we are able to acquire + lock, or *timeout*, if set to a positive float value. + + * If the same thread owns the lock, acquire the lock again, and + return immediately. This is the difference between :class:`Lock` and + :class:`!RLock`; :class:`Lock` handles this case the same as the previous, + blocking until the lock can be acquired. + + When invoked with the *blocking* argument set to ``False``: + + * If no thread owns the lock, acquire the lock and return immediately. + + * If another thread owns the lock, return immediately. + + * If the same thread owns the lock, acquire the lock again and return + immediately. + + In all cases, if the thread was able to acquire the lock, return ``True``. + If the thread was unable to acquire the lock (i.e. if not blocking or + the timeout was reached) return ``False``. + + If called multiple times, failing to call :meth:`~RLock.release` as many times + may lead to deadlock. Consider using :class:`!RLock` as a context manager rather than + calling acquire/release directly. .. versionchanged:: 3.2 The *timeout* parameter is new. @@ -658,7 +704,7 @@ Reentrant locks also support the :ref:`context management protocol ` Only call this method when the calling thread owns the lock. A :exc:`RuntimeError` is raised if this method is called when the lock is - unlocked. + not acquired. There is no return value. @@ -767,7 +813,7 @@ item to the buffer only needs to wake up one consumer thread. occurs. Once awakened or timed out, it re-acquires the lock and returns. When the *timeout* argument is present and not ``None``, it should be a - floating point number specifying a timeout for the operation in seconds + floating-point number specifying a timeout for the operation in seconds (or fractions thereof). When the underlying lock is an :class:`RLock`, it is not released using @@ -991,10 +1037,10 @@ method. The :meth:`~Event.wait` method blocks until the flag is true. has not expired. The return value represents the reason that this blocking method returned; ``True`` if returning because the internal flag is set to true, or ``False`` if a timeout is given and - the the internal flag did not become true within the given wait time. + the internal flag did not become true within the given wait time. When the timeout argument is present and not ``None``, it should be a - floating point number specifying a timeout for the operation in seconds, + floating-point number specifying a timeout for the operation in seconds, or fractions thereof. .. versionchanged:: 3.1 diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 88c15f19bedf8c..804e2679027bd4 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -1,5 +1,5 @@ -:mod:`time` --- Time access and conversions -=========================================== +:mod:`!time` --- Time access and conversions +============================================ .. module:: time :synopsis: Time access and conversions. @@ -69,7 +69,7 @@ An explanation of some terminology and conventions is in order. systems, the clock "ticks" only 50 or 100 times a second. * On the other hand, the precision of :func:`.time` and :func:`sleep` is better - than their Unix equivalents: times are expressed as floating point numbers, + than their Unix equivalents: times are expressed as floating-point numbers, :func:`.time` returns the most accurate time available (using Unix :c:func:`!gettimeofday` where available), and :func:`sleep` will accept a time with a nonzero fraction (Unix :c:func:`!select` is used to implement this, where @@ -193,7 +193,7 @@ Functions Use :func:`clock_settime_ns` to avoid the precision loss caused by the :class:`float` type. - .. availability:: Unix. + .. availability:: Unix, not Android, not iOS. .. versionadded:: 3.3 @@ -202,7 +202,7 @@ Functions Similar to :func:`clock_settime` but set time with nanoseconds. - .. availability:: Unix. + .. availability:: Unix, not Android, not iOS. .. versionadded:: 3.7 @@ -273,7 +273,7 @@ Functions This is the inverse function of :func:`localtime`. Its argument is the :class:`struct_time` or full 9-tuple (since the dst flag is needed; use ``-1`` as the dst flag if it is unknown) which expresses the time in *local* time, not - UTC. It returns a floating point number, for compatibility with :func:`.time`. + UTC. It returns a floating-point number, for compatibility with :func:`.time`. If the input value cannot be represented as a valid time, either :exc:`OverflowError` or :exc:`ValueError` will be raised (which depends on whether the invalid value is caught by Python or the underlying C libraries). @@ -287,6 +287,15 @@ Functions The reference point of the returned value is undefined, so that only the difference between the results of two calls is valid. + Clock: + + * On Windows, call ``QueryPerformanceCounter()`` and + ``QueryPerformanceFrequency()``. + * On macOS, call ``mach_absolute_time()`` and ``mach_timebase_info()``. + * On HP-UX, call ``gethrtime()``. + * Call ``clock_gettime(CLOCK_HIGHRES)`` if available. + * Otherwise, call ``clock_gettime(CLOCK_MONOTONIC)``. + Use :func:`monotonic_ns` to avoid the precision loss caused by the :class:`float` type. @@ -316,6 +325,11 @@ Functions point of the returned value is undefined, so that only the difference between the results of two calls is valid. + .. impl-detail:: + + On CPython, use the same clock as :func:`time.monotonic` and is a + monotonic clock, i.e. a clock that cannot go backwards. + Use :func:`perf_counter_ns` to avoid the precision loss caused by the :class:`float` type. @@ -324,6 +338,10 @@ Functions .. versionchanged:: 3.10 On Windows, the function is now system-wide. + .. versionchanged:: 3.13 + Use the same clock as :func:`time.monotonic`. + + .. function:: perf_counter_ns() -> int Similar to :func:`perf_counter`, but return time as nanoseconds. @@ -358,7 +376,7 @@ Functions .. function:: sleep(secs) Suspend execution of the calling thread for the given number of seconds. - The argument may be a floating point number to indicate a more precise sleep + The argument may be a floating-point number to indicate a more precise sleep time. If the sleep is interrupted by a signal and no exception is raised by the @@ -367,31 +385,39 @@ Functions The suspension time may be longer than requested by an arbitrary amount, because of the scheduling of other activity in the system. + .. rubric:: Windows implementation + On Windows, if *secs* is zero, the thread relinquishes the remainder of its time slice to any other thread that is ready to run. If there are no other threads ready to run, the function returns immediately, and the thread continues execution. On Windows 8.1 and newer the implementation uses a `high-resolution timer - `_ + `_ which provides resolution of 100 nanoseconds. If *secs* is zero, ``Sleep(0)`` is used. - Unix implementation: + .. rubric:: Unix implementation * Use ``clock_nanosleep()`` if available (resolution: 1 nanosecond); * Or use ``nanosleep()`` if available (resolution: 1 nanosecond); * Or use ``select()`` (resolution: 1 microsecond). - .. audit-event:: time.sleep secs + .. note:: - .. versionchanged:: 3.11 - On Unix, the ``clock_nanosleep()`` and ``nanosleep()`` functions are now - used if available. On Windows, a waitable timer is now used. + To emulate a "no-op", use :keyword:`pass` instead of ``time.sleep(0)``. + + To voluntarily relinquish the CPU, specify a real-time :ref:`scheduling + policy ` and use :func:`os.sched_yield` instead. + + .. audit-event:: time.sleep secs .. versionchanged:: 3.5 The function now sleeps at least *secs* even if the sleep is interrupted by a signal, except if the signal handler raises an exception (see :pep:`475` for the rationale). + .. versionchanged:: 3.11 + On Unix, the ``clock_nanosleep()`` and ``nanosleep()`` functions are now + used if available. On Windows, a waitable timer is now used. .. versionchanged:: 3.13 Raises an auditing event. @@ -466,6 +492,9 @@ Functions | | | | | | | | +-----------+------------------------------------------------+-------+ + | ``%u`` | Day of the week (Monday is 1; Sunday is 7) | | + | | as a decimal number [1, 7]. | | + +-----------+------------------------------------------------+-------+ | ``%w`` | Weekday as a decimal number [0(Sunday),6]. | | | | | | +-----------+------------------------------------------------+-------+ @@ -498,6 +527,16 @@ Functions | ``%Z`` | Time zone name (no characters if no time zone | | | | exists). Deprecated. [1]_ | | +-----------+------------------------------------------------+-------+ + | ``%G`` | ISO 8601 year (similar to ``%Y`` but follows | | + | | the rules for the ISO 8601 calendar year). | | + | | The year starts with the week that contains | | + | | the first Thursday of the calendar year. | | + +-----------+------------------------------------------------+-------+ + | ``%V`` | ISO 8601 week number (as a decimal number | | + | | [01,53]). The first week of the year is the | | + | | one that contains the first Thursday of the | | + | | year. Weeks start on Monday. | | + +-----------+------------------------------------------------+-------+ | ``%%`` | A literal ``'%'`` character. | | +-----------+------------------------------------------------+-------+ @@ -600,7 +639,7 @@ Functions - range [1, 12] * - 2 - - .. attribute:: tm_day + - .. attribute:: tm_mday - range [1, 31] * - 3 @@ -648,13 +687,13 @@ Functions .. function:: time() -> float - Return the time in seconds since the epoch_ as a floating point + Return the time in seconds since the epoch_ as a floating-point number. The handling of `leap seconds`_ is platform dependent. On Windows and most Unix systems, the leap seconds are not counted towards the time in seconds since the epoch_. This is commonly referred to as `Unix time `_. - Note that even though the time is always returned as a floating point + Note that even though the time is always returned as a floating-point number, not all systems provide time with a better precision than 1 second. While this function normally returns non-decreasing values, it can return a lower value than a previous call if the system clock has been set back @@ -667,6 +706,12 @@ Functions :class:`struct_time` object is returned, from which the components of the calendar date may be accessed as attributes. + Clock: + + * On Windows, call ``GetSystemTimeAsFileTime()``. + * Call ``clock_gettime(CLOCK_REALTIME)`` if available. + * Otherwise, call ``gettimeofday()``. + Use :func:`time_ns` to avoid the precision loss caused by the :class:`float` type. diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 616f8365b80f6c..548a3ee0540506 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -1,5 +1,5 @@ -:mod:`timeit` --- Measure execution time of small code snippets -=============================================================== +:mod:`!timeit` --- Measure execution time of small code snippets +================================================================ .. module:: timeit :synopsis: Measure the execution time of small code snippets. diff --git a/Doc/library/tkinter.colorchooser.rst b/Doc/library/tkinter.colorchooser.rst index 6e8479c1dea1e2..df2b324fd5d3a7 100644 --- a/Doc/library/tkinter.colorchooser.rst +++ b/Doc/library/tkinter.colorchooser.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.colorchooser` --- Color choosing dialog -===================================================== +:mod:`!tkinter.colorchooser` --- Color choosing dialog +====================================================== .. module:: tkinter.colorchooser :platform: Tk diff --git a/Doc/library/tkinter.dnd.rst b/Doc/library/tkinter.dnd.rst index 02de0fd331958d..62298d96c26459 100644 --- a/Doc/library/tkinter.dnd.rst +++ b/Doc/library/tkinter.dnd.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.dnd` --- Drag and drop support -============================================ +:mod:`!tkinter.dnd` --- Drag and drop support +============================================= .. module:: tkinter.dnd :platform: Tk @@ -25,8 +25,8 @@ Selection of a target object occurs as follows: #. Top-down search of area under mouse for target widget * Target widget should have a callable *dnd_accept* attribute - * If *dnd_accept* is not present or returns None, search moves to parent widget - * If no target widget is found, then the target object is None + * If *dnd_accept* is not present or returns ``None``, search moves to parent widget + * If no target widget is found, then the target object is ``None`` 2. Call to *.dnd_leave(source, event)* #. Call to *.dnd_enter(source, event)* diff --git a/Doc/library/tkinter.font.rst b/Doc/library/tkinter.font.rst index c7c2b7b566cf8f..ed01bd5f483943 100644 --- a/Doc/library/tkinter.font.rst +++ b/Doc/library/tkinter.font.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.font` --- Tkinter font wrapper -============================================ +:mod:`!tkinter.font` --- Tkinter font wrapper +============================================= .. module:: tkinter.font :platform: Tk diff --git a/Doc/library/tkinter.messagebox.rst b/Doc/library/tkinter.messagebox.rst index 56090a0a0e424b..0dc9632ca73304 100644 --- a/Doc/library/tkinter.messagebox.rst +++ b/Doc/library/tkinter.messagebox.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.messagebox` --- Tkinter message prompts -===================================================== +:mod:`!tkinter.messagebox` --- Tkinter message prompts +====================================================== .. module:: tkinter.messagebox :platform: Tk diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index e084d8554c7c09..f284988daf2d4e 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -1,5 +1,5 @@ -:mod:`tkinter` --- Python interface to Tcl/Tk -============================================= +:mod:`!tkinter` --- Python interface to Tcl/Tk +============================================== .. module:: tkinter :synopsis: Interface to Tcl/Tk for graphical user interfaces @@ -58,7 +58,7 @@ details that are unchanged. * `Modern Tkinter for Busy Python Developers `_ By Mark Roseman. (ISBN 978-1999149567) - * `Python GUI programming with Tkinter `_ + * `Python GUI programming with Tkinter `_ By Alan D. Moore. (ISBN 978-1788835886) * `Programming Python `_ @@ -979,6 +979,15 @@ of :class:`tkinter.Image`: Either type of image is created through either the ``file`` or the ``data`` option (other options are available as well). +.. versionchanged:: 3.13 + Added the :class:`!PhotoImage` method :meth:`!copy_replace` to copy a region + from one image to other image, possibly with pixel zooming and/or + subsampling. + Add *from_coords* parameter to :class:`!PhotoImage` methods :meth:`!copy`, + :meth:`!zoom` and :meth:`!subsample`. + Add *zoom* and *subsample* parameters to :class:`!PhotoImage` method + :meth:`!copy`. + The image object can then be used wherever an ``image`` option is supported by some widget (e.g. labels, buttons, menus). In these cases, Tk will not keep a reference to the image. When the last Python reference to the image object is diff --git a/Doc/library/tkinter.scrolledtext.rst b/Doc/library/tkinter.scrolledtext.rst index d20365baa38690..763e24929d74b5 100644 --- a/Doc/library/tkinter.scrolledtext.rst +++ b/Doc/library/tkinter.scrolledtext.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.scrolledtext` --- Scrolled Text Widget -==================================================== +:mod:`!tkinter.scrolledtext` --- Scrolled Text Widget +===================================================== .. module:: tkinter.scrolledtext :platform: Tk diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index bd0d8b3799a0f1..628e9f945ac365 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -1,5 +1,5 @@ -:mod:`tkinter.ttk` --- Tk themed widgets -======================================== +:mod:`!tkinter.ttk` --- Tk themed widgets +========================================= .. module:: tkinter.ttk :synopsis: Tk themed widget set diff --git a/Doc/library/token.rst b/Doc/library/token.rst index e6dc37d7ad852c..40982f32b4beee 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -1,5 +1,5 @@ -:mod:`token` --- Constants used with Python parse trees -======================================================= +:mod:`!token` --- Constants used with Python parse trees +======================================================== .. module:: token :synopsis: Constants representing terminal nodes of the parse tree. @@ -75,10 +75,17 @@ the :mod:`tokenize` module. :noindex: Token value indicating that a type comment was recognized. Such - tokens are only produced when :func:`ast.parse()` is invoked with + tokens are only produced when :func:`ast.parse` is invoked with ``type_comments=True``. +.. data:: EXACT_TOKEN_TYPES + + A dictionary mapping the string representation of a token to its numeric code. + + .. versionadded:: 3.8 + + .. versionchanged:: 3.5 Added :data:`!AWAIT` and :data:`!ASYNC` tokens. diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst index 92bdb052267a68..b80917eae66f8b 100644 --- a/Doc/library/tokenize.rst +++ b/Doc/library/tokenize.rst @@ -1,5 +1,5 @@ -:mod:`tokenize` --- Tokenizer for Python source -=============================================== +:mod:`!tokenize` --- Tokenizer for Python source +================================================ .. module:: tokenize :synopsis: Lexical scanner for Python source code. @@ -91,11 +91,10 @@ write back the modified script. sequences with at least two elements, the token type and the token string. Any additional sequence elements are ignored. - The reconstructed script is returned as a single string. The result is - guaranteed to tokenize back to match the input so that the conversion is - lossless and round-trips are assured. The guarantee applies only to the - token type and token string as the spacing between tokens (column - positions) may change. + The result is guaranteed to tokenize back to match the input so that the + conversion is lossless and round-trips are assured. The guarantee applies + only to the token type and token string as the spacing between tokens + (column positions) may change. It returns bytes, encoded using the :data:`~token.ENCODING` token, which is the first token sequence output by :func:`.tokenize`. If there is no diff --git a/Doc/library/tomllib.rst b/Doc/library/tomllib.rst index f9e2dfeb13dc87..30d7ff50a1acc1 100644 --- a/Doc/library/tomllib.rst +++ b/Doc/library/tomllib.rst @@ -1,5 +1,5 @@ -:mod:`tomllib` --- Parse TOML files -=================================== +:mod:`!tomllib` --- Parse TOML files +==================================== .. module:: tomllib :synopsis: Parse TOML files. @@ -13,20 +13,20 @@ -------------- -This module provides an interface for parsing TOML (Tom's Obvious Minimal +This module provides an interface for parsing TOML 1.0.0 (Tom's Obvious Minimal Language, `https://toml.io `_). This module does not support writing TOML. .. seealso:: - The `Tomli-W package `__ + The :pypi:`Tomli-W package ` is a TOML writer that can be used in conjunction with this module, providing a write API familiar to users of the standard library :mod:`marshal` and :mod:`pickle` modules. .. seealso:: - The `TOML Kit package `__ + The :pypi:`TOML Kit package ` is a style-preserving TOML library with both read and write capability. It is a recommended replacement for this module for editing already existing TOML files. @@ -60,9 +60,36 @@ This module defines the following functions: The following exceptions are available: -.. exception:: TOMLDecodeError +.. exception:: TOMLDecodeError(msg, doc, pos) - Subclass of :exc:`ValueError`. + Subclass of :exc:`ValueError` with the following additional attributes: + + .. attribute:: msg + + The unformatted error message. + + .. attribute:: doc + + The TOML document being parsed. + + .. attribute:: pos + + The index of *doc* where parsing failed. + + .. attribute:: lineno + + The line corresponding to *pos*. + + .. attribute:: colno + + The column corresponding to *pos*. + + .. versionchanged:: 3.14 + Added the *msg*, *doc* and *pos* parameters. + Added the :attr:`msg`, :attr:`doc`, :attr:`pos`, :attr:`lineno` and :attr:`colno` attributes. + + .. deprecated:: 3.14 + Passing free-form positional arguments is deprecated. Examples diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index 8854905e192b45..cae94ea08e17e5 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -1,5 +1,5 @@ -:mod:`trace` --- Trace or track Python statement execution -========================================================== +:mod:`!trace` --- Trace or track Python statement execution +=========================================================== .. module:: trace :synopsis: Trace or track Python statement execution. diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index ab83e0df10b709..b0ee3fc56ad735 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -1,5 +1,5 @@ -:mod:`traceback` --- Print or retrieve a stack traceback -======================================================== +:mod:`!traceback` --- Print or retrieve a stack traceback +========================================================= .. module:: traceback :synopsis: Print or retrieve a stack traceback. @@ -8,11 +8,15 @@ -------------- -This module provides a standard interface to extract, format and print stack -traces of Python programs. It exactly mimics the behavior of the Python -interpreter when it prints a stack trace. This is useful when you want to print -stack traces under program control, such as in a "wrapper" around the -interpreter. +This module provides a standard interface to extract, format and print +stack traces of Python programs. It is more flexible than the +interpreter's default traceback display, and therefore makes it +possible to configure certain aspects of the output. Finally, +it contains a utility for capturing enough information about an +exception to print it later, without the need to save a reference +to the actual exception. Since exceptions can be the roots of large +objects graph, this utility can significantly improve +memory management. .. index:: pair: object; traceback @@ -29,7 +33,24 @@ which are assigned to the :attr:`~BaseException.__traceback__` field of Module :mod:`pdb` Interactive source code debugger for Python programs. -The module defines the following functions: +The module's API can be divided into two parts: + +* Module-level functions offering basic functionality, which are useful for interactive + inspection of exceptions and tracebacks. + +* :class:`TracebackException` class and its helper classes + :class:`StackSummary` and :class:`FrameSummary`. These offer both more + flexibility in the output generated and the ability to store the information + necessary for later formatting without holding references to actual exception + and traceback objects. + +.. versionadded:: 3.13 + Output is colorized by default and can be + :ref:`controlled using environment variables `. + + +Module-Level Functions +---------------------- .. function:: print_tb(tb, limit=None, file=None) @@ -42,6 +63,14 @@ The module defines the following functions: :term:`file ` or :term:`file-like object` to receive the output. + .. note:: + + The meaning of the *limit* parameter is different than the meaning + of :const:`sys.tracebacklimit`. A negative *limit* value corresponds to + a positive value of :const:`!sys.tracebacklimit`, whereas the behaviour of + a positive *limit* value cannot be achieved with + :const:`!sys.tracebacklimit`. + .. versionchanged:: 3.5 Added negative *limit* support. @@ -132,6 +161,13 @@ The module defines the following functions: arguments have the same meaning as for :func:`print_stack`. +.. function:: print_list(extracted_list, file=None) + + Print the list of tuples as returned by :func:`extract_tb` or + :func:`extract_stack` as a formatted stack trace to the given file. + If *file* is ``None``, the output is written to :data:`sys.stderr`. + + .. function:: format_list(extracted_list) Given a list of tuples or :class:`FrameSummary` objects as returned by @@ -229,7 +265,6 @@ The module defines the following functions: .. versionadded:: 3.5 -The module also defines the following classes: :class:`!TracebackException` Objects ------------------------------------ @@ -237,12 +272,17 @@ The module also defines the following classes: .. versionadded:: 3.5 :class:`!TracebackException` objects are created from actual exceptions to -capture data for later printing in a lightweight fashion. +capture data for later printing. They offer a more lightweight method of +storing this information by avoiding holding references to +:ref:`traceback` and :ref:`frame` objects. +In addition, they expose more options to configure the output compared to +the module-level functions described above. .. class:: TracebackException(exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, max_group_width=15, max_group_depth=10) - Capture an exception for later rendering. *limit*, *lookup_lines* and - *capture_locals* are as for the :class:`StackSummary` class. + Capture an exception for later rendering. The meaning of *limit*, + *lookup_lines* and *capture_locals* are as for the :class:`StackSummary` + class. If *compact* is true, only data that is required by :class:`!TracebackException`'s :meth:`format` method @@ -473,7 +513,7 @@ in a :ref:`traceback `. attribute accessed (which also happens when casting it to a :class:`tuple`). :attr:`~FrameSummary.line` may be directly provided, and will prevent line lookups happening at all. *locals* is an optional local variable - dictionary, and if supplied the variable representations are stored in the + mapping, and if supplied the variable representations are stored in the summary for later display. :class:`!FrameSummary` instances have the following attributes: @@ -501,8 +541,8 @@ in a :ref:`traceback `. .. _traceback-example: -Traceback Examples ------------------- +Examples of Using the Module-Level Functions +-------------------------------------------- This simple example implements a basic read-eval-print loop, similar to (but less useful than) the standard Python interactive interpreter loop. For a more @@ -541,8 +581,7 @@ exception and traceback: try: lumberjack() - except IndexError: - exc = sys.exception() + except IndexError as exc: print("*** print_tb:") traceback.print_tb(exc.__traceback__, limit=1, file=sys.stdout) print("*** print_exception:") @@ -645,5 +684,88 @@ This last example demonstrates the final few formatting functions: [' File "spam.py", line 3, in \n spam.eggs()\n', ' File "eggs.py", line 42, in eggs\n return "bacon"\n'] >>> an_error = IndexError('tuple index out of range') - >>> traceback.format_exception_only(type(an_error), an_error) + >>> traceback.format_exception_only(an_error) ['IndexError: tuple index out of range\n'] + + +Examples of Using :class:`TracebackException` +--------------------------------------------- + +With the helper class, we have more options:: + + >>> import sys + >>> from traceback import TracebackException + >>> + >>> def lumberjack(): + ... bright_side_of_life() + ... + >>> def bright_side_of_life(): + ... t = "bright", "side", "of", "life" + ... return t[5] + ... + >>> try: + ... lumberjack() + ... except IndexError as e: + ... exc = e + ... + >>> try: + ... try: + ... lumberjack() + ... except: + ... 1/0 + ... except Exception as e: + ... chained_exc = e + ... + >>> # limit works as with the module-level functions + >>> TracebackException.from_exception(exc, limit=-2).print() + Traceback (most recent call last): + File "", line 6, in lumberjack + bright_side_of_life() + ~~~~~~~~~~~~~~~~~~~^^ + File "", line 10, in bright_side_of_life + return t[5] + ~^^^ + IndexError: tuple index out of range + + >>> # capture_locals adds local variables in frames + >>> TracebackException.from_exception(exc, limit=-2, capture_locals=True).print() + Traceback (most recent call last): + File "", line 6, in lumberjack + bright_side_of_life() + ~~~~~~~~~~~~~~~~~~~^^ + File "", line 10, in bright_side_of_life + return t[5] + ~^^^ + t = ("bright", "side", "of", "life") + IndexError: tuple index out of range + + >>> # The *chain* kwarg to print() controls whether chained + >>> # exceptions are displayed + >>> TracebackException.from_exception(chained_exc).print() + Traceback (most recent call last): + File "", line 4, in + lumberjack() + ~~~~~~~~~~^^ + File "", line 7, in lumberjack + bright_side_of_life() + ~~~~~~~~~~~~~~~~~~~^^ + File "", line 11, in bright_side_of_life + return t[5] + ~^^^ + IndexError: tuple index out of range + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "", line 6, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + + >>> TracebackException.from_exception(chained_exc).print(chain=False) + Traceback (most recent call last): + File "", line 6, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst index 68432aeaecbcc1..2370d927292eb0 100644 --- a/Doc/library/tracemalloc.rst +++ b/Doc/library/tracemalloc.rst @@ -1,5 +1,5 @@ -:mod:`tracemalloc` --- Trace memory allocations -=============================================== +:mod:`!tracemalloc` --- Trace memory allocations +================================================ .. module:: tracemalloc :synopsis: Trace memory allocations. diff --git a/Doc/library/tty.rst b/Doc/library/tty.rst index ed63561c40de24..37778bf20bdcc7 100644 --- a/Doc/library/tty.rst +++ b/Doc/library/tty.rst @@ -1,5 +1,5 @@ -:mod:`tty` --- Terminal control functions -========================================= +:mod:`!tty` --- Terminal control functions +========================================== .. module:: tty :platform: Unix @@ -53,7 +53,7 @@ The :mod:`tty` module defines the following functions: is saved before setting *fd* to raw mode; this value is returned. .. versionchanged:: 3.12 - The return value is now the original tty attributes, instead of None. + The return value is now the original tty attributes, instead of ``None``. .. function:: setcbreak(fd, when=termios.TCSAFLUSH) @@ -67,7 +67,7 @@ The :mod:`tty` module defines the following functions: the minimum input to 1 byte with no delay. .. versionchanged:: 3.12 - The return value is now the original tty attributes, instead of None. + The return value is now the original tty attributes, instead of ``None``. .. versionchanged:: 3.12.2 The ``ICRNL`` flag is no longer cleared. This restores the behavior diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 88b1f09eb3c8b7..98dfa0f53bc6a5 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -14,6 +14,11 @@ from turtle import * turtle = Turtle() +.. testcleanup:: + + import os + os.remove("my_drawing.ps") + -------------- Introduction @@ -120,7 +125,7 @@ off-screen):: home() The home position is at the center of the turtle's screen. If you ever need to -know them, get the turtle's x-y co-ordinates with:: +know them, get the turtle's x-y coordinates with:: pos() @@ -208,6 +213,31 @@ useful when working with learners for whom typing is not a skill. use turtle graphics with a learner. +Automatically begin and end filling +----------------------------------- + +Starting with Python 3.14, you can use the :func:`fill` :term:`context manager` +instead of :func:`begin_fill` and :func:`end_fill` to automatically begin and +end fill. Here is an example:: + + with fill(): + for i in range(4): + forward(100) + right(90) + + forward(200) + +The code above is equivalent to:: + + begin_fill() + for i in range(4): + forward(100) + right(90) + end_fill() + + forward(200) + + Use the ``turtle`` module namespace ----------------------------------- @@ -346,6 +376,7 @@ Pen control Filling | :func:`filling` + | :func:`fill` | :func:`begin_fill` | :func:`end_fill` @@ -376,6 +407,7 @@ Using events | :func:`ondrag` Special Turtle methods + | :func:`poly` | :func:`begin_poly` | :func:`end_poly` | :func:`get_poly` @@ -398,6 +430,7 @@ Window control | :func:`setworldcoordinates` Animation control + | :func:`no_animation` | :func:`delay` | :func:`tracer` | :func:`update` @@ -427,6 +460,7 @@ Input methods Methods specific to Screen | :func:`bye` | :func:`exitonclick` + | :func:`save` | :func:`setup` | :func:`title` @@ -606,7 +640,7 @@ Turtle motion >>> turtle.pos() (20.00,30.00) - .. versionadded: 3.12 + .. versionadded:: 3.12 .. function:: setx(x) @@ -987,8 +1021,8 @@ Settings for measurement >>> turtle.heading() 90.0 - Change angle measurement unit to grad (also known as gon, - grade, or gradian and equals 1/100-th of the right angle.) + >>> # Change angle measurement unit to grad (also known as gon, + >>> # grade, or gradian and equals 1/100-th of the right angle.) >>> turtle.degrees(400.0) >>> turtle.heading() 100.0 @@ -1269,6 +1303,29 @@ Filling ... else: ... turtle.pensize(3) +.. function:: fill() + + Fill the shape drawn in the ``with turtle.fill():`` block. + + .. doctest:: + :skipif: _tkinter is None + + >>> turtle.color("black", "red") + >>> with turtle.fill(): + ... turtle.circle(80) + + Using :func:`!fill` is equivalent to adding the :func:`begin_fill` before the + fill-block and :func:`end_fill` after the fill-block: + + .. doctest:: + :skipif: _tkinter is None + + >>> turtle.color("black", "red") + >>> turtle.begin_fill() + >>> turtle.circle(80) + >>> turtle.end_fill() + + .. versionadded:: next .. function:: begin_fill() @@ -1642,6 +1699,23 @@ Using events Special Turtle methods ---------------------- + +.. function:: poly() + + Record the vertices of a polygon drawn in the ``with turtle.poly():`` block. + The first and last vertices will be connected. + + .. doctest:: + :skipif: _tkinter is None + + >>> with turtle.poly(): + ... turtle.forward(100) + ... turtle.right(60) + ... turtle.forward(100) + + .. versionadded:: next + + .. function:: begin_poly() Start recording the vertices of a polygon. Current turtle position is first @@ -1817,7 +1891,8 @@ Window control .. function:: bgpic(picname=None) - :param picname: a string, name of a gif-file or ``"nopic"``, or ``None`` + :param picname: a string, name of an image file (PNG, GIF, PGM, and PPM) + or ``"nopic"``, or ``None`` Set background image or return name of current backgroundimage. If *picname* is a filename, set the corresponding image as background. If *picname* is @@ -1919,6 +1994,23 @@ Window control Animation control ----------------- +.. function:: no_animation() + + Temporarily disable turtle animation. The code written inside the + ``no_animation`` block will not be animated; + once the code block is exited, the drawing will appear. + + .. doctest:: + :skipif: _tkinter is None + + >>> with screen.no_animation(): + ... for dist in range(2, 400, 2): + ... fd(dist) + ... rt(90) + + .. versionadded:: next + + .. function:: delay(delay=None) :param delay: positive integer @@ -2194,9 +2286,9 @@ Settings and special methods .. function:: register_shape(name, shape=None) addshape(name, shape=None) - There are three different ways to call this function: + There are four different ways to call this function: - (1) *name* is the name of a gif-file and *shape* is ``None``: Install the + (1) *name* is the name of an image file (PNG, GIF, PGM, and PPM) and *shape* is ``None``: Install the corresponding image shape. :: >>> screen.register_shape("turtle.gif") @@ -2205,7 +2297,16 @@ Settings and special methods Image shapes *do not* rotate when turning the turtle, so they do not display the heading of the turtle! - (2) *name* is an arbitrary string and *shape* is a tuple of pairs of + (2) *name* is an arbitrary string and *shape* is the name of an image file (PNG, GIF, PGM, and PPM): Install the + corresponding image shape. :: + + >>> screen.register_shape("turtle", "turtle.gif") + + .. note:: + Image shapes *do not* rotate when turning the turtle, so they do not + display the heading of the turtle! + + (3) *name* is an arbitrary string and *shape* is a tuple of pairs of coordinates: Install the corresponding polygon shape. .. doctest:: @@ -2213,12 +2314,16 @@ Settings and special methods >>> screen.register_shape("triangle", ((5,-3), (0,5), (-5,-3))) - (3) *name* is an arbitrary string and *shape* is a (compound) :class:`Shape` + (4) *name* is an arbitrary string and *shape* is a (compound) :class:`Shape` object: Install the corresponding compound shape. Add a turtle shape to TurtleScreen's shapelist. Only thusly registered shapes can be used by issuing the command ``shape(shapename)``. + .. versionchanged:: 3.14 + Added support for PNG, PGM, and PPM image formats. + Both a shape name and an image file name can be specified. + .. function:: turtles() @@ -2269,6 +2374,24 @@ Methods specific to Screen, not inherited from TurtleScreen client script. +.. function:: save(filename, overwrite=False) + + Save the current turtle drawing (and turtles) as a PostScript file. + + :param filename: the path of the saved PostScript file + :param overwrite: if ``False`` and there already exists a file with the given + filename, then the function will raise a + ``FileExistsError``. If it is ``True``, the file will be + overwritten. + + .. doctest:: + :skipif: _tkinter is None + + >>> screen.save("my_drawing.ps") + >>> screen.save("my_drawing.ps", overwrite=True) + + .. versionadded:: 3.14 + .. function:: setup(width=_CFG["width"], height=_CFG["height"], startx=_CFG["leftright"], starty=_CFG["topbottom"]) Set the size and position of the main window. Default values of arguments @@ -2754,9 +2877,6 @@ Changes since Python 3.0 :func:`Screen.numinput `. These pop up input dialogs and return strings and numbers respectively. -- Two example scripts :file:`tdemo_nim.py` and :file:`tdemo_round_dance.py` - have been added to the :file:`Lib/turtledemo` directory. - .. doctest:: :skipif: _tkinter is None diff --git a/Doc/library/types.rst b/Doc/library/types.rst index b856544e44207c..439e119461f798 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -1,5 +1,5 @@ -:mod:`types` --- Dynamic type creation and names for built-in types -=================================================================== +:mod:`!types` --- Dynamic type creation and names for built-in types +==================================================================== .. module:: types :synopsis: Names for built-in types. @@ -91,8 +91,8 @@ Dynamic Type Creation For classes that have an ``__orig_bases__`` attribute, this function returns the value of ``cls.__orig_bases__``. - For classes without the ``__orig_bases__`` attribute, ``cls.__bases__`` is - returned. + For classes without the ``__orig_bases__`` attribute, + :attr:`cls.__bases__ ` is returned. Examples:: @@ -199,7 +199,7 @@ Standard names are defined for the following types: .. data:: CellType The type for cell objects: such objects are used as containers for - a function's free variables. + a function's :term:`closure variables `. .. versionadded:: 3.8 @@ -260,63 +260,18 @@ Standard names are defined for the following types: The type of :term:`modules `. The constructor takes the name of the module to be created and optionally its :term:`docstring`. - .. note:: - Use :func:`importlib.util.module_from_spec` to create a new module if you - wish to set the various import-controlled attributes. - - .. attribute:: __doc__ - - The :term:`docstring` of the module. Defaults to ``None``. - - .. attribute:: __loader__ - - The :term:`loader` which loaded the module. Defaults to ``None``. - - This attribute is to match :attr:`importlib.machinery.ModuleSpec.loader` - as stored in the :attr:`__spec__` object. - - .. note:: - A future version of Python may stop setting this attribute by default. - To guard against this potential change, preferably read from the - :attr:`__spec__` attribute instead or use - ``getattr(module, "__loader__", None)`` if you explicitly need to use - this attribute. - - .. versionchanged:: 3.4 - Defaults to ``None``. Previously the attribute was optional. - - .. attribute:: __name__ - - The name of the module. Expected to match - :attr:`importlib.machinery.ModuleSpec.name`. - - .. attribute:: __package__ - - Which :term:`package` a module belongs to. If the module is top-level - (i.e. not a part of any specific package) then the attribute should be set - to ``''``, else it should be set to the name of the package (which can be - :attr:`__name__` if the module is a package itself). Defaults to ``None``. - - This attribute is to match :attr:`importlib.machinery.ModuleSpec.parent` - as stored in the :attr:`__spec__` object. - - .. note:: - A future version of Python may stop setting this attribute by default. - To guard against this potential change, preferably read from the - :attr:`__spec__` attribute instead or use - ``getattr(module, "__package__", None)`` if you explicitly need to use - this attribute. - - .. versionchanged:: 3.4 - Defaults to ``None``. Previously the attribute was optional. - - .. attribute:: __spec__ - - A record of the module's import-system-related state. Expected to be an - instance of :class:`importlib.machinery.ModuleSpec`. + .. seealso:: - .. versionadded:: 3.4 + :ref:`Documentation on module objects ` + Provides details on the special attributes that can be found on + instances of :class:`!ModuleType`. + :func:`importlib.util.module_from_spec` + Modules created using the :class:`!ModuleType` constructor are + created with many of their special attributes unset or set to default + values. :func:`!module_from_spec` provides a more robust way of + creating :class:`!ModuleType` instances which ensures the various + attributes are set appropriately. .. data:: EllipsisType @@ -392,7 +347,7 @@ Standard names are defined for the following types: In addition, when a class is defined with a :attr:`~object.__slots__` attribute, then for each slot, an instance of :class:`!MemberDescriptorType` will be added as an attribute - on the class. This allows the slot to appear in the class's :attr:`~object.__dict__`. + on the class. This allows the slot to appear in the class's :attr:`~type.__dict__`. .. impl-detail:: @@ -481,14 +436,25 @@ Additional Utility Classes and Functions A simple :class:`object` subclass that provides attribute access to its namespace, as well as a meaningful repr. - Unlike :class:`object`, with ``SimpleNamespace`` you can add and remove - attributes. If a ``SimpleNamespace`` object is initialized with keyword - arguments, those are directly added to the underlying namespace. + Unlike :class:`object`, with :class:`!SimpleNamespace` you can add and remove + attributes. + + :py:class:`SimpleNamespace` objects may be initialized + in the same way as :class:`dict`: either with keyword arguments, + with a single positional argument, or with both. + When initialized with keyword arguments, + those are directly added to the underlying namespace. + Alternatively, when initialized with a positional argument, + the underlying namespace will be updated with key-value pairs + from that argument (either a mapping object or + an :term:`iterable` object producing key-value pairs). + All such keys must be strings. The type is roughly equivalent to the following code:: class SimpleNamespace: - def __init__(self, /, **kwargs): + def __init__(self, mapping_or_iterable=(), /, **kwargs): + self.__dict__.update(mapping_or_iterable) self.__dict__.update(kwargs) def __repr__(self): @@ -512,6 +478,9 @@ Additional Utility Classes and Functions Attribute order in the repr changed from alphabetical to insertion (like ``dict``). + .. versionchanged:: 3.13 + Added support for an optional positional argument. + .. function:: DynamicClassAttribute(fget=None, fset=None, fdel=None, doc=None) Route attribute access on a class to __getattr__. diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 63bd62d1f6679b..0fee782121b0af 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -23,27 +23,26 @@ -------------- -This module provides runtime support for type hints. For the original -specification of the typing system, see :pep:`484`. For a simplified -introduction to type hints, see :pep:`483`. +This module provides runtime support for type hints. +Consider the function below:: -The function below takes and returns a string and is annotated as follows:: + def surface_area_of_cube(edge_length: float) -> str: + return f"The surface area of the cube is {6 * edge_length ** 2}." - def greeting(name: str) -> str: - return 'Hello ' + name +The function ``surface_area_of_cube`` takes an argument expected to +be an instance of :class:`float`, as indicated by the :term:`type hint` +``edge_length: float``. The function is expected to return an instance +of :class:`str`, as indicated by the ``-> str`` hint. -In the function ``greeting``, the argument ``name`` is expected to be of type -:class:`str` and the return type :class:`str`. Subtypes are accepted as -arguments. +While type hints can be simple classes like :class:`float` or :class:`str`, +they can also be more complex. The :mod:`typing` module provides a vocabulary of +more advanced type hints. New features are frequently added to the ``typing`` module. -The `typing_extensions `_ package +The :pypi:`typing_extensions` package provides backports of these new features to older versions of Python. -For a summary of deprecated features and a deprecation timeline, please see -`Deprecation Timeline of Major Features`_. - .. seealso:: `"Typing cheat sheet" `_ @@ -61,67 +60,11 @@ For a summary of deprecated features and a deprecation timeline, please see .. _relevant-peps: -Relevant PEPs -============= - -Since the initial introduction of type hints in :pep:`484` and :pep:`483`, a -number of PEPs have modified and enhanced Python's framework for type -annotations: - -.. raw:: html - -
- The full list of PEPs - -* :pep:`526`: Syntax for Variable Annotations - *Introducing* syntax for annotating variables outside of function - definitions, and :data:`ClassVar` -* :pep:`544`: Protocols: Structural subtyping (static duck typing) - *Introducing* :class:`Protocol` and the - :func:`@runtime_checkable` decorator -* :pep:`585`: Type Hinting Generics In Standard Collections - *Introducing* :class:`types.GenericAlias` and the ability to use standard - library classes as :ref:`generic types` -* :pep:`586`: Literal Types - *Introducing* :data:`Literal` -* :pep:`589`: TypedDict: Type Hints for Dictionaries with a Fixed Set of Keys - *Introducing* :class:`TypedDict` -* :pep:`591`: Adding a final qualifier to typing - *Introducing* :data:`Final` and the :func:`@final` decorator -* :pep:`593`: Flexible function and variable annotations - *Introducing* :data:`Annotated` -* :pep:`604`: Allow writing union types as ``X | Y`` - *Introducing* :data:`types.UnionType` and the ability to use - the binary-or operator ``|`` to signify a - :ref:`union of types` -* :pep:`612`: Parameter Specification Variables - *Introducing* :class:`ParamSpec` and :data:`Concatenate` -* :pep:`613`: Explicit Type Aliases - *Introducing* :data:`TypeAlias` -* :pep:`646`: Variadic Generics - *Introducing* :data:`TypeVarTuple` -* :pep:`647`: User-Defined Type Guards - *Introducing* :data:`TypeGuard` -* :pep:`655`: Marking individual TypedDict items as required or potentially missing - *Introducing* :data:`Required` and :data:`NotRequired` -* :pep:`673`: Self type - *Introducing* :data:`Self` -* :pep:`675`: Arbitrary Literal String Type - *Introducing* :data:`LiteralString` -* :pep:`681`: Data Class Transforms - *Introducing* the :func:`@dataclass_transform` decorator -* :pep:`692`: Using ``TypedDict`` for more precise ``**kwargs`` typing - *Introducing* a new way of typing ``**kwargs`` with :data:`Unpack` and - :data:`TypedDict` -* :pep:`695`: Type Parameter Syntax - *Introducing* builtin syntax for creating generic functions, classes, and type aliases. -* :pep:`698`: Adding an override decorator to typing - *Introducing* the :func:`@override` decorator - -.. raw:: html - -
-
+Specification for the Python Type System +======================================== + +The canonical, up-to-date specification of the Python type system can be +found at `"Specification for the Python type system" `_. .. _type-aliases: @@ -155,8 +98,9 @@ Type aliases are useful for simplifying complex type signatures. For example:: # The static type checker will treat the previous type signature as # being exactly equivalent to this one. def broadcast_message( - message: str, - servers: Sequence[tuple[tuple[str, int], dict[str, str]]]) -> None: + message: str, + servers: Sequence[tuple[tuple[str, int], dict[str, str]]] + ) -> None: ... The :keyword:`type` statement is new in Python 3.12. For backwards @@ -264,7 +208,7 @@ Annotating callable objects =========================== Functions -- or other :term:`callable` objects -- can be annotated using -:class:`collections.abc.Callable` or :data:`typing.Callable`. +:class:`collections.abc.Callable` or deprecated :data:`typing.Callable`. ``Callable[[int], str]`` signifies a function that takes a single parameter of type :class:`int` and returns a :class:`str`. @@ -457,7 +401,7 @@ The type of class objects ========================= A variable annotated with ``C`` may accept a value of type ``C``. In -contrast, a variable annotated with ``type[C]`` (or +contrast, a variable annotated with ``type[C]`` (or deprecated :class:`typing.Type[C] `) may accept values that are classes themselves -- specifically, it will accept the *class object* of ``C``. For example:: @@ -497,6 +441,87 @@ For example:: ``type[Any]`` is equivalent to :class:`type`, which is the root of Python's :ref:`metaclass hierarchy `. + +.. _annotating-generators-and-coroutines: + +Annotating generators and coroutines +==================================== + +A generator can be annotated using the generic type +:class:`Generator[YieldType, SendType, ReturnType] `. +For example:: + + def echo_round() -> Generator[int, float, str]: + sent = yield 0 + while sent >= 0: + sent = yield round(sent) + return 'Done' + +Note that unlike many other generic classes in the standard library, +the ``SendType`` of :class:`~collections.abc.Generator` behaves +contravariantly, not covariantly or invariantly. + +The ``SendType`` and ``ReturnType`` parameters default to :const:`!None`:: + + def infinite_stream(start: int) -> Generator[int]: + while True: + yield start + start += 1 + +It is also possible to set these types explicitly:: + + def infinite_stream(start: int) -> Generator[int, None, None]: + while True: + yield start + start += 1 + +Simple generators that only ever yield values can also be annotated +as having a return type of either +:class:`Iterable[YieldType] ` +or :class:`Iterator[YieldType] `:: + + def infinite_stream(start: int) -> Iterator[int]: + while True: + yield start + start += 1 + +Async generators are handled in a similar fashion, but don't +expect a ``ReturnType`` type argument +(:class:`AsyncGenerator[YieldType, SendType] `). +The ``SendType`` argument defaults to :const:`!None`, so the following definitions +are equivalent:: + + async def infinite_stream(start: int) -> AsyncGenerator[int]: + while True: + yield start + start = await increment(start) + + async def infinite_stream(start: int) -> AsyncGenerator[int, None]: + while True: + yield start + start = await increment(start) + +As in the synchronous case, +:class:`AsyncIterable[YieldType] ` +and :class:`AsyncIterator[YieldType] ` are +available as well:: + + async def infinite_stream(start: int) -> AsyncIterator[int]: + while True: + yield start + start = await increment(start) + +Coroutines can be annotated using +:class:`Coroutine[YieldType, SendType, ReturnType] `. +Generic arguments correspond to those of :class:`~collections.abc.Generator`, +for example:: + + from collections.abc import Coroutine + c: Coroutine[list[str], str, int] # Some coroutine defined elsewhere + x = c.send('hi') # Inferred type of 'x' is list[str] + async def bar() -> None: + y = await c # Inferred type of 'y' is int + .. _user-defined-generics: User-defined generic types @@ -910,14 +935,25 @@ using ``[]``. .. versionadded:: 3.11 .. data:: Never + NoReturn - The `bottom type `_, + :data:`!Never` and :data:`!NoReturn` represent the + `bottom type `_, a type that has no members. - This can be used to define a function that should never be - called, or a function that never returns:: + They can be used to indicate that a function never returns, + such as :func:`sys.exit`:: + + from typing import Never # or NoReturn + + def stop() -> Never: + raise RuntimeError('no way') + + Or to define a function that should never be + called, as there are no valid arguments, such as + :func:`assert_never`:: - from typing import Never + from typing import Never # or NoReturn def never_call_me(arg: Never) -> None: pass @@ -930,32 +966,18 @@ using ``[]``. case str(): print("It's a str") case _: - never_call_me(arg) # OK, arg is of type Never + never_call_me(arg) # OK, arg is of type Never (or NoReturn) - .. versionadded:: 3.11 - - On older Python versions, :data:`NoReturn` may be used to express the - same concept. ``Never`` was added to make the intended meaning more explicit. - -.. data:: NoReturn - - Special type indicating that a function never returns. - - For example:: + :data:`!Never` and :data:`!NoReturn` have the same meaning in the type system + and static type checkers treat both equivalently. - from typing import NoReturn + .. versionadded:: 3.6.2 - def stop() -> NoReturn: - raise RuntimeError('no way') + Added :data:`NoReturn`. - ``NoReturn`` can also be used as a - `bottom type `_, a type that - has no values. Starting in Python 3.11, the :data:`Never` type should - be used for this concept instead. Type checkers should treat the two - equivalently. + .. versionadded:: 3.11 - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.2 + Added :data:`Never`. .. data:: Self @@ -1234,6 +1256,10 @@ These can be used as types in annotations. They all support subscription using .. versionadded:: 3.5.3 + .. versionchanged:: 3.13 + + :data:`ClassVar` can now be nested in :data:`Final` and vice versa. + .. data:: Final Special typing construct to indicate final names to type checkers. @@ -1257,6 +1283,10 @@ These can be used as types in annotations. They all support subscription using .. versionadded:: 3.8 + .. versionchanged:: 3.13 + + :data:`Final` can now be nested in :data:`ClassVar` and vice versa. + .. data:: Required Special typing construct to mark a :class:`TypedDict` key as required. @@ -1275,6 +1305,26 @@ These can be used as types in annotations. They all support subscription using .. versionadded:: 3.11 +.. data:: ReadOnly + + A special typing construct to mark an item of a :class:`TypedDict` as read-only. + + For example:: + + class Movie(TypedDict): + title: ReadOnly[str] + year: int + + def mutate_movie(m: Movie) -> None: + m["year"] = 1999 # allowed + m["title"] = "The Matrix" # typechecker error + + There is no runtime checking for this property. + + See :class:`TypedDict` and :pep:`705` for more details. + + .. versionadded:: 3.13 + .. data:: Annotated Special typing form to add context-specific metadata to an annotation. @@ -1408,6 +1458,23 @@ These can be used as types in annotations. They all support subscription using >>> X.__metadata__ ('very', 'important', 'metadata') + * At runtime, if you want to retrieve the original + type wrapped by ``Annotated``, use the :attr:`!__origin__` attribute: + + .. doctest:: + + >>> from typing import Annotated, get_origin + >>> Password = Annotated[str, "secret"] + >>> Password.__origin__ + + + Note that using :func:`get_origin` will return ``Annotated`` itself: + + .. doctest:: + + >>> get_origin(Password) + typing.Annotated + .. seealso:: :pep:`593` - Flexible function and variable annotations @@ -1416,22 +1483,23 @@ These can be used as types in annotations. They all support subscription using .. versionadded:: 3.9 -.. data:: TypeGuard +.. data:: TypeIs - Special typing construct for marking user-defined type guard functions. + Special typing construct for marking user-defined type predicate functions. - ``TypeGuard`` can be used to annotate the return type of a user-defined - type guard function. ``TypeGuard`` only accepts a single type argument. - At runtime, functions marked this way should return a boolean. + ``TypeIs`` can be used to annotate the return type of a user-defined + type predicate function. ``TypeIs`` only accepts a single type argument. + At runtime, functions marked this way should return a boolean and take at + least one positional argument. - ``TypeGuard`` aims to benefit *type narrowing* -- a technique used by static + ``TypeIs`` aims to benefit *type narrowing* -- a technique used by static type checkers to determine a more precise type of an expression within a program's code flow. Usually type narrowing is done by analyzing conditional code flow and applying the narrowing to a block of code. The - conditional expression here is sometimes referred to as a "type guard":: + conditional expression here is sometimes referred to as a "type predicate":: def is_str(val: str | float): - # "isinstance" type guard + # "isinstance" type predicate if isinstance(val, str): # Type of ``val`` is narrowed to ``str`` ... @@ -1440,8 +1508,73 @@ These can be used as types in annotations. They all support subscription using ... Sometimes it would be convenient to use a user-defined boolean function - as a type guard. Such a function should use ``TypeGuard[...]`` as its - return type to alert static type checkers to this intention. + as a type predicate. Such a function should use ``TypeIs[...]`` or + :data:`TypeGuard` as its return type to alert static type checkers to + this intention. ``TypeIs`` usually has more intuitive behavior than + ``TypeGuard``, but it cannot be used when the input and output types + are incompatible (e.g., ``list[object]`` to ``list[int]``) or when the + function does not return ``True`` for all instances of the narrowed type. + + Using ``-> TypeIs[NarrowedType]`` tells the static type checker that for a given + function: + + 1. The return value is a boolean. + 2. If the return value is ``True``, the type of its argument + is the intersection of the argument's original type and ``NarrowedType``. + 3. If the return value is ``False``, the type of its argument + is narrowed to exclude ``NarrowedType``. + + For example:: + + from typing import assert_type, final, TypeIs + + class Parent: pass + class Child(Parent): pass + @final + class Unrelated: pass + + def is_parent(val: object) -> TypeIs[Parent]: + return isinstance(val, Parent) + + def run(arg: Child | Unrelated): + if is_parent(arg): + # Type of ``arg`` is narrowed to the intersection + # of ``Parent`` and ``Child``, which is equivalent to + # ``Child``. + assert_type(arg, Child) + else: + # Type of ``arg`` is narrowed to exclude ``Parent``, + # so only ``Unrelated`` is left. + assert_type(arg, Unrelated) + + The type inside ``TypeIs`` must be consistent with the type of the + function's argument; if it is not, static type checkers will raise + an error. An incorrectly written ``TypeIs`` function can lead to + unsound behavior in the type system; it is the user's responsibility + to write such functions in a type-safe manner. + + If a ``TypeIs`` function is a class or instance method, then the type in + ``TypeIs`` maps to the type of the second parameter (after ``cls`` or + ``self``). + + In short, the form ``def foo(arg: TypeA) -> TypeIs[TypeB]: ...``, + means that if ``foo(arg)`` returns ``True``, then ``arg`` is an instance + of ``TypeB``, and if it returns ``False``, it is not an instance of ``TypeB``. + + ``TypeIs`` also works with type variables. For more information, see + :pep:`742` (Narrowing types with ``TypeIs``). + + .. versionadded:: 3.13 + + +.. data:: TypeGuard + + Special typing construct for marking user-defined type predicate functions. + + Type predicate functions are user-defined functions that return whether their + argument is an instance of a particular type. + ``TypeGuard`` works similarly to :data:`TypeIs`, but has subtly different + effects on type checking behavior (see below). Using ``-> TypeGuard`` tells the static type checker that for a given function: @@ -1450,6 +1583,8 @@ These can be used as types in annotations. They all support subscription using 2. If the return value is ``True``, the type of its argument is the type inside ``TypeGuard``. + ``TypeGuard`` also works with type variables. See :pep:`647` for more details. + For example:: def is_str_list(val: list[object]) -> TypeGuard[list[str]]: @@ -1464,23 +1599,19 @@ These can be used as types in annotations. They all support subscription using # Type of ``val`` remains as ``list[object]``. print("Not a list of strings!") - If ``is_str_list`` is a class or instance method, then the type in - ``TypeGuard`` maps to the type of the second parameter after ``cls`` or - ``self``. - - In short, the form ``def foo(arg: TypeA) -> TypeGuard[TypeB]: ...``, - means that if ``foo(arg)`` returns ``True``, then ``arg`` narrows from - ``TypeA`` to ``TypeB``. - - .. note:: - - ``TypeB`` need not be a narrower form of ``TypeA`` -- it can even be a - wider form. The main reason is to allow for things like - narrowing ``list[object]`` to ``list[str]`` even though the latter - is not a subtype of the former, since ``list`` is invariant. - The responsibility of writing type-safe type guards is left to the user. + ``TypeIs`` and ``TypeGuard`` differ in the following ways: - ``TypeGuard`` also works with type variables. See :pep:`647` for more details. + * ``TypeIs`` requires the narrowed type to be a subtype of the input type, while + ``TypeGuard`` does not. The main reason is to allow for things like + narrowing ``list[object]`` to ``list[str]`` even though the latter + is not a subtype of the former, since ``list`` is invariant. + * When a ``TypeGuard`` function returns ``True``, type checkers narrow the type of the + variable to exactly the ``TypeGuard`` type. When a ``TypeIs`` function returns ``True``, + type checkers can infer a more precise type combining the previously known type of the + variable with the ``TypeIs`` type. (Technically, this is known as an intersection type.) + * When a ``TypeGuard`` function returns ``False``, type checkers cannot narrow the type of + the variable at all. When a ``TypeIs`` function returns ``False``, type checkers can narrow + the type of the variable to exclude the ``TypeIs`` type. .. versionadded:: 3.10 @@ -1583,7 +1714,7 @@ without the dedicated syntax, as documented below. .. _typevar: -.. class:: TypeVar(name, *constraints, bound=None, covariant=False, contravariant=False, infer_variance=False) +.. class:: TypeVar(name, *constraints, bound=None, covariant=False, contravariant=False, infer_variance=False, default=typing.NoDefault) Type variable. @@ -1595,11 +1726,11 @@ without the dedicated syntax, as documented below. class Sequence[T]: # T is a TypeVar ... - This syntax can also be used to create bound and constrained type + This syntax can also be used to create bounded and constrained type variables:: - class StrSequence[S: str]: # S is a TypeVar bound to str - ... + class StrSequence[S: str]: # S is a TypeVar with a `str` upper bound; + ... # we can say that S is "bounded by `str`" class StrOrBytesSequence[A: (str, bytes)]: # A is a TypeVar constrained to str or bytes @@ -1632,8 +1763,8 @@ without the dedicated syntax, as documented below. """Add two strings or bytes objects together.""" return x + y - Note that type variables can be *bound*, *constrained*, or neither, but - cannot be both bound *and* constrained. + Note that type variables can be *bounded*, *constrained*, or neither, but + cannot be both bounded *and* constrained. The variance of type variables is inferred by type checkers when they are created through the :ref:`type parameter syntax ` or when @@ -1643,8 +1774,8 @@ without the dedicated syntax, as documented below. By default, manually created type variables are invariant. See :pep:`484` and :pep:`695` for more details. - Bound type variables and constrained type variables have different - semantics in several important ways. Using a *bound* type variable means + Bounded type variables and constrained type variables have different + semantics in several important ways. Using a *bounded* type variable means that the ``TypeVar`` will be solved using the most specific type possible:: x = print_capitalized('a string') @@ -1658,8 +1789,8 @@ without the dedicated syntax, as documented below. z = print_capitalized(45) # error: int is not a subtype of str - Type variables can be bound to concrete types, abstract types (ABCs or - protocols), and even unions of types:: + The upper bound of a type variable can be a concrete type, abstract type + (ABC or Protocol), or even a union of types:: # Can be anything with an __abs__ method def print_abs[T: SupportsAbs](arg: T) -> None: @@ -1703,7 +1834,7 @@ without the dedicated syntax, as documented below. .. attribute:: __bound__ - The bound of the type variable, if any. + The upper bound of the type variable, if any. .. versionchanged:: 3.12 @@ -1711,6 +1842,16 @@ without the dedicated syntax, as documented below. the bound is evaluated only when the attribute is accessed, not when the type variable is created (see :ref:`lazy-evaluation`). + .. method:: evaluate_bound + + An :term:`evaluate function` corresponding to the :attr:`~TypeVar.__bound__` attribute. + When called directly, this method supports only the :attr:`~annotationlib.Format.VALUE` + format, which is equivalent to accessing the :attr:`~TypeVar.__bound__` attribute directly, + but the method object can be passed to :func:`annotationlib.call_evaluate_function` + to evaluate the value in a different format. + + .. versionadded:: 3.14 + .. attribute:: __constraints__ A tuple containing the constraints of the type variable, if any. @@ -1721,15 +1862,55 @@ without the dedicated syntax, as documented below. the constraints are evaluated only when the attribute is accessed, not when the type variable is created (see :ref:`lazy-evaluation`). + .. method:: evaluate_constraints + + An :term:`evaluate function` corresponding to the :attr:`~TypeVar.__constraints__` attribute. + When called directly, this method supports only the :attr:`~annotationlib.Format.VALUE` + format, which is equivalent to accessing the :attr:`~TypeVar.__constraints__` attribute directly, + but the method object can be passed to :func:`annotationlib.call_evaluate_function` + to evaluate the value in a different format. + + .. versionadded:: 3.14 + + .. attribute:: __default__ + + The default value of the type variable, or :data:`typing.NoDefault` if it + has no default. + + .. versionadded:: 3.13 + + .. method:: evaluate_default + + An :term:`evaluate function` corresponding to the :attr:`~TypeVar.__default__` attribute. + When called directly, this method supports only the :attr:`~annotationlib.Format.VALUE` + format, which is equivalent to accessing the :attr:`~TypeVar.__default__` attribute directly, + but the method object can be passed to :func:`annotationlib.call_evaluate_function` + to evaluate the value in a different format. + + .. versionadded:: 3.14 + + .. method:: has_default() + + Return whether or not the type variable has a default value. This is equivalent + to checking whether :attr:`__default__` is not the :data:`typing.NoDefault` + singleton, except that it does not force evaluation of the + :ref:`lazily evaluated ` default value. + + .. versionadded:: 3.13 + .. versionchanged:: 3.12 Type variables can now be declared using the :ref:`type parameter ` syntax introduced by :pep:`695`. The ``infer_variance`` parameter was added. + .. versionchanged:: 3.13 + + Support for default values was added. + .. _typevartuple: -.. class:: TypeVarTuple(name) +.. class:: TypeVarTuple(name, *, default=typing.NoDefault) Type variable tuple. A specialized form of :ref:`type variable ` that enables *variadic* generics. @@ -1820,8 +2001,8 @@ without the dedicated syntax, as documented below. of ``*args``:: def call_soon[*Ts]( - callback: Callable[[*Ts], None], - *args: *Ts + callback: Callable[[*Ts], None], + *args: *Ts ) -> None: ... callback(*args) @@ -1839,6 +2020,32 @@ without the dedicated syntax, as documented below. The name of the type variable tuple. + .. attribute:: __default__ + + The default value of the type variable tuple, or :data:`typing.NoDefault` if it + has no default. + + .. versionadded:: 3.13 + + .. method:: evaluate_default + + An :term:`evaluate function` corresponding to the :attr:`~TypeVarTuple.__default__` attribute. + When called directly, this method supports only the :attr:`~annotationlib.Format.VALUE` + format, which is equivalent to accessing the :attr:`~TypeVarTuple.__default__` attribute directly, + but the method object can be passed to :func:`annotationlib.call_evaluate_function` + to evaluate the value in a different format. + + .. versionadded:: 3.14 + + .. method:: has_default() + + Return whether or not the type variable tuple has a default value. This is equivalent + to checking whether :attr:`__default__` is not the :data:`typing.NoDefault` + singleton, except that it does not force evaluation of the + :ref:`lazily evaluated ` default value. + + .. versionadded:: 3.13 + .. versionadded:: 3.11 .. versionchanged:: 3.12 @@ -1846,7 +2053,11 @@ without the dedicated syntax, as documented below. Type variable tuples can now be declared using the :ref:`type parameter ` syntax introduced by :pep:`695`. -.. class:: ParamSpec(name, *, bound=None, covariant=False, contravariant=False) + .. versionchanged:: 3.13 + + Support for default values was added. + +.. class:: ParamSpec(name, *, bound=None, covariant=False, contravariant=False, default=typing.NoDefault) Parameter specification variable. A specialized version of :ref:`type variables `. @@ -1889,7 +2100,7 @@ without the dedicated syntax, as documented below. return x + y Without ``ParamSpec``, the simplest way to annotate this previously was to - use a :class:`TypeVar` with bound ``Callable[..., Any]``. However this + use a :class:`TypeVar` with upper bound ``Callable[..., Any]``. However this causes two problems: 1. The type checker can't type check the ``inner`` function because @@ -1915,6 +2126,32 @@ without the dedicated syntax, as documented below. The name of the parameter specification. + .. attribute:: __default__ + + The default value of the parameter specification, or :data:`typing.NoDefault` if it + has no default. + + .. versionadded:: 3.13 + + .. method:: evaluate_default + + An :term:`evaluate function` corresponding to the :attr:`~ParamSpec.__default__` attribute. + When called directly, this method supports only the :attr:`~annotationlib.Format.VALUE` + format, which is equivalent to accessing the :attr:`~ParamSpec.__default__` attribute directly, + but the method object can be passed to :func:`annotationlib.call_evaluate_function` + to evaluate the value in a different format. + + .. versionadded:: 3.14 + + .. method:: has_default() + + Return whether or not the parameter specification has a default value. This is equivalent + to checking whether :attr:`__default__` is not the :data:`typing.NoDefault` + singleton, except that it does not force evaluation of the + :ref:`lazily evaluated ` default value. + + .. versionadded:: 3.13 + Parameter specification variables created with ``covariant=True`` or ``contravariant=True`` can be used to declare covariant or contravariant generic types. The ``bound`` argument is also accepted, similar to @@ -1928,6 +2165,10 @@ without the dedicated syntax, as documented below. Parameter specifications can now be declared using the :ref:`type parameter ` syntax introduced by :pep:`695`. + .. versionchanged:: 3.13 + + Support for default values was added. + .. note:: Only parameter specification variables defined in global scope can be pickled. @@ -1939,7 +2180,7 @@ without the dedicated syntax, as documented below. * :ref:`annotating-callables` .. data:: ParamSpecArgs -.. data:: ParamSpecKwargs + ParamSpecKwargs Arguments and keyword arguments attributes of a :class:`ParamSpec`. The ``P.args`` attribute of a ``ParamSpec`` is an instance of ``ParamSpecArgs``, @@ -2026,6 +2267,32 @@ without the dedicated syntax, as documented below. >>> Recursive.__value__ Mutually + .. method:: evaluate_value + + An :term:`evaluate function` corresponding to the :attr:`__value__` attribute. + When called directly, this method supports only the :attr:`~annotationlib.Format.VALUE` + format, which is equivalent to accessing the :attr:`__value__` attribute directly, + but the method object can be passed to :func:`annotationlib.call_evaluate_function` + to evaluate the value in a different format: + + .. doctest:: + + >>> type Alias = undefined + >>> Alias.__value__ + Traceback (most recent call last): + ... + NameError: name 'undefined' is not defined + >>> from annotationlib import Format, call_evaluate_function + >>> Alias.evaluate_value(Format.VALUE) + Traceback (most recent call last): + ... + NameError: name 'undefined' is not defined + >>> call_evaluate_function(Alias.evaluate_value, Format.FORWARDREF) + ForwardRef('undefined') + + .. versionadded:: 3.14 + + Other special directives """""""""""""""""""""""" @@ -2082,7 +2349,9 @@ types. Backward-compatible usage:: - # For creating a generic NamedTuple on Python 3.11 or lower + # For creating a generic NamedTuple on Python 3.11 + T = TypeVar("T") + class Group(NamedTuple, Generic[T]): key: T group: list[T] @@ -2407,7 +2676,7 @@ types. This attribute reflects *only* the value of the ``total`` argument to the current ``TypedDict`` class, not whether the class is semantically - total. For example, a ``TypedDict`` with ``__total__`` set to True may + total. For example, a ``TypedDict`` with ``__total__`` set to ``True`` may have keys marked with :data:`NotRequired`, or it may inherit from another ``TypedDict`` with ``total=False``. Therefore, it is generally better to use :attr:`__required_keys__` and :attr:`__optional_keys__` for introspection. @@ -2455,6 +2724,22 @@ types. ``__required_keys__`` and ``__optional_keys__`` rely on may not work properly, and the values of the attributes may be incorrect. + Support for :data:`ReadOnly` is reflected in the following attributes: + + .. attribute:: __readonly_keys__ + + A :class:`frozenset` containing the names of all read-only keys. Keys + are read-only if they carry the :data:`ReadOnly` qualifier. + + .. versionadded:: 3.13 + + .. attribute:: __mutable_keys__ + + A :class:`frozenset` containing the names of all mutable keys. Keys + are mutable if they do not carry the :data:`ReadOnly` qualifier. + + .. versionadded:: 3.13 + See :pep:`589` for more examples and detailed rules of using ``TypedDict``. .. versionadded:: 3.8 @@ -2469,6 +2754,9 @@ types. .. versionchanged:: 3.13 Removed support for the keyword-argument method of creating ``TypedDict``\ s. + .. versionchanged:: 3.13 + Support for the :data:`ReadOnly` qualifier was added. + .. deprecated-removed:: 3.13 3.15 When using the functional syntax to create a TypedDict class, failing to pass a value to the 'fields' parameter (``TD = TypedDict("TD")``) is @@ -2597,7 +2885,7 @@ Functions and decorators .. seealso:: `Unreachable Code and Exhaustiveness Checking - `__ has more + `__ has more information about exhaustiveness checking with static typing. .. versionadded:: 3.11 @@ -2970,33 +3258,38 @@ Introspection helpers Return a dictionary containing type hints for a function, method, module or class object. - This is often the same as ``obj.__annotations__``. In addition, - forward references encoded as string literals are handled by evaluating - them in ``globals`` and ``locals`` namespaces. For a class ``C``, return - a dictionary constructed by merging all the ``__annotations__`` along - ``C.__mro__`` in reverse order. - - The function recursively replaces all ``Annotated[T, ...]`` with ``T``, - unless ``include_extras`` is set to ``True`` (see :class:`Annotated` for - more information). For example: - - .. testcode:: - - class Student(NamedTuple): - name: Annotated[str, 'some marker'] - - assert get_type_hints(Student) == {'name': str} - assert get_type_hints(Student, include_extras=False) == {'name': str} - assert get_type_hints(Student, include_extras=True) == { - 'name': Annotated[str, 'some marker'] - } + This is often the same as ``obj.__annotations__``, but this function makes + the following changes to the annotations dictionary: + + * Forward references encoded as string literals or :class:`ForwardRef` + objects are handled by evaluating them in *globalns*, *localns*, and + (where applicable) *obj*'s :ref:`type parameter ` namespace. + If *globalns* or *localns* is not given, appropriate namespace + dictionaries are inferred from *obj*. + * ``None`` is replaced with :class:`types.NoneType`. + * If :func:`@no_type_check ` has been applied to *obj*, an + empty dictionary is returned. + * If *obj* is a class ``C``, the function returns a dictionary that merges + annotations from ``C``'s base classes with those on ``C`` directly. This + is done by traversing :attr:`C.__mro__ ` and iteratively + combining + ``__annotations__`` dictionaries. Annotations on classes appearing + earlier in the :term:`method resolution order` always take precedence over + annotations on classes appearing later in the method resolution order. + * The function recursively replaces all occurrences of ``Annotated[T, ...]`` + with ``T``, unless *include_extras* is set to ``True`` (see + :class:`Annotated` for more information). + + See also :func:`inspect.get_annotations`, a lower-level function that + returns annotations more directly. .. note:: - :func:`get_type_hints` does not work with imported - :ref:`type aliases ` that include forward references. - Enabling postponed evaluation of annotations (:pep:`563`) may remove - the need for most forward references. + If any forward references in the annotations of *obj* are not resolvable + or are not valid Python code, this function will raise an exception + such as :exc:`NameError`. For example, this can happen with imported + :ref:`type aliases ` that include forward references, + or with names imported under :data:`if TYPE_CHECKING `. .. versionchanged:: 3.9 Added ``include_extras`` parameter as part of :pep:`593`. @@ -3025,6 +3318,7 @@ Introspection helpers assert get_origin(str) is None assert get_origin(Dict[str, int]) is dict assert get_origin(Union[int, str]) is Union + assert get_origin(Annotated[str, "metadata"]) is Annotated P = ParamSpec('P') assert get_origin(P.args) is P assert get_origin(P.kwargs) is P @@ -3109,7 +3403,7 @@ Introspection helpers Class used for internal typing representation of string forward references. For example, ``List["SomeClass"]`` is implicitly transformed into - ``List[ForwardRef("SomeClass")]``. ``ForwardRef`` should not be instantiated by + ``List[ForwardRef("SomeClass")]``. :class:`!ForwardRef` should not be instantiated by a user, but may be used by introspection tools. .. note:: @@ -3119,6 +3413,55 @@ Introspection helpers .. versionadded:: 3.7.4 + .. versionchanged:: 3.14 + This is now an alias for :class:`annotationlib.ForwardRef`. + +.. function:: evaluate_forward_ref(forward_ref, *, owner=None, globals=None, locals=None, type_params=None, format=annotationlib.Format.VALUE) + + Evaluate an :class:`annotationlib.ForwardRef` as a :term:`type hint`. + + This is similar to calling :meth:`annotationlib.ForwardRef.evaluate`, + but unlike that method, :func:`!evaluate_forward_ref` also: + + * Recursively evaluates forward references nested within the type hint. + * Raises :exc:`TypeError` when it encounters certain objects that are + not valid type hints. + * Replaces type hints that evaluate to :const:`!None` with + :class:`types.NoneType`. + * Supports the :attr:`~annotationlib.Format.FORWARDREF` and + :attr:`~annotationlib.Format.STRING` formats. + + *forward_ref* must be an instance of :class:`~annotationlib.ForwardRef`. + *owner*, if given, should be the object that holds the annotations that + the forward reference derived from, such as a module, class object, or function. + It is used to infer the namespaces to use for looking up names. + *globals* and *locals* can also be explicitly given to provide + the global and local namespaces. + *type_params* is a tuple of :ref:`type parameters ` that + are in scope when evaluating the forward reference. + This parameter must be provided (though it may be an empty tuple) if *owner* + is not given and the forward reference does not already have an owner set. + *format* specifies the format of the annotation and is a member of + the :class:`annotationlib.Format` enum. + + .. versionadded:: 3.14 + +.. data:: NoDefault + + A sentinel object used to indicate that a type parameter has no default + value. For example: + + .. doctest:: + + >>> T = TypeVar("T") + >>> T.__default__ is typing.NoDefault + True + >>> S = TypeVar("S", default=None) + >>> S.__default__ is None + True + + .. versionadded:: 3.13 + Constant -------- @@ -3186,14 +3529,9 @@ Aliases to built-in types Deprecated alias to :class:`dict`. Note that to annotate arguments, it is preferred - to use an abstract collection type such as :class:`Mapping` + to use an abstract collection type such as :class:`~collections.abc.Mapping` rather than to use :class:`dict` or :class:`!typing.Dict`. - This type can be used as follows:: - - def count_words(text: str) -> Dict[str, int]: - ... - .. deprecated:: 3.9 :class:`builtins.dict ` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. @@ -3203,16 +3541,9 @@ Aliases to built-in types Deprecated alias to :class:`list`. Note that to annotate arguments, it is preferred - to use an abstract collection type such as :class:`Sequence` or - :class:`Iterable` rather than to use :class:`list` or :class:`!typing.List`. - - This type may be used as follows:: - - def vec2[T: (int, float)](x: T, y: T) -> List[T]: - return [x, y] - - def keep_positives[T: (int, float)](vector: Sequence[T]) -> List[T]: - return [item for item in vector if item > 0] + to use an abstract collection type such as + :class:`~collections.abc.Sequence` or :class:`~collections.abc.Iterable` + rather than to use :class:`list` or :class:`!typing.List`. .. deprecated:: 3.9 :class:`builtins.list ` now supports subscripting (``[]``). @@ -3223,8 +3554,8 @@ Aliases to built-in types Deprecated alias to :class:`builtins.set `. Note that to annotate arguments, it is preferred - to use an abstract collection type such as :class:`AbstractSet` - rather than to use :class:`set` or :class:`!typing.Set`. + to use an abstract collection type such as :class:`collections.abc.Set` + rather than to use :class:`set` or :class:`typing.Set`. .. deprecated:: 3.9 :class:`builtins.set ` now supports subscripting (``[]``). @@ -3292,7 +3623,6 @@ Aliases to types in :mod:`collections` Deprecated alias to :class:`collections.ChainMap`. - .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. deprecated:: 3.9 @@ -3303,7 +3633,6 @@ Aliases to types in :mod:`collections` Deprecated alias to :class:`collections.Counter`. - .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. deprecated:: 3.9 @@ -3314,7 +3643,6 @@ Aliases to types in :mod:`collections` Deprecated alias to :class:`collections.deque`. - .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. deprecated:: 3.9 @@ -3377,19 +3705,11 @@ Aliases to container ABCs in :mod:`collections.abc` :class:`collections.abc.Set` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. -.. class:: ByteString(Sequence[int]) - - This type represents the types :class:`bytes`, :class:`bytearray`, - and :class:`memoryview` of byte sequences. - - .. deprecated-removed:: 3.9 3.14 - Prefer :class:`collections.abc.Buffer`, or a union like ``bytes | bytearray | memoryview``. - .. class:: Collection(Sized, Iterable[T_co], Container[T_co]) Deprecated alias to :class:`collections.abc.Collection`. - .. versionadded:: 3.6.0 + .. versionadded:: 3.6 .. deprecated:: 3.9 :class:`collections.abc.Collection` now supports subscripting (``[]``). @@ -3423,11 +3743,6 @@ Aliases to container ABCs in :mod:`collections.abc` Deprecated alias to :class:`collections.abc.Mapping`. - This type can be used as follows:: - - def get_position_in_index(word_list: Mapping[str, int], word: str) -> int: - return word_list[word] - .. deprecated:: 3.9 :class:`collections.abc.Mapping` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. @@ -3491,14 +3806,9 @@ Aliases to asynchronous ABCs in :mod:`collections.abc` Deprecated alias to :class:`collections.abc.Coroutine`. - The variance and order of type variables - correspond to those of :class:`Generator`, for example:: - - from collections.abc import Coroutine - c: Coroutine[list[str], str, int] # Some coroutine defined elsewhere - x = c.send('hi') # Inferred type of 'x' is list[str] - async def bar() -> None: - y = await c # Inferred type of 'y' is int + See :ref:`annotating-generators-and-coroutines` + for details on using :class:`collections.abc.Coroutine` + and ``typing.Coroutine`` in type annotations. .. versionadded:: 3.5.3 @@ -3510,34 +3820,9 @@ Aliases to asynchronous ABCs in :mod:`collections.abc` Deprecated alias to :class:`collections.abc.AsyncGenerator`. - An async generator can be annotated by the generic type - ``AsyncGenerator[YieldType, SendType]``. For example:: - - async def echo_round() -> AsyncGenerator[int, float]: - sent = yield 0 - while sent >= 0.0: - rounded = await round(sent) - sent = yield rounded - - Unlike normal generators, async generators cannot return a value, so there - is no ``ReturnType`` type parameter. As with :class:`Generator`, the - ``SendType`` behaves contravariantly. - - If your generator will only yield values, set the ``SendType`` to - ``None``:: - - async def infinite_stream(start: int) -> AsyncGenerator[int, None]: - while True: - yield start - start = await increment(start) - - Alternatively, annotate your generator as having a return type of - either ``AsyncIterable[YieldType]`` or ``AsyncIterator[YieldType]``:: - - async def infinite_stream(start: int) -> AsyncIterator[int]: - while True: - yield start - start = await increment(start) + See :ref:`annotating-generators-and-coroutines` + for details on using :class:`collections.abc.AsyncGenerator` + and ``typing.AsyncGenerator`` in type annotations. .. versionadded:: 3.6.1 @@ -3546,6 +3831,9 @@ Aliases to asynchronous ABCs in :mod:`collections.abc` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. + .. versionchanged:: 3.13 + The ``SendType`` parameter now has a default. + .. class:: AsyncIterable(Generic[T_co]) Deprecated alias to :class:`collections.abc.AsyncIterable`. @@ -3616,39 +3904,17 @@ Aliases to other ABCs in :mod:`collections.abc` Deprecated alias to :class:`collections.abc.Generator`. - A generator can be annotated by the generic type - ``Generator[YieldType, SendType, ReturnType]``. For example:: - - def echo_round() -> Generator[int, float, str]: - sent = yield 0 - while sent >= 0: - sent = yield round(sent) - return 'Done' - - Note that unlike many other generics in the typing module, the ``SendType`` - of :class:`Generator` behaves contravariantly, not covariantly or - invariantly. - - If your generator will only yield values, set the ``SendType`` and - ``ReturnType`` to ``None``:: - - def infinite_stream(start: int) -> Generator[int, None, None]: - while True: - yield start - start += 1 - - Alternatively, annotate your generator as having a return type of - either ``Iterable[YieldType]`` or ``Iterator[YieldType]``:: - - def infinite_stream(start: int) -> Iterator[int]: - while True: - yield start - start += 1 + See :ref:`annotating-generators-and-coroutines` + for details on using :class:`collections.abc.Generator` + and ``typing.Generator`` in type annotations. .. deprecated:: 3.9 :class:`collections.abc.Generator` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. + .. versionchanged:: 3.13 + Default values for the send and return types were added. + .. class:: Hashable Deprecated alias to :class:`collections.abc.Hashable`. @@ -3676,23 +3942,34 @@ Aliases to other ABCs in :mod:`collections.abc` Aliases to :mod:`contextlib` ABCs """"""""""""""""""""""""""""""""" -.. class:: ContextManager(Generic[T_co]) +.. class:: ContextManager(Generic[T_co, ExitT_co]) Deprecated alias to :class:`contextlib.AbstractContextManager`. + The first type parameter, ``T_co``, represents the type returned by + the :meth:`~object.__enter__` method. The optional second type parameter, ``ExitT_co``, + which defaults to ``bool | None``, represents the type returned by the + :meth:`~object.__exit__` method. + .. versionadded:: 3.5.4 - .. versionadded:: 3.6.0 .. deprecated:: 3.9 :class:`contextlib.AbstractContextManager` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. -.. class:: AsyncContextManager(Generic[T_co]) + .. versionchanged:: 3.13 + Added the optional second type parameter, ``ExitT_co``. + +.. class:: AsyncContextManager(Generic[T_co, AExitT_co]) Deprecated alias to :class:`contextlib.AbstractAsyncContextManager`. - .. versionadded:: 3.5.4 + The first type parameter, ``T_co``, represents the type returned by + the :meth:`~object.__aenter__` method. The optional second type parameter, ``AExitT_co``, + which defaults to ``bool | None``, represents the type returned by the + :meth:`~object.__aexit__` method. + .. versionadded:: 3.6.2 .. deprecated:: 3.9 @@ -3700,6 +3977,9 @@ Aliases to :mod:`contextlib` ABCs now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. + .. versionchanged:: 3.13 + Added the optional second type parameter, ``AExitT_co``. + Deprecation Timeline of Major Features ====================================== @@ -3718,10 +3998,6 @@ convenience. This is subject to change, and not all deprecations are listed. - 3.9 - Undecided (see :ref:`deprecated-aliases` for more information) - :pep:`585` - * - :class:`typing.ByteString` - - 3.9 - - 3.14 - - :gh:`91896` * - :data:`typing.Text` - 3.11 - Undecided diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index 7db47d48022a0e..0aef597d064e0e 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -1,5 +1,5 @@ -:mod:`unicodedata` --- Unicode Database -======================================= +:mod:`!unicodedata` --- Unicode Database +======================================== .. module:: unicodedata :synopsis: Access the Unicode Database. @@ -17,8 +17,8 @@ This module provides access to the Unicode Character Database (UCD) which defines character properties for all Unicode characters. The data contained in -this database is compiled from the `UCD version 15.1.0 -`_. +this database is compiled from the `UCD version 16.0.0 +`_. The module uses the same names and symbols as defined by Unicode Standard Annex #44, `"Unicode Character Database" @@ -175,6 +175,6 @@ Examples: .. rubric:: Footnotes -.. [#] https://www.unicode.org/Public/15.1.0/ucd/NameAliases.txt +.. [#] https://www.unicode.org/Public/16.0.0/ucd/NameAliases.txt -.. [#] https://www.unicode.org/Public/15.1.0/ucd/NamedSequences.txt +.. [#] https://www.unicode.org/Public/16.0.0/ucd/NamedSequences.txt diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index f2bdde80bdbd64..00cc9bfc0a5f2b 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -1,5 +1,5 @@ -:mod:`unittest.mock` --- getting started -======================================== +:mod:`!unittest.mock` --- getting started +========================================= .. moduleauthor:: Michael Foord .. currentmodule:: unittest.mock diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index 1f25a16f544da8..eae3ef2888eae0 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -1,6 +1,5 @@ - -:mod:`unittest.mock` --- mock object library -============================================ +:mod:`!unittest.mock` --- mock object library +============================================= .. module:: unittest.mock :synopsis: Mock object library. @@ -35,7 +34,7 @@ is based on the 'action -> assertion' pattern instead of 'record -> replay' used by many mocking frameworks. There is a backport of :mod:`unittest.mock` for earlier versions of Python, -available as `mock on PyPI `_. +available as :pypi:`mock` on PyPI. Quick Guide @@ -69,7 +68,7 @@ available, and then make assertions about how they have been used: 3 >>> thing.method.assert_called_with(3, 4, 5, key='value') -:attr:`side_effect` allows you to perform side effects, including raising an +:attr:`~Mock.side_effect` allows you to perform side effects, including raising an exception when a mock is called: >>> from unittest.mock import Mock @@ -240,7 +239,7 @@ the *new_callable* argument to :func:`patch`. Accessing any attribute not in this list will raise an :exc:`AttributeError`. If *spec* is an object (rather than a list of strings) then - :attr:`~instance.__class__` returns the class of the spec object. This + :attr:`~object.__class__` returns the class of the spec object. This allows mocks to pass :func:`isinstance` tests. * *spec_set*: A stricter variant of *spec*. If used, attempting to *set* @@ -402,6 +401,8 @@ the *new_callable* argument to :func:`patch`. The reset_mock method resets all the call attributes on a mock object: + .. doctest:: + >>> mock = Mock(return_value=None) >>> mock('hello') >>> mock.called @@ -410,20 +411,41 @@ the *new_callable* argument to :func:`patch`. >>> mock.called False - .. versionchanged:: 3.6 - Added two keyword-only arguments to the reset_mock function. - This can be useful where you want to make a series of assertions that - reuse the same object. Note that :meth:`reset_mock` *doesn't* clear the - return value, :attr:`side_effect` or any child attributes you have - set using normal assignment by default. In case you want to reset - *return_value* or :attr:`side_effect`, then pass the corresponding - parameter as ``True``. Child mocks and the return value mock - (if any) are reset as well. + reuse the same object. + + *return_value* parameter when set to ``True`` resets :attr:`return_value`: - .. note:: *return_value*, and :attr:`side_effect` are keyword-only - arguments. + .. doctest:: + >>> mock = Mock(return_value=5) + >>> mock('hello') + 5 + >>> mock.reset_mock(return_value=True) + >>> mock('hello') # doctest: +ELLIPSIS + + + *side_effect* parameter when set to ``True`` resets :attr:`side_effect`: + + .. doctest:: + + >>> mock = Mock(side_effect=ValueError) + >>> mock('hello') + Traceback (most recent call last): + ... + ValueError + >>> mock.reset_mock(side_effect=True) + >>> mock('hello') # doctest: +ELLIPSIS + + + Note that :meth:`reset_mock` *doesn't* clear the + :attr:`return_value`, :attr:`side_effect` or any child attributes you have + set using normal assignment by default. + + Child mocks are reset as well. + + .. versionchanged:: 3.6 + Added two keyword-only arguments to the reset_mock function. .. method:: mock_add_spec(spec, spec_set=False) @@ -738,8 +760,8 @@ the *new_callable* argument to :func:`patch`. .. attribute:: __class__ - Normally the :attr:`__class__` attribute of an object will return its type. - For a mock object with a :attr:`spec`, ``__class__`` returns the spec class + Normally the :attr:`!__class__` attribute of an object will return its type. + For a mock object with a :attr:`!spec`, :attr:`!__class__` returns the spec class instead. This allows mock objects to pass :func:`isinstance` tests for the object they are replacing / masquerading as: @@ -747,7 +769,7 @@ the *new_callable* argument to :func:`patch`. >>> isinstance(mock, int) True - :attr:`__class__` is assignable to, this allows a mock to pass an + :attr:`!__class__` is assignable to, this allows a mock to pass an :func:`isinstance` check without forcing you to use a spec: >>> mock = Mock() @@ -761,8 +783,8 @@ the *new_callable* argument to :func:`patch`. meaning of :class:`Mock`, with the exception of *return_value* and *side_effect* which have no meaning on a non-callable mock. -Mock objects that use a class or an instance as a :attr:`spec` or -:attr:`spec_set` are able to pass :func:`isinstance` tests: +Mock objects that use a class or an instance as a :attr:`!spec` or +:attr:`!spec_set` are able to pass :func:`isinstance` tests: >>> mock = Mock(spec=SomeClass) >>> isinstance(mock, SomeClass) @@ -861,6 +883,20 @@ object:: 3 >>> p.assert_called_once_with() +.. caution:: + + If an :exc:`AttributeError` is raised by :class:`PropertyMock`, + it will be interpreted as a missing descriptor and + :meth:`~object.__getattr__` will be called on the parent mock:: + + >>> m = MagicMock() + >>> no_attribute = PropertyMock(side_effect=AttributeError) + >>> type(m).my_property = no_attribute + >>> m.my_property + + + See :meth:`~object.__getattr__` for details. + .. class:: AsyncMock(spec=None, side_effect=None, return_value=DEFAULT, wraps=None, name=None, spec_set=None, unsafe=False, **kwargs) @@ -869,7 +905,7 @@ object:: call is an awaitable. >>> mock = AsyncMock() - >>> asyncio.iscoroutinefunction(mock) + >>> inspect.iscoroutinefunction(mock) True >>> inspect.isawaitable(mock()) # doctest: +SKIP True @@ -1162,7 +1198,7 @@ Calls made to the object will be recorded in the attributes like :attr:`~Mock.call_args` and :attr:`~Mock.call_args_list`. If :attr:`~Mock.side_effect` is set then it will be called after the call has -been recorded, so if :attr:`side_effect` raises an exception the call is still +been recorded, so if :attr:`!side_effect` raises an exception the call is still recorded. The simplest way to make a mock raise an exception when called is to make @@ -1183,8 +1219,8 @@ The simplest way to make a mock raise an exception when called is to make >>> m.mock_calls [call(1, 2, 3), call('two', 'three', 'four')] -If :attr:`side_effect` is a function then whatever that function returns is what -calls to the mock return. The :attr:`side_effect` function is called with the +If :attr:`~Mock.side_effect` is a function then whatever that function returns is what +calls to the mock return. The :attr:`!side_effect` function is called with the same arguments as the mock. This allows you to vary the return value of the call dynamically, based on the input: @@ -1201,7 +1237,7 @@ call dynamically, based on the input: If you want the mock to still return the default return value (a new mock), or any set return value, then there are two ways of doing this. Either return -:attr:`mock.return_value` from inside :attr:`side_effect`, or return :data:`DEFAULT`: +:attr:`~Mock.return_value` from inside :attr:`~Mock.side_effect`, or return :data:`DEFAULT`: >>> m = MagicMock() >>> def side_effect(*args, **kwargs): @@ -1218,8 +1254,8 @@ any set return value, then there are two ways of doing this. Either return >>> m() 3 -To remove a :attr:`side_effect`, and return to the default behaviour, set the -:attr:`side_effect` to ``None``: +To remove a :attr:`~Mock.side_effect`, and return to the default behaviour, set the +:attr:`!side_effect` to ``None``: >>> m = MagicMock(return_value=6) >>> def side_effect(*args, **kwargs): @@ -1232,7 +1268,7 @@ To remove a :attr:`side_effect`, and return to the default behaviour, set the >>> m() 6 -The :attr:`side_effect` can also be any iterable object. Repeated calls to the mock +The :attr:`~Mock.side_effect` can also be any iterable object. Repeated calls to the mock will return values from the iterable (until the iterable is exhausted and a :exc:`StopIteration` is raised): @@ -1273,7 +1309,7 @@ objects of any type. You may want a mock object to return ``False`` to a :func:`hasattr` call, or raise an :exc:`AttributeError` when an attribute is fetched. You can do this by providing -an object as a :attr:`spec` for a mock, but that isn't always convenient. +an object as a :attr:`!spec` for a mock, but that isn't always convenient. You "block" attributes by deleting them. Once deleted, accessing an attribute will raise an :exc:`AttributeError`. @@ -1442,7 +1478,7 @@ patch If you are patching builtins in a module then you don't need to pass ``create=True``, it will be added by default. - Patch can be used as a :class:`TestCase` class decorator. It works by + Patch can be used as a :class:`~unittest.TestCase` class decorator. It works by decorating each test method in the class. This reduces the boilerplate code when your test methods share a common patchings set. :func:`patch` finds tests by looking for method names that start with ``patch.TEST_PREFIX``. @@ -1480,7 +1516,7 @@ If the class is instantiated multiple times you could use can set the *return_value* to be anything you want. To configure return values on methods of *instances* on the patched class -you must do this on the :attr:`return_value`. For example:: +you must do this on the :attr:`~Mock.return_value`. For example:: >>> class Class: ... def method(self): @@ -1625,7 +1661,8 @@ patch.dict .. function:: patch.dict(in_dict, values=(), clear=False, **kwargs) Patch a dictionary, or dictionary like object, and restore the dictionary - to its original state after the test. + to its original state after the test, where the restored dictionary is a + copy of the dictionary as it was before the test. *in_dict* can be a dictionary or a mapping like container. If it is a mapping then it must at least support getting, setting and deleting items @@ -1801,13 +1838,13 @@ context manager is a dictionary where created mocks are keyed by name:: patch methods: start and stop ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -All the patchers have :meth:`start` and :meth:`stop` methods. These make it simpler to do +All the patchers have :meth:`!start` and :meth:`!stop` methods. These make it simpler to do patching in ``setUp`` methods or where you want to do multiple patches without nesting decorators or with statements. To use them call :func:`patch`, :func:`patch.object` or :func:`patch.dict` as normal and keep a reference to the returned ``patcher`` object. You can then -call :meth:`start` to put the patch in place and :meth:`stop` to undo it. +call :meth:`!start` to put the patch in place and :meth:`!stop` to undo it. If you are using :func:`patch` to create a mock for you then it will be returned by the call to ``patcher.start``. :: @@ -1824,7 +1861,7 @@ the call to ``patcher.start``. :: A typical use case for this might be for doing multiple patches in the ``setUp`` -method of a :class:`TestCase`:: +method of a :class:`~unittest.TestCase`:: >>> class MyTest(unittest.TestCase): ... def setUp(self): @@ -2497,7 +2534,7 @@ behaviour you can switch it off by setting the module level switch Alternatively you can just use ``vars(my_mock)`` (instance members) and ``dir(type(my_mock))`` (type members) to bypass the filtering irrespective of -:const:`mock.FILTER_DIR`. +:const:`FILTER_DIR`. mock_open @@ -2512,7 +2549,7 @@ mock_open default) then a :class:`MagicMock` will be created for you, with the API limited to methods or attributes available on standard file handles. - *read_data* is a string for the :meth:`~io.IOBase.read`, + *read_data* is a string for the :meth:`~io.RawIOBase.read`, :meth:`~io.IOBase.readline`, and :meth:`~io.IOBase.readlines` methods of the file handle to return. Calls to those methods will take data from *read_data* until it is depleted. The mock of these methods is pretty @@ -2524,7 +2561,7 @@ mock_open .. versionchanged:: 3.4 Added :meth:`~io.IOBase.readline` and :meth:`~io.IOBase.readlines` support. - The mock of :meth:`~io.IOBase.read` changed to consume *read_data* rather + The mock of :meth:`~io.RawIOBase.read` changed to consume *read_data* rather than returning it on each call. .. versionchanged:: 3.5 @@ -2576,7 +2613,7 @@ And for reading files:: Autospeccing ~~~~~~~~~~~~ -Autospeccing is based on the existing :attr:`spec` feature of mock. It limits the +Autospeccing is based on the existing :attr:`!spec` feature of mock. It limits the api of mocks to the api of an original object (the spec), but it is recursive (implemented lazily) so that attributes of mocks only have the same api as the attributes of the spec. In addition mocked functions / methods have the @@ -2585,48 +2622,24 @@ called incorrectly. Before I explain how auto-speccing works, here's why it is needed. -:class:`Mock` is a very powerful and flexible object, but it suffers from two flaws -when used to mock out objects from a system under test. One of these flaws is -specific to the :class:`Mock` api and the other is a more general problem with using -mock objects. - -First the problem specific to :class:`Mock`. :class:`Mock` has two assert methods that are -extremely handy: :meth:`~Mock.assert_called_with` and -:meth:`~Mock.assert_called_once_with`. +:class:`Mock` is a very powerful and flexible object, but it suffers from a flaw which +is general to mocking. If you refactor some of your code, rename members and so on, any +tests for code that is still using the *old api* but uses mocks instead of the real +objects will still pass. This means your tests can all pass even though your code is +broken. - >>> mock = Mock(name='Thing', return_value=None) - >>> mock(1, 2, 3) - >>> mock.assert_called_once_with(1, 2, 3) - >>> mock(1, 2, 3) - >>> mock.assert_called_once_with(1, 2, 3) - Traceback (most recent call last): - ... - AssertionError: Expected 'mock' to be called once. Called 2 times. - -Because mocks auto-create attributes on demand, and allow you to call them -with arbitrary arguments, if you misspell one of these assert methods then -your assertion is gone: - -.. code-block:: pycon - - >>> mock = Mock(name='Thing', return_value=None) - >>> mock(1, 2, 3) - >>> mock.assret_called_once_with(4, 5, 6) # Intentional typo! - -Your tests can pass silently and incorrectly because of the typo. +.. versionchanged:: 3.5 -The second issue is more general to mocking. If you refactor some of your -code, rename members and so on, any tests for code that is still using the -*old api* but uses mocks instead of the real objects will still pass. This -means your tests can all pass even though your code is broken. + Before 3.5, tests with a typo in the word assert would silently pass when they should + raise an error. You can still achieve this behavior by passing ``unsafe=True`` to Mock. Note that this is another reason why you need integration tests as well as unit tests. Testing everything in isolation is all fine and dandy, but if you don't test how your units are "wired together" there is still lots of room for bugs that tests might have caught. -:mod:`mock` already provides a feature to help with this, called speccing. If you -use a class or instance as the :attr:`spec` for a mock then you can only access +:mod:`unittest.mock` already provides a feature to help with this, called speccing. If you +use a class or instance as the :attr:`!spec` for a mock then you can only access attributes on the mock that exist on the real class: >>> from urllib import request @@ -2664,7 +2677,7 @@ Here's an example of it in use:: >>> mock_request.Request -You can see that :class:`request.Request` has a spec. :class:`request.Request` takes two +You can see that :class:`!request.Request` has a spec. :class:`!request.Request` takes two arguments in the constructor (one of which is *self*). Here's what happens if we try to call it incorrectly:: @@ -2680,8 +2693,8 @@ specced mocks):: >>> req -:class:`Request` objects are not callable, so the return value of instantiating our -mocked out :class:`request.Request` is a non-callable mock. With the spec in place +:class:`!Request` objects are not callable, so the return value of instantiating our +mocked out :class:`!request.Request` is a non-callable mock. With the spec in place any typos in our asserts will raise the correct error:: >>> req.add_header('spam', 'eggs') @@ -2831,3 +2844,123 @@ Sealing mocks >>> mock.not_submock.attribute2 # This won't raise. .. versionadded:: 3.7 + + +Order of precedence of :attr:`!side_effect`, :attr:`!return_value` and *wraps* +------------------------------------------------------------------------------ + +The order of their precedence is: + +1. :attr:`~Mock.side_effect` +2. :attr:`~Mock.return_value` +3. *wraps* + +If all three are set, mock will return the value from :attr:`~Mock.side_effect`, +ignoring :attr:`~Mock.return_value` and the wrapped object altogether. If any +two are set, the one with the higher precedence will return the value. +Regardless of the order of which was set first, the order of precedence +remains unchanged. + + >>> from unittest.mock import Mock + >>> class Order: + ... @staticmethod + ... def get_value(): + ... return "third" + ... + >>> order_mock = Mock(spec=Order, wraps=Order) + >>> order_mock.get_value.side_effect = ["first"] + >>> order_mock.get_value.return_value = "second" + >>> order_mock.get_value() + 'first' + +As ``None`` is the default value of :attr:`~Mock.side_effect`, if you reassign +its value back to ``None``, the order of precedence will be checked between +:attr:`~Mock.return_value` and the wrapped object, ignoring +:attr:`~Mock.side_effect`. + + >>> order_mock.get_value.side_effect = None + >>> order_mock.get_value() + 'second' + +If the value being returned by :attr:`~Mock.side_effect` is :data:`DEFAULT`, +it is ignored and the order of precedence moves to the successor to obtain the +value to return. + + >>> from unittest.mock import DEFAULT + >>> order_mock.get_value.side_effect = [DEFAULT] + >>> order_mock.get_value() + 'second' + +When :class:`Mock` wraps an object, the default value of +:attr:`~Mock.return_value` will be :data:`DEFAULT`. + + >>> order_mock = Mock(spec=Order, wraps=Order) + >>> order_mock.return_value + sentinel.DEFAULT + >>> order_mock.get_value.return_value + sentinel.DEFAULT + +The order of precedence will ignore this value and it will move to the last +successor which is the wrapped object. + +As the real call is being made to the wrapped object, creating an instance of +this mock will return the real instance of the class. The positional arguments, +if any, required by the wrapped object must be passed. + + >>> order_mock_instance = order_mock() + >>> isinstance(order_mock_instance, Order) + True + >>> order_mock_instance.get_value() + 'third' + + >>> order_mock.get_value.return_value = DEFAULT + >>> order_mock.get_value() + 'third' + + >>> order_mock.get_value.return_value = "second" + >>> order_mock.get_value() + 'second' + +But if you assign ``None`` to it, this will not be ignored as it is an +explicit assignment. So, the order of precedence will not move to the wrapped +object. + + >>> order_mock.get_value.return_value = None + >>> order_mock.get_value() is None + True + +Even if you set all three at once when initializing the mock, the order of +precedence remains the same: + + >>> order_mock = Mock(spec=Order, wraps=Order, + ... **{"get_value.side_effect": ["first"], + ... "get_value.return_value": "second"} + ... ) + ... + >>> order_mock.get_value() + 'first' + >>> order_mock.get_value.side_effect = None + >>> order_mock.get_value() + 'second' + >>> order_mock.get_value.return_value = DEFAULT + >>> order_mock.get_value() + 'third' + +If :attr:`~Mock.side_effect` is exhausted, the order of precedence will not +cause a value to be obtained from the successors. Instead, ``StopIteration`` +exception is raised. + + >>> order_mock = Mock(spec=Order, wraps=Order) + >>> order_mock.get_value.side_effect = ["first side effect value", + ... "another side effect value"] + >>> order_mock.get_value.return_value = "second" + + >>> order_mock.get_value() + 'first side effect value' + >>> order_mock.get_value() + 'another side effect value' + + >>> order_mock.get_value() + Traceback (most recent call last): + ... + StopIteration diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index e6140ac70eb87a..5bb7f88d585e3b 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1,5 +1,5 @@ -:mod:`unittest` --- Unit testing framework -========================================== +:mod:`!unittest` --- Unit testing framework +=========================================== .. module:: unittest :synopsis: Unit testing framework for Python. @@ -46,7 +46,6 @@ test runner a textual interface, or return a special value to indicate the results of executing the tests. - .. seealso:: Module :mod:`doctest` @@ -198,6 +197,9 @@ For a list of all the command-line options:: In earlier versions it was only possible to run individual test methods and not modules or classes. +.. versionadded:: 3.14 + Output is colorized by default and can be + :ref:`controlled using environment variables `. Command-line options ~~~~~~~~~~~~~~~~~~~~ @@ -340,28 +342,21 @@ Test modules and packages can customize test loading and discovery by through the `load_tests protocol`_. .. versionchanged:: 3.4 - Test discovery supports :term:`namespace packages ` - for the start directory. Note that you need to specify the top level - directory too (e.g. - ``python -m unittest discover -s root/namespace -t root``). + Test discovery supports :term:`namespace packages `. .. versionchanged:: 3.11 - :mod:`unittest` dropped the :term:`namespace packages ` - support in Python 3.11. It has been broken since Python 3.7. Start directory and - subdirectories containing tests must be regular package that have - ``__init__.py`` file. - - Directories containing start directory still can be a namespace package. - In this case, you need to specify start directory as dotted package name, - and target directory explicitly. For example:: + Test discovery dropped the :term:`namespace packages ` + support. It has been broken since Python 3.7. + Start directory and its subdirectories containing tests must be regular + package that have ``__init__.py`` file. - # proj/ <-- current directory - # namespace/ - # mypkg/ - # __init__.py - # test_mypkg.py + If the start directory is the dotted name of the package, the ancestor packages + can be namespace packages. - python -m unittest discover -s namespace.mypkg -t . +.. versionchanged:: 3.14 + Test discovery supports :term:`namespace package` as start directory again. + To avoid scanning directories unrelated to Python, + tests are not searched in subdirectories that do not contain ``__init__.py``. .. _organizing-tests: @@ -888,6 +883,12 @@ Test cases | :meth:`assertNotIsInstance(a, b) | ``not isinstance(a, b)`` | 3.2 | | ` | | | +-----------------------------------------+-----------------------------+---------------+ + | :meth:`assertIsSubclass(a, b) | ``issubclass(a, b)`` | 3.14 | + | ` | | | + +-----------------------------------------+-----------------------------+---------------+ + | :meth:`assertNotIsSubclass(a, b) | ``not issubclass(a, b)`` | 3.14 | + | ` | | | + +-----------------------------------------+-----------------------------+---------------+ All the assert methods accept a *msg* argument that, if specified, is used as the error message on failure (see also :data:`longMessage`). @@ -966,6 +967,15 @@ Test cases .. versionadded:: 3.2 + .. method:: assertIsSubclass(cls, superclass, msg=None) + assertNotIsSubclass(cls, superclass, msg=None) + + Test that *cls* is (or is not) a subclass of *superclass* (which can be a + class or a tuple of classes, as supported by :func:`issubclass`). + To check for the exact type, use :func:`assertIs(cls, superclass) `. + + .. versionadded:: 3.14 + It is also possible to check the production of exceptions, warnings, and log messages using the following methods: @@ -1215,6 +1225,24 @@ Test cases | ` | elements in the same number, | | | | regardless of their order. | | +---------------------------------------+--------------------------------+--------------+ + | :meth:`assertStartsWith(a, b) | ``a.startswith(b)`` | 3.14 | + | ` | | | + +---------------------------------------+--------------------------------+--------------+ + | :meth:`assertNotStartsWith(a, b) | ``not a.startswith(b)`` | 3.14 | + | ` | | | + +---------------------------------------+--------------------------------+--------------+ + | :meth:`assertEndsWith(a, b) | ``a.endswith(b)`` | 3.14 | + | ` | | | + +---------------------------------------+--------------------------------+--------------+ + | :meth:`assertNotEndsWith(a, b) | ``not a.endswith(b)`` | 3.14 | + | ` | | | + +---------------------------------------+--------------------------------+--------------+ + | :meth:`assertHasAttr(a, b) | ``hastattr(a, b)`` | 3.14 | + | ` | | | + +---------------------------------------+--------------------------------+--------------+ + | :meth:`assertNotHasAttr(a, b) | ``not hastattr(a, b)`` | 3.14 | + | ` | | | + +---------------------------------------+--------------------------------+--------------+ .. method:: assertAlmostEqual(first, second, places=7, msg=None, delta=None) @@ -1284,6 +1312,34 @@ Test cases .. versionadded:: 3.2 + .. method:: assertStartsWith(s, prefix, msg=None) + .. method:: assertNotStartsWith(s, prefix, msg=None) + + Test that the Unicode or byte string *s* starts (or does not start) + with a *prefix*. + *prefix* can also be a tuple of strings to try. + + .. versionadded:: 3.14 + + + .. method:: assertEndsWith(s, suffix, msg=None) + .. method:: assertNotEndsWith(s, suffix, msg=None) + + Test that the Unicode or byte string *s* ends (or does not end) + with a *suffix*. + *suffix* can also be a tuple of strings to try. + + .. versionadded:: 3.14 + + + .. method:: assertHasAttr(obj, name, msg=None) + .. method:: assertNotHasAttr(obj, name, msg=None) + + Test that the object *obj* has (or has not) an attribute *name*. + + .. versionadded:: 3.14 + + .. _type-specific-methods: The :meth:`assertEqual` method dispatches the equality check for objects of @@ -1880,8 +1936,8 @@ Loading and running tests Python identifiers) will be loaded. All test modules must be importable from the top level of the project. If - the start directory is not the top level directory then the top level - directory must be specified separately. + the start directory is not the top level directory then *top_level_dir* + must be specified separately. If importing a module fails, for example due to a syntax error, then this will be recorded as a single error and discovery will continue. If @@ -1901,9 +1957,11 @@ Loading and running tests package. The pattern is deliberately not stored as a loader attribute so that - packages can continue discovery themselves. *top_level_dir* is stored so - ``load_tests`` does not need to pass this argument in to - ``loader.discover()``. + packages can continue discovery themselves. + + *top_level_dir* is stored internally, and used as a default to any + nested calls to ``discover()``. That is, if a package's ``load_tests`` + calls ``loader.discover()``, it does not need to pass this argument. *start_dir* can be a dotted module name as well as a directory. @@ -1913,10 +1971,8 @@ Loading and running tests Modules that raise :exc:`SkipTest` on import are recorded as skips, not errors. - .. versionchanged:: 3.4 *start_dir* can be a :term:`namespace packages `. - .. versionchanged:: 3.4 Paths are sorted before being imported so that execution order is the same even if the underlying file system's ordering is not dependent on file name. @@ -1928,8 +1984,13 @@ Loading and running tests .. versionchanged:: 3.11 *start_dir* can not be a :term:`namespace packages `. - It has been broken since Python 3.7 and Python 3.11 officially remove it. + It has been broken since Python 3.7, and Python 3.11 officially removes it. + + .. versionchanged:: 3.13 + *top_level_dir* is only stored for the duration of *discover* call. + .. versionchanged:: 3.14 + *start_dir* can once again be a :term:`namespace package`. The following attributes of a :class:`TestLoader` can be configured either by subclassing or assignment on an instance: @@ -2311,8 +2372,8 @@ Loading and running tests (see :ref:`Warning control `), otherwise it will be set to ``'default'``. - Calling ``main`` actually returns an instance of the ``TestProgram`` class. - This stores the result of the tests run as the ``result`` attribute. + Calling ``main`` returns an object with the ``result`` attribute that contains + the result of the tests run as a :class:`unittest.TestResult`. .. versionchanged:: 3.1 The *exit* parameter was added. @@ -2524,7 +2585,7 @@ Signal Handling .. versionadded:: 3.2 The :option:`-c/--catch ` command-line option to unittest, -along with the ``catchbreak`` parameter to :func:`unittest.main()`, provide +along with the ``catchbreak`` parameter to :func:`unittest.main`, provide more friendly handling of control-C during a test run. With catch break behavior enabled control-C will allow the currently running test to complete, and the test run will then end and report all the results so far. A second diff --git a/Doc/library/urllib.error.rst b/Doc/library/urllib.error.rst index facb11f42a40c5..1686ddd09caa48 100644 --- a/Doc/library/urllib.error.rst +++ b/Doc/library/urllib.error.rst @@ -1,5 +1,5 @@ -:mod:`urllib.error` --- Exception classes raised by urllib.request -================================================================== +:mod:`!urllib.error` --- Exception classes raised by urllib.request +=================================================================== .. module:: urllib.error :synopsis: Exception classes raised by urllib.request. diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 3c898c3e826304..44a9c79cba2216 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -1,5 +1,5 @@ -:mod:`urllib.parse` --- Parse URLs into components -================================================== +:mod:`!urllib.parse` --- Parse URLs into components +=================================================== .. module:: urllib.parse :synopsis: Parse URLs into or assemble them from components. @@ -22,15 +22,28 @@ to an absolute URL given a "base URL." The module has been designed to match the internet RFC on Relative Uniform Resource Locators. It supports the following URL schemes: ``file``, ``ftp``, -``gopher``, ``hdl``, ``http``, ``https``, ``imap``, ``mailto``, ``mms``, +``gopher``, ``hdl``, ``http``, ``https``, ``imap``, ``itms-services``, ``mailto``, ``mms``, ``news``, ``nntp``, ``prospero``, ``rsync``, ``rtsp``, ``rtsps``, ``rtspu``, ``sftp``, ``shttp``, ``sip``, ``sips``, ``snews``, ``svn``, ``svn+ssh``, ``telnet``, ``wais``, ``ws``, ``wss``. +.. impl-detail:: + + The inclusion of the ``itms-services`` URL scheme can prevent an app from + passing Apple's App Store review process for the macOS and iOS App Stores. + Handling for the ``itms-services`` scheme is always removed on iOS; on + macOS, it *may* be removed if CPython has been built with the + :option:`--with-app-store-compliance` option. + The :mod:`urllib.parse` module defines functions that fall into two broad categories: URL parsing and URL quoting. These are covered in detail in the following sections. +This module's functions use the deprecated term ``netloc`` (or ``net_loc``), +which was introduced in :rfc:`1808`. However, this term has been obsoleted by +:rfc:`3986`, which introduced the term ``authority`` as its replacement. +The use of ``netloc`` is continued for backward compatibility. + URL Parsing ----------- @@ -168,7 +181,7 @@ or on combining URL components into a URL string. Added IPv6 URL parsing capabilities. .. versionchanged:: 3.3 - The fragment is now parsed for all URL schemes (unless *allow_fragment* is + The fragment is now parsed for all URL schemes (unless *allow_fragments* is false), in accordance with :rfc:`3986`. Previously, an allowlist of schemes that support fragments existed. @@ -226,6 +239,10 @@ or on combining URL components into a URL string. query parameter separator. This has been changed to allow only a single separator key, with ``&`` as the default separator. + .. deprecated:: 3.14 + Accepting objects with false values (like ``0`` and ``[]``) except empty + strings and byte-like objects and ``None`` is now deprecated. + .. function:: parse_qsl(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None, separator='&') @@ -390,6 +407,15 @@ or on combining URL components into a URL string. If you do not want that behavior, preprocess the *url* with :func:`urlsplit` and :func:`urlunsplit`, removing possible *scheme* and *netloc* parts. + .. warning:: + + Because an absolute URL may be passed as the ``url`` parameter, it is + generally **not secure** to use ``urljoin`` with an attacker-controlled + ``url``. For example in, + ``urljoin("https://website.com/users/", username)``, if ``username`` can + contain an absolute URL, the result of ``urljoin`` will be the absolute + URL. + .. versionchanged:: 3.5 @@ -732,6 +758,10 @@ task isn't already covered by the URL parsing functions above. .. versionchanged:: 3.5 Added the *quote_via* parameter. + .. deprecated:: 3.14 + Accepting objects with false values (like ``0`` and ``[]``) except empty + strings and byte-like objects and ``None`` is now deprecated. + .. seealso:: diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index affdce144cd5fc..b3efde3f189566 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -1,5 +1,5 @@ -:mod:`urllib.request` --- Extensible library for opening URLs -============================================================= +:mod:`!urllib.request` --- Extensible library for opening URLs +============================================================== .. module:: urllib.request :synopsis: Extensible library for opening URLs. @@ -67,8 +67,7 @@ The :mod:`urllib.request` module defines the following functions: the response headers as it is specified in the documentation for :class:`~http.client.HTTPResponse`. - For FTP, file, and data URLs and requests explicitly handled by legacy - :class:`URLopener` and :class:`FancyURLopener` classes, this function + For FTP, file, and data URLs, this function returns a :class:`urllib.response.addinfourl` object. Raises :exc:`~urllib.error.URLError` on protocol errors. @@ -97,11 +96,9 @@ The :mod:`urllib.request` module defines the following functions: .. versionchanged:: 3.2 *cafile* and *capath* were added. - .. versionchanged:: 3.2 HTTPS virtual hosts are now supported if possible (that is, if :const:`ssl.HAS_SNI` is true). - .. versionadded:: 3.2 *data* can be an iterable object. .. versionchanged:: 3.3 @@ -115,9 +112,9 @@ The :mod:`urllib.request` module defines the following functions: ``http/1.1`` when no *context* is given. Custom *context* should set ALPN protocols with :meth:`~ssl.SSLContext.set_alpn_protocols`. - .. versionchanged:: 3.13 - Remove *cafile*, *capath* and *cadefault* parameters: use the *context* - parameter instead. + .. versionchanged:: 3.13 + Remove *cafile*, *capath* and *cadefault* parameters: use the *context* + parameter instead. .. function:: install_opener(opener) @@ -151,16 +148,42 @@ The :mod:`urllib.request` module defines the following functions: .. function:: pathname2url(path) - Convert the pathname *path* from the local syntax for a path to the form used in - the path component of a URL. This does not produce a complete URL. The return - value will already be quoted using the :func:`~urllib.parse.quote` function. + Convert the given local path to a ``file:`` URL. This function uses + :func:`~urllib.parse.quote` function to encode the path. For historical + reasons, the return value omits the ``file:`` scheme prefix. This example + shows the function being used on Windows:: + + >>> from urllib.request import pathname2url + >>> path = 'C:\\Program Files' + >>> 'file:' + pathname2url(path) + 'file:///C:/Program%20Files' + + .. versionchanged:: 3.14 + Paths beginning with a slash are converted to URLs with authority + sections. For example, the path ``/etc/hosts`` is converted to + the URL ``///etc/hosts``. + + .. versionchanged:: 3.14 + Windows drive letters are no longer converted to uppercase, and ``:`` + characters not following a drive letter no longer cause an + :exc:`OSError` exception to be raised on Windows. + + +.. function:: url2pathname(url) + Convert the given ``file:`` URL to a local path. This function uses + :func:`~urllib.parse.unquote` to decode the URL. For historical reasons, + the given value *must* omit the ``file:`` scheme prefix. This example shows + the function being used on Windows:: -.. function:: url2pathname(path) + >>> from urllib.request import url2pathname + >>> url = 'file:///C:/Program%20Files' + >>> url2pathname(url.removeprefix('file:')) + 'C:\\Program Files' + + .. versionchanged:: 3.14 + Windows drive letters are no longer converted to uppercase. - Convert the path component *path* from a percent-encoded URL to the local syntax for a - path. This does not accept a complete URL. This function uses - :func:`~urllib.parse.unquote` to decode *path*. .. function:: getproxies() @@ -220,7 +243,7 @@ The following classes are provided: An appropriate ``Content-Type`` header should be included if the *data* argument is present. If this header has not been provided and *data* - is not None, ``Content-Type: application/x-www-form-urlencoded`` will + is not ``None``, ``Content-Type: application/x-www-form-urlencoded`` will be added as a default. The next two arguments are only of interest for correct handling @@ -243,7 +266,7 @@ The following classes are provided: *method* should be a string that indicates the HTTP request method that will be used (e.g. ``'HEAD'``). If provided, its value is stored in the - :attr:`~Request.method` attribute and is used by :meth:`get_method()`. + :attr:`~Request.method` attribute and is used by :meth:`get_method`. The default is ``'GET'`` if *data* is ``None`` or ``'POST'`` otherwise. Subclasses may indicate a different default method by setting the :attr:`~Request.method` attribute in the class itself. @@ -388,6 +411,9 @@ The following classes are provided: :ref:`http-password-mgr` for information on the interface that must be supported. + .. versionchanged:: 3.14 + Added support for HTTP digest authentication algorithm ``SHA-256``. + .. class:: HTTPDigestAuthHandler(password_mgr=None) @@ -1094,7 +1120,7 @@ FileHandler Objects .. versionchanged:: 3.2 This method is applicable only for local hostnames. When a remote - hostname is given, an :exc:`~urllib.error.URLError` is raised. + hostname is given, a :exc:`~urllib.error.URLError` is raised. .. _data-handler-objects: @@ -1109,7 +1135,7 @@ DataHandler Objects ignores white spaces in base64 encoded data URLs so the URL may be wrapped in whatever source file it comes from. But even though some browsers don't mind about a missing padding at the end of a base64 encoded data URL, this - implementation will raise an :exc:`ValueError` in that case. + implementation will raise a :exc:`ValueError` in that case. .. _ftp-handler-objects: @@ -1341,7 +1367,7 @@ environment settings:: >>> import urllib.request >>> proxies = {'http': 'http://proxy.example.com:8080/'} - >>> opener = urllib.request.FancyURLopener(proxies) + >>> opener = urllib.request.build_opener(urllib.request.ProxyHandler(proxies)) >>> with opener.open("http://www.python.org") as f: ... f.read().decode('utf-8') ... @@ -1349,7 +1375,7 @@ environment settings:: The following example uses no proxies at all, overriding environment settings:: >>> import urllib.request - >>> opener = urllib.request.FancyURLopener({}) + >>> opener = urllib.request.build_opener(urllib.request.ProxyHandler({}})) >>> with opener.open("http://www.python.org/") as f: ... f.read().decode('utf-8') ... @@ -1414,121 +1440,6 @@ some point in the future. Cleans up temporary files that may have been left behind by previous calls to :func:`urlretrieve`. -.. class:: URLopener(proxies=None, **x509) - - .. deprecated:: 3.3 - - Base class for opening and reading URLs. Unless you need to support opening - objects using schemes other than :file:`http:`, :file:`ftp:`, or :file:`file:`, - you probably want to use :class:`FancyURLopener`. - - By default, the :class:`URLopener` class sends a :mailheader:`User-Agent` header - of ``urllib/VVV``, where *VVV* is the :mod:`urllib` version number. - Applications can define their own :mailheader:`User-Agent` header by subclassing - :class:`URLopener` or :class:`FancyURLopener` and setting the class attribute - :attr:`version` to an appropriate string value in the subclass definition. - - The optional *proxies* parameter should be a dictionary mapping scheme names to - proxy URLs, where an empty dictionary turns proxies off completely. Its default - value is ``None``, in which case environmental proxy settings will be used if - present, as discussed in the definition of :func:`urlopen`, above. - - Additional keyword parameters, collected in *x509*, may be used for - authentication of the client when using the :file:`https:` scheme. The keywords - *key_file* and *cert_file* are supported to provide an SSL key and certificate; - both are needed to support client authentication. - - :class:`URLopener` objects will raise an :exc:`OSError` exception if the server - returns an error code. - - .. method:: open(fullurl, data=None) - - Open *fullurl* using the appropriate protocol. This method sets up cache and - proxy information, then calls the appropriate open method with its input - arguments. If the scheme is not recognized, :meth:`open_unknown` is called. - The *data* argument has the same meaning as the *data* argument of - :func:`urlopen`. - - This method always quotes *fullurl* using :func:`~urllib.parse.quote`. - - .. method:: open_unknown(fullurl, data=None) - - Overridable interface to open unknown URL types. - - - .. method:: retrieve(url, filename=None, reporthook=None, data=None) - - Retrieves the contents of *url* and places it in *filename*. The return value - is a tuple consisting of a local filename and either an - :class:`email.message.Message` object containing the response headers (for remote - URLs) or ``None`` (for local URLs). The caller must then open and read the - contents of *filename*. If *filename* is not given and the URL refers to a - local file, the input filename is returned. If the URL is non-local and - *filename* is not given, the filename is the output of :func:`tempfile.mktemp` - with a suffix that matches the suffix of the last path component of the input - URL. If *reporthook* is given, it must be a function accepting three numeric - parameters: A chunk number, the maximum size chunks are read in and the total size of the download - (-1 if unknown). It will be called once at the start and after each chunk of data is read from the - network. *reporthook* is ignored for local URLs. - - If the *url* uses the :file:`http:` scheme identifier, the optional *data* - argument may be given to specify a ``POST`` request (normally the request type - is ``GET``). The *data* argument must in standard - :mimetype:`application/x-www-form-urlencoded` format; see the - :func:`urllib.parse.urlencode` function. - - - .. attribute:: version - - Variable that specifies the user agent of the opener object. To get - :mod:`urllib` to tell servers that it is a particular user agent, set this in a - subclass as a class variable or in the constructor before calling the base - constructor. - - -.. class:: FancyURLopener(...) - - .. deprecated:: 3.3 - - :class:`FancyURLopener` subclasses :class:`URLopener` providing default handling - for the following HTTP response codes: 301, 302, 303, 307 and 401. For the 30x - response codes listed above, the :mailheader:`Location` header is used to fetch - the actual URL. For 401 response codes (authentication required), basic HTTP - authentication is performed. For the 30x response codes, recursion is bounded - by the value of the *maxtries* attribute, which defaults to 10. - - For all other response codes, the method :meth:`~BaseHandler.http_error_default` is called - which you can override in subclasses to handle the error appropriately. - - .. note:: - - According to the letter of :rfc:`2616`, 301 and 302 responses to POST requests - must not be automatically redirected without confirmation by the user. In - reality, browsers do allow automatic redirection of these responses, changing - the POST to a GET, and :mod:`urllib` reproduces this behaviour. - - The parameters to the constructor are the same as those for :class:`URLopener`. - - .. note:: - - When performing basic authentication, a :class:`FancyURLopener` instance calls - its :meth:`prompt_user_passwd` method. The default implementation asks the - users for the required information on the controlling terminal. A subclass may - override this method to support more appropriate behavior if needed. - - The :class:`FancyURLopener` class offers one additional method that should be - overloaded to provide the appropriate behavior: - - .. method:: prompt_user_passwd(host, realm) - - Return information needed to authenticate the user at the given host in the - specified security realm. The return value should be a tuple, ``(user, - password)``, which can be used for basic authentication. - - The implementation prompts for this information on the terminal; an application - should override this method to use an appropriate interaction model in the local - environment. - :mod:`urllib.request` Restrictions ---------------------------------- @@ -1580,8 +1491,7 @@ some point in the future. you try to fetch a file whose read permissions make it inaccessible; the FTP code will try to read it, fail with a 550 error, and then perform a directory listing for the unreadable file. If fine-grained control is needed, consider - using the :mod:`ftplib` module, subclassing :class:`FancyURLopener`, or changing - *_urlopener* to meet your needs. + using the :mod:`ftplib` module. diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index b5a49d9c592387..016fcdc75da67a 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -1,5 +1,5 @@ -:mod:`urllib.robotparser` --- Parser for robots.txt -==================================================== +:mod:`!urllib.robotparser` --- Parser for robots.txt +===================================================== .. module:: urllib.robotparser :synopsis: Load a robots.txt file and answer questions about diff --git a/Doc/library/urllib.rst b/Doc/library/urllib.rst index 624e164625556a..7d9f39ef070fc3 100644 --- a/Doc/library/urllib.rst +++ b/Doc/library/urllib.rst @@ -1,5 +1,5 @@ -:mod:`urllib` --- URL handling modules -====================================== +:mod:`!urllib` --- URL handling modules +======================================= .. module:: urllib diff --git a/Doc/library/uu.rst b/Doc/library/uu.rst new file mode 100644 index 00000000000000..0636d180294d47 --- /dev/null +++ b/Doc/library/uu.rst @@ -0,0 +1,15 @@ +:mod:`!uu` --- Encode and decode uuencode files +=============================================== + +.. module:: uu + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!uu` module was +`Python 3.12 `_. diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst index e2d231da38fd9a..09b4d4ac4ffd56 100644 --- a/Doc/library/uuid.rst +++ b/Doc/library/uuid.rst @@ -1,8 +1,8 @@ -:mod:`uuid` --- UUID objects according to :rfc:`4122` -===================================================== +:mod:`!uuid` --- UUID objects according to :rfc:`9562` +====================================================== .. module:: uuid - :synopsis: UUID objects (universally unique identifiers) according to RFC 4122 + :synopsis: UUID objects (universally unique identifiers) according to RFC 9562 .. moduleauthor:: Ka-Ping Yee .. sectionauthor:: George Yoshida @@ -11,8 +11,9 @@ -------------- This module provides immutable :class:`UUID` objects (the :class:`UUID` class) -and the functions :func:`uuid1`, :func:`uuid3`, :func:`uuid4`, :func:`uuid5` for -generating version 1, 3, 4, and 5 UUIDs as specified in :rfc:`4122`. +and the functions :func:`uuid1`, :func:`uuid3`, :func:`uuid4`, :func:`uuid5`, +and :func:`uuid.uuid8` for generating version 1, 3, 4, 5, and 8 UUIDs as +specified in :rfc:`9562` (which supersedes :rfc:`4122`). If all you want is a unique ID, you should probably call :func:`uuid1` or :func:`uuid4`. Note that :func:`uuid1` may compromise privacy since it creates @@ -65,7 +66,7 @@ which relays any information about the UUID's safety, using this enumeration: Exactly one of *hex*, *bytes*, *bytes_le*, *fields*, or *int* must be given. The *version* argument is optional; if given, the resulting UUID will have its - variant and version number set according to :rfc:`4122`, overriding bits in the + variant and version number set according to :rfc:`9562`, overriding bits in the given *hex*, *bytes*, *bytes_le*, *fields*, or *int*. Comparison of UUID objects are made by way of comparing their @@ -137,7 +138,7 @@ which relays any information about the UUID's safety, using this enumeration: .. attribute:: UUID.urn - The UUID as a URN as specified in :rfc:`4122`. + The UUID as a URN as specified in :rfc:`9562`. .. attribute:: UUID.variant @@ -149,9 +150,13 @@ which relays any information about the UUID's safety, using this enumeration: .. attribute:: UUID.version - The UUID version number (1 through 5, meaningful only when the variant is + The UUID version number (1 through 8, meaningful only when the variant is :const:`RFC_4122`). + .. versionchanged:: 3.14 + Added UUID version 8. + + .. attribute:: UUID.is_safe An enumeration of :class:`SafeUUID` which indicates whether the platform @@ -216,6 +221,23 @@ The :mod:`uuid` module defines the following functions: .. index:: single: uuid5 + +.. function:: uuid8(a=None, b=None, c=None) + + Generate a pseudo-random UUID according to + :rfc:`RFC 9562, §5.8 <9562#section-5.8>`. + + When specified, the parameters *a*, *b* and *c* are expected to be + positive integers of 48, 12 and 62 bits respectively. If they exceed + their expected bit count, only their least significant bits are kept; + non-specified arguments are substituted for a pseudo-random integer of + appropriate size. + + .. versionadded:: 3.14 + +.. index:: single: uuid8 + + The :mod:`uuid` module defines the following namespace identifiers for use with :func:`uuid3` or :func:`uuid5`. @@ -252,7 +274,9 @@ of the :attr:`~UUID.variant` attribute: .. data:: RFC_4122 - Specifies the UUID layout given in :rfc:`4122`. + Specifies the UUID layout given in :rfc:`4122`. This constant is kept + for backward compatibility even though :rfc:`4122` has been superseded + by :rfc:`9562`. .. data:: RESERVED_MICROSOFT @@ -265,9 +289,28 @@ of the :attr:`~UUID.variant` attribute: Reserved for future definition. +The :mod:`uuid` module defines the special Nil and Max UUID values: + + +.. data:: NIL + + A special form of UUID that is specified to have all 128 bits set to zero + according to :rfc:`RFC 9562, §5.9 <9562#section-5.9>`. + + .. versionadded:: next + + +.. data:: MAX + + A special form of UUID that is specified to have all 128 bits set to one + according to :rfc:`RFC 9562, §5.10 <9562#section-5.10>`. + + .. versionadded:: next + + .. seealso:: - :rfc:`4122` - A Universally Unique IDentifier (UUID) URN Namespace + :rfc:`9562` - A Universally Unique IDentifier (UUID) URN Namespace This specification defines a Uniform Resource Name namespace for UUIDs, the internal format of UUIDs, and methods of generating UUIDs. @@ -283,7 +326,7 @@ The :mod:`uuid` module can be executed as a script from the command line. .. code-block:: sh - python -m uuid [-h] [-u {uuid1,uuid3,uuid4,uuid5}] [-n NAMESPACE] [-N NAME] + python -m uuid [-h] [-u {uuid1,uuid3,uuid4,uuid5,uuid8}] [-n NAMESPACE] [-N NAME] The following options are accepted: @@ -299,6 +342,9 @@ The following options are accepted: Specify the function name to use to generate the uuid. By default :func:`uuid4` is used. + .. versionadded:: 3.14 + Allow generating UUID version 8. + .. option:: -n --namespace @@ -353,6 +399,14 @@ Here are some examples of typical usage of the :mod:`uuid` module:: >>> uuid.UUID(bytes=x.bytes) UUID('00010203-0405-0607-0809-0a0b0c0d0e0f') + >>> # get the Nil UUID + >>> uuid.NIL + UUID('00000000-0000-0000-0000-000000000000') + + >>> # get the Max UUID + >>> uuid.MAX + UUID('ffffffff-ffff-ffff-ffff-ffffffffffff') + .. _uuid-cli-example: diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 2e7ff345a06234..bed799aedfdfb1 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -1,5 +1,5 @@ -:mod:`venv` --- Creation of virtual environments -================================================ +:mod:`!venv` --- Creation of virtual environments +================================================= .. module:: venv :synopsis: Creation of virtual environments. @@ -25,9 +25,12 @@ A virtual environment is created on top of an existing Python installation, known as the virtual environment's "base" Python, and may optionally be isolated from the packages in the base environment, so only those explicitly installed in the virtual environment are available. +See :ref:`sys-path-init-virtual-environments` and :mod:`site`'s +:ref:`virtual environments documentation ` +for more information. When used from within a virtual environment, common installation tools such as -`pip`_ will install Python packages into a virtual environment +:pypi:`pip` will install Python packages into a virtual environment without needing to be told to do so explicitly. A virtual environment is (amongst other things): @@ -37,14 +40,14 @@ A virtual environment is (amongst other things): are by default isolated from software in other virtual environments and Python interpreters and libraries installed in the operating system. -* Contained in a directory, conventionally either named ``venv`` or ``.venv`` in +* Contained in a directory, conventionally named ``.venv`` or ``venv`` in the project directory, or under a container directory for lots of virtual environments, such as ``~/.virtualenvs``. * Not checked into source control systems such as Git. * Considered as disposable -- it should be simple to delete and recreate it from - scratch. You don't place any project code in the environment + scratch. You don't place any project code in the environment. * Not considered as movable or copyable -- you just recreate the same environment in the target location. @@ -54,14 +57,134 @@ See :pep:`405` for more background on Python virtual environments. .. seealso:: `Python Packaging User Guide: Creating and using virtual environments - `__ + `__ -.. include:: ../includes/wasm-notavail.rst +.. include:: ../includes/wasm-mobile-notavail.rst Creating virtual environments ----------------------------- -.. include:: /using/venv-create.inc +:ref:`Virtual environments ` are created by executing the ``venv`` +module: + +.. code-block:: shell + + python -m venv /path/to/new/virtual/environment + +This creates the target directory (including parent directories as needed) +and places a :file:`pyvenv.cfg` file in it with a ``home`` key +pointing to the Python installation from which the command was run. +It also creates a :file:`bin` (or :file:`Scripts` on Windows) subdirectory +containing a copy or symlink of the Python executable +(as appropriate for the platform or arguments used at environment creation time). +It also creates a :file:`lib/pythonX.Y/site-packages` subdirectory +(on Windows, this is :file:`Lib\site-packages`). +If an existing directory is specified, it will be re-used. + +.. versionchanged:: 3.5 + The use of ``venv`` is now recommended for creating virtual environments. + +.. deprecated-removed:: 3.6 3.8 + :program:`pyvenv` was the recommended tool for creating virtual environments + for Python 3.3 and 3.4, and replaced in 3.5 by executing ``venv`` directly. + +.. highlight:: none + +On Windows, invoke the ``venv`` command as follows: + +.. code-block:: ps1con + + PS> python -m venv C:\path\to\new\virtual\environment + +The command, if run with ``-h``, will show the available options:: + + usage: venv [-h] [--system-site-packages] [--symlinks | --copies] [--clear] + [--upgrade] [--without-pip] [--prompt PROMPT] [--upgrade-deps] + [--without-scm-ignore-files] + ENV_DIR [ENV_DIR ...] + + Creates virtual Python environments in one or more target directories. + + positional arguments: + ENV_DIR A directory to create the environment in. + + options: + -h, --help show this help message and exit + --system-site-packages + Give the virtual environment access to the system + site-packages dir. + --symlinks Try to use symlinks rather than copies, when + symlinks are not the default for the platform. + --copies Try to use copies rather than symlinks, even when + symlinks are the default for the platform. + --clear Delete the contents of the environment directory + if it already exists, before environment creation. + --upgrade Upgrade the environment directory to use this + version of Python, assuming Python has been + upgraded in-place. + --without-pip Skips installing or upgrading pip in the virtual + environment (pip is bootstrapped by default) + --prompt PROMPT Provides an alternative prompt prefix for this + environment. + --upgrade-deps Upgrade core dependencies (pip) to the latest + version in PyPI + --without-scm-ignore-files + Skips adding SCM ignore files to the environment + directory (Git is supported by default). + + Once an environment has been created, you may wish to activate it, e.g. by + sourcing an activate script in its bin directory. + + +.. versionchanged:: 3.4 + Installs pip by default, added the ``--without-pip`` and ``--copies`` + options. + +.. versionchanged:: 3.4 + In earlier versions, if the target directory already existed, an error was + raised, unless the ``--clear`` or ``--upgrade`` option was provided. + +.. versionchanged:: 3.9 + Add ``--upgrade-deps`` option to upgrade pip + setuptools to the latest on PyPI. + +.. versionchanged:: 3.12 + + ``setuptools`` is no longer a core venv dependency. + +.. versionchanged:: 3.13 + + Added the ``--without-scm-ignore-files`` option. +.. versionchanged:: 3.13 + ``venv`` now creates a :file:`.gitignore` file for Git by default. + +.. note:: + While symlinks are supported on Windows, they are not recommended. Of + particular note is that double-clicking ``python.exe`` in File Explorer + will resolve the symlink eagerly and ignore the virtual environment. + +.. note:: + On Microsoft Windows, it may be required to enable the ``Activate.ps1`` + script by setting the execution policy for the user. You can do this by + issuing the following PowerShell command: + + .. code-block:: powershell + + PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + + See `About Execution Policies + `_ + for more information. + +The created :file:`pyvenv.cfg` file also includes the +``include-system-site-packages`` key, set to ``true`` if ``venv`` is +run with the ``--system-site-packages`` option, ``false`` otherwise. + +Unless the ``--without-pip`` option is given, :mod:`ensurepip` will be +invoked to bootstrap ``pip`` into the virtual environment. + +Multiple paths can be given to ``venv``, in which case an identical virtual +environment will be created, according to the given options, at each provided +path. .. _venv-explanation: @@ -95,7 +218,7 @@ containing the virtual environment): | +------------+--------------------------------------------------+ | | csh/tcsh | :samp:`$ source {}/bin/activate.csh` | | +------------+--------------------------------------------------+ -| | PowerShell | :samp:`$ {}/bin/Activate.ps1` | +| | pwsh | :samp:`$ {}/bin/Activate.ps1` | +-------------+------------+--------------------------------------------------+ | Windows | cmd.exe | :samp:`C:\\> {}\\Scripts\\activate.bat` | | +------------+--------------------------------------------------+ @@ -117,7 +240,7 @@ should be runnable without activating it. In order to achieve this, scripts installed into virtual environments have a "shebang" line which points to the environment's Python interpreter, -i.e. :samp:`#!/{}/bin/python`. +:samp:`#!/{}/bin/python`. This means that the script will run with that interpreter regardless of the value of :envvar:`PATH`. On Windows, "shebang" line processing is supported if you have the :ref:`launcher` installed. Thus, double-clicking an installed @@ -168,31 +291,31 @@ creation according to their needs, the :class:`EnvBuilder` class. The :class:`EnvBuilder` class accepts the following keyword arguments on instantiation: - * ``system_site_packages`` -- a Boolean value indicating that the system Python + * *system_site_packages* -- a boolean value indicating that the system Python site-packages should be available to the environment (defaults to ``False``). - * ``clear`` -- a Boolean value which, if true, will delete the contents of + * *clear* -- a boolean value which, if true, will delete the contents of any existing target directory, before creating the environment. - * ``symlinks`` -- a Boolean value indicating whether to attempt to symlink the + * *symlinks* -- a boolean value indicating whether to attempt to symlink the Python binary rather than copying. - * ``upgrade`` -- a Boolean value which, if true, will upgrade an existing + * *upgrade* -- a boolean value which, if true, will upgrade an existing environment with the running Python - for use when that Python has been upgraded in-place (defaults to ``False``). - * ``with_pip`` -- a Boolean value which, if true, ensures pip is + * *with_pip* -- a boolean value which, if true, ensures pip is installed in the virtual environment. This uses :mod:`ensurepip` with the ``--default-pip`` option. - * ``prompt`` -- a String to be used after virtual environment is activated + * *prompt* -- a string to be used after virtual environment is activated (defaults to ``None`` which means directory name of the environment would be used). If the special string ``"."`` is provided, the basename of the current directory is used as the prompt. - * ``upgrade_deps`` -- Update the base venv modules to the latest on PyPI + * *upgrade_deps* -- Update the base venv modules to the latest on PyPI - * ``scm_ignore_files`` -- Create ignore files based for the specified source + * *scm_ignore_files* -- Create ignore files based for the specified source control managers (SCM) in the iterable. Support is defined by having a method named ``create_{scm}_ignore_file``. The only value supported by default is ``"git"`` via :meth:`create_git_ignore_file`. @@ -210,10 +333,7 @@ creation according to their needs, the :class:`EnvBuilder` class. .. versionchanged:: 3.13 Added the ``scm_ignore_files`` parameter - Creators of third-party virtual environment tools will be free to use the - provided :class:`EnvBuilder` class as a base class. - - The returned env-builder is an object which has a method, ``create``: + :class:`EnvBuilder` may be used as a base class. .. method:: create(env_dir) @@ -313,14 +433,14 @@ creation according to their needs, the :class:`EnvBuilder` class. .. method:: upgrade_dependencies(context) - Upgrades the core venv dependency packages (currently ``pip``) + Upgrades the core venv dependency packages (currently :pypi:`pip`) in the environment. This is done by shelling out to the ``pip`` executable in the environment. .. versionadded:: 3.9 .. versionchanged:: 3.12 - ``setuptools`` is no longer a core venv dependency. + :pypi:`setuptools` is no longer a core venv dependency. .. method:: post_setup(context) @@ -328,25 +448,15 @@ creation according to their needs, the :class:`EnvBuilder` class. implementations to pre-install packages in the virtual environment or perform other post-creation steps. - .. versionchanged:: 3.7.2 - Windows now uses redirector scripts for ``python[w].exe`` instead of - copying the actual binaries. In 3.7.2 only :meth:`setup_python` does - nothing unless running from a build in the source tree. - - .. versionchanged:: 3.7.3 - Windows copies the redirector scripts as part of :meth:`setup_python` - instead of :meth:`setup_scripts`. This was not the case in 3.7.2. - When using symlinks, the original executables will be linked. - - In addition, :class:`EnvBuilder` provides this utility method that can be - called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to - assist in installing custom scripts into the virtual environment. - .. method:: install_scripts(context, path) + This method can be + called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to + assist in installing custom scripts into the virtual environment. + *path* is the path to a directory that should contain subdirectories - "common", "posix", "nt", each containing scripts destined for the bin - directory in the environment. The contents of "common" and the + ``common``, ``posix``, ``nt``; each containing scripts destined for the + ``bin`` directory in the environment. The contents of ``common`` and the directory corresponding to :data:`os.name` are copied after some text replacement of placeholders: @@ -371,10 +481,20 @@ creation according to their needs, the :class:`EnvBuilder` class. .. method:: create_git_ignore_file(context) Creates a ``.gitignore`` file within the virtual environment that causes - the entire directory to be ignored by the ``git`` source control manager. + the entire directory to be ignored by the Git source control manager. .. versionadded:: 3.13 + .. versionchanged:: 3.7.2 + Windows now uses redirector scripts for ``python[w].exe`` instead of + copying the actual binaries. In 3.7.2 only :meth:`setup_python` does + nothing unless running from a build in the source tree. + + .. versionchanged:: 3.7.3 + Windows copies the redirector scripts as part of :meth:`setup_python` + instead of :meth:`setup_scripts`. This was not the case in 3.7.2. + When using symlinks, the original executables will be linked. + There is also a module-level convenience function: .. function:: create(env_dir, system_site_packages=False, clear=False, \ @@ -387,16 +507,16 @@ There is also a module-level convenience function: .. versionadded:: 3.3 .. versionchanged:: 3.4 - Added the ``with_pip`` parameter + Added the *with_pip* parameter .. versionchanged:: 3.6 - Added the ``prompt`` parameter + Added the *prompt* parameter .. versionchanged:: 3.9 - Added the ``upgrade_deps`` parameter + Added the *upgrade_deps* parameter .. versionchanged:: 3.13 - Added the ``scm_ignore_files`` parameter + Added the *scm_ignore_files* parameter An example of extending ``EnvBuilder`` -------------------------------------- @@ -614,7 +734,3 @@ subclass which installs setuptools and pip into a created virtual environment:: This script is also available for download `online `_. - - -.. _setuptools: https://pypi.org/project/setuptools/ -.. _pip: https://pypi.org/project/pip/ diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 500398636e11ae..0c7e8543f331db 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -1,5 +1,5 @@ -:mod:`warnings` --- Warning control -=================================== +:mod:`!warnings` --- Warning control +==================================== .. module:: warnings :synopsis: Issue warning messages and control their disposition. @@ -145,6 +145,8 @@ the disposition of the match. Each entry is a tuple of the form (*action*, +---------------+----------------------------------------------+ | ``"always"`` | always print matching warnings | +---------------+----------------------------------------------+ + | ``"all"`` | alias to "always" | + +---------------+----------------------------------------------+ | ``"module"`` | print the first occurrence of matching | | | warnings for each module where the warning | | | is issued (regardless of line number) | @@ -178,6 +180,19 @@ If a warning is reported and doesn't match any registered filter then the "default" action is applied (hence its name). + +.. _repeated-warning-suppression-criteria: + +Repeated Warning Suppression Criteria +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The filters that suppress repeated warnings apply the following criteria to determine if a warning is considered a repeat: + +- ``"default"``: A warning is considered a repeat only if the (*message*, *category*, *module*, *lineno*) are all the same. +- ``"module"``: A warning is considered a repeat if the (*message*, *category*, *module*) are the same, ignoring the line number. +- ``"once"``: A warning is considered a repeat if the (*message*, *category*) are the same, ignoring the module and line number. + + .. _describing-warning-filters: Describing Warning Filters @@ -396,7 +411,7 @@ Available Functions ------------------- -.. function:: warn(message, category=None, stacklevel=1, source=None, *, skip_file_prefixes=None) +.. function:: warn(message, category=None, stacklevel=1, source=None, *, skip_file_prefixes=()) Issue a warning, or maybe ignore it or raise an exception. The *category* argument, if given, must be a :ref:`warning category class `; it @@ -595,6 +610,9 @@ Available Context Managers passed to :func:`simplefilter` as if it were called immediately on entering the context. + See :ref:`warning-filter` for the meaning of the *category* and *lineno* + parameters. + .. note:: The :class:`catch_warnings` manager works by replacing and diff --git a/Doc/library/wave.rst b/Doc/library/wave.rst index 55b029bc742b24..36c2bde87fb8fb 100644 --- a/Doc/library/wave.rst +++ b/Doc/library/wave.rst @@ -1,5 +1,5 @@ -:mod:`wave` --- Read and write WAV files -======================================== +:mod:`!wave` --- Read and write WAV files +========================================= .. module:: wave :synopsis: Provide an interface to the WAV sound format. @@ -46,8 +46,8 @@ The :mod:`wave` module defines the following function and exception: the file object. The :func:`.open` function may be used in a :keyword:`with` statement. When - the :keyword:`!with` block completes, the :meth:`Wave_read.close()` or - :meth:`Wave_write.close()` method is called. + the :keyword:`!with` block completes, the :meth:`Wave_read.close` or + :meth:`Wave_write.close` method is called. .. versionchanged:: 3.4 Added support for unseekable files. diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index d6e062df945c64..2a25ed045c68bd 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -197,7 +197,7 @@ See :ref:`__slots__ documentation ` for details. >>> del k1 # d = {k2: 2} .. versionchanged:: 3.9 - Added support for ``|`` and ``|=`` operators, specified in :pep:`584`. + Added support for ``|`` and ``|=`` operators, as specified in :pep:`584`. :class:`WeakKeyDictionary` objects have an additional method that exposes the internal references directly. The references are not guaranteed to diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index 4667b81e38ada2..2d19c514ce43b6 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -1,5 +1,5 @@ -:mod:`webbrowser` --- Convenient web-browser controller -======================================================= +:mod:`!webbrowser` --- Convenient web-browser controller +======================================================== .. module:: webbrowser :synopsis: Easy-to-use controller for web browsers. @@ -33,15 +33,25 @@ allow the remote browser to maintain its own windows on the display. If remote browsers are not available on Unix, the controlling process will launch a new browser and wait. +On iOS, the :envvar:`BROWSER` environment variable, as well as any arguments +controlling autoraise, browser preference, and new tab/window creation will be +ignored. Web pages will *always* be opened in the user's preferred browser, in +a new tab, with the browser being brought to the foreground. The use of the +:mod:`webbrowser` module on iOS requires the :mod:`ctypes` module. If +:mod:`ctypes` isn't available, calls to :func:`.open` will fail. + The script :program:`webbrowser` can be used as a command-line interface for the module. It accepts a URL as the argument. It accepts the following optional -parameters: ``-n`` opens the URL in a new browser window, if possible; -``-t`` opens the URL in a new browser page ("tab"). The options are, -naturally, mutually exclusive. Usage example:: +parameters: + +* ``-n``/``--new-window`` opens the URL in a new browser window, if possible. +* ``-t``/``--new-tab`` opens the URL in a new browser page ("tab"). + +The options are, naturally, mutually exclusive. Usage example:: python -m webbrowser -t "https://www.python.org" -.. include:: ../includes/wasm-notavail.rst +.. availability:: not WASI, not Android. The following exception is defined: @@ -62,6 +72,8 @@ The following functions are defined: (note that under many window managers this will occur regardless of the setting of this variable). + Returns ``True`` if a browser was successfully launched, ``False`` otherwise. + Note that on some platforms, trying to open a filename using this function, may work and start the operating system's associated program. However, this is neither supported nor portable. @@ -74,11 +86,16 @@ The following functions are defined: Open *url* in a new window of the default browser, if possible, otherwise, open *url* in the only browser window. + Returns ``True`` if a browser was successfully launched, ``False`` otherwise. + + .. function:: open_new_tab(url) Open *url* in a new page ("tab") of the default browser, if possible, otherwise equivalent to :func:`open_new`. + Returns ``True`` if a browser was successfully launched, ``False`` otherwise. + .. function:: get(using=None) @@ -147,6 +164,8 @@ for the controller classes, all defined in this module. +------------------------+-----------------------------------------+-------+ | ``'chromium-browser'`` | ``Chromium('chromium-browser')`` | | +------------------------+-----------------------------------------+-------+ +| ``'iosbrowser'`` | ``IOSBrowser`` | \(4) | ++------------------------+-----------------------------------------+-------+ Notes: @@ -161,7 +180,10 @@ Notes: Only on Windows platforms. (3) - Only on macOS platform. + Only on macOS. + +(4) + Only on iOS. .. versionadded:: 3.2 A new :class:`!MacOSXOSAScript` class has been added @@ -176,6 +198,9 @@ Notes: Removed browsers include Grail, Mosaic, Netscape, Galeon, Skipstone, Iceape, and Firefox versions 35 and below. +.. versionchanged:: 3.13 + Support for iOS has been added. + Here are some simple examples:: url = 'https://docs.python.org/' diff --git a/Doc/library/winreg.rst b/Doc/library/winreg.rst index 06bd4d87eb03c6..b3a824fb69a49f 100644 --- a/Doc/library/winreg.rst +++ b/Doc/library/winreg.rst @@ -1,5 +1,5 @@ -:mod:`winreg` --- Windows registry access -========================================= +:mod:`!winreg` --- Windows registry access +========================================== .. module:: winreg :platform: Windows diff --git a/Doc/library/winsound.rst b/Doc/library/winsound.rst index 370c5216652ba7..f7ca9dc57bbe28 100644 --- a/Doc/library/winsound.rst +++ b/Doc/library/winsound.rst @@ -1,5 +1,5 @@ -:mod:`winsound` --- Sound-playing interface for Windows -======================================================= +:mod:`!winsound` --- Sound-playing interface for Windows +======================================================== .. module:: winsound :platform: Windows diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index c2b0ba7046967e..8d4c5eb6600e8b 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -1,5 +1,5 @@ -:mod:`wsgiref` --- WSGI Utilities and Reference Implementation -============================================================== +:mod:`!wsgiref` --- WSGI Utilities and Reference Implementation +=============================================================== .. module:: wsgiref :synopsis: WSGI Utilities and Reference Implementation. @@ -783,8 +783,8 @@ in :pep:`3333`. .. class:: StartResponse() - A :class:`typing.Protocol` describing `start_response() - `_ + A :class:`typing.Protocol` describing :pep:`start_response() + <3333#the-start-response-callable>` callables (:pep:`3333`). .. data:: WSGIEnvironment @@ -797,18 +797,18 @@ in :pep:`3333`. .. class:: InputStream() - A :class:`typing.Protocol` describing a `WSGI Input Stream - `_. + A :class:`typing.Protocol` describing a :pep:`WSGI Input Stream + <3333#input-and-error-streams>`. .. class:: ErrorStream() - A :class:`typing.Protocol` describing a `WSGI Error Stream - `_. + A :class:`typing.Protocol` describing a :pep:`WSGI Error Stream + <3333#input-and-error-streams>`. .. class:: FileWrapper() - A :class:`typing.Protocol` describing a `file wrapper - `_. + A :class:`typing.Protocol` describing a :pep:`file wrapper + <3333#optional-platform-specific-file-handling>`. See :class:`wsgiref.util.FileWrapper` for a concrete implementation of this protocol. @@ -865,7 +865,7 @@ directory and port number (default: 8000) on the command line:: fn = os.path.join(path, environ["PATH_INFO"][1:]) if "." not in fn.split(os.path.sep)[-1]: fn = os.path.join(fn, "index.html") - mime_type = mimetypes.guess_type(fn)[0] + mime_type = mimetypes.guess_file_type(fn)[0] # Return 200 OK if file exists, otherwise 404 Not Found if os.path.exists(fn): diff --git a/Doc/library/xdrlib.rst b/Doc/library/xdrlib.rst new file mode 100644 index 00000000000000..59b801c8e4072e --- /dev/null +++ b/Doc/library/xdrlib.rst @@ -0,0 +1,15 @@ +:mod:`!xdrlib` --- Encode and decode XDR data +============================================= + +.. module:: xdrlib + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!xdrlib` module was +`Python 3.12 `_. diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 72a7a98c2ac4f2..00a18751207e7a 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -1,5 +1,5 @@ -:mod:`xml.dom.minidom` --- Minimal DOM implementation -===================================================== +:mod:`!xml.dom.minidom` --- Minimal DOM implementation +====================================================== .. module:: xml.dom.minidom :synopsis: Minimal Document Object Model (DOM) implementation. diff --git a/Doc/library/xml.dom.pulldom.rst b/Doc/library/xml.dom.pulldom.rst index 843c2fd7fdb937..fd96765cbe3c96 100644 --- a/Doc/library/xml.dom.pulldom.rst +++ b/Doc/library/xml.dom.pulldom.rst @@ -1,5 +1,5 @@ -:mod:`xml.dom.pulldom` --- Support for building partial DOM trees -================================================================= +:mod:`!xml.dom.pulldom` --- Support for building partial DOM trees +================================================================== .. module:: xml.dom.pulldom :synopsis: Support for building partial DOM trees from SAX events. diff --git a/Doc/library/xml.dom.rst b/Doc/library/xml.dom.rst index d0e1b248d595d1..f33b19bc2724d0 100644 --- a/Doc/library/xml.dom.rst +++ b/Doc/library/xml.dom.rst @@ -1,5 +1,5 @@ -:mod:`xml.dom` --- The Document Object Model API -================================================ +:mod:`!xml.dom` --- The Document Object Model API +================================================= .. module:: xml.dom :synopsis: Document Object Model API for Python. diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 19c7af452e2b71..1daf6628013bf0 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -1,5 +1,5 @@ -:mod:`xml.etree.ElementTree` --- The ElementTree XML API -======================================================== +:mod:`!xml.etree.ElementTree` --- The ElementTree XML API +========================================================= .. module:: xml.etree.ElementTree :synopsis: Implementation of the ElementTree API. @@ -49,7 +49,7 @@ and its sub-elements are done on the :class:`Element` level. Parsing XML ^^^^^^^^^^^ -We'll be using the following XML document as the sample data for this section: +We'll be using the fictive :file:`country_data.xml` XML document as the sample data for this section: .. code-block:: xml @@ -508,7 +508,7 @@ Functions `C14N 2.0 `_ transformation function. Canonicalization is a way to normalise XML output in a way that allows - byte-by-byte comparisons and digital signatures. It reduced the freedom + byte-by-byte comparisons and digital signatures. It reduces the freedom that XML serializers have and instead generates a more constrained XML representation. The main restrictions regard the placement of namespace declarations, the ordering of attributes, and ignorable whitespace. @@ -840,33 +840,28 @@ Functions .. module:: xml.etree.ElementInclude -.. function:: xml.etree.ElementInclude.default_loader( href, parse, encoding=None) - :module: +.. function:: default_loader(href, parse, encoding=None) - Default loader. This default loader reads an included resource from disk. *href* is a URL. - *parse* is for parse mode either "xml" or "text". *encoding* - is an optional text encoding. If not given, encoding is ``utf-8``. Returns the - expanded resource. If the parse mode is ``"xml"``, this is an ElementTree - instance. If the parse mode is "text", this is a Unicode string. If the - loader fails, it can return None or raise an exception. + Default loader. This default loader reads an included resource from disk. + *href* is a URL. *parse* is for parse mode either "xml" or "text". + *encoding* is an optional text encoding. If not given, encoding is ``utf-8``. + Returns the expanded resource. + If the parse mode is ``"xml"``, this is an :class:`~xml.etree.ElementTree.Element` instance. + If the parse mode is ``"text"``, this is a string. + If the loader fails, it can return ``None`` or raise an exception. -.. function:: xml.etree.ElementInclude.include( elem, loader=None, base_url=None, \ - max_depth=6) - :module: +.. function:: include(elem, loader=None, base_url=None, max_depth=6) - This function expands XInclude directives. *elem* is the root element. *loader* is - an optional resource loader. If omitted, it defaults to :func:`default_loader`. + This function expands XInclude directives in-place in tree pointed by *elem*. + *elem* is either the root :class:`~xml.etree.ElementTree.Element` or an + :class:`~xml.etree.ElementTree.ElementTree` instance to find such element. + *loader* is an optional resource loader. If omitted, it defaults to :func:`default_loader`. If given, it should be a callable that implements the same interface as :func:`default_loader`. *base_url* is base URL of the original file, to resolve relative include file references. *max_depth* is the maximum number of recursive - inclusions. Limited to reduce the risk of malicious content explosion. Pass a - negative value to disable the limitation. - - Returns the expanded resource. If the parse mode is - ``"xml"``, this is an ElementTree instance. If the parse mode is "text", - this is a Unicode string. If the loader fails, it can return None or - raise an exception. + inclusions. Limited to reduce the risk of malicious content explosion. + Pass ``None`` to disable the limitation. .. versionchanged:: 3.9 Added the *base_url* and *max_depth* parameters. @@ -879,6 +874,7 @@ Element Objects .. module:: xml.etree.ElementTree :noindex: + :no-index: .. class:: Element(tag, attrib={}, **extra) @@ -975,7 +971,7 @@ Element Objects .. method:: extend(subelements) - Appends *subelements* from a sequence object with zero or more elements. + Appends *subelements* from an iterable of elements. Raises :exc:`TypeError` if a subelement is not an :class:`Element`. .. versionadded:: 3.2 @@ -1063,9 +1059,10 @@ Element Objects :meth:`~object.__getitem__`, :meth:`~object.__setitem__`, :meth:`~object.__len__`. - Caution: Elements with no subelements will test as ``False``. Testing the - truth value of an Element is deprecated and will raise an exception in - Python 3.14. Use specific ``len(elem)`` or ``elem is None`` test instead.:: + Caution: Elements with no subelements will test as ``False``. In a future + release of Python, all elements will test as ``True`` regardless of whether + subelements exist. Instead, prefer explicit ``len(elem)`` or + ``elem is not None`` tests.:: element = root.find('foo') @@ -1378,7 +1375,7 @@ XMLParser Objects .. versionchanged:: 3.8 Parameters are now :ref:`keyword-only `. - The *html* argument no longer supported. + The *html* argument is no longer supported. .. method:: close() @@ -1402,6 +1399,11 @@ XMLParser Objects Disabling reparse deferral has security consequences; please see :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` for details. + Note that :meth:`flush` has been backported to some prior releases of + CPython as a security fix. Check for availability of :meth:`flush` + using :func:`hasattr` if used in code running across a variety of Python + versions. + .. versionadded:: 3.13 @@ -1475,6 +1477,11 @@ XMLPullParser Objects Disabling reparse deferral has security consequences; please see :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` for details. + Note that :meth:`flush` has been backported to some prior releases of + CPython as a security fix. Check for availability of :meth:`flush` + using :func:`hasattr` if used in code running across a variety of Python + versions. + .. versionadded:: 3.13 .. method:: close() diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index 662cc459197e2c..d495995398959d 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -124,10 +124,9 @@ large tokens Expat needs to re-parse unfinished tokens; without the protection introduced in Expat 2.6.0, this can lead to quadratic runtime that can be used to cause denial of service in the application parsing XML. - The issue is known as - `CVE-2023-52425 `_. + The issue is known as :cve:`2023-52425`. -The documentation for `defusedxml`_ on PyPI has further information about +The documentation for :pypi:`defusedxml` on PyPI has further information about all known attack vectors with examples and references. .. _defusedxml-package: @@ -135,14 +134,13 @@ all known attack vectors with examples and references. The :mod:`!defusedxml` Package ------------------------------ -`defusedxml`_ is a pure Python package with modified subclasses of all stdlib +:pypi:`defusedxml` is a pure Python package with modified subclasses of all stdlib XML parsers that prevent any potentially malicious operation. Use of this package is recommended for any server code that parses untrusted XML data. The package also ships with example exploits and extended documentation on more XML exploits such as XPath injection. -.. _defusedxml: https://pypi.org/project/defusedxml/ .. _Billion Laughs: https://en.wikipedia.org/wiki/Billion_laughs .. _ZIP bomb: https://en.wikipedia.org/wiki/Zip_bomb .. _DTD: https://en.wikipedia.org/wiki/Document_type_definition diff --git a/Doc/library/xml.sax.handler.rst b/Doc/library/xml.sax.handler.rst index e2f28e3244cb09..c2c9d6424b5072 100644 --- a/Doc/library/xml.sax.handler.rst +++ b/Doc/library/xml.sax.handler.rst @@ -1,5 +1,5 @@ -:mod:`xml.sax.handler` --- Base classes for SAX handlers -======================================================== +:mod:`!xml.sax.handler` --- Base classes for SAX handlers +========================================================= .. module:: xml.sax.handler :synopsis: Base classes for SAX event handlers. diff --git a/Doc/library/xml.sax.reader.rst b/Doc/library/xml.sax.reader.rst index 113e9e93fb04ff..b0bc84062e0719 100644 --- a/Doc/library/xml.sax.reader.rst +++ b/Doc/library/xml.sax.reader.rst @@ -1,5 +1,5 @@ -:mod:`xml.sax.xmlreader` --- Interface for XML parsers -====================================================== +:mod:`!xml.sax.xmlreader` --- Interface for XML parsers +======================================================= .. module:: xml.sax.xmlreader :synopsis: Interface which SAX-compliant XML parsers must implement. diff --git a/Doc/library/xml.sax.rst b/Doc/library/xml.sax.rst index 6d351dfb4d7072..c60e9e505f7544 100644 --- a/Doc/library/xml.sax.rst +++ b/Doc/library/xml.sax.rst @@ -1,5 +1,5 @@ -:mod:`xml.sax` --- Support for SAX2 parsers -=========================================== +:mod:`!xml.sax` --- Support for SAX2 parsers +============================================ .. module:: xml.sax :synopsis: Package containing SAX2 base classes and convenience functions. diff --git a/Doc/library/xml.sax.utils.rst b/Doc/library/xml.sax.utils.rst index 3a524c9c0d5a9f..5ee11d58c3dd26 100644 --- a/Doc/library/xml.sax.utils.rst +++ b/Doc/library/xml.sax.utils.rst @@ -1,5 +1,5 @@ -:mod:`xml.sax.saxutils` --- SAX Utilities -========================================= +:mod:`!xml.sax.saxutils` --- SAX Utilities +========================================== .. module:: xml.sax.saxutils :synopsis: Convenience functions and classes for use with SAX. diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index f7f23007fb0522..971e65605841e7 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -1,5 +1,5 @@ -:mod:`xmlrpc.client` --- XML-RPC client access -============================================== +:mod:`!xmlrpc.client` --- XML-RPC client access +=============================================== .. module:: xmlrpc.client :synopsis: XML-RPC client access. @@ -64,11 +64,11 @@ between conformable Python objects and XML on the wire. The obsolete *use_datetime* flag is similar to *use_builtin_types* but it applies only to date/time values. -.. versionchanged:: 3.3 - The *use_builtin_types* flag was added. + .. versionchanged:: 3.3 + The *use_builtin_types* flag was added. -.. versionchanged:: 3.8 - The *headers* parameter was added. + .. versionchanged:: 3.8 + The *headers* parameter was added. Both the HTTP and HTTPS transports support the URL syntax extension for HTTP Basic Authentication: ``http://user:pass@host:port/path``. The ``user:pass`` @@ -165,7 +165,7 @@ between conformable Python objects and XML on the wire. A good description of XML-RPC operation and client software in several languages. Contains pretty much everything an XML-RPC client developer needs to know. - `XML-RPC Introspection `_ + `XML-RPC Introspection `_ Describes the XML-RPC protocol extension for introspection. `XML-RPC Specification `_ diff --git a/Doc/library/xmlrpc.rst b/Doc/library/xmlrpc.rst index 5f0a2cf68d01f9..a93d08f78cfba7 100644 --- a/Doc/library/xmlrpc.rst +++ b/Doc/library/xmlrpc.rst @@ -1,6 +1,9 @@ :mod:`!xmlrpc` --- XMLRPC server and client modules =================================================== +.. module:: xmlrpc + :synopsis: Server and client modules implementing XML-RPC. + XML-RPC is a Remote Procedure Call method that uses XML passed via HTTP as a transport. With it, a client can call methods with parameters on a remote server (the server is named by a URI) and get back structured data. diff --git a/Doc/library/xmlrpc.server.rst b/Doc/library/xmlrpc.server.rst index ca1ea455f0acfc..06169c7eca8b0c 100644 --- a/Doc/library/xmlrpc.server.rst +++ b/Doc/library/xmlrpc.server.rst @@ -1,5 +1,5 @@ -:mod:`xmlrpc.server` --- Basic XML-RPC servers -============================================== +:mod:`!xmlrpc.server` --- Basic XML-RPC servers +=============================================== .. module:: xmlrpc.server :synopsis: Basic XML-RPC server implementations. diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst index c8a059bdb1cb93..cdaba07ab46c8f 100644 --- a/Doc/library/zipapp.rst +++ b/Doc/library/zipapp.rst @@ -1,5 +1,5 @@ -:mod:`zipapp` --- Manage executable Python zip archives -======================================================= +:mod:`!zipapp` --- Manage executable Python zip archives +======================================================== .. module:: zipapp :synopsis: Manage executable Python zip archives @@ -332,7 +332,7 @@ Formally, the Python zip application format is therefore: interpreter name, and then a newline (``b'\n'``) character. The interpreter name can be anything acceptable to the OS "shebang" processing, or the Python launcher on Windows. The interpreter should be encoded in UTF-8 on Windows, - and in :func:`sys.getfilesystemencoding()` on POSIX. + and in :func:`sys.getfilesystemencoding` on POSIX. 2. Standard zipfile data, as generated by the :mod:`zipfile` module. The zipfile content *must* include a file called ``__main__.py`` (which must be in the "root" of the zipfile - i.e., it cannot be in a subdirectory). The diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index b6f881fd2dfd70..afe1cd5c75fcbb 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -1,5 +1,5 @@ -:mod:`zipfile` --- Work with ZIP archives -========================================= +:mod:`!zipfile` --- Work with ZIP archives +========================================== .. module:: zipfile :synopsis: Read and write ZIP-format archive files. @@ -84,6 +84,17 @@ The module defines the following items: formerly protected :attr:`!_compresslevel`. The older protected name continues to work as a property for backwards compatibility. + + .. method:: _for_archive(archive) + + Resolve the date_time, compression attributes, and external attributes + to suitable defaults as used by :meth:`ZipFile.writestr`. + + Returns self for chaining. + + .. versionadded:: 3.14 + + .. function:: is_zipfile(filename) Returns ``True`` if *filename* is a valid ZIP file based on its magic number, @@ -301,6 +312,10 @@ ZipFile Objects attempting to read or write other files in the ZIP file will raise a :exc:`ValueError`. + In both cases the file-like object has also attributes :attr:`!name`, + which is equivalent to the name of a file within the archive, and + :attr:`!mode`, which is ``'rb'`` or ``'wb'`` depending on the input mode. + When writing a file, if the file size is not known in advance but may exceed 2 GiB, pass ``force_zip64=True`` to ensure that the header format is capable of supporting large files. If the file size is known in advance, @@ -325,6 +340,12 @@ ZipFile Objects Calling :meth:`.open` on a closed ZipFile will raise a :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. + .. versionchanged:: 3.13 + Added attributes :attr:`!name` and :attr:`!mode` for the writeable + file-like object. + The value of the :attr:`!mode` attribute for the readable file-like + object was changed from ``'r'`` to ``'rb'``. + .. method:: ZipFile.extract(member, path=None, pwd=None) @@ -575,6 +596,15 @@ Path objects are traversable using the ``/`` operator or ``joinpath``. Return ``True`` if the current context references a file. +.. method:: Path.is_symlink() + + Return ``True`` if the current context references a symbolic link. + + .. versionadded:: 3.12 + + .. versionchanged:: 3.13 + Previously, ``is_symlink`` would unconditionally return ``False``. + .. method:: Path.exists() Return ``True`` if the current context references a file or @@ -632,7 +662,7 @@ Path objects are traversable using the ``/`` operator or ``joinpath``. Prior to 3.10, ``joinpath`` was undocumented and accepted exactly one parameter. -The `zipp `_ project provides backports +The :pypi:`zipp` project provides backports of the latest path object functionality to older Pythons. Use ``zipp.Path`` in place of ``zipfile.Path`` for early access to changes. diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst index 47c81f0e63603d..9353a45bdcecba 100644 --- a/Doc/library/zipimport.rst +++ b/Doc/library/zipimport.rst @@ -1,5 +1,5 @@ -:mod:`zipimport` --- Import modules from Zip archives -===================================================== +:mod:`!zipimport` --- Import modules from Zip archives +====================================================== .. module:: zipimport :synopsis: Support for importing Python modules from ZIP archives. @@ -30,6 +30,9 @@ Any files may be present in the ZIP archive, but importers are only invoked for corresponding :file:`.pyc` file, meaning that if a ZIP archive doesn't contain :file:`.pyc` files, importing may be rather slow. +.. versionchanged:: 3.13 + ZIP64 is supported + .. versionchanged:: 3.8 Previously, ZIP archives with an archive comment were not supported. diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index ac179722dee2be..965b82a3daffb9 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -1,5 +1,5 @@ -:mod:`zlib` --- Compression compatible with :program:`gzip` -=========================================================== +:mod:`!zlib` --- Compression compatible with :program:`gzip` +============================================================ .. module:: zlib :synopsis: Low-level interface to compression and decompression routines diff --git a/Doc/library/zoneinfo.rst b/Doc/library/zoneinfo.rst index f8624da6e51dbb..a57f3b8b3e858c 100644 --- a/Doc/library/zoneinfo.rst +++ b/Doc/library/zoneinfo.rst @@ -1,5 +1,5 @@ -:mod:`zoneinfo` --- IANA time zone support -========================================== +:mod:`!zoneinfo` --- IANA time zone support +=========================================== .. module:: zoneinfo :synopsis: IANA time zone support @@ -17,7 +17,7 @@ The :mod:`zoneinfo` module provides a concrete time zone implementation to support the IANA time zone database as originally specified in :pep:`615`. By default, :mod:`zoneinfo` uses the system's time zone data if available; if no system time zone data is available, the library will fall back to using the -first-party `tzdata`_ package available on PyPI. +first-party :pypi:`tzdata` package available on PyPI. .. seealso:: @@ -25,7 +25,7 @@ first-party `tzdata`_ package available on PyPI. Provides the :class:`~datetime.time` and :class:`~datetime.datetime` types with which the :class:`ZoneInfo` class is designed to be used. - Package `tzdata`_ + Package :pypi:`tzdata` First-party package maintained by the CPython core developers to supply time zone data via PyPI. @@ -93,7 +93,7 @@ Data sources The ``zoneinfo`` module does not directly provide time zone data, and instead pulls time zone information from the system time zone database or the -first-party PyPI package `tzdata`_, if available. Some systems, including +first-party PyPI package :pypi:`tzdata`, if available. Some systems, including notably Windows systems, do not have an IANA database available, and so for projects targeting cross-platform compatibility that require time zone data, it is recommended to declare a dependency on tzdata. If neither system data nor @@ -413,5 +413,3 @@ Exceptions and warnings be filtered out, such as a relative path. .. Links and references: - -.. _tzdata: https://pypi.org/project/tzdata/ diff --git a/Doc/license.rst b/Doc/license.rst index cbe918bd1acfe3..61a26ab2867498 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -11,59 +11,63 @@ History of the software ======================= Python was created in the early 1990s by Guido van Rossum at Stichting -Mathematisch Centrum (CWI, see https://www.cwi.nl/) in the Netherlands as a +Mathematisch Centrum (CWI, see https://www.cwi.nl) in the Netherlands as a successor of a language called ABC. Guido remains Python's principal author, although it includes many contributions from others. In 1995, Guido continued his work on Python at the Corporation for National -Research Initiatives (CNRI, see https://www.cnri.reston.va.us/) in Reston, +Research Initiatives (CNRI, see https://www.cnri.reston.va.us) in Reston, Virginia where he released several versions of the software. In May 2000, Guido and the Python core development team moved to BeOpen.com to form the BeOpen PythonLabs team. In October of the same year, the PythonLabs -team moved to Digital Creations (now Zope Corporation; see -https://www.zope.org/). In 2001, the Python Software Foundation (PSF, see +team moved to Digital Creations, which became +Zope Corporation. In 2001, the Python Software Foundation (PSF, see https://www.python.org/psf/) was formed, a non-profit organization created -specifically to own Python-related Intellectual Property. Zope Corporation is a +specifically to own Python-related Intellectual Property. Zope Corporation was a sponsoring member of the PSF. -All Python releases are Open Source (see https://opensource.org/ for the Open +All Python releases are Open Source (see https://opensource.org for the Open Source Definition). Historically, most, but not all, Python releases have also been GPL-compatible; the table below summarizes the various releases. -+----------------+--------------+------------+------------+-----------------+ -| Release | Derived from | Year | Owner | GPL compatible? | -+================+==============+============+============+=================+ -| 0.9.0 thru 1.2 | n/a | 1991-1995 | CWI | yes | -+----------------+--------------+------------+------------+-----------------+ -| 1.3 thru 1.5.2 | 1.2 | 1995-1999 | CNRI | yes | -+----------------+--------------+------------+------------+-----------------+ -| 1.6 | 1.5.2 | 2000 | CNRI | no | -+----------------+--------------+------------+------------+-----------------+ -| 2.0 | 1.6 | 2000 | BeOpen.com | no | -+----------------+--------------+------------+------------+-----------------+ -| 1.6.1 | 1.6 | 2001 | CNRI | no | -+----------------+--------------+------------+------------+-----------------+ -| 2.1 | 2.0+1.6.1 | 2001 | PSF | no | -+----------------+--------------+------------+------------+-----------------+ -| 2.0.1 | 2.0+1.6.1 | 2001 | PSF | yes | -+----------------+--------------+------------+------------+-----------------+ -| 2.1.1 | 2.1+2.0.1 | 2001 | PSF | yes | -+----------------+--------------+------------+------------+-----------------+ -| 2.1.2 | 2.1.1 | 2002 | PSF | yes | -+----------------+--------------+------------+------------+-----------------+ -| 2.1.3 | 2.1.2 | 2002 | PSF | yes | -+----------------+--------------+------------+------------+-----------------+ -| 2.2 and above | 2.1.1 | 2001-now | PSF | yes | -+----------------+--------------+------------+------------+-----------------+ ++----------------+--------------+------------+------------+---------------------+ +| Release | Derived from | Year | Owner | GPL-compatible? (1) | ++================+==============+============+============+=====================+ +| 0.9.0 thru 1.2 | n/a | 1991-1995 | CWI | yes | ++----------------+--------------+------------+------------+---------------------+ +| 1.3 thru 1.5.2 | 1.2 | 1995-1999 | CNRI | yes | ++----------------+--------------+------------+------------+---------------------+ +| 1.6 | 1.5.2 | 2000 | CNRI | no | ++----------------+--------------+------------+------------+---------------------+ +| 2.0 | 1.6 | 2000 | BeOpen.com | no | ++----------------+--------------+------------+------------+---------------------+ +| 1.6.1 | 1.6 | 2001 | CNRI | yes (2) | ++----------------+--------------+------------+------------+---------------------+ +| 2.1 | 2.0+1.6.1 | 2001 | PSF | no | ++----------------+--------------+------------+------------+---------------------+ +| 2.0.1 | 2.0+1.6.1 | 2001 | PSF | yes | ++----------------+--------------+------------+------------+---------------------+ +| 2.1.1 | 2.1+2.0.1 | 2001 | PSF | yes | ++----------------+--------------+------------+------------+---------------------+ +| 2.1.2 | 2.1.1 | 2002 | PSF | yes | ++----------------+--------------+------------+------------+---------------------+ +| 2.1.3 | 2.1.2 | 2002 | PSF | yes | ++----------------+--------------+------------+------------+---------------------+ +| 2.2 and above | 2.1.1 | 2001-now | PSF | yes | ++----------------+--------------+------------+------------+---------------------+ .. note:: - GPL-compatible doesn't mean that we're distributing Python under the GPL. All - Python licenses, unlike the GPL, let you distribute a modified version without - making your changes open source. The GPL-compatible licenses make it possible to - combine Python with other software that is released under the GPL; the others - don't. + (1) GPL-compatible doesn't mean that we're distributing Python under the GPL. + All Python licenses, unlike the GPL, let you distribute a modified version + without making your changes open source. The GPL-compatible licenses make + it possible to combine Python with other software that is released under + the GPL; the others don't. + + (2) According to Richard Stallman, 1.6.1 is not GPL-compatible, because its license + has a choice of law clause. According to CNRI, however, Stallman's lawyer has + told CNRI's lawyer that 1.6.1 is "not incompatible" with the GPL. Thanks to the many outside volunteers who have worked under Guido's direction to make these releases possible. @@ -73,10 +77,10 @@ Terms and conditions for accessing or otherwise using Python ============================================================ Python software and documentation are licensed under the -:ref:`PSF License Agreement `. +Python Software Foundation License Version 2. Starting with Python 3.8.6, examples, recipes, and other code in -the documentation are dual licensed under the PSF License Agreement +the documentation are dual licensed under the PSF License Version 2 and the :ref:`Zero-Clause BSD license `. Some software incorporated into Python is under different licenses. @@ -86,39 +90,38 @@ See :ref:`OtherLicenses` for an incomplete list of these licenses. .. _PSF-license: -PSF LICENSE AGREEMENT FOR PYTHON |release| ------------------------------------------- +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- .. parsed-literal:: 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and - the Individual or Organization ("Licensee") accessing and otherwise using Python - |release| software in source or binary form and its associated documentation. + the Individual or Organization ("Licensee") accessing and otherwise using this + software ("Python") in source or binary form and its associated documentation. 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, - distribute, and otherwise use Python |release| alone or in any derivative + distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of - copyright, i.e., "Copyright © 2001-2024 Python Software Foundation; All Rights - Reserved" are retained in Python |release| alone or in any derivative version + copyright, i.e., "Copyright © 2001 Python Software Foundation; All Rights + Reserved" are retained in Python alone or in any derivative version prepared by Licensee. 3. In the event Licensee prepares a derivative work that is based on or - incorporates Python |release| or any part thereof, and wants to make the + incorporates Python or any part thereof, and wants to make the derivative work available to others as provided herein, then Licensee hereby - agrees to include in any such work a brief summary of the changes made to Python - |release|. + agrees to include in any such work a brief summary of the changes made to Python. - 4. PSF is making Python |release| available to Licensee on an "AS IS" basis. + 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE - USE OF PYTHON |release| WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. + USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. - 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON |release| + 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF - MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON |release|, OR ANY DERIVATIVE + MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. 6. This License Agreement will automatically terminate upon a material breach of @@ -130,7 +133,7 @@ PSF LICENSE AGREEMENT FOR PYTHON |release| trademark sense to endorse or promote products or services of Licensee, or any third party. - 8. By copying, installing or otherwise using Python |release|, Licensee agrees + 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this License Agreement. @@ -205,7 +208,7 @@ CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 Agreement. This Agreement together with Python 1.6.1 may be located on the internet using the following unique, persistent identifier (known as a handle): 1895.22/1013. This Agreement may also be obtained from a proxy server on the - internet using the following URL: http://hdl.handle.net/1895.22/1013." + internet using the following URL: http://hdl.handle.net/1895.22/1013". 3. In the event Licensee prepares a derivative work that is based on or incorporates Python 1.6.1 or any part thereof, and wants to make the derivative @@ -273,8 +276,8 @@ CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 .. _BSD0: -ZERO-CLAUSE BSD LICENSE FOR CODE IN THE PYTHON |release| DOCUMENTATION ----------------------------------------------------------------------- +ZERO-CLAUSE BSD LICENSE FOR CODE IN THE PYTHON DOCUMENTATION +------------------------------------------------------------ .. parsed-literal:: @@ -1042,30 +1045,32 @@ https://www.w3.org/TR/xml-c14n2-testcases/ and is distributed under the OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +.. _mimalloc-license: + mimalloc -------- -MIT License +MIT License:: -Copyright (c) 2018-2021 Microsoft Corporation, Daan Leijen + Copyright (c) 2018-2021 Microsoft Corporation, Daan Leijen -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. asyncio diff --git a/Doc/make.bat b/Doc/make.bat index 87d8359ef112bb..ede793ed3c6d70 100644 --- a/Doc/make.bat +++ b/Doc/make.bat @@ -144,12 +144,12 @@ if exist ..\Misc\NEWS ( ) if defined PAPER ( - set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS% + set SPHINXOPTS=--define latex_elements.papersize=%PAPER% %SPHINXOPTS% ) if "%1" EQU "htmlhelp" ( - set SPHINXOPTS=-D html_theme_options.body_max_width=none %SPHINXOPTS% + set SPHINXOPTS=--define html_theme_options.body_max_width=none %SPHINXOPTS% ) -cmd /S /C "%SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . "%BUILDDIR%\%1" %2 %3 %4 %5 %6 %7 %8 %9" +cmd /S /C "%SPHINXBUILD% %SPHINXOPTS% --builder %1 --doctree-dir build\doctrees . "%BUILDDIR%\%1" %2 %3 %4 %5 %6 %7 %8 %9" if "%1" EQU "htmlhelp" ( "%HTMLHELP%" "%BUILDDIR%\htmlhelp\python%DISTVERSION:.=%.hhp" diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 374404bf33abbe..71cc0c83de567e 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -245,13 +245,12 @@ handler is started. This search inspects the :keyword:`!except` clauses in turn until one is found that matches the exception. An expression-less :keyword:`!except` clause, if present, must be last; it matches any exception. -For an :keyword:`!except` clause with an expression, -that expression is evaluated, and the clause matches the exception -if the resulting object is "compatible" with the exception. An object is -compatible with an exception if the object is the class or a -:term:`non-virtual base class ` of the exception object, -or a tuple containing an item that is the class or a non-virtual base class -of the exception object. + +For an :keyword:`!except` clause with an expression, the +expression must evaluate to an exception type or a tuple of exception types. +The raised exception matches an :keyword:`!except` clause whose expression evaluates +to the class or a :term:`non-virtual base class ` of the exception object, +or to a tuple that contains such a class. If no :keyword:`!except` clause matches the exception, the search for an exception handler @@ -378,8 +377,10 @@ exception group with an empty message string. :: ... ExceptionGroup('', (BlockingIOError())) -An :keyword:`!except*` clause must have a matching type, -and this type cannot be a subclass of :exc:`BaseExceptionGroup`. +An :keyword:`!except*` clause must have a matching expression; it cannot be ``except*:``. +Furthermore, this expression cannot contain exception group types, because that would +have ambiguous semantics. + It is not possible to mix :keyword:`except` and :keyword:`!except*` in the same :keyword:`try`. :keyword:`break`, :keyword:`continue` and :keyword:`return` @@ -840,7 +841,7 @@ A literal pattern corresponds to most : | "None" : | "True" : | "False" - : | `signed_number`: NUMBER | "-" NUMBER + signed_number: ["-"] NUMBER The rule ``strings`` and the token ``NUMBER`` are defined in the :doc:`standard Python grammar <./grammar>`. Triple-quoted strings are @@ -1216,9 +1217,12 @@ A function definition defines a user-defined function object (see section : | `parameter_list_no_posonly` parameter_list_no_posonly: `defparameter` ("," `defparameter`)* ["," [`parameter_list_starargs`]] : | `parameter_list_starargs` - parameter_list_starargs: "*" [`parameter`] ("," `defparameter`)* ["," ["**" `parameter` [","]]] - : | "**" `parameter` [","] + parameter_list_starargs: "*" [`star_parameter`] ("," `defparameter`)* ["," [`parameter_star_kwargs`]] + : "*" ("," `defparameter`)+ ["," [`parameter_star_kwargs`]] + : | `parameter_star_kwargs` + parameter_star_kwargs: "**" `parameter` [","] parameter: `identifier` [":" `expression`] + star_parameter: `identifier` [":" ["*"] `expression`] defparameter: `parameter` ["=" `expression`] funcname: `identifier` @@ -1325,16 +1329,15 @@ and may only be passed by positional arguments. Parameters may have an :term:`annotation ` of the form "``: expression``" following the parameter name. Any parameter may have an annotation, even those of the form -``*identifier`` or ``**identifier``. Functions may have "return" annotation of +``*identifier`` or ``**identifier``. (As a special case, parameters of the form +``*identifier`` may have an annotation "``: *expression``".) Functions may have "return" annotation of the form "``-> expression``" after the parameter list. These annotations can be any valid Python expression. The presence of annotations does not change the -semantics of a function. The annotation values are available as values of -a dictionary keyed by the parameters' names in the :attr:`__annotations__` -attribute of the function object. If the ``annotations`` import from -:mod:`__future__` is used, annotations are preserved as strings at runtime which -enables postponed evaluation. Otherwise, they are evaluated when the function -definition is executed. In this case annotations may be evaluated in -a different order than they appear in the source code. +semantics of a function. See :ref:`annotations` for more information on annotations. + +.. versionchanged:: 3.11 + Parameters of the form "``*identifier``" may have an annotation + "``: *expression``". See :pep:`646`. .. index:: pair: lambda; expression @@ -1421,7 +1424,7 @@ dictionary. The class name is bound to this class object in the original local namespace. The order in which attributes are defined in the class body is preserved -in the new class's ``__dict__``. Note that this is reliable only right +in the new class's :attr:`~type.__dict__`. Note that this is reliable only right after the class is created and only for classes that were defined using the definition syntax. @@ -1452,8 +1455,8 @@ decorators. The result is then bound to the class name. A list of :ref:`type parameters ` may be given in square brackets immediately after the class's name. This indicates to static type checkers that the class is generic. At runtime, -the type parameters can be retrieved from the class's ``__type_params__`` -attribute. See :ref:`generic-classes` for more. +the type parameters can be retrieved from the class's +:attr:`~type.__type_params__` attribute. See :ref:`generic-classes` for more. .. versionchanged:: 3.12 Type parameter lists are new in Python 3.12. @@ -1620,15 +1623,18 @@ Type parameter lists .. versionadded:: 3.12 +.. versionchanged:: 3.13 + Support for default values was added (see :pep:`696`). + .. index:: single: type parameters .. productionlist:: python-grammar type_params: "[" `type_param` ("," `type_param`)* "]" type_param: `typevar` | `typevartuple` | `paramspec` - typevar: `identifier` (":" `expression`)? - typevartuple: "*" `identifier` - paramspec: "**" `identifier` + typevar: `identifier` (":" `expression`)? ("=" `expression`)? + typevartuple: "*" `identifier` ("=" `expression`)? + paramspec: "**" `identifier` ("=" `expression`)? :ref:`Functions ` (including :ref:`coroutines `), :ref:`classes ` and :ref:`type aliases ` may @@ -1663,8 +1669,8 @@ with more precision. The scope of type parameters is modeled with a special function (technically, an :ref:`annotation scope `) that wraps the creation of the generic object. -Generic functions, classes, and type aliases have a :attr:`!__type_params__` -attribute listing their type parameters. +Generic functions, classes, and type aliases have a +:attr:`~definition.__type_params__` attribute listing their type parameters. Type parameters come in three kinds: @@ -1694,19 +1700,31 @@ evaluated in a separate :ref:`annotation scope `. :data:`typing.TypeVarTuple`\ s and :data:`typing.ParamSpec`\ s cannot have bounds or constraints. +All three flavors of type parameters can also have a *default value*, which is used +when the type parameter is not explicitly provided. This is added by appending +a single equals sign (``=``) followed by an expression. Like the bounds and +constraints of type variables, the default value is not evaluated when the +object is created, but only when the type parameter's ``__default__`` attribute +is accessed. To this end, the default value is evaluated in a separate +:ref:`annotation scope `. If no default value is specified +for a type parameter, the ``__default__`` attribute is set to the special +sentinel object :data:`typing.NoDefault`. + The following example indicates the full set of allowed type parameter declarations:: def overly_generic[ SimpleTypeVar, + TypeVarWithDefault = int, TypeVarWithBound: int, TypeVarWithConstraints: (str, bytes), - *SimpleTypeVarTuple, - **SimpleParamSpec, + *SimpleTypeVarTuple = (int, float), + **SimpleParamSpec = (str, bytearray), ]( a: SimpleTypeVar, - b: TypeVarWithBound, - c: Callable[SimpleParamSpec, TypeVarWithConstraints], - *d: SimpleTypeVarTuple, + b: TypeVarWithDefault, + c: TypeVarWithBound, + d: Callable[SimpleParamSpec, TypeVarWithConstraints], + *e: SimpleTypeVarTuple, ): ... .. _generic-functions: @@ -1836,6 +1854,44 @@ Here, ``annotation-def`` (not a real keyword) indicates an :ref:`annotation scope `. The capitalized names like ``TYPE_PARAMS_OF_ListOrSet`` are not actually bound at runtime. +.. _annotations: + +Annotations +=========== + +.. versionchanged:: 3.14 + Annotations are now lazily evaluated by default. + +Variables and function parameters may carry :term:`annotations `, +created by adding a colon after the name, followed by an expression:: + + x: annotation = 1 + def f(param: annotation): ... + +Functions may also carry a return annotation following an arrow:: + + def f() -> annotation: ... + +Annotations are conventionally used for :term:`type hints `, but this +is not enforced by the language, and in general annotations may contain arbitrary +expressions. The presence of annotations does not change the runtime semantics of +the code, except if some mechanism is used that introspects and uses the annotations +(such as :mod:`dataclasses` or :func:`functools.singledispatch`). + +By default, annotations are lazily evaluated in a :ref:`annotation scope `. +This means that they are not evaluated when the code containing the annotation is evaluated. +Instead, the interpreter saves information that can be used to evaluate the annotation later +if requested. The :mod:`annotationlib` module provides tools for evaluating annotations. + +If the :ref:`future statement ` ``from __future__ import annotations`` is present, +all annotations are instead stored as strings:: + + >>> from __future__ import annotations + >>> def f(param: annotation): ... + >>> f.__annotations__ + {'param': 'annotation'} + + .. rubric:: Footnotes .. [#] The exception is propagated to the invocation stack unless @@ -1876,5 +1932,5 @@ like ``TYPE_PARAMS_OF_ListOrSet`` are not actually bound at runtime. therefore the function's :term:`docstring`. .. [#] A string literal appearing as the first statement in the class body is - transformed into the namespace's ``__doc__`` item and therefore the class's - :term:`docstring`. + transformed into the namespace's :attr:`~type.__doc__` item and therefore + the class's :term:`docstring`. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 75b656f385d34b..66b836eaf0008a 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -106,12 +106,16 @@ that mutable object is changed. Types affect almost all aspects of object behavior. Even the importance of object identity is affected in some sense: for immutable types, operations that compute new values may actually return a reference to any existing object with -the same type and value, while for mutable objects this is not allowed. E.g., -after ``a = 1; b = 1``, ``a`` and ``b`` may or may not refer to the same object -with the value one, depending on the implementation, but after ``c = []; d = -[]``, ``c`` and ``d`` are guaranteed to refer to two different, unique, newly -created empty lists. (Note that ``c = d = []`` assigns the same object to both -``c`` and ``d``.) +the same type and value, while for mutable objects this is not allowed. +For example, after ``a = 1; b = 1``, *a* and *b* may or may not refer to +the same object with the value one, depending on the implementation. +This is because :class:`int` is an immutable type, so the reference to ``1`` +can be reused. This behaviour depends on the implementation used, so should +not be relied upon, but is something to be aware of when making use of object +identity tests. +However, after ``c = []; d = []``, *c* and *d* are guaranteed to refer to two +different, unique, newly created empty lists. (Note that ``e = f = []`` assigns +the *same* object to both *e* and *f*.) .. _types: @@ -170,9 +174,12 @@ See for more details. .. versionchanged:: 3.9 - Evaluating :data:`NotImplemented` in a boolean context is deprecated. While - it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. - It will raise a :exc:`TypeError` in a future version of Python. + Evaluating :data:`NotImplemented` in a boolean context was deprecated. + +.. versionchanged:: 3.14 + Evaluating :data:`NotImplemented` in a boolean context now raises a :exc:`TypeError`. + It previously evaluated to :const:`True` and emitted a :exc:`DeprecationWarning` + since Python 3.9. Ellipsis @@ -215,7 +222,7 @@ properties: * A sign is shown only when the number is negative. -Python distinguishes between integers, floating point numbers, and complex +Python distinguishes between integers, floating-point numbers, and complex numbers: @@ -259,18 +266,18 @@ Booleans (:class:`bool`) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. index:: - pair: object; floating point - pair: floating point; number + pair: object; floating-point + pair: floating-point; number pair: C; language pair: Java; language -These represent machine-level double precision floating point numbers. You are +These represent machine-level double precision floating-point numbers. You are at the mercy of the underlying machine architecture (and C or Java implementation) for the accepted range and handling of overflow. Python does not -support single-precision floating point numbers; the savings in processor and +support single-precision floating-point numbers; the savings in processor and memory usage that are usually the reason for using these are dwarfed by the overhead of using objects in Python, so there is no reason to complicate the -language with two kinds of floating point numbers. +language with two kinds of floating-point numbers. :class:`numbers.Complex` (:class:`complex`) @@ -281,7 +288,7 @@ language with two kinds of floating point numbers. pair: complex; number These represent complex numbers as a pair of machine-level double precision -floating point numbers. The same caveats apply as for floating point numbers. +floating-point numbers. The same caveats apply as for floating-point numbers. The real and imaginary parts of a complex number ``z`` can be retrieved through the read-only attributes ``z.real`` and ``z.imag``. @@ -299,14 +306,17 @@ Sequences These represent finite ordered sets indexed by non-negative numbers. The built-in function :func:`len` returns the number of items of a sequence. When the length of a sequence is *n*, the index set contains the numbers 0, 1, -..., *n*-1. Item *i* of sequence *a* is selected by ``a[i]``. +..., *n*-1. Item *i* of sequence *a* is selected by ``a[i]``. Some sequences, +including built-in sequences, interpret negative subscripts by adding the +sequence length. For example, ``a[-2]`` equals ``a[n-2]``, the second to last +item of sequence a with length ``n``. .. index:: single: slicing Sequences also support slicing: ``a[i:j]`` selects all items with index *k* such that *i* ``<=`` *k* ``<`` *j*. When used as an expression, a slice is a -sequence of the same type. This implies that the index set is renumbered so -that it starts at 0. +sequence of the same type. The comment above about negative indexes also applies +to negative slice positions. Some sequences also support "extended slicing" with a third "step" parameter: ``a[i:j:k]`` selects all items of *a* with index *x* where ``x = i + n*k``, *n* @@ -370,7 +380,7 @@ Bytes A bytes object is an immutable array. The items are 8-bit bytes, represented by integers in the range 0 <= x < 256. Bytes literals - (like ``b'abc'``) and the built-in :func:`bytes()` constructor + (like ``b'abc'``) and the built-in :func:`bytes` constructor can be used to create bytes objects. Also, bytes objects can be decoded to strings via the :meth:`~bytes.decode` method. @@ -489,7 +499,7 @@ in the same order they were added sequentially over the dictionary. Replacing an existing key does not change the order, however removing a key and re-inserting it will add it to the end instead of keeping its old place. -Dictionaries are mutable; they can be created by the ``{...}`` notation (see +Dictionaries are mutable; they can be created by the ``{}`` notation (see section :ref:`dict`). .. index:: @@ -554,8 +564,9 @@ Special read-only attributes in which the function was defined. * - .. attribute:: function.__closure__ - - ``None`` or a :class:`tuple` of cells that contain bindings for the - function's free variables. + - ``None`` or a :class:`tuple` of cells that contain bindings for the names specified + in the :attr:`~codeobject.co_freevars` attribute of the function's + :attr:`code object `. A cell object has the attribute ``cell_contents``. This can be used to get the value of the cell, as well as set the value. @@ -571,6 +582,7 @@ Special writable attributes single: __defaults__ (function attribute) single: __code__ (function attribute) single: __annotations__ (function attribute) + single: __annotate__ (function attribute) single: __kwdefaults__ (function attribute) single: __type_params__ (function attribute) @@ -584,7 +596,6 @@ Most of these attributes check the type of the assigned value: * - .. attribute:: function.__doc__ - The function's documentation string, or ``None`` if unavailable. - Not inherited by subclasses. * - .. attribute:: function.__name__ - The function's name. @@ -618,7 +629,17 @@ Most of these attributes check the type of the assigned value: :term:`parameters `. The keys of the dictionary are the parameter names, and ``'return'`` for the return annotation, if provided. - See also: :ref:`annotations-howto`. + See also: :attr:`object.__annotations__`. + + .. versionchanged:: 3.14 + Annotations are now :ref:`lazily evaluated `. + See :pep:`649`. + + * - .. attribute:: function.__annotate__ + - The :term:`annotate function` for this function, or ``None`` + if the function has no annotations. See :attr:`object.__annotate__`. + + .. versionadded:: 3.14 * - .. attribute:: function.__kwdefaults__ - A :class:`dictionary ` containing defaults for keyword-only @@ -724,14 +745,7 @@ When an instance method object is derived from a :class:`classmethod` object, th itself, so that calling either ``x.f(1)`` or ``C.f(1)`` is equivalent to calling ``f(C,1)`` where ``f`` is the underlying function. -Note that the transformation from :ref:`function object ` -to instance method -object happens each time the attribute is retrieved from the instance. In -some cases, a fruitful optimization is to assign the attribute to a local -variable and call that local variable. Also notice that this -transformation only happens for user-defined functions; other callable -objects (and all non-callable objects) are retrieved without -transformation. It is also important to note that user-defined functions +It is important to note that user-defined functions which are attributes of a class instance are not converted to bound methods; this *only* happens when the function is an attribute of the class. @@ -832,6 +846,7 @@ this case, the special read-only attribute :attr:`!__self__` is set to the objec denoted by *alist*. (The attribute has the same semantics as it does with :attr:`other instance methods `.) +.. _classes: Classes ^^^^^^^ @@ -850,6 +865,8 @@ Instances of arbitrary classes can be made callable by defining a :meth:`~object.__call__` method in their class. +.. _module-objects: + Modules ------- @@ -875,48 +892,247 @@ Attribute assignment updates the module's namespace dictionary, e.g., .. index:: single: __name__ (module attribute) - single: __doc__ (module attribute) + single: __spec__ (module attribute) + single: __package__ (module attribute) + single: __loader__ (module attribute) + single: __path__ (module attribute) single: __file__ (module attribute) + single: __cached__ (module attribute) + single: __doc__ (module attribute) single: __annotations__ (module attribute) + single: __annotate__ (module attribute) pair: module; namespace -Predefined (writable) attributes: +.. _import-mod-attrs: + +Import-related attributes on module objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Module objects have the following attributes that relate to the +:ref:`import system `. When a module is created using the machinery associated +with the import system, these attributes are filled in based on the module's +:term:`spec `, before the :term:`loader` executes and loads the +module. + +To create a module dynamically rather than using the import system, +it's recommended to use :func:`importlib.util.module_from_spec`, +which will set the various import-controlled attributes to appropriate values. +It's also possible to use the :class:`types.ModuleType` constructor to create +modules directly, but this technique is more error-prone, as most attributes +must be manually set on the module object after it has been created when using +this approach. + +.. caution:: + + With the exception of :attr:`~module.__name__`, it is **strongly** + recommended that you rely on :attr:`~module.__spec__` and its attributes + instead of any of the other individual attributes listed in this subsection. + Note that updating an attribute on :attr:`!__spec__` will not update the + corresponding attribute on the module itself: + + .. doctest:: + + >>> import typing + >>> typing.__name__, typing.__spec__.name + ('typing', 'typing') + >>> typing.__spec__.name = 'spelling' + >>> typing.__name__, typing.__spec__.name + ('typing', 'spelling') + >>> typing.__name__ = 'keyboard_smashing' + >>> typing.__name__, typing.__spec__.name + ('keyboard_smashing', 'spelling') + +.. attribute:: module.__name__ + + The name used to uniquely identify the module in the import system. + For a directly executed module, this will be set to ``"__main__"``. + + This attribute must be set to the fully qualified name of the module. + It is expected to match the value of + :attr:`module.__spec__.name `. + +.. attribute:: module.__spec__ + + A record of the module's import-system-related state. + + Set to the :class:`module spec ` that was + used when importing the module. See :ref:`module-specs` for more details. + + .. versionadded:: 3.4 + +.. attribute:: module.__package__ + + The :term:`package` a module belongs to. + + If the module is top-level (that is, not a part of any specific package) + then the attribute should be set to ``''`` (the empty string). Otherwise, + it should be set to the name of the module's package (which can be equal to + :attr:`module.__name__` if the module itself is a package). See :pep:`366` + for further details. + + This attribute is used instead of :attr:`~module.__name__` to calculate + explicit relative imports for main modules. It defaults to ``None`` for + modules created dynamically using the :class:`types.ModuleType` constructor; + use :func:`importlib.util.module_from_spec` instead to ensure the attribute + is set to a :class:`str`. + + It is **strongly** recommended that you use + :attr:`module.__spec__.parent ` + instead of :attr:`!module.__package__`. :attr:`__package__` is now only used + as a fallback if :attr:`!__spec__.parent` is not set, and this fallback + path is deprecated. + + .. versionchanged:: 3.4 + This attribute now defaults to ``None`` for modules created dynamically + using the :class:`types.ModuleType` constructor. + Previously the attribute was optional. + + .. versionchanged:: 3.6 + The value of :attr:`!__package__` is expected to be the same as + :attr:`__spec__.parent `. + :attr:`__package__` is now only used as a fallback during import + resolution if :attr:`!__spec__.parent` is not defined. + + .. versionchanged:: 3.10 + :exc:`ImportWarning` is raised if an import resolution falls back to + :attr:`!__package__` instead of + :attr:`__spec__.parent `. - :attr:`__name__` - The module's name. + .. versionchanged:: 3.12 + Raise :exc:`DeprecationWarning` instead of :exc:`ImportWarning` when + falling back to :attr:`!__package__` during import resolution. + + .. deprecated-removed:: 3.13 3.15 + :attr:`!__package__` will cease to be set or taken into consideration + by the import system or standard library. + +.. attribute:: module.__loader__ + + The :term:`loader` object that the import machinery used to load the module. + + This attribute is mostly useful for introspection, but can be used for + additional loader-specific functionality, for example getting data + associated with a loader. + + :attr:`!__loader__` defaults to ``None`` for modules created dynamically + using the :class:`types.ModuleType` constructor; + use :func:`importlib.util.module_from_spec` instead to ensure the attribute + is set to a :term:`loader` object. + + It is **strongly** recommended that you use + :attr:`module.__spec__.loader ` + instead of :attr:`!module.__loader__`. + + .. versionchanged:: 3.4 + This attribute now defaults to ``None`` for modules created dynamically + using the :class:`types.ModuleType` constructor. + Previously the attribute was optional. + + .. deprecated-removed:: 3.12 3.16 + Setting :attr:`!__loader__` on a module while failing to set + :attr:`!__spec__.loader` is deprecated. In Python 3.16, + :attr:`!__loader__` will cease to be set or taken into consideration by + the import system or the standard library. + +.. attribute:: module.__path__ + + A (possibly empty) :term:`sequence` of strings enumerating the locations + where the package's submodules will be found. Non-package modules should + not have a :attr:`!__path__` attribute. See :ref:`package-path-rules` for + more details. + + It is **strongly** recommended that you use + :attr:`module.__spec__.submodule_search_locations ` + instead of :attr:`!module.__path__`. + +.. attribute:: module.__file__ +.. attribute:: module.__cached__ + + :attr:`!__file__` and :attr:`!__cached__` are both optional attributes that + may or may not be set. Both attributes should be a :class:`str` when they + are available. + + :attr:`!__file__` indicates the pathname of the file from which the module + was loaded (if loaded from a file), or the pathname of the shared library + file for extension modules loaded dynamically from a shared library. + It might be missing for certain types of modules, such as C modules that are + statically linked into the interpreter, and the + :ref:`import system ` may opt to leave it unset if it + has no semantic meaning (for example, a module loaded from a database). + + If :attr:`!__file__` is set then the :attr:`!__cached__` attribute might + also be set, which is the path to any compiled version of + the code (for example, a byte-compiled file). The file does not need to exist + to set this attribute; the path can simply point to where the + compiled file *would* exist (see :pep:`3147`). + + Note that :attr:`!__cached__` may be set even if :attr:`!__file__` is not + set. However, that scenario is quite atypical. Ultimately, the + :term:`loader` is what makes use of the module spec provided by the + :term:`finder` (from which :attr:`!__file__` and :attr:`!__cached__` are + derived). So if a loader can load from a cached module but otherwise does + not load from a file, that atypical scenario may be appropriate. + + It is **strongly** recommended that you use + :attr:`module.__spec__.cached ` + instead of :attr:`!module.__cached__`. + + .. deprecated-removed:: 3.13 3.15 + Setting :attr:`!__cached__` on a module while failing to set + :attr:`!__spec__.cached` is deprecated. In Python 3.15, + :attr:`!__cached__` will cease to be set or taken into consideration by + the import system or standard library. + +Other writable attributes on module objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As well as the import-related attributes listed above, module objects also have +the following writable attributes: + +.. attribute:: module.__doc__ - :attr:`__doc__` - The module's documentation string, or ``None`` if - unavailable. + The module's documentation string, or ``None`` if unavailable. + See also: :attr:`__doc__ attributes `. - :attr:`__file__` - The pathname of the file from which the - module was loaded, if it was loaded from a file. - The :attr:`__file__` - attribute may be missing for certain types of modules, such as C modules - that are statically linked into the interpreter. For extension modules - loaded dynamically from a shared library, it's the pathname of the shared - library file. +.. attribute:: module.__annotations__ - :attr:`__annotations__` - A dictionary containing - :term:`variable annotations ` collected during - module body execution. For best practices on working - with :attr:`__annotations__`, please see :ref:`annotations-howto`. + A dictionary containing :term:`variable annotations ` + collected during module body execution. For best practices on working with + :attr:`!__annotations__`, see :mod:`annotationlib`. + + .. versionchanged:: 3.14 + Annotations are now :ref:`lazily evaluated `. + See :pep:`649`. + +.. attribute:: module.__annotate__ + + The :term:`annotate function` for this module, or ``None`` if the module has + no annotations. See also: :attr:`~object.__annotate__` attributes. + + .. versionadded:: 3.14 + +Module dictionaries +^^^^^^^^^^^^^^^^^^^ + +Module objects also have the following special read-only attribute: .. index:: single: __dict__ (module attribute) +.. attribute:: module.__dict__ -Special read-only attribute: :attr:`~object.__dict__` is the module's -namespace as a dictionary object. + The module's namespace as a dictionary object. Uniquely among the attributes + listed here, :attr:`!__dict__` cannot be accessed as a global variable from + within a module; it can only be accessed as an attribute on module objects. -.. impl-detail:: + .. impl-detail:: - Because of the way CPython clears module dictionaries, the module - dictionary will be cleared when the module falls out of scope even if the - dictionary still has live references. To avoid this, copy the dictionary - or keep the module around while using its dictionary directly. + Because of the way CPython clears module dictionaries, the module + dictionary will be cleared when the module falls out of scope even if the + dictionary still has live references. To avoid this, copy the dictionary + or keep the module around while using its dictionary directly. +.. _class-attrs-and-methods: + Custom classes -------------- @@ -929,11 +1145,8 @@ name is not found there, the attribute search continues in the base classes. This search of the base classes uses the C3 method resolution order which behaves correctly even in the presence of 'diamond' inheritance structures where there are multiple inheritance paths leading back to a common ancestor. -Additional details on the C3 MRO used by Python can be found in the -documentation accompanying the 2.3 release at -https://www.python.org/download/releases/2.3/mro/. - -.. XXX: Could we add that MRO doc as an appendix to the language ref? +Additional details on the C3 MRO used by Python can be found at +:ref:`python_2.3_mro`. .. index:: pair: object; class @@ -962,6 +1175,9 @@ of a base class. A class object can be called (see above) to yield a class instance (see below). +Special attributes +^^^^^^^^^^^^^^^^^^ + .. index:: single: __name__ (class attribute) single: __module__ (class attribute) @@ -969,37 +1185,126 @@ A class object can be called (see above) to yield a class instance (see below). single: __bases__ (class attribute) single: __doc__ (class attribute) single: __annotations__ (class attribute) + single: __annotate__ (class attribute) single: __type_params__ (class attribute) + single: __static_attributes__ (class attribute) + single: __firstlineno__ (class attribute) -Special attributes: +.. list-table:: + :header-rows: 1 - :attr:`~definition.__name__` - The class name. + * - Attribute + - Meaning - :attr:`__module__` - The name of the module in which the class was defined. + * - .. attribute:: type.__name__ + - The class's name. + See also: :attr:`__name__ attributes `. - :attr:`~object.__dict__` - The dictionary containing the class's namespace. + * - .. attribute:: type.__qualname__ + - The class's :term:`qualified name`. + See also: :attr:`__qualname__ attributes `. + + * - .. attribute:: type.__module__ + - The name of the module in which the class was defined. + + * - .. attribute:: type.__dict__ + - A :class:`mapping proxy ` + providing a read-only view of the class's namespace. + See also: :attr:`__dict__ attributes `. + + * - .. attribute:: type.__bases__ + - A :class:`tuple` containing the class's bases. + In most cases, for a class defined as ``class X(A, B, C)``, + ``X.__bases__`` will be exactly equal to ``(A, B, C)``. + + * - .. attribute:: type.__doc__ + - The class's documentation string, or ``None`` if undefined. + Not inherited by subclasses. + + * - .. attribute:: type.__annotations__ + - A dictionary containing + :term:`variable annotations ` + collected during class body execution. See also: + :attr:`__annotations__ attributes `. + + For best practices on working with :attr:`~object.__annotations__`, + please see :mod:`annotationlib`. + + .. caution:: + + Accessing the :attr:`!__annotations__` attribute of a class + object directly may yield incorrect results in the presence of + metaclasses. In addition, the attribute may not exist for + some classes. Use :func:`annotationlib.get_annotations` to + retrieve class annotations safely. + + .. versionchanged:: 3.14 + Annotations are now :ref:`lazily evaluated `. + See :pep:`649`. + + * - .. method:: type.__annotate__ + - The :term:`annotate function` for this class, or ``None`` + if the class has no annotations. + See also: :attr:`__annotate__ attributes `. + + .. caution:: + + Accessing the :attr:`!__annotate__` attribute of a class + object directly may yield incorrect results in the presence of + metaclasses. Use :func:`annotationlib.get_annotate_function` to + retrieve the annotate function safely. - :attr:`~class.__bases__` - A tuple containing the base classes, in the order of - their occurrence in the base class list. + .. versionadded:: 3.14 - :attr:`__doc__` - The class's documentation string, or ``None`` if undefined. + * - .. attribute:: type.__type_params__ + - A :class:`tuple` containing the :ref:`type parameters ` of + a :ref:`generic class `. + + .. versionadded:: 3.12 + + * - .. attribute:: type.__static_attributes__ + - A :class:`tuple` containing names of attributes of this class which are + assigned through ``self.X`` from any function in its body. + + .. versionadded:: 3.13 - :attr:`__annotations__` - A dictionary containing - :term:`variable annotations ` - collected during class body execution. For best practices on - working with :attr:`__annotations__`, please see - :ref:`annotations-howto`. + * - .. attribute:: type.__firstlineno__ + - The line number of the first line of the class definition, + including decorators. + Setting the :attr:`__module__` attribute removes the + :attr:`!__firstlineno__` item from the type's dictionary. - :attr:`__type_params__` - A tuple containing the :ref:`type parameters ` of - a :ref:`generic class `. + .. versionadded:: 3.13 + + * - .. attribute:: type.__mro__ + - The :class:`tuple` of classes that are considered when looking for + base classes during method resolution. + + +Special methods +^^^^^^^^^^^^^^^ +In addition to the special attributes described above, all Python classes also +have the following two methods available: + +.. method:: type.mro + + This method can be overridden by a metaclass to customize the method + resolution order for its instances. It is called at class instantiation, + and its result is stored in :attr:`~type.__mro__`. + +.. method:: type.__subclasses__ + + Each class keeps a list of weak references to its immediate subclasses. This + method returns a list of all those references still alive. The list is in + definition order. Example: + + .. doctest:: + + >>> class A: pass + >>> class B(A): pass + >>> A.__subclasses__() + [] Class instances --------------- @@ -1039,12 +1344,22 @@ dictionary directly. Class instances can pretend to be numbers, sequences, or mappings if they have methods with certain special names. See section :ref:`specialnames`. +Special attributes +^^^^^^^^^^^^^^^^^^ + .. index:: single: __dict__ (instance attribute) single: __class__ (instance attribute) -Special attributes: :attr:`~object.__dict__` is the attribute dictionary; -:attr:`~instance.__class__` is the instance's class. +.. attribute:: object.__class__ + + The class to which a class instance belongs. + +.. attribute:: object.__dict__ + + A dictionary or other mapping object used to store an object's (writable) + attributes. Not all instances have a :attr:`!__dict__` attribute; see the + section on :ref:`slots` for more details. I/O objects (also known as file objects) @@ -1159,10 +1474,14 @@ Special read-only attributes * - .. attribute:: codeobject.co_cellvars - A :class:`tuple` containing the names of :ref:`local variables ` - that are referenced by nested functions inside the function + that are referenced from at least one :term:`nested scope` inside the function * - .. attribute:: codeobject.co_freevars - - A :class:`tuple` containing the names of free variables in the function + - A :class:`tuple` containing the names of + :term:`free (closure) variables ` that a :term:`nested scope` + references in an outer scope. See also :attr:`function.__closure__`. + + Note: references to global and builtin names are *not* included. * - .. attribute:: codeobject.co_code - A string representing the sequence of :term:`bytecode` instructions in @@ -1188,7 +1507,7 @@ Special read-only attributes .. deprecated:: 3.12 This attribute of code objects is deprecated, and may be removed in - Python 3.14. + Python 3.15. * - .. attribute:: codeobject.co_stacksize - The required stack size of the code object @@ -1217,9 +1536,9 @@ Other bits in :attr:`~codeobject.co_flags` are reserved for internal use. .. index:: single: documentation string -If a code object represents a function, the first item in -:attr:`~codeobject.co_consts` is -the documentation string of the function, or ``None`` if undefined. +If a code object represents a function and has a docstring, +the first item in :attr:`~codeobject.co_consts` is +the docstring of the function. Methods on code objects ~~~~~~~~~~~~~~~~~~~~~~~ @@ -1231,7 +1550,7 @@ Methods on code objects The iterator returns :class:`tuple`\s containing the ``(start_line, end_line, start_column, end_column)``. The *i-th* tuple corresponds to the - position of the source code that compiled to the *i-th* instruction. + position of the source code that compiled to the *i-th* code unit. Column information is 0-indexed utf-8 byte offsets on the given source line. @@ -1335,8 +1654,13 @@ Special read-only attributes ``object.__getattr__`` with arguments ``obj`` and ``"f_code"``. * - .. attribute:: frame.f_locals - - The dictionary used by the frame to look up - :ref:`local variables ` + - The mapping used by the frame to look up + :ref:`local variables `. + If the frame refers to an :term:`optimized scope`, + this may return a write-through proxy object. + + .. versionchanged:: 3.13 + Return a proxy for optimized scopes. * - .. attribute:: frame.f_globals - The dictionary used by the frame to look up @@ -1650,6 +1974,8 @@ Basic customization It is not guaranteed that :meth:`__del__` methods are called for objects that still exist when the interpreter exits. + :class:`weakref.finalize` provides a straightforward way to register + a cleanup function to be called when an object is garbage collected. .. note:: @@ -1707,7 +2033,8 @@ Basic customization "informal" string representation of instances of that class is required. This is typically used for debugging, so it is important that the representation - is information-rich and unambiguous. + is information-rich and unambiguous. A default implementation is provided by the + :class:`object` class itself. .. index:: single: string; __str__() (object method) @@ -1717,10 +2044,10 @@ Basic customization .. method:: object.__str__(self) - Called by :func:`str(object) ` and the built-in functions - :func:`format` and :func:`print` to compute the "informal" or nicely + Called by :func:`str(object) `, the default :meth:`__format__` implementation, + and the built-in function :func:`print`, to compute the "informal" or nicely printable string representation of an object. The return value must be a - :ref:`string ` object. + :ref:`str ` object. This method differs from :meth:`object.__repr__` in that there is no expectation that :meth:`__str__` return a valid Python expression: a more @@ -1737,7 +2064,8 @@ Basic customization .. index:: pair: built-in function; bytes Called by :ref:`bytes ` to compute a byte-string representation - of an object. This should return a :class:`bytes` object. + of an object. This should return a :class:`bytes` object. The :class:`object` + class itself does not provide this method. .. index:: single: string; __format__() (object method) @@ -1761,6 +2089,9 @@ Basic customization The return value must be a string object. + The default implementation by the :class:`object` class should be given + an empty *format_spec* string. It delegates to :meth:`__str__`. + .. versionchanged:: 3.4 The __format__ method of ``object`` itself raises a :exc:`TypeError` if passed any non-empty string. @@ -1803,6 +2134,12 @@ Basic customization ``(x` with multiple slotted parent @@ -2554,7 +2892,7 @@ in the local namespace as the defined class. When a new class is created by ``type.__new__``, the object provided as the namespace parameter is copied to a new ordered mapping and the original object is discarded. The new copy is wrapped in a read-only proxy, which -becomes the :attr:`~object.__dict__` attribute of the class object. +becomes the :attr:`~type.__dict__` attribute of the class object. .. seealso:: @@ -2582,14 +2920,14 @@ order to allow the addition of Abstract Base Classes (ABCs) as "virtual base classes" to any class or type (including built-in types), including other ABCs. -.. method:: class.__instancecheck__(self, instance) +.. method:: type.__instancecheck__(self, instance) Return true if *instance* should be considered a (direct or indirect) instance of *class*. If defined, called to implement ``isinstance(instance, class)``. -.. method:: class.__subclasscheck__(self, subclass) +.. method:: type.__subclasscheck__(self, subclass) Return true if *subclass* should be considered a (direct or indirect) subclass of *class*. If defined, called to implement ``issubclass(subclass, @@ -2605,8 +2943,8 @@ case the instance is itself a class. :pep:`3119` - Introducing Abstract Base Classes Includes the specification for customizing :func:`isinstance` and - :func:`issubclass` behavior through :meth:`~class.__instancecheck__` and - :meth:`~class.__subclasscheck__`, with motivation for this functionality + :func:`issubclass` behavior through :meth:`~type.__instancecheck__` and + :meth:`~type.__subclasscheck__`, with motivation for this functionality in the context of adding Abstract Base Classes (see the :mod:`abc` module) to the language. @@ -2764,6 +3102,7 @@ Emulating callable objects Called when the instance is "called" as a function; if this method is defined, ``x(arg1, arg2, ...)`` roughly translates to ``type(x).__call__(x, arg1, ...)``. + The :class:`object` class itself does not provide this method. .. _sequence-types: @@ -2771,10 +3110,11 @@ Emulating callable objects Emulating container types ------------------------- -The following methods can be defined to implement container objects. Containers -usually are :term:`sequences ` (such as :class:`lists ` or +The following methods can be defined to implement container objects. None of them +are provided by the :class:`object` class itself. Containers usually are +:term:`sequences ` (such as :class:`lists ` or :class:`tuples `) or :term:`mappings ` (like -:class:`dictionaries `), +:term:`dictionaries `), but can represent other containers as well. The first set of methods is used either to emulate a sequence or to emulate a mapping; the difference is that for a sequence, the allowable keys should be the integers *k* for which ``0 <= k < @@ -3008,12 +3348,13 @@ left undefined. These methods are called to implement the binary arithmetic operations (``+``, ``-``, ``*``, ``@``, ``/``, ``//``, ``%``, :func:`divmod`, :func:`pow`, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with reflected - (swapped) operands. These functions are only called if the left operand does - not support the corresponding operation [#]_ and the operands are of different - types. [#]_ For instance, to evaluate the expression ``x - y``, where *y* is - an instance of a class that has an :meth:`__rsub__` method, - ``type(y).__rsub__(y, x)`` is called if ``type(x).__sub__(x, y)`` returns - :data:`NotImplemented`. + (swapped) operands. These functions are only called if the operands + are of different types, when the left operand does not support the corresponding + operation [#]_, or the right operand's class is derived from the left operand's + class. [#]_ For instance, to evaluate the expression ``x - y``, where *y* is + an instance of a class that has an :meth:`__rsub__` method, ``type(y).__rsub__(y, x)`` + is called if ``type(x).__sub__(x, y)`` returns :data:`NotImplemented` or ``type(y)`` + is a subclass of ``type(x)``. [#]_ .. index:: pair: built-in function; pow @@ -3028,7 +3369,6 @@ left undefined. non-reflected method. This behavior allows subclasses to override their ancestors' operations. - .. method:: object.__iadd__(self, other) object.__isub__(self, other) object.__imul__(self, other) @@ -3110,11 +3450,8 @@ left undefined. return the value of the object truncated to an :class:`~numbers.Integral` (typically an :class:`int`). - The built-in function :func:`int` falls back to :meth:`__trunc__` if neither - :meth:`__int__` nor :meth:`__index__` is defined. - - .. versionchanged:: 3.11 - The delegation of :func:`int` to :meth:`__trunc__` is deprecated. + .. versionchanged:: 3.14 + :func:`int` no longer delegates to the :meth:`~object.__trunc__` method. .. _context-managers: @@ -3137,6 +3474,7 @@ Typical uses of context managers include saving and restoring various kinds of global state, locking and unlocking resources, closing opened files, etc. For more information on context managers, see :ref:`typecontextmanager`. +The :class:`object` class itself does not provide the context manager methods. .. method:: object.__enter__(self) @@ -3240,6 +3578,51 @@ implement the protocol in Python. :class:`collections.abc.Buffer` ABC for buffer types. +Annotations +----------- + +Functions, classes, and modules may contain :term:`annotations `, +which are a way to associate information (usually :term:`type hints `) +with a symbol. + +.. attribute:: object.__annotations__ + + This attribute contains the annotations for an object. It is + :ref:`lazily evaluated `, so accessing the attribute may + execute arbitrary code and raise exceptions. If evaluation is successful, the + attribute is set to a dictionary mapping from variable names to annotations. + + .. versionchanged:: 3.14 + Annotations are now lazily evaluated. + +.. method:: object.__annotate__(format) + + An :term:`annotate function`. + Returns a new dictionary object mapping attribute/parameter names to their annotation values. + + Takes a format parameter specifying the format in which annotations values should be provided. + It must be a member of the :class:`annotationlib.Format` enum, or an integer with + a value corresponding to a member of the enum. + + If an annotate function doesn't support the requested format, it must raise + :exc:`NotImplementedError`. Annotate functions must always support + :attr:`~annotationlib.Format.VALUE` format; they must not raise + :exc:`NotImplementedError()` when called with this format. + + When called with :attr:`~annotationlib.Format.VALUE` format, an annotate function may raise + :exc:`NameError`; it must not raise :exc:`!NameError` when called requesting any other format. + + If an object does not have any annotations, :attr:`~object.__annotate__` should preferably be set + to ``None`` (it can’t be deleted), rather than set to a function that returns an empty dict. + + .. versionadded:: 3.14 + +.. seealso:: + + :pep:`649` --- Deferred evaluation of annotation using descriptors + Introduces lazy evaluation of annotations and the ``__annotate__`` function. + + .. _special-lookup: Special method lookup @@ -3341,6 +3724,8 @@ are awaitable. Must return an :term:`iterator`. Should be used to implement :term:`awaitable` objects. For instance, :class:`asyncio.Future` implements this method to be compatible with the :keyword:`await` expression. + The :class:`object` class itself is not awaitable and does not provide + this method. .. note:: @@ -3426,6 +3811,9 @@ its ``__anext__`` method. Asynchronous iterators can be used in an :keyword:`async for` statement. +The :class:`object` class itself does not provide these methods. + + .. method:: object.__aiter__(self) Must return an *asynchronous iterator* object. @@ -3472,6 +3860,8 @@ suspend execution in its ``__aenter__`` and ``__aexit__`` methods. Asynchronous context managers can be used in an :keyword:`async with` statement. +The :class:`object` class itself does not provide these methods. + .. method:: object.__aenter__(self) Semantically similar to :meth:`~object.__enter__`, the only @@ -3513,7 +3903,10 @@ An example of an asynchronous context manager class:: method—that will instead have the opposite effect of explicitly *blocking* such fallback. -.. [#] For operands of the same type, it is assumed that if the non-reflected - method -- such as :meth:`~object.__add__` -- fails then the overall - operation is not - supported, which is why the reflected method is not called. +.. [#] For operands of the same type, it is assumed that if the non-reflected method + (such as :meth:`~object.__add__`) fails then the operation is not supported, which is why the + reflected method is not called. + +.. [#] If the right operand's type is a subclass of the left operand's type, the + reflected method having precedence allows subclasses to override their ancestors' + operations. diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index cea3a56ba51644..cb6c524dd97a30 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -90,7 +90,7 @@ If a name is bound in a block, it is a local variable of that block, unless declared as :keyword:`nonlocal` or :keyword:`global`. If a name is bound at the module level, it is a global variable. (The variables of the module code block are local and global.) If a variable is used in a code block but not -defined there, it is a :dfn:`free variable`. +defined there, it is a :term:`free variable`. Each occurrence of a name in the program text refers to the :dfn:`binding` of that name established by the following name resolution rules. @@ -139,8 +139,9 @@ namespace. Names are resolved in the top-level namespace by searching the global namespace, i.e. the namespace of the module containing the code block, and the builtins namespace, the namespace of the module :mod:`builtins`. The global namespace is searched first. If the names are not found there, the -builtins namespace is searched. The :keyword:`!global` statement must precede -all uses of the listed names. +builtins namespace is searched next. If the names are also not found in the +builtins namespace, new variables are created in the global namespace. +The global statement must precede all uses of the listed names. The :keyword:`global` statement has the same scope as a name binding operation in the same block. If the nearest enclosing scope for a free variable contains @@ -189,14 +190,15 @@ However, the following will succeed:: Annotation scopes ----------------- -:ref:`Type parameter lists ` and :keyword:`type` statements +:term:`Annotations `, :ref:`type parameter lists ` +and :keyword:`type` statements introduce *annotation scopes*, which behave mostly like function scopes, -but with some exceptions discussed below. :term:`Annotations ` -currently do not use annotation scopes, but they are expected to use -annotation scopes in Python 3.13 when :pep:`649` is implemented. +but with some exceptions discussed below. Annotation scopes are used in the following contexts: +* :term:`Function annotations `. +* :term:`Variable annotations `. * Type parameter lists for :ref:`generic type aliases `. * Type parameter lists for :ref:`generic functions `. A generic function's annotations are @@ -204,7 +206,7 @@ Annotation scopes are used in the following contexts: * Type parameter lists for :ref:`generic classes `. A generic class's base classes and keyword arguments are executed within the annotation scope, but its decorators are not. -* The bounds and constraints for type variables +* The bounds, constraints, and default values for type parameters (:ref:`lazily evaluated `). * The value of type aliases (:ref:`lazily evaluated `). @@ -224,24 +226,34 @@ Annotation scopes differ from function scopes in the following ways: statements in inner scopes. This includes only type parameters, as no other syntactic elements that can appear within annotation scopes can introduce new names. * While annotation scopes have an internal name, that name is not reflected in the - :term:`__qualname__ ` of objects defined within the scope. - Instead, the :attr:`!__qualname__` + :term:`qualified name` of objects defined within the scope. + Instead, the :attr:`~definition.__qualname__` of such objects is as if the object were defined in the enclosing scope. .. versionadded:: 3.12 Annotation scopes were introduced in Python 3.12 as part of :pep:`695`. +.. versionchanged:: 3.13 + Annotation scopes are also used for type parameter defaults, as + introduced by :pep:`696`. + +.. versionchanged:: 3.14 + Annotation scopes are now also used for annotations, as specified in + :pep:`649` and :pep:`749`. + .. _lazy-evaluation: Lazy evaluation --------------- -The values of type aliases created through the :keyword:`type` statement are -*lazily evaluated*. The same applies to the bounds and constraints of type +Most annotation scopes are *lazily evaluated*. This includes annotations, +the values of type aliases created through the :keyword:`type` statement, and +the bounds, constraints, and default values of type variables created through the :ref:`type parameter syntax `. This means that they are not evaluated when the type alias or type variable is -created. Instead, they are only evaluated when doing so is necessary to resolve -an attribute access. +created, or when the object carrying annotations is created. Instead, they +are only evaluated when necessary, for example when the ``__value__`` +attribute on a type alias is accessed. Example: @@ -325,6 +337,9 @@ enclosing namespace, but in the global namespace. [#]_ The :func:`exec` and :func:`eval` functions have optional arguments to override the global and local namespace. If only one namespace is specified, it is used for both. +.. XXX(ncoghlan) above is only accurate for string execution. When executing code objects, + closure cells may now be passed explicitly to resolve co_freevars references. + Docs issue: https://github.com/python/cpython/issues/122826 .. _exceptions: diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 00b57effd3e1c0..7c95b207b1aed2 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -28,13 +28,12 @@ Arithmetic conversions .. index:: pair: arithmetic; conversion When a description of an arithmetic operator below uses the phrase "the numeric -arguments are converted to a common type", this means that the operator +arguments are converted to a common real type", this means that the operator implementation for built-in types works as follows: -* If either argument is a complex number, the other is converted to complex; +* If both arguments are complex numbers, no conversion is performed; -* otherwise, if either argument is a floating point number, the other is - converted to floating point; +* if either argument is a complex or a floating-point number, the other is converted to a floating-point number; * otherwise, both must be integers and no conversion is necessary. @@ -83,18 +82,47 @@ exception. pair: name; mangling pair: private; names -**Private name mangling:** When an identifier that textually occurs in a class -definition begins with two or more underscore characters and does not end in two -or more underscores, it is considered a :dfn:`private name` of that class. -Private names are transformed to a longer form before code is generated for -them. The transformation inserts the class name, with leading underscores -removed and a single underscore inserted, in front of the name. For example, -the identifier ``__spam`` occurring in a class named ``Ham`` will be transformed -to ``_Ham__spam``. This transformation is independent of the syntactical -context in which the identifier is used. If the transformed name is extremely -long (longer than 255 characters), implementation defined truncation may happen. -If the class name consists only of underscores, no transformation is done. +Private name mangling +^^^^^^^^^^^^^^^^^^^^^ +When an identifier that textually occurs in a class definition begins with two +or more underscore characters and does not end in two or more underscores, it +is considered a :dfn:`private name` of that class. + +.. seealso:: + + The :ref:`class specifications `. + +More precisely, private names are transformed to a longer form before code is +generated for them. If the transformed name is longer than 255 characters, +implementation-defined truncation may happen. + +The transformation is independent of the syntactical context in which the +identifier is used but only the following private identifiers are mangled: + +- Any name used as the name of a variable that is assigned or read or any + name of an attribute being accessed. + + The :attr:`~definition.__name__` attribute of nested functions, classes, and + type aliases is however not mangled. + +- The name of imported modules, e.g., ``__spam`` in ``import __spam``. + If the module is part of a package (i.e., its name contains a dot), + the name is *not* mangled, e.g., the ``__foo`` in ``import __foo.bar`` + is not mangled. + +- The name of an imported member, e.g., ``__f`` in ``from spam import __f``. + +The transformation rule is defined as follows: + +- The class name, with leading underscores removed and a single leading + underscore inserted, is inserted in front of the identifier, e.g., the + identifier ``__spam`` occurring in a class named ``Foo``, ``_Foo`` or + ``__Foo`` is transformed to ``_Foo__spam``. + +- If the class name consists only of underscores, the transformation is the + identity, e.g., the identifier ``__spam`` occurring in a class named ``_`` + or ``__`` is left as is. .. _atom-literals: @@ -110,8 +138,8 @@ Python supports string and bytes literals and various numeric literals: : | `integer` | `floatnumber` | `imagnumber` Evaluation of a literal yields an object of the given type (string, bytes, -integer, floating point number, complex number) with the given value. The value -may be approximated in the case of floating point and imaginary (complex) +integer, floating-point number, complex number) with the given value. The value +may be approximated in the case of floating-point and imaginary (complex) literals. See section :ref:`literals` for details. .. index:: @@ -218,10 +246,12 @@ A comprehension in an :keyword:`!async def` function may consist of either a :keyword:`!for` or :keyword:`!async for` clause following the leading expression, may contain additional :keyword:`!for` or :keyword:`!async for` clauses, and may also use :keyword:`await` expressions. -If a comprehension contains either :keyword:`!async for` clauses or -:keyword:`!await` expressions or other asynchronous comprehensions it is called -an :dfn:`asynchronous comprehension`. An asynchronous comprehension may -suspend the execution of the coroutine function in which it appears. + +If a comprehension contains :keyword:`!async for` clauses, or if it contains +:keyword:`!await` expressions or other asynchronous comprehensions anywhere except +the iterable expression in the leftmost :keyword:`!for` clause, it is called an +:dfn:`asynchronous comprehension`. An asynchronous comprehension may suspend the +execution of the coroutine function in which it appears. See also :pep:`530`. .. versionadded:: 3.6 @@ -253,7 +283,7 @@ A list display is a possibly empty series of expressions enclosed in square brackets: .. productionlist:: python-grammar - list_display: "[" [`starred_list` | `comprehension`] "]" + list_display: "[" [`flexible_expression_list` | `comprehension`] "]" A list display yields a new list object, the contents being specified by either a list of expressions or a comprehension. When a comma-separated list of @@ -278,7 +308,7 @@ A set display is denoted by curly braces and distinguishable from dictionary displays by the lack of colons separating keys and values: .. productionlist:: python-grammar - set_display: "{" (`starred_list` | `comprehension`) "}" + set_display: "{" (`flexible_expression_list` | `comprehension`) "}" A set display yields a new mutable set object, the contents being specified by either a sequence of expressions or a comprehension. When a comma-separated @@ -423,7 +453,7 @@ Yield expressions .. productionlist:: python-grammar yield_atom: "(" `yield_expression` ")" yield_from: "yield" "from" `expression` - yield_expression: "yield" `expression_list` | `yield_from` + yield_expression: "yield" `yield_list` | `yield_from` The yield expression is used when defining a :term:`generator` function or an :term:`asynchronous generator` function and @@ -454,9 +484,9 @@ When a generator function is called, it returns an iterator known as a generator. That generator then controls the execution of the generator function. The execution starts when one of the generator's methods is called. At that time, the execution proceeds to the first yield expression, where it is -suspended again, returning the value of :token:`~python-grammar:expression_list` +suspended again, returning the value of :token:`~python-grammar:yield_list` to the generator's caller, -or ``None`` if :token:`~python-grammar:expression_list` is omitted. +or ``None`` if :token:`~python-grammar:yield_list` is omitted. By suspended, we mean that all local state is retained, including the current bindings of local variables, the instruction pointer, the internal evaluation stack, and the state of any exception handling. @@ -545,7 +575,7 @@ is already executing raises a :exc:`ValueError` exception. :meth:`~generator.__next__` method, the current yield expression always evaluates to :const:`None`. The execution then continues to the next yield expression, where the generator is suspended again, and the value of the - :token:`~python-grammar:expression_list` is returned to :meth:`__next__`'s + :token:`~python-grammar:yield_list` is returned to :meth:`__next__`'s caller. If the generator exits without yielding another value, a :exc:`StopIteration` exception is raised. @@ -664,7 +694,7 @@ how a generator object would be used in a :keyword:`for` statement. Calling one of the asynchronous generator's methods returns an :term:`awaitable` object, and the execution starts when this object is awaited on. At that time, the execution proceeds to the first yield expression, where it is suspended -again, returning the value of :token:`~python-grammar:expression_list` to the +again, returning the value of :token:`~python-grammar:yield_list` to the awaiting coroutine. As with a generator, suspension means that all local state is retained, including the current bindings of local variables, the instruction pointer, the internal evaluation stack, and the state of any exception handling. @@ -728,7 +758,7 @@ which are used to control the execution of a generator function. asynchronous generator function is resumed with an :meth:`~agen.__anext__` method, the current yield expression always evaluates to :const:`None` in the returned awaitable, which when run will continue to the next yield - expression. The value of the :token:`~python-grammar:expression_list` of the + expression. The value of the :token:`~python-grammar:yield_list` of the yield expression is the value of the :exc:`StopIteration` exception raised by the completing coroutine. If the asynchronous generator exits without yielding another value, the awaitable instead raises a @@ -741,7 +771,7 @@ which are used to control the execution of a generator function. .. coroutinemethod:: agen.asend(value) Returns an awaitable which when run resumes the execution of the - asynchronous generator. As with the :meth:`~generator.send()` method for a + asynchronous generator. As with the :meth:`~generator.send` method for a generator, this "sends" a value into the asynchronous generator function, and the *value* argument becomes the result of the current yield expression. The awaitable returned by the :meth:`asend` method will return the next @@ -861,7 +891,7 @@ will generally select an element from the container. The subscription of a :ref:`GenericAlias ` object. .. productionlist:: python-grammar - subscription: `primary` "[" `expression_list` "]" + subscription: `primary` "[" `flexible_expression_list` "]" When an object is subscripted, the interpreter will evaluate the primary and the expression list. @@ -873,9 +903,13 @@ primary is subscripted, the evaluated result of the expression list will be passed to one of these methods. For more details on when ``__class_getitem__`` is called instead of ``__getitem__``, see :ref:`classgetitem-versus-getitem`. -If the expression list contains at least one comma, it will evaluate to a -:class:`tuple` containing the items of the expression list. Otherwise, the -expression list will evaluate to the value of the list's sole member. +If the expression list contains at least one comma, or if any of the expressions +are starred, the expression list will evaluate to a :class:`tuple` containing +the items of the expression list. Otherwise, the expression list will evaluate +to the value of the list's sole member. + +.. versionchanged:: 3.11 + Expressions in an expression list may be starred. See :pep:`646`. For built-in objects, there are two types of objects that support subscription via :meth:`~object.__getitem__`: @@ -1121,7 +1155,8 @@ a user-defined function: first thing the code block will do is bind the formal parameters to the arguments; this is described in section :ref:`function`. When the code block executes a :keyword:`return` statement, this specifies the return value of the - function call. + function call. If execution reaches the end of the code block without + executing a :keyword:`return` statement, the return value is ``None``. a built-in function or method: .. index:: @@ -1211,7 +1246,8 @@ Raising ``0.0`` to a negative power results in a :exc:`ZeroDivisionError`. Raising a negative number to a fractional power results in a :class:`complex` number. (In earlier versions it raised a :exc:`ValueError`.) -This operation can be customized using the special :meth:`~object.__pow__` method. +This operation can be customized using the special :meth:`~object.__pow__` and +:meth:`~object.__rpow__` methods. .. _unary: @@ -1286,12 +1322,16 @@ operators and one for additive operators: The ``*`` (multiplication) operator yields the product of its arguments. The arguments must either both be numbers, or one argument must be an integer and the other must be a sequence. In the former case, the numbers are converted to a -common type and then multiplied together. In the latter case, sequence +common real type and then multiplied together. In the latter case, sequence repetition is performed; a negative repetition factor yields an empty sequence. This operation can be customized using the special :meth:`~object.__mul__` and :meth:`~object.__rmul__` methods. +.. versionchanged:: 3.14 + If only one operand is a complex number, the other operand is converted + to a floating-point number. + .. index:: single: matrix multiplication pair: operator; @ (at) @@ -1299,6 +1339,9 @@ This operation can be customized using the special :meth:`~object.__mul__` and The ``@`` (at) operator is intended to be used for matrix multiplication. No builtin Python types implement this operator. +This operation can be customized using the special :meth:`~object.__matmul__` and +:meth:`~object.__rmatmul__` methods. + .. versionadded:: 3.5 .. index:: @@ -1314,8 +1357,10 @@ integer; the result is that of mathematical division with the 'floor' function applied to the result. Division by zero raises the :exc:`ZeroDivisionError` exception. -This operation can be customized using the special :meth:`~object.__truediv__` and -:meth:`~object.__floordiv__` methods. +The division operation can be customized using the special :meth:`~object.__truediv__` +and :meth:`~object.__rtruediv__` methods. +The floor division operation can be customized using the special +:meth:`~object.__floordiv__` and :meth:`~object.__rfloordiv__` methods. .. index:: single: modulo @@ -1324,7 +1369,7 @@ This operation can be customized using the special :meth:`~object.__truediv__` a The ``%`` (modulo) operator yields the remainder from the division of the first argument by the second. The numeric arguments are first converted to a common type. A zero right argument raises the :exc:`ZeroDivisionError` exception. The -arguments may be floating point numbers, e.g., ``3.14%0.7`` equals ``0.34`` +arguments may be floating-point numbers, e.g., ``3.14%0.7`` equals ``0.34`` (since ``3.14`` equals ``4*0.7 + 0.34``.) The modulo operator always yields a result with the same sign as its second operand (or zero); the absolute value of the result is strictly smaller than the absolute value of the second operand @@ -1340,11 +1385,12 @@ also overloaded by string objects to perform old-style string formatting (also known as interpolation). The syntax for string formatting is described in the Python Library Reference, section :ref:`old-string-formatting`. -The *modulo* operation can be customized using the special :meth:`~object.__mod__` method. +The *modulo* operation can be customized using the special :meth:`~object.__mod__` +and :meth:`~object.__rmod__` methods. The floor division operator, the modulo operator, and the :func:`divmod` -function are not defined for complex numbers. Instead, convert to a floating -point number using the :func:`abs` function if appropriate. +function are not defined for complex numbers. Instead, convert to a +floating-point number using the :func:`abs` function if appropriate. .. index:: single: addition @@ -1353,21 +1399,30 @@ point number using the :func:`abs` function if appropriate. The ``+`` (addition) operator yields the sum of its arguments. The arguments must either both be numbers or both be sequences of the same type. In the -former case, the numbers are converted to a common type and then added together. +former case, the numbers are converted to a common real type and then added together. In the latter case, the sequences are concatenated. This operation can be customized using the special :meth:`~object.__add__` and :meth:`~object.__radd__` methods. +.. versionchanged:: 3.14 + If only one operand is a complex number, the other operand is converted + to a floating-point number. + .. index:: single: subtraction single: operator; - (minus) single: - (minus); binary operator The ``-`` (subtraction) operator yields the difference of its arguments. The -numeric arguments are first converted to a common type. +numeric arguments are first converted to a common real type. + +This operation can be customized using the special :meth:`~object.__sub__` and +:meth:`~object.__rsub__` methods. -This operation can be customized using the special :meth:`~object.__sub__` method. +.. versionchanged:: 3.14 + If only one operand is a complex number, the other operand is converted + to a floating-point number. .. _shifting: @@ -1388,8 +1443,10 @@ The shifting operations have lower priority than the arithmetic operations: These operators accept integers as arguments. They shift the first argument to the left or right by the number of bits given by the second argument. -This operation can be customized using the special :meth:`~object.__lshift__` and -:meth:`~object.__rshift__` methods. +The left shift operation can be customized using the special :meth:`~object.__lshift__` +and :meth:`~object.__rlshift__` methods. +The right shift operation can be customized using the special :meth:`~object.__rshift__` +and :meth:`~object.__rrshift__` methods. .. index:: pair: exception; ValueError @@ -1762,6 +1819,9 @@ returns a boolean value regardless of the type of its argument single: assignment expression single: walrus operator single: named expression + pair: assignment; expression + +.. _assignment-expressions: Assignment expressions ====================== @@ -1864,10 +1924,12 @@ Expression lists single: , (comma); expression list .. productionlist:: python-grammar + starred_expression: ["*"] `or_expr` + flexible_expression: `assignment_expression` | `starred_expression` + flexible_expression_list: `flexible_expression` ("," `flexible_expression`)* [","] + starred_expression_list: `starred_expression` ("," `starred_expression`)* [","] expression_list: `expression` ("," `expression`)* [","] - starred_list: `starred_item` ("," `starred_item`)* [","] - starred_expression: `expression` | (`starred_item` ",")* [`starred_item`] - starred_item: `assignment_expression` | "*" `or_expr` + yield_list: `expression_list` | `starred_expression` "," [`starred_expression_list`] .. index:: pair: object; tuple @@ -1888,6 +1950,9 @@ the unpacking. .. versionadded:: 3.5 Iterable unpacking in expression lists, originally proposed by :pep:`448`. +.. versionadded:: 3.11 + Any item in an expression list may be starred. See :pep:`646`. + .. index:: pair: trailing; comma A trailing comma is required only to create a one-item tuple, diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index f8c9724114da9e..48fdd0f5d021c7 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -281,7 +281,7 @@ When the named module is not found in :data:`sys.modules`, Python next searches :data:`sys.meta_path`, which contains a list of meta path finder objects. These finders are queried in order to see if they know how to handle the named module. Meta path finders must implement a method called -:meth:`~importlib.abc.MetaPathFinder.find_spec()` which takes three arguments: +:meth:`~importlib.abc.MetaPathFinder.find_spec` which takes three arguments: a name, an import path, and (optionally) a target module. The meta path finder can use any strategy it wants to determine whether it can handle the named module or not. @@ -292,7 +292,7 @@ spec object. If it cannot handle the named module, it returns ``None``. If a spec, then a :exc:`ModuleNotFoundError` is raised. Any other exceptions raised are simply propagated up, aborting the import process. -The :meth:`~importlib.abc.MetaPathFinder.find_spec()` method of meta path +The :meth:`~importlib.abc.MetaPathFinder.find_spec` method of meta path finders is called with two or three arguments. The first is the fully qualified name of the module being imported, for example ``foo.bar.baz``. The second argument is the path entries to use for the module search. For @@ -513,8 +513,10 @@ holding is that if you have ``sys.modules['spam']`` and ``sys.modules['spam.foo']`` (as you would after the above import), the latter must appear as the ``foo`` attribute of the former. -Module spec ------------ +.. _module-specs: + +Module specs +------------ The import machinery uses a variety of information about each module during import, especially before loading. Most of the information is @@ -527,155 +529,44 @@ and the loader that executes it. Most importantly, it allows the import machinery to perform the boilerplate operations of loading, whereas without a module spec the loader had that responsibility. -The module's spec is exposed as the ``__spec__`` attribute on a module object. +The module's spec is exposed as :attr:`module.__spec__`. Setting +:attr:`!__spec__` appropriately applies equally to +:ref:`modules initialized during interpreter startup `. +The one exception is ``__main__``, where :attr:`!__spec__` is +:ref:`set to None in some cases `. + See :class:`~importlib.machinery.ModuleSpec` for details on the contents of the module spec. .. versionadded:: 3.4 -.. _import-mod-attrs: - -Import-related module attributes --------------------------------- - -The import machinery fills in these attributes on each module object -during loading, based on the module's spec, before the loader executes -the module. - -It is **strongly** recommended that you rely on :attr:`__spec__` and -its attributes instead of any of the other individual attributes -listed below. - -.. attribute:: __name__ - - The ``__name__`` attribute must be set to the fully qualified name of - the module. This name is used to uniquely identify the module in - the import system. - -.. attribute:: __loader__ - - The ``__loader__`` attribute must be set to the loader object that - the import machinery used when loading the module. This is mostly - for introspection, but can be used for additional loader-specific - functionality, for example getting data associated with a loader. - - It is **strongly** recommended that you rely on :attr:`__spec__` - instead of this attribute. - - .. versionchanged:: 3.12 - The value of ``__loader__`` is expected to be the same as - ``__spec__.loader``. The use of ``__loader__`` is deprecated and slated - for removal in Python 3.14. - -.. attribute:: __package__ - - The module's ``__package__`` attribute may be set. Its value must - be a string, but it can be the same value as its ``__name__``. When - the module is a package, its ``__package__`` value should be set to - its ``__name__``. When the module is not a package, ``__package__`` - should be set to the empty string for top-level modules, or for - submodules, to the parent package's name. See :pep:`366` for further - details. - - This attribute is used instead of ``__name__`` to calculate explicit - relative imports for main modules, as defined in :pep:`366`. - - It is **strongly** recommended that you rely on :attr:`__spec__` - instead of this attribute. - - .. versionchanged:: 3.6 - The value of ``__package__`` is expected to be the same as - ``__spec__.parent``. - - .. versionchanged:: 3.10 - :exc:`ImportWarning` is raised if import falls back to - ``__package__`` instead of - :attr:`~importlib.machinery.ModuleSpec.parent`. - - .. versionchanged:: 3.12 - Raise :exc:`DeprecationWarning` instead of :exc:`ImportWarning` - when falling back to ``__package__``. - - -.. attribute:: __spec__ - - The ``__spec__`` attribute must be set to the module spec that was - used when importing the module. Setting ``__spec__`` - appropriately applies equally to :ref:`modules initialized during - interpreter startup `. The one exception is ``__main__``, - where ``__spec__`` is :ref:`set to None in some cases `. - - When ``__spec__.parent`` is not set, ``__package__`` is used as - a fallback. - - .. versionadded:: 3.4 - - .. versionchanged:: 3.6 - ``__spec__.parent`` is used as a fallback when ``__package__`` is - not defined. - -.. attribute:: __path__ - - If the module is a package (either regular or namespace), the module - object's ``__path__`` attribute must be set. The value must be - iterable, but may be empty if ``__path__`` has no further significance. - If ``__path__`` is not empty, it must produce strings when iterated - over. More details on the semantics of ``__path__`` are given - :ref:`below `. - - Non-package modules should not have a ``__path__`` attribute. - -.. attribute:: __file__ -.. attribute:: __cached__ - - ``__file__`` is optional (if set, value must be a string). It indicates - the pathname of the file from which the module was loaded (if - loaded from a file), or the pathname of the shared library file - for extension modules loaded dynamically from a shared library. - It might be missing for certain types of modules, such as C - modules that are statically linked into the interpreter, and the - import system may opt to leave it unset if it has no semantic - meaning (e.g. a module loaded from a database). - - If ``__file__`` is set then the ``__cached__`` attribute might also - be set, which is the path to any compiled version of - the code (e.g. byte-compiled file). The file does not need to exist - to set this attribute; the path can simply point to where the - compiled file would exist (see :pep:`3147`). - - Note that ``__cached__`` may be set even if ``__file__`` is not - set. However, that scenario is quite atypical. Ultimately, the - loader is what makes use of the module spec provided by the finder - (from which ``__file__`` and ``__cached__`` are derived). So - if a loader can load from a cached module but otherwise does not load - from a file, that atypical scenario may be appropriate. - - It is **strongly** recommended that you rely on :attr:`__spec__` - instead of ``__cached__``. - .. _package-path-rules: -module.__path__ ---------------- +__path__ attributes on modules +------------------------------ -By definition, if a module has a ``__path__`` attribute, it is a package. +The :attr:`~module.__path__` attribute should be a (possibly empty) +:term:`sequence` of strings enumerating the locations where the package's +submodules will be found. By definition, if a module has a :attr:`!__path__` +attribute, it is a :term:`package`. -A package's ``__path__`` attribute is used during imports of its subpackages. +A package's :attr:`~module.__path__` attribute is used during imports of its +subpackages. Within the import machinery, it functions much the same as :data:`sys.path`, i.e. providing a list of locations to search for modules during import. -However, ``__path__`` is typically much more constrained than -:data:`sys.path`. +However, :attr:`!__path__` is typically much more constrained than +:data:`!sys.path`. -``__path__`` must be an iterable of strings, but it may be empty. The same rules used for :data:`sys.path` also apply to a package's -``__path__``, and :data:`sys.path_hooks` (described below) are -consulted when traversing a package's ``__path__``. +:attr:`!__path__`. :data:`sys.path_hooks` (described below) are +consulted when traversing a package's :attr:`!__path__`. -A package's ``__init__.py`` file may set or alter the package's ``__path__`` +A package's ``__init__.py`` file may set or alter the package's +:attr:`~module.__path__` attribute, and this was typically the way namespace packages were implemented prior to :pep:`420`. With the adoption of :pep:`420`, namespace packages no -longer need to supply ``__init__.py`` files containing only ``__path__`` -manipulation code; the import machinery automatically sets ``__path__`` +longer need to supply ``__init__.py`` files containing only :attr:`!__path__` +manipulation code; the import machinery automatically sets :attr:`!__path__` correctly for the namespace package. Module reprs @@ -871,10 +762,10 @@ module. The current working directory -- denoted by an empty string -- is handled slightly differently from other entries on :data:`sys.path`. First, if the -current working directory is found to not exist, no value is stored in -:data:`sys.path_importer_cache`. Second, the value for the current working -directory is looked up fresh for each module lookup. Third, the path used for -:data:`sys.path_importer_cache` and returned by +current working directory cannot be determined or is found not to exist, no +value is stored in :data:`sys.path_importer_cache`. Second, the value for the +current working directory is looked up fresh for each module lookup. Third, +the path used for :data:`sys.path_importer_cache` and returned by :meth:`importlib.machinery.PathFinder.find_spec` will be the actual current working directory and not the empty string. diff --git a/Doc/reference/introduction.rst b/Doc/reference/introduction.rst index cf186705e6e987..b7b70e6be5a5b7 100644 --- a/Doc/reference/introduction.rst +++ b/Doc/reference/introduction.rst @@ -74,7 +74,7 @@ PyPy and a Just in Time compiler. One of the goals of the project is to encourage experimentation with the language itself by making it easier to modify the interpreter (since it is written in Python). Additional information is - available on `the PyPy project's home page `_. + available on `the PyPy project's home page `_. Each of these implementations varies in some way from the language as documented in this manual, or introduces specific information beyond what's covered in the diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 0adfb0365934e4..f7167032ad7df9 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -96,10 +96,9 @@ which is recognized also by GNU Emacs, and :: which is recognized by Bram Moolenaar's VIM. -If no encoding declaration is found, the default encoding is UTF-8. In -addition, if the first bytes of the file are the UTF-8 byte-order mark -(``b'\xef\xbb\xbf'``), the declared file encoding is UTF-8 (this is supported, -among others, by Microsoft's :program:`notepad`). +If no encoding declaration is found, the default encoding is UTF-8. If the +implicit or explicit encoding of a file is UTF-8, an initial UTF-8 byte-order +mark (b'\xef\xbb\xbf') is ignored rather than being a syntax error. If an encoding is declared, the encoding name must be recognized by Python (see :ref:`standard-encodings`). The @@ -285,11 +284,10 @@ UAX-31, with elaboration and changes as defined below; see also :pep:`3131` for further details. Within the ASCII range (U+0001..U+007F), the valid characters for identifiers -are the same as in Python 2.x: the uppercase and lowercase letters ``A`` through +include the uppercase and lowercase letters ``A`` through ``Z``, the underscore ``_`` and, except for the first character, the digits ``0`` through ``9``. - -Python 3.0 introduces additional characters from outside the ASCII range (see +Python 3.0 introduced additional characters from outside the ASCII range (see :pep:`3131`). For these characters, the classification uses the version of the Unicode Character Database as included in the :mod:`unicodedata` module. @@ -315,7 +313,7 @@ The Unicode category codes mentioned above stand for: * *Nd* - decimal numbers * *Pc* - connector punctuations * *Other_ID_Start* - explicit list of characters in `PropList.txt - `_ to support backwards + `_ to support backwards compatibility * *Other_ID_Continue* - likewise @@ -323,8 +321,8 @@ All identifiers are converted into the normal form NFKC while parsing; compariso of identifiers is based on NFKC. A non-normative HTML file listing all valid identifier characters for Unicode -15.1.0 can be found at -https://www.unicode.org/Public/15.1.0/ucd/DerivedCoreProperties.txt +16.0.0 can be found at +https://www.unicode.org/Public/16.0.0/ucd/DerivedCoreProperties.txt .. _keywords: @@ -504,17 +502,15 @@ must be expressed with escapes. single: r"; raw string literal Both string and bytes literals may optionally be prefixed with a letter ``'r'`` -or ``'R'``; such strings are called :dfn:`raw strings` and treat backslashes as -literal characters. As a result, in string literals, ``'\U'`` and ``'\u'`` -escapes in raw strings are not treated specially. Given that Python 2.x's raw -unicode literals behave differently than Python 3.x's the ``'ur'`` syntax -is not supported. +or ``'R'``; such constructs are called :dfn:`raw string literals` +and :dfn:`raw bytes literals` respectively and treat backslashes as +literal characters. As a result, in raw string literals, ``'\U'`` and ``'\u'`` +escapes are not treated specially. .. versionadded:: 3.3 The ``'rb'`` prefix of raw bytes literals has been added as a synonym of ``'br'``. -.. versionadded:: 3.3 Support for the unicode legacy literal (``u'value'``) was reintroduced to simplify the maintenance of dual Python 2.x and 3.x codebases. See :pep:`414` for more information. @@ -734,7 +730,7 @@ for the contents of the string is: : ("," `conditional_expression` | "," "*" `or_expr`)* [","] : | `yield_expression` conversion: "s" | "r" | "a" - format_spec: (`literal_char` | NULL | `replacement_field`)* + format_spec: (`literal_char` | `replacement_field`)* literal_char: The parts of the string outside curly braces are treated literally, @@ -881,10 +877,10 @@ Numeric literals ---------------- .. index:: number, numeric literal, integer literal - floating point literal, hexadecimal literal + floating-point literal, hexadecimal literal octal literal, binary literal, decimal literal, imaginary literal, complex literal -There are three types of numeric literals: integers, floating point numbers, and +There are three types of numeric literals: integers, floating-point numbers, and imaginary numbers. There are no complex literals (complex numbers can be formed by adding a real number and an imaginary number). @@ -945,10 +941,10 @@ Some examples of integer literals:: single: _ (underscore); in numeric literal .. _floating: -Floating point literals +Floating-point literals ----------------------- -Floating point literals are described by the following lexical definitions: +Floating-point literals are described by the following lexical definitions: .. productionlist:: python-grammar floatnumber: `pointfloat` | `exponentfloat` @@ -960,10 +956,10 @@ Floating point literals are described by the following lexical definitions: Note that the integer and exponent parts are always interpreted using radix 10. For example, ``077e010`` is legal, and denotes the same number as ``77e10``. The -allowed range of floating point literals is implementation-dependent. As in +allowed range of floating-point literals is implementation-dependent. As in integer literals, underscores are supported for digit grouping. -Some examples of floating point literals:: +Some examples of floating-point literals:: 3.14 10. .001 1e100 3.14e-10 0e0 3.14_15_93 @@ -984,9 +980,9 @@ Imaginary literals are described by the following lexical definitions: imagnumber: (`floatnumber` | `digitpart`) ("j" | "J") An imaginary literal yields a complex number with a real part of 0.0. Complex -numbers are represented as a pair of floating point numbers and have the same +numbers are represented as a pair of floating-point numbers and have the same restrictions on their range. To create a complex number with a nonzero real -part, add a floating point number to it, e.g., ``(3+4j)``. Some examples of +part, add a floating-point number to it, e.g., ``(3+4j)``. Some examples of imaginary literals:: 3.14j 10.j 10j .001j 1e100j 3.14e-10j 3.14_15_93j @@ -1021,9 +1017,9 @@ The following tokens serve as delimiters in the grammar: .. code-block:: none ( ) [ ] { } - , : . ; @ = -> - += -= *= /= //= %= @= - &= |= ^= >>= <<= **= + , : ! . ; @ = + -> += -= *= /= //= %= + @= &= |= ^= >>= <<= **= The period can also occur in floating-point and imaginary literals. A sequence of three periods has a special meaning as an ellipsis literal. The second half @@ -1047,4 +1043,4 @@ occurrence outside string literals and comments is an unconditional error: .. rubric:: Footnotes -.. [#] https://www.unicode.org/Public/15.1.0/ucd/NameAliases.txt +.. [#] https://www.unicode.org/Public/16.0.0/ucd/NameAliases.txt diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 04132c78ce77a6..2a72af4e9a3299 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -293,7 +293,7 @@ statements, cannot be an unpacking) and the expression list, performs the binary operation specific to the type of assignment on the two operands, and assigns the result to the original target. The target is only evaluated once. -An augmented assignment expression like ``x += 1`` can be rewritten as ``x = x + +An augmented assignment statement like ``x += 1`` can be rewritten as ``x = x + 1`` to achieve a similar, but not exactly equal effect. In the augmented version, ``x`` is only evaluated once. Also, when possible, the actual operation is performed *in-place*, meaning that rather than creating a new object and @@ -333,23 +333,24 @@ statement, of a variable or attribute annotation and an optional assignment stat The difference from normal :ref:`assignment` is that only a single target is allowed. -For simple names as assignment targets, if in class or module scope, -the annotations are evaluated and stored in a special class or module -attribute :attr:`__annotations__` -that is a dictionary mapping from variable names (mangled if private) to -evaluated annotations. This attribute is writable and is automatically -created at the start of class or module body execution, if annotations -are found statically. +The assignment target is considered "simple" if it consists of a single +name that is not enclosed in parentheses. +For simple assignment targets, if in class or module scope, +the annotations are gathered in a lazily evaluated +:ref:`annotation scope `. The annotations can be +evaluated using the :attr:`~object.__annotations__` attribute of a +class or module, or using the facilities in the :mod:`annotationlib` +module. -For expressions as assignment targets, the annotations are evaluated if -in class or module scope, but not stored. +If the assignment target is not simple (an attribute, subscript node, or +parenthesized name), the annotation is never evaluated. If a name is annotated in a function scope, then this name is local for that scope. Annotations are never evaluated and stored in function scopes. If the right hand side is present, an annotated -assignment performs the actual assignment before evaluating annotations -(where applicable). If the right hand side is not present for an expression +assignment performs the actual assignment as if there was no annotation +present. If the right hand side is not present for an expression target, then the interpreter evaluates the target except for the last :meth:`~object.__setitem__` or :meth:`~object.__setattr__` call. @@ -370,6 +371,10 @@ target, then the interpreter evaluates the target except for the last regular assignments. Previously, some expressions (like un-parenthesized tuple expressions) caused a syntax error. +.. versionchanged:: 3.14 + Annotations are now lazily evaluated in a separate :ref:`annotation scope `. + If the assignment target is not simple, annotations are never evaluated. + .. _assert: @@ -403,9 +408,9 @@ The extended form, ``assert expression1, expression2``, is equivalent to :: These equivalences assume that :const:`__debug__` and :exc:`AssertionError` refer to the built-in variables with those names. In the current implementation, the -built-in variable :const:`__debug__` is ``True`` under normal circumstances, +built-in variable ``__debug__`` is ``True`` under normal circumstances, ``False`` when optimization is requested (command line option :option:`-O`). The current -code generator emits no code for an assert statement when optimization is +code generator emits no code for an :keyword:`assert` statement when optimization is requested at compile time. Note that it is unnecessary to include the source code for the expression that failed in the error message; it will be displayed as part of the stack trace. @@ -528,8 +533,8 @@ The :keyword:`!yield` statement yield_stmt: `yield_expression` A :keyword:`yield` statement is semantically equivalent to a :ref:`yield -expression `. The yield statement can be used to omit the parentheses -that would otherwise be required in the equivalent yield expression +expression `. The ``yield`` statement can be used to omit the +parentheses that would otherwise be required in the equivalent yield expression statement. For example, the yield statements :: yield @@ -541,7 +546,7 @@ are equivalent to the yield expression statements :: (yield from ) Yield expressions and statements are only used when defining a :term:`generator` -function, and are only used in the body of the generator function. Using yield +function, and are only used in the body of the generator function. Using :keyword:`yield` in a function definition is sufficient to cause that definition to create a generator function instead of a normal function. @@ -664,8 +669,7 @@ and information about handling exceptions is in section :ref:`try`. .. versionchanged:: 3.3 :const:`None` is now permitted as ``Y`` in ``raise X from Y``. -.. versionadded:: 3.3 - The :attr:`~BaseException.__suppress_context__` attribute to suppress + Added the :attr:`~BaseException.__suppress_context__` attribute to suppress automatic display of the exception context. .. versionchanged:: 3.11 @@ -962,25 +966,14 @@ The :keyword:`!global` statement .. productionlist:: python-grammar global_stmt: "global" `identifier` ("," `identifier`)* -The :keyword:`global` statement is a declaration which holds for the entire -current code block. It means that the listed identifiers are to be interpreted -as globals. It would be impossible to assign to a global variable without +The :keyword:`global` statement causes the listed identifiers to be interpreted +as globals. It would be impossible to assign to a global variable without :keyword:`!global`, although free variables may refer to globals without being declared global. -Names listed in a :keyword:`global` statement must not be used in the same code -block textually preceding that :keyword:`!global` statement. - -Names listed in a :keyword:`global` statement must not be defined as formal -parameters, or as targets in :keyword:`with` statements or :keyword:`except` clauses, or in a :keyword:`for` target list, :keyword:`class` -definition, function definition, :keyword:`import` statement, or variable -annotation. - -.. impl-detail:: - - The current implementation does not enforce some of these restrictions, but - programs should not abuse this freedom, as future implementations may enforce - them or silently change the meaning of the program. +The :keyword:`global` statement applies to the entire scope of a function or +class body. A :exc:`SyntaxError` is raised if a variable is used or +assigned to prior to its global declaration in the scope. .. index:: pair: built-in function; exec @@ -1007,25 +1000,29 @@ The :keyword:`!nonlocal` statement .. productionlist:: python-grammar nonlocal_stmt: "nonlocal" `identifier` ("," `identifier`)* -The :keyword:`nonlocal` statement causes the listed identifiers to refer to -previously bound variables in the nearest enclosing scope excluding globals. -This is important because the default behavior for binding is to search the -local namespace first. The statement allows encapsulated code to rebind -variables outside of the local scope besides the global (module) scope. - -Names listed in a :keyword:`nonlocal` statement, unlike those listed in a -:keyword:`global` statement, must refer to pre-existing bindings in an -enclosing scope (the scope in which a new binding should be created cannot -be determined unambiguously). +When the definition of a function or class is nested (enclosed) within +the definitions of other functions, its nonlocal scopes are the local +scopes of the enclosing functions. The :keyword:`nonlocal` statement +causes the listed identifiers to refer to names previously bound in +nonlocal scopes. It allows encapsulated code to rebind such nonlocal +identifiers. If a name is bound in more than one nonlocal scope, the +nearest binding is used. If a name is not bound in any nonlocal scope, +or if there is no nonlocal scope, a :exc:`SyntaxError` is raised. -Names listed in a :keyword:`nonlocal` statement must not collide with -pre-existing bindings in the local scope. +The :keyword:`nonlocal` statement applies to the entire scope of a function or +class body. A :exc:`SyntaxError` is raised if a variable is used or +assigned to prior to its nonlocal declaration in the scope. .. seealso:: :pep:`3104` - Access to Names in Outer Scopes The specification for the :keyword:`nonlocal` statement. +**Programmer's note:** :keyword:`nonlocal` is a directive to the parser +and applies only to code parsed along with it. See the note for the +:keyword:`global` statement. + + .. _type: The :keyword:`!type` statement diff --git a/Doc/requirements-oldest-sphinx.txt b/Doc/requirements-oldest-sphinx.txt deleted file mode 100644 index 597341d99ffeaa..00000000000000 --- a/Doc/requirements-oldest-sphinx.txt +++ /dev/null @@ -1,35 +0,0 @@ -# Requirements to build the Python documentation, for the oldest supported -# Sphinx version. -# -# We pin Sphinx and all of its dependencies to ensure a consistent environment. - -blurb -python-docs-theme>=2022.1 - -# Generated from: -# pip install "Sphinx~=4.2.0" -# pip freeze -# -# Sphinx 4.2 comes from ``needs_sphinx = '4.2'`` in ``Doc/conf.py``. - -alabaster==0.7.13 -Babel==2.13.0 -certifi==2023.7.22 -charset-normalizer==3.3.0 -docutils==0.17.1 -idna==3.4 -imagesize==1.4.1 -Jinja2==3.1.2 -MarkupSafe==2.1.3 -packaging==23.2 -Pygments==2.16.1 -requests==2.31.0 -snowballstemmer==2.2.0 -Sphinx==4.2.0 -sphinxcontrib-applehelp==1.0.4 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.1 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 -urllib3==2.0.7 diff --git a/Doc/requirements.txt b/Doc/requirements.txt index 118e6c322b4be2..32ff8f74d05bb6 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -3,16 +3,16 @@ # Note that when updating this file, you will likely also have to update # the Doc/constraints.txt file. -# Sphinx version is pinned so that new versions that introduce new warnings +# The Sphinx version is pinned so that new versions that introduce new warnings # won't suddenly cause build failures. Updating the version is fine as long # as no warnings are raised by doing so. -sphinx~=7.2.0 +# Keep this version in sync with ``Doc/conf.py``. +sphinx~=8.1.0 blurb -sphinx-autobuild -sphinxext-opengraph==0.7.5 -sphinx-notfound-page==1.0.0 +sphinxext-opengraph~=0.9.0 +sphinx-notfound-page~=1.0.0 # The theme used by the documentation is stored separately, so we need # to install that as well. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 5fbc24c6ee65a5..ad24fe82f754fc 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -12,12 +12,8 @@ Doc/c-api/stable.rst Doc/c-api/type.rst Doc/c-api/typeobj.rst Doc/extending/extending.rst -Doc/glossary.rst -Doc/howto/descriptor.rst -Doc/howto/enum.rst Doc/library/ast.rst Doc/library/asyncio-extending.rst -Doc/library/asyncio-policy.rst Doc/library/asyncio-subprocess.rst Doc/library/collections.rst Doc/library/dbm.rst @@ -26,9 +22,7 @@ Doc/library/email.charset.rst Doc/library/email.compat32-message.rst Doc/library/email.errors.rst Doc/library/email.parser.rst -Doc/library/email.policy.rst Doc/library/exceptions.rst -Doc/library/faulthandler.rst Doc/library/functools.rst Doc/library/http.cookiejar.rst Doc/library/http.server.rst @@ -44,7 +38,6 @@ Doc/library/pickletools.rst Doc/library/platform.rst Doc/library/plistlib.rst Doc/library/profile.rst -Doc/library/pydoc.rst Doc/library/pyexpat.rst Doc/library/readline.rst Doc/library/resource.rst @@ -76,13 +69,11 @@ Doc/library/xmlrpc.server.rst Doc/library/zlib.rst Doc/reference/compound_stmts.rst Doc/reference/datamodel.rst -Doc/tutorial/datastructures.rst Doc/using/windows.rst Doc/whatsnew/2.4.rst Doc/whatsnew/2.5.rst Doc/whatsnew/2.6.rst Doc/whatsnew/2.7.rst -Doc/whatsnew/3.0.rst Doc/whatsnew/3.3.rst Doc/whatsnew/3.4.rst Doc/whatsnew/3.5.rst diff --git a/Doc/tools/check-warnings.py b/Doc/tools/check-warnings.py index 809a8d63087e12..c686eecf8d9271 100644 --- a/Doc/tools/check-warnings.py +++ b/Doc/tools/check-warnings.py @@ -2,6 +2,7 @@ """ Check the output of running Sphinx in nit-picky mode (missing references). """ + from __future__ import annotations import argparse @@ -13,6 +14,9 @@ from pathlib import Path from typing import TextIO +# Fail if NEWS nit found before this line number +NEWS_NIT_THRESHOLD = 1700 + # Exclude these whether they're dirty or clean, # because they trigger a rebuild of dirty files. EXCLUDE_FILES = { @@ -203,7 +207,9 @@ def annotate_diff( def fail_if_regression( - warnings: list[str], files_with_expected_nits: set[str], files_with_nits: set[str] + warnings: list[str], + files_with_expected_nits: set[str], + files_with_nits: set[str], ) -> int: """ Ensure some files always pass Sphinx nit-picky mode (no missing references). @@ -245,6 +251,26 @@ def fail_if_improved( return 0 +def fail_if_new_news_nit(warnings: list[str], threshold: int) -> int: + """ + Ensure no warnings are found in the NEWS file before a given line number. + """ + news_nits = (warning for warning in warnings if "/build/NEWS:" in warning) + + # Nits found before the threshold line + new_news_nits = [ + nit for nit in news_nits if int(nit.split(":")[1]) <= threshold + ] + + if new_news_nits: + print("\nError: new NEWS nits:\n") + for warning in new_news_nits: + print(warning) + return -1 + + return 0 + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser() parser.add_argument( @@ -264,6 +290,14 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Fail if new files with no nits are found", ) + parser.add_argument( + "--fail-if-new-news-nit", + metavar="threshold", + type=int, + nargs="?", + const=NEWS_NIT_THRESHOLD, + help="Fail if new NEWS nit found before threshold line number", + ) args = parser.parse_args(argv) if args.annotate_diff is not None and len(args.annotate_diff) > 2: @@ -274,7 +308,8 @@ def main(argv: list[str] | None = None) -> int: exit_code = 0 wrong_directory_msg = "Must run this script from the repo root" - assert Path("Doc").exists() and Path("Doc").is_dir(), wrong_directory_msg + if not Path("Doc").exists() or not Path("Doc").is_dir(): + raise RuntimeError(wrong_directory_msg) with Path("Doc/sphinx-warnings.txt").open(encoding="UTF-8") as f: warnings = f.read().splitlines() @@ -302,7 +337,12 @@ def main(argv: list[str] | None = None) -> int: ) if args.fail_if_improved: - exit_code += fail_if_improved(files_with_expected_nits, files_with_nits) + exit_code += fail_if_improved( + files_with_expected_nits, files_with_nits + ) + + if args.fail_if_new_news_nit: + exit_code += fail_if_new_news_nit(warnings, args.fail_if_new_news_nit) return exit_code diff --git a/Doc/tools/extensions/audit_events.py b/Doc/tools/extensions/audit_events.py new file mode 100644 index 00000000000000..23d82c0f4414bf --- /dev/null +++ b/Doc/tools/extensions/audit_events.py @@ -0,0 +1,264 @@ +"""Support for documenting audit events.""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING + +from docutils import nodes +from sphinx.errors import NoUri +from sphinx.locale import _ as sphinx_gettext +from sphinx.transforms.post_transforms import SphinxPostTransform +from sphinx.util import logging +from sphinx.util.docutils import SphinxDirective + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sphinx.application import Sphinx + from sphinx.builders import Builder + from sphinx.environment import BuildEnvironment + +logger = logging.getLogger(__name__) + +# This list of sets are allowable synonyms for event argument names. +# If two names are in the same set, they are treated as equal for the +# purposes of warning. This won't help if the number of arguments is +# different! +_SYNONYMS = [ + frozenset({"file", "path", "fd"}), +] + + +class AuditEvents: + def __init__(self) -> None: + self.events: dict[str, list[str]] = {} + self.sources: dict[str, list[tuple[str, str]]] = {} + + def __iter__(self) -> Iterator[tuple[str, list[str], tuple[str, str]]]: + for name, args in self.events.items(): + for source in self.sources[name]: + yield name, args, source + + def add_event( + self, name, args: list[str], source: tuple[str, str] + ) -> None: + if name in self.events: + self._check_args_match(name, args) + else: + self.events[name] = args + self.sources.setdefault(name, []).append(source) + + def _check_args_match(self, name: str, args: list[str]) -> None: + current_args = self.events[name] + msg = ( + f"Mismatched arguments for audit-event {name}: " + f"{current_args!r} != {args!r}" + ) + if current_args == args: + return + if len(current_args) != len(args): + logger.warning(msg) + return + for a1, a2 in zip(current_args, args, strict=False): + if a1 == a2: + continue + if any(a1 in s and a2 in s for s in _SYNONYMS): + continue + logger.warning(msg) + return + + def id_for(self, name) -> str: + source_count = len(self.sources.get(name, ())) + name_clean = re.sub(r"\W", "_", name) + return f"audit_event_{name_clean}_{source_count}" + + def rows(self) -> Iterator[tuple[str, list[str], list[tuple[str, str]]]]: + for name in sorted(self.events.keys()): + yield name, self.events[name], self.sources[name] + + +def initialise_audit_events(app: Sphinx) -> None: + """Initialise the audit_events attribute on the environment.""" + if not hasattr(app.env, "audit_events"): + app.env.audit_events = AuditEvents() + + +def audit_events_purge( + app: Sphinx, env: BuildEnvironment, docname: str +) -> None: + """This is to remove traces of removed documents from env.audit_events.""" + fresh_audit_events = AuditEvents() + for name, args, (doc, target) in env.audit_events: + if doc != docname: + fresh_audit_events.add_event(name, args, (doc, target)) + + +def audit_events_merge( + app: Sphinx, + env: BuildEnvironment, + docnames: list[str], + other: BuildEnvironment, +) -> None: + """In Sphinx parallel builds, this merges audit_events from subprocesses.""" + for name, args, source in other.audit_events: + env.audit_events.add_event(name, args, source) + + +class AuditEvent(SphinxDirective): + has_content = True + required_arguments = 1 + optional_arguments = 2 + final_argument_whitespace = True + + _label = [ + sphinx_gettext( + "Raises an :ref:`auditing event ` " + "{name} with no arguments." + ), + sphinx_gettext( + "Raises an :ref:`auditing event ` " + "{name} with argument {args}." + ), + sphinx_gettext( + "Raises an :ref:`auditing event ` " + "{name} with arguments {args}." + ), + ] + + def run(self) -> list[nodes.paragraph]: + name = self.arguments[0] + if len(self.arguments) >= 2 and self.arguments[1]: + args = [ + arg + for argument in self.arguments[1].strip("'\"").split(",") + if (arg := argument.strip()) + ] + else: + args = [] + ids = [] + try: + target = self.arguments[2].strip("\"'") + except (IndexError, TypeError): + target = None + if not target: + target = self.env.audit_events.id_for(name) + ids.append(target) + self.env.audit_events.add_event(name, args, (self.env.docname, target)) + + node = nodes.paragraph("", classes=["audit-hook"], ids=ids) + self.set_source_info(node) + if self.content: + node.rawsource = '\n'.join(self.content) # for gettext + self.state.nested_parse(self.content, self.content_offset, node) + else: + num_args = min(2, len(args)) + text = self._label[num_args].format( + name=f"``{name}``", + args=", ".join(f"``{a}``" for a in args), + ) + node.rawsource = text # for gettext + parsed, messages = self.state.inline_text(text, self.lineno) + node += parsed + node += messages + return [node] + + +class audit_event_list(nodes.General, nodes.Element): # noqa: N801 + pass + + +class AuditEventListDirective(SphinxDirective): + def run(self) -> list[audit_event_list]: + return [audit_event_list()] + + +class AuditEventListTransform(SphinxPostTransform): + default_priority = 500 + + def run(self) -> None: + if self.document.next_node(audit_event_list) is None: + return + + table = self._make_table(self.app.builder, self.env.docname) + for node in self.document.findall(audit_event_list): + node.replace_self(table) + + def _make_table(self, builder: Builder, docname: str) -> nodes.table: + table = nodes.table(cols=3) + group = nodes.tgroup( + "", + nodes.colspec(colwidth=30), + nodes.colspec(colwidth=55), + nodes.colspec(colwidth=15), + cols=3, + ) + head = nodes.thead() + body = nodes.tbody() + + table += group + group += head + group += body + + head += nodes.row( + "", + nodes.entry("", nodes.paragraph("", "Audit event")), + nodes.entry("", nodes.paragraph("", "Arguments")), + nodes.entry("", nodes.paragraph("", "References")), + ) + + for name, args, sources in builder.env.audit_events.rows(): + body += self._make_row(builder, docname, name, args, sources) + + return table + + @staticmethod + def _make_row( + builder: Builder, + docname: str, + name: str, + args: list[str], + sources: list[tuple[str, str]], + ) -> nodes.row: + row = nodes.row() + name_node = nodes.paragraph("", nodes.Text(name)) + row += nodes.entry("", name_node) + + args_node = nodes.paragraph() + for arg in args: + args_node += nodes.literal(arg, arg) + args_node += nodes.Text(", ") + if len(args_node.children) > 0: + args_node.children.pop() # remove trailing comma + row += nodes.entry("", args_node) + + backlinks_node = nodes.paragraph() + backlinks = enumerate(sorted(set(sources)), start=1) + for i, (doc, label) in backlinks: + if isinstance(label, str): + ref = nodes.reference("", f"[{i}]", internal=True) + try: + target = ( + f"{builder.get_relative_uri(docname, doc)}#{label}" + ) + except NoUri: + continue + else: + ref["refuri"] = target + backlinks_node += ref + row += nodes.entry("", backlinks_node) + return row + + +def setup(app: Sphinx): + app.add_directive("audit-event", AuditEvent) + app.add_directive("audit-event-table", AuditEventListDirective) + app.add_post_transform(AuditEventListTransform) + app.connect("builder-inited", initialise_audit_events) + app.connect("env-purge-doc", audit_events_purge) + app.connect("env-merge-info", audit_events_merge) + return { + "version": "1.0", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/Doc/tools/extensions/availability.py b/Doc/tools/extensions/availability.py new file mode 100644 index 00000000000000..47833fdcb87590 --- /dev/null +++ b/Doc/tools/extensions/availability.py @@ -0,0 +1,125 @@ +"""Support for documenting platform availability""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from docutils import nodes +from sphinx import addnodes +from sphinx.util import logging +from sphinx.util.docutils import SphinxDirective + +if TYPE_CHECKING: + from sphinx.application import Sphinx + from sphinx.util.typing import ExtensionMetadata + +logger = logging.getLogger("availability") + +# known platform, libc, and threading implementations +_PLATFORMS = frozenset({ + "AIX", + "Android", + "BSD", + "DragonFlyBSD", + "Emscripten", + "FreeBSD", + "GNU/kFreeBSD", + "iOS", + "Linux", + "macOS", + "NetBSD", + "OpenBSD", + "POSIX", + "Solaris", + "Unix", + "VxWorks", + "WASI", + "Windows", +}) +_LIBC = frozenset({ + "BSD libc", + "glibc", + "musl", +}) +_THREADING = frozenset({ + # POSIX platforms with pthreads + "pthreads", +}) +KNOWN_PLATFORMS = _PLATFORMS | _LIBC | _THREADING + + +class Availability(SphinxDirective): + has_content = True + required_arguments = 1 + optional_arguments = 0 + final_argument_whitespace = True + + def run(self) -> list[nodes.container]: + title = "Availability" + refnode = addnodes.pending_xref( + title, + nodes.inline(title, title, classes=["xref", "std", "std-ref"]), + refdoc=self.env.docname, + refdomain="std", + refexplicit=True, + reftarget="availability", + reftype="ref", + refwarn=True, + ) + sep = nodes.Text(": ") + parsed, msgs = self.state.inline_text(self.arguments[0], self.lineno) + pnode = nodes.paragraph(title, "", refnode, sep, *parsed, *msgs) + self.set_source_info(pnode) + cnode = nodes.container("", pnode, classes=["availability"]) + self.set_source_info(cnode) + if self.content: + self.state.nested_parse(self.content, self.content_offset, cnode) + self.parse_platforms() + + return [cnode] + + def parse_platforms(self) -> dict[str, str | bool]: + """Parse platform information from arguments + + Arguments is a comma-separated string of platforms. A platform may + be prefixed with "not " to indicate that a feature is not available. + + Example:: + + .. availability:: Windows, Linux >= 4.2, not WASI + + Arguments like "Linux >= 3.17 with glibc >= 2.27" are currently not + parsed into separate tokens. + """ + platforms = {} + for arg in self.arguments[0].rstrip(".").split(","): + arg = arg.strip() + platform, _, version = arg.partition(" >= ") + if platform.startswith("not "): + version = False + platform = platform.removeprefix("not ") + elif not version: + version = True + platforms[platform] = version + + if unknown := set(platforms).difference(KNOWN_PLATFORMS): + logger.warning( + "Unknown platform%s or syntax '%s' in '.. availability:: %s', " + "see %s:KNOWN_PLATFORMS for a set of known platforms.", + "s" if len(platforms) != 1 else "", + " ".join(sorted(unknown)), + self.arguments[0], + __file__, + ) + + return platforms + + +def setup(app: Sphinx) -> ExtensionMetadata: + app.add_directive("availability", Availability) + + return { + "version": "1.0", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index a8b6d8995e3f40..089614a1f6c421 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -1,235 +1,291 @@ -# -*- coding: utf-8 -*- -""" - c_annotations.py - ~~~~~~~~~~~~~~~~ - - Supports annotations for C API elements: +"""Support annotations for C API elements. - * reference count annotations for C API functions. Based on - refcount.py and anno-api.py in the old Python documentation tools. +* Reference count annotations for C API functions. +* Stable ABI annotations +* Limited API annotations - * stable API annotations +Configuration: +* Set ``refcount_file`` to the path to the reference count data file. +* Set ``stable_abi_file`` to the path to stable ABI list. +""" - Usage: - * Set the `refcount_file` config value to the path to the reference - count data file. - * Set the `stable_abi_file` config value to the path to stable ABI list. +from __future__ import annotations - :copyright: Copyright 2007-2014 by Georg Brandl. - :license: Python license. -""" +import csv +import dataclasses +from pathlib import Path +from typing import TYPE_CHECKING -from os import path -import docutils from docutils import nodes -from docutils.parsers.rst import directives -from docutils.parsers.rst import Directive from docutils.statemachine import StringList -from sphinx.locale import _ as sphinx_gettext -import csv - from sphinx import addnodes -from sphinx.domains.c import CObject - - -REST_ROLE_MAP = { - 'function': 'func', - 'var': 'data', - 'type': 'type', - 'macro': 'macro', - 'type': 'type', - 'member': 'member', +from sphinx.locale import _ as sphinx_gettext +from sphinx.util.docutils import SphinxDirective + +if TYPE_CHECKING: + from sphinx.application import Sphinx + from sphinx.util.typing import ExtensionMetadata + +ROLE_TO_OBJECT_TYPE = { + "func": "function", + "macro": "macro", + "member": "member", + "type": "type", + "data": "var", } -# Monkeypatch nodes.Node.findall for forwards compatibility -# This patch can be dropped when the minimum Sphinx version is 4.4.0 -# or the minimum Docutils version is 0.18.1. -if docutils.__version_info__ < (0, 18, 1): - def findall(self, *args, **kwargs): - return iter(self.traverse(*args, **kwargs)) - - nodes.Node.findall = findall - - -class RCEntry: - def __init__(self, name): - self.name = name - self.args = [] - self.result_type = '' - self.result_refs = None - - -class Annotations: - def __init__(self, refcount_filename, stable_abi_file): - self.refcount_data = {} - with open(refcount_filename, 'r') as fp: - for line in fp: - line = line.strip() - if line[:1] in ("", "#"): - # blank lines and comments - continue - parts = line.split(":", 4) - if len(parts) != 5: - raise ValueError("Wrong field count in %r" % line) - function, type, arg, refcount, comment = parts - # Get the entry, creating it if needed: - try: - entry = self.refcount_data[function] - except KeyError: - entry = self.refcount_data[function] = RCEntry(function) - if not refcount or refcount == "null": - refcount = None - else: - refcount = int(refcount) - # Update the entry with the new parameter or the result - # information. - if arg: - entry.args.append((arg, type, refcount)) - else: - entry.result_type = type - entry.result_refs = refcount - - self.stable_abi_data = {} - with open(stable_abi_file, 'r') as fp: - for record in csv.DictReader(fp): - role = record['role'] - name = record['name'] - self.stable_abi_data[name] = record - - def add_annotations(self, app, doctree): - for node in doctree.findall(addnodes.desc_content): - par = node.parent - if par['domain'] != 'c': - continue - if not par[0].has_key('ids') or not par[0]['ids']: - continue - name = par[0]['ids'][0] - if name.startswith("c."): - name = name[2:] - - objtype = par['objtype'] - - # Stable ABI annotation. These have two forms: - # Part of the [Stable ABI](link). - # Part of the [Stable ABI](link) since version X.Y. - # For structs, there's some more info in the message: - # Part of the [Limited API](link) (as an opaque struct). - # Part of the [Stable ABI](link) (including all members). - # Part of the [Limited API](link) (Only some members are part - # of the stable ABI.). - # ... all of which can have "since version X.Y" appended. - record = self.stable_abi_data.get(name) - if record: - if record['role'] != objtype: - raise ValueError( - f"Object type mismatch in limited API annotation " - f"for {name}: {record['role']!r} != {objtype!r}") - stable_added = record['added'] - message = sphinx_gettext('Part of the') - message = message.center(len(message) + 2) - emph_node = nodes.emphasis(message, message, - classes=['stableabi']) - ref_node = addnodes.pending_xref( - 'Stable ABI', refdomain="std", reftarget='stable', - reftype='ref', refexplicit="False") - struct_abi_kind = record['struct_abi_kind'] - if struct_abi_kind in {'opaque', 'members'}: - ref_node += nodes.Text(sphinx_gettext('Limited API')) - else: - ref_node += nodes.Text(sphinx_gettext('Stable ABI')) - emph_node += ref_node - if struct_abi_kind == 'opaque': - emph_node += nodes.Text(' ' + sphinx_gettext('(as an opaque struct)')) - elif struct_abi_kind == 'full-abi': - emph_node += nodes.Text(' ' + sphinx_gettext('(including all members)')) - if record['ifdef_note']: - emph_node += nodes.Text(' ' + record['ifdef_note']) - if stable_added == '3.2': - # Stable ABI was introduced in 3.2. - pass - else: - emph_node += nodes.Text(' ' + sphinx_gettext('since version %s') % stable_added) - emph_node += nodes.Text('.') - if struct_abi_kind == 'members': - emph_node += nodes.Text( - ' ' + sphinx_gettext('(Only some members are part of the stable ABI.)')) - node.insert(0, emph_node) - - # Unstable API annotation. - if name.startswith('PyUnstable'): - warn_node = nodes.admonition( - classes=['unstable-c-api', 'warning']) - message = sphinx_gettext('This is') + ' ' - emph_node = nodes.emphasis(message, message) - ref_node = addnodes.pending_xref( - 'Unstable API', refdomain="std", - reftarget='unstable-c-api', - reftype='ref', refexplicit="False") - ref_node += nodes.Text(sphinx_gettext('Unstable API')) - emph_node += ref_node - emph_node += nodes.Text(sphinx_gettext('. It may change without warning in minor releases.')) - warn_node += emph_node - node.insert(0, warn_node) - - # Return value annotation - if objtype != 'function': - continue - entry = self.refcount_data.get(name) - if not entry: - continue - elif not entry.result_type.endswith("Object*"): - continue - if entry.result_refs is None: - rc = sphinx_gettext('Return value: Always NULL.') - elif entry.result_refs: - rc = sphinx_gettext('Return value: New reference.') - else: - rc = sphinx_gettext('Return value: Borrowed reference.') - node.insert(0, nodes.emphasis(rc, rc, classes=['refcount'])) - - -def init_annotations(app): - annotations = Annotations( - path.join(app.srcdir, app.config.refcount_file), - path.join(app.srcdir, app.config.stable_abi_file), +@dataclasses.dataclass(slots=True) +class RefCountEntry: + # Name of the function. + name: str + # List of (argument name, type, refcount effect) tuples. + # (Currently not used. If it was, a dataclass might work better.) + args: list = dataclasses.field(default_factory=list) + # Return type of the function. + result_type: str = "" + # Reference count effect for the return value. + result_refs: int | None = None + + +@dataclasses.dataclass(frozen=True, slots=True) +class StableABIEntry: + # Role of the object. + # Source: Each [item_kind] in stable_abi.toml is mapped to a C Domain role. + role: str + # Name of the object. + # Source: [.*] in stable_abi.toml. + name: str + # Version when the object was added to the stable ABI. + # (Source: [.*.added] in stable_abi.toml. + added: str + # An explananatory blurb for the ifdef. + # Source: ``feature_macro.*.doc`` in stable_abi.toml. + ifdef_note: str + # Defines how much of the struct is exposed. Only relevant for structs. + # Source: [.*.struct_abi_kind] in stable_abi.toml. + struct_abi_kind: str + + +def read_refcount_data(refcount_filename: Path) -> dict[str, RefCountEntry]: + refcount_data = {} + refcounts = refcount_filename.read_text(encoding="utf8") + for line in refcounts.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + # blank lines and comments + continue + + # Each line is of the form + # function ':' type ':' [param name] ':' [refcount effect] ':' [comment] + parts = line.split(":", 4) + if len(parts) != 5: + raise ValueError(f"Wrong field count in {line!r}") + function, type, arg, refcount, _comment = parts + + # Get the entry, creating it if needed: + try: + entry = refcount_data[function] + except KeyError: + entry = refcount_data[function] = RefCountEntry(function) + if not refcount or refcount == "null": + refcount = None + else: + refcount = int(refcount) + # Update the entry with the new parameter + # or the result information. + if arg: + entry.args.append((arg, type, refcount)) + else: + entry.result_type = type + entry.result_refs = refcount + + return refcount_data + + +def read_stable_abi_data(stable_abi_file: Path) -> dict[str, StableABIEntry]: + stable_abi_data = {} + with open(stable_abi_file, encoding="utf8") as fp: + for record in csv.DictReader(fp): + name = record["name"] + stable_abi_data[name] = StableABIEntry(**record) + + return stable_abi_data + + +def add_annotations(app: Sphinx, doctree: nodes.document) -> None: + state = app.env.domaindata["c_annotations"] + refcount_data = state["refcount_data"] + stable_abi_data = state["stable_abi_data"] + for node in doctree.findall(addnodes.desc_content): + par = node.parent + if par["domain"] != "c": + continue + if not par[0].get("ids", None): + continue + name = par[0]["ids"][0].removeprefix("c.") + objtype = par["objtype"] + + # Stable ABI annotation. + if record := stable_abi_data.get(name): + if ROLE_TO_OBJECT_TYPE[record.role] != objtype: + msg = ( + f"Object type mismatch in limited API annotation for {name}: " + f"{ROLE_TO_OBJECT_TYPE[record.role]!r} != {objtype!r}" + ) + raise ValueError(msg) + annotation = _stable_abi_annotation(record) + node.insert(0, annotation) + + # Unstable API annotation. + if name.startswith("PyUnstable"): + annotation = _unstable_api_annotation() + node.insert(0, annotation) + + # Return value annotation + if objtype != "function": + continue + if name not in refcount_data: + continue + entry = refcount_data[name] + if not entry.result_type.endswith("Object*"): + continue + annotation = _return_value_annotation(entry.result_refs) + node.insert(0, annotation) + + +def _stable_abi_annotation(record: StableABIEntry) -> nodes.emphasis: + """Create the Stable ABI annotation. + + These have two forms: + Part of the `Stable ABI `_. + Part of the `Stable ABI `_ since version X.Y. + For structs, there's some more info in the message: + Part of the `Limited API `_ (as an opaque struct). + Part of the `Stable ABI `_ (including all members). + Part of the `Limited API `_ (Only some members are part + of the stable ABI.). + ... all of which can have "since version X.Y" appended. + """ + stable_added = record.added + message = sphinx_gettext("Part of the") + message = message.center(len(message) + 2) + emph_node = nodes.emphasis(message, message, classes=["stableabi"]) + ref_node = addnodes.pending_xref( + "Stable ABI", + refdomain="std", + reftarget="stable", + reftype="ref", + refexplicit="False", + ) + struct_abi_kind = record.struct_abi_kind + if struct_abi_kind in {"opaque", "members"}: + ref_node += nodes.Text(sphinx_gettext("Limited API")) + else: + ref_node += nodes.Text(sphinx_gettext("Stable ABI")) + emph_node += ref_node + if struct_abi_kind == "opaque": + emph_node += nodes.Text(" " + sphinx_gettext("(as an opaque struct)")) + elif struct_abi_kind == "full-abi": + emph_node += nodes.Text( + " " + sphinx_gettext("(including all members)") + ) + if record.ifdef_note: + emph_node += nodes.Text(f" {record.ifdef_note}") + if stable_added == "3.2": + # Stable ABI was introduced in 3.2. + pass + else: + emph_node += nodes.Text( + " " + sphinx_gettext("since version %s") % stable_added + ) + emph_node += nodes.Text(".") + if struct_abi_kind == "members": + msg = " " + sphinx_gettext( + "(Only some members are part of the stable ABI.)" + ) + emph_node += nodes.Text(msg) + return emph_node + + +def _unstable_api_annotation() -> nodes.admonition: + ref_node = addnodes.pending_xref( + "Unstable API", + nodes.Text(sphinx_gettext("Unstable API")), + refdomain="std", + reftarget="unstable-c-api", + reftype="ref", + refexplicit="False", + ) + emph_node = nodes.emphasis( + "This is ", + sphinx_gettext("This is") + " ", + ref_node, + nodes.Text( + sphinx_gettext( + ". It may change without warning in minor releases." + ) + ), + ) + return nodes.admonition( + "", + emph_node, + classes=["unstable-c-api", "warning"], ) - app.connect('doctree-read', annotations.add_annotations) - - class LimitedAPIList(Directive): - - has_content = False - required_arguments = 0 - optional_arguments = 0 - final_argument_whitespace = True - - def run(self): - content = [] - for record in annotations.stable_abi_data.values(): - role = REST_ROLE_MAP[record['role']] - name = record['name'] - content.append(f'* :c:{role}:`{name}`') - pnode = nodes.paragraph() - self.state.nested_parse(StringList(content), 0, pnode) - return [pnode] - app.add_directive('limited-api-list', LimitedAPIList) +def _return_value_annotation(result_refs: int | None) -> nodes.emphasis: + classes = ["refcount"] + if result_refs is None: + rc = sphinx_gettext("Return value: Always NULL.") + classes.append("return_null") + elif result_refs: + rc = sphinx_gettext("Return value: New reference.") + classes.append("return_new_ref") + else: + rc = sphinx_gettext("Return value: Borrowed reference.") + classes.append("return_borrowed_ref") + return nodes.emphasis(rc, rc, classes=classes) + + +class LimitedAPIList(SphinxDirective): + has_content = False + required_arguments = 0 + optional_arguments = 0 + final_argument_whitespace = True + + def run(self) -> list[nodes.Node]: + state = self.env.domaindata["c_annotations"] + content = [ + f"* :c:{record.role}:`{record.name}`" + for record in state["stable_abi_data"].values() + ] + node = nodes.paragraph() + self.state.nested_parse(StringList(content), 0, node) + return [node] + + +def init_annotations(app: Sphinx) -> None: + # Using domaindata is a bit hack-ish, + # but allows storing state without a global variable or closure. + app.env.domaindata["c_annotations"] = state = {} + state["refcount_data"] = read_refcount_data( + Path(app.srcdir, app.config.refcount_file) + ) + state["stable_abi_data"] = read_stable_abi_data( + Path(app.srcdir, app.config.stable_abi_file) + ) -def setup(app): - app.add_config_value('refcount_file', '', True) - app.add_config_value('stable_abi_file', '', True) - app.connect('builder-inited', init_annotations) +def setup(app: Sphinx) -> ExtensionMetadata: + app.add_config_value("refcount_file", "", "env", types={str}) + app.add_config_value("stable_abi_file", "", "env", types={str}) + app.add_directive("limited-api-list", LimitedAPIList) + app.connect("builder-inited", init_annotations) + app.connect("doctree-read", add_annotations) - # monkey-patch C object... - CObject.option_spec = { - 'noindex': directives.flag, - 'stableabi': directives.flag, + return { + "version": "1.0", + "parallel_read_safe": True, + "parallel_write_safe": True, } - old_handle_signature = CObject.handle_signature - def new_handle_signature(self, sig, signode): - signode.parent['stableabi'] = 'stableabi' in self.options - return old_handle_signature(self, sig, signode) - CObject.handle_signature = new_handle_signature - return {'version': '1.0', 'parallel_read_safe': True} diff --git a/Doc/tools/extensions/changes.py b/Doc/tools/extensions/changes.py new file mode 100644 index 00000000000000..8de5e7f78c6627 --- /dev/null +++ b/Doc/tools/extensions/changes.py @@ -0,0 +1,90 @@ +"""Support for documenting version of changes, additions, deprecations.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sphinx.domains.changeset import ( + VersionChange, + versionlabel_classes, + versionlabels, +) +from sphinx.locale import _ as sphinx_gettext + +if TYPE_CHECKING: + from docutils.nodes import Node + from sphinx.application import Sphinx + from sphinx.util.typing import ExtensionMetadata + + +def expand_version_arg(argument: str, release: str) -> str: + """Expand "next" to the current version""" + if argument == "next": + return sphinx_gettext("{} (unreleased)").format(release) + return argument + + +class PyVersionChange(VersionChange): + def run(self) -> list[Node]: + # Replace the 'next' special token with the current development version + self.arguments[0] = expand_version_arg( + self.arguments[0], self.config.release + ) + return super().run() + + +class DeprecatedRemoved(VersionChange): + required_arguments = 2 + + _deprecated_label = sphinx_gettext( + "Deprecated since version %s, will be removed in version %s" + ) + _removed_label = sphinx_gettext( + "Deprecated since version %s, removed in version %s" + ) + + def run(self) -> list[Node]: + # Replace the first two arguments (deprecated version and removed version) + # with a single tuple of both versions. + version_deprecated = expand_version_arg( + self.arguments[0], self.config.release + ) + version_removed = self.arguments.pop(1) + if version_removed == "next": + raise ValueError( + "deprecated-removed:: second argument cannot be `next`" + ) + self.arguments[0] = version_deprecated, version_removed + + # Set the label based on if we have reached the removal version + current_version = tuple(map(int, self.config.version.split("."))) + removed_version = tuple(map(int, version_removed.split("."))) + if current_version < removed_version: + versionlabels[self.name] = self._deprecated_label + versionlabel_classes[self.name] = "deprecated" + else: + versionlabels[self.name] = self._removed_label + versionlabel_classes[self.name] = "removed" + try: + return super().run() + finally: + # reset versionlabels and versionlabel_classes + versionlabels[self.name] = "" + versionlabel_classes[self.name] = "" + + +def setup(app: Sphinx) -> ExtensionMetadata: + # Override Sphinx's directives with support for 'next' + app.add_directive("versionadded", PyVersionChange, override=True) + app.add_directive("versionchanged", PyVersionChange, override=True) + app.add_directive("versionremoved", PyVersionChange, override=True) + app.add_directive("deprecated", PyVersionChange, override=True) + + # Register the ``.. deprecated-removed::`` directive + app.add_directive("deprecated-removed", DeprecatedRemoved) + + return { + "version": "1.0", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/Doc/tools/extensions/escape4chm.py b/Doc/tools/extensions/escape4chm.py deleted file mode 100644 index 89970975b9032b..00000000000000 --- a/Doc/tools/extensions/escape4chm.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -Escape the `body` part of .chm source file to 7-bit ASCII, to fix visual -effect on some MBCS Windows systems. - -https://bugs.python.org/issue32174 -""" - -import pathlib -import re -from html.entities import codepoint2name - -from sphinx.util.logging import getLogger - -# escape the characters which codepoint > 0x7F -def _process(string): - def escape(matchobj): - codepoint = ord(matchobj.group(0)) - - name = codepoint2name.get(codepoint) - if name is None: - return '&#%d;' % codepoint - else: - return '&%s;' % name - - return re.sub(r'[^\x00-\x7F]', escape, string) - -def escape_for_chm(app, pagename, templatename, context, doctree): - # only works for .chm output - if getattr(app.builder, 'name', '') != 'htmlhelp': - return - - # escape the `body` part to 7-bit ASCII - body = context.get('body') - if body is not None: - context['body'] = _process(body) - -def fixup_keywords(app, exception): - # only works for .chm output - if getattr(app.builder, 'name', '') != 'htmlhelp' or exception: - return - - getLogger(__name__).info('fixing HTML escapes in keywords file...') - outdir = pathlib.Path(app.builder.outdir) - outname = app.builder.config.htmlhelp_basename - with open(outdir / (outname + '.hhk'), 'rb') as f: - index = f.read() - with open(outdir / (outname + '.hhk'), 'wb') as f: - f.write(index.replace(b''', b''')) - -def setup(app): - # `html-page-context` event emitted when the HTML builder has - # created a context dictionary to render a template with. - app.connect('html-page-context', escape_for_chm) - # `build-finished` event emitted when all the files have been - # output. - app.connect('build-finished', fixup_keywords) - - return {'version': '1.0', 'parallel_read_safe': True} diff --git a/Doc/tools/extensions/glossary_search.py b/Doc/tools/extensions/glossary_search.py index 59a6862ea3d3f4..502b6cd95bcb94 100644 --- a/Doc/tools/extensions/glossary_search.py +++ b/Doc/tools/extensions/glossary_search.py @@ -1,63 +1,63 @@ -# -*- coding: utf-8 -*- -""" - glossary_search.py - ~~~~~~~~~~~~~~~~ +"""Feature search results for glossary items prominently.""" - Feature search results for glossary items prominently. +from __future__ import annotations - :license: Python license. -""" import json -import os.path -from docutils.nodes import definition_list_item +from pathlib import Path +from typing import TYPE_CHECKING + +from docutils import nodes from sphinx.addnodes import glossary from sphinx.util import logging +if TYPE_CHECKING: + from sphinx.application import Sphinx + from sphinx.util.typing import ExtensionMetadata logger = logging.getLogger(__name__) -STATIC_DIR = '_static' -JSON = 'glossary.json' -def process_glossary_nodes(app, doctree, fromdocname): - if app.builder.format != 'html': +def process_glossary_nodes( + app: Sphinx, + doctree: nodes.document, + _docname: str, +) -> None: + if app.builder.format != 'html' or app.builder.embedded: return - terms = {} + if hasattr(app.env, 'glossary_terms'): + terms = app.env.glossary_terms + else: + terms = app.env.glossary_terms = {} - for node in doctree.traverse(glossary): - for glossary_item in node.traverse(definition_list_item): - term = glossary_item[0].astext().lower() - definition = glossary_item[1] + for node in doctree.findall(glossary): + for glossary_item in node.findall(nodes.definition_list_item): + term = glossary_item[0].astext() + definition = glossary_item[-1] rendered = app.builder.render_partial(definition) - terms[term] = { - 'title': glossary_item[0].astext(), - 'body': rendered['html_body'] + terms[term.lower()] = { + 'title': term, + 'body': rendered['html_body'], } - if hasattr(app.env, 'glossary_terms'): - app.env.glossary_terms.update(terms) - else: - app.env.glossary_terms = terms -def on_build_finish(app, exc): - if not hasattr(app.env, 'glossary_terms'): - return - if not app.env.glossary_terms: +def write_glossary_json(app: Sphinx, _exc: Exception) -> None: + if not getattr(app.env, 'glossary_terms', None): return - logger.info(f'Writing {JSON}', color='green') - - dest_dir = os.path.join(app.outdir, STATIC_DIR) - os.makedirs(dest_dir, exist_ok=True) - - with open(os.path.join(dest_dir, JSON), 'w') as f: - json.dump(app.env.glossary_terms, f) + logger.info('Writing glossary.json', color='green') + dest = Path(app.outdir, '_static', 'glossary.json') + dest.parent.mkdir(exist_ok=True) + dest.write_text(json.dumps(app.env.glossary_terms), encoding='utf-8') -def setup(app): +def setup(app: Sphinx) -> ExtensionMetadata: app.connect('doctree-resolved', process_glossary_nodes) - app.connect('build-finished', on_build_finish) + app.connect('build-finished', write_glossary_json) - return {'version': '0.1', 'parallel_read_safe': True} + return { + 'version': '1.0', + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/Doc/tools/extensions/lexers/__init__.py b/Doc/tools/extensions/lexers/__init__.py new file mode 100644 index 00000000000000..e12ac5be8139cc --- /dev/null +++ b/Doc/tools/extensions/lexers/__init__.py @@ -0,0 +1,15 @@ +from .asdl_lexer import ASDLLexer +from .peg_lexer import PEGLexer + + +def setup(app): + # Used for highlighting Parser/Python.asdl in library/ast.rst + app.add_lexer("asdl", ASDLLexer) + # Used for highlighting Grammar/python.gram in reference/grammar.rst + app.add_lexer("peg", PEGLexer) + + return { + "version": "1.0", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/Doc/tools/extensions/asdl_highlight.py b/Doc/tools/extensions/lexers/asdl_lexer.py similarity index 62% rename from Doc/tools/extensions/asdl_highlight.py rename to Doc/tools/extensions/lexers/asdl_lexer.py index 42863a4b3bcd6a..3a74174a1f7dfb 100644 --- a/Doc/tools/extensions/asdl_highlight.py +++ b/Doc/tools/extensions/lexers/asdl_lexer.py @@ -1,15 +1,6 @@ -import sys -from pathlib import Path +from pygments.lexer import RegexLexer, bygroups, include +from pygments.token import Comment, Keyword, Name, Operator, Punctuation, Text -CPYTHON_ROOT = Path(__file__).resolve().parent.parent.parent.parent -sys.path.append(str(CPYTHON_ROOT / "Parser")) - -from pygments.lexer import RegexLexer, bygroups, include, words -from pygments.token import (Comment, Keyword, Name, Operator, - Punctuation, Text) - -from asdl import builtin_types -from sphinx.highlighting import lexers class ASDLLexer(RegexLexer): name = "ASDL" @@ -34,7 +25,10 @@ class ASDLLexer(RegexLexer): r"(\w+)(\*\s|\?\s|\s)(\w+)", bygroups(Name.Builtin.Pseudo, Operator, Name), ), - (words(builtin_types), Name.Builtin), + # Keep in line with ``builtin_types`` from Parser/asdl.py. + # ASDL's 4 builtin types are + # constant, identifier, int, string + ("constant|identifier|int|string", Name.Builtin), (r"attributes", Name.Builtin), ( _name + _text_ws + "(=)", @@ -46,8 +40,3 @@ class ASDLLexer(RegexLexer): (r".", Text), ], } - - -def setup(app): - lexers["asdl"] = ASDLLexer() - return {'version': '1.0', 'parallel_read_safe': True} diff --git a/Doc/tools/extensions/peg_highlight.py b/Doc/tools/extensions/lexers/peg_lexer.py similarity index 94% rename from Doc/tools/extensions/peg_highlight.py rename to Doc/tools/extensions/lexers/peg_lexer.py index 4bdc2ee1861334..827af205583f61 100644 --- a/Doc/tools/extensions/peg_highlight.py +++ b/Doc/tools/extensions/lexers/peg_lexer.py @@ -1,8 +1,6 @@ from pygments.lexer import RegexLexer, bygroups, include from pygments.token import Comment, Keyword, Name, Operator, Punctuation, Text -from sphinx.highlighting import lexers - class PEGLexer(RegexLexer): """Pygments Lexer for PEG grammar (.gram) files @@ -79,8 +77,3 @@ class PEGLexer(RegexLexer): (r".", Text), ], } - - -def setup(app): - lexers["peg"] = PEGLexer() - return {"version": "1.0", "parallel_read_safe": True} diff --git a/Doc/tools/extensions/patchlevel.py b/Doc/tools/extensions/patchlevel.py index 617f28c2527ddf..9ccaec3dd5ce0f 100644 --- a/Doc/tools/extensions/patchlevel.py +++ b/Doc/tools/extensions/patchlevel.py @@ -1,68 +1,81 @@ -# -*- coding: utf-8 -*- -""" - patchlevel.py - ~~~~~~~~~~~~~ +"""Extract version information from Include/patchlevel.h.""" - Extract version info from Include/patchlevel.h. - Adapted from Doc/tools/getversioninfo. +import re +import sys +from pathlib import Path +from typing import Literal, NamedTuple - :copyright: 2007-2008 by Georg Brandl. - :license: Python license. -""" +CPYTHON_ROOT = Path( + __file__, # cpython/Doc/tools/extensions/patchlevel.py + "..", # cpython/Doc/tools/extensions + "..", # cpython/Doc/tools + "..", # cpython/Doc + "..", # cpython +).resolve() +PATCHLEVEL_H = CPYTHON_ROOT / "Include" / "patchlevel.h" -from __future__ import print_function +RELEASE_LEVELS = { + "PY_RELEASE_LEVEL_ALPHA": "alpha", + "PY_RELEASE_LEVEL_BETA": "beta", + "PY_RELEASE_LEVEL_GAMMA": "candidate", + "PY_RELEASE_LEVEL_FINAL": "final", +} -import os -import re -import sys -def get_header_version_info(srcdir): - patchlevel_h = os.path.join(srcdir, '..', 'Include', 'patchlevel.h') +class version_info(NamedTuple): # noqa: N801 + major: int #: Major release number + minor: int #: Minor release number + micro: int #: Patch release number + releaselevel: Literal["alpha", "beta", "candidate", "final"] + serial: int #: Serial release number - # This won't pick out all #defines, but it will pick up the ones we - # care about. - rx = re.compile(r'\s*#define\s+([a-zA-Z][a-zA-Z_0-9]*)\s+([a-zA-Z_0-9]+)') - d = {} - with open(patchlevel_h) as f: - for line in f: - m = rx.match(line) - if m is not None: - name, value = m.group(1, 2) - d[name] = value +def get_header_version_info() -> version_info: + # Capture PY_ prefixed #defines. + pat = re.compile(r"\s*#define\s+(PY_\w*)\s+(\w+)", re.ASCII) - release = version = '%s.%s' % (d['PY_MAJOR_VERSION'], d['PY_MINOR_VERSION']) - micro = int(d['PY_MICRO_VERSION']) - release += '.' + str(micro) + defines = {} + patchlevel_h = PATCHLEVEL_H.read_text(encoding="utf-8") + for line in patchlevel_h.splitlines(): + if (m := pat.match(line)) is not None: + name, value = m.groups() + defines[name] = value - level = d['PY_RELEASE_LEVEL'] - suffixes = { - 'PY_RELEASE_LEVEL_ALPHA': 'a', - 'PY_RELEASE_LEVEL_BETA': 'b', - 'PY_RELEASE_LEVEL_GAMMA': 'rc', - } - if level != 'PY_RELEASE_LEVEL_FINAL': - release += suffixes[level] + str(int(d['PY_RELEASE_SERIAL'])) - return version, release + return version_info( + major=int(defines["PY_MAJOR_VERSION"]), + minor=int(defines["PY_MINOR_VERSION"]), + micro=int(defines["PY_MICRO_VERSION"]), + releaselevel=RELEASE_LEVELS[defines["PY_RELEASE_LEVEL"]], + serial=int(defines["PY_RELEASE_SERIAL"]), + ) -def get_sys_version_info(): - major, minor, micro, level, serial = sys.version_info - release = version = '%s.%s' % (major, minor) - release += '.%s' % micro - if level != 'final': - release += '%s%s' % (level[0], serial) +def format_version_info(info: version_info) -> tuple[str, str]: + version = f"{info.major}.{info.minor}" + release = f"{info.major}.{info.minor}.{info.micro}" + if info.releaselevel != "final": + suffix = {"alpha": "a", "beta": "b", "candidate": "rc"} + release += f"{suffix[info.releaselevel]}{info.serial}" return version, release def get_version_info(): try: - return get_header_version_info('.') - except (IOError, OSError): - version, release = get_sys_version_info() - print('Can\'t get version info from Include/patchlevel.h, ' \ - 'using version of this interpreter (%s).' % release, file=sys.stderr) + info = get_header_version_info() + return format_version_info(info) + except OSError: + version, release = format_version_info(sys.version_info) + print( + f"Failed to get version info from Include/patchlevel.h, " + f"using version of this interpreter ({release}).", + file=sys.stderr, + ) return version, release -if __name__ == '__main__': - print(get_header_version_info('.')[1]) + +if __name__ == "__main__": + short_ver, full_ver = format_version_info(get_header_version_info()) + if sys.argv[1:2] == ["--short"]: + print(short_ver) + else: + print(full_ver) diff --git a/Doc/tools/extensions/pydoc_topics.py b/Doc/tools/extensions/pydoc_topics.py new file mode 100644 index 00000000000000..6e43df2e4bf9de --- /dev/null +++ b/Doc/tools/extensions/pydoc_topics.py @@ -0,0 +1,187 @@ +"""Support for building "topic help" for pydoc.""" + +from __future__ import annotations + +from time import asctime +from typing import TYPE_CHECKING + +from sphinx.builders.text import TextBuilder +from sphinx.util import logging +from sphinx.util.display import status_iterator +from sphinx.util.docutils import new_document +from sphinx.writers.text import TextTranslator + +if TYPE_CHECKING: + from collections.abc import Sequence, Set + + from sphinx.application import Sphinx + from sphinx.util.typing import ExtensionMetadata + +logger = logging.getLogger(__name__) + +_PYDOC_TOPIC_LABELS: Sequence[str] = sorted({ + "assert", + "assignment", + "assignment-expressions", + "async", + "atom-identifiers", + "atom-literals", + "attribute-access", + "attribute-references", + "augassign", + "await", + "binary", + "bitwise", + "bltin-code-objects", + "bltin-ellipsis-object", + "bltin-null-object", + "bltin-type-objects", + "booleans", + "break", + "callable-types", + "calls", + "class", + "comparisons", + "compound", + "context-managers", + "continue", + "conversions", + "customization", + "debugger", + "del", + "dict", + "dynamic-features", + "else", + "exceptions", + "execmodel", + "exprlists", + "floating", + "for", + "formatstrings", + "function", + "global", + "id-classes", + "identifiers", + "if", + "imaginary", + "import", + "in", + "integers", + "lambda", + "lists", + "naming", + "nonlocal", + "numbers", + "numeric-types", + "objects", + "operator-summary", + "pass", + "power", + "raise", + "return", + "sequence-types", + "shifting", + "slicings", + "specialattrs", + "specialnames", + "string-methods", + "strings", + "subscriptions", + "truth", + "try", + "types", + "typesfunctions", + "typesmapping", + "typesmethods", + "typesmodules", + "typesseq", + "typesseq-mutable", + "unary", + "while", + "with", + "yield", +}) + + +class PydocTopicsBuilder(TextBuilder): + name = "pydoc-topics" + + def init(self) -> None: + super().init() + self.topics: dict[str, str] = {} + + def get_outdated_docs(self) -> str: + # Return a string describing what an update build will build. + return "all pydoc topics" + + def write_documents(self, _docnames: Set[str]) -> None: + env = self.env + + labels: dict[str, tuple[str, str, str]] + labels = env.domains.standard_domain.labels + + # docname -> list of (topic_label, label_id) pairs + doc_labels: dict[str, list[tuple[str, str]]] = {} + for topic_label in _PYDOC_TOPIC_LABELS: + try: + docname, label_id, _section_name = labels[topic_label] + except KeyError: + logger.warning("label %r not in documentation", topic_label) + continue + doc_labels.setdefault(docname, []).append((topic_label, label_id)) + + for docname, label_ids in status_iterator( + doc_labels.items(), + "building topics... ", + length=len(doc_labels), + stringify_func=_display_labels, + ): + doctree = env.get_and_resolve_doctree(docname, builder=self) + doc_ids = doctree.ids + for topic_label, label_id in label_ids: + document = new_document("
") + document.append(doc_ids[label_id]) + visitor = TextTranslator(document, builder=self) + document.walkabout(visitor) + self.topics[topic_label] = visitor.body + + def finish(self) -> None: + topics_repr = "\n".join( + f" '{topic}': {_repr(self.topics[topic])}," + for topic in sorted(self.topics) + ) + topics = f"""\ +# Autogenerated by Sphinx on {asctime()} +# as part of the release process. + +topics = {{ +{topics_repr} +}} +""" + self.outdir.joinpath("topics.py").write_text(topics, encoding="utf-8") + + +def _display_labels(item: tuple[str, Sequence[tuple[str, str]]]) -> str: + _docname, label_ids = item + labels = [name for name, _id in label_ids] + if len(labels) > 4: + return f"{labels[0]}, {labels[1]}, ..., {labels[-2]}, {labels[-1]}" + return ", ".join(labels) + + +def _repr(text: str, /) -> str: + """Return a triple-single-quoted representation of text.""" + if "'''" not in text: + return f"r'''{text}'''" + text = text.replace("\\", "\\\\").replace("'''", r"\'\'\'") + return f"'''{text}'''" + + +def setup(app: Sphinx) -> ExtensionMetadata: + app.add_builder(PydocTopicsBuilder) + + return { + "version": "1.0", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index cd441836f62bde..f363dfd4216929 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -12,33 +12,19 @@ import re import io from os import getenv, path -from time import asctime -from pprint import pformat -from docutils import nodes, utils -from docutils.io import StringOutput -from docutils.parsers.rst import Directive -from docutils.utils import new_document +from docutils import nodes +from docutils.parsers.rst import directives +from docutils.utils import unescape from sphinx import addnodes -from sphinx.builders import Builder -from sphinx.domains.python import PyFunction, PyMethod -from sphinx.errors import NoUri +from sphinx.domains.python import PyFunction, PyMethod, PyModule from sphinx.locale import _ as sphinx_gettext -from sphinx.util import logging from sphinx.util.docutils import SphinxDirective -from sphinx.util.nodes import split_explicit_title -from sphinx.writers.text import TextWriter, TextTranslator - -try: - # Sphinx 6+ - from sphinx.util.display import status_iterator -except ImportError: - # Deprecated in Sphinx 6.1, will be removed in Sphinx 8 - from sphinx.util import status_iterator ISSUE_URI = 'https://bugs.python.org/issue?@action=redirect&bpo=%s' GH_ISSUE_URI = 'https://github.com/python/cpython/issues/%s' +# Used in conf.py and updated here by python/release-tools/run_release.py SOURCE_URI = 'https://github.com/python/cpython/tree/main/%s' # monkey-patch reST parser to disable alphabetic and roman enumerated lists @@ -48,16 +34,10 @@ Body.enum.converters['lowerroman'] = \ Body.enum.converters['upperroman'] = lambda x: None -# monkey-patch the productionlist directive to allow hyphens in group names -# https://github.com/sphinx-doc/sphinx/issues/11854 -from sphinx.domains import std - -std.token_re = re.compile(r'`((~?[\w-]*:)?\w+)`') - # Support for marking up and linking to bugs.python.org issues def issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): - issue = utils.unescape(text) + issue = unescape(text) # sanity check: there are no bpo issues within these two values if 47261 < int(issue) < 400000: msg = inliner.reporter.error(f'The BPO ID {text!r} seems too high -- ' @@ -72,7 +52,7 @@ def issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): # Support for marking up and linking to GitHub issues def gh_issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): - issue = utils.unescape(text) + issue = unescape(text) # sanity check: all GitHub issues have ID >= 32426 # even though some of them are also valid BPO IDs if int(issue) < 32426: @@ -85,19 +65,9 @@ def gh_issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): return [refnode], [] -# Support for linking to Python source files easily - -def source_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): - has_t, title, target = split_explicit_title(text) - title = utils.unescape(title) - target = utils.unescape(target) - refnode = nodes.reference(title, title, refuri=SOURCE_URI % target) - return [refnode], [] - - # Support for marking up implementation details -class ImplementationDetail(Directive): +class ImplementationDetail(SphinxDirective): has_content = True final_argument_whitespace = True @@ -120,244 +90,6 @@ def run(self): return [pnode] -# Support for documenting platform availability - -class Availability(SphinxDirective): - - has_content = True - required_arguments = 1 - optional_arguments = 0 - final_argument_whitespace = True - - # known platform, libc, and threading implementations - known_platforms = frozenset({ - "AIX", "Android", "BSD", "DragonFlyBSD", "Emscripten", "FreeBSD", - "GNU/kFreeBSD", "Linux", "NetBSD", "OpenBSD", "POSIX", "Solaris", - "Unix", "VxWorks", "WASI", "Windows", "macOS", - # libc - "BSD libc", "glibc", "musl", - # POSIX platforms with pthreads - "pthreads", - }) - - def run(self): - availability_ref = ':ref:`Availability `: ' - avail_nodes, avail_msgs = self.state.inline_text( - availability_ref + self.arguments[0], - self.lineno) - pnode = nodes.paragraph(availability_ref + self.arguments[0], - '', *avail_nodes, *avail_msgs) - self.set_source_info(pnode) - cnode = nodes.container("", pnode, classes=["availability"]) - self.set_source_info(cnode) - if self.content: - self.state.nested_parse(self.content, self.content_offset, cnode) - self.parse_platforms() - - return [cnode] - - def parse_platforms(self): - """Parse platform information from arguments - - Arguments is a comma-separated string of platforms. A platform may - be prefixed with "not " to indicate that a feature is not available. - - Example:: - - .. availability:: Windows, Linux >= 4.2, not Emscripten, not WASI - - Arguments like "Linux >= 3.17 with glibc >= 2.27" are currently not - parsed into separate tokens. - """ - platforms = {} - for arg in self.arguments[0].rstrip(".").split(","): - arg = arg.strip() - platform, _, version = arg.partition(" >= ") - if platform.startswith("not "): - version = False - platform = platform[4:] - elif not version: - version = True - platforms[platform] = version - - unknown = set(platforms).difference(self.known_platforms) - if unknown: - cls = type(self) - logger = logging.getLogger(cls.__qualname__) - logger.warning( - f"Unknown platform(s) or syntax '{' '.join(sorted(unknown))}' " - f"in '.. availability:: {self.arguments[0]}', see " - f"{__file__}:{cls.__qualname__}.known_platforms for a set " - "known platforms." - ) - - return platforms - - - -# Support for documenting audit event - -def audit_events_purge(app, env, docname): - """This is to remove from env.all_audit_events old traces of removed - documents. - """ - if not hasattr(env, 'all_audit_events'): - return - fresh_all_audit_events = {} - for name, event in env.all_audit_events.items(): - event["source"] = [(d, t) for d, t in event["source"] if d != docname] - if event["source"]: - # Only keep audit_events that have at least one source. - fresh_all_audit_events[name] = event - env.all_audit_events = fresh_all_audit_events - - -def audit_events_merge(app, env, docnames, other): - """In Sphinx parallel builds, this merges env.all_audit_events from - subprocesses. - - all_audit_events is a dict of names, with values like: - {'source': [(docname, target), ...], 'args': args} - """ - if not hasattr(other, 'all_audit_events'): - return - if not hasattr(env, 'all_audit_events'): - env.all_audit_events = {} - for name, value in other.all_audit_events.items(): - if name in env.all_audit_events: - env.all_audit_events[name]["source"].extend(value["source"]) - else: - env.all_audit_events[name] = value - - -class AuditEvent(Directive): - - has_content = True - required_arguments = 1 - optional_arguments = 2 - final_argument_whitespace = True - - _label = [ - sphinx_gettext("Raises an :ref:`auditing event ` {name} with no arguments."), - sphinx_gettext("Raises an :ref:`auditing event ` {name} with argument {args}."), - sphinx_gettext("Raises an :ref:`auditing event ` {name} with arguments {args}."), - ] - - @property - def logger(self): - cls = type(self) - return logging.getLogger(cls.__module__ + "." + cls.__name__) - - def run(self): - name = self.arguments[0] - if len(self.arguments) >= 2 and self.arguments[1]: - args = (a.strip() for a in self.arguments[1].strip("'\"").split(",")) - args = [a for a in args if a] - else: - args = [] - - label = self._label[min(2, len(args))] - text = label.format(name="``{}``".format(name), - args=", ".join("``{}``".format(a) for a in args if a)) - - env = self.state.document.settings.env - if not hasattr(env, 'all_audit_events'): - env.all_audit_events = {} - - new_info = { - 'source': [], - 'args': args - } - info = env.all_audit_events.setdefault(name, new_info) - if info is not new_info: - if not self._do_args_match(info['args'], new_info['args']): - self.logger.warning( - "Mismatched arguments for audit-event {}: {!r} != {!r}" - .format(name, info['args'], new_info['args']) - ) - - ids = [] - try: - target = self.arguments[2].strip("\"'") - except (IndexError, TypeError): - target = None - if not target: - target = "audit_event_{}_{}".format( - re.sub(r'\W', '_', name), - len(info['source']), - ) - ids.append(target) - - info['source'].append((env.docname, target)) - - pnode = nodes.paragraph(text, classes=["audit-hook"], ids=ids) - pnode.line = self.lineno - if self.content: - self.state.nested_parse(self.content, self.content_offset, pnode) - else: - n, m = self.state.inline_text(text, self.lineno) - pnode.extend(n + m) - - return [pnode] - - # This list of sets are allowable synonyms for event argument names. - # If two names are in the same set, they are treated as equal for the - # purposes of warning. This won't help if number of arguments is - # different! - _SYNONYMS = [ - {"file", "path", "fd"}, - ] - - def _do_args_match(self, args1, args2): - if args1 == args2: - return True - if len(args1) != len(args2): - return False - for a1, a2 in zip(args1, args2): - if a1 == a2: - continue - if any(a1 in s and a2 in s for s in self._SYNONYMS): - continue - return False - return True - - -class audit_event_list(nodes.General, nodes.Element): - pass - - -class AuditEventListDirective(Directive): - - def run(self): - return [audit_event_list('')] - - -# Support for documenting decorators - -class PyDecoratorMixin(object): - def handle_signature(self, sig, signode): - ret = super(PyDecoratorMixin, self).handle_signature(sig, signode) - signode.insert(0, addnodes.desc_addname('@', '@')) - return ret - - def needs_arglist(self): - return False - - -class PyDecoratorFunction(PyDecoratorMixin, PyFunction): - def run(self): - # a decorator function is a function after all - self.name = 'py:function' - return PyFunction.run(self) - - -# TODO: Use sphinx.domains.python.PyDecoratorMethod when possible -class PyDecoratorMethod(PyDecoratorMixin, PyMethod): - def run(self): - self.name = 'py:method' - return PyMethod.run(self) - - class PyCoroutineMixin(object): def handle_signature(self, sig, signode): ret = super(PyCoroutineMixin, self).handle_signature(sig, signode) @@ -409,62 +141,6 @@ def run(self): return PyMethod.run(self) -# Support for documenting version of removal in deprecations - -class DeprecatedRemoved(Directive): - has_content = True - required_arguments = 2 - optional_arguments = 1 - final_argument_whitespace = True - option_spec = {} - - _deprecated_label = sphinx_gettext('Deprecated since version {deprecated}, will be removed in version {removed}') - _removed_label = sphinx_gettext('Deprecated since version {deprecated}, removed in version {removed}') - - def run(self): - node = addnodes.versionmodified() - node.document = self.state.document - node['type'] = 'deprecated-removed' - version = (self.arguments[0], self.arguments[1]) - node['version'] = version - env = self.state.document.settings.env - current_version = tuple(int(e) for e in env.config.version.split('.')) - removed_version = tuple(int(e) for e in self.arguments[1].split('.')) - if current_version < removed_version: - label = self._deprecated_label - else: - label = self._removed_label - - text = label.format(deprecated=self.arguments[0], removed=self.arguments[1]) - if len(self.arguments) == 3: - inodes, messages = self.state.inline_text(self.arguments[2], - self.lineno+1) - para = nodes.paragraph(self.arguments[2], '', *inodes, translatable=False) - node.append(para) - else: - messages = [] - if self.content: - self.state.nested_parse(self.content, self.content_offset, node) - if len(node): - if isinstance(node[0], nodes.paragraph) and node[0].rawsource: - content = nodes.inline(node[0].rawsource, translatable=True) - content.source = node[0].source - content.line = node[0].line - content += node[0].children - node[0].replace_self(nodes.paragraph('', '', content, translatable=False)) - node[0].insert(0, nodes.inline('', '%s: ' % text, - classes=['versionmodified'])) - else: - para = nodes.paragraph('', '', - nodes.inline('', '%s.' % text, - classes=['versionmodified']), - translatable=False) - node.append(para) - env = self.state.document.settings.env - env.get_domain('changeset').note_changeset(node) - return [node] + messages - - # Support for including Misc/NEWS issue_re = re.compile('(?:[Ii]ssue #|bpo-)([0-9]+)', re.I) @@ -472,7 +148,7 @@ def run(self): whatsnew_re = re.compile(r"(?im)^what's new in (.*?)\??$") -class MiscNews(Directive): +class MiscNews(SphinxDirective): has_content = False required_arguments = 1 optional_arguments = 0 @@ -487,7 +163,7 @@ def run(self): if not source_dir: source_dir = path.dirname(path.abspath(source)) fpath = path.join(source_dir, fname) - self.state.document.settings.record_dependencies.add(fpath) + self.env.note_dependency(path.abspath(fpath)) try: with io.open(fpath, encoding='utf-8') as fp: content = fp.read() @@ -505,69 +181,6 @@ def run(self): return [] -# Support for building "topic help" for pydoc - -pydoc_topic_labels = [ - 'assert', 'assignment', 'async', 'atom-identifiers', 'atom-literals', - 'attribute-access', 'attribute-references', 'augassign', 'await', - 'binary', 'bitwise', 'bltin-code-objects', 'bltin-ellipsis-object', - 'bltin-null-object', 'bltin-type-objects', 'booleans', - 'break', 'callable-types', 'calls', 'class', 'comparisons', 'compound', - 'context-managers', 'continue', 'conversions', 'customization', 'debugger', - 'del', 'dict', 'dynamic-features', 'else', 'exceptions', 'execmodel', - 'exprlists', 'floating', 'for', 'formatstrings', 'function', 'global', - 'id-classes', 'identifiers', 'if', 'imaginary', 'import', 'in', 'integers', - 'lambda', 'lists', 'naming', 'nonlocal', 'numbers', 'numeric-types', - 'objects', 'operator-summary', 'pass', 'power', 'raise', 'return', - 'sequence-types', 'shifting', 'slicings', 'specialattrs', 'specialnames', - 'string-methods', 'strings', 'subscriptions', 'truth', 'try', 'types', - 'typesfunctions', 'typesmapping', 'typesmethods', 'typesmodules', - 'typesseq', 'typesseq-mutable', 'unary', 'while', 'with', 'yield' -] - - -class PydocTopicsBuilder(Builder): - name = 'pydoc-topics' - - default_translator_class = TextTranslator - - def init(self): - self.topics = {} - self.secnumbers = {} - - def get_outdated_docs(self): - return 'all pydoc topics' - - def get_target_uri(self, docname, typ=None): - return '' # no URIs - - def write(self, *ignored): - writer = TextWriter(self) - for label in status_iterator(pydoc_topic_labels, - 'building topics... ', - length=len(pydoc_topic_labels)): - if label not in self.env.domaindata['std']['labels']: - self.env.logger.warning(f'label {label!r} not in documentation') - continue - docname, labelid, sectname = self.env.domaindata['std']['labels'][label] - doctree = self.env.get_and_resolve_doctree(docname, self) - document = new_document('
') - document.append(doctree.ids[labelid]) - destination = StringOutput(encoding='utf-8') - writer.write(document, destination) - self.topics[label] = writer.output - - def finish(self): - f = open(path.join(self.outdir, 'topics.py'), 'wb') - try: - f.write('# -*- coding: utf-8 -*-\n'.encode('utf-8')) - f.write(('# Autogenerated by Sphinx on %s\n' % asctime()).encode('utf-8')) - f.write('# as part of the release process.\n'.encode('utf-8')) - f.write(('topics = ' + pformat(self.topics) + '\n').encode('utf-8')) - finally: - f.close() - - # Support for documenting Opcodes opcode_sig_re = re.compile(r'(\w+(?:\+\d)?)(?:\s*\((.*)\))?') @@ -619,70 +232,6 @@ def parse_monitoring_event(env, sig, signode): return sig -def process_audit_events(app, doctree, fromdocname): - for node in doctree.traverse(audit_event_list): - break - else: - return - - env = app.builder.env - - table = nodes.table(cols=3) - group = nodes.tgroup( - '', - nodes.colspec(colwidth=30), - nodes.colspec(colwidth=55), - nodes.colspec(colwidth=15), - cols=3, - ) - head = nodes.thead() - body = nodes.tbody() - - table += group - group += head - group += body - - row = nodes.row() - row += nodes.entry('', nodes.paragraph('', nodes.Text('Audit event'))) - row += nodes.entry('', nodes.paragraph('', nodes.Text('Arguments'))) - row += nodes.entry('', nodes.paragraph('', nodes.Text('References'))) - head += row - - for name in sorted(getattr(env, "all_audit_events", ())): - audit_event = env.all_audit_events[name] - - row = nodes.row() - node = nodes.paragraph('', nodes.Text(name)) - row += nodes.entry('', node) - - node = nodes.paragraph() - for i, a in enumerate(audit_event['args']): - if i: - node += nodes.Text(", ") - node += nodes.literal(a, nodes.Text(a)) - row += nodes.entry('', node) - - node = nodes.paragraph() - backlinks = enumerate(sorted(set(audit_event['source'])), start=1) - for i, (doc, label) in backlinks: - if isinstance(label, str): - ref = nodes.reference("", nodes.Text("[{}]".format(i)), internal=True) - try: - ref['refuri'] = "{}#{}".format( - app.builder.get_relative_uri(fromdocname, doc), - label, - ) - except NoUri: - continue - node += ref - row += nodes.entry('', node) - - body += row - - for node in doctree.traverse(audit_event_list): - node.replace_self(table) - - def patch_pairindextypes(app, _env) -> None: """Remove all entries from ``pairindextypes`` before writing POT files. @@ -710,18 +259,10 @@ def patch_pairindextypes(app, _env) -> None: def setup(app): app.add_role('issue', issue_role) app.add_role('gh', gh_issue_role) - app.add_role('source', source_role) app.add_directive('impl-detail', ImplementationDetail) - app.add_directive('availability', Availability) - app.add_directive('audit-event', AuditEvent) - app.add_directive('audit-event-table', AuditEventListDirective) - app.add_directive('deprecated-removed', DeprecatedRemoved) - app.add_builder(PydocTopicsBuilder) app.add_object_type('opcode', 'opcode', '%s (opcode)', parse_opcode_signature) app.add_object_type('pdbcommand', 'pdbcmd', '%s (pdb command)', parse_pdb_command) app.add_object_type('monitoring-event', 'monitoring-event', '%s (monitoring event)', parse_monitoring_event) - app.add_directive_to_domain('py', 'decorator', PyDecoratorFunction) - app.add_directive_to_domain('py', 'decoratormethod', PyDecoratorMethod) app.add_directive_to_domain('py', 'coroutinefunction', PyCoroutineFunction) app.add_directive_to_domain('py', 'coroutinemethod', PyCoroutineMethod) app.add_directive_to_domain('py', 'awaitablefunction', PyAwaitableFunction) @@ -729,7 +270,4 @@ def setup(app): app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod) app.add_directive('miscnews', MiscNews) app.connect('env-check-consistency', patch_pairindextypes) - app.connect('doctree-resolved', process_audit_events) - app.connect('env-merge-info', audit_events_merge) - app.connect('env-purge-doc', audit_events_purge) return {'version': '1.0', 'parallel_read_safe': True} diff --git a/Doc/tools/static/glossary_search.js b/Doc/tools/static/glossary_search.js new file mode 100644 index 00000000000000..13d728dc027f1d --- /dev/null +++ b/Doc/tools/static/glossary_search.js @@ -0,0 +1,47 @@ +"use strict"; + +const GLOSSARY_PAGE = "glossary.html"; + +const glossary_search = async () => { + const response = await fetch("_static/glossary.json"); + if (!response.ok) { + throw new Error("Failed to fetch glossary.json"); + } + const glossary = await response.json(); + + const params = new URLSearchParams(document.location.search).get("q"); + if (!params) { + return; + } + + const searchParam = params.toLowerCase(); + const glossaryItem = glossary[searchParam]; + if (!glossaryItem) { + return; + } + + // set up the title text with a link to the glossary page + const glossaryTitle = document.getElementById("glossary-title"); + glossaryTitle.textContent = "Glossary: " + glossaryItem.title; + const linkTarget = searchParam.replace(/ /g, "-"); + glossaryTitle.href = GLOSSARY_PAGE + "#term-" + linkTarget; + + // rewrite any anchor links (to other glossary terms) + // to have a full reference to the glossary page + const glossaryBody = document.getElementById("glossary-body"); + glossaryBody.innerHTML = glossaryItem.body; + const anchorLinks = glossaryBody.querySelectorAll('a[href^="#"]'); + anchorLinks.forEach(function (link) { + const currentUrl = link.getAttribute("href"); + link.href = GLOSSARY_PAGE + currentUrl; + }); + + const glossaryResult = document.getElementById("glossary-result"); + glossaryResult.style.display = ""; +}; + +if (document.readyState !== "loading") { + glossary_search().catch(console.error); +} else { + document.addEventListener("DOMContentLoaded", glossary_search); +} diff --git a/Doc/tools/static/rtd_switcher.js b/Doc/tools/static/rtd_switcher.js new file mode 100644 index 00000000000000..2bf01a002db90c --- /dev/null +++ b/Doc/tools/static/rtd_switcher.js @@ -0,0 +1,55 @@ + function onSwitch(event) { + const option = event.target.selectedIndex; + const item = event.target.options[option]; + window.location.href = item.dataset.url; + } + + document.addEventListener("readthedocs-addons-data-ready", function(event) { + const config = event.detail.data() + const versionSelect = ` + + `; + + // Prepend the current language to the options on the selector + let languages = config.projects.translations.concat(config.projects.current); + languages = languages.sort((a, b) => a.language.name.localeCompare(b.language.name)); + + const languageSelect = ` + + `; + + // Query all the placeholders because there are different ones for Desktop/Mobile + const versionPlaceholders = document.querySelectorAll(".version_switcher_placeholder"); + for (placeholder of versionPlaceholders) { + placeholder.innerHTML = versionSelect; + let selectElement = placeholder.querySelector("select"); + selectElement.addEventListener("change", onSwitch); + } + + const languagePlaceholders = document.querySelectorAll(".language_switcher_placeholder"); + for (placeholder of languagePlaceholders) { + placeholder.innerHTML = languageSelect; + let selectElement = placeholder.querySelector("select"); + selectElement.addEventListener("change", onSwitch); + } + }); diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index b5353d6fb77ab4..45ec436fee72d7 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -1,64 +1,75 @@ {% extends "layout.html" %} -{% set title = 'Download' %} +{% set title = _('Download') %} {% if daily is defined %} - {% set dlbase = pathto('archives', 1) %} + {% set dl_base = pathto('archives', resource=True) %} + {% set dl_version = version %} {% else %} {# The link below returns HTTP 404 until the first related alpha release. This is expected; use daily documentation builds for CPython development. #} - {% set dlbase = 'https://docs.python.org/ftp/python/doc/' + release %} + {% set dl_base = 'https://www.python.org/ftp/python/doc/' + release %} + {% set dl_version = release %} {% endif %} {% block body %} -

Download Python {{ release }} Documentation

+

{% trans %}Download Python {{ dl_version }} Documentation{% endtrans %}

-{% if last_updated %}

Last updated on: {{ last_updated }}.

{% endif %} +{% if last_updated %}

{% trans %}Last updated on: {{ last_updated }}.{% endtrans %}

{% endif %} -

To download an archive containing all the documents for this version of -Python in one of various formats, follow one of links in this table.

+

{% trans %}To download an archive containing all the documents for this version of +Python in one of various formats, follow one of links in this table.{% endtrans %}

- - - - + + + + - - - + + + + - - - + + + + - - - + + + + - - + + + + + + + +
FormatPacked as .zipPacked as .tar.bz2
PDF (US-Letter paper size)Download (ca. 13 MiB)Download (ca. 13 MiB)
{% trans %}Format{% endtrans %}{% trans %}Packed as .zip{% endtrans %}{% trans %}Packed as .tar.bz2{% endtrans %}
PDF (A4 paper size)Download (ca. 13 MiB)Download (ca. 13 MiB)
{% trans %}PDF{% endtrans %}{% trans download_size="17" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans download_size="17" %}Download (ca. {{ download_size }} MiB){% endtrans %}
HTMLDownload (ca. 9 MiB)Download (ca. 6 MiB)
{% trans %}HTML{% endtrans %}{% trans download_size="13" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans download_size="8" %}Download (ca. {{ download_size }} MiB){% endtrans %}
Plain TextDownload (ca. 3 MiB)Download (ca. 2 MiB)
{% trans %}Plain text{% endtrans %}{% trans download_size="4" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans download_size="3" %}Download (ca. {{ download_size }} MiB){% endtrans %}
EPUBDownload (ca. 5 MiB)
{% trans %}Texinfo{% endtrans %}{% trans download_size="9" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans download_size="7" %}Download (ca. {{ download_size }} MiB){% endtrans %}
{% trans %}EPUB{% endtrans %}{% trans download_size="6" %}Download (ca. {{ download_size }} MiB){% endtrans %}
-

These archives contain all the content in the documentation.

+

{% trans %}These archives contain all the content in the documentation.{% endtrans %}

-

Unpacking

+

{% trans %}Unpacking{% endtrans %}

-

Unix users should download the .tar.bz2 archives; these are bzipped tar +

{% trans %}Unix users should download the .tar.bz2 archives; these are bzipped tar archives and can be handled in the usual way using tar and the bzip2 program. The Info-ZIP unzip program can be used to handle the ZIP archives if desired. The .tar.bz2 archives provide the -best compression and fastest download times.

+best compression and fastest download times.{% endtrans %}

-

Windows users can use the ZIP archives since those are customary on that -platform. These are created on Unix using the Info-ZIP zip program.

+

{% trans %}Windows users can use the ZIP archives since those are customary on that +platform. These are created on Unix using the Info-ZIP zip program.{% endtrans %}

-

Problems

+

{% trans %}Problems{% endtrans %}

-

If you have comments or suggestions for the Python documentation, please send -email to docs@python.org.

+

{% trans %}If you have comments or suggestions for the Python documentation, please send +email to docs@python.org.{% endtrans %}

{% endblock %} diff --git a/Doc/tools/templates/indexcontent.html b/Doc/tools/templates/indexcontent.html index 1e3ab7cfe02fee..2686f48dad2a95 100644 --- a/Doc/tools/templates/indexcontent.html +++ b/Doc/tools/templates/indexcontent.html @@ -7,62 +7,64 @@

{{ docstitle|e }}

{% trans %}Welcome! This is the official documentation for Python {{ release }}.{% endtrans %}

-

{% trans %}Parts of the documentation:{% endtrans %}

+

{% trans %}Documentation sections:{% endtrans %}

+ {% trans whatsnew_index=pathto("whatsnew/index") %}Or all "What's new" documents since Python 2.0{% endtrans %}

- - - + {% trans %}Start here: a tour of Python's syntax and features{% endtrans %}

+ + + + {% trans %}In-depth topic manuals{% endtrans %}

- - - - + + + + + {% trans %}Frequently asked questions (with answers!){% endtrans %}

+
-

{% trans %}Indices and tables:{% endtrans %}

+

{% trans %}Indices, glossary, and search:{% endtrans %}

- - + + + {% trans %}Terms explained{% endtrans %}

- + {% trans %}Search this documentation{% endtrans %}

+
-

{% trans %}Meta information:{% endtrans %}

+

{% trans %}Project information:{% endtrans %}

- - - + + + - + - +
{% endblock %} diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 9498b2ccc5af92..b09fd21a8ddcc9 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -26,6 +26,9 @@ {% endblock %} {% block extrahead %} + {% if builder == "html" and enable_analytics %} + + {% endif %} {% if builder != "htmlhelp" %} {% if pagename == 'whatsnew/changelog' and not embedded %} @@ -41,4 +44,9 @@ {{ "}" }} {{ super() }} + +{%- if not embedded %} + + +{%- endif %} {% endblock %} diff --git a/Doc/tools/templates/search.html b/Doc/tools/templates/search.html index 852974461380f2..6ddac5f828bab1 100644 --- a/Doc/tools/templates/search.html +++ b/Doc/tools/templates/search.html @@ -2,61 +2,16 @@ {% block extrahead %} {{ super() }} - + +{% endblock %} +{% block searchresults %} +
+ {# For glossary_search.js #} + +
{% endblock %} diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index 4bea0d8a49ce20..6a1611afadb57c 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -10,6 +10,28 @@ Appendix Interactive Mode ================ +There are two variants of the interactive :term:`REPL`. The classic +basic interpreter is supported on all platforms with minimal line +control capabilities. + +On Windows, or Unix-like systems with :mod:`curses` support, +a new interactive shell is used by default. +This one supports color, multiline editing, history browsing, and +paste mode. To disable color, see :ref:`using-on-controlling-color` for +details. Function keys provide some additional functionality. +:kbd:`F1` enters the interactive help browser :mod:`pydoc`. +:kbd:`F2` allows for browsing command-line history with neither output nor the +:term:`>>>` and :term:`...` prompts. :kbd:`F3` enters "paste mode", which +makes pasting larger blocks of code easier. Press :kbd:`F3` to return to +the regular prompt. + +When using the new interactive shell, exit the shell by typing :kbd:`exit` +or :kbd:`quit`. Adding call parentheses after those commands is not +required. + +If the new interactive shell is not desired, it can be disabled via +the :envvar:`PYTHON_BASIC_REPL` environment variable. + .. _tut-error: Error Handling @@ -40,7 +62,7 @@ Executable Python Scripts On BSD'ish Unix systems, Python scripts can be made directly executable, like shell scripts, by putting the line :: - #!/usr/bin/env python3.5 + #!/usr/bin/env python3 (assuming that the interpreter is on the user's :envvar:`PATH`) at the beginning of the script and giving the file an executable mode. The ``#!`` must be the @@ -107,7 +129,7 @@ of your user site-packages directory. Start Python and run this code:: >>> import site >>> site.getusersitepackages() - '/home/user/.local/lib/python3.5/site-packages' + '/home/user/.local/lib/python3.x/site-packages' Now you can create a file named :file:`usercustomize.py` in that directory and put anything you want in it. It will affect every invocation of Python, unless diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index d1c303ef037027..9d0fab8861d2a9 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -276,8 +276,8 @@ definition looked like this:: then ``MyClass.i`` and ``MyClass.f`` are valid attribute references, returning an integer and a function object, respectively. Class attributes can also be assigned to, so you can change the value of ``MyClass.i`` by assignment. -:attr:`!__doc__` is also a valid attribute, returning the docstring belonging to -the class: ``"A simple example class"``. +:attr:`~type.__doc__` is also a valid attribute, returning the docstring +belonging to the class: ``"A simple example class"``. Class *instantiation* uses function notation. Just pretend that the class object is a parameterless function that returns a new instance of the class. @@ -325,7 +325,7 @@ Now what can we do with instance objects? The only operations understood by instance objects are attribute references. There are two kinds of valid attribute names: data attributes and methods. -*data attributes* correspond to "instance variables" in Smalltalk, and to "data +*Data attributes* correspond to "instance variables" in Smalltalk, and to "data members" in C++. Data attributes need not be declared; like local variables, they spring into existence when they are first assigned to. For example, if ``x`` is the instance of :class:`!MyClass` created above, the following piece of @@ -338,11 +338,7 @@ code will print the value ``16``, without leaving a trace:: del x.counter The other kind of instance attribute reference is a *method*. A method is a -function that "belongs to" an object. (In Python, the term method is not unique -to class instances: other object types can have methods as well. For example, -list objects have methods called append, insert, remove, sort, and so on. -However, in the following discussion, we'll use the term method exclusively to -mean methods of class instance objects, unless explicitly stated otherwise.) +function that "belongs to" an object. .. index:: pair: object; method @@ -665,7 +661,7 @@ class, that calls each parent only once, and that is monotonic (meaning that a class can be subclassed without affecting the precedence order of its parents). Taken together, these properties make it possible to design reliable and extensible classes with multiple inheritance. For more detail, see -https://www.python.org/download/releases/2.3/mro/. +:ref:`python_2.3_mro`. .. _tut-private: @@ -692,6 +688,11 @@ current class name with leading underscore(s) stripped. This mangling is done without regard to the syntactic position of the identifier, as long as it occurs within the definition of a class. +.. seealso:: + + The :ref:`private name mangling specifications ` + for details and special cases. + Name mangling is helpful for letting subclasses override methods without breaking intraclass method calls. For example:: @@ -931,6 +932,6 @@ Examples:: .. [#] Except for one thing. Module objects have a secret read-only attribute called :attr:`~object.__dict__` which returns the dictionary used to implement the module's - namespace; the name :attr:`~object.__dict__` is an attribute but not a global name. + namespace; the name ``__dict__`` is an attribute but not a global name. Obviously, using this violates the abstraction of namespace implementation, and should be restricted to things like post-mortem debuggers. diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 77444f9cb8358d..8261bbdbfb7a01 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -61,7 +61,7 @@ they appear in the sequence. For example (no pun intended): :: >>> # Measure some strings: - ... words = ['cat', 'window', 'defenestrate'] + >>> words = ['cat', 'window', 'defenestrate'] >>> for w in words: ... print(w, len(w)) ... @@ -160,21 +160,60 @@ arguments. In chapter :ref:`tut-structures`, we will discuss in more detail abo .. _tut-break: -:keyword:`!break` and :keyword:`!continue` Statements, and :keyword:`!else` Clauses on Loops -============================================================================================ +:keyword:`!break` and :keyword:`!continue` Statements +===================================================== The :keyword:`break` statement breaks out of the innermost enclosing -:keyword:`for` or :keyword:`while` loop. +:keyword:`for` or :keyword:`while` loop:: -A :keyword:`!for` or :keyword:`!while` loop can include an :keyword:`!else` clause. + >>> for n in range(2, 10): + ... for x in range(2, n): + ... if n % x == 0: + ... print(f"{n} equals {x} * {n//x}") + ... break + ... + 4 equals 2 * 2 + 6 equals 2 * 3 + 8 equals 2 * 4 + 9 equals 3 * 3 + +The :keyword:`continue` statement continues with the next +iteration of the loop:: + + >>> for num in range(2, 10): + ... if num % 2 == 0: + ... print(f"Found an even number {num}") + ... continue + ... print(f"Found an odd number {num}") + ... + Found an even number 2 + Found an odd number 3 + Found an even number 4 + Found an odd number 5 + Found an even number 6 + Found an odd number 7 + Found an even number 8 + Found an odd number 9 + +.. _tut-for-else: +.. _break-and-continue-statements-and-else-clauses-on-loops: + +:keyword:`!else` Clauses on Loops +================================= + +In a :keyword:`!for` or :keyword:`!while` loop the :keyword:`!break` statement +may be paired with an :keyword:`!else` clause. If the loop finishes without +executing the :keyword:`!break`, the :keyword:`!else` clause executes. In a :keyword:`for` loop, the :keyword:`!else` clause is executed -after the loop reaches its final iteration. +after the loop finishes its final iteration, that is, if no break occurred. In a :keyword:`while` loop, it's executed after the loop's condition becomes false. -In either kind of loop, the :keyword:`!else` clause is **not** executed -if the loop was terminated by a :keyword:`break`. +In either kind of loop, the :keyword:`!else` clause is **not** executed if the +loop was terminated by a :keyword:`break`. Of course, other ways of ending the +loop early, such as a :keyword:`return` or a raised exception, will also skip +execution of the :keyword:`else` clause. This is exemplified in the following :keyword:`!for` loop, which searches for prime numbers:: @@ -198,32 +237,19 @@ which searches for prime numbers:: 9 equals 3 * 3 (Yes, this is the correct code. Look closely: the ``else`` clause belongs to -the :keyword:`for` loop, **not** the :keyword:`if` statement.) +the ``for`` loop, **not** the ``if`` statement.) -When used with a loop, the ``else`` clause has more in common with the -``else`` clause of a :keyword:`try` statement than it does with that of -:keyword:`if` statements: a :keyword:`try` statement's ``else`` clause runs -when no exception occurs, and a loop's ``else`` clause runs when no ``break`` -occurs. For more on the :keyword:`!try` statement and exceptions, see -:ref:`tut-handling`. - -The :keyword:`continue` statement, also borrowed from C, continues with the next -iteration of the loop:: +One way to think of the else clause is to imagine it paired with the ``if`` +inside the loop. As the loop executes, it will run a sequence like +if/if/if/else. The ``if`` is inside the loop, encountered a number of times. If +the condition is ever true, a ``break`` will happen. If the condition is never +true, the ``else`` clause outside the loop will execute. - >>> for num in range(2, 10): - ... if num % 2 == 0: - ... print("Found an even number", num) - ... continue - ... print("Found an odd number", num) - ... - Found an even number 2 - Found an odd number 3 - Found an even number 4 - Found an odd number 5 - Found an even number 6 - Found an odd number 7 - Found an even number 8 - Found an odd number 9 +When used with a loop, the ``else`` clause has more in common with the ``else`` +clause of a :keyword:`try` statement than it does with that of ``if`` +statements: a ``try`` statement's ``else`` clause runs when no exception +occurs, and a loop's ``else`` clause runs when no ``break`` occurs. For more on +the ``try`` statement and exceptions, see :ref:`tut-handling`. .. _tut-pass: @@ -436,8 +462,8 @@ Defining Functions We can create a function that writes the Fibonacci series to an arbitrary boundary:: - >>> def fib(n): # write Fibonacci series up to n - ... """Print a Fibonacci series up to n.""" + >>> def fib(n): # write Fibonacci series less than n + ... """Print a Fibonacci series less than n.""" ... a, b = 0, 1 ... while a < n: ... print(a, end=' ') @@ -445,7 +471,7 @@ boundary:: ... print() ... >>> # Now call the function we just defined: - ... fib(2000) + >>> fib(2000) 0 1 1 2 3 5 8 13 21 34 55 89 144 233 377 610 987 1597 .. index:: @@ -807,7 +833,7 @@ parameters as there is a ``/`` in the function definition:: File "", line 1, in TypeError: pos_only_arg() got some positional-only arguments passed as keyword arguments: 'arg' -The third function ``kwd_only_args`` only allows keyword arguments as indicated +The third function ``kwd_only_arg`` only allows keyword arguments as indicated by a ``*`` in the function definition:: >>> kwd_only_arg(3) diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index de2827461e2f24..cbe780e075baf5 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -19,13 +19,13 @@ objects: .. method:: list.append(x) :noindex: - Add an item to the end of the list. Equivalent to ``a[len(a):] = [x]``. + Add an item to the end of the list. Similar to ``a[len(a):] = [x]``. .. method:: list.extend(iterable) :noindex: - Extend the list by appending all the items from the iterable. Equivalent to + Extend the list by appending all the items from the iterable. Similar to ``a[len(a):] = iterable``. @@ -56,7 +56,7 @@ objects: .. method:: list.clear() :noindex: - Remove all items from the list. Equivalent to ``del a[:]``. + Remove all items from the list. Similar to ``del a[:]``. .. method:: list.index(x[, start[, end]]) @@ -93,7 +93,7 @@ objects: .. method:: list.copy() :noindex: - Return a shallow copy of the list. Equivalent to ``a[:]``. + Return a shallow copy of the list. Similar to ``a[:]``. An example that uses most of the list methods:: @@ -126,7 +126,7 @@ Python. Another thing you might notice is that not all data can be sorted or compared. For instance, ``[None, 'hello', 10]`` doesn't sort because -integers can't be compared to strings and *None* can't be compared to +integers can't be compared to strings and ``None`` can't be compared to other types. Also, there are some types that don't have a defined ordering relation. For example, ``3+4j < 5+7j`` isn't a valid comparison. @@ -142,8 +142,8 @@ Using Lists as Stacks The list methods make it very easy to use a list as a stack, where the last element added is the first element retrieved ("last-in, first-out"). To add an -item to the top of the stack, use :meth:`~list.append`. To retrieve an item from the -top of the stack, use :meth:`~list.pop` without an explicit index. For example:: +item to the top of the stack, use :meth:`!append`. To retrieve an item from the +top of the stack, use :meth:`!pop` without an explicit index. For example:: >>> stack = [3, 4, 5] >>> stack.append(6) @@ -340,7 +340,7 @@ The :keyword:`!del` statement ============================= There is a way to remove an item from a list given its index instead of its -value: the :keyword:`del` statement. This differs from the :meth:`~list.pop` method +value: the :keyword:`del` statement. This differs from the :meth:`!pop` method which returns a value. The :keyword:`!del` statement can also be used to remove slices from a list or clear the entire list (which we did earlier by assignment of an empty list to the slice). For example:: @@ -383,16 +383,16 @@ A tuple consists of a number of values separated by commas, for instance:: >>> t (12345, 54321, 'hello!') >>> # Tuples may be nested: - ... u = t, (1, 2, 3, 4, 5) + >>> u = t, (1, 2, 3, 4, 5) >>> u ((12345, 54321, 'hello!'), (1, 2, 3, 4, 5)) >>> # Tuples are immutable: - ... t[0] = 88888 + >>> t[0] = 88888 Traceback (most recent call last): File "", line 1, in TypeError: 'tuple' object does not support item assignment >>> # but they can contain mutable objects: - ... v = ([1, 2, 3], [3, 2, 1]) + >>> v = ([1, 2, 3], [3, 2, 1]) >>> v ([1, 2, 3], [3, 2, 1]) @@ -465,7 +465,7 @@ Here is a brief demonstration:: False >>> # Demonstrate set operations on unique letters from two words - ... + >>> >>> a = set('abracadabra') >>> b = set('alacazam') >>> a # unique letters in a @@ -500,8 +500,8 @@ any immutable type; strings and numbers can always be keys. Tuples can be used as keys if they contain only strings, numbers, or tuples; if a tuple contains any mutable object either directly or indirectly, it cannot be used as a key. You can't use lists as keys, since lists can be modified in place using index -assignments, slice assignments, or methods like :meth:`~list.append` and -:meth:`~list.extend`. +assignments, slice assignments, or methods like :meth:`!append` and +:meth:`!extend`. It is best to think of a dictionary as a set of *key: value* pairs, with the requirement that the keys are unique (within one dictionary). A pair of diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index 4058ebe8efdb42..c01cb8c14a0360 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -20,12 +20,12 @@ complaint you get while you are still learning Python:: >>> while True print('Hello world') File "", line 1 while True print('Hello world') - ^ + ^^^^^ SyntaxError: invalid syntax -The parser repeats the offending line and displays a little 'arrow' pointing at -the earliest point in the line where the error was detected. The error is -caused by (or at least detected at) the token *preceding* the arrow: in the +The parser repeats the offending line and displays little arrows pointing +at the token in the line where the error was detected. The error may be +caused by the absence of a token *before* the indicated token. In the example, the error is detected at the function :func:`print`, since a colon (``':'``) is missing before it. File name and line number are printed so you know where to look in case the input came from a script. @@ -45,14 +45,20 @@ programs, however, and result in error messages as shown here:: >>> 10 * (1/0) Traceback (most recent call last): File "", line 1, in + 10 * (1/0) + ~^~ ZeroDivisionError: division by zero >>> 4 + spam*3 Traceback (most recent call last): File "", line 1, in + 4 + spam*3 + ^^^^ NameError: name 'spam' is not defined >>> '2' + 2 Traceback (most recent call last): File "", line 1, in + '2' + 2 + ~~~~^~~ TypeError: can only concatenate str (not "int") to str The last line of the error message indicates what happened. Exceptions come in @@ -119,9 +125,9 @@ may name multiple exceptions as a parenthesized tuple, for example:: ... except (RuntimeError, TypeError, NameError): ... pass -A class in an :keyword:`except` clause is compatible with an exception if it is -the same class or a base class thereof (but not the other way around --- an -*except clause* listing a derived class is not compatible with a base class). +A class in an :keyword:`except` clause matches exceptions which are instances of the +class itself or one of its derived classes (but not the other way around --- an +*except clause* listing a derived class does not match instances of its base classes). For example, the following code will print B, C, D in that order:: class B(Exception): @@ -252,6 +258,7 @@ exception to occur. For example:: >>> raise NameError('HiThere') Traceback (most recent call last): File "", line 1, in + raise NameError('HiThere') NameError: HiThere The sole argument to :keyword:`raise` indicates the exception to be raised. @@ -275,6 +282,7 @@ re-raise the exception:: An exception flew by! Traceback (most recent call last): File "", line 2, in + raise NameError('HiThere') NameError: HiThere @@ -294,12 +302,15 @@ message:: ... Traceback (most recent call last): File "", line 2, in + open("database.sqlite") + ~~~~^^^^^^^^^^^^^^^^^^^ FileNotFoundError: [Errno 2] No such file or directory: 'database.sqlite' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "", line 4, in + raise RuntimeError("unable to handle error") RuntimeError: unable to handle error To indicate that an exception is a direct consequence of another, the @@ -320,6 +331,8 @@ This can be useful when you are transforming exceptions. For example:: ... Traceback (most recent call last): File "", line 2, in + func() + ~~~~^^ File "", line 2, in func ConnectionError @@ -327,6 +340,7 @@ This can be useful when you are transforming exceptions. For example:: Traceback (most recent call last): File "", line 4, in + raise RuntimeError('Failed to open database') from exc RuntimeError: Failed to open database It also allows disabling automatic exception chaining using the ``from None`` @@ -339,6 +353,7 @@ idiom:: ... Traceback (most recent call last): File "", line 4, in + raise RuntimeError from None RuntimeError For more information about chaining mechanics, see :ref:`bltin-exceptions`. @@ -381,6 +396,7 @@ example:: Goodbye, world! Traceback (most recent call last): File "", line 2, in + raise KeyboardInterrupt KeyboardInterrupt If a :keyword:`finally` clause is present, the :keyword:`!finally` @@ -448,7 +464,11 @@ A more complicated example:: executing finally clause Traceback (most recent call last): File "", line 1, in + divide("2", "1") + ~~~~~~^^^^^^^^^^ File "", line 3, in divide + result = x / y + ~~^~~ TypeError: unsupported operand type(s) for /: 'str' and 'str' As you can see, the :keyword:`finally` clause is executed in any event. The @@ -511,8 +531,11 @@ caught like any other exception. :: >>> f() + Exception Group Traceback (most recent call last): | File "", line 1, in + | f() + | ~^^ | File "", line 3, in f - | ExceptionGroup: there were problems + | raise ExceptionGroup('there were problems', excs) + | ExceptionGroup: there were problems (2 sub-exceptions) +-+---------------- 1 ---------------- | OSError: error 1 +---------------- 2 ---------------- @@ -560,10 +583,15 @@ other clauses and eventually to be reraised. :: There were SystemErrors + Exception Group Traceback (most recent call last): | File "", line 2, in + | f() + | ~^^ | File "", line 2, in f - | ExceptionGroup: group1 + | raise ExceptionGroup( + | ...<12 lines>... + | ) + | ExceptionGroup: group1 (1 sub-exception) +-+---------------- 1 ---------------- - | ExceptionGroup: group2 + | ExceptionGroup: group2 (1 sub-exception) +-+---------------- 1 ---------------- | RecursionError: 4 +------------------------------------ @@ -607,6 +635,7 @@ includes all notes, in the order they were added, after the exception. :: ... Traceback (most recent call last): File "", line 2, in + raise TypeError('bad type') TypeError: bad type Add some information Add some more information @@ -630,23 +659,33 @@ exception in the group has a note indicating when this error has occurred. :: >>> raise ExceptionGroup('We have some problems', excs) + Exception Group Traceback (most recent call last): | File "", line 1, in + | raise ExceptionGroup('We have some problems', excs) | ExceptionGroup: We have some problems (3 sub-exceptions) +-+---------------- 1 ---------------- | Traceback (most recent call last): | File "", line 3, in + | f() + | ~^^ | File "", line 2, in f + | raise OSError('operation failed') | OSError: operation failed | Happened in Iteration 1 +---------------- 2 ---------------- | Traceback (most recent call last): | File "", line 3, in + | f() + | ~^^ | File "", line 2, in f + | raise OSError('operation failed') | OSError: operation failed | Happened in Iteration 2 +---------------- 3 ---------------- | Traceback (most recent call last): | File "", line 3, in + | f() + | ~^^ | File "", line 2, in f + | raise OSError('operation failed') | OSError: operation failed | Happened in Iteration 3 +------------------------------------ diff --git a/Doc/tutorial/floatingpoint.rst b/Doc/tutorial/floatingpoint.rst index 0795e2fef98830..dfe2d1d3a8378f 100644 --- a/Doc/tutorial/floatingpoint.rst +++ b/Doc/tutorial/floatingpoint.rst @@ -6,7 +6,7 @@ .. _tut-fp-issues: ************************************************** -Floating Point Arithmetic: Issues and Limitations +Floating-Point Arithmetic: Issues and Limitations ************************************************** .. sectionauthor:: Tim Peters @@ -88,7 +88,7 @@ the one with 17 significant digits, ``0.10000000000000001``. Starting with Python 3.1, Python (on most systems) is now able to choose the shortest of these and simply display ``0.1``. -Note that this is in the very nature of binary floating-point: this is not a bug +Note that this is in the very nature of binary floating point: this is not a bug in Python, and it is not a bug in your code either. You'll see the same kind of thing in all languages that support your hardware's floating-point arithmetic (although some languages may not *display* the difference by default, or in all @@ -148,13 +148,13 @@ Binary floating-point arithmetic holds many surprises like this. The problem with "0.1" is explained in precise detail below, in the "Representation Error" section. See `Examples of Floating Point Problems `_ for -a pleasant summary of how binary floating-point works and the kinds of +a pleasant summary of how binary floating point works and the kinds of problems commonly encountered in practice. Also see `The Perils of Floating Point `_ for a more complete account of other common surprises. As that says near the end, "there are no easy answers." Still, don't be unduly -wary of floating-point! The errors in Python float operations are inherited +wary of floating point! The errors in Python float operations are inherited from the floating-point hardware, and on most machines are on the order of no more than 1 part in 2\*\*53 per operation. That's more than adequate for most tasks, but you do need to keep in mind that it's not decimal arithmetic and @@ -230,7 +230,7 @@ accumulate to the point where they affect the final total: >>> sum([0.1] * 10) == 1.0 True -The :func:`math.fsum()` goes further and tracks all of the "lost digits" +The :func:`math.fsum` goes further and tracks all of the "lost digits" as values are added onto a running total so that the result has only a single rounding. This is slower than :func:`sum` but will be more accurate in uncommon cases where large magnitude inputs mostly cancel diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index fe9ca9ccb9c7e0..35b8c7cd8eb049 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -37,16 +37,23 @@ printing space-separated values. There are several ways to format output. * The :meth:`str.format` method of strings requires more manual effort. You'll still use ``{`` and ``}`` to mark where a variable will be substituted and can provide detailed formatting directives, - but you'll also need to provide the information to be formatted. + but you'll also need to provide the information to be formatted. In the following code + block there are two examples of how to format variables: + :: >>> yes_votes = 42_572_654 - >>> no_votes = 43_132_495 - >>> percentage = yes_votes / (yes_votes + no_votes) + >>> total_votes = 85_705_149 + >>> percentage = yes_votes / total_votes >>> '{:-9} YES votes {:2.2%}'.format(yes_votes, percentage) ' 42572654 YES votes 49.67%' + Notice how the ``yes_votes`` are padded with spaces and a negative sign only for negative numbers. + The example also prints ``percentage`` multiplied by 100, with 2 decimal + places and followed by a percent sign (see :ref:`formatspec` for details). + + * Finally, you can do all the string handling yourself by using string slicing and concatenation operations to create any layout you can imagine. The string type has some methods that perform useful operations for padding @@ -80,12 +87,12 @@ Some examples:: >>> print(s) The value of x is 32.5, and y is 40000... >>> # The repr() of a string adds string quotes and backslashes: - ... hello = 'hello, world\n' + >>> hello = 'hello, world\n' >>> hellos = repr(hello) >>> print(hellos) 'hello, world\n' >>> # The argument to repr() may be any Python object: - ... repr((x, y, ('spam', 'eggs'))) + >>> repr((x, y, ('spam', 'eggs'))) "(32.5, 40000, ('spam', 'eggs'))" The :mod:`string` module contains a :class:`~string.Template` class that offers @@ -93,6 +100,13 @@ yet another way to substitute values into strings, using placeholders like ``$x`` and replacing them with values from a dictionary, but offers much less control of the formatting. +.. index:: + single: formatted string literal + single: interpolated string literal + single: string; formatted literal + single: string; interpolated literal + single: f-string + single: fstring .. _tut-f-strings: @@ -197,7 +211,12 @@ notation. :: Jack: 4098; Sjoerd: 4127; Dcab: 8637678 This is particularly useful in combination with the built-in function -:func:`vars`, which returns a dictionary containing all local variables. +:func:`vars`, which returns a dictionary containing all local variables:: + + >>> table = {k: str(v) for k, v in vars().items()} + >>> message = " ".join([f'{k}: ' + '{' + k +'};' for k in table.keys()]) + >>> print(message.format(**table)) + __name__: __main__; __doc__: None; __package__: None; __loader__: ... As an example, the following lines produce a tidily aligned set of columns giving integers and their squares and cubes:: @@ -267,9 +286,11 @@ left with zeros. It understands about plus and minus signs:: Old string formatting --------------------- -The % operator (modulo) can also be used for string formatting. Given ``'string' -% values``, instances of ``%`` in ``string`` are replaced with zero or more -elements of ``values``. This operation is commonly known as string +The % operator (modulo) can also be used for string formatting. +Given ``format % values`` (where *format* is a string), +``%`` conversion specifications in *format* are replaced with +zero or more elements of *values*. +This operation is commonly known as string interpolation. For example:: >>> import math diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index 299b6c2777adc0..02e7de77322e99 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -16,7 +16,7 @@ Unix shell's search path makes it possible to start it by typing the command: .. code-block:: text - python3.13 + python3.14 to the shell. [#]_ Since the choice of the directory where the interpreter lives is an installation option, other places are possible; check with your local @@ -97,8 +97,8 @@ before printing the first prompt: .. code-block:: shell-session - $ python3.13 - Python 3.13 (default, April 4 2023, 09:25:04) + $ python3.14 + Python 3.14 (default, April 4 2024, 09:25:04) [GCC 10.2.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 0f16dae8b1418f..65e3b1938bca9c 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -62,7 +62,7 @@ For example:: 20 >>> (50 - 5*6) / 4 5.0 - >>> 8 / 5 # division always returns a floating point number + >>> 8 / 5 # division always returns a floating-point number 1.6 The integer numbers (e.g. ``2``, ``4``, ``20``) have type :class:`int`, @@ -197,21 +197,19 @@ and workarounds. String literals can span multiple lines. One way is using triple-quotes: ``"""..."""`` or ``'''...'''``. End of lines are automatically included in the string, but it's possible to prevent this by adding a ``\`` at -the end of the line. The following example:: - - print("""\ +the end of the line. In the following example, the initial newline is not +included:: + + >>> print("""\ + ... Usage: thingy [OPTIONS] + ... -h Display this usage message + ... -H hostname Hostname to connect to + ... """) Usage: thingy [OPTIONS] -h Display this usage message -H hostname Hostname to connect to - """) - -produces the following output (note that the initial newline is not included): -.. code-block:: text - - Usage: thingy [OPTIONS] - -h Display this usage message - -H hostname Hostname to connect to + >>> Strings can be concatenated (glued together) with the ``+`` operator, and repeated with ``*``:: @@ -501,8 +499,8 @@ together. For instance, we can write an initial sub-sequence of the as follows:: >>> # Fibonacci series: - ... # the sum of two elements defines the next - ... a, b = 0, 1 + >>> # the sum of two elements defines the next + >>> a, b = 0, 1 >>> while a < 10: ... print(a) ... a, b = b, a+b @@ -544,7 +542,7 @@ This example introduces several new features. * The :func:`print` function writes the value of the argument(s) it is given. It differs from just writing the expression you want to write (as we did earlier in the calculator examples) in the way it handles multiple arguments, - floating point quantities, and strings. Strings are printed without quotes, + floating-point quantities, and strings. Strings are printed without quotes, and a space is inserted between items, so you can format things nicely, like this:: diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index 0316239e776a95..de7aa0e2342946 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -585,8 +585,9 @@ as the main module of a Python application must always use absolute imports. Packages in Multiple Directories -------------------------------- -Packages support one more special attribute, :attr:`__path__`. This is -initialized to be a list containing the name of the directory holding the +Packages support one more special attribute, :attr:`~module.__path__`. This is +initialized to be a :term:`sequence` of strings containing the name of the +directory holding the package's :file:`__init__.py` before the code in that file is executed. This variable can be modified; doing so affects future searches for modules and subpackages contained in the package. diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 9def2a5714950b..4b3eef313e76d7 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -15,7 +15,7 @@ operating system:: >>> import os >>> os.getcwd() # Return the current working directory - 'C:\\Python313' + 'C:\\Python314' >>> os.chdir('/server/accesslogs') # Change current working directory >>> os.system('mkdir today') # Run the command mkdir in the system shell 0 @@ -138,7 +138,7 @@ Mathematics =========== The :mod:`math` module gives access to the underlying C library functions for -floating point math:: +floating-point math:: >>> import math >>> math.cos(math.pi / 4) diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index 09b6f3d91bcfed..a2f96b34b2dead 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -279,7 +279,7 @@ applications include caching objects that are expensive to create:: Traceback (most recent call last): File "", line 1, in d['primary'] # entry was automatically removed - File "C:/python313/lib/weakref.py", line 46, in __getitem__ + File "C:/python314/lib/weakref.py", line 46, in __getitem__ o = self.data[key]() KeyError: 'primary' @@ -293,7 +293,7 @@ Many data structure needs can be met with the built-in list type. However, sometimes there is a need for alternative implementations with different performance trade-offs. -The :mod:`array` module provides an :class:`~array.array()` object that is like +The :mod:`array` module provides an :class:`~array.array` object that is like a list that stores only homogeneous data and stores it more compactly. The following example shows an array of numbers stored as two byte unsigned binary numbers (typecode ``"H"``) rather than the usual 16 bytes per entry for regular @@ -306,7 +306,7 @@ lists of Python int objects:: >>> a[1:3] array('H', [10, 700]) -The :mod:`collections` module provides a :class:`~collections.deque()` object +The :mod:`collections` module provides a :class:`~collections.deque` object that is like a list with faster appends and pops from the left side but slower lookups in the middle. These objects are well suited for implementing queues and breadth first tree searches:: @@ -352,11 +352,11 @@ not want to run a full list sort:: .. _tut-decimal-fp: -Decimal Floating Point Arithmetic +Decimal Floating-Point Arithmetic ================================= The :mod:`decimal` module offers a :class:`~decimal.Decimal` datatype for -decimal floating point arithmetic. Compared to the built-in :class:`float` +decimal floating-point arithmetic. Compared to the built-in :class:`float` implementation of binary floating point, the class is especially helpful for * financial applications and other uses which require exact decimal diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index a6dead2eac11f6..f362e1943b666f 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -36,10 +36,10 @@ Creating Virtual Environments ============================= The module used to create and manage virtual environments is called -:mod:`venv`. :mod:`venv` will usually install the most recent version of -Python that you have available. If you have multiple versions of Python on your -system, you can select a specific Python version by running ``python3`` or -whichever version you want. +:mod:`venv`. :mod:`venv` will install the Python version from which +the command was run (as reported by the :option:`--version` option). +For instance, executing the command with ``python3.12`` will install +version 3.12. To create a virtual environment, decide upon a directory where you want to place it, and run the :mod:`venv` module as a script with the directory path:: @@ -76,7 +76,7 @@ virtual environment you're using, and modify the environment so that running ``python`` will get you that particular version and installation of Python. For example: -.. code-block:: bash +.. code-block:: console $ source ~/envs/tutorial-env/bin/activate (tutorial-env) $ python @@ -108,7 +108,7 @@ complete documentation for ``pip``.) You can install the latest version of a package by specifying a package's name: -.. code-block:: bash +.. code-block:: console (tutorial-env) $ python -m pip install novas Collecting novas @@ -120,7 +120,7 @@ You can install the latest version of a package by specifying a package's name: You can also install a specific version of a package by giving the package name followed by ``==`` and the version number: -.. code-block:: bash +.. code-block:: console (tutorial-env) $ python -m pip install requests==2.6.0 Collecting requests==2.6.0 @@ -133,7 +133,7 @@ version is already installed and do nothing. You can supply a different version number to get that version, or you can run ``python -m pip install --upgrade`` to upgrade the package to the latest version: -.. code-block:: bash +.. code-block:: console (tutorial-env) $ python -m pip install --upgrade requests Collecting requests @@ -148,7 +148,7 @@ remove the packages from the virtual environment. ``python -m pip show`` will display information about a particular package: -.. code-block:: bash +.. code-block:: console (tutorial-env) $ python -m pip show requests --- @@ -166,7 +166,7 @@ remove the packages from the virtual environment. ``python -m pip list`` will display all of the packages installed in the virtual environment: -.. code-block:: bash +.. code-block:: console (tutorial-env) $ python -m pip list novas (3.1.1.3) @@ -179,7 +179,7 @@ the virtual environment: but the output uses the format that ``python -m pip install`` expects. A common convention is to put this list in a ``requirements.txt`` file: -.. code-block:: bash +.. code-block:: console (tutorial-env) $ python -m pip freeze > requirements.txt (tutorial-env) $ cat requirements.txt @@ -191,7 +191,7 @@ The ``requirements.txt`` can then be committed to version control and shipped as part of an application. Users can then install all the necessary packages with ``install -r``: -.. code-block:: bash +.. code-block:: console (tutorial-env) $ python -m pip install -r requirements.txt Collecting novas==3.1.1.3 (from -r requirements.txt (line 1)) diff --git a/Doc/using/android.rst b/Doc/using/android.rst new file mode 100644 index 00000000000000..957705f7f5e189 --- /dev/null +++ b/Doc/using/android.rst @@ -0,0 +1,65 @@ +.. _using-android: + +======================= +Using Python on Android +======================= + +Python on Android is unlike Python on desktop platforms. On a desktop platform, +Python is generally installed as a system resource that can be used by any user +of that computer. Users then interact with Python by running a :program:`python` +executable and entering commands at an interactive prompt, or by running a +Python script. + +On Android, there is no concept of installing as a system resource. The only unit +of software distribution is an "app". There is also no console where you could +run a :program:`python` executable, or interact with a Python REPL. + +As a result, the only way you can use Python on Android is in embedded mode – that +is, by writing a native Android application, embedding a Python interpreter +using ``libpython``, and invoking Python code using the :ref:`Python embedding +API `. The full Python interpreter, the standard library, and all +your Python code is then packaged into your app for its own private use. + +The Python standard library has some notable omissions and restrictions on +Android. See the :ref:`API availability guide ` for +details. + +Adding Python to an Android app +------------------------------- + +These instructions are only needed if you're planning to compile Python for +Android yourself. Most users should *not* need to do this. Instead, use one of +the following tools, which will provide a much easier experience: + +* `Briefcase `__, from the BeeWare project +* `Buildozer `__, from the Kivy project +* `Chaquopy `__ +* `pyqtdeploy `__ +* `Termux `__ + +If you're sure you want to do all of this manually, read on. You can use the +:source:`testbed app ` as a guide; each step below contains a +link to the relevant file. + +* Build Python by following the instructions in :source:`Android/README.md`. + +* Add code to your :source:`build.gradle ` + file to copy the following items into your project. All except your own Python + code can be copied from ``cross-build/HOST/prefix/lib``: + + * In your JNI libraries: + + * ``libpython*.*.so`` + * ``lib*_python.so`` (external libraries such as OpenSSL) + + * In your assets: + + * ``python*.*`` (the Python standard library) + * ``python*.*/site-packages`` (your own Python code) + +* Add code to your app to :source:`extract the assets to the filesystem + `. + +* Add code to your app to :source:`start Python in embedded mode + `. This will need to be C code + called via JNI. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 53c95ca1a05c9b..2a59cf3f62d4c5 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -24,7 +24,7 @@ Command line When invoking Python, you may specify any of these options:: - python [-bBdEhiIOqsSuvVWx?] [-c command | -m module-name | script | - ] [args] + python [-bBdEhiIOPqRsSuvVWx?] [-c command | -m module-name | script | - ] [args] The most common use case is, of course, a simple invocation of a script:: @@ -42,6 +42,7 @@ additional methods of invocation: * When called with standard input connected to a tty device, it prompts for commands and executes them until an EOF (an end-of-file character, you can produce that with :kbd:`Ctrl-D` on UNIX or :kbd:`Ctrl-Z, Enter` on Windows) is read. + For more on interactive mode, see :ref:`tut-interac`. * When called with a file name argument or with a file as standard input, it reads and executes a script from that file. * When called with a directory name argument, it reads and executes an @@ -242,12 +243,13 @@ Miscellaneous options .. option:: -b - Issue a warning when comparing :class:`bytes` or :class:`bytearray` with - :class:`str` or :class:`bytes` with :class:`int`. Issue an error when the - option is given twice (:option:`!-bb`). + Issue a warning when converting :class:`bytes` or :class:`bytearray` to + :class:`str` without specifying encoding or comparing :class:`!bytes` or + :class:`!bytearray` with :class:`!str` or :class:`!bytes` with :class:`int`. + Issue an error when the option is given twice (:option:`!-bb`). .. versionchanged:: 3.5 - Affects comparisons of :class:`bytes` with :class:`int`. + Affects also comparisons of :class:`bytes` with :class:`int`. .. option:: -B @@ -288,9 +290,15 @@ Miscellaneous options .. option:: -i - When a script is passed as first argument or the :option:`-c` option is used, - enter interactive mode after executing the script or the command, even when - :data:`sys.stdin` does not appear to be a terminal. The + Enter interactive mode after execution. + + Using the :option:`-i` option will enter interactive mode in any of the following circumstances\: + + * When a script is passed as first argument + * When the :option:`-c` option is used + * When the :option:`-m` option is used + + Interactive mode will start even when :data:`sys.stdin` does not appear to be a terminal. The :envvar:`PYTHONSTARTUP` file is not read. This can be useful to inspect global variables or a stack trace when a script @@ -375,17 +383,19 @@ Miscellaneous options :envvar:`PYTHONHASHSEED` allows you to set a fixed value for the hash seed secret. + .. versionadded:: 3.2.3 + .. versionchanged:: 3.7 The option is no longer ignored. - .. versionadded:: 3.2.3 - .. option:: -s Don't add the :data:`user site-packages directory ` to :data:`sys.path`. + See also :envvar:`PYTHONNOUSERSITE`. + .. seealso:: :pep:`370` -- Per user site-packages directory @@ -437,6 +447,7 @@ Miscellaneous options -Wdefault # Warn once per call location -Werror # Convert to exceptions -Walways # Warn every time + -Wall # Same as -Walways -Wmodule # Warn once per calling module -Wonce # Warn once per Python process -Wignore # Never warn @@ -497,43 +508,73 @@ Miscellaneous options * ``-X faulthandler`` to enable :mod:`faulthandler`. See also :envvar:`PYTHONFAULTHANDLER`. + + .. versionadded:: 3.3 + * ``-X showrefcount`` to output the total reference count and number of used memory blocks when the program finishes or after each statement in the interactive interpreter. This only works on :ref:`debug builds `. + + .. versionadded:: 3.4 + * ``-X tracemalloc`` to start tracing Python memory allocations using the :mod:`tracemalloc` module. By default, only the most recent frame is stored in a traceback of a trace. Use ``-X tracemalloc=NFRAME`` to start tracing with a traceback limit of *NFRAME* frames. See :func:`tracemalloc.start` and :envvar:`PYTHONTRACEMALLOC` for more information. + + .. versionadded:: 3.4 + * ``-X int_max_str_digits`` configures the :ref:`integer string conversion length limitation `. See also :envvar:`PYTHONINTMAXSTRDIGITS`. + + .. versionadded:: 3.11 + * ``-X importtime`` to show how long each import takes. It shows module name, cumulative time (including nested imports) and self time (excluding nested imports). Note that its output may be broken in multi-threaded application. Typical usage is ``python3 -X importtime -c 'import asyncio'``. See also :envvar:`PYTHONPROFILEIMPORTTIME`. + + .. versionadded:: 3.7 + * ``-X dev``: enable :ref:`Python Development Mode `, introducing additional runtime checks that are too expensive to be enabled by - default. + default. See also :envvar:`PYTHONDEVMODE`. + + .. versionadded:: 3.7 + * ``-X utf8`` enables the :ref:`Python UTF-8 Mode `. ``-X utf8=0`` explicitly disables :ref:`Python UTF-8 Mode ` (even when it would otherwise activate automatically). See also :envvar:`PYTHONUTF8`. + + .. versionadded:: 3.7 + * ``-X pycache_prefix=PATH`` enables writing ``.pyc`` files to a parallel tree rooted at the given directory instead of to the code tree. See also :envvar:`PYTHONPYCACHEPREFIX`. + + .. versionadded:: 3.8 + * ``-X warn_default_encoding`` issues a :class:`EncodingWarning` when the locale-specific default encoding is used for opening files. See also :envvar:`PYTHONWARNDEFAULTENCODING`. + + .. versionadded:: 3.10 + * ``-X no_debug_ranges`` disables the inclusion of the tables mapping extra location information (end line, start column offset and end column offset) to every instruction in code objects. This is useful when smaller code objects and pyc files are desired as well as suppressing the extra visual location indicators when the interpreter displays tracebacks. See also :envvar:`PYTHONNODEBUGRANGES`. + + .. versionadded:: 3.11 + * ``-X frozen_modules`` determines whether or not frozen modules are ignored by the import machinery. A value of ``on`` means they get imported and ``off`` means they are ignored. The default is ``on`` @@ -542,17 +583,35 @@ Miscellaneous options Note that the :mod:`!importlib_bootstrap` and :mod:`!importlib_bootstrap_external` frozen modules are always used, even if this flag is set to ``off``. See also :envvar:`PYTHON_FROZEN_MODULES`. + + .. versionadded:: 3.11 + * ``-X perf`` enables support for the Linux ``perf`` profiler. When this option is provided, the ``perf`` profiler will be able to report Python calls. This option is only available on some platforms and will do nothing if is not supported on the current system. The default value is "off". See also :envvar:`PYTHONPERFSUPPORT` and :ref:`perf_profiling`. + + .. versionadded:: 3.12 + + * ``-X perf_jit`` enables support for the Linux ``perf`` profiler with DWARF + support. When this option is provided, the ``perf`` profiler will be able + to report Python calls using DWARF information. This option is only available on + some platforms and will do nothing if is not supported on the current + system. The default value is "off". See also :envvar:`PYTHON_PERF_JIT_SUPPORT` + and :ref:`perf_profiling`. + + .. versionadded:: 3.13 + * :samp:`-X cpu_count={n}` overrides :func:`os.cpu_count`, :func:`os.process_cpu_count`, and :func:`multiprocessing.cpu_count`. *n* must be greater than or equal to 1. This option may be useful for users who need to limit CPU resources of a container system. See also :envvar:`PYTHON_CPU_COUNT`. If *n* is ``default``, nothing is overridden. + + .. versionadded:: 3.13 + * :samp:`-X presite={package.module}` specifies a module that should be imported before the :mod:`site` module is executed and before the :mod:`__main__` module exists. Therefore, the imported module isn't @@ -560,56 +619,26 @@ Miscellaneous options initialization. Python needs to be :ref:`built in debug mode ` for this option to exist. See also :envvar:`PYTHON_PRESITE`. - It also allows passing arbitrary values and retrieving them through the - :data:`sys._xoptions` dictionary. - - .. versionchanged:: 3.2 - The :option:`-X` option was added. + .. versionadded:: 3.13 - .. versionadded:: 3.3 - The ``-X faulthandler`` option. + * :samp:`-X gil={0,1}` forces the GIL to be disabled or enabled, + respectively. Setting to ``0`` is only available in builds configured with + :option:`--disable-gil`. See also :envvar:`PYTHON_GIL` and + :ref:`whatsnew313-free-threaded-cpython`. - .. versionadded:: 3.4 - The ``-X showrefcount`` and ``-X tracemalloc`` options. + .. versionadded:: 3.13 - .. versionadded:: 3.6 - The ``-X showalloccount`` option. - - .. versionadded:: 3.7 - The ``-X importtime``, ``-X dev`` and ``-X utf8`` options. + It also allows passing arbitrary values and retrieving them through the + :data:`sys._xoptions` dictionary. - .. versionadded:: 3.8 - The ``-X pycache_prefix`` option. The ``-X dev`` option now logs - ``close()`` exceptions in :class:`io.IOBase` destructor. + .. versionadded:: 3.2 .. versionchanged:: 3.9 - Using ``-X dev`` option, check *encoding* and *errors* arguments on - string encoding and decoding operations. - - The ``-X showalloccount`` option has been removed. + Removed the ``-X showalloccount`` option. - .. versionadded:: 3.10 - The ``-X warn_default_encoding`` option. + .. versionchanged:: 3.10 Removed the ``-X oldparser`` option. - .. versionadded:: 3.11 - The ``-X no_debug_ranges`` option. - - .. versionadded:: 3.11 - The ``-X frozen_modules`` option. - - .. versionadded:: 3.11 - The ``-X int_max_str_digits`` option. - - .. versionadded:: 3.12 - The ``-X perf`` option. - - .. versionadded:: 3.13 - The ``-X cpu_count`` option. - - .. versionadded:: 3.13 - The ``-X presite`` option. - .. _using-on-controlling-color: Controlling color @@ -621,11 +650,11 @@ behavior can be controlled by setting different environment variables. Setting the environment variable ``TERM`` to ``dumb`` will disable color. -If the environment variable ``FORCE_COLOR`` is set, then color will be +If the |FORCE_COLOR|_ environment variable is set, then color will be enabled regardless of the value of TERM. This is useful on CI systems which aren’t terminals but can still display ANSI escape sequences. -If the environment variable ``NO_COLOR`` is set, Python will disable all color +If the |NO_COLOR|_ environment variable is set, Python will disable all color in the output. This takes precedence over ``FORCE_COLOR``. All these environment variables are used also by other tools to control color @@ -756,6 +785,15 @@ conflict. This variable can also be modified by Python code using :data:`os.environ` to force inspect mode on program termination. + .. audit-event:: cpython.run_stdin "" "" + + .. versionchanged:: 3.12.5 (also 3.11.10, 3.10.15, 3.9.20, and 3.8.20) + Emits audit events. + + .. versionchanged:: 3.13 + Uses PyREPL if possible, in which case :envvar:`PYTHONSTARTUP` is + also executed. Emits audit events. + .. envvar:: PYTHONUNBUFFERED @@ -879,6 +917,7 @@ conflict. PYTHONWARNINGS=default # Warn once per call location PYTHONWARNINGS=error # Convert to exceptions PYTHONWARNINGS=always # Warn every time + PYTHONWARNINGS=all # Same as PYTHONWARNINGS=always PYTHONWARNINGS=module # Warn once per calling module PYTHONWARNINGS=once # Warn once per Python process PYTHONWARNINGS=ignore # Never warn @@ -955,11 +994,11 @@ conflict. * ``pymalloc_debug``: same as ``pymalloc`` but also install debug hooks. * ``mimalloc_debug``: same as ``mimalloc`` but also install debug hooks. + .. versionadded:: 3.6 + .. versionchanged:: 3.7 Added the ``"default"`` allocator. - .. versionadded:: 3.6 - .. envvar:: PYTHONMALLOCSTATS @@ -984,7 +1023,7 @@ conflict. 'surrogatepass' are used. This may also be enabled at runtime with - :func:`sys._enablelegacywindowsfsencoding()`. + :func:`sys._enablelegacywindowsfsencoding`. .. availability:: Windows. @@ -1108,6 +1147,21 @@ conflict. .. versionadded:: 3.12 +.. envvar:: PYTHON_PERF_JIT_SUPPORT + + If this variable is set to a nonzero value, it enables support for + the Linux ``perf`` profiler so Python calls can be detected by it + using DWARF information. + + If set to ``0``, disable Linux ``perf`` profiler support. + + See also the :option:`-X perf_jit <-X>` command-line option + and :ref:`perf_profiling`. + + .. versionadded:: 3.13 + + + .. envvar:: PYTHON_CPU_COUNT If this variable is set to a positive integer, it overrides the return @@ -1139,6 +1193,15 @@ conflict. .. versionadded:: 3.13 +.. envvar:: PYTHON_BASIC_REPL + + If this variable is set to any value, the interpreter will not attempt to + load the Python-based :term:`REPL` that requires :mod:`curses` and + :mod:`readline`, and will instead use the traditional parser-based + :term:`REPL`. + + .. versionadded:: 3.13 + .. envvar:: PYTHON_HISTORY This environment variable can be used to set the location of a @@ -1147,6 +1210,17 @@ conflict. .. versionadded:: 3.13 +.. envvar:: PYTHON_GIL + + If this variable is set to ``1``, the global interpreter lock (GIL) will be + forced on. Setting it to ``0`` forces the GIL off (needs Python configured with + the :option:`--disable-gil` build option). + + See also the :option:`-X gil <-X>` command-line option, which takes + precedence over this variable, and :ref:`whatsnew313-free-threaded-cpython`. + + .. versionadded:: 3.13 + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 26e355a735deaa..629859e36cb654 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -16,8 +16,8 @@ Features and minimum versions required to build CPython: * On Windows, Microsoft Visual Studio 2017 or later is required. -* Support for `IEEE 754 `_ floating - point numbers and `floating point Not-a-Number (NaN) +* Support for `IEEE 754 `_ + floating-point numbers and `floating-point Not-a-Number (NaN) `_. * Support for threads. @@ -29,7 +29,7 @@ Features and minimum versions required to build CPython: * Tcl/Tk 8.5.12 for the :mod:`tkinter` module. -* Autoconf 2.71 and aclocal 1.16.4 are required to regenerate the +* Autoconf 2.72 and aclocal 1.16.5 are required to regenerate the :file:`configure` script. .. versionchanged:: 3.1 @@ -56,7 +56,10 @@ Features and minimum versions required to build CPython: Tcl/Tk version 8.5.12 is now required for the :mod:`tkinter` module. .. versionchanged:: 3.13 - Autoconf 2.71, aclocal 1.16.4 and SQLite 3.15.2 are now required. + Autoconf 2.71, aclocal 1.16.5 and SQLite 3.15.2 are now required. + +.. versionchanged:: 3.14 + Autoconf 2.72 is now required. See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform support". @@ -183,12 +186,6 @@ General Options See :envvar:`PYTHONCOERCECLOCALE` and the :pep:`538`. -.. option:: --without-freelists - - Disable all freelists except the empty tuple singleton. - - .. versionadded:: 3.11 - .. option:: --with-platlibdir=DIRNAME Python library directory name (default is ``lib``). @@ -299,7 +296,24 @@ General Options Defines the ``Py_GIL_DISABLED`` macro and adds ``"t"`` to :data:`sys.abiflags`. - See :pep:`703` "Making the Global Interpreter Lock Optional in CPython". + See :ref:`whatsnew313-free-threaded-cpython` for more detail. + + .. versionadded:: 3.13 + +.. option:: --enable-experimental-jit=[no|yes|yes-off|interpreter] + + Indicate how to integrate the :ref:`JIT compiler `. + + * ``no`` - build the interpreter without the JIT. + * ``yes`` - build the interpreter with the JIT. + * ``yes-off`` - build the interpreter with the JIT but disable it by default. + * ``interpreter`` - build the interpreter without the JIT, but with the tier 2 enabled interpreter. + + By convention, ``--enable-experimental-jit`` is a shorthand for ``--enable-experimental-jit=yes``. + + .. note:: + + When building CPython with JIT enabled, ensure that your system has Python 3.11 or later installed. .. versionadded:: 3.13 @@ -389,6 +403,17 @@ Options for third-party dependencies C compiler and linker flags for ``libffi``, used by :mod:`ctypes` module, overriding ``pkg-config``. +.. option:: LIBMPDEC_CFLAGS +.. option:: LIBMPDEC_LIBS + + C compiler and linker flags for ``libmpdec``, used by :mod:`decimal` module, + overriding ``pkg-config``. + + .. note:: + + These environment variables have no effect unless + :option:`--with-system-libmpdec` is specified. + .. option:: LIBLZMA_CFLAGS .. option:: LIBLZMA_LIBS @@ -416,7 +441,7 @@ Options for third-party dependencies .. option:: PANEL_CFLAGS .. option:: PANEL_LIBS - C compiler and Linker flags for PANEL, overriding ``pkg-config``. + C compiler and linker flags for PANEL, overriding ``pkg-config``. C compiler and linker flags for ``libpanel`` or ``libpanelw``, used by :mod:`curses.panel` module, overriding ``pkg-config``. @@ -436,15 +461,6 @@ Options for third-party dependencies WebAssembly Options ------------------- -.. option:: --with-emscripten-target=[browser|node] - - Set build flavor for ``wasm32-emscripten``. - - * ``browser`` (default): preload minimal stdlib, default MEMFS. - * ``node``: NODERAWFS and pthread support. - - .. versionadded:: 3.11 - .. option:: --enable-wasm-dynamic-linking Turn on dynamic linking support for WASM. @@ -518,6 +534,15 @@ also be used to improve performance. GCC is used: add ``-fno-semantic-interposition`` to the compiler and linker flags. + .. note:: + + During the build, you may encounter compiler warnings about + profile data not being available for some source files. + These warnings are harmless, as only a subset of the code is exercised + during profile data acquisition. + To disable these warnings on Clang, manually suppress them by adding + ``-Wno-profile-instr-unprofiled`` to :envvar:`CFLAGS`. + .. versionadded:: 3.6 .. versionchanged:: 3.10 @@ -595,7 +620,7 @@ also be used to improve performance. .. option:: --without-mimalloc - Disable the fast mimalloc allocator :ref:`mimalloc ` + Disable the fast :ref:`mimalloc ` allocator (enabled by default). See also :envvar:`PYTHONMALLOC` environment variable. @@ -688,7 +713,7 @@ Debug options Effects: * Define the ``Py_TRACE_REFS`` macro. - * Add :func:`!sys.getobjects` function. + * Add :func:`sys.getobjects` function. * Add :envvar:`PYTHONDUMPREFS` environment variable. The :envvar:`PYTHONDUMPREFS` environment variable can be used to dump @@ -784,11 +809,20 @@ Libraries options .. option:: --with-system-libmpdec - Build the ``_decimal`` extension module using an installed ``mpdec`` - library, see the :mod:`decimal` module (default is no). + Build the ``_decimal`` extension module using an installed ``mpdecimal`` + library, see the :mod:`decimal` module (default is yes). .. versionadded:: 3.3 + .. versionchanged:: 3.13 + Default to using the installed ``mpdecimal`` library. + + .. deprecated-removed:: 3.13 3.15 + A copy of the ``mpdecimal`` library sources will no longer be distributed + with Python 3.15. + + .. seealso:: :option:`LIBMPDEC_CFLAGS` and :option:`LIBMPDEC_LIBS`. + .. option:: --with-readline=readline|editline Designate a backend library for the :mod:`readline` module. @@ -878,10 +912,42 @@ Security Options The settings ``python`` and *STRING* also set TLS 1.2 as minimum protocol version. +.. option:: --disable-safety + + Disable compiler options that are `recommended by OpenSSF`_ for security reasons with no performance overhead. + If this option is not enabled, CPython will be built based on safety compiler options with no slow down. + When this option is enabled, CPython will not be built with the compiler options listed below. + + The following compiler options are disabled with :option:`!--disable-safety`: + + * `-fstack-protector-strong`_: Enable run-time checks for stack-based buffer overflows. + * `-Wtrampolines`_: Enable warnings about trampolines that require executable stacks. + + .. _recommended by OpenSSF: https://github.com/ossf/wg-best-practices-os-developers/blob/main/docs/Compiler-Hardening-Guides/Compiler-Options-Hardening-Guide-for-C-and-C++.md + .. _-fstack-protector-strong: https://github.com/ossf/wg-best-practices-os-developers/blob/main/docs/Compiler-Hardening-Guides/Compiler-Options-Hardening-Guide-for-C-and-C++.md#enable-run-time-checks-for-stack-based-buffer-overflows + .. _-Wtrampolines: https://github.com/ossf/wg-best-practices-os-developers/blob/main/docs/Compiler-Hardening-Guides/Compiler-Options-Hardening-Guide-for-C-and-C++.md#enable-warning-about-trampolines-that-require-executable-stacks + + .. versionadded:: 3.14 + +.. option:: --enable-slower-safety + + Enable compiler options that are `recommended by OpenSSF`_ for security reasons which require overhead. + If this option is not enabled, CPython will not be built based on safety compiler options which performance impact. + When this option is enabled, CPython will be built with the compiler options listed below. + + The following compiler options are enabled with :option:`!--enable-slower-safety`: + + * `-D_FORTIFY_SOURCE=3`_: Fortify sources with compile- and run-time checks for unsafe libc usage and buffer overflows. + + .. _-D_FORTIFY_SOURCE=3: https://github.com/ossf/wg-best-practices-os-developers/blob/main/docs/Compiler-Hardening-Guides/Compiler-Options-Hardening-Guide-for-C-and-C++.md#fortify-sources-for-unsafe-libc-usage-and-buffer-overflows + + .. versionadded:: 3.14 + + macOS Options ------------- -See ``Mac/README.rst``. +See :source:`Mac/README.rst`. .. option:: --enable-universalsdk .. option:: --enable-universalsdk=SDKDIR @@ -916,6 +982,31 @@ See ``Mac/README.rst``. Specify the name for the python framework on macOS only valid when :option:`--enable-framework` is set (default: ``Python``). +.. option:: --with-app-store-compliance +.. option:: --with-app-store-compliance=PATCH-FILE + + The Python standard library contains strings that are known to trigger + automated inspection tool errors when submitted for distribution by + the macOS and iOS App Stores. If enabled, this option will apply the list of + patches that are known to correct app store compliance. A custom patch + file can also be specified. This option is disabled by default. + + .. versionadded:: 3.13 + +iOS Options +----------- + +See :source:`iOS/README.rst`. + +.. option:: --enable-framework=INSTALLDIR + + Create a Python.framework. Unlike macOS, the *INSTALLDIR* argument + specifying the installation path is mandatory. + +.. option:: --with-framework-name=FRAMEWORK + + Specify the name for the framework (default: ``Python``). + Cross Compiling Options ----------------------- @@ -991,32 +1082,108 @@ Main build steps Main Makefile targets --------------------- -* ``make``: Build Python with the standard library. -* ``make platform:``: build the ``python`` program, but don't build the - standard library extension modules. -* ``make profile-opt``: build Python using Profile Guided Optimization (PGO). - You can use the configure :option:`--enable-optimizations` option to make - this the default target of the ``make`` command (``make all`` or just - ``make``). - -* ``make test``: Build Python and run the Python test suite with ``--fast-ci`` - option. Variables: - - * ``TESTOPTS``: additional regrtest command line options. - * ``TESTPYTHONOPTS``: additional Python command line options. - * ``TESTTIMEOUT``: timeout in seconds (default: 20 minutes). - -* ``make buildbottest``: Similar to ``make test``, but use ``--slow-ci`` - option and default timeout of 20 minutes, instead of ``--fast-ci`` option - and a default timeout of 10 minutes. - -* ``make install``: Build and install Python. -* ``make regen-all``: Regenerate (almost) all generated files; - ``make regen-stdlib-module-names`` and ``autoconf`` must be run separately - for the remaining generated files. -* ``make clean``: Remove built files. -* ``make distclean``: Same than ``make clean``, but remove also files created - by the configure script. +make +^^^^ + +For the most part, when rebuilding after editing some code or +refreshing your checkout from upstream, all you need to do is execute +``make``, which (per Make's semantics) builds the default target, the +first one defined in the Makefile. By tradition (including in the +CPython project) this is usually the ``all`` target. The +``configure`` script expands an ``autoconf`` variable, +``@DEF_MAKE_ALL_RULE@`` to describe precisely which targets ``make +all`` will build. The three choices are: + +* ``profile-opt`` (configured with ``--enable-optimizations``) +* ``build_wasm`` (chosen if the host platform matches ``wasm32-wasi*`` or + ``wasm32-emscripten``) +* ``build_all`` (configured without explicitly using either of the others) + +Depending on the most recent source file changes, Make will rebuild +any targets (object files and executables) deemed out-of-date, +including running ``configure`` again if necessary. Source/target +dependencies are many and maintained manually however, so Make +sometimes doesn't have all the information necessary to correctly +detect all targets which need to be rebuilt. Depending on which +targets aren't rebuilt, you might experience a number of problems. If +you have build or test problems which you can't otherwise explain, +``make clean && make`` should work around most dependency problems, at +the expense of longer build times. + + +make platform +^^^^^^^^^^^^^ + +Build the ``python`` program, but don't build the standard library +extension modules. This generates a file named ``platform`` which +contains a single line describing the details of the build platform, +e.g., ``macosx-14.3-arm64-3.12`` or ``linux-x86_64-3.13``. + + +make profile-opt +^^^^^^^^^^^^^^^^ + +Build Python using profile-guided optimization (PGO). You can use the +configure :option:`--enable-optimizations` option to make this the +default target of the ``make`` command (``make all`` or just +``make``). + + + +make clean +^^^^^^^^^^ + +Remove built files. + + +make distclean +^^^^^^^^^^^^^^ + +In addition to the work done by ``make clean``, remove files +created by the configure script. ``configure`` will have to be run +before building again. [#]_ + + +make install +^^^^^^^^^^^^ + +Build the ``all`` target and install Python. + + +make test +^^^^^^^^^ + +Build the ``all`` target and run the Python test suite with the +``--fast-ci`` option without GUI tests. Variables: + +* ``TESTOPTS``: additional regrtest command-line options. +* ``TESTPYTHONOPTS``: additional Python command-line options. +* ``TESTTIMEOUT``: timeout in seconds (default: 10 minutes). + + +make ci +^^^^^^^ + +This is similar to ``make test``, but uses the ``-ugui`` to also run GUI tests. + +.. versionadded:: 3.14 + + +make buildbottest +^^^^^^^^^^^^^^^^^ + +This is similar to ``make test``, but uses the ``--slow-ci`` +option and default timeout of 20 minutes, instead of ``--fast-ci`` option. + + +make regen-all +^^^^^^^^^^^^^^ + +Regenerate (almost) all generated files. These include (but are not +limited to) bytecode cases, and parser generator file. +``make regen-stdlib-module-names`` and ``autoconf`` must be run +separately for the remaining `generated files <#generated-files>`_. + C extensions ------------ @@ -1311,3 +1478,14 @@ Linker flags Linker flags used for building the interpreter object files. .. versionadded:: 3.8 + + +.. rubric:: Footnotes + +.. [#] ``git clean -fdx`` is an even more extreme way to "clean" your + checkout. It removes all files not known to Git. + When bug hunting using ``git bisect``, this is + `recommended between probes `_ + to guarantee a completely clean build. **Use with care**, as it + will delete all files not checked into Git, including your + new, uncommitted work. diff --git a/Doc/using/index.rst b/Doc/using/index.rst index e1a3111f36a44f..90fdfc0bec0583 100644 --- a/Doc/using/index.rst +++ b/Doc/using/index.rst @@ -12,10 +12,13 @@ interpreter and things that make working with Python easier. .. toctree:: :numbered: + :maxdepth: 2 cmdline.rst unix.rst configure.rst windows.rst mac.rst + android.rst + ios.rst editors.rst diff --git a/Doc/using/ios.rst b/Doc/using/ios.rst new file mode 100644 index 00000000000000..aa43f75ec35a6c --- /dev/null +++ b/Doc/using/ios.rst @@ -0,0 +1,388 @@ +.. _using-ios: + +=================== +Using Python on iOS +=================== + +:Authors: + Russell Keith-Magee (2024-03) + +Python on iOS is unlike Python on desktop platforms. On a desktop platform, +Python is generally installed as a system resource that can be used by any user +of that computer. Users then interact with Python by running a :program:`python` +executable and entering commands at an interactive prompt, or by running a +Python script. + +On iOS, there is no concept of installing as a system resource. The only unit +of software distribution is an "app". There is also no console where you could +run a :program:`python` executable, or interact with a Python REPL. + +As a result, the only way you can use Python on iOS is in embedded mode - that +is, by writing a native iOS application, and embedding a Python interpreter +using ``libPython``, and invoking Python code using the :ref:`Python embedding +API `. The full Python interpreter, the standard library, and all +your Python code is then packaged as a standalone bundle that can be +distributed via the iOS App Store. + +If you're looking to experiment for the first time with writing an iOS app in +Python, projects such as `BeeWare `__ and `Kivy +`__ will provide a much more approachable user experience. +These projects manage the complexities associated with getting an iOS project +running, so you only need to deal with the Python code itself. + +Python at runtime on iOS +======================== + +iOS version compatibility +------------------------- + +The minimum supported iOS version is specified at compile time, using the +:option:`--host` option to ``configure``. By default, when compiled for iOS, +Python will be compiled with a minimum supported iOS version of 13.0. To use a +different minimum iOS version, provide the version number as part of the +:option:`!--host` argument - for example, +``--host=arm64-apple-ios15.4-simulator`` would compile an ARM64 simulator build +with a deployment target of 15.4. + +Platform identification +----------------------- + +When executing on iOS, ``sys.platform`` will report as ``ios``. This value will +be returned on an iPhone or iPad, regardless of whether the app is running on +the simulator or a physical device. + +Information about the specific runtime environment, including the iOS version, +device model, and whether the device is a simulator, can be obtained using +:func:`platform.ios_ver`. :func:`platform.system` will report ``iOS`` or +``iPadOS``, depending on the device. + +:func:`os.uname` reports kernel-level details; it will report a name of +``Darwin``. + +Standard library availability +----------------------------- + +The Python standard library has some notable omissions and restrictions on +iOS. See the :ref:`API availability guide for iOS ` for +details. + +Binary extension modules +------------------------ + +One notable difference about iOS as a platform is that App Store distribution +imposes hard requirements on the packaging of an application. One of these +requirements governs how binary extension modules are distributed. + +The iOS App Store requires that *all* binary modules in an iOS app must be +dynamic libraries, contained in a framework with appropriate metadata, stored +in the ``Frameworks`` folder of the packaged app. There can be only a single +binary per framework, and there can be no executable binary material outside +the ``Frameworks`` folder. + +This conflicts with the usual Python approach for distributing binaries, which +allows a binary extension module to be loaded from any location on +``sys.path``. To ensure compliance with App Store policies, an iOS project must +post-process any Python packages, converting ``.so`` binary modules into +individual standalone frameworks with appropriate metadata and signing. For +details on how to perform this post-processing, see the guide for :ref:`adding +Python to your project `. + +To help Python discover binaries in their new location, the original ``.so`` +file on ``sys.path`` is replaced with a ``.fwork`` file. This file is a text +file containing the location of the framework binary, relative to the app +bundle. To allow the framework to resolve back to the original location, the +framework must contain a ``.origin`` file that contains the location of the +``.fwork`` file, relative to the app bundle. + +For example, consider the case of an import ``from foo.bar import _whiz``, +where ``_whiz`` is implemented with the binary module +``sources/foo/bar/_whiz.abi3.so``, with ``sources`` being the location +registered on ``sys.path``, relative to the application bundle. This module +*must* be distributed as ``Frameworks/foo.bar._whiz.framework/foo.bar._whiz`` +(creating the framework name from the full import path of the module), with an +``Info.plist`` file in the ``.framework`` directory identifying the binary as a +framework. The ``foo.bar._whiz`` module would be represented in the original +location with a ``sources/foo/bar/_whiz.abi3.fwork`` marker file, containing +the path ``Frameworks/foo.bar._whiz/foo.bar._whiz``. The framework would also +contain ``Frameworks/foo.bar._whiz.framework/foo.bar._whiz.origin``, containing +the path to the ``.fwork`` file. + +When running on iOS, the Python interpreter will install an +:class:`~importlib.machinery.AppleFrameworkLoader` that is able to read and +import ``.fwork`` files. Once imported, the ``__file__`` attribute of the +binary module will report as the location of the ``.fwork`` file. However, the +:class:`~importlib.machinery.ModuleSpec` for the loaded module will report the +``origin`` as the location of the binary in the framework folder. + +Compiler stub binaries +---------------------- + +Xcode doesn't expose explicit compilers for iOS; instead, it uses an ``xcrun`` +script that resolves to a full compiler path (e.g., ``xcrun --sdk iphoneos +clang`` to get the ``clang`` for an iPhone device). However, using this script +poses two problems: + +* The output of ``xcrun`` includes paths that are machine specific, resulting + in a sysconfig module that cannot be shared between users; and + +* It results in ``CC``/``CPP``/``LD``/``AR`` definitions that include spaces. + There is a lot of C ecosystem tooling that assumes that you can split a + command line at the first space to get the path to the compiler executable; + this isn't the case when using ``xcrun``. + +To avoid these problems, Python provided stubs for these tools. These stubs are +shell script wrappers around the underingly ``xcrun`` tools, distributed in a +``bin`` folder distributed alongside the compiled iOS framework. These scripts +are relocatable, and will always resolve to the appropriate local system paths. +By including these scripts in the bin folder that accompanies a framework, the +contents of the ``sysconfig`` module becomes useful for end-users to compile +their own modules. When compiling third-party Python modules for iOS, you +should ensure these stub binaries are on your path. + +Installing Python on iOS +======================== + +Tools for building iOS apps +--------------------------- + +Building for iOS requires the use of Apple's Xcode tooling. It is strongly +recommended that you use the most recent stable release of Xcode. This will +require the use of the most (or second-most) recently released macOS version, +as Apple does not maintain Xcode for older macOS versions. The Xcode Command +Line Tools are not sufficient for iOS development; you need a *full* Xcode +install. + +If you want to run your code on the iOS simulator, you'll also need to install +an iOS Simulator Platform. You should be prompted to select an iOS Simulator +Platform when you first run Xcode. Alternatively, you can add an iOS Simulator +Platform by selecting from the Platforms tab of the Xcode Settings panel. + +.. _adding-ios: + +Adding Python to an iOS project +------------------------------- + +Python can be added to any iOS project, using either Swift or Objective C. The +following examples will use Objective C; if you are using Swift, you may find a +library like `PythonKit `__ to be +helpful. + +To add Python to an iOS Xcode project: + +1. Build or obtain a Python ``XCFramework``. See the instructions in + :source:`iOS/README.rst` (in the CPython source distribution) for details on + how to build a Python ``XCFramework``. At a minimum, you will need a build + that supports ``arm64-apple-ios``, plus one of either + ``arm64-apple-ios-simulator`` or ``x86_64-apple-ios-simulator``. + +2. Drag the ``XCframework`` into your iOS project. In the following + instructions, we'll assume you've dropped the ``XCframework`` into the root + of your project; however, you can use any other location that you want by + adjusting paths as needed. + +3. Drag the ``iOS/Resources/dylib-Info-template.plist`` file into your project, + and ensure it is associated with the app target. + +4. Add your application code as a folder in your Xcode project. In the + following instructions, we'll assume that your user code is in a folder + named ``app`` in the root of your project; you can use any other location by + adjusting paths as needed. Ensure that this folder is associated with your + app target. + +5. Select the app target by selecting the root node of your Xcode project, then + the target name in the sidebar that appears. + +6. In the "General" settings, under "Frameworks, Libraries and Embedded + Content", add ``Python.xcframework``, with "Embed & Sign" selected. + +7. In the "Build Settings" tab, modify the following: + + - Build Options + + * User Script Sandboxing: No + * Enable Testability: Yes + + - Search Paths + + * Framework Search Paths: ``$(PROJECT_DIR)`` + * Header Search Paths: ``"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers"`` + + - Apple Clang - Warnings - All languages + + * Quoted Include In Framework Header: No + +8. Add a build step that copies the Python standard library into your app. In + the "Build Phases" tab, add a new "Run Script" build step *before* the + "Embed Frameworks" step, but *after* the "Copy Bundle Resources" step. Name + the step "Install Target Specific Python Standard Library", disable the + "Based on dependency analysis" checkbox, and set the script content to: + + .. code-block:: bash + + set -e + + mkdir -p "$CODESIGNING_FOLDER_PATH/python/lib" + if [ "$EFFECTIVE_PLATFORM_NAME" = "-iphonesimulator" ]; then + echo "Installing Python modules for iOS Simulator" + rsync -au --delete "$PROJECT_DIR/Python.xcframework/ios-arm64_x86_64-simulator/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" + else + echo "Installing Python modules for iOS Device" + rsync -au --delete "$PROJECT_DIR/Python.xcframework/ios-arm64/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" + fi + + Note that the name of the simulator "slice" in the XCframework may be + different, depending the CPU architectures your ``XCFramework`` supports. + +9. Add a second build step that processes the binary extension modules in the + standard library into "Framework" format. Add a "Run Script" build step + *directly after* the one you added in step 8, named "Prepare Python Binary + Modules". It should also have "Based on dependency analysis" unchecked, with + the following script content: + + .. code-block:: bash + + set -e + + install_dylib () { + INSTALL_BASE=$1 + FULL_EXT=$2 + + # The name of the extension file + EXT=$(basename "$FULL_EXT") + # The location of the extension file, relative to the bundle + RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} + # The path to the extension file, relative to the install base + PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/} + # The full dotted name of the extension module, constructed from the file path. + FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d "." -f 1 | tr "/" "."); + # A bundle identifier; not actually used, but required by Xcode framework packaging + FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr "_" "-") + # The name of the framework folder. + FRAMEWORK_FOLDER="Frameworks/$FULL_MODULE_NAME.framework" + + # If the framework folder doesn't exist, create it. + if [ ! -d "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" ]; then + echo "Creating framework for $RELATIVE_EXT" + mkdir -p "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" + cp "$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" + plutil -replace CFBundleExecutable -string "$FULL_MODULE_NAME" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" + plutil -replace CFBundleIdentifier -string "$FRAMEWORK_BUNDLE_ID" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" + fi + + echo "Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME" + mv "$FULL_EXT" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" + # Create a placeholder .fwork file where the .so was + echo "$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" > ${FULL_EXT%.so}.fwork + # Create a back reference to the .so file location in the framework + echo "${RELATIVE_EXT%.so}.fwork" > "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin" + } + + PYTHON_VER=$(ls -1 "$CODESIGNING_FOLDER_PATH/python/lib") + echo "Install Python $PYTHON_VER standard library extension modules..." + find "$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload" -name "*.so" | while read FULL_EXT; do + install_dylib python/lib/$PYTHON_VER/lib-dynload/ "$FULL_EXT" + done + + # Clean up dylib template + rm -f "$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist" + + echo "Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)..." + find "$CODESIGNING_FOLDER_PATH/Frameworks" -name "*.framework" -exec /usr/bin/codesign --force --sign "$EXPANDED_CODE_SIGN_IDENTITY" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der "{}" \; + +10. Add Objective C code to initialize and use a Python interpreter in embedded + mode. You should ensure that: + + * UTF-8 mode (:c:member:`PyPreConfig.utf8_mode`) is *enabled*; + * Buffered stdio (:c:member:`PyConfig.buffered_stdio`) is *disabled*; + * Writing bytecode (:c:member:`PyConfig.write_bytecode`) is *disabled*; + * Signal handlers (:c:member:`PyConfig.install_signal_handlers`) are *enabled*; + * System logging (:c:member:`PyConfig.use_system_logger`) is *enabled* + (optional, but strongly recommended); + * ``PYTHONHOME`` for the interpreter is configured to point at the + ``python`` subfolder of your app's bundle; and + * The ``PYTHONPATH`` for the interpreter includes: + + - the ``python/lib/python3.X`` subfolder of your app's bundle, + - the ``python/lib/python3.X/lib-dynload`` subfolder of your app's bundle, and + - the ``app`` subfolder of your app's bundle + + Your app's bundle location can be determined using ``[[NSBundle mainBundle] + resourcePath]``. + +Steps 8, 9 and 10 of these instructions assume that you have a single folder of +pure Python application code, named ``app``. If you have third-party binary +modules in your app, some additional steps will be required: + +* You need to ensure that any folders containing third-party binaries are + either associated with the app target, or copied in as part of step 8. Step 8 + should also purge any binaries that are not appropriate for the platform a + specific build is targeting (i.e., delete any device binaries if you're + building an app targeting the simulator). + +* Any folders that contain third-party binaries must be processed into + framework form by step 9. The invocation of ``install_dylib`` that processes + the ``lib-dynload`` folder can be copied and adapted for this purpose. + +* If you're using a separate folder for third-party packages, ensure that folder + is included as part of the ``PYTHONPATH`` configuration in step 10. + +Testing a Python package +------------------------ + +The CPython source tree contains :source:`a testbed project ` that +is used to run the CPython test suite on the iOS simulator. This testbed can also +be used as a testbed project for running your Python library's test suite on iOS. + +After building or obtaining an iOS XCFramework (See :source:`iOS/README.rst` +for details), create a clone of the Python iOS testbed project by running: + +.. code-block:: bash + + $ python iOS/testbed clone --framework --app --app app-testbed + +You will need to modify the ``iOS/testbed`` reference to point to that +directory in the CPython source tree; any folders specified with the ``--app`` +flag will be copied into the cloned testbed project. The resulting testbed will +be created in the ``app-testbed`` folder. In this example, the ``module1`` and +``module2`` would be importable modules at runtime. If your project has +additional dependencies, they can be installed into the +``app-testbed/iOSTestbed/app_packages`` folder (using ``pip install --target +app-testbed/iOSTestbed/app_packages`` or similar). + +You can then use the ``app-testbed`` folder to run the test suite for your app, +For example, if ``module1.tests`` was the entry point to your test suite, you +could run: + +.. code-block:: bash + + $ python app-testbed run -- module1.tests + +This is the equivalent of running ``python -m module1.tests`` on a desktop +Python build. Any arguments after the ``--`` will be passed to the testbed as +if they were arguments to ``python -m`` on a desktop machine. + +You can also open the testbed project in Xcode by running: + +.. code-block:: bash + + $ open app-testbed/iOSTestbed.xcodeproj + +This will allow you to use the full Xcode suite of tools for debugging. + +App Store Compliance +==================== + +The only mechanism for distributing apps to third-party iOS devices is to +submit the app to the iOS App Store; apps submitted for distribution must pass +Apple's app review process. This process includes a set of automated validation +rules that inspect the submitted application bundle for problematic code. + +The Python standard library contains some code that is known to violate these +automated rules. While these violations appear to be false positives, Apple's +review rules cannot be challenged; so, it is necessary to modify the Python +standard library for an app to pass App Store review. + +The Python source tree contains +:source:`a patch file ` that will remove +all code that is known to cause issues with the App Store review process. This +patch is applied automatically when building for iOS. diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index e99993238895f9..4b6c884f3d4f25 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -2,140 +2,223 @@ .. _using-on-mac: ********************* -Using Python on a Mac +Using Python on macOS ********************* -:Author: Bob Savage +.. sectionauthor:: Bob Savage +.. sectionauthor:: Ned Deily +This document aims to give an overview of macOS-specific behavior you should +know about to get started with Python on Mac computers. +Python on a Mac running macOS is very similar to Python on other Unix-derived platforms, +but there are some differences in installation and some features. -Python on a Mac running macOS is in principle very similar to Python on -any other Unix platform, but there are a number of additional features such as -the integrated development environment (IDE) and the Package Manager that are -worth pointing out. +There are various ways to obtain and install Python for macOS. +Pre-built versions of the most recent versions of Python are available +from a number of distributors. Much of this document describes use of +the Pythons provided by the CPython release team for download from +the `python.org website `_. See +:ref:`alternative_bundles` for some other options. +.. |usemac_x_dot_y| replace:: 3.13 +.. |usemac_python_x_dot_y_literal| replace:: ``python3.13`` +.. |usemac_python_x_dot_y_t_literal| replace:: ``python3.13t`` +.. |usemac_python_x_dot_y_t_literal_config| replace:: ``python3.13t-config`` +.. |usemac_applications_folder_name| replace:: ``Python 3.13`` +.. |usemac_applications_folder_version| replace:: ``/Applications/Python 3.13/`` .. _getting-osx: .. _getting-and-installing-macpython: -Getting and Installing Python -============================= +Using Python for macOS from ``python.org`` +========================================== -macOS used to come with Python 2.7 pre-installed between versions -10.8 and `12.3 `_. -You are invited to install the most recent version of Python 3 from the `Python -website `__. -A current "universal2 binary" build of Python, which runs natively on the Mac's -new Apple Silicon and legacy Intel processors, is available there. +Installation steps +------------------ -What you get after installing is a number of things: +For `current Python versions `_ +(other than those in ``security`` status), the release team produces a +**Python for macOS** installer package for each new release. +A list of available installers +is available `here `_. +We recommend using the most recent supported Python version where possible. +Current installers provide a +`universal2 binary `_ build +of Python which runs natively on all Macs (Apple Silicon and Intel) that are +supported by a wide range of macOS versions, +currently typically from at least **macOS 10.13 High Sierra** on. -* A |python_version_literal| folder in your :file:`Applications` folder. In here - you find IDLE, the development environment that is a standard part of official +The downloaded file is a standard macOS installer package file (``.pkg``). +File integrity information (checksum, size, sigstore signature, etc) for each file is included +on the release download page. Installer packages and their contents are signed and notarized +with ``Python Software Foundation`` Apple Developer ID certificates +to meet `macOS Gatekeeper requirements `_. + +For a default installation, double-click on the downloaded installer package file. +This should launch the standard macOS Installer app and display the first of several +installer windows steps. + +.. image:: mac_installer_01_introduction.png + +Clicking on the **Continue** button brings up the **Read Me** for this installer. +Besides other important information, the **Read Me** documents which Python version is +going to be installed and on what versions of macOS it is supported. You may need +to scroll through to read the whole file. By default, this **Read Me** will also be +installed in |usemac_applications_folder_version| and available to read anytime. + +.. image:: mac_installer_02_readme.png + +Clicking on **Continue** proceeds to display the license for Python and for +other included software. You will then need to **Agree** to the license terms +before proceeding to the next step. This license file will also be installed +and available to be read later. + +.. image:: mac_installer_03_license.png + +After the license terms are accepted, the next step is the **Installation Type** +display. For most uses, the standard set of installation operations is appropriate. + +.. image:: mac_installer_04_installation_type.png + +By pressing the **Customize** button, you can choose to omit or select certain package +components of the installer. Click on each package name to see a description of +what it installs. +To also install support for the optional experimental free-threaded feature, +see :ref:`install-freethreaded-macos`. + +.. image:: mac_installer_05_custom_install.png + +In either case, clicking **Install** will begin the install process by asking +permission to install new software. A macOS user name with ``Administrator`` privilege +is needed as the installed Python will be available to all users of the Mac. + +When the installation is complete, the **Summary** window will appear. + +.. image:: mac_installer_06_summary.png + +Double-click on the :command:`Install Certificates.command` +icon or file in the |usemac_applications_folder_version| window to complete the +installation. + +.. image:: mac_installer_07_applications.png + +This will open a temporary :program:`Terminal` shell window that +will use the new Python to download and install SSL root certificates +for its use. + +.. image:: mac_installer_08_install_certificates.png + +If ``Successfully installed certifi`` and ``update complete`` appears +in the terminal window, the installation is complete. +Close this terminal window and the installer window. + +A default install will include: + +* A |usemac_applications_folder_name| folder in your :file:`Applications` folder. In here + you find :program:`IDLE`, the development environment that is a standard part of official Python distributions; and :program:`Python Launcher`, which handles double-clicking Python - scripts from the Finder. + scripts from the macOS `Finder `_. * A framework :file:`/Library/Frameworks/Python.framework`, which includes the Python executable and libraries. The installer adds this location to your shell - path. To uninstall Python, you can remove these three things. A - symlink to the Python executable is placed in :file:`/usr/local/bin/`. + path. To uninstall Python, you can remove these three things. + Symlinks to the Python executable are placed in :file:`/usr/local/bin/`. .. note:: - On macOS 10.8-12.3, the Apple-provided build of Python is installed in - :file:`/System/Library/Frameworks/Python.framework` and :file:`/usr/bin/python`, - respectively. You should never modify or delete these, as they are - Apple-controlled and are used by Apple- or third-party software. Remember that - if you choose to install a newer Python version from python.org, you will have - two different but functional Python installations on your computer, so it will - be important that your paths and usages are consistent with what you want to do. - -IDLE includes a Help menu that allows you to access Python documentation. If you -are completely new to Python you should start reading the tutorial introduction -in that document. - -If you are familiar with Python on other Unix platforms you should read the -section on running Python scripts from the Unix shell. - + Recent versions of macOS include a :command:`python3` command in :file:`/usr/bin/python3` + that links to a usually older and incomplete version of Python provided by and for use by + the Apple development tools, :program:`Xcode` or the :program:`Command Line Tools for Xcode`. + You should never modify or attempt to delete this installation, as it is + Apple-controlled and is used by Apple-provided or third-party software. If + you choose to install a newer Python version from ``python.org``, you will have + two different but functional Python installations on your computer that + can co-exist. The default installer options should ensure that its :command:`python3` + will be used instead of the system :command:`python3`. How to run a Python script -------------------------- -Your best way to get started with Python on macOS is through the IDLE -integrated development environment; see section :ref:`ide` and use the Help menu -when the IDE is running. +There are two ways to invoke the Python interpreter. +If you are familiar with using a Unix shell in a terminal +window, you can invoke |usemac_python_x_dot_y_literal| or ``python3`` optionally +followed by one or more command line options (described in :ref:`using-on-general`). +The Python tutorial also has a useful section on +:ref:`using Python interactively from a shell `. + +You can also invoke the interpreter through an integrated +development environment. +:ref:`idle` is a basic editor and interpreter environment +which is included with the standard distribution of Python. +:program:`IDLE` includes a Help menu that allows you to access Python documentation. If you +are completely new to Python, you can read the tutorial introduction +in that document. + +There are many other editors and IDEs available, see :ref:`editors` +for more information. -If you want to run Python scripts from the Terminal window command line or from -the Finder you first need an editor to create your script. macOS comes with a -number of standard Unix command line editors, :program:`vim` -:program:`nano` among them. If you want a more Mac-like editor, -:program:`BBEdit` from Bare Bones Software (see -https://www.barebones.com/products/bbedit/index.html) are good choices, as is -:program:`TextMate` (see https://macromates.com). Other editors include -:program:`MacVim` (https://macvim.org) and :program:`Aquamacs` -(https://aquamacs.org). +To run a Python script file from the terminal window, you can +invoke the interpreter with the name of the script file: -To run your script from the Terminal window you must make sure that -:file:`/usr/local/bin` is in your shell search path. + |usemac_python_x_dot_y_literal| ``myscript.py`` -To run your script from the Finder you have two options: +To run your script from the Finder, you can either: * Drag it to :program:`Python Launcher`. * Select :program:`Python Launcher` as the default application to open your - script (or any ``.py`` script) through the finder Info window and double-click it. + script (or any ``.py`` script) through the Finder Info window and double-click it. :program:`Python Launcher` has various preferences to control how your script is launched. Option-dragging allows you to change these for one invocation, or use - its Preferences menu to change things globally. - - -.. _osx-gui-scripts: - -Running scripts with a GUI --------------------------- + its ``Preferences`` menu to change things globally. -With older versions of Python, there is one macOS quirk that you need to be -aware of: programs that talk to the Aqua window manager (in other words, -anything that has a GUI) need to be run in a special way. Use :program:`pythonw` -instead of :program:`python` to start such scripts. +Be aware that running the script directly from the macOS Finder might +produce different results than when running from a terminal window as +the script will not be run in the usual shell environment including +any setting of environment variables in shell profiles. +And, as with any other script or program, +be certain of what you are about to run. -With Python 3.9, you can use either :program:`python` or :program:`pythonw`. +.. _alternative_bundles: +Alternative Distributions +========================= -Configuration -------------- +Besides the standard ``python.org`` for macOS installer, there are third-party +distributions for macOS that may include additional functionality. +Some popular distributions and their key features: -Python on macOS honors all standard Unix environment variables such as -:envvar:`PYTHONPATH`, but setting these variables for programs started from the -Finder is non-standard as the Finder does not read your :file:`.profile` or -:file:`.cshrc` at startup. You need to create a file -:file:`~/.MacOSX/environment.plist`. See Apple's -`Technical Q&A QA1067 `__ -for details. +`ActivePython `_ + Installer with multi-platform compatibility, documentation -For more information on installation Python packages, see section -:ref:`mac-package-manager`. +`Anaconda `_ + Popular scientific modules (such as numpy, scipy, and pandas) and the + ``conda`` package manager. +`Homebrew `_ + Package manager for macOS including multiple versions of Python and many + third-party Python-based packages (including numpy, scipy, and pandas). -.. _ide: - -The IDE -======= - -Python ships with the standard IDLE development environment. A good -introduction to using IDLE can be found at -https://www.hashcollision.org/hkn/python/idle_intro/index.html. +`MacPorts `_ + Another package manager for macOS including multiple versions of Python and many + third-party Python-based packages. May include pre-built versions of Python and + many packages for older versions of macOS. +Note that distributions might not include the latest versions of Python or +other libraries, and are not maintained or supported by the core Python team. .. _mac-package-manager: Installing Additional Python Packages ===================================== -This section has moved to the `Python Packaging User Guide`_. +Refer to the `Python Packaging User Guide`_ for more information. .. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/installing-packages/ +.. _osx-gui-scripts: + .. _gui-programming-on-the-mac: GUI Programming @@ -143,45 +226,249 @@ GUI Programming There are several options for building GUI applications on the Mac with Python. -*PyObjC* is a Python binding to Apple's Objective-C/Cocoa framework, which is -the foundation of most modern Mac development. Information on PyObjC is -available from https://pypi.org/project/pyobjc/. - The standard Python GUI toolkit is :mod:`tkinter`, based on the cross-platform -Tk toolkit (https://www.tcl.tk). An Aqua-native version of Tk is bundled with -macOS by Apple, and the latest version can be downloaded and installed from -https://www.activestate.com; it can also be built from source. +Tk toolkit (https://www.tcl.tk). A macOS-native version of Tk is included with +the installer. + +*PyObjC* is a Python binding to Apple's Objective-C/Cocoa framework. +Information on PyObjC is available from :pypi:`pyobjc`. + +A number of alternative macOS GUI toolkits are available including: + +* `PySide `_: Official Python bindings to the + `Qt GUI toolkit `_. -*wxPython* is another popular cross-platform GUI toolkit that runs natively on -macOS. Packages and documentation are available from https://www.wxpython.org. +* `PyQt `_: Alternative + Python bindings to Qt. -*PyQt* is another popular cross-platform GUI toolkit that runs natively on -macOS. More information can be found at -https://riverbankcomputing.com/software/pyqt/intro. +* `Kivy `_: A cross-platform GUI toolkit that supports + desktop and mobile platforms. -*PySide* is another cross-platform Qt-based toolkit. More information at -https://www.qt.io/qt-for-python. +* `Toga `_: Part of the `BeeWare Project + `_; supports desktop, mobile, web and console apps. +* `wxPython `_: A cross-platform toolkit that + supports desktop operating systems. + + +Advanced Topics +=============== + +.. _install-freethreaded-macos: + +Installing Free-threaded Binaries +--------------------------------- + +.. versionadded:: 3.13 (Experimental) + +.. note:: + + Everything described in this section is considered experimental, + and should be expected to change in future releases. + +The ``python.org`` :ref:`Python for macOS ` +installer package can optionally install an additional build of +Python |usemac_x_dot_y| that supports :pep:`703`, the experimental free-threading feature +(running with the :term:`global interpreter lock` disabled). +Check the release page on ``python.org`` for possible updated information. + +Because this feature is still considered experimental, the support for it +is not installed by default. It is packaged as a separate install option, +available by clicking the **Customize** button on the **Installation Type** +step of the installer as described above. + +.. image:: mac_installer_09_custom_install_free_threaded.png + +If the box next to the **Free-threaded Python** package name is checked, +a separate :file:`PythonT.framework` will also be installed +alongside the normal :file:`Python.framework` in :file:`/Library/Frameworks`. +This configuration allows a free-threaded Python |usemac_x_dot_y| build to co-exist +on your system with a traditional (GIL only) Python |usemac_x_dot_y| build with +minimal risk while installing or testing. This installation layout is itself +experimental and is subject to change in future releases. + +Known cautions and limitations: + +- The **UNIX command-line tools** package, which is selected by default, + will install links in :file:`/usr/local/bin` for |usemac_python_x_dot_y_t_literal|, + the free-threaded interpreter, and |usemac_python_x_dot_y_t_literal_config|, + a configuration utility which may be useful for package builders. + Since :file:`/usr/local/bin` is typically included in your shell ``PATH``, + in most cases no changes to your ``PATH`` environment variables should + be needed to use |usemac_python_x_dot_y_t_literal|. + +- For this release, the **Shell profile updater** package and the + :file:`Update Shell Profile.command` in |usemac_applications_folder_version| + do not support the free-threaded package. + +- The free-threaded build and the traditional build have separate search + paths and separate :file:`site-packages` directories so, by default, + if you need a package available in both builds, it may need to be installed in both. + The free-threaded package will install a separate instance of :program:`pip` for use + with |usemac_python_x_dot_y_t_literal|. + + - To install a package using :command:`pip` without a :command:`venv`: + + |usemac_python_x_dot_y_t_literal| ``-m pip install `` + +- When working with multiple Python environments, it is usually safest and easiest + to :ref:`create and use virtual environments `. + This can avoid possible command name conflicts and confusion about which Python is in use: + + |usemac_python_x_dot_y_t_literal| ``-m venv `` + + then :command:`activate`. + +- To run a free-threaded version of IDLE: + + |usemac_python_x_dot_y_t_literal| ``-m idlelib`` + +- The interpreters in both builds respond to the same + :ref:`PYTHON environment variables ` + which may have unexpected results, for example, if you have ``PYTHONPATH`` + set in a shell profile. If necessary, there are + :ref:`command line options ` like ``-E`` + to ignore these environment variables. + +- The free-threaded build links to the third-party shared libraries, + such as ``OpenSSL`` and ``Tk``, installed in the traditional framework. + This means that both builds also share one set of trust certificates + as installed by the :command:`Install Certificates.command` script, + thus it only needs to be run once. + +- If you cannot depend on the link in ``/usr/local/bin`` pointing to the + ``python.org`` free-threaded |usemac_python_x_dot_y_t_literal| (for example, if you want + to install your own version there or some other distribution does), + you can explicitly set your shell ``PATH`` environment variable to + include the ``PythonT`` framework ``bin`` directory: + + .. code-block:: sh + + export PATH="/Library/Frameworks/PythonT.framework/Versions/3.13/bin":"$PATH" + + The traditional framework installation by default does something similar, + except for :file:`Python.framework`. Be aware that having both framework ``bin`` + directories in ``PATH`` can lead to confusion if there are duplicate names + like ``python3.13`` in both; which one is actually used depends on the order + they appear in ``PATH``. The ``which python3.x`` or ``which python3.xt`` + commands can show which path is being used. Using virtual environments + can help avoid such ambiguities. Another option might be to create + a shell :command:`alias` to the desired interpreter, like: + + .. code-block:: sh + + alias py3.13="/Library/Frameworks/Python.framework/Versions/3.13/bin/python3.13" + alias py3.13t="/Library/Frameworks/PythonT.framework/Versions/3.13/bin/python3.13t" + +Installing using the command line +--------------------------------- + +If you want to use automation to install the ``python.org`` installer package +(rather than by using the familiar macOS :program:`Installer` GUI app), +the macOS command line :command:`installer` utility lets you select non-default +options, too. If you are not familiar with :command:`installer`, it can be +somewhat cryptic (see :command:`man installer` for more information). +As an example, the following shell snippet shows one way to do it, +using the ``3.13.0b2`` release and selecting the free-threaded interpreter +option: + +.. code-block:: sh + + RELEASE="python-3.13.0b2-macos11.pkg" + + # download installer pkg + curl -O https://www.python.org/ftp/python/3.13.0/${RELEASE} + + # create installer choicechanges to customize the install: + # enable the PythonTFramework-3.13 package + # while accepting the other defaults (install all other packages) + cat > ./choicechanges.plist < + + + + + attributeSetting + 1 + choiceAttribute + selected + choiceIdentifier + org.python.Python.PythonTFramework-3.13 + + + + EOF + + sudo installer -pkg ./${RELEASE} -applyChoiceChangesXML ./choicechanges.plist -target / + + +You can then test that both installer builds are now available with something like: + +.. code-block:: console + + $ # test that the free-threaded interpreter was installed if the Unix Command Tools package was enabled + $ /usr/local/bin/python3.13t -VV + Python 3.13.0b2 experimental free-threading build (v3.13.0b2:3a83b172af, Jun 5 2024, 12:57:31) [Clang 15.0.0 (clang-1500.3.9.4)] + $ # and the traditional interpreter + $ /usr/local/bin/python3.13 -VV + Python 3.13.0b2 (v3.13.0b2:3a83b172af, Jun 5 2024, 12:50:24) [Clang 15.0.0 (clang-1500.3.9.4)] + $ # test that they are also available without the prefix if /usr/local/bin is on $PATH + $ python3.13t -VV + Python 3.13.0b2 experimental free-threading build (v3.13.0b2:3a83b172af, Jun 5 2024, 12:57:31) [Clang 15.0.0 (clang-1500.3.9.4)] + $ python3.13 -VV + Python 3.13.0b2 (v3.13.0b2:3a83b172af, Jun 5 2024, 12:50:24) [Clang 15.0.0 (clang-1500.3.9.4)] + +.. note:: + + Current ``python.org`` installers only install to fixed locations like + :file:`/Library/Frameworks/`, :file:`/Applications`, and :file:`/usr/local/bin`. + You cannot use the :command:`installer` ``-domain`` option to install to + other locations. .. _distributing-python-applications-on-the-mac: Distributing Python Applications -================================ +-------------------------------- -The standard tool for deploying standalone Python applications on the Mac is -:program:`py2app`. More information on installing and using :program:`py2app` -can be found at https://pypi.org/project/py2app/. +A range of tools exist for converting your Python code into a standalone +distributable application: +* :pypi:`py2app`: Supports creating macOS ``.app`` + bundles from a Python project. -Other Resources -=============== +* `Briefcase `_: Part of the `BeeWare Project + `_; a cross-platform packaging tool that supports + creation of ``.app`` bundles on macOS, as well as managing signing and + notarization. + +* `PyInstaller `_: A cross-platform packaging tool that creates + a single file or folder as a distributable artifact. + +App Store Compliance +-------------------- -The Pythonmac-SIG mailing list is an excellent support resource for Python users -and developers on the Mac: +Apps submitted for distribution through the macOS App Store must pass Apple's +app review process. This process includes a set of automated validation rules +that inspect the submitted application bundle for problematic code. -https://www.python.org/community/sigs/current/pythonmac-sig/ +The Python standard library contains some code that is known to violate these +automated rules. While these violations appear to be false positives, Apple's +review rules cannot be challenged. Therefore, it is necessary to modify the +Python standard library for an app to pass App Store review. -Another useful resource is the MacPython wiki: +The Python source tree contains +:source:`a patch file ` that will remove +all code that is known to cause issues with the App Store review process. This +patch is applied automatically when CPython is configured with the +:option:`--with-app-store-compliance` option. -https://wiki.python.org/moin/MacPython +This patch is not normally required to use CPython on a Mac; nor is it required +if you are distributing an app *outside* the macOS App Store. It is *only* +required if you are using the macOS App Store as a distribution channel. + +Other Resources +=============== +The `python.org Help page `_ has links to many useful resources. +The `Pythonmac-SIG mailing list `_ +is another support resource specifically for Python users and developers on the Mac. diff --git a/Doc/using/mac_installer_01_introduction.png b/Doc/using/mac_installer_01_introduction.png new file mode 100644 index 00000000000000..1999f3a3759093 Binary files /dev/null and b/Doc/using/mac_installer_01_introduction.png differ diff --git a/Doc/using/mac_installer_02_readme.png b/Doc/using/mac_installer_02_readme.png new file mode 100644 index 00000000000000..a36efaf7d50fd6 Binary files /dev/null and b/Doc/using/mac_installer_02_readme.png differ diff --git a/Doc/using/mac_installer_03_license.png b/Doc/using/mac_installer_03_license.png new file mode 100644 index 00000000000000..598c22a13d9e62 Binary files /dev/null and b/Doc/using/mac_installer_03_license.png differ diff --git a/Doc/using/mac_installer_04_installation_type.png b/Doc/using/mac_installer_04_installation_type.png new file mode 100644 index 00000000000000..9498fd06240a4e Binary files /dev/null and b/Doc/using/mac_installer_04_installation_type.png differ diff --git a/Doc/using/mac_installer_05_custom_install.png b/Doc/using/mac_installer_05_custom_install.png new file mode 100644 index 00000000000000..3a201d2f44655a Binary files /dev/null and b/Doc/using/mac_installer_05_custom_install.png differ diff --git a/Doc/using/mac_installer_06_summary.png b/Doc/using/mac_installer_06_summary.png new file mode 100644 index 00000000000000..1af6eee2c668cd Binary files /dev/null and b/Doc/using/mac_installer_06_summary.png differ diff --git a/Doc/using/mac_installer_07_applications.png b/Doc/using/mac_installer_07_applications.png new file mode 100644 index 00000000000000..940219cad6f61c Binary files /dev/null and b/Doc/using/mac_installer_07_applications.png differ diff --git a/Doc/using/mac_installer_08_install_certificates.png b/Doc/using/mac_installer_08_install_certificates.png new file mode 100644 index 00000000000000..c125eeb18aa0c1 Binary files /dev/null and b/Doc/using/mac_installer_08_install_certificates.png differ diff --git a/Doc/using/mac_installer_09_custom_install_free_threaded.png b/Doc/using/mac_installer_09_custom_install_free_threaded.png new file mode 100644 index 00000000000000..0f69c55eddb228 Binary files /dev/null and b/Doc/using/mac_installer_09_custom_install_free_threaded.png differ diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index 58838c28e6eb86..a2bcdab09a9282 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -17,12 +17,12 @@ On Linux Python comes preinstalled on most Linux distributions, and is available as a package on all others. However there are certain features you might want to use -that are not available on your distro's package. You can easily compile the +that are not available on your distro's package. You can compile the latest version of Python from source. -In the event that Python doesn't come preinstalled and isn't in the repositories as -well, you can easily make packages for your own distro. Have a look at the -following links: +In the event that the latest version of Python doesn't come preinstalled and isn't +in the repositories as well, you can make packages for your own distro. Have a +look at the following links: .. seealso:: diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc deleted file mode 100644 index 354eb1541ceac2..00000000000000 --- a/Doc/using/venv-create.inc +++ /dev/null @@ -1,121 +0,0 @@ -Creation of :ref:`virtual environments ` is done by executing the -command ``venv``:: - - python -m venv /path/to/new/virtual/environment - -Running this command creates the target directory (creating any parent -directories that don't exist already) and places a ``pyvenv.cfg`` file in it -with a ``home`` key pointing to the Python installation from which the command -was run (a common name for the target directory is ``.venv``). It also creates -a ``bin`` (or ``Scripts`` on Windows) subdirectory containing a copy/symlink -of the Python binary/binaries (as appropriate for the platform or arguments -used at environment creation time). It also creates an (initially empty) -``lib/pythonX.Y/site-packages`` subdirectory (on Windows, this is -``Lib\site-packages``). If an existing directory is specified, it will be -re-used. - -.. versionchanged:: 3.5 - The use of ``venv`` is now recommended for creating virtual environments. - -.. deprecated:: 3.6 - ``pyvenv`` was the recommended tool for creating virtual environments for - Python 3.3 and 3.4, and is - :ref:`deprecated in Python 3.6 `. - -.. highlight:: none - -On Windows, invoke the ``venv`` command as follows:: - - c:\>Python35\python -m venv c:\path\to\myenv - -Alternatively, if you configured the ``PATH`` and ``PATHEXT`` variables for -your :ref:`Python installation `:: - - c:\>python -m venv c:\path\to\myenv - -The command, if run with ``-h``, will show the available options:: - - usage: venv [-h] [--system-site-packages] [--symlinks | --copies] [--clear] - [--upgrade] [--without-pip] [--prompt PROMPT] [--upgrade-deps] - [--without-scm-ignore-file] - ENV_DIR [ENV_DIR ...] - - Creates virtual Python environments in one or more target directories. - - positional arguments: - ENV_DIR A directory to create the environment in. - - options: - -h, --help show this help message and exit - --system-site-packages - Give the virtual environment access to the system - site-packages dir. - --symlinks Try to use symlinks rather than copies, when - symlinks are not the default for the platform. - --copies Try to use copies rather than symlinks, even when - symlinks are the default for the platform. - --clear Delete the contents of the environment directory if - it already exists, before environment creation. - --upgrade Upgrade the environment directory to use this - version of Python, assuming Python has been upgraded - in-place. - --without-pip Skips installing or upgrading pip in the virtual - environment (pip is bootstrapped by default) - --prompt PROMPT Provides an alternative prompt prefix for this - environment. - --upgrade-deps Upgrade core dependencies (pip) to the latest - version in PyPI - --without-scm-ignore-file - Skips adding the default SCM ignore file to the - environment directory (the default is a .gitignore - file). - - Once an environment has been created, you may wish to activate it, e.g. by - sourcing an activate script in its bin directory. - -.. versionchanged:: 3.13 - - ``--without-scm-ignore-file`` was added along with creating an ignore file - for ``git`` by default. - -.. versionchanged:: 3.12 - - ``setuptools`` is no longer a core venv dependency. - -.. versionchanged:: 3.9 - Add ``--upgrade-deps`` option to upgrade pip + setuptools to the latest on PyPI - -.. versionchanged:: 3.4 - Installs pip by default, added the ``--without-pip`` and ``--copies`` - options - -.. versionchanged:: 3.4 - In earlier versions, if the target directory already existed, an error was - raised, unless the ``--clear`` or ``--upgrade`` option was provided. - -.. note:: - While symlinks are supported on Windows, they are not recommended. Of - particular note is that double-clicking ``python.exe`` in File Explorer - will resolve the symlink eagerly and ignore the virtual environment. - -.. note:: - On Microsoft Windows, it may be required to enable the ``Activate.ps1`` - script by setting the execution policy for the user. You can do this by - issuing the following PowerShell command: - - PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser - - See `About Execution Policies - `_ - for more information. - -The created ``pyvenv.cfg`` file also includes the -``include-system-site-packages`` key, set to ``true`` if ``venv`` is -run with the ``--system-site-packages`` option, ``false`` otherwise. - -Unless the ``--without-pip`` option is given, :mod:`ensurepip` will be -invoked to bootstrap ``pip`` into the virtual environment. - -Multiple paths can be given to ``venv``, in which case an identical virtual -environment will be created, according to the given options, at each provided -path. diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index cc4db34b04d900..1a6322d72341ff 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -23,8 +23,9 @@ available for application-local distributions. As specified in :pep:`11`, a Python release only supports a Windows platform while Microsoft considers the platform under extended support. This means that -Python |version| supports Windows 8.1 and newer. If you require Windows 7 -support, please install Python 3.8. +Python |version| supports Windows 10 and newer. If you require Windows 7 +support, please install Python 3.8. If you require Windows 8.1 support, +please install Python 3.12. There are a number of different installers available for Windows, each with certain benefits and downsides. @@ -434,7 +435,7 @@ When writing to the Windows Registry, the following behaviors exist: For more detail on the technical basis for these limitations, please consult Microsoft's documentation on packaged full-trust apps, currently available at `docs.microsoft.com/en-us/windows/msix/desktop/desktop-to-uwp-behind-the-scenes -`_ +`_ .. _windows-nuget: @@ -535,7 +536,7 @@ dependents, such as Idle), pip and the Python documentation are not included. .. note:: The embedded distribution does not include the `Microsoft C Runtime - `_ and it is + `_ and it is the responsibility of the application installer to provide this. The runtime may have already been installed on a user's system previously or automatically via Windows Update, and can be detected by finding @@ -608,7 +609,7 @@ key features: Popular scientific modules (such as numpy, scipy and pandas) and the ``conda`` package manager. -`Enthought Deployment Manager `_ +`Enthought Deployment Manager `_ "The Next Generation Python Environment and Package Manager". Previously Enthought provided Canopy, but it `reached end of life in 2016 @@ -678,13 +679,13 @@ System variables, you need non-restricted access to your machine .. seealso:: - https://docs.microsoft.com/en-us/windows/win32/procthread/environment-variables + https://learn.microsoft.com/windows/win32/procthread/environment-variables Overview of environment variables on Windows - https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/set_1 + https://learn.microsoft.com/windows-server/administration/windows-commands/set_1 The ``set`` command, for temporarily modifying environment variables - https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/setx + https://learn.microsoft.com/windows-server/administration/windows-commands/setx The ``setx`` command, for permanently modifying environment variables @@ -837,8 +838,8 @@ The short form of the argument (``-3``) only ever selects from core Python releases, and not other distributions. However, the longer form (``-V:3``) will select from any. -The Company is matched on the full string, case-insenitive. The Tag is matched -oneither the full string, or a prefix, provided the next character is a dot or a +The Company is matched on the full string, case-insensitive. The Tag is matched +on either the full string, or a prefix, provided the next character is a dot or a hyphen. This allows ``-V:3.1`` to match ``3.1-32``, but not ``3.10``. Tags are sorted using numerical ordering (``3.10`` is newer than ``3.1``), but are compared using text (``-V:3.01`` does not match ``3.1``). @@ -1285,18 +1286,18 @@ The Windows-specific standard modules are documented in PyWin32 ------- -The `PyWin32 `_ module by Mark Hammond +The :pypi:`PyWin32` module by Mark Hammond is a collection of modules for advanced Windows-specific support. This includes utilities for: * `Component Object Model - `_ + `_ (COM) * Win32 API calls * Registry * Event log * `Microsoft Foundation Classes - `_ + `_ (MFC) user interfaces `PythonWin `_ + `Win32 How Do I...? `_ by Tim Golden `Python and COM `_ diff --git a/Doc/whatsnew/2.1.rst b/Doc/whatsnew/2.1.rst index b4002f06e92adc..f23f27c994d717 100644 --- a/Doc/whatsnew/2.1.rst +++ b/Doc/whatsnew/2.1.rst @@ -443,8 +443,8 @@ Python syntax:: f.grammar = "A ::= B (C D)*" The dictionary containing attributes can be accessed as the function's -:attr:`~object.__dict__`. Unlike the :attr:`~object.__dict__` attribute of class instances, in -functions you can actually assign a new dictionary to :attr:`~object.__dict__`, though +:attr:`~function.__dict__`. Unlike the :attr:`~type.__dict__` attribute of class instances, in +functions you can actually assign a new dictionary to :attr:`~function.__dict__`, though the new value is restricted to a regular Python dictionary; you *can't* be tricky and set it to a :class:`!UserDict` instance, or any other random object that behaves like a mapping. @@ -644,9 +644,9 @@ New and Improved Modules lists the function arguments and the local variables for each frame. * Various functions in the :mod:`time` module, such as :func:`~time.asctime` and - :func:`~time.localtime`, require a floating point argument containing the time in + :func:`~time.localtime`, require a floating-point argument containing the time in seconds since the epoch. The most common use of these functions is to work with - the current time, so the floating point argument has been made optional; when a + the current time, so the floating-point argument has been made optional; when a value isn't provided, the current time will be used. For example, log file entries usually need a string containing the current time; in Python 2.1, ``time.asctime()`` can be used, instead of the lengthier diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index e6c13f957b8d54..856be5ecfa56ad 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -171,7 +171,7 @@ attributes of their own: * :attr:`~definition.__name__` is the attribute's name. -* :attr:`!__doc__` is the attribute's docstring. +* :attr:`~definition.__doc__` is the attribute's docstring. * ``__get__(object)`` is a method that retrieves the attribute value from *object*. @@ -186,7 +186,8 @@ are:: descriptor = obj.__class__.x descriptor.__get__(obj) -For methods, :meth:`!descriptor.__get__` returns a temporary object that's +For methods, :meth:`descriptor.__get__ ` returns a temporary +object that's callable, and wraps up the instance and the method to be called on it. This is also why static methods and class methods are now possible; they have descriptors that wrap up just the method, or the method and the class. As a @@ -1062,7 +1063,7 @@ code, none of the changes described here will affect you very much. simply been changed to use the new C-level interface. (Contributed by Fred L. Drake, Jr.) -* Another low-level API, primarily of interest to implementors of Python +* Another low-level API, primarily of interest to implementers of Python debuggers and development tools, was added. :c:func:`PyInterpreterState_Head` and :c:func:`PyInterpreterState_Next` let a caller walk through all the existing interpreter objects; :c:func:`PyInterpreterState_ThreadHead` and @@ -1249,7 +1250,7 @@ Some of the more notable changes are: * The :func:`pow` built-in function no longer supports 3 arguments when floating-point numbers are supplied. ``pow(x, y, z)`` returns ``(x**y) % z``, - but this is never useful for floating point numbers, and the final result varies + but this is never useful for floating-point numbers, and the final result varies unpredictably depending on the platform. A call such as ``pow(2.0, 8.0, 7.0)`` will now raise a :exc:`TypeError` exception. diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index 37cd41add8132c..ac463f82cfb8ca 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -1084,7 +1084,7 @@ Here are all of the changes that Python 2.3 makes to the core Python language. C3 algorithm as described in the paper `"A Monotonic Superclass Linearization for Dylan" `_. To understand the motivation for this change, read Michele Simionato's article - `"Python 2.3 Method Resolution Order" `_, or + :ref:`python_2.3_mro`, or read the thread on python-dev starting with the message at https://mail.python.org/pipermail/python-dev/2002-October/029035.html. Samuele Pedroni first pointed out the problem and also implemented the fix by coding the @@ -1113,10 +1113,10 @@ Here are all of the changes that Python 2.3 makes to the core Python language. * One of the noted incompatibilities between old- and new-style classes has been - removed: you can now assign to the :attr:`~definition.__name__` and :attr:`~class.__bases__` + removed: you can now assign to the :attr:`~type.__name__` and :attr:`~type.__bases__` attributes of new-style classes. There are some restrictions on what can be - assigned to :attr:`~class.__bases__` along the lines of those relating to assigning to - an instance's :attr:`~instance.__class__` attribute. + assigned to :attr:`!__bases__` along the lines of those relating to assigning to + an instance's :attr:`~object.__class__` attribute. .. ====================================================================== @@ -1382,7 +1382,7 @@ complete list of changes, or look through the CVS logs for all the details. In Python 2.4, the default will change to always returning floats. Application developers should enable this feature only if all their libraries - work properly when confronted with floating point time stamps, or if they use + work properly when confronted with floating-point time stamps, or if they use the tuple API. If used, the feature should be activated on an application level instead of trying to enable it on a per-use basis. @@ -1925,8 +1925,8 @@ Changes to Python's build process and to the C API include: dependence on a system version or local installation of Expat. * If you dynamically allocate type objects in your extension, you should be - aware of a change in the rules relating to the :attr:`!__module__` and - :attr:`~definition.__name__` attributes. In summary, you will want to ensure the type's + aware of a change in the rules relating to the :attr:`~type.__module__` and + :attr:`~type.__name__` attributes. In summary, you will want to ensure the type's dictionary contains a ``'__module__'`` key; making the module name the part of the type name leading up to the final period will no longer have the desired effect. For more detail, read the API reference documentation or the source. diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index 7e235d4370edaa..7628cfefe0ec96 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -684,11 +684,11 @@ includes a quick-start tutorial and a reference. Written by Facundo Batista and implemented by Facundo Batista, Eric Price, Raymond Hettinger, Aahz, and Tim Peters. - http://www.lahey.com/float.htm + `http://www.lahey.com/float.htm `__ The article uses Fortran code to illustrate many of the problems that floating-point inaccuracy can cause. - http://speleotrove.com/decimal/ + https://speleotrove.com/decimal/ A description of a decimal-based representation. This representation is being proposed as a standard, and underlies the new Python decimal type. Much of this material was written by Mike Cowlishaw, designer of the Rexx language. @@ -757,7 +757,7 @@ API that perform ASCII-only conversions, ignoring the locale setting: :c:expr:`double` to an ASCII string. The code for these functions came from the GLib library -(https://developer-old.gnome.org/glib/2.26/), whose developers kindly +(`https://developer-old.gnome.org/glib/2.26/ `__), whose developers kindly relicensed the relevant functions and donated them to the Python Software Foundation. The :mod:`locale` module can now change the numeric locale, letting extensions such as GTK+ produce the correct results. diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index 2ae26e7a106a0b..3430ac8668e280 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -1724,7 +1724,7 @@ attribute of the function object to change this:: :mod:`ctypes` also provides a wrapper for Python's C API as the ``ctypes.pythonapi`` object. This object does *not* release the global interpreter lock before calling a function, because the lock must be held when -calling into the interpreter's code. There's a :class:`py_object()` type +calling into the interpreter's code. There's a :class:`~ctypes.py_object` type constructor that will create a :c:expr:`PyObject *` pointer. A simple usage:: import ctypes @@ -1734,7 +1734,7 @@ constructor that will create a :c:expr:`PyObject *` pointer. A simple usage:: ctypes.py_object("abc"), ctypes.py_object(1)) # d is now {'abc', 1}. -Don't forget to use :class:`py_object()`; if it's omitted you end up with a +Don't forget to use :func:`~ctypes.py_object`; if it's omitted you end up with a segmentation fault. :mod:`ctypes` has been around for a while, but people still write and diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index e4ade5ecd82b9d..fdccfb7deb1ed7 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -502,12 +502,12 @@ Python's :option:`-m` switch allows running a module as a script. When you ran a module that was located inside a package, relative imports didn't work correctly. -The fix for Python 2.6 adds a :attr:`__package__` attribute to -modules. When this attribute is present, relative imports will be +The fix for Python 2.6 adds a :attr:`module.__package__` attribute. +When this attribute is present, relative imports will be relative to the value of this attribute instead of the -:attr:`__name__` attribute. +:attr:`~module.__name__` attribute. -PEP 302-style importers can then set :attr:`__package__` as necessary. +PEP 302-style importers can then set :attr:`~module.__package__` as necessary. The :mod:`runpy` module that implements the :option:`-m` switch now does this, so relative imports will now work correctly in scripts running from inside a package. @@ -1453,7 +1453,7 @@ that will be the numerator and denominator of the resulting fraction. :: Fraction(5, 3) For converting floating-point numbers to rationals, -the float type now has an :meth:`as_integer_ratio()` method that returns +the float type now has an :meth:`as_integer_ratio` method that returns the numerator and denominator for a fraction that evaluates to the same floating-point value:: @@ -2273,7 +2273,7 @@ changes, or look through the Subversion logs for all the details. (Contributed by Guido van Rossum from work for Google App Engine; :issue:`3487`.) -* The :mod:`rlcompleter` module's :meth:`Completer.complete()` method +* The :mod:`rlcompleter` module's :meth:`Completer.complete` method will now ignore exceptions triggered while evaluating a name. (Fixed by Lorenz Quack; :issue:`2250`.) @@ -2566,7 +2566,7 @@ changes, or look through the Subversion logs for all the details. :meth:`tracer`, and :meth:`speed` methods. * The ability to set new shapes for the turtle, and to define a new coordinate system. - * Turtles now have an :meth:`undo()` method that can roll back actions. + * Turtles now have an :meth:`undo` method that can roll back actions. * Simple support for reacting to input events such as mouse and keyboard activity, making it possible to write simple games. * A :file:`turtle.cfg` file can be used to customize the starting appearance @@ -3015,8 +3015,7 @@ Changes to Python's build process and to the C API include: ``PyRun_SimpleString("sys.path.pop(0)\n")`` afterwards to discard the first ``sys.path`` component. - Security issue reported as `CVE-2008-5983 - `_; + Security issue reported as :cve:`2008-5983`; discussed in :gh:`50003`, and fixed by Antoine Pitrou. * The BerkeleyDB module now has a C API object, available as @@ -3052,7 +3051,7 @@ Changes to Python's build process and to the C API include: * Several functions return information about the platform's floating-point support. :c:func:`PyFloat_GetMax` returns - the maximum representable floating point value, + the maximum representable floating-point value, and :c:func:`PyFloat_GetMin` returns the minimum positive value. :c:func:`PyFloat_GetInfo` returns an object containing more information from the :file:`float.h` file, such as diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 5c99fbc503ba65..0e4dee0bd24fb2 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -291,7 +291,7 @@ modules. configuration files can now be read, modified, and then written back in their original order. -* The :meth:`~collections.somenamedtuple._asdict()` method for +* The :meth:`~collections.somenamedtuple._asdict` method for :func:`collections.namedtuple` now returns an ordered dictionary with the values appearing in the same order as the underlying tuple indices. @@ -1198,7 +1198,7 @@ changes, or look through the Subversion logs for all the details. of the operands. Previously such comparisons would fall back to Python's default rules for comparing objects, which produced arbitrary results based on their type. Note that you still cannot combine - :class:`!Decimal` and floating-point in other operations such as addition, + :class:`!Decimal` and floating point in other operations such as addition, since you should be explicitly choosing how to convert between float and :class:`!Decimal`. (Fixed by Mark Dickinson; :issue:`2531`.) @@ -1548,7 +1548,7 @@ changes, or look through the Subversion logs for all the details. *ciphers* argument that's a string listing the encryption algorithms to be allowed; the format of the string is described `in the OpenSSL documentation - `__. + `__. (Added by Antoine Pitrou; :issue:`8322`.) Another change makes the extension load all of OpenSSL's ciphers and @@ -1738,7 +1738,7 @@ New module: importlib Python 3.1 includes the :mod:`importlib` package, a re-implementation of the logic underlying Python's :keyword:`import` statement. -:mod:`importlib` is useful for implementors of Python interpreters and +:mod:`importlib` is useful for implementers of Python interpreters and to users who wish to write new importers that can participate in the import process. Python 2.7 doesn't contain the complete :mod:`importlib` package, but instead has a tiny subset that contains @@ -1831,8 +1831,7 @@ The :mod:`unittest` module was greatly enhanced; many new features were added. Most of these features were implemented by Michael Foord, unless otherwise noted. The enhanced version of the module is downloadable separately for use with Python versions 2.4 to 2.6, -packaged as the :mod:`!unittest2` package, from -https://pypi.org/project/unittest2. +packaged as the :mod:`!unittest2` package, from :pypi:`unittest2`. When used from the command line, the module can automatically discover tests. It's not as fancy as `py.test `__ or @@ -2178,8 +2177,7 @@ Changes to Python's build process and to the C API include: whether the application should be using :c:func:`!PySys_SetArgvEx` with *updatepath* set to false. - Security issue reported as `CVE-2008-5983 - `_; + Security issue reported as :cve:`2008-5983`; discussed in :issue:`5753`, and fixed by Antoine Pitrou. * New macros: the Python header files now define the following macros: @@ -2626,7 +2624,7 @@ with the first of those changes appearing in the Python 2.7.7 release. 2 applications. (Contributed by Alex Gaynor; :issue:`21304`.) * OpenSSL 1.0.1h was upgraded for the official Windows installers published on - python.org. (contributed by Zachary Ware in :issue:`21671` for CVE-2014-0224) + python.org. (Contributed by Zachary Ware in :issue:`21671` for :cve:`2014-0224`.) :pep:`466` related features added in Python 2.7.9: @@ -2682,14 +2680,12 @@ automatic ``PATH`` modifications to have ``pip`` available from the command line by default, otherwise it can still be accessed through the Python launcher for Windows as ``py -m pip``. -As `discussed in the PEP`__, platform packagers may choose not to install +As :pep:`discussed in the PEP <0477#disabling-ensurepip-by-downstream-distributors>`, +platform packagers may choose not to install these commands by default, as long as, when invoked, they provide clear and simple directions on how to install them on that platform (usually using the system package manager). -__ https://peps.python.org/pep-0477/#disabling-ensurepip-by-downstream-distributors - - Documentation Changes ~~~~~~~~~~~~~~~~~~~~~ diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index 888e6279754fc2..6e1fda22ed203a 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -150,8 +150,8 @@ Some well-known APIs no longer return lists: sorted(d)`` instead (this works in Python 2.5 too and is just as efficient). -* Also, the :meth:`dict.iterkeys`, :meth:`dict.iteritems` and - :meth:`dict.itervalues` methods are no longer supported. +* Also, the :meth:`!dict.iterkeys`, :meth:`!dict.iteritems` and + :meth:`!dict.itervalues` methods are no longer supported. * :func:`map` and :func:`filter` return iterators. If you really need a list and the input sequences are all of equal length, a quick @@ -170,7 +170,7 @@ Some well-known APIs no longer return lists: :func:`itertools.zip_longest`, e.g. ``map(func, *sequences)`` becomes ``list(map(func, itertools.zip_longest(*sequences)))``. -* :func:`range` now behaves like :func:`xrange` used to behave, except +* :func:`range` now behaves like :func:`!xrange` used to behave, except it works with values of arbitrary size. The latter no longer exists. @@ -192,33 +192,33 @@ Python 3.0 has simplified the rules for ordering comparisons: operators: objects of different incomparable types always compare unequal to each other. -* :meth:`builtin.sorted` and :meth:`list.sort` no longer accept the +* :meth:`sorted` and :meth:`list.sort` no longer accept the *cmp* argument providing a comparison function. Use the *key* argument instead. N.B. the *key* and *reverse* arguments are now "keyword-only". -* The :func:`cmp` function should be treated as gone, and the :meth:`__cmp__` - special method is no longer supported. Use :meth:`__lt__` for sorting, - :meth:`__eq__` with :meth:`__hash__`, and other rich comparisons as needed. - (If you really need the :func:`cmp` functionality, you could use the +* The :func:`!cmp` function should be treated as gone, and the :meth:`!__cmp__` + special method is no longer supported. Use :meth:`~object.__lt__` for sorting, + :meth:`~object.__eq__` with :meth:`~object.__hash__`, and other rich comparisons as needed. + (If you really need the :func:`!cmp` functionality, you could use the expression ``(a > b) - (a < b)`` as the equivalent for ``cmp(a, b)``.) Integers -------- -* :pep:`237`: Essentially, :class:`long` renamed to :class:`int`. +* :pep:`237`: Essentially, :class:`!long` renamed to :class:`int`. That is, there is only one built-in integral type, named - :class:`int`; but it behaves mostly like the old :class:`long` type. + :class:`int`; but it behaves mostly like the old :class:`!long` type. * :pep:`238`: An expression like ``1/2`` returns a float. Use ``1//2`` to get the truncating behavior. (The latter syntax has existed for years, at least since Python 2.2.) -* The :data:`sys.maxint` constant was removed, since there is no +* The :data:`!sys.maxint` constant was removed, since there is no longer a limit to the value of integers. However, :data:`sys.maxsize` can be used as an integer larger than any practical list or string index. It conforms to the implementation's "natural" integer size - and is typically the same as :data:`sys.maxint` in previous releases + and is typically the same as :data:`!sys.maxint` in previous releases on the same platform (assuming the same build options). * The :func:`repr` of a long integer doesn't include the trailing ``L`` @@ -251,7 +251,7 @@ changed. that uses Unicode, encodings or binary data most likely has to change. The change is for the better, as in the 2.x world there were numerous bugs having to do with mixing encoded and unencoded - text. To be prepared in Python 2.x, start using :class:`unicode` + text. To be prepared in Python 2.x, start using :class:`!unicode` for all unencoded text, and :class:`str` for binary or encoded data only. Then the ``2to3`` tool will do most of the work for you. @@ -269,7 +269,7 @@ changed. separate *mutable* type to hold buffered binary data, :class:`bytearray`. Nearly all APIs that accept :class:`bytes` also accept :class:`bytearray`. The mutable API is based on - :class:`collections.MutableSequence`. + :class:`collections.MutableSequence `. * All backslashes in raw string literals are interpreted literally. This means that ``'\U'`` and ``'\u'`` escapes in raw strings are not @@ -278,11 +278,11 @@ changed. single "euro" character. (Of course, this change only affects raw string literals; the euro character is ``'\u20ac'`` in Python 3.0.) -* The built-in :class:`basestring` abstract type was removed. Use +* The built-in :class:`!basestring` abstract type was removed. Use :class:`str` instead. The :class:`str` and :class:`bytes` types don't have functionality enough in common to warrant a shared base class. The ``2to3`` tool (see below) replaces every occurrence of - :class:`basestring` with :class:`str`. + :class:`!basestring` with :class:`str`. * Files opened as text files (still the default mode for :func:`open`) always use an encoding to map between strings (in memory) and bytes @@ -357,8 +357,8 @@ New Syntax provides a standardized way of annotating a function's parameters and return value. There are no semantics attached to such annotations except that they can be introspected at runtime using - the :attr:`__annotations__` attribute. The intent is to encourage - experimentation through metaclasses, decorators or frameworks. + the :attr:`~object.__annotations__` attribute. The intent is to + encourage experimentation through metaclasses, decorators or frameworks. * :pep:`3102`: Keyword-only arguments. Named parameters occurring after ``*args`` in the parameter list *must* be specified using @@ -428,7 +428,7 @@ Changed Syntax class C(metaclass=M): ... - The module-global :data:`__metaclass__` variable is no longer + The module-global :data:`!__metaclass__` variable is no longer supported. (It was a crutch to make it easier to default to new-style classes without deriving every class from :class:`object`.) @@ -522,19 +522,19 @@ consulted for longer descriptions. *encoding*, *errors*, *newline* and *closefd*. Also note that an invalid *mode* argument now raises :exc:`ValueError`, not :exc:`IOError`. The binary file object underlying a text file - object can be accessed as :attr:`f.buffer` (but beware that the + object can be accessed as :attr:`!f.buffer` (but beware that the text object maintains a buffer of itself in order to speed up the encoding and decoding operations). -* :ref:`pep-3118`. The old builtin :func:`buffer` is now really gone; +* :ref:`pep-3118`. The old builtin :func:`!buffer` is now really gone; the new builtin :func:`memoryview` provides (mostly) similar functionality. * :ref:`pep-3119`. The :mod:`abc` module and the ABCs defined in the :mod:`collections` module plays a somewhat more prominent role in the language now, and built-in collection types like :class:`dict` - and :class:`list` conform to the :class:`collections.MutableMapping` - and :class:`collections.MutableSequence` ABCs, respectively. + and :class:`list` conform to the :class:`collections.MutableMapping ` + and :class:`collections.MutableSequence ` ABCs, respectively. * :ref:`pep-3127`. As mentioned above, the new octal literal notation is the only one supported, and binary literals have been @@ -612,7 +612,7 @@ review: :mod:`!CGIHTTPServer`, :mod:`!SimpleHTTPServer`, :mod:`!Cookie`, :mod:`!cookielib`). - * :mod:`tkinter` (all :mod:`Tkinter`-related modules except + * :mod:`tkinter` (all ``Tkinter``-related modules except :mod:`turtle`). The target audience of :mod:`turtle` doesn't really care about :mod:`tkinter`. Also note that as of Python 2.6, the functionality of :mod:`turtle` has been greatly enhanced. @@ -628,47 +628,47 @@ Some other changes to standard library modules, not covered by * Killed :mod:`!sets`. Use the built-in :func:`set` class. -* Cleanup of the :mod:`sys` module: removed :func:`sys.exitfunc`, - :func:`sys.exc_clear`, :data:`sys.exc_type`, :data:`sys.exc_value`, - :data:`sys.exc_traceback`. (Note that :data:`sys.last_type` +* Cleanup of the :mod:`sys` module: removed :func:`!sys.exitfunc`, + :func:`!sys.exc_clear`, :data:`!sys.exc_type`, :data:`!sys.exc_value`, + :data:`!sys.exc_traceback`. (Note that :data:`sys.last_type` etc. remain.) -* Cleanup of the :class:`array.array` type: the :meth:`read` and - :meth:`write` methods are gone; use :meth:`fromfile` and - :meth:`tofile` instead. Also, the ``'c'`` typecode for array is +* Cleanup of the :class:`array.array` type: the :meth:`!read` and + :meth:`!write` methods are gone; use :meth:`~array.array.fromfile` and + :meth:`~array.array.tofile` instead. Also, the ``'c'`` typecode for array is gone -- use either ``'b'`` for bytes or ``'u'`` for Unicode characters. * Cleanup of the :mod:`operator` module: removed - :func:`sequenceIncludes` and :func:`isCallable`. + :func:`!sequenceIncludes` and :func:`!isCallable`. * Cleanup of the :mod:`!thread` module: :func:`!acquire_lock` and :func:`!release_lock` are gone; use :meth:`~threading.Lock.acquire` and :meth:`~threading.Lock.release` instead. -* Cleanup of the :mod:`random` module: removed the :func:`jumpahead` API. +* Cleanup of the :mod:`random` module: removed the :func:`!jumpahead` API. * The :mod:`!new` module is gone. -* The functions :func:`os.tmpnam`, :func:`os.tempnam` and - :func:`os.tmpfile` have been removed in favor of the :mod:`tempfile` +* The functions :func:`!os.tmpnam`, :func:`!os.tempnam` and + :func:`!os.tmpfile` have been removed in favor of the :mod:`tempfile` module. * The :mod:`tokenize` module has been changed to work with bytes. The main entry point is now :func:`tokenize.tokenize`, instead of generate_tokens. -* :data:`string.letters` and its friends (:data:`string.lowercase` and - :data:`string.uppercase`) are gone. Use +* :data:`!string.letters` and its friends (:data:`!string.lowercase` and + :data:`!string.uppercase`) are gone. Use :data:`string.ascii_letters` etc. instead. (The reason for the - removal is that :data:`string.letters` and friends had + removal is that :data:`!string.letters` and friends had locale-specific behavior, which is a bad idea for such attractively named global "constants".) -* Renamed module :mod:`__builtin__` to :mod:`builtins` (removing the - underscores, adding an 's'). The :data:`__builtins__` variable +* Renamed module :mod:`!__builtin__` to :mod:`builtins` (removing the + underscores, adding an 's'). The :data:`!__builtins__` variable found in most global namespaces is unchanged. To modify a builtin, - you should use :mod:`builtins`, not :data:`__builtins__`! + you should use :mod:`builtins`, not :data:`!__builtins__`! :pep:`3101`: A New Approach To String Formatting @@ -702,9 +702,9 @@ new powerful features added: idiom for handling all exceptions except for this latter category is to use :keyword:`except` :exc:`Exception`. -* :exc:`StandardError` was removed. +* :exc:`!StandardError` was removed. -* Exceptions no longer behave as sequences. Use the :attr:`args` +* Exceptions no longer behave as sequences. Use the :attr:`~BaseException.args` attribute instead. * :pep:`3109`: Raising exceptions. You must now use :samp:`raise @@ -765,20 +765,20 @@ Operators And Special Methods When referencing a method as a class attribute, you now get a plain function object. -* :meth:`__getslice__`, :meth:`__setslice__` and :meth:`__delslice__` +* :meth:`!__getslice__`, :meth:`!__setslice__` and :meth:`!__delslice__` were killed. The syntax ``a[i:j]`` now translates to - ``a.__getitem__(slice(i, j))`` (or :meth:`__setitem__` or - :meth:`__delitem__`, when used as an assignment or deletion target, + ``a.__getitem__(slice(i, j))`` (or :meth:`~object.__setitem__` or + :meth:`~object.__delitem__`, when used as an assignment or deletion target, respectively). * :pep:`3114`: the standard :meth:`next` method has been renamed to :meth:`~iterator.__next__`. -* The :meth:`__oct__` and :meth:`__hex__` special methods are removed - -- :func:`oct` and :func:`hex` use :meth:`__index__` now to convert +* The :meth:`!__oct__` and :meth:`!__hex__` special methods are removed + -- :func:`oct` and :func:`hex` use :meth:`~object.__index__` now to convert the argument to an integer. -* Removed support for :attr:`__members__` and :attr:`__methods__`. +* Removed support for :attr:`!__members__` and :attr:`!__methods__`. * The function attributes named :attr:`!func_X` have been renamed to use the :attr:`!__X__` form, freeing up these names in the function @@ -802,7 +802,7 @@ Builtins instance will automatically be chosen. With arguments, the behavior of :func:`super` is unchanged. -* :pep:`3111`: :func:`raw_input` was renamed to :func:`input`. That +* :pep:`3111`: :func:`!raw_input` was renamed to :func:`input`. That is, the new :func:`input` function reads a line from :data:`sys.stdin` and returns it with the trailing newline stripped. It raises :exc:`EOFError` if the input is terminated prematurely. @@ -820,31 +820,31 @@ Builtins argument and a value of the same type as ``x`` when called with two arguments. -* Moved :func:`intern` to :func:`sys.intern`. +* Moved :func:`!intern` to :func:`sys.intern`. -* Removed: :func:`apply`. Instead of ``apply(f, args)`` use +* Removed: :func:`!apply`. Instead of ``apply(f, args)`` use ``f(*args)``. * Removed :func:`callable`. Instead of ``callable(f)`` you can use - ``isinstance(f, collections.Callable)``. The :func:`operator.isCallable` + ``isinstance(f, collections.Callable)``. The :func:`!operator.isCallable` function is also gone. -* Removed :func:`coerce`. This function no longer serves a purpose +* Removed :func:`!coerce`. This function no longer serves a purpose now that classic classes are gone. -* Removed :func:`execfile`. Instead of ``execfile(fn)`` use +* Removed :func:`!execfile`. Instead of ``execfile(fn)`` use ``exec(open(fn).read())``. -* Removed the :class:`file` type. Use :func:`open`. There are now several +* Removed the :class:`!file` type. Use :func:`open`. There are now several different kinds of streams that open can return in the :mod:`io` module. -* Removed :func:`reduce`. Use :func:`functools.reduce` if you really +* Removed :func:`!reduce`. Use :func:`functools.reduce` if you really need it; however, 99 percent of the time an explicit :keyword:`for` loop is more readable. -* Removed :func:`reload`. Use :func:`!imp.reload`. +* Removed :func:`!reload`. Use :func:`!imp.reload`. -* Removed. :meth:`dict.has_key` -- use the :keyword:`in` operator +* Removed. :meth:`!dict.has_key` -- use the :keyword:`in` operator instead. .. ====================================================================== diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst index 69b273e58385d2..b9606beb5f9ef9 100644 --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -205,9 +205,9 @@ Some smaller changes made to the core Python language are: (Contributed by Mark Dickinson; :issue:`4707`.) -* Python now uses David Gay's algorithm for finding the shortest floating - point representation that doesn't change its value. This should help - mitigate some of the confusion surrounding binary floating point +* Python now uses David Gay's algorithm for finding the shortest floating-point + representation that doesn't change its value. This should help + mitigate some of the confusion surrounding binary floating-point numbers. The significance is easily seen with a number like ``1.1`` which does not @@ -215,7 +215,7 @@ Some smaller changes made to the core Python language are: equivalent, an expression like ``float('1.1')`` evaluates to the nearest representable value which is ``0x1.199999999999ap+0`` in hex or ``1.100000000000000088817841970012523233890533447265625`` in decimal. That - nearest value was and still is used in subsequent floating point + nearest value was and still is used in subsequent floating-point calculations. What is new is how the number gets displayed. Formerly, Python used a @@ -224,7 +224,7 @@ Some smaller changes made to the core Python language are: using 17 digits was that it relied on IEEE-754 guarantees to assure that ``eval(repr(1.1))`` would round-trip exactly to its original value. The disadvantage is that many people found the output to be confusing (mistaking - intrinsic limitations of binary floating point representation as being a + intrinsic limitations of binary floating-point representation as being a problem with Python itself). The new algorithm for ``repr(1.1)`` is smarter and returns ``'1.1'``. @@ -236,8 +236,8 @@ Some smaller changes made to the core Python language are: it does not change the underlying values. So, it is still the case that ``1.1 + 2.2 != 3.3`` even though the representations may suggest otherwise. - The new algorithm depends on certain features in the underlying floating - point implementation. If the required features are not found, the old + The new algorithm depends on certain features in the underlying floating-point + implementation. If the required features are not found, the old algorithm will continue to be used. Also, the text pickle protocols assure cross-platform portability by using the old algorithm. @@ -550,7 +550,7 @@ Porting to Python 3.1 This section lists previously described changes and other bugfixes that may require changes to your code: -* The new floating point string representations can break existing doctests. +* The new floating-point string representations can break existing doctests. For example:: def e(): diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index e35179a2d8e513..e4699fbf8edaf7 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -352,7 +352,7 @@ was expecting an indentation, including the location of the statement: AttributeErrors ~~~~~~~~~~~~~~~ -When printing :exc:`AttributeError`, :c:func:`PyErr_Display` will offer +When printing :exc:`AttributeError`, :c:func:`!PyErr_Display` will offer suggestions of similar attribute names in the object that the exception was raised from: @@ -366,14 +366,14 @@ raised from: (Contributed by Pablo Galindo in :issue:`38530`.) .. warning:: - Notice this won't work if :c:func:`PyErr_Display` is not called to display the error + Notice this won't work if :c:func:`!PyErr_Display` is not called to display the error which can happen if some other custom error display function is used. This is a common scenario in some REPLs like IPython. NameErrors ~~~~~~~~~~ -When printing :exc:`NameError` raised by the interpreter, :c:func:`PyErr_Display` +When printing :exc:`NameError` raised by the interpreter, :c:func:`!PyErr_Display` will offer suggestions of similar variable names in the function that the exception was raised from: @@ -388,7 +388,7 @@ was raised from: (Contributed by Pablo Galindo in :issue:`38530`.) .. warning:: - Notice this won't work if :c:func:`PyErr_Display` is not called to display the error, + Notice this won't work if :c:func:`!PyErr_Display` is not called to display the error, which can happen if some other custom error display function is used. This is a common scenario in some REPLs like IPython. @@ -690,7 +690,7 @@ are in :pep:`635`, and a longer tutorial is in :pep:`636`. Optional ``EncodingWarning`` and ``encoding="locale"`` option ------------------------------------------------------------- -The default encoding of :class:`TextIOWrapper` and :func:`open` is +The default encoding of :class:`~io.TextIOWrapper` and :func:`open` is platform and locale dependent. Since UTF-8 is used on most Unix platforms, omitting ``encoding`` option when opening UTF-8 files (e.g. JSON, YAML, TOML, Markdown) is a very common bug. For example:: @@ -785,7 +785,7 @@ especially when forward references or invalid types were involved. Compare:: StrCache = 'Cache[str]' # a type alias LOG_PREFIX = 'LOG[DEBUG]' # a module constant -Now the :mod:`typing` module has a special value :data:`TypeAlias` +Now the :mod:`typing` module has a special value :data:`~typing.TypeAlias` which lets you declare type aliases more explicitly:: StrCache: TypeAlias = 'Cache[str]' # a type alias @@ -798,10 +798,10 @@ See :pep:`613` for more details. PEP 647: User-Defined Type Guards --------------------------------- -:data:`TypeGuard` has been added to the :mod:`typing` module to annotate +:data:`~typing.TypeGuard` has been added to the :mod:`typing` module to annotate type guard functions and improve information provided to static type checkers -during type narrowing. For more information, please see :data:`TypeGuard`\ 's -documentation, and :pep:`647`. +during type narrowing. For more information, please see +:data:`~typing.TypeGuard`\ 's documentation, and :pep:`647`. (Contributed by Ken Jin and Guido van Rossum in :issue:`43766`. PEP written by Eric Traut.) @@ -972,8 +972,8 @@ and objects representing asynchronously released resources. Add asynchronous context manager support to :func:`contextlib.nullcontext`. (Contributed by Tom Gringauz in :issue:`41543`.) -Add :class:`AsyncContextDecorator`, for supporting usage of async context managers -as decorators. +Add :class:`~contextlib.AsyncContextDecorator`, for supporting usage of async +context managers as decorators. curses ------ @@ -1089,8 +1089,8 @@ encodings enum ---- -:class:`Enum` :func:`__repr__` now returns ``enum_name.member_name`` and -:func:`__str__` now returns ``member_name``. Stdlib enums available as +:class:`~enum.Enum` :func:`~object.__repr__` now returns ``enum_name.member_name`` and +:func:`~object.__str__` now returns ``member_name``. Stdlib enums available as module constants have a :func:`repr` of ``module_name.member_name``. (Contributed by Ethan Furman in :issue:`40066`.) @@ -1104,7 +1104,7 @@ Add *encoding* and *errors* parameters in :func:`fileinput.input` and :class:`fileinput.FileInput`. (Contributed by Inada Naoki in :issue:`43712`.) -:func:`fileinput.hook_compressed` now returns :class:`TextIOWrapper` object +:func:`fileinput.hook_compressed` now returns :class:`~io.TextIOWrapper` object when *mode* is "r" and file is compressed, like uncompressed files. (Contributed by Inada Naoki in :issue:`5758`.) @@ -1202,12 +1202,12 @@ Feature parity with ``importlib_metadata`` 4.6 :ref:`importlib.metadata entry points ` now provide a nicer experience for selecting entry points by group and name through a new -:class:`importlib.metadata.EntryPoints` class. See the Compatibility +:ref:`importlib.metadata.EntryPoints ` class. See the Compatibility Note in the docs for more info on the deprecation and usage. -Added :func:`importlib.metadata.packages_distributions` for resolving -top-level Python modules and packages to their -:class:`importlib.metadata.Distribution`. +Added :ref:`importlib.metadata.packages_distributions() ` +for resolving top-level Python modules and packages to their +:ref:`importlib.metadata.Distribution `. inspect ------- @@ -1224,7 +1224,7 @@ best practice for accessing the annotations dict defined on any Python object; for more information on best practices for working with annotations, please see :ref:`annotations-howto`. Relatedly, :func:`inspect.signature`, -:func:`inspect.Signature.from_callable`, and :func:`inspect.Signature.from_function` +:func:`inspect.Signature.from_callable`, and :func:`!inspect.Signature.from_function` now call :func:`inspect.get_annotations` to retrieve annotations. This means :func:`inspect.signature` and :func:`inspect.Signature.from_callable` can also now un-stringize stringized annotations. @@ -1233,7 +1233,7 @@ also now un-stringize stringized annotations. itertools --------- -Add :func:`itertools.pairwise()`. +Add :func:`itertools.pairwise`. (Contributed by Raymond Hettinger in :issue:`38200`.) linecache @@ -1245,14 +1245,14 @@ When a module does not define ``__loader__``, fall back to ``__spec__.loader``. os -- -Add :func:`os.cpu_count()` support for VxWorks RTOS. +Add :func:`os.cpu_count` support for VxWorks RTOS. (Contributed by Peixing Xin in :issue:`41440`.) Add a new function :func:`os.eventfd` and related helpers to wrap the ``eventfd2`` syscall on Linux. (Contributed by Christian Heimes in :issue:`41001`.) -Add :func:`os.splice()` that allows to move data between two file +Add :func:`os.splice` that allows to move data between two file descriptors without copying between kernel address space and user address space, where one of the file descriptors must refer to a pipe. (Contributed by Pablo Galindo in :issue:`41625`.) @@ -1292,7 +1292,7 @@ functions in the :mod:`os` module. platform -------- -Add :func:`platform.freedesktop_os_release()` to retrieve operation system +Add :func:`platform.freedesktop_os_release` to retrieve operation system identification from `freedesktop.org os-release `_ standard file. (Contributed by Christian Heimes in :issue:`28468`.) @@ -1484,9 +1484,9 @@ is a :class:`typing.TypedDict`. Subclasses of ``typing.Protocol`` which only have data variables declared will now raise a ``TypeError`` when checked with ``isinstance`` unless they -are decorated with :func:`runtime_checkable`. Previously, these checks +are decorated with :func:`~typing.runtime_checkable`. Previously, these checks passed silently. Users should decorate their -subclasses with the :func:`runtime_checkable` decorator +subclasses with the :func:`!runtime_checkable` decorator if they want runtime protocols. (Contributed by Yurii Karabas in :issue:`38908`.) @@ -1595,8 +1595,8 @@ Optimizations :func:`map`, :func:`filter`, :func:`reversed`, :func:`bool` and :func:`float`. (Contributed by Donghee Na and Jeroen Demeyer in :issue:`43575`, :issue:`43287`, :issue:`41922`, :issue:`41873` and :issue:`41870`.) -* :class:`BZ2File` performance is improved by removing internal ``RLock``. - This makes :class:`BZ2File` thread unsafe in the face of multiple simultaneous +* :class:`~bz2.BZ2File` performance is improved by removing internal ``RLock``. + This makes :class:`!BZ2File` thread unsafe in the face of multiple simultaneous readers or writers, just like its equivalent classes in :mod:`gzip` and :mod:`lzma` have always been. (Contributed by Inada Naoki in :issue:`43785`.) @@ -1620,7 +1620,7 @@ Deprecated cleaning up old import semantics that were kept for Python 2.7 compatibility. Specifically, :meth:`!find_loader`/:meth:`!find_module` - (superseded by :meth:`~importlib.abc.Finder.find_spec`), + (superseded by :meth:`~importlib.abc.MetaPathFinder.find_spec`), :meth:`~importlib.abc.Loader.load_module` (superseded by :meth:`~importlib.abc.Loader.exec_module`), :meth:`!module_repr` (which the import system @@ -1647,7 +1647,7 @@ Deprecated :meth:`~importlib.abc.Loader.exec_module` instead. (Contributed by Brett Cannon in :issue:`26131`.) -* :meth:`zimport.zipimporter.load_module` has been deprecated in +* :meth:`!zimport.zipimporter.load_module` has been deprecated in preference for :meth:`~zipimport.zipimporter.exec_module`. (Contributed by Brett Cannon in :issue:`26131`.) @@ -1759,23 +1759,23 @@ Deprecated * The following :mod:`ssl` features have been deprecated since Python 3.6, Python 3.7, or OpenSSL 1.1.0 and will be removed in 3.11: - * :data:`~ssl.OP_NO_SSLv2`, :data:`~ssl.OP_NO_SSLv3`, :data:`~ssl.OP_NO_TLSv1`, - :data:`~ssl.OP_NO_TLSv1_1`, :data:`~ssl.OP_NO_TLSv1_2`, and - :data:`~ssl.OP_NO_TLSv1_3` are replaced by - :attr:`sslSSLContext.minimum_version` and - :attr:`sslSSLContext.maximum_version`. + * :data:`!OP_NO_SSLv2`, :data:`!OP_NO_SSLv3`, :data:`!OP_NO_TLSv1`, + :data:`!OP_NO_TLSv1_1`, :data:`!OP_NO_TLSv1_2`, and + :data:`!OP_NO_TLSv1_3` are replaced by + :attr:`~ssl.SSLContext.minimum_version` and + :attr:`~ssl.SSLContext.maximum_version`. - * :data:`~ssl.PROTOCOL_SSLv2`, :data:`~ssl.PROTOCOL_SSLv3`, - :data:`~ssl.PROTOCOL_SSLv23`, :data:`~ssl.PROTOCOL_TLSv1`, - :data:`~ssl.PROTOCOL_TLSv1_1`, :data:`~ssl.PROTOCOL_TLSv1_2`, and - :const:`~ssl.PROTOCOL_TLS` are deprecated in favor of + * :data:`!PROTOCOL_SSLv2`, :data:`!PROTOCOL_SSLv3`, + :data:`!PROTOCOL_SSLv23`, :data:`!PROTOCOL_TLSv1`, + :data:`!PROTOCOL_TLSv1_1`, :data:`!PROTOCOL_TLSv1_2`, and + :const:`!PROTOCOL_TLS` are deprecated in favor of :const:`~ssl.PROTOCOL_TLS_CLIENT` and :const:`~ssl.PROTOCOL_TLS_SERVER` - * :func:`~ssl.wrap_socket` is replaced by :meth:`ssl.SSLContext.wrap_socket` + * :func:`!wrap_socket` is replaced by :meth:`ssl.SSLContext.wrap_socket` - * :func:`~ssl.match_hostname` + * :func:`!match_hostname` - * :func:`~ssl.RAND_pseudo_bytes`, :func:`~ssl.RAND_egd` + * :func:`!RAND_pseudo_bytes`, :func:`!RAND_egd` * NPN features like :meth:`ssl.SSLSocket.selected_npn_protocol` and :meth:`ssl.SSLContext.set_npn_protocols` are replaced by ALPN. @@ -2331,8 +2331,7 @@ Converting between :class:`int` and :class:`str` in bases other than 2 (binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) now raises a :exc:`ValueError` if the number of digits in string form is above a limit to avoid potential denial of service attacks due to the -algorithmic complexity. This is a mitigation for `CVE-2020-10735 -`_. +algorithmic complexity. This is a mitigation for :cve:`2020-10735`. This limit can be configured or disabled by environment variable, command line flag, or :mod:`sys` APIs. See the :ref:`integer string conversion length limitation ` documentation. The default limit diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 4f4c1de8d8d596..ed41ecd50b0011 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -544,8 +544,7 @@ Other CPython Implementation Changes (binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) now raises a :exc:`ValueError` if the number of digits in string form is above a limit to avoid potential denial of service attacks due to the - algorithmic complexity. This is a mitigation for `CVE-2020-10735 - `_. + algorithmic complexity. This is a mitigation for :cve:`2020-10735`. This limit can be configured or disabled by environment variable, command line flag, or :mod:`sys` APIs. See the :ref:`integer string conversion length limitation ` documentation. The default limit @@ -769,6 +768,21 @@ functools (Contributed by Yurii Karabas in :issue:`46014`.) +.. _whatsnew311-gzip: + +gzip +---- + +* The :func:`gzip.compress` function is now faster when used with the + **mtime=0** argument as it delegates the compression entirely to a single + :func:`zlib.compress` operation. There is one side effect of this change: The + gzip file header contains an "OS" byte in its header. That was traditionally + always set to a value of 255 representing "unknown" by the :mod:`gzip` + module. Now, when using :func:`~gzip.compress` with **mtime=0**, it may be + set to a different value by the underlying zlib C library Python was linked + against. + (See :gh:`112346` for details on the side effect.) + .. _whatsnew311-hashlib: hashlib @@ -1656,7 +1670,7 @@ Replaced opcodes | | | for each direction | | | | | +------------------------------------+------------------------------------+-----------------------------------------+ -| | :opcode:`!SETUP_WITH` | :opcode:`BEFORE_WITH` | :keyword:`with` block setup | +| | :opcode:`!SETUP_WITH` | :opcode:`!BEFORE_WITH` | :keyword:`with` block setup | | | :opcode:`!SETUP_ASYNC_WITH` | | | +------------------------------------+------------------------------------+-----------------------------------------+ @@ -2018,8 +2032,8 @@ Removed C APIs are :ref:`listed separately `. It was introduced in Python 3.4 but has been broken since Python 3.7. (Contributed by Inada Naoki in :issue:`23882`.) -* Removed the undocumented private :meth:`!float.__set_format__()` method, - previously known as :meth:`!float.__setformat__()` in Python 3.7. +* Removed the undocumented private :meth:`!float.__set_format__` method, + previously known as :meth:`!float.__setformat__` in Python 3.7. Its docstring said: "You probably don't want to use this function. It exists mainly to be used in Python's test suite." (Contributed by Victor Stinner in :issue:`46852`.) @@ -2028,7 +2042,7 @@ Removed C APIs are :ref:`listed separately `. (and corresponding :c:macro:`!EXPERIMENTAL_ISOLATED_SUBINTERPRETERS` macro) have been removed. -* `Pynche `_ +* :pypi:`Pynche` --- The Pythonically Natural Color and Hue Editor --- has been moved out of ``Tools/scripts`` and is `being developed independently `_ from the Python source tree. @@ -2124,7 +2138,7 @@ Build Changes :issue:`45440` and :issue:`46640`.) * Support for `IEEE 754 `_ - floating point numbers. + floating-point numbers. (Contributed by Victor Stinner in :issue:`46917`.) * The :c:macro:`!Py_NO_NAN` macro has been removed. @@ -2154,7 +2168,7 @@ Build Changes (Contributed by Donghee Na and Brett Holman in :issue:`44340`.) * Freelists for object structs can now be disabled. A new :program:`configure` - option :option:`--without-freelists` can be used to disable all freelists + option ``--without-freelists`` can be used to disable all freelists except empty tuple singleton. (Contributed by Christian Heimes in :issue:`45522`.) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index b986e638498abd..4fffc78a237791 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -59,7 +59,7 @@ Summary -- Release highlights .. This section singles out the most important changes in Python 3.12. Brevity is key. -Python 3.12 is the latest stable release of the Python programming language, +Python 3.12 is a stable release of the Python programming language, with a mix of changes to the language and the standard library. The library changes focus on cleaning up deprecated APIs, usability, and correctness. Of note, the :mod:`!distutils` package has been removed from the standard library. @@ -154,7 +154,7 @@ Important deprecations, removals or restrictions: reducing the size of every :class:`str` object by at least 8 bytes. * :pep:`632`: Remove the :mod:`!distutils` package. - See `the migration guide `_ + See :pep:`the migration guide <0632#migration-advice>` for advice replacing the APIs it provided. The third-party `Setuptools `__ package continues to provide :mod:`!distutils`, @@ -359,7 +359,7 @@ create an interpreter with its own GIL: /* The new interpreter is now active in the current thread. */ For further examples how to use the C-API for sub-interpreters with a -per-interpreter GIL, see :source:`Modules/_xxsubinterpretersmodule.c`. +per-interpreter GIL, see ``Modules/_xxsubinterpretersmodule.c``. (Contributed by Eric Snow in :gh:`104210`, etc.) @@ -652,14 +652,14 @@ asyncio making some use-cases 2x to 5x faster. (Contributed by Jacob Bower & Itamar Oren in :gh:`102853`, :gh:`104140`, and :gh:`104138`) -* On Linux, :mod:`asyncio` uses :class:`asyncio.PidfdChildWatcher` by default +* On Linux, :mod:`asyncio` uses :class:`!asyncio.PidfdChildWatcher` by default if :func:`os.pidfd_open` is available and functional instead of - :class:`asyncio.ThreadedChildWatcher`. + :class:`!asyncio.ThreadedChildWatcher`. (Contributed by Kumar Aditya in :gh:`98024`.) * The event loop now uses the best available child watcher for each platform - (:class:`asyncio.PidfdChildWatcher` if supported and - :class:`asyncio.ThreadedChildWatcher` otherwise), so manually + (:class:`!asyncio.PidfdChildWatcher` if supported and + :class:`!asyncio.ThreadedChildWatcher` otherwise), so manually configuring a child watcher is not recommended. (Contributed by Kumar Aditya in :gh:`94597`.) @@ -726,7 +726,7 @@ inspect * Add :func:`inspect.markcoroutinefunction` to mark sync functions that return a :term:`coroutine` for use with :func:`inspect.iscoroutinefunction`. - (Contributed Carlton Gibson in :gh:`99247`.) + (Contributed by Carlton Gibson in :gh:`99247`.) * Add :func:`inspect.getasyncgenstate` and :func:`inspect.getasyncgenlocals` for determining the current state of asynchronous generators. @@ -734,13 +734,12 @@ inspect * The performance of :func:`inspect.getattr_static` has been considerably improved. Most calls to the function should be at least 2x faster than they - were in Python 3.11, and some may be 6x faster or more. (Contributed by Alex - Waygood in :gh:`103193`.) + were in Python 3.11. (Contributed by Alex Waygood in :gh:`103193`.) itertools --------- -* Add :class:`itertools.batched()` for collecting into even-sized +* Add :func:`itertools.batched` for collecting into even-sized tuples where the last batch may be shorter than the rest. (Contributed by Raymond Hettinger in :gh:`98363`.) @@ -751,8 +750,8 @@ math (Contributed by Raymond Hettinger in :gh:`100485`.) * Extend :func:`math.nextafter` to include a *steps* argument - for moving up or down multiple steps at a time. - (By Matthias Goergens, Mark Dickinson, and Raymond Hettinger in :gh:`94906`.) + for moving up or down multiple steps at a time. (Contributed by + Matthias Goergens, Mark Dickinson, and Raymond Hettinger in :gh:`94906`.) os -- @@ -927,8 +926,6 @@ tempfile * :func:`tempfile.mkdtemp` now always returns an absolute path, even if the argument provided to the *dir* parameter is a relative path. -.. _whatsnew-typing-py312: - threading --------- @@ -963,6 +960,8 @@ types :ref:`user-defined-generics` when subclassed. (Contributed by James Hilton-Balfe and Alex Waygood in :gh:`101827`.) +.. _whatsnew-typing-py312: + typing ------ @@ -1006,8 +1005,8 @@ typing :func:`runtime-checkable protocols ` has changed significantly. Most ``isinstance()`` checks against protocols with only a few members should be at least 2x faster than in 3.11, and some may be 20x - faster or more. However, ``isinstance()`` checks against protocols with fourteen - or more members may be slower than in Python 3.11. (Contributed by Alex + faster or more. However, ``isinstance()`` checks against protocols with many + members may be slower than in Python 3.11. (Contributed by Alex Waygood in :gh:`74690` and :gh:`103193`.) * All :data:`typing.TypedDict` and :data:`typing.NamedTuple` classes now have the @@ -1124,7 +1123,7 @@ CPython bytecode changes * Add the :opcode:`LOAD_SUPER_ATTR` instruction. (Contributed by Carl Meyer and Vladimir Matveev in :gh:`103497`.) -* Add the :opcode:`RETURN_CONST` instruction. (Contributed by Wenyang Wang in :gh:`101632`.) +* Add the ``RETURN_CONST`` instruction. (Contributed by Wenyang Wang in :gh:`101632`.) Demos and Tools =============== @@ -1163,15 +1162,15 @@ Deprecated * :mod:`asyncio`: - * The child watcher classes :class:`asyncio.MultiLoopChildWatcher`, - :class:`asyncio.FastChildWatcher`, :class:`asyncio.AbstractChildWatcher` - and :class:`asyncio.SafeChildWatcher` are deprecated and + * The child watcher classes :class:`!asyncio.MultiLoopChildWatcher`, + :class:`!asyncio.FastChildWatcher`, :class:`!asyncio.AbstractChildWatcher` + and :class:`!asyncio.SafeChildWatcher` are deprecated and will be removed in Python 3.14. (Contributed by Kumar Aditya in :gh:`94597`.) - * :func:`asyncio.set_child_watcher`, :func:`asyncio.get_child_watcher`, - :meth:`asyncio.AbstractEventLoopPolicy.set_child_watcher` and - :meth:`asyncio.AbstractEventLoopPolicy.get_child_watcher` are deprecated + * :func:`!asyncio.set_child_watcher`, :func:`!asyncio.get_child_watcher`, + :meth:`!asyncio.AbstractEventLoopPolicy.set_child_watcher` and + :meth:`!asyncio.AbstractEventLoopPolicy.get_child_watcher` are deprecated and will be removed in Python 3.14. (Contributed by Kumar Aditya in :gh:`94597`.) @@ -1184,7 +1183,7 @@ Deprecated replaced by :data:`calendar.JANUARY` and :data:`calendar.FEBRUARY`. (Contributed by Prince Roshan in :gh:`103636`.) -* :mod:`collections.abc`: Deprecated :class:`collections.abc.ByteString`. +* :mod:`collections.abc`: Deprecated :class:`!collections.abc.ByteString`. Prefer :class:`Sequence` or :class:`collections.abc.Buffer`. For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. (Contributed by Shantanu Jain in :gh:`91896`.) @@ -1230,7 +1229,7 @@ Deprecated your code *requires* ``'fork'``. See :ref:`contexts and start methods `. -* :mod:`pkgutil`: :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` +* :mod:`pkgutil`: :func:`!pkgutil.find_loader` and :func:`!pkgutil.get_loader` are deprecated and will be removed in Python 3.14; use :func:`importlib.util.find_spec` instead. (Contributed by Nikita Sobolev in :gh:`97850`.) @@ -1252,10 +1251,10 @@ Deprecated :exc:`DeprecationWarning` when it can detect being called from a multithreaded process. There has always been a fundamental incompatibility with the POSIX platform when doing so. Even if such code *appeared* to work. - We added the warning to to raise awareness as issues encounted by code doing + We added the warning to raise awareness as issues encountered by code doing this are becoming more frequent. See the :func:`os.fork` documentation for more details along with `this discussion on fork being incompatible with threads - `_ for *why* we're now surfacing this + `_ for *why* we're now surfacing this longstanding platform compatibility problem to developers. When this warning appears due to usage of :mod:`multiprocessing` or @@ -1294,7 +1293,7 @@ Deprecated :class:`collections.abc.Hashable` and :class:`collections.abc.Sized` respectively, are deprecated. (:gh:`94309`.) - * :class:`typing.ByteString`, deprecated since Python 3.9, now causes a + * :class:`!typing.ByteString`, deprecated since Python 3.9, now causes a :exc:`DeprecationWarning` to be emitted when it is used. (Contributed by Alex Waygood in :gh:`91896`.) @@ -1310,173 +1309,39 @@ Deprecated may be removed in a future version of Python. Use the single-arg versions of these functions instead. (Contributed by Ofey Chan in :gh:`89874`.) -* :exc:`DeprecationWarning` is now raised when ``__package__`` on a - module differs from ``__spec__.parent`` (previously it was - :exc:`ImportWarning`). +* :exc:`DeprecationWarning` is now raised when :attr:`~module.__package__` on a + module differs from + :attr:`__spec__.parent ` (previously + it was :exc:`ImportWarning`). (Contributed by Brett Cannon in :gh:`65961`.) -* Setting ``__package__`` or ``__cached__`` on a module is deprecated, - and will cease to be set or taken into consideration by the import system in Python 3.14. - (Contributed by Brett Cannon in :gh:`65961`.) +* Setting :attr:`~module.__package__` or :attr:`~module.__cached__` on a + module is deprecated, and will cease to be set or taken into consideration by + the import system in Python 3.14. (Contributed by Brett Cannon in :gh:`65961`.) * The bitwise inversion operator (``~``) on bool is deprecated. It will throw an - error in Python 3.14. Use ``not`` for logical negation of bools instead. + error in Python 3.16. Use ``not`` for logical negation of bools instead. In the rare case that you really need the bitwise inversion of the underlying ``int``, convert to int explicitly: ``~int(x)``. (Contributed by Tim Hoffmann in :gh:`103487`.) * Accessing :attr:`~codeobject.co_lnotab` on code objects was deprecated in Python 3.10 via :pep:`626`, - but it only got a proper :exc:`DeprecationWarning` in 3.12, - therefore it will be removed in 3.14. + but it only got a proper :exc:`DeprecationWarning` in 3.12. + May be removed in 3.15. (Contributed by Nikita Sobolev in :gh:`101866`.) -Pending Removal in Python 3.13 ------------------------------- - -The following modules and APIs have been deprecated in earlier Python releases, -and will be removed in Python 3.13. - -Modules (see :pep:`594`): - -* :mod:`!aifc` -* :mod:`!audioop` -* :mod:`!cgi` -* :mod:`!cgitb` -* :mod:`!chunk` -* :mod:`!crypt` -* :mod:`!imghdr` -* :mod:`!mailcap` -* :mod:`!msilib` -* :mod:`!nis` -* :mod:`!nntplib` -* :mod:`!ossaudiodev` -* :mod:`!pipes` -* :mod:`!sndhdr` -* :mod:`!spwd` -* :mod:`!sunau` -* :mod:`!telnetlib` -* :mod:`!uu` -* :mod:`!xdrlib` - -Other modules: - -* :mod:`!lib2to3`, and the :program:`2to3` program (:gh:`84540`) - -APIs: - -* :class:`!configparser.LegacyInterpolation` (:gh:`90765`) -* ``locale.resetlocale()`` (:gh:`90817`) -* :meth:`!turtle.RawTurtle.settiltangle` (:gh:`50096`) -* :func:`!unittest.findTestCases` (:gh:`50096`) -* :func:`!unittest.getTestCaseNames` (:gh:`50096`) -* :func:`!unittest.makeSuite` (:gh:`50096`) -* :meth:`!unittest.TestProgram.usageExit` (:gh:`67048`) -* :class:`!webbrowser.MacOSX` (:gh:`86421`) -* :class:`classmethod` descriptor chaining (:gh:`89519`) -* :mod:`importlib.resources` deprecated methods: - - * ``contents()`` - * ``is_resource()`` - * ``open_binary()`` - * ``open_text()`` - * ``path()`` - * ``read_binary()`` - * ``read_text()`` - - Use :func:`importlib.resources.files()` instead. Refer to `importlib-resources: Migrating from Legacy - `_ (:gh:`106531`) - -Pending Removal in Python 3.14 ------------------------------- - -The following APIs have been deprecated -and will be removed in Python 3.14. - -* :mod:`argparse`: The *type*, *choices*, and *metavar* parameters - of :class:`!argparse.BooleanOptionalAction` - -* :mod:`ast`: - - * :class:`!ast.Num` - * :class:`!ast.Str` - * :class:`!ast.Bytes` - * :class:`!ast.NameConstant` - * :class:`!ast.Ellipsis` - -* :mod:`asyncio`: - - * :class:`!asyncio.MultiLoopChildWatcher` - * :class:`!asyncio.FastChildWatcher` - * :class:`!asyncio.AbstractChildWatcher` - * :class:`!asyncio.SafeChildWatcher` - * :func:`!asyncio.set_child_watcher` - * :func:`!asyncio.get_child_watcher`, - * :meth:`!asyncio.AbstractEventLoopPolicy.set_child_watcher` - * :meth:`!asyncio.AbstractEventLoopPolicy.get_child_watcher` - -* :mod:`collections.abc`: :class:`!collections.abc.ByteString`. - -* :mod:`email`: the *isdst* parameter in :func:`email.utils.localtime`. - -* :mod:`importlib.abc`: - - * :class:`!importlib.abc.ResourceReader` - * :class:`!importlib.abc.Traversable` - * :class:`!importlib.abc.TraversableResources` - -* :mod:`itertools`: Support for copy, deepcopy, and pickle operations. +.. include:: ../deprecations/pending-removal-in-3.13.rst -* :mod:`pkgutil`: +.. include:: ../deprecations/pending-removal-in-3.14.rst - * :func:`!pkgutil.find_loader` - * :func:`!pkgutil.get_loader`. - -* :mod:`pty`: - - * :func:`!pty.master_open` - * :func:`!pty.slave_open` - -* :mod:`shutil`: The *onerror* argument of :func:`shutil.rmtree` - -* :mod:`typing`: :class:`!typing.ByteString` - -* :mod:`xml.etree.ElementTree`: Testing the truth value of an :class:`xml.etree.ElementTree.Element`. - -* The ``__package__`` and ``__cached__`` attributes on module objects. - -* The :attr:`~codeobject.co_lnotab` attribute of code objects. - -Pending Removal in Python 3.15 ------------------------------- +.. include:: ../deprecations/pending-removal-in-3.15.rst -The following APIs have been deprecated -and will be removed in Python 3.15. +.. include:: ../deprecations/pending-removal-in-3.16.rst -APIs: - -* :func:`locale.getdefaultlocale` (:gh:`90817`) - - -Pending Removal in Future Versions ----------------------------------- - -The following APIs were deprecated in earlier Python versions and will be removed, -although there is currently no date scheduled for their removal. - -* :mod:`array`'s ``'u'`` format code (:gh:`57281`) - -* :class:`typing.Text` (:gh:`92332`) - -* Currently Python accepts numeric literals immediately followed by keywords, - for example ``0in x``, ``1or x``, ``0if 1else 2``. It allows confusing - and ambiguous expressions like ``[0x1for x in y]`` (which can be - interpreted as ``[0x1 for x in y]`` or ``[0x1f or x in y]``). - A syntax warning is raised if the numeric literal is - immediately followed by one of keywords :keyword:`and`, :keyword:`else`, - :keyword:`for`, :keyword:`if`, :keyword:`in`, :keyword:`is` and :keyword:`or`. - In a future release it will be changed to a syntax error. (:gh:`87999`) +.. include:: ../deprecations/pending-removal-in-future.rst +.. _whatsnew312-removed: Removed ======= @@ -1503,6 +1368,8 @@ configparser * :class:`configparser.ConfigParser` no longer has a ``readfp`` method. Use :meth:`~configparser.ConfigParser.read_file` instead. +.. _whatsnew312-removed-distutils: + distutils --------- @@ -1560,9 +1427,9 @@ hashlib ------- * Remove the pure Python implementation of :mod:`hashlib`'s - :func:`hashlib.pbkdf2_hmac()`, deprecated in Python 3.10. Python 3.10 and + :func:`hashlib.pbkdf2_hmac`, deprecated in Python 3.10. Python 3.10 and newer requires OpenSSL 1.1.1 (:pep:`644`): this OpenSSL version provides - a C implementation of :func:`~hashlib.pbkdf2_hmac()` which is faster. + a C implementation of :func:`~hashlib.pbkdf2_hmac` which is faster. (Contributed by Victor Stinner in :gh:`94199`.) importlib @@ -1571,7 +1438,7 @@ importlib * Many previously deprecated cleanups in :mod:`importlib` have now been completed: - * References to, and support for :meth:`!module_repr()` has been removed. + * References to, and support for :meth:`!module_repr` has been removed. (Contributed by Barry Warsaw in :gh:`97850`.) * ``importlib.util.set_package``, ``importlib.util.set_loader`` and @@ -1584,6 +1451,8 @@ importlib * ``importlib.abc.Finder``, ``pkgutil.ImpImporter``, and ``pkgutil.ImpLoader`` have been removed. (Contributed by Barry Warsaw in :gh:`98040`.) +.. _whatsnew312-removed-imp: + imp --- @@ -1659,12 +1528,10 @@ smtpd * The ``smtpd`` module has been removed according to the schedule in :pep:`594`, having been deprecated in Python 3.4.7 and 3.5.4. - Use aiosmtpd_ PyPI module or any other + Use the :pypi:`aiosmtpd` PyPI module or any other :mod:`asyncio`-based server instead. (Contributed by Oleg Iarygin in :gh:`93243`.) -.. _aiosmtpd: https://pypi.org/project/aiosmtpd/ - sqlite3 ------- @@ -1701,9 +1568,8 @@ ssl instead, create a :class:`ssl.SSLContext` object and call its :class:`ssl.SSLContext.wrap_socket` method. Any package that still uses :func:`!ssl.wrap_socket` is broken and insecure. The function neither sends a - SNI TLS extension nor validates server hostname. Code is subject to `CWE-295 - `_: Improper Certificate - Validation. + SNI TLS extension nor validates the server hostname. Code is subject to :cwe:`295` + (Improper Certificate Validation). (Contributed by Victor Stinner in :gh:`94199`.) unittest @@ -1743,7 +1609,7 @@ unittest * Undocumented :meth:`TestLoader.loadTestsFromModule ` parameter *use_load_tests* - (deprecated and ignored since Python 3.2). + (deprecated and ignored since Python 3.5). * An alias of the :class:`~unittest.TextTestResult` class: ``_TextTestResult`` (deprecated in Python 3.2). @@ -1832,7 +1698,7 @@ Changes in the Python API * Remove the ``asyncore``-based ``smtpd`` module deprecated in Python 3.4.7 and 3.5.4. A recommended replacement is the - :mod:`asyncio`-based aiosmtpd_ PyPI module. + :mod:`asyncio`-based :pypi:`aiosmtpd` PyPI module. * :func:`shlex.split`: Passing ``None`` for *s* argument now raises an exception, rather than reading :data:`sys.stdin`. The feature was deprecated @@ -2028,7 +1894,7 @@ New Features The :c:macro:`Py_TPFLAGS_MANAGED_DICT` and :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` flags have been added. This allows extensions classes to support object - ``__dict__`` and weakrefs with less bookkeeping, + :attr:`~object.__dict__` and weakrefs with less bookkeeping, using less memory and with faster access. * API for performing calls using @@ -2147,7 +2013,7 @@ Porting to Python 3.12 internal-only field directly. To get a list of subclasses, call the Python method - :py:meth:`~class.__subclasses__` (using :c:func:`PyObject_CallMethod`, + :py:meth:`~type.__subclasses__` (using :c:func:`PyObject_CallMethod`, for example). * Add support of more formatting options (left aligning, octals, uppercase @@ -2166,7 +2032,7 @@ Porting to Python 3.12 :c:func:`PyUnicode_FromFormatV`. (Contributed by Philip Georgi in :gh:`95504`.) -* Extension classes wanting to add a ``__dict__`` or weak reference slot +* Extension classes wanting to add a :attr:`~object.__dict__` or weak reference slot should use :c:macro:`Py_TPFLAGS_MANAGED_DICT` and :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead of ``tp_dictoffset`` and ``tp_weaklistoffset``, respectively. @@ -2351,92 +2217,13 @@ Deprecated overrides :c:member:`~PyTypeObject.tp_new` is deprecated. Call the metaclass instead. -Pending Removal in Python 3.14 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. Add deprecations above alphabetically, not here at the end. -* The ``ma_version_tag`` field in :c:type:`PyDictObject` for extension modules - (:pep:`699`; :gh:`101193`). +.. include:: ../deprecations/c-api-pending-removal-in-3.14.rst -* Global configuration variables: +.. include:: ../deprecations/c-api-pending-removal-in-3.15.rst - * :c:var:`Py_DebugFlag`: use :c:member:`PyConfig.parser_debug` - * :c:var:`Py_VerboseFlag`: use :c:member:`PyConfig.verbose` - * :c:var:`Py_QuietFlag`: use :c:member:`PyConfig.quiet` - * :c:var:`Py_InteractiveFlag`: use :c:member:`PyConfig.interactive` - * :c:var:`Py_InspectFlag`: use :c:member:`PyConfig.inspect` - * :c:var:`Py_OptimizeFlag`: use :c:member:`PyConfig.optimization_level` - * :c:var:`Py_NoSiteFlag`: use :c:member:`PyConfig.site_import` - * :c:var:`Py_BytesWarningFlag`: use :c:member:`PyConfig.bytes_warning` - * :c:var:`Py_FrozenFlag`: use :c:member:`PyConfig.pathconfig_warnings` - * :c:var:`Py_IgnoreEnvironmentFlag`: use :c:member:`PyConfig.use_environment` - * :c:var:`Py_DontWriteBytecodeFlag`: use :c:member:`PyConfig.write_bytecode` - * :c:var:`Py_NoUserSiteDirectory`: use :c:member:`PyConfig.user_site_directory` - * :c:var:`Py_UnbufferedStdioFlag`: use :c:member:`PyConfig.buffered_stdio` - * :c:var:`Py_HashRandomizationFlag`: use :c:member:`PyConfig.use_hash_seed` - and :c:member:`PyConfig.hash_seed` - * :c:var:`Py_IsolatedFlag`: use :c:member:`PyConfig.isolated` - * :c:var:`Py_LegacyWindowsFSEncodingFlag`: use :c:member:`PyPreConfig.legacy_windows_fs_encoding` - * :c:var:`Py_LegacyWindowsStdioFlag`: use :c:member:`PyConfig.legacy_windows_stdio` - * :c:var:`!Py_FileSystemDefaultEncoding`: use :c:member:`PyConfig.filesystem_encoding` - * :c:var:`!Py_HasFileSystemDefaultEncoding`: use :c:member:`PyConfig.filesystem_encoding` - * :c:var:`!Py_FileSystemDefaultEncodeErrors`: use :c:member:`PyConfig.filesystem_errors` - * :c:var:`!Py_UTF8Mode`: use :c:member:`PyPreConfig.utf8_mode` (see :c:func:`Py_PreInitialize`) - - The :c:func:`Py_InitializeFromConfig` API should be used with - :c:type:`PyConfig` instead. - -* Creating :c:data:`immutable types ` with mutable - bases (:gh:`95388`). - -Pending Removal in Python 3.15 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -* :c:func:`PyImport_ImportModuleNoBlock`: use :c:func:`PyImport_ImportModule` -* :c:type:`!Py_UNICODE_WIDE` type: use :c:type:`wchar_t` -* :c:type:`Py_UNICODE` type: use :c:type:`wchar_t` -* Python initialization functions: - - * :c:func:`PySys_ResetWarnOptions`: clear :data:`sys.warnoptions` and - :data:`!warnings.filters` - * :c:func:`Py_GetExecPrefix`: get :data:`sys.exec_prefix` - * :c:func:`Py_GetPath`: get :data:`sys.path` - * :c:func:`Py_GetPrefix`: get :data:`sys.prefix` - * :c:func:`Py_GetProgramFullPath`: get :data:`sys.executable` - * :c:func:`Py_GetProgramName`: get :data:`sys.executable` - * :c:func:`Py_GetPythonHome`: get :c:member:`PyConfig.home` or - the :envvar:`PYTHONHOME` environment variable - -Pending Removal in Future Versions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The following APIs are deprecated and will be removed, -although there is currently no date scheduled for their removal. - -* :c:macro:`Py_TPFLAGS_HAVE_FINALIZE`: unneeded since Python 3.8 -* :c:func:`PyErr_Fetch`: use :c:func:`PyErr_GetRaisedException` -* :c:func:`PyErr_NormalizeException`: use :c:func:`PyErr_GetRaisedException` -* :c:func:`PyErr_Restore`: use :c:func:`PyErr_SetRaisedException` -* :c:func:`PyModule_GetFilename`: use :c:func:`PyModule_GetFilenameObject` -* :c:func:`PyOS_AfterFork`: use :c:func:`PyOS_AfterFork_Child` -* :c:func:`PySlice_GetIndicesEx`: use :c:func:`PySlice_Unpack` and :c:func:`PySlice_AdjustIndices` -* :c:func:`!PyUnicode_AsDecodedObject`: use :c:func:`PyCodec_Decode` -* :c:func:`!PyUnicode_AsDecodedUnicode`: use :c:func:`PyCodec_Decode` -* :c:func:`!PyUnicode_AsEncodedObject`: use :c:func:`PyCodec_Encode` -* :c:func:`!PyUnicode_AsEncodedUnicode`: use :c:func:`PyCodec_Encode` -* :c:func:`PyUnicode_READY`: unneeded since Python 3.12 -* :c:func:`!PyErr_Display`: use :c:func:`PyErr_DisplayException` -* :c:func:`!_PyErr_ChainExceptions`: use ``_PyErr_ChainExceptions1`` -* :c:member:`!PyBytesObject.ob_shash` member: - call :c:func:`PyObject_Hash` instead -* :c:member:`!PyDictObject.ma_version_tag` member -* Thread Local Storage (TLS) API: - - * :c:func:`PyThread_create_key`: use :c:func:`PyThread_tss_alloc` - * :c:func:`PyThread_delete_key`: use :c:func:`PyThread_tss_free` - * :c:func:`PyThread_set_key_value`: use :c:func:`PyThread_tss_set` - * :c:func:`PyThread_get_key_value`: use :c:func:`PyThread_tss_get` - * :c:func:`PyThread_delete_key_value`: use :c:func:`PyThread_tss_delete` - * :c:func:`PyThread_ReInitTLS`: unneeded since Python 3.7 +.. include:: ../deprecations/c-api-pending-removal-in-future.rst Removed ------- diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 96c8aee5da075a..f2a8c711f4df88 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -3,7 +3,7 @@ What's New In Python 3.13 **************************** -:Editor: TBD +:Editors: Adam Turner and Thomas Wouters .. Rules for maintenance: @@ -46,492 +46,1251 @@ when researching a change. This article explains the new features in Python 3.13, compared to 3.12. - +Python 3.13 was released on October 7, 2024. For full details, see the :ref:`changelog `. -.. note:: +.. seealso:: - Prerelease users should be aware that this document is currently in draft - form. It will be updated substantially as Python 3.13 moves towards release, - so it's worth checking back even after reading earlier versions. + :pep:`719` -- Python 3.13 Release Schedule -Summary -- Release highlights +Summary -- Release Highlights ============================= .. This section singles out the most important changes in Python 3.13. Brevity is key. +Python 3.13 is the latest stable release of the Python programming +language, with a mix of changes to the language, the implementation +and the standard library. +The biggest changes include a new `interactive interpreter +`_, +experimental support for running in a `free-threaded mode +`_ (:pep:`703`), +and a `Just-In-Time compiler `_ (:pep:`744`). + +Error messages continue to improve, with tracebacks now highlighted in color +by default. The :func:`locals` builtin now has :ref:`defined semantics +` for changing the returned mapping, +and type parameters now support default values. + +The library changes contain removal of deprecated APIs and modules, +as well as the usual improvements in user-friendliness and correctness. +Several legacy standard library modules have now `been removed +`_ following their deprecation in Python 3.11 (:pep:`594`). + +This article doesn't attempt to provide a complete specification +of all new features, but instead gives a convenient overview. +For full details refer to the documentation, +such as the :ref:`Library Reference ` +and :ref:`Language Reference `. +To understand the complete implementation and design rationale for a change, +refer to the PEP for a particular new feature; +but note that PEPs usually are not kept up-to-date +once a feature has been fully implemented. +See `Porting to Python 3.13`_ for guidance on upgrading from +earlier versions of Python. + +-------------- .. PEP-sized items next. -Important deprecations, removals or restrictions: +Interpreter improvements: + +* A greatly improved :ref:`interactive interpreter + ` and + :ref:`improved error messages `. +* :pep:`667`: The :func:`locals` builtin now has + :ref:`defined semantics ` when mutating the + returned mapping. Python debuggers and similar tools may now more reliably + update local variables in optimized scopes even during concurrent code + execution. +* :pep:`703`: CPython 3.13 has experimental support for running with the + :term:`global interpreter lock` disabled. See :ref:`Free-threaded CPython + ` for more details. +* :pep:`744`: A basic :ref:`JIT compiler ` was added. + It is currently disabled by default (though we may turn it on later). + Performance improvements are modest -- we expect to improve this + over the next few releases. +* Color support in the new :ref:`interactive interpreter + `, + as well as in :ref:`tracebacks ` + and :ref:`doctest ` output. + This can be disabled through the :envvar:`PYTHON_COLORS` and |NO_COLOR|_ + environment variables. + +Python data model improvements: + +* :attr:`~type.__static_attributes__` stores the names of attributes accessed + through ``self.X`` in any function in a class body. +* :attr:`~type.__firstlineno__` records the first line number of a class + definition. + +Significant improvements in the standard library: + +* Add a new :exc:`PythonFinalizationError` exception, raised when an operation + is blocked during :term:`finalization `. +* The :mod:`argparse` module now supports deprecating command-line options, + positional arguments, and subcommands. +* The new functions :func:`base64.z85encode` and :func:`base64.z85decode` + support encoding and decoding `Z85 data`_. +* The :mod:`copy` module now has a :func:`copy.replace` function, + with support for many builtin types and any class defining + the :func:`~object.__replace__` method. +* The new :mod:`dbm.sqlite3` module is now the default :mod:`dbm` backend. +* The :mod:`os` module has a :ref:`suite of new functions ` + for working with Linux's timer notification file descriptors. +* The :mod:`random` module now has a :ref:`command-line interface `. + +Security improvements: + +* :func:`ssl.create_default_context` sets :data:`ssl.VERIFY_X509_PARTIAL_CHAIN` + and :data:`ssl.VERIFY_X509_STRICT` as default flags. + +C API improvements: + +* The :c:data:`Py_mod_gil` slot is now used to indicate that + an extension module supports running with the :term:`GIL` disabled. +* The :doc:`PyTime C API ` has been added, + providing access to system clocks. +* :c:type:`PyMutex` is a new lightweight mutex that occupies a single byte. +* There is a new :ref:`suite of functions ` + for generating :pep:`669` monitoring events in the C API. + +New typing features: + +* :pep:`696`: Type parameters (:data:`typing.TypeVar`, :data:`typing.ParamSpec`, + and :data:`typing.TypeVarTuple`) now support defaults. +* :pep:`702`: The new :func:`warnings.deprecated` decorator adds support + for marking deprecations in the type system and at runtime. +* :pep:`705`: :data:`typing.ReadOnly` can be used to mark an item of a + :class:`typing.TypedDict` as read-only for type checkers. +* :pep:`742`: :data:`typing.TypeIs` provides more intuitive + type narrowing behavior, as an alternative to :data:`typing.TypeGuard`. + +Platform support: + +* :pep:`730`: Apple's iOS is now an :ref:`officially supported platform + `, at :pep:`tier 3 <11#tier-3>`. +* :pep:`738`: Android is now an :ref:`officially supported platform + `, at :pep:`tier 3 <11#tier-3>`. +* ``wasm32-wasi`` is now supported as a :pep:`tier 2 <11#tier-2>` platform. +* ``wasm32-emscripten`` is no longer an officially supported platform. + +Important removals: * :ref:`PEP 594 `: The remaining 19 "dead batteries" - have been removed from the standard library: + (legacy stdlib modules) have been removed from the standard library: :mod:`!aifc`, :mod:`!audioop`, :mod:`!cgi`, :mod:`!cgitb`, :mod:`!chunk`, :mod:`!crypt`, :mod:`!imghdr`, :mod:`!mailcap`, :mod:`!msilib`, :mod:`!nis`, - :mod:`!nntplib`, :mod:`!ossaudiodev`, :mod:`!pipes`, :mod:`!sndhdr`, :mod:`!spwd`, - :mod:`!sunau`, :mod:`!telnetlib`, :mod:`!uu` and :mod:`!xdrlib`. - -* :pep:`602` ("Annual Release Cycle for Python") has been updated: + :mod:`!nntplib`, :mod:`!ossaudiodev`, :mod:`!pipes`, :mod:`!sndhdr`, + :mod:`!spwd`, :mod:`!sunau`, :mod:`!telnetlib`, :mod:`!uu` and :mod:`!xdrlib`. +* Remove the :program:`2to3` tool and :mod:`!lib2to3` module + (deprecated in Python 3.11). +* Remove the :mod:`!tkinter.tix` module (deprecated in Python 3.6). +* Remove the :func:`!locale.resetlocale` function. +* Remove the :mod:`!typing.io` and :mod:`!typing.re` namespaces. +* Remove chained :class:`classmethod` descriptors. - * Python 3.9 - 3.12 have one and a half years of full support, - followed by three and a half years of security fixes. - * Python 3.13 and later have two years of full support, - followed by three years of security fixes. +Release schedule changes: -Interpreter improvements: +:pep:`602` ("Annual Release Cycle for Python") has been updated +to extend the full support ('bugfix') period for new releases to two years. +This updated policy means that: -* A basic :ref:`JIT compiler ` was added. - It is currently disabled by default (though we may turn it on later). - Performance improvements are modest -- we expect to be improving this - over the next few releases. +* Python 3.9--3.12 have one and a half years of full support, + followed by three and a half years of security fixes. +* Python 3.13 and later have two years of full support, + followed by three years of security fixes. New Features ============ -Improved Error Messages + +.. _whatsnew313-better-interactive-interpreter: + +A better interactive interpreter +-------------------------------- + +Python now uses a new :term:`interactive` shell by default, based on code +from the `PyPy project`_. +When the user starts the :term:`REPL` from an interactive terminal, +the following new features are now supported: + +* Multiline editing with history preservation. +* Direct support for REPL-specific commands like :kbd:`help`, :kbd:`exit`, + and :kbd:`quit`, without the need to call them as functions. +* Prompts and tracebacks with :ref:`color enabled by default + `. +* Interactive help browsing using :kbd:`F1` with a separate command + history. +* History browsing using :kbd:`F2` that skips output as well as the + :term:`>>>` and :term:`...` prompts. +* "Paste mode" with :kbd:`F3` that makes pasting larger blocks of code + easier (press :kbd:`F3` again to return to the regular prompt). + +To disable the new interactive shell, +set the :envvar:`PYTHON_BASIC_REPL` environment variable. +For more on interactive mode, see :ref:`tut-interac`. + +(Contributed by Pablo Galindo Salgado, Łukasz Langa, and +Lysandros Nikolaou in :gh:`111201` based on code from the PyPy project. +Windows support contributed by Dino Viehland and Anthony Shaw.) + +.. _`PyPy project`: https://pypy.org/ + + +.. _whatsnew313-improved-error-messages: + +Improved error messages ----------------------- -* The interpreter now colorizes error messages when displaying tracebacks by default. - This feature can be controlled via the new :envvar:`PYTHON_COLORS` environment - variable as well as the canonical ``NO_COLOR`` and ``FORCE_COLOR`` environment - variables. See also :ref:`using-on-controlling-color`. +* The interpreter now uses color by default when displaying tracebacks in the + terminal. This feature :ref:`can be controlled ` + via the new :envvar:`PYTHON_COLORS` environment variable as well as + the canonical |NO_COLOR|_ and |FORCE_COLOR|_ environment variables. (Contributed by Pablo Galindo Salgado in :gh:`112730`.) -* When an incorrect keyword argument is passed to a function, the error message - now potentially suggests the correct keyword argument. +* A common mistake is to write a script with the same name as a + standard library module. When this results in errors, we now + display a more helpful error message: + + .. code-block:: pytb + + $ python random.py + Traceback (most recent call last): + File "/home/me/random.py", line 1, in + import random + File "/home/me/random.py", line 3, in + print(random.randint(5)) + ^^^^^^^^^^^^^^ + AttributeError: module 'random' has no attribute 'randint' (consider renaming '/home/me/random.py' since it has the same name as the standard library module named 'random' and prevents importing that standard library module) + + Similarly, if a script has the same name as a third-party + module that it attempts to import and this results in errors, + we also display a more helpful error message: + + .. code-block:: pytb + + $ python numpy.py + Traceback (most recent call last): + File "/home/me/numpy.py", line 1, in + import numpy as np + File "/home/me/numpy.py", line 3, in + np.array([1, 2, 3]) + ^^^^^^^^ + AttributeError: module 'numpy' has no attribute 'array' (consider renaming '/home/me/numpy.py' if it has the same name as a library you intended to import) + + (Contributed by Shantanu Jain in :gh:`95754`.) + +* The error message now tries to suggest the correct keyword argument + when an incorrect keyword argument is passed to a function. + + .. code-block:: pycon + + >>> "Better error messages!".split(max_split=1) + Traceback (most recent call last): + File "", line 1, in + "Better error messages!".split(max_split=1) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^ + TypeError: split() got an unexpected keyword argument 'max_split'. Did you mean 'maxsplit'? + (Contributed by Pablo Galindo Salgado and Shantanu Jain in :gh:`107944`.) - >>> "better error messages!".split(max_split=1) - Traceback (most recent call last): - File "", line 1, in - "better error messages!".split(max_split=1) - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^ - TypeError: split() got an unexpected keyword argument 'max_split'. Did you mean 'maxsplit'? + +.. _whatsnew313-free-threaded-cpython: + +Free-threaded CPython +--------------------- + +CPython now has experimental support for running in a free-threaded mode, +with the :term:`global interpreter lock` (GIL) disabled. +This is an experimental feature and therefore is not enabled by default. +The free-threaded mode requires a different executable, +usually called ``python3.13t`` or ``python3.13t.exe``. +Pre-built binaries marked as *free-threaded* can be installed as part of +the official :ref:`Windows ` +and :ref:`macOS ` installers, +or CPython can be built from source with the :option:`--disable-gil` option. + +Free-threaded execution allows for full utilization of the available +processing power by running threads in parallel on available CPU cores. +While not all software will benefit from this automatically, programs +designed with threading in mind will run faster on multi-core hardware. +**The free-threaded mode is experimental** and work is ongoing to improve it: +expect some bugs and a substantial single-threaded performance hit. +Free-threaded builds of CPython support optionally running with the GIL +enabled at runtime using the environment variable :envvar:`PYTHON_GIL` or +the command-line option :option:`-X gil=1`. + +To check if the current interpreter supports free-threading, :option:`python -VV <-V>` +and :attr:`sys.version` contain "experimental free-threading build". +The new :func:`!sys._is_gil_enabled` function can be used to check whether +the GIL is actually disabled in the running process. + +C-API extension modules need to be built specifically for the free-threaded +build. Extensions that support running with the :term:`GIL` disabled should +use the :c:data:`Py_mod_gil` slot. Extensions using single-phase init should +use :c:func:`PyUnstable_Module_SetGIL` to indicate whether they support +running with the GIL disabled. Importing C extensions that don't use these +mechanisms will cause the GIL to be enabled, unless the GIL was explicitly +disabled with the :envvar:`PYTHON_GIL` environment variable or the +:option:`-X gil=0` option. +pip 24.1 or newer is required to install packages with C extensions in the +free-threaded build. + +This work was made possible thanks to many individuals and +organizations, including the large community of contributors to Python +and third-party projects to test and enable free-threading support. +Notable contributors include: +Sam Gross, Ken Jin, Donghee Na, Itamar Oren, Matt Page, Brett Simmers, +Dino Viehland, Carl Meyer, Nathan Goldbaum, Ralf Gommers, +Lysandros Nikolaou, and many others. +Many of these contributors are employed by Meta, which has +provided significant engineering resources to support this project. + +.. seealso:: + + :pep:`703` "Making the Global Interpreter Lock Optional in CPython" + contains rationale and information surrounding this work. + + `Porting Extension Modules to Support Free-Threading + `_: A community-maintained + porting guide for extension authors. + + +.. _whatsnew313-jit-compiler: + +An experimental just-in-time (JIT) compiler +------------------------------------------- + +When CPython is configured and built using +the :option:`!--enable-experimental-jit` option, +a just-in-time (JIT) compiler is added which may speed up some Python programs. +On Windows, use ``PCbuild/build.bat --experimental-jit`` to enable the JIT +or ``--experimental-jit-interpreter`` to enable the Tier 2 interpreter. +Build requirements and further supporting information `are contained at`__ +:file:`Tools/jit/README.md`. + +__ https://github.com/python/cpython/blob/main/Tools/jit/README.md + +The :option:`!--enable-experimental-jit` option takes these (optional) values, +defaulting to ``yes`` if :option:`!--enable-experimental-jit` is present +without the optional value. + +* ``no``: Disable the entire Tier 2 and JIT pipeline. +* ``yes``: Enable the JIT. + To disable the JIT at runtime, pass the environment variable ``PYTHON_JIT=0``. +* ``yes-off``: Build the JIT but disable it by default. + To enable the JIT at runtime, pass the environment variable ``PYTHON_JIT=1``. +* ``interpreter``: Enable the Tier 2 interpreter but disable the JIT. + The interpreter can be disabled by running with ``PYTHON_JIT=0``. + +The internal architecture is roughly as follows: + +* We start with specialized *Tier 1 bytecode*. + See :ref:`What's new in 3.11 ` for details. +* When the Tier 1 bytecode gets hot enough, it gets translated + to a new purely internal intermediate representation (IR), + called the *Tier 2 IR*, and sometimes referred to as micro-ops ("uops"). +* The Tier 2 IR uses the same stack-based virtual machine as Tier 1, + but the instruction format is better suited to translation to machine code. +* We have several optimization passes for Tier 2 IR, which are applied + before it is interpreted or translated to machine code. +* There is a Tier 2 interpreter, but it is mostly intended for debugging + the earlier stages of the optimization pipeline. + The Tier 2 interpreter can be enabled by configuring Python + with ``--enable-experimental-jit=interpreter``. +* When the JIT is enabled, the optimized + Tier 2 IR is translated to machine code, which is then executed. +* The machine code translation process uses a technique called + *copy-and-patch*. It has no runtime dependencies, but there is a new + build-time dependency on LLVM. + +.. seealso:: :pep:`744` + +(JIT by Brandt Bucher, inspired by a paper by Haoran Xu and Fredrik Kjolstad. +Tier 2 IR by Mark Shannon and Guido van Rossum. +Tier 2 optimizer by Ken Jin.) + + +.. _whatsnew313-locals-semantics: + +Defined mutation semantics for :py:func:`locals` +------------------------------------------------ + +Historically, the expected result of mutating the return value of +:func:`locals` has been left to individual Python implementations to define. +Starting from Python 3.13, :pep:`667` standardises +the historical behavior of CPython for most code execution scopes, +but changes :term:`optimized scopes ` +(functions, generators, coroutines, comprehensions, and generator expressions) +to explicitly return independent snapshots of the currently assigned local +variables, including locally referenced nonlocal variables captured in closures. + +This change to the semantics of :func:`locals` in optimized scopes also +affects the default behavior of code execution functions that implicitly +target :func:`!locals` if no explicit namespace is provided +(such as :func:`exec` and :func:`eval`). +In previous versions, whether or not changes could be accessed by calling +:func:`!locals` after calling the code execution function was +implementation-dependent. In CPython specifically, such code would typically +appear to work as desired, but could sometimes fail in optimized scopes based +on other code (including debuggers and code execution tracing tools) +potentially resetting the shared snapshot in that scope. +Now, the code will always run against an independent snapshot of +the local variables in optimized scopes, and hence the changes will never +be visible in subsequent calls to :func:`!locals`. +To access the changes made in these cases, an explicit namespace reference +must now be passed to the relevant function. +Alternatively, it may make sense to update affected code to use a higher level +code execution API that returns the resulting code execution namespace +(e.g. :func:`runpy.run_path` when executing Python files from disk). + +To ensure debuggers and similar tools can reliably update local variables in +scopes affected by this change, :attr:`FrameType.f_locals ` now +returns a write-through proxy to the frame's local and locally referenced +nonlocal variables in these scopes, rather than returning an inconsistently +updated shared ``dict`` instance with undefined runtime semantics. + +See :pep:`667` for more details, including related C API changes +and deprecations. Porting notes are also provided below for the affected +:ref:`Python APIs ` and :ref:`C APIs +`. + +(PEP and implementation contributed by Mark Shannon and Tian Gao in +:gh:`74929`. Documentation updates provided by Guido van Rossum and +Alyssa Coghlan.) + + +.. _whatsnew313-platform-support: + +Support for mobile platforms +---------------------------- + +:pep:`730`: iOS is now a :pep:`11` supported platform, with the +``arm64-apple-ios`` and ``arm64-apple-ios-simulator`` targets at tier 3 +(iPhone and iPad devices released after 2013 and the Xcode iOS simulator +running on Apple silicon hardware, respectively). +``x86_64-apple-ios-simulator`` +(the Xcode iOS simulator running on older ``x86_64`` hardware) +is not a tier 3 supported platform, but will have best-effort support. +(PEP written and implementation contributed by Russell Keith-Magee in +:gh:`114099`.) + +:pep:`738`: Android is now a :pep:`11` supported platform, with the +``aarch64-linux-android`` and ``x86_64-linux-android`` targets at tier 3. +The 32-bit targets ``arm-linux-androideabi`` and ``i686-linux-android`` +are not tier 3 supported platforms, but will have best-effort support. +(PEP written and implementation contributed by Malcolm Smith in +:gh:`116622`.) + +.. seealso:: :pep:`730`, :pep:`738` + Other Language Changes ====================== -* Allow the *count* argument of :meth:`str.replace` to be a keyword. - (Contributed by Hugo van Kemenade in :gh:`106487`.) +* The compiler now strips common leading whitespace + from every line in a docstring. + This reduces the size of the :term:`bytecode cache ` + (such as ``.pyc`` files), with reductions in file size of around 5%, + for example in :mod:`!sqlalchemy.orm.session` from SQLAlchemy 2.0. + This change affects tools that use docstrings, such as :mod:`doctest`. + + .. doctest:: + + >>> def spam(): + ... """ + ... This is a docstring with + ... leading whitespace. + ... + ... It even has multiple paragraphs! + ... """ + ... + >>> spam.__doc__ + '\nThis is a docstring with\n leading whitespace.\n\nIt even has multiple paragraphs!\n' -* Compiler now strip indents from docstrings. - This will reduce the size of :term:`bytecode cache ` (e.g. ``.pyc`` file). - For example, cache file size for ``sqlalchemy.orm.session`` in SQLAlchemy 2.0 - is reduced by about 5%. - This change will affect tools using docstrings, like :mod:`doctest`. (Contributed by Inada Naoki in :gh:`81283`.) -* The :func:`compile` built-in can now accept a new flag, - ``ast.PyCF_OPTIMIZED_AST``, which is similar to ``ast.PyCF_ONLY_AST`` - except that the returned ``AST`` is optimized according to the value - of the ``optimize`` argument. - (Contributed by Irit Katriel in :gh:`108113`). +* :ref:`Annotation scopes ` within class scopes + can now contain lambdas and comprehensions. + Comprehensions that are located within class scopes + are not inlined into their parent scope. -* :mod:`multiprocessing`, :mod:`concurrent.futures`, :mod:`compileall`: - Replace :func:`os.cpu_count` with :func:`os.process_cpu_count` to select the - default number of worker threads and processes. Get the CPU affinity - if supported. - (Contributed by Victor Stinner in :gh:`109649`.) + .. code-block:: python -* :func:`os.path.realpath` now resolves MS-DOS style file names even if - the file is not accessible. - (Contributed by Moonsik Park in :gh:`82367`.) + class C[T]: + type Alias = lambda: T -* Fixed a bug where a :keyword:`global` declaration in an :keyword:`except` block - is rejected when the global is used in the :keyword:`else` block. - (Contributed by Irit Katriel in :gh:`111123`.) + (Contributed by Jelle Zijlstra in :gh:`109118` and :gh:`118160`.) -* Many functions now emit a warning if a boolean value is passed as - a file descriptor argument. - This can help catch some errors earlier. - (Contributed by Serhiy Storchaka in :gh:`82626`.) +* :ref:`Future statements ` are no longer triggered by + relative imports of the :mod:`__future__` module, + meaning that statements of the form ``from .__future__ import ...`` + are now simply standard relative imports, with no special features activated. + (Contributed by Jeremiah Gabriel Pascual in :gh:`118216`.) + +* :keyword:`global` declarations are now permitted in :keyword:`except` blocks + when that global is used in the :keyword:`else` block. + Previously this raised an erroneous :exc:`SyntaxError`. + (Contributed by Irit Katriel in :gh:`111123`.) -* Added a new environment variable :envvar:`PYTHON_FROZEN_MODULES`. It - determines whether or not frozen modules are ignored by the import machinery, - equivalent of the :option:`-X frozen_modules <-X>` command-line option. +* Add :envvar:`PYTHON_FROZEN_MODULES`, a new environment variable that + determines whether frozen modules are ignored by the import machinery, + equivalent to the :option:`-X frozen_modules <-X>` command-line option. (Contributed by Yilei Yang in :gh:`111374`.) -* The new :envvar:`PYTHON_HISTORY` environment variable can be used to change - the location of a ``.python_history`` file. - (Contributed by Levi Sabah, Zackery Spytz and Hugo van Kemenade in - :gh:`73965`.) +* Add :ref:`support for the perf profiler ` working + without `frame pointers `_ through + the new environment variable :envvar:`PYTHON_PERF_JIT_SUPPORT` + and command-line option :option:`-X perf_jit <-X>`. + (Contributed by Pablo Galindo in :gh:`118518`.) + +* The location of a :file:`.python_history` file can be changed via the + new :envvar:`PYTHON_HISTORY` environment variable. + (Contributed by Levi Sabah, Zackery Spytz and Hugo van Kemenade + in :gh:`73965`.) + +* Classes have a new :attr:`~type.__static_attributes__` attribute. + This is populated by the compiler with a tuple of the class's attribute names + which are assigned through ``self.`` from any function in its body. + (Contributed by Irit Katriel in :gh:`115775`.) + +* The compiler now creates a :attr:`!__firstlineno__` attribute on classes + with the line number of the first line of the class definition. + (Contributed by Serhiy Storchaka in :gh:`118465`.) + +* The :func:`exec` and :func:`eval` builtins now accept + the *globals* and *locals* arguments as keywords. + (Contributed by Raphael Gaschignard in :gh:`105879`) + +* The :func:`compile` builtin now accepts a new flag, + ``ast.PyCF_OPTIMIZED_AST``, which is similar to ``ast.PyCF_ONLY_AST`` + except that the returned AST is optimized according to + the value of the *optimize* argument. + (Contributed by Irit Katriel in :gh:`108113`). -* Add :exc:`PythonFinalizationError` exception. This exception derived from - :exc:`RuntimeError` is raised when an operation is blocked during - the :term:`Python finalization `. +* Add a :attr:`~property.__name__` attribute on :class:`property` objects. + (Contributed by Eugene Toder in :gh:`101860`.) - The following functions now raise PythonFinalizationError, instead of - :exc:`RuntimeError`: +* Add :exc:`PythonFinalizationError`, a new exception derived from + :exc:`RuntimeError` and used to signal when operations are blocked + during :term:`finalization `. + The following callables now raise :exc:`!PythonFinalizationError`, + instead of :exc:`RuntimeError`: - * :func:`_thread.start_new_thread`. - * :class:`subprocess.Popen`. - * :func:`os.fork`. - * :func:`os.forkpty`. + * :func:`_thread.start_new_thread` + * :func:`os.fork` + * :func:`os.forkpty` + * :class:`subprocess.Popen` (Contributed by Victor Stinner in :gh:`114570`.) -* Allow controlling Expat >=2.6.0 reparse deferral (CVE-2023-52425) - by adding five new methods: +* Allow the *count* argument of :meth:`str.replace` to be a keyword. + (Contributed by Hugo van Kemenade in :gh:`106487`.) - * :meth:`xml.etree.ElementTree.XMLParser.flush` - * :meth:`xml.etree.ElementTree.XMLPullParser.flush` - * :meth:`xml.parsers.expat.xmlparser.GetReparseDeferralEnabled` - * :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` - * :meth:`!xml.sax.expatreader.ExpatParser.flush` +* Many functions now emit a warning if a boolean value is passed as + a file descriptor argument. + This can help catch some errors earlier. + (Contributed by Serhiy Storchaka in :gh:`82626`.) - (Contributed by Sebastian Pipping in :gh:`115623`.) +* Added :attr:`!name` and :attr:`!mode` attributes + for compressed and archived file-like objects in + the :mod:`bz2`, :mod:`lzma`, :mod:`tarfile`, and :mod:`zipfile` modules. + (Contributed by Serhiy Storchaka in :gh:`115961`.) New Modules =========== -* None yet. +* :mod:`dbm.sqlite3`: An SQLite backend for :mod:`dbm`. + (Contributed by Raymond Hettinger and Erlend E. Aasland in :gh:`100414`.) Improved Modules ================ + argparse -------- -* Add parameter *deprecated* in methods - :meth:`~argparse.ArgumentParser.add_argument` and :meth:`!add_parser` - which allows to deprecate command-line options, positional arguments and - subcommands. - (Contributed by Serhiy Storchaka in :gh:`83648`). +* Add the *deprecated* parameter to the + :meth:`~argparse.ArgumentParser.add_argument` + and :meth:`!add_parser` methods, to enable deprecating + command-line options, positional arguments, and subcommands. + (Contributed by Serhiy Storchaka in :gh:`83648`.) + array ----- -* Add ``'w'`` type code (``Py_UCS4``) that can be used for Unicode strings. - It can be used instead of ``'u'`` type code, which is deprecated. +* Add the ``'w'`` type code (``Py_UCS4``) for Unicode characters. + It should be used instead of the deprecated ``'u'`` type code. (Contributed by Inada Naoki in :gh:`80480`.) -* Add ``clear()`` method in order to implement ``MutableSequence``. +* Register :class:`array.array` as a :class:`~collections.abc.MutableSequence` + by implementing the :meth:`~array.array.clear` method. (Contributed by Mike Zimin in :gh:`114894`.) + ast --- -* The constructors of node types in the :mod:`ast` module are now stricter - in the arguments they accept, and have more intuitive behaviour when - arguments are omitted. +* The constructors of node types in the :mod:`ast` module are now + stricter in the arguments they accept, + with more intuitive behavior when arguments are omitted. If an optional field on an AST node is not included as an argument when constructing an instance, the field will now be set to ``None``. Similarly, - if a list field is omitted, that field will now be set to an empty list. - (Previously, in both cases, the attribute would be missing on the newly + if a list field is omitted, that field will now be set to an empty list, + and if an :class:`!expr_context` field is omitted, it defaults to + :class:`Load() `. + (Previously, in all cases, the attribute would be missing on the newly constructed AST node instance.) - If other arguments are omitted, a :exc:`DeprecationWarning` is emitted. - This will cause an exception in Python 3.15. Similarly, passing a keyword - argument that does not map to a field on the AST node is now deprecated, + In all other cases, where a required argument is omitted, + the node constructor will emit a :exc:`DeprecationWarning`. + This will raise an exception in Python 3.15. + Similarly, passing a keyword argument to the constructor + that does not map to a field on the AST node is now deprecated, and will raise an exception in Python 3.15. -* :func:`ast.parse` now accepts an optional argument ``optimize`` - which is passed on to the :func:`compile` built-in. This makes it - possible to obtain an optimized ``AST``. + These changes do not apply to user-defined subclasses of :class:`ast.AST` + unless the class opts in to the new behavior + by defining the :attr:`.AST._field_types` mapping. + + (Contributed by Jelle Zijlstra in :gh:`105858`, :gh:`117486`, and :gh:`118851`.) + +* :func:`ast.parse` now accepts an optional argument *optimize* + which is passed on to :func:`compile`. + This makes it possible to obtain an optimized AST. (Contributed by Irit Katriel in :gh:`108113`.) + asyncio ------- +* :func:`asyncio.as_completed` now returns an object that is both an + :term:`asynchronous iterator` and a plain :term:`iterator` + of :term:`awaitables `. + The awaitables yielded by asynchronous iteration include original task + or future objects that were passed in, + making it easier to associate results with the tasks being completed. + (Contributed by Justin Arthur in :gh:`77714`.) + * :meth:`asyncio.loop.create_unix_server` will now automatically remove the Unix socket when the server is closed. (Contributed by Pierre Ossman in :gh:`111246`.) -* :meth:`asyncio.DatagramTransport.sendto` will now send zero-length - datagrams if called with an empty bytes object. The transport flow - control also now accounts for the datagram header when calculating - the buffer size. +* :meth:`.DatagramTransport.sendto` will now send zero-length + datagrams if called with an empty bytes object. + The transport flow control also now accounts for the datagram header + when calculating the buffer size. (Contributed by Jamie Phan in :gh:`115199`.) +* Add :meth:`Queue.shutdown ` + and :exc:`~asyncio.QueueShutDown` to manage queue termination. + (Contributed by Laurie Opperman and Yves Duprat in :gh:`104228`.) + +* Add the :meth:`.Server.close_clients` and :meth:`.Server.abort_clients` + methods, which more forcefully close an asyncio server. + (Contributed by Pierre Ossman in :gh:`113538`.) + +* Accept a tuple of separators in :meth:`.StreamReader.readuntil`, + stopping when any one of them is encountered. + (Contributed by Bruce Merry in :gh:`81322`.) + +* Improve the behavior of :class:`~asyncio.TaskGroup` when + an external cancellation collides with an internal cancellation. + For example, when two task groups are nested + and both experience an exception in a child task simultaneously, + it was possible that the outer task group would hang, + because its internal cancellation was swallowed by the inner task group. + + In the case where a task group is cancelled externally + and also must raise an :exc:`ExceptionGroup`, + it will now call the parent task's :meth:`~asyncio.Task.cancel` method. + This ensures that a :exc:`~asyncio.CancelledError` will be raised + at the next :keyword:`await`, so the cancellation is not lost. + + An added benefit of these changes is that task groups now preserve + the cancellation count (:meth:`~asyncio.Task.cancelling`). + + In order to handle some corner cases, :meth:`~asyncio.Task.uncancel` may now + reset the undocumented ``_must_cancel`` flag + when the cancellation count reaches zero. + + (Inspired by an issue reported by Arthur Tacca in :gh:`116720`.) + +* When :meth:`.TaskGroup.create_task` is called on an inactive + :class:`~asyncio.TaskGroup`, the given coroutine will be closed (which + prevents a :exc:`RuntimeWarning` about the given coroutine being + never awaited). + (Contributed by Arthur Tacca and Jason Zhang in :gh:`115957`.) + + base64 ---- +------ -* Add :func:`base64.z85encode` and :func:`base64.z85decode` functions which allow encoding - and decoding z85 data. - See `Z85 specification `_ for more information. +* Add :func:`~base64.z85encode` and :func:`~base64.z85decode` functions + for encoding :class:`bytes` as `Z85 data`_ + and decoding Z85-encoded data to :class:`!bytes`. (Contributed by Matan Perelman in :gh:`75299`.) + .. _Z85 data: https://rfc.zeromq.org/spec/32/ + + +compileall +---------- + +* The default number of worker threads and processes is now selected using + :func:`os.process_cpu_count` instead of :func:`os.cpu_count`. + (Contributed by Victor Stinner in :gh:`109649`.) + + +concurrent.futures +------------------ + +* The default number of worker threads and processes is now selected using + :func:`os.process_cpu_count` instead of :func:`os.cpu_count`. + (Contributed by Victor Stinner in :gh:`109649`.) + + +configparser +------------ + +* :class:`~configparser.ConfigParser` now has support for unnamed sections, + which allows for top-level key-value pairs. + This can be enabled with the new *allow_unnamed_section* parameter. + (Contributed by Pedro Sousa Lacerda in :gh:`66449`.) + + copy ---- -* Add :func:`copy.replace` function which allows to create a modified copy of - an object, which is especially useful for immutable objects. - It supports named tuples created with the factory function - :func:`collections.namedtuple`, :class:`~dataclasses.dataclass` instances, - various :mod:`datetime` objects, :class:`~inspect.Signature` objects, - :class:`~inspect.Parameter` objects, :ref:`code object `, and - any user classes which define the :meth:`!__replace__` method. +* The new :func:`~copy.replace` function and the :meth:`replace protocol + ` make creating modified copies of objects much simpler. + This is especially useful when working with immutable objects. + The following types support the :func:`~copy.replace` function + and implement the replace protocol: + + * :func:`collections.namedtuple` + * :class:`dataclasses.dataclass` + * :class:`datetime.datetime`, :class:`datetime.date`, :class:`datetime.time` + * :class:`inspect.Signature`, :class:`inspect.Parameter` + * :class:`types.SimpleNamespace` + * :ref:`code objects ` + + Any user-defined class can also support :func:`copy.replace` by defining + the :meth:`~object.__replace__` method. (Contributed by Serhiy Storchaka in :gh:`108751`.) + +ctypes +------ + +* As a consequence of necessary internal refactoring, initialization of + internal metaclasses now happens in ``__init__`` rather + than in ``__new__``. This affects projects that subclass these internal + metaclasses to provide custom initialization. + Generally: + + - Custom logic that was done in ``__new__`` after calling ``super().__new__`` + should be moved to ``__init__``. + - To create a class, call the metaclass, not only the metaclass's + ``__new__`` method. + + See :gh:`124520` for discussion and links to changes in some affected + projects. + +* :class:`ctypes.Structure` objects have a new :attr:`~ctypes.Structure._align_` + attribute which allows the alignment of the structure being packed to/from + memory to be specified explicitly. + (Contributed by Matt Sanderson in :gh:`112433`) + dbm --- -* Add :meth:`dbm.gnu.gdbm.clear` and :meth:`dbm.ndbm.ndbm.clear` methods that remove all items - from the database. +* Add :mod:`dbm.sqlite3`, a new module which implements an SQLite backend, + and make it the default :mod:`!dbm` backend. + (Contributed by Raymond Hettinger and Erlend E. Aasland in :gh:`100414`.) + +* Allow removing all items from the database through + the new :meth:`.gdbm.clear` and :meth:`.ndbm.clear` methods. (Contributed by Donghee Na in :gh:`107122`.) + dis --- * Change the output of :mod:`dis` module functions to show logical labels for jump targets and exception handlers, rather than offsets. - The offsets can be added with the new ``-O`` command line option or - the ``show_offsets`` parameter. + The offsets can be added with the new + :option:`-O ` command-line option + or the *show_offsets* argument. (Contributed by Irit Katriel in :gh:`112137`.) -dbm ---- +* :meth:`~dis.get_instructions` no longer represents cache entries + as separate instructions. + Instead, it returns them as part of the :class:`~dis.Instruction`, + in the new *cache_info* field. + The *show_caches* argument to :meth:`~dis.get_instructions` is deprecated + and no longer has any effect. + (Contributed by Irit Katriel in :gh:`112962`.) -* Add :meth:`dbm.gnu.gdbm.clear` and :meth:`dbm.ndbm.ndbm.clear` methods that remove all items - from the database. - (Contributed by Donghee Na in :gh:`107122`.) -* Add new :mod:`dbm.sqlite3` backend, and make it the default :mod:`!dbm` backend. - (Contributed by Raymond Hettinger and Erlend E. Aasland in :gh:`100414`.) +.. _whatsnew313-doctest: doctest ------- -* The :meth:`doctest.DocTestRunner.run` method now counts the number of skipped - tests. Add :attr:`doctest.DocTestRunner.skips` and - :attr:`doctest.TestResults.skipped` attributes. +* :mod:`doctest` output is now colored by default. + This can be controlled via the new :envvar:`PYTHON_COLORS` environment + variable as well as the canonical |NO_COLOR|_ + and |FORCE_COLOR|_ environment variables. + See also :ref:`using-on-controlling-color`. + (Contributed by Hugo van Kemenade in :gh:`117225`.) + +* The :meth:`.DocTestRunner.run` method now counts the number of skipped tests. + Add the :attr:`.DocTestRunner.skips` and :attr:`.TestResults.skipped` attributes. (Contributed by Victor Stinner in :gh:`108794`.) + email ----- -* :func:`email.utils.getaddresses` and :func:`email.utils.parseaddr` now return - ``('', '')`` 2-tuples in more situations where invalid email addresses are - encountered instead of potentially inaccurate values. Add optional *strict* - parameter to these two functions: use ``strict=False`` to get the old - behavior, accept malformed inputs. - ``getattr(email.utils, 'supports_strict_parsing', False)`` can be use to - check if the *strict* parameter is available. +* Headers with embedded newlines are now quoted on output. + The :mod:`~email.generator` will now refuse to serialize (write) headers + that are improperly folded or delimited, such that they would be parsed as + multiple headers or joined with adjacent data. + If you need to turn this safety feature off, + set :attr:`~email.policy.Policy.verify_generated_headers`. + (Contributed by Bas Bloemsaat and Petr Viktorin in :gh:`121650`.) + +* :func:`~email.utils.getaddresses` and :func:`~email.utils.parseaddr` now + return ``('', '')`` pairs in more situations where invalid email addresses + are encountered instead of potentially inaccurate values. + The two functions have a new optional *strict* parameter (default ``True``). + To get the old behavior (accepting malformed input), use ``strict=False``. + ``getattr(email.utils, 'supports_strict_parsing', False)`` can be used + to check if the *strict* parameter is available. (Contributed by Thomas Dwyer and Victor Stinner for :gh:`102988` to improve - the CVE-2023-27043 fix.) + the :cve:`2023-27043` fix.) + + +enum +---- + +* :class:`~enum.EnumDict` has been made public to better support subclassing + :class:`~enum.EnumType`. + fractions --------- -* Formatting for objects of type :class:`fractions.Fraction` now supports - the standard format specification mini-language rules for fill, alignment, - sign handling, minimum width and grouping. (Contributed by Mark Dickinson - in :gh:`111320`.) +* :class:`~fractions.Fraction` objects now support the standard + :ref:`format specification mini-language ` rules + for fill, alignment, sign handling, minimum width, and grouping. + (Contributed by Mark Dickinson in :gh:`111320`.) + glob ---- -* Add :func:`glob.translate` function that converts a path specification with - shell-style wildcards to a regular expression. +* Add :func:`~glob.translate`, a function to convert a path specification + with shell-style wildcards to a regular expression. (Contributed by Barney Gale in :gh:`72904`.) + +importlib +--------- + +* The following functions in :mod:`importlib.resources` now allow accessing + a directory (or tree) of resources, using multiple positional arguments + (the *encoding* and *errors* arguments in the text-reading functions + are now keyword-only): + + * :func:`~importlib.resources.is_resource` + * :func:`~importlib.resources.open_binary` + * :func:`~importlib.resources.open_text` + * :func:`~importlib.resources.path` + * :func:`~importlib.resources.read_binary` + * :func:`~importlib.resources.read_text` + + These functions are no longer deprecated and are not scheduled for removal. + (Contributed by Petr Viktorin in :gh:`116608`.) + +* :func:`~importlib.resources.contents` remains deprecated in favor of + the fully-featured :class:`~importlib.resources.abc.Traversable` API. + However, there is now no plan to remove it. + (Contributed by Petr Viktorin in :gh:`116608`.) + + io -- -The :class:`io.IOBase` finalizer now logs the ``close()`` method errors with -:data:`sys.unraisablehook`. Previously, errors were ignored silently by default, -and only logged in :ref:`Python Development Mode ` or on :ref:`Python -built on debug mode `. -(Contributed by Victor Stinner in :gh:`62948`.) +* The :class:`~io.IOBase` finalizer now logs any errors raised by + the :meth:`~io.IOBase.close` method with :data:`sys.unraisablehook`. + Previously, errors were ignored silently by default, + and only logged in :ref:`Python Development Mode ` + or when using a :ref:`Python debug build `. + (Contributed by Victor Stinner in :gh:`62948`.) + ipaddress --------- -* Add the :attr:`ipaddress.IPv4Address.ipv6_mapped` property, which returns the IPv4-mapped IPv6 address. +* Add the :attr:`.IPv4Address.ipv6_mapped` property, + which returns the IPv4-mapped IPv6 address. (Contributed by Charles Machalow in :gh:`109466`.) +* Fix ``is_global`` and ``is_private`` behavior in + :class:`~ipaddress.IPv4Address`, :class:`~ipaddress.IPv6Address`, + :class:`~ipaddress.IPv4Network`, and :class:`~ipaddress.IPv6Network`. + (Contributed by Jakub Stasiak in :gh:`113171`.) + + itertools --------- -* Added a ``strict`` option to :func:`itertools.batched`. - This raises a :exc:`ValueError` if the final batch is shorter +* :func:`~itertools.batched` has a new *strict* parameter, + which raises a :exc:`ValueError` if the final batch is shorter than the specified batch size. (Contributed by Raymond Hettinger in :gh:`113202`.) + marshal ------- * Add the *allow_code* parameter in module functions. - Passing ``allow_code=False`` prevents serialization and de-serialization of - code objects which are incompatible between Python versions. + Passing ``allow_code=False`` prevents serialization and de-serialization + of code objects which are incompatible between Python versions. (Contributed by Serhiy Storchaka in :gh:`113626`.) + +math +---- + +* The new function :func:`~math.fma` performs fused multiply-add operations. + This computes ``x * y + z`` with only a single round, + and so avoids any intermediate loss of precision. + It wraps the ``fma()`` function provided by C99, + and follows the specification of the IEEE 754 "fusedMultiplyAdd" operation + for special cases. + (Contributed by Mark Dickinson and Victor Stinner in :gh:`73468`.) + + +mimetypes +--------- + +* Add the :func:`~mimetypes.guess_file_type` function to guess a MIME type + from a filesystem path. + Using paths with :func:`~mimetypes.guess_type` is now :term:`soft deprecated`. + (Contributed by Serhiy Storchaka in :gh:`66543`.) + + mmap ---- -* The :class:`mmap.mmap` class now has an :meth:`~mmap.mmap.seekable` method +* :class:`~mmap.mmap` is now protected from crashing on Windows when the + mapped memory is inaccessible due to file system errors or access violations. + (Contributed by Jannis Weigend in :gh:`118209`.) + +* :class:`~mmap.mmap` has a new :meth:`~mmap.mmap.seekable` method that can be used when a seekable file-like object is required. The :meth:`~mmap.mmap.seek` method now returns the new absolute position. (Contributed by Donghee Na and Sylvie Liberman in :gh:`111835`.) -* :class:`mmap.mmap` now has a *trackfd* parameter on Unix; if it is ``False``, - the file descriptor specified by *fileno* will not be duplicated. + +* The new UNIX-only *trackfd* parameter for :class:`~mmap.mmap` controls + file descriptor duplication; + if false, the file descriptor specified by *fileno* will not be duplicated. (Contributed by Zackery Spytz and Petr Viktorin in :gh:`78502`.) -opcode ------- -* Move ``opcode.ENABLE_SPECIALIZATION`` to ``_opcode.ENABLE_SPECIALIZATION``. - This field was added in 3.12, it was never documented and is not intended for - external usage. (Contributed by Irit Katriel in :gh:`105481`.) +multiprocessing +--------------- + +* The default number of worker threads and processes is now selected using + :func:`os.process_cpu_count` instead of :func:`os.cpu_count`. + (Contributed by Victor Stinner in :gh:`109649`.) -* Removed ``opcode.is_pseudo``, ``opcode.MIN_PSEUDO_OPCODE`` and - ``opcode.MAX_PSEUDO_OPCODE``, which were added in 3.12, were never - documented or exposed through ``dis``, and were not intended to be - used externally. os -- -* Add :func:`os.process_cpu_count` function to get the number of logical CPUs - usable by the calling thread of the current process. +* Add :func:`~os.process_cpu_count` function to get the number + of logical CPU cores usable by the calling thread of the current process. (Contributed by Victor Stinner in :gh:`109649`.) -* Add a low level interface for Linux's timer notification file descriptors - via :func:`os.timerfd_create`, - :func:`os.timerfd_settime`, :func:`os.timerfd_settime_ns`, - :func:`os.timerfd_gettime`, and :func:`os.timerfd_gettime_ns`, - :const:`os.TFD_NONBLOCK`, :const:`os.TFD_CLOEXEC`, - :const:`os.TFD_TIMER_ABSTIME`, and :const:`os.TFD_TIMER_CANCEL_ON_SET` - (Contributed by Masaru Tsuchiyama in :gh:`108277`.) - -* :func:`os.cpu_count` and :func:`os.process_cpu_count` can be overridden through - the new environment variable :envvar:`PYTHON_CPU_COUNT` or the new command-line option - :option:`-X cpu_count <-X>`. This option is useful for users who need to limit - CPU resources of a container system without having to modify the container (application code). +* :func:`~os.cpu_count` and :func:`~os.process_cpu_count` can be overridden + through the new environment variable :envvar:`PYTHON_CPU_COUNT` + or the new command-line option :option:`-X cpu_count <-X>`. + This option is useful for users who need to limit CPU resources + of a container system without having to modify application code + or the container itself. (Contributed by Donghee Na in :gh:`109595`.) -* Add support of :func:`os.lchmod` and the *follow_symlinks* argument - in :func:`os.chmod` on Windows. - Note that the default value of *follow_symlinks* in :func:`!os.lchmod` is - ``False`` on Windows. +* Add a :ref:`low level interface ` to Linux's + :manpage:`timer file descriptors ` + via :func:`~os.timerfd_create`, + :func:`~os.timerfd_settime`, :func:`~os.timerfd_settime_ns`, + :func:`~os.timerfd_gettime`, :func:`~os.timerfd_gettime_ns`, + :const:`~os.TFD_NONBLOCK`, :const:`~os.TFD_CLOEXEC`, + :const:`~os.TFD_TIMER_ABSTIME`, and :const:`~os.TFD_TIMER_CANCEL_ON_SET` + (Contributed by Masaru Tsuchiyama in :gh:`108277`.) + +* :func:`~os.lchmod` and the *follow_symlinks* argument of :func:`~os.chmod` + are both now available on Windows. + Note that the default value of *follow_symlinks* + in :func:`!lchmod` is ``False`` on Windows. (Contributed by Serhiy Storchaka in :gh:`59616`.) -* Add support of :func:`os.fchmod` and a file descriptor - in :func:`os.chmod` on Windows. +* :func:`~os.fchmod` and support for file descriptors in :func:`~os.chmod` + are both now available on Windows. (Contributed by Serhiy Storchaka in :gh:`113191`.) -* :func:`os.posix_spawn` now accepts ``env=None``, which makes the newly spawned - process use the current process environment. +* On Windows, :func:`~os.mkdir` and :func:`~os.makedirs` now support passing + a *mode* value of ``0o700`` to apply access control to the new directory. + This implicitly affects :func:`tempfile.mkdtemp` + and is a mitigation for :cve:`2024-4030`. + Other values for *mode* continue to be ignored. + (Contributed by Steve Dower in :gh:`118486`.) + +* :func:`~os.posix_spawn` now accepts ``None`` for the *env* argument, + which makes the newly spawned process use the current process environment. (Contributed by Jakub Kulik in :gh:`113119`.) -* :func:`os.posix_spawn` gains an :attr:`os.POSIX_SPAWN_CLOSEFROM` attribute for - use in ``file_actions=`` on platforms that support +* :func:`~os.posix_spawn` can now use the :attr:`~os.POSIX_SPAWN_CLOSEFROM` + attribute in the *file_actions* parameter on platforms that support :c:func:`!posix_spawn_file_actions_addclosefrom_np`. (Contributed by Jakub Kulik in :gh:`113117`.) + os.path ------- -* Add :func:`os.path.isreserved` to check if a path is reserved on the current - system. This function is only available on Windows. +* Add :func:`~os.path.isreserved` to check if a path is reserved + on the current system. + This function is only available on Windows. (Contributed by Barney Gale in :gh:`88569`.) -* On Windows, :func:`os.path.isabs` no longer considers paths starting with - exactly one (back)slash to be absolute. + +* On Windows, :func:`~os.path.isabs` no longer considers paths + starting with exactly one slash (``\`` or ``/``) to be absolute. (Contributed by Barney Gale and Jon Foster in :gh:`44626`.) +* :func:`~os.path.realpath` now resolves MS-DOS style file names + even if the file is not accessible. + (Contributed by Moonsik Park in :gh:`82367`.) + + pathlib ------- -* Add :exc:`pathlib.UnsupportedOperation`, which is raised instead of +* Add :exc:`~pathlib.UnsupportedOperation`, which is raised instead of :exc:`NotImplementedError` when a path operation isn't supported. (Contributed by Barney Gale in :gh:`89812`.) -* Add :meth:`pathlib.Path.from_uri`, a new constructor to create a :class:`pathlib.Path` - object from a 'file' URI (``file://``). +* Add a new constructor for creating :class:`~pathlib.Path` objects + from 'file' URIs (``file:///``), :meth:`.Path.from_uri`. (Contributed by Barney Gale in :gh:`107465`.) -* Add :meth:`pathlib.PurePath.full_match` for matching paths with +* Add :meth:`.PurePath.full_match` for matching paths with shell-style wildcards, including the recursive wildcard "``**``". (Contributed by Barney Gale in :gh:`73435`.) -* Add *follow_symlinks* keyword-only argument to :meth:`pathlib.Path.glob`, - :meth:`~pathlib.Path.rglob`, :meth:`~pathlib.Path.is_file`, - :meth:`~pathlib.Path.is_dir`, :meth:`~pathlib.Path.owner`, - :meth:`~pathlib.Path.group`. - (Contributed by Barney Gale in :gh:`77609` and :gh:`105793`, and - Kamil Turek in :gh:`107962`). +* Add the :attr:`.PurePath.parser` class attribute to store the + implementation of :mod:`os.path` used + for low-level path parsing and joining. + This will be either :mod:`!posixpath` or :mod:`!ntpath`. + +* Add *recurse_symlinks* keyword-only argument to + :meth:`.Path.glob` and :meth:`~pathlib.Path.rglob`. + (Contributed by Barney Gale in :gh:`77609`.) + +* :meth:`.Path.glob` and :meth:`~pathlib.Path.rglob` + now return files and directories when given a pattern that ends with "``**``". + Previously, only directories were returned. + (Contributed by Barney Gale in :gh:`70303`.) + +* Add the *follow_symlinks* keyword-only argument to + :meth:`Path.is_file `, + :meth:`Path.is_dir `, + :meth:`.Path.owner`, and :meth:`.Path.group`. + (Contributed by Barney Gale in :gh:`105793` and Kamil Turek in :gh:`107962`.) -* Return files and directories from :meth:`pathlib.Path.glob` and - :meth:`~pathlib.Path.rglob` when given a pattern that ends with "``**``". In - earlier versions, only directories were returned. - (Contributed by Barney Gale in :gh:`70303`). pdb --- -* Add ability to move between chained exceptions during post mortem debugging in :func:`~pdb.pm` using - the new ``exceptions [exc_number]`` command for Pdb. (Contributed by Matthias - Bussonnier in :gh:`106676`.) +* :func:`breakpoint` and :func:`~pdb.set_trace` now enter the debugger immediately + rather than on the next line of code to be executed. This change prevents the + debugger from breaking outside of the context when :func:`!breakpoint` is positioned + at the end of the context. + (Contributed by Tian Gao in :gh:`118579`.) -* Expressions/statements whose prefix is a pdb command are now correctly +* ``sys.path[0]`` is no longer replaced by the directory of the script + being debugged when :attr:`sys.flags.safe_path` is set. + (Contributed by Tian Gao and Christian Walther in :gh:`111762`.) + +* :mod:`zipapp` is now supported as a debugging target. + (Contributed by Tian Gao in :gh:`118501`.) + +* Add ability to move between chained exceptions during + post-mortem debugging in :func:`~pdb.pm` using + the new :pdbcmd:`exceptions [exc_number] ` command for Pdb. + (Contributed by Matthias Bussonnier in :gh:`106676`.) + +* Expressions and statements whose prefix is a pdb command are now correctly identified and executed. (Contributed by Tian Gao in :gh:`108464`.) -* ``sys.path[0]`` will no longer be replaced by the directory of the script - being debugged when ``sys.flags.safe_path`` is set (via the :option:`-P` - command line option or :envvar:`PYTHONSAFEPATH` environment variable). - (Contributed by Tian Gao and Christian Walther in :gh:`111762`.) queue ----- -* Add :meth:`queue.Queue.shutdown` (along with :exc:`queue.ShutDown`) for queue - termination. +* Add :meth:`Queue.shutdown ` and :exc:`~queue.ShutDown` + to manage queue termination. (Contributed by Laurie Opperman and Yves Duprat in :gh:`104750`.) + +random +------ + +* Add a :ref:`command-line interface `. + (Contributed by Hugo van Kemenade in :gh:`118131`.) + + re -- -* Rename :exc:`!re.error` to :exc:`re.PatternError` for improved clarity. + +* Rename :exc:`!re.error` to :exc:`~re.PatternError` for improved clarity. :exc:`!re.error` is kept for backward compatibility. + +shutil +------ + +* Support the *dir_fd* and *follow_symlinks* keyword arguments + in :func:`~shutil.chown`. + (Contributed by Berker Peksag and Tahia K in :gh:`62308`) + + +site +---- + +* :file:`.pth` files are now decoded using UTF-8 first, + and then with the :term:`locale encoding` if UTF-8 decoding fails. + (Contributed by Inada Naoki in :gh:`117802`.) + + sqlite3 ------- -* A :exc:`ResourceWarning` is now emitted if a :class:`sqlite3.Connection` +* A :exc:`ResourceWarning` is now emitted if a :class:`~sqlite3.Connection` object is not :meth:`closed ` explicitly. (Contributed by Erlend E. Aasland in :gh:`105539`.) -* Add *filter* keyword-only parameter to :meth:`sqlite3.Connection.iterdump` +* Add the *filter* keyword-only parameter to :meth:`.Connection.iterdump` for filtering database objects to dump. (Contributed by Mariusz Felisiak in :gh:`91602`.) + +ssl +--- + +* The :func:`~ssl.create_default_context` API now includes + :data:`~ssl.VERIFY_X509_PARTIAL_CHAIN` and :data:`~ssl.VERIFY_X509_STRICT` + in its default flags. + + .. note:: + + :data:`~ssl.VERIFY_X509_STRICT` may reject pre-:rfc:`5280` + or malformed certificates that the underlying OpenSSL implementation + might otherwise accept. + Whilst disabling this is not recommended, you can do so using: + + .. code-block:: python + + import ssl + + ctx = ssl.create_default_context() + ctx.verify_flags &= ~ssl.VERIFY_X509_STRICT + + (Contributed by William Woodruff in :gh:`112389`.) + + statistics ---------- -* Add :func:`statistics.kde` for kernel density estimation. +* Add :func:`~statistics.kde` for kernel density estimation. This makes it possible to estimate a continuous probability density function from a fixed number of discrete samples. (Contributed by Raymond Hettinger in :gh:`115863`.) +* Add :func:`~statistics.kde_random` for sampling from an + estimated probability density function created by :func:`~statistics.kde`. + (Contributed by Raymond Hettinger in :gh:`115863`.) + + +.. _whatsnew313-subprocess: + subprocess ---------- -* The :mod:`subprocess` module now uses the :func:`os.posix_spawn` function in - more situations. Notably in the default case of ``close_fds=True`` on more - recent versions of platforms including Linux, FreeBSD, and Solaris where the - C library provides :c:func:`!posix_spawn_file_actions_addclosefrom_np`. - On Linux this should perform similar to our existing Linux :c:func:`!vfork` - based code. A private control knob :attr:`!subprocess._USE_POSIX_SPAWN` can - be set to ``False`` if you need to force :mod:`subprocess` not to ever use - :func:`os.posix_spawn`. Please report your reason and platform details in - the CPython issue tracker if you set this so that we can improve our API - selection logic for everyone. +* The :mod:`subprocess` module now uses the :func:`~os.posix_spawn` function in + more situations. + + Notably, when *close_fds* is ``True`` (the default), + :func:`~os.posix_spawn` will be used when the C library provides + :c:func:`!posix_spawn_file_actions_addclosefrom_np`, + which includes recent versions of Linux, FreeBSD, and Solaris. + On Linux, this should perform similarly to the existing + Linux :c:func:`!vfork` based code. + + A private control knob :attr:`!subprocess._USE_POSIX_SPAWN` can + be set to ``False`` if you need to force :mod:`subprocess` + to never use :func:`~os.posix_spawn`. + Please report your reason and platform details in + the :ref:`issue tracker ` if you set this + so that we can improve our API selection logic for everyone. (Contributed by Jakub Kulik in :gh:`113117`.) + sys --- -* Add the :func:`sys._is_interned` function to test if the string was interned. +* Add the :func:`~sys._is_interned` function to test if a string was interned. This function is not guaranteed to exist in all implementations of Python. (Contributed by Serhiy Storchaka in :gh:`78573`.) + +tempfile +-------- + +* On Windows, the default mode ``0o700`` used by :func:`tempfile.mkdtemp` now + limits access to the new directory due to changes to :func:`os.mkdir`. + This is a mitigation for :cve:`2024-4030`. + (Contributed by Steve Dower in :gh:`118486`.) + + +time +---- + +* On Windows, :func:`~time.monotonic` now uses the + ``QueryPerformanceCounter()`` clock for a resolution of 1 microsecond, + instead of the ``GetTickCount64()`` clock which has + a resolution of 15.6 milliseconds. + (Contributed by Victor Stinner in :gh:`88494`.) + +* On Windows, :func:`~time.time` now uses the + ``GetSystemTimePreciseAsFileTime()`` clock for a resolution of 1 microsecond, + instead of the ``GetSystemTimeAsFileTime()`` clock which has + a resolution of 15.6 milliseconds. + (Contributed by Victor Stinner in :gh:`63207`.) + + tkinter ------- @@ -543,901 +1302,723 @@ tkinter * The :mod:`tkinter` widget method :meth:`!wm_attributes` now accepts the attribute name without the minus prefix to get window attributes, - e.g. ``w.wm_attributes('alpha')`` and allows to specify attributes and - values to set as keyword arguments, e.g. ``w.wm_attributes(alpha=0.5)``. - Add new optional keyword-only parameter *return_python_dict*: calling - ``w.wm_attributes(return_python_dict=True)`` returns the attributes as - a dict instead of a tuple. + for example ``w.wm_attributes('alpha')`` + and allows specifying attributes and values to set as keyword arguments, + for example ``w.wm_attributes(alpha=0.5)``. + (Contributed by Serhiy Storchaka in :gh:`43457`.) + +* :meth:`!wm_attributes` can now return attributes as a :class:`dict`, + by using the new optional keyword-only parameter *return_python_dict*. (Contributed by Serhiy Storchaka in :gh:`43457`.) -* Add new optional keyword-only parameter *return_ints* in - the :meth:`!Text.count` method. - Passing ``return_ints=True`` makes it always returning the single count - as an integer instead of a 1-tuple or ``None``. +* :meth:`!Text.count` can now return a simple :class:`int` + when the new optional keyword-only parameter *return_ints* is used. + Otherwise, the single count is returned as a 1-tuple or ``None``. (Contributed by Serhiy Storchaka in :gh:`97928`.) -* Add support of the "vsapi" element type in +* Support the "vsapi" element type in the :meth:`~tkinter.ttk.Style.element_create` method of :class:`tkinter.ttk.Style`. (Contributed by Serhiy Storchaka in :gh:`68166`.) +* Add the :meth:`!after_info` method for Tkinter widgets. + (Contributed by Cheryl Sabella in :gh:`77020`.) + +* Add a new :meth:`!copy_replace` method to :class:`!PhotoImage` + to copy a region from one image to another, + possibly with pixel zooming, subsampling, or both. + (Contributed by Serhiy Storchaka in :gh:`118225`.) + +* Add *from_coords* parameter to the :class:`!PhotoImage` methods + :meth:`!copy`, :meth:`!zoom` and :meth:`!subsample`. + Add *zoom* and *subsample* parameters to the :class:`!PhotoImage` method + :meth:`!copy`. + (Contributed by Serhiy Storchaka in :gh:`118225`.) + +* Add the :class:`!PhotoImage` methods + :meth:`!read` to read an image from a file + and :meth:`!data` to get the image data. + Add *background* and *grayscale* parameters to the :meth:`!write` method. + (Contributed by Serhiy Storchaka in :gh:`118271`.) + + traceback --------- -* Add *show_group* parameter to :func:`traceback.TracebackException.format_exception_only` - to format the nested exceptions of a :exc:`BaseExceptionGroup` instance, recursively. +* Add the :attr:`~traceback.TracebackException.exc_type_str` attribute + to :class:`~traceback.TracebackException`, + which holds a string display of the *exc_type*. + Deprecate the :attr:`~traceback.TracebackException.exc_type` attribute, + which holds the type object itself. + Add parameter *save_exc_type* (default ``True``) + to indicate whether ``exc_type`` should be saved. + (Contributed by Irit Katriel in :gh:`112332`.) + +* Add a new *show_group* keyword-only parameter to + :meth:`.TracebackException.format_exception_only` to (recursively) format + the nested exceptions of a :exc:`BaseExceptionGroup` instance. (Contributed by Irit Katriel in :gh:`105292`.) -* Add the field *exc_type_str* to :class:`~traceback.TracebackException`, which - holds a string display of the *exc_type*. Deprecate the field *exc_type* - which holds the type object itself. Add parameter *save_exc_type* (default - ``True``) to indicate whether ``exc_type`` should be saved. - (Contributed by Irit Katriel in :gh:`112332`.) + +types +----- + +* :class:`~types.SimpleNamespace` can now take a single positional argument + to initialise the namespace's arguments. + This argument must either be a mapping or an iterable of key-value pairs. + (Contributed by Serhiy Storchaka in :gh:`108191`.) + typing ------ -* Add :func:`typing.get_protocol_members` to return the set of members - defining a :class:`typing.Protocol`. Add :func:`typing.is_protocol` to - check whether a class is a :class:`typing.Protocol`. (Contributed by Jelle Zijlstra in - :gh:`104873`.) +* :pep:`705`: Add :data:`~typing.ReadOnly`, a special typing construct + to mark a :class:`~typing.TypedDict` item as read-only for type checkers. -unicodedata ------------ +* :pep:`742`: Add :data:`~typing.TypeIs`, a typing construct + that can be used to instruct a type checker how to narrow a type. -* The Unicode database has been updated to version 15.1.0. (Contributed by - James Gerity in :gh:`109559`.) +* Add :data:`~typing.NoDefault`, a sentinel object used to represent + the defaults of some parameters in the :mod:`typing` module. + (Contributed by Jelle Zijlstra in :gh:`116126`.) -venv ----- +* Add :func:`~typing.get_protocol_members` to return the set of members + defining a :class:`typing.Protocol`. + (Contributed by Jelle Zijlstra in :gh:`104873`.) -* Add support for adding source control management (SCM) ignore files to a - virtual environment's directory. By default, Git is supported. This is - implemented as opt-in via the API which can be extended to support other SCMs - (:class:`venv.EnvBuilder` and :func:`venv.create`), and opt-out via the CLI - (using ``--without-scm-ignore-files``). (Contributed by Brett Cannon in - :gh:`108125`.) +* Add :func:`~typing.is_protocol` to check whether a class + is a :class:`~typing.Protocol`. + (Contributed by Jelle Zijlstra in :gh:`104873`.) -warnings --------- +* :data:`~typing.ClassVar` can now be nested in :data:`~typing.Final`, + and vice versa. + (Contributed by Mehdi Drissi in :gh:`89547`.) -* The new :func:`warnings.deprecated` decorator provides a way to communicate - deprecations to :term:`static type checkers ` and - to warn on usage of deprecated classes and functions. A runtime deprecation - warning may also be emitted when a decorated function or class is used at runtime. - See :pep:`702`. (Contributed by Jelle Zijlstra in :gh:`104003`.) -xml.etree.ElementTree ---------------------- +unicodedata +----------- -* Add the :meth:`!close` method for the iterator returned by - :func:`~xml.etree.ElementTree.iterparse` for explicit cleaning up. - (Contributed by Serhiy Storchaka in :gh:`69893`.) +* Update the Unicode database to `version 15.1.0`__. + (Contributed by James Gerity in :gh:`109559`.) + __ https://www.unicode.org/versions/Unicode15.1.0/ -Optimizations -============= -* :func:`textwrap.indent` is now ~30% faster than before for large input. - (Contributed by Inada Naoki in :gh:`107369`.) +venv +---- -* The :mod:`subprocess` module uses :func:`os.posix_spawn` in more situations - including the default where ``close_fds=True`` on many modern platforms. This - should provide a noteworthy performance increase launching processes on - FreeBSD and Solaris. See the ``subprocess`` section above for details. - (Contributed by Jakub Kulik in :gh:`113117`.) +* Add support for creating source control management (SCM) ignore files + in a virtual environment's directory. + By default, Git is supported. + This is implemented as opt-in via the API, + which can be extended to support other SCMs + (:class:`~venv.EnvBuilder` and :func:`~venv.create`), + and opt-out via the CLI, using :option:`!--without-scm-ignore-files`. + (Contributed by Brett Cannon in :gh:`108125`.) -.. _whatsnew313-jit-compiler: -Experimental JIT Compiler -========================= +warnings +-------- -When CPython is configured using the ``--enable-experimental-jit`` option, -a just-in-time compiler is added which can speed up some Python programs. +* :pep:`702`: The new :func:`warnings.deprecated` decorator provides a way to + communicate deprecations to a :term:`static type checker` + and to warn on usage of deprecated classes and functions. + A :exc:`DeprecationWarning` may also be emitted when + a decorated function or class is used at runtime. + (Contributed by Jelle Zijlstra in :gh:`104003`.) -The internal architecture is roughly as follows. -* We start with specialized *Tier 1 bytecode*. - See :ref:`What's new in 3.11 ` for details. +xml +--- -* When the Tier 1 bytecode gets hot enough, it gets translated - to a new, purely internal *Tier 2 IR*, a.k.a. micro-ops ("uops"). +* Allow controlling Expat >=2.6.0 reparse deferral (:cve:`2023-52425`) + by adding five new methods: -* The Tier 2 IR uses the same stack-based VM as Tier 1, but the - instruction format is better suited to translation to machine code. + * :meth:`xml.etree.ElementTree.XMLParser.flush` + * :meth:`xml.etree.ElementTree.XMLPullParser.flush` + * :meth:`xml.parsers.expat.xmlparser.GetReparseDeferralEnabled` + * :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` + * :meth:`!xml.sax.expatreader.ExpatParser.flush` -* We have several optimization passes for Tier 2 IR, which are applied - before it is interpreted or translated to machine code. + (Contributed by Sebastian Pipping in :gh:`115623`.) -* There is a Tier 2 interpreter, but it is mostly intended for debugging - the earlier stages of the optimization pipeline. If the JIT is not - enabled, the Tier 2 interpreter can be invoked by passing Python the - ``-X uops`` option or by setting the ``PYTHON_UOPS`` environment - variable to ``1``. +* Add the :meth:`!close` method for the iterator returned by + :func:`~xml.etree.ElementTree.iterparse` for explicit cleanup. + (Contributed by Serhiy Storchaka in :gh:`69893`.) -* When the ``--enable-experimental-jit`` option is used, the optimized - Tier 2 IR is translated to machine code, which is then executed. - This does not require additional runtime options. -* The machine code translation process uses an architecture called - *copy-and-patch*. It has no runtime dependencies, but there is a new - build-time dependency on LLVM. +zipimport +--------- -(JIT by Brandt Bucher, inspired by a paper by Haoran Xu and Fredrik Kjolstad. -Tier 2 IR by Mark Shannon and Guido van Rossum. -Tier 2 optimizer by Ken Jin.) +* Add support for ZIP64_ format files. + Everybody loves huge data, right? + (Contributed by Tim Hatch in :gh:`94146`.) + .. _ZIP64: https://en.wikipedia.org/wiki/Zip_(file_format)#ZIP64 -Deprecated -========== +Optimizations +============= -* :mod:`array`: :mod:`array`'s ``'u'`` format code, deprecated in docs since Python 3.3, - emits :exc:`DeprecationWarning` since 3.13 - and will be removed in Python 3.16. - Use the ``'w'`` format code instead. - (contributed by Hugo van Kemenade in :gh:`80480`) +* Several standard library modules have had + their import times significantly improved. + For example, the import time of the :mod:`typing` module + has been reduced by around a third by removing dependencies + on :mod:`re` and :mod:`contextlib`. + Other modules to enjoy import-time speedups include + :mod:`email.utils`, :mod:`enum`, :mod:`functools`, + :mod:`importlib.metadata`, and :mod:`threading`. + (Contributed by Alex Waygood, Shantanu Jain, Adam Turner, Daniel Hollas, + and others in :gh:`109653`.) + +* :func:`textwrap.indent` is now around 30% faster than before for large input. + (Contributed by Inada Naoki in :gh:`107369`.) -* :mod:`ctypes`: Deprecate undocumented :func:`!ctypes.SetPointerType` - and :func:`!ctypes.ARRAY` functions. - Replace ``ctypes.ARRAY(item_type, size)`` with ``item_type * size``. - (Contributed by Victor Stinner in :gh:`105733`.) +* The :mod:`subprocess` module now uses the :func:`~os.posix_spawn` function in + more situations, including when *close_fds* is ``True`` (the default) + on many modern platforms. + This should provide a notable performance increase + when launching processes on FreeBSD and Solaris. + See the :ref:`subprocess ` section above for details. + (Contributed by Jakub Kulik in :gh:`113117`.) -* :mod:`decimal`: Deprecate non-standard format specifier "N" for - :class:`decimal.Decimal`. - It was not documented and only supported in the C implementation. - (Contributed by Serhiy Storchaka in :gh:`89902`.) -* :mod:`dis`: The ``dis.HAVE_ARGUMENT`` separator is deprecated. Check - membership in :data:`~dis.hasarg` instead. - (Contributed by Irit Katriel in :gh:`109319`.) +Removed Modules And APIs +======================== -* :mod:`getopt` and :mod:`optparse` modules: They are now - :term:`soft deprecated`: the :mod:`argparse` module should be used for new projects. - Previously, the :mod:`optparse` module was already deprecated, its removal - was not scheduled, and no warnings was emitted: so there is no change in - practice. - (Contributed by Victor Stinner in :gh:`106535`.) -* :mod:`gettext`: Emit deprecation warning for non-integer numbers in - :mod:`gettext` functions and methods that consider plural forms even if the - translation was not found. - (Contributed by Serhiy Storchaka in :gh:`88434`.) +.. _whatsnew313-pep594: -* :mod:`http.server`: :class:`http.server.CGIHTTPRequestHandler` now emits a - :exc:`DeprecationWarning` as it will be removed in 3.15. Process-based CGI - HTTP servers have been out of favor for a very long time. This code was - outdated, unmaintained, and rarely used. It has a high potential for both - security and functionality bugs. This includes removal of the ``--cgi`` - flag to the ``python -m http.server`` command line in 3.15. +PEP 594: Remove "dead batteries" from the standard library +---------------------------------------------------------- -* :mod:`pathlib`: - :meth:`pathlib.PurePath.is_reserved` is deprecated and scheduled for - removal in Python 3.15. Use :func:`os.path.isreserved` to detect reserved - paths on Windows. +:pep:`594` proposed removing 19 modules from the standard library, +colloquially referred to as 'dead batteries' due to their +historic, obsolete, or insecure status. +All of the following modules were deprecated in Python 3.11, +and are now removed: -* :mod:`pydoc`: Deprecate undocumented :func:`!pydoc.ispackage` function. - (Contributed by Zackery Spytz in :gh:`64020`.) +* :mod:`!aifc` + + * :pypi:`standard-aifc`: + Use the redistribution of ``aifc`` library from PyPI. -* :mod:`sqlite3`: Passing more than one positional argument to - :func:`sqlite3.connect` and the :class:`sqlite3.Connection` constructor is - deprecated. The remaining parameters will become keyword-only in Python 3.15. +* :mod:`!audioop` - Deprecate passing name, number of arguments, and the callable as keyword - arguments for the following :class:`sqlite3.Connection` APIs: + * :pypi:`audioop-lts`: + Use ``audioop-lts`` library from PyPI. - * :meth:`~sqlite3.Connection.create_function` - * :meth:`~sqlite3.Connection.create_aggregate` +* :mod:`!chunk` - Deprecate passing the callback callable by keyword for the following - :class:`sqlite3.Connection` APIs: + * :pypi:`standard-chunk`: + Use the redistribution of ``chunk`` library from PyPI. - * :meth:`~sqlite3.Connection.set_authorizer` - * :meth:`~sqlite3.Connection.set_progress_handler` - * :meth:`~sqlite3.Connection.set_trace_callback` +* :mod:`!cgi` and :mod:`!cgitb` - The affected parameters will become positional-only in Python 3.15. + * :class:`!cgi.FieldStorage` can typically be replaced with + :func:`urllib.parse.parse_qsl` for ``GET`` and ``HEAD`` requests, + and the :mod:`email.message` module or the :pypi:`multipart` library + for ``POST`` and ``PUT`` requests. - (Contributed by Erlend E. Aasland in :gh:`107948` and :gh:`108278`.) + * :func:`!cgi.parse` can be replaced by calling + :func:`urllib.parse.parse_qs` directly on the desired query string, + unless the input is ``multipart/form-data``, + which should be replaced as described below for :func:`!cgi.parse_multipart`. -* :mod:`sys`: :func:`sys._enablelegacywindowsfsencoding` function. - Replace it with the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable. - (Contributed by Inada Naoki in :gh:`73427`.) + * :func:`!cgi.parse_header` can be replaced with the functionality + in the :mod:`email` package, which implements the same MIME RFCs. + For example, with :class:`email.message.EmailMessage`: -* :mod:`traceback`: The field *exc_type* of :class:`traceback.TracebackException` - is deprecated. Use *exc_type_str* instead. + .. code-block:: python -* :mod:`typing`: + from email.message import EmailMessage - * Creating a :class:`typing.NamedTuple` class using keyword arguments to denote - the fields (``NT = NamedTuple("NT", x=int, y=int)``) is deprecated, and will - be disallowed in Python 3.15. Use the class-based syntax or the functional - syntax instead. (Contributed by Alex Waygood in :gh:`105566`.) - - * When using the functional syntax to create a :class:`typing.NamedTuple` - class or a :class:`typing.TypedDict` class, failing to pass a value to the - 'fields' parameter (``NT = NamedTuple("NT")`` or ``TD = TypedDict("TD")``) is - deprecated. Passing ``None`` to the 'fields' parameter - (``NT = NamedTuple("NT", None)`` or ``TD = TypedDict("TD", None)``) is also - deprecated. Both will be disallowed in Python 3.15. To create a NamedTuple - class with 0 fields, use ``class NT(NamedTuple): pass`` or - ``NT = NamedTuple("NT", [])``. To create a TypedDict class with 0 fields, use - ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. - (Contributed by Alex Waygood in :gh:`105566` and :gh:`105570`.) + msg = EmailMessage() + msg['content-type'] = 'application/json; charset="utf8"' + main, params = msg.get_content_type(), msg['content-type'].params - * :func:`typing.no_type_check_decorator` is deprecated, and scheduled for - removal in Python 3.15. After eight years in the :mod:`typing` module, it - has yet to be supported by any major type checkers. - (Contributed by Alex Waygood in :gh:`106309`.) + * :func:`!cgi.parse_multipart` can be replaced with the functionality + in the :mod:`email` package, which implements the same MIME RFCs, + or with the :pypi:`multipart` library. + For example, the :class:`email.message.EmailMessage` + and :class:`email.message.Message` classes. - * :data:`typing.AnyStr` is deprecated. In Python 3.16, it will be removed from - ``typing.__all__``, and a :exc:`DeprecationWarning` will be emitted when it - is imported or accessed. It will be removed entirely in Python 3.18. Use - the new :ref:`type parameter syntax ` instead. - (Contributed by Michael The in :gh:`107116`.) + * :pypi:`standard-cgi`: and :pypi:`standard-cgitb`: + Use the redistribution of ``cgi`` and ``cgitb`` library from PyPI. -* :mod:`wave`: Deprecate the ``getmark()``, ``setmark()`` and ``getmarkers()`` - methods of the :class:`wave.Wave_read` and :class:`wave.Wave_write` classes. - They will be removed in Python 3.15. - (Contributed by Victor Stinner in :gh:`105096`.) +* :mod:`!crypt` and the private :mod:`!_crypt` extension. + The :mod:`hashlib` module may be an appropriate replacement + when simply hashing a value is required. + Otherwise, various third-party libraries on PyPI are available: -* Calling :meth:`frame.clear` on a suspended frame raises :exc:`RuntimeError` - (as has always been the case for an executing frame). - (Contributed by Irit Katriel in :gh:`79932`.) + * :pypi:`bcrypt`: + Modern password hashing for your software and your servers. + * :pypi:`passlib`: + Comprehensive password hashing framework supporting over 30 schemes. + * :pypi:`argon2-cffi`: + The secure Argon2 password hashing algorithm. + * :pypi:`legacycrypt`: + :mod:`ctypes` wrapper to the POSIX crypt library call + and associated functionality. + * :pypi:`crypt_r`: + Fork of the :mod:`!crypt` module, + wrapper to the :manpage:`crypt_r(3)` library call + and associated functionality. + * :pypi:`standard-crypt` and :pypi:`deprecated-crypt-alternative`: + Use the redistribution of ``crypt`` and reimplementation of ``_crypt`` libraries from PyPI. -* Assignment to a function's :attr:`~function.__code__` attribute where the new code - object's type does not match the function's type, is deprecated. The - different types are: plain function, generator, async generator and - coroutine. - (Contributed by Irit Katriel in :gh:`81137`.) +* :mod:`!imghdr`: + The :pypi:`filetype`, :pypi:`puremagic`, or :pypi:`python-magic` libraries + should be used as replacements. + For example, the :func:`!puremagic.what` function can be used + to replace the :func:`!imghdr.what` function for all file formats + that were supported by :mod:`!imghdr`. -* The undocumented and unused ``tarfile`` attribute of :class:`tarfile.TarFile` - is deprecated and scheduled for removal in Python 3.16. + * :pypi:`standard-imghdr`: + Use the redistribution of ``imghdr`` library from PyPI. +* :mod:`!mailcap`: + Use the :mod:`mimetypes` module instead. -Pending Removal in Python 3.14 ------------------------------- + * :pypi:`standard-mailcap`: + Use the redistribution of ``mailcap`` library from PyPI. -* :mod:`argparse`: The *type*, *choices*, and *metavar* parameters - of :class:`!argparse.BooleanOptionalAction` are deprecated - and will be removed in 3.14. - (Contributed by Nikita Sobolev in :gh:`92248`.) +* :mod:`!msilib` +* :mod:`!nis` +* :mod:`!nntplib`: + Use the :pypi:`pynntp` library from PyPI instead. -* :mod:`ast`: The following features have been deprecated in documentation - since Python 3.8, now cause a :exc:`DeprecationWarning` to be emitted at - runtime when they are accessed or used, and will be removed in Python 3.14: + * :pypi:`standard-nntplib`: + Use the redistribution of ``nntplib`` library from PyPI. - * :class:`!ast.Num` - * :class:`!ast.Str` - * :class:`!ast.Bytes` - * :class:`!ast.NameConstant` - * :class:`!ast.Ellipsis` +* :mod:`!ossaudiodev`: + For audio playback, use the :pypi:`pygame` library from PyPI instead. +* :mod:`!pipes`: + Use the :mod:`subprocess` module instead. + Use :func:`shlex.quote` to replace the undocumented ``pipes.quote`` + function. - Use :class:`ast.Constant` instead. - (Contributed by Serhiy Storchaka in :gh:`90953`.) + * :pypi:`standard-pipes`: + Use the redistribution of ``pipes`` library from PyPI. -* :mod:`collections.abc`: Deprecated :class:`~collections.abc.ByteString`. - Prefer :class:`!Sequence` or :class:`~collections.abc.Buffer`. - For use in typing, prefer a union, like ``bytes | bytearray``, - or :class:`collections.abc.Buffer`. - (Contributed by Shantanu Jain in :gh:`91896`.) +* :mod:`!sndhdr`: + The :pypi:`filetype`, :pypi:`puremagic`, or :pypi:`python-magic` libraries + should be used as replacements. -* :mod:`email`: Deprecated the *isdst* parameter in :func:`email.utils.localtime`. - (Contributed by Alan Williams in :gh:`72346`.) + * :pypi:`standard-sndhdr`: + Use the redistribution of ``sndhdr`` library from PyPI. -* :mod:`importlib`: ``__package__`` and ``__cached__`` will cease to be set or - taken into consideration by the import system (:gh:`97879`). +* :mod:`!spwd`: + Use the :pypi:`python-pam` library from PyPI instead. +* :mod:`!sunau` -* :mod:`importlib.abc` deprecated classes: + * :pypi:`standard-sunau`: + Use the redistribution of ``sunau`` library from PyPI. - * :class:`!importlib.abc.ResourceReader` - * :class:`!importlib.abc.Traversable` - * :class:`!importlib.abc.TraversableResources` +* :mod:`!telnetlib`, + Use the :pypi:`telnetlib3` or :pypi:`Exscript` libraries from PyPI instead. - Use :mod:`importlib.resources.abc` classes instead: + * :pypi:`standard-telnetlib`: + Use the redistribution of ``telnetlib`` library from PyPI. - * :class:`importlib.resources.abc.Traversable` - * :class:`importlib.resources.abc.TraversableResources` +* :mod:`!uu`: + Use the :mod:`base64` module instead, as a modern alternative. - (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) + * :pypi:`standard-uu`: + Use the redistribution of ``uu`` library from PyPI. -* :mod:`itertools` had undocumented, inefficient, historically buggy, - and inconsistent support for copy, deepcopy, and pickle operations. - This will be removed in 3.14 for a significant reduction in code - volume and maintenance burden. - (Contributed by Raymond Hettinger in :gh:`101588`.) +* :mod:`!xdrlib` -* :mod:`multiprocessing`: The default start method will change to a safer one on - Linux, BSDs, and other non-macOS POSIX platforms where ``'fork'`` is currently - the default (:gh:`84559`). Adding a runtime warning about this was deemed too - disruptive as the majority of code is not expected to care. Use the - :func:`~multiprocessing.get_context` or - :func:`~multiprocessing.set_start_method` APIs to explicitly specify when - your code *requires* ``'fork'``. See :ref:`multiprocessing-start-methods`. + * :pypi:`standard-xdrlib`: + Use the redistribution of ``xdrlib`` library from PyPI. -* :mod:`pathlib`: :meth:`~pathlib.PurePath.is_relative_to` and - :meth:`~pathlib.PurePath.relative_to`: passing additional arguments is - deprecated. +(Contributed by Victor Stinner and Zachary Ware in :gh:`104773` and :gh:`104780`.) -* :mod:`pkgutil`: :func:`~pkgutil.find_loader` and :func:`~pkgutil.get_loader` - now raise :exc:`DeprecationWarning`; - use :func:`importlib.util.find_spec` instead. - (Contributed by Nikita Sobolev in :gh:`97850`.) -* :mod:`pty`: +2to3 +---- - * ``master_open()``: use :func:`pty.openpty`. - * ``slave_open()``: use :func:`pty.openpty`. +* Remove the :program:`2to3` program and the :mod:`!lib2to3` module, + previously deprecated in Python 3.11. + (Contributed by Victor Stinner in :gh:`104780`.) -* :func:`shutil.rmtree` *onerror* parameter is deprecated in 3.12, - and will be removed in 3.14: use the *onexc* parameter instead. -* :mod:`sqlite3`: +builtins +-------- - * :data:`~sqlite3.version` and :data:`~sqlite3.version_info`. +* Remove support for chained :class:`classmethod` descriptors + (introduced in :gh:`63272`). + These can no longer be used to wrap other descriptors, + such as :class:`property`. + The core design of this feature was flawed and led to several problems. + To "pass-through" a :class:`classmethod`, consider using + the :attr:`!__wrapped__` attribute that was added in Python 3.10. + (Contributed by Raymond Hettinger in :gh:`89519`.) + +* Raise a :exc:`RuntimeError` when calling :meth:`frame.clear` + on a suspended frame (as has always been the case for an executing frame). + (Contributed by Irit Katriel in :gh:`79932`.) - * :meth:`~sqlite3.Cursor.execute` and :meth:`~sqlite3.Cursor.executemany` - if :ref:`named placeholders ` are used and - *parameters* is a sequence instead of a :class:`dict`. - * date and datetime adapter, date and timestamp converter: - see the :mod:`sqlite3` documentation for suggested replacement recipes. +configparser +------------ -* :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was - deprecated in :pep:`626` - since 3.10 and was planned to be removed in 3.12, - but it only got a proper :exc:`DeprecationWarning` in 3.12. - May be removed in 3.14. - (Contributed by Nikita Sobolev in :gh:`101866`.) +* Remove the undocumented :class:`!LegacyInterpolation` class, + deprecated in the docstring since Python 3.2, + and at runtime since Python 3.11. + (Contributed by Hugo van Kemenade in :gh:`104886`.) -* :mod:`typing`: :class:`~typing.ByteString`, deprecated since Python 3.9, - now causes a :exc:`DeprecationWarning` to be emitted when it is used. -* :class:`!urllib.parse.Quoter` is deprecated: it was not intended to be a - public API. - (Contributed by Gregory P. Smith in :gh:`88168`.) +importlib.metadata +------------------ -* :mod:`xml.etree.ElementTree`: Testing the truth value of an - :class:`~xml.etree.ElementTree.Element` is deprecated and will raise an - exception in Python 3.14. +* Remove deprecated subscript (:meth:`~object.__getitem__`) access for + :ref:`EntryPoint ` objects. + (Contributed by Jason R. Coombs in :gh:`113175`.) -Pending Removal in Python 3.15 ------------------------------- -* :class:`http.server.CGIHTTPRequestHandler` will be removed along with its - related ``--cgi`` flag to ``python -m http.server``. It was obsolete and - rarely used. No direct replacement exists. *Anything* is better than CGI - to interface a web server with a request handler. +locale +------ -* :class:`locale`: :func:`locale.getdefaultlocale` was deprecated in Python 3.11 - and originally planned for removal in Python 3.13 (:gh:`90817`), - but removal has been postponed to Python 3.15. - Use :func:`locale.setlocale()`, :func:`locale.getencoding()` and - :func:`locale.getlocale()` instead. - (Contributed by Hugo van Kemenade in :gh:`111187`.) +* Remove the :func:`!locale.resetlocale` function, deprecated in Python 3.11. + Use ``locale.setlocale(locale.LC_ALL, "")`` instead. + (Contributed by Victor Stinner in :gh:`104783`.) -* :mod:`pathlib`: - :meth:`pathlib.PurePath.is_reserved` is deprecated and scheduled for - removal in Python 3.15. Use :func:`os.path.isreserved` to detect reserved - paths on Windows. - -* :mod:`threading`: - Passing any arguments to :func:`threading.RLock` is now deprecated. - C version allows any numbers of args and kwargs, - but they are just ignored. Python version does not allow any arguments. - All arguments will be removed from :func:`threading.RLock` in Python 3.15. - (Contributed by Nikita Sobolev in :gh:`102029`.) - -* :class:`typing.NamedTuple`: - - * The undocumented keyword argument syntax for creating NamedTuple classes - (``NT = NamedTuple("NT", x=int)``) is deprecated, and will be disallowed in - 3.15. Use the class-based syntax or the functional syntax instead. - - * When using the functional syntax to create a NamedTuple class, failing to - pass a value to the 'fields' parameter (``NT = NamedTuple("NT")``) is - deprecated. Passing ``None`` to the 'fields' parameter - (``NT = NamedTuple("NT", None)``) is also deprecated. Both will be - disallowed in Python 3.15. To create a NamedTuple class with 0 fields, use - ``class NT(NamedTuple): pass`` or ``NT = NamedTuple("NT", [])``. - -* :class:`typing.TypedDict`: When using the functional syntax to create a - TypedDict class, failing to pass a value to the 'fields' parameter (``TD = - TypedDict("TD")``) is deprecated. Passing ``None`` to the 'fields' parameter - (``TD = TypedDict("TD", None)``) is also deprecated. Both will be disallowed - in Python 3.15. To create a TypedDict class with 0 fields, use ``class - TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. - -* :mod:`wave`: Deprecate the ``getmark()``, ``setmark()`` and ``getmarkers()`` - methods of the :class:`wave.Wave_read` and :class:`wave.Wave_write` classes. - They will be removed in Python 3.15. - (Contributed by Victor Stinner in :gh:`105096`.) - -Pending Removal in Python 3.16 ------------------------------- - -* :class:`array.array` ``'u'`` type (:c:type:`wchar_t`): - use the ``'w'`` type instead (``Py_UCS4``). - -Pending Removal in Future Versions ----------------------------------- - -The following APIs were deprecated in earlier Python versions and will be removed, -although there is currently no date scheduled for their removal. - -* :mod:`argparse`: Nesting argument groups and nesting mutually exclusive - groups are deprecated. - -* :mod:`builtins`: - - * ``~bool``, bitwise inversion on bool. - * ``bool(NotImplemented)``. - * Generators: ``throw(type, exc, tb)`` and ``athrow(type, exc, tb)`` - signature is deprecated: use ``throw(exc)`` and ``athrow(exc)`` instead, - the single argument signature. - * Currently Python accepts numeric literals immediately followed by keywords, - for example ``0in x``, ``1or x``, ``0if 1else 2``. It allows confusing and - ambiguous expressions like ``[0x1for x in y]`` (which can be interpreted as - ``[0x1 for x in y]`` or ``[0x1f or x in y]``). A syntax warning is raised - if the numeric literal is immediately followed by one of keywords - :keyword:`and`, :keyword:`else`, :keyword:`for`, :keyword:`if`, - :keyword:`in`, :keyword:`is` and :keyword:`or`. In a future release it - will be changed to a syntax error. (:gh:`87999`) - * Support for ``__index__()`` and ``__int__()`` method returning non-int type: - these methods will be required to return an instance of a strict subclass of - :class:`int`. - * Support for ``__float__()`` method returning a strict subclass of - :class:`float`: these methods will be required to return an instance of - :class:`float`. - * Support for ``__complex__()`` method returning a strict subclass of - :class:`complex`: these methods will be required to return an instance of - :class:`complex`. - * Delegation of ``int()`` to ``__trunc__()`` method. - -* :mod:`calendar`: ``calendar.January`` and ``calendar.February`` constants are - deprecated and replaced by :data:`calendar.JANUARY` and - :data:`calendar.FEBRUARY`. - (Contributed by Prince Roshan in :gh:`103636`.) - -* :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method - instead. - -* :mod:`datetime`: - - * :meth:`~datetime.datetime.utcnow`: - use ``datetime.datetime.now(tz=datetime.UTC)``. - * :meth:`~datetime.datetime.utcfromtimestamp`: - use ``datetime.datetime.fromtimestamp(timestamp, tz=datetime.UTC)``. - -* :mod:`gettext`: Plural value must be an integer. - -* :mod:`importlib`: - - * ``load_module()`` method: use ``exec_module()`` instead. - * :func:`~importlib.util.cache_from_source` *debug_override* parameter is - deprecated: use the *optimization* parameter instead. - -* :mod:`importlib.metadata`: - - * ``EntryPoints`` tuple interface. - * Implicit ``None`` on return values. - -* :mod:`mailbox`: Use of StringIO input and text mode is deprecated, use - BytesIO and binary mode instead. - -* :mod:`os`: Calling :func:`os.register_at_fork` in multi-threaded process. - -* :class:`!pydoc.ErrorDuringImport`: A tuple value for *exc_info* parameter is - deprecated, use an exception instance. - -* :mod:`re`: More strict rules are now applied for numerical group references - and group names in regular expressions. Only sequence of ASCII digits is now - accepted as a numerical reference. The group name in bytes patterns and - replacement strings can now only contain ASCII letters and digits and - underscore. - (Contributed by Serhiy Storchaka in :gh:`91760`.) - -* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. - -* :mod:`ssl` options and protocols: - - * :class:`ssl.SSLContext` without protocol argument is deprecated. - * :class:`ssl.SSLContext`: :meth:`~ssl.SSLContext.set_npn_protocols` and - :meth:`!selected_npn_protocol` are deprecated: use ALPN - instead. - * ``ssl.OP_NO_SSL*`` options - * ``ssl.OP_NO_TLS*`` options - * ``ssl.PROTOCOL_SSLv3`` - * ``ssl.PROTOCOL_TLS`` - * ``ssl.PROTOCOL_TLSv1`` - * ``ssl.PROTOCOL_TLSv1_1`` - * ``ssl.PROTOCOL_TLSv1_2`` - * ``ssl.TLSVersion.SSLv3`` - * ``ssl.TLSVersion.TLSv1`` - * ``ssl.TLSVersion.TLSv1_1`` - -* :func:`sysconfig.is_python_build` *check_home* parameter is deprecated and - ignored. - -* :mod:`threading` methods: - - * :meth:`!threading.Condition.notifyAll`: use :meth:`~threading.Condition.notify_all`. - * :meth:`!threading.Event.isSet`: use :meth:`~threading.Event.is_set`. - * :meth:`!threading.Thread.isDaemon`, :meth:`threading.Thread.setDaemon`: - use :attr:`threading.Thread.daemon` attribute. - * :meth:`!threading.Thread.getName`, :meth:`threading.Thread.setName`: - use :attr:`threading.Thread.name` attribute. - * :meth:`!threading.currentThread`: use :meth:`threading.current_thread`. - * :meth:`!threading.activeCount`: use :meth:`threading.active_count`. - -* :class:`typing.Text` (:gh:`92332`). - -* :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value - that is not ``None`` from a test case. - -* :mod:`urllib.parse` deprecated functions: :func:`~urllib.parse.urlparse` instead - - * ``splitattr()`` - * ``splithost()`` - * ``splitnport()`` - * ``splitpasswd()`` - * ``splitport()`` - * ``splitquery()`` - * ``splittag()`` - * ``splittype()`` - * ``splituser()`` - * ``splitvalue()`` - * ``to_bytes()`` -* :mod:`urllib.request`: :class:`~urllib.request.URLopener` and - :class:`~urllib.request.FancyURLopener` style of invoking requests is - deprecated. Use newer :func:`~urllib.request.urlopen` functions and methods. +opcode +------ -* :mod:`wsgiref`: ``SimpleHandler.stdout.write()`` should not do partial - writes. +* Move :attr:`!opcode.ENABLE_SPECIALIZATION` to :attr:`!_opcode.ENABLE_SPECIALIZATION`. + This field was added in 3.12, it was never documented, + and is not intended for external use. + (Contributed by Irit Katriel in :gh:`105481`.) -* :meth:`zipimport.zipimporter.load_module` is deprecated: - use :meth:`~zipimport.zipimporter.exec_module` instead. +* Remove :func:`!opcode.is_pseudo`, :attr:`!opcode.MIN_PSEUDO_OPCODE`, + and :attr:`!opcode.MAX_PSEUDO_OPCODE`, which were added in Python 3.12, + but were neither documented nor exposed through :mod:`dis`, + and were not intended to be used externally. + (Contributed by Irit Katriel in :gh:`105481`.) -Removed -======= +optparse +-------- -.. _whatsnew313-pep594: +* This module is no longer considered :term:`soft deprecated`. + While :mod:`argparse` remains preferred for new projects that + aren't using a third party command line argument processing + library, there are aspects of the way ``argparse`` works that + mean the lower level ``optparse`` module may provide a better + foundation for *writing* argument processing libraries, and + for implementing command line applications which adhere more + strictly than ``argparse`` does to various Unix command line + processing conventions that originate in the behaviour of the + C :c:func:`!getopt` function . + (Contributed by Alyssa Coghlan and Serhiy Storchaka in :gh:`126180`.) -PEP 594: dead batteries ------------------------ -* :pep:`594` removed 19 modules from the standard library, - deprecated in Python 3.11: +pathlib +------- - * :mod:`!aifc`. - (Contributed by Victor Stinner in :gh:`104773`.) +* Remove the ability to use :class:`~pathlib.Path` objects as context managers. + This functionality was deprecated and has had no effect since Python 3.9. + (Contributed by Barney Gale in :gh:`83863`.) - * :mod:`!audioop`. - (Contributed by Victor Stinner in :gh:`104773`.) - * :mod:`!chunk`. - (Contributed by Victor Stinner in :gh:`104773`.) +re +-- - * :mod:`!cgi` and :mod:`!cgitb`. +* Remove the undocumented, deprecated, and broken + :func:`!re.template` function and :attr:`!re.TEMPLATE` / :attr:`!re.T` flag. + (Contributed by Serhiy Storchaka and Nikita Sobolev in :gh:`105687`.) - * ``cgi.FieldStorage`` can typically be replaced with - :func:`urllib.parse.parse_qsl` for ``GET`` and ``HEAD`` requests, - and the :mod:`email.message` module or `multipart - `__ PyPI project for ``POST`` and - ``PUT``. - * ``cgi.parse()`` can be replaced by calling :func:`urllib.parse.parse_qs` - directly on the desired query string, except for ``multipart/form-data`` - input, which can be handled as described for ``cgi.parse_multipart()``. +tkinter.tix +----------- - * ``cgi.parse_header()`` can be replaced with the functionality in the - :mod:`email` package, which implements the same MIME RFCs. For example, - with :class:`email.message.EmailMessage`:: +* Remove the :mod:`!tkinter.tix` module, deprecated in Python 3.6. + The third-party Tix library which the module wrapped is unmaintained. + (Contributed by Zachary Ware in :gh:`75552`.) - from email.message import EmailMessage - msg = EmailMessage() - msg['content-type'] = 'application/json; charset="utf8"' - main, params = msg.get_content_type(), msg['content-type'].params - * ``cgi.parse_multipart()`` can be replaced with the functionality in the - :mod:`email` package (e.g. :class:`email.message.EmailMessage` and - :class:`email.message.Message`) which implements the same MIME RFCs, or - with the `multipart `__ PyPI project. +turtle +------ - (Contributed by Victor Stinner in :gh:`104773`.) +* Remove the :meth:`!RawTurtle.settiltangle` method, + deprecated in the documentation since Python 3.1 + and at runtime since Python 3.11. + (Contributed by Hugo van Kemenade in :gh:`104876`.) - * :mod:`!crypt` module and its private :mod:`!_crypt` extension. - The :mod:`hashlib` module is a potential replacement for certain use cases. - Otherwise, the following PyPI projects can be used: - * `bcrypt `_: - Modern password hashing for your software and your servers. - * `passlib `_: - Comprehensive password hashing framework supporting over 30 schemes. - * `argon2-cffi `_: - The secure Argon2 password hashing algorithm. - * `legacycrypt `_: - Wrapper to the POSIX crypt library call and associated functionality. +typing +------ - (Contributed by Victor Stinner in :gh:`104773`.) +* Remove the :mod:`!typing.io` and :mod:`!typing.re` namespaces, + deprecated since Python 3.8. + The items in those namespaces can be imported directly + from the :mod:`typing` module. + (Contributed by Sebastian Rittau in :gh:`92871`.) - * :mod:`!imghdr`: use the projects - `filetype `_, - `puremagic `_, - or `python-magic `_ instead. - (Contributed by Victor Stinner in :gh:`104773`.) +* Remove the keyword-argument method of creating + :class:`~typing.TypedDict` types, deprecated in Python 3.11. + (Contributed by Tomas Roun in :gh:`104786`.) - * :mod:`!mailcap`. - The :mod:`mimetypes` module provides an alternative. - (Contributed by Victor Stinner in :gh:`104773`.) - * :mod:`!msilib`. - (Contributed by Zachary Ware in :gh:`104773`.) +unittest +-------- - * :mod:`!nis`. - (Contributed by Victor Stinner in :gh:`104773`.) +* Remove the following :mod:`unittest` functions, deprecated in Python 3.11: - * :mod:`!nntplib`: - the `PyPI nntplib project `_ - can be used instead. - (Contributed by Victor Stinner in :gh:`104773`.) + * :func:`!unittest.findTestCases` + * :func:`!unittest.makeSuite` + * :func:`!unittest.getTestCaseNames` - * :mod:`!ossaudiodev`: use the - `pygame project `_ for audio playback. - (Contributed by Victor Stinner in :gh:`104780`.) + Use :class:`~unittest.TestLoader` methods instead: - * :mod:`!pipes`: use the :mod:`subprocess` module instead. - (Contributed by Victor Stinner in :gh:`104773`.) + * :meth:`~unittest.TestLoader.loadTestsFromModule` + * :meth:`~unittest.TestLoader.loadTestsFromTestCase` + * :meth:`~unittest.TestLoader.getTestCaseNames` - * :mod:`!sndhdr`: use the projects - `filetype `_, - `puremagic `_, or - `python-magic `_ instead. - (Contributed by Victor Stinner in :gh:`104773`.) + (Contributed by Hugo van Kemenade in :gh:`104835`.) - * :mod:`!spwd`: - the `python-pam project `_ - can be used instead. - (Contributed by Victor Stinner in :gh:`104773`.) +* Remove the untested and undocumented :meth:`!TestProgram.usageExit` + method, deprecated in Python 3.11. + (Contributed by Hugo van Kemenade in :gh:`104992`.) - * :mod:`!sunau`. - (Contributed by Victor Stinner in :gh:`104773`.) - * :mod:`!telnetlib`, use the projects - `telnetlib3 `_ or - `Exscript `_ instead. - (Contributed by Victor Stinner in :gh:`104773`.) +urllib +------ - * :mod:`!uu`: the :mod:`base64` module is a modern alternative. - (Contributed by Victor Stinner in :gh:`104773`.) +* Remove the *cafile*, *capath*, and *cadefault* parameters of the + :func:`urllib.request.urlopen` function, deprecated in Python 3.6. + Use the *context* parameter instead with an :class:`~ssl.SSLContext` instance. + The :meth:`ssl.SSLContext.load_cert_chain` function + can be used to load specific certificates, + or let :func:`ssl.create_default_context` select + the operating system's trusted certificate authority (CA) certificates. + (Contributed by Victor Stinner in :gh:`105382`.) - * :mod:`!xdrlib`. - (Contributed by Victor Stinner in :gh:`104773`.) -2to3 ----- +webbrowser +---------- -* Remove the ``2to3`` program and the :mod:`!lib2to3` module, +* Remove the untested and undocumented :class:`!MacOSX` class, deprecated in Python 3.11. - (Contributed by Victor Stinner in :gh:`104780`.) - -configparser ------------- - -* Remove the undocumented :class:`!configparser.LegacyInterpolation` class, - deprecated in the docstring since Python 3.2, - and with a deprecation warning since Python 3.11. - (Contributed by Hugo van Kemenade in :gh:`104886`.) + Use the :class:`!MacOSXOSAScript` class (introduced in Python 3.2) instead. + (Contributed by Hugo van Kemenade in :gh:`104804`.) -importlib ---------- +* Remove the deprecated :attr:`!MacOSXOSAScript._name` attribute. + Use the :attr:`MacOSXOSAScript.name ` + attribute instead. + (Contributed by Nikita Sobolev in :gh:`105546`.) -* Remove :mod:`importlib.resources` deprecated methods: - * ``contents()`` - * ``is_resource()`` - * ``open_binary()`` - * ``open_text()`` - * ``path()`` - * ``read_binary()`` - * ``read_text()`` +New Deprecations +================ - Use :func:`importlib.resources.files()` instead. Refer to `importlib-resources: Migrating from Legacy - `_ - for migration advice. - (Contributed by Jason R. Coombs in :gh:`106532`.) +* :ref:`User-defined functions `: -* Remove deprecated :meth:`~object.__getitem__` access for - :class:`!importlib.metadata.EntryPoint` objects. - (Contributed by Jason R. Coombs in :gh:`113175`.) + * Deprecate assignment to a function's :attr:`~function.__code__` attribute, + where the new code object's type does not match the function's type. + The different types are: + plain function, generator, async generator, and coroutine. + (Contributed by Irit Katriel in :gh:`81137`.) -locale ------- +* :mod:`array`: -* Remove ``locale.resetlocale()`` function deprecated in Python 3.11: - use ``locale.setlocale(locale.LC_ALL, "")`` instead. - (Contributed by Victor Stinner in :gh:`104783`.) + * Deprecate the ``'u'`` format code (:c:type:`wchar_t`) at runtime. + This format code has been deprecated in documentation since Python 3.3, + and will be removed in Python 3.16. + Use the ``'w'`` format code (:c:type:`Py_UCS4`) + for Unicode characters instead. + (Contributed by Hugo van Kemenade in :gh:`80480`.) -logging -------- +* :mod:`ctypes`: -* :mod:`logging`: Remove undocumented and untested ``Logger.warn()`` and - ``LoggerAdapter.warn()`` methods and ``logging.warn()`` function. Deprecated - since Python 3.3, they were aliases to the :meth:`logging.Logger.warning` - method, :meth:`!logging.LoggerAdapter.warning` method and - :func:`logging.warning` function. - (Contributed by Victor Stinner in :gh:`105376`.) + * Deprecate the undocumented :func:`!SetPointerType` function, + to be removed in Python 3.15. + (Contributed by Victor Stinner in :gh:`105733`.) -pathlib -------- + * :term:`Soft-deprecate ` the :func:`~ctypes.ARRAY` + function in favour of ``type * length`` multiplication. + (Contributed by Victor Stinner in :gh:`105733`.) -* Remove support for using :class:`pathlib.Path` objects as context managers. - This functionality was deprecated and made a no-op in Python 3.9. +* :mod:`decimal`: -re --- + * Deprecate the non-standard and undocumented :class:`~decimal.Decimal` + format specifier ``'N'``, + which is only supported in the :mod:`!decimal` module's C implementation. + (Contributed by Serhiy Storchaka in :gh:`89902`.) -* Remove undocumented, never working, and deprecated ``re.template`` function - and ``re.TEMPLATE`` flag (and ``re.T`` alias). - (Contributed by Serhiy Storchaka and Nikita Sobolev in :gh:`105687`.) +* :mod:`dis`: -tkinter -------- + * Deprecate the :attr:`!HAVE_ARGUMENT` separator. + Check membership in :data:`~dis.hasarg` instead. + (Contributed by Irit Katriel in :gh:`109319`.) -* Remove the :mod:`!tkinter.tix` module, deprecated in Python 3.6. The - third-party Tix library which the module wrapped is unmaintained. - (Contributed by Zachary Ware in :gh:`75552`.) +* :mod:`gettext`: -turtle ------- + * Deprecate non-integer numbers as arguments to functions and methods + that consider plural forms in the :mod:`!gettext` module, + even if no translation was found. + (Contributed by Serhiy Storchaka in :gh:`88434`.) -* Remove the :meth:`!turtle.RawTurtle.settiltangle` method, - deprecated in docs since Python 3.1 - and with a deprecation warning since Python 3.11. - (Contributed by Hugo van Kemenade in :gh:`104876`.) +* :mod:`glob`: -typing ------- + * Deprecate the undocumented :func:`!glob0` and :func:`!glob1` functions. + Use :func:`~glob.glob` and pass a :term:`path-like object` specifying + the root directory to the *root_dir* parameter instead. + (Contributed by Barney Gale in :gh:`117337`.) -* Namespaces ``typing.io`` and ``typing.re``, deprecated in Python 3.8, - are now removed. The items in those namespaces can be imported directly - from :mod:`typing`. (Contributed by Sebastian Rittau in :gh:`92871`.) +* :mod:`http.server`: -* Remove support for the keyword-argument method of creating - :class:`typing.TypedDict` types, deprecated in Python 3.11. - (Contributed by Tomas Roun in :gh:`104786`.) + * Deprecate :class:`~http.server.CGIHTTPRequestHandler`, + to be removed in Python 3.15. + Process-based CGI HTTP servers have been out of favor for a very long time. + This code was outdated, unmaintained, and rarely used. + It has a high potential for both security and functionality bugs. + (Contributed by Gregory P. Smith in :gh:`109096`.) -unittest --------- + * Deprecate the :option:`!--cgi` flag to + the :program:`python -m http.server` command-line interface, + to be removed in Python 3.15. + (Contributed by Gregory P. Smith in :gh:`109096`.) -* Removed the following :mod:`unittest` functions, deprecated in Python 3.11: +* :mod:`mimetypes`: - * :func:`!unittest.findTestCases` - * :func:`!unittest.makeSuite` - * :func:`!unittest.getTestCaseNames` + * :term:`Soft-deprecate ` file path arguments + to :func:`~mimetypes.guess_type`, + use :func:`~mimetypes.guess_file_type` instead. + (Contributed by Serhiy Storchaka in :gh:`66543`.) - Use :class:`~unittest.TestLoader` methods instead: +* :mod:`re`: - * :meth:`unittest.TestLoader.loadTestsFromModule` - * :meth:`unittest.TestLoader.loadTestsFromTestCase` - * :meth:`unittest.TestLoader.getTestCaseNames` + * Deprecate passing the optional *maxsplit*, *count*, or *flags* arguments + as positional arguments to the module-level + :func:`~re.split`, :func:`~re.sub`, and :func:`~re.subn` functions. + These parameters will become :ref:`keyword-only ` + in a future version of Python. + (Contributed by Serhiy Storchaka in :gh:`56166`.) - (Contributed by Hugo van Kemenade in :gh:`104835`.) +* :mod:`pathlib`: -* Remove the untested and undocumented :meth:`!unittest.TestProgram.usageExit` - method, deprecated in Python 3.11. - (Contributed by Hugo van Kemenade in :gh:`104992`.) + * Deprecate :meth:`.PurePath.is_reserved`, + to be removed in Python 3.15. + Use :func:`os.path.isreserved` to detect reserved paths on Windows. + (Contributed by Barney Gale in :gh:`88569`.) -urllib ------- +* :mod:`platform`: -* Remove *cafile*, *capath* and *cadefault* parameters of the - :func:`urllib.request.urlopen` function, deprecated in Python 3.6: use the - *context* parameter instead. Please use - :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. - (Contributed by Victor Stinner in :gh:`105382`.) + * Deprecate :func:`~platform.java_ver`, + to be removed in Python 3.15. + This function is only useful for Jython support, has a confusing API, + and is largely untested. + (Contributed by Nikita Sobolev in :gh:`116349`.) -webbrowser ----------- +* :mod:`pydoc`: -* Remove the untested and undocumented :mod:`webbrowser` :class:`!MacOSX` class, - deprecated in Python 3.11. - Use the :class:`!MacOSXOSAScript` class (introduced in Python 3.2) instead. - (Contributed by Hugo van Kemenade in :gh:`104804`.) + * Deprecate the undocumented :func:`!ispackage` function. + (Contributed by Zackery Spytz in :gh:`64020`.) -* Remove deprecated ``webbrowser.MacOSXOSAScript._name`` attribute. - Use :attr:`webbrowser.MacOSXOSAScript.name ` - attribute instead. - (Contributed by Nikita Sobolev in :gh:`105546`.) +* :mod:`sqlite3`: -Others ------- + * Deprecate passing more than one positional argument to + the :func:`~sqlite3.connect` function + and the :class:`~sqlite3.Connection` constructor. + The remaining parameters will become keyword-only in Python 3.15. + (Contributed by Erlend E. Aasland in :gh:`107948`.) -* None yet + * Deprecate passing name, number of arguments, and the callable as keyword + arguments for :meth:`.Connection.create_function` + and :meth:`.Connection.create_aggregate` + These parameters will become positional-only in Python 3.15. + (Contributed by Erlend E. Aasland in :gh:`108278`.) -CPython bytecode changes -======================== + * Deprecate passing the callback callable by keyword for the + :meth:`~sqlite3.Connection.set_authorizer`, + :meth:`~sqlite3.Connection.set_progress_handler`, and + :meth:`~sqlite3.Connection.set_trace_callback` + :class:`~sqlite3.Connection` methods. + The callback callables will become positional-only in Python 3.15. + (Contributed by Erlend E. Aasland in :gh:`108278`.) -* The oparg of ``YIELD_VALUE`` is now ``1`` if the yield is part of a - yield-from or await, and ``0`` otherwise. The oparg of ``RESUME`` was - changed to add a bit indicating whether the except-depth is 1, which - is needed to optimize closing of generators. - (Contributed by Irit Katriel in :gh:`111354`.) +* :mod:`sys`: -Porting to Python 3.13 -====================== + * Deprecate the :func:`~sys._enablelegacywindowsfsencoding` function, + to be removed in Python 3.16. + Use the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable instead. + (Contributed by Inada Naoki in :gh:`73427`.) -This section lists previously described changes and other bugfixes -that may require changes to your code. +* :mod:`tarfile`: -Changes in the Python API -------------------------- + * Deprecate the undocumented and unused :attr:`!TarFile.tarfile` attribute, + to be removed in Python 3.16. + (Contributed in :gh:`115256`.) -* Functions :c:func:`PyDict_GetItem`, :c:func:`PyDict_GetItemString`, - :c:func:`PyMapping_HasKey`, :c:func:`PyMapping_HasKeyString`, - :c:func:`PyObject_HasAttr`, :c:func:`PyObject_HasAttrString`, and - :c:func:`PySys_GetObject`, which clear all errors which occurred when calling - them, now report them using :func:`sys.unraisablehook`. - You may replace them with other functions as - recommended in the documentation. - (Contributed by Serhiy Storchaka in :gh:`106672`.) +* :mod:`traceback`: -* An :exc:`OSError` is now raised by :func:`getpass.getuser` for any failure to - retrieve a username, instead of :exc:`ImportError` on non-Unix platforms or - :exc:`KeyError` on Unix platforms where the password database is empty. + * Deprecate the :attr:`.TracebackException.exc_type` attribute. + Use :attr:`.TracebackException.exc_type_str` instead. + (Contributed by Irit Katriel in :gh:`112332`.) -* The :mod:`threading` module now expects the :mod:`!_thread` module to have - an ``_is_main_interpreter`` attribute. It is a function with no - arguments that return ``True`` if the current interpreter is the - main interpreter. +* :mod:`typing`: - Any library or application that provides a custom ``_thread`` module - must provide ``_is_main_interpreter()``, just like the module's - other "private" attributes. - (See :gh:`112826`.) + * Deprecate the undocumented keyword argument syntax for creating + :class:`~typing.NamedTuple` classes + (e.g. ``Point = NamedTuple("Point", x=int, y=int)``), + to be removed in Python 3.15. + Use the class-based syntax or the functional syntax instead. + (Contributed by Alex Waygood in :gh:`105566`.) + + * Deprecate omitting the *fields* parameter when creating + a :class:`~typing.NamedTuple` or :class:`typing.TypedDict` class, + and deprecate passing ``None`` to the *fields* parameter of both types. + Python 3.15 will require a valid sequence for the *fields* parameter. + To create a NamedTuple class with zero fields, + use ``class NT(NamedTuple): pass`` or ``NT = NamedTuple("NT", ())``. + To create a TypedDict class with zero fields, + use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. + (Contributed by Alex Waygood in :gh:`105566` and :gh:`105570`.) -* :class:`mailbox.Maildir` now ignores files with a leading dot. - (Contributed by Zackery Spytz in :gh:`65559`.) + * Deprecate the :func:`typing.no_type_check_decorator` decorator function, + to be removed in in Python 3.15. + After eight years in the :mod:`typing` module, + it has yet to be supported by any major type checker. + (Contributed by Alex Waygood in :gh:`106309`.) -* :meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob` now return both - files and directories if a pattern that ends with "``**``" is given, rather - than directories only. Users may add a trailing slash to match only - directories. + * Deprecate :data:`typing.AnyStr`. + In Python 3.16, it will be removed from ``typing.__all__``, + and a :exc:`DeprecationWarning` will be emitted at runtime + when it is imported or accessed. + It will be removed entirely in Python 3.18. + Use the new :ref:`type parameter syntax ` instead. + (Contributed by Michael The in :gh:`107116`.) +* :mod:`wave`: -Build Changes -============= + * Deprecate the :meth:`~wave.Wave_read.getmark`, :meth:`!setmark`, + and :meth:`~wave.Wave_read.getmarkers` methods of + the :class:`~wave.Wave_read` and :class:`~wave.Wave_write` classes, + to be removed in Python 3.15. + (Contributed by Victor Stinner in :gh:`105096`.) -* Autoconf 2.71 and aclocal 1.16.4 is now required to regenerate - the :file:`configure` script. - (Contributed by Christian Heimes in :gh:`89886`.) +.. Add deprecations above alphabetically, not here at the end. -* SQLite 3.15.2 or newer is required to build the :mod:`sqlite3` extension module. - (Contributed by Erlend Aasland in :gh:`105875`.) +.. include:: ../deprecations/pending-removal-in-3.14.rst -* Python built with :file:`configure` :option:`--with-trace-refs` (tracing - references) is now ABI compatible with the Python release build and - :ref:`debug build `. - (Contributed by Victor Stinner in :gh:`108634`.) +.. include:: ../deprecations/pending-removal-in-3.15.rst -* Building CPython now requires a compiler with support for the C11 atomic - library, GCC built-in atomic functions, or MSVC interlocked intrinsics. +.. include:: ../deprecations/pending-removal-in-3.16.rst -* The ``errno``, ``md5``, ``resource``, ``winsound``, ``_ctypes_test``, - ``_multiprocessing.posixshmem``, ``_scproxy``, ``_stat``, - ``_testimportmultiple`` and ``_uuid`` C extensions are now built with the - :ref:`limited C API `. - (Contributed by Victor Stinner in :gh:`85283`.) +.. include:: ../deprecations/pending-removal-in-future.rst -* ``wasm32-wasi`` is now a tier 2 platform. - (Contributed by Brett Cannon in :gh:`115192`.) +CPython Bytecode Changes +======================== -* ``wasm32-emscripten`` is no longer a supported platform. - (Contributed by Brett Cannon in :gh:`115192`.) +* The oparg of :opcode:`YIELD_VALUE` is now + ``1`` if the yield is part of a yield-from or await, and ``0`` otherwise. + The oparg of :opcode:`RESUME` was changed to add a bit indicating + if the except-depth is 1, which is needed to optimize closing of generators. + (Contributed by Irit Katriel in :gh:`111354`.) C API Changes @@ -1446,324 +2027,328 @@ C API Changes New Features ------------ -* You no longer have to define the ``PY_SSIZE_T_CLEAN`` macro before including - :file:`Python.h` when using ``#`` formats in - :ref:`format codes `. - APIs accepting the format codes always use ``Py_ssize_t`` for ``#`` formats. - (Contributed by Inada Naoki in :gh:`104922`.) - -* The *keywords* parameter of :c:func:`PyArg_ParseTupleAndKeywords` and - :c:func:`PyArg_VaParseTupleAndKeywords` now has type :c:expr:`char * const *` - in C and :c:expr:`const char * const *` in C++, instead of :c:expr:`char **`. - It makes these functions compatible with arguments of type - :c:expr:`const char * const *`, :c:expr:`const char **` or - :c:expr:`char * const *` in C++ and :c:expr:`char * const *` in C - without an explicit type cast. - This can be overridden with the :c:macro:`PY_CXX_CONST` macro. - (Contributed by Serhiy Storchaka in :gh:`65210`.) - -* Add :c:func:`PyImport_AddModuleRef`: similar to - :c:func:`PyImport_AddModule`, but return a :term:`strong reference` instead - of a :term:`borrowed reference`. - (Contributed by Victor Stinner in :gh:`105922`.) +* Add the :ref:`PyMonitoring C API ` + for generating :pep:`669` monitoring events: + + * :c:type:`PyMonitoringState` + * :c:func:`PyMonitoring_FirePyStartEvent` + * :c:func:`PyMonitoring_FirePyResumeEvent` + * :c:func:`PyMonitoring_FirePyReturnEvent` + * :c:func:`PyMonitoring_FirePyYieldEvent` + * :c:func:`PyMonitoring_FireCallEvent` + * :c:func:`PyMonitoring_FireLineEvent` + * :c:func:`PyMonitoring_FireJumpEvent` + * ``PyMonitoring_FireBranchEvent`` + * :c:func:`PyMonitoring_FireCReturnEvent` + * :c:func:`PyMonitoring_FirePyThrowEvent` + * :c:func:`PyMonitoring_FireRaiseEvent` + * :c:func:`PyMonitoring_FireCRaiseEvent` + * :c:func:`PyMonitoring_FireReraiseEvent` + * :c:func:`PyMonitoring_FireExceptionHandledEvent` + * :c:func:`PyMonitoring_FirePyUnwindEvent` + * :c:func:`PyMonitoring_FireStopIterationEvent` + * :c:func:`PyMonitoring_EnterScope` + * :c:func:`PyMonitoring_ExitScope` + + (Contributed by Irit Katriel in :gh:`111997`). + +* Add :c:type:`PyMutex`, a lightweight mutex that occupies a single byte, + and the new :c:func:`PyMutex_Lock` and :c:func:`PyMutex_Unlock` functions. + :c:func:`!PyMutex_Lock` will release the :term:`GIL` (if currently held) + if the operation needs to block. + (Contributed by Sam Gross in :gh:`108724`.) + +* Add the :ref:`PyTime C API ` to provide access to system clocks: + + * :c:type:`PyTime_t`. + * :c:var:`PyTime_MIN` and :c:var:`PyTime_MAX`. + * :c:func:`PyTime_AsSecondsDouble`. + * :c:func:`PyTime_Monotonic`. + * :c:func:`PyTime_MonotonicRaw`. + * :c:func:`PyTime_PerfCounter`. + * :c:func:`PyTime_PerfCounterRaw`. + * :c:func:`PyTime_Time`. + * :c:func:`PyTime_TimeRaw`. -* Add :c:func:`PyWeakref_GetRef` function: similar to - :c:func:`PyWeakref_GetObject` but returns a :term:`strong reference`, or - ``NULL`` if the referent is no longer live. - (Contributed by Victor Stinner in :gh:`105927`.) + (Contributed by Victor Stinner and Petr Viktorin in :gh:`110850`.) -* Add :c:func:`PyObject_GetOptionalAttr` and - :c:func:`PyObject_GetOptionalAttrString`, variants of - :c:func:`PyObject_GetAttr` and :c:func:`PyObject_GetAttrString` which - don't raise :exc:`AttributeError` if the attribute is not found. - These variants are more convenient and faster if the missing attribute - should not be treated as a failure. - (Contributed by Serhiy Storchaka in :gh:`106521`.) +* Add the :c:func:`PyDict_ContainsString` function + with the same behavior as :c:func:`PyDict_Contains`, + but *key* is specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + (Contributed by Victor Stinner in :gh:`108314`.) -* Add :c:func:`PyMapping_GetOptionalItem` and - :c:func:`PyMapping_GetOptionalItemString`: variants of - :c:func:`PyObject_GetItem` and :c:func:`PyMapping_GetItemString` which don't - raise :exc:`KeyError` if the key is not found. - These variants are more convenient and faster if the missing key should not - be treated as a failure. - (Contributed by Serhiy Storchaka in :gh:`106307`.) +* Add the :c:func:`PyDict_GetItemRef` and :c:func:`PyDict_GetItemStringRef` + functions, + which behave similarly to :c:func:`PyDict_GetItemWithError`, + but return a :term:`strong reference` instead of a :term:`borrowed reference`. + Moreover, these functions return ``-1`` on error, + removing the need to check :c:func:`!PyErr_Occurred`. + (Contributed by Victor Stinner in :gh:`106004`.) -* Add fixed variants of functions which silently ignore errors: +* Add the :c:func:`PyDict_SetDefaultRef` function, + which behaves similarly to :c:func:`PyDict_SetDefault`, + but returns a :term:`strong reference` instead of a :term:`borrowed reference`. + This function returns ``-1`` on error, + ``0`` on insertion, + and ``1`` if the key was already present in the dictionary. + (Contributed by Sam Gross in :gh:`112066`.) - - :c:func:`PyObject_HasAttrWithError` replaces :c:func:`PyObject_HasAttr`. - - :c:func:`PyObject_HasAttrStringWithError` replaces :c:func:`PyObject_HasAttrString`. - - :c:func:`PyMapping_HasKeyWithError` replaces :c:func:`PyMapping_HasKey`. - - :c:func:`PyMapping_HasKeyStringWithError` replaces :c:func:`PyMapping_HasKeyString`. +* Add the :c:func:`PyDict_Pop` and :c:func:`PyDict_PopString` functions + to remove a key from a dictionary and optionally return the removed value. + This is similar to :meth:`dict.pop`, + though there is no default value, + and :exc:`KeyError` is not raised for missing keys. + (Contributed by Stefan Behnel and Victor Stinner in :gh:`111262`.) - New functions return not only ``1`` for true and ``0`` for false, but also - ``-1`` for error. +* Add the :c:func:`PyMapping_GetOptionalItem` + and :c:func:`PyMapping_GetOptionalItemString` functions + as alternatives to :c:func:`PyObject_GetItem` + and :c:func:`PyMapping_GetItemString` respectively. + The new functions do not raise :exc:`KeyError` + if the requested key is missing from the mapping. + These variants are more convenient and faster + if a missing key should not be treated as a failure. + (Contributed by Serhiy Storchaka in :gh:`106307`.) - (Contributed by Serhiy Storchaka in :gh:`108511`.) +* Add the :c:func:`PyObject_GetOptionalAttr` + and :c:func:`PyObject_GetOptionalAttrString` functions + as alternatives to :c:func:`PyObject_GetAttr` + and :c:func:`PyObject_GetAttrString` respectively. + The new functions do not raise :exc:`AttributeError` + if the requested attribute is not found on the object. + These variants are more convenient and faster + if the missing attribute should not be treated as a failure. + (Contributed by Serhiy Storchaka in :gh:`106521`.) -* If Python is built in :ref:`debug mode ` or :option:`with - assertions <--with-assertions>`, :c:func:`PyTuple_SET_ITEM` and - :c:func:`PyList_SET_ITEM` now check the index argument with an assertion. - (Contributed by Victor Stinner in :gh:`106168`.) +* Add the :c:func:`PyErr_FormatUnraisable` function + as an extension to :c:func:`PyErr_WriteUnraisable` + that allows customizing the warning message. + (Contributed by Serhiy Storchaka in :gh:`108082`.) -* Add :c:func:`PyModule_Add` function: similar to - :c:func:`PyModule_AddObjectRef` and :c:func:`PyModule_AddObject` but - always steals a reference to the value. - (Contributed by Serhiy Storchaka in :gh:`86493`.) +* Add new functions that return a :term:`strong reference` instead of + a :term:`borrowed reference` for frame locals, globals, and builtins, + as part of :ref:`PEP 667 `: -* Add :c:func:`PyDict_GetItemRef` and :c:func:`PyDict_GetItemStringRef` - functions: similar to :c:func:`PyDict_GetItemWithError` but returning a - :term:`strong reference` instead of a :term:`borrowed reference`. Moreover, - these functions return -1 on error and so checking ``PyErr_Occurred()`` is - not needed. - (Contributed by Victor Stinner in :gh:`106004`.) + * :c:func:`PyEval_GetFrameBuiltins` replaces :c:func:`PyEval_GetBuiltins` + * :c:func:`PyEval_GetFrameGlobals` replaces :c:func:`PyEval_GetGlobals` + * :c:func:`PyEval_GetFrameLocals` replaces :c:func:`PyEval_GetLocals` -* Added :c:func:`PyDict_SetDefaultRef`, which is similar to - :c:func:`PyDict_SetDefault` but returns a :term:`strong reference` instead of - a :term:`borrowed reference`. This function returns ``-1`` on error, ``0`` on - insertion, and ``1`` if the key was already present in the dictionary. - (Contributed by Sam Gross in :gh:`112066`.) + (Contributed by Mark Shannon and Tian Gao in :gh:`74929`.) -* Add :c:func:`PyDict_ContainsString` function: same as - :c:func:`PyDict_Contains`, but *key* is specified as a :c:expr:`const char*` - UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. - (Contributed by Victor Stinner in :gh:`108314`.) +* Add the :c:func:`Py_GetConstant` and :c:func:`Py_GetConstantBorrowed` + functions to get :term:`strong ` + or :term:`borrowed ` references to constants. + For example, ``Py_GetConstant(Py_CONSTANT_ZERO)`` returns a strong reference + to the constant zero. + (Contributed by Victor Stinner in :gh:`115754`.) -* Added :c:func:`PyList_GetItemRef` function: similar to - :c:func:`PyList_GetItem` but returns a :term:`strong reference` instead of - a :term:`borrowed reference`. +* Add the :c:func:`PyImport_AddModuleRef` function + as a replacement for :c:func:`PyImport_AddModule` + that returns a :term:`strong reference` instead of a :term:`borrowed reference`. + (Contributed by Victor Stinner in :gh:`105922`.) -* Add :c:func:`Py_IsFinalizing` function: check if the main Python interpreter is +* Add the :c:func:`Py_IsFinalizing` function to check + whether the main Python interpreter is :term:`shutting down `. (Contributed by Victor Stinner in :gh:`108014`.) -* Add :c:func:`PyLong_AsInt` function: similar to :c:func:`PyLong_AsLong`, but - store the result in a C :c:expr:`int` instead of a C :c:expr:`long`. - Previously, it was known as the private function :c:func:`!_PyLong_AsInt` - (with an underscore prefix). - (Contributed by Victor Stinner in :gh:`108014`.) +* Add the :c:func:`PyList_GetItemRef` function + as a replacement for :c:func:`PyList_GetItem` + that returns a :term:`strong reference` instead of a :term:`borrowed reference`. + (Contributed by Sam Gross in :gh:`114329`.) -* Python built with :file:`configure` :option:`--with-trace-refs` (tracing - references) now supports the :ref:`Limited API `. - (Contributed by Victor Stinner in :gh:`108634`.) - -* Add :c:func:`PyObject_VisitManagedDict` and - :c:func:`PyObject_ClearManagedDict` functions which must be called by the - traverse and clear functions of a type using - :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag. The `pythoncapi-compat project - `__ can be used to get these - functions on Python 3.11 and 3.12. - (Contributed by Victor Stinner in :gh:`107073`.) +* Add the :c:func:`PyList_Extend` and :c:func:`PyList_Clear` functions, + mirroring the Python :meth:`!list.extend` and :meth:`!list.clear` methods. + (Contributed by Victor Stinner in :gh:`111138`.) -* Add :c:func:`PyUnicode_EqualToUTF8AndSize` and :c:func:`PyUnicode_EqualToUTF8` - functions: compare Unicode object with a :c:expr:`const char*` UTF-8 encoded - string and return true (``1``) if they are equal, or false (``0``) otherwise. - These functions do not raise exceptions. - (Contributed by Serhiy Storchaka in :gh:`110289`.) +* Add the :c:func:`PyLong_AsInt` function. + It behaves similarly to :c:func:`PyLong_AsLong`, + but stores the result in a C :c:expr:`int` instead of a C :c:expr:`long`. + (Contributed by Victor Stinner in :gh:`108014`.) -* Add :c:func:`PyThreadState_GetUnchecked()` function: similar to - :c:func:`PyThreadState_Get()`, but don't kill the process with a fatal error - if it is NULL. The caller is responsible to check if the result is NULL. - Previously, the function was private and known as - ``_PyThreadState_UncheckedGet()``. - (Contributed by Victor Stinner in :gh:`108867`.) +* Add the :c:func:`PyLong_AsNativeBytes`, :c:func:`PyLong_FromNativeBytes`, + and :c:func:`PyLong_FromUnsignedNativeBytes` functions + to simplify converting between native integer types + and Python :class:`int` objects. + (Contributed by Steve Dower in :gh:`111140`.) -* Add :c:func:`PySys_AuditTuple` function: similar to :c:func:`PySys_Audit`, - but pass event arguments as a Python :class:`tuple` object. - (Contributed by Victor Stinner in :gh:`85283`.) +* Add :c:func:`PyModule_Add` function, which is similar to + :c:func:`PyModule_AddObjectRef` and :c:func:`PyModule_AddObject`, + but always steals a reference to the value. + (Contributed by Serhiy Storchaka in :gh:`86493`.) -* :c:func:`PyArg_ParseTupleAndKeywords` now supports non-ASCII keyword - parameter names. - (Contributed by Serhiy Storchaka in :gh:`110815`.) +* Add the :c:func:`PyObject_GenericHash` function + that implements the default hashing function of a Python object. + (Contributed by Serhiy Storchaka in :gh:`113024`.) -* Add :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawCalloc`, - :c:func:`PyMem_RawRealloc` and :c:func:`PyMem_RawFree` to the limited C API - (version 3.13). - (Contributed by Victor Stinner in :gh:`85283`.) +* Add the :c:func:`Py_HashPointer` function to hash a raw pointer. + (Contributed by Victor Stinner in :gh:`111545`.) -* Add :c:func:`PySys_Audit` and :c:func:`PySys_AuditTuple` functions to the - limited C API. - (Contributed by Victor Stinner in :gh:`85283`.) +* Add the :c:func:`PyObject_VisitManagedDict` and + :c:func:`PyObject_ClearManagedDict` functions. + which must be called by the traverse and clear functions of a type using + the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag. + The `pythoncapi-compat project`_ can be used to + use these functions with Python 3.11 and 3.12. + (Contributed by Victor Stinner in :gh:`107073`.) -* Add :c:func:`PyErr_FormatUnraisable` function: similar to - :c:func:`PyErr_WriteUnraisable`, but allow customizing the warning message. - (Contributed by Serhiy Storchaka in :gh:`108082`.) +* Add the :c:func:`PyRefTracer_SetTracer` + and :c:func:`PyRefTracer_GetTracer` functions, + which enable tracking object creation and destruction + in the same way that the :mod:`tracemalloc` module does. + (Contributed by Pablo Galindo in :gh:`93502`.) -* Add :c:func:`PyList_Extend` and :c:func:`PyList_Clear` functions: similar to - Python ``list.extend()`` and ``list.clear()`` methods. - (Contributed by Victor Stinner in :gh:`111138`.) +* Add the :c:func:`PySys_AuditTuple` function + as an alternative to :c:func:`PySys_Audit` + that takes event arguments as a Python :class:`tuple` object. + (Contributed by Victor Stinner in :gh:`85283`.) -* Add :c:func:`PyDict_Pop` and :c:func:`PyDict_PopString` functions: remove a - key from a dictionary and optionally return the removed value. This is - similar to :meth:`dict.pop`, but without the default value and not raising - :exc:`KeyError` if the key is missing. - (Contributed by Stefan Behnel and Victor Stinner in :gh:`111262`.) +* Add the :c:func:`PyThreadState_GetUnchecked()` function + as an alternative to :c:func:`PyThreadState_Get()` + that doesn't kill the process with a fatal error if it is ``NULL``. + The caller is responsible for checking if the result is ``NULL``. + (Contributed by Victor Stinner in :gh:`108867`.) -* Add :c:func:`Py_HashPointer` function to hash a pointer. - (Contributed by Victor Stinner in :gh:`111545`.) +* Add the :c:func:`PyType_GetFullyQualifiedName` function + to get the type's fully qualified name. + The module name is prepended if :attr:`type.__module__` is + a string and is not equal to either ``'builtins'`` or ``'__main__'``. + (Contributed by Victor Stinner in :gh:`111696`.) + +* Add the :c:func:`PyType_GetModuleName` function + to get the type's module name. This is equivalent to getting the + :attr:`type.__module__` attribute. + (Contributed by Eric Snow and Victor Stinner in :gh:`111696`.) + +* Add the :c:func:`PyUnicode_EqualToUTF8AndSize` + and :c:func:`PyUnicode_EqualToUTF8` functions + to compare a Unicode object with a :c:expr:`const char*` UTF-8 encoded string + and ``1`` if they are equal or ``0`` otherwise. + These functions do not raise exceptions. + (Contributed by Serhiy Storchaka in :gh:`110289`.) -* Add PyTime C API: +* Add the :c:func:`PyWeakref_GetRef` function + as an alternative to :c:func:`PyWeakref_GetObject` + that returns a :term:`strong reference` + or ``NULL`` if the referent is no longer live. + (Contributed by Victor Stinner in :gh:`105927`.) - * :c:type:`PyTime_t` type. - * :c:var:`PyTime_MIN` and :c:var:`PyTime_MAX` constants. - * :c:func:`PyTime_AsSecondsDouble` - :c:func:`PyTime_Monotonic`, :c:func:`PyTime_PerfCounter`, and - :c:func:`PyTime_Time` functions. +* Add fixed variants of functions which silently ignore errors: - (Contributed by Victor Stinner and Petr Viktorin in :gh:`110850`.) + * :c:func:`PyObject_HasAttrWithError` replaces :c:func:`PyObject_HasAttr`. + * :c:func:`PyObject_HasAttrStringWithError` + replaces :c:func:`PyObject_HasAttrString`. + * :c:func:`PyMapping_HasKeyWithError` replaces :c:func:`PyMapping_HasKey`. + * :c:func:`PyMapping_HasKeyStringWithError` + replaces :c:func:`PyMapping_HasKeyString`. -* Add :c:func:`PyLong_AsNativeBytes`, :c:func:`PyLong_FromNativeBytes` and - :c:func:`PyLong_FromUnsignedNativeBytes` functions to simplify converting - between native integer types and Python :class:`int` objects. - (Contributed by Steve Dower in :gh:`111140`.) + The new functions return ``-1`` for errors + and the standard ``1`` for true and ``0`` for false. + (Contributed by Serhiy Storchaka in :gh:`108511`.) -Porting to Python 3.13 ----------------------- -* ``Python.h`` no longer includes the ```` standard header. It was - included for the ``finite()`` function which is now provided by the - ```` header. It should now be included explicitly if needed. Remove - also the ``HAVE_IEEEFP_H`` macro. - (Contributed by Victor Stinner in :gh:`108765`.) +Changed C APIs +-------------- -* ``Python.h`` no longer includes these standard header files: ````, - ```` and ````. If needed, they should now be - included explicitly. For example, ```` provides the ``clock()`` and - ``gmtime()`` functions, ```` provides the ``select()`` - function, and ```` provides the ``futimes()``, ``gettimeofday()`` - and ``setitimer()`` functions. - (Contributed by Victor Stinner in :gh:`108765`.) +* The *keywords* parameter of :c:func:`PyArg_ParseTupleAndKeywords` + and :c:func:`PyArg_VaParseTupleAndKeywords` + now has type :c:expr:`char * const *` in C + and :c:expr:`const char * const *` in C++, + instead of :c:expr:`char **`. + In C++, this makes these functions compatible with arguments + of type :c:expr:`const char * const *`, :c:expr:`const char **`, + or :c:expr:`char * const *` without an explicit type cast. + In C, the functions only support arguments of type :c:expr:`char * const *`. + This can be overridden with the :c:macro:`PY_CXX_CONST` macro. + (Contributed by Serhiy Storchaka in :gh:`65210`.) -* If the :c:macro:`Py_LIMITED_API` macro is defined, :c:macro:`!Py_BUILD_CORE`, - :c:macro:`!Py_BUILD_CORE_BUILTIN` and :c:macro:`!Py_BUILD_CORE_MODULE` macros - are now undefined by ````. - (Contributed by Victor Stinner in :gh:`85283`.) +* :c:func:`PyArg_ParseTupleAndKeywords` now supports + non-ASCII keyword parameter names. + (Contributed by Serhiy Storchaka in :gh:`110815`.) -* The old trashcan macros ``Py_TRASHCAN_SAFE_BEGIN`` and ``Py_TRASHCAN_SAFE_END`` - were removed. They should be replaced by the new macros ``Py_TRASHCAN_BEGIN`` - and ``Py_TRASHCAN_END``. +* The :c:func:`!PyCode_GetFirstFree` function is now unstable API + and is now named :c:func:`PyUnstable_Code_GetFirstFree`. + (Contributed by Bogdan Romanyuk in :gh:`115781`.) - A tp_dealloc function that has the old macros, such as:: +* The :c:func:`PyDict_GetItem`, :c:func:`PyDict_GetItemString`, + :c:func:`PyMapping_HasKey`, :c:func:`PyMapping_HasKeyString`, + :c:func:`PyObject_HasAttr`, :c:func:`PyObject_HasAttrString`, + and :c:func:`PySys_GetObject` functions, + each of which clears all errors which occurred when calling them + now reports these errors using :func:`sys.unraisablehook`. + You may replace them with other functions as recommended in the documentation. + (Contributed by Serhiy Storchaka in :gh:`106672`.) - static void - mytype_dealloc(mytype *p) - { - PyObject_GC_UnTrack(p); - Py_TRASHCAN_SAFE_BEGIN(p); - ... - Py_TRASHCAN_SAFE_END - } +* Add support for the ``%T``, ``%#T``, ``%N`` and ``%#N`` formats + to :c:func:`PyUnicode_FromFormat`: - should migrate to the new macros as follows:: + * ``%T``: Get the fully qualified name of an object type + * ``%#T``: As above, but use a colon as the separator + * ``%N``: Get the fully qualified name of a type + * ``%#N``: As above, but use a colon as the separator - static void - mytype_dealloc(mytype *p) - { - PyObject_GC_UnTrack(p); - Py_TRASHCAN_BEGIN(p, mytype_dealloc) - ... - Py_TRASHCAN_END - } + See :pep:`737` for more information. + (Contributed by Victor Stinner in :gh:`111696`.) - Note that ``Py_TRASHCAN_BEGIN`` has a second argument which - should be the deallocation function it is in. +* You no longer have to define the ``PY_SSIZE_T_CLEAN`` macro before + including :file:`Python.h` when using ``#`` formats in + :ref:`format codes `. + APIs accepting the format codes always use ``Py_ssize_t`` for ``#`` formats. + (Contributed by Inada Naoki in :gh:`104922`.) -* On Windows, ``Python.h`` no longer includes the ```` standard - header file. If needed, it should now be included explicitly. For example, it - provides ``offsetof()`` function, and ``size_t`` and ``ptrdiff_t`` types. - Including ```` explicitly was already needed by all other - platforms, the ``HAVE_STDDEF_H`` macro is only defined on Windows. - (Contributed by Victor Stinner in :gh:`108765`.) +* If Python is built in :ref:`debug mode ` + or :option:`with assertions <--with-assertions>`, + :c:func:`PyTuple_SET_ITEM` and :c:func:`PyList_SET_ITEM` + now check the index argument with an assertion. + (Contributed by Victor Stinner in :gh:`106168`.) -Deprecated ----------- -* Passing optional arguments *maxsplit*, *count* and *flags* in module-level - functions :func:`re.split`, :func:`re.sub` and :func:`re.subn` as positional - arguments is now deprecated. - In future Python versions these parameters will be - :ref:`keyword-only `. - (Contributed by Serhiy Storchaka in :gh:`56166`.) +Limited C API Changes +--------------------- -* Deprecate the old ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` types: use directly - the :c:type:`wchar_t` type instead. Since Python 3.3, ``Py_UNICODE`` and - ``PY_UNICODE_TYPE`` are just aliases to :c:type:`wchar_t`. - (Contributed by Victor Stinner in :gh:`105156`.) +* The following functions are now included in the Limited C API: -* Deprecate old Python initialization functions: + * :c:func:`PyMem_RawMalloc` + * :c:func:`PyMem_RawCalloc` + * :c:func:`PyMem_RawRealloc` + * :c:func:`PyMem_RawFree` + * :c:func:`PySys_Audit` + * :c:func:`PySys_AuditTuple` + * :c:func:`PyType_GetModuleByDef` - * :c:func:`PySys_ResetWarnOptions`: - clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. - * :c:func:`Py_GetExecPrefix`: get :data:`sys.exec_prefix` instead. - * :c:func:`Py_GetPath`: get :data:`sys.path` instead. - * :c:func:`Py_GetPrefix`: get :data:`sys.prefix` instead. - * :c:func:`Py_GetProgramFullPath`: get :data:`sys.executable` instead. - * :c:func:`Py_GetProgramName`: get :data:`sys.executable` instead. - * :c:func:`Py_GetPythonHome`: get :c:member:`PyConfig.home` or - :envvar:`PYTHONHOME` environment variable instead. - - Functions scheduled for removal in Python 3.15. - (Contributed by Victor Stinner in :gh:`105145`.) + (Contributed by Victor Stinner in :gh:`85283`, :gh:`85283`, and :gh:`116936`.) -* Deprecate the :c:func:`PyImport_ImportModuleNoBlock` function which is just - an alias to :c:func:`PyImport_ImportModule` since Python 3.3. - Scheduled for removal in Python 3.15. - (Contributed by Victor Stinner in :gh:`105396`.) +* Python built with :option:`--with-trace-refs` (tracing references) + now supports the :ref:`Limited API `. + (Contributed by Victor Stinner in :gh:`108634`.) -* Deprecate the :c:func:`PyWeakref_GetObject` and - :c:func:`PyWeakref_GET_OBJECT` functions, which return a :term:`borrowed - reference`: use the new :c:func:`PyWeakref_GetRef` function instead, it - returns a :term:`strong reference`. The `pythoncapi-compat project - `__ can be used to get - :c:func:`PyWeakref_GetRef` on Python 3.12 and older. - (Contributed by Victor Stinner in :gh:`105927`.) -Removed -------- +Removed C APIs +-------------- -* Removed chained :class:`classmethod` descriptors (introduced in - :gh:`63272`). This can no longer be used to wrap other descriptors - such as :class:`property`. The core design of this feature was flawed - and caused a number of downstream problems. To "pass-through" a - :class:`classmethod`, consider using the :attr:`!__wrapped__` - attribute that was added in Python 3.10. (Contributed by Raymond - Hettinger in :gh:`89519`.) - -* Remove many APIs (functions, macros, variables) with names prefixed by - ``_Py`` or ``_PY`` (considered as private API). If your project is affected - by one of these removals and you consider that the removed API should remain - available, please open a new issue to request a public C API and - add ``cc @vstinner`` to the issue to notify Victor Stinner. +* Remove several functions, macros, variables, etc + with names prefixed by ``_Py`` or ``_PY`` (which are considered private). + If your project is affected by one of these removals + and you believe that the removed API should remain available, + please :ref:`open a new issue ` to request a public C API + and add ``cc: @vstinner`` to the issue to notify Victor Stinner. (Contributed by Victor Stinner in :gh:`106320`.) -* Remove functions deprecated in Python 3.9: - - * ``PyEval_CallObject()``, ``PyEval_CallObjectWithKeywords()``: use - :c:func:`PyObject_CallNoArgs` or :c:func:`PyObject_Call` instead. - Warning: :c:func:`PyObject_Call` positional arguments must be a - :class:`tuple` and must not be ``NULL``, keyword arguments must be a - :class:`dict` or ``NULL``, whereas removed functions checked arguments type - and accepted ``NULL`` positional and keyword arguments. - To replace ``PyEval_CallObjectWithKeywords(func, NULL, kwargs)`` with - :c:func:`PyObject_Call`, pass an empty tuple as positional arguments using - :c:func:`PyTuple_New(0) `. - * ``PyEval_CallFunction()``: use :c:func:`PyObject_CallFunction` instead. - * ``PyEval_CallMethod()``: use :c:func:`PyObject_CallMethod` instead. - * ``PyCFunction_Call()``: use :c:func:`PyObject_Call` instead. - - (Contributed by Victor Stinner in :gh:`105107`.) +* Remove old buffer protocols deprecated in Python 3.0. + Use :ref:`bufferobjects` instead. -* Remove old buffer protocols deprecated in Python 3.0. Use :ref:`bufferobjects` instead. - - * :c:func:`!PyObject_CheckReadBuffer`: Use :c:func:`PyObject_CheckBuffer` to - test if the object supports the buffer protocol. - Note that :c:func:`PyObject_CheckBuffer` doesn't guarantee that - :c:func:`PyObject_GetBuffer` will succeed. - To test if the object is actually readable, see the next example - of :c:func:`PyObject_GetBuffer`. + * :c:func:`!PyObject_CheckReadBuffer`: + Use :c:func:`PyObject_CheckBuffer` to test + whether the object supports the buffer protocol. + Note that :c:func:`PyObject_CheckBuffer` doesn't guarantee + that :c:func:`PyObject_GetBuffer` will succeed. + To test if the object is actually readable, + see the next example of :c:func:`PyObject_GetBuffer`. * :c:func:`!PyObject_AsCharBuffer`, :c:func:`!PyObject_AsReadBuffer`: - :c:func:`PyObject_GetBuffer` and :c:func:`PyBuffer_Release` instead: + Use :c:func:`PyObject_GetBuffer` and :c:func:`PyBuffer_Release` instead: .. code-block:: c @@ -1775,8 +2360,8 @@ Removed // You may need to cast buf as `(const char*)view.buf`. PyBuffer_Release(&view); - * :c:func:`!PyObject_AsWriteBuffer`: Use - :c:func:`PyObject_GetBuffer` and :c:func:`PyBuffer_Release` instead: + * :c:func:`!PyObject_AsWriteBuffer`: + Use :c:func:`PyObject_GetBuffer` and :c:func:`PyBuffer_Release` instead: .. code-block:: c @@ -1789,42 +2374,60 @@ Removed (Contributed by Inada Naoki in :gh:`85275`.) +* Remove various functions deprecated in Python 3.9: + + * :c:func:`!PyEval_CallObject`, :c:func:`!PyEval_CallObjectWithKeywords`: + Use :c:func:`PyObject_CallNoArgs` or :c:func:`PyObject_Call` instead. + + .. warning:: + + In :c:func:`PyObject_Call`, positional arguments must be a :class:`tuple` + and must not be ``NULL``, + and keyword arguments must be a :class:`dict` or ``NULL``, + whereas the removed functions checked argument types + and accepted ``NULL`` positional and keyword arguments. + To replace ``PyEval_CallObjectWithKeywords(func, NULL, kwargs)`` with + :c:func:`PyObject_Call`, + pass an empty tuple as positional arguments using + :c:func:`PyTuple_New(0) `. + + * :c:func:`!PyEval_CallFunction`: + Use :c:func:`PyObject_CallFunction` instead. + * :c:func:`!PyEval_CallMethod`: + Use :c:func:`PyObject_CallMethod` instead. + * :c:func:`!PyCFunction_Call`: + Use :c:func:`PyObject_Call` instead. + + (Contributed by Victor Stinner in :gh:`105107`.) + * Remove the following old functions to configure the Python initialization, deprecated in Python 3.11: - * ``PySys_AddWarnOptionUnicode()``: use :c:member:`PyConfig.warnoptions` instead. - * ``PySys_AddWarnOption()``: use :c:member:`PyConfig.warnoptions` instead. - * ``PySys_AddXOption()``: use :c:member:`PyConfig.xoptions` instead. - * ``PySys_HasWarnOptions()``: use :c:member:`PyConfig.xoptions` instead. - * ``PySys_SetArgvEx()``: set :c:member:`PyConfig.argv` instead. - * ``PySys_SetArgv()``: set :c:member:`PyConfig.argv` instead. - * ``PySys_SetPath()``: set :c:member:`PyConfig.module_search_paths` instead. - * ``Py_SetPath()``: set :c:member:`PyConfig.module_search_paths` instead. - * ``Py_SetProgramName()``: set :c:member:`PyConfig.program_name` instead. - * ``Py_SetPythonHome()``: set :c:member:`PyConfig.home` instead. - * ``Py_SetStandardStreamEncoding()``: set :c:member:`PyConfig.stdio_encoding` - instead, and set also maybe :c:member:`PyConfig.legacy_windows_stdio` (on - Windows). - * ``_Py_SetProgramFullPath()``: set :c:member:`PyConfig.executable` instead. + * :c:func:`!PySys_AddWarnOptionUnicode`: + Use :c:member:`PyConfig.warnoptions` instead. + * :c:func:`!PySys_AddWarnOption`: + Use :c:member:`PyConfig.warnoptions` instead. + * :c:func:`!PySys_AddXOption`: + Use :c:member:`PyConfig.xoptions` instead. + * :c:func:`!PySys_HasWarnOptions`: + Use :c:member:`PyConfig.xoptions` instead. + * :c:func:`!PySys_SetPath`: + Set :c:member:`PyConfig.module_search_paths` instead. + * :c:func:`!Py_SetPath`: + Set :c:member:`PyConfig.module_search_paths` instead. + * :c:func:`!Py_SetStandardStreamEncoding`: + Set :c:member:`PyConfig.stdio_encoding` instead, + and set also maybe :c:member:`PyConfig.legacy_windows_stdio` (on Windows). + * :c:func:`!_Py_SetProgramFullPath`: + Set :c:member:`PyConfig.executable` instead. Use the new :c:type:`PyConfig` API of the :ref:`Python Initialization Configuration ` instead (:pep:`587`), added to Python 3.8. (Contributed by Victor Stinner in :gh:`105145`.) -* Remove the old trashcan macros ``Py_TRASHCAN_SAFE_BEGIN`` and - ``Py_TRASHCAN_SAFE_END``. They should be replaced by the new macros - ``Py_TRASHCAN_BEGIN`` and ``Py_TRASHCAN_END``. The new macros were - added in Python 3.8 and the old macros were deprecated in Python 3.11. - (Contributed by Irit Katriel in :gh:`105111`.) - -* Remove ``PyEval_InitThreads()`` and ``PyEval_ThreadsInitialized()`` - functions, deprecated in Python 3.9. Since Python 3.7, ``Py_Initialize()`` - always creates the GIL: calling ``PyEval_InitThreads()`` did nothing and - ``PyEval_ThreadsInitialized()`` always returned non-zero. - (Contributed by Victor Stinner in :gh:`105182`.) - -* Remove ``PyEval_AcquireLock()`` and ``PyEval_ReleaseLock()`` functions, - deprecated in Python 3.2. They didn't update the current thread state. +* Remove :c:func:`!PyEval_AcquireLock` and :c:func:`!PyEval_ReleaseLock` functions, + deprecated in Python 3.2. + They didn't update the current thread state. They can be replaced with: * :c:func:`PyEval_SaveThread` and :c:func:`PyEval_RestoreThread`; @@ -1833,113 +2436,354 @@ Removed (Contributed by Victor Stinner in :gh:`105182`.) -* Remove private ``_PyObject_FastCall()`` function: - use ``PyObject_Vectorcall()`` which is available since Python 3.8 +* Remove the :c:func:`!PyEval_ThreadsInitialized` function, + deprecated in Python 3.9. + Since Python 3.7, :c:func:`!Py_Initialize` always creates the GIL: + calling :c:func:`!PyEval_InitThreads` does nothing and + :c:func:`!PyEval_ThreadsInitialized` always returns non-zero. + (Contributed by Victor Stinner in :gh:`105182`.) + +* Remove the :c:func:`!_PyInterpreterState_Get` alias to + :c:func:`PyInterpreterState_Get()` + which was kept for backward compatibility with Python 3.8. + The `pythoncapi-compat project`_ can be used to get + :c:func:`PyInterpreterState_Get()` on Python 3.8 and older. + (Contributed by Victor Stinner in :gh:`106320`.) + +* Remove the private :c:func:`!_PyObject_FastCall` function: + use :c:func:`!PyObject_Vectorcall` which is available since Python 3.8 (:pep:`590`). (Contributed by Victor Stinner in :gh:`106023`.) -* Remove ``cpython/pytime.h`` header file: it only contained private functions. +* Remove the ``cpython/pytime.h`` header file, + which only contained private functions. (Contributed by Victor Stinner in :gh:`106316`.) -* Remove ``_PyInterpreterState_Get()`` alias to - :c:func:`PyInterpreterState_Get()` which was kept for backward compatibility - with Python 3.8. The `pythoncapi-compat project - `__ can be used to get - :c:func:`PyInterpreterState_Get()` on Python 3.8 and older. - (Contributed by Victor Stinner in :gh:`106320`.) +* Remove the undocumented ``PY_TIMEOUT_MAX`` constant from the limited C API. + (Contributed by Victor Stinner in :gh:`110014`.) + +* Remove the old trashcan macros ``Py_TRASHCAN_SAFE_BEGIN`` + and ``Py_TRASHCAN_SAFE_END``. + Replace both with the new macros ``Py_TRASHCAN_BEGIN`` + and ``Py_TRASHCAN_END``. + (Contributed by Irit Katriel in :gh:`105111`.) + +Deprecated C APIs +----------------- + +* Deprecate old Python initialization functions: -* The :c:func:`PyModule_AddObject` function is now :term:`soft deprecated`: - :c:func:`PyModule_Add` or :c:func:`PyModule_AddObjectRef` functions should - be used instead. + * :c:func:`PySys_ResetWarnOptions`: + Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. + * :c:func:`Py_GetExecPrefix`: + Get :data:`sys.exec_prefix` instead. + * :c:func:`Py_GetPath`: + Get :data:`sys.path` instead. + * :c:func:`Py_GetPrefix`: + Get :data:`sys.prefix` instead. + * :c:func:`Py_GetProgramFullPath`: + Get :data:`sys.executable` instead. + * :c:func:`Py_GetProgramName`: + Get :data:`sys.executable` instead. + * :c:func:`Py_GetPythonHome`: + Get :c:member:`PyConfig.home` + or the :envvar:`PYTHONHOME` environment variable instead. + + (Contributed by Victor Stinner in :gh:`105145`.) + +* :term:`Soft deprecate ` the + :c:func:`PyEval_GetBuiltins`, :c:func:`PyEval_GetGlobals`, + and :c:func:`PyEval_GetLocals` functions, + which return a :term:`borrowed reference`. + (Soft deprecated as part of :pep:`667`.) + +* Deprecate the :c:func:`PyImport_ImportModuleNoBlock` function, + which is just an alias to :c:func:`PyImport_ImportModule` since Python 3.3. + (Contributed by Victor Stinner in :gh:`105396`.) + +* :term:`Soft deprecate ` the + :c:func:`PyModule_AddObject` function. + It should be replaced with :c:func:`PyModule_Add` + or :c:func:`PyModule_AddObjectRef`. (Contributed by Serhiy Storchaka in :gh:`86493`.) -Pending Removal in Python 3.14 ------------------------------- - -* Creating immutable types (:c:macro:`Py_TPFLAGS_IMMUTABLETYPE`) with mutable - bases using the C API. -* Global configuration variables: - - * :c:var:`Py_DebugFlag`: use :c:member:`PyConfig.parser_debug` - * :c:var:`Py_VerboseFlag`: use :c:member:`PyConfig.verbose` - * :c:var:`Py_QuietFlag`: use :c:member:`PyConfig.quiet` - * :c:var:`Py_InteractiveFlag`: use :c:member:`PyConfig.interactive` - * :c:var:`Py_InspectFlag`: use :c:member:`PyConfig.inspect` - * :c:var:`Py_OptimizeFlag`: use :c:member:`PyConfig.optimization_level` - * :c:var:`Py_NoSiteFlag`: use :c:member:`PyConfig.site_import` - * :c:var:`Py_BytesWarningFlag`: use :c:member:`PyConfig.bytes_warning` - * :c:var:`Py_FrozenFlag`: use :c:member:`PyConfig.pathconfig_warnings` - * :c:var:`Py_IgnoreEnvironmentFlag`: use :c:member:`PyConfig.use_environment` - * :c:var:`Py_DontWriteBytecodeFlag`: use :c:member:`PyConfig.write_bytecode` - * :c:var:`Py_NoUserSiteDirectory`: use :c:member:`PyConfig.user_site_directory` - * :c:var:`Py_UnbufferedStdioFlag`: use :c:member:`PyConfig.buffered_stdio` - * :c:var:`Py_HashRandomizationFlag`: use :c:member:`PyConfig.use_hash_seed` - and :c:member:`PyConfig.hash_seed` - * :c:var:`Py_IsolatedFlag`: use :c:member:`PyConfig.isolated` - * :c:var:`Py_LegacyWindowsFSEncodingFlag`: use :c:member:`PyPreConfig.legacy_windows_fs_encoding` - * :c:var:`Py_LegacyWindowsStdioFlag`: use :c:member:`PyConfig.legacy_windows_stdio` - * :c:var:`!Py_FileSystemDefaultEncoding`: use :c:member:`PyConfig.filesystem_encoding` - * :c:var:`!Py_HasFileSystemDefaultEncoding`: use :c:member:`PyConfig.filesystem_encoding` - * :c:var:`!Py_FileSystemDefaultEncodeErrors`: use :c:member:`PyConfig.filesystem_errors` - * :c:var:`!Py_UTF8Mode`: use :c:member:`PyPreConfig.utf8_mode` (see :c:func:`Py_PreInitialize`) - - The :c:func:`Py_InitializeFromConfig` API should be used with - :c:type:`PyConfig` instead. - -Pending Removal in Python 3.15 ------------------------------- - -* :c:func:`PyImport_ImportModuleNoBlock`: use :c:func:`PyImport_ImportModule`. -* :c:func:`PyWeakref_GET_OBJECT`: use :c:func:`PyWeakref_GetRef` instead. -* :c:func:`PyWeakref_GetObject`: use :c:func:`PyWeakref_GetRef` instead. -* :c:type:`!Py_UNICODE_WIDE` type: use :c:type:`wchar_t` instead. -* :c:type:`Py_UNICODE` type: use :c:type:`wchar_t` instead. -* Python initialization functions: - - * :c:func:`PySys_ResetWarnOptions`: clear :data:`sys.warnoptions` and - :data:`!warnings.filters` instead. - * :c:func:`Py_GetExecPrefix`: get :data:`sys.exec_prefix` instead. - * :c:func:`Py_GetPath`: get :data:`sys.path` instead. - * :c:func:`Py_GetPrefix`: get :data:`sys.prefix` instead. - * :c:func:`Py_GetProgramFullPath`: get :data:`sys.executable` instead. - * :c:func:`Py_GetProgramName`: get :data:`sys.executable` instead. - * :c:func:`Py_GetPythonHome`: get :c:member:`PyConfig.home` or - :envvar:`PYTHONHOME` environment variable instead. - -Pending Removal in Future Versions ----------------------------------- - -The following APIs were deprecated in earlier Python versions and will be -removed, although there is currently no date scheduled for their removal. - -* :c:macro:`Py_TPFLAGS_HAVE_FINALIZE`: no needed since Python 3.8. -* :c:func:`PyErr_Fetch`: use :c:func:`PyErr_GetRaisedException`. -* :c:func:`PyErr_NormalizeException`: use :c:func:`PyErr_GetRaisedException`. -* :c:func:`PyErr_Restore`: use :c:func:`PyErr_SetRaisedException`. -* :c:func:`PyModule_GetFilename`: use :c:func:`PyModule_GetFilenameObject`. -* :c:func:`PyOS_AfterFork`: use :c:func:`PyOS_AfterFork_Child()`. -* :c:func:`PySlice_GetIndicesEx`. -* :c:func:`!PyUnicode_AsDecodedObject`. -* :c:func:`!PyUnicode_AsDecodedUnicode`. -* :c:func:`!PyUnicode_AsEncodedObject`. -* :c:func:`!PyUnicode_AsEncodedUnicode`. -* :c:func:`PyUnicode_READY`: not needed since Python 3.12. -* :c:func:`!_PyErr_ChainExceptions`. -* :c:member:`!PyBytesObject.ob_shash` member: - call :c:func:`PyObject_Hash` instead. -* :c:member:`!PyDictObject.ma_version_tag` member. -* TLS API: - - * :c:func:`PyThread_create_key`: use :c:func:`PyThread_tss_alloc`. - * :c:func:`PyThread_delete_key`: use :c:func:`PyThread_tss_free`. - * :c:func:`PyThread_set_key_value`: use :c:func:`PyThread_tss_set`. - * :c:func:`PyThread_get_key_value`: use :c:func:`PyThread_tss_get`. - * :c:func:`PyThread_delete_key_value`: use :c:func:`PyThread_tss_delete`. - * :c:func:`PyThread_ReInitTLS`: no longer needed. - -* Remove undocumented ``PY_TIMEOUT_MAX`` constant from the limited C API. - (Contributed by Victor Stinner in :gh:`110014`.) +* Deprecate the old ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` types + and the :c:macro:`!Py_UNICODE_WIDE` define. + Use the :c:type:`wchar_t` type directly instead. + Since Python 3.3, ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` + are just aliases to :c:type:`!wchar_t`. + (Contributed by Victor Stinner in :gh:`105156`.) + +* Deprecate the :c:func:`PyWeakref_GetObject` and + :c:func:`PyWeakref_GET_OBJECT` functions, + which return a :term:`borrowed reference`. + Replace them with the new :c:func:`PyWeakref_GetRef` function, + which returns a :term:`strong reference`. + The `pythoncapi-compat project`_ can be used to get + :c:func:`PyWeakref_GetRef` on Python 3.12 and older. + (Contributed by Victor Stinner in :gh:`105927`.) + +.. Add deprecations above alphabetically, not here at the end. + +.. include:: ../deprecations/c-api-pending-removal-in-3.14.rst + +.. include:: ../deprecations/c-api-pending-removal-in-3.15.rst + +.. include:: ../deprecations/c-api-pending-removal-in-future.rst + +.. _pythoncapi-compat project: https://github.com/python/pythoncapi-compat/ + +Build Changes +============= + +* ``arm64-apple-ios`` and ``arm64-apple-ios-simulator`` are both + now :pep:`11` tier 3 platforms. + (:ref:`PEP 730 ` written + and implementation contributed by Russell Keith-Magee in :gh:`114099`.) + +* ``aarch64-linux-android`` and ``x86_64-linux-android`` are both + now :pep:`11` tier 3 platforms. + (:ref:`PEP 738 ` written + and implementation contributed by Malcolm Smith in :gh:`116622`.) + +* ``wasm32-wasi`` is now a :pep:`11` tier 2 platform. + (Contributed by Brett Cannon in :gh:`115192`.) + +* ``wasm32-emscripten`` is no longer a :pep:`11` supported platform. + (Contributed by Brett Cannon in :gh:`115192`.) + +* Building CPython now requires a compiler with support for the C11 atomic + library, GCC built-in atomic functions, or MSVC interlocked intrinsics. + +* Autoconf 2.71 and aclocal 1.16.5 are now required to regenerate + the :file:`configure` script. + (Contributed by Christian Heimes in :gh:`89886` and by Victor Stinner in :gh:`112090`.) + +* SQLite 3.15.2 or newer is required to build + the :mod:`sqlite3` extension module. + (Contributed by Erlend Aasland in :gh:`105875`.) + +* CPython now bundles the `mimalloc library`_ by default. + It is licensed under the MIT license; + see :ref:`mimalloc license `. + The bundled mimalloc has custom changes, see :gh:`113141` for details. + (Contributed by Dino Viehland in :gh:`109914`.) + + .. _mimalloc library: https://github.com/microsoft/mimalloc/ + +* The :file:`configure` option :option:`--with-system-libmpdec` + now defaults to ``yes``. + The bundled copy of ``libmpdecimal`` will be removed in Python 3.15. + +* Python built with :file:`configure` :option:`--with-trace-refs` + (tracing references) is now ABI compatible with the Python release build + and :ref:`debug build `. + (Contributed by Victor Stinner in :gh:`108634`.) + +* On POSIX systems, the pkg-config (``.pc``) filenames now include the ABI + flags. For example, the free-threaded build generates ``python-3.13t.pc`` + and the debug build generates ``python-3.13d.pc``. + +* The ``errno``, ``fcntl``, ``grp``, ``md5``, ``pwd``, ``resource``, + ``termios``, ``winsound``, + ``_ctypes_test``, ``_multiprocessing.posixshmem``, ``_scproxy``, ``_stat``, + ``_statistics``, ``_testconsole``, ``_testimportmultiple`` and ``_uuid`` + C extensions are now built with the :ref:`limited C API `. + (Contributed by Victor Stinner in :gh:`85283`.) + + +Porting to Python 3.13 +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + +Changes in the Python API +------------------------- + +.. _pep667-porting-notes-py: + +* :ref:`PEP 667 ` introduces several changes + to the semantics of :func:`locals` and :attr:`f_locals `: + + * Calling :func:`locals` in an :term:`optimized scope` now produces an + independent snapshot on each call, and hence no longer implicitly updates + previously returned references. Obtaining the legacy CPython behavior now + requires explicit calls to update the initially returned dictionary with the + results of subsequent calls to :func:`!locals`. Code execution functions that + implicitly target :func:`!locals` (such as ``exec`` and ``eval``) must be + passed an explicit namespace to access their results in an optimized scope. + (Changed as part of :pep:`667`.) + + * Calling :func:`locals` from a comprehension at module or class scope + (including via ``exec`` or ``eval``) once more behaves as if the comprehension + were running as an independent nested function (i.e. the local variables from + the containing scope are not included). In Python 3.12, this had changed + to include the local variables from the containing scope when implementing + :pep:`709`. (Changed as part of :pep:`667`.) + + * Accessing :attr:`FrameType.f_locals ` in an + :term:`optimized scope` now returns a write-through proxy rather than a + snapshot that gets updated at ill-specified times. If a snapshot is desired, + it must be created explicitly with ``dict`` or the proxy's ``.copy()`` method. + (Changed as part of :pep:`667`.) + +* :class:`functools.partial` now emits a :exc:`FutureWarning` + when used as a method. + The behavior will change in future Python versions. + Wrap it in :func:`staticmethod` if you want to preserve the old behavior. + (Contributed by Serhiy Storchaka in :gh:`121027`.) + +* An :exc:`OSError` is now raised by :func:`getpass.getuser` + for any failure to retrieve a username, + instead of :exc:`ImportError` on non-Unix platforms + or :exc:`KeyError` on Unix platforms where the password database is empty. + +* The value of the :attr:`!mode` attribute of :class:`gzip.GzipFile` + is now a string (``'rb'`` or ``'wb'``) instead of an integer (``1`` or ``2``). + The value of the :attr:`!mode` attribute of the readable file-like object + returned by :meth:`zipfile.ZipFile.open` is now ``'rb'`` instead of ``'r'``. + (Contributed by Serhiy Storchaka in :gh:`115961`.) + +* :class:`mailbox.Maildir` now ignores files with a leading dot (``.``). + (Contributed by Zackery Spytz in :gh:`65559`.) + +* :meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob` now return both + files and directories if a pattern that ends with "``**``" is given, + rather than directories only. + Add a trailing slash to keep the previous behavior and only match directories. + +* The :mod:`threading` module now expects the :mod:`!_thread` module + to have an :func:`!_is_main_interpreter` function. + This function takes no arguments and returns ``True`` + if the current interpreter is the main interpreter. + + Any library or application that provides a custom :mod:`!_thread` module + must provide :func:`!_is_main_interpreter`, + just like the module's other "private" attributes. + (:gh:`112826`.) + + +Changes in the C API +-------------------- + +* ``Python.h`` no longer includes the ```` standard header. It was + included for the :c:func:`!finite` function which is now provided by the + ```` header. It should now be included explicitly if needed. Remove + also the ``HAVE_IEEEFP_H`` macro. + (Contributed by Victor Stinner in :gh:`108765`.) + +* ``Python.h`` no longer includes these standard header files: ````, + ```` and ````. If needed, they should now be + included explicitly. For example, ```` provides the :c:func:`!clock` and + :c:func:`!gmtime` functions, ```` provides the :c:func:`!select` + function, and ```` provides the :c:func:`!futimes`, :c:func:`!gettimeofday` + and :c:func:`!setitimer` functions. + (Contributed by Victor Stinner in :gh:`108765`.) + +* On Windows, ``Python.h`` no longer includes the ```` standard + header file. If needed, it should now be included explicitly. For example, it + provides :c:func:`!offsetof` function, and ``size_t`` and ``ptrdiff_t`` types. + Including ```` explicitly was already needed by all other + platforms, the ``HAVE_STDDEF_H`` macro is only defined on Windows. + (Contributed by Victor Stinner in :gh:`108765`.) + +* If the :c:macro:`Py_LIMITED_API` macro is defined, :c:macro:`!Py_BUILD_CORE`, + :c:macro:`!Py_BUILD_CORE_BUILTIN` and :c:macro:`!Py_BUILD_CORE_MODULE` macros + are now undefined by ````. + (Contributed by Victor Stinner in :gh:`85283`.) + +* The old trashcan macros ``Py_TRASHCAN_SAFE_BEGIN`` and ``Py_TRASHCAN_SAFE_END`` + were removed. They should be replaced by the new macros ``Py_TRASHCAN_BEGIN`` + and ``Py_TRASHCAN_END``. + + A ``tp_dealloc`` function that has the old macros, such as:: + + static void + mytype_dealloc(mytype *p) + { + PyObject_GC_UnTrack(p); + Py_TRASHCAN_SAFE_BEGIN(p); + ... + Py_TRASHCAN_SAFE_END + } + + should migrate to the new macros as follows:: + + static void + mytype_dealloc(mytype *p) + { + PyObject_GC_UnTrack(p); + Py_TRASHCAN_BEGIN(p, mytype_dealloc) + ... + Py_TRASHCAN_END + } + + Note that ``Py_TRASHCAN_BEGIN`` has a second argument which + should be the deallocation function it is in. The new macros were + added in Python 3.8 and the old macros were deprecated in Python 3.11. + (Contributed by Irit Katriel in :gh:`105111`.) +.. _pep667-porting-notes-c: + +* :ref:`PEP 667 ` introduces several changes + to frame-related functions: + + * The effects of mutating the dictionary returned from + :c:func:`PyEval_GetLocals` in an :term:`optimized scope` have changed. + New dict entries added this way will now *only* be visible to + subsequent :c:func:`PyEval_GetLocals` calls in that frame, + as :c:func:`PyFrame_GetLocals`, :func:`locals`, + and :attr:`FrameType.f_locals ` no longer access + the same underlying cached dictionary. + Changes made to entries for actual variable names and names added via + the write-through proxy interfaces will be overwritten on subsequent calls + to :c:func:`PyEval_GetLocals` in that frame. + The recommended code update depends on how the function was being used, + so refer to the deprecation notice on the function for details. + + * Calling :c:func:`PyFrame_GetLocals` in an :term:`optimized scope` + now returns a write-through proxy rather than a snapshot + that gets updated at ill-specified times. + If a snapshot is desired, it must be created explicitly + (e.g. with :c:func:`PyDict_Copy`), + or by calling the new :c:func:`PyEval_GetFrameLocals` API. + + * :c:func:`!PyFrame_FastToLocals` and :c:func:`!PyFrame_FastToLocalsWithError` + no longer have any effect. + Calling these functions has been redundant since Python 3.11, + when :c:func:`PyFrame_GetLocals` was first introduced. + + * :c:func:`!PyFrame_LocalsToFast` no longer has any effect. + Calling this function is redundant now that :c:func:`PyFrame_GetLocals` + returns a write-through proxy for :term:`optimized scopes `. + +* Python 3.13 removed many private functions. Some of them can be replaced using these + alternatives: + + * ``_PyDict_Pop()``: :c:func:`PyDict_Pop` or :c:func:`PyDict_PopString`; + * ``_PyDict_GetItemWithError()``: :c:func:`PyDict_GetItemRef`; + * ``_PyErr_WriteUnraisableMsg()``: :c:func:`PyErr_FormatUnraisable`; + * ``_PyEval_SetTrace()``: :c:func:`PyEval_SetTrace` or :c:func:`PyEval_SetTraceAllThreads`; + * ``_PyList_Extend()``: :c:func:`PyList_Extend`; + * ``_PyLong_AsInt()``: :c:func:`PyLong_AsInt`; + * ``_PyMem_RawStrdup()``: ``strdup()``; + * ``_PyMem_Strdup()``: ``strdup()``; + * ``_PyObject_ClearManagedDict()``: :c:func:`PyObject_ClearManagedDict`; + * ``_PyObject_VisitManagedDict()``: :c:func:`PyObject_VisitManagedDict`; + * ``_PyThreadState_UncheckedGet()``: :c:func:`PyThreadState_GetUnchecked()`; + * ``_PyTime_AsSecondsDouble()``: :c:func:`PyTime_AsSecondsDouble`; + * ``_PyTime_GetMonotonicClock()``: :c:func:`PyTime_Monotonic` or :c:func:`PyTime_MonotonicRaw`; + * ``_PyTime_GetPerfCounter()``: :c:func:`PyTime_PerfCounter` or :c:func:`PyTime_PerfCounterRaw`; + * ``_PyTime_GetSystemClock()``: :c:func:`PyTime_Time` or :c:func:`PyTime_TimeRaw`; + * ``_PyTime_MAX``: :c:var:`PyTime_MAX`; + * ``_PyTime_MIN``: :c:var:`PyTime_MIN`; + * ``_PyTime_t``: :c:type:`PyTime_t`; + * ``_Py_HashPointer()``: :c:func:`Py_HashPointer`; + * ``_Py_IsFinalizing()``: :c:func:`Py_IsFinalizing`. + + The `pythoncapi-compat project`_ can be used to get most of these new + functions on Python 3.12 and older. Regression Test Changes ======================= diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst new file mode 100644 index 00000000000000..2c10d7fefd44ab --- /dev/null +++ b/Doc/whatsnew/3.14.rst @@ -0,0 +1,1440 @@ + +**************************** + What's new in Python 3.14 +**************************** + +:Editor: TBD + +.. Rules for maintenance: + + * Anyone can add text to this document. Do not spend very much time + on the wording of your changes, because your text will probably + get rewritten to some degree. + + * The maintainer will go through Misc/NEWS periodically and add + changes; it's therefore more important to add your changes to + Misc/NEWS than to this file. + + * This is not a complete list of every single change; completeness + is the purpose of Misc/NEWS. Some changes I consider too small + or esoteric to include. If such a change is added to the text, + I'll just remove it. (This is another reason you shouldn't spend + too much time on writing your addition.) + + * If you want to draw your new text to the attention of the + maintainer, add 'XXX' to the beginning of the paragraph or + section. + + * It's OK to just add a fragmentary note about a change. For + example: "XXX Describe the transmogrify() function added to the + socket module." The maintainer will research the change and + write the necessary text. + + * You can comment out your additions if you like, but it's not + necessary (especially when a final release is some months away). + + * Credit the author of a patch or bugfix. Just the name is + sufficient; the e-mail address isn't necessary. + + * It's helpful to add the issue number as a comment: + + XXX Describe the transmogrify() function added to the socket + module. + (Contributed by P.Y. Developer in :gh:`12345`.) + + This saves the maintainer the effort of going through the VCS log + when researching a change. + +This article explains the new features in Python 3.14, compared to 3.13. + +For full details, see the :ref:`changelog `. + +.. note:: + + Prerelease users should be aware that this document is currently in draft + form. It will be updated substantially as Python 3.14 moves towards release, + so it's worth checking back even after reading earlier versions. + + +Summary -- release highlights +============================= + +.. This section singles out the most important changes in Python 3.14. + Brevity is key. + + +.. PEP-sized items next. + +* :ref:`PEP 649: deferred evaluation of annotations ` +* :ref:`PEP 741: Python Configuration C API ` +* :ref:`PEP 761: Discontinuation of PGP signatures ` + + +New features +============ + +.. _whatsnew314-pep649: + +PEP 649: deferred evaluation of annotations +------------------------------------------- + +The :term:`annotations ` on functions, classes, and modules are no +longer evaluated eagerly. Instead, annotations are stored in special-purpose +:term:`annotate functions ` and evaluated only when +necessary. This is specified in :pep:`649` and :pep:`749`. + +This change is designed to make annotations in Python more performant and more +usable in most circumstances. The runtime cost for defining annotations is +minimized, but it remains possible to introspect annotations at runtime. +It is usually no longer necessary to enclose annotations in strings if they +contain forward references. + +The new :mod:`annotationlib` module provides tools for inspecting deferred +annotations. Annotations may be evaluated in the :attr:`~annotationlib.Format.VALUE` +format (which evaluates annotations to runtime values, similar to the behavior in +earlier Python versions), the :attr:`~annotationlib.Format.FORWARDREF` format +(which replaces undefined names with special markers), and the +:attr:`~annotationlib.Format.STRING` format (which returns annotations as strings). + +This example shows how these formats behave: + +.. doctest:: + + >>> from annotationlib import get_annotations, Format + >>> def func(arg: Undefined): + ... pass + >>> get_annotations(func, format=Format.VALUE) + Traceback (most recent call last): + ... + NameError: name 'Undefined' is not defined + >>> get_annotations(func, format=Format.FORWARDREF) + {'arg': ForwardRef('Undefined')} + >>> get_annotations(func, format=Format.STRING) + {'arg': 'Undefined'} + +Implications for annotated code +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you define annotations in your code (for example, for use with a static type +checker), then this change probably does not affect you: you can keep +writing annotations the same way you did with previous versions of Python. + +You will likely be able to remove quoted strings in annotations, which are frequently +used for forward references. Similarly, if you use ``from __future__ import annotations`` +to avoid having to write strings in annotations, you may well be able to +remove that import. However, if you rely on third-party libraries that read annotations, +those libraries may need changes to support unquoted annotations before they +work as expected. + +Implications for readers of ``__annotations__`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If your code reads the ``__annotations__`` attribute on objects, you may want +to make changes in order to support code that relies on deferred evaluation of +annotations. For example, you may want to use :func:`annotationlib.get_annotations` +with the :attr:`~annotationlib.Format.FORWARDREF` format, as the :mod:`dataclasses` +module now does. + +Related changes +^^^^^^^^^^^^^^^ + +The changes in Python 3.14 are designed to rework how ``__annotations__`` +works at runtime while minimizing breakage to code that contains +annotations in source code and to code that reads ``__annotations__``. However, +if you rely on undocumented details of the annotation behavior or on private +functions in the standard library, there are many ways in which your code may +not work in Python 3.14. To safeguard your code against future changes, +use only the documented functionality of the :mod:`annotationlib` module. + +``from __future__ import annotations`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In Python 3.7, :pep:`563` introduced the ``from __future__ import annotations`` +directive, which turns all annotations into strings. This directive is now +considered deprecated and it is expected to be removed in a future version of Python. +However, this removal will not happen until after Python 3.13, the last version of +Python without deferred evaluation of annotations, reaches its end of life in 2029. +In Python 3.14, the behavior of code using ``from __future__ import annotations`` +is unchanged. + + +Improved error messages +----------------------- + +* When unpacking assignment fails due to incorrect number of variables, the + error message prints the received number of values in more cases than before. + (Contributed by Tushar Sadhwani in :gh:`122239`.) + + .. code-block:: pycon + + >>> x, y, z = 1, 2, 3, 4 + Traceback (most recent call last): + File "", line 1, in + x, y, z = 1, 2, 3, 4 + ^^^^^^^ + ValueError: too many values to unpack (expected 3, got 4) + + +.. _whatsnew314-pep741: + +PEP 741: Python Configuration C API +----------------------------------- + +Add a :ref:`PyInitConfig C API ` to configure the Python +initialization without relying on C structures and the ability to make +ABI-compatible changes in the future. + +Complete the :pep:`587` :ref:`PyConfig C API ` by adding +:c:func:`PyInitConfig_AddModule` which can be used to add a built-in extension +module; feature previously referred to as the “inittab”. + +Add :c:func:`PyConfig_Get` and :c:func:`PyConfig_Set` functions to get and set +the current runtime configuration. + +PEP 587 “Python Initialization Configuration” unified all the ways to configure +the Python initialization. This PEP unifies also the configuration of the +Python preinitialization and the Python initialization in a single API. +Moreover, this PEP only provides a single choice to embed Python, instead of +having two “Python” and “Isolated” choices (PEP 587), to simplify the API +further. + +The lower level PEP 587 PyConfig API remains available for use cases with an +intentionally higher level of coupling to CPython implementation details (such +as emulating the full functionality of CPython’s CLI, including its +configuration mechanisms). + +(Contributed by Victor Stinner in :gh:`107954`.) + +.. seealso:: + :pep:`741`. + + +Other language changes +====================== + +* The :func:`map` built-in now has an optional keyword-only *strict* flag + like :func:`zip` to check that all the iterables are of equal length. + (Contributed by Wannes Boeykens in :gh:`119793`.) + +* Incorrect usage of :keyword:`await` and asynchronous comprehensions + is now detected even if the code is optimized away by the :option:`-O` + command-line option. For example, ``python -O -c 'assert await 1'`` + now produces a :exc:`SyntaxError`. (Contributed by Jelle Zijlstra in :gh:`121637`.) + +* Writes to ``__debug__`` are now detected even if the code is optimized + away by the :option:`-O` command-line option. For example, + ``python -O -c 'assert (__debug__ := 1)'`` now produces a + :exc:`SyntaxError`. (Contributed by Irit Katriel in :gh:`122245`.) + +* Add class methods :meth:`float.from_number` and :meth:`complex.from_number` + to convert a number to :class:`float` or :class:`complex` type correspondingly. + They raise an error if the argument is a string. + (Contributed by Serhiy Storchaka in :gh:`84978`.) + +* Implement mixed-mode arithmetic rules combining real and complex numbers as + specified by C standards since C99. + (Contributed by Sergey B Kirpichev in :gh:`69639`.) + +* All Windows code pages are now supported as "cpXXX" codecs on Windows. + (Contributed by Serhiy Storchaka in :gh:`123803`.) + +* :class:`super` objects are now :mod:`pickleable ` and + :mod:`copyable `. + (Contributed by Serhiy Storchaka in :gh:`125767`.) + +* The :class:`memoryview` type now supports subscription, + making it a :term:`generic type`. + (Contributed by Brian Schubert in :gh:`126012`.) + +* ``\B`` in :mod:`regular expression ` now matches empty input string. + Now it is always the opposite of ``\b``. + (Contributed by Serhiy Storchaka in :gh:`124130`.) + +* iOS and macOS apps can now be configured to redirect ``stdout`` and + ``stderr`` content to the system log. (Contributed by Russell Keith-Magee in + :gh:`127592`.) + +* The iOS testbed is now able to stream test output while the test is running. + The testbed can also be used to run the test suite of projects other than + CPython itself. (Contributed by Russell Keith-Magee in :gh:`127592`.) + +New modules +=========== + +* :mod:`annotationlib`: For introspecting :term:`annotations `. + See :pep:`749` for more details. + (Contributed by Jelle Zijlstra in :gh:`119180`.) + + +Improved modules +================ + +argparse +-------- + +* The default value of the :ref:`program name ` for + :class:`argparse.ArgumentParser` now reflects the way the Python + interpreter was instructed to find the ``__main__`` module code. + (Contributed by Serhiy Storchaka and Alyssa Coghlan in :gh:`66436`.) + +* Introduced the optional *suggest_on_error* parameter to + :class:`argparse.ArgumentParser`, enabling suggestions for argument choices + and subparser names if mistyped by the user. + (Contributed by Savannah Ostrowski in :gh:`124456`.) + +ast +--- + +* Add :func:`ast.compare` for comparing two ASTs. + (Contributed by Batuhan Taskaya and Jeremy Hylton in :gh:`60191`.) + +* Add support for :func:`copy.replace` for AST nodes. + (Contributed by Bénédikt Tran in :gh:`121141`.) + +* Docstrings are now removed from an optimized AST in optimization level 2. + (Contributed by Irit Katriel in :gh:`123958`.) + +* The ``repr()`` output for AST nodes now includes more information. + (Contributed by Tomas R in :gh:`116022`.) + + +calendar +-------- + +* By default, today's date is highlighted in color in :mod:`calendar`'s + :ref:`command-line ` text output. + This can be controlled via the :envvar:`PYTHON_COLORS` environment + variable as well as the canonical |NO_COLOR|_ + and |FORCE_COLOR|_ environment variables. + See also :ref:`using-on-controlling-color`. + (Contributed by Hugo van Kemenade in :gh:`128317`.) + + +concurrent.futures +------------------ + +* Add :class:`~concurrent.futures.InterpreterPoolExecutor`, + which exposes "subinterpreters (multiple Python interpreters in the + same process) to Python code. This is separate from the proposed API + in :pep:`734`. + (Contributed by Eric Snow in :gh:`124548`.) +* The default ``ProcessPoolExecutor`` start method (see + :ref:`multiprocessing-start-methods`) changed from *fork* to *forkserver* on + platforms other than macOS & Windows. If you require the threading + incompatible *fork* start method you must explicitly request it by + supplying a *mp_context* to :class:`concurrent.futures.ProcessPoolExecutor`. + (Contributed by Gregory P. Smith in :gh:`84559`.) + +ctypes +------ + +* The layout of :ref:`bit fields ` + in :class:`~ctypes.Structure` and :class:`~ctypes.Union` + now matches platform defaults (GCC/Clang or MVSC) more closely. + In particular, fields no longer overlap. + (Contributed by Matthias Görgens in :gh:`97702`.) + +* The :attr:`.Structure._layout_` class attribute can now be set + to help match a non-default ABI. + (Contributed by Petr Viktorin in :gh:`97702`.) + +* On Windows, the :exc:`~ctypes.COMError` exception is now public. + (Contributed by Jun Komoda in :gh:`126686`.) + +* On Windows, the :func:`~ctypes.CopyComPointer` function is now public. + (Contributed by Jun Komoda in :gh:`127275`.) + +* :func:`ctypes.memoryview_at` now exists to create a + :class:`memoryview` object that refers to the supplied pointer and + length. This works like :func:`ctypes.string_at` except it avoids a + buffer copy, and is typically useful when implementing pure Python + callback functions that are passed dynamically-sized buffers. + (Contributed by Rian Hunter in :gh:`112018`.) + +* Complex types, :class:`~ctypes.c_float_complex`, + :class:`~ctypes.c_double_complex` and :class:`~ctypes.c_longdouble_complex`, + are now available if both the compiler and the ``libffi`` library support + complex C types. + (Contributed by Sergey B Kirpichev in :gh:`61103`). + + +datetime +-------- + +* Add :meth:`datetime.time.strptime` and :meth:`datetime.date.strptime`. + (Contributed by Wannes Boeykens in :gh:`41431`.) + +decimal +------- + +* Add alternative :class:`~decimal.Decimal` constructor + :meth:`Decimal.from_number() `. + (Contributed by Serhiy Storchaka in :gh:`121798`.) + +dis +--- + +* Add support for rendering full source location information of + :class:`instructions `, rather than only the line number. + This feature is added to the following interfaces via the *show_positions* + keyword argument: + + - :class:`dis.Bytecode` + - :func:`dis.dis` + - :func:`dis.distb` + - :func:`dis.disassemble` + + This feature is also exposed via :option:`dis --show-positions`. + (Contributed by Bénédikt Tran in :gh:`123165`.) + +* Add the :option:`dis --specialized` command-line option to + show specialized bytecode. + (Contributed by Bénédikt Tran in :gh:`127413`.) + + +errno +----- + +* Add :data:`errno.EHWPOISON` error code. + (Contributed by James Roy in :gh:`126585`.) + + +fractions +--------- + +* Add support for converting any objects that have the + :meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`. + (Contributed by Serhiy Storchaka in :gh:`82017`.) + +* Add alternative :class:`~fractions.Fraction` constructor + :meth:`Fraction.from_number() `. + (Contributed by Serhiy Storchaka in :gh:`121797`.) + + +functools +--------- + +* Add support to :func:`functools.partial` and + :func:`functools.partialmethod` for :data:`functools.Placeholder` sentinels + to reserve a place for positional arguments. + (Contributed by Dominykas Grigonis in :gh:`119127`.) + +* Allow the *initial* parameter of :func:`functools.reduce` to be passed + as a keyword argument. + (Contributed by Sayandip Dutta in :gh:`125916`.) + + +getopt +------ + +* Add support for options with optional arguments. + (Contributed by Serhiy Storchaka in :gh:`126374`.) + +* Add support for returning intermixed options and non-option arguments in order. + (Contributed by Serhiy Storchaka in :gh:`126390`.) + +http +---- + +* Directory lists and error pages generated by the :mod:`http.server` + module allow the browser to apply its default dark mode. + (Contributed by Yorik Hansen in :gh:`123430`.) + + +inspect +------- + +* :func:`inspect.signature` takes a new argument *annotation_format* to control + the :class:`annotationlib.Format` used for representing annotations. + (Contributed by Jelle Zijlstra in :gh:`101552`.) + +* :meth:`inspect.Signature.format` takes a new argument *unquote_annotations*. + If true, string :term:`annotations ` are displayed without surrounding quotes. + (Contributed by Jelle Zijlstra in :gh:`101552`.) + +* Add function :func:`inspect.ispackage` to determine whether an object is a + :term:`package` or not. + (Contributed by Zhikang Yan in :gh:`125634`.) + + + +io +-- + +* Reading text from a non-blocking stream with ``read`` may now raise a + :exc:`BlockingIOError` if the operation cannot immediately return bytes. + (Contributed by Giovanni Siragusa in :gh:`109523`.) + + +json +---- + +* Add notes for JSON serialization errors that allow to identify the source + of the error. + (Contributed by Serhiy Storchaka in :gh:`122163`.) + +* Enable the :mod:`json` module to work as a script using the :option:`-m` + switch: :program:`python -m json`. + See the :ref:`JSON command-line interface ` documentation. + (Contributed by Trey Hunner in :gh:`122873`.) + + +mimetypes +--------- + +* Add MS and :rfc:`8081` MIME types for fonts: + + * Embedded OpenType: ``application/vnd.ms-fontobject`` + * OpenType Layout (OTF) ``font/otf`` + * TrueType: ``font/ttf`` + * WOFF 1.0 ``font/woff`` + * WOFF 2.0 ``font/woff2`` + + (Contributed by Sahil Prajapati and Hugo van Kemenade in :gh:`84852`.) + +* Add :rfc:`9559` MIME types for Matroska audiovisual data container + structures, containing: + + * audio with no video: ``audio/matroska`` (``.mka``) + * video: ``video/matroska`` (``.mkv``) + * stereoscopic video: ``video/matroska-3d`` (``.mk3d``) + + (Contributed by Hugo van Kemenade in :gh:`89416`.) + +* Add MIME types for images with RFCs: + + * :rfc:`1494`: CCITT Group 3 (``.g3``) + * :rfc:`3362`: Real-time Facsimile, T.38 (``.t38``) + * :rfc:`3745`: JPEG 2000 (``.jp2``), extension (``.jpx``) and compound (``.jpm``) + * :rfc:`3950`: Tag Image File Format Fax eXtended, TIFF-FX (``.tfx``) + * :rfc:`4047`: Flexible Image Transport System (``.fits``) + * :rfc:`7903`: Enhanced Metafile (``.emf``) and Windows Metafile (``.wmf``) + + (Contributed by Hugo van Kemenade in :gh:`85957`.) + + +multiprocessing +--------------- + +* The default start method (see :ref:`multiprocessing-start-methods`) changed + from *fork* to *forkserver* on platforms other than macOS & Windows where + it was already *spawn*. If you require the threading incompatible *fork* + start method you must explicitly request it using a context from + :func:`multiprocessing.get_context` (preferred) or change the default via + :func:`multiprocessing.set_start_method`. + (Contributed by Gregory P. Smith in :gh:`84559`.) +* :mod:`multiprocessing`'s ``"forkserver"`` start method now authenticates + its control socket to avoid solely relying on filesystem permissions + to restrict what other processes could cause the forkserver to spawn workers + and run code. + (Contributed by Gregory P. Smith for :gh:`97514`.) +* The :ref:`multiprocessing proxy objects ` + for *list* and *dict* types gain previously overlooked missing methods: + + * :meth:`!clear` and :meth:`!copy` for proxies of :class:`list`. + * :meth:`~dict.fromkeys`, ``reversed(d)``, ``d | {}``, ``{} | d``, + ``d |= {'b': 2}`` for proxies of :class:`dict`. + + (Contributed by Roy Hyunjin Han for :gh:`103134`.) + + +operator +-------- + +* Two new functions :func:`operator.is_none` and :func:`operator.is_not_none` + have been added, such that ``operator.is_none(obj)`` is equivalent + to ``obj is None`` and ``operator.is_not_none(obj)`` is equivalent + to ``obj is not None``. + (Contributed by Raymond Hettinger and Nico Mexis in :gh:`115808`.) + + +os +-- + +* Add the :func:`os.reload_environ` function to update :data:`os.environ` and + :data:`os.environb` with changes to the environment made by + :func:`os.putenv`, by :func:`os.unsetenv`, or made outside Python in the + same process. + (Contributed by Victor Stinner in :gh:`120057`.) + +* Add the :data:`~os.SCHED_DEADLINE` and :data:`~os.SCHED_NORMAL` constants + to the :mod:`os` module. + (Contributed by James Roy in :gh:`127688`.) + +* Add the :func:`os.readinto` function to read into a + :ref:`buffer object ` from a file descriptor. + (Contributed by Cody Maloney in :gh:`129205`.) + + +pathlib +------- + +* Add methods to :class:`pathlib.Path` to recursively copy or move files and + directories: + + * :meth:`~pathlib.Path.copy` copies a file or directory tree to a destination. + * :meth:`~pathlib.Path.copy_into` copies *into* a destination directory. + * :meth:`~pathlib.Path.move` moves a file or directory tree to a destination. + * :meth:`~pathlib.Path.move_into` moves *into* a destination directory. + + (Contributed by Barney Gale in :gh:`73991`.) + + +pdb +--- + +* Hardcoded breakpoints (:func:`breakpoint` and :func:`pdb.set_trace`) now + reuse the most recent :class:`~pdb.Pdb` instance that calls + :meth:`~pdb.Pdb.set_trace`, instead of creating a new one each time. + As a result, all the instance specific data like :pdbcmd:`display` and + :pdbcmd:`commands` are preserved across hardcoded breakpoints. + (Contributed by Tian Gao in :gh:`121450`.) + +* Add a new argument *mode* to :class:`pdb.Pdb`. Disable the ``restart`` + command when :mod:`pdb` is in ``inline`` mode. + (Contributed by Tian Gao in :gh:`123757`.) + + +pickle +------ + +* Set the default protocol version on the :mod:`pickle` module to 5. + For more details, see :ref:`pickle protocols `. + +* Add notes for pickle serialization errors that allow to identify the source + of the error. + (Contributed by Serhiy Storchaka in :gh:`122213`.) + + +platform +-------- + +* Add :func:`platform.invalidate_caches` to invalidate the cached results. + + (Contributed by Bénédikt Tran in :gh:`122549`.) + + +pydoc +----- + +* :term:`Annotations ` in help output are now usually + displayed in a format closer to that in the original source. + (Contributed by Jelle Zijlstra in :gh:`101552`.) + + +ssl +--- + +* Indicate through :data:`ssl.HAS_PHA` whether the :mod:`ssl` module supports + TLSv1.3 post-handshake client authentication (PHA). + (Contributed by Will Childs-Klein in :gh:`128036`.) + + +symtable +-------- + +* Expose the following :class:`symtable.Symbol` methods: + + * :meth:`~symtable.Symbol.is_comp_cell` + * :meth:`~symtable.Symbol.is_comp_iter` + * :meth:`~symtable.Symbol.is_free_class` + + (Contributed by Bénédikt Tran in :gh:`120029`.) + + +sys +--- + +* The previously undocumented special function :func:`sys.getobjects`, + which only exists in specialized builds of Python, may now return objects + from other interpreters than the one it's called in. + +sys.monitoring +-------------- + +* Two new events are added: :monitoring-event:`BRANCH_LEFT` and + :monitoring-event:`BRANCH_RIGHT`. The ``BRANCH`` event is deprecated. + +threading +--------- + +* :meth:`threading.Thread.start` now sets the operating system thread name + to :attr:`threading.Thread.name`. + (Contributed by Victor Stinner in :gh:`59705`.) + +tkinter +------- + +* Make tkinter widget methods :meth:`!after` and :meth:`!after_idle` accept + arguments passed by keyword. + (Contributed by Zhikang Yan in :gh:`126899`.) + + +turtle +------ + +* Add context managers for :func:`turtle.fill`, :func:`turtle.poly` + and :func:`turtle.no_animation`. + (Contributed by Marie Roald and Yngve Mardal Moe in :gh:`126350`.) + + +unicodedata +----------- + +* The Unicode database has been updated to Unicode 16.0.0. + + +unittest +-------- + +* :mod:`unittest` output is now colored by default. + This can be controlled via the :envvar:`PYTHON_COLORS` environment + variable as well as the canonical |NO_COLOR|_ + and |FORCE_COLOR|_ environment variables. + See also :ref:`using-on-controlling-color`. + (Contributed by Hugo van Kemenade in :gh:`127221`.) + +* unittest discovery supports :term:`namespace package` as start + directory again. It was removed in Python 3.11. + (Contributed by Jacob Walls in :gh:`80958`.) + +* A number of new methods were added in the :class:`~unittest.TestCase` class + that provide more specialized tests. + + - :meth:`~unittest.TestCase.assertHasAttr` and + :meth:`~unittest.TestCase.assertNotHasAttr` check whether the object + has a particular attribute. + - :meth:`~unittest.TestCase.assertIsSubclass` and + :meth:`~unittest.TestCase.assertNotIsSubclass` check whether the object + is a subclass of a particular class, or of one of a tuple of classes. + - :meth:`~unittest.TestCase.assertStartsWith`, + :meth:`~unittest.TestCase.assertNotStartsWith`, + :meth:`~unittest.TestCase.assertEndsWith` and + :meth:`~unittest.TestCase.assertNotEndsWith` check whether the Unicode + or byte string starts or ends with particular string(s). + + (Contributed by Serhiy Storchaka in :gh:`71339`.) + + +urllib +------ + +* Upgrade HTTP digest authentication algorithm for :mod:`urllib.request` by + supporting SHA-256 digest authentication as specified in :rfc:`7616`. + (Contributed by Calvin Bui in :gh:`128193`.) + + +uuid +---- + +* Add support for UUID version 8 via :func:`uuid.uuid8` as specified + in :rfc:`9562`. + (Contributed by Bénédikt Tran in :gh:`89083`.) + +* :const:`uuid.NIL` and :const:`uuid.MAX` are now available to represent the + Nil and Max UUID formats as defined by :rfc:`9562`. + (Contributed by Nick Pope in :gh:`128427`.) + + +zipinfo +------- + +* Added :func:`ZipInfo._for_archive ` + to resolve suitable defaults for a :class:`~zipfile.ZipInfo` object + as used by :func:`ZipFile.writestr `. + + (Contributed by Bénédikt Tran in :gh:`123424`.) + +* :meth:`zipfile.ZipFile.writestr` now respect ``SOURCE_DATE_EPOCH`` that + distributions can set centrally and have build tools consume this in order + to produce reproducible output. + + (Contributed by Jiahao Li in :gh:`91279`.) + +.. Add improved modules above alphabetically, not here at the end. + +Optimizations +============= + +asyncio +------- + +* :mod:`asyncio` now uses double linked list implementation for native tasks + which speeds up execution by 10% on standard pyperformance benchmarks and + reduces memory usage. + (Contributed by Kumar Aditya in :gh:`107803`.) + +* :mod:`asyncio` has new utility functions for introspecting and printing + the program's call graph: :func:`asyncio.capture_call_graph` and + :func:`asyncio.print_call_graph`. + (Contributed by Yury Selivanov, Pablo Galindo Salgado, and Łukasz Langa + in :gh:`91048`.) + +base64 +------ + +* Improve the performance of :func:`base64.b16decode` by up to ten times, + and reduce the import time of :mod:`base64` by up to six times. + (Contributed by Bénédikt Tran, Chris Markiewicz, and Adam Turner in :gh:`118761`.) + + +io +--- +* :mod:`io` which provides the built-in :func:`open` makes less system calls + when opening regular files as well as reading whole files. Reading a small + operating system cached file in full is up to 15% faster. + :func:`pathlib.Path.read_bytes` has the most optimizations for reading a + file's bytes in full. (Contributed by Cody Maloney and Victor Stinner in + :gh:`120754` and :gh:`90102`.) + + +uuid +---- + +* Improve generation of :class:`~uuid.UUID` objects via their dedicated + functions: + + * :func:`~uuid.uuid3` and :func:`~uuid.uuid5` are both roughly 40% faster + for 16-byte names and 20% faster for 1024-byte names. Performance for + longer names remains unchanged. + * :func:`~uuid.uuid4` and :func:`~uuid.uuid8` are 30% and 40% faster + respectively. + + (Contributed by Bénédikt Tran in :gh:`128150`.) + + +Deprecated +========== + +* :mod:`argparse`: + + * Passing the undocumented keyword argument *prefix_chars* to + :meth:`~argparse.ArgumentParser.add_argument_group` is now + deprecated. + (Contributed by Savannah Ostrowski in :gh:`125563`.) + * Deprecated the :class:`argparse.FileType` type converter. + Anything with resource management should be done downstream after the + arguments are parsed. + (Contributed by Serhiy Storchaka in :gh:`58032`.) + +* :mod:`asyncio`: + + * :func:`!asyncio.iscoroutinefunction` is deprecated + and will be removed in Python 3.16; + use :func:`inspect.iscoroutinefunction` instead. + (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) + + * :mod:`asyncio` policy system is deprecated and will be removed in Python 3.16. + In particular, the following classes and functions are deprecated: + + * :class:`asyncio.AbstractEventLoopPolicy` + * :class:`asyncio.DefaultEventLoopPolicy` + * :class:`asyncio.WindowsSelectorEventLoopPolicy` + * :class:`asyncio.WindowsProactorEventLoopPolicy` + * :func:`asyncio.get_event_loop_policy` + * :func:`asyncio.set_event_loop_policy` + * :func:`asyncio.set_event_loop` + + Users should use :func:`asyncio.run` or :class:`asyncio.Runner` with + *loop_factory* to use the desired event loop implementation. + + For example, to use :class:`asyncio.SelectorEventLoop` on Windows:: + + import asyncio + + async def main(): + ... + + asyncio.run(main(), loop_factory=asyncio.SelectorEventLoop) + + (Contributed by Kumar Aditya in :gh:`127949`.) + +* :mod:`builtins`: + Passing a complex number as the *real* or *imag* argument in the + :func:`complex` constructor is now deprecated; it should only be passed + as a single positional argument. + (Contributed by Serhiy Storchaka in :gh:`109218`.) + +* :mod:`functools`: + Calling the Python implementation of :func:`functools.reduce` with *function* + or *sequence* as keyword arguments is now deprecated. + (Contributed by Kirill Podoprigora in :gh:`121676`.) + +* :mod:`os`: + :term:`Soft deprecate ` :func:`os.popen` and + :func:`os.spawn* ` functions. They should no longer be used to + write new code. The :mod:`subprocess` module is recommended instead. + (Contributed by Victor Stinner in :gh:`120743`.) + +* :mod:`symtable`: + Deprecate :meth:`symtable.Class.get_methods` due to the lack of interest. + (Contributed by Bénédikt Tran in :gh:`119698`.) + +* :mod:`urllib.parse`: + Accepting objects with false values (like ``0`` and ``[]``) except empty + strings, byte-like objects and ``None`` in :mod:`urllib.parse` functions + :func:`~urllib.parse.parse_qsl` and :func:`~urllib.parse.parse_qs` is now + deprecated. + (Contributed by Serhiy Storchaka in :gh:`116897`.) + +.. Add deprecations above alphabetically, not here at the end. + +.. include:: ../deprecations/pending-removal-in-3.15.rst + +.. include:: ../deprecations/pending-removal-in-3.16.rst + +.. include:: ../deprecations/pending-removal-in-future.rst + +Removed +======= + +argparse +-------- + +* Remove the *type*, *choices*, and *metavar* parameters + of :class:`!argparse.BooleanOptionalAction`. + They were deprecated since 3.12. + +* Calling :meth:`~argparse.ArgumentParser.add_argument_group` on an argument + group, and calling :meth:`~argparse.ArgumentParser.add_argument_group` or + :meth:`~argparse.ArgumentParser.add_mutually_exclusive_group` on a mutually + exclusive group now raise exceptions. This nesting was never supported, + often failed to work correctly, and was unintentionally exposed through + inheritance. This functionality has been deprecated since Python 3.11. + (Contributed by Savannah Ostrowski in :gh:`127186`.) + +ast +--- + +* Remove the following classes. They were all deprecated since Python 3.8, + and have emitted deprecation warnings since Python 3.12: + + * :class:`!ast.Bytes` + * :class:`!ast.Ellipsis` + * :class:`!ast.NameConstant` + * :class:`!ast.Num` + * :class:`!ast.Str` + + Use :class:`ast.Constant` instead. As a consequence of these removals, + user-defined ``visit_Num``, ``visit_Str``, ``visit_Bytes``, + ``visit_NameConstant`` and ``visit_Ellipsis`` methods on custom + :class:`ast.NodeVisitor` subclasses will no longer be called when the + :class:`!NodeVisitor` subclass is visiting an AST. Define a ``visit_Constant`` + method instead. + + Also, remove the following deprecated properties on :class:`ast.Constant`, + which were present for compatibility with the now-removed AST classes: + + * :attr:`!ast.Constant.n` + * :attr:`!ast.Constant.s` + + Use :attr:`!ast.Constant.value` instead. + + (Contributed by Alex Waygood in :gh:`119562`.) + +asyncio +------- + +* Remove the following classes and functions. They were all deprecated and + emitted deprecation warnings since Python 3.12: + + * :func:`!asyncio.get_child_watcher` + * :func:`!asyncio.set_child_watcher` + * :meth:`!asyncio.AbstractEventLoopPolicy.get_child_watcher` + * :meth:`!asyncio.AbstractEventLoopPolicy.set_child_watcher` + * :class:`!asyncio.AbstractChildWatcher` + * :class:`!asyncio.FastChildWatcher` + * :class:`!asyncio.MultiLoopChildWatcher` + * :class:`!asyncio.PidfdChildWatcher` + * :class:`!asyncio.SafeChildWatcher` + * :class:`!asyncio.ThreadedChildWatcher` + + (Contributed by Kumar Aditya in :gh:`120804`.) + +* Removed implicit creation of event loop by :func:`asyncio.get_event_loop`. + It now raises a :exc:`RuntimeError` if there is no current event loop. + (Contributed by Kumar Aditya in :gh:`126353`.) + + There's a few patterns that use :func:`asyncio.get_event_loop`, most + of them can be replaced with :func:`asyncio.run`. + + If you're running an async function, simply use :func:`asyncio.run`. + + Before:: + + async def main(): + ... + + + loop = asyncio.get_event_loop() + try: + loop.run_until_complete(main()) + finally: + loop.close() + + After:: + + async def main(): + ... + + asyncio.run(main()) + + If you need to start something, e.g. a server listening on a socket + and then run forever, use :func:`asyncio.run` and an + :class:`asyncio.Event`. + + Before:: + + def start_server(loop): + ... + + loop = asyncio.get_event_loop() + try: + start_server(loop) + loop.run_forever() + finally: + loop.close() + + After:: + + def start_server(loop): + ... + + async def main(): + start_server(asyncio.get_running_loop()) + await asyncio.Event().wait() + + asyncio.run(main()) + + If you need to run something in an event loop, then run some blocking + code around it, use :class:`asyncio.Runner`. + + Before:: + + async def operation_one(): + ... + + def blocking_code(): + ... + + async def operation_two(): + ... + + loop = asyncio.get_event_loop() + try: + loop.run_until_complete(operation_one()) + blocking_code() + loop.run_until_complete(operation_two()) + finally: + loop.close() + + After:: + + async def operation_one(): + ... + + def blocking_code(): + ... + + async def operation_two(): + ... + + with asyncio.Runner() as runner: + runner.run(operation_one()) + blocking_code() + runner.run(operation_two()) + + + +collections.abc +--------------- + +* Remove :class:`!collections.abc.ByteString`. It had previously raised a + :exc:`DeprecationWarning` since Python 3.12. + +email +----- + +* Remove the *isdst* parameter from :func:`email.utils.localtime`. + (Contributed by Hugo van Kemenade in :gh:`118798`.) + +importlib +--------- + +* Remove deprecated :mod:`importlib.abc` classes: + + * :class:`!importlib.abc.ResourceReader` + * :class:`!importlib.abc.Traversable` + * :class:`!importlib.abc.TraversableResources` + + Use :mod:`importlib.resources.abc` classes instead: + + * :class:`importlib.resources.abc.Traversable` + * :class:`importlib.resources.abc.TraversableResources` + + (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) + +itertools +--------- + +* Remove :mod:`itertools` support for copy, deepcopy, and pickle operations. + These had previously raised a :exc:`DeprecationWarning` since Python 3.12. + (Contributed by Raymond Hettinger in :gh:`101588`.) + +pathlib +------- + +* Remove support for passing additional keyword arguments to + :class:`pathlib.Path`. In previous versions, any such arguments are ignored. +* Remove support for passing additional positional arguments to + :meth:`pathlib.PurePath.relative_to` and + :meth:`~pathlib.PurePath.is_relative_to`. In previous versions, any such + arguments are joined onto *other*. + +pkgutil +------- + +* Remove deprecated :func:`!pkgutil.get_loader` and :func:`!pkgutil.find_loader`. + These had previously raised a :exc:`DeprecationWarning` since Python 3.12. + (Contributed by Bénédikt Tran in :gh:`97850`.) + +pty +--- + +* Remove deprecated :func:`!pty.master_open` and :func:`!pty.slave_open`. + They had previously raised a :exc:`DeprecationWarning` since Python 3.12. + Use :func:`pty.openpty` instead. + (Contributed by Nikita Sobolev in :gh:`118824`.) + +sqlite3 +------- + +* Remove :data:`!version` and :data:`!version_info` from :mod:`sqlite3`. + (Contributed by Hugo van Kemenade in :gh:`118924`.) + +* Disallow using a sequence of parameters with named placeholders. + This had previously raised a :exc:`DeprecationWarning` since Python 3.12; + it will now raise a :exc:`sqlite3.ProgrammingError`. + (Contributed by Erlend E. Aasland in :gh:`118928` and :gh:`101693`.) + +typing +------ + +* Remove :class:`!typing.ByteString`. It had previously raised a + :exc:`DeprecationWarning` since Python 3.12. + +urllib +------ + +* Remove deprecated :class:`!Quoter` class from :mod:`urllib.parse`. + It had previously raised a :exc:`DeprecationWarning` since Python 3.11. + (Contributed by Nikita Sobolev in :gh:`118827`.) +* Remove deprecated :class:`!URLopener` and :class:`!FancyURLopener` classes + from :mod:`urllib.request`. They had previously raised a + :exc:`DeprecationWarning` since Python 3.3. + + ``myopener.open()`` can be replaced with :func:`~urllib.request.urlopen`, + and ``myopener.retrieve()`` can be replaced with + :func:`~urllib.request.urlretrieve`. Customizations to the opener + classes can be replaced by passing customized handlers to + :func:`~urllib.request.build_opener`. + + (Contributed by Barney Gale in :gh:`84850`.) + +Others +------ + +* Using :data:`NotImplemented` in a boolean context will now raise a :exc:`TypeError`. + It had previously raised a :exc:`DeprecationWarning` since Python 3.9. (Contributed + by Jelle Zijlstra in :gh:`118767`.) + +* The :func:`int` built-in no longer delegates to + :meth:`~object.__trunc__`. Classes that want to support conversion to + integer must implement either :meth:`~object.__int__` or + :meth:`~object.__index__`. (Contributed by Mark Dickinson in :gh:`119743`.) + + +Porting to Python 3.14 +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + +Changes in the Python API +------------------------- + +* :class:`functools.partial` is now a method descriptor. + Wrap it in :func:`staticmethod` if you want to preserve the old behavior. + (Contributed by Serhiy Storchaka and Dominykas Grigonis in :gh:`121027`.) + +* The :func:`locale.nl_langinfo` function now sets temporarily the ``LC_CTYPE`` + locale in some cases. + This temporary change affects other threads. + (Contributed by Serhiy Storchaka in :gh:`69998`.) + + +Build changes +============= + +* GNU Autoconf 2.72 is now required to generate :file:`configure`. + (Contributed by Erlend Aasland in :gh:`115765`.) + +.. _whatsnew314-pep761: + +PEP 761: Discontinuation of PGP signatures +------------------------------------------ + +PGP signatures will not be available for CPython 3.14 and onwards. +Users verifying artifacts must use `Sigstore verification materials`_ for +verifying CPython artifacts. This change in release process is specified +in :pep:`761`. + +.. _Sigstore verification materials: https://www.python.org/downloads/metadata/sigstore/ + + +C API changes +============= + +New features +------------ + +* Add :c:func:`PyLong_GetSign` function to get the sign of :class:`int` objects. + (Contributed by Sergey B Kirpichev in :gh:`116560`.) + +* Add a new :c:type:`PyUnicodeWriter` API to create a Python :class:`str` + object: + + * :c:func:`PyUnicodeWriter_Create` + * :c:func:`PyUnicodeWriter_DecodeUTF8Stateful` + * :c:func:`PyUnicodeWriter_Discard` + * :c:func:`PyUnicodeWriter_Finish` + * :c:func:`PyUnicodeWriter_Format` + * :c:func:`PyUnicodeWriter_WriteChar` + * :c:func:`PyUnicodeWriter_WriteRepr` + * :c:func:`PyUnicodeWriter_WriteStr` + * :c:func:`PyUnicodeWriter_WriteSubstring` + * :c:func:`PyUnicodeWriter_WriteUCS4` + * :c:func:`PyUnicodeWriter_WriteUTF8` + * :c:func:`PyUnicodeWriter_WriteWideChar` + + (Contributed by Victor Stinner in :gh:`119182`.) + +* Add :c:func:`PyIter_NextItem` to replace :c:func:`PyIter_Next`, + which has an ambiguous return value. + (Contributed by Irit Katriel and Erlend Aasland in :gh:`105201`.) + +* Add :c:func:`PyLong_IsPositive`, :c:func:`PyLong_IsNegative` + and :c:func:`PyLong_IsZero` for checking if :c:type:`PyLongObject` + is positive, negative, or zero, respectively. + (Contributed by James Roy and Sergey B Kirpichev in :gh:`126061`.) + +* Add new functions to convert C ```` numbers from/to Python + :class:`int`: + + * :c:func:`PyLong_AsInt32` + * :c:func:`PyLong_AsInt64` + * :c:func:`PyLong_AsUInt32` + * :c:func:`PyLong_AsUInt64` + * :c:func:`PyLong_FromInt32` + * :c:func:`PyLong_FromInt64` + * :c:func:`PyLong_FromUInt32` + * :c:func:`PyLong_FromUInt64` + + (Contributed by Victor Stinner in :gh:`120389`.) + +* Add :c:func:`PyBytes_Join(sep, iterable) ` function, + similar to ``sep.join(iterable)`` in Python. + (Contributed by Victor Stinner in :gh:`121645`.) + +* Add :c:func:`Py_HashBuffer` to compute and return the hash value of a buffer. + (Contributed by Antoine Pitrou and Victor Stinner in :gh:`122854`.) + +* Add functions to get and set the current runtime Python configuration + (:pep:`741`): + + * :c:func:`PyConfig_Get` + * :c:func:`PyConfig_GetInt` + * :c:func:`PyConfig_Set` + * :c:func:`PyConfig_Names` + + (Contributed by Victor Stinner in :gh:`107954`.) + +* Add functions to configure the Python initialization (:pep:`741`): + + * :c:func:`Py_InitializeFromInitConfig` + * :c:func:`PyInitConfig_AddModule` + * :c:func:`PyInitConfig_Create` + * :c:func:`PyInitConfig_Free` + * :c:func:`PyInitConfig_FreeStrList` + * :c:func:`PyInitConfig_GetError` + * :c:func:`PyInitConfig_GetExitCode` + * :c:func:`PyInitConfig_GetInt` + * :c:func:`PyInitConfig_GetStr` + * :c:func:`PyInitConfig_GetStrList` + * :c:func:`PyInitConfig_HasOption` + * :c:func:`PyInitConfig_SetInt` + * :c:func:`PyInitConfig_SetStr` + * :c:func:`PyInitConfig_SetStrList` + + (Contributed by Victor Stinner in :gh:`107954`.) + +* Add a new import and export API for Python :class:`int` objects (:pep:`757`): + + * :c:func:`PyLong_GetNativeLayout`; + * :c:func:`PyLong_Export`; + * :c:func:`PyLong_FreeExport`; + * :c:func:`PyLongWriter_Create`; + * :c:func:`PyLongWriter_Finish`; + * :c:func:`PyLongWriter_Discard`. + + (Contributed by Sergey B Kirpichev and Victor Stinner in :gh:`102471`.) + +* Add :c:func:`PyType_GetBaseByToken` and :c:data:`Py_tp_token` slot for easier + superclass identification, which attempts to resolve the `type checking issue + `__ mentioned in :pep:`630` + (:gh:`124153`). + +* Add :c:func:`PyUnicode_Equal` function to the limited C API: + test if two strings are equal. + (Contributed by Victor Stinner in :gh:`124502`.) + +* Add :c:func:`PyType_Freeze` function to make a type immutable. + (Contributed by Victor Stinner in :gh:`121654`.) + +* Add :c:func:`PyUnstable_Object_EnableDeferredRefcount` for enabling + deferred reference counting, as outlined in :pep:`703`. + +* Add :c:func:`PyMonitoring_FireBranchLeftEvent` and + :c:func:`PyMonitoring_FireBranchRightEvent` for generating + :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT` + events, respectively. + +* Add :c:func:`Py_fopen` function to open a file. Similar to the + :c:func:`!fopen` function, but the *path* parameter is a Python object and an + exception is set on error. Add also :c:func:`Py_fclose` function to close a + file. + (Contributed by Victor Stinner in :gh:`127350`.) + +* Add macros :c:func:`Py_PACK_VERSION` and :c:func:`Py_PACK_FULL_VERSION` for + bit-packing Python version numbers. + (Contributed by Petr Viktorin in :gh:`128629`.) + +* Add :c:func:`PyUnstable_IsImmortal` for determining whether an object is :term:`immortal`, + for debugging purposes. + + +Limited C API changes +--------------------- + +* In the limited C API 3.14 and newer, :c:func:`Py_TYPE` and + :c:func:`Py_REFCNT` are now implemented as an opaque function call to hide + implementation details. + (Contributed by Victor Stinner in :gh:`120600` and :gh:`124127`.) + + +Porting to Python 3.14 +---------------------- + +* :c:func:`Py_Finalize` now deletes all interned strings. This + is backwards incompatible to any C-Extension that holds onto an interned + string after a call to :c:func:`Py_Finalize` and is then reused after a + call to :c:func:`Py_Initialize`. Any issues arising from this behavior will + normally result in crashes during the execution of the subsequent call to + :c:func:`Py_Initialize` from accessing uninitialized memory. To fix, use + an address sanitizer to identify any use-after-free coming from + an interned string and deallocate it during module shutdown. + (Contributed by Eddie Elizondo in :gh:`113601`.) + +* The :ref:`Unicode Exception Objects ` C API + now raises a :exc:`TypeError` if its exception argument is not + a :exc:`UnicodeError` object. + (Contributed by Bénédikt Tran in :gh:`127691`.) + +* Private functions promoted to public C APIs: + + * ``_PyBytes_Join()``: :c:func:`PyBytes_Join`; + * ``_PyLong_IsNegative()``: :c:func:`PyLong_IsNegative`; + * ``_PyLong_IsPositive()``: :c:func:`PyLong_IsPositive`; + * ``_PyLong_IsZero()``: :c:func:`PyLong_IsZero`; + * ``_PyLong_Sign()``: :c:func:`PyLong_GetSign`; + * ``_PyUnicodeWriter_Dealloc()``: :c:func:`PyUnicodeWriter_Discard`; + * ``_PyUnicodeWriter_Finish()``: :c:func:`PyUnicodeWriter_Finish`; + * ``_PyUnicodeWriter_Init()``: :c:func:`PyUnicodeWriter_Create`; + * ``_PyUnicodeWriter_WriteChar()``: :c:func:`PyUnicodeWriter_WriteChar`; + * ``_PyUnicodeWriter_WriteStr()``: :c:func:`PyUnicodeWriter_WriteStr`; + * ``_PyUnicodeWriter_WriteSubstring()``: :c:func:`PyUnicodeWriter_WriteSubstring`; + * ``_PyUnicode_EQ()``: :c:func:`PyUnicode_Equal`; + * ``_PyUnicode_Equal()``: :c:func:`PyUnicode_Equal`; + * ``_Py_GetConfig()``: :c:func:`PyConfig_Get` and :c:func:`PyConfig_GetInt`; + * ``_Py_HashBytes()``: :c:func:`Py_HashBuffer`; + * ``_Py_fopen_obj()``: :c:func:`Py_fopen`. + + The `pythoncapi-compat project`_ can be used to get most of these new + functions on Python 3.13 and older. + +.. _pythoncapi-compat project: https://github.com/python/pythoncapi-compat/ + + +Deprecated +---------- + +* The :c:macro:`!Py_HUGE_VAL` macro is :term:`soft deprecated`, + use :c:macro:`!Py_INFINITY` instead. + (Contributed by Sergey B Kirpichev in :gh:`120026`.) + +* Macros :c:macro:`!Py_IS_NAN`, :c:macro:`!Py_IS_INFINITY` + and :c:macro:`!Py_IS_FINITE` are :term:`soft deprecated`, + use instead :c:macro:`!isnan`, :c:macro:`!isinf` and + :c:macro:`!isfinite` available from :file:`math.h` + since C99. (Contributed by Sergey B Kirpichev in :gh:`119613`.) + +* The previously undocumented function :c:func:`PySequence_In` is :term:`soft deprecated`. + Use :c:func:`PySequence_Contains` instead. + (Contributed by Yuki Kobayashi in :gh:`127896`.) + +.. Add C API deprecations above alphabetically, not here at the end. + +* The ``PyMonitoring_FireBranchEvent`` function is deprecated and should + be replaced with calls to :c:func:`PyMonitoring_FireBranchLeftEvent` + and :c:func:`PyMonitoring_FireBranchRightEvent`. + +* The following private functions are deprecated and planned for removal in + Python 3.18: + + * :c:func:`!_PyBytes_Join`: use :c:func:`PyBytes_Join`. + * :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`. + * :c:func:`!_PyDict_Pop()`: use :c:func:`PyDict_Pop`. + * :c:func:`!_PyLong_Sign()`: use :c:func:`PyLong_GetSign`. + * :c:func:`!_PyLong_FromDigits` and :c:func:`!_PyLong_New`: + use :c:func:`PyLongWriter_Create`. + * :c:func:`!_PyThreadState_UncheckedGet`: use :c:func:`PyThreadState_GetUnchecked`. + * :c:func:`!_PyUnicode_AsString`: use :c:func:`PyUnicode_AsUTF8`. + * :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`. + * :c:func:`!_Py_fopen_obj`: use :c:func:`Py_fopen`. + + The `pythoncapi-compat project`_ can be used to get these new public + functions on Python 3.13 and older. + + (Contributed by Victor Stinner in :gh:`128863`.) + + +.. include:: ../deprecations/c-api-pending-removal-in-3.15.rst + +.. include:: ../deprecations/c-api-pending-removal-in-3.18.rst + +.. include:: ../deprecations/c-api-pending-removal-in-future.rst + + +Removed +------- + +* Creating :c:data:`immutable types ` with mutable + bases was deprecated since 3.12 and now raises a :exc:`TypeError`. + +* Remove ``PyDictObject.ma_version_tag`` member which was deprecated since + Python 3.12. Use the :c:func:`PyDict_AddWatcher` API instead. + (Contributed by Sam Gross in :gh:`124296`.) + +* Remove the private ``_Py_InitializeMain()`` function. It was a + :term:`provisional API` added to Python 3.8 by :pep:`587`. + (Contributed by Victor Stinner in :gh:`129033`.) diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 52474517f5facc..7104904c956a7a 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -312,8 +312,8 @@ cluttering source directories, the *pyc* files are now collected in a Aside from the filenames and target directories, the new scheme has a few aspects that are visible to the programmer: -* Imported modules now have a :attr:`__cached__` attribute which stores the name - of the actual file that was imported: +* Imported modules now have a :attr:`~module.__cached__` attribute which stores + the name of the actual file that was imported: >>> import collections >>> collections.__cached__ # doctest: +SKIP @@ -531,7 +531,7 @@ Some smaller changes made to the core Python language are: (Proposed and implemented by Mark Dickinson; :issue:`9337`.) -* :class:`memoryview` objects now have a :meth:`~memoryview.release()` method +* :class:`memoryview` objects now have a :meth:`~memoryview.release` method and they also now support the context management protocol. This allows timely release of any resources that were acquired when requesting a buffer from the original object. @@ -785,8 +785,8 @@ functools (Contributed by Raymond Hettinger and incorporating design ideas from Jim Baker, Miki Tebeka, and Nick Coghlan; see `recipe 498245 - `_\, `recipe 577479 - `_\, :issue:`10586`, and + `_\, `recipe 577479 + `_\, :issue:`10586`, and :issue:`10593`.) * The :func:`functools.wraps` decorator now adds a :attr:`__wrapped__` attribute @@ -1312,7 +1312,7 @@ An early decision to limit the interoperability of various numeric types has been relaxed. It is still unsupported (and ill-advised) to have implicit mixing in arithmetic expressions such as ``Decimal('1.1') + float('1.1')`` because the latter loses information in the process of constructing the binary -float. However, since existing floating point value can be converted losslessly +float. However, since existing floating-point value can be converted losslessly to either a decimal or rational representation, it makes sense to add them to the constructor and to support mixed-type comparisons. @@ -1325,7 +1325,7 @@ the constructor and to support mixed-type comparisons. and :class:`fractions.Fraction` (:issue:`2531` and :issue:`8188`). Similar changes were made to :class:`fractions.Fraction` so that the -:meth:`~fractions.Fraction.from_float()` and :meth:`~fractions.Fraction.from_decimal` +:meth:`~fractions.Fraction.from_float` and :meth:`~fractions.Fraction.from_decimal` methods are no longer needed (:issue:`8294`): >>> from decimal import Decimal @@ -1622,7 +1622,7 @@ socket The :mod:`socket` module has two new improvements. -* Socket objects now have a :meth:`~socket.socket.detach()` method which puts +* Socket objects now have a :meth:`~socket.socket.detach` method which puts the socket into closed state without actually closing the underlying file descriptor. The latter can then be reused for other purposes. (Added by Antoine Pitrou; :issue:`8524`.) @@ -1650,7 +1650,7 @@ for secure (encrypted, authenticated) internet connections: * The :func:`ssl.wrap_socket() ` constructor function now takes a *ciphers* argument. The *ciphers* string lists the allowed encryption algorithms using the format described in the `OpenSSL documentation - `__. + `__. * When linked against recent versions of OpenSSL, the :mod:`ssl` module now supports the Server Name Indication extension to the TLS protocol, allowing @@ -1859,11 +1859,11 @@ asyncore -------- :class:`!asyncore.dispatcher` now provides a -:meth:`!handle_accepted()` method +:meth:`!handle_accepted` method returning a ``(sock, addr)`` pair which is called when a connection has actually been established with a new remote endpoint. This is supposed to be used as a -replacement for old :meth:`!handle_accept()` and avoids -the user to call :meth:`!accept()` directly. +replacement for old :meth:`!handle_accept` and avoids +the user to call :meth:`!accept` directly. (Contributed by Giampaolo Rodolà; :issue:`6706`.) @@ -2321,7 +2321,7 @@ Multi-threading intervals and reduced overhead due to lock contention and the number of ensuing system calls. The notion of a "check interval" to allow thread switches has been abandoned and replaced by an absolute duration expressed in - seconds. This parameter is tunable through :func:`sys.setswitchinterval()`. + seconds. This parameter is tunable through :func:`sys.setswitchinterval`. It currently defaults to 5 milliseconds. Additional details about the implementation can be read from a `python-dev diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 29b4034e32821c..7a8eb47cbdb354 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -549,9 +549,11 @@ separation of binary and text data). PEP 3155: Qualified name for classes and functions ================================================== -Functions and class objects have a new ``__qualname__`` attribute representing +Functions and class objects have a new :attr:`~definition.__qualname__` +attribute representing the "path" from the module top-level to their definition. For global functions -and classes, this is the same as ``__name__``. For other functions and classes, +and classes, this is the same as :attr:`~definition.__name__`. +For other functions and classes, it provides better information about where they were actually defined, and how they might be accessible from the global scope. @@ -779,8 +781,8 @@ Other Language Changes Some smaller changes made to the core Python language are: * Added support for Unicode name aliases and named sequences. - Both :func:`unicodedata.lookup()` and ``'\N{...}'`` now resolve name aliases, - and :func:`unicodedata.lookup()` resolves named sequences too. + Both :func:`unicodedata.lookup` and ``'\N{...}'`` now resolve name aliases, + and :func:`unicodedata.lookup` resolves named sequences too. (Contributed by Ezio Melotti in :issue:`12753`.) @@ -1097,12 +1099,12 @@ decimal C-module and libmpdec written by Stefan Krah. The new C version of the decimal module integrates the high speed libmpdec -library for arbitrary precision correctly rounded decimal floating point +library for arbitrary precision correctly rounded decimal floating-point arithmetic. libmpdec conforms to IBM's General Decimal Arithmetic Specification. Performance gains range from 10x for database applications to 100x for numerically intensive applications. These numbers are expected gains -for standard precisions used in decimal floating point arithmetic. Since +for standard precisions used in decimal floating-point arithmetic. Since the precision is user configurable, the exact figures may vary. For example, in integer bignum arithmetic the differences can be significantly higher. @@ -1145,8 +1147,8 @@ API changes | :const:`MIN_EMIN` | ``-425000000`` | ``-999999999999999999`` | +-------------------+----------------+-------------------------+ -* In the context templates (:class:`~decimal.DefaultContext`, - :class:`~decimal.BasicContext` and :class:`~decimal.ExtendedContext`) +* In the context templates (:const:`~decimal.DefaultContext`, + :const:`~decimal.BasicContext` and :const:`~decimal.ExtendedContext`) the magnitude of :attr:`~decimal.Context.Emax` and :attr:`~decimal.Context.Emin` has changed to ``999999``. diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 3dd400c3771ed2..71b186aeed7359 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -215,13 +215,12 @@ automatic ``PATH`` modifications to have ``pip`` available from the command line by default, otherwise it can still be accessed through the Python launcher for Windows as ``py -m pip``. -As `discussed in the PEP`__, platform packagers may choose not to install +As :pep:`discussed in the PEP <0453#recommendations-for-downstream-distributors>` +platform packagers may choose not to install these commands by default, as long as, when invoked, they provide clear and simple directions on how to install them on that platform (usually using the system package manager). -__ https://peps.python.org/pep-0453/#recommendations-for-downstream-distributors - .. note:: To avoid conflicts between parallel Python 2 and Python 3 installations, @@ -1495,7 +1494,7 @@ The dictionary returned by :meth:`.SSLSocket.getpeercert` contains additional stat ---- -The :mod:`stat` module is now backed by a C implementation in :mod:`_stat`. A C +The :mod:`stat` module is now backed by a C implementation in :mod:`!_stat`. A C implementation is required as most of the values aren't standardized and are platform-dependent. (Contributed by Christian Heimes in :issue:`11016`.) @@ -1963,11 +1962,11 @@ Other Improvements `_ will build python, run the test suite, and generate an HTML coverage report for the C codebase using ``gcov`` and `lcov - `_. + `_. * The ``-R`` option to the :ref:`python regression test suite ` now also checks for memory allocation leaks, using - :func:`sys.getallocatedblocks()`. (Contributed by Antoine Pitrou in + :func:`sys.getallocatedblocks`. (Contributed by Antoine Pitrou in :issue:`13390`.) * ``python -m`` now works with namespace packages. @@ -1980,7 +1979,7 @@ Other Improvements now works correctly (previously it silently returned the first python module in the file). (Contributed by Václav Šmilauer in :issue:`16421`.) -* A new opcode, :opcode:`LOAD_CLASSDEREF`, has been added to fix a bug in the +* A new opcode, :opcode:`!LOAD_CLASSDEREF`, has been added to fix a bug in the loading of free variables in class bodies that could be triggered by certain uses of :ref:`__prepare__ `. (Contributed by Benjamin Peterson in :issue:`17853`.) @@ -2272,7 +2271,8 @@ Changes in the Python API :func:`super` and falling through all the way to the ABCs. For compatibility, catch both :exc:`NotImplementedError` or the appropriate exception as needed. -* The module type now initializes the :attr:`__package__` and :attr:`__loader__` +* The module type now initializes the :attr:`~module.__package__` and + :attr:`~module.__loader__` attributes to ``None`` by default. To determine if these attributes were set in a backwards-compatible fashion, use e.g. ``getattr(module, '__loader__', None) is not None``. (:issue:`17115`.) @@ -2413,7 +2413,7 @@ Changes in the Python API formal public interface the naming has been made consistent (:issue:`18532`). * Because :mod:`unittest.TestSuite` now drops references to tests after they - are run, test harnesses that re-use a :class:`~unittest.TestSuite` to re-run + are run, test harnesses that reuse a :class:`~unittest.TestSuite` to re-run a set of tests may fail. Test suites should not be re-used in this fashion since it means state is retained between test runs, breaking the test isolation that :mod:`unittest` is designed to provide. However, if the lack diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index 5c2ec230441b42..3f3f634171dab6 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -423,8 +423,8 @@ are declared in the annotations:: return 'Hello ' + name While these annotations are available at runtime through the usual -:attr:`__annotations__` attribute, *no automatic type checking happens at -runtime*. Instead, it is assumed that a separate off-line type checker +:attr:`~object.__annotations__` attribute, *no automatic type checking happens +at runtime*. Instead, it is assumed that a separate off-line type checker (e.g. `mypy `_) will be used for on-demand source code analysis. @@ -951,7 +951,7 @@ New :class:`~collections.abc.Awaitable`, :class:`~collections.abc.Coroutine`, (Contributed by Yury Selivanov in :issue:`24184`.) For earlier Python versions, a backport of the new ABCs is available in an -external `PyPI package `_. +external :pypi:`PyPI package `. compileall @@ -1667,7 +1667,7 @@ Both the :class:`!SMTPServer` and :class:`!SMTPChannel` classes now accept a *decode_data* keyword argument to determine if the ``DATA`` portion of the SMTP transaction is decoded using the ``"utf-8"`` codec or is instead provided to the -:meth:`!SMTPServer.process_message()` +:meth:`!SMTPServer.process_message` method as a byte string. The default is ``True`` for backward compatibility reasons, but will change to ``False`` in Python 3.6. If *decode_data* is set to ``False``, the ``process_message`` method must be prepared to accept keyword @@ -1677,14 +1677,14 @@ arguments. The :class:`!SMTPServer` class now advertises the ``8BITMIME`` extension (:rfc:`6152`) if *decode_data* has been set ``True``. If the client specifies ``BODY=8BITMIME`` on the ``MAIL`` command, it is passed to -:meth:`!SMTPServer.process_message()` +:meth:`!SMTPServer.process_message` via the *mail_options* keyword. (Contributed by Milan Oberkirch and R. David Murray in :issue:`21795`.) The :class:`!SMTPServer` class now also supports the ``SMTPUTF8`` extension (:rfc:`6531`: Internationalized Email). If the client specified ``SMTPUTF8 BODY=8BITMIME`` on the ``MAIL`` command, they are passed to -:meth:`!SMTPServer.process_message()` +:meth:`!SMTPServer.process_message` via the *mail_options* keyword. It is the responsibility of the ``process_message`` method to correctly handle the ``SMTPUTF8`` data. (Contributed by Milan Oberkirch in :issue:`21725`.) @@ -1935,8 +1935,8 @@ specifying the namespace in which the code will be running. tkinter ------- -The :mod:`tkinter._fix` module used for setting up the Tcl/Tk environment -on Windows has been replaced by a private function in the :mod:`_tkinter` +The :mod:`!tkinter._fix` module used for setting up the Tcl/Tk environment +on Windows has been replaced by a private function in the :mod:`!_tkinter` module which makes no permanent changes to environment variables. (Contributed by Zachary Ware in :issue:`20035`.) @@ -2405,7 +2405,7 @@ Changes in the Python API error-prone and has been removed in Python 3.5. See :issue:`13936` for full details. -* The :meth:`ssl.SSLSocket.send()` method now raises either +* The :meth:`ssl.SSLSocket.send` method now raises either :exc:`ssl.SSLWantReadError` or :exc:`ssl.SSLWantWriteError` on a non-blocking socket if the operation would block. Previously, it would return ``0``. (Contributed by Nikolaus Rath in :issue:`20951`.) @@ -2526,9 +2526,9 @@ Changes in the C API to format the :func:`repr` of the object. (Contributed by Serhiy Storchaka in :issue:`22453`.) -* Because the lack of the :attr:`__module__` attribute breaks pickling and +* Because the lack of the :attr:`~type.__module__` attribute breaks pickling and introspection, a deprecation warning is now raised for builtin types without - the :attr:`__module__` attribute. This would be an AttributeError in + the :attr:`~type.__module__` attribute. This will be an :exc:`AttributeError` in the future. (Contributed by Serhiy Storchaka in :issue:`20204`.) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index d62beb0bdc8672..1fcc5d7cbfb387 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -261,7 +261,7 @@ allowed. The :ref:`string formatting ` language also now has support for the ``'_'`` option to signal the use of an underscore for a thousands -separator for floating point presentation types and for integer +separator for floating-point presentation types and for integer presentation type ``'d'``. For integer presentation types ``'b'``, ``'o'``, ``'x'``, and ``'X'``, underscores will be inserted every 4 digits:: @@ -511,10 +511,10 @@ correct. Prior to Python 3.6, data loss could result when using bytes paths on Windows. With this change, using bytes to represent paths is now supported on Windows, provided those bytes are encoded with the encoding returned by -:func:`sys.getfilesystemencoding()`, which now defaults to ``'utf-8'``. +:func:`sys.getfilesystemencoding`, which now defaults to ``'utf-8'``. Applications that do not use str to represent paths should use -:func:`os.fsencode()` and :func:`os.fsdecode()` to ensure their bytes are +:func:`os.fsencode` and :func:`os.fsdecode` to ensure their bytes are correctly encoded. To revert to the previous behaviour, set :envvar:`PYTHONLEGACYWINDOWSFSENCODING` or call :func:`sys._enablelegacywindowsfsencoding`. @@ -549,7 +549,7 @@ PEP 520: Preserving Class Attribute Definition Order Attributes in a class definition body have a natural ordering: the same order in which the names appear in the source. This order is now -preserved in the new class's :attr:`~object.__dict__` attribute. +preserved in the new class's :attr:`~type.__dict__` attribute. Also, the effective default class *execution* namespace (returned from :ref:`type.__prepare__() `) is now an insertion-order-preserving @@ -780,7 +780,7 @@ for managing secrets, such as account authentication, tokens, and similar. Note that the pseudo-random generators in the :mod:`random` module should *NOT* be used for security purposes. Use :mod:`secrets` - on Python 3.6+ and :func:`os.urandom()` on Python 3.5 and earlier. + on Python 3.6+ and :func:`os.urandom` on Python 3.5 and earlier. .. seealso:: @@ -934,7 +934,7 @@ asynchronous generators. The :func:`~collections.namedtuple` function now accepts an optional keyword argument *module*, which, when specified, is used for -the ``__module__`` attribute of the returned named tuple class. +the :attr:`~type.__module__` attribute of the returned named tuple class. (Contributed by Raymond Hettinger in :issue:`17941`.) The *verbose* and *rename* arguments for @@ -1316,7 +1316,7 @@ Storchaka in :issue:`24164`.) pickletools ----------- -:func:`pickletools.dis()` now outputs the implicit memo index for the +:func:`pickletools.dis` now outputs the implicit memo index for the ``MEMOIZE`` opcode. (Contributed by Serhiy Storchaka in :issue:`25382`.) @@ -2336,10 +2336,10 @@ Changes in the Python API * With the introduction of :exc:`ModuleNotFoundError`, import system consumers may start expecting import system replacements to raise that more specific exception when appropriate, rather than the less-specific :exc:`ImportError`. - To provide future compatibility with such consumers, implementors of + To provide future compatibility with such consumers, implementers of alternative import systems that completely replace :func:`__import__` will need to update their implementations to raise the new subclass when a module - can't be found at all. Implementors of compliant plugins to the default + can't be found at all. Implementers of compliant plugins to the default import system shouldn't need to make any changes, as the default import system will raise the new subclass when appropriate. @@ -2366,27 +2366,27 @@ There have been several major changes to the :term:`bytecode` in Python 3.6. (Contributed by Demur Rumed with input and reviews from Serhiy Storchaka and Victor Stinner in :issue:`26647` and :issue:`28050`.) -* The new :opcode:`FORMAT_VALUE` and :opcode:`BUILD_STRING` opcodes as part +* The new :opcode:`!FORMAT_VALUE` and :opcode:`BUILD_STRING` opcodes as part of the :ref:`formatted string literal ` implementation. (Contributed by Eric Smith in :issue:`25483` and Serhiy Storchaka in :issue:`27078`.) -* The new :opcode:`BUILD_CONST_KEY_MAP` opcode to optimize the creation +* The new :opcode:`!BUILD_CONST_KEY_MAP` opcode to optimize the creation of dictionaries with constant keys. (Contributed by Serhiy Storchaka in :issue:`27140`.) * The function call opcodes have been heavily reworked for better performance and simpler implementation. - The :opcode:`MAKE_FUNCTION`, :opcode:`CALL_FUNCTION`, - :opcode:`CALL_FUNCTION_KW` and :opcode:`BUILD_MAP_UNPACK_WITH_CALL` opcodes + The :opcode:`MAKE_FUNCTION`, :opcode:`!CALL_FUNCTION`, + :opcode:`!CALL_FUNCTION_KW` and :opcode:`!BUILD_MAP_UNPACK_WITH_CALL` opcodes have been modified, the new :opcode:`CALL_FUNCTION_EX` and - :opcode:`BUILD_TUPLE_UNPACK_WITH_CALL` have been added, and + :opcode:`!BUILD_TUPLE_UNPACK_WITH_CALL` have been added, and ``CALL_FUNCTION_VAR``, ``CALL_FUNCTION_VAR_KW`` and ``MAKE_CLOSURE`` opcodes have been removed. (Contributed by Demur Rumed in :issue:`27095`, and Serhiy Storchaka in :issue:`27213`, :issue:`28257`.) -* The new :opcode:`SETUP_ANNOTATIONS` and :opcode:`STORE_ANNOTATION` opcodes +* The new :opcode:`SETUP_ANNOTATIONS` and :opcode:`!STORE_ANNOTATION` opcodes have been added to support the new :term:`variable annotation` syntax. (Contributed by Ivan Levkivskyi in :issue:`27985`.) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 8122e0ee129b0d..f420fa5c04479b 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -339,7 +339,7 @@ PEP 564: New Time Functions With Nanosecond Resolution ------------------------------------------------------ The resolution of clocks in modern systems can exceed the limited precision -of a floating point number returned by the :func:`time.time` function +of a floating-point number returned by the :func:`time.time` function and its variants. To avoid loss of precision, :pep:`564` adds six new "nanosecond" variants of the existing timer functions to the :mod:`time` module: @@ -353,7 +353,7 @@ module: The new functions return the number of nanoseconds as an integer value. -`Measurements `_ +:pep:`Measurements <0564#annex-clocks-resolution-in-python>` show that on Linux and Windows the resolution of :func:`time.time_ns` is approximately 3 times better than that of :func:`time.time`. @@ -603,7 +603,7 @@ The new :mod:`importlib.resources` module provides several new APIs and one new ABC for access to, opening, and reading *resources* inside packages. Resources are roughly similar to files inside packages, but they needn't be actual files on the physical file system. Module loaders can provide a -:meth:`get_resource_reader()` function which returns +:meth:`get_resource_reader` function which returns a :class:`importlib.abc.ResourceReader` instance to support this new API. Built-in file path loaders and zip file loaders both support this. @@ -669,8 +669,8 @@ include: * The new :func:`asyncio.current_task` function returns the currently running :class:`~asyncio.Task` instance, and the new :func:`asyncio.all_tasks` function returns a set of all existing ``Task`` instances in a given loop. - The :meth:`Task.current_task() ` and - :meth:`Task.all_tasks() ` methods have been deprecated. + The :meth:`!Task.current_task` and + :meth:`!Task.all_tasks` methods have been deprecated. (Contributed by Andrew Svetlov in :issue:`32250`.) * The new *provisional* :class:`~asyncio.BufferedProtocol` class allows @@ -1133,7 +1133,7 @@ The MIME type of .bmp has been changed from ``'image/x-ms-bmp'`` to msilib ------ -The new :meth:`!Database.Close()` method can be used +The new :meth:`!Database.Close` method can be used to close the :abbr:`MSI` database. (Contributed by Berker Peksag in :issue:`20486`.) @@ -1969,7 +1969,7 @@ asynchronous context manager must be used in order to acquire and release the synchronization resource. (Contributed by Andrew Svetlov in :issue:`32253`.) -The :meth:`asyncio.Task.current_task` and :meth:`asyncio.Task.all_tasks` +The :meth:`!asyncio.Task.current_task` and :meth:`!asyncio.Task.all_tasks` methods have been deprecated. (Contributed by Andrew Svetlov in :issue:`32250`.) @@ -2017,11 +2017,11 @@ importlib --------- Methods -:meth:`!MetaPathFinder.find_module()` +:meth:`!MetaPathFinder.find_module` (replaced by :meth:`MetaPathFinder.find_spec() `) and -:meth:`!PathEntryFinder.find_loader()` +:meth:`!PathEntryFinder.find_loader` (replaced by :meth:`PathEntryFinder.find_spec() `) both deprecated in Python 3.4 now emit :exc:`DeprecationWarning`. @@ -2048,7 +2048,7 @@ The :mod:`macpath` is now deprecated and will be removed in Python 3.8. threading --------- -:mod:`dummy_threading` and :mod:`_dummy_thread` have been deprecated. It is +:mod:`!dummy_threading` and :mod:`!_dummy_thread` have been deprecated. It is no longer possible to build Python with threading disabled. Use :mod:`threading` instead. (Contributed by Antoine Pitrou in :issue:`31370`.) @@ -2184,7 +2184,7 @@ The following features and APIs have been removed from Python 3.7: ``socket.socketpair`` on Python 3.5 and newer. * :mod:`asyncio` no longer exports the :mod:`selectors` and - :mod:`_overlapped` modules as ``asyncio.selectors`` and + :mod:`!_overlapped` modules as ``asyncio.selectors`` and ``asyncio._overlapped``. Replace ``from asyncio import selectors`` with ``import selectors``. @@ -2366,7 +2366,7 @@ Changes in the Python API positions 2--3. To match only blank lines, the pattern should be rewritten as ``r'(?m)^[^\S\n]*$'``. - :func:`re.sub()` now replaces empty matches adjacent to a previous + :func:`re.sub` now replaces empty matches adjacent to a previous non-empty match. For example ``re.sub('x*', '-', 'abxd')`` returns now ``'-a-b--d-'`` instead of ``'-a-b-d-'`` (the first minus between 'b' and 'd' replaces 'x', and the second minus replaces an empty string between @@ -2425,7 +2425,7 @@ Changes in the Python API to :meth:`ArgumentParser.add_subparsers() `. (Contributed by Anthony Sottile in :issue:`26510`.) -* :meth:`ast.literal_eval()` is now stricter. Addition and subtraction of +* :meth:`ast.literal_eval` is now stricter. Addition and subtraction of arbitrary numbers are no longer allowed. (Contributed by Serhiy Storchaka in :issue:`31778`.) @@ -2476,10 +2476,10 @@ avoiding possible problems use new functions :c:func:`PySlice_Unpack` and CPython bytecode changes ------------------------ -There are two new opcodes: :opcode:`LOAD_METHOD` and :opcode:`CALL_METHOD`. +There are two new opcodes: :opcode:`LOAD_METHOD` and :opcode:`!CALL_METHOD`. (Contributed by Yury Selivanov and INADA Naoki in :issue:`26110`.) -The :opcode:`STORE_ANNOTATION` opcode has been removed. +The :opcode:`!STORE_ANNOTATION` opcode has been removed. (Contributed by Mark Shannon in :issue:`32550`.) @@ -2609,8 +2609,7 @@ Converting between :class:`int` and :class:`str` in bases other than 2 (binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) now raises a :exc:`ValueError` if the number of digits in string form is above a limit to avoid potential denial of service attacks due to the -algorithmic complexity. This is a mitigation for `CVE-2020-10735 -`_. +algorithmic complexity. This is a mitigation for :cve:`2020-10735`. This limit can be configured or disabled by environment variable, command line flag, or :mod:`sys` APIs. See the :ref:`integer string conversion length limitation ` documentation. The default limit diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 9a2652f5e33605..7aca35b2959cd2 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -428,9 +428,9 @@ Other Language Changes normal assignment syntax:: >>> def parse(family): - lastname, *members = family.split() - return lastname.upper(), *members - + ... lastname, *members = family.split() + ... return lastname.upper(), *members + ... >>> parse('simpsons homer marge bart lisa maggie') ('SIMPSONS', 'homer', 'marge', 'bart', 'lisa', 'maggie') @@ -936,7 +936,7 @@ Add option ``--json-lines`` to parse every input line as a separate JSON object. logging ------- -Added a *force* keyword argument to :func:`logging.basicConfig()` +Added a *force* keyword argument to :func:`logging.basicConfig`. When set to true, any existing handlers attached to the root logger are removed and closed before carrying out the configuration specified by the other arguments. @@ -1077,16 +1077,16 @@ pathlib ------- :mod:`pathlib.Path` methods that return a boolean result like -:meth:`~pathlib.Path.exists()`, :meth:`~pathlib.Path.is_dir()`, -:meth:`~pathlib.Path.is_file()`, :meth:`~pathlib.Path.is_mount()`, -:meth:`~pathlib.Path.is_symlink()`, :meth:`~pathlib.Path.is_block_device()`, -:meth:`~pathlib.Path.is_char_device()`, :meth:`~pathlib.Path.is_fifo()`, -:meth:`~pathlib.Path.is_socket()` now return ``False`` instead of raising +:meth:`~pathlib.Path.exists`, :meth:`~pathlib.Path.is_dir`, +:meth:`~pathlib.Path.is_file`, :meth:`~pathlib.Path.is_mount`, +:meth:`~pathlib.Path.is_symlink`, :meth:`~pathlib.Path.is_block_device`, +:meth:`~pathlib.Path.is_char_device`, :meth:`~pathlib.Path.is_fifo`, +:meth:`~pathlib.Path.is_socket` now return ``False`` instead of raising :exc:`ValueError` or its subclass :exc:`UnicodeEncodeError` for paths that contain characters unrepresentable at the OS level. (Contributed by Serhiy Storchaka in :issue:`33721`.) -Added :meth:`!pathlib.Path.link_to()` which creates a hard link pointing +Added :meth:`!pathlib.Path.link_to` which creates a hard link pointing to a path. (Contributed by Joannah Nanjekye in :issue:`26978`) Note that ``link_to`` was deprecated in 3.10 and removed in 3.12 in @@ -1170,13 +1170,13 @@ recursively removing their contents first. socket ------ -Added :meth:`~socket.create_server()` and :meth:`~socket.has_dualstack_ipv6()` +Added :meth:`~socket.create_server` and :meth:`~socket.has_dualstack_ipv6` convenience functions to automate the necessary tasks usually involved when creating a server socket, including accepting both IPv4 and IPv6 connections on the same socket. (Contributed by Giampaolo Rodolà in :issue:`17561`.) -The :func:`socket.if_nameindex()`, :func:`socket.if_nametoindex()`, and -:func:`socket.if_indextoname()` functions have been implemented on Windows. +The :func:`socket.if_nameindex`, :func:`socket.if_nametoindex`, and +:func:`socket.if_indextoname` functions have been implemented on Windows. (Contributed by Zackery Spytz in :issue:`37007`.) @@ -1192,11 +1192,11 @@ post-handshake authentication. statistics ---------- -Added :func:`statistics.fmean` as a faster, floating point variant of -:func:`statistics.mean()`. (Contributed by Raymond Hettinger and +Added :func:`statistics.fmean` as a faster, floating-point variant of +:func:`statistics.mean`. (Contributed by Raymond Hettinger and Steven D'Aprano in :issue:`35904`.) -Added :func:`statistics.geometric_mean()` +Added :func:`statistics.geometric_mean` (Contributed by Raymond Hettinger in :issue:`27181`.) Added :func:`statistics.multimode` that returns a list of the most @@ -1367,10 +1367,10 @@ Added :class:`~unittest.mock.AsyncMock` to support an asynchronous version of have been added as well. (Contributed by Lisa Roach in :issue:`26467`). -Added :func:`~unittest.addModuleCleanup()` and -:meth:`~unittest.TestCase.addClassCleanup()` to unittest to support -cleanups for :func:`~unittest.setUpModule()` and -:meth:`~unittest.TestCase.setUpClass()`. +Added :func:`~unittest.addModuleCleanup` and +:meth:`~unittest.TestCase.addClassCleanup` to unittest to support +cleanups for :func:`~unittest.setUpModule` and +:meth:`~unittest.TestCase.setUpClass`. (Contributed by Lisa Roach in :issue:`24412`.) Several mock assert functions now also print a list of actual calls upon @@ -1432,7 +1432,7 @@ and ``{namespace}*`` which returns all tags in the given namespace. (Contributed by Stefan Behnel in :issue:`28238`.) The :mod:`xml.etree.ElementTree` module provides a new function -:func:`–xml.etree.ElementTree.canonicalize()` that implements C14N 2.0. +:func:`–xml.etree.ElementTree.canonicalize` that implements C14N 2.0. (Contributed by Stefan Behnel in :issue:`13611`.) The target object of :class:`xml.etree.ElementTree.XMLParser` can @@ -1712,7 +1712,7 @@ Deprecated the ``l*gettext()`` functions. (Contributed by Serhiy Storchaka in :issue:`33710`.) -* The :meth:`~threading.Thread.isAlive()` method of :class:`threading.Thread` +* The :meth:`~threading.Thread.isAlive` method of :class:`threading.Thread` has been deprecated. (Contributed by Donghee Na in :issue:`35283`.) @@ -1755,7 +1755,7 @@ The following features and APIs have been removed from Python 3.8: * Starting with Python 3.3, importing ABCs from :mod:`collections` was deprecated, and importing should be done from :mod:`collections.abc`. Being able to import from collections was marked for removal in 3.8, but has been - delayed to 3.9. (See :issue:`36952`.) + delayed to 3.9. (See :gh:`81134`.) * The :mod:`macpath` module, deprecated in Python 3.7, has been removed. (Contributed by Victor Stinner in :issue:`35471`.) @@ -2152,11 +2152,11 @@ CPython bytecode changes cleaning-up code for :keyword:`break`, :keyword:`continue` and :keyword:`return`. - Removed opcodes :opcode:`BREAK_LOOP`, :opcode:`CONTINUE_LOOP`, - :opcode:`SETUP_LOOP` and :opcode:`SETUP_EXCEPT`. Added new opcodes - :opcode:`ROT_FOUR`, :opcode:`BEGIN_FINALLY`, :opcode:`CALL_FINALLY` and - :opcode:`POP_FINALLY`. Changed the behavior of :opcode:`END_FINALLY` - and :opcode:`WITH_CLEANUP_START`. + Removed opcodes :opcode:`!BREAK_LOOP`, :opcode:`!CONTINUE_LOOP`, + :opcode:`!SETUP_LOOP` and :opcode:`!SETUP_EXCEPT`. Added new opcodes + :opcode:`!ROT_FOUR`, :opcode:`!BEGIN_FINALLY`, :opcode:`!CALL_FINALLY` and + :opcode:`!POP_FINALLY`. Changed the behavior of :opcode:`!END_FINALLY` + and :opcode:`!WITH_CLEANUP_START`. (Contributed by Mark Shannon, Antoine Pitrou and Serhiy Storchaka in :issue:`17611`.) @@ -2337,8 +2337,7 @@ Converting between :class:`int` and :class:`str` in bases other than 2 (binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) now raises a :exc:`ValueError` if the number of digits in string form is above a limit to avoid potential denial of service attacks due to the -algorithmic complexity. This is a mitigation for `CVE-2020-10735 -`_. +algorithmic complexity. This is a mitigation for :cve:`2020-10735`. This limit can be configured or disabled by environment variable, command line flag, or :mod:`sys` APIs. See the :ref:`integer string conversion length limitation ` documentation. The default limit diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 49d926b0edcd0f..b062e6b4c9bca0 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -81,13 +81,13 @@ Interpreter improvements: * a number of Python builtins (range, tuple, set, frozenset, list, dict) are now sped up using :pep:`590` vectorcall; * garbage collection does not block on resurrected objects; -* a number of Python modules (:mod:`_abc`, :mod:`!audioop`, :mod:`_bz2`, - :mod:`_codecs`, :mod:`_contextvars`, :mod:`!_crypt`, :mod:`_functools`, - :mod:`_json`, :mod:`_locale`, :mod:`math`, :mod:`operator`, :mod:`resource`, - :mod:`time`, :mod:`_weakref`) now use multiphase initialization as defined +* a number of Python modules (:mod:`!_abc`, :mod:`!audioop`, :mod:`!_bz2`, + :mod:`!_codecs`, :mod:`!_contextvars`, :mod:`!_crypt`, :mod:`!_functools`, + :mod:`!_json`, :mod:`!_locale`, :mod:`math`, :mod:`operator`, :mod:`resource`, + :mod:`time`, :mod:`!_weakref`) now use multiphase initialization as defined by PEP 489; * a number of standard library modules (:mod:`!audioop`, :mod:`ast`, :mod:`grp`, - :mod:`_hashlib`, :mod:`pwd`, :mod:`_posixsubprocess`, :mod:`random`, + :mod:`!_hashlib`, :mod:`pwd`, :mod:`!_posixsubprocess`, :mod:`random`, :mod:`select`, :mod:`struct`, :mod:`termios`, :mod:`zlib`) are now using the stable ABI defined by PEP 384. @@ -203,7 +203,7 @@ The :mod:`ast` module uses the new parser and produces the same AST as the old parser. In Python 3.10, the old parser will be deleted and so will all -functionality that depends on it (primarily the :mod:`parser` module, +functionality that depends on it (primarily the :mod:`!parser` module, which has long been deprecated). In Python 3.9 *only*, you can switch back to the LL(1) parser using a command line switch (``-X oldparser``) or an environment variable (``PYTHONOLDPARSER=1``). @@ -300,12 +300,9 @@ Example:: As a fall-back source of data for platforms that don't ship the IANA database, -the |tzdata|_ module was released as a first-party package -- distributed via +the :pypi:`tzdata` module was released as a first-party package -- distributed via PyPI and maintained by the CPython core team. -.. |tzdata| replace:: ``tzdata`` -.. _tzdata: https://pypi.org/project/tzdata/ - .. seealso:: :pep:`615` -- Support for the IANA Time Zone Database in the Standard Library @@ -355,7 +352,7 @@ that schedules a shutdown for the default executor that waits on the :func:`asyncio.run` has been updated to use the new :term:`coroutine`. (Contributed by Kyle Stanley in :issue:`34037`.) -Added :class:`asyncio.PidfdChildWatcher`, a Linux-specific child watcher +Added :class:`!asyncio.PidfdChildWatcher`, a Linux-specific child watcher implementation that polls process file descriptors. (:issue:`38692`) Added a new :term:`coroutine` :func:`asyncio.to_thread`. It is mainly used for @@ -369,7 +366,7 @@ wait until the cancellation is complete also in the case when *timeout* is <= 0, like it does with positive timeouts. (Contributed by Elvis Pranskevichus in :issue:`32751`.) -:mod:`asyncio` now raises :exc:`TyperError` when calling incompatible +:mod:`asyncio` now raises :exc:`TypeError` when calling incompatible methods with an :class:`ssl.SSLSocket` socket. (Contributed by Ido Michael in :issue:`37404`.) @@ -411,8 +408,8 @@ Added :func:`curses.get_escdelay`, :func:`curses.set_escdelay`, datetime -------- -The :meth:`~datetime.date.isocalendar()` of :class:`datetime.date` -and :meth:`~datetime.datetime.isocalendar()` of :class:`datetime.datetime` +The :meth:`~datetime.date.isocalendar` of :class:`datetime.date` +and :meth:`~datetime.datetime.isocalendar` of :class:`datetime.datetime` methods now returns a :func:`~collections.namedtuple` instead of a :class:`tuple`. (Contributed by Donghee Na in :issue:`24416`.) @@ -592,7 +589,7 @@ a non-blocking socket. (Contributed by Donghee Na in :issue:`39259`.) os -- -Added :const:`~os.CLD_KILLED` and :const:`~os.CLD_STOPPED` for :attr:`si_code`. +Added :const:`~os.CLD_KILLED` and :const:`~os.CLD_STOPPED` for :attr:`!si_code`. (Contributed by Donghee Na in :issue:`38493`.) Exposed the Linux-specific :func:`os.pidfd_open` (:issue:`38692`) and @@ -613,7 +610,7 @@ convert a wait status to an exit code. pathlib ------- -Added :meth:`pathlib.Path.readlink()` which acts similarly to +Added :meth:`pathlib.Path.readlink` which acts similarly to :func:`os.readlink`. (Contributed by Girts Folkmanis in :issue:`30618`) @@ -640,7 +637,8 @@ pydoc ----- The documentation string is now shown not only for class, function, -method etc, but for any object that has its own ``__doc__`` attribute. +method etc, but for any object that has its own :attr:`~definition.__doc__` +attribute. (Contributed by Serhiy Storchaka in :issue:`40257`.) random @@ -864,7 +862,7 @@ Deprecated Python versions it will raise a :exc:`TypeError` for all floats. (Contributed by Serhiy Storchaka in :issue:`37315`.) -* The :mod:`parser` and :mod:`symbol` modules are deprecated and will be +* The :mod:`!parser` and :mod:`!symbol` modules are deprecated and will be removed in future versions of Python. For the majority of use cases, users can leverage the Abstract Syntax Tree (AST) generation and compilation stage, using the :mod:`ast` module. @@ -892,20 +890,20 @@ Deprecated it for writing and silencing a warning. (Contributed by Serhiy Storchaka in :issue:`28286`.) -* Deprecated the ``split()`` method of :class:`_tkinter.TkappType` in +* Deprecated the ``split()`` method of :class:`!_tkinter.TkappType` in favour of the ``splitlist()`` method which has more consistent and - predicable behavior. + predictable behavior. (Contributed by Serhiy Storchaka in :issue:`38371`.) * The explicit passing of coroutine objects to :func:`asyncio.wait` has been deprecated and will be removed in version 3.11. (Contributed by Yury Selivanov and Kyle Stanley in :issue:`34790`.) -* binhex4 and hexbin4 standards are now deprecated. The :mod:`binhex` module +* binhex4 and hexbin4 standards are now deprecated. The :mod:`!binhex` module and the following :mod:`binascii` functions are now deprecated: - * :func:`~binascii.b2a_hqx`, :func:`~binascii.a2b_hqx` - * :func:`~binascii.rlecode_hqx`, :func:`~binascii.rledecode_hqx` + * :func:`!b2a_hqx`, :func:`!a2b_hqx` + * :func:`!rlecode_hqx`, :func:`!rledecode_hqx` (Contributed by Victor Stinner in :issue:`39353`.) @@ -953,7 +951,7 @@ Deprecated Removed ======= -* The erroneous version at :data:`unittest.mock.__version__` has been removed. +* The erroneous version at :data:`!unittest.mock.__version__` has been removed. * :class:`!nntplib.NNTP`: ``xpath()`` and ``xgtitle()`` methods have been removed. These methods are deprecated since Python 3.3. Generally, these extensions @@ -986,13 +984,13 @@ Removed (Contributed by Victor Stinner in :issue:`37312`.) * ``aifc.openfp()`` alias to ``aifc.open()``, ``sunau.openfp()`` alias to - ``sunau.open()``, and ``wave.openfp()`` alias to :func:`wave.open()` have been + ``sunau.open()``, and ``wave.openfp()`` alias to :func:`wave.open` have been removed. They were deprecated since Python 3.7. (Contributed by Victor Stinner in :issue:`37320`.) -* The :meth:`~threading.Thread.isAlive()` method of :class:`threading.Thread` +* The :meth:`!isAlive` method of :class:`threading.Thread` has been removed. It was deprecated since Python 3.8. - Use :meth:`~threading.Thread.is_alive()` instead. + Use :meth:`~threading.Thread.is_alive` instead. (Contributed by Donghee Na in :issue:`37804`.) * Methods ``getchildren()`` and ``getiterator()`` of classes @@ -1038,7 +1036,7 @@ Removed ``asyncio.Condition`` and ``asyncio.Semaphore``. (Contributed by Andrew Svetlov in :issue:`34793`.) -* The :func:`sys.getcounts` function, the ``-X showalloccount`` command line +* The :func:`!sys.getcounts` function, the ``-X showalloccount`` command line option and the ``show_alloc_count`` field of the C structure :c:type:`PyConfig` have been removed. They required a special Python build by defining ``COUNT_ALLOCS`` macro. @@ -1049,11 +1047,11 @@ Removed the ``__annotations__`` attribute instead. (Contributed by Serhiy Storchaka in :issue:`40182`.) -* The :meth:`symtable.SymbolTable.has_exec` method has been removed. It was +* The :meth:`!symtable.SymbolTable.has_exec` method has been removed. It was deprecated since 2006, and only returning ``False`` when it's called. (Contributed by Batuhan Taskaya in :issue:`40208`) -* The :meth:`asyncio.Task.current_task` and :meth:`asyncio.Task.all_tasks` +* The :meth:`!asyncio.Task.current_task` and :meth:`!asyncio.Task.all_tasks` have been removed. They were deprecated since Python 3.7 and you can use :func:`asyncio.current_task` and :func:`asyncio.all_tasks` instead. (Contributed by Rémi Lapeyre in :issue:`40967`) @@ -1205,7 +1203,7 @@ Changes in the C API CPython bytecode changes ------------------------ -* The :opcode:`LOAD_ASSERTION_ERROR` opcode was added for handling the +* The :opcode:`!LOAD_ASSERTION_ERROR` opcode was added for handling the :keyword:`assert` statement. Previously, the assert statement would not work correctly if the :exc:`AssertionError` exception was being shadowed. (Contributed by Zackery Spytz in :issue:`34880`.) @@ -1233,7 +1231,7 @@ Build Changes * The ``COUNT_ALLOCS`` special build macro has been removed. (Contributed by Victor Stinner in :issue:`39489`.) -* On non-Windows platforms, the :c:func:`setenv` and :c:func:`unsetenv` +* On non-Windows platforms, the :c:func:`!setenv` and :c:func:`!unsetenv` functions are now required to build Python. (Contributed by Victor Stinner in :issue:`39395`.) @@ -1322,7 +1320,7 @@ New Features the garbage collector respectively. (Contributed by Pablo Galindo Salgado in :issue:`40241`.) -* Added :c:func:`_PyObject_FunctionStr` to get a user-friendly string +* Added :c:func:`!_PyObject_FunctionStr` to get a user-friendly string representation of a function-like object. (Patch by Jeroen Demeyer in :issue:`37645`.) @@ -1364,7 +1362,7 @@ Porting to Python 3.9 and refers to a constant string. (Contributed by Serhiy Storchaka in :issue:`38650`.) -* The :c:type:`PyGC_Head` structure is now opaque. It is only defined in the +* The :c:type:`!PyGC_Head` structure is now opaque. It is only defined in the internal C API (``pycore_gc.h``). (Contributed by Victor Stinner in :issue:`40241`.) @@ -1387,12 +1385,12 @@ Porting to Python 3.9 * :c:func:`PyObject_IS_GC` macro was converted to a function. - * The :c:func:`PyObject_NEW` macro becomes an alias to the - :c:macro:`PyObject_New` macro, and the :c:func:`PyObject_NEW_VAR` macro + * The :c:func:`!PyObject_NEW` macro becomes an alias to the + :c:macro:`PyObject_New` macro, and the :c:func:`!PyObject_NEW_VAR` macro becomes an alias to the :c:macro:`PyObject_NewVar` macro. They no longer access directly the :c:member:`PyTypeObject.tp_basicsize` member. - * :c:func:`PyObject_GET_WEAKREFS_LISTPTR` macro was converted to a function: + * :c:func:`!PyObject_GET_WEAKREFS_LISTPTR` macro was converted to a function: the macro accessed directly the :c:member:`PyTypeObject.tp_weaklistoffset` member. @@ -1592,8 +1590,7 @@ Converting between :class:`int` and :class:`str` in bases other than 2 (binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) now raises a :exc:`ValueError` if the number of digits in string form is above a limit to avoid potential denial of service attacks due to the -algorithmic complexity. This is a mitigation for `CVE-2020-10735 -`_. +algorithmic complexity. This is a mitigation for :cve:`2020-10735`. This limit can be configured or disabled by environment variable, command line flag, or :mod:`sys` APIs. See the :ref:`integer string conversion length limitation ` documentation. The default limit diff --git a/Doc/whatsnew/index.rst b/Doc/whatsnew/index.rst index b9c19602653219..6ff722a1894585 100644 --- a/Doc/whatsnew/index.rst +++ b/Doc/whatsnew/index.rst @@ -11,6 +11,7 @@ anyone wishing to stay up-to-date after a new release. .. toctree:: :maxdepth: 2 + 3.14.rst 3.13.rst 3.12.rst 3.11.rst @@ -34,8 +35,8 @@ anyone wishing to stay up-to-date after a new release. 2.1.rst 2.0.rst -The "Changelog" is an HTML version of the `file built -`_ from the contents of the +The "Changelog" is an HTML version of the :pypi:`file built` +from the contents of the :source:`Misc/NEWS.d` directory tree, which contains *all* nontrivial changes to Python for the current version. diff --git a/Grammar/python.gram b/Grammar/python.gram index 797c195a0a91ba..b47028460b94f4 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -78,6 +78,9 @@ _PyPegen_parse(Parser *p) # Fail if e can be parsed, without consuming any input. # ~ # Commit to the current alternative, even if it fails to parse. +# &&e +# Eager parse e. The parser will not backtrack and will immediately +# fail with SyntaxError if e cannot be parsed. # # STARTING RULES @@ -124,7 +127,6 @@ simple_stmt[stmt_ty] (memo): | &'nonlocal' nonlocal_stmt compound_stmt[stmt_ty]: - | invalid_compound_stmt | &('def' | '@' | 'async') function_def | &'if' if_stmt | &('class' | '@') class_def @@ -149,9 +151,9 @@ assignment[stmt_ty]: | a=('(' b=single_target ')' { b } | single_subscript_attribute_target) ':' b=expression c=['=' d=annotated_rhs { d }] { CHECK_VERSION(stmt_ty, 6, "Variable annotations syntax is", _PyAST_AnnAssign(a, b, c, 0, EXTRA)) } - | a[asdl_expr_seq*]=(z=star_targets '=' { z })+ b=(yield_expr | star_expressions) !'=' tc=[TYPE_COMMENT] { + | a[asdl_expr_seq*]=(z=star_targets '=' { z })+ b=annotated_rhs !'=' tc=[TYPE_COMMENT] { _PyAST_Assign(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } - | a=single_target b=augassign ~ c=(yield_expr | star_expressions) { + | a=single_target b=augassign ~ c=annotated_rhs { _PyAST_AugAssign(a, b->kind, c, EXTRA) } | invalid_assignment @@ -266,11 +268,11 @@ function_def[stmt_ty]: function_def_raw[stmt_ty]: | invalid_def_raw - | 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block { + | 'def' n=NAME t=[type_params] '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block { _PyAST_FunctionDef(n->v.Name.id, (params) ? params : CHECK(arguments_ty, _PyPegen_empty_arguments(p)), b, NULL, a, NEW_TYPE_COMMENT(p, tc), t, EXTRA) } - | 'async' 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block { + | 'async' 'def' n=NAME t=[type_params] '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block { CHECK_VERSION( stmt_ty, 5, @@ -432,7 +434,7 @@ except_star_block[excepthandler_ty]: | invalid_except_star_stmt_indent | 'except' '*' e=expression t=['as' z=NAME { z }] ':' b=block { _PyAST_ExceptHandler(e, (t) ? ((expr_ty) t)->v.Name.id : NULL, b, EXTRA) } - | invalid_except_stmt + | invalid_except_star_stmt finally_block[asdl_stmt_seq*]: | invalid_finally_stmt | 'finally' &&':' a=block { a } @@ -638,27 +640,24 @@ type_alias[stmt_ty]: # Type parameter declaration # -------------------------- -type_params[asdl_type_param_seq*]: '[' t=type_param_seq ']' { +type_params[asdl_type_param_seq*]: + | invalid_type_params + | '[' t=type_param_seq ']' { CHECK_VERSION(asdl_type_param_seq *, 12, "Type parameter lists are", t) } type_param_seq[asdl_type_param_seq*]: a[asdl_type_param_seq*]=','.type_param+ [','] { a } type_param[type_param_ty] (memo): - | a=NAME b=[type_param_bound] { _PyAST_TypeVar(a->v.Name.id, b, EXTRA) } - | '*' a=NAME colon=':' e=expression { - RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind - ? "cannot use constraints with TypeVarTuple" - : "cannot use bound with TypeVarTuple") - } - | '*' a=NAME { _PyAST_TypeVarTuple(a->v.Name.id, EXTRA) } - | '**' a=NAME colon=':' e=expression { - RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind - ? "cannot use constraints with ParamSpec" - : "cannot use bound with ParamSpec") - } - | '**' a=NAME { _PyAST_ParamSpec(a->v.Name.id, EXTRA) } + | a=NAME b=[type_param_bound] c=[type_param_default] { _PyAST_TypeVar(a->v.Name.id, b, c, EXTRA) } + | invalid_type_param + | '*' a=NAME b=[type_param_starred_default] { _PyAST_TypeVarTuple(a->v.Name.id, b, EXTRA) } + | '**' a=NAME b=[type_param_default] { _PyAST_ParamSpec(a->v.Name.id, b, EXTRA) } type_param_bound[expr_ty]: ':' e=expression { e } +type_param_default[expr_ty]: '=' e=expression { + CHECK_VERSION(expr_ty, 13, "Type parameter defaults are", e) } +type_param_starred_default[expr_ty]: '=' e=star_expression { + CHECK_VERSION(expr_ty, 13, "Type parameter defaults are", e) } # EXPRESSIONS # ----------- @@ -778,6 +777,7 @@ bitwise_and[expr_ty]: shift_expr[expr_ty]: | a=shift_expr '<<' b=sum { _PyAST_BinOp(a, LShift, b, EXTRA) } | a=shift_expr '>>' b=sum { _PyAST_BinOp(a, RShift, b, EXTRA) } + | invalid_arithmetic | sum # Arithmetic operators @@ -794,6 +794,7 @@ term[expr_ty]: | a=term '//' b=factor { _PyAST_BinOp(a, FloorDiv, b, EXTRA) } | a=term '%' b=factor { _PyAST_BinOp(a, Mod, b, EXTRA) } | a=term '@' b=factor { CHECK_VERSION(expr_ty, 5, "The '@' operator is", _PyAST_BinOp(a, MatMult, b, EXTRA)) } + | invalid_factor | factor factor[expr_ty] (memo): @@ -913,7 +914,7 @@ fstring_middle[expr_ty]: | fstring_replacement_field | t=FSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) } fstring_replacement_field[expr_ty]: - | '{' a=(yield_expr | star_expressions) debug_expr='='? conversion=[fstring_conversion] format=[fstring_full_format_spec] rbrace='}' { + | '{' a=annotated_rhs debug_expr='='? conversion=[fstring_conversion] format=[fstring_full_format_spec] rbrace='}' { _PyPegen_formatted_value(p, a, debug_expr, conversion, format, rbrace, EXTRA) } | invalid_replacement_field fstring_conversion[ResultTokenWithMetadata*]: @@ -968,8 +969,7 @@ for_if_clause[comprehension_ty]: CHECK_VERSION(comprehension_ty, 6, "Async comprehensions are", _PyAST_comprehension(a, b, c, 1, p->arena)) } | 'for' a=star_targets 'in' ~ b=disjunction c[asdl_expr_seq*]=('if' z=disjunction { z })* { _PyAST_comprehension(a, b, c, 0, p->arena) } - | 'async'? 'for' (bitwise_or (',' bitwise_or)* [',']) !'in' { - RAISE_SYNTAX_ERROR("'in' expected after for-loop variables") } + | invalid_for_if_clause | invalid_for_target listcomp[expr_ty]: @@ -1009,8 +1009,9 @@ kwargs[asdl_seq*]: | ','.kwarg_or_double_starred+ starred_expression[expr_ty]: - | invalid_starred_expression + | invalid_starred_expression_unpacking | '*' a=expression { _PyAST_Starred(a, Load, EXTRA) } + | invalid_starred_expression kwarg_or_starred[KeywordOrStarred*]: | invalid_kwarg @@ -1131,8 +1132,8 @@ func_type_comment[Token*]: # From here on, there are rules for invalid syntax with specialised error messages invalid_arguments: - | ((','.(starred_expression | ( assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' b='*' { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(b, "iterable argument unpacking follows keyword argument unpacking") } + | ((','.(starred_expression | ( assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) a=',' ','.(starred_expression !'=')+ { + RAISE_SYNTAX_ERROR_STARTING_FROM(a, "iterable argument unpacking follows keyword argument unpacking") } | a=expression b=for_if_clauses ',' [args | expression for_if_clauses] { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, _PyPegen_get_last_comprehension_item(PyPegen_last_item(b, comprehension_ty)), "Generator expression must be parenthesized") } | a=NAME b='=' expression for_if_clauses { @@ -1164,6 +1165,18 @@ invalid_legacy_expression: _PyPegen_check_legacy_stmt(p, a) ? RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "Missing parentheses in call to '%U'. Did you mean %U(...)?", a->v.Name.id, a->v.Name.id) : NULL} +invalid_type_param: + | '*' a=NAME colon=':' e=expression { + RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind + ? "cannot use constraints with TypeVarTuple" + : "cannot use bound with TypeVarTuple") + } + | '**' a=NAME colon=':' e=expression { + RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind + ? "cannot use constraints with ParamSpec" + : "cannot use bound with ParamSpec") + } + invalid_expression: # !(NAME STRING) is not matched so we don't show this error with some invalid string prefixes like: kf"dsfsdf" # Soft keywords need to also be ignored because they can be parsed as NAME NAME @@ -1198,7 +1211,7 @@ invalid_assignment: | (star_targets '=')* a=star_expressions '=' { RAISE_SYNTAX_ERROR_INVALID_TARGET(STAR_TARGETS, a) } | (star_targets '=')* a=yield_expr '=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "assignment to yield expression not possible") } - | a=star_expressions augassign (yield_expr | star_expressions) { + | a=star_expressions augassign annotated_rhs { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( a, "'%s' is an illegal expression for augmented assignment", @@ -1284,6 +1297,10 @@ invalid_with_item: | expression 'as' a=expression &(',' | ')' | ':') { RAISE_SYNTAX_ERROR_INVALID_TARGET(STAR_TARGETS, a) } +invalid_for_if_clause: + | 'async'? 'for' (bitwise_or (',' bitwise_or)* [',']) !'in' { + RAISE_SYNTAX_ERROR("'in' expected after for-loop variables") } + invalid_for_target: | 'async'? 'for' a=star_expressions { RAISE_SYNTAX_ERROR_INVALID_TARGET(FOR_TARGETS, a) } @@ -1296,14 +1313,14 @@ invalid_group: invalid_import: | a='import' ','.dotted_name+ 'from' dotted_name { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "Did you mean to use 'from ... import ...' instead?") } + | 'import' token=NEWLINE { + RAISE_SYNTAX_ERROR_STARTING_FROM(token, "Expected one or more names after 'import'") } invalid_import_from_targets: | import_from_as_names ',' NEWLINE { RAISE_SYNTAX_ERROR("trailing comma not allowed without surrounding parentheses") } - -invalid_compound_stmt: - | a='elif' named_expression ':' { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "'elif' must match an if-statement here") } - | a='else' ':' { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "'else' must match a valid statement here") } + | token=NEWLINE { + RAISE_SYNTAX_ERROR_STARTING_FROM(token, "Expected one or more names after 'import'") } invalid_with_stmt: | ['async'] 'with' ','.(expression ['as' star_target])+ NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } @@ -1323,11 +1340,21 @@ invalid_try_stmt: | 'try' ':' block* except_star_block+ a='except' [expression ['as' NAME]] ':' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot have both 'except' and 'except*' on the same 'try'") } invalid_except_stmt: - | 'except' '*'? a=expression ',' expressions ['as' NAME ] ':' { + | 'except' a=expression ',' expressions ['as' NAME ] ':' { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "multiple exception types must be parenthesized") } - | a='except' '*'? expression ['as' NAME ] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } + | a='except' expression ['as' NAME ] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } | a='except' NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } + | 'except' expression 'as' a=expression { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + a, "cannot use except statement with %s", _PyPegen_get_expr_name(a)) } +invalid_except_star_stmt: + | 'except' '*' a=expression ',' expressions ['as' NAME ] ':' { + RAISE_SYNTAX_ERROR_STARTING_FROM(a, "multiple exception types must be parenthesized") } + | a='except' '*' expression ['as' NAME ] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } | a='except' '*' (NEWLINE | ':') { RAISE_SYNTAX_ERROR("expected one or more exception types") } + | 'except' '*' expression 'as' a=expression { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + a, "cannot use except* statement with %s", _PyPegen_get_expr_name(a)) } invalid_finally_stmt: | a='finally' ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after 'finally' statement on line %d", a->lineno) } @@ -1348,7 +1375,9 @@ invalid_case_block: RAISE_INDENTATION_ERROR("expected an indented block after 'case' statement on line %d", a->lineno) } invalid_as_pattern: | or_pattern 'as' a="_" { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use '_' as a target") } - | or_pattern 'as' !NAME a=expression { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "invalid pattern target") } + | or_pattern 'as' a=expression { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + a, "cannot use %s as pattern target", _PyPegen_get_expr_name(a)) } invalid_class_pattern: | name_or_attr '(' a=invalid_class_argument_pattern { RAISE_SYNTAX_ERROR_KNOWN_RANGE( PyPegen_first_item(a, pattern_ty), @@ -1378,6 +1407,7 @@ invalid_for_stmt: invalid_def_raw: | ['async'] a='def' NAME [type_params] '(' [params] ')' ['->' expression] ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after function definition on line %d", a->lineno) } + | ['async'] 'def' NAME [type_params] &&'(' [params] ')' ['->' expression] &&':' [func_type_comment] block invalid_class_def_raw: | 'class' NAME [type_params] ['(' [arguments] ')'] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } | a='class' NAME [type_params] ['(' [arguments] ')'] ':' NEWLINE !INDENT { @@ -1392,26 +1422,40 @@ invalid_kvpair: RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, a->lineno, a->end_col_offset - 1, a->end_lineno, -1, "':' expected after dictionary key") } | expression ':' a='*' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use a starred expression in a dictionary value") } | expression a=':' &('}'|',') {RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") } -invalid_starred_expression: +invalid_starred_expression_unpacking: | a='*' expression '=' b=expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to iterable argument unpacking") } +invalid_starred_expression: + | '*' { RAISE_SYNTAX_ERROR("Invalid star expression") } + invalid_replacement_field: | '{' a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '='") } | '{' a='!' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '!'") } | '{' a=':' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before ':'") } | '{' a='}' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '}'") } - | '{' !(yield_expr | star_expressions) { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting a valid expression after '{'")} - | '{' (yield_expr | star_expressions) !('=' | '!' | ':' | '}') { + | '{' !annotated_rhs { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting a valid expression after '{'")} + | '{' annotated_rhs !('=' | '!' | ':' | '}') { PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '=', or '!', or ':', or '}'") } - | '{' (yield_expr | star_expressions) '=' !('!' | ':' | '}') { + | '{' annotated_rhs '=' !('!' | ':' | '}') { PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '!', or ':', or '}'") } - | '{' (yield_expr | star_expressions) '='? invalid_conversion_character - | '{' (yield_expr | star_expressions) '='? ['!' NAME] !(':' | '}') { + | '{' annotated_rhs '='? invalid_conversion_character + | '{' annotated_rhs '='? ['!' NAME] !(':' | '}') { PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting ':' or '}'") } - | '{' (yield_expr | star_expressions) '='? ['!' NAME] ':' fstring_format_spec* !'}' { + | '{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}' { PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '}', or format specs") } - | '{' (yield_expr | star_expressions) '='? ['!' NAME] !'}' { + | '{' annotated_rhs '='? ['!' NAME] !'}' { PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '}'") } invalid_conversion_character: | '!' &(':' | '}') { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: missing conversion character") } | '!' !NAME { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: invalid conversion character") } + +invalid_arithmetic: + | sum ('+'|'-'|'*'|'/'|'%'|'//'|'@') a='not' b=inversion { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") } +invalid_factor: + | ('+' | '-' | '~') a='not' b=factor { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") } + +invalid_type_params: + | '[' token=']' { + RAISE_SYNTAX_ERROR_STARTING_FROM( + token, + "Type parameter list cannot be empty")} diff --git a/Include/Python.h b/Include/Python.h index 01fc45137a17bb..64be80145890a3 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -45,6 +45,19 @@ # endif #endif +// gh-111506: The free-threaded build is not compatible with the limited API +// or the stable ABI. +#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) +# error "The limited API is not currently supported in the free-threaded build" +#endif + +#if defined(Py_GIL_DISABLED) && defined(_MSC_VER) +# include // __readgsqword() +#endif + +#if defined(Py_GIL_DISABLED) && defined(__MINGW32__) +# include // __readgsqword() +#endif // Include Python header files #include "pyport.h" @@ -55,7 +68,10 @@ #include "pybuffer.h" #include "pystats.h" #include "pyatomic.h" +#include "lock.h" +#include "critical_section.h" #include "object.h" +#include "refcount.h" #include "objimpl.h" #include "typeslots.h" #include "pyhash.h" @@ -63,6 +79,7 @@ #include "bytearrayobject.h" #include "bytesobject.h" #include "unicodeobject.h" +#include "pyerrors.h" #include "longobject.h" #include "cpython/longintrepr.h" #include "boolobject.h" @@ -78,6 +95,7 @@ #include "setobject.h" #include "methodobject.h" #include "moduleobject.h" +#include "monitoring.h" #include "cpython/funcobject.h" #include "cpython/classobject.h" #include "fileobject.h" @@ -99,7 +117,6 @@ #include "cpython/picklebufobject.h" #include "cpython/pytime.h" #include "codecs.h" -#include "pyerrors.h" #include "pythread.h" #include "cpython/context.h" #include "modsupport.h" @@ -108,6 +125,7 @@ #include "pylifecycle.h" #include "ceval.h" #include "sysmodule.h" +#include "audit.h" #include "osmodule.h" #include "intrcheck.h" #include "import.h" @@ -119,6 +137,5 @@ #include "fileutils.h" #include "cpython/pyfpe.h" #include "cpython/tracemalloc.h" -#include "cpython/optimizer.h" #endif /* !Py_PYTHON_H */ diff --git a/Include/abstract.h b/Include/abstract.h index bd12a54963c13f..7cfee1332ccaa4 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -397,13 +397,23 @@ PyAPI_FUNC(int) PyIter_Check(PyObject *); This function always succeeds. */ PyAPI_FUNC(int) PyAIter_Check(PyObject *); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030e0000 +/* Return 1 and set 'item' to the next item of 'iter' on success. + * Return 0 and set 'item' to NULL when there are no remaining values. + * Return -1, set 'item' to NULL and set an exception on error. + */ +PyAPI_FUNC(int) PyIter_NextItem(PyObject *iter, PyObject **item); +#endif + /* Takes an iterator object and calls its tp_iternext slot, returning the next value. If the iterator is exhausted, this returns NULL without setting an exception. - NULL with an exception means an error occurred. */ + NULL with an exception means an error occurred. + + Prefer PyIter_NextItem() instead. */ PyAPI_FUNC(PyObject *) PyIter_Next(PyObject *); #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 @@ -852,15 +862,15 @@ PyAPI_FUNC(int) PyMapping_HasKeyWithError(PyObject *o, PyObject *key); PyAPI_FUNC(int) PyMapping_HasKeyStringWithError(PyObject *o, const char *key); -/* On success, return a list or tuple of the keys in mapping object 'o'. +/* On success, return a list of the keys in mapping object 'o'. On failure, return NULL. */ PyAPI_FUNC(PyObject *) PyMapping_Keys(PyObject *o); -/* On success, return a list or tuple of the values in mapping object 'o'. +/* On success, return a list of the values in mapping object 'o'. On failure, return NULL. */ PyAPI_FUNC(PyObject *) PyMapping_Values(PyObject *o); -/* On success, return a list or tuple of the items in mapping object 'o', +/* On success, return a list of the items in mapping object 'o', where each item is a tuple containing a key-value pair. On failure, return NULL. */ PyAPI_FUNC(PyObject *) PyMapping_Items(PyObject *o); diff --git a/Include/audit.h b/Include/audit.h new file mode 100644 index 00000000000000..793b7077e1027b --- /dev/null +++ b/Include/audit.h @@ -0,0 +1,30 @@ +#ifndef Py_AUDIT_H +#define Py_AUDIT_H +#ifdef __cplusplus +extern "C" { +#endif + + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +PyAPI_FUNC(int) PySys_Audit( + const char *event, + const char *argFormat, + ...); + +PyAPI_FUNC(int) PySys_AuditTuple( + const char *event, + PyObject *args); +#endif + + +#ifndef Py_LIMITED_API +# define Py_CPYTHON_AUDIT_H +# include "cpython/audit.h" +# undef Py_CPYTHON_AUDIT_H +#endif + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_AUDIT_H */ diff --git a/Include/boolobject.h b/Include/boolobject.h index 19aef5b1b87c6a..3037e61bbf6d0c 100644 --- a/Include/boolobject.h +++ b/Include/boolobject.h @@ -18,8 +18,13 @@ PyAPI_DATA(PyLongObject) _Py_FalseStruct; PyAPI_DATA(PyLongObject) _Py_TrueStruct; /* Use these macros */ -#define Py_False _PyObject_CAST(&_Py_FalseStruct) -#define Py_True _PyObject_CAST(&_Py_TrueStruct) +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030D0000 +# define Py_False Py_GetConstantBorrowed(Py_CONSTANT_FALSE) +# define Py_True Py_GetConstantBorrowed(Py_CONSTANT_TRUE) +#else +# define Py_False _PyObject_CAST(&_Py_FalseStruct) +# define Py_True _PyObject_CAST(&_Py_TrueStruct) +#endif // Test if an object is the True singleton, the same as "x is True" in Python. PyAPI_FUNC(int) Py_IsTrue(PyObject *x); diff --git a/Include/ceval.h b/Include/ceval.h index 9885bdb7febc21..32ab38972e548f 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -22,6 +22,10 @@ PyAPI_FUNC(PyObject *) PyEval_GetGlobals(void); PyAPI_FUNC(PyObject *) PyEval_GetLocals(void); PyAPI_FUNC(PyFrameObject *) PyEval_GetFrame(void); +PyAPI_FUNC(PyObject *) PyEval_GetFrameBuiltins(void); +PyAPI_FUNC(PyObject *) PyEval_GetFrameGlobals(void); +PyAPI_FUNC(PyObject *) PyEval_GetFrameLocals(void); + PyAPI_FUNC(int) Py_AddPendingCall(int (*func)(void *), void *arg); PyAPI_FUNC(int) Py_MakePendingCalls(void); @@ -38,7 +42,7 @@ PyAPI_FUNC(int) Py_MakePendingCalls(void); level exceeds "current recursion limit + 50". By construction, this protection can only be triggered when the "overflowed" flag is set. It means the cleanup code has itself gone into an infinite loop, or the - RecursionError has been mistakingly ignored. When this protection is + RecursionError has been mistakenly ignored. When this protection is triggered, the interpreter aborts with a Fatal Error. In addition, the "overflowed" flag is automatically reset when the @@ -107,6 +111,8 @@ PyAPI_FUNC(PyObject *) PyEval_EvalFrameEx(PyFrameObject *f, int exc); PyAPI_FUNC(PyThreadState *) PyEval_SaveThread(void); PyAPI_FUNC(void) PyEval_RestoreThread(PyThreadState *); +Py_DEPRECATED(3.9) PyAPI_FUNC(void) PyEval_InitThreads(void); + PyAPI_FUNC(void) PyEval_AcquireThread(PyThreadState *tstate); PyAPI_FUNC(void) PyEval_ReleaseThread(PyThreadState *tstate); @@ -127,6 +133,13 @@ PyAPI_FUNC(void) PyEval_ReleaseThread(PyThreadState *tstate); #define FVS_MASK 0x4 #define FVS_HAVE_SPEC 0x4 +/* Special methods used by LOAD_SPECIAL */ +#define SPECIAL___ENTER__ 0 +#define SPECIAL___EXIT__ 1 +#define SPECIAL___AENTER__ 2 +#define SPECIAL___AEXIT__ 3 +#define SPECIAL_MAX 3 + #ifndef Py_LIMITED_API # define Py_CPYTHON_CEVAL_H # include "cpython/ceval.h" diff --git a/Include/cpython/audit.h b/Include/cpython/audit.h new file mode 100644 index 00000000000000..3c5c7a8c06091d --- /dev/null +++ b/Include/cpython/audit.h @@ -0,0 +1,8 @@ +#ifndef Py_CPYTHON_AUDIT_H +# error "this header file must not be included directly" +#endif + + +typedef int(*Py_AuditHookFunction)(const char *, PyObject *, void *); + +PyAPI_FUNC(int) PySys_AddAuditHook(Py_AuditHookFunction, void*); diff --git a/Include/cpython/bytesobject.h b/Include/cpython/bytesobject.h index 816823716e9a6f..71c133f173f157 100644 --- a/Include/cpython/bytesobject.h +++ b/Include/cpython/bytesobject.h @@ -31,3 +31,12 @@ static inline Py_ssize_t PyBytes_GET_SIZE(PyObject *op) { return Py_SIZE(self); } #define PyBytes_GET_SIZE(self) PyBytes_GET_SIZE(_PyObject_CAST(self)) + +PyAPI_FUNC(PyObject*) PyBytes_Join(PyObject *sep, PyObject *iterable); + +// Deprecated alias kept for backward compatibility +Py_DEPRECATED(3.14) static inline PyObject* +_PyBytes_Join(PyObject *sep, PyObject *iterable) +{ + return PyBytes_Join(sep, iterable); +} diff --git a/Include/cpython/cellobject.h b/Include/cpython/cellobject.h index 47a6a491497ea0..85a63a13747d87 100644 --- a/Include/cpython/cellobject.h +++ b/Include/cpython/cellobject.h @@ -22,10 +22,14 @@ PyAPI_FUNC(PyObject *) PyCell_Get(PyObject *); PyAPI_FUNC(int) PyCell_Set(PyObject *, PyObject *); static inline PyObject* PyCell_GET(PyObject *op) { + PyObject *res; PyCellObject *cell; assert(PyCell_Check(op)); cell = _Py_CAST(PyCellObject*, op); - return cell->ob_ref; + Py_BEGIN_CRITICAL_SECTION(cell); + res = cell->ob_ref; + Py_END_CRITICAL_SECTION(); + return res; } #define PyCell_GET(op) PyCell_GET(_PyObject_CAST(op)) @@ -33,7 +37,9 @@ static inline void PyCell_SET(PyObject *op, PyObject *value) { PyCellObject *cell; assert(PyCell_Check(op)); cell = _Py_CAST(PyCellObject*, op); + Py_BEGIN_CRITICAL_SECTION(cell); cell->ob_ref = value; + Py_END_CRITICAL_SECTION(); } #define PyCell_SET(op, value) PyCell_SET(_PyObject_CAST(op), (value)) diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h index 78f7405661662f..ca8109e3248a8d 100644 --- a/Include/cpython/ceval.h +++ b/Include/cpython/ceval.h @@ -23,3 +23,21 @@ _PyEval_RequestCodeExtraIndex(freefunc f) { PyAPI_FUNC(int) _PyEval_SliceIndex(PyObject *, Py_ssize_t *); PyAPI_FUNC(int) _PyEval_SliceIndexNotNone(PyObject *, Py_ssize_t *); + + +// Trampoline API + +typedef struct { + FILE* perf_map; + PyThread_type_lock map_lock; +} PerfMapState; + +PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void); +PyAPI_FUNC(int) PyUnstable_WritePerfMapEntry( + const void *code_addr, + unsigned int code_size, + const char *entry_name); +PyAPI_FUNC(void) PyUnstable_PerfMapState_Fini(void); +PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename); +PyAPI_FUNC(int) PyUnstable_PerfTrampoline_CompileCode(PyCodeObject *); +PyAPI_FUNC(int) PyUnstable_PerfTrampoline_SetPersistAfterFork(int enable); diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 1f47d99fb60443..2bd3e08631f0ad 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -8,12 +8,14 @@ extern "C" { #endif +/* Total tool ids available */ +#define _PY_MONITORING_TOOL_IDS 8 /* Count of all local monitoring events */ -#define _PY_MONITORING_LOCAL_EVENTS 10 +#define _PY_MONITORING_LOCAL_EVENTS 11 /* Count of all "real" monitoring events (not derived from other events) */ -#define _PY_MONITORING_UNGROUPED_EVENTS 15 +#define _PY_MONITORING_UNGROUPED_EVENTS 16 /* Count of all monitoring events */ -#define _PY_MONITORING_EVENTS 17 +#define _PY_MONITORING_EVENTS 19 /* Tables of which tools are active for each monitored event. */ typedef struct _Py_LocalMonitors { @@ -24,47 +26,6 @@ typedef struct _Py_GlobalMonitors { uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS]; } _Py_GlobalMonitors; -/* Each instruction in a code object is a fixed-width value, - * currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG - * opcode allows for larger values but the current limit is 3 uses - * of EXTENDED_ARG (see Python/compile.c), for a maximum - * 32-bit value. This aligns with the note in Python/compile.c - * (compiler_addop_i_line) indicating that the max oparg value is - * 2**32 - 1, rather than INT_MAX. - */ - -typedef union { - uint16_t cache; - struct { - uint8_t code; - uint8_t arg; - } op; -} _Py_CODEUNIT; - - -/* These macros only remain defined for compatibility. */ -#define _Py_OPCODE(word) ((word).op.code) -#define _Py_OPARG(word) ((word).op.arg) - -static inline _Py_CODEUNIT -_py_make_codeunit(uint8_t opcode, uint8_t oparg) -{ - // No designated initialisers because of C++ compat - _Py_CODEUNIT word; - word.op.code = opcode; - word.op.arg = oparg; - return word; -} - -static inline void -_py_set_opcode(_Py_CODEUNIT *word, uint8_t opcode) -{ - word->op.code = opcode; -} - -#define _Py_MAKE_CODEUNIT(opcode, oparg) _py_make_codeunit((opcode), (oparg)) -#define _Py_SET_OPCODE(word, opcode) _py_set_opcode(&(word), (opcode)) - typedef struct { PyObject *_co_code; @@ -73,12 +34,13 @@ typedef struct { PyObject *_co_freevars; } _PyCoCached; -/* Ancilliary data structure used for instrumentation. - Line instrumentation creates an array of - these. One entry per code unit.*/ +/* Ancillary data structure used for instrumentation. + Line instrumentation creates this with sufficient + space for one entry per code unit. The total size + of the data will be `bytes_per_entry * Py_SIZE(code)` */ typedef struct { - uint8_t original_opcode; - int8_t line_delta; + uint8_t bytes_per_entry; + uint8_t data[1]; } _PyCoLineInstrumentationData; @@ -98,6 +60,8 @@ typedef struct { _Py_LocalMonitors active_monitors; /* The tools that are to be notified for events for the matching code unit */ uint8_t *tools; + /* The version of tools when they instrument the code */ + uintptr_t tool_versions[_PY_MONITORING_TOOL_IDS]; /* Information to support line events */ _PyCoLineInstrumentationData *lines; /* The tools that are to be notified for line events for the matching code unit */ @@ -109,6 +73,24 @@ typedef struct { uint8_t *per_instruction_tools; } _PyCoMonitoringData; +#ifdef Py_GIL_DISABLED + +/* Each thread specializes a thread-local copy of the bytecode in free-threaded + * builds. These copies are stored on the code object in a `_PyCodeArray`. The + * first entry in the array always points to the "main" copy of the bytecode + * that is stored at the end of the code object. + */ +typedef struct { + Py_ssize_t size; + char *entries[1]; +} _PyCodeArray; + +#define _PyCode_DEF_THREAD_LOCAL_BYTECODE() \ + _PyCodeArray *co_tlbc; +#else +#define _PyCode_DEF_THREAD_LOCAL_BYTECODE() +#endif + // To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are // defined in this macro: #define _PyCode_DEF(SIZE) { \ @@ -150,7 +132,8 @@ typedef struct { \ /* redundant values (derived from co_localsplusnames and \ co_localspluskinds) */ \ - int co_nlocalsplus; /* number of local + cell + free variables */ \ + int co_nlocalsplus; /* number of spaces for holding local, cell, \ + and free variables */ \ int co_framesize; /* Size of frame in words */ \ int co_nlocals; /* number of local variables */ \ int co_ncellvars; /* total number of cell variables */ \ @@ -169,11 +152,13 @@ typedef struct { _PyCoCached *_co_cached; /* cached co_* attributes */ \ uintptr_t _co_instrumentation_version; /* current instrumentation version */ \ _PyCoMonitoringData *_co_monitoring; /* Monitoring data */ \ + Py_ssize_t _co_unique_id; /* ID used for per-thread refcounting */ \ int _co_firsttraceable; /* index of first traceable instruction */ \ /* Scratch space for extra data relating to the code object. \ Type is a void* to keep the format private in codeobject.c to force \ people to go through the proper APIs. */ \ void *co_extra; \ + _PyCode_DEF_THREAD_LOCAL_BYTECODE() \ char co_code_adaptive[(SIZE)]; \ } @@ -210,12 +195,20 @@ struct PyCodeObject _PyCode_DEF(1); #define CO_NO_MONITORING_EVENTS 0x2000000 +/* Whether the code object has a docstring, + If so, it will be the first item in co_consts +*/ +#define CO_HAS_DOCSTRING 0x4000000 + +/* A function defined in class scope */ +#define CO_METHOD 0x8000000 + /* This should be defined if a future statement modifies the syntax. For example, when a keyword is added. */ #define PY_PARSER_REQUIRES_FUTURE_KEYWORD -#define CO_MAXBLOCKS 20 /* Max static block nesting within a function */ +#define CO_MAXBLOCKS 21 /* Max static block nesting within a function */ PyAPI_DATA(PyTypeObject) PyCode_Type; @@ -226,13 +219,14 @@ static inline Py_ssize_t PyCode_GetNumFree(PyCodeObject *op) { return op->co_nfreevars; } -static inline int PyCode_GetFirstFree(PyCodeObject *op) { +static inline int PyUnstable_Code_GetFirstFree(PyCodeObject *op) { assert(PyCode_Check(op)); return op->co_nlocalsplus - op->co_nfreevars; } -#define _PyCode_CODE(CO) _Py_RVALUE((_Py_CODEUNIT *)(CO)->co_code_adaptive) -#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) +Py_DEPRECATED(3.13) static inline int PyCode_GetFirstFree(PyCodeObject *op) { + return PyUnstable_Code_GetFirstFree(op); +} /* Unstable public interface */ PyAPI_FUNC(PyCodeObject *) PyUnstable_Code_New( diff --git a/Include/cpython/compile.h b/Include/cpython/compile.h index 0d587505ef7f85..cfdb7080d45f2b 100644 --- a/Include/cpython/compile.h +++ b/Include/cpython/compile.h @@ -32,28 +32,8 @@ typedef struct { #define _PyCompilerFlags_INIT \ (PyCompilerFlags){.cf_flags = 0, .cf_feature_version = PY_MINOR_VERSION} -/* source location information */ -typedef struct { - int lineno; - int end_lineno; - int col_offset; - int end_col_offset; -} _PyCompilerSrcLocation; - -#define SRC_LOCATION_FROM_AST(n) \ - (_PyCompilerSrcLocation){ \ - .lineno = (n)->lineno, \ - .end_lineno = (n)->end_lineno, \ - .col_offset = (n)->col_offset, \ - .end_col_offset = (n)->end_col_offset } - /* Future feature support */ -typedef struct { - int ff_features; /* flags set by future statements */ - _PyCompilerSrcLocation ff_location; /* location of last future statement */ -} PyFutureFeatures; - #define FUTURE_NESTED_SCOPES "nested_scopes" #define FUTURE_GENERATORS "generators" #define FUTURE_DIVISION "division" diff --git a/Include/cpython/complexobject.h b/Include/cpython/complexobject.h index fbdc6a91fe895c..28576afad0b6b5 100644 --- a/Include/cpython/complexobject.h +++ b/Include/cpython/complexobject.h @@ -9,10 +9,16 @@ typedef struct { // Operations on complex numbers. PyAPI_FUNC(Py_complex) _Py_c_sum(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) _Py_cr_sum(Py_complex, double); PyAPI_FUNC(Py_complex) _Py_c_diff(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) _Py_cr_diff(Py_complex, double); +PyAPI_FUNC(Py_complex) _Py_rc_diff(double, Py_complex); PyAPI_FUNC(Py_complex) _Py_c_neg(Py_complex); PyAPI_FUNC(Py_complex) _Py_c_prod(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) _Py_cr_prod(Py_complex, double); PyAPI_FUNC(Py_complex) _Py_c_quot(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) _Py_cr_quot(Py_complex, double); +PyAPI_FUNC(Py_complex) _Py_rc_quot(double, Py_complex); PyAPI_FUNC(Py_complex) _Py_c_pow(Py_complex, Py_complex); PyAPI_FUNC(double) _Py_c_abs(Py_complex); diff --git a/Include/cpython/context.h b/Include/cpython/context.h index a3249fc29b082e..3a7a4b459c09ad 100644 --- a/Include/cpython/context.h +++ b/Include/cpython/context.h @@ -27,6 +27,39 @@ PyAPI_FUNC(PyObject *) PyContext_CopyCurrent(void); PyAPI_FUNC(int) PyContext_Enter(PyObject *); PyAPI_FUNC(int) PyContext_Exit(PyObject *); +typedef enum { + /* + * The current context has switched to a different context. The object + * passed to the watch callback is the now-current contextvars.Context + * object, or None if no context is current. + */ + Py_CONTEXT_SWITCHED = 1, +} PyContextEvent; + +/* + * Context object watcher callback function. The object passed to the callback + * is event-specific; see PyContextEvent for details. + * + * if the callback returns with an exception set, it must return -1. Otherwise + * it should return 0 + */ +typedef int (*PyContext_WatchCallback)(PyContextEvent, PyObject *); + +/* + * Register a per-interpreter callback that will be invoked for context object + * enter/exit events. + * + * Returns a handle that may be passed to PyContext_ClearWatcher on success, + * or -1 and sets and error if no more handles are available. + */ +PyAPI_FUNC(int) PyContext_AddWatcher(PyContext_WatchCallback callback); + +/* + * Clear the watcher associated with the watcher_id handle. + * + * Returns 0 on success or -1 if no watcher exists for the provided id. + */ +PyAPI_FUNC(int) PyContext_ClearWatcher(int watcher_id); /* Create a new context variable. diff --git a/Include/cpython/critical_section.h b/Include/cpython/critical_section.h new file mode 100644 index 00000000000000..35db3fb6a59ce6 --- /dev/null +++ b/Include/cpython/critical_section.h @@ -0,0 +1,134 @@ +#ifndef Py_CPYTHON_CRITICAL_SECTION_H +# error "this header file must not be included directly" +#endif + +// Python critical sections +// +// Conceptually, critical sections are a deadlock avoidance layer on top of +// per-object locks. These helpers, in combination with those locks, replace +// our usage of the global interpreter lock to provide thread-safety for +// otherwise thread-unsafe objects, such as dict. +// +// NOTE: These APIs are no-ops in non-free-threaded builds. +// +// Straightforward per-object locking could introduce deadlocks that were not +// present when running with the GIL. Threads may hold locks for multiple +// objects simultaneously because Python operations can nest. If threads were +// to acquire the same locks in different orders, they would deadlock. +// +// One way to avoid deadlocks is to allow threads to hold only the lock (or +// locks) for a single operation at a time (typically a single lock, but some +// operations involve two locks). When a thread begins a nested operation it +// could suspend the locks for any outer operation: before beginning the nested +// operation, the locks for the outer operation are released and when the +// nested operation completes, the locks for the outer operation are +// reacquired. +// +// To improve performance, this API uses a variation of the above scheme. +// Instead of immediately suspending locks any time a nested operation begins, +// locks are only suspended if the thread would block. This reduces the number +// of lock acquisitions and releases for nested operations, while still +// avoiding deadlocks. +// +// Additionally, the locks for any active operation are suspended around +// other potentially blocking operations, such as I/O. This is because the +// interaction between locks and blocking operations can lead to deadlocks in +// the same way as the interaction between multiple locks. +// +// Each thread's critical sections and their corresponding locks are tracked in +// a stack in `PyThreadState.critical_section`. When a thread calls +// `_PyThreadState_Detach()`, such as before a blocking I/O operation or when +// waiting to acquire a lock, the thread suspends all of its active critical +// sections, temporarily releasing the associated locks. When the thread calls +// `_PyThreadState_Attach()`, it resumes the top-most (i.e., most recent) +// critical section by reacquiring the associated lock or locks. See +// `_PyCriticalSection_Resume()`. +// +// NOTE: Only the top-most critical section is guaranteed to be active. +// Operations that need to lock two objects at once must use +// `Py_BEGIN_CRITICAL_SECTION2()`. You *CANNOT* use nested critical sections +// to lock more than one object at once, because the inner critical section +// may suspend the outer critical sections. This API does not provide a way +// to lock more than two objects at once (though it could be added later +// if actually needed). +// +// NOTE: Critical sections implicitly behave like reentrant locks because +// attempting to acquire the same lock will suspend any outer (earlier) +// critical sections. However, they are less efficient for this use case than +// purposefully designed reentrant locks. +// +// Example usage: +// Py_BEGIN_CRITICAL_SECTION(op); +// ... +// Py_END_CRITICAL_SECTION(); +// +// To lock two objects at once: +// Py_BEGIN_CRITICAL_SECTION2(op1, op2); +// ... +// Py_END_CRITICAL_SECTION2(); + +typedef struct PyCriticalSection PyCriticalSection; +typedef struct PyCriticalSection2 PyCriticalSection2; + +PyAPI_FUNC(void) +PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op); + +PyAPI_FUNC(void) +PyCriticalSection_End(PyCriticalSection *c); + +PyAPI_FUNC(void) +PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b); + +PyAPI_FUNC(void) +PyCriticalSection2_End(PyCriticalSection2 *c); + +#ifndef Py_GIL_DISABLED +# define Py_BEGIN_CRITICAL_SECTION(op) \ + { +# define Py_END_CRITICAL_SECTION() \ + } +# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ + { +# define Py_END_CRITICAL_SECTION2() \ + } +#else /* !Py_GIL_DISABLED */ + +// NOTE: the contents of this struct are private and may change betweeen +// Python releases without a deprecation period. +struct PyCriticalSection { + // Tagged pointer to an outer active critical section (or 0). + uintptr_t _cs_prev; + + // Mutex used to protect critical section + PyMutex *_cs_mutex; +}; + +// A critical section protected by two mutexes. Use +// Py_BEGIN_CRITICAL_SECTION2 and Py_END_CRITICAL_SECTION2. +// NOTE: the contents of this struct are private and may change betweeen +// Python releases without a deprecation period. +struct PyCriticalSection2 { + PyCriticalSection _cs_base; + + PyMutex *_cs_mutex2; +}; + +# define Py_BEGIN_CRITICAL_SECTION(op) \ + { \ + PyCriticalSection _py_cs; \ + PyCriticalSection_Begin(&_py_cs, _PyObject_CAST(op)) + +# define Py_END_CRITICAL_SECTION() \ + PyCriticalSection_End(&_py_cs); \ + } + +# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ + { \ + PyCriticalSection2 _py_cs2; \ + PyCriticalSection2_Begin(&_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b)) + +# define Py_END_CRITICAL_SECTION2() \ + PyCriticalSection2_End(&_py_cs2); \ + } + +#endif diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index 35b6a822a0dfff..df9ec7050fca1a 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -14,16 +14,13 @@ typedef struct { /* Number of items in the dictionary */ Py_ssize_t ma_used; - /* Dictionary version: globally unique, value change each time - the dictionary is modified */ -#ifdef Py_BUILD_CORE - /* Bits 0-7 are for dict watchers. + /* This is a private field for CPython's internal use. + * Bits 0-7 are for dict watchers. * Bits 8-11 are for the watched mutation counter (used by tier2 optimization) - * The remaining bits (12-63) are the actual version tag. */ - uint64_t ma_version_tag; -#else - Py_DEPRECATED(3.12) uint64_t ma_version_tag; -#endif + * Bits 12-31 are currently unused + * Bits 32-63 are a unique id in the free threading build (used for per-thread refcounting) + */ + uint64_t _ma_watcher_tag; PyDictKeysObject *ma_keys; @@ -37,7 +34,8 @@ typedef struct { PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, Py_hash_t hash); -PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObject *, const char *); +// PyDict_GetItemStringRef() can be used instead +Py_DEPRECATED(3.14) PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObject *, const char *); PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); @@ -56,7 +54,11 @@ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { PyDictObject *mp; assert(PyDict_Check(op)); mp = _Py_CAST(PyDictObject*, op); +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&mp->ma_used); +#else return mp->ma_used; +#endif } #define PyDict_GET_SIZE(op) PyDict_GET_SIZE(_PyObject_CAST(op)) @@ -66,7 +68,12 @@ PyAPI_FUNC(PyObject *) _PyDict_NewPresized(Py_ssize_t minused); PyAPI_FUNC(int) PyDict_Pop(PyObject *dict, PyObject *key, PyObject **result); PyAPI_FUNC(int) PyDict_PopString(PyObject *dict, const char *key, PyObject **result); -PyAPI_FUNC(PyObject *) _PyDict_Pop(PyObject *dict, PyObject *key, PyObject *default_value); + +// Use PyDict_Pop() instead +Py_DEPRECATED(3.14) PyAPI_FUNC(PyObject *) _PyDict_Pop( + PyObject *dict, + PyObject *key, + PyObject *default_value); /* Dictionary watchers */ diff --git a/Include/cpython/fileutils.h b/Include/cpython/fileutils.h index b386ad107bde1f..626b1ad57b3846 100644 --- a/Include/cpython/fileutils.h +++ b/Include/cpython/fileutils.h @@ -2,7 +2,15 @@ # error "this header file must not be included directly" #endif -// Used by _testcapi which must not use the internal C API -PyAPI_FUNC(FILE*) _Py_fopen_obj( +PyAPI_FUNC(FILE*) Py_fopen( PyObject *path, const char *mode); + +// Deprecated alias kept for backward compatibility +Py_DEPRECATED(3.14) static inline FILE* +_Py_fopen_obj(PyObject *path, const char *mode) +{ + return Py_fopen(path, mode); +} + +PyAPI_FUNC(int) Py_fclose(FILE *file); diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index 4e19535c656f2c..dbbfbb5105ba7a 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -27,3 +27,9 @@ PyAPI_FUNC(int) _PyFrame_IsEntryFrame(PyFrameObject *frame); PyAPI_FUNC(int) PyFrame_FastToLocalsWithError(PyFrameObject *f); PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); + + +typedef struct { + PyObject_HEAD + PyFrameObject* frame; +} PyFrameLocalsProxyObject; diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index de2013323d2c72..598cd330bc9ca9 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -8,7 +8,7 @@ extern "C" { #endif -#define COMMON_FIELDS(PREFIX) \ +#define _Py_COMMON_FIELDS(PREFIX) \ PyObject *PREFIX ## globals; \ PyObject *PREFIX ## builtins; \ PyObject *PREFIX ## name; \ @@ -19,7 +19,7 @@ extern "C" { PyObject *PREFIX ## closure; /* NULL or a tuple of cell objects */ typedef struct { - COMMON_FIELDS(fc_) + _Py_COMMON_FIELDS(fc_) } PyFrameConstructor; /* Function objects and code objects should not be confused with each other: @@ -35,12 +35,13 @@ typedef struct { typedef struct { PyObject_HEAD - COMMON_FIELDS(func_) + _Py_COMMON_FIELDS(func_) PyObject *func_doc; /* The __doc__ attribute, can be anything */ PyObject *func_dict; /* The __dict__ attribute, a dict or NULL */ PyObject *func_weakreflist; /* List of weak references */ PyObject *func_module; /* The __module__ attribute, can be anything */ PyObject *func_annotations; /* Annotations, a dict or NULL */ + PyObject *func_annotate; /* Callable to fill the annotations dictionary */ PyObject *func_typeparams; /* Tuple of active type variables or NULL */ vectorcallfunc vectorcall; /* Version number for use by specializer. @@ -60,6 +61,8 @@ typedef struct { */ } PyFunctionObject; +#undef _Py_COMMON_FIELDS + PyAPI_DATA(PyTypeObject) PyFunction_Type; #define PyFunction_Check(op) Py_IS_TYPE((op), &PyFunction_Type) diff --git a/Include/cpython/genobject.h b/Include/cpython/genobject.h index 49e46c277d75ae..f75884e597e2c2 100644 --- a/Include/cpython/genobject.h +++ b/Include/cpython/genobject.h @@ -9,29 +9,7 @@ extern "C" { /* --- Generators --------------------------------------------------------- */ -/* _PyGenObject_HEAD defines the initial segment of generator - and coroutine objects. */ -#define _PyGenObject_HEAD(prefix) \ - PyObject_HEAD \ - /* List of weak reference. */ \ - PyObject *prefix##_weakreflist; \ - /* Name of the generator. */ \ - PyObject *prefix##_name; \ - /* Qualified name of the generator. */ \ - PyObject *prefix##_qualname; \ - _PyErr_StackItem prefix##_exc_state; \ - PyObject *prefix##_origin_or_finalizer; \ - char prefix##_hooks_inited; \ - char prefix##_closed; \ - char prefix##_running_async; \ - /* The frame */ \ - int8_t prefix##_frame_state; \ - PyObject *prefix##_iframe[1]; \ - -typedef struct { - /* The gi_ prefix is intended to remind of generator-iterator. */ - _PyGenObject_HEAD(gi) -} PyGenObject; +typedef struct _PyGenObject PyGenObject; PyAPI_DATA(PyTypeObject) PyGen_Type; @@ -46,9 +24,7 @@ PyAPI_FUNC(PyCodeObject *) PyGen_GetCode(PyGenObject *gen); /* --- PyCoroObject ------------------------------------------------------- */ -typedef struct { - _PyGenObject_HEAD(cr) -} PyCoroObject; +typedef struct _PyCoroObject PyCoroObject; PyAPI_DATA(PyTypeObject) PyCoro_Type; @@ -59,9 +35,7 @@ PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *, /* --- Asynchronous Generators -------------------------------------------- */ -typedef struct { - _PyGenObject_HEAD(ag) -} PyAsyncGenObject; +typedef struct _PyAsyncGenObject PyAsyncGenObject; PyAPI_DATA(PyTypeObject) PyAsyncGen_Type; PyAPI_DATA(PyTypeObject) _PyAsyncGenASend_Type; @@ -73,7 +47,6 @@ PyAPI_FUNC(PyObject *) PyAsyncGen_New(PyFrameObject *, #define PyAsyncGenASend_CheckExact(op) Py_IS_TYPE((op), &_PyAsyncGenASend_Type) - #undef _PyGenObject_HEAD #ifdef __cplusplus diff --git a/Include/cpython/import.h b/Include/cpython/import.h index 7daf0b84fcf71b..0fd61c28cafa0e 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -2,8 +2,6 @@ # error "this header file must not be included directly" #endif -PyMODINIT_FUNC PyInit__imp(void); - struct _inittab { const char *name; /* ASCII encoded string */ PyObject* (*initfunc)(void); diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 87c059c521cbc9..8ef19f677066c2 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -179,8 +179,15 @@ typedef struct PyConfig { int use_frozen_modules; int safe_path; int int_max_str_digits; +#ifdef __APPLE__ + int use_system_logger; +#endif int cpu_count; +#ifdef Py_GIL_DISABLED + int enable_gil; + int tlbc_enabled; +#endif /* --- Path configuration inputs ------------ */ int pathconfig_warnings; @@ -257,6 +264,14 @@ PyAPI_FUNC(PyStatus) PyConfig_SetWideStringList(PyConfig *config, Py_ssize_t length, wchar_t **items); +/* --- PyConfig_Get() ----------------------------------------- */ + +PyAPI_FUNC(PyObject*) PyConfig_Get(const char *name); +PyAPI_FUNC(int) PyConfig_GetInt(const char *name, int *value); +PyAPI_FUNC(PyObject*) PyConfig_Names(void); +PyAPI_FUNC(int) PyConfig_Set(const char *name, PyObject *value); + + /* --- Helper functions --------------------------------------- */ /* Get the original command line arguments, before Python modified them. @@ -264,6 +279,51 @@ PyAPI_FUNC(PyStatus) PyConfig_SetWideStringList(PyConfig *config, See also PyConfig.orig_argv. */ PyAPI_FUNC(void) Py_GetArgcArgv(int *argc, wchar_t ***argv); + +// --- PyInitConfig --------------------------------------------------------- + +typedef struct PyInitConfig PyInitConfig; + +PyAPI_FUNC(PyInitConfig*) PyInitConfig_Create(void); +PyAPI_FUNC(void) PyInitConfig_Free(PyInitConfig *config); + +PyAPI_FUNC(int) PyInitConfig_GetError(PyInitConfig* config, + const char **err_msg); +PyAPI_FUNC(int) PyInitConfig_GetExitCode(PyInitConfig* config, + int *exitcode); + +PyAPI_FUNC(int) PyInitConfig_HasOption(PyInitConfig *config, + const char *name); +PyAPI_FUNC(int) PyInitConfig_GetInt(PyInitConfig *config, + const char *name, + int64_t *value); +PyAPI_FUNC(int) PyInitConfig_GetStr(PyInitConfig *config, + const char *name, + char **value); +PyAPI_FUNC(int) PyInitConfig_GetStrList(PyInitConfig *config, + const char *name, + size_t *length, + char ***items); +PyAPI_FUNC(void) PyInitConfig_FreeStrList(size_t length, char **items); + +PyAPI_FUNC(int) PyInitConfig_SetInt(PyInitConfig *config, + const char *name, + int64_t value); +PyAPI_FUNC(int) PyInitConfig_SetStr(PyInitConfig *config, + const char *name, + const char *value); +PyAPI_FUNC(int) PyInitConfig_SetStrList(PyInitConfig *config, + const char *name, + size_t length, + char * const *items); + +PyAPI_FUNC(int) PyInitConfig_AddModule(PyInitConfig *config, + const char *name, + PyObject* (*initfunc)(void)); + +PyAPI_FUNC(int) Py_InitializeFromInitConfig(PyInitConfig *config); + + #ifdef __cplusplus } #endif diff --git a/Include/cpython/interpreteridobject.h b/Include/cpython/interpreteridobject.h deleted file mode 100644 index 4ab9ad5d315f80..00000000000000 --- a/Include/cpython/interpreteridobject.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef Py_CPYTHON_INTERPRETERIDOBJECT_H -# error "this header file must not be included directly" -#endif - -/* Interpreter ID Object */ - -PyAPI_DATA(PyTypeObject) PyInterpreterID_Type; - -PyAPI_FUNC(PyObject *) PyInterpreterID_New(int64_t); -PyAPI_FUNC(PyObject *) PyInterpreterState_GetIDObject(PyInterpreterState *); -PyAPI_FUNC(PyInterpreterState *) PyInterpreterID_LookUp(PyObject *); diff --git a/Include/cpython/lock.h b/Include/cpython/lock.h new file mode 100644 index 00000000000000..8ee03e82f74dfd --- /dev/null +++ b/Include/cpython/lock.h @@ -0,0 +1,63 @@ +#ifndef Py_CPYTHON_LOCK_H +# error "this header file must not be included directly" +#endif + +#define _Py_UNLOCKED 0 +#define _Py_LOCKED 1 + +// A mutex that occupies one byte. The lock can be zero initialized to +// represent the unlocked state. +// +// Typical initialization: +// PyMutex m = (PyMutex){0}; +// +// Or initialize as global variables: +// static PyMutex m; +// +// Typical usage: +// PyMutex_Lock(&m); +// ... +// PyMutex_Unlock(&m); +// +// The contents of the PyMutex are not part of the public API, but are +// described to aid in understanding the implementation and debugging. Only +// the two least significant bits are used. The remaining bits are always zero: +// 0b00: unlocked +// 0b01: locked +// 0b10: unlocked and has parked threads +// 0b11: locked and has parked threads +typedef struct PyMutex { + uint8_t _bits; // (private) +} PyMutex; + +// exported function for locking the mutex +PyAPI_FUNC(void) PyMutex_Lock(PyMutex *m); + +// exported function for unlocking the mutex +PyAPI_FUNC(void) PyMutex_Unlock(PyMutex *m); + +// Locks the mutex. +// +// If the mutex is currently locked, the calling thread will be parked until +// the mutex is unlocked. If the current thread holds the GIL, then the GIL +// will be released while the thread is parked. +static inline void +_PyMutex_Lock(PyMutex *m) +{ + uint8_t expected = _Py_UNLOCKED; + if (!_Py_atomic_compare_exchange_uint8(&m->_bits, &expected, _Py_LOCKED)) { + PyMutex_Lock(m); + } +} +#define PyMutex_Lock _PyMutex_Lock + +// Unlocks the mutex. +static inline void +_PyMutex_Unlock(PyMutex *m) +{ + uint8_t expected = _Py_LOCKED; + if (!_Py_atomic_compare_exchange_uint8(&m->_bits, &expected, _Py_UNLOCKED)) { + PyMutex_Unlock(m); + } +} +#define PyMutex_Unlock _PyMutex_Unlock diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index f037c7bb90deda..4b6f97a5e475d6 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -76,8 +76,8 @@ typedef long stwodigits; /* signed variant of twodigits */ - 1: Zero - 2: Negative - The third lowest bit of lv_tag is reserved for an immortality flag, but is - not currently used. + The third lowest bit of lv_tag is + set to 1 for the small ints. In a normalized number, ob_digit[ndigits-1] (the most significant digit) is never zero. Also, in all cases, for all valid i, @@ -100,12 +100,12 @@ struct _longobject { _PyLongValue long_value; }; -PyAPI_FUNC(PyLongObject*) _PyLong_New(Py_ssize_t); +Py_DEPRECATED(3.14) PyAPI_FUNC(PyLongObject*) _PyLong_New(Py_ssize_t); // Return a copy of src. PyAPI_FUNC(PyObject*) _PyLong_Copy(PyLongObject *src); -PyAPI_FUNC(PyLongObject*) _PyLong_FromDigits( +Py_DEPRECATED(3.14) PyAPI_FUNC(PyLongObject*) _PyLong_FromDigits( int negative, Py_ssize_t digit_count, digit *digits); @@ -120,7 +120,7 @@ PyAPI_FUNC(PyLongObject*) _PyLong_FromDigits( static inline int _PyLong_IsCompact(const PyLongObject* op) { - assert(PyType_HasFeature((op)->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); + assert(PyType_HasFeature(op->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); return op->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS); } @@ -129,15 +129,54 @@ _PyLong_IsCompact(const PyLongObject* op) { static inline Py_ssize_t _PyLong_CompactValue(const PyLongObject *op) { - assert(PyType_HasFeature((op)->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); + Py_ssize_t sign; + assert(PyType_HasFeature(op->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); assert(PyUnstable_Long_IsCompact(op)); - Py_ssize_t sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK); + sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK); return sign * (Py_ssize_t)op->long_value.ob_digit[0]; } #define PyUnstable_Long_CompactValue _PyLong_CompactValue +/* --- Import/Export API -------------------------------------------------- */ + +typedef struct PyLongLayout { + uint8_t bits_per_digit; + uint8_t digit_size; + int8_t digits_order; + int8_t digit_endianness; +} PyLongLayout; + +PyAPI_FUNC(const PyLongLayout*) PyLong_GetNativeLayout(void); + +typedef struct PyLongExport { + int64_t value; + uint8_t negative; + Py_ssize_t ndigits; + const void *digits; + // Member used internally, must not be used for other purpose. + Py_uintptr_t _reserved; +} PyLongExport; + +PyAPI_FUNC(int) PyLong_Export( + PyObject *obj, + PyLongExport *export_long); +PyAPI_FUNC(void) PyLong_FreeExport( + PyLongExport *export_long); + + +/* --- PyLongWriter API --------------------------------------------------- */ + +typedef struct PyLongWriter PyLongWriter; + +PyAPI_FUNC(PyLongWriter*) PyLongWriter_Create( + int negative, + Py_ssize_t ndigits, + void **digits); +PyAPI_FUNC(PyObject*) PyLongWriter_Finish(PyLongWriter *writer); +PyAPI_FUNC(void) PyLongWriter_Discard(PyLongWriter *writer); + #ifdef __cplusplus } #endif diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h index 07251db6bcc203..7f28ad60b7467b 100644 --- a/Include/cpython/longobject.h +++ b/Include/cpython/longobject.h @@ -2,13 +2,32 @@ # error "this header file must not be included directly" #endif +#define _PyLong_CAST(op) \ + (assert(PyLong_Check(op)), _Py_CAST(PyLongObject*, (op))) + PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base); +#define Py_ASNATIVEBYTES_DEFAULTS -1 +#define Py_ASNATIVEBYTES_BIG_ENDIAN 0 +#define Py_ASNATIVEBYTES_LITTLE_ENDIAN 1 +#define Py_ASNATIVEBYTES_NATIVE_ENDIAN 3 +#define Py_ASNATIVEBYTES_UNSIGNED_BUFFER 4 +#define Py_ASNATIVEBYTES_REJECT_NEGATIVE 8 +#define Py_ASNATIVEBYTES_ALLOW_INDEX 16 + /* PyLong_AsNativeBytes: Copy the integer value to a native variable. buffer points to the first byte of the variable. n_bytes is the number of bytes available in the buffer. Pass 0 to request the required size for the value. - endianness is -1 for native endian, 0 for big endian or 1 for little. + flags is a bitfield of the following flags: + * 1 - little endian + * 2 - native endian + * 4 - unsigned destination (e.g. don't reject copying 255 into one byte) + * 8 - raise an exception for negative inputs + * 16 - call __index__ on non-int types + If flags is -1 (all bits set), native endian is used, value truncation + behaves most like C (allows negative inputs and allow MSB set), and non-int + objects will raise a TypeError. Big endian mode will write the most significant byte into the address directly referenced by buffer; little endian will write the least significant byte into that address. @@ -24,27 +43,58 @@ PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base); calculate the bit length of an integer object. */ PyAPI_FUNC(Py_ssize_t) PyLong_AsNativeBytes(PyObject* v, void* buffer, - Py_ssize_t n_bytes, int endianness); + Py_ssize_t n_bytes, int flags); /* PyLong_FromNativeBytes: Create an int value from a native integer n_bytes is the number of bytes to read from the buffer. Passing 0 will always produce the zero int. PyLong_FromUnsignedNativeBytes always produces a non-negative int. - endianness is -1 for native endian, 0 for big endian or 1 for little. + flags is the same as for PyLong_AsNativeBytes, but only supports selecting + the endianness or forcing an unsigned buffer. Returns the int object, or NULL with an exception set. */ PyAPI_FUNC(PyObject*) PyLong_FromNativeBytes(const void* buffer, size_t n_bytes, - int endianness); + int flags); PyAPI_FUNC(PyObject*) PyLong_FromUnsignedNativeBytes(const void* buffer, - size_t n_bytes, int endianness); + size_t n_bytes, int flags); PyAPI_FUNC(int) PyUnstable_Long_IsCompact(const PyLongObject* op); PyAPI_FUNC(Py_ssize_t) PyUnstable_Long_CompactValue(const PyLongObject* op); -// _PyLong_Sign. Return 0 if v is 0, -1 if v < 0, +1 if v > 0. -// v must not be NULL, and must be a normalized long. -// There are no error cases. -PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); +/* PyLong_IsPositive. Check if the integer object is positive. + + - On success, return 1 if *obj is positive, and 0 otherwise. + - On failure, set an exception, and return -1. */ +PyAPI_FUNC(int) PyLong_IsPositive(PyObject *obj); + +/* PyLong_IsNegative. Check if the integer object is negative. + + - On success, return 1 if *obj is negative, and 0 otherwise. + - On failure, set an exception, and return -1. */ +PyAPI_FUNC(int) PyLong_IsNegative(PyObject *obj); + +/* PyLong_IsZero. Check if the integer object is zero. + + - On success, return 1 if *obj is zero, and 0 if it is non-zero. + - On failure, set an exception, and return -1. */ +PyAPI_FUNC(int) PyLong_IsZero(PyObject *obj); + +/* PyLong_GetSign. Get the sign of an integer object: + 0, -1 or +1 for zero, negative or positive integer, respectively. + + - On success, set '*sign' to the integer sign, and return 0. + - On failure, set an exception, and return -1. */ +PyAPI_FUNC(int) PyLong_GetSign(PyObject *v, int *sign); + +Py_DEPRECATED(3.14) PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); + +/* _PyLong_NumBits. Return the number of bits needed to represent the + absolute value of a long. For example, this returns 1 for 1 and -1, 2 + for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. + v must not be NULL, and must be a normalized long. + Always successful. +*/ +PyAPI_FUNC(int64_t) _PyLong_NumBits(PyObject *v); /* _PyLong_FromByteArray: View the n unsigned bytes as a binary integer in base 256, and return a Python int with the same numeric value. diff --git a/Include/cpython/modsupport.h b/Include/cpython/modsupport.h new file mode 100644 index 00000000000000..d3b88f58c82ca3 --- /dev/null +++ b/Include/cpython/modsupport.h @@ -0,0 +1,26 @@ +#ifndef Py_CPYTHON_MODSUPPORT_H +# error "this header file must not be included directly" +#endif + +// A data structure that can be used to run initialization code once in a +// thread-safe manner. The C++11 equivalent is std::call_once. +typedef struct { + uint8_t v; +} _PyOnceFlag; + +typedef struct _PyArg_Parser { + const char *format; + const char * const *keywords; + const char *fname; + const char *custom_msg; + _PyOnceFlag once; /* atomic one-time initialization flag */ + int is_kwtuple_owned; /* does this parser own the kwtuple object? */ + int pos; /* number of positional-only arguments */ + int min; /* minimal number of arguments */ + int max; /* maximal number of positional arguments */ + PyObject *kwtuple; /* tuple of keyword parameter names */ + struct _PyArg_Parser *next; +} _PyArg_Parser; + +PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywordsFast(PyObject *, PyObject *, + struct _PyArg_Parser *, ...); diff --git a/Include/cpython/monitoring.h b/Include/cpython/monitoring.h new file mode 100644 index 00000000000000..ce92942404c9f7 --- /dev/null +++ b/Include/cpython/monitoring.h @@ -0,0 +1,269 @@ +#ifndef Py_CPYTHON_MONITORING_H +# error "this header file must not be included directly" +#endif + +/* Local events. + * These require bytecode instrumentation */ + +#define PY_MONITORING_EVENT_PY_START 0 +#define PY_MONITORING_EVENT_PY_RESUME 1 +#define PY_MONITORING_EVENT_PY_RETURN 2 +#define PY_MONITORING_EVENT_PY_YIELD 3 +#define PY_MONITORING_EVENT_CALL 4 +#define PY_MONITORING_EVENT_LINE 5 +#define PY_MONITORING_EVENT_INSTRUCTION 6 +#define PY_MONITORING_EVENT_JUMP 7 +#define PY_MONITORING_EVENT_BRANCH_LEFT 8 +#define PY_MONITORING_EVENT_BRANCH_RIGHT 9 +#define PY_MONITORING_EVENT_STOP_ITERATION 10 + +#define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ + ((ev) < _PY_MONITORING_LOCAL_EVENTS) + +/* Other events, mainly exceptions */ + +#define PY_MONITORING_EVENT_RAISE 11 +#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 12 +#define PY_MONITORING_EVENT_PY_UNWIND 13 +#define PY_MONITORING_EVENT_PY_THROW 14 +#define PY_MONITORING_EVENT_RERAISE 15 + + +/* Ancillary events */ + +#define PY_MONITORING_EVENT_C_RETURN 16 +#define PY_MONITORING_EVENT_C_RAISE 17 +#define PY_MONITORING_EVENT_BRANCH 18 + + +typedef struct _PyMonitoringState { + uint8_t active; + uint8_t opaque; +} PyMonitoringState; + + +PyAPI_FUNC(int) +PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, + const uint8_t *event_types, Py_ssize_t length); + +PyAPI_FUNC(int) +PyMonitoring_ExitScope(void); + + +PyAPI_FUNC(int) +_PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval); + +PyAPI_FUNC(int) +_PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* callable, PyObject *arg0); + +PyAPI_FUNC(int) +_PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + int lineno); + +PyAPI_FUNC(int) +_PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset); + +Py_DEPRECATED(3.14) PyAPI_FUNC(int) +_PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireBranchRightEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireBranchLeftEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value); + + +#define _PYMONITORING_IF_ACTIVE(STATE, X) \ + if ((STATE)->active) { \ + return (X); \ + } \ + else { \ + return 0; \ + } + +static inline int +PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyStartEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyResumeEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyReturnEvent(state, codelike, offset, retval)); +} + +static inline int +PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyYieldEvent(state, codelike, offset, retval)); +} + +static inline int +PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* callable, PyObject *arg0) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireCallEvent(state, codelike, offset, callable, arg0)); +} + +static inline int +PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + int lineno) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireLineEvent(state, codelike, offset, lineno)); +} + +static inline int +PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireJumpEvent(state, codelike, offset, target_offset)); +} + +static inline int +PyMonitoring_FireBranchRightEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireBranchRightEvent(state, codelike, offset, target_offset)); +} + +static inline int +PyMonitoring_FireBranchLeftEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireBranchLeftEvent(state, codelike, offset, target_offset)); +} + +static inline int +PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireCReturnEvent(state, codelike, offset, retval)); +} + +static inline int +PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyThrowEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireRaiseEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireReraiseEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireExceptionHandledEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireCRaiseEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyUnwindEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireStopIterationEvent(state, codelike, offset, value)); +} + +#undef _PYMONITORING_IF_ACTIVE diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 7512bb70c760fd..71bd01884426ad 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -221,7 +221,9 @@ struct _typeobject { PyObject *tp_weaklist; /* not used for static builtin types */ destructor tp_del; - /* Type attribute cache version tag. Added in version 2.6 */ + /* Type attribute cache version tag. Added in version 2.6. + * If zero, the cache is invalid and must be initialized. + */ unsigned int tp_version_tag; destructor tp_finalize; @@ -229,9 +231,17 @@ struct _typeobject { /* bitset of which type-watchers care about this type */ unsigned char tp_watched; + + /* Number of tp_version_tag values used. + * Set to _Py_ATTR_CACHE_UNUSED if the attribute cache is + * disabled for this type (e.g. due to custom MRO entries). + * Otherwise, limited to MAX_VERSIONS_PER_CLASS (defined elsewhere). + */ uint16_t tp_versions_used; }; +#define _Py_ATTR_CACHE_UNUSED (30000) // (see tp_versions_used) + /* This struct is used by the specializer * It should be treated as an opaque blob * by code other than the specializer and interpreter. */ @@ -269,13 +279,17 @@ typedef struct _heaptypeobject { struct _dictkeysobject *ht_cached_keys; PyObject *ht_module; char *_ht_tpname; // Storage for "tp_name"; see PyType_FromModuleAndSpec + void *ht_token; // Storage for the "Py_tp_token" slot struct _specialization_cache _spec_cache; // For use by the specializer. +#ifdef Py_GIL_DISABLED + Py_ssize_t unique_id; // ID used for per-thread refcounting +#endif /* here are optional user slots, followed by the members. */ } PyHeapTypeObject; PyAPI_FUNC(const char *) _PyType_Name(PyTypeObject *); PyAPI_FUNC(PyObject *) _PyType_Lookup(PyTypeObject *, PyObject *); -PyAPI_FUNC(PyObject *) PyType_GetModuleByDef(PyTypeObject *, PyModuleDef *); +PyAPI_FUNC(PyObject *) _PyType_LookupRef(PyTypeObject *, PyObject *); PyAPI_FUNC(PyObject *) PyType_GetDict(PyTypeObject *); PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); @@ -288,6 +302,8 @@ PyAPI_FUNC(PyObject **) _PyObject_GetDictPtr(PyObject *); PyAPI_FUNC(void) PyObject_CallFinalizer(PyObject *); PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *); +PyAPI_FUNC(void) PyUnstable_Object_ClearWeakRefsNoCallbacks(PyObject *); + /* Same as PyObject_Generic{Get,Set}Attr, but passing the attributes dict as the last parameter. */ PyAPI_FUNC(PyObject *) @@ -313,7 +329,7 @@ PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *); * triggered as a side-effect of `dst` getting torn down no longer believes * `dst` points to a valid object. * - * Temporary variables are used to only evalutate macro arguments once and so + * Temporary variables are used to only evaluate macro arguments once and so * avoid the duplication of side effects. _Py_TYPEOF() or memcpy() is used to * avoid a miscompilation caused by type punning. See Py_CLEAR() comment for * implementation details about type punning. @@ -449,8 +465,8 @@ without deallocating anything (and so unbounded call-stack depth is avoided). When the call stack finishes unwinding again, code generated by the END macro notices this, and calls another routine to deallocate all the objects that may have been added to the list of deferred deallocations. In effect, a -chain of N deallocations is broken into (N-1)/(_PyTrash_UNWIND_LEVEL-1) pieces, -with the call stack never exceeding a depth of _PyTrash_UNWIND_LEVEL. +chain of N deallocations is broken into (N-1)/(Py_TRASHCAN_HEADROOM-1) pieces, +with the call stack never exceeding a depth of Py_TRASHCAN_HEADROOM. Since the tp_dealloc of a subclass typically calls the tp_dealloc of the base class, we need to ensure that the trashcan is only triggered on the tp_dealloc @@ -459,38 +475,39 @@ partially-deallocated object. To check this, the tp_dealloc function must be passed as second argument to Py_TRASHCAN_BEGIN(). */ -/* Python 3.9 private API, invoked by the macros below. */ -PyAPI_FUNC(int) _PyTrash_begin(PyThreadState *tstate, PyObject *op); -PyAPI_FUNC(void) _PyTrash_end(PyThreadState *tstate); + +PyAPI_FUNC(void) _PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op); +PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(PyThreadState *tstate); + + /* Python 3.10 private API, invoked by the Py_TRASHCAN_BEGIN(). */ -PyAPI_FUNC(int) _PyTrash_cond(PyObject *op, destructor dealloc); -#define Py_TRASHCAN_BEGIN_CONDITION(op, cond) \ - do { \ - PyThreadState *_tstate = NULL; \ - /* If "cond" is false, then _tstate remains NULL and the deallocator \ - * is run normally without involving the trashcan */ \ - if (cond) { \ - _tstate = PyThreadState_GetUnchecked(); \ - if (_PyTrash_begin(_tstate, _PyObject_CAST(op))) { \ - break; \ - } \ - } - /* The body of the deallocator is here. */ -#define Py_TRASHCAN_END \ - if (_tstate) { \ - _PyTrash_end(_tstate); \ - } \ - } while (0); +/* To avoid raising recursion errors during dealloc trigger trashcan before we reach + * recursion limit. To avoid trashing, we don't attempt to empty the trashcan until + * we have headroom above the trigger limit */ +#define Py_TRASHCAN_HEADROOM 50 #define Py_TRASHCAN_BEGIN(op, dealloc) \ - Py_TRASHCAN_BEGIN_CONDITION((op), \ - _PyTrash_cond(_PyObject_CAST(op), (destructor)(dealloc))) +do { \ + PyThreadState *tstate = PyThreadState_Get(); \ + if (tstate->c_recursion_remaining <= Py_TRASHCAN_HEADROOM && Py_TYPE(op)->tp_dealloc == (destructor)dealloc) { \ + _PyTrash_thread_deposit_object(tstate, (PyObject *)op); \ + break; \ + } \ + tstate->c_recursion_remaining--; + /* The body of the deallocator is here. */ +#define Py_TRASHCAN_END \ + tstate->c_recursion_remaining++; \ + if (tstate->delete_later && tstate->c_recursion_remaining > (Py_TRASHCAN_HEADROOM*2)) { \ + _PyTrash_thread_destroy_chain(tstate); \ + } \ +} while (0); PyAPI_FUNC(void *) PyObject_GetItemData(PyObject *obj); PyAPI_FUNC(int) PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg); +PyAPI_FUNC(int) _PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict); PyAPI_FUNC(void) PyObject_ClearManagedDict(PyObject *obj); #define TYPE_MAX_WATCHERS 8 @@ -507,3 +524,29 @@ PyAPI_FUNC(int) PyType_Unwatch(int watcher_id, PyObject *type); * assigned, or 0 if a new tag could not be assigned. */ PyAPI_FUNC(int) PyUnstable_Type_AssignVersionTag(PyTypeObject *type); + + +typedef enum { + PyRefTracer_CREATE = 0, + PyRefTracer_DESTROY = 1, +} PyRefTracerEvent; + +typedef int (*PyRefTracer)(PyObject *, PyRefTracerEvent event, void *); +PyAPI_FUNC(int) PyRefTracer_SetTracer(PyRefTracer tracer, void *data); +PyAPI_FUNC(PyRefTracer) PyRefTracer_GetTracer(void**); + +/* Enable PEP-703 deferred reference counting on the object. + * + * Returns 1 if deferred reference counting was successfully enabled, and + * 0 if the runtime ignored it. This function cannot fail. + */ +PyAPI_FUNC(int) PyUnstable_Object_EnableDeferredRefcount(PyObject *); + +/* Check whether the object is immortal. This cannot fail. */ +PyAPI_FUNC(int) PyUnstable_IsImmortal(PyObject *); + +// Increments the reference count of the object, if it's not zero. +// PyUnstable_EnableTryIncRef() should be called on the object +// before calling this function in order to avoid spurious failures. +PyAPI_FUNC(int) PyUnstable_TryIncRef(PyObject *); +PyAPI_FUNC(void) PyUnstable_EnableTryIncRef(PyObject *); diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h deleted file mode 100644 index 6d7b8bc3c1433a..00000000000000 --- a/Include/cpython/optimizer.h +++ /dev/null @@ -1,122 +0,0 @@ - -#ifndef Py_LIMITED_API -#ifndef Py_OPTIMIZER_H -#define Py_OPTIMIZER_H -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct _PyExecutorLinkListNode { - struct _PyExecutorObject *next; - struct _PyExecutorObject *previous; -} _PyExecutorLinkListNode; - - -/* Bloom filter with m = 256 - * https://en.wikipedia.org/wiki/Bloom_filter */ -#define BLOOM_FILTER_WORDS 8 - -typedef struct _bloom_filter { - uint32_t bits[BLOOM_FILTER_WORDS]; -} _PyBloomFilter; - -typedef struct { - uint8_t opcode; - uint8_t oparg; - uint8_t valid; - int index; // Index of ENTER_EXECUTOR (if code isn't NULL, below). - _PyBloomFilter bloom; - _PyExecutorLinkListNode links; - PyCodeObject *code; // Weak (NULL if no corresponding ENTER_EXECUTOR). -} _PyVMData; - -typedef struct { - uint16_t opcode; - uint16_t oparg; - union { - uint32_t target; - uint32_t exit_index; - }; - uint64_t operand; // A cache entry -} _PyUOpInstruction; - -typedef struct _exit_data { - uint32_t target; - int16_t temperature; - const struct _PyExecutorObject *executor; -} _PyExitData; - -typedef struct _PyExecutorObject { - PyObject_VAR_HEAD - const _PyUOpInstruction *trace; - _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ - uint32_t exit_count; - uint32_t code_size; - size_t jit_size; - void *jit_code; - _PyExitData exits[1]; -} _PyExecutorObject; - -typedef struct _PyOptimizerObject _PyOptimizerObject; - -/* Should return > 0 if a new executor is created. O if no executor is produced and < 0 if an error occurred. */ -typedef int (*optimize_func)( - _PyOptimizerObject* self, struct _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, _PyExecutorObject **exec_ptr, - int curr_stackentries); - -typedef struct _PyOptimizerObject { - PyObject_HEAD - optimize_func optimize; - /* These thresholds are treated as signed so do not exceed INT16_MAX - * Use INT16_MAX to indicate that the optimizer should never be called */ - uint16_t resume_threshold; - uint16_t side_threshold; - uint16_t backedge_threshold; - /* Data needed by the optimizer goes here, but is opaque to the VM */ -} _PyOptimizerObject; - -/** Test support **/ -typedef struct { - _PyOptimizerObject base; - int64_t count; -} _PyCounterOptimizerObject; - -PyAPI_FUNC(int) PyUnstable_Replace_Executor(PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutorObject *executor); - -_PyOptimizerObject *_Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject* optimizer); - -PyAPI_FUNC(int) PyUnstable_SetOptimizer(_PyOptimizerObject* optimizer); - -PyAPI_FUNC(_PyOptimizerObject *) PyUnstable_GetOptimizer(void); - -PyAPI_FUNC(_PyExecutorObject *) PyUnstable_GetExecutor(PyCodeObject *code, int offset); - -void _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *); -void _Py_ExecutorClear(_PyExecutorObject *); -void _Py_BloomFilter_Init(_PyBloomFilter *); -void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); -PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); -PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation); -extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation); - -/* For testing */ -PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void); -PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewUOpOptimizer(void); - -#define OPTIMIZER_BITS_IN_COUNTER 4 -/* Minimum of 16 additional executions before retry */ -#define MIN_TIER2_BACKOFF 4 -#define MAX_TIER2_BACKOFF (15 - OPTIMIZER_BITS_IN_COUNTER) -#define OPTIMIZER_BITS_MASK ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1) -/* A value <= UINT16_MAX but large enough that when shifted is > UINT16_MAX */ -#define OPTIMIZER_UNREACHABLE_THRESHOLD UINT16_MAX - -#define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3 -#define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6 - -#ifdef __cplusplus -} -#endif -#endif /* !Py_OPTIMIZER_H */ -#endif /* Py_LIMITED_API */ diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index c3e132d3877ca5..2a0c11e7b3ad66 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -321,6 +321,27 @@ _Py_atomic_load_ptr(const void *obj); static inline int _Py_atomic_load_int_relaxed(const int *obj); +static inline char +_Py_atomic_load_char_relaxed(const char *obj); + +static inline unsigned char +_Py_atomic_load_uchar_relaxed(const unsigned char *obj); + +static inline short +_Py_atomic_load_short_relaxed(const short *obj); + +static inline unsigned short +_Py_atomic_load_ushort_relaxed(const unsigned short *obj); + +static inline long +_Py_atomic_load_long_relaxed(const long *obj); + +static inline double +_Py_atomic_load_double_relaxed(const double *obj); + +static inline long long +_Py_atomic_load_llong_relaxed(const long long *obj); + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj); @@ -458,6 +479,30 @@ static inline void _Py_atomic_store_ullong_relaxed(unsigned long long *obj, unsigned long long value); +static inline void +_Py_atomic_store_char_relaxed(char *obj, char value); + +static inline void +_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value); + +static inline void +_Py_atomic_store_short_relaxed(short *obj, short value); + +static inline void +_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value); + +static inline void +_Py_atomic_store_long_relaxed(long *obj, long value); + +static inline void +_Py_atomic_store_float_relaxed(float *obj, float value); + +static inline void +_Py_atomic_store_double_relaxed(double *obj, double value); + +static inline void +_Py_atomic_store_llong_relaxed(long long *obj, long long value); + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ @@ -465,10 +510,16 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj, static inline void * _Py_atomic_load_ptr_acquire(const void *obj); +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj); + // Stores `*obj = value` (release operation) static inline void _Py_atomic_store_ptr_release(void *obj, void *value); +static inline void +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value); + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value); @@ -478,6 +529,9 @@ _Py_atomic_store_int_release(int *obj, int value); static inline int _Py_atomic_load_int_acquire(const int *obj); +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value); + static inline void _Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value); @@ -491,6 +545,8 @@ static inline Py_ssize_t _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); + + // --- _Py_atomic_fence ------------------------------------------------------ // Sequential consistency fence. C11 fences have complex semantics. When @@ -499,6 +555,9 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); // See https://en.cppreference.com/w/cpp/atomic/atomic_thread_fence static inline void _Py_atomic_fence_seq_cst(void); +// Acquire fence +static inline void _Py_atomic_fence_acquire(void); + // Release fence static inline void _Py_atomic_fence_release(void); @@ -515,15 +574,15 @@ static inline void _Py_atomic_fence_release(void); #if _Py_USE_GCC_BUILTIN_ATOMICS # define Py_ATOMIC_GCC_H -# include "cpython/pyatomic_gcc.h" +# include "pyatomic_gcc.h" # undef Py_ATOMIC_GCC_H #elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__) # define Py_ATOMIC_STD_H -# include "cpython/pyatomic_std.h" +# include "pyatomic_std.h" # undef Py_ATOMIC_STD_H #elif defined(_MSC_VER) # define Py_ATOMIC_MSC_H -# include "cpython/pyatomic_msc.h" +# include "pyatomic_msc.h" # undef Py_ATOMIC_MSC_H #else # error "no available pyatomic implementation for this platform/compiler" diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 0b40f81bd8736d..b179e4c9a185a9 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -297,7 +297,7 @@ _Py_atomic_load_ssize(const Py_ssize_t *obj) static inline void * _Py_atomic_load_ptr(const void *obj) -{ return (void *)__atomic_load_n((void **)obj, __ATOMIC_SEQ_CST); } +{ return (void *)__atomic_load_n((void * const *)obj, __ATOMIC_SEQ_CST); } // --- _Py_atomic_load_relaxed ----------------------------------------------- @@ -306,6 +306,34 @@ static inline int _Py_atomic_load_int_relaxed(const int *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } +static inline char +_Py_atomic_load_char_relaxed(const char *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline unsigned char +_Py_atomic_load_uchar_relaxed(const unsigned char *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline short +_Py_atomic_load_short_relaxed(const short *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline unsigned short +_Py_atomic_load_ushort_relaxed(const unsigned short *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline long +_Py_atomic_load_long_relaxed(const long *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline float +_Py_atomic_load_float_relaxed(const float *obj) +{ float ret; __atomic_load(obj, &ret, __ATOMIC_RELAXED); return ret; } + +static inline double +_Py_atomic_load_double_relaxed(const double *obj) +{ double ret; __atomic_load(obj, &ret, __ATOMIC_RELAXED); return ret; } + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } @@ -356,12 +384,16 @@ _Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj) static inline void * _Py_atomic_load_ptr_relaxed(const void *obj) -{ return (void *)__atomic_load_n((const void **)obj, __ATOMIC_RELAXED); } +{ return (void *)__atomic_load_n((void * const *)obj, __ATOMIC_RELAXED); } static inline unsigned long long _Py_atomic_load_ullong_relaxed(const unsigned long long *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } +static inline long long +_Py_atomic_load_llong_relaxed(const long long *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + // --- _Py_atomic_store ------------------------------------------------------ @@ -485,17 +517,57 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj, unsigned long long value) { __atomic_store_n(obj, value, __ATOMIC_RELAXED); } +static inline void +_Py_atomic_store_char_relaxed(char *obj, char value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +static inline void +_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_short_relaxed(short *obj, short value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_long_relaxed(long *obj, long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_float_relaxed(float *obj, float value) +{ __atomic_store(obj, &value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_double_relaxed(double *obj, double value) +{ __atomic_store(obj, &value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_llong_relaxed(long long *obj, long long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ static inline void * _Py_atomic_load_ptr_acquire(const void *obj) -{ return (void *)__atomic_load_n((void **)obj, __ATOMIC_ACQUIRE); } +{ return (void *)__atomic_load_n((void * const *)obj, __ATOMIC_ACQUIRE); } + +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +{ return (uintptr_t)__atomic_load_n(obj, __ATOMIC_ACQUIRE); } static inline void _Py_atomic_store_ptr_release(void *obj, void *value) { __atomic_store_n((void **)obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_int_release(int *obj, int value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } @@ -508,6 +580,10 @@ static inline int _Py_atomic_load_int_acquire(const int *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } @@ -530,6 +606,10 @@ static inline void _Py_atomic_fence_seq_cst(void) { __atomic_thread_fence(__ATOMIC_SEQ_CST); } + static inline void +_Py_atomic_fence_acquire(void) +{ __atomic_thread_fence(__ATOMIC_ACQUIRE); } + static inline void _Py_atomic_fence_release(void) { __atomic_thread_fence(__ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 3205e253b28546..d155955df0cddf 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -634,6 +634,48 @@ _Py_atomic_load_int_relaxed(const int *obj) return *(volatile int *)obj; } +static inline char +_Py_atomic_load_char_relaxed(const char *obj) +{ + return *(volatile char *)obj; +} + +static inline unsigned char +_Py_atomic_load_uchar_relaxed(const unsigned char *obj) +{ + return *(volatile unsigned char *)obj; +} + +static inline short +_Py_atomic_load_short_relaxed(const short *obj) +{ + return *(volatile short *)obj; +} + +static inline unsigned short +_Py_atomic_load_ushort_relaxed(const unsigned short *obj) +{ + return *(volatile unsigned short *)obj; +} + +static inline long +_Py_atomic_load_long_relaxed(const long *obj) +{ + return *(volatile long *)obj; +} + +static inline float +_Py_atomic_load_float_relaxed(const float *obj) +{ + return *(volatile float *)obj; +} + +static inline double +_Py_atomic_load_double_relaxed(const double *obj) +{ + return *(volatile double *)obj; +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -718,6 +760,12 @@ _Py_atomic_load_ullong_relaxed(const unsigned long long *obj) return *(volatile unsigned long long *)obj; } +static inline long long +_Py_atomic_load_llong_relaxed(const long long *obj) +{ + return *(volatile long long *)obj; +} + // --- _Py_atomic_store ------------------------------------------------------ @@ -899,6 +947,60 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj, *(volatile unsigned long long *)obj = value; } +static inline void +_Py_atomic_store_char_relaxed(char *obj, char value) +{ + *(volatile char *)obj = value; +} + +static inline void +_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value) +{ + *(volatile unsigned char *)obj = value; +} + +static inline void +_Py_atomic_store_short_relaxed(short *obj, short value) +{ + *(volatile short *)obj = value; +} + +static inline void +_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) +{ + *(volatile unsigned short *)obj = value; +} + +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ + *(volatile unsigned int *)obj = value; +} + +static inline void +_Py_atomic_store_long_relaxed(long *obj, long value) +{ + *(volatile long *)obj = value; +} + +static inline void +_Py_atomic_store_float_relaxed(float *obj, float value) +{ + *(volatile float *)obj = value; +} + +static inline void +_Py_atomic_store_double_relaxed(double *obj, double value) +{ + *(volatile double *)obj = value; +} + +static inline void +_Py_atomic_store_llong_relaxed(long long *obj, long long value) +{ + *(volatile long long *)obj = value; +} + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ @@ -914,6 +1016,18 @@ _Py_atomic_load_ptr_acquire(const void *obj) #endif } +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(uintptr_t volatile *)obj; +#elif defined(_M_ARM64) + return (uintptr_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uintptr_acquire" +#endif +} + static inline void _Py_atomic_store_ptr_release(void *obj, void *value) { @@ -926,6 +1040,19 @@ _Py_atomic_store_ptr_release(void *obj, void *value) #endif } +static inline void +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uintptr_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_uintptr_release" +#endif +} + static inline void _Py_atomic_store_int_release(int *obj, int value) { @@ -964,6 +1091,19 @@ _Py_atomic_load_int_acquire(const int *obj) #endif } +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uint32_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of _Py_atomic_store_uint32_release" +#endif +} + static inline void _Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) { @@ -1028,6 +1168,18 @@ _Py_atomic_fence_seq_cst(void) #else # error "no implementation of _Py_atomic_fence_seq_cst" #endif +} + + static inline void +_Py_atomic_fence_acquire(void) +{ +#if defined(_M_ARM64) + __dmb(_ARM64_BARRIER_ISHLD); +#elif defined(_M_X64) || defined(_M_IX86) + _ReadBarrier(); +#else +# error "no implementation of _Py_atomic_fence_acquire" +#endif } static inline void diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index f3970a45df24f9..69a8b9e615ea5f 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -515,6 +515,62 @@ _Py_atomic_load_int_relaxed(const int *obj) memory_order_relaxed); } +static inline char +_Py_atomic_load_char_relaxed(const char *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(char)*)obj, + memory_order_relaxed); +} + +static inline unsigned char +_Py_atomic_load_uchar_relaxed(const unsigned char *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(unsigned char)*)obj, + memory_order_relaxed); +} + +static inline short +_Py_atomic_load_short_relaxed(const short *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(short)*)obj, + memory_order_relaxed); +} + +static inline unsigned short +_Py_atomic_load_ushort_relaxed(const unsigned short *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(unsigned short)*)obj, + memory_order_relaxed); +} + +static inline long +_Py_atomic_load_long_relaxed(const long *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(long)*)obj, + memory_order_relaxed); +} + +static inline float +_Py_atomic_load_float_relaxed(const float *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(float)*)obj, + memory_order_relaxed); +} + +static inline double +_Py_atomic_load_double_relaxed(const double *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(double)*)obj, + memory_order_relaxed); +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -627,6 +683,14 @@ _Py_atomic_load_ullong_relaxed(const unsigned long long *obj) memory_order_relaxed); } +static inline long long +_Py_atomic_load_llong_relaxed(const long long *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(long long)*)obj, + memory_order_relaxed); +} + // --- _Py_atomic_store ------------------------------------------------------ @@ -852,6 +916,78 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj, memory_order_relaxed); } +static inline void +_Py_atomic_store_char_relaxed(char *obj, char value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(char)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned char)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_short_relaxed(short *obj, short value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(short)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned short)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned int)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_long_relaxed(long *obj, long value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(long)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_float_relaxed(float *obj, float value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(float)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_double_relaxed(double *obj, double value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(double)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_llong_relaxed(long long *obj, long long value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(long long)*)obj, value, + memory_order_relaxed); +} + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ @@ -863,6 +999,14 @@ _Py_atomic_load_ptr_acquire(const void *obj) memory_order_acquire); } +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uintptr_t)*)obj, + memory_order_acquire); +} + static inline void _Py_atomic_store_ptr_release(void *obj, void *value) { @@ -871,6 +1015,14 @@ _Py_atomic_store_ptr_release(void *obj, void *value) memory_order_release); } +static inline void +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uintptr_t)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_int_release(int *obj, int value) { @@ -895,6 +1047,14 @@ _Py_atomic_load_int_acquire(const int *obj) memory_order_acquire); } +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint32_t)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) { @@ -916,6 +1076,7 @@ _Py_atomic_load_uint32_acquire(const uint32_t *obj) { _Py_USING_STD; return atomic_load_explicit((const _Atomic(uint32_t)*)obj, + memory_order_acquire); } static inline Py_ssize_t @@ -923,6 +1084,7 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) { _Py_USING_STD; return atomic_load_explicit((const _Atomic(Py_ssize_t)*)obj, + memory_order_acquire); } @@ -935,6 +1097,13 @@ _Py_atomic_fence_seq_cst(void) atomic_thread_fence(memory_order_seq_cst); } + static inline void +_Py_atomic_fence_acquire(void) +{ + _Py_USING_STD; + atomic_thread_fence(memory_order_acquire); +} + static inline void _Py_atomic_fence_release(void) { diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 32c5884cd21341..b36b4681f5dddb 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -88,6 +88,10 @@ typedef PyOSErrorObject PyEnvironmentErrorObject; typedef PyOSErrorObject PyWindowsErrorObject; #endif +/* Context manipulation (PEP 3134) */ + +PyAPI_FUNC(void) _PyErr_ChainExceptions1(PyObject *); + /* In exceptions.c */ PyAPI_FUNC(PyObject*) PyUnstable_Exc_PrepReraiseStar( @@ -96,7 +100,7 @@ PyAPI_FUNC(PyObject*) PyUnstable_Exc_PrepReraiseStar( /* In signalmodule.c */ -int PySignal_SetWakeupFd(int fd); +PyAPI_FUNC(int) PySignal_SetWakeupFd(int fd); /* Support for adding program text to SyntaxErrors */ diff --git a/Include/cpython/pyframe.h b/Include/cpython/pyframe.h index c5adbbe4868f69..51529763923ec3 100644 --- a/Include/cpython/pyframe.h +++ b/Include/cpython/pyframe.h @@ -3,8 +3,10 @@ #endif PyAPI_DATA(PyTypeObject) PyFrame_Type; +PyAPI_DATA(PyTypeObject) PyFrameLocalsProxy_Type; #define PyFrame_Check(op) Py_IS_TYPE((op), &PyFrame_Type) +#define PyFrameLocalsProxy_Check(op) Py_IS_TYPE((op), &PyFrameLocalsProxy_Type) PyAPI_FUNC(PyFrameObject *) PyFrame_GetBack(PyFrameObject *frame); PyAPI_FUNC(PyObject *) PyFrame_GetLocals(PyFrameObject *frame); @@ -26,7 +28,7 @@ struct _PyInterpreterFrame; * Does not raise an exception. */ PyAPI_FUNC(PyObject *) PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame); -/* Returns a byte ofsset into the last executed instruction. +/* Returns a byte offset into the last executed instruction. * Does not raise an exception. */ PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame); diff --git a/Include/cpython/pyhash.h b/Include/cpython/pyhash.h index 396c208e1b106a..a33ba10b8d3a37 100644 --- a/Include/cpython/pyhash.h +++ b/Include/cpython/pyhash.h @@ -3,28 +3,32 @@ #endif /* Prime multiplier used in string and various other hashes. */ -#define _PyHASH_MULTIPLIER 1000003UL /* 0xf4243 */ +#define PyHASH_MULTIPLIER 1000003UL /* 0xf4243 */ /* Parameters used for the numeric hash implementation. See notes for _Py_HashDouble in Python/pyhash.c. Numeric hashes are based on reduction modulo the prime 2**_PyHASH_BITS - 1. */ #if SIZEOF_VOID_P >= 8 -# define _PyHASH_BITS 61 +# define PyHASH_BITS 61 #else -# define _PyHASH_BITS 31 +# define PyHASH_BITS 31 #endif -#define _PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) -#define _PyHASH_INF 314159 -#define _PyHASH_IMAG _PyHASH_MULTIPLIER +#define PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) +#define PyHASH_INF 314159 +#define PyHASH_IMAG PyHASH_MULTIPLIER + +/* Aliases kept for backward compatibility with Python 3.12 */ +#define _PyHASH_MULTIPLIER PyHASH_MULTIPLIER +#define _PyHASH_BITS PyHASH_BITS +#define _PyHASH_MODULUS PyHASH_MODULUS +#define _PyHASH_INF PyHASH_INF +#define _PyHASH_IMAG PyHASH_IMAG /* Helpers for hash functions */ PyAPI_FUNC(Py_hash_t) _Py_HashDouble(PyObject *, double); -// Kept for backward compatibility -#define _Py_HashPointer Py_HashPointer - /* hash function definition */ typedef struct { @@ -37,3 +41,14 @@ typedef struct { PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); PyAPI_FUNC(Py_hash_t) Py_HashPointer(const void *ptr); + +// Deprecated alias kept for backward compatibility +Py_DEPRECATED(3.14) static inline Py_hash_t +_Py_HashPointer(const void *ptr) +{ + return Py_HashPointer(ptr); +} + +PyAPI_FUNC(Py_hash_t) PyObject_GenericHash(PyObject *); + +PyAPI_FUNC(Py_hash_t) Py_HashBuffer(const void *ptr, Py_ssize_t len); diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index d425a233f71000..86ce6e6f79824a 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -25,9 +25,6 @@ PyAPI_FUNC(PyStatus) Py_PreInitializeFromArgs( PyAPI_FUNC(PyStatus) Py_InitializeFromConfig( const PyConfig *config); -// Python 3.8 provisional API (PEP 587) -PyAPI_FUNC(PyStatus) _Py_InitializeMain(void); - PyAPI_FUNC(int) Py_RunMain(void); @@ -63,6 +60,15 @@ typedef struct { .gil = PyInterpreterConfig_OWN_GIL, \ } +// gh-117649: The free-threaded build does not currently support single-phase +// init extensions in subinterpreters. For now, we ensure that +// `check_multi_interp_extensions` is always `1`, even in the legacy config. +#ifdef Py_GIL_DISABLED +# define _PyInterpreterConfig_LEGACY_CHECK_MULTI_INTERP_EXTENSIONS 1 +#else +# define _PyInterpreterConfig_LEGACY_CHECK_MULTI_INTERP_EXTENSIONS 0 +#endif + #define _PyInterpreterConfig_LEGACY_INIT \ { \ .use_main_obmalloc = 1, \ @@ -70,7 +76,7 @@ typedef struct { .allow_exec = 1, \ .allow_threads = 1, \ .allow_daemon_threads = 1, \ - .check_multi_interp_extensions = 0, \ + .check_multi_interp_extensions = _PyInterpreterConfig_LEGACY_CHECK_MULTI_INTERP_EXTENSIONS, \ .gil = PyInterpreterConfig_SHARED_GIL, \ } diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index ac7ff83748dbfc..cd6d9582496850 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -56,11 +56,6 @@ typedef struct _stack_chunk { PyObject * data[1]; /* Variable sized */ } _PyStackChunk; -struct _py_trashcan { - int delete_nesting; - PyObject *delete_later; -}; - struct _ts { /* See Python/ceval.c for comments explaining most fields */ @@ -88,6 +83,8 @@ struct _ts { unsigned int bound_gilstate:1; /* Currently in use (maybe holds the GIL). */ unsigned int active:1; + /* Currently holds the GIL. */ + unsigned int holds_gil:1; /* various stages of finalization */ unsigned int finalizing:1; @@ -95,15 +92,16 @@ struct _ts { unsigned int finalized:1; /* padding to align to 4 bytes */ - unsigned int :24; + unsigned int :23; } _status; #ifdef Py_BUILD_CORE # define _PyThreadState_WHENCE_NOTSET -1 # define _PyThreadState_WHENCE_UNKNOWN 0 -# define _PyThreadState_WHENCE_INTERP 1 -# define _PyThreadState_WHENCE_THREADING 2 -# define _PyThreadState_WHENCE_GILSTATE 3 -# define _PyThreadState_WHENCE_EXEC 4 +# define _PyThreadState_WHENCE_INIT 1 +# define _PyThreadState_WHENCE_FINI 2 +# define _PyThreadState_WHENCE_THREADING 3 +# define _PyThreadState_WHENCE_GILSTATE 4 +# define _PyThreadState_WHENCE_EXEC 5 #endif int _whence; @@ -152,7 +150,7 @@ struct _ts { */ unsigned long native_thread_id; - struct _py_trashcan trash; + PyObject *delete_later; /* Tagged pointer to top-most critical section, or zero if there is no * active critical section. Critical sections are only used in @@ -161,32 +159,6 @@ struct _ts { */ uintptr_t critical_section; - /* Called when a thread state is deleted normally, but not when it - * is destroyed after fork(). - * Pain: to prevent rare but fatal shutdown errors (issue 18808), - * Thread.join() must wait for the join'ed thread's tstate to be unlinked - * from the tstate chain. That happens at the end of a thread's life, - * in pystate.c. - * The obvious way doesn't quite work: create a lock which the tstate - * unlinking code releases, and have Thread.join() wait to acquire that - * lock. The problem is that we _are_ at the end of the thread's life: - * if the thread holds the last reference to the lock, decref'ing the - * lock will delete the lock, and that may trigger arbitrary Python code - * if there's a weakref, with a callback, to the lock. But by this time - * _PyRuntime.gilstate.tstate_current is already NULL, so only the simplest - * of C code can be allowed to run (in particular it must not be possible to - * release the GIL). - * So instead of holding the lock directly, the tstate holds a weakref to - * the lock: that's the value of on_delete_data below. Decref'ing a - * weakref is harmless. - * on_delete points to _threadmodule.c's static release_sentinel() function. - * After the tstate is unlinked, release_sentinel is called with the - * weakref-to-lock (on_delete_data) argument, and release_sentinel releases - * the indirectly held lock. - */ - void (*on_delete)(void *); - void *on_delete_data; - int coroutine_origin_tracking_depth; PyObject *async_gen_firstiter; @@ -219,26 +191,42 @@ struct _ts { PyObject *previous_executor; + uint64_t dict_global_version; + + /* Used to store/retrieve `threading.local` keys/values for this thread */ + PyObject *threading_local_key; + + /* Used by `threading.local`s to be remove keys/values for dying threads. + The PyThreadObject must hold the only reference to this value. + */ + PyObject *threading_local_sentinel; }; #ifdef Py_DEBUG // A debug build is likely built with low optimization level which implies // higher stack memory usage than a release build: use a lower limit. -# if defined(__wasi__) - // Based on wasmtime 16. -# define Py_C_RECURSION_LIMIT 150 -# else -# define Py_C_RECURSION_LIMIT 500 -# endif -#elif defined(__wasi__) - // Based on wasmtime 16. # define Py_C_RECURSION_LIMIT 500 #elif defined(__s390x__) # define Py_C_RECURSION_LIMIT 800 +#elif defined(_WIN32) && defined(_M_ARM64) +# define Py_C_RECURSION_LIMIT 1000 #elif defined(_WIN32) # define Py_C_RECURSION_LIMIT 3000 +#elif defined(__ANDROID__) + // On an ARM64 emulator, API level 34 was OK with 10000, but API level 21 + // crashed in test_compiler_recursion_limit. +# define Py_C_RECURSION_LIMIT 3000 #elif defined(_Py_ADDRESS_SANITIZER) # define Py_C_RECURSION_LIMIT 4000 +#elif defined(__sparc__) + // test_descr crashed on sparc64 with >7000 but let's keep a margin of error. +# define Py_C_RECURSION_LIMIT 4000 +#elif defined(__wasi__) + // Based on wasmtime 16. +# define Py_C_RECURSION_LIMIT 5000 +#elif defined(__hppa__) || defined(__powerpc64__) + // test_descr crashed with >8000 but let's keep a margin of error. +# define Py_C_RECURSION_LIMIT 5000 #else // This value is duplicated in Lib/test/support/__init__.py # define Py_C_RECURSION_LIMIT 10000 @@ -251,8 +239,12 @@ struct _ts { * if it is NULL. */ PyAPI_FUNC(PyThreadState *) PyThreadState_GetUnchecked(void); -// Alias kept for backward compatibility -#define _PyThreadState_UncheckedGet PyThreadState_GetUnchecked +// Deprecated alias kept for backward compatibility +Py_DEPRECATED(3.14) static inline PyThreadState* +_PyThreadState_UncheckedGet(void) +{ + return PyThreadState_GetUnchecked(); +} // Disable tracing and profiling. diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index 887fbbedf88502..ee8885cda7b60d 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -18,12 +18,20 @@ // // Define _PY_INTERPRETER macro to increment interpreter_increfs and // interpreter_decrefs. Otherwise, increment increfs and decrefs. +// +// The number of incref operations counted by `incref` and +// `interpreter_incref` is the number of increment operations, which is +// not equal to the total of all reference counts. A single increment +// operation may increase the reference count of an object by more than +// one. For example, see `_Py_RefcntAdd`. #ifndef Py_CPYTHON_PYSTATS_H # error "this header file must not be included directly" #endif -#define SPECIALIZATION_FAILURE_KINDS 36 +#define PYSTATS_MAX_UOP_ID 512 + +#define SPECIALIZATION_FAILURE_KINDS 37 /* Stats for determining who is calling PyEval_EvalFrame */ #define EVAL_CALL_TOTAL 0 @@ -68,6 +76,10 @@ typedef struct _object_stats { uint64_t decrefs; uint64_t interpreter_increfs; uint64_t interpreter_decrefs; + uint64_t immortal_increfs; + uint64_t immortal_decrefs; + uint64_t interpreter_immortal_increfs; + uint64_t interpreter_immortal_decrefs; uint64_t allocations; uint64_t allocations512; uint64_t allocations4k; @@ -75,12 +87,11 @@ typedef struct _object_stats { uint64_t frees; uint64_t to_freelist; uint64_t from_freelist; - uint64_t new_values; + uint64_t inline_values; uint64_t dict_materialized_on_request; uint64_t dict_materialized_new_key; uint64_t dict_materialized_too_big; uint64_t dict_materialized_str_subclass; - uint64_t dict_dematerialized; uint64_t type_cache_hits; uint64_t type_cache_misses; uint64_t type_cache_dunder_hits; @@ -94,11 +105,14 @@ typedef struct _gc_stats { uint64_t collections; uint64_t object_visits; uint64_t objects_collected; + uint64_t objects_transitively_reachable; + uint64_t objects_not_transitively_reachable; } GCStats; typedef struct _uop_stats { uint64_t execution_count; uint64_t miss; + uint64_t pair_count[PYSTATS_MAX_UOP_ID + 1]; } UOpStats; #define _Py_UOP_HIST_SIZE 32 @@ -116,7 +130,7 @@ typedef struct _optimization_stats { uint64_t recursive_call; uint64_t low_confidence; uint64_t executors_invalidated; - UOpStats opcode[512]; + UOpStats opcode[PYSTATS_MAX_UOP_ID + 1]; uint64_t unsupported_opcode[256]; uint64_t trace_length_hist[_Py_UOP_HIST_SIZE]; uint64_t trace_run_length_hist[_Py_UOP_HIST_SIZE]; @@ -124,6 +138,9 @@ typedef struct _optimization_stats { uint64_t optimizer_attempts; uint64_t optimizer_successes; uint64_t optimizer_failure_reason_no_memory; + uint64_t remove_globals_builtins_changed; + uint64_t remove_globals_incorrect_keys; + uint64_t error_in_opcode[PYSTATS_MAX_UOP_ID + 1]; } OptimizationStats; typedef struct _rare_event_stats { @@ -158,7 +175,11 @@ PyAPI_DATA(PyStats*) _Py_stats; #ifdef _PY_INTERPRETER # define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_increfs++; } while (0) # define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_decrefs++; } while (0) +# define _Py_INCREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_immortal_increfs++; } while (0) +# define _Py_DECREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_immortal_decrefs++; } while (0) #else # define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.increfs++; } while (0) # define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.decrefs++; } while (0) +# define _Py_INCREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.immortal_increfs++; } while (0) +# define _Py_DECREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.immortal_decrefs++; } while (0) #endif diff --git a/Include/cpython/pythread.h b/Include/cpython/pythread.h index 03f710a9f7ef2e..e658b35bd90700 100644 --- a/Include/cpython/pythread.h +++ b/Include/cpython/pythread.h @@ -22,7 +22,7 @@ PyAPI_DATA(const long long) PY_TIMEOUT_MAX; */ # define NATIVE_TSS_KEY_T unsigned long #elif defined(HAVE_PTHREAD_STUBS) -# include "cpython/pthread_stubs.h" +# include "pthread_stubs.h" # define NATIVE_TSS_KEY_T pthread_key_t #else # error "Require native threads. See https://bugs.python.org/issue31370" diff --git a/Include/cpython/pytime.h b/Include/cpython/pytime.h index d8244700d614ce..5c68110aeedb86 100644 --- a/Include/cpython/pytime.h +++ b/Include/cpython/pytime.h @@ -16,6 +16,10 @@ PyAPI_FUNC(int) PyTime_Monotonic(PyTime_t *result); PyAPI_FUNC(int) PyTime_PerfCounter(PyTime_t *result); PyAPI_FUNC(int) PyTime_Time(PyTime_t *result); +PyAPI_FUNC(int) PyTime_MonotonicRaw(PyTime_t *result); +PyAPI_FUNC(int) PyTime_PerfCounterRaw(PyTime_t *result); +PyAPI_FUNC(int) PyTime_TimeRaw(PyTime_t *result); + #ifdef __cplusplus } #endif diff --git a/Include/cpython/setobject.h b/Include/cpython/setobject.h index 1778c778a05324..89565cb29212fc 100644 --- a/Include/cpython/setobject.h +++ b/Include/cpython/setobject.h @@ -62,6 +62,10 @@ typedef struct { (assert(PyAnySet_Check(so)), _Py_CAST(PySetObject*, so)) static inline Py_ssize_t PySet_GET_SIZE(PyObject *so) { +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&(_PySet_CAST(so)->used)); +#else return _PySet_CAST(so)->used; +#endif } #define PySet_GET_SIZE(so) PySet_GET_SIZE(_PyObject_CAST(so)) diff --git a/Include/cpython/sysmodule.h b/Include/cpython/sysmodule.h deleted file mode 100644 index a3ac07f538a94f..00000000000000 --- a/Include/cpython/sysmodule.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef Py_CPYTHON_SYSMODULE_H -# error "this header file must not be included directly" -#endif - -typedef int(*Py_AuditHookFunction)(const char *, PyObject *, void *); - -PyAPI_FUNC(int) PySys_AddAuditHook(Py_AuditHookFunction, void*); - -typedef struct { - FILE* perf_map; - PyThread_type_lock map_lock; -} PerfMapState; - -PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void); -PyAPI_FUNC(int) PyUnstable_WritePerfMapEntry( - const void *code_addr, - unsigned int code_size, - const char *entry_name); -PyAPI_FUNC(void) PyUnstable_PerfMapState_Fini(void); -PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename); -PyAPI_FUNC(int) PyUnstable_PerfTrampoline_CompileCode(PyCodeObject *); -PyAPI_FUNC(int) PyUnstable_PerfTrampoline_SetPersistAfterFork(int enable); diff --git a/Include/cpython/tracemalloc.h b/Include/cpython/tracemalloc.h index 61a16ea9a9f3eb..6d094291ae2e90 100644 --- a/Include/cpython/tracemalloc.h +++ b/Include/cpython/tracemalloc.h @@ -1,6 +1,9 @@ #ifndef Py_LIMITED_API #ifndef Py_TRACEMALLOC_H #define Py_TRACEMALLOC_H +#ifdef __cplusplus +extern "C" { +#endif /* Track an allocated memory block in the tracemalloc module. Return 0 on success, return -1 on error (failed to allocate memory to store @@ -22,5 +25,8 @@ PyAPI_FUNC(int) PyTraceMalloc_Untrack( unsigned int domain, uintptr_t ptr); +#ifdef __cplusplus +} +#endif #endif // !Py_TRACEMALLOC_H #endif // !Py_LIMITED_API diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index d9b54bce83202d..287de52b96202c 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -109,7 +109,7 @@ typedef struct { 3: Interned, Immortal, and Static This categorization allows the runtime to determine the right cleanup mechanism at runtime shutdown. */ - unsigned int interned:2; + uint16_t interned; /* Character size: - PyUnicode_1BYTE_KIND (1): @@ -132,21 +132,23 @@ typedef struct { * all characters are in the range U+0000-U+10FFFF * at least one character is in the range U+10000-U+10FFFF */ - unsigned int kind:3; + unsigned short kind:3; /* Compact is with respect to the allocation scheme. Compact unicode objects only require one memory block while non-compact objects use one block for the PyUnicodeObject struct and another for its data buffer. */ - unsigned int compact:1; + unsigned short compact:1; /* The string only contains characters in the range U+0000-U+007F (ASCII) and the kind is PyUnicode_1BYTE_KIND. If ascii is set and compact is set, use the PyASCIIObject structure. */ - unsigned int ascii:1; + unsigned short ascii:1; /* The object is statically allocated. */ - unsigned int statically_allocated:1; + unsigned short statically_allocated:1; /* Padding to ensure that PyUnicode_DATA() is always aligned to - 4 bytes (see issue #19537 on m68k). */ - unsigned int :24; + 4 bytes (see issue #19537 on m68k) and we use unsigned short to avoid + the extra four bytes on 32-bit Windows. This is restricted features + for specific compilers including GCC, MSVC, Clang and IBM's XL compiler. */ + unsigned short :10; } state; } PyASCIIObject; @@ -195,7 +197,11 @@ typedef struct { /* Use only if you know it's a string */ static inline unsigned int PyUnicode_CHECK_INTERNED(PyObject *op) { +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_uint16_relaxed(&_PyASCIIObject_CAST(op)->state.interned); +#else return _PyASCIIObject_CAST(op)->state.interned; +#endif } #define PyUnicode_CHECK_INTERNED(op) PyUnicode_CHECK_INTERNED(_PyObject_CAST(op)) @@ -444,7 +450,54 @@ PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( Py_ssize_t size); -/* --- _PyUnicodeWriter API ----------------------------------------------- */ +/* --- Public PyUnicodeWriter API ----------------------------------------- */ + +typedef struct PyUnicodeWriter PyUnicodeWriter; + +PyAPI_FUNC(PyUnicodeWriter*) PyUnicodeWriter_Create(Py_ssize_t length); +PyAPI_FUNC(void) PyUnicodeWriter_Discard(PyUnicodeWriter *writer); +PyAPI_FUNC(PyObject*) PyUnicodeWriter_Finish(PyUnicodeWriter *writer); + +PyAPI_FUNC(int) PyUnicodeWriter_WriteChar( + PyUnicodeWriter *writer, + Py_UCS4 ch); +PyAPI_FUNC(int) PyUnicodeWriter_WriteUTF8( + PyUnicodeWriter *writer, + const char *str, + Py_ssize_t size); +PyAPI_FUNC(int) PyUnicodeWriter_WriteWideChar( + PyUnicodeWriter *writer, + const wchar_t *str, + Py_ssize_t size); +PyAPI_FUNC(int) PyUnicodeWriter_WriteUCS4( + PyUnicodeWriter *writer, + Py_UCS4 *str, + Py_ssize_t size); + +PyAPI_FUNC(int) PyUnicodeWriter_WriteStr( + PyUnicodeWriter *writer, + PyObject *obj); +PyAPI_FUNC(int) PyUnicodeWriter_WriteRepr( + PyUnicodeWriter *writer, + PyObject *obj); +PyAPI_FUNC(int) PyUnicodeWriter_WriteSubstring( + PyUnicodeWriter *writer, + PyObject *str, + Py_ssize_t start, + Py_ssize_t end); +PyAPI_FUNC(int) PyUnicodeWriter_Format( + PyUnicodeWriter *writer, + const char *format, + ...); +PyAPI_FUNC(int) PyUnicodeWriter_DecodeUTF8Stateful( + PyUnicodeWriter *writer, + const char *string, /* UTF-8 encoded string */ + Py_ssize_t length, /* size of string */ + const char *errors, /* error handling */ + Py_ssize_t *consumed); /* bytes consumed */ + + +/* --- Private _PyUnicodeWriter API --------------------------------------- */ typedef struct { PyObject *buffer; @@ -466,7 +519,7 @@ typedef struct { /* If readonly is 1, buffer is a shared string (cannot be modified) and size is set to 0. */ unsigned char readonly; -} _PyUnicodeWriter ; +} _PyUnicodeWriter; // Initialize a Unicode writer. // @@ -577,8 +630,12 @@ _PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer); PyAPI_FUNC(const char *) PyUnicode_AsUTF8(PyObject *unicode); -// Alias kept for backward compatibility -#define _PyUnicode_AsString PyUnicode_AsUTF8 +// Deprecated alias kept for backward compatibility +Py_DEPRECATED(3.14) static inline const char* +_PyUnicode_AsString(PyObject *unicode) +{ + return PyUnicode_AsUTF8(unicode); +} /* === Characters Type APIs =============================================== */ diff --git a/Include/cpython/weakrefobject.h b/Include/cpython/weakrefobject.h index 1559e2def61260..da8e77cddaca63 100644 --- a/Include/cpython/weakrefobject.h +++ b/Include/cpython/weakrefobject.h @@ -30,15 +30,28 @@ struct _PyWeakReference { PyWeakReference *wr_prev; PyWeakReference *wr_next; vectorcallfunc vectorcall; + +#ifdef Py_GIL_DISABLED + /* Pointer to the lock used when clearing in free-threaded builds. + * Normally this can be derived from wr_object, but in some cases we need + * to lock after wr_object has been set to Py_None. + */ + PyMutex *weakrefs_lock; +#endif }; +PyAPI_FUNC(void) _PyWeakref_ClearRef(PyWeakReference *self); + +#define _PyWeakref_CAST(op) \ + (assert(PyWeakref_Check(op)), _Py_CAST(PyWeakReference*, (op))) + +// Test if a weak reference is dead. +PyAPI_FUNC(int) PyWeakref_IsDead(PyObject *ref); + Py_DEPRECATED(3.13) static inline PyObject* PyWeakref_GET_OBJECT(PyObject *ref_obj) { - PyWeakReference *ref; - PyObject *obj; - assert(PyWeakref_Check(ref_obj)); - ref = _Py_CAST(PyWeakReference*, ref_obj); - obj = ref->wr_object; + PyWeakReference *ref = _PyWeakref_CAST(ref_obj); + PyObject *obj = ref->wr_object; // Explanation for the Py_REFCNT() check: when a weakref's target is part // of a long chain of deallocations which triggers the trashcan mechanism, // clearing the weakrefs can be delayed long after the target's refcount diff --git a/Include/critical_section.h b/Include/critical_section.h new file mode 100644 index 00000000000000..3b37615a8b17e2 --- /dev/null +++ b/Include/critical_section.h @@ -0,0 +1,16 @@ +#ifndef Py_CRITICAL_SECTION_H +#define Py_CRITICAL_SECTION_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_LIMITED_API +# define Py_CPYTHON_CRITICAL_SECTION_H +# include "cpython/critical_section.h" +# undef Py_CPYTHON_CRITICAL_SECTION_H +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_CRITICAL_SECTION_H */ diff --git a/Include/exports.h b/Include/exports.h index ce601216f17156..0c646d5beb6ad6 100644 --- a/Include/exports.h +++ b/Include/exports.h @@ -41,11 +41,8 @@ * we may still need to support gcc >= 4, as some Ubuntu LTS and Centos versions * have 4 < gcc < 5. */ - #ifndef __has_attribute - #define __has_attribute(x) 0 // Compatibility with non-clang compilers. - #endif #if (defined(__GNUC__) && (__GNUC__ >= 4)) ||\ - (defined(__clang__) && __has_attribute(visibility)) + (defined(__clang__) && _Py__has_attribute(visibility)) #define Py_IMPORTED_SYMBOL __attribute__ ((visibility ("default"))) #define Py_EXPORTED_SYMBOL __attribute__ ((visibility ("default"))) #define Py_LOCAL_SYMBOL __attribute__ ((visibility ("hidden"))) diff --git a/Include/floatobject.h b/Include/floatobject.h index 999441ac536e1d..4d24a76edd5de1 100644 --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -2,7 +2,7 @@ /* Float object interface */ /* -PyFloatObject represents a (double precision) floating point number. +PyFloatObject represents a (double precision) floating-point number. */ #ifndef Py_FLOATOBJECT_H @@ -21,10 +21,10 @@ PyAPI_DATA(PyTypeObject) PyFloat_Type; #define Py_RETURN_INF(sign) \ do { \ if (copysign(1., sign) == 1.) { \ - return PyFloat_FromDouble(Py_HUGE_VAL); \ + return PyFloat_FromDouble(Py_INFINITY); \ } \ else { \ - return PyFloat_FromDouble(-Py_HUGE_VAL); \ + return PyFloat_FromDouble(-Py_INFINITY); \ } \ } while(0) diff --git a/Include/internal/mimalloc/mimalloc/atomic.h b/Include/internal/mimalloc/mimalloc/atomic.h index eb8478ceed6adf..cdd9c372beafd5 100644 --- a/Include/internal/mimalloc/mimalloc/atomic.h +++ b/Include/internal/mimalloc/mimalloc/atomic.h @@ -11,7 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file // -------------------------------------------------------------------------------------------- // Atomics // We need to be portable between C, C++, and MSVC. -// We base the primitives on the C/C++ atomics and create a mimimal wrapper for MSVC in C compilation mode. +// We base the primitives on the C/C++ atomics and create a minimal wrapper for MSVC in C compilation mode. // This is why we try to use only `uintptr_t` and `*` as atomic types. // To gain better insight in the range of used atomics, we use explicitly named memory order operations // instead of passing the memory order as a parameter. @@ -23,7 +23,9 @@ terms of the MIT license. A copy of the license can be found in the file #define _Atomic(tp) std::atomic #define mi_atomic(name) std::atomic_##name #define mi_memory_order(name) std::memory_order_##name -#if !defined(ATOMIC_VAR_INIT) || (__cplusplus >= 202002L) // c++20, see issue #571 +#if (__cplusplus >= 202002L) // c++20, see issue #571 + #define MI_ATOMIC_VAR_INIT(x) x +#elif !defined(ATOMIC_VAR_INIT) #define MI_ATOMIC_VAR_INIT(x) x #else #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) @@ -39,7 +41,9 @@ terms of the MIT license. A copy of the license can be found in the file #include #define mi_atomic(name) atomic_##name #define mi_memory_order(name) memory_order_##name -#if !defined(ATOMIC_VAR_INIT) || (__STDC_VERSION__ >= 201710L) // c17, see issue #735 +#if (__STDC_VERSION__ >= 201710L) // c17, see issue #735 + #define MI_ATOMIC_VAR_INIT(x) x +#elif !defined(ATOMIC_VAR_INIT) #define MI_ATOMIC_VAR_INIT(x) x #else #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) diff --git a/Include/internal/mimalloc/mimalloc/internal.h b/Include/internal/mimalloc/mimalloc/internal.h index 8af841cfdffc01..d97f51b8eefbe5 100644 --- a/Include/internal/mimalloc/mimalloc/internal.h +++ b/Include/internal/mimalloc/mimalloc/internal.h @@ -10,12 +10,12 @@ terms of the MIT license. A copy of the license can be found in the file // -------------------------------------------------------------------------- -// This file contains the interal API's of mimalloc and various utility +// This file contains the internal API's of mimalloc and various utility // functions and macros. // -------------------------------------------------------------------------- -#include "mimalloc/types.h" -#include "mimalloc/track.h" +#include "types.h" +#include "track.h" #if (MI_DEBUG>0) #define mi_trace_message(...) _mi_trace_message(__VA_ARGS__) diff --git a/Include/internal/mimalloc/mimalloc/prim.h b/Include/internal/mimalloc/mimalloc/prim.h index 8a60d528458e6c..322ab29e6b41c2 100644 --- a/Include/internal/mimalloc/mimalloc/prim.h +++ b/Include/internal/mimalloc/mimalloc/prim.h @@ -151,9 +151,9 @@ static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { // If you test on another platform and it works please send a PR :-) // see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register. #elif defined(__GNUC__) && ( \ - (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \ + (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \ || (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__))) \ - || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \ + || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \ || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ || (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ ) diff --git a/Include/internal/mimalloc/mimalloc/types.h b/Include/internal/mimalloc/mimalloc/types.h index ed93e45062c2db..354839ba955b36 100644 --- a/Include/internal/mimalloc/mimalloc/types.h +++ b/Include/internal/mimalloc/mimalloc/types.h @@ -21,7 +21,7 @@ terms of the MIT license. A copy of the license can be found in the file #include // ptrdiff_t #include // uintptr_t, uint16_t, etc -#include "mimalloc/atomic.h" // _Atomic +#include "atomic.h" // _Atomic #ifdef _MSC_VER #pragma warning(disable:4214) // bitfield is not int @@ -311,6 +311,7 @@ typedef struct mi_page_s { uint32_t slice_offset; // distance from the actual page data slice (0 if a page) uint8_t is_committed : 1; // `true` if the page virtual memory is committed uint8_t is_zero_init : 1; // `true` if the page was initially zero initialized + uint8_t use_qsbr : 1; // delay page freeing using qsbr uint8_t tag : 4; // tag from the owning heap uint8_t debug_offset; // number of bytes to preserve when filling freed or uninitialized memory @@ -336,8 +337,13 @@ typedef struct mi_page_s { struct mi_page_s* next; // next page owned by this thread with the same `block_size` struct mi_page_s* prev; // previous page owned by this thread with the same `block_size` +#ifdef Py_GIL_DISABLED + struct llist_node qsbr_node; + uint64_t qsbr_goal; +#endif + // 64-bit 9 words, 32-bit 12 words, (+2 for secure) - #if MI_INTPTR_SIZE==8 + #if MI_INTPTR_SIZE==8 && !defined(Py_GIL_DISABLED) uintptr_t padding[1]; #endif } mi_page_t; @@ -555,6 +561,7 @@ struct mi_heap_s { bool no_reclaim; // `true` if this heap should not reclaim abandoned pages uint8_t tag; // custom identifier for this heap uint8_t debug_offset; // number of bytes to preserve when filling freed or uninitialized memory + bool page_use_qsbr; // should freeing pages be delayed using QSBR }; diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index f222d485e0b54b..f5bf1205a82be9 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -657,14 +657,17 @@ struct _type_param { struct { identifier name; expr_ty bound; + expr_ty default_value; } TypeVar; struct { identifier name; + expr_ty default_value; } ParamSpec; struct { identifier name; + expr_ty default_value; } TypeVarTuple; } v; @@ -892,14 +895,15 @@ pattern_ty _PyAST_MatchOr(asdl_pattern_seq * patterns, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); type_ignore_ty _PyAST_TypeIgnore(int lineno, string tag, PyArena *arena); -type_param_ty _PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int - col_offset, int end_lineno, int end_col_offset, - PyArena *arena); -type_param_ty _PyAST_ParamSpec(identifier name, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); -type_param_ty _PyAST_TypeVarTuple(identifier name, int lineno, int col_offset, - int end_lineno, int end_col_offset, PyArena - *arena); +type_param_ty _PyAST_TypeVar(identifier name, expr_ty bound, expr_ty + default_value, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +type_param_ty _PyAST_ParamSpec(identifier name, expr_ty default_value, int + lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +type_param_ty _PyAST_TypeVarTuple(identifier name, expr_ty default_value, int + lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); PyObject* PyAST_mod2obj(mod_ty t); diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index f1b1786264803b..09ae95465495c0 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -184,6 +184,7 @@ struct ast_state { PyObject *conversion; PyObject *ctx; PyObject *decorator_list; + PyObject *default_value; PyObject *defaults; PyObject *elt; PyObject *elts; diff --git a/Include/internal/pycore_atexit.h b/Include/internal/pycore_atexit.h index 4dcda8f517c787..db1e5568e09413 100644 --- a/Include/internal/pycore_atexit.h +++ b/Include/internal/pycore_atexit.h @@ -36,25 +36,30 @@ typedef struct atexit_callback { struct atexit_callback *next; } atexit_callback; -typedef struct { - PyObject *func; - PyObject *args; - PyObject *kwargs; -} atexit_py_callback; - struct atexit_state { +#ifdef Py_GIL_DISABLED + PyMutex ll_callbacks_lock; +#endif atexit_callback *ll_callbacks; - atexit_callback *last_ll_callback; // XXX The rest of the state could be moved to the atexit module state // and a low-level callback added for it during module exec. // For the moment we leave it here. - atexit_py_callback **callbacks; - int ncallbacks; - int callback_len; + + // List containing tuples with callback information. + // e.g. [(func, args, kwargs), ...] + PyObject *callbacks; }; -// Export for '_xxinterpchannels' shared extension +#ifdef Py_GIL_DISABLED +# define _PyAtExit_LockCallbacks(state) PyMutex_Lock(&state->ll_callbacks_lock); +# define _PyAtExit_UnlockCallbacks(state) PyMutex_Unlock(&state->ll_callbacks_lock); +#else +# define _PyAtExit_LockCallbacks(state) +# define _PyAtExit_UnlockCallbacks(state) +#endif + +// Export for '_interpchannels' shared extension PyAPI_FUNC(int) _Py_AtExit( PyInterpreterState *interp, atexit_datacallbackfunc func, diff --git a/Include/internal/pycore_audit.h b/Include/internal/pycore_audit.h new file mode 100644 index 00000000000000..2811aaa6236123 --- /dev/null +++ b/Include/internal/pycore_audit.h @@ -0,0 +1,35 @@ +#ifndef Py_INTERNAL_AUDIT_H +#define Py_INTERNAL_AUDIT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +/* Runtime audit hook state */ + +typedef struct _Py_AuditHookEntry { + struct _Py_AuditHookEntry *next; + Py_AuditHookFunction hookCFunction; + void *userData; +} _Py_AuditHookEntry; + + +extern int _PySys_Audit( + PyThreadState *tstate, + const char *event, + const char *argFormat, + ...); + +// _PySys_ClearAuditHooks() must not be exported: use extern rather than +// PyAPI_FUNC(). We want minimal exposure of this function. +extern void _PySys_ClearAuditHooks(PyThreadState *tstate); + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_AUDIT_H */ diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h new file mode 100644 index 00000000000000..b5e33fa8b7abc0 --- /dev/null +++ b/Include/internal/pycore_backoff.h @@ -0,0 +1,139 @@ + +#ifndef Py_INTERNAL_BACKOFF_H +#define Py_INTERNAL_BACKOFF_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include +#include +#include + + +typedef struct { + uint16_t value_and_backoff; +} _Py_BackoffCounter; + + +/* 16-bit countdown counters using exponential backoff. + + These are used by the adaptive specializer to count down until + it is time to specialize an instruction. If specialization fails + the counter is reset using exponential backoff. + + Another use is for the Tier 2 optimizer to decide when to create + a new Tier 2 trace (executor). Again, exponential backoff is used. + + The 16-bit counter is structured as a 12-bit unsigned 'value' + and a 4-bit 'backoff' field. When resetting the counter, the + backoff field is incremented (until it reaches a limit) and the + value is set to a bit mask representing the value 2**backoff - 1. + The maximum backoff is 12 (the number of bits in the value). + + There is an exceptional value which must not be updated, 0xFFFF. +*/ + +#define BACKOFF_BITS 4 +#define MAX_BACKOFF 12 +#define UNREACHABLE_BACKOFF 15 + +static inline bool +is_unreachable_backoff_counter(_Py_BackoffCounter counter) +{ + return counter.value_and_backoff == UNREACHABLE_BACKOFF; +} + +static inline _Py_BackoffCounter +make_backoff_counter(uint16_t value, uint16_t backoff) +{ + assert(backoff <= 15); + assert(value <= 0xFFF); + _Py_BackoffCounter result; + result.value_and_backoff = (value << BACKOFF_BITS) | backoff; + return result; +} + +static inline _Py_BackoffCounter +forge_backoff_counter(uint16_t counter) +{ + _Py_BackoffCounter result; + result.value_and_backoff = counter; + return result; +} + +static inline _Py_BackoffCounter +restart_backoff_counter(_Py_BackoffCounter counter) +{ + assert(!is_unreachable_backoff_counter(counter)); + int backoff = counter.value_and_backoff & 15; + if (backoff < MAX_BACKOFF) { + return make_backoff_counter((1 << (backoff + 1)) - 1, backoff + 1); + } + else { + return make_backoff_counter((1 << MAX_BACKOFF) - 1, MAX_BACKOFF); + } +} + +static inline _Py_BackoffCounter +pause_backoff_counter(_Py_BackoffCounter counter) +{ + _Py_BackoffCounter result; + result.value_and_backoff = counter.value_and_backoff | (1 << BACKOFF_BITS); + return result; +} + +static inline _Py_BackoffCounter +advance_backoff_counter(_Py_BackoffCounter counter) +{ + _Py_BackoffCounter result; + result.value_and_backoff = counter.value_and_backoff - (1 << BACKOFF_BITS); + return result; +} + +static inline bool +backoff_counter_triggers(_Py_BackoffCounter counter) +{ + /* Test whether the value is zero and the backoff is not UNREACHABLE_BACKOFF */ + return counter.value_and_backoff < UNREACHABLE_BACKOFF; +} + +/* Initial JUMP_BACKWARD counter. + * This determines when we create a trace for a loop. */ +#define JUMP_BACKWARD_INITIAL_VALUE 4095 +#define JUMP_BACKWARD_INITIAL_BACKOFF 12 +static inline _Py_BackoffCounter +initial_jump_backoff_counter(void) +{ + return make_backoff_counter(JUMP_BACKWARD_INITIAL_VALUE, + JUMP_BACKWARD_INITIAL_BACKOFF); +} + +/* Initial exit temperature. + * Must be larger than ADAPTIVE_COOLDOWN_VALUE, + * otherwise when a side exit warms up we may construct + * a new trace before the Tier 1 code has properly re-specialized. */ +#define SIDE_EXIT_INITIAL_VALUE 4095 +#define SIDE_EXIT_INITIAL_BACKOFF 12 + +static inline _Py_BackoffCounter +initial_temperature_backoff_counter(void) +{ + return make_backoff_counter(SIDE_EXIT_INITIAL_VALUE, + SIDE_EXIT_INITIAL_BACKOFF); +} + +/* Unreachable backoff counter. */ +static inline _Py_BackoffCounter +initial_unreachable_backoff_counter(void) +{ + return make_backoff_counter(0, UNREACHABLE_BACKOFF); +} + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_BACKOFF_H */ diff --git a/Include/internal/pycore_bytes_methods.h b/Include/internal/pycore_bytes_methods.h index 11e8ab20e91367..059dc2599bbd77 100644 --- a/Include/internal/pycore_bytes_methods.h +++ b/Include/internal/pycore_bytes_methods.h @@ -26,14 +26,23 @@ extern void _Py_bytes_title(char *result, const char *s, Py_ssize_t len); extern void _Py_bytes_capitalize(char *result, const char *s, Py_ssize_t len); extern void _Py_bytes_swapcase(char *result, const char *s, Py_ssize_t len); -extern PyObject *_Py_bytes_find(const char *str, Py_ssize_t len, PyObject *args); -extern PyObject *_Py_bytes_index(const char *str, Py_ssize_t len, PyObject *args); -extern PyObject *_Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *args); -extern PyObject *_Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *args); -extern PyObject *_Py_bytes_count(const char *str, Py_ssize_t len, PyObject *args); +extern PyObject *_Py_bytes_find(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); +extern PyObject *_Py_bytes_index(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); +extern PyObject *_Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); +extern PyObject *_Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); +extern PyObject *_Py_bytes_count(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); extern int _Py_bytes_contains(const char *str, Py_ssize_t len, PyObject *arg); -extern PyObject *_Py_bytes_startswith(const char *str, Py_ssize_t len, PyObject *args); -extern PyObject *_Py_bytes_endswith(const char *str, Py_ssize_t len, PyObject *args); +extern PyObject *_Py_bytes_startswith(const char *str, Py_ssize_t len, + PyObject *subobj, Py_ssize_t start, + Py_ssize_t end); +extern PyObject *_Py_bytes_endswith(const char *str, Py_ssize_t len, + PyObject *subobj, Py_ssize_t start, + Py_ssize_t end); /* The maketrans() static method. */ extern PyObject* _Py_bytes_maketrans(Py_buffer *frm, Py_buffer *to); diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h index 94d421a9eb742a..300e7f4896a39e 100644 --- a/Include/internal/pycore_bytesobject.h +++ b/Include/internal/pycore_bytesobject.h @@ -23,10 +23,6 @@ extern PyObject* _PyBytes_FromHex( PyAPI_FUNC(PyObject*) _PyBytes_DecodeEscape(const char *, Py_ssize_t, const char *, const char **); -/* _PyBytes_Join(sep, x) is like sep.join(x). sep must be PyBytesObject*, - x must be an iterable object. */ -extern PyObject* _PyBytes_Join(PyObject *sep, PyObject *x); - // Substring Search. // diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index c92028a01299e2..49f5c3322de267 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -8,7 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_identifier.h" // _Py_Identifier #include "pycore_pystate.h" // _PyThreadState_GET() /* Suggested size (number of positional arguments) for arrays of PyObject* diff --git a/Include/internal/pycore_cell.h b/Include/internal/pycore_cell.h new file mode 100644 index 00000000000000..27f67d57b2fb79 --- /dev/null +++ b/Include/internal/pycore_cell.h @@ -0,0 +1,48 @@ +#ifndef Py_INTERNAL_CELL_H +#define Py_INTERNAL_CELL_H + +#include "pycore_critical_section.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// Sets the cell contents to `value` and return previous contents. Steals a +// reference to `value`. +static inline PyObject * +PyCell_SwapTakeRef(PyCellObject *cell, PyObject *value) +{ + PyObject *old_value; + Py_BEGIN_CRITICAL_SECTION(cell); + old_value = cell->ob_ref; + cell->ob_ref = value; + Py_END_CRITICAL_SECTION(); + return old_value; +} + +static inline void +PyCell_SetTakeRef(PyCellObject *cell, PyObject *value) +{ + PyObject *old_value = PyCell_SwapTakeRef(cell, value); + Py_XDECREF(old_value); +} + +// Gets the cell contents. Returns a new reference. +static inline PyObject * +PyCell_GetRef(PyCellObject *cell) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(cell); + res = Py_XNewRef(cell->ob_ref); + Py_END_CRITICAL_SECTION(); + return res; +} + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_CELL_H */ diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 6eab2ba1daedf8..fea8665ae39ab5 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -48,8 +48,12 @@ extern void _PyEval_SignalReceived(void); #define _Py_PENDING_MAINTHREADONLY 1 #define _Py_PENDING_RAWFREE 2 +typedef int _Py_add_pending_call_result; +#define _Py_ADD_PENDING_SUCCESS 0 +#define _Py_ADD_PENDING_FULL -1 + // Export for '_testinternalcapi' shared extension -PyAPI_FUNC(int) _PyEval_AddPendingCall( +PyAPI_FUNC(_Py_add_pending_call_result) _PyEval_AddPendingCall( PyInterpreterState *interp, _Py_pending_call_func func, void *arg, @@ -79,9 +83,6 @@ extern void _PyEval_Fini(void); extern PyObject* _PyEval_GetBuiltins(PyThreadState *tstate); -extern PyObject* _PyEval_BuiltinsFromGlobals( - PyThreadState *tstate, - PyObject *globals); // Trampoline API @@ -104,6 +105,7 @@ extern int _PyIsPerfTrampolineActive(void); extern PyStatus _PyPerfTrampoline_AfterFork_Child(void); #ifdef PY_HAVE_PERF_TRAMPOLINE extern _PyPerf_Callbacks _Py_perfmap_callbacks; +extern _PyPerf_Callbacks _Py_perfmap_jit_callbacks; #endif static inline PyObject* @@ -127,7 +129,64 @@ extern void _PyEval_InitGIL(PyThreadState *tstate, int own_gil); extern void _PyEval_FiniGIL(PyInterpreterState *interp); extern void _PyEval_AcquireLock(PyThreadState *tstate); -extern void _PyEval_ReleaseLock(PyInterpreterState *, PyThreadState *); + +extern void _PyEval_ReleaseLock(PyInterpreterState *, PyThreadState *, + int final_release); + +#ifdef Py_GIL_DISABLED +// Returns 0 or 1 if the GIL for the given thread's interpreter is disabled or +// enabled, respectively. +// +// The enabled state of the GIL will not change while one or more threads are +// attached. +static inline int +_PyEval_IsGILEnabled(PyThreadState *tstate) +{ + struct _gil_runtime_state *gil = tstate->interp->ceval.gil; + return _Py_atomic_load_int_relaxed(&gil->enabled) != 0; +} + +// Enable or disable the GIL used by the interpreter that owns tstate, which +// must be the current thread. This may affect other interpreters, if the GIL +// is shared. All three functions will be no-ops (and return 0) if the +// interpreter's `enable_gil' config is not _PyConfig_GIL_DEFAULT. +// +// Every call to _PyEval_EnableGILTransient() must be paired with exactly one +// call to either _PyEval_EnableGILPermanent() or +// _PyEval_DisableGIL(). _PyEval_EnableGILPermanent() and _PyEval_DisableGIL() +// must only be called while the GIL is enabled from a call to +// _PyEval_EnableGILTransient(). +// +// _PyEval_EnableGILTransient() returns 1 if it enabled the GIL, or 0 if the +// GIL was already enabled, whether transiently or permanently. The caller will +// hold the GIL upon return. +// +// _PyEval_EnableGILPermanent() returns 1 if it permanently enabled the GIL +// (which must already be enabled), or 0 if it was already permanently +// enabled. Once _PyEval_EnableGILPermanent() has been called once, all +// subsequent calls to any of the three functions will be no-ops. +// +// _PyEval_DisableGIL() returns 1 if it disabled the GIL, or 0 if the GIL was +// kept enabled because of another request, whether transient or permanent. +// +// All three functions must be called by an attached thread (this implies that +// if the GIL is enabled, the current thread must hold it). +extern int _PyEval_EnableGILTransient(PyThreadState *tstate); +extern int _PyEval_EnableGILPermanent(PyThreadState *tstate); +extern int _PyEval_DisableGIL(PyThreadState *state); + + +static inline _Py_CODEUNIT * +_PyEval_GetExecutableCode(PyThreadState *tstate, PyCodeObject *co) +{ + _Py_CODEUNIT *bc = _PyCode_GetTLBCFast(tstate, co); + if (bc != NULL) { + return bc; + } + return _PyCode_GetTLBC(co); +} + +#endif extern void _PyEval_DeactivateOpCache(void); @@ -138,12 +197,12 @@ extern void _PyEval_DeactivateOpCache(void); /* With USE_STACKCHECK macro defined, trigger stack checks in _Py_CheckRecursiveCall() on every 64th call to _Py_EnterRecursiveCall. */ static inline int _Py_MakeRecCheck(PyThreadState *tstate) { - return (tstate->c_recursion_remaining-- <= 0 + return (tstate->c_recursion_remaining-- < 0 || (tstate->c_recursion_remaining & 63) == 0); } #else static inline int _Py_MakeRecCheck(PyThreadState *tstate) { - return tstate->c_recursion_remaining-- <= 0; + return tstate->c_recursion_remaining-- < 0; } #endif @@ -161,6 +220,11 @@ static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate, return (_Py_MakeRecCheck(tstate) && _Py_CheckRecursiveCall(tstate, where)); } +static inline void _Py_EnterRecursiveCallTstateUnchecked(PyThreadState *tstate) { + assert(tstate->c_recursion_remaining > 0); + tstate->c_recursion_remaining--; +} + static inline int _Py_EnterRecursiveCall(const char *where) { PyThreadState *tstate = _PyThreadState_GET(); return _Py_EnterRecursiveCallTstate(tstate, where); @@ -177,7 +241,7 @@ static inline void _Py_LeaveRecursiveCall(void) { extern struct _PyInterpreterFrame* _PyEval_GetFrame(void); -extern PyObject* _Py_MakeCoro(PyFunctionObject *func); +PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func); /* Handle signals, pending calls, GIL drop request and asynchronous exception */ @@ -190,18 +254,38 @@ typedef PyObject *(*conversion_func)(PyObject *); PyAPI_DATA(const binaryfunc) _PyEval_BinaryOps[]; PyAPI_DATA(const conversion_func) _PyEval_ConversionFuncs[]; +typedef struct _special_method { + PyObject *name; + const char *error; +} _Py_SpecialMethod; + +PyAPI_DATA(const _Py_SpecialMethod) _Py_SpecialMethods[]; +PyAPI_DATA(const size_t) _Py_FunctionAttributeOffsets[]; + PyAPI_FUNC(int) _PyEval_CheckExceptStarTypeValid(PyThreadState *tstate, PyObject* right); PyAPI_FUNC(int) _PyEval_CheckExceptTypeValid(PyThreadState *tstate, PyObject* right); -PyAPI_FUNC(int) _PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, PyObject **match, PyObject **rest); +PyAPI_FUNC(int) _PyEval_ExceptionGroupMatch(_PyInterpreterFrame *, PyObject* exc_value, PyObject *match_type, PyObject **match, PyObject **rest); PyAPI_FUNC(void) _PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg); PyAPI_FUNC(void) _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, const char *format_str, PyObject *obj); PyAPI_FUNC(void) _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg); PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs); +PyAPI_FUNC(PyObject *) _PyEval_ImportFrom(PyThreadState *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyEval_ImportName(PyThreadState *, _PyInterpreterFrame *, PyObject *, PyObject *, PyObject *); PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); -PyAPI_FUNC(int) _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, PyObject **sp); +PyAPI_FUNC(void) _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); +PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, _PyStackRef v, int argcnt, int argcntafter, _PyStackRef *sp); PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); +PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch); +PyAPI_FUNC(void) _PyObjectArray_Free(PyObject **array, PyObject **scratch); + +PyAPI_FUNC(PyObject *) _PyEval_GetANext(PyObject *aiter); +PyAPI_FUNC(void) _PyEval_LoadGlobalStackRef(PyObject *globals, PyObject *builtins, PyObject *name, _PyStackRef *writeto); +PyAPI_FUNC(PyObject *) _PyEval_GetAwaitable(PyObject *iterable, int oparg); +PyAPI_FUNC(PyObject *) _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *name); +PyAPI_FUNC(int) +_Py_Check_ArgsIterable(PyThreadState *tstate, PyObject *func, PyObject *args); /* Bits that can be set in PyThreadState.eval_breaker */ #define _PY_GIL_DROP_REQUEST_BIT (1U << 0) @@ -211,6 +295,7 @@ PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterF #define _PY_GC_SCHEDULED_BIT (1U << 4) #define _PY_EVAL_PLEASE_STOP_BIT (1U << 5) #define _PY_EVAL_EXPLICIT_MERGE_BIT (1U << 6) +#define _PY_EVAL_JIT_INVALIDATE_COLD_BIT (1U << 7) /* Reserve a few bits for future use */ #define _PY_EVAL_EVENTS_BITS 8 @@ -240,6 +325,8 @@ _Py_eval_breaker_bit_is_set(PyThreadState *tstate, uintptr_t bit) void _Py_set_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit); void _Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit); +PyAPI_FUNC(PyObject *) _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyStackRef right, double value); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index b453328f15649e..009a1ea41eb985 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -14,28 +14,58 @@ extern "C" { typedef int (*_Py_pending_call_func)(void *); +struct _pending_call { + _Py_pending_call_func func; + void *arg; + int flags; +}; + +#define PENDINGCALLSARRAYSIZE 300 + +#define MAXPENDINGCALLS PENDINGCALLSARRAYSIZE +/* For interpreter-level pending calls, we want to avoid spending too + much time on pending calls in any one thread, so we apply a limit. */ +#if MAXPENDINGCALLS > 100 +# define MAXPENDINGCALLSLOOP 100 +#else +# define MAXPENDINGCALLSLOOP MAXPENDINGCALLS +#endif + +/* We keep the number small to preserve as much compatibility + as possible with earlier versions. */ +#define MAXPENDINGCALLS_MAIN 32 +/* For the main thread, we want to make sure all pending calls are + run at once, for the sake of prompt signal handling. This is + unlikely to cause any problems since there should be very few + pending calls for the main thread. */ +#define MAXPENDINGCALLSLOOP_MAIN 0 + struct _pending_calls { - int busy; + PyThreadState *handling_thread; PyMutex mutex; /* Request for running pending calls. */ - int32_t calls_to_do; -#define NPENDINGCALLS 32 - struct _pending_call { - _Py_pending_call_func func; - void *arg; - int flags; - } calls[NPENDINGCALLS]; + int32_t npending; + /* The maximum allowed number of pending calls. + If the queue fills up to this point then _PyEval_AddPendingCall() + will return _Py_ADD_PENDING_FULL. */ + int32_t max; + /* We don't want a flood of pending calls to interrupt any one thread + for too long, so we keep a limit on the number handled per pass. + A value of 0 means there is no limit (other than the maximum + size of the list of pending calls). */ + int32_t maxloop; + struct _pending_call calls[PENDINGCALLSARRAYSIZE]; int first; - int last; + int next; }; + typedef enum { PERF_STATUS_FAILED = -1, // Perf trampoline is in an invalid state PERF_STATUS_NO_INIT = 0, // Perf trampoline is not initialized PERF_STATUS_OK = 1, // Perf trampoline is ready to be executed } perf_status_t; - #ifdef PY_HAVE_PERF_TRAMPOLINE struct code_arena_st; @@ -45,13 +75,16 @@ struct trampoline_api_st { unsigned int code_size, PyCodeObject* code); int (*free_state)(void* state); void *state; + Py_ssize_t code_padding; }; #endif + struct _ceval_runtime_state { struct { #ifdef PY_HAVE_PERF_TRAMPOLINE perf_status_t status; + int perf_trampoline_type; Py_ssize_t extra_code_index; struct code_arena_st *code_arena; struct trampoline_api_st trampoline_api; @@ -62,9 +95,15 @@ struct _ceval_runtime_state { #endif } perf; /* Pending calls to be made only on the main thread. */ + // The signal machinery falls back on this + // so it must be especially stable and efficient. + // For example, we use a preallocated array + // for the list of pending calls. struct _pending_calls pending_mainthread; + PyMutex sys_trace_profile_mutex; }; + #ifdef PY_HAVE_PERF_TRAMPOLINE # define _PyEval_RUNTIME_PERF_INIT \ { \ diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index a4e64825743ada..01d41446fdb0cf 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -8,6 +8,57 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_stackref.h" // _PyStackRef +#include "pycore_lock.h" // PyMutex +#include "pycore_backoff.h" // _Py_BackoffCounter +#include "pycore_tstate.h" // _PyThreadStateImpl + + +/* Each instruction in a code object is a fixed-width value, + * currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG + * opcode allows for larger values but the current limit is 3 uses + * of EXTENDED_ARG (see Python/compile.c), for a maximum + * 32-bit value. This aligns with the note in Python/compile.c + * (compiler_addop_i_line) indicating that the max oparg value is + * 2**32 - 1, rather than INT_MAX. + */ + +typedef union { + uint16_t cache; + struct { + uint8_t code; + uint8_t arg; + } op; + _Py_BackoffCounter counter; // First cache entry of specializable op +} _Py_CODEUNIT; + +#define _PyCode_CODE(CO) _Py_RVALUE((_Py_CODEUNIT *)(CO)->co_code_adaptive) +#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) + + +/* These macros only remain defined for compatibility. */ +#define _Py_OPCODE(word) ((word).op.code) +#define _Py_OPARG(word) ((word).op.arg) + +static inline _Py_CODEUNIT +_py_make_codeunit(uint8_t opcode, uint8_t oparg) +{ + // No designated initialisers because of C++ compat + _Py_CODEUNIT word; + word.op.code = opcode; + word.op.arg = oparg; + return word; +} + +static inline void +_py_set_opcode(_Py_CODEUNIT *word, uint8_t opcode) +{ + word->op.code = opcode; +} + +#define _Py_MAKE_CODEUNIT(opcode, oparg) _py_make_codeunit((opcode), (oparg)) +#define _Py_SET_OPCODE(word, opcode) _py_set_opcode(&(word), (opcode)) + // We hide some of the newer PyCodeObject fields behind macros. // This helps with backporting certain changes to 3.12. @@ -16,6 +67,14 @@ extern "C" { #define _PyCode_HAS_INSTRUMENTATION(CODE) \ (CODE->_co_instrumentation_version > 0) +struct _py_code_state { + PyMutex mutex; + // Interned constants from code objects. Used by the free-threaded build. + struct _Py_hashtable_t *constants; +}; + +extern PyStatus _PyCode_Init(PyInterpreterState *interp); +extern void _PyCode_Fini(PyInterpreterState *interp); #define CODE_MAX_WATCHERS 8 @@ -31,7 +90,7 @@ extern "C" { #define CACHE_ENTRIES(cache) (sizeof(cache)/sizeof(_Py_CODEUNIT)) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; uint16_t module_keys_version; uint16_t builtin_keys_version; uint16_t index; @@ -40,46 +99,50 @@ typedef struct { #define INLINE_CACHE_ENTRIES_LOAD_GLOBAL CACHE_ENTRIES(_PyLoadGlobalCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; + uint16_t external_cache[4]; } _PyBinaryOpCache; #define INLINE_CACHE_ENTRIES_BINARY_OP CACHE_ENTRIES(_PyBinaryOpCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyUnpackSequenceCache; #define INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE \ CACHE_ENTRIES(_PyUnpackSequenceCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyCompareOpCache; #define INLINE_CACHE_ENTRIES_COMPARE_OP CACHE_ENTRIES(_PyCompareOpCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyBinarySubscrCache; #define INLINE_CACHE_ENTRIES_BINARY_SUBSCR CACHE_ENTRIES(_PyBinarySubscrCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PySuperAttrCache; #define INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR CACHE_ENTRIES(_PySuperAttrCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; uint16_t version[2]; uint16_t index; } _PyAttrCache; typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; uint16_t type_version[2]; - uint16_t keys_version[2]; + union { + uint16_t keys_version[2]; + uint16_t dict_offset; + }; uint16_t descr[4]; } _PyLoadMethodCache; @@ -90,37 +153,44 @@ typedef struct { #define INLINE_CACHE_ENTRIES_STORE_ATTR CACHE_ENTRIES(_PyAttrCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; uint16_t func_version[2]; } _PyCallCache; #define INLINE_CACHE_ENTRIES_CALL CACHE_ENTRIES(_PyCallCache) +#define INLINE_CACHE_ENTRIES_CALL_KW CACHE_ENTRIES(_PyCallCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyStoreSubscrCache; #define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PyForIterCache; #define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; } _PySendCache; #define INLINE_CACHE_ENTRIES_SEND CACHE_ENTRIES(_PySendCache) typedef struct { - uint16_t counter; + _Py_BackoffCounter counter; uint16_t version[2]; } _PyToBoolCache; #define INLINE_CACHE_ENTRIES_TO_BOOL CACHE_ENTRIES(_PyToBoolCache) +typedef struct { + _Py_BackoffCounter counter; +} _PyContainsOpCache; + +#define INLINE_CACHE_ENTRIES_CONTAINS_OP CACHE_ENTRIES(_PyContainsOpCache) + // Borrowed references to common callables: struct callable_cache { PyObject *isinstance; @@ -245,43 +315,47 @@ extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); /** API for executors */ extern void _PyCode_Clear_Executors(PyCodeObject *code); + #ifdef Py_GIL_DISABLED // gh-115999 tracks progress on addressing this. #define ENABLE_SPECIALIZATION 0 +// Use this to enable specialization families once they are thread-safe. All +// uses will be replaced with ENABLE_SPECIALIZATION once all families are +// thread-safe. +#define ENABLE_SPECIALIZATION_FT 1 #else #define ENABLE_SPECIALIZATION 1 +#define ENABLE_SPECIALIZATION_FT ENABLE_SPECIALIZATION #endif /* Specialization functions */ -extern void _Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, +extern void _Py_Specialize_LoadSuperAttr(_PyStackRef global_super, _PyStackRef cls, _Py_CODEUNIT *instr, int load_method); -extern void _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, +extern void _Py_Specialize_LoadAttr(_PyStackRef owner, _Py_CODEUNIT *instr, PyObject *name); -extern void _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, +extern void _Py_Specialize_StoreAttr(_PyStackRef owner, _Py_CODEUNIT *instr, PyObject *name); extern void _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name); -extern void _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, +extern void _Py_Specialize_BinarySubscr(_PyStackRef sub, _PyStackRef container, _Py_CODEUNIT *instr); -extern void _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, +extern void _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub, _Py_CODEUNIT *instr); -extern void _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, +extern void _Py_Specialize_Call(_PyStackRef callable, _Py_CODEUNIT *instr, int nargs); -extern void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, - int oparg, PyObject **locals); -extern void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, +extern void _Py_Specialize_CallKw(_PyStackRef callable, _Py_CODEUNIT *instr, + int nargs); +extern void _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, + int oparg, _PyStackRef *locals); +extern void _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, +extern void _Py_Specialize_UnpackSequence(_PyStackRef seq, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr); -extern void _Py_Specialize_ToBool(PyObject *value, _Py_CODEUNIT *instr); - -/* Finalizer function for static codeobjects used in deepfreeze.py */ -extern void _PyStaticCode_Fini(PyCodeObject *co); -/* Function to intern strings of codeobjects and quicken the bytecode */ -extern int _PyStaticCode_Init(PyCodeObject *co); +extern void _Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg); +extern void _Py_Specialize_Send(_PyStackRef receiver, _Py_CODEUNIT *instr); +extern void _Py_Specialize_ToBool(_PyStackRef value, _Py_CODEUNIT *instr); +extern void _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *instr); #ifdef Py_STATS @@ -300,7 +374,15 @@ extern int _PyStaticCode_Init(PyCodeObject *co); #define GC_STAT_ADD(gen, name, n) do { if (_Py_stats) _Py_stats->gc_stats[(gen)].name += (n); } while (0) #define OPT_STAT_INC(name) do { if (_Py_stats) _Py_stats->optimization_stats.name++; } while (0) #define UOP_STAT_INC(opname, name) do { if (_Py_stats) { assert(opname < 512); _Py_stats->optimization_stats.opcode[opname].name++; } } while (0) +#define UOP_PAIR_INC(uopcode, lastuop) \ + do { \ + if (lastuop && _Py_stats) { \ + _Py_stats->optimization_stats.opcode[lastuop].pair_count[uopcode]++; \ + } \ + lastuop = uopcode; \ + } while (0) #define OPT_UNSUPPORTED_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.unsupported_opcode[opname]++; } while (0) +#define OPT_ERROR_IN_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.error_in_opcode[opname]++; } while (0) #define OPT_HIST(length, name) \ do { \ if (_Py_stats) { \ @@ -310,6 +392,7 @@ extern int _PyStaticCode_Init(PyCodeObject *co); } \ } while (0) #define RARE_EVENT_STAT_INC(name) do { if (_Py_stats) _Py_stats->rare_event_stats.name++; } while (0) +#define OPCODE_DEFERRED_INC(opname) do { if (_Py_stats && opcode == opname) _Py_stats->opcode_stats[opname].specialization.deferred++; } while (0) // Export for '_opcode' shared extension PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); @@ -326,9 +409,12 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define GC_STAT_ADD(gen, name, n) ((void)0) #define OPT_STAT_INC(name) ((void)0) #define UOP_STAT_INC(opname, name) ((void)0) +#define UOP_PAIR_INC(uopcode, lastuop) ((void)0) #define OPT_UNSUPPORTED_OPCODE(opname) ((void)0) +#define OPT_ERROR_IN_OPCODE(opname) ((void)0) #define OPT_HIST(length, name) ((void)0) #define RARE_EVENT_STAT_INC(name) ((void)0) +#define OPCODE_DEFERRED_INC(opname) ((void)0) #endif // !Py_STATS // Utility functions for reading/writing 32/64-bit values in the inline caches. @@ -353,7 +439,7 @@ write_u64(uint16_t *p, uint64_t val) } static inline void -write_obj(uint16_t *p, PyObject *val) +write_ptr(uint16_t *p, void *val) { memcpy(p, &val, sizeof(val)); } @@ -439,18 +525,14 @@ write_location_entry_start(uint8_t *ptr, int code, int length) /** Counters * The first 16-bit value in each inline cache is a counter. - * When counting misses, the counter is treated as a simple unsigned value. * * When counting executions until the next specialization attempt, * exponential backoff is used to reduce the number of specialization failures. - * The high 12 bits store the counter, the low 4 bits store the backoff exponent. - * On a specialization failure, the backoff exponent is incremented and the - * counter set to (2**backoff - 1). - * Backoff == 6 -> starting counter == 63, backoff == 10 -> starting counter == 1023. + * See pycore_backoff.h for more details. + * On a specialization failure, the backoff counter is restarted. */ -/* With a 16-bit counter, we have 12 bits for the counter value, and 4 bits for the backoff */ -#define ADAPTIVE_BACKOFF_BITS 4 +#include "pycore_backoff.h" // A value of 1 means that we attempt to specialize the *second* time each // instruction is executed. Executing twice is a much better indicator of @@ -468,38 +550,43 @@ write_location_entry_start(uint8_t *ptr, int code, int length) #define ADAPTIVE_COOLDOWN_VALUE 52 #define ADAPTIVE_COOLDOWN_BACKOFF 0 -#define MAX_BACKOFF_VALUE (16 - ADAPTIVE_BACKOFF_BITS) - +// Can't assert this in pycore_backoff.h because of header order dependencies +#if SIDE_EXIT_INITIAL_VALUE <= ADAPTIVE_COOLDOWN_VALUE +# error "Cold exit value should be larger than adaptive cooldown value" +#endif -static inline uint16_t +static inline _Py_BackoffCounter adaptive_counter_bits(uint16_t value, uint16_t backoff) { - return ((value << ADAPTIVE_BACKOFF_BITS) - | (backoff & ((1 << ADAPTIVE_BACKOFF_BITS) - 1))); + return make_backoff_counter(value, backoff); } -static inline uint16_t +static inline _Py_BackoffCounter adaptive_counter_warmup(void) { return adaptive_counter_bits(ADAPTIVE_WARMUP_VALUE, ADAPTIVE_WARMUP_BACKOFF); } -static inline uint16_t +static inline _Py_BackoffCounter adaptive_counter_cooldown(void) { return adaptive_counter_bits(ADAPTIVE_COOLDOWN_VALUE, ADAPTIVE_COOLDOWN_BACKOFF); } -static inline uint16_t -adaptive_counter_backoff(uint16_t counter) { - uint16_t backoff = counter & ((1 << ADAPTIVE_BACKOFF_BITS) - 1); - backoff++; - if (backoff > MAX_BACKOFF_VALUE) { - backoff = MAX_BACKOFF_VALUE; - } - uint16_t value = (uint16_t)(1 << backoff) - 1; - return adaptive_counter_bits(value, backoff); +static inline _Py_BackoffCounter +adaptive_counter_backoff(_Py_BackoffCounter counter) { + return restart_backoff_counter(counter); } +/* Specialization Extensions */ + +/* callbacks for an external specialization */ +typedef int (*binaryopguardfunc)(PyObject *lhs, PyObject *rhs); +typedef PyObject *(*binaryopactionfunc)(PyObject *lhs, PyObject *rhs); + +typedef struct { + binaryopguardfunc guard; + binaryopactionfunc action; +} _PyBinaryOpSpecializationDescr; /* Comparison bit masks. */ @@ -523,10 +610,57 @@ adaptive_counter_backoff(uint16_t counter) { extern int _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); -extern int _Py_GetBaseOpcode(PyCodeObject *code, int offset); +extern _Py_CODEUNIT _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); extern int _PyInstruction_GetLength(PyCodeObject *code, int offset); +extern PyObject *_PyInstrumentation_BranchesIterator(PyCodeObject *code); + +struct _PyCode8 _PyCode_DEF(8); + +PyAPI_DATA(const struct _PyCode8) _Py_InitCleanup; + +#ifdef Py_GIL_DISABLED + +static inline _PyCodeArray * +_PyCode_GetTLBCArray(PyCodeObject *co) +{ + return _Py_STATIC_CAST(_PyCodeArray *, + _Py_atomic_load_ptr_acquire(&co->co_tlbc)); +} + +// Return a pointer to the thread-local bytecode for the current thread, if it +// exists. +static inline _Py_CODEUNIT * +_PyCode_GetTLBCFast(PyThreadState *tstate, PyCodeObject *co) +{ + _PyCodeArray *code = _PyCode_GetTLBCArray(co); + int32_t idx = ((_PyThreadStateImpl*) tstate)->tlbc_index; + if (idx < code->size && code->entries[idx] != NULL) { + return (_Py_CODEUNIT *) code->entries[idx]; + } + return NULL; +} + +// Return a pointer to the thread-local bytecode for the current thread, +// creating it if necessary. +extern _Py_CODEUNIT *_PyCode_GetTLBC(PyCodeObject *co); + +// Reserve an index for the current thread into thread-local bytecode +// arrays +// +// Returns the reserved index or -1 on error. +extern int32_t _Py_ReserveTLBCIndex(PyInterpreterState *interp); + +// Release the current thread's index into thread-local bytecode arrays +extern void _Py_ClearTLBCIndex(_PyThreadStateImpl *tstate); + +// Free all TLBC copies not associated with live threads. +// +// Returns 0 on success or -1 on error. +extern int _Py_ClearUnusedTLBC(PyInterpreterState *interp); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_codecs.h b/Include/internal/pycore_codecs.h index a2a7151d50ade7..4400be8b33dee7 100644 --- a/Include/internal/pycore_codecs.h +++ b/Include/internal/pycore_codecs.h @@ -8,8 +8,30 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_lock.h" // PyMutex + +/* Initialize codecs-related state for the given interpreter, including + registering the first codec search function. Must be called before any other + PyCodec-related functions, and while only one thread is active. */ +extern PyStatus _PyCodec_InitRegistry(PyInterpreterState *interp); + +/* Finalize codecs-related state for the given interpreter. No PyCodec-related + functions other than PyCodec_Unregister() may be called after this. */ +extern void _PyCodec_Fini(PyInterpreterState *interp); + extern PyObject* _PyCodec_Lookup(const char *encoding); +/* + * Un-register the error handling callback function registered under + * the given 'name'. Only custom error handlers can be un-registered. + * + * - Return -1 and set an exception if 'name' refers to a built-in + * error handling name (e.g., 'strict'), or if an error occurred. + * - Return 0 if no custom error handler can be found for 'name'. + * - Return 1 if the custom error handler was successfully removed. + */ +extern int _PyCodec_UnregisterError(const char *name); + /* Text codec specific encoding and decoding API. Checks the encoding against a list of codecs which do not @@ -48,6 +70,26 @@ extern PyObject* _PyCodecInfo_GetIncrementalEncoder( PyObject *codec_info, const char *errors); +// Per-interpreter state used by codecs.c. +struct codecs_state { + // A list of callable objects used to search for codecs. + PyObject *search_path; + + // A dict mapping codec names to codecs returned from a callable in + // search_path. + PyObject *search_cache; + + // A dict mapping error handling strategies to functions to implement them. + PyObject *error_registry; + +#ifdef Py_GIL_DISABLED + // Used to safely delete a specific item from search_path. + PyMutex search_path_mutex; +#endif + + // Whether or not the rest of the state is initialized. + int initialized; +}; #ifdef __cplusplus } diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index e5870759ba74f1..9f0ca33892a43b 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -8,6 +8,12 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include + +#include "pycore_ast.h" // mod_ty +#include "pycore_symtable.h" // _Py_SourceLocation +#include "pycore_instruction_sequence.h" + struct _arena; // Type defined in pycore_pyarena.h struct _mod; // Type defined in pycore_ast.h @@ -27,45 +33,12 @@ extern int _PyCompile_AstOptimize( int optimize, struct _arena *arena); -static const _PyCompilerSrcLocation NO_LOCATION = {-1, -1, -1, -1}; - extern int _PyAST_Optimize( struct _mod *, struct _arena *arena, int optimize, int ff_features); -typedef struct { - int h_label; - int h_startdepth; - int h_preserve_lasti; -} _PyCompile_ExceptHandlerInfo; - -typedef struct { - int i_opcode; - int i_oparg; - _PyCompilerSrcLocation i_loc; - _PyCompile_ExceptHandlerInfo i_except_handler_info; - - /* Used by the assembler */ - int i_target; - int i_offset; -} _PyCompile_Instruction; - -typedef struct { - _PyCompile_Instruction *s_instrs; - int s_allocated; - int s_used; - - int *s_labelmap; /* label id --> instr offset */ - int s_labelmap_size; - int s_next_free_label; /* next free label id */ -} _PyCompile_InstructionSequence; - -int _PyCompile_InstructionSequence_UseLabel(_PyCompile_InstructionSequence *seq, int lbl); -int _PyCompile_InstructionSequence_Addop(_PyCompile_InstructionSequence *seq, - int opcode, int oparg, - _PyCompilerSrcLocation loc); typedef struct { PyObject *u_name; @@ -91,6 +64,115 @@ typedef struct { int u_firstlineno; /* the first lineno of the block */ } _PyCompile_CodeUnitMetadata; +struct _PyCompiler; + +typedef enum { + COMPILE_OP_FAST, + COMPILE_OP_GLOBAL, + COMPILE_OP_DEREF, + COMPILE_OP_NAME, +} _PyCompile_optype; + +/* _PyCompile_FBlockInfo tracks the current frame block. + * + * A frame block is used to handle loops, try/except, and try/finally. + * It's called a frame block to distinguish it from a basic block in the + * compiler IR. + */ + +enum _PyCompile_FBlockType { + COMPILE_FBLOCK_WHILE_LOOP, + COMPILE_FBLOCK_FOR_LOOP, + COMPILE_FBLOCK_TRY_EXCEPT, + COMPILE_FBLOCK_FINALLY_TRY, + COMPILE_FBLOCK_FINALLY_END, + COMPILE_FBLOCK_WITH, + COMPILE_FBLOCK_ASYNC_WITH, + COMPILE_FBLOCK_HANDLER_CLEANUP, + COMPILE_FBLOCK_POP_VALUE, + COMPILE_FBLOCK_EXCEPTION_HANDLER, + COMPILE_FBLOCK_EXCEPTION_GROUP_HANDLER, + COMPILE_FBLOCK_ASYNC_COMPREHENSION_GENERATOR, + COMPILE_FBLOCK_STOP_ITERATION, +}; + +typedef struct { + enum _PyCompile_FBlockType fb_type; + _PyJumpTargetLabel fb_block; + _Py_SourceLocation fb_loc; + /* (optional) type-specific exit or cleanup block */ + _PyJumpTargetLabel fb_exit; + /* (optional) additional information required for unwinding */ + void *fb_datum; +} _PyCompile_FBlockInfo; + + +int _PyCompile_PushFBlock(struct _PyCompiler *c, _Py_SourceLocation loc, + enum _PyCompile_FBlockType t, + _PyJumpTargetLabel block_label, + _PyJumpTargetLabel exit, void *datum); +void _PyCompile_PopFBlock(struct _PyCompiler *c, enum _PyCompile_FBlockType t, + _PyJumpTargetLabel block_label); +_PyCompile_FBlockInfo *_PyCompile_TopFBlock(struct _PyCompiler *c); + +int _PyCompile_EnterScope(struct _PyCompiler *c, identifier name, int scope_type, + void *key, int lineno, PyObject *private, + _PyCompile_CodeUnitMetadata *umd); +void _PyCompile_ExitScope(struct _PyCompiler *c); +Py_ssize_t _PyCompile_AddConst(struct _PyCompiler *c, PyObject *o); +_PyInstructionSequence *_PyCompile_InstrSequence(struct _PyCompiler *c); +int _PyCompile_FutureFeatures(struct _PyCompiler *c); +PyObject *_PyCompile_DeferredAnnotations(struct _PyCompiler *c); +PyObject *_PyCompile_Mangle(struct _PyCompiler *c, PyObject *name); +PyObject *_PyCompile_MaybeMangle(struct _PyCompiler *c, PyObject *name); +int _PyCompile_MaybeAddStaticAttributeToClass(struct _PyCompiler *c, expr_ty e); +int _PyCompile_GetRefType(struct _PyCompiler *c, PyObject *name); +int _PyCompile_LookupCellvar(struct _PyCompiler *c, PyObject *name); +int _PyCompile_ResolveNameop(struct _PyCompiler *c, PyObject *mangled, int scope, + _PyCompile_optype *optype, Py_ssize_t *arg); + +int _PyCompile_IsInteractiveTopLevel(struct _PyCompiler *c); +int _PyCompile_IsInInlinedComp(struct _PyCompiler *c); +int _PyCompile_ScopeType(struct _PyCompiler *c); +int _PyCompile_OptimizationLevel(struct _PyCompiler *c); +int _PyCompile_LookupArg(struct _PyCompiler *c, PyCodeObject *co, PyObject *name); +PyObject *_PyCompile_Qualname(struct _PyCompiler *c); +_PyCompile_CodeUnitMetadata *_PyCompile_Metadata(struct _PyCompiler *c); +PyObject *_PyCompile_StaticAttributesAsTuple(struct _PyCompiler *c); + +struct symtable *_PyCompile_Symtable(struct _PyCompiler *c); +PySTEntryObject *_PyCompile_SymtableEntry(struct _PyCompiler *c); + +enum { + COMPILE_SCOPE_MODULE, + COMPILE_SCOPE_CLASS, + COMPILE_SCOPE_FUNCTION, + COMPILE_SCOPE_ASYNC_FUNCTION, + COMPILE_SCOPE_LAMBDA, + COMPILE_SCOPE_COMPREHENSION, + COMPILE_SCOPE_ANNOTATIONS, +}; + + +typedef struct { + PyObject *pushed_locals; + PyObject *temp_symbols; + PyObject *fast_hidden; + _PyJumpTargetLabel cleanup; +} _PyCompile_InlinedComprehensionState; + +int _PyCompile_TweakInlinedComprehensionScopes(struct _PyCompiler *c, _Py_SourceLocation loc, + PySTEntryObject *entry, + _PyCompile_InlinedComprehensionState *state); +int _PyCompile_RevertInlinedComprehensionScopes(struct _PyCompiler *c, _Py_SourceLocation loc, + _PyCompile_InlinedComprehensionState *state); +int _PyCompile_AddDeferredAnnotaion(struct _PyCompiler *c, stmt_ty s); + +int _PyCodegen_AddReturnAtEnd(struct _PyCompiler *c, int addNone); +int _PyCodegen_EnterAnonymousScope(struct _PyCompiler* c, mod_ty mod); +int _PyCodegen_Expression(struct _PyCompiler *c, expr_ty e); +int _PyCodegen_Body(struct _PyCompiler *c, _Py_SourceLocation loc, asdl_stmt_seq *stmts, + bool is_interactive); /* Utility for a number of growing arrays used in the compiler */ int _PyCompile_EnsureArrayLargeEnough( @@ -102,17 +184,13 @@ int _PyCompile_EnsureArrayLargeEnough( int _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj); +PyCodeObject *_PyCompile_OptimizeAndAssemble(struct _PyCompiler *c, int addNone); -// Export for '_opcode' extention module -PyAPI_FUNC(int) _PyCompile_OpcodeIsValid(int opcode); -PyAPI_FUNC(int) _PyCompile_OpcodeHasArg(int opcode); -PyAPI_FUNC(int) _PyCompile_OpcodeHasConst(int opcode); -PyAPI_FUNC(int) _PyCompile_OpcodeHasName(int opcode); -PyAPI_FUNC(int) _PyCompile_OpcodeHasJump(int opcode); -PyAPI_FUNC(int) _PyCompile_OpcodeHasFree(int opcode); -PyAPI_FUNC(int) _PyCompile_OpcodeHasLocal(int opcode); -PyAPI_FUNC(int) _PyCompile_OpcodeHasExc(int opcode); +Py_ssize_t _PyCompile_DictAddObj(PyObject *dict, PyObject *o); +int _PyCompile_Error(struct _PyCompiler *c, _Py_SourceLocation loc, const char *format, ...); +int _PyCompile_Warn(struct _PyCompiler *c, _Py_SourceLocation loc, const char *format, ...); +// Export for '_opcode' extension module PyAPI_FUNC(PyObject*) _PyCompile_GetUnaryIntrinsicName(int index); PyAPI_FUNC(PyObject*) _PyCompile_GetBinaryIntrinsicName(int index); @@ -129,12 +207,6 @@ PyAPI_FUNC(PyObject*) _PyCompile_CodeGen( int optimize, int compile_mode); -// Export for '_testinternalcapi' shared extension -PyAPI_FUNC(PyObject*) _PyCompile_OptimizeCfg( - PyObject *instructions, - PyObject *consts, - int nlocals); - // Export for '_testinternalcapi' shared extension PyAPI_FUNC(PyCodeObject*) _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index ae5c47f195eb7f..c2b98d15da68fa 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -5,9 +5,9 @@ # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState #include "pycore_hamt.h" // PyHamtObject +#define CONTEXT_MAX_WATCHERS 8 extern PyTypeObject _PyContextTokenMissing_Type; @@ -35,9 +35,11 @@ struct _pycontextvarobject { PyObject_HEAD PyObject *var_name; PyObject *var_default; +#ifndef Py_GIL_DISABLED PyObject *var_cached; uint64_t var_cached_tsid; uint64_t var_cached_tsver; +#endif Py_hash_t var_hash; }; diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 9163b5cf0f2e8a..e66d6d805c1b3b 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -13,111 +13,48 @@ extern "C" { #endif -// Implementation of Python critical sections -// -// Conceptually, critical sections are a deadlock avoidance layer on top of -// per-object locks. These helpers, in combination with those locks, replace -// our usage of the global interpreter lock to provide thread-safety for -// otherwise thread-unsafe objects, such as dict. -// -// NOTE: These APIs are no-ops in non-free-threaded builds. -// -// Straightforward per-object locking could introduce deadlocks that were not -// present when running with the GIL. Threads may hold locks for multiple -// objects simultaneously because Python operations can nest. If threads were -// to acquire the same locks in different orders, they would deadlock. -// -// One way to avoid deadlocks is to allow threads to hold only the lock (or -// locks) for a single operation at a time (typically a single lock, but some -// operations involve two locks). When a thread begins a nested operation it -// could suspend the locks for any outer operation: before beginning the nested -// operation, the locks for the outer operation are released and when the -// nested operation completes, the locks for the outer operation are -// reacquired. -// -// To improve performance, this API uses a variation of the above scheme. -// Instead of immediately suspending locks any time a nested operation begins, -// locks are only suspended if the thread would block. This reduces the number -// of lock acquisitions and releases for nested operations, while still -// avoiding deadlocks. -// -// Additionally, the locks for any active operation are suspended around -// other potentially blocking operations, such as I/O. This is because the -// interaction between locks and blocking operations can lead to deadlocks in -// the same way as the interaction between multiple locks. -// -// Each thread's critical sections and their corresponding locks are tracked in -// a stack in `PyThreadState.critical_section`. When a thread calls -// `_PyThreadState_Detach()`, such as before a blocking I/O operation or when -// waiting to acquire a lock, the thread suspends all of its active critical -// sections, temporarily releasing the associated locks. When the thread calls -// `_PyThreadState_Attach()`, it resumes the top-most (i.e., most recent) -// critical section by reacquiring the associated lock or locks. See -// `_PyCriticalSection_Resume()`. -// -// NOTE: Only the top-most critical section is guaranteed to be active. -// Operations that need to lock two objects at once must use -// `Py_BEGIN_CRITICAL_SECTION2()`. You *CANNOT* use nested critical sections -// to lock more than one object at once, because the inner critical section -// may suspend the outer critical sections. This API does not provide a way -// to lock more than two objects at once (though it could be added later -// if actually needed). -// -// NOTE: Critical sections implicitly behave like reentrant locks because -// attempting to acquire the same lock will suspend any outer (earlier) -// critical sections. However, they are less efficient for this use case than -// purposefully designed reentrant locks. -// -// Example usage: -// Py_BEGIN_CRITICAL_SECTION(op); -// ... -// Py_END_CRITICAL_SECTION(); -// -// To lock two objects at once: -// Py_BEGIN_CRITICAL_SECTION2(op1, op2); -// ... -// Py_END_CRITICAL_SECTION2(); - - // Tagged pointers to critical sections use the two least significant bits to // mark if the pointed-to critical section is inactive and whether it is a -// _PyCriticalSection2 object. +// PyCriticalSection2 object. #define _Py_CRITICAL_SECTION_INACTIVE 0x1 #define _Py_CRITICAL_SECTION_TWO_MUTEXES 0x2 #define _Py_CRITICAL_SECTION_MASK 0x3 #ifdef Py_GIL_DISABLED -# define Py_BEGIN_CRITICAL_SECTION(op) \ +# define Py_BEGIN_CRITICAL_SECTION_MUT(mutex) \ { \ - _PyCriticalSection _cs; \ - _PyCriticalSection_Begin(&_cs, &_PyObject_CAST(op)->ob_mutex) - -# define Py_END_CRITICAL_SECTION() \ - _PyCriticalSection_End(&_cs); \ - } + PyCriticalSection _py_cs; \ + _PyCriticalSection_BeginMutex(&_py_cs, mutex) -# define Py_XBEGIN_CRITICAL_SECTION(op) \ +# define Py_BEGIN_CRITICAL_SECTION2_MUT(m1, m2) \ { \ - _PyCriticalSection _cs_opt = {0}; \ - _PyCriticalSection_XBegin(&_cs_opt, _PyObject_CAST(op)) - -# define Py_XEND_CRITICAL_SECTION() \ - _PyCriticalSection_XEnd(&_cs_opt); \ - } - -# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ + PyCriticalSection2 _py_cs2; \ + _PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) + +// Specialized version of critical section locking to safely use +// PySequence_Fast APIs without the GIL. For performance, the argument *to* +// PySequence_Fast() is provided to the macro, not the *result* of +// PySequence_Fast(), which would require an extra test to determine if the +// lock must be acquired. +# define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original) \ { \ - _PyCriticalSection2 _cs2; \ - _PyCriticalSection2_Begin(&_cs2, &_PyObject_CAST(a)->ob_mutex, &_PyObject_CAST(b)->ob_mutex) + PyObject *_orig_seq = _PyObject_CAST(original); \ + const bool _should_lock_cs = PyList_CheckExact(_orig_seq); \ + PyCriticalSection _cs; \ + if (_should_lock_cs) { \ + _PyCriticalSection_Begin(&_cs, _orig_seq); \ + } -# define Py_END_CRITICAL_SECTION2() \ - _PyCriticalSection2_End(&_cs2); \ +# define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() \ + if (_should_lock_cs) { \ + PyCriticalSection_End(&_cs); \ + } \ } // Asserts that the mutex is locked. The mutex must be held by the // top-most critical section otherwise there's the possibility // that the mutex would be swalled out in some code paths. -#define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) \ +#define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) \ _PyCriticalSection_AssertHeld(mutex) // Asserts that the mutex for the given object is locked. The mutex must @@ -125,72 +62,57 @@ extern "C" { // possibility that the mutex would be swalled out in some code paths. #ifdef Py_DEBUG -#define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) \ +# define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) \ if (Py_REFCNT(op) != 1) { \ _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(&_PyObject_CAST(op)->ob_mutex); \ } #else /* Py_DEBUG */ -#define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) +# define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) #endif /* Py_DEBUG */ #else /* !Py_GIL_DISABLED */ // The critical section APIs are no-ops with the GIL. -# define Py_BEGIN_CRITICAL_SECTION(op) -# define Py_END_CRITICAL_SECTION() -# define Py_XBEGIN_CRITICAL_SECTION(op) -# define Py_XEND_CRITICAL_SECTION() -# define Py_BEGIN_CRITICAL_SECTION2(a, b) -# define Py_END_CRITICAL_SECTION2() +# define Py_BEGIN_CRITICAL_SECTION_MUT(mut) { +# define Py_BEGIN_CRITICAL_SECTION2_MUT(m1, m2) { +# define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original) { +# define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() } # define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) # define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) #endif /* !Py_GIL_DISABLED */ -typedef struct { - // Tagged pointer to an outer active critical section (or 0). - // The two least-significant-bits indicate whether the pointed-to critical - // section is inactive and whether it is a _PyCriticalSection2 object. - uintptr_t prev; - - // Mutex used to protect critical section - PyMutex *mutex; -} _PyCriticalSection; - -// A critical section protected by two mutexes. Use -// _PyCriticalSection2_Begin and _PyCriticalSection2_End. -typedef struct { - _PyCriticalSection base; - - PyMutex *mutex2; -} _PyCriticalSection2; - -static inline int -_PyCriticalSection_IsActive(uintptr_t tag) -{ - return tag != 0 && (tag & _Py_CRITICAL_SECTION_INACTIVE) == 0; -} - // Resumes the top-most critical section. PyAPI_FUNC(void) _PyCriticalSection_Resume(PyThreadState *tstate); // (private) slow path for locking the mutex PyAPI_FUNC(void) -_PyCriticalSection_BeginSlow(_PyCriticalSection *c, PyMutex *m); +_PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m); PyAPI_FUNC(void) -_PyCriticalSection2_BeginSlow(_PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, +_PyCriticalSection2_BeginSlow(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, int is_m1_locked); +PyAPI_FUNC(void) +_PyCriticalSection_SuspendAll(PyThreadState *tstate); + +#ifdef Py_GIL_DISABLED + +static inline int +_PyCriticalSection_IsActive(uintptr_t tag) +{ + return tag != 0 && (tag & _Py_CRITICAL_SECTION_INACTIVE) == 0; +} + static inline void -_PyCriticalSection_Begin(_PyCriticalSection *c, PyMutex *m) +_PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m) { - if (PyMutex_LockFast(&m->v)) { + if (PyMutex_LockFast(m)) { PyThreadState *tstate = _PyThreadState_GET(); - c->mutex = m; - c->prev = tstate->critical_section; + c->_cs_mutex = m; + c->_cs_prev = tstate->critical_section; tstate->critical_section = (uintptr_t)c; } else { @@ -199,23 +121,20 @@ _PyCriticalSection_Begin(_PyCriticalSection *c, PyMutex *m) } static inline void -_PyCriticalSection_XBegin(_PyCriticalSection *c, PyObject *op) +_PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op) { -#ifdef Py_GIL_DISABLED - if (op != NULL) { - _PyCriticalSection_Begin(c, &_PyObject_CAST(op)->ob_mutex); - } -#endif + _PyCriticalSection_BeginMutex(c, &op->ob_mutex); } +#define PyCriticalSection_Begin _PyCriticalSection_Begin // Removes the top-most critical section from the thread's stack of critical // sections. If the new top-most critical section is inactive, then it is // resumed. static inline void -_PyCriticalSection_Pop(_PyCriticalSection *c) +_PyCriticalSection_Pop(PyCriticalSection *c) { PyThreadState *tstate = _PyThreadState_GET(); - uintptr_t prev = c->prev; + uintptr_t prev = c->_cs_prev; tstate->critical_section = prev; if ((prev & _Py_CRITICAL_SECTION_INACTIVE) != 0) { @@ -224,28 +143,27 @@ _PyCriticalSection_Pop(_PyCriticalSection *c) } static inline void -_PyCriticalSection_End(_PyCriticalSection *c) +_PyCriticalSection_End(PyCriticalSection *c) { - PyMutex_Unlock(c->mutex); - _PyCriticalSection_Pop(c); -} - -static inline void -_PyCriticalSection_XEnd(_PyCriticalSection *c) -{ - if (c->mutex) { - _PyCriticalSection_End(c); + // If the mutex is NULL, we used the fast path in + // _PyCriticalSection_BeginSlow for locks already held in the top-most + // critical section, and we shouldn't unlock or pop this critical section. + if (c->_cs_mutex == NULL) { + return; } + PyMutex_Unlock(c->_cs_mutex); + _PyCriticalSection_Pop(c); } +#define PyCriticalSection_End _PyCriticalSection_End static inline void -_PyCriticalSection2_Begin(_PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) +_PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) { if (m1 == m2) { // If the two mutex arguments are the same, treat this as a critical // section with a single mutex. - c->mutex2 = NULL; - _PyCriticalSection_Begin(&c->base, m1); + c->_cs_mutex2 = NULL; + _PyCriticalSection_BeginMutex(&c->_cs_base, m1); return; } @@ -258,12 +176,12 @@ _PyCriticalSection2_Begin(_PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) m2 = tmp; } - if (PyMutex_LockFast(&m1->v)) { - if (PyMutex_LockFast(&m2->v)) { + if (PyMutex_LockFast(m1)) { + if (PyMutex_LockFast(m2)) { PyThreadState *tstate = _PyThreadState_GET(); - c->base.mutex = m1; - c->mutex2 = m2; - c->base.prev = tstate->critical_section; + c->_cs_base._cs_mutex = m1; + c->_cs_mutex2 = m2; + c->_cs_base._cs_prev = tstate->critical_section; uintptr_t p = (uintptr_t)c | _Py_CRITICAL_SECTION_TWO_MUTEXES; tstate->critical_section = p; @@ -278,19 +196,30 @@ _PyCriticalSection2_Begin(_PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) } static inline void -_PyCriticalSection2_End(_PyCriticalSection2 *c) +_PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) { - if (c->mutex2) { - PyMutex_Unlock(c->mutex2); - } - PyMutex_Unlock(c->base.mutex); - _PyCriticalSection_Pop(&c->base); + _PyCriticalSection2_BeginMutex(c, &a->ob_mutex, &b->ob_mutex); } +#define PyCriticalSection2_Begin _PyCriticalSection2_Begin -PyAPI_FUNC(void) -_PyCriticalSection_SuspendAll(PyThreadState *tstate); - -#ifdef Py_GIL_DISABLED +static inline void +_PyCriticalSection2_End(PyCriticalSection2 *c) +{ + // if mutex1 is NULL, we used the fast path in + // _PyCriticalSection_BeginSlow for mutexes that are already held, + // which should only happen when mutex1 and mutex2 were the same mutex, + // and mutex2 should also be NULL. + if (c->_cs_base._cs_mutex == NULL) { + assert(c->_cs_mutex2 == NULL); + return; + } + if (c->_cs_mutex2) { + PyMutex_Unlock(c->_cs_mutex2); + } + PyMutex_Unlock(c->_cs_base._cs_mutex); + _PyCriticalSection_Pop(&c->_cs_base); +} +#define PyCriticalSection2_End _PyCriticalSection2_End static inline void _PyCriticalSection_AssertHeld(PyMutex *mutex) @@ -299,18 +228,18 @@ _PyCriticalSection_AssertHeld(PyMutex *mutex) PyThreadState *tstate = _PyThreadState_GET(); uintptr_t prev = tstate->critical_section; if (prev & _Py_CRITICAL_SECTION_TWO_MUTEXES) { - _PyCriticalSection2 *cs = (_PyCriticalSection2 *)(prev & ~_Py_CRITICAL_SECTION_MASK); - assert(cs != NULL && (cs->base.mutex == mutex || cs->mutex2 == mutex)); + PyCriticalSection2 *cs = (PyCriticalSection2 *)(prev & ~_Py_CRITICAL_SECTION_MASK); + assert(cs != NULL && (cs->_cs_base._cs_mutex == mutex || cs->_cs_mutex2 == mutex)); } else { - _PyCriticalSection *cs = (_PyCriticalSection *)(tstate->critical_section & ~_Py_CRITICAL_SECTION_MASK); - assert(cs != NULL && cs->mutex == mutex); + PyCriticalSection *cs = (PyCriticalSection *)(tstate->critical_section & ~_Py_CRITICAL_SECTION_MASK); + assert(cs != NULL && cs->_cs_mutex == mutex); } #endif } -#endif +#endif /* Py_GIL_DISABLED */ #ifdef __cplusplus } diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 63abef864ff87f..69a60d73e05c26 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -11,6 +11,7 @@ extern "C" { #include "pycore_lock.h" // PyMutex #include "pycore_pyerrors.h" + /**************/ /* exceptions */ /**************/ @@ -38,28 +39,28 @@ extern int _Py_CallInInterpreterAndRawFree( /* cross-interpreter data */ /**************************/ -typedef struct _xid _PyCrossInterpreterData; -typedef PyObject *(*xid_newobjectfunc)(_PyCrossInterpreterData *); +typedef struct _xidata _PyXIData_t; +typedef PyObject *(*xid_newobjfunc)(_PyXIData_t *); typedef void (*xid_freefunc)(void *); -// _PyCrossInterpreterData is similar to Py_buffer as an effectively +// _PyXIData_t is similar to Py_buffer as an effectively // opaque struct that holds data outside the object machinery. This // is necessary to pass safely between interpreters in the same process. -struct _xid { +struct _xidata { // data is the cross-interpreter-safe derivation of a Python object - // (see _PyObject_GetCrossInterpreterData). It will be NULL if the + // (see _PyObject_GetXIData). It will be NULL if the // new_object func (below) encodes the data. void *data; // obj is the Python object from which the data was derived. This // is non-NULL only if the data remains bound to the object in some // way, such that the object must be "released" (via a decref) when // the data is released. In that case the code that sets the field, - // likely a registered "crossinterpdatafunc", is responsible for + // likely a registered "xidatafunc", is responsible for // ensuring it owns the reference (i.e. incref). PyObject *obj; // interp is the ID of the owning interpreter of the original // object. It corresponds to the active interpreter when - // _PyObject_GetCrossInterpreterData() was called. This should only + // _PyObject_GetXIData() was called. This should only // be set by the cross-interpreter machinery. // // We use the ID rather than the PyInterpreterState to avoid issues @@ -71,130 +72,138 @@ struct _xid { // interpreter given the data. The resulting object (a new // reference) will be equivalent to the original object. This field // is required. - xid_newobjectfunc new_object; + xid_newobjfunc new_object; // free is called when the data is released. If it is NULL then // nothing will be done to free the data. For some types this is // okay (e.g. bytes) and for those types this field should be set // to NULL. However, for most the data was allocated just for // cross-interpreter use, so it must be freed when - // _PyCrossInterpreterData_Release is called or the memory will + // _PyXIData_Release is called or the memory will // leak. In that case, at the very least this field should be set // to PyMem_RawFree (the default if not explicitly set to NULL). // The call will happen with the original interpreter activated. xid_freefunc free; }; -PyAPI_FUNC(_PyCrossInterpreterData *) _PyCrossInterpreterData_New(void); -PyAPI_FUNC(void) _PyCrossInterpreterData_Free(_PyCrossInterpreterData *data); +PyAPI_FUNC(_PyXIData_t *) _PyXIData_New(void); +PyAPI_FUNC(void) _PyXIData_Free(_PyXIData_t *data); -#define _PyCrossInterpreterData_DATA(DATA) ((DATA)->data) -#define _PyCrossInterpreterData_OBJ(DATA) ((DATA)->obj) -#define _PyCrossInterpreterData_INTERPID(DATA) ((DATA)->interpid) +#define _PyXIData_DATA(DATA) ((DATA)->data) +#define _PyXIData_OBJ(DATA) ((DATA)->obj) +#define _PyXIData_INTERPID(DATA) ((DATA)->interpid) // Users should not need getters for "new_object" or "free". +/* getting cross-interpreter data */ + +typedef int (*xidatafunc)(PyThreadState *tstate, PyObject *, _PyXIData_t *); + +typedef struct _xid_lookup_state _PyXIData_lookup_t; + +typedef struct { + _PyXIData_lookup_t *global; + _PyXIData_lookup_t *local; + PyObject *PyExc_NotShareableError; +} _PyXIData_lookup_context_t; + +PyAPI_FUNC(int) _PyXIData_GetLookupContext( + PyInterpreterState *, + _PyXIData_lookup_context_t *); + +PyAPI_FUNC(xidatafunc) _PyXIData_Lookup( + _PyXIData_lookup_context_t *, + PyObject *); +PyAPI_FUNC(int) _PyObject_CheckXIData( + _PyXIData_lookup_context_t *, + PyObject *); +PyAPI_FUNC(int) _PyObject_GetXIData( + _PyXIData_lookup_context_t *, + PyObject *, + _PyXIData_t *); + + +/* using cross-interpreter data */ + +PyAPI_FUNC(PyObject *) _PyXIData_NewObject(_PyXIData_t *); +PyAPI_FUNC(int) _PyXIData_Release(_PyXIData_t *); +PyAPI_FUNC(int) _PyXIData_ReleaseAndRawFree(_PyXIData_t *); + + /* defining cross-interpreter data */ -PyAPI_FUNC(void) _PyCrossInterpreterData_Init( - _PyCrossInterpreterData *data, +PyAPI_FUNC(void) _PyXIData_Init( + _PyXIData_t *data, PyInterpreterState *interp, void *shared, PyObject *obj, - xid_newobjectfunc new_object); -PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize( - _PyCrossInterpreterData *, + xid_newobjfunc new_object); +PyAPI_FUNC(int) _PyXIData_InitWithSize( + _PyXIData_t *, PyInterpreterState *interp, const size_t, PyObject *, - xid_newobjectfunc); -PyAPI_FUNC(void) _PyCrossInterpreterData_Clear( - PyInterpreterState *, _PyCrossInterpreterData *); + xid_newobjfunc); +PyAPI_FUNC(void) _PyXIData_Clear( PyInterpreterState *, _PyXIData_t *); // Normally the Init* functions are sufficient. The only time // additional initialization might be needed is to set the "free" func, // though that should be infrequent. -#define _PyCrossInterpreterData_SET_FREE(DATA, FUNC) \ +#define _PyXIData_SET_FREE(DATA, FUNC) \ do { \ (DATA)->free = (FUNC); \ } while (0) // Additionally, some shareable types are essentially light wrappers -// around other shareable types. The crossinterpdatafunc of the wrapper +// around other shareable types. The xidatafunc of the wrapper // can often be implemented by calling the wrapped object's -// crossinterpdatafunc and then changing the "new_object" function. -// We have _PyCrossInterpreterData_SET_NEW_OBJECT() here for that, +// xidatafunc and then changing the "new_object" function. +// We have _PyXIData_SET_NEW_OBJECT() here for that, // but might be better to have a function like -// _PyCrossInterpreterData_AdaptToWrapper() instead. -#define _PyCrossInterpreterData_SET_NEW_OBJECT(DATA, FUNC) \ +// _PyXIData_AdaptToWrapper() instead. +#define _PyXIData_SET_NEW_OBJECT(DATA, FUNC) \ do { \ (DATA)->new_object = (FUNC); \ } while (0) -/* using cross-interpreter data */ - -PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); -PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); -PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); -PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); -PyAPI_FUNC(int) _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *); - - /* cross-interpreter data registry */ -// For now we use a global registry of shareable classes. An -// alternative would be to add a tp_* slot for a class's -// crossinterpdatafunc. It would be simpler and more efficient. - -typedef int (*crossinterpdatafunc)(PyThreadState *tstate, PyObject *, - _PyCrossInterpreterData *); - -struct _xidregitem; - -struct _xidregitem { - struct _xidregitem *prev; - struct _xidregitem *next; - /* This can be a dangling pointer, but only if weakref is set. */ - PyTypeObject *cls; - /* This is NULL for builtin types. */ - PyObject *weakref; - size_t refcount; - crossinterpdatafunc getdata; -}; - -struct _xidregistry { - int global; /* builtin types or heap types */ - int initialized; - PyMutex mutex; - struct _xidregitem *head; -}; - -PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc); -PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *); -PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); +#define Py_CORE_CROSSINTERP_DATA_REGISTRY_H +#include "pycore_crossinterp_data_registry.h" +#undef Py_CORE_CROSSINTERP_DATA_REGISTRY_H /*****************************/ /* runtime state & lifecycle */ /*****************************/ -struct _xi_runtime_state { +typedef struct { // builtin types - // XXX Remove this field once we have a tp_* slot. - struct _xidregistry registry; -}; + _PyXIData_lookup_t data_lookup; +} _PyXI_global_state_t; -struct _xi_state { +typedef struct { // heap types - // XXX Remove this field once we have a tp_* slot. - struct _xidregistry registry; + _PyXIData_lookup_t data_lookup; - // heap types - PyObject *PyExc_NotShareableError; -}; + struct xi_exceptions { + // static types + PyObject *PyExc_InterpreterError; + PyObject *PyExc_InterpreterNotFoundError; + // heap types + PyObject *PyExc_NotShareableError; + } exceptions; +} _PyXI_state_t; + +#define _PyXI_GET_GLOBAL_STATE(interp) (&(interp)->runtime->xi) +#define _PyXI_GET_STATE(interp) (&(interp)->xi) +#ifndef Py_BUILD_CORE_MODULE extern PyStatus _PyXI_Init(PyInterpreterState *interp); extern void _PyXI_Fini(PyInterpreterState *interp); - extern PyStatus _PyXI_InitTypes(PyInterpreterState *interp); extern void _PyXI_FiniTypes(PyInterpreterState *interp); +#endif // Py_BUILD_CORE_MODULE -#define _PyInterpreterState_GetXIState(interp) (&(interp)->xi) +int _Py_xi_global_state_init(_PyXI_global_state_t *); +void _Py_xi_global_state_fini(_PyXI_global_state_t *); +int _Py_xi_state_init(_PyXI_state_t *, PyInterpreterState *); +void _Py_xi_state_fini(_PyXI_state_t *, PyInterpreterState *); /***************************/ @@ -217,6 +226,11 @@ typedef struct _excinfo { const char *errdisplay; } _PyXI_excinfo; +PyAPI_FUNC(int) _PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc); +PyAPI_FUNC(PyObject *) _PyXI_FormatExcInfo(_PyXI_excinfo *info); +PyAPI_FUNC(PyObject *) _PyXI_ExcInfoAsObject(_PyXI_excinfo *info); +PyAPI_FUNC(void) _PyXI_ClearExcInfo(_PyXI_excinfo *info); + typedef enum error_code { _PyXI_ERR_NO_ERROR = 0, @@ -313,6 +327,22 @@ PyAPI_FUNC(PyObject *) _PyXI_ApplyCapturedException(_PyXI_session *session); PyAPI_FUNC(int) _PyXI_HasCapturedException(_PyXI_session *session); +/*************/ +/* other API */ +/*************/ + +// Export for _testinternalcapi shared extension +PyAPI_FUNC(PyInterpreterState *) _PyXI_NewInterpreter( + PyInterpreterConfig *config, + long *maybe_whence, + PyThreadState **p_tstate, + PyThreadState **p_save_tstate); +PyAPI_FUNC(void) _PyXI_EndInterpreter( + PyInterpreterState *interp, + PyThreadState *tstate, + PyThreadState **p_save_tstate); + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_crossinterp_data_registry.h b/Include/internal/pycore_crossinterp_data_registry.h new file mode 100644 index 00000000000000..bbad4de770857f --- /dev/null +++ b/Include/internal/pycore_crossinterp_data_registry.h @@ -0,0 +1,41 @@ +#ifndef Py_CORE_CROSSINTERP_DATA_REGISTRY_H +# error "this header must not be included directly" +#endif + + +// For now we use a global registry of shareable classes. An +// alternative would be to add a tp_* slot for a class's +// xidatafunc. It would be simpler and more efficient. + +struct _xid_regitem; + +typedef struct _xid_regitem { + struct _xid_regitem *prev; + struct _xid_regitem *next; + /* This can be a dangling pointer, but only if weakref is set. */ + PyTypeObject *cls; + /* This is NULL for builtin types. */ + PyObject *weakref; + size_t refcount; + xidatafunc getdata; +} _PyXIData_regitem_t; + +typedef struct { + int global; /* builtin types or heap types */ + int initialized; + PyMutex mutex; + _PyXIData_regitem_t *head; +} _PyXIData_registry_t; + +PyAPI_FUNC(int) _PyXIData_RegisterClass( + _PyXIData_lookup_context_t *, + PyTypeObject *, + xidatafunc); +PyAPI_FUNC(int) _PyXIData_UnregisterClass( + _PyXIData_lookup_context_t *, + PyTypeObject *); + +struct _xid_lookup_state { + // XXX Remove this field once we have a tp_* slot. + _PyXIData_registry_t registry; +}; diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h new file mode 100644 index 00000000000000..44feb079571a73 --- /dev/null +++ b/Include/internal/pycore_debug_offsets.h @@ -0,0 +1,335 @@ +#ifndef Py_INTERNAL_DEBUG_OFFSETS_H +#define Py_INTERNAL_DEBUG_OFFSETS_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +#define _Py_Debug_Cookie "xdebugpy" + +#if defined(__APPLE__) +# include +#endif + +// Macros to burn global values in custom sections so out-of-process +// profilers can locate them easily. +#define GENERATE_DEBUG_SECTION(name, declaration) \ + _GENERATE_DEBUG_SECTION_WINDOWS(name) \ + _GENERATE_DEBUG_SECTION_APPLE(name) \ + declaration \ + _GENERATE_DEBUG_SECTION_LINUX(name) + +#if defined(MS_WINDOWS) +#define _GENERATE_DEBUG_SECTION_WINDOWS(name) \ + _Pragma(Py_STRINGIFY(section(Py_STRINGIFY(name), read, write))) \ + __declspec(allocate(Py_STRINGIFY(name))) +#else +#define _GENERATE_DEBUG_SECTION_WINDOWS(name) +#endif + +#if defined(__APPLE__) +#define _GENERATE_DEBUG_SECTION_APPLE(name) \ + __attribute__((section(SEG_DATA "," Py_STRINGIFY(name)))) \ + __attribute__((used)) +#else +#define _GENERATE_DEBUG_SECTION_APPLE(name) +#endif + +#if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) +#define _GENERATE_DEBUG_SECTION_LINUX(name) \ + __attribute__((section("." Py_STRINGIFY(name)))) \ + __attribute__((used)) +#else +#define _GENERATE_DEBUG_SECTION_LINUX(name) +#endif + +#ifdef Py_GIL_DISABLED +# define _Py_Debug_gilruntimestate_enabled offsetof(struct _gil_runtime_state, enabled) +# define _Py_Debug_Free_Threaded 1 +#else +# define _Py_Debug_gilruntimestate_enabled 0 +# define _Py_Debug_Free_Threaded 0 +#endif + + +typedef struct _Py_DebugOffsets { + char cookie[8]; + uint64_t version; + uint64_t free_threaded; + // Runtime state offset; + struct _runtime_state { + uint64_t size; + uint64_t finalizing; + uint64_t interpreters_head; + } runtime_state; + + // Interpreter state offset; + struct _interpreter_state { + uint64_t size; + uint64_t id; + uint64_t next; + uint64_t threads_head; + uint64_t gc; + uint64_t imports_modules; + uint64_t sysdict; + uint64_t builtins; + uint64_t ceval_gil; + uint64_t gil_runtime_state; + uint64_t gil_runtime_state_enabled; + uint64_t gil_runtime_state_locked; + uint64_t gil_runtime_state_holder; + } interpreter_state; + + // Thread state offset; + struct _thread_state{ + uint64_t size; + uint64_t prev; + uint64_t next; + uint64_t interp; + uint64_t current_frame; + uint64_t thread_id; + uint64_t native_thread_id; + uint64_t datastack_chunk; + uint64_t status; + } thread_state; + + // InterpreterFrame offset; + struct _interpreter_frame { + uint64_t size; + uint64_t previous; + uint64_t executable; + uint64_t instr_ptr; + uint64_t localsplus; + uint64_t owner; + uint64_t stackpointer; + } interpreter_frame; + + // Code object offset; + struct _code_object { + uint64_t size; + uint64_t filename; + uint64_t name; + uint64_t qualname; + uint64_t linetable; + uint64_t firstlineno; + uint64_t argcount; + uint64_t localsplusnames; + uint64_t localspluskinds; + uint64_t co_code_adaptive; + } code_object; + + // PyObject offset; + struct _pyobject { + uint64_t size; + uint64_t ob_type; + } pyobject; + + // PyTypeObject object offset; + struct _type_object { + uint64_t size; + uint64_t tp_name; + uint64_t tp_repr; + uint64_t tp_flags; + } type_object; + + // PyTuple object offset; + struct _tuple_object { + uint64_t size; + uint64_t ob_item; + uint64_t ob_size; + } tuple_object; + + // PyList object offset; + struct _list_object { + uint64_t size; + uint64_t ob_item; + uint64_t ob_size; + } list_object; + + // PySet object offset; + struct _set_object { + uint64_t size; + uint64_t used; + uint64_t table; + uint64_t mask; + } set_object; + + // PyDict object offset; + struct _dict_object { + uint64_t size; + uint64_t ma_keys; + uint64_t ma_values; + } dict_object; + + // PyFloat object offset; + struct _float_object { + uint64_t size; + uint64_t ob_fval; + } float_object; + + // PyLong object offset; + struct _long_object { + uint64_t size; + uint64_t lv_tag; + uint64_t ob_digit; + } long_object; + + // PyBytes object offset; + struct _bytes_object { + uint64_t size; + uint64_t ob_size; + uint64_t ob_sval; + } bytes_object; + + // Unicode object offset; + struct _unicode_object { + uint64_t size; + uint64_t state; + uint64_t length; + uint64_t asciiobject_size; + } unicode_object; + + // GC runtime state offset; + struct _gc { + uint64_t size; + uint64_t collecting; + } gc; + + // Generator object offset; + struct _gen_object { + uint64_t size; + uint64_t gi_name; + uint64_t gi_iframe; + uint64_t gi_frame_state; + } gen_object; +} _Py_DebugOffsets; + + +#define _Py_DebugOffsets_INIT(debug_cookie) { \ + .cookie = debug_cookie, \ + .version = PY_VERSION_HEX, \ + .free_threaded = _Py_Debug_Free_Threaded, \ + .runtime_state = { \ + .size = sizeof(_PyRuntimeState), \ + .finalizing = offsetof(_PyRuntimeState, _finalizing), \ + .interpreters_head = offsetof(_PyRuntimeState, interpreters.head), \ + }, \ + .interpreter_state = { \ + .size = sizeof(PyInterpreterState), \ + .id = offsetof(PyInterpreterState, id), \ + .next = offsetof(PyInterpreterState, next), \ + .threads_head = offsetof(PyInterpreterState, threads.head), \ + .gc = offsetof(PyInterpreterState, gc), \ + .imports_modules = offsetof(PyInterpreterState, imports.modules), \ + .sysdict = offsetof(PyInterpreterState, sysdict), \ + .builtins = offsetof(PyInterpreterState, builtins), \ + .ceval_gil = offsetof(PyInterpreterState, ceval.gil), \ + .gil_runtime_state = offsetof(PyInterpreterState, _gil), \ + .gil_runtime_state_enabled = _Py_Debug_gilruntimestate_enabled, \ + .gil_runtime_state_locked = offsetof(PyInterpreterState, _gil.locked), \ + .gil_runtime_state_holder = offsetof(PyInterpreterState, _gil.last_holder), \ + }, \ + .thread_state = { \ + .size = sizeof(PyThreadState), \ + .prev = offsetof(PyThreadState, prev), \ + .next = offsetof(PyThreadState, next), \ + .interp = offsetof(PyThreadState, interp), \ + .current_frame = offsetof(PyThreadState, current_frame), \ + .thread_id = offsetof(PyThreadState, thread_id), \ + .native_thread_id = offsetof(PyThreadState, native_thread_id), \ + .datastack_chunk = offsetof(PyThreadState, datastack_chunk), \ + .status = offsetof(PyThreadState, _status), \ + }, \ + .interpreter_frame = { \ + .size = sizeof(_PyInterpreterFrame), \ + .previous = offsetof(_PyInterpreterFrame, previous), \ + .executable = offsetof(_PyInterpreterFrame, f_executable), \ + .instr_ptr = offsetof(_PyInterpreterFrame, instr_ptr), \ + .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ + .owner = offsetof(_PyInterpreterFrame, owner), \ + .stackpointer = offsetof(_PyInterpreterFrame, stackpointer), \ + }, \ + .code_object = { \ + .size = sizeof(PyCodeObject), \ + .filename = offsetof(PyCodeObject, co_filename), \ + .name = offsetof(PyCodeObject, co_name), \ + .qualname = offsetof(PyCodeObject, co_qualname), \ + .linetable = offsetof(PyCodeObject, co_linetable), \ + .firstlineno = offsetof(PyCodeObject, co_firstlineno), \ + .argcount = offsetof(PyCodeObject, co_argcount), \ + .localsplusnames = offsetof(PyCodeObject, co_localsplusnames), \ + .localspluskinds = offsetof(PyCodeObject, co_localspluskinds), \ + .co_code_adaptive = offsetof(PyCodeObject, co_code_adaptive), \ + }, \ + .pyobject = { \ + .size = sizeof(PyObject), \ + .ob_type = offsetof(PyObject, ob_type), \ + }, \ + .type_object = { \ + .size = sizeof(PyTypeObject), \ + .tp_name = offsetof(PyTypeObject, tp_name), \ + .tp_repr = offsetof(PyTypeObject, tp_repr), \ + .tp_flags = offsetof(PyTypeObject, tp_flags), \ + }, \ + .tuple_object = { \ + .size = sizeof(PyTupleObject), \ + .ob_item = offsetof(PyTupleObject, ob_item), \ + .ob_size = offsetof(PyTupleObject, ob_base.ob_size), \ + }, \ + .list_object = { \ + .size = sizeof(PyListObject), \ + .ob_item = offsetof(PyListObject, ob_item), \ + .ob_size = offsetof(PyListObject, ob_base.ob_size), \ + }, \ + .set_object = { \ + .size = sizeof(PySetObject), \ + .used = offsetof(PySetObject, used), \ + .table = offsetof(PySetObject, table), \ + .mask = offsetof(PySetObject, mask), \ + }, \ + .dict_object = { \ + .size = sizeof(PyDictObject), \ + .ma_keys = offsetof(PyDictObject, ma_keys), \ + .ma_values = offsetof(PyDictObject, ma_values), \ + }, \ + .float_object = { \ + .size = sizeof(PyFloatObject), \ + .ob_fval = offsetof(PyFloatObject, ob_fval), \ + }, \ + .long_object = { \ + .size = sizeof(PyLongObject), \ + .lv_tag = offsetof(PyLongObject, long_value.lv_tag), \ + .ob_digit = offsetof(PyLongObject, long_value.ob_digit), \ + }, \ + .bytes_object = { \ + .size = sizeof(PyBytesObject), \ + .ob_size = offsetof(PyBytesObject, ob_base.ob_size), \ + .ob_sval = offsetof(PyBytesObject, ob_sval), \ + }, \ + .unicode_object = { \ + .size = sizeof(PyUnicodeObject), \ + .state = offsetof(PyUnicodeObject, _base._base.state), \ + .length = offsetof(PyUnicodeObject, _base._base.length), \ + .asciiobject_size = sizeof(PyASCIIObject), \ + }, \ + .gc = { \ + .size = sizeof(struct _gc_runtime_state), \ + .collecting = offsetof(struct _gc_runtime_state, collecting), \ + }, \ + .gen_object = { \ + .size = sizeof(PyGenObject), \ + .gi_name = offsetof(PyGenObject, gi_name), \ + .gi_iframe = offsetof(PyGenObject, gi_iframe), \ + .gi_frame_state = offsetof(PyGenObject, gi_frame_state), \ + }, \ +} + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_DEBUG_OFFSETS_H */ diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index cd171a4384db5d..f4c55ca6cf64d2 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -1,4 +1,3 @@ - #ifndef Py_INTERNAL_DICT_H #define Py_INTERNAL_DICT_H #ifdef __cplusplus @@ -9,15 +8,19 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState -#include "pycore_identifier.h" // _Py_Identifier -#include "pycore_object.h" // PyDictOrValues +#include "pycore_object.h" // PyManagedDictPointer +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_ACQUIRE +#include "pycore_stackref.h" // _PyStackRef // Unsafe flavor of PyDict_GetItemWithError(): no error checking extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key); -extern int _PyDict_DelItemIf(PyObject *mp, PyObject *key, - int (*predicate)(PyObject *value)); +// Delete an item from a dict if a predicate is true +// Returns -1 on error, 1 if the item was deleted, 0 otherwise +// Export for '_asyncio' shared extension +PyAPI_FUNC(int) _PyDict_DelItemIf(PyObject *mp, PyObject *key, + int (*predicate)(PyObject *value, void *arg), + void *arg); // "KnownHash" variants // Export for '_asyncio' shared extension @@ -40,8 +43,6 @@ extern int _PyDict_Next( extern int _PyDict_HasOnlyStringKeys(PyObject *mp); -extern void _PyDict_MaybeUntrack(PyObject *mp); - // Export for '_ctypes' shared extension PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); @@ -81,7 +82,7 @@ typedef struct { PyObject *me_value; /* This field is only meaningful for combined tables */ } PyDictUnicodeEntry; -extern PyDictKeysObject *_PyDict_NewKeysForClass(void); +extern PyDictKeysObject *_PyDict_NewKeysForClass(PyHeapTypeObject *); extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); /* Gets a version number unique to the current state of the keys of dict, if possible. @@ -89,6 +90,17 @@ extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); extern uint32_t _PyDictKeys_GetVersionForCurrentState( PyInterpreterState *interp, PyDictKeysObject *dictkeys); +/* Gets a version number unique to the current state of the keys of dict, if possible. + * + * In free-threaded builds ensures that the dict can be used for lock-free + * reads if a version was assigned. + * + * The caller must hold the per-object lock on dict. + * + * Returns the version number, or zero if it was not possible to get a version number. */ +extern uint32_t _PyDict_GetKeysVersionForCurrentState( + PyInterpreterState *interp, PyDictObject *dict); + extern size_t _PyDict_KeysSize(PyDictKeysObject *keys); extern void _PyDictKeys_DecRef(PyDictKeysObject *keys); @@ -97,14 +109,39 @@ extern void _PyDictKeys_DecRef(PyDictKeysObject *keys); * -1 when no entry found, -3 when compare raises error. */ extern Py_ssize_t _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); +extern Py_ssize_t _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); +extern Py_ssize_t _Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject *key, Py_hash_t hash, _PyStackRef *value_addr); extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *); extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); + +/* Look up a string key in an all unicode dict keys, assign the keys object a version, and + * store it in version. + * + * Returns DKIX_ERROR if key is not a string or if the keys object is not all + * strings. + * + * Returns DKIX_EMPTY if the key is not present. + */ +extern Py_ssize_t _PyDictKeys_StringLookupAndVersion(PyDictKeysObject* dictkeys, PyObject *key, uint32_t *version); +extern Py_ssize_t _PyDictKeys_StringLookupSplit(PyDictKeysObject* dictkeys, PyObject *key); PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); +PyAPI_FUNC(void) _PyDict_LoadGlobalStackRef(PyDictObject *, PyDictObject *, PyObject *, _PyStackRef *); + +// Loads the __builtins__ object from the globals dict. Returns a new reference. +extern PyObject *_PyDict_LoadBuiltinsFromGlobals(PyObject *globals); /* Consumes references to key and value */ PyAPI_FUNC(int) _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); -extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *name, PyObject *value); +extern int _PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value); +// Export for '_asyncio' shared extension +PyAPI_FUNC(int) _PyDict_SetItem_KnownHash_LockHeld(PyDictObject *mp, PyObject *key, + PyObject *value, Py_hash_t hash); +// Export for '_asyncio' shared extension +PyAPI_FUNC(int) _PyDict_GetItemRef_KnownHash_LockHeld(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); +extern int _PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); +extern int _PyDict_GetItemRef_Unicode_LockHeld(PyDictObject *op, PyObject *key, PyObject **result); +extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, PyObject *name, PyObject *value); extern int _PyDict_Pop_KnownHash( PyDictObject *dict, @@ -167,7 +204,7 @@ struct _dictkeysobject { char dk_indices[]; /* char is required to avoid strict aliasing. */ /* "PyDictKeyEntry or PyDictUnicodeEntry dk_entries[USABLE_FRACTION(DK_SIZE(dk))];" array follows: - see the DK_ENTRIES() macro */ + see the DK_ENTRIES() / DK_UNICODE_ENTRIES() functions below */ }; /* This must be no more than 250, for the prefix size to fit in one byte. */ @@ -181,6 +218,10 @@ struct _dictkeysobject { * [-1] = prefix size. [-2] = used size. size[-2-n...] = insertion order. */ struct _dictvalues { + uint8_t capacity; + uint8_t size; + uint8_t embedded; + uint8_t valid; PyObject *values[1]; }; @@ -196,6 +237,7 @@ static inline void* _DK_ENTRIES(PyDictKeysObject *dk) { size_t index = (size_t)1 << dk->dk_log2_index_bytes; return (&indices[index]); } + static inline PyDictKeyEntry* DK_ENTRIES(PyDictKeysObject *dk) { assert(dk->dk_kind == DICT_KEYS_GENERAL); return (PyDictKeyEntry*)_DK_ENTRIES(dk); @@ -210,27 +252,18 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DICT_VERSION_INCREMENT (1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) #define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1) #define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) - 1) +#define DICT_UNIQUE_ID_SHIFT (32) +#define DICT_UNIQUE_ID_MAX ((UINT64_C(1) << (64 - DICT_UNIQUE_ID_SHIFT)) - 1) -#define DICT_VALUES_SIZE(values) ((uint8_t *)values)[-1] -#define DICT_VALUES_USED_SIZE(values) ((uint8_t *)values)[-2] - -#ifdef Py_GIL_DISABLED -#define DICT_NEXT_VERSION(INTERP) \ - (_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version, DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT) - -#else -#define DICT_NEXT_VERSION(INTERP) \ - ((INTERP)->dict_state.global_version += DICT_VERSION_INCREMENT) -#endif -void +PyAPI_FUNC(void) _PyDict_SendEvent(int watcher_bits, PyDict_WatchEvent event, PyDictObject *mp, PyObject *key, PyObject *value); -static inline uint64_t +static inline void _PyDict_NotifyEvent(PyInterpreterState *interp, PyDict_WatchEvent event, PyDictObject *mp, @@ -238,33 +271,126 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, PyObject *value) { assert(Py_REFCNT((PyObject*)mp) > 0); - int watcher_bits = mp->ma_version_tag & DICT_WATCHER_MASK; + int watcher_bits = mp->_ma_watcher_tag & DICT_WATCHER_MASK; if (watcher_bits) { RARE_EVENT_STAT_INC(watched_dict_modification); _PyDict_SendEvent(watcher_bits, event, mp, key, value); } - return DICT_NEXT_VERSION(interp) | (mp->ma_version_tag & DICT_WATCHER_AND_MODIFICATION_MASK); } -extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values); -PyAPI_FUNC(bool) _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv); +extern PyDictObject *_PyObject_MaterializeManagedDict(PyObject *obj); + PyAPI_FUNC(PyObject *)_PyDict_FromItems( PyObject *const *keys, Py_ssize_t keys_offset, PyObject *const *values, Py_ssize_t values_offset, Py_ssize_t length); +static inline uint8_t * +get_insertion_order_array(PyDictValues *values) +{ + return (uint8_t *)&values->values[values->capacity]; +} + static inline void _PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix) { assert(ix < SHARED_KEYS_MAX_SIZE); - uint8_t *size_ptr = ((uint8_t *)values)-2; - int size = *size_ptr; - assert(size+2 < DICT_VALUES_SIZE(values)); - size++; - size_ptr[-size] = (uint8_t)ix; - *size_ptr = size; + int size = values->size; + uint8_t *array = get_insertion_order_array(values); + assert(size < values->capacity); + assert(((uint8_t)ix) == ix); + array[size] = (uint8_t)ix; + values->size = size+1; } +static inline size_t +shared_keys_usable_size(PyDictKeysObject *keys) +{ + // dk_usable will decrease for each instance that is created and each + // value that is added. dk_nentries will increase for each value that + // is added. We want to always return the right value or larger. + // We therefore increase dk_nentries first and we decrease dk_usable + // second, and conversely here we read dk_usable first and dk_entries + // second (to avoid the case where we read entries before the increment + // and read usable after the decrement) + Py_ssize_t dk_usable = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_usable); + Py_ssize_t dk_nentries = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_nentries); + return dk_nentries + dk_usable; +} + +static inline size_t +_PyInlineValuesSize(PyTypeObject *tp) +{ + PyDictKeysObject *keys = ((PyHeapTypeObject*)tp)->ht_cached_keys; + assert(keys != NULL); + size_t size = shared_keys_usable_size(keys); + size_t prefix_size = _Py_SIZE_ROUND_UP(size, sizeof(PyObject *)); + assert(prefix_size < 256); + return prefix_size + (size + 1) * sizeof(PyObject *); +} + +int +_PyDict_DetachFromObject(PyDictObject *dict, PyObject *obj); + +// Enables per-thread ref counting on this dict in the free threading build +extern void _PyDict_EnablePerThreadRefcounting(PyObject *op); + +PyDictObject *_PyObject_MaterializeManagedDict_LockHeld(PyObject *); + +// See `_Py_INCREF_TYPE()` in pycore_object.h +#ifndef Py_GIL_DISABLED +# define _Py_INCREF_DICT Py_INCREF +# define _Py_DECREF_DICT Py_DECREF +# define _Py_INCREF_BUILTINS Py_INCREF +# define _Py_DECREF_BUILTINS Py_DECREF +#else +static inline Py_ssize_t +_PyDict_UniqueId(PyDictObject *mp) +{ + return (Py_ssize_t)(mp->_ma_watcher_tag >> DICT_UNIQUE_ID_SHIFT); +} + +static inline void +_Py_INCREF_DICT(PyObject *op) +{ + assert(PyDict_Check(op)); + Py_ssize_t id = _PyDict_UniqueId((PyDictObject *)op); + _Py_THREAD_INCREF_OBJECT(op, id); +} + +static inline void +_Py_DECREF_DICT(PyObject *op) +{ + assert(PyDict_Check(op)); + Py_ssize_t id = _PyDict_UniqueId((PyDictObject *)op); + _Py_THREAD_DECREF_OBJECT(op, id); +} + +// Like `_Py_INCREF_DICT`, but also handles non-dict objects because builtins +// may not be a dict. +static inline void +_Py_INCREF_BUILTINS(PyObject *op) +{ + if (PyDict_CheckExact(op)) { + _Py_INCREF_DICT(op); + } + else { + Py_INCREF(op); + } +} + +static inline void +_Py_DECREF_BUILTINS(PyObject *op) +{ + if (PyDict_CheckExact(op)) { + _Py_DECREF_DICT(op); + } + else { + Py_DECREF(op); + } +} +#endif + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index 1a44755c7a01a3..11932b8d1e1ab6 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -12,10 +12,6 @@ extern "C" { #define DICT_WATCHED_MUTATION_BITS 4 struct _Py_dict_state { - /*Global counter used to set ma_version_tag field of dictionary. - * It is incremented each time that a dictionary is created and each - * time that a dictionary is modified. */ - uint64_t global_version; uint32_t next_keys_version; PyDict_WatchCallback watchers[DICT_MAX_WATCHERS]; }; diff --git a/Include/internal/pycore_dtoa.h b/Include/internal/pycore_dtoa.h index c5cfdf4ce8f823..e4222c5267d6be 100644 --- a/Include/internal/pycore_dtoa.h +++ b/Include/internal/pycore_dtoa.h @@ -11,8 +11,6 @@ extern "C" { #include "pycore_pymath.h" // _PY_SHORT_FLOAT_REPR -#if _PY_SHORT_FLOAT_REPR == 1 - typedef uint32_t ULong; struct @@ -22,15 +20,15 @@ Bigint { ULong x[1]; }; -#ifdef Py_USING_MEMORY_DEBUGGER +#if defined(Py_USING_MEMORY_DEBUGGER) || _PY_SHORT_FLOAT_REPR == 0 struct _dtoa_state { int _not_used; }; -#define _dtoa_interp_state_INIT(INTERP) \ +#define _dtoa_state_INIT(INTERP) \ {0} -#else // !Py_USING_MEMORY_DEBUGGER +#else // !Py_USING_MEMORY_DEBUGGER && _PY_SHORT_FLOAT_REPR != 0 /* The size of the Bigint freelist */ #define Bigint_Kmax 7 @@ -66,8 +64,6 @@ extern char* _Py_dg_dtoa(double d, int mode, int ndigits, int *decpt, int *sign, char **rve); extern void _Py_dg_freedtoa(char *s); -#endif // _PY_SHORT_FLOAT_REPR == 1 - extern PyStatus _PyDtoa_Init(PyInterpreterState *interp); extern void _PyDtoa_Fini(PyInterpreterState *interp); diff --git a/Include/internal/pycore_emscripten_trampoline.h b/Include/internal/pycore_emscripten_trampoline.h index e519c99ad86cce..5546ebbbfcb5c1 100644 --- a/Include/internal/pycore_emscripten_trampoline.h +++ b/Include/internal/pycore_emscripten_trampoline.h @@ -27,24 +27,14 @@ #if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) -void _Py_EmscriptenTrampoline_Init(_PyRuntimeState *runtime); +void +_Py_EmscriptenTrampoline_Init(_PyRuntimeState *runtime); PyObject* -_PyEM_TrampolineCall_JavaScript(PyCFunctionWithKeywords func, - PyObject* self, - PyObject* args, - PyObject* kw); - -PyObject* -_PyEM_TrampolineCall_Reflection(PyCFunctionWithKeywords func, - PyObject* self, - PyObject* args, - PyObject* kw); - -#define _PyEM_TrampolineCall(meth, self, args, kw) \ - ((_PyRuntime.wasm_type_reflection_available) ? \ - (_PyEM_TrampolineCall_Reflection((PyCFunctionWithKeywords)(meth), (self), (args), (kw))) : \ - (_PyEM_TrampolineCall_JavaScript((PyCFunctionWithKeywords)(meth), (self), (args), (kw)))) +_PyEM_TrampolineCall(PyCFunctionWithKeywords func, + PyObject* self, + PyObject* args, + PyObject* kw); #define _PyCFunction_TrampolineCall(meth, self, args) \ _PyEM_TrampolineCall( \ @@ -62,8 +52,6 @@ _PyEM_TrampolineCall_Reflection(PyCFunctionWithKeywords func, #else // defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) -#define _Py_EmscriptenTrampoline_Init(runtime) - #define _PyCFunction_TrampolineCall(meth, self, args) \ (meth)((self), (args)) diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 5c55282fa39e6f..13f86b01bbfe8f 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -290,6 +290,8 @@ extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t extern HRESULT PathCchSkipRoot(const wchar_t *pszPath, const wchar_t **ppszRootEnd); #endif /* defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) */ +extern void _Py_skiproot(const wchar_t *path, Py_ssize_t size, Py_ssize_t *drvsize, Py_ssize_t *rootsize); + // Macros to protect CRT calls against instant termination when passed an // invalid parameter (bpo-23524). IPH stands for Invalid Parameter Handler. // Usage: @@ -324,6 +326,9 @@ extern int _PyFile_Flush(PyObject *); extern int _Py_GetTicksPerSecond(long *ticks_per_second); #endif +// Export for '_testcapi' shared extension +PyAPI_FUNC(int) _Py_IsValidFD(int fd); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index f984df695696c3..f44b081b06cea5 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -8,7 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState #include "pycore_unicodeobject.h" // _PyUnicodeWriter /* runtime lifecycle */ @@ -55,6 +54,8 @@ extern PyObject* _Py_string_to_number_with_underscores( extern double _Py_parse_inf_or_nan(const char *p, char **endptr); +extern int _Py_convert_int_to_double(PyObject **v, double *dbl); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_flowgraph.h b/Include/internal/pycore_flowgraph.h index 58fed46886ea45..5043260d2fd99f 100644 --- a/Include/internal/pycore_flowgraph.h +++ b/Include/internal/pycore_flowgraph.h @@ -8,17 +8,14 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_opcode_utils.h" #include "pycore_compile.h" - -typedef struct { - int id; -} _PyCfgJumpTargetLabel; +#include "pycore_instruction_sequence.h" +#include "pycore_opcode_utils.h" struct _PyCfgBuilder; -int _PyCfgBuilder_UseLabel(struct _PyCfgBuilder *g, _PyCfgJumpTargetLabel lbl); -int _PyCfgBuilder_Addop(struct _PyCfgBuilder *g, int opcode, int oparg, _PyCompilerSrcLocation loc); +int _PyCfgBuilder_UseLabel(struct _PyCfgBuilder *g, _PyJumpTargetLabel lbl); +int _PyCfgBuilder_Addop(struct _PyCfgBuilder *g, int opcode, int oparg, _Py_SourceLocation loc); struct _PyCfgBuilder* _PyCfgBuilder_New(void); void _PyCfgBuilder_Free(struct _PyCfgBuilder *g); @@ -27,16 +24,23 @@ int _PyCfgBuilder_CheckSize(struct _PyCfgBuilder* g); int _PyCfg_OptimizeCodeUnit(struct _PyCfgBuilder *g, PyObject *consts, PyObject *const_cache, int nlocals, int nparams, int firstlineno); -int _PyCfg_ToInstructionSequence(struct _PyCfgBuilder *g, _PyCompile_InstructionSequence *seq); +struct _PyCfgBuilder* _PyCfg_FromInstructionSequence(_PyInstructionSequence *seq); +int _PyCfg_ToInstructionSequence(struct _PyCfgBuilder *g, _PyInstructionSequence *seq); int _PyCfg_OptimizedCfgToInstructionSequence(struct _PyCfgBuilder *g, _PyCompile_CodeUnitMetadata *umd, int code_flags, int *stackdepth, int *nlocalsplus, - _PyCompile_InstructionSequence *seq); + _PyInstructionSequence *seq); PyCodeObject * _PyAssemble_MakeCodeObject(_PyCompile_CodeUnitMetadata *u, PyObject *const_cache, - PyObject *consts, int maxdepth, _PyCompile_InstructionSequence *instrs, + PyObject *consts, int maxdepth, _PyInstructionSequence *instrs, int nlocalsplus, int code_flags, PyObject *filename); +// Export for '_testinternalcapi' shared extension +PyAPI_FUNC(PyObject*) _PyCompile_OptimizeCfg( + PyObject *instructions, + PyObject *consts, + int nlocals); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 0f9e7333cf1e1c..8cc3504723b64c 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -11,8 +11,9 @@ extern "C" { #include #include // offsetof() #include "pycore_code.h" // STATS +#include "pycore_stackref.h" // _PyStackRef -/* See Objects/frame_layout.md for an explanation of the frame stack +/* See InternalDocs/frames.md for an explanation of the frame stack * including explanation of the PyFrameObject and _PyInterpreterFrame * structs. */ @@ -25,7 +26,11 @@ struct _frame { int f_lineno; /* Current line number. Only valid if non-zero */ char f_trace_lines; /* Emit per-line trace events? */ char f_trace_opcodes; /* Emit per-opcode trace events? */ - char f_fast_as_locals; /* Have the fast locals of this frame been converted to a dict? */ + PyObject *f_extra_locals; /* Dict for locals set by users using f_locals, could be NULL */ + /* This is purely for backwards compatibility for PyEval_GetLocals. + PyEval_GetLocals requires a borrowed reference so the actual reference + is stored here */ + PyObject *f_locals_cache; /* The frame data, if this frame object owns the frame */ PyObject *_f_frame_data[1]; }; @@ -51,52 +56,83 @@ enum _frameowner { FRAME_OWNED_BY_THREAD = 0, FRAME_OWNED_BY_GENERATOR = 1, FRAME_OWNED_BY_FRAME_OBJECT = 2, - FRAME_OWNED_BY_CSTACK = 3, + FRAME_OWNED_BY_INTERPRETER = 3, + FRAME_OWNED_BY_CSTACK = 4, }; typedef struct _PyInterpreterFrame { - PyObject *f_executable; /* Strong reference */ + _PyStackRef f_executable; /* Deferred or strong reference (code object or None) */ struct _PyInterpreterFrame *previous; - PyObject *f_funcobj; /* Strong reference. Only valid if not on C stack */ + _PyStackRef f_funcobj; /* Deferred or strong reference. Only valid if not on C stack */ PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */ PyObject *f_builtins; /* Borrowed reference. Only valid if not on C stack */ PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */ PyFrameObject *frame_obj; /* Strong reference, may be NULL. Only valid if not on C stack */ _Py_CODEUNIT *instr_ptr; /* Instruction currently executing (or about to begin) */ - int stacktop; /* Offset of TOS from localsplus */ + _PyStackRef *stackpointer; +#ifdef Py_GIL_DISABLED + /* Index of thread-local bytecode containing instr_ptr. */ + int32_t tlbc_index; +#endif uint16_t return_offset; /* Only relevant during a function call */ char owner; +#ifdef Py_DEBUG + uint8_t visited:1; + uint8_t lltrace:7; +#else + uint8_t visited; +#endif /* Locals and stack */ - PyObject *localsplus[1]; + _PyStackRef localsplus[1]; } _PyInterpreterFrame; #define _PyInterpreterFrame_LASTI(IF) \ - ((int)((IF)->instr_ptr - _PyCode_CODE(_PyFrame_GetCode(IF)))) + ((int)((IF)->instr_ptr - _PyFrame_GetBytecode((IF)))) static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) { - assert(PyCode_Check(f->f_executable)); - return (PyCodeObject *)f->f_executable; + PyObject *executable = PyStackRef_AsPyObjectBorrow(f->f_executable); + assert(PyCode_Check(executable)); + return (PyCodeObject *)executable; +} + +static inline _Py_CODEUNIT * +_PyFrame_GetBytecode(_PyInterpreterFrame *f) +{ +#ifdef Py_GIL_DISABLED + PyCodeObject *co = _PyFrame_GetCode(f); + _PyCodeArray *tlbc = _PyCode_GetTLBCArray(co); + assert(f->tlbc_index >= 0 && f->tlbc_index < tlbc->size); + return (_Py_CODEUNIT *)tlbc->entries[f->tlbc_index]; +#else + return _PyCode_CODE(_PyFrame_GetCode(f)); +#endif +} + +static inline PyFunctionObject *_PyFrame_GetFunction(_PyInterpreterFrame *f) { + PyObject *func = PyStackRef_AsPyObjectBorrow(f->f_funcobj); + assert(PyFunction_Check(func)); + return (PyFunctionObject *)func; } -static inline PyObject **_PyFrame_Stackbase(_PyInterpreterFrame *f) { - return f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus; +static inline _PyStackRef *_PyFrame_Stackbase(_PyInterpreterFrame *f) { + return (f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); } -static inline PyObject *_PyFrame_StackPeek(_PyInterpreterFrame *f) { - assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus); - assert(f->localsplus[f->stacktop-1] != NULL); - return f->localsplus[f->stacktop-1]; +static inline _PyStackRef _PyFrame_StackPeek(_PyInterpreterFrame *f) { + assert(f->stackpointer > f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); + assert(!PyStackRef_IsNull(f->stackpointer[-1])); + return f->stackpointer[-1]; } -static inline PyObject *_PyFrame_StackPop(_PyInterpreterFrame *f) { - assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus); - f->stacktop--; - return f->localsplus[f->stacktop]; +static inline _PyStackRef _PyFrame_StackPop(_PyInterpreterFrame *f) { + assert(f->stackpointer > f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); + f->stackpointer--; + return *f->stackpointer; } -static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, PyObject *value) { - f->localsplus[f->stacktop] = value; - f->stacktop++; +static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, _PyStackRef value) { + *f->stackpointer = value; + f->stackpointer++; } #define FRAME_SPECIALS_SIZE ((int)((sizeof(_PyInterpreterFrame)-1)/sizeof(PyObject *))) @@ -110,7 +146,40 @@ _PyFrame_NumSlotsForCodeObject(PyCodeObject *code) return code->co_framesize - FRAME_SPECIALS_SIZE; } -void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest); +static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest) +{ + *dest = *src; + assert(src->stackpointer != NULL); + int stacktop = (int)(src->stackpointer - src->localsplus); + assert(stacktop >= _PyFrame_GetCode(src)->co_nlocalsplus); + dest->stackpointer = dest->localsplus + stacktop; + for (int i = 1; i < stacktop; i++) { + dest->localsplus[i] = src->localsplus[i]; + } + // Don't leave a dangling pointer to the old frame when creating generators + // and coroutines: + dest->previous = NULL; +} + +#ifdef Py_GIL_DISABLED +static inline void +_PyFrame_InitializeTLBC(PyThreadState *tstate, _PyInterpreterFrame *frame, + PyCodeObject *code) +{ + _Py_CODEUNIT *tlbc = _PyCode_GetTLBCFast(tstate, code); + if (tlbc == NULL) { + // No thread-local bytecode exists for this thread yet; use the main + // thread's copy, deferring thread-local bytecode creation to the + // execution of RESUME. + frame->instr_ptr = _PyCode_CODE(code); + frame->tlbc_index = 0; + } + else { + frame->instr_ptr = tlbc; + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + } +} +#endif /* Consumes reference to func and locals. Does not initialize frame->previous, which happens @@ -118,50 +187,62 @@ void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest); */ static inline void _PyFrame_Initialize( - _PyInterpreterFrame *frame, PyFunctionObject *func, - PyObject *locals, PyCodeObject *code, int null_locals_from) + PyThreadState *tstate, _PyInterpreterFrame *frame, _PyStackRef func, + PyObject *locals, PyCodeObject *code, int null_locals_from, _PyInterpreterFrame *previous) { - frame->f_funcobj = (PyObject *)func; - frame->f_executable = Py_NewRef(code); - frame->f_builtins = func->func_builtins; - frame->f_globals = func->func_globals; + frame->previous = previous; + frame->f_funcobj = func; + frame->f_executable = PyStackRef_FromPyObjectNew(code); + PyFunctionObject *func_obj = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(func); + frame->f_builtins = func_obj->func_builtins; + frame->f_globals = func_obj->func_globals; frame->f_locals = locals; - frame->stacktop = code->co_nlocalsplus; + frame->stackpointer = frame->localsplus + code->co_nlocalsplus; frame->frame_obj = NULL; +#ifdef Py_GIL_DISABLED + _PyFrame_InitializeTLBC(tstate, frame, code); +#else + (void)tstate; frame->instr_ptr = _PyCode_CODE(code); +#endif frame->return_offset = 0; frame->owner = FRAME_OWNED_BY_THREAD; + frame->visited = 0; +#ifdef Py_DEBUG + frame->lltrace = 0; +#endif for (int i = null_locals_from; i < code->co_nlocalsplus; i++) { - frame->localsplus[i] = NULL; + frame->localsplus[i] = PyStackRef_NULL; } } /* Gets the pointer to the locals array * that precedes this frame. */ -static inline PyObject** +static inline _PyStackRef* _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) { return frame->localsplus; } -/* Fetches the stack pointer, and sets stacktop to -1. - Having stacktop <= 0 ensures that invalid - values are not visible to the cycle GC. - We choose -1 rather than 0 to assist debugging. */ -static inline PyObject** +/* Fetches the stack pointer, and sets stackpointer to NULL. + Having stackpointer == NULL ensures that invalid + values are not visible to the cycle GC. */ +static inline _PyStackRef* _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { - PyObject **sp = frame->localsplus + frame->stacktop; - frame->stacktop = -1; + assert(frame->stackpointer != NULL); + _PyStackRef *sp = frame->stackpointer; + frame->stackpointer = NULL; return sp; } static inline void -_PyFrame_SetStackPointer(_PyInterpreterFrame *frame, PyObject **stack_pointer) +_PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) { - frame->stacktop = (int)(stack_pointer - frame->localsplus); + assert(frame->stackpointer == NULL); + frame->stackpointer = stack_pointer; } /* Determine whether a frame is incomplete. @@ -175,11 +256,12 @@ _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, PyObject **stack_pointer) static inline bool _PyFrame_IsIncomplete(_PyInterpreterFrame *frame) { - if (frame->owner == FRAME_OWNED_BY_CSTACK) { + if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { return true; } return frame->owner != FRAME_OWNED_BY_GENERATOR && - frame->instr_ptr < _PyCode_CODE(_PyFrame_GetCode(frame)) + _PyFrame_GetCode(frame)->_co_firsttraceable; + frame->instr_ptr < _PyFrame_GetBytecode(frame) + + _PyFrame_GetCode(frame)->_co_firsttraceable; } static inline _PyInterpreterFrame * @@ -204,7 +286,7 @@ _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame); /* Gets the PyFrameObject for this frame, lazily * creating it if necessary. - * Returns a borrowed referennce */ + * Returns a borrowed reference */ static inline PyFrameObject * _PyFrame_GetFrameObject(_PyInterpreterFrame *frame) { @@ -217,6 +299,9 @@ _PyFrame_GetFrameObject(_PyInterpreterFrame *frame) return _PyFrame_MakeAndSetFrameObject(frame); } +void +_PyFrame_ClearLocals(_PyInterpreterFrame *frame); + /* Clears all references in the frame. * If take is non-zero, then the _PyInterpreterFrame frame * may be transferred to the frame object it references @@ -232,14 +317,11 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame * frame); int _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg); -PyObject * -_PyFrame_GetLocals(_PyInterpreterFrame *frame, int include_hidden); +bool +_PyFrame_HasHiddenLocals(_PyInterpreterFrame *frame); -int -_PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame); - -void -_PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear); +PyObject * +_PyFrame_GetLocals(_PyInterpreterFrame *frame); static inline bool _PyThreadState_HasStackSpace(PyThreadState *tstate, int size) @@ -256,54 +338,64 @@ _PyThreadState_HasStackSpace(PyThreadState *tstate, int size) extern _PyInterpreterFrame * _PyThreadState_PushFrame(PyThreadState *tstate, size_t size); -void _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame *frame); +PyAPI_FUNC(void) _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame *frame); /* Pushes a frame without checking for space. * Must be guarded by _PyThreadState_HasStackSpace() * Consumes reference to func. */ static inline _PyInterpreterFrame * -_PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func, int null_locals_from) +_PyFrame_PushUnchecked(PyThreadState *tstate, _PyStackRef func, int null_locals_from, _PyInterpreterFrame * previous) { CALL_STAT_INC(frames_pushed); - PyCodeObject *code = (PyCodeObject *)func->func_code; + PyFunctionObject *func_obj = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(func); + PyCodeObject *code = (PyCodeObject *)func_obj->func_code; _PyInterpreterFrame *new_frame = (_PyInterpreterFrame *)tstate->datastack_top; tstate->datastack_top += code->co_framesize; assert(tstate->datastack_top < tstate->datastack_limit); - _PyFrame_Initialize(new_frame, func, NULL, code, null_locals_from); + _PyFrame_Initialize(tstate, new_frame, func, NULL, code, null_locals_from, + previous); return new_frame; } /* Pushes a trampoline frame without checking for space. * Must be guarded by _PyThreadState_HasStackSpace() */ static inline _PyInterpreterFrame * -_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth) +_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth, _PyInterpreterFrame * previous) { CALL_STAT_INC(frames_pushed); _PyInterpreterFrame *frame = (_PyInterpreterFrame *)tstate->datastack_top; tstate->datastack_top += code->co_framesize; assert(tstate->datastack_top < tstate->datastack_limit); - frame->f_funcobj = Py_None; - frame->f_executable = Py_NewRef(code); + frame->previous = previous; + frame->f_funcobj = PyStackRef_None; + frame->f_executable = PyStackRef_FromPyObjectNew(code); #ifdef Py_DEBUG frame->f_builtins = NULL; frame->f_globals = NULL; #endif frame->f_locals = NULL; - frame->stacktop = code->co_nlocalsplus + stackdepth; + assert(stackdepth <= code->co_stacksize); + frame->stackpointer = frame->localsplus + code->co_nlocalsplus + stackdepth; frame->frame_obj = NULL; +#ifdef Py_GIL_DISABLED + _PyFrame_InitializeTLBC(tstate, frame, code); +#else frame->instr_ptr = _PyCode_CODE(code); +#endif frame->owner = FRAME_OWNED_BY_THREAD; + frame->visited = 0; +#ifdef Py_DEBUG + frame->lltrace = 0; +#endif frame->return_offset = 0; return frame; } -static inline -PyGenObject *_PyFrame_GetGenerator(_PyInterpreterFrame *frame) -{ - assert(frame->owner == FRAME_OWNED_BY_GENERATOR); - size_t offset_in_gen = offsetof(PyGenObject, gi_iframe); - return (PyGenObject *)(((char *)frame) - offset_in_gen); -} +PyAPI_FUNC(_PyInterpreterFrame *) +_PyEvalFramePushAndInit(PyThreadState *tstate, _PyStackRef func, + PyObject *locals, _PyStackRef const* args, + size_t argcount, PyObject *kwnames, + _PyInterpreterFrame *previous); #ifdef __cplusplus } diff --git a/Include/internal/pycore_freelist.h b/Include/internal/pycore_freelist.h index e684e084b8bef8..84a5ab30f3eeea 100644 --- a/Include/internal/pycore_freelist.h +++ b/Include/internal/pycore_freelist.h @@ -8,144 +8,101 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -// PyTuple_MAXSAVESIZE - largest tuple to save on free list -// PyTuple_MAXFREELIST - maximum number of tuples of each size to save - -#ifdef WITH_FREELISTS -// with freelists -# define PyTuple_MAXSAVESIZE 20 -# define PyTuple_NFREELISTS PyTuple_MAXSAVESIZE -# define PyTuple_MAXFREELIST 2000 -# define PyList_MAXFREELIST 80 -# define PyDict_MAXFREELIST 80 -# define PyFloat_MAXFREELIST 100 -# define PyContext_MAXFREELIST 255 -# define _PyAsyncGen_MAXFREELIST 80 -# define _PyObjectStackChunk_MAXFREELIST 4 -#else -# define PyTuple_NFREELISTS 0 -# define PyTuple_MAXFREELIST 0 -# define PyList_MAXFREELIST 0 -# define PyDict_MAXFREELIST 0 -# define PyFloat_MAXFREELIST 0 -# define PyContext_MAXFREELIST 0 -# define _PyAsyncGen_MAXFREELIST 0 -# define _PyObjectStackChunk_MAXFREELIST 0 -#endif - -struct _Py_list_freelist { -#ifdef WITH_FREELISTS - PyListObject *items[PyList_MAXFREELIST]; - int numfree; +#include "pycore_freelist_state.h" // struct _Py_freelists +#include "pycore_object.h" // _PyObject_IS_GC +#include "pycore_pystate.h" // _PyThreadState_GET +#include "pycore_code.h" // OBJECT_STAT_INC + +static inline struct _Py_freelists * +_Py_freelists_GET(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); +#ifdef Py_DEBUG + _Py_EnsureTstateNotNULL(tstate); #endif -}; - -struct _Py_tuple_freelist { -#if WITH_FREELISTS - /* There is one freelist for each size from 1 to PyTuple_MAXSAVESIZE. - The empty tuple is handled separately. - Each tuple stored in the array is the head of the linked list - (and the next available tuple) for that size. The actual tuple - object is used as the linked list node, with its first item - (ob_item[0]) pointing to the next node (i.e. the previous head). - Each linked list is initially NULL. */ - PyTupleObject *items[PyTuple_NFREELISTS]; - int numfree[PyTuple_NFREELISTS]; +#ifdef Py_GIL_DISABLED + return &((_PyThreadStateImpl*)tstate)->freelists; #else - char _unused; // Empty structs are not allowed. -#endif -}; - -struct _Py_float_freelist { -#ifdef WITH_FREELISTS - /* Special free list - free_list is a singly-linked list of available PyFloatObjects, - linked via abuse of their ob_type members. */ - int numfree; - PyFloatObject *items; -#endif -}; - -struct _Py_dict_freelist { -#ifdef WITH_FREELISTS - /* Dictionary reuse scheme to save calls to malloc and free */ - PyDictObject *items[PyDict_MAXFREELIST]; - int numfree; + return &tstate->interp->object_state.freelists; #endif -}; - -struct _Py_dictkeys_freelist { -#ifdef WITH_FREELISTS - /* Dictionary keys reuse scheme to save calls to malloc and free */ - PyDictKeysObject *items[PyDict_MAXFREELIST]; - int numfree; -#endif -}; - -struct _Py_slice_freelist { -#ifdef WITH_FREELISTS - /* Using a cache is very effective since typically only a single slice is - created and then deleted again. */ - PySliceObject *slice_cache; -#endif -}; - -struct _Py_context_freelist { -#ifdef WITH_FREELISTS - // List of free PyContext objects - PyContext *items; - int numfree; -#endif -}; +} -struct _Py_async_gen_freelist { -#ifdef WITH_FREELISTS - /* Freelists boost performance 6-10%; they also reduce memory - fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend - are short-living objects that are instantiated for every - __anext__() call. */ - struct _PyAsyncGenWrappedValue* items[_PyAsyncGen_MAXFREELIST]; - int numfree; -#endif -}; +// Pushes `op` to the freelist, calls `freefunc` if the freelist is full +#define _Py_FREELIST_FREE(NAME, op, freefunc) \ + _PyFreeList_Free(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), \ + Py_ ## NAME ## _MAXFREELIST, freefunc) +// Pushes `op` to the freelist, returns 1 if successful, 0 if the freelist is full +#define _Py_FREELIST_PUSH(NAME, op, limit) \ + _PyFreeList_Push(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), limit) + +// Pops a PyObject from the freelist, returns NULL if the freelist is empty. +#define _Py_FREELIST_POP(TYPE, NAME) \ + _Py_CAST(TYPE*, _PyFreeList_Pop(&_Py_freelists_GET()->NAME)) + +// Pops a non-PyObject data structure from the freelist, returns NULL if the +// freelist is empty. +#define _Py_FREELIST_POP_MEM(NAME) \ + _PyFreeList_PopMem(&_Py_freelists_GET()->NAME) + +#define _Py_FREELIST_SIZE(NAME) (int)((_Py_freelists_GET()->NAME).size) + +static inline int +_PyFreeList_Push(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize) +{ + if (fl->size < maxsize && fl->size >= 0) { + FT_ATOMIC_STORE_PTR_RELAXED(*(void **)obj, fl->freelist); + fl->freelist = obj; + fl->size++; + OBJECT_STAT_INC(to_freelist); + return 1; + } + return 0; +} -struct _Py_async_gen_asend_freelist { -#ifdef WITH_FREELISTS - struct PyAsyncGenASend* items[_PyAsyncGen_MAXFREELIST]; - int numfree; -#endif -}; +static inline void +_PyFreeList_Free(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize, + freefunc dofree) +{ + if (!_PyFreeList_Push(fl, obj, maxsize)) { + dofree(obj); + } +} -struct _PyObjectStackChunk; +static inline void * +_PyFreeList_PopNoStats(struct _Py_freelist *fl) +{ + void *obj = fl->freelist; + if (obj != NULL) { + assert(fl->size > 0); + fl->freelist = *(void **)obj; + fl->size--; + } + return obj; +} -struct _Py_object_stack_freelist { - struct _PyObjectStackChunk *items; - Py_ssize_t numfree; -}; +static inline PyObject * +_PyFreeList_Pop(struct _Py_freelist *fl) +{ + PyObject *op = _PyFreeList_PopNoStats(fl); + if (op != NULL) { + OBJECT_STAT_INC(from_freelist); + _Py_NewReference(op); + } + return op; +} -struct _Py_object_freelists { - struct _Py_float_freelist floats; - struct _Py_tuple_freelist tuples; - struct _Py_list_freelist lists; - struct _Py_dict_freelist dicts; - struct _Py_dictkeys_freelist dictkeys; - struct _Py_slice_freelist slices; - struct _Py_context_freelist contexts; - struct _Py_async_gen_freelist async_gens; - struct _Py_async_gen_asend_freelist async_gen_asends; - struct _Py_object_stack_freelist object_stacks; -}; +static inline void * +_PyFreeList_PopMem(struct _Py_freelist *fl) +{ + void *op = _PyFreeList_PopNoStats(fl); + if (op != NULL) { + OBJECT_STAT_INC(from_freelist); + } + return op; +} -extern void _PyObject_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PyTuple_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PyFloat_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PyList_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PySlice_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PyDict_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PyAsyncGen_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PyContext_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PyObjectStackChunk_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); +extern void _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization); #ifdef __cplusplus } diff --git a/Include/internal/pycore_freelist_state.h b/Include/internal/pycore_freelist_state.h new file mode 100644 index 00000000000000..7c252f5b570c13 --- /dev/null +++ b/Include/internal/pycore_freelist_state.h @@ -0,0 +1,62 @@ +#ifndef Py_INTERNAL_FREELIST_STATE_H +#define Py_INTERNAL_FREELIST_STATE_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +# define PyTuple_MAXSAVESIZE 20 // Largest tuple to save on freelist +# define Py_tuple_MAXFREELIST 2000 // Maximum number of tuples of each size to save +# define Py_lists_MAXFREELIST 80 +# define Py_list_iters_MAXFREELIST 10 +# define Py_tuple_iters_MAXFREELIST 10 +# define Py_dicts_MAXFREELIST 80 +# define Py_dictkeys_MAXFREELIST 80 +# define Py_floats_MAXFREELIST 100 +# define Py_ints_MAXFREELIST 100 +# define Py_slices_MAXFREELIST 1 +# define Py_contexts_MAXFREELIST 255 +# define Py_async_gens_MAXFREELIST 80 +# define Py_async_gen_asends_MAXFREELIST 80 +# define Py_futureiters_MAXFREELIST 255 +# define Py_object_stack_chunks_MAXFREELIST 4 +# define Py_unicode_writers_MAXFREELIST 1 +# define Py_pymethodobjects_MAXFREELIST 20 + +// A generic freelist of either PyObjects or other data structures. +struct _Py_freelist { + // Entries are linked together using the first word of the object. + // For PyObjects, this overlaps with the `ob_refcnt` field or the `ob_tid` + // field. + void *freelist; + + // The number of items in the free list or -1 if the free list is disabled + Py_ssize_t size; +}; + +struct _Py_freelists { + struct _Py_freelist floats; + struct _Py_freelist ints; + struct _Py_freelist tuples[PyTuple_MAXSAVESIZE]; + struct _Py_freelist lists; + struct _Py_freelist list_iters; + struct _Py_freelist tuple_iters; + struct _Py_freelist dicts; + struct _Py_freelist dictkeys; + struct _Py_freelist slices; + struct _Py_freelist contexts; + struct _Py_freelist async_gens; + struct _Py_freelist async_gen_asends; + struct _Py_freelist futureiters; + struct _Py_freelist object_stack_chunks; + struct _Py_freelist unicode_writers; + struct _Py_freelist pymethodobjects; +}; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_FREELIST_STATE_H */ diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index dad6a89af77dec..c45d281125febb 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -4,6 +4,8 @@ extern "C" { #endif +#include "pycore_lock.h" + #ifndef Py_BUILD_CORE # error "this header requires Py_BUILD_CORE define" #endif @@ -16,21 +18,43 @@ extern PyObject* _PyFunction_Vectorcall( #define FUNC_MAX_WATCHERS 8 +#define FUNC_VERSION_UNSET 0 +#define FUNC_VERSION_CLEARED 1 +#define FUNC_VERSION_FIRST_VALID 2 + #define FUNC_VERSION_CACHE_SIZE (1<<12) /* Must be a power of 2 */ + +struct _func_version_cache_item { + PyFunctionObject *func; + PyObject *code; +}; + struct _py_func_state { +#ifdef Py_GIL_DISABLED + // Protects next_version + PyMutex mutex; +#endif + uint32_t next_version; - // Borrowed references to function objects whose + // Borrowed references to function and code objects whose // func_version % FUNC_VERSION_CACHE_SIZE // once was equal to the index in the table. - // They are cleared when the function is deallocated. - PyFunctionObject *func_version_cache[FUNC_VERSION_CACHE_SIZE]; + // They are cleared when the function or code object is deallocated. + struct _func_version_cache_item func_version_cache[FUNC_VERSION_CACHE_SIZE]; }; extern PyFunctionObject* _PyFunction_FromConstructor(PyFrameConstructor *constr); +static inline int +_PyFunction_IsVersionValid(uint32_t version) +{ + return version >= FUNC_VERSION_FIRST_VALID; +} + extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); PyAPI_FUNC(void) _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version); -PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version); +void _PyFunction_ClearCodeByVersion(uint32_t version); +PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version, PyObject **p_code); extern PyObject *_Py_set_function_type_params( PyThreadState* unused, PyObject *func, PyObject *type_params); diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 40414a868518bb..b1806df2706097 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -8,15 +8,13 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState - /* GC information is stored BEFORE the object structure. */ typedef struct { - // Pointer to next object in the list. + // Tagged pointer to next object in the list. // 0 means the object is not tracked uintptr_t _gc_next; - // Pointer to previous object in the list. + // Tagged pointer to previous object in the list. // Lowest two bits are used for flags documented later. uintptr_t _gc_prev; } PyGC_Head; @@ -37,19 +35,53 @@ static inline PyObject* _Py_FROM_GC(PyGC_Head *gc) { } -/* Bit flags for ob_gc_bits (in Py_GIL_DISABLED builds) */ +/* Bit flags for ob_gc_bits (in Py_GIL_DISABLED builds) + * + * Setting the bits requires a relaxed store. The per-object lock must also be + * held, except when the object is only visible to a single thread (e.g. during + * object initialization or destruction). + * + * Reading the bits requires using a relaxed load, but does not require holding + * the per-object lock. + */ +#ifdef Py_GIL_DISABLED +# define _PyGC_BITS_TRACKED (1<<0) // Tracked by the GC +# define _PyGC_BITS_FINALIZED (1<<1) // tp_finalize was called +# define _PyGC_BITS_UNREACHABLE (1<<2) +# define _PyGC_BITS_FROZEN (1<<3) +# define _PyGC_BITS_SHARED (1<<4) +# define _PyGC_BITS_ALIVE (1<<5) // Reachable from a known root. +# define _PyGC_BITS_DEFERRED (1<<6) // Use deferred reference counting +#endif + #ifdef Py_GIL_DISABLED -# define _PyGC_BITS_TRACKED (1) -# define _PyGC_BITS_FINALIZED (2) -# define _PyGC_BITS_UNREACHABLE (4) -# define _PyGC_BITS_FROZEN (8) -# define _PyGC_BITS_SHARED (16) + +static inline void +_PyObject_SET_GC_BITS(PyObject *op, uint8_t new_bits) +{ + uint8_t bits = _Py_atomic_load_uint8_relaxed(&op->ob_gc_bits); + _Py_atomic_store_uint8_relaxed(&op->ob_gc_bits, bits | new_bits); +} + +static inline int +_PyObject_HAS_GC_BITS(PyObject *op, uint8_t bits) +{ + return (_Py_atomic_load_uint8_relaxed(&op->ob_gc_bits) & bits) != 0; +} + +static inline void +_PyObject_CLEAR_GC_BITS(PyObject *op, uint8_t bits_to_clear) +{ + uint8_t bits = _Py_atomic_load_uint8_relaxed(&op->ob_gc_bits); + _Py_atomic_store_uint8_relaxed(&op->ob_gc_bits, bits & ~bits_to_clear); +} + #endif /* True if the object is currently tracked by the GC. */ static inline int _PyObject_GC_IS_TRACKED(PyObject *op) { #ifdef Py_GIL_DISABLED - return (op->ob_gc_bits & _PyGC_BITS_TRACKED) != 0; + return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_TRACKED); #else PyGC_Head *gc = _Py_AS_GC(op); return (gc->_gc_next != 0); @@ -71,16 +103,19 @@ static inline int _PyObject_GC_MAY_BE_TRACKED(PyObject *obj) { #ifdef Py_GIL_DISABLED -/* True if an object is shared between multiple threads and - * needs special purpose when freeing to do the possibility - * of in-flight lock-free reads occuring */ +/* True if memory the object references is shared between + * multiple threads and needs special purpose when freeing + * those references due to the possibility of in-flight + * lock-free reads occurring. The object is responsible + * for calling _PyMem_FreeDelayed on the referenced + * memory. */ static inline int _PyObject_GC_IS_SHARED(PyObject *op) { - return (op->ob_gc_bits & _PyGC_BITS_SHARED) != 0; + return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_SHARED); } #define _PyObject_GC_IS_SHARED(op) _PyObject_GC_IS_SHARED(_Py_CAST(PyObject*, op)) static inline void _PyObject_GC_SET_SHARED(PyObject *op) { - op->ob_gc_bits |= _PyGC_BITS_SHARED; + _PyObject_SET_GC_BITS(op, _PyGC_BITS_SHARED); } #define _PyObject_GC_SET_SHARED(op) _PyObject_GC_SET_SHARED(_Py_CAST(PyObject*, op)) @@ -88,11 +123,26 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) { /* Bit flags for _gc_prev */ /* Bit 0 is set when tp_finalize is called */ -#define _PyGC_PREV_MASK_FINALIZED (1) +#define _PyGC_PREV_MASK_FINALIZED ((uintptr_t)1) /* Bit 1 is set when the object is in generation which is GCed currently. */ -#define _PyGC_PREV_MASK_COLLECTING (2) -/* The (N-2) most significant bits contain the real address. */ -#define _PyGC_PREV_SHIFT (2) +#define _PyGC_PREV_MASK_COLLECTING ((uintptr_t)2) + +/* Bit 0 in _gc_next is the old space bit. + * It is set as follows: + * Young: gcstate->visited_space + * old[0]: 0 + * old[1]: 1 + * permanent: 0 + * + * During a collection all objects handled should have the bit set to + * gcstate->visited_space, as objects are moved from the young gen + * and the increment into old[gcstate->visited_space]. + * When object are moved from the pending space, old[gcstate->visited_space^1] + * into the increment, the old space bit is flipped. +*/ +#define _PyGC_NEXT_MASK_OLD_SPACE_1 1 + +#define _PyGC_PREV_SHIFT 2 #define _PyGC_PREV_MASK (((uintptr_t) -1) << _PyGC_PREV_SHIFT) /* set for debugging information */ @@ -118,11 +168,13 @@ typedef enum { // Lowest bit of _gc_next is used for flags only in GC. // But it is always 0 for normal code. static inline PyGC_Head* _PyGCHead_NEXT(PyGC_Head *gc) { - uintptr_t next = gc->_gc_next; + uintptr_t next = gc->_gc_next & _PyGC_PREV_MASK; return (PyGC_Head*)next; } static inline void _PyGCHead_SET_NEXT(PyGC_Head *gc, PyGC_Head *next) { - gc->_gc_next = (uintptr_t)next; + uintptr_t unext = (uintptr_t)next; + assert((unext & ~_PyGC_PREV_MASK) == 0); + gc->_gc_next = (gc->_gc_next & ~_PyGC_PREV_MASK) | unext; } // Lowest two bits of _gc_prev is used for _PyGC_PREV_MASK_* flags. @@ -130,6 +182,7 @@ static inline PyGC_Head* _PyGCHead_PREV(PyGC_Head *gc) { uintptr_t prev = (gc->_gc_prev & _PyGC_PREV_MASK); return (PyGC_Head*)prev; } + static inline void _PyGCHead_SET_PREV(PyGC_Head *gc, PyGC_Head *prev) { uintptr_t uprev = (uintptr_t)prev; assert((uprev & ~_PyGC_PREV_MASK) == 0); @@ -138,7 +191,7 @@ static inline void _PyGCHead_SET_PREV(PyGC_Head *gc, PyGC_Head *prev) { static inline int _PyGC_FINALIZED(PyObject *op) { #ifdef Py_GIL_DISABLED - return (op->ob_gc_bits & _PyGC_BITS_FINALIZED) != 0; + return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_FINALIZED); #else PyGC_Head *gc = _Py_AS_GC(op); return ((gc->_gc_prev & _PyGC_PREV_MASK_FINALIZED) != 0); @@ -146,7 +199,7 @@ static inline int _PyGC_FINALIZED(PyObject *op) { } static inline void _PyGC_SET_FINALIZED(PyObject *op) { #ifdef Py_GIL_DISABLED - op->ob_gc_bits |= _PyGC_BITS_FINALIZED; + _PyObject_SET_GC_BITS(op, _PyGC_BITS_FINALIZED); #else PyGC_Head *gc = _Py_AS_GC(op); gc->_gc_prev |= _PyGC_PREV_MASK_FINALIZED; @@ -154,7 +207,7 @@ static inline void _PyGC_SET_FINALIZED(PyObject *op) { } static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) { #ifdef Py_GIL_DISABLED - op->ob_gc_bits &= ~_PyGC_BITS_FINALIZED; + _PyObject_CLEAR_GC_BITS(op, _PyGC_BITS_FINALIZED); #else PyGC_Head *gc = _Py_AS_GC(op); gc->_gc_prev &= ~_PyGC_PREV_MASK_FINALIZED; @@ -215,6 +268,13 @@ struct gc_generation { generations */ }; +struct gc_collection_stats { + /* number of collected objects */ + Py_ssize_t collected; + /* total number of uncollectable objects (put into gc.garbage) */ + Py_ssize_t uncollectable; +}; + /* Running stats per generation */ struct gc_generation_stats { /* total number of collections */ @@ -225,6 +285,11 @@ struct gc_generation_stats { Py_ssize_t uncollectable; }; +enum _GCPhase { + GC_PHASE_MARK = 0, + GC_PHASE_COLLECT = 1 +}; + struct _gc_runtime_state { /* List of objects that still need to be cleaned up, singly linked * via their gc headers' gc_prev pointers. */ @@ -236,8 +301,8 @@ struct _gc_runtime_state { int enabled; int debug; /* linked lists of container objects */ - struct gc_generation generations[NUM_GENERATIONS]; - PyGC_Head *generation0; + struct gc_generation young; + struct gc_generation old[2]; /* a permanent generation which won't be collected */ struct gc_generation permanent_generation; struct gc_generation_stats generation_stats[NUM_GENERATIONS]; @@ -247,6 +312,14 @@ struct _gc_runtime_state { PyObject *garbage; /* a list of callbacks to be invoked when collection is performed */ PyObject *callbacks; + + Py_ssize_t heap_size; + Py_ssize_t work_to_do; + /* Which of the old spaces is the visited space */ + int visited_space; + int phase; + +#ifdef Py_GIL_DISABLED /* This is the number of objects that survived the last full collection. It approximates the number of long lived objects tracked by the GC. @@ -258,6 +331,10 @@ struct _gc_runtime_state { collections, and are awaiting to undergo a full collection for the first time. */ Py_ssize_t long_lived_pending; + + /* True if gc.freeze() has been used. */ + int freeze_active; +#endif }; #ifdef Py_GIL_DISABLED @@ -270,9 +347,8 @@ struct _gc_thread_state { extern void _PyGC_InitState(struct _gc_runtime_state *); -extern Py_ssize_t _PyGC_Collect(PyThreadState *tstate, int generation, - _PyGC_Reason reason); -extern Py_ssize_t _PyGC_CollectNoFail(PyThreadState *tstate); +extern Py_ssize_t _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason); +extern void _PyGC_CollectNoFail(PyThreadState *tstate); /* Freeze objects tracked by the GC and ignore them in future collections. */ extern void _PyGC_Freeze(PyInterpreterState *interp); @@ -281,7 +357,7 @@ extern void _PyGC_Unfreeze(PyInterpreterState *interp); /* Number of frozen objects */ extern Py_ssize_t _PyGC_GetFreezeCount(PyInterpreterState *interp); -extern PyObject *_PyGC_GetObjects(PyInterpreterState *interp, Py_ssize_t generation); +extern PyObject *_PyGC_GetObjects(PyInterpreterState *interp, int generation); extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs); // Functions to clear types free lists @@ -289,6 +365,27 @@ extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); extern void _Py_ScheduleGC(PyThreadState *tstate); extern void _Py_RunGC(PyThreadState *tstate); +union _PyStackRef; + +// GC visit callback for tracked interpreter frames +extern int _PyGC_VisitFrameStack(struct _PyInterpreterFrame *frame, visitproc visit, void *arg); +extern int _PyGC_VisitStackRef(union _PyStackRef *ref, visitproc visit, void *arg); + +// Like Py_VISIT but for _PyStackRef fields +#define _Py_VISIT_STACKREF(ref) \ + do { \ + if (!PyStackRef_IsNull(ref)) { \ + int vret = _PyGC_VisitStackRef(&(ref), visit, arg); \ + if (vret) \ + return vret; \ + } \ + } while (0) + +#ifdef Py_GIL_DISABLED +extern void _PyGC_VisitObjectsWorldStopped(PyInterpreterState *interp, + gcvisitobjects_t callback, void *arg); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index 9463c822ad8669..f6d7e6d367177b 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -8,7 +8,49 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" +#include "pycore_frame.h" + +/* _PyGenObject_HEAD defines the initial segment of generator + and coroutine objects. */ +#define _PyGenObject_HEAD(prefix) \ + PyObject_HEAD \ + /* List of weak reference. */ \ + PyObject *prefix##_weakreflist; \ + /* Name of the generator. */ \ + PyObject *prefix##_name; \ + /* Qualified name of the generator. */ \ + PyObject *prefix##_qualname; \ + _PyErr_StackItem prefix##_exc_state; \ + PyObject *prefix##_origin_or_finalizer; \ + char prefix##_hooks_inited; \ + char prefix##_closed; \ + char prefix##_running_async; \ + /* The frame */ \ + int8_t prefix##_frame_state; \ + struct _PyInterpreterFrame prefix##_iframe; \ + +struct _PyGenObject { + /* The gi_ prefix is intended to remind of generator-iterator. */ + _PyGenObject_HEAD(gi) +}; + +struct _PyCoroObject { + _PyGenObject_HEAD(cr) +}; + +struct _PyAsyncGenObject { + _PyGenObject_HEAD(ag) +}; + +#undef _PyGenObject_HEAD + +static inline +PyGenObject *_PyGen_GetGeneratorFromFrame(_PyInterpreterFrame *frame) +{ + assert(frame->owner == FRAME_OWNED_BY_GENERATOR); + size_t offset_in_gen = offsetof(PyGenObject, gi_iframe); + return (PyGenObject *)(((char *)frame) - offset_in_gen); +} PyAPI_FUNC(PyObject *)_PyGen_yf(PyGenObject *); extern void _PyGen_Finalize(PyObject *self); diff --git a/Include/internal/pycore_gil.h b/Include/internal/pycore_gil.h index 19b0d23a68568a..a2de5077371eba 100644 --- a/Include/internal/pycore_gil.h +++ b/Include/internal/pycore_gil.h @@ -20,6 +20,23 @@ extern "C" { #define FORCE_SWITCHING struct _gil_runtime_state { +#ifdef Py_GIL_DISABLED + /* If this GIL is disabled, enabled == 0. + + If this GIL is enabled transiently (most likely to initialize a module + of unknown safety), enabled indicates the number of active transient + requests. + + If this GIL is enabled permanently, enabled == INT_MAX. + + It must not be modified directly; use _PyEval_EnableGILTransiently(), + _PyEval_EnableGILPermanently(), and _PyEval_DisableGIL() + + It is always read and written atomically, but a thread can assume its + value will be stable as long as that thread is attached or knows that no + other threads are attached (e.g., during a stop-the-world.). */ + int enabled; +#endif /* microseconds (the Python API uses seconds, though) */ unsigned long interval; /* Last PyThreadState holding / having held the GIL. This helps us diff --git a/Include/internal/pycore_global_objects.h b/Include/internal/pycore_global_objects.h index 327fcc24cb29f1..e3f7ac707f0c37 100644 --- a/Include/internal/pycore_global_objects.h +++ b/Include/internal/pycore_global_objects.h @@ -66,9 +66,6 @@ struct _Py_static_objects { struct _Py_interp_cached_objects { PyObject *interned_strings; - /* AST */ - PyObject *str_replace_inf; - /* object.__reduce__ */ PyObject *objreduce; PyObject *type_slots_pname; @@ -81,6 +78,7 @@ struct _Py_interp_cached_objects { PyTypeObject *paramspec_type; PyTypeObject *paramspecargs_type; PyTypeObject *paramspeckwargs_type; + PyTypeObject *constevaluator_type; }; #define _Py_INTERP_STATIC_OBJECT(interp, NAME) \ diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index e8b62bd0dcd369..90214a314031d1 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -11,7 +11,7 @@ extern "C" { #ifdef Py_DEBUG static inline void _PyStaticObject_CheckRefcnt(PyObject *obj) { - if (Py_REFCNT(obj) < _Py_IMMORTAL_REFCNT) { + if (!_Py_IsImmortal(obj)) { fprintf(stderr, "Immortal Object has less refcnt than expected.\n"); _PyObject_Dump(obj); } @@ -551,21 +551,18 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_setcomp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_string)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_unknown)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(close_br)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dbl_close_br)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dbl_open_br)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dbl_percent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(defaults)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot_locals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(empty)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(format)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(generic_base)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(json_decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(kwdefaults)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(list_err)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(newline)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(open_br)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(percent)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(str_replace_inf)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(type_params)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(utf_8)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CANCELLED)); @@ -577,7 +574,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(TextIOWrapper)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(True)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(WarningMessage)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_WindowsConsoleIO)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__IOBase_closed)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__abc_tpflags__)); @@ -590,9 +586,9 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__all__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__and__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__anext__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotate__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotations__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__args__)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__asyncio_running_event_loop__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__await__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__bases__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__bool__)); @@ -609,7 +605,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__classdictcell__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__complex__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__contains__)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__copy__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__ctypes_from_outparam__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__del__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__delattr__)); @@ -624,6 +619,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__eq__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__exit__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__file__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__firstlineno__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__float__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__floordiv__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__format__)); @@ -724,12 +720,12 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__slotnames__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__slots__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__spec__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__static_attributes__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__str__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__sub__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__subclasscheck__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__subclasshook__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__truediv__)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__trunc__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__type_params__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__typing_is_unpacked_typevartuple__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__typing_prepare_subst__)); @@ -742,8 +738,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abc_impl)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abstract_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_active)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_align_)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_annotation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_anonymous_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_argtypes_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_as_parameter_)); @@ -764,23 +758,25 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_initializing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_io)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_is_text_encoding)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_isatty_open_only)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_length_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_limbo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_lock_unlock_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_loop)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_needs_com_addref_)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_pack_)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_only_immortal)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_restype_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_showwarnmsg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_shutdown)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_slotnames)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime_datetime)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_swappedbytes_)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime_datetime_date)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime_datetime_datetime)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime_datetime_time)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_type_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_uninitialized_submodules)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_warn_unawaited_coroutine)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_xoptions)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(a)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(abs_tol)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(access)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aclose)); @@ -790,8 +786,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aggregate_class)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(align)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(allow_code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argdefs)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(args)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arguments)); @@ -803,12 +801,12 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(attribute)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(authorizer_callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(autocommit)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(b)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(backtick)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(base)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(before)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(big)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(binary_form)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bit_size)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(block)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bound)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(buffer)); @@ -821,10 +819,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(byteorder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bytes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bytes_per_sep)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_call)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_exception)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_return)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_datetime_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_statements)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cadata)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cafile)); @@ -862,6 +860,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(co_stacksize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(co_varnames)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(code)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(col_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(command)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(comment_factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(compile_mode)); @@ -875,7 +874,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(count)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(covariant)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cwd)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(d)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(data)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(database)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(day)); @@ -901,16 +899,17 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(displayhook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dklen)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(doc)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(done)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dont_inherit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dst)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dst_dir_fd)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(e)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eager_start)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(effective_ids)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(element_factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(encode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(encoding)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end_col_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end_lineno)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(endpos)); @@ -927,7 +926,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extend)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extra_tokens)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(f)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(facility)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(false)); @@ -937,6 +935,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fd2)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fdel)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fget)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fields)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(file)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(file_actions)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(filename)); @@ -953,6 +952,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fold)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(follow_symlinks)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format_spec)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(from_param)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fromlist)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fromtimestamp)); @@ -960,7 +960,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(func)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(future)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(g)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(generation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(genexpr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get)); @@ -974,9 +973,9 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(globals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(groupindex)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(groups)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(h)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(handle)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(handle_seq)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(has_location)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hash_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(header)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(headers)); @@ -991,6 +990,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(importlib)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(in_fd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(incoming)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(index)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(indexgroup)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(inf)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(infer_variance)); @@ -1011,6 +1011,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(intersection)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(interval)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(is_running)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(is_struct)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isatty)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isinstance)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isoformat)); @@ -1032,6 +1033,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw1)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw2)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kwdefaults)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(label)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(lambda)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last_exc)); @@ -1084,7 +1086,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(msg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mutex)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mycmp)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n_arg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n_fields)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n_sequence_fields)); @@ -1095,6 +1096,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespaces)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(narg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ndigits)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(nested)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_file_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_limit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(newline)); @@ -1127,9 +1129,9 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(origin)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(out_fd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(outgoing)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(outpath)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(overlapped)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(owner)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(p)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pages)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(password)); @@ -1157,7 +1159,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ps2)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(query)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(quotetabs)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(r)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(raw)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(read)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(read1)); @@ -1181,7 +1182,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(return)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reverse)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reversed)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(s)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(salt)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sched_priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(scheduler)); @@ -1214,6 +1214,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sort)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(source)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(source_traceback)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(spam)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(src)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(src_dir_fd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stacklevel)); @@ -1232,6 +1233,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(strict_mode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(string)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sub_key)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(subcalls)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(symmetric_difference_update)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tabsize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tag)); @@ -1249,8 +1251,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(threading)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(throw)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timeout)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timer)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(times)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timetuple)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timeunit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(top)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(trace_callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(traceback)); @@ -1275,6 +1279,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(version)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(volume)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(wait_all)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(warn_on_full_buffer)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(warnings)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(warnoptions)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(wbits)); @@ -1286,7 +1291,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(writable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(write)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(write_through)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(x)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(year)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(zdict)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(strings).ascii[0]); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index b41f5c8952021e..97a75d0c46c867 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -37,21 +37,18 @@ struct _Py_global_strings { STRUCT_FOR_STR(anon_setcomp, "") STRUCT_FOR_STR(anon_string, "") STRUCT_FOR_STR(anon_unknown, "") - STRUCT_FOR_STR(close_br, "}") STRUCT_FOR_STR(dbl_close_br, "}}") STRUCT_FOR_STR(dbl_open_br, "{{") STRUCT_FOR_STR(dbl_percent, "%%") STRUCT_FOR_STR(defaults, ".defaults") - STRUCT_FOR_STR(dot, ".") STRUCT_FOR_STR(dot_locals, ".") STRUCT_FOR_STR(empty, "") + STRUCT_FOR_STR(format, ".format") STRUCT_FOR_STR(generic_base, ".generic_base") STRUCT_FOR_STR(json_decoder, "json.decoder") STRUCT_FOR_STR(kwdefaults, ".kwdefaults") STRUCT_FOR_STR(list_err, "list index out of range") - STRUCT_FOR_STR(newline, "\n") - STRUCT_FOR_STR(open_br, "{") - STRUCT_FOR_STR(percent, "%") + STRUCT_FOR_STR(str_replace_inf, "1e309") STRUCT_FOR_STR(type_params, ".type_params") STRUCT_FOR_STR(utf_8, "utf-8") } literals; @@ -66,7 +63,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(TextIOWrapper) STRUCT_FOR_ID(True) STRUCT_FOR_ID(WarningMessage) - STRUCT_FOR_ID(_) STRUCT_FOR_ID(_WindowsConsoleIO) STRUCT_FOR_ID(__IOBase_closed) STRUCT_FOR_ID(__abc_tpflags__) @@ -79,9 +75,9 @@ struct _Py_global_strings { STRUCT_FOR_ID(__all__) STRUCT_FOR_ID(__and__) STRUCT_FOR_ID(__anext__) + STRUCT_FOR_ID(__annotate__) STRUCT_FOR_ID(__annotations__) STRUCT_FOR_ID(__args__) - STRUCT_FOR_ID(__asyncio_running_event_loop__) STRUCT_FOR_ID(__await__) STRUCT_FOR_ID(__bases__) STRUCT_FOR_ID(__bool__) @@ -98,7 +94,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(__classdictcell__) STRUCT_FOR_ID(__complex__) STRUCT_FOR_ID(__contains__) - STRUCT_FOR_ID(__copy__) STRUCT_FOR_ID(__ctypes_from_outparam__) STRUCT_FOR_ID(__del__) STRUCT_FOR_ID(__delattr__) @@ -113,6 +108,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__eq__) STRUCT_FOR_ID(__exit__) STRUCT_FOR_ID(__file__) + STRUCT_FOR_ID(__firstlineno__) STRUCT_FOR_ID(__float__) STRUCT_FOR_ID(__floordiv__) STRUCT_FOR_ID(__format__) @@ -213,12 +209,12 @@ struct _Py_global_strings { STRUCT_FOR_ID(__slotnames__) STRUCT_FOR_ID(__slots__) STRUCT_FOR_ID(__spec__) + STRUCT_FOR_ID(__static_attributes__) STRUCT_FOR_ID(__str__) STRUCT_FOR_ID(__sub__) STRUCT_FOR_ID(__subclasscheck__) STRUCT_FOR_ID(__subclasshook__) STRUCT_FOR_ID(__truediv__) - STRUCT_FOR_ID(__trunc__) STRUCT_FOR_ID(__type_params__) STRUCT_FOR_ID(__typing_is_unpacked_typevartuple__) STRUCT_FOR_ID(__typing_prepare_subst__) @@ -231,8 +227,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(_abc_impl) STRUCT_FOR_ID(_abstract_) STRUCT_FOR_ID(_active) - STRUCT_FOR_ID(_align_) - STRUCT_FOR_ID(_annotation) STRUCT_FOR_ID(_anonymous_) STRUCT_FOR_ID(_argtypes_) STRUCT_FOR_ID(_as_parameter_) @@ -253,23 +247,25 @@ struct _Py_global_strings { STRUCT_FOR_ID(_initializing) STRUCT_FOR_ID(_io) STRUCT_FOR_ID(_is_text_encoding) + STRUCT_FOR_ID(_isatty_open_only) STRUCT_FOR_ID(_length_) STRUCT_FOR_ID(_limbo) STRUCT_FOR_ID(_lock_unlock_module) STRUCT_FOR_ID(_loop) STRUCT_FOR_ID(_needs_com_addref_) - STRUCT_FOR_ID(_pack_) + STRUCT_FOR_ID(_only_immortal) STRUCT_FOR_ID(_restype_) STRUCT_FOR_ID(_showwarnmsg) STRUCT_FOR_ID(_shutdown) STRUCT_FOR_ID(_slotnames) - STRUCT_FOR_ID(_strptime_datetime) - STRUCT_FOR_ID(_swappedbytes_) + STRUCT_FOR_ID(_strptime) + STRUCT_FOR_ID(_strptime_datetime_date) + STRUCT_FOR_ID(_strptime_datetime_datetime) + STRUCT_FOR_ID(_strptime_datetime_time) STRUCT_FOR_ID(_type_) STRUCT_FOR_ID(_uninitialized_submodules) STRUCT_FOR_ID(_warn_unawaited_coroutine) STRUCT_FOR_ID(_xoptions) - STRUCT_FOR_ID(a) STRUCT_FOR_ID(abs_tol) STRUCT_FOR_ID(access) STRUCT_FOR_ID(aclose) @@ -279,8 +275,10 @@ struct _Py_global_strings { STRUCT_FOR_ID(after_in_parent) STRUCT_FOR_ID(aggregate_class) STRUCT_FOR_ID(alias) + STRUCT_FOR_ID(align) STRUCT_FOR_ID(allow_code) STRUCT_FOR_ID(append) + STRUCT_FOR_ID(arg) STRUCT_FOR_ID(argdefs) STRUCT_FOR_ID(args) STRUCT_FOR_ID(arguments) @@ -292,12 +290,12 @@ struct _Py_global_strings { STRUCT_FOR_ID(attribute) STRUCT_FOR_ID(authorizer_callback) STRUCT_FOR_ID(autocommit) - STRUCT_FOR_ID(b) STRUCT_FOR_ID(backtick) STRUCT_FOR_ID(base) STRUCT_FOR_ID(before) STRUCT_FOR_ID(big) STRUCT_FOR_ID(binary_form) + STRUCT_FOR_ID(bit_size) STRUCT_FOR_ID(block) STRUCT_FOR_ID(bound) STRUCT_FOR_ID(buffer) @@ -310,10 +308,10 @@ struct _Py_global_strings { STRUCT_FOR_ID(byteorder) STRUCT_FOR_ID(bytes) STRUCT_FOR_ID(bytes_per_sep) - STRUCT_FOR_ID(c) STRUCT_FOR_ID(c_call) STRUCT_FOR_ID(c_exception) STRUCT_FOR_ID(c_return) + STRUCT_FOR_ID(cached_datetime_module) STRUCT_FOR_ID(cached_statements) STRUCT_FOR_ID(cadata) STRUCT_FOR_ID(cafile) @@ -351,6 +349,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(co_stacksize) STRUCT_FOR_ID(co_varnames) STRUCT_FOR_ID(code) + STRUCT_FOR_ID(col_offset) STRUCT_FOR_ID(command) STRUCT_FOR_ID(comment_factory) STRUCT_FOR_ID(compile_mode) @@ -364,7 +363,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(count) STRUCT_FOR_ID(covariant) STRUCT_FOR_ID(cwd) - STRUCT_FOR_ID(d) STRUCT_FOR_ID(data) STRUCT_FOR_ID(database) STRUCT_FOR_ID(day) @@ -390,16 +388,17 @@ struct _Py_global_strings { STRUCT_FOR_ID(displayhook) STRUCT_FOR_ID(dklen) STRUCT_FOR_ID(doc) + STRUCT_FOR_ID(done) STRUCT_FOR_ID(dont_inherit) STRUCT_FOR_ID(dst) STRUCT_FOR_ID(dst_dir_fd) - STRUCT_FOR_ID(e) STRUCT_FOR_ID(eager_start) STRUCT_FOR_ID(effective_ids) STRUCT_FOR_ID(element_factory) STRUCT_FOR_ID(encode) STRUCT_FOR_ID(encoding) STRUCT_FOR_ID(end) + STRUCT_FOR_ID(end_col_offset) STRUCT_FOR_ID(end_lineno) STRUCT_FOR_ID(end_offset) STRUCT_FOR_ID(endpos) @@ -416,7 +415,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(exp) STRUCT_FOR_ID(extend) STRUCT_FOR_ID(extra_tokens) - STRUCT_FOR_ID(f) STRUCT_FOR_ID(facility) STRUCT_FOR_ID(factory) STRUCT_FOR_ID(false) @@ -426,6 +424,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(fd2) STRUCT_FOR_ID(fdel) STRUCT_FOR_ID(fget) + STRUCT_FOR_ID(fields) STRUCT_FOR_ID(file) STRUCT_FOR_ID(file_actions) STRUCT_FOR_ID(filename) @@ -442,6 +441,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(fold) STRUCT_FOR_ID(follow_symlinks) STRUCT_FOR_ID(format) + STRUCT_FOR_ID(format_spec) STRUCT_FOR_ID(from_param) STRUCT_FOR_ID(fromlist) STRUCT_FOR_ID(fromtimestamp) @@ -449,7 +449,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(fset) STRUCT_FOR_ID(func) STRUCT_FOR_ID(future) - STRUCT_FOR_ID(g) STRUCT_FOR_ID(generation) STRUCT_FOR_ID(genexpr) STRUCT_FOR_ID(get) @@ -463,9 +462,9 @@ struct _Py_global_strings { STRUCT_FOR_ID(globals) STRUCT_FOR_ID(groupindex) STRUCT_FOR_ID(groups) - STRUCT_FOR_ID(h) STRUCT_FOR_ID(handle) STRUCT_FOR_ID(handle_seq) + STRUCT_FOR_ID(has_location) STRUCT_FOR_ID(hash_name) STRUCT_FOR_ID(header) STRUCT_FOR_ID(headers) @@ -480,6 +479,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(importlib) STRUCT_FOR_ID(in_fd) STRUCT_FOR_ID(incoming) + STRUCT_FOR_ID(index) STRUCT_FOR_ID(indexgroup) STRUCT_FOR_ID(inf) STRUCT_FOR_ID(infer_variance) @@ -500,6 +500,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(intersection) STRUCT_FOR_ID(interval) STRUCT_FOR_ID(is_running) + STRUCT_FOR_ID(is_struct) STRUCT_FOR_ID(isatty) STRUCT_FOR_ID(isinstance) STRUCT_FOR_ID(isoformat) @@ -521,6 +522,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(kw1) STRUCT_FOR_ID(kw2) STRUCT_FOR_ID(kwdefaults) + STRUCT_FOR_ID(label) STRUCT_FOR_ID(lambda) STRUCT_FOR_ID(last) STRUCT_FOR_ID(last_exc) @@ -573,7 +575,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(msg) STRUCT_FOR_ID(mutex) STRUCT_FOR_ID(mycmp) - STRUCT_FOR_ID(n) STRUCT_FOR_ID(n_arg) STRUCT_FOR_ID(n_fields) STRUCT_FOR_ID(n_sequence_fields) @@ -584,6 +585,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(namespaces) STRUCT_FOR_ID(narg) STRUCT_FOR_ID(ndigits) + STRUCT_FOR_ID(nested) STRUCT_FOR_ID(new_file_name) STRUCT_FOR_ID(new_limit) STRUCT_FOR_ID(newline) @@ -616,9 +618,9 @@ struct _Py_global_strings { STRUCT_FOR_ID(origin) STRUCT_FOR_ID(out_fd) STRUCT_FOR_ID(outgoing) + STRUCT_FOR_ID(outpath) STRUCT_FOR_ID(overlapped) STRUCT_FOR_ID(owner) - STRUCT_FOR_ID(p) STRUCT_FOR_ID(pages) STRUCT_FOR_ID(parent) STRUCT_FOR_ID(password) @@ -646,7 +648,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(ps2) STRUCT_FOR_ID(query) STRUCT_FOR_ID(quotetabs) - STRUCT_FOR_ID(r) STRUCT_FOR_ID(raw) STRUCT_FOR_ID(read) STRUCT_FOR_ID(read1) @@ -670,7 +671,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(return) STRUCT_FOR_ID(reverse) STRUCT_FOR_ID(reversed) - STRUCT_FOR_ID(s) STRUCT_FOR_ID(salt) STRUCT_FOR_ID(sched_priority) STRUCT_FOR_ID(scheduler) @@ -703,6 +703,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(sort) STRUCT_FOR_ID(source) STRUCT_FOR_ID(source_traceback) + STRUCT_FOR_ID(spam) STRUCT_FOR_ID(src) STRUCT_FOR_ID(src_dir_fd) STRUCT_FOR_ID(stacklevel) @@ -721,6 +722,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(strict_mode) STRUCT_FOR_ID(string) STRUCT_FOR_ID(sub_key) + STRUCT_FOR_ID(subcalls) STRUCT_FOR_ID(symmetric_difference_update) STRUCT_FOR_ID(tabsize) STRUCT_FOR_ID(tag) @@ -738,8 +740,10 @@ struct _Py_global_strings { STRUCT_FOR_ID(threading) STRUCT_FOR_ID(throw) STRUCT_FOR_ID(timeout) + STRUCT_FOR_ID(timer) STRUCT_FOR_ID(times) STRUCT_FOR_ID(timetuple) + STRUCT_FOR_ID(timeunit) STRUCT_FOR_ID(top) STRUCT_FOR_ID(trace_callback) STRUCT_FOR_ID(traceback) @@ -764,6 +768,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(version) STRUCT_FOR_ID(volume) STRUCT_FOR_ID(wait_all) + STRUCT_FOR_ID(warn_on_full_buffer) STRUCT_FOR_ID(warnings) STRUCT_FOR_ID(warnoptions) STRUCT_FOR_ID(wbits) @@ -775,7 +780,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(writable) STRUCT_FOR_ID(write) STRUCT_FOR_ID(write_through) - STRUCT_FOR_ID(x) STRUCT_FOR_ID(year) STRUCT_FOR_ID(zdict) } identifiers; @@ -798,6 +802,10 @@ struct _Py_global_strings { (_Py_SINGLETON(strings.identifiers._py_ ## NAME._ascii.ob_base)) #define _Py_STR(NAME) \ (_Py_SINGLETON(strings.literals._py_ ## NAME._ascii.ob_base)) +#define _Py_LATIN1_CHR(CH) \ + ((CH) < 128 \ + ? (PyObject*)&_Py_SINGLETON(strings).ascii[(CH)] \ + : (PyObject*)&_Py_SINGLETON(strings).latin1[(CH) - 128]) /* _Py_DECLARE_STR() should precede all uses of _Py_STR() in a function. diff --git a/Include/internal/pycore_identifier.h b/Include/internal/pycore_identifier.h deleted file mode 100644 index cda28810a48196..00000000000000 --- a/Include/internal/pycore_identifier.h +++ /dev/null @@ -1,20 +0,0 @@ -/* String Literals: _Py_Identifier API */ - -#ifndef Py_INTERNAL_IDENTIFIER_H -#define Py_INTERNAL_IDENTIFIER_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - -extern PyObject* _PyType_LookupId(PyTypeObject *, _Py_Identifier *); -extern PyObject* _PyObject_LookupSpecialId(PyObject *, _Py_Identifier *); -extern int _PyObject_SetAttrId(PyObject *, _Py_Identifier *, PyObject *); - -#ifdef __cplusplus -} -#endif -#endif // !Py_INTERNAL_IDENTIFIER_H diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index eb8a9a0db46c22..318c712bdfa174 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -20,15 +20,16 @@ PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module); extern int _PyImport_SetModuleString(const char *name, PyObject* module); extern void _PyImport_AcquireLock(PyInterpreterState *interp); -extern int _PyImport_ReleaseLock(PyInterpreterState *interp); +extern void _PyImport_ReleaseLock(PyInterpreterState *interp); +extern void _PyImport_ReInitLock(PyInterpreterState *interp); +// This is used exclusively for the sys and builtins modules: extern int _PyImport_FixupBuiltin( + PyThreadState *tstate, PyObject *mod, const char *name, /* UTF-8 encoded string */ PyObject *modules ); -extern int _PyImport_FixupExtensionObject(PyObject*, PyObject *, - PyObject *, PyObject *); // Export for many shared extensions, like '_json' PyAPI_FUNC(PyObject*) _PyImport_GetModuleAttr(PyObject *, PyObject *); @@ -50,9 +51,9 @@ struct _import_runtime_state { PyMutex mutex; /* The actual cache of (filename, name, PyModuleDef) for modules. Only legacy (single-phase init) extension modules are added - and only if they support multiple initialization (m_size >- 0) + and only if they support multiple initialization (m_size >= 0) or are imported in the main interpreter. - This is initialized lazily in _PyImport_FixupExtensionObject(). + This is initialized lazily in fix_up_extension() in import.c. Modules are added there and looked up in _imp.find_extension(). */ _Py_hashtable_t *hashtable; } extensions; @@ -94,11 +95,7 @@ struct _import_state { #endif PyObject *import_func; /* The global import lock. */ - struct { - PyThread_type_lock mutex; - unsigned long thread; - int level; - } lock; + _PyRecursiveMutex lock; /* diagnostic info in PyImport_ImportModuleLevelObject() */ struct { int import_level; @@ -123,11 +120,6 @@ struct _import_state { #define IMPORTS_INIT \ { \ DLOPENFLAGS_INIT \ - .lock = { \ - .mutex = NULL, \ - .thread = PYTHREAD_INVALID_THREAD_ID, \ - .level = 0, \ - }, \ .find_and_load = { \ .header = 1, \ }, \ @@ -180,11 +172,6 @@ extern void _PyImport_FiniCore(PyInterpreterState *interp); extern void _PyImport_FiniExternal(PyInterpreterState *interp); -#ifdef HAVE_FORK -extern PyStatus _PyImport_ReInitLock(PyInterpreterState *interp); -#endif - - extern PyObject* _PyImport_GetBuiltinModuleNames(void); struct _module_alias { @@ -206,6 +193,19 @@ extern int _PyImport_CheckSubinterpIncompatibleExtensionAllowed( // Export for '_testinternalcapi' shared extension PyAPI_FUNC(int) _PyImport_ClearExtension(PyObject *name, PyObject *filename); +#ifdef Py_GIL_DISABLED +// Assuming that the GIL is enabled from a call to +// _PyEval_EnableGILTransient(), resolve the transient request depending on the +// state of the module argument: +// - If module is NULL or a PyModuleObject with md_gil == Py_MOD_GIL_NOT_USED, +// call _PyEval_DisableGIL(). +// - Otherwise, call _PyEval_EnableGILPermanent(). If the GIL was not already +// enabled permanently, issue a warning referencing the module's name. +// +// This function may raise an exception. +extern int _PyImport_CheckGILForModule(PyObject *module, PyObject *module_name); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index c8583582b358ac..525a16f6b97274 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -14,9 +14,91 @@ extern "C" { extern const char *_PyImport_DynLoadFiletab[]; -extern PyObject *_PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *); +typedef enum ext_module_kind { + _Py_ext_module_kind_UNKNOWN = 0, + _Py_ext_module_kind_SINGLEPHASE = 1, + _Py_ext_module_kind_MULTIPHASE = 2, + _Py_ext_module_kind_INVALID = 3, +} _Py_ext_module_kind; + +typedef enum ext_module_origin { + _Py_ext_module_origin_CORE = 1, + _Py_ext_module_origin_BUILTIN = 2, + _Py_ext_module_origin_DYNAMIC = 3, +} _Py_ext_module_origin; + +/* Input for loading an extension module. */ +struct _Py_ext_module_loader_info { + PyObject *filename; +#ifndef MS_WINDOWS + PyObject *filename_encoded; +#endif + PyObject *name; + PyObject *name_encoded; + /* path is always a borrowed ref of name or filename, + * depending on if it's builtin or not. */ + PyObject *path; + _Py_ext_module_origin origin; + const char *hook_prefix; + const char *newcontext; +}; +extern void _Py_ext_module_loader_info_clear( + struct _Py_ext_module_loader_info *info); +extern int _Py_ext_module_loader_info_init( + struct _Py_ext_module_loader_info *info, + PyObject *name, + PyObject *filename, + _Py_ext_module_origin origin); +extern int _Py_ext_module_loader_info_init_for_core( + struct _Py_ext_module_loader_info *p_info, + PyObject *name); +extern int _Py_ext_module_loader_info_init_for_builtin( + struct _Py_ext_module_loader_info *p_info, + PyObject *name); +#ifdef HAVE_DYNAMIC_LOADING +extern int _Py_ext_module_loader_info_init_from_spec( + struct _Py_ext_module_loader_info *info, + PyObject *spec); +#endif + +/* The result from running an extension module's init function. */ +struct _Py_ext_module_loader_result { + PyModuleDef *def; + PyObject *module; + _Py_ext_module_kind kind; + struct _Py_ext_module_loader_result_error *err; + struct _Py_ext_module_loader_result_error { + enum _Py_ext_module_loader_result_error_kind { + _Py_ext_module_loader_result_EXCEPTION = 0, + _Py_ext_module_loader_result_ERR_MISSING = 1, + _Py_ext_module_loader_result_ERR_UNREPORTED_EXC = 2, + _Py_ext_module_loader_result_ERR_UNINITIALIZED = 3, + _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE = 4, + _Py_ext_module_loader_result_ERR_NOT_MODULE = 5, + _Py_ext_module_loader_result_ERR_MISSING_DEF = 6, + } kind; + PyObject *exc; + } _err; +}; +extern void _Py_ext_module_loader_result_clear( + struct _Py_ext_module_loader_result *res); +extern void _Py_ext_module_loader_result_apply_error( + struct _Py_ext_module_loader_result *res, + const char *name); + +/* The module init function. */ typedef PyObject *(*PyModInitFunction)(void); +#ifdef HAVE_DYNAMIC_LOADING +extern PyModInitFunction _PyImport_GetModInitFunc( + struct _Py_ext_module_loader_info *info, + FILE *fp); +#endif +extern int _PyImport_RunModInitFunc( + PyModInitFunction p0, + struct _Py_ext_module_loader_info *info, + struct _Py_ext_module_loader_result *p_res); + /* Max length of module suffix searched for -- accommodates "module.slb" */ #define MAXSUFFIXSIZE 12 diff --git a/Include/internal/pycore_index_pool.h b/Include/internal/pycore_index_pool.h new file mode 100644 index 00000000000000..e81bfd4d6ed03d --- /dev/null +++ b/Include/internal/pycore_index_pool.h @@ -0,0 +1,56 @@ +#ifndef Py_INTERNAL_INDEX_POOL_H +#define Py_INTERNAL_INDEX_POOL_H + +#include "Python.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef Py_GIL_DISABLED + +// This contains code for allocating unique indices in an array. It is used by +// the free-threaded build to assign each thread a globally unique index into +// each code object's thread-local bytecode array. + +// A min-heap of indices +typedef struct _PyIndexHeap { + int32_t *values; + + // Number of items stored in values + Py_ssize_t size; + + // Maximum number of items that can be stored in values + Py_ssize_t capacity; +} _PyIndexHeap; + +// An unbounded pool of indices. Indices are allocated starting from 0. They +// may be released back to the pool once they are no longer in use. +typedef struct _PyIndexPool { + PyMutex mutex; + + // Min heap of indices available for allocation + _PyIndexHeap free_indices; + + // Next index to allocate if no free indices are available + int32_t next_index; +} _PyIndexPool; + +// Allocate the smallest available index. Returns -1 on error. +extern int32_t _PyIndexPool_AllocIndex(_PyIndexPool *indices); + +// Release `index` back to the pool +extern void _PyIndexPool_FreeIndex(_PyIndexPool *indices, int32_t index); + +extern void _PyIndexPool_Fini(_PyIndexPool *indices); + +#endif // Py_GIL_DISABLED + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_INDEX_POOL_H diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index c86988234f6a05..25ec8cec3bd249 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -153,6 +153,18 @@ typedef enum { _PyConfig_INIT_ISOLATED = 3 } _PyConfigInitEnum; +typedef enum { + /* For now, this means the GIL is enabled. + + gh-116329: This will eventually change to "the GIL is disabled but can + be re-enabled by loading an incompatible extension module." */ + _PyConfig_GIL_DEFAULT = -1, + + /* The GIL has been forced off or on, and will not be affected by module loading. */ + _PyConfig_GIL_DISABLE = 0, + _PyConfig_GIL_ENABLE = 1, +} _PyConfigGILEnum; + // Export for '_testembed' program PyAPI_FUNC(void) _PyConfig_InitCompatConfig(PyConfig *config); @@ -169,7 +181,7 @@ extern PyStatus _PyConfig_Write(const PyConfig *config, extern PyStatus _PyConfig_SetPyArgv( PyConfig *config, const _PyArgv *args); - +extern PyObject* _PyConfig_CreateXOptionsDict(const PyConfig *config); extern void _Py_DumpPathConfig(PyThreadState *tstate); diff --git a/Include/internal/pycore_instruction_sequence.h b/Include/internal/pycore_instruction_sequence.h new file mode 100644 index 00000000000000..099f2fd124007a --- /dev/null +++ b/Include/internal/pycore_instruction_sequence.h @@ -0,0 +1,78 @@ +#ifndef Py_INTERNAL_INSTRUCTION_SEQUENCE_H +#define Py_INTERNAL_INSTRUCTION_SEQUENCE_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_symtable.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +typedef struct { + int h_label; + int h_startdepth; + int h_preserve_lasti; +} _PyExceptHandlerInfo; + +typedef struct { + int i_opcode; + int i_oparg; + _Py_SourceLocation i_loc; + _PyExceptHandlerInfo i_except_handler_info; + + /* Temporary fields, used by the assembler and in instr_sequence_to_cfg */ + int i_target; + int i_offset; +} _PyInstruction; + +typedef struct instruction_sequence { + PyObject_HEAD + _PyInstruction *s_instrs; + int s_allocated; + int s_used; + + int s_next_free_label; /* next free label id */ + + /* Map of a label id to instruction offset (index into s_instrs). + * If s_labelmap is NULL, then each label id is the offset itself. + */ + int *s_labelmap; + int s_labelmap_size; + + /* PyList of instruction sequences of nested functions */ + PyObject *s_nested; +} _PyInstructionSequence; + +typedef struct { + int id; +} _PyJumpTargetLabel; + +#define NO_LABEL ((const _PyJumpTargetLabel){-1}) + +#define SAME_JUMP_TARGET_LABEL(L1, L2) ((L1).id == (L2).id) +#define IS_JUMP_TARGET_LABEL(L) (!SAME_JUMP_TARGET_LABEL((L), (NO_LABEL))) + +PyAPI_FUNC(PyObject*)_PyInstructionSequence_New(void); + +int _PyInstructionSequence_UseLabel(_PyInstructionSequence *seq, int lbl); +int _PyInstructionSequence_Addop(_PyInstructionSequence *seq, + int opcode, int oparg, + _Py_SourceLocation loc); +_PyJumpTargetLabel _PyInstructionSequence_NewLabel(_PyInstructionSequence *seq); +int _PyInstructionSequence_ApplyLabelMap(_PyInstructionSequence *seq); +int _PyInstructionSequence_InsertInstruction(_PyInstructionSequence *seq, int pos, + int opcode, int oparg, _Py_SourceLocation loc); +int _PyInstructionSequence_AddNested(_PyInstructionSequence *seq, _PyInstructionSequence *nested); +void PyInstructionSequence_Fini(_PyInstructionSequence *seq); + +extern PyTypeObject _PyInstructionSequence_Type; +#define _PyInstructionSequence_Check(v) Py_IS_TYPE((v), &_PyInstructionSequence_Type) + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_INSTRUCTION_SEQUENCE_H */ diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index eae8371ef7f9b8..92d8f056f402fc 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -13,38 +13,6 @@ extern "C" { #define PY_MONITORING_TOOL_IDS 8 -/* Local events. - * These require bytecode instrumentation */ - -#define PY_MONITORING_EVENT_PY_START 0 -#define PY_MONITORING_EVENT_PY_RESUME 1 -#define PY_MONITORING_EVENT_PY_RETURN 2 -#define PY_MONITORING_EVENT_PY_YIELD 3 -#define PY_MONITORING_EVENT_CALL 4 -#define PY_MONITORING_EVENT_LINE 5 -#define PY_MONITORING_EVENT_INSTRUCTION 6 -#define PY_MONITORING_EVENT_JUMP 7 -#define PY_MONITORING_EVENT_BRANCH 8 -#define PY_MONITORING_EVENT_STOP_ITERATION 9 - -#define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ - ((ev) < _PY_MONITORING_LOCAL_EVENTS) - -/* Other events, mainly exceptions */ - -#define PY_MONITORING_EVENT_RAISE 10 -#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11 -#define PY_MONITORING_EVENT_PY_UNWIND 12 -#define PY_MONITORING_EVENT_PY_THROW 13 -#define PY_MONITORING_EVENT_RERAISE 14 - - -/* Ancilliary events */ - -#define PY_MONITORING_EVENT_C_RETURN 15 -#define PY_MONITORING_EVENT_C_RAISE 16 - - typedef uint32_t _PyMonitoringEventSet; /* Tool IDs */ @@ -55,7 +23,7 @@ typedef uint32_t _PyMonitoringEventSet; #define PY_MONITORING_PROFILER_ID 2 #define PY_MONITORING_OPTIMIZER_ID 5 -/* Internal IDs used to suuport sys.setprofile() and sys.settrace() */ +/* Internal IDs used to support sys.setprofile() and sys.settrace() */ #define PY_MONITORING_SYS_PROFILE_ID 6 #define PY_MONITORING_SYS_TRACE_ID 7 @@ -80,8 +48,8 @@ _Py_call_instrumentation_instruction( _Py_CODEUNIT * _Py_call_instrumentation_jump( - PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target); + _Py_CODEUNIT *instr, PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *dest); extern int _Py_call_instrumentation_arg(PyThreadState *tstate, int event, diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 6a00aafea73779..6f00eca8de05af 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -14,25 +14,30 @@ extern "C" { #include "pycore_atexit.h" // struct atexit_state #include "pycore_ceval_state.h" // struct _ceval_state #include "pycore_code.h" // struct callable_cache +#include "pycore_codecs.h" // struct codecs_state #include "pycore_context.h" // struct _Py_context_state -#include "pycore_crossinterp.h" // struct _xidregistry +#include "pycore_crossinterp.h" // _PyXI_state_t #include "pycore_dict_state.h" // struct _Py_dict_state #include "pycore_dtoa.h" // struct _dtoa_state #include "pycore_exceptions.h" // struct _Py_exc_state #include "pycore_floatobject.h" // struct _Py_float_state #include "pycore_function.h" // FUNC_MAX_WATCHERS #include "pycore_gc.h" // struct _gc_runtime_state -#include "pycore_genobject.h" // struct _Py_async_gen_state +#include "pycore_genobject.h" // _PyGen_FetchStopIterationValue #include "pycore_global_objects.h"// struct _Py_interp_cached_objects #include "pycore_import.h" // struct _import_state +#include "pycore_index_pool.h" // _PyIndexPool #include "pycore_instruments.h" // _PY_MONITORING_EVENTS #include "pycore_list.h" // struct _Py_list_state #include "pycore_mimalloc.h" // struct _mimalloc_interp_state #include "pycore_object_state.h" // struct _py_object_state +#include "pycore_optimizer.h" // _PyExecutorObject #include "pycore_obmalloc.h" // struct _obmalloc_state #include "pycore_qsbr.h" // struct _qsbr_state +#include "pycore_stackref.h" // Py_STACKREF_DEBUG #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_tuple.h" // struct _Py_tuple_state +#include "pycore_uniqueid.h" // struct _Py_unique_id_pool #include "pycore_typeobject.h" // struct types_state #include "pycore_unicodeobject.h" // struct _Py_unicode_state #include "pycore_warnings.h" // struct _warnings_runtime_state @@ -59,6 +64,12 @@ struct _stoptheworld_state { PyThreadState *requester; // Thread that requested the pause (may be NULL). }; +#ifdef Py_GIL_DISABLED +// This should be prime but otherwise the choice is arbitrary. A larger value +// increases concurrency at the expense of memory. +# define NUM_WEAKREF_LIST_LOCKS 127 +#endif + /* cross-interpreter data registry */ /* Tracks some rare events per-interpreter, used by the optimizer to turn on/off @@ -85,7 +96,7 @@ typedef struct _rare_events { */ struct _is { - /* This struct countains the eval_breaker, + /* This struct contains the eval_breaker, * which is by far the hottest field in this struct * and should be placed at the beginning. */ struct _ceval_state ceval; @@ -93,15 +104,26 @@ struct _is { PyInterpreterState *next; int64_t id; - int64_t id_refcount; + Py_ssize_t id_refcount; int requires_idref; - PyThread_type_lock id_mutex; + +#define _PyInterpreterState_WHENCE_NOTSET -1 +#define _PyInterpreterState_WHENCE_UNKNOWN 0 +#define _PyInterpreterState_WHENCE_RUNTIME 1 +#define _PyInterpreterState_WHENCE_LEGACY_CAPI 2 +#define _PyInterpreterState_WHENCE_CAPI 3 +#define _PyInterpreterState_WHENCE_XI 4 +#define _PyInterpreterState_WHENCE_STDLIB 5 +#define _PyInterpreterState_WHENCE_MAX 5 + long _whence; /* Has been initialized to a safe state. In order to be effective, this must be set to 0 during or right after allocation. */ int _initialized; + /* Has been fully initialized via pylifecycle.c. */ + int _ready; int finalizing; uintptr_t last_restart_version; @@ -109,6 +131,7 @@ struct _is { uint64_t next_unique_id; /* The linked list of threads, newest first. */ PyThreadState *head; + _PyThreadStateImpl *preallocated; /* The thread currently executing in the __main__ module, if any. */ PyThreadState *main; /* Used in Modules/_threadmodule.c. */ @@ -164,10 +187,7 @@ struct _is { possible to facilitate out-of-process observability tools. */ - PyObject *codec_search_path; - PyObject *codec_search_cache; - PyObject *codec_error_registry; - int codecs_initialized; + struct codecs_state codecs; PyConfig config; unsigned long feature_flags; @@ -187,7 +207,7 @@ struct _is { freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS]; /* cross-interpreter data and utils */ - struct _xi_state xi; + _PyXI_state_t xi; #ifdef HAVE_FORK PyObject *before_forkers; @@ -203,6 +223,9 @@ struct _is { #if defined(Py_GIL_DISABLED) struct _mimalloc_interp_state mimalloc; struct _brc_state brc; // biased reference counting state + struct _Py_unique_id_pool unique_ids; // object ids for per-thread refcounts + PyMutex weakref_locks[NUM_WEAKREF_LIST_LOCKS]; + _PyIndexPool tlbc_indices; #endif // Per-interpreter state for the obmalloc allocator. For the main @@ -220,14 +243,17 @@ struct _is { PyObject *audit_hooks; PyType_WatchCallback type_watchers[TYPE_MAX_WATCHERS]; PyCode_WatchCallback code_watchers[CODE_MAX_WATCHERS]; + PyContext_WatchCallback context_watchers[CONTEXT_MAX_WATCHERS]; // One bit is set for each non-NULL entry in code_watchers uint8_t active_code_watchers; + uint8_t active_context_watchers; struct _py_object_state object_state; struct _Py_unicode_state unicode; struct _Py_long_state long_state; struct _dtoa_state dtoa; struct _py_func_state func_state; + struct _py_code_state code_state; struct _Py_dict_state dict_state; struct _Py_exc_state exc_state; @@ -236,16 +262,9 @@ struct _is { struct ast_state ast; struct types_state types; struct callable_cache callable_cache; - _PyOptimizerObject *optimizer; + bool jit; _PyExecutorObject *executor_list_head; - - /* These two values are shifted and offset to speed up check in JUMP_BACKWARD */ - uint32_t optimizer_resume_threshold; - uint32_t optimizer_backedge_threshold; - - uint16_t optimizer_side_threshold; - - uint32_t next_func_version; + size_t trace_run_counter; _rare_events rare_events; PyDict_WatchCallback builtins_dict_watcher; @@ -256,13 +275,22 @@ struct _is { Py_ssize_t sys_tracing_threads; /* Count of threads with c_tracefunc set */ PyObject *monitoring_callables[PY_MONITORING_TOOL_IDS][_PY_MONITORING_EVENTS]; PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; + uintptr_t monitoring_tool_versions[PY_MONITORING_TOOL_IDS]; struct _Py_interp_cached_objects cached_objects; struct _Py_interp_static_objects static_objects; + Py_ssize_t _interactive_src_count; + /* the initial PyInterpreterState.threads.head */ _PyThreadStateImpl _initial_thread; - Py_ssize_t _interactive_src_count; + // _initial_thread should be the last field of PyInterpreterState. + // See https://github.com/python/cpython/issues/127117. + +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) + uint64_t next_stackref; + _Py_hashtable_t *stackref_debug_table; +#endif }; @@ -296,51 +324,22 @@ _PyInterpreterState_SetFinalizing(PyInterpreterState *interp, PyThreadState *tst } -// Export for the _xxinterpchannels module. -PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(int64_t); -PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *); -PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *); +// Exports for the _testinternalcapi module. +PyAPI_FUNC(int64_t) _PyInterpreterState_ObjectToID(PyObject *); +PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(int64_t); +PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpIDObject(PyObject *); +PyAPI_FUNC(void) _PyInterpreterState_IDIncref(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *); -extern const PyConfig* _PyInterpreterState_GetConfig(PyInterpreterState *interp); +PyAPI_FUNC(int) _PyInterpreterState_IsReady(PyInterpreterState *interp); -// Get a copy of the current interpreter configuration. -// -// Return 0 on success. Raise an exception and return -1 on error. -// -// The caller must initialize 'config', using PyConfig_InitPythonConfig() -// for example. -// -// Python must be preinitialized to call this method. -// The caller must hold the GIL. -// -// Once done with the configuration, PyConfig_Clear() must be called to clear -// it. -// -// Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(int) _PyInterpreterState_GetConfigCopy( - struct PyConfig *config); - -// Set the configuration of the current interpreter. -// -// This function should be called during or just after the Python -// initialization. -// -// Update the sys module with the new configuration. If the sys module was -// modified directly after the Python initialization, these changes are lost. -// -// Some configuration like faulthandler or warnoptions can be updated in the -// configuration, but don't reconfigure Python (don't enable/disable -// faulthandler and don't reconfigure warnings filters). -// -// Return 0 on success. Raise an exception and return -1 on error. -// -// The configuration should come from _PyInterpreterState_GetConfigCopy(). -// -// Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(int) _PyInterpreterState_SetConfig( - const struct PyConfig *config); +PyAPI_FUNC(long) _PyInterpreterState_GetWhence(PyInterpreterState *interp); +extern void _PyInterpreterState_SetWhence( + PyInterpreterState *interp, + long whence); + +extern const PyConfig* _PyInterpreterState_GetConfig(PyInterpreterState *interp); /* @@ -381,7 +380,10 @@ PyAPI_FUNC(PyStatus) _PyInterpreterState_New( #define RARE_EVENT_INTERP_INC(interp, name) \ do { \ /* saturating add */ \ - if (interp->rare_events.name < UINT8_MAX) interp->rare_events.name++; \ + int val = FT_ATOMIC_LOAD_UINT8_RELAXED(interp->rare_events.name); \ + if (val < UINT8_MAX) { \ + FT_ATOMIC_STORE_UINT8(interp->rare_events.name, val + 1); \ + } \ RARE_EVENT_STAT_INC(name); \ } while (0); \ diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index 8fa88ea3f74caa..39c2a30f6e979d 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -28,8 +28,9 @@ #define INTRINSIC_TYPEVAR_WITH_BOUND 2 #define INTRINSIC_TYPEVAR_WITH_CONSTRAINTS 3 #define INTRINSIC_SET_FUNCTION_TYPE_PARAMS 4 +#define INTRINSIC_SET_TYPEPARAM_DEFAULT 5 -#define MAX_INTRINSIC_2 4 +#define MAX_INTRINSIC_2 5 typedef PyObject *(*intrinsic_func1)(PyThreadState* tstate, PyObject *value); typedef PyObject *(*intrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2); diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h index 17bd23f0752be2..4d6cc35a7a3de7 100644 --- a/Include/internal/pycore_jit.h +++ b/Include/internal/pycore_jit.h @@ -11,7 +11,7 @@ extern "C" { #ifdef _Py_JIT -typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate); +typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length); void _PyJIT_Free(_PyExecutorObject *executor); diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 2a82912e41d557..5d817891408481 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -8,10 +8,11 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState - PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *); extern void _PyList_DebugMallocStats(FILE *out); +// _PyList_GetItemRef should be used only when the object is known as a list +// because it doesn't raise TypeError when the object is not a list, whereas PyList_GetItemRef does. +extern PyObject* _PyList_GetItemRef(PyListObject *, Py_ssize_t i); #define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item) @@ -28,7 +29,11 @@ _PyList_AppendTakeRef(PyListObject *self, PyObject *newitem) Py_ssize_t allocated = self->allocated; assert((size_t)len + 1 < PY_SSIZE_T_MAX); if (allocated > len) { +#ifdef Py_GIL_DISABLED + _Py_atomic_store_ptr_release(&self->ob_item[len], newitem); +#else PyList_SET_ITEM(self, len, newitem); +#endif Py_SET_SIZE(self, len + 1); return 0; } @@ -43,7 +48,7 @@ _Py_memory_repeat(char* dest, Py_ssize_t len_dest, Py_ssize_t len_src) Py_ssize_t copied = len_src; while (copied < len_dest) { Py_ssize_t bytes_to_copy = Py_MIN(copied, len_dest - copied); - memcpy(dest + copied, dest, bytes_to_copy); + memcpy(dest + copied, dest, (size_t)bytes_to_copy); copied += bytes_to_copy; } } @@ -54,7 +59,10 @@ typedef struct { PyListObject *it_seq; /* Set to NULL when iterator is exhausted */ } _PyListIterObject; -PyAPI_FUNC(PyObject *)_PyList_FromArraySteal(PyObject *const *src, Py_ssize_t n); +union _PyStackRef; + +PyAPI_FUNC(PyObject *)_PyList_FromStackRefStealOnSuccess(const union _PyStackRef *src, Py_ssize_t n); +PyAPI_FUNC(PyObject *)_PyList_AsTupleAndClear(PyListObject *v); #ifdef __cplusplus } diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index f648be496ea4af..8bcb23a6ce9f9d 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -13,84 +13,23 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif - -// A mutex that occupies one byte. The lock can be zero initialized. -// -// Only the two least significant bits are used. The remaining bits should be -// zero: -// 0b00: unlocked -// 0b01: locked -// 0b10: unlocked and has parked threads -// 0b11: locked and has parked threads -// -// Typical initialization: -// PyMutex m = (PyMutex){0}; -// -// Or initialize as global variables: -// static PyMutex m; -// -// Typical usage: -// PyMutex_Lock(&m); -// ... -// PyMutex_Unlock(&m); - -// NOTE: In Py_GIL_DISABLED builds, `struct _PyMutex` is defined in Include/object.h. -// The Py_GIL_DISABLED builds need the definition in Include/object.h for the -// `ob_mutex` field in PyObject. For the default (non-free-threaded) build, -// we define the struct here to avoid exposing it in the public API. -#ifndef Py_GIL_DISABLED -struct _PyMutex { uint8_t v; }; -#endif - -typedef struct _PyMutex PyMutex; - -#define _Py_UNLOCKED 0 -#define _Py_LOCKED 1 +//_Py_UNLOCKED is defined as 0 and _Py_LOCKED as 1 in Include/cpython/lock.h #define _Py_HAS_PARKED 2 #define _Py_ONCE_INITIALIZED 4 -// (private) slow path for locking the mutex -PyAPI_FUNC(void) _PyMutex_LockSlow(PyMutex *m); - -// (private) slow path for unlocking the mutex -PyAPI_FUNC(void) _PyMutex_UnlockSlow(PyMutex *m); - static inline int -PyMutex_LockFast(uint8_t *lock_bits) +PyMutex_LockFast(PyMutex *m) { uint8_t expected = _Py_UNLOCKED; + uint8_t *lock_bits = &m->_bits; return _Py_atomic_compare_exchange_uint8(lock_bits, &expected, _Py_LOCKED); } -// Locks the mutex. -// -// If the mutex is currently locked, the calling thread will be parked until -// the mutex is unlocked. If the current thread holds the GIL, then the GIL -// will be released while the thread is parked. -static inline void -PyMutex_Lock(PyMutex *m) -{ - uint8_t expected = _Py_UNLOCKED; - if (!_Py_atomic_compare_exchange_uint8(&m->v, &expected, _Py_LOCKED)) { - _PyMutex_LockSlow(m); - } -} - -// Unlocks the mutex. -static inline void -PyMutex_Unlock(PyMutex *m) -{ - uint8_t expected = _Py_LOCKED; - if (!_Py_atomic_compare_exchange_uint8(&m->v, &expected, _Py_UNLOCKED)) { - _PyMutex_UnlockSlow(m); - } -} - // Checks if the mutex is currently locked. static inline int PyMutex_IsLocked(PyMutex *m) { - return (_Py_atomic_load_uint8(&m->v) & _Py_LOCKED) != 0; + return (_Py_atomic_load_uint8(&m->_bits) & _Py_LOCKED) != 0; } // Re-initializes the mutex after a fork to the unlocked state. @@ -116,18 +55,18 @@ typedef enum _PyLockFlags { extern PyLockStatus _PyMutex_LockTimed(PyMutex *m, PyTime_t timeout_ns, _PyLockFlags flags); -// Lock a mutex with aditional options. See _PyLockFlags for details. +// Lock a mutex with additional options. See _PyLockFlags for details. static inline void PyMutex_LockFlags(PyMutex *m, _PyLockFlags flags) { uint8_t expected = _Py_UNLOCKED; - if (!_Py_atomic_compare_exchange_uint8(&m->v, &expected, _Py_LOCKED)) { + if (!_Py_atomic_compare_exchange_uint8(&m->_bits, &expected, _Py_LOCKED)) { _PyMutex_LockTimed(m, -1, flags); } } -// Unlock a mutex, returns 0 if the mutex is not locked (used for improved -// error messages). +// Unlock a mutex, returns -1 if the mutex is not locked (used for improved +// error messages) otherwise returns 0. extern int _PyMutex_TryUnlock(PyMutex *m); @@ -150,18 +89,10 @@ PyAPI_FUNC(void) PyEvent_Wait(PyEvent *evt); // Wait for the event to be set, or until the timeout expires. If the event is // already set, then this returns immediately. Returns 1 if the event was set, -// and 0 if the timeout expired or thread was interrupted. -PyAPI_FUNC(int) PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns); - -// A one-time event notification with reference counting. -typedef struct _PyEventRc { - PyEvent event; - Py_ssize_t refcount; -} _PyEventRc; - -_PyEventRc *_PyEventRc_New(void); -void _PyEventRc_Incref(_PyEventRc *erc); -void _PyEventRc_Decref(_PyEventRc *erc); +// and 0 if the timeout expired or thread was interrupted. If `detach` is +// true, then the thread will detach/release the GIL while waiting. +PyAPI_FUNC(int) +PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns, int detach); // _PyRawMutex implements a word-sized mutex that that does not depend on the // parking lot API, and therefore can be used in the parking lot @@ -198,12 +129,6 @@ _PyRawMutex_Unlock(_PyRawMutex *m) _PyRawMutex_UnlockSlow(m); } -// A data structure that can be used to run initialization code once in a -// thread-safe manner. The C++11 equivalent is std::call_once. -typedef struct { - uint8_t v; -} _PyOnceFlag; - // Type signature for one-time initialization functions. The function should // return 0 on success and -1 on failure. typedef int _Py_once_fn_t(void *arg); @@ -227,6 +152,19 @@ _PyOnceFlag_CallOnce(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg) return _PyOnceFlag_CallOnceSlow(flag, fn, arg); } +// A recursive mutex. The mutex should zero-initialized. +typedef struct { + PyMutex mutex; + unsigned long long thread; // i.e., PyThread_get_thread_ident_ex() + size_t level; +} _PyRecursiveMutex; + +PyAPI_FUNC(int) _PyRecursiveMutex_IsLockedByCurrentThread(_PyRecursiveMutex *m); +PyAPI_FUNC(void) _PyRecursiveMutex_Lock(_PyRecursiveMutex *m); +extern PyLockStatus _PyRecursiveMutex_LockTimed(_PyRecursiveMutex *m, PyTime_t timeout, _PyLockFlags flags); +PyAPI_FUNC(void) _PyRecursiveMutex_Unlock(_PyRecursiveMutex *m); +extern int _PyRecursiveMutex_TryUnlock(_PyRecursiveMutex *m); + // A readers-writer (RW) lock. The lock supports multiple concurrent readers or // a single writer. The lock is write-preferring: if a writer is waiting while // the lock is read-locked then, new readers will be blocked. This avoids @@ -271,7 +209,7 @@ PyAPI_FUNC(void) _PyRWMutex_Unlock(_PyRWMutex *rwmutex); // underlying data and then read the sequence number again after reading the data. If the // sequence has not changed the data is valid. // -// Differs a little bit in that we use CAS on sequence as the lock, instead of a seperate spin lock. +// Differs a little bit in that we use CAS on sequence as the lock, instead of a separate spin lock. // The writer can also detect that the undelering data has not changed and abandon the write // and restore the previous sequence. typedef struct { @@ -284,7 +222,7 @@ PyAPI_FUNC(void) _PySeqLock_LockWrite(_PySeqLock *seqlock); // Unlock the sequence lock and move to the next sequence number. PyAPI_FUNC(void) _PySeqLock_UnlockWrite(_PySeqLock *seqlock); -// Abandon the current update indicating that no mutations have occured +// Abandon the current update indicating that no mutations have occurred // and restore the previous sequence value. PyAPI_FUNC(void) _PySeqLock_AbandonWrite(_PySeqLock *seqlock); @@ -292,12 +230,12 @@ PyAPI_FUNC(void) _PySeqLock_AbandonWrite(_PySeqLock *seqlock); PyAPI_FUNC(uint32_t) _PySeqLock_BeginRead(_PySeqLock *seqlock); // End the read operation and confirm that the sequence number has not changed. -// Returns 1 if the read was successful or 0 if the read should be re-tried. -PyAPI_FUNC(uint32_t) _PySeqLock_EndRead(_PySeqLock *seqlock, uint32_t previous); +// Returns 1 if the read was successful or 0 if the read should be retried. +PyAPI_FUNC(int) _PySeqLock_EndRead(_PySeqLock *seqlock, uint32_t previous); // Check if the lock was held during a fork and clear the lock. Returns 1 -// if the lock was held and any associated datat should be cleared. -PyAPI_FUNC(uint32_t) _PySeqLock_AfterFork(_PySeqLock *seqlock); +// if the lock was held and any associated data should be cleared. +PyAPI_FUNC(int) _PySeqLock_AfterFork(_PySeqLock *seqlock); #ifdef __cplusplus } diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index f04f66d053bab9..c52eb77692dd6a 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -47,17 +47,6 @@ extern "C" { # error "_PY_LONG_DEFAULT_MAX_STR_DIGITS smaller than threshold." #endif -// _PyLong_NumBits. Return the number of bits needed to represent the -// absolute value of a long. For example, this returns 1 for 1 and -1, 2 -// for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. -// v must not be NULL, and must be a normalized long. -// (size_t)-1 is returned and OverflowError set if the true result doesn't -// fit in a size_t. -// -// Export for 'math' shared extension. -PyAPI_FUNC(size_t) _PyLong_NumBits(PyObject *v); - - /* runtime lifecycle */ extern PyStatus _PyLong_InitTypes(PyInterpreterState *); @@ -66,6 +55,8 @@ extern void _PyLong_FiniTypes(PyInterpreterState *interp); /* other API */ +PyAPI_FUNC(void) _PyLong_ExactDealloc(PyObject *self); + #define _PyLong_SMALL_INTS _Py_SINGLETON(small_ints) // _PyLong_GetZero() and _PyLong_GetOne() must always be available @@ -90,14 +81,13 @@ static inline PyObject* _PyLong_FromUnsignedChar(unsigned char i) } // _PyLong_Frexp returns a double x and an exponent e such that the -// true value is approximately equal to x * 2**e. e is >= 0. x is +// true value is approximately equal to x * 2**e. x is // 0.0 if and only if the input is 0 (in which case, e and x are both -// zeroes); otherwise, 0.5 <= abs(x) < 1.0. On overflow, which is -// possible if the number of bits doesn't fit into a Py_ssize_t, sets -// OverflowError and returns -1.0 for x, 0 for e. +// zeroes); otherwise, 0.5 <= abs(x) < 1.0. +// Always successful. // // Export for 'math' shared extension -PyAPI_DATA(double) _PyLong_Frexp(PyLongObject *a, Py_ssize_t *e); +PyAPI_DATA(double) _PyLong_Frexp(PyLongObject *a, int64_t *e); extern PyObject* _PyLong_FromBytes(const char *, Py_ssize_t, int); @@ -116,10 +106,10 @@ PyAPI_DATA(PyObject*) _PyLong_DivmodNear(PyObject *, PyObject *); PyAPI_DATA(PyObject*) _PyLong_Format(PyObject *obj, int base); // Export for 'math' shared extension -PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, size_t); +PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, int64_t); // Export for 'math' shared extension -PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, size_t); +PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, int64_t); PyAPI_FUNC(PyObject*) _PyLong_Add(PyLongObject *left, PyLongObject *right); PyAPI_FUNC(PyObject*) _PyLong_Multiply(PyLongObject *left, PyLongObject *right); @@ -169,13 +159,14 @@ PyAPI_FUNC(int) _PyLong_Size_t_Converter(PyObject *, void *); /* Long value tag bits: * 0-1: Sign bits value = (1-sign), ie. negative=2, positive=0, zero=1. - * 2: Reserved for immortality bit + * 2: Set to 1 for the small ints * 3+ Unsigned digit count */ #define SIGN_MASK 3 #define SIGN_ZERO 1 #define SIGN_NEGATIVE 2 #define NON_SIZE_BITS 3 +#define IMMORTALITY_BIT_MASK (1 << 2) /* The functions _PyLong_IsCompact and _PyLong_CompactValue are defined * in Include/cpython/longobject.h, since they need to be inline. @@ -189,8 +180,12 @@ PyAPI_FUNC(int) _PyLong_Size_t_Converter(PyObject *, void *); * we define them to the numbers in both places and then assert that * they're the same. */ -static_assert(SIGN_MASK == _PyLong_SIGN_MASK, "SIGN_MASK does not match _PyLong_SIGN_MASK"); -static_assert(NON_SIZE_BITS == _PyLong_NON_SIZE_BITS, "NON_SIZE_BITS does not match _PyLong_NON_SIZE_BITS"); +#if SIGN_MASK != _PyLong_SIGN_MASK +# error "SIGN_MASK does not match _PyLong_SIGN_MASK" +#endif +#if NON_SIZE_BITS != _PyLong_NON_SIZE_BITS +# error "NON_SIZE_BITS does not match _PyLong_NON_SIZE_BITS" +#endif /* All *compact" values are guaranteed to fit into * a Py_ssize_t with at least one bit to spare. @@ -202,7 +197,7 @@ static_assert(NON_SIZE_BITS == _PyLong_NON_SIZE_BITS, "NON_SIZE_BITS does not ma static inline int _PyLong_IsNonNegativeCompact(const PyLongObject* op) { assert(PyLong_Check(op)); - return op->long_value.lv_tag <= (1 << NON_SIZE_BITS); + return ((op->long_value.lv_tag & ~IMMORTALITY_BIT_MASK) <= (1 << NON_SIZE_BITS)); } @@ -235,7 +230,7 @@ static inline Py_ssize_t _PyLong_DigitCount(const PyLongObject *op) { assert(PyLong_Check(op)); - return op->long_value.lv_tag >> NON_SIZE_BITS; + return (Py_ssize_t)(op->long_value.lv_tag >> NON_SIZE_BITS); } /* Equivalent to _PyLong_DigitCount(op) * _PyLong_NonCompactSign(op) */ @@ -251,7 +246,7 @@ static inline int _PyLong_CompactSign(const PyLongObject *op) { assert(PyLong_Check(op)); - assert(_PyLong_IsCompact(op)); + assert(_PyLong_IsCompact((PyLongObject *)op)); return 1 - (op->long_value.lv_tag & SIGN_MASK); } @@ -259,7 +254,7 @@ static inline int _PyLong_NonCompactSign(const PyLongObject *op) { assert(PyLong_Check(op)); - assert(!_PyLong_IsCompact(op)); + assert(!_PyLong_IsCompact((PyLongObject *)op)); return 1 - (op->long_value.lv_tag & SIGN_MASK); } @@ -270,7 +265,8 @@ _PyLong_SameSign(const PyLongObject *a, const PyLongObject *b) return (a->long_value.lv_tag & SIGN_MASK) == (b->long_value.lv_tag & SIGN_MASK); } -#define TAG_FROM_SIGN_AND_SIZE(sign, size) ((1 - (sign)) | ((size) << NON_SIZE_BITS)) +#define TAG_FROM_SIGN_AND_SIZE(sign, size) \ + ((uintptr_t)(1 - (sign)) | ((uintptr_t)(size) << NON_SIZE_BITS)) static inline void _PyLong_SetSignAndDigitCount(PyLongObject *op, int sign, Py_ssize_t size) @@ -278,7 +274,7 @@ _PyLong_SetSignAndDigitCount(PyLongObject *op, int sign, Py_ssize_t size) assert(size >= 0); assert(-1 <= sign && sign <= 1); assert(sign != 0 || size == 0); - op->long_value.lv_tag = TAG_FROM_SIGN_AND_SIZE(sign, (size_t)size); + op->long_value.lv_tag = TAG_FROM_SIGN_AND_SIZE(sign, size); } static inline void @@ -288,7 +284,7 @@ _PyLong_SetDigitCount(PyLongObject *op, Py_ssize_t size) op->long_value.lv_tag = (((size_t)size) << NON_SIZE_BITS) | (op->long_value.lv_tag & SIGN_MASK); } -#define NON_SIZE_MASK ~((1 << NON_SIZE_BITS) - 1) +#define NON_SIZE_MASK ~(uintptr_t)((1 << NON_SIZE_BITS) - 1) static inline void _PyLong_FlipSign(PyLongObject *op) { @@ -303,7 +299,7 @@ _PyLong_FlipSign(PyLongObject *op) { .long_value = { \ .lv_tag = TAG_FROM_SIGN_AND_SIZE( \ (val) == 0 ? 0 : ((val) < 0 ? -1 : 1), \ - (val) == 0 ? 0 : 1), \ + (val) == 0 ? 0 : 1) | IMMORTALITY_BIT_MASK, \ { ((val) >= 0 ? (val) : -(val)) }, \ } \ } diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h new file mode 100644 index 00000000000000..5a0b6dae8a5ad6 --- /dev/null +++ b/Include/internal/pycore_magic_number.h @@ -0,0 +1,293 @@ +#ifndef Py_INTERNAL_MAGIC_NUMBER_H +#define Py_INTERNAL_MAGIC_NUMBER_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +/* + +Magic number to reject .pyc files generated by other Python versions. +It should change for each incompatible change to the bytecode. + +PYC_MAGIC_NUMBER must change whenever the bytecode emitted by the compiler may +no longer be understood by older implementations of the eval loop (usually due +to the addition of new opcodes). + +The value of CR and LF is incorporated so if you ever read or write +a .pyc file in text mode the magic number will be wrong; also, the +Apple MPW compiler swaps their values, botching string constants. + +There were a variety of old schemes for setting the magic number. Starting with +Python 3.11, Python 3.n starts with magic number 2900+50n. Within each minor +version, the magic number is incremented by 1 each time the file format changes. + +Known values: + Python 1.5: 20121 + Python 1.5.1: 20121 + Python 1.5.2: 20121 + Python 1.6: 50428 + Python 2.0: 50823 + Python 2.0.1: 50823 + Python 2.1: 60202 + Python 2.1.1: 60202 + Python 2.1.2: 60202 + Python 2.2: 60717 + Python 2.3a0: 62011 + Python 2.3a0: 62021 + Python 2.3a0: 62011 (!) + Python 2.4a0: 62041 + Python 2.4a3: 62051 + Python 2.4b1: 62061 + Python 2.5a0: 62071 + Python 2.5a0: 62081 (ast-branch) + Python 2.5a0: 62091 (with) + Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) + Python 2.5b3: 62101 (fix wrong code: for x, in ...) + Python 2.5b3: 62111 (fix wrong code: x += yield) + Python 2.5c1: 62121 (fix wrong lnotab with for loops and + storing constants that should have been removed) + Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) + Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) + Python 2.6a1: 62161 (WITH_CLEANUP optimization) + Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND) + Python 2.7a0: 62181 (optimize conditional branches: + introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) + Python 2.7a0 62191 (introduce SETUP_WITH) + Python 2.7a0 62201 (introduce BUILD_SET) + Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD) + Python 3000: 3000 + 3010 (removed UNARY_CONVERT) + 3020 (added BUILD_SET) + 3030 (added keyword-only parameters) + 3040 (added signature annotations) + 3050 (print becomes a function) + 3060 (PEP 3115 metaclass syntax) + 3061 (string literals become unicode) + 3071 (PEP 3109 raise changes) + 3081 (PEP 3137 make __file__ and __name__ unicode) + 3091 (kill str8 interning) + 3101 (merge from 2.6a0, see 62151) + 3103 (__file__ points to source file) + Python 3.0a4: 3111 (WITH_CLEANUP optimization). + Python 3.0b1: 3131 (lexical exception stacking, including POP_EXCEPT + #3021) + Python 3.1a1: 3141 (optimize list, set and dict comprehensions: + change LIST_APPEND and SET_ADD, add MAP_ADD #2183) + Python 3.1a1: 3151 (optimize conditional branches: + introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE + #4715) + Python 3.2a1: 3160 (add SETUP_WITH #6101) + Python 3.2a2: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR #9225) + Python 3.2a3 3180 (add DELETE_DEREF #4617) + Python 3.3a1 3190 (__class__ super closure changed) + Python 3.3a1 3200 (PEP 3155 __qualname__ added #13448) + Python 3.3a1 3210 (added size modulo 2**32 to the pyc header #13645) + Python 3.3a2 3220 (changed PEP 380 implementation #14230) + Python 3.3a4 3230 (revert changes to implicit __class__ closure #14857) + Python 3.4a1 3250 (evaluate positional default arguments before + keyword-only defaults #16967) + Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override + free vars #17853) + Python 3.4a1 3270 (various tweaks to the __class__ closure #12370) + Python 3.4a1 3280 (remove implicit class argument) + Python 3.4a4 3290 (changes to __qualname__ computation #19301) + Python 3.4a4 3300 (more changes to __qualname__ computation #19301) + Python 3.4rc2 3310 (alter __qualname__ computation #20625) + Python 3.5a1 3320 (PEP 465: Matrix multiplication operator #21176) + Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations #2292) + Python 3.5b2 3340 (fix dictionary display evaluation order #11205) + Python 3.5b3 3350 (add GET_YIELD_FROM_ITER opcode #24400) + Python 3.5.2 3351 (fix BUILD_MAP_UNPACK_WITH_CALL opcode #27286) + Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483) + Python 3.6a1 3361 (lineno delta of code.co_lnotab becomes signed #26107) + Python 3.6a2 3370 (16 bit wordcode #26647) + Python 3.6a2 3371 (add BUILD_CONST_KEY_MAP opcode #27140) + Python 3.6a2 3372 (MAKE_FUNCTION simplification, remove MAKE_CLOSURE + #27095) + Python 3.6b1 3373 (add BUILD_STRING opcode #27078) + Python 3.6b1 3375 (add SETUP_ANNOTATIONS and STORE_ANNOTATION opcodes + #27985) + Python 3.6b1 3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL + #27213) + Python 3.6b1 3377 (set __class__ cell from type.__new__ #23722) + Python 3.6b2 3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257) + Python 3.6rc1 3379 (more thorough __class__ validation #23722) + Python 3.7a1 3390 (add LOAD_METHOD and CALL_METHOD opcodes #26110) + Python 3.7a2 3391 (update GET_AITER #31709) + Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650) + Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550) + Python 3.7b5 3394 (restored docstring as the first stmt in the body; + this might affected the first line number #32911) + Python 3.8a1 3400 (move frame block handling to compiler #17611) + Python 3.8a1 3401 (add END_ASYNC_FOR #33041) + Python 3.8a1 3410 (PEP570 Python Positional-Only Parameters #36540) + Python 3.8b2 3411 (Reverse evaluation order of key: value in dict + comprehensions #35224) + Python 3.8b2 3412 (Swap the position of positional args and positional + only args in ast.arguments #37593) + Python 3.8b4 3413 (Fix "break" and "continue" in "finally" #37830) + Python 3.9a0 3420 (add LOAD_ASSERTION_ERROR #34880) + Python 3.9a0 3421 (simplified bytecode for with blocks #32949) + Python 3.9a0 3422 (remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY, POP_FINALLY bytecodes #33387) + Python 3.9a2 3423 (add IS_OP, CONTAINS_OP and JUMP_IF_NOT_EXC_MATCH bytecodes #39156) + Python 3.9a2 3424 (simplify bytecodes for *value unpacking) + Python 3.9a2 3425 (simplify bytecodes for **value unpacking) + Python 3.10a1 3430 (Make 'annotations' future by default) + Python 3.10a1 3431 (New line number table format -- PEP 626) + Python 3.10a2 3432 (Function annotation for MAKE_FUNCTION is changed from dict to tuple bpo-42202) + Python 3.10a2 3433 (RERAISE restores f_lasti if oparg != 0) + Python 3.10a6 3434 (PEP 634: Structural Pattern Matching) + Python 3.10a7 3435 Use instruction offsets (as opposed to byte offsets). + Python 3.10b1 3436 (Add GEN_START bytecode #43683) + Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!) + Python 3.10b1 3438 Safer line number table handling. + Python 3.10b1 3439 (Add ROT_N) + Python 3.11a1 3450 Use exception table for unwinding ("zero cost" exception handling) + Python 3.11a1 3451 (Add CALL_METHOD_KW) + Python 3.11a1 3452 (drop nlocals from marshaled code objects) + Python 3.11a1 3453 (add co_fastlocalnames and co_fastlocalkinds) + Python 3.11a1 3454 (compute cell offsets relative to locals bpo-43693) + Python 3.11a1 3455 (add MAKE_CELL bpo-43693) + Python 3.11a1 3456 (interleave cell args bpo-43693) + Python 3.11a1 3457 (Change localsplus to a bytes object bpo-43693) + Python 3.11a1 3458 (imported objects now don't use LOAD_METHOD/CALL_METHOD) + Python 3.11a1 3459 (PEP 657: add end line numbers and column offsets for instructions) + Python 3.11a1 3460 (Add co_qualname field to PyCodeObject bpo-44530) + Python 3.11a1 3461 (JUMP_ABSOLUTE must jump backwards) + Python 3.11a2 3462 (bpo-44511: remove COPY_DICT_WITHOUT_KEYS, change + MATCH_CLASS and MATCH_KEYS, and add COPY) + Python 3.11a3 3463 (bpo-45711: JUMP_IF_NOT_EXC_MATCH no longer pops the + active exception) + Python 3.11a3 3464 (bpo-45636: Merge numeric BINARY_*INPLACE_* into + BINARY_OP) + Python 3.11a3 3465 (Add COPY_FREE_VARS opcode) + Python 3.11a4 3466 (bpo-45292: PEP-654 except*) + Python 3.11a4 3467 (Change CALL_xxx opcodes) + Python 3.11a4 3468 (Add SEND opcode) + Python 3.11a4 3469 (bpo-45711: remove type, traceback from exc_info) + Python 3.11a4 3470 (bpo-46221: PREP_RERAISE_STAR no longer pushes lasti) + Python 3.11a4 3471 (bpo-46202: remove pop POP_EXCEPT_AND_RERAISE) + Python 3.11a4 3472 (bpo-46009: replace GEN_START with POP_TOP) + Python 3.11a4 3473 (Add POP_JUMP_IF_NOT_NONE/POP_JUMP_IF_NONE opcodes) + Python 3.11a4 3474 (Add RESUME opcode) + Python 3.11a5 3475 (Add RETURN_GENERATOR opcode) + Python 3.11a5 3476 (Add ASYNC_GEN_WRAP opcode) + Python 3.11a5 3477 (Replace DUP_TOP/DUP_TOP_TWO with COPY and + ROT_TWO/ROT_THREE/ROT_FOUR/ROT_N with SWAP) + Python 3.11a5 3478 (New CALL opcodes) + Python 3.11a5 3479 (Add PUSH_NULL opcode) + Python 3.11a5 3480 (New CALL opcodes, second iteration) + Python 3.11a5 3481 (Use inline cache for BINARY_OP) + Python 3.11a5 3482 (Use inline caching for UNPACK_SEQUENCE and LOAD_GLOBAL) + Python 3.11a5 3483 (Use inline caching for COMPARE_OP and BINARY_SUBSCR) + Python 3.11a5 3484 (Use inline caching for LOAD_ATTR, LOAD_METHOD, and + STORE_ATTR) + Python 3.11a5 3485 (Add an oparg to GET_AWAITABLE) + Python 3.11a6 3486 (Use inline caching for PRECALL and CALL) + Python 3.11a6 3487 (Remove the adaptive "oparg counter" mechanism) + Python 3.11a6 3488 (LOAD_GLOBAL can push additional NULL) + Python 3.11a6 3489 (Add JUMP_BACKWARD, remove JUMP_ABSOLUTE) + Python 3.11a6 3490 (remove JUMP_IF_NOT_EXC_MATCH, add CHECK_EXC_MATCH) + Python 3.11a6 3491 (remove JUMP_IF_NOT_EG_MATCH, add CHECK_EG_MATCH, + add JUMP_BACKWARD_NO_INTERRUPT, make JUMP_NO_INTERRUPT virtual) + Python 3.11a7 3492 (make POP_JUMP_IF_NONE/NOT_NONE/TRUE/FALSE relative) + Python 3.11a7 3493 (Make JUMP_IF_TRUE_OR_POP/JUMP_IF_FALSE_OR_POP relative) + Python 3.11a7 3494 (New location info table) + Python 3.11b4 3495 (Set line number of module's RESUME instr to 0 per PEP 626) + Python 3.12a1 3500 (Remove PRECALL opcode) + Python 3.12a1 3501 (YIELD_VALUE oparg == stack_depth) + Python 3.12a1 3502 (LOAD_FAST_CHECK, no NULL-check in LOAD_FAST) + Python 3.12a1 3503 (Shrink LOAD_METHOD cache) + Python 3.12a1 3504 (Merge LOAD_METHOD back into LOAD_ATTR) + Python 3.12a1 3505 (Specialization/Cache for FOR_ITER) + Python 3.12a1 3506 (Add BINARY_SLICE and STORE_SLICE instructions) + Python 3.12a1 3507 (Set lineno of module's RESUME to 0) + Python 3.12a1 3508 (Add CLEANUP_THROW) + Python 3.12a1 3509 (Conditional jumps only jump forward) + Python 3.12a2 3510 (FOR_ITER leaves iterator on the stack) + Python 3.12a2 3511 (Add STOPITERATION_ERROR instruction) + Python 3.12a2 3512 (Remove all unused consts from code objects) + Python 3.12a4 3513 (Add CALL_INTRINSIC_1 instruction, removed STOPITERATION_ERROR, PRINT_EXPR, IMPORT_STAR) + Python 3.12a4 3514 (Remove ASYNC_GEN_WRAP, LIST_TO_TUPLE, and UNARY_POSITIVE) + Python 3.12a5 3515 (Embed jump mask in COMPARE_OP oparg) + Python 3.12a5 3516 (Add COMPARE_AND_BRANCH instruction) + Python 3.12a5 3517 (Change YIELD_VALUE oparg to exception block depth) + Python 3.12a6 3518 (Add RETURN_CONST instruction) + Python 3.12a6 3519 (Modify SEND instruction) + Python 3.12a6 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2) + Python 3.12a7 3521 (Shrink the LOAD_GLOBAL caches) + Python 3.12a7 3522 (Removed JUMP_IF_FALSE_OR_POP/JUMP_IF_TRUE_OR_POP) + Python 3.12a7 3523 (Convert COMPARE_AND_BRANCH back to COMPARE_OP) + Python 3.12a7 3524 (Shrink the BINARY_SUBSCR caches) + Python 3.12b1 3525 (Shrink the CALL caches) + Python 3.12b1 3526 (Add instrumentation support) + Python 3.12b1 3527 (Add LOAD_SUPER_ATTR) + Python 3.12b1 3528 (Add LOAD_SUPER_ATTR_METHOD specialization) + Python 3.12b1 3529 (Inline list/dict/set comprehensions) + Python 3.12b1 3530 (Shrink the LOAD_SUPER_ATTR caches) + Python 3.12b1 3531 (Add PEP 695 changes) + Python 3.13a1 3550 (Plugin optimizer support) + Python 3.13a1 3551 (Compact superinstructions) + Python 3.13a1 3552 (Remove LOAD_FAST__LOAD_CONST and LOAD_CONST__LOAD_FAST) + Python 3.13a1 3553 (Add SET_FUNCTION_ATTRIBUTE) + Python 3.13a1 3554 (more efficient bytecodes for f-strings) + Python 3.13a1 3555 (generate specialized opcodes metadata from bytecodes.c) + Python 3.13a1 3556 (Convert LOAD_CLOSURE to a pseudo-op) + Python 3.13a1 3557 (Make the conversion to boolean in jumps explicit) + Python 3.13a1 3558 (Reorder the stack items for CALL) + Python 3.13a1 3559 (Generate opcode IDs from bytecodes.c) + Python 3.13a1 3560 (Add RESUME_CHECK instruction) + Python 3.13a1 3561 (Add cache entry to branch instructions) + Python 3.13a1 3562 (Assign opcode IDs for internal ops in separate range) + Python 3.13a1 3563 (Add CALL_KW and remove KW_NAMES) + Python 3.13a1 3564 (Removed oparg from YIELD_VALUE, changed oparg values of RESUME) + Python 3.13a1 3565 (Oparg of YIELD_VALUE indicates whether it is in a yield-from) + Python 3.13a1 3566 (Emit JUMP_NO_INTERRUPT instead of JUMP for non-loop no-lineno cases) + Python 3.13a1 3567 (Reimplement line number propagation by the compiler) + Python 3.13a1 3568 (Change semantics of END_FOR) + Python 3.13a5 3569 (Specialize CONTAINS_OP) + Python 3.13a6 3570 (Add __firstlineno__ class attribute) + Python 3.13b1 3571 (Fix miscompilation of private names in generic classes) + Python 3.14a1 3600 (Add LOAD_COMMON_CONSTANT) + Python 3.14a1 3601 (Fix miscompilation of private names in generic classes) + Python 3.14a1 3602 (Add LOAD_SPECIAL. Remove BEFORE_WITH and BEFORE_ASYNC_WITH) + Python 3.14a1 3603 (Remove BUILD_CONST_KEY_MAP) + Python 3.14a1 3604 (Do not duplicate test at end of while statements) + Python 3.14a1 3605 (Move ENTER_EXECUTOR to opcode 255) + Python 3.14a1 3606 (Specialize CALL_KW) + Python 3.14a1 3607 (Add pseudo instructions JUMP_IF_TRUE/FALSE) + Python 3.14a1 3608 (Add support for slices) + Python 3.14a2 3609 (Add LOAD_SMALL_INT and LOAD_CONST_IMMORTAL instructions, remove RETURN_CONST) + Python 3.14a4 3610 (Add VALUE_WITH_FAKE_GLOBALS format to annotationlib) + Python 3.14a4 3611 (Add NOT_TAKEN instruction) + Python 3.14a4 3612 (Add POP_ITER and INSTRUMENTED_POP_ITER) + Python 3.14a4 3613 (Add LOAD_CONST_MORTAL instruction) + Python 3.14a5 3614 (Add BINARY_OP_EXTEND) + Python 3.14a5 3615 (CALL_FUNCTION_EX always take a kwargs argument) + + Python 3.15 will start with 3650 + + Please don't copy-paste the same pre-release tag for new entries above!!! + You should always use the *upcoming* tag. For example, if 3.12a6 came out + a week ago, I should put "Python 3.12a7" next to my new magic number. + +Whenever PYC_MAGIC_NUMBER is changed, the ranges in the magic_values array in +PC/launcher.c must also be updated. + +*/ + +#define PYC_MAGIC_NUMBER 3615 +/* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes + (little-endian) and then appending b'\r\n'. */ +#define PYC_MAGIC_NUMBER_TOKEN \ + ((uint32_t)PYC_MAGIC_NUMBER | ((uint32_t)'\r' << 16) | ((uint32_t)'\n' << 24)) + + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_MAGIC_NUMBER_H diff --git a/Include/internal/pycore_mimalloc.h b/Include/internal/pycore_mimalloc.h index 14c98108ec131e..d870d01beb702c 100644 --- a/Include/internal/pycore_mimalloc.h +++ b/Include/internal/pycore_mimalloc.h @@ -27,14 +27,27 @@ typedef enum { # define MI_DEBUG_FREED PYMEM_DEADBYTE # define MI_DEBUG_PADDING PYMEM_FORBIDDENBYTE #ifdef Py_DEBUG -# define MI_DEBUG 1 +# define MI_DEBUG 2 #else # define MI_DEBUG 0 #endif -#include "mimalloc.h" -#include "mimalloc/types.h" -#include "mimalloc/internal.h" +#ifdef _Py_THREAD_SANITIZER +# define MI_TSAN 1 +#endif + +#ifdef __cplusplus +extern "C++" { +#endif + +#include "mimalloc/mimalloc.h" +#include "mimalloc/mimalloc/types.h" +#include "mimalloc/mimalloc/internal.h" + +#ifdef __cplusplus +} +#endif + #endif #ifdef Py_GIL_DISABLED @@ -48,6 +61,8 @@ struct _mimalloc_thread_state { mi_heap_t *current_object_heap; mi_heap_t heaps[_Py_MIMALLOC_HEAP_COUNT]; mi_tld_t tld; + int initialized; + struct llist_node page_list; }; #endif diff --git a/Include/internal/pycore_modsupport.h b/Include/internal/pycore_modsupport.h index 3d3cd6722528e9..614e9f93751834 100644 --- a/Include/internal/pycore_modsupport.h +++ b/Include/internal/pycore_modsupport.h @@ -1,8 +1,6 @@ #ifndef Py_INTERNAL_MODSUPPORT_H #define Py_INTERNAL_MODSUPPORT_H -#include "pycore_lock.h" // _PyOnceFlag - #ifdef __cplusplus extern "C" { #endif @@ -67,24 +65,6 @@ PyAPI_FUNC(void) _PyArg_BadArgument( // --- _PyArg_Parser API --------------------------------------------------- -typedef struct _PyArg_Parser { - const char *format; - const char * const *keywords; - const char *fname; - const char *custom_msg; - _PyOnceFlag once; /* atomic one-time initialization flag */ - int is_kwtuple_owned; /* does this parser own the kwtuple object? */ - int pos; /* number of positional-only arguments */ - int min; /* minimal number of arguments */ - int max; /* maximal number of positional arguments */ - PyObject *kwtuple; /* tuple of keyword parameter names */ - struct _PyArg_Parser *next; -} _PyArg_Parser; - -// Export for '_testclinic' shared extension -PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywordsFast(PyObject *, PyObject *, - struct _PyArg_Parser *, ...); - // Export for '_dbm' shared extension PyAPI_FUNC(int) _PyArg_ParseStackAndKeywords( PyObject *const *args, @@ -103,20 +83,14 @@ PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywords( int minpos, int maxpos, int minkw, + int varpos, PyObject **buf); -#define _PyArg_UnpackKeywords(args, nargs, kwargs, kwnames, parser, minpos, maxpos, minkw, buf) \ +#define _PyArg_UnpackKeywords(args, nargs, kwargs, kwnames, parser, minpos, maxpos, minkw, varpos, buf) \ (((minkw) == 0 && (kwargs) == NULL && (kwnames) == NULL && \ - (minpos) <= (nargs) && (nargs) <= (maxpos) && (args) != NULL) ? (args) : \ + (minpos) <= (nargs) && ((varpos) || (nargs) <= (maxpos)) && (args) != NULL) ? \ + (args) : \ _PyArg_UnpackKeywords((args), (nargs), (kwargs), (kwnames), (parser), \ - (minpos), (maxpos), (minkw), (buf))) - -// Export for '_testclinic' shared extension -PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywordsWithVararg( - PyObject *const *args, Py_ssize_t nargs, - PyObject *kwargs, PyObject *kwnames, - struct _PyArg_Parser *parser, - int minpos, int maxpos, int minkw, - int vararg, PyObject **buf); + (minpos), (maxpos), (minkw), (varpos), (buf))) #ifdef __cplusplus } diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index 5644bbe5e0552b..9bb282a13a9659 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -11,6 +11,8 @@ extern "C" { extern void _PyModule_Clear(PyObject *); extern void _PyModule_ClearDict(PyObject *); extern int _PyModuleSpec_IsInitializing(PyObject *); +extern int _PyModuleSpec_GetFileOrigin(PyObject *, PyObject **); +extern int _PyModule_IsPossiblyShadowing(PyObject *); extern int _PyModule_IsExtension(PyObject *obj); @@ -22,6 +24,9 @@ typedef struct { PyObject *md_weaklist; // for logging purposes after md_dict is cleared PyObject *md_name; +#ifdef Py_GIL_DISABLED + void *md_gil; +#endif } PyModuleObject; static inline PyModuleDef* _PyModule_GetDef(PyObject *mod) { @@ -43,7 +48,7 @@ static inline PyObject* _PyModule_GetDict(PyObject *mod) { } PyObject* _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress); -PyObject* _Py_module_getattro(PyModuleObject *m, PyObject *name); +PyObject* _Py_module_getattro(PyObject *m, PyObject *name); #ifdef __cplusplus } diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 9809f5f2e0271a..0b1df7e68b8dfa 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -12,7 +12,22 @@ extern "C" { #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_emscripten_trampoline.h" // _PyCFunction_TrampolineCall() #include "pycore_interp.h" // PyInterpreterState.gc +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_PTR_RELAXED #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_stackref.h" +#include "pycore_uniqueid.h" // _PyObject_ThreadIncrefSlow() + +// This value is added to `ob_ref_shared` for objects that use deferred +// reference counting so that they are not immediately deallocated when the +// non-deferred reference count drops to zero. +// +// The value is half the maximum shared refcount because the low two bits of +// `ob_ref_shared` are used for flags. +#define _Py_REF_DEFERRED (PY_SSIZE_T_MAX / 8) + +/* For backwards compatibility -- Do not use this */ +#define _Py_IsImmortalLoose(op) _Py_IsImmortal + /* Check if an object is consistent. For example, ensure that the reference counter is greater than or equal to 1, and ensure that ob_type is not NULL. @@ -47,7 +62,7 @@ extern void _Py_ForgetReference(PyObject *); PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); /* We need to maintain an internal copy of Py{Var}Object_HEAD_INIT to avoid - designated initializer conflicts in C++20. If we use the deinition in + designated initializer conflicts in C++20. If we use the definition in object.h, we will be mixing designated and non-designated initializers in pycore objects which is forbiddent in C++20. However, if we then use designated initializers in object.h then Extensions without designated break. @@ -58,15 +73,25 @@ PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); #define _PyObject_HEAD_INIT(type) \ { \ .ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL, \ + .ob_flags = _Py_STATICALLY_ALLOCATED_FLAG, \ .ob_type = (type) \ } #else +#if SIZEOF_VOID_P > 4 +#define _PyObject_HEAD_INIT(type) \ + { \ + .ob_refcnt = _Py_IMMORTAL_INITIAL_REFCNT, \ + .ob_flags = _Py_STATICALLY_ALLOCATED_FLAG, \ + .ob_type = (type) \ + } +#else #define _PyObject_HEAD_INIT(type) \ { \ - .ob_refcnt = _Py_IMMORTAL_REFCNT, \ + .ob_refcnt = _Py_STATIC_IMMORTAL_INITIAL_REFCNT, \ .ob_type = (type) \ } #endif +#endif #define _PyVarObject_HEAD_INIT(type, size) \ { \ .ob_base = _PyObject_HEAD_INIT(type), \ @@ -80,15 +105,23 @@ PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( #define _Py_FatalRefcountError(message) \ _Py_FatalRefcountErrorFunc(__func__, (message)) +#define _PyReftracerTrack(obj, operation) \ + do { \ + struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; \ + if (tracer->tracer_func != NULL) { \ + void *data = tracer->tracer_data; \ + tracer->tracer_func((obj), (operation), data); \ + } \ + } while(0) #ifdef Py_REF_DEBUG /* The symbol is only exposed in the API for the sake of extensions built against the pre-3.12 stable ABI. */ PyAPI_DATA(Py_ssize_t) _Py_RefTotal; -extern void _Py_AddRefTotal(PyInterpreterState *, Py_ssize_t); -extern void _Py_IncRefTotal(PyInterpreterState *); -extern void _Py_DecRefTotal(PyInterpreterState *); +extern void _Py_AddRefTotal(PyThreadState *, Py_ssize_t); +extern PyAPI_FUNC(void) _Py_IncRefTotal(PyThreadState *); +extern void _Py_DecRefTotal(PyThreadState *); # define _Py_DEC_REFTOTAL(interp) \ interp->object_state.reftotal-- @@ -98,13 +131,18 @@ extern void _Py_DecRefTotal(PyInterpreterState *); static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) { if (_Py_IsImmortal(op)) { + _Py_INCREF_IMMORTAL_STAT_INC(); return; } #ifdef Py_REF_DEBUG - _Py_AddRefTotal(_PyInterpreterState_GET(), n); + _Py_AddRefTotal(_PyThreadState_GET(), n); #endif #if !defined(Py_GIL_DISABLED) +#if SIZEOF_VOID_P > 4 + op->ob_refcnt += (PY_UINT32_T)n; +#else op->ob_refcnt += n; +#endif #else if (_Py_IsOwnedByCurrentThread(op)) { uint32_t local = op->ob_ref_local; @@ -122,26 +160,36 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) _Py_atomic_add_ssize(&op->ob_ref_shared, (n << _Py_REF_SHARED_SHIFT)); } #endif + // Although the ref count was increased by `n` (which may be greater than 1) + // it is only a single increment (i.e. addition) operation, so only 1 refcnt + // increment operation is counted. + _Py_INCREF_STAT_INC(); } #define _Py_RefcntAdd(op, n) _Py_RefcntAdd(_PyObject_CAST(op), n) -static inline void _Py_SetImmortal(PyObject *op) +// Checks if an object has a single, unique reference. If the caller holds a +// unique reference, it may be able to safely modify the object in-place. +static inline int +_PyObject_IsUniquelyReferenced(PyObject *ob) { - if (op) { -#ifdef Py_GIL_DISABLED - op->ob_tid = _Py_UNOWNED_TID; - op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; - op->ob_ref_shared = 0; +#if !defined(Py_GIL_DISABLED) + return Py_REFCNT(ob) == 1; #else - op->ob_refcnt = _Py_IMMORTAL_REFCNT; + // NOTE: the entire ob_ref_shared field must be zero, including flags, to + // ensure that other threads cannot concurrently create new references to + // this object. + return (_Py_IsOwnedByCurrentThread(ob) && + _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local) == 1 && + _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared) == 0); #endif - } } -#define _Py_SetImmortal(op) _Py_SetImmortal(_PyObject_CAST(op)) + +PyAPI_FUNC(void) _Py_SetImmortal(PyObject *op); +PyAPI_FUNC(void) _Py_SetImmortalUntracked(PyObject *op); // Makes an immortal object mortal again with the specified refcnt. Should only // be used during runtime finalization. -static inline void _Py_SetMortal(PyObject *op, Py_ssize_t refcnt) +static inline void _Py_SetMortal(PyObject *op, short refcnt) { if (op) { assert(_Py_IsImmortal(op)); @@ -174,6 +222,7 @@ static inline void _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) { if (_Py_IsImmortal(op)) { + _Py_DECREF_IMMORTAL_STAT_INC(); return; } _Py_DECREF_STAT_INC(); @@ -187,6 +236,7 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) #ifdef Py_TRACE_REFS _Py_ForgetReference(op); #endif + _PyReftracerTrack(op, PyRefTracer_DESTROY); destruct(op); } } @@ -195,6 +245,7 @@ static inline void _Py_DECREF_NO_DEALLOC(PyObject *op) { if (_Py_IsImmortal(op)) { + _Py_DECREF_IMMORTAL_STAT_INC(); return; } _Py_DECREF_STAT_INC(); @@ -248,16 +299,10 @@ Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra); extern int _PyType_CheckConsistency(PyTypeObject *type); extern int _PyDict_CheckConsistency(PyObject *mp, int check_content); -/* Update the Python traceback of an object. This function must be called - when a memory block is reused from a free list. - - Internal function called by _Py_NewReference(). */ -extern int _PyTraceMalloc_NewReference(PyObject *op); - // Fast inlined version of PyType_HasFeature() static inline int _PyType_HasFeature(PyTypeObject *type, unsigned long feature) { - return ((type->tp_flags & feature) != 0); + return ((FT_ATOMIC_LOAD_ULONG_RELAXED(type->tp_flags) & feature) != 0); } extern void _PyType_InitCache(PyInterpreterState *interp); @@ -266,6 +311,117 @@ extern PyStatus _PyObject_InitState(PyInterpreterState *interp); extern void _PyObject_FiniState(PyInterpreterState *interp); extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj); +// Macros used for per-thread reference counting in the free threading build. +// They resolve to normal Py_INCREF/DECREF calls in the default build. +// +// The macros are used for only a few references that would otherwise cause +// scaling bottlenecks in the free threading build: +// - The reference from an object to `ob_type`. +// - The reference from a function to `func_code`. +// - The reference from a function to `func_globals` and `func_builtins`. +// +// It's safe, but not performant or necessary, to use these macros for other +// references to code, type, or dict objects. It's also safe to mix their +// usage with normal Py_INCREF/DECREF calls. +// +// See also Include/internal/pycore_dict.h for _Py_INCREF_DICT/_Py_DECREF_DICT. +#ifndef Py_GIL_DISABLED +# define _Py_INCREF_TYPE Py_INCREF +# define _Py_DECREF_TYPE Py_DECREF +# define _Py_INCREF_CODE Py_INCREF +# define _Py_DECREF_CODE Py_DECREF +#else +static inline void +_Py_THREAD_INCREF_OBJECT(PyObject *obj, Py_ssize_t unique_id) +{ + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + + // The table index is `unique_id - 1` because 0 is not a valid unique id. + // Unsigned comparison so that `idx=-1` is handled by the "else". + size_t idx = (size_t)(unique_id - 1); + if (idx < (size_t)tstate->refcounts.size) { +# ifdef Py_REF_DEBUG + _Py_INCREF_IncRefTotal(); +# endif + _Py_INCREF_STAT_INC(); + tstate->refcounts.values[idx]++; + } + else { + // The slow path resizes the per-thread refcount array if necessary. + // It handles the unique_id=0 case to keep the inlinable function smaller. + _PyObject_ThreadIncrefSlow(obj, idx); + } +} + +static inline void +_Py_INCREF_TYPE(PyTypeObject *type) +{ + if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { + assert(_Py_IsImmortal(type)); + _Py_INCREF_IMMORTAL_STAT_INC(); + return; + } + + // gh-122974: GCC 11 warns about the access to PyHeapTypeObject fields when + // _Py_INCREF_TYPE() is called on a statically allocated type. The + // _PyType_HasFeature check above ensures that the type is a heap type. +#if defined(__GNUC__) && __GNUC__ >= 11 +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Warray-bounds" +#endif + _Py_THREAD_INCREF_OBJECT((PyObject *)type, ((PyHeapTypeObject *)type)->unique_id); +#if defined(__GNUC__) && __GNUC__ >= 11 +# pragma GCC diagnostic pop +#endif +} + +static inline void +_Py_INCREF_CODE(PyCodeObject *co) +{ + _Py_THREAD_INCREF_OBJECT((PyObject *)co, co->_co_unique_id); +} + +static inline void +_Py_THREAD_DECREF_OBJECT(PyObject *obj, Py_ssize_t unique_id) +{ + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + + // The table index is `unique_id - 1` because 0 is not a valid unique id. + // Unsigned comparison so that `idx=-1` is handled by the "else". + size_t idx = (size_t)(unique_id - 1); + if (idx < (size_t)tstate->refcounts.size) { +# ifdef Py_REF_DEBUG + _Py_DECREF_DecRefTotal(); +# endif + _Py_DECREF_STAT_INC(); + tstate->refcounts.values[idx]--; + } + else { + // Directly decref the object if the id is not assigned or if + // per-thread refcounting has been disabled on this object. + Py_DECREF(obj); + } +} + +static inline void +_Py_DECREF_TYPE(PyTypeObject *type) +{ + if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { + assert(_Py_IsImmortal(type)); + _Py_DECREF_IMMORTAL_STAT_INC(); + return; + } + PyHeapTypeObject *ht = (PyHeapTypeObject *)type; + _Py_THREAD_DECREF_OBJECT((PyObject *)type, ht->unique_id); +} + +static inline void +_Py_DECREF_CODE(PyCodeObject *co) +{ + _Py_THREAD_DECREF_OBJECT((PyObject *)co, co->_co_unique_id); +} +#endif + /* Inline functions trading binary compatibility for speed: _PyObject_Init() is the fast version of PyObject_Init(), and _PyObject_InitVar() is the fast version of PyObject_InitVar(). @@ -276,9 +432,8 @@ _PyObject_Init(PyObject *op, PyTypeObject *typeobj) { assert(op != NULL); Py_SET_TYPE(op, typeobj); - if (_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE)) { - Py_INCREF(typeobj); - } + assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortal(typeobj)); + _Py_INCREF_TYPE(typeobj); _Py_NewReference(op); } @@ -316,7 +471,7 @@ static inline void _PyObject_GC_TRACK( "object already tracked by the garbage collector", filename, lineno, __func__); #ifdef Py_GIL_DISABLED - op->ob_gc_bits |= _PyGC_BITS_TRACKED; + _PyObject_SET_GC_BITS(op, _PyGC_BITS_TRACKED); #else PyGC_Head *gc = _Py_AS_GC(op); _PyObject_ASSERT_FROM(op, @@ -325,11 +480,12 @@ static inline void _PyObject_GC_TRACK( filename, lineno, __func__); PyInterpreterState *interp = _PyInterpreterState_GET(); - PyGC_Head *generation0 = interp->gc.generation0; + PyGC_Head *generation0 = &interp->gc.young.head; PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); _PyGCHead_SET_NEXT(last, gc); _PyGCHead_SET_PREV(gc, last); - _PyGCHead_SET_NEXT(gc, generation0); + uintptr_t not_visited = 1 ^ interp->gc.visited_space; + gc->_gc_next = ((uintptr_t)generation0) | not_visited; generation0->_gc_prev = (uintptr_t)gc; #endif } @@ -356,7 +512,7 @@ static inline void _PyObject_GC_UNTRACK( filename, lineno, __func__); #ifdef Py_GIL_DISABLED - op->ob_gc_bits &= ~_PyGC_BITS_TRACKED; + _PyObject_CLEAR_GC_BITS(op, _PyGC_BITS_TRACKED); #else PyGC_Head *gc = _Py_AS_GC(op); PyGC_Head *prev = _PyGCHead_PREV(gc); @@ -398,13 +554,14 @@ _Py_TryIncrefFast(PyObject *op) { local += 1; if (local == 0) { // immortal + _Py_INCREF_IMMORTAL_STAT_INC(); return 1; } if (_Py_IsOwnedByCurrentThread(op)) { _Py_INCREF_STAT_INC(); _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif return 1; } @@ -427,7 +584,7 @@ _Py_TryIncRefShared(PyObject *op) &shared, shared + (1 << _Py_REF_SHARED_SHIFT))) { #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif _Py_INCREF_STAT_INC(); return 1; @@ -437,7 +594,7 @@ _Py_TryIncRefShared(PyObject *op) /* Tries to incref the object op and ensures that *src still points to it. */ static inline int -_Py_TryIncref(PyObject **src, PyObject *op) +_Py_TryIncrefCompare(PyObject **src, PyObject *op) { if (_Py_TryIncrefFast(op)) { return 1; @@ -452,6 +609,20 @@ _Py_TryIncref(PyObject **src, PyObject *op) return 1; } +static inline int +_Py_TryIncrefCompareStackRef(PyObject **src, PyObject *op, _PyStackRef *out) +{ + if (_Py_IsImmortal(op) || _PyObject_HasDeferredRefcount(op)) { + *out = (_PyStackRef){ .bits = (intptr_t)op | Py_TAG_DEFERRED }; + return 1; + } + if (_Py_TryIncrefCompare(src, op)) { + *out = PyStackRef_FromPyObjectSteal(op); + return 1; + } + return 0; +} + /* Loads and increfs an object from ptr, which may contain a NULL value. Safe with concurrent (atomic) updates to ptr. NOTE: The writer must set maybe-weakref on the stored object! */ @@ -463,7 +634,7 @@ _Py_XGetRef(PyObject **ptr) if (value == NULL) { return value; } - if (_Py_TryIncref(ptr, value)) { + if (_Py_TryIncrefCompare(ptr, value)) { return value; } } @@ -478,7 +649,7 @@ _Py_TryXGetRef(PyObject **ptr) if (value == NULL) { return value; } - if (_Py_TryIncref(ptr, value)) { + if (_Py_TryIncrefCompare(ptr, value)) { return value; } return NULL; @@ -492,6 +663,9 @@ _Py_NewRefWithLock(PyObject *op) if (_Py_TryIncrefFast(op)) { return op; } +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyThreadState_GET()); +#endif _Py_INCREF_STAT_INC(); for (;;) { Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); @@ -517,8 +691,86 @@ _Py_XNewRefWithLock(PyObject *obj) return _Py_NewRefWithLock(obj); } +static inline void +_PyObject_SetMaybeWeakref(PyObject *op) +{ + if (_Py_IsImmortal(op)) { + return; + } + for (;;) { + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); + if ((shared & _Py_REF_SHARED_FLAG_MASK) != 0) { + // Nothing to do if it's in WEAKREFS, QUEUED, or MERGED states. + return; + } + if (_Py_atomic_compare_exchange_ssize( + &op->ob_ref_shared, &shared, shared | _Py_REF_MAYBE_WEAKREF)) { + return; + } + } +} + +extern int _PyObject_ResurrectEndSlow(PyObject *op); #endif +// Temporarily resurrects an object during deallocation. The refcount is set +// to one. +static inline void +_PyObject_ResurrectStart(PyObject *op) +{ + assert(Py_REFCNT(op) == 0); +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyThreadState_GET()); +#endif +#ifdef Py_GIL_DISABLED + _Py_atomic_store_uintptr_relaxed(&op->ob_tid, _Py_ThreadId()); + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 1); + _Py_atomic_store_ssize_relaxed(&op->ob_ref_shared, 0); +#else + Py_SET_REFCNT(op, 1); +#endif +} + +// Undoes an object resurrection by decrementing the refcount without calling +// _Py_Dealloc(). Returns 0 if the object is dead (the normal case), and +// deallocation should continue. Returns 1 if the object is still alive. +static inline int +_PyObject_ResurrectEnd(PyObject *op) +{ +#ifdef Py_REF_DEBUG + _Py_DecRefTotal(_PyThreadState_GET()); +#endif +#ifndef Py_GIL_DISABLED + Py_SET_REFCNT(op, Py_REFCNT(op) - 1); + return Py_REFCNT(op) != 0; +#else + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + Py_ssize_t shared = _Py_atomic_load_ssize_acquire(&op->ob_ref_shared); + if (_Py_IsOwnedByCurrentThread(op) && local == 1 && shared == 0) { + // Fast-path: object has a single refcount and is owned by this thread + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 0); + return 0; + } + // Slow-path: object has a shared refcount or is not owned by this thread + return _PyObject_ResurrectEndSlow(op); +#endif +} + +/* Tries to incref op and returns 1 if successful or 0 otherwise. */ +static inline int +_Py_TryIncref(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + return _Py_TryIncrefFast(op) || _Py_TryIncRefShared(op); +#else + if (Py_REFCNT(op) > 0) { + Py_INCREF(op); + return 1; + } + return 0; +#endif +} + #ifdef Py_REF_DEBUG extern void _PyInterpreterState_FinalizeRefTotal(PyInterpreterState *); extern void _Py_FinalizeRefTotal(_PyRuntimeState *); @@ -547,7 +799,7 @@ _PyObject_GET_WEAKREFS_LISTPTR(PyObject *op) if (PyType_Check(op) && ((PyTypeObject *)op)->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState( + managed_static_type_state *state = _PyStaticType_GetState( interp, (PyTypeObject *)op); return _PyStaticType_GET_WEAKREFS_LISTPTR(state); } @@ -577,27 +829,40 @@ _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(PyObject *op) return (PyWeakReference **)((char *)op + offset); } +// Fast inlined version of PyType_IS_GC() +#define _PyType_IS_GC(t) _PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) // Fast inlined version of PyObject_IS_GC() static inline int _PyObject_IS_GC(PyObject *obj) { PyTypeObject *type = Py_TYPE(obj); - return (PyType_IS_GC(type) + return (_PyType_IS_GC(type) && (type->tp_is_gc == NULL || type->tp_is_gc(obj))); } -// Fast inlined version of PyType_IS_GC() -#define _PyType_IS_GC(t) _PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) +// Fast inlined version of PyObject_Hash() +static inline Py_hash_t +_PyObject_HashFast(PyObject *op) +{ + if (PyUnicode_CheckExact(op)) { + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED( + _PyASCIIObject_CAST(op)->hash); + if (hash != -1) { + return hash; + } + } + return PyObject_Hash(op); +} static inline size_t _PyType_PreHeaderSize(PyTypeObject *tp) { return ( #ifndef Py_GIL_DISABLED - _PyType_IS_GC(tp) * sizeof(PyGC_Head) + + (size_t)_PyType_IS_GC(tp) * sizeof(PyGC_Head) + #endif - _PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER) * 2 * sizeof(PyObject *) + (size_t)_PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER) * 2 * sizeof(PyObject *) ); } @@ -615,18 +880,32 @@ static inline int _PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) { } extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); -extern PyObject *_PyType_NewManagedObject(PyTypeObject *type); +PyAPI_FUNC(PyObject *) _PyType_NewManagedObject(PyTypeObject *type); extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *); extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int); - -extern int _PyObject_InitializeDict(PyObject *obj); -int _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); -extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, - PyObject *name, PyObject *value); -PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, - PyObject *name); +extern int _PyObject_SetAttributeErrorContext(PyObject *v, PyObject* name); + +void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); +extern int _PyObject_StoreInstanceAttribute(PyObject *obj, + PyObject *name, PyObject *value); +extern bool _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, + PyObject **attr); +extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *, + unsigned int *); + +// Cache the provided init method in the specialization cache of type if the +// provided type version matches the current version of the type. +// +// The cached value is borrowed and is only valid if guarded by a type +// version check. In free-threaded builds the init method must also use +// deferred reference counting. +// +// Returns 1 if the value was cached or 0 otherwise. +extern int _PyType_CacheInitForSpecialization(PyHeapTypeObject *type, + PyObject *init, + unsigned int tp_version); #ifdef Py_GIL_DISABLED # define MANAGED_DICT_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-1) @@ -637,50 +916,39 @@ PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, #endif typedef union { - PyObject *dict; - /* Use a char* to generate a warning if directly assigning a PyDictValues */ - char *values; -} PyDictOrValues; + PyDictObject *dict; +} PyManagedDictPointer; -static inline PyDictOrValues * -_PyObject_DictOrValuesPointer(PyObject *obj) +static inline PyManagedDictPointer * +_PyObject_ManagedDictPointer(PyObject *obj) { assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - return (PyDictOrValues *)((char *)obj + MANAGED_DICT_OFFSET); + return (PyManagedDictPointer *)((char *)obj + MANAGED_DICT_OFFSET); } -static inline int -_PyDictOrValues_IsValues(PyDictOrValues dorv) +static inline PyDictObject * +_PyObject_GetManagedDict(PyObject *obj) { - return ((uintptr_t)dorv.values) & 1; + PyManagedDictPointer *dorv = _PyObject_ManagedDictPointer(obj); + return (PyDictObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(dorv->dict); } static inline PyDictValues * -_PyDictOrValues_GetValues(PyDictOrValues dorv) -{ - assert(_PyDictOrValues_IsValues(dorv)); - return (PyDictValues *)(dorv.values + 1); -} - -static inline PyObject * -_PyDictOrValues_GetDict(PyDictOrValues dorv) +_PyObject_InlineValues(PyObject *obj) { - assert(!_PyDictOrValues_IsValues(dorv)); - return dorv.dict; -} - -static inline void -_PyDictOrValues_SetValues(PyDictOrValues *ptr, PyDictValues *values) -{ - ptr->values = ((char *)values) - 1; + PyTypeObject *tp = Py_TYPE(obj); + assert(tp->tp_basicsize > 0 && (size_t)tp->tp_basicsize % sizeof(PyObject *) == 0); + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + return (PyDictValues *)((char *)obj + tp->tp_basicsize); } extern PyObject ** _PyObject_ComputedDictPointer(PyObject *); -extern void _PyObject_FreeInstanceAttributes(PyObject *obj); extern int _PyObject_IsInstanceDictEmpty(PyObject *); // Export for 'math' shared extension PyAPI_FUNC(PyObject*) _PyObject_LookupSpecial(PyObject *, PyObject *); +PyAPI_FUNC(PyObject*) _PyObject_LookupSpecialMethod(PyObject *self, PyObject *attr, PyObject **self_or_null); extern int _PyObject_IsAbstract(PyObject *); @@ -705,13 +973,7 @@ PyAPI_FUNC(PyObject*) _PyObject_GetState(PyObject *); * Third party code unintentionally rely on problematic fpcasts. The call * trampoline mitigates common occurrences of bad fpcasts on Emscripten. */ -#if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) -#define _PyCFunction_TrampolineCall(meth, self, args) \ - _PyCFunctionWithKeywords_TrampolineCall( \ - (*(PyCFunctionWithKeywords)(void(*)(void))(meth)), (self), (args), NULL) -extern PyObject* _PyCFunctionWithKeywords_TrampolineCall( - PyCFunctionWithKeywords meth, PyObject *, PyObject *, PyObject *); -#else +#if !(defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE)) #define _PyCFunction_TrampolineCall(meth, self, args) \ (meth)((self), (args)) #define _PyCFunctionWithKeywords_TrampolineCall(meth, self, args, kw) \ @@ -726,6 +988,15 @@ PyAPI_DATA(PyTypeObject) _PyNotImplemented_Type; // Export for the stable ABI. PyAPI_DATA(int) _Py_SwappedOp[]; +extern void _Py_GetConstant_Init(void); + +enum _PyAnnotateFormat { + _Py_ANNOTATE_FORMAT_VALUE = 1, + _Py_ANNOTATE_FORMAT_VALUE_WITH_FAKE_GLOBALS = 2, + _Py_ANNOTATE_FORMAT_FORWARDREF = 3, + _Py_ANNOTATE_FORMAT_STRING = 4, +}; + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_object_deferred.h b/Include/internal/pycore_object_deferred.h new file mode 100644 index 00000000000000..c070d768b7d771 --- /dev/null +++ b/Include/internal/pycore_object_deferred.h @@ -0,0 +1,32 @@ +#ifndef Py_INTERNAL_OBJECT_DEFERRED_H +#define Py_INTERNAL_OBJECT_DEFERRED_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "pycore_gc.h" + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// Mark an object as supporting deferred reference counting. This is a no-op +// in the default (with GIL) build. Objects that use deferred reference +// counting should be tracked by the GC so that they are eventually collected. +extern void _PyObject_SetDeferredRefcount(PyObject *op); + +static inline int +_PyObject_HasDeferredRefcount(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_DEFERRED); +#else + return 0; +#endif +} + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_OBJECT_DEFERRED_H diff --git a/Include/internal/pycore_object_stack.h b/Include/internal/pycore_object_stack.h index fc130b1e9920b4..39e69b7cde52a1 100644 --- a/Include/internal/pycore_object_stack.h +++ b/Include/internal/pycore_object_stack.h @@ -1,8 +1,6 @@ #ifndef Py_INTERNAL_OBJECT_STACK_H #define Py_INTERNAL_OBJECT_STACK_H -#include "pycore_freelist.h" // _PyFreeListState - #ifdef __cplusplus extern "C" { #endif @@ -73,6 +71,16 @@ _PyObjectStack_Pop(_PyObjectStack *stack) return obj; } +static inline Py_ssize_t +_PyObjectStack_Size(_PyObjectStack *stack) +{ + Py_ssize_t size = 0; + for (_PyObjectStackChunk *buf = stack->head; buf != NULL; buf = buf->prev) { + size += buf->n; + } + return size; +} + // Merge src into dst, leaving src empty extern void _PyObjectStack_Merge(_PyObjectStack *dst, _PyObjectStack *src); diff --git a/Include/internal/pycore_object_state.h b/Include/internal/pycore_object_state.h index cd7c9335b3e611..8a47a6d9e6eb0d 100644 --- a/Include/internal/pycore_object_state.h +++ b/Include/internal/pycore_object_state.h @@ -8,8 +8,16 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyObject_freelists -#include "pycore_hashtable.h" // _Py_hashtable_t +#include "pycore_freelist_state.h" // _Py_freelists +#include "pycore_hashtable.h" // _Py_hashtable_t + + +/* Reference tracer state */ +struct _reftracer_runtime_state { + PyRefTracer tracer_func; + void* tracer_data; +}; + struct _py_object_runtime_state { #ifdef Py_REF_DEBUG @@ -20,7 +28,7 @@ struct _py_object_runtime_state { struct _py_object_state { #if !defined(Py_GIL_DISABLED) - struct _Py_object_freelists freelists; + struct _Py_freelists freelists; #endif #ifdef Py_REF_DEBUG Py_ssize_t reftotal; diff --git a/Include/internal/pycore_obmalloc.h b/Include/internal/pycore_obmalloc.h index 9140d8f08f0af1..a7ba8f340737aa 100644 --- a/Include/internal/pycore_obmalloc.h +++ b/Include/internal/pycore_obmalloc.h @@ -255,8 +255,8 @@ struct pool_header { union { pymem_block *_padding; uint count; } ref; /* number of allocated blocks */ pymem_block *freeblock; /* pool's free list head */ - struct pool_header *nextpool; /* next pool of this size class */ - struct pool_header *prevpool; /* previous pool "" */ + struct pool_header *nextpool; /* see "Pool table" for meaning */ + struct pool_header *prevpool; /* " */ uint arenaindex; /* index into arenas of base adr */ uint szidx; /* block size class index */ uint nextoffset; /* bytes to virgin block */ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index ab34366ab1066c..fe791c090120c9 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -20,12 +20,10 @@ extern "C" { #define IS_PSEUDO_INSTR(OP) ( \ ((OP) == LOAD_CLOSURE) || \ ((OP) == STORE_FAST_MAYBE_NULL) || \ - ((OP) == LOAD_SUPER_METHOD) || \ - ((OP) == LOAD_ZERO_SUPER_METHOD) || \ - ((OP) == LOAD_ZERO_SUPER_ATTR) || \ - ((OP) == LOAD_METHOD) || \ ((OP) == JUMP) || \ ((OP) == JUMP_NO_INTERRUPT) || \ + ((OP) == JUMP_IF_FALSE) || \ + ((OP) == JUMP_IF_TRUE) || \ ((OP) == SETUP_FINALLY) || \ ((OP) == SETUP_CLEANUP) || \ ((OP) == SETUP_WITH) || \ @@ -37,10 +35,6 @@ extern int _PyOpcode_num_popped(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA int _PyOpcode_num_popped(int opcode, int oparg) { switch(opcode) { - case BEFORE_ASYNC_WITH: - return 1; - case BEFORE_WITH: - return 1; case BINARY_OP: return 2; case BINARY_OP_ADD_FLOAT: @@ -49,6 +43,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2; case BINARY_OP_ADD_UNICODE: return 2; + case BINARY_OP_EXTEND: + return 2; case BINARY_OP_INPLACE_ADD_UNICODE: return 2; case BINARY_OP_MULTIPLY_FLOAT: @@ -73,8 +69,6 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2; case BINARY_SUBSCR_TUPLE_INT: return 2; - case BUILD_CONST_KEY_MAP: - return 1 + oparg; case BUILD_LIST: return oparg; case BUILD_MAP: @@ -82,7 +76,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case BUILD_SET: return oparg; case BUILD_SLICE: - return 2 + ((oparg == 3) ? 1 : 0); + return oparg; case BUILD_STRING: return oparg; case BUILD_TUPLE: @@ -95,6 +89,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2 + oparg; case CALL_BOUND_METHOD_EXACT_ARGS: return 2 + oparg; + case CALL_BOUND_METHOD_GENERAL: + return 2 + oparg; case CALL_BUILTIN_CLASS: return 2 + oparg; case CALL_BUILTIN_FAST: @@ -104,7 +100,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case CALL_BUILTIN_O: return 2 + oparg; case CALL_FUNCTION_EX: - return 3 + (oparg & 1); + return 4; case CALL_INTRINSIC_1: return 1; case CALL_INTRINSIC_2: @@ -113,10 +109,16 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2 + oparg; case CALL_KW: return 3 + oparg; + case CALL_KW_BOUND_METHOD: + return 3 + oparg; + case CALL_KW_NON_PY: + return 3 + oparg; + case CALL_KW_PY: + return 3 + oparg; case CALL_LEN: return 2 + oparg; case CALL_LIST_APPEND: - return 2 + oparg; + return 3; case CALL_METHOD_DESCRIPTOR_FAST: return 2 + oparg; case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: @@ -125,16 +127,18 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2 + oparg; case CALL_METHOD_DESCRIPTOR_O: return 2 + oparg; + case CALL_NON_PY_GENERAL: + return 2 + oparg; case CALL_PY_EXACT_ARGS: return 2 + oparg; - case CALL_PY_WITH_DEFAULTS: + case CALL_PY_GENERAL: return 2 + oparg; case CALL_STR_1: - return 2 + oparg; + return 3; case CALL_TUPLE_1: - return 2 + oparg; + return 3; case CALL_TYPE_1: - return 2 + oparg; + return 3; case CHECK_EG_MATCH: return 2; case CHECK_EXC_MATCH: @@ -151,6 +155,10 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2; case CONTAINS_OP: return 2; + case CONTAINS_OP_DICT: + return 2; + case CONTAINS_OP_SET: + return 2; case CONVERT_VALUE: return 1; case COPY: @@ -216,7 +224,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case IMPORT_NAME: return 2; case INSTRUMENTED_CALL: - return 0; + return 2 + oparg; case INSTRUMENTED_CALL_FUNCTION_EX: return 0; case INSTRUMENTED_CALL_KW: @@ -233,8 +241,14 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case INSTRUMENTED_JUMP_FORWARD: return 0; + case INSTRUMENTED_LINE: + return 0; case INSTRUMENTED_LOAD_SUPER_ATTR: - return 3; + return 0; + case INSTRUMENTED_NOT_TAKEN: + return 0; + case INSTRUMENTED_POP_ITER: + return 1; case INSTRUMENTED_POP_JUMP_IF_FALSE: return 0; case INSTRUMENTED_POP_JUMP_IF_NONE: @@ -245,8 +259,6 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case INSTRUMENTED_RESUME: return 0; - case INSTRUMENTED_RETURN_CONST: - return 0; case INSTRUMENTED_RETURN_VALUE: return 1; case INSTRUMENTED_YIELD_VALUE: @@ -255,22 +267,34 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case IS_OP: return 2; + case JUMP: + return 0; case JUMP_BACKWARD: return 0; + case JUMP_BACKWARD_JIT: + return 0; case JUMP_BACKWARD_NO_INTERRUPT: return 0; + case JUMP_BACKWARD_NO_JIT: + return 0; case JUMP_FORWARD: return 0; + case JUMP_IF_FALSE: + return 1; + case JUMP_IF_TRUE: + return 1; + case JUMP_NO_INTERRUPT: + return 0; case LIST_APPEND: return 2 + (oparg-1); case LIST_EXTEND: return 2 + (oparg-1); - case LOAD_ASSERTION_ERROR: - return 0; case LOAD_ATTR: return 1; case LOAD_ATTR_CLASS: return 1; + case LOAD_ATTR_CLASS_WITH_METACLASS_CHECK: + return 1; case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: return 1; case LOAD_ATTR_INSTANCE_VALUE: @@ -295,8 +319,16 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case LOAD_BUILD_CLASS: return 0; + case LOAD_CLOSURE: + return 0; + case LOAD_COMMON_CONSTANT: + return 0; case LOAD_CONST: return 0; + case LOAD_CONST_IMMORTAL: + return 0; + case LOAD_CONST_MORTAL: + return 0; case LOAD_DEREF: return 0; case LOAD_FAST: @@ -321,6 +353,10 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case LOAD_NAME: return 0; + case LOAD_SMALL_INT: + return 0; + case LOAD_SPECIAL: + return 1; case LOAD_SUPER_ATTR: return 3; case LOAD_SUPER_ATTR_ATTR: @@ -343,8 +379,14 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case NOP: return 0; + case NOT_TAKEN: + return 0; + case POP_BLOCK: + return 0; case POP_EXCEPT: return 1; + case POP_ITER: + return 1; case POP_JUMP_IF_FALSE: return 1; case POP_JUMP_IF_NONE: @@ -369,8 +411,6 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case RESUME_CHECK: return 0; - case RETURN_CONST: - return 0; case RETURN_GENERATOR: return 0; case RETURN_VALUE: @@ -381,6 +421,12 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2; case SETUP_ANNOTATIONS: return 0; + case SETUP_CLEANUP: + return 0; + case SETUP_FINALLY: + return 0; + case SETUP_WITH: + return 0; case SET_ADD: return 2 + (oparg-1); case SET_FUNCTION_ATTRIBUTE: @@ -401,6 +447,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case STORE_FAST_LOAD_FAST: return 1; + case STORE_FAST_MAYBE_NULL: + return 1; case STORE_FAST_STORE_FAST: return 2; case STORE_GLOBAL: @@ -448,7 +496,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case UNPACK_SEQUENCE_TWO_TUPLE: return 1; case WITH_EXCEPT_START: - return 4; + return 5; case YIELD_VALUE: return 1; default: @@ -462,10 +510,6 @@ extern int _PyOpcode_num_pushed(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA int _PyOpcode_num_pushed(int opcode, int oparg) { switch(opcode) { - case BEFORE_ASYNC_WITH: - return 2; - case BEFORE_WITH: - return 2; case BINARY_OP: return 1; case BINARY_OP_ADD_FLOAT: @@ -474,6 +518,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case BINARY_OP_ADD_UNICODE: return 1; + case BINARY_OP_EXTEND: + return 1; case BINARY_OP_INPLACE_ADD_UNICODE: return 0; case BINARY_OP_MULTIPLY_FLOAT: @@ -491,15 +537,13 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case BINARY_SUBSCR_DICT: return 1; case BINARY_SUBSCR_GETITEM: - return 1; + return 0; case BINARY_SUBSCR_LIST_INT: return 1; case BINARY_SUBSCR_STR_INT: return 1; case BINARY_SUBSCR_TUPLE_INT: return 1; - case BUILD_CONST_KEY_MAP: - return 1; case BUILD_LIST: return 1; case BUILD_MAP: @@ -517,9 +561,11 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case CALL: return 1; case CALL_ALLOC_AND_ENTER_INIT: - return 1; + return 0; case CALL_BOUND_METHOD_EXACT_ARGS: return 0; + case CALL_BOUND_METHOD_GENERAL: + return 0; case CALL_BUILTIN_CLASS: return 1; case CALL_BUILTIN_FAST: @@ -538,10 +584,16 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case CALL_KW: return 1; + case CALL_KW_BOUND_METHOD: + return 0; + case CALL_KW_NON_PY: + return 1; + case CALL_KW_PY: + return 0; case CALL_LEN: return 1; case CALL_LIST_APPEND: - return 1; + return 0; case CALL_METHOD_DESCRIPTOR_FAST: return 1; case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: @@ -550,10 +602,12 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case CALL_METHOD_DESCRIPTOR_O: return 1; + case CALL_NON_PY_GENERAL: + return 1; case CALL_PY_EXACT_ARGS: return 0; - case CALL_PY_WITH_DEFAULTS: - return 1; + case CALL_PY_GENERAL: + return 0; case CALL_STR_1: return 1; case CALL_TUPLE_1: @@ -576,6 +630,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case CONTAINS_OP: return 1; + case CONTAINS_OP_DICT: + return 1; + case CONTAINS_OP_SET: + return 1; case CONVERT_VALUE: return 1; case COPY: @@ -617,7 +675,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case FOR_ITER: return 2; case FOR_ITER_GEN: - return 2; + return 1; case FOR_ITER_LIST: return 2; case FOR_ITER_RANGE: @@ -641,7 +699,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case IMPORT_NAME: return 1; case INSTRUMENTED_CALL: - return 0; + return 1; case INSTRUMENTED_CALL_FUNCTION_EX: return 0; case INSTRUMENTED_CALL_KW: @@ -658,8 +716,14 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case INSTRUMENTED_JUMP_FORWARD: return 0; + case INSTRUMENTED_LINE: + return 0; case INSTRUMENTED_LOAD_SUPER_ATTR: - return 1 + (oparg & 1); + return 0; + case INSTRUMENTED_NOT_TAKEN: + return 0; + case INSTRUMENTED_POP_ITER: + return 0; case INSTRUMENTED_POP_JUMP_IF_FALSE: return 0; case INSTRUMENTED_POP_JUMP_IF_NONE: @@ -670,32 +734,42 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case INSTRUMENTED_RESUME: return 0; - case INSTRUMENTED_RETURN_CONST: - return 0; case INSTRUMENTED_RETURN_VALUE: - return 0; + return 1; case INSTRUMENTED_YIELD_VALUE: return 1; case INTERPRETER_EXIT: return 0; case IS_OP: return 1; + case JUMP: + return 0; case JUMP_BACKWARD: return 0; + case JUMP_BACKWARD_JIT: + return 0; case JUMP_BACKWARD_NO_INTERRUPT: return 0; + case JUMP_BACKWARD_NO_JIT: + return 0; case JUMP_FORWARD: return 0; + case JUMP_IF_FALSE: + return 1; + case JUMP_IF_TRUE: + return 1; + case JUMP_NO_INTERRUPT: + return 0; case LIST_APPEND: return 1 + (oparg-1); case LIST_EXTEND: return 1 + (oparg-1); - case LOAD_ASSERTION_ERROR: - return 1; case LOAD_ATTR: - return 1 + (oparg & 1); + return 1 + (oparg&1); case LOAD_ATTR_CLASS: return 1 + (oparg & 1); + case LOAD_ATTR_CLASS_WITH_METACLASS_CHECK: + return 1 + (oparg & 1); case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: return 1; case LOAD_ATTR_INSTANCE_VALUE: @@ -713,15 +787,23 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: return 1; case LOAD_ATTR_PROPERTY: - return 1; + return 0; case LOAD_ATTR_SLOT: return 1 + (oparg & 1); case LOAD_ATTR_WITH_HINT: return 1 + (oparg & 1); case LOAD_BUILD_CLASS: return 1; + case LOAD_CLOSURE: + return 1; + case LOAD_COMMON_CONSTANT: + return 1; case LOAD_CONST: return 1; + case LOAD_CONST_IMMORTAL: + return 1; + case LOAD_CONST_MORTAL: + return 1; case LOAD_DEREF: return 1; case LOAD_FAST: @@ -746,6 +828,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case LOAD_NAME: return 1; + case LOAD_SMALL_INT: + return 1; + case LOAD_SPECIAL: + return 2; case LOAD_SUPER_ATTR: return 1 + (oparg & 1); case LOAD_SUPER_ATTR_ATTR: @@ -768,8 +854,14 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 2; case NOP: return 0; + case NOT_TAKEN: + return 0; + case POP_BLOCK: + return 0; case POP_EXCEPT: return 0; + case POP_ITER: + return 0; case POP_JUMP_IF_FALSE: return 0; case POP_JUMP_IF_NONE: @@ -794,18 +886,22 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case RESUME_CHECK: return 0; - case RETURN_CONST: - return 0; case RETURN_GENERATOR: - return 0; + return 1; case RETURN_VALUE: - return 0; + return 1; case SEND: return 2; case SEND_GEN: - return 2; + return 1; case SETUP_ANNOTATIONS: return 0; + case SETUP_CLEANUP: + return 2; + case SETUP_FINALLY: + return 1; + case SETUP_WITH: + return 1; case SET_ADD: return 1 + (oparg-1); case SET_FUNCTION_ATTRIBUTE: @@ -826,6 +922,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case STORE_FAST_LOAD_FAST: return 1; + case STORE_FAST_MAYBE_NULL: + return 0; case STORE_FAST_STORE_FAST: return 0; case STORE_GLOBAL: @@ -863,7 +961,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case UNARY_NOT: return 1; case UNPACK_EX: - return 1 + (oparg >> 8) + (oparg & 0xFF); + return 1 + (oparg & 0xFF) + (oparg >> 8); case UNPACK_SEQUENCE: return oparg; case UNPACK_SEQUENCE_LIST: @@ -871,9 +969,9 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case UNPACK_SEQUENCE_TUPLE: return oparg; case UNPACK_SEQUENCE_TWO_TUPLE: - return oparg; + return 2; case WITH_EXCEPT_START: - return 5; + return 6; case YIELD_VALUE: return 1; default: @@ -883,20 +981,981 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { #endif +extern int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect); +#ifdef NEED_OPCODE_METADATA +int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect) { + switch(opcode) { + case BINARY_OP: { + *effect = 0; + return 0; + } + case BINARY_OP_ADD_FLOAT: { + *effect = 0; + return 0; + } + case BINARY_OP_ADD_INT: { + *effect = 0; + return 0; + } + case BINARY_OP_ADD_UNICODE: { + *effect = 0; + return 0; + } + case BINARY_OP_EXTEND: { + *effect = 0; + return 0; + } + case BINARY_OP_INPLACE_ADD_UNICODE: { + *effect = 0; + return 0; + } + case BINARY_OP_MULTIPLY_FLOAT: { + *effect = 0; + return 0; + } + case BINARY_OP_MULTIPLY_INT: { + *effect = 0; + return 0; + } + case BINARY_OP_SUBTRACT_FLOAT: { + *effect = 0; + return 0; + } + case BINARY_OP_SUBTRACT_INT: { + *effect = 0; + return 0; + } + case BINARY_SLICE: { + *effect = 0; + return 0; + } + case BINARY_SUBSCR: { + *effect = 1; + return 0; + } + case BINARY_SUBSCR_DICT: { + *effect = -1; + return 0; + } + case BINARY_SUBSCR_GETITEM: { + *effect = 1; + return 0; + } + case BINARY_SUBSCR_LIST_INT: { + *effect = -1; + return 0; + } + case BINARY_SUBSCR_STR_INT: { + *effect = -1; + return 0; + } + case BINARY_SUBSCR_TUPLE_INT: { + *effect = -1; + return 0; + } + case BUILD_LIST: { + *effect = 1 - oparg; + return 0; + } + case BUILD_MAP: { + *effect = 1 - oparg*2; + return 0; + } + case BUILD_SET: { + *effect = 1 - oparg; + return 0; + } + case BUILD_SLICE: { + *effect = 1 - oparg; + return 0; + } + case BUILD_STRING: { + *effect = 1 - oparg; + return 0; + } + case BUILD_TUPLE: { + *effect = 1 - oparg; + return 0; + } + case CACHE: { + *effect = 0; + return 0; + } + case CALL: { + int max_eff = Py_MAX(0, -1 - oparg); + max_eff = Py_MAX(max_eff, -2 - oparg); + *effect = max_eff; + return 0; + } + case CALL_ALLOC_AND_ENTER_INIT: { + int max_eff = Py_MAX(0, -1 - oparg); + max_eff = Py_MAX(max_eff, -2 - oparg); + *effect = max_eff; + return 0; + } + case CALL_BOUND_METHOD_EXACT_ARGS: { + int max_eff = Py_MAX(0, -1 - oparg); + max_eff = Py_MAX(max_eff, -2 - oparg); + *effect = max_eff; + return 0; + } + case CALL_BOUND_METHOD_GENERAL: { + int max_eff = Py_MAX(0, -1 - oparg); + max_eff = Py_MAX(max_eff, -2 - oparg); + *effect = max_eff; + return 0; + } + case CALL_BUILTIN_CLASS: { + *effect = -1 - oparg; + return 0; + } + case CALL_BUILTIN_FAST: { + *effect = -1 - oparg; + return 0; + } + case CALL_BUILTIN_FAST_WITH_KEYWORDS: { + *effect = -1 - oparg; + return 0; + } + case CALL_BUILTIN_O: { + *effect = -1 - oparg; + return 0; + } + case CALL_FUNCTION_EX: { + *effect = 0; + return 0; + } + case CALL_INTRINSIC_1: { + *effect = 0; + return 0; + } + case CALL_INTRINSIC_2: { + *effect = -1; + return 0; + } + case CALL_ISINSTANCE: { + *effect = -1 - oparg; + return 0; + } + case CALL_KW: { + int max_eff = Py_MAX(0, -2 - oparg); + max_eff = Py_MAX(max_eff, -3 - oparg); + *effect = max_eff; + return 0; + } + case CALL_KW_BOUND_METHOD: { + int max_eff = Py_MAX(0, -2 - oparg); + max_eff = Py_MAX(max_eff, -3 - oparg); + *effect = max_eff; + return 0; + } + case CALL_KW_NON_PY: { + *effect = Py_MAX(0, -2 - oparg); + return 0; + } + case CALL_KW_PY: { + int max_eff = Py_MAX(0, -2 - oparg); + max_eff = Py_MAX(max_eff, -3 - oparg); + *effect = max_eff; + return 0; + } + case CALL_LEN: { + *effect = -1 - oparg; + return 0; + } + case CALL_LIST_APPEND: { + *effect = -3; + return 0; + } + case CALL_METHOD_DESCRIPTOR_FAST: { + *effect = -1 - oparg; + return 0; + } + case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { + *effect = -1 - oparg; + return 0; + } + case CALL_METHOD_DESCRIPTOR_NOARGS: { + *effect = -1 - oparg; + return 0; + } + case CALL_METHOD_DESCRIPTOR_O: { + *effect = -1 - oparg; + return 0; + } + case CALL_NON_PY_GENERAL: { + *effect = Py_MAX(0, -1 - oparg); + return 0; + } + case CALL_PY_EXACT_ARGS: { + int max_eff = Py_MAX(0, -1 - oparg); + max_eff = Py_MAX(max_eff, -2 - oparg); + *effect = max_eff; + return 0; + } + case CALL_PY_GENERAL: { + int max_eff = Py_MAX(0, -1 - oparg); + max_eff = Py_MAX(max_eff, -2 - oparg); + *effect = max_eff; + return 0; + } + case CALL_STR_1: { + *effect = -2; + return 0; + } + case CALL_TUPLE_1: { + *effect = -2; + return 0; + } + case CALL_TYPE_1: { + *effect = -2; + return 0; + } + case CHECK_EG_MATCH: { + *effect = 0; + return 0; + } + case CHECK_EXC_MATCH: { + *effect = 0; + return 0; + } + case CLEANUP_THROW: { + *effect = -1; + return 0; + } + case COMPARE_OP: { + *effect = 0; + return 0; + } + case COMPARE_OP_FLOAT: { + *effect = 0; + return 0; + } + case COMPARE_OP_INT: { + *effect = 0; + return 0; + } + case COMPARE_OP_STR: { + *effect = 0; + return 0; + } + case CONTAINS_OP: { + *effect = 0; + return 0; + } + case CONTAINS_OP_DICT: { + *effect = -1; + return 0; + } + case CONTAINS_OP_SET: { + *effect = -1; + return 0; + } + case CONVERT_VALUE: { + *effect = 0; + return 0; + } + case COPY: { + *effect = 1; + return 0; + } + case COPY_FREE_VARS: { + *effect = 0; + return 0; + } + case DELETE_ATTR: { + *effect = -1; + return 0; + } + case DELETE_DEREF: { + *effect = 0; + return 0; + } + case DELETE_FAST: { + *effect = 0; + return 0; + } + case DELETE_GLOBAL: { + *effect = 0; + return 0; + } + case DELETE_NAME: { + *effect = 0; + return 0; + } + case DELETE_SUBSCR: { + *effect = -2; + return 0; + } + case DICT_MERGE: { + *effect = -1; + return 0; + } + case DICT_UPDATE: { + *effect = -1; + return 0; + } + case END_ASYNC_FOR: { + *effect = -2; + return 0; + } + case END_FOR: { + *effect = -1; + return 0; + } + case END_SEND: { + *effect = -1; + return 0; + } + case ENTER_EXECUTOR: { + *effect = 0; + return 0; + } + case EXIT_INIT_CHECK: { + *effect = -1; + return 0; + } + case EXTENDED_ARG: { + *effect = 0; + return 0; + } + case FORMAT_SIMPLE: { + *effect = 0; + return 0; + } + case FORMAT_WITH_SPEC: { + *effect = -1; + return 0; + } + case FOR_ITER: { + *effect = 1; + return 0; + } + case FOR_ITER_GEN: { + *effect = 1; + return 0; + } + case FOR_ITER_LIST: { + *effect = 1; + return 0; + } + case FOR_ITER_RANGE: { + *effect = 1; + return 0; + } + case FOR_ITER_TUPLE: { + *effect = 1; + return 0; + } + case GET_AITER: { + *effect = 0; + return 0; + } + case GET_ANEXT: { + *effect = 1; + return 0; + } + case GET_AWAITABLE: { + *effect = 0; + return 0; + } + case GET_ITER: { + *effect = 0; + return 0; + } + case GET_LEN: { + *effect = 1; + return 0; + } + case GET_YIELD_FROM_ITER: { + *effect = 0; + return 0; + } + case IMPORT_FROM: { + *effect = 1; + return 0; + } + case IMPORT_NAME: { + *effect = -1; + return 0; + } + case INSTRUMENTED_CALL: { + *effect = Py_MAX(0, -1 - oparg); + return 0; + } + case INSTRUMENTED_CALL_FUNCTION_EX: { + *effect = 0; + return 0; + } + case INSTRUMENTED_CALL_KW: { + *effect = 0; + return 0; + } + case INSTRUMENTED_END_FOR: { + *effect = -1; + return 0; + } + case INSTRUMENTED_END_SEND: { + *effect = -1; + return 0; + } + case INSTRUMENTED_FOR_ITER: { + *effect = 0; + return 0; + } + case INSTRUMENTED_INSTRUCTION: { + *effect = 0; + return 0; + } + case INSTRUMENTED_JUMP_BACKWARD: { + *effect = 0; + return 0; + } + case INSTRUMENTED_JUMP_FORWARD: { + *effect = 0; + return 0; + } + case INSTRUMENTED_LINE: { + *effect = 0; + return 0; + } + case INSTRUMENTED_LOAD_SUPER_ATTR: { + *effect = 0; + return 0; + } + case INSTRUMENTED_NOT_TAKEN: { + *effect = 0; + return 0; + } + case INSTRUMENTED_POP_ITER: { + *effect = -1; + return 0; + } + case INSTRUMENTED_POP_JUMP_IF_FALSE: { + *effect = 0; + return 0; + } + case INSTRUMENTED_POP_JUMP_IF_NONE: { + *effect = 0; + return 0; + } + case INSTRUMENTED_POP_JUMP_IF_NOT_NONE: { + *effect = 0; + return 0; + } + case INSTRUMENTED_POP_JUMP_IF_TRUE: { + *effect = 0; + return 0; + } + case INSTRUMENTED_RESUME: { + *effect = 0; + return 0; + } + case INSTRUMENTED_RETURN_VALUE: { + *effect = 0; + return 0; + } + case INSTRUMENTED_YIELD_VALUE: { + *effect = 0; + return 0; + } + case INTERPRETER_EXIT: { + *effect = -1; + return 0; + } + case IS_OP: { + *effect = -1; + return 0; + } + case JUMP: { + *effect = 0; + return 0; + } + case JUMP_BACKWARD: { + *effect = 0; + return 0; + } + case JUMP_BACKWARD_JIT: { + *effect = 0; + return 0; + } + case JUMP_BACKWARD_NO_INTERRUPT: { + *effect = 0; + return 0; + } + case JUMP_BACKWARD_NO_JIT: { + *effect = 0; + return 0; + } + case JUMP_FORWARD: { + *effect = 0; + return 0; + } + case JUMP_IF_FALSE: { + *effect = 0; + return 0; + } + case JUMP_IF_TRUE: { + *effect = 0; + return 0; + } + case JUMP_NO_INTERRUPT: { + *effect = 0; + return 0; + } + case LIST_APPEND: { + *effect = -1; + return 0; + } + case LIST_EXTEND: { + *effect = -1; + return 0; + } + case LOAD_ATTR: { + int max_eff = Py_MAX(1, (oparg & 1)); + max_eff = Py_MAX(max_eff, (oparg&1)); + *effect = max_eff; + return 0; + } + case LOAD_ATTR_CLASS: { + *effect = Py_MAX(0, (oparg & 1)); + return 0; + } + case LOAD_ATTR_CLASS_WITH_METACLASS_CHECK: { + *effect = Py_MAX(0, (oparg & 1)); + return 0; + } + case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: { + *effect = 0; + return 0; + } + case LOAD_ATTR_INSTANCE_VALUE: { + *effect = Py_MAX(0, (oparg & 1)); + return 0; + } + case LOAD_ATTR_METHOD_LAZY_DICT: { + *effect = 1; + return 0; + } + case LOAD_ATTR_METHOD_NO_DICT: { + *effect = 1; + return 0; + } + case LOAD_ATTR_METHOD_WITH_VALUES: { + *effect = 1; + return 0; + } + case LOAD_ATTR_MODULE: { + *effect = Py_MAX(1, (oparg & 1)); + return 0; + } + case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { + *effect = 0; + return 0; + } + case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { + *effect = 0; + return 0; + } + case LOAD_ATTR_PROPERTY: { + *effect = 0; + return 0; + } + case LOAD_ATTR_SLOT: { + *effect = Py_MAX(0, (oparg & 1)); + return 0; + } + case LOAD_ATTR_WITH_HINT: { + *effect = Py_MAX(1, (oparg & 1)); + return 0; + } + case LOAD_BUILD_CLASS: { + *effect = 1; + return 0; + } + case LOAD_CLOSURE: { + *effect = 1; + return 0; + } + case LOAD_COMMON_CONSTANT: { + *effect = 1; + return 0; + } + case LOAD_CONST: { + *effect = 1; + return 0; + } + case LOAD_CONST_IMMORTAL: { + *effect = 1; + return 0; + } + case LOAD_CONST_MORTAL: { + *effect = 1; + return 0; + } + case LOAD_DEREF: { + *effect = 1; + return 0; + } + case LOAD_FAST: { + *effect = 1; + return 0; + } + case LOAD_FAST_AND_CLEAR: { + *effect = 1; + return 0; + } + case LOAD_FAST_CHECK: { + *effect = 1; + return 0; + } + case LOAD_FAST_LOAD_FAST: { + *effect = 2; + return 0; + } + case LOAD_FROM_DICT_OR_DEREF: { + *effect = 0; + return 0; + } + case LOAD_FROM_DICT_OR_GLOBALS: { + *effect = 0; + return 0; + } + case LOAD_GLOBAL: { + *effect = Py_MAX(1, 1 + (oparg & 1)); + return 0; + } + case LOAD_GLOBAL_BUILTIN: { + *effect = Py_MAX(1, 1 + (oparg & 1)); + return 0; + } + case LOAD_GLOBAL_MODULE: { + *effect = Py_MAX(1, 1 + (oparg & 1)); + return 0; + } + case LOAD_LOCALS: { + *effect = 1; + return 0; + } + case LOAD_NAME: { + *effect = 1; + return 0; + } + case LOAD_SMALL_INT: { + *effect = 1; + return 0; + } + case LOAD_SPECIAL: { + *effect = 1; + return 0; + } + case LOAD_SUPER_ATTR: { + *effect = Py_MAX(0, -2 + (oparg & 1)); + return 0; + } + case LOAD_SUPER_ATTR_ATTR: { + *effect = -2; + return 0; + } + case LOAD_SUPER_ATTR_METHOD: { + *effect = -1; + return 0; + } + case MAKE_CELL: { + *effect = 0; + return 0; + } + case MAKE_FUNCTION: { + *effect = 0; + return 0; + } + case MAP_ADD: { + *effect = -2; + return 0; + } + case MATCH_CLASS: { + *effect = -2; + return 0; + } + case MATCH_KEYS: { + *effect = 1; + return 0; + } + case MATCH_MAPPING: { + *effect = 1; + return 0; + } + case MATCH_SEQUENCE: { + *effect = 1; + return 0; + } + case NOP: { + *effect = 0; + return 0; + } + case NOT_TAKEN: { + *effect = 0; + return 0; + } + case POP_BLOCK: { + *effect = 0; + return 0; + } + case POP_EXCEPT: { + *effect = -1; + return 0; + } + case POP_ITER: { + *effect = -1; + return 0; + } + case POP_JUMP_IF_FALSE: { + *effect = -1; + return 0; + } + case POP_JUMP_IF_NONE: { + *effect = 0; + return 0; + } + case POP_JUMP_IF_NOT_NONE: { + *effect = 0; + return 0; + } + case POP_JUMP_IF_TRUE: { + *effect = -1; + return 0; + } + case POP_TOP: { + *effect = -1; + return 0; + } + case PUSH_EXC_INFO: { + *effect = 1; + return 0; + } + case PUSH_NULL: { + *effect = 1; + return 0; + } + case RAISE_VARARGS: { + *effect = -oparg; + return 0; + } + case RERAISE: { + *effect = -1; + return 0; + } + case RESERVED: { + *effect = 0; + return 0; + } + case RESUME: { + *effect = 0; + return 0; + } + case RESUME_CHECK: { + *effect = 0; + return 0; + } + case RETURN_GENERATOR: { + *effect = 1; + return 0; + } + case RETURN_VALUE: { + *effect = 0; + return 0; + } + case SEND: { + *effect = 0; + return 0; + } + case SEND_GEN: { + *effect = 0; + return 0; + } + case SETUP_ANNOTATIONS: { + *effect = 0; + return 0; + } + case SETUP_CLEANUP: { + *effect = 2; + return 0; + } + case SETUP_FINALLY: { + *effect = 1; + return 0; + } + case SETUP_WITH: { + *effect = 1; + return 0; + } + case SET_ADD: { + *effect = -1; + return 0; + } + case SET_FUNCTION_ATTRIBUTE: { + *effect = -1; + return 0; + } + case SET_UPDATE: { + *effect = -1; + return 0; + } + case STORE_ATTR: { + *effect = 0; + return 0; + } + case STORE_ATTR_INSTANCE_VALUE: { + *effect = 0; + return 0; + } + case STORE_ATTR_SLOT: { + *effect = 0; + return 0; + } + case STORE_ATTR_WITH_HINT: { + *effect = 0; + return 0; + } + case STORE_DEREF: { + *effect = -1; + return 0; + } + case STORE_FAST: { + *effect = -1; + return 0; + } + case STORE_FAST_LOAD_FAST: { + *effect = 0; + return 0; + } + case STORE_FAST_MAYBE_NULL: { + *effect = -1; + return 0; + } + case STORE_FAST_STORE_FAST: { + *effect = -2; + return 0; + } + case STORE_GLOBAL: { + *effect = -1; + return 0; + } + case STORE_NAME: { + *effect = -1; + return 0; + } + case STORE_SLICE: { + *effect = 0; + return 0; + } + case STORE_SUBSCR: { + *effect = 0; + return 0; + } + case STORE_SUBSCR_DICT: { + *effect = -3; + return 0; + } + case STORE_SUBSCR_LIST_INT: { + *effect = -3; + return 0; + } + case SWAP: { + *effect = 0; + return 0; + } + case TO_BOOL: { + *effect = 0; + return 0; + } + case TO_BOOL_ALWAYS_TRUE: { + *effect = 0; + return 0; + } + case TO_BOOL_BOOL: { + *effect = 0; + return 0; + } + case TO_BOOL_INT: { + *effect = 0; + return 0; + } + case TO_BOOL_LIST: { + *effect = 0; + return 0; + } + case TO_BOOL_NONE: { + *effect = 0; + return 0; + } + case TO_BOOL_STR: { + *effect = 0; + return 0; + } + case UNARY_INVERT: { + *effect = 0; + return 0; + } + case UNARY_NEGATIVE: { + *effect = 0; + return 0; + } + case UNARY_NOT: { + *effect = 0; + return 0; + } + case UNPACK_EX: { + *effect = (oparg & 0xFF) + (oparg >> 8); + return 0; + } + case UNPACK_SEQUENCE: { + *effect = Py_MAX(1, -1 + oparg); + return 0; + } + case UNPACK_SEQUENCE_LIST: { + *effect = -1 + oparg; + return 0; + } + case UNPACK_SEQUENCE_TUPLE: { + *effect = -1 + oparg; + return 0; + } + case UNPACK_SEQUENCE_TWO_TUPLE: { + *effect = 1; + return 0; + } + case WITH_EXCEPT_START: { + *effect = 1; + return 0; + } + case YIELD_VALUE: { + *effect = 0; + return 0; + } + default: + return -1; + } +} + +#endif + enum InstructionFormat { INSTR_FMT_IB = 1, INSTR_FMT_IBC = 2, INSTR_FMT_IBC00 = 3, INSTR_FMT_IBC000 = 4, - INSTR_FMT_IBC00000000 = 5, - INSTR_FMT_IX = 6, - INSTR_FMT_IXC = 7, - INSTR_FMT_IXC00 = 8, - INSTR_FMT_IXC000 = 9, + INSTR_FMT_IBC0000 = 5, + INSTR_FMT_IBC00000000 = 6, + INSTR_FMT_IX = 7, + INSTR_FMT_IXC = 8, + INSTR_FMT_IXC00 = 9, + INSTR_FMT_IXC000 = 10, + INSTR_FMT_IXC0000 = 11, }; #define IS_VALID_OPCODE(OP) \ - (((OP) >= 0) && ((OP) < 268) && \ + (((OP) >= 0) && ((OP) < 266) && \ (_PyOpcode_opcode_metadata[(OP)].valid_entry)) #define HAS_ARG_FLAG (1) @@ -913,6 +1972,8 @@ enum InstructionFormat { #define HAS_PURE_FLAG (2048) #define HAS_PASSTHROUGH_FLAG (4096) #define HAS_OPARG_AND_1_FLAG (8192) +#define HAS_ERROR_NO_POP_FLAG (16384) +#define HAS_NO_SAVE_IP_FLAG (32768) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -927,6 +1988,8 @@ enum InstructionFormat { #define OPCODE_HAS_PURE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PURE_FLAG)) #define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG)) #define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG)) +#define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) +#define OPCODE_HAS_NO_SAVE_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NO_SAVE_IP_FLAG)) #define OPARG_FULL 0 #define OPARG_CACHE_1 1 @@ -939,159 +2002,173 @@ enum InstructionFormat { struct opcode_metadata { uint8_t valid_entry; - int8_t instr_format; - int16_t flags; + uint8_t instr_format; + uint16_t flags; }; -extern const struct opcode_metadata _PyOpcode_opcode_metadata[268]; +extern const struct opcode_metadata _PyOpcode_opcode_metadata[266]; #ifdef NEED_OPCODE_METADATA -const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { - [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, +const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { + [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_EXTEND] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BUILD_CONST_KEY_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [BINARY_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CACHE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, + [CACHE] = { true, INSTR_FMT_IX, 0 }, + [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_PY_WITH_DEFAULTS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_NON_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [CHECK_EG_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [CONTAINS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, + [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, + [CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, - [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [END_FOR] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, - [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, - [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, - [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, + [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, - [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, + [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, + [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IXC, 0 }, + [INSTRUMENTED_NOT_TAKEN] = { true, INSTR_FMT_IX, 0 }, + [INSTRUMENTED_POP_ITER] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [INSTRUMENTED_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, 0 }, [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [JUMP_BACKWARD_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD_NO_INTERRUPT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [JUMP_BACKWARD_NO_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ASSERTION_ERROR] = { true, INSTR_FMT_IX, 0 }, [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG }, + [LOAD_COMMON_CONSTANT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, + [LOAD_CONST_IMMORTAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, + [LOAD_CONST_MORTAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, + [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SMALL_INT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [LOAD_SPECIAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1099,7 +2176,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, [NOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [NOT_TAKEN] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [POP_ITER] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, @@ -1107,28 +2186,27 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [POP_TOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, - [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RESERVED] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [RESERVED] = { true, INSTR_FMT_IX, 0 }, + [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ESCAPES_FLAG }, [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [RETURN_VALUE] = { true, INSTR_FMT_IX, 0 }, [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, - [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG }, + [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG }, + [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG }, [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1137,12 +2215,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, @@ -1152,23 +2230,21 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [JUMP_IF_TRUE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_NO_INTERRUPT] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, - [LOAD_METHOD] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_SUPER_METHOD] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ZERO_SUPER_ATTR] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ZERO_SUPER_METHOD] = { true, -1, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [POP_BLOCK] = { true, -1, HAS_PURE_FLAG }, [SETUP_CLEANUP] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, [SETUP_FINALLY] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, [SETUP_WITH] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, - [STORE_FAST_MAYBE_NULL] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST_MAYBE_NULL] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG }, }; #endif -#define MAX_UOP_PER_EXPANSION 8 +#define MAX_UOP_PER_EXPANSION 9 struct opcode_macro_expansion { int nuops; struct { int16_t uop; int8_t size; int8_t offset; } uops[MAX_UOP_PER_EXPANSION]; @@ -1178,12 +2254,12 @@ extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[256]; #ifdef NEED_OPCODE_METADATA const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { - [BEFORE_ASYNC_WITH] = { .nuops = 1, .uops = { { _BEFORE_ASYNC_WITH, 0, 0 } } }, - [BEFORE_WITH] = { .nuops = 1, .uops = { { _BEFORE_WITH, 0, 0 } } }, [BINARY_OP] = { .nuops = 1, .uops = { { _BINARY_OP, 0, 0 } } }, [BINARY_OP_ADD_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_ADD_FLOAT, 0, 0 } } }, [BINARY_OP_ADD_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_ADD_INT, 0, 0 } } }, [BINARY_OP_ADD_UNICODE] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _BINARY_OP_ADD_UNICODE, 0, 0 } } }, + [BINARY_OP_EXTEND] = { .nuops = 2, .uops = { { _GUARD_BINARY_OP_EXTEND, 4, 1 }, { _BINARY_OP_EXTEND, 4, 1 } } }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _BINARY_OP_INPLACE_ADD_UNICODE, 0, 0 } } }, [BINARY_OP_MULTIPLY_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_MULTIPLY_FLOAT, 0, 0 } } }, [BINARY_OP_MULTIPLY_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_MULTIPLY_INT, 0, 0 } } }, [BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, 0, 0 } } }, @@ -1191,32 +2267,40 @@ _PyOpcode_macro_expansion[256] = { [BINARY_SLICE] = { .nuops = 1, .uops = { { _BINARY_SLICE, 0, 0 } } }, [BINARY_SUBSCR] = { .nuops = 1, .uops = { { _BINARY_SUBSCR, 0, 0 } } }, [BINARY_SUBSCR_DICT] = { .nuops = 1, .uops = { { _BINARY_SUBSCR_DICT, 0, 0 } } }, + [BINARY_SUBSCR_GETITEM] = { .nuops = 4, .uops = { { _CHECK_PEP_523, 0, 0 }, { _BINARY_SUBSCR_CHECK_FUNC, 0, 0 }, { _BINARY_SUBSCR_INIT_CALL, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, [BINARY_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { _BINARY_SUBSCR_LIST_INT, 0, 0 } } }, [BINARY_SUBSCR_STR_INT] = { .nuops = 1, .uops = { { _BINARY_SUBSCR_STR_INT, 0, 0 } } }, [BINARY_SUBSCR_TUPLE_INT] = { .nuops = 1, .uops = { { _BINARY_SUBSCR_TUPLE_INT, 0, 0 } } }, - [BUILD_CONST_KEY_MAP] = { .nuops = 1, .uops = { { _BUILD_CONST_KEY_MAP, 0, 0 } } }, [BUILD_LIST] = { .nuops = 1, .uops = { { _BUILD_LIST, 0, 0 } } }, [BUILD_MAP] = { .nuops = 1, .uops = { { _BUILD_MAP, 0, 0 } } }, [BUILD_SET] = { .nuops = 1, .uops = { { _BUILD_SET, 0, 0 } } }, [BUILD_SLICE] = { .nuops = 1, .uops = { { _BUILD_SLICE, 0, 0 } } }, [BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, 0, 0 } } }, [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, 0, 0 } } }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, - [CALL_BUILTIN_CLASS] = { .nuops = 1, .uops = { { _CALL_BUILTIN_CLASS, 0, 0 } } }, - [CALL_BUILTIN_FAST] = { .nuops = 1, .uops = { { _CALL_BUILTIN_FAST, 0, 0 } } }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, 0, 0 } } }, - [CALL_BUILTIN_O] = { .nuops = 1, .uops = { { _CALL_BUILTIN_O, 0, 0 } } }, + [CALL_ALLOC_AND_ENTER_INIT] = { .nuops = 4, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_AND_ALLOCATE_OBJECT, 2, 1 }, { _CREATE_INIT_FRAME, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 9, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, 0, 0 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_BOUND_METHOD_GENERAL] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, 0, 0 }, { _PY_FRAME_GENERAL, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_BUILTIN_CLASS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_CLASS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_BUILTIN_FAST] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_BUILTIN_O] = { .nuops = 2, .uops = { { _CALL_BUILTIN_O, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_1, 0, 0 } } }, [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_2, 0, 0 } } }, [CALL_ISINSTANCE] = { .nuops = 1, .uops = { { _CALL_ISINSTANCE, 0, 0 } } }, + [CALL_KW_BOUND_METHOD] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, 0, 0 }, { _PY_FRAME_KW, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, 0, 0 }, { _CALL_KW_NON_PY, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_KW_PY] = { .nuops = 5, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _PY_FRAME_KW, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_LEN] = { .nuops = 1, .uops = { { _CALL_LEN, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_O, 0, 0 } } }, - [CALL_PY_EXACT_ARGS] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, - [CALL_STR_1] = { .nuops = 1, .uops = { { _CALL_STR_1, 0, 0 } } }, - [CALL_TUPLE_1] = { .nuops = 1, .uops = { { _CALL_TUPLE_1, 0, 0 } } }, + [CALL_LIST_APPEND] = { .nuops = 1, .uops = { { _CALL_LIST_APPEND, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_O, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_NON_PY_GENERAL] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE, 0, 0 }, { _CALL_NON_PY_GENERAL, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_PY_EXACT_ARGS] = { .nuops = 7, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, 0, 0 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_PY_GENERAL] = { .nuops = 5, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _PY_FRAME_GENERAL, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_STR_1] = { .nuops = 2, .uops = { { _CALL_STR_1, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_TUPLE_1] = { .nuops = 2, .uops = { { _CALL_TUPLE_1, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_TYPE_1] = { .nuops = 1, .uops = { { _CALL_TYPE_1, 0, 0 } } }, [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { _CHECK_EG_MATCH, 0, 0 } } }, [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { _CHECK_EXC_MATCH, 0, 0 } } }, @@ -1225,6 +2309,8 @@ _PyOpcode_macro_expansion[256] = { [COMPARE_OP_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _COMPARE_OP_INT, 0, 0 } } }, [COMPARE_OP_STR] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _COMPARE_OP_STR, 0, 0 } } }, [CONTAINS_OP] = { .nuops = 1, .uops = { { _CONTAINS_OP, 0, 0 } } }, + [CONTAINS_OP_DICT] = { .nuops = 1, .uops = { { _CONTAINS_OP_DICT, 0, 0 } } }, + [CONTAINS_OP_SET] = { .nuops = 1, .uops = { { _CONTAINS_OP_SET, 0, 0 } } }, [CONVERT_VALUE] = { .nuops = 1, .uops = { { _CONVERT_VALUE, 0, 0 } } }, [COPY] = { .nuops = 1, .uops = { { _COPY, 0, 0 } } }, [COPY_FREE_VARS] = { .nuops = 1, .uops = { { _COPY_FREE_VARS, 0, 0 } } }, @@ -1236,12 +2322,13 @@ _PyOpcode_macro_expansion[256] = { [DELETE_SUBSCR] = { .nuops = 1, .uops = { { _DELETE_SUBSCR, 0, 0 } } }, [DICT_MERGE] = { .nuops = 1, .uops = { { _DICT_MERGE, 0, 0 } } }, [DICT_UPDATE] = { .nuops = 1, .uops = { { _DICT_UPDATE, 0, 0 } } }, - [END_FOR] = { .nuops = 1, .uops = { { _POP_TOP, 0, 0 } } }, + [END_FOR] = { .nuops = 1, .uops = { { _END_FOR, 0, 0 } } }, [END_SEND] = { .nuops = 1, .uops = { { _END_SEND, 0, 0 } } }, [EXIT_INIT_CHECK] = { .nuops = 1, .uops = { { _EXIT_INIT_CHECK, 0, 0 } } }, [FORMAT_SIMPLE] = { .nuops = 1, .uops = { { _FORMAT_SIMPLE, 0, 0 } } }, [FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { _FORMAT_WITH_SPEC, 0, 0 } } }, [FOR_ITER] = { .nuops = 1, .uops = { { _FOR_ITER, 9, 0 } } }, + [FOR_ITER_GEN] = { .nuops = 3, .uops = { { _CHECK_PEP_523, 0, 0 }, { _FOR_ITER_GEN_FRAME, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, [FOR_ITER_LIST] = { .nuops = 3, .uops = { { _ITER_CHECK_LIST, 0, 0 }, { _ITER_JUMP_LIST, 9, 1 }, { _ITER_NEXT_LIST, 0, 0 } } }, [FOR_ITER_RANGE] = { .nuops = 3, .uops = { { _ITER_CHECK_RANGE, 0, 0 }, { _ITER_JUMP_RANGE, 9, 1 }, { _ITER_NEXT_RANGE, 0, 0 } } }, [FOR_ITER_TUPLE] = { .nuops = 3, .uops = { { _ITER_CHECK_TUPLE, 0, 0 }, { _ITER_JUMP_TUPLE, 9, 1 }, { _ITER_NEXT_TUPLE, 0, 0 } } }, @@ -1251,35 +2338,41 @@ _PyOpcode_macro_expansion[256] = { [GET_ITER] = { .nuops = 1, .uops = { { _GET_ITER, 0, 0 } } }, [GET_LEN] = { .nuops = 1, .uops = { { _GET_LEN, 0, 0 } } }, [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { _GET_YIELD_FROM_ITER, 0, 0 } } }, + [IMPORT_FROM] = { .nuops = 1, .uops = { { _IMPORT_FROM, 0, 0 } } }, + [IMPORT_NAME] = { .nuops = 1, .uops = { { _IMPORT_NAME, 0, 0 } } }, [IS_OP] = { .nuops = 1, .uops = { { _IS_OP, 0, 0 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { _LIST_APPEND, 0, 0 } } }, [LIST_EXTEND] = { .nuops = 1, .uops = { { _LIST_EXTEND, 0, 0 } } }, - [LOAD_ASSERTION_ERROR] = { .nuops = 1, .uops = { { _LOAD_ASSERTION_ERROR, 0, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, 0, 0 } } }, - [LOAD_ATTR_CLASS] = { .nuops = 2, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 } } }, - [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 0, 0 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, + [LOAD_ATTR_CLASS] = { .nuops = 3, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, 0, 0 } } }, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { .nuops = 4, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _GUARD_TYPE_VERSION, 2, 3 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, 0, 0 } } }, + [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 }, { _PUSH_NULL_CONDITIONAL, 0, 0 } } }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 1, 3 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, - [LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, 1, 3 } } }, + [LOAD_ATTR_MODULE] = { .nuops = 3, .uops = { { _CHECK_ATTR_MODULE_PUSH_KEYS, 2, 1 }, { _LOAD_ATTR_MODULE_FROM_KEYS, 1, 3 }, { _PUSH_NULL_CONDITIONAL, 0, 0 } } }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, - [LOAD_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 } } }, - [LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_WITH_HINT, 0, 0 }, { _LOAD_ATTR_WITH_HINT, 1, 3 } } }, + [LOAD_ATTR_PROPERTY] = { .nuops = 5, .uops = { { _CHECK_PEP_523, 0, 0 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_PROPERTY_FRAME, 4, 5 }, { _SAVE_RETURN_OFFSET, 7, 9 }, { _PUSH_FRAME, 0, 0 } } }, + [LOAD_ATTR_SLOT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 }, { _PUSH_NULL_CONDITIONAL, 0, 0 } } }, + [LOAD_ATTR_WITH_HINT] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_WITH_HINT, 0, 0 }, { _LOAD_ATTR_WITH_HINT, 1, 3 }, { _PUSH_NULL_CONDITIONAL, 0, 0 } } }, [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { _LOAD_BUILD_CLASS, 0, 0 } } }, - [LOAD_CONST] = { .nuops = 1, .uops = { { _LOAD_CONST, 0, 0 } } }, + [LOAD_COMMON_CONSTANT] = { .nuops = 1, .uops = { { _LOAD_COMMON_CONSTANT, 0, 0 } } }, + [LOAD_CONST_IMMORTAL] = { .nuops = 1, .uops = { { _LOAD_CONST_IMMORTAL, 0, 0 } } }, + [LOAD_CONST_MORTAL] = { .nuops = 1, .uops = { { _LOAD_CONST_MORTAL, 0, 0 } } }, [LOAD_DEREF] = { .nuops = 1, .uops = { { _LOAD_DEREF, 0, 0 } } }, [LOAD_FAST] = { .nuops = 1, .uops = { { _LOAD_FAST, 0, 0 } } }, [LOAD_FAST_AND_CLEAR] = { .nuops = 1, .uops = { { _LOAD_FAST_AND_CLEAR, 0, 0 } } }, [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { _LOAD_FAST_CHECK, 0, 0 } } }, [LOAD_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { _LOAD_FAST, 5, 0 }, { _LOAD_FAST, 6, 0 } } }, [LOAD_FROM_DICT_OR_DEREF] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_DEREF, 0, 0 } } }, - [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, - [LOAD_GLOBAL] = { .nuops = 1, .uops = { { _LOAD_GLOBAL, 0, 0 } } }, - [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, - [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, + [LOAD_GLOBAL] = { .nuops = 2, .uops = { { _LOAD_GLOBAL, 0, 0 }, { _PUSH_NULL_CONDITIONAL, 0, 0 } } }, + [LOAD_GLOBAL_BUILTIN] = { .nuops = 4, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION_PUSH_KEYS, 1, 2 }, { _LOAD_GLOBAL_BUILTINS_FROM_KEYS, 1, 3 }, { _PUSH_NULL_CONDITIONAL, 0, 0 } } }, + [LOAD_GLOBAL_MODULE] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION_PUSH_KEYS, 1, 1 }, { _LOAD_GLOBAL_MODULE_FROM_KEYS, 1, 3 }, { _PUSH_NULL_CONDITIONAL, 0, 0 } } }, [LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, 0, 0 } } }, [LOAD_NAME] = { .nuops = 1, .uops = { { _LOAD_NAME, 0, 0 } } }, + [LOAD_SMALL_INT] = { .nuops = 1, .uops = { { _LOAD_SMALL_INT, 0, 0 } } }, + [LOAD_SPECIAL] = { .nuops = 1, .uops = { { _LOAD_SPECIAL, 0, 0 } } }, [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_ATTR, 0, 0 } } }, [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_METHOD, 0, 0 } } }, [MAKE_CELL] = { .nuops = 1, .uops = { { _MAKE_CELL, 0, 0 } } }, @@ -1290,7 +2383,9 @@ _PyOpcode_macro_expansion[256] = { [MATCH_MAPPING] = { .nuops = 1, .uops = { { _MATCH_MAPPING, 0, 0 } } }, [MATCH_SEQUENCE] = { .nuops = 1, .uops = { { _MATCH_SEQUENCE, 0, 0 } } }, [NOP] = { .nuops = 1, .uops = { { _NOP, 0, 0 } } }, + [NOT_TAKEN] = { .nuops = 1, .uops = { { _NOP, 0, 0 } } }, [POP_EXCEPT] = { .nuops = 1, .uops = { { _POP_EXCEPT, 0, 0 } } }, + [POP_ITER] = { .nuops = 1, .uops = { { _POP_TOP, 0, 0 } } }, [POP_JUMP_IF_FALSE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_FALSE, 9, 1 } } }, [POP_JUMP_IF_NONE] = { .nuops = 2, .uops = { { _IS_NONE, 0, 0 }, { _POP_JUMP_IF_TRUE, 9, 1 } } }, [POP_JUMP_IF_NOT_NONE] = { .nuops = 2, .uops = { { _IS_NONE, 0, 0 }, { _POP_JUMP_IF_FALSE, 9, 1 } } }, @@ -1299,15 +2394,17 @@ _PyOpcode_macro_expansion[256] = { [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { _PUSH_EXC_INFO, 0, 0 } } }, [PUSH_NULL] = { .nuops = 1, .uops = { { _PUSH_NULL, 0, 0 } } }, [RESUME_CHECK] = { .nuops = 1, .uops = { { _RESUME_CHECK, 0, 0 } } }, - [RETURN_CONST] = { .nuops = 2, .uops = { { _LOAD_CONST, 0, 0 }, { _POP_FRAME, 0, 0 } } }, - [RETURN_VALUE] = { .nuops = 1, .uops = { { _POP_FRAME, 0, 0 } } }, + [RETURN_GENERATOR] = { .nuops = 1, .uops = { { _RETURN_GENERATOR, 0, 0 } } }, + [RETURN_VALUE] = { .nuops = 1, .uops = { { _RETURN_VALUE, 0, 0 } } }, + [SEND_GEN] = { .nuops = 3, .uops = { { _CHECK_PEP_523, 0, 0 }, { _SEND_GEN_FRAME, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, [SETUP_ANNOTATIONS] = { .nuops = 1, .uops = { { _SETUP_ANNOTATIONS, 0, 0 } } }, [SET_ADD] = { .nuops = 1, .uops = { { _SET_ADD, 0, 0 } } }, [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, 0, 0 } } }, [SET_UPDATE] = { .nuops = 1, .uops = { { _SET_UPDATE, 0, 0 } } }, [STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, 0, 0 } } }, - [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, + [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION_AND_LOCK, 2, 1 }, { _GUARD_DORV_NO_DICT, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, + [STORE_ATTR_WITH_HINT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_WITH_HINT, 1, 3 } } }, [STORE_DEREF] = { .nuops = 1, .uops = { { _STORE_DEREF, 0, 0 } } }, [STORE_FAST] = { .nuops = 1, .uops = { { _STORE_FAST, 0, 0 } } }, [STORE_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { _STORE_FAST, 5, 0 }, { _LOAD_FAST, 6, 0 } } }, @@ -1320,7 +2417,7 @@ _PyOpcode_macro_expansion[256] = { [STORE_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { _STORE_SUBSCR_LIST_INT, 0, 0 } } }, [SWAP] = { .nuops = 1, .uops = { { _SWAP, 0, 0 } } }, [TO_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL, 0, 0 } } }, - [TO_BOOL_ALWAYS_TRUE] = { .nuops = 1, .uops = { { _TO_BOOL_ALWAYS_TRUE, 2, 1 } } }, + [TO_BOOL_ALWAYS_TRUE] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _REPLACE_WITH_TRUE, 0, 0 } } }, [TO_BOOL_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL_BOOL, 0, 0 } } }, [TO_BOOL_INT] = { .nuops = 1, .uops = { { _TO_BOOL_INT, 0, 0 } } }, [TO_BOOL_LIST] = { .nuops = 1, .uops = { { _TO_BOOL_LIST, 0, 0 } } }, @@ -1335,18 +2432,18 @@ _PyOpcode_macro_expansion[256] = { [UNPACK_SEQUENCE_TUPLE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE_TUPLE, 0, 0 } } }, [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE_TWO_TUPLE, 0, 0 } } }, [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { _WITH_EXCEPT_START, 0, 0 } } }, + [YIELD_VALUE] = { .nuops = 1, .uops = { { _YIELD_VALUE, 0, 0 } } }, }; #endif // NEED_OPCODE_METADATA -extern const char *_PyOpcode_OpName[268]; +extern const char *_PyOpcode_OpName[266]; #ifdef NEED_OPCODE_METADATA -const char *_PyOpcode_OpName[268] = { - [BEFORE_ASYNC_WITH] = "BEFORE_ASYNC_WITH", - [BEFORE_WITH] = "BEFORE_WITH", +const char *_PyOpcode_OpName[266] = { [BINARY_OP] = "BINARY_OP", [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", + [BINARY_OP_EXTEND] = "BINARY_OP_EXTEND", [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", @@ -1359,7 +2456,6 @@ const char *_PyOpcode_OpName[268] = { [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", [BINARY_SUBSCR_STR_INT] = "BINARY_SUBSCR_STR_INT", [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", - [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_LIST] = "BUILD_LIST", [BUILD_MAP] = "BUILD_MAP", [BUILD_SET] = "BUILD_SET", @@ -1370,6 +2466,7 @@ const char *_PyOpcode_OpName[268] = { [CALL] = "CALL", [CALL_ALLOC_AND_ENTER_INIT] = "CALL_ALLOC_AND_ENTER_INIT", [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", + [CALL_BOUND_METHOD_GENERAL] = "CALL_BOUND_METHOD_GENERAL", [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [CALL_BUILTIN_FAST] = "CALL_BUILTIN_FAST", [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", @@ -1379,14 +2476,18 @@ const char *_PyOpcode_OpName[268] = { [CALL_INTRINSIC_2] = "CALL_INTRINSIC_2", [CALL_ISINSTANCE] = "CALL_ISINSTANCE", [CALL_KW] = "CALL_KW", + [CALL_KW_BOUND_METHOD] = "CALL_KW_BOUND_METHOD", + [CALL_KW_NON_PY] = "CALL_KW_NON_PY", + [CALL_KW_PY] = "CALL_KW_PY", [CALL_LEN] = "CALL_LEN", [CALL_LIST_APPEND] = "CALL_LIST_APPEND", [CALL_METHOD_DESCRIPTOR_FAST] = "CALL_METHOD_DESCRIPTOR_FAST", [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [CALL_METHOD_DESCRIPTOR_NOARGS] = "CALL_METHOD_DESCRIPTOR_NOARGS", [CALL_METHOD_DESCRIPTOR_O] = "CALL_METHOD_DESCRIPTOR_O", + [CALL_NON_PY_GENERAL] = "CALL_NON_PY_GENERAL", [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", - [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", + [CALL_PY_GENERAL] = "CALL_PY_GENERAL", [CALL_STR_1] = "CALL_STR_1", [CALL_TUPLE_1] = "CALL_TUPLE_1", [CALL_TYPE_1] = "CALL_TYPE_1", @@ -1398,6 +2499,8 @@ const char *_PyOpcode_OpName[268] = { [COMPARE_OP_INT] = "COMPARE_OP_INT", [COMPARE_OP_STR] = "COMPARE_OP_STR", [CONTAINS_OP] = "CONTAINS_OP", + [CONTAINS_OP_DICT] = "CONTAINS_OP_DICT", + [CONTAINS_OP_SET] = "CONTAINS_OP_SET", [CONVERT_VALUE] = "CONVERT_VALUE", [COPY] = "COPY", [COPY_FREE_VARS] = "COPY_FREE_VARS", @@ -1441,26 +2544,31 @@ const char *_PyOpcode_OpName[268] = { [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", [INSTRUMENTED_LOAD_SUPER_ATTR] = "INSTRUMENTED_LOAD_SUPER_ATTR", + [INSTRUMENTED_NOT_TAKEN] = "INSTRUMENTED_NOT_TAKEN", + [INSTRUMENTED_POP_ITER] = "INSTRUMENTED_POP_ITER", [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", - [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", [INSTRUMENTED_YIELD_VALUE] = "INSTRUMENTED_YIELD_VALUE", [INTERPRETER_EXIT] = "INTERPRETER_EXIT", [IS_OP] = "IS_OP", [JUMP] = "JUMP", [JUMP_BACKWARD] = "JUMP_BACKWARD", + [JUMP_BACKWARD_JIT] = "JUMP_BACKWARD_JIT", [JUMP_BACKWARD_NO_INTERRUPT] = "JUMP_BACKWARD_NO_INTERRUPT", + [JUMP_BACKWARD_NO_JIT] = "JUMP_BACKWARD_NO_JIT", [JUMP_FORWARD] = "JUMP_FORWARD", + [JUMP_IF_FALSE] = "JUMP_IF_FALSE", + [JUMP_IF_TRUE] = "JUMP_IF_TRUE", [JUMP_NO_INTERRUPT] = "JUMP_NO_INTERRUPT", [LIST_APPEND] = "LIST_APPEND", [LIST_EXTEND] = "LIST_EXTEND", - [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [LOAD_ATTR] = "LOAD_ATTR", [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = "LOAD_ATTR_CLASS_WITH_METACLASS_CHECK", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", @@ -1474,7 +2582,10 @@ const char *_PyOpcode_OpName[268] = { [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", [LOAD_CLOSURE] = "LOAD_CLOSURE", + [LOAD_COMMON_CONSTANT] = "LOAD_COMMON_CONSTANT", [LOAD_CONST] = "LOAD_CONST", + [LOAD_CONST_IMMORTAL] = "LOAD_CONST_IMMORTAL", + [LOAD_CONST_MORTAL] = "LOAD_CONST_MORTAL", [LOAD_DEREF] = "LOAD_DEREF", [LOAD_FAST] = "LOAD_FAST", [LOAD_FAST_AND_CLEAR] = "LOAD_FAST_AND_CLEAR", @@ -1486,14 +2597,12 @@ const char *_PyOpcode_OpName[268] = { [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [LOAD_LOCALS] = "LOAD_LOCALS", - [LOAD_METHOD] = "LOAD_METHOD", [LOAD_NAME] = "LOAD_NAME", + [LOAD_SMALL_INT] = "LOAD_SMALL_INT", + [LOAD_SPECIAL] = "LOAD_SPECIAL", [LOAD_SUPER_ATTR] = "LOAD_SUPER_ATTR", [LOAD_SUPER_ATTR_ATTR] = "LOAD_SUPER_ATTR_ATTR", [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", - [LOAD_SUPER_METHOD] = "LOAD_SUPER_METHOD", - [LOAD_ZERO_SUPER_ATTR] = "LOAD_ZERO_SUPER_ATTR", - [LOAD_ZERO_SUPER_METHOD] = "LOAD_ZERO_SUPER_METHOD", [MAKE_CELL] = "MAKE_CELL", [MAKE_FUNCTION] = "MAKE_FUNCTION", [MAP_ADD] = "MAP_ADD", @@ -1502,8 +2611,10 @@ const char *_PyOpcode_OpName[268] = { [MATCH_MAPPING] = "MATCH_MAPPING", [MATCH_SEQUENCE] = "MATCH_SEQUENCE", [NOP] = "NOP", + [NOT_TAKEN] = "NOT_TAKEN", [POP_BLOCK] = "POP_BLOCK", [POP_EXCEPT] = "POP_EXCEPT", + [POP_ITER] = "POP_ITER", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_NONE] = "POP_JUMP_IF_NONE", [POP_JUMP_IF_NOT_NONE] = "POP_JUMP_IF_NOT_NONE", @@ -1516,7 +2627,6 @@ const char *_PyOpcode_OpName[268] = { [RESERVED] = "RESERVED", [RESUME] = "RESUME", [RESUME_CHECK] = "RESUME_CHECK", - [RETURN_CONST] = "RETURN_CONST", [RETURN_GENERATOR] = "RETURN_GENERATOR", [RETURN_VALUE] = "RETURN_VALUE", [SEND] = "SEND", @@ -1567,7 +2677,6 @@ const char *_PyOpcode_OpName[268] = { extern const uint8_t _PyOpcode_Caches[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Caches[256] = { - [JUMP_BACKWARD] = 1, [TO_BOOL] = 3, [BINARY_SUBSCR] = 1, [STORE_SUBSCR] = 1, @@ -1578,25 +2687,27 @@ const uint8_t _PyOpcode_Caches[256] = { [LOAD_SUPER_ATTR] = 1, [LOAD_ATTR] = 9, [COMPARE_OP] = 1, + [CONTAINS_OP] = 1, + [JUMP_BACKWARD] = 1, [POP_JUMP_IF_TRUE] = 1, [POP_JUMP_IF_FALSE] = 1, [POP_JUMP_IF_NONE] = 1, [POP_JUMP_IF_NOT_NONE] = 1, [FOR_ITER] = 1, [CALL] = 3, - [BINARY_OP] = 1, + [CALL_KW] = 3, + [BINARY_OP] = 5, }; #endif extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { - [BEFORE_ASYNC_WITH] = BEFORE_ASYNC_WITH, - [BEFORE_WITH] = BEFORE_WITH, [BINARY_OP] = BINARY_OP, [BINARY_OP_ADD_FLOAT] = BINARY_OP, [BINARY_OP_ADD_INT] = BINARY_OP, [BINARY_OP_ADD_UNICODE] = BINARY_OP, + [BINARY_OP_EXTEND] = BINARY_OP, [BINARY_OP_INPLACE_ADD_UNICODE] = BINARY_OP, [BINARY_OP_MULTIPLY_FLOAT] = BINARY_OP, [BINARY_OP_MULTIPLY_INT] = BINARY_OP, @@ -1609,7 +2720,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [BINARY_SUBSCR_LIST_INT] = BINARY_SUBSCR, [BINARY_SUBSCR_STR_INT] = BINARY_SUBSCR, [BINARY_SUBSCR_TUPLE_INT] = BINARY_SUBSCR, - [BUILD_CONST_KEY_MAP] = BUILD_CONST_KEY_MAP, [BUILD_LIST] = BUILD_LIST, [BUILD_MAP] = BUILD_MAP, [BUILD_SET] = BUILD_SET, @@ -1620,6 +2730,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL] = CALL, [CALL_ALLOC_AND_ENTER_INIT] = CALL, [CALL_BOUND_METHOD_EXACT_ARGS] = CALL, + [CALL_BOUND_METHOD_GENERAL] = CALL, [CALL_BUILTIN_CLASS] = CALL, [CALL_BUILTIN_FAST] = CALL, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, @@ -1629,14 +2740,18 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_INTRINSIC_2] = CALL_INTRINSIC_2, [CALL_ISINSTANCE] = CALL, [CALL_KW] = CALL_KW, + [CALL_KW_BOUND_METHOD] = CALL_KW, + [CALL_KW_NON_PY] = CALL_KW, + [CALL_KW_PY] = CALL_KW, [CALL_LEN] = CALL, [CALL_LIST_APPEND] = CALL, [CALL_METHOD_DESCRIPTOR_FAST] = CALL, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, [CALL_METHOD_DESCRIPTOR_NOARGS] = CALL, [CALL_METHOD_DESCRIPTOR_O] = CALL, + [CALL_NON_PY_GENERAL] = CALL, [CALL_PY_EXACT_ARGS] = CALL, - [CALL_PY_WITH_DEFAULTS] = CALL, + [CALL_PY_GENERAL] = CALL, [CALL_STR_1] = CALL, [CALL_TUPLE_1] = CALL, [CALL_TYPE_1] = CALL, @@ -1648,6 +2763,8 @@ const uint8_t _PyOpcode_Deopt[256] = { [COMPARE_OP_INT] = COMPARE_OP, [COMPARE_OP_STR] = COMPARE_OP, [CONTAINS_OP] = CONTAINS_OP, + [CONTAINS_OP_DICT] = CONTAINS_OP, + [CONTAINS_OP_SET] = CONTAINS_OP, [CONVERT_VALUE] = CONVERT_VALUE, [COPY] = COPY, [COPY_FREE_VARS] = COPY_FREE_VARS, @@ -1691,24 +2808,27 @@ const uint8_t _PyOpcode_Deopt[256] = { [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, [INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR, + [INSTRUMENTED_NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN, + [INSTRUMENTED_POP_ITER] = INSTRUMENTED_POP_ITER, [INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, [INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, - [INSTRUMENTED_RETURN_CONST] = INSTRUMENTED_RETURN_CONST, [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, [INTERPRETER_EXIT] = INTERPRETER_EXIT, [IS_OP] = IS_OP, [JUMP_BACKWARD] = JUMP_BACKWARD, + [JUMP_BACKWARD_JIT] = JUMP_BACKWARD, [JUMP_BACKWARD_NO_INTERRUPT] = JUMP_BACKWARD_NO_INTERRUPT, + [JUMP_BACKWARD_NO_JIT] = JUMP_BACKWARD, [JUMP_FORWARD] = JUMP_FORWARD, [LIST_APPEND] = LIST_APPEND, [LIST_EXTEND] = LIST_EXTEND, - [LOAD_ASSERTION_ERROR] = LOAD_ASSERTION_ERROR, [LOAD_ATTR] = LOAD_ATTR, [LOAD_ATTR_CLASS] = LOAD_ATTR, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = LOAD_ATTR, [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = LOAD_ATTR, [LOAD_ATTR_INSTANCE_VALUE] = LOAD_ATTR, [LOAD_ATTR_METHOD_LAZY_DICT] = LOAD_ATTR, @@ -1721,7 +2841,10 @@ const uint8_t _PyOpcode_Deopt[256] = { [LOAD_ATTR_SLOT] = LOAD_ATTR, [LOAD_ATTR_WITH_HINT] = LOAD_ATTR, [LOAD_BUILD_CLASS] = LOAD_BUILD_CLASS, + [LOAD_COMMON_CONSTANT] = LOAD_COMMON_CONSTANT, [LOAD_CONST] = LOAD_CONST, + [LOAD_CONST_IMMORTAL] = LOAD_CONST, + [LOAD_CONST_MORTAL] = LOAD_CONST, [LOAD_DEREF] = LOAD_DEREF, [LOAD_FAST] = LOAD_FAST, [LOAD_FAST_AND_CLEAR] = LOAD_FAST_AND_CLEAR, @@ -1734,6 +2857,8 @@ const uint8_t _PyOpcode_Deopt[256] = { [LOAD_GLOBAL_MODULE] = LOAD_GLOBAL, [LOAD_LOCALS] = LOAD_LOCALS, [LOAD_NAME] = LOAD_NAME, + [LOAD_SMALL_INT] = LOAD_SMALL_INT, + [LOAD_SPECIAL] = LOAD_SPECIAL, [LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR, [LOAD_SUPER_ATTR_ATTR] = LOAD_SUPER_ATTR, [LOAD_SUPER_ATTR_METHOD] = LOAD_SUPER_ATTR, @@ -1745,7 +2870,9 @@ const uint8_t _PyOpcode_Deopt[256] = { [MATCH_MAPPING] = MATCH_MAPPING, [MATCH_SEQUENCE] = MATCH_SEQUENCE, [NOP] = NOP, + [NOT_TAKEN] = NOT_TAKEN, [POP_EXCEPT] = POP_EXCEPT, + [POP_ITER] = POP_ITER, [POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE, [POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE, [POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE, @@ -1758,7 +2885,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [RESERVED] = RESERVED, [RESUME] = RESUME, [RESUME_CHECK] = RESUME, - [RETURN_CONST] = RETURN_CONST, [RETURN_GENERATOR] = RETURN_GENERATOR, [RETURN_VALUE] = RETURN_VALUE, [SEND] = SEND, @@ -1804,6 +2930,7 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ + case 118: \ case 119: \ case 120: \ case 121: \ @@ -1834,49 +2961,33 @@ const uint8_t _PyOpcode_Deopt[256] = { case 146: \ case 147: \ case 148: \ - case 219: \ - case 220: \ - case 221: \ - case 222: \ - case 223: \ - case 224: \ - case 225: \ - case 226: \ - case 227: \ - case 228: \ - case 229: \ - case 230: \ - case 231: \ case 232: \ case 233: \ case 234: \ - case 235: \ - case 255: \ ; struct pseudo_targets { - uint8_t targets[3]; + uint8_t as_sequence; + uint8_t targets[4]; }; -extern const struct pseudo_targets _PyOpcode_PseudoTargets[12]; +extern const struct pseudo_targets _PyOpcode_PseudoTargets[10]; #ifdef NEED_OPCODE_METADATA -const struct pseudo_targets _PyOpcode_PseudoTargets[12] = { - [LOAD_CLOSURE-256] = { { LOAD_FAST, 0, 0 } }, - [STORE_FAST_MAYBE_NULL-256] = { { STORE_FAST, 0, 0 } }, - [LOAD_SUPER_METHOD-256] = { { LOAD_SUPER_ATTR, 0, 0 } }, - [LOAD_ZERO_SUPER_METHOD-256] = { { LOAD_SUPER_ATTR, 0, 0 } }, - [LOAD_ZERO_SUPER_ATTR-256] = { { LOAD_SUPER_ATTR, 0, 0 } }, - [LOAD_METHOD-256] = { { LOAD_ATTR, 0, 0 } }, - [JUMP-256] = { { JUMP_FORWARD, JUMP_BACKWARD, 0 } }, - [JUMP_NO_INTERRUPT-256] = { { JUMP_FORWARD, JUMP_BACKWARD_NO_INTERRUPT, 0 } }, - [SETUP_FINALLY-256] = { { NOP, 0, 0 } }, - [SETUP_CLEANUP-256] = { { NOP, 0, 0 } }, - [SETUP_WITH-256] = { { NOP, 0, 0 } }, - [POP_BLOCK-256] = { { NOP, 0, 0 } }, +const struct pseudo_targets _PyOpcode_PseudoTargets[10] = { + [LOAD_CLOSURE-256] = { 0, { LOAD_FAST, 0, 0, 0 } }, + [STORE_FAST_MAYBE_NULL-256] = { 0, { STORE_FAST, 0, 0, 0 } }, + [JUMP-256] = { 0, { JUMP_FORWARD, JUMP_BACKWARD, 0, 0 } }, + [JUMP_NO_INTERRUPT-256] = { 0, { JUMP_FORWARD, JUMP_BACKWARD_NO_INTERRUPT, 0, 0 } }, + [JUMP_IF_FALSE-256] = { 1, { COPY, TO_BOOL, POP_JUMP_IF_FALSE, 0 } }, + [JUMP_IF_TRUE-256] = { 1, { COPY, TO_BOOL, POP_JUMP_IF_TRUE, 0 } }, + [SETUP_FINALLY-256] = { 0, { NOP, 0, 0, 0 } }, + [SETUP_CLEANUP-256] = { 0, { NOP, 0, 0, 0 } }, + [SETUP_WITH-256] = { 0, { NOP, 0, 0, 0 } }, + [POP_BLOCK-256] = { 0, { NOP, 0, 0, 0 } }, }; #endif // NEED_OPCODE_METADATA static inline bool is_pseudo_target(int pseudo, int target) { - if (pseudo < 256 || pseudo >= 268) { + if (pseudo < 256 || pseudo >= 266) { return false; } for (int i = 0; _PyOpcode_PseudoTargets[pseudo-256].targets[i]; i++) { diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index 208bfb2f75308b..0872231d1f2d11 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -45,9 +45,14 @@ extern "C" { (opcode) == JUMP_BACKWARD || \ (opcode) == JUMP_BACKWARD_NO_INTERRUPT) +#define IS_CONDITIONAL_JUMP_OPCODE(opcode) \ + ((opcode) == POP_JUMP_IF_FALSE || \ + (opcode) == POP_JUMP_IF_TRUE || \ + (opcode) == POP_JUMP_IF_NONE || \ + (opcode) == POP_JUMP_IF_NOT_NONE) + #define IS_SCOPE_EXIT_OPCODE(opcode) \ ((opcode) == RETURN_VALUE || \ - (opcode) == RETURN_CONST || \ (opcode) == RAISE_VARARGS || \ (opcode) == RERAISE) @@ -57,6 +62,12 @@ extern "C" { #define MAKE_FUNCTION_KWDEFAULTS 0x02 #define MAKE_FUNCTION_ANNOTATIONS 0x04 #define MAKE_FUNCTION_CLOSURE 0x08 +#define MAKE_FUNCTION_ANNOTATE 0x10 + +/* Values used as the oparg for LOAD_COMMON_CONSTANT */ +#define CONSTANT_ASSERTIONERROR 0 +#define CONSTANT_NOTIMPLEMENTEDERROR 1 +#define NUM_COMMON_CONSTANTS 2 /* Values used in the oparg for RESUME */ #define RESUME_AT_FUNC_START 0 diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index d32e6c0174f680..e806e306d2d57f 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -11,29 +11,139 @@ extern "C" { #include "pycore_uop_ids.h" #include + +typedef struct _PyExecutorLinkListNode { + struct _PyExecutorObject *next; + struct _PyExecutorObject *previous; +} _PyExecutorLinkListNode; + + +/* Bloom filter with m = 256 + * https://en.wikipedia.org/wiki/Bloom_filter */ +#define _Py_BLOOM_FILTER_WORDS 8 + +typedef struct { + uint32_t bits[_Py_BLOOM_FILTER_WORDS]; +} _PyBloomFilter; + +typedef struct { + uint8_t opcode; + uint8_t oparg; + uint8_t valid:1; + uint8_t linked:1; + uint8_t chain_depth:6; // Must be big enough for MAX_CHAIN_DEPTH - 1. + bool warm; + int index; // Index of ENTER_EXECUTOR (if code isn't NULL, below). + _PyBloomFilter bloom; + _PyExecutorLinkListNode links; + PyCodeObject *code; // Weak (NULL if no corresponding ENTER_EXECUTOR). +} _PyVMData; + +/* Depending on the format, + * the 32 bits between the oparg and operand are: + * UOP_FORMAT_TARGET: + * uint32_t target; + * UOP_FORMAT_JUMP + * uint16_t jump_target; + * uint16_t error_target; + */ +typedef struct { + uint16_t opcode:15; + uint16_t format:1; + uint16_t oparg; + union { + uint32_t target; + struct { + uint16_t jump_target; + uint16_t error_target; + }; + }; + uint64_t operand0; // A cache entry + uint64_t operand1; +#ifdef Py_STATS + uint64_t execution_count; +#endif +} _PyUOpInstruction; + +typedef struct { + uint32_t target; + _Py_BackoffCounter temperature; + const struct _PyExecutorObject *executor; +} _PyExitData; + +typedef struct _PyExecutorObject { + PyObject_VAR_HEAD + const _PyUOpInstruction *trace; + _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ + uint32_t exit_count; + uint32_t code_size; + size_t jit_size; + void *jit_code; + void *jit_side_entry; + _PyExitData exits[1]; +} _PyExecutorObject; + + +// Export for '_opcode' shared extension (JIT compiler). +PyAPI_FUNC(_PyExecutorObject*) _Py_GetExecutor(PyCodeObject *code, int offset); + +void _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *); +void _Py_ExecutorDetach(_PyExecutorObject *); +void _Py_BloomFilter_Init(_PyBloomFilter *); +void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); +PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); + +#define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3 +#define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6 + +#ifdef _Py_TIER2 +PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation); +PyAPI_FUNC(void) _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation); +PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); + +#else +# define _Py_Executors_InvalidateDependency(A, B, C) ((void)0) +# define _Py_Executors_InvalidateAll(A, B) ((void)0) +# define _Py_Executors_InvalidateCold(A) ((void)0) + +#endif + +// Used as the threshold to trigger executor invalidation when +// trace_run_counter is greater than this value. +#define JIT_CLEANUP_THRESHOLD 100000 + // This is the length of the trace we project initially. -#define UOP_MAX_TRACE_LENGTH 512 +#define UOP_MAX_TRACE_LENGTH 800 #define TRACE_STACK_SIZE 5 -int _Py_uop_analyze_and_optimize(_PyInterpreterFrame *frame, +int _Py_uop_analyze_and_optimize(struct _PyInterpreterFrame *frame, _PyUOpInstruction *trace, int trace_len, int curr_stackentries, _PyBloomFilter *dependencies); -extern PyTypeObject _PyCounterExecutor_Type; -extern PyTypeObject _PyCounterOptimizer_Type; -extern PyTypeObject _PyDefaultOptimizer_Type; extern PyTypeObject _PyUOpExecutor_Type; -extern PyTypeObject _PyUOpOptimizer_Type; -/* Symbols */ -/* See explanation in optimizer_symbols.c */ -struct _Py_UopsSymbol { - int flags; // 0 bits: Top; 2 or more bits: Bottom - PyTypeObject *typ; // Borrowed reference - PyObject *const_val; // Owned reference (!) -}; +#define UOP_FORMAT_TARGET 0 +#define UOP_FORMAT_JUMP 1 + +static inline uint32_t uop_get_target(const _PyUOpInstruction *inst) +{ + assert(inst->format == UOP_FORMAT_TARGET); + return inst->target; +} + +static inline uint16_t uop_get_jump_target(const _PyUOpInstruction *inst) +{ + assert(inst->format == UOP_FORMAT_JUMP); + return inst->jump_target; +} + +static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) +{ + assert(inst->format != UOP_FORMAT_TARGET); + return inst->error_target; +} // Holds locals, stack, locals, stack ... co_consts (in that order) #define MAX_ABSTRACT_INTERP_SIZE 4096 @@ -43,16 +153,69 @@ struct _Py_UopsSymbol { // Need extras for root frame and for overflow frame (see TRACE_STACK_PUSH()) #define MAX_ABSTRACT_FRAME_DEPTH (TRACE_STACK_SIZE + 2) -typedef struct _Py_UopsSymbol _Py_UopsSymbol; +// The maximum number of side exits that we can take before requiring forward +// progress (and inserting a new ENTER_EXECUTOR instruction). In practice, this +// is the "maximum amount of polymorphism" that an isolated trace tree can +// handle before rejoining the rest of the program. +#define MAX_CHAIN_DEPTH 4 + +/* Symbols */ +/* See explanation in optimizer_symbols.c */ + + +typedef enum _JitSymType { + JIT_SYM_UNKNOWN_TAG = 1, + JIT_SYM_NULL_TAG = 2, + JIT_SYM_NON_NULL_TAG = 3, + JIT_SYM_BOTTOM_TAG = 4, + JIT_SYM_TYPE_VERSION_TAG = 5, + JIT_SYM_KNOWN_CLASS_TAG = 6, + JIT_SYM_KNOWN_VALUE_TAG = 7, + JIT_SYM_TUPLE_TAG = 8, +} JitSymType; + +typedef struct _jit_opt_known_class { + uint8_t tag; + uint32_t version; + PyTypeObject *type; +} JitOptKnownClass; + +typedef struct _jit_opt_known_version { + uint8_t tag; + uint32_t version; +} JitOptKnownVersion; + +typedef struct _jit_opt_known_value { + uint8_t tag; + PyObject *value; +} JitOptKnownValue; + +#define MAX_SYMBOLIC_TUPLE_SIZE 7 + +typedef struct _jit_opt_tuple { + uint8_t tag; + uint8_t length; + uint16_t items[MAX_SYMBOLIC_TUPLE_SIZE]; +} JitOptTuple; + +typedef union _jit_opt_symbol { + uint8_t tag; + JitOptKnownClass cls; + JitOptKnownValue value; + JitOptKnownVersion version; + JitOptTuple tuple; +} JitOptSymbol; + + struct _Py_UOpsAbstractFrame { // Max stacklen int stack_len; int locals_len; - _Py_UopsSymbol **stack_pointer; - _Py_UopsSymbol **stack; - _Py_UopsSymbol **locals; + JitOptSymbol **stack_pointer; + JitOptSymbol **stack; + JitOptSymbol **locals; }; typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; @@ -60,11 +223,13 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; typedef struct ty_arena { int ty_curr_number; int ty_max_number; - _Py_UopsSymbol arena[TY_ARENA_SIZE]; + JitOptSymbol arena[TY_ARENA_SIZE]; } ty_arena; -struct _Py_UOpsContext { - PyObject_HEAD +typedef struct _JitOptContext { + char done; + char out_of_space; + bool contradiction; // The current "executing" frame. _Py_UOpsAbstractFrame *frame; _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH]; @@ -73,45 +238,63 @@ struct _Py_UOpsContext { // Arena for the symbolic types. ty_arena t_arena; - _Py_UopsSymbol **n_consumed; - _Py_UopsSymbol **limit; - _Py_UopsSymbol *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; -}; + JitOptSymbol **n_consumed; + JitOptSymbol **limit; + JitOptSymbol *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; +} JitOptContext; + +extern bool _Py_uop_sym_is_null(JitOptSymbol *sym); +extern bool _Py_uop_sym_is_not_null(JitOptSymbol *sym); +extern bool _Py_uop_sym_is_const(JitOptSymbol *sym); +extern PyObject *_Py_uop_sym_get_const(JitOptSymbol *sym); +extern JitOptSymbol *_Py_uop_sym_new_unknown(JitOptContext *ctx); +extern JitOptSymbol *_Py_uop_sym_new_not_null(JitOptContext *ctx); +extern JitOptSymbol *_Py_uop_sym_new_type( + JitOptContext *ctx, PyTypeObject *typ); +extern JitOptSymbol *_Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val); +extern JitOptSymbol *_Py_uop_sym_new_null(JitOptContext *ctx); +extern bool _Py_uop_sym_has_type(JitOptSymbol *sym); +extern bool _Py_uop_sym_matches_type(JitOptSymbol *sym, PyTypeObject *typ); +extern bool _Py_uop_sym_matches_type_version(JitOptSymbol *sym, unsigned int version); +extern void _Py_uop_sym_set_null(JitOptContext *ctx, JitOptSymbol *sym); +extern void _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym); +extern void _Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeObject *typ); +extern bool _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int version); +extern void _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val); +extern bool _Py_uop_sym_is_bottom(JitOptSymbol *sym); +extern int _Py_uop_sym_truthiness(JitOptSymbol *sym); +extern PyTypeObject *_Py_uop_sym_get_type(JitOptSymbol *sym); +extern bool _Py_uop_sym_is_immortal(JitOptSymbol *sym); +extern JitOptSymbol *_Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptSymbol **args); +extern JitOptSymbol *_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol *sym, int item); +extern int _Py_uop_sym_tuple_length(JitOptSymbol *sym); -typedef struct _Py_UOpsContext _Py_UOpsContext; - -extern bool _Py_uop_sym_is_null(_Py_UopsSymbol *sym); -extern bool _Py_uop_sym_is_not_null(_Py_UopsSymbol *sym); -extern bool _Py_uop_sym_is_const(_Py_UopsSymbol *sym); -extern PyObject *_Py_uop_sym_get_const(_Py_UopsSymbol *sym); -extern _Py_UopsSymbol *_Py_uop_sym_new_unknown(_Py_UOpsContext *ctx); -extern _Py_UopsSymbol *_Py_uop_sym_new_not_null(_Py_UOpsContext *ctx); -extern _Py_UopsSymbol *_Py_uop_sym_new_type( - _Py_UOpsContext *ctx, PyTypeObject *typ); -extern _Py_UopsSymbol *_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val); -extern _Py_UopsSymbol *_Py_uop_sym_new_null(_Py_UOpsContext *ctx); -extern bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ); -extern bool _Py_uop_sym_set_null(_Py_UopsSymbol *sym); -extern bool _Py_uop_sym_set_non_null(_Py_UopsSymbol *sym); -extern bool _Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *typ); -extern bool _Py_uop_sym_set_const(_Py_UopsSymbol *sym, PyObject *const_val); -extern bool _Py_uop_sym_is_bottom(_Py_UopsSymbol *sym); - - -extern int _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx); -extern void _Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx); +extern void _Py_uop_abstractcontext_init(JitOptContext *ctx); +extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx); extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( - _Py_UOpsContext *ctx, + JitOptContext *ctx, PyCodeObject *co, - _Py_UopsSymbol **localsplus_start, - int n_locals_already_filled, - int curr_stackentries); -extern int _Py_uop_frame_pop(_Py_UOpsContext *ctx); + int curr_stackentries, + JitOptSymbol **args, + int arg_len); +extern int _Py_uop_frame_pop(JitOptContext *ctx); PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); -PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer, _PyExecutorObject **exec_ptr); +PyAPI_FUNC(int) _PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyStackRef *stack_pointer, _PyExecutorObject **exec_ptr, int chain_depth); + +static inline int is_terminator(const _PyUOpInstruction *uop) +{ + int opcode = uop->opcode; + return ( + opcode == _EXIT_TRACE || + opcode == _JUMP_TO_TOP || + opcode == _DYNAMIC_EXIT + ); +} + +PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); #ifdef __cplusplus } diff --git a/Include/internal/pycore_parser.h b/Include/internal/pycore_parser.h index 067b34c12c4e7f..b16084aaa15515 100644 --- a/Include/internal/pycore_parser.h +++ b/Include/internal/pycore_parser.h @@ -21,6 +21,9 @@ extern "C" { struct _parser_runtime_state { #ifdef Py_DEBUG long memo_statistics[_PYPEGEN_NSTATISTICS]; +#ifdef Py_GIL_DISABLED + PyMutex mutex; +#endif #else int _not_used; #endif @@ -28,8 +31,10 @@ struct _parser_runtime_state { }; _Py_DECLARE_STR(empty, "") +#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED) #define _parser_runtime_state_INIT \ { \ + .mutex = {0}, \ .dummy_name = { \ .kind = Name_kind, \ .v.Name.id = &_Py_STR(empty), \ @@ -40,6 +45,20 @@ _Py_DECLARE_STR(empty, "") .end_col_offset = 0, \ }, \ } +#else +#define _parser_runtime_state_INIT \ + { \ + .dummy_name = { \ + .kind = Name_kind, \ + .v.Name.id = &_Py_STR(empty), \ + .v.Name.ctx = Load, \ + .lineno = 1, \ + .col_offset = 0, \ + .end_lineno = 1, \ + .end_col_offset = 0, \ + }, \ + } +#endif extern struct _mod* _PyParser_ASTFromString( const char *str, diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index e441600d54e1aa..d755d03a5fa190 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -20,21 +20,143 @@ extern "C" { #endif #ifdef Py_GIL_DISABLED +#define FT_ATOMIC_LOAD_PTR(value) _Py_atomic_load_ptr(&value) +#define FT_ATOMIC_STORE_PTR(value, new_value) _Py_atomic_store_ptr(&value, new_value) #define FT_ATOMIC_LOAD_SSIZE(value) _Py_atomic_load_ssize(&value) +#define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) \ + _Py_atomic_load_ssize_acquire(&value) #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) \ _Py_atomic_load_ssize_relaxed(&value) +#define FT_ATOMIC_STORE_PTR(value, new_value) \ + _Py_atomic_store_ptr(&value, new_value) +#define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) \ + _Py_atomic_load_ptr_acquire(&value) +#define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) \ + _Py_atomic_load_uintptr_acquire(&value) +#define FT_ATOMIC_LOAD_PTR_RELAXED(value) \ + _Py_atomic_load_ptr_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT8(value) \ + _Py_atomic_load_uint8(&value) +#define FT_ATOMIC_STORE_UINT8(value, new_value) \ + _Py_atomic_store_uint8(&value, new_value) +#define FT_ATOMIC_LOAD_UINT8_RELAXED(value) \ + _Py_atomic_load_uint8_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT16_RELAXED(value) \ + _Py_atomic_load_uint16_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT32_RELAXED(value) \ + _Py_atomic_load_uint32_relaxed(&value) +#define FT_ATOMIC_LOAD_ULONG_RELAXED(value) \ + _Py_atomic_load_ulong_relaxed(&value) #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \ _Py_atomic_store_ptr_relaxed(&value, new_value) #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) \ _Py_atomic_store_ptr_release(&value, new_value) +#define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) \ + _Py_atomic_store_uintptr_release(&value, new_value) #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) \ _Py_atomic_store_ssize_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) \ + _Py_atomic_store_uint8_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) \ + _Py_atomic_store_uint16_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) \ + _Py_atomic_store_uint32_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) \ + _Py_atomic_store_char_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_CHAR_RELAXED(value) \ + _Py_atomic_load_char_relaxed(&value) +#define FT_ATOMIC_STORE_UCHAR_RELAXED(value, new_value) \ + _Py_atomic_store_uchar_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) \ + _Py_atomic_load_uchar_relaxed(&value) +#define FT_ATOMIC_STORE_SHORT_RELAXED(value, new_value) \ + _Py_atomic_store_short_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_SHORT_RELAXED(value) \ + _Py_atomic_load_short_relaxed(&value) +#define FT_ATOMIC_STORE_USHORT_RELAXED(value, new_value) \ + _Py_atomic_store_ushort_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_USHORT_RELAXED(value) \ + _Py_atomic_load_ushort_relaxed(&value) +#define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) \ + _Py_atomic_store_int_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_INT_RELAXED(value) \ + _Py_atomic_load_int_relaxed(&value) +#define FT_ATOMIC_STORE_UINT_RELAXED(value, new_value) \ + _Py_atomic_store_uint_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_UINT_RELAXED(value) \ + _Py_atomic_load_uint_relaxed(&value) +#define FT_ATOMIC_STORE_LONG_RELAXED(value, new_value) \ + _Py_atomic_store_long_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_LONG_RELAXED(value) \ + _Py_atomic_load_long_relaxed(&value) +#define FT_ATOMIC_STORE_ULONG_RELAXED(value, new_value) \ + _Py_atomic_store_ulong_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) \ + _Py_atomic_store_ssize_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_FLOAT_RELAXED(value, new_value) \ + _Py_atomic_store_float_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) \ + _Py_atomic_load_float_relaxed(&value) +#define FT_ATOMIC_STORE_DOUBLE_RELAXED(value, new_value) \ + _Py_atomic_store_double_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_DOUBLE_RELAXED(value) \ + _Py_atomic_load_double_relaxed(&value) +#define FT_ATOMIC_STORE_LLONG_RELAXED(value, new_value) \ + _Py_atomic_store_llong_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_LLONG_RELAXED(value) \ + _Py_atomic_load_llong_relaxed(&value) +#define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) \ + _Py_atomic_store_ullong_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) \ + _Py_atomic_load_ullong_relaxed(&value) + #else +#define FT_ATOMIC_LOAD_PTR(value) value +#define FT_ATOMIC_STORE_PTR(value, new_value) value = new_value #define FT_ATOMIC_LOAD_SSIZE(value) value +#define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) value #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value +#define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) value +#define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) value +#define FT_ATOMIC_LOAD_PTR_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT8(value) value +#define FT_ATOMIC_STORE_UINT8(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_UINT8_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT16_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT32_RELAXED(value) value +#define FT_ATOMIC_LOAD_ULONG_RELAXED(value) value #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_CHAR_RELAXED(value) value +#define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) value +#define FT_ATOMIC_STORE_UCHAR_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_SHORT_RELAXED(value) value +#define FT_ATOMIC_STORE_SHORT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_USHORT_RELAXED(value) value +#define FT_ATOMIC_STORE_USHORT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_INT_RELAXED(value) value +#define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_UINT_RELAXED(value) value +#define FT_ATOMIC_STORE_UINT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_LONG_RELAXED(value) value +#define FT_ATOMIC_STORE_LONG_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_ULONG_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) value +#define FT_ATOMIC_STORE_FLOAT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_DOUBLE_RELAXED(value) value +#define FT_ATOMIC_STORE_DOUBLE_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_LLONG_RELAXED(value) value +#define FT_ATOMIC_STORE_LLONG_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) value +#define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value + #endif #ifdef __cplusplus diff --git a/Include/internal/pycore_pybuffer.h b/Include/internal/pycore_pybuffer.h index 3cbc290b2ea3ee..9439d2bd770587 100644 --- a/Include/internal/pycore_pybuffer.h +++ b/Include/internal/pycore_pybuffer.h @@ -9,7 +9,7 @@ extern "C" { #endif -// Exported for the _xxinterpchannels module. +// Exported for the _interpchannels module. PyAPI_FUNC(int) _PyBuffer_ReleaseInInterpreter( PyInterpreterState *interp, Py_buffer *view); PyAPI_FUNC(int) _PyBuffer_ReleaseInInterpreterAndRawFree( diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 910335fd2cf33b..fa7d9ee36d095d 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -120,11 +120,28 @@ extern void _PyErr_SetNone(PyThreadState *tstate, PyObject *exception); extern PyObject* _PyErr_NoMemory(PyThreadState *tstate); +extern int _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset, + int end_lineno, int end_col_offset); +extern void _PyErr_RaiseSyntaxError(PyObject *msg, PyObject *filename, int lineno, int col_offset, + int end_lineno, int end_col_offset); + PyAPI_FUNC(void) _PyErr_SetString( PyThreadState *tstate, PyObject *exception, const char *string); +/* + * Set an exception with the error message decoded from the current locale + * encoding (LC_CTYPE). + * + * Exceptions occurring in decoding take priority over the desired exception. + * + * Exported for '_ctypes' shared extensions. + */ +PyAPI_FUNC(void) _PyErr_SetLocaleString( + PyObject *exception, + const char *string); + PyAPI_FUNC(PyObject*) _PyErr_Format( PyThreadState *tstate, PyObject *exception, @@ -161,14 +178,26 @@ extern PyObject* _Py_Offer_Suggestions(PyObject* exception); PyAPI_FUNC(Py_ssize_t) _Py_UTF8_Edit_Cost(PyObject *str_a, PyObject *str_b, Py_ssize_t max_cost); -void _PyErr_FormatNote(const char *format, ...); +// Export for '_json' shared extension +PyAPI_FUNC(void) _PyErr_FormatNote(const char *format, ...); /* Context manipulation (PEP 3134) */ Py_DEPRECATED(3.12) extern void _PyErr_ChainExceptions(PyObject *, PyObject *, PyObject *); -// Export for '_zoneinfo' shared extension -PyAPI_FUNC(void) _PyErr_ChainExceptions1(PyObject *); +// implementation detail for the codeop module. +// Exported for test.test_peg_generator.test_c_parser +PyAPI_DATA(PyTypeObject) _PyExc_IncompleteInputError; +#define PyExc_IncompleteInputError ((PyObject *)(&_PyExc_IncompleteInputError)) + +extern int _PyUnicodeError_GetParams( + PyObject *self, + PyObject **obj, + Py_ssize_t *objlen, + Py_ssize_t *start, + Py_ssize_t *end, + Py_ssize_t *slen, + int as_bytes); #ifdef __cplusplus } diff --git a/Include/internal/pycore_pyhash.h b/Include/internal/pycore_pyhash.h index 0ce08900e96f0b..9414e7761171d2 100644 --- a/Include/internal/pycore_pyhash.h +++ b/Include/internal/pycore_pyhash.h @@ -20,9 +20,6 @@ _Py_HashPointerRaw(const void *ptr) return (Py_hash_t)x; } -// Export for '_datetime' shared extension -PyAPI_FUNC(Py_hash_t) _Py_HashBytes(const void*, Py_ssize_t); - /* Hash secret * * memory layout on 64 bit systems diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index c675098685764c..f426ae0e103b9c 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -42,7 +42,6 @@ extern PyStatus _Py_HashRandomization_Init(const PyConfig *); extern PyStatus _PyGC_Init(PyInterpreterState *interp); extern PyStatus _PyAtExit_Init(PyInterpreterState *interp); -extern int _Py_Deepfreeze_Init(void); /* Various internal finalizers */ @@ -58,7 +57,6 @@ extern void _PyWarnings_Fini(PyInterpreterState *interp); extern void _PyAST_Fini(PyInterpreterState *interp); extern void _PyAtExit_Fini(PyInterpreterState *interp); extern void _PyThread_FiniType(PyInterpreterState *interp); -extern void _Py_Deepfreeze_Fini(void); extern void _PyArg_Fini(void); extern void _Py_FinalizeAllocatedBlocks(_PyRuntimeState *); @@ -116,6 +114,22 @@ PyAPI_FUNC(char*) _Py_SetLocaleFromEnv(int category); // Export for special main.c string compiling with source tracebacks int _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompilerFlags *flags); + +/* interpreter config */ + +// Export for _testinternalcapi shared extension +PyAPI_FUNC(int) _PyInterpreterConfig_InitFromState( + PyInterpreterConfig *, + PyInterpreterState *); +PyAPI_FUNC(PyObject *) _PyInterpreterConfig_AsDict(PyInterpreterConfig *); +PyAPI_FUNC(int) _PyInterpreterConfig_InitFromDict( + PyInterpreterConfig *, + PyObject *); +PyAPI_FUNC(int) _PyInterpreterConfig_UpdateFromDict( + PyInterpreterConfig *, + PyObject *); + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pymath.h b/Include/internal/pycore_pymath.h index 7a4e1c1eb714f7..eea8996ba68ca0 100644 --- a/Include/internal/pycore_pymath.h +++ b/Include/internal/pycore_pymath.h @@ -33,7 +33,7 @@ extern "C" { static inline void _Py_ADJUST_ERANGE1(double x) { if (errno == 0) { - if (x == Py_HUGE_VAL || x == -Py_HUGE_VAL) { + if (x == Py_INFINITY || x == -Py_INFINITY) { errno = ERANGE; } } @@ -44,8 +44,8 @@ static inline void _Py_ADJUST_ERANGE1(double x) static inline void _Py_ADJUST_ERANGE2(double x, double y) { - if (x == Py_HUGE_VAL || x == -Py_HUGE_VAL || - y == Py_HUGE_VAL || y == -Py_HUGE_VAL) + if (x == Py_INFINITY || x == -Py_INFINITY || + y == Py_INFINITY || y == -Py_INFINITY) { if (errno == 0) { errno = ERANGE; diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 1aea91abc5d69f..5bb34001aab1b4 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -119,9 +119,26 @@ extern int _PyMem_DebugEnabled(void); // Enqueue a pointer to be freed possibly after some delay. extern void _PyMem_FreeDelayed(void *ptr); +// Enqueue an object to be freed possibly after some delay +#ifdef Py_GIL_DISABLED +extern void _PyObject_XDecRefDelayed(PyObject *obj); +#else +static inline void _PyObject_XDecRefDelayed(PyObject *obj) +{ + Py_XDECREF(obj); +} +#endif + // Periodically process delayed free requests. extern void _PyMem_ProcessDelayed(PyThreadState *tstate); + +// Periodically process delayed free requests when the world is stopped. +// Notify of any objects whic should be freeed. +typedef void (*delayed_dealloc_cb)(PyObject *, void *); +extern void _PyMem_ProcessDelayedNoDealloc(PyThreadState *tstate, + delayed_dealloc_cb cb, void *state); + // Abandon all thread-local delayed free requests and push them to the // interpreter's queue. extern void _PyMem_AbandonDelayed(PyThreadState *tstate); diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 6f9e6a332a7830..ff3b222b157810 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -8,11 +8,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState #include "pycore_runtime.h" // _PyRuntime #include "pycore_tstate.h" // _PyThreadStateImpl - // Values for PyThreadState.state. A thread must be in the "attached" state // before calling most Python APIs. If the GIL is enabled, then "attached" // implies that the thread holds the GIL and "detached" implies that the @@ -77,11 +75,17 @@ _Py_IsMainInterpreterFinalizing(PyInterpreterState *interp) interp == &_PyRuntime._main_interpreter); } -// Export for _xxsubinterpreters module. +// Export for _interpreters module. +PyAPI_FUNC(PyObject *) _PyInterpreterState_GetIDObject(PyInterpreterState *); + +// Export for _interpreters module. PyAPI_FUNC(int) _PyInterpreterState_SetRunningMain(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_SetNotRunningMain(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_IsRunningMain(PyInterpreterState *); -PyAPI_FUNC(int) _PyInterpreterState_FailIfRunningMain(PyInterpreterState *); +PyAPI_FUNC(void) _PyErr_SetInterpreterAlreadyRunning(void); + +extern int _PyThreadState_IsRunningMain(PyThreadState *); +extern void _PyInterpreterState_ReinitRunningMain(PyThreadState *); static inline const PyConfig * @@ -137,6 +141,12 @@ _PyThreadState_GET(void) #endif } +static inline int +_PyThreadState_IsAttached(PyThreadState *tstate) +{ + return (_Py_atomic_load_int_relaxed(&tstate->state) == _Py_THREAD_ATTACHED); +} + // Attaches the current thread to the interpreter. // // This may block while acquiring the GIL (if the GIL is enabled) or while @@ -180,10 +190,18 @@ static inline void _Py_EnsureFuncTstateNotNULL(const char *func, PyThreadState *tstate) { if (tstate == NULL) { +#ifndef Py_GIL_DISABLED _Py_FatalErrorFunc(func, "the function must be called with the GIL held, " "after Python initialization and before Python finalization, " "but the GIL is released (the current Python thread state is NULL)"); +#else + _Py_FatalErrorFunc(func, + "the function must be called with an active thread state, " + "after Python initialization and before Python finalization, " + "but it was called without an active thread state. " + "Are you trying to call the C API inside of a Py_BEGIN_ALLOW_THREADS block?"); +#endif } } @@ -211,11 +229,16 @@ static inline PyInterpreterState* _PyInterpreterState_GET(void) { // PyThreadState functions -extern PyThreadState * _PyThreadState_New( +// Export for _testinternalcapi +PyAPI_FUNC(PyThreadState *) _PyThreadState_New( PyInterpreterState *interp, int whence); extern void _PyThreadState_Bind(PyThreadState *tstate); -extern void _PyThreadState_DeleteExcept(PyThreadState *tstate); +PyAPI_FUNC(PyThreadState *) _PyThreadState_NewBound( + PyInterpreterState *interp, + int whence); +extern PyThreadState * _PyThreadState_RemoveExcept(PyThreadState *tstate); +extern void _PyThreadState_DeleteList(PyThreadState *list); extern void _PyThreadState_ClearMimallocHeaps(PyThreadState *tstate); // Export for '_testinternalcapi' shared extension @@ -254,6 +277,15 @@ extern int _PyOS_InterruptOccurred(PyThreadState *tstate); #define HEAD_UNLOCK(runtime) \ PyMutex_Unlock(&(runtime)->interpreters.mutex) +#define _Py_FOR_EACH_TSTATE_UNLOCKED(interp, t) \ + for (PyThreadState *t = interp->threads.head; t; t = t->next) +#define _Py_FOR_EACH_TSTATE_BEGIN(interp, t) \ + HEAD_LOCK(interp->runtime); \ + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, t) +#define _Py_FOR_EACH_TSTATE_END(interp) \ + HEAD_UNLOCK(interp->runtime) + + // Get the configuration of the current interpreter. // The caller must hold the GIL. // Export for test_peg_generator. @@ -268,19 +300,18 @@ PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); // See also PyInterpreterState_Get() and _PyInterpreterState_GET(). extern PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe(void); -static inline struct _Py_object_freelists* _Py_object_freelists_GET(void) +#ifndef NDEBUG +/* Modern equivalent of assert(PyGILState_Check()) */ +static inline void +_Py_AssertHoldsTstateFunc(const char *func) { PyThreadState *tstate = _PyThreadState_GET(); -#ifdef Py_DEBUG - _Py_EnsureTstateNotNULL(tstate); -#endif - -#ifdef Py_GIL_DISABLED - return &((_PyThreadStateImpl*)tstate)->freelists; + _Py_EnsureFuncTstateNotNULL(func, tstate); +} +#define _Py_AssertHoldsTstate() _Py_AssertHoldsTstateFunc(__func__) #else - return &tstate->interp->object_state.freelists; +#define _Py_AssertHoldsTstate() #endif -} #ifdef __cplusplus } diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h index d2e7cc2a206ced..a1e084cf67d58d 100644 --- a/Include/internal/pycore_pythread.h +++ b/Include/internal/pycore_pythread.h @@ -78,7 +78,7 @@ struct _pythread_runtime_state { } stubs; #endif - // Linked list of ThreadHandleObjects + // Linked list of ThreadHandles struct llist_node handles; }; @@ -99,7 +99,7 @@ extern void _PyThread_AfterFork(struct _pythread_runtime_state *state); // unset: -1 seconds, in nanoseconds #define PyThread_UNSET_TIMEOUT ((PyTime_t)(-1 * 1000 * 1000 * 1000)) -// Exported for the _xxinterpchannels module. +// Exported for the _interpchannels module. PyAPI_FUNC(int) PyThread_ParseTimeoutArg( PyObject *arg, int blocking, @@ -111,7 +111,7 @@ PyAPI_FUNC(int) PyThread_ParseTimeoutArg( * are returned, depending on whether the lock can be acquired within the * timeout. */ -// Exported for the _xxinterpchannels module. +// Exported for the _interpchannels module. PyAPI_FUNC(PyLockStatus) PyThread_acquire_lock_timed_with_retries( PyThread_type_lock, PY_TIMEOUT_T microseconds); @@ -147,11 +147,24 @@ PyAPI_FUNC(int) PyThread_start_joinable_thread(void (*func)(void *), PyAPI_FUNC(int) PyThread_join_thread(PyThread_handle_t); /* * Detach a thread started with `PyThread_start_joinable_thread`, such - * that its resources are relased as soon as it exits. + * that its resources are released as soon as it exits. * This function cannot be interrupted. It returns 0 on success, * a non-zero value on failure. */ PyAPI_FUNC(int) PyThread_detach_thread(PyThread_handle_t); +/* + * Hangs the thread indefinitely without exiting it. + * + * gh-87135: There is no safe way to exit a thread other than returning + * normally from its start function. This is used during finalization in lieu + * of actually exiting the thread. Since the program is expected to terminate + * soon anyway, it does not matter if the thread stack stays around until then. + * + * This is unfortunate for embedders who may not be terminating their process + * when they're done with the interpreter, but our C API design does not allow + * for safely exiting threads attempting to re-enter Python post finalization. + */ +void _Py_NO_RETURN PyThread_hang_thread(void); #ifdef __cplusplus } diff --git a/Include/internal/pycore_qsbr.h b/Include/internal/pycore_qsbr.h index 475f00deedc226..20e643e172b38d 100644 --- a/Include/internal/pycore_qsbr.h +++ b/Include/internal/pycore_qsbr.h @@ -29,6 +29,12 @@ extern "C" { #define QSBR_INITIAL 1 #define QSBR_INCR 2 +// Wrap-around safe comparison. This is a holdover from the FreeBSD +// implementation, which uses 32-bit sequence numbers. We currently use 64-bit +// sequence numbers, so wrap-around is unlikely. +#define QSBR_LT(a, b) ((int64_t)((a)-(b)) < 0) +#define QSBR_LEQ(a, b) ((int64_t)((a)-(b)) <= 0) + struct _qsbr_shared; struct _PyThreadStateImpl; // forward declare to avoid circular dependency @@ -89,6 +95,15 @@ _Py_qsbr_quiescent_state(struct _qsbr_thread_state *qsbr) _Py_atomic_store_uint64_release(&qsbr->seq, seq); } +// Have the read sequences advanced to the given goal? Like `_Py_qsbr_poll()`, +// but does not perform a scan of threads. +static inline bool +_Py_qbsr_goal_reached(struct _qsbr_thread_state *qsbr, uint64_t goal) +{ + uint64_t rd_seq = _Py_atomic_load_uint64(&qsbr->shared->rd_seq); + return QSBR_LEQ(goal, rd_seq); +} + // Advance the write sequence and return the new goal. This should be called // after data is removed. The returned goal is used with `_Py_qsbr_poll()` to // determine when it is safe to reclaim (free) the memory. @@ -125,7 +140,7 @@ _Py_qsbr_register(struct _PyThreadStateImpl *tstate, // Disassociates a PyThreadState from the QSBR state and frees the QSBR state. extern void -_Py_qsbr_unregister(struct _PyThreadStateImpl *tstate); +_Py_qsbr_unregister(PyThreadState *tstate); extern void _Py_qsbr_fini(PyInterpreterState *interp); diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index dc6f6f100f7a92..cf123791eba9ac 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -9,8 +9,10 @@ extern "C" { #endif #include "pycore_atexit.h" // struct _atexit_runtime_state +#include "pycore_audit.h" // _Py_AuditHookEntry #include "pycore_ceval_state.h" // struct _ceval_runtime_state -#include "pycore_crossinterp.h" // struct _xidregistry +#include "pycore_crossinterp.h" // _PyXI_global_state_t +#include "pycore_debug_offsets.h" // _Py_DebugOffsets #include "pycore_faulthandler.h" // struct _faulthandler_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_import.h" // struct _import_runtime_state @@ -25,117 +27,12 @@ extern "C" { #include "pycore_typeobject.h" // struct _types_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_state -struct _getargs_runtime_state { - struct _PyArg_Parser *static_parsers; -}; - -/* GIL state */ - -struct _gilstate_runtime_state { - /* bpo-26558: Flag to disable PyGILState_Check(). - If set to non-zero, PyGILState_Check() always return 1. */ - int check_enabled; - /* The single PyInterpreterState used by this process' - GILState implementation - */ - /* TODO: Given interp_main, it may be possible to kill this ref */ - PyInterpreterState *autoInterpreterState; -}; - -/* Runtime audit hook state */ - -typedef struct _Py_AuditHookEntry { - struct _Py_AuditHookEntry *next; - Py_AuditHookFunction hookCFunction; - void *userData; -} _Py_AuditHookEntry; - -typedef struct _Py_DebugOffsets { - char cookie[8]; - uint64_t version; - // Runtime state offset; - struct _runtime_state { - uint64_t finalizing; - uint64_t interpreters_head; - } runtime_state; - - // Interpreter state offset; - struct _interpreter_state { - uint64_t next; - uint64_t threads_head; - uint64_t gc; - uint64_t imports_modules; - uint64_t sysdict; - uint64_t builtins; - uint64_t ceval_gil; - uint64_t gil_runtime_state_locked; - uint64_t gil_runtime_state_holder; - } interpreter_state; - - // Thread state offset; - struct _thread_state{ - uint64_t prev; - uint64_t next; - uint64_t interp; - uint64_t current_frame; - uint64_t thread_id; - uint64_t native_thread_id; - } thread_state; - - // InterpreterFrame offset; - struct _interpreter_frame { - uint64_t previous; - uint64_t executable; - uint64_t instr_ptr; - uint64_t localsplus; - uint64_t owner; - } interpreter_frame; - - // CFrame offset; - struct _cframe { - uint64_t current_frame; - uint64_t previous; - } cframe; - - // Code object offset; - struct _code_object { - uint64_t filename; - uint64_t name; - uint64_t linetable; - uint64_t firstlineno; - uint64_t argcount; - uint64_t localsplusnames; - uint64_t localspluskinds; - uint64_t co_code_adaptive; - } code_object; - - // PyObject offset; - struct _pyobject { - uint64_t ob_type; - } pyobject; - - // PyTypeObject object offset; - struct _type_object { - uint64_t tp_name; - } type_object; - - // PyTuple object offset; - struct _tuple_object { - uint64_t ob_item; - } tuple_object; - - // Unicode object offset; - struct _unicode_object { - uint64_t state; - uint64_t length; - size_t asciiobject_size; - } unicode_object; -} _Py_DebugOffsets; /* Full Python runtime state */ /* _PyRuntimeState holds the global state for the CPython runtime. - That data is exposed in the internal API as a static variable (_PyRuntime). + That data is exported by the internal API as a global variable + (_PyRuntime, defined near the top of pylifecycle.c). */ typedef struct pyruntimestate { /* This field must be first to facilitate locating it by out of process @@ -209,7 +106,7 @@ typedef struct pyruntimestate { tools. */ /* cross-interpreter data and utils */ - struct _xi_runtime_state xi; + _PyXI_global_state_t xi; struct _pymem_allocators allocators; struct _obmalloc_global_state obmalloc; @@ -231,11 +128,23 @@ typedef struct pyruntimestate { struct _import_runtime_state imports; struct _ceval_runtime_state ceval; - struct _gilstate_runtime_state gilstate; - struct _getargs_runtime_state getargs; + struct _gilstate_runtime_state { + /* bpo-26558: Flag to disable PyGILState_Check(). + If set to non-zero, PyGILState_Check() always return 1. */ + int check_enabled; + /* The single PyInterpreterState used by this process' + GILState implementation + */ + /* TODO: Given interp_main, it may be possible to kill this ref */ + PyInterpreterState *autoInterpreterState; + } gilstate; + struct _getargs_runtime_state { + struct _PyArg_Parser *static_parsers; + } getargs; struct _fileutils_state fileutils; struct _faulthandler_runtime_state faulthandler; struct _tracemalloc_runtime_state tracemalloc; + struct _reftracer_runtime_state ref_tracer; // The rwmutex is used to prevent overlapping global and per-interpreter // stop-the-world events. Global stop-the-world events lock the mutex @@ -260,6 +169,12 @@ typedef struct pyruntimestate { struct _Py_unicode_runtime_state unicode_state; struct _types_runtime_state types; +#if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) + // Used in "Python/emscripten_trampoline.c" to choose between type + // reflection trampoline and EM_JS trampoline. + int (*emscripten_count_args_function)(PyCFunctionWithKeywords func); +#endif + /* All the objects that are shared by the runtime's interpreters. */ struct _Py_cached_objects cached_objects; struct _Py_static_objects static_objects; @@ -280,13 +195,8 @@ typedef struct pyruntimestate { /* _PyRuntimeState.interpreters.main */ PyInterpreterState _main_interpreter; - -#if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) - // Used in "Python/emscripten_trampoline.c" to choose between type - // reflection trampoline and EM_JS trampoline. - bool wasm_type_reflection_available; -#endif - + // _main_interpreter should be the last field of _PyRuntimeState. + // See https://github.com/python/cpython/issues/127117. } _PyRuntimeState; @@ -335,6 +245,7 @@ _PyRuntimeState_SetFinalizing(_PyRuntimeState *runtime, PyThreadState *tstate) { } } + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index cc47b9a82e2879..1260b957ce9482 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -9,8 +9,10 @@ extern "C" { #endif #include "pycore_ceval_state.h" // _PyEval_RUNTIME_PERF_INIT +#include "pycore_debug_offsets.h" // _Py_DebugOffsets_INIT() #include "pycore_faulthandler.h" // _faulthandler_runtime_state_INIT #include "pycore_floatobject.h" // _py_float_format_unknown +#include "pycore_function.h" #include "pycore_object.h" // _PyObject_HEAD_INIT #include "pycore_obmalloc_init.h" // _obmalloc_global_state_INIT #include "pycore_parser.h" // _parser_runtime_state_INIT @@ -29,66 +31,9 @@ extern PyTypeObject _PyExc_MemoryError; /* The static initializers defined here should only be used in the runtime init code (in pystate.c and pylifecycle.c). */ -#define _PyRuntimeState_INIT(runtime) \ +#define _PyRuntimeState_INIT(runtime, debug_cookie) \ { \ - .debug_offsets = { \ - .cookie = "xdebugpy", \ - .version = PY_VERSION_HEX, \ - .runtime_state = { \ - .finalizing = offsetof(_PyRuntimeState, _finalizing), \ - .interpreters_head = offsetof(_PyRuntimeState, interpreters.head), \ - }, \ - .interpreter_state = { \ - .next = offsetof(PyInterpreterState, next), \ - .threads_head = offsetof(PyInterpreterState, threads.head), \ - .gc = offsetof(PyInterpreterState, gc), \ - .imports_modules = offsetof(PyInterpreterState, imports.modules), \ - .sysdict = offsetof(PyInterpreterState, sysdict), \ - .builtins = offsetof(PyInterpreterState, builtins), \ - .ceval_gil = offsetof(PyInterpreterState, ceval.gil), \ - .gil_runtime_state_locked = offsetof(PyInterpreterState, _gil.locked), \ - .gil_runtime_state_holder = offsetof(PyInterpreterState, _gil.last_holder), \ - }, \ - .thread_state = { \ - .prev = offsetof(PyThreadState, prev), \ - .next = offsetof(PyThreadState, next), \ - .interp = offsetof(PyThreadState, interp), \ - .current_frame = offsetof(PyThreadState, current_frame), \ - .thread_id = offsetof(PyThreadState, thread_id), \ - .native_thread_id = offsetof(PyThreadState, native_thread_id), \ - }, \ - .interpreter_frame = { \ - .previous = offsetof(_PyInterpreterFrame, previous), \ - .executable = offsetof(_PyInterpreterFrame, f_executable), \ - .instr_ptr = offsetof(_PyInterpreterFrame, instr_ptr), \ - .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ - .owner = offsetof(_PyInterpreterFrame, owner), \ - }, \ - .code_object = { \ - .filename = offsetof(PyCodeObject, co_filename), \ - .name = offsetof(PyCodeObject, co_name), \ - .linetable = offsetof(PyCodeObject, co_linetable), \ - .firstlineno = offsetof(PyCodeObject, co_firstlineno), \ - .argcount = offsetof(PyCodeObject, co_argcount), \ - .localsplusnames = offsetof(PyCodeObject, co_localsplusnames), \ - .localspluskinds = offsetof(PyCodeObject, co_localspluskinds), \ - .co_code_adaptive = offsetof(PyCodeObject, co_code_adaptive), \ - }, \ - .pyobject = { \ - .ob_type = offsetof(PyObject, ob_type), \ - }, \ - .type_object = { \ - .tp_name = offsetof(PyTypeObject, tp_name), \ - }, \ - .tuple_object = { \ - .ob_item = offsetof(PyTupleObject, ob_item), \ - }, \ - .unicode_object = { \ - .state = offsetof(PyUnicodeObject, _base._base.state), \ - .length = offsetof(PyUnicodeObject, _base._base.length), \ - .asciiobject_size = sizeof(PyASCIIObject), \ - }, \ - }, \ + .debug_offsets = _Py_DebugOffsets_INIT(debug_cookie), \ .allocators = { \ .standard = _pymem_allocators_standard_INIT(runtime), \ .debug = _pymem_allocators_debug_INIT, \ @@ -105,8 +50,10 @@ extern PyTypeObject _PyExc_MemoryError; .next_id = -1, \ }, \ .xi = { \ - .registry = { \ - .global = 1, \ + .data_lookup = { \ + .registry = { \ + .global = 1, \ + }, \ }, \ }, \ /* A TSS key must be initialized with Py_tss_NEEDS_INIT \ @@ -114,6 +61,10 @@ extern PyTypeObject _PyExc_MemoryError; .autoTSSkey = Py_tss_NEEDS_INIT, \ .parser = _parser_runtime_state_INIT, \ .ceval = { \ + .pending_mainthread = { \ + .max = MAXPENDINGCALLS_MAIN, \ + .maxloop = MAXPENDINGCALLSLOOP_MAIN, \ + }, \ .perf = _PyEval_RUNTIME_PERF_INIT, \ }, \ .gilstate = { \ @@ -124,6 +75,10 @@ extern PyTypeObject _PyExc_MemoryError; }, \ .faulthandler = _faulthandler_runtime_state_INIT, \ .tracemalloc = _tracemalloc_runtime_state_INIT, \ + .ref_tracer = { \ + .tracer_func = NULL, \ + .tracer_data = NULL, \ + }, \ .stoptheworld = { \ .is_global = 1, \ }, \ @@ -132,7 +87,7 @@ extern PyTypeObject _PyExc_MemoryError; .double_format = _py_float_format_unknown, \ }, \ .types = { \ - .next_version_tag = 1, \ + .next_version_tag = _Py_TYPE_VERSION_NEXT, \ }, \ .static_objects = { \ .singletons = { \ @@ -162,18 +117,27 @@ extern PyTypeObject _PyExc_MemoryError; #define _PyInterpreterState_INIT(INTERP) \ { \ .id_refcount = -1, \ + ._whence = _PyInterpreterState_WHENCE_NOTSET, \ + .threads = { \ + .preallocated = &(INTERP)._initial_thread, \ + }, \ .imports = IMPORTS_INIT, \ .ceval = { \ .recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \ + .pending = { \ + .max = MAXPENDINGCALLS, \ + .maxloop = MAXPENDINGCALLSLOOP, \ + }, \ }, \ .gc = { \ .enabled = 1, \ - .generations = { \ - /* .head is set in _PyGC_InitState(). */ \ - { .threshold = 700, }, \ - { .threshold = 10, }, \ + .young = { .threshold = 2000, }, \ + .old = { \ { .threshold = 10, }, \ + { .threshold = 0, }, \ }, \ + .work_to_do = -5000, \ + .phase = GC_PHASE_MARK, \ }, \ .qsbr = { \ .wr_seq = QSBR_INITIAL, \ @@ -183,7 +147,7 @@ extern PyTypeObject _PyExc_MemoryError; .dict_state = _dict_state_INIT, \ .mem_free_queue = _Py_mem_free_queue_INIT(INTERP.mem_free_queue), \ .func_state = { \ - .next_version = 1, \ + .next_version = FUNC_VERSION_FIRST_VALID, \ }, \ .types = { \ .next_version_tag = _Py_TYPE_BASE_VERSION_TAG, \ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 016eae02a2103d..4f928cc050bf8e 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -546,21 +546,18 @@ extern "C" { INIT_STR(anon_setcomp, ""), \ INIT_STR(anon_string, ""), \ INIT_STR(anon_unknown, ""), \ - INIT_STR(close_br, "}"), \ INIT_STR(dbl_close_br, "}}"), \ INIT_STR(dbl_open_br, "{{"), \ INIT_STR(dbl_percent, "%%"), \ INIT_STR(defaults, ".defaults"), \ - INIT_STR(dot, "."), \ INIT_STR(dot_locals, "."), \ INIT_STR(empty, ""), \ + INIT_STR(format, ".format"), \ INIT_STR(generic_base, ".generic_base"), \ INIT_STR(json_decoder, "json.decoder"), \ INIT_STR(kwdefaults, ".kwdefaults"), \ INIT_STR(list_err, "list index out of range"), \ - INIT_STR(newline, "\n"), \ - INIT_STR(open_br, "{"), \ - INIT_STR(percent, "%"), \ + INIT_STR(str_replace_inf, "1e309"), \ INIT_STR(type_params, ".type_params"), \ INIT_STR(utf_8, "utf-8"), \ } @@ -575,7 +572,6 @@ extern "C" { INIT_ID(TextIOWrapper), \ INIT_ID(True), \ INIT_ID(WarningMessage), \ - INIT_ID(_), \ INIT_ID(_WindowsConsoleIO), \ INIT_ID(__IOBase_closed), \ INIT_ID(__abc_tpflags__), \ @@ -588,9 +584,9 @@ extern "C" { INIT_ID(__all__), \ INIT_ID(__and__), \ INIT_ID(__anext__), \ + INIT_ID(__annotate__), \ INIT_ID(__annotations__), \ INIT_ID(__args__), \ - INIT_ID(__asyncio_running_event_loop__), \ INIT_ID(__await__), \ INIT_ID(__bases__), \ INIT_ID(__bool__), \ @@ -607,7 +603,6 @@ extern "C" { INIT_ID(__classdictcell__), \ INIT_ID(__complex__), \ INIT_ID(__contains__), \ - INIT_ID(__copy__), \ INIT_ID(__ctypes_from_outparam__), \ INIT_ID(__del__), \ INIT_ID(__delattr__), \ @@ -622,6 +617,7 @@ extern "C" { INIT_ID(__eq__), \ INIT_ID(__exit__), \ INIT_ID(__file__), \ + INIT_ID(__firstlineno__), \ INIT_ID(__float__), \ INIT_ID(__floordiv__), \ INIT_ID(__format__), \ @@ -722,12 +718,12 @@ extern "C" { INIT_ID(__slotnames__), \ INIT_ID(__slots__), \ INIT_ID(__spec__), \ + INIT_ID(__static_attributes__), \ INIT_ID(__str__), \ INIT_ID(__sub__), \ INIT_ID(__subclasscheck__), \ INIT_ID(__subclasshook__), \ INIT_ID(__truediv__), \ - INIT_ID(__trunc__), \ INIT_ID(__type_params__), \ INIT_ID(__typing_is_unpacked_typevartuple__), \ INIT_ID(__typing_prepare_subst__), \ @@ -740,8 +736,6 @@ extern "C" { INIT_ID(_abc_impl), \ INIT_ID(_abstract_), \ INIT_ID(_active), \ - INIT_ID(_align_), \ - INIT_ID(_annotation), \ INIT_ID(_anonymous_), \ INIT_ID(_argtypes_), \ INIT_ID(_as_parameter_), \ @@ -762,23 +756,25 @@ extern "C" { INIT_ID(_initializing), \ INIT_ID(_io), \ INIT_ID(_is_text_encoding), \ + INIT_ID(_isatty_open_only), \ INIT_ID(_length_), \ INIT_ID(_limbo), \ INIT_ID(_lock_unlock_module), \ INIT_ID(_loop), \ INIT_ID(_needs_com_addref_), \ - INIT_ID(_pack_), \ + INIT_ID(_only_immortal), \ INIT_ID(_restype_), \ INIT_ID(_showwarnmsg), \ INIT_ID(_shutdown), \ INIT_ID(_slotnames), \ - INIT_ID(_strptime_datetime), \ - INIT_ID(_swappedbytes_), \ + INIT_ID(_strptime), \ + INIT_ID(_strptime_datetime_date), \ + INIT_ID(_strptime_datetime_datetime), \ + INIT_ID(_strptime_datetime_time), \ INIT_ID(_type_), \ INIT_ID(_uninitialized_submodules), \ INIT_ID(_warn_unawaited_coroutine), \ INIT_ID(_xoptions), \ - INIT_ID(a), \ INIT_ID(abs_tol), \ INIT_ID(access), \ INIT_ID(aclose), \ @@ -788,8 +784,10 @@ extern "C" { INIT_ID(after_in_parent), \ INIT_ID(aggregate_class), \ INIT_ID(alias), \ + INIT_ID(align), \ INIT_ID(allow_code), \ INIT_ID(append), \ + INIT_ID(arg), \ INIT_ID(argdefs), \ INIT_ID(args), \ INIT_ID(arguments), \ @@ -801,12 +799,12 @@ extern "C" { INIT_ID(attribute), \ INIT_ID(authorizer_callback), \ INIT_ID(autocommit), \ - INIT_ID(b), \ INIT_ID(backtick), \ INIT_ID(base), \ INIT_ID(before), \ INIT_ID(big), \ INIT_ID(binary_form), \ + INIT_ID(bit_size), \ INIT_ID(block), \ INIT_ID(bound), \ INIT_ID(buffer), \ @@ -819,10 +817,10 @@ extern "C" { INIT_ID(byteorder), \ INIT_ID(bytes), \ INIT_ID(bytes_per_sep), \ - INIT_ID(c), \ INIT_ID(c_call), \ INIT_ID(c_exception), \ INIT_ID(c_return), \ + INIT_ID(cached_datetime_module), \ INIT_ID(cached_statements), \ INIT_ID(cadata), \ INIT_ID(cafile), \ @@ -860,6 +858,7 @@ extern "C" { INIT_ID(co_stacksize), \ INIT_ID(co_varnames), \ INIT_ID(code), \ + INIT_ID(col_offset), \ INIT_ID(command), \ INIT_ID(comment_factory), \ INIT_ID(compile_mode), \ @@ -873,7 +872,6 @@ extern "C" { INIT_ID(count), \ INIT_ID(covariant), \ INIT_ID(cwd), \ - INIT_ID(d), \ INIT_ID(data), \ INIT_ID(database), \ INIT_ID(day), \ @@ -899,16 +897,17 @@ extern "C" { INIT_ID(displayhook), \ INIT_ID(dklen), \ INIT_ID(doc), \ + INIT_ID(done), \ INIT_ID(dont_inherit), \ INIT_ID(dst), \ INIT_ID(dst_dir_fd), \ - INIT_ID(e), \ INIT_ID(eager_start), \ INIT_ID(effective_ids), \ INIT_ID(element_factory), \ INIT_ID(encode), \ INIT_ID(encoding), \ INIT_ID(end), \ + INIT_ID(end_col_offset), \ INIT_ID(end_lineno), \ INIT_ID(end_offset), \ INIT_ID(endpos), \ @@ -925,7 +924,6 @@ extern "C" { INIT_ID(exp), \ INIT_ID(extend), \ INIT_ID(extra_tokens), \ - INIT_ID(f), \ INIT_ID(facility), \ INIT_ID(factory), \ INIT_ID(false), \ @@ -935,6 +933,7 @@ extern "C" { INIT_ID(fd2), \ INIT_ID(fdel), \ INIT_ID(fget), \ + INIT_ID(fields), \ INIT_ID(file), \ INIT_ID(file_actions), \ INIT_ID(filename), \ @@ -951,6 +950,7 @@ extern "C" { INIT_ID(fold), \ INIT_ID(follow_symlinks), \ INIT_ID(format), \ + INIT_ID(format_spec), \ INIT_ID(from_param), \ INIT_ID(fromlist), \ INIT_ID(fromtimestamp), \ @@ -958,7 +958,6 @@ extern "C" { INIT_ID(fset), \ INIT_ID(func), \ INIT_ID(future), \ - INIT_ID(g), \ INIT_ID(generation), \ INIT_ID(genexpr), \ INIT_ID(get), \ @@ -972,9 +971,9 @@ extern "C" { INIT_ID(globals), \ INIT_ID(groupindex), \ INIT_ID(groups), \ - INIT_ID(h), \ INIT_ID(handle), \ INIT_ID(handle_seq), \ + INIT_ID(has_location), \ INIT_ID(hash_name), \ INIT_ID(header), \ INIT_ID(headers), \ @@ -989,6 +988,7 @@ extern "C" { INIT_ID(importlib), \ INIT_ID(in_fd), \ INIT_ID(incoming), \ + INIT_ID(index), \ INIT_ID(indexgroup), \ INIT_ID(inf), \ INIT_ID(infer_variance), \ @@ -1009,6 +1009,7 @@ extern "C" { INIT_ID(intersection), \ INIT_ID(interval), \ INIT_ID(is_running), \ + INIT_ID(is_struct), \ INIT_ID(isatty), \ INIT_ID(isinstance), \ INIT_ID(isoformat), \ @@ -1030,6 +1031,7 @@ extern "C" { INIT_ID(kw1), \ INIT_ID(kw2), \ INIT_ID(kwdefaults), \ + INIT_ID(label), \ INIT_ID(lambda), \ INIT_ID(last), \ INIT_ID(last_exc), \ @@ -1082,7 +1084,6 @@ extern "C" { INIT_ID(msg), \ INIT_ID(mutex), \ INIT_ID(mycmp), \ - INIT_ID(n), \ INIT_ID(n_arg), \ INIT_ID(n_fields), \ INIT_ID(n_sequence_fields), \ @@ -1093,6 +1094,7 @@ extern "C" { INIT_ID(namespaces), \ INIT_ID(narg), \ INIT_ID(ndigits), \ + INIT_ID(nested), \ INIT_ID(new_file_name), \ INIT_ID(new_limit), \ INIT_ID(newline), \ @@ -1125,9 +1127,9 @@ extern "C" { INIT_ID(origin), \ INIT_ID(out_fd), \ INIT_ID(outgoing), \ + INIT_ID(outpath), \ INIT_ID(overlapped), \ INIT_ID(owner), \ - INIT_ID(p), \ INIT_ID(pages), \ INIT_ID(parent), \ INIT_ID(password), \ @@ -1155,7 +1157,6 @@ extern "C" { INIT_ID(ps2), \ INIT_ID(query), \ INIT_ID(quotetabs), \ - INIT_ID(r), \ INIT_ID(raw), \ INIT_ID(read), \ INIT_ID(read1), \ @@ -1179,7 +1180,6 @@ extern "C" { INIT_ID(return), \ INIT_ID(reverse), \ INIT_ID(reversed), \ - INIT_ID(s), \ INIT_ID(salt), \ INIT_ID(sched_priority), \ INIT_ID(scheduler), \ @@ -1212,6 +1212,7 @@ extern "C" { INIT_ID(sort), \ INIT_ID(source), \ INIT_ID(source_traceback), \ + INIT_ID(spam), \ INIT_ID(src), \ INIT_ID(src_dir_fd), \ INIT_ID(stacklevel), \ @@ -1230,6 +1231,7 @@ extern "C" { INIT_ID(strict_mode), \ INIT_ID(string), \ INIT_ID(sub_key), \ + INIT_ID(subcalls), \ INIT_ID(symmetric_difference_update), \ INIT_ID(tabsize), \ INIT_ID(tag), \ @@ -1247,8 +1249,10 @@ extern "C" { INIT_ID(threading), \ INIT_ID(throw), \ INIT_ID(timeout), \ + INIT_ID(timer), \ INIT_ID(times), \ INIT_ID(timetuple), \ + INIT_ID(timeunit), \ INIT_ID(top), \ INIT_ID(trace_callback), \ INIT_ID(traceback), \ @@ -1273,6 +1277,7 @@ extern "C" { INIT_ID(version), \ INIT_ID(volume), \ INIT_ID(wait_all), \ + INIT_ID(warn_on_full_buffer), \ INIT_ID(warnings), \ INIT_ID(warnoptions), \ INIT_ID(wbits), \ @@ -1284,7 +1289,6 @@ extern "C" { INIT_ID(writable), \ INIT_ID(write), \ INIT_ID(write_through), \ - INIT_ID(x), \ INIT_ID(year), \ INIT_ID(zdict), \ } diff --git a/Include/internal/pycore_setobject.h b/Include/internal/pycore_setobject.h index 34a00e6d45fe69..0494c07fe1869d 100644 --- a/Include/internal/pycore_setobject.h +++ b/Include/internal/pycore_setobject.h @@ -8,19 +8,31 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -// Export for '_pickle' shared extension +// Export for '_abc' shared extension PyAPI_FUNC(int) _PySet_NextEntry( PyObject *set, Py_ssize_t *pos, PyObject **key, Py_hash_t *hash); +// Export for '_pickle' shared extension +PyAPI_FUNC(int) _PySet_NextEntryRef( + PyObject *set, + Py_ssize_t *pos, + PyObject **key, + Py_hash_t *hash); + // Export for '_pickle' shared extension PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable); // Export for the gdb plugin's (python-gdb.py) benefit PyAPI_DATA(PyObject *) _PySet_Dummy; +PyAPI_FUNC(int) _PySet_Contains(PySetObject *so, PyObject *key); + +// Clears the set without acquiring locks. Used by _PyCode_Fini. +extern void _PySet_ClearInternal(PySetObject *so); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h new file mode 100644 index 00000000000000..1ae62cc69bb364 --- /dev/null +++ b/Include/internal/pycore_stackref.h @@ -0,0 +1,383 @@ +#ifndef Py_INTERNAL_STACKREF_H +#define Py_INTERNAL_STACKREF_H +#ifdef __cplusplus +extern "C" { +#endif + +// Define this to get precise tracking of stackrefs. +// #define Py_STACKREF_DEBUG 1 + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_object_deferred.h" + +#include +#include + +/* + This file introduces a new API for handling references on the stack, called + _PyStackRef. This API is inspired by HPy. + + There are 3 main operations, that convert _PyStackRef to PyObject* and + vice versa: + + 1. Borrow (discouraged) + 2. Steal + 3. New + + Borrow means that the reference is converted without any change in ownership. + This is discouraged because it makes verification much harder. It also makes + unboxed integers harder in the future. + + Steal means that ownership is transferred to something else. The total + number of references to the object stays the same. + + New creates a new reference from the old reference. The old reference + is still valid. + + With these 3 API, a strict stack discipline must be maintained. All + _PyStackRef must be operated on by the new reference operations: + + 1. DUP + 2. CLOSE + + DUP is roughly equivalent to Py_NewRef. It creates a new reference from an old + reference. The old reference remains unchanged. + + CLOSE is roughly equivalent to Py_DECREF. It destroys a reference. + + Note that it is unsafe to borrow a _PyStackRef and then do normal + CPython refcounting operations on it! +*/ + + +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) + + + +typedef union _PyStackRef { + uint64_t index; +} _PyStackRef; + +#define Py_TAG_BITS 0 + +PyAPI_FUNC(PyObject *) _Py_stackref_get_object(_PyStackRef ref); +PyAPI_FUNC(PyObject *) _Py_stackref_close(_PyStackRef ref); +PyAPI_FUNC(_PyStackRef) _Py_stackref_create(PyObject *obj, const char *filename, int linenumber); +PyAPI_FUNC(void) _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber); +extern void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref); + +static const _PyStackRef PyStackRef_NULL = { .index = 0 }; + +#define PyStackRef_None ((_PyStackRef){ .index = 1 } ) +#define PyStackRef_False ((_PyStackRef){ .index = 2 }) +#define PyStackRef_True ((_PyStackRef){ .index = 3 }) + +#define LAST_PREDEFINED_STACKREF_INDEX 3 + +static inline int +PyStackRef_IsNull(_PyStackRef ref) +{ + return ref.index == 0; +} + +static inline int +PyStackRef_IsTrue(_PyStackRef ref) +{ + return _Py_stackref_get_object(ref) == Py_True; +} + +static inline int +PyStackRef_IsFalse(_PyStackRef ref) +{ + return _Py_stackref_get_object(ref) == Py_False; +} + +static inline int +PyStackRef_IsNone(_PyStackRef ref) +{ + return _Py_stackref_get_object(ref) == Py_None; +} + +static inline PyObject * +_PyStackRef_AsPyObjectBorrow(_PyStackRef ref, const char *filename, int linenumber) +{ + _Py_stackref_record_borrow(ref, filename, linenumber); + return _Py_stackref_get_object(ref); +} + +#define PyStackRef_AsPyObjectBorrow(REF) _PyStackRef_AsPyObjectBorrow((REF), __FILE__, __LINE__) + +static inline PyObject * +PyStackRef_AsPyObjectSteal(_PyStackRef ref) +{ + return _Py_stackref_close(ref); +} + +static inline _PyStackRef +_PyStackRef_FromPyObjectNew(PyObject *obj, const char *filename, int linenumber) +{ + Py_INCREF(obj); + return _Py_stackref_create(obj, filename, linenumber); +} +#define PyStackRef_FromPyObjectNew(obj) _PyStackRef_FromPyObjectNew(_PyObject_CAST(obj), __FILE__, __LINE__) + +static inline _PyStackRef +_PyStackRef_FromPyObjectSteal(PyObject *obj, const char *filename, int linenumber) +{ + return _Py_stackref_create(obj, filename, linenumber); +} +#define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj), __FILE__, __LINE__) + +static inline _PyStackRef +_PyStackRef_FromPyObjectImmortal(PyObject *obj, const char *filename, int linenumber) +{ + assert(_Py_IsImmortal(obj)); + return _Py_stackref_create(obj, filename, linenumber); +} +#define PyStackRef_FromPyObjectImmortal(obj) _PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj), __FILE__, __LINE__) + +static inline void +PyStackRef_CLOSE(_PyStackRef ref) +{ + PyObject *obj = _Py_stackref_close(ref); + Py_DECREF(obj); +} + +static inline _PyStackRef +_PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber) +{ + PyObject *obj = _Py_stackref_get_object(ref); + Py_INCREF(obj); + return _Py_stackref_create(obj, filename, linenumber); +} +#define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__) + +#define PyStackRef_CLOSE_SPECIALIZED(stackref, dealloc) PyStackRef_CLOSE(stackref) + +#else + +typedef union _PyStackRef { + uintptr_t bits; +} _PyStackRef; + + +#define Py_TAG_DEFERRED (1) + +#define Py_TAG_PTR ((uintptr_t)0) +#define Py_TAG_BITS ((uintptr_t)1) + +#ifdef Py_GIL_DISABLED + +static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED}; +#define PyStackRef_IsNull(stackref) ((stackref).bits == PyStackRef_NULL.bits) +#define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_DEFERRED }) +#define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED }) +#define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_DEFERRED }) + +static inline PyObject * +PyStackRef_AsPyObjectBorrow(_PyStackRef stackref) +{ + PyObject *cleared = ((PyObject *)((stackref).bits & (~Py_TAG_BITS))); + return cleared; +} + +#define PyStackRef_IsDeferred(ref) (((ref).bits & Py_TAG_BITS) == Py_TAG_DEFERRED) + +static inline PyObject * +PyStackRef_NotDeferred_AsPyObject(_PyStackRef stackref) +{ + assert(!PyStackRef_IsDeferred(stackref)); + return (PyObject *)stackref.bits; +} + +static inline PyObject * +PyStackRef_AsPyObjectSteal(_PyStackRef stackref) +{ + assert(!PyStackRef_IsNull(stackref)); + if (PyStackRef_IsDeferred(stackref)) { + return Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)); + } + return PyStackRef_AsPyObjectBorrow(stackref); +} + +static inline _PyStackRef +_PyStackRef_FromPyObjectSteal(PyObject *obj) +{ + assert(obj != NULL); + // Make sure we don't take an already tagged value. + assert(((uintptr_t)obj & Py_TAG_BITS) == 0); + return (_PyStackRef){ .bits = (uintptr_t)obj }; +} +# define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj)) + +static inline _PyStackRef +PyStackRef_FromPyObjectNew(PyObject *obj) +{ + // Make sure we don't take an already tagged value. + assert(((uintptr_t)obj & Py_TAG_BITS) == 0); + assert(obj != NULL); + if (_Py_IsImmortal(obj) || _PyObject_HasDeferredRefcount(obj)) { + return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; + } + else { + return (_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR }; + } +} +#define PyStackRef_FromPyObjectNew(obj) PyStackRef_FromPyObjectNew(_PyObject_CAST(obj)) + +static inline _PyStackRef +PyStackRef_FromPyObjectImmortal(PyObject *obj) +{ + // Make sure we don't take an already tagged value. + assert(((uintptr_t)obj & Py_TAG_BITS) == 0); + assert(obj != NULL); + assert(_Py_IsImmortal(obj)); + return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; +} +#define PyStackRef_FromPyObjectImmortal(obj) PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj)) + +#define PyStackRef_CLOSE(REF) \ + do { \ + _PyStackRef _close_tmp = (REF); \ + assert(!PyStackRef_IsNull(_close_tmp)); \ + if (!PyStackRef_IsDeferred(_close_tmp)) { \ + Py_DECREF(PyStackRef_AsPyObjectBorrow(_close_tmp)); \ + } \ + } while (0) + +static inline _PyStackRef +PyStackRef_DUP(_PyStackRef stackref) +{ + assert(!PyStackRef_IsNull(stackref)); + if (PyStackRef_IsDeferred(stackref)) { + assert(_Py_IsImmortal(PyStackRef_AsPyObjectBorrow(stackref)) || + _PyObject_HasDeferredRefcount(PyStackRef_AsPyObjectBorrow(stackref)) + ); + return stackref; + } + Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref)); + return stackref; +} + +// Convert a possibly deferred reference to a strong reference. +static inline _PyStackRef +PyStackRef_AsStrongReference(_PyStackRef stackref) +{ + return PyStackRef_FromPyObjectSteal(PyStackRef_AsPyObjectSteal(stackref)); +} + +#define PyStackRef_CLOSE_SPECIALIZED(stackref, dealloc) PyStackRef_CLOSE(stackref) + + +#else // Py_GIL_DISABLED + +// With GIL +static const _PyStackRef PyStackRef_NULL = { .bits = 0 }; +#define PyStackRef_IsNull(stackref) ((stackref).bits == 0) +#define PyStackRef_True ((_PyStackRef){.bits = (uintptr_t)&_Py_TrueStruct }) +#define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) }) +#define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) }) + +#define PyStackRef_AsPyObjectBorrow(stackref) ((PyObject *)(stackref).bits) + +#define PyStackRef_AsPyObjectSteal(stackref) PyStackRef_AsPyObjectBorrow(stackref) + +#define PyStackRef_FromPyObjectSteal(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj))}) + +#define PyStackRef_FromPyObjectNew(obj) ((_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) }) + +#define PyStackRef_FromPyObjectImmortal(obj) ((_PyStackRef){ .bits = (uintptr_t)(obj) }) + +#define PyStackRef_CLOSE(stackref) Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref)) + +#define PyStackRef_DUP(stackref) PyStackRef_FromPyObjectSteal(Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref))) + +#define PyStackRef_CLOSE_SPECIALIZED(stackref, dealloc) _Py_DECREF_SPECIALIZED(PyStackRef_AsPyObjectBorrow(stackref), dealloc) + +#endif // Py_GIL_DISABLED + +// Check if a stackref is exactly the same as another stackref, including the +// the deferred bit. This can only be used safely if you know that the deferred +// bits of `a` and `b` match. +#define PyStackRef_IsExactly(a, b) \ + (assert(((a).bits & Py_TAG_BITS) == ((b).bits & Py_TAG_BITS)), (a).bits == (b).bits) + +// Checks that mask out the deferred bit in the free threading build. +#define PyStackRef_IsNone(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_None) +#define PyStackRef_IsTrue(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_True) +#define PyStackRef_IsFalse(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_False) + +#endif + +// Converts a PyStackRef back to a PyObject *, converting the +// stackref to a new reference. +#define PyStackRef_AsPyObjectNew(stackref) Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)) + +#define PyStackRef_TYPE(stackref) Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref)) + + +#define PyStackRef_CLEAR(op) \ + do { \ + _PyStackRef *_tmp_op_ptr = &(op); \ + _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \ + if (!PyStackRef_IsNull(_tmp_old_op)) { \ + *_tmp_op_ptr = PyStackRef_NULL; \ + PyStackRef_CLOSE(_tmp_old_op); \ + } \ + } while (0) + +#define PyStackRef_XCLOSE(stackref) \ + do { \ + _PyStackRef _tmp = (stackref); \ + if (!PyStackRef_IsNull(_tmp)) { \ + PyStackRef_CLOSE(_tmp); \ + } \ + } while (0); + + + +// StackRef type checks + +static inline bool +PyStackRef_GenCheck(_PyStackRef stackref) +{ + return PyGen_Check(PyStackRef_AsPyObjectBorrow(stackref)); +} + +static inline bool +PyStackRef_BoolCheck(_PyStackRef stackref) +{ + return PyBool_Check(PyStackRef_AsPyObjectBorrow(stackref)); +} + +static inline bool +PyStackRef_LongCheck(_PyStackRef stackref) +{ + return PyLong_Check(PyStackRef_AsPyObjectBorrow(stackref)); +} + +static inline bool +PyStackRef_ExceptionInstanceCheck(_PyStackRef stackref) +{ + return PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(stackref)); +} + +static inline bool +PyStackRef_CodeCheck(_PyStackRef stackref) +{ + return PyCode_Check(PyStackRef_AsPyObjectBorrow(stackref)); +} + +static inline bool +PyStackRef_FunctionCheck(_PyStackRef stackref) +{ + return PyFunction_Check(PyStackRef_AsPyObjectBorrow(stackref)); +} + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_STACKREF_H */ diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index b44393b5644673..b7e274296112aa 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -12,14 +12,27 @@ struct _mod; // Type defined in pycore_ast.h typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock, - // Used for annotations if 'from __future__ import annotations' is active. - // Annotation blocks cannot bind names and are not evaluated. + // Used for annotations. If 'from __future__ import annotations' is active, + // annotation blocks cannot bind names and are not evaluated. Otherwise, they + // are lazily evaluated (see PEP 649). AnnotationBlock, - // Used for generics and type aliases. These work mostly like functions - // (see PEP 695 for details). The three different blocks function identically; - // they are different enum entries only so that error messages can be more - // precise. - TypeVarBoundBlock, TypeAliasBlock, TypeParamBlock + + // The following blocks are used for generics and type aliases. These work + // mostly like functions (see PEP 695 for details). The three different + // blocks function identically; they are different enum entries only so + // that error messages can be more precise. + + // The block to enter when processing a "type" (PEP 695) construction, + // e.g., "type MyGeneric[T] = list[T]". + TypeAliasBlock, + // The block to enter when processing a "generic" (PEP 695) object, + // e.g., "def foo[T](): pass" or "class A[T]: pass". + TypeParametersBlock, + // The block to enter when processing the bound, the constraint tuple + // or the default value of a single "type variable" in the formal sense, + // i.e., a TypeVar, a TypeVarTuple or a ParamSpec object (the latter two + // do not support a bound or a constraint tuple). + TypeVariableBlock, } _Py_block_ty; typedef enum _comprehension_type { @@ -29,6 +42,29 @@ typedef enum _comprehension_type { SetComprehension = 3, GeneratorExpression = 4 } _Py_comprehension_ty; +/* source location information */ +typedef struct { + int lineno; + int end_lineno; + int col_offset; + int end_col_offset; +} _Py_SourceLocation; + +#define SRC_LOCATION_FROM_AST(n) \ + (_Py_SourceLocation){ \ + .lineno = (n)->lineno, \ + .end_lineno = (n)->end_lineno, \ + .col_offset = (n)->col_offset, \ + .end_col_offset = (n)->end_col_offset } + +static const _Py_SourceLocation NO_LOCATION = {-1, -1, -1, -1}; + +/* __future__ information */ +typedef struct { + int ff_features; /* flags set by future statements */ + _Py_SourceLocation ff_location; /* location of last future statement */ +} _PyFutureFeatures; + struct _symtable_entry; struct symtable { @@ -44,7 +80,7 @@ struct symtable { consistency with the corresponding compiler structure */ PyObject *st_private; /* name of current class or NULL */ - PyFutureFeatures *st_future; /* module's future features that affect + _PyFutureFeatures *st_future; /* module's future features that affect the symbol table */ int recursion_depth; /* current recursion depth */ int recursion_limit; /* recursion limit */ @@ -58,13 +94,21 @@ typedef struct _symtable_entry { PyObject *ste_varnames; /* list of function parameters */ PyObject *ste_children; /* list of child blocks */ PyObject *ste_directives;/* locations of global and nonlocal statements */ + PyObject *ste_mangled_names; /* set of names for which mangling should be applied */ + _Py_block_ty ste_type; + // Optional string set by symtable.c and used when reporting errors. + // The content of that string is a description of the current "context". + // + // For instance, if we are processing the default value of the type + // variable "T" in "def foo[T = int](): pass", `ste_scope_info` is + // set to "a TypeVar default". + const char *ste_scope_info; + int ste_nested; /* true if block is nested */ - unsigned ste_free : 1; /* true if block has free variables */ - unsigned ste_child_free : 1; /* true if a child block has free vars, - including free refs to globals */ unsigned ste_generator : 1; /* true if namespace is a generator */ unsigned ste_coroutine : 1; /* true if namespace is a coroutine */ + unsigned ste_annotations_used : 1; /* true if there are any annotations in this scope */ _Py_comprehension_ty ste_comprehension; /* Kind of comprehension (if any) */ unsigned ste_varargs : 1; /* true if block has varargs */ unsigned ste_varkeywords : 1; /* true if block has varkeywords */ @@ -79,13 +123,11 @@ typedef struct _symtable_entry { unsigned ste_comp_iter_target : 1; /* true if visiting comprehension target */ unsigned ste_can_see_class_scope : 1; /* true if this block can see names bound in an enclosing class scope */ + unsigned ste_has_docstring : 1; /* true if docstring present */ + unsigned ste_method : 1; /* true if block is a function block defined in class scope */ int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */ - int ste_lineno; /* first line of block */ - int ste_col_offset; /* offset of first line of block */ - int ste_end_lineno; /* end line of block */ - int ste_end_col_offset; /* end offset of first line of block */ - int ste_opt_lineno; /* lineno of last exec or import * */ - int ste_opt_col_offset; /* offset of last exec or import * */ + _Py_SourceLocation ste_loc; /* source location of block */ + struct _symtable_entry *ste_annotation_block; /* symbol table entry for this entry's annotations */ struct symtable *ste_table; } PySTEntryObject; @@ -100,11 +142,13 @@ extern int _PyST_IsFunctionLike(PySTEntryObject *); extern struct symtable* _PySymtable_Build( struct _mod *mod, PyObject *filename, - PyFutureFeatures *future); + _PyFutureFeatures *future); extern PySTEntryObject* _PySymtable_Lookup(struct symtable *, void *); +extern int _PySymtable_LookupOptional(struct symtable *, void *, PySTEntryObject **); extern void _PySymtable_Free(struct symtable *); +extern PyObject *_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name); extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); /* Flags for def-use information */ @@ -114,7 +158,6 @@ extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); #define DEF_PARAM (2<<1) /* formal parameter */ #define DEF_NONLOCAL (2<<2) /* nonlocal stmt */ #define USE (2<<3) /* name is used */ -#define DEF_FREE (2<<4) /* name used but not defined in nested block */ #define DEF_FREE_CLASS (2<<5) /* free variable from class's method */ #define DEF_IMPORT (2<<6) /* assignment occurred via import */ #define DEF_ANNOT (2<<7) /* this name is annotated */ @@ -125,11 +168,12 @@ extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); #define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT) /* GLOBAL_EXPLICIT and GLOBAL_IMPLICIT are used internally by the symbol - table. GLOBAL is returned from PyST_GetScope() for either of them. + table. GLOBAL is returned from _PyST_GetScope() for either of them. It is stored in ste_symbols at bits 13-16. */ #define SCOPE_OFFSET 12 #define SCOPE_MASK (DEF_GLOBAL | DEF_LOCAL | DEF_PARAM | DEF_NONLOCAL) +#define SYMBOL_TO_SCOPE(S) (((S) >> SCOPE_OFFSET) & SCOPE_MASK) #define LOCAL 1 #define GLOBAL_EXPLICIT 2 @@ -137,9 +181,6 @@ extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); #define FREE 4 #define CELL 5 -#define GENERATOR 1 -#define GENERATOR_EXPRESSION 2 - // Used by symtablemodule.c extern struct symtable* _Py_SymtableStringObjectFlags( const char *str, @@ -150,7 +191,7 @@ extern struct symtable* _Py_SymtableStringObjectFlags( int _PyFuture_FromAST( struct _mod * mod, PyObject *filename, - PyFutureFeatures* futures); + _PyFutureFeatures* futures); #ifdef __cplusplus } diff --git a/Include/internal/pycore_sysmodule.h b/Include/internal/pycore_sysmodule.h index 9b8eafd3d6cfd3..99968df54a45f6 100644 --- a/Include/internal/pycore_sysmodule.h +++ b/Include/internal/pycore_sysmodule.h @@ -14,21 +14,14 @@ PyAPI_FUNC(PyObject*) _PySys_GetAttr(PyThreadState *tstate, PyObject *name); // Export for '_pickle' shared extension PyAPI_FUNC(size_t) _PySys_GetSizeOf(PyObject *); -extern int _PySys_Audit( - PyThreadState *tstate, - const char *event, - const char *argFormat, - ...); - -// _PySys_ClearAuditHooks() must not be exported: use extern rather than -// PyAPI_FUNC(). We want minimal exposure of this function. -extern void _PySys_ClearAuditHooks(PyThreadState *tstate); - extern int _PySys_SetAttr(PyObject *, PyObject *); extern int _PySys_ClearAttrString(PyInterpreterState *interp, const char *name, int verbose); +extern int _PySys_SetFlagObj(Py_ssize_t pos, PyObject *new_value); +extern int _PySys_SetIntMaxStrDigits(int maxdigits); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h index 40b28e0ba221ae..205ac5d3781ddd 100644 --- a/Include/internal/pycore_time.h +++ b/Include/internal/pycore_time.h @@ -6,7 +6,7 @@ // Time formats: // // * Seconds. -// * Seconds as a floating point number (C double). +// * Seconds as a floating-point number (C double). // * Milliseconds (10^-3 seconds). // * Microseconds (10^-6 seconds). // * 100 nanoseconds (10^-7 seconds), used on Windows. @@ -132,8 +132,10 @@ PyAPI_FUNC(int) _PyTime_ObjectToTimespec( PyAPI_FUNC(PyTime_t) _PyTime_FromSeconds(int seconds); // Create a timestamp from a number of seconds in double. -// Export for '_socket' shared extension. -PyAPI_FUNC(PyTime_t) _PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round); +extern int _PyTime_FromSecondsDouble( + double seconds, + _PyTime_round_t round, + PyTime_t *result); // Macro to create a timestamp from a number of seconds, no integer overflow. // Only use the macro for small values, prefer _PyTime_FromSeconds(). @@ -247,14 +249,6 @@ typedef struct { double resolution; } _Py_clock_info_t; -// Similar to PyTime_Time() but silently ignore the error and return 0 if the -// internal clock fails. -// -// Use _PyTime_TimeWithInfo() or the public PyTime_Time() to check -// for failure. -// Export for '_random' shared extension. -PyAPI_FUNC(PyTime_t) _PyTime_TimeUnchecked(void); - // Get the current time from the system clock. // On success, set *t and *info (if not NULL), and return 0. // On error, raise an exception and return -1. @@ -262,14 +256,6 @@ extern int _PyTime_TimeWithInfo( PyTime_t *t, _Py_clock_info_t *info); -// Similar to PyTime_Monotonic() but silently ignore the error and return 0 if -// the internal clock fails. -// -// Use _PyTime_MonotonicWithInfo() or the public PyTime_Monotonic() -// to check for failure. -// Export for '_random' shared extension. -PyAPI_FUNC(PyTime_t) _PyTime_MonotonicUnchecked(void); - // Get the time of a monotonic clock, i.e. a clock that cannot go backwards. // The clock is not affected by system clock updates. The reference point of // the returned value is undefined, so that only the difference between the @@ -294,14 +280,6 @@ PyAPI_FUNC(int) _PyTime_localtime(time_t t, struct tm *tm); // Export for '_datetime' shared extension. PyAPI_FUNC(int) _PyTime_gmtime(time_t t, struct tm *tm); -// Similar to PyTime_PerfCounter() but silently ignore the error and return 0 -// if the internal clock fails. -// -// Use _PyTime_PerfCounterWithInfo() or the public PyTime_PerfCounter() to -// check for failure. -// Export for '_lsprof' shared extension. -PyAPI_FUNC(PyTime_t) _PyTime_PerfCounterUnchecked(void); - // Get the performance counter: clock with the highest available resolution to // measure a short duration. @@ -317,12 +295,12 @@ extern int _PyTime_PerfCounterWithInfo( // --- _PyDeadline ----------------------------------------------------------- // Create a deadline. -// Pseudo code: _PyTime_MonotonicUnchecked() + timeout. +// Pseudo code: return PyTime_MonotonicRaw() + timeout // Export for '_ssl' shared extension. PyAPI_FUNC(PyTime_t) _PyDeadline_Init(PyTime_t timeout); // Get remaining time from a deadline. -// Pseudo code: deadline - _PyTime_MonotonicUnchecked(). +// Pseudo code: return deadline - PyTime_MonotonicRaw() // Export for '_ssl' shared extension. PyAPI_FUNC(PyTime_t) _PyDeadline_Get(PyTime_t deadline); diff --git a/Include/internal/pycore_tracemalloc.h b/Include/internal/pycore_tracemalloc.h index 7ddc5bac5d10af..572e8025876319 100644 --- a/Include/internal/pycore_tracemalloc.h +++ b/Include/internal/pycore_tracemalloc.h @@ -11,10 +11,6 @@ extern "C" { #include "pycore_hashtable.h" // _Py_hashtable_t -/* Trace memory blocks allocated by PyMem_RawMalloc() */ -#define TRACE_RAW_MALLOC - - struct _PyTraceMalloc_Config { /* Module initialized? Variable protected by the GIL */ @@ -25,7 +21,7 @@ struct _PyTraceMalloc_Config { } initialized; /* Is tracemalloc tracing memory allocations? - Variable protected by the GIL */ + Variable protected by the TABLES_LOCK(). */ int tracing; /* limit of the number of frames in a traceback, 1 by default. @@ -74,9 +70,7 @@ struct _tracemalloc_runtime_state { PyMemAllocatorEx obj; } allocators; -#if defined(TRACE_RAW_MALLOC) - PyThread_type_lock tables_lock; -#endif + PyMutex tables_lock; /* Size in bytes of currently traced memory. Protected by TABLES_LOCK(). */ size_t traced_memory; @@ -85,14 +79,14 @@ struct _tracemalloc_runtime_state { size_t peak_traced_memory; /* Hash table used as a set to intern filenames: PyObject* => PyObject*. - Protected by the GIL */ + Protected by the TABLES_LOCK(). */ _Py_hashtable_t *filenames; /* Buffer to store a new traceback in traceback_new(). - Protected by the GIL. */ + Protected by the TABLES_LOCK(). */ struct tracemalloc_traceback *traceback; /* Hash table used as a set to intern tracebacks: traceback_t* => traceback_t* - Protected by the GIL */ + Protected by the TABLES_LOCK(). */ _Py_hashtable_t *tracebacks; /* pointer (void*) => trace (trace_t*). Protected by TABLES_LOCK(). */ @@ -144,7 +138,7 @@ extern PyObject* _PyTraceMalloc_GetTraces(void); extern PyObject* _PyTraceMalloc_GetObjectTraceback(PyObject *obj); /* Initialize tracemalloc */ -extern int _PyTraceMalloc_Init(void); +extern PyStatus _PyTraceMalloc_Init(void); /* Start tracemalloc */ extern int _PyTraceMalloc_Start(int max_nframe); diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index e268e6fbbb087b..74e1452763e56c 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -8,17 +8,10 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_brc.h" // struct _brc_thread_state -#include "pycore_freelist.h" // struct _Py_freelist_state -#include "pycore_mimalloc.h" // struct _mimalloc_thread_state -#include "pycore_qsbr.h" // struct qsbr - - -static inline void -_PyThreadState_SetWhence(PyThreadState *tstate, int whence) -{ - tstate->_whence = whence; -} +#include "pycore_brc.h" // struct _brc_thread_state +#include "pycore_freelist_state.h" // struct _Py_freelists +#include "pycore_mimalloc.h" // struct _mimalloc_thread_state +#include "pycore_qsbr.h" // struct qsbr // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The @@ -28,18 +21,40 @@ typedef struct _PyThreadStateImpl { // semi-public fields are in PyThreadState. PyThreadState base; + PyObject *asyncio_running_loop; // Strong reference + PyObject *asyncio_running_task; // Strong reference + struct _qsbr_thread_state *qsbr; // only used by free-threaded build struct llist_node mem_free_queue; // delayed free queue #ifdef Py_GIL_DISABLED struct _gc_thread_state gc; struct _mimalloc_thread_state mimalloc; - struct _Py_object_freelists freelists; + struct _Py_freelists freelists; struct _brc_thread_state brc; + struct { + // The per-thread refcounts + Py_ssize_t *values; + + // Size of the refcounts array. + Py_ssize_t size; + + // If set, don't use per-thread refcounts + int is_finalized; + } refcounts; + + // Index to use to retrieve thread-local bytecode for this thread + int32_t tlbc_index; + + // When >1, code objects do not immortalize their non-string constants. + int suppress_co_const_immortalization; #endif -} _PyThreadStateImpl; +#if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) + Py_ssize_t reftotal; // this thread's total refcount operations +#endif +} _PyThreadStateImpl; #ifdef __cplusplus } diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index 14a9e42c3a324c..dc68d6648b9ec8 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -20,7 +20,8 @@ extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *); #define _PyTuple_ITEMS(op) _Py_RVALUE(_PyTuple_CAST(op)->ob_item) -extern PyObject *_PyTuple_FromArray(PyObject *const *, Py_ssize_t); +PyAPI_FUNC(PyObject *)_PyTuple_FromArray(PyObject *const *, Py_ssize_t); +PyAPI_FUNC(PyObject *)_PyTuple_FromStackRefStealOnSuccess(const union _PyStackRef *, Py_ssize_t); PyAPI_FUNC(PyObject *)_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); typedef struct { diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index c214111fed6f97..581153344a8e05 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -14,14 +14,43 @@ extern "C" { /* state */ +#define _Py_TYPE_VERSION_INT 1 +#define _Py_TYPE_VERSION_FLOAT 2 +#define _Py_TYPE_VERSION_LIST 3 +#define _Py_TYPE_VERSION_TUPLE 4 +#define _Py_TYPE_VERSION_STR 5 +#define _Py_TYPE_VERSION_SET 6 +#define _Py_TYPE_VERSION_FROZEN_SET 7 +#define _Py_TYPE_VERSION_DICT 8 +#define _Py_TYPE_VERSION_BYTEARRAY 9 +#define _Py_TYPE_VERSION_BYTES 10 +#define _Py_TYPE_VERSION_COMPLEX 11 + +#define _Py_TYPE_VERSION_NEXT 16 + + #define _Py_TYPE_BASE_VERSION_TAG (2<<16) #define _Py_MAX_GLOBAL_TYPE_VERSION_TAG (_Py_TYPE_BASE_VERSION_TAG - 1) +/* For now we hard-code this to a value for which we are confident + all the static builtin types will fit (for all builds). */ +#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES 200 +#define _Py_MAX_MANAGED_STATIC_EXT_TYPES 10 +#define _Py_MAX_MANAGED_STATIC_TYPES \ + (_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES + _Py_MAX_MANAGED_STATIC_EXT_TYPES) + struct _types_runtime_state { /* Used to set PyTypeObject.tp_version_tag for core static types. */ // bpo-42745: next_version_tag remains shared by all interpreters // because of static types. unsigned int next_version_tag; + + struct { + struct { + PyTypeObject *type; + int64_t interp_count; + } types[_Py_MAX_MANAGED_STATIC_TYPES]; + } managed_static; }; @@ -42,12 +71,9 @@ struct type_cache { struct type_cache_entry hashtable[1 << MCACHE_SIZE_EXP]; }; -/* For now we hard-code this to a value for which we are confident - all the static builtin types will fit (for all builds). */ -#define _Py_MAX_STATIC_BUILTIN_TYPES 200 - typedef struct { PyTypeObject *type; + int isbuiltin; int readying; int ready; // XXX tp_dict can probably be statically allocated, @@ -59,7 +85,9 @@ typedef struct { are also some diagnostic uses for the list of weakrefs, so we still keep it. */ PyObject *tp_weaklist; -} static_builtin_state; +} managed_static_type_state; + +#define TYPE_VERSION_CACHE_SIZE (1<<12) /* Must be a power of 2 */ struct types_state { /* Used to set PyTypeObject.tp_version_tag. @@ -68,9 +96,60 @@ struct types_state { unsigned int next_version_tag; struct type_cache type_cache; - size_t num_builtins_initialized; - static_builtin_state builtins[_Py_MAX_STATIC_BUILTIN_TYPES]; + + /* Every static builtin type is initialized for each interpreter + during its own initialization, including for the main interpreter + during global runtime initialization. This is done by calling + _PyStaticType_InitBuiltin(). + + The first time a static builtin type is initialized, all the + normal PyType_Ready() stuff happens. The only difference from + normal is that there are three PyTypeObject fields holding + objects which are stored here (on PyInterpreterState) rather + than in the corresponding PyTypeObject fields. Those are: + tp_dict (cls.__dict__), tp_subclasses (cls.__subclasses__), + and tp_weaklist. + + When a subinterpreter is initialized, each static builtin type + is still initialized, but only the interpreter-specific portion, + namely those three objects. + + Those objects are stored in the PyInterpreterState.types.builtins + array, at the index corresponding to each specific static builtin + type. That index (a size_t value) is stored in the tp_subclasses + field. For static builtin types, we re-purposed the now-unused + tp_subclasses to avoid adding another field to PyTypeObject. + In all other cases tp_subclasses holds a dict like before. + (The field was previously defined as PyObject*, but is now void* + to reflect its dual use.) + + The index for each static builtin type isn't statically assigned. + Instead it is calculated the first time a type is initialized + (by the main interpreter). The index matches the order in which + the type was initialized relative to the others. The actual + value comes from the current value of num_builtins_initialized, + as each type is initialized for the main interpreter. + + num_builtins_initialized is incremented once for each static + builtin type. Once initialization is over for a subinterpreter, + the value will be the same as for all other interpreters. */ + struct { + size_t num_initialized; + managed_static_type_state initialized[_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES]; + } builtins; + /* We apply a similar strategy for managed extension modules. */ + struct { + size_t num_initialized; + size_t next_index; + managed_static_type_state initialized[_Py_MAX_MANAGED_STATIC_EXT_TYPES]; + } for_extensions; PyMutex mutex; + + // Borrowed references to type objects whose + // tp_version_tag % TYPE_VERSION_CACHE_SIZE + // once was equal to the index in the table. + // They are cleared when the type object is deallocated. + PyTypeObject *type_version_cache[TYPE_VERSION_CACHE_SIZE]; }; @@ -78,6 +157,7 @@ struct types_state { extern PyStatus _PyTypes_InitTypes(PyInterpreterState *); extern void _PyTypes_FiniTypes(PyInterpreterState *); +extern void _PyTypes_FiniExtTypes(PyInterpreterState *interp); extern void _PyTypes_Fini(PyInterpreterState *); extern void _PyTypes_AfterFork(void); @@ -93,12 +173,34 @@ typedef struct wrapperbase pytype_slotdef; static inline PyObject ** -_PyStaticType_GET_WEAKREFS_LISTPTR(static_builtin_state *state) +_PyStaticType_GET_WEAKREFS_LISTPTR(managed_static_type_state *state) { assert(state != NULL); return &state->tp_weaklist; } +extern int _PyStaticType_InitBuiltin( + PyInterpreterState *interp, + PyTypeObject *type); +extern void _PyStaticType_FiniBuiltin( + PyInterpreterState *interp, + PyTypeObject *type); +extern void _PyStaticType_ClearWeakRefs( + PyInterpreterState *interp, + PyTypeObject *type); +extern managed_static_type_state * _PyStaticType_GetState( + PyInterpreterState *interp, + PyTypeObject *type); + +// Export for '_datetime' shared extension. +PyAPI_FUNC(int) _PyStaticType_InitForExtension( + PyInterpreterState *interp, + PyTypeObject *self); + +// Export for _testinternalcapi extension. +PyAPI_FUNC(PyObject *) _PyStaticType_GetBuiltins(void); + + /* Like PyType_GetModuleState, but skips verification * that type is a heap type with an associated module */ static inline void * @@ -114,11 +216,6 @@ _PyType_GetModuleState(PyTypeObject *type) } -extern int _PyStaticType_InitBuiltin(PyInterpreterState *, PyTypeObject *type); -extern static_builtin_state * _PyStaticType_GetState(PyInterpreterState *, PyTypeObject *); -extern void _PyStaticType_ClearWeakRefs(PyInterpreterState *, PyTypeObject *type); -extern void _PyStaticType_Dealloc(PyInterpreterState *, PyTypeObject *); - // Export for 'math' shared extension, used via _PyType_IsReady() static inline // function PyAPI_FUNC(PyObject *) _PyType_GetDict(PyTypeObject *); @@ -128,6 +225,9 @@ extern PyObject * _PyType_GetMRO(PyTypeObject *type); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); extern int _PyType_HasSubclasses(PyTypeObject *); +// Export for _testinternalcapi extension. +PyAPI_FUNC(PyObject *) _PyType_GetSlotWrapperNames(void); + // PyType_Ready() must be called if _PyType_IsReady() is false. // See also the Py_TPFLAGS_READY flag. static inline int @@ -150,10 +250,35 @@ extern PyTypeObject _PyBufferWrapper_Type; PyAPI_FUNC(PyObject*) _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, PyObject *name, int *meth_found); - -// This is exported for the _testinternalcapi module. -PyAPI_FUNC(PyObject *) _PyType_GetModuleName(PyTypeObject *); - +extern PyObject* _PyType_GetFullyQualifiedName(PyTypeObject *type, char sep); + +// Perform the following operation, in a thread-safe way when required by the +// build mode. +// +// self->tp_flags = (self->tp_flags & ~mask) | flags; +extern void _PyType_SetFlags(PyTypeObject *self, unsigned long mask, + unsigned long flags); +extern int _PyType_AddMethod(PyTypeObject *, PyMethodDef *); + +// Like _PyType_SetFlags(), but apply the operation to self and any of its +// subclasses without Py_TPFLAGS_IMMUTABLETYPE set. +extern void _PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask, + unsigned long flags); + +extern unsigned int _PyType_GetVersionForCurrentState(PyTypeObject *tp); +PyAPI_FUNC(void) _PyType_SetVersion(PyTypeObject *tp, unsigned int version); +PyTypeObject *_PyType_LookupByVersion(unsigned int version); + +// Function pointer type for user-defined validation function that will be +// called by _PyType_Validate(). +// It should return 0 if the validation is passed, otherwise it will return -1. +typedef int (*_py_validate_type)(PyTypeObject *); + +// It will verify the ``ty`` through user-defined validation function ``validate``, +// and if the validation is passed, it will set the ``tp_version`` as valid +// tp_version_tag from the ``ty``. +extern int _PyType_Validate(PyTypeObject *ty, _py_validate_type validate, unsigned int *tp_version); +extern int _PyType_CacheGetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor, uint32_t tp_version); #ifdef __cplusplus } diff --git a/Include/internal/pycore_typevarobject.h b/Include/internal/pycore_typevarobject.h index c9fa97d6820757..4d7556e68cdaee 100644 --- a/Include/internal/pycore_typevarobject.h +++ b/Include/internal/pycore_typevarobject.h @@ -13,10 +13,14 @@ extern PyObject *_Py_make_paramspec(PyThreadState *, PyObject *); extern PyObject *_Py_make_typevartuple(PyThreadState *, PyObject *); extern PyObject *_Py_make_typealias(PyThreadState *, PyObject *); extern PyObject *_Py_subscript_generic(PyThreadState *, PyObject *); +extern PyObject *_Py_set_typeparam_default(PyThreadState *, PyObject *, PyObject *); extern int _Py_initialize_generic(PyInterpreterState *); extern void _Py_clear_generic_types(PyInterpreterState *); +extern int _Py_typing_type_repr(PyUnicodeWriter *, PyObject *); extern PyTypeObject _PyTypeAlias_Type; +extern PyTypeObject _PyNoDefault_Type; +extern PyObject _Py_NoDefaultStruct; #ifdef __cplusplus } diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index fea5ceea0954f4..a60372f58295a9 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -10,8 +10,8 @@ extern "C" { #include "pycore_lock.h" // PyMutex #include "pycore_fileutils.h" // _Py_error_handler -#include "pycore_identifier.h" // _Py_Identifier #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI +#include "pycore_global_objects.h" // _Py_SINGLETON /* --- Characters Type APIs ----------------------------------------------- */ @@ -33,6 +33,7 @@ PyAPI_FUNC(int) _PyUnicode_CheckConsistency( PyAPI_FUNC(void) _PyUnicode_ExactDealloc(PyObject *op); extern Py_ssize_t _PyUnicode_InternedSize(void); +extern Py_ssize_t _PyUnicode_InternedSize_Immortal(void); // Get a copy of a Unicode string. // Export for '_datetime' shared extension. @@ -189,7 +190,7 @@ extern PyObject* _PyUnicode_EncodeCharmap( /* --- Decimal Encoder ---------------------------------------------------- */ -// Coverts a Unicode object holding a decimal value to an ASCII string +// Converts a Unicode object holding a decimal value to an ASCII string // for using in int, float and complex parsers. // Transforms code points that have decimal digit property to the // corresponding ASCII digit code points. Transforms spaces to ASCII. @@ -251,11 +252,7 @@ extern Py_ssize_t _PyUnicode_InsertThousandsGrouping( extern PyObject* _PyUnicode_FormatLong(PyObject *, int, int, int); -/* Fast equality check when the inputs are known to be exact unicode types - and where the hash values are equal (i.e. a very probable match) */ -extern int _PyUnicode_EQ(PyObject *, PyObject *); - -// Equality check. +// Fast equality check when the inputs are known to be exact unicode types. // Export for '_pickle' shared extension. PyAPI_FUNC(int) _PyUnicode_Equal(PyObject *, PyObject *); @@ -275,6 +272,18 @@ extern void _PyUnicode_FiniTypes(PyInterpreterState *); extern PyTypeObject _PyUnicodeASCIIIter_Type; +/* --- Interning ---------------------------------------------------------- */ + +// All these are "ref-neutral", like the public PyUnicode_InternInPlace. + +// Explicit interning routines: +PyAPI_FUNC(void) _PyUnicode_InternMortal(PyInterpreterState *interp, PyObject **); +PyAPI_FUNC(void) _PyUnicode_InternImmortal(PyInterpreterState *interp, PyObject **); +// Left here to help backporting: +PyAPI_FUNC(void) _PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p); +// Only for singletons in the _PyRuntime struct: +extern void _PyUnicode_InternStatic(PyInterpreterState *interp, PyObject **); + /* --- Other API ---------------------------------------------------------- */ struct _Py_unicode_runtime_ids { @@ -311,7 +320,6 @@ struct _Py_unicode_state { struct _Py_unicode_ids ids; }; -extern void _PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p); extern void _PyUnicode_ClearInterned(PyInterpreterState *interp); // Like PyUnicode_AsUTF8(), but check for embedded null characters. diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 64c4cf8c077056..5b78d038fc1192 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -13,2168 +13,3009 @@ static inline void _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { PyObject *string; string = &_Py_ID(CANCELLED); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(FINISHED); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(False); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(JSONDecodeError); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(PENDING); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(Py_Repr); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(TextIOWrapper); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(True); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(WarningMessage); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(_); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_WindowsConsoleIO); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__IOBase_closed); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__abc_tpflags__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__abs__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__abstractmethods__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__add__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__aenter__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__aexit__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__aiter__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__all__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__and__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__anext__); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(__annotate__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__annotations__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__args__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(__asyncio_running_event_loop__); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__await__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__bases__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__bool__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__buffer__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__build_class__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__builtins__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__bytes__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__call__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__cantrace__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__class__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__class_getitem__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__classcell__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__classdict__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__classdictcell__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__complex__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__contains__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(__copy__); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ctypes_from_outparam__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__del__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__delattr__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__delete__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__delitem__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__dict__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__dictoffset__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__dir__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__divmod__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__doc__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__enter__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__eq__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__exit__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__file__); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(__firstlineno__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__float__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__floordiv__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__format__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__fspath__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ge__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__get__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__getattr__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__getattribute__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__getinitargs__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__getitem__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__getnewargs__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__getnewargs_ex__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__getstate__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__gt__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__hash__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__iadd__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__iand__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ifloordiv__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ilshift__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__imatmul__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__imod__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__import__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__imul__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__index__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__init__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__init_subclass__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__instancecheck__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__int__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__invert__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ior__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ipow__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__irshift__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__isabstractmethod__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__isub__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__iter__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__itruediv__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ixor__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__le__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__len__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__length_hint__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__lltrace__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__loader__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__lshift__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__lt__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__main__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__match_args__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__matmul__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__missing__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__mod__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__module__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__mro_entries__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__mul__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__name__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ne__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__neg__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__new__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__newobj__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__newobj_ex__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__next__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__notes__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__or__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__orig_class__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__origin__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__package__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__parameters__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__path__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__pos__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__pow__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__prepare__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__qualname__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__radd__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rand__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rdivmod__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__reduce__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__reduce_ex__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__release_buffer__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__repr__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__reversed__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rfloordiv__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rlshift__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rmatmul__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rmod__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rmul__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__ror__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__round__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rpow__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rrshift__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rshift__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rsub__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rtruediv__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__rxor__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__set__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__set_name__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__setattr__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__setitem__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__setstate__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__sizeof__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__slotnames__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__slots__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__spec__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(__static_attributes__); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__str__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__sub__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__subclasscheck__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__subclasshook__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__truediv__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(__trunc__); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__type_params__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__typing_is_unpacked_typevartuple__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__typing_prepare_subst__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__typing_subst__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__typing_unpacked_tuple_args__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__warningregistry__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__weaklistoffset__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__weakref__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__xor__); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_abc_impl); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_abstract_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_active); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(_align_); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(_annotation); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_anonymous_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_argtypes_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_as_parameter_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_asyncio_future_blocking); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_blksize); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_bootstrap); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_check_retval_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_dealloc_warn); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_feature_version); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_field_types); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_fields_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_finalizing); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_find_and_load); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_fix_up_module); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_flags_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_get_sourcefile); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_handle_fromlist); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_initializing); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_io); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_is_text_encoding); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_isatty_open_only); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_length_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_limbo); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_lock_unlock_module); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_loop); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_needs_com_addref_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(_pack_); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_only_immortal); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_restype_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_showwarnmsg); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_shutdown); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_slotnames); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_strptime); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(_strptime_datetime); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_strptime_datetime_date); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(_swappedbytes_); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_strptime_datetime_datetime); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_strptime_datetime_time); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_type_); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_uninitialized_submodules); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_warn_unawaited_coroutine); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_xoptions); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(a); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(abs_tol); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(access); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(aclose); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(add); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(add_done_callback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(after_in_child); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(after_in_parent); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(aggregate_class); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(alias); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(align); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(allow_code); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(append); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(arg); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(argdefs); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(args); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(arguments); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(argv); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(as_integer_ratio); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(asend); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ast); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(athrow); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(attribute); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(authorizer_callback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(autocommit); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(b); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(backtick); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(base); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(before); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(big); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(binary_form); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(bit_size); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(block); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(bound); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(buffer); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(buffer_callback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(buffer_size); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(buffering); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(buffers); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(bufsize); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(builtins); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(byteorder); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(bytes); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(bytes_per_sep); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(c); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(c_call); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(c_exception); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(c_return); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(cached_datetime_module); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cached_statements); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cadata); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cafile); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(call); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(call_exception_handler); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(call_soon); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(callback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cancel); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(capath); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(category); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cb_type); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(certfile); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(check_same_thread); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(clear); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(close); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(closed); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(closefd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(closure); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_argcount); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_cellvars); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_code); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_consts); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_exceptiontable); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_filename); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_firstlineno); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_flags); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_freevars); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_kwonlyargcount); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_linetable); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_name); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_names); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_nlocals); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_posonlyargcount); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_qualname); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_stacksize); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(co_varnames); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(code); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(col_offset); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(command); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(comment_factory); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(compile_mode); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(consts); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(context); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(contravariant); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cookie); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(copy); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(copyreg); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(coro); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(count); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(covariant); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cwd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(d); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(data); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(database); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(day); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(decode); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(decoder); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(default); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(defaultaction); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(delete); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(depth); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(desired_access); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(detect_types); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(deterministic); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(device); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dict); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dictcomp); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(difference_update); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(digest); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(digest_size); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(digestmod); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dir_fd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(discard); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dispatch_table); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(displayhook); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dklen); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(doc); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(done); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dont_inherit); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dst); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dst_dir_fd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(e); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(eager_start); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(effective_ids); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(element_factory); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(encode); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(encoding); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(end); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(end_col_offset); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(end_lineno); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(end_offset); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(endpos); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(entrypoint); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(env); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(errors); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(event); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(eventmask); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(exc_type); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(exc_value); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(excepthook); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(exception); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(existing_file_name); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(exp); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(extend); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(extra_tokens); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(f); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(facility); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(factory); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(false); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(family); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fanout); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fd2); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fdel); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fget); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(fields); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(file); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(file_actions); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(filename); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fileno); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(filepath); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fillvalue); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(filter); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(filters); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(final); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(find_class); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fix_imports); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(flags); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(flush); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fold); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(follow_symlinks); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(format); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(format_spec); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(from_param); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fromlist); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fromtimestamp); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fromutc); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(fset); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(func); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(future); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(g); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(generation); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(genexpr); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(get); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(get_debug); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(get_event_loop); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(get_loop); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(get_source); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(getattr); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(getstate); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(gid); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(globals); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(groupindex); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(groups); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(h); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(handle); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(handle_seq); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(has_location); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(hash_name); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(header); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(headers); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(hi); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(hook); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(hour); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(id); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ident); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(identity_hint); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ignore); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(imag); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(importlib); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(in_fd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(incoming); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(index); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(indexgroup); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(inf); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(infer_variance); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(inherit_handle); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(inheritable); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(initial); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(initial_bytes); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(initial_owner); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(initial_state); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(initial_value); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(initval); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(inner_size); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(input); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(insert_comments); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(insert_pis); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(instructions); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(intern); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(intersection); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(interval); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(is_running); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(is_struct); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(isatty); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(isinstance); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(isoformat); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(isolation_level); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(istext); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(item); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(items); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(iter); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(iterable); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(iterations); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(join); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(jump); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(keepends); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(key); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(keyfile); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(keys); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(kind); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(kw); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(kw1); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(kw2); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(kwdefaults); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(label); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(lambda); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(last); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(last_exc); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(last_node); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(last_traceback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(last_type); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(last_value); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(latin1); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(leaf_size); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(len); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(length); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(level); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(limit); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(line); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(line_buffering); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(lineno); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(listcomp); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(little); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(lo); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(locale); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(locals); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(logoption); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(loop); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(manual_reset); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(mapping); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(match); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(max_length); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxdigits); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxevents); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxlen); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxmem); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxsplit); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxvalue); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(memLevel); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(memlimit); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(message); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(metaclass); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(metadata); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(method); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(microsecond); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(milliseconds); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(minute); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(mod); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(mode); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(module); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(module_globals); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(modules); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(month); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(mro); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(msg); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(mutex); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(mycmp); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(n); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(n_arg); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(n_fields); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(n_sequence_fields); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(n_unnamed_fields); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(name); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(name_from); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(namespace_separator); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(namespaces); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(narg); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ndigits); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(nested); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(new_file_name); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(new_limit); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(newline); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(newlines); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(next); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(nlocals); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(node_depth); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(node_offset); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ns); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(nstype); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(nt); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(null); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(number); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(obj); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(object); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(offset); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(offset_dst); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(offset_src); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(on_type_read); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(onceregistry); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(only_keys); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(oparg); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(opcode); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(open); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(opener); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(operation); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(optimize); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(options); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(order); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(origin); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(out_fd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(outgoing); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(outpath); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(overlapped); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(owner); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(p); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pages); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(parent); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(password); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(path); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pattern); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(peek); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(persistent_id); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(persistent_load); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(person); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pi_factory); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pid); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(policy); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pos); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pos1); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pos2); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(posix); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(print_file_and_line); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(priority); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(progress); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(progress_handler); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(progress_routine); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(proto); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(protocol); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ps1); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ps2); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(query); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(quotetabs); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(r); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(raw); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(read); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(read1); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(readable); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(readall); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(readinto); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(readinto1); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(readline); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(readonly); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(real); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reducer_override); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(registry); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(rel_tol); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(release); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reload); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(repl); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(replace); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reserved); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reset); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(resetids); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(return); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reverse); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reversed); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(s); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(salt); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sched_priority); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(scheduler); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(second); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(security_attributes); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(seek); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(seekable); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(selectors); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(self); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(send); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sep); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sequence); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(server_hostname); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(server_side); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(session); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(setcomp); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(setpgroup); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(setsid); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(setsigdef); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(setsigmask); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(setstate); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(shape); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(show_cmd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(signed); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(size); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sizehint); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(skip_file_prefixes); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sleep); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sock); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sort); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(source); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(source_traceback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(spam); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(src); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(src_dir_fd); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(stacklevel); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(start); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(statement); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(status); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(stderr); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(stdin); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(stdout); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(step); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(steps); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(store_name); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(strategy); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(strftime); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(strict); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(strict_mode); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(string); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sub_key); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(subcalls); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(symmetric_difference_update); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tabsize); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tag); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(target); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(target_is_directory); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(task); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tb_frame); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tb_lasti); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tb_lineno); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tb_next); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tell); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(template); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(term); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(text); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(threading); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(throw); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(timeout); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(timer); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(times); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(timetuple); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(timeunit); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(top); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(trace_callback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(traceback); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(trailers); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(translate); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(true); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(truncate); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(twice); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(txt); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(type); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(type_params); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tz); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tzinfo); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tzname); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(uid); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(unlink); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(unraisablehook); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(uri); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(usedforsecurity); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(value); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(values); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(version); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(volume); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(wait_all); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(warn_on_full_buffer); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(warnings); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(warnoptions); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(wbits); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(week); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(weekday); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(which); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(who); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(withdata); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(writable); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(write); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(write_through); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(x); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(year); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(zdict); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(empty); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(dbl_percent); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(dot_locals); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(defaults); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(format); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(generic_base); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(kwdefaults); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(type_params); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(str_replace_inf); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_null); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_dictcomp); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_genexpr); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_lambda); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_listcomp); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_module); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_setcomp); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_string); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(anon_unknown); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(json_decoder); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(list_err); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(utf_8); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(dbl_open_br); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(dbl_close_br); + _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); + assert(PyUnicode_GET_LENGTH(string) != 1); } /* End auto-generated code */ #ifdef __cplusplus diff --git a/Include/internal/pycore_uniqueid.h b/Include/internal/pycore_uniqueid.h new file mode 100644 index 00000000000000..9d3c866a704894 --- /dev/null +++ b/Include/internal/pycore_uniqueid.h @@ -0,0 +1,77 @@ +#ifndef Py_INTERNAL_UNIQUEID_H +#define Py_INTERNAL_UNIQUEID_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef Py_GIL_DISABLED + +// This contains code for allocating unique ids to objects for per-thread +// reference counting. +// +// Per-thread reference counting is used along with deferred reference +// counting to avoid scaling bottlenecks due to reference count contention. +// +// An id of 0 is used to indicate that an object doesn't use per-thread +// refcounting. This value is used when the object is finalized by the GC +// and during interpreter shutdown to allow the object to be +// deallocated promptly when the object's refcount reaches zero. +// +// Each entry implicitly represents a unique id based on its offset in the +// table. Non-allocated entries form a free-list via the 'next' pointer. +// Allocated entries store the corresponding PyObject. +typedef union _Py_unique_id_entry { + // Points to the next free type id, when part of the freelist + union _Py_unique_id_entry *next; + + // Stores the object when the id is assigned + PyObject *obj; +} _Py_unique_id_entry; + +struct _Py_unique_id_pool { + PyMutex mutex; + + // combined table of object with allocated unique ids and unallocated ids. + _Py_unique_id_entry *table; + + // Next entry to allocate inside 'table' or NULL + _Py_unique_id_entry *freelist; + + // size of 'table' + Py_ssize_t size; +}; + +#define _Py_INVALID_UNIQUE_ID 0 + +// Assigns the next id from the pool of ids. +extern Py_ssize_t _PyObject_AssignUniqueId(PyObject *obj); + +// Releases the allocated id back to the pool. +extern void _PyObject_ReleaseUniqueId(Py_ssize_t unique_id); + +// Releases the allocated id back to the pool. +extern void _PyObject_DisablePerThreadRefcounting(PyObject *obj); + +// Merges the per-thread reference counts into the corresponding objects. +extern void _PyObject_MergePerThreadRefcounts(_PyThreadStateImpl *tstate); + +// Like _PyObject_MergePerThreadRefcounts, but also frees the per-thread +// array of refcounts. +extern void _PyObject_FinalizePerThreadRefcounts(_PyThreadStateImpl *tstate); + +// Frees the interpreter's pool of type ids. +extern void _PyObject_FinalizeUniqueIdPool(PyInterpreterState *interp); + +// Increfs the object, resizing the thread-local refcount array if necessary. +PyAPI_FUNC(void) _PyObject_ThreadIncrefSlow(PyObject *obj, size_t idx); + +#endif /* Py_GIL_DISABLED */ + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_UNIQUEID_H */ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 8f71eab44d914d..7a6c0d22fe24e5 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -11,249 +11,289 @@ extern "C" { #define _EXIT_TRACE 300 #define _SET_IP 301 -#define _BEFORE_ASYNC_WITH BEFORE_ASYNC_WITH -#define _BEFORE_WITH BEFORE_WITH #define _BINARY_OP 302 #define _BINARY_OP_ADD_FLOAT 303 #define _BINARY_OP_ADD_INT 304 #define _BINARY_OP_ADD_UNICODE 305 -#define _BINARY_OP_MULTIPLY_FLOAT 306 -#define _BINARY_OP_MULTIPLY_INT 307 -#define _BINARY_OP_SUBTRACT_FLOAT 308 -#define _BINARY_OP_SUBTRACT_INT 309 -#define _BINARY_SLICE BINARY_SLICE -#define _BINARY_SUBSCR 310 +#define _BINARY_OP_EXTEND 306 +#define _BINARY_OP_INPLACE_ADD_UNICODE 307 +#define _BINARY_OP_MULTIPLY_FLOAT 308 +#define _BINARY_OP_MULTIPLY_INT 309 +#define _BINARY_OP_SUBTRACT_FLOAT 310 +#define _BINARY_OP_SUBTRACT_INT 311 +#define _BINARY_SLICE 312 +#define _BINARY_SUBSCR 313 +#define _BINARY_SUBSCR_CHECK_FUNC 314 #define _BINARY_SUBSCR_DICT BINARY_SUBSCR_DICT -#define _BINARY_SUBSCR_GETITEM BINARY_SUBSCR_GETITEM +#define _BINARY_SUBSCR_INIT_CALL 315 #define _BINARY_SUBSCR_LIST_INT BINARY_SUBSCR_LIST_INT #define _BINARY_SUBSCR_STR_INT BINARY_SUBSCR_STR_INT #define _BINARY_SUBSCR_TUPLE_INT BINARY_SUBSCR_TUPLE_INT -#define _BUILD_CONST_KEY_MAP BUILD_CONST_KEY_MAP #define _BUILD_LIST BUILD_LIST #define _BUILD_MAP BUILD_MAP #define _BUILD_SET BUILD_SET #define _BUILD_SLICE BUILD_SLICE #define _BUILD_STRING BUILD_STRING #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL 311 -#define _CALL_ALLOC_AND_ENTER_INIT CALL_ALLOC_AND_ENTER_INIT -#define _CALL_BUILTIN_CLASS CALL_BUILTIN_CLASS -#define _CALL_BUILTIN_FAST CALL_BUILTIN_FAST -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS CALL_BUILTIN_FAST_WITH_KEYWORDS -#define _CALL_BUILTIN_O CALL_BUILTIN_O -#define _CALL_FUNCTION_EX CALL_FUNCTION_EX +#define _CALL_BUILTIN_CLASS 316 +#define _CALL_BUILTIN_FAST 317 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 318 +#define _CALL_BUILTIN_O 319 #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 #define _CALL_ISINSTANCE CALL_ISINSTANCE -#define _CALL_KW CALL_KW +#define _CALL_KW_NON_PY 320 #define _CALL_LEN CALL_LEN -#define _CALL_METHOD_DESCRIPTOR_FAST CALL_METHOD_DESCRIPTOR_FAST -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS -#define _CALL_METHOD_DESCRIPTOR_NOARGS CALL_METHOD_DESCRIPTOR_NOARGS -#define _CALL_METHOD_DESCRIPTOR_O CALL_METHOD_DESCRIPTOR_O -#define _CALL_PY_WITH_DEFAULTS CALL_PY_WITH_DEFAULTS -#define _CALL_STR_1 CALL_STR_1 -#define _CALL_TUPLE_1 CALL_TUPLE_1 +#define _CALL_LIST_APPEND CALL_LIST_APPEND +#define _CALL_METHOD_DESCRIPTOR_FAST 321 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 322 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 323 +#define _CALL_METHOD_DESCRIPTOR_O 324 +#define _CALL_NON_PY_GENERAL 325 +#define _CALL_STR_1 326 +#define _CALL_TUPLE_1 327 #define _CALL_TYPE_1 CALL_TYPE_1 -#define _CHECK_ATTR_CLASS 312 -#define _CHECK_ATTR_METHOD_LAZY_DICT 313 -#define _CHECK_ATTR_MODULE 314 -#define _CHECK_ATTR_WITH_HINT 315 -#define _CHECK_BUILTINS 316 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 317 +#define _CHECK_AND_ALLOCATE_OBJECT 328 +#define _CHECK_ATTR_CLASS 329 +#define _CHECK_ATTR_METHOD_LAZY_DICT 330 +#define _CHECK_ATTR_MODULE_PUSH_KEYS 331 +#define _CHECK_ATTR_WITH_HINT 332 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 333 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION_EXACT_ARGS 318 -#define _CHECK_GLOBALS 319 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 320 -#define _CHECK_PEP_523 321 -#define _CHECK_STACK_SPACE 322 -#define _CHECK_VALIDITY 323 -#define _CHECK_VALIDITY_AND_SET_IP 324 -#define _COLD_EXIT 325 -#define _COMPARE_OP 326 -#define _COMPARE_OP_FLOAT 327 -#define _COMPARE_OP_INT 328 -#define _COMPARE_OP_STR 329 -#define _CONTAINS_OP CONTAINS_OP +#define _CHECK_FUNCTION 334 +#define _CHECK_FUNCTION_EXACT_ARGS 335 +#define _CHECK_FUNCTION_VERSION 336 +#define _CHECK_FUNCTION_VERSION_INLINE 337 +#define _CHECK_FUNCTION_VERSION_KW 338 +#define _CHECK_IS_NOT_PY_CALLABLE 339 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 340 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 341 +#define _CHECK_METHOD_VERSION 342 +#define _CHECK_METHOD_VERSION_KW 343 +#define _CHECK_PEP_523 344 +#define _CHECK_PERIODIC 345 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 346 +#define _CHECK_STACK_SPACE 347 +#define _CHECK_STACK_SPACE_OPERAND 348 +#define _CHECK_VALIDITY 349 +#define _CHECK_VALIDITY_AND_SET_IP 350 +#define _COMPARE_OP 351 +#define _COMPARE_OP_FLOAT 352 +#define _COMPARE_OP_INT 353 +#define _COMPARE_OP_STR 354 +#define _CONTAINS_OP 355 +#define _CONTAINS_OP_DICT CONTAINS_OP_DICT +#define _CONTAINS_OP_SET CONTAINS_OP_SET #define _CONVERT_VALUE CONVERT_VALUE #define _COPY COPY #define _COPY_FREE_VARS COPY_FREE_VARS +#define _CREATE_INIT_FRAME 356 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR +#define _DEOPT 357 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE +#define _DO_CALL 358 +#define _DO_CALL_FUNCTION_EX 359 +#define _DO_CALL_KW 360 +#define _DYNAMIC_EXIT 361 +#define _END_FOR END_FOR #define _END_SEND END_SEND +#define _ERROR_POP_N 362 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _FATAL_ERROR 330 +#define _EXPAND_METHOD 363 +#define _EXPAND_METHOD_KW 364 +#define _FATAL_ERROR 365 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 331 -#define _FOR_ITER_GEN FOR_ITER_GEN -#define _FOR_ITER_TIER_TWO 332 +#define _FOR_ITER 366 +#define _FOR_ITER_GEN_FRAME 367 +#define _FOR_ITER_TIER_TWO 368 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 333 -#define _GUARD_BOTH_INT 334 -#define _GUARD_BOTH_UNICODE 335 -#define _GUARD_BUILTINS_VERSION 336 -#define _GUARD_DORV_VALUES 337 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 338 -#define _GUARD_GLOBALS_VERSION 339 -#define _GUARD_IS_FALSE_POP 340 -#define _GUARD_IS_NONE_POP 341 -#define _GUARD_IS_NOT_NONE_POP 342 -#define _GUARD_IS_TRUE_POP 343 -#define _GUARD_KEYS_VERSION 344 -#define _GUARD_NOT_EXHAUSTED_LIST 345 -#define _GUARD_NOT_EXHAUSTED_RANGE 346 -#define _GUARD_NOT_EXHAUSTED_TUPLE 347 -#define _GUARD_TYPE_VERSION 348 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 349 -#define _INIT_CALL_PY_EXACT_ARGS 350 -#define _INIT_CALL_PY_EXACT_ARGS_0 351 -#define _INIT_CALL_PY_EXACT_ARGS_1 352 -#define _INIT_CALL_PY_EXACT_ARGS_2 353 -#define _INIT_CALL_PY_EXACT_ARGS_3 354 -#define _INIT_CALL_PY_EXACT_ARGS_4 355 -#define _INSTRUMENTED_CALL INSTRUMENTED_CALL +#define _GUARD_BINARY_OP_EXTEND 369 +#define _GUARD_BOTH_FLOAT 370 +#define _GUARD_BOTH_INT 371 +#define _GUARD_BOTH_UNICODE 372 +#define _GUARD_BUILTINS_VERSION_PUSH_KEYS 373 +#define _GUARD_DORV_NO_DICT 374 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 375 +#define _GUARD_GLOBALS_VERSION 376 +#define _GUARD_GLOBALS_VERSION_PUSH_KEYS 377 +#define _GUARD_IS_FALSE_POP 378 +#define _GUARD_IS_NONE_POP 379 +#define _GUARD_IS_NOT_NONE_POP 380 +#define _GUARD_IS_TRUE_POP 381 +#define _GUARD_KEYS_VERSION 382 +#define _GUARD_NOS_FLOAT 383 +#define _GUARD_NOS_INT 384 +#define _GUARD_NOT_EXHAUSTED_LIST 385 +#define _GUARD_NOT_EXHAUSTED_RANGE 386 +#define _GUARD_NOT_EXHAUSTED_TUPLE 387 +#define _GUARD_TOS_FLOAT 388 +#define _GUARD_TOS_INT 389 +#define _GUARD_TYPE_VERSION 390 +#define _GUARD_TYPE_VERSION_AND_LOCK 391 +#define _IMPORT_FROM IMPORT_FROM +#define _IMPORT_NAME IMPORT_NAME +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 392 +#define _INIT_CALL_PY_EXACT_ARGS 393 +#define _INIT_CALL_PY_EXACT_ARGS_0 394 +#define _INIT_CALL_PY_EXACT_ARGS_1 395 +#define _INIT_CALL_PY_EXACT_ARGS_2 396 +#define _INIT_CALL_PY_EXACT_ARGS_3 397 +#define _INIT_CALL_PY_EXACT_ARGS_4 398 #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION -#define _INSTRUMENTED_JUMP_BACKWARD INSTRUMENTED_JUMP_BACKWARD #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD +#define _INSTRUMENTED_LINE INSTRUMENTED_LINE #define _INSTRUMENTED_LOAD_SUPER_ATTR INSTRUMENTED_LOAD_SUPER_ATTR +#define _INSTRUMENTED_NOT_TAKEN INSTRUMENTED_NOT_TAKEN #define _INSTRUMENTED_POP_JUMP_IF_FALSE INSTRUMENTED_POP_JUMP_IF_FALSE #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _INSTRUMENTED_RESUME INSTRUMENTED_RESUME -#define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST -#define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE -#define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 356 -#define _IS_NONE 357 +#define _IS_NONE 399 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 358 -#define _ITER_CHECK_RANGE 359 -#define _ITER_CHECK_TUPLE 360 -#define _ITER_JUMP_LIST 361 -#define _ITER_JUMP_RANGE 362 -#define _ITER_JUMP_TUPLE 363 -#define _ITER_NEXT_LIST 364 -#define _ITER_NEXT_RANGE 365 -#define _ITER_NEXT_TUPLE 366 -#define _JUMP_TO_TOP 367 +#define _ITER_CHECK_LIST 400 +#define _ITER_CHECK_RANGE 401 +#define _ITER_CHECK_TUPLE 402 +#define _ITER_JUMP_LIST 403 +#define _ITER_JUMP_RANGE 404 +#define _ITER_JUMP_TUPLE 405 +#define _ITER_NEXT_LIST 406 +#define _ITER_NEXT_RANGE 407 +#define _ITER_NEXT_TUPLE 408 +#define _JUMP_TO_TOP 409 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR -#define _LOAD_ATTR 368 -#define _LOAD_ATTR_CLASS 369 -#define _LOAD_ATTR_CLASS_0 370 -#define _LOAD_ATTR_CLASS_1 371 +#define _LOAD_ATTR 410 +#define _LOAD_ATTR_CLASS 411 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 372 -#define _LOAD_ATTR_INSTANCE_VALUE_0 373 -#define _LOAD_ATTR_INSTANCE_VALUE_1 374 -#define _LOAD_ATTR_METHOD_LAZY_DICT 375 -#define _LOAD_ATTR_METHOD_NO_DICT 376 -#define _LOAD_ATTR_METHOD_WITH_VALUES 377 -#define _LOAD_ATTR_MODULE 378 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 379 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 380 -#define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY -#define _LOAD_ATTR_SLOT 381 -#define _LOAD_ATTR_SLOT_0 382 -#define _LOAD_ATTR_SLOT_1 383 -#define _LOAD_ATTR_WITH_HINT 384 +#define _LOAD_ATTR_INSTANCE_VALUE 412 +#define _LOAD_ATTR_METHOD_LAZY_DICT 413 +#define _LOAD_ATTR_METHOD_NO_DICT 414 +#define _LOAD_ATTR_METHOD_WITH_VALUES 415 +#define _LOAD_ATTR_MODULE 416 +#define _LOAD_ATTR_MODULE_FROM_KEYS 417 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 418 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 419 +#define _LOAD_ATTR_PROPERTY_FRAME 420 +#define _LOAD_ATTR_SLOT 421 +#define _LOAD_ATTR_WITH_HINT 422 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS +#define _LOAD_BYTECODE 423 +#define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 385 -#define _LOAD_CONST_INLINE_BORROW 386 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 387 -#define _LOAD_CONST_INLINE_WITH_NULL 388 +#define _LOAD_CONST_IMMORTAL LOAD_CONST_IMMORTAL +#define _LOAD_CONST_INLINE 424 +#define _LOAD_CONST_INLINE_BORROW 425 +#define _LOAD_CONST_MORTAL LOAD_CONST_MORTAL #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 389 -#define _LOAD_FAST_0 390 -#define _LOAD_FAST_1 391 -#define _LOAD_FAST_2 392 -#define _LOAD_FAST_3 393 -#define _LOAD_FAST_4 394 -#define _LOAD_FAST_5 395 -#define _LOAD_FAST_6 396 -#define _LOAD_FAST_7 397 +#define _LOAD_FAST 426 +#define _LOAD_FAST_0 427 +#define _LOAD_FAST_1 428 +#define _LOAD_FAST_2 429 +#define _LOAD_FAST_3 430 +#define _LOAD_FAST_4 431 +#define _LOAD_FAST_5 432 +#define _LOAD_FAST_6 433 +#define _LOAD_FAST_7 434 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 398 -#define _LOAD_GLOBAL_BUILTINS 399 -#define _LOAD_GLOBAL_MODULE 400 +#define _LOAD_GLOBAL 435 +#define _LOAD_GLOBAL_BUILTINS 436 +#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 437 +#define _LOAD_GLOBAL_MODULE 438 +#define _LOAD_GLOBAL_MODULE_FROM_KEYS 439 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME +#define _LOAD_SMALL_INT 440 +#define _LOAD_SMALL_INT_0 441 +#define _LOAD_SMALL_INT_1 442 +#define _LOAD_SMALL_INT_2 443 +#define _LOAD_SMALL_INT_3 444 +#define _LOAD_SPECIAL LOAD_SPECIAL #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD +#define _MAKE_CALLARGS_A_TUPLE 445 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION +#define _MAKE_WARM 446 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE +#define _MAYBE_EXPAND_METHOD 447 +#define _MAYBE_EXPAND_METHOD_KW 448 +#define _MONITOR_CALL 449 +#define _MONITOR_JUMP_BACKWARD 450 +#define _MONITOR_RESUME 451 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_FRAME 401 -#define _POP_JUMP_IF_FALSE 402 -#define _POP_JUMP_IF_TRUE 403 +#define _POP_JUMP_IF_FALSE 452 +#define _POP_JUMP_IF_TRUE 453 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 404 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 454 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 405 +#define _PUSH_FRAME 455 #define _PUSH_NULL PUSH_NULL +#define _PUSH_NULL_CONDITIONAL 456 +#define _PY_FRAME_GENERAL 457 +#define _PY_FRAME_KW 458 +#define _QUICKEN_RESUME 459 +#define _REPLACE_WITH_TRUE 460 #define _RESUME_CHECK RESUME_CHECK -#define _SAVE_RETURN_OFFSET 406 -#define _SEND 407 -#define _SEND_GEN SEND_GEN +#define _RETURN_GENERATOR RETURN_GENERATOR +#define _RETURN_VALUE RETURN_VALUE +#define _SAVE_RETURN_OFFSET 461 +#define _SEND 462 +#define _SEND_GEN_FRAME 463 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 408 -#define _STORE_ATTR 409 -#define _STORE_ATTR_INSTANCE_VALUE 410 -#define _STORE_ATTR_SLOT 411 -#define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT +#define _START_EXECUTOR 464 +#define _STORE_ATTR 465 +#define _STORE_ATTR_INSTANCE_VALUE 466 +#define _STORE_ATTR_SLOT 467 +#define _STORE_ATTR_WITH_HINT 468 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 412 -#define _STORE_FAST_0 413 -#define _STORE_FAST_1 414 -#define _STORE_FAST_2 415 -#define _STORE_FAST_3 416 -#define _STORE_FAST_4 417 -#define _STORE_FAST_5 418 -#define _STORE_FAST_6 419 -#define _STORE_FAST_7 420 +#define _STORE_FAST 469 +#define _STORE_FAST_0 470 +#define _STORE_FAST_1 471 +#define _STORE_FAST_2 472 +#define _STORE_FAST_3 473 +#define _STORE_FAST_4 474 +#define _STORE_FAST_5 475 +#define _STORE_FAST_6 476 +#define _STORE_FAST_7 477 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 421 +#define _STORE_SLICE 478 +#define _STORE_SUBSCR 479 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TO_BOOL 422 -#define _TO_BOOL_ALWAYS_TRUE TO_BOOL_ALWAYS_TRUE +#define _TIER2_RESUME_CHECK 480 +#define _TO_BOOL 481 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -263,12 +303,13 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 423 +#define _UNPACK_SEQUENCE 482 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define MAX_UOP_ID 423 +#define _YIELD_VALUE YIELD_VALUE +#define MAX_UOP_ID 482 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 7f921a6cd3f4c8..80f89defb7e2eb 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -15,11 +15,15 @@ extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1]; extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1]; extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; +extern int _PyUop_num_popped(int opcode, int oparg); + #ifdef NEED_OPCODE_METADATA const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_NOP] = HAS_PURE_FLAG, + [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_RESUME_CHECK] = HAS_DEOPT_FLAG, - [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG, + [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, @@ -31,48 +35,64 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG, - [_STORE_FAST_0] = HAS_LOCAL_FLAG, - [_STORE_FAST_1] = HAS_LOCAL_FLAG, - [_STORE_FAST_2] = HAS_LOCAL_FLAG, - [_STORE_FAST_3] = HAS_LOCAL_FLAG, - [_STORE_FAST_4] = HAS_LOCAL_FLAG, - [_STORE_FAST_5] = HAS_LOCAL_FLAG, - [_STORE_FAST_6] = HAS_LOCAL_FLAG, - [_STORE_FAST_7] = HAS_LOCAL_FLAG, - [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, + [_LOAD_CONST_MORTAL] = HAS_ARG_FLAG | HAS_CONST_FLAG, + [_LOAD_CONST_IMMORTAL] = HAS_ARG_FLAG | HAS_CONST_FLAG, + [_LOAD_SMALL_INT_0] = 0, + [_LOAD_SMALL_INT_1] = 0, + [_LOAD_SMALL_INT_2] = 0, + [_LOAD_SMALL_INT_3] = 0, + [_LOAD_SMALL_INT] = HAS_ARG_FLAG, + [_STORE_FAST_0] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_STORE_FAST_1] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_STORE_FAST_2] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_STORE_FAST_3] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_STORE_FAST_4] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_STORE_FAST_5] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_STORE_FAST_6] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_STORE_FAST_7] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, [_POP_TOP] = HAS_PURE_FLAG, [_PUSH_NULL] = HAS_PURE_FLAG, + [_END_FOR] = HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG, [_END_SEND] = HAS_PURE_FLAG, [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNARY_NOT] = HAS_PURE_FLAG, [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_TO_BOOL_BOOL] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, - [_TO_BOOL_INT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_TO_BOOL_LIST] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_TO_BOOL_NONE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_TO_BOOL_STR] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_TO_BOOL_ALWAYS_TRUE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_TO_BOOL_BOOL] = HAS_EXIT_FLAG, + [_TO_BOOL_INT] = HAS_EXIT_FLAG, + [_TO_BOOL_LIST] = HAS_EXIT_FLAG, + [_TO_BOOL_NONE] = HAS_EXIT_FLAG, + [_TO_BOOL_STR] = HAS_EXIT_FLAG, + [_REPLACE_WITH_TRUE] = 0, [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_BOTH_INT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_BOTH_INT] = HAS_EXIT_FLAG, + [_GUARD_NOS_INT] = HAS_EXIT_FLAG, + [_GUARD_TOS_INT] = HAS_EXIT_FLAG, [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_GUARD_BOTH_FLOAT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, - [_BINARY_OP_MULTIPLY_FLOAT] = HAS_PURE_FLAG, - [_BINARY_OP_ADD_FLOAT] = HAS_PURE_FLAG, - [_BINARY_OP_SUBTRACT_FLOAT] = HAS_PURE_FLAG, - [_GUARD_BOTH_UNICODE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, - [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_GUARD_BOTH_FLOAT] = HAS_EXIT_FLAG, + [_GUARD_NOS_FLOAT] = HAS_EXIT_FLAG, + [_GUARD_TOS_FLOAT] = HAS_EXIT_FLAG, + [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_GUARD_BOTH_UNICODE] = HAS_EXIT_FLAG, + [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, + [_GUARD_BINARY_OP_EXTEND] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_EXTEND] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG, - [_BINARY_SUBSCR_STR_INT] = HAS_DEOPT_FLAG, - [_BINARY_SUBSCR_TUPLE_INT] = HAS_DEOPT_FLAG, + [_BINARY_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_SUBSCR_STR_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_SUBSCR_TUPLE_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SUBSCR_DICT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_SUBSCR_CHECK_FUNC] = HAS_DEOPT_FLAG, + [_BINARY_SUBSCR_INIT_CALL] = 0, [_LIST_APPEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_SET_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -81,15 +101,17 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_POP_FRAME] = HAS_ESCAPES_FLAG, + [_RETURN_VALUE] = 0, [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, - [_LOAD_ASSERTION_ERROR] = 0, + [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, @@ -98,64 +120,65 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_LOCALS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_FROM_DICT_OR_GLOBALS] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_PUSH_NULL_CONDITIONAL] = HAS_ARG_FLAG, [_GUARD_GLOBALS_VERSION] = HAS_DEOPT_FLAG, - [_GUARD_BUILTINS_VERSION] = HAS_DEOPT_FLAG, - [_LOAD_GLOBAL_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_LOAD_GLOBAL_BUILTINS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_DELETE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG, - [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_GLOBALS_VERSION_PUSH_KEYS] = HAS_DEOPT_FLAG, + [_GUARD_BUILTINS_VERSION_PUSH_KEYS] = HAS_DEOPT_FLAG, + [_LOAD_GLOBAL_MODULE_FROM_KEYS] = HAS_DEOPT_FLAG, + [_LOAD_GLOBAL_BUILTINS_FROM_KEYS] = HAS_DEOPT_FLAG, + [_DELETE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG, [_COPY_FREE_VARS] = HAS_ARG_FLAG, - [_BUILD_STRING] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BUILD_TUPLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BUILD_LIST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BUILD_STRING] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_BUILD_TUPLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_BUILD_LIST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, [_LIST_EXTEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SET_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SETUP_ANNOTATIONS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BUILD_CONST_KEY_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DICT_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DICT_MERGE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAP_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_TYPE_VERSION] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, - [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_INSTANCE_VALUE_0] = HAS_DEOPT_FLAG, - [_LOAD_ATTR_INSTANCE_VALUE_1] = HAS_DEOPT_FLAG, - [_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG, - [_CHECK_ATTR_MODULE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_ATTR_SLOT_0] = HAS_DEOPT_FLAG, - [_LOAD_ATTR_SLOT_1] = HAS_DEOPT_FLAG, - [_LOAD_ATTR_SLOT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG, - [_CHECK_ATTR_CLASS] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_CLASS_0] = 0, - [_LOAD_ATTR_CLASS_1] = 0, - [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG | HAS_OPARG_AND_1_FLAG, - [_GUARD_DORV_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_TYPE_VERSION] = HAS_EXIT_FLAG, + [_GUARD_TYPE_VERSION_AND_LOCK] = HAS_EXIT_FLAG, + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_INSTANCE_VALUE] = HAS_DEOPT_FLAG, + [_CHECK_ATTR_MODULE_PUSH_KEYS] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_MODULE_FROM_KEYS] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_ATTR_WITH_HINT] = HAS_EXIT_FLAG, + [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG, + [_LOAD_ATTR_SLOT] = HAS_DEOPT_FLAG, + [_CHECK_ATTR_CLASS] = HAS_EXIT_FLAG, + [_LOAD_ATTR_CLASS] = 0, + [_LOAD_ATTR_PROPERTY_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_GUARD_DORV_NO_DICT] = HAS_EXIT_FLAG, [_STORE_ATTR_INSTANCE_VALUE] = HAS_ESCAPES_FLAG, - [_STORE_ATTR_SLOT] = HAS_ESCAPES_FLAG, + [_STORE_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_STORE_ATTR_SLOT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_COMPARE_OP_STR] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG, + [_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_COMPARE_OP_STR] = HAS_ARG_FLAG, [_IS_OP] = HAS_ARG_FLAG, [_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CONTAINS_OP_SET] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CONTAINS_OP_DICT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_EG_MATCH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_EXC_MATCH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_IMPORT_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_IMPORT_FROM] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_IS_NONE] = 0, [_GET_LEN] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MATCH_CLASS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -163,111 +186,137 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MATCH_SEQUENCE] = 0, [_MATCH_KEYS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_FOR_ITER_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_ITER_CHECK_LIST] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_GUARD_NOT_EXHAUSTED_LIST] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, + [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, [_ITER_NEXT_LIST] = 0, - [_ITER_CHECK_TUPLE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_GUARD_NOT_EXHAUSTED_TUPLE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_ITER_CHECK_TUPLE] = HAS_EXIT_FLAG, + [_GUARD_NOT_EXHAUSTED_TUPLE] = HAS_EXIT_FLAG, [_ITER_NEXT_TUPLE] = 0, - [_ITER_CHECK_RANGE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_GUARD_NOT_EXHAUSTED_RANGE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BEFORE_ASYNC_WITH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BEFORE_WITH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_ITER_CHECK_RANGE] = HAS_EXIT_FLAG, + [_GUARD_NOT_EXHAUSTED_RANGE] = HAS_EXIT_FLAG, + [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG, + [_FOR_ITER_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_LOAD_SPECIAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_EXC_INFO] = 0, - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_GUARD_KEYS_VERSION] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_METHOD_WITH_VALUES] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_ATTR_METHOD_NO_DICT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_DEOPT_FLAG, + [_GUARD_KEYS_VERSION] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_METHOD_WITH_VALUES] = HAS_ARG_FLAG, + [_LOAD_ATTR_METHOD_NO_DICT] = HAS_ARG_FLAG, [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = HAS_ARG_FLAG, [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = HAS_ARG_FLAG, - [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG, + [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG, + [_MAYBE_EXPAND_METHOD] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_FUNCTION_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CHECK_FUNCTION_VERSION_INLINE] = HAS_EXIT_FLAG, + [_CHECK_METHOD_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_EXPAND_METHOD] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_IS_NOT_PY_CALLABLE] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_NON_PY_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_CHECK_PEP_523] = HAS_DEOPT_FLAG, - [_CHECK_FUNCTION_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_CHECK_STACK_SPACE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_INIT_CALL_PY_EXACT_ARGS_0] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_INIT_CALL_PY_EXACT_ARGS_1] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_INIT_CALL_PY_EXACT_ARGS_2] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_PUSH_FRAME] = HAS_ESCAPES_FLAG, - [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_CALL_STR_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, - [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_LEN] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_ISINSTANCE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_FUNCTION_EXACT_ARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CHECK_STACK_SPACE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_0] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_1] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_2] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, + [_PUSH_FRAME] = 0, + [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_CALL_STR_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_AND_ALLOCATE_OBJECT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CREATE_INIT_FRAME] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_LEN] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_ISINSTANCE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_LIST_APPEND] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAYBE_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_FUNCTION_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CHECK_METHOD_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_IS_NOT_PY_CALLABLE_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_KW_NON_PY] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_CALLARGS_A_TUPLE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG, + [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_WITH_SPEC] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_COPY] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_GUARD_IS_TRUE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_GUARD_IS_FALSE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_GUARD_IS_NONE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_GUARD_IS_NOT_NONE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_JUMP_TO_TOP] = HAS_EVAL_BREAK_FLAG, + [_GUARD_IS_TRUE_POP] = HAS_EXIT_FLAG, + [_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG, + [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_IS_NOT_NONE_POP] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_JUMP_TO_TOP] = 0, [_SET_IP] = 0, + [_CHECK_STACK_SPACE_OPERAND] = HAS_DEOPT_FLAG, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, - [_EXIT_TRACE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_EXIT_TRACE] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, - [_POP_TOP_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, - [_LOAD_CONST_INLINE_WITH_NULL] = HAS_PURE_FLAG, - [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = HAS_PURE_FLAG, - [_CHECK_GLOBALS] = HAS_DEOPT_FLAG, - [_CHECK_BUILTINS] = HAS_DEOPT_FLAG, - [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, - [_COLD_EXIT] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_START_EXECUTOR] = 0, - [_FATAL_ERROR] = HAS_ESCAPES_FLAG, + [_POP_TOP_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_CHECK_FUNCTION] = HAS_DEOPT_FLAG, + [_LOAD_GLOBAL_MODULE] = HAS_DEOPT_FLAG, + [_LOAD_GLOBAL_BUILTINS] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_MODULE] = HAS_DEOPT_FLAG, + [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, + [_START_EXECUTOR] = HAS_ESCAPES_FLAG, + [_MAKE_WARM] = 0, + [_FATAL_ERROR] = 0, [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, + [_DEOPT] = 0, + [_ERROR_POP_N] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_TIER2_RESUME_CHECK] = HAS_DEOPT_FLAG, }; const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { [_LOAD_FAST] = 8, + [_LOAD_SMALL_INT] = 4, [_STORE_FAST] = 8, [_INIT_CALL_PY_EXACT_ARGS] = 5, }; const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { - [_BEFORE_ASYNC_WITH] = "_BEFORE_ASYNC_WITH", - [_BEFORE_WITH] = "_BEFORE_WITH", [_BINARY_OP] = "_BINARY_OP", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", + [_BINARY_OP_EXTEND] = "_BINARY_OP_EXTEND", + [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", [_BINARY_SLICE] = "_BINARY_SLICE", [_BINARY_SUBSCR] = "_BINARY_SUBSCR", + [_BINARY_SUBSCR_CHECK_FUNC] = "_BINARY_SUBSCR_CHECK_FUNC", [_BINARY_SUBSCR_DICT] = "_BINARY_SUBSCR_DICT", + [_BINARY_SUBSCR_INIT_CALL] = "_BINARY_SUBSCR_INIT_CALL", [_BINARY_SUBSCR_LIST_INT] = "_BINARY_SUBSCR_LIST_INT", [_BINARY_SUBSCR_STR_INT] = "_BINARY_SUBSCR_STR_INT", [_BINARY_SUBSCR_TUPLE_INT] = "_BINARY_SUBSCR_TUPLE_INT", - [_BUILD_CONST_KEY_MAP] = "_BUILD_CONST_KEY_MAP", [_BUILD_LIST] = "_BUILD_LIST", [_BUILD_MAP] = "_BUILD_MAP", [_BUILD_SET] = "_BUILD_SET", @@ -281,52 +330,74 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CALL_INTRINSIC_1] = "_CALL_INTRINSIC_1", [_CALL_INTRINSIC_2] = "_CALL_INTRINSIC_2", [_CALL_ISINSTANCE] = "_CALL_ISINSTANCE", + [_CALL_KW_NON_PY] = "_CALL_KW_NON_PY", [_CALL_LEN] = "_CALL_LEN", + [_CALL_LIST_APPEND] = "_CALL_LIST_APPEND", [_CALL_METHOD_DESCRIPTOR_FAST] = "_CALL_METHOD_DESCRIPTOR_FAST", [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [_CALL_METHOD_DESCRIPTOR_NOARGS] = "_CALL_METHOD_DESCRIPTOR_NOARGS", [_CALL_METHOD_DESCRIPTOR_O] = "_CALL_METHOD_DESCRIPTOR_O", + [_CALL_NON_PY_GENERAL] = "_CALL_NON_PY_GENERAL", [_CALL_STR_1] = "_CALL_STR_1", [_CALL_TUPLE_1] = "_CALL_TUPLE_1", [_CALL_TYPE_1] = "_CALL_TYPE_1", + [_CHECK_AND_ALLOCATE_OBJECT] = "_CHECK_AND_ALLOCATE_OBJECT", [_CHECK_ATTR_CLASS] = "_CHECK_ATTR_CLASS", [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", - [_CHECK_ATTR_MODULE] = "_CHECK_ATTR_MODULE", + [_CHECK_ATTR_MODULE_PUSH_KEYS] = "_CHECK_ATTR_MODULE_PUSH_KEYS", [_CHECK_ATTR_WITH_HINT] = "_CHECK_ATTR_WITH_HINT", - [_CHECK_BUILTINS] = "_CHECK_BUILTINS", [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", [_CHECK_EG_MATCH] = "_CHECK_EG_MATCH", [_CHECK_EXC_MATCH] = "_CHECK_EXC_MATCH", + [_CHECK_FUNCTION] = "_CHECK_FUNCTION", [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", - [_CHECK_GLOBALS] = "_CHECK_GLOBALS", + [_CHECK_FUNCTION_VERSION] = "_CHECK_FUNCTION_VERSION", + [_CHECK_FUNCTION_VERSION_INLINE] = "_CHECK_FUNCTION_VERSION_INLINE", + [_CHECK_FUNCTION_VERSION_KW] = "_CHECK_FUNCTION_VERSION_KW", + [_CHECK_IS_NOT_PY_CALLABLE] = "_CHECK_IS_NOT_PY_CALLABLE", + [_CHECK_IS_NOT_PY_CALLABLE_KW] = "_CHECK_IS_NOT_PY_CALLABLE_KW", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", + [_CHECK_METHOD_VERSION] = "_CHECK_METHOD_VERSION", + [_CHECK_METHOD_VERSION_KW] = "_CHECK_METHOD_VERSION_KW", [_CHECK_PEP_523] = "_CHECK_PEP_523", + [_CHECK_PERIODIC] = "_CHECK_PERIODIC", + [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = "_CHECK_PERIODIC_IF_NOT_YIELD_FROM", [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", + [_CHECK_STACK_SPACE_OPERAND] = "_CHECK_STACK_SPACE_OPERAND", [_CHECK_VALIDITY] = "_CHECK_VALIDITY", [_CHECK_VALIDITY_AND_SET_IP] = "_CHECK_VALIDITY_AND_SET_IP", - [_COLD_EXIT] = "_COLD_EXIT", [_COMPARE_OP] = "_COMPARE_OP", [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT", [_COMPARE_OP_INT] = "_COMPARE_OP_INT", [_COMPARE_OP_STR] = "_COMPARE_OP_STR", [_CONTAINS_OP] = "_CONTAINS_OP", + [_CONTAINS_OP_DICT] = "_CONTAINS_OP_DICT", + [_CONTAINS_OP_SET] = "_CONTAINS_OP_SET", [_CONVERT_VALUE] = "_CONVERT_VALUE", [_COPY] = "_COPY", [_COPY_FREE_VARS] = "_COPY_FREE_VARS", + [_CREATE_INIT_FRAME] = "_CREATE_INIT_FRAME", [_DELETE_ATTR] = "_DELETE_ATTR", [_DELETE_DEREF] = "_DELETE_DEREF", [_DELETE_FAST] = "_DELETE_FAST", [_DELETE_GLOBAL] = "_DELETE_GLOBAL", [_DELETE_NAME] = "_DELETE_NAME", [_DELETE_SUBSCR] = "_DELETE_SUBSCR", + [_DEOPT] = "_DEOPT", [_DICT_MERGE] = "_DICT_MERGE", [_DICT_UPDATE] = "_DICT_UPDATE", + [_DYNAMIC_EXIT] = "_DYNAMIC_EXIT", + [_END_FOR] = "_END_FOR", [_END_SEND] = "_END_SEND", + [_ERROR_POP_N] = "_ERROR_POP_N", [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", [_EXIT_TRACE] = "_EXIT_TRACE", + [_EXPAND_METHOD] = "_EXPAND_METHOD", + [_EXPAND_METHOD_KW] = "_EXPAND_METHOD_KW", [_FATAL_ERROR] = "_FATAL_ERROR", [_FORMAT_SIMPLE] = "_FORMAT_SIMPLE", [_FORMAT_WITH_SPEC] = "_FORMAT_WITH_SPEC", + [_FOR_ITER_GEN_FRAME] = "_FOR_ITER_GEN_FRAME", [_FOR_ITER_TIER_TWO] = "_FOR_ITER_TIER_TWO", [_GET_AITER] = "_GET_AITER", [_GET_ANEXT] = "_GET_ANEXT", @@ -334,22 +405,31 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GET_ITER] = "_GET_ITER", [_GET_LEN] = "_GET_LEN", [_GET_YIELD_FROM_ITER] = "_GET_YIELD_FROM_ITER", + [_GUARD_BINARY_OP_EXTEND] = "_GUARD_BINARY_OP_EXTEND", [_GUARD_BOTH_FLOAT] = "_GUARD_BOTH_FLOAT", [_GUARD_BOTH_INT] = "_GUARD_BOTH_INT", [_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE", - [_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION", - [_GUARD_DORV_VALUES] = "_GUARD_DORV_VALUES", + [_GUARD_BUILTINS_VERSION_PUSH_KEYS] = "_GUARD_BUILTINS_VERSION_PUSH_KEYS", + [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT", [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", + [_GUARD_GLOBALS_VERSION_PUSH_KEYS] = "_GUARD_GLOBALS_VERSION_PUSH_KEYS", [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", [_GUARD_IS_TRUE_POP] = "_GUARD_IS_TRUE_POP", [_GUARD_KEYS_VERSION] = "_GUARD_KEYS_VERSION", + [_GUARD_NOS_FLOAT] = "_GUARD_NOS_FLOAT", + [_GUARD_NOS_INT] = "_GUARD_NOS_INT", [_GUARD_NOT_EXHAUSTED_LIST] = "_GUARD_NOT_EXHAUSTED_LIST", [_GUARD_NOT_EXHAUSTED_RANGE] = "_GUARD_NOT_EXHAUSTED_RANGE", [_GUARD_NOT_EXHAUSTED_TUPLE] = "_GUARD_NOT_EXHAUSTED_TUPLE", + [_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT", + [_GUARD_TOS_INT] = "_GUARD_TOS_INT", [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", + [_GUARD_TYPE_VERSION_AND_LOCK] = "_GUARD_TYPE_VERSION_AND_LOCK", + [_IMPORT_FROM] = "_IMPORT_FROM", + [_IMPORT_NAME] = "_IMPORT_NAME", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", [_INIT_CALL_PY_EXACT_ARGS_0] = "_INIT_CALL_PY_EXACT_ARGS_0", @@ -357,7 +437,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_2] = "_INIT_CALL_PY_EXACT_ARGS_2", [_INIT_CALL_PY_EXACT_ARGS_3] = "_INIT_CALL_PY_EXACT_ARGS_3", [_INIT_CALL_PY_EXACT_ARGS_4] = "_INIT_CALL_PY_EXACT_ARGS_4", - [_INTERNAL_INCREMENT_OPT_COUNTER] = "_INTERNAL_INCREMENT_OPT_COUNTER", [_IS_NONE] = "_IS_NONE", [_IS_OP] = "_IS_OP", [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", @@ -369,30 +448,25 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_JUMP_TO_TOP] = "_JUMP_TO_TOP", [_LIST_APPEND] = "_LIST_APPEND", [_LIST_EXTEND] = "_LIST_EXTEND", - [_LOAD_ASSERTION_ERROR] = "_LOAD_ASSERTION_ERROR", [_LOAD_ATTR] = "_LOAD_ATTR", [_LOAD_ATTR_CLASS] = "_LOAD_ATTR_CLASS", - [_LOAD_ATTR_CLASS_0] = "_LOAD_ATTR_CLASS_0", - [_LOAD_ATTR_CLASS_1] = "_LOAD_ATTR_CLASS_1", [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", - [_LOAD_ATTR_INSTANCE_VALUE_0] = "_LOAD_ATTR_INSTANCE_VALUE_0", - [_LOAD_ATTR_INSTANCE_VALUE_1] = "_LOAD_ATTR_INSTANCE_VALUE_1", [_LOAD_ATTR_METHOD_LAZY_DICT] = "_LOAD_ATTR_METHOD_LAZY_DICT", [_LOAD_ATTR_METHOD_NO_DICT] = "_LOAD_ATTR_METHOD_NO_DICT", [_LOAD_ATTR_METHOD_WITH_VALUES] = "_LOAD_ATTR_METHOD_WITH_VALUES", [_LOAD_ATTR_MODULE] = "_LOAD_ATTR_MODULE", + [_LOAD_ATTR_MODULE_FROM_KEYS] = "_LOAD_ATTR_MODULE_FROM_KEYS", [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT", [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + [_LOAD_ATTR_PROPERTY_FRAME] = "_LOAD_ATTR_PROPERTY_FRAME", [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", - [_LOAD_ATTR_SLOT_0] = "_LOAD_ATTR_SLOT_0", - [_LOAD_ATTR_SLOT_1] = "_LOAD_ATTR_SLOT_1", [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS", - [_LOAD_CONST] = "_LOAD_CONST", + [_LOAD_COMMON_CONSTANT] = "_LOAD_COMMON_CONSTANT", + [_LOAD_CONST_IMMORTAL] = "_LOAD_CONST_IMMORTAL", [_LOAD_CONST_INLINE] = "_LOAD_CONST_INLINE", [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW", - [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = "_LOAD_CONST_INLINE_BORROW_WITH_NULL", - [_LOAD_CONST_INLINE_WITH_NULL] = "_LOAD_CONST_INLINE_WITH_NULL", + [_LOAD_CONST_MORTAL] = "_LOAD_CONST_MORTAL", [_LOAD_DEREF] = "_LOAD_DEREF", [_LOAD_FAST] = "_LOAD_FAST", [_LOAD_FAST_0] = "_LOAD_FAST_0", @@ -407,31 +481,48 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_FAST_CHECK] = "_LOAD_FAST_CHECK", [_LOAD_FAST_LOAD_FAST] = "_LOAD_FAST_LOAD_FAST", [_LOAD_FROM_DICT_OR_DEREF] = "_LOAD_FROM_DICT_OR_DEREF", - [_LOAD_FROM_DICT_OR_GLOBALS] = "_LOAD_FROM_DICT_OR_GLOBALS", [_LOAD_GLOBAL] = "_LOAD_GLOBAL", [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", + [_LOAD_GLOBAL_BUILTINS_FROM_KEYS] = "_LOAD_GLOBAL_BUILTINS_FROM_KEYS", [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", + [_LOAD_GLOBAL_MODULE_FROM_KEYS] = "_LOAD_GLOBAL_MODULE_FROM_KEYS", [_LOAD_LOCALS] = "_LOAD_LOCALS", [_LOAD_NAME] = "_LOAD_NAME", + [_LOAD_SMALL_INT] = "_LOAD_SMALL_INT", + [_LOAD_SMALL_INT_0] = "_LOAD_SMALL_INT_0", + [_LOAD_SMALL_INT_1] = "_LOAD_SMALL_INT_1", + [_LOAD_SMALL_INT_2] = "_LOAD_SMALL_INT_2", + [_LOAD_SMALL_INT_3] = "_LOAD_SMALL_INT_3", + [_LOAD_SPECIAL] = "_LOAD_SPECIAL", [_LOAD_SUPER_ATTR_ATTR] = "_LOAD_SUPER_ATTR_ATTR", [_LOAD_SUPER_ATTR_METHOD] = "_LOAD_SUPER_ATTR_METHOD", + [_MAKE_CALLARGS_A_TUPLE] = "_MAKE_CALLARGS_A_TUPLE", [_MAKE_CELL] = "_MAKE_CELL", [_MAKE_FUNCTION] = "_MAKE_FUNCTION", + [_MAKE_WARM] = "_MAKE_WARM", [_MAP_ADD] = "_MAP_ADD", [_MATCH_CLASS] = "_MATCH_CLASS", [_MATCH_KEYS] = "_MATCH_KEYS", [_MATCH_MAPPING] = "_MATCH_MAPPING", [_MATCH_SEQUENCE] = "_MATCH_SEQUENCE", + [_MAYBE_EXPAND_METHOD] = "_MAYBE_EXPAND_METHOD", + [_MAYBE_EXPAND_METHOD_KW] = "_MAYBE_EXPAND_METHOD_KW", [_NOP] = "_NOP", [_POP_EXCEPT] = "_POP_EXCEPT", - [_POP_FRAME] = "_POP_FRAME", [_POP_TOP] = "_POP_TOP", [_POP_TOP_LOAD_CONST_INLINE_BORROW] = "_POP_TOP_LOAD_CONST_INLINE_BORROW", [_PUSH_EXC_INFO] = "_PUSH_EXC_INFO", [_PUSH_FRAME] = "_PUSH_FRAME", [_PUSH_NULL] = "_PUSH_NULL", + [_PUSH_NULL_CONDITIONAL] = "_PUSH_NULL_CONDITIONAL", + [_PY_FRAME_GENERAL] = "_PY_FRAME_GENERAL", + [_PY_FRAME_KW] = "_PY_FRAME_KW", + [_REPLACE_WITH_TRUE] = "_REPLACE_WITH_TRUE", [_RESUME_CHECK] = "_RESUME_CHECK", + [_RETURN_GENERATOR] = "_RETURN_GENERATOR", + [_RETURN_VALUE] = "_RETURN_VALUE", [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", + [_SEND_GEN_FRAME] = "_SEND_GEN_FRAME", [_SETUP_ANNOTATIONS] = "_SETUP_ANNOTATIONS", [_SET_ADD] = "_SET_ADD", [_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE", @@ -441,6 +532,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_STORE_ATTR] = "_STORE_ATTR", [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", + [_STORE_ATTR_WITH_HINT] = "_STORE_ATTR_WITH_HINT", [_STORE_DEREF] = "_STORE_DEREF", [_STORE_FAST] = "_STORE_FAST", [_STORE_FAST_0] = "_STORE_FAST_0", @@ -460,8 +552,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_STORE_SUBSCR_DICT] = "_STORE_SUBSCR_DICT", [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", [_SWAP] = "_SWAP", + [_TIER2_RESUME_CHECK] = "_TIER2_RESUME_CHECK", [_TO_BOOL] = "_TO_BOOL", - [_TO_BOOL_ALWAYS_TRUE] = "_TO_BOOL_ALWAYS_TRUE", [_TO_BOOL_BOOL] = "_TO_BOOL_BOOL", [_TO_BOOL_INT] = "_TO_BOOL_INT", [_TO_BOOL_LIST] = "_TO_BOOL_LIST", @@ -476,7 +568,556 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_UNPACK_SEQUENCE_TUPLE] = "_UNPACK_SEQUENCE_TUPLE", [_UNPACK_SEQUENCE_TWO_TUPLE] = "_UNPACK_SEQUENCE_TWO_TUPLE", [_WITH_EXCEPT_START] = "_WITH_EXCEPT_START", + [_YIELD_VALUE] = "_YIELD_VALUE", }; +int _PyUop_num_popped(int opcode, int oparg) +{ + switch(opcode) { + case _NOP: + return 0; + case _CHECK_PERIODIC: + return 0; + case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: + return 0; + case _RESUME_CHECK: + return 0; + case _LOAD_FAST_CHECK: + return 0; + case _LOAD_FAST_0: + return 0; + case _LOAD_FAST_1: + return 0; + case _LOAD_FAST_2: + return 0; + case _LOAD_FAST_3: + return 0; + case _LOAD_FAST_4: + return 0; + case _LOAD_FAST_5: + return 0; + case _LOAD_FAST_6: + return 0; + case _LOAD_FAST_7: + return 0; + case _LOAD_FAST: + return 0; + case _LOAD_FAST_AND_CLEAR: + return 0; + case _LOAD_FAST_LOAD_FAST: + return 0; + case _LOAD_CONST_MORTAL: + return 0; + case _LOAD_CONST_IMMORTAL: + return 0; + case _LOAD_SMALL_INT_0: + return 0; + case _LOAD_SMALL_INT_1: + return 0; + case _LOAD_SMALL_INT_2: + return 0; + case _LOAD_SMALL_INT_3: + return 0; + case _LOAD_SMALL_INT: + return 0; + case _STORE_FAST_0: + return 1; + case _STORE_FAST_1: + return 1; + case _STORE_FAST_2: + return 1; + case _STORE_FAST_3: + return 1; + case _STORE_FAST_4: + return 1; + case _STORE_FAST_5: + return 1; + case _STORE_FAST_6: + return 1; + case _STORE_FAST_7: + return 1; + case _STORE_FAST: + return 1; + case _STORE_FAST_LOAD_FAST: + return 1; + case _STORE_FAST_STORE_FAST: + return 2; + case _POP_TOP: + return 1; + case _PUSH_NULL: + return 0; + case _END_FOR: + return 1; + case _END_SEND: + return 2; + case _UNARY_NEGATIVE: + return 1; + case _UNARY_NOT: + return 1; + case _TO_BOOL: + return 1; + case _TO_BOOL_BOOL: + return 0; + case _TO_BOOL_INT: + return 1; + case _TO_BOOL_LIST: + return 1; + case _TO_BOOL_NONE: + return 1; + case _TO_BOOL_STR: + return 1; + case _REPLACE_WITH_TRUE: + return 1; + case _UNARY_INVERT: + return 1; + case _GUARD_BOTH_INT: + return 0; + case _GUARD_NOS_INT: + return 0; + case _GUARD_TOS_INT: + return 0; + case _BINARY_OP_MULTIPLY_INT: + return 2; + case _BINARY_OP_ADD_INT: + return 2; + case _BINARY_OP_SUBTRACT_INT: + return 2; + case _GUARD_BOTH_FLOAT: + return 0; + case _GUARD_NOS_FLOAT: + return 0; + case _GUARD_TOS_FLOAT: + return 0; + case _BINARY_OP_MULTIPLY_FLOAT: + return 2; + case _BINARY_OP_ADD_FLOAT: + return 2; + case _BINARY_OP_SUBTRACT_FLOAT: + return 2; + case _GUARD_BOTH_UNICODE: + return 0; + case _BINARY_OP_ADD_UNICODE: + return 2; + case _BINARY_OP_INPLACE_ADD_UNICODE: + return 2; + case _GUARD_BINARY_OP_EXTEND: + return 0; + case _BINARY_OP_EXTEND: + return 2; + case _BINARY_SUBSCR: + return 2; + case _BINARY_SLICE: + return 3; + case _STORE_SLICE: + return 4; + case _BINARY_SUBSCR_LIST_INT: + return 2; + case _BINARY_SUBSCR_STR_INT: + return 2; + case _BINARY_SUBSCR_TUPLE_INT: + return 2; + case _BINARY_SUBSCR_DICT: + return 2; + case _BINARY_SUBSCR_CHECK_FUNC: + return 0; + case _BINARY_SUBSCR_INIT_CALL: + return 3; + case _LIST_APPEND: + return 1; + case _SET_ADD: + return 1; + case _STORE_SUBSCR: + return 3; + case _STORE_SUBSCR_LIST_INT: + return 3; + case _STORE_SUBSCR_DICT: + return 3; + case _DELETE_SUBSCR: + return 2; + case _CALL_INTRINSIC_1: + return 1; + case _CALL_INTRINSIC_2: + return 2; + case _RETURN_VALUE: + return 1; + case _GET_AITER: + return 1; + case _GET_ANEXT: + return 0; + case _GET_AWAITABLE: + return 1; + case _SEND_GEN_FRAME: + return 1; + case _YIELD_VALUE: + return 1; + case _POP_EXCEPT: + return 1; + case _LOAD_COMMON_CONSTANT: + return 0; + case _LOAD_BUILD_CLASS: + return 0; + case _STORE_NAME: + return 1; + case _DELETE_NAME: + return 0; + case _UNPACK_SEQUENCE: + return 1; + case _UNPACK_SEQUENCE_TWO_TUPLE: + return 1; + case _UNPACK_SEQUENCE_TUPLE: + return 1; + case _UNPACK_SEQUENCE_LIST: + return 1; + case _UNPACK_EX: + return 1; + case _STORE_ATTR: + return 2; + case _DELETE_ATTR: + return 1; + case _STORE_GLOBAL: + return 1; + case _DELETE_GLOBAL: + return 0; + case _LOAD_LOCALS: + return 0; + case _LOAD_NAME: + return 0; + case _LOAD_GLOBAL: + return 0; + case _PUSH_NULL_CONDITIONAL: + return 0; + case _GUARD_GLOBALS_VERSION: + return 0; + case _GUARD_GLOBALS_VERSION_PUSH_KEYS: + return 0; + case _GUARD_BUILTINS_VERSION_PUSH_KEYS: + return 0; + case _LOAD_GLOBAL_MODULE_FROM_KEYS: + return 1; + case _LOAD_GLOBAL_BUILTINS_FROM_KEYS: + return 1; + case _DELETE_FAST: + return 0; + case _MAKE_CELL: + return 0; + case _DELETE_DEREF: + return 0; + case _LOAD_FROM_DICT_OR_DEREF: + return 1; + case _LOAD_DEREF: + return 0; + case _STORE_DEREF: + return 1; + case _COPY_FREE_VARS: + return 0; + case _BUILD_STRING: + return oparg; + case _BUILD_TUPLE: + return oparg; + case _BUILD_LIST: + return oparg; + case _LIST_EXTEND: + return 1; + case _SET_UPDATE: + return 1; + case _BUILD_SET: + return oparg; + case _BUILD_MAP: + return oparg*2; + case _SETUP_ANNOTATIONS: + return 0; + case _DICT_UPDATE: + return 1; + case _DICT_MERGE: + return 1; + case _MAP_ADD: + return 2; + case _LOAD_SUPER_ATTR_ATTR: + return 3; + case _LOAD_SUPER_ATTR_METHOD: + return 3; + case _LOAD_ATTR: + return 1; + case _GUARD_TYPE_VERSION: + return 0; + case _GUARD_TYPE_VERSION_AND_LOCK: + return 0; + case _CHECK_MANAGED_OBJECT_HAS_VALUES: + return 0; + case _LOAD_ATTR_INSTANCE_VALUE: + return 1; + case _CHECK_ATTR_MODULE_PUSH_KEYS: + return 0; + case _LOAD_ATTR_MODULE_FROM_KEYS: + return 2; + case _CHECK_ATTR_WITH_HINT: + return 0; + case _LOAD_ATTR_WITH_HINT: + return 2; + case _LOAD_ATTR_SLOT: + return 1; + case _CHECK_ATTR_CLASS: + return 0; + case _LOAD_ATTR_CLASS: + return 1; + case _LOAD_ATTR_PROPERTY_FRAME: + return 1; + case _GUARD_DORV_NO_DICT: + return 0; + case _STORE_ATTR_INSTANCE_VALUE: + return 2; + case _STORE_ATTR_WITH_HINT: + return 2; + case _STORE_ATTR_SLOT: + return 2; + case _COMPARE_OP: + return 2; + case _COMPARE_OP_FLOAT: + return 2; + case _COMPARE_OP_INT: + return 2; + case _COMPARE_OP_STR: + return 2; + case _IS_OP: + return 2; + case _CONTAINS_OP: + return 2; + case _CONTAINS_OP_SET: + return 2; + case _CONTAINS_OP_DICT: + return 2; + case _CHECK_EG_MATCH: + return 2; + case _CHECK_EXC_MATCH: + return 1; + case _IMPORT_NAME: + return 2; + case _IMPORT_FROM: + return 0; + case _IS_NONE: + return 1; + case _GET_LEN: + return 0; + case _MATCH_CLASS: + return 3; + case _MATCH_MAPPING: + return 0; + case _MATCH_SEQUENCE: + return 0; + case _MATCH_KEYS: + return 0; + case _GET_ITER: + return 1; + case _GET_YIELD_FROM_ITER: + return 1; + case _FOR_ITER_TIER_TWO: + return 0; + case _ITER_CHECK_LIST: + return 0; + case _GUARD_NOT_EXHAUSTED_LIST: + return 0; + case _ITER_NEXT_LIST: + return 0; + case _ITER_CHECK_TUPLE: + return 0; + case _GUARD_NOT_EXHAUSTED_TUPLE: + return 0; + case _ITER_NEXT_TUPLE: + return 0; + case _ITER_CHECK_RANGE: + return 0; + case _GUARD_NOT_EXHAUSTED_RANGE: + return 0; + case _ITER_NEXT_RANGE: + return 0; + case _FOR_ITER_GEN_FRAME: + return 0; + case _LOAD_SPECIAL: + return 1; + case _WITH_EXCEPT_START: + return 0; + case _PUSH_EXC_INFO: + return 1; + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: + return 0; + case _GUARD_KEYS_VERSION: + return 0; + case _LOAD_ATTR_METHOD_WITH_VALUES: + return 1; + case _LOAD_ATTR_METHOD_NO_DICT: + return 1; + case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: + return 1; + case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + return 1; + case _CHECK_ATTR_METHOD_LAZY_DICT: + return 0; + case _LOAD_ATTR_METHOD_LAZY_DICT: + return 1; + case _MAYBE_EXPAND_METHOD: + return 2 + oparg; + case _PY_FRAME_GENERAL: + return 2 + oparg; + case _CHECK_FUNCTION_VERSION: + return 0; + case _CHECK_FUNCTION_VERSION_INLINE: + return 0; + case _CHECK_METHOD_VERSION: + return 0; + case _EXPAND_METHOD: + return 2 + oparg; + case _CHECK_IS_NOT_PY_CALLABLE: + return 0; + case _CALL_NON_PY_GENERAL: + return 2 + oparg; + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: + return 0; + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: + return 2 + oparg; + case _CHECK_PEP_523: + return 0; + case _CHECK_FUNCTION_EXACT_ARGS: + return 0; + case _CHECK_STACK_SPACE: + return 0; + case _INIT_CALL_PY_EXACT_ARGS_0: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_1: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_2: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_3: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_4: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS: + return 2 + oparg; + case _PUSH_FRAME: + return 1; + case _CALL_TYPE_1: + return 3; + case _CALL_STR_1: + return 3; + case _CALL_TUPLE_1: + return 3; + case _CHECK_AND_ALLOCATE_OBJECT: + return 2 + oparg; + case _CREATE_INIT_FRAME: + return 2 + oparg; + case _EXIT_INIT_CHECK: + return 1; + case _CALL_BUILTIN_CLASS: + return 2 + oparg; + case _CALL_BUILTIN_O: + return 2 + oparg; + case _CALL_BUILTIN_FAST: + return 2 + oparg; + case _CALL_BUILTIN_FAST_WITH_KEYWORDS: + return 2 + oparg; + case _CALL_LEN: + return 2 + oparg; + case _CALL_ISINSTANCE: + return 2 + oparg; + case _CALL_LIST_APPEND: + return 3; + case _CALL_METHOD_DESCRIPTOR_O: + return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: + return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_NOARGS: + return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_FAST: + return 2 + oparg; + case _MAYBE_EXPAND_METHOD_KW: + return 3 + oparg; + case _PY_FRAME_KW: + return 3 + oparg; + case _CHECK_FUNCTION_VERSION_KW: + return 0; + case _CHECK_METHOD_VERSION_KW: + return 0; + case _EXPAND_METHOD_KW: + return 3 + oparg; + case _CHECK_IS_NOT_PY_CALLABLE_KW: + return 0; + case _CALL_KW_NON_PY: + return 3 + oparg; + case _MAKE_CALLARGS_A_TUPLE: + return 2; + case _MAKE_FUNCTION: + return 1; + case _SET_FUNCTION_ATTRIBUTE: + return 2; + case _RETURN_GENERATOR: + return 0; + case _BUILD_SLICE: + return oparg; + case _CONVERT_VALUE: + return 1; + case _FORMAT_SIMPLE: + return 1; + case _FORMAT_WITH_SPEC: + return 2; + case _COPY: + return 0; + case _BINARY_OP: + return 2; + case _SWAP: + return 2 + (oparg-2); + case _GUARD_IS_TRUE_POP: + return 1; + case _GUARD_IS_FALSE_POP: + return 1; + case _GUARD_IS_NONE_POP: + return 1; + case _GUARD_IS_NOT_NONE_POP: + return 1; + case _JUMP_TO_TOP: + return 0; + case _SET_IP: + return 0; + case _CHECK_STACK_SPACE_OPERAND: + return 0; + case _SAVE_RETURN_OFFSET: + return 0; + case _EXIT_TRACE: + return 0; + case _CHECK_VALIDITY: + return 0; + case _LOAD_CONST_INLINE: + return 0; + case _LOAD_CONST_INLINE_BORROW: + return 0; + case _POP_TOP_LOAD_CONST_INLINE_BORROW: + return 1; + case _CHECK_FUNCTION: + return 0; + case _LOAD_GLOBAL_MODULE: + return 0; + case _LOAD_GLOBAL_BUILTINS: + return 0; + case _LOAD_ATTR_MODULE: + return 1; + case _DYNAMIC_EXIT: + return 0; + case _START_EXECUTOR: + return 0; + case _MAKE_WARM: + return 0; + case _FATAL_ERROR: + return 0; + case _CHECK_VALIDITY_AND_SET_IP: + return 0; + case _DEOPT: + return 0; + case _ERROR_POP_N: + return oparg; + case _TIER2_RESUME_CHECK: + return 0; + default: + return -1; + } +} + #endif // NEED_OPCODE_METADATA diff --git a/Include/internal/pycore_warnings.h b/Include/internal/pycore_warnings.h index 9785d7cc467de2..672228cd6fbd19 100644 --- a/Include/internal/pycore_warnings.h +++ b/Include/internal/pycore_warnings.h @@ -14,6 +14,7 @@ struct _warnings_runtime_state { PyObject *filters; /* List */ PyObject *once_registry; /* Dict */ PyObject *default_action; /* String */ + _PyRecursiveMutex lock; long filters_version; }; diff --git a/Include/internal/pycore_weakref.h b/Include/internal/pycore_weakref.h index dea267b49039e7..94aadb2c1547dd 100644 --- a/Include/internal/pycore_weakref.h +++ b/Include/internal/pycore_weakref.h @@ -9,7 +9,35 @@ extern "C" { #endif #include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() +#include "pycore_lock.h" #include "pycore_object.h" // _Py_REF_IS_MERGED() +#include "pycore_pyatomic_ft_wrappers.h" + +#ifdef Py_GIL_DISABLED + +#define WEAKREF_LIST_LOCK(obj) \ + _PyInterpreterState_GET() \ + ->weakref_locks[((uintptr_t)obj) % NUM_WEAKREF_LIST_LOCKS] + +// Lock using the referenced object +#define LOCK_WEAKREFS(obj) \ + PyMutex_LockFlags(&WEAKREF_LIST_LOCK(obj), _Py_LOCK_DONT_DETACH) +#define UNLOCK_WEAKREFS(obj) PyMutex_Unlock(&WEAKREF_LIST_LOCK(obj)) + +// Lock using a weakref +#define LOCK_WEAKREFS_FOR_WR(wr) \ + PyMutex_LockFlags(wr->weakrefs_lock, _Py_LOCK_DONT_DETACH) +#define UNLOCK_WEAKREFS_FOR_WR(wr) PyMutex_Unlock(wr->weakrefs_lock) + +#else + +#define LOCK_WEAKREFS(obj) +#define UNLOCK_WEAKREFS(obj) + +#define LOCK_WEAKREFS_FOR_WR(wr) +#define UNLOCK_WEAKREFS_FOR_WR(wr) + +#endif static inline int _is_dead(PyObject *obj) { @@ -30,53 +58,62 @@ static inline int _is_dead(PyObject *obj) static inline PyObject* _PyWeakref_GET_REF(PyObject *ref_obj) { assert(PyWeakref_Check(ref_obj)); - PyObject *ret = NULL; - Py_BEGIN_CRITICAL_SECTION(ref_obj); PyWeakReference *ref = _Py_CAST(PyWeakReference*, ref_obj); - PyObject *obj = ref->wr_object; + PyObject *obj = FT_ATOMIC_LOAD_PTR(ref->wr_object); if (obj == Py_None) { // clear_weakref() was called - goto end; + return NULL; } - if (_is_dead(obj)) { - goto end; + LOCK_WEAKREFS(obj); +#ifdef Py_GIL_DISABLED + if (ref->wr_object == Py_None) { + // clear_weakref() was called + UNLOCK_WEAKREFS(obj); + return NULL; } -#if !defined(Py_GIL_DISABLED) - assert(Py_REFCNT(obj) > 0); #endif - ret = Py_NewRef(obj); -end: - Py_END_CRITICAL_SECTION(); - return ret; + if (_Py_TryIncref(obj)) { + UNLOCK_WEAKREFS(obj); + return obj; + } + UNLOCK_WEAKREFS(obj); + return NULL; } static inline int _PyWeakref_IS_DEAD(PyObject *ref_obj) { assert(PyWeakref_Check(ref_obj)); int ret = 0; - Py_BEGIN_CRITICAL_SECTION(ref_obj); PyWeakReference *ref = _Py_CAST(PyWeakReference*, ref_obj); - PyObject *obj = ref->wr_object; + PyObject *obj = FT_ATOMIC_LOAD_PTR(ref->wr_object); if (obj == Py_None) { // clear_weakref() was called ret = 1; } else { + LOCK_WEAKREFS(obj); // See _PyWeakref_GET_REF() for the rationale of this test +#ifdef Py_GIL_DISABLED + ret = (ref->wr_object == Py_None) || _is_dead(obj); +#else ret = _is_dead(obj); +#endif + UNLOCK_WEAKREFS(obj); } - Py_END_CRITICAL_SECTION(); return ret; } -extern Py_ssize_t _PyWeakref_GetWeakrefCount(PyWeakReference *head); +extern Py_ssize_t _PyWeakref_GetWeakrefCount(PyObject *obj); -extern void _PyWeakref_ClearRef(PyWeakReference *self); +// Clear all the weak references to obj but leave their callbacks uncalled and +// intact. +extern void _PyWeakref_ClearWeakRefsNoCallbacks(PyObject *obj); + +PyAPI_FUNC(int) _PyWeakref_IsDead(PyObject *weakref); #ifdef __cplusplus } #endif #endif /* !Py_INTERNAL_WEAKREF_H */ - diff --git a/Include/interpreteridobject.h b/Include/interpreteridobject.h deleted file mode 100644 index 8432632f339e92..00000000000000 --- a/Include/interpreteridobject.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef Py_INTERPRETERIDOBJECT_H -#define Py_INTERPRETERIDOBJECT_H - -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_INTERPRETERIDOBJECT_H -# include "cpython/interpreteridobject.h" -# undef Py_CPYTHON_INTERPRETERIDOBJECT_H -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_INTERPRETERIDOBJECT_H */ diff --git a/Include/listobject.h b/Include/listobject.h index 4e4084b43483a2..e1e059b0ba7466 100644 --- a/Include/listobject.h +++ b/Include/listobject.h @@ -29,7 +29,9 @@ PyAPI_FUNC(PyObject *) PyList_New(Py_ssize_t size); PyAPI_FUNC(Py_ssize_t) PyList_Size(PyObject *); PyAPI_FUNC(PyObject *) PyList_GetItem(PyObject *, Py_ssize_t); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 PyAPI_FUNC(PyObject *) PyList_GetItemRef(PyObject *, Py_ssize_t); +#endif PyAPI_FUNC(int) PyList_SetItem(PyObject *, Py_ssize_t, PyObject *); PyAPI_FUNC(int) PyList_Insert(PyObject *, Py_ssize_t, PyObject *); PyAPI_FUNC(int) PyList_Append(PyObject *, PyObject *); diff --git a/Include/lock.h b/Include/lock.h new file mode 100644 index 00000000000000..782b9dbc70d056 --- /dev/null +++ b/Include/lock.h @@ -0,0 +1,16 @@ +#ifndef Py_LOCK_H +#define Py_LOCK_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_LIMITED_API +# define Py_CPYTHON_LOCK_H +# include "cpython/lock.h" +# undef Py_CPYTHON_LOCK_H +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_LOCK_H */ diff --git a/Include/longobject.h b/Include/longobject.h index 51005efff636fa..45c0d218c13f2f 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -30,6 +30,18 @@ PyAPI_FUNC(unsigned long) PyLong_AsUnsignedLongMask(PyObject *); PyAPI_FUNC(int) PyLong_AsInt(PyObject *); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030e0000 +PyAPI_FUNC(PyObject*) PyLong_FromInt32(int32_t value); +PyAPI_FUNC(PyObject*) PyLong_FromUInt32(uint32_t value); +PyAPI_FUNC(PyObject*) PyLong_FromInt64(int64_t value); +PyAPI_FUNC(PyObject*) PyLong_FromUInt64(uint64_t value); + +PyAPI_FUNC(int) PyLong_AsInt32(PyObject *obj, int32_t *value); +PyAPI_FUNC(int) PyLong_AsUInt32(PyObject *obj, uint32_t *value); +PyAPI_FUNC(int) PyLong_AsInt64(PyObject *obj, int64_t *value); +PyAPI_FUNC(int) PyLong_AsUInt64(PyObject *obj, uint64_t *value); +#endif + PyAPI_FUNC(PyObject *) PyLong_GetInfo(void); /* It may be useful in the future. I've added it in the PyInt -> PyLong @@ -40,7 +52,24 @@ PyAPI_FUNC(PyObject *) PyLong_GetInfo(void); #if !defined(SIZEOF_PID_T) || SIZEOF_PID_T == SIZEOF_INT #define _Py_PARSE_PID "i" #define PyLong_FromPid PyLong_FromLong -#define PyLong_AsPid PyLong_AsLong +# if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +# define PyLong_AsPid PyLong_AsInt +# elif SIZEOF_INT == SIZEOF_LONG +# define PyLong_AsPid PyLong_AsLong +# else +static inline int +PyLong_AsPid(PyObject *obj) +{ + int overflow; + long result = PyLong_AsLongAndOverflow(obj, &overflow); + if (overflow || result > INT_MAX || result < INT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C int"); + return -1; + } + return (int)result; +} +# endif #elif SIZEOF_PID_T == SIZEOF_LONG #define _Py_PARSE_PID "l" #define PyLong_FromPid PyLong_FromLong diff --git a/Include/marshal.h b/Include/marshal.h index f8b0de80cfc38d..f773587bdd0429 100644 --- a/Include/marshal.h +++ b/Include/marshal.h @@ -13,7 +13,7 @@ PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(const char *, Py_ssize_t); PyAPI_FUNC(PyObject *) PyMarshal_WriteObjectToString(PyObject *, int); -#define Py_MARSHAL_VERSION 4 +#define Py_MARSHAL_VERSION 5 PyAPI_FUNC(long) PyMarshal_ReadLongFromFile(FILE *); PyAPI_FUNC(int) PyMarshal_ReadShortFromFile(FILE *); diff --git a/Include/modsupport.h b/Include/modsupport.h index ea4c0fce9f4562..af995f567b004c 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -134,6 +134,12 @@ PyAPI_FUNC(PyObject *) PyModule_FromDefAndSpec2(PyModuleDef *def, #endif /* New in 3.5 */ +#ifndef Py_LIMITED_API +# define Py_CPYTHON_MODSUPPORT_H +# include "cpython/modsupport.h" +# undef Py_CPYTHON_MODSUPPORT_H +#endif + #ifdef __cplusplus } #endif diff --git a/Include/moduleobject.h b/Include/moduleobject.h index 42b87cc4e91012..2a17c891dda811 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -53,7 +53,7 @@ typedef struct PyModuleDef_Base { /* A copy of the module's __dict__ after the first time it was loaded. This is only set/used for legacy modules that do not support multiple initializations. - It is set by _PyImport_FixupExtensionObject(). */ + It is set by fix_up_extension() in import.c. */ PyObject* m_copy; } PyModuleDef_Base; @@ -76,9 +76,13 @@ struct PyModuleDef_Slot { #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030c0000 # define Py_mod_multiple_interpreters 3 #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +# define Py_mod_gil 4 +#endif + #ifndef Py_LIMITED_API -#define _Py_mod_LAST_SLOT 3 +#define _Py_mod_LAST_SLOT 4 #endif #endif /* New in 3.5 */ @@ -90,6 +94,16 @@ struct PyModuleDef_Slot { # define Py_MOD_PER_INTERPRETER_GIL_SUPPORTED ((void *)2) #endif +/* for Py_mod_gil: */ +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +# define Py_MOD_GIL_USED ((void *)0) +# define Py_MOD_GIL_NOT_USED ((void *)1) +#endif + +#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) +PyAPI_FUNC(int) PyUnstable_Module_SetGIL(PyObject *module, void *gil); +#endif + struct PyModuleDef { PyModuleDef_Base m_base; const char* m_name; diff --git a/Include/monitoring.h b/Include/monitoring.h new file mode 100644 index 00000000000000..985f7f230e44e3 --- /dev/null +++ b/Include/monitoring.h @@ -0,0 +1,18 @@ +#ifndef Py_MONITORING_H +#define Py_MONITORING_H +#ifdef __cplusplus +extern "C" { +#endif + +// There is currently no limited API for monitoring + +#ifndef Py_LIMITED_API +# define Py_CPYTHON_MONITORING_H +# include "cpython/monitoring.h" +# undef Py_CPYTHON_MONITORING_H +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_MONITORING_H */ diff --git a/Include/object.h b/Include/object.h index 05187fe5dc4f20..da7b3668c033f4 100644 --- a/Include/object.h +++ b/Include/object.h @@ -59,88 +59,32 @@ whose size is determined when the object is allocated. /* PyObject_HEAD defines the initial segment of every PyObject. */ #define PyObject_HEAD PyObject ob_base; -/* -Immortalization: - -The following indicates the immortalization strategy depending on the amount -of available bits in the reference count field. All strategies are backwards -compatible but the specific reference count value or immortalization check -might change depending on the specializations for the underlying system. - -Proper deallocation of immortal instances requires distinguishing between -statically allocated immortal instances vs those promoted by the runtime to be -immortal. The latter should be the only instances that require -cleanup during runtime finalization. -*/ - -#if SIZEOF_VOID_P > 4 -/* -In 64+ bit systems, an object will be marked as immortal by setting all of the -lower 32 bits of the reference count field, which is equal to: 0xFFFFFFFF - -Using the lower 32 bits makes the value backwards compatible by allowing -C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely -increase and decrease the objects reference count. The object would lose its -immortality, but the execution would still be correct. - -Reference count increases will use saturated arithmetic, taking advantage of -having all the lower 32 bits set, which will avoid the reference count to go -beyond the refcount limit. Immortality checks for reference count decreases will -be done by checking the bit sign flag in the lower 32 bits. -*/ -#define _Py_IMMORTAL_REFCNT _Py_CAST(Py_ssize_t, UINT_MAX) - -#else -/* -In 32 bit systems, an object will be marked as immortal by setting all of the -lower 30 bits of the reference count field, which is equal to: 0x3FFFFFFF - -Using the lower 30 bits makes the value backwards compatible by allowing -C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely -increase and decrease the objects reference count. The object would lose its -immortality, but the execution would still be correct. - -Reference count increases and decreases will first go through an immortality -check by comparing the reference count field to the immortality reference count. -*/ -#define _Py_IMMORTAL_REFCNT _Py_CAST(Py_ssize_t, UINT_MAX >> 2) -#endif - -// Py_GIL_DISABLED builds indicate immortal objects using `ob_ref_local`, which is -// always 32-bits. -#ifdef Py_GIL_DISABLED -#define _Py_IMMORTAL_REFCNT_LOCAL UINT32_MAX -#endif - // Kept for backward compatibility. It was needed by Py_TRACE_REFS build. #define _PyObject_EXTRA_INIT -// Make all internal uses of PyObject_HEAD_INIT immortal while preserving the -// C-API expectation that the refcnt will be set to 1. +/* Make all uses of PyObject_HEAD_INIT immortal. + * + * Statically allocated objects might be shared between + * interpreters, so must be marked as immortal. + */ #if defined(Py_GIL_DISABLED) #define PyObject_HEAD_INIT(type) \ { \ 0, \ - 0, \ + _Py_STATICALLY_ALLOCATED_FLAG, \ { 0 }, \ 0, \ _Py_IMMORTAL_REFCNT_LOCAL, \ 0, \ (type), \ }, -#elif defined(Py_BUILD_CORE) +#else #define PyObject_HEAD_INIT(type) \ { \ - { _Py_IMMORTAL_REFCNT }, \ + { _Py_STATIC_IMMORTAL_INITIAL_REFCNT }, \ (type) \ }, -#else -#define PyObject_HEAD_INIT(type) \ - { \ - { 1 }, \ - (type) \ - }, -#endif /* Py_BUILD_CORE */ +#endif #define PyVarObject_HEAD_INIT(type, size) \ { \ @@ -176,9 +120,19 @@ struct _object { __pragma(warning(disable: 4201)) #endif union { - Py_ssize_t ob_refcnt; #if SIZEOF_VOID_P > 4 - PY_UINT32_T ob_refcnt_split[2]; + PY_INT64_T ob_refcnt_full; /* This field is needed for efficient initialization with Clang on ARM */ + struct { +# if PY_BIG_ENDIAN + PY_UINT32_T ob_flags; + PY_UINT32_T ob_refcnt; +# else + PY_UINT32_T ob_refcnt; + PY_UINT32_T ob_flags; +# endif + }; +#else + Py_ssize_t ob_refcnt; #endif }; #ifdef _MSC_VER @@ -193,31 +147,13 @@ struct _object { // fields have been merged. #define _Py_UNOWNED_TID 0 -// The shared reference count uses the two least-significant bits to store -// flags. The remaining bits are used to store the reference count. -#define _Py_REF_SHARED_SHIFT 2 -#define _Py_REF_SHARED_FLAG_MASK 0x3 - -// The shared flags are initialized to zero. -#define _Py_REF_SHARED_INIT 0x0 -#define _Py_REF_MAYBE_WEAKREF 0x1 -#define _Py_REF_QUEUED 0x2 -#define _Py_REF_MERGED 0x3 - -// Create a shared field from a refcnt and desired flags -#define _Py_REF_SHARED(refcnt, flags) (((refcnt) << _Py_REF_SHARED_SHIFT) + (flags)) - -// NOTE: In non-free-threaded builds, `struct _PyMutex` is defined in -// pycore_lock.h. See pycore_lock.h for more details. -struct _PyMutex { uint8_t v; }; - struct _object { // ob_tid stores the thread id (or zero). It is also used by the GC and the // trashcan mechanism as a linked list pointer and by the GC to store the // computed "gc_refs" refcount. uintptr_t ob_tid; - uint16_t _padding; - struct _PyMutex ob_mutex; // per-object lock + uint16_t ob_flags; + PyMutex ob_mutex; // per-object lock uint8_t ob_gc_bits; // gc-related state uint32_t ob_ref_local; // local reference count Py_ssize_t ob_ref_shared; // shared (atomic) reference count @@ -254,13 +190,19 @@ _Py_ThreadId(void) tid = __readfsdword(24); #elif defined(_MSC_VER) && defined(_M_ARM64) tid = __getReg(18); +#elif defined(__MINGW32__) && defined(_M_X64) + tid = __readgsqword(48); +#elif defined(__MINGW32__) && defined(_M_IX86) + tid = __readfsdword(24); +#elif defined(__MINGW32__) && defined(_M_ARM64) + tid = __getReg(18); #elif defined(__i386__) __asm__("movl %%gs:0, %0" : "=r" (tid)); // 32-bit always uses GS #elif defined(__MACH__) && defined(__x86_64__) __asm__("movq %%gs:0, %0" : "=r" (tid)); // x86_64 macOSX uses GS #elif defined(__x86_64__) __asm__("movq %%fs:0, %0" : "=r" (tid)); // x86_64 Linux, BSD uses FS -#elif defined(__arm__) +#elif defined(__arm__) && __ARM_ARCH >= 7 __asm__ ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tid)); #elif defined(__aarch64__) && defined(__APPLE__) __asm__ ("mrs %0, tpidrro_el0" : "=r" (tid)); @@ -306,34 +248,30 @@ _Py_ThreadId(void) static inline Py_ALWAYS_INLINE int _Py_IsOwnedByCurrentThread(PyObject *ob) { - return ob->ob_tid == _Py_ThreadId(); -} -#endif - -static inline Py_ssize_t Py_REFCNT(PyObject *ob) { -#if !defined(Py_GIL_DISABLED) - return ob->ob_refcnt; +#ifdef _Py_THREAD_SANITIZER + return _Py_atomic_load_uintptr_relaxed(&ob->ob_tid) == _Py_ThreadId(); #else - uint32_t local = _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local); - if (local == _Py_IMMORTAL_REFCNT_LOCAL) { - return _Py_IMMORTAL_REFCNT; - } - Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared); - return _Py_STATIC_CAST(Py_ssize_t, local) + - Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, shared, _Py_REF_SHARED_SHIFT); + return ob->ob_tid == _Py_ThreadId(); #endif } -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_REFCNT(ob) Py_REFCNT(_PyObject_CAST(ob)) #endif +// Py_TYPE() implementation for the stable ABI +PyAPI_FUNC(PyTypeObject*) Py_TYPE(PyObject *ob); -// bpo-39573: The Py_SET_TYPE() function must be used to set an object type. -static inline PyTypeObject* Py_TYPE(PyObject *ob) { - return ob->ob_type; -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_TYPE(ob) Py_TYPE(_PyObject_CAST(ob)) +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030e0000 + // Stable ABI implements Py_TYPE() as a function call + // on limited C API version 3.14 and newer. +#else + static inline PyTypeObject* _Py_TYPE(PyObject *ob) + { + return ob->ob_type; + } + #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 + # define Py_TYPE(ob) _Py_TYPE(_PyObject_CAST(ob)) + #else + # define Py_TYPE(ob) _Py_TYPE(ob) + #endif #endif PyAPI_DATA(PyTypeObject) PyLong_Type; @@ -341,27 +279,14 @@ PyAPI_DATA(PyTypeObject) PyBool_Type; // bpo-39573: The Py_SET_SIZE() function must be used to set an object size. static inline Py_ssize_t Py_SIZE(PyObject *ob) { - assert(ob->ob_type != &PyLong_Type); - assert(ob->ob_type != &PyBool_Type); - PyVarObject *var_ob = _PyVarObject_CAST(ob); - return var_ob->ob_size; + assert(Py_TYPE(ob) != &PyLong_Type); + assert(Py_TYPE(ob) != &PyBool_Type); + return _PyVarObject_CAST(ob)->ob_size; } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SIZE(ob) Py_SIZE(_PyObject_CAST(ob)) #endif -static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) -{ -#if defined(Py_GIL_DISABLED) - return (op->ob_ref_local == _Py_IMMORTAL_REFCNT_LOCAL); -#elif SIZEOF_VOID_P > 4 - return (_Py_CAST(PY_INT32_T, op->ob_refcnt) < 0); -#else - return (op->ob_refcnt == _Py_IMMORTAL_REFCNT); -#endif -} -#define _Py_IsImmortal(op) _Py_IsImmortal(_PyObject_CAST(op)) - static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { return Py_TYPE(ob) == type; } @@ -370,55 +295,6 @@ static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { #endif -// Py_SET_REFCNT() implementation for stable ABI -PyAPI_FUNC(void) _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt); - -static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { -#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030d0000 - // Stable ABI implements Py_SET_REFCNT() as a function call - // on limited C API version 3.13 and newer. - _Py_SetRefcnt(ob, refcnt); -#else - // This immortal check is for code that is unaware of immortal objects. - // The runtime tracks these objects and we should avoid as much - // as possible having extensions inadvertently change the refcnt - // of an immortalized object. - if (_Py_IsImmortal(ob)) { - return; - } - -#ifndef Py_GIL_DISABLED - ob->ob_refcnt = refcnt; -#else - if (_Py_IsOwnedByCurrentThread(ob)) { - if ((size_t)refcnt > (size_t)UINT32_MAX) { - // On overflow, make the object immortal - ob->ob_tid = _Py_UNOWNED_TID; - ob->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; - ob->ob_ref_shared = 0; - } - else { - // Set local refcount to desired refcount and shared refcount - // to zero, but preserve the shared refcount flags. - ob->ob_ref_local = _Py_STATIC_CAST(uint32_t, refcnt); - ob->ob_ref_shared &= _Py_REF_SHARED_FLAG_MASK; - } - } - else { - // Set local refcount to zero and shared refcount to desired refcount. - // Mark the object as merged. - ob->ob_tid = _Py_UNOWNED_TID; - ob->ob_ref_local = 0; - ob->ob_ref_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED); - } -#endif // Py_GIL_DISABLED -#endif // Py_LIMITED_API+0 < 0x030d0000 -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SET_REFCNT(ob, refcnt) Py_SET_REFCNT(_PyObject_CAST(ob), (refcnt)) -#endif - - static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { ob->ob_type = type; } @@ -427,8 +303,8 @@ static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { #endif static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { - assert(ob->ob_base.ob_type != &PyLong_Type); - assert(ob->ob_base.ob_type != &PyBool_Type); + assert(Py_TYPE(_PyObject_CAST(ob)) != &PyLong_Type); + assert(Py_TYPE(_PyObject_CAST(ob)) != &PyBool_Type); #ifdef Py_GIL_DISABLED _Py_atomic_store_ssize_relaxed(&ob->ob_size, size); #else @@ -522,11 +398,19 @@ PyAPI_FUNC(void *) PyType_GetModuleState(PyTypeObject *); PyAPI_FUNC(PyObject *) PyType_GetName(PyTypeObject *); PyAPI_FUNC(PyObject *) PyType_GetQualName(PyTypeObject *); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030D0000 +PyAPI_FUNC(PyObject *) PyType_GetFullyQualifiedName(PyTypeObject *type); +PyAPI_FUNC(PyObject *) PyType_GetModuleName(PyTypeObject *type); +#endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030C0000 PyAPI_FUNC(PyObject *) PyType_FromMetaclass(PyTypeObject*, PyObject*, PyType_Spec*, PyObject*); PyAPI_FUNC(void *) PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls); PyAPI_FUNC(Py_ssize_t) PyType_GetTypeDataSize(PyTypeObject *cls); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030E0000 +PyAPI_FUNC(int) PyType_GetBaseByToken(PyTypeObject *, void *, PyTypeObject **); +#define Py_TP_USE_SPEC NULL +#endif /* Generic type check */ PyAPI_FUNC(int) PyType_IsSubtype(PyTypeObject *, PyTypeObject *); @@ -626,13 +510,18 @@ given type object has a specified feature. /* Track types initialized using _PyStaticType_InitBuiltin(). */ #define _Py_TPFLAGS_STATIC_BUILTIN (1 << 1) +/* The values array is placed inline directly after the rest of + * the object. Implies Py_TPFLAGS_HAVE_GC. + */ +#define Py_TPFLAGS_INLINE_VALUES (1 << 2) + /* Placement of weakref pointers are managed by the VM, not by the type. * The VM will automatically set tp_weaklistoffset. */ #define Py_TPFLAGS_MANAGED_WEAKREF (1 << 3) /* Placement of dict (and values) pointers are managed by the VM, not by the type. - * The VM will automatically set tp_dictoffset. + * The VM will automatically set tp_dictoffset. Implies Py_TPFLAGS_HAVE_GC. */ #define Py_TPFLAGS_MANAGED_DICT (1 << 4) @@ -685,7 +574,7 @@ given type object has a specified feature. /* Objects behave like an unbound method */ #define Py_TPFLAGS_METHOD_DESCRIPTOR (1UL << 17) -/* Object has up-to-date type attribute cache */ +/* Unused. Legacy flag */ #define Py_TPFLAGS_VALID_VERSION_TAG (1UL << 19) /* Type is abstract and cannot be instantiated */ @@ -730,338 +619,20 @@ given type object has a specified feature. #define Py_TPFLAGS_HAVE_VERSION_TAG (1UL << 18) -/* -The macros Py_INCREF(op) and Py_DECREF(op) are used to increment or decrement -reference counts. Py_DECREF calls the object's deallocator function when -the refcount falls to 0; for -objects that don't contain references to other objects or heap memory -this can be the standard function free(). Both macros can be used -wherever a void expression is allowed. The argument must not be a -NULL pointer. If it may be NULL, use Py_XINCREF/Py_XDECREF instead. -The macro _Py_NewReference(op) initialize reference counts to 1, and -in special builds (Py_REF_DEBUG, Py_TRACE_REFS) performs additional -bookkeeping appropriate to the special build. - -We assume that the reference count field can never overflow; this can -be proven when the size of the field is the same as the pointer size, so -we ignore the possibility. Provided a C int is at least 32 bits (which -is implicitly assumed in many parts of this code), that's enough for -about 2**31 references to an object. - -XXX The following became out of date in Python 2.2, but I'm not sure -XXX what the full truth is now. Certainly, heap-allocated type objects -XXX can and should be deallocated. -Type objects should never be deallocated; the type pointer in an object -is not considered to be a reference to the type object, to save -complications in the deallocation function. (This is actually a -decision that's up to the implementer of each new type so if you want, -you can count such references to the type object.) -*/ - -#if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API) -PyAPI_FUNC(void) _Py_NegativeRefcount(const char *filename, int lineno, - PyObject *op); -PyAPI_FUNC(void) _Py_INCREF_IncRefTotal(void); -PyAPI_FUNC(void) _Py_DECREF_DecRefTotal(void); -#endif // Py_REF_DEBUG && !Py_LIMITED_API - -PyAPI_FUNC(void) _Py_Dealloc(PyObject *); - -/* -These are provided as conveniences to Python runtime embedders, so that -they can have object code that is not dependent on Python compilation flags. -*/ -PyAPI_FUNC(void) Py_IncRef(PyObject *); -PyAPI_FUNC(void) Py_DecRef(PyObject *); - -// Similar to Py_IncRef() and Py_DecRef() but the argument must be non-NULL. -// Private functions used by Py_INCREF() and Py_DECREF(). -PyAPI_FUNC(void) _Py_IncRef(PyObject *); -PyAPI_FUNC(void) _Py_DecRef(PyObject *); +#define Py_CONSTANT_NONE 0 +#define Py_CONSTANT_FALSE 1 +#define Py_CONSTANT_TRUE 2 +#define Py_CONSTANT_ELLIPSIS 3 +#define Py_CONSTANT_NOT_IMPLEMENTED 4 +#define Py_CONSTANT_ZERO 5 +#define Py_CONSTANT_ONE 6 +#define Py_CONSTANT_EMPTY_STR 7 +#define Py_CONSTANT_EMPTY_BYTES 8 +#define Py_CONSTANT_EMPTY_TUPLE 9 -static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) -{ -#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) - // Stable ABI implements Py_INCREF() as a function call on limited C API - // version 3.12 and newer, and on Python built in debug mode. _Py_IncRef() - // was added to Python 3.10.0a7, use Py_IncRef() on older Python versions. - // Py_IncRef() accepts NULL whereas _Py_IncRef() doesn't. -# if Py_LIMITED_API+0 >= 0x030a00A7 - _Py_IncRef(op); -# else - Py_IncRef(op); -# endif -#else - // Non-limited C API and limited C API for Python 3.9 and older access - // directly PyObject.ob_refcnt. -#if defined(Py_GIL_DISABLED) - uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); - uint32_t new_local = local + 1; - if (new_local == 0) { - // local is equal to _Py_IMMORTAL_REFCNT: do nothing - return; - } - if (_Py_IsOwnedByCurrentThread(op)) { - _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, new_local); - } - else { - _Py_atomic_add_ssize(&op->ob_ref_shared, (1 << _Py_REF_SHARED_SHIFT)); - } -#elif SIZEOF_VOID_P > 4 - // Portable saturated add, branching on the carry flag and set low bits - PY_UINT32_T cur_refcnt = op->ob_refcnt_split[PY_BIG_ENDIAN]; - PY_UINT32_T new_refcnt = cur_refcnt + 1; - if (new_refcnt == 0) { - // cur_refcnt is equal to _Py_IMMORTAL_REFCNT: the object is immortal, - // do nothing - return; - } - op->ob_refcnt_split[PY_BIG_ENDIAN] = new_refcnt; -#else - // Explicitly check immortality against the immortal value - if (_Py_IsImmortal(op)) { - return; - } - op->ob_refcnt++; -#endif - _Py_INCREF_STAT_INC(); -#ifdef Py_REF_DEBUG - _Py_INCREF_IncRefTotal(); -#endif -#endif -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op)) -#endif - - -#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) -// Implements Py_DECREF on objects not owned by the current thread. -PyAPI_FUNC(void) _Py_DecRefShared(PyObject *); -PyAPI_FUNC(void) _Py_DecRefSharedDebug(PyObject *, const char *, int); - -// Called from Py_DECREF by the owning thread when the local refcount reaches -// zero. The call will deallocate the object if the shared refcount is also -// zero. Otherwise, the thread gives up ownership and merges the reference -// count fields. -PyAPI_FUNC(void) _Py_MergeZeroLocalRefcount(PyObject *); -#endif - -#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) -// Stable ABI implements Py_DECREF() as a function call on limited C API -// version 3.12 and newer, and on Python built in debug mode. _Py_DecRef() was -// added to Python 3.10.0a7, use Py_DecRef() on older Python versions. -// Py_DecRef() accepts NULL whereas _Py_IncRef() doesn't. -static inline void Py_DECREF(PyObject *op) { -# if Py_LIMITED_API+0 >= 0x030a00A7 - _Py_DecRef(op); -# else - Py_DecRef(op); -# endif -} -#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) - -#elif defined(Py_GIL_DISABLED) && defined(Py_REF_DEBUG) -static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) -{ - uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); - if (local == _Py_IMMORTAL_REFCNT_LOCAL) { - return; - } - _Py_DECREF_STAT_INC(); - _Py_DECREF_DecRefTotal(); - if (_Py_IsOwnedByCurrentThread(op)) { - if (local == 0) { - _Py_NegativeRefcount(filename, lineno, op); - } - local--; - _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); - if (local == 0) { - _Py_MergeZeroLocalRefcount(op); - } - } - else { - _Py_DecRefSharedDebug(op, filename, lineno); - } -} -#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) - -#elif defined(Py_GIL_DISABLED) -static inline void Py_DECREF(PyObject *op) -{ - uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); - if (local == _Py_IMMORTAL_REFCNT_LOCAL) { - return; - } - _Py_DECREF_STAT_INC(); - if (_Py_IsOwnedByCurrentThread(op)) { - local--; - _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); - if (local == 0) { - _Py_MergeZeroLocalRefcount(op); - } - } - else { - _Py_DecRefShared(op); - } -} -#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) - -#elif defined(Py_REF_DEBUG) -static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) -{ - if (op->ob_refcnt <= 0) { - _Py_NegativeRefcount(filename, lineno, op); - } - if (_Py_IsImmortal(op)) { - return; - } - _Py_DECREF_STAT_INC(); - _Py_DECREF_DecRefTotal(); - if (--op->ob_refcnt == 0) { - _Py_Dealloc(op); - } -} -#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) - -#else -static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op) -{ - // Non-limited C API and limited C API for Python 3.9 and older access - // directly PyObject.ob_refcnt. - if (_Py_IsImmortal(op)) { - return; - } - _Py_DECREF_STAT_INC(); - if (--op->ob_refcnt == 0) { - _Py_Dealloc(op); - } -} -#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) -#endif - - -/* Safely decref `op` and set `op` to NULL, especially useful in tp_clear - * and tp_dealloc implementations. - * - * Note that "the obvious" code can be deadly: - * - * Py_XDECREF(op); - * op = NULL; - * - * Typically, `op` is something like self->containee, and `self` is done - * using its `containee` member. In the code sequence above, suppose - * `containee` is non-NULL with a refcount of 1. Its refcount falls to - * 0 on the first line, which can trigger an arbitrary amount of code, - * possibly including finalizers (like __del__ methods or weakref callbacks) - * coded in Python, which in turn can release the GIL and allow other threads - * to run, etc. Such code may even invoke methods of `self` again, or cause - * cyclic gc to trigger, but-- oops! --self->containee still points to the - * object being torn down, and it may be in an insane state while being torn - * down. This has in fact been a rich historic source of miserable (rare & - * hard-to-diagnose) segfaulting (and other) bugs. - * - * The safe way is: - * - * Py_CLEAR(op); - * - * That arranges to set `op` to NULL _before_ decref'ing, so that any code - * triggered as a side-effect of `op` getting torn down no longer believes - * `op` points to a valid object. - * - * There are cases where it's safe to use the naive code, but they're brittle. - * For example, if `op` points to a Python integer, you know that destroying - * one of those can't cause problems -- but in part that relies on that - * Python integers aren't currently weakly referencable. Best practice is - * to use Py_CLEAR() even if you can't think of a reason for why you need to. - * - * gh-98724: Use a temporary variable to only evaluate the macro argument once, - * to avoid the duplication of side effects if the argument has side effects. - * - * gh-99701: If the PyObject* type is used with casting arguments to PyObject*, - * the code can be miscompiled with strict aliasing because of type punning. - * With strict aliasing, a compiler considers that two pointers of different - * types cannot read or write the same memory which enables optimization - * opportunities. - * - * If available, use _Py_TYPEOF() to use the 'op' type for temporary variables, - * and so avoid type punning. Otherwise, use memcpy() which causes type erasure - * and so prevents the compiler to reuse an old cached 'op' value after - * Py_CLEAR(). - */ -#ifdef _Py_TYPEOF -#define Py_CLEAR(op) \ - do { \ - _Py_TYPEOF(op)* _tmp_op_ptr = &(op); \ - _Py_TYPEOF(op) _tmp_old_op = (*_tmp_op_ptr); \ - if (_tmp_old_op != NULL) { \ - *_tmp_op_ptr = _Py_NULL; \ - Py_DECREF(_tmp_old_op); \ - } \ - } while (0) -#else -#define Py_CLEAR(op) \ - do { \ - PyObject **_tmp_op_ptr = _Py_CAST(PyObject**, &(op)); \ - PyObject *_tmp_old_op = (*_tmp_op_ptr); \ - if (_tmp_old_op != NULL) { \ - PyObject *_null_ptr = _Py_NULL; \ - memcpy(_tmp_op_ptr, &_null_ptr, sizeof(PyObject*)); \ - Py_DECREF(_tmp_old_op); \ - } \ - } while (0) -#endif - - -/* Function to use in case the object pointer can be NULL: */ -static inline void Py_XINCREF(PyObject *op) -{ - if (op != _Py_NULL) { - Py_INCREF(op); - } -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_XINCREF(op) Py_XINCREF(_PyObject_CAST(op)) -#endif - -static inline void Py_XDECREF(PyObject *op) -{ - if (op != _Py_NULL) { - Py_DECREF(op); - } -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_XDECREF(op) Py_XDECREF(_PyObject_CAST(op)) -#endif - -// Create a new strong reference to an object: -// increment the reference count of the object and return the object. -PyAPI_FUNC(PyObject*) Py_NewRef(PyObject *obj); - -// Similar to Py_NewRef(), but the object can be NULL. -PyAPI_FUNC(PyObject*) Py_XNewRef(PyObject *obj); - -static inline PyObject* _Py_NewRef(PyObject *obj) -{ - Py_INCREF(obj); - return obj; -} - -static inline PyObject* _Py_XNewRef(PyObject *obj) -{ - Py_XINCREF(obj); - return obj; -} - -// Py_NewRef() and Py_XNewRef() are exported as functions for the stable ABI. -// Names overridden with macros by static inline functions for best -// performances. -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj)) -# define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj)) -#else -# define Py_NewRef(obj) _Py_NewRef(obj) -# define Py_XNewRef(obj) _Py_XNewRef(obj) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +PyAPI_FUNC(PyObject*) Py_GetConstant(unsigned int constant_id); +PyAPI_FUNC(PyObject*) Py_GetConstantBorrowed(unsigned int constant_id); #endif @@ -1070,7 +641,12 @@ _Py_NoneStruct is an object of undefined type which can be used in contexts where NULL (nil) is not suitable (since NULL often means 'error'). */ PyAPI_DATA(PyObject) _Py_NoneStruct; /* Don't use this directly */ -#define Py_None (&_Py_NoneStruct) + +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030D0000 +# define Py_None Py_GetConstantBorrowed(Py_CONSTANT_NONE) +#else +# define Py_None (&_Py_NoneStruct) +#endif // Test if an object is the None singleton, the same as "x is None" in Python. PyAPI_FUNC(int) Py_IsNone(PyObject *x); @@ -1084,7 +660,12 @@ Py_NotImplemented is a singleton used to signal that an operation is not implemented for a given type combination. */ PyAPI_DATA(PyObject) _Py_NotImplementedStruct; /* Don't use this directly */ -#define Py_NotImplemented (&_Py_NotImplementedStruct) + +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030D0000 +# define Py_NotImplemented Py_GetConstantBorrowed(Py_CONSTANT_NOT_IMPLEMENTED) +#else +# define Py_NotImplemented (&_Py_NotImplementedStruct) +#endif /* Macro for returning Py_NotImplemented from a function */ #define Py_RETURN_NOTIMPLEMENTED return Py_NotImplemented @@ -1193,7 +774,11 @@ PyType_HasFeature(PyTypeObject *type, unsigned long feature) // PyTypeObject is opaque in the limited C API flags = PyType_GetFlags(type); #else - flags = type->tp_flags; +# ifdef Py_GIL_DISABLED + flags = _Py_atomic_load_ulong_relaxed(&type->tp_flags); +# else + flags = type->tp_flags; +# endif #endif return ((flags & feature) != 0); } @@ -1217,6 +802,14 @@ static inline int PyType_CheckExact(PyObject *op) { # define PyType_CheckExact(op) PyType_CheckExact(_PyObject_CAST(op)) #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +PyAPI_FUNC(PyObject *) PyType_GetModuleByDef(PyTypeObject *, PyModuleDef *); +#endif + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030e0000 +PyAPI_FUNC(int) PyType_Freeze(PyTypeObject *type); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index fe969342ee79e7..4a9fc15dcd2880 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -11,37 +11,37 @@ extern "C" { /* Instruction opcodes for compiled code */ #define CACHE 0 -#define BEFORE_ASYNC_WITH 1 -#define BEFORE_WITH 2 +#define BINARY_SLICE 1 +#define BINARY_SUBSCR 2 #define BINARY_OP_INPLACE_ADD_UNICODE 3 -#define BINARY_SLICE 4 -#define BINARY_SUBSCR 5 -#define CHECK_EG_MATCH 6 -#define CHECK_EXC_MATCH 7 -#define CLEANUP_THROW 8 -#define DELETE_SUBSCR 9 -#define END_ASYNC_FOR 10 -#define END_FOR 11 -#define END_SEND 12 -#define EXIT_INIT_CHECK 13 -#define FORMAT_SIMPLE 14 -#define FORMAT_WITH_SPEC 15 -#define GET_AITER 16 +#define CALL_FUNCTION_EX 4 +#define CHECK_EG_MATCH 5 +#define CHECK_EXC_MATCH 6 +#define CLEANUP_THROW 7 +#define DELETE_SUBSCR 8 +#define END_ASYNC_FOR 9 +#define END_FOR 10 +#define END_SEND 11 +#define EXIT_INIT_CHECK 12 +#define FORMAT_SIMPLE 13 +#define FORMAT_WITH_SPEC 14 +#define GET_AITER 15 +#define GET_ANEXT 16 #define RESERVED 17 -#define GET_ANEXT 18 -#define GET_ITER 19 -#define GET_LEN 20 -#define GET_YIELD_FROM_ITER 21 -#define INTERPRETER_EXIT 22 -#define LOAD_ASSERTION_ERROR 23 -#define LOAD_BUILD_CLASS 24 -#define LOAD_LOCALS 25 -#define MAKE_FUNCTION 26 -#define MATCH_KEYS 27 -#define MATCH_MAPPING 28 -#define MATCH_SEQUENCE 29 -#define NOP 30 -#define POP_EXCEPT 31 +#define GET_ITER 18 +#define GET_LEN 19 +#define GET_YIELD_FROM_ITER 20 +#define INTERPRETER_EXIT 21 +#define LOAD_BUILD_CLASS 22 +#define LOAD_LOCALS 23 +#define MAKE_FUNCTION 24 +#define MATCH_KEYS 25 +#define MATCH_MAPPING 26 +#define MATCH_SEQUENCE 27 +#define NOP 28 +#define NOT_TAKEN 29 +#define POP_EXCEPT 30 +#define POP_ITER 31 #define POP_TOP 32 #define PUSH_EXC_INFO 33 #define PUSH_NULL 34 @@ -56,53 +56,53 @@ extern "C" { #define UNARY_NOT 43 #define WITH_EXCEPT_START 44 #define BINARY_OP 45 -#define BUILD_CONST_KEY_MAP 46 -#define BUILD_LIST 47 -#define BUILD_MAP 48 -#define BUILD_SET 49 -#define BUILD_SLICE 50 -#define BUILD_STRING 51 -#define BUILD_TUPLE 52 -#define CALL 53 -#define CALL_FUNCTION_EX 54 -#define CALL_INTRINSIC_1 55 -#define CALL_INTRINSIC_2 56 -#define CALL_KW 57 -#define COMPARE_OP 58 -#define CONTAINS_OP 59 -#define CONVERT_VALUE 60 -#define COPY 61 -#define COPY_FREE_VARS 62 -#define DELETE_ATTR 63 -#define DELETE_DEREF 64 -#define DELETE_FAST 65 -#define DELETE_GLOBAL 66 -#define DELETE_NAME 67 -#define DICT_MERGE 68 -#define DICT_UPDATE 69 -#define ENTER_EXECUTOR 70 -#define EXTENDED_ARG 71 -#define FOR_ITER 72 -#define GET_AWAITABLE 73 -#define IMPORT_FROM 74 -#define IMPORT_NAME 75 -#define IS_OP 76 -#define JUMP_BACKWARD 77 -#define JUMP_BACKWARD_NO_INTERRUPT 78 -#define JUMP_FORWARD 79 -#define LIST_APPEND 80 -#define LIST_EXTEND 81 -#define LOAD_ATTR 82 -#define LOAD_CONST 83 -#define LOAD_DEREF 84 -#define LOAD_FAST 85 -#define LOAD_FAST_AND_CLEAR 86 -#define LOAD_FAST_CHECK 87 -#define LOAD_FAST_LOAD_FAST 88 -#define LOAD_FROM_DICT_OR_DEREF 89 -#define LOAD_FROM_DICT_OR_GLOBALS 90 -#define LOAD_GLOBAL 91 -#define LOAD_NAME 92 +#define BUILD_LIST 46 +#define BUILD_MAP 47 +#define BUILD_SET 48 +#define BUILD_SLICE 49 +#define BUILD_STRING 50 +#define BUILD_TUPLE 51 +#define CALL 52 +#define CALL_INTRINSIC_1 53 +#define CALL_INTRINSIC_2 54 +#define CALL_KW 55 +#define COMPARE_OP 56 +#define CONTAINS_OP 57 +#define CONVERT_VALUE 58 +#define COPY 59 +#define COPY_FREE_VARS 60 +#define DELETE_ATTR 61 +#define DELETE_DEREF 62 +#define DELETE_FAST 63 +#define DELETE_GLOBAL 64 +#define DELETE_NAME 65 +#define DICT_MERGE 66 +#define DICT_UPDATE 67 +#define EXTENDED_ARG 68 +#define FOR_ITER 69 +#define GET_AWAITABLE 70 +#define IMPORT_FROM 71 +#define IMPORT_NAME 72 +#define IS_OP 73 +#define JUMP_BACKWARD 74 +#define JUMP_BACKWARD_NO_INTERRUPT 75 +#define JUMP_FORWARD 76 +#define LIST_APPEND 77 +#define LIST_EXTEND 78 +#define LOAD_ATTR 79 +#define LOAD_COMMON_CONSTANT 80 +#define LOAD_CONST 81 +#define LOAD_DEREF 82 +#define LOAD_FAST 83 +#define LOAD_FAST_AND_CLEAR 84 +#define LOAD_FAST_CHECK 85 +#define LOAD_FAST_LOAD_FAST 86 +#define LOAD_FROM_DICT_OR_DEREF 87 +#define LOAD_FROM_DICT_OR_GLOBALS 88 +#define LOAD_GLOBAL 89 +#define LOAD_NAME 90 +#define LOAD_SMALL_INT 91 +#define LOAD_SPECIAL 92 #define LOAD_SUPER_ATTR 93 #define MAKE_CELL 94 #define MAP_ADD 95 @@ -113,126 +113,139 @@ extern "C" { #define POP_JUMP_IF_TRUE 100 #define RAISE_VARARGS 101 #define RERAISE 102 -#define RETURN_CONST 103 -#define SEND 104 -#define SET_ADD 105 -#define SET_FUNCTION_ATTRIBUTE 106 -#define SET_UPDATE 107 -#define STORE_ATTR 108 -#define STORE_DEREF 109 -#define STORE_FAST 110 -#define STORE_FAST_LOAD_FAST 111 -#define STORE_FAST_STORE_FAST 112 -#define STORE_GLOBAL 113 -#define STORE_NAME 114 -#define SWAP 115 -#define UNPACK_EX 116 -#define UNPACK_SEQUENCE 117 -#define YIELD_VALUE 118 +#define SEND 103 +#define SET_ADD 104 +#define SET_FUNCTION_ATTRIBUTE 105 +#define SET_UPDATE 106 +#define STORE_ATTR 107 +#define STORE_DEREF 108 +#define STORE_FAST 109 +#define STORE_FAST_LOAD_FAST 110 +#define STORE_FAST_STORE_FAST 111 +#define STORE_GLOBAL 112 +#define STORE_NAME 113 +#define SWAP 114 +#define UNPACK_EX 115 +#define UNPACK_SEQUENCE 116 +#define YIELD_VALUE 117 #define RESUME 149 #define BINARY_OP_ADD_FLOAT 150 #define BINARY_OP_ADD_INT 151 #define BINARY_OP_ADD_UNICODE 152 -#define BINARY_OP_MULTIPLY_FLOAT 153 -#define BINARY_OP_MULTIPLY_INT 154 -#define BINARY_OP_SUBTRACT_FLOAT 155 -#define BINARY_OP_SUBTRACT_INT 156 -#define BINARY_SUBSCR_DICT 157 -#define BINARY_SUBSCR_GETITEM 158 -#define BINARY_SUBSCR_LIST_INT 159 -#define BINARY_SUBSCR_STR_INT 160 -#define BINARY_SUBSCR_TUPLE_INT 161 -#define CALL_ALLOC_AND_ENTER_INIT 162 -#define CALL_BOUND_METHOD_EXACT_ARGS 163 -#define CALL_BUILTIN_CLASS 164 -#define CALL_BUILTIN_FAST 165 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 166 -#define CALL_BUILTIN_O 167 -#define CALL_ISINSTANCE 168 -#define CALL_LEN 169 -#define CALL_LIST_APPEND 170 -#define CALL_METHOD_DESCRIPTOR_FAST 171 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 172 -#define CALL_METHOD_DESCRIPTOR_NOARGS 173 -#define CALL_METHOD_DESCRIPTOR_O 174 -#define CALL_PY_EXACT_ARGS 175 -#define CALL_PY_WITH_DEFAULTS 176 -#define CALL_STR_1 177 -#define CALL_TUPLE_1 178 -#define CALL_TYPE_1 179 -#define COMPARE_OP_FLOAT 180 -#define COMPARE_OP_INT 181 -#define COMPARE_OP_STR 182 -#define FOR_ITER_GEN 183 -#define FOR_ITER_LIST 184 -#define FOR_ITER_RANGE 185 -#define FOR_ITER_TUPLE 186 -#define LOAD_ATTR_CLASS 187 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 188 -#define LOAD_ATTR_INSTANCE_VALUE 189 -#define LOAD_ATTR_METHOD_LAZY_DICT 190 -#define LOAD_ATTR_METHOD_NO_DICT 191 -#define LOAD_ATTR_METHOD_WITH_VALUES 192 -#define LOAD_ATTR_MODULE 193 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 194 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 195 -#define LOAD_ATTR_PROPERTY 196 -#define LOAD_ATTR_SLOT 197 -#define LOAD_ATTR_WITH_HINT 198 -#define LOAD_GLOBAL_BUILTIN 199 -#define LOAD_GLOBAL_MODULE 200 -#define LOAD_SUPER_ATTR_ATTR 201 -#define LOAD_SUPER_ATTR_METHOD 202 -#define RESUME_CHECK 203 -#define SEND_GEN 204 -#define STORE_ATTR_INSTANCE_VALUE 205 -#define STORE_ATTR_SLOT 206 -#define STORE_ATTR_WITH_HINT 207 -#define STORE_SUBSCR_DICT 208 -#define STORE_SUBSCR_LIST_INT 209 -#define TO_BOOL_ALWAYS_TRUE 210 -#define TO_BOOL_BOOL 211 -#define TO_BOOL_INT 212 -#define TO_BOOL_LIST 213 -#define TO_BOOL_NONE 214 -#define TO_BOOL_STR 215 -#define UNPACK_SEQUENCE_LIST 216 -#define UNPACK_SEQUENCE_TUPLE 217 -#define UNPACK_SEQUENCE_TWO_TUPLE 218 -#define INSTRUMENTED_RESUME 236 -#define INSTRUMENTED_END_FOR 237 -#define INSTRUMENTED_END_SEND 238 -#define INSTRUMENTED_RETURN_VALUE 239 -#define INSTRUMENTED_RETURN_CONST 240 -#define INSTRUMENTED_YIELD_VALUE 241 -#define INSTRUMENTED_LOAD_SUPER_ATTR 242 -#define INSTRUMENTED_FOR_ITER 243 -#define INSTRUMENTED_CALL 244 -#define INSTRUMENTED_CALL_KW 245 -#define INSTRUMENTED_CALL_FUNCTION_EX 246 -#define INSTRUMENTED_INSTRUCTION 247 -#define INSTRUMENTED_JUMP_FORWARD 248 -#define INSTRUMENTED_JUMP_BACKWARD 249 -#define INSTRUMENTED_POP_JUMP_IF_TRUE 250 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 251 -#define INSTRUMENTED_POP_JUMP_IF_NONE 252 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 253 +#define BINARY_OP_EXTEND 153 +#define BINARY_OP_MULTIPLY_FLOAT 154 +#define BINARY_OP_MULTIPLY_INT 155 +#define BINARY_OP_SUBTRACT_FLOAT 156 +#define BINARY_OP_SUBTRACT_INT 157 +#define BINARY_SUBSCR_DICT 158 +#define BINARY_SUBSCR_GETITEM 159 +#define BINARY_SUBSCR_LIST_INT 160 +#define BINARY_SUBSCR_STR_INT 161 +#define BINARY_SUBSCR_TUPLE_INT 162 +#define CALL_ALLOC_AND_ENTER_INIT 163 +#define CALL_BOUND_METHOD_EXACT_ARGS 164 +#define CALL_BOUND_METHOD_GENERAL 165 +#define CALL_BUILTIN_CLASS 166 +#define CALL_BUILTIN_FAST 167 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 168 +#define CALL_BUILTIN_O 169 +#define CALL_ISINSTANCE 170 +#define CALL_KW_BOUND_METHOD 171 +#define CALL_KW_NON_PY 172 +#define CALL_KW_PY 173 +#define CALL_LEN 174 +#define CALL_LIST_APPEND 175 +#define CALL_METHOD_DESCRIPTOR_FAST 176 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 177 +#define CALL_METHOD_DESCRIPTOR_NOARGS 178 +#define CALL_METHOD_DESCRIPTOR_O 179 +#define CALL_NON_PY_GENERAL 180 +#define CALL_PY_EXACT_ARGS 181 +#define CALL_PY_GENERAL 182 +#define CALL_STR_1 183 +#define CALL_TUPLE_1 184 +#define CALL_TYPE_1 185 +#define COMPARE_OP_FLOAT 186 +#define COMPARE_OP_INT 187 +#define COMPARE_OP_STR 188 +#define CONTAINS_OP_DICT 189 +#define CONTAINS_OP_SET 190 +#define FOR_ITER_GEN 191 +#define FOR_ITER_LIST 192 +#define FOR_ITER_RANGE 193 +#define FOR_ITER_TUPLE 194 +#define JUMP_BACKWARD_JIT 195 +#define JUMP_BACKWARD_NO_JIT 196 +#define LOAD_ATTR_CLASS 197 +#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 198 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 199 +#define LOAD_ATTR_INSTANCE_VALUE 200 +#define LOAD_ATTR_METHOD_LAZY_DICT 201 +#define LOAD_ATTR_METHOD_NO_DICT 202 +#define LOAD_ATTR_METHOD_WITH_VALUES 203 +#define LOAD_ATTR_MODULE 204 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 205 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 206 +#define LOAD_ATTR_PROPERTY 207 +#define LOAD_ATTR_SLOT 208 +#define LOAD_ATTR_WITH_HINT 209 +#define LOAD_CONST_IMMORTAL 210 +#define LOAD_CONST_MORTAL 211 +#define LOAD_GLOBAL_BUILTIN 212 +#define LOAD_GLOBAL_MODULE 213 +#define LOAD_SUPER_ATTR_ATTR 214 +#define LOAD_SUPER_ATTR_METHOD 215 +#define RESUME_CHECK 216 +#define SEND_GEN 217 +#define STORE_ATTR_INSTANCE_VALUE 218 +#define STORE_ATTR_SLOT 219 +#define STORE_ATTR_WITH_HINT 220 +#define STORE_SUBSCR_DICT 221 +#define STORE_SUBSCR_LIST_INT 222 +#define TO_BOOL_ALWAYS_TRUE 223 +#define TO_BOOL_BOOL 224 +#define TO_BOOL_INT 225 +#define TO_BOOL_LIST 226 +#define TO_BOOL_NONE 227 +#define TO_BOOL_STR 228 +#define UNPACK_SEQUENCE_LIST 229 +#define UNPACK_SEQUENCE_TUPLE 230 +#define UNPACK_SEQUENCE_TWO_TUPLE 231 +#define INSTRUMENTED_END_FOR 235 +#define INSTRUMENTED_POP_ITER 236 +#define INSTRUMENTED_END_SEND 237 +#define INSTRUMENTED_LOAD_SUPER_ATTR 238 +#define INSTRUMENTED_FOR_ITER 239 +#define INSTRUMENTED_CALL_KW 240 +#define INSTRUMENTED_CALL_FUNCTION_EX 241 +#define INSTRUMENTED_INSTRUCTION 242 +#define INSTRUMENTED_JUMP_FORWARD 243 +#define INSTRUMENTED_NOT_TAKEN 244 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 245 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 246 +#define INSTRUMENTED_POP_JUMP_IF_NONE 247 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 248 +#define INSTRUMENTED_RESUME 249 +#define INSTRUMENTED_RETURN_VALUE 250 +#define INSTRUMENTED_YIELD_VALUE 251 +#define INSTRUMENTED_CALL 252 +#define INSTRUMENTED_JUMP_BACKWARD 253 #define INSTRUMENTED_LINE 254 +#define ENTER_EXECUTOR 255 #define JUMP 256 -#define JUMP_NO_INTERRUPT 257 -#define LOAD_CLOSURE 258 -#define LOAD_METHOD 259 -#define LOAD_SUPER_METHOD 260 -#define LOAD_ZERO_SUPER_ATTR 261 -#define LOAD_ZERO_SUPER_METHOD 262 -#define POP_BLOCK 263 -#define SETUP_CLEANUP 264 -#define SETUP_FINALLY 265 -#define SETUP_WITH 266 -#define STORE_FAST_MAYBE_NULL 267 +#define JUMP_IF_FALSE 257 +#define JUMP_IF_TRUE 258 +#define JUMP_NO_INTERRUPT 259 +#define LOAD_CLOSURE 260 +#define POP_BLOCK 261 +#define SETUP_CLEANUP 262 +#define SETUP_FINALLY 263 +#define SETUP_WITH 264 +#define STORE_FAST_MAYBE_NULL 265 #define HAVE_ARGUMENT 44 -#define MIN_INSTRUMENTED_OPCODE 236 +#define MIN_SPECIALIZED_OPCODE 150 +#define MIN_INSTRUMENTED_OPCODE 235 #ifdef __cplusplus } diff --git a/Include/patchlevel.h b/Include/patchlevel.h index ae2d22f80817d5..40e7d73b7a6634 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -1,4 +1,5 @@ - +#ifndef _Py_PATCHLEVEL_H +#define _Py_PATCHLEVEL_H /* Python version identification scheme. When the major or minor version changes, the VERSION variable in @@ -17,19 +18,32 @@ /* Version parsed out into numeric values */ /*--start constants--*/ #define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 13 +#define PY_MINOR_VERSION 14 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA #define PY_RELEASE_SERIAL 4 /* Version as a string */ -#define PY_VERSION "3.13.0a4+" +#define PY_VERSION "3.14.0a4+" /*--end constants--*/ + +#define _Py_PACK_FULL_VERSION(X, Y, Z, LEVEL, SERIAL) ( \ + (((X) & 0xff) << 24) | \ + (((Y) & 0xff) << 16) | \ + (((Z) & 0xff) << 8) | \ + (((LEVEL) & 0xf) << 4) | \ + (((SERIAL) & 0xf) << 0)) + /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. Use this for numeric comparisons, e.g. #if PY_VERSION_HEX >= ... */ -#define PY_VERSION_HEX ((PY_MAJOR_VERSION << 24) | \ - (PY_MINOR_VERSION << 16) | \ - (PY_MICRO_VERSION << 8) | \ - (PY_RELEASE_LEVEL << 4) | \ - (PY_RELEASE_SERIAL << 0)) +#define PY_VERSION_HEX _Py_PACK_FULL_VERSION( \ + PY_MAJOR_VERSION, \ + PY_MINOR_VERSION, \ + PY_MICRO_VERSION, \ + PY_RELEASE_LEVEL, \ + PY_RELEASE_SERIAL) + +// Public Py_PACK_VERSION is declared in pymacro.h; it needs . + +#endif //_Py_PATCHLEVEL_H diff --git a/Include/py_curses.h b/Include/py_curses.h index e46b08e9cc414e..e11bfedb17d205 100644 --- a/Include/py_curses.h +++ b/Include/py_curses.h @@ -23,23 +23,40 @@ # endif #endif -#if !defined(HAVE_CURSES_IS_PAD) && defined(WINDOW_HAS_FLAGS) -/* The following definition is necessary for ncurses 5.7; without it, - some of [n]curses.h set NCURSES_OPAQUE to 1, and then Python - can't get at the WINDOW flags field. */ +#if defined(WINDOW_HAS_FLAGS) && defined(__APPLE__) +/* gh-109617, gh-115383: we can rely on the default value for NCURSES_OPAQUE on + most platforms, but not on macOS. This is because, starting with Xcode 15, + Apple-provided ncurses.h comes from ncurses 6 (which defaults to opaque + structs) but can still be linked to older versions of ncurses dynamic + libraries which don't provide functions such as is_pad() to deal with opaque + structs. Setting NCURSES_OPAQUE to 0 is harmless in all ncurses releases to + this date (provided that a thread-safe implementation is not required), but + this might change in the future. This fix might become irrelevant once + support for macOS 13 or earlier is dropped. */ #define NCURSES_OPAQUE 0 #endif -#ifdef HAVE_NCURSES_H -#include -#else -#include +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include #endif -#ifdef HAVE_NCURSES_H +#ifdef NCURSES_VERSION /* configure was checking , but we will use , which has some or all these features. */ -#if !defined(WINDOW_HAS_FLAGS) && !(NCURSES_OPAQUE+0) +#if !defined(WINDOW_HAS_FLAGS) && \ + (NCURSES_VERSION_PATCH+0 < 20070303 || !(NCURSES_OPAQUE+0)) +/* the WINDOW flags field was always accessible in ncurses prior to 20070303; + after that, it depends on the value of NCURSES_OPAQUE. */ #define WINDOW_HAS_FLAGS 1 #endif #if !defined(HAVE_CURSES_IS_PAD) && NCURSES_VERSION_PATCH+0 >= 20090906 @@ -64,8 +81,6 @@ typedef struct { char *encoding; } PyCursesWindowObject; -#define PyCursesWindow_Check(v) Py_IS_TYPE((v), &PyCursesWindow_Type) - #define PyCurses_CAPSULE_NAME "_curses._C_API" @@ -82,6 +97,8 @@ static void **PyCurses_API; #define PyCursesInitialised {if (! ((int (*)(void))PyCurses_API[2]) () ) return NULL;} #define PyCursesInitialisedColor {if (! ((int (*)(void))PyCurses_API[3]) () ) return NULL;} +#define PyCursesWindow_Check(v) Py_IS_TYPE((v), &PyCursesWindow_Type) + #define import_curses() \ PyCurses_API = (void **)PyCapsule_Import(PyCurses_CAPSULE_NAME, 1); diff --git a/Include/pyerrors.h b/Include/pyerrors.h index 68d7985dac8876..5d0028c116e2d8 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -108,7 +108,6 @@ PyAPI_DATA(PyObject *) PyExc_NotImplementedError; PyAPI_DATA(PyObject *) PyExc_SyntaxError; PyAPI_DATA(PyObject *) PyExc_IndentationError; PyAPI_DATA(PyObject *) PyExc_TabError; -PyAPI_DATA(PyObject *) PyExc_IncompleteInputError; PyAPI_DATA(PyObject *) PyExc_ReferenceError; PyAPI_DATA(PyObject *) PyExc_SystemError; PyAPI_DATA(PyObject *) PyExc_SystemExit; diff --git a/Include/pyexpat.h b/Include/pyexpat.h index e0cff33b53227a..9824d099c3df7d 100644 --- a/Include/pyexpat.h +++ b/Include/pyexpat.h @@ -3,7 +3,7 @@ /* note: you must import expat.h before importing this module! */ -#define PyExpat_CAPI_MAGIC "pyexpat.expat_CAPI 1.2" +#define PyExpat_CAPI_MAGIC "pyexpat.expat_CAPI 1.1" #define PyExpat_CAPSULE_NAME "pyexpat.expat_CAPI" struct PyExpat_CAPI diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index c1e2bc5e323358..de1bcb1d2cb632 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -34,8 +34,12 @@ PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv); PyAPI_FUNC(int) Py_BytesMain(int argc, char **argv); /* In pathconfig.c */ +Py_DEPRECATED(3.11) PyAPI_FUNC(void) Py_SetProgramName(const wchar_t *); Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); + +Py_DEPRECATED(3.11) PyAPI_FUNC(void) Py_SetPythonHome(const wchar_t *); Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void); + Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void); Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetPrefix(void); Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void); diff --git a/Include/pymacro.h b/Include/pymacro.h index cd6fc4eba9c2ed..a82f347866e8d0 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -15,11 +15,11 @@ // MSVC makes static_assert a keyword in C11-17, contrary to the standards. // // In C++11 and C2x, static_assert is a keyword, redefining is undefined -// behaviour. So only define if building as C (if __STDC_VERSION__ is defined), -// not C++, and only for C11-17. +// behaviour. So only define if building as C, not C++ (if __cplusplus is +// not defined), and only for C11-17. #if !defined(static_assert) && (defined(__GNUC__) || defined(__clang__)) \ - && defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \ - && __STDC_VERSION__ <= 201710L + && !defined(__cplusplus) && defined(__STDC_VERSION__) \ + && __STDC_VERSION__ >= 201112L && __STDC_VERSION__ <= 201710L # define static_assert _Static_assert #endif @@ -46,24 +46,42 @@ /* Argument must be a char or an int in [-128, 127] or [0, 255]. */ #define Py_CHARMASK(c) ((unsigned char)((c) & 0xff)) -/* Assert a build-time dependency, as an expression. - - Your compile will fail if the condition isn't true, or can't be evaluated - by the compiler. This can be used in an expression: its value is 0. - - Example: - - #define foo_to_char(foo) \ - ((char *)(foo) \ - + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) - - Written by Rusty Russell, public domain, http://ccodearchive.net/ */ -#define Py_BUILD_ASSERT_EXPR(cond) \ +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \ + && !defined(__cplusplus) && !defined(_MSC_VER)) +# define Py_BUILD_ASSERT_EXPR(cond) \ + ((void)sizeof(struct { int dummy; _Static_assert(cond, #cond); }), \ + 0) +#else + /* Assert a build-time dependency, as an expression. + * + * Your compile will fail if the condition isn't true, or can't be evaluated + * by the compiler. This can be used in an expression: its value is 0. + * + * Example: + * + * #define foo_to_char(foo) \ + * ((char *)(foo) \ + * + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) + * + * Written by Rusty Russell, public domain, http://ccodearchive.net/ + */ +# define Py_BUILD_ASSERT_EXPR(cond) \ (sizeof(char [1 - 2*!(cond)]) - 1) +#endif -#define Py_BUILD_ASSERT(cond) do { \ - (void)Py_BUILD_ASSERT_EXPR(cond); \ - } while(0) +#if ((defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \ + || (defined(__cplusplus) && __cplusplus >= 201103L)) + // Use static_assert() on C11 and newer +# define Py_BUILD_ASSERT(cond) \ + do { \ + static_assert((cond), #cond); \ + } while (0) +#else +# define Py_BUILD_ASSERT(cond) \ + do { \ + (void)Py_BUILD_ASSERT_EXPR(cond); \ + } while(0) +#endif /* Get the number of elements in a visible array @@ -172,4 +190,13 @@ // "comparison of unsigned expression in '< 0' is always false". #define _Py_IS_TYPE_SIGNED(type) ((type)(-1) <= 0) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030E0000 // 3.14 +// Version helpers. These are primarily macros, but have exported equivalents. +PyAPI_FUNC(uint32_t) Py_PACK_FULL_VERSION(int x, int y, int z, int level, int serial); +PyAPI_FUNC(uint32_t) Py_PACK_VERSION(int x, int y); +#define Py_PACK_FULL_VERSION _Py_PACK_FULL_VERSION +#define Py_PACK_VERSION(X, Y) Py_PACK_FULL_VERSION(X, Y, 0, 0, 0) +#endif // Py_LIMITED_API < 3.14 + + #endif /* Py_PYMACRO_H */ diff --git a/Include/pymath.h b/Include/pymath.h index 4c1e3d9984894b..0ead1f95670fde 100644 --- a/Include/pymath.h +++ b/Include/pymath.h @@ -29,14 +29,17 @@ // Py_IS_NAN(X) // Return 1 if float or double arg is a NaN, else 0. +// Soft deprecated since Python 3.14, use isnan() instead. #define Py_IS_NAN(X) isnan(X) // Py_IS_INFINITY(X) // Return 1 if float or double arg is an infinity, else 0. +// Soft deprecated since Python 3.14, use isinf() instead. #define Py_IS_INFINITY(X) isinf(X) // Py_IS_FINITE(X) // Return 1 if float or double arg is neither infinite nor NAN, else 0. +// Soft deprecated since Python 3.14, use isfinite() instead. #define Py_IS_FINITE(X) isfinite(X) // Py_INFINITY: Value that evaluates to a positive double infinity. @@ -46,7 +49,7 @@ /* Py_HUGE_VAL should always be the same as Py_INFINITY. But historically * this was not reliable and Python did not require IEEE floats and C99 - * conformity. Prefer Py_INFINITY for new code. + * conformity. The macro was soft deprecated in Python 3.14, use Py_INFINITY instead. */ #ifndef Py_HUGE_VAL # define Py_HUGE_VAL HUGE_VAL diff --git a/Include/pyport.h b/Include/pyport.h index 9d7ef0061806ad..aabd094df54a74 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -9,6 +9,24 @@ #endif +// Preprocessor check for a builtin preprocessor function. Always return 0 +// if __has_builtin() macro is not defined. +// +// __has_builtin() is available on clang and GCC 10. +#ifdef __has_builtin +# define _Py__has_builtin(x) __has_builtin(x) +#else +# define _Py__has_builtin(x) 0 +#endif + +// Preprocessor check for a compiler __attribute__. Always return 0 +// if __has_attribute() macro is not defined. +#ifdef __has_attribute +# define _Py__has_attribute(x) __has_attribute(x) +#else +# define _Py__has_attribute(x) 0 +#endif + // Macro to use C++ static_cast<> in the Python C API. #ifdef __cplusplus # define _Py_STATIC_CAST(type, expr) static_cast(expr) @@ -180,6 +198,7 @@ typedef Py_ssize_t Py_ssize_clean_t; # define Py_LOCAL_INLINE(type) static inline type #endif +// Soft deprecated since Python 3.14, use memcpy() instead. #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_MEMCPY memcpy #endif @@ -531,16 +550,6 @@ extern "C" { #endif -// Preprocessor check for a builtin preprocessor function. Always return 0 -// if __has_builtin() macro is not defined. -// -// __has_builtin() is available on clang and GCC 10. -#ifdef __has_builtin -# define _Py__has_builtin(x) __has_builtin(x) -#else -# define _Py__has_builtin(x) 0 -#endif - // _Py_TYPEOF(expr) gets the type of an expression. // // Example: _Py_TYPEOF(x) x_copy = (x); @@ -572,6 +581,9 @@ extern "C" { # if defined(__SANITIZE_ADDRESS__) # define _Py_ADDRESS_SANITIZER # endif +# if defined(__SANITIZE_THREAD__) +# define _Py_THREAD_SANITIZER +# endif #endif @@ -603,4 +615,26 @@ extern "C" { # define _SGI_MP_SOURCE #endif +// Explicit fallthrough in switch case to avoid warnings +// with compiler flag -Wimplicit-fallthrough. +// +// Usage example: +// +// switch (value) { +// case 1: _Py_FALLTHROUGH; +// case 2: code; break; +// } +// +// __attribute__((fallthrough)) was introduced in GCC 7 and Clang 10 / +// Apple Clang 12.0. Earlier Clang versions support only the C++11 +// style fallthrough attribute, not the GCC extension syntax used here, +// and __has_attribute(fallthrough) evaluates to 1. +#if _Py__has_attribute(fallthrough) && (!defined(__clang__) || \ + (!defined(__apple_build_version__) && __clang_major__ >= 10) || \ + (defined(__apple_build_version__) && __clang_major__ >= 12)) +# define _Py_FALLTHROUGH __attribute__((fallthrough)) +#else +# define _Py_FALLTHROUGH do { } while (0) +#endif + #endif /* Py_PYPORT_H */ diff --git a/Include/pystats.h b/Include/pystats.h index acfa32201711e0..a515570d1bb3bc 100644 --- a/Include/pystats.h +++ b/Include/pystats.h @@ -18,6 +18,8 @@ extern "C" { #else # define _Py_INCREF_STAT_INC() ((void)0) # define _Py_DECREF_STAT_INC() ((void)0) +# define _Py_INCREF_IMMORTAL_STAT_INC() ((void)0) +# define _Py_DECREF_IMMORTAL_STAT_INC() ((void)0) #endif // !Py_STATS #ifdef __cplusplus diff --git a/Include/pythread.h b/Include/pythread.h index a3216c51d66165..82247daf8e0aa0 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -17,7 +17,26 @@ typedef enum PyLockStatus { PyAPI_FUNC(void) PyThread_init_thread(void); PyAPI_FUNC(unsigned long) PyThread_start_new_thread(void (*)(void *), void *); -PyAPI_FUNC(void) _Py_NO_RETURN PyThread_exit_thread(void); +/* Terminates the current thread. Considered unsafe. + * + * WARNING: This function is only safe to call if all functions in the full call + * stack are written to safely allow it. Additionally, the behavior is + * platform-dependent. This function should be avoided, and is no longer called + * by Python itself. It is retained only for compatibility with existing C + * extension code. + * + * With pthreads, calls `pthread_exit` causes some libcs (glibc?) to attempt to + * unwind the stack and call C++ destructors; if a `noexcept` function is + * reached, they may terminate the process. Others (macOS) do unwinding. + * + * On Windows, calls `_endthreadex` which kills the thread without calling C++ + * destructors. + * + * In either case there is a risk of invalid references remaining to data on the + * thread stack. + */ +Py_DEPRECATED(3.14) PyAPI_FUNC(void) _Py_NO_RETURN PyThread_exit_thread(void); + PyAPI_FUNC(unsigned long) PyThread_get_thread_ident(void); #if (defined(__APPLE__) || defined(__linux__) || defined(_WIN32) \ diff --git a/Include/refcount.h b/Include/refcount.h new file mode 100644 index 00000000000000..d98b2dfcf37202 --- /dev/null +++ b/Include/refcount.h @@ -0,0 +1,543 @@ +#ifndef Py_REFCOUNT_H +#define Py_REFCOUNT_H +#ifdef __cplusplus +extern "C" { +#endif + + +/* +Immortalization: + +The following indicates the immortalization strategy depending on the amount +of available bits in the reference count field. All strategies are backwards +compatible but the specific reference count value or immortalization check +might change depending on the specializations for the underlying system. + +Proper deallocation of immortal instances requires distinguishing between +statically allocated immortal instances vs those promoted by the runtime to be +immortal. The latter should be the only instances that require +cleanup during runtime finalization. +*/ + +/* Leave the low bits for refcount overflow for old stable ABI code */ +#define _Py_STATICALLY_ALLOCATED_FLAG (1 << 7) + +#if SIZEOF_VOID_P > 4 +/* +In 64+ bit systems, any object whose 32 bit reference count is >= 2**31 +will be treated as immortal. + +Using the lower 32 bits makes the value backwards compatible by allowing +C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely +increase and decrease the objects reference count. + +In order to offer sufficient resilience to C extensions using the stable ABI +compiled against 3.11 or earlier, we set the initial value near the +middle of the range (2**31, 2**32). That way the the refcount can be +off by ~1 billion without affecting immortality. + +Reference count increases will use saturated arithmetic, taking advantage of +having all the lower 32 bits set, which will avoid the reference count to go +beyond the refcount limit. Immortality checks for reference count decreases will +be done by checking the bit sign flag in the lower 32 bits. + +*/ +#define _Py_IMMORTAL_INITIAL_REFCNT (3UL << 30) +#define _Py_STATIC_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(_Py_IMMORTAL_INITIAL_REFCNT | (((Py_ssize_t)_Py_STATICALLY_ALLOCATED_FLAG) << 32))) + +#else +/* +In 32 bit systems, an object will be treated as immortal if its reference +count equals or exceeds _Py_IMMORTAL_MINIMUM_REFCNT (2**30). + +Using the lower 30 bits makes the value backwards compatible by allowing +C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely +increase and decrease the objects reference count. The object would lose its +immortality, but the execution would still be correct. + +Reference count increases and decreases will first go through an immortality +check by comparing the reference count field to the minimum immortality refcount. +*/ +#define _Py_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(5L << 28)) +#define _Py_IMMORTAL_MINIMUM_REFCNT ((Py_ssize_t)(1L << 30)) +#define _Py_STATIC_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(7L << 28)) +#define _Py_STATIC_IMMORTAL_MINIMUM_REFCNT ((Py_ssize_t)(6L << 28)) +#endif + +// Py_GIL_DISABLED builds indicate immortal objects using `ob_ref_local`, which is +// always 32-bits. +#ifdef Py_GIL_DISABLED +#define _Py_IMMORTAL_REFCNT_LOCAL UINT32_MAX +#endif + + +#ifdef Py_GIL_DISABLED + // The shared reference count uses the two least-significant bits to store + // flags. The remaining bits are used to store the reference count. +# define _Py_REF_SHARED_SHIFT 2 +# define _Py_REF_SHARED_FLAG_MASK 0x3 + + // The shared flags are initialized to zero. +# define _Py_REF_SHARED_INIT 0x0 +# define _Py_REF_MAYBE_WEAKREF 0x1 +# define _Py_REF_QUEUED 0x2 +# define _Py_REF_MERGED 0x3 + + // Create a shared field from a refcnt and desired flags +# define _Py_REF_SHARED(refcnt, flags) \ + (((refcnt) << _Py_REF_SHARED_SHIFT) + (flags)) +#endif // Py_GIL_DISABLED + + +// Py_REFCNT() implementation for the stable ABI +PyAPI_FUNC(Py_ssize_t) Py_REFCNT(PyObject *ob); + +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030e0000 + // Stable ABI implements Py_REFCNT() as a function call + // on limited C API version 3.14 and newer. +#else + static inline Py_ssize_t _Py_REFCNT(PyObject *ob) { + #if !defined(Py_GIL_DISABLED) + return ob->ob_refcnt; + #else + uint32_t local = _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local); + if (local == _Py_IMMORTAL_REFCNT_LOCAL) { + return _Py_IMMORTAL_INITIAL_REFCNT; + } + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared); + return _Py_STATIC_CAST(Py_ssize_t, local) + + Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, shared, _Py_REF_SHARED_SHIFT); + #endif + } + #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 + # define Py_REFCNT(ob) _Py_REFCNT(_PyObject_CAST(ob)) + #endif +#endif + + +static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) +{ +#if defined(Py_GIL_DISABLED) + return (_Py_atomic_load_uint32_relaxed(&op->ob_ref_local) == + _Py_IMMORTAL_REFCNT_LOCAL); +#elif SIZEOF_VOID_P > 4 + return _Py_CAST(PY_INT32_T, op->ob_refcnt) < 0; +#else + return op->ob_refcnt >= _Py_IMMORTAL_MINIMUM_REFCNT; +#endif +} +#define _Py_IsImmortal(op) _Py_IsImmortal(_PyObject_CAST(op)) + + +static inline Py_ALWAYS_INLINE int _Py_IsStaticImmortal(PyObject *op) +{ +#if defined(Py_GIL_DISABLED) || SIZEOF_VOID_P > 4 + return (op->ob_flags & _Py_STATICALLY_ALLOCATED_FLAG) != 0; +#else + return op->ob_refcnt >= _Py_STATIC_IMMORTAL_MINIMUM_REFCNT; +#endif +} +#define _Py_IsStaticImmortal(op) _Py_IsStaticImmortal(_PyObject_CAST(op)) + +// Py_SET_REFCNT() implementation for stable ABI +PyAPI_FUNC(void) _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt); + +static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { + assert(refcnt >= 0); +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030d0000 + // Stable ABI implements Py_SET_REFCNT() as a function call + // on limited C API version 3.13 and newer. + _Py_SetRefcnt(ob, refcnt); +#else + // This immortal check is for code that is unaware of immortal objects. + // The runtime tracks these objects and we should avoid as much + // as possible having extensions inadvertently change the refcnt + // of an immortalized object. + if (_Py_IsImmortal(ob)) { + return; + } +#ifndef Py_GIL_DISABLED +#if SIZEOF_VOID_P > 4 + ob->ob_refcnt = (PY_UINT32_T)refcnt; +#else + ob->ob_refcnt = refcnt; +#endif +#else + if (_Py_IsOwnedByCurrentThread(ob)) { + if ((size_t)refcnt > (size_t)UINT32_MAX) { + // On overflow, make the object immortal + ob->ob_tid = _Py_UNOWNED_TID; + ob->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; + ob->ob_ref_shared = 0; + } + else { + // Set local refcount to desired refcount and shared refcount + // to zero, but preserve the shared refcount flags. + ob->ob_ref_local = _Py_STATIC_CAST(uint32_t, refcnt); + ob->ob_ref_shared &= _Py_REF_SHARED_FLAG_MASK; + } + } + else { + // Set local refcount to zero and shared refcount to desired refcount. + // Mark the object as merged. + ob->ob_tid = _Py_UNOWNED_TID; + ob->ob_ref_local = 0; + ob->ob_ref_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED); + } +#endif // Py_GIL_DISABLED +#endif // Py_LIMITED_API+0 < 0x030d0000 +} +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_SET_REFCNT(ob, refcnt) Py_SET_REFCNT(_PyObject_CAST(ob), (refcnt)) +#endif + + +/* +The macros Py_INCREF(op) and Py_DECREF(op) are used to increment or decrement +reference counts. Py_DECREF calls the object's deallocator function when +the refcount falls to 0; for +objects that don't contain references to other objects or heap memory +this can be the standard function free(). Both macros can be used +wherever a void expression is allowed. The argument must not be a +NULL pointer. If it may be NULL, use Py_XINCREF/Py_XDECREF instead. +The macro _Py_NewReference(op) initialize reference counts to 1, and +in special builds (Py_REF_DEBUG, Py_TRACE_REFS) performs additional +bookkeeping appropriate to the special build. + +We assume that the reference count field can never overflow; this can +be proven when the size of the field is the same as the pointer size, so +we ignore the possibility. Provided a C int is at least 32 bits (which +is implicitly assumed in many parts of this code), that's enough for +about 2**31 references to an object. + +XXX The following became out of date in Python 2.2, but I'm not sure +XXX what the full truth is now. Certainly, heap-allocated type objects +XXX can and should be deallocated. +Type objects should never be deallocated; the type pointer in an object +is not considered to be a reference to the type object, to save +complications in the deallocation function. (This is actually a +decision that's up to the implementer of each new type so if you want, +you can count such references to the type object.) +*/ + +#if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API) +PyAPI_FUNC(void) _Py_NegativeRefcount(const char *filename, int lineno, + PyObject *op); +PyAPI_FUNC(void) _Py_INCREF_IncRefTotal(void); +PyAPI_FUNC(void) _Py_DECREF_DecRefTotal(void); +#endif // Py_REF_DEBUG && !Py_LIMITED_API + +PyAPI_FUNC(void) _Py_Dealloc(PyObject *); + + +/* +These are provided as conveniences to Python runtime embedders, so that +they can have object code that is not dependent on Python compilation flags. +*/ +PyAPI_FUNC(void) Py_IncRef(PyObject *); +PyAPI_FUNC(void) Py_DecRef(PyObject *); + +// Similar to Py_IncRef() and Py_DecRef() but the argument must be non-NULL. +// Private functions used by Py_INCREF() and Py_DECREF(). +PyAPI_FUNC(void) _Py_IncRef(PyObject *); +PyAPI_FUNC(void) _Py_DecRef(PyObject *); + +static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) +{ +#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) + // Stable ABI implements Py_INCREF() as a function call on limited C API + // version 3.12 and newer, and on Python built in debug mode. _Py_IncRef() + // was added to Python 3.10.0a7, use Py_IncRef() on older Python versions. + // Py_IncRef() accepts NULL whereas _Py_IncRef() doesn't. +# if Py_LIMITED_API+0 >= 0x030a00A7 + _Py_IncRef(op); +# else + Py_IncRef(op); +# endif +#else + // Non-limited C API and limited C API for Python 3.9 and older access + // directly PyObject.ob_refcnt. +#if defined(Py_GIL_DISABLED) + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + uint32_t new_local = local + 1; + if (new_local == 0) { + _Py_INCREF_IMMORTAL_STAT_INC(); + // local is equal to _Py_IMMORTAL_REFCNT_LOCAL: do nothing + return; + } + if (_Py_IsOwnedByCurrentThread(op)) { + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, new_local); + } + else { + _Py_atomic_add_ssize(&op->ob_ref_shared, (1 << _Py_REF_SHARED_SHIFT)); + } +#elif SIZEOF_VOID_P > 4 + PY_UINT32_T cur_refcnt = op->ob_refcnt; + if (((int32_t)cur_refcnt) < 0) { + // the object is immortal + _Py_INCREF_IMMORTAL_STAT_INC(); + return; + } + op->ob_refcnt = cur_refcnt + 1; +#else + if (_Py_IsImmortal(op)) { + _Py_INCREF_IMMORTAL_STAT_INC(); + return; + } + op->ob_refcnt++; +#endif + _Py_INCREF_STAT_INC(); +#ifdef Py_REF_DEBUG + _Py_INCREF_IncRefTotal(); +#endif +#endif +} +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op)) +#endif + + +#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) +// Implements Py_DECREF on objects not owned by the current thread. +PyAPI_FUNC(void) _Py_DecRefShared(PyObject *); +PyAPI_FUNC(void) _Py_DecRefSharedDebug(PyObject *, const char *, int); + +// Called from Py_DECREF by the owning thread when the local refcount reaches +// zero. The call will deallocate the object if the shared refcount is also +// zero. Otherwise, the thread gives up ownership and merges the reference +// count fields. +PyAPI_FUNC(void) _Py_MergeZeroLocalRefcount(PyObject *); +#endif + +#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) +// Stable ABI implements Py_DECREF() as a function call on limited C API +// version 3.12 and newer, and on Python built in debug mode. _Py_DecRef() was +// added to Python 3.10.0a7, use Py_DecRef() on older Python versions. +// Py_DecRef() accepts NULL whereas _Py_IncRef() doesn't. +static inline void Py_DECREF(PyObject *op) { +# if Py_LIMITED_API+0 >= 0x030a00A7 + _Py_DecRef(op); +# else + Py_DecRef(op); +# endif +} +#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) + +#elif defined(Py_GIL_DISABLED) && defined(Py_REF_DEBUG) +static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) +{ + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + if (local == _Py_IMMORTAL_REFCNT_LOCAL) { + _Py_DECREF_IMMORTAL_STAT_INC(); + return; + } + _Py_DECREF_STAT_INC(); + _Py_DECREF_DecRefTotal(); + if (_Py_IsOwnedByCurrentThread(op)) { + if (local == 0) { + _Py_NegativeRefcount(filename, lineno, op); + } + local--; + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); + if (local == 0) { + _Py_MergeZeroLocalRefcount(op); + } + } + else { + _Py_DecRefSharedDebug(op, filename, lineno); + } +} +#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) + +#elif defined(Py_GIL_DISABLED) +static inline void Py_DECREF(PyObject *op) +{ + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + if (local == _Py_IMMORTAL_REFCNT_LOCAL) { + _Py_DECREF_IMMORTAL_STAT_INC(); + return; + } + _Py_DECREF_STAT_INC(); + if (_Py_IsOwnedByCurrentThread(op)) { + local--; + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); + if (local == 0) { + _Py_MergeZeroLocalRefcount(op); + } + } + else { + _Py_DecRefShared(op); + } +} +#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) + +#elif defined(Py_REF_DEBUG) +static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) +{ +#if SIZEOF_VOID_P > 4 + /* If an object has been freed, it will have a negative full refcnt + * If it has not it been freed, will have a very large refcnt */ + if (op->ob_refcnt_full <= 0 || op->ob_refcnt > (((PY_UINT32_T)-1) - (1<<20))) { +#else + if (op->ob_refcnt <= 0) { +#endif + _Py_NegativeRefcount(filename, lineno, op); + } + if (_Py_IsImmortal(op)) { + _Py_DECREF_IMMORTAL_STAT_INC(); + return; + } + _Py_DECREF_STAT_INC(); + _Py_DECREF_DecRefTotal(); + if (--op->ob_refcnt == 0) { + _Py_Dealloc(op); + } +} +#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) + +#else +static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op) +{ + // Non-limited C API and limited C API for Python 3.9 and older access + // directly PyObject.ob_refcnt. + if (_Py_IsImmortal(op)) { + _Py_DECREF_IMMORTAL_STAT_INC(); + return; + } + _Py_DECREF_STAT_INC(); + if (--op->ob_refcnt == 0) { + _Py_Dealloc(op); + } +} +#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) +#endif + + +/* Safely decref `op` and set `op` to NULL, especially useful in tp_clear + * and tp_dealloc implementations. + * + * Note that "the obvious" code can be deadly: + * + * Py_XDECREF(op); + * op = NULL; + * + * Typically, `op` is something like self->containee, and `self` is done + * using its `containee` member. In the code sequence above, suppose + * `containee` is non-NULL with a refcount of 1. Its refcount falls to + * 0 on the first line, which can trigger an arbitrary amount of code, + * possibly including finalizers (like __del__ methods or weakref callbacks) + * coded in Python, which in turn can release the GIL and allow other threads + * to run, etc. Such code may even invoke methods of `self` again, or cause + * cyclic gc to trigger, but-- oops! --self->containee still points to the + * object being torn down, and it may be in an insane state while being torn + * down. This has in fact been a rich historic source of miserable (rare & + * hard-to-diagnose) segfaulting (and other) bugs. + * + * The safe way is: + * + * Py_CLEAR(op); + * + * That arranges to set `op` to NULL _before_ decref'ing, so that any code + * triggered as a side-effect of `op` getting torn down no longer believes + * `op` points to a valid object. + * + * There are cases where it's safe to use the naive code, but they're brittle. + * For example, if `op` points to a Python integer, you know that destroying + * one of those can't cause problems -- but in part that relies on that + * Python integers aren't currently weakly referencable. Best practice is + * to use Py_CLEAR() even if you can't think of a reason for why you need to. + * + * gh-98724: Use a temporary variable to only evaluate the macro argument once, + * to avoid the duplication of side effects if the argument has side effects. + * + * gh-99701: If the PyObject* type is used with casting arguments to PyObject*, + * the code can be miscompiled with strict aliasing because of type punning. + * With strict aliasing, a compiler considers that two pointers of different + * types cannot read or write the same memory which enables optimization + * opportunities. + * + * If available, use _Py_TYPEOF() to use the 'op' type for temporary variables, + * and so avoid type punning. Otherwise, use memcpy() which causes type erasure + * and so prevents the compiler to reuse an old cached 'op' value after + * Py_CLEAR(). + */ +#ifdef _Py_TYPEOF +#define Py_CLEAR(op) \ + do { \ + _Py_TYPEOF(op)* _tmp_op_ptr = &(op); \ + _Py_TYPEOF(op) _tmp_old_op = (*_tmp_op_ptr); \ + if (_tmp_old_op != NULL) { \ + *_tmp_op_ptr = _Py_NULL; \ + Py_DECREF(_tmp_old_op); \ + } \ + } while (0) +#else +#define Py_CLEAR(op) \ + do { \ + PyObject **_tmp_op_ptr = _Py_CAST(PyObject**, &(op)); \ + PyObject *_tmp_old_op = (*_tmp_op_ptr); \ + if (_tmp_old_op != NULL) { \ + PyObject *_null_ptr = _Py_NULL; \ + memcpy(_tmp_op_ptr, &_null_ptr, sizeof(PyObject*)); \ + Py_DECREF(_tmp_old_op); \ + } \ + } while (0) +#endif + + +/* Function to use in case the object pointer can be NULL: */ +static inline void Py_XINCREF(PyObject *op) +{ + if (op != _Py_NULL) { + Py_INCREF(op); + } +} +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_XINCREF(op) Py_XINCREF(_PyObject_CAST(op)) +#endif + +static inline void Py_XDECREF(PyObject *op) +{ + if (op != _Py_NULL) { + Py_DECREF(op); + } +} +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_XDECREF(op) Py_XDECREF(_PyObject_CAST(op)) +#endif + +// Create a new strong reference to an object: +// increment the reference count of the object and return the object. +PyAPI_FUNC(PyObject*) Py_NewRef(PyObject *obj); + +// Similar to Py_NewRef(), but the object can be NULL. +PyAPI_FUNC(PyObject*) Py_XNewRef(PyObject *obj); + +static inline PyObject* _Py_NewRef(PyObject *obj) +{ + Py_INCREF(obj); + return obj; +} + +static inline PyObject* _Py_XNewRef(PyObject *obj) +{ + Py_XINCREF(obj); + return obj; +} + +// Py_NewRef() and Py_XNewRef() are exported as functions for the stable ABI. +// Names overridden with macros by static inline functions for best +// performances. +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj)) +# define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj)) +#else +# define Py_NewRef(obj) _Py_NewRef(obj) +# define Py_XNewRef(obj) _Py_XNewRef(obj) +#endif + + +#ifdef __cplusplus +} +#endif +#endif // !Py_REFCOUNT_H diff --git a/Include/sliceobject.h b/Include/sliceobject.h index c13863f27c2e63..35e2ea254ca80a 100644 --- a/Include/sliceobject.h +++ b/Include/sliceobject.h @@ -8,7 +8,11 @@ extern "C" { PyAPI_DATA(PyObject) _Py_EllipsisObject; /* Don't use this directly */ -#define Py_Ellipsis (&_Py_EllipsisObject) +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030D0000 +# define Py_Ellipsis Py_GetConstantBorrowed(Py_CONSTANT_ELLIPSIS) +#else +# define Py_Ellipsis (&_Py_EllipsisObject) +#endif /* Slice object interface */ diff --git a/Include/structmember.h b/Include/structmember.h index f6e8fd829892f4..5f29fbcfed99e3 100644 --- a/Include/structmember.h +++ b/Include/structmember.h @@ -11,7 +11,7 @@ extern "C" { * New definitions are in descrobject.h. * * However, there's nothing wrong with old code continuing to use it, - * and there's not much mainenance overhead in maintaining a few aliases. + * and there's not much maintenance overhead in maintaining a few aliases. * So, don't be too eager to convert old code. * * It uses names not prefixed with Py_. diff --git a/Include/sysmodule.h b/Include/sysmodule.h index 7b14f72ee2e494..c1d5f610fe08a5 100644 --- a/Include/sysmodule.h +++ b/Include/sysmodule.h @@ -7,6 +7,9 @@ extern "C" { PyAPI_FUNC(PyObject *) PySys_GetObject(const char *); PyAPI_FUNC(int) PySys_SetObject(const char *, PyObject *); +Py_DEPRECATED(3.11) PyAPI_FUNC(void) PySys_SetArgv(int, wchar_t **); +Py_DEPRECATED(3.11) PyAPI_FUNC(void) PySys_SetArgvEx(int, wchar_t **, int); + PyAPI_FUNC(void) PySys_WriteStdout(const char *format, ...) Py_GCC_ATTRIBUTE((format(printf, 1, 2))); PyAPI_FUNC(void) PySys_WriteStderr(const char *format, ...) @@ -18,23 +21,6 @@ Py_DEPRECATED(3.13) PyAPI_FUNC(void) PySys_ResetWarnOptions(void); PyAPI_FUNC(PyObject *) PySys_GetXOptions(void); -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 -PyAPI_FUNC(int) PySys_Audit( - const char *event, - const char *argFormat, - ...); - -PyAPI_FUNC(int) PySys_AuditTuple( - const char *event, - PyObject *args); -#endif - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_SYSMODULE_H -# include "cpython/sysmodule.h" -# undef Py_CPYTHON_SYSMODULE_H -#endif - #ifdef __cplusplus } #endif diff --git a/Include/typeslots.h b/Include/typeslots.h index 506b05580de146..a7f3017ec02e92 100644 --- a/Include/typeslots.h +++ b/Include/typeslots.h @@ -86,3 +86,11 @@ /* New in 3.10 */ #define Py_am_send 81 #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030E0000 +/* New in 3.14 */ +#define Py_tp_vectorcall 82 +#endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030E0000 +/* New in 3.14 */ +#define Py_tp_token 83 +#endif diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index dee00715b3c51d..2ce3a008b7129e 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -966,6 +966,10 @@ PyAPI_FUNC(int) PyUnicode_EqualToUTF8(PyObject *, const char *); PyAPI_FUNC(int) PyUnicode_EqualToUTF8AndSize(PyObject *, const char *, Py_ssize_t); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030e0000 +PyAPI_FUNC(int) PyUnicode_Equal(PyObject *str1, PyObject *str2); +#endif + /* Rich compare two strings and return one of the following: - NULL in case an exception was raised diff --git a/Include/weakrefobject.h b/Include/weakrefobject.h index 727ba6934bbacb..a6e71eb178b124 100644 --- a/Include/weakrefobject.h +++ b/Include/weakrefobject.h @@ -28,7 +28,10 @@ PyAPI_FUNC(PyObject *) PyWeakref_NewRef(PyObject *ob, PyAPI_FUNC(PyObject *) PyWeakref_NewProxy(PyObject *ob, PyObject *callback); Py_DEPRECATED(3.13) PyAPI_FUNC(PyObject *) PyWeakref_GetObject(PyObject *ref); + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030D0000 PyAPI_FUNC(int) PyWeakref_GetRef(PyObject *ref, PyObject **pobj); +#endif #ifndef Py_LIMITED_API diff --git a/InternalDocs/README.md b/InternalDocs/README.md new file mode 100644 index 00000000000000..4502902307cd5c --- /dev/null +++ b/InternalDocs/README.md @@ -0,0 +1,43 @@ +# CPython Internals Documentation + +The documentation in this folder is intended for CPython maintainers. +It describes implementation details of CPython, which should not be +assumed to be part of the Python language specification. These details +can change between any two CPython versions and should not be assumed +to hold for other implementations of the Python language. + +The core dev team attempts to keep this documentation up to date. If +it is not, please report that through the +[issue tracker](https://github.com/python/cpython/issues). + + +Compiling Python Source Code +--- + +- [Guide to the parser](parser.md) + +- [Compiler Design](compiler.md) + +- [Changing Python's Grammar](changing_grammar.md) + +Runtime Objects +--- + +- [Code Objects](code_objects.md) + +- [Generators](generators.md) + +- [Frames](frames.md) + +- [String Interning](string_interning.md) + +Program Execution +--- + +- [The Bytecode Interpreter](interpreter.md) + +- [The JIT](jit.md) + +- [Garbage Collector Design](garbage_collector.md) + +- [Exception Handling](exception_handling.md) diff --git a/InternalDocs/changing_grammar.md b/InternalDocs/changing_grammar.md new file mode 100644 index 00000000000000..c6b895135a360d --- /dev/null +++ b/InternalDocs/changing_grammar.md @@ -0,0 +1,63 @@ +# Changing CPython's grammar + +There's more to changing Python's grammar than editing +[`Grammar/python.gram`](../Grammar/python.gram). +Below is a checklist of things that may need to change. + +> [!NOTE] +> +> Many of these changes require re-generating some of the derived +> files. If things mysteriously don't work, it may help to run +> ``make clean``. + +## Checklist + +* [`Grammar/python.gram`](../Grammar/python.gram): The grammar definition, + with actions that build AST nodes. + After changing it, run ``make regen-pegen`` (or ``build.bat --regen`` on Windows), + to regenerate [`Parser/parser.c`](../Parser/parser.c). + (This runs Python's parser generator, [`Tools/peg_generator`](../Tools/peg_generator)). + +* [`Grammar/Tokens`](../Grammar/Tokens) is a place for adding new token types. After + changing it, run ``make regen-token`` to regenerate + [`Include/internal/pycore_token.h`](../Include/internal/pycore_token.h), + [`Parser/token.c`](../Parser/token.c), [`Lib/token.py`](../Lib/token.py) + and [`Doc/library/token-list.inc`](../Doc/library/token-list.inc). + If you change both ``python.gram`` and ``Tokens``, run ``make regen-token`` + before ``make regen-pegen``. + On Windows, ``build.bat --regen`` will regenerate both at the same time. + +* [`Parser/Python.asdl`](../Parser/Python.asdl) may need changes to match the grammar. + Then run ``make regen-ast`` to regenerate + [`Include/internal/pycore_ast.h`](../Include/internal/pycore_ast.h) and + [`Python/Python-ast.c`](../Python/Python-ast.c). + +* [`Parser/lexer/`](../Parser/lexer) contains the tokenization code. + This is where you would add a new type of comment or string literal, for example. + +* [`Python/ast.c`](../Python/ast.c) will need changes to validate AST objects + involved with the grammar change. + +* [`Python/ast_unparse.c`](../Python/ast_unparse.c) will need changes to unparse + AST involved with the grammar change ("unparsing" is used to turn annotations + into strings per [PEP 563](https://peps.python.org/pep-0563/). + +* The [`compiler`](compiler.md) may need to change when there are changes + to the `AST`. + +* ``_Unparser`` in the [`Lib/ast.py`](../Lib/ast.py) file may need changes + to accommodate any modifications in the AST nodes. + +* [`Doc/library/ast.rst`](../Doc/library/ast.rst) may need to be updated + to reflect changes to AST nodes. + +* Add some usage of your new syntax to ``test_grammar.py``. + +* Certain changes may require tweaks to the library module + [`pyclbr`](https://docs.python.org/3/library/pyclbr.html#module-pyclbr). + +* [`Lib/tokenize.py`](../Lib/tokenize.py) needs changes to match changes + to the tokenizer. + +* Documentation must be written! Specifically, one or more of the pages in + [`Doc/reference/`](../Doc/reference) will need to be updated. diff --git a/InternalDocs/code_objects.md b/InternalDocs/code_objects.md new file mode 100644 index 00000000000000..a91a7043c1b8d4 --- /dev/null +++ b/InternalDocs/code_objects.md @@ -0,0 +1,179 @@ +# Code objects + +A `CodeObject` is a builtin Python type that represents a compiled executable, +such as a compiled function or class. +It contains a sequence of bytecode instructions along with its associated +metadata: data which is necessary to execute the bytecode instructions (such +as the values of the constants they access) or context information such as +the source code location, which is useful for debuggers and other tools. + +Since 3.11, the final field of the `PyCodeObject` C struct is an array +of indeterminate length containing the bytecode, `code->co_code_adaptive`. +(In older versions the code object was a +[`bytes`](https://docs.python.org/dev/library/stdtypes.html#bytes) +object, `code->co_code`; this was changed to save an allocation and to +allow it to be mutated.) + +Code objects are typically produced by the bytecode [compiler](compiler.md), +although they are often written to disk by one process and read back in by another. +The disk version of a code object is serialized using the +[marshal](https://docs.python.org/dev/library/marshal.html) protocol. +When a `CodeObject` is created, the function `_PyCode_Quicken()` from +[`Python/specialize.c`](../Python/specialize.c) is called to initialize +the caches of all adaptive instructions. This is required because the +on-disk format is a sequence of bytes, and some of the caches need to be +initialized with 16-bit values. + +Code objects are nominally immutable. +Some fields (including `co_code_adaptive` and fields for runtime +information such as `_co_monitoring`) are mutable, but mutable fields are +not included when code objects are hashed or compared. + +## Source code locations + +Whenever an exception occurs, the interpreter adds a traceback entry to +the exception for the current frame, as well as each frame on the stack that +it unwinds. +The `tb_lineno` field of a traceback entry is (lazily) set to the line +number of the instruction that was executing in the frame at the time of +the exception. +This field is computed from the locations table, `co_linetable`, by the function +[`PyCode_Addr2Line`](https://docs.python.org/dev/c-api/code.html#c.PyCode_Addr2Line). +Despite its name, `co_linetable` includes more than line numbers; it represents +a 4-number source location for every instruction, indicating the precise line +and column at which it begins and ends. This is a significant amount of data, +so a compact format is very important. + +Note that traceback objects don't store all this information -- they store the start line +number, for backward compatibility, and the "last instruction" value. +The rest can be computed from the last instruction (`tb_lasti`) with the help of the +locations table. For Python code, there is a convenience method +(`codeobject.co_positions`)[https://docs.python.org/dev/reference/datamodel.html#codeobject.co_positions] +which returns an iterator of `({line}, {endline}, {column}, {endcolumn})` tuples, +one per instruction. +There is also `co_lines()` which returns an iterator of `({start}, {end}, {line})` tuples, +where `{start}` and `{end}` are bytecode offsets. +The latter is described by [`PEP 626`](https://peps.python.org/pep-0626/); it is more +compact, but doesn't return end line numbers or column offsets. +From C code, you need to call +[`PyCode_Addr2Location`](https://docs.python.org/dev/c-api/code.html#c.PyCode_Addr2Location). + +As the locations table is only consulted when displaying a traceback and when +tracing (to pass the line number to the tracing function), lookup is not +performance critical. +In order to reduce the overhead during tracing, the mapping from instruction offset to +line number is cached in the ``_co_linearray`` field. + +### Format of the locations table + +The `co_linetable` bytes object of code objects contains a compact +representation of the source code positions of instructions, which are +returned by the `co_positions()` iterator. + +> [!NOTE] +> `co_linetable` is not to be confused with `co_lnotab`. +> For backwards compatibility, `co_lnotab` exposes the format +> as it existed in Python 3.10 and lower: this older format +> stores only the start line for each instruction. +> It is lazily created from `co_linetable` when accessed. +> See [`Objects/lnotab_notes.txt`](../Objects/lnotab_notes.txt) for more details. + +`co_linetable` consists of a sequence of location entries. +Each entry starts with a byte with the most significant bit set, followed by +zero or more bytes with the most significant bit unset. + +Each entry contains the following information: + +* The number of code units covered by this entry (length) +* The start line +* The end line +* The start column +* The end column + +The first byte has the following format: + +| Bit 7 | Bits 3-6 | Bits 0-2 | +|-------|----------|----------------------------| +| 1 | Code | Length (in code units) - 1 | + +The codes are enumerated in the `_PyCodeLocationInfoKind` enum. + +### Variable-length integer encodings + +Integers are often encoded using a variable length integer encoding + +#### Unsigned integers (`varint`) + +Unsigned integers are encoded in 6-bit chunks, least significant first. +Each chunk but the last has bit 6 set. +For example: + +* 63 is encoded as `0x3f` +* 200 is encoded as `0x48`, `0x03` since ``200 = (0x03 << 6) | 0x48``. + +The following helper can be used to convert an integer into a `varint`: + +```py +def encode_varint(s): + ret = [] + while s >= 64: + ret.append(((s & 0x3F) | 0x40) & 0x3F) + s >>= 6 + ret.append(s & 0x3F) + return bytes(ret) +``` + +To convert a `varint` into an unsigned integer: + +```py +def decode_varint(chunks): + ret = 0 + for chunk in reversed(chunks): + ret = (ret << 6) | chunk + return ret +``` + +#### Signed integers (`svarint`) + +Signed integers are encoded by converting them to unsigned integers, using the following function: + +```py +def svarint_to_varint(s): + if s < 0: + return ((-s) << 1) | 1 + else: + return s << 1 +``` + +To convert a `varint` into a signed integer: + +```py +def varint_to_svarint(uval): + return -(uval >> 1) if uval & 1 else (uval >> 1) +``` + +### Location entries + +The meaning of the codes and the following bytes are as follows: + +| Code | Meaning | Start line | End line | Start column | End column | +|-------|----------------|---------------|----------|---------------|---------------| +| 0-9 | Short form | Δ 0 | Δ 0 | See below | See below | +| 10-12 | One line form | Δ (code - 10) | Δ 0 | unsigned byte | unsigned byte | +| 13 | No column info | Δ svarint | Δ 0 | None | None | +| 14 | Long form | Δ svarint | Δ varint | varint | varint | +| 15 | No location | None | None | None | None | + +The Δ means the value is encoded as a delta from another value: + +* Start line: Delta from the previous start line, or `co_firstlineno` for the first entry. +* End line: Delta from the start line. + +### The short forms + +Codes 0-9 are the short forms. The short form consists of two bytes, +the second byte holding additional column information. The code is the +start column divided by 8 (and rounded down). + +* Start column: `(code*8) + ((second_byte>>4)&7)` +* End column: `start_column + (second_byte&15)` diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md new file mode 100644 index 00000000000000..8ca19a42b91b83 --- /dev/null +++ b/InternalDocs/compiler.md @@ -0,0 +1,607 @@ +Compiler design +=============== + +Abstract +-------- + +In CPython, the compilation from source code to bytecode involves several steps: + +1. Tokenize the source code [Parser/lexer/](../Parser/lexer) + and [Parser/tokenizer/](../Parser/tokenizer). +2. Parse the stream of tokens into an Abstract Syntax Tree + [Parser/parser.c](../Parser/parser.c). +3. Transform AST into an instruction sequence + [Python/compile.c](../Python/compile.c). +4. Construct a Control Flow Graph and apply optimizations to it + [Python/flowgraph.c](../Python/flowgraph.c). +5. Emit bytecode based on the Control Flow Graph + [Python/assemble.c](../Python/assemble.c). + +This document outlines how these steps of the process work. + +This document only describes parsing in enough depth to explain what is needed +for understanding compilation. This document provides a detailed, though not +exhaustive, view of the how the entire system works. You will most likely need +to read some source code to have an exact understanding of all details. + + +Parsing +======= + +As of Python 3.9, Python's parser is a PEG parser of a somewhat +unusual design. It is unusual in the sense that the parser's input is a stream +of tokens rather than a stream of characters which is more common with PEG +parsers. + +The grammar file for Python can be found in +[Grammar/python.gram](../Grammar/python.gram). +The definitions for literal tokens (such as `:`, numbers, etc.) can be found in +[Grammar/Tokens](../Grammar/Tokens). Various C files, including +[Parser/parser.c](../Parser/parser.c) are generated from these. + +See Also: + +* [Guide to the parser](parser.md) + for a detailed description of the parser. + +* [Changing CPython’s grammar](changing_grammar.md) + for a detailed description of the grammar. + + +Abstract syntax trees (AST) +=========================== + + +The abstract syntax tree (AST) is a high-level representation of the +program structure without the necessity of containing the source code; +it can be thought of as an abstract representation of the source code. The +specification of the AST nodes is specified using the Zephyr Abstract +Syntax Definition Language (ASDL) [^1], [^2]. + +The definition of the AST nodes for Python is found in the file +[Parser/Python.asdl](../Parser/Python.asdl). + +Each AST node (representing statements, expressions, and several +specialized types, like list comprehensions and exception handlers) is +defined by the ASDL. Most definitions in the AST correspond to a +particular source construct, such as an 'if' statement or an attribute +lookup. The definition is independent of its realization in any +particular programming language. + +The following fragment of the Python ASDL construct demonstrates the +approach and syntax: + +``` + module Python + { + stmt = FunctionDef(identifier name, arguments args, stmt* body, + expr* decorators) + | Return(expr? value) | Yield(expr? value) + attributes (int lineno) + } +``` + +The preceding example describes two different kinds of statements and an +expression: function definitions, return statements, and yield expressions. +All three kinds are considered of type `stmt` as shown by `|` separating +the various kinds. They all take arguments of various kinds and amounts. + +Modifiers on the argument type specify the number of values needed; `?` +means it is optional, `*` means 0 or more, while no modifier means only one +value for the argument and it is required. `FunctionDef`, for instance, +takes an `identifier` for the *name*, `arguments` for *args*, zero or more +`stmt` arguments for *body*, and zero or more `expr` arguments for +*decorators*. + +Do notice that something like 'arguments', which is a node type, is +represented as a single AST node and not as a sequence of nodes as with +stmt as one might expect. + +All three kinds also have an 'attributes' argument; this is shown by the +fact that 'attributes' lacks a '|' before it. + +The statement definitions above generate the following C structure type: + + +``` + typedef struct _stmt *stmt_ty; + + struct _stmt { + enum { FunctionDef_kind=1, Return_kind=2, Yield_kind=3 } kind; + union { + struct { + identifier name; + arguments_ty args; + asdl_seq *body; + } FunctionDef; + + struct { + expr_ty value; + } Return; + + struct { + expr_ty value; + } Yield; + } v; + int lineno; + } +``` + +Also generated are a series of constructor functions that allocate (in +this case) a `stmt_ty` struct with the appropriate initialization. The +`kind` field specifies which component of the union is initialized. The +`FunctionDef()` constructor function sets 'kind' to `FunctionDef_kind` and +initializes the *name*, *args*, *body*, and *attributes* fields. + +See also [Green Tree Snakes - The missing Python AST docs]( +https://greentreesnakes.readthedocs.io/en/latest) by Thomas Kluyver. + +Memory management +================= + +Before discussing the actual implementation of the compiler, a discussion of +how memory is handled is in order. To make memory management simple, an **arena** +is used that pools memory in a single location for easy +allocation and removal. This enables the removal of explicit memory +deallocation. Because memory allocation for all needed memory in the compiler +registers that memory with the arena, a single call to free the arena is all +that is needed to completely free all memory used by the compiler. + +In general, unless you are working on the critical core of the compiler, memory +management can be completely ignored. But if you are working at either the +very beginning of the compiler or the end, you need to care about how the arena +works. All code relating to the arena is in either +[Include/internal/pycore_pyarena.h](../Include/internal/pycore_pyarena.h) +or [Python/pyarena.c](../Python/pyarena.c). + +`PyArena_New()` will create a new arena. The returned `PyArena` structure +will store pointers to all memory given to it. This does the bookkeeping of +what memory needs to be freed when the compiler is finished with the memory it +used. That freeing is done with `PyArena_Free()`. This only needs to be +called in strategic areas where the compiler exits. + +As stated above, in general you should not have to worry about memory +management when working on the compiler. The technical details of memory +management have been designed to be hidden from you for most cases. + +The only exception comes about when managing a PyObject. Since the rest +of Python uses reference counting, there is extra support added +to the arena to cleanup each PyObject that was allocated. These cases +are very rare. However, if you've allocated a PyObject, you must tell +the arena about it by calling `PyArena_AddPyObject()`. + + +Source code to AST +================== + +The AST is generated from source code using the function +`_PyParser_ASTFromString()` or `_PyParser_ASTFromFile()` +[Parser/peg_api.c](../Parser/peg_api.c). + +After some checks, a helper function in +[Parser/parser.c](../Parser/parser.c) +begins applying production rules on the source code it receives; converting source +code to tokens and matching these tokens recursively to their corresponding rule. The +production rule's corresponding rule function is called on every match. These rule +functions follow the format `xx_rule`. Where *xx* is the grammar rule +that the function handles and is automatically derived from +[Grammar/python.gram](../Grammar/python.gram) by +[Tools/peg_generator/pegen/c_generator.py](../Tools/peg_generator/pegen/c_generator.py). + +Each rule function in turn creates an AST node as it goes along. It does this +by allocating all the new nodes it needs, calling the proper AST node creation +functions for any required supporting functions and connecting them as needed. +This continues until all nonterminal symbols are replaced with terminals. If an +error occurs, the rule functions backtrack and try another rule function. If +there are no more rules, an error is set and the parsing ends. + +The AST node creation helper functions have the name `_PyAST_{xx}` +where *xx* is the AST node that the function creates. These are defined by the +ASDL grammar and contained in [Python/Python-ast.c](../Python/Python-ast.c) +(which is generated by [Parser/asdl_c.py](../Parser/asdl_c.py) +from [Parser/Python.asdl](../Parser/Python.asdl)). +This all leads to a sequence of AST nodes stored in `asdl_seq` structs. + +To demonstrate everything explained so far, here's the +rule function responsible for a simple named import statement such as +`import sys`. Note that error-checking and debugging code has been +omitted. Removed parts are represented by `...`. +Furthermore, some comments have been added for explanation. These comments +may not be present in the actual code. + + +``` + // This is the production rule (from python.gram) the rule function + // corresponds to: + // import_name: 'import' dotted_as_names + static stmt_ty + import_name_rule(Parser *p) + { + ... + stmt_ty _res = NULL; + { // 'import' dotted_as_names + ... + Token * _keyword; + asdl_alias_seq* a; + // The tokenizing steps. + if ( + (_keyword = _PyPegen_expect_token(p, 513)) // token='import' + && + (a = dotted_as_names_rule(p)) // dotted_as_names + ) + { + ... + // Generate an AST for the import statement. + _res = _PyAST_Import ( a , ...); + ... + goto done; + } + ... + } + _res = NULL; + done: + ... + return _res; + } +``` + + +To improve backtracking performance, some rules (chosen by applying a +`(memo)` flag in the grammar file) are memoized. Each rule function checks if +a memoized version exists and returns that if so, else it continues in the +manner stated in the previous paragraphs. + +There are macros for creating and using `asdl_xx_seq *` types, where *xx* is +a type of the ASDL sequence. Three main types are defined +manually -- `generic`, `identifier` and `int`. These types are found in +[Python/asdl.c](../Python/asdl.c) and its corresponding header file +[Include/internal/pycore_asdl.h](../Include/internal/pycore_asdl.h). +Functions and macros for creating `asdl_xx_seq *` types are as follows: + +* `_Py_asdl_generic_seq_new(Py_ssize_t, PyArena *)`: + Allocate memory for an `asdl_generic_seq` of the specified length +* `_Py_asdl_identifier_seq_new(Py_ssize_t, PyArena *)`: + Allocate memory for an `asdl_identifier_seq` of the specified length +* `_Py_asdl_int_seq_new(Py_ssize_t, PyArena *)`: + Allocate memory for an `asdl_int_seq` of the specified length + +In addition to the three types mentioned above, some ASDL sequence types are +automatically generated by [Parser/asdl_c.py](../Parser/asdl_c.py) and found in +[Include/internal/pycore_ast.h](../Include/internal/pycore_ast.h). +Macros for using both manually defined and automatically generated ASDL +sequence types are as follows: + +* `asdl_seq_GET(asdl_xx_seq *, int)`: + Get item held at a specific position in an `asdl_xx_seq` +* `asdl_seq_SET(asdl_xx_seq *, int, stmt_ty)`: + Set a specific index in an `asdl_xx_seq` to the specified value + +Untyped counterparts exist for some of the typed macros. These are useful +when a function needs to manipulate a generic ASDL sequence: + +* `asdl_seq_GET_UNTYPED(asdl_seq *, int)`: + Get item held at a specific position in an `asdl_seq` +* `asdl_seq_SET_UNTYPED(asdl_seq *, int, stmt_ty)`: + Set a specific index in an `asdl_seq` to the specified value +* `asdl_seq_LEN(asdl_seq *)`: + Return the length of an `asdl_seq` or `asdl_xx_seq` + +Note that typed macros and functions are recommended over their untyped +counterparts. Typed macros carry out checks in debug mode and aid +debugging errors caused by incorrectly casting from `void *`. + +If you are working with statements, you must also worry about keeping +track of what line number generated the statement. Currently the line +number is passed as the last parameter to each `stmt_ty` function. + +See also [PEP 617: New PEG parser for CPython](https://peps.python.org/pep-0617/). + + +Control flow graphs +=================== + +A **control flow graph** (often referenced by its acronym, **CFG**) is a +directed graph that models the flow of a program. A node of a CFG is +not an individual bytecode instruction, but instead represents a +sequence of bytecode instructions that always execute sequentially. +Each node is called a *basic block* and must always execute from +start to finish, with a single entry point at the beginning and a +single exit point at the end. If some bytecode instruction *a* needs +to jump to some other bytecode instruction *b*, then *a* must occur at +the end of its basic block, and *b* must occur at the start of its +basic block. + +As an example, consider the following code snippet: + +```python +if x < 10: + f1() + f2() +else: + g() +end() +``` + +The `x < 10` guard is represented by its own basic block that +compares `x` with `10` and then ends in a conditional jump based on +the result of the comparison. This conditional jump allows the block +to point to both the body of the `if` and the body of the `else`. The +`if` basic block contains the `f1()` and `f2()` calls and points to +the `end()` basic block. The `else` basic block contains the `g()` +call and similarly points to the `end()` block. + +Note that more complex code in the guard, the `if` body, or the `else` +body may be represented by multiple basic blocks. For instance, +short-circuiting boolean logic in a guard like `if x or y:` +will produce one basic block that tests the truth value of `x` +and then points both (1) to the start of the `if` body and (2) to +a different basic block that tests the truth value of y. + +CFGs are useful as an intermediate representation of the code because +they are a convenient data structure for optimizations. + +AST to CFG to bytecode +====================== + +The conversion of an `AST` to bytecode is initiated by a call to the function +`_PyAST_Compile()` in [Python/compile.c](../Python/compile.c). + +The first step is to construct the symbol table. This is implemented by +`_PySymtable_Build()` in [Python/symtable.c](../Python/symtable.c). +This function begins by entering the starting code block for the AST (passed-in) +and then calling the proper `symtable_visit_{xx}` function (with *xx* being the +AST node type). Next, the AST tree is walked with the various code blocks that +delineate the reach of a local variable as blocks are entered and exited using +`symtable_enter_block()` and `symtable_exit_block()`, respectively. + +Once the symbol table is created, the `AST` is transformed by `compiler_codegen()` +in [Python/compile.c](../Python/compile.c) into a sequence of pseudo instructions. +These are similar to bytecode, but in some cases they are more abstract, and are +resolved later into actual bytecode. The construction of this instruction sequence +is handled by several functions that break the task down by various AST node types. +The functions are all named `compiler_visit_{xx}` where *xx* is the name of the node +type (such as `stmt`, `expr`, etc.). Each function receives a `struct compiler *` +and `{xx}_ty` where *xx* is the AST node type. Typically these functions +consist of a large 'switch' statement, branching based on the kind of +node type passed to it. Simple things are handled inline in the +'switch' statement with more complex transformations farmed out to other +functions named `compiler_{xx}` with *xx* being a descriptive name of what is +being handled. + +When transforming an arbitrary AST node, use the `VISIT()` macro. +The appropriate `compiler_visit_{xx}` function is called, based on the value +passed in for (so `VISIT({c}, expr, {node})` calls +`compiler_visit_expr({c}, {node})`). The `VISIT_SEQ()` macro is very similar, +but is called on AST node sequences (those values that were created as +arguments to a node that used the '*' modifier). + +Emission of bytecode is handled by the following macros: + +* `ADDOP(struct compiler *, location, int)`: + add a specified opcode +* `ADDOP_IN_SCOPE(struct compiler *, location, int)`: + like `ADDOP`, but also exits current scope; used for adding return value + opcodes in lambdas and closures +* `ADDOP_I(struct compiler *, location, int, Py_ssize_t)`: + add an opcode that takes an integer argument +* `ADDOP_O(struct compiler *, location, int, PyObject *, TYPE)`: + add an opcode with the proper argument based on the position of the + specified PyObject in PyObject sequence object, but with no handling of + mangled names; used for when you + need to do named lookups of objects such as globals, consts, or + parameters where name mangling is not possible and the scope of the + name is known; *TYPE* is the name of PyObject sequence + (`names` or `varnames`) +* `ADDOP_N(struct compiler *, location, int, PyObject *, TYPE)`: + just like `ADDOP_O`, but steals a reference to PyObject +* `ADDOP_NAME(struct compiler *, location, int, PyObject *, TYPE)`: + just like `ADDOP_O`, but name mangling is also handled; used for + attribute loading or importing based on name +* `ADDOP_LOAD_CONST(struct compiler *, location, PyObject *)`: + add the `LOAD_CONST` opcode with the proper argument based on the + position of the specified PyObject in the consts table. +* `ADDOP_LOAD_CONST_NEW(struct compiler *, location, PyObject *)`: + just like `ADDOP_LOAD_CONST`, but steals a reference to PyObject +* `ADDOP_JUMP(struct compiler *, location, int, basicblock *)`: + create a jump to a basic block + +The `location` argument is a struct with the source location to be +associated with this instruction. It is typically extracted from an +`AST` node with the `LOC` macro. The `NO_LOCATION` can be used +for *synthetic* instructions, which we do not associate with a line +number at this stage. For example, the implicit `return None` +which is added at the end of a function is not associated with any +line in the source code. + +There are several helper functions that will emit pseudo-instructions +and are named `compiler_{xx}()` where *xx* is what the function helps +with (`list`, `boolop`, etc.). A rather useful one is `compiler_nameop()`. +This function looks up the scope of a variable and, based on the +expression context, emits the proper opcode to load, store, or delete +the variable. + +Once the instruction sequence is created, it is transformed into a CFG +by `_PyCfg_FromInstructionSequence()`. Then `_PyCfg_OptimizeCodeUnit()` +applies various peephole optimizations, and +`_PyCfg_OptimizedCfgToInstructionSequence()` converts the optimized `CFG` +back into an instruction sequence. These conversions and optimizations are +implemented in [Python/flowgraph.c](../Python/flowgraph.c). + +Finally, the sequence of pseudo-instructions is converted into actual +bytecode. This includes transforming pseudo instructions into actual instructions, +converting jump targets from logical labels to relative offsets, and +construction of the [exception table](exception_handling.md) and +[locations table](code_objects.md#source-code-locations). +The bytecode and tables are then wrapped into a `PyCodeObject` along with additional +metadata, including the `consts` and `names` arrays, information about function +reference to the source code (filename, etc). All of this is implemented by +`_PyAssemble_MakeCodeObject()` in [Python/assemble.c](../Python/assemble.c). + + +Code objects +============ + +The result of `_PyAST_Compile()` is a `PyCodeObject` which is defined in +[Include/cpython/code.h](../Include/cpython/code.h). +And with that you now have executable Python bytecode! + +The code objects (byte code) are executed in `_PyEval_EvalFrameDefault()` +in [Python/ceval.c](../Python/ceval.c). + +Important files +=============== + +* [Parser/](../Parser) + + * [Parser/Python.asdl](../Parser/Python.asdl): + ASDL syntax file. + + * [Parser/asdl.py](../Parser/asdl.py): + Parser for ASDL definition files. + Reads in an ASDL description and parses it into an AST that describes it. + + * [Parser/asdl_c.py](../Parser/asdl_c.py): + Generate C code from an ASDL description. Generates + [Python/Python-ast.c](../Python/Python-ast.c) and + [Include/internal/pycore_ast.h](../Include/internal/pycore_ast.h). + + * [Parser/parser.c](../Parser/parser.c): + The new PEG parser introduced in Python 3.9. Generated by + [Tools/peg_generator/pegen/c_generator.py](../Tools/peg_generator/pegen/c_generator.py) + from the grammar [Grammar/python.gram](../Grammar/python.gram). + Creates the AST from source code. Rule functions for their corresponding production + rules are found here. + + * [Parser/peg_api.c](../Parser/peg_api.c): + Contains high-level functions which are used by the interpreter to create + an AST from source code. + + * [Parser/pegen.c](../Parser/pegen.c): + Contains helper functions which are used by functions in + [Parser/parser.c](../Parser/parser.c) to construct the AST. Also contains + helper functions which help raise better error messages when parsing source code. + + * [Parser/pegen.h](../Parser/pegen.h): + Header file for the corresponding [Parser/pegen.c](../Parser/pegen.c). + Also contains definitions of the `Parser` and `Token` structs. + +* [Python/](../Python) + + * [Python/Python-ast.c](../Python/Python-ast.c): + Creates C structs corresponding to the ASDL types. Also contains code for + marshalling AST nodes (core ASDL types have marshalling code in + [Python/asdl.c](../Python/asdl.c)). + File automatically generated by [Parser/asdl_c.py](../Parser/asdl_c.py). + This file must be committed separately after every grammar change + is committed since the `__version__` value is set to the latest + grammar change revision number. + + * [Python/asdl.c](../Python/asdl.c): + Contains code to handle the ASDL sequence type. + Also has code to handle marshalling the core ASDL types, such as number + and identifier. Used by [Python/Python-ast.c](../Python/Python-ast.c) + for marshalling AST nodes. + + * [Python/ast.c](../Python/ast.c): + Used for validating the AST. + + * [Python/ast_opt.c](../Python/ast_opt.c): + Optimizes the AST. + + * [Python/ast_unparse.c](../Python/ast_unparse.c): + Converts the AST expression node back into a string (for string annotations). + + * [Python/ceval.c](../Python/ceval.c): + Executes byte code (aka, eval loop). + + * [Python/symtable.c](../Python/symtable.c): + Generates a symbol table from AST. + + * [Python/pyarena.c](../Python/pyarena.c): + Implementation of the arena memory manager. + + * [Python/compile.c](../Python/compile.c): + Emits pseudo bytecode based on the AST. + + * [Python/flowgraph.c](../Python/flowgraph.c): + Implements peephole optimizations. + + * [Python/assemble.c](../Python/assemble.c): + Constructs a code object from a sequence of pseudo instructions. + + * [Python/instruction_sequence.c](../Python/instruction_sequence.c): + A data structure representing a sequence of bytecode-like pseudo-instructions. + +* [Include/](../Include) + + * [Include/cpython/code.h](../Include/cpython/code.h) + : Header file for [Objects/codeobject.c](../Objects/codeobject.c); + contains definition of `PyCodeObject`. + + * [Include/opcode.h](../Include/opcode.h) + : One of the files that must be modified whenever + [Lib/opcode.py](../Lib/opcode.py) is. + + * [Include/internal/pycore_ast.h](../Include/internal/pycore_ast.h) + : Contains the actual definitions of the C structs as generated by + [Python/Python-ast.c](../Python/Python-ast.c). + Automatically generated by [Parser/asdl_c.py](../Parser/asdl_c.py). + + * [Include/internal/pycore_asdl.h](../Include/internal/pycore_asdl.h) + : Header for the corresponding [Python/ast.c](../Python/ast.c). + + * [Include/internal/pycore_ast.h](../Include/internal/pycore_ast.h) + : Declares `_PyAST_Validate()` external (from [Python/ast.c](../Python/ast.c)). + + * [Include/internal/pycore_symtable.h](../Include/internal/pycore_symtable.h) + : Header for [Python/symtable.c](../Python/symtable.c). + `struct symtable` and `PySTEntryObject` are defined here. + + * [Include/internal/pycore_parser.h](../Include/internal/pycore_parser.h) + : Header for the corresponding [Parser/peg_api.c](../Parser/peg_api.c). + + * [Include/internal/pycore_pyarena.h](../Include/internal/pycore_pyarena.h) + : Header file for the corresponding [Python/pyarena.c](../Python/pyarena.c). + + * [Include/opcode_ids.h](../Include/opcode_ids.h) + : List of opcodes. Generated from [Python/bytecodes.c](../Python/bytecodes.c) + by + [Tools/cases_generator/opcode_id_generator.py](../Tools/cases_generator/opcode_id_generator.py). + +* [Objects/](../Objects) + + * [Objects/codeobject.c](../Objects/codeobject.c) + : Contains PyCodeObject-related code. + + * [Objects/frameobject.c](../Objects/frameobject.c) + : Contains the `frame_setlineno()` function which should determine whether it is allowed + to make a jump between two points in a bytecode. + +* [Lib/](../Lib) + + * [Lib/opcode.py](../Lib/opcode.py) + : opcode utilities exposed to Python. + + * [Include/core/pycore_magic_number.h](../Include/internal/pycore_magic_number.h) + : Home of the magic number (named `MAGIC_NUMBER`) for bytecode versioning. + + +Objects +======= + +* [Locations](code_objects.md#source-code-locations): Describes the location table +* [Frames](frames.md): Describes frames and the frame stack +* [Objects/object_layout.md](../Objects/object_layout.md): Describes object layout for 3.11 and later +* [Exception Handling](exception_handling.md): Describes the exception table + + +References +========== + +[^1]: Daniel C. Wang, Andrew W. Appel, Jeff L. Korn, and Chris + S. Serra. `The Zephyr Abstract Syntax Description Language.`_ + In Proceedings of the Conference on Domain-Specific Languages, + pp. 213--227, 1997. + +[^2]: The Zephyr Abstract Syntax Description Language.: + https://www.cs.princeton.edu/research/techreps/TR-554-97 diff --git a/InternalDocs/exception_handling.md b/InternalDocs/exception_handling.md new file mode 100644 index 00000000000000..28589787e1fad7 --- /dev/null +++ b/InternalDocs/exception_handling.md @@ -0,0 +1,194 @@ +Description of exception handling +--------------------------------- + +Python uses a technique known as "zero-cost" exception handling, which +minimizes the cost of supporting exceptions. In the common case (where +no exception is raised) the cost is reduced to zero (or close to zero). +The cost of raising an exception is increased, but not by much. + +The following code: + +``` +try: + g(0) +except: + res = "fail" + +``` + +compiles into intermediate code like the following: + +``` + RESUME 0 + + 1 SETUP_FINALLY 8 (to L1) + + 2 LOAD_NAME 0 (g) + PUSH_NULL + LOAD_CONST 0 (0) + CALL 1 + POP_TOP + POP_BLOCK + + -- L1: PUSH_EXC_INFO + + 3 POP_TOP + + 4 LOAD_CONST 1 ('fail') + STORE_NAME 1 (res) +``` + +`SETUP_FINALLY` and `POP_BLOCK` are pseudo-instructions. This means +that they can appear in intermediate code but they are not bytecode +instructions. `SETUP_FINALLY` specifies that henceforth, exceptions +are handled by the code at label L1. The `POP_BLOCK` instruction +reverses the effect of the last `SETUP` instruction, so that the +active exception handler reverts to what it was before. + +`SETUP_FINALLY` and `POP_BLOCK` have no effect when no exceptions +are raised. The idea of zero-cost exception handling is to replace +these pseudo-instructions by metadata which is stored alongside the +bytecode, and which is inspected only when an exception occurs. +This metadata is the exception table, and it is stored in the code +object's `co_exceptiontable` field. + +When the pseudo-instructions are translated into bytecode, +`SETUP_FINALLY` and `POP_BLOCK` are removed, and the exception +table is constructed, mapping each instruction to the exception +handler that covers it, if any. Instructions which are not +covered by any exception handler within the same code object's +bytecode, do not appear in the exception table at all. + +For the code object in our example above, the table has a single +entry specifying that all instructions that were between the +`SETUP_FINALLY` and the `POP_BLOCK` are covered by the exception +handler located at label `L1`. + +Handling Exceptions +------------------- + +At runtime, when an exception occurs, the interpreter calls +`get_exception_handler()` in [Python/ceval.c](../Python/ceval.c) +to look up the offset of the current instruction in the exception +table. If it finds a handler, control flow transfers to it. Otherwise, the +exception bubbles up to the caller, and the caller's frame is +checked for a handler covering the `CALL` instruction. This +repeats until a handler is found or the topmost frame is reached. +If no handler is found, then the interpreter function +(`_PyEval_EvalFrameDefault()`) returns NULL. During unwinding, +the traceback is constructed as each frame is added to it by +`PyTraceBack_Here()`, which is in [Python/traceback.c](../Python/traceback.c). + +Along with the location of an exception handler, each entry of the +exception table also contains the stack depth of the `try` instruction +and a boolean `lasti` value, which indicates whether the instruction +offset of the raising instruction should be pushed to the stack. + +Handling an exception, once an exception table entry is found, consists +of the following steps: + +1. pop values from the stack until it matches the stack depth for the handler. +2. if `lasti` is true, then push the offset that the exception was raised at. +3. push the exception to the stack. +4. jump to the target offset and resume execution. + + +Reraising Exceptions and `lasti` +-------------------------------- + +The purpose of pushing `lasti` to the stack is for cases where an exception +needs to be re-raised, and be associated with the original instruction that +raised it. This happens, for example, at the end of a `finally` block, when +any in-flight exception needs to be propagated on. As the frame's instruction +pointer now points into the finally block, a `RERAISE` instruction +(with `oparg > 0`) sets it to the `lasti` value from the stack. + +Format of the exception table +----------------------------- + +Conceptually, the exception table consists of a sequence of 5-tuples: + +1. `start-offset` (inclusive) +2. `end-offset` (exclusive) +3. `target` +4. `stack-depth` +5. `push-lasti` (boolean) + +All offsets and lengths are in code units, not bytes. + +We want the format to be compact, but quickly searchable. +For it to be compact, it needs to have variable sized entries so that we can store common (small) offsets compactly, but handle large offsets if needed. +For it to be searchable quickly, we need to support binary search giving us log(n) performance in all cases. +Binary search typically assumes fixed size entries, but that is not necessary, as long as we can identify the start of an entry. + +It is worth noting that the size (end-start) is always smaller than the end, so we encode the entries as: +`start, size, target, depth, push-lasti`. + +Also, sizes are limited to 2**30 as the code length cannot exceed 2**31 and each code unit takes 2 bytes. +It also happens that depth is generally quite small. + +So, we need to encode: + +``` +start (up to 30 bits) +size (up to 30 bits) +target (up to 30 bits) +depth (up to ~8 bits) +lasti (1 bit) +``` + +We need a marker for the start of the entry, so the first byte of entry will have the most significant bit set. +Since the most significant bit is reserved for marking the start of an entry, we have 7 bits per byte to encode offsets. +Encoding uses a standard varint encoding, but with only 7 bits instead of the usual 8. +The 8 bits of a byte are (msb left) SXdddddd where S is the start bit. X is the extend bit meaning that the next byte is required to extend the offset. + +In addition, we combine `depth` and `lasti` into a single value, `((depth<<1)+lasti)`, before encoding. + +For example, the exception entry: + +``` +start: 20 +end: 28 +target: 100 +depth: 3 +lasti: False +``` + +is encoded by first converting to the more compact four value form: + +``` +start: 20 +size: 8 +target: 100 +depth<<1+lasti: 6 +``` + +which is then encoded as: + +``` +148 (MSB + 20 for start) +8 (size) +65 (Extend bit + 1) +36 (Remainder of target, 100 == (1<<6)+36) +6 +``` + +for a total of five bytes. + +The code to construct the exception table is in `assemble_exception_table()` +in [Python/assemble.c](../Python/assemble.c). + +The interpreter's function to lookup the table by instruction offset is +`get_exception_handler()` in [Python/ceval.c](../Python/ceval.c). +The Python function `_parse_exception_table()` in [Lib/dis.py](../Lib/dis.py) +returns the exception table content as a list of namedtuple instances. + +Exception Chaining Implementation +--------------------------------- + +[Exception chaining](https://docs.python.org/dev/tutorial/errors.html#exception-chaining) +refers to setting the `__context__` and `__cause__` fields of an exception as it is +being raised. The `__context__` field is set by `_PyErr_SetObject()` in +[Python/errors.c](../Python/errors.c) (which is ultimately called by all +`PyErr_Set*()` functions). The `__cause__` field (explicit chaining) is set by +the `RAISE_VARARGS` bytecode. diff --git a/Objects/frame_layout.md b/InternalDocs/frames.md similarity index 61% rename from Objects/frame_layout.md rename to InternalDocs/frames.md index b348e85689f507..1a909009eea610 100644 --- a/Objects/frame_layout.md +++ b/InternalDocs/frames.md @@ -1,51 +1,47 @@ -# The Frame Stack - -Each call to a Python function has an activation record, -commonly known as a "frame". -Python semantics allows frames to outlive the activation, -so they have (before 3.11) been allocated on the heap. -This is expensive as it requires many allocations and -results in poor locality of reference. - -In 3.11, rather than have these frames scattered about memory, -as happens for heap-allocated objects, frames are allocated -contiguously in a per-thread stack. -This improves performance significantly for two reasons: -* It reduces allocation overhead to a pointer comparison and increment. -* Stack allocated data has the best possible locality and will always be in - CPU cache. - -Generator and coroutines still need heap allocated activation records, but -can be linked into the per-thread stack so as to not impact performance too much. +# Frames -## Layout - -Each activation record consists of four conceptual sections: +Each call to a Python function has an activation record, commonly known as a +"frame". It contains information about the function being executed, consisting +of three conceptual sections: * Local variables (including arguments, cells and free variables) * Evaluation stack -* Specials: The per-frame object references needed by the VM: globals dict, - code object, etc. -* Linkage: Pointer to the previous activation record, stack depth, etc. +* Specials: The per-frame object references needed by the VM, including + globals dict, code object, instruction pointer, stack depth, the + previous frame, etc. + +The definition of the `_PyInterpreterFrame` struct is in +[Include/internal/pycore_frame.h](../Include/internal/pycore_frame.h). + +# Allocation -### Layout +Python semantics allows frames to outlive the activation, so they need to +be allocated outside the C call stack. To reduce overhead and improve locality +of reference, most frames are allocated contiguously in a per-thread stack +(see `_PyThreadState_PushFrame` in [Python/pystate.c](../Python/pystate.c)). -The specials and linkage sections are a fixed size, so are grouped together. +Frames of generators and coroutines are embedded in the generator and coroutine +objects, so are not allocated in the per-thread stack. See `PyGenObject` in +[Include/internal/pycore_genobject.h](../Include/internal/pycore_genobject.h). + +## Layout Each activation record is laid out as: -* Specials and linkage + +* Specials * Locals * Stack This seems to provide the best performance without excessive complexity. -It needs the interpreter to hold two pointers, a frame pointer and a stack pointer. +The specials have a fixed size, so the offset of the locals is known. The +interpreter needs to hold two pointers, a frame pointer and a stack pointer. #### Alternative layout An alternative layout that was used for part of 3.11 alpha was: * Locals -* Specials and linkage +* Specials * Stack This has the advantage that no copying is required when making a call, @@ -53,23 +49,10 @@ as the arguments on the stack are (usually) already in the correct location for the parameters. However, it requires the VM to maintain an extra pointer for the locals, which can hurt performance. -A variant that only needs the need two pointers is to reverse the numbering -of the locals, so that the last one is numbered `0`, and the first in memory -is numbered `N-1`. -This allows the locals, specials and linkage to accessed from the frame pointer. -We may implement this in the future. - -#### Note: - -> In a contiguous stack, we would need to save one fewer registers, as the -> top of the caller's activation record would be the same at the base of the -> callee's. However, since some activation records are kept on the heap we -> cannot do this. - ### Generators and Coroutines Generators and coroutines contain a `_PyInterpreterFrame` -The specials sections contains the following pointers: +The specials section contains the following pointers: * Globals dict * Builtins dict @@ -86,31 +69,28 @@ and builtins, than strong references to both globals and builtins. When creating a backtrace or when calling `sys._getframe()` the frame becomes visible to Python code. When this happens a new `PyFrameObject` is created -and a strong reference to it placed in the `frame_obj` field of the specials +and a strong reference to it is placed in the `frame_obj` field of the specials section. The `frame_obj` field is initially `NULL`. The `PyFrameObject` may outlive a stack-allocated `_PyInterpreterFrame`. If it does then `_PyInterpreterFrame` is copied into the `PyFrameObject`, except the evaluation stack which must be empty at this point. -The linkage section is updated to reflect the new location of the frame. +The previous frame link is updated to reflect the new location of the frame. This mechanism provides the appearance of persistent, heap-allocated frames for each activation, but with low runtime overhead. ### Generators and Coroutines - -Generator objects have a `_PyInterpreterFrame` embedded in them. -This means that creating a generator requires only a single allocation, -reducing allocation overhead and improving locality of reference. -The embedded frame is linked into the per-thread frame when iterated or -awaited. +Generators (objects of type `PyGen_Type`, `PyCoro_Type` or +`PyAsyncGen_Type`) have a `_PyInterpreterFrame` embedded in them, so +that they can be created with a single memory allocation. +When such an embedded frame is iterated or awaited, it can be linked with +frames on the per-thread stack via the linkage fields. If a frame object associated with a generator outlives the generator, then -the embedded `_PyInterpreterFrame` is copied into the frame object. - - -All the above applies to coroutines and async generators as well. +the embedded `_PyInterpreterFrame` is copied into the frame object (see +`take_ownership()` in [Python/frame.c](../Python/frame.c)). ### Field names @@ -119,7 +99,7 @@ Thus, some of the field names may be a bit misleading. For example the `f_globals` field has a `f_` prefix implying it belongs to the `PyFrameObject` struct, although it belongs to the `_PyInterpreterFrame` struct. -We may rationalize this naming scheme for 3.12. +We may rationalize this naming scheme for a later version. ### Shim frames @@ -148,7 +128,7 @@ The `return_offset` field determines where a `RETURN` should go in the caller, relative to `instr_ptr`. It is only meaningful to the callee, so it needs to be set in any instruction that implements a call (to a Python function), including CALL, SEND and BINARY_SUBSCR_GETITEM, among others. If there is no -callee, then return_offset is meaningless. It is necessary to have a separate +callee, then return_offset is meaningless. It is necessary to have a separate field for the return offset because (1) if we apply this offset to `instr_ptr` while executing the `RETURN`, this is too early and would lose us information about the previous instruction which we could need for introspecting and diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md new file mode 100644 index 00000000000000..e4cb9e45c9e96a --- /dev/null +++ b/InternalDocs/garbage_collector.md @@ -0,0 +1,654 @@ +Garbage collector design +======================== + +Abstract +======== + +The main garbage collection algorithm used by CPython is reference counting. The basic idea is +that CPython counts how many different places there are that have a reference to an +object. Such a place could be another object, or a global (or static) C variable, or +a local variable in some C function. When an object’s reference count becomes zero, +the object is deallocated. If it contains references to other objects, their +reference counts are decremented. Those other objects may be deallocated in turn, if +this decrement makes their reference count become zero, and so on. The reference +count field can be examined using the `sys.getrefcount()` function (notice that the +value returned by this function is always 1 more as the function also has a reference +to the object when called): + +```pycon +>>> x = object() +>>> sys.getrefcount(x) +2 +>>> y = x +>>> sys.getrefcount(x) +3 +>>> del y +>>> sys.getrefcount(x) +2 +``` + +The main problem with the reference counting scheme is that it does not handle reference +cycles. For instance, consider this code: + +```pycon +>>> container = [] +>>> container.append(container) +>>> sys.getrefcount(container) +3 +>>> del container +``` + +In this example, `container` holds a reference to itself, so even when we remove +our reference to it (the variable "container") the reference count never falls to 0 +because it still has its own internal reference. Therefore it would never be +cleaned just by simple reference counting. For this reason some additional machinery +is needed to clean these reference cycles between objects once they become +unreachable. This is the cyclic garbage collector, usually called just Garbage +Collector (GC), even though reference counting is also a form of garbage collection. + +Starting in version 3.13, CPython contains two GC implementations: + +- The default build implementation relies on the + [global interpreter lock](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) + for thread safety. +- The free-threaded build implementation pauses other executing threads when + performing a collection for thread safety. + +Both implementations use the same basic algorithms, but operate on different +data structures. See the section on +[Differences between GC implementations](#Differences-between-GC-implementations) +for the details. + + +Memory layout and object structure +================================== + +The garbage collector requires additional fields in Python objects to support +garbage collection. These extra fields are different in the default and the +free-threaded builds. + + +GC for the default build +------------------------ + +Normally the C structure supporting a regular Python object looks as follows: + +``` + object -----> +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ \ + | ob_refcnt | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ | PyObject_HEAD + | *ob_type | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ / + | ... | +``` + +In order to support the garbage collector, the memory layout of objects is altered +to accommodate extra information **before** the normal layout: + +``` + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ \ + | *_gc_next | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ | PyGC_Head + | *_gc_prev | | + object -----> +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ / + | ob_refcnt | \ + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ | PyObject_HEAD + | *ob_type | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ / + | ... | +``` + + +In this way the object can be treated as a normal python object and when the extra +information associated to the GC is needed the previous fields can be accessed by a +simple type cast from the original object: `((PyGC_Head *)(the_object)-1)`. + +As is explained later in the +[Optimization: reusing fields to save memory](#optimization-reusing-fields-to-save-memory) +section, these two extra fields are normally used to keep doubly linked lists of all the +objects tracked by the garbage collector (these lists are the GC generations, more on +that in the [Optimization: incremental collection](#Optimization-incremental-collection) section), but +they are also reused to fulfill other purposes when the full doubly linked list +structure is not needed as a memory optimization. + +Doubly linked lists are used because they efficiently support the most frequently required operations. In +general, the collection of all objects tracked by GC is partitioned into disjoint sets, each in its own +doubly linked list. Between collections, objects are partitioned into "generations", reflecting how +often they've survived collection attempts. During collections, the generation(s) being collected +are further partitioned into, for example, sets of reachable and unreachable objects. Doubly linked lists +support moving an object from one partition to another, adding a new object, removing an object +entirely (objects tracked by GC are most often reclaimed by the refcounting system when GC +isn't running at all!), and merging partitions, all with a small constant number of pointer updates. +With care, they also support iterating over a partition while objects are being added to - and +removed from - it, which is frequently required while GC is running. + +GC for the free-threaded build +------------------------------ + +In the free-threaded build, Python objects contain a 1-byte field +`ob_gc_bits` that is used to track garbage collection related state. The +field exists in all objects, including ones that do not support cyclic +garbage collection. The field is used to identify objects that are tracked +by the collector, ensure that finalizers are called only once per object, +and, during garbage collection, differentiate reachable vs. unreachable objects. + +``` + object -----> +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ \ + | ob_tid | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ | + | pad | ob_mutex | ob_gc_bits | ob_ref_local | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ | PyObject_HEAD + | ob_ref_shared | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ | + | *ob_type | | + +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ / + | ... | +``` + +Note that not all fields are to scale. `pad` is two bytes, `ob_mutex` and +`ob_gc_bits` are each one byte, and `ob_ref_local` is four bytes. The +other fields, `ob_tid`, `ob_ref_shared`, and `ob_type`, are all +pointer-sized (that is, eight bytes on a 64-bit platform). + + +The garbage collector also temporarily repurposes the `ob_tid` (thread ID) +and `ob_ref_local` (local reference count) fields for other purposes during +collections. + + +C APIs +------ + +Specific APIs are offered to allocate, deallocate, initialize, track, and untrack +objects with GC support. These APIs can be found in the +[Garbage Collector C API documentation](https://docs.python.org/3/c-api/gcsupport.html). + +Apart from this object structure, the type object for objects supporting garbage +collection must include the `Py_TPFLAGS_HAVE_GC` in its `tp_flags` slot and +provide an implementation of the `tp_traverse` handler. Unless it can be proven +that the objects cannot form reference cycles with only objects of its type or unless +the type is immutable, a `tp_clear` implementation must also be provided. + + +Identifying reference cycles +============================ + +The algorithm that CPython uses to detect those reference cycles is +implemented in the `gc` module. The garbage collector **only focuses** +on cleaning container objects (that is, objects that can contain a reference +to one or more objects). These can be arrays, dictionaries, lists, custom +class instances, classes in extension modules, etc. One could think that +cycles are uncommon but the truth is that many internal references needed by +the interpreter create cycles everywhere. Some notable examples: + +- Exceptions contain traceback objects that contain a list of frames that + contain the exception itself. +- Module-level functions reference the module's dict (which is needed to resolve globals), + which in turn contains entries for the module-level functions. +- Instances have references to their class which itself references its module, and the module + contains references to everything that is inside (and maybe other modules) + and this can lead back to the original instance. +- When representing data structures like graphs, it is very typical for them to + have internal links to themselves. + +To correctly dispose of these objects once they become unreachable, they need +to be identified first. To understand how the algorithm works, let’s take +the case of a circular linked list which has one link referenced by a +variable `A`, and one self-referencing object which is completely +unreachable: + +```pycon +>>> import gc +>>> +>>> class Link: +... def __init__(self, next_link=None): +... self.next_link = next_link +... +>>> link_3 = Link() +>>> link_2 = Link(link_3) +>>> link_1 = Link(link_2) +>>> link_3.next_link = link_1 +>>> A = link_1 +>>> del link_1, link_2, link_3 +>>> +>>> link_4 = Link() +>>> link_4.next_link = link_4 +>>> del link_4 +>>> +>>> # Collect the unreachable Link object (and its .__dict__ dict). +>>> gc.collect() +2 +``` + +The GC starts with a set of candidate objects it wants to scan. In the +default build, these "objects to scan" might be all container objects or a +smaller subset (or "generation"). In the free-threaded build, the collector +always scans all container objects. + +The objective is to identify all the unreachable objects. The collector does +this by identifying reachable objects; the remaining objects must be +unreachable. The first step is to identify all of the "to scan" objects that +are **directly** reachable from outside the set of candidate objects. These +objects have a refcount larger than the number of incoming references from +within the candidate set. + +Every object that supports garbage collection will have an extra reference +count field initialized to the reference count (`gc_ref` in the figures) +of that object when the algorithm starts. This is because the algorithm needs +to modify the reference count to do the computations and in this way the +interpreter will not modify the real reference count field. + +![gc-image1](images/python-cyclic-gc-1-new-page.png) + +The GC then iterates over all containers in the first list and decrements by one the +`gc_ref` field of any other object that container is referencing. Doing +this makes use of the `tp_traverse` slot in the container class (implemented +using the C API or inherited by a superclass) to know what objects are referenced by +each container. After all the objects have been scanned, only the objects that have +references from outside the “objects to scan” list will have `gc_ref > 0`. + +![gc-image2](images/python-cyclic-gc-2-new-page.png) + +Notice that having `gc_ref == 0` does not imply that the object is unreachable. +This is because another object that is reachable from the outside (`gc_ref > 0`) +can still have references to it. For instance, the `link_2` object in our example +ended having `gc_ref == 0` but is referenced still by the `link_1` object that +is reachable from the outside. To obtain the set of objects that are really +unreachable, the garbage collector re-scans the container objects using the +`tp_traverse` slot; this time with a different traverse function that marks objects with +`gc_ref == 0` as "tentatively unreachable" and then moves them to the +tentatively unreachable list. The following image depicts the state of the lists in a +moment when the GC processed the `link_3` and `link_4` objects but has not +processed `link_1` and `link_2` yet. + +![gc-image3](images/python-cyclic-gc-3-new-page.png) + +Then the GC scans the next `link_1` object. Because it has `gc_ref == 1`, +the gc does not do anything special because it knows it has to be reachable (and is +already in what will become the reachable list): + +![gc-image4](images/python-cyclic-gc-4-new-page.png) + +When the GC encounters an object which is reachable (`gc_ref > 0`), it traverses +its references using the `tp_traverse` slot to find all the objects that are +reachable from it, moving them to the end of the list of reachable objects (where +they started originally) and setting its `gc_ref` field to 1. This is what happens +to `link_2` and `link_3` below as they are reachable from `link_1`. From the +state in the previous image and after examining the objects referred to by `link_1` +the GC knows that `link_3` is reachable after all, so it is moved back to the +original list and its `gc_ref` field is set to 1 so that if the GC visits it again, +it will know that it's reachable. To avoid visiting an object twice, the GC marks all +objects that have already been visited once (by unsetting the `PREV_MASK_COLLECTING` +flag) so that if an object that has already been processed is referenced by some other +object, the GC does not process it twice. + +![gc-image5](images/python-cyclic-gc-5-new-page.png) + +Notice that an object that was marked as "tentatively unreachable" and was later +moved back to the reachable list will be visited again by the garbage collector +as now all the references that that object has need to be processed as well. This +process is really a breadth first search over the object graph. Once all the objects +are scanned, the GC knows that all container objects in the tentatively unreachable +list are really unreachable and can thus be garbage collected. + +Pragmatically, it's important to note that no recursion is required by any of this, +and neither does it in any other way require additional memory proportional to the +number of objects, number of pointers, or the lengths of pointer chains. Apart from +`O(1)` storage for internal C needs, the objects themselves contain all the storage +the GC algorithms require. + +Why moving unreachable objects is better +---------------------------------------- + +It sounds logical to move the unreachable objects under the premise that most objects +are usually reachable, until you think about it: the reason it pays isn't actually +obvious. + +Suppose we create objects A, B, C in that order. They appear in the young generation +in the same order. If B points to A, and C to B, and C is reachable from outside, +then the adjusted refcounts after the first step of the algorithm runs will be 0, 0, +and 1 respectively because the only reachable object from the outside is C. + +When the next step of the algorithm finds A, A is moved to the unreachable list. The +same for B when it's first encountered. Then C is traversed, B is moved *back* to +the reachable list. B is eventually traversed, and then A is moved back to the reachable +list. + +So instead of not moving at all, the reachable objects B and A are each moved twice. +Why is this a win? A straightforward algorithm to move the reachable objects instead +would move A, B, and C once each. The key is that this dance leaves the objects in +order C, B, A - it's reversed from the original order. On all *subsequent* scans, +none of them will move. Since most objects aren't in cycles, this can save an +unbounded number of moves across an unbounded number of later collections. The only +time the cost can be higher is the first time the chain is scanned. + +Destroying unreachable objects +============================== + +Once the GC knows the list of unreachable objects, a very delicate process starts +with the objective of completely destroying these objects. Roughly, the process +follows these steps in order: + +1. Handle and clear weak references (if any). Weak references to unreachable objects + are set to `None`. If the weak reference has an associated callback, the callback + is enqueued to be called once the clearing of weak references is finished. We only + invoke callbacks for weak references that are themselves reachable. If both the weak + reference and the pointed-to object are unreachable we do not execute the callback. + This is partly for historical reasons: the callback could resurrect an unreachable + object and support for weak references predates support for object resurrection. + Ignoring the weak reference's callback is fine because both the object and the weakref + are going away, so it's legitimate to say the weak reference is going away first. +2. If an object has legacy finalizers (`tp_del` slot) move it to the + `gc.garbage` list. +3. Call the finalizers (`tp_finalize` slot) and mark the objects as already + finalized to avoid calling finalizers twice if the objects are resurrected or + if other finalizers have removed the object first. +4. Deal with resurrected objects. If some objects have been resurrected, the GC + finds the new subset of objects that are still unreachable by running the cycle + detection algorithm again and continues with them. +5. Call the `tp_clear` slot of every object so all internal links are broken and + the reference counts fall to 0, triggering the destruction of all unreachable + objects. + +Optimization: incremental collection +==================================== + +In order to bound the length of each garbage collection pause, the GC implementation +for the default build uses incremental collection with two generations. + +Generational garbage collection takes advantage of what is known as the weak +generational hypothesis: Most objects die young. +This has proven to be very close to the reality of many Python +programs as many temporary objects are created and destroyed very quickly. + +To take advantage of this fact, all container objects are segregated into +two generations: young and old. Every new object starts in the young generation. +Each garbage collection scans the entire young generation and part of the old generation. + +The time taken to scan the young generation can be controlled by controlling its +size, but the size of the old generation cannot be controlled. +In order to keep pause times down, scanning of the old generation of the heap +occurs in increments. + +To keep track of what has been scanned, the old generation contains two lists: + +* Those objects that have not yet been scanned, referred to as the `pending` list. +* Those objects that have been scanned, referred to as the `visited` list. + +To detect and collect all unreachable objects in the heap, the garbage collector +must scan the whole heap. This whole heap scan is called a full scavenge. + +Increments +---------- + +Each full scavenge is performed in a series of increments. +For each full scavenge, the combined increments will cover the whole heap. + +Each increment is made up of: + +* The young generation +* The old generation's least recently scanned objects +* All objects reachable from those objects that have not yet been scanned this full scavenge + +The surviving objects (those that are not collected) are moved to the back of the +`visited` list in the old generation. + +When a full scavenge starts, no objects in the heap are considered to have been scanned, +so all objects in the old generation must be in the `pending` space. +When all objects in the heap have been scanned a cycle ends, and all objects are moved +to the `pending` list again. To avoid having to traverse the entire list, which list is +`pending` and which is `visited` is determined by a field in the `GCState` struct. +The `visited` and `pending` lists can be swapped by toggling this bit. + +Correctness +----------- + +The [algorithm for identifying cycles](#Identifying-reference-cycles) will find all +unreachable cycles in a list of objects, but will not find any cycles that are +even partly outside of that list. +Therefore, to be guaranteed that a full scavenge will find all unreachable cycles, +each cycle must be fully contained within a single increment. + +To make sure that no partial cycles are included in the increment we perform a +[transitive closure](https://en.wikipedia.org/wiki/Transitive_closure) +over reachable, unscanned objects from the initial increment. +Since the transitive closure of objects reachable from an object must be a (non-strict) +superset of any unreachable cycle including that object, we are guaranteed that a +transitive closure cannot contain any partial cycles. +We can exclude scanned objects, as they must have been reachable when scanned. +If a scanned object becomes part of an unreachable cycle after being scanned, it will +not be collected this at this time, but it will be collected in the next full scavenge. + +> [!NOTE] +> The GC implementation for the free-threaded build does not use incremental collection. +> Every collection operates on the entire heap. + +In order to decide when to run, the collector keeps track of the number of object +allocations and deallocations since the last collection. When the number of +allocations minus the number of deallocations exceeds `threshold0`, +collection starts. `threshold1` determines the fraction of the old +collection that is included in the increment. +The fraction is inversely proportional to `threshold1`, +as historically a larger `threshold1` meant that old generation +collections were performed less frequently. +`threshold2` is ignored. + +These thresholds can be examined using the +[`gc.get_threshold()`](https://docs.python.org/3/library/gc.html#gc.get_threshold) +function: + +```pycon +>>> import gc +>>> gc.get_threshold() +(700, 10, 10) +``` + +The content of these generations can be examined using the +`gc.get_objects(generation=NUM)` function and collections can be triggered +specifically in a generation by calling `gc.collect(generation=NUM)`. + +```pycon +>>> import gc +>>> class MyObj: +... pass +... +>>> # Move everything to the old generation so it's easier to inspect +>>> # the young generation. +>>> gc.collect() +0 +>>> # Create a reference cycle. +>>> x = MyObj() +>>> x.self = x +>>> +>>> # Initially the object is in the young generation. +>>> gc.get_objects(generation=0) +[..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] +>>> +>>> # After a collection of the youngest generation the object +>>> # moves to the old generation. +>>> gc.collect(generation=0) +0 +>>> gc.get_objects(generation=0) +[] +>>> gc.get_objects(generation=1) +[] +>>> gc.get_objects(generation=2) +[..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] +``` + + +Optimization: visiting reachable objects +======================================== + +An object cannot be garbage if it can be reached. + +To avoid having to identify reference cycles across the whole heap, we can +reduce the amount of work done considerably by first moving most reachable objects +to the `visited` space. Empirically, most reachable objects can be reached from a +small set of global objects and local variables. +This step does much less work per object, so reduces the time spent +performing garbage collection by at least half. + +> [!NOTE] +> Objects that are not determined to be reachable by this pass are not necessarily +> unreachable. We still need to perform the main algorithm to determine which objects +> are actually unreachable. +We use the same technique of forming a transitive closure as the incremental +collector does to find reachable objects, seeding the list with some global +objects and the currently executing frames. + +This phase moves objects to the `visited` space, as follows: + +1. All objects directly referred to by any builtin class, the `sys` module, the `builtins` +module and all objects directly referred to from stack frames are added to a working +set of reachable objects. +2. Until this working set is empty: + 1. Pop an object from the set and move it to the `visited` space + 2. For each object directly reachable from that object: + * If it is not already in `visited` space and it is a GC object, + add it to the working set + + +Before each increment of collection is performed, the stacks are scanned +to check for any new stack frames that have been created since the last +increment. All objects directly referred to from those stack frames are +added to the working set. +Then the above algorithm is repeated, starting from step 2. + +Optimization: reusing fields to save memory +=========================================== + +In order to save memory, the two linked list pointers in every object with GC +support are reused for several purposes. This is a common optimization known +as "fat pointers" or "tagged pointers": pointers that carry additional data, +"folded" into the pointer, meaning stored inline in the data representing the +address, taking advantage of certain properties of memory addressing. This is +possible as most architectures align certain types of data +to the size of the data, often a word or multiple thereof. This discrepancy +leaves a few of the least significant bits of the pointer unused, which can be +used for tags or to keep other information – most often as a bit field (each +bit a separate tag) – as long as code that uses the pointer masks out these +bits before accessing memory. For example, on a 32-bit architecture (for both +addresses and word size), a word is 32 bits = 4 bytes, so word-aligned +addresses are always a multiple of 4, hence end in `00`, leaving the last 2 bits +available; while on a 64-bit architecture, a word is 64 bits = 8 bytes, so +word-aligned addresses end in `000`, leaving the last 3 bits available. + +The CPython GC makes use of two fat pointers that correspond to the extra fields +of `PyGC_Head` discussed in the `Memory layout and object structure`_ section: + +> [!WARNING] +> Because the presence of extra information, "tagged" or "fat" pointers cannot be +> dereferenced directly and the extra information must be stripped off before +> obtaining the real memory address. Special care needs to be taken with +> functions that directly manipulate the linked lists, as these functions +> normally assume the pointers inside the lists are in a consistent state. + + +- The `_gc_prev` field is normally used as the "previous" pointer to maintain the + doubly linked list but its lowest two bits are used to keep the flags + `PREV_MASK_COLLECTING` and `_PyGC_PREV_MASK_FINALIZED`. Between collections, + the only flag that can be present is `_PyGC_PREV_MASK_FINALIZED` that indicates + if an object has been already finalized. During collections `_gc_prev` is + temporarily used for storing a copy of the reference count (`gc_ref`), in + addition to two flags, and the GC linked list becomes a singly linked list until + `_gc_prev` is restored. + +- The `_gc_next` field is used as the "next" pointer to maintain the doubly linked + list but during collection its lowest bit is used to keep the + `NEXT_MASK_UNREACHABLE` flag that indicates if an object is tentatively + unreachable during the cycle detection algorithm. This is a drawback to using only + doubly linked lists to implement partitions: while most needed operations are + constant-time, there is no efficient way to determine which partition an object is + currently in. Instead, when that's needed, ad hoc tricks (like the + `NEXT_MASK_UNREACHABLE` flag) are employed. + +Optimization: delayed untracking containers +=========================================== + +Certain types of containers cannot participate in a reference cycle, and so do +not need to be tracked by the garbage collector. Untracking these objects +reduces the cost of garbage collection. However, determining which objects may +be untracked is not free, and the costs must be weighed against the benefits +for garbage collection. There are two possible strategies for when to untrack +a container: + +1. When the container is created. +2. When the container is examined by the garbage collector. + +As a general rule, instances of atomic types aren't tracked and instances of +non-atomic types (containers, user-defined objects...) are. + +Tuples containing only immutable objects (integers, strings etc, +and recursively, tuples of immutable objects) do not need to be tracked. The +interpreter creates a large number of tuples, many of which will not survive +until garbage collection. It is therefore not worthwhile to untrack eligible +tuples at creation time. Instead, all tuples except the empty tuple are tracked +when created. During garbage collection it is determined whether any surviving +tuples can be untracked. A tuple can be untracked if all of its contents are +already not tracked. Tuples are examined for untracking in all garbage collection +cycles. + +The garbage collector module provides the Python function `is_tracked(obj)`, which returns +the current tracking status of the object. Subsequent garbage collections may change the +tracking status of the object. + +```pycon +>>> gc.is_tracked(0) +False +>>> gc.is_tracked("a") +False +>>> gc.is_tracked([]) +True +>>> gc.is_tracked(()) +False +>>> gc.is_tracked({}) +True +>>> gc.is_tracked({"a": 1}) +True +``` + +Differences between GC implementations +====================================== + +This section summarizes the differences between the GC implementation in the +default build and the implementation in the free-threaded build. + +The default build implementation makes extensive use of the `PyGC_Head` data +structure, while the free-threaded build implementation does not use that +data structure. + +- The default build implementation stores all tracked objects in a doubly + linked list using `PyGC_Head`. The free-threaded build implementation + instead relies on the embedded mimalloc memory allocator to scan the heap + for tracked objects. +- The default build implementation uses `PyGC_Head` for the unreachable + object list. The free-threaded build implementation repurposes the + `ob_tid` field to store a unreachable objects linked list. +- The default build implementation stores flags in the `_gc_prev` field of + `PyGC_Head`. The free-threaded build implementation stores these flags + in `ob_gc_bits`. + + +The default build implementation relies on the +[global interpreter lock](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) +for thread safety. The free-threaded build implementation has two "stop the +world" pauses, in which all other executing threads are temporarily paused so +that the GC can safely access reference counts and object attributes. + +The default build implementation is a generational collector. The +free-threaded build is non-generational; each collection scans the entire +heap. + +- Keeping track of object generations is simple and inexpensive in the default + build. The free-threaded build relies on mimalloc for finding tracked + objects; identifying "young" objects without scanning the entire heap would + be more difficult. + + +> [!NOTE] +> **Document history** +> +> Pablo Galindo Salgado - Original author +> +> Irit Katriel - Convert to Markdown diff --git a/InternalDocs/generators.md b/InternalDocs/generators.md new file mode 100644 index 00000000000000..87fbb91236844b --- /dev/null +++ b/InternalDocs/generators.md @@ -0,0 +1,106 @@ + +Generators and Coroutines +========================= + +Generators +---------- + +Generators in CPython are implemented with the struct `PyGenObject`. +They consist of a [`frame`](frames.md) and metadata about the generator's +execution state. + +A generator object resumes execution in its frame when its `send()` +method is called. This is analogous to a function executing in its own +frame when it is called, but a function returns to the calling frame only once, +while a generator "returns" execution to the caller's frame every time +it emits a new item with a +[`yield` expression](https://docs.python.org/dev/reference/expressions.html#yield-expressions). +This is implemented by the +[`YIELD_VALUE`](https://docs.python.org/dev/library/dis.html#opcode-YIELD_VALUE) +bytecode, which is similar to +[`RETURN_VALUE`](https://docs.python.org/dev/library/dis.html#opcode-RETURN_VALUE) +in the sense that it puts a value on the stack and returns execution to the +calling frame, but it also needs to perform additional work to leave the generator +frame in a state that allows it to be resumed. In particular, it updates the frame's +instruction pointer and stores the interpreter's exception state on the generator +object. When the generator is resumed, this exception state is copied back to the +interpreter state. + +The `frame` of a generator is embedded in the generator object struct as a +[`_PyInterpreterFrame`](frames.md) (see `_PyGenObject_HEAD` in +[`pycore_genobject.h`](../Include/internal/pycore_genobject.h)). +This means that we can get the frame from the generator or the generator +from the frame (see `_PyGen_GetGeneratorFromFrame` in the same file). +Other fields of the generator struct include metadata (such as the name of +the generator function) and runtime state information (such as whether its +frame is executing, suspended, cleared, etc.). + +Generator Object Creation and Destruction +----------------------------------------- + +The bytecode of a generator function begins with a +[`RETURN_GENERATOR`](https://docs.python.org/dev/library/dis.html#opcode-RETURN_GENERATOR) +instruction, which creates a generator object, including its embedded frame. +The generator's frame is initialized as a copy of the frame in which +`RETURN_GENERATOR` is executing, but its `owner` field is overwritten to indicate +that it is owned by a generator. Finally, `RETURN_GENERATOR` pushes the new generator +object to the stack and returns to the caller of the generator function (at +which time its frame is destroyed). When the generator is next resumed by +[`gen_send_ex2()`](../Objects/genobject.c), `_PyEval_EvalFrame()` is called +to continue executing the generator function, in the frame that is embedded in +the generator object. + +When a generator object is destroyed in [`gen_dealloc`](../Objects/genobject.c), +its embedded `_PyInterpreterFrame` field may need to be preserved, if it is exposed +to Python as part of a [`PyFrameObject`](frames.md#frame-objects). This is detected +in [`_PyFrame_ClearExceptCode`](../Python/frame.c) by the fact that the interpreter +frame's `frame_obj` field is set, and the frame object it points to has refcount +greater than 1. If so, the `take_ownership()` function is called to create a new +copy of the interpreter frame and transfer ownership of it from the generator to +the frame object. + +Iteration +--------- + +The [`FOR_ITER`](https://docs.python.org/dev/library/dis.html#opcode-FOR_ITER) +instruction calls `__next__` on the iterator which is on the top of the stack, +and pushes the result to the stack. It has [`specializations`](adaptive.md) +for a few common iterator types, including `FOR_ITER_GEN`, for iterating over +a generator. `FOR_ITER_GEN` bypasses the call to `__next__`, and instead +directly pushes the generator stack and resumes its execution from the +instruction that follows the last yield. + +Chained Generators +------------------ + +A `yield from` expression creates a generator that efficiently yields the +sequence created by another generator. This is implemented with the +[`SEND` instruction](https://docs.python.org/dev/library/dis.html#opcode-SEND), +which pushes the value of its arg to the stack of the generator's frame, sets +the exception state on this frame, and resumes execution of the chained generator. +On return from `SEND`, the value at the top of the stack is sent back up +the generator chain with a `YIELD_VALUE`. This sequence of `SEND` followed by +`YIELD_VALUE` is repeated in a loop, until a `StopIteration` exception is +raised to indicate that the generator has no more values to emit. + +The [`CLEANUP_THROW`](https://docs.python.org/dev/library/dis.html#opcode-CLEANUP_THROW) +instruction is used to handle exceptions raised from the send-yield loop. +Exceptions of type `StopIteration` is handled, their `value` field hold the +value to be returned by the generator's `close()` function. Any other +exception is re-raised by `CLEANUP_THROW`. + +Coroutines +---------- + +Coroutines are generators that use the value returned from a `yield` expression, +i.e., the argument that was passed to the `.send()` call that resumed it after +it yielded. This makes it possible for data to flow in both directions: from +the generator to the caller via the argument of the `yield` expression, and +from the caller to the generator via the send argument to the `send()` call. +A `yield from` expression passes the `send` argument to the chained generator, +so this data flow works along the chain (see `gen_send_ex2()` in +[`genobject.c`](../Objects/genobject.c)). + +Recall that a generator's `__next__` function simply calls `self.send(None)`, +so all this works the same in generators and coroutines, but only coroutines +use the value of the argument to `send`. diff --git a/InternalDocs/images/python-cyclic-gc-1-new-page.png b/InternalDocs/images/python-cyclic-gc-1-new-page.png new file mode 100644 index 00000000000000..2ddac50f4b5575 Binary files /dev/null and b/InternalDocs/images/python-cyclic-gc-1-new-page.png differ diff --git a/InternalDocs/images/python-cyclic-gc-2-new-page.png b/InternalDocs/images/python-cyclic-gc-2-new-page.png new file mode 100644 index 00000000000000..159aeeb05024a3 Binary files /dev/null and b/InternalDocs/images/python-cyclic-gc-2-new-page.png differ diff --git a/InternalDocs/images/python-cyclic-gc-3-new-page.png b/InternalDocs/images/python-cyclic-gc-3-new-page.png new file mode 100644 index 00000000000000..29fab0498e5b10 Binary files /dev/null and b/InternalDocs/images/python-cyclic-gc-3-new-page.png differ diff --git a/InternalDocs/images/python-cyclic-gc-4-new-page.png b/InternalDocs/images/python-cyclic-gc-4-new-page.png new file mode 100644 index 00000000000000..51a2b1065ea64e Binary files /dev/null and b/InternalDocs/images/python-cyclic-gc-4-new-page.png differ diff --git a/InternalDocs/images/python-cyclic-gc-5-new-page.png b/InternalDocs/images/python-cyclic-gc-5-new-page.png new file mode 100644 index 00000000000000..fe67a6896fe4b0 Binary files /dev/null and b/InternalDocs/images/python-cyclic-gc-5-new-page.png differ diff --git a/InternalDocs/interpreter.md b/InternalDocs/interpreter.md new file mode 100644 index 00000000000000..52702792c6cb7b --- /dev/null +++ b/InternalDocs/interpreter.md @@ -0,0 +1,515 @@ +# The bytecode interpreter + +This document describes the workings and implementation of the bytecode +interpreter, the part of python that executes compiled Python code. Its +entry point is in [Python/ceval.c](../Python/ceval.c). + +At a high level, the interpreter consists of a loop that iterates over the +bytecode instructions, executing each of them via a switch statement that +has a case implementing each opcode. This switch statement is generated +from the instruction definitions in [Python/bytecodes.c](../Python/bytecodes.c) +which are written in [a DSL](../Tools/cases_generator/interpreter_definition.md) +developed for this purpose. + +Recall that the [Python Compiler](compiler.md) produces a [`CodeObject`](code_objects.md), +which contains the bytecode instructions along with static data that is required to execute them, +such as the consts list, variable names, +[exception table](exception_handling.md#format-of-the-exception-table), and so on. + +When the interpreter's +[`PyEval_EvalCode()`](https://docs.python.org/3.14/c-api/veryhigh.html#c.PyEval_EvalCode) +function is called to execute a `CodeObject`, it constructs a [`Frame`](frames.md) and calls +[`_PyEval_EvalFrame()`](https://docs.python.org/3.14/c-api/veryhigh.html#c.PyEval_EvalCode) +to execute the code object in this frame. The frame holds the dynamic state of the +`CodeObject`'s execution, including the instruction pointer, the globals and builtins. +It also has a reference to the `CodeObject` itself. + +In addition to the frame, `_PyEval_EvalFrame()` also receives a +[`Thread State`](https://docs.python.org/3/c-api/init.html#c.PyThreadState) +object, `tstate`, which includes things like the exception state and the +recursion depth. The thread state also provides access to the per-interpreter +state (`tstate->interp`), which has a pointer to the per-runtime (that is, +truly global) state (`tstate->interp->runtime`). + +Finally, `_PyEval_EvalFrame()` receives an integer argument `throwflag` +which, when nonzero, indicates that the interpreter should just raise the current exception +(this is used in the implementation of +[`gen.throw`](https://docs.python.org/3.14/reference/expressions.html#generator.throw). + +By default, [`_PyEval_EvalFrame()`](https://docs.python.org/3.14/c-api/veryhigh.html#c.PyEval_EvalCode) +simply calls [`_PyEval_EvalFrameDefault()`] to execute the frame. However, as per +[`PEP 523`](https://peps.python.org/pep-0523/) this is configurable by setting +`interp->eval_frame`. In the following, we describe the default function, +`_PyEval_EvalFrameDefault()`. + + +## Instruction decoding + +The first task of the interpreter is to decode the bytecode instructions. +Bytecode is stored as an array of 16-bit code units (`_Py_CODEUNIT`). +Each code unit contains an 8-bit `opcode` and an 8-bit argument (`oparg`), both unsigned. +In order to make the bytecode format independent of the machine byte order when stored on disk, +`opcode` is always the first byte and `oparg` is always the second byte. +Macros are used to extract the `opcode` and `oparg` from a code unit +(`_Py_OPCODE(word)` and `_Py_OPARG(word)`). +Some instructions (for example, `NOP` or `POP_TOP`) have no argument -- in this case +we ignore `oparg`. + +A simplified version of the interpreter's main loop looks like this: + +```c + _Py_CODEUNIT *first_instr = code->co_code_adaptive; + _Py_CODEUNIT *next_instr = first_instr; + while (1) { + _Py_CODEUNIT word = *next_instr++; + unsigned char opcode = _Py_OPCODE(word); + unsigned int oparg = _Py_OPARG(word); + switch (opcode) { + // ... A case for each opcode ... + } + } +``` + +This loop iterates over the instructions, decoding each into its `opcode` +and `oparg`, and then executes the switch case that implements this `opcode`. + +The instruction format supports 256 different opcodes, which is sufficient. +However, it also limits `oparg` to 8-bit values, which is too restrictive. +To overcome this, the `EXTENDED_ARG` opcode allows us to prefix any instruction +with one or more additional data bytes, which combine into a larger oparg. +For example, this sequence of code units: + + EXTENDED_ARG 1 + EXTENDED_ARG 0 + LOAD_CONST 2 + +would set `opcode` to `LOAD_CONST` and `oparg` to `65538` (that is, `0x1_00_02`). +The compiler should limit itself to at most three `EXTENDED_ARG` prefixes, to allow the +resulting `oparg` to fit in 32 bits, but the interpreter does not check this. + +In the following, a `code unit` is always two bytes, while an `instruction` is a +sequence of code units consisting of zero to three `EXTENDED_ARG` opcodes followed by +a primary opcode. + +The following loop, to be inserted just above the `switch` statement, will make the above +snippet decode a complete instruction: + +```c + while (opcode == EXTENDED_ARG) { + word = *next_instr++; + opcode = _Py_OPCODE(word); + oparg = (oparg << 8) | _Py_OPARG(word); + } +``` + +For various reasons we'll get to later (mostly efficiency, given that `EXTENDED_ARG` +is rare) the actual code is different. + +## Jumps + +Note that when the `switch` statement is reached, `next_instr` (the "instruction offset") +already points to the next instruction. +Thus, jump instructions can be implemented by manipulating `next_instr`: + +- A jump forward (`JUMP_FORWARD`) sets `next_instr += oparg`. +- A jump backward sets `next_instr -= oparg`. + +## Inline cache entries + +Some (specialized or specializable) instructions have an associated "inline cache". +The inline cache consists of one or more two-byte entries included in the bytecode +array as additional words following the `opcode`/`oparg` pair. +The size of the inline cache for a particular instruction is fixed by its `opcode`. +Moreover, the inline cache size for all instructions in a +[family of specialized/specializable instructions](#Specialization) +(for example, `LOAD_ATTR`, `LOAD_ATTR_SLOT`, `LOAD_ATTR_MODULE`) must all be +the same. Cache entries are reserved by the compiler and initialized with zeros. +Although they are represented by code units, cache entries do not conform to the +`opcode` / `oparg` format. + +If an instruction has an inline cache, the layout of its cache is described in +the instruction's definition in [`Python/bytecodes.c`](../Python/bytecodes.c). +The structs defined in [`pycore_code.h`](../Include/internal/pycore_code.h) +allow us to access the cache by casting `next_instr` to a pointer to the relevant +`struct`. The size of such a `struct` must be independent of the machine +architecture, word size and alignment requirements. For a 32-bit field, the +`struct` should use `_Py_CODEUNIT field[2]`. + +The instruction implementation is responsible for advancing `next_instr` past the inline cache. +For example, if an instruction's inline cache is four bytes (that is, two code units) in size, +the code for the instruction must contain `next_instr += 2;`. +This is equivalent to a relative forward jump by that many code units. +(In the interpreter definition DSL, this is coded as `JUMPBY(n)`, where `n` is the number +of code units to jump, typically given as a named constant.) + +Serializing non-zero cache entries would present a problem because the serialization +(:mod:`marshal`) format must be independent of the machine byte order. + +More information about the use of inline caches can be found in +[PEP 659](https://peps.python.org/pep-0659/#ancillary-data). + +## The evaluation stack + +Most instructions read or write some data in the form of object references (`PyObject *`). +The CPython bytecode interpreter is a stack machine, meaning that its instructions operate +by pushing data onto and popping it off the stack. +The stack forms part of the frame for the code object. Its maximum depth is calculated +by the compiler and stored in the `co_stacksize` field of the code object, so that the +stack can be pre-allocated as a contiguous array of `PyObject*` pointers, when the frame +is created. + +The stack effects of each instruction are also exposed through the +[opcode metadata](../Include/internal/pycore_opcode_metadata.h) through two +functions that report how many stack elements the instructions consumes, +and how many it produces (`_PyOpcode_num_popped` and `_PyOpcode_num_pushed`). +For example, the `BINARY_OP` instruction pops two objects from the stack and pushes the +result back onto the stack. + +The stack grows up in memory; the operation `PUSH(x)` is equivalent to `*stack_pointer++ = x`, +whereas `x = POP()` means `x = *--stack_pointer`. +Overflow and underflow checks are active in debug mode, but are otherwise optimized away. + +At any point during execution, the stack level is knowable based on the instruction pointer +alone, and some properties of each item on the stack are also known. +In particular, only a few instructions may push a `NULL` onto the stack, and the positions +that may be `NULL` are known. +A few other instructions (`GET_ITER`, `FOR_ITER`) push or pop an object that is known to +be an iterator. + +Instruction sequences that do not allow statically knowing the stack depth are deemed illegal; +the bytecode compiler never generates such sequences. +For example, the following sequence is illegal, because it keeps pushing items on the stack: + + LOAD_FAST 0 + JUMP_BACKWARD 2 + +> [!NOTE] +> Do not confuse the evaluation stack with the call stack, which is used to implement calling +> and returning from functions. + +## Error handling + +When the implementation of an opcode raises an exception, it jumps to the +`exception_unwind` label in [Python/ceval.c](../Python/ceval.c). +The exception is then handled as described in the +[`exception handling documentation`](exception_handling.md#handling-exceptions). + +## Python-to-Python calls + +The `_PyEval_EvalFrameDefault()` function is recursive, because sometimes +the interpreter calls some C function that calls back into the interpreter. +In 3.10 and before, this was the case even when a Python function called +another Python function: +The `CALL` opcode would call the `tp_call` dispatch function of the +callee, which would extract the code object, create a new frame for the call +stack, and then call back into the interpreter. This approach is very general +but consumes several C stack frames for each nested Python call, thereby +increasing the risk of an (unrecoverable) C stack overflow. + +Since 3.11, the `CALL` instruction special-cases function objects to "inline" +the call. When a call gets inlined, a new frame gets pushed onto the call +stack and the interpreter "jumps" to the start of the callee's bytecode. +When an inlined callee executes a `RETURN_VALUE` instruction, the frame is +popped off the call stack and the interpreter returns to its caller, +by popping a frame off the call stack and "jumping" to the return address. +There is a flag in the frame (`frame->is_entry`) that indicates whether +the frame was inlined (set if it wasn't). +If `RETURN_VALUE` finds this flag set, it performs the usual cleanup and +returns from `_PyEval_EvalFrameDefault()` altogether, to a C caller. + +A similar check is performed when an unhandled exception occurs. + +## The call stack + +Up through 3.10, the call stack was implemented as a singly-linked list of +[frame objects](frames.md). This was expensive because each call would require a +heap allocation for the stack frame. + +Since 3.11, frames are no longer fully-fledged objects. Instead, a leaner internal +`_PyInterpreterFrame` structure is used, which is allocated using a custom allocator +function (`_PyThreadState_BumpFramePointer()`), which allocates and initializes a +frame structure. Usually a frame allocation is just a pointer bump, which improves +memory locality. + +Sometimes an actual `PyFrameObject` is needed, such as when Python code calls +`sys._getframe()` or an extension module calls +[`PyEval_GetFrame()`](https://docs.python.org/3/c-api/reflection.html#c.PyEval_GetFrame). +In this case we allocate a proper `PyFrameObject` and initialize it from the +`_PyInterpreterFrame`. + +Things get more complicated when generators are involved, since those do not +follow the push/pop model. This includes async functions, which are based on +the same mechanism. A generator object has space for a `_PyInterpreterFrame` +structure, including the variable-size part (used for locals and the eval stack). +When a generator (or async) function is first called, a special opcode +`RETURN_GENERATOR` is executed, which is responsible for creating the +generator object. The generator object's `_PyInterpreterFrame` is initialized +with a copy of the current stack frame. The current stack frame is then popped +off the frame stack and the generator object is returned. +(Details differ depending on the `is_entry` flag.) +When the generator is resumed, the interpreter pushes its `_PyInterpreterFrame` +onto the frame stack and resumes execution. +See also the [generators](generators.md) section. + + + + + +## Introducing a new bytecode instruction + +It is occasionally necessary to add a new opcode in order to implement +a new feature or change the way that existing features are compiled. +This section describes the changes required to do this. + +First, you must choose a name for the bytecode, implement it in +[`Python/bytecodes.c`](../Python/bytecodes.c) and add a documentation +entry in [`Doc/library/dis.rst`](../Doc/library/dis.rst). +Then run `make regen-cases` to assign a number for it (see +[`Include/opcode_ids.h`](../Include/opcode_ids.h)) and regenerate a +number of files with the actual implementation of the bytecode in +[`Python/generated_cases.c.h`](../Python/generated_cases.c.h) and +metadata about it in additional files. + +With a new bytecode you must also change what is called the "magic number" for +.pyc files: bump the value of the variable `MAGIC_NUMBER` in +[`Lib/importlib/_bootstrap_external.py`](../Lib/importlib/_bootstrap_external.py). +Changing this number will lead to all .pyc files with the old `MAGIC_NUMBER` +to be recompiled by the interpreter on import. Whenever `MAGIC_NUMBER` is +changed, the ranges in the `magic_values` array in +[`PC/launcher.c`](../PC/launcher.c) may also need to be updated. Changes to +[`Lib/importlib/_bootstrap_external.py`](../Lib/importlib/_bootstrap_external.py) +will take effect only after running `make regen-importlib`. + +> [!NOTE] +> Running `make regen-importlib` before adding the new bytecode target to +> [`Python/bytecodes.c`](../Python/bytecodes.c) +> (followed by `make regen-cases`) will result in an error. You should only run +> `make regen-importlib` after the new bytecode target has been added. + +> [!NOTE] +> On Windows, running the `./build.bat` script will automatically +> regenerate the required files without requiring additional arguments. + +Finally, you need to introduce the use of the new bytecode. Update +[`Python/codegen.c`](../Python/codegen.c) to emit code with this bytecode. +Optimizations in [`Python/flowgraph.c`](../Python/flowgraph.c) may also +need to be updated. If the new opcode affects a control flow or the block +stack, you may have to update the `frame_setlineno()` function in +[`Objects/frameobject.c`](../Objects/frameobject.c). It may also be necessary +to update [`Lib/dis.py`](../Lib/dis.py) if the new opcode interprets its +argument in a special way (like `FORMAT_VALUE` or `MAKE_FUNCTION`). + +If you make a change here that can affect the output of bytecode that +is already in existence and you do not change the magic number, make +sure to delete your old .py(c|o) files! Even though you will end up changing +the magic number if you change the bytecode, while you are debugging your work +you may be changing the bytecode output without constantly bumping up the +magic number. This can leave you with stale .pyc files that will not be +recreated. +Running `find . -name '*.py[co]' -exec rm -f '{}' +` should delete all .pyc +files you have, forcing new ones to be created and thus allow you test out your +new bytecode properly. Run `make regen-importlib` for updating the +bytecode of frozen importlib files. You have to run `make` again after this +to recompile the generated C files. + +## Specialization + +Bytecode specialization, which was introduced in +[PEP 659](https://peps.python.org/pep-0659/), speeds up program execution by +rewriting instructions based on runtime information. This is done by replacing +a generic instruction with a faster version that works for the case that this +program encounters. Each specializable instruction is responsible for rewriting +itself, using its [inline caches](#inline-cache-entries) for +bookkeeping. + +When an adaptive instruction executes, it may attempt to specialize itself, +depending on the argument and the contents of its cache. This is done +by calling one of the `_Py_Specialize_XXX` functions in +[`Python/specialize.c`](../Python/specialize.c). + + +The specialized instructions are responsible for checking that the special-case +assumptions still apply, and de-optimizing back to the generic version if not. + +## Families of instructions + +A *family* of instructions consists of an adaptive instruction along with the +specialized instructions that it can be replaced by. +It has the following fundamental properties: + +* It corresponds to a single instruction in the code + generated by the bytecode compiler. +* It has a single adaptive instruction that records an execution count and, + at regular intervals, attempts to specialize itself. If not specializing, + it executes the base implementation. +* It has at least one specialized form of the instruction that is tailored + for a particular value or set of values at runtime. +* All members of the family must have the same number of inline cache entries, + to ensure correct execution. + Individual family members do not need to use all of the entries, + but must skip over any unused entries when executing. + +The current implementation also requires the following, +although these are not fundamental and may change: + +* All families use one or more inline cache entries, + the first entry is always the counter. +* All instruction names should start with the name of the adaptive + instruction. +* Specialized forms should have names describing their specialization. + +## Example family + +The `LOAD_GLOBAL` instruction (in [Python/bytecodes.c](../Python/bytecodes.c)) +already has an adaptive family that serves as a relatively simple example. + +The `LOAD_GLOBAL` instruction performs adaptive specialization, +calling `_Py_Specialize_LoadGlobal()` when the counter reaches zero. + +There are two specialized instructions in the family, `LOAD_GLOBAL_MODULE` +which is specialized for global variables in the module, and +`LOAD_GLOBAL_BUILTIN` which is specialized for builtin variables. + +## Performance analysis + +The benefit of a specialization can be assessed with the following formula: +`Tbase/Tadaptive`. + +Where `Tbase` is the mean time to execute the base instruction, +and `Tadaptive` is the mean time to execute the specialized and adaptive forms. + +`Tadaptive = (sum(Ti*Ni) + Tmiss*Nmiss)/(sum(Ni)+Nmiss)` + +`Ti` is the time to execute the `i`th instruction in the family and `Ni` is +the number of times that instruction is executed. +`Tmiss` is the time to process a miss, including de-optimzation +and the time to execute the base instruction. + +The ideal situation is where misses are rare and the specialized +forms are much faster than the base instruction. +`LOAD_GLOBAL` is near ideal, `Nmiss/sum(Ni) ≈ 0`. +In which case we have `Tadaptive ≈ sum(Ti*Ni)`. +Since we can expect the specialized forms `LOAD_GLOBAL_MODULE` and +`LOAD_GLOBAL_BUILTIN` to be much faster than the adaptive base instruction, +we would expect the specialization of `LOAD_GLOBAL` to be profitable. + +## Design considerations + +While `LOAD_GLOBAL` may be ideal, instructions like `LOAD_ATTR` and +`CALL_FUNCTION` are not. For maximum performance we want to keep `Ti` +low for all specialized instructions and `Nmiss` as low as possible. + +Keeping `Nmiss` low means that there should be specializations for almost +all values seen by the base instruction. Keeping `sum(Ti*Ni)` low means +keeping `Ti` low which means minimizing branches and dependent memory +accesses (pointer chasing). These two objectives may be in conflict, +requiring judgement and experimentation to design the family of instructions. + +The size of the inline cache should as small as possible, +without impairing performance, to reduce the number of +`EXTENDED_ARG` jumps, and to reduce pressure on the CPU's data cache. + +### Gathering data + +Before choosing how to specialize an instruction, it is important to gather +some data. What are the patterns of usage of the base instruction? +Data can best be gathered by instrumenting the interpreter. Since a +specialization function and adaptive instruction are going to be required, +instrumentation can most easily be added in the specialization function. + +### Choice of specializations + +The performance of the specializing adaptive interpreter relies on the +quality of specialization and keeping the overhead of specialization low. + +Specialized instructions must be fast. In order to be fast, +specialized instructions should be tailored for a particular +set of values that allows them to: + +1. Verify that incoming value is part of that set with low overhead. +2. Perform the operation quickly. + +This requires that the set of values is chosen such that membership can be +tested quickly and that membership is sufficient to allow the operation to be +performed quickly. + +For example, `LOAD_GLOBAL_MODULE` is specialized for `globals()` +dictionaries that have a keys with the expected version. + +This can be tested quickly: + +* `globals->keys->dk_version == expected_version` + +and the operation can be performed quickly: + +* `value = entries[cache->index].me_value;`. + +Because it is impossible to measure the performance of an instruction without +also measuring unrelated factors, the assessment of the quality of a +specialization will require some judgement. + +As a general rule, specialized instructions should be much faster than the +base instruction. + +### Implementation of specialized instructions + +In general, specialized instructions should be implemented in two parts: + +1. A sequence of guards, each of the form + `DEOPT_IF(guard-condition-is-false, BASE_NAME)`. +2. The operation, which should ideally have no branches and + a minimum number of dependent memory accesses. + +In practice, the parts may overlap, as data required for guards +can be re-used in the operation. + +If there are branches in the operation, then consider further specialization +to eliminate the branches. + +### Maintaining stats + +Finally, take care that stats are gathered correctly. +After the last `DEOPT_IF` has passed, a hit should be recorded with +`STAT_INC(BASE_INSTRUCTION, hit)`. +After an optimization has been deferred in the adaptive instruction, +that should be recorded with `STAT_INC(BASE_INSTRUCTION, deferred)`. + + +Additional resources +-------------------- + +* Brandt Bucher's talk about the specializing interpreter at PyCon US 2023. + [Slides](https://github.com/brandtbucher/brandtbucher/blob/master/2023/04/21/inside_cpython_311s_new_specializing_adaptive_interpreter.pdf) + [Video](https://www.youtube.com/watch?v=PGZPSWZSkJI&t=1470s) diff --git a/InternalDocs/jit.md b/InternalDocs/jit.md new file mode 100644 index 00000000000000..2c204f39792d6a --- /dev/null +++ b/InternalDocs/jit.md @@ -0,0 +1,130 @@ +# The JIT + +The [adaptive interpreter](interpreter.md) consists of a main loop that +executes the bytecode instructions generated by the +[bytecode compiler](compiler.md) and their +[specializations](interpreter.md#Specialization). Runtime optimization in +this interpreter can only be done for one instruction at a time. The JIT +is based on a mechanism to replace an entire sequence of bytecode instructions, +and this enables optimizations that span multiple instructions. + +Historically, the adaptive interpreter was referred to as `tier 1` and +the JIT as `tier 2`. You will see remnants of this in the code. + +## The Optimizer and Executors + +The program begins running on the adaptive interpreter, until a `JUMP_BACKWARD` +instruction determines that it is "hot" because the counter in its +[inline cache](interpreter.md#inline-cache-entries) indicates that it +executed more than some threshold number of times (see +[`backoff_counter_triggers`](../Include/internal/pycore_backoff.h)). +It then calls the function `_PyOptimizer_Optimize()` in +[`Python/optimizer.c`](../Python/optimizer.c), passing it the current +[frame](frames.md) and instruction pointer. `_PyOptimizer_Optimize()` +constructs an object of type +[`_PyExecutorObject`](Include/internal/pycore_optimizer.h) which implements +an optimized version of the instruction trace beginning at this jump. + +The optimizer determines where the trace ends, and the executor is set up +to either return to the adaptive interpreter and resume execution, or +transfer control to another executor (see `_PyExitData` in +Include/internal/pycore_optimizer.h). + +The executor is stored on the [`code object`](code_objects.md) of the frame, +in the `co_executors` field which is an array of executors. The start +instruction of the trace (the `JUMP_BACKWARD`) is replaced by an +`ENTER_EXECUTOR` instruction whose `oparg` is equal to the index of the +executor in `co_executors`. + +## The micro-op optimizer + +The micro-op (abbreviated `uop` to approximate `μop`) optimizer is defined in +[`Python/optimizer.c`](../Python/optimizer.c) as `_PyOptimizer_Optimize`. +It translates an instruction trace into a sequence of micro-ops by replacing +each bytecode by an equivalent sequence of micro-ops (see +`_PyOpcode_macro_expansion` in +[pycore_opcode_metadata.h](../Include/internal/pycore_opcode_metadata.h) +which is generated from [`Python/bytecodes.c`](../Python/bytecodes.c)). +The micro-op sequence is then optimized by +`_Py_uop_analyze_and_optimize` in +[`Python/optimizer_analysis.c`](../Python/optimizer_analysis.c) +and an instance of `_PyUOpExecutor_Type` is created to contain it. + +## The JIT interpreter + +After a `JUMP_BACKWARD` instruction invokes the uop optimizer to create a uop +executor, it transfers control to this executor via the `GOTO_TIER_TWO` macro. + +CPython implements two executors. Here we describe the JIT interpreter, +which is the simpler of them and is therefore useful for debugging and analyzing +the uops generation and optimization stages. To run it, we configure the +JIT to run on its interpreter (i.e., python is configured with +[`--enable-experimental-jit=interpreter`](https://docs.python.org/dev/using/configure.html#cmdoption-enable-experimental-jit)). + +When invoked, the executor jumps to the `tier2_dispatch:` label in +[`Python/ceval.c`](../Python/ceval.c), where there is a loop that +executes the micro-ops. The body of this loop is a switch statement over +the uops IDs, resembling the one used in the adaptive interpreter. + +The swtich implementing the uops is in [`Python/executor_cases.c.h`](../Python/executor_cases.c.h), +which is generated by the build script +[`Tools/cases_generator/tier2_generator.py`](../Tools/cases_generator/tier2_generator.py) +from the bytecode definitions in +[`Python/bytecodes.c`](../Python/bytecodes.c). + +When an `_EXIT_TRACE` or `_DEOPT` uop is reached, the uop interpreter exits +and execution returns to the adaptive interpreter. + +## Invalidating Executors + +In addition to being stored on the code object, each executor is also +inserted into a list of all executors, which is stored in the interpreter +state's `executor_list_head` field. This list is used when it is necessary +to invalidate executors because values they used in their construction may +have changed. + +## The JIT + +When the full jit is enabled (python was configured with +[`--enable-experimental-jit`](https://docs.python.org/dev/using/configure.html#cmdoption-enable-experimental-jit), +the uop executor's `jit_code` field is populated with a pointer to a compiled +C function that implements the executor logic. This function's signature is +defined by `jit_func` in [`pycore_jit.h`](Include/internal/pycore_jit.h). +When the executor is invoked by `ENTER_EXECUTOR`, instead of jumping to +the uop interpreter at `tier2_dispatch`, the executor runs the function +that `jit_code` points to. This function returns the instruction pointer +of the next Tier 1 instruction that needs to execute. + +The generation of the jitted functions uses the copy-and-patch technique +which is described in +[Haoran Xu's article](https://sillycross.github.io/2023/05/12/2023-05-12/). +At its core are statically generated `stencils` for the implementation +of the micro ops, which are completed with runtime information while +the jitted code is constructed for an executor by +[`_PyJIT_Compile`](../Python/jit.c). + +The stencils are generated at build time under the Makefile target `regen-jit` +by the scripts in [`/Tools/jit`](/Tools/jit). This script reads +[`Python/executor_cases.c.h`](../Python/executor_cases.c.h) (which is +generated from [`Python/bytecodes.c`](../Python/bytecodes.c)). For +each opcode, it constructs a `.c` file that contains a function for +implementing this opcode, with some runtime information injected. +This is done by replacing `CASE` by the bytecode definition in the +template file [`Tools/jit/template.c`](../Tools/jit/template.c). + +Each of the `.c` files is compiled by LLVM, to produce an object file +that contains a function that executes the opcode. These compiled +functions are used to generate the file +[`jit_stencils.h`](../jit_stencils.h), which contains the functions +that the JIT can use to emit code for each of the bytecodes. + +For Python maintainers this means that changes to the bytecodes and +their implementations do not require changes related to the stencils, +because everything is automatically generated from +[`Python/bytecodes.c`](../Python/bytecodes.c) at build time. + +See Also: + +* [Copy-and-Patch Compilation: A fast compilation algorithm for high-level languages and bytecode](https://arxiv.org/abs/2011.13127) + +* [PyCon 2024: Building a JIT compiler for CPython](https://www.youtube.com/watch?v=kMO3Ju0QCDo) diff --git a/InternalDocs/parser.md b/InternalDocs/parser.md new file mode 100644 index 00000000000000..be47efe24356d4 --- /dev/null +++ b/InternalDocs/parser.md @@ -0,0 +1,881 @@ +Guide to the parser +=================== + +Abstract +-------- + +Python's Parser is currently a +[`PEG` (Parser Expression Grammar)](https://en.wikipedia.org/wiki/Parsing_expression_grammar) +parser. It was introduced in +[PEP 617: New PEG parser for CPython](https://peps.python.org/pep-0617/) to replace +the original [`LL(1)`](https://en.wikipedia.org/wiki/LL_parser) parser. + +The code implementing the parser is generated from a grammar definition by a +[parser generator](https://en.wikipedia.org/wiki/Compiler-compiler). +Therefore, changes to the Python language are made by modifying the +[grammar file](../Grammar/python.gram). +Developers rarely need to modify the generator itself. + +See [Changing CPython's grammar](changing_grammar.md) +for a detailed description of the grammar and the process for changing it. + +How PEG parsers work +==================== + +A PEG (Parsing Expression Grammar) grammar differs from a +[context-free grammar](https://en.wikipedia.org/wiki/Context-free_grammar) +in that the way it is written more closely reflects how the parser will operate +when parsing. The fundamental technical difference is that the choice operator +is ordered. This means that when writing: + +``` + rule: A | B | C +``` + +a parser that implements a context-free-grammar (such as an `LL(1)` parser) will +generate constructions that, given an input string, *deduce* which alternative +(`A`, `B` or `C`) must be expanded. On the other hand, a PEG parser will +check each alternative, in the order in which they are specified, and select +that first one that succeeds. + +This means that in a PEG grammar, the choice operator is not commutative. +Furthermore, unlike context-free grammars, the derivation according to a +PEG grammar cannot be ambiguous: if a string parses, it has exactly one +valid parse tree. + +PEG parsers are usually constructed as a recursive descent parser in which every +rule in the grammar corresponds to a function in the program implementing the +parser, and the parsing expression (the "expansion" or "definition" of the rule) +represents the "code" in said function. Each parsing function conceptually takes +an input string as its argument, and yields one of the following results: + +* A "success" result. This result indicates that the expression can be parsed by + that rule and the function may optionally move forward or consume one or more + characters of the input string supplied to it. +* A "failure" result, in which case no input is consumed. + +Note that "failure" results do not imply that the program is incorrect, nor do +they necessarily mean that the parsing has failed. Since the choice operator is +ordered, a failure very often merely indicates "try the following option". A +direct implementation of a PEG parser as a recursive descent parser will present +exponential time performance in the worst case, because PEG parsers have +infinite lookahead (this means that they can consider an arbitrary number of +tokens before deciding for a rule). Usually, PEG parsers avoid this exponential +time complexity with a technique called +["packrat parsing"](https://pdos.csail.mit.edu/~baford/packrat/thesis/) +which not only loads the entire program in memory before parsing it but also +allows the parser to backtrack arbitrarily. This is made efficient by memoizing +the rules already matched for each position. The cost of the memoization cache +is that the parser will naturally use more memory than a simple `LL(1)` parser, +which normally are table-based. + + +Key ideas +--------- + +- Alternatives are ordered ( `A | B` is not the same as `B | A` ). +- If a rule returns a failure, it doesn't mean that the parsing has failed, + it just means "try something else". +- By default PEG parsers run in exponential time, which can be optimized to linear by + using memoization. +- If parsing fails completely (no rule succeeds in parsing all the input text), the + PEG parser doesn't have a concept of "where the + [`SyntaxError`](https://docs.python.org/3/library/exceptions.html#SyntaxError) is". + + +> [!IMPORTANT] +> Don't try to reason about a PEG grammar in the same way you would to with an +> [EBNF](https://en.wikipedia.org/wiki/Extended_Backus–Naur_form) +> or context free grammar. PEG is optimized to describe **how** input strings will +> be parsed, while context-free grammars are optimized to generate strings of the +> language they describe (in EBNF, to know whether a given string is in the +> language, you need to do work to find out as it is not immediately obvious from +> the grammar). + + +Consequences of the ordered choice operator +------------------------------------------- + +Although PEG may look like EBNF, its meaning is quite different. The fact +that the alternatives are ordered in a PEG grammer (which is at the core of +how PEG parsers work) has deep consequences, other than removing ambiguity. + +If a rule has two alternatives and the first of them succeeds, the second one is +**not** attempted even if the caller rule fails to parse the rest of the input. +Thus the parser is said to be "eager". To illustrate this, consider +the following two rules (in these examples, a token is an individual character): + +``` + first_rule: ( 'a' | 'aa' ) 'a' + second_rule: ('aa' | 'a' ) 'a' +``` + +In a regular EBNF grammar, both rules specify the language `{aa, aaa}` but +in PEG, one of these two rules accepts the string `aaa` but not the string +`aa`. The other does the opposite -- it accepts the string `aa` +but not the string `aaa`. The rule `('a'|'aa')'a'` does +not accept `aaa` because `'a'|'aa'` consumes the first `a`, letting the +final `a` in the rule consume the second, and leaving out the third `a`. +As the rule has succeeded, no attempt is ever made to go back and let +`'a'|'aa'` try the second alternative. The expression `('aa'|'a')'a'` does +not accept `aa` because `'aa'|'a'` accepts all of `aa`, leaving nothing +for the final `a`. Again, the second alternative of `'aa'|'a'` is not +tried. + +> [!CAUTION] +> The effects of ordered choice, such as the ones illustrated above, may be +> hidden by many levels of rules. + +For this reason, writing rules where an alternative is contained in the next +one is in almost all cases a mistake, for example: + +``` + my_rule: + | 'if' expression 'then' block + | 'if' expression 'then' block 'else' block +``` + +In this example, the second alternative will never be tried because the first one will +succeed first (even if the input string has an `'else' block` that follows). To correctly +write this rule you can simply alter the order: + +``` + my_rule: + | 'if' expression 'then' block 'else' block + | 'if' expression 'then' block +``` + +In this case, if the input string doesn't have an `'else' block`, the first alternative +will fail and the second will be attempted. + +Grammar Syntax +============== + +The grammar consists of a sequence of rules of the form: + +``` + rule_name: expression +``` + +Optionally, a type can be included right after the rule name, which +specifies the return type of the C or Python function corresponding to +the rule: + +``` + rule_name[return_type]: expression +``` + +If the return type is omitted, then a `void *` is returned in C and an +`Any` in Python. + +Grammar expressions +------------------- + +| Expression | Description and Example | +|-----------------|-----------------------------------------------------------------------------------------------------------------------| +| `# comment` | Python-style comments. | +| `e1 e2` | Match `e1`, then match `e2`.
`rule_name: first_rule second_rule` | +| `e1 \| e2` | Match `e1` or `e2`.
`rule_name[return_type]:`
` \| first_alt`
` \| second_alt` | +| `( e )` | Grouping operator: Match `e`.
`rule_name: (e)`
`rule_name: (e1 e2)*` | +| `[ e ]` or `e?` | Optionally match `e`.
`rule_name: [e]`
`rule_name: e (',' e)* [',']` | +| `e*` | Match zero or more occurrences of `e`.
`rule_name: (e1 e2)*` | +| `e+` | Match one or more occurrences of `e`.
`rule_name: (e1 e2)+` | +| `s.e+` | Match one or more occurrences of `e`, separated by `s`.
`rule_name: ','.e+` | +| `&e` | Positive lookahead: Succeed if `e` can be parsed, without consuming input. | +| `!e` | Negative lookahead: Fail if `e` can be parsed, without consuming input.
`primary: atom !'.' !'(' !'['` | +| `~` | Commit to the current alternative, even if it fails to parse (cut).
`rule_name: '(' ~ some_rule ')' \| some_alt` | + + +Left recursion +-------------- + +PEG parsers normally do not support left recursion, but CPython's parser +generator implements a technique similar to the one described in +[Medeiros et al.](https://arxiv.org/pdf/1207.0443) but using the memoization +cache instead of static variables. This approach is closer to the one described +in [Warth et al.](http://web.cs.ucla.edu/~todd/research/pepm08.pdf). This +allows us to write not only simple left-recursive rules but also more +complicated rules that involve indirect left-recursion like: + +``` + rule1: rule2 | 'a' + rule2: rule3 | 'b' + rule3: rule1 | 'c' +``` + +and "hidden left-recursion" like: + +``` + rule: 'optional'? rule '@' some_other_rule +``` + +Variables in the grammar +------------------------ + +A sub-expression can be named by preceding it with an identifier and an +`=` sign. The name can then be used in the action (see below), like this: + +``` + rule_name[return_type]: '(' a=some_other_rule ')' { a } +``` + +Grammar actions +--------------- + +To avoid the intermediate steps that obscure the relationship between the +grammar and the AST generation, the PEG parser allows directly generating AST +nodes for a rule via grammar actions. Grammar actions are language-specific +expressions that are evaluated when a grammar rule is successfully parsed. These +expressions can be written in Python or C depending on the desired output of the +parser generator. This means that if one would want to generate a parser in +Python and another in C, two grammar files should be written, each one with a +different set of actions, keeping everything else apart from said actions +identical in both files. As an example of a grammar with Python actions, the +piece of the parser generator that parses grammar files is bootstrapped from a +meta-grammar file with Python actions that generate the grammar tree as a result +of the parsing. + +In the specific case of the PEG grammar for Python, having actions allows +directly describing how the AST is composed in the grammar itself, making it +more clear and maintainable. This AST generation process is supported by the use +of some helper functions that factor out common AST object manipulations and +some other required operations that are not directly related to the grammar. + +To indicate these actions each alternative can be followed by the action code +inside curly-braces, which specifies the return value of the alternative: + +``` + rule_name[return_type]: + | first_alt1 first_alt2 { first_alt1 } + | second_alt1 second_alt2 { second_alt1 } +``` + +If the action is omitted, a default action is generated: + +- If there is a single name in the rule, it gets returned. +- If there are multiple names in the rule, a collection with all parsed + expressions gets returned (the type of the collection will be different + in C and Python). + +This default behaviour is primarily made for very simple situations and for +debugging purposes. + +> [!WARNING] +> It's important that the actions don't mutate any AST nodes that are passed +> into them via variables referring to other rules. The reason for mutation +> being not allowed is that the AST nodes are cached by memoization and could +> potentially be reused in a different context, where the mutation would be +> invalid. If an action needs to change an AST node, it should instead make a +> new copy of the node and change that. + +The full meta-grammar for the grammars supported by the PEG generator is: + +``` + start[Grammar]: grammar ENDMARKER { grammar } + + grammar[Grammar]: + | metas rules { Grammar(rules, metas) } + | rules { Grammar(rules, []) } + + metas[MetaList]: + | meta metas { [meta] + metas } + | meta { [meta] } + + meta[MetaTuple]: + | "@" NAME NEWLINE { (name.string, None) } + | "@" a=NAME b=NAME NEWLINE { (a.string, b.string) } + | "@" NAME STRING NEWLINE { (name.string, literal_eval(string.string)) } + + rules[RuleList]: + | rule rules { [rule] + rules } + | rule { [rule] } + + rule[Rule]: + | rulename ":" alts NEWLINE INDENT more_alts DEDENT { + Rule(rulename[0], rulename[1], Rhs(alts.alts + more_alts.alts)) } + | rulename ":" NEWLINE INDENT more_alts DEDENT { Rule(rulename[0], rulename[1], more_alts) } + | rulename ":" alts NEWLINE { Rule(rulename[0], rulename[1], alts) } + + rulename[RuleName]: + | NAME '[' type=NAME '*' ']' {(name.string, type.string+"*")} + | NAME '[' type=NAME ']' {(name.string, type.string)} + | NAME {(name.string, None)} + + alts[Rhs]: + | alt "|" alts { Rhs([alt] + alts.alts)} + | alt { Rhs([alt]) } + + more_alts[Rhs]: + | "|" alts NEWLINE more_alts { Rhs(alts.alts + more_alts.alts) } + | "|" alts NEWLINE { Rhs(alts.alts) } + + alt[Alt]: + | items '$' action { Alt(items + [NamedItem(None, NameLeaf('ENDMARKER'))], action=action) } + | items '$' { Alt(items + [NamedItem(None, NameLeaf('ENDMARKER'))], action=None) } + | items action { Alt(items, action=action) } + | items { Alt(items, action=None) } + + items[NamedItemList]: + | named_item items { [named_item] + items } + | named_item { [named_item] } + + named_item[NamedItem]: + | NAME '=' ~ item {NamedItem(name.string, item)} + | item {NamedItem(None, item)} + | it=lookahead {NamedItem(None, it)} + + lookahead[LookaheadOrCut]: + | '&' ~ atom {PositiveLookahead(atom)} + | '!' ~ atom {NegativeLookahead(atom)} + | '~' {Cut()} + + item[Item]: + | '[' ~ alts ']' {Opt(alts)} + | atom '?' {Opt(atom)} + | atom '*' {Repeat0(atom)} + | atom '+' {Repeat1(atom)} + | sep=atom '.' node=atom '+' {Gather(sep, node)} + | atom {atom} + + atom[Plain]: + | '(' ~ alts ')' {Group(alts)} + | NAME {NameLeaf(name.string) } + | STRING {StringLeaf(string.string)} + + # Mini-grammar for the actions + + action[str]: "{" ~ target_atoms "}" { target_atoms } + + target_atoms[str]: + | target_atom target_atoms { target_atom + " " + target_atoms } + | target_atom { target_atom } + + target_atom[str]: + | "{" ~ target_atoms "}" { "{" + target_atoms + "}" } + | NAME { name.string } + | NUMBER { number.string } + | STRING { string.string } + | "?" { "?" } + | ":" { ":" } +``` + +As an illustrative example this simple grammar file allows directly +generating a full parser that can parse simple arithmetic expressions and that +returns a valid C-based Python AST: + +``` + start[mod_ty]: a=expr_stmt* ENDMARKER { _PyAST_Module(a, NULL, p->arena) } + expr_stmt[stmt_ty]: a=expr NEWLINE { _PyAST_Expr(a, EXTRA) } + + expr[expr_ty]: + | l=expr '+' r=term { _PyAST_BinOp(l, Add, r, EXTRA) } + | l=expr '-' r=term { _PyAST_BinOp(l, Sub, r, EXTRA) } + | term + + term[expr_ty]: + | l=term '*' r=factor { _PyAST_BinOp(l, Mult, r, EXTRA) } + | l=term '/' r=factor { _PyAST_BinOp(l, Div, r, EXTRA) } + | factor + + factor[expr_ty]: + | '(' e=expr ')' { e } + | atom + + atom[expr_ty]: + | NAME + | NUMBER +``` + +Here `EXTRA` is a macro that expands to `start_lineno, start_col_offset, +end_lineno, end_col_offset, p->arena`, those being variables automatically +injected by the parser; `p` points to an object that holds on to all state +for the parser. + +A similar grammar written to target Python AST objects: + +``` + start[ast.Module]: a=expr_stmt* ENDMARKER { ast.Module(body=a or [] } + expr_stmt: a=expr NEWLINE { ast.Expr(value=a, EXTRA) } + + expr: + | l=expr '+' r=term { ast.BinOp(left=l, op=ast.Add(), right=r, EXTRA) } + | l=expr '-' r=term { ast.BinOp(left=l, op=ast.Sub(), right=r, EXTRA) } + | term + + term: + | l=term '*' r=factor { ast.BinOp(left=l, op=ast.Mult(), right=r, EXTRA) } + | l=term '/' r=factor { ast.BinOp(left=l, op=ast.Div(), right=r, EXTRA) } + | factor + + factor: + | '(' e=expr ')' { e } + | atom + + atom: + | NAME + | NUMBER +``` + +Pegen +===== + +Pegen is the parser generator used in CPython to produce the final PEG parser +used by the interpreter. It is the program that can be used to read the python +grammar located in [`Grammar/python.gram`](../Grammar/python.gram) and produce +the final C parser. It contains the following pieces: + +- A parser generator that can read a grammar file and produce a PEG parser + written in Python or C that can parse said grammar. The generator is located at + [`Tools/peg_generator/pegen`](../Tools/peg_generator/pegen). +- A PEG meta-grammar that automatically generates a Python parser which is used + for the parser generator itself (this means that there are no manually-written + parsers). The meta-grammar is located at + [`Tools/peg_generator/pegen/metagrammar.gram`](../Tools/peg_generator/pegen/metagrammar.gram). +- A generated parser (using the parser generator) that can directly produce C and Python AST objects. + +The source code for Pegen lives at [`Tools/peg_generator/pegen`](../Tools/peg_generator/pegen) +but normally all typical commands to interact with the parser generator are executed from +the main makefile. + +How to regenerate the parser +---------------------------- + +Once you have made the changes to the grammar files, to regenerate the `C` +parser (the one used by the interpreter) just execute: + +```shell +$ make regen-pegen +``` + +using the `Makefile` in the main directory. If you are on Windows you can +use the Visual Studio project files to regenerate the parser or to execute: + +```dos +PCbuild/build.bat --regen +``` + +The generated parser file is located at [`Parser/parser.c`](../Parser/parser.c). + +How to regenerate the meta-parser +--------------------------------- + +The meta-grammar (the grammar that describes the grammar for the grammar files +themselves) is located at +[`Tools/peg_generator/pegen/metagrammar.gram`](../Tools/peg_generator/pegen/metagrammar.gram). +Although it is very unlikely that you will ever need to modify it, if you make +any modifications to this file (in order to implement new Pegen features) you will +need to regenerate the meta-parser (the parser that parses the grammar files). +To do so just execute: + +```shell +$ make regen-pegen-metaparser +``` + +If you are on Windows you can use the Visual Studio project files +to regenerate the parser or to execute: + +```dos +PCbuild/build.bat --regen +``` + + +Grammatical elements and rules +------------------------------ + +Pegen has some special grammatical elements and rules: + +- Strings with single quotes (') (for example, `'class'`) denote KEYWORDS. +- Strings with double quotes (") (for example, `"match"`) denote SOFT KEYWORDS. +- Uppercase names (for example, `NAME`) denote tokens in the + [`Grammar/Tokens`](../Grammar/Tokens) file. +- Rule names starting with `invalid_` are used for specialized syntax errors. + + - These rules are NOT used in the first pass of the parser. + - Only if the first pass fails to parse, a second pass including the invalid + rules will be executed. + - If the parser fails in the second phase with a generic syntax error, the + location of the generic failure of the first pass will be used (this avoids + reporting incorrect locations due to the invalid rules). + - The order of the alternatives involving invalid rules matter + (like any rule in PEG). + +Tokenization +------------ + +It is common among PEG parser frameworks that the parser does both the parsing +and the tokenization, but this does not happen in Pegen. The reason is that the +Python language needs a custom tokenizer to handle things like indentation +boundaries, some special keywords like `ASYNC` and `AWAIT` (for +compatibility purposes), backtracking errors (such as unclosed parenthesis), +dealing with encoding, interactive mode and much more. Some of these reasons +are also there for historical purposes, and some others are useful even today. + +The list of tokens (all uppercase names in the grammar) that you can use can +be found in the [`Grammar/Tokens`](../Grammar/Tokens) +file. If you change this file to add new tokens, make sure to regenerate the +files by executing: + +```shell +$ make regen-token +``` + +If you are on Windows you can use the Visual Studio project files to regenerate +the tokens or to execute: + +```dos +PCbuild/build.bat --regen +``` + +How tokens are generated and the rules governing this are completely up to the tokenizer +([`Parser/lexer`](../Parser/lexer) and [`Parser/tokenizer`](../Parser/tokenizer)); +the parser just receives tokens from it. + +Memoization +----------- + +As described previously, to avoid exponential time complexity in the parser, +memoization is used. + +The C parser used by Python is highly optimized and memoization can be expensive +both in memory and time. Although the memory cost is obvious (the parser needs +memory for storing previous results in the cache) the execution time cost comes +from continuously checking if the given rule has a cache hit or not. In many +situations, just parsing it again can be faster. Pegen **disables memoization +by default** except for rules with the special marker `memo` after the rule +name (and type, if present): + +``` +rule_name[typr] (memo): + ... +``` + +By selectively turning on memoization for a handful of rules, the parser becomes +faster and uses less memory. + +> [!NOTE] +> Left-recursive rules always use memoization, since the implementation of +> left-recursion depends on it. + +To determine whether a new rule needs memoization or not, benchmarking is required +(comparing execution times and memory usage of some considerably large files with +and without memoization). There is a very simple instrumentation API available +in the generated C parse code that allows to measure how much each rule uses +memoization (check the [`Parser/pegen.c`](../Parser/pegen.c) +file for more information) but it needs to be manually activated. + +Automatic variables +------------------- + +To make writing actions easier, Pegen injects some automatic variables in the +namespace available when writing actions. In the C parser, some of these +automatic variable names are: + +- `p`: The parser structure. +- `EXTRA`: This is a macro that expands to + `(_start_lineno, _start_col_offset, _end_lineno, _end_col_offset, p->arena)`, + which is normally used to create AST nodes as almost all constructors need these + attributes to be provided. All of the location variables are taken from the + location information of the current token. + +Hard and soft keywords +---------------------- + +> [!NOTE] +> In the grammar files, keywords are defined using **single quotes** (for example, +> `'class'`) while soft keywords are defined using **double quotes** (for example, +> `"match"`). + +There are two kinds of keywords allowed in pegen grammars: *hard* and *soft* +keywords. The difference between hard and soft keywords is that hard keywords +are always reserved words, even in positions where they make no sense +(for example, `x = class + 1`), while soft keywords only get a special +meaning in context. Trying to use a hard keyword as a variable will always +fail: + +```pycon +>>> class = 3 +File "", line 1 + class = 3 + ^ +SyntaxError: invalid syntax +>>> foo(class=3) +File "", line 1 + foo(class=3) + ^^^^^ +SyntaxError: invalid syntax +``` + +While soft keywords don't have this limitation if used in a context other than +one where they are defined as keywords: + +```pycon +>>> match = 45 +>>> foo(match="Yeah!") +``` + +The `match` and `case` keywords are soft keywords, so that they are +recognized as keywords at the beginning of a match statement or case block +respectively, but are allowed to be used in other places as variable or +argument names. + +You can get a list of all keywords defined in the grammar from Python: + +```pycon +>>> import keyword +>>> keyword.kwlist +['False', 'None', 'True', 'and', 'as', 'assert', 'async', 'await', 'break', +'class', 'continue', 'def', 'del', 'elif', 'else', 'except', 'finally', 'for', +'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'nonlocal', 'not', 'or', +'pass', 'raise', 'return', 'try', 'while', 'with', 'yield'] +``` + +as well as soft keywords: + +```pycon +>>> import keyword +>>> keyword.softkwlist +['_', 'case', 'match'] +``` + +> [!CAUTION] +> Soft keywords can be a bit challenging to manage as they can be accepted in +> places you don't intend, given how the order alternatives behave in PEG +> parsers (see the +> [consequences of ordered choice](#consequences-of-the-ordered-choice-operator) +> section for some background on this). In general, try to define them in places +> where there are not many alternatives. + +Error handling +-------------- + +When a pegen-generated parser detects that an exception is raised, it will +**automatically stop parsing**, no matter what the current state of the parser +is, and it will unwind the stack and report the exception. This means that if a +[rule action](#grammar-actions) raises an exception, all parsing will +stop at that exact point. This is done to allow to correctly propagate any +exception set by calling Python's C API functions. This also includes +[`SyntaxError`](https://docs.python.org/3/library/exceptions.html#SyntaxError) +exceptions and it is the main mechanism the parser uses to report custom syntax +error messages. + +> [!NOTE] +> Tokenizer errors are normally reported by raising exceptions but some special +> tokenizer errors such as unclosed parenthesis will be reported only after the +> parser finishes without returning anything. + +How syntax errors are reported +------------------------------ + +As described previously in the [how PEG parsers work](#how-peg-parsers-work) +section, PEG parsers don't have a defined concept of where errors happened +in the grammar, because a rule failure doesn't imply a parsing failure like +in context free grammars. This means that a heuristic has to be used to report +generic errors unless something is explicitly declared as an error in the +grammar. + +To report generic syntax errors, pegen uses a common heuristic in PEG parsers: +the location of *generic* syntax errors is reported to be the furthest token that +was attempted to be matched but failed. This is only done if parsing has failed +(the parser returns `NULL` in C or `None` in Python) but no exception has +been raised. + +As the Python grammar was primordially written as an `LL(1)` grammar, this heuristic +has an extremely high success rate, but some PEG features, such as lookaheads, +can impact this. + +> [!CAUTION] +> Positive and negative lookaheads will try to match a token so they will affect +> the location of generic syntax errors. Use them carefully at boundaries +> between rules. + +To generate more precise syntax errors, custom rules are used. This is a common +practice also in context free grammars: the parser will try to accept some +construct that is known to be incorrect just to report a specific syntax error +for that construct. In pegen grammars, these rules start with the `invalid_` +prefix. This is because trying to match these rules normally has a performance +impact on parsing (and can also affect the 'correct' grammar itself in some +tricky cases, depending on the ordering of the rules) so the generated parser +acts in two phases: + +1. The first phase will try to parse the input stream without taking into + account rules that start with the `invalid_` prefix. If the parsing + succeeds it will return the generated AST and the second phase will be + skipped. + +2. If the first phase failed, a second parsing attempt is done including the + rules that start with an `invalid_` prefix. By design this attempt + **cannot succeed** and is only executed to give to the invalid rules a + chance to detect specific situations where custom, more precise, syntax + errors can be raised. This also allows to trade a bit of performance for + precision reporting errors: given that we know that the input text is + invalid, there is typically no need to be fast because execution is going + to stop anyway. + +> [!IMPORTANT] +> When defining invalid rules: +> +> - Make sure all custom invalid rules raise +> [`SyntaxError`](https://docs.python.org/3/library/exceptions.html#SyntaxError) +> exceptions (or a subclass of it). +> - Make sure **all** invalid rules start with the `invalid_` prefix to not +> impact performance of parsing correct Python code. +> - Make sure the parser doesn't behave differently for regular rules when you introduce invalid rules +> (see the [how PEG parsers work](#how-peg-parsers-work) section for more information). + +You can find a collection of macros to raise specialized syntax errors in the +[`Parser/pegen.h`](../Parser/pegen.h) +header file. These macros allow also to report ranges for +the custom errors, which will be highlighted in the tracebacks that will be +displayed when the error is reported. + + +> [!TIP] +> A good way to test whether an invalid rule will be triggered when you expect +> is to test if introducing a syntax error **after** valid code triggers the +> rule or not. For example: + +``` + $ 42 +``` + +should trigger the syntax error in the `$` character. If your rule is not correctly defined this +won't happen. As another example, suppose that you try to define a rule to match Python 2 style +`print` statements in order to create a better error message and you define it as: + +``` +invalid_print: "print" expression +``` + +This will **seem** to work because the parser will correctly parse `print(something)` because it is valid +code and the second phase will never execute but if you try to parse `print(something) $ 3` the first pass +of the parser will fail (because of the `$`) and in the second phase, the rule will match the +`print(something)` as `print` followed by the variable `something` between parentheses and the error +will be reported there instead of the `$` character. + +Generating AST objects +---------------------- + +The output of the C parser used by CPython, which is generated from the +[grammar file](../Grammar/python.gram), is a Python AST object (using C +structures). This means that the actions in the grammar file generate AST +objects when they succeed. Constructing these objects can be quite cumbersome +(see the [AST compiler section](compiler.md#abstract-syntax-trees-ast) +for more information on how these objects are constructed and how they are used +by the compiler), so special helper functions are used. These functions are +declared in the [`Parser/pegen.h`](../Parser/pegen.h) header file and defined +in the [`Parser/action_helpers.c`](../Parser/action_helpers.c) file. The +helpers include functions that join AST sequences, get specific elements +from them or to perform extra processing on the generated tree. + + +> [!CAUTION] +> Actions must **never** be used to accept or reject rules. It may be tempting +> in some situations to write a very generic rule and then check the generated +> AST to decide whether it is valid or not, but this will render the +> (official grammar)[https://docs.python.org/3/reference/grammar.html] partially +> incorrect (because it does not include actions) and will make it more difficult +> for other Python implementations to adapt the grammar to their own needs. + +As a general rule, if an action spawns multiple lines or requires something more +complicated than a single expression of C code, is normally better to create a +custom helper in [`Parser/action_helpers.c`](../Parser/action_helpers.c) +and expose it in the [`Parser/pegen.h`](../Parser/pegen.h) header file so that +it can be used from the grammar. + +When parsing succeeds, the parser **must** return a **valid** AST object. + +Testing +======= + +There are three files that contain tests for the grammar and the parser: + +- [test_grammar.py](../Lib/test/test_grammar.py) +- [test_syntax.py](../Lib/test/test_syntax.py) +- [test_exceptions.py](../Lib/test/test_exceptions.py) + +Check the contents of these files to know which is the best place for new +tests, depending on the nature of the new feature you are adding. + +Tests for the parser generator itself can be found in the +[test_peg_generator](../Lib/test/test_peg_generator) directory. + + +Debugging generated parsers +=========================== + +Making experiments +------------------ + +As the generated C parser is the one used by Python, this means that if +something goes wrong when adding some new rules to the grammar, you cannot +correctly compile and execute Python anymore. This makes it a bit challenging +to debug when something goes wrong, especially when experimenting. + +For this reason it is a good idea to experiment first by generating a Python +parser. To do this, you can go to the [Tools/peg_generator](../Tools/peg_generator) +directory on the CPython repository and manually call the parser generator by executing: + +```shell +$ python -m pegen python +``` + +This will generate a file called `parse.py` in the same directory that you +can use to parse some input: + +```shell +$ python parse.py file_with_source_code_to_test.py +``` + +As the generated `parse.py` file is just Python code, you can modify it +and add breakpoints to debug or better understand some complex situations. + + +Verbose mode +------------ + +When Python is compiled in debug mode (by adding `--with-pydebug` when +running the configure step in Linux or by adding `-d` when calling the +[PCbuild/build.bat](../PCbuild/build.bat)), it is possible to activate a +**very** verbose mode in the generated parser. This is very useful to +debug the generated parser and to understand how it works, but it +can be a bit hard to understand at first. + +> [!NOTE] +> When activating verbose mode in the Python parser, it is better to not use +> interactive mode as it can be much harder to understand, because interactive +> mode involves some special steps compared to regular parsing. + +To activate verbose mode you can add the `-d` flag when executing Python: + +```shell +$ python -d file_to_test.py +``` + +This will print **a lot** of output to `stderr` so it is probably better to dump +it to a file for further analysis. The output consists of trace lines with the +following structure:: + +``` + ('>'|'-'|'+'|'!') []: ... +``` + +Every line is indented by a different amount (``) depending on how +deep the call stack is. The next character marks the type of the trace: + +- `>` indicates that a rule is going to be attempted to be parsed. +- `-` indicates that a rule has failed to be parsed. +- `+` indicates that a rule has been parsed correctly. +- `!` indicates that an exception or an error has been detected and the parser is unwinding. + +The `` part indicates the current index in the token array, +the `` part indicates what rule is being parsed and +the `` part indicates what alternative within that rule +is being attempted. + + +> [!NOTE] +> **Document history** +> +> Pablo Galindo Salgado - Original author +> +> Irit Katriel and Jacob Coffee - Convert to Markdown diff --git a/InternalDocs/string_interning.md b/InternalDocs/string_interning.md new file mode 100644 index 00000000000000..26a5197c6e70f3 --- /dev/null +++ b/InternalDocs/string_interning.md @@ -0,0 +1,121 @@ +# String interning + +*Interned* strings are conceptually part of an interpreter-global +*set* of interned strings, meaning that: + +- no two interned strings have the same content (across an interpreter); +- two interned strings can be safely compared using pointer equality + (Python `is`). + +This is used to optimize dict and attribute lookups, among other things. + +Python uses two different mechanisms to intern strings: singletons and +dynamic interning. + +## Singletons + +The 256 possible one-character latin-1 strings, which can be retrieved with +`_Py_LATIN1_CHR(c)`, are stored in statically allocated arrays, +`_PyRuntime.static_objects.strings.ascii` and +`_PyRuntime.static_objects.strings.latin1`. + +Longer singleton strings are marked in C source with `_Py_ID` (if the string +is a valid C identifier fragment) or `_Py_STR` (if it needs a separate +C-compatible name.) +These are also stored in statically allocated arrays. +They are collected from CPython sources using `make regen-global-objects` +(`Tools/build/generate_global_objects.py`), which generates code +for declaration, initialization and finalization. + +The empty string is one of the singletons: `_Py_STR(empty)`. + +The three sets of singletons (`_Py_LATIN1_CHR`, `_Py_ID`, `_Py_STR`) +are disjoint. +If you have such a singleton, it (and no other copy) will be interned. + +These singletons are interned in a runtime-global lookup table, +`_PyRuntime.cached_objects.interned_strings` (`INTERNED_STRINGS`), +at runtime initialization, and immutable until it's torn down +at runtime finalization. +It is shared across threads and interpreters without any synchronization. + + +## Dynamically allocated strings + +All other strings are allocated dynamically, and have their +`_PyUnicode_STATE(s).statically_allocated` flag set to zero. +When interned, such strings are added to an interpreter-wide dict, +`PyInterpreterState.cached_objects.interned_strings`. + +The key and value of each entry in this dict reference the same object. + + +## Immortality and reference counting + +Invariant: Every immortal string is interned. + +In practice, this means that you must not use `_Py_SetImmortal` on +a string. (If you know it's already immortal, don't immortalize it; +if you know it's not interned you might be immortalizing a redundant copy; +if it's interned and mortal it needs extra processing in +`_PyUnicode_InternImmortal`.) + +The converse is not true: interned strings can be mortal. +For mortal interned strings: + +- the 2 references from the interned dict (key & value) are excluded from + their refcount +- the deallocator (`unicode_dealloc`) removes the string from the interned dict +- at shutdown, when the interned dict is cleared, the references are added back + +As with any type, you should only immortalize strings that will live until +interpreter shutdown. +We currently also immortalize strings contained in code objects and similar, +specifically in the compiler and in `marshal`. +These are “close enough” to immortal: even in use cases like hot reloading +or `eval`-ing user input, the number of distinct identifiers and string +constants is expected to stay low. + + +## Internal API + +We have the following *internal* API for interning: + +- `_PyUnicode_InternMortal`: just intern the string +- `_PyUnicode_InternImmortal`: intern, and immortalize the result +- `_PyUnicode_InternStatic`: intern a static singleton (`_Py_STR`, `_Py_ID` + or one-byte). Not for general use. + +All take an interpreter state, and a pointer to a `PyObject*` which they +modify in place. + +The functions take ownership of (“steal”) the reference to their argument, +and update the argument with a *new* reference. +This means: + +- They're “reference neutral”. +- They must not be called with a borrowed reference. + + +## State + +The intern state (retrieved by `PyUnicode_CHECK_INTERNED(s)`; +stored in `_PyUnicode_STATE(s).interned`) can be: + +- `SSTATE_NOT_INTERNED` (defined as 0, which is useful in a boolean context) +- `SSTATE_INTERNED_MORTAL` (1) +- `SSTATE_INTERNED_IMMORTAL` (2) +- `SSTATE_INTERNED_IMMORTAL_STATIC` (3) + +The valid transitions between these states are: + +- For dynamically allocated strings: + + - 0 -> 1 (`_PyUnicode_InternMortal`) + - 1 -> 2 or 0 -> 2 (`_PyUnicode_InternImmortal`) + + Using `_PyUnicode_InternStatic` on these is an error; the other cases + don't change the state. + +- Singletons are interned (0 -> 3) at runtime init; + after that all interning functions don't change the state. diff --git a/LICENSE b/LICENSE index 14603b95c2e23b..20cf39097c68ba 100644 --- a/LICENSE +++ b/LICENSE @@ -83,7 +83,7 @@ grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, -i.e., "Copyright (c) 2001-2024 Python Software Foundation; All Rights Reserved" +i.e., "Copyright (c) 2001 Python Software Foundation; All Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee. 3. In the event Licensee prepares a derivative work that is based on diff --git a/Lib/_android_support.py b/Lib/_android_support.py new file mode 100644 index 00000000000000..ae506f6a4b57b8 --- /dev/null +++ b/Lib/_android_support.py @@ -0,0 +1,181 @@ +import io +import sys +from threading import RLock +from time import sleep, time + +# The maximum length of a log message in bytes, including the level marker and +# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD at +# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log.h;l=71. +# Messages longer than this will be truncated by logcat. This limit has already +# been reduced at least once in the history of Android (from 4076 to 4068 between +# API level 23 and 26), so leave some headroom. +MAX_BYTES_PER_WRITE = 4000 + +# UTF-8 uses a maximum of 4 bytes per character, so limiting text writes to this +# size ensures that we can always avoid exceeding MAX_BYTES_PER_WRITE. +# However, if the actual number of bytes per character is smaller than that, +# then we may still join multiple consecutive text writes into binary +# writes containing a larger number of characters. +MAX_CHARS_PER_WRITE = MAX_BYTES_PER_WRITE // 4 + + +# When embedded in an app on current versions of Android, there's no easy way to +# monitor the C-level stdout and stderr. The testbed comes with a .c file to +# redirect them to the system log using a pipe, but that wouldn't be convenient +# or appropriate for all apps. So we redirect at the Python level instead. +def init_streams(android_log_write, stdout_prio, stderr_prio): + if sys.executable: + return # Not embedded in an app. + + global logcat + logcat = Logcat(android_log_write) + + sys.stdout = TextLogStream( + stdout_prio, "python.stdout", sys.stdout.fileno()) + sys.stderr = TextLogStream( + stderr_prio, "python.stderr", sys.stderr.fileno()) + + +class TextLogStream(io.TextIOWrapper): + def __init__(self, prio, tag, fileno=None, **kwargs): + # The default is surrogateescape for stdout and backslashreplace for + # stderr, but in the context of an Android log, readability is more + # important than reversibility. + kwargs.setdefault("encoding", "UTF-8") + kwargs.setdefault("errors", "backslashreplace") + + super().__init__(BinaryLogStream(prio, tag, fileno), **kwargs) + self._lock = RLock() + self._pending_bytes = [] + self._pending_bytes_count = 0 + + def __repr__(self): + return f"" + + def write(self, s): + if not isinstance(s, str): + raise TypeError( + f"write() argument must be str, not {type(s).__name__}") + + # In case `s` is a str subclass that writes itself to stdout or stderr + # when we call its methods, convert it to an actual str. + s = str.__str__(s) + + # We want to emit one log message per line wherever possible, so split + # the string into lines first. Note that "".splitlines() == [], so + # nothing will be logged for an empty string. + with self._lock: + for line in s.splitlines(keepends=True): + while line: + chunk = line[:MAX_CHARS_PER_WRITE] + line = line[MAX_CHARS_PER_WRITE:] + self._write_chunk(chunk) + + return len(s) + + # The size and behavior of TextIOWrapper's buffer is not part of its public + # API, so we handle buffering ourselves to avoid truncation. + def _write_chunk(self, s): + b = s.encode(self.encoding, self.errors) + if self._pending_bytes_count + len(b) > MAX_BYTES_PER_WRITE: + self.flush() + + self._pending_bytes.append(b) + self._pending_bytes_count += len(b) + if ( + self.write_through + or b.endswith(b"\n") + or self._pending_bytes_count > MAX_BYTES_PER_WRITE + ): + self.flush() + + def flush(self): + with self._lock: + self.buffer.write(b"".join(self._pending_bytes)) + self._pending_bytes.clear() + self._pending_bytes_count = 0 + + # Since this is a line-based logging system, line buffering cannot be turned + # off, i.e. a newline always causes a flush. + @property + def line_buffering(self): + return True + + +class BinaryLogStream(io.RawIOBase): + def __init__(self, prio, tag, fileno=None): + self.prio = prio + self.tag = tag + self._fileno = fileno + + def __repr__(self): + return f"" + + def writable(self): + return True + + def write(self, b): + if type(b) is not bytes: + try: + b = bytes(memoryview(b)) + except TypeError: + raise TypeError( + f"write() argument must be bytes-like, not {type(b).__name__}" + ) from None + + # Writing an empty string to the stream should have no effect. + if b: + logcat.write(self.prio, self.tag, b) + return len(b) + + # This is needed by the test suite --timeout option, which uses faulthandler. + def fileno(self): + if self._fileno is None: + raise io.UnsupportedOperation("fileno") + return self._fileno + + +# When a large volume of data is written to logcat at once, e.g. when a test +# module fails in --verbose3 mode, there's a risk of overflowing logcat's own +# buffer and losing messages. We avoid this by imposing a rate limit using the +# token bucket algorithm, based on a conservative estimate of how fast `adb +# logcat` can consume data. +MAX_BYTES_PER_SECOND = 1024 * 1024 + +# The logcat buffer size of a device can be determined by running `logcat -g`. +# We set the token bucket size to half of the buffer size of our current minimum +# API level, because other things on the system will be producing messages as +# well. +BUCKET_SIZE = 128 * 1024 + +# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39 +PER_MESSAGE_OVERHEAD = 28 + + +class Logcat: + def __init__(self, android_log_write): + self.android_log_write = android_log_write + self._lock = RLock() + self._bucket_level = 0 + self._prev_write_time = time() + + def write(self, prio, tag, message): + # Encode null bytes using "modified UTF-8" to avoid them truncating the + # message. + message = message.replace(b"\x00", b"\xc0\x80") + + with self._lock: + now = time() + self._bucket_level += ( + (now - self._prev_write_time) * MAX_BYTES_PER_SECOND) + + # If the bucket level is still below zero, the clock must have gone + # backwards, so reset it to zero and continue. + self._bucket_level = max(0, min(self._bucket_level, BUCKET_SIZE)) + self._prev_write_time = now + + self._bucket_level -= PER_MESSAGE_OVERHEAD + len(tag) + len(message) + if self._bucket_level < 0: + sleep(-self._bucket_level / MAX_BYTES_PER_SECOND) + + self.android_log_write(prio, tag, message) diff --git a/Lib/_apple_support.py b/Lib/_apple_support.py new file mode 100644 index 00000000000000..92febdcf587070 --- /dev/null +++ b/Lib/_apple_support.py @@ -0,0 +1,66 @@ +import io +import sys + + +def init_streams(log_write, stdout_level, stderr_level): + # Redirect stdout and stderr to the Apple system log. This method is + # invoked by init_apple_streams() (initconfig.c) if config->use_system_logger + # is enabled. + sys.stdout = SystemLog(log_write, stdout_level, errors=sys.stderr.errors) + sys.stderr = SystemLog(log_write, stderr_level, errors=sys.stderr.errors) + + +class SystemLog(io.TextIOWrapper): + def __init__(self, log_write, level, **kwargs): + kwargs.setdefault("encoding", "UTF-8") + kwargs.setdefault("line_buffering", True) + super().__init__(LogStream(log_write, level), **kwargs) + + def __repr__(self): + return f"" + + def write(self, s): + if not isinstance(s, str): + raise TypeError( + f"write() argument must be str, not {type(s).__name__}") + + # In case `s` is a str subclass that writes itself to stdout or stderr + # when we call its methods, convert it to an actual str. + s = str.__str__(s) + + # We want to emit one log message per line, so split + # the string before sending it to the superclass. + for line in s.splitlines(keepends=True): + super().write(line) + + return len(s) + + +class LogStream(io.RawIOBase): + def __init__(self, log_write, level): + self.log_write = log_write + self.level = level + + def __repr__(self): + return f"" + + def writable(self): + return True + + def write(self, b): + if type(b) is not bytes: + try: + b = bytes(memoryview(b)) + except TypeError: + raise TypeError( + f"write() argument must be bytes-like, not {type(b).__name__}" + ) from None + + # Writing an empty string to the stream should have no effect. + if b: + # Encode null bytes using "modified UTF-8" to avoid truncating the + # message. This should not affect the return value, as the caller + # may be expecting it to match the length of the input. + self.log_write(self.level, b.replace(b"\x00", b"\xc0\x80")) + + return len(b) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 601107d2d86771..06667b7434ccef 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -49,7 +49,7 @@ def _f(): pass "Mapping", "MutableMapping", "MappingView", "KeysView", "ItemsView", "ValuesView", "Sequence", "MutableSequence", - "ByteString", "Buffer", + "Buffer", ] # This module has been renamed from collections.abc to _collections_abc to @@ -85,6 +85,10 @@ def _f(): pass dict_items = type({}.items()) ## misc ## mappingproxy = type(type.__dict__) +def _get_framelocalsproxy(): + return type(sys._getframe().f_locals) +framelocalsproxy = _get_framelocalsproxy() +del _get_framelocalsproxy generator = type((lambda: (yield))()) ## coroutine ## async def _coro(): pass @@ -481,9 +485,10 @@ def __new__(cls, origin, args): def __repr__(self): if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]): return super().__repr__() + from annotationlib import value_to_string return (f'collections.abc.Callable' - f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' - f'{_type_repr(self.__args__[-1])}]') + f'[[{", ".join([value_to_string(a) for a in self.__args__[:-1]])}], ' + f'{value_to_string(self.__args__[-1])}]') def __reduce__(self): args = self.__args__ @@ -520,23 +525,6 @@ def _is_param_expr(obj): names = ('ParamSpec', '_ConcatenateGenericAlias') return obj.__module__ == 'typing' and any(obj.__name__ == name for name in names) -def _type_repr(obj): - """Return the repr() of an object, special-casing types (internal helper). - - Copied from :mod:`typing` since collections.abc - shouldn't depend on that module. - (Keep this roughly in sync with the typing version.) - """ - if isinstance(obj, type): - if obj.__module__ == 'builtins': - return obj.__qualname__ - return f'{obj.__module__}.{obj.__qualname__}' - if obj is Ellipsis: - return '...' - if isinstance(obj, FunctionType): - return obj.__name__ - return repr(obj) - class Callable(metaclass=ABCMeta): @@ -836,6 +824,7 @@ def __eq__(self, other): __reversed__ = None Mapping.register(mappingproxy) +Mapping.register(framelocalsproxy) class MappingView(Sized): @@ -973,7 +962,7 @@ def clear(self): def update(self, other=(), /, **kwds): ''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F. - If E present and has a .keys() method, does: for k in E: D[k] = E[k] + If E present and has a .keys() method, does: for k in E.keys(): D[k] = E[k] If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v In either case, this is followed by: for k, v in F.items(): D[k] = v ''' @@ -1068,40 +1057,10 @@ def count(self, value): Sequence.register(tuple) Sequence.register(str) +Sequence.register(bytes) Sequence.register(range) Sequence.register(memoryview) -class _DeprecateByteStringMeta(ABCMeta): - def __new__(cls, name, bases, namespace, **kwargs): - if name != "ByteString": - import warnings - - warnings._deprecated( - "collections.abc.ByteString", - remove=(3, 14), - ) - return super().__new__(cls, name, bases, namespace, **kwargs) - - def __instancecheck__(cls, instance): - import warnings - - warnings._deprecated( - "collections.abc.ByteString", - remove=(3, 14), - ) - return super().__instancecheck__(instance) - -class ByteString(Sequence, metaclass=_DeprecateByteStringMeta): - """This unifies bytes and bytearray. - - XXX Should add all their methods. - """ - - __slots__ = () - -ByteString.register(bytes) -ByteString.register(bytearray) - class MutableSequence(Sequence): """All the operations on a read-write sequence. @@ -1170,4 +1129,4 @@ def __iadd__(self, values): MutableSequence.register(list) -MutableSequence.register(bytearray) # Multiply inheriting, see ByteString +MutableSequence.register(bytearray) diff --git a/Lib/_colorize.py b/Lib/_colorize.py new file mode 100644 index 00000000000000..41e818f2a747ff --- /dev/null +++ b/Lib/_colorize.py @@ -0,0 +1,69 @@ +import io +import os +import sys + +COLORIZE = True + + +class ANSIColors: + BACKGROUND_YELLOW = "\x1b[43m" + BOLD_GREEN = "\x1b[1;32m" + BOLD_MAGENTA = "\x1b[1;35m" + BOLD_RED = "\x1b[1;31m" + BLACK = "\x1b[30m" + GREEN = "\x1b[32m" + GREY = "\x1b[90m" + MAGENTA = "\x1b[35m" + RED = "\x1b[31m" + RESET = "\x1b[0m" + YELLOW = "\x1b[33m" + + +NoColors = ANSIColors() + +for attr in dir(NoColors): + if not attr.startswith("__"): + setattr(NoColors, attr, "") + + +def get_colors(colorize: bool = False, *, file=None) -> ANSIColors: + if colorize or can_colorize(file=file): + return ANSIColors() + else: + return NoColors + + +def can_colorize(*, file=None) -> bool: + if file is None: + file = sys.stdout + + if not sys.flags.ignore_environment: + if os.environ.get("PYTHON_COLORS") == "0": + return False + if os.environ.get("PYTHON_COLORS") == "1": + return True + if os.environ.get("NO_COLOR"): + return False + if not COLORIZE: + return False + if os.environ.get("FORCE_COLOR"): + return True + if os.environ.get("TERM") == "dumb": + return False + + if not hasattr(file, "fileno"): + return False + + if sys.platform == "win32": + try: + import nt + + if not nt._supports_virtual_terminal(): + return False + except (ImportError, AttributeError): + return False + + try: + return os.isatty(file.fileno()) + except io.UnsupportedOperation: + return file.isatty() diff --git a/Lib/_ios_support.py b/Lib/_ios_support.py new file mode 100644 index 00000000000000..20467a7c2bcaeb --- /dev/null +++ b/Lib/_ios_support.py @@ -0,0 +1,71 @@ +import sys +try: + from ctypes import cdll, c_void_p, c_char_p, util +except ImportError: + # ctypes is an optional module. If it's not present, we're limited in what + # we can tell about the system, but we don't want to prevent the module + # from working. + print("ctypes isn't available; iOS system calls will not be available", file=sys.stderr) + objc = None +else: + # ctypes is available. Load the ObjC library, and wrap the objc_getClass, + # sel_registerName methods + lib = util.find_library("objc") + if lib is None: + # Failed to load the objc library + raise ImportError("ObjC runtime library couldn't be loaded") + + objc = cdll.LoadLibrary(lib) + objc.objc_getClass.restype = c_void_p + objc.objc_getClass.argtypes = [c_char_p] + objc.sel_registerName.restype = c_void_p + objc.sel_registerName.argtypes = [c_char_p] + + +def get_platform_ios(): + # Determine if this is a simulator using the multiarch value + is_simulator = sys.implementation._multiarch.endswith("simulator") + + # We can't use ctypes; abort + if not objc: + return None + + # Most of the methods return ObjC objects + objc.objc_msgSend.restype = c_void_p + # All the methods used have no arguments. + objc.objc_msgSend.argtypes = [c_void_p, c_void_p] + + # Equivalent of: + # device = [UIDevice currentDevice] + UIDevice = objc.objc_getClass(b"UIDevice") + SEL_currentDevice = objc.sel_registerName(b"currentDevice") + device = objc.objc_msgSend(UIDevice, SEL_currentDevice) + + # Equivalent of: + # device_systemVersion = [device systemVersion] + SEL_systemVersion = objc.sel_registerName(b"systemVersion") + device_systemVersion = objc.objc_msgSend(device, SEL_systemVersion) + + # Equivalent of: + # device_systemName = [device systemName] + SEL_systemName = objc.sel_registerName(b"systemName") + device_systemName = objc.objc_msgSend(device, SEL_systemName) + + # Equivalent of: + # device_model = [device model] + SEL_model = objc.sel_registerName(b"model") + device_model = objc.objc_msgSend(device, SEL_model) + + # UTF8String returns a const char*; + SEL_UTF8String = objc.sel_registerName(b"UTF8String") + objc.objc_msgSend.restype = c_char_p + + # Equivalent of: + # system = [device_systemName UTF8String] + # release = [device_systemVersion UTF8String] + # model = [device_model UTF8String] + system = objc.objc_msgSend(device_systemName, SEL_UTF8String).decode() + release = objc.objc_msgSend(device_systemVersion, SEL_UTF8String).decode() + model = objc.objc_msgSend(device_model, SEL_UTF8String).decode() + + return system, release, model, is_simulator diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index fdb099bd0c2ecf..12c41374592185 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -6,6 +6,10 @@ "RESUME": [ "RESUME_CHECK", ], + "LOAD_CONST": [ + "LOAD_CONST_MORTAL", + "LOAD_CONST_IMMORTAL", + ], "TO_BOOL": [ "TO_BOOL_ALWAYS_TRUE", "TO_BOOL_BOOL", @@ -22,6 +26,7 @@ "BINARY_OP_ADD_FLOAT", "BINARY_OP_SUBTRACT_FLOAT", "BINARY_OP_ADD_UNICODE", + "BINARY_OP_EXTEND", "BINARY_OP_INPLACE_ADD_UNICODE", ], "BINARY_SUBSCR": [ @@ -62,6 +67,7 @@ "LOAD_ATTR_WITH_HINT", "LOAD_ATTR_SLOT", "LOAD_ATTR_CLASS", + "LOAD_ATTR_CLASS_WITH_METACLASS_CHECK", "LOAD_ATTR_PROPERTY", "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", "LOAD_ATTR_METHOD_WITH_VALUES", @@ -75,6 +81,14 @@ "COMPARE_OP_INT", "COMPARE_OP_STR", ], + "CONTAINS_OP": [ + "CONTAINS_OP_SET", + "CONTAINS_OP_DICT", + ], + "JUMP_BACKWARD": [ + "JUMP_BACKWARD_NO_JIT", + "JUMP_BACKWARD_JIT", + ], "FOR_ITER": [ "FOR_ITER_LIST", "FOR_ITER_TUPLE", @@ -84,7 +98,6 @@ "CALL": [ "CALL_BOUND_METHOD_EXACT_ARGS", "CALL_PY_EXACT_ARGS", - "CALL_PY_WITH_DEFAULTS", "CALL_TYPE_1", "CALL_STR_1", "CALL_TUPLE_1", @@ -100,6 +113,14 @@ "CALL_METHOD_DESCRIPTOR_NOARGS", "CALL_METHOD_DESCRIPTOR_FAST", "CALL_ALLOC_AND_ENTER_INIT", + "CALL_PY_GENERAL", + "CALL_BOUND_METHOD_GENERAL", + "CALL_NON_PY_GENERAL", + ], + "CALL_KW": [ + "CALL_KW_BOUND_METHOD", + "CALL_KW_PY", + "CALL_KW_NON_PY", ], } @@ -107,73 +128,86 @@ 'BINARY_OP_ADD_FLOAT': 150, 'BINARY_OP_ADD_INT': 151, 'BINARY_OP_ADD_UNICODE': 152, + 'BINARY_OP_EXTEND': 153, 'BINARY_OP_INPLACE_ADD_UNICODE': 3, - 'BINARY_OP_MULTIPLY_FLOAT': 153, - 'BINARY_OP_MULTIPLY_INT': 154, - 'BINARY_OP_SUBTRACT_FLOAT': 155, - 'BINARY_OP_SUBTRACT_INT': 156, - 'BINARY_SUBSCR_DICT': 157, - 'BINARY_SUBSCR_GETITEM': 158, - 'BINARY_SUBSCR_LIST_INT': 159, - 'BINARY_SUBSCR_STR_INT': 160, - 'BINARY_SUBSCR_TUPLE_INT': 161, - 'CALL_ALLOC_AND_ENTER_INIT': 162, - 'CALL_BOUND_METHOD_EXACT_ARGS': 163, - 'CALL_BUILTIN_CLASS': 164, - 'CALL_BUILTIN_FAST': 165, - 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 166, - 'CALL_BUILTIN_O': 167, - 'CALL_ISINSTANCE': 168, - 'CALL_LEN': 169, - 'CALL_LIST_APPEND': 170, - 'CALL_METHOD_DESCRIPTOR_FAST': 171, - 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 172, - 'CALL_METHOD_DESCRIPTOR_NOARGS': 173, - 'CALL_METHOD_DESCRIPTOR_O': 174, - 'CALL_PY_EXACT_ARGS': 175, - 'CALL_PY_WITH_DEFAULTS': 176, - 'CALL_STR_1': 177, - 'CALL_TUPLE_1': 178, - 'CALL_TYPE_1': 179, - 'COMPARE_OP_FLOAT': 180, - 'COMPARE_OP_INT': 181, - 'COMPARE_OP_STR': 182, - 'FOR_ITER_GEN': 183, - 'FOR_ITER_LIST': 184, - 'FOR_ITER_RANGE': 185, - 'FOR_ITER_TUPLE': 186, - 'LOAD_ATTR_CLASS': 187, - 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 188, - 'LOAD_ATTR_INSTANCE_VALUE': 189, - 'LOAD_ATTR_METHOD_LAZY_DICT': 190, - 'LOAD_ATTR_METHOD_NO_DICT': 191, - 'LOAD_ATTR_METHOD_WITH_VALUES': 192, - 'LOAD_ATTR_MODULE': 193, - 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 194, - 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 195, - 'LOAD_ATTR_PROPERTY': 196, - 'LOAD_ATTR_SLOT': 197, - 'LOAD_ATTR_WITH_HINT': 198, - 'LOAD_GLOBAL_BUILTIN': 199, - 'LOAD_GLOBAL_MODULE': 200, - 'LOAD_SUPER_ATTR_ATTR': 201, - 'LOAD_SUPER_ATTR_METHOD': 202, - 'RESUME_CHECK': 203, - 'SEND_GEN': 204, - 'STORE_ATTR_INSTANCE_VALUE': 205, - 'STORE_ATTR_SLOT': 206, - 'STORE_ATTR_WITH_HINT': 207, - 'STORE_SUBSCR_DICT': 208, - 'STORE_SUBSCR_LIST_INT': 209, - 'TO_BOOL_ALWAYS_TRUE': 210, - 'TO_BOOL_BOOL': 211, - 'TO_BOOL_INT': 212, - 'TO_BOOL_LIST': 213, - 'TO_BOOL_NONE': 214, - 'TO_BOOL_STR': 215, - 'UNPACK_SEQUENCE_LIST': 216, - 'UNPACK_SEQUENCE_TUPLE': 217, - 'UNPACK_SEQUENCE_TWO_TUPLE': 218, + 'BINARY_OP_MULTIPLY_FLOAT': 154, + 'BINARY_OP_MULTIPLY_INT': 155, + 'BINARY_OP_SUBTRACT_FLOAT': 156, + 'BINARY_OP_SUBTRACT_INT': 157, + 'BINARY_SUBSCR_DICT': 158, + 'BINARY_SUBSCR_GETITEM': 159, + 'BINARY_SUBSCR_LIST_INT': 160, + 'BINARY_SUBSCR_STR_INT': 161, + 'BINARY_SUBSCR_TUPLE_INT': 162, + 'CALL_ALLOC_AND_ENTER_INIT': 163, + 'CALL_BOUND_METHOD_EXACT_ARGS': 164, + 'CALL_BOUND_METHOD_GENERAL': 165, + 'CALL_BUILTIN_CLASS': 166, + 'CALL_BUILTIN_FAST': 167, + 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 168, + 'CALL_BUILTIN_O': 169, + 'CALL_ISINSTANCE': 170, + 'CALL_KW_BOUND_METHOD': 171, + 'CALL_KW_NON_PY': 172, + 'CALL_KW_PY': 173, + 'CALL_LEN': 174, + 'CALL_LIST_APPEND': 175, + 'CALL_METHOD_DESCRIPTOR_FAST': 176, + 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 177, + 'CALL_METHOD_DESCRIPTOR_NOARGS': 178, + 'CALL_METHOD_DESCRIPTOR_O': 179, + 'CALL_NON_PY_GENERAL': 180, + 'CALL_PY_EXACT_ARGS': 181, + 'CALL_PY_GENERAL': 182, + 'CALL_STR_1': 183, + 'CALL_TUPLE_1': 184, + 'CALL_TYPE_1': 185, + 'COMPARE_OP_FLOAT': 186, + 'COMPARE_OP_INT': 187, + 'COMPARE_OP_STR': 188, + 'CONTAINS_OP_DICT': 189, + 'CONTAINS_OP_SET': 190, + 'FOR_ITER_GEN': 191, + 'FOR_ITER_LIST': 192, + 'FOR_ITER_RANGE': 193, + 'FOR_ITER_TUPLE': 194, + 'JUMP_BACKWARD_JIT': 195, + 'JUMP_BACKWARD_NO_JIT': 196, + 'LOAD_ATTR_CLASS': 197, + 'LOAD_ATTR_CLASS_WITH_METACLASS_CHECK': 198, + 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 199, + 'LOAD_ATTR_INSTANCE_VALUE': 200, + 'LOAD_ATTR_METHOD_LAZY_DICT': 201, + 'LOAD_ATTR_METHOD_NO_DICT': 202, + 'LOAD_ATTR_METHOD_WITH_VALUES': 203, + 'LOAD_ATTR_MODULE': 204, + 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 205, + 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 206, + 'LOAD_ATTR_PROPERTY': 207, + 'LOAD_ATTR_SLOT': 208, + 'LOAD_ATTR_WITH_HINT': 209, + 'LOAD_CONST_IMMORTAL': 210, + 'LOAD_CONST_MORTAL': 211, + 'LOAD_GLOBAL_BUILTIN': 212, + 'LOAD_GLOBAL_MODULE': 213, + 'LOAD_SUPER_ATTR_ATTR': 214, + 'LOAD_SUPER_ATTR_METHOD': 215, + 'RESUME_CHECK': 216, + 'SEND_GEN': 217, + 'STORE_ATTR_INSTANCE_VALUE': 218, + 'STORE_ATTR_SLOT': 219, + 'STORE_ATTR_WITH_HINT': 220, + 'STORE_SUBSCR_DICT': 221, + 'STORE_SUBSCR_LIST_INT': 222, + 'TO_BOOL_ALWAYS_TRUE': 223, + 'TO_BOOL_BOOL': 224, + 'TO_BOOL_INT': 225, + 'TO_BOOL_LIST': 226, + 'TO_BOOL_NONE': 227, + 'TO_BOOL_STR': 228, + 'UNPACK_SEQUENCE_LIST': 229, + 'UNPACK_SEQUENCE_TUPLE': 230, + 'UNPACK_SEQUENCE_TWO_TUPLE': 231, } opmap = { @@ -181,35 +215,36 @@ 'RESERVED': 17, 'RESUME': 149, 'INSTRUMENTED_LINE': 254, - 'BEFORE_ASYNC_WITH': 1, - 'BEFORE_WITH': 2, - 'BINARY_SLICE': 4, - 'BINARY_SUBSCR': 5, - 'CHECK_EG_MATCH': 6, - 'CHECK_EXC_MATCH': 7, - 'CLEANUP_THROW': 8, - 'DELETE_SUBSCR': 9, - 'END_ASYNC_FOR': 10, - 'END_FOR': 11, - 'END_SEND': 12, - 'EXIT_INIT_CHECK': 13, - 'FORMAT_SIMPLE': 14, - 'FORMAT_WITH_SPEC': 15, - 'GET_AITER': 16, - 'GET_ANEXT': 18, - 'GET_ITER': 19, - 'GET_LEN': 20, - 'GET_YIELD_FROM_ITER': 21, - 'INTERPRETER_EXIT': 22, - 'LOAD_ASSERTION_ERROR': 23, - 'LOAD_BUILD_CLASS': 24, - 'LOAD_LOCALS': 25, - 'MAKE_FUNCTION': 26, - 'MATCH_KEYS': 27, - 'MATCH_MAPPING': 28, - 'MATCH_SEQUENCE': 29, - 'NOP': 30, - 'POP_EXCEPT': 31, + 'ENTER_EXECUTOR': 255, + 'BINARY_SLICE': 1, + 'BINARY_SUBSCR': 2, + 'CALL_FUNCTION_EX': 4, + 'CHECK_EG_MATCH': 5, + 'CHECK_EXC_MATCH': 6, + 'CLEANUP_THROW': 7, + 'DELETE_SUBSCR': 8, + 'END_ASYNC_FOR': 9, + 'END_FOR': 10, + 'END_SEND': 11, + 'EXIT_INIT_CHECK': 12, + 'FORMAT_SIMPLE': 13, + 'FORMAT_WITH_SPEC': 14, + 'GET_AITER': 15, + 'GET_ANEXT': 16, + 'GET_ITER': 18, + 'GET_LEN': 19, + 'GET_YIELD_FROM_ITER': 20, + 'INTERPRETER_EXIT': 21, + 'LOAD_BUILD_CLASS': 22, + 'LOAD_LOCALS': 23, + 'MAKE_FUNCTION': 24, + 'MATCH_KEYS': 25, + 'MATCH_MAPPING': 26, + 'MATCH_SEQUENCE': 27, + 'NOP': 28, + 'NOT_TAKEN': 29, + 'POP_EXCEPT': 30, + 'POP_ITER': 31, 'POP_TOP': 32, 'PUSH_EXC_INFO': 33, 'PUSH_NULL': 34, @@ -224,53 +259,53 @@ 'UNARY_NOT': 43, 'WITH_EXCEPT_START': 44, 'BINARY_OP': 45, - 'BUILD_CONST_KEY_MAP': 46, - 'BUILD_LIST': 47, - 'BUILD_MAP': 48, - 'BUILD_SET': 49, - 'BUILD_SLICE': 50, - 'BUILD_STRING': 51, - 'BUILD_TUPLE': 52, - 'CALL': 53, - 'CALL_FUNCTION_EX': 54, - 'CALL_INTRINSIC_1': 55, - 'CALL_INTRINSIC_2': 56, - 'CALL_KW': 57, - 'COMPARE_OP': 58, - 'CONTAINS_OP': 59, - 'CONVERT_VALUE': 60, - 'COPY': 61, - 'COPY_FREE_VARS': 62, - 'DELETE_ATTR': 63, - 'DELETE_DEREF': 64, - 'DELETE_FAST': 65, - 'DELETE_GLOBAL': 66, - 'DELETE_NAME': 67, - 'DICT_MERGE': 68, - 'DICT_UPDATE': 69, - 'ENTER_EXECUTOR': 70, - 'EXTENDED_ARG': 71, - 'FOR_ITER': 72, - 'GET_AWAITABLE': 73, - 'IMPORT_FROM': 74, - 'IMPORT_NAME': 75, - 'IS_OP': 76, - 'JUMP_BACKWARD': 77, - 'JUMP_BACKWARD_NO_INTERRUPT': 78, - 'JUMP_FORWARD': 79, - 'LIST_APPEND': 80, - 'LIST_EXTEND': 81, - 'LOAD_ATTR': 82, - 'LOAD_CONST': 83, - 'LOAD_DEREF': 84, - 'LOAD_FAST': 85, - 'LOAD_FAST_AND_CLEAR': 86, - 'LOAD_FAST_CHECK': 87, - 'LOAD_FAST_LOAD_FAST': 88, - 'LOAD_FROM_DICT_OR_DEREF': 89, - 'LOAD_FROM_DICT_OR_GLOBALS': 90, - 'LOAD_GLOBAL': 91, - 'LOAD_NAME': 92, + 'BUILD_LIST': 46, + 'BUILD_MAP': 47, + 'BUILD_SET': 48, + 'BUILD_SLICE': 49, + 'BUILD_STRING': 50, + 'BUILD_TUPLE': 51, + 'CALL': 52, + 'CALL_INTRINSIC_1': 53, + 'CALL_INTRINSIC_2': 54, + 'CALL_KW': 55, + 'COMPARE_OP': 56, + 'CONTAINS_OP': 57, + 'CONVERT_VALUE': 58, + 'COPY': 59, + 'COPY_FREE_VARS': 60, + 'DELETE_ATTR': 61, + 'DELETE_DEREF': 62, + 'DELETE_FAST': 63, + 'DELETE_GLOBAL': 64, + 'DELETE_NAME': 65, + 'DICT_MERGE': 66, + 'DICT_UPDATE': 67, + 'EXTENDED_ARG': 68, + 'FOR_ITER': 69, + 'GET_AWAITABLE': 70, + 'IMPORT_FROM': 71, + 'IMPORT_NAME': 72, + 'IS_OP': 73, + 'JUMP_BACKWARD': 74, + 'JUMP_BACKWARD_NO_INTERRUPT': 75, + 'JUMP_FORWARD': 76, + 'LIST_APPEND': 77, + 'LIST_EXTEND': 78, + 'LOAD_ATTR': 79, + 'LOAD_COMMON_CONSTANT': 80, + 'LOAD_CONST': 81, + 'LOAD_DEREF': 82, + 'LOAD_FAST': 83, + 'LOAD_FAST_AND_CLEAR': 84, + 'LOAD_FAST_CHECK': 85, + 'LOAD_FAST_LOAD_FAST': 86, + 'LOAD_FROM_DICT_OR_DEREF': 87, + 'LOAD_FROM_DICT_OR_GLOBALS': 88, + 'LOAD_GLOBAL': 89, + 'LOAD_NAME': 90, + 'LOAD_SMALL_INT': 91, + 'LOAD_SPECIAL': 92, 'LOAD_SUPER_ATTR': 93, 'MAKE_CELL': 94, 'MAP_ADD': 95, @@ -281,53 +316,51 @@ 'POP_JUMP_IF_TRUE': 100, 'RAISE_VARARGS': 101, 'RERAISE': 102, - 'RETURN_CONST': 103, - 'SEND': 104, - 'SET_ADD': 105, - 'SET_FUNCTION_ATTRIBUTE': 106, - 'SET_UPDATE': 107, - 'STORE_ATTR': 108, - 'STORE_DEREF': 109, - 'STORE_FAST': 110, - 'STORE_FAST_LOAD_FAST': 111, - 'STORE_FAST_STORE_FAST': 112, - 'STORE_GLOBAL': 113, - 'STORE_NAME': 114, - 'SWAP': 115, - 'UNPACK_EX': 116, - 'UNPACK_SEQUENCE': 117, - 'YIELD_VALUE': 118, - 'INSTRUMENTED_RESUME': 236, - 'INSTRUMENTED_END_FOR': 237, - 'INSTRUMENTED_END_SEND': 238, - 'INSTRUMENTED_RETURN_VALUE': 239, - 'INSTRUMENTED_RETURN_CONST': 240, - 'INSTRUMENTED_YIELD_VALUE': 241, - 'INSTRUMENTED_LOAD_SUPER_ATTR': 242, - 'INSTRUMENTED_FOR_ITER': 243, - 'INSTRUMENTED_CALL': 244, - 'INSTRUMENTED_CALL_KW': 245, - 'INSTRUMENTED_CALL_FUNCTION_EX': 246, - 'INSTRUMENTED_INSTRUCTION': 247, - 'INSTRUMENTED_JUMP_FORWARD': 248, - 'INSTRUMENTED_JUMP_BACKWARD': 249, - 'INSTRUMENTED_POP_JUMP_IF_TRUE': 250, - 'INSTRUMENTED_POP_JUMP_IF_FALSE': 251, - 'INSTRUMENTED_POP_JUMP_IF_NONE': 252, - 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 253, + 'SEND': 103, + 'SET_ADD': 104, + 'SET_FUNCTION_ATTRIBUTE': 105, + 'SET_UPDATE': 106, + 'STORE_ATTR': 107, + 'STORE_DEREF': 108, + 'STORE_FAST': 109, + 'STORE_FAST_LOAD_FAST': 110, + 'STORE_FAST_STORE_FAST': 111, + 'STORE_GLOBAL': 112, + 'STORE_NAME': 113, + 'SWAP': 114, + 'UNPACK_EX': 115, + 'UNPACK_SEQUENCE': 116, + 'YIELD_VALUE': 117, + 'INSTRUMENTED_END_FOR': 235, + 'INSTRUMENTED_POP_ITER': 236, + 'INSTRUMENTED_END_SEND': 237, + 'INSTRUMENTED_LOAD_SUPER_ATTR': 238, + 'INSTRUMENTED_FOR_ITER': 239, + 'INSTRUMENTED_CALL_KW': 240, + 'INSTRUMENTED_CALL_FUNCTION_EX': 241, + 'INSTRUMENTED_INSTRUCTION': 242, + 'INSTRUMENTED_JUMP_FORWARD': 243, + 'INSTRUMENTED_NOT_TAKEN': 244, + 'INSTRUMENTED_POP_JUMP_IF_TRUE': 245, + 'INSTRUMENTED_POP_JUMP_IF_FALSE': 246, + 'INSTRUMENTED_POP_JUMP_IF_NONE': 247, + 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 248, + 'INSTRUMENTED_RESUME': 249, + 'INSTRUMENTED_RETURN_VALUE': 250, + 'INSTRUMENTED_YIELD_VALUE': 251, + 'INSTRUMENTED_CALL': 252, + 'INSTRUMENTED_JUMP_BACKWARD': 253, 'JUMP': 256, - 'JUMP_NO_INTERRUPT': 257, - 'LOAD_CLOSURE': 258, - 'LOAD_METHOD': 259, - 'LOAD_SUPER_METHOD': 260, - 'LOAD_ZERO_SUPER_ATTR': 261, - 'LOAD_ZERO_SUPER_METHOD': 262, - 'POP_BLOCK': 263, - 'SETUP_CLEANUP': 264, - 'SETUP_FINALLY': 265, - 'SETUP_WITH': 266, - 'STORE_FAST_MAYBE_NULL': 267, + 'JUMP_IF_FALSE': 257, + 'JUMP_IF_TRUE': 258, + 'JUMP_NO_INTERRUPT': 259, + 'LOAD_CLOSURE': 260, + 'POP_BLOCK': 261, + 'SETUP_CLEANUP': 262, + 'SETUP_FINALLY': 263, + 'SETUP_WITH': 264, + 'STORE_FAST_MAYBE_NULL': 265, } HAVE_ARGUMENT = 44 -MIN_INSTRUMENTED_OPCODE = 236 +MIN_INSTRUMENTED_OPCODE = 235 diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index b7d569cc41740e..be90c9b1315d53 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -1,8 +1,4 @@ -"""Concrete date/time and related types. - -See http://www.iana.org/time-zones/repository/tz-link.html for -time zone and DST data sources. -""" +"""Pure Python implementation of the datetime module.""" __all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo", "MINYEAR", "MAXYEAR", "UTC") @@ -204,6 +200,28 @@ def _format_offset(off, sep=':'): s += '.%06d' % ss.microseconds return s +_normalize_century = None +def _need_normalize_century(): + global _normalize_century + if _normalize_century is None: + try: + _normalize_century = ( + _time.strftime("%Y", (99, 1, 1, 0, 0, 0, 0, 1, 0)) != "0099") + except ValueError: + _normalize_century = True + return _normalize_century + +_supports_c99 = None +def _can_support_c99(): + global _supports_c99 + if _supports_c99 is None: + try: + _supports_c99 = ( + _time.strftime("%F", (1900, 1, 1, 0, 0, 0, 0, 1, 0)) == "1900-01-01") + except ValueError: + _supports_c99 = False + return _supports_c99 + # Correctly substitute for %z and %Z escapes in strftime formats. def _wrap_strftime(object, format, timetuple): # Don't call utcoffset() or tzname() unless actually needed. @@ -261,6 +279,20 @@ def _wrap_strftime(object, format, timetuple): # strftime is going to have at this: escape % Zreplace = s.replace('%', '%%') newformat.append(Zreplace) + # Note that datetime(1000, 1, 1).strftime('%G') == '1000' so + # year 1000 for %G can go on the fast path. + elif ((ch in 'YG' or ch in 'FC' and _can_support_c99()) and + object.year < 1000 and _need_normalize_century()): + if ch == 'G': + year = int(_time.strftime("%G", timetuple)) + else: + year = object.year + if ch == 'C': + push('{:02}'.format(year // 100)) + else: + push('{:04}'.format(year)) + if ch == 'F': + push('-{:02}-{:02}'.format(*timetuple[1:3])) else: push('%') push(ch) @@ -431,6 +463,17 @@ def _parse_isoformat_time(tstr): time_comps = _parse_hh_mm_ss_ff(timestr) + hour, minute, second, microsecond = time_comps + became_next_day = False + error_from_components = False + if (hour == 24): + if all(time_comp == 0 for time_comp in time_comps[1:]): + hour = 0 + time_comps[0] = hour + became_next_day = True + else: + error_from_components = True + tzi = None if tz_pos == len_str and tstr[-1] == 'Z': tzi = timezone.utc @@ -463,7 +506,7 @@ def _parse_isoformat_time(tstr): time_comps.append(tzi) - return time_comps + return time_comps, became_next_day, error_from_components # tuple[int, int, int] -> tuple[int, int, int] version of date.fromisocalendar def _isoweek_to_gregorian(year, week, day): @@ -608,7 +651,19 @@ def __new__(cls, days=0, seconds=0, microseconds=0, # guide the C implementation; it's way more convoluted than speed- # ignoring auto-overflow-to-long idiomatic Python could be. - # XXX Check that all inputs are ints or floats. + for name, value in ( + ("days", days), + ("seconds", seconds), + ("microseconds", microseconds), + ("milliseconds", milliseconds), + ("minutes", minutes), + ("hours", hours), + ("weeks", weeks) + ): + if not isinstance(value, (int, float)): + raise TypeError( + f"unsupported type for timedelta {name} component: {type(value).__name__}" + ) # Final values, all integer. # s and us fit in 32-bit signed ints; d isn't bounded. @@ -908,6 +963,7 @@ class date: fromtimestamp() today() fromordinal() + strptime() Operators: @@ -966,6 +1022,8 @@ def __new__(cls, year, month=None, day=None): @classmethod def fromtimestamp(cls, t): "Construct a date from a POSIX timestamp (like time.time())." + if t is None: + raise TypeError("'NoneType' object cannot be interpreted as an integer") y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) return cls(y, m, d) @@ -1006,6 +1064,12 @@ def fromisocalendar(cls, year, week, day): This is the inverse of the date.isocalendar() function""" return cls(*_isoweek_to_gregorian(year, week, day)) + @classmethod + def strptime(cls, date_string, format): + """Parse a date string according to the given format (like time.strptime()).""" + import _strptime + return _strptime._strptime_datetime_date(cls, date_string, format) + # Conversions to string def __repr__(self): @@ -1326,6 +1390,7 @@ class time: Constructors: __new__() + strptime() Operators: @@ -1384,6 +1449,12 @@ def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold self._fold = fold return self + @classmethod + def strptime(cls, date_string, format): + """string, format -> new time parsed from a string (like time.strptime()).""" + import _strptime + return _strptime._strptime_datetime_time(cls, date_string, format) + # Read-only field accessors @property def hour(self): @@ -1554,7 +1625,7 @@ def fromisoformat(cls, time_string): time_string = time_string.removeprefix('T') try: - return cls(*_parse_isoformat_time(time_string)) + return cls(*_parse_isoformat_time(time_string)[0]) except Exception: raise ValueError(f'Invalid isoformat string: {time_string!r}') @@ -1868,10 +1939,27 @@ def fromisoformat(cls, date_string): if tstr: try: - time_components = _parse_isoformat_time(tstr) + time_components, became_next_day, error_from_components = _parse_isoformat_time(tstr) except ValueError: raise ValueError( f'Invalid isoformat string: {date_string!r}') from None + else: + if error_from_components: + raise ValueError("minute, second, and microsecond must be 0 when hour is 24") + + if became_next_day: + year, month, day = date_components + # Only wrap day/month when it was previously valid + if month <= 12 and day <= (days_in_month := _days_in_month(year, month)): + # Calculate midnight of the next day + day += 1 + if day > days_in_month: + day = 1 + month += 1 + if month > 12: + month = 1 + year += 1 + date_components = [year, month, day] else: time_components = [0, 0, 0, 0, None] @@ -2090,7 +2178,7 @@ def __str__(self): def strptime(cls, date_string, format): 'string, format -> new datetime parsed from a string (like time.strptime()).' import _strptime - return _strptime._strptime_datetime(cls, date_string, format) + return _strptime._strptime_datetime_datetime(cls, date_string, format) def utcoffset(self): """Return the timezone offset as timedelta positive east of UTC (negative west of @@ -2304,7 +2392,6 @@ def __reduce__(self): def _isoweek1monday(year): # Helper to calculate the day number of the Monday starting week 1 - # XXX This could be done more efficiently THURSDAY = 3 firstday = _ymd2ord(year, 1, 1) firstweekday = (firstday + 6) % 7 # See weekday() above diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 2692f2fcba45bf..ec036199331396 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -13,104 +13,7 @@ # bug) and will be backported. At this point the spec is stabilizing # and the updates are becoming fewer, smaller, and less significant. -""" -This is an implementation of decimal floating point arithmetic based on -the General Decimal Arithmetic Specification: - - http://speleotrove.com/decimal/decarith.html - -and IEEE standard 854-1987: - - http://en.wikipedia.org/wiki/IEEE_854-1987 - -Decimal floating point has finite precision with arbitrarily large bounds. - -The purpose of this module is to support arithmetic using familiar -"schoolhouse" rules and to avoid some of the tricky representation -issues associated with binary floating point. The package is especially -useful for financial applications or for contexts where users have -expectations that are at odds with binary floating point (for instance, -in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead -of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected -Decimal('0.00')). - -Here are some examples of using the decimal module: - ->>> from decimal import * ->>> setcontext(ExtendedContext) ->>> Decimal(0) -Decimal('0') ->>> Decimal('1') -Decimal('1') ->>> Decimal('-.0123') -Decimal('-0.0123') ->>> Decimal(123456) -Decimal('123456') ->>> Decimal('123.45e12345678') -Decimal('1.2345E+12345680') ->>> Decimal('1.33') + Decimal('1.27') -Decimal('2.60') ->>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41') -Decimal('-2.20') ->>> dig = Decimal(1) ->>> print(dig / Decimal(3)) -0.333333333 ->>> getcontext().prec = 18 ->>> print(dig / Decimal(3)) -0.333333333333333333 ->>> print(dig.sqrt()) -1 ->>> print(Decimal(3).sqrt()) -1.73205080756887729 ->>> print(Decimal(3) ** 123) -4.85192780976896427E+58 ->>> inf = Decimal(1) / Decimal(0) ->>> print(inf) -Infinity ->>> neginf = Decimal(-1) / Decimal(0) ->>> print(neginf) --Infinity ->>> print(neginf + inf) -NaN ->>> print(neginf * inf) --Infinity ->>> print(dig / 0) -Infinity ->>> getcontext().traps[DivisionByZero] = 1 ->>> print(dig / 0) -Traceback (most recent call last): - ... - ... - ... -decimal.DivisionByZero: x / 0 ->>> c = Context() ->>> c.traps[InvalidOperation] = 0 ->>> print(c.flags[InvalidOperation]) -0 ->>> c.divide(Decimal(0), Decimal(0)) -Decimal('NaN') ->>> c.traps[InvalidOperation] = 1 ->>> print(c.flags[InvalidOperation]) -1 ->>> c.flags[InvalidOperation] = 0 ->>> print(c.flags[InvalidOperation]) -0 ->>> print(c.divide(Decimal(0), Decimal(0))) -Traceback (most recent call last): - ... - ... - ... -decimal.InvalidOperation: 0 / 0 ->>> print(c.flags[InvalidOperation]) -1 ->>> c.flags[InvalidOperation] = 0 ->>> c.traps[InvalidOperation] = 0 ->>> print(c.divide(Decimal(0), Decimal(0))) -NaN ->>> print(c.flags[InvalidOperation]) -1 ->>> -""" +"""Python decimal arithmetic module""" __all__ = [ # Two major classes @@ -194,7 +97,7 @@ class DecimalException(ArithmeticError): Used exceptions derive from this. If an exception derives from another exception besides this (such as - Underflow (Inexact, Rounded, Subnormal) that indicates that it is only + Underflow (Inexact, Rounded, Subnormal)) that indicates that it is only called if the others are present. This isn't actually used for anything, though. @@ -242,7 +145,7 @@ class InvalidOperation(DecimalException): x ** (+-)INF An operand is invalid - The result of the operation after these is a quiet positive NaN, + The result of the operation after this is a quiet positive NaN, except when the cause is a signaling NaN, in which case the result is also a quiet NaN, but with the original sign, and an optional diagnostic information. @@ -521,7 +424,7 @@ def sin(x): # numbers.py for more detail. class Decimal(object): - """Floating point class for decimal arithmetic.""" + """Floating-point class for decimal arithmetic.""" __slots__ = ('_exp','_int','_sign', '_is_special') # Generally, the value of the Decimal instance is given by @@ -679,6 +582,21 @@ def __new__(cls, value="0", context=None): raise TypeError("Cannot convert %r to Decimal" % value) + @classmethod + def from_number(cls, number): + """Converts a real number to a decimal number, exactly. + + >>> Decimal.from_number(314) # int + Decimal('314') + >>> Decimal.from_number(0.1) # float + Decimal('0.1000000000000000055511151231257827021181583404541015625') + >>> Decimal.from_number(Decimal('3.14')) # another decimal instance + Decimal('3.14') + """ + if isinstance(number, (int, Decimal, float)): + return cls(number) + raise TypeError("Cannot convert %r to Decimal" % number) + @classmethod def from_float(cls, f): """Converts a float to a decimal number, exactly. @@ -2228,10 +2146,16 @@ def _power_exact(self, other, p): else: return None - if xc >= 10**p: + # An exact power of 10 is representable, but can convert to a + # string of any length. But an exact power of 10 shouldn't be + # possible at this point. + assert xc > 1, self + assert xc % 10 != 0, self + strxc = str(xc) + if len(strxc) > p: return None xe = -e-xe - return _dec_from_triple(0, str(xc), xe) + return _dec_from_triple(0, strxc, xe) # now y is positive; find m and n such that y = m/n if ye >= 0: @@ -2281,13 +2205,18 @@ def _power_exact(self, other, p): return None xc = xc**m xe *= m - if xc > 10**p: + # An exact power of 10 is representable, but can convert to a string + # of any length. But an exact power of 10 shouldn't be possible at + # this point. + assert xc > 1, self + assert xc % 10 != 0, self + str_xc = str(xc) + if len(str_xc) > p: return None # by this point the result *is* exactly representable # adjust the exponent to get as close as possible to the ideal # exponent, if necessary - str_xc = str(xc) if other._isinteger() and other._sign == 0: ideal_exponent = self._exp*int(other) zeros = min(xe-ideal_exponent, p-len(str_xc)) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index a3fede699218a1..023478aa78c6a0 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -16,7 +16,7 @@ _setmode = None import io -from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END) +from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END) # noqa: F401 valid_seek_flags = {0, 1, 2} # Hardwired values if hasattr(os, 'SEEK_HOLE') : @@ -238,18 +238,11 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, result = raw try: line_buffering = False - if buffering == 1 or buffering < 0 and raw.isatty(): + if buffering == 1 or buffering < 0 and raw._isatty_open_only(): buffering = -1 line_buffering = True if buffering < 0: - buffering = DEFAULT_BUFFER_SIZE - try: - bs = os.fstat(raw.fileno()).st_blksize - except (OSError, AttributeError): - pass - else: - if bs > 1: - buffering = bs + buffering = raw._blksize if buffering < 0: raise ValueError("invalid buffering size") if buffering == 0: @@ -1487,6 +1480,7 @@ def __init__(self, file, mode='r', closefd=True, opener=None): """ if self._fd >= 0: # Have to close the existing file first. + self._stat_atopen = None try: if self._closefd: os.close(self._fd) @@ -1565,18 +1559,15 @@ def __init__(self, file, mode='r', closefd=True, opener=None): os.set_inheritable(fd, False) self._closefd = closefd - fdfstat = os.fstat(fd) + self._stat_atopen = os.fstat(fd) try: - if stat.S_ISDIR(fdfstat.st_mode): + if stat.S_ISDIR(self._stat_atopen.st_mode): raise IsADirectoryError(errno.EISDIR, os.strerror(errno.EISDIR), file) except AttributeError: # Ignore the AttributeError if stat.S_ISDIR or errno.EISDIR # don't exist. pass - self._blksize = getattr(fdfstat, 'st_blksize', 0) - if self._blksize <= 1: - self._blksize = DEFAULT_BUFFER_SIZE if _setmode: # don't translate newlines (\r\n <=> \n) @@ -1593,6 +1584,7 @@ def __init__(self, file, mode='r', closefd=True, opener=None): if e.errno != errno.ESPIPE: raise except: + self._stat_atopen = None if owned_fd is not None: os.close(owned_fd) raise @@ -1622,6 +1614,17 @@ def __repr__(self): return ('<%s name=%r mode=%r closefd=%r>' % (class_name, name, self.mode, self._closefd)) + @property + def _blksize(self): + if self._stat_atopen is None: + return DEFAULT_BUFFER_SIZE + + blksize = getattr(self._stat_atopen, "st_blksize", 0) + # WASI sets blsize to 0 + if not blksize: + return DEFAULT_BUFFER_SIZE + return blksize + def _checkReadable(self): if not self._readable: raise UnsupportedOperation('File not open for reading') @@ -1654,14 +1657,22 @@ def readall(self): """ self._checkClosed() self._checkReadable() - bufsize = DEFAULT_BUFFER_SIZE - try: - pos = os.lseek(self._fd, 0, SEEK_CUR) - end = os.fstat(self._fd).st_size - if end >= pos: - bufsize = end - pos + 1 - except OSError: - pass + if self._stat_atopen is None or self._stat_atopen.st_size <= 0: + bufsize = DEFAULT_BUFFER_SIZE + else: + # In order to detect end of file, need a read() of at least 1 + # byte which returns size 0. Oversize the buffer by 1 byte so the + # I/O can be completed with two read() calls (one for all data, one + # for EOF) without needing to resize the buffer. + bufsize = self._stat_atopen.st_size + 1 + + if self._stat_atopen.st_size > 65536: + try: + pos = os.lseek(self._fd, 0, SEEK_CUR) + if self._stat_atopen.st_size >= pos: + bufsize = self._stat_atopen.st_size - pos + 1 + except OSError: + pass result = bytearray() while True: @@ -1681,13 +1692,14 @@ def readall(self): return bytes(result) - def readinto(self, b): + def readinto(self, buffer): """Same as RawIOBase.readinto().""" - m = memoryview(b).cast('B') - data = self.read(len(m)) - n = len(data) - m[:n] = data - return n + self._checkClosed() + self._checkReadable() + try: + return os.readinto(self._fd, buffer) + except BlockingIOError: + return None def write(self, b): """Write bytes b to file, return number written. @@ -1737,6 +1749,7 @@ def truncate(self, size=None): if size is None: size = self.tell() os.ftruncate(self._fd, size) + self._stat_atopen = None return size def close(self): @@ -1746,6 +1759,7 @@ def close(self): called more than once without error. """ if not self.closed: + self._stat_atopen = None try: if self._closefd: os.close(self._fd) @@ -1784,6 +1798,21 @@ def isatty(self): self._checkClosed() return os.isatty(self._fd) + def _isatty_open_only(self): + """Checks whether the file is a TTY using an open-only optimization. + + TTYs are always character devices. If the interpreter knows a file is + not a character device when it would call ``isatty``, can skip that + call. Inside ``open()`` there is a fresh stat result that contains that + information. Use the stat result to skip a system call. Outside of that + context TOCTOU issues (the fd could be arbitrarily modified by + surrounding code). + """ + if (self._stat_atopen is not None + and not stat.S_ISCHR(self._stat_atopen.st_mode)): + return False + return os.isatty(self._fd) + @property def closefd(self): """True if the file descriptor will be closed by close().""" @@ -2517,9 +2546,12 @@ def read(self, size=None): size = size_index() decoder = self._decoder or self._get_decoder() if size < 0: + chunk = self.buffer.read() + if chunk is None: + raise BlockingIOError("Read returned None.") # Read everything. result = (self._get_decoded_chars() + - decoder.decode(self.buffer.read(), final=True)) + decoder.decode(chunk, final=True)) if self._snapshot is not None: self._set_decoded_chars('') self._snapshot = None diff --git a/Lib/_pylong.py b/Lib/_pylong.py index 936346e187ff69..be1acd17ce3592 100644 --- a/Lib/_pylong.py +++ b/Lib/_pylong.py @@ -14,7 +14,127 @@ import re import decimal - +try: + import _decimal +except ImportError: + _decimal = None + +# A number of functions have this form, where `w` is a desired number of +# digits in base `base`: +# +# def inner(...w...): +# if w <= LIMIT: +# return something +# lo = w >> 1 +# hi = w - lo +# something involving base**lo, inner(...lo...), j, and inner(...hi...) +# figure out largest w needed +# result = inner(w) +# +# They all had some on-the-fly scheme to cache `base**lo` results for reuse. +# Power is costly. +# +# This routine aims to compute all amd only the needed powers in advance, as +# efficiently as reasonably possible. This isn't trivial, and all the +# on-the-fly methods did needless work in many cases. The driving code above +# changes to: +# +# figure out largest w needed +# mycache = compute_powers(w, base, LIMIT) +# result = inner(w) +# +# and `mycache[lo]` replaces `base**lo` in the inner function. +# +# If an algorithm wants the powers of ceiling(w/2) instead of the floor, +# pass keyword argument `need_hi=True`. +# +# While this does give minor speedups (a few percent at best), the +# primary intent is to simplify the functions using this, by eliminating +# the need for them to craft their own ad-hoc caching schemes. +# +# See code near end of file for a block of code that can be enabled to +# run millions of tests. +def compute_powers(w, base, more_than, *, need_hi=False, show=False): + seen = set() + need = set() + ws = {w} + while ws: + w = ws.pop() # any element is fine to use next + if w in seen or w <= more_than: + continue + seen.add(w) + lo = w >> 1 + hi = w - lo + # only _need_ one here; the other may, or may not, be needed + which = hi if need_hi else lo + need.add(which) + ws.add(which) + if lo != hi: + ws.add(w - which) + + # `need` is the set of exponents needed. To compute them all + # efficiently, possibly add other exponents to `extra`. The goal is + # to ensure that each exponent can be gotten from a smaller one via + # multiplying by the base, squaring it, or squaring and then + # multiplying by the base. + # + # If need_hi is False, this is already the case (w can always be + # gotten from w >> 1 via one of the squaring strategies). But we do + # the work anyway, just in case ;-) + # + # Note that speed is irrelevant. These loops are working on little + # ints (exponents) and go around O(log w) times. The total cost is + # insignificant compared to just one of the bigint multiplies. + cands = need.copy() + extra = set() + while cands: + w = max(cands) + cands.remove(w) + lo = w >> 1 + if lo > more_than and w-1 not in cands and lo not in cands: + extra.add(lo) + cands.add(lo) + assert need_hi or not extra + + d = {} + for n in sorted(need | extra): + lo = n >> 1 + hi = n - lo + if n-1 in d: + if show: + print("* base", end="") + result = d[n-1] * base # cheap! + elif lo in d: + # Multiplying a bigint by itself is about twice as fast + # in CPython provided it's the same object. + if show: + print("square", end="") + result = d[lo] * d[lo] # same object + if hi != lo: + if show: + print(" * base", end="") + assert 2 * lo + 1 == n + result *= base + else: # rare + if show: + print("pow", end='') + result = base ** n + if show: + print(" at", n, "needed" if n in need else "extra") + d[n] = result + + assert need <= d.keys() + if excess := d.keys() - need: + assert need_hi + for n in excess: + del d[n] + return d + +_unbounded_dec_context = decimal.getcontext().copy() +_unbounded_dec_context.prec = decimal.MAX_PREC +_unbounded_dec_context.Emax = decimal.MAX_EMAX +_unbounded_dec_context.Emin = decimal.MIN_EMIN +_unbounded_dec_context.traps[decimal.Inexact] = 1 # sanity check def int_to_decimal(n): """Asymptotically fast conversion of an 'int' to Decimal.""" @@ -29,61 +149,76 @@ def int_to_decimal(n): # "clever" recursive way. If we want a string representation, we # apply str to _that_. - D = decimal.Decimal - D2 = D(2) - - BITLIM = 128 - - mem = {} - - def w2pow(w): - """Return D(2)**w and store the result. Also possibly save some - intermediate results. In context, these are likely to be reused - across various levels of the conversion to Decimal.""" - if (result := mem.get(w)) is None: - if w <= BITLIM: - result = D2**w - elif w - 1 in mem: - result = (t := mem[w - 1]) + t - else: - w2 = w >> 1 - # If w happens to be odd, w-w2 is one larger then w2 - # now. Recurse on the smaller first (w2), so that it's - # in the cache and the larger (w-w2) can be handled by - # the cheaper `w-1 in mem` branch instead. - result = w2pow(w2) * w2pow(w - w2) - mem[w] = result - return result + from decimal import Decimal as D + BITLIM = 200 + # Don't bother caching the "lo" mask in this; the time to compute it is + # tiny compared to the multiply. def inner(n, w): if w <= BITLIM: return D(n) w2 = w >> 1 hi = n >> w2 - lo = n - (hi << w2) - return inner(lo, w2) + inner(hi, w - w2) * w2pow(w2) - - with decimal.localcontext() as ctx: - ctx.prec = decimal.MAX_PREC - ctx.Emax = decimal.MAX_EMAX - ctx.Emin = decimal.MIN_EMIN - ctx.traps[decimal.Inexact] = 1 + lo = n & ((1 << w2) - 1) + return inner(lo, w2) + inner(hi, w - w2) * w2pow[w2] + with decimal.localcontext(_unbounded_dec_context): + nbits = n.bit_length() + w2pow = compute_powers(nbits, D(2), BITLIM) if n < 0: negate = True n = -n else: negate = False - result = inner(n, n.bit_length()) + result = inner(n, nbits) if negate: result = -result return result - def int_to_decimal_string(n): """Asymptotically fast conversion of an 'int' to a decimal string.""" - return str(int_to_decimal(n)) - + w = n.bit_length() + if w > 450_000 and _decimal is not None: + # It is only usable with the C decimal implementation. + # _pydecimal.py calls str() on very large integers, which in its + # turn calls int_to_decimal_string(), causing very deep recursion. + return str(int_to_decimal(n)) + + # Fallback algorithm for the case when the C decimal module isn't + # available. This algorithm is asymptotically worse than the algorithm + # using the decimal module, but better than the quadratic time + # implementation in longobject.c. + + DIGLIM = 1000 + def inner(n, w): + if w <= DIGLIM: + return str(n) + w2 = w >> 1 + hi, lo = divmod(n, pow10[w2]) + return inner(hi, w - w2) + inner(lo, w2).zfill(w2) + + # The estimation of the number of decimal digits. + # There is no harm in small error. If we guess too large, there may + # be leading 0's that need to be stripped. If we guess too small, we + # may need to call str() recursively for the remaining highest digits, + # which can still potentially be a large integer. This is manifested + # only if the number has way more than 10**15 digits, that exceeds + # the 52-bit physical address limit in both Intel64 and AMD64. + w = int(w * 0.3010299956639812 + 1) # log10(2) + pow10 = compute_powers(w, 5, DIGLIM) + for k, v in pow10.items(): + pow10[k] = v << k # 5**k << k == 5**k * 2**k == 10**k + if n < 0: + n = -n + sign = '-' + else: + sign = '' + s = inner(n, w) + if s[0] == '0' and n: + # If our guess of w is too large, there may be leading 0's that + # need to be stripped. + s = s.lstrip('0') + return sign + s def _str_to_int_inner(s): """Asymptotically fast conversion of a 'str' to an 'int'.""" @@ -100,38 +235,157 @@ def _str_to_int_inner(s): DIGLIM = 2048 - mem = {} - - def w5pow(w): - """Return 5**w and store the result. - Also possibly save some intermediate results. In context, these - are likely to be reused across various levels of the conversion - to 'int'. - """ - if (result := mem.get(w)) is None: - if w <= DIGLIM: - result = 5**w - elif w - 1 in mem: - result = mem[w - 1] * 5 - else: - w2 = w >> 1 - # If w happens to be odd, w-w2 is one larger then w2 - # now. Recurse on the smaller first (w2), so that it's - # in the cache and the larger (w-w2) can be handled by - # the cheaper `w-1 in mem` branch instead. - result = w5pow(w2) * w5pow(w - w2) - mem[w] = result - return result - def inner(a, b): if b - a <= DIGLIM: return int(s[a:b]) mid = (a + b + 1) >> 1 - return inner(mid, b) + ((inner(a, mid) * w5pow(b - mid)) << (b - mid)) + return (inner(mid, b) + + ((inner(a, mid) * w5pow[b - mid]) + << (b - mid))) + w5pow = compute_powers(len(s), 5, DIGLIM) return inner(0, len(s)) +# Asymptotically faster version, using the C decimal module. See +# comments at the end of the file. This uses decimal arithmetic to +# convert from base 10 to base 256. The latter is just a string of +# bytes, which CPython can convert very efficiently to a Python int. + +# log of 10 to base 256 with best-possible 53-bit precision. Obtained +# via: +# from mpmath import mp +# mp.prec = 1000 +# print(float(mp.log(10, 256)).hex()) +_LOG_10_BASE_256 = float.fromhex('0x1.a934f0979a371p-2') # about 0.415 + +# _spread is for internal testing. It maps a key to the number of times +# that condition obtained in _dec_str_to_int_inner: +# key 0 - quotient guess was right +# key 1 - quotient had to be boosted by 1, one time +# key 999 - one adjustment wasn't enough, so fell back to divmod +from collections import defaultdict +_spread = defaultdict(int) +del defaultdict + +def _dec_str_to_int_inner(s, *, GUARD=8): + # Yes, BYTELIM is "large". Large enough that CPython will usually + # use the Karatsuba _str_to_int_inner to convert the string. This + # allowed reducing the cutoff for calling _this_ function from 3.5M + # to 2M digits. We could almost certainly do even better by + # fine-tuning this and/or using a larger output base than 256. + BYTELIM = 100_000 + D = decimal.Decimal + result = bytearray() + # See notes at end of file for discussion of GUARD. + assert GUARD > 0 # if 0, `decimal` can blow up - .prec 0 not allowed + + def inner(n, w): + #assert n < D256 ** w # required, but too expensive to check + if w <= BYTELIM: + # XXX Stefan Pochmann discovered that, for 1024-bit ints, + # `int(Decimal)` took 2.5x longer than `int(str(Decimal))`. + # Worse, `int(Decimal) is still quadratic-time for much + # larger ints. So unless/until all that is repaired, the + # seemingly redundant `str(Decimal)` is crucial to speed. + result.extend(int(str(n)).to_bytes(w)) # big-endian default + return + w1 = w >> 1 + w2 = w - w1 + if 0: + # This is maximally clear, but "too slow". `decimal` + # division is asymptotically fast, but we have no way to + # tell it to reuse the high-precision reciprocal it computes + # for pow256[w2], so it has to recompute it over & over & + # over again :-( + hi, lo = divmod(n, pow256[w2][0]) + else: + p256, recip = pow256[w2] + # The integer part will have a number of digits about equal + # to the difference between the log10s of `n` and `pow256` + # (which, since these are integers, is roughly approximated + # by `.adjusted()`). That's the working precision we need, + ctx.prec = max(n.adjusted() - p256.adjusted(), 0) + GUARD + hi = +n * +recip # unary `+` chops back to ctx.prec digits + ctx.prec = decimal.MAX_PREC + hi = hi.to_integral_value() # lose the fractional digits + lo = n - hi * p256 + # Because we've been uniformly rounding down, `hi` is a + # lower bound on the correct quotient. + assert lo >= 0 + # Adjust quotient up if needed. It usually isn't. In random + # testing on inputs through 5 billion digit strings, the + # test triggered once in about 200 thousand tries. + count = 0 + if lo >= p256: + count = 1 + lo -= p256 + hi += 1 + if lo >= p256: + # Complete correction via an exact computation. I + # believe it's not possible to get here provided + # GUARD >= 3. It's tested by reducing GUARD below + # that. + count = 999 + hi2, lo = divmod(lo, p256) + hi += hi2 + _spread[count] += 1 + # The assert should always succeed, but way too slow to keep + # enabled. + #assert hi, lo == divmod(n, pow256[w2][0]) + inner(hi, w1) + del hi # at top levels, can free a lot of RAM "early" + inner(lo, w2) + + # How many base 256 digits are needed?. Mathematically, exactly + # floor(log256(int(s))) + 1. There is no cheap way to compute this. + # But we can get an upper bound, and that's necessary for our error + # analysis to make sense. int(s) < 10**len(s), so the log needed is + # < log256(10**len(s)) = len(s) * log256(10). However, using + # finite-precision floating point for this, it's possible that the + # computed value is a little less than the true value. If the true + # value is at - or a little higher than - an integer, we can get an + # off-by-1 error too low. So we add 2 instead of 1 if chopping lost + # a fraction > 0.9. + + # The "WASI" test platform can complain about `len(s)` if it's too + # large to fit in its idea of "an index-sized integer". + lenS = s.__len__() + log_ub = lenS * _LOG_10_BASE_256 + log_ub_as_int = int(log_ub) + w = log_ub_as_int + 1 + (log_ub - log_ub_as_int > 0.9) + # And what if we've plain exhausted the limits of HW floats? We + # could compute the log to any desired precision using `decimal`, + # but it's not plausible that anyone will pass a string requiring + # trillions of bytes (unless they're just trying to "break things"). + if w.bit_length() >= 46: + # "Only" had < 53 - 46 = 7 bits to spare in IEEE-754 double. + raise ValueError(f"cannot convert string of len {lenS} to int") + with decimal.localcontext(_unbounded_dec_context) as ctx: + D256 = D(256) + pow256 = compute_powers(w, D256, BYTELIM, need_hi=True) + rpow256 = compute_powers(w, 1 / D256, BYTELIM, need_hi=True) + # We're going to do inexact, chopped arithmetic, multiplying by + # an approximation to the reciprocal of 256**i. We chop to get a + # lower bound on the true integer quotient. Our approximation is + # a lower bound, the multiplication is chopped too, and + # to_integral_value() is also chopped. + ctx.traps[decimal.Inexact] = 0 + ctx.rounding = decimal.ROUND_DOWN + for k, v in pow256.items(): + # No need to save much more precision in the reciprocal than + # the power of 256 has, plus some guard digits to absorb + # most relevant rounding errors. This is highly significant: + # 1/2**i has the same number of significant decimal digits + # as 5**i, generally over twice the number in 2**i, + ctx.prec = v.adjusted() + GUARD + 1 + # The unary "+" chops the reciprocal back to that precision. + pow256[k] = v, +rpow256[k] + del rpow256 # exact reciprocals no longer needed + ctx.prec = decimal.MAX_PREC + inner(D(s), w) + return int.from_bytes(result) + def int_from_string(s): """Asymptotically fast version of PyLong_FromString(), conversion of a string of decimal digits into an 'int'.""" @@ -140,8 +394,10 @@ def int_from_string(s): # and underscores, and stripped leading whitespace. The input can still # contain underscores and have trailing whitespace. s = s.rstrip().replace('_', '') - return _str_to_int_inner(s) - + func = _str_to_int_inner + if len(s) >= 2_000_000 and _decimal is not None: + func = _dec_str_to_int_inner + return func(s) def str_to_int(s): """Asymptotically fast version of decimal string to 'int' conversion.""" @@ -274,7 +530,7 @@ def int_divmod(a, b): Its time complexity is O(n**1.58), where n = #bits(a) + #bits(b). """ if b == 0: - raise ZeroDivisionError + raise ZeroDivisionError('division by zero') elif b < 0: q, r = int_divmod(-a, -b) return q, -r @@ -283,3 +539,191 @@ def int_divmod(a, b): return ~q, b + ~r else: return _divmod_pos(a, b) + + +# Notes on _dec_str_to_int_inner: +# +# Stefan Pochmann worked up a str->int function that used the decimal +# module to, in effect, convert from base 10 to base 256. This is +# "unnatural", in that it requires multiplying and dividing by large +# powers of 2, which `decimal` isn't naturally suited to. But +# `decimal`'s `*` and `/` are asymptotically superior to CPython's, so +# at _some_ point it could be expected to win. +# +# Alas, the crossover point was too high to be of much real interest. I +# (Tim) then worked on ways to replace its division with multiplication +# by a cached reciprocal approximation instead, fixing up errors +# afterwards. This reduced the crossover point significantly, +# +# I revisited the code, and found ways to improve and simplify it. The +# crossover point is at about 3.4 million digits now. +# +# About .adjusted() +# ----------------- +# Restrict to Decimal values x > 0. We don't use negative numbers in the +# code, and I don't want to have to keep typing, e.g., "absolute value". +# +# For convenience, I'll use `x.a` to mean `x.adjusted()`. x.a doesn't +# look at the digits of x, but instead returns an integer giving x's +# order of magnitude. These are equivalent: +# +# - x.a is the power-of-10 exponent of x's most significant digit. +# - x.a = the infinitely precise floor(log10(x)) +# - x can be written in this form, where f is a real with 1 <= f < 10: +# x = f * 10**x.a +# +# Observation; if x is an integer, len(str(x)) = x.a + 1. +# +# Lemma 1: (x * y).a = x.a + y.a, or one larger +# +# Proof: Write x = f * 10**x.a and y = g * 10**y.a, where f and g are in +# [1, 10). Then x*y = f*g * 10**(x.a + y.a), where 1 <= f*g < 100. If +# f*g < 10, (x*y).a is x.a+y.a. Else divide f*g by 10 to bring it back +# into [1, 10], and add 1 to the exponent to compensate. Then (x*y).a is +# x.a+y.a+1. +# +# Lemma 2: ceiling(log10(x/y)) <= x.a - y.a + 1 +# +# Proof: Express x and y as in Lemma 1. Then x/y = f/g * 10**(x.a - +# y.a), where 1/10 < f/g < 10. If 1 <= f/g, (x/y).a is x.a-y.a. Else +# multiply f/g by 10 to bring it back into [1, 10], and subtract 1 from +# the exponent to compensate. Then (x/y).a is x.a-y.a-1. So the largest +# (x/y).a can be is x.a-y.a. Since that's the floor of log10(x/y). the +# ceiling is at most 1 larger (with equality iff f/g = 1 exactly). +# +# GUARD digits +# ------------ +# We only want the integer part of divisions, so don't need to build +# the full multiplication tree. But using _just_ the number of +# digits expected in the integer part ignores too much. What's left +# out can have a very significant effect on the quotient. So we use +# GUARD additional digits. +# +# The default 8 is more than enough so no more than 1 correction step +# was ever needed for all inputs tried through 2.5 billion digits. In +# fact, I believe 3 guard digits are always enough - but the proof is +# very involved, so better safe than sorry. +# +# Short course: +# +# If prec is the decimal precision in effect, and we're rounding down, +# the result of an operation is exactly equal to the infinitely precise +# result times 1-e for some real e with 0 <= e < 10**(1-prec). In +# +# ctx.prec = max(n.adjusted() - p256.adjusted(), 0) + GUARD +# hi = +n * +recip # unary `+` chops to ctx.prec digits +# +# we have 3 visible chopped operations, but there's also a 4th: +# precomputing a truncated `recip` as part of setup. +# +# So the computed product is exactly equal to the true product times +# (1-e1)*(1-e2)*(1-e3)*(1-e4); since the e's are all very small, an +# excellent approximation to the second factor is 1-(e1+e2+e3+e4) (the +# 2nd and higher order terms in the expanded product are too tiny to +# matter). If they're all as large as possible, that's +# +# 1 - 4*10**(1-prec). This, BTW, is all bog-standard FP error analysis. +# +# That implies the computed product is within 1 of the true product +# provided prec >= log10(true_product) + 1.602. +# +# Here are telegraphic details, rephrasing the initial condition in +# equivalent ways, step by step: +# +# prod - prod * (1 - 4*10**(1-prec)) <= 1 +# prod - prod + prod * 4*10**(1-prec)) <= 1 +# prod * 4*10**(1-prec)) <= 1 +# 10**(log10(prod)) * 4*10**(1-prec)) <= 1 +# 4*10**(1-prec+log10(prod))) <= 1 +# 10**(1-prec+log10(prod))) <= 1/4 +# 1-prec+log10(prod) <= log10(1/4) = -0.602 +# -prec <= -1.602 - log10(prod) +# prec >= log10(prod) + 1.602 +# +# The true product is the same as the true ratio n/p256. By Lemma 2 +# above, n.a - p256.a + 1 is an upper bound on the ceiling of +# log10(prod). Then 2 is the ceiling of 1.602. so n.a - p256.a + 3 is an +# upper bound on the right hand side of the inequality. Any prec >= that +# will work. +# +# But since this is just a sketch of a proof ;-), the code uses the +# empirically tested 8 instead of 3. 5 digits more or less makes no +# practical difference to speed - these ints are huge. And while +# increasing GUARD above 3 may not be necessary, every increase cuts the +# percentage of cases that need a correction at all. +# +# On Computing Reciprocals +# ------------------------ +# In general, the exact reciprocals we compute have over twice as many +# significant digits as needed. 1/256**i has the same number of +# significant decimal digits as 5**i. It's a significant waste of RAM +# to store all those unneeded digits. +# +# So we cut exact reciprocals back to the least precision that can +# be needed so that the error analysis above is valid, +# +# [Note: turns out it's very significantly faster to do it this way than +# to compute 1 / 256**i directly to the desired precision, because the +# power method doesn't require division. It's also faster than computing +# (1/256)**i directly to the desired precision - no material division +# there, but `compute_powers()` is much smarter about _how_ to compute +# all the powers needed than repeated applications of `**` - that +# function invokes `**` for at most the few smallest powers needed.] +# +# The hard part is that chopping back to a shorter width occurs +# _outside_ of `inner`. We can't know then what `prec` `inner()` will +# need. We have to pick, for each value of `w2`, the largest possible +# value `prec` can become when `inner()` is working on `w2`. +# +# This is the `prec` inner() uses: +# max(n.a - p256.a, 0) + GUARD +# and what setup uses (renaming its `v` to `p256` - same thing): +# p256.a + GUARD + 1 +# +# We need that the second is always at least as large as the first, +# which is the same as requiring +# +# n.a - 2 * p256.a <= 1 +# +# What's the largest n can be? n < 255**w = 256**(w2 + (w - w2)). The +# worst case in this context is when w ix even. and then w = 2*w2, so +# n < 256**(2*w2) = (256**w2)**2 = p256**2. By Lemma 1, then, n.a +# is at most p256.a + p256.a + 1. +# +# So the most n.a - 2 * p256.a can be is +# p256.a + p256.a + 1 - 2 * p256.a = 1. QED +# +# Note: an earlier version of the code split on floor(e/2) instead of on +# the ceiling. The worst case then is odd `w`, and a more involved proof +# was needed to show that adding 4 (instead of 1) may be necessary. +# Basically because, in that case, n may be up to 256 times larger than +# p256**2. Curiously enough, by splitting on the ceiling instead, +# nothing in any proof here actually depends on the output base (256). + +# Enable for brute-force testing of compute_powers(). This takes about a +# minute, because it tries millions of cases. +if 0: + def consumer(w, limit, need_hi): + seen = set() + need = set() + def inner(w): + if w <= limit: + return + if w in seen: + return + seen.add(w) + lo = w >> 1 + hi = w - lo + need.add(hi if need_hi else lo) + inner(lo) + inner(hi) + inner(w) + exp = compute_powers(w, 1, limit, need_hi=need_hi) + assert exp.keys() == need + + from itertools import chain + for need_hi in (False, True): + for limit in (0, 1, 10, 100, 1_000, 10_000, 100_000): + for w in chain(range(1, 100_000), + (10**i for i in range(5, 30))): + consumer(w, limit, need_hi) diff --git a/Lib/_pyrepl/__init__.py b/Lib/_pyrepl/__init__.py new file mode 100644 index 00000000000000..1693cbd0b98b74 --- /dev/null +++ b/Lib/_pyrepl/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/Lib/_pyrepl/__main__.py b/Lib/_pyrepl/__main__.py new file mode 100644 index 00000000000000..3fa992eee8eeff --- /dev/null +++ b/Lib/_pyrepl/__main__.py @@ -0,0 +1,6 @@ +# Important: don't add things to this module, as they will end up in the REPL's +# default globals. Use _pyrepl.main instead. + +if __name__ == "__main__": + from .main import interactive_console as __pyrepl_interactive_console + __pyrepl_interactive_console() diff --git a/Lib/_pyrepl/_minimal_curses.py b/Lib/_pyrepl/_minimal_curses.py new file mode 100644 index 00000000000000..d884f880f50ac7 --- /dev/null +++ b/Lib/_pyrepl/_minimal_curses.py @@ -0,0 +1,68 @@ +"""Minimal '_curses' module, the low-level interface for curses module +which is not meant to be used directly. + +Based on ctypes. It's too incomplete to be really called '_curses', so +to use it, you have to import it and stick it in sys.modules['_curses'] +manually. + +Note that there is also a built-in module _minimal_curses which will +hide this one if compiled in. +""" + +import ctypes +import ctypes.util + + +class error(Exception): + pass + + +def _find_clib() -> str: + trylibs = ["ncursesw", "ncurses", "curses"] + + for lib in trylibs: + path = ctypes.util.find_library(lib) + if path: + return path + raise ModuleNotFoundError("curses library not found", name="_pyrepl._minimal_curses") + + +_clibpath = _find_clib() +clib = ctypes.cdll.LoadLibrary(_clibpath) + +clib.setupterm.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)] +clib.setupterm.restype = ctypes.c_int + +clib.tigetstr.argtypes = [ctypes.c_char_p] +clib.tigetstr.restype = ctypes.c_ssize_t + +clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int] # type: ignore[operator] +clib.tparm.restype = ctypes.c_char_p + +OK = 0 +ERR = -1 + +# ____________________________________________________________ + + +def setupterm(termstr, fd): + err = ctypes.c_int(0) + result = clib.setupterm(termstr, fd, ctypes.byref(err)) + if result == ERR: + raise error("setupterm() failed (err=%d)" % err.value) + + +def tigetstr(cap): + if not isinstance(cap, bytes): + cap = cap.encode("ascii") + result = clib.tigetstr(cap) + if result == ERR: + return None + return ctypes.cast(result, ctypes.c_char_p).value + + +def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): + result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) + if result is None: + raise error("tparm() returned NULL") + return result diff --git a/Lib/_pyrepl/_threading_handler.py b/Lib/_pyrepl/_threading_handler.py new file mode 100644 index 00000000000000..82f5e8650a2072 --- /dev/null +++ b/Lib/_pyrepl/_threading_handler.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +import traceback + + +TYPE_CHECKING = False +if TYPE_CHECKING: + from threading import Thread + from types import TracebackType + from typing import Protocol + + class ExceptHookArgs(Protocol): + @property + def exc_type(self) -> type[BaseException]: ... + @property + def exc_value(self) -> BaseException | None: ... + @property + def exc_traceback(self) -> TracebackType | None: ... + @property + def thread(self) -> Thread | None: ... + + class ShowExceptions(Protocol): + def __call__(self) -> int: ... + def add(self, s: str) -> None: ... + + from .reader import Reader + + +def install_threading_hook(reader: Reader) -> None: + import threading + + @dataclass + class ExceptHookHandler: + lock: threading.Lock = field(default_factory=threading.Lock) + messages: list[str] = field(default_factory=list) + + def show(self) -> int: + count = 0 + with self.lock: + if not self.messages: + return 0 + reader.restore() + for tb in self.messages: + count += 1 + if tb: + print(tb) + self.messages.clear() + reader.scheduled_commands.append("ctrl-c") + reader.prepare() + return count + + def add(self, s: str) -> None: + with self.lock: + self.messages.append(s) + + def exception(self, args: ExceptHookArgs) -> None: + lines = traceback.format_exception( + args.exc_type, + args.exc_value, + args.exc_traceback, + colorize=reader.can_colorize, + ) # type: ignore[call-overload] + pre = f"\nException in {args.thread.name}:\n" if args.thread else "\n" + tb = pre + "".join(lines) + self.add(tb) + + def __call__(self) -> int: + return self.show() + + + handler = ExceptHookHandler() + reader.threading_hook = handler + threading.excepthook = handler.exception diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py new file mode 100644 index 00000000000000..503ca1da329eaa --- /dev/null +++ b/Lib/_pyrepl/commands.py @@ -0,0 +1,489 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations +import os + +# Categories of actions: +# killing +# yanking +# motion +# editing +# history +# finishing +# [completion] + + +# types +if False: + from .historical_reader import HistoricalReader + + +class Command: + finish: bool = False + kills_digit_arg: bool = True + + def __init__( + self, reader: HistoricalReader, event_name: str, event: list[str] + ) -> None: + # Reader should really be "any reader" but there's too much usage of + # HistoricalReader methods and fields in the code below for us to + # refactor at the moment. + + self.reader = reader + self.event = event + self.event_name = event_name + + def do(self) -> None: + pass + + +class KillCommand(Command): + def kill_range(self, start: int, end: int) -> None: + if start == end: + return + r = self.reader + b = r.buffer + text = b[start:end] + del b[start:end] + if is_kill(r.last_command): + if start < r.pos: + r.kill_ring[-1] = text + r.kill_ring[-1] + else: + r.kill_ring[-1] = r.kill_ring[-1] + text + else: + r.kill_ring.append(text) + r.pos = start + r.dirty = True + + +class YankCommand(Command): + pass + + +class MotionCommand(Command): + pass + + +class EditCommand(Command): + pass + + +class FinishCommand(Command): + finish = True + pass + + +def is_kill(command: type[Command] | None) -> bool: + return command is not None and issubclass(command, KillCommand) + + +def is_yank(command: type[Command] | None) -> bool: + return command is not None and issubclass(command, YankCommand) + + +# etc + + +class digit_arg(Command): + kills_digit_arg = False + + def do(self) -> None: + r = self.reader + c = self.event[-1] + if c == "-": + if r.arg is not None: + r.arg = -r.arg + else: + r.arg = -1 + else: + d = int(c) + if r.arg is None: + r.arg = d + else: + if r.arg < 0: + r.arg = 10 * r.arg - d + else: + r.arg = 10 * r.arg + d + r.dirty = True + + +class clear_screen(Command): + def do(self) -> None: + r = self.reader + r.console.clear() + r.dirty = True + + +class refresh(Command): + def do(self) -> None: + self.reader.dirty = True + + +class repaint(Command): + def do(self) -> None: + self.reader.dirty = True + self.reader.console.repaint() + + +class kill_line(KillCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + eol = r.eol() + for c in b[r.pos : eol]: + if not c.isspace(): + self.kill_range(r.pos, eol) + return + else: + self.kill_range(r.pos, eol + 1) + + +class unix_line_discard(KillCommand): + def do(self) -> None: + r = self.reader + self.kill_range(r.bol(), r.pos) + + +class unix_word_rubout(KillCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + self.kill_range(r.bow(), r.pos) + + +class kill_word(KillCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + self.kill_range(r.pos, r.eow()) + + +class backward_kill_word(KillCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + self.kill_range(r.bow(), r.pos) + + +class yank(YankCommand): + def do(self) -> None: + r = self.reader + if not r.kill_ring: + r.error("nothing to yank") + return + r.insert(r.kill_ring[-1]) + + +class yank_pop(YankCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + if not r.kill_ring: + r.error("nothing to yank") + return + if not is_yank(r.last_command): + r.error("previous command was not a yank") + return + repl = len(r.kill_ring[-1]) + r.kill_ring.insert(0, r.kill_ring.pop()) + t = r.kill_ring[-1] + b[r.pos - repl : r.pos] = t + r.pos = r.pos - repl + len(t) + r.dirty = True + + +class interrupt(FinishCommand): + def do(self) -> None: + import signal + + self.reader.console.finish() + self.reader.finish() + os.kill(os.getpid(), signal.SIGINT) + + +class ctrl_c(Command): + def do(self) -> None: + self.reader.console.finish() + self.reader.finish() + raise KeyboardInterrupt + + +class suspend(Command): + def do(self) -> None: + import signal + + r = self.reader + p = r.pos + r.console.finish() + os.kill(os.getpid(), signal.SIGSTOP) + ## this should probably be done + ## in a handler for SIGCONT? + r.console.prepare() + r.pos = p + # r.posxy = 0, 0 # XXX this is invalid + r.dirty = True + r.console.screen = [] + + +class up(MotionCommand): + def do(self) -> None: + r = self.reader + for _ in range(r.get_arg()): + x, y = r.pos2xy() + new_y = y - 1 + + if r.bol() == 0: + if r.historyi > 0: + r.select_item(r.historyi - 1) + return + r.pos = 0 + r.error("start of buffer") + return + + if ( + x + > ( + new_x := r.max_column(new_y) + ) # we're past the end of the previous line + or x == r.max_column(y) + and any( + not i.isspace() for i in r.buffer[r.bol() :] + ) # move between eols + ): + x = new_x + + r.setpos_from_xy(x, new_y) + + +class down(MotionCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + for _ in range(r.get_arg()): + x, y = r.pos2xy() + new_y = y + 1 + + if r.eol() == len(b): + if r.historyi < len(r.history): + r.select_item(r.historyi + 1) + r.pos = r.eol(0) + return + r.pos = len(b) + r.error("end of buffer") + return + + if ( + x + > ( + new_x := r.max_column(new_y) + ) # we're past the end of the previous line + or x == r.max_column(y) + and any( + not i.isspace() for i in r.buffer[r.bol() :] + ) # move between eols + ): + x = new_x + + r.setpos_from_xy(x, new_y) + + +class left(MotionCommand): + def do(self) -> None: + r = self.reader + for _ in range(r.get_arg()): + p = r.pos - 1 + if p >= 0: + r.pos = p + else: + self.reader.error("start of buffer") + + +class right(MotionCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + for _ in range(r.get_arg()): + p = r.pos + 1 + if p <= len(b): + r.pos = p + else: + self.reader.error("end of buffer") + + +class beginning_of_line(MotionCommand): + def do(self) -> None: + self.reader.pos = self.reader.bol() + + +class end_of_line(MotionCommand): + def do(self) -> None: + self.reader.pos = self.reader.eol() + + +class home(MotionCommand): + def do(self) -> None: + self.reader.pos = 0 + + +class end(MotionCommand): + def do(self) -> None: + self.reader.pos = len(self.reader.buffer) + + +class forward_word(MotionCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + r.pos = r.eow() + + +class backward_word(MotionCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + r.pos = r.bow() + + +class self_insert(EditCommand): + def do(self) -> None: + r = self.reader + text = self.event * r.get_arg() + r.insert(text) + + +class insert_nl(EditCommand): + def do(self) -> None: + r = self.reader + r.insert("\n" * r.get_arg()) + + +class transpose_characters(EditCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + s = r.pos - 1 + if s < 0: + r.error("cannot transpose at start of buffer") + else: + if s == len(b): + s -= 1 + t = min(s + r.get_arg(), len(b) - 1) + c = b[s] + del b[s] + b.insert(t, c) + r.pos = t + r.dirty = True + + +class backspace(EditCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + for i in range(r.get_arg()): + if r.pos > 0: + r.pos -= 1 + del b[r.pos] + r.dirty = True + else: + self.reader.error("can't backspace at start") + + +class delete(EditCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + if ( + r.pos == 0 + and len(b) == 0 # this is something of a hack + and self.event[-1] == "\004" + ): + r.update_screen() + r.console.finish() + raise EOFError + for i in range(r.get_arg()): + if r.pos != len(b): + del b[r.pos] + r.dirty = True + else: + self.reader.error("end of buffer") + + +class accept(FinishCommand): + def do(self) -> None: + pass + + +class help(Command): + def do(self) -> None: + import _sitebuiltins + + with self.reader.suspend(): + self.reader.msg = _sitebuiltins._Helper()() # type: ignore[assignment, call-arg] + + +class invalid_key(Command): + def do(self) -> None: + pending = self.reader.console.getpending() + s = "".join(self.event) + pending.data + self.reader.error("`%r' not bound" % s) + + +class invalid_command(Command): + def do(self) -> None: + s = self.event_name + self.reader.error("command `%s' not known" % s) + + +class show_history(Command): + def do(self) -> None: + from .pager import get_pager + from site import gethistoryfile # type: ignore[attr-defined] + + history = os.linesep.join(self.reader.history[:]) + self.reader.console.restore() + pager = get_pager() + pager(history, gethistoryfile()) + self.reader.console.prepare() + + # We need to copy over the state so that it's consistent between + # console and reader, and console does not overwrite/append stuff + self.reader.console.screen = self.reader.screen.copy() + self.reader.console.posxy = self.reader.cxy + + +class paste_mode(Command): + + def do(self) -> None: + self.reader.paste_mode = not self.reader.paste_mode + self.reader.dirty = True + + +class enable_bracketed_paste(Command): + def do(self) -> None: + self.reader.paste_mode = True + self.reader.in_bracketed_paste = True + +class disable_bracketed_paste(Command): + def do(self) -> None: + self.reader.paste_mode = False + self.reader.in_bracketed_paste = False + self.reader.dirty = True diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py new file mode 100644 index 00000000000000..1cd4b6367ca8b1 --- /dev/null +++ b/Lib/_pyrepl/completing_reader.py @@ -0,0 +1,295 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +from dataclasses import dataclass, field + +import re +from . import commands, console, reader +from .reader import Reader + + +# types +Command = commands.Command +if False: + from .types import KeySpec, CommandName + + +def prefix(wordlist: list[str], j: int = 0) -> str: + d = {} + i = j + try: + while 1: + for word in wordlist: + d[word[i]] = 1 + if len(d) > 1: + return wordlist[0][j:i] + i += 1 + d = {} + except IndexError: + return wordlist[0][j:i] + return "" + + +STRIPCOLOR_REGEX = re.compile(r"\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[m|K]") + +def stripcolor(s: str) -> str: + return STRIPCOLOR_REGEX.sub('', s) + + +def real_len(s: str) -> int: + return len(stripcolor(s)) + + +def left_align(s: str, maxlen: int) -> str: + stripped = stripcolor(s) + if len(stripped) > maxlen: + # too bad, we remove the color + return stripped[:maxlen] + padding = maxlen - len(stripped) + return s + ' '*padding + + +def build_menu( + cons: console.Console, + wordlist: list[str], + start: int, + use_brackets: bool, + sort_in_column: bool, +) -> tuple[list[str], int]: + if use_brackets: + item = "[ %s ]" + padding = 4 + else: + item = "%s " + padding = 2 + maxlen = min(max(map(real_len, wordlist)), cons.width - padding) + cols = int(cons.width / (maxlen + padding)) + rows = int((len(wordlist) - 1)/cols + 1) + + if sort_in_column: + # sort_in_column=False (default) sort_in_column=True + # A B C A D G + # D E F B E + # G C F + # + # "fill" the table with empty words, so we always have the same amount + # of rows for each column + missing = cols*rows - len(wordlist) + wordlist = wordlist + ['']*missing + indexes = [(i % cols) * rows + i // cols for i in range(len(wordlist))] + wordlist = [wordlist[i] for i in indexes] + menu = [] + i = start + for r in range(rows): + row = [] + for col in range(cols): + row.append(item % left_align(wordlist[i], maxlen)) + i += 1 + if i >= len(wordlist): + break + menu.append(''.join(row)) + if i >= len(wordlist): + i = 0 + break + if r + 5 > cons.height: + menu.append(" %d more... " % (len(wordlist) - i)) + break + return menu, i + +# this gets somewhat user interface-y, and as a result the logic gets +# very convoluted. +# +# To summarise the summary of the summary:- people are a problem. +# -- The Hitch-Hikers Guide to the Galaxy, Episode 12 + +#### Desired behaviour of the completions commands. +# the considerations are: +# (1) how many completions are possible +# (2) whether the last command was a completion +# (3) if we can assume that the completer is going to return the same set of +# completions: this is controlled by the ``assume_immutable_completions`` +# variable on the reader, which is True by default to match the historical +# behaviour of pyrepl, but e.g. False in the ReadlineAlikeReader to match +# more closely readline's semantics (this is needed e.g. by +# fancycompleter) +# +# if there's no possible completion, beep at the user and point this out. +# this is easy. +# +# if there's only one possible completion, stick it in. if the last thing +# user did was a completion, point out that he isn't getting anywhere, but +# only if the ``assume_immutable_completions`` is True. +# +# now it gets complicated. +# +# for the first press of a completion key: +# if there's a common prefix, stick it in. + +# irrespective of whether anything got stuck in, if the word is now +# complete, show the "complete but not unique" message + +# if there's no common prefix and if the word is not now complete, +# beep. + +# common prefix -> yes no +# word complete \/ +# yes "cbnu" "cbnu" +# no - beep + +# for the second bang on the completion key +# there will necessarily be no common prefix +# show a menu of the choices. + +# for subsequent bangs, rotate the menu around (if there are sufficient +# choices). + + +class complete(commands.Command): + def do(self) -> None: + r: CompletingReader + r = self.reader # type: ignore[assignment] + last_is_completer = r.last_command_is(self.__class__) + immutable_completions = r.assume_immutable_completions + completions_unchangable = last_is_completer and immutable_completions + stem = r.get_stem() + if not completions_unchangable: + r.cmpltn_menu_choices = r.get_completions(stem) + + completions = r.cmpltn_menu_choices + if not completions: + r.error("no matches") + elif len(completions) == 1: + if completions_unchangable and len(completions[0]) == len(stem): + r.msg = "[ sole completion ]" + r.dirty = True + r.insert(completions[0][len(stem):]) + else: + p = prefix(completions, len(stem)) + if p: + r.insert(p) + if last_is_completer: + r.cmpltn_menu_visible = True + r.cmpltn_message_visible = False + r.cmpltn_menu, r.cmpltn_menu_end = build_menu( + r.console, completions, r.cmpltn_menu_end, + r.use_brackets, r.sort_in_column) + r.dirty = True + elif not r.cmpltn_menu_visible: + r.cmpltn_message_visible = True + if stem + p in completions: + r.msg = "[ complete but not unique ]" + r.dirty = True + else: + r.msg = "[ not unique ]" + r.dirty = True + + +class self_insert(commands.self_insert): + def do(self) -> None: + r: CompletingReader + r = self.reader # type: ignore[assignment] + + commands.self_insert.do(self) + if r.cmpltn_menu_visible: + stem = r.get_stem() + if len(stem) < 1: + r.cmpltn_reset() + else: + completions = [w for w in r.cmpltn_menu_choices + if w.startswith(stem)] + if completions: + r.cmpltn_menu, r.cmpltn_menu_end = build_menu( + r.console, completions, 0, + r.use_brackets, r.sort_in_column) + else: + r.cmpltn_reset() + + +@dataclass +class CompletingReader(Reader): + """Adds completion support""" + + ### Class variables + # see the comment for the complete command + assume_immutable_completions = True + use_brackets = True # display completions inside [] + sort_in_column = False + + ### Instance variables + cmpltn_menu: list[str] = field(init=False) + cmpltn_menu_visible: bool = field(init=False) + cmpltn_message_visible: bool = field(init=False) + cmpltn_menu_end: int = field(init=False) + cmpltn_menu_choices: list[str] = field(init=False) + + def __post_init__(self) -> None: + super().__post_init__() + self.cmpltn_reset() + for c in (complete, self_insert): + self.commands[c.__name__] = c + self.commands[c.__name__.replace('_', '-')] = c + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r'\t', 'complete'),) + + def after_command(self, cmd: Command) -> None: + super().after_command(cmd) + if not isinstance(cmd, (complete, self_insert)): + self.cmpltn_reset() + + def calc_screen(self) -> list[str]: + screen = super().calc_screen() + if self.cmpltn_menu_visible: + # We display the completions menu below the current prompt + ly = self.lxy[1] + 1 + screen[ly:ly] = self.cmpltn_menu + # If we're not in the middle of multiline edit, don't append to screeninfo + # since that screws up the position calculation in pos2xy function. + # This is a hack to prevent the cursor jumping + # into the completions menu when pressing left or down arrow. + if self.pos != len(self.buffer): + self.screeninfo[ly:ly] = [(0, [])]*len(self.cmpltn_menu) + return screen + + def finish(self) -> None: + super().finish() + self.cmpltn_reset() + + def cmpltn_reset(self) -> None: + self.cmpltn_menu = [] + self.cmpltn_menu_visible = False + self.cmpltn_message_visible = False + self.cmpltn_menu_end = 0 + self.cmpltn_menu_choices = [] + + def get_stem(self) -> str: + st = self.syntax_table + SW = reader.SYNTAX_WORD + b = self.buffer + p = self.pos - 1 + while p >= 0 and st.get(b[p], SW) == SW: + p -= 1 + return ''.join(b[p+1:self.pos]) + + def get_completions(self, stem: str) -> list[str]: + return [] diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py new file mode 100644 index 00000000000000..0d78890b4f45d5 --- /dev/null +++ b/Lib/_pyrepl/console.py @@ -0,0 +1,213 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +import _colorize # type: ignore[import-not-found] + +from abc import ABC, abstractmethod +import ast +import code +from dataclasses import dataclass, field +import os.path +import sys + + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import IO + from typing import Callable + + +@dataclass +class Event: + evt: str + data: str + raw: bytes = b"" + + +@dataclass +class Console(ABC): + posxy: tuple[int, int] + screen: list[str] = field(default_factory=list) + height: int = 25 + width: int = 80 + + def __init__( + self, + f_in: IO[bytes] | int = 0, + f_out: IO[bytes] | int = 1, + term: str = "", + encoding: str = "", + ): + self.encoding = encoding or sys.getdefaultencoding() + + if isinstance(f_in, int): + self.input_fd = f_in + else: + self.input_fd = f_in.fileno() + + if isinstance(f_out, int): + self.output_fd = f_out + else: + self.output_fd = f_out.fileno() + + @abstractmethod + def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: ... + + @abstractmethod + def prepare(self) -> None: ... + + @abstractmethod + def restore(self) -> None: ... + + @abstractmethod + def move_cursor(self, x: int, y: int) -> None: ... + + @abstractmethod + def set_cursor_vis(self, visible: bool) -> None: ... + + @abstractmethod + def getheightwidth(self) -> tuple[int, int]: + """Return (height, width) where height and width are the height + and width of the terminal window in characters.""" + ... + + @abstractmethod + def get_event(self, block: bool = True) -> Event | None: + """Return an Event instance. Returns None if |block| is false + and there is no event pending, otherwise waits for the + completion of an event.""" + ... + + @abstractmethod + def push_char(self, char: int | bytes) -> None: + """ + Push a character to the console event queue. + """ + ... + + @abstractmethod + def beep(self) -> None: ... + + @abstractmethod + def clear(self) -> None: + """Wipe the screen""" + ... + + @abstractmethod + def finish(self) -> None: + """Move the cursor to the end of the display and otherwise get + ready for end. XXX could be merged with restore? Hmm.""" + ... + + @abstractmethod + def flushoutput(self) -> None: + """Flush all output to the screen (assuming there's some + buffering going on somewhere).""" + ... + + @abstractmethod + def forgetinput(self) -> None: + """Forget all pending, but not yet processed input.""" + ... + + @abstractmethod + def getpending(self) -> Event: + """Return the characters that have been typed but not yet + processed.""" + ... + + @abstractmethod + def wait(self, timeout: float | None) -> bool: + """Wait for an event. The return value is True if an event is + available, False if the timeout has been reached. If timeout is + None, wait forever. The timeout is in milliseconds.""" + ... + + @property + def input_hook(self) -> Callable[[], int] | None: + """Returns the current input hook.""" + ... + + @abstractmethod + def repaint(self) -> None: ... + + +class InteractiveColoredConsole(code.InteractiveConsole): + def __init__( + self, + locals: dict[str, object] | None = None, + filename: str = "", + *, + local_exit: bool = False, + ) -> None: + super().__init__(locals=locals, filename=filename, local_exit=local_exit) # type: ignore[call-arg] + self.can_colorize = _colorize.can_colorize() + + def showsyntaxerror(self, filename=None, **kwargs): + super().showsyntaxerror(filename=filename, **kwargs) + + def _excepthook(self, typ, value, tb): + import traceback + lines = traceback.format_exception( + typ, value, tb, + colorize=self.can_colorize, + limit=traceback.BUILTIN_EXCEPTION_LIMIT) + self.write(''.join(lines)) + + def runsource(self, source, filename="", symbol="single"): + try: + tree = self.compile.compiler( + source, + filename, + "exec", + ast.PyCF_ONLY_AST, + incomplete_input=False, + ) + except (SyntaxError, OverflowError, ValueError): + self.showsyntaxerror(filename, source=source) + return False + if tree.body: + *_, last_stmt = tree.body + for stmt in tree.body: + wrapper = ast.Interactive if stmt is last_stmt else ast.Module + the_symbol = symbol if stmt is last_stmt else "exec" + item = wrapper([stmt]) + try: + code = self.compile.compiler(item, filename, the_symbol) + except SyntaxError as e: + if e.args[0] == "'await' outside function": + python = os.path.basename(sys.executable) + e.add_note( + f"Try the asyncio REPL ({python} -m asyncio) to use" + f" top-level 'await' and run background asyncio tasks." + ) + self.showsyntaxerror(filename, source=source) + return False + except (OverflowError, ValueError): + self.showsyntaxerror(filename, source=source) + return False + + if code is None: + return True + + self.runcode(code) + return False diff --git a/Lib/_pyrepl/curses.py b/Lib/_pyrepl/curses.py new file mode 100644 index 00000000000000..3a624d9f6835d1 --- /dev/null +++ b/Lib/_pyrepl/curses.py @@ -0,0 +1,33 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + +try: + import _curses +except ImportError: + try: + import curses as _curses # type: ignore[no-redef] + except ImportError: + from . import _minimal_curses as _curses # type: ignore[no-redef] + +setupterm = _curses.setupterm +tigetstr = _curses.tigetstr +tparm = _curses.tparm +error = _curses.error diff --git a/Lib/_pyrepl/fancy_termios.py b/Lib/_pyrepl/fancy_termios.py new file mode 100644 index 00000000000000..0468b9a2670267 --- /dev/null +++ b/Lib/_pyrepl/fancy_termios.py @@ -0,0 +1,76 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +import termios + + +class TermState: + def __init__(self, tuples): + ( + self.iflag, + self.oflag, + self.cflag, + self.lflag, + self.ispeed, + self.ospeed, + self.cc, + ) = tuples + + def as_list(self): + return [ + self.iflag, + self.oflag, + self.cflag, + self.lflag, + self.ispeed, + self.ospeed, + # Always return a copy of the control characters list to ensure + # there are not any additional references to self.cc + self.cc[:], + ] + + def copy(self): + return self.__class__(self.as_list()) + + +def tcgetattr(fd): + return TermState(termios.tcgetattr(fd)) + + +def tcsetattr(fd, when, attrs): + termios.tcsetattr(fd, when, attrs.as_list()) + + +class Term(TermState): + TS__init__ = TermState.__init__ + + def __init__(self, fd=0): + self.TS__init__(termios.tcgetattr(fd)) + self.fd = fd + self.stack = [] + + def save(self): + self.stack.append(self.as_list()) + + def set(self, when=termios.TCSANOW): + termios.tcsetattr(self.fd, when, self.as_list()) + + def restore(self): + self.TS__init__(self.stack.pop()) + self.set() diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py new file mode 100644 index 00000000000000..c4b95fa2e81ee6 --- /dev/null +++ b/Lib/_pyrepl/historical_reader.py @@ -0,0 +1,419 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass, field + +from . import commands, input +from .reader import Reader + + +if False: + from .types import SimpleContextManager, KeySpec, CommandName + + +isearch_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple( + [("\\%03o" % c, "isearch-end") for c in range(256) if chr(c) != "\\"] + + [(c, "isearch-add-character") for c in map(chr, range(32, 127)) if c != "\\"] + + [ + ("\\%03o" % c, "isearch-add-character") + for c in range(256) + if chr(c).isalpha() and chr(c) != "\\" + ] + + [ + ("\\\\", "self-insert"), + (r"\C-r", "isearch-backwards"), + (r"\C-s", "isearch-forwards"), + (r"\C-c", "isearch-cancel"), + (r"\C-g", "isearch-cancel"), + (r"\", "isearch-backspace"), + ] +) + +ISEARCH_DIRECTION_NONE = "" +ISEARCH_DIRECTION_BACKWARDS = "r" +ISEARCH_DIRECTION_FORWARDS = "f" + + +class next_history(commands.Command): + def do(self) -> None: + r = self.reader + if r.historyi == len(r.history): + r.error("end of history list") + return + r.select_item(r.historyi + 1) + + +class previous_history(commands.Command): + def do(self) -> None: + r = self.reader + if r.historyi == 0: + r.error("start of history list") + return + r.select_item(r.historyi - 1) + + +class history_search_backward(commands.Command): + def do(self) -> None: + r = self.reader + r.search_next(forwards=False) + + +class history_search_forward(commands.Command): + def do(self) -> None: + r = self.reader + r.search_next(forwards=True) + + +class restore_history(commands.Command): + def do(self) -> None: + r = self.reader + if r.historyi != len(r.history): + if r.get_unicode() != r.history[r.historyi]: + r.buffer = list(r.history[r.historyi]) + r.pos = len(r.buffer) + r.dirty = True + + +class first_history(commands.Command): + def do(self) -> None: + self.reader.select_item(0) + + +class last_history(commands.Command): + def do(self) -> None: + self.reader.select_item(len(self.reader.history)) + + +class operate_and_get_next(commands.FinishCommand): + def do(self) -> None: + self.reader.next_history = self.reader.historyi + 1 + + +class yank_arg(commands.Command): + def do(self) -> None: + r = self.reader + if r.last_command is self.__class__: + r.yank_arg_i += 1 + else: + r.yank_arg_i = 0 + if r.historyi < r.yank_arg_i: + r.error("beginning of history list") + return + a = r.get_arg(-1) + # XXX how to split? + words = r.get_item(r.historyi - r.yank_arg_i - 1).split() + if a < -len(words) or a >= len(words): + r.error("no such arg") + return + w = words[a] + b = r.buffer + if r.yank_arg_i > 0: + o = len(r.yank_arg_yanked) + else: + o = 0 + b[r.pos - o : r.pos] = list(w) + r.yank_arg_yanked = w + r.pos += len(w) - o + r.dirty = True + + +class forward_history_isearch(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_FORWARDS + r.isearch_start = r.historyi, r.pos + r.isearch_term = "" + r.dirty = True + r.push_input_trans(r.isearch_trans) + + +class reverse_history_isearch(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS + r.dirty = True + r.isearch_term = "" + r.push_input_trans(r.isearch_trans) + r.isearch_start = r.historyi, r.pos + + +class isearch_cancel(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_NONE + r.pop_input_trans() + r.select_item(r.isearch_start[0]) + r.pos = r.isearch_start[1] + r.dirty = True + + +class isearch_add_character(commands.Command): + def do(self) -> None: + r = self.reader + b = r.buffer + r.isearch_term += self.event[-1] + r.dirty = True + p = r.pos + len(r.isearch_term) - 1 + if b[p : p + 1] != [r.isearch_term[-1]]: + r.isearch_next() + + +class isearch_backspace(commands.Command): + def do(self) -> None: + r = self.reader + if len(r.isearch_term) > 0: + r.isearch_term = r.isearch_term[:-1] + r.dirty = True + else: + r.error("nothing to rubout") + + +class isearch_forwards(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_FORWARDS + r.isearch_next() + + +class isearch_backwards(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS + r.isearch_next() + + +class isearch_end(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_NONE + r.console.forgetinput() + r.pop_input_trans() + r.dirty = True + + +@dataclass +class HistoricalReader(Reader): + """Adds history support (with incremental history searching) to the + Reader class. + """ + + history: list[str] = field(default_factory=list) + historyi: int = 0 + next_history: int | None = None + transient_history: dict[int, str] = field(default_factory=dict) + isearch_term: str = "" + isearch_direction: str = ISEARCH_DIRECTION_NONE + isearch_start: tuple[int, int] = field(init=False) + isearch_trans: input.KeymapTranslator = field(init=False) + yank_arg_i: int = 0 + yank_arg_yanked: str = "" + + def __post_init__(self) -> None: + super().__post_init__() + for c in [ + next_history, + previous_history, + restore_history, + first_history, + last_history, + yank_arg, + forward_history_isearch, + reverse_history_isearch, + isearch_end, + isearch_add_character, + isearch_cancel, + isearch_add_character, + isearch_backspace, + isearch_forwards, + isearch_backwards, + operate_and_get_next, + history_search_backward, + history_search_forward, + ]: + self.commands[c.__name__] = c + self.commands[c.__name__.replace("_", "-")] = c + self.isearch_start = self.historyi, self.pos + self.isearch_trans = input.KeymapTranslator( + isearch_keymap, invalid_cls=isearch_end, character_cls=isearch_add_character + ) + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r"\C-n", "next-history"), + (r"\C-p", "previous-history"), + (r"\C-o", "operate-and-get-next"), + (r"\C-r", "reverse-history-isearch"), + (r"\C-s", "forward-history-isearch"), + (r"\M-r", "restore-history"), + (r"\M-.", "yank-arg"), + (r"\", "history-search-forward"), + (r"\x1b[6~", "history-search-forward"), + (r"\", "history-search-backward"), + (r"\x1b[5~", "history-search-backward"), + ) + + def select_item(self, i: int) -> None: + self.transient_history[self.historyi] = self.get_unicode() + buf = self.transient_history.get(i) + if buf is None: + buf = self.history[i].rstrip() + self.buffer = list(buf) + self.historyi = i + self.pos = len(self.buffer) + self.dirty = True + self.last_refresh_cache.invalidated = True + + def get_item(self, i: int) -> str: + if i != len(self.history): + return self.transient_history.get(i, self.history[i]) + else: + return self.transient_history.get(i, self.get_unicode()) + + @contextmanager + def suspend(self) -> SimpleContextManager: + with super().suspend(), self.suspend_history(): + yield + + @contextmanager + def suspend_history(self) -> SimpleContextManager: + try: + old_history = self.history[:] + del self.history[:] + yield + finally: + self.history[:] = old_history + + def prepare(self) -> None: + super().prepare() + try: + self.transient_history = {} + if self.next_history is not None and self.next_history < len(self.history): + self.historyi = self.next_history + self.buffer[:] = list(self.history[self.next_history]) + self.pos = len(self.buffer) + self.transient_history[len(self.history)] = "" + else: + self.historyi = len(self.history) + self.next_history = None + except: + self.restore() + raise + + def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: + if cursor_on_line and self.isearch_direction != ISEARCH_DIRECTION_NONE: + d = "rf"[self.isearch_direction == ISEARCH_DIRECTION_FORWARDS] + return "(%s-search `%s') " % (d, self.isearch_term) + else: + return super().get_prompt(lineno, cursor_on_line) + + def search_next(self, *, forwards: bool) -> None: + """Search history for the current line contents up to the cursor. + + Selects the first item found. If nothing is under the cursor, any next + item in history is selected. + """ + pos = self.pos + s = self.get_unicode() + history_index = self.historyi + + # In multiline contexts, we're only interested in the current line. + nl_index = s.rfind('\n', 0, pos) + prefix = s[nl_index + 1:pos] + pos = len(prefix) + + match_prefix = len(prefix) + len_item = 0 + if history_index < len(self.history): + len_item = len(self.get_item(history_index)) + if len_item and pos == len_item: + match_prefix = False + elif not pos: + match_prefix = False + + while 1: + if forwards: + out_of_bounds = history_index >= len(self.history) - 1 + else: + out_of_bounds = history_index == 0 + if out_of_bounds: + if forwards and not match_prefix: + self.pos = 0 + self.buffer = [] + self.dirty = True + else: + self.error("not found") + return + + history_index += 1 if forwards else -1 + s = self.get_item(history_index) + + if not match_prefix: + self.select_item(history_index) + return + + len_acc = 0 + for i, line in enumerate(s.splitlines(keepends=True)): + if line.startswith(prefix): + self.select_item(history_index) + self.pos = pos + len_acc + return + len_acc += len(line) + + def isearch_next(self) -> None: + st = self.isearch_term + p = self.pos + i = self.historyi + s = self.get_unicode() + forwards = self.isearch_direction == ISEARCH_DIRECTION_FORWARDS + while 1: + if forwards: + p = s.find(st, p + 1) + else: + p = s.rfind(st, 0, p + len(st) - 1) + if p != -1: + self.select_item(i) + self.pos = p + return + elif (forwards and i >= len(self.history) - 1) or (not forwards and i == 0): + self.error("not found") + return + else: + if forwards: + i += 1 + s = self.get_item(i) + p = -1 + else: + i -= 1 + s = self.get_item(i) + p = len(s) + + def finish(self) -> None: + super().finish() + ret = self.get_unicode() + for i, t in self.transient_history.items(): + if i < len(self.history) and i != self.historyi: + self.history[i] = t + if ret and should_auto_add_history: + self.history.append(ret) + + +should_auto_add_history = True diff --git a/Lib/_pyrepl/input.py b/Lib/_pyrepl/input.py new file mode 100644 index 00000000000000..21c24eb5cde3e3 --- /dev/null +++ b/Lib/_pyrepl/input.py @@ -0,0 +1,114 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +# (naming modules after builtin functions is not such a hot idea...) + +# an KeyTrans instance translates Event objects into Command objects + +# hmm, at what level do we want [C-i] and [tab] to be equivalent? +# [meta-a] and [esc a]? obviously, these are going to be equivalent +# for the UnixConsole, but should they be for PygameConsole? + +# it would in any situation seem to be a bad idea to bind, say, [tab] +# and [C-i] to *different* things... but should binding one bind the +# other? + +# executive, temporary decision: [tab] and [C-i] are distinct, but +# [meta-key] is identified with [esc key]. We demand that any console +# class does quite a lot towards emulating a unix terminal. + +from __future__ import annotations + +from abc import ABC, abstractmethod +import unicodedata +from collections import deque + + +# types +if False: + from .types import EventTuple + + +class InputTranslator(ABC): + @abstractmethod + def push(self, evt: EventTuple) -> None: + pass + + @abstractmethod + def get(self) -> EventTuple | None: + return None + + @abstractmethod + def empty(self) -> bool: + return True + + +class KeymapTranslator(InputTranslator): + def __init__(self, keymap, verbose=False, invalid_cls=None, character_cls=None): + self.verbose = verbose + from .keymap import compile_keymap, parse_keys + + self.keymap = keymap + self.invalid_cls = invalid_cls + self.character_cls = character_cls + d = {} + for keyspec, command in keymap: + keyseq = tuple(parse_keys(keyspec)) + d[keyseq] = command + if self.verbose: + print(d) + self.k = self.ck = compile_keymap(d, ()) + self.results = deque() + self.stack = [] + + def push(self, evt): + if self.verbose: + print("pushed", evt.data, end="") + key = evt.data + d = self.k.get(key) + if isinstance(d, dict): + if self.verbose: + print("transition") + self.stack.append(key) + self.k = d + else: + if d is None: + if self.verbose: + print("invalid") + if self.stack or len(key) > 1 or unicodedata.category(key) == "C": + self.results.append((self.invalid_cls, self.stack + [key])) + else: + # small optimization: + self.k[key] = self.character_cls + self.results.append((self.character_cls, [key])) + else: + if self.verbose: + print("matched", d) + self.results.append((d, self.stack + [key])) + self.stack = [] + self.k = self.ck + + def get(self): + if self.results: + return self.results.popleft() + else: + return None + + def empty(self) -> bool: + return not self.results diff --git a/Lib/_pyrepl/keymap.py b/Lib/_pyrepl/keymap.py new file mode 100644 index 00000000000000..d11df4b5164696 --- /dev/null +++ b/Lib/_pyrepl/keymap.py @@ -0,0 +1,213 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +""" +Keymap contains functions for parsing keyspecs and turning keyspecs into +appropriate sequences. + +A keyspec is a string representing a sequence of key presses that can +be bound to a command. All characters other than the backslash represent +themselves. In the traditional manner, a backslash introduces an escape +sequence. + +pyrepl uses its own keyspec format that is meant to be a strict superset of +readline's KEYSEQ format. This means that if a spec is found that readline +accepts that this doesn't, it should be logged as a bug. Note that this means +we're using the '\\C-o' style of readline's keyspec, not the 'Control-o' sort. + +The extension to readline is that the sequence \\ denotes the +sequence of characters produced by hitting KEY. + +Examples: +'a' - what you get when you hit the 'a' key +'\\EOA' - Escape - O - A (up, on my terminal) +'\\' - the up arrow key +'\\' - ditto (keynames are case-insensitive) +'\\C-o', '\\c-o' - control-o +'\\M-.' - meta-period +'\\E.' - ditto (that's how meta works for pyrepl) +'\\', '\\', '\\t', '\\011', '\\x09', '\\X09', '\\C-i', '\\C-I' + - all of these are the tab character. +""" + +_escapes = { + "\\": "\\", + "'": "'", + '"': '"', + "a": "\a", + "b": "\b", + "e": "\033", + "f": "\f", + "n": "\n", + "r": "\r", + "t": "\t", + "v": "\v", +} + +_keynames = { + "backspace": "backspace", + "delete": "delete", + "down": "down", + "end": "end", + "enter": "\r", + "escape": "\033", + "f1": "f1", + "f2": "f2", + "f3": "f3", + "f4": "f4", + "f5": "f5", + "f6": "f6", + "f7": "f7", + "f8": "f8", + "f9": "f9", + "f10": "f10", + "f11": "f11", + "f12": "f12", + "f13": "f13", + "f14": "f14", + "f15": "f15", + "f16": "f16", + "f17": "f17", + "f18": "f18", + "f19": "f19", + "f20": "f20", + "home": "home", + "insert": "insert", + "left": "left", + "page down": "page down", + "page up": "page up", + "return": "\r", + "right": "right", + "space": " ", + "tab": "\t", + "up": "up", +} + + +class KeySpecError(Exception): + pass + + +def parse_keys(keys: str) -> list[str]: + """Parse keys in keyspec format to a sequence of keys.""" + s = 0 + r: list[str] = [] + while s < len(keys): + k, s = _parse_single_key_sequence(keys, s) + r.extend(k) + return r + + +def _parse_single_key_sequence(key: str, s: int) -> tuple[list[str], int]: + ctrl = 0 + meta = 0 + ret = "" + while not ret and s < len(key): + if key[s] == "\\": + c = key[s + 1].lower() + if c in _escapes: + ret = _escapes[c] + s += 2 + elif c == "c": + if key[s + 2] != "-": + raise KeySpecError( + "\\C must be followed by `-' (char %d of %s)" + % (s + 2, repr(key)) + ) + if ctrl: + raise KeySpecError( + "doubled \\C- (char %d of %s)" % (s + 1, repr(key)) + ) + ctrl = 1 + s += 3 + elif c == "m": + if key[s + 2] != "-": + raise KeySpecError( + "\\M must be followed by `-' (char %d of %s)" + % (s + 2, repr(key)) + ) + if meta: + raise KeySpecError( + "doubled \\M- (char %d of %s)" % (s + 1, repr(key)) + ) + meta = 1 + s += 3 + elif c.isdigit(): + n = key[s + 1 : s + 4] + ret = chr(int(n, 8)) + s += 4 + elif c == "x": + n = key[s + 2 : s + 4] + ret = chr(int(n, 16)) + s += 4 + elif c == "<": + t = key.find(">", s) + if t == -1: + raise KeySpecError( + "unterminated \\< starting at char %d of %s" + % (s + 1, repr(key)) + ) + ret = key[s + 2 : t].lower() + if ret not in _keynames: + raise KeySpecError( + "unrecognised keyname `%s' at char %d of %s" + % (ret, s + 2, repr(key)) + ) + ret = _keynames[ret] + s = t + 1 + else: + raise KeySpecError( + "unknown backslash escape %s at char %d of %s" + % (repr(c), s + 2, repr(key)) + ) + else: + ret = key[s] + s += 1 + if ctrl: + if len(ret) == 1: + ret = chr(ord(ret) & 0x1F) # curses.ascii.ctrl() + elif ret in {"left", "right"}: + ret = f"ctrl {ret}" + else: + raise KeySpecError("\\C- followed by invalid key") + + result = [ret], s + if meta: + result[0].insert(0, "\033") + return result + + +def compile_keymap(keymap, empty=b""): + r = {} + for key, value in keymap.items(): + if isinstance(key, bytes): + first = key[:1] + else: + first = key[0] + r.setdefault(first, {})[key[1:]] = value + for key, value in r.items(): + if empty in value: + if len(value) != 1: + raise KeySpecError("key definitions for %s clash" % (value.values(),)) + else: + r[key] = value[empty] + else: + r[key] = compile_keymap(value, empty) + return r diff --git a/Lib/_pyrepl/main.py b/Lib/_pyrepl/main.py new file mode 100644 index 00000000000000..a6f824dcc4ad14 --- /dev/null +++ b/Lib/_pyrepl/main.py @@ -0,0 +1,59 @@ +import errno +import os +import sys + + +CAN_USE_PYREPL: bool +FAIL_REASON: str +try: + if sys.platform == "win32" and sys.getwindowsversion().build < 10586: + raise RuntimeError("Windows 10 TH2 or later required") + if not os.isatty(sys.stdin.fileno()): + raise OSError(errno.ENOTTY, "tty required", "stdin") + from .simple_interact import check + if err := check(): + raise RuntimeError(err) +except Exception as e: + CAN_USE_PYREPL = False + FAIL_REASON = f"warning: can't use pyrepl: {e}" +else: + CAN_USE_PYREPL = True + FAIL_REASON = "" + + +def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): + if not CAN_USE_PYREPL: + if not os.getenv('PYTHON_BASIC_REPL') and FAIL_REASON: + from .trace import trace + trace(FAIL_REASON) + print(FAIL_REASON, file=sys.stderr) + return sys._baserepl() + + if mainmodule: + namespace = mainmodule.__dict__ + else: + import __main__ + namespace = __main__.__dict__ + namespace.pop("__pyrepl_interactive_console", None) + + # sys._baserepl() above does this internally, we do it here + startup_path = os.getenv("PYTHONSTARTUP") + if pythonstartup and startup_path: + sys.audit("cpython.run_startup", startup_path) + + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") + exec(startup_code, namespace) + + # set sys.{ps1,ps2} just before invoking the interactive interpreter. This + # mimics what CPython does in pythonrun.c + if not hasattr(sys, "ps1"): + sys.ps1 = ">>> " + if not hasattr(sys, "ps2"): + sys.ps2 = "... " + + from .console import InteractiveColoredConsole + from .simple_interact import run_multiline_interactive_console + console = InteractiveColoredConsole(namespace, filename="") + run_multiline_interactive_console(console) diff --git a/Lib/_pyrepl/mypy.ini b/Lib/_pyrepl/mypy.ini new file mode 100644 index 00000000000000..395f5945ab740b --- /dev/null +++ b/Lib/_pyrepl/mypy.ini @@ -0,0 +1,24 @@ +# Config file for running mypy on _pyrepl. +# Run mypy by invoking `mypy --config-file Lib/_pyrepl/mypy.ini` +# on the command-line from the repo root + +[mypy] +files = Lib/_pyrepl +explicit_package_bases = True +python_version = 3.12 +platform = linux +pretty = True + +# Enable most stricter settings +enable_error_code = ignore-without-code,redundant-expr +strict = True + +# Various stricter settings that we can't yet enable +# Try to enable these in the following order: +disallow_untyped_calls = False +disallow_untyped_defs = False +check_untyped_defs = False + +# Various internal modules that typeshed deliberately doesn't have stubs for: +[mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*] +ignore_missing_imports = True diff --git a/Lib/_pyrepl/pager.py b/Lib/_pyrepl/pager.py new file mode 100644 index 00000000000000..1fddc63e3ee3ad --- /dev/null +++ b/Lib/_pyrepl/pager.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import io +import os +import re +import sys + + +# types +if False: + from typing import Protocol + class Pager(Protocol): + def __call__(self, text: str, title: str = "") -> None: + ... + + +def get_pager() -> Pager: + """Decide what method to use for paging through text.""" + if not hasattr(sys.stdin, "isatty"): + return plain_pager + if not hasattr(sys.stdout, "isatty"): + return plain_pager + if not sys.stdin.isatty() or not sys.stdout.isatty(): + return plain_pager + if sys.platform == "emscripten": + return plain_pager + use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER') + if use_pager: + if sys.platform == 'win32': # pipes completely broken in Windows + return lambda text, title='': tempfile_pager(plain(text), use_pager) + elif os.environ.get('TERM') in ('dumb', 'emacs'): + return lambda text, title='': pipe_pager(plain(text), use_pager, title) + else: + return lambda text, title='': pipe_pager(text, use_pager, title) + if os.environ.get('TERM') in ('dumb', 'emacs'): + return plain_pager + if sys.platform == 'win32': + return lambda text, title='': tempfile_pager(plain(text), 'more <') + if hasattr(os, 'system') and os.system('(pager) 2>/dev/null') == 0: + return lambda text, title='': pipe_pager(text, 'pager', title) + if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: + return lambda text, title='': pipe_pager(text, 'less', title) + + import tempfile + (fd, filename) = tempfile.mkstemp() + os.close(fd) + try: + if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0: + return lambda text, title='': pipe_pager(text, 'more', title) + else: + return tty_pager + finally: + os.unlink(filename) + + +def escape_stdout(text: str) -> str: + # Escape non-encodable characters to avoid encoding errors later + encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' + return text.encode(encoding, 'backslashreplace').decode(encoding) + + +def escape_less(s: str) -> str: + return re.sub(r'([?:.%\\])', r'\\\1', s) + + +def plain(text: str) -> str: + """Remove boldface formatting from text.""" + return re.sub('.\b', '', text) + + +def tty_pager(text: str, title: str = '') -> None: + """Page through text on a text terminal.""" + lines = plain(escape_stdout(text)).split('\n') + has_tty = False + try: + import tty + import termios + fd = sys.stdin.fileno() + old = termios.tcgetattr(fd) + tty.setcbreak(fd) + has_tty = True + + def getchar() -> str: + return sys.stdin.read(1) + + except (ImportError, AttributeError, io.UnsupportedOperation): + def getchar() -> str: + return sys.stdin.readline()[:-1][:1] + + try: + try: + h = int(os.environ.get('LINES', 0)) + except ValueError: + h = 0 + if h <= 1: + h = 25 + r = inc = h - 1 + sys.stdout.write('\n'.join(lines[:inc]) + '\n') + while lines[r:]: + sys.stdout.write('-- more --') + sys.stdout.flush() + c = getchar() + + if c in ('q', 'Q'): + sys.stdout.write('\r \r') + break + elif c in ('\r', '\n'): + sys.stdout.write('\r \r' + lines[r] + '\n') + r = r + 1 + continue + if c in ('b', 'B', '\x1b'): + r = r - inc - inc + if r < 0: r = 0 + sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n') + r = r + inc + + finally: + if has_tty: + termios.tcsetattr(fd, termios.TCSAFLUSH, old) + + +def plain_pager(text: str, title: str = '') -> None: + """Simply print unformatted text. This is the ultimate fallback.""" + sys.stdout.write(plain(escape_stdout(text))) + + +def pipe_pager(text: str, cmd: str, title: str = '') -> None: + """Page through text by feeding it to another program.""" + import subprocess + env = os.environ.copy() + if title: + title += ' ' + esc_title = escape_less(title) + prompt_string = ( + f' {esc_title}' + + '?ltline %lt?L/%L.' + ':byte %bB?s/%s.' + '.' + '?e (END):?pB %pB\\%..' + ' (press h for help or q to quit)') + env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string) + proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, + errors='backslashreplace', env=env) + assert proc.stdin is not None + try: + with proc.stdin as pipe: + try: + pipe.write(text) + except KeyboardInterrupt: + # We've hereby abandoned whatever text hasn't been written, + # but the pager is still in control of the terminal. + pass + except OSError: + pass # Ignore broken pipes caused by quitting the pager program. + while True: + try: + proc.wait() + break + except KeyboardInterrupt: + # Ignore ctl-c like the pager itself does. Otherwise the pager is + # left running and the terminal is in raw mode and unusable. + pass + + +def tempfile_pager(text: str, cmd: str, title: str = '') -> None: + """Page through text by invoking a program on a temporary file.""" + import tempfile + with tempfile.TemporaryDirectory() as tempdir: + filename = os.path.join(tempdir, 'pydoc.out') + with open(filename, 'w', errors='backslashreplace', + encoding=os.device_encoding(0) if + sys.platform == 'win32' else None + ) as file: + file.write(text) + os.system(cmd + ' "' + filename + '"') diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py new file mode 100644 index 00000000000000..4b0700d069c621 --- /dev/null +++ b/Lib/_pyrepl/reader.py @@ -0,0 +1,815 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +import sys + +from contextlib import contextmanager +from dataclasses import dataclass, field, fields +import unicodedata +from _colorize import can_colorize, ANSIColors # type: ignore[import-not-found] + + +from . import commands, console, input +from .utils import ANSI_ESCAPE_SEQUENCE, wlen, str_width +from .trace import trace + + +# types +Command = commands.Command +from .types import Callback, SimpleContextManager, KeySpec, CommandName + + +def disp_str(buffer: str) -> tuple[str, list[int]]: + """disp_str(buffer:string) -> (string, [int]) + + Return the string that should be the printed representation of + |buffer| and a list detailing where the characters of |buffer| + get used up. E.g.: + + >>> disp_str(chr(3)) + ('^C', [1, 0]) + + """ + b: list[int] = [] + s: list[str] = [] + for c in buffer: + if c == '\x1a': + s.append(c) + b.append(2) + elif ord(c) < 128: + s.append(c) + b.append(1) + elif unicodedata.category(c).startswith("C"): + c = r"\u%04x" % ord(c) + s.append(c) + b.extend([0] * (len(c) - 1)) + else: + s.append(c) + b.append(str_width(c)) + return "".join(s), b + + +# syntax classes: + +SYNTAX_WHITESPACE, SYNTAX_WORD, SYNTAX_SYMBOL = range(3) + + +def make_default_syntax_table() -> dict[str, int]: + # XXX perhaps should use some unicodedata here? + st: dict[str, int] = {} + for c in map(chr, range(256)): + st[c] = SYNTAX_SYMBOL + for c in [a for a in map(chr, range(256)) if a.isalnum()]: + st[c] = SYNTAX_WORD + st["\n"] = st[" "] = SYNTAX_WHITESPACE + return st + + +def make_default_commands() -> dict[CommandName, type[Command]]: + result: dict[CommandName, type[Command]] = {} + for v in vars(commands).values(): + if isinstance(v, type) and issubclass(v, Command) and v.__name__[0].islower(): + result[v.__name__] = v + result[v.__name__.replace("_", "-")] = v + return result + + +default_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple( + [ + (r"\C-a", "beginning-of-line"), + (r"\C-b", "left"), + (r"\C-c", "interrupt"), + (r"\C-d", "delete"), + (r"\C-e", "end-of-line"), + (r"\C-f", "right"), + (r"\C-g", "cancel"), + (r"\C-h", "backspace"), + (r"\C-j", "accept"), + (r"\", "accept"), + (r"\C-k", "kill-line"), + (r"\C-l", "clear-screen"), + (r"\C-m", "accept"), + (r"\C-t", "transpose-characters"), + (r"\C-u", "unix-line-discard"), + (r"\C-w", "unix-word-rubout"), + (r"\C-x\C-u", "upcase-region"), + (r"\C-y", "yank"), + *(() if sys.platform == "win32" else ((r"\C-z", "suspend"), )), + (r"\M-b", "backward-word"), + (r"\M-c", "capitalize-word"), + (r"\M-d", "kill-word"), + (r"\M-f", "forward-word"), + (r"\M-l", "downcase-word"), + (r"\M-t", "transpose-words"), + (r"\M-u", "upcase-word"), + (r"\M-y", "yank-pop"), + (r"\M--", "digit-arg"), + (r"\M-0", "digit-arg"), + (r"\M-1", "digit-arg"), + (r"\M-2", "digit-arg"), + (r"\M-3", "digit-arg"), + (r"\M-4", "digit-arg"), + (r"\M-5", "digit-arg"), + (r"\M-6", "digit-arg"), + (r"\M-7", "digit-arg"), + (r"\M-8", "digit-arg"), + (r"\M-9", "digit-arg"), + (r"\M-\n", "accept"), + ("\\\\", "self-insert"), + (r"\x1b[200~", "enable_bracketed_paste"), + (r"\x1b[201~", "disable_bracketed_paste"), + (r"\x03", "ctrl-c"), + ] + + [(c, "self-insert") for c in map(chr, range(32, 127)) if c != "\\"] + + [(c, "self-insert") for c in map(chr, range(128, 256)) if c.isalpha()] + + [ + (r"\", "up"), + (r"\", "down"), + (r"\", "left"), + (r"\C-\", "backward-word"), + (r"\", "right"), + (r"\C-\", "forward-word"), + (r"\", "delete"), + (r"\x1b[3~", "delete"), + (r"\", "backspace"), + (r"\M-\", "backward-kill-word"), + (r"\", "end-of-line"), # was 'end' + (r"\", "beginning-of-line"), # was 'home' + (r"\", "help"), + (r"\", "show-history"), + (r"\", "paste-mode"), + (r"\EOF", "end"), # the entries in the terminfo database for xterms + (r"\EOH", "home"), # seem to be wrong. this is a less than ideal + # workaround + ] +) + + +@dataclass(slots=True) +class Reader: + """The Reader class implements the bare bones of a command reader, + handling such details as editing and cursor motion. What it does + not support are such things as completion or history support - + these are implemented elsewhere. + + Instance variables of note include: + + * buffer: + A *list* (*not* a string at the moment :-) containing all the + characters that have been entered. + * console: + Hopefully encapsulates the OS dependent stuff. + * pos: + A 0-based index into 'buffer' for where the insertion point + is. + * screeninfo: + Ahem. This list contains some info needed to move the + insertion point around reasonably efficiently. + * cxy, lxy: + the position of the insertion point in screen ... + * syntax_table: + Dictionary mapping characters to 'syntax class'; read the + emacs docs to see what this means :-) + * commands: + Dictionary mapping command names to command classes. + * arg: + The emacs-style prefix argument. It will be None if no such + argument has been provided. + * dirty: + True if we need to refresh the display. + * kill_ring: + The emacs-style kill-ring; manipulated with yank & yank-pop + * ps1, ps2, ps3, ps4: + prompts. ps1 is the prompt for a one-line input; for a + multiline input it looks like: + ps2> first line of input goes here + ps3> second and further + ps3> lines get ps3 + ... + ps4> and the last one gets ps4 + As with the usual top-level, you can set these to instances if + you like; str() will be called on them (once) at the beginning + of each command. Don't put really long or newline containing + strings here, please! + This is just the default policy; you can change it freely by + overriding get_prompt() (and indeed some standard subclasses + do). + * finished: + handle1 will set this to a true value if a command signals + that we're done. + """ + + console: console.Console + + ## state + buffer: list[str] = field(default_factory=list) + pos: int = 0 + ps1: str = "->> " + ps2: str = "/>> " + ps3: str = "|.. " + ps4: str = R"\__ " + kill_ring: list[list[str]] = field(default_factory=list) + msg: str = "" + arg: int | None = None + dirty: bool = False + finished: bool = False + paste_mode: bool = False + in_bracketed_paste: bool = False + commands: dict[str, type[Command]] = field(default_factory=make_default_commands) + last_command: type[Command] | None = None + syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table) + keymap: tuple[tuple[str, str], ...] = () + input_trans: input.KeymapTranslator = field(init=False) + input_trans_stack: list[input.KeymapTranslator] = field(default_factory=list) + screen: list[str] = field(default_factory=list) + screeninfo: list[tuple[int, list[int]]] = field(init=False) + cxy: tuple[int, int] = field(init=False) + lxy: tuple[int, int] = field(init=False) + scheduled_commands: list[str] = field(default_factory=list) + can_colorize: bool = False + threading_hook: Callback | None = None + + ## cached metadata to speed up screen refreshes + @dataclass + class RefreshCache: + in_bracketed_paste: bool = False + screen: list[str] = field(default_factory=list) + screeninfo: list[tuple[int, list[int]]] = field(init=False) + line_end_offsets: list[int] = field(default_factory=list) + pos: int = field(init=False) + cxy: tuple[int, int] = field(init=False) + dimensions: tuple[int, int] = field(init=False) + invalidated: bool = False + + def update_cache(self, + reader: Reader, + screen: list[str], + screeninfo: list[tuple[int, list[int]]], + ) -> None: + self.in_bracketed_paste = reader.in_bracketed_paste + self.screen = screen.copy() + self.screeninfo = screeninfo.copy() + self.pos = reader.pos + self.cxy = reader.cxy + self.dimensions = reader.console.width, reader.console.height + self.invalidated = False + + def valid(self, reader: Reader) -> bool: + if self.invalidated: + return False + dimensions = reader.console.width, reader.console.height + dimensions_changed = dimensions != self.dimensions + paste_changed = reader.in_bracketed_paste != self.in_bracketed_paste + return not (dimensions_changed or paste_changed) + + def get_cached_location(self, reader: Reader) -> tuple[int, int]: + if self.invalidated: + raise ValueError("Cache is invalidated") + offset = 0 + earliest_common_pos = min(reader.pos, self.pos) + num_common_lines = len(self.line_end_offsets) + while num_common_lines > 0: + offset = self.line_end_offsets[num_common_lines - 1] + if earliest_common_pos > offset: + break + num_common_lines -= 1 + else: + offset = 0 + return offset, num_common_lines + + last_refresh_cache: RefreshCache = field(default_factory=RefreshCache) + + def __post_init__(self) -> None: + # Enable the use of `insert` without a `prepare` call - necessary to + # facilitate the tab completion hack implemented for + # . + self.keymap = self.collect_keymap() + self.input_trans = input.KeymapTranslator( + self.keymap, invalid_cls="invalid-key", character_cls="self-insert" + ) + self.screeninfo = [(0, [])] + self.cxy = self.pos2xy() + self.lxy = (self.pos, 0) + self.can_colorize = can_colorize() + + self.last_refresh_cache.screeninfo = self.screeninfo + self.last_refresh_cache.pos = self.pos + self.last_refresh_cache.cxy = self.cxy + self.last_refresh_cache.dimensions = (0, 0) + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return default_keymap + + def calc_screen(self) -> list[str]: + """Translate changes in self.buffer into changes in self.console.screen.""" + # Since the last call to calc_screen: + # screen and screeninfo may differ due to a completion menu being shown + # pos and cxy may differ due to edits, cursor movements, or completion menus + + # Lines that are above both the old and new cursor position can't have changed, + # unless the terminal has been resized (which might cause reflowing) or we've + # entered or left paste mode (which changes prompts, causing reflowing). + num_common_lines = 0 + offset = 0 + if self.last_refresh_cache.valid(self): + offset, num_common_lines = self.last_refresh_cache.get_cached_location(self) + + screen = self.last_refresh_cache.screen + del screen[num_common_lines:] + + screeninfo = self.last_refresh_cache.screeninfo + del screeninfo[num_common_lines:] + + last_refresh_line_end_offsets = self.last_refresh_cache.line_end_offsets + del last_refresh_line_end_offsets[num_common_lines:] + + pos = self.pos + pos -= offset + + prompt_from_cache = (offset and self.buffer[offset - 1] != "\n") + + lines = "".join(self.buffer[offset:]).split("\n") + + cursor_found = False + lines_beyond_cursor = 0 + for ln, line in enumerate(lines, num_common_lines): + ll = len(line) + if 0 <= pos <= ll: + self.lxy = pos, ln + cursor_found = True + elif cursor_found: + lines_beyond_cursor += 1 + if lines_beyond_cursor > self.console.height: + # No need to keep formatting lines. + # The console can't show them. + break + if prompt_from_cache: + # Only the first line's prompt can come from the cache + prompt_from_cache = False + prompt = "" + else: + prompt = self.get_prompt(ln, ll >= pos >= 0) + while "\n" in prompt: + pre_prompt, _, prompt = prompt.partition("\n") + last_refresh_line_end_offsets.append(offset) + screen.append(pre_prompt) + screeninfo.append((0, [])) + pos -= ll + 1 + prompt, lp = self.process_prompt(prompt) + l, l2 = disp_str(line) + wrapcount = (wlen(l) + lp) // self.console.width + if wrapcount == 0: + offset += ll + 1 # Takes all of the line plus the newline + last_refresh_line_end_offsets.append(offset) + screen.append(prompt + l) + screeninfo.append((lp, l2)) + else: + i = 0 + while l: + prelen = lp if i == 0 else 0 + index_to_wrap_before = 0 + column = 0 + for character_width in l2: + if column + character_width >= self.console.width - prelen: + break + index_to_wrap_before += 1 + column += character_width + pre = prompt if i == 0 else "" + if len(l) > index_to_wrap_before: + offset += index_to_wrap_before + post = "\\" + after = [1] + else: + offset += index_to_wrap_before + 1 # Takes the newline + post = "" + after = [] + last_refresh_line_end_offsets.append(offset) + screen.append(pre + l[:index_to_wrap_before] + post) + screeninfo.append((prelen, l2[:index_to_wrap_before] + after)) + l = l[index_to_wrap_before:] + l2 = l2[index_to_wrap_before:] + i += 1 + self.screeninfo = screeninfo + self.cxy = self.pos2xy() + if self.msg: + for mline in self.msg.split("\n"): + screen.append(mline) + screeninfo.append((0, [])) + + self.last_refresh_cache.update_cache(self, screen, screeninfo) + return screen + + @staticmethod + def process_prompt(prompt: str) -> tuple[str, int]: + """Process the prompt. + + This means calculate the length of the prompt. The character \x01 + and \x02 are used to bracket ANSI control sequences and need to be + excluded from the length calculation. So also a copy of the prompt + is returned with these control characters removed.""" + + # The logic below also ignores the length of common escape + # sequences if they were not explicitly within \x01...\x02. + # They are CSI (or ANSI) sequences ( ESC [ ... LETTER ) + + # wlen from utils already excludes ANSI_ESCAPE_SEQUENCE chars, + # which breaks the logic below so we redefine it here. + def wlen(s: str) -> int: + return sum(str_width(i) for i in s) + + out_prompt = "" + l = wlen(prompt) + pos = 0 + while True: + s = prompt.find("\x01", pos) + if s == -1: + break + e = prompt.find("\x02", s) + if e == -1: + break + # Found start and end brackets, subtract from string length + l = l - (e - s + 1) + keep = prompt[pos:s] + l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep))) + out_prompt += keep + prompt[s + 1 : e] + pos = e + 1 + keep = prompt[pos:] + l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep))) + out_prompt += keep + return out_prompt, l + + def bow(self, p: int | None = None) -> int: + """Return the 0-based index of the word break preceding p most + immediately. + + p defaults to self.pos; word boundaries are determined using + self.syntax_table.""" + if p is None: + p = self.pos + st = self.syntax_table + b = self.buffer + p -= 1 + while p >= 0 and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD: + p -= 1 + while p >= 0 and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD: + p -= 1 + return p + 1 + + def eow(self, p: int | None = None) -> int: + """Return the 0-based index of the word break following p most + immediately. + + p defaults to self.pos; word boundaries are determined using + self.syntax_table.""" + if p is None: + p = self.pos + st = self.syntax_table + b = self.buffer + while p < len(b) and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD: + p += 1 + while p < len(b) and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD: + p += 1 + return p + + def bol(self, p: int | None = None) -> int: + """Return the 0-based index of the line break preceding p most + immediately. + + p defaults to self.pos.""" + if p is None: + p = self.pos + b = self.buffer + p -= 1 + while p >= 0 and b[p] != "\n": + p -= 1 + return p + 1 + + def eol(self, p: int | None = None) -> int: + """Return the 0-based index of the line break following p most + immediately. + + p defaults to self.pos.""" + if p is None: + p = self.pos + b = self.buffer + while p < len(b) and b[p] != "\n": + p += 1 + return p + + def max_column(self, y: int) -> int: + """Return the last x-offset for line y""" + return self.screeninfo[y][0] + sum(self.screeninfo[y][1]) + + def max_row(self) -> int: + return len(self.screeninfo) - 1 + + def get_arg(self, default: int = 1) -> int: + """Return any prefix argument that the user has supplied, + returning 'default' if there is None. Defaults to 1. + """ + if self.arg is None: + return default + return self.arg + + def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: + """Return what should be in the left-hand margin for line + 'lineno'.""" + if self.arg is not None and cursor_on_line: + prompt = f"(arg: {self.arg}) " + elif self.paste_mode and not self.in_bracketed_paste: + prompt = "(paste) " + elif "\n" in self.buffer: + if lineno == 0: + prompt = self.ps2 + elif self.ps4 and lineno == self.buffer.count("\n"): + prompt = self.ps4 + else: + prompt = self.ps3 + else: + prompt = self.ps1 + + if self.can_colorize: + prompt = f"{ANSIColors.BOLD_MAGENTA}{prompt}{ANSIColors.RESET}" + return prompt + + def push_input_trans(self, itrans: input.KeymapTranslator) -> None: + self.input_trans_stack.append(self.input_trans) + self.input_trans = itrans + + def pop_input_trans(self) -> None: + self.input_trans = self.input_trans_stack.pop() + + def setpos_from_xy(self, x: int, y: int) -> None: + """Set pos according to coordinates x, y""" + pos = 0 + i = 0 + while i < y: + prompt_len, character_widths = self.screeninfo[i] + offset = len(character_widths) - character_widths.count(0) + in_wrapped_line = prompt_len + sum(character_widths) >= self.console.width + if in_wrapped_line: + pos += offset - 1 # -1 cause backslash is not in buffer + else: + pos += offset + 1 # +1 cause newline is in buffer + i += 1 + + j = 0 + cur_x = self.screeninfo[i][0] + while cur_x < x: + if self.screeninfo[i][1][j] == 0: + continue + cur_x += self.screeninfo[i][1][j] + j += 1 + pos += 1 + + self.pos = pos + + def pos2xy(self) -> tuple[int, int]: + """Return the x, y coordinates of position 'pos'.""" + # this *is* incomprehensible, yes. + y = 0 + pos = self.pos + assert 0 <= pos <= len(self.buffer) + if pos == len(self.buffer): + y = len(self.screeninfo) - 1 + p, l2 = self.screeninfo[y] + return p + sum(l2) + l2.count(0), y + + for p, l2 in self.screeninfo: + l = len(l2) - l2.count(0) + in_wrapped_line = p + sum(l2) >= self.console.width + offset = l - 1 if in_wrapped_line else l # need to remove backslash + if offset >= pos: + break + + if p + sum(l2) >= self.console.width: + pos -= l - 1 # -1 cause backslash is not in buffer + else: + pos -= l + 1 # +1 cause newline is in buffer + y += 1 + return p + sum(l2[:pos]), y + + def insert(self, text: str | list[str]) -> None: + """Insert 'text' at the insertion point.""" + self.buffer[self.pos : self.pos] = list(text) + self.pos += len(text) + self.dirty = True + + def update_cursor(self) -> None: + """Move the cursor to reflect changes in self.pos""" + self.cxy = self.pos2xy() + self.console.move_cursor(*self.cxy) + + def after_command(self, cmd: Command) -> None: + """This function is called to allow post command cleanup.""" + if getattr(cmd, "kills_digit_arg", True): + if self.arg is not None: + self.dirty = True + self.arg = None + + def prepare(self) -> None: + """Get ready to run. Call restore when finished. You must not + write to the console in between the calls to prepare and + restore.""" + try: + self.console.prepare() + self.arg = None + self.finished = False + del self.buffer[:] + self.pos = 0 + self.dirty = True + self.last_command = None + self.calc_screen() + except BaseException: + self.restore() + raise + + while self.scheduled_commands: + cmd = self.scheduled_commands.pop() + self.do_cmd((cmd, [])) + + def last_command_is(self, cls: type) -> bool: + if not self.last_command: + return False + return issubclass(cls, self.last_command) + + def restore(self) -> None: + """Clean up after a run.""" + self.console.restore() + + @contextmanager + def suspend(self) -> SimpleContextManager: + """A context manager to delegate to another reader.""" + prev_state = {f.name: getattr(self, f.name) for f in fields(self)} + try: + self.restore() + yield + finally: + for arg in ("msg", "ps1", "ps2", "ps3", "ps4", "paste_mode"): + setattr(self, arg, prev_state[arg]) + self.prepare() + + def finish(self) -> None: + """Called when a command signals that we're finished.""" + pass + + def error(self, msg: str = "none") -> None: + self.msg = "! " + msg + " " + self.dirty = True + self.console.beep() + + def update_screen(self) -> None: + if self.dirty: + self.refresh() + + def refresh(self) -> None: + """Recalculate and refresh the screen.""" + if self.in_bracketed_paste and self.buffer and not self.buffer[-1] == "\n": + return + + # this call sets up self.cxy, so call it first. + self.screen = self.calc_screen() + self.console.refresh(self.screen, self.cxy) + self.dirty = False + + def do_cmd(self, cmd: tuple[str, list[str]]) -> None: + """`cmd` is a tuple of "event_name" and "event", which in the current + implementation is always just the "buffer" which happens to be a list + of single-character strings.""" + + trace("received command {cmd}", cmd=cmd) + if isinstance(cmd[0], str): + command_type = self.commands.get(cmd[0], commands.invalid_command) + elif isinstance(cmd[0], type): + command_type = cmd[0] + else: + return # nothing to do + + command = command_type(self, *cmd) # type: ignore[arg-type] + command.do() + + self.after_command(command) + + if self.dirty: + self.refresh() + else: + self.update_cursor() + + if not isinstance(cmd, commands.digit_arg): + self.last_command = command_type + + self.finished = bool(command.finish) + if self.finished: + self.console.finish() + self.finish() + + def run_hooks(self) -> None: + threading_hook = self.threading_hook + if threading_hook is None and 'threading' in sys.modules: + from ._threading_handler import install_threading_hook + install_threading_hook(self) + if threading_hook is not None: + try: + threading_hook() + except Exception: + pass + + input_hook = self.console.input_hook + if input_hook: + try: + input_hook() + except Exception: + pass + + def handle1(self, block: bool = True) -> bool: + """Handle a single event. Wait as long as it takes if block + is true (the default), otherwise return False if no event is + pending.""" + + if self.msg: + self.msg = "" + self.dirty = True + + while True: + # We use the same timeout as in readline.c: 100ms + self.run_hooks() + self.console.wait(100) + event = self.console.get_event(block=False) + if not event: + if block: + continue + return False + + translate = True + + if event.evt == "key": + self.input_trans.push(event) + elif event.evt == "scroll": + self.refresh() + elif event.evt == "resize": + self.refresh() + else: + translate = False + + if translate: + cmd = self.input_trans.get() + else: + cmd = [event.evt, event.data] + + if cmd is None: + if block: + continue + return False + + self.do_cmd(cmd) + return True + + def push_char(self, char: int | bytes) -> None: + self.console.push_char(char) + self.handle1(block=False) + + def readline(self, startup_hook: Callback | None = None) -> str: + """Read a line. The implementation of this method also shows + how to drive Reader if you want more control over the event + loop.""" + self.prepare() + try: + if startup_hook is not None: + startup_hook() + self.refresh() + while not self.finished: + self.handle1() + return self.get_unicode() + + finally: + self.restore() + + def bind(self, spec: KeySpec, command: CommandName) -> None: + self.keymap = self.keymap + ((spec, command),) + self.input_trans = input.KeymapTranslator( + self.keymap, invalid_cls="invalid-key", character_cls="self-insert" + ) + + def get_unicode(self) -> str: + """Return the current buffer as a unicode string.""" + return "".join(self.buffer) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py new file mode 100644 index 00000000000000..888185eb03be66 --- /dev/null +++ b/Lib/_pyrepl/readline.py @@ -0,0 +1,598 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Alex Gaynor +# Antonio Cuni +# Armin Rigo +# Holger Krekel +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""A compatibility wrapper reimplementing the 'readline' standard module +on top of pyrepl. Not all functionalities are supported. Contains +extensions for multiline input. +""" + +from __future__ import annotations + +import warnings +from dataclasses import dataclass, field + +import os +from site import gethistoryfile # type: ignore[attr-defined] +import sys +from rlcompleter import Completer as RLCompleter + +from . import commands, historical_reader +from .completing_reader import CompletingReader +from .console import Console as ConsoleType + +Console: type[ConsoleType] +_error: tuple[type[Exception], ...] | type[Exception] +try: + from .unix_console import UnixConsole as Console, _error +except ImportError: + from .windows_console import WindowsConsole as Console, _error + +ENCODING = sys.getdefaultencoding() or "latin1" + + +# types +Command = commands.Command +from collections.abc import Callable, Collection +from .types import Callback, Completer, KeySpec, CommandName + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import Any, Mapping + + +MoreLinesCallable = Callable[[str], bool] + + +__all__ = [ + "add_history", + "clear_history", + "get_begidx", + "get_completer", + "get_completer_delims", + "get_current_history_length", + "get_endidx", + "get_history_item", + "get_history_length", + "get_line_buffer", + "insert_text", + "parse_and_bind", + "read_history_file", + # "read_init_file", + # "redisplay", + "remove_history_item", + "replace_history_item", + "set_auto_history", + "set_completer", + "set_completer_delims", + "set_history_length", + # "set_pre_input_hook", + "set_startup_hook", + "write_history_file", + # ---- multiline extensions ---- + "multiline_input", +] + +# ____________________________________________________________ + +@dataclass +class ReadlineConfig: + readline_completer: Completer | None = None + completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?") + + +@dataclass(kw_only=True) +class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader): + # Class fields + assume_immutable_completions = False + use_brackets = False + sort_in_column = True + + # Instance fields + config: ReadlineConfig + more_lines: MoreLinesCallable | None = None + last_used_indentation: str | None = None + + def __post_init__(self) -> None: + super().__post_init__() + self.commands["maybe_accept"] = maybe_accept + self.commands["maybe-accept"] = maybe_accept + self.commands["backspace_dedent"] = backspace_dedent + self.commands["backspace-dedent"] = backspace_dedent + + def error(self, msg: str = "none") -> None: + pass # don't show error messages by default + + def get_stem(self) -> str: + b = self.buffer + p = self.pos - 1 + completer_delims = self.config.completer_delims + while p >= 0 and b[p] not in completer_delims: + p -= 1 + return "".join(b[p + 1 : self.pos]) + + def get_completions(self, stem: str) -> list[str]: + if len(stem) == 0 and self.more_lines is not None: + b = self.buffer + p = self.pos + while p > 0 and b[p - 1] != "\n": + p -= 1 + num_spaces = 4 - ((self.pos - p) % 4) + return [" " * num_spaces] + result = [] + function = self.config.readline_completer + if function is not None: + try: + stem = str(stem) # rlcompleter.py seems to not like unicode + except UnicodeEncodeError: + pass # but feed unicode anyway if we have no choice + state = 0 + while True: + try: + next = function(stem, state) + except Exception: + break + if not isinstance(next, str): + break + result.append(next) + state += 1 + # emulate the behavior of the standard readline that sorts + # the completions before displaying them. + result.sort() + return result + + def get_trimmed_history(self, maxlength: int) -> list[str]: + if maxlength >= 0: + cut = len(self.history) - maxlength + if cut < 0: + cut = 0 + else: + cut = 0 + return self.history[cut:] + + def update_last_used_indentation(self) -> None: + indentation = _get_first_indentation(self.buffer) + if indentation is not None: + self.last_used_indentation = indentation + + # --- simplified support for reading multiline Python statements --- + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r"\n", "maybe-accept"), + (r"\", "backspace-dedent"), + ) + + def after_command(self, cmd: Command) -> None: + super().after_command(cmd) + if self.more_lines is None: + # Force single-line input if we are in raw_input() mode. + # Although there is no direct way to add a \n in this mode, + # multiline buffers can still show up using various + # commands, e.g. navigating the history. + try: + index = self.buffer.index("\n") + except ValueError: + pass + else: + self.buffer = self.buffer[:index] + if self.pos > len(self.buffer): + self.pos = len(self.buffer) + + +def set_auto_history(_should_auto_add_history: bool) -> None: + """Enable or disable automatic history""" + historical_reader.should_auto_add_history = bool(_should_auto_add_history) + + +def _get_this_line_indent(buffer: list[str], pos: int) -> int: + indent = 0 + while pos > 0 and buffer[pos - 1] in " \t": + indent += 1 + pos -= 1 + if pos > 0 and buffer[pos - 1] == "\n": + return indent + return 0 + + +def _get_previous_line_indent(buffer: list[str], pos: int) -> tuple[int, int | None]: + prevlinestart = pos + while prevlinestart > 0 and buffer[prevlinestart - 1] != "\n": + prevlinestart -= 1 + prevlinetext = prevlinestart + while prevlinetext < pos and buffer[prevlinetext] in " \t": + prevlinetext += 1 + if prevlinetext == pos: + indent = None + else: + indent = prevlinetext - prevlinestart + return prevlinestart, indent + + +def _get_first_indentation(buffer: list[str]) -> str | None: + indented_line_start = None + for i in range(len(buffer)): + if (i < len(buffer) - 1 + and buffer[i] == "\n" + and buffer[i + 1] in " \t" + ): + indented_line_start = i + 1 + elif indented_line_start is not None and buffer[i] not in " \t\n": + return ''.join(buffer[indented_line_start : i]) + return None + + +def _should_auto_indent(buffer: list[str], pos: int) -> bool: + # check if last character before "pos" is a colon, ignoring + # whitespaces and comments. + last_char = None + while pos > 0: + pos -= 1 + if last_char is None: + if buffer[pos] not in " \t\n#": # ignore whitespaces and comments + last_char = buffer[pos] + else: + # even if we found a non-whitespace character before + # original pos, we keep going back until newline is reached + # to make sure we ignore comments + if buffer[pos] == "\n": + break + if buffer[pos] == "#": + last_char = None + return last_char == ":" + + +class maybe_accept(commands.Command): + def do(self) -> None: + r: ReadlineAlikeReader + r = self.reader # type: ignore[assignment] + r.dirty = True # this is needed to hide the completion menu, if visible + + if self.reader.in_bracketed_paste: + r.insert("\n") + return + + # if there are already several lines and the cursor + # is not on the last one, always insert a new \n. + text = r.get_unicode() + + if "\n" in r.buffer[r.pos :] or ( + r.more_lines is not None and r.more_lines(text) + ): + def _newline_before_pos(): + before_idx = r.pos - 1 + while before_idx > 0 and text[before_idx].isspace(): + before_idx -= 1 + return text[before_idx : r.pos].count("\n") > 0 + + # if there's already a new line before the cursor then + # even if the cursor is followed by whitespace, we assume + # the user is trying to terminate the block + if _newline_before_pos() and text[r.pos:].isspace(): + self.finish = True + return + + # auto-indent the next line like the previous line + prevlinestart, indent = _get_previous_line_indent(r.buffer, r.pos) + r.insert("\n") + if not self.reader.paste_mode: + if indent: + for i in range(prevlinestart, prevlinestart + indent): + r.insert(r.buffer[i]) + r.update_last_used_indentation() + if _should_auto_indent(r.buffer, r.pos): + if r.last_used_indentation is not None: + indentation = r.last_used_indentation + else: + # default + indentation = " " * 4 + r.insert(indentation) + elif not self.reader.paste_mode: + self.finish = True + else: + r.insert("\n") + + +class backspace_dedent(commands.Command): + def do(self) -> None: + r = self.reader + b = r.buffer + if r.pos > 0: + repeat = 1 + if b[r.pos - 1] != "\n": + indent = _get_this_line_indent(b, r.pos) + if indent > 0: + ls = r.pos - indent + while ls > 0: + ls, pi = _get_previous_line_indent(b, ls - 1) + if pi is not None and pi < indent: + repeat = indent - pi + break + r.pos -= repeat + del b[r.pos : r.pos + repeat] + r.dirty = True + else: + self.reader.error("can't backspace at start") + + +# ____________________________________________________________ + + +@dataclass(slots=True) +class _ReadlineWrapper: + f_in: int = -1 + f_out: int = -1 + reader: ReadlineAlikeReader | None = field(default=None, repr=False) + saved_history_length: int = -1 + startup_hook: Callback | None = None + config: ReadlineConfig = field(default_factory=ReadlineConfig, repr=False) + + def __post_init__(self) -> None: + if self.f_in == -1: + self.f_in = os.dup(0) + if self.f_out == -1: + self.f_out = os.dup(1) + + def get_reader(self) -> ReadlineAlikeReader: + if self.reader is None: + console = Console(self.f_in, self.f_out, encoding=ENCODING) + self.reader = ReadlineAlikeReader(console=console, config=self.config) + return self.reader + + def input(self, prompt: object = "") -> str: + try: + reader = self.get_reader() + except _error: + assert raw_input is not None + return raw_input(prompt) + prompt_str = str(prompt) + reader.ps1 = prompt_str + sys.audit("builtins.input", prompt_str) + result = reader.readline(startup_hook=self.startup_hook) + sys.audit("builtins.input/result", result) + return result + + def multiline_input(self, more_lines: MoreLinesCallable, ps1: str, ps2: str) -> str: + """Read an input on possibly multiple lines, asking for more + lines as long as 'more_lines(unicodetext)' returns an object whose + boolean value is true. + """ + reader = self.get_reader() + saved = reader.more_lines + try: + reader.more_lines = more_lines + reader.ps1 = ps1 + reader.ps2 = ps1 + reader.ps3 = ps2 + reader.ps4 = "" + with warnings.catch_warnings(action="ignore"): + return reader.readline() + finally: + reader.more_lines = saved + reader.paste_mode = False + + def parse_and_bind(self, string: str) -> None: + pass # XXX we don't support parsing GNU-readline-style init files + + def set_completer(self, function: Completer | None = None) -> None: + self.config.readline_completer = function + + def get_completer(self) -> Completer | None: + return self.config.readline_completer + + def set_completer_delims(self, delimiters: Collection[str]) -> None: + self.config.completer_delims = frozenset(delimiters) + + def get_completer_delims(self) -> str: + return "".join(sorted(self.config.completer_delims)) + + def _histline(self, line: str) -> str: + line = line.rstrip("\n") + return line + + def get_history_length(self) -> int: + return self.saved_history_length + + def set_history_length(self, length: int) -> None: + self.saved_history_length = length + + def get_current_history_length(self) -> int: + return len(self.get_reader().history) + + def read_history_file(self, filename: str = gethistoryfile()) -> None: + # multiline extension (really a hack) for the end of lines that + # are actually continuations inside a single multiline_input() + # history item: we use \r\n instead of just \n. If the history + # file is passed to GNU readline, the extra \r are just ignored. + history = self.get_reader().history + + with open(os.path.expanduser(filename), 'rb') as f: + is_editline = f.readline().startswith(b"_HiStOrY_V2_") + if is_editline: + encoding = "unicode-escape" + else: + f.seek(0) + encoding = "utf-8" + + lines = [line.decode(encoding, errors='replace') for line in f.read().split(b'\n')] + buffer = [] + for line in lines: + if line.endswith("\r"): + buffer.append(line+'\n') + else: + line = self._histline(line) + if buffer: + line = self._histline("".join(buffer).replace("\r", "") + line) + del buffer[:] + if line: + history.append(line) + + def write_history_file(self, filename: str = gethistoryfile()) -> None: + maxlength = self.saved_history_length + history = self.get_reader().get_trimmed_history(maxlength) + f = open(os.path.expanduser(filename), "w", + encoding="utf-8", newline="\n") + with f: + for entry in history: + entry = entry.replace("\n", "\r\n") # multiline history support + f.write(entry + "\n") + + def clear_history(self) -> None: + del self.get_reader().history[:] + + def get_history_item(self, index: int) -> str | None: + history = self.get_reader().history + if 1 <= index <= len(history): + return history[index - 1] + else: + return None # like readline.c + + def remove_history_item(self, index: int) -> None: + history = self.get_reader().history + if 0 <= index < len(history): + del history[index] + else: + raise ValueError("No history item at position %d" % index) + # like readline.c + + def replace_history_item(self, index: int, line: str) -> None: + history = self.get_reader().history + if 0 <= index < len(history): + history[index] = self._histline(line) + else: + raise ValueError("No history item at position %d" % index) + # like readline.c + + def add_history(self, line: str) -> None: + self.get_reader().history.append(self._histline(line)) + + def set_startup_hook(self, function: Callback | None = None) -> None: + self.startup_hook = function + + def get_line_buffer(self) -> str: + return self.get_reader().get_unicode() + + def _get_idxs(self) -> tuple[int, int]: + start = cursor = self.get_reader().pos + buf = self.get_line_buffer() + for i in range(cursor - 1, -1, -1): + if buf[i] in self.get_completer_delims(): + break + start = i + return start, cursor + + def get_begidx(self) -> int: + return self._get_idxs()[0] + + def get_endidx(self) -> int: + return self._get_idxs()[1] + + def insert_text(self, text: str) -> None: + self.get_reader().insert(text) + + +_wrapper = _ReadlineWrapper() + +# ____________________________________________________________ +# Public API + +parse_and_bind = _wrapper.parse_and_bind +set_completer = _wrapper.set_completer +get_completer = _wrapper.get_completer +set_completer_delims = _wrapper.set_completer_delims +get_completer_delims = _wrapper.get_completer_delims +get_history_length = _wrapper.get_history_length +set_history_length = _wrapper.set_history_length +get_current_history_length = _wrapper.get_current_history_length +read_history_file = _wrapper.read_history_file +write_history_file = _wrapper.write_history_file +clear_history = _wrapper.clear_history +get_history_item = _wrapper.get_history_item +remove_history_item = _wrapper.remove_history_item +replace_history_item = _wrapper.replace_history_item +add_history = _wrapper.add_history +set_startup_hook = _wrapper.set_startup_hook +get_line_buffer = _wrapper.get_line_buffer +get_begidx = _wrapper.get_begidx +get_endidx = _wrapper.get_endidx +insert_text = _wrapper.insert_text + +# Extension +multiline_input = _wrapper.multiline_input + +# Internal hook +_get_reader = _wrapper.get_reader + +# ____________________________________________________________ +# Stubs + + +def _make_stub(_name: str, _ret: object) -> None: + def stub(*args: object, **kwds: object) -> None: + import warnings + + warnings.warn("readline.%s() not implemented" % _name, stacklevel=2) + + stub.__name__ = _name + globals()[_name] = stub + + +for _name, _ret in [ + ("read_init_file", None), + ("redisplay", None), + ("set_pre_input_hook", None), +]: + assert _name not in globals(), _name + _make_stub(_name, _ret) + +# ____________________________________________________________ + + +def _setup(namespace: Mapping[str, Any]) -> None: + global raw_input + if raw_input is not None: + return # don't run _setup twice + + try: + f_in = sys.stdin.fileno() + f_out = sys.stdout.fileno() + except (AttributeError, ValueError): + return + if not os.isatty(f_in) or not os.isatty(f_out): + return + + _wrapper.f_in = f_in + _wrapper.f_out = f_out + + # set up namespace in rlcompleter, which requires it to be a bona fide dict + if not isinstance(namespace, dict): + namespace = dict(namespace) + _wrapper.config.readline_completer = RLCompleter(namespace).complete + + # this is not really what readline.c does. Better than nothing I guess + import builtins + raw_input = builtins.input + builtins.input = _wrapper.input + + +raw_input: Callable[[object], str] | None = None diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py new file mode 100644 index 00000000000000..a065174ad42fb6 --- /dev/null +++ b/Lib/_pyrepl/simple_interact.py @@ -0,0 +1,167 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""This is an alternative to python_reader which tries to emulate +the CPython prompt as closely as possible, with the exception of +allowing multiline input and multiline history entries. +""" + +from __future__ import annotations + +import _sitebuiltins +import linecache +import functools +import os +import sys +import code + +from .readline import _get_reader, multiline_input + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import Any + + +_error: tuple[type[Exception], ...] | type[Exception] +try: + from .unix_console import _error +except ModuleNotFoundError: + from .windows_console import _error + +def check() -> str: + """Returns the error message if there is a problem initializing the state.""" + try: + _get_reader() + except _error as e: + if term := os.environ.get("TERM", ""): + term = f"; TERM={term}" + return str(str(e) or repr(e) or "unknown error") + term + return "" + + +def _strip_final_indent(text: str) -> str: + # kill spaces and tabs at the end, but only if they follow '\n'. + # meant to remove the auto-indentation only (although it would of + # course also remove explicitly-added indentation). + short = text.rstrip(" \t") + n = len(short) + if n > 0 and text[n - 1] == "\n": + return short + return text + + +def _clear_screen(): + reader = _get_reader() + reader.scheduled_commands.append("clear_screen") + + +REPL_COMMANDS = { + "exit": _sitebuiltins.Quitter('exit', ''), + "quit": _sitebuiltins.Quitter('quit' ,''), + "copyright": _sitebuiltins._Printer('copyright', sys.copyright), + "help": _sitebuiltins._Helper(), + "clear": _clear_screen, + "\x1a": _sitebuiltins.Quitter('\x1a', ''), +} + + +def _more_lines(console: code.InteractiveConsole, unicodetext: str) -> bool: + # ooh, look at the hack: + src = _strip_final_indent(unicodetext) + try: + code = console.compile(src, "", "single") + except (OverflowError, SyntaxError, ValueError): + lines = src.splitlines(keepends=True) + if len(lines) == 1: + return False + + last_line = lines[-1] + was_indented = last_line.startswith((" ", "\t")) + not_empty = last_line.strip() != "" + incomplete = not last_line.endswith("\n") + return (was_indented or not_empty) and incomplete + else: + return code is None + + +def run_multiline_interactive_console( + console: code.InteractiveConsole, + *, + future_flags: int = 0, +) -> None: + from .readline import _setup + _setup(console.locals) + if future_flags: + console.compile.compiler.flags |= future_flags + + more_lines = functools.partial(_more_lines, console) + input_n = 0 + + def maybe_run_command(statement: str) -> bool: + statement = statement.strip() + if statement in console.locals or statement not in REPL_COMMANDS: + return False + + reader = _get_reader() + reader.history.pop() # skip internal commands in history + command = REPL_COMMANDS[statement] + if callable(command): + # Make sure that history does not change because of commands + with reader.suspend_history(): + command() + return True + return False + + while True: + try: + try: + sys.stdout.flush() + except Exception: + pass + + ps1 = getattr(sys, "ps1", ">>> ") + ps2 = getattr(sys, "ps2", "... ") + try: + statement = multiline_input(more_lines, ps1, ps2) + except EOFError: + break + + if maybe_run_command(statement): + continue + + input_name = f"" + linecache._register_code(input_name, statement, "") # type: ignore[attr-defined] + more = console.push(_strip_final_indent(statement), filename=input_name, _symbol="single") # type: ignore[call-arg] + assert not more + input_n += 1 + except KeyboardInterrupt: + r = _get_reader() + if r.input_trans is r.isearch_trans: + r.do_cmd(("isearch-end", [""])) + r.pos = len(r.get_unicode()) + r.dirty = True + r.refresh() + r.in_bracketed_paste = False + console.write("\nKeyboardInterrupt\n") + console.resetbuffer() + except MemoryError: + console.write("\nMemoryError\n") + console.resetbuffer() diff --git a/Lib/_pyrepl/trace.py b/Lib/_pyrepl/trace.py new file mode 100644 index 00000000000000..a8eb2433cd3cce --- /dev/null +++ b/Lib/_pyrepl/trace.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import os + +# types +if False: + from typing import IO + + +trace_file: IO[str] | None = None +if trace_filename := os.environ.get("PYREPL_TRACE"): + trace_file = open(trace_filename, "a") + + +def trace(line: str, *k: object, **kw: object) -> None: + if trace_file is None: + return + if k or kw: + line = line.format(*k, **kw) + trace_file.write(line + "\n") + trace_file.flush() diff --git a/Lib/_pyrepl/types.py b/Lib/_pyrepl/types.py new file mode 100644 index 00000000000000..f9d48b828c720b --- /dev/null +++ b/Lib/_pyrepl/types.py @@ -0,0 +1,8 @@ +from collections.abc import Callable, Iterator + +Callback = Callable[[], object] +SimpleContextManager = Iterator[None] +KeySpec = str # like r"\C-c" +CommandName = str # like "interrupt" +EventTuple = tuple[CommandName, str] +Completer = Callable[[str, int], str | None] diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py new file mode 100644 index 00000000000000..96379bc20f3357 --- /dev/null +++ b/Lib/_pyrepl/unix_console.py @@ -0,0 +1,810 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +import errno +import os +import re +import select +import signal +import struct +import termios +import time +import platform +from fcntl import ioctl + +from . import curses +from .console import Console, Event +from .fancy_termios import tcgetattr, tcsetattr +from .trace import trace +from .unix_eventqueue import EventQueue +from .utils import wlen + + +TYPE_CHECKING = False + +# types +if TYPE_CHECKING: + from typing import IO, Literal, overload +else: + overload = lambda func: None + + +class InvalidTerminal(RuntimeError): + pass + + +_error = (termios.error, curses.error, InvalidTerminal) + +SIGWINCH_EVENT = "repaint" + +FIONREAD = getattr(termios, "FIONREAD", None) +TIOCGWINSZ = getattr(termios, "TIOCGWINSZ", None) + +# ------------ start of baudrate definitions ------------ + +# Add (possibly) missing baudrates (check termios man page) to termios + + +def add_baudrate_if_supported(dictionary: dict[int, int], rate: int) -> None: + baudrate_name = "B%d" % rate + if hasattr(termios, baudrate_name): + dictionary[getattr(termios, baudrate_name)] = rate + + +# Check the termios man page (Line speed) to know where these +# values come from. +potential_baudrates = [ + 0, + 110, + 115200, + 1200, + 134, + 150, + 1800, + 19200, + 200, + 230400, + 2400, + 300, + 38400, + 460800, + 4800, + 50, + 57600, + 600, + 75, + 9600, +] + +ratedict: dict[int, int] = {} +for rate in potential_baudrates: + add_baudrate_if_supported(ratedict, rate) + +# Clean up variables to avoid unintended usage +del rate, add_baudrate_if_supported + +# ------------ end of baudrate definitions ------------ + +delayprog = re.compile(b"\\$<([0-9]+)((?:/|\\*){0,2})>") + +try: + poll: type[select.poll] = select.poll +except AttributeError: + # this is exactly the minimum necessary to support what we + # do with poll objects + class MinimalPoll: + def __init__(self): + pass + + def register(self, fd, flag): + self.fd = fd + # note: The 'timeout' argument is received as *milliseconds* + def poll(self, timeout: float | None = None) -> list[int]: + if timeout is None: + r, w, e = select.select([self.fd], [], []) + else: + r, w, e = select.select([self.fd], [], [], timeout/1000) + return r + + poll = MinimalPoll # type: ignore[assignment] + + +class UnixConsole(Console): + def __init__( + self, + f_in: IO[bytes] | int = 0, + f_out: IO[bytes] | int = 1, + term: str = "", + encoding: str = "", + ): + """ + Initialize the UnixConsole. + + Parameters: + - f_in (int or file-like object): Input file descriptor or object. + - f_out (int or file-like object): Output file descriptor or object. + - term (str): Terminal name. + - encoding (str): Encoding to use for I/O operations. + """ + super().__init__(f_in, f_out, term, encoding) + + self.pollob = poll() + self.pollob.register(self.input_fd, select.POLLIN) + self.input_buffer = b"" + self.input_buffer_pos = 0 + curses.setupterm(term or None, self.output_fd) + self.term = term + + @overload + def _my_getstr(cap: str, optional: Literal[False] = False) -> bytes: ... + + @overload + def _my_getstr(cap: str, optional: bool) -> bytes | None: ... + + def _my_getstr(cap: str, optional: bool = False) -> bytes | None: + r = curses.tigetstr(cap) + if not optional and r is None: + raise InvalidTerminal( + f"terminal doesn't have the required {cap} capability" + ) + return r + + self._bel = _my_getstr("bel") + self._civis = _my_getstr("civis", optional=True) + self._clear = _my_getstr("clear") + self._cnorm = _my_getstr("cnorm", optional=True) + self._cub = _my_getstr("cub", optional=True) + self._cub1 = _my_getstr("cub1", optional=True) + self._cud = _my_getstr("cud", optional=True) + self._cud1 = _my_getstr("cud1", optional=True) + self._cuf = _my_getstr("cuf", optional=True) + self._cuf1 = _my_getstr("cuf1", optional=True) + self._cup = _my_getstr("cup") + self._cuu = _my_getstr("cuu", optional=True) + self._cuu1 = _my_getstr("cuu1", optional=True) + self._dch1 = _my_getstr("dch1", optional=True) + self._dch = _my_getstr("dch", optional=True) + self._el = _my_getstr("el") + self._hpa = _my_getstr("hpa", optional=True) + self._ich = _my_getstr("ich", optional=True) + self._ich1 = _my_getstr("ich1", optional=True) + self._ind = _my_getstr("ind", optional=True) + self._pad = _my_getstr("pad", optional=True) + self._ri = _my_getstr("ri", optional=True) + self._rmkx = _my_getstr("rmkx", optional=True) + self._smkx = _my_getstr("smkx", optional=True) + + self.__setup_movement() + + self.event_queue = EventQueue(self.input_fd, self.encoding) + self.cursor_visible = 1 + + def more_in_buffer(self) -> bool: + return bool( + self.input_buffer + and self.input_buffer_pos < len(self.input_buffer) + ) + + def __read(self, n: int) -> bytes: + if not self.more_in_buffer(): + self.input_buffer = os.read(self.input_fd, 10000) + + ret = self.input_buffer[self.input_buffer_pos : self.input_buffer_pos + n] + self.input_buffer_pos += len(ret) + if self.input_buffer_pos >= len(self.input_buffer): + self.input_buffer = b"" + self.input_buffer_pos = 0 + return ret + + + def change_encoding(self, encoding: str) -> None: + """ + Change the encoding used for I/O operations. + + Parameters: + - encoding (str): New encoding to use. + """ + self.encoding = encoding + + def refresh(self, screen, c_xy): + """ + Refresh the console screen. + + Parameters: + - screen (list): List of strings representing the screen contents. + - c_xy (tuple): Cursor position (x, y) on the screen. + """ + cx, cy = c_xy + if not self.__gone_tall: + while len(self.screen) < min(len(screen), self.height): + self.__hide_cursor() + self.__move(0, len(self.screen) - 1) + self.__write("\n") + self.posxy = 0, len(self.screen) + self.screen.append("") + else: + while len(self.screen) < len(screen): + self.screen.append("") + + if len(screen) > self.height: + self.__gone_tall = 1 + self.__move = self.__move_tall + + px, py = self.posxy + old_offset = offset = self.__offset + height = self.height + + # we make sure the cursor is on the screen, and that we're + # using all of the screen if we can + if cy < offset: + offset = cy + elif cy >= offset + height: + offset = cy - height + 1 + elif offset > 0 and len(screen) < offset + height: + offset = max(len(screen) - height, 0) + screen.append("") + + oldscr = self.screen[old_offset : old_offset + height] + newscr = screen[offset : offset + height] + + # use hardware scrolling if we have it. + if old_offset > offset and self._ri: + self.__hide_cursor() + self.__write_code(self._cup, 0, 0) + self.posxy = 0, old_offset + for i in range(old_offset - offset): + self.__write_code(self._ri) + oldscr.pop(-1) + oldscr.insert(0, "") + elif old_offset < offset and self._ind: + self.__hide_cursor() + self.__write_code(self._cup, self.height - 1, 0) + self.posxy = 0, old_offset + self.height - 1 + for i in range(offset - old_offset): + self.__write_code(self._ind) + oldscr.pop(0) + oldscr.append("") + + self.__offset = offset + + for ( + y, + oldline, + newline, + ) in zip(range(offset, offset + height), oldscr, newscr): + if oldline != newline: + self.__write_changed_line(y, oldline, newline, px) + + y = len(newscr) + while y < len(oldscr): + self.__hide_cursor() + self.__move(0, y) + self.posxy = 0, y + self.__write_code(self._el) + y += 1 + + self.__show_cursor() + + self.screen = screen.copy() + self.move_cursor(cx, cy) + self.flushoutput() + + def move_cursor(self, x, y): + """ + Move the cursor to the specified position on the screen. + + Parameters: + - x (int): X coordinate. + - y (int): Y coordinate. + """ + if y < self.__offset or y >= self.__offset + self.height: + self.event_queue.insert(Event("scroll", None)) + else: + self.__move(x, y) + self.posxy = x, y + self.flushoutput() + + def prepare(self): + """ + Prepare the console for input/output operations. + """ + self.__svtermstate = tcgetattr(self.input_fd) + raw = self.__svtermstate.copy() + raw.iflag &= ~(termios.INPCK | termios.ISTRIP | termios.IXON) + raw.oflag &= ~(termios.OPOST) + raw.cflag &= ~(termios.CSIZE | termios.PARENB) + raw.cflag |= termios.CS8 + raw.iflag |= termios.BRKINT + raw.lflag &= ~(termios.ICANON | termios.ECHO | termios.IEXTEN) + raw.lflag |= termios.ISIG + raw.cc[termios.VMIN] = 1 + raw.cc[termios.VTIME] = 0 + tcsetattr(self.input_fd, termios.TCSADRAIN, raw) + + # In macOS terminal we need to deactivate line wrap via ANSI escape code + if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + os.write(self.output_fd, b"\033[?7l") + + self.screen = [] + self.height, self.width = self.getheightwidth() + + self.__buffer = [] + + self.posxy = 0, 0 + self.__gone_tall = 0 + self.__move = self.__move_short + self.__offset = 0 + + self.__maybe_write_code(self._smkx) + + try: + self.old_sigwinch = signal.signal(signal.SIGWINCH, self.__sigwinch) + except ValueError: + pass + + self.__enable_bracketed_paste() + + def restore(self): + """ + Restore the console to the default state + """ + self.__disable_bracketed_paste() + self.__maybe_write_code(self._rmkx) + self.flushoutput() + tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate) + + if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + os.write(self.output_fd, b"\033[?7h") + + if hasattr(self, "old_sigwinch"): + signal.signal(signal.SIGWINCH, self.old_sigwinch) + del self.old_sigwinch + + def push_char(self, char: int | bytes) -> None: + """ + Push a character to the console event queue. + """ + trace("push char {char!r}", char=char) + self.event_queue.push(char) + + def get_event(self, block: bool = True) -> Event | None: + """ + Get an event from the console event queue. + + Parameters: + - block (bool): Whether to block until an event is available. + + Returns: + - Event: Event object from the event queue. + """ + if not block and not self.wait(timeout=0): + return None + + while self.event_queue.empty(): + while True: + try: + self.push_char(self.__read(1)) + except OSError as err: + if err.errno == errno.EINTR: + if not self.event_queue.empty(): + return self.event_queue.get() + else: + continue + else: + raise + else: + break + return self.event_queue.get() + + def wait(self, timeout: float | None = None) -> bool: + """ + Wait for events on the console. + """ + return ( + not self.event_queue.empty() + or self.more_in_buffer() + or bool(self.pollob.poll(timeout)) + ) + + def set_cursor_vis(self, visible): + """ + Set the visibility of the cursor. + + Parameters: + - visible (bool): Visibility flag. + """ + if visible: + self.__show_cursor() + else: + self.__hide_cursor() + + if TIOCGWINSZ: + + def getheightwidth(self): + """ + Get the height and width of the console. + + Returns: + - tuple: Height and width of the console. + """ + try: + return int(os.environ["LINES"]), int(os.environ["COLUMNS"]) + except (KeyError, TypeError, ValueError): + try: + size = ioctl(self.input_fd, TIOCGWINSZ, b"\000" * 8) + except OSError: + return 25, 80 + height, width = struct.unpack("hhhh", size)[0:2] + if not height: + return 25, 80 + return height, width + + else: + + def getheightwidth(self): + """ + Get the height and width of the console. + + Returns: + - tuple: Height and width of the console. + """ + try: + return int(os.environ["LINES"]), int(os.environ["COLUMNS"]) + except (KeyError, TypeError, ValueError): + return 25, 80 + + def forgetinput(self): + """ + Discard any pending input on the console. + """ + termios.tcflush(self.input_fd, termios.TCIFLUSH) + + def flushoutput(self): + """ + Flush the output buffer. + """ + for text, iscode in self.__buffer: + if iscode: + self.__tputs(text) + else: + os.write(self.output_fd, text.encode(self.encoding, "replace")) + del self.__buffer[:] + + def finish(self): + """ + Finish console operations and flush the output buffer. + """ + y = len(self.screen) - 1 + while y >= 0 and not self.screen[y]: + y -= 1 + self.__move(0, min(y, self.height + self.__offset - 1)) + self.__write("\n\r") + self.flushoutput() + + def beep(self): + """ + Emit a beep sound. + """ + self.__maybe_write_code(self._bel) + self.flushoutput() + + if FIONREAD: + + def getpending(self): + """ + Get pending events from the console event queue. + + Returns: + - Event: Pending event from the event queue. + """ + e = Event("key", "", b"") + + while not self.event_queue.empty(): + e2 = self.event_queue.get() + e.data += e2.data + e.raw += e.raw + + amount = struct.unpack("i", ioctl(self.input_fd, FIONREAD, b"\0\0\0\0"))[0] + raw = self.__read(amount) + data = str(raw, self.encoding, "replace") + e.data += data + e.raw += raw + return e + + else: + + def getpending(self): + """ + Get pending events from the console event queue. + + Returns: + - Event: Pending event from the event queue. + """ + e = Event("key", "", b"") + + while not self.event_queue.empty(): + e2 = self.event_queue.get() + e.data += e2.data + e.raw += e.raw + + amount = 10000 + raw = self.__read(amount) + data = str(raw, self.encoding, "replace") + e.data += data + e.raw += raw + return e + + def clear(self): + """ + Clear the console screen. + """ + self.__write_code(self._clear) + self.__gone_tall = 1 + self.__move = self.__move_tall + self.posxy = 0, 0 + self.screen = [] + + @property + def input_hook(self): + try: + import posix + except ImportError: + return None + if posix._is_inputhook_installed(): + return posix._inputhook + + def __enable_bracketed_paste(self) -> None: + os.write(self.output_fd, b"\x1b[?2004h") + + def __disable_bracketed_paste(self) -> None: + os.write(self.output_fd, b"\x1b[?2004l") + + def __setup_movement(self): + """ + Set up the movement functions based on the terminal capabilities. + """ + if 0 and self._hpa: # hpa don't work in windows telnet :-( + self.__move_x = self.__move_x_hpa + elif self._cub and self._cuf: + self.__move_x = self.__move_x_cub_cuf + elif self._cub1 and self._cuf1: + self.__move_x = self.__move_x_cub1_cuf1 + else: + raise RuntimeError("insufficient terminal (horizontal)") + + if self._cuu and self._cud: + self.__move_y = self.__move_y_cuu_cud + elif self._cuu1 and self._cud1: + self.__move_y = self.__move_y_cuu1_cud1 + else: + raise RuntimeError("insufficient terminal (vertical)") + + if self._dch1: + self.dch1 = self._dch1 + elif self._dch: + self.dch1 = curses.tparm(self._dch, 1) + else: + self.dch1 = None + + if self._ich1: + self.ich1 = self._ich1 + elif self._ich: + self.ich1 = curses.tparm(self._ich, 1) + else: + self.ich1 = None + + self.__move = self.__move_short + + def __write_changed_line(self, y, oldline, newline, px_coord): + # this is frustrating; there's no reason to test (say) + # self.dch1 inside the loop -- but alternative ways of + # structuring this function are equally painful (I'm trying to + # avoid writing code generators these days...) + minlen = min(wlen(oldline), wlen(newline)) + x_pos = 0 + x_coord = 0 + + px_pos = 0 + j = 0 + for c in oldline: + if j >= px_coord: + break + j += wlen(c) + px_pos += 1 + + # reuse the oldline as much as possible, but stop as soon as we + # encounter an ESCAPE, because it might be the start of an escape + # sequence + while ( + x_coord < minlen + and oldline[x_pos] == newline[x_pos] + and newline[x_pos] != "\x1b" + ): + x_coord += wlen(newline[x_pos]) + x_pos += 1 + + # if we need to insert a single character right after the first detected change + if oldline[x_pos:] == newline[x_pos + 1 :] and self.ich1: + if ( + y == self.posxy[1] + and x_coord > self.posxy[0] + and oldline[px_pos:x_pos] == newline[px_pos + 1 : x_pos + 1] + ): + x_pos = px_pos + x_coord = px_coord + character_width = wlen(newline[x_pos]) + self.__move(x_coord, y) + self.__write_code(self.ich1) + self.__write(newline[x_pos]) + self.posxy = x_coord + character_width, y + + # if it's a single character change in the middle of the line + elif ( + x_coord < minlen + and oldline[x_pos + 1 :] == newline[x_pos + 1 :] + and wlen(oldline[x_pos]) == wlen(newline[x_pos]) + ): + character_width = wlen(newline[x_pos]) + self.__move(x_coord, y) + self.__write(newline[x_pos]) + self.posxy = x_coord + character_width, y + + # if this is the last character to fit in the line and we edit in the middle of the line + elif ( + self.dch1 + and self.ich1 + and wlen(newline) == self.width + and x_coord < wlen(newline) - 2 + and newline[x_pos + 1 : -1] == oldline[x_pos:-2] + ): + self.__hide_cursor() + self.__move(self.width - 2, y) + self.posxy = self.width - 2, y + self.__write_code(self.dch1) + + character_width = wlen(newline[x_pos]) + self.__move(x_coord, y) + self.__write_code(self.ich1) + self.__write(newline[x_pos]) + self.posxy = character_width + 1, y + + else: + self.__hide_cursor() + self.__move(x_coord, y) + if wlen(oldline) > wlen(newline): + self.__write_code(self._el) + self.__write(newline[x_pos:]) + self.posxy = wlen(newline), y + + if "\x1b" in newline: + # ANSI escape characters are present, so we can't assume + # anything about the position of the cursor. Moving the cursor + # to the left margin should work to get to a known position. + self.move_cursor(0, y) + + def __write(self, text): + self.__buffer.append((text, 0)) + + def __write_code(self, fmt, *args): + self.__buffer.append((curses.tparm(fmt, *args), 1)) + + def __maybe_write_code(self, fmt, *args): + if fmt: + self.__write_code(fmt, *args) + + def __move_y_cuu1_cud1(self, y): + assert self._cud1 is not None + assert self._cuu1 is not None + dy = y - self.posxy[1] + if dy > 0: + self.__write_code(dy * self._cud1) + elif dy < 0: + self.__write_code((-dy) * self._cuu1) + + def __move_y_cuu_cud(self, y): + dy = y - self.posxy[1] + if dy > 0: + self.__write_code(self._cud, dy) + elif dy < 0: + self.__write_code(self._cuu, -dy) + + def __move_x_hpa(self, x: int) -> None: + if x != self.posxy[0]: + self.__write_code(self._hpa, x) + + def __move_x_cub1_cuf1(self, x: int) -> None: + assert self._cuf1 is not None + assert self._cub1 is not None + dx = x - self.posxy[0] + if dx > 0: + self.__write_code(self._cuf1 * dx) + elif dx < 0: + self.__write_code(self._cub1 * (-dx)) + + def __move_x_cub_cuf(self, x: int) -> None: + dx = x - self.posxy[0] + if dx > 0: + self.__write_code(self._cuf, dx) + elif dx < 0: + self.__write_code(self._cub, -dx) + + def __move_short(self, x, y): + self.__move_x(x) + self.__move_y(y) + + def __move_tall(self, x, y): + assert 0 <= y - self.__offset < self.height, y - self.__offset + self.__write_code(self._cup, y - self.__offset, x) + + def __sigwinch(self, signum, frame): + self.height, self.width = self.getheightwidth() + self.event_queue.insert(Event("resize", None)) + + def __hide_cursor(self): + if self.cursor_visible: + self.__maybe_write_code(self._civis) + self.cursor_visible = 0 + + def __show_cursor(self): + if not self.cursor_visible: + self.__maybe_write_code(self._cnorm) + self.cursor_visible = 1 + + def repaint(self): + if not self.__gone_tall: + self.posxy = 0, self.posxy[1] + self.__write("\r") + ns = len(self.screen) * ["\000" * self.width] + self.screen = ns + else: + self.posxy = 0, self.__offset + self.__move(0, self.__offset) + ns = self.height * ["\000" * self.width] + self.screen = ns + + def __tputs(self, fmt, prog=delayprog): + """A Python implementation of the curses tputs function; the + curses one can't really be wrapped in a sane manner. + + I have the strong suspicion that this is complexity that + will never do anyone any good.""" + # using .get() means that things will blow up + # only if the bps is actually needed (which I'm + # betting is pretty unlkely) + bps = ratedict.get(self.__svtermstate.ospeed) + while True: + m = prog.search(fmt) + if not m: + os.write(self.output_fd, fmt) + break + x, y = m.span() + os.write(self.output_fd, fmt[:x]) + fmt = fmt[y:] + delay = int(m.group(1)) + if b"*" in m.group(2): + delay *= self.height + if self._pad and bps is not None: + nchars = (bps * delay) / 1000 + os.write(self.output_fd, self._pad * nchars) + else: + time.sleep(float(delay) / 1000.0) diff --git a/Lib/_pyrepl/unix_eventqueue.py b/Lib/_pyrepl/unix_eventqueue.py new file mode 100644 index 00000000000000..70cfade26e23b1 --- /dev/null +++ b/Lib/_pyrepl/unix_eventqueue.py @@ -0,0 +1,152 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from collections import deque + +from . import keymap +from .console import Event +from . import curses +from .trace import trace +from termios import tcgetattr, VERASE +import os + + +# Mapping of human-readable key names to their terminal-specific codes +TERMINAL_KEYNAMES = { + "delete": "kdch1", + "down": "kcud1", + "end": "kend", + "enter": "kent", + "home": "khome", + "insert": "kich1", + "left": "kcub1", + "page down": "knp", + "page up": "kpp", + "right": "kcuf1", + "up": "kcuu1", +} + + +# Function keys F1-F20 mapping +TERMINAL_KEYNAMES.update(("f%d" % i, "kf%d" % i) for i in range(1, 21)) + +# Known CTRL-arrow keycodes +CTRL_ARROW_KEYCODES= { + # for xterm, gnome-terminal, xfce terminal, etc. + b'\033[1;5D': 'ctrl left', + b'\033[1;5C': 'ctrl right', + # for rxvt + b'\033Od': 'ctrl left', + b'\033Oc': 'ctrl right', +} + +def get_terminal_keycodes() -> dict[bytes, str]: + """ + Generates a dictionary mapping terminal keycodes to human-readable names. + """ + keycodes = {} + for key, terminal_code in TERMINAL_KEYNAMES.items(): + keycode = curses.tigetstr(terminal_code) + trace('key {key} tiname {terminal_code} keycode {keycode!r}', **locals()) + if keycode: + keycodes[keycode] = key + keycodes.update(CTRL_ARROW_KEYCODES) + return keycodes + +class EventQueue: + def __init__(self, fd: int, encoding: str) -> None: + self.keycodes = get_terminal_keycodes() + if os.isatty(fd): + backspace = tcgetattr(fd)[6][VERASE] + self.keycodes[backspace] = "backspace" + self.compiled_keymap = keymap.compile_keymap(self.keycodes) + self.keymap = self.compiled_keymap + trace("keymap {k!r}", k=self.keymap) + self.encoding = encoding + self.events: deque[Event] = deque() + self.buf = bytearray() + + def get(self) -> Event | None: + """ + Retrieves the next event from the queue. + """ + if self.events: + return self.events.popleft() + else: + return None + + def empty(self) -> bool: + """ + Checks if the queue is empty. + """ + return not self.events + + def flush_buf(self) -> bytearray: + """ + Flushes the buffer and returns its contents. + """ + old = self.buf + self.buf = bytearray() + return old + + def insert(self, event: Event) -> None: + """ + Inserts an event into the queue. + """ + trace('added event {event}', event=event) + self.events.append(event) + + def push(self, char: int | bytes) -> None: + """ + Processes a character by updating the buffer and handling special key mappings. + """ + ord_char = char if isinstance(char, int) else ord(char) + char = bytes(bytearray((ord_char,))) + self.buf.append(ord_char) + if char in self.keymap: + if self.keymap is self.compiled_keymap: + #sanity check, buffer is empty when a special key comes + assert len(self.buf) == 1 + k = self.keymap[char] + trace('found map {k!r}', k=k) + if isinstance(k, dict): + self.keymap = k + else: + self.insert(Event('key', k, self.flush_buf())) + self.keymap = self.compiled_keymap + + elif self.buf and self.buf[0] == 27: # escape + # escape sequence not recognized by our keymap: propagate it + # outside so that i can be recognized as an M-... key (see also + # the docstring in keymap.py + trace('unrecognized escape sequence, propagating...') + self.keymap = self.compiled_keymap + self.insert(Event('key', '\033', bytearray(b'\033'))) + for _c in self.flush_buf()[1:]: + self.push(_c) + + else: + try: + decoded = bytes(self.buf).decode(self.encoding) + except UnicodeError: + return + else: + self.insert(Event('key', decoded, self.flush_buf())) + self.keymap = self.compiled_keymap diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py new file mode 100644 index 00000000000000..4651717bd7e121 --- /dev/null +++ b/Lib/_pyrepl/utils.py @@ -0,0 +1,25 @@ +import re +import unicodedata +import functools + +ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]") + + +@functools.cache +def str_width(c: str) -> int: + if ord(c) < 128: + return 1 + w = unicodedata.east_asian_width(c) + if w in ('N', 'Na', 'H', 'A'): + return 1 + return 2 + + +def wlen(s: str) -> int: + if len(s) == 1 and s != '\x1a': + return str_width(s) + length = sum(str_width(i) for i in s) + # remove lengths of any escape sequences + sequence = ANSI_ESCAPE_SEQUENCE.findall(s) + ctrl_z_cnt = s.count('\x1a') + return length - sum(len(i) for i in sequence) + ctrl_z_cnt diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py new file mode 100644 index 00000000000000..e1ecd9845aefb4 --- /dev/null +++ b/Lib/_pyrepl/windows_console.py @@ -0,0 +1,628 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +import io +import os +import sys +import time +import msvcrt + +from collections import deque +import ctypes +from ctypes.wintypes import ( + _COORD, + WORD, + SMALL_RECT, + BOOL, + HANDLE, + CHAR, + DWORD, + WCHAR, + SHORT, +) +from ctypes import Structure, POINTER, Union +from .console import Event, Console +from .trace import trace +from .utils import wlen + +try: + from ctypes import GetLastError, WinDLL, windll, WinError # type: ignore[attr-defined] +except: + # Keep MyPy happy off Windows + from ctypes import CDLL as WinDLL, cdll as windll + + def GetLastError() -> int: + return 42 + + class WinError(OSError): # type: ignore[no-redef] + def __init__(self, err: int | None, descr: str | None = None) -> None: + self.err = err + self.descr = descr + + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import IO + +# Virtual-Key Codes: https://learn.microsoft.com/en-us/windows/win32/inputdev/virtual-key-codes +VK_MAP: dict[int, str] = { + 0x23: "end", # VK_END + 0x24: "home", # VK_HOME + 0x25: "left", # VK_LEFT + 0x26: "up", # VK_UP + 0x27: "right", # VK_RIGHT + 0x28: "down", # VK_DOWN + 0x2E: "delete", # VK_DELETE + 0x70: "f1", # VK_F1 + 0x71: "f2", # VK_F2 + 0x72: "f3", # VK_F3 + 0x73: "f4", # VK_F4 + 0x74: "f5", # VK_F5 + 0x75: "f6", # VK_F6 + 0x76: "f7", # VK_F7 + 0x77: "f8", # VK_F8 + 0x78: "f9", # VK_F9 + 0x79: "f10", # VK_F10 + 0x7A: "f11", # VK_F11 + 0x7B: "f12", # VK_F12 + 0x7C: "f13", # VK_F13 + 0x7D: "f14", # VK_F14 + 0x7E: "f15", # VK_F15 + 0x7F: "f16", # VK_F16 + 0x80: "f17", # VK_F17 + 0x81: "f18", # VK_F18 + 0x82: "f19", # VK_F19 + 0x83: "f20", # VK_F20 +} + +# Console escape codes: https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences +ERASE_IN_LINE = "\x1b[K" +MOVE_LEFT = "\x1b[{}D" +MOVE_RIGHT = "\x1b[{}C" +MOVE_UP = "\x1b[{}A" +MOVE_DOWN = "\x1b[{}B" +CLEAR = "\x1b[H\x1b[J" + +# State of control keys: https://learn.microsoft.com/en-us/windows/console/key-event-record-str +ALT_ACTIVE = 0x01 | 0x02 +CTRL_ACTIVE = 0x04 | 0x08 + + +class _error(Exception): + pass + + +class WindowsConsole(Console): + def __init__( + self, + f_in: IO[bytes] | int = 0, + f_out: IO[bytes] | int = 1, + term: str = "", + encoding: str = "", + ): + super().__init__(f_in, f_out, term, encoding) + + SetConsoleMode( + OutHandle, + ENABLE_WRAP_AT_EOL_OUTPUT + | ENABLE_PROCESSED_OUTPUT + | ENABLE_VIRTUAL_TERMINAL_PROCESSING, + ) + self.screen: list[str] = [] + self.width = 80 + self.height = 25 + self.__offset = 0 + self.event_queue: deque[Event] = deque() + try: + self.out = io._WindowsConsoleIO(self.output_fd, "w") # type: ignore[attr-defined] + except ValueError: + # Console I/O is redirected, fallback... + self.out = None + + def refresh(self, screen: list[str], c_xy: tuple[int, int]) -> None: + """ + Refresh the console screen. + + Parameters: + - screen (list): List of strings representing the screen contents. + - c_xy (tuple): Cursor position (x, y) on the screen. + """ + cx, cy = c_xy + + while len(self.screen) < min(len(screen), self.height): + self._hide_cursor() + self._move_relative(0, len(self.screen) - 1) + self.__write("\n") + self.posxy = 0, len(self.screen) + self.screen.append("") + + px, py = self.posxy + old_offset = offset = self.__offset + height = self.height + + # we make sure the cursor is on the screen, and that we're + # using all of the screen if we can + if cy < offset: + offset = cy + elif cy >= offset + height: + offset = cy - height + 1 + scroll_lines = offset - old_offset + + # Scrolling the buffer as the current input is greater than the visible + # portion of the window. We need to scroll the visible portion and the + # entire history + self._scroll(scroll_lines, self._getscrollbacksize()) + self.posxy = self.posxy[0], self.posxy[1] + scroll_lines + self.__offset += scroll_lines + + for i in range(scroll_lines): + self.screen.append("") + elif offset > 0 and len(screen) < offset + height: + offset = max(len(screen) - height, 0) + screen.append("") + + oldscr = self.screen[old_offset : old_offset + height] + newscr = screen[offset : offset + height] + + self.__offset = offset + + self._hide_cursor() + for ( + y, + oldline, + newline, + ) in zip(range(offset, offset + height), oldscr, newscr): + if oldline != newline: + self.__write_changed_line(y, oldline, newline, px) + + y = len(newscr) + while y < len(oldscr): + self._move_relative(0, y) + self.posxy = 0, y + self._erase_to_end() + y += 1 + + self._show_cursor() + + self.screen = screen + self.move_cursor(cx, cy) + + @property + def input_hook(self): + try: + import nt + except ImportError: + return None + if nt._is_inputhook_installed(): + return nt._inputhook + + def __write_changed_line( + self, y: int, oldline: str, newline: str, px_coord: int + ) -> None: + # this is frustrating; there's no reason to test (say) + # self.dch1 inside the loop -- but alternative ways of + # structuring this function are equally painful (I'm trying to + # avoid writing code generators these days...) + minlen = min(wlen(oldline), wlen(newline)) + x_pos = 0 + x_coord = 0 + + px_pos = 0 + j = 0 + for c in oldline: + if j >= px_coord: + break + j += wlen(c) + px_pos += 1 + + # reuse the oldline as much as possible, but stop as soon as we + # encounter an ESCAPE, because it might be the start of an escape + # sequence + while ( + x_coord < minlen + and oldline[x_pos] == newline[x_pos] + and newline[x_pos] != "\x1b" + ): + x_coord += wlen(newline[x_pos]) + x_pos += 1 + + self._hide_cursor() + self._move_relative(x_coord, y) + if wlen(oldline) > wlen(newline): + self._erase_to_end() + + self.__write(newline[x_pos:]) + if wlen(newline) == self.width: + # If we wrapped we want to start at the next line + self._move_relative(0, y + 1) + self.posxy = 0, y + 1 + else: + self.posxy = wlen(newline), y + + if "\x1b" in newline or y != self.posxy[1] or '\x1a' in newline: + # ANSI escape characters are present, so we can't assume + # anything about the position of the cursor. Moving the cursor + # to the left margin should work to get to a known position. + self.move_cursor(0, y) + + def _scroll( + self, top: int, bottom: int, left: int | None = None, right: int | None = None + ) -> None: + scroll_rect = SMALL_RECT() + scroll_rect.Top = SHORT(top) + scroll_rect.Bottom = SHORT(bottom) + scroll_rect.Left = SHORT(0 if left is None else left) + scroll_rect.Right = SHORT( + self.getheightwidth()[1] - 1 if right is None else right + ) + destination_origin = _COORD() + fill_info = CHAR_INFO() + fill_info.UnicodeChar = " " + + if not ScrollConsoleScreenBuffer( + OutHandle, scroll_rect, None, destination_origin, fill_info + ): + raise WinError(GetLastError()) + + def _hide_cursor(self): + self.__write("\x1b[?25l") + + def _show_cursor(self): + self.__write("\x1b[?25h") + + def _enable_blinking(self): + self.__write("\x1b[?12h") + + def _disable_blinking(self): + self.__write("\x1b[?12l") + + def __write(self, text: str) -> None: + if "\x1a" in text: + text = ''.join(["^Z" if x == '\x1a' else x for x in text]) + + if self.out is not None: + self.out.write(text.encode(self.encoding, "replace")) + self.out.flush() + else: + os.write(self.output_fd, text.encode(self.encoding, "replace")) + + @property + def screen_xy(self) -> tuple[int, int]: + info = CONSOLE_SCREEN_BUFFER_INFO() + if not GetConsoleScreenBufferInfo(OutHandle, info): + raise WinError(GetLastError()) + return info.dwCursorPosition.X, info.dwCursorPosition.Y + + def _erase_to_end(self) -> None: + self.__write(ERASE_IN_LINE) + + def prepare(self) -> None: + trace("prepare") + self.screen = [] + self.height, self.width = self.getheightwidth() + + self.posxy = 0, 0 + self.__gone_tall = 0 + self.__offset = 0 + + def restore(self) -> None: + pass + + def _move_relative(self, x: int, y: int) -> None: + """Moves relative to the current posxy""" + dx = x - self.posxy[0] + dy = y - self.posxy[1] + if dx < 0: + self.__write(MOVE_LEFT.format(-dx)) + elif dx > 0: + self.__write(MOVE_RIGHT.format(dx)) + + if dy < 0: + self.__write(MOVE_UP.format(-dy)) + elif dy > 0: + self.__write(MOVE_DOWN.format(dy)) + + def move_cursor(self, x: int, y: int) -> None: + if x < 0 or y < 0: + raise ValueError(f"Bad cursor position {x}, {y}") + + if y < self.__offset or y >= self.__offset + self.height: + self.event_queue.insert(0, Event("scroll", "")) + else: + self._move_relative(x, y) + self.posxy = x, y + + def set_cursor_vis(self, visible: bool) -> None: + if visible: + self._show_cursor() + else: + self._hide_cursor() + + def getheightwidth(self) -> tuple[int, int]: + """Return (height, width) where height and width are the height + and width of the terminal window in characters.""" + info = CONSOLE_SCREEN_BUFFER_INFO() + if not GetConsoleScreenBufferInfo(OutHandle, info): + raise WinError(GetLastError()) + return ( + info.srWindow.Bottom - info.srWindow.Top + 1, + info.srWindow.Right - info.srWindow.Left + 1, + ) + + def _getscrollbacksize(self) -> int: + info = CONSOLE_SCREEN_BUFFER_INFO() + if not GetConsoleScreenBufferInfo(OutHandle, info): + raise WinError(GetLastError()) + + return info.srWindow.Bottom # type: ignore[no-any-return] + + def _read_input(self, block: bool = True) -> INPUT_RECORD | None: + if not block: + events = DWORD() + if not GetNumberOfConsoleInputEvents(InHandle, events): + raise WinError(GetLastError()) + if not events.value: + return None + + rec = INPUT_RECORD() + read = DWORD() + if not ReadConsoleInput(InHandle, rec, 1, read): + raise WinError(GetLastError()) + + return rec + + def get_event(self, block: bool = True) -> Event | None: + """Return an Event instance. Returns None if |block| is false + and there is no event pending, otherwise waits for the + completion of an event.""" + if self.event_queue: + return self.event_queue.pop() + + while True: + rec = self._read_input(block) + if rec is None: + return None + + if rec.EventType == WINDOW_BUFFER_SIZE_EVENT: + return Event("resize", "") + + if rec.EventType != KEY_EVENT or not rec.Event.KeyEvent.bKeyDown: + # Only process keys and keydown events + if block: + continue + return None + + key_event = rec.Event.KeyEvent + raw_key = key = key_event.uChar.UnicodeChar + + if key == "\r": + # Make enter unix-like + return Event(evt="key", data="\n", raw=b"\n") + elif key_event.wVirtualKeyCode == 8: + # Turn backspace directly into the command + key = "backspace" + elif key == "\x00": + # Handle special keys like arrow keys and translate them into the appropriate command + key = VK_MAP.get(key_event.wVirtualKeyCode) + if key: + if key_event.dwControlKeyState & CTRL_ACTIVE: + key = f"ctrl {key}" + elif key_event.dwControlKeyState & ALT_ACTIVE: + # queue the key, return the meta command + self.event_queue.insert(0, Event(evt="key", data=key, raw=key)) + return Event(evt="key", data="\033") # keymap.py uses this for meta + return Event(evt="key", data=key, raw=key) + if block: + continue + + return None + + if key_event.dwControlKeyState & ALT_ACTIVE: + # queue the key, return the meta command + self.event_queue.insert(0, Event(evt="key", data=key, raw=raw_key)) + return Event(evt="key", data="\033") # keymap.py uses this for meta + + return Event(evt="key", data=key, raw=raw_key) + + def push_char(self, char: int | bytes) -> None: + """ + Push a character to the console event queue. + """ + raise NotImplementedError("push_char not supported on Windows") + + def beep(self) -> None: + self.__write("\x07") + + def clear(self) -> None: + """Wipe the screen""" + self.__write(CLEAR) + self.posxy = 0, 0 + self.screen = [""] + + def finish(self) -> None: + """Move the cursor to the end of the display and otherwise get + ready for end. XXX could be merged with restore? Hmm.""" + y = len(self.screen) - 1 + while y >= 0 and not self.screen[y]: + y -= 1 + self._move_relative(0, min(y, self.height + self.__offset - 1)) + self.__write("\r\n") + + def flushoutput(self) -> None: + """Flush all output to the screen (assuming there's some + buffering going on somewhere). + + All output on Windows is unbuffered so this is a nop""" + pass + + def forgetinput(self) -> None: + """Forget all pending, but not yet processed input.""" + if not FlushConsoleInputBuffer(InHandle): + raise WinError(GetLastError()) + + def getpending(self) -> Event: + """Return the characters that have been typed but not yet + processed.""" + return Event("key", "", b"") + + def wait(self, timeout: float | None) -> bool: + """Wait for an event.""" + # Poor man's Windows select loop + start_time = time.time() + while True: + if msvcrt.kbhit(): # type: ignore[attr-defined] + return True + if timeout and time.time() - start_time > timeout / 1000: + return False + time.sleep(0.01) + + def repaint(self) -> None: + raise NotImplementedError("No repaint support") + + +# Windows interop +class CONSOLE_SCREEN_BUFFER_INFO(Structure): + _fields_ = [ + ("dwSize", _COORD), + ("dwCursorPosition", _COORD), + ("wAttributes", WORD), + ("srWindow", SMALL_RECT), + ("dwMaximumWindowSize", _COORD), + ] + + +class CONSOLE_CURSOR_INFO(Structure): + _fields_ = [ + ("dwSize", DWORD), + ("bVisible", BOOL), + ] + + +class CHAR_INFO(Structure): + _fields_ = [ + ("UnicodeChar", WCHAR), + ("Attributes", WORD), + ] + + +class Char(Union): + _fields_ = [ + ("UnicodeChar", WCHAR), + ("Char", CHAR), + ] + + +class KeyEvent(ctypes.Structure): + _fields_ = [ + ("bKeyDown", BOOL), + ("wRepeatCount", WORD), + ("wVirtualKeyCode", WORD), + ("wVirtualScanCode", WORD), + ("uChar", Char), + ("dwControlKeyState", DWORD), + ] + + +class WindowsBufferSizeEvent(ctypes.Structure): + _fields_ = [("dwSize", _COORD)] + + +class ConsoleEvent(ctypes.Union): + _fields_ = [ + ("KeyEvent", KeyEvent), + ("WindowsBufferSizeEvent", WindowsBufferSizeEvent), + ] + + +class INPUT_RECORD(Structure): + _fields_ = [("EventType", WORD), ("Event", ConsoleEvent)] + + +KEY_EVENT = 0x01 +FOCUS_EVENT = 0x10 +MENU_EVENT = 0x08 +MOUSE_EVENT = 0x02 +WINDOW_BUFFER_SIZE_EVENT = 0x04 + +ENABLE_PROCESSED_OUTPUT = 0x01 +ENABLE_WRAP_AT_EOL_OUTPUT = 0x02 +ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x04 + +STD_INPUT_HANDLE = -10 +STD_OUTPUT_HANDLE = -11 + +if sys.platform == "win32": + _KERNEL32 = WinDLL("kernel32", use_last_error=True) + + GetStdHandle = windll.kernel32.GetStdHandle + GetStdHandle.argtypes = [DWORD] + GetStdHandle.restype = HANDLE + + GetConsoleScreenBufferInfo = _KERNEL32.GetConsoleScreenBufferInfo + GetConsoleScreenBufferInfo.argtypes = [ + HANDLE, + ctypes.POINTER(CONSOLE_SCREEN_BUFFER_INFO), + ] + GetConsoleScreenBufferInfo.restype = BOOL + + ScrollConsoleScreenBuffer = _KERNEL32.ScrollConsoleScreenBufferW + ScrollConsoleScreenBuffer.argtypes = [ + HANDLE, + POINTER(SMALL_RECT), + POINTER(SMALL_RECT), + _COORD, + POINTER(CHAR_INFO), + ] + ScrollConsoleScreenBuffer.restype = BOOL + + SetConsoleMode = _KERNEL32.SetConsoleMode + SetConsoleMode.argtypes = [HANDLE, DWORD] + SetConsoleMode.restype = BOOL + + ReadConsoleInput = _KERNEL32.ReadConsoleInputW + ReadConsoleInput.argtypes = [HANDLE, POINTER(INPUT_RECORD), DWORD, POINTER(DWORD)] + ReadConsoleInput.restype = BOOL + + GetNumberOfConsoleInputEvents = _KERNEL32.GetNumberOfConsoleInputEvents + GetNumberOfConsoleInputEvents.argtypes = [HANDLE, POINTER(DWORD)] + GetNumberOfConsoleInputEvents.restype = BOOL + + FlushConsoleInputBuffer = _KERNEL32.FlushConsoleInputBuffer + FlushConsoleInputBuffer.argtypes = [HANDLE] + FlushConsoleInputBuffer.restype = BOOL + + OutHandle = GetStdHandle(STD_OUTPUT_HANDLE) + InHandle = GetStdHandle(STD_INPUT_HANDLE) +else: + + def _win_only(*args, **kwargs): + raise NotImplementedError("Windows only") + + GetStdHandle = _win_only + GetConsoleScreenBufferInfo = _win_only + ScrollConsoleScreenBuffer = _win_only + SetConsoleMode = _win_only + ReadConsoleInput = _win_only + GetNumberOfConsoleInputEvents = _win_only + FlushConsoleInputBuffer = _win_only + OutHandle = 0 + InHandle = 0 diff --git a/Lib/_strptime.py b/Lib/_strptime.py index 798cf9f9d3fffe..e6e23596db6f99 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -10,10 +10,12 @@ strptime -- Calculates the time struct represented by the passed-in string """ +import os import time import locale import calendar from re import compile as re_compile +from re import sub as re_sub from re import IGNORECASE from re import escape as re_escape from datetime import (date as datetime_date, @@ -27,6 +29,18 @@ def _getlang(): # Figure out what the current language is set to. return locale.getlocale(locale.LC_TIME) +def _findall(haystack, needle): + # Find all positions of needle in haystack. + if not needle: + return + i = 0 + while True: + i = haystack.find(needle, i) + if i < 0: + break + yield i + i += len(needle) + class LocaleTime(object): """Stores and handles locale-specific information related to time. @@ -101,7 +115,8 @@ def __calc_am_pm(self): am_pm = [] for hour in (1, 22): time_tuple = time.struct_time((1999,3,17,hour,44,55,2,76,0)) - am_pm.append(time.strftime("%p", time_tuple).lower()) + # br_FR has AM/PM info (' ',' '). + am_pm.append(time.strftime("%p", time_tuple).lower().strip()) self.am_pm = am_pm def __calc_date_time(self): @@ -113,42 +128,130 @@ def __calc_date_time(self): # values within the format string is very important; it eliminates # possible ambiguity for what something represents. time_tuple = time.struct_time((1999,3,17,22,44,55,2,76,0)) - date_time = [None, None, None] - date_time[0] = time.strftime("%c", time_tuple).lower() - date_time[1] = time.strftime("%x", time_tuple).lower() - date_time[2] = time.strftime("%X", time_tuple).lower() - replacement_pairs = [('%', '%%'), (self.f_weekday[2], '%A'), - (self.f_month[3], '%B'), (self.a_weekday[2], '%a'), - (self.a_month[3], '%b'), (self.am_pm[1], '%p'), - ('1999', '%Y'), ('99', '%y'), ('22', '%H'), - ('44', '%M'), ('55', '%S'), ('76', '%j'), - ('17', '%d'), ('03', '%m'), ('3', '%m'), - # '3' needed for when no leading zero. - ('2', '%w'), ('10', '%I')] - replacement_pairs.extend([(tz, "%Z") for tz_values in self.timezone - for tz in tz_values]) - for offset,directive in ((0,'%c'), (1,'%x'), (2,'%X')): - current_format = date_time[offset] - for old, new in replacement_pairs: + time_tuple2 = time.struct_time((1999,1,3,1,1,1,6,3,0)) + replacement_pairs = [ + ('1999', '%Y'), ('99', '%y'), ('22', '%H'), + ('44', '%M'), ('55', '%S'), ('76', '%j'), + ('17', '%d'), ('03', '%m'), ('3', '%m'), + # '3' needed for when no leading zero. + ('2', '%w'), ('10', '%I'), + # Non-ASCII digits + ('\u0661\u0669\u0669\u0669', '%Y'), + ('\u0669\u0669', '%Oy'), + ('\u0662\u0662', '%OH'), + ('\u0664\u0664', '%OM'), + ('\u0665\u0665', '%OS'), + ('\u0661\u0667', '%Od'), + ('\u0660\u0663', '%Om'), + ('\u0663', '%Om'), + ('\u0662', '%Ow'), + ('\u0661\u0660', '%OI'), + ] + date_time = [] + for directive in ('%c', '%x', '%X'): + current_format = time.strftime(directive, time_tuple).lower() + current_format = current_format.replace('%', '%%') + # The month and the day of the week formats are treated specially + # because of a possible ambiguity in some locales where the full + # and abbreviated names are equal or names of different types + # are equal. See doc of __find_month_format for more details. + lst, fmt = self.__find_weekday_format(directive) + if lst: + current_format = current_format.replace(lst[2], fmt, 1) + lst, fmt = self.__find_month_format(directive) + if lst: + current_format = current_format.replace(lst[3], fmt, 1) + if self.am_pm[1]: # Must deal with possible lack of locale info # manifesting itself as the empty string (e.g., Swedish's # lack of AM/PM info) or a platform returning a tuple of empty # strings (e.g., MacOS 9 having timezone as ('','')). - if old: - current_format = current_format.replace(old, new) + current_format = current_format.replace(self.am_pm[1], '%p') + for tz_values in self.timezone: + for tz in tz_values: + if tz: + current_format = current_format.replace(tz, "%Z") + # Transform all non-ASCII digits to digits in range U+0660 to U+0669. + current_format = re_sub(r'\d(?3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])", 'f': r"(?P[0-9]{1,6})", 'H': r"(?P2[0-3]|[0-1]\d|\d)", - 'I': r"(?P1[0-2]|0[1-9]|[1-9])", + 'I': r"(?P1[0-2]|0[1-9]|[1-9]| [1-9])", 'G': r"(?P\d\d\d\d)", 'j': r"(?P36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])", 'm': r"(?P1[0-2]|0[1-9]|[1-9])", @@ -198,8 +301,6 @@ def __init__(self, locale_time=None): 'V': r"(?P5[0-3]|0[1-9]|[1-4]\d|\d)", # W is set below by using 'U' 'y': r"(?P\d\d)", - #XXX: Does 'Y' need to worry about having less or more than - # 4 digits? 'Y': r"(?P\d\d\d\d)", 'z': r"(?P[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|(?-i:Z))", 'A': self.__seqToRE(self.locale_time.f_weekday, 'A'), @@ -210,11 +311,15 @@ def __init__(self, locale_time=None): 'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone for tz in tz_names), 'Z'), - '%': '%'}) - base.__setitem__('W', base.__getitem__('U').replace('U', 'W')) - base.__setitem__('c', self.pattern(self.locale_time.LC_date_time)) - base.__setitem__('x', self.pattern(self.locale_time.LC_date)) + '%': '%'} + for d in 'dmyHIMS': + mapping['O' + d] = r'(?P<%s>\d\d|\d| \d)' % d + mapping['Ow'] = r'(?P\d)' + mapping['W'] = mapping['U'].replace('U', 'W') + base.__init__(mapping) base.__setitem__('X', self.pattern(self.locale_time.LC_time)) + base.__setitem__('x', self.pattern(self.locale_time.LC_date)) + base.__setitem__('c', self.pattern(self.locale_time.LC_date_time)) def __seqToRE(self, to_convert, directive): """Convert a list to a regex string for matching a directive. @@ -242,21 +347,36 @@ def pattern(self, format): regex syntax are escaped. """ - processed_format = '' # The sub() call escapes all characters that might be misconstrued # as regex syntax. Cannot use re.escape since we have to deal with # format directives (%m, etc.). - regex_chars = re_compile(r"([\\.^$*+?\(\){}\[\]|])") - format = regex_chars.sub(r"\\\1", format) - whitespace_replacement = re_compile(r'\s+') - format = whitespace_replacement.sub(r'\\s+', format) - while '%' in format: - directive_index = format.index('%')+1 - processed_format = "%s%s%s" % (processed_format, - format[:directive_index-1], - self[format[directive_index]]) - format = format[directive_index+1:] - return "%s%s" % (processed_format, format) + format = re_sub(r"([\\.^$*+?\(\){}\[\]|])", r"\\\1", format) + format = re_sub(r'\s+', r'\\s+', format) + format = re_sub(r"'", "['\u02bc]", format) # needed for br_FR + year_in_format = False + day_of_month_in_format = False + def repl(m): + format_char = m[1] + match format_char: + case 'Y' | 'y' | 'G': + nonlocal year_in_format + year_in_format = True + case 'd': + nonlocal day_of_month_in_format + day_of_month_in_format = True + return self[format_char] + format = re_sub(r'%(O?.)', repl, format) + if day_of_month_in_format and not year_in_format: + import warnings + warnings.warn("""\ +Parsing dates involving a day of month without a year specified is ambiguous +and fails to parse leap day. The default behavior will change in Python 3.15 +to either always raise an exception or to use a different default year (TBD). +To avoid trouble, add a specific year to the input & format. +See https://github.com/python/cpython/issues/70647.""", + DeprecationWarning, + skip_file_prefixes=(os.path.dirname(__file__),)) + return format def compile(self, format): """Return a compiled re object for the format string.""" @@ -548,18 +668,40 @@ def _strptime_time(data_string, format="%a %b %d %H:%M:%S %Y"): tt = _strptime(data_string, format)[0] return time.struct_time(tt[:time._STRUCT_TM_ITEMS]) -def _strptime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"): - """Return a class cls instance based on the input string and the +def _strptime_datetime_date(cls, data_string, format="%a %b %d %Y"): + """Return a date instance based on the input string and the + format string.""" + tt, _, _ = _strptime(data_string, format) + args = tt[:3] + return cls(*args) + +def _parse_tz(tzname, gmtoff, gmtoff_fraction): + tzdelta = datetime_timedelta(seconds=gmtoff, microseconds=gmtoff_fraction) + if tzname: + return datetime_timezone(tzdelta, tzname) + else: + return datetime_timezone(tzdelta) + +def _strptime_datetime_time(cls, data_string, format="%H:%M:%S"): + """Return a time instance based on the input string and the format string.""" tt, fraction, gmtoff_fraction = _strptime(data_string, format) tzname, gmtoff = tt[-2:] - args = tt[:6] + (fraction,) - if gmtoff is not None: - tzdelta = datetime_timedelta(seconds=gmtoff, microseconds=gmtoff_fraction) - if tzname: - tz = datetime_timezone(tzdelta, tzname) - else: - tz = datetime_timezone(tzdelta) - args += (tz,) + args = tt[3:6] + (fraction,) + if gmtoff is None: + return cls(*args) + else: + tz = _parse_tz(tzname, gmtoff, gmtoff_fraction) + return cls(*args, tz) - return cls(*args) +def _strptime_datetime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"): + """Return a datetime instance based on the input string and the + format string.""" + tt, fraction, gmtoff_fraction = _strptime(data_string, format) + tzname, gmtoff = tt[-2:] + args = tt[:6] + (fraction,) + if gmtoff is None: + return cls(*args) + else: + tz = _parse_tz(tzname, gmtoff, gmtoff_fraction) + return cls(*args, tz) diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py index 489eec714e0d48..d1c7fcaeec9821 100644 --- a/Lib/_weakrefset.py +++ b/Lib/_weakrefset.py @@ -8,69 +8,29 @@ __all__ = ['WeakSet'] -class _IterationGuard: - # This context manager registers itself in the current iterators of the - # weak container, such as to delay all removals until the context manager - # exits. - # This technique should be relatively thread-safe (since sets are). - - def __init__(self, weakcontainer): - # Don't create cycles - self.weakcontainer = ref(weakcontainer) - - def __enter__(self): - w = self.weakcontainer() - if w is not None: - w._iterating.add(self) - return self - - def __exit__(self, e, t, b): - w = self.weakcontainer() - if w is not None: - s = w._iterating - s.remove(self) - if not s: - w._commit_removals() - - class WeakSet: def __init__(self, data=None): self.data = set() + def _remove(item, selfref=ref(self)): self = selfref() if self is not None: - if self._iterating: - self._pending_removals.append(item) - else: - self.data.discard(item) + self.data.discard(item) + self._remove = _remove - # A list of keys to be removed - self._pending_removals = [] - self._iterating = set() if data is not None: self.update(data) - def _commit_removals(self): - pop = self._pending_removals.pop - discard = self.data.discard - while True: - try: - item = pop() - except IndexError: - return - discard(item) - def __iter__(self): - with _IterationGuard(self): - for itemref in self.data: - item = itemref() - if item is not None: - # Caveat: the iterator will keep a strong reference to - # `item` until it is resumed or closed. - yield item + for itemref in self.data.copy(): + item = itemref() + if item is not None: + # Caveat: the iterator will keep a strong reference to + # `item` until it is resumed or closed. + yield item def __len__(self): - return len(self.data) - len(self._pending_removals) + return len(self.data) def __contains__(self, item): try: @@ -83,21 +43,15 @@ def __reduce__(self): return self.__class__, (list(self),), self.__getstate__() def add(self, item): - if self._pending_removals: - self._commit_removals() self.data.add(ref(item, self._remove)) def clear(self): - if self._pending_removals: - self._commit_removals() self.data.clear() def copy(self): return self.__class__(self) def pop(self): - if self._pending_removals: - self._commit_removals() while True: try: itemref = self.data.pop() @@ -108,18 +62,12 @@ def pop(self): return item def remove(self, item): - if self._pending_removals: - self._commit_removals() self.data.remove(ref(item)) def discard(self, item): - if self._pending_removals: - self._commit_removals() self.data.discard(ref(item)) def update(self, other): - if self._pending_removals: - self._commit_removals() for element in other: self.add(element) @@ -136,8 +84,6 @@ def difference(self, other): def difference_update(self, other): self.__isub__(other) def __isub__(self, other): - if self._pending_removals: - self._commit_removals() if self is other: self.data.clear() else: @@ -151,8 +97,6 @@ def intersection(self, other): def intersection_update(self, other): self.__iand__(other) def __iand__(self, other): - if self._pending_removals: - self._commit_removals() self.data.intersection_update(ref(item) for item in other) return self @@ -184,8 +128,6 @@ def symmetric_difference(self, other): def symmetric_difference_update(self, other): self.__ixor__(other) def __ixor__(self, other): - if self._pending_removals: - self._commit_removals() if self is other: self.data.clear() else: diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py new file mode 100644 index 00000000000000..42f1f3877514d9 --- /dev/null +++ b/Lib/annotationlib.py @@ -0,0 +1,853 @@ +"""Helpers for introspecting and wrapping annotations.""" + +import ast +import builtins +import enum +import functools +import keyword +import sys +import types + +__all__ = [ + "Format", + "ForwardRef", + "call_annotate_function", + "call_evaluate_function", + "get_annotate_function", + "get_annotations", + "annotations_to_string", + "value_to_string", +] + + +class Format(enum.IntEnum): + VALUE = 1 + VALUE_WITH_FAKE_GLOBALS = 2 + FORWARDREF = 3 + STRING = 4 + + +_Union = None +_sentinel = object() + +# Slots shared by ForwardRef and _Stringifier. The __forward__ names must be +# preserved for compatibility with the old typing.ForwardRef class. The remaining +# names are private. +_SLOTS = ( + "__forward_evaluated__", + "__forward_value__", + "__forward_is_argument__", + "__forward_is_class__", + "__forward_module__", + "__weakref__", + "__arg__", + "__ast_node__", + "__code__", + "__globals__", + "__owner__", + "__cell__", + "__stringifier_dict__", +) + + +class ForwardRef: + """Wrapper that holds a forward reference. + + Constructor arguments: + * arg: a string representing the code to be evaluated. + * module: the module where the forward reference was created. + Must be a string, not a module object. + * owner: The owning object (module, class, or function). + * is_argument: Does nothing, retained for compatibility. + * is_class: True if the forward reference was created in class scope. + + """ + + __slots__ = _SLOTS + + def __init__( + self, + arg, + *, + module=None, + owner=None, + is_argument=True, + is_class=False, + ): + if not isinstance(arg, str): + raise TypeError(f"Forward reference must be a string -- got {arg!r}") + + self.__arg__ = arg + self.__forward_evaluated__ = False + self.__forward_value__ = None + self.__forward_is_argument__ = is_argument + self.__forward_is_class__ = is_class + self.__forward_module__ = module + self.__code__ = None + self.__ast_node__ = None + self.__globals__ = None + self.__cell__ = None + self.__owner__ = owner + + def __init_subclass__(cls, /, *args, **kwds): + raise TypeError("Cannot subclass ForwardRef") + + def evaluate(self, *, globals=None, locals=None, type_params=None, owner=None): + """Evaluate the forward reference and return the value. + + If the forward reference cannot be evaluated, raise an exception. + """ + if self.__forward_evaluated__: + return self.__forward_value__ + if self.__cell__ is not None: + try: + value = self.__cell__.cell_contents + except ValueError: + pass + else: + self.__forward_evaluated__ = True + self.__forward_value__ = value + return value + if owner is None: + owner = self.__owner__ + + if globals is None and self.__forward_module__ is not None: + globals = getattr( + sys.modules.get(self.__forward_module__, None), "__dict__", None + ) + if globals is None: + globals = self.__globals__ + if globals is None: + if isinstance(owner, type): + module_name = getattr(owner, "__module__", None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + globals = getattr(module, "__dict__", None) + elif isinstance(owner, types.ModuleType): + globals = getattr(owner, "__dict__", None) + elif callable(owner): + globals = getattr(owner, "__globals__", None) + + # If we pass None to eval() below, the globals of this module are used. + if globals is None: + globals = {} + + if locals is None: + locals = {} + if isinstance(owner, type): + locals.update(vars(owner)) + + if type_params is None and owner is not None: + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + type_params = getattr(owner, "__type_params__", None) + + # type parameters require some special handling, + # as they exist in their own scope + # but `eval()` does not have a dedicated parameter for that scope. + # For classes, names in type parameter scopes should override + # names in the global scope (which here are called `localns`!), + # but should in turn be overridden by names in the class scope + # (which here are called `globalns`!) + if type_params is not None: + globals = dict(globals) + locals = dict(locals) + for param in type_params: + param_name = param.__name__ + if not self.__forward_is_class__ or param_name not in globals: + globals[param_name] = param + locals.pop(param_name, None) + + arg = self.__forward_arg__ + if arg.isidentifier() and not keyword.iskeyword(arg): + if arg in locals: + value = locals[arg] + elif arg in globals: + value = globals[arg] + elif hasattr(builtins, arg): + return getattr(builtins, arg) + else: + raise NameError(arg) + else: + code = self.__forward_code__ + value = eval(code, globals=globals, locals=locals) + self.__forward_evaluated__ = True + self.__forward_value__ = value + return value + + def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard): + import typing + import warnings + + if type_params is _sentinel: + typing._deprecation_warning_for_no_type_params_passed( + "typing.ForwardRef._evaluate" + ) + type_params = () + warnings._deprecated( + "ForwardRef._evaluate", + "{name} is a private API and is retained for compatibility, but will be removed" + " in Python 3.16. Use ForwardRef.evaluate() or typing.evaluate_forward_ref() instead.", + remove=(3, 16), + ) + return typing.evaluate_forward_ref( + self, + globals=globalns, + locals=localns, + type_params=type_params, + _recursive_guard=recursive_guard, + ) + + @property + def __forward_arg__(self): + if self.__arg__ is not None: + return self.__arg__ + if self.__ast_node__ is not None: + self.__arg__ = ast.unparse(self.__ast_node__) + return self.__arg__ + raise AssertionError( + "Attempted to access '__forward_arg__' on an uninitialized ForwardRef" + ) + + @property + def __forward_code__(self): + if self.__code__ is not None: + return self.__code__ + arg = self.__forward_arg__ + # If we do `def f(*args: *Ts)`, then we'll have `arg = '*Ts'`. + # Unfortunately, this isn't a valid expression on its own, so we + # do the unpacking manually. + if arg.startswith("*"): + arg_to_compile = f"({arg},)[0]" # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] + else: + arg_to_compile = arg + try: + self.__code__ = compile(arg_to_compile, "", "eval") + except SyntaxError: + raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}") + return self.__code__ + + def __eq__(self, other): + if not isinstance(other, ForwardRef): + return NotImplemented + if self.__forward_evaluated__ and other.__forward_evaluated__: + return ( + self.__forward_arg__ == other.__forward_arg__ + and self.__forward_value__ == other.__forward_value__ + ) + return ( + self.__forward_arg__ == other.__forward_arg__ + and self.__forward_module__ == other.__forward_module__ + ) + + def __hash__(self): + return hash((self.__forward_arg__, self.__forward_module__)) + + def __or__(self, other): + global _Union + if _Union is None: + from typing import Union as _Union + return _Union[self, other] + + def __ror__(self, other): + global _Union + if _Union is None: + from typing import Union as _Union + return _Union[other, self] + + def __repr__(self): + if self.__forward_module__ is None: + module_repr = "" + else: + module_repr = f", module={self.__forward_module__!r}" + return f"ForwardRef({self.__forward_arg__!r}{module_repr})" + + +class _Stringifier: + # Must match the slots on ForwardRef, so we can turn an instance of one into an + # instance of the other in place. + __slots__ = _SLOTS + + def __init__( + self, + node, + globals=None, + owner=None, + is_class=False, + cell=None, + *, + stringifier_dict, + ): + # Either an AST node or a simple str (for the common case where a ForwardRef + # represent a single name). + assert isinstance(node, (ast.AST, str)) + self.__arg__ = None + self.__forward_evaluated__ = False + self.__forward_value__ = None + self.__forward_is_argument__ = False + self.__forward_is_class__ = is_class + self.__forward_module__ = None + self.__code__ = None + self.__ast_node__ = node + self.__globals__ = globals + self.__cell__ = cell + self.__owner__ = owner + self.__stringifier_dict__ = stringifier_dict + + def __convert_to_ast(self, other): + if isinstance(other, _Stringifier): + if isinstance(other.__ast_node__, str): + return ast.Name(id=other.__ast_node__) + return other.__ast_node__ + elif isinstance(other, slice): + return ast.Slice( + lower=( + self.__convert_to_ast(other.start) + if other.start is not None + else None + ), + upper=( + self.__convert_to_ast(other.stop) + if other.stop is not None + else None + ), + step=( + self.__convert_to_ast(other.step) + if other.step is not None + else None + ), + ) + else: + return ast.Constant(value=other) + + def __get_ast(self): + node = self.__ast_node__ + if isinstance(node, str): + return ast.Name(id=node) + return node + + def __make_new(self, node): + stringifier = _Stringifier( + node, + self.__globals__, + self.__owner__, + self.__forward_is_class__, + stringifier_dict=self.__stringifier_dict__, + ) + self.__stringifier_dict__.stringifiers.append(stringifier) + return stringifier + + # Must implement this since we set __eq__. We hash by identity so that + # stringifiers in dict keys are kept separate. + def __hash__(self): + return id(self) + + def __getitem__(self, other): + # Special case, to avoid stringifying references to class-scoped variables + # as '__classdict__["x"]'. + if self.__ast_node__ == "__classdict__": + raise KeyError + if isinstance(other, tuple): + elts = [self.__convert_to_ast(elt) for elt in other] + other = ast.Tuple(elts) + else: + other = self.__convert_to_ast(other) + assert isinstance(other, ast.AST), repr(other) + return self.__make_new(ast.Subscript(self.__get_ast(), other)) + + def __getattr__(self, attr): + return self.__make_new(ast.Attribute(self.__get_ast(), attr)) + + def __call__(self, *args, **kwargs): + return self.__make_new( + ast.Call( + self.__get_ast(), + [self.__convert_to_ast(arg) for arg in args], + [ + ast.keyword(key, self.__convert_to_ast(value)) + for key, value in kwargs.items() + ], + ) + ) + + def __iter__(self): + yield self.__make_new(ast.Starred(self.__get_ast())) + + def __repr__(self): + if isinstance(self.__ast_node__, str): + return self.__ast_node__ + return ast.unparse(self.__ast_node__) + + def __format__(self, format_spec): + raise TypeError("Cannot stringify annotation containing string formatting") + + def _make_binop(op: ast.AST): + def binop(self, other): + return self.__make_new( + ast.BinOp(self.__get_ast(), op, self.__convert_to_ast(other)) + ) + + return binop + + __add__ = _make_binop(ast.Add()) + __sub__ = _make_binop(ast.Sub()) + __mul__ = _make_binop(ast.Mult()) + __matmul__ = _make_binop(ast.MatMult()) + __truediv__ = _make_binop(ast.Div()) + __mod__ = _make_binop(ast.Mod()) + __lshift__ = _make_binop(ast.LShift()) + __rshift__ = _make_binop(ast.RShift()) + __or__ = _make_binop(ast.BitOr()) + __xor__ = _make_binop(ast.BitXor()) + __and__ = _make_binop(ast.BitAnd()) + __floordiv__ = _make_binop(ast.FloorDiv()) + __pow__ = _make_binop(ast.Pow()) + + del _make_binop + + def _make_rbinop(op: ast.AST): + def rbinop(self, other): + return self.__make_new( + ast.BinOp(self.__convert_to_ast(other), op, self.__get_ast()) + ) + + return rbinop + + __radd__ = _make_rbinop(ast.Add()) + __rsub__ = _make_rbinop(ast.Sub()) + __rmul__ = _make_rbinop(ast.Mult()) + __rmatmul__ = _make_rbinop(ast.MatMult()) + __rtruediv__ = _make_rbinop(ast.Div()) + __rmod__ = _make_rbinop(ast.Mod()) + __rlshift__ = _make_rbinop(ast.LShift()) + __rrshift__ = _make_rbinop(ast.RShift()) + __ror__ = _make_rbinop(ast.BitOr()) + __rxor__ = _make_rbinop(ast.BitXor()) + __rand__ = _make_rbinop(ast.BitAnd()) + __rfloordiv__ = _make_rbinop(ast.FloorDiv()) + __rpow__ = _make_rbinop(ast.Pow()) + + del _make_rbinop + + def _make_compare(op): + def compare(self, other): + return self.__make_new( + ast.Compare( + left=self.__get_ast(), + ops=[op], + comparators=[self.__convert_to_ast(other)], + ) + ) + + return compare + + __lt__ = _make_compare(ast.Lt()) + __le__ = _make_compare(ast.LtE()) + __eq__ = _make_compare(ast.Eq()) + __ne__ = _make_compare(ast.NotEq()) + __gt__ = _make_compare(ast.Gt()) + __ge__ = _make_compare(ast.GtE()) + + del _make_compare + + def _make_unary_op(op): + def unary_op(self): + return self.__make_new(ast.UnaryOp(op, self.__get_ast())) + + return unary_op + + __invert__ = _make_unary_op(ast.Invert()) + __pos__ = _make_unary_op(ast.UAdd()) + __neg__ = _make_unary_op(ast.USub()) + + del _make_unary_op + + +class _StringifierDict(dict): + def __init__(self, namespace, globals=None, owner=None, is_class=False): + super().__init__(namespace) + self.namespace = namespace + self.globals = globals + self.owner = owner + self.is_class = is_class + self.stringifiers = [] + + def __missing__(self, key): + fwdref = _Stringifier( + key, + globals=self.globals, + owner=self.owner, + is_class=self.is_class, + stringifier_dict=self, + ) + self.stringifiers.append(fwdref) + return fwdref + + +def call_evaluate_function(evaluate, format, *, owner=None): + """Call an evaluate function. Evaluate functions are normally generated for + the value of type aliases and the bounds, constraints, and defaults of + type parameter objects. + """ + return call_annotate_function(evaluate, format, owner=owner, _is_evaluate=True) + + +def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): + """Call an __annotate__ function. __annotate__ functions are normally + generated by the compiler to defer the evaluation of annotations. They + can be called with any of the format arguments in the Format enum, but + compiler-generated __annotate__ functions only support the VALUE format. + This function provides additional functionality to call __annotate__ + functions with the FORWARDREF and STRING formats. + + *annotate* must be an __annotate__ function, which takes a single argument + and returns a dict of annotations. + + *format* must be a member of the Format enum or one of the corresponding + integer values. + + *owner* can be the object that owns the annotations (i.e., the module, + class, or function that the __annotate__ function derives from). With the + FORWARDREF format, it is used to provide better evaluation capabilities + on the generated ForwardRef objects. + + """ + if format == Format.VALUE_WITH_FAKE_GLOBALS: + raise ValueError("The VALUE_WITH_FAKE_GLOBALS format is for internal use only") + try: + return annotate(format) + except NotImplementedError: + pass + if format == Format.STRING: + # STRING is implemented by calling the annotate function in a special + # environment where every name lookup results in an instance of _Stringifier. + # _Stringifier supports every dunder operation and returns a new _Stringifier. + # At the end, we get a dictionary that mostly contains _Stringifier objects (or + # possibly constants if the annotate function uses them directly). We then + # convert each of those into a string to get an approximation of the + # original source. + globals = _StringifierDict({}) + if annotate.__closure__: + freevars = annotate.__code__.co_freevars + new_closure = [] + for i, cell in enumerate(annotate.__closure__): + if i < len(freevars): + name = freevars[i] + else: + name = "__cell__" + fwdref = _Stringifier(name, stringifier_dict=globals) + new_closure.append(types.CellType(fwdref)) + closure = tuple(new_closure) + else: + closure = None + func = types.FunctionType( + annotate.__code__, + globals, + closure=closure, + argdefs=annotate.__defaults__, + kwdefaults=annotate.__kwdefaults__, + ) + annos = func(Format.VALUE_WITH_FAKE_GLOBALS) + if _is_evaluate: + return annos if isinstance(annos, str) else repr(annos) + return { + key: val if isinstance(val, str) else repr(val) + for key, val in annos.items() + } + elif format == Format.FORWARDREF: + # FORWARDREF is implemented similarly to STRING, but there are two changes, + # at the beginning and the end of the process. + # First, while STRING uses an empty dictionary as the namespace, so that all + # name lookups result in _Stringifier objects, FORWARDREF uses the globals + # and builtins, so that defined names map to their real values. + # Second, instead of returning strings, we want to return either real values + # or ForwardRef objects. To do this, we keep track of all _Stringifier objects + # created while the annotation is being evaluated, and at the end we convert + # them all to ForwardRef objects by assigning to __class__. To make this + # technique work, we have to ensure that the _Stringifier and ForwardRef + # classes share the same attributes. + # We use this technique because while the annotations are being evaluated, + # we want to support all operations that the language allows, including even + # __getattr__ and __eq__, and return new _Stringifier objects so we can accurately + # reconstruct the source. But in the dictionary that we eventually return, we + # want to return objects with more user-friendly behavior, such as an __eq__ + # that returns a bool and an defined set of attributes. + namespace = {**annotate.__builtins__, **annotate.__globals__} + is_class = isinstance(owner, type) + globals = _StringifierDict(namespace, annotate.__globals__, owner, is_class) + if annotate.__closure__: + freevars = annotate.__code__.co_freevars + new_closure = [] + for i, cell in enumerate(annotate.__closure__): + try: + cell.cell_contents + except ValueError: + if i < len(freevars): + name = freevars[i] + else: + name = "__cell__" + fwdref = _Stringifier( + name, + cell=cell, + owner=owner, + globals=annotate.__globals__, + is_class=is_class, + stringifier_dict=globals, + ) + globals.stringifiers.append(fwdref) + new_closure.append(types.CellType(fwdref)) + else: + new_closure.append(cell) + closure = tuple(new_closure) + else: + closure = None + func = types.FunctionType( + annotate.__code__, + globals, + closure=closure, + argdefs=annotate.__defaults__, + kwdefaults=annotate.__kwdefaults__, + ) + result = func(Format.VALUE_WITH_FAKE_GLOBALS) + for obj in globals.stringifiers: + obj.__class__ = ForwardRef + obj.__stringifier_dict__ = None # not needed for ForwardRef + if isinstance(obj.__ast_node__, str): + obj.__arg__ = obj.__ast_node__ + obj.__ast_node__ = None + return result + elif format == Format.VALUE: + # Should be impossible because __annotate__ functions must not raise + # NotImplementedError for this format. + raise RuntimeError("annotate function does not support VALUE format") + else: + raise ValueError(f"Invalid format: {format!r}") + + +# We use the descriptors from builtins.type instead of accessing +# .__annotations__ and .__annotate__ directly on class objects, because +# otherwise we could get wrong results in some cases involving metaclasses. +# See PEP 749. +_BASE_GET_ANNOTATE = type.__dict__["__annotate__"].__get__ +_BASE_GET_ANNOTATIONS = type.__dict__["__annotations__"].__get__ + + +def get_annotate_function(obj): + """Get the __annotate__ function for an object. + + obj may be a function, class, or module, or a user-defined type with + an `__annotate__` attribute. + + Returns the __annotate__ function or None. + """ + if isinstance(obj, type): + try: + return _BASE_GET_ANNOTATE(obj) + except AttributeError: + # AttributeError is raised for static types. + return None + return getattr(obj, "__annotate__", None) + + +def get_annotations( + obj, *, globals=None, locals=None, eval_str=False, format=Format.VALUE +): + """Compute the annotations dict for an object. + + obj may be a callable, class, or module. + Passing in an object of any other type raises TypeError. + + Returns a dict. get_annotations() returns a new dict every time + it's called; calling it twice on the same object will return two + different but equivalent dicts. + + This function handles several details for you: + + * If eval_str is true, values of type str will + be un-stringized using eval(). This is intended + for use with stringized annotations + ("from __future__ import annotations"). + * If obj doesn't have an annotations dict, returns an + empty dict. (Functions and methods always have an + annotations dict; classes, modules, and other types of + callables may not.) + * Ignores inherited annotations on classes. If a class + doesn't have its own annotations dict, returns an empty dict. + * All accesses to object members and dict values are done + using getattr() and dict.get() for safety. + * Always, always, always returns a freshly-created dict. + + eval_str controls whether or not values of type str are replaced + with the result of calling eval() on those values: + + * If eval_str is true, eval() is called on values of type str. + * If eval_str is false (the default), values of type str are unchanged. + + globals and locals are passed in to eval(); see the documentation + for eval() for more information. If either globals or locals is + None, this function may replace that value with a context-specific + default, contingent on type(obj): + + * If obj is a module, globals defaults to obj.__dict__. + * If obj is a class, globals defaults to + sys.modules[obj.__module__].__dict__ and locals + defaults to the obj class namespace. + * If obj is a callable, globals defaults to obj.__globals__, + although if obj is a wrapped function (using + functools.update_wrapper()) it is first unwrapped. + """ + if eval_str and format != Format.VALUE: + raise ValueError("eval_str=True is only supported with format=Format.VALUE") + + match format: + case Format.VALUE: + # For VALUE, we only look at __annotations__ + ann = _get_dunder_annotations(obj) + case Format.FORWARDREF: + # For FORWARDREF, we use __annotations__ if it exists + try: + return dict(_get_dunder_annotations(obj)) + except NameError: + pass + + # But if __annotations__ threw a NameError, we try calling __annotate__ + ann = _get_and_call_annotate(obj, format) + if ann is not None: + return ann + + # If that didn't work either, we have a very weird object: evaluating + # __annotations__ threw NameError and there is no __annotate__. In that case, + # we fall back to trying __annotations__ again. + return dict(_get_dunder_annotations(obj)) + case Format.STRING: + # For STRING, we try to call __annotate__ + ann = _get_and_call_annotate(obj, format) + if ann is not None: + return ann + # But if we didn't get it, we use __annotations__ instead. + ann = _get_dunder_annotations(obj) + return annotations_to_string(ann) + case Format.VALUE_WITH_FAKE_GLOBALS: + raise ValueError("The VALUE_WITH_FAKE_GLOBALS format is for internal use only") + case _: + raise ValueError(f"Unsupported format {format!r}") + + if not ann: + return {} + + if not eval_str: + return dict(ann) + + if isinstance(obj, type): + # class + obj_globals = None + module_name = getattr(obj, "__module__", None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + obj_globals = getattr(module, "__dict__", None) + obj_locals = dict(vars(obj)) + unwrap = obj + elif isinstance(obj, types.ModuleType): + # module + obj_globals = getattr(obj, "__dict__") + obj_locals = None + unwrap = None + elif callable(obj): + # this includes types.Function, types.BuiltinFunctionType, + # types.BuiltinMethodType, functools.partial, functools.singledispatch, + # "class funclike" from Lib/test/test_inspect... on and on it goes. + obj_globals = getattr(obj, "__globals__", None) + obj_locals = None + unwrap = obj + elif ann is not None: + obj_globals = obj_locals = unwrap = None + else: + raise TypeError(f"{obj!r} is not a module, class, or callable.") + + if unwrap is not None: + while True: + if hasattr(unwrap, "__wrapped__"): + unwrap = unwrap.__wrapped__ + continue + if isinstance(unwrap, functools.partial): + unwrap = unwrap.func + continue + break + if hasattr(unwrap, "__globals__"): + obj_globals = unwrap.__globals__ + + if globals is None: + globals = obj_globals + if locals is None: + locals = obj_locals + + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + if type_params := getattr(obj, "__type_params__", ()): + if locals is None: + locals = {} + locals = {param.__name__: param for param in type_params} | locals + + return_value = { + key: value if not isinstance(value, str) else eval(value, globals, locals) + for key, value in ann.items() + } + return return_value + + +def value_to_string(value): + """Convert a Python value to a format suitable for use with the STRING format. + + This is inteded as a helper for tools that support the STRING format but do + not have access to the code that originally produced the annotations. It uses + repr() for most objects. + + """ + if isinstance(value, type): + if value.__module__ == "builtins": + return value.__qualname__ + return f"{value.__module__}.{value.__qualname__}" + if value is ...: + return "..." + if isinstance(value, (types.FunctionType, types.BuiltinFunctionType)): + return value.__name__ + return repr(value) + + +def annotations_to_string(annotations): + """Convert an annotation dict containing values to approximately the STRING format.""" + return { + n: t if isinstance(t, str) else value_to_string(t) + for n, t in annotations.items() + } + + +def _get_and_call_annotate(obj, format): + annotate = get_annotate_function(obj) + if annotate is not None: + ann = call_annotate_function(annotate, format, owner=obj) + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotate__ returned a non-dict") + return dict(ann) + return None + + +def _get_dunder_annotations(obj): + if isinstance(obj, type): + try: + ann = _BASE_GET_ANNOTATIONS(obj) + except AttributeError: + # For static types, the descriptor raises AttributeError. + return {} + else: + ann = getattr(obj, "__annotations__", None) + if ann is None: + return {} + + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") + return dict(ann) diff --git a/Lib/argparse.py b/Lib/argparse.py index 0dbdd67a82f391..d24fa72e573d4f 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -18,11 +18,12 @@ 'integers', metavar='int', nargs='+', type=int, help='an integer to be summed') parser.add_argument( - '--log', default=sys.stdout, type=argparse.FileType('w'), + '--log', help='the file where the sum should be written') args = parser.parse_args() - args.log.write('%s' % sum(args.integers)) - args.log.close() + with (open(args.log, 'w') if args.log is not None + else contextlib.nullcontext(sys.stdout)) as log: + log.write('%s' % sum(args.integers)) The module contains the following public classes: @@ -39,7 +40,8 @@ - FileType -- A factory for defining types of files to be created. As the example above shows, instances of FileType are typically passed as - the type= argument of add_argument() calls. + the type= argument of add_argument() calls. Deprecated since + Python 3.14. - Action -- The base class for parser actions. Typically actions are selected by passing strings like 'store_true' or 'append_const' to @@ -261,13 +263,12 @@ def add_argument(self, action): # find all invocations get_invocation = self._format_action_invocation - invocations = [get_invocation(action)] + invocation_lengths = [len(get_invocation(action)) + self._current_indent] for subaction in self._iter_indented_subactions(action): - invocations.append(get_invocation(subaction)) + invocation_lengths.append(len(get_invocation(subaction)) + self._current_indent) # update the maximum item length - invocation_length = max(map(len, invocations)) - action_length = invocation_length + self._current_indent + action_length = max(invocation_lengths) self._action_max_length = max(self._action_max_length, action_length) @@ -328,17 +329,8 @@ def _format_usage(self, usage, actions, groups, prefix): if len(prefix) + len(usage) > text_width: # break usage into wrappable parts - part_regexp = ( - r'\(.*?\)+(?=\s|$)|' - r'\[.*?\]+(?=\s|$)|' - r'\S+' - ) - opt_usage = format(optionals, groups) - pos_usage = format(positionals, groups) - opt_parts = _re.findall(part_regexp, opt_usage) - pos_parts = _re.findall(part_regexp, pos_usage) - assert ' '.join(opt_parts) == opt_usage - assert ' '.join(pos_parts) == pos_usage + opt_parts = self._get_actions_usage_parts(optionals, groups) + pos_parts = self._get_actions_usage_parts(positionals, groups) # helper for wrapping lines def get_lines(parts, indent, prefix=None): @@ -391,6 +383,9 @@ def get_lines(parts, indent, prefix=None): return '%s%s\n\n' % (prefix, usage) def _format_actions_usage(self, actions, groups): + return ' '.join(self._get_actions_usage_parts(actions, groups)) + + def _get_actions_usage_parts(self, actions, groups): # find group indices and identify actions in groups group_actions = set() inserts = {} @@ -398,58 +393,26 @@ def _format_actions_usage(self, actions, groups): if not group._group_actions: raise ValueError(f'empty group {group}') + if all(action.help is SUPPRESS for action in group._group_actions): + continue + try: - start = actions.index(group._group_actions[0]) + start = min(actions.index(item) for item in group._group_actions) except ValueError: continue else: - group_action_count = len(group._group_actions) - end = start + group_action_count - if actions[start:end] == group._group_actions: - - suppressed_actions_count = 0 - for action in group._group_actions: - group_actions.add(action) - if action.help is SUPPRESS: - suppressed_actions_count += 1 - - exposed_actions_count = group_action_count - suppressed_actions_count - if not exposed_actions_count: - continue - - if not group.required: - if start in inserts: - inserts[start] += ' [' - else: - inserts[start] = '[' - if end in inserts: - inserts[end] += ']' - else: - inserts[end] = ']' - elif exposed_actions_count > 1: - if start in inserts: - inserts[start] += ' (' - else: - inserts[start] = '(' - if end in inserts: - inserts[end] += ')' - else: - inserts[end] = ')' - for i in range(start + 1, end): - inserts[i] = '|' + end = start + len(group._group_actions) + if set(actions[start:end]) == set(group._group_actions): + group_actions.update(group._group_actions) + inserts[start, end] = group # collect all actions format strings parts = [] - for i, action in enumerate(actions): + for action in actions: # suppressed arguments are marked with None - # remove | separators for suppressed arguments if action.help is SUPPRESS: - parts.append(None) - if inserts.get(i) == '|': - inserts.pop(i) - elif inserts.get(i + 1) == '|': - inserts.pop(i + 1) + part = None # produce all arg strings elif not action.option_strings: @@ -461,9 +424,6 @@ def _format_actions_usage(self, actions, groups): if part[0] == '[' and part[-1] == ']': part = part[1:-1] - # add the action string to the list - parts.append(part) - # produce the first way to invoke the option in brackets else: option_string = action.option_strings[0] @@ -484,26 +444,32 @@ def _format_actions_usage(self, actions, groups): if not action.required and action not in group_actions: part = '[%s]' % part - # add the action string to the list - parts.append(part) - - # insert things at the necessary indices - for i in sorted(inserts, reverse=True): - parts[i:i] = [inserts[i]] - - # join all the action items with spaces - text = ' '.join([item for item in parts if item is not None]) + # add the action string to the list + parts.append(part) - # clean up separators for mutually exclusive groups - open = r'[\[(]' - close = r'[\])]' - text = _re.sub(r'(%s) ' % open, r'\1', text) - text = _re.sub(r' (%s)' % close, r'\1', text) - text = _re.sub(r'%s *%s' % (open, close), r'', text) - text = text.strip() - - # return the text - return text + # group mutually exclusive actions + inserted_separators_indices = set() + for start, end in sorted(inserts, reverse=True): + group = inserts[start, end] + group_parts = [item for item in parts[start:end] if item is not None] + group_size = len(group_parts) + if group.required: + open, close = "()" if group_size > 1 else ("", "") + else: + open, close = "[]" + group_parts[0] = open + group_parts[0] + group_parts[-1] = group_parts[-1] + close + for i, part in enumerate(group_parts[:-1], start=start): + # insert a separator if not already done in a nested group + if i not in inserted_separators_indices: + parts[i] = part + ' |' + inserted_separators_indices.add(i) + parts[start + group_size - 1] = group_parts[-1] + for i in range(start + group_size, end): + parts[i] = None + + # return the usage parts + return [item for item in parts if item is not None] def _format_text(self, text): if '%(prog)' in text: @@ -563,8 +529,7 @@ def _format_action(self, action): def _format_action_invocation(self, action): if not action.option_strings: default = self._get_default_metavar_for_positional(action) - metavar, = self._metavar_formatter(action, default)(1) - return metavar + return ' '.join(self._metavar_formatter(action, default)(1)) else: @@ -584,8 +549,7 @@ def _metavar_formatter(self, action, default_metavar): if action.metavar is not None: result = action.metavar elif action.choices is not None: - choice_strs = [str(choice) for choice in action.choices] - result = '{%s}' % ','.join(choice_strs) + result = '{%s}' % ','.join(map(str, action.choices)) else: result = default_metavar @@ -625,17 +589,19 @@ def _format_args(self, action, default_metavar): return result def _expand_help(self, action): + help_string = self._get_help_string(action) + if '%' not in help_string: + return help_string params = dict(vars(action), prog=self._prog) for name in list(params): - if params[name] is SUPPRESS: + value = params[name] + if value is SUPPRESS: del params[name] - for name in list(params): - if hasattr(params[name], '__name__'): - params[name] = params[name].__name__ + elif hasattr(value, '__name__'): + params[name] = value.__name__ if params.get('choices') is not None: - choices_str = ', '.join([str(c) for c in params['choices']]) - params['choices'] = choices_str - return self._get_help_string(action) % params + params['choices'] = ', '.join(map(str, params['choices'])) + return help_string % params def _iter_indented_subactions(self, action): try: @@ -739,11 +705,19 @@ def _get_action_name(argument): elif argument.option_strings: return '/'.join(argument.option_strings) elif argument.metavar not in (None, SUPPRESS): - return argument.metavar + metavar = argument.metavar + if not isinstance(metavar, tuple): + return metavar + if argument.nargs == ZERO_OR_MORE and len(metavar) == 2: + return '%s[, %s]' % metavar + elif argument.nargs == ONE_OR_MORE: + return '%s[, %s]' % metavar + else: + return ', '.join(metavar) elif argument.dest not in (None, SUPPRESS): return argument.dest elif argument.choices: - return '{' + ','.join(argument.choices) + '}' + return '{%s}' % ','.join(map(str, argument.choices)) else: return None @@ -872,22 +846,16 @@ def format_usage(self): return self.option_strings[0] def __call__(self, parser, namespace, values, option_string=None): - raise NotImplementedError(_('.__call__() not defined')) - + raise NotImplementedError('.__call__() not defined') -# FIXME: remove together with `BooleanOptionalAction` deprecated arguments. -_deprecated_default = object() class BooleanOptionalAction(Action): def __init__(self, option_strings, dest, default=None, - type=_deprecated_default, - choices=_deprecated_default, required=False, help=None, - metavar=_deprecated_default, deprecated=False): _option_strings = [] @@ -895,38 +863,19 @@ def __init__(self, _option_strings.append(option_string) if option_string.startswith('--'): + if option_string.startswith('--no-'): + raise ValueError(f'invalid option name {option_string!r} ' + f'for BooleanOptionalAction') option_string = '--no-' + option_string[2:] _option_strings.append(option_string) - # We need `_deprecated` special value to ban explicit arguments that - # match default value. Like: - # parser.add_argument('-f', action=BooleanOptionalAction, type=int) - for field_name in ('type', 'choices', 'metavar'): - if locals()[field_name] is not _deprecated_default: - import warnings - warnings._deprecated( - field_name, - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - if type is _deprecated_default: - type = None - if choices is _deprecated_default: - choices = None - if metavar is _deprecated_default: - metavar = None - super().__init__( option_strings=_option_strings, dest=dest, nargs=0, default=default, - type=type, - choices=choices, required=required, help=help, - metavar=metavar, deprecated=deprecated) @@ -1226,20 +1175,23 @@ def add_parser(self, name, *, deprecated=False, **kwargs): aliases = kwargs.pop('aliases', ()) if name in self._name_parser_map: - raise ArgumentError(self, _('conflicting subparser: %s') % name) + raise ValueError(f'conflicting subparser: {name}') for alias in aliases: if alias in self._name_parser_map: - raise ArgumentError( - self, _('conflicting subparser alias: %s') % alias) + raise ValueError(f'conflicting subparser alias: {alias}') # create a pseudo-action to hold the choice help if 'help' in kwargs: help = kwargs.pop('help') choice_action = self._ChoicesPseudoAction(name, aliases, help) self._choices_actions.append(choice_action) + else: + choice_action = None # create the parser and add it to the map parser = self._parser_class(**kwargs) + if choice_action is not None: + parser._check_help(choice_action) self._name_parser_map[name] = parser # make parser available under aliases also @@ -1288,7 +1240,8 @@ def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, key, value) if arg_strings: - vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) + if not hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR): + setattr(namespace, _UNRECOGNIZED_ARGS_ATTR, []) getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) class _ExtendAction(_AppendAction): @@ -1303,7 +1256,7 @@ def __call__(self, parser, namespace, values, option_string=None): # ============== class FileType(object): - """Factory for creating file object types + """Deprecated factory for creating file object types Instances of FileType are typically passed as type= arguments to the ArgumentParser add_argument() method. @@ -1320,6 +1273,12 @@ class FileType(object): """ def __init__(self, mode='r', bufsize=-1, encoding=None, errors=None): + import warnings + warnings.warn( + "FileType is deprecated. Simply open files after parsing arguments.", + category=PendingDeprecationWarning, + stacklevel=2 + ) self._mode = mode self._bufsize = bufsize self._encoding = encoding @@ -1423,7 +1382,7 @@ def __init__(self, self._defaults = {} # determines whether an "option" looks like a negative number - self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$') + self._negative_number_matcher = _re.compile(r'-\.?\d') # whether or not there are any optionals that look like negative # numbers -- uses a list so it can be shared and edited @@ -1473,7 +1432,8 @@ def add_argument(self, *args, **kwargs): chars = self.prefix_chars if not args or len(args) == 1 and args[0][0] not in chars: if args and 'dest' in kwargs: - raise ValueError('dest supplied twice for positional argument') + raise TypeError('dest supplied twice for positional argument,' + ' did you mean metavar?') kwargs = self._get_positional_kwargs(*args, **kwargs) # otherwise, we're adding an optional argument @@ -1489,27 +1449,34 @@ def add_argument(self, *args, **kwargs): kwargs['default'] = self.argument_default # create the action object, and add it to the parser + action_name = kwargs.get('action') action_class = self._pop_action_class(kwargs) if not callable(action_class): - raise ValueError('unknown action "%s"' % (action_class,)) + raise ValueError('unknown action {action_class!r}') action = action_class(**kwargs) + # raise an error if action for positional argument does not + # consume arguments + if not action.option_strings and action.nargs == 0: + raise ValueError(f'action {action_name!r} is not valid for positional arguments') + # raise an error if the action type is not callable type_func = self._registry_get('type', action.type, action.type) if not callable(type_func): - raise ValueError('%r is not callable' % (type_func,)) + raise TypeError(f'{type_func!r} is not callable') if type_func is FileType: - raise ValueError('%r is a FileType class object, instance of it' - ' must be passed' % (type_func,)) + raise TypeError(f'{type_func!r} is a FileType class object, ' + f'instance of it must be passed') # raise an error if the metavar does not match the type if hasattr(self, "_get_formatter"): + formatter = self._get_formatter() try: - self._get_formatter()._format_args(action, None) + formatter._format_args(action, None) except TypeError: raise ValueError("length of metavar tuple does not match nargs") - + self._check_help(action) return self._add_action(action) def add_argument_group(self, *args, **kwargs): @@ -1553,8 +1520,8 @@ def _add_container_actions(self, container): if group.title in title_group_map: # This branch could happen if a derived class added # groups with duplicated titles in __init__ - msg = _('cannot merge actions - two groups are named %r') - raise ValueError(msg % (group.title)) + msg = f'cannot merge actions - two groups are named {group.title!r}' + raise ValueError(msg) title_group_map[group.title] = group # map each action to its group @@ -1577,7 +1544,11 @@ def _add_container_actions(self, container): # NOTE: if add_mutually_exclusive_group ever gains title= and # description= then this code will need to be expanded as above for group in container._mutually_exclusive_groups: - mutex_group = self.add_mutually_exclusive_group( + if group._container is container: + cont = self + else: + cont = title_group_map[group._container.title] + mutex_group = cont.add_mutually_exclusive_group( required=group.required) # map the actions to their new mutex group @@ -1591,14 +1562,15 @@ def _add_container_actions(self, container): def _get_positional_kwargs(self, dest, **kwargs): # make sure required is not specified if 'required' in kwargs: - msg = _("'required' is an invalid argument for positionals") + msg = "'required' is an invalid argument for positionals" raise TypeError(msg) # mark positional arguments as required if at least one is # always required - if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]: - kwargs['required'] = True - if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs: + nargs = kwargs.get('nargs') + if nargs == 0: + raise ValueError('nargs for positionals must be != 0') + if nargs not in [OPTIONAL, ZERO_OR_MORE, REMAINDER, SUPPRESS]: kwargs['required'] = True # return the keyword arguments with no option strings @@ -1611,11 +1583,9 @@ def _get_optional_kwargs(self, *args, **kwargs): for option_string in args: # error on strings that don't start with an appropriate prefix if not option_string[0] in self.prefix_chars: - args = {'option': option_string, - 'prefix_chars': self.prefix_chars} - msg = _('invalid option string %(option)r: ' - 'must start with a character %(prefix_chars)r') - raise ValueError(msg % args) + raise ValueError( + f'invalid option string {option_string!r}: ' + f'must start with a character {self.prefix_chars!r}') # strings starting with two prefix characters are long options option_strings.append(option_string) @@ -1631,8 +1601,8 @@ def _get_optional_kwargs(self, *args, **kwargs): dest_option_string = option_strings[0] dest = dest_option_string.lstrip(self.prefix_chars) if not dest: - msg = _('dest= is required for options like %r') - raise ValueError(msg % option_string) + msg = f'dest= is required for options like {option_string!r}' + raise TypeError(msg) dest = dest.replace('-', '_') # return the updated keyword arguments @@ -1648,8 +1618,8 @@ def _get_handler(self): try: return getattr(self, handler_func_name) except AttributeError: - msg = _('invalid conflict_resolution value: %r') - raise ValueError(msg % self.conflict_handler) + msg = f'invalid conflict_resolution value: {self.conflict_handler!r}' + raise ValueError(msg) def _check_conflict(self, action): @@ -1688,10 +1658,26 @@ def _handle_conflict_resolve(self, action, conflicting_actions): if not action.option_strings: action.container._remove_action(action) + def _check_help(self, action): + if action.help and hasattr(self, "_get_formatter"): + formatter = self._get_formatter() + try: + formatter._expand_help(action) + except (ValueError, TypeError, KeyError) as exc: + raise ValueError('badly formed help string') from exc + class _ArgumentGroup(_ActionsContainer): def __init__(self, container, title=None, description=None, **kwargs): + if 'prefix_chars' in kwargs: + import warnings + depr_msg = ( + "The use of the undocumented 'prefix_chars' parameter in " + "ArgumentParser.add_argument_group() is deprecated." + ) + warnings.warn(depr_msg, DeprecationWarning, stacklevel=3) + # add any missing keyword arguments by checking the container update = kwargs.setdefault update('conflict_handler', container.conflict_handler) @@ -1723,14 +1709,7 @@ def _remove_action(self, action): self._group_actions.remove(action) def add_argument_group(self, *args, **kwargs): - import warnings - warnings.warn( - "Nesting argument groups is deprecated.", - category=DeprecationWarning, - stacklevel=2 - ) - return super().add_argument_group(*args, **kwargs) - + raise ValueError('argument groups cannot be nested') class _MutuallyExclusiveGroup(_ArgumentGroup): @@ -1741,7 +1720,7 @@ def __init__(self, container, required=False): def _add_action(self, action): if action.required: - msg = _('mutually exclusive arguments must be optional') + msg = 'mutually exclusive arguments must be optional' raise ValueError(msg) action = self._container._add_action(action) self._group_actions.append(action) @@ -1751,14 +1730,29 @@ def _remove_action(self, action): self._container._remove_action(action) self._group_actions.remove(action) - def add_mutually_exclusive_group(self, *args, **kwargs): - import warnings - warnings.warn( - "Nesting mutually exclusive groups is deprecated.", - category=DeprecationWarning, - stacklevel=2 - ) - return super().add_mutually_exclusive_group(*args, **kwargs) + def add_mutually_exclusive_group(self, **kwargs): + raise ValueError('mutually exclusive groups cannot be nested') + +def _prog_name(prog=None): + if prog is not None: + return prog + arg0 = _sys.argv[0] + try: + modspec = _sys.modules['__main__'].__spec__ + except (KeyError, AttributeError): + # possibly PYTHONSTARTUP or -X presite or other weird edge case + # no good answer here, so fall back to the default + modspec = None + if modspec is None: + # simple script + return _os.path.basename(arg0) + py = _os.path.basename(_sys.executable) + if modspec.name != '__main__': + # imported module or package + modname = modspec.name.removesuffix('.__main__') + return f'{py} -m {modname}' + # directory or ZIP file + return f'{py} {arg0}' class ArgumentParser(_AttributeHolder, _ActionsContainer): @@ -1781,6 +1775,8 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer): - allow_abbrev -- Allow long options to be abbreviated unambiguously - exit_on_error -- Determines whether or not ArgumentParser exits with error info when an error occurs + - suggest_on_error - Enables suggestions for mistyped argument choices + and subparser names. (default: ``False``) """ def __init__(self, @@ -1796,7 +1792,8 @@ def __init__(self, conflict_handler='error', add_help=True, allow_abbrev=True, - exit_on_error=True): + exit_on_error=True, + suggest_on_error=False): superinit = super(ArgumentParser, self).__init__ superinit(description=description, @@ -1804,11 +1801,7 @@ def __init__(self, argument_default=argument_default, conflict_handler=conflict_handler) - # default setting for prog - if prog is None: - prog = _os.path.basename(_sys.argv[0]) - - self.prog = prog + self.prog = _prog_name(prog) self.usage = usage self.epilog = epilog self.formatter_class = formatter_class @@ -1816,6 +1809,7 @@ def __init__(self, self.add_help = add_help self.allow_abbrev = allow_abbrev self.exit_on_error = exit_on_error + self.suggest_on_error = suggest_on_error add_group = self.add_argument_group self._positionals = add_group(_('positional arguments')) @@ -1863,14 +1857,14 @@ def _get_kwargs(self): # ================================== def add_subparsers(self, **kwargs): if self._subparsers is not None: - self.error(_('cannot have multiple subparser arguments')) + raise ValueError('cannot have multiple subparser arguments') # add the parser class to the arguments if it's not present kwargs.setdefault('parser_class', type(self)) if 'title' in kwargs or 'description' in kwargs: - title = _(kwargs.pop('title', 'subcommands')) - description = _(kwargs.pop('description', None)) + title = kwargs.pop('title', _('subcommands')) + description = kwargs.pop('description', None) self._subparsers = self.add_argument_group(title, description) else: self._subparsers = self._positionals @@ -1881,12 +1875,13 @@ def add_subparsers(self, **kwargs): formatter = self._get_formatter() positionals = self._get_positional_actions() groups = self._mutually_exclusive_groups - formatter.add_usage(self.usage, positionals, groups, '') + formatter.add_usage(None, positionals, groups, '') kwargs['prog'] = formatter.format_help().strip() # create the parsers action and add it to the positionals list parsers_class = self._pop_action_class(kwargs, 'parsers') action = parsers_class(option_strings=[], **kwargs) + self._check_help(action) self._subparsers._add_action(action) # return the created parsers action @@ -1915,11 +1910,17 @@ def _get_positional_actions(self): def parse_args(self, args=None, namespace=None): args, argv = self.parse_known_args(args, namespace) if argv: - msg = _('unrecognized arguments: %s') - self.error(msg % ' '.join(argv)) + msg = _('unrecognized arguments: %s') % ' '.join(argv) + if self.exit_on_error: + self.error(msg) + else: + raise ArgumentError(None, msg) return args def parse_known_args(self, args=None, namespace=None): + return self._parse_known_args2(args, namespace, intermixed=False) + + def _parse_known_args2(self, args, namespace, intermixed): if args is None: # args default to the system args args = _sys.argv[1:] @@ -1946,18 +1947,18 @@ def parse_known_args(self, args=None, namespace=None): # parse the arguments and exit if there are any errors if self.exit_on_error: try: - namespace, args = self._parse_known_args(args, namespace) + namespace, args = self._parse_known_args(args, namespace, intermixed) except ArgumentError as err: self.error(str(err)) else: - namespace, args = self._parse_known_args(args, namespace) + namespace, args = self._parse_known_args(args, namespace, intermixed) if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR): args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR)) delattr(namespace, _UNRECOGNIZED_ARGS_ATTR) return namespace, args - def _parse_known_args(self, arg_strings, namespace): + def _parse_known_args(self, arg_strings, namespace, intermixed): # replace arg strings that are file references if self.fromfile_prefix_chars is not None: arg_strings = self._read_args_from_files(arg_strings) @@ -1989,11 +1990,11 @@ def _parse_known_args(self, arg_strings, namespace): # otherwise, add the arg to the arg strings # and note the index if it was an option else: - option_tuple = self._parse_optional(arg_string) - if option_tuple is None: + option_tuples = self._parse_optional(arg_string) + if option_tuples is None: pattern = 'A' else: - option_string_indices[i] = option_tuple + option_string_indices[i] = option_tuples pattern = 'O' arg_string_pattern_parts.append(pattern) @@ -2010,9 +2011,8 @@ def take_action(action, argument_strings, option_string=None): argument_values = self._get_values(action, argument_strings) # error if this argument is not allowed with other previously - # seen arguments, assuming that actions that use the default - # value don't really count as "present" - if argument_values is not action.default: + # seen arguments + if action.option_strings or argument_strings: seen_non_default_actions.add(action) for conflict_action in action_conflicts.get(action, []): if conflict_action in seen_non_default_actions: @@ -2029,8 +2029,16 @@ def take_action(action, argument_strings, option_string=None): def consume_optional(start_index): # get the optional identified at this index - option_tuple = option_string_indices[start_index] - action, option_string, sep, explicit_arg = option_tuple + option_tuples = option_string_indices[start_index] + # if multiple actions match, the option string was ambiguous + if len(option_tuples) > 1: + options = ', '.join([option_string + for action, option_string, sep, explicit_arg in option_tuples]) + args = {'option': arg_strings[start_index], 'matches': options} + msg = _('ambiguous option: %(option)s could match %(matches)s') + raise ArgumentError(None, msg % args) + + action, option_string, sep, explicit_arg = option_tuples[0] # identify additional optionals in the same arg string # (e.g. -xyz is the same as -x -y -z if no args are required) @@ -2041,6 +2049,7 @@ def consume_optional(start_index): # if we found no optional action, skip it if action is None: extras.append(arg_strings[start_index]) + extras_pattern.append('O') return start_index + 1 # if there is an explicit argument, try to match the @@ -2076,6 +2085,7 @@ def consume_optional(start_index): sep = '' else: extras.append(char + explicit_arg) + extras_pattern.append('O') stop = start_index + 1 break # if the action expect exactly one argument, we've @@ -2130,6 +2140,15 @@ def consume_positionals(start_index): # and add the Positional and its args to the list for action, arg_count in zip(positionals, arg_counts): args = arg_strings[start_index: start_index + arg_count] + # Strip out the first '--' if it is not in REMAINDER arg. + if action.nargs == PARSER: + if arg_strings_pattern[start_index] == '-': + assert args[0] == '--' + args.remove('--') + elif action.nargs != REMAINDER: + if (arg_strings_pattern.find('-', start_index, + start_index + arg_count) >= 0): + args.remove('--') start_index += arg_count if args and action.deprecated and action.dest not in warned: self._warning(_("argument '%(argument_name)s' is deprecated") % @@ -2145,6 +2164,7 @@ def consume_positionals(start_index): # consume Positionals and Optionals alternately, until we have # passed the last option string extras = [] + extras_pattern = [] start_index = 0 if option_string_indices: max_option_string_index = max(option_string_indices) @@ -2158,7 +2178,7 @@ def consume_positionals(start_index): if next_option_string_index in option_string_indices: break next_option_string_index += 1 - if start_index != next_option_string_index: + if not intermixed and start_index != next_option_string_index: positionals_end_index = consume_positionals(start_index) # only try to parse the next optional if we didn't consume @@ -2174,16 +2194,35 @@ def consume_positionals(start_index): if start_index not in option_string_indices: strings = arg_strings[start_index:next_option_string_index] extras.extend(strings) + extras_pattern.extend(arg_strings_pattern[start_index:next_option_string_index]) start_index = next_option_string_index # consume the next optional and any arguments for it start_index = consume_optional(start_index) - # consume any positionals following the last Optional - stop_index = consume_positionals(start_index) + if not intermixed: + # consume any positionals following the last Optional + stop_index = consume_positionals(start_index) - # if we didn't consume all the argument strings, there were extras - extras.extend(arg_strings[stop_index:]) + # if we didn't consume all the argument strings, there were extras + extras.extend(arg_strings[stop_index:]) + else: + extras.extend(arg_strings[start_index:]) + extras_pattern.extend(arg_strings_pattern[start_index:]) + extras_pattern = ''.join(extras_pattern) + assert len(extras_pattern) == len(extras) + # consume all positionals + arg_strings = [s for s, c in zip(extras, extras_pattern) if c != 'O'] + arg_strings_pattern = extras_pattern.replace('O', '') + stop_index = consume_positionals(0) + # leave unknown optionals and non-consumed positionals in extras + for i, c in enumerate(extras_pattern): + if not stop_index: + break + if c != 'O': + stop_index -= 1 + extras[i] = None + extras = [s for s in extras if s is not None] # make sure all required actions were present and also convert # action defaults which were not given as arguments @@ -2205,7 +2244,7 @@ def consume_positionals(start_index): self._get_value(action, action.default)) if required_actions: - self.error(_('the following arguments are required: %s') % + raise ArgumentError(None, _('the following arguments are required: %s') % ', '.join(required_actions)) # make sure all required groups had one option present @@ -2221,7 +2260,7 @@ def consume_positionals(start_index): for action in group._group_actions if action.help is not SUPPRESS] msg = _('one of the arguments %s is required') - self.error(msg % ' '.join(names)) + raise ArgumentError(None, msg % ' '.join(names)) # return the updated namespace and the extra arguments return namespace, extras @@ -2248,7 +2287,7 @@ def _read_args_from_files(self, arg_strings): arg_strings = self._read_args_from_files(arg_strings) new_arg_strings.extend(arg_strings) except OSError as err: - self.error(str(err)) + raise ArgumentError(None, str(err)) # return the modified argument list return new_arg_strings @@ -2281,18 +2320,19 @@ def _match_argument(self, action, arg_strings_pattern): def _match_arguments_partial(self, actions, arg_strings_pattern): # progressively shorten the actions list by slicing off the # final actions until we find a match - result = [] for i in range(len(actions), 0, -1): actions_slice = actions[:i] pattern = ''.join([self._get_nargs_pattern(action) for action in actions_slice]) match = _re.match(pattern, arg_strings_pattern) if match is not None: - result.extend([len(string) for string in match.groups()]) - break - - # return the list of arg string counts - return result + result = [len(string) for string in match.groups()] + if (match.end() < len(arg_strings_pattern) + and arg_strings_pattern[match.end()] == 'O'): + while result and not result[-1]: + del result[-1] + return result + return [] def _parse_optional(self, arg_string): # if it's an empty string, it was meant to be a positional @@ -2306,7 +2346,7 @@ def _parse_optional(self, arg_string): # if the option string is present in the parser, return the action if arg_string in self._option_string_actions: action = self._option_string_actions[arg_string] - return action, arg_string, None, None + return [(action, arg_string, None, None)] # if it's just a single character, it was meant to be positional if len(arg_string) == 1: @@ -2316,25 +2356,14 @@ def _parse_optional(self, arg_string): option_string, sep, explicit_arg = arg_string.partition('=') if sep and option_string in self._option_string_actions: action = self._option_string_actions[option_string] - return action, option_string, sep, explicit_arg + return [(action, option_string, sep, explicit_arg)] # search through all possible prefixes of the option string # and all actions in the parser for possible interpretations option_tuples = self._get_option_tuples(arg_string) - # if multiple actions match, the option string was ambiguous - if len(option_tuples) > 1: - options = ', '.join([option_string - for action, option_string, sep, explicit_arg in option_tuples]) - args = {'option': arg_string, 'matches': options} - msg = _('ambiguous option: %(option)s could match %(matches)s') - self.error(msg % args) - - # if exactly one action matched, this segmentation is good, - # so return the parsed action - elif len(option_tuples) == 1: - option_tuple, = option_tuples - return option_tuple + if option_tuples: + return option_tuples # if it was not found as an option, but it looks like a negative # number, it was meant to be positional @@ -2349,7 +2378,7 @@ def _parse_optional(self, arg_string): # it was meant to be an optional but there is no such option # in this parser (though it might be a valid option in a subparser) - return None, arg_string, None, None + return [(None, arg_string, None, None)] def _get_option_tuples(self, option_string): result = [] @@ -2372,7 +2401,9 @@ def _get_option_tuples(self, option_string): # but multiple character options always have to have their argument # separate elif option_string[0] in chars and option_string[1] not in chars: - option_prefix = option_string + option_prefix, sep, explicit_arg = option_string.partition('=') + if not sep: + sep = explicit_arg = None short_option_prefix = option_string[:2] short_explicit_arg = option_string[2:] @@ -2381,14 +2412,14 @@ def _get_option_tuples(self, option_string): action = self._option_string_actions[option_string] tup = action, option_string, '', short_explicit_arg result.append(tup) - elif option_string.startswith(option_prefix): + elif self.allow_abbrev and option_string.startswith(option_prefix): action = self._option_string_actions[option_string] - tup = action, option_string, None, None + tup = action, option_string, sep, explicit_arg result.append(tup) # shouldn't ever get here else: - self.error(_('unexpected option string: %s') % option_string) + raise ArgumentError(None, _('unexpected option string: %s') % option_string) # return the collected option tuples return result @@ -2397,43 +2428,40 @@ def _get_nargs_pattern(self, action): # in all examples below, we have to allow for '--' args # which are represented as '-' in the pattern nargs = action.nargs + # if this is an optional action, -- is not allowed + option = action.option_strings # the default (None) is assumed to be a single argument if nargs is None: - nargs_pattern = '(-*A-*)' + nargs_pattern = '([A])' if option else '(-*A-*)' # allow zero or one arguments elif nargs == OPTIONAL: - nargs_pattern = '(-*A?-*)' + nargs_pattern = '(A?)' if option else '(-*A?-*)' # allow zero or more arguments elif nargs == ZERO_OR_MORE: - nargs_pattern = '(-*[A-]*)' + nargs_pattern = '(A*)' if option else '(-*[A-]*)' # allow one or more arguments elif nargs == ONE_OR_MORE: - nargs_pattern = '(-*A[A-]*)' + nargs_pattern = '(A+)' if option else '(-*A[A-]*)' # allow any number of options or arguments elif nargs == REMAINDER: - nargs_pattern = '([-AO]*)' + nargs_pattern = '([AO]*)' if option else '(.*)' # allow one argument followed by any number of options or arguments elif nargs == PARSER: - nargs_pattern = '(-*A[-AO]*)' + nargs_pattern = '(A[AO]*)' if option else '(-*A[-AO]*)' # suppress action, like nargs=0 elif nargs == SUPPRESS: - nargs_pattern = '(-*-*)' + nargs_pattern = '()' if option else '(-*)' # all others should be integers else: - nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs) - - # if this is an optional action, -- is not allowed - if action.option_strings: - nargs_pattern = nargs_pattern.replace('-*', '') - nargs_pattern = nargs_pattern.replace('-', '') + nargs_pattern = '([AO]{%d})' % nargs if option else '((?:-*A){%d}-*)' % nargs # return the pattern return nargs_pattern @@ -2445,8 +2473,11 @@ def _get_nargs_pattern(self, action): def parse_intermixed_args(self, args=None, namespace=None): args, argv = self.parse_known_intermixed_args(args, namespace) if argv: - msg = _('unrecognized arguments: %s') - self.error(msg % ' '.join(argv)) + msg = _('unrecognized arguments: %s') % ' '.join(argv) + if self.exit_on_error: + self.error(msg) + else: + raise ArgumentError(None, msg) return args def parse_known_intermixed_args(self, args=None, namespace=None): @@ -2457,10 +2488,6 @@ def parse_known_intermixed_args(self, args=None, namespace=None): # are then parsed. If the parser definition is incompatible with the # intermixed assumptions (e.g. use of REMAINDER, subparsers) a # TypeError is raised. - # - # positionals are 'deactivated' by setting nargs and default to - # SUPPRESS. This blocks the addition of that positional to the - # namespace positionals = self._get_positional_actions() a = [action for action in positionals @@ -2469,80 +2496,20 @@ def parse_known_intermixed_args(self, args=None, namespace=None): raise TypeError('parse_intermixed_args: positional arg' ' with nargs=%s'%a[0].nargs) - if [action.dest for group in self._mutually_exclusive_groups - for action in group._group_actions if action in positionals]: - raise TypeError('parse_intermixed_args: positional in' - ' mutuallyExclusiveGroup') - - try: - save_usage = self.usage - try: - if self.usage is None: - # capture the full usage for use in error messages - self.usage = self.format_usage()[7:] - for action in positionals: - # deactivate positionals - action.save_nargs = action.nargs - # action.nargs = 0 - action.nargs = SUPPRESS - action.save_default = action.default - action.default = SUPPRESS - namespace, remaining_args = self.parse_known_args(args, - namespace) - for action in positionals: - # remove the empty positional values from namespace - if (hasattr(namespace, action.dest) - and getattr(namespace, action.dest)==[]): - from warnings import warn - warn('Do not expect %s in %s' % (action.dest, namespace)) - delattr(namespace, action.dest) - finally: - # restore nargs and usage before exiting - for action in positionals: - action.nargs = action.save_nargs - action.default = action.save_default - optionals = self._get_optional_actions() - try: - # parse positionals. optionals aren't normally required, but - # they could be, so make sure they aren't. - for action in optionals: - action.save_required = action.required - action.required = False - for group in self._mutually_exclusive_groups: - group.save_required = group.required - group.required = False - namespace, extras = self.parse_known_args(remaining_args, - namespace) - finally: - # restore parser values before exiting - for action in optionals: - action.required = action.save_required - for group in self._mutually_exclusive_groups: - group.required = group.save_required - finally: - self.usage = save_usage - return namespace, extras + return self._parse_known_args2(args, namespace, intermixed=True) # ======================== # Value conversion methods # ======================== def _get_values(self, action, arg_strings): - # for everything but PARSER, REMAINDER args, strip out first '--' - if not action.option_strings and action.nargs not in [PARSER, REMAINDER]: - try: - arg_strings.remove('--') - except ValueError: - pass - # optional argument produces a default when not present if not arg_strings and action.nargs == OPTIONAL: if action.option_strings: value = action.const else: value = action.default - if isinstance(value, str): + if isinstance(value, str) and value is not SUPPRESS: value = self._get_value(action, value) - self._check_value(action, value) # when nargs='*' on a positional, if there were no command-line # args, use the default if it is anything other than None @@ -2550,11 +2517,8 @@ def _get_values(self, action, arg_strings): not action.option_strings): if action.default is not None: value = action.default - self._check_value(action, value) else: - # since arg_strings is always [] at this point - # there is no need to use self._check_value(action, value) - value = arg_strings + value = [] # single argument or optional argument produces a single value elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: @@ -2587,8 +2551,7 @@ def _get_values(self, action, arg_strings): def _get_value(self, action, arg_string): type_func = self._registry_get('type', action.type, action.type) if not callable(type_func): - msg = _('%r is not callable') - raise ArgumentError(action, msg % type_func) + raise TypeError(f'{type_func!r} is not callable') # convert the value to the appropriate type try: @@ -2611,10 +2574,27 @@ def _get_value(self, action, arg_string): def _check_value(self, action, value): # converted value must be one of the choices (if specified) - if action.choices is not None and value not in action.choices: - args = {'value': value, - 'choices': ', '.join(map(repr, action.choices))} + choices = action.choices + if choices is None: + return + + if isinstance(choices, str): + choices = iter(choices) + + if value not in choices: + args = {'value': str(value), + 'choices': ', '.join(map(str, action.choices))} msg = _('invalid choice: %(value)r (choose from %(choices)s)') + + if self.suggest_on_error and isinstance(value, str): + if all(isinstance(choice, str) for choice in action.choices): + import difflib + suggestions = difflib.get_close_matches(value, action.choices, 1) + if suggestions: + args['closest'] = suggestions[0] + msg = _('invalid choice: %(value)r, maybe you meant %(closest)r? ' + '(choose from %(choices)s)') + raise ArgumentError(action, msg % args) # ======================= diff --git a/Lib/ast.py b/Lib/ast.py index b8c4ce6f919e6b..154d2c8c1f9ebb 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -25,7 +25,6 @@ :license: Python License. """ import sys -import re from _ast import * from contextlib import contextmanager, nullcontext from enum import IntEnum, auto, _simple_enum @@ -114,7 +113,11 @@ def _convert(node): return _convert(node_or_string) -def dump(node, annotate_fields=True, include_attributes=False, *, indent=None): +def dump( + node, annotate_fields=True, include_attributes=False, + *, + indent=None, show_empty=False, +): """ Return a formatted dump of the tree in node. This is mainly useful for debugging purposes. If annotate_fields is true (by default), @@ -125,6 +128,8 @@ def dump(node, annotate_fields=True, include_attributes=False, *, indent=None): include_attributes can be set to true. If indent is a non-negative integer or string, then the tree will be pretty-printed with that indent level. None (the default) selects the single line representation. + If show_empty is False, then empty lists and fields that are None + will be omitted from the output for better readability. """ def _format(node, level=0): if indent is not None: @@ -137,6 +142,7 @@ def _format(node, level=0): if isinstance(node, AST): cls = type(node) args = [] + args_buffer = [] allsimple = True keywords = annotate_fields for name in node._fields: @@ -148,6 +154,18 @@ def _format(node, level=0): if value is None and getattr(cls, name, ...) is None: keywords = True continue + if ( + not show_empty + and (value is None or value == []) + # Special cases: + # `Constant(value=None)` and `MatchSingleton(value=None)` + and not isinstance(node, (Constant, MatchSingleton)) + ): + args_buffer.append(repr(value)) + continue + elif not keywords: + args.extend(args_buffer) + args_buffer = [] value, simple = _format(value, level) allsimple = allsimple and simple if keywords: @@ -306,12 +324,18 @@ def get_docstring(node, clean=True): return text -_line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))") +_line_pattern = None def _splitlines_no_ff(source, maxlines=None): """Split a string into lines ignoring form feed and other chars. This mimics how the Python parser splits source code. """ + global _line_pattern + if _line_pattern is None: + # lazily computed to speedup import time of `ast` + import re + _line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))") + lines = [] for lineno, match in enumerate(_line_pattern.finditer(source), 1): if maxlines is not None and lineno > maxlines: @@ -382,6 +406,88 @@ def walk(node): yield node +def compare( + a, + b, + /, + *, + compare_attributes=False, +): + """Recursively compares two ASTs. + + compare_attributes affects whether AST attributes are considered + in the comparison. If compare_attributes is False (default), then + attributes are ignored. Otherwise they must all be equal. This + option is useful to check whether the ASTs are structurally equal but + might differ in whitespace or similar details. + """ + + sentinel = object() # handle the possibility of a missing attribute/field + + def _compare(a, b): + # Compare two fields on an AST object, which may themselves be + # AST objects, lists of AST objects, or primitive ASDL types + # like identifiers and constants. + if isinstance(a, AST): + return compare( + a, + b, + compare_attributes=compare_attributes, + ) + elif isinstance(a, list): + # If a field is repeated, then both objects will represent + # the value as a list. + if len(a) != len(b): + return False + for a_item, b_item in zip(a, b): + if not _compare(a_item, b_item): + return False + else: + return True + else: + return type(a) is type(b) and a == b + + def _compare_fields(a, b): + if a._fields != b._fields: + return False + for field in a._fields: + a_field = getattr(a, field, sentinel) + b_field = getattr(b, field, sentinel) + if a_field is sentinel and b_field is sentinel: + # both nodes are missing a field at runtime + continue + if a_field is sentinel or b_field is sentinel: + # one of the node is missing a field + return False + if not _compare(a_field, b_field): + return False + else: + return True + + def _compare_attributes(a, b): + if a._attributes != b._attributes: + return False + # Attributes are always ints. + for attr in a._attributes: + a_attr = getattr(a, attr, sentinel) + b_attr = getattr(b, attr, sentinel) + if a_attr is sentinel and b_attr is sentinel: + # both nodes are missing an attribute at runtime + continue + if a_attr != b_attr: + return False + else: + return True + + if type(a) is not type(b): + return False + if not _compare_fields(a, b): + return False + if compare_attributes and not _compare_attributes(a, b): + return False + return True + + class NodeVisitor(object): """ A node visitor base class that walks the abstract syntax tree and calls a @@ -418,27 +524,6 @@ def generic_visit(self, node): elif isinstance(value, AST): self.visit(value) - def visit_Constant(self, node): - value = node.value - type_name = _const_node_type_names.get(type(value)) - if type_name is None: - for cls, name in _const_node_type_names.items(): - if isinstance(value, cls): - type_name = name - break - if type_name is not None: - method = 'visit_' + type_name - try: - visitor = getattr(self, method) - except AttributeError: - pass - else: - import warnings - warnings.warn(f"{method} is deprecated; add visit_Constant", - DeprecationWarning, 2) - return visitor(node) - return self.generic_visit(node) - class NodeTransformer(NodeVisitor): """ @@ -498,151 +583,6 @@ def generic_visit(self, node): setattr(node, field, new_node) return node - -_DEPRECATED_VALUE_ALIAS_MESSAGE = ( - "{name} is deprecated and will be removed in Python {remove}; use value instead" -) -_DEPRECATED_CLASS_MESSAGE = ( - "{name} is deprecated and will be removed in Python {remove}; " - "use ast.Constant instead" -) - - -# If the ast module is loaded more than once, only add deprecated methods once -if not hasattr(Constant, 'n'): - # The following code is for backward compatibility. - # It will be removed in future. - - def _n_getter(self): - """Deprecated. Use value instead.""" - import warnings - warnings._deprecated( - "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - return self.value - - def _n_setter(self, value): - import warnings - warnings._deprecated( - "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - self.value = value - - def _s_getter(self): - """Deprecated. Use value instead.""" - import warnings - warnings._deprecated( - "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - return self.value - - def _s_setter(self, value): - import warnings - warnings._deprecated( - "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - self.value = value - - Constant.n = property(_n_getter, _n_setter) - Constant.s = property(_s_getter, _s_setter) - -class _ABC(type): - - def __init__(cls, *args): - cls.__doc__ = """Deprecated AST node class. Use ast.Constant instead""" - - def __instancecheck__(cls, inst): - if cls in _const_types: - import warnings - warnings._deprecated( - f"ast.{cls.__qualname__}", - message=_DEPRECATED_CLASS_MESSAGE, - remove=(3, 14) - ) - if not isinstance(inst, Constant): - return False - if cls in _const_types: - try: - value = inst.value - except AttributeError: - return False - else: - return ( - isinstance(value, _const_types[cls]) and - not isinstance(value, _const_types_not.get(cls, ())) - ) - return type.__instancecheck__(cls, inst) - -def _new(cls, *args, **kwargs): - for key in kwargs: - if key not in cls._fields: - # arbitrary keyword arguments are accepted - continue - pos = cls._fields.index(key) - if pos < len(args): - raise TypeError(f"{cls.__name__} got multiple values for argument {key!r}") - if cls in _const_types: - import warnings - warnings._deprecated( - f"ast.{cls.__qualname__}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return Constant(*args, **kwargs) - return Constant.__new__(cls, *args, **kwargs) - -class Num(Constant, metaclass=_ABC): - _fields = ('n',) - __new__ = _new - -class Str(Constant, metaclass=_ABC): - _fields = ('s',) - __new__ = _new - -class Bytes(Constant, metaclass=_ABC): - _fields = ('s',) - __new__ = _new - -class NameConstant(Constant, metaclass=_ABC): - __new__ = _new - -class Ellipsis(Constant, metaclass=_ABC): - _fields = () - - def __new__(cls, *args, **kwargs): - if cls is _ast_Ellipsis: - import warnings - warnings._deprecated( - "ast.Ellipsis", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return Constant(..., *args, **kwargs) - return Constant.__new__(cls, *args, **kwargs) - -# Keep another reference to Ellipsis in the global namespace -# so it can be referenced in Ellipsis.__new__ -# (The original "Ellipsis" name is removed from the global namespace later on) -_ast_Ellipsis = Ellipsis - -_const_types = { - Num: (int, float, complex), - Str: (str,), - Bytes: (bytes,), - NameConstant: (type(None), bool), - Ellipsis: (type(...),), -} -_const_types_not = { - Num: (bool,), -} - -_const_node_type_names = { - bool: 'NameConstant', # should be before int - type(None): 'NameConstant', - int: 'Num', - float: 'Num', - complex: 'Num', - str: 'Str', - bytes: 'Bytes', - type(...): 'Ellipsis', -} - class slice(AST): """Deprecated AST node class.""" @@ -1105,12 +1045,21 @@ def visit_TypeVar(self, node): if node.bound: self.write(": ") self.traverse(node.bound) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) def visit_TypeVarTuple(self, node): self.write("*" + node.name) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) def visit_ParamSpec(self, node): self.write("**" + node.name) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) def visit_TypeAlias(self, node): self.fill("type ") @@ -1785,31 +1734,16 @@ def visit_MatchOr(self, node): self.set_precedence(_Precedence.BOR.next(), *node.patterns) self.interleave(lambda: self.write(" | "), self.traverse, node.patterns) + def unparse(ast_obj): unparser = _Unparser() return unparser.visit(ast_obj) -_deprecated_globals = { - name: globals().pop(name) - for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis') -} - -def __getattr__(name): - if name in _deprecated_globals: - globals()[name] = value = _deprecated_globals[name] - import warnings - warnings._deprecated( - f"ast.{name}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return value - raise AttributeError(f"module 'ast' has no attribute '{name}'") - - def main(): import argparse - parser = argparse.ArgumentParser(prog='python -m ast') + parser = argparse.ArgumentParser() parser.add_argument('infile', nargs='?', default='-', help='the file to parse; defaults to stdin') parser.add_argument('-m', '--mode', default='exec', diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index 03165a425eb7d2..4be7112fa017d4 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -10,6 +10,7 @@ from .events import * from .exceptions import * from .futures import * +from .graph import * from .locks import * from .protocols import * from .runners import * @@ -27,6 +28,7 @@ events.__all__ + exceptions.__all__ + futures.__all__ + + graph.__all__ + locks.__all__ + protocols.__all__ + runners.__all__ + @@ -45,3 +47,19 @@ else: from .unix_events import * # pragma: no cover __all__ += unix_events.__all__ + +def __getattr__(name: str): + import warnings + + deprecated = { + "AbstractEventLoopPolicy", + "DefaultEventLoopPolicy", + "WindowsSelectorEventLoopPolicy", + "WindowsProactorEventLoopPolicy", + } + if name in deprecated: + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + # deprecated things have underscores in front of them + return globals()["_" + name] + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index cbc1d7c93ef76f..662ba649aa08be 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -1,42 +1,51 @@ import ast import asyncio -import code import concurrent.futures +import contextvars import inspect +import os import site import sys import threading import types import warnings +from _colorize import can_colorize, ANSIColors # type: ignore[import-not-found] +from _pyrepl.console import InteractiveColoredConsole + from . import futures -class AsyncIOInteractiveConsole(code.InteractiveConsole): +class AsyncIOInteractiveConsole(InteractiveColoredConsole): def __init__(self, locals, loop): - super().__init__(locals) + super().__init__(locals, filename="") self.compile.compiler.flags |= ast.PyCF_ALLOW_TOP_LEVEL_AWAIT self.loop = loop + self.context = contextvars.copy_context() def runcode(self, code): + global return_code future = concurrent.futures.Future() def callback(): + global return_code global repl_future - global repl_future_interrupted + global keyboard_interrupted repl_future = None - repl_future_interrupted = False + keyboard_interrupted = False func = types.FunctionType(code, self.locals) try: coro = func() - except SystemExit: - raise + except SystemExit as se: + return_code = se.code + self.loop.stop() + return except KeyboardInterrupt as ex: - repl_future_interrupted = True + keyboard_interrupted = True future.set_exception(ex) return except BaseException as ex: @@ -48,19 +57,21 @@ def callback(): return try: - repl_future = self.loop.create_task(coro) + repl_future = self.loop.create_task(coro, context=self.context) futures._chain_future(repl_future, future) except BaseException as exc: future.set_exception(exc) - loop.call_soon_threadsafe(callback) + loop.call_soon_threadsafe(callback, context=self.context) try: return future.result() - except SystemExit: - raise + except SystemExit as se: + return_code = se.code + self.loop.stop() + return except BaseException: - if repl_future_interrupted: + if keyboard_interrupted: self.write("\nKeyboardInterrupt\n") else: self.showtraceback() @@ -69,18 +80,47 @@ def callback(): class REPLThread(threading.Thread): def run(self): + global return_code + try: banner = ( f'asyncio REPL {sys.version} on {sys.platform}\n' f'Use "await" directly instead of "asyncio.run()".\n' f'Type "help", "copyright", "credits" or "license" ' f'for more information.\n' - f'{getattr(sys, "ps1", ">>> ")}import asyncio' ) - console.interact( - banner=banner, - exitmsg='exiting asyncio REPL...') + console.write(banner) + + if startup_path := os.getenv("PYTHONSTARTUP"): + sys.audit("cpython.run_startup", startup_path) + + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") + exec(startup_code, console.locals) + + ps1 = getattr(sys, "ps1", ">>> ") + if can_colorize() and CAN_USE_PYREPL: + ps1 = f"{ANSIColors.BOLD_MAGENTA}{ps1}{ANSIColors.RESET}" + console.write(f"{ps1}import asyncio\n") + + if CAN_USE_PYREPL: + from _pyrepl.simple_interact import ( + run_multiline_interactive_console, + ) + try: + run_multiline_interactive_console(console) + except SystemExit: + # expected via the `exit` and `quit` commands + pass + except BaseException: + # unexpected issue + console.showtraceback() + console.write("Internal error, ") + return_code = 1 + else: + console.interact(banner="", exitmsg="") finally: warnings.filterwarnings( 'ignore', @@ -89,10 +129,27 @@ def run(self): loop.call_soon_threadsafe(loop.stop) + def interrupt(self) -> None: + if not CAN_USE_PYREPL: + return + + from _pyrepl.simple_interact import _get_reader + r = _get_reader() + if r.threading_hook is not None: + r.threading_hook.add("") # type: ignore + if __name__ == '__main__': + sys.audit("cpython.run_stdin") + + if os.getenv('PYTHON_BASIC_REPL'): + CAN_USE_PYREPL = False + else: + from _pyrepl.main import CAN_USE_PYREPL + + return_code = 0 loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + asyncio._set_event_loop(loop) repl_locals = {'asyncio': asyncio} for key in {'__name__', '__package__', @@ -103,16 +160,17 @@ def run(self): console = AsyncIOInteractiveConsole(repl_locals, loop) repl_future = None - repl_future_interrupted = False + keyboard_interrupted = False try: import readline # NoQA except ImportError: - pass + readline = None interactive_hook = getattr(sys, "__interactivehook__", None) if interactive_hook is not None: + sys.audit("cpython.run_interactivehook", interactive_hook) interactive_hook() if interactive_hook is site.register_readline: @@ -122,10 +180,11 @@ def run(self): except: pass else: - completer = rlcompleter.Completer(console.locals) - readline.set_completer(completer.complete) + if readline is not None: + completer = rlcompleter.Completer(console.locals) + readline.set_completer(completer.complete) - repl_thread = REPLThread() + repl_thread = REPLThread(name="Interactive thread") repl_thread.daemon = True repl_thread.start() @@ -133,9 +192,13 @@ def run(self): try: loop.run_forever() except KeyboardInterrupt: + keyboard_interrupted = True if repl_future and not repl_future.done(): repl_future.cancel() - repl_future_interrupted = True + repl_thread.interrupt() continue else: break + + console.write('exiting asyncio REPL...\n') + sys.exit(return_code) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 6c5cf28e7c59d4..ed852421e44212 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -17,7 +17,6 @@ import collections.abc import concurrent.futures import errno -import functools import heapq import itertools import os @@ -279,7 +278,9 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, ssl_handshake_timeout, ssl_shutdown_timeout=None): self._loop = loop self._sockets = sockets - self._active_count = 0 + # Weak references so we don't break Transport's ability to + # detect abandoned transports + self._clients = weakref.WeakSet() self._waiters = [] self._protocol_factory = protocol_factory self._backlog = backlog @@ -292,14 +293,13 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, def __repr__(self): return f'<{self.__class__.__name__} sockets={self.sockets!r}>' - def _attach(self): + def _attach(self, transport): assert self._sockets is not None - self._active_count += 1 + self._clients.add(transport) - def _detach(self): - assert self._active_count > 0 - self._active_count -= 1 - if self._active_count == 0 and self._sockets is None: + def _detach(self, transport): + self._clients.discard(transport) + if len(self._clients) == 0 and self._sockets is None: self._wakeup() def _wakeup(self): @@ -348,9 +348,17 @@ def close(self): self._serving_forever_fut.cancel() self._serving_forever_fut = None - if self._active_count == 0: + if len(self._clients) == 0: self._wakeup() + def close_clients(self): + for transport in self._clients.copy(): + transport.close() + + def abort_clients(self): + for transport in self._clients.copy(): + transport.abort() + async def start_serving(self): self._start_serving() # Skip one loop iteration so that all 'loop.add_reader' @@ -450,26 +458,24 @@ def create_future(self): """Create a Future object attached to the loop.""" return futures.Future(loop=self) - def create_task(self, coro, *, name=None, context=None): + def create_task(self, coro, **kwargs): """Schedule a coroutine object. Return a task object. """ self._check_closed() - if self._task_factory is None: - task = tasks.Task(coro, loop=self, name=name, context=context) - if task._source_traceback: - del task._source_traceback[-1] - else: - if context is None: - # Use legacy API if context is not needed - task = self._task_factory(self, coro) - else: - task = self._task_factory(self, coro, context=context) - - task.set_name(name) + if self._task_factory is not None: + return self._task_factory(self, coro, **kwargs) - return task + task = tasks.Task(coro, loop=self, **kwargs) + if task._source_traceback: + del task._source_traceback[-1] + try: + return task + finally: + # gh-128552: prevent a refcycle of + # task.exception().__traceback__->BaseEventLoop.create_task->task + del task def set_task_factory(self, factory): """Set a task factory that will be used by loop.create_task(). @@ -477,9 +483,10 @@ def set_task_factory(self, factory): If factory is None the default task factory will be set. If factory is a callable, it should have a signature matching - '(loop, coro)', where 'loop' will be a reference to the active - event loop, 'coro' will be a coroutine object. The callable - must return a Future. + '(loop, coro, **kwargs)', where 'loop' will be a reference to the active + event loop, 'coro' will be a coroutine object, and **kwargs will be + arbitrary keyword arguments that should be passed on to Task. + The callable must return a Task. """ if factory is not None and not callable(factory): raise TypeError('task factory must be a callable or None') @@ -828,7 +835,7 @@ def call_soon(self, callback, *args, context=None): def _check_callback(self, callback, method): if (coroutines.iscoroutine(callback) or - coroutines.iscoroutinefunction(callback)): + coroutines._iscoroutinefunction(callback)): raise TypeError( f"coroutines cannot be used with {method}()") if not callable(callback): @@ -865,7 +872,10 @@ def call_soon_threadsafe(self, callback, *args, context=None): self._check_closed() if self._debug: self._check_callback(callback, 'call_soon_threadsafe') - handle = self._call_soon(callback, args, context) + handle = events._ThreadSafeHandle(callback, args, self, context) + self._ready.append(handle) + if handle._source_traceback: + del handle._source_traceback[-1] if handle._source_traceback: del handle._source_traceback[-1] self._write_to_self() @@ -1019,8 +1029,7 @@ async def _connect_sock(self, exceptions, addr_info, local_addr_infos=None): except OSError as exc: msg = ( f'error while attempting to bind on ' - f'address {laddr!r}: ' - f'{exc.strerror.lower()}' + f'address {laddr!r}: {str(exc).lower()}' ) exc = OSError(exc.errno, msg) my_exceptions.append(exc) @@ -1132,11 +1141,18 @@ async def create_connection( except OSError: continue else: # using happy eyeballs - sock, _, _ = await staggered.staggered_race( - (functools.partial(self._connect_sock, - exceptions, addrinfo, laddr_infos) - for addrinfo in infos), - happy_eyeballs_delay, loop=self) + sock = (await staggered.staggered_race( + ( + # can't use functools.partial as it keeps a reference + # to exceptions + lambda addrinfo=addrinfo: self._connect_sock( + exceptions, addrinfo, laddr_infos + ) + for addrinfo in infos + ), + happy_eyeballs_delay, + loop=self, + ))[0] # can't use sock, _, _ as it keeks a reference to exceptions if sock is None: exceptions = [exc for sub in exceptions for exc in sub] @@ -1571,7 +1587,9 @@ async def create_server( if reuse_address: sock.setsockopt( socket.SOL_SOCKET, socket.SO_REUSEADDR, True) - if reuse_port: + # Since Linux 6.12.9, SO_REUSEPORT is not allowed + # on other address families than AF_INET/AF_INET6. + if reuse_port and af in (socket.AF_INET, socket.AF_INET6): _set_reuseport(sock) if keep_alive: sock.setsockopt( @@ -1590,7 +1608,7 @@ async def create_server( except OSError as err: msg = ('error while attempting ' 'to bind on address %r: %s' - % (sa, err.strerror.lower())) + % (sa, str(err).lower())) if err.errno == errno.EADDRNOTAVAIL: # Assume the family is not enabled (bpo-30945) sockets.pop() diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index 6dbde2b696ad1f..9c2ba679ce2bf1 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -1,6 +1,9 @@ import collections import subprocess import warnings +import os +import signal +import sys from . import protocols from . import transports @@ -142,17 +145,31 @@ def _check_proc(self): if self._proc is None: raise ProcessLookupError() - def send_signal(self, signal): - self._check_proc() - self._proc.send_signal(signal) + if sys.platform == 'win32': + def send_signal(self, signal): + self._check_proc() + self._proc.send_signal(signal) + + def terminate(self): + self._check_proc() + self._proc.terminate() + + def kill(self): + self._check_proc() + self._proc.kill() + else: + def send_signal(self, signal): + self._check_proc() + try: + os.kill(self._proc.pid, signal) + except ProcessLookupError: + pass - def terminate(self): - self._check_proc() - self._proc.terminate() + def terminate(self): + self.send_signal(signal.SIGTERM) - def kill(self): - self._check_proc() - self._proc.kill() + def kill(self): + self.send_signal(signal.SIGKILL) async def _connect_pipes(self, waiter): try: diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index ab4f30eb51ba2c..a51319cb72a6a9 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -18,7 +18,16 @@ def _is_debug_mode(): def iscoroutinefunction(func): + import warnings """Return True if func is a decorated coroutine function.""" + warnings._deprecated("asyncio.iscoroutinefunction", + f"{warnings._DEPRECATED_MSG}; " + "use inspect.iscoroutinefunction() instead", + remove=(3,16)) + return _iscoroutinefunction(func) + + +def _iscoroutinefunction(func): return (inspect.iscoroutinefunction(func) or getattr(func, '_is_coroutine', None) is _is_coroutine) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 680749325025db..2e45b4fe6fa2dd 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -5,14 +5,22 @@ # SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io __all__ = ( - 'AbstractEventLoopPolicy', - 'AbstractEventLoop', 'AbstractServer', - 'Handle', 'TimerHandle', - 'get_event_loop_policy', 'set_event_loop_policy', - 'get_event_loop', 'set_event_loop', 'new_event_loop', - 'get_child_watcher', 'set_child_watcher', - '_set_running_loop', 'get_running_loop', - '_get_running_loop', + "_AbstractEventLoopPolicy", + "AbstractEventLoop", + "AbstractServer", + "Handle", + "TimerHandle", + "_get_event_loop_policy", + "get_event_loop_policy", + "_set_event_loop_policy", + "set_event_loop_policy", + "get_event_loop", + "_set_event_loop", + "set_event_loop", + "new_event_loop", + "_set_running_loop", + "get_running_loop", + "_get_running_loop", ) import contextvars @@ -22,6 +30,7 @@ import subprocess import sys import threading +import warnings from . import format_helpers @@ -104,6 +113,34 @@ def _run(self): self._loop.call_exception_handler(context) self = None # Needed to break cycles when an exception occurs. +# _ThreadSafeHandle is used for callbacks scheduled with call_soon_threadsafe +# and is thread safe unlike Handle which is not thread safe. +class _ThreadSafeHandle(Handle): + + __slots__ = ('_lock',) + + def __init__(self, callback, args, loop, context=None): + super().__init__(callback, args, loop, context) + self._lock = threading.RLock() + + def cancel(self): + with self._lock: + return super().cancel() + + def cancelled(self): + with self._lock: + return super().cancelled() + + def _run(self): + # The event loop checks for cancellation without holding the lock + # It is possible that the handle is cancelled after the check + # but before the callback is called so check it again after acquiring + # the lock and return without calling the callback if it is cancelled. + with self._lock: + if self._cancelled: + return + return super()._run() + class TimerHandle(Handle): """Object returned by timed callback registration methods.""" @@ -175,6 +212,14 @@ def close(self): """Stop serving. This leaves existing connections open.""" raise NotImplementedError + def close_clients(self): + """Close all active connections.""" + raise NotImplementedError + + def abort_clients(self): + """Close all active connections immediately.""" + raise NotImplementedError + def get_loop(self): """Get the event loop the Server object is attached to.""" raise NotImplementedError @@ -284,7 +329,7 @@ def create_future(self): # Method scheduling a coroutine object: create a task. - def create_task(self, coro, *, name=None, context=None): + def create_task(self, coro, **kwargs): raise NotImplementedError # Methods for interacting with threads. @@ -621,7 +666,7 @@ def set_debug(self, enabled): raise NotImplementedError -class AbstractEventLoopPolicy: +class _AbstractEventLoopPolicy: """Abstract policy for accessing the event loop.""" def get_event_loop(self): @@ -644,18 +689,7 @@ def new_event_loop(self): the current context, set_event_loop must be called explicitly.""" raise NotImplementedError - # Child processes handling (Unix only). - - def get_child_watcher(self): - "Get the watcher for child processes." - raise NotImplementedError - - def set_child_watcher(self, watcher): - """Set the watcher for child processes.""" - raise NotImplementedError - - -class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy): +class _BaseDefaultEventLoopPolicy(_AbstractEventLoopPolicy): """Default policy implementation for accessing the event loop. In this policy, each thread has its own event loop. However, we @@ -672,7 +706,6 @@ class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy): class _Local(threading.local): _loop = None - _set_called = False def __init__(self): self._local = self._Local() @@ -682,28 +715,6 @@ def get_event_loop(self): Returns an instance of EventLoop or raises an exception. """ - if (self._local._loop is None and - not self._local._set_called and - threading.current_thread() is threading.main_thread()): - stacklevel = 2 - try: - f = sys._getframe(1) - except AttributeError: - pass - else: - # Move up the call stack so that the warning is attached - # to the line outside asyncio itself. - while f: - module = f.f_globals.get('__name__') - if not (module == 'asyncio' or module.startswith('asyncio.')): - break - f = f.f_back - stacklevel += 1 - import warnings - warnings.warn('There is no current event loop', - DeprecationWarning, stacklevel=stacklevel) - self.set_event_loop(self.new_event_loop()) - if self._local._loop is None: raise RuntimeError('There is no current event loop in thread %r.' % threading.current_thread().name) @@ -712,7 +723,6 @@ def get_event_loop(self): def set_event_loop(self, loop): """Set the event loop.""" - self._local._set_called = True if loop is not None and not isinstance(loop, AbstractEventLoop): raise TypeError(f"loop must be an instance of AbstractEventLoop or None, not '{type(loop).__name__}'") self._local._loop = loop @@ -782,26 +792,32 @@ def _init_event_loop_policy(): global _event_loop_policy with _lock: if _event_loop_policy is None: # pragma: no branch - from . import DefaultEventLoopPolicy - _event_loop_policy = DefaultEventLoopPolicy() + from . import _DefaultEventLoopPolicy + _event_loop_policy = _DefaultEventLoopPolicy() -def get_event_loop_policy(): +def _get_event_loop_policy(): """Get the current event loop policy.""" if _event_loop_policy is None: _init_event_loop_policy() return _event_loop_policy +def get_event_loop_policy(): + warnings._deprecated('asyncio.get_event_loop_policy', remove=(3, 16)) + return _get_event_loop_policy() -def set_event_loop_policy(policy): +def _set_event_loop_policy(policy): """Set the current event loop policy. If policy is None, the default policy is restored.""" global _event_loop_policy - if policy is not None and not isinstance(policy, AbstractEventLoopPolicy): + if policy is not None and not isinstance(policy, _AbstractEventLoopPolicy): raise TypeError(f"policy must be an instance of AbstractEventLoopPolicy or None, not '{type(policy).__name__}'") _event_loop_policy = policy +def set_event_loop_policy(policy): + warnings._deprecated('asyncio.set_event_loop_policy', remove=(3,16)) + _set_event_loop_policy(policy) def get_event_loop(): """Return an asyncio event loop. @@ -816,28 +832,21 @@ def get_event_loop(): current_loop = _get_running_loop() if current_loop is not None: return current_loop - return get_event_loop_policy().get_event_loop() + return _get_event_loop_policy().get_event_loop() +def _set_event_loop(loop): + _get_event_loop_policy().set_event_loop(loop) + def set_event_loop(loop): """Equivalent to calling get_event_loop_policy().set_event_loop(loop).""" - get_event_loop_policy().set_event_loop(loop) + warnings._deprecated('asyncio.set_event_loop', remove=(3,16)) + _set_event_loop(loop) def new_event_loop(): """Equivalent to calling get_event_loop_policy().new_event_loop().""" - return get_event_loop_policy().new_event_loop() - - -def get_child_watcher(): - """Equivalent to calling get_event_loop_policy().get_child_watcher().""" - return get_event_loop_policy().get_child_watcher() - - -def set_child_watcher(watcher): - """Equivalent to calling - get_event_loop_policy().set_child_watcher(watcher).""" - return get_event_loop_policy().set_child_watcher(watcher) + return _get_event_loop_policy().new_event_loop() # Alias pure-Python implementations for testing purposes. @@ -867,7 +876,7 @@ def set_child_watcher(watcher): def on_fork(): # Reset the loop and wakeupfd in the forked child process. if _event_loop_policy is not None: - _event_loop_policy._local = BaseDefaultEventLoopPolicy._Local() + _event_loop_policy._local = _BaseDefaultEventLoopPolicy._Local() _set_running_loop(None) signal.set_wakeup_fd(-1) diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 5d35321db7943b..d1df6707302277 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -2,6 +2,7 @@ __all__ = ( 'Future', 'wrap_future', 'isfuture', + 'future_add_to_awaited_by', 'future_discard_from_awaited_by', ) import concurrent.futures @@ -43,7 +44,6 @@ class Future: - This class is not compatible with the wait() and as_completed() methods in the concurrent.futures package. - (In Python 3.4 or later we may be able to unify the implementations.) """ # Class variables serving as defaults for instance variables. @@ -61,12 +61,15 @@ class Future: # the Future protocol (i.e. is intended to be duck-type compatible). # The value must also be not-None, to enable a subclass to declare # that it is not compatible by setting this to None. - # - It is set by __iter__() below so that Task._step() can tell + # - It is set by __iter__() below so that Task.__step() can tell # the difference between - # `await Future()` or`yield from Future()` (correct) vs. + # `await Future()` or `yield from Future()` (correct) vs. # `yield Future()` (incorrect). _asyncio_future_blocking = False + # Used by the capture_call_stack() API. + __asyncio_awaited_by = None + __log_traceback = False def __init__(self, *, loop=None): @@ -116,6 +119,12 @@ def _log_traceback(self, val): raise ValueError('_log_traceback can only be set to False') self.__log_traceback = False + @property + def _asyncio_awaited_by(self): + if self.__asyncio_awaited_by is None: + return None + return frozenset(self.__asyncio_awaited_by) + def get_loop(self): """Return the event loop the Future is bound to.""" loop = self._loop @@ -191,8 +200,7 @@ def result(self): the future is done and has an exception set, this exception is raised. """ if self._state == _CANCELLED: - exc = self._make_cancelled_error() - raise exc + raise self._make_cancelled_error() if self._state != _FINISHED: raise exceptions.InvalidStateError('Result is not ready.') self.__log_traceback = False @@ -209,8 +217,7 @@ def exception(self): InvalidStateError. """ if self._state == _CANCELLED: - exc = self._make_cancelled_error() - raise exc + raise self._make_cancelled_error() if self._state != _FINISHED: raise exceptions.InvalidStateError('Exception is not set.') self.__log_traceback = False @@ -319,11 +326,9 @@ def _set_result_unless_cancelled(fut, result): def _convert_future_exc(exc): exc_class = type(exc) if exc_class is concurrent.futures.CancelledError: - return exceptions.CancelledError(*exc.args) - elif exc_class is concurrent.futures.TimeoutError: - return exceptions.TimeoutError(*exc.args) + return exceptions.CancelledError(*exc.args).with_traceback(exc.__traceback__) elif exc_class is concurrent.futures.InvalidStateError: - return exceptions.InvalidStateError(*exc.args) + return exceptions.InvalidStateError(*exc.args).with_traceback(exc.__traceback__) else: return exc @@ -420,6 +425,49 @@ def wrap_future(future, *, loop=None): return new_future +def future_add_to_awaited_by(fut, waiter, /): + """Record that `fut` is awaited on by `waiter`.""" + # For the sake of keeping the implementation minimal and assuming + # that most of asyncio users use the built-in Futures and Tasks + # (or their subclasses), we only support native Future objects + # and their subclasses. + # + # Longer version: tracking requires storing the caller-callee + # dependency somewhere. One obvious choice is to store that + # information right in the future itself in a dedicated attribute. + # This means that we'd have to require all duck-type compatible + # futures to implement a specific attribute used by asyncio for + # the book keeping. Another solution would be to store that in + # a global dictionary. The downside here is that that would create + # strong references and any scenario where the "add" call isn't + # followed by a "discard" call would lead to a memory leak. + # Using WeakDict would resolve that issue, but would complicate + # the C code (_asynciomodule.c). The bottom line here is that + # it's not clear that all this work would be worth the effort. + # + # Note that there's an accelerated version of this function + # shadowing this implementation later in this file. + if isinstance(fut, _PyFuture) and isinstance(waiter, _PyFuture): + if fut._Future__asyncio_awaited_by is None: + fut._Future__asyncio_awaited_by = set() + fut._Future__asyncio_awaited_by.add(waiter) + + +def future_discard_from_awaited_by(fut, waiter, /): + """Record that `fut` is no longer awaited on by `waiter`.""" + # See the comment in "future_add_to_awaited_by()" body for + # details on implementation. + # + # Note that there's an accelerated version of this function + # shadowing this implementation later in this file. + if isinstance(fut, _PyFuture) and isinstance(waiter, _PyFuture): + if fut._Future__asyncio_awaited_by is not None: + fut._Future__asyncio_awaited_by.discard(waiter) + + +_py_future_add_to_awaited_by = future_add_to_awaited_by +_py_future_discard_from_awaited_by = future_discard_from_awaited_by + try: import _asyncio except ImportError: @@ -427,3 +475,7 @@ def wrap_future(future, *, loop=None): else: # _CFuture is needed for tests. Future = _CFuture = _asyncio.Future + future_add_to_awaited_by = _asyncio.future_add_to_awaited_by + future_discard_from_awaited_by = _asyncio.future_discard_from_awaited_by + _c_future_add_to_awaited_by = future_add_to_awaited_by + _c_future_discard_from_awaited_by = future_discard_from_awaited_by diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py new file mode 100644 index 00000000000000..d8df7c9919abbf --- /dev/null +++ b/Lib/asyncio/graph.py @@ -0,0 +1,278 @@ +"""Introspection utils for tasks call graphs.""" + +import dataclasses +import sys +import types + +from . import events +from . import futures +from . import tasks + +__all__ = ( + 'capture_call_graph', + 'format_call_graph', + 'print_call_graph', + 'FrameCallGraphEntry', + 'FutureCallGraph', +) + +if False: # for type checkers + from typing import TextIO + +# Sadly, we can't re-use the traceback module's datastructures as those +# are tailored for error reporting, whereas we need to represent an +# async call graph. +# +# Going with pretty verbose names as we'd like to export them to the +# top level asyncio namespace, and want to avoid future name clashes. + + +@dataclasses.dataclass(frozen=True, slots=True) +class FrameCallGraphEntry: + frame: types.FrameType + + +@dataclasses.dataclass(frozen=True, slots=True) +class FutureCallGraph: + future: futures.Future + call_stack: tuple["FrameCallGraphEntry", ...] + awaited_by: tuple["FutureCallGraph", ...] + + +def _build_graph_for_future( + future: futures.Future, + *, + limit: int | None = None, +) -> FutureCallGraph: + if not isinstance(future, futures.Future): + raise TypeError( + f"{future!r} object does not appear to be compatible " + f"with asyncio.Future" + ) + + coro = None + if get_coro := getattr(future, 'get_coro', None): + coro = get_coro() if limit != 0 else None + + st: list[FrameCallGraphEntry] = [] + awaited_by: list[FutureCallGraph] = [] + + while coro is not None: + if hasattr(coro, 'cr_await'): + # A native coroutine or duck-type compatible iterator + st.append(FrameCallGraphEntry(coro.cr_frame)) + coro = coro.cr_await + elif hasattr(coro, 'ag_await'): + # A native async generator or duck-type compatible iterator + st.append(FrameCallGraphEntry(coro.cr_frame)) + coro = coro.ag_await + else: + break + + if future._asyncio_awaited_by: + for parent in future._asyncio_awaited_by: + awaited_by.append(_build_graph_for_future(parent, limit=limit)) + + if limit is not None: + if limit > 0: + st = st[:limit] + elif limit < 0: + st = st[limit:] + st.reverse() + return FutureCallGraph(future, tuple(st), tuple(awaited_by)) + + +def capture_call_graph( + future: futures.Future | None = None, + /, + *, + depth: int = 1, + limit: int | None = None, +) -> FutureCallGraph | None: + """Capture the async call graph for the current task or the provided Future. + + The graph is represented with three data structures: + + * FutureCallGraph(future, call_stack, awaited_by) + + Where 'future' is an instance of asyncio.Future or asyncio.Task. + + 'call_stack' is a tuple of FrameGraphEntry objects. + + 'awaited_by' is a tuple of FutureCallGraph objects. + + * FrameCallGraphEntry(frame) + + Where 'frame' is a frame object of a regular Python function + in the call stack. + + Receives an optional 'future' argument. If not passed, + the current task will be used. If there's no current task, the function + returns None. + + If "capture_call_graph()" is introspecting *the current task*, the + optional keyword-only 'depth' argument can be used to skip the specified + number of frames from top of the stack. + + If the optional keyword-only 'limit' argument is provided, each call stack + in the resulting graph is truncated to include at most ``abs(limit)`` + entries. If 'limit' is positive, the entries left are the closest to + the invocation point. If 'limit' is negative, the topmost entries are + left. If 'limit' is omitted or None, all entries are present. + If 'limit' is 0, the call stack is not captured at all, only + "awaited by" information is present. + """ + + loop = events._get_running_loop() + + if future is not None: + # Check if we're in a context of a running event loop; + # if yes - check if the passed future is the currently + # running task or not. + if loop is None or future is not tasks.current_task(loop=loop): + return _build_graph_for_future(future, limit=limit) + # else: future is the current task, move on. + else: + if loop is None: + raise RuntimeError( + 'capture_call_graph() is called outside of a running ' + 'event loop and no *future* to introspect was provided') + future = tasks.current_task(loop=loop) + + if future is None: + # This isn't a generic call stack introspection utility. If we + # can't determine the current task and none was provided, we + # just return. + return None + + if not isinstance(future, futures.Future): + raise TypeError( + f"{future!r} object does not appear to be compatible " + f"with asyncio.Future" + ) + + call_stack: list[FrameCallGraphEntry] = [] + + f = sys._getframe(depth) if limit != 0 else None + try: + while f is not None: + is_async = f.f_generator is not None + call_stack.append(FrameCallGraphEntry(f)) + + if is_async: + if f.f_back is not None and f.f_back.f_generator is None: + # We've reached the bottom of the coroutine stack, which + # must be the Task that runs it. + break + + f = f.f_back + finally: + del f + + awaited_by = [] + if future._asyncio_awaited_by: + for parent in future._asyncio_awaited_by: + awaited_by.append(_build_graph_for_future(parent, limit=limit)) + + if limit is not None: + limit *= -1 + if limit > 0: + call_stack = call_stack[:limit] + elif limit < 0: + call_stack = call_stack[limit:] + + return FutureCallGraph(future, tuple(call_stack), tuple(awaited_by)) + + +def format_call_graph( + future: futures.Future | None = None, + /, + *, + depth: int = 1, + limit: int | None = None, +) -> str: + """Return the async call graph as a string for `future`. + + If `future` is not provided, format the call graph for the current task. + """ + + def render_level(st: FutureCallGraph, buf: list[str], level: int) -> None: + def add_line(line: str) -> None: + buf.append(level * ' ' + line) + + if isinstance(st.future, tasks.Task): + add_line( + f'* Task(name={st.future.get_name()!r}, id={id(st.future):#x})' + ) + else: + add_line( + f'* Future(id={id(st.future):#x})' + ) + + if st.call_stack: + add_line( + f' + Call stack:' + ) + for ste in st.call_stack: + f = ste.frame + + if f.f_generator is None: + f = ste.frame + add_line( + f' | File {f.f_code.co_filename!r},' + f' line {f.f_lineno}, in' + f' {f.f_code.co_qualname}()' + ) + else: + c = f.f_generator + + try: + f = c.cr_frame + code = c.cr_code + tag = 'async' + except AttributeError: + try: + f = c.ag_frame + code = c.ag_code + tag = 'async generator' + except AttributeError: + f = c.gi_frame + code = c.gi_code + tag = 'generator' + + add_line( + f' | File {f.f_code.co_filename!r},' + f' line {f.f_lineno}, in' + f' {tag} {code.co_qualname}()' + ) + + if st.awaited_by: + add_line( + f' + Awaited by:' + ) + for fut in st.awaited_by: + render_level(fut, buf, level + 1) + + graph = capture_call_graph(future, depth=depth + 1, limit=limit) + if graph is None: + return "" + + buf: list[str] = [] + try: + render_level(graph, buf, 0) + finally: + # 'graph' has references to frames so we should + # make sure it's GC'ed as soon as we don't need it. + del graph + return '\n'.join(buf) + +def print_call_graph( + future: futures.Future | None = None, + /, + *, + file: TextIO | None = None, + depth: int = 1, + limit: int | None = None, +) -> None: + """Print the async call graph for the current task or the provided Future.""" + print(format_call_graph(future, depth=depth, limit=limit), file=file) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index aaee8ff0702923..fa3a94764b507a 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -341,9 +341,9 @@ def _notify(self, n): fut.set_result(False) def notify_all(self): - """Wake up all threads waiting on this condition. This method acts - like notify(), but wakes up all waiting threads instead of one. If the - calling thread has not acquired the lock when this method is called, + """Wake up all tasks waiting on this condition. This method acts + like notify(), but wakes up all waiting tasks instead of one. If the + calling task has not acquired the lock when this method is called, a RuntimeError is raised. """ self.notify(len(self._waiters)) @@ -485,7 +485,7 @@ class Barrier(mixins._LoopBoundMixin): def __init__(self, parties): """Create a barrier, initialised to 'parties' tasks.""" if parties < 1: - raise ValueError('parties must be > 0') + raise ValueError('parties must be >= 1') self._cond = Condition() # notify all tasks when state changes diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index a512db6367b20a..7eb55bd63ddb73 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -63,7 +63,7 @@ def __init__(self, loop, sock, protocol, waiter=None, self._called_connection_lost = False self._eof_written = False if self._server is not None: - self._server._attach() + self._server._attach(self) self._loop.call_soon(self._protocol.connection_made, self) if waiter is not None: # only wake up the waiter when connection_made() has been called @@ -167,7 +167,7 @@ def _call_connection_lost(self, exc): self._sock = None server = self._server if server is not None: - server._detach() + server._detach(self) self._server = None self._called_connection_lost = True @@ -721,6 +721,8 @@ async def sock_sendto(self, sock, data, address): return await self._proactor.sendto(sock, data, 0, address) async def sock_connect(self, sock, address): + if self._debug and sock.gettimeout() != 0: + raise ValueError("the socket must be non-blocking") return await self._proactor.connect(sock, address) async def sock_accept(self, sock): diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index a9656a6df561ba..2f3865114a84f9 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -1,4 +1,11 @@ -__all__ = ('Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty') +__all__ = ( + 'Queue', + 'PriorityQueue', + 'LifoQueue', + 'QueueFull', + 'QueueEmpty', + 'QueueShutDown', +) import collections import heapq @@ -18,6 +25,11 @@ class QueueFull(Exception): pass +class QueueShutDown(Exception): + """Raised when putting on to or getting from a shut-down Queue.""" + pass + + class Queue(mixins._LoopBoundMixin): """A queue, useful for coordinating producer and consumer coroutines. @@ -41,6 +53,7 @@ def __init__(self, maxsize=0): self._finished = locks.Event() self._finished.set() self._init(maxsize) + self._is_shutdown = False # These three are overridable in subclasses. @@ -81,6 +94,8 @@ def _format(self): result += f' _putters[{len(self._putters)}]' if self._unfinished_tasks: result += f' tasks={self._unfinished_tasks}' + if self._is_shutdown: + result += ' shutdown' return result def qsize(self): @@ -112,8 +127,12 @@ async def put(self, item): Put an item into the queue. If the queue is full, wait until a free slot is available before adding item. + + Raises QueueShutDown if the queue has been shut down. """ while self.full(): + if self._is_shutdown: + raise QueueShutDown putter = self._get_loop().create_future() self._putters.append(putter) try: @@ -125,7 +144,7 @@ async def put(self, item): self._putters.remove(putter) except ValueError: # The putter could be removed from self._putters by a - # previous get_nowait call. + # previous get_nowait call or a shutdown call. pass if not self.full() and not putter.cancelled(): # We were woken up by get_nowait(), but can't take @@ -138,7 +157,11 @@ def put_nowait(self, item): """Put an item into the queue without blocking. If no free slot is immediately available, raise QueueFull. + + Raises QueueShutDown if the queue has been shut down. """ + if self._is_shutdown: + raise QueueShutDown if self.full(): raise QueueFull self._put(item) @@ -150,8 +173,13 @@ async def get(self): """Remove and return an item from the queue. If queue is empty, wait until an item is available. + + Raises QueueShutDown if the queue has been shut down and is empty, or + if the queue has been shut down immediately. """ while self.empty(): + if self._is_shutdown and self.empty(): + raise QueueShutDown getter = self._get_loop().create_future() self._getters.append(getter) try: @@ -163,7 +191,7 @@ async def get(self): self._getters.remove(getter) except ValueError: # The getter could be removed from self._getters by a - # previous put_nowait call. + # previous put_nowait call, or a shutdown call. pass if not self.empty() and not getter.cancelled(): # We were woken up by put_nowait(), but can't take @@ -176,8 +204,13 @@ def get_nowait(self): """Remove and return an item from the queue. Return an item if one is immediately available, else raise QueueEmpty. + + Raises QueueShutDown if the queue has been shut down and is empty, or + if the queue has been shut down immediately. """ if self.empty(): + if self._is_shutdown: + raise QueueShutDown raise QueueEmpty item = self._get() self._wakeup_next(self._putters) @@ -194,6 +227,9 @@ def task_done(self): been processed (meaning that a task_done() call was received for every item that had been put() into the queue). + shutdown(immediate=True) calls task_done() for each remaining item in + the queue. + Raises ValueError if called more times than there were items placed in the queue. """ @@ -214,6 +250,34 @@ async def join(self): if self._unfinished_tasks > 0: await self._finished.wait() + def shutdown(self, immediate=False): + """Shut-down the queue, making queue gets and puts raise QueueShutDown. + + By default, gets will only raise once the queue is empty. Set + 'immediate' to True to make gets raise immediately instead. + + All blocked callers of put() and get() will be unblocked. If + 'immediate', a task is marked as done for each item remaining in + the queue, which may unblock callers of join(). + """ + self._is_shutdown = True + if immediate: + while not self.empty(): + self._get() + if self._unfinished_tasks > 0: + self._unfinished_tasks -= 1 + if self._unfinished_tasks == 0: + self._finished.set() + # All getters need to re-check queue-empty to raise ShutDown + while self._getters: + getter = self._getters.popleft() + if not getter.done(): + getter.set_result(None) + while self._putters: + putter = self._putters.popleft() + if not putter.done(): + putter.set_result(None) + class PriorityQueue(Queue): """A subclass of Queue; retrieves entries in priority order (lowest first). diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index 1b89236599aad7..14397b4ad0c732 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -3,6 +3,7 @@ import contextvars import enum import functools +import inspect import threading import signal from . import coroutines @@ -73,7 +74,7 @@ def close(self): loop.shutdown_default_executor(constants.THREAD_JOIN_TIMEOUT)) finally: if self._set_event_loop: - events.set_event_loop(None) + events._set_event_loop(None) loop.close() self._loop = None self._state = _State.CLOSED @@ -84,10 +85,7 @@ def get_loop(self): return self._loop def run(self, coro, *, context=None): - """Run a coroutine inside the embedded event loop.""" - if not coroutines.iscoroutine(coro): - raise ValueError("a coroutine was expected, got {!r}".format(coro)) - + """Run code in the embedded event loop.""" if events._get_running_loop() is not None: # fail fast with short traceback raise RuntimeError( @@ -95,8 +93,19 @@ def run(self, coro, *, context=None): self._lazy_init() + if not coroutines.iscoroutine(coro): + if inspect.isawaitable(coro): + async def _wrap_awaitable(awaitable): + return await awaitable + + coro = _wrap_awaitable(coro) + else: + raise TypeError('An asyncio.Future, a coroutine or an ' + 'awaitable is required') + if context is None: context = self._context + task = self._loop.create_task(coro, context=context) if (threading.current_thread() is threading.main_thread() @@ -138,7 +147,7 @@ def _lazy_init(self): if not self._set_event_loop: # Call set_event_loop only once to avoid calling # attach_loop multiple times on child watchers - events.set_event_loop(self._loop) + events._set_event_loop(self._loop) self._set_event_loop = True else: self._loop = self._loop_factory() @@ -168,6 +177,7 @@ def run(main, *, debug=None, loop_factory=None): running in the same thread. If debug is True, the event loop will be run in debug mode. + If loop_factory is passed, it is used for new event loop creation. This function always creates a new event loop and closes it at the end. It should be used as a main entry point for asyncio programs, and should diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 8e888d26ea0737..50992a607b3a1c 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -180,9 +180,13 @@ def _accept_connection( logger.debug("%r got a new connection from %r: %r", server, addr, conn) conn.setblocking(False) - except (BlockingIOError, InterruptedError, ConnectionAbortedError): - # Early exit because the socket accept buffer is empty. - return None + except ConnectionAbortedError: + # Discard connections that were aborted before accept(). + continue + except (BlockingIOError, InterruptedError): + # Early exit because of a signal or + # the socket accept buffer is empty. + return except OSError as exc: # There's nowhere to send the error, so just log it. if exc.errno in (errno.EMFILE, errno.ENFILE, @@ -791,7 +795,7 @@ def __init__(self, loop, sock, protocol, extra=None, server=None): self._paused = False # Set when pause_reading() called if self._server is not None: - self._server._attach() + self._server._attach(self) loop._transports[self._sock_fd] = self def __repr__(self): @@ -868,6 +872,8 @@ def __del__(self, _warn=warnings.warn): if self._sock is not None: _warn(f"unclosed transport {self!r}", ResourceWarning, source=self) self._sock.close() + if self._server is not None: + self._server._detach(self) def _fatal_error(self, exc, message='Fatal error on transport'): # Should be called from exception handler only. @@ -906,7 +912,7 @@ def _call_connection_lost(self, exc): self._loop = None server = self._server if server is not None: - server._detach() + server._detach(self) self._server = None def get_write_buffer_size(self): @@ -1173,6 +1179,7 @@ def writelines(self, list_of_data): # If the entire buffer couldn't be written, register a write handler if self._buffer: self._loop._add_writer(self._sock_fd, self._write_ready) + self._maybe_pause_protocol() def can_write_eof(self): return True diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index fa99d4533aa0a6..74c5f0d5ca0609 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -101,7 +101,7 @@ def get_protocol(self): return self._ssl_protocol._app_protocol def is_closing(self): - return self._closed + return self._closed or self._ssl_protocol._is_transport_closing() def close(self): """Close the transport. @@ -379,6 +379,9 @@ def _get_app_transport(self): self._app_transport_created = True return self._app_transport + def _is_transport_closing(self): + return self._transport is not None and self._transport.is_closing() + def connection_made(self, transport): """Called when the low-level connection is made. diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py index e180cde0243b15..2ad65d8648e6c5 100644 --- a/Lib/asyncio/staggered.py +++ b/Lib/asyncio/staggered.py @@ -8,6 +8,7 @@ from . import exceptions as exceptions_mod from . import locks from . import tasks +from . import futures async def staggered_race(coro_fns, delay, *, loop=None): @@ -63,14 +64,38 @@ async def staggered_race(coro_fns, delay, *, loop=None): """ # TODO: when we have aiter() and anext(), allow async iterables in coro_fns. loop = loop or events.get_running_loop() + parent_task = tasks.current_task(loop) enum_coro_fns = enumerate(coro_fns) winner_result = None winner_index = None + unhandled_exceptions = [] exceptions = [] - running_tasks = [] + running_tasks = set() + on_completed_fut = None + + def task_done(task): + running_tasks.discard(task) + futures.future_discard_from_awaited_by(task, parent_task) + if ( + on_completed_fut is not None + and not on_completed_fut.done() + and not running_tasks + ): + on_completed_fut.set_result(None) + + if task.cancelled(): + return + + exc = task.exception() + if exc is None: + return + unhandled_exceptions.append(exc) - async def run_one_coro( - previous_failed: typing.Optional[locks.Event]) -> None: + async def run_one_coro(ok_to_start, previous_failed) -> None: + # in eager tasks this waits for the calling task to append this task + # to running_tasks, in regular tasks this wait is a no-op that does + # not yield a future. See gh-124309. + await ok_to_start.wait() # Wait for the previous task to finish, or for delay seconds if previous_failed is not None: with contextlib.suppress(exceptions_mod.TimeoutError): @@ -86,9 +111,14 @@ async def run_one_coro( return # Start task that will run the next coroutine this_failed = locks.Event() - next_task = loop.create_task(run_one_coro(this_failed)) - running_tasks.append(next_task) - assert len(running_tasks) == this_index + 2 + next_ok_to_start = locks.Event() + next_task = loop.create_task(run_one_coro(next_ok_to_start, this_failed)) + futures.future_add_to_awaited_by(next_task, parent_task) + running_tasks.add(next_task) + next_task.add_done_callback(task_done) + # next_task has been appended to running_tasks so next_task is ok to + # start. + next_ok_to_start.set() # Prepare place to put this coroutine's exceptions if not won exceptions.append(None) assert len(exceptions) == this_index + 1 @@ -113,27 +143,37 @@ async def run_one_coro( # up as done() == True, cancelled() == False, exception() == # asyncio.CancelledError. This behavior is specified in # https://bugs.python.org/issue30048 - for i, t in enumerate(running_tasks): - if i != this_index: + current_task = tasks.current_task(loop) + for t in running_tasks: + if t is not current_task: t.cancel() - first_task = loop.create_task(run_one_coro(None)) - running_tasks.append(first_task) + propagate_cancellation_error = None try: - # Wait for a growing list of tasks to all finish: poor man's version of - # curio's TaskGroup or trio's nursery - done_count = 0 - while done_count != len(running_tasks): - done, _ = await tasks.wait(running_tasks) - done_count = len(done) + ok_to_start = locks.Event() + first_task = loop.create_task(run_one_coro(ok_to_start, None)) + futures.future_add_to_awaited_by(first_task, parent_task) + running_tasks.add(first_task) + first_task.add_done_callback(task_done) + # first_task has been appended to running_tasks so first_task is ok to start. + ok_to_start.set() + propagate_cancellation_error = None + # Make sure no tasks are left running if we leave this function + while running_tasks: + on_completed_fut = loop.create_future() + try: + await on_completed_fut + except exceptions_mod.CancelledError as ex: + propagate_cancellation_error = ex + for task in running_tasks: + task.cancel(*ex.args) + on_completed_fut = None + if __debug__ and unhandled_exceptions: # If run_one_coro raises an unhandled exception, it's probably a # programming error, and I want to see it. - if __debug__: - for d in done: - if d.done() and not d.cancelled() and d.exception(): - raise d.exception() + raise ExceptionGroup("staggered race failed", unhandled_exceptions) + if propagate_cancellation_error is not None: + raise propagate_cancellation_error return winner_result, winner_index, exceptions finally: - # Make sure no tasks are left running if we leave this function - for t in running_tasks: - t.cancel() + del exceptions, propagate_cancellation_error, unhandled_exceptions, parent_task diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index 3fe52dbac25c91..64aac4cc50d15a 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -590,20 +590,34 @@ async def readuntil(self, separator=b'\n'): If the data cannot be read because of over limit, a LimitOverrunError exception will be raised, and the data will be left in the internal buffer, so it can be read again. + + The ``separator`` may also be a tuple of separators. In this + case the return value will be the shortest possible that has any + separator as the suffix. For the purposes of LimitOverrunError, + the shortest possible separator is considered to be the one that + matched. """ - seplen = len(separator) - if seplen == 0: + if isinstance(separator, tuple): + # Makes sure shortest matches wins + separator = sorted(separator, key=len) + else: + separator = [separator] + if not separator: + raise ValueError('Separator should contain at least one element') + min_seplen = len(separator[0]) + max_seplen = len(separator[-1]) + if min_seplen == 0: raise ValueError('Separator should be at least one-byte string') if self._exception is not None: raise self._exception # Consume whole buffer except last bytes, which length is - # one less than seplen. Let's check corner cases with - # separator='SEPARATOR': + # one less than max_seplen. Let's check corner cases with + # separator[-1]='SEPARATOR': # * we have received almost complete separator (without last # byte). i.e buffer='some textSEPARATO'. In this case we - # can safely consume len(separator) - 1 bytes. + # can safely consume max_seplen - 1 bytes. # * last byte of buffer is first byte of separator, i.e. # buffer='abcdefghijklmnopqrS'. We may safely consume # everything except that last byte, but this require to @@ -616,26 +630,35 @@ async def readuntil(self, separator=b'\n'): # messages :) # `offset` is the number of bytes from the beginning of the buffer - # where there is no occurrence of `separator`. + # where there is no occurrence of any `separator`. offset = 0 - # Loop until we find `separator` in the buffer, exceed the buffer size, + # Loop until we find a `separator` in the buffer, exceed the buffer size, # or an EOF has happened. while True: buflen = len(self._buffer) - # Check if we now have enough data in the buffer for `separator` to - # fit. - if buflen - offset >= seplen: - isep = self._buffer.find(separator, offset) - - if isep != -1: - # `separator` is in the buffer. `isep` will be used later - # to retrieve the data. + # Check if we now have enough data in the buffer for shortest + # separator to fit. + if buflen - offset >= min_seplen: + match_start = None + match_end = None + for sep in separator: + isep = self._buffer.find(sep, offset) + + if isep != -1: + # `separator` is in the buffer. `match_start` and + # `match_end` will be used later to retrieve the + # data. + end = isep + len(sep) + if match_end is None or end < match_end: + match_end = end + match_start = isep + if match_end is not None: break # see upper comment for explanation. - offset = buflen + 1 - seplen + offset = max(0, buflen + 1 - max_seplen) if offset > self._limit: raise exceptions.LimitOverrunError( 'Separator is not found, and chunk exceed the limit', @@ -644,7 +667,7 @@ async def readuntil(self, separator=b'\n'): # Complete message (with full separator) may be present in buffer # even when EOF flag is set. This may happen when the last chunk # adds data which makes separator be found. That's why we check for - # EOF *ater* inspecting the buffer. + # EOF *after* inspecting the buffer. if self._eof: chunk = bytes(self._buffer) self._buffer.clear() @@ -653,12 +676,12 @@ async def readuntil(self, separator=b'\n'): # _wait_for_data() will resume reading if stream was paused. await self._wait_for_data('readuntil') - if isep > self._limit: + if match_start > self._limit: raise exceptions.LimitOverrunError( - 'Separator is found, but chunk is longer than limit', isep) + 'Separator is found, but chunk is longer than limit', match_start) - chunk = self._buffer[:isep + seplen] - del self._buffer[:isep + seplen] + chunk = self._buffer[:match_end] + del self._buffer[:match_end] self._maybe_resume_transport() return bytes(chunk) diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index f322b1f6653f6a..1633478d1c87c2 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -6,6 +6,7 @@ from . import events from . import exceptions +from . import futures from . import tasks @@ -66,6 +67,20 @@ async def __aenter__(self): return self async def __aexit__(self, et, exc, tb): + tb = None + try: + return await self._aexit(et, exc) + finally: + # Exceptions are heavy objects that can have object + # cycles (bad for GC); let's not keep a reference to + # a bunch of them. It would be nicer to use a try/finally + # in __aexit__ directly but that introduced some diff noise + self._parent_task = None + self._errors = None + self._base_error = None + exc = None + + async def _aexit(self, et, exc): self._exiting = True if (exc is not None and @@ -77,12 +92,6 @@ async def __aexit__(self, et, exc, tb): propagate_cancellation_error = exc else: propagate_cancellation_error = None - if self._parent_cancel_requested: - # If this flag is set we *must* call uncancel(). - if self._parent_task.uncancel() == 0: - # If there are no pending cancellations left, - # don't propagate CancelledError. - propagate_cancellation_error = None if et is not None: if not self._aborting: @@ -128,25 +137,47 @@ async def __aexit__(self, et, exc, tb): assert not self._tasks if self._base_error is not None: - raise self._base_error + try: + raise self._base_error + finally: + exc = None + + if self._parent_cancel_requested: + # If this flag is set we *must* call uncancel(). + if self._parent_task.uncancel() == 0: + # If there are no pending cancellations left, + # don't propagate CancelledError. + propagate_cancellation_error = None # Propagate CancelledError if there is one, except if there # are other errors -- those have priority. - if propagate_cancellation_error is not None and not self._errors: - raise propagate_cancellation_error + try: + if propagate_cancellation_error is not None and not self._errors: + try: + raise propagate_cancellation_error + finally: + exc = None + finally: + propagate_cancellation_error = None if et is not None and not issubclass(et, exceptions.CancelledError): self._errors.append(exc) if self._errors: - # Exceptions are heavy objects that can have object - # cycles (bad for GC); let's not keep a reference to - # a bunch of them. + # If the parent task is being cancelled from the outside + # of the taskgroup, un-cancel and re-cancel the parent task, + # which will keep the cancel count stable. + if self._parent_task.cancelling(): + self._parent_task.uncancel() + self._parent_task.cancel() try: - me = BaseExceptionGroup('unhandled errors in a TaskGroup', self._errors) - raise me from None + raise BaseExceptionGroup( + 'unhandled errors in a TaskGroup', + self._errors, + ) from None finally: - self._errors = None + exc = None + def create_task(self, coro, *, name=None, context=None): """Create a new task in this group and return it. @@ -154,25 +185,33 @@ def create_task(self, coro, *, name=None, context=None): Similar to `asyncio.create_task`. """ if not self._entered: + coro.close() raise RuntimeError(f"TaskGroup {self!r} has not been entered") if self._exiting and not self._tasks: + coro.close() raise RuntimeError(f"TaskGroup {self!r} is finished") if self._aborting: + coro.close() raise RuntimeError(f"TaskGroup {self!r} is shutting down") if context is None: task = self._loop.create_task(coro, name=name) else: task = self._loop.create_task(coro, name=name, context=context) - # optimization: Immediately call the done callback if the task is + futures.future_add_to_awaited_by(task, self._parent_task) + + # Always schedule the done callback even if the task is # already done (e.g. if the coro was able to complete eagerly), - # and skip scheduling a done callback - if task.done(): - self._on_task_done(task) - else: - self._tasks.add(task) - task.add_done_callback(self._on_task_done) - return task + # otherwise if the task completes with an exception then it will cancel + # the current task too early. gh-128550, gh-128588 + self._tasks.add(task) + task.add_done_callback(self._on_task_done) + try: + return task + finally: + # gh-128552: prevent a refcycle of + # task.exception().__traceback__->TaskGroup.create_task->task + del task # Since Python 3.8 Tasks propagate all exceptions correctly, # except for KeyboardInterrupt and SystemExit which are @@ -192,6 +231,8 @@ def _abort(self): def _on_task_done(self, task): self._tasks.discard(task) + futures.future_discard_from_awaited_by(task, self._parent_task) + if self._on_completed_fut is not None and not self._tasks: if not self._on_completed_fut.done(): self._on_completed_fut.set_result(True) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index fafee3e738f6aa..a25854cc4bd69e 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -25,6 +25,7 @@ from . import events from . import exceptions from . import futures +from . import queues from . import timeouts # Helper to generate new task names @@ -47,23 +48,8 @@ def all_tasks(loop=None): # capturing the set of eager tasks first, so if an eager task "graduates" # to a regular task in another thread, we don't risk missing it. eager_tasks = list(_eager_tasks) - # Looping over the WeakSet isn't safe as it can be updated from another - # thread, therefore we cast it to list prior to filtering. The list cast - # itself requires iteration, so we repeat it several times ignoring - # RuntimeErrors (which are not very likely to occur). - # See issues 34970 and 36607 for details. - scheduled_tasks = None - i = 0 - while True: - try: - scheduled_tasks = list(_scheduled_tasks) - except RuntimeError: - i += 1 - if i >= 1000: - raise - else: - break - return {t for t in itertools.chain(scheduled_tasks, eager_tasks) + + return {t for t in itertools.chain(_scheduled_tasks, eager_tasks) if futures._get_loop(t) is loop and not t.done()} @@ -254,6 +240,8 @@ def uncancel(self): """ if self._num_cancels_requested > 0: self._num_cancels_requested -= 1 + if self._num_cancels_requested == 0: + self._must_cancel = False return self._num_cancels_requested def __eager_start(self): @@ -278,7 +266,7 @@ def __eager_start(self): def __step(self, exc=None): if self.done(): raise exceptions.InvalidStateError( - f'_step(): already done: {self!r}, {exc!r}') + f'__step(): already done: {self!r}, {exc!r}') if self._must_cancel: if not isinstance(exc, exceptions.CancelledError): exc = self._make_cancelled_error() @@ -334,6 +322,7 @@ def __step_run_and_handle_result(self, exc): self._loop.call_soon( self.__step, new_exc, context=self._context) else: + futures.future_add_to_awaited_by(result, self) result._asyncio_future_blocking = False result.add_done_callback( self.__wakeup, context=self._context) @@ -368,6 +357,7 @@ def __step_run_and_handle_result(self, exc): self = None # Needed to break cycles when an exception occurs. def __wakeup(self, future): + futures.future_discard_from_awaited_by(future, self) try: future.result() except BaseException as exc: @@ -376,7 +366,7 @@ def __wakeup(self, future): else: # Don't pass the value of `future.result()` explicitly, # as `Future.__iter__` and `Future.__await__` don't need it. - # If we call `_step(value, None)` instead of `_step()`, + # If we call `__step(value, None)` instead of `__step()`, # Python eval loop would use `.send(value)` method call, # instead of `__next__()`, which is slower for futures # that return non-generator iterators from their `__iter__`. @@ -464,7 +454,7 @@ async def wait_for(fut, timeout): If the wait is cancelled, the task is also cancelled. - If the task supresses the cancellation and returns a value instead, + If the task suppresses the cancellation and returns a value instead, that value is returned. This function is a coroutine. @@ -514,6 +504,7 @@ async def _wait(fs, timeout, return_when, loop): if timeout is not None: timeout_handle = loop.call_later(timeout, _release_waiter, waiter) counter = len(fs) + cur_task = current_task() def _on_completion(f): nonlocal counter @@ -526,9 +517,11 @@ def _on_completion(f): timeout_handle.cancel() if not waiter.done(): waiter.set_result(None) + futures.future_discard_from_awaited_by(f, cur_task) for f in fs: f.add_done_callback(_on_completion) + futures.future_add_to_awaited_by(f, cur_task) try: await waiter @@ -564,62 +557,125 @@ async def _cancel_and_wait(fut): fut.remove_done_callback(cb) -# This is *not* a @coroutine! It is just an iterator (yielding Futures). +class _AsCompletedIterator: + """Iterator of awaitables representing tasks of asyncio.as_completed. + + As an asynchronous iterator, iteration yields futures as they finish. As a + plain iterator, new coroutines are yielded that will return or raise the + result of the next underlying future to complete. + """ + def __init__(self, aws, timeout): + self._done = queues.Queue() + self._timeout_handle = None + + loop = events.get_event_loop() + todo = {ensure_future(aw, loop=loop) for aw in set(aws)} + for f in todo: + f.add_done_callback(self._handle_completion) + if todo and timeout is not None: + self._timeout_handle = ( + loop.call_later(timeout, self._handle_timeout) + ) + self._todo = todo + self._todo_left = len(todo) + + def __aiter__(self): + return self + + def __iter__(self): + return self + + async def __anext__(self): + if not self._todo_left: + raise StopAsyncIteration + assert self._todo_left > 0 + self._todo_left -= 1 + return await self._wait_for_one() + + def __next__(self): + if not self._todo_left: + raise StopIteration + assert self._todo_left > 0 + self._todo_left -= 1 + return self._wait_for_one(resolve=True) + + def _handle_timeout(self): + for f in self._todo: + f.remove_done_callback(self._handle_completion) + self._done.put_nowait(None) # Sentinel for _wait_for_one(). + self._todo.clear() # Can't do todo.remove(f) in the loop. + + def _handle_completion(self, f): + if not self._todo: + return # _handle_timeout() was here first. + self._todo.remove(f) + self._done.put_nowait(f) + if not self._todo and self._timeout_handle is not None: + self._timeout_handle.cancel() + + async def _wait_for_one(self, resolve=False): + # Wait for the next future to be done and return it unless resolve is + # set, in which case return either the result of the future or raise + # an exception. + f = await self._done.get() + if f is None: + # Dummy value from _handle_timeout(). + raise exceptions.TimeoutError + return f.result() if resolve else f + + def as_completed(fs, *, timeout=None): - """Return an iterator whose values are coroutines. + """Create an iterator of awaitables or their results in completion order. - When waiting for the yielded coroutines you'll get the results (or - exceptions!) of the original Futures (or coroutines), in the order - in which and as soon as they complete. + Run the supplied awaitables concurrently. The returned object can be + iterated to obtain the results of the awaitables as they finish. - This differs from PEP 3148; the proper way to use this is: + The object returned can be iterated as an asynchronous iterator or a plain + iterator. When asynchronous iteration is used, the originally-supplied + awaitables are yielded if they are tasks or futures. This makes it easy to + correlate previously-scheduled tasks with their results: - for f in as_completed(fs): - result = await f # The 'await' may raise. - # Use result. + ipv4_connect = create_task(open_connection("127.0.0.1", 80)) + ipv6_connect = create_task(open_connection("::1", 80)) + tasks = [ipv4_connect, ipv6_connect] - If a timeout is specified, the 'await' will raise - TimeoutError when the timeout occurs before all Futures are done. + async for earliest_connect in as_completed(tasks): + # earliest_connect is done. The result can be obtained by + # awaiting it or calling earliest_connect.result() + reader, writer = await earliest_connect - Note: The futures 'f' are not necessarily members of fs. - """ - if futures.isfuture(fs) or coroutines.iscoroutine(fs): - raise TypeError(f"expect an iterable of futures, not {type(fs).__name__}") + if earliest_connect is ipv6_connect: + print("IPv6 connection established.") + else: + print("IPv4 connection established.") - from .queues import Queue # Import here to avoid circular import problem. - done = Queue() + During asynchronous iteration, implicitly-created tasks will be yielded for + supplied awaitables that aren't tasks or futures. - loop = events.get_event_loop() - todo = {ensure_future(f, loop=loop) for f in set(fs)} - timeout_handle = None + When used as a plain iterator, each iteration yields a new coroutine that + returns the result or raises the exception of the next completed awaitable. + This pattern is compatible with Python versions older than 3.13: - def _on_timeout(): - for f in todo: - f.remove_done_callback(_on_completion) - done.put_nowait(None) # Queue a dummy value for _wait_for_one(). - todo.clear() # Can't do todo.remove(f) in the loop. + ipv4_connect = create_task(open_connection("127.0.0.1", 80)) + ipv6_connect = create_task(open_connection("::1", 80)) + tasks = [ipv4_connect, ipv6_connect] - def _on_completion(f): - if not todo: - return # _on_timeout() was here first. - todo.remove(f) - done.put_nowait(f) - if not todo and timeout_handle is not None: - timeout_handle.cancel() + for next_connect in as_completed(tasks): + # next_connect is not one of the original task objects. It must be + # awaited to obtain the result value or raise the exception of the + # awaitable that finishes next. + reader, writer = await next_connect - async def _wait_for_one(): - f = await done.get() - if f is None: - # Dummy value from _on_timeout(). - raise exceptions.TimeoutError - return f.result() # May raise f.exception(). + A TimeoutError is raised if the timeout occurs before all awaitables are + done. This is raised by the async for loop during asynchronous iteration or + by the coroutines yielded during plain iteration. + """ + if inspect.isawaitable(fs): + raise TypeError( + f"expects an iterable of awaitables, not {type(fs).__name__}" + ) - for f in todo: - f.add_done_callback(_on_completion) - if todo and timeout is not None: - timeout_handle = loop.call_later(timeout, _on_timeout) - for _ in range(len(todo)): - yield _wait_for_one() + return _AsCompletedIterator(fs, timeout) @types.coroutine @@ -751,10 +807,19 @@ def gather(*coros_or_futures, return_exceptions=False): outer.set_result([]) return outer - def _done_callback(fut): + loop = events._get_running_loop() + if loop is not None: + cur_task = current_task(loop) + else: + cur_task = None + + def _done_callback(fut, cur_task=cur_task): nonlocal nfinished nfinished += 1 + if cur_task is not None: + futures.future_discard_from_awaited_by(fut, cur_task) + if outer is None or outer.done(): if not fut.cancelled(): # Mark exception retrieved. @@ -811,7 +876,6 @@ def _done_callback(fut): nfuts = 0 nfinished = 0 done_futs = [] - loop = None outer = None # bpo-46672 for arg in coros_or_futures: if arg not in arg_to_fut: @@ -824,12 +888,13 @@ def _done_callback(fut): # can't control it, disable the "destroy pending task" # warning. fut._log_destroy_pending = False - nfuts += 1 arg_to_fut[arg] = fut if fut.done(): done_futs.append(fut) else: + if cur_task is not None: + futures.future_add_to_awaited_by(fut, cur_task) fut.add_done_callback(_done_callback) else: @@ -889,7 +954,15 @@ def shield(arg): loop = futures._get_loop(inner) outer = loop.create_future() - def _inner_done_callback(inner): + if loop is not None and (cur_task := current_task(loop)) is not None: + futures.future_add_to_awaited_by(inner, cur_task) + else: + cur_task = None + + def _inner_done_callback(inner, cur_task=cur_task): + if cur_task is not None: + futures.future_discard_from_awaited_by(inner, cur_task) + if outer.cancelled(): if not inner.cancelled(): # Mark inner's result as retrieved. @@ -1031,14 +1104,14 @@ def _unregister_eager_task(task): _py_enter_task = _enter_task _py_leave_task = _leave_task _py_swap_current_task = _swap_current_task - +_py_all_tasks = all_tasks try: from _asyncio import (_register_task, _register_eager_task, _unregister_task, _unregister_eager_task, _enter_task, _leave_task, _swap_current_task, _scheduled_tasks, _eager_tasks, _current_tasks, - current_task) + current_task, all_tasks) except ImportError: pass else: @@ -1050,3 +1123,4 @@ def _unregister_eager_task(task): _c_enter_task = _enter_task _c_leave_task = _leave_task _c_swap_current_task = _swap_current_task + _c_all_tasks = all_tasks diff --git a/Lib/asyncio/timeouts.py b/Lib/asyncio/timeouts.py index e6f5100691d362..09342dc7c1310b 100644 --- a/Lib/asyncio/timeouts.py +++ b/Lib/asyncio/timeouts.py @@ -1,7 +1,6 @@ import enum from types import TracebackType -from typing import final, Optional, Type from . import events from . import exceptions @@ -23,14 +22,13 @@ class _State(enum.Enum): EXITED = "finished" -@final class Timeout: """Asynchronous context manager for cancelling overdue coroutines. Use `timeout()` or `timeout_at()` rather than instantiating this class directly. """ - def __init__(self, when: Optional[float]) -> None: + def __init__(self, when: float | None) -> None: """Schedule a timeout that will trigger at a given loop time. - If `when` is `None`, the timeout will never trigger. @@ -39,15 +37,15 @@ def __init__(self, when: Optional[float]) -> None: """ self._state = _State.CREATED - self._timeout_handler: Optional[events.TimerHandle] = None - self._task: Optional[tasks.Task] = None + self._timeout_handler: events.TimerHandle | None = None + self._task: tasks.Task | None = None self._when = when - def when(self) -> Optional[float]: + def when(self) -> float | None: """Return the current deadline.""" return self._when - def reschedule(self, when: Optional[float]) -> None: + def reschedule(self, when: float | None) -> None: """Reschedule the timeout.""" if self._state is not _State.ENTERED: if self._state is _State.CREATED: @@ -96,10 +94,10 @@ async def __aenter__(self) -> "Timeout": async def __aexit__( self, - exc_type: Optional[Type[BaseException]], - exc_val: Optional[BaseException], - exc_tb: Optional[TracebackType], - ) -> Optional[bool]: + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> bool | None: assert self._state in (_State.ENTERED, _State.EXPIRING) if self._timeout_handler is not None: @@ -142,7 +140,7 @@ def _insert_timeout_error(exc_val: BaseException) -> None: exc_val = exc_val.__context__ -def timeout(delay: Optional[float]) -> Timeout: +def timeout(delay: float | None) -> Timeout: """Timeout async context manager. Useful in cases when you want to apply timeout logic around block @@ -162,7 +160,7 @@ def timeout(delay: Optional[float]) -> Timeout: return Timeout(loop.time() + delay if delay is not None else None) -def timeout_at(when: Optional[float]) -> Timeout: +def timeout_at(when: float | None) -> Timeout: """Schedule the timeout at absolute time. Like timeout() but argument gives absolute time in the same clock system diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 41ccf1b78fb93b..f69c6a64c39ae6 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -28,10 +28,7 @@ __all__ = ( 'SelectorEventLoop', - 'AbstractChildWatcher', 'SafeChildWatcher', - 'FastChildWatcher', 'PidfdChildWatcher', - 'MultiLoopChildWatcher', 'ThreadedChildWatcher', - 'DefaultEventLoopPolicy', + '_DefaultEventLoopPolicy', 'EventLoop', ) @@ -65,6 +62,10 @@ def __init__(self, selector=None): super().__init__(selector) self._signal_handlers = {} self._unix_server_sockets = {} + if can_use_pidfd(): + self._watcher = _PidfdChildWatcher() + else: + self._watcher = _ThreadedChildWatcher() def close(self): super().close() @@ -94,7 +95,7 @@ def add_signal_handler(self, sig, callback, *args): Raise RuntimeError if there is a problem setting up the handler. """ if (coroutines.iscoroutine(callback) or - coroutines.iscoroutinefunction(callback)): + coroutines._iscoroutinefunction(callback)): raise TypeError("coroutines cannot be used " "with add_signal_handler()") self._check_signal(sig) @@ -197,33 +198,22 @@ def _make_write_pipe_transport(self, pipe, protocol, waiter=None, async def _make_subprocess_transport(self, protocol, args, shell, stdin, stdout, stderr, bufsize, extra=None, **kwargs): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = events.get_child_watcher() - - with watcher: - if not watcher.is_active(): - # Check early. - # Raising exception before process creation - # prevents subprocess execution if the watcher - # is not ready to handle it. - raise RuntimeError("asyncio.get_child_watcher() is not activated, " - "subprocess support is not installed.") - waiter = self.create_future() - transp = _UnixSubprocessTransport(self, protocol, args, shell, - stdin, stdout, stderr, bufsize, - waiter=waiter, extra=extra, - **kwargs) - watcher.add_child_handler(transp.get_pid(), - self._child_watcher_callback, transp) - try: - await waiter - except (SystemExit, KeyboardInterrupt): - raise - except BaseException: - transp.close() - await transp._wait() - raise + watcher = self._watcher + waiter = self.create_future() + transp = _UnixSubprocessTransport(self, protocol, args, shell, + stdin, stdout, stderr, bufsize, + waiter=waiter, extra=extra, + **kwargs) + watcher.add_child_handler(transp.get_pid(), + self._child_watcher_callback, transp) + try: + await waiter + except (SystemExit, KeyboardInterrupt): + raise + except BaseException: + transp.close() + await transp._wait() + raise return transp @@ -865,93 +855,7 @@ def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): stdin_w.close() -class AbstractChildWatcher: - """Abstract base class for monitoring child processes. - - Objects derived from this class monitor a collection of subprocesses and - report their termination or interruption by a signal. - - New callbacks are registered with .add_child_handler(). Starting a new - process must be done within a 'with' block to allow the watcher to suspend - its activity until the new process if fully registered (this is needed to - prevent a race condition in some implementations). - - Example: - with watcher: - proc = subprocess.Popen("sleep 1") - watcher.add_child_handler(proc.pid, callback) - - Notes: - Implementations of this class must be thread-safe. - - Since child watcher objects may catch the SIGCHLD signal and call - waitpid(-1), there should be only one active object per process. - """ - - def __init_subclass__(cls) -> None: - if cls.__module__ != __name__: - warnings._deprecated("AbstractChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def add_child_handler(self, pid, callback, *args): - """Register a new child handler. - - Arrange for callback(pid, returncode, *args) to be called when - process 'pid' terminates. Specifying another callback for the same - process replaces the previous handler. - - Note: callback() must be thread-safe. - """ - raise NotImplementedError() - - def remove_child_handler(self, pid): - """Removes the handler for process 'pid'. - - The function returns True if the handler was successfully removed, - False if there was nothing to remove.""" - - raise NotImplementedError() - - def attach_loop(self, loop): - """Attach the watcher to an event loop. - - If the watcher was previously attached to an event loop, then it is - first detached before attaching to the new loop. - - Note: loop may be None. - """ - raise NotImplementedError() - - def close(self): - """Close the watcher. - - This must be called to make sure that any underlying resource is freed. - """ - raise NotImplementedError() - - def is_active(self): - """Return ``True`` if the watcher is active and is used by the event loop. - - Return True if the watcher is installed and ready to handle process exit - notifications. - - """ - raise NotImplementedError() - - def __enter__(self): - """Enter the watcher's context and allow starting new processes - - This function must return self""" - raise NotImplementedError() - - def __exit__(self, a, b, c): - """Exit the watcher's context""" - raise NotImplementedError() - - -class PidfdChildWatcher(AbstractChildWatcher): +class _PidfdChildWatcher: """Child watcher implementation using Linux's pid file descriptors. This child watcher polls process file descriptors (pidfds) to await child @@ -963,21 +867,6 @@ class PidfdChildWatcher(AbstractChildWatcher): recent (5.3+) kernels. """ - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, exc_traceback): - pass - - def is_active(self): - return True - - def close(self): - pass - - def attach_loop(self, loop): - pass - def add_child_handler(self, pid, callback, *args): loop = events.get_running_loop() pidfd = os.pidfd_open(pid) @@ -1002,386 +891,7 @@ def _do_wait(self, pid, pidfd, callback, args): os.close(pidfd) callback(pid, returncode, *args) - def remove_child_handler(self, pid): - # asyncio never calls remove_child_handler() !!! - # The method is no-op but is implemented because - # abstract base classes require it. - return True - - -class BaseChildWatcher(AbstractChildWatcher): - - def __init__(self): - self._loop = None - self._callbacks = {} - - def close(self): - self.attach_loop(None) - - def is_active(self): - return self._loop is not None and self._loop.is_running() - - def _do_waitpid(self, expected_pid): - raise NotImplementedError() - - def _do_waitpid_all(self): - raise NotImplementedError() - - def attach_loop(self, loop): - assert loop is None or isinstance(loop, events.AbstractEventLoop) - - if self._loop is not None and loop is None and self._callbacks: - warnings.warn( - 'A loop is being detached ' - 'from a child watcher with pending handlers', - RuntimeWarning) - - if self._loop is not None: - self._loop.remove_signal_handler(signal.SIGCHLD) - - self._loop = loop - if loop is not None: - loop.add_signal_handler(signal.SIGCHLD, self._sig_chld) - - # Prevent a race condition in case a child terminated - # during the switch. - self._do_waitpid_all() - - def _sig_chld(self): - try: - self._do_waitpid_all() - except (SystemExit, KeyboardInterrupt): - raise - except BaseException as exc: - # self._loop should always be available here - # as '_sig_chld' is added as a signal handler - # in 'attach_loop' - self._loop.call_exception_handler({ - 'message': 'Unknown exception in SIGCHLD handler', - 'exception': exc, - }) - - -class SafeChildWatcher(BaseChildWatcher): - """'Safe' child watcher implementation. - - This implementation avoids disrupting other code spawning processes by - polling explicitly each process in the SIGCHLD handler instead of calling - os.waitpid(-1). - - This is a safe solution but it has a significant overhead when handling a - big number of children (O(n) each time SIGCHLD is raised) - """ - - def __init__(self): - super().__init__() - warnings._deprecated("SafeChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def close(self): - self._callbacks.clear() - super().close() - - def __enter__(self): - return self - - def __exit__(self, a, b, c): - pass - - def add_child_handler(self, pid, callback, *args): - self._callbacks[pid] = (callback, args) - - # Prevent a race condition in case the child is already terminated. - self._do_waitpid(pid) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def _do_waitpid_all(self): - - for pid in list(self._callbacks): - self._do_waitpid(pid) - - def _do_waitpid(self, expected_pid): - assert expected_pid > 0 - - try: - pid, status = os.waitpid(expected_pid, os.WNOHANG) - except ChildProcessError: - # The child process is already reaped - # (may happen if waitpid() is called elsewhere). - pid = expected_pid - returncode = 255 - logger.warning( - "Unknown child process pid %d, will report returncode 255", - pid) - else: - if pid == 0: - # The child process is still alive. - return - - returncode = waitstatus_to_exitcode(status) - if self._loop.get_debug(): - logger.debug('process %s exited with returncode %s', - expected_pid, returncode) - - try: - callback, args = self._callbacks.pop(pid) - except KeyError: # pragma: no cover - # May happen if .remove_child_handler() is called - # after os.waitpid() returns. - if self._loop.get_debug(): - logger.warning("Child watcher got an unexpected pid: %r", - pid, exc_info=True) - else: - callback(pid, returncode, *args) - - -class FastChildWatcher(BaseChildWatcher): - """'Fast' child watcher implementation. - - This implementation reaps every terminated processes by calling - os.waitpid(-1) directly, possibly breaking other code spawning processes - and waiting for their termination. - - There is no noticeable overhead when handling a big number of children - (O(1) each time a child terminates). - """ - def __init__(self): - super().__init__() - self._lock = threading.Lock() - self._zombies = {} - self._forks = 0 - warnings._deprecated("FastChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def close(self): - self._callbacks.clear() - self._zombies.clear() - super().close() - - def __enter__(self): - with self._lock: - self._forks += 1 - - return self - - def __exit__(self, a, b, c): - with self._lock: - self._forks -= 1 - - if self._forks or not self._zombies: - return - - collateral_victims = str(self._zombies) - self._zombies.clear() - - logger.warning( - "Caught subprocesses termination from unknown pids: %s", - collateral_victims) - - def add_child_handler(self, pid, callback, *args): - assert self._forks, "Must use the context manager" - - with self._lock: - try: - returncode = self._zombies.pop(pid) - except KeyError: - # The child is running. - self._callbacks[pid] = callback, args - return - - # The child is dead already. We can fire the callback. - callback(pid, returncode, *args) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def _do_waitpid_all(self): - # Because of signal coalescing, we must keep calling waitpid() as - # long as we're able to reap a child. - while True: - try: - pid, status = os.waitpid(-1, os.WNOHANG) - except ChildProcessError: - # No more child processes exist. - return - else: - if pid == 0: - # A child process is still alive. - return - - returncode = waitstatus_to_exitcode(status) - - with self._lock: - try: - callback, args = self._callbacks.pop(pid) - except KeyError: - # unknown child - if self._forks: - # It may not be registered yet. - self._zombies[pid] = returncode - if self._loop.get_debug(): - logger.debug('unknown process %s exited ' - 'with returncode %s', - pid, returncode) - continue - callback = None - else: - if self._loop.get_debug(): - logger.debug('process %s exited with returncode %s', - pid, returncode) - - if callback is None: - logger.warning( - "Caught subprocess termination from unknown pid: " - "%d -> %d", pid, returncode) - else: - callback(pid, returncode, *args) - - -class MultiLoopChildWatcher(AbstractChildWatcher): - """A watcher that doesn't require running loop in the main thread. - - This implementation registers a SIGCHLD signal handler on - instantiation (which may conflict with other code that - install own handler for this signal). - - The solution is safe but it has a significant overhead when - handling a big number of processes (*O(n)* each time a - SIGCHLD is received). - """ - - # Implementation note: - # The class keeps compatibility with AbstractChildWatcher ABC - # To achieve this it has empty attach_loop() method - # and doesn't accept explicit loop argument - # for add_child_handler()/remove_child_handler() - # but retrieves the current loop by get_running_loop() - - def __init__(self): - self._callbacks = {} - self._saved_sighandler = None - warnings._deprecated("MultiLoopChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def is_active(self): - return self._saved_sighandler is not None - - def close(self): - self._callbacks.clear() - if self._saved_sighandler is None: - return - - handler = signal.getsignal(signal.SIGCHLD) - if handler != self._sig_chld: - logger.warning("SIGCHLD handler was changed by outside code") - else: - signal.signal(signal.SIGCHLD, self._saved_sighandler) - self._saved_sighandler = None - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - - def add_child_handler(self, pid, callback, *args): - loop = events.get_running_loop() - self._callbacks[pid] = (loop, callback, args) - - # Prevent a race condition in case the child is already terminated. - self._do_waitpid(pid) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def attach_loop(self, loop): - # Don't save the loop but initialize itself if called first time - # The reason to do it here is that attach_loop() is called from - # unix policy only for the main thread. - # Main thread is required for subscription on SIGCHLD signal - if self._saved_sighandler is not None: - return - - self._saved_sighandler = signal.signal(signal.SIGCHLD, self._sig_chld) - if self._saved_sighandler is None: - logger.warning("Previous SIGCHLD handler was set by non-Python code, " - "restore to default handler on watcher close.") - self._saved_sighandler = signal.SIG_DFL - - # Set SA_RESTART to limit EINTR occurrences. - signal.siginterrupt(signal.SIGCHLD, False) - - def _do_waitpid_all(self): - for pid in list(self._callbacks): - self._do_waitpid(pid) - - def _do_waitpid(self, expected_pid): - assert expected_pid > 0 - - try: - pid, status = os.waitpid(expected_pid, os.WNOHANG) - except ChildProcessError: - # The child process is already reaped - # (may happen if waitpid() is called elsewhere). - pid = expected_pid - returncode = 255 - logger.warning( - "Unknown child process pid %d, will report returncode 255", - pid) - debug_log = False - else: - if pid == 0: - # The child process is still alive. - return - - returncode = waitstatus_to_exitcode(status) - debug_log = True - try: - loop, callback, args = self._callbacks.pop(pid) - except KeyError: # pragma: no cover - # May happen if .remove_child_handler() is called - # after os.waitpid() returns. - logger.warning("Child watcher got an unexpected pid: %r", - pid, exc_info=True) - else: - if loop.is_closed(): - logger.warning("Loop %r that handles pid %r is closed", loop, pid) - else: - if debug_log and loop.get_debug(): - logger.debug('process %s exited with returncode %s', - expected_pid, returncode) - loop.call_soon_threadsafe(callback, pid, returncode, *args) - - def _sig_chld(self, signum, frame): - try: - self._do_waitpid_all() - except (SystemExit, KeyboardInterrupt): - raise - except BaseException: - logger.warning('Unknown exception in SIGCHLD handler', exc_info=True) - - -class ThreadedChildWatcher(AbstractChildWatcher): +class _ThreadedChildWatcher: """Threaded child watcher implementation. The watcher uses a thread per process @@ -1398,18 +908,6 @@ def __init__(self): self._pid_counter = itertools.count(0) self._threads = {} - def is_active(self): - return True - - def close(self): - pass - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - def __del__(self, _warn=warnings.warn): threads = [thread for thread in list(self._threads.values()) if thread.is_alive()] @@ -1427,15 +925,6 @@ def add_child_handler(self, pid, callback, *args): self._threads[pid] = thread thread.start() - def remove_child_handler(self, pid): - # asyncio never calls remove_child_handler() !!! - # The method is no-op but is implemented because - # abstract base classes require it. - return True - - def attach_loop(self, loop): - pass - def _do_waitpid(self, loop, expected_pid, callback, args): assert expected_pid > 0 @@ -1474,63 +963,11 @@ def can_use_pidfd(): return True -class _UnixDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy): - """UNIX event loop policy with a watcher for child processes.""" +class _UnixDefaultEventLoopPolicy(events._BaseDefaultEventLoopPolicy): + """UNIX event loop policy""" _loop_factory = _UnixSelectorEventLoop - def __init__(self): - super().__init__() - self._watcher = None - - def _init_watcher(self): - with events._lock: - if self._watcher is None: # pragma: no branch - if can_use_pidfd(): - self._watcher = PidfdChildWatcher() - else: - self._watcher = ThreadedChildWatcher() - - def set_event_loop(self, loop): - """Set the event loop. - - As a side effect, if a child watcher was set before, then calling - .set_event_loop() from the main thread will call .attach_loop(loop) on - the child watcher. - """ - - super().set_event_loop(loop) - - if (self._watcher is not None and - threading.current_thread() is threading.main_thread()): - self._watcher.attach_loop(loop) - - def get_child_watcher(self): - """Get the watcher for child processes. - - If not yet set, a ThreadedChildWatcher object is automatically created. - """ - if self._watcher is None: - self._init_watcher() - - warnings._deprecated("get_child_watcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", remove=(3, 14)) - return self._watcher - - def set_child_watcher(self, watcher): - """Set the watcher for child processes.""" - - assert watcher is None or isinstance(watcher, AbstractChildWatcher) - - if self._watcher is not None: - self._watcher.close() - - self._watcher = watcher - warnings._deprecated("set_child_watcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", remove=(3, 14)) - SelectorEventLoop = _UnixSelectorEventLoop -DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy +_DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy EventLoop = SelectorEventLoop diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index b62ea75fee3858..5f75b17d8ca649 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -8,6 +8,7 @@ import _overlapped import _winapi import errno +from functools import partial import math import msvcrt import socket @@ -28,8 +29,8 @@ __all__ = ( 'SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor', - 'DefaultEventLoopPolicy', 'WindowsSelectorEventLoopPolicy', - 'WindowsProactorEventLoopPolicy', 'EventLoop', + '_DefaultEventLoopPolicy', '_WindowsSelectorEventLoopPolicy', + '_WindowsProactorEventLoopPolicy', 'EventLoop', ) @@ -324,13 +325,13 @@ def _run_forever_cleanup(self): if self._self_reading_future is not None: ov = self._self_reading_future._ov self._self_reading_future.cancel() - # self_reading_future was just cancelled so if it hasn't been - # finished yet, it never will be (it's possible that it has - # already finished and its callback is waiting in the queue, - # where it could still happen if the event loop is restarted). - # Unregister it otherwise IocpProactor.close will wait for it - # forever - if ov is not None: + # self_reading_future always uses IOCP, so even though it's + # been cancelled, we need to make sure that the IOCP message + # is received so that the kernel is not holding on to the + # memory, possibly causing memory corruption later. Only + # unregister it if IO is complete in all respects. Otherwise + # we need another _poll() later to complete the IO. + if ov is not None and not ov.pending: self._proactor._unregister(ov) self._self_reading_future = None @@ -467,6 +468,18 @@ def finish_socket_func(trans, key, ov): else: raise + @classmethod + def _finish_recvfrom(cls, trans, key, ov, *, empty_result): + try: + return cls.finish_socket_func(trans, key, ov) + except OSError as exc: + # WSARecvFrom will report ERROR_PORT_UNREACHABLE when the same + # socket is used to send to an address that is not listening. + if exc.winerror == _overlapped.ERROR_PORT_UNREACHABLE: + return empty_result, None + else: + raise + def recv(self, conn, nbytes, flags=0): self._register_with_iocp(conn) ov = _overlapped.Overlapped(NULL) @@ -501,7 +514,8 @@ def recvfrom(self, conn, nbytes, flags=0): except BrokenPipeError: return self._result((b'', None)) - return self._register(ov, conn, self.finish_socket_func) + return self._register(ov, conn, partial(self._finish_recvfrom, + empty_result=b'')) def recvfrom_into(self, conn, buf, flags=0): self._register_with_iocp(conn) @@ -511,17 +525,8 @@ def recvfrom_into(self, conn, buf, flags=0): except BrokenPipeError: return self._result((0, None)) - def finish_recv(trans, key, ov): - try: - return ov.getresult() - except OSError as exc: - if exc.winerror in (_overlapped.ERROR_NETNAME_DELETED, - _overlapped.ERROR_OPERATION_ABORTED): - raise ConnectionResetError(*exc.args) - else: - raise - - return self._register(ov, conn, finish_recv) + return self._register(ov, conn, partial(self._finish_recvfrom, + empty_result=0)) def sendto(self, conn, buf, flags=0, addr=None): self._register_with_iocp(conn) @@ -886,13 +891,13 @@ def callback(f): SelectorEventLoop = _WindowsSelectorEventLoop -class WindowsSelectorEventLoopPolicy(events.BaseDefaultEventLoopPolicy): +class _WindowsSelectorEventLoopPolicy(events._BaseDefaultEventLoopPolicy): _loop_factory = SelectorEventLoop -class WindowsProactorEventLoopPolicy(events.BaseDefaultEventLoopPolicy): +class _WindowsProactorEventLoopPolicy(events._BaseDefaultEventLoopPolicy): _loop_factory = ProactorEventLoop -DefaultEventLoopPolicy = WindowsProactorEventLoopPolicy +_DefaultEventLoopPolicy = _WindowsProactorEventLoopPolicy EventLoop = ProactorEventLoop diff --git a/Lib/base64.py b/Lib/base64.py old mode 100755 new mode 100644 index 25164d1a1df4fc..5d78cc09f40cd3 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -1,12 +1,9 @@ -#! /usr/bin/env python3 - """Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings""" # Modified 04-Oct-1995 by Jack Jansen to use binascii module # Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support # Modified 22-May-2007 by Guido van Rossum to use bytes everywhere -import re import struct import binascii @@ -286,7 +283,7 @@ def b16decode(s, casefold=False): s = _bytes_from_decode_data(s) if casefold: s = s.upper() - if re.search(b'[^0-9A-F]', s): + if s.translate(None, delete=b'0123456789ABCDEF'): raise binascii.Error('Non-base16 digit found') return binascii.unhexlify(s) @@ -332,7 +329,7 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): wrapcol controls whether the output should have newline (b'\\n') characters added to it. If this is non-zero, each output line will be at most this - many characters long. + many characters long, excluding the trailing newline. pad controls whether the input is padded to a multiple of 4 before encoding. Note that the btoa implementation always pads. diff --git a/Lib/bdb.py b/Lib/bdb.py index 1acf7957f0d669..a741628e32a981 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -3,6 +3,8 @@ import fnmatch import sys import os +import weakref +from contextlib import contextmanager from inspect import CO_GENERATOR, CO_COROUTINE, CO_ASYNC_GENERATOR __all__ = ["BdbQuit", "Bdb", "Breakpoint"] @@ -32,8 +34,11 @@ def __init__(self, skip=None): self.skip = set(skip) if skip else None self.breaks = {} self.fncache = {} - self.frame_trace_lines = {} + self.frame_trace_lines_opcodes = {} self.frame_returning = None + self.trace_opcodes = False + self.enterframe = None + self.code_linenos = weakref.WeakKeyDictionary() self._load_breaks() @@ -61,6 +66,12 @@ def reset(self): self.botframe = None self._set_stopinfo(None, None) + @contextmanager + def set_enterframe(self, frame): + self.enterframe = frame + yield + self.enterframe = None + def trace_dispatch(self, frame, event, arg): """Dispatch a trace function for debugged frames based on the event. @@ -85,24 +96,28 @@ def trace_dispatch(self, frame, event, arg): The arg parameter depends on the previous event. """ - if self.quitting: - return # None - if event == 'line': - return self.dispatch_line(frame) - if event == 'call': - return self.dispatch_call(frame, arg) - if event == 'return': - return self.dispatch_return(frame, arg) - if event == 'exception': - return self.dispatch_exception(frame, arg) - if event == 'c_call': - return self.trace_dispatch - if event == 'c_exception': - return self.trace_dispatch - if event == 'c_return': + + with self.set_enterframe(frame): + if self.quitting: + return # None + if event == 'line': + return self.dispatch_line(frame) + if event == 'call': + return self.dispatch_call(frame, arg) + if event == 'return': + return self.dispatch_return(frame, arg) + if event == 'exception': + return self.dispatch_exception(frame, arg) + if event == 'c_call': + return self.trace_dispatch + if event == 'c_exception': + return self.trace_dispatch + if event == 'c_return': + return self.trace_dispatch + if event == 'opcode': + return self.dispatch_opcode(frame, arg) + print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) return self.trace_dispatch - print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) - return self.trace_dispatch def dispatch_line(self, frame): """Invoke user function and return trace function for line event. @@ -148,6 +163,9 @@ def dispatch_return(self, frame, arg): if self.stop_here(frame) or frame == self.returnframe: # Ignore return events in generator except when stepping. if self.stopframe and frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: + # It's possible to trigger a StopIteration exception in + # the caller so we must set the trace function in the caller + self._set_caller_tracefunc(frame) return self.trace_dispatch try: self.frame_returning = frame @@ -158,6 +176,11 @@ def dispatch_return(self, frame, arg): # The user issued a 'next' or 'until' command. if self.stopframe is frame and self.stoplineno != -1: self._set_stopinfo(None, None) + # The previous frame might not have f_trace set, unless we are + # issuing a command that does not expect to stop, we should set + # f_trace + if self.stoplineno != -1: + self._set_caller_tracefunc(frame) return self.trace_dispatch def dispatch_exception(self, frame, arg): @@ -187,6 +210,17 @@ def dispatch_exception(self, frame, arg): return self.trace_dispatch + def dispatch_opcode(self, frame, arg): + """Invoke user function and return trace function for opcode event. + If the debugger stops on the current opcode, invoke + self.user_opcode(). Raise BdbQuit if self.quitting is set. + Return self.trace_dispatch to continue tracing in this scope. + """ + if self.stop_here(frame) or self.break_here(frame): + self.user_opcode(frame) + if self.quitting: raise BdbQuit + return self.trace_dispatch + # Normally derived classes don't override the following # methods, but they may if they want to redefine the # definition of stopping and breakpoints. @@ -250,9 +284,25 @@ def do_clear(self, arg): raise NotImplementedError("subclass of bdb must implement do_clear()") def break_anywhere(self, frame): - """Return True if there is any breakpoint for frame's filename. + """Return True if there is any breakpoint in that frame """ - return self.canonic(frame.f_code.co_filename) in self.breaks + filename = self.canonic(frame.f_code.co_filename) + if filename not in self.breaks: + return False + for lineno in self.breaks[filename]: + if self._lineno_in_frame(lineno, frame): + return True + return False + + def _lineno_in_frame(self, lineno, frame): + """Return True if the line number is in the frame's code object. + """ + code = frame.f_code + if lineno < code.co_firstlineno: + return False + if code not in self.code_linenos: + self.code_linenos[code] = set(lineno for _, _, lineno in code.co_lines()) + return lineno in self.code_linenos[code] # Derived classes should override the user_* methods # to gain control. @@ -273,7 +323,21 @@ def user_exception(self, frame, exc_info): """Called when we stop on an exception.""" pass - def _set_stopinfo(self, stopframe, returnframe, stoplineno=0): + def user_opcode(self, frame): + """Called when we are about to execute an opcode.""" + pass + + def _set_trace_opcodes(self, trace_opcodes): + if trace_opcodes != self.trace_opcodes: + self.trace_opcodes = trace_opcodes + frame = self.enterframe + while frame is not None: + frame.f_trace_opcodes = trace_opcodes + if frame is self.botframe: + break + frame = frame.f_back + + def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): """Set the attributes for stopping. If stoplineno is greater than or equal to 0, then stop at line @@ -286,6 +350,17 @@ def _set_stopinfo(self, stopframe, returnframe, stoplineno=0): # stoplineno >= 0 means: stop at line >= the stoplineno # stoplineno -1 means: don't stop at all self.stoplineno = stoplineno + self._set_trace_opcodes(opcode) + + def _set_caller_tracefunc(self, current_frame): + # Issue #13183: pdb skips frames after hitting a breakpoint and running + # step commands. + # Restore the trace function in the caller (that may not have been set + # for performance reasons) when returning from the current frame, unless + # the caller is the botframe. + caller_frame = current_frame.f_back + if caller_frame and not caller_frame.f_trace and caller_frame is not self.botframe: + caller_frame.f_trace = self.trace_dispatch # Derived classes and clients can call the following methods # to affect the stepping state. @@ -300,16 +375,12 @@ def set_until(self, frame, lineno=None): def set_step(self): """Stop after one line of code.""" - # Issue #13183: pdb skips frames after hitting a breakpoint and running - # step commands. - # Restore the trace function in the caller (that may not have been set - # for performance reasons) when returning from the current frame. - if self.frame_returning: - caller_frame = self.frame_returning.f_back - if caller_frame and not caller_frame.f_trace: - caller_frame.f_trace = self.trace_dispatch self._set_stopinfo(None, None) + def set_stepinstr(self): + """Stop before the next instruction.""" + self._set_stopinfo(None, None, opcode=True) + def set_next(self, frame): """Stop on the next line in or below the given frame.""" self._set_stopinfo(frame, None) @@ -317,7 +388,7 @@ def set_next(self, frame): def set_return(self, frame): """Stop when returning from the given frame.""" if frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: - self._set_stopinfo(frame, None, -1) + self._set_stopinfo(frame, frame, -1) else: self._set_stopinfo(frame.f_back, frame) @@ -326,17 +397,19 @@ def set_trace(self, frame=None): If frame is not specified, debugging starts from caller's frame. """ + sys.settrace(None) if frame is None: frame = sys._getframe().f_back self.reset() - while frame: - frame.f_trace = self.trace_dispatch - self.botframe = frame - # We need f_trace_liens == True for the debugger to work - self.frame_trace_lines[frame] = frame.f_trace_lines - frame.f_trace_lines = True - frame = frame.f_back - self.set_step() + with self.set_enterframe(frame): + while frame: + frame.f_trace = self.trace_dispatch + self.botframe = frame + self.frame_trace_lines_opcodes[frame] = (frame.f_trace_lines, frame.f_trace_opcodes) + # We need f_trace_lines == True for the debugger to work + frame.f_trace_lines = True + frame = frame.f_back + self.set_stepinstr() sys.settrace(self.trace_dispatch) def set_continue(self): @@ -353,9 +426,9 @@ def set_continue(self): while frame and frame is not self.botframe: del frame.f_trace frame = frame.f_back - for frame, prev_trace_lines in self.frame_trace_lines.items(): - frame.f_trace_lines = prev_trace_lines - self.frame_trace_lines = {} + for frame, (trace_lines, trace_opcodes) in self.frame_trace_lines_opcodes.items(): + frame.f_trace_lines, frame.f_trace_opcodes = trace_lines, trace_opcodes + self.frame_trace_lines_opcodes = {} def set_quit(self): """Set quitting attribute to True. @@ -394,6 +467,14 @@ def set_break(self, filename, lineno, temporary=False, cond=None, return 'Line %s:%d does not exist' % (filename, lineno) self._add_to_breaks(filename, lineno) bp = Breakpoint(filename, lineno, temporary, cond, funcname) + # After we set a new breakpoint, we need to search through all frames + # and set f_trace to trace_dispatch if there could be a breakpoint in + # that frame. + frame = self.enterframe + while frame: + if self.break_anywhere(frame): + frame.f_trace = self.trace_dispatch + frame = frame.f_back return None def _load_breaks(self): diff --git a/Lib/bz2.py b/Lib/bz2.py index fabe4f73c8d808..2420cd019069b4 100644 --- a/Lib/bz2.py +++ b/Lib/bz2.py @@ -17,7 +17,7 @@ from _bz2 import BZ2Compressor, BZ2Decompressor -_MODE_CLOSED = 0 +# Value 0 no longer used _MODE_READ = 1 # Value 2 no longer used _MODE_WRITE = 3 @@ -54,7 +54,7 @@ def __init__(self, filename, mode="r", *, compresslevel=9): """ self._fp = None self._closefp = False - self._mode = _MODE_CLOSED + self._mode = None if not (1 <= compresslevel <= 9): raise ValueError("compresslevel must be between 1 and 9") @@ -100,7 +100,7 @@ def close(self): May be called more than once without error. Once the file is closed, any other operation on it will raise a ValueError. """ - if self._mode == _MODE_CLOSED: + if self.closed: return try: if self._mode == _MODE_READ: @@ -115,13 +115,21 @@ def close(self): finally: self._fp = None self._closefp = False - self._mode = _MODE_CLOSED self._buffer = None @property def closed(self): """True if this file is closed.""" - return self._mode == _MODE_CLOSED + return self._fp is None + + @property + def name(self): + self._check_not_closed() + return self._fp.name + + @property + def mode(self): + return 'wb' if self._mode == _MODE_WRITE else 'rb' def fileno(self): """Return the file descriptor for the underlying file.""" diff --git a/Lib/cProfile.py b/Lib/cProfile.py old mode 100755 new mode 100644 index 9c132372dc4ee0..e7c868b8d55543 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """Python interface for the 'lsprof' profiler. Compatible with the 'profile' module. """ diff --git a/Lib/calendar.py b/Lib/calendar.py index 833ce331b14a0c..d2e5e08d02dbb9 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -27,7 +27,9 @@ error = ValueError # Exceptions raised for bad input -class IllegalMonthError(ValueError): +# This is trick for backward compatibility. Since 3.13, we will raise IllegalMonthError instead of +# IndexError for bad month number(out of 1-12). But we can't remove IndexError for backward compatibility. +class IllegalMonthError(ValueError, IndexError): def __init__(self, month): self.month = month def __str__(self): @@ -158,11 +160,14 @@ def weekday(year, month, day): return Day(datetime.date(year, month, day).weekday()) -def monthrange(year, month): - """Return weekday (0-6 ~ Mon-Sun) and number of days (28-31) for - year, month.""" +def _validate_month(month): if not 1 <= month <= 12: raise IllegalMonthError(month) + +def monthrange(year, month): + """Return weekday of first day of month (0-6 ~ Mon-Sun) + and number of days (28-31) for year, month.""" + _validate_month(month) day1 = weekday(year, month, 1) ndays = mdays[month] + (month == FEBRUARY and isleap(year)) return day1, ndays @@ -344,11 +349,27 @@ def formatday(self, day, weekday, width): s = '%2i' % day # right-align single-digit days return s.center(width) - def formatweek(self, theweek, width): + def formatweek(self, theweek, width, *, highlight_day=None): """ Returns a single week in a string (no newline). """ - return ' '.join(self.formatday(d, wd, width) for (d, wd) in theweek) + if highlight_day: + from _colorize import get_colors + + ansi = get_colors() + highlight = f"{ansi.BLACK}{ansi.BACKGROUND_YELLOW}" + reset = ansi.RESET + else: + highlight = reset = "" + + return ' '.join( + ( + f"{highlight}{self.formatday(d, wd, width)}{reset}" + if d == highlight_day + else self.formatday(d, wd, width) + ) + for (d, wd) in theweek + ) def formatweekday(self, day, width): """ @@ -370,6 +391,8 @@ def formatmonthname(self, theyear, themonth, width, withyear=True): """ Return a formatted month name. """ + _validate_month(themonth) + s = month_name[themonth] if withyear: s = "%s %r" % (s, theyear) @@ -381,10 +404,11 @@ def prmonth(self, theyear, themonth, w=0, l=0): """ print(self.formatmonth(theyear, themonth, w, l), end='') - def formatmonth(self, theyear, themonth, w=0, l=0): + def formatmonth(self, theyear, themonth, w=0, l=0, *, highlight_day=None): """ Return a month's calendar string (multi-line). """ + highlight_day = highlight_day.day if highlight_day else None w = max(2, w) l = max(1, l) s = self.formatmonthname(theyear, themonth, 7 * (w + 1) - 1) @@ -393,11 +417,11 @@ def formatmonth(self, theyear, themonth, w=0, l=0): s += self.formatweekheader(w).rstrip() s += '\n' * l for week in self.monthdays2calendar(theyear, themonth): - s += self.formatweek(week, w).rstrip() + s += self.formatweek(week, w, highlight_day=highlight_day).rstrip() s += '\n' * l return s - def formatyear(self, theyear, w=2, l=1, c=6, m=3): + def formatyear(self, theyear, w=2, l=1, c=6, m=3, *, highlight_day=None): """ Returns a year's calendar as a multi-line string. """ @@ -421,15 +445,24 @@ def formatyear(self, theyear, w=2, l=1, c=6, m=3): headers = (header for k in months) a(formatstring(headers, colwidth, c).rstrip()) a('\n'*l) + + if highlight_day and highlight_day.month in months: + month_pos = months.index(highlight_day.month) + else: + month_pos = None + # max number of weeks for this row height = max(len(cal) for cal in row) for j in range(height): weeks = [] - for cal in row: + for k, cal in enumerate(row): if j >= len(cal): weeks.append('') else: - weeks.append(self.formatweek(cal[j], w)) + day = highlight_day.day if k == month_pos else None + weeks.append( + self.formatweek(cal[j], w, highlight_day=day) + ) a(formatstring(weeks, colwidth, c).rstrip()) a('\n' * l) return ''.join(v) @@ -500,6 +533,7 @@ def formatmonthname(self, theyear, themonth, withyear=True): """ Return a month name as a table row. """ + _validate_month(themonth) if withyear: s = '%s %s' % (month_name[themonth], theyear) else: @@ -757,6 +791,7 @@ def main(args=None): sys.exit(1) locale = options.locale, options.encoding + today = datetime.date.today() if options.type == "html": if options.month: @@ -773,7 +808,7 @@ def main(args=None): optdict = dict(encoding=encoding, css=options.css) write = sys.stdout.buffer.write if options.year is None: - write(cal.formatyearpage(datetime.date.today().year, **optdict)) + write(cal.formatyearpage(today.year, **optdict)) else: write(cal.formatyearpage(options.year, **optdict)) else: @@ -786,11 +821,18 @@ def main(args=None): if options.month is None: optdict["c"] = options.spacing optdict["m"] = options.months + if options.month is not None: + _validate_month(options.month) if options.year is None: - result = cal.formatyear(datetime.date.today().year, **optdict) + optdict["highlight_day"] = today + result = cal.formatyear(today.year, **optdict) elif options.month is None: + if options.year == today.year: + optdict["highlight_day"] = today result = cal.formatyear(options.year, **optdict) else: + if options.year == today.year and options.month == today.month: + optdict["highlight_day"] = today result = cal.formatmonth(options.year, options.month, **optdict) write = sys.stdout.write if options.encoding: diff --git a/Lib/cmd.py b/Lib/cmd.py index a37d16cd7bde16..c333e099bd8c9a 100644 --- a/Lib/cmd.py +++ b/Lib/cmd.py @@ -5,16 +5,16 @@ 1. End of file on input is processed as the command 'EOF'. 2. A command is parsed out of each line by collecting the prefix composed of characters in the identchars member. -3. A command `foo' is dispatched to a method 'do_foo()'; the do_ method +3. A command 'foo' is dispatched to a method 'do_foo()'; the do_ method is passed a single argument consisting of the remainder of the line. 4. Typing an empty line repeats the last command. (Actually, it calls the - method `emptyline', which may be overridden in a subclass.) -5. There is a predefined `help' method. Given an argument `topic', it - calls the command `help_topic'. With no arguments, it lists all topics + method 'emptyline', which may be overridden in a subclass.) +5. There is a predefined 'help' method. Given an argument 'topic', it + calls the command 'help_topic'. With no arguments, it lists all topics with defined help_ functions, broken into up to three topics; documented commands, miscellaneous help topics, and undocumented commands. -6. The command '?' is a synonym for `help'. The command '!' is a synonym - for `shell', if a do_shell method exists. +6. The command '?' is a synonym for 'help'. The command '!' is a synonym + for 'shell', if a do_shell method exists. 7. If completion is enabled, completing commands will be done automatically, and completing of commands args is done by calling complete_foo() with arguments text, line, begidx, endidx. text is string we are matching @@ -23,21 +23,21 @@ indexes of the text being matched, which could be used to provide different completion depending upon which position the argument is in. -The `default' method may be overridden to intercept commands for which there +The 'default' method may be overridden to intercept commands for which there is no do_ method. -The `completedefault' method may be overridden to intercept completions for +The 'completedefault' method may be overridden to intercept completions for commands that have no complete_ method. -The data member `self.ruler' sets the character used to draw separator lines +The data member 'self.ruler' sets the character used to draw separator lines in the help messages. If empty, no ruler line is drawn. It defaults to "=". -If the value of `self.intro' is nonempty when the cmdloop method is called, +If the value of 'self.intro' is nonempty when the cmdloop method is called, it is printed out on interpreter startup. This value may be overridden via an optional argument to the cmdloop() method. -The data members `self.doc_header', `self.misc_header', and -`self.undoc_header' set the headers used for the help function's +The data members 'self.doc_header', 'self.misc_header', and +'self.undoc_header' set the headers used for the help function's listings of documented functions, miscellaneous topics, and undocumented functions respectively. """ diff --git a/Lib/code.py b/Lib/code.py index f4aecddeca7813..1cc2ed8b1dbf28 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -25,10 +25,10 @@ class InteractiveInterpreter: def __init__(self, locals=None): """Constructor. - The optional 'locals' argument specifies the dictionary in - which code will be executed; it defaults to a newly created - dictionary with key "__name__" set to "__console__" and key - "__doc__" set to None. + The optional 'locals' argument specifies a mapping to use as the + namespace in which code will be executed; it defaults to a newly + created dictionary with key "__name__" set to "__console__" and + key "__doc__" set to None. """ if locals is None: @@ -64,7 +64,7 @@ def runsource(self, source, filename="", symbol="single"): code = self.compile(source, filename, symbol) except (OverflowError, SyntaxError, ValueError): # Case 1 - self.showsyntaxerror(filename) + self.showsyntaxerror(filename, source=source) return False if code is None: @@ -94,7 +94,7 @@ def runcode(self, code): except: self.showtraceback() - def showsyntaxerror(self, filename=None): + def showsyntaxerror(self, filename=None, **kwargs): """Display the syntax error that just occurred. This doesn't display a stack trace because there isn't one. @@ -106,29 +106,14 @@ def showsyntaxerror(self, filename=None): The output is written by self.write(), below. """ - type, value, tb = sys.exc_info() - sys.last_exc = value - sys.last_type = type - sys.last_value = value - sys.last_traceback = tb - if filename and type is SyntaxError: - # Work hard to stuff the correct filename in the exception - try: - msg, (dummy_filename, lineno, offset, line) = value.args - except ValueError: - # Not the format we expect; leave it alone - pass - else: - # Stuff in the right filename - value = SyntaxError(msg, (filename, lineno, offset, line)) - sys.last_exc = sys.last_value = value - if sys.excepthook is sys.__excepthook__: - lines = traceback.format_exception_only(type, value) - self.write(''.join(lines)) - else: - # If someone has set sys.excepthook, we let that take precedence - # over self.write - sys.excepthook(type, value, tb) + try: + typ, value, tb = sys.exc_info() + if filename and issubclass(typ, SyntaxError): + value.filename = filename + source = kwargs.pop('source', "") + self._showtraceback(typ, value, None, source) + finally: + typ = value = tb = None def showtraceback(self): """Display the exception that just occurred. @@ -138,19 +123,46 @@ def showtraceback(self): The output is written by self.write(), below. """ - sys.last_type, sys.last_value, last_tb = ei = sys.exc_info() - sys.last_traceback = last_tb - sys.last_exc = ei[1] try: - lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next) - if sys.excepthook is sys.__excepthook__: - self.write(''.join(lines)) - else: - # If someone has set sys.excepthook, we let that take precedence - # over self.write - sys.excepthook(ei[0], ei[1], last_tb) + typ, value, tb = sys.exc_info() + self._showtraceback(typ, value, tb.tb_next, "") finally: - last_tb = ei = None + typ = value = tb = None + + def _showtraceback(self, typ, value, tb, source): + sys.last_type = typ + sys.last_traceback = tb + value = value.with_traceback(tb) + # Set the line of text that the exception refers to + lines = source.splitlines() + if (source and typ is SyntaxError + and not value.text and value.lineno is not None + and len(lines) >= value.lineno): + value.text = lines[value.lineno - 1] + sys.last_exc = sys.last_value = value + if sys.excepthook is sys.__excepthook__: + self._excepthook(typ, value, tb) + else: + # If someone has set sys.excepthook, we let that take precedence + # over self.write + try: + sys.excepthook(typ, value, tb) + except SystemExit: + raise + except BaseException as e: + e.__context__ = None + e = e.with_traceback(e.__traceback__.tb_next) + print('Error in sys.excepthook:', file=sys.stderr) + sys.__excepthook__(type(e), e, e.__traceback__) + print(file=sys.stderr) + print('Original exception was:', file=sys.stderr) + sys.__excepthook__(typ, value, tb) + + def _excepthook(self, typ, value, tb): + # This method is being overwritten in + # _pyrepl.console.InteractiveColoredConsole + lines = traceback.format_exception(typ, value, tb) + self.write(''.join(lines)) def write(self, data): """Write a string. @@ -170,7 +182,7 @@ class InteractiveConsole(InteractiveInterpreter): """ - def __init__(self, locals=None, filename="", local_exit=False): + def __init__(self, locals=None, filename="", *, local_exit=False): """Constructor. The optional locals argument will be passed to the @@ -280,7 +292,7 @@ def interact(self, banner=None, exitmsg=None): elif exitmsg != '': self.write('%s\n' % exitmsg) - def push(self, line): + def push(self, line, filename=None, _symbol="single"): """Push a line to the interpreter. The line should not have a trailing newline; it may have @@ -296,7 +308,9 @@ def push(self, line): """ self.buffer.append(line) source = "\n".join(self.buffer) - more = self.runsource(source, self.filename) + if filename is None: + filename = self.filename + more = self.runsource(source, filename, symbol=_symbol) if not more: self.resetbuffer() return more @@ -351,7 +365,7 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=Fa console.raw_input = readfunc else: try: - import readline + import readline # noqa: F401 except ImportError: pass console.interact(banner, exitmsg) diff --git a/Lib/codecs.py b/Lib/codecs.py index 9b35b6127dd01c..e365e6cf22929f 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -1109,24 +1109,15 @@ def make_encoding_map(decoding_map): ### error handlers -try: - strict_errors = lookup_error("strict") - ignore_errors = lookup_error("ignore") - replace_errors = lookup_error("replace") - xmlcharrefreplace_errors = lookup_error("xmlcharrefreplace") - backslashreplace_errors = lookup_error("backslashreplace") - namereplace_errors = lookup_error("namereplace") -except LookupError: - # In --disable-unicode builds, these error handler are missing - strict_errors = None - ignore_errors = None - replace_errors = None - xmlcharrefreplace_errors = None - backslashreplace_errors = None - namereplace_errors = None +strict_errors = lookup_error("strict") +ignore_errors = lookup_error("ignore") +replace_errors = lookup_error("replace") +xmlcharrefreplace_errors = lookup_error("xmlcharrefreplace") +backslashreplace_errors = lookup_error("backslashreplace") +namereplace_errors = lookup_error("namereplace") # Tell modulefinder that using codecs probably needs the encodings # package _false = 0 if _false: - import encodings + import encodings # noqa: F401 diff --git a/Lib/codeop.py b/Lib/codeop.py index 6ad60e7f85098d..adf000ba29f88c 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -44,6 +44,7 @@ # Caveat emptor: These flags are undocumented on purpose and depending # on their effect outside the standard library is **unsupported**. PyCF_DONT_IMPLY_DEDENT = 0x200 +PyCF_ONLY_AST = 0x400 PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000 def _maybe_compile(compiler, source, filename, symbol): @@ -65,7 +66,7 @@ def _maybe_compile(compiler, source, filename, symbol): try: compiler(source + "\n", filename, symbol) return None - except IncompleteInputError as e: + except _IncompleteInputError as e: return None except SyntaxError as e: pass @@ -109,12 +110,14 @@ class Compile: def __init__(self): self.flags = PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT - def __call__(self, source, filename, symbol, **kwargs): - flags = self.flags + def __call__(self, source, filename, symbol, flags=0, **kwargs): + flags |= self.flags if kwargs.get('incomplete_input', True) is False: flags &= ~PyCF_DONT_IMPLY_DEDENT flags &= ~PyCF_ALLOW_INCOMPLETE_INPUT codeob = compile(source, filename, symbol, flags, True) + if flags & PyCF_ONLY_AST: + return codeob # this is an ast.Module in this case for feature in _features: if codeob.co_flags & feature.compiler_flag: self.flags |= feature.compiler_flag diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 2e527dfd810c43..78229ac54b80da 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -29,6 +29,9 @@ import _collections_abc import sys as _sys +_sys.modules['collections.abc'] = _collections_abc +abc = _collections_abc + from itertools import chain as _chain from itertools import repeat as _repeat from itertools import starmap as _starmap @@ -46,7 +49,8 @@ _collections_abc.MutableSequence.register(deque) try: - from _collections import _deque_iterator + # Expose _deque_iterator to support pickling deque iterators + from _collections import _deque_iterator # noqa: F401 except ImportError: pass @@ -55,6 +59,8 @@ except ImportError: pass +heapq = None # Lazily imported + ################################################################################ ### OrderedDict @@ -629,7 +635,10 @@ def most_common(self, n=None): return sorted(self.items(), key=_itemgetter(1), reverse=True) # Lazy import to speedup Python startup time - import heapq + global heapq + if heapq is None: + import heapq + return heapq.nlargest(n, self.items(), key=_itemgetter(1)) def elements(self): @@ -639,7 +648,8 @@ def elements(self): >>> sorted(c.elements()) ['A', 'A', 'B', 'B', 'C', 'C'] - # Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1 + Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1 + >>> import math >>> prime_factors = Counter({2: 2, 3: 3, 17: 1}) >>> math.prod(prime_factors.elements()) @@ -680,7 +690,7 @@ def update(self, iterable=None, /, **kwds): ''' # The regular dict.update() operation makes no sense here because the - # replace behavior results in the some of original untouched counts + # replace behavior results in some of the original untouched counts # being mixed-in with all of the other counts for a mismash that # doesn't have a straight-forward interpretation in most counting # contexts. Instead, we implement straight-addition. Both the inputs @@ -1015,7 +1025,7 @@ def __getitem__(self, key): return self.__missing__(key) # support subclasses that define __missing__ def get(self, key, default=None): - return self[key] if key in self else default + return self[key] if key in self else default # needs to make use of __contains__ def __len__(self): return len(set().union(*self.maps)) # reuses stored hash values if possible @@ -1027,7 +1037,10 @@ def __iter__(self): return iter(d) def __contains__(self, key): - return any(key in m for m in self.maps) + for mapping in self.maps: + if key in mapping: + return True + return False def __bool__(self): return any(self.maps) @@ -1037,9 +1050,9 @@ def __repr__(self): return f'{self.__class__.__name__}({", ".join(map(repr, self.maps))})' @classmethod - def fromkeys(cls, iterable, *args): - 'Create a ChainMap with a single dict created from the iterable.' - return cls(dict.fromkeys(iterable, *args)) + def fromkeys(cls, iterable, value=None, /): + 'Create a new ChainMap with keys from iterable and values set to value.' + return cls(dict.fromkeys(iterable, value)) def copy(self): 'New ChainMap or subclass with a new copy of maps[0] and refs to maps[1:]' diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py deleted file mode 100644 index 86ca8b8a8414b3..00000000000000 --- a/Lib/collections/abc.py +++ /dev/null @@ -1,3 +0,0 @@ -from _collections_abc import * -from _collections_abc import __all__ -from _collections_abc import _CallableGenericAlias diff --git a/Lib/colorsys.py b/Lib/colorsys.py index bc897bd0f99298..e97f91718a3a30 100644 --- a/Lib/colorsys.py +++ b/Lib/colorsys.py @@ -24,7 +24,7 @@ __all__ = ["rgb_to_yiq","yiq_to_rgb","rgb_to_hls","hls_to_rgb", "rgb_to_hsv","hsv_to_rgb"] -# Some floating point constants +# Some floating-point constants ONE_THIRD = 1.0/3.0 ONE_SIXTH = 1.0/6.0 diff --git a/Lib/compileall.py b/Lib/compileall.py index 9b53086bf41380..47e2446356e7d7 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -116,7 +116,8 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, prependdir=prependdir, limit_sl_dest=limit_sl_dest, hardlink_dupes=hardlink_dupes), - files) + files, + chunksize=4) success = min(results, default=True) else: for file in files: diff --git a/Lib/concurrent/futures/__init__.py b/Lib/concurrent/futures/__init__.py index 292e886d5a88ac..7ada7431c1ab8c 100644 --- a/Lib/concurrent/futures/__init__.py +++ b/Lib/concurrent/futures/__init__.py @@ -23,11 +23,13 @@ 'ALL_COMPLETED', 'CancelledError', 'TimeoutError', + 'InvalidStateError', 'BrokenExecutor', 'Future', 'Executor', 'wait', 'as_completed', + 'InterpreterPoolExecutor', 'ProcessPoolExecutor', 'ThreadPoolExecutor', ) @@ -38,7 +40,7 @@ def __dir__(): def __getattr__(name): - global ProcessPoolExecutor, ThreadPoolExecutor + global ProcessPoolExecutor, ThreadPoolExecutor, InterpreterPoolExecutor if name == 'ProcessPoolExecutor': from .process import ProcessPoolExecutor as pe @@ -50,4 +52,13 @@ def __getattr__(name): ThreadPoolExecutor = te return te + if name == 'InterpreterPoolExecutor': + try: + from .interpreter import InterpreterPoolExecutor as ie + except ModuleNotFoundError: + ie = InterpreterPoolExecutor = None + else: + InterpreterPoolExecutor = ie + return ie + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py index 6742a07753c921..707fcdfde79acd 100644 --- a/Lib/concurrent/futures/_base.py +++ b/Lib/concurrent/futures/_base.py @@ -23,14 +23,6 @@ CANCELLED_AND_NOTIFIED = 'CANCELLED_AND_NOTIFIED' FINISHED = 'FINISHED' -_FUTURE_STATES = [ - PENDING, - RUNNING, - CANCELLED, - CANCELLED_AND_NOTIFIED, - FINISHED -] - _STATE_TO_DESCRIPTION_MAP = { PENDING: "pending", RUNNING: "running", diff --git a/Lib/concurrent/futures/interpreter.py b/Lib/concurrent/futures/interpreter.py new file mode 100644 index 00000000000000..d17688dc9d7346 --- /dev/null +++ b/Lib/concurrent/futures/interpreter.py @@ -0,0 +1,242 @@ +"""Implements InterpreterPoolExecutor.""" + +import contextlib +import pickle +import textwrap +from . import thread as _thread +import _interpreters +import _interpqueues + + +class ExecutionFailed(_interpreters.InterpreterError): + """An unhandled exception happened during execution.""" + + def __init__(self, excinfo): + msg = excinfo.formatted + if not msg: + if excinfo.type and excinfo.msg: + msg = f'{excinfo.type.__name__}: {excinfo.msg}' + else: + msg = excinfo.type.__name__ or excinfo.msg + super().__init__(msg) + self.excinfo = excinfo + + def __str__(self): + try: + formatted = self.excinfo.errdisplay + except Exception: + return super().__str__() + else: + return textwrap.dedent(f""" +{super().__str__()} + +Uncaught in the interpreter: + +{formatted} + """.strip()) + + +UNBOUND = 2 # error; this should not happen. + + +class WorkerContext(_thread.WorkerContext): + + @classmethod + def prepare(cls, initializer, initargs, shared): + def resolve_task(fn, args, kwargs): + if isinstance(fn, str): + # XXX Circle back to this later. + raise TypeError('scripts not supported') + if args or kwargs: + raise ValueError(f'a script does not take args or kwargs, got {args!r} and {kwargs!r}') + data = textwrap.dedent(fn) + kind = 'script' + # Make sure the script compiles. + # Ideally we wouldn't throw away the resulting code + # object. However, there isn't much to be done until + # code objects are shareable and/or we do a better job + # of supporting code objects in _interpreters.exec(). + compile(data, '', 'exec') + else: + # Functions defined in the __main__ module can't be pickled, + # so they can't be used here. In the future, we could possibly + # borrow from multiprocessing to work around this. + data = pickle.dumps((fn, args, kwargs)) + kind = 'function' + return (data, kind) + + if initializer is not None: + try: + initdata = resolve_task(initializer, initargs, {}) + except ValueError: + if isinstance(initializer, str) and initargs: + raise ValueError(f'an initializer script does not take args, got {initargs!r}') + raise # re-raise + else: + initdata = None + def create_context(): + return cls(initdata, shared) + return create_context, resolve_task + + @classmethod + @contextlib.contextmanager + def _capture_exc(cls, resultsid): + try: + yield + except BaseException as exc: + # Send the captured exception out on the results queue, + # but still leave it unhandled for the interpreter to handle. + err = pickle.dumps(exc) + _interpqueues.put(resultsid, (None, err), 1, UNBOUND) + raise # re-raise + + @classmethod + def _send_script_result(cls, resultsid): + _interpqueues.put(resultsid, (None, None), 0, UNBOUND) + + @classmethod + def _call(cls, func, args, kwargs, resultsid): + with cls._capture_exc(resultsid): + res = func(*args or (), **kwargs or {}) + # Send the result back. + try: + _interpqueues.put(resultsid, (res, None), 0, UNBOUND) + except _interpreters.NotShareableError: + res = pickle.dumps(res) + _interpqueues.put(resultsid, (res, None), 1, UNBOUND) + + @classmethod + def _call_pickled(cls, pickled, resultsid): + with cls._capture_exc(resultsid): + fn, args, kwargs = pickle.loads(pickled) + cls._call(fn, args, kwargs, resultsid) + + def __init__(self, initdata, shared=None): + self.initdata = initdata + self.shared = dict(shared) if shared else None + self.interpid = None + self.resultsid = None + + def __del__(self): + if self.interpid is not None: + self.finalize() + + def _exec(self, script): + assert self.interpid is not None + excinfo = _interpreters.exec(self.interpid, script, restrict=True) + if excinfo is not None: + raise ExecutionFailed(excinfo) + + def initialize(self): + assert self.interpid is None, self.interpid + self.interpid = _interpreters.create(reqrefs=True) + try: + _interpreters.incref(self.interpid) + + maxsize = 0 + fmt = 0 + self.resultsid = _interpqueues.create(maxsize, fmt, UNBOUND) + + self._exec(f'from {__name__} import WorkerContext') + + if self.shared: + _interpreters.set___main___attrs( + self.interpid, self.shared, restrict=True) + + if self.initdata: + self.run(self.initdata) + except BaseException: + self.finalize() + raise # re-raise + + def finalize(self): + interpid = self.interpid + resultsid = self.resultsid + self.resultsid = None + self.interpid = None + if resultsid is not None: + try: + _interpqueues.destroy(resultsid) + except _interpqueues.QueueNotFoundError: + pass + if interpid is not None: + try: + _interpreters.decref(interpid) + except _interpreters.InterpreterNotFoundError: + pass + + def run(self, task): + data, kind = task + if kind == 'script': + raise NotImplementedError('script kind disabled') + script = f""" +with WorkerContext._capture_exc({self.resultsid}): +{textwrap.indent(data, ' ')} +WorkerContext._send_script_result({self.resultsid})""" + elif kind == 'function': + script = f'WorkerContext._call_pickled({data!r}, {self.resultsid})' + else: + raise NotImplementedError(kind) + + try: + self._exec(script) + except ExecutionFailed as exc: + exc_wrapper = exc + else: + exc_wrapper = None + + # Return the result, or raise the exception. + while True: + try: + obj = _interpqueues.get(self.resultsid) + except _interpqueues.QueueNotFoundError: + raise # re-raise + except _interpqueues.QueueError: + continue + except ModuleNotFoundError: + # interpreters.queues doesn't exist, which means + # QueueEmpty doesn't. Act as though it does. + continue + else: + break + (res, excdata), pickled, unboundop = obj + assert unboundop is None, unboundop + if excdata is not None: + assert res is None, res + assert pickled + assert exc_wrapper is not None + exc = pickle.loads(excdata) + raise exc from exc_wrapper + return pickle.loads(res) if pickled else res + + +class BrokenInterpreterPool(_thread.BrokenThreadPool): + """ + Raised when a worker thread in an InterpreterPoolExecutor failed initializing. + """ + + +class InterpreterPoolExecutor(_thread.ThreadPoolExecutor): + + BROKEN = BrokenInterpreterPool + + @classmethod + def prepare_context(cls, initializer, initargs, shared): + return WorkerContext.prepare(initializer, initargs, shared) + + def __init__(self, max_workers=None, thread_name_prefix='', + initializer=None, initargs=(), shared=None): + """Initializes a new InterpreterPoolExecutor instance. + + Args: + max_workers: The maximum number of interpreters that can be used to + execute the given calls. + thread_name_prefix: An optional name prefix to give our threads. + initializer: A callable or script used to initialize + each worker interpreter. + initargs: A tuple of arguments to pass to the initializer. + shared: A mapping of shareabled objects to be inserted into + each worker interpreter. + """ + super().__init__(max_workers, thread_name_prefix, + initializer, initargs, shared=shared) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index ca843e11eeb83d..42eee72bc1457f 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -68,27 +68,31 @@ class _ThreadWakeup: def __init__(self): self._closed = False + self._lock = threading.Lock() self._reader, self._writer = mp.Pipe(duplex=False) def close(self): - # Please note that we do not take the shutdown lock when + # Please note that we do not take the self._lock when # calling clear() (to avoid deadlocking) so this method can # only be called safely from the same thread as all calls to - # clear() even if you hold the shutdown lock. Otherwise we + # clear() even if you hold the lock. Otherwise we # might try to read from the closed pipe. - if not self._closed: - self._closed = True - self._writer.close() - self._reader.close() + with self._lock: + if not self._closed: + self._closed = True + self._writer.close() + self._reader.close() def wakeup(self): - if not self._closed: - self._writer.send_bytes(b"") + with self._lock: + if not self._closed: + self._writer.send_bytes(b"") def clear(self): - if not self._closed: - while self._reader.poll(): - self._reader.recv_bytes() + if self._closed: + raise RuntimeError('operation on closed _ThreadWakeup') + while self._reader.poll(): + self._reader.recv_bytes() def _python_exit(): @@ -167,10 +171,8 @@ def __init__(self, work_id, fn, args, kwargs): class _SafeQueue(Queue): """Safe Queue set exception to the future object linked to a job""" - def __init__(self, max_size=0, *, ctx, pending_work_items, shutdown_lock, - thread_wakeup): + def __init__(self, max_size=0, *, ctx, pending_work_items, thread_wakeup): self.pending_work_items = pending_work_items - self.shutdown_lock = shutdown_lock self.thread_wakeup = thread_wakeup super().__init__(max_size, ctx=ctx) @@ -179,8 +181,7 @@ def _on_queue_feeder_error(self, e, obj): tb = format_exception(type(e), e, e.__traceback__) e.__cause__ = _RemoteTraceback('\n"""\n{}"""'.format(''.join(tb))) work_item = self.pending_work_items.pop(obj.work_id, None) - with self.shutdown_lock: - self.thread_wakeup.wakeup() + self.thread_wakeup.wakeup() # work_item can be None if another process terminated. In this # case, the executor_manager_thread fails all work_items # with BrokenProcessPool @@ -296,11 +297,10 @@ def __init__(self, executor): # if there is no pending work item. def weakref_cb(_, thread_wakeup=self.thread_wakeup, - shutdown_lock=self.shutdown_lock): - mp.util.debug('Executor collected: triggering callback for' + mp_util_debug=mp.util.debug): + mp_util_debug('Executor collected: triggering callback for' ' QueueManager wakeup') - with shutdown_lock: - thread_wakeup.wakeup() + thread_wakeup.wakeup() self.executor_reference = weakref.ref(executor, weakref_cb) @@ -428,11 +428,6 @@ def wait_result_broken_or_wakeup(self): elif wakeup_reader in ready: is_broken = False - # No need to hold the _shutdown_lock here because: - # 1. we're the only thread to use the wakeup reader - # 2. we're also the only thread to call thread_wakeup.close() - # 3. we want to avoid a possible deadlock when both reader and writer - # would block (gh-105829) self.thread_wakeup.clear() return result_item, is_broken, cause @@ -588,7 +583,7 @@ def _check_system_limits(): raise NotImplementedError(_system_limited) _system_limits_checked = True try: - import multiprocessing.synchronize + import multiprocessing.synchronize # noqa: F401 except ImportError: _system_limited = ( "This Python build lacks multiprocessing.synchronize, usually due " @@ -720,10 +715,9 @@ def __init__(self, max_workers=None, mp_context=None, # as it could result in a deadlock if a worker process dies with the # _result_queue write lock still acquired. # - # _shutdown_lock must be locked to access _ThreadWakeup.close() and - # .wakeup(). Care must also be taken to not call clear or close from - # more than one thread since _ThreadWakeup.clear() is not protected by - # the _shutdown_lock + # Care must be taken to only call clear and close from the + # executor_manager_thread, since _ThreadWakeup.clear() is not protected + # by a lock. self._executor_manager_thread_wakeup = _ThreadWakeup() # Create communication channels for the executor @@ -734,7 +728,6 @@ def __init__(self, max_workers=None, mp_context=None, self._call_queue = _SafeQueue( max_size=queue_size, ctx=self._mp_context, pending_work_items=self._pending_work_items, - shutdown_lock=self._shutdown_lock, thread_wakeup=self._executor_manager_thread_wakeup) # Killed worker processes can produce spurious "broken pipe" # tracebacks in the queue's own worker thread. But we detect killed diff --git a/Lib/concurrent/futures/thread.py b/Lib/concurrent/futures/thread.py index a024033f35fb54..909359b648709f 100644 --- a/Lib/concurrent/futures/thread.py +++ b/Lib/concurrent/futures/thread.py @@ -41,21 +41,49 @@ def _python_exit(): os.register_at_fork(before=_global_shutdown_lock.acquire, after_in_child=_global_shutdown_lock._at_fork_reinit, after_in_parent=_global_shutdown_lock.release) + os.register_at_fork(after_in_child=_threads_queues.clear) + + +class WorkerContext: + + @classmethod + def prepare(cls, initializer, initargs): + if initializer is not None: + if not callable(initializer): + raise TypeError("initializer must be a callable") + def create_context(): + return cls(initializer, initargs) + def resolve_task(fn, args, kwargs): + return (fn, args, kwargs) + return create_context, resolve_task + + def __init__(self, initializer, initargs): + self.initializer = initializer + self.initargs = initargs + + def initialize(self): + if self.initializer is not None: + self.initializer(*self.initargs) + + def finalize(self): + pass + + def run(self, task): + fn, args, kwargs = task + return fn(*args, **kwargs) class _WorkItem: - def __init__(self, future, fn, args, kwargs): + def __init__(self, future, task): self.future = future - self.fn = fn - self.args = args - self.kwargs = kwargs + self.task = task - def run(self): + def run(self, ctx): if not self.future.set_running_or_notify_cancel(): return try: - result = self.fn(*self.args, **self.kwargs) + result = ctx.run(self.task) except BaseException as exc: self.future.set_exception(exc) # Break a reference cycle with the exception 'exc' @@ -66,16 +94,15 @@ def run(self): __class_getitem__ = classmethod(types.GenericAlias) -def _worker(executor_reference, work_queue, initializer, initargs): - if initializer is not None: - try: - initializer(*initargs) - except BaseException: - _base.LOGGER.critical('Exception in initializer:', exc_info=True) - executor = executor_reference() - if executor is not None: - executor._initializer_failed() - return +def _worker(executor_reference, ctx, work_queue): + try: + ctx.initialize() + except BaseException: + _base.LOGGER.critical('Exception in initializer:', exc_info=True) + executor = executor_reference() + if executor is not None: + executor._initializer_failed() + return try: while True: try: @@ -89,7 +116,7 @@ def _worker(executor_reference, work_queue, initializer, initargs): work_item = work_queue.get(block=True) if work_item is not None: - work_item.run() + work_item.run(ctx) # Delete references to object. See GH-60488 del work_item continue @@ -110,6 +137,8 @@ def _worker(executor_reference, work_queue, initializer, initargs): del executor except BaseException: _base.LOGGER.critical('Exception in worker', exc_info=True) + finally: + ctx.finalize() class BrokenThreadPool(_base.BrokenExecutor): @@ -120,11 +149,17 @@ class BrokenThreadPool(_base.BrokenExecutor): class ThreadPoolExecutor(_base.Executor): + BROKEN = BrokenThreadPool + # Used to assign unique thread names when thread_name_prefix is not supplied. _counter = itertools.count().__next__ + @classmethod + def prepare_context(cls, initializer, initargs): + return WorkerContext.prepare(initializer, initargs) + def __init__(self, max_workers=None, thread_name_prefix='', - initializer=None, initargs=()): + initializer=None, initargs=(), **ctxkwargs): """Initializes a new ThreadPoolExecutor instance. Args: @@ -133,6 +168,7 @@ def __init__(self, max_workers=None, thread_name_prefix='', thread_name_prefix: An optional name prefix to give our threads. initializer: A callable used to initialize worker threads. initargs: A tuple of arguments to pass to the initializer. + ctxkwargs: Additional arguments to cls.prepare_context(). """ if max_workers is None: # ThreadPoolExecutor is often used to: @@ -146,8 +182,9 @@ def __init__(self, max_workers=None, thread_name_prefix='', if max_workers <= 0: raise ValueError("max_workers must be greater than 0") - if initializer is not None and not callable(initializer): - raise TypeError("initializer must be a callable") + (self._create_worker_context, + self._resolve_work_item_task, + ) = type(self).prepare_context(initializer, initargs, **ctxkwargs) self._max_workers = max_workers self._work_queue = queue.SimpleQueue() @@ -158,13 +195,11 @@ def __init__(self, max_workers=None, thread_name_prefix='', self._shutdown_lock = threading.Lock() self._thread_name_prefix = (thread_name_prefix or ("ThreadPoolExecutor-%d" % self._counter())) - self._initializer = initializer - self._initargs = initargs def submit(self, fn, /, *args, **kwargs): with self._shutdown_lock, _global_shutdown_lock: if self._broken: - raise BrokenThreadPool(self._broken) + raise self.BROKEN(self._broken) if self._shutdown: raise RuntimeError('cannot schedule new futures after shutdown') @@ -173,7 +208,8 @@ def submit(self, fn, /, *args, **kwargs): 'interpreter shutdown') f = _base.Future() - w = _WorkItem(f, fn, args, kwargs) + task = self._resolve_work_item_task(fn, args, kwargs) + w = _WorkItem(f, task) self._work_queue.put(w) self._adjust_thread_count() @@ -196,9 +232,8 @@ def weakref_cb(_, q=self._work_queue): num_threads) t = threading.Thread(name=thread_name, target=_worker, args=(weakref.ref(self, weakref_cb), - self._work_queue, - self._initializer, - self._initargs)) + self._create_worker_context(), + self._work_queue)) t.start() self._threads.add(t) _threads_queues[t] = self._work_queue @@ -214,7 +249,7 @@ def _initializer_failed(self): except queue.Empty: break if work_item is not None: - work_item.future.set_exception(BrokenThreadPool(self._broken)) + work_item.future.set_exception(self.BROKEN(self._broken)) def shutdown(self, wait=True, *, cancel_futures=False): with self._shutdown_lock: diff --git a/Lib/configparser.py b/Lib/configparser.py index 71362d23ec3757..9dc4fa515cfcbe 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -18,8 +18,8 @@ delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section='DEFAULT', - interpolation=, converters=): - + interpolation=, converters=, + allow_unnamed_section=False): Create the parser. When `defaults` is given, it is initialized into the dictionary or intrinsic defaults. The keys must be strings, the values must be appropriate for %()s string interpolation. @@ -68,6 +68,10 @@ converter gets its corresponding get*() method on the parser object and section proxies. + When `allow_unnamed_section` is True (default: False), options + without section are accepted: the section for these is + ``configparser.UNNAMED_SECTION``. + sections() Return all the configuration section names, sans DEFAULT. @@ -139,23 +143,28 @@ between keys and values are surrounded by spaces. """ -from collections.abc import MutableMapping +# Do not import dataclasses; overhead is unacceptable (gh-117703) + +from collections.abc import Iterable, MutableMapping from collections import ChainMap as _ChainMap +import contextlib import functools import io import itertools import os import re import sys +import types __all__ = ("NoSectionError", "DuplicateOptionError", "DuplicateSectionError", "NoOptionError", "InterpolationError", "InterpolationDepthError", "InterpolationMissingOptionError", "InterpolationSyntaxError", "ParsingError", "MissingSectionHeaderError", + "MultilineContinuationError", "UnnamedSectionDisabledError", "ConfigParser", "RawConfigParser", "Interpolation", "BasicInterpolation", "ExtendedInterpolation", "SectionProxy", "ConverterMapping", - "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH") + "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH", "UNNAMED_SECTION") _default_dict = dict DEFAULTSECT = "DEFAULT" @@ -297,15 +306,33 @@ def __init__(self, option, section, rawval): class ParsingError(Error): """Raised when a configuration file does not follow legal syntax.""" - def __init__(self, source): + def __init__(self, source, *args): super().__init__(f'Source contains parsing errors: {source!r}') self.source = source self.errors = [] self.args = (source, ) + if args: + self.append(*args) def append(self, lineno, line): self.errors.append((lineno, line)) - self.message += '\n\t[line %2d]: %s' % (lineno, line) + self.message += '\n\t[line %2d]: %s' % (lineno, repr(line)) + + def combine(self, others): + for other in others: + for error in other.errors: + self.append(*error) + return self + + @staticmethod + def _raise_all(exceptions: Iterable['ParsingError']): + """ + Combine any number of ParsingErrors into one and raise it. + """ + exceptions = iter(exceptions) + with contextlib.suppress(StopIteration): + raise next(exceptions).combine(exceptions) + class MissingSectionHeaderError(ParsingError): @@ -322,6 +349,36 @@ def __init__(self, filename, lineno, line): self.args = (filename, lineno, line) +class MultilineContinuationError(ParsingError): + """Raised when a key without value is followed by continuation line""" + def __init__(self, filename, lineno, line): + Error.__init__( + self, + "Key without value continued with an indented line.\n" + "file: %r, line: %d\n%r" + %(filename, lineno, line)) + self.source = filename + self.lineno = lineno + self.line = line + self.args = (filename, lineno, line) + + +class UnnamedSectionDisabledError(Error): + """Raised when an attempt to use UNNAMED_SECTION is made with the + feature disabled.""" + def __init__(self): + Error.__init__(self, "Support for UNNAMED_SECTION is disabled.") + + +class _UnnamedSection: + + def __repr__(self): + return "" + + +UNNAMED_SECTION = _UnnamedSection() + + # Used in parser getters to indicate the default behaviour when a specific # option is not found it to raise an exception. Created to enable `None` as # a valid fallback value. @@ -490,6 +547,52 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, "found: %r" % (rest,)) +class _ReadState: + elements_added : set[str] + cursect : dict[str, str] | None = None + sectname : str | None = None + optname : str | None = None + lineno : int = 0 + indent_level : int = 0 + errors : list[ParsingError] + + def __init__(self): + self.elements_added = set() + self.errors = list() + + +class _Line(str): + + def __new__(cls, val, *args, **kwargs): + return super().__new__(cls, val) + + def __init__(self, val, prefixes): + self.prefixes = prefixes + + @functools.cached_property + def clean(self): + return self._strip_full() and self._strip_inline() + + @property + def has_comments(self): + return self.strip() != self.clean + + def _strip_inline(self): + """ + Search for the earliest prefix at the beginning of the line or following a space. + """ + matcher = re.compile( + '|'.join(fr'(^|\s)({re.escape(prefix)})' for prefix in self.prefixes.inline) + # match nothing if no prefixes + or '(?!)' + ) + match = matcher.search(self) + return self[:match.start() if match else None].strip() + + def _strip_full(self): + return '' if any(map(self.strip().startswith, self.prefixes.full)) else True + + class RawConfigParser(MutableMapping): """ConfigParser that does not do interpolation.""" @@ -536,7 +639,8 @@ def __init__(self, defaults=None, dict_type=_default_dict, comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=DEFAULTSECT, - interpolation=_UNSET, converters=_UNSET): + interpolation=_UNSET, converters=_UNSET, + allow_unnamed_section=False,): self._dict = dict_type self._sections = self._dict() @@ -555,8 +659,10 @@ def __init__(self, defaults=None, dict_type=_default_dict, else: self._optcre = re.compile(self._OPT_TMPL.format(delim=d), re.VERBOSE) - self._comment_prefixes = tuple(comment_prefixes or ()) - self._inline_comment_prefixes = tuple(inline_comment_prefixes or ()) + self._prefixes = types.SimpleNamespace( + full=tuple(comment_prefixes or ()), + inline=tuple(inline_comment_prefixes or ()), + ) self._strict = strict self._allow_no_value = allow_no_value self._empty_lines_in_values = empty_lines_in_values @@ -575,6 +681,7 @@ def __init__(self, defaults=None, dict_type=_default_dict, self._converters.update(converters) if defaults: self._read_defaults(defaults) + self._allow_unnamed_section = allow_unnamed_section def defaults(self): return self._defaults @@ -593,6 +700,10 @@ def add_section(self, section): if section == self.default_section: raise ValueError('Invalid section name: %r' % section) + if section is UNNAMED_SECTION: + if not self._allow_unnamed_section: + raise UnnamedSectionDisabledError + if section in self._sections: raise DuplicateSectionError(section) self._sections[section] = self._dict() @@ -848,13 +959,19 @@ def write(self, fp, space_around_delimiters=True): if self._defaults: self._write_section(fp, self.default_section, self._defaults.items(), d) + if UNNAMED_SECTION in self._sections: + self._write_section(fp, UNNAMED_SECTION, self._sections[UNNAMED_SECTION].items(), d, unnamed=True) + for section in self._sections: + if section is UNNAMED_SECTION: + continue self._write_section(fp, section, self._sections[section].items(), d) - def _write_section(self, fp, section_name, section_items, delimiter): - """Write a single section to the specified `fp`.""" - fp.write("[{}]\n".format(section_name)) + def _write_section(self, fp, section_name, section_items, delimiter, unnamed=False): + """Write a single section to the specified 'fp'.""" + if not unnamed: + fp.write("[{}]\n".format(section_name)) for key, value in section_items: value = self._interpolation.before_write(self, section_name, key, value) @@ -940,110 +1057,113 @@ def _read(self, fp, fpname): in an otherwise empty line or may be entered in lines holding values or section names. Please note that comments get stripped off when reading configuration files. """ - elements_added = set() - cursect = None # None, or a dictionary - sectname = None - optname = None - lineno = 0 - indent_level = 0 - e = None # None, or an exception - for lineno, line in enumerate(fp, start=1): - comment_start = sys.maxsize - # strip inline comments - inline_prefixes = {p: -1 for p in self._inline_comment_prefixes} - while comment_start == sys.maxsize and inline_prefixes: - next_prefixes = {} - for prefix, index in inline_prefixes.items(): - index = line.find(prefix, index+1) - if index == -1: - continue - next_prefixes[prefix] = index - if index == 0 or (index > 0 and line[index-1].isspace()): - comment_start = min(comment_start, index) - inline_prefixes = next_prefixes - # strip full line comments - for prefix in self._comment_prefixes: - if line.strip().startswith(prefix): - comment_start = 0 - break - if comment_start == sys.maxsize: - comment_start = None - value = line[:comment_start].strip() - if not value: + + try: + ParsingError._raise_all(self._read_inner(fp, fpname)) + finally: + self._join_multiline_values() + + def _read_inner(self, fp, fpname): + st = _ReadState() + + Line = functools.partial(_Line, prefixes=self._prefixes) + for st.lineno, line in enumerate(map(Line, fp), start=1): + if not line.clean: if self._empty_lines_in_values: # add empty line to the value, but only if there was no # comment on the line - if (comment_start is None and - cursect is not None and - optname and - cursect[optname] is not None): - cursect[optname].append('') # newlines added at join + if (not line.has_comments and + st.cursect is not None and + st.optname and + st.cursect[st.optname] is not None): + st.cursect[st.optname].append('') # newlines added at join else: # empty line marks end of value - indent_level = sys.maxsize + st.indent_level = sys.maxsize continue - # continuation line? + first_nonspace = self.NONSPACECRE.search(line) - cur_indent_level = first_nonspace.start() if first_nonspace else 0 - if (cursect is not None and optname and - cur_indent_level > indent_level): - cursect[optname].append(value) - # a section header or option header? - else: - indent_level = cur_indent_level - # is it a section header? - mo = self.SECTCRE.match(value) - if mo: - sectname = mo.group('header') - if sectname in self._sections: - if self._strict and sectname in elements_added: - raise DuplicateSectionError(sectname, fpname, - lineno) - cursect = self._sections[sectname] - elements_added.add(sectname) - elif sectname == self.default_section: - cursect = self._defaults - else: - cursect = self._dict() - self._sections[sectname] = cursect - self._proxies[sectname] = SectionProxy(self, sectname) - elements_added.add(sectname) - # So sections can't start with a continuation line - optname = None - # no section header in the file? - elif cursect is None: - raise MissingSectionHeaderError(fpname, lineno, line) - # an option line? - else: - mo = self._optcre.match(value) - if mo: - optname, vi, optval = mo.group('option', 'vi', 'value') - if not optname: - e = self._handle_error(e, fpname, lineno, line) - optname = self.optionxform(optname.rstrip()) - if (self._strict and - (sectname, optname) in elements_added): - raise DuplicateOptionError(sectname, optname, - fpname, lineno) - elements_added.add((sectname, optname)) - # This check is fine because the OPTCRE cannot - # match if it would set optval to None - if optval is not None: - optval = optval.strip() - cursect[optname] = [optval] - else: - # valueless option handling - cursect[optname] = None - else: - # a non-fatal parsing error occurred. set up the - # exception but keep going. the exception will be - # raised at the end of the file and will contain a - # list of all bogus lines - e = self._handle_error(e, fpname, lineno, line) - self._join_multiline_values() - # if any parsing errors occurred, raise an exception - if e: - raise e + st.cur_indent_level = first_nonspace.start() if first_nonspace else 0 + + if self._handle_continuation_line(st, line, fpname): + continue + + self._handle_rest(st, line, fpname) + + return st.errors + + def _handle_continuation_line(self, st, line, fpname): + # continuation line? + is_continue = (st.cursect is not None and st.optname and + st.cur_indent_level > st.indent_level) + if is_continue: + if st.cursect[st.optname] is None: + raise MultilineContinuationError(fpname, st.lineno, line) + st.cursect[st.optname].append(line.clean) + return is_continue + + def _handle_rest(self, st, line, fpname): + # a section header or option header? + if self._allow_unnamed_section and st.cursect is None: + self._handle_header(st, UNNAMED_SECTION, fpname) + + st.indent_level = st.cur_indent_level + # is it a section header? + mo = self.SECTCRE.match(line.clean) + + if not mo and st.cursect is None: + raise MissingSectionHeaderError(fpname, st.lineno, line) + + self._handle_header(st, mo.group('header'), fpname) if mo else self._handle_option(st, line, fpname) + + def _handle_header(self, st, sectname, fpname): + st.sectname = sectname + if st.sectname in self._sections: + if self._strict and st.sectname in st.elements_added: + raise DuplicateSectionError(st.sectname, fpname, + st.lineno) + st.cursect = self._sections[st.sectname] + st.elements_added.add(st.sectname) + elif st.sectname == self.default_section: + st.cursect = self._defaults + else: + st.cursect = self._dict() + self._sections[st.sectname] = st.cursect + self._proxies[st.sectname] = SectionProxy(self, st.sectname) + st.elements_added.add(st.sectname) + # So sections can't start with a continuation line + st.optname = None + + def _handle_option(self, st, line, fpname): + # an option line? + st.indent_level = st.cur_indent_level + + mo = self._optcre.match(line.clean) + if not mo: + # a non-fatal parsing error occurred. set up the + # exception but keep going. the exception will be + # raised at the end of the file and will contain a + # list of all bogus lines + st.errors.append(ParsingError(fpname, st.lineno, line)) + return + + st.optname, vi, optval = mo.group('option', 'vi', 'value') + if not st.optname: + st.errors.append(ParsingError(fpname, st.lineno, line)) + st.optname = self.optionxform(st.optname.rstrip()) + if (self._strict and + (st.sectname, st.optname) in st.elements_added): + raise DuplicateOptionError(st.sectname, st.optname, + fpname, st.lineno) + st.elements_added.add((st.sectname, st.optname)) + # This check is fine because the OPTCRE cannot + # match if it would set optval to None + if optval is not None: + optval = optval.strip() + st.cursect[st.optname] = [optval] + else: + # valueless option handling + st.cursect[st.optname] = None def _join_multiline_values(self): defaults = self.default_section, self._defaults @@ -1063,12 +1183,6 @@ def _read_defaults(self, defaults): for key, value in defaults.items(): self._defaults[self.optionxform(key)] = value - def _handle_error(self, exc, fpname, lineno, line): - if not exc: - exc = ParsingError(fpname) - exc.append(lineno, repr(line)) - return exc - def _unify_values(self, section, vars): """Create a sequence of lookups with 'vars' taking priority over the 'section' which takes priority over the DEFAULTSECT. @@ -1097,20 +1211,20 @@ def _convert_to_boolean(self, value): return self.BOOLEAN_STATES[value.lower()] def _validate_value_types(self, *, section="", option="", value=""): - """Raises a TypeError for non-string values. + """Raises a TypeError for illegal non-string values. - The only legal non-string value if we allow valueless - options is None, so we need to check if the value is a - string if: - - we do not allow valueless options, or - - we allow valueless options but the value is not None + Legal non-string values are UNNAMED_SECTION and falsey values if + they are allowed. For compatibility reasons this method is not used in classic set() for RawConfigParsers. It is invoked in every case for mapping protocol access and in ConfigParser.set(). """ - if not isinstance(section, str): - raise TypeError("section names must be strings") + if section is UNNAMED_SECTION: + if not self._allow_unnamed_section: + raise UnnamedSectionDisabledError + elif not isinstance(section, str): + raise TypeError("section names must be strings or UNNAMED_SECTION") if not isinstance(option, str): raise TypeError("option keys must be strings") if not self._allow_no_value or value: diff --git a/Lib/contextvars.py b/Lib/contextvars.py index d78c80dfe6f99c..14514f185e069d 100644 --- a/Lib/contextvars.py +++ b/Lib/contextvars.py @@ -1,4 +1,8 @@ +import _collections_abc from _contextvars import Context, ContextVar, Token, copy_context __all__ = ('Context', 'ContextVar', 'Token', 'copy_context') + + +_collections_abc.Mapping.register(Context) diff --git a/Lib/copy.py b/Lib/copy.py index a69bc4e78c20b3..c64fc0761793f5 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -4,8 +4,9 @@ import copy - x = copy.copy(y) # make a shallow copy of y - x = copy.deepcopy(y) # make a deep copy of y + x = copy.copy(y) # make a shallow copy of y + x = copy.deepcopy(y) # make a deep copy of y + x = copy.replace(y, a=1, b=2) # new object with fields replaced, as defined by `__replace__` For module specific errors, copy.Error is raised. @@ -56,7 +57,7 @@ class Error(Exception): pass error = Error # backward compatibility -__all__ = ["Error", "copy", "deepcopy"] +__all__ = ["Error", "copy", "deepcopy", "replace"] def copy(x): """Shallow copy operation on arbitrary Python objects. @@ -66,13 +67,15 @@ def copy(x): cls = type(x) - copier = _copy_dispatch.get(cls) - if copier: - return copier(x) + if cls in _copy_atomic_types: + return x + if cls in _copy_builtin_containers: + return cls.copy(x) + if issubclass(cls, type): # treat it as a regular class: - return _copy_immutable(x) + return x copier = getattr(cls, "__copy__", None) if copier is not None: @@ -97,23 +100,12 @@ def copy(x): return _reconstruct(x, None, *rv) -_copy_dispatch = d = {} - -def _copy_immutable(x): - return x -for t in (types.NoneType, int, float, bool, complex, str, tuple, +_copy_atomic_types = {types.NoneType, int, float, bool, complex, str, tuple, bytes, frozenset, type, range, slice, property, types.BuiltinFunctionType, types.EllipsisType, types.NotImplementedType, types.FunctionType, types.CodeType, - weakref.ref): - d[t] = _copy_immutable - -d[list] = list.copy -d[dict] = dict.copy -d[set] = set.copy -d[bytearray] = bytearray.copy - -del d, t + weakref.ref, super} +_copy_builtin_containers = {list, dict, set, bytearray} def deepcopy(x, memo=None, _nil=[]): """Deep copy operation on arbitrary Python objects. @@ -121,6 +113,11 @@ def deepcopy(x, memo=None, _nil=[]): See the module's __doc__ string for more info. """ + cls = type(x) + + if cls in _atomic_types: + return x + d = id(x) if memo is None: memo = {} @@ -129,14 +126,12 @@ def deepcopy(x, memo=None, _nil=[]): if y is not _nil: return y - cls = type(x) - copier = _deepcopy_dispatch.get(cls) if copier is not None: y = copier(x, memo) else: if issubclass(cls, type): - y = _deepcopy_atomic(x, memo) + y = x # atomic copy else: copier = getattr(x, "__deepcopy__", None) if copier is not None: @@ -167,26 +162,12 @@ def deepcopy(x, memo=None, _nil=[]): _keep_alive(x, memo) # Make sure x lives at least as long as d return y +_atomic_types = {types.NoneType, types.EllipsisType, types.NotImplementedType, + int, float, bool, complex, bytes, str, types.CodeType, type, range, + types.BuiltinFunctionType, types.FunctionType, weakref.ref, property} + _deepcopy_dispatch = d = {} -def _deepcopy_atomic(x, memo): - return x -d[types.NoneType] = _deepcopy_atomic -d[types.EllipsisType] = _deepcopy_atomic -d[types.NotImplementedType] = _deepcopy_atomic -d[int] = _deepcopy_atomic -d[float] = _deepcopy_atomic -d[bool] = _deepcopy_atomic -d[complex] = _deepcopy_atomic -d[bytes] = _deepcopy_atomic -d[str] = _deepcopy_atomic -d[types.CodeType] = _deepcopy_atomic -d[type] = _deepcopy_atomic -d[range] = _deepcopy_atomic -d[types.BuiltinFunctionType] = _deepcopy_atomic -d[types.FunctionType] = _deepcopy_atomic -d[weakref.ref] = _deepcopy_atomic -d[property] = _deepcopy_atomic def _deepcopy_list(x, memo, deepcopy=deepcopy): y = [] diff --git a/Lib/copyreg.py b/Lib/copyreg.py index 578392409b403c..17c5dde67c887c 100644 --- a/Lib/copyreg.py +++ b/Lib/copyreg.py @@ -36,6 +36,11 @@ def pickle_union(obj): pickle(type(int | str), pickle_union) +def pickle_super(obj): + return super, (obj.__thisclass__, obj.__self__) + +pickle(super, pickle_super) + # Support for pickling new-style objects def _reconstructor(cls, base, state): diff --git a/Lib/csv.py b/Lib/csv.py index 75e35b23236795..0a627ba7a512fa 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -47,7 +47,7 @@ class excel: field contains either the quotechar or the delimiter csv.QUOTE_ALL means that quotes are always placed around fields. csv.QUOTE_NONNUMERIC means that quotes are always placed around - fields which do not parse as integers or floating point + fields which do not parse as integers or floating-point numbers. csv.QUOTE_STRINGS means that quotes are always placed around fields which are strings. Note that the Python value None @@ -63,7 +63,6 @@ class excel: written as two quotes """ -import re import types from _csv import Error, writer, reader, register_dialect, \ unregister_dialect, get_dialect, list_dialects, \ @@ -281,6 +280,7 @@ def _guess_quote_and_delimiter(self, data, delimiters): If there is no quotechar the delimiter can't be determined this way. """ + import re matches = [] for restr in (r'(?P[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?P=delim)', # ,".*?", diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index d54ee05b15f5bd..8e2a2926f7a853 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -19,7 +19,7 @@ raise Exception("Version number mismatch", __version__, _ctypes_version) if _os.name == "nt": - from _ctypes import FormatError + from _ctypes import COMError, CopyComPointer, FormatError DEFAULT_MODE = RTLD_LOCAL if _os.name == "posix" and _sys.platform == "darwin": @@ -205,6 +205,18 @@ class c_longdouble(_SimpleCData): if sizeof(c_longdouble) == sizeof(c_double): c_longdouble = c_double +try: + class c_double_complex(_SimpleCData): + _type_ = "C" + _check_size(c_double_complex) + class c_float_complex(_SimpleCData): + _type_ = "E" + _check_size(c_float_complex) + class c_longdouble_complex(_SimpleCData): + _type_ = "F" +except AttributeError: + pass + if _calcsize("l") == _calcsize("q"): # if long and long long have the same size, make c_longlong an alias for c_long c_longlong = c_long @@ -314,8 +326,6 @@ def SetPointerType(pointer, cls): del _pointer_type_cache[id(pointer)] def ARRAY(typ, len): - import warnings - warnings._deprecated("ctypes.ARRAY", remove=(3, 15)) return typ * len ################################################################ @@ -348,6 +358,17 @@ def __init__(self, name, mode=DEFAULT_MODE, handle=None, winmode=None): if name: name = _os.fspath(name) + + # If the filename that has been provided is an iOS/tvOS/watchOS + # .fwork file, dereference the location to the true origin of the + # binary. + if name.endswith(".fwork"): + with open(name) as f: + name = _os.path.join( + _os.path.dirname(_sys.executable), + f.read().strip() + ) + self._name = name flags = self._func_flags_ if use_errno: @@ -468,7 +489,7 @@ def LoadLibrary(self, name): if _os.name == "nt": pythonapi = PyDLL("python dll", None, _sys.dllhandle) -elif hasattr(_sys, "getandroidapilevel"): +elif _sys.platform == "android": pythonapi = PyDLL("libpython%d.%d.so" % _sys.version_info[:2]) elif _sys.platform == "cygwin": pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2]) @@ -503,6 +524,7 @@ def WinError(code=None, descr=None): # functions from _ctypes import _memmove_addr, _memset_addr, _string_at_addr, _cast_addr +from _ctypes import _memoryview_at_addr ## void *memmove(void *, const void *, size_t); memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr) @@ -523,11 +545,19 @@ def cast(obj, typ): _string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr) def string_at(ptr, size=-1): - """string_at(addr[, size]) -> string + """string_at(ptr[, size]) -> string - Return the string at addr.""" + Return the byte string at void *ptr.""" return _string_at(ptr, size) +_memoryview_at = PYFUNCTYPE( + py_object, c_void_p, c_ssize_t, c_int)(_memoryview_at_addr) +def memoryview_at(ptr, size, readonly=False): + """memoryview_at(ptr, size[, readonly]) -> memoryview + + Return a memoryview representing the memory at void *ptr.""" + return _memoryview_at(ptr, size, bool(readonly)) + try: from _ctypes import _wstring_at_addr except ImportError: @@ -535,9 +565,9 @@ def string_at(ptr, size=-1): else: _wstring_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr) def wstring_at(ptr, size=-1): - """wstring_at(addr[, size]) -> string + """wstring_at(ptr[, size]) -> string - Return the string at addr.""" + Return the wide-character string at void *ptr.""" return _wstring_at(ptr, size) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py new file mode 100644 index 00000000000000..e30db598ab22e1 --- /dev/null +++ b/Lib/ctypes/_layout.py @@ -0,0 +1,337 @@ +"""Python implementation of computing the layout of a struct/union + +This code is internal and tightly coupled to the C part. The interface +may change at any time. +""" + +import sys +import warnings +import struct + +from _ctypes import CField, buffer_info +import ctypes + +def round_down(n, multiple): + assert n >= 0 + assert multiple > 0 + return (n // multiple) * multiple + +def round_up(n, multiple): + assert n >= 0 + assert multiple > 0 + return ((n + multiple - 1) // multiple) * multiple + +def LOW_BIT(offset): + return offset & 0xFFFF + +def NUM_BITS(bitsize): + return bitsize >> 16 + +def BUILD_SIZE(bitsize, offset): + assert 0 <= offset, offset + assert offset <= 0xFFFF, offset + # We don't support zero length bitfields. + # And GET_BITFIELD uses NUM_BITS(size) == 0, + # to figure out whether we are handling a bitfield. + assert bitsize > 0, bitsize + result = (bitsize << 16) + offset + assert bitsize == NUM_BITS(result), (bitsize, result) + assert offset == LOW_BIT(result), (offset, result) + return result + +def build_size(bit_size, bit_offset, big_endian, type_size): + if big_endian: + return BUILD_SIZE(bit_size, 8 * type_size - bit_offset - bit_size) + return BUILD_SIZE(bit_size, bit_offset) + +_INT_MAX = (1 << (ctypes.sizeof(ctypes.c_int) * 8) - 1) - 1 + + +class StructUnionLayout: + def __init__(self, fields, size, align, format_spec): + # sequence of CField objects + self.fields = fields + + # total size of the aggregate (rounded up to alignment) + self.size = size + + # total alignment requirement of the aggregate + self.align = align + + # buffer format specification (as a string, UTF-8 but bes + # kept ASCII-only) + self.format_spec = format_spec + + +def get_layout(cls, input_fields, is_struct, base): + """Return a StructUnionLayout for the given class. + + Called by PyCStructUnionType_update_stginfo when _fields_ is assigned + to a class. + """ + # Currently there are two modes, selectable using the '_layout_' attribute: + # + # 'gcc-sysv' mode places fields one after another, bit by bit. + # But "each bit field must fit within a single object of its specified + # type" (GCC manual, section 15.8 "Bit Field Packing"). When it doesn't, + # we insert a few bits of padding to avoid that. + # + # 'ms' mode works similar except for bitfield packing. Adjacent + # bit-fields are packed into the same 1-, 2-, or 4-byte allocation unit + # if the integral types are the same size and if the next bit-field fits + # into the current allocation unit without crossing the boundary imposed + # by the common alignment requirements of the bit-fields. + # + # See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html#index-mms-bitfields + # for details. + + # We do not support zero length bitfields (we use bitsize != 0 + # elsewhere to indicate a bitfield). Here, non-bitfields have bit_size + # set to size*8. + + # For clarity, variables that count bits have `bit` in their names. + + layout = getattr(cls, '_layout_', None) + if layout is None: + if sys.platform == 'win32' or getattr(cls, '_pack_', None): + gcc_layout = False + else: + gcc_layout = True + elif layout == 'ms': + gcc_layout = False + elif layout == 'gcc-sysv': + gcc_layout = True + else: + raise ValueError(f'unknown _layout_: {layout!r}') + + align = getattr(cls, '_align_', 1) + if align < 0: + raise ValueError('_align_ must be a non-negative integer') + elif align == 0: + # Setting `_align_ = 0` amounts to using the default alignment + align == 1 + + if base: + align = max(ctypes.alignment(base), align) + + swapped_bytes = hasattr(cls, '_swappedbytes_') + if swapped_bytes: + big_endian = sys.byteorder == 'little' + else: + big_endian = sys.byteorder == 'big' + + pack = getattr(cls, '_pack_', None) + if pack is not None: + try: + pack = int(pack) + except (TypeError, ValueError): + raise ValueError("_pack_ must be an integer") + if pack < 0: + raise ValueError("_pack_ must be a non-negative integer") + if pack > _INT_MAX: + raise ValueError("_pack_ too big") + if gcc_layout: + raise ValueError('_pack_ is not compatible with gcc-sysv layout') + + result_fields = [] + + if is_struct: + format_spec_parts = ["T{"] + else: + format_spec_parts = ["B"] + + last_field_bit_size = 0 # used in MS layout only + + # `8 * next_byte_offset + next_bit_offset` points to where the + # next field would start. + next_bit_offset = 0 + next_byte_offset = 0 + + # size if this was a struct (sum of field sizes, plus padding) + struct_size = 0 + # max of field sizes; only meaningful for unions + union_size = 0 + + if base: + struct_size = ctypes.sizeof(base) + if gcc_layout: + next_bit_offset = struct_size * 8 + else: + next_byte_offset = struct_size + + last_size = struct_size + for i, field in enumerate(input_fields): + if not is_struct: + # Unions start fresh each time + last_field_bit_size = 0 + next_bit_offset = 0 + next_byte_offset = 0 + + # Unpack the field + field = tuple(field) + try: + name, ctype = field + except (ValueError, TypeError): + try: + name, ctype, bit_size = field + except (ValueError, TypeError) as exc: + raise ValueError( + '_fields_ must be a sequence of (name, C type) pairs ' + + 'or (name, C type, bit size) triples') from exc + is_bitfield = True + if bit_size <= 0: + raise ValueError( + f'number of bits invalid for bit field {name!r}') + type_size = ctypes.sizeof(ctype) + if bit_size > type_size * 8: + raise ValueError( + f'number of bits invalid for bit field {name!r}') + else: + is_bitfield = False + type_size = ctypes.sizeof(ctype) + bit_size = type_size * 8 + + type_bit_size = type_size * 8 + type_align = ctypes.alignment(ctype) or 1 + type_bit_align = type_align * 8 + + if gcc_layout: + # We don't use next_byte_offset here + assert pack is None + assert next_byte_offset == 0 + + # Determine whether the bit field, if placed at the next + # free bit, fits within a single object of its specified type. + # That is: determine a "slot", sized & aligned for the + # specified type, which contains the bitfield's beginning: + slot_start_bit = round_down(next_bit_offset, type_bit_align) + slot_end_bit = slot_start_bit + type_bit_size + # And see if it also contains the bitfield's last bit: + field_end_bit = next_bit_offset + bit_size + if field_end_bit > slot_end_bit: + # It doesn't: add padding (bump up to the next + # alignment boundary) + next_bit_offset = round_up(next_bit_offset, type_bit_align) + + offset = round_down(next_bit_offset, type_bit_align) // 8 + if is_bitfield: + effective_bit_offset = next_bit_offset - 8 * offset + size = build_size(bit_size, effective_bit_offset, + big_endian, type_size) + assert effective_bit_offset <= type_bit_size + else: + assert offset == next_bit_offset / 8 + size = type_size + + next_bit_offset += bit_size + struct_size = round_up(next_bit_offset, 8) // 8 + else: + if pack: + type_align = min(pack, type_align) + + # next_byte_offset points to end of current bitfield. + # next_bit_offset is generally non-positive, + # and 8 * next_byte_offset + next_bit_offset points just behind + # the end of the last field we placed. + if ( + (0 < next_bit_offset + bit_size) + or (type_bit_size != last_field_bit_size) + ): + # Close the previous bitfield (if any) + # and start a new bitfield + next_byte_offset = round_up(next_byte_offset, type_align) + + next_byte_offset += type_size + + last_field_bit_size = type_bit_size + # Reminder: 8 * (next_byte_offset) + next_bit_offset + # points to where we would start a new field, namely + # just behind where we placed the last field plus an + # allowance for alignment. + next_bit_offset = -last_field_bit_size + + assert type_bit_size == last_field_bit_size + + offset = next_byte_offset - last_field_bit_size // 8 + if is_bitfield: + assert 0 <= (last_field_bit_size + next_bit_offset) + size = build_size(bit_size, + last_field_bit_size + next_bit_offset, + big_endian, type_size) + else: + size = type_size + if type_bit_size: + assert (last_field_bit_size + next_bit_offset) < type_bit_size + + next_bit_offset += bit_size + struct_size = next_byte_offset + + assert (not is_bitfield) or (LOW_BIT(size) <= size * 8) + + # Add the format spec parts + if is_struct: + padding = offset - last_size + format_spec_parts.append(padding_spec(padding)) + + fieldfmt, bf_ndim, bf_shape = buffer_info(ctype) + + if bf_shape: + format_spec_parts.extend(( + "(", + ','.join(str(n) for n in bf_shape), + ")", + )) + + if fieldfmt is None: + fieldfmt = "B" + if isinstance(name, bytes): + # a bytes name would be rejected later, but we check early + # to avoid a BytesWarning with `python -bb` + raise TypeError( + "field {name!r}: name must be a string, not bytes") + format_spec_parts.append(f"{fieldfmt}:{name}:") + + result_fields.append(CField( + name=name, + type=ctype, + size=size, + offset=offset, + bit_size=bit_size if is_bitfield else None, + index=i, + )) + if is_bitfield and not gcc_layout: + assert type_bit_size > 0 + + align = max(align, type_align) + last_size = struct_size + if not is_struct: + union_size = max(struct_size, union_size) + + if is_struct: + total_size = struct_size + else: + total_size = union_size + + # Adjust the size according to the alignment requirements + aligned_size = round_up(total_size, align) + + # Finish up the format spec + if is_struct: + padding = aligned_size - total_size + format_spec_parts.append(padding_spec(padding)) + format_spec_parts.append("}") + + return StructUnionLayout( + fields=result_fields, + size=aligned_size, + align=align, + format_spec="".join(format_spec_parts), + ) + + +def padding_spec(padding): + if padding <= 0: + return "" + if padding == 1: + return "x" + return f"{padding}x" diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index c550883e7c7d4b..117bf06cb01013 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -67,7 +67,7 @@ def find_library(name): return fname return None -elif os.name == "posix" and sys.platform == "darwin": +elif os.name == "posix" and sys.platform in {"darwin", "ios", "tvos", "watchos"}: from ctypes.macholib.dyld import dyld_find as _dyld_find def find_library(name): possible = ['lib%s.dylib' % name, @@ -89,6 +89,15 @@ def find_library(name): from ctypes._aix import find_library +elif sys.platform == "android": + def find_library(name): + directory = "/system/lib" + if "64" in os.uname().machine: + directory += "64" + + fname = f"{directory}/lib{name}.so" + return fname if os.path.isfile(fname) else None + elif os.name == "posix": # Andreas Degert's find functions, using gcc, /sbin/ldconfig, objdump import re, tempfile diff --git a/Lib/ctypes/wintypes.py b/Lib/ctypes/wintypes.py index 9c4e721438aad5..4beba0d19513e2 100644 --- a/Lib/ctypes/wintypes.py +++ b/Lib/ctypes/wintypes.py @@ -63,10 +63,16 @@ def __repr__(self): HBITMAP = HANDLE HBRUSH = HANDLE HCOLORSPACE = HANDLE +HCONV = HANDLE +HCONVLIST = HANDLE +HCURSOR = HANDLE HDC = HANDLE +HDDEDATA = HANDLE HDESK = HANDLE +HDROP = HANDLE HDWP = HANDLE HENHMETAFILE = HANDLE +HFILE = INT HFONT = HANDLE HGDIOBJ = HANDLE HGLOBAL = HANDLE @@ -82,9 +88,11 @@ def __repr__(self): HMONITOR = HANDLE HPALETTE = HANDLE HPEN = HANDLE +HRESULT = LONG HRGN = HANDLE HRSRC = HANDLE HSTR = HANDLE +HSZ = HANDLE HTASK = HANDLE HWINSTA = HANDLE HWND = HANDLE diff --git a/Lib/curses/__init__.py b/Lib/curses/__init__.py index 69270bfcd2b205..6165fe6c9875c0 100644 --- a/Lib/curses/__init__.py +++ b/Lib/curses/__init__.py @@ -53,7 +53,7 @@ def start_color(): try: has_key except NameError: - from .has_key import has_key + from .has_key import has_key # noqa: F401 # Wrapper for the entire curses-based application. Runs a function which # should be the rest of your curses-based application. If the application diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 3335821f3f32e9..7a24f8a9e5ccee 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -4,11 +4,10 @@ import types import inspect import keyword -import functools import itertools +import annotationlib import abc -import _thread -from types import FunctionType, GenericAlias +from reprlib import recursive_repr __all__ = ['dataclass', @@ -245,25 +244,6 @@ def __repr__(self): property, }) -# This function's logic is copied from "recursive_repr" function in -# reprlib module to avoid dependency. -def _recursive_repr(user_function): - # Decorator to make a repr function return "..." for a recursive - # call. - repr_running = set() - - @functools.wraps(user_function) - def wrapper(self): - key = id(self), _thread.get_ident() - if key in repr_running: - return '...' - repr_running.add(key) - try: - result = user_function(self) - finally: - repr_running.discard(key) - return result - return wrapper class InitVar: __slots__ = ('type', ) @@ -303,11 +283,12 @@ class Field: 'compare', 'metadata', 'kw_only', + 'doc', '_field_type', # Private: not to be used by user code. ) def __init__(self, default, default_factory, init, repr, hash, compare, - metadata, kw_only): + metadata, kw_only, doc): self.name = None self.type = None self.default = default @@ -320,9 +301,10 @@ def __init__(self, default, default_factory, init, repr, hash, compare, if metadata is None else types.MappingProxyType(metadata)) self.kw_only = kw_only + self.doc = doc self._field_type = None - @_recursive_repr + @recursive_repr() def __repr__(self): return ('Field(' f'name={self.name!r},' @@ -335,6 +317,7 @@ def __repr__(self): f'compare={self.compare!r},' f'metadata={self.metadata!r},' f'kw_only={self.kw_only!r},' + f'doc={self.doc!r},' f'_field_type={self._field_type}' ')') @@ -353,7 +336,7 @@ def __set_name__(self, owner, name): # it. func(self.default, owner, name) - __class_getitem__ = classmethod(GenericAlias) + __class_getitem__ = classmethod(types.GenericAlias) class _DataclassParams: @@ -402,7 +385,7 @@ def __repr__(self): # so that a type checker can be told (via overloads) that this is a # function whose type depends on its parameters. def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, - hash=None, compare=True, metadata=None, kw_only=MISSING): + hash=None, compare=True, metadata=None, kw_only=MISSING, doc=None): """Return an object to identify dataclass fields. default is the default value of the field. default_factory is a @@ -414,7 +397,7 @@ def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, comparison functions. metadata, if specified, must be a mapping which is stored but not otherwise examined by dataclass. If kw_only is true, the field will become a keyword-only parameter to - __init__(). + __init__(). doc is an optional docstring for this field. It is an error to specify both default and default_factory. """ @@ -422,7 +405,7 @@ def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, if default is not MISSING and default_factory is not MISSING: raise ValueError('cannot specify both default and default_factory') return Field(default, default_factory, init, repr, hash, compare, - metadata, kw_only) + metadata, kw_only, doc) def _fields_in_init_order(fields): @@ -446,32 +429,95 @@ def _tuple_str(obj_name, fields): return f'({",".join([f"{obj_name}.{f.name}" for f in fields])},)' -def _create_fn(name, args, body, *, globals=None, locals=None, - return_type=MISSING): - # Note that we may mutate locals. Callers beware! - # The only callers are internal to this module, so no - # worries about external callers. - if locals is None: - locals = {} - return_annotation = '' - if return_type is not MISSING: - locals['__dataclass_return_type__'] = return_type - return_annotation = '->__dataclass_return_type__' - args = ','.join(args) - body = '\n'.join(f' {b}' for b in body) - - # Compute the text of the entire function. - txt = f' def {name}({args}){return_annotation}:\n{body}' - - # Free variables in exec are resolved in the global namespace. - # The global namespace we have is user-provided, so we can't modify it for - # our purposes. So we put the things we need into locals and introduce a - # scope to allow the function we're creating to close over them. - local_vars = ', '.join(locals.keys()) - txt = f"def __create_fn__({local_vars}):\n{txt}\n return {name}" - ns = {} - exec(txt, globals, ns) - return ns['__create_fn__'](**locals) +class _FuncBuilder: + def __init__(self, globals): + self.names = [] + self.src = [] + self.globals = globals + self.locals = {} + self.overwrite_errors = {} + self.unconditional_adds = {} + + def add_fn(self, name, args, body, *, locals=None, return_type=MISSING, + overwrite_error=False, unconditional_add=False, decorator=None): + if locals is not None: + self.locals.update(locals) + + # Keep track if this method is allowed to be overwritten if it already + # exists in the class. The error is method-specific, so keep it with + # the name. We'll use this when we generate all of the functions in + # the add_fns_to_class call. overwrite_error is either True, in which + # case we'll raise an error, or it's a string, in which case we'll + # raise an error and append this string. + if overwrite_error: + self.overwrite_errors[name] = overwrite_error + + # Should this function always overwrite anything that's already in the + # class? The default is to not overwrite a function that already + # exists. + if unconditional_add: + self.unconditional_adds[name] = True + + self.names.append(name) + + if return_type is not MISSING: + self.locals[f'__dataclass_{name}_return_type__'] = return_type + return_annotation = f'->__dataclass_{name}_return_type__' + else: + return_annotation = '' + args = ','.join(args) + body = '\n'.join(body) + + # Compute the text of the entire function, add it to the text we're generating. + self.src.append(f'{f' {decorator}\n' if decorator else ''} def {name}({args}){return_annotation}:\n{body}') + + def add_fns_to_class(self, cls): + # The source to all of the functions we're generating. + fns_src = '\n'.join(self.src) + + # The locals they use. + local_vars = ','.join(self.locals.keys()) + + # The names of all of the functions, used for the return value of the + # outer function. Need to handle the 0-tuple specially. + if len(self.names) == 0: + return_names = '()' + else: + return_names =f'({",".join(self.names)},)' + + # txt is the entire function we're going to execute, including the + # bodies of the functions we're defining. Here's a greatly simplified + # version: + # def __create_fn__(): + # def __init__(self, x, y): + # self.x = x + # self.y = y + # @recursive_repr + # def __repr__(self): + # return f"cls(x={self.x!r},y={self.y!r})" + # return __init__,__repr__ + + txt = f"def __create_fn__({local_vars}):\n{fns_src}\n return {return_names}" + ns = {} + exec(txt, self.globals, ns) + fns = ns['__create_fn__'](**self.locals) + + # Now that we've generated the functions, assign them into cls. + for name, fn in zip(self.names, fns): + fn.__qualname__ = f"{cls.__qualname__}.{fn.__name__}" + if self.unconditional_adds.get(name, False): + setattr(cls, name, fn) + else: + already_exists = _set_new_attribute(cls, name, fn) + + # See if it's an error to overwrite this particular function. + if already_exists and (msg_extra := self.overwrite_errors.get(name)): + error_msg = (f'Cannot overwrite attribute {fn.__name__} ' + f'in class {cls.__name__}') + if not msg_extra is True: + error_msg = f'{error_msg} {msg_extra}' + + raise TypeError(error_msg) def _field_assign(frozen, name, value, self_name): @@ -482,8 +528,8 @@ def _field_assign(frozen, name, value, self_name): # self_name is what "self" is called in this function: don't # hard-code "self", since that might be a field name. if frozen: - return f'__dataclass_builtins_object__.__setattr__({self_name},{name!r},{value})' - return f'{self_name}.{name}={value}' + return f' __dataclass_builtins_object__.__setattr__({self_name},{name!r},{value})' + return f' {self_name}.{name}={value}' def _field_init(f, frozen, globals, self_name, slots): @@ -566,7 +612,7 @@ def _init_param(f): def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, - self_name, globals, slots): + self_name, func_builder, slots): # fields contains both real fields and InitVar pseudo-fields. # Make sure we don't have fields without defaults following fields @@ -585,11 +631,11 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, raise TypeError(f'non-default argument {f.name!r} ' f'follows default argument {seen_default.name!r}') - locals = {f'__dataclass_type_{f.name}__': f.type for f in fields} - locals.update({ - '__dataclass_HAS_DEFAULT_FACTORY__': _HAS_DEFAULT_FACTORY, - '__dataclass_builtins_object__': object, - }) + locals = {**{f'__dataclass_type_{f.name}__': f.type for f in fields}, + **{'__dataclass_HAS_DEFAULT_FACTORY__': _HAS_DEFAULT_FACTORY, + '__dataclass_builtins_object__': object, + } + } body_lines = [] for f in fields: @@ -603,89 +649,52 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, if has_post_init: params_str = ','.join(f.name for f in fields if f._field_type is _FIELD_INITVAR) - body_lines.append(f'{self_name}.{_POST_INIT_NAME}({params_str})') + body_lines.append(f' {self_name}.{_POST_INIT_NAME}({params_str})') # If no body lines, use 'pass'. if not body_lines: - body_lines = ['pass'] + body_lines = [' pass'] _init_params = [_init_param(f) for f in std_fields] if kw_only_fields: # Add the keyword-only args. Because the * can only be added if # there's at least one keyword-only arg, there needs to be a test here - # (instead of just concatenting the lists together). + # (instead of just concatenating the lists together). _init_params += ['*'] _init_params += [_init_param(f) for f in kw_only_fields] - return _create_fn('__init__', - [self_name] + _init_params, - body_lines, - locals=locals, - globals=globals, - return_type=None) - - -def _repr_fn(fields, globals): - fn = _create_fn('__repr__', - ('self',), - ['return f"{self.__class__.__qualname__}(' + - ', '.join([f"{f.name}={{self.{f.name}!r}}" - for f in fields]) + - ')"'], - globals=globals) - return _recursive_repr(fn) - - -def _frozen_get_del_attr(cls, fields, globals): + func_builder.add_fn('__init__', + [self_name] + _init_params, + body_lines, + locals=locals, + return_type=None) + + +def _frozen_get_del_attr(cls, fields, func_builder): locals = {'cls': cls, 'FrozenInstanceError': FrozenInstanceError} condition = 'type(self) is cls' if fields: condition += ' or name in {' + ', '.join(repr(f.name) for f in fields) + '}' - return (_create_fn('__setattr__', - ('self', 'name', 'value'), - (f'if {condition}:', - ' raise FrozenInstanceError(f"cannot assign to field {name!r}")', - f'super(cls, self).__setattr__(name, value)'), - locals=locals, - globals=globals), - _create_fn('__delattr__', - ('self', 'name'), - (f'if {condition}:', - ' raise FrozenInstanceError(f"cannot delete field {name!r}")', - f'super(cls, self).__delattr__(name)'), - locals=locals, - globals=globals), - ) - -def _cmp_fn(name, op, self_tuple, other_tuple, globals): - # Create a comparison function. If the fields in the object are - # named 'x' and 'y', then self_tuple is the string - # '(self.x,self.y)' and other_tuple is the string - # '(other.x,other.y)'. - - return _create_fn(name, - ('self', 'other'), - [ 'if other.__class__ is self.__class__:', - f' return {self_tuple}{op}{other_tuple}', - 'return NotImplemented'], - globals=globals) - - -def _hash_fn(fields, globals): - self_tuple = _tuple_str('self', fields) - return _create_fn('__hash__', - ('self',), - [f'return hash({self_tuple})'], - globals=globals) + func_builder.add_fn('__setattr__', + ('self', 'name', 'value'), + (f' if {condition}:', + ' raise FrozenInstanceError(f"cannot assign to field {name!r}")', + f' super(cls, self).__setattr__(name, value)'), + locals=locals, + overwrite_error=True) + func_builder.add_fn('__delattr__', + ('self', 'name'), + (f' if {condition}:', + ' raise FrozenInstanceError(f"cannot delete field {name!r}")', + f' super(cls, self).__delattr__(name)'), + locals=locals, + overwrite_error=True) def _is_classvar(a_type, typing): - # This test uses a typing internal class, but it's the best way to - # test if this is a ClassVar. return (a_type is typing.ClassVar - or (type(a_type) is typing._GenericAlias - and a_type.__origin__ is typing.ClassVar)) + or (typing.get_origin(a_type) is typing.ClassVar)) def _is_initvar(a_type, dataclasses): @@ -854,19 +863,11 @@ def _get_field(cls, a_name, a_type, default_kw_only): return f -def _set_qualname(cls, value): - # Ensure that the functions returned from _create_fn uses the proper - # __qualname__ (the class they belong to). - if isinstance(value, FunctionType): - value.__qualname__ = f"{cls.__qualname__}.{value.__name__}" - return value - def _set_new_attribute(cls, name, value): # Never overwrites an existing attribute. Returns True if the # attribute already exists. if name in cls.__dict__: return True - _set_qualname(cls, value) setattr(cls, name, value) return False @@ -876,14 +877,22 @@ def _set_new_attribute(cls, name, value): # take. The common case is to do nothing, so instead of providing a # function that is a no-op, use None to signify that. -def _hash_set_none(cls, fields, globals): - return None +def _hash_set_none(cls, fields, func_builder): + # It's sort of a hack that I'm setting this here, instead of at + # func_builder.add_fns_to_class time, but since this is an exceptional case + # (it's not setting an attribute to a function, but to a scalar value), + # just do it directly here. I might come to regret this. + cls.__hash__ = None -def _hash_add(cls, fields, globals): +def _hash_add(cls, fields, func_builder): flds = [f for f in fields if (f.compare if f.hash is None else f.hash)] - return _set_qualname(cls, _hash_fn(flds, globals)) + self_tuple = _tuple_str('self', flds) + func_builder.add_fn('__hash__', + ('self',), + [f' return hash({self_tuple})'], + unconditional_add=True) -def _hash_exception(cls, fields, globals): +def _hash_exception(cls, fields, func_builder): # Raise an exception. raise TypeError(f'Cannot overwrite attribute __hash__ ' f'in class {cls.__name__}') @@ -973,7 +982,8 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, # actual default value. Pseudo-fields ClassVars and InitVars are # included, despite the fact that they're not real fields. That's # dealt with later. - cls_annotations = inspect.get_annotations(cls) + cls_annotations = annotationlib.get_annotations( + cls, format=annotationlib.Format.FORWARDREF) # Now find fields in our class. While doing so, validate some # things, and set the default values (as class attributes) where @@ -1061,24 +1071,26 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, (std_init_fields, kw_only_init_fields) = _fields_in_init_order(all_init_fields) + func_builder = _FuncBuilder(globals) + if init: # Does this class have a post-init function? has_post_init = hasattr(cls, _POST_INIT_NAME) - _set_new_attribute(cls, '__init__', - _init_fn(all_init_fields, - std_init_fields, - kw_only_init_fields, - frozen, - has_post_init, - # The name to use for the "self" - # param in __init__. Use "self" - # if possible. - '__dataclass_self__' if 'self' in fields - else 'self', - globals, - slots, - )) + _init_fn(all_init_fields, + std_init_fields, + kw_only_init_fields, + frozen, + has_post_init, + # The name to use for the "self" + # param in __init__. Use "self" + # if possible. + '__dataclass_self__' if 'self' in fields + else 'self', + func_builder, + slots, + ) + _set_new_attribute(cls, '__replace__', _replace) # Get the fields as a list, and include only real fields. This is @@ -1087,7 +1099,13 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, if repr: flds = [f for f in field_list if f.repr] - _set_new_attribute(cls, '__repr__', _repr_fn(flds, globals)) + func_builder.add_fn('__repr__', + ('self',), + [' return f"{self.__class__.__qualname__}(' + + ', '.join([f"{f.name}={{self.{f.name}!r}}" + for f in flds]) + ')"'], + locals={'__dataclasses_recursive_repr': recursive_repr}, + decorator="@__dataclasses_recursive_repr()") if eq: # Create __eq__ method. There's no need for a __ne__ method, @@ -1095,14 +1113,13 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, cmp_fields = (field for field in field_list if field.compare) terms = [f'self.{field.name}==other.{field.name}' for field in cmp_fields] field_comparisons = ' and '.join(terms) or 'True' - body = [f'if other.__class__ is self.__class__:', - f' return {field_comparisons}', - f'return NotImplemented'] - func = _create_fn('__eq__', - ('self', 'other'), - body, - globals=globals) - _set_new_attribute(cls, '__eq__', func) + func_builder.add_fn('__eq__', + ('self', 'other'), + [ ' if self is other:', + ' return True', + ' if other.__class__ is self.__class__:', + f' return {field_comparisons}', + ' return NotImplemented']) if order: # Create and set the ordering methods. @@ -1114,18 +1131,19 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, ('__gt__', '>'), ('__ge__', '>='), ]: - if _set_new_attribute(cls, name, - _cmp_fn(name, op, self_tuple, other_tuple, - globals=globals)): - raise TypeError(f'Cannot overwrite attribute {name} ' - f'in class {cls.__name__}. Consider using ' - 'functools.total_ordering') + # Create a comparison function. If the fields in the object are + # named 'x' and 'y', then self_tuple is the string + # '(self.x,self.y)' and other_tuple is the string + # '(other.x,other.y)'. + func_builder.add_fn(name, + ('self', 'other'), + [ ' if other.__class__ is self.__class__:', + f' return {self_tuple}{op}{other_tuple}', + ' return NotImplemented'], + overwrite_error='Consider using functools.total_ordering') if frozen: - for fn in _frozen_get_del_attr(cls, field_list, globals): - if _set_new_attribute(cls, fn.__name__, fn): - raise TypeError(f'Cannot overwrite attribute {fn.__name__} ' - f'in class {cls.__name__}') + _frozen_get_del_attr(cls, field_list, func_builder) # Decide if/how we're going to create a hash function. hash_action = _hash_action[bool(unsafe_hash), @@ -1133,9 +1151,12 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, bool(frozen), has_explicit_hash] if hash_action: - # No need to call _set_new_attribute here, since by the time - # we're here the overwriting is unconditional. - cls.__hash__ = hash_action(cls, field_list, globals) + cls.__hash__ = hash_action(cls, field_list, func_builder) + + # Generate the methods and add them to the class. This needs to be done + # before the __doc__ logic below, since inspect will look at the __init__ + # signature. + func_builder.add_fns_to_class(cls) if not getattr(cls, '__doc__'): # Create a class doc-string. @@ -1148,7 +1169,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, cls.__doc__ = (cls.__name__ + text_sig) if match_args: - # I could probably compute this once + # I could probably compute this once. _set_new_attribute(cls, '__match_args__', tuple(f.name for f in std_init_fields)) @@ -1156,7 +1177,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, if weakref_slot and not slots: raise TypeError('weakref_slot is True but slots is False') if slots: - cls = _add_slots(cls, frozen, weakref_slot) + cls = _add_slots(cls, frozen, weakref_slot, fields) abc.update_abstractmethods(cls) @@ -1179,8 +1200,17 @@ def _dataclass_setstate(self, state): def _get_slots(cls): match cls.__dict__.get('__slots__'): + # `__dictoffset__` and `__weakrefoffset__` can tell us whether + # the base type has dict/weakref slots, in a way that works correctly + # for both Python classes and C extension types. Extension types + # don't use `__slots__` for slot creation case None: - return + slots = [] + if getattr(cls, '__weakrefoffset__', -1) != 0: + slots.append('__weakref__') + if getattr(cls, '__dictoffset__', -1) != 0: + slots.append('__dict__') + yield from slots case str(slot): yield slot # Slots may be any iterable, but we cannot handle an iterator @@ -1191,9 +1221,56 @@ def _get_slots(cls): raise TypeError(f"Slots of '{cls.__name__}' cannot be determined") -def _add_slots(cls, is_frozen, weakref_slot): - # Need to create a new class, since we can't set __slots__ - # after a class has been created. +def _update_func_cell_for__class__(f, oldcls, newcls): + # Returns True if we update a cell, else False. + if f is None: + # f will be None in the case of a property where not all of + # fget, fset, and fdel are used. Nothing to do in that case. + return False + try: + idx = f.__code__.co_freevars.index("__class__") + except ValueError: + # This function doesn't reference __class__, so nothing to do. + return False + # Fix the cell to point to the new class, if it's already pointing + # at the old class. I'm not convinced that the "is oldcls" test + # is needed, but other than performance can't hurt. + closure = f.__closure__[idx] + if closure.cell_contents is oldcls: + closure.cell_contents = newcls + return True + return False + + +def _create_slots(defined_fields, inherited_slots, field_names, weakref_slot): + # The slots for our class. Remove slots from our base classes. Add + # '__weakref__' if weakref_slot was given, unless it is already present. + seen_docs = False + slots = {} + for slot in itertools.filterfalse( + inherited_slots.__contains__, + itertools.chain( + # gh-93521: '__weakref__' also needs to be filtered out if + # already present in inherited_slots + field_names, ('__weakref__',) if weakref_slot else () + ) + ): + doc = getattr(defined_fields.get(slot), 'doc', None) + if doc is not None: + seen_docs = True + slots.update({slot: doc}) + + # We only return dict if there's at least one doc member, + # otherwise we return tuple, which is the old default format. + if seen_docs: + return slots + return tuple(slots) + + +def _add_slots(cls, is_frozen, weakref_slot, defined_fields): + # Need to create a new class, since we can't set __slots__ after a + # class has been created, and the @dataclass decorator is called + # after the class is created. # Make sure __slots__ isn't already set. if '__slots__' in cls.__dict__: @@ -1206,17 +1283,9 @@ def _add_slots(cls, is_frozen, weakref_slot): inherited_slots = set( itertools.chain.from_iterable(map(_get_slots, cls.__mro__[1:-1])) ) - # The slots for our class. Remove slots from our base classes. Add - # '__weakref__' if weakref_slot was given, unless it is already present. - cls_dict["__slots__"] = tuple( - itertools.filterfalse( - inherited_slots.__contains__, - itertools.chain( - # gh-93521: '__weakref__' also needs to be filtered out if - # already present in inherited_slots - field_names, ('__weakref__',) if weakref_slot else () - ) - ), + + cls_dict["__slots__"] = _create_slots( + defined_fields, inherited_slots, field_names, weakref_slot, ) for field_name in field_names: @@ -1232,18 +1301,37 @@ def _add_slots(cls, is_frozen, weakref_slot): # And finally create the class. qualname = getattr(cls, '__qualname__', None) - cls = type(cls)(cls.__name__, cls.__bases__, cls_dict) + newcls = type(cls)(cls.__name__, cls.__bases__, cls_dict) if qualname is not None: - cls.__qualname__ = qualname + newcls.__qualname__ = qualname if is_frozen: # Need this for pickling frozen classes with slots. if '__getstate__' not in cls_dict: - cls.__getstate__ = _dataclass_getstate + newcls.__getstate__ = _dataclass_getstate if '__setstate__' not in cls_dict: - cls.__setstate__ = _dataclass_setstate - - return cls + newcls.__setstate__ = _dataclass_setstate + + # Fix up any closures which reference __class__. This is used to + # fix zero argument super so that it points to the correct class + # (the newly created one, which we're returning) and not the + # original class. We can break out of this loop as soon as we + # make an update, since all closures for a class will share a + # given cell. + for member in newcls.__dict__.values(): + # If this is a wrapped function, unwrap it. + member = inspect.unwrap(member) + + if isinstance(member, types.FunctionType): + if _update_func_cell_for__class__(member, cls, newcls): + break + elif isinstance(member, property): + if (_update_func_cell_for__class__(member.fget, cls, newcls) + or _update_func_cell_for__class__(member.fset, cls, newcls) + or _update_func_cell_for__class__(member.fdel, cls, newcls)): + break + + return newcls def dataclass(cls=None, /, *, init=True, repr=True, eq=True, order=False, @@ -1462,7 +1550,7 @@ def _astuple_inner(obj, tuple_factory): def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, - weakref_slot=False, module=None): + weakref_slot=False, module=None, decorator=dataclass): """Return a new dynamically created dataclass. The dataclass name will be 'cls_name'. 'fields' is an iterable @@ -1542,8 +1630,8 @@ def exec_body_callback(ns): if module is not None: cls.__module__ = module - # Apply the normal decorator. - return dataclass(cls, init=init, repr=repr, eq=eq, order=order, + # Apply the normal provided decorator. + return decorator(cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen, match_args=match_args, kw_only=kw_only, slots=slots, weakref_slot=weakref_slot) diff --git a/Lib/datetime.py b/Lib/datetime.py index a33d2d724cb33d..14f30556584e32 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1,9 +1,13 @@ +"""Specific date/time and related types. + +See https://data.iana.org/time-zones/tz-link.html for +time zone and DST data sources. +""" + try: from _datetime import * - from _datetime import __doc__ except ImportError: from _pydatetime import * - from _pydatetime import __doc__ __all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo", "MINYEAR", "MAXYEAR", "UTC") diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index 74c9d9b7e2f1d8..7e0ae2a29e3a64 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -1,6 +1,5 @@ import os import sqlite3 -import sys from pathlib import Path from contextlib import suppress, closing from collections.abc import MutableMapping diff --git a/Lib/decimal.py b/Lib/decimal.py index 7746ea2601024c..530bdfb38953d9 100644 --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -1,11 +1,109 @@ +"""Decimal fixed-point and floating-point arithmetic. + +This is an implementation of decimal floating-point arithmetic based on +the General Decimal Arithmetic Specification: + + http://speleotrove.com/decimal/decarith.html + +and IEEE standard 854-1987: + + http://en.wikipedia.org/wiki/IEEE_854-1987 + +Decimal floating point has finite precision with arbitrarily large bounds. + +The purpose of this module is to support arithmetic using familiar +"schoolhouse" rules and to avoid some of the tricky representation +issues associated with binary floating point. The package is especially +useful for financial applications or for contexts where users have +expectations that are at odds with binary floating point (for instance, +in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead +of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected +Decimal('0.00')). + +Here are some examples of using the decimal module: + +>>> from decimal import * +>>> setcontext(ExtendedContext) +>>> Decimal(0) +Decimal('0') +>>> Decimal('1') +Decimal('1') +>>> Decimal('-.0123') +Decimal('-0.0123') +>>> Decimal(123456) +Decimal('123456') +>>> Decimal('123.45e12345678') +Decimal('1.2345E+12345680') +>>> Decimal('1.33') + Decimal('1.27') +Decimal('2.60') +>>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41') +Decimal('-2.20') +>>> dig = Decimal(1) +>>> print(dig / Decimal(3)) +0.333333333 +>>> getcontext().prec = 18 +>>> print(dig / Decimal(3)) +0.333333333333333333 +>>> print(dig.sqrt()) +1 +>>> print(Decimal(3).sqrt()) +1.73205080756887729 +>>> print(Decimal(3) ** 123) +4.85192780976896427E+58 +>>> inf = Decimal(1) / Decimal(0) +>>> print(inf) +Infinity +>>> neginf = Decimal(-1) / Decimal(0) +>>> print(neginf) +-Infinity +>>> print(neginf + inf) +NaN +>>> print(neginf * inf) +-Infinity +>>> print(dig / 0) +Infinity +>>> getcontext().traps[DivisionByZero] = 1 +>>> print(dig / 0) +Traceback (most recent call last): + ... + ... + ... +decimal.DivisionByZero: x / 0 +>>> c = Context() +>>> c.traps[InvalidOperation] = 0 +>>> print(c.flags[InvalidOperation]) +0 +>>> c.divide(Decimal(0), Decimal(0)) +Decimal('NaN') +>>> c.traps[InvalidOperation] = 1 +>>> print(c.flags[InvalidOperation]) +1 +>>> c.flags[InvalidOperation] = 0 +>>> print(c.flags[InvalidOperation]) +0 +>>> print(c.divide(Decimal(0), Decimal(0))) +Traceback (most recent call last): + ... + ... + ... +decimal.InvalidOperation: 0 / 0 +>>> print(c.flags[InvalidOperation]) +1 +>>> c.flags[InvalidOperation] = 0 +>>> c.traps[InvalidOperation] = 0 +>>> print(c.divide(Decimal(0), Decimal(0))) +NaN +>>> print(c.flags[InvalidOperation]) +1 +>>> +""" try: from _decimal import * - from _decimal import __doc__ - from _decimal import __version__ - from _decimal import __libmpdec_version__ + from _decimal import __version__ # noqa: F401 + from _decimal import __libmpdec_version__ # noqa: F401 except ImportError: - from _pydecimal import * - from _pydecimal import __doc__ - from _pydecimal import __version__ - from _pydecimal import __libmpdec_version__ + import _pydecimal + import sys + _pydecimal.__doc__ = __doc__ + sys.modules[__name__] = _pydecimal diff --git a/Lib/difflib.py b/Lib/difflib.py index ba0b256969ebff..7f595b6c72e641 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -908,87 +908,85 @@ def _fancy_replace(self, a, alo, ahi, b, blo, bhi): + abcdefGhijkl ? ^ ^ ^ """ - - # don't synch up unless the lines have a similarity score of at - # least cutoff; best_ratio tracks the best score seen so far - best_ratio, cutoff = 0.74, 0.75 + # Don't synch up unless the lines have a similarity score above + # cutoff. Previously only the smallest pair was handled here, + # and if there are many pairs with the best ratio, recursion + # could grow very deep, and runtime cubic. See: + # https://github.com/python/cpython/issues/119105 + # + # Later, more pathological cases prompted removing recursion + # entirely. + cutoff = 0.74999 cruncher = SequenceMatcher(self.charjunk) - eqi, eqj = None, None # 1st indices of equal lines (if any) + crqr = cruncher.real_quick_ratio + cqr = cruncher.quick_ratio + cr = cruncher.ratio - # search for the pair that matches best without being identical - # (identical lines must be junk lines, & we don't want to synch up - # on junk -- unless we have to) + WINDOW = 10 + best_i = best_j = None + dump_i, dump_j = alo, blo # smallest indices not yet resolved for j in range(blo, bhi): - bj = b[j] - cruncher.set_seq2(bj) - for i in range(alo, ahi): - ai = a[i] - if ai == bj: - if eqi is None: - eqi, eqj = i, j - continue - cruncher.set_seq1(ai) - # computing similarity is expensive, so use the quick - # upper bounds first -- have seen this speed up messy - # compares by a factor of 3. - # note that ratio() is only expensive to compute the first - # time it's called on a sequence pair; the expensive part - # of the computation is cached by cruncher - if cruncher.real_quick_ratio() > best_ratio and \ - cruncher.quick_ratio() > best_ratio and \ - cruncher.ratio() > best_ratio: - best_ratio, best_i, best_j = cruncher.ratio(), i, j - if best_ratio < cutoff: - # no non-identical "pretty close" pair - if eqi is None: - # no identical pair either -- treat it as a straight replace - yield from self._plain_replace(a, alo, ahi, b, blo, bhi) - return - # no close pair, but an identical pair -- synch up on that - best_i, best_j, best_ratio = eqi, eqj, 1.0 - else: - # there's a close pair, so forget the identical pair (if any) - eqi = None - - # a[best_i] very similar to b[best_j]; eqi is None iff they're not - # identical - - # pump out diffs from before the synch point - yield from self._fancy_helper(a, alo, best_i, b, blo, best_j) - - # do intraline marking on the synch pair - aelt, belt = a[best_i], b[best_j] - if eqi is None: - # pump out a '-', '?', '+', '?' quad for the synched lines - atags = btags = "" - cruncher.set_seqs(aelt, belt) - for tag, ai1, ai2, bj1, bj2 in cruncher.get_opcodes(): - la, lb = ai2 - ai1, bj2 - bj1 - if tag == 'replace': - atags += '^' * la - btags += '^' * lb - elif tag == 'delete': - atags += '-' * la - elif tag == 'insert': - btags += '+' * lb - elif tag == 'equal': - atags += ' ' * la - btags += ' ' * lb - else: - raise ValueError('unknown tag %r' % (tag,)) - yield from self._qformat(aelt, belt, atags, btags) - else: - # the synch pair is identical - yield ' ' + aelt + cruncher.set_seq2(b[j]) + # Search the corresponding i's within WINDOW for rhe highest + # ratio greater than `cutoff`. + aequiv = alo + (j - blo) + arange = range(max(aequiv - WINDOW, dump_i), + min(aequiv + WINDOW + 1, ahi)) + if not arange: # likely exit if `a` is shorter than `b` + break + best_ratio = cutoff + for i in arange: + cruncher.set_seq1(a[i]) + # Ordering by cheapest to most expensive ratio is very + # valuable, most often getting out early. + if (crqr() > best_ratio + and cqr() > best_ratio + and cr() > best_ratio): + best_i, best_j, best_ratio = i, j, cr() + + if best_i is None: + # found nothing to synch on yet - move to next j + continue - # pump out diffs from after the synch point - yield from self._fancy_helper(a, best_i+1, ahi, b, best_j+1, bhi) + # pump out straight replace from before this synch pair + yield from self._fancy_helper(a, dump_i, best_i, + b, dump_j, best_j) + # do intraline marking on the synch pair + aelt, belt = a[best_i], b[best_j] + if aelt != belt: + # pump out a '-', '?', '+', '?' quad for the synched lines + atags = btags = "" + cruncher.set_seqs(aelt, belt) + for tag, ai1, ai2, bj1, bj2 in cruncher.get_opcodes(): + la, lb = ai2 - ai1, bj2 - bj1 + if tag == 'replace': + atags += '^' * la + btags += '^' * lb + elif tag == 'delete': + atags += '-' * la + elif tag == 'insert': + btags += '+' * lb + elif tag == 'equal': + atags += ' ' * la + btags += ' ' * lb + else: + raise ValueError('unknown tag %r' % (tag,)) + yield from self._qformat(aelt, belt, atags, btags) + else: + # the synch pair is identical + yield ' ' + aelt + dump_i, dump_j = best_i + 1, best_j + 1 + best_i = best_j = None + + # pump out straight replace from after the last synch pair + yield from self._fancy_helper(a, dump_i, ahi, + b, dump_j, bhi) def _fancy_helper(self, a, alo, ahi, b, blo, bhi): g = [] if alo < ahi: if blo < bhi: - g = self._fancy_replace(a, alo, ahi, b, blo, bhi) + g = self._plain_replace(a, alo, ahi, b, blo, bhi) else: g = self._dump('-', a, alo, ahi) elif blo < bhi: @@ -1266,6 +1264,12 @@ def _check_types(a, b, *args): if b and not isinstance(b[0], str): raise TypeError('lines to compare must be str, not %s (%r)' % (type(b[0]).__name__, b[0])) + if isinstance(a, str): + raise TypeError('input must be a sequence of strings, not %s' % + type(a).__name__) + if isinstance(b, str): + raise TypeError('input must be a sequence of strings, not %s' % + type(b).__name__) for arg in args: if not isinstance(arg, str): raise TypeError('all arguments must be str, not: %r' % (arg,)) diff --git a/Lib/dis.py b/Lib/dis.py index d146bcbb5097ef..109c986bbe3d7d 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -11,12 +11,16 @@ _cache_format, _inline_cache_entries, _nb_ops, + _common_constants, _intrinsic_1_descs, _intrinsic_2_descs, + _special_method_names, _specializations, _specialized_opmap, ) +from _opcode import get_executor + __all__ = ["code_info", "dis", "disassemble", "distb", "disco", "findlinestarts", "findlabels", "show_code", "get_instructions", "Instruction", "Bytecode"] + _opcodes_all @@ -28,12 +32,11 @@ CONVERT_VALUE = opmap['CONVERT_VALUE'] SET_FUNCTION_ATTRIBUTE = opmap['SET_FUNCTION_ATTRIBUTE'] -FUNCTION_ATTR_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure') +FUNCTION_ATTR_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure', 'annotate') ENTER_EXECUTOR = opmap['ENTER_EXECUTOR'] -LOAD_CONST = opmap['LOAD_CONST'] -RETURN_CONST = opmap['RETURN_CONST'] LOAD_GLOBAL = opmap['LOAD_GLOBAL'] +LOAD_SMALL_INT = opmap['LOAD_SMALL_INT'] BINARY_OP = opmap['BINARY_OP'] JUMP_BACKWARD = opmap['JUMP_BACKWARD'] FOR_ITER = opmap['FOR_ITER'] @@ -42,9 +45,13 @@ LOAD_SUPER_ATTR = opmap['LOAD_SUPER_ATTR'] CALL_INTRINSIC_1 = opmap['CALL_INTRINSIC_1'] CALL_INTRINSIC_2 = opmap['CALL_INTRINSIC_2'] +LOAD_COMMON_CONSTANT = opmap['LOAD_COMMON_CONSTANT'] +LOAD_SPECIAL = opmap['LOAD_SPECIAL'] LOAD_FAST_LOAD_FAST = opmap['LOAD_FAST_LOAD_FAST'] STORE_FAST_LOAD_FAST = opmap['STORE_FAST_LOAD_FAST'] STORE_FAST_STORE_FAST = opmap['STORE_FAST_STORE_FAST'] +IS_OP = opmap['IS_OP'] +CONTAINS_OP = opmap['CONTAINS_OP'] CACHE = opmap["CACHE"] @@ -74,7 +81,7 @@ def _try_compile(source, name): return compile(source, name, 'exec') def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, - show_offsets=False): + show_offsets=False, show_positions=False): """Disassemble classes, methods, functions, and other compiled objects. With no argument, disassemble the last traceback. @@ -85,7 +92,7 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, """ if x is None: distb(file=file, show_caches=show_caches, adaptive=adaptive, - show_offsets=show_offsets) + show_offsets=show_offsets, show_positions=show_positions) return # Extract functions from methods. if hasattr(x, '__func__'): @@ -106,12 +113,12 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, if isinstance(x1, _have_code): print("Disassembly of %s:" % name, file=file) try: - dis(x1, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) + dis(x1, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) except TypeError as msg: print("Sorry:", msg, file=file) print(file=file) elif hasattr(x, 'co_code'): # Code object - _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) + _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) elif isinstance(x, (bytes, bytearray)): # Raw bytecode labels_map = _make_labels_map(x) label_width = 4 + len(str(len(labels_map))) @@ -122,12 +129,12 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, arg_resolver = ArgResolver(labels_map=labels_map) _disassemble_bytes(x, arg_resolver=arg_resolver, formatter=formatter) elif isinstance(x, str): # Source code - _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) + _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) else: raise TypeError("don't know how to disassemble %s objects" % type(x).__name__) -def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets=False): +def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False): """Disassemble a traceback (default: last traceback).""" if tb is None: try: @@ -138,22 +145,24 @@ def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets except AttributeError: raise RuntimeError("no last traceback to disassemble") from None while tb.tb_next: tb = tb.tb_next - disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) + disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) # The inspect module interrogates this dictionary to build its # list of CO_* constants. It is also used by pretty_flags to # turn the co_flags field into a human readable list. COMPILER_FLAG_NAMES = { - 1: "OPTIMIZED", - 2: "NEWLOCALS", - 4: "VARARGS", - 8: "VARKEYWORDS", - 16: "NESTED", - 32: "GENERATOR", - 64: "NOFREE", - 128: "COROUTINE", - 256: "ITERABLE_COROUTINE", - 512: "ASYNC_GENERATOR", + 1: "OPTIMIZED", + 2: "NEWLOCALS", + 4: "VARARGS", + 8: "VARKEYWORDS", + 16: "NESTED", + 32: "GENERATOR", + 64: "NOFREE", + 128: "COROUTINE", + 256: "ITERABLE_COROUTINE", + 512: "ASYNC_GENERATOR", + 0x4000000: "HAS_DOCSTRING", + 0x8000000: "METHOD", } def pretty_flags(flags): @@ -205,7 +214,27 @@ def _deoptop(op): return _all_opmap[deoptmap[name]] if name in deoptmap else op def _get_code_array(co, adaptive): - return co._co_code_adaptive if adaptive else co.co_code + if adaptive: + code = co._co_code_adaptive + res = [] + found = False + for i in range(0, len(code), 2): + op, arg = code[i], code[i+1] + if op == ENTER_EXECUTOR: + try: + ex = get_executor(co, i) + except (ValueError, RuntimeError): + ex = None + + if ex: + op, arg = ex.get_opcode(), ex.get_oparg() + found = True + + res.append(op.to_bytes()) + res.append(arg.to_bytes()) + return code if not found else b''.join(res) + else: + return co.co_code def code_info(x): """Formatted details of methods, functions, or code.""" @@ -401,21 +430,25 @@ def __str__(self): class Formatter: def __init__(self, file=None, lineno_width=0, offset_width=0, label_width=0, - line_offset=0, show_caches=False): + line_offset=0, show_caches=False, *, show_positions=False): """Create a Formatter *file* where to write the output - *lineno_width* sets the width of the line number field (0 omits it) + *lineno_width* sets the width of the source location field (0 omits it). + Should be large enough for a line number or full positions (depending + on the value of *show_positions*). *offset_width* sets the width of the instruction offset field *label_width* sets the width of the label field *show_caches* is a boolean indicating whether to display cache lines - + *show_positions* is a boolean indicating whether full positions should + be reported instead of only the line numbers. """ self.file = file self.lineno_width = lineno_width self.offset_width = offset_width self.label_width = label_width self.show_caches = show_caches + self.show_positions = show_positions def print_instruction(self, instr, mark_as_current=False): self.print_instruction_line(instr, mark_as_current) @@ -448,15 +481,27 @@ def print_instruction_line(self, instr, mark_as_current): print(file=self.file) fields = [] - # Column: Source code line number + # Column: Source code locations information if lineno_width: - if instr.starts_line: - lineno_fmt = "%%%dd" if instr.line_number is not None else "%%%ds" - lineno_fmt = lineno_fmt % lineno_width - lineno = _NO_LINENO if instr.line_number is None else instr.line_number - fields.append(lineno_fmt % lineno) + if self.show_positions: + # reporting positions instead of just line numbers + if instr_positions := instr.positions: + if all(p is None for p in instr_positions): + positions_str = _NO_LINENO + else: + ps = tuple('?' if p is None else p for p in instr_positions) + positions_str = f"{ps[0]}:{ps[2]}-{ps[1]}:{ps[3]}" + fields.append(f'{positions_str:{lineno_width}}') + else: + fields.append(' ' * lineno_width) else: - fields.append(' ' * lineno_width) + if instr.starts_line: + lineno_fmt = "%%%dd" if instr.line_number is not None else "%%%ds" + lineno_fmt = lineno_fmt % lineno_width + lineno = _NO_LINENO if instr.line_number is None else instr.line_number + fields.append(lineno_fmt % lineno) + else: + fields.append(' ' * lineno_width) # Column: Label if instr.label is not None: lbl = f"L{instr.label}:" @@ -514,8 +559,6 @@ def offset_from_jump_arg(self, op, arg, offset): argval = offset + 2 + signed_arg*2 caches = _get_cache_size(_all_opname[deop]) argval += 2 * caches - if deop == ENTER_EXECUTOR: - argval += 2 return argval return None @@ -581,6 +624,18 @@ def get_argval_argrepr(self, op, arg, offset): argrepr = _intrinsic_1_descs[arg] elif deop == CALL_INTRINSIC_2: argrepr = _intrinsic_2_descs[arg] + elif deop == LOAD_COMMON_CONSTANT: + obj = _common_constants[arg] + if isinstance(obj, type): + argrepr = obj.__name__ + else: + argrepr = repr(obj) + elif deop == LOAD_SPECIAL: + argrepr = _special_method_names[arg] + elif deop == IS_OP: + argrepr = 'is not' if argval else 'is' + elif deop == CONTAINS_OP: + argrepr = 'not in' if argval else 'in' return argval, argrepr def get_instructions(x, *, first_line=None, show_caches=None, adaptive=False): @@ -620,8 +675,10 @@ def _get_const_value(op, arg, co_consts): Otherwise (if it is a LOAD_CONST and co_consts is not provided) returns the dis.UNKNOWN sentinel. """ - assert op in hasconst + assert op in hasconst or op == LOAD_SMALL_INT + if op == LOAD_SMALL_INT: + return arg argval = UNKNOWN if co_consts is not None: argval = co_consts[arg] @@ -680,8 +737,7 @@ def _parse_exception_table(code): def _is_backward_jump(op): return opname[op] in ('JUMP_BACKWARD', - 'JUMP_BACKWARD_NO_INTERRUPT', - 'ENTER_EXECUTOR') + 'JUMP_BACKWARD_NO_INTERRUPT') def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=None, original_code=None, arg_resolver=None): @@ -725,8 +781,10 @@ def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=N if caches: cache_info = [] + cache_offset = offset for name, size in _cache_format[opname[deop]].items(): - data = code[offset + 2: offset + 2 + 2 * size] + data = code[cache_offset + 2: cache_offset + 2 + 2 * size] + cache_offset += size * 2 cache_info.append((name, size, data)) else: cache_info = None @@ -738,17 +796,22 @@ def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=N def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False, - show_offsets=False): + show_offsets=False, show_positions=False): """Disassemble a code object.""" linestarts = dict(findlinestarts(co)) exception_entries = _parse_exception_table(co) + if show_positions: + lineno_width = _get_positions_width(co) + else: + lineno_width = _get_lineno_width(linestarts) labels_map = _make_labels_map(co.co_code, exception_entries=exception_entries) label_width = 4 + len(str(len(labels_map))) formatter = Formatter(file=file, - lineno_width=_get_lineno_width(linestarts), + lineno_width=lineno_width, offset_width=len(str(max(len(co.co_code) - 2, 9999))) if show_offsets else 0, label_width=label_width, - show_caches=show_caches) + show_caches=show_caches, + show_positions=show_positions) arg_resolver = ArgResolver(co_consts=co.co_consts, names=co.co_names, varname_from_oparg=co._varname_from_oparg, @@ -757,8 +820,8 @@ def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False, exception_entries=exception_entries, co_positions=co.co_positions(), original_code=co.co_code, arg_resolver=arg_resolver, formatter=formatter) -def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adaptive=False, show_offsets=False): - disassemble(co, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) +def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False): + disassemble(co, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) if depth is None or depth > 0: if depth is not None: depth = depth - 1 @@ -768,7 +831,7 @@ def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adap print("Disassembly of %r:" % (x,), file=file) _disassemble_recursive( x, file=file, depth=depth, show_caches=show_caches, - adaptive=adaptive, show_offsets=show_offsets + adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions ) @@ -801,6 +864,22 @@ def _get_lineno_width(linestarts): lineno_width = len(_NO_LINENO) return lineno_width +def _get_positions_width(code): + # Positions are formatted as 'LINE:COL-ENDLINE:ENDCOL ' (note trailing space). + # A missing component appears as '?', and when all components are None, we + # render '_NO_LINENO'. thus the minimum width is 1 + len(_NO_LINENO). + # + # If all values are missing, positions are not printed (i.e. positions_width = 0). + has_value = False + values_width = 0 + for positions in code.co_positions(): + has_value |= any(isinstance(p, int) for p in positions) + width = sum(1 if p is None else len(str(p)) for p in positions) + values_width = max(width, values_width) + if has_value: + # 3 = number of separators in a normal format + return 1 + max(len(_NO_LINENO), 3 + values_width) + return 0 def _disassemble_bytes(code, lasti=-1, linestarts=None, *, line_offset=0, exception_entries=(), @@ -918,7 +997,8 @@ def _find_imports(co): if op == IMPORT_NAME and i >= 2: from_op = opargs[i-1] level_op = opargs[i-2] - if (from_op[0] in hasconst and level_op[0] in hasconst): + if (from_op[0] in hasconst and + (level_op[0] in hasconst or level_op[0] == LOAD_SMALL_INT)): level = _get_const_value(level_op[0], level_op[1], consts) fromlist = _get_const_value(from_op[0], from_op[1], consts) yield (names[oparg], level, fromlist) @@ -947,7 +1027,7 @@ class Bytecode: Iterating over this yields the bytecode operations as Instruction instances. """ - def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False, adaptive=False, show_offsets=False): + def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False): self.codeobj = co = _get_code_object(x) if first_line is None: self.first_line = co.co_firstlineno @@ -962,6 +1042,7 @@ def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False self.show_caches = show_caches self.adaptive = adaptive self.show_offsets = show_offsets + self.show_positions = show_positions def __iter__(self): co = self.codeobj @@ -1005,16 +1086,19 @@ def dis(self): with io.StringIO() as output: code = _get_code_array(co, self.adaptive) offset_width = len(str(max(len(code) - 2, 9999))) if self.show_offsets else 0 - - + if self.show_positions: + lineno_width = _get_positions_width(co) + else: + lineno_width = _get_lineno_width(self._linestarts) labels_map = _make_labels_map(co.co_code, self.exception_entries) label_width = 4 + len(str(len(labels_map))) formatter = Formatter(file=output, - lineno_width=_get_lineno_width(self._linestarts), + lineno_width=lineno_width, offset_width=offset_width, label_width=label_width, line_offset=self._line_offset, - show_caches=self.show_caches) + show_caches=self.show_caches, + show_positions=self.show_positions) arg_resolver = ArgResolver(co_consts=co.co_consts, names=co.co_names, @@ -1032,7 +1116,7 @@ def dis(self): return output.getvalue() -def main(): +def main(args=None): import argparse parser = argparse.ArgumentParser() @@ -1040,8 +1124,12 @@ def main(): help='show inline caches') parser.add_argument('-O', '--show-offsets', action='store_true', help='show instruction offsets') + parser.add_argument('-P', '--show-positions', action='store_true', + help='show instruction positions') + parser.add_argument('-S', '--specialized', action='store_true', + help='show specialized bytecode') parser.add_argument('infile', nargs='?', default='-') - args = parser.parse_args() + args = parser.parse_args(args=args) if args.infile == '-': name = '' source = sys.stdin.buffer.read() @@ -1050,7 +1138,8 @@ def main(): with open(args.infile, 'rb') as infile: source = infile.read() code = compile(source, name, "exec") - dis(code, show_caches=args.show_caches, show_offsets=args.show_offsets) + dis(code, show_caches=args.show_caches, adaptive=args.specialized, + show_offsets=args.show_offsets, show_positions=args.show_positions) if __name__ == "__main__": main() diff --git a/Lib/doctest.py b/Lib/doctest.py index 6049423b5147a5..e02e73ed722f7e 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -104,6 +104,8 @@ def _test(): import unittest from io import StringIO, IncrementalNewlineDecoder from collections import namedtuple +import _colorize # Used in doctests +from _colorize import ANSIColors, can_colorize class TestResults(namedtuple('TestResults', 'failed attempted')): @@ -387,11 +389,11 @@ def __init__(self, out): # still use input() to get user input self.use_rawinput = 1 - def set_trace(self, frame=None): + def set_trace(self, frame=None, *, commands=None): self.__debugger_used = True if frame is None: frame = sys._getframe().f_back - pdb.Pdb.set_trace(self, frame) + pdb.Pdb.set_trace(self, frame, commands=commands) def set_continue(self): # Calling set_continue unconditionally would break unit test @@ -1140,7 +1142,14 @@ def _find_lineno(self, obj, source_lines): obj = obj.fget if inspect.isfunction(obj) and getattr(obj, '__doc__', None): # We don't use `docstring` var here, because `obj` can be changed. - obj = inspect.unwrap(obj).__code__ + obj = inspect.unwrap(obj) + try: + obj = obj.__code__ + except AttributeError: + # Functions implemented in C don't necessarily + # have a __code__ attribute. + # If there's no code, there's no lineno + return None if inspect.istraceback(obj): obj = obj.tb_frame if inspect.isframe(obj): obj = obj.f_code if inspect.iscode(obj): @@ -1172,6 +1181,9 @@ class DocTestRunner: The `run` method is used to process a single DocTest case. It returns a TestResults instance. + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> tests = DocTestFinder().find(_TestClass) >>> runner = DocTestRunner(verbose=False) >>> tests.sort(key = lambda test: test.name) @@ -1191,9 +1203,9 @@ class DocTestRunner: 2 tests in _TestClass 2 tests in _TestClass.__init__ 2 tests in _TestClass.get - 1 tests in _TestClass.square + 1 test in _TestClass.square 7 tests in 4 items. - 7 passed and 0 failed. + 7 passed. Test passed. TestResults(failed=0, attempted=7) @@ -1215,13 +1227,15 @@ class DocTestRunner: `OutputChecker` to the constructor. The test runner's display output can be controlled in two ways. - First, an output function (`out) can be passed to + First, an output function (`out`) can be passed to `TestRunner.run`; this function will be called with strings that should be displayed. It defaults to `sys.stdout.write`. If capturing the output is not sufficient, then the display output can be also customized by subclassing DocTestRunner, and overriding the methods `report_start`, `report_success`, `report_unexpected_exception`, and `report_failure`. + + >>> _colorize.COLORIZE = save_colorize """ # This divider string is used to separate failure messages, and to # separate sections of the summary. @@ -1300,7 +1314,10 @@ def report_unexpected_exception(self, out, test, example, exc_info): 'Exception raised:\n' + _indent(_exception_traceback(exc_info))) def _failure_header(self, test, example): - out = [self.DIVIDER] + red, reset = ( + (ANSIColors.RED, ANSIColors.RESET) if can_colorize() else ("", "") + ) + out = [f"{red}{self.DIVIDER}{reset}"] if test.filename: if test.lineno is not None and example.lineno is not None: lineno = test.lineno + example.lineno + 1 @@ -1540,7 +1557,11 @@ def out(s): # Make sure sys.displayhook just prints the value to stdout save_displayhook = sys.displayhook sys.displayhook = sys.__displayhook__ - + saved_can_colorize = _colorize.can_colorize + _colorize.can_colorize = lambda *args, **kwargs: False + color_variables = {"PYTHON_COLORS": None, "FORCE_COLOR": None} + for key in color_variables: + color_variables[key] = os.environ.pop(key, None) try: return self.__run(test, compileflags, out) finally: @@ -1549,6 +1570,10 @@ def out(s): sys.settrace(save_trace) linecache.getlines = self.save_linecache_getlines sys.displayhook = save_displayhook + _colorize.can_colorize = saved_can_colorize + for key, value in color_variables.items(): + if value is not None: + os.environ[key] = value if clear_globs: test.globs.clear() import builtins @@ -1568,49 +1593,70 @@ def summarize(self, verbose=None): """ if verbose is None: verbose = self._verbose - notests = [] - passed = [] - failed = [] + + notests, passed, failed = [], [], [] total_tries = total_failures = total_skips = 0 - for item in self._stats.items(): - name, (failures, tries, skips) = item + + for name, (failures, tries, skips) in self._stats.items(): assert failures <= tries total_tries += tries total_failures += failures total_skips += skips + if tries == 0: notests.append(name) elif failures == 0: passed.append((name, tries)) else: - failed.append(item) + failed.append((name, (failures, tries, skips))) + + ansi = _colorize.get_colors() + bold_green = ansi.BOLD_GREEN + bold_red = ansi.BOLD_RED + green = ansi.GREEN + red = ansi.RED + reset = ansi.RESET + yellow = ansi.YELLOW + if verbose: if notests: - print(f"{len(notests)} items had no tests:") + print(f"{_n_items(notests)} had no tests:") notests.sort() for name in notests: print(f" {name}") + if passed: - print(f"{len(passed)} items passed all tests:") - passed.sort() - for name, count in passed: - print(f" {count:3d} tests in {name}") + print(f"{green}{_n_items(passed)} passed all tests:{reset}") + for name, count in sorted(passed): + s = "" if count == 1 else "s" + print(f" {green}{count:3d} test{s} in {name}{reset}") + if failed: - print(self.DIVIDER) - print(f"{len(failed)} items had failures:") - failed.sort() - for name, (failures, tries, skips) in failed: + print(f"{red}{self.DIVIDER}{reset}") + print(f"{_n_items(failed)} had failures:") + for name, (failures, tries, skips) in sorted(failed): print(f" {failures:3d} of {tries:3d} in {name}") + if verbose: - print(f"{total_tries} tests in {len(self._stats)} items.") - print(f"{total_tries - total_failures} passed and {total_failures} failed.") + s = "" if total_tries == 1 else "s" + print(f"{total_tries} test{s} in {_n_items(self._stats)}.") + + and_f = ( + f" and {red}{total_failures} failed{reset}" + if total_failures else "" + ) + print(f"{green}{total_tries - total_failures} passed{reset}{and_f}.") + if total_failures: - msg = f"***Test Failed*** {total_failures} failures" + s = "" if total_failures == 1 else "s" + msg = f"{bold_red}***Test Failed*** {total_failures} failure{s}{reset}" if total_skips: - msg = f"{msg} and {total_skips} skipped tests" + s = "" if total_skips == 1 else "s" + msg = f"{msg} and {yellow}{total_skips} skipped test{s}{reset}" print(f"{msg}.") elif verbose: - print("Test passed.") + print(f"{bold_green}Test passed.{reset}") + return TestResults(total_failures, total_tries, skipped=total_skips) #///////////////////////////////////////////////////////////////// @@ -1627,9 +1673,18 @@ def merge(self, other): d[name] = (failures, tries, skips) +def _n_items(items: list | dict) -> str: + """ + Helper to pluralise the number of items in a list. + """ + n = len(items) + s = "" if n == 1 else "s" + return f"{n} item{s}" + + class OutputChecker: """ - A class used to check the whether the actual output from a doctest + A class used to check whether the actual output from a doctest example matches the expected output. `OutputChecker` defines two methods: `check_output`, which compares a given pair of outputs, and returns true if they match; and `output_difference`, which @@ -2262,12 +2317,13 @@ def runTest(self): try: runner.DIVIDER = "-"*70 - failures, tries = runner.run( - test, out=new.write, clear_globs=False) + results = runner.run(test, out=new.write, clear_globs=False) + if results.skipped == results.attempted: + raise unittest.SkipTest("all examples were skipped") finally: sys.stdout = old - if failures: + if results.failed: raise self.failureException(self.format_failure(new.getvalue())) def format_failure(self, err): @@ -2678,7 +2734,7 @@ def testsource(module, name): return testsrc def debug_src(src, pm=False, globs=None): - """Debug a single doctest docstring, in argument `src`'""" + """Debug a single doctest docstring, in argument `src`""" testsrc = script_from_examples(src) debug_script(testsrc, pm, globs) diff --git a/Lib/email/__init__.py b/Lib/email/__init__.py index 9fa47783004185..6d597006e5eefe 100644 --- a/Lib/email/__init__.py +++ b/Lib/email/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2007 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index e4a342d446f6a3..3d845c09d415f6 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -92,9 +92,19 @@ ASPECIALS = TSPECIALS | set("*'%") ATTRIBUTE_ENDS = ASPECIALS | WSP EXTENDED_ATTRIBUTE_ENDS = ATTRIBUTE_ENDS - set('%') +NLSET = {'\n', '\r'} +SPECIALSNL = SPECIALS | NLSET + + +def make_quoted_pairs(value): + """Escape dquote and backslash for use within a quoted-string.""" + return str(value).replace('\\', '\\\\').replace('"', '\\"') + def quote_string(value): - return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"' + escaped = make_quoted_pairs(value) + return f'"{escaped}"' + # Match a RFC 2047 word, looks like =?utf-8?q?someword?= rfc2047_matcher = re.compile(r''' @@ -566,12 +576,14 @@ def display_name(self): if res[0].token_type == 'cfws': res.pop(0) else: - if res[0][0].token_type == 'cfws': + if (isinstance(res[0], TokenList) and + res[0][0].token_type == 'cfws'): res[0] = TokenList(res[0][1:]) if res[-1].token_type == 'cfws': res.pop() else: - if res[-1][-1].token_type == 'cfws': + if (isinstance(res[-1], TokenList) and + res[-1][-1].token_type == 'cfws'): res[-1] = TokenList(res[-1][:-1]) return res.value @@ -586,9 +598,13 @@ def value(self): quote = True if len(self) != 0 and quote: pre = post = '' - if self[0].token_type=='cfws' or self[0][0].token_type=='cfws': + if (self[0].token_type == 'cfws' or + isinstance(self[0], TokenList) and + self[0][0].token_type == 'cfws'): pre = ' ' - if self[-1].token_type=='cfws' or self[-1][-1].token_type=='cfws': + if (self[-1].token_type == 'cfws' or + isinstance(self[-1], TokenList) and + self[-1][-1].token_type == 'cfws'): post = ' ' return pre+quote_string(self.display_name)+post else: @@ -950,6 +966,7 @@ class _InvalidEwError(errors.HeaderParseError): DOT = ValueTerminal('.', 'dot') ListSeparator = ValueTerminal(',', 'list-separator') ListSeparator.as_ew_allowed = False +ListSeparator.syntactic_break = False RouteComponentMarker = ValueTerminal('@', 'route-component-marker') # @@ -1207,7 +1224,7 @@ def get_bare_quoted_string(value): value is the text between the quote marks, with whitespace preserved and quoted pairs decoded. """ - if value[0] != '"': + if not value or value[0] != '"': raise errors.HeaderParseError( "expected '\"' but found '{}'".format(value)) bare_quoted_string = BareQuotedString() @@ -1448,7 +1465,7 @@ def get_local_part(value): """ local_part = LocalPart() leader = None - if value[0] in CFWS_LEADER: + if value and value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value: raise errors.HeaderParseError( @@ -1514,13 +1531,18 @@ def get_obs_local_part(value): raise token, value = get_cfws(value) obs_local_part.append(token) + if not obs_local_part: + raise errors.HeaderParseError( + "expected obs-local-part but found '{}'".format(value)) if (obs_local_part[0].token_type == 'dot' or obs_local_part[0].token_type=='cfws' and + len(obs_local_part) > 1 and obs_local_part[1].token_type=='dot'): obs_local_part.defects.append(errors.InvalidHeaderDefect( "Invalid leading '.' in local part")) if (obs_local_part[-1].token_type == 'dot' or obs_local_part[-1].token_type=='cfws' and + len(obs_local_part) > 1 and obs_local_part[-2].token_type=='dot'): obs_local_part.defects.append(errors.InvalidHeaderDefect( "Invalid trailing '.' in local part")) @@ -1602,7 +1624,7 @@ def get_domain(value): """ domain = Domain() leader = None - if value[0] in CFWS_LEADER: + if value and value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value: raise errors.HeaderParseError( @@ -1678,6 +1700,8 @@ def get_obs_route(value): if value[0] in CFWS_LEADER: token, value = get_cfws(value) obs_route.append(token) + if not value: + break if value[0] == '@': obs_route.append(RouteComponentMarker) token, value = get_domain(value[1:]) @@ -1696,7 +1720,7 @@ def get_angle_addr(value): """ angle_addr = AngleAddr() - if value[0] in CFWS_LEADER: + if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) angle_addr.append(token) if not value or value[0] != '<': @@ -1706,7 +1730,7 @@ def get_angle_addr(value): value = value[1:] # Although it is not legal per RFC5322, SMTP uses '<>' in certain # circumstances. - if value[0] == '>': + if value and value[0] == '>': angle_addr.append(ValueTerminal('>', 'angle-addr-end')) angle_addr.defects.append(errors.InvalidHeaderDefect( "null addr-spec in angle-addr")) @@ -1758,6 +1782,9 @@ def get_name_addr(value): name_addr = NameAddr() # Both the optional display name and the angle-addr can start with cfws. leader = None + if not value: + raise errors.HeaderParseError( + "expected name-addr but found '{}'".format(value)) if value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value: @@ -1772,7 +1799,10 @@ def get_name_addr(value): raise errors.HeaderParseError( "expected name-addr but found '{}'".format(token)) if leader is not None: - token[0][:0] = [leader] + if isinstance(token[0], TokenList): + token[0][:0] = [leader] + else: + token[:0] = [leader] leader = None name_addr.append(token) token, value = get_angle_addr(value) @@ -2765,11 +2795,15 @@ def _refold_parse_tree(parse_tree, *, policy): # max_line_length 0/None means no limit, ie: infinitely long. maxlen = policy.max_line_length or sys.maxsize encoding = 'utf-8' if policy.utf8 else 'us-ascii' - lines = [''] - last_ew = None + lines = [''] # Folded lines to be output + leading_whitespace = '' # When we have whitespace between two encoded + # words, we may need to encode the whitespace + # at the beginning of the second word. + last_ew = None # Points to the last encoded character if there's an ew on + # the line last_charset = None wrap_as_ew_blocked = 0 - want_encoding = False + want_encoding = False # This is set to True if we need to encode this part end_ew_not_allowed = Terminal('', 'wrap_as_ew_blocked') parts = list(parse_tree) while parts: @@ -2778,9 +2812,13 @@ def _refold_parse_tree(parse_tree, *, policy): wrap_as_ew_blocked -= 1 continue tstr = str(part) - if part.token_type == 'ptext' and set(tstr) & SPECIALS: - # Encode if tstr contains special characters. - want_encoding = True + if not want_encoding: + if part.token_type == 'ptext': + # Encode if tstr contains special characters. + want_encoding = not SPECIALSNL.isdisjoint(tstr) + else: + # Encode if tstr contains newlines. + want_encoding = not NLSET.isdisjoint(tstr) try: tstr.encode(encoding) charset = encoding @@ -2793,10 +2831,12 @@ def _refold_parse_tree(parse_tree, *, policy): # 'charset' property on the policy. charset = 'utf-8' want_encoding = True + if part.token_type == 'mime-parameters': # Mime parameter folding (using RFC2231) is extra special. _fold_mime_parameters(part, lines, maxlen, encoding) continue + if want_encoding and not wrap_as_ew_blocked: if not part.as_ew_allowed: want_encoding = False @@ -2819,7 +2859,9 @@ def _refold_parse_tree(parse_tree, *, policy): if not hasattr(part, 'encode'): # It's not a Terminal, do each piece individually. parts = list(part) + parts - else: + want_encoding = False + continue + elif part.as_ew_allowed: # It's a terminal, wrap it as an encoded word, possibly # combining it with previously encoded words if allowed. if (last_ew is not None and @@ -2828,26 +2870,58 @@ def _refold_parse_tree(parse_tree, *, policy): last_charset == 'utf-8' and charset != 'us-ascii')): last_ew = None last_ew = _fold_as_ew(tstr, lines, maxlen, last_ew, - part.ew_combine_allowed, charset) + part.ew_combine_allowed, charset, leading_whitespace) + # This whitespace has been added to the lines in _fold_as_ew() + # so clear it now. + leading_whitespace = '' last_charset = charset - want_encoding = False - continue + want_encoding = False + continue + else: + # It's a terminal which should be kept non-encoded + # (e.g. a ListSeparator). + last_ew = None + want_encoding = False + # fall through + if len(tstr) <= maxlen - len(lines[-1]): lines[-1] += tstr continue + # This part is too long to fit. The RFC wants us to break at # "major syntactic breaks", so unless we don't consider this # to be one, check if it will fit on the next line by itself. + leading_whitespace = '' if (part.syntactic_break and len(tstr) + 1 <= maxlen): newline = _steal_trailing_WSP_if_exists(lines) if newline or part.startswith_fws(): + # We're going to fold the data onto a new line here. Due to + # the way encoded strings handle continuation lines, we need to + # be prepared to encode any whitespace if the next line turns + # out to start with an encoded word. lines.append(newline + tstr) + + whitespace_accumulator = [] + for char in lines[-1]: + if char not in WSP: + break + whitespace_accumulator.append(char) + leading_whitespace = ''.join(whitespace_accumulator) last_ew = None continue if not hasattr(part, 'encode'): # It's not a terminal, try folding the subparts. newparts = list(part) + if part.token_type == 'bare-quoted-string': + # To fold a quoted string we need to create a list of terminal + # tokens that will render the leading and trailing quotes + # and use quoted pairs in the value as appropriate. + newparts = ( + [ValueTerminal('"', 'ptext')] + + [ValueTerminal(make_quoted_pairs(p), 'ptext') + for p in newparts] + + [ValueTerminal('"', 'ptext')]) if not part.as_ew_allowed: wrap_as_ew_blocked += 1 newparts.append(end_ew_not_allowed) @@ -2866,9 +2940,10 @@ def _refold_parse_tree(parse_tree, *, policy): else: # We can't fold it onto the next line either... lines[-1] += tstr + return policy.linesep.join(lines) + policy.linesep -def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): +def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset, leading_whitespace): """Fold string to_encode into lines as encoded word, combining if allowed. Return the new value for last_ew, or None if ew_combine_allowed is False. @@ -2883,7 +2958,7 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): to_encode = str( get_unstructured(lines[-1][last_ew:] + to_encode)) lines[-1] = lines[-1][:last_ew] - if to_encode[0] in WSP: + elif to_encode[0] in WSP: # We're joining this to non-encoded text, so don't encode # the leading blank. leading_wsp = to_encode[0] @@ -2891,6 +2966,7 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): if (len(lines[-1]) == maxlen): lines.append(_steal_trailing_WSP_if_exists(lines)) lines[-1] += leading_wsp + trailing_wsp = '' if to_encode[-1] in WSP: # Likewise for the trailing space. @@ -2910,11 +2986,20 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): while to_encode: remaining_space = maxlen - len(lines[-1]) - text_space = remaining_space - chrome_len + text_space = remaining_space - chrome_len - len(leading_whitespace) if text_space <= 0: lines.append(' ') continue + # If we are at the start of a continuation line, prepend whitespace + # (we only want to do this when the line starts with an encoded word + # but if we're folding in this helper function, then we know that we + # are going to be writing out an encoded word.) + if len(lines) > 1 and len(lines[-1]) == 1 and leading_whitespace: + encoded_word = _ew.encode(leading_whitespace, charset=encode_as) + lines[-1] += encoded_word + leading_whitespace = '' + to_encode_word = to_encode[:text_space] encoded_word = _ew.encode(to_encode_word, charset=encode_as) excess = len(encoded_word) - remaining_space @@ -2926,6 +3011,7 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): excess = len(encoded_word) - remaining_space lines[-1] += encoded_word to_encode = to_encode[len(to_encode_word):] + leading_whitespace = '' if to_encode: lines.append(' ') diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py index 0f1bf8e4253ec4..84917038874ba1 100644 --- a/Lib/email/_parseaddr.py +++ b/Lib/email/_parseaddr.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2007 Python Software Foundation +# Copyright (C) 2002 Python Software Foundation # Contact: email-sig@python.org """Email address parsing code. @@ -224,7 +224,7 @@ class AddrlistClass: def __init__(self, field): """Initialize a new instance. - `field' is an unparsed address header field, containing + 'field' is an unparsed address header field, containing one or more addresses. """ self.specials = '()<>@,:;.\"[]' @@ -233,7 +233,7 @@ def __init__(self, field): self.CR = '\r\n' self.FWS = self.LWS + self.CR self.atomends = self.specials + self.LWS + self.CR - # Note that RFC 2822 now specifies `.' as obs-phrase, meaning that it + # Note that RFC 2822 now specifies '.' as obs-phrase, meaning that it # is obsolete syntax. RFC 2822 requires that we recognize obsolete # syntax, so allow dots in phrases. self.phraseends = self.atomends.replace('.', '') @@ -423,14 +423,14 @@ def getdomain(self): def getdelimited(self, beginchar, endchars, allowcomments=True): """Parse a header fragment delimited by special characters. - `beginchar' is the start character for the fragment. - If self is not looking at an instance of `beginchar' then + 'beginchar' is the start character for the fragment. + If self is not looking at an instance of 'beginchar' then getdelimited returns the empty string. - `endchars' is a sequence of allowable end-delimiting characters. + 'endchars' is a sequence of allowable end-delimiting characters. Parsing stops when one of these is encountered. - If `allowcomments' is non-zero, embedded RFC 2822 comments are allowed + If 'allowcomments' is non-zero, embedded RFC 2822 comments are allowed within the parsed fragment. """ if self.field[self.pos] != beginchar: @@ -474,7 +474,7 @@ def getatom(self, atomends=None): Optional atomends specifies a different set of end token delimiters (the default is to use self.atomends). This is used e.g. in - getphraselist() since phrase endings must not include the `.' (which + getphraselist() since phrase endings must not include the '.' (which is legal in phrases).""" atomlist = [''] if atomends is None: diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py index c9cbadd2a80c48..4b63b97217a835 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -150,13 +150,20 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta): wrapping is done. Default is 78. mangle_from_ -- a flag that, when True escapes From_ lines in the - body of the message by putting a `>' in front of + body of the message by putting a '>' in front of them. This is used when the message is being - serialized by a generator. Default: True. + serialized by a generator. Default: False. message_factory -- the class to use to create new message objects. If the value is None, the default is Message. + verify_generated_headers + -- if true, the generator verifies that each header + they are properly folded, so that a parser won't + treat it as multiple headers, start-of-body, or + part of another header. + This is a check against custom Header & fold() + implementations. """ raise_on_defect = False @@ -165,6 +172,7 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta): max_line_length = 78 mangle_from_ = False message_factory = None + verify_generated_headers = True def handle_defect(self, obj, defect): """Based on policy, either raise defect or call register_defect. @@ -294,12 +302,12 @@ def header_source_parse(self, sourcelines): """+ The name is parsed as everything up to the ':' and returned unmodified. The value is determined by stripping leading whitespace off the - remainder of the first line, joining all subsequent lines together, and + remainder of the first line joined with all subsequent lines, and stripping any trailing carriage return or linefeed characters. """ name, value = sourcelines[0].split(':', 1) - value = value.lstrip(' \t') + ''.join(sourcelines[1:]) + value = ''.join((value, *sourcelines[1:])).lstrip(' \t\r\n') return (name, value.rstrip('\r\n')) def header_store_parse(self, name, value): diff --git a/Lib/email/base64mime.py b/Lib/email/base64mime.py index 4cdf22666e3016..a5a3f737a97b51 100644 --- a/Lib/email/base64mime.py +++ b/Lib/email/base64mime.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2007 Python Software Foundation +# Copyright (C) 2002 Python Software Foundation # Author: Ben Gertzfield # Contact: email-sig@python.org @@ -15,7 +15,7 @@ with Base64 encoding. RFC 2045 defines a method for including character set information in an -`encoded-word' in a header. This method is commonly used for 8-bit real names +'encoded-word' in a header. This method is commonly used for 8-bit real names in To:, From:, Cc:, etc. fields, as well as Subject: lines. This module does not do the line wrapping or end-of-line character conversion diff --git a/Lib/email/charset.py b/Lib/email/charset.py index 043801107b60e5..5036c3f58a5633 100644 --- a/Lib/email/charset.py +++ b/Lib/email/charset.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2007 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Ben Gertzfield, Barry Warsaw # Contact: email-sig@python.org @@ -175,7 +175,7 @@ class Charset: module expose the following information about a character set: input_charset: The initial character set specified. Common aliases - are converted to their `official' email names (e.g. latin_1 + are converted to their 'official' email names (e.g. latin_1 is converted to iso-8859-1). Defaults to 7-bit us-ascii. header_encoding: If the character set must be encoded before it can be @@ -245,7 +245,7 @@ def __eq__(self, other): def get_body_encoding(self): """Return the content-transfer-encoding used for body encoding. - This is either the string `quoted-printable' or `base64' depending on + This is either the string 'quoted-printable' or 'base64' depending on the encoding used, or it is a function in which case you should call the function with a single argument, the Message object being encoded. The function should then set the Content-Transfer-Encoding diff --git a/Lib/email/encoders.py b/Lib/email/encoders.py index 17bd1ab7b19f32..55741a22a07b20 100644 --- a/Lib/email/encoders.py +++ b/Lib/email/encoders.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2006 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/errors.py b/Lib/email/errors.py index 3ad00565549968..6bc744bd59c5bb 100644 --- a/Lib/email/errors.py +++ b/Lib/email/errors.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2006 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org @@ -29,6 +29,10 @@ class CharsetError(MessageError): """An illegal charset was given.""" +class HeaderWriteError(MessageError): + """Error while writing headers.""" + + # These are parsing defects which the parser was able to work around. class MessageDefect(ValueError): """Base class for a message defect.""" diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py index 06d6b4a3afcd07..b2bc4afc1cc26f 100644 --- a/Lib/email/feedparser.py +++ b/Lib/email/feedparser.py @@ -1,4 +1,4 @@ -# Copyright (C) 2004-2006 Python Software Foundation +# Copyright (C) 2004 Python Software Foundation # Authors: Baxter, Wouters and Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/generator.py b/Lib/email/generator.py index 7ccbe10eb76856..ab5bd0653e440c 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2010 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org @@ -14,12 +14,14 @@ from copy import deepcopy from io import StringIO, BytesIO from email.utils import _has_surrogates +from email.errors import HeaderWriteError UNDERSCORE = '_' NL = '\n' # XXX: no longer used by the code below. NLCRE = re.compile(r'\r\n|\r|\n') fcre = re.compile(r'^From ', re.MULTILINE) +NEWLINE_WITHOUT_FWSP = re.compile(r'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]') class Generator: @@ -41,7 +43,7 @@ def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, *, Optional mangle_from_ is a flag that, when True (the default if policy is not set), escapes From_ lines in the body of the message by putting - a `>' in front of them. + a '>' in front of them. Optional maxheaderlen specifies the longest length for a non-continued header. When a header line is longer (in characters, with tabs @@ -74,7 +76,7 @@ def flatten(self, msg, unixfrom=False, linesep=None): unixfrom is a flag that forces the printing of a Unix From_ delimiter before the first object in the message tree. If the original message - has no From_ delimiter, a `standard' one is crafted. By default, this + has no From_ delimiter, a 'standard' one is crafted. By default, this is False to inhibit the printing of any From_ delimiter. Note that for subobjects, no From_ line is printed. @@ -222,7 +224,16 @@ def _dispatch(self, msg): def _write_headers(self, msg): for h, v in msg.raw_items(): - self.write(self.policy.fold(h, v)) + folded = self.policy.fold(h, v) + if self.policy.verify_generated_headers: + linesep = self.policy.linesep + if not folded.endswith(linesep): + raise HeaderWriteError( + f'folded header does not end with {linesep!r}: {folded!r}') + if NEWLINE_WITHOUT_FWSP.search(folded.removesuffix(linesep)): + raise HeaderWriteError( + f'folded header contains newline: {folded!r}') + self.write(folded) # A blank line always separates headers from body self.write(self._NL) @@ -243,7 +254,7 @@ def _handle_text(self, msg): # existing message. msg = deepcopy(msg) del msg['content-transfer-encoding'] - msg.set_payload(payload, charset) + msg.set_payload(msg._payload, charset) payload = msg.get_payload() self._munge_cte = (msg['content-transfer-encoding'], msg['content-type']) @@ -456,7 +467,7 @@ def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, fmt=None, *, argument is allowed. Walks through all subparts of a message. If the subpart is of main - type `text', then it prints the decoded payload of the subpart. + type 'text', then it prints the decoded payload of the subpart. Otherwise, fmt is a format string that is used instead of the message payload. fmt is expanded with the following keywords (in diff --git a/Lib/email/header.py b/Lib/email/header.py index 984851a7d9a679..113a81f41314ec 100644 --- a/Lib/email/header.py +++ b/Lib/email/header.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2007 Python Software Foundation +# Copyright (C) 2002 Python Software Foundation # Author: Ben Gertzfield, Barry Warsaw # Contact: email-sig@python.org @@ -192,7 +192,7 @@ def __init__(self, s=None, charset=None, The maximum line length can be specified explicitly via maxlinelen. For splitting the first line to a shorter value (to account for the field - header which isn't included in s, e.g. `Subject') pass in the name of + header which isn't included in s, e.g. 'Subject') pass in the name of the field in header_name. The default maxlinelen is 78 as recommended by RFC 2822. @@ -276,7 +276,7 @@ def append(self, s, charset=None, errors='strict'): output codec of the charset. If the string cannot be encoded to the output codec, a UnicodeError will be raised. - Optional `errors' is passed as the errors argument to the decode + Optional 'errors' is passed as the errors argument to the decode call if s is a byte string. """ if charset is None: @@ -326,7 +326,7 @@ def encode(self, splitchars=';, \t', maxlinelen=None, linesep='\n'): Optional splitchars is a string containing characters which should be given extra weight by the splitting algorithm during normal header - wrapping. This is in very rough support of RFC 2822's `higher level + wrapping. This is in very rough support of RFC 2822's 'higher level syntactic breaks': split points preceded by a splitchar are preferred during line splitting, with the characters preferred in the order in which they appear in the string. Space and tab may be included in the diff --git a/Lib/email/iterators.py b/Lib/email/iterators.py index 3410935e38f476..08ede3ec679613 100644 --- a/Lib/email/iterators.py +++ b/Lib/email/iterators.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2006 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org @@ -43,8 +43,8 @@ def body_line_iterator(msg, decode=False): def typed_subpart_iterator(msg, maintype='text', subtype=None): """Iterate over the subparts with a given MIME type. - Use `maintype' as the main MIME type to match against; this defaults to - "text". Optional `subtype' is the MIME subtype to match against; if + Use 'maintype' as the main MIME type to match against; this defaults to + "text". Optional 'subtype' is the MIME subtype to match against; if omitted, only the main type is matched. """ for subpart in msg.walk(): diff --git a/Lib/email/message.py b/Lib/email/message.py index fe769580fed5d0..87fcab68868d46 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2007 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org @@ -21,7 +21,7 @@ SEMISPACE = '; ' -# Regular expression that matches `special' characters in parameters, the +# Regular expression that matches 'special' characters in parameters, the # existence of which force quoting of the parameter value. tspecials = re.compile(r'[ \(\)<>@,;:\\"/\[\]\?=]') @@ -141,7 +141,7 @@ class Message: multipart or a message/rfc822), then the payload is a list of Message objects, otherwise it is a string. - Message objects implement part of the `mapping' interface, which assumes + Message objects implement part of the 'mapping' interface, which assumes there is exactly one occurrence of the header per message. Some headers do in fact appear multiple times (e.g. Received) and for those headers, you must use the explicit API to set or get all the headers. Not all of @@ -286,15 +286,19 @@ def get_payload(self, i=None, decode=False): if i is not None and not isinstance(self._payload, list): raise TypeError('Expected list, got %s' % type(self._payload)) payload = self._payload - # cte might be a Header, so for now stringify it. - cte = str(self.get('content-transfer-encoding', '')).lower() + cte = self.get('content-transfer-encoding', '') + if hasattr(cte, 'cte'): + cte = cte.cte + else: + # cte might be a Header, so for now stringify it. + cte = str(cte).strip().lower() # payload may be bytes here. if not decode: if isinstance(payload, str) and utils._has_surrogates(payload): try: bpayload = payload.encode('ascii', 'surrogateescape') try: - payload = bpayload.decode(self.get_param('charset', 'ascii'), 'replace') + payload = bpayload.decode(self.get_content_charset('ascii'), 'replace') except LookupError: payload = bpayload.decode('ascii', 'replace') except UnicodeEncodeError: @@ -340,7 +344,7 @@ def set_payload(self, payload, charset=None): return if not isinstance(charset, Charset): charset = Charset(charset) - payload = payload.encode(charset.output_charset) + payload = payload.encode(charset.output_charset, 'surrogateescape') if hasattr(payload, 'decode'): self._payload = payload.decode('ascii', 'surrogateescape') else: @@ -597,7 +601,7 @@ def get_content_type(self): """Return the message's content type. The returned string is coerced to lower case of the form - `maintype/subtype'. If there was no Content-Type header in the + 'maintype/subtype'. If there was no Content-Type header in the message, the default type as given by get_default_type() will be returned. Since according to RFC 2045, messages always have a default type this will always return a value. @@ -620,7 +624,7 @@ def get_content_type(self): def get_content_maintype(self): """Return the message's main content type. - This is the `maintype' part of the string returned by + This is the 'maintype' part of the string returned by get_content_type(). """ ctype = self.get_content_type() @@ -629,14 +633,14 @@ def get_content_maintype(self): def get_content_subtype(self): """Returns the message's sub-content type. - This is the `subtype' part of the string returned by + This is the 'subtype' part of the string returned by get_content_type(). """ ctype = self.get_content_type() return ctype.split('/')[1] def get_default_type(self): - """Return the `default' content type. + """Return the 'default' content type. Most messages have a default content type of text/plain, except for messages that are subparts of multipart/digest containers. Such @@ -645,7 +649,7 @@ def get_default_type(self): return self._default_type def set_default_type(self, ctype): - """Set the `default' content type. + """Set the 'default' content type. ctype should be either "text/plain" or "message/rfc822", although this is not enforced. The default content type is not stored in the @@ -678,8 +682,8 @@ def get_params(self, failobj=None, header='content-type', unquote=True): """Return the message's Content-Type parameters, as a list. The elements of the returned list are 2-tuples of key/value pairs, as - split on the `=' sign. The left hand side of the `=' is the key, - while the right hand side is the value. If there is no `=' sign in + split on the '=' sign. The left hand side of the '=' is the key, + while the right hand side is the value. If there is no '=' sign in the parameter the value is the empty string. The value is as described in the get_param() method. @@ -839,9 +843,9 @@ def get_filename(self, failobj=None): """Return the filename associated with the payload if present. The filename is extracted from the Content-Disposition header's - `filename' parameter, and it is unquoted. If that header is missing - the `filename' parameter, this method falls back to looking for the - `name' parameter. + 'filename' parameter, and it is unquoted. If that header is missing + the 'filename' parameter, this method falls back to looking for the + 'name' parameter. """ missing = object() filename = self.get_param('filename', missing, 'content-disposition') @@ -854,7 +858,7 @@ def get_filename(self, failobj=None): def get_boundary(self, failobj=None): """Return the boundary associated with the payload if present. - The boundary is extracted from the Content-Type header's `boundary' + The boundary is extracted from the Content-Type header's 'boundary' parameter, and it is unquoted. """ missing = object() diff --git a/Lib/email/mime/application.py b/Lib/email/mime/application.py index f67cbad3f03407..9a9d213d2a940d 100644 --- a/Lib/email/mime/application.py +++ b/Lib/email/mime/application.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2006 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Keith Dart # Contact: email-sig@python.org diff --git a/Lib/email/mime/audio.py b/Lib/email/mime/audio.py index aa0c4905cbb2b4..85f4a955238c52 100644 --- a/Lib/email/mime/audio.py +++ b/Lib/email/mime/audio.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2007 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Anthony Baxter # Contact: email-sig@python.org diff --git a/Lib/email/mime/base.py b/Lib/email/mime/base.py index f601f621cec393..da4c6e591a5cb8 100644 --- a/Lib/email/mime/base.py +++ b/Lib/email/mime/base.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2006 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/mime/image.py b/Lib/email/mime/image.py index 4b7f2f9cbad425..dab9685848172b 100644 --- a/Lib/email/mime/image.py +++ b/Lib/email/mime/image.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2006 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/mime/message.py b/Lib/email/mime/message.py index 61836b5a7861fc..13d9ff599f86db 100644 --- a/Lib/email/mime/message.py +++ b/Lib/email/mime/message.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2006 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/mime/multipart.py b/Lib/email/mime/multipart.py index 94d81c771a474e..1abb84d5fed0bb 100644 --- a/Lib/email/mime/multipart.py +++ b/Lib/email/mime/multipart.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2006 Python Software Foundation +# Copyright (C) 2002 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org @@ -21,7 +21,7 @@ def __init__(self, _subtype='mixed', boundary=None, _subparts=None, Content-Type and MIME-Version headers. _subtype is the subtype of the multipart content type, defaulting to - `mixed'. + 'mixed'. boundary is the multipart boundary string. By default it is calculated as needed. diff --git a/Lib/email/mime/nonmultipart.py b/Lib/email/mime/nonmultipart.py index a41386eb148c0c..5beab3a441e2bc 100644 --- a/Lib/email/mime/nonmultipart.py +++ b/Lib/email/mime/nonmultipart.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2006 Python Software Foundation +# Copyright (C) 2002 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/mime/text.py b/Lib/email/mime/text.py index 7672b789138600..aa4da7f8217e43 100644 --- a/Lib/email/mime/text.py +++ b/Lib/email/mime/text.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2006 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org diff --git a/Lib/email/parser.py b/Lib/email/parser.py index 06d99b17f2f9c4..039f03cba74fa0 100644 --- a/Lib/email/parser.py +++ b/Lib/email/parser.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2007 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw, Thomas Wouters, Anthony Baxter # Contact: email-sig@python.org @@ -22,7 +22,7 @@ def __init__(self, _class=None, *, policy=compat32): textual representation of the message. The string must be formatted as a block of RFC 2822 headers and header - continuation lines, optionally preceded by a `Unix-from' header. The + continuation lines, optionally preceded by a 'Unix-from' header. The header block is terminated either by the end of the string or by a blank line. @@ -82,7 +82,7 @@ def __init__(self, *args, **kw): textual representation of the message. The input must be formatted as a block of RFC 2822 headers and header - continuation lines, optionally preceded by a `Unix-from' header. The + continuation lines, optionally preceded by a 'Unix-from' header. The header block is terminated either by the end of the input or by a blank line. diff --git a/Lib/email/policy.py b/Lib/email/policy.py index 8816c84ed175a7..6e109b65011a44 100644 --- a/Lib/email/policy.py +++ b/Lib/email/policy.py @@ -21,7 +21,7 @@ 'HTTP', ] -linesep_splitter = re.compile(r'\n|\r') +linesep_splitter = re.compile(r'\n|\r\n?') @_extend_docstrings class EmailPolicy(Policy): @@ -119,13 +119,13 @@ def header_source_parse(self, sourcelines): """+ The name is parsed as everything up to the ':' and returned unmodified. The value is determined by stripping leading whitespace off the - remainder of the first line, joining all subsequent lines together, and + remainder of the first line joined with all subsequent lines, and stripping any trailing carriage return or linefeed characters. (This is the same as Compat32). """ name, value = sourcelines[0].split(':', 1) - value = value.lstrip(' \t') + ''.join(sourcelines[1:]) + value = ''.join((value, *sourcelines[1:])).lstrip(' \t\r\n') return (name, value.rstrip('\r\n')) def header_store_parse(self, name, value): @@ -205,7 +205,8 @@ def _fold(self, name, value, refold_binary=False): if hasattr(value, 'name'): return value.fold(policy=self) maxlen = self.max_line_length if self.max_line_length else sys.maxsize - lines = value.splitlines() + # We can't use splitlines here because it splits on more than \r and \n. + lines = linesep_splitter.split(value) refold = (self.refold_source == 'all' or self.refold_source == 'long' and (lines and len(lines[0])+len(name)+2 > maxlen or diff --git a/Lib/email/quoprimime.py b/Lib/email/quoprimime.py index 27fcbb5a26e3ae..27c7ea55c7871f 100644 --- a/Lib/email/quoprimime.py +++ b/Lib/email/quoprimime.py @@ -1,11 +1,11 @@ -# Copyright (C) 2001-2006 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Ben Gertzfield # Contact: email-sig@python.org """Quoted-printable content transfer encoding per RFCs 2045-2047. This module handles the content transfer encoding method defined in RFC 2045 -to encode US ASCII-like 8-bit data called `quoted-printable'. It is used to +to encode US ASCII-like 8-bit data called 'quoted-printable'. It is used to safely encode text that is in a character set similar to the 7-bit US ASCII character set, but that includes some 8-bit characters that are normally not allowed in email bodies or headers. @@ -17,7 +17,7 @@ with quoted-printable encoding. RFC 2045 defines a method for including character set information in an -`encoded-word' in a header. This method is commonly used for 8-bit real names +'encoded-word' in a header. This method is commonly used for 8-bit real names in To:/From:/Cc: etc. fields, as well as Subject: lines. This module does not do the line wrapping or end-of-line character @@ -127,7 +127,7 @@ def quote(c): def header_encode(header_bytes, charset='iso-8859-1'): """Encode a single header line with quoted-printable (like) encoding. - Defined in RFC 2045, this `Q' encoding is similar to quoted-printable, but + Defined in RFC 2045, this 'Q' encoding is similar to quoted-printable, but used specifically for email header fields to allow charsets with mostly 7 bit characters (and some 8 bit) to remain more or less readable in non-RFC 2045 aware mail clients. @@ -290,7 +290,7 @@ def _unquote_match(match): # Header decoding is done a bit differently def header_decode(s): - """Decode a string encoded with RFC 2045 MIME header `Q' encoding. + """Decode a string encoded with RFC 2045 MIME header 'Q' encoding. This function does not parse a full MIME header value encoded with quoted-printable (like =?iso-8859-1?q?Hello_World?=) -- please use diff --git a/Lib/email/utils.py b/Lib/email/utils.py index 103cef61a83538..7eab74dc0db9df 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2010 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Author: Barry Warsaw # Contact: email-sig@python.org @@ -241,7 +241,7 @@ def formatdate(timeval=None, localtime=False, usegmt=False): Fri, 09 Nov 2001 01:08:47 -0000 - Optional timeval if given is a floating point time value as accepted by + Optional timeval if given is a floating-point time value as accepted by gmtime() and localtime(), otherwise the current time is used. Optional localtime is a flag that when True, interprets timeval, and @@ -466,23 +466,15 @@ def collapse_rfc2231_value(value, errors='replace', # better than not having it. # -def localtime(dt=None, isdst=None): +def localtime(dt=None): """Return local time as an aware datetime object. If called without arguments, return current time. Otherwise *dt* argument should be a datetime instance, and it is converted to the local time zone according to the system time zone database. If *dt* is naive (that is, dt.tzinfo is None), it is assumed to be in local time. - The isdst parameter is ignored. """ - if isdst is not None: - import warnings - warnings._deprecated( - "The 'isdst' parameter to 'localtime'", - message='{name} is deprecated and slated for removal in Python {remove}', - remove=(3, 14), - ) if dt is None: dt = datetime.datetime.now() return dt.astimezone() diff --git a/Lib/encodings/__init__.py b/Lib/encodings/__init__.py index f9075b8f0d98ac..298177eb8003a7 100644 --- a/Lib/encodings/__init__.py +++ b/Lib/encodings/__init__.py @@ -156,19 +156,22 @@ def search_function(encoding): codecs.register(search_function) if sys.platform == 'win32': - # bpo-671666, bpo-46668: If Python does not implement a codec for current - # Windows ANSI code page, use the "mbcs" codec instead: - # WideCharToMultiByte() and MultiByteToWideChar() functions with CP_ACP. - # Python does not support custom code pages. - def _alias_mbcs(encoding): + from ._win_cp_codecs import create_win32_code_page_codec + + def win32_code_page_search_function(encoding): + encoding = encoding.lower() + if not encoding.startswith('cp'): + return None try: - import _winapi - ansi_code_page = "cp%s" % _winapi.GetACP() - if encoding == ansi_code_page: - import encodings.mbcs - return encodings.mbcs.getregentry() - except ImportError: - # Imports may fail while we are shutting down - pass + cp = int(encoding[2:]) + except ValueError: + return None + # Test if the code page is supported + try: + codecs.code_page_encode(cp, 'x') + except (OverflowError, OSError): + return None + + return create_win32_code_page_codec(cp) - codecs.register(_alias_mbcs) + codecs.register(win32_code_page_search_function) diff --git a/Lib/encodings/_win_cp_codecs.py b/Lib/encodings/_win_cp_codecs.py new file mode 100644 index 00000000000000..4f8eb886794404 --- /dev/null +++ b/Lib/encodings/_win_cp_codecs.py @@ -0,0 +1,36 @@ +import codecs + +def create_win32_code_page_codec(cp): + from codecs import code_page_encode, code_page_decode + + def encode(input, errors='strict'): + return code_page_encode(cp, input, errors) + + def decode(input, errors='strict'): + return code_page_decode(cp, input, errors, True) + + class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input, final=False): + return code_page_encode(cp, input, self.errors)[0] + + class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def _buffer_decode(self, input, errors, final): + return code_page_decode(cp, input, errors, final) + + class StreamWriter(codecs.StreamWriter): + def encode(self, input, errors='strict'): + return code_page_encode(cp, input, errors) + + class StreamReader(codecs.StreamReader): + def decode(self, input, errors, final): + return code_page_decode(cp, input, errors, final) + + return codecs.CodecInfo( + name=f'cp{cp}', + encode=encode, + decode=decode, + incrementalencoder=IncrementalEncoder, + incrementaldecoder=IncrementalDecoder, + streamreader=StreamReader, + streamwriter=StreamWriter, + ) diff --git a/Lib/encodings/idna.py b/Lib/encodings/idna.py index 5396047a7fb0b8..60a8d5eb227f82 100644 --- a/Lib/encodings/idna.py +++ b/Lib/encodings/idna.py @@ -11,7 +11,7 @@ sace_prefix = "xn--" # This assumes query strings, so AllowUnassigned is true -def nameprep(label): +def nameprep(label): # type: (str) -> str # Map newlabel = [] for c in label: @@ -25,7 +25,7 @@ def nameprep(label): label = unicodedata.normalize("NFKC", label) # Prohibit - for c in label: + for i, c in enumerate(label): if stringprep.in_table_c12(c) or \ stringprep.in_table_c22(c) or \ stringprep.in_table_c3(c) or \ @@ -35,7 +35,7 @@ def nameprep(label): stringprep.in_table_c7(c) or \ stringprep.in_table_c8(c) or \ stringprep.in_table_c9(c): - raise UnicodeError("Invalid character %r" % c) + raise UnicodeEncodeError("idna", label, i, i+1, f"Invalid character {c!r}") # Check bidi RandAL = [stringprep.in_table_d1(x) for x in label] @@ -46,29 +46,38 @@ def nameprep(label): # This is table C.8, which was already checked # 2) If a string contains any RandALCat character, the string # MUST NOT contain any LCat character. - if any(stringprep.in_table_d2(x) for x in label): - raise UnicodeError("Violation of BIDI requirement 2") + for i, x in enumerate(label): + if stringprep.in_table_d2(x): + raise UnicodeEncodeError("idna", label, i, i+1, + "Violation of BIDI requirement 2") # 3) If a string contains any RandALCat character, a # RandALCat character MUST be the first character of the # string, and a RandALCat character MUST be the last # character of the string. - if not RandAL[0] or not RandAL[-1]: - raise UnicodeError("Violation of BIDI requirement 3") + if not RandAL[0]: + raise UnicodeEncodeError("idna", label, 0, 1, + "Violation of BIDI requirement 3") + if not RandAL[-1]: + raise UnicodeEncodeError("idna", label, len(label)-1, len(label), + "Violation of BIDI requirement 3") return label -def ToASCII(label): +def ToASCII(label): # type: (str) -> bytes try: # Step 1: try ASCII - label = label.encode("ascii") - except UnicodeError: + label_ascii = label.encode("ascii") + except UnicodeEncodeError: pass else: # Skip to step 3: UseSTD3ASCIIRules is false, so # Skip to step 8. - if 0 < len(label) < 64: - return label - raise UnicodeError("label empty or too long") + if 0 < len(label_ascii) < 64: + return label_ascii + if len(label) == 0: + raise UnicodeEncodeError("idna", label, 0, 1, "label empty") + else: + raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") # Step 2: nameprep label = nameprep(label) @@ -76,29 +85,34 @@ def ToASCII(label): # Step 3: UseSTD3ASCIIRules is false # Step 4: try ASCII try: - label = label.encode("ascii") - except UnicodeError: + label_ascii = label.encode("ascii") + except UnicodeEncodeError: pass else: # Skip to step 8. if 0 < len(label) < 64: - return label - raise UnicodeError("label empty or too long") + return label_ascii + if len(label) == 0: + raise UnicodeEncodeError("idna", label, 0, 1, "label empty") + else: + raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") # Step 5: Check ACE prefix - if label.startswith(sace_prefix): - raise UnicodeError("Label starts with ACE prefix") + if label.lower().startswith(sace_prefix): + raise UnicodeEncodeError( + "idna", label, 0, len(sace_prefix), "Label starts with ACE prefix") # Step 6: Encode with PUNYCODE - label = label.encode("punycode") + label_ascii = label.encode("punycode") # Step 7: Prepend ACE prefix - label = ace_prefix + label + label_ascii = ace_prefix + label_ascii # Step 8: Check size - if 0 < len(label) < 64: - return label - raise UnicodeError("label empty or too long") + # do not check for empty as we prepend ace_prefix. + if len(label_ascii) < 64: + return label_ascii + raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") def ToUnicode(label): if len(label) > 1024: @@ -110,7 +124,9 @@ def ToUnicode(label): # per https://www.rfc-editor.org/rfc/rfc3454#section-3.1 while still # preventing us from wasting time decoding a big thing that'll just # hit the actual <= 63 length limit in Step 6. - raise UnicodeError("label way too long") + if isinstance(label, str): + label = label.encode("utf-8", errors="backslashreplace") + raise UnicodeDecodeError("idna", label, 0, len(label), "label way too long") # Step 1: Check for ASCII if isinstance(label, bytes): pure_ascii = True @@ -118,25 +134,32 @@ def ToUnicode(label): try: label = label.encode("ascii") pure_ascii = True - except UnicodeError: + except UnicodeEncodeError: pure_ascii = False if not pure_ascii: + assert isinstance(label, str) # Step 2: Perform nameprep label = nameprep(label) # It doesn't say this, but apparently, it should be ASCII now try: label = label.encode("ascii") - except UnicodeError: - raise UnicodeError("Invalid character in IDN label") + except UnicodeEncodeError as exc: + raise UnicodeEncodeError("idna", label, exc.start, exc.end, + "Invalid character in IDN label") # Step 3: Check for ACE prefix - if not label.startswith(ace_prefix): + assert isinstance(label, bytes) + if not label.lower().startswith(ace_prefix): return str(label, "ascii") # Step 4: Remove ACE prefix label1 = label[len(ace_prefix):] # Step 5: Decode using PUNYCODE - result = label1.decode("punycode") + try: + result = label1.decode("punycode") + except UnicodeDecodeError as exc: + offset = len(ace_prefix) + raise UnicodeDecodeError("idna", label, offset+exc.start, offset+exc.end, exc.reason) # Step 6: Apply ToASCII label2 = ToASCII(result) @@ -144,7 +167,8 @@ def ToUnicode(label): # Step 7: Compare the result of step 6 with the one of step 3 # label2 will already be in lower case. if str(label, "ascii").lower() != str(label2, "ascii"): - raise UnicodeError("IDNA does not round-trip", label, label2) + raise UnicodeDecodeError("idna", label, 0, len(label), + f"IDNA does not round-trip, '{label!r}' != '{label2!r}'") # Step 8: return the result of step 5 return result @@ -156,7 +180,7 @@ def encode(self, input, errors='strict'): if errors != 'strict': # IDNA is quite clear that implementations must be strict - raise UnicodeError("unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return b'', 0 @@ -168,11 +192,16 @@ def encode(self, input, errors='strict'): else: # ASCII name: fast path labels = result.split(b'.') - for label in labels[:-1]: - if not (0 < len(label) < 64): - raise UnicodeError("label empty or too long") - if len(labels[-1]) >= 64: - raise UnicodeError("label too long") + for i, label in enumerate(labels[:-1]): + if len(label) == 0: + offset = sum(len(l) for l in labels[:i]) + i + raise UnicodeEncodeError("idna", input, offset, offset+1, + "label empty") + for i, label in enumerate(labels): + if len(label) >= 64: + offset = sum(len(l) for l in labels[:i]) + i + raise UnicodeEncodeError("idna", input, offset, offset+len(label), + "label too long") return result, len(input) result = bytearray() @@ -182,17 +211,27 @@ def encode(self, input, errors='strict'): del labels[-1] else: trailing_dot = b'' - for label in labels: + for i, label in enumerate(labels): if result: # Join with U+002E result.extend(b'.') - result.extend(ToASCII(label)) + try: + result.extend(ToASCII(label)) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + offset = sum(len(l) for l in labels[:i]) + i + raise UnicodeEncodeError( + "idna", + input, + offset + exc.start, + offset + exc.end, + exc.reason, + ) return bytes(result+trailing_dot), len(input) def decode(self, input, errors='strict'): if errors != 'strict': - raise UnicodeError("Unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return "", 0 @@ -202,7 +241,7 @@ def decode(self, input, errors='strict'): # XXX obviously wrong, see #3232 input = bytes(input) - if ace_prefix not in input: + if ace_prefix not in input.lower(): # Fast path try: return input.decode('ascii'), len(input) @@ -218,8 +257,15 @@ def decode(self, input, errors='strict'): trailing_dot = '' result = [] - for label in labels: - result.append(ToUnicode(label)) + for i, label in enumerate(labels): + try: + u_label = ToUnicode(label) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + offset = sum(len(x) for x in labels[:i]) + len(labels[:i]) + raise UnicodeDecodeError( + "idna", input, offset+exc.start, offset+exc.end, exc.reason) + else: + result.append(u_label) return ".".join(result)+trailing_dot, len(input) @@ -227,7 +273,7 @@ class IncrementalEncoder(codecs.BufferedIncrementalEncoder): def _buffer_encode(self, input, errors, final): if errors != 'strict': # IDNA is quite clear that implementations must be strict - raise UnicodeError("unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return (b'', 0) @@ -251,7 +297,16 @@ def _buffer_encode(self, input, errors, final): # Join with U+002E result.extend(b'.') size += 1 - result.extend(ToASCII(label)) + try: + result.extend(ToASCII(label)) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise UnicodeEncodeError( + "idna", + input, + size + exc.start, + size + exc.end, + exc.reason, + ) size += len(label) result += trailing_dot @@ -261,7 +316,7 @@ def _buffer_encode(self, input, errors, final): class IncrementalDecoder(codecs.BufferedIncrementalDecoder): def _buffer_decode(self, input, errors, final): if errors != 'strict': - raise UnicodeError("Unsupported error handling "+errors) + raise UnicodeError("Unsupported error handling: {errors}") if not input: return ("", 0) @@ -271,7 +326,11 @@ def _buffer_decode(self, input, errors, final): labels = dots.split(input) else: # Must be ASCII string - input = str(input, "ascii") + try: + input = str(input, "ascii") + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise UnicodeDecodeError("idna", input, + exc.start, exc.end, exc.reason) labels = input.split(".") trailing_dot = '' @@ -288,7 +347,18 @@ def _buffer_decode(self, input, errors, final): result = [] size = 0 for label in labels: - result.append(ToUnicode(label)) + try: + u_label = ToUnicode(label) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise UnicodeDecodeError( + "idna", + input.encode("ascii", errors="backslashreplace"), + size + exc.start, + size + exc.end, + exc.reason, + ) + else: + result.append(u_label) if size: size += 1 size += len(label) diff --git a/Lib/encodings/punycode.py b/Lib/encodings/punycode.py index 1c5726447077b1..4622fc8c9206f3 100644 --- a/Lib/encodings/punycode.py +++ b/Lib/encodings/punycode.py @@ -1,4 +1,4 @@ -""" Codec for the Punicode encoding, as specified in RFC 3492 +""" Codec for the Punycode encoding, as specified in RFC 3492 Written by Martin v. Löwis. """ @@ -131,10 +131,11 @@ def decode_generalized_number(extended, extpos, bias, errors): j = 0 while 1: try: - char = ord(extended[extpos]) + char = extended[extpos] except IndexError: if errors == "strict": - raise UnicodeError("incomplete punicode string") + raise UnicodeDecodeError("punycode", extended, extpos, extpos+1, + "incomplete punycode string") return extpos + 1, None extpos += 1 if 0x41 <= char <= 0x5A: # A-Z @@ -142,8 +143,8 @@ def decode_generalized_number(extended, extpos, bias, errors): elif 0x30 <= char <= 0x39: digit = char - 22 # 0x30-26 elif errors == "strict": - raise UnicodeError("Invalid extended code point '%s'" - % extended[extpos-1]) + raise UnicodeDecodeError("punycode", extended, extpos-1, extpos, + f"Invalid extended code point '{extended[extpos-1]}'") else: return extpos, None t = T(j, bias) @@ -155,11 +156,14 @@ def decode_generalized_number(extended, extpos, bias, errors): def insertion_sort(base, extended, errors): - """3.2 Insertion unsort coding""" + """3.2 Insertion sort coding""" + # This function raises UnicodeDecodeError with position in the extended. + # Caller should add the offset. char = 0x80 pos = -1 bias = 72 extpos = 0 + while extpos < len(extended): newpos, delta = decode_generalized_number(extended, extpos, bias, errors) @@ -171,7 +175,9 @@ def insertion_sort(base, extended, errors): char += pos // (len(base) + 1) if char > 0x10FFFF: if errors == "strict": - raise UnicodeError("Invalid character U+%x" % char) + raise UnicodeDecodeError( + "punycode", extended, pos-1, pos, + f"Invalid character U+{char:x}") char = ord('?') pos = pos % (len(base) + 1) base = base[:pos] + chr(char) + base[pos:] @@ -187,11 +193,21 @@ def punycode_decode(text, errors): pos = text.rfind(b"-") if pos == -1: base = "" - extended = str(text, "ascii").upper() + extended = text.upper() else: - base = str(text[:pos], "ascii", errors) - extended = str(text[pos+1:], "ascii").upper() - return insertion_sort(base, extended, errors) + try: + base = str(text[:pos], "ascii", errors) + except UnicodeDecodeError as exc: + raise UnicodeDecodeError("ascii", text, exc.start, exc.end, + exc.reason) from None + extended = text[pos+1:].upper() + try: + return insertion_sort(base, extended, errors) + except UnicodeDecodeError as exc: + offset = pos + 1 + raise UnicodeDecodeError("punycode", text, + offset+exc.start, offset+exc.end, + exc.reason) from None ### Codec APIs @@ -203,7 +219,7 @@ def encode(self, input, errors='strict'): def decode(self, input, errors='strict'): if errors not in ('strict', 'replace', 'ignore'): - raise UnicodeError("Unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") res = punycode_decode(input, errors) return res, len(input) @@ -214,7 +230,7 @@ def encode(self, input, final=False): class IncrementalDecoder(codecs.IncrementalDecoder): def decode(self, input, final=False): if self.errors not in ('strict', 'replace', 'ignore'): - raise UnicodeError("Unsupported error handling "+self.errors) + raise UnicodeError(f"Unsupported error handling: {self.errors}") return punycode_decode(input, self.errors) class StreamWriter(Codec,codecs.StreamWriter): diff --git a/Lib/encodings/undefined.py b/Lib/encodings/undefined.py index 4690288355c710..082771e1c86677 100644 --- a/Lib/encodings/undefined.py +++ b/Lib/encodings/undefined.py @@ -1,6 +1,6 @@ """ Python 'undefined' Codec - This codec will always raise a ValueError exception when being + This codec will always raise a UnicodeError exception when being used. It is intended for use by the site.py file to switch off automatic string to Unicode coercion. diff --git a/Lib/encodings/utf_16.py b/Lib/encodings/utf_16.py index c61248242be8c7..d3b9980026666f 100644 --- a/Lib/encodings/utf_16.py +++ b/Lib/encodings/utf_16.py @@ -64,7 +64,7 @@ def _buffer_decode(self, input, errors, final): elif byteorder == 1: self.decoder = codecs.utf_16_be_decode elif consumed >= 2: - raise UnicodeError("UTF-16 stream does not start with BOM") + raise UnicodeDecodeError("utf-16", input, 0, 2, "Stream does not start with BOM") return (output, consumed) return self.decoder(input, self.errors, final) @@ -138,7 +138,7 @@ def decode(self, input, errors='strict'): elif byteorder == 1: self.decode = codecs.utf_16_be_decode elif consumed>=2: - raise UnicodeError("UTF-16 stream does not start with BOM") + raise UnicodeDecodeError("utf-16", input, 0, 2, "Stream does not start with BOM") return (object, consumed) ### encodings module API diff --git a/Lib/encodings/utf_32.py b/Lib/encodings/utf_32.py index cdf84d14129a62..1924bedbb74c68 100644 --- a/Lib/encodings/utf_32.py +++ b/Lib/encodings/utf_32.py @@ -59,7 +59,7 @@ def _buffer_decode(self, input, errors, final): elif byteorder == 1: self.decoder = codecs.utf_32_be_decode elif consumed >= 4: - raise UnicodeError("UTF-32 stream does not start with BOM") + raise UnicodeDecodeError("utf-32", input, 0, 4, "Stream does not start with BOM") return (output, consumed) return self.decoder(input, self.errors, final) @@ -132,8 +132,8 @@ def decode(self, input, errors='strict'): self.decode = codecs.utf_32_le_decode elif byteorder == 1: self.decode = codecs.utf_32_be_decode - elif consumed>=4: - raise UnicodeError("UTF-32 stream does not start with BOM") + elif consumed >= 4: + raise UnicodeDecodeError("utf-32", input, 0, 4, "Stream does not start with BOM") return (object, consumed) ### encodings module API diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index e8dd253bb55520..645ad998129348 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -10,7 +10,7 @@ __all__ = ["version", "bootstrap"] -_PIP_VERSION = "24.0" +_PIP_VERSION = "24.3.1" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora @@ -205,7 +205,7 @@ def _uninstall_helper(*, verbosity=0): def _main(argv=None): import argparse - parser = argparse.ArgumentParser(prog="python -m ensurepip") + parser = argparse.ArgumentParser() parser.add_argument( "--version", action="version", diff --git a/Lib/ensurepip/_bundled/pip-24.0-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-24.0-py3-none-any.whl deleted file mode 100644 index 2e6aa9d2cb9923..00000000000000 Binary files a/Lib/ensurepip/_bundled/pip-24.0-py3-none-any.whl and /dev/null differ diff --git a/Lib/ensurepip/_bundled/pip-24.3.1-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-24.3.1-py3-none-any.whl new file mode 100644 index 00000000000000..5f1d35be6dd56b Binary files /dev/null and b/Lib/ensurepip/_bundled/pip-24.3.1-py3-none-any.whl differ diff --git a/Lib/ensurepip/_uninstall.py b/Lib/ensurepip/_uninstall.py index b257904328d2f5..4183c28a809008 100644 --- a/Lib/ensurepip/_uninstall.py +++ b/Lib/ensurepip/_uninstall.py @@ -6,7 +6,7 @@ def _main(argv=None): - parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall") + parser = argparse.ArgumentParser() parser.add_argument( "--version", action="version", diff --git a/Lib/enum.py b/Lib/enum.py index 22963cca4466f2..04443471b40bff 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -162,6 +162,11 @@ def _dedent(text): lines[j] = l[i:] return '\n'.join(lines) +class _not_given: + def __repr__(self): + return('') +_not_given = _not_given() + class _auto_null: def __repr__(self): return '_auto_null' @@ -202,7 +207,7 @@ def __get__(self, instance, ownerclass=None): # use previous enum.property return self.fget(instance) elif self._attr_type == 'attr': - # look up previous attibute + # look up previous attribute return getattr(self._cls_type, self.name) elif self._attr_type == 'desc': # use previous descriptor @@ -322,6 +327,8 @@ def __set_name__(self, enum_class, member_name): # to the map, and by-value lookups for this value will be # linear. enum_class._value2member_map_.setdefault(value, enum_member) + if value not in enum_class._hashable_values_: + enum_class._hashable_values_.append(value) except TypeError: # keep track of the value in a list so containment checks are quick enum_class._unhashable_values_.append(value) @@ -335,12 +342,13 @@ class EnumDict(dict): EnumType will use the names found in self._member_names as the enumeration member names. """ - def __init__(self): + def __init__(self, cls_name=None): super().__init__() self._member_names = {} # use a dict -- faster look-up than a list, and keeps insertion order since 3.7 self._last_values = [] self._ignore = [] self._auto_called = False + self._cls_name = cls_name def __setitem__(self, key, value): """ @@ -351,7 +359,7 @@ def __setitem__(self, key, value): Single underscore (sunder) names are reserved. """ - if _is_private(self._cls_name, key): + if self._cls_name is not None and _is_private(self._cls_name, key): # do nothing, name will be a normal attribute pass elif _is_sunder(key): @@ -360,7 +368,10 @@ def __setitem__(self, key, value): '_generate_next_value_', '_numeric_repr_', '_missing_', '_ignore_', '_iter_member_', '_iter_member_by_value_', '_iter_member_by_def_', '_add_alias_', '_add_value_alias_', - ): + # While not in use internally, those are common for pretty + # printing and thus excluded from Enum's reservation of + # _sunder_ names + ) and not key.startswith('_repr_'): raise ValueError( '_sunder_ names, such as %r, are reserved for future Enum use' % (key, ) @@ -396,7 +407,7 @@ def __setitem__(self, key, value): value = value.value elif _is_descriptor(value): pass - elif _is_internal_class(self._cls_name, value): + elif self._cls_name is not None and _is_internal_class(self._cls_name, value): # do nothing, name will be a normal attribute pass else: @@ -434,7 +445,7 @@ def __setitem__(self, key, value): # accepts iterable as multiple arguments? value = t(auto_valued) except TypeError: - # then pass them in singlely + # then pass them in singly value = t(*auto_valued) self._member_names[key] = None if non_auto_store: @@ -468,8 +479,7 @@ def __prepare__(metacls, cls, bases, **kwds): # check that previous enum members do not exist metacls._check_for_existing_members_(cls, bases) # create the namespace dict - enum_dict = EnumDict() - enum_dict._cls_name = cls + enum_dict = EnumDict(cls) # inherit previous flags and _generate_next_value_ function member_type, first_enum = metacls._get_mixins_(cls, bases) if first_enum is not None: @@ -530,7 +540,8 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k classdict['_member_names_'] = [] classdict['_member_map_'] = {} classdict['_value2member_map_'] = {} - classdict['_unhashable_values_'] = [] + classdict['_hashable_values_'] = [] # for comparing with non-hashable types + classdict['_unhashable_values_'] = [] # e.g. frozenset() with set() classdict['_unhashable_values_map_'] = {} classdict['_member_type_'] = member_type # now set the __repr__ for the value @@ -546,22 +557,16 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k classdict['_all_bits_'] = 0 classdict['_inverted_'] = None try: - exc = None classdict['_%s__in_progress' % cls] = True enum_class = super().__new__(metacls, cls, bases, classdict, **kwds) classdict['_%s__in_progress' % cls] = False delattr(enum_class, '_%s__in_progress' % cls) except Exception as e: - # since 3.12 the line "Error calling __set_name__ on '_proto_member' instance ..." - # is tacked on to the error instead of raising a RuntimeError - # recreate the exception to discard - exc = type(e)(str(e)) - exc.__cause__ = e.__cause__ - exc.__context__ = e.__context__ - tb = e.__traceback__ - if exc is not None: - raise exc.with_traceback(tb) - # + # since 3.12 the note "Error calling __set_name__ on '_proto_member' instance ..." + # is tacked on to the error instead of raising a RuntimeError, so discard it + if hasattr(e, '__notes__'): + del e.__notes__ + raise # update classdict with any changes made by __init_subclass__ classdict.update(enum_class.__dict__) # @@ -680,7 +685,7 @@ def __bool__(cls): """ return True - def __call__(cls, value, names=None, *values, module=None, qualname=None, type=None, start=1, boundary=None): + def __call__(cls, value, names=_not_given, *values, module=None, qualname=None, type=None, start=1, boundary=None): """ Either returns an existing member, or creates a new enum class. @@ -709,18 +714,18 @@ def __call__(cls, value, names=None, *values, module=None, qualname=None, type=N """ if cls._member_map_: # simple value lookup if members exist - if names: + if names is not _not_given: value = (value, names) + values return cls.__new__(cls, value) # otherwise, functional API: we're creating a new Enum type - if names is None and type is None: + if names is _not_given and type is None: # no body? no data-type? possibly wrong usage raise TypeError( f"{cls} has no members; specify `names=()` if you meant to create a new, empty, enum" ) return cls._create_( class_name=value, - names=names, + names=None if names is _not_given else names, module=module, qualname=qualname, type=type, @@ -740,7 +745,10 @@ def __contains__(cls, value): try: return value in cls._value2member_map_ except TypeError: - return value in cls._unhashable_values_ + return ( + value in cls._unhashable_values_ # both structures are lists + or value in cls._hashable_values_ + ) def __delattr__(cls, attr): # nicer error message when someone tries to delete an attribute @@ -1083,8 +1091,21 @@ def _add_member_(cls, name, member): setattr(cls, name, member) # now add to _member_map_ (even aliases) cls._member_map_[name] = member - # - cls._member_map_[name] = member + + @property + def __signature__(cls): + from inspect import Parameter, Signature + if cls._member_names_: + return Signature([Parameter('values', Parameter.VAR_POSITIONAL)]) + else: + return Signature([Parameter('new_class_name', Parameter.POSITIONAL_ONLY), + Parameter('names', Parameter.POSITIONAL_OR_KEYWORD), + Parameter('module', Parameter.KEYWORD_ONLY, default=None), + Parameter('qualname', Parameter.KEYWORD_ONLY, default=None), + Parameter('type', Parameter.KEYWORD_ONLY, default=None), + Parameter('start', Parameter.KEYWORD_ONLY, default=1), + Parameter('boundary', Parameter.KEYWORD_ONLY, default=None)]) + EnumMeta = EnumType # keep EnumMeta name for backwards compatibility @@ -1129,13 +1150,6 @@ class Enum(metaclass=EnumType): attributes -- see the documentation for details. """ - @classmethod - def __signature__(cls): - if cls._member_names_: - return '(*values)' - else: - return '(new_class_name, /, names, *, module=None, qualname=None, type=None, start=1, boundary=None)' - def __new__(cls, value): # all enum instances are actually created during class construction # without calling this method; this method is called by the metaclass' @@ -1152,8 +1166,11 @@ def __new__(cls, value): pass except TypeError: # not there, now do long search -- O(n) behavior - for name, values in cls._unhashable_values_map_.items(): - if value in values: + for name, unhashable_values in cls._unhashable_values_map_.items(): + if value in unhashable_values: + return cls[name] + for name, member in cls._member_map_.items(): + if value == member._value_: return cls[name] # still not found -- verify that members exist, in-case somebody got here mistakenly # (such as via super when trying to override __new__) @@ -1194,9 +1211,6 @@ def __new__(cls, value): exc = None ve_exc = None - def __init__(self, *args, **kwds): - pass - def _add_alias_(self, name): self.__class__._add_member_(name, self) @@ -1219,6 +1233,7 @@ def _add_value_alias_(self, value): # to the map, and by-value lookups for this value will be # linear. cls._value2member_map_.setdefault(value, self) + cls._hashable_values_.append(value) except TypeError: # keep track of the value in a list so containment checks are quick cls._unhashable_values_.append(value) @@ -1675,7 +1690,7 @@ def global_flag_repr(self): cls_name = self.__class__.__name__ if self._name_ is None: return "%s.%s(%r)" % (module, cls_name, self._value_) - if _is_single_bit(self): + if _is_single_bit(self._value_): return '%s.%s' % (module, self._name_) if self._boundary_ is not FlagBoundary.KEEP: return '|'.join(['%s.%s' % (module, name) for name in self.name.split('|')]) @@ -1749,6 +1764,7 @@ def convert_class(cls): body['_member_names_'] = member_names = [] body['_member_map_'] = member_map = {} body['_value2member_map_'] = value2member_map = {} + body['_hashable_values_'] = hashable_values = [] body['_unhashable_values_'] = unhashable_values = [] body['_unhashable_values_map_'] = {} body['_member_type_'] = member_type = etype._member_type_ @@ -1797,20 +1813,31 @@ def convert_class(cls): for name, value in attrs.items(): if isinstance(value, auto) and auto.value is _auto_null: value = gnv(name, 1, len(member_names), gnv_last_values) - if value in value2member_map or value in unhashable_values: + # create basic member (possibly isolate value for alias check) + if use_args: + if not isinstance(value, tuple): + value = (value, ) + member = new_member(enum_class, *value) + value = value[0] + else: + member = new_member(enum_class) + if __new__ is None: + member._value_ = value + # now check if alias + try: + contained = value2member_map.get(member._value_) + except TypeError: + contained = None + if member._value_ in unhashable_values or member.value in hashable_values: + for m in enum_class: + if m._value_ == member._value_: + contained = m + break + if contained is not None: # an alias to an existing member - enum_class(value)._add_alias_(name) + contained._add_alias_(name) else: - # create the member - if use_args: - if not isinstance(value, tuple): - value = (value, ) - member = new_member(enum_class, *value) - value = value[0] - else: - member = new_member(enum_class) - if __new__ is None: - member._value_ = value + # finish creating member member._name_ = name member.__objclass__ = enum_class member.__init__(value) @@ -1821,6 +1848,7 @@ def convert_class(cls): else: enum_class._add_member_(name, member) value2member_map[value] = member + hashable_values.append(value) if _is_single_bit(value): # not a multi-bit alias, record in _member_names_ and _flag_mask_ member_names.append(name) @@ -1842,24 +1870,31 @@ def convert_class(cls): if value.value is _auto_null: value.value = gnv(name, 1, len(member_names), gnv_last_values) value = value.value + # create basic member (possibly isolate value for alias check) + if use_args: + if not isinstance(value, tuple): + value = (value, ) + member = new_member(enum_class, *value) + value = value[0] + else: + member = new_member(enum_class) + if __new__ is None: + member._value_ = value + # now check if alias try: - contained = value in value2member_map + contained = value2member_map.get(member._value_) except TypeError: - contained = value in unhashable_values - if contained: + contained = None + if member._value_ in unhashable_values or member._value_ in hashable_values: + for m in enum_class: + if m._value_ == member._value_: + contained = m + break + if contained is not None: # an alias to an existing member - enum_class(value)._add_alias_(name) + contained._add_alias_(name) else: - # create the member - if use_args: - if not isinstance(value, tuple): - value = (value, ) - member = new_member(enum_class, *value) - value = value[0] - else: - member = new_member(enum_class) - if __new__ is None: - member._value_ = value + # finish creating member member._name_ = name member.__objclass__ = enum_class member.__init__(value) @@ -1876,6 +1911,8 @@ def convert_class(cls): # to the map, and by-value lookups for this value will be # linear. enum_class._value2member_map_.setdefault(value, member) + if value not in hashable_values: + hashable_values.append(value) except TypeError: # keep track of the value in a list so containment checks are quick enum_class._unhashable_values_.append(value) @@ -2013,7 +2050,8 @@ def _test_simple_enum(checked_enum, simple_enum): + list(simple_enum._member_map_.keys()) ) for key in set(checked_keys + simple_keys): - if key in ('__module__', '_member_map_', '_value2member_map_', '__doc__'): + if key in ('__module__', '_member_map_', '_value2member_map_', '__doc__', + '__static_attributes__', '__firstlineno__'): # keys known to be different, or very long continue elif key in member_names: diff --git a/Lib/filecmp.py b/Lib/filecmp.py index 6ffc71fc059a80..c5b8d854d77de3 100644 --- a/Lib/filecmp.py +++ b/Lib/filecmp.py @@ -88,7 +88,7 @@ def _do_cmp(f1, f2): class dircmp: """A class that manages the comparison of 2 directories. - dircmp(a, b, ignore=None, hide=None, shallow=True) + dircmp(a, b, ignore=None, hide=None, *, shallow=True) A and B are directories. IGNORE is a list of names to ignore, defaults to DEFAULT_IGNORES. @@ -124,7 +124,7 @@ class dircmp: in common_dirs. """ - def __init__(self, a, b, ignore=None, hide=None, shallow=True): # Initialize + def __init__(self, a, b, ignore=None, hide=None, *, shallow=True): # Initialize self.left = a self.right = b if hide is None: @@ -164,12 +164,14 @@ def phase2(self): # Distinguish files, directories, funnies ok = True try: a_stat = os.stat(a_path) - except OSError: + except (OSError, ValueError): + # See https://github.com/python/cpython/issues/122400 + # for the rationale for protecting against ValueError. # print('Can\'t stat', a_path, ':', why.args[1]) ok = False try: b_stat = os.stat(b_path) - except OSError: + except (OSError, ValueError): # print('Can\'t stat', b_path, ':', why.args[1]) ok = False @@ -201,7 +203,7 @@ def phase4(self): # Find out differences between common subdirectories a_x = os.path.join(self.left, x) b_x = os.path.join(self.right, x) self.subdirs[x] = self.__class__(a_x, b_x, self.ignore, self.hide, - self.shallow) + shallow=self.shallow) def phase4_closure(self): # Recursively call phase4() on subdirectories self.phase4() @@ -285,12 +287,12 @@ def cmpfiles(a, b, common, shallow=True): # Return: # 0 for equal # 1 for different -# 2 for funny cases (can't stat, etc.) +# 2 for funny cases (can't stat, NUL bytes, etc.) # def _cmp(a, b, sh, abs=abs, cmp=cmp): try: return not abs(cmp(a, b, sh)) - except OSError: + except (OSError, ValueError): return 2 diff --git a/Lib/fnmatch.py b/Lib/fnmatch.py index 73acb1fe8d4106..865baea23467ea 100644 --- a/Lib/fnmatch.py +++ b/Lib/fnmatch.py @@ -77,24 +77,30 @@ def translate(pat): There is no way to quote meta-characters. """ - STAR = object() - parts = _translate(pat, STAR, '.') - return _join_translated_parts(parts, STAR) + parts, star_indices = _translate(pat, '*', '.') + return _join_translated_parts(parts, star_indices) +_re_setops_sub = re.compile(r'([&~|])').sub +_re_escape = functools.lru_cache(maxsize=512)(re.escape) -def _translate(pat, STAR, QUESTION_MARK): +def _translate(pat, star, question_mark): res = [] add = res.append + star_indices = [] + i, n = 0, len(pat) while i < n: c = pat[i] i = i+1 if c == '*': + # store the position of the wildcard + star_indices.append(len(res)) + add(star) # compress consecutive `*` into one - if (not res) or res[-1] is not STAR: - add(STAR) + while i < n and pat[i] == '*': + i += 1 elif c == '?': - add(QUESTION_MARK) + add(question_mark) elif c == '[': j = i if j < n and pat[j] == '!': @@ -133,8 +139,6 @@ def _translate(pat, STAR, QUESTION_MARK): # Hyphens that create ranges shouldn't be escaped. stuff = '-'.join(s.replace('\\', r'\\').replace('-', r'\-') for s in chunks) - # Escape set operations (&&, ~~ and ||). - stuff = re.sub(r'([&~|])', r'\\\1', stuff) i = j+1 if not stuff: # Empty range: never match. @@ -143,50 +147,40 @@ def _translate(pat, STAR, QUESTION_MARK): # Negated empty range: match any character. add('.') else: + # Escape set operations (&&, ~~ and ||). + stuff = _re_setops_sub(r'\\\1', stuff) if stuff[0] == '!': stuff = '^' + stuff[1:] elif stuff[0] in ('^', '['): stuff = '\\' + stuff add(f'[{stuff}]') else: - add(re.escape(c)) - assert i == n - return res - - -def _join_translated_parts(inp, STAR): - # Deal with STARs. - res = [] - add = res.append - i, n = 0, len(inp) - # Fixed pieces at the start? - while i < n and inp[i] is not STAR: - add(inp[i]) - i += 1 - # Now deal with STAR fixed STAR fixed ... - # For an interior `STAR fixed` pairing, we want to do a minimal - # .*? match followed by `fixed`, with no possibility of backtracking. - # Atomic groups ("(?>...)") allow us to spell that directly. - # Note: people rely on the undocumented ability to join multiple - # translate() results together via "|" to build large regexps matching - # "one of many" shell patterns. - while i < n: - assert inp[i] is STAR - i += 1 - if i == n: - add(".*") - break - assert inp[i] is not STAR - fixed = [] - while i < n and inp[i] is not STAR: - fixed.append(inp[i]) - i += 1 - fixed = "".join(fixed) - if i == n: - add(".*") - add(fixed) - else: - add(f"(?>.*?{fixed})") + add(_re_escape(c)) assert i == n - res = "".join(res) + return res, star_indices + + +def _join_translated_parts(parts, star_indices): + if not star_indices: + return fr'(?s:{"".join(parts)})\Z' + iter_star_indices = iter(star_indices) + j = next(iter_star_indices) + buffer = parts[:j] # fixed pieces at the start + append, extend = buffer.append, buffer.extend + i = j + 1 + for j in iter_star_indices: + # Now deal with STAR fixed STAR fixed ... + # For an interior `STAR fixed` pairing, we want to do a minimal + # .*? match followed by `fixed`, with no possibility of backtracking. + # Atomic groups ("(?>...)") allow us to spell that directly. + # Note: people rely on the undocumented ability to join multiple + # translate() results together via "|" to build large regexps matching + # "one of many" shell patterns. + append('(?>.*?') + extend(parts[i:j]) + append(')') + i = j + 1 + append('.*') + extend(parts[i:]) + res = ''.join(buffer) return fr'(?s:{res})\Z' diff --git a/Lib/fractions.py b/Lib/fractions.py index f8c6c9c438c737..f0cbc8c2e6c012 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -3,7 +3,6 @@ """Fraction, infinite-precision, rational numbers.""" -from decimal import Decimal import functools import math import numbers @@ -244,7 +243,9 @@ def __new__(cls, numerator=0, denominator=None): self._denominator = numerator.denominator return self - elif isinstance(numerator, (float, Decimal)): + elif (isinstance(numerator, float) or + (not isinstance(numerator, type) and + hasattr(numerator, 'as_integer_ratio'))): # Exact conversion self._numerator, self._denominator = numerator.as_integer_ratio() return self @@ -278,8 +279,8 @@ def __new__(cls, numerator=0, denominator=None): numerator = -numerator else: - raise TypeError("argument should be a string " - "or a Rational instance") + raise TypeError("argument should be a string or a Rational " + "instance or have the as_integer_ratio() method") elif type(numerator) is int is type(denominator): pass # *very* normal case @@ -305,6 +306,28 @@ def __new__(cls, numerator=0, denominator=None): self._denominator = denominator return self + @classmethod + def from_number(cls, number): + """Converts a finite real number to a rational number, exactly. + + Beware that Fraction.from_number(0.3) != Fraction(3, 10). + + """ + if type(number) is int: + return cls._from_coprime_ints(number, 1) + + elif isinstance(number, numbers.Rational): + return cls._from_coprime_ints(number.numerator, number.denominator) + + elif (isinstance(number, float) or + (not isinstance(number, type) and + hasattr(number, 'as_integer_ratio'))): + return cls._from_coprime_ints(*number.as_integer_ratio()) + + else: + raise TypeError("argument should be a Rational instance or " + "have the as_integer_ratio() method") + @classmethod def from_float(cls, f): """Converts a finite float to a rational number, exactly. @@ -668,7 +691,7 @@ def forward(a, b): elif isinstance(b, float): return fallback_operator(float(a), b) elif handle_complex and isinstance(b, complex): - return fallback_operator(complex(a), b) + return fallback_operator(float(a), b) else: return NotImplemented forward.__name__ = '__' + fallback_operator.__name__ + '__' @@ -681,7 +704,7 @@ def reverse(b, a): elif isinstance(a, numbers.Real): return fallback_operator(float(a), float(b)) elif handle_complex and isinstance(a, numbers.Complex): - return fallback_operator(complex(a), complex(b)) + return fallback_operator(complex(a), float(b)) else: return NotImplemented reverse.__name__ = '__r' + fallback_operator.__name__ + '__' @@ -848,7 +871,7 @@ def _mod(a, b): __mod__, __rmod__ = _operator_fallbacks(_mod, operator.mod, False) - def __pow__(a, b): + def __pow__(a, b, modulo=None): """a ** b If b is not an integer, the result will be a float or complex @@ -856,6 +879,8 @@ def __pow__(a, b): result will be rational. """ + if modulo is not None: + return NotImplemented if isinstance(b, numbers.Rational): if b.denominator == 1: power = b.numerator @@ -875,8 +900,10 @@ def __pow__(a, b): # A fractional power will generally produce an # irrational number. return float(a) ** float(b) - else: + elif isinstance(b, (float, complex)): return float(a) ** b + else: + return NotImplemented def __rpow__(b, a): """a ** b""" diff --git a/Lib/ftplib.py b/Lib/ftplib.py index 10c5d1ea08ab11..50771e8c17c250 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -343,7 +343,7 @@ def ntransfercmd(self, cmd, rest=None): connection and the expected size of the transfer. The expected size may be None if it could not be determined. - Optional `rest' argument can be a string that is sent as the + Optional 'rest' argument can be a string that is sent as the argument to a REST command. This is essentially a server marker used to tell the server to skip over any data up to the given marker. diff --git a/Lib/functools.py b/Lib/functools.py index 601cb8e7c0b74b..fd33f0ae479ddc 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -6,23 +6,22 @@ # Written by Nick Coghlan , # Raymond Hettinger , # and Łukasz Langa . -# Copyright (C) 2006-2013 Python Software Foundation. +# Copyright (C) 2006 Python Software Foundation. # See C source code for _functools credits/copyright __all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES', 'total_ordering', 'cache', 'cmp_to_key', 'lru_cache', 'reduce', 'partial', 'partialmethod', 'singledispatch', 'singledispatchmethod', - 'cached_property'] + 'cached_property', 'Placeholder'] from abc import get_cache_token from collections import namedtuple -# import types, weakref # Deferred to single_dispatch() +# import weakref # Deferred to single_dispatch() +from operator import itemgetter from reprlib import recursive_repr +from types import GenericAlias, MethodType, MappingProxyType, UnionType from _thread import RLock -# Avoid importing types, so we can speedup import time -GenericAlias = type(list[int]) - ################################################################################ ### update_wrapper() and wraps() decorator ################################################################################ @@ -31,7 +30,7 @@ # wrapper functions that can handle naive introspection WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__qualname__', '__doc__', - '__annotations__', '__type_params__') + '__annotate__', '__type_params__') WRAPPER_UPDATES = ('__dict__',) def update_wrapper(wrapper, wrapped, @@ -237,14 +236,16 @@ def __ge__(self, other): def reduce(function, sequence, initial=_initial_missing): """ - reduce(function, iterable[, initial], /) -> value - - Apply a function of two arguments cumulatively to the items of a sequence - or iterable, from left to right, so as to reduce the iterable to a single - value. For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates - ((((1+2)+3)+4)+5). If initial is present, it is placed before the items - of the iterable in the calculation, and serves as a default when the - iterable is empty. + reduce(function, iterable, /[, initial]) -> value + + Apply a function of two arguments cumulatively to the items of an iterable, from left to right. + + This effectively reduces the iterable to a single value. If initial is present, + it is placed before the items of the iterable in the calculation, and serves as + a default when the iterable is empty. + + For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) + calculates ((((1 + 2) + 3) + 4) + 5). """ it = iter(sequence) @@ -263,53 +264,135 @@ def reduce(function, sequence, initial=_initial_missing): return value -try: - from _functools import reduce -except ImportError: - pass - ################################################################################ ### partial() argument application ################################################################################ -# Purely functional, no descriptor behaviour -class partial: - """New function with partial application of the given arguments - and keywords. + +class _PlaceholderType: + """The type of the Placeholder singleton. + + Used as a placeholder for partial arguments. """ + __instance = None + __slots__ = () - __slots__ = "func", "args", "keywords", "__dict__", "__weakref__" + def __init_subclass__(cls, *args, **kwargs): + raise TypeError(f"type '{cls.__name__}' is not an acceptable base type") - def __new__(cls, func, /, *args, **keywords): + def __new__(cls): + if cls.__instance is None: + cls.__instance = object.__new__(cls) + return cls.__instance + + def __repr__(self): + return 'Placeholder' + + def __reduce__(self): + return 'Placeholder' + +Placeholder = _PlaceholderType() + +def _partial_prepare_merger(args): + if not args: + return 0, None + nargs = len(args) + order = [] + j = nargs + for i, a in enumerate(args): + if a is Placeholder: + order.append(j) + j += 1 + else: + order.append(i) + phcount = j - nargs + merger = itemgetter(*order) if phcount else None + return phcount, merger + +def _partial_new(cls, func, /, *args, **keywords): + if issubclass(cls, partial): + base_cls = partial if not callable(func): raise TypeError("the first argument must be callable") + else: + base_cls = partialmethod + # func could be a descriptor like classmethod which isn't callable + if not callable(func) and not hasattr(func, "__get__"): + raise TypeError(f"the first argument {func!r} must be a callable " + "or a descriptor") + if args and args[-1] is Placeholder: + raise TypeError("trailing Placeholders are not allowed") + if isinstance(func, base_cls): + pto_phcount = func._phcount + tot_args = func.args + if args: + tot_args += args + if pto_phcount: + # merge args with args of `func` which is `partial` + nargs = len(args) + if nargs < pto_phcount: + tot_args += (Placeholder,) * (pto_phcount - nargs) + tot_args = func._merger(tot_args) + if nargs > pto_phcount: + tot_args += args[pto_phcount:] + phcount, merger = _partial_prepare_merger(tot_args) + else: # works for both pto_phcount == 0 and != 0 + phcount, merger = pto_phcount, func._merger + keywords = {**func.keywords, **keywords} + func = func.func + else: + tot_args = args + phcount, merger = _partial_prepare_merger(tot_args) + + self = object.__new__(cls) + self.func = func + self.args = tot_args + self.keywords = keywords + self._phcount = phcount + self._merger = merger + return self + +def _partial_repr(self): + cls = type(self) + module = cls.__module__ + qualname = cls.__qualname__ + args = [repr(self.func)] + args.extend(map(repr, self.args)) + args.extend(f"{k}={v!r}" for k, v in self.keywords.items()) + return f"{module}.{qualname}({', '.join(args)})" - if hasattr(func, "func"): - args = func.args + args - keywords = {**func.keywords, **keywords} - func = func.func +# Purely functional, no descriptor behaviour +class partial: + """New function with partial application of the given arguments + and keywords. + """ - self = super(partial, cls).__new__(cls) + __slots__ = ("func", "args", "keywords", "_phcount", "_merger", + "__dict__", "__weakref__") - self.func = func - self.args = args - self.keywords = keywords - return self + __new__ = _partial_new + __repr__ = recursive_repr()(_partial_repr) def __call__(self, /, *args, **keywords): + phcount = self._phcount + if phcount: + try: + pto_args = self._merger(self.args + args) + args = args[phcount:] + except IndexError: + raise TypeError("missing positional arguments " + "in 'partial' call; expected " + f"at least {phcount}, got {len(args)}") + else: + pto_args = self.args keywords = {**self.keywords, **keywords} - return self.func(*self.args, *args, **keywords) + return self.func(*pto_args, *args, **keywords) - @recursive_repr() - def __repr__(self): - cls = type(self) - qualname = cls.__qualname__ - module = cls.__module__ - args = [repr(self.func)] - args.extend(repr(x) for x in self.args) - args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items()) - return f"{module}.{qualname}({', '.join(args)})" + def __get__(self, obj, objtype=None): + if obj is None: + return self + return MethodType(self, obj) def __reduce__(self): return type(self), (self.func,), (self.func, self.args, @@ -326,6 +409,10 @@ def __setstate__(self, state): (namespace is not None and not isinstance(namespace, dict))): raise TypeError("invalid partial state") + if args and args[-1] is Placeholder: + raise TypeError("trailing Placeholders are not allowed") + phcount, merger = _partial_prepare_merger(args) + args = tuple(args) # just in case it's a subclass if kwds is None: kwds = {} @@ -338,55 +425,43 @@ def __setstate__(self, state): self.func = func self.args = args self.keywords = kwds + self._phcount = phcount + self._merger = merger + + __class_getitem__ = classmethod(GenericAlias) + try: - from _functools import partial + from _functools import partial, Placeholder, _PlaceholderType except ImportError: pass # Descriptor version -class partialmethod(object): +class partialmethod: """Method descriptor with partial application of the given arguments and keywords. Supports wrapping existing descriptors and handles non-descriptor callables as instance methods. """ - - def __init__(self, func, /, *args, **keywords): - if not callable(func) and not hasattr(func, "__get__"): - raise TypeError("{!r} is not callable or a descriptor" - .format(func)) - - # func could be a descriptor like classmethod which isn't callable, - # so we can't inherit from partial (it verifies func is callable) - if isinstance(func, partialmethod): - # flattening is mandatory in order to place cls/self before all - # other arguments - # it's also more efficient since only one function will be called - self.func = func.func - self.args = func.args + args - self.keywords = {**func.keywords, **keywords} - else: - self.func = func - self.args = args - self.keywords = keywords - - def __repr__(self): - args = ", ".join(map(repr, self.args)) - keywords = ", ".join("{}={!r}".format(k, v) - for k, v in self.keywords.items()) - format_string = "{module}.{cls}({func}, {args}, {keywords})" - return format_string.format(module=self.__class__.__module__, - cls=self.__class__.__qualname__, - func=self.func, - args=args, - keywords=keywords) + __new__ = _partial_new + __repr__ = _partial_repr def _make_unbound_method(self): def _method(cls_or_self, /, *args, **keywords): + phcount = self._phcount + if phcount: + try: + pto_args = self._merger(self.args + args) + args = args[phcount:] + except IndexError: + raise TypeError("missing positional arguments " + "in 'partialmethod' call; expected " + f"at least {phcount}, got {len(args)}") + else: + pto_args = self.args keywords = {**self.keywords, **keywords} - return self.func(cls_or_self, *self.args, *args, **keywords) + return self.func(cls_or_self, *pto_args, *args, **keywords) _method.__isabstractmethod__ = self.__isabstractmethod__ _method.__partialmethod__ = self return _method @@ -673,7 +748,7 @@ def cache(user_function, /): def _c3_merge(sequences): """Merges MROs in *sequences* to a single MRO using the C3 algorithm. - Adapted from https://www.python.org/download/releases/2.3/mro/. + Adapted from https://docs.python.org/3/howto/mro.html. """ result = [] @@ -822,7 +897,7 @@ def singledispatch(func): # There are many programs that use functools without singledispatch, so we # trade-off making singledispatch marginally slower for the benefit of # making start-up of such applications slightly faster. - import types, weakref + import weakref registry = {} dispatch_cache = weakref.WeakKeyDictionary() @@ -853,7 +928,7 @@ def dispatch(cls): def _is_union_type(cls): from typing import get_origin, Union - return get_origin(cls) in {Union, types.UnionType} + return get_origin(cls) in {Union, UnionType} def _is_valid_dispatch_type(cls): if isinstance(cls, type): @@ -878,8 +953,8 @@ def register(cls, func=None): f"Invalid first argument to `register()`. " f"{cls!r} is not a class or union type." ) - ann = getattr(cls, '__annotations__', {}) - if not ann: + ann = getattr(cls, '__annotate__', None) + if ann is None: raise TypeError( f"Invalid first argument to `register()`: {cls!r}. " f"Use either `@register(some_class)` or plain `@register` " @@ -889,13 +964,19 @@ def register(cls, func=None): # only import typing if annotation parsing is necessary from typing import get_type_hints - argname, cls = next(iter(get_type_hints(func).items())) + from annotationlib import Format, ForwardRef + argname, cls = next(iter(get_type_hints(func, format=Format.FORWARDREF).items())) if not _is_valid_dispatch_type(cls): if _is_union_type(cls): raise TypeError( f"Invalid annotation for {argname!r}. " f"{cls!r} not all arguments are classes." ) + elif isinstance(cls, ForwardRef): + raise TypeError( + f"Invalid annotation for {argname!r}. " + f"{cls!r} is an unresolved forward reference." + ) else: raise TypeError( f"Invalid annotation for {argname!r}. " @@ -924,7 +1005,7 @@ def wrapper(*args, **kw): registry[object] = func wrapper.register = register wrapper.dispatch = dispatch - wrapper.registry = types.MappingProxyType(registry) + wrapper.registry = MappingProxyType(registry) wrapper._clear_cache = dispatch_cache.clear update_wrapper(wrapper, func) return wrapper @@ -1038,3 +1119,31 @@ def __get__(self, instance, owner=None): return val __class_getitem__ = classmethod(GenericAlias) + +def _warn_python_reduce_kwargs(py_reduce): + @wraps(py_reduce) + def wrapper(*args, **kwargs): + if 'function' in kwargs or 'sequence' in kwargs: + import os + import warnings + warnings.warn( + 'Calling functools.reduce with keyword arguments ' + '"function" or "sequence" ' + 'is deprecated in Python 3.14 and will be ' + 'forbidden in Python 3.16.', + DeprecationWarning, + skip_file_prefixes=(os.path.dirname(__file__),)) + return py_reduce(*args, **kwargs) + return wrapper + +reduce = _warn_python_reduce_kwargs(reduce) +del _warn_python_reduce_kwargs + +# The import of the C accelerated version of reduce() has been moved +# here due to gh-121676. In Python 3.16, _warn_python_reduce_kwargs() +# should be removed and the import block should be moved back right +# after the definition of reduce(). +try: + from _functools import reduce +except ImportError: + pass diff --git a/Lib/genericpath.py b/Lib/genericpath.py index 1bd5b3897c3af9..ba7b0a13c7f81d 100644 --- a/Lib/genericpath.py +++ b/Lib/genericpath.py @@ -7,8 +7,8 @@ import stat __all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime', - 'getsize', 'isdir', 'isfile', 'islink', 'samefile', 'sameopenfile', - 'samestat'] + 'getsize', 'isdevdrive', 'isdir', 'isfile', 'isjunction', 'islink', + 'lexists', 'samefile', 'sameopenfile', 'samestat'] # Does a path exist? @@ -22,6 +22,15 @@ def exists(path): return True +# Being true for dangling symbolic links is also useful. +def lexists(path): + """Test whether a path exists. Returns True for broken symbolic links""" + try: + os.lstat(path) + except (OSError, ValueError): + return False + return True + # This follows symbolic links, so both islink() and isdir() can be true # for the same path on systems that support symlinks def isfile(path): @@ -57,6 +66,21 @@ def islink(path): return stat.S_ISLNK(st.st_mode) +# Is a path a junction? +def isjunction(path): + """Test whether a path is a junction + Junctions are not supported on the current platform""" + os.fspath(path) + return False + + +def isdevdrive(path): + """Determines whether the specified path is on a Windows Dev Drive. + Dev Drives are not supported on the current platform""" + os.fspath(path) + return False + + def getsize(filename): """Return the size of a file, reported by os.stat().""" return os.stat(filename).st_size diff --git a/Lib/getopt.py b/Lib/getopt.py index 5419d77f5d774e..25f3e2439b3e35 100644 --- a/Lib/getopt.py +++ b/Lib/getopt.py @@ -2,8 +2,8 @@ This module helps scripts to parse the command line arguments in sys.argv. It supports the same conventions as the Unix getopt() -function (including the special meanings of arguments of the form `-' -and `--'). Long options similar to those supported by GNU software +function (including the special meanings of arguments of the form '-' +and '--'). Long options similar to those supported by GNU software may be used as well via an optional third argument. This module provides two functions and an exception: @@ -24,21 +24,14 @@ # TODO for gnu_getopt(): # # - GNU getopt_long_only mechanism -# - allow the caller to specify ordering -# - RETURN_IN_ORDER option -# - GNU extension with '-' as first character of option string -# - optional arguments, specified by double colons # - an option string with a W followed by semicolon should # treat "-W foo" as "--foo" __all__ = ["GetoptError","error","getopt","gnu_getopt"] import os -try: - from gettext import gettext as _ -except ImportError: - # Bootstrapping Python: gettext's dependencies not built yet - def _(s): return s +from gettext import gettext as _ + class GetoptError(Exception): opt = '' @@ -61,12 +54,14 @@ def getopt(args, shortopts, longopts = []): running program. Typically, this means "sys.argv[1:]". shortopts is the string of option letters that the script wants to recognize, with options that require an argument followed by a - colon (i.e., the same format that Unix getopt() uses). If + colon and options that accept an optional argument followed by + two colons (i.e., the same format that Unix getopt() uses). If specified, longopts is a list of strings with the names of the long options which should be supported. The leading '--' characters should not be included in the option name. Options which require an argument should be followed by an equal sign - ('='). + ('='). Options which accept an optional argument should be + followed by an equal sign and question mark ('=?'). The return value consists of two elements: the first is a list of (option, value) pairs; the second is the list of program arguments @@ -105,7 +100,7 @@ def gnu_getopt(args, shortopts, longopts = []): processing options as soon as a non-option argument is encountered. - If the first character of the option string is `+', or if the + If the first character of the option string is '+', or if the environment variable POSIXLY_CORRECT is set, then option processing stops as soon as a non-option argument is encountered. @@ -118,8 +113,13 @@ def gnu_getopt(args, shortopts, longopts = []): else: longopts = list(longopts) + return_in_order = False + if shortopts.startswith('-'): + shortopts = shortopts[1:] + all_options_first = False + return_in_order = True # Allow options after non-option arguments? - if shortopts.startswith('+'): + elif shortopts.startswith('+'): shortopts = shortopts[1:] all_options_first = True elif os.environ.get("POSIXLY_CORRECT"): @@ -133,8 +133,14 @@ def gnu_getopt(args, shortopts, longopts = []): break if args[0][:2] == '--': + if return_in_order and prog_args: + opts.append((None, prog_args)) + prog_args = [] opts, args = do_longs(opts, args[0][2:], longopts, args[1:]) elif args[0][:1] == '-' and args[0] != '-': + if return_in_order and prog_args: + opts.append((None, prog_args)) + prog_args = [] opts, args = do_shorts(opts, args[0][1:], shortopts, args[1:]) else: if all_options_first: @@ -156,7 +162,7 @@ def do_longs(opts, opt, longopts, args): has_arg, opt = long_has_args(opt, longopts) if has_arg: - if optarg is None: + if optarg is None and has_arg != '?': if not args: raise GetoptError(_('option --%s requires argument') % opt, opt) optarg, args = args[0], args[1:] @@ -177,13 +183,19 @@ def long_has_args(opt, longopts): return False, opt elif opt + '=' in possibilities: return True, opt - # No exact match, so better be unique. + elif opt + '=?' in possibilities: + return '?', opt + # Possibilities must be unique to be accepted if len(possibilities) > 1: - # XXX since possibilities contains all valid continuations, might be - # nice to work them into the error msg - raise GetoptError(_('option --%s not a unique prefix') % opt, opt) + raise GetoptError( + _("option --%s not a unique prefix; possible options: %s") + % (opt, ", ".join(possibilities)), + opt, + ) assert len(possibilities) == 1 unique_match = possibilities[0] + if unique_match.endswith('=?'): + return '?', unique_match[:-2] has_arg = unique_match.endswith('=') if has_arg: unique_match = unique_match[:-1] @@ -192,8 +204,9 @@ def long_has_args(opt, longopts): def do_shorts(opts, optstring, shortopts, args): while optstring != '': opt, optstring = optstring[0], optstring[1:] - if short_has_arg(opt, shortopts): - if optstring == '': + has_arg = short_has_arg(opt, shortopts) + if has_arg: + if optstring == '' and has_arg != '?': if not args: raise GetoptError(_('option -%s requires argument') % opt, opt) @@ -207,7 +220,11 @@ def do_shorts(opts, optstring, shortopts, args): def short_has_arg(opt, shortopts): for i in range(len(shortopts)): if opt == shortopts[i] != ':': - return shortopts.startswith(':', i+1) + if not shortopts.startswith(':', i+1): + return False + if shortopts.startswith('::', i+1): + return '?' + return True raise GetoptError(_('option -%s not recognized') % opt, opt) if __name__ == '__main__': diff --git a/Lib/gettext.py b/Lib/gettext.py index 62cff81b7b3d49..4c1f9427459b14 100644 --- a/Lib/gettext.py +++ b/Lib/gettext.py @@ -48,7 +48,6 @@ import operator import os -import re import sys @@ -70,22 +69,26 @@ # https://www.gnu.org/software/gettext/manual/gettext.html#Plural-forms # http://git.savannah.gnu.org/cgit/gettext.git/tree/gettext-runtime/intl/plural.y -_token_pattern = re.compile(r""" - (?P[ \t]+) | # spaces and horizontal tabs - (?P[0-9]+\b) | # decimal integer - (?Pn\b) | # only n is allowed - (?P[()]) | - (?P[-*/%+?:]|[>, - # <=, >=, ==, !=, &&, ||, - # ? : - # unary and bitwise ops - # not allowed - (?P\w+|.) # invalid token - """, re.VERBOSE|re.DOTALL) - +_token_pattern = None def _tokenize(plural): - for mo in re.finditer(_token_pattern, plural): + global _token_pattern + if _token_pattern is None: + import re + _token_pattern = re.compile(r""" + (?P[ \t]+) | # spaces and horizontal tabs + (?P[0-9]+\b) | # decimal integer + (?Pn\b) | # only n is allowed + (?P[()]) | + (?P[-*/%+?:]|[>, + # <=, >=, ==, !=, &&, ||, + # ? : + # unary and bitwise ops + # not allowed + (?P\w+|.) # invalid token + """, re.VERBOSE|re.DOTALL) + + for mo in _token_pattern.finditer(plural): kind = mo.lastgroup if kind == 'WHITESPACES': continue @@ -648,7 +651,7 @@ def npgettext(context, msgid1, msgid2, n): # import gettext # cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR) # _ = cat.gettext -# print _('Hello World') +# print(_('Hello World')) # The resulting catalog object currently don't support access through a # dictionary API, which was supported (but apparently unused) in GNOME diff --git a/Lib/glob.py b/Lib/glob.py index 343be78a73b20a..690ab1b8b9fb1d 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -4,11 +4,14 @@ import os import re import fnmatch +import functools import itertools +import operator import stat import sys -__all__ = ["glob", "iglob", "escape"] + +__all__ = ["glob", "iglob", "escape", "translate"] def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, include_hidden=False): @@ -104,8 +107,8 @@ def _iglob(pathname, root_dir, dir_fd, recursive, dironly, def _glob1(dirname, pattern, dir_fd, dironly, include_hidden=False): names = _listdir(dirname, dir_fd, dironly) - if include_hidden or not _ishidden(pattern): - names = (x for x in names if include_hidden or not _ishidden(x)) + if not (include_hidden or _ishidden(pattern)): + names = (x for x in names if not _ishidden(x)) return fnmatch.filter(names, pattern) def _glob0(dirname, basename, dir_fd, dironly, include_hidden=False): @@ -119,12 +122,19 @@ def _glob0(dirname, basename, dir_fd, dironly, include_hidden=False): return [basename] return [] -# Following functions are not public but can be used by third-party code. +_deprecated_function_message = ( + "{name} is deprecated and will be removed in Python {remove}. Use " + "glob.glob and pass a directory to its root_dir argument instead." +) def glob0(dirname, pattern): + import warnings + warnings._deprecated("glob.glob0", _deprecated_function_message, remove=(3, 15)) return _glob0(dirname, pattern, None, False) def glob1(dirname, pattern): + import warnings + warnings._deprecated("glob.glob1", _deprecated_function_message, remove=(3, 15)) return _glob1(dirname, pattern, None, False) # This helper function recursively yields relative pathnames inside a literal @@ -249,15 +259,16 @@ def escape(pathname): return drive + pathname +_special_parts = ('', '.', '..') _dir_open_flags = os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0) +_no_recurse_symlinks = object() def translate(pat, *, recursive=False, include_hidden=False, seps=None): """Translate a pathname with shell wildcards to a regular expression. If `recursive` is true, the pattern segment '**' will match any number of - path segments; if '**' appears outside its own segment, ValueError will be - raised. + path segments. If `include_hidden` is true, wildcards can match path segments beginning with a dot ('.'). @@ -291,22 +302,245 @@ def translate(pat, *, recursive=False, include_hidden=False, seps=None): for idx, part in enumerate(parts): if part == '*': results.append(one_segment if idx < last_part_idx else one_last_segment) - continue - if recursive: - if part == '**': - if idx < last_part_idx: - if parts[idx + 1] != '**': - results.append(any_segments) - else: - results.append(any_last_segments) - continue - elif '**' in part: - raise ValueError("Invalid pattern: '**' can only be an entire path component") - if part: - if not include_hidden and part[0] in '*?': - results.append(r'(?!\.)') - results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)) - if idx < last_part_idx: - results.append(any_sep) + elif recursive and part == '**': + if idx < last_part_idx: + if parts[idx + 1] != '**': + results.append(any_segments) + else: + results.append(any_last_segments) + else: + if part: + if not include_hidden and part[0] in '*?': + results.append(r'(?!\.)') + results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)[0]) + if idx < last_part_idx: + results.append(any_sep) res = ''.join(results) return fr'(?s:{res})\Z' + + +@functools.lru_cache(maxsize=512) +def _compile_pattern(pat, sep, case_sensitive, recursive=True): + """Compile given glob pattern to a re.Pattern object (observing case + sensitivity).""" + flags = re.NOFLAG if case_sensitive else re.IGNORECASE + regex = translate(pat, recursive=recursive, include_hidden=True, seps=sep) + return re.compile(regex, flags=flags).match + + +class _GlobberBase: + """Abstract class providing shell-style pattern matching and globbing. + """ + + def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False): + self.sep = sep + self.case_sensitive = case_sensitive + self.case_pedantic = case_pedantic + self.recursive = recursive + + # Abstract methods + + @staticmethod + def lexists(path): + """Implements os.path.lexists(). + """ + raise NotImplementedError + + @staticmethod + def scandir(path): + """Implements os.scandir(). + """ + raise NotImplementedError + + @staticmethod + def add_slash(path): + """Returns a path with a trailing slash added. + """ + raise NotImplementedError + + @staticmethod + def concat_path(path, text): + """Implements path concatenation. + """ + raise NotImplementedError + + # High-level methods + + def compile(self, pat): + return _compile_pattern(pat, self.sep, self.case_sensitive, self.recursive) + + def selector(self, parts): + """Returns a function that selects from a given path, walking and + filtering according to the glob-style pattern parts in *parts*. + """ + if not parts: + return self.select_exists + part = parts.pop() + if self.recursive and part == '**': + selector = self.recursive_selector + elif part in _special_parts: + selector = self.special_selector + elif not self.case_pedantic and magic_check.search(part) is None: + selector = self.literal_selector + else: + selector = self.wildcard_selector + return selector(part, parts) + + def special_selector(self, part, parts): + """Returns a function that selects special children of the given path. + """ + select_next = self.selector(parts) + + def select_special(path, exists=False): + path = self.concat_path(self.add_slash(path), part) + return select_next(path, exists) + return select_special + + def literal_selector(self, part, parts): + """Returns a function that selects a literal descendant of a path. + """ + + # Optimization: consume and join any subsequent literal parts here, + # rather than leaving them for the next selector. This reduces the + # number of string concatenation operations and calls to add_slash(). + while parts and magic_check.search(parts[-1]) is None: + part += self.sep + parts.pop() + + select_next = self.selector(parts) + + def select_literal(path, exists=False): + path = self.concat_path(self.add_slash(path), part) + return select_next(path, exists=False) + return select_literal + + def wildcard_selector(self, part, parts): + """Returns a function that selects direct children of a given path, + filtering by pattern. + """ + + match = None if part == '*' else self.compile(part) + dir_only = bool(parts) + if dir_only: + select_next = self.selector(parts) + + def select_wildcard(path, exists=False): + try: + # We must close the scandir() object before proceeding to + # avoid exhausting file descriptors when globbing deep trees. + with self.scandir(path) as scandir_it: + entries = list(scandir_it) + except OSError: + pass + else: + prefix = self.add_slash(path) + for entry in entries: + if match is None or match(entry.name): + if dir_only: + try: + if not entry.is_dir(): + continue + except OSError: + continue + entry_path = self.concat_path(prefix, entry.name) + if dir_only: + yield from select_next(entry_path, exists=True) + else: + yield entry_path + return select_wildcard + + def recursive_selector(self, part, parts): + """Returns a function that selects a given path and all its children, + recursively, filtering by pattern. + """ + # Optimization: consume following '**' parts, which have no effect. + while parts and parts[-1] == '**': + parts.pop() + + # Optimization: consume and join any following non-special parts here, + # rather than leaving them for the next selector. They're used to + # build a regular expression, which we use to filter the results of + # the recursive walk. As a result, non-special pattern segments + # following a '**' wildcard don't require additional filesystem access + # to expand. + follow_symlinks = self.recursive is not _no_recurse_symlinks + if follow_symlinks: + while parts and parts[-1] not in _special_parts: + part += self.sep + parts.pop() + + match = None if part == '**' else self.compile(part) + dir_only = bool(parts) + select_next = self.selector(parts) + + def select_recursive(path, exists=False): + path = self.add_slash(path) + match_pos = len(str(path)) + if match is None or match(str(path), match_pos): + yield from select_next(path, exists) + stack = [path] + while stack: + yield from select_recursive_step(stack, match_pos) + + def select_recursive_step(stack, match_pos): + path = stack.pop() + try: + # We must close the scandir() object before proceeding to + # avoid exhausting file descriptors when globbing deep trees. + with self.scandir(path) as scandir_it: + entries = list(scandir_it) + except OSError: + pass + else: + prefix = self.add_slash(path) + for entry in entries: + is_dir = False + try: + if entry.is_dir(follow_symlinks=follow_symlinks): + is_dir = True + except OSError: + pass + + if is_dir or not dir_only: + entry_path = self.concat_path(prefix, entry.name) + if match is None or match(str(entry_path), match_pos): + if dir_only: + yield from select_next(entry_path, exists=True) + else: + # Optimization: directly yield the path if this is + # last pattern part. + yield entry_path + if is_dir: + stack.append(entry_path) + + return select_recursive + + def select_exists(self, path, exists=False): + """Yields the given path, if it exists. + """ + if exists: + # Optimization: this path is already known to exist, e.g. because + # it was returned from os.scandir(), so we skip calling lstat(). + yield path + elif self.lexists(path): + yield path + + +class _StringGlobber(_GlobberBase): + """Provides shell-style pattern matching and globbing for string paths. + """ + lexists = staticmethod(os.path.lexists) + scandir = staticmethod(os.scandir) + concat_path = operator.add + + if os.name == 'nt': + @staticmethod + def add_slash(pathname): + tail = os.path.splitroot(pathname)[2] + if not tail or tail[-1] in '\\/': + return pathname + return f'{pathname}\\' + else: + @staticmethod + def add_slash(pathname): + if not pathname or pathname[-1] == '/': + return pathname + return f'{pathname}/' diff --git a/Lib/graphlib.py b/Lib/graphlib.py index 636545648e12d3..1438a5fc54b9cb 100644 --- a/Lib/graphlib.py +++ b/Lib/graphlib.py @@ -103,7 +103,7 @@ def prepare(self): # nodes as possible before cycles block more progress cycle = self._find_cycle() if cycle: - raise CycleError(f"nodes are in a cycle", cycle) + raise CycleError("nodes are in a cycle", cycle) def get_ready(self): """Return a tuple of all the nodes that are ready. diff --git a/Lib/gzip.py b/Lib/gzip.py index fda93e0261e028..1a3c82ce7e0711 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -15,7 +15,8 @@ FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16 -READ, WRITE = 1, 2 +READ = 'rb' +WRITE = 'wb' _COMPRESS_LEVEL_FAST = 1 _COMPRESS_LEVEL_TRADEOFF = 6 @@ -178,9 +179,10 @@ def __init__(self, filename=None, mode=None, and 9 is slowest and produces the most compression. 0 is no compression at all. The default is 9. - The mtime argument is an optional numeric timestamp to be written - to the last modification time field in the stream when compressing. - If omitted or None, the current time is used. + The optional mtime argument is the timestamp requested by gzip. The time + is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. + If mtime is omitted or None, the current time is used. Use mtime = 0 + to generate a compressed stream that does not depend on creation time. """ @@ -578,43 +580,21 @@ def _rewind(self): self._new_member = True -def _create_simple_gzip_header(compresslevel: int, - mtime = None) -> bytes: - """ - Write a simple gzip header with no extra fields. - :param compresslevel: Compresslevel used to determine the xfl bytes. - :param mtime: The mtime (must support conversion to a 32-bit integer). - :return: A bytes object representing the gzip header. - """ - if mtime is None: - mtime = time.time() - if compresslevel == _COMPRESS_LEVEL_BEST: - xfl = 2 - elif compresslevel == _COMPRESS_LEVEL_FAST: - xfl = 4 - else: - xfl = 0 - # Pack ID1 and ID2 magic bytes, method (8=deflate), header flags (no extra - # fields added to header), mtime, xfl and os (255 for unknown OS). - return struct.pack("= 0: if self.length is not None and amt > self.length: # clip the read to the "end of response" amt = self.length @@ -590,6 +590,8 @@ def _get_chunk_left(self): def _read_chunked(self, amt=None): assert self.chunked != _UNKNOWN + if amt is not None and amt < 0: + amt = None value = [] try: while (chunk_left := self._get_chunk_left()) is not None: @@ -1025,7 +1027,7 @@ def close(self): response.close() def send(self, data): - """Send `data' to the server. + """Send 'data' to the server. ``data`` can be a string object, a bytes object, an array object, a file-like object that supports a .read() method, or an iterable object. """ @@ -1137,10 +1139,10 @@ def putrequest(self, method, url, skip_host=False, skip_accept_encoding=False): """Send a request to the server. - `method' specifies an HTTP request method, e.g. 'GET'. - `url' specifies the object being requested, e.g. '/index.html'. - `skip_host' if True does not add automatically a 'Host:' header - `skip_accept_encoding' if True does not add automatically an + 'method' specifies an HTTP request method, e.g. 'GET'. + 'url' specifies the object being requested, e.g. '/index.html'. + 'skip_host' if True does not add automatically a 'Host:' header + 'skip_accept_encoding' if True does not add automatically an 'Accept-Encoding:' header """ diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index bd89370e16831e..fb0fd2e97999af 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -1986,7 +1986,7 @@ class MozillaCookieJar(FileCookieJar): This class differs from CookieJar only in the format it uses to save and load cookies to and from a file. This class uses the Mozilla/Netscape - `cookies.txt' format. curl and lynx use this file format, too. + 'cookies.txt' format. curl and lynx use this file format, too. Don't expect cookies saved while the browser is running to be noticed by the browser (in fact, Mozilla on unix will overwrite your saved cookies if diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index 35ac2dc6ae280c..694b1b09a0567c 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -184,8 +184,13 @@ def _quote(str): return '"' + str.translate(_Translator) + '"' -_OctalPatt = re.compile(r"\\[0-3][0-7][0-7]") -_QuotePatt = re.compile(r"[\\].") +_unquote_sub = re.compile(r'\\(?:([0-3][0-7][0-7])|(.))').sub + +def _unquote_replace(m): + if m[1]: + return chr(int(m[1], 8)) + else: + return m[2] def _unquote(str): # If there aren't any doublequotes, @@ -205,36 +210,13 @@ def _unquote(str): # \012 --> \n # \" --> " # - i = 0 - n = len(str) - res = [] - while 0 <= i < n: - o_match = _OctalPatt.search(str, i) - q_match = _QuotePatt.search(str, i) - if not o_match and not q_match: # Neither matched - res.append(str[i:]) - break - # else: - j = k = -1 - if o_match: - j = o_match.start(0) - if q_match: - k = q_match.start(0) - if q_match and (not o_match or k < j): # QuotePatt matched - res.append(str[i:k]) - res.append(str[k+1]) - i = k + 2 - else: # OctalPatt matched - res.append(str[i:j]) - res.append(chr(int(str[j+1:j+4], 8))) - i = j + 4 - return _nulljoin(res) + return _unquote_sub(_unquote_replace, str) # The _getdate() routine is used to set the expiration time in the cookie's HTTP # header. By default, _getdate() returns the current time in the appropriate # "expires" format for a Set-Cookie header. The one optional argument is an # offset from now, in seconds. For example, an offset of -3600 means "one hour -# ago". The offset may be a floating point number. +# ago". The offset may be a floating-point number. # _weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] @@ -282,17 +264,19 @@ class Morsel(dict): "httponly" : "HttpOnly", "version" : "Version", "samesite" : "SameSite", + "partitioned": "Partitioned", } - _flags = {'secure', 'httponly'} + _reserved_defaults = dict.fromkeys(_reserved, "") + + _flags = {'secure', 'httponly', 'partitioned'} def __init__(self): # Set defaults self._key = self._value = self._coded_value = None # Set default attributes - for key in self._reserved: - dict.__setitem__(self, key, "") + dict.update(self, self._reserved_defaults) @property def key(self): @@ -442,9 +426,11 @@ def OutputString(self, attrs=None): ( # Optional group: there may not be a value. \s*=\s* # Equal Sign (?P # Start of group 'val' - "(?:[^\\"]|\\.)*" # Any doublequoted string + "(?:[^\\"]|\\.)*" # Any double-quoted string | # or - \w{3},\s[\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Special case for "expires" attr + # Special case for "expires" attr + (\w{3,6}day|\w{3}),\s # Day of the week or abbreviated day + [\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Date and time in specific format | # or [""" + _LegalValueChars + r"""]* # Any word or empty string ) # End of group 'val' diff --git a/Lib/http/server.py b/Lib/http/server.py index ee7a9b6aa55b88..a90c8d34c394db 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -99,7 +99,7 @@ import posixpath import select import shutil -import socket # For gethostbyaddr() +import socket import socketserver import sys import time @@ -114,6 +114,11 @@ + Error response @@ -129,7 +134,8 @@ class HTTPServer(socketserver.TCPServer): - allow_reuse_address = 1 # Seems to make sense in testing environment + allow_reuse_address = True # Seems to make sense in testing environment + allow_reuse_port = True def server_bind(self): """Override server_bind to store the server name.""" @@ -803,6 +809,7 @@ def list_directory(self, path): r.append('') r.append('') r.append(f'') + r.append('') r.append(f'{title}\n') r.append(f'\n

{title}

') r.append('
\n
    ') @@ -897,7 +904,7 @@ def guess_type(self, path): ext = ext.lower() if ext in self.extensions_map: return self.extensions_map[ext] - guess, _ = mimetypes.guess_type(path) + guess, _ = mimetypes.guess_file_type(path) if guess: return guess return 'application/octet-stream' diff --git a/Lib/idlelib/HISTORY.txt b/Lib/idlelib/HISTORY.txt index 731fabd185fbbf..a601b25b5f838f 100644 --- a/Lib/idlelib/HISTORY.txt +++ b/Lib/idlelib/HISTORY.txt @@ -277,7 +277,7 @@ Command to format a paragraph. Debug menu: JIT (Just-In-Time) stack viewer toggle -- if set, the stack viewer -automaticall pops up when you get a traceback. +automatically pops up when you get a traceback. Windows menu: diff --git a/Lib/idlelib/Icons/README.txt b/Lib/idlelib/Icons/README.txt index d91c4d5d8d8cfa..e245bc0b26e121 100644 --- a/Lib/idlelib/Icons/README.txt +++ b/Lib/idlelib/Icons/README.txt @@ -1,13 +1,51 @@ -The IDLE icons are from https://bugs.python.org/issue1490384 +IDLE-PYTHON LOGOS -Created by Andrew Clover. +These are sent to tk on Windows, *NIX, and non-Aqua macOS +in pyshell following "# set application icon". -The original sources are available from Andrew's website: + +2006?: Andrew Clover made variously sized python icons for win23. https://www.doxdesk.com/software/py/pyicons.html -Various different formats and sizes are available at this GitHub Pull Request: -https://github.com/python/cpython/pull/17473 +2006: 16, 32, and 48 bit .png versions were copied to CPython +as Python application icons, maybe in PC/icons/py.ico. +https://github.com/python/cpython/issues/43372 + +2014: They were copied (perhaps a bit revised) to idlelib/Icons. +https://github.com/python/cpython/issues/64605 +.gif versions were also added. + +2020: Add Clover's 256-bit image. +https://github.com/python/cpython/issues/82620 +Other fixups were done. + +The idle.ico file used for Windows was created with ImageMagick: + $ convert idle_16.png idle_32.png idle_48.png idle_256.png idle.ico +** This needs redoing whenever files are changed. +?? Do Start, Desktop, and Taskbar use idlelib/Icons files? + +Issue added Windows Store PC/icons/idlex44.png and .../idlex150.png. +https://github.com/python/cpython/pull/22817 +?? Should these be updated with major changes? + +2022: Optimize .png images in CPython repository with external program. +https://github.com/python/cpython/pull/21348 +idle.ico (and idlex##) were not updated. + +The idlexx.gif files are only needed for *nix running tcl/tk 8.5. +As of 2022, this was known true for 1 'major' Linux distribution. +(Same would be true for any non-Aqua macOS with 8.5, but now none?) +Can be deleted when we require 8.6 or it is known always used. + +Future: Derivatives of Python logo should be submitted for approval. +PSF Trademark Working Group / Committee psf-trademarks@python.org +https://www.python.org/community/logos/ # Original files +https://www.python.org/psf/trademarks-faq/ +https://www.python.org/psf/trademarks/ # Usage. + + +OTHER GIFS: These are used by browsers using idlelib.tree. +At least some will not be used when tree is replaced by ttk.Treeview. -The idle.ico file was created with ImageMagick: - $ convert idle_16.png idle_32.png idle_48.png idle_256.png idle.ico +Edited 2024 August 26 by TJR. diff --git a/Lib/idlelib/News3.txt b/Lib/idlelib/News3.txt index 241b1f48e5c1d8..37ff93f9866e3c 100644 --- a/Lib/idlelib/News3.txt +++ b/Lib/idlelib/News3.txt @@ -4,6 +4,18 @@ Released on 2024-10-xx ========================= +gh-120083: Add explicit black IDLE Hovertip foreground color needed for +recent macOS. Fixes Sonoma showing unreadable white on pale yellow. +Patch by John Riggles. + +gh-122482: Change About IDLE to direct users to discuss.python.org +instead of the now unused idle-dev email and mailing list. + +gh-78889: Stop Shell freezes by blocking user access to non-method +sys.stdout.shell attributes, which are all private. + +gh-78955: Use user-selected color theme for Help => IDLE Doc. + gh-96905: In idlelib code, stop redefining built-ins 'dict' and 'object'. gh-72284: Improve the lists of features, editor key bindings, @@ -566,14 +578,14 @@ bpo-33679: Enable theme-specific color configuration for Code Context. color setting, default or custom, on the extensions tab, that applied to all themes.) For built-in themes, the foreground is the same as normal text and the background is a contrasting gray. Context colors for -custom themes are set on the Hightlights tab along with other colors. +custom themes are set on the Highlights tab along with other colors. When one starts IDLE from a console and loads a custom theme without definitions for 'context', one will see a warning message on the console. bpo-33642: Display up to maxlines non-blank lines for Code Context. If there is no current context, show a single blank line. (Previously, -the Code Contex had numlines lines, usually with some blank.) The use +the Code Context had numlines lines, usually with some blank.) The use of a new option, 'maxlines' (default 15), avoids possible interference with user settings of the old option, 'numlines' (default 3). @@ -727,7 +739,7 @@ not affect their keyset-specific customization after 3.6.3. and vice versa. Initial patch by Charles Wohlganger, revised by Terry Jan Reedy. -bpo-31051: Rearrange condigdialog General tab. +bpo-31051: Rearrange configdialog General tab. Sort non-Help options into Window (Shell+Editor) and Editor (only). Leave room for the addition of new options. Patch by Terry Jan Reedy. diff --git a/Lib/idlelib/TODO.txt b/Lib/idlelib/TODO.txt index e2f1ac0f274001..41b86b0c6d5bbd 100644 --- a/Lib/idlelib/TODO.txt +++ b/Lib/idlelib/TODO.txt @@ -179,7 +179,7 @@ it -- i.e. you can only edit the current command, and the cursor can't escape from the command area. (Albert Brandl) - Set X11 class to "idle/Idle", set icon and title to something -beginning with "idle" -- for window manangers. (Randall Hopper) +beginning with "idle" -- for window managers. (Randall Hopper) - Config files editable through a preferences dialog. (me) DONE diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py index 92992fd9cce9cd..d10c88a43f9231 100644 --- a/Lib/idlelib/config.py +++ b/Lib/idlelib/config.py @@ -158,8 +158,9 @@ def __init__(self, _utest=False): self.defaultCfg = {} self.userCfg = {} self.cfg = {} # TODO use to select userCfg vs defaultCfg + + # See https://bugs.python.org/issue4630#msg356516 for following. # self.blink_off_time = ['insertofftime'] - # See https:/bugs.python.org/issue4630, msg356516. if not _utest: self.CreateConfigHandlers() @@ -599,7 +600,7 @@ def GetCoreKeys(self, keySetName=None): """ # TODO: = dict(sorted([(v-event, keys), ...]))? keyBindings={ - # vitual-event: list of key events. + # virtual-event: list of key events. '<>': ['', ''], '<>': ['', ''], '<>': ['', ''], diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index eedf97bf74fe6a..4d2adb48570d49 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -111,7 +111,7 @@ def create_widgets(self): load_configs: Load pages except for extensions. activate_config_changes: Tell editors to reload. """ - self.frame = frame = Frame(self, padding="5px") + self.frame = frame = Frame(self, padding=5) self.frame.grid(sticky="nwes") self.note = note = Notebook(frame) self.extpage = ExtPage(note) diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 7bfa0932500d81..c76db20c58792d 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -914,7 +914,7 @@ def RemoveKeybindings(self): def ApplyKeybindings(self): """Apply the virtual, configurable keybindings. - Alse update hotkeys to current keyset. + Also update hotkeys to current keyset. """ # Called from configdialog.activate_config_changes. self.mainmenu.default_keydefs = keydefs = idleConf.GetCurrentKeySet() diff --git a/Lib/idlelib/extend.txt b/Lib/idlelib/extend.txt index b482f76c4fb0f7..2522758ceb4f70 100644 --- a/Lib/idlelib/extend.txt +++ b/Lib/idlelib/extend.txt @@ -52,7 +52,7 @@ should probably be refined in the future.) Extensions are not required to define menu entries for all the events they implement. (They are also not required to create keybindings, but in that -case there must be empty bindings in cofig-extensions.def) +case there must be empty bindings in config-extensions.def) Here is a partial example from zzdummy.py: diff --git a/Lib/idlelib/grep.py b/Lib/idlelib/grep.py index ef14349960bfa2..42048ff2395fe1 100644 --- a/Lib/idlelib/grep.py +++ b/Lib/idlelib/grep.py @@ -190,7 +190,7 @@ def grep_it(self, prog, path): def _grep_dialog(parent): # htest # - from tkinter import Toplevel, Text, SEL, END + from tkinter import Toplevel, Text, SEL from tkinter.ttk import Frame, Button from idlelib.pyshell import PyShellFileList diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index 827d230b54e159..2a4adc6a4d395f 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -5,7 +5,7 @@ - IDLE — Python 3.13.0a2 documentation + IDLE — Python 3.14.0a0 documentation @@ -18,7 +18,7 @@ @@ -26,6 +26,7 @@ + @@ -45,6 +46,8 @@ + + @@ -184,7 +187,7 @@

    Navigation

  • - 3.13.0a2 Documentation » + 3.14.0a0 Documentation »
  • @@ -554,7 +557,7 @@

    Key bindingsControl key on Windows and -Unix and the Command key on macOS. (And all such dicussions +Unix and the Command key on macOS. (And all such discussions assume that the keys have not been re-bound to something else.)

    @@ -694,7 +697,7 @@

    Shell window -
  • C-c attemps to interrupt statement execution (but may fail).

  • +
  • C-c attempts to interrupt statement execution (but may fail).

  • C-d closes Shell if typed at a >>> prompt.

  • Alt-p and Alt-n (C-p and C-n on macOS) retrieve to the current prompt the previous or next previously @@ -1136,7 +1139,7 @@

    Navigation

  • - 3.13.0a2 Documentation » + 3.14.0a0 Documentation »
  • @@ -1180,7 +1183,7 @@

    Navigation



    - Last updated on Jan 17, 2024 (06:57 UTC). + Last updated on Oct 14, 2024 (20:27 UTC). Found a bug?
    diff --git a/Lib/idlelib/help.py b/Lib/idlelib/help.py index bdf4b2b29f11a2..d8613b2eadd6aa 100644 --- a/Lib/idlelib/help.py +++ b/Lib/idlelib/help.py @@ -33,6 +33,7 @@ from tkinter import font as tkfont from idlelib.config import idleConf +from idlelib.colorizer import color_config ## About IDLE ## @@ -177,14 +178,16 @@ def __init__(self, parent, filename): normalfont = self.findfont(['TkDefaultFont', 'arial', 'helvetica']) fixedfont = self.findfont(['TkFixedFont', 'monaco', 'courier']) + color_config(self) self['font'] = (normalfont, 12) self.tag_configure('em', font=(normalfont, 12, 'italic')) self.tag_configure('h1', font=(normalfont, 20, 'bold')) self.tag_configure('h2', font=(normalfont, 18, 'bold')) self.tag_configure('h3', font=(normalfont, 15, 'bold')) - self.tag_configure('pre', font=(fixedfont, 12), background='#f6f6ff') + self.tag_configure('pre', font=(fixedfont, 12)) + preback = self['selectbackground'] self.tag_configure('preblock', font=(fixedfont, 10), lmargin1=25, - borderwidth=1, relief='solid', background='#eeffcc') + background=preback) self.tag_configure('l1', lmargin1=25, lmargin2=25) self.tag_configure('l2', lmargin1=50, lmargin2=50) self.tag_configure('l3', lmargin1=75, lmargin2=75) diff --git a/Lib/idlelib/help_about.py b/Lib/idlelib/help_about.py index aa1c352897f9e7..81c65f6264e7b9 100644 --- a/Lib/idlelib/help_about.py +++ b/Lib/idlelib/help_about.py @@ -85,15 +85,18 @@ def create_widgets(self): byline = Label(frame_background, text=byline_text, justify=LEFT, fg=self.fg, bg=self.bg) byline.grid(row=2, column=0, sticky=W, columnspan=3, padx=10, pady=5) - email = Label(frame_background, text='email: idle-dev@python.org', - justify=LEFT, fg=self.fg, bg=self.bg) - email.grid(row=6, column=0, columnspan=2, sticky=W, padx=10, pady=0) + + forums_url = "https://discuss.python.org" + forums = Label(frame_background, text="Python forums: "+forums_url, + justify=LEFT, fg=self.fg, bg=self.bg) + forums.grid(row=6, column=0, sticky=W, padx=10, pady=0) + forums.bind("", lambda event: webbrowser.open(forums_url)) docs_url = ("https://docs.python.org/%d.%d/library/idle.html" % sys.version_info[:2]) docs = Label(frame_background, text=docs_url, justify=LEFT, fg=self.fg, bg=self.bg) docs.grid(row=7, column=0, columnspan=2, sticky=W, padx=10, pady=0) - docs.bind("", lambda event: webbrowser.open(docs['text'])) + docs.bind("", lambda event: webbrowser.open(docs_url)) Frame(frame_background, borderwidth=1, relief=SUNKEN, height=2, bg=self.bg).grid(row=8, column=0, sticky=EW, @@ -123,9 +126,7 @@ def create_widgets(self): height=2, bg=self.bg).grid(row=11, column=0, sticky=EW, columnspan=3, padx=5, pady=5) - idle = Label(frame_background, - text='IDLE', - fg=self.fg, bg=self.bg) + idle = Label(frame_background, text='IDLE', fg=self.fg, bg=self.bg) idle.grid(row=12, column=0, sticky=W, padx=10, pady=0) idle_buttons = Frame(frame_background, bg=self.bg) idle_buttons.grid(row=13, column=0, columnspan=3, sticky=NSEW) diff --git a/Lib/idlelib/idle_test/example_stub.pyi b/Lib/idlelib/idle_test/example_stub.pyi index 17b58010a9d8de..abcdbc17529974 100644 --- a/Lib/idlelib/idle_test/example_stub.pyi +++ b/Lib/idlelib/idle_test/example_stub.pyi @@ -1,4 +1,4 @@ -" Example to test recognition of .pyi file as Python source code. +# An example file to test recognition of a .pyi file as Python source code. class Example: def method(self, argument1: str, argument2: list[int]) -> None: ... diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index 5099d093382445..2773ed7ce614b5 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -98,8 +98,8 @@ def test_click_help(self): dialog.buttons['Help'].invoke() title, contents = view.kwds['title'], view.kwds['contents'] self.assertEqual(title, 'Help for IDLE preferences') - self.assertTrue(contents.startswith('When you click') and - contents.endswith('a different name.\n')) + self.assertStartsWith(contents, 'When you click') + self.assertEndsWith(contents,'a different name.\n') class FontPageTest(unittest.TestCase): diff --git a/Lib/idlelib/idle_test/test_debugger.py b/Lib/idlelib/idle_test/test_debugger.py index d1c9638dd5d711..9ca3b332648b31 100644 --- a/Lib/idlelib/idle_test/test_debugger.py +++ b/Lib/idlelib/idle_test/test_debugger.py @@ -256,7 +256,7 @@ def test_init(self): flist = None master_window = self.root sv = debugger.StackViewer(master_window, flist, gui) - self.assertTrue(hasattr(sv, 'stack')) + self.assertHasAttr(sv, 'stack') def test_load_stack(self): # Test the .load_stack() method against a fixed test stack. diff --git a/Lib/idlelib/idle_test/test_grep.py b/Lib/idlelib/idle_test/test_grep.py index a0b5b69171879c..d67dba76911fcf 100644 --- a/Lib/idlelib/idle_test/test_grep.py +++ b/Lib/idlelib/idle_test/test_grep.py @@ -143,7 +143,7 @@ def test_found(self): self.assertIn(pat, lines[0]) self.assertIn('py: 1:', lines[1]) # line number 1 self.assertIn('2', lines[3]) # hits found 2 - self.assertTrue(lines[4].startswith('(Hint:')) + self.assertStartsWith(lines[4], '(Hint:') class Default_commandTest(unittest.TestCase): diff --git a/Lib/idlelib/idle_test/test_multicall.py b/Lib/idlelib/idle_test/test_multicall.py index b3a3bfb88f9c31..67f28db6b0875c 100644 --- a/Lib/idlelib/idle_test/test_multicall.py +++ b/Lib/idlelib/idle_test/test_multicall.py @@ -27,7 +27,7 @@ def tearDownClass(cls): def test_creator(self): mc = self.mc self.assertIs(multicall._multicall_dict[Text], mc) - self.assertTrue(issubclass(mc, Text)) + self.assertIsSubclass(mc, Text) mc2 = multicall.MultiCallCreator(Text) self.assertIs(mc, mc2) diff --git a/Lib/idlelib/idle_test/test_outwin.py b/Lib/idlelib/idle_test/test_outwin.py index d6e85ad674417c..81f4aad7e95e95 100644 --- a/Lib/idlelib/idle_test/test_outwin.py +++ b/Lib/idlelib/idle_test/test_outwin.py @@ -1,6 +1,7 @@ "Test outwin, coverage 76%." from idlelib import outwin +import sys import unittest from test.support import requires from tkinter import Tk, Text @@ -18,6 +19,10 @@ def setUpClass(cls): root.withdraw() w = cls.window = outwin.OutputWindow(None, None, None, root) cls.text = w.text = Text(root) + if sys.platform == 'darwin': # Issue 112938 + cls.text.update = cls.text.update_idletasks + # Without this, test write, writelines, and goto... fail. + # The reasons and why macOS-specific are unclear. @classmethod def tearDownClass(cls): diff --git a/Lib/idlelib/idle_test/test_query.py b/Lib/idlelib/idle_test/test_query.py index bb12b2b08652d5..a6ef858a8c954a 100644 --- a/Lib/idlelib/idle_test/test_query.py +++ b/Lib/idlelib/idle_test/test_query.py @@ -134,10 +134,10 @@ def test_c_source_name(self): def test_good_module_name(self): dialog = self.Dummy_ModuleName('idlelib') - self.assertTrue(dialog.entry_ok().endswith('__init__.py')) + self.assertEndsWith(dialog.entry_ok(), '__init__.py') self.assertEqual(dialog.entry_error['text'], '') dialog = self.Dummy_ModuleName('idlelib.idle') - self.assertTrue(dialog.entry_ok().endswith('idle.py')) + self.assertEndsWith(dialog.entry_ok(), 'idle.py') self.assertEqual(dialog.entry_error['text'], '') @@ -389,7 +389,7 @@ def test_click_module_name(self): self.assertEqual(dialog.text0, 'idlelib') self.assertEqual(dialog.entry.get(), 'idlelib') dialog.button_ok.invoke() - self.assertTrue(dialog.result.endswith('__init__.py')) + self.assertEndsWith(dialog.result, '__init__.py') root.destroy() diff --git a/Lib/idlelib/idle_test/test_redirector.py b/Lib/idlelib/idle_test/test_redirector.py index a97b3002afcf12..bd486d7da66010 100644 --- a/Lib/idlelib/idle_test/test_redirector.py +++ b/Lib/idlelib/idle_test/test_redirector.py @@ -34,7 +34,7 @@ def test_close(self): redir.register('insert', Func) redir.close() self.assertEqual(redir._operations, {}) - self.assertFalse(hasattr(self.text, 'widget')) + self.assertNotHasAttr(self.text, 'widget') class WidgetRedirectorTest(unittest.TestCase): diff --git a/Lib/idlelib/idle_test/test_run.py b/Lib/idlelib/idle_test/test_run.py index a38e43dcb9d1c4..83ecbffa2a197e 100644 --- a/Lib/idlelib/idle_test/test_run.py +++ b/Lib/idlelib/idle_test/test_run.py @@ -8,6 +8,7 @@ from unittest import mock import idlelib from idlelib.idle_test.mock_idle import Func +from test.support import force_not_colorized idlelib.testing = True # Use {} for executing test user code. @@ -46,6 +47,7 @@ def __eq__(self, other): "Did you mean: 'real'?\n"), ) + @force_not_colorized def test_get_message(self): for code, exc, msg in self.data: with self.subTest(code=code): @@ -57,6 +59,7 @@ def test_get_message(self): expect = f'{exc.__name__}: {msg}' self.assertEqual(actual, expect) + @force_not_colorized @mock.patch.object(run, 'cleanup_traceback', new_callable=lambda: (lambda t, e: None)) def test_get_multiple_message(self, mock): diff --git a/Lib/idlelib/idle_test/test_sidebar.py b/Lib/idlelib/idle_test/test_sidebar.py index 605e7a892570d7..4157a4b4dcdd2a 100644 --- a/Lib/idlelib/idle_test/test_sidebar.py +++ b/Lib/idlelib/idle_test/test_sidebar.py @@ -725,7 +725,7 @@ def test_copy(self): text.tag_add('sel', f'{first_line}.0', 'end-1c') selected_text = text.get('sel.first', 'sel.last') - self.assertTrue(selected_text.startswith('if True:\n')) + self.assertStartsWith(selected_text, 'if True:\n') self.assertIn('\n1\n', selected_text) text.event_generate('<>') @@ -749,7 +749,7 @@ def test_copy_with_prompts(self): text.tag_add('sel', f'{first_line}.3', 'end-1c') selected_text = text.get('sel.first', 'sel.last') - self.assertTrue(selected_text.startswith('True:\n')) + self.assertStartsWith(selected_text, 'True:\n') selected_lines_text = text.get('sel.first linestart', 'sel.last') selected_lines = selected_lines_text.split('\n') diff --git a/Lib/idlelib/outwin.py b/Lib/idlelib/outwin.py index 5ed3f35a7af655..8baa657550de94 100644 --- a/Lib/idlelib/outwin.py +++ b/Lib/idlelib/outwin.py @@ -112,7 +112,7 @@ def write(self, s, tags=(), mark="insert"): assert isinstance(s, str) self.text.insert(mark, s, tags) self.text.see(mark) - self.text.update_idletasks() + self.text.update() return len(s) def writelines(self, lines): diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 1524fccd5d20f8..66fbbd4a97b7af 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -11,15 +11,9 @@ "Your Python may not be configured for Tk. **", file=sys.__stderr__) raise SystemExit(1) -# Valid arguments for the ...Awareness call below are defined in the following. -# https://msdn.microsoft.com/en-us/library/windows/desktop/dn280512(v=vs.85).aspx if sys.platform == 'win32': - try: - import ctypes - PROCESS_SYSTEM_DPI_AWARE = 1 # Int required. - ctypes.OleDLL('shcore').SetProcessDpiAwareness(PROCESS_SYSTEM_DPI_AWARE) - except (ImportError, AttributeError, OSError): - pass + from idlelib.util import fix_win_hidpi + fix_win_hidpi() from tkinter import messagebox @@ -430,7 +424,9 @@ def __init__(self, tkconsole): def spawn_subprocess(self): if self.subprocess_arglist is None: self.subprocess_arglist = self.build_subprocess_arglist() - self.rpcsubproc = subprocess.Popen(self.subprocess_arglist) + # gh-127060: Disable traceback colors + env = dict(os.environ, TERM='dumb') + self.rpcsubproc = subprocess.Popen(self.subprocess_arglist, env=env) def build_subprocess_arglist(self): assert (self.port!=0), ( @@ -712,7 +708,7 @@ def prepend_syspath(self, filename): del _filename, _sys, _dirname, _dir \n""".format(filename)) - def showsyntaxerror(self, filename=None): + def showsyntaxerror(self, filename=None, **kwargs): """Override Interactive Interpreter method: Use Colorizing Color the offending position instead of printing it and pointing at it diff --git a/Lib/idlelib/redirector.py b/Lib/idlelib/redirector.py index 08728956abd900..8e2ba68d3815bf 100644 --- a/Lib/idlelib/redirector.py +++ b/Lib/idlelib/redirector.py @@ -106,6 +106,7 @@ def dispatch(self, operation, *args): to *args to accomplish that. For an example, see colorizer.py. ''' + operation = str(operation) # can be a Tcl_Obj m = self._operations.get(operation) try: if m: diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index 53e80a9b42801f..a30db99a619a93 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -91,21 +91,28 @@ def capture_warnings(capture): _warnings_showwarning = None capture_warnings(True) -tcl = tkinter.Tcl() -def handle_tk_events(tcl=tcl): - """Process any tk events that are ready to be dispatched if tkinter - has been imported, a tcl interpreter has been created and tk has been - loaded.""" - tcl.eval("update") +if idlelib.testing: + # gh-121008: When testing IDLE, don't create a Tk object to avoid side + # effects such as installing a PyOS_InputHook hook. + def handle_tk_events(): + pass +else: + tcl = tkinter.Tcl() + + def handle_tk_events(tcl=tcl): + """Process any tk events that are ready to be dispatched if tkinter + has been imported, a tcl interpreter has been created and tk has been + loaded.""" + tcl.eval("update") # Thread shared globals: Establish a queue between a subthread (which handles # the socket) and the main thread (which runs user code), plus global -# completion, exit and interruptable (the main thread) flags: +# completion, exit and interruptible (the main thread) flags: exit_now = False quitting = False -interruptable = False +interruptible = False def main(del_exitfunc=False): """Start the Python execution server in a subprocess @@ -436,6 +443,9 @@ class StdioFile(io.TextIOBase): def __init__(self, shell, tags, encoding='utf-8', errors='strict'): self.shell = shell + # GH-78889: accessing unpickleable attributes freezes Shell. + # IDLE only needs methods; allow 'width' for possible use. + self.shell._RPCProxy__attributes = {'width': 1} self.tags = tags self._encoding = encoding self._errors = errors @@ -572,14 +582,14 @@ def __init__(self, rpchandler): self.locals = {} def runcode(self, code): - global interruptable + global interruptible try: self.user_exc_info = None - interruptable = True + interruptible = True try: exec(code, self.locals) finally: - interruptable = False + interruptible = False except SystemExit as e: if e.args: # SystemExit called with an argument. ob = e.args[0] @@ -605,7 +615,7 @@ def runcode(self, code): flush_stdout() def interrupt_the_server(self): - if interruptable: + if interruptible: thread.interrupt_main() def start_the_debugger(self, gui_adap_oid): diff --git a/Lib/idlelib/searchbase.py b/Lib/idlelib/searchbase.py index 64ed50c7364be3..c68a6ca339af04 100644 --- a/Lib/idlelib/searchbase.py +++ b/Lib/idlelib/searchbase.py @@ -86,7 +86,7 @@ def create_widgets(self): top.wm_iconname(self.icon) _setup_dialog(top) self.top = top - self.frame = Frame(top, padding="5px") + self.frame = Frame(top, padding=5) self.frame.grid(sticky="nwes") top.grid_columnconfigure(0, weight=100) top.grid_rowconfigure(0, weight=100) diff --git a/Lib/idlelib/tooltip.py b/Lib/idlelib/tooltip.py index 3983690dd41177..df5b1fe1dcfb08 100644 --- a/Lib/idlelib/tooltip.py +++ b/Lib/idlelib/tooltip.py @@ -144,7 +144,8 @@ def hidetip(self): class Hovertip(OnHoverTooltipBase): "A tooltip that pops up when a mouse hovers over an anchor widget." - def __init__(self, anchor_widget, text, hover_delay=1000): + def __init__(self, anchor_widget, text, hover_delay=1000, + foreground="#000000", background="#ffffe0"): """Create a text tooltip with a mouse hover delay. anchor_widget: the widget next to which the tooltip will be shown @@ -156,10 +157,13 @@ def __init__(self, anchor_widget, text, hover_delay=1000): """ super().__init__(anchor_widget, hover_delay=hover_delay) self.text = text + self.foreground = foreground + self.background = background def showcontents(self): label = Label(self.tipwindow, text=self.text, justify=LEFT, - background="#ffffe0", relief=SOLID, borderwidth=1) + relief=SOLID, borderwidth=1, + foreground=self.foreground, background=self.background) label.pack() diff --git a/Lib/idlelib/tree.py b/Lib/idlelib/tree.py index 0726d7e23660f6..182ce7189614da 100644 --- a/Lib/idlelib/tree.py +++ b/Lib/idlelib/tree.py @@ -83,6 +83,8 @@ def wheel_event(event, widget=None): class TreeNode: + dy = 0 + def __init__(self, canvas, parent, item): self.canvas = canvas self.parent = parent @@ -199,23 +201,22 @@ def update(self): def draw(self, x, y): # XXX This hard-codes too many geometry constants! - dy = 20 self.x, self.y = x, y self.drawicon() self.drawtext() if self.state != 'expanded': - return y + dy + return y + TreeNode.dy # draw children if not self.children: sublist = self.item._GetSubList() if not sublist: # _IsExpandable() was mistaken; that's allowed - return y+17 + return y + TreeNode.dy for item in sublist: child = self.__class__(self.canvas, self, item) self.children.append(child) cx = x+20 - cy = y + dy + cy = y + TreeNode.dy cylast = 0 for child in self.children: cylast = cy @@ -289,6 +290,11 @@ def drawtext(self): self.label.bind("", lambda e: wheel_event(e, self.canvas)) self.label.bind("", lambda e: wheel_event(e, self.canvas)) self.text_id = id + if TreeNode.dy == 0: + # The first row doesn't matter what the dy is, just measure its + # size to get the value of the subsequent dy + coords = self.canvas.bbox(id) + TreeNode.dy = max(20, coords[3] - coords[1] - 3) def select_or_edit(self, event=None): if self.selected and self.item.IsEditable(): diff --git a/Lib/idlelib/util.py b/Lib/idlelib/util.py index a7ae74b0579004..e05604ab4853f6 100644 --- a/Lib/idlelib/util.py +++ b/Lib/idlelib/util.py @@ -12,11 +12,26 @@ * std streams (pyshell, run), * warning stuff (pyshell, run). """ +import sys # .pyw is for Windows; .pyi is for typing stub files. # The extension order is needed for iomenu open/save dialogs. py_extensions = ('.py', '.pyw', '.pyi') + +# Fix for HiDPI screens on Windows. CALL BEFORE ANY TK OPERATIONS! +# URL for arguments for the ...Awareness call below. +# https://msdn.microsoft.com/en-us/library/windows/desktop/dn280512(v=vs.85).aspx +if sys.platform == 'win32': # pragma: no cover + def fix_win_hidpi(): # Called in pyshell and turtledemo. + try: + import ctypes + PROCESS_SYSTEM_DPI_AWARE = 1 # Int required. + ctypes.OleDLL('shcore').SetProcessDpiAwareness(PROCESS_SYSTEM_DPI_AWARE) + except (ImportError, AttributeError, OSError): + pass + + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_util', verbosity=2) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 577b4b9b03a88d..db708580a0abf6 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -52,6 +52,9 @@ # search command can be quite large, so we now use 1M. _MAXLINE = 1000000 +# Data larger than this will be read in chunks, to prevent extreme +# overallocation. +_SAFE_BUF_SIZE = 1 << 20 # Commands @@ -239,7 +242,7 @@ def _connect(self): if __debug__: self._cmd_log_len = 10 self._cmd_log_idx = 0 - self._cmd_log = {} # Last `_cmd_log_len' interactions + self._cmd_log = {} # Last '_cmd_log_len' interactions if self.debug >= 1: self._mesg('imaplib version %s' % __version__) self._mesg('new IMAP4 connection, tag=%s' % self.tagpre) @@ -315,7 +318,13 @@ def open(self, host='', port=IMAP4_PORT, timeout=None): def read(self, size): """Read 'size' bytes from remote.""" - return self.file.read(size) + cursize = min(size, _SAFE_BUF_SIZE) + data = self.file.read(cursize) + while cursize < size and len(data) == cursize: + delta = min(cursize, size - cursize) + data += self.file.read(delta) + cursize += delta + return data def readline(self): @@ -396,7 +405,7 @@ def append(self, mailbox, flags, date_time, message): (typ, [data]) = .append(mailbox, flags, date_time, message) - All args except `message' can be None. + All args except 'message' can be None. """ name = 'APPEND' if not mailbox: @@ -927,7 +936,7 @@ def xatom(self, name, *args): (typ, [data]) = .xatom(name, arg, ...) - Returns response appropriate to extension command `name'. + Returns response appropriate to extension command 'name'. """ name = name.upper() #if not name in self.capabilities: # Let the server decide! @@ -1167,7 +1176,7 @@ def _get_tagged_response(self, tag, expect_bye=False): # Some have reported "unexpected response" exceptions. # Note that ignoring them here causes loops. # Instead, send me details of the unexpected response and - # I'll update the code in `_get_response()'. + # I'll update the code in '_get_response()'. try: self._get_response() @@ -1259,7 +1268,7 @@ def _dump_ur(self, untagged_resp_dict): self._mesg('untagged responses dump:' + '\n\t\t'.join(items)) def _log(self, line): - # Keep log of last `_cmd_log_len' interactions for debugging. + # Keep log of last '_cmd_log_len' interactions for debugging. self._cmd_log[self._cmd_log_idx] = (line, time.time()) self._cmd_log_idx += 1 if self._cmd_log_idx >= self._cmd_log_len: diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py index f38fe5c1ab461a..a7d57561ead046 100644 --- a/Lib/importlib/__init__.py +++ b/Lib/importlib/__init__.py @@ -103,7 +103,7 @@ def reload(module): try: name = module.__name__ except AttributeError: - raise TypeError("reload() argument must be a module") + raise TypeError("reload() argument must be a module") from None if sys.modules.get(name) is not module: raise ImportError(f"module {name} not in sys.modules", name=name) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 6d6292f9559253..b70d09b32abce6 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -1134,7 +1134,7 @@ def find_spec(cls, fullname, path=None, target=None): # part of the importer), instead of here (the finder part). # The loader is the usual place to get the data that will # be loaded into the module. (For example, see _LoaderBasics - # in _bootstra_external.py.) Most importantly, this importer + # in _bootstrap_external.py.) Most importantly, this importer # is simpler if we wait to get the data. # However, getting as much data in the finder as possible # to later load the module is okay, and sometimes important. @@ -1241,7 +1241,6 @@ def _find_spec(name, path, target=None): """Find a module's spec.""" meta_path = sys.meta_path if meta_path is None: - # PyImport_Cleanup() is running or has been called. raise ImportError("sys.meta_path is None, Python is likely " "shutting down") diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 2a9aef03179f6f..8bcd741c446bd2 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -52,7 +52,7 @@ # Bootstrap-related code ###################################################### _CASE_INSENSITIVE_PLATFORMS_STR_KEY = 'win', -_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin' +_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin', 'ios', 'tvos', 'watchos' _CASE_INSENSITIVE_PLATFORMS = (_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY + _CASE_INSENSITIVE_PLATFORMS_STR_KEY) @@ -81,6 +81,11 @@ def _pack_uint32(x): return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little') +def _unpack_uint64(data): + """Convert 8 bytes in little-endian to an integer.""" + assert len(data) == 8 + return int.from_bytes(data, 'little') + def _unpack_uint32(data): """Convert 4 bytes in little-endian to an integer.""" assert len(data) == 4 @@ -204,7 +209,11 @@ def _write_atomic(path, data, mode=0o666): # We first write data to a temporary file, and then use os.replace() to # perform an atomic rename. with _io.FileIO(fd, 'wb') as file: - file.write(data) + bytes_written = file.write(data) + if bytes_written != len(data): + # Raise an OSError so the 'except' below cleans up the partially + # written file. + raise OSError("os.write() didn't write the full pyc file") _os.replace(path_tmp, path) except OSError: try: @@ -216,274 +225,7 @@ def _write_atomic(path, data, mode=0o666): _code_type = type(_write_atomic.__code__) - -# Finder/loader utility code ############################################### - -# Magic word to reject .pyc files generated by other Python versions. -# It should change for each incompatible change to the bytecode. -# -# The value of CR and LF is incorporated so if you ever read or write -# a .pyc file in text mode the magic number will be wrong; also, the -# Apple MPW compiler swaps their values, botching string constants. -# -# There were a variety of old schemes for setting the magic number. -# The current working scheme is to increment the previous value by -# 10. -# -# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic -# number also includes a new "magic tag", i.e. a human readable string used -# to represent the magic number in __pycache__ directories. When you change -# the magic number, you must also set a new unique magic tag. Generally this -# can be named after the Python major version of the magic number bump, but -# it can really be anything, as long as it's different than anything else -# that's come before. The tags are included in the following table, starting -# with Python 3.2a0. -# -# Known values: -# Python 1.5: 20121 -# Python 1.5.1: 20121 -# Python 1.5.2: 20121 -# Python 1.6: 50428 -# Python 2.0: 50823 -# Python 2.0.1: 50823 -# Python 2.1: 60202 -# Python 2.1.1: 60202 -# Python 2.1.2: 60202 -# Python 2.2: 60717 -# Python 2.3a0: 62011 -# Python 2.3a0: 62021 -# Python 2.3a0: 62011 (!) -# Python 2.4a0: 62041 -# Python 2.4a3: 62051 -# Python 2.4b1: 62061 -# Python 2.5a0: 62071 -# Python 2.5a0: 62081 (ast-branch) -# Python 2.5a0: 62091 (with) -# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) -# Python 2.5b3: 62101 (fix wrong code: for x, in ...) -# Python 2.5b3: 62111 (fix wrong code: x += yield) -# Python 2.5c1: 62121 (fix wrong lnotab with for loops and -# storing constants that should have been removed) -# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) -# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) -# Python 2.6a1: 62161 (WITH_CLEANUP optimization) -# Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND) -# Python 2.7a0: 62181 (optimize conditional branches: -# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) -# Python 2.7a0 62191 (introduce SETUP_WITH) -# Python 2.7a0 62201 (introduce BUILD_SET) -# Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD) -# Python 3000: 3000 -# 3010 (removed UNARY_CONVERT) -# 3020 (added BUILD_SET) -# 3030 (added keyword-only parameters) -# 3040 (added signature annotations) -# 3050 (print becomes a function) -# 3060 (PEP 3115 metaclass syntax) -# 3061 (string literals become unicode) -# 3071 (PEP 3109 raise changes) -# 3081 (PEP 3137 make __file__ and __name__ unicode) -# 3091 (kill str8 interning) -# 3101 (merge from 2.6a0, see 62151) -# 3103 (__file__ points to source file) -# Python 3.0a4: 3111 (WITH_CLEANUP optimization). -# Python 3.0b1: 3131 (lexical exception stacking, including POP_EXCEPT - #3021) -# Python 3.1a1: 3141 (optimize list, set and dict comprehensions: -# change LIST_APPEND and SET_ADD, add MAP_ADD #2183) -# Python 3.1a1: 3151 (optimize conditional branches: -# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE - #4715) -# Python 3.2a1: 3160 (add SETUP_WITH #6101) -# tag: cpython-32 -# Python 3.2a2: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR #9225) -# tag: cpython-32 -# Python 3.2a3 3180 (add DELETE_DEREF #4617) -# Python 3.3a1 3190 (__class__ super closure changed) -# Python 3.3a1 3200 (PEP 3155 __qualname__ added #13448) -# Python 3.3a1 3210 (added size modulo 2**32 to the pyc header #13645) -# Python 3.3a2 3220 (changed PEP 380 implementation #14230) -# Python 3.3a4 3230 (revert changes to implicit __class__ closure #14857) -# Python 3.4a1 3250 (evaluate positional default arguments before -# keyword-only defaults #16967) -# Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override -# free vars #17853) -# Python 3.4a1 3270 (various tweaks to the __class__ closure #12370) -# Python 3.4a1 3280 (remove implicit class argument) -# Python 3.4a4 3290 (changes to __qualname__ computation #19301) -# Python 3.4a4 3300 (more changes to __qualname__ computation #19301) -# Python 3.4rc2 3310 (alter __qualname__ computation #20625) -# Python 3.5a1 3320 (PEP 465: Matrix multiplication operator #21176) -# Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations #2292) -# Python 3.5b2 3340 (fix dictionary display evaluation order #11205) -# Python 3.5b3 3350 (add GET_YIELD_FROM_ITER opcode #24400) -# Python 3.5.2 3351 (fix BUILD_MAP_UNPACK_WITH_CALL opcode #27286) -# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483) -# Python 3.6a1 3361 (lineno delta of code.co_lnotab becomes signed #26107) -# Python 3.6a2 3370 (16 bit wordcode #26647) -# Python 3.6a2 3371 (add BUILD_CONST_KEY_MAP opcode #27140) -# Python 3.6a2 3372 (MAKE_FUNCTION simplification, remove MAKE_CLOSURE -# #27095) -# Python 3.6b1 3373 (add BUILD_STRING opcode #27078) -# Python 3.6b1 3375 (add SETUP_ANNOTATIONS and STORE_ANNOTATION opcodes -# #27985) -# Python 3.6b1 3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL - #27213) -# Python 3.6b1 3377 (set __class__ cell from type.__new__ #23722) -# Python 3.6b2 3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257) -# Python 3.6rc1 3379 (more thorough __class__ validation #23722) -# Python 3.7a1 3390 (add LOAD_METHOD and CALL_METHOD opcodes #26110) -# Python 3.7a2 3391 (update GET_AITER #31709) -# Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650) -# Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550) -# Python 3.7b5 3394 (restored docstring as the first stmt in the body; -# this might affected the first line number #32911) -# Python 3.8a1 3400 (move frame block handling to compiler #17611) -# Python 3.8a1 3401 (add END_ASYNC_FOR #33041) -# Python 3.8a1 3410 (PEP570 Python Positional-Only Parameters #36540) -# Python 3.8b2 3411 (Reverse evaluation order of key: value in dict -# comprehensions #35224) -# Python 3.8b2 3412 (Swap the position of positional args and positional -# only args in ast.arguments #37593) -# Python 3.8b4 3413 (Fix "break" and "continue" in "finally" #37830) -# Python 3.9a0 3420 (add LOAD_ASSERTION_ERROR #34880) -# Python 3.9a0 3421 (simplified bytecode for with blocks #32949) -# Python 3.9a0 3422 (remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY, POP_FINALLY bytecodes #33387) -# Python 3.9a2 3423 (add IS_OP, CONTAINS_OP and JUMP_IF_NOT_EXC_MATCH bytecodes #39156) -# Python 3.9a2 3424 (simplify bytecodes for *value unpacking) -# Python 3.9a2 3425 (simplify bytecodes for **value unpacking) -# Python 3.10a1 3430 (Make 'annotations' future by default) -# Python 3.10a1 3431 (New line number table format -- PEP 626) -# Python 3.10a2 3432 (Function annotation for MAKE_FUNCTION is changed from dict to tuple bpo-42202) -# Python 3.10a2 3433 (RERAISE restores f_lasti if oparg != 0) -# Python 3.10a6 3434 (PEP 634: Structural Pattern Matching) -# Python 3.10a7 3435 Use instruction offsets (as opposed to byte offsets). -# Python 3.10b1 3436 (Add GEN_START bytecode #43683) -# Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!) -# Python 3.10b1 3438 Safer line number table handling. -# Python 3.10b1 3439 (Add ROT_N) -# Python 3.11a1 3450 Use exception table for unwinding ("zero cost" exception handling) -# Python 3.11a1 3451 (Add CALL_METHOD_KW) -# Python 3.11a1 3452 (drop nlocals from marshaled code objects) -# Python 3.11a1 3453 (add co_fastlocalnames and co_fastlocalkinds) -# Python 3.11a1 3454 (compute cell offsets relative to locals bpo-43693) -# Python 3.11a1 3455 (add MAKE_CELL bpo-43693) -# Python 3.11a1 3456 (interleave cell args bpo-43693) -# Python 3.11a1 3457 (Change localsplus to a bytes object bpo-43693) -# Python 3.11a1 3458 (imported objects now don't use LOAD_METHOD/CALL_METHOD) -# Python 3.11a1 3459 (PEP 657: add end line numbers and column offsets for instructions) -# Python 3.11a1 3460 (Add co_qualname field to PyCodeObject bpo-44530) -# Python 3.11a1 3461 (JUMP_ABSOLUTE must jump backwards) -# Python 3.11a2 3462 (bpo-44511: remove COPY_DICT_WITHOUT_KEYS, change -# MATCH_CLASS and MATCH_KEYS, and add COPY) -# Python 3.11a3 3463 (bpo-45711: JUMP_IF_NOT_EXC_MATCH no longer pops the -# active exception) -# Python 3.11a3 3464 (bpo-45636: Merge numeric BINARY_*/INPLACE_* into -# BINARY_OP) -# Python 3.11a3 3465 (Add COPY_FREE_VARS opcode) -# Python 3.11a4 3466 (bpo-45292: PEP-654 except*) -# Python 3.11a4 3467 (Change CALL_xxx opcodes) -# Python 3.11a4 3468 (Add SEND opcode) -# Python 3.11a4 3469 (bpo-45711: remove type, traceback from exc_info) -# Python 3.11a4 3470 (bpo-46221: PREP_RERAISE_STAR no longer pushes lasti) -# Python 3.11a4 3471 (bpo-46202: remove pop POP_EXCEPT_AND_RERAISE) -# Python 3.11a4 3472 (bpo-46009: replace GEN_START with POP_TOP) -# Python 3.11a4 3473 (Add POP_JUMP_IF_NOT_NONE/POP_JUMP_IF_NONE opcodes) -# Python 3.11a4 3474 (Add RESUME opcode) -# Python 3.11a5 3475 (Add RETURN_GENERATOR opcode) -# Python 3.11a5 3476 (Add ASYNC_GEN_WRAP opcode) -# Python 3.11a5 3477 (Replace DUP_TOP/DUP_TOP_TWO with COPY and -# ROT_TWO/ROT_THREE/ROT_FOUR/ROT_N with SWAP) -# Python 3.11a5 3478 (New CALL opcodes) -# Python 3.11a5 3479 (Add PUSH_NULL opcode) -# Python 3.11a5 3480 (New CALL opcodes, second iteration) -# Python 3.11a5 3481 (Use inline cache for BINARY_OP) -# Python 3.11a5 3482 (Use inline caching for UNPACK_SEQUENCE and LOAD_GLOBAL) -# Python 3.11a5 3483 (Use inline caching for COMPARE_OP and BINARY_SUBSCR) -# Python 3.11a5 3484 (Use inline caching for LOAD_ATTR, LOAD_METHOD, and -# STORE_ATTR) -# Python 3.11a5 3485 (Add an oparg to GET_AWAITABLE) -# Python 3.11a6 3486 (Use inline caching for PRECALL and CALL) -# Python 3.11a6 3487 (Remove the adaptive "oparg counter" mechanism) -# Python 3.11a6 3488 (LOAD_GLOBAL can push additional NULL) -# Python 3.11a6 3489 (Add JUMP_BACKWARD, remove JUMP_ABSOLUTE) -# Python 3.11a6 3490 (remove JUMP_IF_NOT_EXC_MATCH, add CHECK_EXC_MATCH) -# Python 3.11a6 3491 (remove JUMP_IF_NOT_EG_MATCH, add CHECK_EG_MATCH, -# add JUMP_BACKWARD_NO_INTERRUPT, make JUMP_NO_INTERRUPT virtual) -# Python 3.11a7 3492 (make POP_JUMP_IF_NONE/NOT_NONE/TRUE/FALSE relative) -# Python 3.11a7 3493 (Make JUMP_IF_TRUE_OR_POP/JUMP_IF_FALSE_OR_POP relative) -# Python 3.11a7 3494 (New location info table) -# Python 3.11b4 3495 (Set line number of module's RESUME instr to 0 per PEP 626) -# Python 3.12a1 3500 (Remove PRECALL opcode) -# Python 3.12a1 3501 (YIELD_VALUE oparg == stack_depth) -# Python 3.12a1 3502 (LOAD_FAST_CHECK, no NULL-check in LOAD_FAST) -# Python 3.12a1 3503 (Shrink LOAD_METHOD cache) -# Python 3.12a1 3504 (Merge LOAD_METHOD back into LOAD_ATTR) -# Python 3.12a1 3505 (Specialization/Cache for FOR_ITER) -# Python 3.12a1 3506 (Add BINARY_SLICE and STORE_SLICE instructions) -# Python 3.12a1 3507 (Set lineno of module's RESUME to 0) -# Python 3.12a1 3508 (Add CLEANUP_THROW) -# Python 3.12a1 3509 (Conditional jumps only jump forward) -# Python 3.12a2 3510 (FOR_ITER leaves iterator on the stack) -# Python 3.12a2 3511 (Add STOPITERATION_ERROR instruction) -# Python 3.12a2 3512 (Remove all unused consts from code objects) -# Python 3.12a4 3513 (Add CALL_INTRINSIC_1 instruction, removed STOPITERATION_ERROR, PRINT_EXPR, IMPORT_STAR) -# Python 3.12a4 3514 (Remove ASYNC_GEN_WRAP, LIST_TO_TUPLE, and UNARY_POSITIVE) -# Python 3.12a5 3515 (Embed jump mask in COMPARE_OP oparg) -# Python 3.12a5 3516 (Add COMPARE_AND_BRANCH instruction) -# Python 3.12a5 3517 (Change YIELD_VALUE oparg to exception block depth) -# Python 3.12a6 3518 (Add RETURN_CONST instruction) -# Python 3.12a6 3519 (Modify SEND instruction) -# Python 3.12a6 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2) -# Python 3.12a7 3521 (Shrink the LOAD_GLOBAL caches) -# Python 3.12a7 3522 (Removed JUMP_IF_FALSE_OR_POP/JUMP_IF_TRUE_OR_POP) -# Python 3.12a7 3523 (Convert COMPARE_AND_BRANCH back to COMPARE_OP) -# Python 3.12a7 3524 (Shrink the BINARY_SUBSCR caches) -# Python 3.12b1 3525 (Shrink the CALL caches) -# Python 3.12b1 3526 (Add instrumentation support) -# Python 3.12b1 3527 (Add LOAD_SUPER_ATTR) -# Python 3.12b1 3528 (Add LOAD_SUPER_ATTR_METHOD specialization) -# Python 3.12b1 3529 (Inline list/dict/set comprehensions) -# Python 3.12b1 3530 (Shrink the LOAD_SUPER_ATTR caches) -# Python 3.12b1 3531 (Add PEP 695 changes) -# Python 3.13a1 3550 (Plugin optimizer support) -# Python 3.13a1 3551 (Compact superinstructions) -# Python 3.13a1 3552 (Remove LOAD_FAST__LOAD_CONST and LOAD_CONST__LOAD_FAST) -# Python 3.13a1 3553 (Add SET_FUNCTION_ATTRIBUTE) -# Python 3.13a1 3554 (more efficient bytecodes for f-strings) -# Python 3.13a1 3555 (generate specialized opcodes metadata from bytecodes.c) -# Python 3.13a1 3556 (Convert LOAD_CLOSURE to a pseudo-op) -# Python 3.13a1 3557 (Make the conversion to boolean in jumps explicit) -# Python 3.13a1 3558 (Reorder the stack items for CALL) -# Python 3.13a1 3559 (Generate opcode IDs from bytecodes.c) -# Python 3.13a1 3560 (Add RESUME_CHECK instruction) -# Python 3.13a1 3561 (Add cache entry to branch instructions) -# Python 3.13a1 3562 (Assign opcode IDs for internal ops in separate range) -# Python 3.13a1 3563 (Add CALL_KW and remove KW_NAMES) -# Python 3.13a1 3564 (Removed oparg from YIELD_VALUE, changed oparg values of RESUME) -# Python 3.13a1 3565 (Oparg of YIELD_VALUE indicates whether it is in a yield-from) -# Python 3.13a1 3566 (Emit JUMP_NO_INTERRUPT instead of JUMP for non-loop no-lineno cases) -# Python 3.13a1 3567 (Reimplement line number propagation by the compiler) -# Python 3.13a1 3568 (Change semantics of END_FOR) - -# Python 3.14 will start with 3600 - -# Please don't copy-paste the same pre-release tag for new entries above!!! -# You should always use the *upcoming* tag. For example, if 3.12a6 came out -# a week ago, I should put "Python 3.12a7" next to my new magic number. - -# MAGIC must change whenever the bytecode emitted by the compiler may no -# longer be understood by older implementations of the eval loop (usually -# due to the addition of new opcodes). -# -# Starting with Python 3.11, Python 3.n starts with magic number 2900+50n. -# -# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array -# in PC/launcher.c must also be updated. - -MAGIC_NUMBER = (3568).to_bytes(2, 'little') + b'\r\n' - -_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c +MAGIC_NUMBER = _imp.pyc_magic_number_token.to_bytes(4, 'little') _PYCACHE = '__pycache__' _OPT = 'opt-' @@ -974,6 +716,12 @@ def _search_registry(cls, fullname): @classmethod def find_spec(cls, fullname, path=None, target=None): + _warnings.warn('importlib.machinery.WindowsRegistryFinder is ' + 'deprecated; use site configuration instead. ' + 'Future versions of Python may not enable this ' + 'finder by default.', + DeprecationWarning, stacklevel=2) + filepath = cls._search_registry(fullname) if filepath is None: return None @@ -1122,7 +870,7 @@ def get_code(self, fullname): _imp.check_hash_based_pycs == 'always')): source_bytes = self.get_data(source_path) source_hash = _imp.source_hash( - _RAW_MAGIC_NUMBER, + _imp.pyc_magic_number_token, source_bytes, ) _validate_hash_pyc(data, source_hash, fullname, @@ -1151,7 +899,7 @@ def get_code(self, fullname): source_mtime is not None): if hash_based: if source_hash is None: - source_hash = _imp.source_hash(_RAW_MAGIC_NUMBER, + source_hash = _imp.source_hash(_imp.pyc_magic_number_token, source_bytes) data = _code_to_hash_pyc(code_object, source_hash, check_source) else: @@ -1457,7 +1205,7 @@ class PathFinder: @staticmethod def invalidate_caches(): """Call the invalidate_caches() method on all path entry finders - stored in sys.path_importer_caches (where implemented).""" + stored in sys.path_importer_cache (where implemented).""" for name, finder in list(sys.path_importer_cache.items()): # Drop entry if finder name is a relative path. The current # working directory may have changed. @@ -1469,6 +1217,9 @@ def invalidate_caches(): # https://bugs.python.org/issue45703 _NamespacePath._epoch += 1 + from importlib.metadata import MetadataPathFinder + MetadataPathFinder.invalidate_caches() + @staticmethod def _path_hooks(path): """Search sys.path_hooks for a finder for 'path'.""" @@ -1493,7 +1244,7 @@ def _path_importer_cache(cls, path): if path == '': try: path = _os.getcwd() - except FileNotFoundError: + except (FileNotFoundError, PermissionError): # Don't cache the failure as the cwd can easily change to # a valid directory later on. return None @@ -1710,6 +1461,46 @@ def __repr__(self): return f'FileFinder({self.path!r})' +class AppleFrameworkLoader(ExtensionFileLoader): + """A loader for modules that have been packaged as frameworks for + compatibility with Apple's iOS App Store policies. + """ + def create_module(self, spec): + # If the ModuleSpec has been created by the FileFinder, it will have + # been created with an origin pointing to the .fwork file. We need to + # redirect this to the location in the Frameworks folder, using the + # content of the .fwork file. + if spec.origin.endswith(".fwork"): + with _io.FileIO(spec.origin, 'r') as file: + framework_binary = file.read().decode().strip() + bundle_path = _path_split(sys.executable)[0] + spec.origin = _path_join(bundle_path, framework_binary) + + # If the loader is created based on the spec for a loaded module, the + # path will be pointing at the Framework location. If this occurs, + # get the original .fwork location to use as the module's __file__. + if self.path.endswith(".fwork"): + path = self.path + else: + with _io.FileIO(self.path + ".origin", 'r') as file: + origin = file.read().decode().strip() + bundle_path = _path_split(sys.executable)[0] + path = _path_join(bundle_path, origin) + + module = _bootstrap._call_with_frames_removed(_imp.create_dynamic, spec) + + _bootstrap._verbose_message( + "Apple framework extension module {!r} loaded from {!r} (path {!r})", + spec.name, + spec.origin, + path, + ) + + # Ensure that the __file__ points at the .fwork location + module.__file__ = path + + return module + # Import setup ############################################################### def _fix_up_module(ns, name, pathname, cpathname=None): @@ -1742,10 +1533,17 @@ def _get_supported_file_loaders(): Each item is a tuple (loader, suffixes). """ - extensions = ExtensionFileLoader, _imp.extension_suffixes() + extension_loaders = [] + if hasattr(_imp, 'create_dynamic'): + if sys.platform in {"ios", "tvos", "watchos"}: + extension_loaders = [(AppleFrameworkLoader, [ + suffix.replace(".so", ".fwork") + for suffix in _imp.extension_suffixes() + ])] + extension_loaders.append((ExtensionFileLoader, _imp.extension_suffixes())) source = SourceFileLoader, SOURCE_SUFFIXES bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES - return [extensions, source, bytecode] + return extension_loaders + [source, bytecode] def _set_bootstrap_module(_bootstrap_module): diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index b56fa94eb9c135..29f01f77eff4a0 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -13,9 +13,6 @@ _frozen_importlib_external = _bootstrap_external from ._abc import Loader import abc -import warnings - -from .resources import abc as _resources_abc __all__ = [ @@ -25,19 +22,6 @@ ] -def __getattr__(name): - """ - For backwards compatibility, continue to make names - from _resources_abc available through this module. #93963 - """ - if name in _resources_abc.__all__: - obj = getattr(_resources_abc, name) - warnings._deprecated(f"{__name__}.{name}", remove=(3, 14)) - globals()[name] = obj - return obj - raise AttributeError(f'module {__name__!r} has no attribute {name!r}') - - def _register(abstract_cls, *classes): for cls in classes: abstract_cls.register(cls) @@ -86,6 +70,15 @@ class ResourceLoader(Loader): """ + def __init__(self): + import warnings + warnings.warn('importlib.abc.ResourceLoader is deprecated in ' + 'favour of supporting resource loading through ' + 'importlib.resources.abc.TraversableResources.', + DeprecationWarning, stacklevel=2) + super().__init__() + + @abc.abstractmethod def get_data(self, path): """Abstract method which when implemented should return the bytes for @@ -180,7 +173,11 @@ def get_code(self, fullname): else: return self.source_to_code(source, path) -_register(ExecutionLoader, machinery.ExtensionFileLoader) +_register( + ExecutionLoader, + machinery.ExtensionFileLoader, + machinery.AppleFrameworkLoader, +) class FileLoader(_bootstrap_external.FileLoader, ResourceLoader, ExecutionLoader): @@ -211,6 +208,10 @@ class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLo def path_mtime(self, path): """Return the (int) modification time for the path (str).""" + import warnings + warnings.warn('SourceLoader.path_mtime is deprecated in favour of ' + 'SourceLoader.path_stats().', + DeprecationWarning, stacklevel=2) if self.path_stats.__func__ is SourceLoader.path_stats: raise OSError return int(self.path_stats(path)['mtime']) diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index d9a19a13f7b275..63d726445c3d96 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -3,18 +3,48 @@ from ._bootstrap import ModuleSpec from ._bootstrap import BuiltinImporter from ._bootstrap import FrozenImporter -from ._bootstrap_external import (SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES, - OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES, - EXTENSION_SUFFIXES) +from ._bootstrap_external import ( + SOURCE_SUFFIXES, BYTECODE_SUFFIXES, EXTENSION_SUFFIXES, + DEBUG_BYTECODE_SUFFIXES as _DEBUG_BYTECODE_SUFFIXES, + OPTIMIZED_BYTECODE_SUFFIXES as _OPTIMIZED_BYTECODE_SUFFIXES +) from ._bootstrap_external import WindowsRegistryFinder from ._bootstrap_external import PathFinder from ._bootstrap_external import FileFinder from ._bootstrap_external import SourceFileLoader from ._bootstrap_external import SourcelessFileLoader from ._bootstrap_external import ExtensionFileLoader +from ._bootstrap_external import AppleFrameworkLoader from ._bootstrap_external import NamespaceLoader def all_suffixes(): """Returns a list of all recognized module suffixes for this process""" return SOURCE_SUFFIXES + BYTECODE_SUFFIXES + EXTENSION_SUFFIXES + + +__all__ = ['AppleFrameworkLoader', 'BYTECODE_SUFFIXES', 'BuiltinImporter', + 'DEBUG_BYTECODE_SUFFIXES', 'EXTENSION_SUFFIXES', + 'ExtensionFileLoader', 'FileFinder', 'FrozenImporter', 'ModuleSpec', + 'NamespaceLoader', 'OPTIMIZED_BYTECODE_SUFFIXES', 'PathFinder', + 'SOURCE_SUFFIXES', 'SourceFileLoader', 'SourcelessFileLoader', + 'WindowsRegistryFinder', 'all_suffixes'] + + +def __getattr__(name): + import warnings + + if name == 'DEBUG_BYTECODE_SUFFIXES': + warnings.warn('importlib.machinery.DEBUG_BYTECODE_SUFFIXES is ' + 'deprecated; use importlib.machinery.BYTECODE_SUFFIXES ' + 'instead.', + DeprecationWarning, stacklevel=2) + return _DEBUG_BYTECODE_SUFFIXES + elif name == 'OPTIMIZED_BYTECODE_SUFFIXES': + warnings.warn('importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES is ' + 'deprecated; use importlib.machinery.BYTECODE_SUFFIXES ' + 'instead.', + DeprecationWarning, stacklevel=2) + return _OPTIMIZED_BYTECODE_SUFFIXES + + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py index c612fbefee2e80..8ce62dd864fc27 100644 --- a/Lib/importlib/metadata/__init__.py +++ b/Lib/importlib/metadata/__init__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import re import abc @@ -16,7 +18,7 @@ import posixpath import collections -from . import _adapters, _meta +from . import _meta from ._collections import FreezableDefaultDict, Pair from ._functools import method_cache, pass_none from ._itertools import always_iterable, unique_everseen @@ -26,7 +28,7 @@ from importlib import import_module from importlib.abc import MetaPathFinder from itertools import starmap -from typing import Iterable, List, Mapping, Optional, Set, Union, cast +from typing import Any, Iterable, List, Mapping, Match, Optional, Set, cast __all__ = [ 'Distribution', @@ -163,17 +165,17 @@ class EntryPoint: value: str group: str - dist: Optional['Distribution'] = None + dist: Optional[Distribution] = None def __init__(self, name: str, value: str, group: str) -> None: vars(self).update(name=name, value=value, group=group) - def load(self): + def load(self) -> Any: """Load the entry point from its definition. If only a module is indicated by the value, return that module. Otherwise, return the named object. """ - match = self.pattern.match(self.value) + match = cast(Match, self.pattern.match(self.value)) module = import_module(match.group('module')) attrs = filter(None, (match.group('attr') or '').split('.')) return functools.reduce(getattr, attrs, module) @@ -268,7 +270,7 @@ def __repr__(self): """ return '%s(%r)' % (self.__class__.__name__, tuple(self)) - def select(self, **params): + def select(self, **params) -> EntryPoints: """ Select entry points from self that match the given parameters (typically group and/or name). @@ -304,19 +306,17 @@ def _from_text(text): class PackagePath(pathlib.PurePosixPath): """A reference to a path in a package""" - hash: Optional["FileHash"] + hash: Optional[FileHash] size: int - dist: "Distribution" + dist: Distribution def read_text(self, encoding: str = 'utf-8') -> str: # type: ignore[override] - with self.locate().open(encoding=encoding) as stream: - return stream.read() + return self.locate().read_text(encoding=encoding) def read_binary(self) -> bytes: - with self.locate().open('rb') as stream: - return stream.read() + return self.locate().read_bytes() - def locate(self) -> pathlib.Path: + def locate(self) -> SimplePath: """Return a path-like object for this path""" return self.dist.locate_file(self) @@ -330,6 +330,7 @@ def __repr__(self) -> str: class DeprecatedNonAbstract: + # Required until Python 3.14 def __new__(cls, *args, **kwargs): all_names = { name for subclass in inspect.getmro(cls) for name in vars(subclass) @@ -349,25 +350,48 @@ def __new__(cls, *args, **kwargs): class Distribution(DeprecatedNonAbstract): - """A Python distribution package.""" + """ + An abstract Python distribution package. + + Custom providers may derive from this class and define + the abstract methods to provide a concrete implementation + for their environment. Some providers may opt to override + the default implementation of some properties to bypass + the file-reading mechanism. + """ @abc.abstractmethod def read_text(self, filename) -> Optional[str]: """Attempt to load metadata file given by the name. + Python distribution metadata is organized by blobs of text + typically represented as "files" in the metadata directory + (e.g. package-1.0.dist-info). These files include things + like: + + - METADATA: The distribution metadata including fields + like Name and Version and Description. + - entry_points.txt: A series of entry points as defined in + `the entry points spec `_. + - RECORD: A record of files according to + `this recording spec `_. + + A package may provide any set of files, including those + not listed here or none at all. + :param filename: The name of the file in the distribution info. :return: The text if found, otherwise None. """ @abc.abstractmethod - def locate_file(self, path: Union[str, os.PathLike[str]]) -> pathlib.Path: + def locate_file(self, path: str | os.PathLike[str]) -> SimplePath: """ - Given a path to a file in this distribution, return a path + Given a path to a file in this distribution, return a SimplePath to it. """ @classmethod - def from_name(cls, name: str) -> "Distribution": + def from_name(cls, name: str) -> Distribution: """Return the Distribution for the given package name. :param name: The name of the distribution package to search for. @@ -385,16 +409,18 @@ def from_name(cls, name: str) -> "Distribution": raise PackageNotFoundError(name) @classmethod - def discover(cls, **kwargs) -> Iterable["Distribution"]: + def discover( + cls, *, context: Optional[DistributionFinder.Context] = None, **kwargs + ) -> Iterable[Distribution]: """Return an iterable of Distribution objects for all packages. Pass a ``context`` or pass keyword arguments for constructing a context. :context: A ``DistributionFinder.Context`` object. - :return: Iterable of Distribution objects for all packages. + :return: Iterable of Distribution objects for packages matching + the context. """ - context = kwargs.pop('context', None) if context and kwargs: raise ValueError("cannot accept context and kwargs") context = context or DistributionFinder.Context(**kwargs) @@ -403,8 +429,8 @@ def discover(cls, **kwargs) -> Iterable["Distribution"]: ) @staticmethod - def at(path: Union[str, os.PathLike[str]]) -> "Distribution": - """Return a Distribution for the indicated metadata path + def at(path: str | os.PathLike[str]) -> Distribution: + """Return a Distribution for the indicated metadata path. :param path: a string or path-like object :return: a concrete Distribution instance for the path @@ -413,7 +439,7 @@ def at(path: Union[str, os.PathLike[str]]) -> "Distribution": @staticmethod def _discover_resolvers(): - """Search the meta_path for resolvers.""" + """Search the meta_path for resolvers (MetadataPathFinders).""" declared = ( getattr(finder, 'find_distributions', None) for finder in sys.meta_path ) @@ -424,8 +450,15 @@ def metadata(self) -> _meta.PackageMetadata: """Return the parsed metadata for this Distribution. The returned object will have keys that name the various bits of - metadata. See PEP 566 for details. + metadata per the + `Core metadata specifications `_. + + Custom providers may provide the METADATA file or override this + property. """ + # deferred for performance (python/cpython#109829) + from . import _adapters + opt_text = ( self.read_text('METADATA') or self.read_text('PKG-INFO') @@ -454,6 +487,12 @@ def version(self) -> str: @property def entry_points(self) -> EntryPoints: + """ + Return EntryPoints for this distribution. + + Custom providers may provide the ``entry_points.txt`` file + or override this property. + """ return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self) @property @@ -466,6 +505,10 @@ def files(self) -> Optional[List[PackagePath]]: (i.e. RECORD for dist-info, or installed-files.txt or SOURCES.txt for egg-info) is missing. Result may be empty if the metadata exists but is empty. + + Custom providers are recommended to provide a "RECORD" file (in + ``read_text``) or override this property to allow for callers to be + able to resolve filenames provided by the package. """ def make_file(name, hash=None, size_str=None): @@ -497,7 +540,7 @@ def skip_missing_files(package_paths): def _read_files_distinfo(self): """ - Read the lines of RECORD + Read the lines of RECORD. """ text = self.read_text('RECORD') return text and text.splitlines() @@ -524,7 +567,7 @@ def _read_files_egginfo_installed(self): paths = ( (subdir / name) .resolve() - .relative_to(self.locate_file('').resolve()) + .relative_to(self.locate_file('').resolve(), walk_up=True) .as_posix() for name in text.splitlines() ) @@ -611,6 +654,9 @@ def _load_json(self, filename): class DistributionFinder(MetaPathFinder): """ A MetaPathFinder capable of discovering installed distributions. + + Custom providers should implement this interface in order to + supply metadata. """ class Context: @@ -623,6 +669,17 @@ class Context: Each DistributionFinder may expect any parameters and should attempt to honor the canonical parameters defined below when appropriate. + + This mechanism gives a custom provider a means to + solicit additional details from the caller beyond + "name" and "path" when searching distributions. + For example, imagine a provider that exposes suites + of packages in either a "public" or "private" ``realm``. + A caller may wish to query only for distributions in + a particular realm and could call + ``distributions(realm="private")`` to signal to the + custom provider to only include distributions from that + realm. """ name = None @@ -658,11 +715,18 @@ def find_distributions(self, context=Context()) -> Iterable[Distribution]: class FastPath: """ - Micro-optimized class for searching a path for - children. + Micro-optimized class for searching a root for children. + + Root is a path on the file system that may contain metadata + directories either as natural directories or within a zip file. >>> FastPath('').children() ['...'] + + FastPath objects are cached and recycled for any given root. + + >>> FastPath('foobar') is FastPath('foobar') + True """ @functools.lru_cache() # type: ignore @@ -704,7 +768,19 @@ def lookup(self, mtime): class Lookup: + """ + A micro-optimized class for searching a (fast) path for metadata. + """ + def __init__(self, path: FastPath): + """ + Calculate all of the children representing metadata. + + From the children in the path, calculate early all of the + children that appear to represent metadata (infos) or legacy + metadata (eggs). + """ + base = os.path.basename(path.root).lower() base_is_egg = base.endswith(".egg") self.infos = FreezableDefaultDict(list) @@ -725,7 +801,10 @@ def __init__(self, path: FastPath): self.infos.freeze() self.eggs.freeze() - def search(self, prepared): + def search(self, prepared: Prepared): + """ + Yield all infos and eggs matching the Prepared query. + """ infos = ( self.infos[prepared.normalized] if prepared @@ -741,13 +820,28 @@ def search(self, prepared): class Prepared: """ - A prepared search for metadata on a possibly-named package. + A prepared search query for metadata on a possibly-named package. + + Pre-calculates the normalization to prevent repeated operations. + + >>> none = Prepared(None) + >>> none.normalized + >>> none.legacy_normalized + >>> bool(none) + False + >>> sample = Prepared('Sample__Pkg-name.foo') + >>> sample.normalized + 'sample_pkg_name_foo' + >>> sample.legacy_normalized + 'sample__pkg_name.foo' + >>> bool(sample) + True """ normalized = None legacy_normalized = None - def __init__(self, name): + def __init__(self, name: Optional[str]): self.name = name if name is None: return @@ -777,7 +871,7 @@ class MetadataPathFinder(DistributionFinder): @classmethod def find_distributions( cls, context=DistributionFinder.Context() - ) -> Iterable["PathDistribution"]: + ) -> Iterable[PathDistribution]: """ Find distributions. @@ -797,6 +891,7 @@ def _search_paths(cls, name, paths): path.search(prepared) for path in map(FastPath, paths) ) + @classmethod def invalidate_caches(cls) -> None: FastPath.__new__.cache_clear() @@ -809,7 +904,7 @@ def __init__(self, path: SimplePath) -> None: """ self._path = path - def read_text(self, filename: Union[str, os.PathLike[str]]) -> Optional[str]: + def read_text(self, filename: str | os.PathLike[str]) -> Optional[str]: with suppress( FileNotFoundError, IsADirectoryError, @@ -823,7 +918,7 @@ def read_text(self, filename: Union[str, os.PathLike[str]]) -> Optional[str]: read_text.__doc__ = Distribution.read_text.__doc__ - def locate_file(self, path: Union[str, os.PathLike[str]]) -> pathlib.Path: + def locate_file(self, path: str | os.PathLike[str]) -> SimplePath: return self._path.parent / path @property diff --git a/Lib/importlib/metadata/_meta.py b/Lib/importlib/metadata/_meta.py index f670016de7fef2..1927d0f624d82f 100644 --- a/Lib/importlib/metadata/_meta.py +++ b/Lib/importlib/metadata/_meta.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import os from typing import Protocol from typing import Any, Dict, Iterator, List, Optional, TypeVar, Union, overload @@ -6,30 +9,27 @@ class PackageMetadata(Protocol): - def __len__(self) -> int: - ... # pragma: no cover + def __len__(self) -> int: ... # pragma: no cover - def __contains__(self, item: str) -> bool: - ... # pragma: no cover + def __contains__(self, item: str) -> bool: ... # pragma: no cover - def __getitem__(self, key: str) -> str: - ... # pragma: no cover + def __getitem__(self, key: str) -> str: ... # pragma: no cover - def __iter__(self) -> Iterator[str]: - ... # pragma: no cover + def __iter__(self) -> Iterator[str]: ... # pragma: no cover @overload - def get(self, name: str, failobj: None = None) -> Optional[str]: - ... # pragma: no cover + def get( + self, name: str, failobj: None = None + ) -> Optional[str]: ... # pragma: no cover @overload - def get(self, name: str, failobj: _T) -> Union[str, _T]: - ... # pragma: no cover + def get(self, name: str, failobj: _T) -> Union[str, _T]: ... # pragma: no cover # overload per python/importlib_metadata#435 @overload - def get_all(self, name: str, failobj: None = None) -> Optional[List[Any]]: - ... # pragma: no cover + def get_all( + self, name: str, failobj: None = None + ) -> Optional[List[Any]]: ... # pragma: no cover @overload def get_all(self, name: str, failobj: _T) -> Union[List[Any], _T]: @@ -44,20 +44,24 @@ def json(self) -> Dict[str, Union[str, List[str]]]: """ -class SimplePath(Protocol[_T]): +class SimplePath(Protocol): """ - A minimal subset of pathlib.Path required by PathDistribution. + A minimal subset of pathlib.Path required by Distribution. """ - def joinpath(self, other: Union[str, _T]) -> _T: - ... # pragma: no cover + def joinpath( + self, other: Union[str, os.PathLike[str]] + ) -> SimplePath: ... # pragma: no cover - def __truediv__(self, other: Union[str, _T]) -> _T: - ... # pragma: no cover + def __truediv__( + self, other: Union[str, os.PathLike[str]] + ) -> SimplePath: ... # pragma: no cover @property - def parent(self) -> _T: - ... # pragma: no cover + def parent(self) -> SimplePath: ... # pragma: no cover + + def read_text(self, encoding=None) -> str: ... # pragma: no cover + + def read_bytes(self) -> bytes: ... # pragma: no cover - def read_text(self) -> str: - ... # pragma: no cover + def exists(self) -> bool: ... # pragma: no cover diff --git a/Lib/importlib/resources/__init__.py b/Lib/importlib/resources/__init__.py index ae83cd07c4d4fb..723c9f9eb33ce1 100644 --- a/Lib/importlib/resources/__init__.py +++ b/Lib/importlib/resources/__init__.py @@ -1,4 +1,11 @@ -"""Read resources contained within a package.""" +""" +Read resources contained within a package. + +This codebase is shared between importlib.resources in the stdlib +and importlib_resources in PyPI. See +https://github.com/python/importlib_metadata/wiki/Development-Methodology +for more detail. +""" from ._common import ( as_file, @@ -7,6 +14,16 @@ Anchor, ) +from ._functional import ( + contents, + is_resource, + open_binary, + open_text, + path, + read_binary, + read_text, +) + from .abc import ResourceReader @@ -16,4 +33,11 @@ 'ResourceReader', 'as_file', 'files', + 'contents', + 'is_resource', + 'open_binary', + 'open_text', + 'path', + 'read_binary', + 'read_text', ] diff --git a/Lib/importlib/resources/_common.py b/Lib/importlib/resources/_common.py index a3902535342612..4e9014c45a056e 100644 --- a/Lib/importlib/resources/_common.py +++ b/Lib/importlib/resources/_common.py @@ -12,8 +12,6 @@ from typing import Union, Optional, cast from .abc import ResourceReader, Traversable -from ._adapters import wrap_spec - Package = Union[types.ModuleType, str] Anchor = Package @@ -27,6 +25,8 @@ def package_to_anchor(func): >>> files('a', 'b') Traceback (most recent call last): TypeError: files() takes from 0 to 1 positional arguments but 2 were given + + Remove this compatibility in Python 3.14. """ undefined = object() @@ -66,10 +66,10 @@ def get_resource_reader(package: types.ModuleType) -> Optional[ResourceReader]: # zipimport.zipimporter does not support weak references, resulting in a # TypeError. That seems terrible. spec = package.__spec__ - reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore + reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore[union-attr] if reader is None: return None - return reader(spec.name) # type: ignore + return reader(spec.name) # type: ignore[union-attr] @functools.singledispatch @@ -93,12 +93,13 @@ def _infer_caller(): """ def is_this_file(frame_info): - return frame_info.filename == __file__ + return frame_info.filename == stack[0].filename def is_wrapper(frame_info): return frame_info.function == 'wrapper' - not_this_file = itertools.filterfalse(is_this_file, inspect.stack()) + stack = inspect.stack() + not_this_file = itertools.filterfalse(is_this_file, stack) # also exclude 'wrapper' due to singledispatch in the call stack callers = itertools.filterfalse(is_wrapper, not_this_file) return next(callers).frame @@ -109,6 +110,9 @@ def from_package(package: types.ModuleType): Return a Traversable object for the given package. """ + # deferred for performance (python/cpython#109829) + from ._adapters import wrap_spec + spec = wrap_spec(package) reader = spec.loader.get_resource_reader(spec.name) return reader.files() @@ -179,7 +183,7 @@ def _(path): @contextlib.contextmanager def _temp_path(dir: tempfile.TemporaryDirectory): """ - Wrap tempfile.TemporyDirectory to return a pathlib object. + Wrap tempfile.TemporaryDirectory to return a pathlib object. """ with dir as result: yield pathlib.Path(result) diff --git a/Lib/importlib/resources/_functional.py b/Lib/importlib/resources/_functional.py new file mode 100644 index 00000000000000..f59416f2dd627d --- /dev/null +++ b/Lib/importlib/resources/_functional.py @@ -0,0 +1,81 @@ +"""Simplified function-based API for importlib.resources""" + +import warnings + +from ._common import files, as_file + + +_MISSING = object() + + +def open_binary(anchor, *path_names): + """Open for binary reading the *resource* within *package*.""" + return _get_resource(anchor, path_names).open('rb') + + +def open_text(anchor, *path_names, encoding=_MISSING, errors='strict'): + """Open for text reading the *resource* within *package*.""" + encoding = _get_encoding_arg(path_names, encoding) + resource = _get_resource(anchor, path_names) + return resource.open('r', encoding=encoding, errors=errors) + + +def read_binary(anchor, *path_names): + """Read and return contents of *resource* within *package* as bytes.""" + return _get_resource(anchor, path_names).read_bytes() + + +def read_text(anchor, *path_names, encoding=_MISSING, errors='strict'): + """Read and return contents of *resource* within *package* as str.""" + encoding = _get_encoding_arg(path_names, encoding) + resource = _get_resource(anchor, path_names) + return resource.read_text(encoding=encoding, errors=errors) + + +def path(anchor, *path_names): + """Return the path to the *resource* as an actual file system path.""" + return as_file(_get_resource(anchor, path_names)) + + +def is_resource(anchor, *path_names): + """Return ``True`` if there is a resource named *name* in the package, + + Otherwise returns ``False``. + """ + return _get_resource(anchor, path_names).is_file() + + +def contents(anchor, *path_names): + """Return an iterable over the named resources within the package. + + The iterable returns :class:`str` resources (e.g. files). + The iterable does not recurse into subdirectories. + """ + warnings.warn( + "importlib.resources.contents is deprecated. " + "Use files(anchor).iterdir() instead.", + DeprecationWarning, + stacklevel=1, + ) + return (resource.name for resource in _get_resource(anchor, path_names).iterdir()) + + +def _get_encoding_arg(path_names, encoding): + # For compatibility with versions where *encoding* was a positional + # argument, it needs to be given explicitly when there are multiple + # *path_names*. + # This limitation can be removed in Python 3.15. + if encoding is _MISSING: + if len(path_names) > 1: + raise TypeError( + "'encoding' argument required with multiple path names", + ) + else: + return 'utf-8' + return encoding + + +def _get_resource(anchor, path_names): + if anchor is None: + raise TypeError("anchor must be module or string, got None") + return files(anchor).joinpath(*path_names) diff --git a/Lib/importlib/resources/readers.py b/Lib/importlib/resources/readers.py index c3cdf769cbecb0..70fc7e2b9c0145 100644 --- a/Lib/importlib/resources/readers.py +++ b/Lib/importlib/resources/readers.py @@ -1,8 +1,14 @@ +from __future__ import annotations + import collections +import contextlib import itertools import pathlib import operator +import re +import warnings import zipfile +from collections.abc import Iterator from . import abc @@ -31,8 +37,10 @@ def files(self): class ZipReader(abc.TraversableResources): def __init__(self, loader, module): - _, _, name = module.rpartition('.') - self.prefix = loader.prefix.replace('\\', '/') + name + '/' + self.prefix = loader.prefix.replace('\\', '/') + if loader.is_package(module): + _, _, name = module.rpartition('.') + self.prefix += name + '/' self.archive = loader.archive def open_resource(self, resource): @@ -62,7 +70,7 @@ class MultiplexedPath(abc.Traversable): """ def __init__(self, *paths): - self._paths = list(map(pathlib.Path, remove_duplicates(paths))) + self._paths = list(map(_ensure_traversable, remove_duplicates(paths))) if not self._paths: message = 'MultiplexedPath must contain at least one path' raise FileNotFoundError(message) @@ -130,7 +138,40 @@ class NamespaceReader(abc.TraversableResources): def __init__(self, namespace_path): if 'NamespacePath' not in str(namespace_path): raise ValueError('Invalid path') - self.path = MultiplexedPath(*list(namespace_path)) + self.path = MultiplexedPath(*filter(bool, map(self._resolve, namespace_path))) + + @classmethod + def _resolve(cls, path_str) -> abc.Traversable | None: + r""" + Given an item from a namespace path, resolve it to a Traversable. + + path_str might be a directory on the filesystem or a path to a + zipfile plus the path within the zipfile, e.g. ``/foo/bar`` or + ``/foo/baz.zip/inner_dir`` or ``foo\baz.zip\inner_dir\sub``. + + path_str might also be a sentinel used by editable packages to + trigger other behaviors (see python/importlib_resources#311). + In that case, return None. + """ + dirs = (cand for cand in cls._candidate_paths(path_str) if cand.is_dir()) + return next(dirs, None) + + @classmethod + def _candidate_paths(cls, path_str: str) -> Iterator[abc.Traversable]: + yield pathlib.Path(path_str) + yield from cls._resolve_zip_path(path_str) + + @staticmethod + def _resolve_zip_path(path_str: str): + for match in reversed(list(re.finditer(r'[\\/]', path_str))): + with contextlib.suppress( + FileNotFoundError, + IsADirectoryError, + NotADirectoryError, + PermissionError, + ): + inner = path_str[match.end() :].replace('\\', '/') + '/' + yield zipfile.Path(path_str[: match.start()], inner.lstrip('/')) def resource_path(self, resource): """ @@ -142,3 +183,21 @@ def resource_path(self, resource): def files(self): return self.path + + +def _ensure_traversable(path): + """ + Convert deprecated string arguments to traversables (pathlib.Path). + + Remove with Python 3.15. + """ + if not isinstance(path, str): + return path + + warnings.warn( + "String arguments are deprecated. Pass a Traversable instead.", + DeprecationWarning, + stacklevel=3, + ) + + return pathlib.Path(path) diff --git a/Lib/importlib/resources/simple.py b/Lib/importlib/resources/simple.py index 96f117fec62c10..2e75299b13aabf 100644 --- a/Lib/importlib/resources/simple.py +++ b/Lib/importlib/resources/simple.py @@ -77,7 +77,7 @@ class ResourceHandle(Traversable): def __init__(self, parent: ResourceContainer, name: str): self.parent = parent - self.name = name # type: ignore + self.name = name # type: ignore[misc] def is_file(self): return True diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index ff4f12fb1af70c..2b564e9b52e0cb 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -5,7 +5,6 @@ from ._bootstrap import spec_from_loader from ._bootstrap import _find_spec from ._bootstrap_external import MAGIC_NUMBER -from ._bootstrap_external import _RAW_MAGIC_NUMBER from ._bootstrap_external import cache_from_source from ._bootstrap_external import decode_source from ._bootstrap_external import source_from_cache @@ -13,13 +12,12 @@ import _imp import sys -import threading import types def source_hash(source_bytes): "Return the hash of *source_bytes* as used in hash-based pyc files." - return _imp.source_hash(_RAW_MAGIC_NUMBER, source_bytes) + return _imp.source_hash(_imp.pyc_magic_number_token, source_bytes) def resolve_name(name, package): @@ -146,7 +144,7 @@ class _incompatible_extension_module_restrictions: You can get the same effect as this function by implementing the basic interface of multi-phase init (PEP 489) and lying about - support for mulitple interpreters (or per-interpreter GIL). + support for multiple interpreters (or per-interpreter GIL). """ def __init__(self, *, disable_check): @@ -178,16 +176,17 @@ def __getattribute__(self, attr): # Only the first thread to get the lock should trigger the load # and reset the module's class. The rest can now getattr(). if object.__getattribute__(self, '__class__') is _LazyModule: - # The first thread comes here multiple times as it descends the - # call stack. The first time, it sets is_loading and triggers - # exec_module(), which will access module.__dict__, module.__name__, - # and/or module.__spec__, reentering this method. These accesses - # need to be allowed to proceed without triggering the load again. - if loader_state['is_loading'] and attr.startswith('__') and attr.endswith('__'): - return object.__getattribute__(self, attr) + __class__ = loader_state['__class__'] + + # Reentrant calls from the same thread must be allowed to proceed without + # triggering the load again. + # exec_module() and self-referential imports are the primary ways this can + # happen, but in any case we must return something to avoid deadlock. + if loader_state['is_loading']: + return __class__.__getattribute__(self, attr) loader_state['is_loading'] = True - __dict__ = object.__getattribute__(self, '__dict__') + __dict__ = __class__.__getattribute__(self, '__dict__') # All module metadata must be gathered from __spec__ in order to avoid # using mutated values. @@ -217,8 +216,10 @@ def __getattribute__(self, attr): # Update after loading since that's what would happen in an eager # loading situation. __dict__.update(attrs_updated) - # Finally, stop triggering this method. - self.__class__ = types.ModuleType + # Finally, stop triggering this method, if the module did not + # already update its own __class__. + if isinstance(self, _LazyModule): + object.__setattr__(self, '__class__', __class__) return getattr(self, attr) @@ -254,6 +255,9 @@ def create_module(self, spec): def exec_module(self, module): """Make the module load lazily.""" + # Threading is only needed for lazy loading, and importlib.util can + # be pulled in at interpreter startup, so defer until needed. + import threading module.__spec__.loader = self.loader module.__loader__ = self.loader # Don't need to worry about deep-copying as trying to set an attribute @@ -267,3 +271,9 @@ def exec_module(self, module): loader_state['is_loading'] = False module.__spec__.loader_state = loader_state module.__class__ = _LazyModule + + +__all__ = ['LazyLoader', 'Loader', 'MAGIC_NUMBER', + 'cache_from_source', 'decode_source', 'find_spec', + 'module_from_spec', 'resolve_name', 'source_from_cache', + 'source_hash', 'spec_from_file_location', 'spec_from_loader'] diff --git a/Lib/inspect.py b/Lib/inspect.py index 8a2b2c96e993b5..facad478103668 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -6,9 +6,9 @@ Here are some of the useful functions provided by this module: - ismodule(), isclass(), ismethod(), isfunction(), isgeneratorfunction(), - isgenerator(), istraceback(), isframe(), iscode(), isbuiltin(), - isroutine() - check object types + ismodule(), isclass(), ismethod(), ispackage(), isfunction(), + isgeneratorfunction(), isgenerator(), istraceback(), isframe(), + iscode(), isbuiltin(), isroutine() - check object types getmembers() - get members of an object that satisfy a given condition getfile(), getsourcefile(), getsource() - find an object's source code @@ -24,8 +24,6 @@ stack(), trace() - get info about frames on the stack or in a traceback signature() - get a Signature object for the callable - - get_annotations() - safely compute an object's annotations """ # This module is in the public domain. No warranties. @@ -58,6 +56,8 @@ "CO_OPTIMIZED", "CO_VARARGS", "CO_VARKEYWORDS", + "CO_HAS_DOCSTRING", + "CO_METHOD", "ClassFoundException", "ClosureVars", "EndOfBlock", @@ -130,6 +130,7 @@ "ismethoddescriptor", "ismethodwrapper", "ismodule", + "ispackage", "isroutine", "istraceback", "markcoroutinefunction", @@ -142,6 +143,8 @@ import abc +from annotationlib import Format +from annotationlib import get_annotations # re-exported import ast import dis import collections.abc @@ -160,6 +163,7 @@ from keyword import iskeyword from operator import attrgetter from collections import namedtuple, OrderedDict +from weakref import ref as make_weakref # Create constants for the compiler flags in Include/code.h # We try to get them from dis to avoid duplication @@ -172,121 +176,6 @@ TPFLAGS_IS_ABSTRACT = 1 << 20 -def get_annotations(obj, *, globals=None, locals=None, eval_str=False): - """Compute the annotations dict for an object. - - obj may be a callable, class, or module. - Passing in an object of any other type raises TypeError. - - Returns a dict. get_annotations() returns a new dict every time - it's called; calling it twice on the same object will return two - different but equivalent dicts. - - This function handles several details for you: - - * If eval_str is true, values of type str will - be un-stringized using eval(). This is intended - for use with stringized annotations - ("from __future__ import annotations"). - * If obj doesn't have an annotations dict, returns an - empty dict. (Functions and methods always have an - annotations dict; classes, modules, and other types of - callables may not.) - * Ignores inherited annotations on classes. If a class - doesn't have its own annotations dict, returns an empty dict. - * All accesses to object members and dict values are done - using getattr() and dict.get() for safety. - * Always, always, always returns a freshly-created dict. - - eval_str controls whether or not values of type str are replaced - with the result of calling eval() on those values: - - * If eval_str is true, eval() is called on values of type str. - * If eval_str is false (the default), values of type str are unchanged. - - globals and locals are passed in to eval(); see the documentation - for eval() for more information. If either globals or locals is - None, this function may replace that value with a context-specific - default, contingent on type(obj): - - * If obj is a module, globals defaults to obj.__dict__. - * If obj is a class, globals defaults to - sys.modules[obj.__module__].__dict__ and locals - defaults to the obj class namespace. - * If obj is a callable, globals defaults to obj.__globals__, - although if obj is a wrapped function (using - functools.update_wrapper()) it is first unwrapped. - """ - if isinstance(obj, type): - # class - obj_dict = getattr(obj, '__dict__', None) - if obj_dict and hasattr(obj_dict, 'get'): - ann = obj_dict.get('__annotations__', None) - if isinstance(ann, types.GetSetDescriptorType): - ann = None - else: - ann = None - - obj_globals = None - module_name = getattr(obj, '__module__', None) - if module_name: - module = sys.modules.get(module_name, None) - if module: - obj_globals = getattr(module, '__dict__', None) - obj_locals = dict(vars(obj)) - unwrap = obj - elif isinstance(obj, types.ModuleType): - # module - ann = getattr(obj, '__annotations__', None) - obj_globals = getattr(obj, '__dict__') - obj_locals = None - unwrap = None - elif callable(obj): - # this includes types.Function, types.BuiltinFunctionType, - # types.BuiltinMethodType, functools.partial, functools.singledispatch, - # "class funclike" from Lib/test/test_inspect... on and on it goes. - ann = getattr(obj, '__annotations__', None) - obj_globals = getattr(obj, '__globals__', None) - obj_locals = None - unwrap = obj - else: - raise TypeError(f"{obj!r} is not a module, class, or callable.") - - if ann is None: - return {} - - if not isinstance(ann, dict): - raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") - - if not ann: - return {} - - if not eval_str: - return dict(ann) - - if unwrap is not None: - while True: - if hasattr(unwrap, '__wrapped__'): - unwrap = unwrap.__wrapped__ - continue - if isinstance(unwrap, functools.partial): - unwrap = unwrap.func - continue - break - if hasattr(unwrap, "__globals__"): - obj_globals = unwrap.__globals__ - - if globals is None: - globals = obj_globals - if locals is None: - locals = obj_locals - - return_value = {key: - value if not isinstance(value, str) else eval(value, globals, locals) - for key, value in ann.items() } - return return_value - - # ----------------------------------------------------------- type-checking def ismodule(object): """Return true if the object is a module.""" @@ -300,15 +189,20 @@ def ismethod(object): """Return true if the object is an instance method.""" return isinstance(object, types.MethodType) +def ispackage(object): + """Return true if the object is a package.""" + return ismodule(object) and hasattr(object, "__path__") + def ismethoddescriptor(object): """Return true if the object is a method descriptor. But not if ismethod() or isclass() or isfunction() are true. This is new in Python 2.2, and, for example, is true of int.__add__. - An object passing this test has a __get__ attribute but not a __set__ - attribute, but beyond that the set of attributes varies. __name__ is - usually sensible, and __doc__ often is. + An object passing this test has a __get__ attribute, but not a + __set__ attribute or a __delete__ attribute. Beyond that, the set + of attributes varies; __name__ is usually sensible, and __doc__ + often is. Methods implemented via descriptors that also pass one of the other tests return false from the ismethoddescriptor() test, simply because @@ -318,7 +212,9 @@ def ismethoddescriptor(object): # mutual exclusion return False tp = type(object) - return hasattr(tp, "__get__") and not hasattr(tp, "__set__") + return (hasattr(tp, "__get__") + and not hasattr(tp, "__set__") + and not hasattr(tp, "__delete__")) def isdatadescriptor(object): """Return true if the object is a data descriptor. @@ -374,11 +270,16 @@ def isfunction(object): Function objects provide these attributes: __doc__ documentation string __name__ name with which this function was defined + __qualname__ qualified name of this function + __module__ name of the module the function was defined in or None __code__ code object containing compiled function bytecode __defaults__ tuple of any default values for arguments __globals__ global namespace in which this function was defined __annotations__ dict of parameter annotations - __kwdefaults__ dict of keyword only parameters with defaults""" + __kwdefaults__ dict of keyword only parameters with defaults + __dict__ namespace which is supporting arbitrary function attributes + __closure__ a tuple of cells or None + __type_params__ tuple of type parameters""" return isinstance(object, types.FunctionType) def _has_code_flag(f, flag): @@ -402,13 +303,13 @@ def isgeneratorfunction(obj): return _has_code_flag(obj, CO_GENERATOR) # A marker for markcoroutinefunction and iscoroutinefunction. -_is_coroutine_marker = object() +_is_coroutine_mark = object() def _has_coroutine_mark(f): while ismethod(f): f = f.__func__ f = functools._unwrap_partial(f) - return getattr(f, "_is_coroutine_marker", None) is _is_coroutine_marker + return getattr(f, "_is_coroutine_marker", None) is _is_coroutine_mark def markcoroutinefunction(func): """ @@ -416,7 +317,7 @@ def markcoroutinefunction(func): """ if hasattr(func, '__func__'): func = func.__func__ - func._is_coroutine_marker = _is_coroutine_marker + func._is_coroutine_marker = _is_coroutine_mark return func def iscoroutinefunction(obj): @@ -443,17 +344,18 @@ def isgenerator(object): """Return true if the object is a generator. Generator objects provide these attributes: - __iter__ defined to support iteration over container - close raises a new GeneratorExit exception inside the - generator to terminate the iteration gi_code code object gi_frame frame object or possibly None once the generator has been exhausted gi_running set to 1 when generator is executing, 0 otherwise - next return the next item from the container - send resumes the generator and "sends" a value that becomes + gi_yieldfrom object being iterated by yield from or None + + __iter__() defined to support iteration over container + close() raises a new GeneratorExit exception inside the + generator to terminate the iteration + send() resumes the generator and "sends" a value that becomes the result of the current yield-expression - throw used to raise an exception inside the generator""" + throw() used to raise an exception inside the generator""" return isinstance(object, types.GeneratorType) def iscoroutine(object): @@ -488,7 +390,11 @@ def isframe(object): f_lasti index of last attempted instruction in bytecode f_lineno current line number in Python source code f_locals local namespace seen by this frame - f_trace tracing function for this frame, or None""" + f_trace tracing function for this frame, or None + f_trace_lines is a tracing event triggered for each source line? + f_trace_opcodes are per-opcode events being requested? + + clear() used to clear all references to local variables""" return isinstance(object, types.FrameType) def iscode(object): @@ -505,6 +411,7 @@ def iscode(object): co_flags bitmap: 1=optimized | 2=newlocals | 4=*arg | 8=**arg | 16=nested | 32=generator | 64=nofree | 128=coroutine | 256=iterable_coroutine | 512=async_generator + | 0x4000000=has_docstring co_freevars tuple of names of free variables co_posonlyargcount number of positional only arguments co_kwonlyargcount number of keyword only arguments (not including ** arg) @@ -513,7 +420,12 @@ def iscode(object): co_names tuple of names other than arguments and function locals co_nlocals number of local variables co_stacksize virtual machine stack space required - co_varnames tuple of names of arguments and local variables""" + co_varnames tuple of names of arguments and local variables + co_qualname fully qualified function name + + co_lines() returns an iterator that yields successive bytecode ranges + co_positions() returns an iterator of source code positions for each bytecode instruction + replace() returns a copy of the code object with a new values""" return isinstance(object, types.CodeType) def isbuiltin(object): @@ -946,14 +858,17 @@ def getsourcefile(object): Return None if no way can be identified to get the source. """ filename = getfile(object) - all_bytecode_suffixes = importlib.machinery.DEBUG_BYTECODE_SUFFIXES[:] - all_bytecode_suffixes += importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES[:] + all_bytecode_suffixes = importlib.machinery.BYTECODE_SUFFIXES[:] if any(filename.endswith(s) for s in all_bytecode_suffixes): filename = (os.path.splitext(filename)[0] + importlib.machinery.SOURCE_SUFFIXES[0]) elif any(filename.endswith(s) for s in importlib.machinery.EXTENSION_SUFFIXES): return None + elif filename.endswith(".fwork"): + # Apple mobile framework markers are another type of non-source file + return None + # return a filename found in the linecache even if it doesn't exist on disk if filename in linecache.cache: return filename @@ -984,6 +899,7 @@ def getmodule(object, _filename=None): return object if hasattr(object, '__module__'): return sys.modules.get(object.__module__) + # Try the filename to modulename cache if _filename is not None and _filename in modulesbyfile: return sys.modules.get(modulesbyfile[_filename]) @@ -1029,79 +945,6 @@ class ClassFoundException(Exception): pass -class _ClassFinder(ast.NodeVisitor): - - def __init__(self, cls, tree, lines, qualname): - self.stack = [] - self.cls = cls - self.tree = tree - self.lines = lines - self.qualname = qualname - self.lineno_found = [] - - def visit_FunctionDef(self, node): - self.stack.append(node.name) - self.stack.append('') - self.generic_visit(node) - self.stack.pop() - self.stack.pop() - - visit_AsyncFunctionDef = visit_FunctionDef - - def visit_ClassDef(self, node): - self.stack.append(node.name) - if self.qualname == '.'.join(self.stack): - # Return the decorator for the class if present - if node.decorator_list: - line_number = node.decorator_list[0].lineno - else: - line_number = node.lineno - - # decrement by one since lines starts with indexing by zero - self.lineno_found.append((line_number - 1, node.end_lineno)) - self.generic_visit(node) - self.stack.pop() - - def get_lineno(self): - self.visit(self.tree) - lineno_found_number = len(self.lineno_found) - if lineno_found_number == 0: - raise OSError('could not find class definition') - elif lineno_found_number == 1: - return self.lineno_found[0][0] - else: - # We have multiple candidates for the class definition. - # Now we have to guess. - - # First, let's see if there are any method definitions - for member in self.cls.__dict__.values(): - if (isinstance(member, types.FunctionType) and - member.__module__ == self.cls.__module__): - for lineno, end_lineno in self.lineno_found: - if lineno <= member.__code__.co_firstlineno <= end_lineno: - return lineno - - class_strings = [(''.join(self.lines[lineno: end_lineno]), lineno) - for lineno, end_lineno in self.lineno_found] - - # Maybe the class has a docstring and it's unique? - if self.cls.__doc__: - ret = None - for candidate, lineno in class_strings: - if self.cls.__doc__.strip() in candidate: - if ret is None: - ret = lineno - else: - break - else: - if ret is not None: - return ret - - # We are out of ideas, just return the last one found, which is - # slightly better than previous ones - return self.lineno_found[-1][0] - - def findsource(object): """Return the entire source file and starting line number for an object. @@ -1119,7 +962,7 @@ def findsource(object): # Allow filenames in form of "" to pass through. # `doctest` monkeypatches `linecache` module to enable # inspection, so let `linecache.getlines` to be called. - if not (file.startswith('<') and file.endswith('>')): + if (not (file.startswith('<') and file.endswith('>'))) or file.endswith('.fwork'): raise OSError('source code not available') module = getmodule(object, file) @@ -1134,11 +977,13 @@ def findsource(object): return lines, 0 if isclass(object): - qualname = object.__qualname__ - source = ''.join(lines) - tree = ast.parse(source) - class_finder = _ClassFinder(object, tree, lines, qualname) - return lines, class_finder.get_lineno() + try: + lnum = vars(object)['__firstlineno__'] - 1 + except (TypeError, KeyError): + raise OSError('source code not available') + if lnum >= len(lines): + raise OSError('lineno is out of bounds') + return lines, lnum if ismethod(object): object = object.__func__ @@ -1152,15 +997,8 @@ def findsource(object): if not hasattr(object, 'co_firstlineno'): raise OSError('could not find function definition') lnum = object.co_firstlineno - 1 - pat = re.compile(r'^(\s*def\s)|(\s*async\s+def\s)|(.*(? 0: - try: - line = lines[lnum] - except IndexError: - raise OSError('lineno is out of bounds') - if pat.match(line): - break - lnum = lnum - 1 + if lnum >= len(lines): + raise OSError('lineno is out of bounds') return lines, lnum raise OSError('could not find code object') @@ -1489,7 +1327,9 @@ def getargvalues(frame): args, varargs, varkw = getargs(frame.f_code) return ArgInfo(args, varargs, varkw, frame.f_locals) -def formatannotation(annotation, base_module=None): +def formatannotation(annotation, base_module=None, *, quote_annotation_strings=True): + if not quote_annotation_strings and isinstance(annotation, str): + return annotation if getattr(annotation, '__module__', None) == 'typing': def repl(match): text = match.group() @@ -1667,11 +1507,15 @@ def getclosurevars(func): global_vars = {} builtin_vars = {} unbound_names = set() - for name in code.co_names: - if name in ("None", "True", "False"): - # Because these used to be builtins instead of keywords, they - # may still show up as name references. We ignore them. - continue + global_names = set() + for instruction in dis.get_instructions(code): + opname = instruction.opname + name = instruction.argval + if opname == "LOAD_ATTR": + unbound_names.add(name) + elif opname == "LOAD_GLOBAL": + global_names.add(name) + for name in global_names: try: global_vars[name] = global_ns[name] except KeyError: @@ -1834,9 +1678,16 @@ def _check_class(klass, attr): return entry.__dict__[attr] return _sentinel + @functools.lru_cache() -def _shadowed_dict_from_mro_tuple(mro): - for entry in mro: +def _shadowed_dict_from_weakref_mro_tuple(*weakref_mro): + for weakref_entry in weakref_mro: + # Normally we'd have to check whether the result of weakref_entry() + # is None here, in case the object the weakref is pointing to has died. + # In this specific case, however, we know that the only caller of this + # function is `_shadowed_dict()`, and that therefore this weakref is + # guaranteed to point to an object that is still alive. + entry = weakref_entry() dunder_dict = _get_dunder_dict_of_class(entry) if '__dict__' in dunder_dict: class_dict = dunder_dict['__dict__'] @@ -1846,8 +1697,19 @@ def _shadowed_dict_from_mro_tuple(mro): return class_dict return _sentinel + def _shadowed_dict(klass): - return _shadowed_dict_from_mro_tuple(_static_getmro(klass)) + # gh-118013: the inner function here is decorated with lru_cache for + # performance reasons, *but* make sure not to pass strong references + # to the items in the mro. Doing so can lead to unexpected memory + # consumption in cases where classes are dynamically created and + # destroyed, and the dynamically created classes happen to be the only + # objects that hold strong references to other objects that take up a + # significant amount of memory. + return _shadowed_dict_from_weakref_mro_tuple( + *[make_weakref(entry) for entry in _static_getmro(klass)] + ) + def getattr_static(obj, attr, default=_sentinel): """Retrieve attributes without triggering dynamic lookup via the @@ -2084,7 +1946,12 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()): if param.kind is _POSITIONAL_ONLY: # If positional-only parameter is bound by partial, # it effectively disappears from the signature - new_params.pop(param_name) + # However, if it is a Placeholder it is not removed + # And also looses default value + if arg_value is functools.Placeholder: + new_params[param_name] = param.replace(default=_empty) + else: + new_params.pop(param_name) continue if param.kind is _POSITIONAL_OR_KEYWORD: @@ -2106,7 +1973,17 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()): new_params[param_name] = param.replace(default=arg_value) else: # was passed as a positional argument - new_params.pop(param.name) + # Do not pop if it is a Placeholder + # also change kind to positional only + # and remove default + if arg_value is functools.Placeholder: + new_param = param.replace( + kind=_POSITIONAL_ONLY, + default=_empty + ) + new_params[param_name] = new_param + else: + new_params.pop(param_name) continue if param.kind is _KEYWORD_ONLY: @@ -2162,8 +2039,10 @@ def _signature_is_builtin(obj): ismethoddescriptor(obj) or isinstance(obj, _NonUserDefinedCallables) or # Can't test 'isinstance(type)' here, as it would - # also be True for regular python classes - obj in (type, object)) + # also be True for regular python classes. + # Can't use the `in` operator here, as it would + # invoke the custom __eq__ method. + obj is type or obj is object) def _signature_is_functionlike(obj): @@ -2266,7 +2145,12 @@ def _signature_fromstr(cls, obj, s, skip_bound_arg=True): module = None module_dict = {} + module_name = getattr(obj, '__module__', None) + if not module_name: + objclass = getattr(obj, '__objclass__', None) + module_name = getattr(objclass, '__module__', None) + if module_name: module = sys.modules.get(module_name, None) if module: @@ -2400,7 +2284,8 @@ def _signature_from_builtin(cls, func, skip_bound_arg=True): def _signature_from_function(cls, func, skip_bound_arg=True, - globals=None, locals=None, eval_str=False): + globals=None, locals=None, eval_str=False, + *, annotation_format=Format.VALUE): """Private helper: constructs Signature for the given python function.""" is_duck_function = False @@ -2426,7 +2311,8 @@ def _signature_from_function(cls, func, skip_bound_arg=True, positional = arg_names[:pos_count] keyword_only_count = func_code.co_kwonlyargcount keyword_only = arg_names[pos_count:pos_count + keyword_only_count] - annotations = get_annotations(func, globals=globals, locals=locals, eval_str=eval_str) + annotations = get_annotations(func, globals=globals, locals=locals, eval_str=eval_str, + format=annotation_format) defaults = func.__defaults__ kwdefaults = func.__kwdefaults__ @@ -2509,7 +2395,8 @@ def _signature_from_callable(obj, *, globals=None, locals=None, eval_str=False, - sigcls): + sigcls, + annotation_format=Format.VALUE): """Private helper function to get signature for arbitrary callable objects. @@ -2521,7 +2408,8 @@ def _signature_from_callable(obj, *, globals=globals, locals=locals, sigcls=sigcls, - eval_str=eval_str) + eval_str=eval_str, + annotation_format=annotation_format) if not callable(obj): raise TypeError('{!r} is not a callable object'.format(obj)) @@ -2554,18 +2442,10 @@ def _signature_from_callable(obj, *, pass else: if sig is not None: - # since __text_signature__ is not writable on classes, __signature__ - # may contain text (or be a callable that returns text); - # if so, convert it - o_sig = sig - if not isinstance(sig, (Signature, str)) and callable(sig): - sig = sig() - if isinstance(sig, str): - sig = _signature_fromstr(sigcls, obj, sig) if not isinstance(sig, Signature): raise TypeError( 'unexpected object {!r} in __signature__ ' - 'attribute'.format(o_sig)) + 'attribute'.format(sig)) return sig try: @@ -2593,24 +2473,30 @@ def _signature_from_callable(obj, *, sig_params = tuple(sig.parameters.values()) assert (not sig_params or first_wrapped_param is not sig_params[0]) + # If there were placeholders set, + # first param is transformed to positional only + if partialmethod.args.count(functools.Placeholder): + first_wrapped_param = first_wrapped_param.replace( + kind=Parameter.POSITIONAL_ONLY) new_params = (first_wrapped_param,) + sig_params return sig.replace(parameters=new_params) + if isinstance(obj, functools.partial): + wrapped_sig = _get_signature_of(obj.func) + return _signature_get_partial(wrapped_sig, obj) + if isfunction(obj) or _signature_is_functionlike(obj): # If it's a pure Python function, or an object that is duck type # of a Python function (Cython functions, for instance), then: return _signature_from_function(sigcls, obj, skip_bound_arg=skip_bound_arg, - globals=globals, locals=locals, eval_str=eval_str) + globals=globals, locals=locals, eval_str=eval_str, + annotation_format=annotation_format) if _signature_is_builtin(obj): return _signature_from_builtin(sigcls, obj, skip_bound_arg=skip_bound_arg) - if isinstance(obj, functools.partial): - wrapped_sig = _get_signature_of(obj.func) - return _signature_get_partial(wrapped_sig, obj) - if isinstance(obj, type): # obj is a class or a metaclass @@ -2675,6 +2561,13 @@ def _signature_from_callable(obj, *, # An object with __call__ call = getattr_static(type(obj), '__call__', None) if call is not None: + try: + text_sig = obj.__text_signature__ + except AttributeError: + pass + else: + if text_sig: + return _signature_fromstr(sigcls, obj, text_sig) call = _descriptor_get(call, obj) return _get_signature_of(call) @@ -2833,13 +2726,17 @@ def replace(self, *, name=_void, kind=_void, return type(self)(name, kind, default=default, annotation=annotation) def __str__(self): + return self._format() + + def _format(self, *, quote_annotation_strings=True): kind = self.kind formatted = self._name # Add annotation and default value if self._annotation is not _empty: - formatted = '{}: {}'.format(formatted, - formatannotation(self._annotation)) + annotation = formatannotation(self._annotation, + quote_annotation_strings=quote_annotation_strings) + formatted = '{}: {}'.format(formatted, annotation) if self._default is not _empty: if self._annotation is not _empty: @@ -3046,11 +2943,19 @@ def __init__(self, parameters=None, *, return_annotation=_empty, params = OrderedDict() top_kind = _POSITIONAL_ONLY seen_default = False + seen_var_parameters = set() for param in parameters: kind = param.kind name = param.name + if kind in (_VAR_POSITIONAL, _VAR_KEYWORD): + if kind in seen_var_parameters: + msg = f'more than one {kind.description} parameter' + raise ValueError(msg) + + seen_var_parameters.add(kind) + if kind < top_kind: msg = ( 'wrong parameter order: {} parameter before {} ' @@ -3087,11 +2992,13 @@ def __init__(self, parameters=None, *, return_annotation=_empty, @classmethod def from_callable(cls, obj, *, - follow_wrapped=True, globals=None, locals=None, eval_str=False): + follow_wrapped=True, globals=None, locals=None, eval_str=False, + annotation_format=Format.VALUE): """Constructs Signature for the given callable object.""" return _signature_from_callable(obj, sigcls=cls, follow_wrapper_chains=follow_wrapped, - globals=globals, locals=locals, eval_str=eval_str) + globals=globals, locals=locals, eval_str=eval_str, + annotation_format=annotation_format) @property def parameters(self): @@ -3148,6 +3055,8 @@ def _bind(self, args, kwargs, *, partial=False): parameters_ex = () arg_vals = iter(args) + pos_only_param_in_kwargs = [] + while True: # Let's iterate through the positional arguments and corresponding # parameters @@ -3168,10 +3077,10 @@ def _bind(self, args, kwargs, *, partial=False): break elif param.name in kwargs: if param.kind == _POSITIONAL_ONLY: - msg = '{arg!r} parameter is positional only, ' \ - 'but was passed as a keyword' - msg = msg.format(arg=param.name) - raise TypeError(msg) from None + # Raise a TypeError once we are sure there is no + # **kwargs param later. + pos_only_param_in_kwargs.append(param) + continue parameters_ex = (param,) break elif (param.kind == _VAR_KEYWORD or @@ -3253,20 +3162,22 @@ def _bind(self, args, kwargs, *, partial=False): format(arg=param_name)) from None else: - if param.kind == _POSITIONAL_ONLY: - # This should never happen in case of a properly built - # Signature object (but let's have this check here - # to ensure correct behaviour just in case) - raise TypeError('{arg!r} parameter is positional only, ' - 'but was passed as a keyword'. \ - format(arg=param.name)) - arguments[param_name] = arg_val if kwargs: if kwargs_param is not None: # Process our '**kwargs'-like parameter arguments[kwargs_param.name] = kwargs + elif pos_only_param_in_kwargs: + raise TypeError( + 'got some positional-only arguments passed as ' + 'keyword arguments: {arg!r}'.format( + arg=', '.join( + param.name + for param in pos_only_param_in_kwargs + ), + ), + ) else: raise TypeError( 'got an unexpected keyword argument {arg!r}'.format( @@ -3302,19 +3213,24 @@ def __repr__(self): def __str__(self): return self.format() - def format(self, *, max_width=None): + def format(self, *, max_width=None, quote_annotation_strings=True): """Create a string representation of the Signature object. If *max_width* integer is passed, signature will try to fit into the *max_width*. If signature is longer than *max_width*, all parameters will be on separate lines. + + If *quote_annotation_strings* is False, annotations + in the signature are displayed without opening and closing quotation + marks. This is useful when the signature was created with the + STRING format or when ``from __future__ import annotations`` was used. """ result = [] render_pos_only_separator = False render_kw_only_separator = True for param in self.parameters.values(): - formatted = str(param) + formatted = param._format(quote_annotation_strings=quote_annotation_strings) kind = param.kind @@ -3351,16 +3267,19 @@ def format(self, *, max_width=None): rendered = '(\n {}\n)'.format(',\n '.join(result)) if self.return_annotation is not _empty: - anno = formatannotation(self.return_annotation) + anno = formatannotation(self.return_annotation, + quote_annotation_strings=quote_annotation_strings) rendered += ' -> {}'.format(anno) return rendered -def signature(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=False): +def signature(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=False, + annotation_format=Format.VALUE): """Get a signature object for the passed callable.""" return Signature.from_callable(obj, follow_wrapped=follow_wrapped, - globals=globals, locals=locals, eval_str=eval_str) + globals=globals, locals=locals, eval_str=eval_str, + annotation_format=annotation_format) class BufferFlags(enum.IntFlag): diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index e398cc138308d9..703fa289dda1fb 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -239,7 +239,7 @@ def summarize_address_range(first, last): else: raise ValueError('unknown IP version') - ip_bits = first._max_prefixlen + ip_bits = first.max_prefixlen first_int = first._ip last_int = last._ip while first_int <= last_int: @@ -310,7 +310,7 @@ def collapse_addresses(addresses): [IPv4Network('192.0.2.0/24')] Args: - addresses: An iterator of IPv4Network or IPv6Network objects. + addresses: An iterable of IPv4Network or IPv6Network objects. Returns: An iterator of the collapsed IPv(4|6)Network objects. @@ -326,12 +326,12 @@ def collapse_addresses(addresses): # split IP addresses and networks for ip in addresses: if isinstance(ip, _BaseAddress): - if ips and ips[-1]._version != ip._version: + if ips and ips[-1].version != ip.version: raise TypeError("%s and %s are not of the same version" % ( ip, ips[-1])) ips.append(ip) - elif ip._prefixlen == ip._max_prefixlen: - if ips and ips[-1]._version != ip._version: + elif ip._prefixlen == ip.max_prefixlen: + if ips and ips[-1].version != ip.version: raise TypeError("%s and %s are not of the same version" % ( ip, ips[-1])) try: @@ -339,7 +339,7 @@ def collapse_addresses(addresses): except AttributeError: ips.append(ip.network_address) else: - if nets and nets[-1]._version != ip._version: + if nets and nets[-1].version != ip.version: raise TypeError("%s and %s are not of the same version" % ( ip, nets[-1])) nets.append(ip) @@ -407,26 +407,21 @@ def reverse_pointer(self): """ return self._reverse_pointer() - @property - def version(self): - msg = '%200s has no version specified' % (type(self),) - raise NotImplementedError(msg) - def _check_int_address(self, address): if address < 0: msg = "%d (< 0) is not permitted as an IPv%d address" - raise AddressValueError(msg % (address, self._version)) + raise AddressValueError(msg % (address, self.version)) if address > self._ALL_ONES: msg = "%d (>= 2**%d) is not permitted as an IPv%d address" - raise AddressValueError(msg % (address, self._max_prefixlen, - self._version)) + raise AddressValueError(msg % (address, self.max_prefixlen, + self.version)) def _check_packed_address(self, address, expected_len): address_len = len(address) if address_len != expected_len: msg = "%r (len %d != %d) is not permitted as an IPv%d address" raise AddressValueError(msg % (address, address_len, - expected_len, self._version)) + expected_len, self.version)) @classmethod def _ip_int_from_prefix(cls, prefixlen): @@ -455,12 +450,12 @@ def _prefix_from_ip_int(cls, ip_int): ValueError: If the input intermingles zeroes & ones """ trailing_zeroes = _count_righthand_zero_bits(ip_int, - cls._max_prefixlen) - prefixlen = cls._max_prefixlen - trailing_zeroes + cls.max_prefixlen) + prefixlen = cls.max_prefixlen - trailing_zeroes leading_ones = ip_int >> trailing_zeroes all_ones = (1 << prefixlen) - 1 if leading_ones != all_ones: - byteslen = cls._max_prefixlen // 8 + byteslen = cls.max_prefixlen // 8 details = ip_int.to_bytes(byteslen, 'big') msg = 'Netmask pattern %r mixes zeroes & ones' raise ValueError(msg % details) @@ -492,7 +487,7 @@ def _prefix_from_prefix_string(cls, prefixlen_str): prefixlen = int(prefixlen_str) except ValueError: cls._report_invalid_netmask(prefixlen_str) - if not (0 <= prefixlen <= cls._max_prefixlen): + if not (0 <= prefixlen <= cls.max_prefixlen): cls._report_invalid_netmask(prefixlen_str) return prefixlen @@ -542,7 +537,7 @@ def _split_addr_prefix(cls, address): """ # a packed address or integer if isinstance(address, (bytes, int)): - return address, cls._max_prefixlen + return address, cls.max_prefixlen if not isinstance(address, tuple): # Assume input argument to be string or any object representation @@ -552,7 +547,7 @@ def _split_addr_prefix(cls, address): # Constructing from a tuple (addr, [mask]) if len(address) > 1: return address - return address[0], cls._max_prefixlen + return address[0], cls.max_prefixlen def __reduce__(self): return self.__class__, (str(self),) @@ -577,14 +572,14 @@ def __int__(self): def __eq__(self, other): try: return (self._ip == other._ip - and self._version == other._version) + and self.version == other.version) except AttributeError: return NotImplemented def __lt__(self, other): if not isinstance(other, _BaseAddress): return NotImplemented - if self._version != other._version: + if self.version != other.version: raise TypeError('%s and %s are not of the same version' % ( self, other)) if self._ip != other._ip: @@ -613,7 +608,7 @@ def __hash__(self): return hash(hex(int(self._ip))) def _get_address_key(self): - return (self._version, self) + return (self.version, self) def __reduce__(self): return self.__class__, (self._ip,) @@ -649,15 +644,15 @@ def __format__(self, fmt): # Set some defaults if fmt_base == 'n': - if self._version == 4: + if self.version == 4: fmt_base = 'b' # Binary is default for ipv4 else: fmt_base = 'x' # Hex is default for ipv6 if fmt_base == 'b': - padlen = self._max_prefixlen + padlen = self.max_prefixlen else: - padlen = self._max_prefixlen // 4 + padlen = self.max_prefixlen // 4 if grouping: padlen += padlen // 4 - 1 @@ -716,7 +711,7 @@ def __getitem__(self, n): def __lt__(self, other): if not isinstance(other, _BaseNetwork): return NotImplemented - if self._version != other._version: + if self.version != other.version: raise TypeError('%s and %s are not of the same version' % ( self, other)) if self.network_address != other.network_address: @@ -727,7 +722,7 @@ def __lt__(self, other): def __eq__(self, other): try: - return (self._version == other._version and + return (self.version == other.version and self.network_address == other.network_address and int(self.netmask) == int(other.netmask)) except AttributeError: @@ -738,7 +733,7 @@ def __hash__(self): def __contains__(self, other): # always false if one is v4 and the other is v6. - if self._version != other._version: + if self.version != other.version: return False # dealing with another network. if isinstance(other, _BaseNetwork): @@ -829,7 +824,7 @@ def address_exclude(self, other): ValueError: If other is not completely contained by self. """ - if not self._version == other._version: + if not self.version == other.version: raise TypeError("%s and %s are not of the same version" % ( self, other)) @@ -901,10 +896,10 @@ def compare_networks(self, other): """ # does this need to raise a ValueError? - if self._version != other._version: + if self.version != other.version: raise TypeError('%s and %s are not of the same type' % ( self, other)) - # self._version == other._version below here: + # self.version == other.version below here: if self.network_address < other.network_address: return -1 if self.network_address > other.network_address: @@ -924,7 +919,7 @@ def _get_networks_key(self): and list.sort(). """ - return (self._version, self.network_address, self.netmask) + return (self.version, self.network_address, self.netmask) def subnets(self, prefixlen_diff=1, new_prefix=None): """The subnets which join to make the current subnet. @@ -952,7 +947,7 @@ def subnets(self, prefixlen_diff=1, new_prefix=None): number means a larger network) """ - if self._prefixlen == self._max_prefixlen: + if self._prefixlen == self.max_prefixlen: yield self return @@ -967,7 +962,7 @@ def subnets(self, prefixlen_diff=1, new_prefix=None): raise ValueError('prefix length diff must be > 0') new_prefixlen = self._prefixlen + prefixlen_diff - if new_prefixlen > self._max_prefixlen: + if new_prefixlen > self.max_prefixlen: raise ValueError( 'prefix length diff %d is invalid for netblock %s' % ( new_prefixlen, self)) @@ -1036,7 +1031,7 @@ def is_multicast(self): def _is_subnet_of(a, b): try: # Always false if one is v4 and the other is v6. - if a._version != b._version: + if a.version != b.version: raise TypeError(f"{a} and {b} are not of the same version") return (b.network_address <= a.network_address and b.broadcast_address >= a.broadcast_address) @@ -1086,7 +1081,11 @@ def is_private(self): """ return any(self.network_address in priv_network and self.broadcast_address in priv_network - for priv_network in self._constants._private_networks) + for priv_network in self._constants._private_networks) and all( + self.network_address not in network and + self.broadcast_address not in network + for network in self._constants._private_networks_exceptions + ) @property def is_global(self): @@ -1142,11 +1141,11 @@ class _BaseV4: """ __slots__ = () - _version = 4 + version = 4 # Equivalent to 255.255.255.255 or 32 bits of 1's. _ALL_ONES = (2**IPV4LENGTH) - 1 - _max_prefixlen = IPV4LENGTH + max_prefixlen = IPV4LENGTH # There are only a handful of valid v4 netmasks, so we cache them all # when constructed (see _make_netmask()). _netmask_cache = {} @@ -1166,7 +1165,7 @@ def _make_netmask(cls, arg): if arg not in cls._netmask_cache: if isinstance(arg, int): prefixlen = arg - if not (0 <= prefixlen <= cls._max_prefixlen): + if not (0 <= prefixlen <= cls.max_prefixlen): cls._report_invalid_netmask(prefixlen) else: try: @@ -1264,15 +1263,6 @@ def _reverse_pointer(self): reverse_octets = str(self).split('.')[::-1] return '.'.join(reverse_octets) + '.in-addr.arpa' - @property - def max_prefixlen(self): - return self._max_prefixlen - - @property - def version(self): - return self._version - - class IPv4Address(_BaseV4, _BaseAddress): """Represent and manipulate single IPv4 Addresses.""" @@ -1333,18 +1323,41 @@ def is_reserved(self): @property @functools.lru_cache() def is_private(self): - """Test if this address is allocated for private networks. + """``True`` if the address is defined as not globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exceptions: - Returns: - A boolean, True if the address is reserved per - iana-ipv4-special-registry. + * ``is_private`` is ``False`` for ``100.64.0.0/10`` + * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_private == address.ipv4_mapped.is_private + ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. """ - return any(self in net for net in self._constants._private_networks) + return ( + any(self in net for net in self._constants._private_networks) + and all(self not in net for net in self._constants._private_networks_exceptions) + ) @property @functools.lru_cache() def is_global(self): + """``True`` if the address is defined as globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exception: + + For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_global == address.ipv4_mapped.is_global + + ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. + """ return self not in self._constants._public_network and not self.is_private @property @@ -1529,9 +1542,9 @@ def __init__(self, address, strict=True): self.network_address = IPv4Address(packed & int(self.netmask)) - if self._prefixlen == (self._max_prefixlen - 1): + if self._prefixlen == (self.max_prefixlen - 1): self.hosts = self.__iter__ - elif self._prefixlen == (self._max_prefixlen): + elif self._prefixlen == (self.max_prefixlen): self.hosts = lambda: [IPv4Address(addr)] @property @@ -1558,13 +1571,15 @@ class _IPv4Constants: _public_network = IPv4Network('100.64.0.0/10') + # Not globally reachable address blocks listed on + # https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml _private_networks = [ IPv4Network('0.0.0.0/8'), IPv4Network('10.0.0.0/8'), IPv4Network('127.0.0.0/8'), IPv4Network('169.254.0.0/16'), IPv4Network('172.16.0.0/12'), - IPv4Network('192.0.0.0/29'), + IPv4Network('192.0.0.0/24'), IPv4Network('192.0.0.170/31'), IPv4Network('192.0.2.0/24'), IPv4Network('192.168.0.0/16'), @@ -1575,6 +1590,11 @@ class _IPv4Constants: IPv4Network('255.255.255.255/32'), ] + _private_networks_exceptions = [ + IPv4Network('192.0.0.9/32'), + IPv4Network('192.0.0.10/32'), + ] + _reserved_network = IPv4Network('240.0.0.0/4') _unspecified_address = IPv4Address('0.0.0.0') @@ -1594,11 +1614,11 @@ class _BaseV6: """ __slots__ = () - _version = 6 + version = 6 _ALL_ONES = (2**IPV6LENGTH) - 1 _HEXTET_COUNT = 8 _HEX_DIGITS = frozenset('0123456789ABCDEFabcdef') - _max_prefixlen = IPV6LENGTH + max_prefixlen = IPV6LENGTH # There are only a bunch of valid v6 netmasks, so we cache them all # when constructed (see _make_netmask()). @@ -1616,7 +1636,7 @@ def _make_netmask(cls, arg): if arg not in cls._netmask_cache: if isinstance(arg, int): prefixlen = arg - if not (0 <= prefixlen <= cls._max_prefixlen): + if not (0 <= prefixlen <= cls.max_prefixlen): cls._report_invalid_netmask(prefixlen) else: prefixlen = cls._prefix_from_prefix_string(arg) @@ -1878,15 +1898,6 @@ def _split_scope_id(ip_str): raise AddressValueError('Invalid IPv6 address: "%r"' % ip_str) return addr, scope_id - @property - def max_prefixlen(self): - return self._max_prefixlen - - @property - def version(self): - return self._version - - class IPv6Address(_BaseV6, _BaseAddress): """Represent and manipulate single IPv6 Addresses.""" @@ -1936,12 +1947,21 @@ def __init__(self, address): def _explode_shorthand_ip_string(self): ipv4_mapped = self.ipv4_mapped if ipv4_mapped is None: - long_form = super()._explode_shorthand_ip_string() - else: - prefix_len = 30 - raw_exploded_str = super()._explode_shorthand_ip_string() - long_form = "%s%s" % (raw_exploded_str[:prefix_len], str(ipv4_mapped)) - return long_form + return super()._explode_shorthand_ip_string() + prefix_len = 30 + raw_exploded_str = super()._explode_shorthand_ip_string() + return f"{raw_exploded_str[:prefix_len]}{ipv4_mapped!s}" + + def _reverse_pointer(self): + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is None: + return super()._reverse_pointer() + prefix_len = 30 + raw_exploded_str = super()._explode_shorthand_ip_string()[:prefix_len] + # ipv4 encoded using hexadecimal nibbles instead of decimals + ipv4_int = ipv4_mapped._ip + reverse_chars = f"{raw_exploded_str}{ipv4_int:008x}"[::-1].replace(':', '') + return '.'.join(reverse_chars) + '.ip6.arpa' def _ipv4_mapped_ipv6_to_str(self): """Return convenient text representation of IPv4-mapped IPv6 address @@ -2009,6 +2029,9 @@ def is_multicast(self): See RFC 2373 2.7 for details. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_multicast return self in self._constants._multicast_network @property @@ -2020,6 +2043,9 @@ def is_reserved(self): reserved IPv6 Network ranges. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_reserved return any(self in x for x in self._constants._reserved_networks) @property @@ -2030,6 +2056,9 @@ def is_link_local(self): A boolean, True if the address is reserved per RFC 4291. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_link_local return self in self._constants._linklocal_network @property @@ -2049,28 +2078,46 @@ def is_site_local(self): @property @functools.lru_cache() def is_private(self): - """Test if this address is allocated for private networks. + """``True`` if the address is defined as not globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exceptions: - Returns: - A boolean, True if the address is reserved per - iana-ipv6-special-registry, or is ipv4_mapped and is - reserved in the iana-ipv4-special-registry. + * ``is_private`` is ``False`` for ``100.64.0.0/10`` + * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_private == address.ipv4_mapped.is_private + ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. """ ipv4_mapped = self.ipv4_mapped if ipv4_mapped is not None: return ipv4_mapped.is_private - return any(self in net for net in self._constants._private_networks) + return ( + any(self in net for net in self._constants._private_networks) + and all(self not in net for net in self._constants._private_networks_exceptions) + ) @property def is_global(self): - """Test if this address is allocated for public networks. + """``True`` if the address is defined as globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exception: - Returns: - A boolean, true if the address is not reserved per - iana-ipv6-special-registry. + For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_global == address.ipv4_mapped.is_global + ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_global return not self.is_private @property @@ -2082,6 +2129,9 @@ def is_unspecified(self): RFC 2373 2.5.2. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_unspecified return self._ip == 0 @property @@ -2093,6 +2143,9 @@ def is_loopback(self): RFC 2373 2.5.3. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_loopback return self._ip == 1 @property @@ -2209,7 +2262,7 @@ def is_unspecified(self): @property def is_loopback(self): - return self._ip == 1 and self.network.is_loopback + return super().is_loopback and self.network.is_loopback class IPv6Network(_BaseV6, _BaseNetwork): @@ -2271,9 +2324,9 @@ def __init__(self, address, strict=True): self.network_address = IPv6Address(packed & int(self.netmask)) - if self._prefixlen == (self._max_prefixlen - 1): + if self._prefixlen == (self.max_prefixlen - 1): self.hosts = self.__iter__ - elif self._prefixlen == self._max_prefixlen: + elif self._prefixlen == self.max_prefixlen: self.hosts = lambda: [IPv6Address(addr)] def hosts(self): @@ -2310,19 +2363,33 @@ class _IPv6Constants: _multicast_network = IPv6Network('ff00::/8') + # Not globally reachable address blocks listed on + # https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml _private_networks = [ IPv6Network('::1/128'), IPv6Network('::/128'), IPv6Network('::ffff:0:0/96'), + IPv6Network('64:ff9b:1::/48'), IPv6Network('100::/64'), IPv6Network('2001::/23'), - IPv6Network('2001:2::/48'), IPv6Network('2001:db8::/32'), - IPv6Network('2001:10::/28'), + # IANA says N/A, let's consider it not globally reachable to be safe + IPv6Network('2002::/16'), + # RFC 9637: https://www.rfc-editor.org/rfc/rfc9637.html#section-6-2.2 + IPv6Network('3fff::/20'), IPv6Network('fc00::/7'), IPv6Network('fe80::/10'), ] + _private_networks_exceptions = [ + IPv6Network('2001:1::1/128'), + IPv6Network('2001:1::2/128'), + IPv6Network('2001:3::/32'), + IPv6Network('2001:4:112::/48'), + IPv6Network('2001:20::/28'), + IPv6Network('2001:30::/28'), + ] + _reserved_networks = [ IPv6Network('::/8'), IPv6Network('100::/8'), IPv6Network('200::/7'), IPv6Network('400::/6'), diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index ed2c74771ea87d..1d972d22ded072 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -86,13 +86,13 @@ '[2.0, 1.0]' -Using json.tool from the shell to validate and pretty-print:: +Using json from the shell to validate and pretty-print:: - $ echo '{"json":"obj"}' | python -m json.tool + $ echo '{"json":"obj"}' | python -m json { "json": "obj" } - $ echo '{ 1.2:3.4}' | python -m json.tool + $ echo '{ 1.2:3.4}' | python -m json Expecting property name enclosed in double quotes: line 1 column 3 (char 2) """ __version__ = '2.0.9' diff --git a/Lib/json/__main__.py b/Lib/json/__main__.py new file mode 100644 index 00000000000000..1808eaddb62776 --- /dev/null +++ b/Lib/json/__main__.py @@ -0,0 +1,20 @@ +"""Command-line tool to validate and pretty-print JSON + +Usage:: + + $ echo '{"json":"obj"}' | python -m json + { + "json": "obj" + } + $ echo '{ 1.2:3.4}' | python -m json + Expecting property name enclosed in double quotes: line 1 column 3 (char 2) + +""" +import json.tool + + +if __name__ == '__main__': + try: + json.tool.main() + except BrokenPipeError as exc: + raise SystemExit(exc.errno) diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py index d69a45d6793069..ff4bfcdcc407b9 100644 --- a/Lib/json/decoder.py +++ b/Lib/json/decoder.py @@ -50,17 +50,18 @@ def __reduce__(self): } +HEXDIGITS = re.compile(r'[0-9A-Fa-f]{4}', FLAGS) STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) BACKSLASH = { '"': '"', '\\': '\\', '/': '/', 'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t', } -def _decode_uXXXX(s, pos): - esc = s[pos + 1:pos + 5] - if len(esc) == 4 and esc[1] not in 'xX': +def _decode_uXXXX(s, pos, _m=HEXDIGITS.match): + esc = _m(s, pos + 1) + if esc is not None: try: - return int(esc, 16) + return int(esc.group(), 16) except ValueError: pass msg = "Invalid \\uXXXX escape" diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 597849eca0524a..b804224098e14f 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -244,15 +244,18 @@ def floatstr(o, allow_nan=self.allow_nan, return text - if (_one_shot and c_make_encoder is not None - and self.indent is None): + if self.indent is None or isinstance(self.indent, str): + indent = self.indent + else: + indent = ' ' * self.indent + if _one_shot and c_make_encoder is not None: _iterencode = c_make_encoder( - markers, self.default, _encoder, self.indent, + markers, self.default, _encoder, indent, self.key_separator, self.item_separator, self.sort_keys, self.skipkeys, self.allow_nan) else: _iterencode = _make_iterencode( - markers, self.default, _encoder, self.indent, floatstr, + markers, self.default, _encoder, indent, floatstr, self.key_separator, self.item_separator, self.sort_keys, self.skipkeys, _one_shot) return _iterencode(o, 0) @@ -272,9 +275,6 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _intstr=int.__repr__, ): - if _indent is not None and not isinstance(_indent, str): - _indent = ' ' * _indent - def _iterencode_list(lst, _current_indent_level): if not lst: yield '[]' @@ -293,37 +293,40 @@ def _iterencode_list(lst, _current_indent_level): else: newline_indent = None separator = _item_separator - first = True - for value in lst: - if first: - first = False - else: + for i, value in enumerate(lst): + if i: buf = separator - if isinstance(value, str): - yield buf + _encoder(value) - elif value is None: - yield buf + 'null' - elif value is True: - yield buf + 'true' - elif value is False: - yield buf + 'false' - elif isinstance(value, int): - # Subclasses of int/float may override __repr__, but we still - # want to encode them as integers/floats in JSON. One example - # within the standard library is IntEnum. - yield buf + _intstr(value) - elif isinstance(value, float): - # see comment above for int - yield buf + _floatstr(value) - else: - yield buf - if isinstance(value, (list, tuple)): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) + try: + if isinstance(value, str): + yield buf + _encoder(value) + elif value is None: + yield buf + 'null' + elif value is True: + yield buf + 'true' + elif value is False: + yield buf + 'false' + elif isinstance(value, int): + # Subclasses of int/float may override __repr__, but we still + # want to encode them as integers/floats in JSON. One example + # within the standard library is IntEnum. + yield buf + _intstr(value) + elif isinstance(value, float): + # see comment above for int + yield buf + _floatstr(value) else: - chunks = _iterencode(value, _current_indent_level) - yield from chunks + yield buf + if isinstance(value, (list, tuple)): + chunks = _iterencode_list(value, _current_indent_level) + elif isinstance(value, dict): + chunks = _iterencode_dict(value, _current_indent_level) + else: + chunks = _iterencode(value, _current_indent_level) + yield from chunks + except GeneratorExit: + raise + except BaseException as exc: + exc.add_note(f'when serializing {type(lst).__name__} item {i}') + raise if newline_indent is not None: _current_indent_level -= 1 yield '\n' + _indent * _current_indent_level @@ -382,28 +385,34 @@ def _iterencode_dict(dct, _current_indent_level): yield item_separator yield _encoder(key) yield _key_separator - if isinstance(value, str): - yield _encoder(value) - elif value is None: - yield 'null' - elif value is True: - yield 'true' - elif value is False: - yield 'false' - elif isinstance(value, int): - # see comment for int/float in _make_iterencode - yield _intstr(value) - elif isinstance(value, float): - # see comment for int/float in _make_iterencode - yield _floatstr(value) - else: - if isinstance(value, (list, tuple)): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) + try: + if isinstance(value, str): + yield _encoder(value) + elif value is None: + yield 'null' + elif value is True: + yield 'true' + elif value is False: + yield 'false' + elif isinstance(value, int): + # see comment for int/float in _make_iterencode + yield _intstr(value) + elif isinstance(value, float): + # see comment for int/float in _make_iterencode + yield _floatstr(value) else: - chunks = _iterencode(value, _current_indent_level) - yield from chunks + if isinstance(value, (list, tuple)): + chunks = _iterencode_list(value, _current_indent_level) + elif isinstance(value, dict): + chunks = _iterencode_dict(value, _current_indent_level) + else: + chunks = _iterencode(value, _current_indent_level) + yield from chunks + except GeneratorExit: + raise + except BaseException as exc: + exc.add_note(f'when serializing {type(dct).__name__} item {key!r}') + raise if newline_indent is not None: _current_indent_level -= 1 yield '\n' + _indent * _current_indent_level @@ -436,8 +445,14 @@ def _iterencode(o, _current_indent_level): if markerid in markers: raise ValueError("Circular reference detected") markers[markerid] = o - o = _default(o) - yield from _iterencode(o, _current_indent_level) + newobj = _default(o) + try: + yield from _iterencode(newobj, _current_indent_level) + except GeneratorExit: + raise + except BaseException as exc: + exc.add_note(f'when serializing {type(o).__name__} object') + raise if markers is not None: del markers[markerid] return _iterencode diff --git a/Lib/json/scanner.py b/Lib/json/scanner.py index 7a61cfc2d24dce..090897515fe2f3 100644 --- a/Lib/json/scanner.py +++ b/Lib/json/scanner.py @@ -9,7 +9,7 @@ __all__ = ['make_scanner'] NUMBER_RE = re.compile( - r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?', + r'(-?(?:0|[1-9][0-9]*))(\.[0-9]+)?([eE][-+]?[0-9]+)?', (re.VERBOSE | re.MULTILINE | re.DOTALL)) def py_make_scanner(context): diff --git a/Lib/json/tool.py b/Lib/json/tool.py index fdfc3372bcca02..1ba91384c81f27 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py @@ -1,14 +1,7 @@ -r"""Command-line tool to validate and pretty-print JSON - -Usage:: - - $ echo '{"json":"obj"}' | python -m json.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m json.tool - Expecting property name enclosed in double quotes: line 1 column 3 (char 2) +"""Command-line tool to validate and pretty-print JSON +See `json.__main__` for a usage example (invocation as +`python -m json.tool` is supported for backwards compatibility). """ import argparse import json @@ -16,10 +9,9 @@ def main(): - prog = 'python -m json.tool' description = ('A simple command line interface for json module ' 'to validate and pretty-print JSON objects.') - parser = argparse.ArgumentParser(prog=prog, description=description) + parser = argparse.ArgumentParser(description=description) parser.add_argument('infile', nargs='?', help='a JSON file to be validated or pretty-printed', default='-') @@ -86,4 +78,4 @@ def main(): try: main() except BrokenPipeError as exc: - sys.exit(exc.errno) + raise SystemExit(exc.errno) diff --git a/Lib/linecache.py b/Lib/linecache.py index 04c8f45a6c60ca..8ba2df73d5a8fb 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -5,9 +5,6 @@ that name. """ -import sys -import os - __all__ = ["getline", "clearcache", "checkcache", "lazycache"] @@ -52,23 +49,31 @@ def checkcache(filename=None): (This is not checked upon each call!)""" if filename is None: - filenames = list(cache.keys()) - elif filename in cache: - filenames = [filename] + # get keys atomically + filenames = cache.copy().keys() else: - return + filenames = [filename] for filename in filenames: - entry = cache[filename] + try: + entry = cache[filename] + except KeyError: + continue + if len(entry) == 1: # lazy cache entry, leave it lazy. continue size, mtime, lines, fullname = entry if mtime is None: continue # no-op for files loaded via a __loader__ + try: + # This import can fail if the interpreter is shutting down + import os + except ImportError: + return try: stat = os.stat(fullname) - except OSError: + except (OSError, ValueError): cache.pop(filename, None) continue if size != stat.st_size or mtime != stat.st_mtime: @@ -80,6 +85,11 @@ def updatecache(filename, module_globals=None): If something's wrong, print a message, discard the cache entry, and return an empty list.""" + # These imports are not at top level because linecache is in the critical + # path of the interpreter startup and importing os and sys take a lot of time + # and slows down the startup sequence. + import os + import sys import tokenize if filename in cache: @@ -128,16 +138,20 @@ def updatecache(filename, module_globals=None): try: stat = os.stat(fullname) break - except OSError: + except (OSError, ValueError): pass else: return [] + except ValueError: # may be raised by os.stat() + return [] try: with tokenize.open(fullname) as fp: lines = fp.readlines() except (OSError, UnicodeDecodeError, SyntaxError): return [] - if lines and not lines[-1].endswith('\n'): + if not lines: + lines = ['\n'] + elif not lines[-1].endswith('\n'): lines[-1] += '\n' size, mtime = stat.st_size, stat.st_mtime cache[filename] = size, mtime, lines, fullname diff --git a/Lib/locale.py b/Lib/locale.py index e0cb4c5449d556..d8c09f1123d318 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -1460,7 +1460,8 @@ def getpreferredencoding(do_setlocale=True): # to include every locale up to Windows Vista. # # NOTE: this mapping is incomplete. If your language is missing, please -# submit a bug report to the Python bug tracker at http://bugs.python.org/ +# submit a bug report as detailed in the Python devguide at: +# https://devguide.python.org/triage/issue-tracker/ # Make sure you include the missing language identifier and the suggested # locale code. # diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index fcec9e76b98661..aa9b79d8cab4bb 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -37,7 +37,7 @@ 'captureWarnings', 'critical', 'debug', 'disable', 'error', 'exception', 'fatal', 'getLevelName', 'getLogger', 'getLoggerClass', 'info', 'log', 'makeLogRecord', 'setLoggerClass', 'shutdown', - 'warning', 'getLogRecordFactory', 'setLogRecordFactory', + 'warn', 'warning', 'getLogRecordFactory', 'setLogRecordFactory', 'lastResort', 'raiseExceptions', 'getLevelNamesMapping', 'getHandlerByName', 'getHandlerNames'] @@ -56,7 +56,7 @@ # #_startTime is used as the base when calculating the relative time of events # -_startTime = time.time() +_startTime = time.time_ns() # #raiseExceptions is used to see if exceptions during handling should be @@ -300,7 +300,7 @@ def __init__(self, name, level, pathname, lineno, """ Initialize a logging record with interesting information. """ - ct = time.time() + ct = time.time_ns() self.name = name self.msg = msg # @@ -339,9 +339,17 @@ def __init__(self, name, level, pathname, lineno, self.stack_info = sinfo self.lineno = lineno self.funcName = func - self.created = ct - self.msecs = int((ct - int(ct)) * 1000) + 0.0 # see gh-89047 - self.relativeCreated = (self.created - _startTime) * 1000 + self.created = ct / 1e9 # ns to float seconds + # Get the number of whole milliseconds (0-999) in the fractional part of seconds. + # Eg: 1_677_903_920_999_998_503 ns --> 999_998_503 ns--> 999 ms + # Convert to float by adding 0.0 for historical reasons. See gh-89047 + self.msecs = (ct % 1_000_000_000) // 1_000_000 + 0.0 + if self.msecs == 999.0 and int(self.created) != ct // 1_000_000_000: + # ns -> sec conversion can round up, e.g: + # 1_677_903_920_999_999_900 ns --> 1_677_903_921.0 sec + self.msecs = 0.0 + + self.relativeCreated = (ct - _startTime) / 1e6 if logThreads: self.thread = threading.get_ident() self.threadName = threading.current_thread().name @@ -572,7 +580,7 @@ class Formatter(object): %(lineno)d Source line number where the logging call was issued (if available) %(funcName)s Function name - %(created)f Time when the LogRecord was created (time.time() + %(created)f Time when the LogRecord was created (time.time_ns() / 1e9 return value) %(asctime)s Textual time when the LogRecord was created %(msecs)d Millisecond portion of the creation time @@ -1522,6 +1530,11 @@ def warning(self, msg, *args, **kwargs): if self.isEnabledFor(WARNING): self._log(WARNING, msg, args, **kwargs) + def warn(self, msg, *args, **kwargs): + warnings.warn("The 'warn' method is deprecated, " + "use 'warning' instead", DeprecationWarning, 2) + self.warning(msg, *args, **kwargs) + def error(self, msg, *args, **kwargs): """ Log 'msg % args' with severity 'ERROR'. @@ -1898,6 +1911,11 @@ def warning(self, msg, *args, **kwargs): """ self.log(WARNING, msg, *args, **kwargs) + def warn(self, msg, *args, **kwargs): + warnings.warn("The 'warn' method is deprecated, " + "use 'warning' instead", DeprecationWarning, 2) + self.warning(msg, *args, **kwargs) + def error(self, msg, *args, **kwargs): """ Delegate an error call to the underlying logger. @@ -2013,7 +2031,7 @@ def basicConfig(**kwargs): that this argument is incompatible with 'filename' - if both are present, 'stream' is ignored. handlers If specified, this should be an iterable of already created - handlers, which will be added to the root handler. Any handler + handlers, which will be added to the root logger. Any handler in the list which does not have a formatter assigned will be assigned the formatter created in this function. force If this keyword is specified as true, any existing handlers @@ -2161,6 +2179,11 @@ def warning(msg, *args, **kwargs): basicConfig() root.warning(msg, *args, **kwargs) +def warn(msg, *args, **kwargs): + warnings.warn("The 'warn' function is deprecated, " + "use 'warning' instead", DeprecationWarning, 2) + warning(msg, *args, **kwargs) + def info(msg, *args, **kwargs): """ Log a message with severity 'INFO' on the root logger. If the logger has diff --git a/Lib/logging/config.py b/Lib/logging/config.py index ea37dd7544564a..6a6a7f726f7e0c 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -497,6 +497,33 @@ def as_tuple(self, value): value = tuple(value) return value +def _is_queue_like_object(obj): + """Check that *obj* implements the Queue API.""" + if isinstance(obj, (queue.Queue, queue.SimpleQueue)): + return True + # defer importing multiprocessing as much as possible + from multiprocessing.queues import Queue as MPQueue + if isinstance(obj, MPQueue): + return True + # Depending on the multiprocessing start context, we cannot create + # a multiprocessing.managers.BaseManager instance 'mm' to get the + # runtime type of mm.Queue() or mm.JoinableQueue() (see gh-119819). + # + # Since we only need an object implementing the Queue API, we only + # do a protocol check, but we do not use typing.runtime_checkable() + # and typing.Protocol to reduce import time (see gh-121723). + # + # Ideally, we would have wanted to simply use strict type checking + # instead of a protocol-based type checking since the latter does + # not check the method signatures. + # + # Note that only 'put_nowait' and 'get' are required by the logging + # queue handler and queue listener (see gh-124653) and that other + # methods are either optional or unused. + minimal_queue_interface = ['put_nowait', 'get'] + return all(callable(getattr(obj, method, None)) + for method in minimal_queue_interface) + class DictConfigurator(BaseConfigurator): """ Configure logging using a dictionary-like object to describe the @@ -725,16 +752,16 @@ def add_filters(self, filterer, filters): def _configure_queue_handler(self, klass, **kwargs): if 'queue' in kwargs: - q = kwargs['queue'] + q = kwargs.pop('queue') else: q = queue.Queue() # unbounded - rhl = kwargs.get('respect_handler_level', False) - if 'listener' in kwargs: - lklass = kwargs['listener'] - else: - lklass = logging.handlers.QueueListener - listener = lklass(q, *kwargs.get('handlers', []), respect_handler_level=rhl) - handler = klass(q) + + rhl = kwargs.pop('respect_handler_level', False) + lklass = kwargs.pop('listener', logging.handlers.QueueListener) + handlers = kwargs.pop('handlers', []) + + listener = lklass(q, *handlers, respect_handler_level=rhl) + handler = klass(q, **kwargs) handler.listener = listener return handler @@ -761,38 +788,39 @@ def configure_handler(self, config): klass = cname else: klass = self.resolve(cname) - if issubclass(klass, logging.handlers.MemoryHandler) and\ - 'target' in config: - # Special case for handler which refers to another handler - try: - tn = config['target'] - th = self.config['handlers'][tn] - if not isinstance(th, logging.Handler): - config.update(config_copy) # restore for deferred cfg - raise TypeError('target not configured yet') - config['target'] = th - except Exception as e: - raise ValueError('Unable to set target handler %r' % tn) from e + if issubclass(klass, logging.handlers.MemoryHandler): + if 'flushLevel' in config: + config['flushLevel'] = logging._checkLevel(config['flushLevel']) + if 'target' in config: + # Special case for handler which refers to another handler + try: + tn = config['target'] + th = self.config['handlers'][tn] + if not isinstance(th, logging.Handler): + config.update(config_copy) # restore for deferred cfg + raise TypeError('target not configured yet') + config['target'] = th + except Exception as e: + raise ValueError('Unable to set target handler %r' % tn) from e elif issubclass(klass, logging.handlers.QueueHandler): # Another special case for handler which refers to other handlers # if 'handlers' not in config: # raise ValueError('No handlers specified for a QueueHandler') if 'queue' in config: - from multiprocessing.queues import Queue as MPQueue qspec = config['queue'] - if not isinstance(qspec, (queue.Queue, MPQueue)): - if isinstance(qspec, str): - q = self.resolve(qspec) - if not callable(q): - raise TypeError('Invalid queue specifier %r' % qspec) - q = q() - elif isinstance(qspec, dict): - if '()' not in qspec: - raise TypeError('Invalid queue specifier %r' % qspec) - q = self.configure_custom(dict(qspec)) - else: + + if isinstance(qspec, str): + q = self.resolve(qspec) + if not callable(q): raise TypeError('Invalid queue specifier %r' % qspec) - config['queue'] = q + config['queue'] = q() + elif isinstance(qspec, dict): + if '()' not in qspec: + raise TypeError('Invalid queue specifier %r' % qspec) + config['queue'] = self.configure_custom(dict(qspec)) + elif not _is_queue_like_object(qspec): + raise TypeError('Invalid queue specifier %r' % qspec) + if 'listener' in config: lspec = config['listener'] if isinstance(lspec, type): @@ -978,7 +1006,8 @@ class ConfigSocketReceiver(ThreadingTCPServer): A simple TCP socket-based logging config receiver. """ - allow_reuse_address = 1 + allow_reuse_address = True + allow_reuse_port = True def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT, handler=None, ready=None, verify=None): diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index cdb014bca04b6e..017c9ab409b7bc 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -23,11 +23,17 @@ To use, simply 'import logging.handlers' and log away! """ -import io, logging, socket, os, pickle, struct, time, re -from stat import ST_DEV, ST_INO, ST_MTIME +import copy +import io +import logging +import os +import pickle import queue +import re +import socket +import struct import threading -import copy +import time # # Some constants... @@ -187,15 +193,18 @@ def shouldRollover(self, record): Basically, see if the supplied record would cause the file to exceed the size limit we have. """ - # See bpo-45401: Never rollover anything other than regular files - if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename): - return False if self.stream is None: # delay was set... self.stream = self._open() if self.maxBytes > 0: # are we rolling over? + pos = self.stream.tell() + if not pos: + # gh-116263: Never rollover an empty file + return False msg = "%s\n" % self.format(record) - self.stream.seek(0, 2) #due to non-posix-compliant Windows feature - if self.stream.tell() + len(msg) >= self.maxBytes: + if pos + len(msg) >= self.maxBytes: + # See bpo-45401: Never rollover anything other than regular files + if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename): + return False return True return False @@ -269,7 +278,7 @@ def __init__(self, filename, when='h', interval=1, backupCount=0, # path object (see Issue #27493), but self.baseFilename will be a string filename = self.baseFilename if os.path.exists(filename): - t = os.stat(filename)[ST_MTIME] + t = int(os.stat(filename).st_mtime) else: t = int(time.time()) self.rolloverAt = self.computeRollover(t) @@ -334,7 +343,10 @@ def computeRollover(self, currentTime): daysToWait = self.dayOfWeek - day else: daysToWait = 6 - day + self.dayOfWeek + 1 - result += daysToWait * (60 * 60 * 24) + result += daysToWait * _MIDNIGHT + result += self.interval - _MIDNIGHT * 7 + else: + result += self.interval - _MIDNIGHT if not self.utc: dstNow = t[-1] dstAtRollover = time.localtime(result)[-1] @@ -459,8 +471,7 @@ class WatchedFileHandler(logging.FileHandler): This handler is not appropriate for use under Windows, because under Windows open files cannot be moved or renamed - logging opens the files with exclusive locks - and so there is no need - for such a handler. Furthermore, ST_INO is not supported under - Windows; stat always returns zero for this value. + for such a handler. This handler is based on a suggestion and patch by Chad J. Schroeder. @@ -476,9 +487,11 @@ def __init__(self, filename, mode='a', encoding=None, delay=False, self._statstream() def _statstream(self): - if self.stream: - sres = os.fstat(self.stream.fileno()) - self.dev, self.ino = sres[ST_DEV], sres[ST_INO] + if self.stream is None: + return + sres = os.fstat(self.stream.fileno()) + self.dev = sres.st_dev + self.ino = sres.st_ino def reopenIfNeeded(self): """ @@ -488,6 +501,9 @@ def reopenIfNeeded(self): has, close the old stream and reopen the file to get the current stream. """ + if self.stream is None: + return + # Reduce the chance of race conditions by stat'ing by path only # once and then fstat'ing our new fd if we opened a new log stream. # See issue #14632: Thanks to John Mulligan for the problem report @@ -495,18 +511,23 @@ def reopenIfNeeded(self): try: # stat the file by path, checking for existence sres = os.stat(self.baseFilename) + + # compare file system stat with that of our stream file handle + reopen = (sres.st_dev != self.dev or sres.st_ino != self.ino) except FileNotFoundError: - sres = None - # compare file system stat with that of our stream file handle - if not sres or sres[ST_DEV] != self.dev or sres[ST_INO] != self.ino: - if self.stream is not None: - # we have an open file handle, clean it up - self.stream.flush() - self.stream.close() - self.stream = None # See Issue #21742: _open () might fail. - # open a new file handle and get new stat info from that fd - self.stream = self._open() - self._statstream() + reopen = True + + if not reopen: + return + + # we have an open file handle, clean it up + self.stream.flush() + self.stream.close() + self.stream = None # See Issue #21742: _open () might fail. + + # open a new file handle and get new stat info from that fd + self.stream = self._open() + self._statstream() def emit(self, record): """ @@ -834,7 +855,7 @@ class SysLogHandler(logging.Handler): } def __init__(self, address=('localhost', SYSLOG_UDP_PORT), - facility=LOG_USER, socktype=None): + facility=LOG_USER, socktype=None, timeout=None): """ Initialize a handler. @@ -851,6 +872,7 @@ def __init__(self, address=('localhost', SYSLOG_UDP_PORT), self.address = address self.facility = facility self.socktype = socktype + self.timeout = timeout self.socket = None self.createSocket() @@ -912,6 +934,8 @@ def createSocket(self): err = sock = None try: sock = socket.socket(af, socktype, proto) + if self.timeout: + sock.settimeout(self.timeout) if socktype == socket.SOCK_STREAM: sock.connect(sa) break diff --git a/Lib/lzma.py b/Lib/lzma.py index 800f52198fbb79..946066aa0fba56 100644 --- a/Lib/lzma.py +++ b/Lib/lzma.py @@ -25,11 +25,11 @@ import io import os from _lzma import * -from _lzma import _encode_filter_properties, _decode_filter_properties +from _lzma import _encode_filter_properties, _decode_filter_properties # noqa: F401 import _compression -_MODE_CLOSED = 0 +# Value 0 no longer used _MODE_READ = 1 # Value 2 no longer used _MODE_WRITE = 3 @@ -92,7 +92,7 @@ def __init__(self, filename=None, mode="r", *, """ self._fp = None self._closefp = False - self._mode = _MODE_CLOSED + self._mode = None if mode in ("r", "rb"): if check != -1: @@ -137,7 +137,7 @@ def close(self): May be called more than once without error. Once the file is closed, any other operation on it will raise a ValueError. """ - if self._mode == _MODE_CLOSED: + if self.closed: return try: if self._mode == _MODE_READ: @@ -153,12 +153,20 @@ def close(self): finally: self._fp = None self._closefp = False - self._mode = _MODE_CLOSED @property def closed(self): """True if this file is closed.""" - return self._mode == _MODE_CLOSED + return self._fp is None + + @property + def name(self): + self._check_not_closed() + return self._fp.name + + @property + def mode(self): + return 'wb' if self._mode == _MODE_WRITE else 'rb' def fileno(self): """Return the file descriptor for the underlying file.""" diff --git a/Lib/mailbox.py b/Lib/mailbox.py index 746811bd559412..b00d9e8634c785 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -750,9 +750,13 @@ def flush(self): _sync_close(new_file) # self._file is about to get replaced, so no need to sync. self._file.close() - # Make sure the new file's mode is the same as the old file's - mode = os.stat(self._path).st_mode - os.chmod(new_file.name, mode) + # Make sure the new file's mode and owner are the same as the old file's + info = os.stat(self._path) + os.chmod(new_file.name, info.st_mode) + try: + os.chown(new_file.name, info.st_uid, info.st_gid) + except (AttributeError, OSError): + pass try: os.rename(new_file.name, self._path) except FileExistsError: diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 51b99701c9d727..753238354f6d36 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -23,11 +23,6 @@ read_mime_types(file) -- parse one file, return a dictionary or None """ -import os -import sys -import posixpath -import urllib.parse - try: from _winapi import _mimetypes_read_windows_registry except ImportError: @@ -40,7 +35,7 @@ __all__ = [ "knownfiles", "inited", "MimeTypes", - "guess_type", "guess_all_extensions", "guess_extension", + "guess_type", "guess_file_type", "guess_all_extensions", "guess_extension", "add_type", "init", "read_mime_types", "suffix_map", "encodings_map", "types_map", "common_types" ] @@ -116,11 +111,21 @@ def guess_type(self, url, strict=True): mapped to '.tar.gz'. (This is table-driven too, using the dictionary suffix_map.) - Optional `strict' argument when False adds a bunch of commonly found, + Optional 'strict' argument when False adds a bunch of commonly found, but non-standard types. """ + # Lazy import to improve module import time + import os + import urllib.parse + + # TODO: Deprecate accepting file paths (in particular path-like objects). url = os.fspath(url) - scheme, url = urllib.parse._splittype(url) + p = urllib.parse.urlparse(url) + if p.scheme and len(p.scheme) > 1: + scheme = p.scheme + url = p.path + else: + return self.guess_file_type(url, strict=strict) if scheme == 'data': # syntax of data URLs: # dataurl := "data:" [ mediatype ] [ ";base64" ] "," data @@ -140,13 +145,32 @@ def guess_type(self, url, strict=True): if '=' in type or '/' not in type: type = 'text/plain' return type, None # never compressed, so encoding is None - base, ext = posixpath.splitext(url) + + # Lazy import to improve module import time + import posixpath + + return self._guess_file_type(url, strict, posixpath.splitext) + + def guess_file_type(self, path, *, strict=True): + """Guess the type of a file based on its path. + + Similar to guess_type(), but takes file path instead of URL. + """ + # Lazy import to improve module import time + import os + + path = os.fsdecode(path) + path = os.path.splitdrive(path)[1] + return self._guess_file_type(path, strict, os.path.splitext) + + def _guess_file_type(self, path, strict, splitext): + base, ext = splitext(path) while (ext_lower := ext.lower()) in self.suffix_map: - base, ext = posixpath.splitext(base + self.suffix_map[ext_lower]) + base, ext = splitext(base + self.suffix_map[ext_lower]) # encodings_map is case sensitive if ext in self.encodings_map: encoding = self.encodings_map[ext] - base, ext = posixpath.splitext(base) + base, ext = splitext(base) else: encoding = None ext = ext.lower() @@ -167,9 +191,9 @@ def guess_all_extensions(self, type, strict=True): Return value is a list of strings giving the possible filename extensions, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data stream, - but would be mapped to the MIME type `type' by guess_type(). + but would be mapped to the MIME type 'type' by guess_type(). - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ type = type.lower() @@ -186,11 +210,11 @@ def guess_extension(self, type, strict=True): Return value is a string giving a filename extension, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data - stream, but would be mapped to the MIME type `type' by - guess_type(). If no extension can be guessed for `type', None + stream, but would be mapped to the MIME type 'type' by + guess_type(). If no extension can be guessed for 'type', None is returned. - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ extensions = self.guess_all_extensions(type, strict) @@ -296,7 +320,7 @@ def guess_type(url, strict=True): to ".tar.gz". (This is table-driven too, using the dictionary suffix_map). - Optional `strict' argument when false adds a bunch of commonly found, but + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ if _db is None: @@ -304,17 +328,27 @@ def guess_type(url, strict=True): return _db.guess_type(url, strict) +def guess_file_type(path, *, strict=True): + """Guess the type of a file based on its path. + + Similar to guess_type(), but takes file path instead of URL. + """ + if _db is None: + init() + return _db.guess_file_type(path, strict=strict) + + def guess_all_extensions(type, strict=True): """Guess the extensions for a file based on its MIME type. Return value is a list of strings giving the possible filename extensions, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data - stream, but would be mapped to the MIME type `type' by - guess_type(). If no extension can be guessed for `type', None + stream, but would be mapped to the MIME type 'type' by + guess_type(). If no extension can be guessed for 'type', None is returned. - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ if _db is None: @@ -327,10 +361,10 @@ def guess_extension(type, strict=True): Return value is a string giving a filename extension, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data stream, but would be mapped to the - MIME type `type' by guess_type(). If no extension can be guessed for - `type', None is returned. + MIME type 'type' by guess_type(). If no extension can be guessed for + 'type', None is returned. - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ if _db is None: @@ -371,6 +405,9 @@ def init(files=None): else: db = _db + # Lazy import to improve module import time + import os + for file in files: if os.path.isfile(file): db.read(file) @@ -417,7 +454,7 @@ def _default_mime_types(): } # Before adding new types, make sure they are either registered with IANA, - # at http://www.iana.org/assignments/media-types + # at https://www.iana.org/assignments/media-types/media-types.xhtml # or extensions, i.e. using the x- prefix # If you add to these, please keep them sorted by mime type. @@ -451,6 +488,7 @@ def _default_mime_types(): '.m3u8' : 'application/vnd.apple.mpegurl', '.xls' : 'application/vnd.ms-excel', '.xlb' : 'application/vnd.ms-excel', + '.eot' : 'application/vnd.ms-fontobject', '.ppt' : 'application/vnd.ms-powerpoint', '.pot' : 'application/vnd.ms-powerpoint', '.ppa' : 'application/vnd.ms-powerpoint', @@ -506,6 +544,7 @@ def _default_mime_types(): '.ass' : 'audio/aac', '.au' : 'audio/basic', '.snd' : 'audio/basic', + '.mka' : 'audio/matroska', '.mp3' : 'audio/mpeg', '.mp2' : 'audio/mpeg', '.opus' : 'audio/opus', @@ -514,21 +553,34 @@ def _default_mime_types(): '.aiff' : 'audio/x-aiff', '.ra' : 'audio/x-pn-realaudio', '.wav' : 'audio/x-wav', + '.otf' : 'font/otf', + '.ttf' : 'font/ttf', + '.woff' : 'font/woff', + '.woff2' : 'font/woff2', '.avif' : 'image/avif', '.bmp' : 'image/bmp', + '.emf' : 'image/emf', + '.fits' : 'image/fits', + '.g3' : 'image/g3fax', '.gif' : 'image/gif', '.ief' : 'image/ief', + '.jp2' : 'image/jp2', '.jpg' : 'image/jpeg', '.jpe' : 'image/jpeg', '.jpeg' : 'image/jpeg', + '.jpm' : 'image/jpm', + '.jpx' : 'image/jpx', '.heic' : 'image/heic', '.heif' : 'image/heif', '.png' : 'image/png', '.svg' : 'image/svg+xml', + '.t38' : 'image/t38', '.tiff' : 'image/tiff', '.tif' : 'image/tiff', + '.tfx' : 'image/tiff-fx', '.ico' : 'image/vnd.microsoft.icon', '.webp' : 'image/webp', + '.wmf' : 'image/wmf', '.ras' : 'image/x-cmu-raster', '.pnm' : 'image/x-portable-anymap', '.pbm' : 'image/x-portable-bitmap', @@ -546,6 +598,8 @@ def _default_mime_types(): '.csv' : 'text/csv', '.html' : 'text/html', '.htm' : 'text/html', + '.md' : 'text/markdown', + '.markdown': 'text/markdown', '.n3' : 'text/n3', '.txt' : 'text/plain', '.bat' : 'text/plain', @@ -555,14 +609,18 @@ def _default_mime_types(): '.pl' : 'text/plain', '.srt' : 'text/plain', '.rtx' : 'text/richtext', + '.rtf' : 'text/rtf', '.tsv' : 'text/tab-separated-values', '.vtt' : 'text/vtt', '.py' : 'text/x-python', + '.rst' : 'text/x-rst', '.etx' : 'text/x-setext', '.sgm' : 'text/x-sgml', '.sgml' : 'text/x-sgml', '.vcf' : 'text/x-vcard', '.xml' : 'text/xml', + '.mkv' : 'video/matroska', + '.mk3d' : 'video/matroska-3d', '.mp4' : 'video/mp4', '.mpeg' : 'video/mpeg', '.m1v' : 'video/mpeg', @@ -597,6 +655,7 @@ def _default_mime_types(): def _main(): import getopt + import sys USAGE = """\ Usage: mimetypes.py [options] type diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index a0a020f9eeb9b4..ac478ee7f51722 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -72,7 +72,12 @@ def _find_module(name, path=None): if isinstance(spec.loader, importlib.machinery.SourceFileLoader): kind = _PY_SOURCE - elif isinstance(spec.loader, importlib.machinery.ExtensionFileLoader): + elif isinstance( + spec.loader, ( + importlib.machinery.ExtensionFileLoader, + importlib.machinery.AppleFrameworkLoader, + ) + ): kind = _C_EXTENSION elif isinstance(spec.loader, importlib.machinery.SourcelessFileLoader): diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index b7e1e132172d02..d429212d447380 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -11,13 +11,13 @@ import errno import io +import itertools import os import sys import socket import struct -import time import tempfile -import itertools +import time from . import util @@ -39,7 +39,9 @@ # # -BUFSIZE = 8192 +# 64 KiB is the default PIPE buffer size of most POSIX platforms. +BUFSIZE = 64 * 1024 + # A very generous timeout when it comes to local connections... CONNECTION_TIMEOUT = 20. @@ -179,6 +181,10 @@ def close(self): finally: self._handle = None + def _detach(self): + """Stop managing the underlying file descriptor or handle.""" + self._handle = None + def send_bytes(self, buf, offset=0, size=None): """Send the bytes data from a bytes-like object""" self._check_closed() @@ -392,7 +398,8 @@ def _recv(self, size, read=_read): handle = self._handle remaining = size while remaining > 0: - chunk = read(handle, remaining) + to_read = min(BUFSIZE, remaining) + chunk = read(handle, to_read) n = len(chunk) if n == 0: if remaining == size: @@ -846,7 +853,7 @@ def PipeClient(address): _LEGACY_LENGTHS = (_MD5ONLY_MESSAGE_LENGTH, _MD5_DIGEST_LEN) -def _get_digest_name_and_payload(message: bytes) -> (str, bytes): +def _get_digest_name_and_payload(message): # type: (bytes) -> tuple[str, bytes] """Returns a digest name and the payload for a response hash. If a legacy protocol is detected based on the message length @@ -956,7 +963,7 @@ def answer_challenge(connection, authkey: bytes): f'Protocol error, expected challenge: {message=}') message = message[len(_CHALLENGE):] if len(message) < _MD5ONLY_MESSAGE_LENGTH: - raise AuthenticationError('challenge too short: {len(message)} bytes') + raise AuthenticationError(f'challenge too short: {len(message)} bytes') digest = _create_response(authkey, message) connection.send_bytes(digest) response = connection.recv_bytes(256) # reject large message diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index de8a264829dff3..d0a3ad00e53ad8 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -167,7 +167,7 @@ def allow_connection_pickling(self): ''' # This is undocumented. In previous versions of multiprocessing # its only effect was to make socket objects inheritable on Windows. - from . import connection + from . import connection # noqa: F401 def set_executable(self, executable): '''Sets the path to a python.exe or pythonw.exe binary used to run @@ -259,13 +259,12 @@ def get_start_method(self, allow_none=False): def get_all_start_methods(self): """Returns a list of the supported start methods, default first.""" - if sys.platform == 'win32': - return ['spawn'] - else: - methods = ['spawn', 'fork'] if sys.platform == 'darwin' else ['fork', 'spawn'] - if reduction.HAVE_SEND_HANDLE: - methods.append('forkserver') - return methods + default = self._default_context.get_start_method() + start_method_names = [default] + start_method_names.extend( + name for name in _concrete_contexts if name != default + ) + return start_method_names # @@ -320,14 +319,15 @@ def _check_available(self): 'spawn': SpawnContext(), 'forkserver': ForkServerContext(), } - if sys.platform == 'darwin': - # bpo-33725: running arbitrary code after fork() is no longer reliable - # on macOS since macOS 10.14 (Mojave). Use spawn by default instead. - _default_context = DefaultContext(_concrete_contexts['spawn']) + # bpo-33725: running arbitrary code after fork() is no longer reliable + # on macOS since macOS 10.14 (Mojave). Use spawn by default instead. + # gh-84559: We changed everyones default to a thread safeish one in 3.14. + if reduction.HAVE_SEND_HANDLE and sys.platform != 'darwin': + _default_context = DefaultContext(_concrete_contexts['forkserver']) else: - _default_context = DefaultContext(_concrete_contexts['fork']) + _default_context = DefaultContext(_concrete_contexts['spawn']) -else: +else: # Windows class SpawnProcess(process.BaseProcess): _start_method = 'spawn' diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 4642707dae2f4e..df9b9be9d1898b 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -1,3 +1,4 @@ +import atexit import errno import os import selectors @@ -8,6 +9,7 @@ import threading import warnings +from . import AuthenticationError from . import connection from . import process from .context import reduction @@ -24,6 +26,7 @@ MAXFDS_TO_SEND = 256 SIGNED_STRUCT = struct.Struct('q') # large enough for pid_t +_AUTHKEY_LEN = 32 # <= PIPEBUF so it fits a single write to an empty pipe. # # Forkserver class @@ -32,6 +35,7 @@ class ForkServer(object): def __init__(self): + self._forkserver_authkey = None self._forkserver_address = None self._forkserver_alive_fd = None self._forkserver_pid = None @@ -58,6 +62,7 @@ def _stop_unlocked(self): if not util.is_abstract_socket_namespace(self._forkserver_address): os.unlink(self._forkserver_address) self._forkserver_address = None + self._forkserver_authkey = None def set_forkserver_preload(self, modules_names): '''Set list of module names to try to load in forkserver process.''' @@ -82,6 +87,7 @@ def connect_to_new_process(self, fds): process data. ''' self.ensure_running() + assert self._forkserver_authkey if len(fds) + 4 >= MAXFDS_TO_SEND: raise ValueError('too many fds') with socket.socket(socket.AF_UNIX) as client: @@ -92,6 +98,18 @@ def connect_to_new_process(self, fds): resource_tracker.getfd()] allfds += fds try: + client.setblocking(True) + wrapped_client = connection.Connection(client.fileno()) + # The other side of this exchange happens in the child as + # implemented in main(). + try: + connection.answer_challenge( + wrapped_client, self._forkserver_authkey) + connection.deliver_challenge( + wrapped_client, self._forkserver_authkey) + finally: + wrapped_client._detach() + del wrapped_client reduction.sendfds(client, allfds) return parent_r, parent_w except: @@ -119,6 +137,7 @@ def ensure_running(self): return # dead, launch it again os.close(self._forkserver_alive_fd) + self._forkserver_authkey = None self._forkserver_address = None self._forkserver_alive_fd = None self._forkserver_pid = None @@ -129,9 +148,9 @@ def ensure_running(self): if self._preload_modules: desired_keys = {'main_path', 'sys_path'} data = spawn.get_preparation_data('ignore') - data = {x: y for x, y in data.items() if x in desired_keys} + main_kws = {x: y for x, y in data.items() if x in desired_keys} else: - data = {} + main_kws = {} with socket.socket(socket.AF_UNIX) as listener: address = connection.arbitrary_address('AF_UNIX') @@ -143,19 +162,31 @@ def ensure_running(self): # all client processes own the write end of the "alive" pipe; # when they all terminate the read end becomes ready. alive_r, alive_w = os.pipe() + # A short lived pipe to initialize the forkserver authkey. + authkey_r, authkey_w = os.pipe() try: - fds_to_pass = [listener.fileno(), alive_r] + fds_to_pass = [listener.fileno(), alive_r, authkey_r] + main_kws['authkey_r'] = authkey_r cmd %= (listener.fileno(), alive_r, self._preload_modules, - data) + main_kws) exe = spawn.get_executable() args = [exe] + util._args_from_interpreter_flags() args += ['-c', cmd] pid = util.spawnv_passfds(exe, args, fds_to_pass) except: os.close(alive_w) + os.close(authkey_w) raise finally: os.close(alive_r) + os.close(authkey_r) + # Authenticate our control socket to prevent access from + # processes we have not shared this key with. + try: + self._forkserver_authkey = os.urandom(_AUTHKEY_LEN) + os.write(authkey_w, self._forkserver_authkey) + finally: + os.close(authkey_w) self._forkserver_address = address self._forkserver_alive_fd = alive_w self._forkserver_pid = pid @@ -164,9 +195,21 @@ def ensure_running(self): # # -def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): - '''Run forkserver.''' +def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, + *, authkey_r=None): + """Run forkserver.""" + if authkey_r is not None: + try: + authkey = os.read(authkey_r, _AUTHKEY_LEN) + assert len(authkey) == _AUTHKEY_LEN, f'{len(authkey)} < {_AUTHKEY_LEN}' + finally: + os.close(authkey_r) + else: + authkey = b'' + if preload: + if sys_path is not None: + sys.path[:] = sys_path if '__main__' in preload and main_path is not None: process.current_process()._inheriting = True try: @@ -254,8 +297,24 @@ def sigchld_handler(*_unused): if listener in rfds: # Incoming fork request with listener.accept()[0] as s: - # Receive fds from client - fds = reduction.recvfds(s, MAXFDS_TO_SEND + 1) + try: + if authkey: + wrapped_s = connection.Connection(s.fileno()) + # The other side of this exchange happens in + # in connect_to_new_process(). + try: + connection.deliver_challenge( + wrapped_s, authkey) + connection.answer_challenge( + wrapped_s, authkey) + finally: + wrapped_s._detach() + del wrapped_s + # Receive fds from client + fds = reduction.recvfds(s, MAXFDS_TO_SEND + 1) + except (EOFError, BrokenPipeError, AuthenticationError): + s.close() + continue if len(fds) > MAXFDS_TO_SEND: raise RuntimeError( "Too many ({0:n}) fds to send".format( @@ -271,6 +330,8 @@ def sigchld_handler(*_unused): selector.close() unused_fds = [alive_r, child_w, sig_r, sig_w] unused_fds.extend(pid_to_fd.values()) + atexit._clear() + atexit.register(util._exit_function) code = _serve_one(child_r, fds, unused_fds, old_handlers) @@ -278,6 +339,7 @@ def sigchld_handler(*_unused): sys.excepthook(*sys.exc_info()) sys.stderr.flush() finally: + atexit._run_exitfuncs() os._exit(code) else: # Send pid to client process diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 76b915de74d94e..040f4674d735c0 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -18,6 +18,7 @@ import threading import signal import array +import collections.abc import queue import time import types @@ -758,22 +759,29 @@ class BaseProxy(object): _address_to_local = {} _mutex = util.ForkAwareThreadLock() + # Each instance gets a `_serial` number. Unlike `id(...)`, this number + # is never reused. + _next_serial = 1 + def __init__(self, token, serializer, manager=None, authkey=None, exposed=None, incref=True, manager_owned=False): with BaseProxy._mutex: - tls_idset = BaseProxy._address_to_local.get(token.address, None) - if tls_idset is None: - tls_idset = util.ForkAwareLocal(), ProcessLocalSet() - BaseProxy._address_to_local[token.address] = tls_idset + tls_serials = BaseProxy._address_to_local.get(token.address, None) + if tls_serials is None: + tls_serials = util.ForkAwareLocal(), ProcessLocalSet() + BaseProxy._address_to_local[token.address] = tls_serials + + self._serial = BaseProxy._next_serial + BaseProxy._next_serial += 1 # self._tls is used to record the connection used by this # thread to communicate with the manager at token.address - self._tls = tls_idset[0] + self._tls = tls_serials[0] - # self._idset is used to record the identities of all shared - # objects for which the current process owns references and + # self._all_serials is a set used to record the identities of all + # shared objects for which the current process owns references and # which are in the manager at token.address - self._idset = tls_idset[1] + self._all_serials = tls_serials[1] self._token = token self._id = self._token.id @@ -856,20 +864,20 @@ def _incref(self): dispatch(conn, None, 'incref', (self._id,)) util.debug('INCREF %r', self._token.id) - self._idset.add(self._id) + self._all_serials.add(self._serial) state = self._manager and self._manager._state self._close = util.Finalize( self, BaseProxy._decref, - args=(self._token, self._authkey, state, - self._tls, self._idset, self._Client), + args=(self._token, self._serial, self._authkey, state, + self._tls, self._all_serials, self._Client), exitpriority=10 ) @staticmethod - def _decref(token, authkey, state, tls, idset, _Client): - idset.discard(token.id) + def _decref(token, serial, authkey, state, tls, idset, _Client): + idset.discard(serial) # check whether manager is still alive if state is None or state.value == State.STARTED: @@ -1152,10 +1160,10 @@ def set(self, value): BaseListProxy = MakeProxyType('BaseListProxy', ( - '__add__', '__contains__', '__delitem__', '__getitem__', '__len__', - '__mul__', '__reversed__', '__rmul__', '__setitem__', - 'append', 'count', 'extend', 'index', 'insert', 'pop', 'remove', - 'reverse', 'sort', '__imul__' + '__add__', '__contains__', '__delitem__', '__getitem__', '__imul__', + '__len__', '__mul__', '__reversed__', '__rmul__', '__setitem__', + 'append', 'clear', 'copy', 'count', 'extend', 'index', 'insert', 'pop', + 'remove', 'reverse', 'sort', )) class ListProxy(BaseListProxy): def __iadd__(self, value): @@ -1167,18 +1175,25 @@ def __imul__(self, value): __class_getitem__ = classmethod(types.GenericAlias) +collections.abc.MutableSequence.register(BaseListProxy) -_BaseDictProxy = MakeProxyType('DictProxy', ( - '__contains__', '__delitem__', '__getitem__', '__iter__', '__len__', - '__setitem__', 'clear', 'copy', 'get', 'items', +_BaseDictProxy = MakeProxyType('_BaseDictProxy', ( + '__contains__', '__delitem__', '__getitem__', '__ior__', '__iter__', + '__len__', '__or__', '__reversed__', '__ror__', + '__setitem__', 'clear', 'copy', 'fromkeys', 'get', 'items', 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values' )) _BaseDictProxy._method_to_typeid_ = { '__iter__': 'Iterator', } class DictProxy(_BaseDictProxy): + def __ior__(self, value): + self._callmethod('__ior__', (value,)) + return self + __class_getitem__ = classmethod(types.GenericAlias) +collections.abc.MutableMapping.register(_BaseDictProxy) ArrayProxy = MakeProxyType('ArrayProxy', ( '__len__', '__getitem__', '__setitem__' diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index 625981cf47627c..a57ef6bdad5ccc 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -1,3 +1,4 @@ +import atexit import os import signal @@ -66,10 +67,13 @@ def _launch(self, process_obj): self.pid = os.fork() if self.pid == 0: try: + atexit._clear() + atexit.register(util._exit_function) os.close(parent_r) os.close(parent_w) code = process_obj._bootstrap(parent_sentinel=child_r) finally: + atexit._run_exitfuncs() os._exit(code) else: os.close(child_w) diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index 49d4c7eea22411..62fb0ddbf91a5d 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -3,6 +3,7 @@ import signal import sys import _winapi +from subprocess import STARTUPINFO, STARTF_FORCEOFFFEEDBACK from .context import reduction, get_spawning_popen, set_spawning_popen from . import spawn @@ -74,7 +75,8 @@ def __init__(self, process_obj): try: hp, ht, pid, tid = _winapi.CreateProcess( python_exe, cmd, - None, None, False, 0, env, None, None) + None, None, False, 0, env, None, + STARTUPINFO(dwFlags=STARTF_FORCEOFFFEEDBACK)) _winapi.CloseHandle(ht) except: _winapi.CloseHandle(rhandle) diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 271ba3fd325138..b45f7df476f7d8 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -310,11 +310,8 @@ def _bootstrap(self, parent_sentinel=None): # _run_after_forkers() is executed del old_process util.info('child process calling self.run()') - try: - self.run() - exitcode = 0 - finally: - util._exit_function() + self.run() + exitcode = 0 except SystemExit as e: if e.code is None: exitcode = 0 diff --git a/Lib/multiprocessing/reduction.py b/Lib/multiprocessing/reduction.py index 5593f0682f7fce..fcccd3eef86cc7 100644 --- a/Lib/multiprocessing/reduction.py +++ b/Lib/multiprocessing/reduction.py @@ -139,15 +139,12 @@ def detach(self): __all__ += ['DupFd', 'sendfds', 'recvfds'] import array - # On MacOSX we should acknowledge receipt of fds -- see Issue14669 - ACKNOWLEDGE = sys.platform == 'darwin' - def sendfds(sock, fds): '''Send an array of fds over an AF_UNIX socket.''' fds = array.array('i', fds) msg = bytes([len(fds) % 256]) sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)]) - if ACKNOWLEDGE and sock.recv(1) != b'A': + if sock.recv(1) != b'A': raise RuntimeError('did not receive acknowledgement of fd') def recvfds(sock, size): @@ -158,8 +155,11 @@ def recvfds(sock, size): if not msg and not ancdata: raise EOFError try: - if ACKNOWLEDGE: - sock.send(b'A') + # We send/recv an Ack byte after the fds to work around an old + # macOS bug; it isn't clear if this is still required but it + # makes unit testing fd sending easier. + # See: https://github.com/python/cpython/issues/58874 + sock.send(b'A') # Acknowledge if len(ancdata) != 1: raise RuntimeError('received %d items of ancdata' % len(ancdata)) diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 20ddd9c50e3d88..90e036ae905afa 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -155,13 +155,14 @@ def ensure_running(self): # that can make the child die before it registers signal handlers # for SIGINT and SIGTERM. The mask is unregistered after spawning # the child. + prev_sigmask = None try: if _HAVE_SIGMASK: - signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) + prev_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) pid = util.spawnv_passfds(exe, args, fds_to_pass) finally: - if _HAVE_SIGMASK: - signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS) + if prev_sigmask is not None: + signal.pthread_sigmask(signal.SIG_SETMASK, prev_sigmask) except: os.close(w) raise diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 67e70fdc27cf31..99a8ce3320ad4e 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -539,6 +539,6 @@ def index(self, value): if value == entry: return position else: - raise ValueError(f"{value!r} not in this container") + raise ValueError("ShareableList.index(x): x not in list") __class_getitem__ = classmethod(types.GenericAlias) diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index 3ccbfe311c71f3..edd6c2543a7435 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -21,22 +21,21 @@ from . import process from . import util -# Try to import the mp.synchronize module cleanly, if it fails -# raise ImportError for platforms lacking a working sem_open implementation. -# See issue 3770 +# TODO: Do any platforms still lack a functioning sem_open? try: from _multiprocessing import SemLock, sem_unlink -except (ImportError): +except ImportError: raise ImportError("This platform lacks a functioning sem_open" + - " implementation, therefore, the required" + - " synchronization primitives needed will not" + - " function, see issue 3770.") + " implementation. https://github.com/python/cpython/issues/48020.") # # Constants # -RECURSIVE_MUTEX, SEMAPHORE = list(range(2)) +# These match the enum in Modules/_multiprocessing/semaphore.c +RECURSIVE_MUTEX = 0 +SEMAPHORE = 1 + SEM_VALUE_MAX = _multiprocessing.SemLock.SEM_VALUE_MAX # @@ -174,7 +173,7 @@ def __repr__(self): name = process.current_process().name if threading.current_thread().name != 'MainThread': name += '|' + threading.current_thread().name - elif self._semlock._get_value() == 1: + elif not self._semlock._is_zero(): name = 'None' elif self._semlock._count() > 0: name = 'SomeOtherThread' @@ -200,7 +199,7 @@ def __repr__(self): if threading.current_thread().name != 'MainThread': name += '|' + threading.current_thread().name count = self._semlock._count() - elif self._semlock._get_value() == 1: + elif not self._semlock._is_zero(): name, count = 'None', 0 elif self._semlock._count() > 0: name, count = 'SomeOtherThread', 'nonzero' @@ -360,7 +359,7 @@ def wait(self, timeout=None): return True return False - def __repr__(self) -> str: + def __repr__(self): set_status = 'set' if self.is_set() else 'unset' return f"<{type(self).__qualname__} at {id(self):#x} {set_status}>" # diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 32871850ddec8b..b7192042b9cf47 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -14,7 +14,7 @@ import atexit import threading # we want threading to install it's # cleanup function before multiprocessing does -from subprocess import _args_from_interpreter_flags +from subprocess import _args_from_interpreter_flags # noqa: F401 from . import process @@ -102,11 +102,7 @@ def log_to_stderr(level=None): # Abstract socket support def _platform_supports_abstract_sockets(): - if sys.platform == "linux": - return True - if hasattr(sys, 'getandroidapilevel'): - return True - return False + return sys.platform in ("linux", "android") def is_abstract_socket_namespace(address): @@ -442,15 +438,13 @@ def _flush_std_streams(): def spawnv_passfds(path, args, passfds): import _posixsubprocess - import subprocess passfds = tuple(sorted(map(int, passfds))) errpipe_read, errpipe_write = os.pipe() try: return _posixsubprocess.fork_exec( args, [path], True, passfds, None, None, -1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write, - False, False, -1, None, None, None, -1, None, - subprocess._USE_VFORK) + False, False, -1, None, None, None, -1, None) finally: os.close(errpipe_read) os.close(errpipe_write) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index e7cbfe17ecb3c8..5481bb8888ef59 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -19,7 +19,6 @@ import os import sys -import stat import genericpath from genericpath import * @@ -29,7 +28,8 @@ "ismount","isreserved","expanduser","expandvars","normpath", "abspath","curdir","pardir","sep","pathsep","defpath","altsep", "extsep","devnull","realpath","supports_unicode_filenames","relpath", - "samefile", "sameopenfile", "samestat", "commonpath", "isjunction"] + "samefile", "sameopenfile", "samestat", "commonpath", "isjunction", + "isdevdrive"] def _get_bothseps(path): if isinstance(path, bytes): @@ -101,16 +101,14 @@ def join(path, *paths): if isinstance(path, bytes): sep = b'\\' seps = b'\\/' - colon = b':' + colon_seps = b':\\/' else: sep = '\\' seps = '\\/' - colon = ':' + colon_seps = ':\\/' try: - if not paths: - path[:0] + sep #23780: Ensure compatible data type even if p is null. result_drive, result_root, result_path = splitroot(path) - for p in map(os.fspath, paths): + for p in paths: p_drive, p_root, p_path = splitroot(p) if p_root: # Second path is absolute @@ -134,7 +132,7 @@ def join(path, *paths): result_path = result_path + p_path ## add separator between UNC and non-absolute path if (result_path and not result_root and - result_drive and result_drive[-1:] not in colon + seps): + result_drive and result_drive[-1] not in colon_seps): return result_drive + sep + result_path return result_drive + result_root + result_path except (TypeError, AttributeError, BytesWarning): @@ -168,56 +166,52 @@ def splitdrive(p): return drive, root + tail -def splitroot(p): - """Split a pathname into drive, root and tail. The drive is defined - exactly as in splitdrive(). On Windows, the root may be a single path - separator or an empty string. The tail contains anything after the root. - For example: +try: + from nt import _path_splitroot_ex as splitroot +except ImportError: + def splitroot(p): + """Split a pathname into drive, root and tail. - splitroot('//server/share/') == ('//server/share', '/', '') - splitroot('C:/Users/Barney') == ('C:', '/', 'Users/Barney') - splitroot('C:///spam///ham') == ('C:', '/', '//spam///ham') - splitroot('Windows/notepad') == ('', '', 'Windows/notepad') - """ - p = os.fspath(p) - if isinstance(p, bytes): - sep = b'\\' - altsep = b'/' - colon = b':' - unc_prefix = b'\\\\?\\UNC\\' - empty = b'' - else: - sep = '\\' - altsep = '/' - colon = ':' - unc_prefix = '\\\\?\\UNC\\' - empty = '' - normp = p.replace(altsep, sep) - if normp[:1] == sep: - if normp[1:2] == sep: - # UNC drives, e.g. \\server\share or \\?\UNC\server\share - # Device drives, e.g. \\.\device or \\?\device - start = 8 if normp[:8].upper() == unc_prefix else 2 - index = normp.find(sep, start) - if index == -1: - return p, empty, empty - index2 = normp.find(sep, index + 1) - if index2 == -1: - return p, empty, empty - return p[:index2], p[index2:index2 + 1], p[index2 + 1:] + The tail contains anything after the root.""" + p = os.fspath(p) + if isinstance(p, bytes): + sep = b'\\' + altsep = b'/' + colon = b':' + unc_prefix = b'\\\\?\\UNC\\' + empty = b'' else: - # Relative path with root, e.g. \Windows - return empty, p[:1], p[1:] - elif normp[1:2] == colon: - if normp[2:3] == sep: - # Absolute drive-letter path, e.g. X:\Windows - return p[:2], p[2:3], p[3:] + sep = '\\' + altsep = '/' + colon = ':' + unc_prefix = '\\\\?\\UNC\\' + empty = '' + normp = p.replace(altsep, sep) + if normp[:1] == sep: + if normp[1:2] == sep: + # UNC drives, e.g. \\server\share or \\?\UNC\server\share + # Device drives, e.g. \\.\device or \\?\device + start = 8 if normp[:8].upper() == unc_prefix else 2 + index = normp.find(sep, start) + if index == -1: + return p, empty, empty + index2 = normp.find(sep, index + 1) + if index2 == -1: + return p, empty, empty + return p[:index2], p[index2:index2 + 1], p[index2 + 1:] + else: + # Relative path with root, e.g. \Windows + return empty, p[:1], p[1:] + elif normp[1:2] == colon: + if normp[2:3] == sep: + # Absolute drive-letter path, e.g. X:\Windows + return p[:2], p[2:3], p[3:] + else: + # Relative path with drive, e.g. X:Windows + return p[:2], empty, p[2:] else: - # Relative path with drive, e.g. X:Windows - return p[:2], empty, p[2:] - else: - # Relative path, e.g. Windows - return empty, empty, p + # Relative path, e.g. Windows + return empty, empty, p # Split a path in head (everything up to the last '/') and tail (the @@ -269,33 +263,6 @@ def dirname(p): return split(p)[0] -# Is a path a junction? - -if hasattr(os.stat_result, 'st_reparse_tag'): - def isjunction(path): - """Test whether a path is a junction""" - try: - st = os.lstat(path) - except (OSError, ValueError, AttributeError): - return False - return bool(st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT) -else: - def isjunction(path): - """Test whether a path is a junction""" - os.fspath(path) - return False - - -# Being true for dangling symbolic links is also useful. - -def lexists(path): - """Test whether a path exists. Returns True for broken symbolic links""" - try: - st = os.lstat(path) - except (OSError, ValueError): - return False - return True - # Is a path a mount point? # Any drive letter root (eg c:\) # Any share UNC (eg \\server\share) @@ -351,8 +318,8 @@ def isreserved(path): def _isreservedname(name): """Return true if the filename is reserved by the system.""" # Trailing dots and spaces are reserved. - if name.endswith(('.', ' ')) and name not in ('.', '..'): - return True + if name[-1:] in ('.', ' '): + return name not in ('.', '..') # Wildcards, separators, colon, and pipe (*?"<>/\:|) are reserved. # ASCII control characters (0-31) are reserved. # Colon is reserved for file streams (e.g. "name:stream[:type]"). @@ -361,9 +328,7 @@ def _isreservedname(name): # DOS device names are reserved (e.g. "nul" or "nul .txt"). The rules # are complex and vary across Windows versions. On the side of # caution, return True for names that may not be reserved. - if name.partition('.')[0].rstrip(' ').upper() in _reserved_names: - return True - return False + return name.partition('.')[0].rstrip(' ').upper() in _reserved_names # Expand paths beginning with '~' or '~user'. @@ -381,24 +346,23 @@ def expanduser(path): If user or $HOME is unknown, do nothing.""" path = os.fspath(path) if isinstance(path, bytes): + seps = b'\\/' tilde = b'~' else: + seps = '\\/' tilde = '~' if not path.startswith(tilde): return path i, n = 1, len(path) - while i < n and path[i] not in _get_bothseps(path): + while i < n and path[i] not in seps: i += 1 if 'USERPROFILE' in os.environ: userhome = os.environ['USERPROFILE'] - elif not 'HOMEPATH' in os.environ: + elif 'HOMEPATH' not in os.environ: return path else: - try: - drive = os.environ['HOMEDRIVE'] - except KeyError: - drive = '' + drive = os.environ.get('HOMEDRIVE', '') userhome = join(drive, os.environ['HOMEPATH']) if i != 1: #~user @@ -549,7 +513,7 @@ def expandvars(path): # Previously, this function also truncated pathnames to 8+3 format, # but as this module is called "ntpath", that's obviously wrong! try: - from nt import _path_normpath + from nt import _path_normpath as normpath except ImportError: def normpath(path): @@ -588,37 +552,22 @@ def normpath(path): comps.append(curdir) return prefix + sep.join(comps) -else: - def normpath(path): - """Normalize path, eliminating double slashes, etc.""" - path = os.fspath(path) - if isinstance(path, bytes): - return os.fsencode(_path_normpath(os.fsdecode(path))) or b"." - return _path_normpath(path) or "." - - -def _abspath_fallback(path): - """Return the absolute version of a path as a fallback function in case - `nt._getfullpathname` is not available or raises OSError. See bpo-31047 for - more. - - """ - - path = os.fspath(path) - if not isabs(path): - if isinstance(path, bytes): - cwd = os.getcwdb() - else: - cwd = os.getcwd() - path = join(cwd, path) - return normpath(path) # Return an absolute path. try: from nt import _getfullpathname except ImportError: # not running on Windows - mock up something sensible - abspath = _abspath_fallback + def abspath(path): + """Return the absolute version of a path.""" + path = os.fspath(path) + if not isabs(path): + if isinstance(path, bytes): + cwd = os.getcwdb() + else: + cwd = os.getcwd() + path = join(cwd, path) + return normpath(path) else: # use native Windows method on Windows def abspath(path): @@ -626,7 +575,27 @@ def abspath(path): try: return _getfullpathname(normpath(path)) except (OSError, ValueError): - return _abspath_fallback(path) + # See gh-75230, handle outside for cleaner traceback + pass + path = os.fspath(path) + if not isabs(path): + if isinstance(path, bytes): + sep = b'\\' + getcwd = os.getcwdb + else: + sep = '\\' + getcwd = os.getcwd + drive, root, path = splitroot(path) + # Either drive or root can be nonempty, but not both. + if drive or root: + try: + path = join(_getfullpathname(drive + root), path) + except (OSError, ValueError): + # Drive "\0:" cannot exist; use the root directory. + path = drive + sep + path + else: + path = join(getcwd(), path) + return normpath(path) try: from nt import _findfirstfile, _getfinalpathname, readlink as _nt_readlink @@ -738,7 +707,8 @@ def realpath(path, *, strict=False): new_unc_prefix = b'\\\\' cwd = os.getcwdb() # bpo-38081: Special case for realpath(b'nul') - if normcase(path) == normcase(os.fsencode(devnull)): + devnull = b'nul' + if normcase(path) == devnull: return b'\\\\.\\NUL' else: prefix = '\\\\?\\' @@ -746,7 +716,8 @@ def realpath(path, *, strict=False): new_unc_prefix = '\\\\' cwd = os.getcwd() # bpo-38081: Special case for realpath('nul') - if normcase(path) == normcase(devnull): + devnull = 'nul' + if normcase(path) == devnull: return '\\\\.\\NUL' had_prefix = path.startswith(prefix) if not had_prefix and not isabs(path): @@ -799,6 +770,9 @@ def realpath(path, *, strict=False): def relpath(path, start=None): """Return a relative version of a path""" path = os.fspath(path) + if not path: + raise ValueError("no path specified") + if isinstance(path, bytes): sep = b'\\' curdir = b'.' @@ -810,22 +784,20 @@ def relpath(path, start=None): if start is None: start = curdir + else: + start = os.fspath(start) - if not path: - raise ValueError("no path specified") - - start = os.fspath(start) try: - start_abs = abspath(normpath(start)) - path_abs = abspath(normpath(path)) + start_abs = abspath(start) + path_abs = abspath(path) start_drive, _, start_rest = splitroot(start_abs) path_drive, _, path_rest = splitroot(path_abs) if normcase(start_drive) != normcase(path_drive): raise ValueError("path is on mount %r, start on mount %r" % ( path_drive, start_drive)) - start_list = [x for x in start_rest.split(sep) if x] - path_list = [x for x in path_rest.split(sep) if x] + start_list = start_rest.split(sep) if start_rest else [] + path_list = path_rest.split(sep) if path_rest else [] # Work out how much of the filepath is shared by start and path. i = 0 for e1, e2 in zip(start_list, path_list): @@ -836,29 +808,28 @@ def relpath(path, start=None): rel_list = [pardir] * (len(start_list)-i) + path_list[i:] if not rel_list: return curdir - return join(*rel_list) + return sep.join(rel_list) except (TypeError, ValueError, AttributeError, BytesWarning, DeprecationWarning): genericpath._check_arg_types('relpath', path, start) raise -# Return the longest common sub-path of the sequence of paths given as input. +# Return the longest common sub-path of the iterable of paths given as input. # The function is case-insensitive and 'separator-insensitive', i.e. if the # only difference between two paths is the use of '\' versus '/' as separator, # they are deemed to be equal. # # However, the returned path will have the standard '\' separator (even if the # given paths had the alternative '/' separator) and will have the case of the -# first path given in the sequence. Additionally, any trailing separator is +# first path given in the iterable. Additionally, any trailing separator is # stripped from the returned path. def commonpath(paths): - """Given a sequence of path names, returns the longest common sub-path.""" - + """Given an iterable of path names, returns the longest common sub-path.""" + paths = tuple(map(os.fspath, paths)) if not paths: - raise ValueError('commonpath() arg is an empty sequence') + raise ValueError('commonpath() arg is an empty iterable') - paths = tuple(map(os.fspath, paths)) if isinstance(paths[0], bytes): sep = b'\\' altsep = b'/' @@ -872,9 +843,6 @@ def commonpath(paths): drivesplits = [splitroot(p.replace(altsep, sep).lower()) for p in paths] split_paths = [p.split(sep) for d, r, p in drivesplits] - if len({r for d, r, p in drivesplits}) != 1: - raise ValueError("Can't mix absolute and relative paths") - # Check that all drive letters or UNC paths match. The check is made only # now otherwise type errors for mixing strings and bytes would not be # caught. @@ -882,6 +850,12 @@ def commonpath(paths): raise ValueError("Paths don't have the same drive") drive, root, path = splitroot(paths[0].replace(altsep, sep)) + if len({r for d, r, p in drivesplits}) != 1: + if drive: + raise ValueError("Can't mix absolute and relative paths") + else: + raise ValueError("Can't mix rooted and not-rooted paths") + common = path.split(sep) common = [c for c in common if c and c != curdir] @@ -902,13 +876,15 @@ def commonpath(paths): try: - # The isdir(), isfile(), islink() and exists() implementations in - # genericpath use os.stat(). This is overkill on Windows. Use simpler + # The isdir(), isfile(), islink(), exists() and lexists() implementations + # in genericpath use os.stat(). This is overkill on Windows. Use simpler # builtin functions if they are available. from nt import _path_isdir as isdir from nt import _path_isfile as isfile from nt import _path_islink as islink + from nt import _path_isjunction as isjunction from nt import _path_exists as exists + from nt import _path_lexists as lexists except ImportError: # Use genericpath.* as imported above pass @@ -916,15 +892,12 @@ def commonpath(paths): try: from nt import _path_isdevdrive -except ImportError: - def isdevdrive(path): - """Determines whether the specified path is on a Windows Dev Drive.""" - # Never a Dev Drive - return False -else: def isdevdrive(path): """Determines whether the specified path is on a Windows Dev Drive.""" try: return _path_isdevdrive(abspath(path)) except OSError: return False +except ImportError: + # Use genericpath.isdevdrive as imported above + pass diff --git a/Lib/nturl2path.py b/Lib/nturl2path.py index 61852aff58912d..7e13ae3128333d 100644 --- a/Lib/nturl2path.py +++ b/Lib/nturl2path.py @@ -15,32 +15,29 @@ def url2pathname(url): # become # C:\foo\bar\spam.foo import string, urllib.parse + if url[:3] == '///': + # URL has an empty authority section, so the path begins on the third + # character. + url = url[2:] + elif url[:12] == '//localhost/': + # Skip past 'localhost' authority. + url = url[11:] + if url[:3] == '///': + # Skip past extra slash before UNC drive in URL path. + url = url[1:] # Windows itself uses ":" even in URLs. url = url.replace(':', '|') if not '|' in url: # No drive specifier, just convert slashes - if url[:4] == '////': - # path is something like ////host/path/on/remote/host - # convert this to \\host\path\on\remote\host - # (notice halving of slashes at the start of the path) - url = url[2:] - components = url.split('/') # make sure not to convert quoted slashes :-) - return urllib.parse.unquote('\\'.join(components)) + return urllib.parse.unquote(url.replace('/', '\\')) comp = url.split('|') if len(comp) != 2 or comp[0][-1] not in string.ascii_letters: error = 'Bad URL: ' + url raise OSError(error) - drive = comp[0][-1].upper() - components = comp[1].split('/') - path = drive + ':' - for comp in components: - if comp: - path = path + '\\' + urllib.parse.unquote(comp) - # Issue #11474 - handing url such as |c/| - if path.endswith(':') and url.endswith('/'): - path += '\\' - return path + drive = comp[0][-1] + tail = urllib.parse.unquote(comp[1].replace('/', '\\')) + return drive + ':' + tail def pathname2url(p): """OS-specific conversion from a file system path to a relative URL @@ -49,33 +46,26 @@ def pathname2url(p): # C:\foo\bar\spam.foo # becomes # ///C:/foo/bar/spam.foo + import ntpath import urllib.parse # First, clean up some special forms. We are going to sacrifice # the additional information anyway - if p[:4] == '\\\\?\\': + p = p.replace('\\', '/') + if p[:4] == '//?/': p = p[4:] - if p[:4].upper() == 'UNC\\': - p = '\\' + p[4:] - elif p[1:2] != ':': - raise OSError('Bad path: ' + p) - if not ':' in p: - # No drive specifier, just convert slashes and quote the name - if p[:2] == '\\\\': - # path is something like \\host\path\on\remote\host - # convert this to ////host/path/on/remote/host - # (notice doubling of slashes at the start of the path) - p = '\\\\' + p - components = p.split('\\') - return urllib.parse.quote('/'.join(components)) - comp = p.split(':', maxsplit=2) - if len(comp) != 2 or len(comp[0]) > 1: - error = 'Bad path: ' + p - raise OSError(error) + if p[:4].upper() == 'UNC/': + p = '//' + p[4:] + drive, root, tail = ntpath.splitroot(p) + if drive: + if drive[1:] == ':': + # DOS drive specified. Add three slashes to the start, producing + # an authority section with a zero-length authority, and a path + # section starting with a single slash. + drive = f'///{drive}' + drive = urllib.parse.quote(drive, safe='/:') + elif root: + # Add explicitly empty authority to path beginning with one slash. + root = f'//{root}' - drive = urllib.parse.quote(comp[0].upper()) - components = comp[1].split('\\') - path = '///' + drive + ':' - for comp in components: - if comp: - path = path + '/' + urllib.parse.quote(comp) - return path + tail = urllib.parse.quote(tail) + return drive + root + tail diff --git a/Lib/opcode.py b/Lib/opcode.py index 88f4df7c0e8c38..4ee0d64026bd0a 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -12,13 +12,14 @@ import _opcode from _opcode import stack_effect -from _opcode_metadata import (_specializations, _specialized_opmap, opmap, - HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE) +from _opcode_metadata import (_specializations, _specialized_opmap, opmap, # noqa: F401 + HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE) # noqa: F401 EXTENDED_ARG = opmap['EXTENDED_ARG'] opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] -for op, i in opmap.items(): - opname[i] = op +for m in (opmap, _specialized_opmap): + for op, i in m.items(): + opname[i] = op cmp_op = ('<', '<=', '==', '!=', '>', '>=') @@ -36,6 +37,8 @@ _intrinsic_1_descs = _opcode.get_intrinsic1_descs() _intrinsic_2_descs = _opcode.get_intrinsic2_descs() +_special_method_names = _opcode.get_special_method_names() +_common_constants = [AssertionError, NotImplementedError] _nb_ops = _opcode.get_nb_ops() hascompare = [opmap["COMPARE_OP"]] @@ -49,6 +52,7 @@ }, "BINARY_OP": { "counter": 1, + "descr": 4, }, "UNPACK_SEQUENCE": { "counter": 1, @@ -56,6 +60,9 @@ "COMPARE_OP": { "counter": 1, }, + "CONTAINS_OP": { + "counter": 1, + }, "BINARY_SUBSCR": { "counter": 1, }, @@ -80,6 +87,10 @@ "counter": 1, "func_version": 2, }, + "CALL_KW": { + "counter": 1, + "func_version": 2, + }, "STORE_SUBSCR": { "counter": 1, }, diff --git a/Lib/operator.py b/Lib/operator.py index 30116c1189a499..1b765522f85949 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -14,8 +14,8 @@ 'delitem', 'eq', 'floordiv', 'ge', 'getitem', 'gt', 'iadd', 'iand', 'iconcat', 'ifloordiv', 'ilshift', 'imatmul', 'imod', 'imul', 'index', 'indexOf', 'inv', 'invert', 'ior', 'ipow', 'irshift', - 'is_', 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', - 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', + 'is_', 'is_none', 'is_not', 'is_not_none', 'isub', 'itemgetter', 'itruediv', + 'ixor', 'le', 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', 'mul', 'ne', 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift', 'setitem', 'sub', 'truediv', 'truth', 'xor'] @@ -66,6 +66,14 @@ def is_not(a, b): "Same as a is not b." return a is not b +def is_none(a): + "Same as a is None." + return a is None + +def is_not_none(a): + "Same as a is not None." + return a is not None + # Mathematical/Bitwise Operations *********************************************# def abs(a): @@ -239,7 +247,7 @@ class attrgetter: """ __slots__ = ('_attrs', '_call') - def __init__(self, attr, *attrs): + def __init__(self, attr, /, *attrs): if not attrs: if not isinstance(attr, str): raise TypeError('attribute name must be a string') @@ -257,7 +265,7 @@ def func(obj): return tuple(getter(obj) for getter in getters) self._call = func - def __call__(self, obj): + def __call__(self, obj, /): return self._call(obj) def __repr__(self): @@ -276,7 +284,7 @@ class itemgetter: """ __slots__ = ('_items', '_call') - def __init__(self, item, *items): + def __init__(self, item, /, *items): if not items: self._items = (item,) def func(obj): @@ -288,7 +296,7 @@ def func(obj): return tuple(obj[i] for i in items) self._call = func - def __call__(self, obj): + def __call__(self, obj, /): return self._call(obj) def __repr__(self): @@ -315,7 +323,7 @@ def __init__(self, name, /, *args, **kwargs): self._args = args self._kwargs = kwargs - def __call__(self, obj): + def __call__(self, obj, /): return getattr(obj, self._name)(*self._args, **self._kwargs) def __repr__(self): @@ -415,7 +423,7 @@ def ixor(a, b): except ImportError: pass else: - from _operator import __doc__ + from _operator import __doc__ # noqa: F401 # All of these "__func__ = func" assignments have to happen after importing # from _operator to make sure they're set to the right function diff --git a/Lib/optparse.py b/Lib/optparse.py index 1c450c6fcbe3b6..38cf16d21efffa 100644 --- a/Lib/optparse.py +++ b/Lib/optparse.py @@ -43,7 +43,7 @@ __copyright__ = """ Copyright (c) 2001-2006 Gregory P. Ward. All rights reserved. -Copyright (c) 2002-2006 Python Software Foundation. All rights reserved. +Copyright (c) 2002 Python Software Foundation. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -74,7 +74,8 @@ """ import sys, os -import textwrap +from gettext import gettext as _, ngettext + def _repr(self): return "<%s at 0x%x: %s>" % (self.__class__.__name__, id(self), self) @@ -86,19 +87,6 @@ def _repr(self): # Id: help.py 527 2006-07-23 15:21:30Z greg # Id: errors.py 509 2006-04-20 00:58:24Z gward -try: - from gettext import gettext, ngettext -except ImportError: - def gettext(message): - return message - - def ngettext(singular, plural, n): - if n == 1: - return singular - return plural - -_ = gettext - class OptParseError (Exception): def __init__(self, msg): @@ -263,6 +251,7 @@ def _format_text(self, text): Format a paragraph of free-form text for inclusion in the help output at the current indentation level. """ + import textwrap text_width = max(self.width - self.current_indent, 11) indent = " "*self.current_indent return textwrap.fill(text, @@ -319,6 +308,7 @@ def format_option(self, option): indent_first = 0 result.append(opts) if option.help: + import textwrap help_text = self.expand_default(option) help_lines = textwrap.wrap(help_text, self.help_width) result.append("%*s%s\n" % (indent_first, "", help_lines[0])) diff --git a/Lib/os.py b/Lib/os.py index 7f38e14e7bdd96..9c2258e6ccf5ba 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -64,6 +64,10 @@ def _get_exports_list(module): from posix import _have_functions except ImportError: pass + try: + from posix import _create_environ + except ImportError: + pass import posix __all__.extend(_get_exports_list(posix)) @@ -88,6 +92,10 @@ def _get_exports_list(module): from nt import _have_functions except ImportError: pass + try: + from nt import _create_environ + except ImportError: + pass else: raise ImportError('no os specific module found') @@ -281,6 +289,10 @@ def renames(old, new): __all__.extend(["makedirs", "removedirs", "renames"]) +# Private sentinel that makes walk() classify all symlinks and junctions as +# regular files. +_walk_symlinks_as_files = object() + def walk(top, topdown=True, onerror=None, followlinks=False): """Directory tree generator. @@ -361,58 +373,45 @@ def walk(top, topdown=True, onerror=None, followlinks=False): # minor reason when (say) a thousand readable directories are still # left to visit. try: - scandir_it = scandir(top) + with scandir(top) as entries: + for entry in entries: + try: + if followlinks is _walk_symlinks_as_files: + is_dir = entry.is_dir(follow_symlinks=False) and not entry.is_junction() + else: + is_dir = entry.is_dir() + except OSError: + # If is_dir() raises an OSError, consider the entry not to + # be a directory, same behaviour as os.path.isdir(). + is_dir = False + + if is_dir: + dirs.append(entry.name) + else: + nondirs.append(entry.name) + + if not topdown and is_dir: + # Bottom-up: traverse into sub-directory, but exclude + # symlinks to directories if followlinks is False + if followlinks: + walk_into = True + else: + try: + is_symlink = entry.is_symlink() + except OSError: + # If is_symlink() raises an OSError, consider the + # entry not to be a symbolic link, same behaviour + # as os.path.islink(). + is_symlink = False + walk_into = not is_symlink + + if walk_into: + walk_dirs.append(entry.path) except OSError as error: if onerror is not None: onerror(error) continue - cont = False - with scandir_it: - while True: - try: - try: - entry = next(scandir_it) - except StopIteration: - break - except OSError as error: - if onerror is not None: - onerror(error) - cont = True - break - - try: - is_dir = entry.is_dir() - except OSError: - # If is_dir() raises an OSError, consider the entry not to - # be a directory, same behaviour as os.path.isdir(). - is_dir = False - - if is_dir: - dirs.append(entry.name) - else: - nondirs.append(entry.name) - - if not topdown and is_dir: - # Bottom-up: traverse into sub-directory, but exclude - # symlinks to directories if followlinks is False - if followlinks: - walk_into = True - else: - try: - is_symlink = entry.is_symlink() - except OSError: - # If is_symlink() raises an OSError, consider the - # entry not to be a symbolic link, same behaviour - # as os.path.islink(). - is_symlink = False - walk_into = not is_symlink - - if walk_into: - walk_dirs.append(entry.path) - if cont: - continue - if topdown: # Yield before sub-directory traversal if going top down yield top, dirs, nondirs @@ -471,24 +470,59 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= """ sys.audit("os.fwalk", top, topdown, onerror, follow_symlinks, dir_fd) top = fspath(top) - # Note: To guard against symlink races, we use the standard - # lstat()/open()/fstat() trick. - if not follow_symlinks: - orig_st = stat(top, follow_symlinks=False, dir_fd=dir_fd) - topfd = open(top, O_RDONLY, dir_fd=dir_fd) + stack = [(_fwalk_walk, (True, dir_fd, top, top, None))] + isbytes = isinstance(top, bytes) try: - if (follow_symlinks or (st.S_ISDIR(orig_st.st_mode) and - path.samestat(orig_st, stat(topfd)))): - yield from _fwalk(topfd, top, isinstance(top, bytes), - topdown, onerror, follow_symlinks) + while stack: + yield from _fwalk(stack, isbytes, topdown, onerror, follow_symlinks) finally: - close(topfd) - - def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): + # Close any file descriptors still on the stack. + while stack: + action, value = stack.pop() + if action == _fwalk_close: + close(value) + + # Each item in the _fwalk() stack is a pair (action, args). + _fwalk_walk = 0 # args: (isroot, dirfd, toppath, topname, entry) + _fwalk_yield = 1 # args: (toppath, dirnames, filenames, topfd) + _fwalk_close = 2 # args: dirfd + + def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): # Note: This uses O(depth of the directory tree) file descriptors: if # necessary, it can be adapted to only require O(1) FDs, see issue # #13734. + action, value = stack.pop() + if action == _fwalk_close: + close(value) + return + elif action == _fwalk_yield: + yield value + return + assert action == _fwalk_walk + isroot, dirfd, toppath, topname, entry = value + try: + if not follow_symlinks: + # Note: To guard against symlink races, we use the standard + # lstat()/open()/fstat() trick. + if entry is None: + orig_st = stat(topname, follow_symlinks=False, dir_fd=dirfd) + else: + orig_st = entry.stat(follow_symlinks=False) + topfd = open(topname, O_RDONLY | O_NONBLOCK, dir_fd=dirfd) + except OSError as err: + if isroot: + raise + if onerror is not None: + onerror(err) + return + stack.append((_fwalk_close, topfd)) + if not follow_symlinks: + if isroot and not st.S_ISDIR(orig_st.st_mode): + return + if not path.samestat(orig_st, stat(topfd)): + return + scandir_it = scandir(topfd) dirs = [] nondirs = [] @@ -514,31 +548,18 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): if topdown: yield toppath, dirs, nondirs, topfd + else: + stack.append((_fwalk_yield, (toppath, dirs, nondirs, topfd))) - for name in dirs if entries is None else zip(dirs, entries): - try: - if not follow_symlinks: - if topdown: - orig_st = stat(name, dir_fd=topfd, follow_symlinks=False) - else: - assert entries is not None - name, entry = name - orig_st = entry.stat(follow_symlinks=False) - dirfd = open(name, O_RDONLY, dir_fd=topfd) - except OSError as err: - if onerror is not None: - onerror(err) - continue - try: - if follow_symlinks or path.samestat(orig_st, stat(dirfd)): - dirpath = path.join(toppath, name) - yield from _fwalk(dirfd, dirpath, isbytes, - topdown, onerror, follow_symlinks) - finally: - close(dirfd) - - if not topdown: - yield toppath, dirs, nondirs, topfd + toppath = path.join(toppath, toppath[:0]) # Add trailing slash. + if entries is None: + stack.extend( + (_fwalk_walk, (False, topfd, toppath + name, name, None)) + for name in dirs[::-1]) + else: + stack.extend( + (_fwalk_walk, (False, topfd, toppath + name, name, entry)) + for name, entry in zip(dirs[::-1], entries[::-1])) __all__.append("fwalk") @@ -744,7 +765,7 @@ def __ror__(self, other): new.update(self) return new -def _createenviron(): +def _create_environ_mapping(): if name == 'nt': # Where Env Var Names Must Be UPPERCASE def check_str(value): @@ -774,8 +795,22 @@ def decode(value): encode, decode) # unicode environ -environ = _createenviron() -del _createenviron +environ = _create_environ_mapping() +del _create_environ_mapping + + +if _exists("_create_environ"): + def reload_environ(): + data = _create_environ() + if name == 'nt': + encodekey = environ.encodekey + data = {encodekey(key): value + for key, value in data.items()} + + # modify in-place to keep os.environb in sync + env_data = environ._data + env_data.clear() + env_data.update(data) def getenv(key, default=None): diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index 46834b1a76a6eb..ec1bac9ef49350 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -5,875 +5,6 @@ operating systems. """ -import io -import ntpath -import os -import posixpath -import sys -import warnings -from itertools import chain -from _collections_abc import Sequence +from pathlib._local import * -try: - import pwd -except ImportError: - pwd = None -try: - import grp -except ImportError: - grp = None - -from . import _abc - - -__all__ = [ - "UnsupportedOperation", - "PurePath", "PurePosixPath", "PureWindowsPath", - "Path", "PosixPath", "WindowsPath", - ] - - -class _PathParents(Sequence): - """This object provides sequence-like access to the logical ancestors - of a path. Don't try to construct it yourself.""" - __slots__ = ('_path', '_drv', '_root', '_tail') - - def __init__(self, path): - self._path = path - self._drv = path.drive - self._root = path.root - self._tail = path._tail - - def __len__(self): - return len(self._tail) - - def __getitem__(self, idx): - if isinstance(idx, slice): - return tuple(self[i] for i in range(*idx.indices(len(self)))) - - if idx >= len(self) or idx < -len(self): - raise IndexError(idx) - if idx < 0: - idx += len(self) - return self._path._from_parsed_parts(self._drv, self._root, - self._tail[:-idx - 1]) - - def __repr__(self): - return "<{}.parents>".format(type(self._path).__name__) - - -UnsupportedOperation = _abc.UnsupportedOperation - - -class PurePath(_abc.PurePathBase): - """Base class for manipulating paths without I/O. - - PurePath represents a filesystem path and offers operations which - don't imply any actual filesystem I/O. Depending on your system, - instantiating a PurePath will return either a PurePosixPath or a - PureWindowsPath object. You can also instantiate either of these classes - directly, regardless of your system. - """ - - __slots__ = ( - # The `_raw_paths` slot stores unnormalized string paths. This is set - # in the `__init__()` method. - '_raw_paths', - - # The `_drv`, `_root` and `_tail_cached` slots store parsed and - # normalized parts of the path. They are set when any of the `drive`, - # `root` or `_tail` properties are accessed for the first time. The - # three-part division corresponds to the result of - # `os.path.splitroot()`, except that the tail is further split on path - # separators (i.e. it is a list of strings), and that the root and - # tail are normalized. - '_drv', '_root', '_tail_cached', - - # The `_str` slot stores the string representation of the path, - # computed from the drive, root and tail when `__str__()` is called - # for the first time. It's used to implement `_str_normcase` - '_str', - - # The `_str_normcase_cached` slot stores the string path with - # normalized case. It is set when the `_str_normcase` property is - # accessed for the first time. It's used to implement `__eq__()` - # `__hash__()`, and `_parts_normcase` - '_str_normcase_cached', - - # The `_parts_normcase_cached` slot stores the case-normalized - # string path after splitting on path separators. It's set when the - # `_parts_normcase` property is accessed for the first time. It's used - # to implement comparison methods like `__lt__()`. - '_parts_normcase_cached', - - # The `_hash` slot stores the hash of the case-normalized string - # path. It's set when `__hash__()` is called for the first time. - '_hash', - ) - pathmod = os.path - - def __new__(cls, *args, **kwargs): - """Construct a PurePath from one or several strings and or existing - PurePath objects. The strings and path objects are combined so as - to yield a canonicalized path, which is incorporated into the - new PurePath object. - """ - if cls is PurePath: - cls = PureWindowsPath if os.name == 'nt' else PurePosixPath - return object.__new__(cls) - - def __init__(self, *args): - paths = [] - for arg in args: - if isinstance(arg, PurePath): - if arg.pathmod is ntpath and self.pathmod is posixpath: - # GH-103631: Convert separators for backwards compatibility. - paths.extend(path.replace('\\', '/') for path in arg._raw_paths) - else: - paths.extend(arg._raw_paths) - else: - try: - path = os.fspath(arg) - except TypeError: - path = arg - if not isinstance(path, str): - raise TypeError( - "argument should be a str or an os.PathLike " - "object where __fspath__ returns a str, " - f"not {type(path).__name__!r}") - paths.append(path) - # Avoid calling super().__init__, as an optimisation - self._raw_paths = paths - - def joinpath(self, *pathsegments): - """Combine this path with one or several arguments, and return a - new path representing either a subpath (if all arguments are relative - paths) or a totally different path (if one of the arguments is - anchored). - """ - return self.with_segments(self, *pathsegments) - - def __truediv__(self, key): - try: - return self.with_segments(self, key) - except TypeError: - return NotImplemented - - def __rtruediv__(self, key): - try: - return self.with_segments(key, self) - except TypeError: - return NotImplemented - - def __reduce__(self): - # Using the parts tuple helps share interned path parts - # when pickling related paths. - return (self.__class__, self.parts) - - def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, self.as_posix()) - - def __fspath__(self): - return str(self) - - def __bytes__(self): - """Return the bytes representation of the path. This is only - recommended to use under Unix.""" - return os.fsencode(self) - - @property - def _str_normcase(self): - # String with normalized case, for hashing and equality checks - try: - return self._str_normcase_cached - except AttributeError: - if _abc._is_case_sensitive(self.pathmod): - self._str_normcase_cached = str(self) - else: - self._str_normcase_cached = str(self).lower() - return self._str_normcase_cached - - def __hash__(self): - try: - return self._hash - except AttributeError: - self._hash = hash(self._str_normcase) - return self._hash - - def __eq__(self, other): - if not isinstance(other, PurePath): - return NotImplemented - return self._str_normcase == other._str_normcase and self.pathmod is other.pathmod - - @property - def _parts_normcase(self): - # Cached parts with normalized case, for comparisons. - try: - return self._parts_normcase_cached - except AttributeError: - self._parts_normcase_cached = self._str_normcase.split(self.pathmod.sep) - return self._parts_normcase_cached - - def __lt__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: - return NotImplemented - return self._parts_normcase < other._parts_normcase - - def __le__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: - return NotImplemented - return self._parts_normcase <= other._parts_normcase - - def __gt__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: - return NotImplemented - return self._parts_normcase > other._parts_normcase - - def __ge__(self, other): - if not isinstance(other, PurePath) or self.pathmod is not other.pathmod: - return NotImplemented - return self._parts_normcase >= other._parts_normcase - - def __str__(self): - """Return the string representation of the path, suitable for - passing to system calls.""" - try: - return self._str - except AttributeError: - self._str = self._format_parsed_parts(self.drive, self.root, - self._tail) or '.' - return self._str - - @classmethod - def _format_parsed_parts(cls, drv, root, tail): - if drv or root: - return drv + root + cls.pathmod.sep.join(tail) - elif tail and cls.pathmod.splitdrive(tail[0])[0]: - tail = ['.'] + tail - return cls.pathmod.sep.join(tail) - - def _from_parsed_parts(self, drv, root, tail): - path_str = self._format_parsed_parts(drv, root, tail) - path = self.with_segments(path_str) - path._str = path_str or '.' - path._drv = drv - path._root = root - path._tail_cached = tail - return path - - @classmethod - def _parse_path(cls, path): - if not path: - return '', '', [] - sep = cls.pathmod.sep - altsep = cls.pathmod.altsep - if altsep: - path = path.replace(altsep, sep) - drv, root, rel = cls.pathmod.splitroot(path) - if not root and drv.startswith(sep) and not drv.endswith(sep): - drv_parts = drv.split(sep) - if len(drv_parts) == 4 and drv_parts[2] not in '?.': - # e.g. //server/share - root = sep - elif len(drv_parts) == 6: - # e.g. //?/unc/server/share - root = sep - parsed = [sys.intern(str(x)) for x in rel.split(sep) if x and x != '.'] - return drv, root, parsed - - @property - def _raw_path(self): - """The joined but unnormalized path.""" - paths = self._raw_paths - if len(paths) == 0: - path = '' - elif len(paths) == 1: - path = paths[0] - else: - path = self.pathmod.join(*paths) - return path - - @property - def drive(self): - """The drive prefix (letter or UNC path), if any.""" - try: - return self._drv - except AttributeError: - self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) - return self._drv - - @property - def root(self): - """The root of the path, if any.""" - try: - return self._root - except AttributeError: - self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) - return self._root - - @property - def _tail(self): - try: - return self._tail_cached - except AttributeError: - self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) - return self._tail_cached - - @property - def anchor(self): - """The concatenation of the drive and root, or ''.""" - return self.drive + self.root - - @property - def parts(self): - """An object providing sequence-like access to the - components in the filesystem path.""" - if self.drive or self.root: - return (self.drive + self.root,) + tuple(self._tail) - else: - return tuple(self._tail) - - @property - def parent(self): - """The logical parent of the path.""" - drv = self.drive - root = self.root - tail = self._tail - if not tail: - return self - return self._from_parsed_parts(drv, root, tail[:-1]) - - @property - def parents(self): - """A sequence of this path's logical parents.""" - # The value of this property should not be cached on the path object, - # as doing so would introduce a reference cycle. - return _PathParents(self) - - @property - def name(self): - """The final path component, if any.""" - tail = self._tail - if not tail: - return '' - return tail[-1] - - def with_name(self, name): - """Return a new path with the file name changed.""" - m = self.pathmod - if not name or m.sep in name or (m.altsep and m.altsep in name) or name == '.': - raise ValueError(f"Invalid name {name!r}") - tail = self._tail.copy() - if not tail: - raise ValueError(f"{self!r} has an empty name") - tail[-1] = name - return self._from_parsed_parts(self.drive, self.root, tail) - - def relative_to(self, other, /, *_deprecated, walk_up=False): - """Return the relative path to another path identified by the passed - arguments. If the operation is not possible (because this is not - related to the other path), raise ValueError. - - The *walk_up* parameter controls whether `..` may be used to resolve - the path. - """ - if _deprecated: - msg = ("support for supplying more than one positional argument " - "to pathlib.PurePath.relative_to() is deprecated and " - "scheduled for removal in Python 3.14") - warnings.warn(msg, DeprecationWarning, stacklevel=2) - other = self.with_segments(other, *_deprecated) - elif not isinstance(other, PurePath): - other = self.with_segments(other) - for step, path in enumerate(chain([other], other.parents)): - if path == self or path in self.parents: - break - elif not walk_up: - raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") - elif path.name == '..': - raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") - else: - raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") - parts = ['..'] * step + self._tail[len(path._tail):] - return self._from_parsed_parts('', '', parts) - - def is_relative_to(self, other, /, *_deprecated): - """Return True if the path is relative to another path or False. - """ - if _deprecated: - msg = ("support for supplying more than one argument to " - "pathlib.PurePath.is_relative_to() is deprecated and " - "scheduled for removal in Python 3.14") - warnings.warn(msg, DeprecationWarning, stacklevel=2) - other = self.with_segments(other, *_deprecated) - elif not isinstance(other, PurePath): - other = self.with_segments(other) - return other == self or other in self.parents - - def is_absolute(self): - """True if the path is absolute (has both a root and, if applicable, - a drive).""" - if self.pathmod is posixpath: - # Optimization: work with raw paths on POSIX. - for path in self._raw_paths: - if path.startswith('/'): - return True - return False - return self.pathmod.isabs(self) - - def is_reserved(self): - """Return True if the path contains one of the special names reserved - by the system, if any.""" - msg = ("pathlib.PurePath.is_reserved() is deprecated and scheduled " - "for removal in Python 3.15. Use os.path.isreserved() to " - "detect reserved paths on Windows.") - warnings.warn(msg, DeprecationWarning, stacklevel=2) - if self.pathmod is ntpath: - return self.pathmod.isreserved(self) - return False - - def as_uri(self): - """Return the path as a URI.""" - if not self.is_absolute(): - raise ValueError("relative path can't be expressed as a file URI") - - drive = self.drive - if len(drive) == 2 and drive[1] == ':': - # It's a path on a local drive => 'file:///c:/a/b' - prefix = 'file:///' + drive - path = self.as_posix()[2:] - elif drive: - # It's a path on a network drive => 'file://host/share/a/b' - prefix = 'file:' - path = self.as_posix() - else: - # It's a posix path => 'file:///etc/hosts' - prefix = 'file://' - path = str(self) - from urllib.parse import quote_from_bytes - return prefix + quote_from_bytes(os.fsencode(path)) - - @property - def _pattern_stack(self): - """Stack of path components, to be used with patterns in glob().""" - parts = self._tail.copy() - pattern = self._raw_path - if self.anchor: - raise NotImplementedError("Non-relative patterns are unsupported") - elif not parts: - raise ValueError("Unacceptable pattern: {!r}".format(pattern)) - elif pattern[-1] in (self.pathmod.sep, self.pathmod.altsep): - # GH-65238: pathlib doesn't preserve trailing slash. Add it back. - parts.append('') - parts.reverse() - return parts - - @property - def _pattern_str(self): - """The path expressed as a string, for use in pattern-matching.""" - # The string representation of an empty path is a single dot ('.'). Empty - # paths shouldn't match wildcards, so we change it to the empty string. - path_str = str(self) - return '' if path_str == '.' else path_str - -# Subclassing os.PathLike makes isinstance() checks slower, -# which in turn makes Path construction slower. Register instead! -os.PathLike.register(PurePath) - - -class PurePosixPath(PurePath): - """PurePath subclass for non-Windows systems. - - On a POSIX system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - pathmod = posixpath - __slots__ = () - - -class PureWindowsPath(PurePath): - """PurePath subclass for Windows systems. - - On a Windows system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - pathmod = ntpath - __slots__ = () - - -class Path(_abc.PathBase, PurePath): - """PurePath subclass that can make system calls. - - Path represents a filesystem path but unlike PurePath, also offers - methods to do system calls on path objects. Depending on your system, - instantiating a Path will return either a PosixPath or a WindowsPath - object. You can also instantiate a PosixPath or WindowsPath directly, - but cannot instantiate a WindowsPath on a POSIX system or vice versa. - """ - __slots__ = () - as_uri = PurePath.as_uri - - @classmethod - def _unsupported_msg(cls, attribute): - return f"{cls.__name__}.{attribute} is unsupported on this system" - - def __init__(self, *args, **kwargs): - if kwargs: - msg = ("support for supplying keyword arguments to pathlib.PurePath " - "is deprecated and scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) - super().__init__(*args) - - def __new__(cls, *args, **kwargs): - if cls is Path: - cls = WindowsPath if os.name == 'nt' else PosixPath - return object.__new__(cls) - - def stat(self, *, follow_symlinks=True): - """ - Return the result of the stat() system call on this path, like - os.stat() does. - """ - return os.stat(self, follow_symlinks=follow_symlinks) - - def is_mount(self): - """ - Check if this path is a mount point - """ - return os.path.ismount(self) - - def is_junction(self): - """ - Whether this path is a junction. - """ - return os.path.isjunction(self) - - def open(self, mode='r', buffering=-1, encoding=None, - errors=None, newline=None): - """ - Open the file pointed by this path and return a file object, as - the built-in open() function does. - """ - if "b" not in mode: - encoding = io.text_encoding(encoding) - return io.open(self, mode, buffering, encoding, errors, newline) - - def read_text(self, encoding=None, errors=None, newline=None): - """ - Open the file in text mode, read it, and close the file. - """ - # Call io.text_encoding() here to ensure any warning is raised at an - # appropriate stack level. - encoding = io.text_encoding(encoding) - return _abc.PathBase.read_text(self, encoding, errors, newline) - - def write_text(self, data, encoding=None, errors=None, newline=None): - """ - Open the file in text mode, write to it, and close the file. - """ - # Call io.text_encoding() here to ensure any warning is raised at an - # appropriate stack level. - encoding = io.text_encoding(encoding) - return _abc.PathBase.write_text(self, data, encoding, errors, newline) - - def iterdir(self): - """Yield path objects of the directory contents. - - The children are yielded in arbitrary order, and the - special entries '.' and '..' are not included. - """ - return (self._make_child_relpath(name) for name in os.listdir(self)) - - def _scandir(self): - return os.scandir(self) - - def _direntry_str(self, entry): - # Transform an entry yielded from _scandir() into a path string. - return entry.name if str(self) == '.' else entry.path - - def _make_child_direntry(self, entry): - # Transform an entry yielded from _scandir() into a path object. - path_str = self._direntry_str(entry) - path = self.with_segments(path_str) - path._str = path_str - path._drv = self.drive - path._root = self.root - path._tail_cached = self._tail + [entry.name] - return path - - def _make_child_relpath(self, name): - if not name: - return self - path_str = str(self) - tail = self._tail - if tail: - path_str = f'{path_str}{self.pathmod.sep}{name}' - elif path_str != '.': - path_str = f'{path_str}{name}' - else: - path_str = name - path = self.with_segments(path_str) - path._str = path_str - path._drv = self.drive - path._root = self.root - path._tail_cached = tail + [name] - return path - - def glob(self, pattern, *, case_sensitive=None, follow_symlinks=None): - """Iterate over this subtree and yield all existing files (of any - kind, including directories) matching the given relative pattern. - """ - sys.audit("pathlib.Path.glob", self, pattern) - if not isinstance(pattern, PurePath): - pattern = self.with_segments(pattern) - return _abc.PathBase.glob( - self, pattern, case_sensitive=case_sensitive, follow_symlinks=follow_symlinks) - - def rglob(self, pattern, *, case_sensitive=None, follow_symlinks=None): - """Recursively yield all existing files (of any kind, including - directories) matching the given relative pattern, anywhere in - this subtree. - """ - sys.audit("pathlib.Path.rglob", self, pattern) - if not isinstance(pattern, PurePath): - pattern = self.with_segments(pattern) - pattern = '**' / pattern - return _abc.PathBase.glob( - self, pattern, case_sensitive=case_sensitive, follow_symlinks=follow_symlinks) - - def walk(self, top_down=True, on_error=None, follow_symlinks=False): - """Walk the directory tree from this directory, similar to os.walk().""" - sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) - return _abc.PathBase.walk( - self, top_down=top_down, on_error=on_error, follow_symlinks=follow_symlinks) - - def absolute(self): - """Return an absolute version of this path - No normalization or symlink resolution is performed. - - Use resolve() to resolve symlinks and remove '..' segments. - """ - if self.is_absolute(): - return self - if self.root: - drive = os.path.splitroot(os.getcwd())[0] - return self._from_parsed_parts(drive, self.root, self._tail) - if self.drive: - # There is a CWD on each drive-letter drive. - cwd = os.path.abspath(self.drive) - else: - cwd = os.getcwd() - if not self._tail: - # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). - # We pass only one argument to with_segments() to avoid the cost - # of joining, and we exploit the fact that getcwd() returns a - # fully-normalized string by storing it in _str. This is used to - # implement Path.cwd(). - result = self.with_segments(cwd) - result._str = cwd - return result - drive, root, rel = os.path.splitroot(cwd) - if not rel: - return self._from_parsed_parts(drive, root, self._tail) - tail = rel.split(self.pathmod.sep) - tail.extend(self._tail) - return self._from_parsed_parts(drive, root, tail) - - def resolve(self, strict=False): - """ - Make the path absolute, resolving all symlinks on the way and also - normalizing it. - """ - - return self.with_segments(os.path.realpath(self, strict=strict)) - - if pwd: - def owner(self, *, follow_symlinks=True): - """ - Return the login name of the file owner. - """ - uid = self.stat(follow_symlinks=follow_symlinks).st_uid - return pwd.getpwuid(uid).pw_name - - if grp: - def group(self, *, follow_symlinks=True): - """ - Return the group name of the file gid. - """ - gid = self.stat(follow_symlinks=follow_symlinks).st_gid - return grp.getgrgid(gid).gr_name - - if hasattr(os, "readlink"): - def readlink(self): - """ - Return the path to which the symbolic link points. - """ - return self.with_segments(os.readlink(self)) - - def touch(self, mode=0o666, exist_ok=True): - """ - Create this file with the given access mode, if it doesn't exist. - """ - - if exist_ok: - # First try to bump modification time - # Implementation note: GNU touch uses the UTIME_NOW option of - # the utimensat() / futimens() functions. - try: - os.utime(self, None) - except OSError: - # Avoid exception chaining - pass - else: - return - flags = os.O_CREAT | os.O_WRONLY - if not exist_ok: - flags |= os.O_EXCL - fd = os.open(self, flags, mode) - os.close(fd) - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - """ - Create a new directory at this given path. - """ - try: - os.mkdir(self, mode) - except FileNotFoundError: - if not parents or self.parent == self: - raise - self.parent.mkdir(parents=True, exist_ok=True) - self.mkdir(mode, parents=False, exist_ok=exist_ok) - except OSError: - # Cannot rely on checking for EEXIST, since the operating system - # could give priority to other errors like EACCES or EROFS - if not exist_ok or not self.is_dir(): - raise - - def chmod(self, mode, *, follow_symlinks=True): - """ - Change the permissions of the path, like os.chmod(). - """ - os.chmod(self, mode, follow_symlinks=follow_symlinks) - - def unlink(self, missing_ok=False): - """ - Remove this file or link. - If the path is a directory, use rmdir() instead. - """ - try: - os.unlink(self) - except FileNotFoundError: - if not missing_ok: - raise - - def rmdir(self): - """ - Remove this directory. The directory must be empty. - """ - os.rmdir(self) - - def rename(self, target): - """ - Rename this path to the target path. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - os.rename(self, target) - return self.with_segments(target) - - def replace(self, target): - """ - Rename this path to the target path, overwriting if that path exists. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - os.replace(self, target) - return self.with_segments(target) - - if hasattr(os, "symlink"): - def symlink_to(self, target, target_is_directory=False): - """ - Make this path a symlink pointing to the target path. - Note the order of arguments (link, target) is the reverse of os.symlink. - """ - os.symlink(target, self, target_is_directory) - - if hasattr(os, "link"): - def hardlink_to(self, target): - """ - Make this path a hard link pointing to the same file as *target*. - - Note the order of arguments (self, target) is the reverse of os.link's. - """ - os.link(target, self) - - def expanduser(self): - """ Return a new path with expanded ~ and ~user constructs - (as returned by os.path.expanduser) - """ - if (not (self.drive or self.root) and - self._tail and self._tail[0][:1] == '~'): - homedir = os.path.expanduser(self._tail[0]) - if homedir[:1] == "~": - raise RuntimeError("Could not determine home directory.") - drv, root, tail = self._parse_path(homedir) - return self._from_parsed_parts(drv, root, tail + self._tail[1:]) - - return self - - @classmethod - def from_uri(cls, uri): - """Return a new path from the given 'file' URI.""" - if not uri.startswith('file:'): - raise ValueError(f"URI does not start with 'file:': {uri!r}") - path = uri[5:] - if path[:3] == '///': - # Remove empty authority - path = path[2:] - elif path[:12] == '//localhost/': - # Remove 'localhost' authority - path = path[11:] - if path[:3] == '///' or (path[:1] == '/' and path[2:3] in ':|'): - # Remove slash before DOS device/UNC path - path = path[1:] - if path[1:2] == '|': - # Replace bar with colon in DOS drive - path = path[:1] + ':' + path[2:] - from urllib.parse import unquote_to_bytes - path = cls(os.fsdecode(unquote_to_bytes(path))) - if not path.is_absolute(): - raise ValueError(f"URI is not absolute: {uri!r}") - return path - - -class PosixPath(Path, PurePosixPath): - """Path subclass for non-Windows systems. - - On a POSIX system, instantiating a Path should return this object. - """ - __slots__ = () - - if os.name == 'nt': - def __new__(cls, *args, **kwargs): - raise UnsupportedOperation( - f"cannot instantiate {cls.__name__!r} on your system") - -class WindowsPath(Path, PureWindowsPath): - """Path subclass for Windows systems. - - On a Windows system, instantiating a Path should return this object. - """ - __slots__ = () - - if os.name != 'nt': - def __new__(cls, *args, **kwargs): - raise UnsupportedOperation( - f"cannot instantiate {cls.__name__!r} on your system") +__all__ = _local.__all__ diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 645d62a0f0699a..e498dc78e83b5e 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -7,265 +7,244 @@ it's developed alongside pathlib. If it finds success and maturity as a PyPI package, it could become a public part of the standard library. -Two base classes are defined here -- PurePathBase and PathBase -- that -resemble pathlib's PurePath and Path respectively. +Three base classes are defined here -- JoinablePath, ReadablePath and +WritablePath. """ import functools -from errno import ENOENT, ENOTDIR, EBADF, ELOOP, EINVAL -from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO - -# -# Internals -# - -_WINERROR_NOT_READY = 21 # drive exists but is not accessible -_WINERROR_INVALID_NAME = 123 # fix for bpo-35306 -_WINERROR_CANT_RESOLVE_FILENAME = 1921 # broken symlink pointing to itself - -# EBADF - guard against macOS `stat` throwing EBADF -_IGNORED_ERRNOS = (ENOENT, ENOTDIR, EBADF, ELOOP) - -_IGNORED_WINERRORS = ( - _WINERROR_NOT_READY, - _WINERROR_INVALID_NAME, - _WINERROR_CANT_RESOLVE_FILENAME) - -def _ignore_error(exception): - return (getattr(exception, 'errno', None) in _IGNORED_ERRNOS or - getattr(exception, 'winerror', None) in _IGNORED_WINERRORS) +import io +import operator +import posixpath +from errno import EINVAL +from glob import _GlobberBase, _no_recurse_symlinks +from pathlib._os import copyfileobj @functools.cache -def _is_case_sensitive(pathmod): - return pathmod.normcase('Aa') == 'Aa' - -# -# Globbing helpers -# - -re = glob = None +def _is_case_sensitive(parser): + return parser.normcase('Aa') == 'Aa' -@functools.lru_cache(maxsize=512) -def _compile_pattern(pat, sep, case_sensitive, recursive=True): - """Compile given glob pattern to a re.Pattern object (observing case - sensitivity).""" - global re, glob - if re is None: - import re, glob - - flags = re.NOFLAG if case_sensitive else re.IGNORECASE - regex = glob.translate(pat, recursive=recursive, include_hidden=True, seps=sep) - return re.compile(regex, flags=flags).match - - -def _select_special(paths, part): - """Yield special literal children of the given paths.""" - for path in paths: - yield path._make_child_relpath(part) +def _explode_path(path): + """ + Split the path into a 2-tuple (anchor, parts), where *anchor* is the + uppermost parent of the path (equivalent to path.parents[-1]), and + *parts* is a reversed list of parts following the anchor. + """ + split = path.parser.split + path = str(path) + parent, name = split(path) + names = [] + while path != parent: + names.append(name) + path = parent + parent, name = split(path) + return path, names -def _select_children(parent_paths, dir_only, follow_symlinks, match): - """Yield direct children of given paths, filtering by name and type.""" - if follow_symlinks is None: - follow_symlinks = True - for parent_path in parent_paths: +def magic_open(path, mode='r', buffering=-1, encoding=None, errors=None, + newline=None): + """ + Open the file pointed to by this path and return a file object, as + the built-in open() function does. + """ + try: + return io.open(path, mode, buffering, encoding, errors, newline) + except TypeError: + pass + cls = type(path) + text = 'b' not in mode + mode = ''.join(sorted(c for c in mode if c not in 'bt')) + if text: try: - # We must close the scandir() object before proceeding to - # avoid exhausting file descriptors when globbing deep trees. - with parent_path._scandir() as scandir_it: - entries = list(scandir_it) - except OSError: + attr = getattr(cls, f'__open_{mode}__') + except AttributeError: pass else: - for entry in entries: - if dir_only: - try: - if not entry.is_dir(follow_symlinks=follow_symlinks): - continue - except OSError: - continue - # Avoid cost of making a path object for non-matching paths by - # matching against the os.DirEntry.name string. - if match is None or match(entry.name): - yield parent_path._make_child_direntry(entry) - - -def _select_recursive(parent_paths, dir_only, follow_symlinks, match): - """Yield given paths and all their children, recursively, filtering by - string and type. - """ - if follow_symlinks is None: - follow_symlinks = False - for parent_path in parent_paths: - if match is not None: - # If we're filtering paths through a regex, record the length of - # the parent path. We'll pass it to match(path, pos=...) later. - parent_len = len(str(parent_path._make_child_relpath('_'))) - 1 - paths = [parent_path._make_child_relpath('')] - while paths: - path = paths.pop() - if match is None or match(str(path), parent_len): - # Yield *directory* path that matches pattern (if any). - yield path - try: - # We must close the scandir() object before proceeding to - # avoid exhausting file descriptors when globbing deep trees. - with path._scandir() as scandir_it: - entries = list(scandir_it) - except OSError: - pass - else: - for entry in entries: - # Handle directory entry. - try: - if entry.is_dir(follow_symlinks=follow_symlinks): - # Recurse into this directory. - paths.append(path._make_child_direntry(entry)) - continue - except OSError: - pass - - # Handle file entry. - if not dir_only: - # Avoid cost of making a path object for non-matching - # files by matching against the os.DirEntry object. - if match is None or match(path._direntry_str(entry), parent_len): - # Yield *file* path that matches pattern (if any). - yield path._make_child_direntry(entry) + return attr(path, buffering, encoding, errors, newline) - -def _select_unique(paths): - """Yields the given paths, filtering out duplicates.""" - yielded = set() try: - for path in paths: - path_str = str(path) - if path_str not in yielded: - yield path - yielded.add(path_str) - finally: - yielded.clear() + attr = getattr(cls, f'__open_{mode}b__') + except AttributeError: + pass + else: + stream = attr(path, buffering) + if text: + stream = io.TextIOWrapper(stream, encoding, errors, newline) + return stream + + raise TypeError(f"{cls.__name__} can't be opened with mode {mode!r}") -class UnsupportedOperation(NotImplementedError): - """An exception that is raised when an unsupported operation is called on - a path object. +class PathGlobber(_GlobberBase): + """ + Class providing shell-style globbing for path objects. """ - pass + lexists = operator.methodcaller('exists', follow_symlinks=False) + add_slash = operator.methodcaller('joinpath', '') + scandir = operator.methodcaller('_scandir') -class PathModuleBase: - """Base class for path modules, which do low-level path manipulation. + @staticmethod + def concat_path(path, text): + """Appends text to the given path.""" + return path.with_segments(str(path) + text) - Path modules provide a subset of the os.path API, specifically those - functions needed to provide PurePathBase functionality. Each PurePathBase - subclass references its path module via a 'pathmod' class attribute. - Every method in this base class raises an UnsupportedOperation exception. +class CopyReader: + """ + Class that implements the "read" part of copying between path objects. + An instance of this class is available from the ReadablePath._copy_reader + property. """ + __slots__ = ('_path',) - @classmethod - def _unsupported_msg(cls, attribute): - return f"{cls.__name__}.{attribute} is unsupported" + def __init__(self, path): + self._path = path - @property - def sep(self): - """The character used to separate path components.""" - raise UnsupportedOperation(self._unsupported_msg('sep')) - - def join(self, path, *paths): - """Join path segments.""" - raise UnsupportedOperation(self._unsupported_msg('join()')) - - def split(self, path): - """Split the path into a pair (head, tail), where *head* is everything - before the final path separator, and *tail* is everything after. - Either part may be empty. + _readable_metakeys = frozenset() + + def _read_metadata(self, metakeys, *, follow_symlinks=True): """ - raise UnsupportedOperation(self._unsupported_msg('split()')) + Returns path metadata as a dict with string keys. + """ + raise NotImplementedError + + +class CopyWriter: + """ + Class that implements the "write" part of copying between path objects. An + instance of this class is available from the WritablePath._copy_writer + property. + """ + __slots__ = ('_path',) - def splitdrive(self, path): - """Split the path into a 2-item tuple (drive, tail), where *drive* is - a device name or mount point, and *tail* is everything after the - drive. Either part may be empty.""" - raise UnsupportedOperation(self._unsupported_msg('splitdrive()')) + def __init__(self, path): + self._path = path - def normcase(self, path): - """Normalize the case of the path.""" - raise UnsupportedOperation(self._unsupported_msg('normcase()')) + _writable_metakeys = frozenset() - def isabs(self, path): - """Returns whether the path is absolute, i.e. unaffected by the - current directory or drive.""" - raise UnsupportedOperation(self._unsupported_msg('isabs()')) + def _write_metadata(self, metadata, *, follow_symlinks=True): + """ + Sets path metadata from the given dict with string keys. + """ + raise NotImplementedError + + def _create(self, source, follow_symlinks, dirs_exist_ok, preserve_metadata): + self._ensure_distinct_path(source) + if preserve_metadata: + metakeys = self._writable_metakeys & source._copy_reader._readable_metakeys + else: + metakeys = None + if not follow_symlinks and source.is_symlink(): + self._create_symlink(source, metakeys) + elif source.is_dir(): + self._create_dir(source, metakeys, follow_symlinks, dirs_exist_ok) + else: + self._create_file(source, metakeys) + return self._path + + def _create_dir(self, source, metakeys, follow_symlinks, dirs_exist_ok): + """Copy the given directory to our path.""" + children = list(source.iterdir()) + self._path.mkdir(exist_ok=dirs_exist_ok) + for src in children: + dst = self._path.joinpath(src.name) + if not follow_symlinks and src.is_symlink(): + dst._copy_writer._create_symlink(src, metakeys) + elif src.is_dir(): + dst._copy_writer._create_dir(src, metakeys, follow_symlinks, dirs_exist_ok) + else: + dst._copy_writer._create_file(src, metakeys) + if metakeys: + metadata = source._copy_reader._read_metadata(metakeys) + if metadata: + self._write_metadata(metadata) + + def _create_file(self, source, metakeys): + """Copy the given file to our path.""" + self._ensure_different_file(source) + with magic_open(source, 'rb') as source_f: + try: + with magic_open(self._path, 'wb') as target_f: + copyfileobj(source_f, target_f) + except IsADirectoryError as e: + if not self._path.exists(): + # Raise a less confusing exception. + raise FileNotFoundError( + f'Directory does not exist: {self._path}') from e + raise + if metakeys: + metadata = source._copy_reader._read_metadata(metakeys) + if metadata: + self._write_metadata(metadata) + + def _create_symlink(self, source, metakeys): + """Copy the given symbolic link to our path.""" + self._path.symlink_to(source.readlink()) + if metakeys: + metadata = source._copy_reader._read_metadata(metakeys, follow_symlinks=False) + if metadata: + self._write_metadata(metadata, follow_symlinks=False) + + def _ensure_different_file(self, source): + """ + Raise OSError(EINVAL) if both paths refer to the same file. + """ + pass + + def _ensure_distinct_path(self, source): + """ + Raise OSError(EINVAL) if the other path is within this path. + """ + # Note: there is no straightforward, foolproof algorithm to determine + # if one directory is within another (a particularly perverse example + # would be a single network share mounted in one location via NFS, and + # in another location via CIFS), so we simply checks whether the + # other path is lexically equal to, or within, this path. + if source == self._path: + err = OSError(EINVAL, "Source and target are the same path") + elif source in self._path.parents: + err = OSError(EINVAL, "Source path is a parent of target path") + else: + return + err.filename = str(source) + err.filename2 = str(self._path) + raise err -class PurePathBase: +class JoinablePath: """Base class for pure path objects. This class *does not* provide several magic methods that are defined in - its subclass PurePath. They are: __fspath__, __bytes__, __reduce__, - __hash__, __eq__, __lt__, __le__, __gt__, __ge__. Its initializer and path - joining methods accept only strings, not os.PathLike objects more broadly. + its subclass PurePath. They are: __init__, __fspath__, __bytes__, + __reduce__, __hash__, __eq__, __lt__, __le__, __gt__, __ge__. """ - __slots__ = ( - # The `_raw_path` slot store a joined string path. This is set in the - # `__init__()` method. - '_raw_path', - - # The '_resolving' slot stores a boolean indicating whether the path - # is being processed by `PathBase.resolve()`. This prevents duplicate - # work from occurring when `resolve()` calls `stat()` or `readlink()`. - '_resolving', - ) - pathmod = PathModuleBase() - - def __init__(self, path, *paths): - self._raw_path = self.pathmod.join(path, *paths) if paths else path - if not isinstance(self._raw_path, str): - raise TypeError( - f"path should be a str, not {type(self._raw_path).__name__!r}") - self._resolving = False + __slots__ = () + parser = posixpath def with_segments(self, *pathsegments): """Construct a new path object from any number of path-like objects. Subclasses may override this method to customize how new path objects are created from methods like `iterdir()`. """ - return type(self)(*pathsegments) + raise NotImplementedError def __str__(self): """Return the string representation of the path, suitable for passing to system calls.""" - return self._raw_path - - def as_posix(self): - """Return the string representation of the path with forward (/) - slashes.""" - return str(self).replace(self.pathmod.sep, '/') - - @property - def drive(self): - """The drive prefix (letter or UNC path), if any.""" - return self.pathmod.splitdrive(self.anchor)[0] - - @property - def root(self): - """The root of the path, if any.""" - return self.pathmod.splitdrive(self.anchor)[1] + raise NotImplementedError @property def anchor(self): """The concatenation of the drive and root, or ''.""" - return self._stack[0] + return _explode_path(self)[0] @property def name(self): """The final path component, if any.""" - return self.pathmod.split(self._raw_path)[1] + return self.parser.split(str(self))[1] @property def suffix(self): @@ -274,12 +253,7 @@ def suffix(self): This includes the leading period. For example: '.txt' """ - name = self.name - i = name.rfind('.') - if 0 < i < len(name) - 1: - return name[i:] - else: - return '' + return self.parser.splitext(self.name)[1] @property def suffixes(self): @@ -288,28 +262,25 @@ def suffixes(self): These include the leading periods. For example: ['.tar', '.gz'] """ - name = self.name - if name.endswith('.'): - return [] - name = name.lstrip('.') - return ['.' + suffix for suffix in name.split('.')[1:]] + split = self.parser.splitext + stem, suffix = split(self.name) + suffixes = [] + while suffix: + suffixes.append(suffix) + stem, suffix = split(stem) + return suffixes[::-1] @property def stem(self): """The final path component, minus its last suffix.""" - name = self.name - i = name.rfind('.') - if 0 < i < len(name) - 1: - return name[:i] - else: - return name + return self.parser.splitext(self.name)[0] def with_name(self, name): """Return a new path with the file name changed.""" - split = self.pathmod.split + split = self.parser.split if split(name)[0]: raise ValueError(f"Invalid name {name!r}") - return self.with_segments(split(self._raw_path)[0], name) + return self.with_segments(split(str(self))[0], name) def with_stem(self, stem): """Return a new path with the stem changed.""" @@ -328,66 +299,19 @@ def with_suffix(self, suffix): string, remove the suffix from the path. """ stem = self.stem - if not suffix: - return self.with_name(stem) - elif not stem: + if not stem: # If the stem is empty, we can't make the suffix non-empty. raise ValueError(f"{self!r} has an empty name") - elif suffix.startswith('.') and len(suffix) > 1: - return self.with_name(stem + suffix) - else: + elif suffix and not suffix.startswith('.'): raise ValueError(f"Invalid suffix {suffix!r}") - - def relative_to(self, other, *, walk_up=False): - """Return the relative path to another path identified by the passed - arguments. If the operation is not possible (because this is not - related to the other path), raise ValueError. - - The *walk_up* parameter controls whether `..` may be used to resolve - the path. - """ - if not isinstance(other, PurePathBase): - other = self.with_segments(other) - anchor0, parts0 = self._stack - anchor1, parts1 = other._stack - if anchor0 != anchor1: - raise ValueError(f"{self._raw_path!r} and {other._raw_path!r} have different anchors") - while parts0 and parts1 and parts0[-1] == parts1[-1]: - parts0.pop() - parts1.pop() - for part in parts1: - if not part or part == '.': - pass - elif not walk_up: - raise ValueError(f"{self._raw_path!r} is not in the subpath of {other._raw_path!r}") - elif part == '..': - raise ValueError(f"'..' segment in {other._raw_path!r} cannot be walked") - else: - parts0.append('..') - return self.with_segments('', *reversed(parts0)) - - def is_relative_to(self, other): - """Return True if the path is relative to another path or False. - """ - if not isinstance(other, PurePathBase): - other = self.with_segments(other) - anchor0, parts0 = self._stack - anchor1, parts1 = other._stack - if anchor0 != anchor1: - return False - while parts0 and parts1 and parts0[-1] == parts1[-1]: - parts0.pop() - parts1.pop() - for part in parts1: - if part and part != '.': - return False - return True + else: + return self.with_name(stem + suffix) @property def parts(self): """An object providing sequence-like access to the components in the filesystem path.""" - anchor, parts = self._stack + anchor, parts = _explode_path(self) if anchor: parts.append(anchor) return tuple(reversed(parts)) @@ -398,53 +322,34 @@ def joinpath(self, *pathsegments): paths) or a totally different path (if one of the arguments is anchored). """ - return self.with_segments(self._raw_path, *pathsegments) + return self.with_segments(str(self), *pathsegments) def __truediv__(self, key): try: - return self.with_segments(self._raw_path, key) + return self.with_segments(str(self), key) except TypeError: return NotImplemented def __rtruediv__(self, key): try: - return self.with_segments(key, self._raw_path) + return self.with_segments(key, str(self)) except TypeError: return NotImplemented - @property - def _stack(self): - """ - Split the path into a 2-tuple (anchor, parts), where *anchor* is the - uppermost parent of the path (equivalent to path.parents[-1]), and - *parts* is a reversed list of parts following the anchor. - """ - split = self.pathmod.split - path = self._raw_path - parent, name = split(path) - names = [] - while path != parent: - names.append(name) - path = parent - parent, name = split(path) - return path, names - @property def parent(self): """The logical parent of the path.""" - path = self._raw_path - parent = self.pathmod.split(path)[0] + path = str(self) + parent = self.parser.split(path)[0] if path != parent: - parent = self.with_segments(parent) - parent._resolving = self._resolving - return parent + return self.with_segments(parent) return self @property def parents(self): """A sequence of this path's logical parents.""" - split = self.pathmod.split - path = self._raw_path + split = self.parser.split + path = str(self) parent = split(path)[0] parents = [] while path != parent: @@ -453,71 +358,28 @@ def parents(self): parent = split(path)[0] return tuple(parents) - def is_absolute(self): - """True if the path is absolute (has both a root and, if applicable, - a drive).""" - return self.pathmod.isabs(self._raw_path) - - @property - def _pattern_stack(self): - """Stack of path components, to be used with patterns in glob().""" - anchor, parts = self._stack - if anchor: - raise NotImplementedError("Non-relative patterns are unsupported") - return parts - - @property - def _pattern_str(self): - """The path expressed as a string, for use in pattern-matching.""" - return str(self) - - def match(self, path_pattern, *, case_sensitive=None): - """ - Return True if this path matches the given pattern. If the pattern is - relative, matching is done from the right; otherwise, the entire path - is matched. The recursive wildcard '**' is *not* supported by this - method. - """ - if not isinstance(path_pattern, PurePathBase): - path_pattern = self.with_segments(path_pattern) - if case_sensitive is None: - case_sensitive = _is_case_sensitive(self.pathmod) - sep = path_pattern.pathmod.sep - path_parts = self.parts[::-1] - pattern_parts = path_pattern.parts[::-1] - if not pattern_parts: - raise ValueError("empty pattern") - if len(path_parts) < len(pattern_parts): - return False - if len(path_parts) > len(pattern_parts) and path_pattern.anchor: - return False - for path_part, pattern_part in zip(path_parts, pattern_parts): - match = _compile_pattern(pattern_part, sep, case_sensitive, recursive=False) - if match(path_part) is None: - return False - return True - def full_match(self, pattern, *, case_sensitive=None): """ Return True if this path matches the given glob-style pattern. The pattern is matched against the entire path. """ - if not isinstance(pattern, PurePathBase): + if not isinstance(pattern, JoinablePath): pattern = self.with_segments(pattern) if case_sensitive is None: - case_sensitive = _is_case_sensitive(self.pathmod) - match = _compile_pattern(pattern._pattern_str, pattern.pathmod.sep, case_sensitive) - return match(self._pattern_str) is not None + case_sensitive = _is_case_sensitive(self.parser) + globber = PathGlobber(pattern.parser.sep, case_sensitive, recursive=True) + match = globber.compile(str(pattern)) + return match(str(self)) is not None -class PathBase(PurePathBase): +class ReadablePath(JoinablePath): """Base class for concrete path objects. This class provides dummy implementations for many methods that derived classes can override selectively; the default implementations raise - UnsupportedOperation. The most basic methods, such as stat() and open(), - directly raise UnsupportedOperation; these basic methods are called by + NotImplementedError. The most basic methods, such as stat() and open(), + directly raise NotImplementedError; these basic methods are called by other methods such as is_dir() and read_text(). The Path class derives this class to implement local filesystem paths. @@ -526,30 +388,6 @@ class PathBase(PurePathBase): """ __slots__ = () - # Maximum number of symlinks to follow in resolve() - _max_symlinks = 40 - - @classmethod - def _unsupported_msg(cls, attribute): - return f"{cls.__name__}.{attribute} is unsupported" - - def stat(self, *, follow_symlinks=True): - """ - Return the result of the stat() system call on this path, like - os.stat() does. - """ - raise UnsupportedOperation(self._unsupported_msg('stat()')) - - def lstat(self): - """ - Like stat(), except if the path points to a symlink, the symlink's - status information is returned, rather than its target's. - """ - return self.stat(follow_symlinks=False) - - - # Convenience functions for querying the stat results - def exists(self, *, follow_symlinks=True): """ Whether this path exists. @@ -557,210 +395,56 @@ def exists(self, *, follow_symlinks=True): This method normally follows symlinks; to check whether a symlink exists, add the argument follow_symlinks=False. """ - try: - self.stat(follow_symlinks=follow_symlinks) - except OSError as e: - if not _ignore_error(e): - raise - return False - except ValueError: - # Non-encodable path - return False - return True + raise NotImplementedError def is_dir(self, *, follow_symlinks=True): """ Whether this path is a directory. """ - try: - return S_ISDIR(self.stat(follow_symlinks=follow_symlinks).st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False + raise NotImplementedError def is_file(self, *, follow_symlinks=True): """ Whether this path is a regular file (also True for symlinks pointing to regular files). """ - try: - return S_ISREG(self.stat(follow_symlinks=follow_symlinks).st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_mount(self): - """ - Check if this path is a mount point - """ - # Need to exist and be a dir - if not self.exists() or not self.is_dir(): - return False - - try: - parent_dev = self.parent.stat().st_dev - except OSError: - return False - - dev = self.stat().st_dev - if dev != parent_dev: - return True - ino = self.stat().st_ino - parent_ino = self.parent.stat().st_ino - return ino == parent_ino + raise NotImplementedError def is_symlink(self): """ Whether this path is a symbolic link. """ - try: - return S_ISLNK(self.lstat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist - return False - except ValueError: - # Non-encodable path - return False + raise NotImplementedError - def is_junction(self): + def __open_rb__(self, buffering=-1): """ - Whether this path is a junction. + Open the file pointed to by this path for reading in binary mode and + return a file object, like open(mode='rb'). """ - # Junctions are a Windows-only feature, not present in POSIX nor the - # majority of virtual filesystems. There is no cross-platform idiom - # to check for junctions (using stat().st_mode). - return False - - def is_block_device(self): - """ - Whether this path is a block device. - """ - try: - return S_ISBLK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_char_device(self): - """ - Whether this path is a character device. - """ - try: - return S_ISCHR(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_fifo(self): - """ - Whether this path is a FIFO. - """ - try: - return S_ISFIFO(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_socket(self): - """ - Whether this path is a socket. - """ - try: - return S_ISSOCK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def samefile(self, other_path): - """Return whether other_path is the same or not as this file - (as returned by os.path.samefile()). - """ - st = self.stat() - try: - other_st = other_path.stat() - except AttributeError: - other_st = self.with_segments(other_path).stat() - return (st.st_ino == other_st.st_ino and - st.st_dev == other_st.st_dev) - - def open(self, mode='r', buffering=-1, encoding=None, - errors=None, newline=None): - """ - Open the file pointed by this path and return a file object, as - the built-in open() function does. - """ - raise UnsupportedOperation(self._unsupported_msg('open()')) + raise NotImplementedError def read_bytes(self): """ Open the file in bytes mode, read it, and close the file. """ - with self.open(mode='rb') as f: + with magic_open(self, mode='rb', buffering=0) as f: return f.read() def read_text(self, encoding=None, errors=None, newline=None): """ Open the file in text mode, read it, and close the file. """ - with self.open(mode='r', encoding=encoding, errors=errors, newline=newline) as f: + with magic_open(self, mode='r', encoding=encoding, errors=errors, newline=newline) as f: return f.read() - def write_bytes(self, data): - """ - Open the file in bytes mode, write to it, and close the file. - """ - # type-check for the buffer interface before truncating the file - view = memoryview(data) - with self.open(mode='wb') as f: - return f.write(view) + def _scandir(self): + """Yield os.DirEntry-like objects of the directory contents. - def write_text(self, data, encoding=None, errors=None, newline=None): - """ - Open the file in text mode, write to it, and close the file. + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. """ - if not isinstance(data, str): - raise TypeError('data must be str, not %s' % - data.__class__.__name__) - with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: - return f.write(data) + import contextlib + return contextlib.nullcontext(self.iterdir()) def iterdir(self): """Yield path objects of the directory contents. @@ -768,336 +452,155 @@ def iterdir(self): The children are yielded in arbitrary order, and the special entries '.' and '..' are not included. """ - raise UnsupportedOperation(self._unsupported_msg('iterdir()')) - - def _scandir(self): - # Emulate os.scandir(), which returns an object that can be used as a - # context manager. This method is called by walk() and glob(). - from contextlib import nullcontext - return nullcontext(self.iterdir()) - - def _direntry_str(self, entry): - # Transform an entry yielded from _scandir() into a path string. - # PathBase._scandir() yields PathBase objects, so use str(). - return str(entry) + raise NotImplementedError - def _make_child_direntry(self, entry): - # Transform an entry yielded from _scandir() into a path object. - # PathBase._scandir() yields PathBase objects, so this is a no-op. - return entry - - def _make_child_relpath(self, name): - return self.joinpath(name) - - def glob(self, pattern, *, case_sensitive=None, follow_symlinks=True): + def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=True): """Iterate over this subtree and yield all existing files (of any kind, including directories) matching the given relative pattern. """ - if not isinstance(pattern, PurePathBase): + if not isinstance(pattern, JoinablePath): pattern = self.with_segments(pattern) + anchor, parts = _explode_path(pattern) + if anchor: + raise NotImplementedError("Non-relative patterns are unsupported") if case_sensitive is None: - # TODO: evaluate case-sensitivity of each directory in _select_children(). - case_sensitive = _is_case_sensitive(self.pathmod) - - stack = pattern._pattern_stack - specials = ('', '.', '..') - deduplicate_paths = False - sep = self.pathmod.sep - paths = iter([self] if self.is_dir() else []) - while stack: - part = stack.pop() - if part in specials: - # Join special component (e.g. '..') onto paths. - paths = _select_special(paths, part) - - elif part == '**': - # Consume following '**' components, which have no effect. - while stack and stack[-1] == '**': - stack.pop() - - # Consume following non-special components, provided we're - # treating symlinks consistently. Each component is joined - # onto 'part', which is used to generate an re.Pattern object. - if follow_symlinks is not None: - while stack and stack[-1] not in specials: - part += sep + stack.pop() - - # If the previous loop consumed pattern components, compile an - # re.Pattern object based on those components. - match = _compile_pattern(part, sep, case_sensitive) if part != '**' else None - - # Recursively walk directories, filtering by type and regex. - paths = _select_recursive(paths, bool(stack), follow_symlinks, match) - - # De-duplicate if we've already seen a '**' component. - if deduplicate_paths: - paths = _select_unique(paths) - deduplicate_paths = True - - elif '**' in part: - raise ValueError("Invalid pattern: '**' can only be an entire path component") - - else: - # If the pattern component isn't '*', compile an re.Pattern - # object based on the component. - match = _compile_pattern(part, sep, case_sensitive) if part != '*' else None - - # Iterate over directories' children filtering by type and regex. - paths = _select_children(paths, bool(stack), follow_symlinks, match) - return paths + case_sensitive = _is_case_sensitive(self.parser) + case_pedantic = False + elif case_sensitive == _is_case_sensitive(self.parser): + case_pedantic = False + else: + case_pedantic = True + recursive = True if recurse_symlinks else _no_recurse_symlinks + globber = PathGlobber(self.parser.sep, case_sensitive, case_pedantic, recursive) + select = globber.selector(parts) + return select(self) - def rglob(self, pattern, *, case_sensitive=None, follow_symlinks=True): + def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=True): """Recursively yield all existing files (of any kind, including directories) matching the given relative pattern, anywhere in this subtree. """ - if not isinstance(pattern, PurePathBase): + if not isinstance(pattern, JoinablePath): pattern = self.with_segments(pattern) pattern = '**' / pattern - return self.glob(pattern, case_sensitive=case_sensitive, follow_symlinks=follow_symlinks) + return self.glob(pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks) def walk(self, top_down=True, on_error=None, follow_symlinks=False): """Walk the directory tree from this directory, similar to os.walk().""" paths = [self] - while paths: path = paths.pop() if isinstance(path, tuple): yield path continue - - # We may not have read permission for self, in which case we can't - # get a list of the files the directory contains. os.walk() - # always suppressed the exception in that instance, rather than - # blow up for a minor reason when (say) a thousand readable - # directories are still left to visit. That logic is copied here. + dirnames = [] + filenames = [] + if not top_down: + paths.append((path, dirnames, filenames)) try: - scandir_obj = path._scandir() + with path._scandir() as entries: + for entry in entries: + name = entry.name + try: + if entry.is_dir(follow_symlinks=follow_symlinks): + if not top_down: + paths.append(path.joinpath(name)) + dirnames.append(name) + else: + filenames.append(name) + except OSError: + filenames.append(name) except OSError as error: if on_error is not None: on_error(error) - continue - - with scandir_obj as scandir_it: - dirnames = [] - filenames = [] if not top_down: - paths.append((path, dirnames, filenames)) - for entry in scandir_it: - try: - is_dir = entry.is_dir(follow_symlinks=follow_symlinks) - except OSError: - # Carried over from os.path.isdir(). - is_dir = False - - if is_dir: - if not top_down: - paths.append(path._make_child_direntry(entry)) - dirnames.append(entry.name) - else: - filenames.append(entry.name) - + while not isinstance(paths.pop(), tuple): + pass + continue if top_down: yield path, dirnames, filenames - paths += [path._make_child_relpath(d) for d in reversed(dirnames)] - - def absolute(self): - """Return an absolute version of this path - No normalization or symlink resolution is performed. - - Use resolve() to resolve symlinks and remove '..' segments. - """ - raise UnsupportedOperation(self._unsupported_msg('absolute()')) - - @classmethod - def cwd(cls): - """Return a new path pointing to the current working directory.""" - # We call 'absolute()' rather than using 'os.getcwd()' directly to - # enable users to replace the implementation of 'absolute()' in a - # subclass and benefit from the new behaviour here. This works because - # os.path.abspath('.') == os.getcwd(). - return cls('').absolute() - - def expanduser(self): - """ Return a new path with expanded ~ and ~user constructs - (as returned by os.path.expanduser) - """ - raise UnsupportedOperation(self._unsupported_msg('expanduser()')) - - @classmethod - def home(cls): - """Return a new path pointing to expanduser('~'). - """ - return cls("~").expanduser() + paths += [path.joinpath(d) for d in reversed(dirnames)] def readlink(self): """ Return the path to which the symbolic link points. """ - raise UnsupportedOperation(self._unsupported_msg('readlink()')) - readlink._supported = False + raise NotImplementedError - def resolve(self, strict=False): + _copy_reader = property(CopyReader) + + def copy(self, target, follow_symlinks=True, dirs_exist_ok=False, + preserve_metadata=False): """ - Make the path absolute, resolving all symlinks on the way and also - normalizing it. + Recursively copy this file or directory tree to the given destination. """ - if self._resolving: - return self - path_root, parts = self._stack - path = self.with_segments(path_root) + if not hasattr(target, '_copy_writer'): + target = self.with_segments(target) + + # Delegate to the target path's CopyWriter object. try: - path = path.absolute() - except UnsupportedOperation: - path_tail = [] - else: - path_root, path_tail = path._stack - path_tail.reverse() - - # If the user has *not* overridden the `readlink()` method, then symlinks are unsupported - # and (in non-strict mode) we can improve performance by not calling `stat()`. - querying = strict or getattr(self.readlink, '_supported', True) - link_count = 0 - while parts: - part = parts.pop() - if not part or part == '.': - continue - if part == '..': - if not path_tail: - if path_root: - # Delete '..' segment immediately following root - continue - elif path_tail[-1] != '..': - # Delete '..' segment and its predecessor - path_tail.pop() - continue - path_tail.append(part) - if querying and part != '..': - path = self.with_segments(path_root + self.pathmod.sep.join(path_tail)) - path._resolving = True - try: - st = path.stat(follow_symlinks=False) - if S_ISLNK(st.st_mode): - # Like Linux and macOS, raise OSError(errno.ELOOP) if too many symlinks are - # encountered during resolution. - link_count += 1 - if link_count >= self._max_symlinks: - raise OSError(ELOOP, "Too many symbolic links in path", self._raw_path) - target_root, target_parts = path.readlink()._stack - # If the symlink target is absolute (like '/etc/hosts'), set the current - # path to its uppermost parent (like '/'). - if target_root: - path_root = target_root - path_tail.clear() - else: - path_tail.pop() - # Add the symlink target's reversed tail parts (like ['hosts', 'etc']) to - # the stack of unresolved path parts. - parts.extend(target_parts) - continue - elif parts and not S_ISDIR(st.st_mode): - raise NotADirectoryError(ENOTDIR, "Not a directory", self._raw_path) - except OSError: - if strict: - raise - else: - querying = False - return self.with_segments(path_root + self.pathmod.sep.join(path_tail)) + create = target._copy_writer._create + except AttributeError: + raise TypeError(f"Target is not writable: {target}") from None + return create(self, follow_symlinks, dirs_exist_ok, preserve_metadata) - def symlink_to(self, target, target_is_directory=False): + def copy_into(self, target_dir, *, follow_symlinks=True, + dirs_exist_ok=False, preserve_metadata=False): """ - Make this path a symlink pointing to the target path. - Note the order of arguments (link, target) is the reverse of os.symlink. + Copy this file or directory tree into the given existing directory. """ - raise UnsupportedOperation(self._unsupported_msg('symlink_to()')) + name = self.name + if not name: + raise ValueError(f"{self!r} has an empty name") + elif hasattr(target_dir, '_copy_writer'): + target = target_dir / name + else: + target = self.with_segments(target_dir, name) + return self.copy(target, follow_symlinks=follow_symlinks, + dirs_exist_ok=dirs_exist_ok, + preserve_metadata=preserve_metadata) - def hardlink_to(self, target): - """ - Make this path a hard link pointing to the same file as *target*. - Note the order of arguments (self, target) is the reverse of os.link's. - """ - raise UnsupportedOperation(self._unsupported_msg('hardlink_to()')) +class WritablePath(JoinablePath): + __slots__ = () - def touch(self, mode=0o666, exist_ok=True): + def symlink_to(self, target, target_is_directory=False): """ - Create this file with the given access mode, if it doesn't exist. + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. """ - raise UnsupportedOperation(self._unsupported_msg('touch()')) + raise NotImplementedError def mkdir(self, mode=0o777, parents=False, exist_ok=False): """ Create a new directory at this given path. """ - raise UnsupportedOperation(self._unsupported_msg('mkdir()')) + raise NotImplementedError - def rename(self, target): + def __open_wb__(self, buffering=-1): """ - Rename this path to the target path. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - raise UnsupportedOperation(self._unsupported_msg('rename()')) - - def replace(self, target): + Open the file pointed to by this path for writing in binary mode and + return a file object, like open(mode='wb'). """ - Rename this path to the target path, overwriting if that path exists. + raise NotImplementedError - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - raise UnsupportedOperation(self._unsupported_msg('replace()')) - - def chmod(self, mode, *, follow_symlinks=True): - """ - Change the permissions of the path, like os.chmod(). - """ - raise UnsupportedOperation(self._unsupported_msg('chmod()')) - - def lchmod(self, mode): - """ - Like chmod(), except if the path points to a symlink, the symlink's - permissions are changed, rather than its target's. - """ - self.chmod(mode, follow_symlinks=False) - - def unlink(self, missing_ok=False): - """ - Remove this file or link. - If the path is a directory, use rmdir() instead. - """ - raise UnsupportedOperation(self._unsupported_msg('unlink()')) - - def rmdir(self): - """ - Remove this directory. The directory must be empty. - """ - raise UnsupportedOperation(self._unsupported_msg('rmdir()')) - - def owner(self, *, follow_symlinks=True): + def write_bytes(self, data): """ - Return the login name of the file owner. + Open the file in bytes mode, write to it, and close the file. """ - raise UnsupportedOperation(self._unsupported_msg('owner()')) + # type-check for the buffer interface before truncating the file + view = memoryview(data) + with magic_open(self, mode='wb') as f: + return f.write(view) - def group(self, *, follow_symlinks=True): + def write_text(self, data, encoding=None, errors=None, newline=None): """ - Return the group name of the file gid. + Open the file in text mode, write to it, and close the file. """ - raise UnsupportedOperation(self._unsupported_msg('group()')) - - @classmethod - def from_uri(cls, uri): - """Return a new path from the given 'file' URI.""" - raise UnsupportedOperation(cls._unsupported_msg('from_uri()')) + if not isinstance(data, str): + raise TypeError('data must be str, not %s' % + data.__class__.__name__) + with magic_open(self, mode='w', encoding=encoding, errors=errors, newline=newline) as f: + return f.write(data) - def as_uri(self): - """Return the path as a URI.""" - raise UnsupportedOperation(self._unsupported_msg('as_uri()')) + _copy_writer = property(CopyWriter) diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py new file mode 100644 index 00000000000000..b3ec934f7510de --- /dev/null +++ b/Lib/pathlib/_local.py @@ -0,0 +1,1326 @@ +import io +import ntpath +import operator +import os +import posixpath +import sys +from errno import * +from glob import _StringGlobber, _no_recurse_symlinks +from itertools import chain +from stat import S_IMODE, S_ISDIR, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO +from _collections_abc import Sequence + +try: + import pwd +except ImportError: + pwd = None +try: + import grp +except ImportError: + grp = None + +from pathlib._os import copyfile +from pathlib._abc import CopyReader, CopyWriter, JoinablePath, ReadablePath, WritablePath + + +__all__ = [ + "UnsupportedOperation", + "PurePath", "PurePosixPath", "PureWindowsPath", + "Path", "PosixPath", "WindowsPath", + ] + + +class UnsupportedOperation(NotImplementedError): + """An exception that is raised when an unsupported operation is attempted. + """ + pass + + +class _PathParents(Sequence): + """This object provides sequence-like access to the logical ancestors + of a path. Don't try to construct it yourself.""" + __slots__ = ('_path', '_drv', '_root', '_tail') + + def __init__(self, path): + self._path = path + self._drv = path.drive + self._root = path.root + self._tail = path._tail + + def __len__(self): + return len(self._tail) + + def __getitem__(self, idx): + if isinstance(idx, slice): + return tuple(self[i] for i in range(*idx.indices(len(self)))) + + if idx >= len(self) or idx < -len(self): + raise IndexError(idx) + if idx < 0: + idx += len(self) + return self._path._from_parsed_parts(self._drv, self._root, + self._tail[:-idx - 1]) + + def __repr__(self): + return "<{}.parents>".format(type(self._path).__name__) + + +class _LocalCopyReader(CopyReader): + """This object implements the "read" part of copying local paths. Don't + try to construct it yourself. + """ + __slots__ = () + + _readable_metakeys = {'mode', 'times_ns'} + if hasattr(os.stat_result, 'st_flags'): + _readable_metakeys.add('flags') + if hasattr(os, 'listxattr'): + _readable_metakeys.add('xattrs') + _readable_metakeys = frozenset(_readable_metakeys) + + def _read_metadata(self, metakeys, *, follow_symlinks=True): + metadata = {} + if 'mode' in metakeys or 'times_ns' in metakeys or 'flags' in metakeys: + st = self._path.stat(follow_symlinks=follow_symlinks) + if 'mode' in metakeys: + metadata['mode'] = S_IMODE(st.st_mode) + if 'times_ns' in metakeys: + metadata['times_ns'] = st.st_atime_ns, st.st_mtime_ns + if 'flags' in metakeys: + metadata['flags'] = st.st_flags + if 'xattrs' in metakeys: + try: + metadata['xattrs'] = [ + (attr, os.getxattr(self._path, attr, follow_symlinks=follow_symlinks)) + for attr in os.listxattr(self._path, follow_symlinks=follow_symlinks)] + except OSError as err: + if err.errno not in (EPERM, ENOTSUP, ENODATA, EINVAL, EACCES): + raise + return metadata + + +class _LocalCopyWriter(CopyWriter): + """This object implements the "write" part of copying local paths. Don't + try to construct it yourself. + """ + __slots__ = () + + _writable_metakeys = _LocalCopyReader._readable_metakeys + + def _write_metadata(self, metadata, *, follow_symlinks=True): + def _nop(*args, ns=None, follow_symlinks=None): + pass + + if follow_symlinks: + # use the real function if it exists + def lookup(name): + return getattr(os, name, _nop) + else: + # use the real function only if it exists + # *and* it supports follow_symlinks + def lookup(name): + fn = getattr(os, name, _nop) + if fn in os.supports_follow_symlinks: + return fn + return _nop + + times_ns = metadata.get('times_ns') + if times_ns is not None: + lookup("utime")(self._path, ns=times_ns, follow_symlinks=follow_symlinks) + # We must copy extended attributes before the file is (potentially) + # chmod()'ed read-only, otherwise setxattr() will error with -EACCES. + xattrs = metadata.get('xattrs') + if xattrs is not None: + for attr, value in xattrs: + try: + os.setxattr(self._path, attr, value, follow_symlinks=follow_symlinks) + except OSError as e: + if e.errno not in (EPERM, ENOTSUP, ENODATA, EINVAL, EACCES): + raise + mode = metadata.get('mode') + if mode is not None: + try: + lookup("chmod")(self._path, mode, follow_symlinks=follow_symlinks) + except NotImplementedError: + # if we got a NotImplementedError, it's because + # * follow_symlinks=False, + # * lchown() is unavailable, and + # * either + # * fchownat() is unavailable or + # * fchownat() doesn't implement AT_SYMLINK_NOFOLLOW. + # (it returned ENOSUP.) + # therefore we're out of options--we simply cannot chown the + # symlink. give up, suppress the error. + # (which is what shutil always did in this circumstance.) + pass + flags = metadata.get('flags') + if flags is not None: + try: + lookup("chflags")(self._path, flags, follow_symlinks=follow_symlinks) + except OSError as why: + if why.errno not in (EOPNOTSUPP, ENOTSUP): + raise + + if copyfile: + # Use fast OS routine for local file copying where available. + def _create_file(self, source, metakeys): + """Copy the given file to the given target.""" + try: + source = os.fspath(source) + except TypeError: + if not isinstance(source, WritablePath): + raise + super()._create_file(source, metakeys) + else: + copyfile(source, os.fspath(self._path)) + + if os.name == 'nt': + # Windows: symlink target might not exist yet if we're copying several + # files, so ensure we pass is_dir to os.symlink(). + def _create_symlink(self, source, metakeys): + """Copy the given symlink to the given target.""" + self._path.symlink_to(source.readlink(), source.is_dir()) + if metakeys: + metadata = source._copy_reader._read_metadata(metakeys, follow_symlinks=False) + if metadata: + self._write_metadata(metadata, follow_symlinks=False) + + def _ensure_different_file(self, source): + """ + Raise OSError(EINVAL) if both paths refer to the same file. + """ + try: + if not self._path.samefile(source): + return + except (OSError, ValueError): + return + err = OSError(EINVAL, "Source and target are the same file") + err.filename = str(source) + err.filename2 = str(self._path) + raise err + + +class PurePath(JoinablePath): + """Base class for manipulating paths without I/O. + + PurePath represents a filesystem path and offers operations which + don't imply any actual filesystem I/O. Depending on your system, + instantiating a PurePath will return either a PurePosixPath or a + PureWindowsPath object. You can also instantiate either of these classes + directly, regardless of your system. + """ + + __slots__ = ( + # The `_raw_paths` slot stores unjoined string paths. This is set in + # the `__init__()` method. + '_raw_paths', + + # The `_drv`, `_root` and `_tail_cached` slots store parsed and + # normalized parts of the path. They are set when any of the `drive`, + # `root` or `_tail` properties are accessed for the first time. The + # three-part division corresponds to the result of + # `os.path.splitroot()`, except that the tail is further split on path + # separators (i.e. it is a list of strings), and that the root and + # tail are normalized. + '_drv', '_root', '_tail_cached', + + # The `_str` slot stores the string representation of the path, + # computed from the drive, root and tail when `__str__()` is called + # for the first time. It's used to implement `_str_normcase` + '_str', + + # The `_str_normcase_cached` slot stores the string path with + # normalized case. It is set when the `_str_normcase` property is + # accessed for the first time. It's used to implement `__eq__()` + # `__hash__()`, and `_parts_normcase` + '_str_normcase_cached', + + # The `_parts_normcase_cached` slot stores the case-normalized + # string path after splitting on path separators. It's set when the + # `_parts_normcase` property is accessed for the first time. It's used + # to implement comparison methods like `__lt__()`. + '_parts_normcase_cached', + + # The `_hash` slot stores the hash of the case-normalized string + # path. It's set when `__hash__()` is called for the first time. + '_hash', + ) + parser = os.path + + def __new__(cls, *args, **kwargs): + """Construct a PurePath from one or several strings and or existing + PurePath objects. The strings and path objects are combined so as + to yield a canonicalized path, which is incorporated into the + new PurePath object. + """ + if cls is PurePath: + cls = PureWindowsPath if os.name == 'nt' else PurePosixPath + return object.__new__(cls) + + def __init__(self, *args): + paths = [] + for arg in args: + if isinstance(arg, PurePath): + if arg.parser is not self.parser: + # GH-103631: Convert separators for backwards compatibility. + paths.append(arg.as_posix()) + else: + paths.extend(arg._raw_paths) + else: + try: + path = os.fspath(arg) + except TypeError: + path = arg + if not isinstance(path, str): + raise TypeError( + "argument should be a str or an os.PathLike " + "object where __fspath__ returns a str, " + f"not {type(path).__name__!r}") + paths.append(path) + self._raw_paths = paths + + def with_segments(self, *pathsegments): + """Construct a new path object from any number of path-like objects. + Subclasses may override this method to customize how new path objects + are created from methods like `iterdir()`. + """ + return type(self)(*pathsegments) + + def joinpath(self, *pathsegments): + """Combine this path with one or several arguments, and return a + new path representing either a subpath (if all arguments are relative + paths) or a totally different path (if one of the arguments is + anchored). + """ + return self.with_segments(self, *pathsegments) + + def __truediv__(self, key): + try: + return self.with_segments(self, key) + except TypeError: + return NotImplemented + + def __rtruediv__(self, key): + try: + return self.with_segments(key, self) + except TypeError: + return NotImplemented + + def __reduce__(self): + return self.__class__, tuple(self._raw_paths) + + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + + def __fspath__(self): + return str(self) + + def __bytes__(self): + """Return the bytes representation of the path. This is only + recommended to use under Unix.""" + return os.fsencode(self) + + @property + def _str_normcase(self): + # String with normalized case, for hashing and equality checks + try: + return self._str_normcase_cached + except AttributeError: + if self.parser is posixpath: + self._str_normcase_cached = str(self) + else: + self._str_normcase_cached = str(self).lower() + return self._str_normcase_cached + + def __hash__(self): + try: + return self._hash + except AttributeError: + self._hash = hash(self._str_normcase) + return self._hash + + def __eq__(self, other): + if not isinstance(other, PurePath): + return NotImplemented + return self._str_normcase == other._str_normcase and self.parser is other.parser + + @property + def _parts_normcase(self): + # Cached parts with normalized case, for comparisons. + try: + return self._parts_normcase_cached + except AttributeError: + self._parts_normcase_cached = self._str_normcase.split(self.parser.sep) + return self._parts_normcase_cached + + def __lt__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase < other._parts_normcase + + def __le__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase <= other._parts_normcase + + def __gt__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase > other._parts_normcase + + def __ge__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase >= other._parts_normcase + + def __str__(self): + """Return the string representation of the path, suitable for + passing to system calls.""" + try: + return self._str + except AttributeError: + self._str = self._format_parsed_parts(self.drive, self.root, + self._tail) or '.' + return self._str + + @classmethod + def _format_parsed_parts(cls, drv, root, tail): + if drv or root: + return drv + root + cls.parser.sep.join(tail) + elif tail and cls.parser.splitdrive(tail[0])[0]: + tail = ['.'] + tail + return cls.parser.sep.join(tail) + + def _from_parsed_parts(self, drv, root, tail): + path = self._from_parsed_string(self._format_parsed_parts(drv, root, tail)) + path._drv = drv + path._root = root + path._tail_cached = tail + return path + + def _from_parsed_string(self, path_str): + path = self.with_segments(path_str) + path._str = path_str or '.' + return path + + @classmethod + def _parse_path(cls, path): + if not path: + return '', '', [] + sep = cls.parser.sep + altsep = cls.parser.altsep + if altsep: + path = path.replace(altsep, sep) + drv, root, rel = cls.parser.splitroot(path) + if not root and drv.startswith(sep) and not drv.endswith(sep): + drv_parts = drv.split(sep) + if len(drv_parts) == 4 and drv_parts[2] not in '?.': + # e.g. //server/share + root = sep + elif len(drv_parts) == 6: + # e.g. //?/unc/server/share + root = sep + return drv, root, [x for x in rel.split(sep) if x and x != '.'] + + @classmethod + def _parse_pattern(cls, pattern): + """Parse a glob pattern to a list of parts. This is much like + _parse_path, except: + + - Rather than normalizing and returning the drive and root, we raise + NotImplementedError if either are present. + - If the path has no real parts, we raise ValueError. + - If the path ends in a slash, then a final empty part is added. + """ + drv, root, rel = cls.parser.splitroot(pattern) + if root or drv: + raise NotImplementedError("Non-relative patterns are unsupported") + sep = cls.parser.sep + altsep = cls.parser.altsep + if altsep: + rel = rel.replace(altsep, sep) + parts = [x for x in rel.split(sep) if x and x != '.'] + if not parts: + raise ValueError(f"Unacceptable pattern: {str(pattern)!r}") + elif rel.endswith(sep): + # GH-65238: preserve trailing slash in glob patterns. + parts.append('') + return parts + + def as_posix(self): + """Return the string representation of the path with forward (/) + slashes.""" + return str(self).replace(self.parser.sep, '/') + + @property + def _raw_path(self): + paths = self._raw_paths + if len(paths) == 1: + return paths[0] + elif paths: + # Join path segments from the initializer. + path = self.parser.join(*paths) + # Cache the joined path. + paths.clear() + paths.append(path) + return path + else: + paths.append('') + return '' + + @property + def drive(self): + """The drive prefix (letter or UNC path), if any.""" + try: + return self._drv + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._drv + + @property + def root(self): + """The root of the path, if any.""" + try: + return self._root + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._root + + @property + def _tail(self): + try: + return self._tail_cached + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._tail_cached + + @property + def anchor(self): + """The concatenation of the drive and root, or ''.""" + return self.drive + self.root + + @property + def parts(self): + """An object providing sequence-like access to the + components in the filesystem path.""" + if self.drive or self.root: + return (self.drive + self.root,) + tuple(self._tail) + else: + return tuple(self._tail) + + @property + def parent(self): + """The logical parent of the path.""" + drv = self.drive + root = self.root + tail = self._tail + if not tail: + return self + return self._from_parsed_parts(drv, root, tail[:-1]) + + @property + def parents(self): + """A sequence of this path's logical parents.""" + # The value of this property should not be cached on the path object, + # as doing so would introduce a reference cycle. + return _PathParents(self) + + @property + def name(self): + """The final path component, if any.""" + tail = self._tail + if not tail: + return '' + return tail[-1] + + def with_name(self, name): + """Return a new path with the file name changed.""" + p = self.parser + if not name or p.sep in name or (p.altsep and p.altsep in name) or name == '.': + raise ValueError(f"Invalid name {name!r}") + tail = self._tail.copy() + if not tail: + raise ValueError(f"{self!r} has an empty name") + tail[-1] = name + return self._from_parsed_parts(self.drive, self.root, tail) + + @property + def stem(self): + """The final path component, minus its last suffix.""" + name = self.name + i = name.rfind('.') + if i != -1: + stem = name[:i] + # Stem must contain at least one non-dot character. + if stem.lstrip('.'): + return stem + return name + + @property + def suffix(self): + """ + The final component's last suffix, if any. + + This includes the leading period. For example: '.txt' + """ + name = self.name.lstrip('.') + i = name.rfind('.') + if i != -1: + return name[i:] + return '' + + @property + def suffixes(self): + """ + A list of the final component's suffixes, if any. + + These include the leading periods. For example: ['.tar', '.gz'] + """ + return ['.' + ext for ext in self.name.lstrip('.').split('.')[1:]] + + def relative_to(self, other, *, walk_up=False): + """Return the relative path to another path identified by the passed + arguments. If the operation is not possible (because this is not + related to the other path), raise ValueError. + + The *walk_up* parameter controls whether `..` may be used to resolve + the path. + """ + if not isinstance(other, PurePath): + other = self.with_segments(other) + for step, path in enumerate(chain([other], other.parents)): + if path == self or path in self.parents: + break + elif not walk_up: + raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") + elif path.name == '..': + raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") + else: + raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") + parts = ['..'] * step + self._tail[len(path._tail):] + return self._from_parsed_parts('', '', parts) + + def is_relative_to(self, other): + """Return True if the path is relative to another path or False. + """ + if not isinstance(other, PurePath): + other = self.with_segments(other) + return other == self or other in self.parents + + def is_absolute(self): + """True if the path is absolute (has both a root and, if applicable, + a drive).""" + if self.parser is posixpath: + # Optimization: work with raw paths on POSIX. + for path in self._raw_paths: + if path.startswith('/'): + return True + return False + return self.parser.isabs(self) + + def is_reserved(self): + """Return True if the path contains one of the special names reserved + by the system, if any.""" + import warnings + msg = ("pathlib.PurePath.is_reserved() is deprecated and scheduled " + "for removal in Python 3.15. Use os.path.isreserved() to " + "detect reserved paths on Windows.") + warnings.warn(msg, DeprecationWarning, stacklevel=2) + if self.parser is ntpath: + return self.parser.isreserved(self) + return False + + def as_uri(self): + """Return the path as a URI.""" + if not self.is_absolute(): + raise ValueError("relative path can't be expressed as a file URI") + + drive = self.drive + if len(drive) == 2 and drive[1] == ':': + # It's a path on a local drive => 'file:///c:/a/b' + prefix = 'file:///' + drive + path = self.as_posix()[2:] + elif drive: + # It's a path on a network drive => 'file://host/share/a/b' + prefix = 'file:' + path = self.as_posix() + else: + # It's a posix path => 'file:///etc/hosts' + prefix = 'file://' + path = str(self) + from urllib.parse import quote_from_bytes + return prefix + quote_from_bytes(os.fsencode(path)) + + def full_match(self, pattern, *, case_sensitive=None): + """ + Return True if this path matches the given glob-style pattern. The + pattern is matched against the entire path. + """ + if not isinstance(pattern, PurePath): + pattern = self.with_segments(pattern) + if case_sensitive is None: + case_sensitive = self.parser is posixpath + + # The string representation of an empty path is a single dot ('.'). Empty + # paths shouldn't match wildcards, so we change it to the empty string. + path = str(self) if self.parts else '' + pattern = str(pattern) if pattern.parts else '' + globber = _StringGlobber(self.parser.sep, case_sensitive, recursive=True) + return globber.compile(pattern)(path) is not None + + def match(self, path_pattern, *, case_sensitive=None): + """ + Return True if this path matches the given pattern. If the pattern is + relative, matching is done from the right; otherwise, the entire path + is matched. The recursive wildcard '**' is *not* supported by this + method. + """ + if not isinstance(path_pattern, PurePath): + path_pattern = self.with_segments(path_pattern) + if case_sensitive is None: + case_sensitive = self.parser is posixpath + path_parts = self.parts[::-1] + pattern_parts = path_pattern.parts[::-1] + if not pattern_parts: + raise ValueError("empty pattern") + if len(path_parts) < len(pattern_parts): + return False + if len(path_parts) > len(pattern_parts) and path_pattern.anchor: + return False + globber = _StringGlobber(self.parser.sep, case_sensitive) + for path_part, pattern_part in zip(path_parts, pattern_parts): + match = globber.compile(pattern_part) + if match(path_part) is None: + return False + return True + +# Subclassing os.PathLike makes isinstance() checks slower, +# which in turn makes Path construction slower. Register instead! +os.PathLike.register(PurePath) + + +class PurePosixPath(PurePath): + """PurePath subclass for non-Windows systems. + + On a POSIX system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + parser = posixpath + __slots__ = () + + +class PureWindowsPath(PurePath): + """PurePath subclass for Windows systems. + + On a Windows system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + parser = ntpath + __slots__ = () + + +class Path(WritablePath, ReadablePath, PurePath): + """PurePath subclass that can make system calls. + + Path represents a filesystem path but unlike PurePath, also offers + methods to do system calls on path objects. Depending on your system, + instantiating a Path will return either a PosixPath or a WindowsPath + object. You can also instantiate a PosixPath or WindowsPath directly, + but cannot instantiate a WindowsPath on a POSIX system or vice versa. + """ + __slots__ = () + + def __new__(cls, *args, **kwargs): + if cls is Path: + cls = WindowsPath if os.name == 'nt' else PosixPath + return object.__new__(cls) + + def stat(self, *, follow_symlinks=True): + """ + Return the result of the stat() system call on this path, like + os.stat() does. + """ + return os.stat(self, follow_symlinks=follow_symlinks) + + def lstat(self): + """ + Like stat(), except if the path points to a symlink, the symlink's + status information is returned, rather than its target's. + """ + return os.lstat(self) + + def exists(self, *, follow_symlinks=True): + """ + Whether this path exists. + + This method normally follows symlinks; to check whether a symlink exists, + add the argument follow_symlinks=False. + """ + if follow_symlinks: + return os.path.exists(self) + return os.path.lexists(self) + + def is_dir(self, *, follow_symlinks=True): + """ + Whether this path is a directory. + """ + if follow_symlinks: + return os.path.isdir(self) + try: + return S_ISDIR(self.stat(follow_symlinks=follow_symlinks).st_mode) + except (OSError, ValueError): + return False + + def is_file(self, *, follow_symlinks=True): + """ + Whether this path is a regular file (also True for symlinks pointing + to regular files). + """ + if follow_symlinks: + return os.path.isfile(self) + try: + return S_ISREG(self.stat(follow_symlinks=follow_symlinks).st_mode) + except (OSError, ValueError): + return False + + def is_mount(self): + """ + Check if this path is a mount point + """ + return os.path.ismount(self) + + def is_symlink(self): + """ + Whether this path is a symbolic link. + """ + return os.path.islink(self) + + def is_junction(self): + """ + Whether this path is a junction. + """ + return os.path.isjunction(self) + + def is_block_device(self): + """ + Whether this path is a block device. + """ + try: + return S_ISBLK(self.stat().st_mode) + except (OSError, ValueError): + return False + + def is_char_device(self): + """ + Whether this path is a character device. + """ + try: + return S_ISCHR(self.stat().st_mode) + except (OSError, ValueError): + return False + + def is_fifo(self): + """ + Whether this path is a FIFO. + """ + try: + return S_ISFIFO(self.stat().st_mode) + except (OSError, ValueError): + return False + + def is_socket(self): + """ + Whether this path is a socket. + """ + try: + return S_ISSOCK(self.stat().st_mode) + except (OSError, ValueError): + return False + + def samefile(self, other_path): + """Return whether other_path is the same or not as this file + (as returned by os.path.samefile()). + """ + st = self.stat() + try: + other_st = other_path.stat() + except AttributeError: + other_st = self.with_segments(other_path).stat() + return (st.st_ino == other_st.st_ino and + st.st_dev == other_st.st_dev) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + """ + Open the file pointed to by this path and return a file object, as + the built-in open() function does. + """ + if "b" not in mode: + encoding = io.text_encoding(encoding) + return io.open(self, mode, buffering, encoding, errors, newline) + + def read_bytes(self): + """ + Open the file in bytes mode, read it, and close the file. + """ + with self.open(mode='rb', buffering=0) as f: + return f.read() + + def read_text(self, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, read it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + with self.open(mode='r', encoding=encoding, errors=errors, newline=newline) as f: + return f.read() + + def write_bytes(self, data): + """ + Open the file in bytes mode, write to it, and close the file. + """ + # type-check for the buffer interface before truncating the file + view = memoryview(data) + with self.open(mode='wb') as f: + return f.write(view) + + def write_text(self, data, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, write to it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + if not isinstance(data, str): + raise TypeError('data must be str, not %s' % + data.__class__.__name__) + with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: + return f.write(data) + + _remove_leading_dot = operator.itemgetter(slice(2, None)) + _remove_trailing_slash = operator.itemgetter(slice(-1)) + + def _filter_trailing_slash(self, paths): + sep = self.parser.sep + anchor_len = len(self.anchor) + for path_str in paths: + if len(path_str) > anchor_len and path_str[-1] == sep: + path_str = path_str[:-1] + yield path_str + + def _scandir(self): + """Yield os.DirEntry-like objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + return os.scandir(self) + + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + root_dir = str(self) + with os.scandir(root_dir) as scandir_it: + paths = [entry.path for entry in scandir_it] + if root_dir == '.': + paths = map(self._remove_leading_dot, paths) + return map(self._from_parsed_string, paths) + + def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): + """Iterate over this subtree and yield all existing files (of any + kind, including directories) matching the given relative pattern. + """ + sys.audit("pathlib.Path.glob", self, pattern) + if case_sensitive is None: + case_sensitive = self.parser is posixpath + case_pedantic = False + else: + # The user has expressed a case sensitivity choice, but we don't + # know the case sensitivity of the underlying filesystem, so we + # must use scandir() for everything, including non-wildcard parts. + case_pedantic = True + parts = self._parse_pattern(pattern) + recursive = True if recurse_symlinks else _no_recurse_symlinks + globber = _StringGlobber(self.parser.sep, case_sensitive, case_pedantic, recursive) + select = globber.selector(parts[::-1]) + root = str(self) + paths = select(root) + + # Normalize results + if root == '.': + paths = map(self._remove_leading_dot, paths) + if parts[-1] == '': + paths = map(self._remove_trailing_slash, paths) + elif parts[-1] == '**': + paths = self._filter_trailing_slash(paths) + paths = map(self._from_parsed_string, paths) + return paths + + def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): + """Recursively yield all existing files (of any kind, including + directories) matching the given relative pattern, anywhere in + this subtree. + """ + sys.audit("pathlib.Path.rglob", self, pattern) + pattern = self.parser.join('**', pattern) + return self.glob(pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks) + + def walk(self, top_down=True, on_error=None, follow_symlinks=False): + """Walk the directory tree from this directory, similar to os.walk().""" + sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) + root_dir = str(self) + if not follow_symlinks: + follow_symlinks = os._walk_symlinks_as_files + results = os.walk(root_dir, top_down, on_error, follow_symlinks) + for path_str, dirnames, filenames in results: + if root_dir == '.': + path_str = path_str[2:] + yield self._from_parsed_string(path_str), dirnames, filenames + + def absolute(self): + """Return an absolute version of this path + No normalization or symlink resolution is performed. + + Use resolve() to resolve symlinks and remove '..' segments. + """ + if self.is_absolute(): + return self + if self.root: + drive = os.path.splitroot(os.getcwd())[0] + return self._from_parsed_parts(drive, self.root, self._tail) + if self.drive: + # There is a CWD on each drive-letter drive. + cwd = os.path.abspath(self.drive) + else: + cwd = os.getcwd() + if not self._tail: + # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). + # We pass only one argument to with_segments() to avoid the cost + # of joining, and we exploit the fact that getcwd() returns a + # fully-normalized string by storing it in _str. This is used to + # implement Path.cwd(). + return self._from_parsed_string(cwd) + drive, root, rel = os.path.splitroot(cwd) + if not rel: + return self._from_parsed_parts(drive, root, self._tail) + tail = rel.split(self.parser.sep) + tail.extend(self._tail) + return self._from_parsed_parts(drive, root, tail) + + @classmethod + def cwd(cls): + """Return a new path pointing to the current working directory.""" + cwd = os.getcwd() + path = cls(cwd) + path._str = cwd # getcwd() returns a normalized path + return path + + def resolve(self, strict=False): + """ + Make the path absolute, resolving all symlinks on the way and also + normalizing it. + """ + + return self.with_segments(os.path.realpath(self, strict=strict)) + + if pwd: + def owner(self, *, follow_symlinks=True): + """ + Return the login name of the file owner. + """ + uid = self.stat(follow_symlinks=follow_symlinks).st_uid + return pwd.getpwuid(uid).pw_name + else: + def owner(self, *, follow_symlinks=True): + """ + Return the login name of the file owner. + """ + f = f"{type(self).__name__}.owner()" + raise UnsupportedOperation(f"{f} is unsupported on this system") + + if grp: + def group(self, *, follow_symlinks=True): + """ + Return the group name of the file gid. + """ + gid = self.stat(follow_symlinks=follow_symlinks).st_gid + return grp.getgrgid(gid).gr_name + else: + def group(self, *, follow_symlinks=True): + """ + Return the group name of the file gid. + """ + f = f"{type(self).__name__}.group()" + raise UnsupportedOperation(f"{f} is unsupported on this system") + + if hasattr(os, "readlink"): + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + return self.with_segments(os.readlink(self)) + else: + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + f = f"{type(self).__name__}.readlink()" + raise UnsupportedOperation(f"{f} is unsupported on this system") + + def touch(self, mode=0o666, exist_ok=True): + """ + Create this file with the given access mode, if it doesn't exist. + """ + + if exist_ok: + # First try to bump modification time + # Implementation note: GNU touch uses the UTIME_NOW option of + # the utimensat() / futimens() functions. + try: + os.utime(self, None) + except OSError: + # Avoid exception chaining + pass + else: + return + flags = os.O_CREAT | os.O_WRONLY + if not exist_ok: + flags |= os.O_EXCL + fd = os.open(self, flags, mode) + os.close(fd) + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ + try: + os.mkdir(self, mode) + except FileNotFoundError: + if not parents or self.parent == self: + raise + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) + except OSError: + # Cannot rely on checking for EEXIST, since the operating system + # could give priority to other errors like EACCES or EROFS + if not exist_ok or not self.is_dir(): + raise + + def chmod(self, mode, *, follow_symlinks=True): + """ + Change the permissions of the path, like os.chmod(). + """ + os.chmod(self, mode, follow_symlinks=follow_symlinks) + + def lchmod(self, mode): + """ + Like chmod(), except if the path points to a symlink, the symlink's + permissions are changed, rather than its target's. + """ + self.chmod(mode, follow_symlinks=False) + + def unlink(self, missing_ok=False): + """ + Remove this file or link. + If the path is a directory, use rmdir() instead. + """ + try: + os.unlink(self) + except FileNotFoundError: + if not missing_ok: + raise + + def rmdir(self): + """ + Remove this directory. The directory must be empty. + """ + os.rmdir(self) + + def _delete(self): + """ + Delete this file or directory (including all sub-directories). + """ + if self.is_symlink() or self.is_junction(): + self.unlink() + elif self.is_dir(): + # Lazy import to improve module import time + import shutil + shutil.rmtree(self) + else: + self.unlink() + + def rename(self, target): + """ + Rename this path to the target path. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.rename(self, target) + return self.with_segments(target) + + def replace(self, target): + """ + Rename this path to the target path, overwriting if that path exists. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.replace(self, target) + return self.with_segments(target) + + _copy_reader = property(_LocalCopyReader) + _copy_writer = property(_LocalCopyWriter) + + def move(self, target): + """ + Recursively move this file or directory tree to the given destination. + """ + # Use os.replace() if the target is os.PathLike and on the same FS. + try: + target_str = os.fspath(target) + except TypeError: + pass + else: + if not hasattr(target, '_copy_writer'): + target = self.with_segments(target_str) + target._copy_writer._ensure_different_file(self) + try: + os.replace(self, target_str) + return target + except OSError as err: + if err.errno != EXDEV: + raise + # Fall back to copy+delete. + target = self.copy(target, follow_symlinks=False, preserve_metadata=True) + self._delete() + return target + + def move_into(self, target_dir): + """ + Move this file or directory tree into the given existing directory. + """ + name = self.name + if not name: + raise ValueError(f"{self!r} has an empty name") + elif hasattr(target_dir, '_copy_writer'): + target = target_dir / name + else: + target = self.with_segments(target_dir, name) + return self.move(target) + + if hasattr(os, "symlink"): + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + os.symlink(target, self, target_is_directory) + else: + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + f = f"{type(self).__name__}.symlink_to()" + raise UnsupportedOperation(f"{f} is unsupported on this system") + + if hasattr(os, "link"): + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + os.link(target, self) + else: + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + f = f"{type(self).__name__}.hardlink_to()" + raise UnsupportedOperation(f"{f} is unsupported on this system") + + def expanduser(self): + """ Return a new path with expanded ~ and ~user constructs + (as returned by os.path.expanduser) + """ + if (not (self.drive or self.root) and + self._tail and self._tail[0][:1] == '~'): + homedir = os.path.expanduser(self._tail[0]) + if homedir[:1] == "~": + raise RuntimeError("Could not determine home directory.") + drv, root, tail = self._parse_path(homedir) + return self._from_parsed_parts(drv, root, tail + self._tail[1:]) + + return self + + @classmethod + def home(cls): + """Return a new path pointing to expanduser('~'). + """ + homedir = os.path.expanduser("~") + if homedir == "~": + raise RuntimeError("Could not determine home directory.") + return cls(homedir) + + @classmethod + def from_uri(cls, uri): + """Return a new path from the given 'file' URI.""" + if not uri.startswith('file:'): + raise ValueError(f"URI does not start with 'file:': {uri!r}") + path = uri[5:] + if path[:3] == '///': + # Remove empty authority + path = path[2:] + elif path[:12] == '//localhost/': + # Remove 'localhost' authority + path = path[11:] + if path[:3] == '///' or (path[:1] == '/' and path[2:3] in ':|'): + # Remove slash before DOS device/UNC path + path = path[1:] + if path[1:2] == '|': + # Replace bar with colon in DOS drive + path = path[:1] + ':' + path[2:] + from urllib.parse import unquote_to_bytes + path = cls(os.fsdecode(unquote_to_bytes(path))) + if not path.is_absolute(): + raise ValueError(f"URI is not absolute: {uri!r}") + return path + + +class PosixPath(Path, PurePosixPath): + """Path subclass for non-Windows systems. + + On a POSIX system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name == 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") + +class WindowsPath(Path, PureWindowsPath): + """Path subclass for Windows systems. + + On a Windows system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name != 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") diff --git a/Lib/pathlib/_os.py b/Lib/pathlib/_os.py new file mode 100644 index 00000000000000..57bcaf3d680138 --- /dev/null +++ b/Lib/pathlib/_os.py @@ -0,0 +1,164 @@ +""" +Low-level OS functionality wrappers used by pathlib. +""" + +from errno import * +import os +import sys +try: + import fcntl +except ImportError: + fcntl = None +try: + import posix +except ImportError: + posix = None +try: + import _winapi +except ImportError: + _winapi = None + + +def _get_copy_blocksize(infd): + """Determine blocksize for fastcopying on Linux. + Hopefully the whole file will be copied in a single call. + The copying itself should be performed in a loop 'till EOF is + reached (0 return) so a blocksize smaller or bigger than the actual + file size should not make any difference, also in case the file + content changes while being copied. + """ + try: + blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8 MiB + except OSError: + blocksize = 2 ** 27 # 128 MiB + # On 32-bit architectures truncate to 1 GiB to avoid OverflowError, + # see gh-82500. + if sys.maxsize < 2 ** 32: + blocksize = min(blocksize, 2 ** 30) + return blocksize + + +if fcntl and hasattr(fcntl, 'FICLONE'): + def _ficlone(source_fd, target_fd): + """ + Perform a lightweight copy of two files, where the data blocks are + copied only when modified. This is known as Copy on Write (CoW), + instantaneous copy or reflink. + """ + fcntl.ioctl(target_fd, fcntl.FICLONE, source_fd) +else: + _ficlone = None + + +if posix and hasattr(posix, '_fcopyfile'): + def _fcopyfile(source_fd, target_fd): + """ + Copy a regular file content using high-performance fcopyfile(3) + syscall (macOS). + """ + posix._fcopyfile(source_fd, target_fd, posix._COPYFILE_DATA) +else: + _fcopyfile = None + + +if hasattr(os, 'copy_file_range'): + def _copy_file_range(source_fd, target_fd): + """ + Copy data from one regular mmap-like fd to another by using a + high-performance copy_file_range(2) syscall that gives filesystems + an opportunity to implement the use of reflinks or server-side + copy. + This should work on Linux >= 4.5 only. + """ + blocksize = _get_copy_blocksize(source_fd) + offset = 0 + while True: + sent = os.copy_file_range(source_fd, target_fd, blocksize, + offset_dst=offset) + if sent == 0: + break # EOF + offset += sent +else: + _copy_file_range = None + + +if hasattr(os, 'sendfile'): + def _sendfile(source_fd, target_fd): + """Copy data from one regular mmap-like fd to another by using + high-performance sendfile(2) syscall. + This should work on Linux >= 2.6.33 only. + """ + blocksize = _get_copy_blocksize(source_fd) + offset = 0 + while True: + sent = os.sendfile(target_fd, source_fd, offset, blocksize) + if sent == 0: + break # EOF + offset += sent +else: + _sendfile = None + + +if _winapi and hasattr(_winapi, 'CopyFile2'): + def copyfile(source, target): + """ + Copy from one file to another using CopyFile2 (Windows only). + """ + _winapi.CopyFile2(source, target, 0) +else: + copyfile = None + + +def copyfileobj(source_f, target_f): + """ + Copy data from file-like object source_f to file-like object target_f. + """ + try: + source_fd = source_f.fileno() + target_fd = target_f.fileno() + except Exception: + pass # Fall through to generic code. + else: + try: + # Use OS copy-on-write where available. + if _ficlone: + try: + _ficlone(source_fd, target_fd) + return + except OSError as err: + if err.errno not in (EBADF, EOPNOTSUPP, ETXTBSY, EXDEV): + raise err + + # Use OS copy where available. + if _fcopyfile: + try: + _fcopyfile(source_fd, target_fd) + return + except OSError as err: + if err.errno not in (EINVAL, ENOTSUP): + raise err + if _copy_file_range: + try: + _copy_file_range(source_fd, target_fd) + return + except OSError as err: + if err.errno not in (ETXTBSY, EXDEV): + raise err + if _sendfile: + try: + _sendfile(source_fd, target_fd) + return + except OSError as err: + if err.errno != ENOTSOCK: + raise err + except OSError as err: + # Produce more useful error messages. + err.filename = source_f.name + err.filename2 = target_f.name + raise err + + # Last resort: copy with fileobj read() and write(). + read_source = source_f.read + write_target = target_f.write + while buf := read_source(1024 * 1024): + write_target(buf) diff --git a/Lib/pathlib/_types.py b/Lib/pathlib/_types.py new file mode 100644 index 00000000000000..84032bb5b4ff1a --- /dev/null +++ b/Lib/pathlib/_types.py @@ -0,0 +1,19 @@ +""" +Protocols for supporting classes in pathlib. +""" +from typing import Protocol, runtime_checkable + + +@runtime_checkable +class Parser(Protocol): + """Protocol for path parsers, which do low-level path manipulation. + + Path parsers provide a subset of the os.path API, specifically those + functions needed to provide JoinablePath functionality. Each JoinablePath + subclass references its path parser via a 'parser' class attribute. + """ + + sep: str + def split(self, path: str) -> tuple[str, str]: ... + def splitext(self, path: str) -> tuple[str, str]: ... + def normcase(self, path: str) -> str: ... diff --git a/Lib/pdb.py b/Lib/pdb.py old mode 100755 new mode 100644 index 519c1ccd5640a1..beef74d792250b --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """ The Python Debugger Pdb ======================= @@ -77,17 +75,21 @@ import code import glob import token +import types import codeop import pprint import signal import inspect +import textwrap import tokenize -import functools +import itertools import traceback import linecache +import _colorize from contextlib import contextmanager -from typing import Union +from rlcompleter import Completer +from types import CodeType class Restart(Exception): @@ -116,13 +118,16 @@ def find_first_executable_line(code): return code.co_firstlineno def find_function(funcname, filename): - cre = re.compile(r'def\s+%s\s*[(]' % re.escape(funcname)) + cre = re.compile(r'def\s+%s(\s*\[.+\])?\s*[(]' % re.escape(funcname)) try: fp = tokenize.open(filename) except OSError: - return None + lines = linecache.getlines(filename) + if not lines: + return None + fp = io.StringIO(''.join(lines)) funcdef = "" - funcstart = None + funcstart = 0 # consumer of this info expects the first line to be 1 with fp: for lineno, line in enumerate(fp, start=1): @@ -133,9 +138,12 @@ def find_function(funcname, filename): if funcdef: try: - funccode = compile(funcdef, filename, 'exec').co_consts[0] + code = compile(funcdef, filename, 'exec') except SyntaxError: continue + # We should always be able to find the code object here + funccode = next(c for c in code.co_consts if + isinstance(c, CodeType) and c.co_name == funcname) lineno_offset = find_first_executable_line(funccode) return funcname, filename, funcstart + lineno_offset - 1 return None @@ -155,52 +163,58 @@ def __repr__(self): return self -class _ScriptTarget(str): - def __new__(cls, val): - # Mutate self to be the "real path". - res = super().__new__(cls, os.path.realpath(val)) +class _ExecutableTarget: + filename: str + code: CodeType | str + namespace: dict - # Store the original path for error reporting. - res.orig = val - return res +class _ScriptTarget(_ExecutableTarget): + def __init__(self, target): + self._target = os.path.realpath(target) - def check(self): - if not os.path.exists(self): - print('Error:', self.orig, 'does not exist') + if not os.path.exists(self._target): + print(f'Error: {target} does not exist') sys.exit(1) - if os.path.isdir(self): - print('Error:', self.orig, 'is a directory') + if os.path.isdir(self._target): + print(f'Error: {target} is a directory') sys.exit(1) # If safe_path(-P) is not set, sys.path[0] is the directory # of pdb, and we should replace it with the directory of the script if not sys.flags.safe_path: - sys.path[0] = os.path.dirname(self) + sys.path[0] = os.path.dirname(self._target) + + def __repr__(self): + return self._target @property def filename(self): - return self + return self._target + + @property + def code(self): + # Open the file each time because the file may be modified + with io.open_code(self._target) as fp: + return f"exec(compile({fp.read()!r}, {self._target!r}, 'exec'))" @property def namespace(self): return dict( __name__='__main__', - __file__=self, + __file__=self._target, __builtins__=__builtins__, __spec__=None, ) - @property - def code(self): - with io.open_code(self) as fp: - return f"exec(compile({fp.read()!r}, {self!r}, 'exec'))" +class _ModuleTarget(_ExecutableTarget): + def __init__(self, target): + self._target = target -class _ModuleTarget(str): - def check(self): + import runpy try: - self._details + _, self._spec, self._code = runpy._get_module_details(self._target) except ImportError as e: print(f"ImportError: {e}") sys.exit(1) @@ -208,24 +222,54 @@ def check(self): traceback.print_exc() sys.exit(1) - @functools.cached_property - def _details(self): - import runpy - return runpy._get_module_details(self) + def __repr__(self): + return self._target @property def filename(self): - return self.code.co_filename + return self._code.co_filename @property def code(self): - name, spec, code = self._details - return code + return self._code @property - def _spec(self): - name, spec, code = self._details - return spec + def namespace(self): + return dict( + __name__='__main__', + __file__=os.path.normcase(os.path.abspath(self.filename)), + __package__=self._spec.parent, + __loader__=self._spec.loader, + __spec__=self._spec, + __builtins__=__builtins__, + ) + + +class _ZipTarget(_ExecutableTarget): + def __init__(self, target): + import runpy + + self._target = os.path.realpath(target) + sys.path.insert(0, self._target) + try: + _, self._spec, self._code = runpy._get_main_module_details() + except ImportError as e: + print(f"ImportError: {e}") + sys.exit(1) + except Exception: + traceback.print_exc() + sys.exit(1) + + def __repr__(self): + return self._target + + @property + def filename(self): + return self._code.co_filename + + @property + def code(self): + return self._code @property def namespace(self): @@ -266,8 +310,10 @@ class Pdb(bdb.Bdb, cmd.Cmd): _file_mtime_table = {} + _last_pdb_instance = None + def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, - nosigint=False, readrc=True): + nosigint=False, readrc=True, mode=None): bdb.Bdb.__init__(self, skip=skip) cmd.Cmd.__init__(self, completekey, stdin, stdout) sys.audit("pdb.Pdb") @@ -279,6 +325,7 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, self.mainpyfile = '' self._wait_for_mainpyfile = False self.tb_lineno = {} + self.mode = mode # Try to load readline if it exists try: import readline @@ -307,10 +354,6 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, pass self.commands = {} # associates a command list to breakpoint numbers - self.commands_doprompt = {} # for each bp num, tells if the prompt - # must be disp. after execing the cmd list - self.commands_silent = {} # for each bp num, tells if the stack trace - # must be disp. after execing the cmd list self.commands_defining = False # True while in the process of defining # a command list self.commands_bnum = None # The breakpoint number for which we are @@ -319,6 +362,16 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, self._chained_exceptions = tuple() self._chained_exception_index = 0 + def set_trace(self, frame=None, *, commands=None): + Pdb._last_pdb_instance = self + if frame is None: + frame = sys._getframe().f_back + + if commands is not None: + self.rcLines.extend(commands) + + super().set_trace(frame) + def sigint_handler(self, signum, frame): if self.allow_kbdint: raise KeyboardInterrupt @@ -350,12 +403,10 @@ def setup(self, f, tb): self.tb_lineno[tb.tb_frame] = lineno tb = tb.tb_next self.curframe = self.stack[self.curindex][0] - # The f_locals dictionary is updated from the actual frame - # locals whenever the .f_locals accessor is called, so we - # cache it here to ensure that modifications are not overwritten. - self.curframe_locals = self.curframe.f_locals self.set_convenience_variable(self.curframe, '_frame', self.curframe) + self._save_initial_file_mtime(self.curframe) + if self._chained_exceptions: self.set_convenience_variable( self.curframe, @@ -363,26 +414,12 @@ def setup(self, f, tb): self._chained_exceptions[self._chained_exception_index], ) - return self.execRcLines() - - # Can be executed earlier than 'setup' if desired - def execRcLines(self): - if not self.rcLines: - return - # local copy because of recursion - rcLines = self.rcLines - rcLines.reverse() - # execute every line only once - self.rcLines = [] - while rcLines: - line = rcLines.pop().strip() - if line and line[0] != '#': - if self.onecmd(line): - # if onecmd returns True, the command wants to exit - # from the interaction, save leftover rc lines - # to execute before next interaction - self.rcLines += reversed(rcLines) - return True + if self.rcLines: + self.cmdqueue = [ + line for line in self.rcLines + if line.strip() and not line.strip().startswith("#") + ] + self.rcLines = [] # Override Bdb methods @@ -398,12 +435,20 @@ def user_call(self, frame, argument_list): def user_line(self, frame): """This function is called when we stop or break at this line.""" if self._wait_for_mainpyfile: - if (self.mainpyfile != self.canonic(frame.f_code.co_filename) - or frame.f_lineno <= 0): + if (self.mainpyfile != self.canonic(frame.f_code.co_filename)): return self._wait_for_mainpyfile = False - if self.bp_commands(frame): - self.interaction(frame, None) + if self.trace_opcodes: + # GH-127321 + # We want to avoid stopping at an opcode that does not have + # an associated line number because pdb does not like it + if frame.f_lineno is None: + self.set_stepinstr() + return + self.bp_commands(frame) + self.interaction(frame, None) + + user_opcode = user_line def bp_commands(self, frame): """Call every command that was set for the current active breakpoint @@ -416,18 +461,9 @@ def bp_commands(self, frame): self.currentbp in self.commands: currentbp = self.currentbp self.currentbp = 0 - lastcmd_back = self.lastcmd - self.setup(frame, None) for line in self.commands[currentbp]: - self.onecmd(line) - self.lastcmd = lastcmd_back - if not self.commands_silent[currentbp]: - self.print_stack_entry(self.stack[self.curindex]) - if self.commands_doprompt[currentbp]: - self._cmdloop() - self.forget() - return - return 1 + self.cmdqueue.append(line) + self.cmdqueue.append(f'_pdbcmd_restore_lastcmd {self.lastcmd}') def user_return(self, frame, return_value): """This function is called when a return trap is set here.""" @@ -470,9 +506,21 @@ def _cmdloop(self): except KeyboardInterrupt: self.message('--KeyboardInterrupt--') + def _save_initial_file_mtime(self, frame): + """save the mtime of the all the files in the frame stack in the file mtime table + if they haven't been saved yet.""" + while frame: + filename = frame.f_code.co_filename + if filename not in self._file_mtime_table: + try: + self._file_mtime_table[filename] = os.path.getmtime(filename) + except Exception: + pass + frame = frame.f_back + def _validate_file_mtime(self): - """Check if the source file of the current frame has been modified since - the last time we saw it. If so, give a warning.""" + """Check if the source file of the current frame has been modified. + If so, give a warning and reset the modify time to current.""" try: filename = self.curframe.f_code.co_filename mtime = os.path.getmtime(filename) @@ -482,11 +530,11 @@ def _validate_file_mtime(self): mtime != self._file_mtime_table[filename]): self.message(f"*** WARNING: file '{filename}' was edited, " "running stale code until the program is rerun") - self._file_mtime_table[filename] = mtime + self._file_mtime_table[filename] = mtime # Called before loop, handles display expressions # Set up convenience variable containers - def preloop(self): + def _show_display(self): displaying = self.displaying.get(self.curframe) if displaying: for expr, oldvalue in displaying.items(): @@ -571,13 +619,17 @@ def interaction(self, frame, tb_or_exc): if isinstance(tb_or_exc, BaseException): assert tb is not None, "main exception must have a traceback" with self._hold_exceptions(_chained_exceptions): - if self.setup(frame, tb): - # no interaction desired at this time (happens if .pdbrc contains - # a command like "continue") - self.forget() - return - self.print_stack_entry(self.stack[self.curindex]) + self.setup(frame, tb) + # We should print the stack entry if and only if the user input + # is expected, and we should print it right before the user input. + # We achieve this by appending _pdbcmd_print_frame_status to the + # command queue. If cmdqueue is not exhausted, the user input is + # not expected and we will not print the stack entry. + self.cmdqueue.append('_pdbcmd_print_frame_status') self._cmdloop() + # If _pdbcmd_print_frame_status is not used, pop it out + if self.cmdqueue and self.cmdqueue[-1] == '_pdbcmd_print_frame_status': + self.cmdqueue.pop() self.forget() def displayhook(self, obj): @@ -589,29 +641,108 @@ def displayhook(self, obj): self.message(repr(obj)) @contextmanager - def _disable_tab_completion(self): - if self.use_rawinput and self.completekey == 'tab': - try: - import readline - except ImportError: - yield - return - try: - readline.parse_and_bind('tab: self-insert') - yield - finally: - readline.parse_and_bind('tab: complete') - else: + def _disable_command_completion(self): + completenames = self.completenames + try: + self.completenames = self.completedefault yield + finally: + self.completenames = completenames + return + + def _exec_in_closure(self, source, globals, locals): + """ Run source code in closure so code object created within source + can find variables in locals correctly + + returns True if the source is executed, False otherwise + """ + + # Determine if the source should be executed in closure. Only when the + # source compiled to multiple code objects, we should use this feature. + # Otherwise, we can just raise an exception and normal exec will be used. + + code = compile(source, "", "exec") + if not any(isinstance(const, CodeType) for const in code.co_consts): + return False + + # locals could be a proxy which does not support pop + # copy it first to avoid modifying the original locals + locals_copy = dict(locals) + + locals_copy["__pdb_eval__"] = { + "result": None, + "write_back": {} + } + + # If the source is an expression, we need to print its value + try: + compile(source, "", "eval") + except SyntaxError: + pass + else: + source = "__pdb_eval__['result'] = " + source + + # Add write-back to update the locals + source = ("try:\n" + + textwrap.indent(source, " ") + "\n" + + "finally:\n" + + " __pdb_eval__['write_back'] = locals()") + + # Build a closure source code with freevars from locals like: + # def __pdb_outer(): + # var = None + # def __pdb_scope(): # This is the code object we want to execute + # nonlocal var + # + # return __pdb_scope.__code__ + source_with_closure = ("def __pdb_outer():\n" + + "\n".join(f" {var} = None" for var in locals_copy) + "\n" + + " def __pdb_scope():\n" + + "\n".join(f" nonlocal {var}" for var in locals_copy) + "\n" + + textwrap.indent(source, " ") + "\n" + + " return __pdb_scope.__code__" + ) + + # Get the code object of __pdb_scope() + # The exec fills locals_copy with the __pdb_outer() function and we can call + # that to get the code object of __pdb_scope() + ns = {} + try: + exec(source_with_closure, {}, ns) + except Exception: + return False + code = ns["__pdb_outer"]() + + cells = tuple(types.CellType(locals_copy.get(var)) for var in code.co_freevars) + + try: + exec(code, globals, locals_copy, closure=cells) + except Exception: + return False + + # get the data we need from the statement + pdb_eval = locals_copy["__pdb_eval__"] + + # __pdb_eval__ should not be updated back to locals + pdb_eval["write_back"].pop("__pdb_eval__") + + # Write all local variables back to locals + locals.update(pdb_eval["write_back"]) + eval_result = pdb_eval["result"] + if eval_result is not None: + print(repr(eval_result)) + + return True def default(self, line): if line[:1] == '!': line = line[1:].strip() - locals = self.curframe_locals + locals = self.curframe.f_locals globals = self.curframe.f_globals try: + buffer = line if (code := codeop.compile_command(line + '\n', '', 'single')) is None: # Multi-line mode - with self._disable_tab_completion(): + with self._disable_command_completion(): buffer = line continue_prompt = "... " while (code := codeop.compile_command(buffer, '', 'single')) is None: @@ -634,6 +765,7 @@ def default(self, line): else: line = line.rstrip('\r\n') buffer += '\n' + line + self.lastcmd = buffer save_stdout = sys.stdout save_stdin = sys.stdin save_displayhook = sys.displayhook @@ -641,7 +773,8 @@ def default(self, line): sys.stdin = self.stdin sys.stdout = self.stdout sys.displayhook = self.displayhook - exec(code, globals, locals) + if not self._exec_in_closure(buffer, globals, locals): + exec(code, globals, locals) finally: sys.stdout = save_stdout sys.stdin = save_stdin @@ -657,7 +790,7 @@ def _replace_convenience_variables(self, line): if "$" not in line: return line - dollar_start = dollar_end = -1 + dollar_start = dollar_end = (-1, -1) replace_variables = [] try: for t in tokenize.generate_tokens(io.StringIO(line).readline): @@ -712,7 +845,7 @@ def precmd(self, line): if marker >= 0: # queue up everything after marker next = line[marker+2:].lstrip() - self.cmdqueue.append(next) + self.cmdqueue.insert(0, next) line = line[:marker].rstrip() # Replace all the convenience variables @@ -728,7 +861,10 @@ def onecmd(self, line): a breakpoint command list definition. """ if not self.commands_defining: - self._validate_file_mtime() + if line.startswith('_pdbcmd'): + command, arg, line = self.parseline(line) + if hasattr(self, command): + return getattr(self, command)(arg) return cmd.Cmd.onecmd(self, line) else: return self.handle_command_def(line) @@ -737,14 +873,16 @@ def handle_command_def(self, line): """Handles one command line during command list definition.""" cmd, arg, line = self.parseline(line) if not cmd: - return - if cmd == 'silent': - self.commands_silent[self.commands_bnum] = True - return # continue to handle other cmd def in the cmd list - elif cmd == 'end': - self.cmdqueue = [] - return 1 # end of cmd list + return False + if cmd == 'end': + return True # end of cmd list + elif cmd == 'EOF': + print('') + return True # end of cmd list cmdlist = self.commands[self.commands_bnum] + if cmd == 'silent': + cmdlist.append('_pdbcmd_silence_frame_status') + return False # continue to handle other cmd def in the cmd list if arg: cmdlist.append(cmd+' '+arg) else: @@ -756,10 +894,8 @@ def handle_command_def(self, line): func = self.default # one of the resuming commands if func.__name__ in self.commands_resuming: - self.commands_doprompt[self.commands_bnum] = False - self.cmdqueue = [] - return 1 - return + return True + return False # interface abstraction functions @@ -789,7 +925,10 @@ def completenames(self, text, line, begidx, endidx): if commands: return commands else: - return self._complete_expression(text, line, begidx, endidx) + expressions = self._complete_expression(text, line, begidx, endidx) + if expressions: + return expressions + return self.completedefault(text, line, begidx, endidx) def _complete_location(self, text, line, begidx, endidx): # Complete a file/module/function location for break/tbreak/clear. @@ -824,7 +963,7 @@ def _complete_expression(self, text, line, begidx, endidx): # Collect globals and locals. It is usually not really sensible to also # complete builtins, and they clutter the namespace quite heavily, so we # leave them out. - ns = {**self.curframe.f_globals, **self.curframe_locals} + ns = {**self.curframe.f_globals, **self.curframe.f_locals} if text.startswith("$"): # Complete convenience variables conv_vars = self.curframe.f_globals.get('__pdb_convenience_variables', {}) @@ -846,6 +985,35 @@ def _complete_expression(self, text, line, begidx, endidx): # Complete a simple name. return [n for n in ns.keys() if n.startswith(text)] + def completedefault(self, text, line, begidx, endidx): + if text.startswith("$"): + # Complete convenience variables + conv_vars = self.curframe.f_globals.get('__pdb_convenience_variables', {}) + return [f"${name}" for name in conv_vars if name.startswith(text[1:])] + + # Use rlcompleter to do the completion + state = 0 + matches = [] + completer = Completer(self.curframe.f_globals | self.curframe.f_locals) + while (match := completer.complete(text, state)) is not None: + matches.append(match) + state += 1 + return matches + + # Pdb meta commands, only intended to be used internally by pdb + + def _pdbcmd_print_frame_status(self, arg): + self.print_stack_trace(0) + self._validate_file_mtime() + self._show_display() + + def _pdbcmd_silence_frame_status(self, arg): + if self.cmdqueue and self.cmdqueue[-1] == '_pdbcmd_print_frame_status': + self.cmdqueue.pop() + + def _pdbcmd_restore_lastcmd(self, arg): + self.lastcmd = arg + # Command definitions, called by cmdloop() # The argument is the remaining string on the command line # Return true to exit from the command loop @@ -904,14 +1072,10 @@ def do_commands(self, arg): self.commands_bnum = bnum # Save old definitions for the case of a keyboard interrupt. if bnum in self.commands: - old_command_defs = (self.commands[bnum], - self.commands_doprompt[bnum], - self.commands_silent[bnum]) + old_commands = self.commands[bnum] else: - old_command_defs = None + old_commands = None self.commands[bnum] = [] - self.commands_doprompt[bnum] = True - self.commands_silent[bnum] = False prompt_back = self.prompt self.prompt = '(com) ' @@ -920,14 +1084,10 @@ def do_commands(self, arg): self.cmdloop() except KeyboardInterrupt: # Restore old definitions. - if old_command_defs: - self.commands[bnum] = old_command_defs[0] - self.commands_doprompt[bnum] = old_command_defs[1] - self.commands_silent[bnum] = old_command_defs[2] + if old_commands: + self.commands[bnum] = old_commands else: del self.commands[bnum] - del self.commands_doprompt[bnum] - del self.commands_silent[bnum] self.error('command definition aborted, old commands restored') finally: self.commands_defining = False @@ -935,7 +1095,7 @@ def do_commands(self, arg): complete_commands = _complete_bpnumber - def do_break(self, arg, temporary = 0): + def do_break(self, arg, temporary=False): """b(reak) [ ([filename:]lineno | function) [, condition] ] Without argument, list all breaks. @@ -996,7 +1156,7 @@ def do_break(self, arg, temporary = 0): try: func = eval(arg, self.curframe.f_globals, - self.curframe_locals) + self.curframe.f_locals) except: func = arg try: @@ -1050,7 +1210,7 @@ def do_tbreak(self, arg): Same arguments as break, but sets a temporary breakpoint: it is automatically deleted when first hit. """ - self.do_break(arg, 1) + self.do_break(arg, True) complete_tbreak = _complete_location @@ -1084,7 +1244,7 @@ def lineinfo(self, identifier): if f: fname = f item = parts[1] - answer = find_function(item, fname) + answer = find_function(item, self.canonic(fname)) return answer or failed def checkline(self, filename, lineno): @@ -1276,16 +1436,24 @@ def do_clear(self, arg): complete_cl = _complete_location def do_where(self, arg): - """w(here) + """w(here) [count] - Print a stack trace, with the most recent frame at the bottom. + Print a stack trace. If count is not specified, print the full stack. + If count is 0, print the current frame entry. If count is positive, + print count entries from the most recent frame. If count is negative, + print -count entries from the least recent frame. An arrow indicates the "current frame", which determines the context of most commands. 'bt' is an alias for this command. """ - if arg: - self._print_invalid_arg(arg) - return - self.print_stack_trace() + if not arg: + count = None + else: + try: + count = int(arg) + except ValueError: + self.error('Invalid count (%s)' % arg) + return + self.print_stack_trace(count) do_w = do_where do_bt = do_where @@ -1293,7 +1461,6 @@ def _select_frame(self, number): assert 0 <= number < len(self.stack) self.curindex = number self.curframe = self.stack[self.curindex][0] - self.curframe_locals = self.curframe.f_locals self.set_convenience_variable(self.curframe, '_frame', self.curframe) self.print_stack_entry(self.stack[self.curindex]) self.lineno = None @@ -1447,6 +1614,11 @@ def do_run(self, arg): sys.argv. History, breakpoints, actions and debugger options are preserved. "restart" is an alias for "run". """ + if self.mode == 'inline': + self.error('run/restart command is disabled when pdb is running in inline mode.\n' + 'Use the command line interface to enable restarting your program\n' + 'e.g. "python -m pdb myscript.py"') + return if arg: import shlex argv0 = sys.argv[0:1] @@ -1534,7 +1706,7 @@ def do_debug(self, arg): """ sys.settrace(None) globals = self.curframe.f_globals - locals = self.curframe_locals + locals = self.curframe.f_locals p = Pdb(self.completekey, self.stdin, self.stdout) p.prompt = "(%s) " % self.prompt.strip() self.message("ENTERING RECURSIVE DEBUGGER") @@ -1553,6 +1725,19 @@ def do_quit(self, arg): Quit from the debugger. The program being executed is aborted. """ + if self.mode == 'inline': + while True: + try: + reply = input('Quitting pdb will kill the process. Quit anyway? [y/n] ') + reply = reply.lower().strip() + except EOFError: + reply = 'y' + self.message('') + if reply == 'y' or reply == '': + sys.exit(0) + elif reply.lower() == 'n': + return + self._user_requested_quit = True self.set_quit() return 1 @@ -1566,9 +1751,7 @@ def do_EOF(self, arg): Handles the receipt of EOF as a command. """ self.message('') - self._user_requested_quit = True - self.set_quit() - return 1 + return self.do_quit(arg) def do_args(self, arg): """a(rgs) @@ -1579,7 +1762,7 @@ def do_args(self, arg): self._print_invalid_arg(arg) return co = self.curframe.f_code - dict = self.curframe_locals + dict = self.curframe.f_locals n = co.co_argcount + co.co_kwonlyargcount if co.co_flags & inspect.CO_VARARGS: n = n+1 if co.co_flags & inspect.CO_VARKEYWORDS: n = n+1 @@ -1599,15 +1782,15 @@ def do_retval(self, arg): if arg: self._print_invalid_arg(arg) return - if '__return__' in self.curframe_locals: - self.message(self._safe_repr(self.curframe_locals['__return__'], "retval")) + if '__return__' in self.curframe.f_locals: + self.message(self._safe_repr(self.curframe.f_locals['__return__'], "retval")) else: self.error('Not yet returned!') do_rv = do_retval def _getval(self, arg): try: - return eval(arg, self.curframe.f_globals, self.curframe_locals) + return eval(arg, self.curframe.f_globals, self.curframe.f_locals) except: self._error_exc() raise @@ -1615,7 +1798,7 @@ def _getval(self, arg): def _getval_except(self, arg, frame=None): try: if frame is None: - return eval(arg, self.curframe.f_globals, self.curframe_locals) + return eval(arg, self.curframe.f_globals, self.curframe.f_locals) else: return eval(arg, frame.f_globals, frame.f_locals) except BaseException as exc: @@ -1714,6 +1897,7 @@ def do_list(self, arg): self.message('[EOF]') except KeyboardInterrupt: pass + self._validate_file_mtime() do_l = do_list def do_longlist(self, arg): @@ -1732,6 +1916,7 @@ def do_longlist(self, arg): self.error(err) return self._print_lines(lines, lineno, breaklist, self.curframe) + self._validate_file_mtime() do_ll = do_longlist def do_source(self, arg): @@ -1859,7 +2044,7 @@ def do_interact(self, arg): Start an interactive interpreter whose global namespace contains all the (global and local) names found in the current scope. """ - ns = {**self.curframe.f_globals, **self.curframe_locals} + ns = {**self.curframe.f_globals, **self.curframe.f_locals} console = _PdbInteractiveConsole(ns, message=self.message) console.interact(banner="*pdb interact start*", exitmsg="*exit from pdb interact command*") @@ -1931,7 +2116,7 @@ def complete_unalias(self, text, line, begidx, endidx): # List of all the commands making the program resume execution. commands_resuming = ['do_continue', 'do_step', 'do_next', 'do_return', - 'do_quit', 'do_jump'] + 'do_until', 'do_quit', 'do_jump'] # Print a traceback starting at the top stack frame. # The most recently entered frame is printed last; @@ -1940,10 +2125,22 @@ def complete_unalias(self, text, line, begidx, endidx): # It is also consistent with the up/down commands (which are # compatible with dbx and gdb: up moves towards 'main()' # and down moves towards the most recent stack frame). - - def print_stack_trace(self): + # * if count is None, prints the full stack + # * if count = 0, prints the current frame entry + # * if count < 0, prints -count least recent frame entries + # * if count > 0, prints count most recent frame entries + + def print_stack_trace(self, count=None): + if count is None: + stack_to_print = self.stack + elif count == 0: + stack_to_print = [self.stack[self.curindex]] + elif count < 0: + stack_to_print = self.stack[:-count] + else: + stack_to_print = self.stack[-count:] try: - for frame_lineno in self.stack: + for frame_lineno in stack_to_print: self.print_stack_entry(frame_lineno) except KeyboardInterrupt: pass @@ -2015,17 +2212,23 @@ def lookupmodule(self, filename): lookupmodule() translates (possibly incomplete) file or module name into an absolute file name. + + filename could be in format of: + * an absolute path like '/path/to/file.py' + * a relative path like 'file.py' or 'dir/file.py' + * a module name like 'module' or 'package.module' + + files and modules will be searched in sys.path. """ - if os.path.isabs(filename) and os.path.exists(filename): - return filename - f = os.path.join(sys.path[0], filename) - if os.path.exists(f) and self.canonic(f) == self.mainpyfile: - return f - root, ext = os.path.splitext(filename) - if ext == '': - filename = filename + '.py' + if not filename.endswith('.py'): + # A module is passed in so convert it to equivalent file + filename = filename.replace('.', os.sep) + '.py' + if os.path.isabs(filename): - return filename + if os.path.exists(filename): + return filename + return None + for dirname in sys.path: while os.path.islink(dirname): dirname = os.readlink(dirname) @@ -2034,7 +2237,7 @@ def lookupmodule(self, filename): return fullname return None - def _run(self, target: Union[_ModuleTarget, _ScriptTarget]): + def _run(self, target: _ExecutableTarget): # When bdb sets tracing, a number of call and line events happen # BEFORE debugger even reaches user's code (and the exact sequence of # events depends on python version). Take special measures to @@ -2172,18 +2375,22 @@ def runcall(*args, **kwds): """ return Pdb().runcall(*args, **kwds) -def set_trace(*, header=None): +def set_trace(*, header=None, commands=None): """Enter the debugger at the calling stack frame. This is useful to hard-code a breakpoint at a given point in a program, even if the code is not otherwise being debugged (e.g. when an assertion fails). If given, *header* is printed to the console - just before debugging begins. + just before debugging begins. *commands* is an optional list of + pdb commands to run when the debugger starts. """ - pdb = Pdb() + if Pdb._last_pdb_instance is not None: + pdb = Pdb._last_pdb_instance + else: + pdb = Pdb(mode='inline') if header is not None: pdb.message(header) - pdb.set_trace(sys._getframe().f_back) + pdb.set_trace(sys._getframe().f_back, commands=commands) # Post-Mortem interface @@ -2254,16 +2461,17 @@ def help(): def main(): import argparse - parser = argparse.ArgumentParser(prog="pdb", + parser = argparse.ArgumentParser(usage="%(prog)s [-h] [-c command] (-m module | pyfile) [args ...]", description=_usage, formatter_class=argparse.RawDescriptionHelpFormatter, allow_abbrev=False) - parser.add_argument('-c', '--command', action='append', default=[], metavar='command') - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('-m', metavar='module') - group.add_argument('pyfile', nargs='?') - parser.add_argument('args', nargs="*") + # We need to maunally get the script from args, because the first positional + # arguments could be either the script we need to debug, or the argument + # to the -m module + parser.add_argument('-c', '--command', action='append', default=[], metavar='command', dest='commands', + help='pdb commands to execute as if given in a .pdbrc file') + parser.add_argument('-m', metavar='module', dest='module') if len(sys.argv) == 1: # If no arguments were given (python -m pdb), print the whole help message. @@ -2271,25 +2479,47 @@ def main(): parser.print_help() sys.exit(2) - opts = parser.parse_args() - - if opts.m: - file = opts.m + opts, args = parser.parse_known_args() + + if opts.module: + # If a module is being debugged, we consider the arguments after "-m module" to + # be potential arguments to the module itself. We need to parse the arguments + # before "-m" to check if there is any invalid argument. + # e.g. "python -m pdb -m foo --spam" means passing "--spam" to "foo" + # "python -m pdb --spam -m foo" means passing "--spam" to "pdb" and is invalid + idx = sys.argv.index('-m') + args_to_pdb = sys.argv[1:idx] + # This will raise an error if there are invalid arguments + parser.parse_args(args_to_pdb) + else: + # If a script is being debugged, then pdb expects the script name as the first argument. + # Anything before the script is considered an argument to pdb itself, which would + # be invalid because it's not parsed by argparse. + invalid_args = list(itertools.takewhile(lambda a: a.startswith('-'), args)) + if invalid_args: + parser.error(f"unrecognized arguments: {' '.join(invalid_args)}") + sys.exit(2) + + if opts.module: + file = opts.module target = _ModuleTarget(file) else: - file = opts.pyfile - target = _ScriptTarget(file) - - target.check() + if not args: + parser.error("no module or script to run") + file = args.pop(0) + if file.endswith('.pyz'): + target = _ZipTarget(file) + else: + target = _ScriptTarget(file) - sys.argv[:] = [file] + opts.args # Hide "pdb.py" and pdb options from argument list + sys.argv[:] = [file] + args # Hide "pdb.py" and pdb options from argument list # Note on saving/restoring sys.argv: it's a good idea when sys.argv was # modified by the script being debugged. It's a bad idea when it was # changed by the user from the command line. There is a "restart" command # which allows explicit specification of command line arguments. - pdb = Pdb() - pdb.rcLines.extend(opts.command) + pdb = Pdb(mode='cli') + pdb.rcLines.extend(opts.commands) while True: try: pdb._run(target) @@ -2301,12 +2531,15 @@ def main(): print("The program exited via sys.exit(). Exit status:", end=' ') print(e) except BaseException as e: - traceback.print_exc() + traceback.print_exception(e, colorize=_colorize.can_colorize()) print("Uncaught exception. Entering post mortem debugging") print("Running 'cont' or 'step' will restart the program") - pdb.interaction(None, e) - print("Post mortem debugger finished. The " + target + - " will be restarted") + try: + pdb.interaction(None, e) + except Restart: + print("Restarting", target, "with arguments:") + print("\t" + " ".join(sys.argv[1:])) + continue if pdb._user_requested_quit: break print("The program finished and will be restarted") diff --git a/Lib/pickle.py b/Lib/pickle.py index 33c97c8c5efb28..8afb4aa4285f37 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -26,12 +26,11 @@ from types import FunctionType from copyreg import dispatch_table from copyreg import _extension_registry, _inverted_registry, _extension_cache -from itertools import islice +from itertools import batched from functools import partial import sys from sys import maxsize from struct import pack, unpack -import re import io import codecs import _compat_pickle @@ -51,7 +50,7 @@ bytes_types = (bytes, bytearray) # These are purely informational; no code uses these. -format_version = "4.0" # File format version we write +format_version = "5.0" # File format version we write compatible_formats = ["1.0", # Original protocol 0 "1.1", # Protocol 0 with INST added "1.2", # Original protocol 1 @@ -68,7 +67,7 @@ # The protocol we write by default. May be less than HIGHEST_PROTOCOL. # Only bump this if the oldest still supported version of Python already # includes it. -DEFAULT_PROTOCOL = 4 +DEFAULT_PROTOCOL = 5 class PickleError(Exception): """A common base class for the other pickling exceptions.""" @@ -188,7 +187,7 @@ def __init__(self, value): NEXT_BUFFER = b'\x97' # push next out-of-band buffer READONLY_BUFFER = b'\x98' # make top of stack readonly -__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$", x)]) +__all__.extend(x for x in dir() if x.isupper() and not x.startswith('_')) class _Framer: @@ -313,37 +312,46 @@ def load_frame(self, frame_size): # Tools used for pickling. -def _getattribute(obj, name): - for subpath in name.split('.'): - if subpath == '': - raise AttributeError("Can't get local attribute {!r} on {!r}" - .format(name, obj)) - try: - parent = obj - obj = getattr(obj, subpath) - except AttributeError: - raise AttributeError("Can't get attribute {!r} on {!r}" - .format(name, obj)) from None - return obj, parent +def _getattribute(obj, dotted_path): + for subpath in dotted_path: + obj = getattr(obj, subpath) + return obj def whichmodule(obj, name): """Find the module an object belong to.""" + dotted_path = name.split('.') module_name = getattr(obj, '__module__', None) - if module_name is not None: - return module_name - # Protect the iteration by using a list copy of sys.modules against dynamic - # modules that trigger imports of other modules upon calls to getattr. - for module_name, module in sys.modules.copy().items(): - if (module_name == '__main__' - or module_name == '__mp_main__' # bpo-42406 - or module is None): - continue - try: - if _getattribute(module, name)[0] is obj: - return module_name - except AttributeError: - pass - return '__main__' + if '' in dotted_path: + raise PicklingError(f"Can't pickle local object {obj!r}") + if module_name is None: + # Protect the iteration by using a list copy of sys.modules against dynamic + # modules that trigger imports of other modules upon calls to getattr. + for module_name, module in sys.modules.copy().items(): + if (module_name == '__main__' + or module_name == '__mp_main__' # bpo-42406 + or module is None): + continue + try: + if _getattribute(module, dotted_path) is obj: + return module_name + except AttributeError: + pass + module_name = '__main__' + + try: + __import__(module_name, level=0) + module = sys.modules[module_name] + except (ImportError, ValueError, KeyError) as exc: + raise PicklingError(f"Can't pickle {obj!r}: {exc!s}") + try: + if _getattribute(module, dotted_path) is obj: + return module_name + except AttributeError: + raise PicklingError(f"Can't pickle {obj!r}: " + f"it's not found as {module_name}.{name}") + + raise PicklingError( + f"Can't pickle {obj!r}: it's not the same object as {module_name}.{name}") def encode_long(x): r"""Encode a long to a two's complement little-endian binary string. @@ -395,6 +403,13 @@ def decode_long(data): """ return int.from_bytes(data, byteorder='little', signed=True) +def _T(obj): + cls = type(obj) + module = cls.__module__ + if module in (None, 'builtins', '__main__'): + return cls.__qualname__ + return f'{module}.{cls.__qualname__}' + _NoValue = object() @@ -408,7 +423,7 @@ def __init__(self, file, protocol=None, *, fix_imports=True, The optional *protocol* argument tells the pickler to use the given protocol; supported protocols are 0, 1, 2, 3, 4 and 5. - The default protocol is 4. It was introduced in Python 3.4, and + The default protocol is 5. It was introduced in Python 3.8, and is incompatible with previous versions. Specifying a negative protocol version selects the highest @@ -532,10 +547,11 @@ def save(self, obj, save_persistent_id=True): self.framer.commit_frame() # Check for persistent id (defined by a subclass) - pid = self.persistent_id(obj) - if pid is not None and save_persistent_id: - self.save_pers(pid) - return + if save_persistent_id: + pid = self.persistent_id(obj) + if pid is not None: + self.save_pers(pid) + return # Check the memo x = self.memo.get(id(obj)) @@ -577,26 +593,29 @@ def save(self, obj, save_persistent_id=True): if reduce is not _NoValue: rv = reduce() else: - raise PicklingError("Can't pickle %r object: %r" % - (t.__name__, obj)) + raise PicklingError(f"Can't pickle {_T(t)} object") # Check for string returned by reduce(), meaning "save as global" if isinstance(rv, str): self.save_global(obj, rv) return - # Assert that reduce() returned a tuple - if not isinstance(rv, tuple): - raise PicklingError("%s must return string or tuple" % reduce) - - # Assert that it returned an appropriately sized tuple - l = len(rv) - if not (2 <= l <= 6): - raise PicklingError("Tuple returned by %s must have " - "two to six elements" % reduce) - - # Save the reduce() output and finally memoize the object - self.save_reduce(obj=obj, *rv) + try: + # Assert that reduce() returned a tuple + if not isinstance(rv, tuple): + raise PicklingError(f'__reduce__ must return a string or tuple, not {_T(rv)}') + + # Assert that it returned an appropriately sized tuple + l = len(rv) + if not (2 <= l <= 6): + raise PicklingError("tuple returned by __reduce__ " + "must contain 2 through 6 elements") + + # Save the reduce() output and finally memoize the object + self.save_reduce(obj=obj, *rv) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} object') + raise def persistent_id(self, obj): # This exists so a subclass can override it @@ -618,10 +637,12 @@ def save_reduce(self, func, args, state=None, listitems=None, dictitems=None, state_setter=None, *, obj=None): # This API is called by some subclasses - if not isinstance(args, tuple): - raise PicklingError("args from save_reduce() must be a tuple") if not callable(func): - raise PicklingError("func from save_reduce() must be callable") + raise PicklingError(f"first item of the tuple returned by __reduce__ " + f"must be callable, not {_T(func)}") + if not isinstance(args, tuple): + raise PicklingError(f"second item of the tuple returned by __reduce__ " + f"must be a tuple, not {_T(args)}") save = self.save write = self.write @@ -630,19 +651,30 @@ def save_reduce(self, func, args, state=None, listitems=None, if self.proto >= 2 and func_name == "__newobj_ex__": cls, args, kwargs = args if not hasattr(cls, "__new__"): - raise PicklingError("args[0] from {} args has no __new__" - .format(func_name)) + raise PicklingError("first argument to __newobj_ex__() has no __new__") if obj is not None and cls is not obj.__class__: - raise PicklingError("args[0] from {} args has the wrong class" - .format(func_name)) + raise PicklingError(f"first argument to __newobj_ex__() " + f"must be {obj.__class__!r}, not {cls!r}") if self.proto >= 4: - save(cls) - save(args) - save(kwargs) + try: + save(cls) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} class') + raise + try: + save(args) + save(kwargs) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} __new__ arguments') + raise write(NEWOBJ_EX) else: func = partial(cls.__new__, cls, *args, **kwargs) - save(func) + try: + save(func) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} reconstructor') + raise save(()) write(REDUCE) elif self.proto >= 2 and func_name == "__newobj__": @@ -674,18 +706,33 @@ def save_reduce(self, func, args, state=None, listitems=None, # Python 2.2). cls = args[0] if not hasattr(cls, "__new__"): - raise PicklingError( - "args[0] from __newobj__ args has no __new__") + raise PicklingError("first argument to __newobj__() has no __new__") if obj is not None and cls is not obj.__class__: - raise PicklingError( - "args[0] from __newobj__ args has the wrong class") + raise PicklingError(f"first argument to __newobj__() " + f"must be {obj.__class__!r}, not {cls!r}") args = args[1:] - save(cls) - save(args) + try: + save(cls) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} class') + raise + try: + save(args) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} __new__ arguments') + raise write(NEWOBJ) else: - save(func) - save(args) + try: + save(func) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} reconstructor') + raise + try: + save(args) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} reconstructor arguments') + raise write(REDUCE) if obj is not None: @@ -703,23 +750,35 @@ def save_reduce(self, func, args, state=None, listitems=None, # items and dict items (as (key, value) tuples), or None. if listitems is not None: - self._batch_appends(listitems) + self._batch_appends(listitems, obj) if dictitems is not None: - self._batch_setitems(dictitems) + self._batch_setitems(dictitems, obj) if state is not None: if state_setter is None: - save(state) + try: + save(state) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} state') + raise write(BUILD) else: # If a state_setter is specified, call it instead of load_build # to update obj's with its previous state. # First, push state_setter and its tuple of expected arguments # (obj, state) onto the stack. - save(state_setter) + try: + save(state_setter) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} state setter') + raise save(obj) # simple BINGET opcode as obj is already memoized. - save(state) + try: + save(state) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} state') + raise write(TUPLE2) # Trigger a state_setter(obj, state) function call. write(REDUCE) @@ -782,14 +841,10 @@ def save_float(self, obj): self.write(FLOAT + repr(obj).encode("ascii") + b'\n') dispatch[float] = save_float - def save_bytes(self, obj): - if self.proto < 3: - if not obj: # bytes object is empty - self.save_reduce(bytes, (), obj=obj) - else: - self.save_reduce(codecs.encode, - (str(obj, 'latin1'), 'latin1'), obj=obj) - return + def _save_bytes_no_memo(self, obj): + # helper for writing bytes objects for protocol >= 3 + # without memoizing them + assert self.proto >= 3 n = len(obj) if n <= 0xff: self.write(SHORT_BINBYTES + pack("= 5 + # without memoizing them + assert self.proto >= 5 + n = len(obj) + if n >= self.framer._FRAME_SIZE_TARGET: + self._write_large_bytes(BYTEARRAY8 + pack("= self.framer._FRAME_SIZE_TARGET: - self._write_large_bytes(BYTEARRAY8 + pack("= 5") with obj.raw() as m: if not m.contiguous: @@ -832,10 +903,18 @@ def save_picklebuffer(self, obj): if in_band: # Write data in-band # XXX The C implementation avoids a copy here + buf = m.tobytes() + in_memo = id(buf) in self.memo if m.readonly: - self.save_bytes(m.tobytes()) + if in_memo: + self._save_bytes_no_memo(buf) + else: + self.save_bytes(buf) else: - self.save_bytearray(m.tobytes()) + if in_memo: + self._save_bytearray_no_memo(buf) + else: + self.save_bytearray(buf) else: # Write data out-of-band self.write(NEXT_BUFFER) @@ -879,8 +958,12 @@ def save_tuple(self, obj): save = self.save memo = self.memo if n <= 3 and self.proto >= 2: - for element in obj: - save(element) + for i, element in enumerate(obj): + try: + save(element) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {i}') + raise # Subtle. Same as in the big comment below. if id(obj) in memo: get = self.get(memo[id(obj)][0]) @@ -894,8 +977,12 @@ def save_tuple(self, obj): # has more than 3 elements. write = self.write write(MARK) - for element in obj: - save(element) + for i, element in enumerate(obj): + try: + save(element) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {i}') + raise if id(obj) in memo: # Subtle. d was not in memo when we entered save_tuple(), so @@ -925,38 +1012,47 @@ def save_list(self, obj): self.write(MARK + LIST) self.memoize(obj) - self._batch_appends(obj) + self._batch_appends(obj, obj) dispatch[list] = save_list _BATCHSIZE = 1000 - def _batch_appends(self, items): + def _batch_appends(self, items, obj): # Helper to batch up APPENDS sequences save = self.save write = self.write if not self.bin: - for x in items: - save(x) + for i, x in enumerate(items): + try: + save(x) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {i}') + raise write(APPEND) return - it = iter(items) - while True: - tmp = list(islice(it, self._BATCHSIZE)) - n = len(tmp) - if n > 1: + start = 0 + for batch in batched(items, self._BATCHSIZE): + batch_len = len(batch) + if batch_len != 1: write(MARK) - for x in tmp: - save(x) + for i, x in enumerate(batch, start): + try: + save(x) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {i}') + raise write(APPENDS) - elif n: - save(tmp[0]) + else: + try: + save(batch[0]) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {start}') + raise write(APPEND) - # else tmp is empty, and we're done - if n < self._BATCHSIZE: - return + start += batch_len def save_dict(self, obj): if self.bin: @@ -965,11 +1061,11 @@ def save_dict(self, obj): self.write(MARK + DICT) self.memoize(obj) - self._batch_setitems(obj.items()) + self._batch_setitems(obj.items(), obj) dispatch[dict] = save_dict - def _batch_setitems(self, items): + def _batch_setitems(self, items, obj): # Helper to batch up SETITEMS sequences; proto >= 1 only save = self.save write = self.write @@ -977,28 +1073,34 @@ def _batch_setitems(self, items): if not self.bin: for k, v in items: save(k) - save(v) + try: + save(v) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {k!r}') + raise write(SETITEM) return - it = iter(items) - while True: - tmp = list(islice(it, self._BATCHSIZE)) - n = len(tmp) - if n > 1: + for batch in batched(items, self._BATCHSIZE): + if len(batch) != 1: write(MARK) - for k, v in tmp: + for k, v in batch: save(k) - save(v) + try: + save(v) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {k!r}') + raise write(SETITEMS) - elif n: - k, v = tmp[0] + else: + k, v = batch[0] save(k) - save(v) + try: + save(v) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {k!r}') + raise write(SETITEM) - # else tmp is empty, and we're done - if n < self._BATCHSIZE: - return def save_set(self, obj): save = self.save @@ -1011,17 +1113,15 @@ def save_set(self, obj): write(EMPTY_SET) self.memoize(obj) - it = iter(obj) - while True: - batch = list(islice(it, self._BATCHSIZE)) - n = len(batch) - if n > 0: - write(MARK) + for batch in batched(obj, self._BATCHSIZE): + write(MARK) + try: for item in batch: save(item) - write(ADDITEMS) - if n < self._BATCHSIZE: - return + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} element') + raise + write(ADDITEMS) dispatch[set] = save_set def save_frozenset(self, obj): @@ -1033,8 +1133,12 @@ def save_frozenset(self, obj): return write(MARK) - for item in obj: - save(item) + try: + for item in obj: + save(item) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} element') + raise if id(obj) in self.memo: # If the object is already in the memo, this means it is @@ -1053,48 +1157,59 @@ def save_global(self, obj, name=None): if name is None: name = getattr(obj, '__qualname__', None) - if name is None: - name = obj.__name__ + if name is None: + name = obj.__name__ module_name = whichmodule(obj, name) - try: - __import__(module_name, level=0) - module = sys.modules[module_name] - obj2, parent = _getattribute(module, name) - except (ImportError, KeyError, AttributeError): - raise PicklingError( - "Can't pickle %r: it's not found as %s.%s" % - (obj, module_name, name)) from None - else: - if obj2 is not obj: - raise PicklingError( - "Can't pickle %r: it's not the same object as %s.%s" % - (obj, module_name, name)) - if self.proto >= 2: - code = _extension_registry.get((module_name, name)) - if code: - assert code > 0 + code = _extension_registry.get((module_name, name), _NoValue) + if code is not _NoValue: if code <= 0xff: - write(EXT1 + pack("= 3. + if self.proto >= 4: self.save(module_name) self.save(name) write(STACK_GLOBAL) - elif parent is not module: - self.save_reduce(getattr, (parent, lastname)) - elif self.proto >= 3: - write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + - bytes(name, "utf-8") + b'\n') + elif '.' in name: + # In protocol < 4, objects with multi-part __qualname__ + # are represented as + # getattr(getattr(..., attrname1), attrname2). + dotted_path = name.split('.') + name = dotted_path.pop(0) + save = self.save + for attrname in dotted_path: + save(getattr) + if self.proto < 2: + write(MARK) + self._save_toplevel_by_name(module_name, name) + for attrname in dotted_path: + save(attrname) + if self.proto < 2: + write(TUPLE) + else: + write(TUPLE2) + write(REDUCE) + else: + self._save_toplevel_by_name(module_name, name) + + self.memoize(obj) + + def _save_toplevel_by_name(self, module_name, name): + if self.proto >= 3: + # Non-ASCII identifiers are supported only with protocols >= 3. + encoding = "utf-8" else: if self.fix_imports: r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING @@ -1103,15 +1218,19 @@ def save_global(self, obj, name=None): module_name, name = r_name_mapping[(module_name, name)] elif module_name in r_import_mapping: module_name = r_import_mapping[module_name] - try: - write(GLOBAL + bytes(module_name, "ascii") + b'\n' + - bytes(name, "ascii") + b'\n') - except UnicodeEncodeError: - raise PicklingError( - "can't pickle global identifier '%s.%s' using " - "pickle protocol %i" % (module, name, self.proto)) from None - - self.memoize(obj) + encoding = "ascii" + try: + self.write(GLOBAL + bytes(module_name, encoding) + b'\n') + except UnicodeEncodeError: + raise PicklingError( + f"can't pickle module identifier {module_name!r} using " + f"pickle protocol {self.proto}") + try: + self.write(bytes(name, encoding) + b'\n') + except UnicodeEncodeError: + raise PicklingError( + f"can't pickle global identifier {name!r} using " + f"pickle protocol {self.proto}") def save_type(self, obj): if obj is type(None): @@ -1267,7 +1386,7 @@ def load_int(self): elif data == TRUE[1:]: val = True else: - val = int(data, 0) + val = int(data) self.append(val) dispatch[INT[0]] = load_int @@ -1287,7 +1406,7 @@ def load_long(self): val = self.readline()[:-1] if val and val[-1] == b'L'[0]: val = val[:-1] - self.append(int(val, 0)) + self.append(int(val)) dispatch[LONG[0]] = load_long def load_long1(self): @@ -1548,9 +1667,8 @@ def load_ext4(self): dispatch[EXT4[0]] = load_ext4 def get_extension(self, code): - nil = [] - obj = _extension_cache.get(code, nil) - if obj is not nil: + obj = _extension_cache.get(code, _NoValue) + if obj is not _NoValue: self.append(obj) return key = _inverted_registry.get(code) @@ -1572,8 +1690,13 @@ def find_class(self, module, name): elif module in _compat_pickle.IMPORT_MAPPING: module = _compat_pickle.IMPORT_MAPPING[module] __import__(module, level=0) - if self.proto >= 4: - return _getattribute(sys.modules[module], name)[0] + if self.proto >= 4 and '.' in name: + dotted_path = name.split('.') + try: + return _getattribute(sys.modules[module], dotted_path) + except AttributeError: + raise AttributeError( + f"Can't resolve path {name!r} on module {module!r}") else: return getattr(sys.modules[module], name) diff --git a/Lib/pickletools.py b/Lib/pickletools.py index 51ee4a7a2632ac..d9c4fb1e63e91a 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -312,7 +312,7 @@ def read_uint8(f): doc="Eight-byte unsigned integer, little-endian.") -def read_stringnl(f, decode=True, stripquotes=True): +def read_stringnl(f, decode=True, stripquotes=True, *, encoding='latin-1'): r""" >>> import io >>> read_stringnl(io.BytesIO(b"'abcd'\nefg\n")) @@ -356,7 +356,7 @@ def read_stringnl(f, decode=True, stripquotes=True): raise ValueError("no string quotes around %r" % data) if decode: - data = codecs.escape_decode(data)[0].decode("ascii") + data = codecs.escape_decode(data)[0].decode(encoding) return data stringnl = ArgumentDescriptor( @@ -370,7 +370,7 @@ def read_stringnl(f, decode=True, stripquotes=True): """) def read_stringnl_noescape(f): - return read_stringnl(f, stripquotes=False) + return read_stringnl(f, stripquotes=False, encoding='utf-8') stringnl_noescape = ArgumentDescriptor( name='stringnl_noescape', @@ -2429,8 +2429,6 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): + A memo entry isn't referenced before it's defined. + The markobject isn't stored in the memo. - - + A memo entry isn't redefined. """ # Most of the hair here is for sanity checks, but most of it is needed @@ -2484,7 +2482,7 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): assert opcode.name == "POP" numtopop = 0 else: - errormsg = markmsg = "no MARK exists on stack" + errormsg = "no MARK exists on stack" # Check for correct memo usage. if opcode.name in ("PUT", "BINPUT", "LONG_BINPUT", "MEMOIZE"): @@ -2494,9 +2492,7 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): else: assert arg is not None memo_idx = arg - if memo_idx in memo: - errormsg = "memo key %r already defined" % arg - elif not stack: + if not stack: errormsg = "stack is empty -- can't store into memo" elif stack[-1] is markobject: errormsg = "can't store markobject in the memo" @@ -2513,7 +2509,10 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): # make a mild effort to align arguments line += ' ' * (10 - len(opcode.name)) if arg is not None: - line += ' ' + repr(arg) + if opcode.name in ("STRING", "BINSTRING", "SHORT_BINSTRING"): + line += ' ' + ascii(arg) + else: + line += ' ' + repr(arg) if markmsg: line += ' ' + markmsg if annotate: diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index dccbec52aa731e..b84d72f2395d45 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -12,7 +12,7 @@ import warnings __all__ = [ - 'get_importer', 'iter_importers', 'get_loader', 'find_loader', + 'get_importer', 'iter_importers', 'walk_packages', 'iter_modules', 'get_data', 'read_code', 'extend_path', 'ModuleInfo', @@ -263,59 +263,6 @@ def iter_importers(fullname=""): yield get_importer(item) -def get_loader(module_or_name): - """Get a "loader" object for module_or_name - - Returns None if the module cannot be found or imported. - If the named module is not already imported, its containing package - (if any) is imported, in order to establish the package __path__. - """ - warnings._deprecated("pkgutil.get_loader", - f"{warnings._DEPRECATED_MSG}; " - "use importlib.util.find_spec() instead", - remove=(3, 14)) - if module_or_name in sys.modules: - module_or_name = sys.modules[module_or_name] - if module_or_name is None: - return None - if isinstance(module_or_name, ModuleType): - module = module_or_name - loader = getattr(module, '__loader__', None) - if loader is not None: - return loader - if getattr(module, '__spec__', None) is None: - return None - fullname = module.__name__ - else: - fullname = module_or_name - return find_loader(fullname) - - -def find_loader(fullname): - """Find a "loader" object for fullname - - This is a backwards compatibility wrapper around - importlib.util.find_spec that converts most failures to ImportError - and only returns the loader rather than the full spec - """ - warnings._deprecated("pkgutil.find_loader", - f"{warnings._DEPRECATED_MSG}; " - "use importlib.util.find_spec() instead", - remove=(3, 14)) - if fullname.startswith('.'): - msg = "Relative module name {!r} not supported".format(fullname) - raise ImportError(msg) - try: - spec = importlib.util.find_spec(fullname) - except (ImportError, AttributeError, TypeError, ValueError) as ex: - # This hack fixes an impedance mismatch between pkgutil and - # importlib, where the latter raises other errors for cases where - # pkgutil previously raised ImportError - msg = "Error while finding loader for {!r} ({}: {})" - raise ImportError(msg.format(fullname, type(ex), ex)) from ex - return spec.loader if spec is not None else None - - def extend_path(path, name): """Extend a package's path. diff --git a/Lib/platform.py b/Lib/platform.py old mode 100755 new mode 100644 index b56472235ee9e4..1f6baed66d3df9 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - """ This module tries to retrieve as much platform-identifying data as possible. It makes this information available via function APIs. @@ -10,7 +8,8 @@ """ # This module is maintained by Marc-Andre Lemburg . # If you find problems, please submit bug reports/patches via the -# Python bug tracker (http://bugs.python.org) and assign them to "lemburg". +# Python issue tracker (https://github.com/python/cpython/issues) and +# mention "@malemburg". # # Still needed: # * support for MS-DOS (PythonDX ?) @@ -32,6 +31,7 @@ # # # +# 1.0.9 - added invalidate_caches() function to invalidate cached values # 1.0.8 - changed Windows support to read version from kernel32.dll # 1.0.7 - added DEV_NULL # 1.0.6 - added linux_distribution() @@ -110,7 +110,7 @@ """ -__version__ = '1.0.8' +__version__ = '1.0.9' import collections import os @@ -353,7 +353,8 @@ def _wmi_query(table, *keys): ] _WIN32_SERVER_RELEASES = [ - ((10, 1, 0), "post2022Server"), + ((10, 1, 0), "post2025Server"), + ((10, 0, 26100), "2025Server"), ((10, 0, 20348), "2022Server"), ((10, 0, 17763), "2019Server"), ((6, 4, 0), "2016Server"), @@ -370,10 +371,7 @@ def win32_is_iot(): def win32_edition(): try: - try: - import winreg - except ImportError: - import _winreg as winreg + import winreg except ImportError: pass else: @@ -432,10 +430,7 @@ def _win32_ver(version, csd, ptype): csd = 'SP' + csd[13:] try: - try: - import winreg - except ImportError: - import _winreg as winreg + import winreg except ImportError: pass else: @@ -502,8 +497,32 @@ def mac_ver(release='', versioninfo=('', '', ''), machine=''): # If that also doesn't work return the default values return release, versioninfo, machine -def _java_getprop(name, default): +# A namedtuple for iOS version information. +IOSVersionInfo = collections.namedtuple( + "IOSVersionInfo", + ["system", "release", "model", "is_simulator"] +) + + +def ios_ver(system="", release="", model="", is_simulator=False): + """Get iOS version information, and return it as a namedtuple: + (system, release, model, is_simulator). + + If values can't be determined, they are set to values provided as + parameters. + """ + if sys.platform == "ios": + import _ios_support + result = _ios_support.get_platform_ios() + if result is not None: + return IOSVersionInfo(*result) + + return IOSVersionInfo(system, release, model, is_simulator) + + +def _java_getprop(name, default): + """This private helper is deprecated in 3.13 and will be removed in 3.15""" from java.lang import System try: value = System.getProperty(name) @@ -525,9 +544,11 @@ def java_ver(release='', vendor='', vminfo=('', '', ''), osinfo=('', '', '')): given as parameters (which all default to ''). """ + import warnings + warnings._deprecated('java_ver', remove=(3, 15)) # Import the needed APIs try: - import java.lang + import java.lang # noqa: F401 except ImportError: return release, vendor, vminfo, osinfo @@ -546,6 +567,47 @@ def java_ver(release='', vendor='', vminfo=('', '', ''), osinfo=('', '', '')): return release, vendor, vminfo, osinfo + +AndroidVer = collections.namedtuple( + "AndroidVer", "release api_level manufacturer model device is_emulator") + +def android_ver(release="", api_level=0, manufacturer="", model="", device="", + is_emulator=False): + if sys.platform == "android": + try: + from ctypes import CDLL, c_char_p, create_string_buffer + except ImportError: + pass + else: + # An NDK developer confirmed that this is an officially-supported + # API (https://stackoverflow.com/a/28416743). Use `getattr` to avoid + # private name mangling. + system_property_get = getattr(CDLL("libc.so"), "__system_property_get") + system_property_get.argtypes = (c_char_p, c_char_p) + + def getprop(name, default): + # https://android.googlesource.com/platform/bionic/+/refs/tags/android-5.0.0_r1/libc/include/sys/system_properties.h#39 + PROP_VALUE_MAX = 92 + buffer = create_string_buffer(PROP_VALUE_MAX) + length = system_property_get(name.encode("UTF-8"), buffer) + if length == 0: + # This API doesn’t distinguish between an empty property and + # a missing one. + return default + else: + return buffer.value.decode("UTF-8", "backslashreplace") + + release = getprop("ro.build.version.release", release) + api_level = int(getprop("ro.build.version.sdk", api_level)) + manufacturer = getprop("ro.product.manufacturer", manufacturer) + model = getprop("ro.product.model", model) + device = getprop("ro.product.device", device) + is_emulator = getprop("ro.kernel.qemu", "0") == "1" + + return AndroidVer( + release, api_level, manufacturer, model, device, is_emulator) + + ### System name aliasing def system_alias(system, release, version): @@ -617,7 +679,7 @@ def _platform(*args): if cleaned == platform: break platform = cleaned - while platform[-1] == '-': + while platform and platform[-1] == '-': platform = platform[:-1] return platform @@ -658,7 +720,7 @@ def _syscmd_file(target, default=''): default in case the command should fail. """ - if sys.platform in ('dos', 'win32', 'win16'): + if sys.platform in {'dos', 'win32', 'win16', 'ios', 'tvos', 'watchos'}: # XXX Others too ? return default @@ -822,6 +884,14 @@ def get_OpenVMS(): csid, cpu_number = vms_lib.getsyi('SYI$_CPU', 0) return 'Alpha' if cpu_number >= 128 else 'VAX' + # On the iOS simulator, os.uname returns the architecture as uname.machine. + # On device it returns the model name for some reason; but there's only one + # CPU architecture for iOS devices, so we know the right answer. + def get_ios(): + if sys.implementation._multiarch.endswith("simulator"): + return os.uname().machine + return 'arm64' + def from_subprocess(): """ Fall back to `uname -p` @@ -976,6 +1046,15 @@ def uname(): system = 'Windows' release = 'Vista' + # On Android, return the name and version of the OS rather than the kernel. + if sys.platform == 'android': + system = 'Android' + release = android_ver().release + + # Normalize responses on iOS + if sys.platform == 'ios': + system, release, _, _ = ios_ver() + vals = system, node, release, version, machine # Replace 'unknown' values with the more portable '' _uname_cache = uname_result(*map(_unknown_as_blank, vals)) @@ -1074,17 +1153,16 @@ def _sys_version(sys_version=None): if result is not None: return result - sys_version_parser = re.compile( - r'([\w.+]+)\s*' # "version" - r'\(#?([^,]+)' # "(#buildno" - r'(?:,\s*([\w ]*)' # ", builddate" - r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" - r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" - if sys.platform.startswith('java'): # Jython + jython_sys_version_parser = re.compile( + r'([\w.+]+)\s*' # "version" + r'\(#?([^,]+)' # "(#buildno" + r'(?:,\s*([\w ]*)' # ", builddate" + r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" + r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" name = 'Jython' - match = sys_version_parser.match(sys_version) + match = jython_sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse Jython sys.version: %s' % @@ -1111,7 +1189,14 @@ def _sys_version(sys_version=None): else: # CPython - match = sys_version_parser.match(sys_version) + cpython_sys_version_parser = re.compile( + r'([\w.+]+)\s*' # "version" + r'(?:experimental free-threading build\s+)?' # "free-threading-build" + r'\(#?([^,]+)' # "(#buildno" + r'(?:,\s*([\w ]*)' # ", builddate" + r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" + r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" + match = cpython_sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse CPython sys.version: %s' % @@ -1255,11 +1340,14 @@ def platform(aliased=False, terse=False): system, release, version = system_alias(system, release, version) if system == 'Darwin': - # macOS (darwin kernel) - macos_release = mac_ver()[0] - if macos_release: - system = 'macOS' - release = macos_release + # macOS and iOS both report as a "Darwin" kernel + if sys.platform == "ios": + system, release, _, _ = ios_ver() + else: + macos_release = mac_ver()[0] + if macos_release: + system = 'macOS' + release = macos_release if system == 'Windows': # MS platforms @@ -1355,6 +1443,18 @@ def freedesktop_os_release(): return _os_release_cache.copy() +def invalidate_caches(): + """Invalidate the cached results.""" + global _uname_cache + _uname_cache = None + + global _os_release_cache + _os_release_cache = None + + _sys_version_cache.clear() + _platform_cache.clear() + + ### Command line interface if __name__ == '__main__': diff --git a/Lib/poplib.py b/Lib/poplib.py index 1a1629d175b6d9..beb93a0d57cf93 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -309,7 +309,7 @@ def close(self): # optional commands: def rpop(self, user): - """Not sure what this does.""" + """Send RPOP command to access the mailbox with an alternate user.""" return self._shortcmd('RPOP %s' % user) diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 33943b4403636a..db72ded8826056 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -22,6 +22,7 @@ altsep = None devnull = '/dev/null' +import errno import os import sys import stat @@ -35,7 +36,7 @@ "samefile","sameopenfile","samestat", "curdir","pardir","sep","pathsep","defpath","altsep","extsep", "devnull","realpath","supports_unicode_filenames","relpath", - "commonpath", "isjunction"] + "commonpath", "isjunction","isdevdrive"] def _get_sep(path): @@ -77,12 +78,11 @@ def join(a, *p): sep = _get_sep(a) path = a try: - if not p: - path[:0] + sep #23780: Ensure compatible data type even if p is null. - for b in map(os.fspath, p): - if b.startswith(sep): + for b in p: + b = os.fspath(b) + if b.startswith(sep) or not path: path = b - elif not path or path.endswith(sep): + elif path.endswith(sep): path += b else: path += sep + b @@ -135,33 +135,30 @@ def splitdrive(p): return p[:0], p -def splitroot(p): - """Split a pathname into drive, root and tail. On Posix, drive is always - empty; the root may be empty, a single slash, or two slashes. The tail - contains anything after the root. For example: +try: + from posix import _path_splitroot_ex as splitroot +except ImportError: + def splitroot(p): + """Split a pathname into drive, root and tail. - splitroot('foo/bar') == ('', '', 'foo/bar') - splitroot('/foo/bar') == ('', '/', 'foo/bar') - splitroot('//foo/bar') == ('', '//', 'foo/bar') - splitroot('///foo/bar') == ('', '/', '//foo/bar') - """ - p = os.fspath(p) - if isinstance(p, bytes): - sep = b'/' - empty = b'' - else: - sep = '/' - empty = '' - if p[:1] != sep: - # Relative path, e.g.: 'foo' - return empty, empty, p - elif p[1:2] != sep or p[2:3] == sep: - # Absolute path, e.g.: '/foo', '///foo', '////foo', etc. - return empty, sep, p[1:] - else: - # Precisely two leading slashes, e.g.: '//foo'. Implementation defined per POSIX, see - # https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 - return empty, p[:2], p[2:] + The tail contains anything after the root.""" + p = os.fspath(p) + if isinstance(p, bytes): + sep = b'/' + empty = b'' + else: + sep = '/' + empty = '' + if p[:1] != sep: + # Relative path, e.g.: 'foo' + return empty, empty, p + elif p[1:2] != sep or p[2:3] == sep: + # Absolute path, e.g.: '/foo', '///foo', '////foo', etc. + return empty, sep, p[1:] + else: + # Precisely two leading slashes, e.g.: '//foo'. Implementation defined per POSIX, see + # https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 + return empty, p[:2], p[2:] # Return the tail (basename) part of a path, same as split(path)[1]. @@ -187,26 +184,6 @@ def dirname(p): return head -# Is a path a junction? - -def isjunction(path): - """Test whether a path is a junction - Junctions are not a part of posix semantics""" - os.fspath(path) - return False - - -# Being true for dangling symbolic links is also useful. - -def lexists(path): - """Test whether a path exists. Returns True for broken symbolic links""" - try: - os.lstat(path) - except (OSError, ValueError): - return False - return True - - # Is a path a mount point? # (Does this work for all UNIXes? Is it even guaranteed to work by Posix?) @@ -227,21 +204,17 @@ def ismount(path): parent = join(path, b'..') else: parent = join(path, '..') - parent = realpath(parent) try: s2 = os.lstat(parent) - except (OSError, ValueError): - return False + except OSError: + parent = realpath(parent) + try: + s2 = os.lstat(parent) + except OSError: + return False - dev1 = s1.st_dev - dev2 = s2.st_dev - if dev1 != dev2: - return True # path/.. on a different device as path - ino1 = s1.st_ino - ino2 = s2.st_ino - if ino1 == ino2: - return True # path/.. is the same i-node as path - return False + # path/.. on a different device as path or the same i-node as path + return s1.st_dev != s2.st_dev or s1.st_ino == s2.st_ino # Expand paths beginning with '~' or '~user'. @@ -290,7 +263,7 @@ def expanduser(path): return path name = path[1:i] if isinstance(name, bytes): - name = str(name, 'ASCII') + name = os.fsdecode(name) try: pwent = pwd.getpwnam(name) except KeyError: @@ -303,11 +276,8 @@ def expanduser(path): return path if isinstance(path, bytes): userhome = os.fsencode(userhome) - root = b'/' - else: - root = '/' - userhome = userhome.rstrip(root) - return (userhome + path[i:]) or root + userhome = userhome.rstrip(sep) + return (userhome + path[i:]) or sep # Expand paths containing shell variable substitutions. @@ -371,7 +341,7 @@ def expandvars(path): # if it contains symbolic links! try: - from posix import _path_normpath + from posix import _path_normpath as normpath except ImportError: def normpath(path): @@ -379,21 +349,19 @@ def normpath(path): path = os.fspath(path) if isinstance(path, bytes): sep = b'/' - empty = b'' dot = b'.' dotdot = b'..' else: sep = '/' - empty = '' dot = '.' dotdot = '..' - if path == empty: + if not path: return dot _, initial_slashes, path = splitroot(path) comps = path.split(sep) new_comps = [] for comp in comps: - if comp in (empty, dot): + if not comp or comp == dot: continue if (comp != dotdot or (not initial_slashes and not new_comps) or (new_comps and new_comps[-1] == dotdot)): @@ -404,24 +372,16 @@ def normpath(path): path = initial_slashes + sep.join(comps) return path or dot -else: - def normpath(path): - """Normalize path, eliminating double slashes, etc.""" - path = os.fspath(path) - if isinstance(path, bytes): - return os.fsencode(_path_normpath(os.fsdecode(path))) or b"." - return _path_normpath(path) or "." - def abspath(path): """Return an absolute path.""" path = os.fspath(path) - if not isabs(path): - if isinstance(path, bytes): - cwd = os.getcwdb() - else: - cwd = os.getcwd() - path = join(cwd, path) + if isinstance(path, bytes): + if not path.startswith(b'/'): + path = join(os.getcwdb(), path) + else: + if not path.startswith('/'): + path = join(os.getcwd(), path) return normpath(path) @@ -432,72 +392,114 @@ def realpath(filename, *, strict=False): """Return the canonical path of the specified filename, eliminating any symbolic links encountered in the path.""" filename = os.fspath(filename) - path, ok = _joinrealpath(filename[:0], filename, strict, {}) - return abspath(path) - -# Join two paths, normalizing and eliminating any symbolic links -# encountered in the second path. -def _joinrealpath(path, rest, strict, seen): - if isinstance(path, bytes): + if isinstance(filename, bytes): sep = b'/' curdir = b'.' pardir = b'..' + getcwd = os.getcwdb else: sep = '/' curdir = '.' pardir = '..' - - if isabs(rest): - rest = rest[1:] - path = sep - - while rest: - name, _, rest = rest.partition(sep) + getcwd = os.getcwd + return _realpath(filename, strict, sep, curdir, pardir, getcwd) + +def _realpath(filename, strict=False, sep=sep, curdir=curdir, pardir=pardir, + getcwd=os.getcwd, lstat=os.lstat, readlink=os.readlink, maxlinks=None): + # The stack of unresolved path parts. When popped, a special value of None + # indicates that a symlink target has been resolved, and that the original + # symlink path can be retrieved by popping again. The [::-1] slice is a + # very fast way of spelling list(reversed(...)). + rest = filename.split(sep)[::-1] + + # Number of unprocessed parts in 'rest'. This can differ from len(rest) + # later, because 'rest' might contain markers for unresolved symlinks. + part_count = len(rest) + + # The resolved path, which is absolute throughout this function. + # Note: getcwd() returns a normalized and symlink-free path. + path = sep if filename.startswith(sep) else getcwd() + + # Mapping from symlink paths to *fully resolved* symlink targets. If a + # symlink is encountered but not yet resolved, the value is None. This is + # used both to detect symlink loops and to speed up repeated traversals of + # the same links. + seen = {} + + # Number of symlinks traversed. When the number of traversals is limited + # by *maxlinks*, this is used instead of *seen* to detect symlink loops. + link_count = 0 + + while part_count: + name = rest.pop() + if name is None: + # resolved symlink target + seen[rest.pop()] = path + continue + part_count -= 1 if not name or name == curdir: # current dir continue if name == pardir: # parent dir - if path: - path, name = split(path) - if name == pardir: - path = join(path, pardir, pardir) - else: - path = pardir + path = path[:path.rindex(sep)] or sep continue - newpath = join(path, name) + if path == sep: + newpath = path + name + else: + newpath = path + sep + name try: - st = os.lstat(newpath) + st_mode = lstat(newpath).st_mode + if not stat.S_ISLNK(st_mode): + if strict and part_count and not stat.S_ISDIR(st_mode): + raise OSError(errno.ENOTDIR, os.strerror(errno.ENOTDIR), + newpath) + path = newpath + continue + elif maxlinks is not None: + link_count += 1 + if link_count > maxlinks: + if strict: + raise OSError(errno.ELOOP, os.strerror(errno.ELOOP), + newpath) + path = newpath + continue + elif newpath in seen: + # Already seen this path + path = seen[newpath] + if path is not None: + # use cached value + continue + # The symlink is not resolved, so we must have a symlink loop. + if strict: + raise OSError(errno.ELOOP, os.strerror(errno.ELOOP), + newpath) + path = newpath + continue + target = readlink(newpath) except OSError: if strict: raise - is_link = False - else: - is_link = stat.S_ISLNK(st.st_mode) - if not is_link: path = newpath continue # Resolve the symbolic link - if newpath in seen: - # Already seen this path - path = seen[newpath] - if path is not None: - # use cached value - continue - # The symlink is not resolved, so we must have a symlink loop. - if strict: - # Raise OSError(errno.ELOOP) - os.stat(newpath) - else: - # Return already resolved part + rest of the path unchanged. - return join(newpath, rest), False - seen[newpath] = None # not resolved symlink - path, ok = _joinrealpath(path, os.readlink(newpath), strict, seen) - if not ok: - return join(path, rest), False - seen[newpath] = path # resolved symlink + if target.startswith(sep): + # Symlink target is absolute; reset resolved path. + path = sep + if maxlinks is None: + # Mark this symlink as seen but not fully resolved. + seen[newpath] = None + # Push the symlink path onto the stack, and signal its specialness + # by also pushing None. When these entries are popped, we'll + # record the fully-resolved symlink target in the 'seen' mapping. + rest.append(newpath) + rest.append(None) + # Push the unresolved symlink target parts onto the stack. + target_parts = target.split(sep)[::-1] + rest.extend(target_parts) + part_count += len(target_parts) - return path, True + return path supports_unicode_filenames = (sys.platform == 'darwin') @@ -505,10 +507,10 @@ def _joinrealpath(path, rest, strict, seen): def relpath(path, start=None): """Return a relative version of a path""" + path = os.fspath(path) if not path: raise ValueError("no path specified") - path = os.fspath(path) if isinstance(path, bytes): curdir = b'.' sep = b'/' @@ -524,15 +526,17 @@ def relpath(path, start=None): start = os.fspath(start) try: - start_list = [x for x in abspath(start).split(sep) if x] - path_list = [x for x in abspath(path).split(sep) if x] + start_tail = abspath(start).lstrip(sep) + path_tail = abspath(path).lstrip(sep) + start_list = start_tail.split(sep) if start_tail else [] + path_list = path_tail.split(sep) if path_tail else [] # Work out how much of the filepath is shared by start and path. i = len(commonprefix([start_list, path_list])) rel_list = [pardir] * (len(start_list)-i) + path_list[i:] if not rel_list: return curdir - return join(*rel_list) + return sep.join(rel_list) except (TypeError, AttributeError, BytesWarning, DeprecationWarning): genericpath._check_arg_types('relpath', path, start) raise @@ -562,7 +566,7 @@ def commonpath(paths): split_paths = [path.split(sep) for path in paths] try: - isabs, = set(p[:1] == sep for p in paths) + isabs, = {p.startswith(sep) for p in paths} except ValueError: raise ValueError("Can't mix absolute and relative paths") from None diff --git a/Lib/pprint.py b/Lib/pprint.py index 9314701db340c7..dc0953cec67a58 100644 --- a/Lib/pprint.py +++ b/Lib/pprint.py @@ -35,8 +35,6 @@ """ import collections as _collections -import dataclasses as _dataclasses -import re import sys as _sys import types as _types from io import StringIO as _StringIO @@ -54,6 +52,7 @@ def pprint(object, stream=None, indent=1, width=80, depth=None, *, underscore_numbers=underscore_numbers) printer.pprint(object) + def pformat(object, indent=1, width=80, depth=None, *, compact=False, sort_dicts=True, underscore_numbers=False): """Format a Python object into a pretty-printed representation.""" @@ -61,22 +60,27 @@ def pformat(object, indent=1, width=80, depth=None, *, compact=compact, sort_dicts=sort_dicts, underscore_numbers=underscore_numbers).pformat(object) + def pp(object, *args, sort_dicts=False, **kwargs): """Pretty-print a Python object""" pprint(object, *args, sort_dicts=sort_dicts, **kwargs) + def saferepr(object): """Version of repr() which can handle recursive data structures.""" return PrettyPrinter()._safe_repr(object, {}, None, 0)[0] + def isreadable(object): """Determine if saferepr(object) is readable by eval().""" return PrettyPrinter()._safe_repr(object, {}, None, 0)[1] + def isrecursive(object): """Determine if object requires a recursive representation.""" return PrettyPrinter()._safe_repr(object, {}, None, 0)[2] + class _safe_key: """Helper function for key functions when sorting unorderable objects. @@ -99,10 +103,12 @@ def __lt__(self, other): return ((str(type(self.obj)), id(self.obj)) < \ (str(type(other.obj)), id(other.obj))) + def _safe_tuple(t): "Helper function for comparing 2-tuples" return _safe_key(t[0]), _safe_key(t[1]) + class PrettyPrinter: def __init__(self, indent=1, width=80, depth=None, stream=None, *, compact=False, sort_dicts=True, underscore_numbers=False): @@ -179,12 +185,15 @@ def _format(self, object, stream, indent, allowance, context, level): max_width = self._width - indent - allowance if len(rep) > max_width: p = self._dispatch.get(type(object).__repr__, None) + # Lazy import to improve module import time + from dataclasses import is_dataclass + if p is not None: context[objid] = 1 p(self, object, stream, indent, allowance, context, level + 1) del context[objid] return - elif (_dataclasses.is_dataclass(object) and + elif (is_dataclass(object) and not isinstance(object, type) and object.__dataclass_params__.repr and # Check dataclass has generated repr method. @@ -197,9 +206,12 @@ def _format(self, object, stream, indent, allowance, context, level): stream.write(rep) def _pprint_dataclass(self, object, stream, indent, allowance, context, level): + # Lazy import to improve module import time + from dataclasses import fields as dataclass_fields + cls_name = object.__class__.__name__ indent += len(cls_name) + 1 - items = [(f.name, getattr(object, f.name)) for f in _dataclasses.fields(object) if f.repr] + items = [(f.name, getattr(object, f.name)) for f in dataclass_fields(object) if f.repr] stream.write(cls_name + '(') self._format_namespace_items(items, stream, indent, allowance, context, level) stream.write(')') @@ -291,6 +303,9 @@ def _pprint_str(self, object, stream, indent, allowance, context, level): if len(rep) <= max_width1: chunks.append(rep) else: + # Lazy import to improve module import time + import re + # A list of alternating (non-space, space) strings parts = re.findall(r'\S*\s*', line) assert parts @@ -632,9 +647,11 @@ def _safe_repr(self, object, context, maxlevels, level): rep = repr(object) return rep, (rep and not rep.startswith('<')), False + _builtin_scalars = frozenset({str, bytes, bytearray, float, complex, bool, type(None)}) + def _recursion(object): return ("" % (type(object).__name__, id(object))) diff --git a/Lib/profile.py b/Lib/profile.py old mode 100755 new mode 100644 index f2f8c2f21333e0..a5afb12c9d121a --- a/Lib/profile.py +++ b/Lib/profile.py @@ -1,4 +1,3 @@ -#! /usr/bin/env python3 # # Class for profiling python code. rev 1.0 6/2/94 # diff --git a/Lib/pstats.py b/Lib/pstats.py index 2f054bb4011e7f..becaf35580eaee 100644 --- a/Lib/pstats.py +++ b/Lib/pstats.py @@ -29,7 +29,6 @@ from enum import StrEnum, _simple_enum from functools import cmp_to_key from dataclasses import dataclass -from typing import Dict __all__ = ["Stats", "SortKey", "FunctionProfile", "StatsProfile"] @@ -69,7 +68,7 @@ class FunctionProfile: class StatsProfile: '''Class for keeping track of an item in inventory.''' total_tt: float - func_profiles: Dict[str, FunctionProfile] + func_profiles: dict[str, FunctionProfile] class Stats: """This class is used for creating reports from data generated by the @@ -83,7 +82,7 @@ class Stats: method now take arbitrarily many file names as arguments. All the print methods now take an argument that indicates how many lines - to print. If the arg is a floating point number between 0 and 1.0, then + to print. If the arg is a floating-point number between 0 and 1.0, then it is taken as a decimal percentage of the available lines to be printed (e.g., .1 means print 10% of all available lines). If it is an integer, it is taken to mean the number of lines of data that you wish to have @@ -611,7 +610,7 @@ def f8(x): if __name__ == '__main__': import cmd try: - import readline + import readline # noqa: F401 except ImportError: pass diff --git a/Lib/pty.py b/Lib/pty.py index 1d97994abef3c8..4b25ac32c8da14 100644 --- a/Lib/pty.py +++ b/Lib/pty.py @@ -32,27 +32,18 @@ def openpty(): except (AttributeError, OSError): pass master_fd, slave_name = _open_terminal() - slave_fd = slave_open(slave_name) - return master_fd, slave_fd - -def master_open(): - """master_open() -> (master_fd, slave_name) - Open a pty master and return the fd, and the filename of the slave end. - Deprecated, use openpty() instead.""" - - import warnings - warnings.warn("Use pty.openpty() instead.", DeprecationWarning, stacklevel=2) # Remove API in 3.14 + slave_fd = os.open(slave_name, os.O_RDWR) try: - master_fd, slave_fd = os.openpty() - except (AttributeError, OSError): + from fcntl import ioctl, I_PUSH + except ImportError: + return master_fd, slave_fd + try: + ioctl(slave_fd, I_PUSH, "ptem") + ioctl(slave_fd, I_PUSH, "ldterm") + except OSError: pass - else: - slave_name = os.ttyname(slave_fd) - os.close(slave_fd) - return master_fd, slave_name - - return _open_terminal() + return master_fd, slave_fd def _open_terminal(): """Open pty master and return (master_fd, tty_name).""" @@ -66,26 +57,6 @@ def _open_terminal(): return (fd, '/dev/tty' + x + y) raise OSError('out of pty devices') -def slave_open(tty_name): - """slave_open(tty_name) -> slave_fd - Open the pty slave and acquire the controlling terminal, returning - opened filedescriptor. - Deprecated, use openpty() instead.""" - - import warnings - warnings.warn("Use pty.openpty() instead.", DeprecationWarning, stacklevel=2) # Remove API in 3.14 - - result = os.open(tty_name, os.O_RDWR) - try: - from fcntl import ioctl, I_PUSH - except ImportError: - return result - try: - ioctl(result, I_PUSH, "ptem") - ioctl(result, I_PUSH, "ldterm") - except OSError: - pass - return result def fork(): """fork() -> (pid, master_fd) diff --git a/Lib/pydoc.py b/Lib/pydoc.py old mode 100755 new mode 100644 index 407c0205c7ab66..922946e5fa7ddb --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """Generate Python documentation in HTML or text for interactive use. At the Python interactive prompt, calling help(thing) on a Python object @@ -54,6 +53,7 @@ class or function within a module or module in a package. If the # the current directory is changed with os.chdir(), an incorrect # path will be displayed. +import ast import __future__ import builtins import importlib._bootstrap @@ -72,10 +72,26 @@ class or function within a module or module in a package. If the import tokenize import urllib.parse import warnings +from annotationlib import Format from collections import deque from reprlib import Repr from traceback import format_exception_only +from _pyrepl.pager import (get_pager, pipe_pager, + plain_pager, tempfile_pager, tty_pager) + +# Expose plain() as pydoc.plain() +from _pyrepl.pager import plain # noqa: F401 + + +# --------------------------------------------------------- old names + +getpager = get_pager +pipepager = pipe_pager +plainpager = plain_pager +tempfilepager = tempfile_pager +ttypager = tty_pager + # --------------------------------------------------------- common routines @@ -198,12 +214,12 @@ def splitdoc(doc): def _getargspec(object): try: - signature = inspect.signature(object) + signature = inspect.signature(object, annotation_format=Format.STRING) if signature: name = getattr(object, '__name__', '') # function are always single-line and should not be formatted max_width = (80 - len(name)) if name != '' else None - return signature.format(max_width=max_width) + return signature.format(max_width=max_width, quote_annotation_strings=False) except (ValueError, TypeError): argspec = getattr(object, '__text_signature__', None) if argspec: @@ -313,7 +329,8 @@ def visiblename(name, all=None, obj=None): if name in {'__author__', '__builtins__', '__cached__', '__credits__', '__date__', '__doc__', '__file__', '__spec__', '__loader__', '__module__', '__name__', '__package__', - '__path__', '__qualname__', '__slots__', '__version__'}: + '__path__', '__qualname__', '__slots__', '__version__', + '__static_attributes__', '__firstlineno__'}: return 0 # Private names are hidden, but special names are displayed. if name.startswith('__') and name.endswith('__'): return 1 @@ -368,21 +385,29 @@ def ispackage(path): return False def source_synopsis(file): - line = file.readline() - while line[:1] == '#' or not line.strip(): - line = file.readline() - if not line: break - line = line.strip() - if line[:4] == 'r"""': line = line[1:] - if line[:3] == '"""': - line = line[3:] - if line[-1:] == '\\': line = line[:-1] - while not line.strip(): - line = file.readline() - if not line: break - result = line.split('"""')[0].strip() - else: result = None - return result + """Return the one-line summary of a file object, if present""" + + string = '' + try: + tokens = tokenize.generate_tokens(file.readline) + for tok_type, tok_string, _, _, _ in tokens: + if tok_type == tokenize.STRING: + string += tok_string + elif tok_type == tokenize.NEWLINE: + with warnings.catch_warnings(): + # Ignore the "invalid escape sequence" warning. + warnings.simplefilter("ignore", SyntaxWarning) + docstring = ast.literal_eval(string) + if not isinstance(docstring, str): + return None + return docstring.strip().split('\n')[0].strip() + elif tok_type == tokenize.OP and tok_string in ('(', ')'): + string += tok_string + elif tok_type not in (tokenize.COMMENT, tokenize.NL, tokenize.ENCODING): + return None + except (tokenize.TokenError, UnicodeDecodeError, SyntaxError): + return None + return None def synopsis(filename, cache={}): """Get the one-line summary out of a module file.""" @@ -1410,7 +1435,8 @@ def makename(c, m=object.__module__): # List the built-in subclasses, if any: subclasses = sorted( (str(cls.__name__) for cls in type.__subclasses__(object) - if not cls.__name__.startswith("_") and cls.__module__ == "builtins"), + if (not cls.__name__.startswith("_") and + getattr(cls, '__module__', '') == "builtins")), key=str.lower ) no_of_subclasses = len(subclasses) @@ -1636,149 +1662,11 @@ def bold(self, text): # --------------------------------------------------------- user interfaces -def pager(text): +def pager(text, title=''): """The first time this is called, determine what kind of pager to use.""" global pager - pager = getpager() - pager(text) - -def getpager(): - """Decide what method to use for paging through text.""" - if not hasattr(sys.stdin, "isatty"): - return plainpager - if not hasattr(sys.stdout, "isatty"): - return plainpager - if not sys.stdin.isatty() or not sys.stdout.isatty(): - return plainpager - if sys.platform == "emscripten": - return plainpager - use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER') - if use_pager: - if sys.platform == 'win32': # pipes completely broken in Windows - return lambda text: tempfilepager(plain(text), use_pager) - elif os.environ.get('TERM') in ('dumb', 'emacs'): - return lambda text: pipepager(plain(text), use_pager) - else: - return lambda text: pipepager(text, use_pager) - if os.environ.get('TERM') in ('dumb', 'emacs'): - return plainpager - if sys.platform == 'win32': - return lambda text: tempfilepager(plain(text), 'more <') - if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: - return lambda text: pipepager(text, 'less') - - import tempfile - (fd, filename) = tempfile.mkstemp() - os.close(fd) - try: - if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0: - return lambda text: pipepager(text, 'more') - else: - return ttypager - finally: - os.unlink(filename) - -def plain(text): - """Remove boldface formatting from text.""" - return re.sub('.\b', '', text) - -def pipepager(text, cmd): - """Page through text by feeding it to another program.""" - import subprocess - env = os.environ.copy() - prompt_string = ( - ' ' - '?ltline %lt?L/%L.' - ':byte %bB?s/%s.' - '.' - '?e (END):?pB %pB\\%..' - ' (press h for help or q to quit)') - env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string) - proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, - errors='backslashreplace', env=env) - try: - with proc.stdin as pipe: - try: - pipe.write(text) - except KeyboardInterrupt: - # We've hereby abandoned whatever text hasn't been written, - # but the pager is still in control of the terminal. - pass - except OSError: - pass # Ignore broken pipes caused by quitting the pager program. - while True: - try: - proc.wait() - break - except KeyboardInterrupt: - # Ignore ctl-c like the pager itself does. Otherwise the pager is - # left running and the terminal is in raw mode and unusable. - pass - -def tempfilepager(text, cmd): - """Page through text by invoking a program on a temporary file.""" - import tempfile - with tempfile.TemporaryDirectory() as tempdir: - filename = os.path.join(tempdir, 'pydoc.out') - with open(filename, 'w', errors='backslashreplace', - encoding=os.device_encoding(0) if - sys.platform == 'win32' else None - ) as file: - file.write(text) - os.system(cmd + ' "' + filename + '"') - -def _escape_stdout(text): - # Escape non-encodable characters to avoid encoding errors later - encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' - return text.encode(encoding, 'backslashreplace').decode(encoding) - -def ttypager(text): - """Page through text on a text terminal.""" - lines = plain(_escape_stdout(text)).split('\n') - try: - import tty - fd = sys.stdin.fileno() - old = tty.tcgetattr(fd) - tty.setcbreak(fd) - getchar = lambda: sys.stdin.read(1) - except (ImportError, AttributeError, io.UnsupportedOperation): - tty = None - getchar = lambda: sys.stdin.readline()[:-1][:1] - - try: - try: - h = int(os.environ.get('LINES', 0)) - except ValueError: - h = 0 - if h <= 1: - h = 25 - r = inc = h - 1 - sys.stdout.write('\n'.join(lines[:inc]) + '\n') - while lines[r:]: - sys.stdout.write('-- more --') - sys.stdout.flush() - c = getchar() - - if c in ('q', 'Q'): - sys.stdout.write('\r \r') - break - elif c in ('\r', '\n'): - sys.stdout.write('\r \r' + lines[r] + '\n') - r = r + 1 - continue - if c in ('b', 'B', '\x1b'): - r = r - inc - inc - if r < 0: r = 0 - sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n') - r = r + inc - - finally: - if tty: - tty.tcsetattr(fd, tty.TCSAFLUSH, old) - -def plainpager(text): - """Simply print unformatted text. This is the ultimate fallback.""" - sys.stdout.write(plain(_escape_stdout(text))) + pager = get_pager() + pager(text, title) def describe(thing): """Produce a short description of the given thing.""" @@ -1805,6 +1693,13 @@ def describe(thing): return 'function ' + thing.__name__ if inspect.ismethod(thing): return 'method ' + thing.__name__ + if inspect.ismethodwrapper(thing): + return 'method wrapper ' + thing.__name__ + if inspect.ismethoddescriptor(thing): + try: + return 'method descriptor ' + thing.__name__ + except AttributeError: + pass return type(thing).__name__ def locate(path, forceload=0): @@ -1878,7 +1773,15 @@ def doc(thing, title='Python Library Documentation: %s', forceload=0, """Display text documentation, given an object or a path to an object.""" if output is None: try: - pager(render_doc(thing, title, forceload)) + if isinstance(thing, str): + what = thing + else: + what = getattr(thing, '__qualname__', None) + if not isinstance(what, str): + what = getattr(thing, '__name__', None) + if not isinstance(what, str): + what = type(thing).__name__ + ' object' + pager(render_doc(thing, title, forceload), f'Help on {what!s}') except ImportError as exc: if is_cli: raise @@ -1978,6 +1881,7 @@ class Helper: ':': 'SLICINGS DICTIONARYLITERALS', '@': 'def class', '\\': 'STRINGS', + ':=': 'ASSIGNMENTEXPRESSIONS', '_': 'PRIVATENAMES', '__': 'PRIVATENAMES SPECIALMETHODS', '`': 'BACKQUOTES', @@ -2071,6 +1975,7 @@ class Helper: 'ASSERTION': 'assert', 'ASSIGNMENT': ('assignment', 'AUGMENTEDASSIGNMENT'), 'AUGMENTEDASSIGNMENT': ('augassign', 'NUMBERMETHODS'), + 'ASSIGNMENTEXPRESSIONS': ('assignment-expressions', ''), 'DELETION': 'del', 'RETURNING': 'return', 'IMPORTING': 'import', @@ -2132,7 +2037,7 @@ def interact(self): if (len(request) > 2 and request[0] == request[-1] in ("'", '"') and request[0] not in request[1:-1]): request = request[1:-1] - if request.lower() in ('q', 'quit'): break + if request.lower() in ('q', 'quit', 'exit'): break if request == 'help': self.intro() else: @@ -2159,7 +2064,7 @@ def help(self, request, is_cli=False): elif request in self.symbols: self.showsymbol(request) elif request in ['True', 'False', 'None']: # special case these keywords since they are objects too - doc(eval(request), 'Help on %s:', is_cli=is_cli) + doc(eval(request), 'Help on %s:', output=self._output, is_cli=is_cli) elif request in self.keywords: self.showtopic(request) elif request in self.topics: self.showtopic(request) elif request: doc(request, 'Help on %s:', output=self._output, is_cli=is_cli) @@ -2184,7 +2089,7 @@ def intro(self): enter "modules spam". To quit this help utility and return to the interpreter, -enter "q" or "quit". +enter "q", "quit" or "exit". '''.format('%d.%d' % sys.version_info[:2])) def list(self, items, columns=4, width=80): @@ -2252,7 +2157,11 @@ def showtopic(self, topic, more_xrefs=''): text = 'Related help topics: ' + ', '.join(xrefs.split()) + '\n' wrapped_text = textwrap.wrap(text, 72) doc += '\n%s\n' % '\n'.join(wrapped_text) - pager(doc) + + if self._output is None: + pager(doc, f'Help on {topic!s}') + else: + self.output.write(doc) def _gettopic(self, topic, more_xrefs=''): """Return unbuffered tuple of (topic, xrefs). @@ -2504,6 +2413,7 @@ def __init__(self, urlhandler, host, port): threading.Thread.__init__(self) self.serving = False self.error = None + self.docserver = None def run(self): """Start the server.""" @@ -2536,9 +2446,9 @@ def stop(self): thread = ServerThread(urlhandler, hostname, port) thread.start() - # Wait until thread.serving is True to make sure we are - # really up before returning. - while not thread.error and not thread.serving: + # Wait until thread.serving is True and thread.docserver is set + # to make sure we are really up before returning. + while not thread.error and not (thread.serving and thread.docserver): time.sleep(.01) return thread diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index a49c38a51d39f2..3e8c7ce321edc2 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,16679 +1,12781 @@ -# -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Thu Feb 15 14:30:52 2024 +# Autogenerated by Sphinx on Tue Jan 21 03:33:33 2025 # as part of the release process. -topics = {'assert': 'The "assert" statement\n' - '**********************\n' - '\n' - 'Assert statements are a convenient way to insert debugging ' - 'assertions\n' - 'into a program:\n' - '\n' - ' assert_stmt ::= "assert" expression ["," expression]\n' - '\n' - 'The simple form, "assert expression", is equivalent to\n' - '\n' - ' if __debug__:\n' - ' if not expression: raise AssertionError\n' - '\n' - 'The extended form, "assert expression1, expression2", is ' - 'equivalent to\n' - '\n' - ' if __debug__:\n' - ' if not expression1: raise AssertionError(expression2)\n' - '\n' - 'These equivalences assume that "__debug__" and "AssertionError" ' - 'refer\n' - 'to the built-in variables with those names. In the current\n' - 'implementation, the built-in variable "__debug__" is "True" under\n' - 'normal circumstances, "False" when optimization is requested ' - '(command\n' - 'line option "-O"). The current code generator emits no code for ' - 'an\n' - 'assert statement when optimization is requested at compile time. ' - 'Note\n' - 'that it is unnecessary to include the source code for the ' - 'expression\n' - 'that failed in the error message; it will be displayed as part of ' - 'the\n' - 'stack trace.\n' - '\n' - 'Assignments to "__debug__" are illegal. The value for the ' - 'built-in\n' - 'variable is determined when the interpreter starts.\n', - 'assignment': 'Assignment statements\n' - '*********************\n' - '\n' - 'Assignment statements are used to (re)bind names to values and ' - 'to\n' - 'modify attributes or items of mutable objects:\n' - '\n' - ' assignment_stmt ::= (target_list "=")+ (starred_expression ' - '| yield_expression)\n' - ' target_list ::= target ("," target)* [","]\n' - ' target ::= identifier\n' - ' | "(" [target_list] ")"\n' - ' | "[" [target_list] "]"\n' - ' | attributeref\n' - ' | subscription\n' - ' | slicing\n' - ' | "*" target\n' - '\n' - '(See section Primaries for the syntax definitions for ' - '*attributeref*,\n' - '*subscription*, and *slicing*.)\n' - '\n' - 'An assignment statement evaluates the expression list ' - '(remember that\n' - 'this can be a single expression or a comma-separated list, the ' - 'latter\n' - 'yielding a tuple) and assigns the single resulting object to ' - 'each of\n' - 'the target lists, from left to right.\n' - '\n' - 'Assignment is defined recursively depending on the form of the ' - 'target\n' - '(list). When a target is part of a mutable object (an ' - 'attribute\n' - 'reference, subscription or slicing), the mutable object must\n' - 'ultimately perform the assignment and decide about its ' - 'validity, and\n' - 'may raise an exception if the assignment is unacceptable. The ' - 'rules\n' - 'observed by various types and the exceptions raised are given ' - 'with the\n' - 'definition of the object types (see section The standard type\n' - 'hierarchy).\n' - '\n' - 'Assignment of an object to a target list, optionally enclosed ' - 'in\n' - 'parentheses or square brackets, is recursively defined as ' - 'follows.\n' - '\n' - '* If the target list is a single target with no trailing ' - 'comma,\n' - ' optionally in parentheses, the object is assigned to that ' - 'target.\n' - '\n' - '* Else:\n' - '\n' - ' * If the target list contains one target prefixed with an ' - 'asterisk,\n' - ' called a “starred” target: The object must be an iterable ' - 'with at\n' - ' least as many items as there are targets in the target ' - 'list, minus\n' - ' one. The first items of the iterable are assigned, from ' - 'left to\n' - ' right, to the targets before the starred target. The ' - 'final items\n' - ' of the iterable are assigned to the targets after the ' - 'starred\n' - ' target. A list of the remaining items in the iterable is ' - 'then\n' - ' assigned to the starred target (the list can be empty).\n' - '\n' - ' * Else: The object must be an iterable with the same number ' - 'of items\n' - ' as there are targets in the target list, and the items ' - 'are\n' - ' assigned, from left to right, to the corresponding ' - 'targets.\n' - '\n' - 'Assignment of an object to a single target is recursively ' - 'defined as\n' - 'follows.\n' - '\n' - '* If the target is an identifier (name):\n' - '\n' - ' * If the name does not occur in a "global" or "nonlocal" ' - 'statement\n' - ' in the current code block: the name is bound to the object ' - 'in the\n' - ' current local namespace.\n' - '\n' - ' * Otherwise: the name is bound to the object in the global ' - 'namespace\n' - ' or the outer namespace determined by "nonlocal", ' - 'respectively.\n' - '\n' - ' The name is rebound if it was already bound. This may cause ' - 'the\n' - ' reference count for the object previously bound to the name ' - 'to reach\n' - ' zero, causing the object to be deallocated and its ' - 'destructor (if it\n' - ' has one) to be called.\n' - '\n' - '* If the target is an attribute reference: The primary ' - 'expression in\n' - ' the reference is evaluated. It should yield an object with\n' - ' assignable attributes; if this is not the case, "TypeError" ' - 'is\n' - ' raised. That object is then asked to assign the assigned ' - 'object to\n' - ' the given attribute; if it cannot perform the assignment, it ' - 'raises\n' - ' an exception (usually but not necessarily ' - '"AttributeError").\n' - '\n' - ' Note: If the object is a class instance and the attribute ' - 'reference\n' - ' occurs on both sides of the assignment operator, the ' - 'right-hand side\n' - ' expression, "a.x" can access either an instance attribute or ' - '(if no\n' - ' instance attribute exists) a class attribute. The left-hand ' - 'side\n' - ' target "a.x" is always set as an instance attribute, ' - 'creating it if\n' - ' necessary. Thus, the two occurrences of "a.x" do not ' - 'necessarily\n' - ' refer to the same attribute: if the right-hand side ' - 'expression\n' - ' refers to a class attribute, the left-hand side creates a ' - 'new\n' - ' instance attribute as the target of the assignment:\n' - '\n' - ' class Cls:\n' - ' x = 3 # class variable\n' - ' inst = Cls()\n' - ' inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x ' - 'as 3\n' - '\n' - ' This description does not necessarily apply to descriptor\n' - ' attributes, such as properties created with "property()".\n' - '\n' - '* If the target is a subscription: The primary expression in ' - 'the\n' - ' reference is evaluated. It should yield either a mutable ' - 'sequence\n' - ' object (such as a list) or a mapping object (such as a ' - 'dictionary).\n' - ' Next, the subscript expression is evaluated.\n' - '\n' - ' If the primary is a mutable sequence object (such as a ' - 'list), the\n' - ' subscript must yield an integer. If it is negative, the ' - 'sequence’s\n' - ' length is added to it. The resulting value must be a ' - 'nonnegative\n' - ' integer less than the sequence’s length, and the sequence is ' - 'asked\n' - ' to assign the assigned object to its item with that index. ' - 'If the\n' - ' index is out of range, "IndexError" is raised (assignment to ' - 'a\n' - ' subscripted sequence cannot add new items to a list).\n' - '\n' - ' If the primary is a mapping object (such as a dictionary), ' - 'the\n' - ' subscript must have a type compatible with the mapping’s key ' - 'type,\n' - ' and the mapping is then asked to create a key/value pair ' - 'which maps\n' - ' the subscript to the assigned object. This can either ' - 'replace an\n' - ' existing key/value pair with the same key value, or insert a ' - 'new\n' - ' key/value pair (if no key with the same value existed).\n' - '\n' - ' For user-defined objects, the "__setitem__()" method is ' - 'called with\n' - ' appropriate arguments.\n' - '\n' - '* If the target is a slicing: The primary expression in the ' - 'reference\n' - ' is evaluated. It should yield a mutable sequence object ' - '(such as a\n' - ' list). The assigned object should be a sequence object of ' - 'the same\n' - ' type. Next, the lower and upper bound expressions are ' - 'evaluated,\n' - ' insofar they are present; defaults are zero and the ' - 'sequence’s\n' - ' length. The bounds should evaluate to integers. If either ' - 'bound is\n' - ' negative, the sequence’s length is added to it. The ' - 'resulting\n' - ' bounds are clipped to lie between zero and the sequence’s ' - 'length,\n' - ' inclusive. Finally, the sequence object is asked to replace ' - 'the\n' - ' slice with the items of the assigned sequence. The length ' - 'of the\n' - ' slice may be different from the length of the assigned ' - 'sequence,\n' - ' thus changing the length of the target sequence, if the ' - 'target\n' - ' sequence allows it.\n' - '\n' - '**CPython implementation detail:** In the current ' - 'implementation, the\n' - 'syntax for targets is taken to be the same as for expressions, ' - 'and\n' - 'invalid syntax is rejected during the code generation phase, ' - 'causing\n' - 'less detailed error messages.\n' - '\n' - 'Although the definition of assignment implies that overlaps ' - 'between\n' - 'the left-hand side and the right-hand side are ‘simultaneous’ ' - '(for\n' - 'example "a, b = b, a" swaps two variables), overlaps *within* ' - 'the\n' - 'collection of assigned-to variables occur left-to-right, ' - 'sometimes\n' - 'resulting in confusion. For instance, the following program ' - 'prints\n' - '"[0, 2]":\n' - '\n' - ' x = [0, 1]\n' - ' i = 0\n' - ' i, x[i] = 1, 2 # i is updated, then x[i] is ' - 'updated\n' - ' print(x)\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3132** - Extended Iterable Unpacking\n' - ' The specification for the "*target" feature.\n' - '\n' - '\n' - 'Augmented assignment statements\n' - '===============================\n' - '\n' - 'Augmented assignment is the combination, in a single ' - 'statement, of a\n' - 'binary operation and an assignment statement:\n' - '\n' - ' augmented_assignment_stmt ::= augtarget augop ' - '(expression_list | yield_expression)\n' - ' augtarget ::= identifier | attributeref | ' - 'subscription | slicing\n' - ' augop ::= "+=" | "-=" | "*=" | "@=" | ' - '"/=" | "//=" | "%=" | "**="\n' - ' | ">>=" | "<<=" | "&=" | "^=" | "|="\n' - '\n' - '(See section Primaries for the syntax definitions of the last ' - 'three\n' - 'symbols.)\n' - '\n' - 'An augmented assignment evaluates the target (which, unlike ' - 'normal\n' - 'assignment statements, cannot be an unpacking) and the ' - 'expression\n' - 'list, performs the binary operation specific to the type of ' - 'assignment\n' - 'on the two operands, and assigns the result to the original ' - 'target.\n' - 'The target is only evaluated once.\n' - '\n' - 'An augmented assignment expression like "x += 1" can be ' - 'rewritten as\n' - '"x = x + 1" to achieve a similar, but not exactly equal ' - 'effect. In the\n' - 'augmented version, "x" is only evaluated once. Also, when ' - 'possible,\n' - 'the actual operation is performed *in-place*, meaning that ' - 'rather than\n' - 'creating a new object and assigning that to the target, the ' - 'old object\n' - 'is modified instead.\n' - '\n' - 'Unlike normal assignments, augmented assignments evaluate the ' - 'left-\n' - 'hand side *before* evaluating the right-hand side. For ' - 'example, "a[i]\n' - '+= f(x)" first looks-up "a[i]", then it evaluates "f(x)" and ' - 'performs\n' - 'the addition, and lastly, it writes the result back to ' - '"a[i]".\n' - '\n' - 'With the exception of assigning to tuples and multiple targets ' - 'in a\n' - 'single statement, the assignment done by augmented assignment\n' - 'statements is handled the same way as normal assignments. ' - 'Similarly,\n' - 'with the exception of the possible *in-place* behavior, the ' - 'binary\n' - 'operation performed by augmented assignment is the same as the ' - 'normal\n' - 'binary operations.\n' - '\n' - 'For targets which are attribute references, the same caveat ' - 'about\n' - 'class and instance attributes applies as for regular ' - 'assignments.\n' - '\n' - '\n' - 'Annotated assignment statements\n' - '===============================\n' - '\n' - '*Annotation* assignment is the combination, in a single ' - 'statement, of\n' - 'a variable or attribute annotation and an optional assignment\n' - 'statement:\n' - '\n' - ' annotated_assignment_stmt ::= augtarget ":" expression\n' - ' ["=" (starred_expression | ' - 'yield_expression)]\n' - '\n' - 'The difference from normal Assignment statements is that only ' - 'a single\n' - 'target is allowed.\n' - '\n' - 'For simple names as assignment targets, if in class or module ' - 'scope,\n' - 'the annotations are evaluated and stored in a special class or ' - 'module\n' - 'attribute "__annotations__" that is a dictionary mapping from ' - 'variable\n' - 'names (mangled if private) to evaluated annotations. This ' - 'attribute is\n' - 'writable and is automatically created at the start of class or ' - 'module\n' - 'body execution, if annotations are found statically.\n' - '\n' - 'For expressions as assignment targets, the annotations are ' - 'evaluated\n' - 'if in class or module scope, but not stored.\n' - '\n' - 'If a name is annotated in a function scope, then this name is ' - 'local\n' - 'for that scope. Annotations are never evaluated and stored in ' - 'function\n' - 'scopes.\n' - '\n' - 'If the right hand side is present, an annotated assignment ' - 'performs\n' - 'the actual assignment before evaluating annotations (where\n' - 'applicable). If the right hand side is not present for an ' - 'expression\n' - 'target, then the interpreter evaluates the target except for ' - 'the last\n' - '"__setitem__()" or "__setattr__()" call.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 526** - Syntax for Variable Annotations\n' - ' The proposal that added syntax for annotating the types ' - 'of\n' - ' variables (including class variables and instance ' - 'variables),\n' - ' instead of expressing them through comments.\n' - '\n' - ' **PEP 484** - Type hints\n' - ' The proposal that added the "typing" module to provide a ' - 'standard\n' - ' syntax for type annotations that can be used in static ' - 'analysis\n' - ' tools and IDEs.\n' - '\n' - 'Changed in version 3.8: Now annotated assignments allow the ' - 'same\n' - 'expressions in the right hand side as regular assignments. ' - 'Previously,\n' - 'some expressions (like un-parenthesized tuple expressions) ' - 'caused a\n' - 'syntax error.\n', - 'async': 'Coroutines\n' - '**********\n' - '\n' - 'New in version 3.5.\n' - '\n' - '\n' - 'Coroutine function definition\n' - '=============================\n' - '\n' - ' async_funcdef ::= [decorators] "async" "def" funcname "(" ' - '[parameter_list] ")"\n' - ' ["->" expression] ":" suite\n' - '\n' - 'Execution of Python coroutines can be suspended and resumed at ' - 'many\n' - 'points (see *coroutine*). "await" expressions, "async for" and ' - '"async\n' - 'with" can only be used in the body of a coroutine function.\n' - '\n' - 'Functions defined with "async def" syntax are always coroutine\n' - 'functions, even if they do not contain "await" or "async" ' - 'keywords.\n' - '\n' - 'It is a "SyntaxError" to use a "yield from" expression inside the ' - 'body\n' - 'of a coroutine function.\n' - '\n' - 'An example of a coroutine function:\n' - '\n' - ' async def func(param1, param2):\n' - ' do_stuff()\n' - ' await some_coroutine()\n' - '\n' - 'Changed in version 3.7: "await" and "async" are now keywords;\n' - 'previously they were only treated as such inside the body of a\n' - 'coroutine function.\n' - '\n' - '\n' - 'The "async for" statement\n' - '=========================\n' - '\n' - ' async_for_stmt ::= "async" for_stmt\n' - '\n' - 'An *asynchronous iterable* provides an "__aiter__" method that\n' - 'directly returns an *asynchronous iterator*, which can call\n' - 'asynchronous code in its "__anext__" method.\n' - '\n' - 'The "async for" statement allows convenient iteration over\n' - 'asynchronous iterables.\n' - '\n' - 'The following code:\n' - '\n' - ' async for TARGET in ITER:\n' - ' SUITE\n' - ' else:\n' - ' SUITE2\n' - '\n' - 'Is semantically equivalent to:\n' - '\n' - ' iter = (ITER)\n' - ' iter = type(iter).__aiter__(iter)\n' - ' running = True\n' - '\n' - ' while running:\n' - ' try:\n' - ' TARGET = await type(iter).__anext__(iter)\n' - ' except StopAsyncIteration:\n' - ' running = False\n' - ' else:\n' - ' SUITE\n' - ' else:\n' - ' SUITE2\n' - '\n' - 'See also "__aiter__()" and "__anext__()" for details.\n' - '\n' - 'It is a "SyntaxError" to use an "async for" statement outside the ' - 'body\n' - 'of a coroutine function.\n' - '\n' - '\n' - 'The "async with" statement\n' - '==========================\n' - '\n' - ' async_with_stmt ::= "async" with_stmt\n' - '\n' - 'An *asynchronous context manager* is a *context manager* that is ' - 'able\n' - 'to suspend execution in its *enter* and *exit* methods.\n' - '\n' - 'The following code:\n' - '\n' - ' async with EXPRESSION as TARGET:\n' - ' SUITE\n' - '\n' - 'is semantically equivalent to:\n' - '\n' - ' manager = (EXPRESSION)\n' - ' aenter = type(manager).__aenter__\n' - ' aexit = type(manager).__aexit__\n' - ' value = await aenter(manager)\n' - ' hit_except = False\n' - '\n' - ' try:\n' - ' TARGET = value\n' - ' SUITE\n' - ' except:\n' - ' hit_except = True\n' - ' if not await aexit(manager, *sys.exc_info()):\n' - ' raise\n' - ' finally:\n' - ' if not hit_except:\n' - ' await aexit(manager, None, None, None)\n' - '\n' - 'See also "__aenter__()" and "__aexit__()" for details.\n' - '\n' - 'It is a "SyntaxError" to use an "async with" statement outside the\n' - 'body of a coroutine function.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 492** - Coroutines with async and await syntax\n' - ' The proposal that made coroutines a proper standalone concept ' - 'in\n' - ' Python, and added supporting syntax.\n', - 'atom-identifiers': 'Identifiers (Names)\n' - '*******************\n' - '\n' - 'An identifier occurring as an atom is a name. See ' - 'section Identifiers\n' - 'and keywords for lexical definition and section Naming ' - 'and binding for\n' - 'documentation of naming and binding.\n' - '\n' - 'When the name is bound to an object, evaluation of the ' - 'atom yields\n' - 'that object. When a name is not bound, an attempt to ' - 'evaluate it\n' - 'raises a "NameError" exception.\n' - '\n' - '**Private name mangling:** When an identifier that ' - 'textually occurs in\n' - 'a class definition begins with two or more underscore ' - 'characters and\n' - 'does not end in two or more underscores, it is ' - 'considered a *private\n' - 'name* of that class. Private names are transformed to a ' - 'longer form\n' - 'before code is generated for them. The transformation ' - 'inserts the\n' - 'class name, with leading underscores removed and a ' - 'single underscore\n' - 'inserted, in front of the name. For example, the ' - 'identifier "__spam"\n' - 'occurring in a class named "Ham" will be transformed to ' - '"_Ham__spam".\n' - 'This transformation is independent of the syntactical ' - 'context in which\n' - 'the identifier is used. If the transformed name is ' - 'extremely long\n' - '(longer than 255 characters), implementation defined ' - 'truncation may\n' - 'happen. If the class name consists only of underscores, ' - 'no\n' - 'transformation is done.\n', - 'atom-literals': 'Literals\n' - '********\n' - '\n' - 'Python supports string and bytes literals and various ' - 'numeric\n' - 'literals:\n' - '\n' - ' literal ::= stringliteral | bytesliteral\n' - ' | integer | floatnumber | imagnumber\n' - '\n' - 'Evaluation of a literal yields an object of the given type ' - '(string,\n' - 'bytes, integer, floating point number, complex number) with ' - 'the given\n' - 'value. The value may be approximated in the case of ' - 'floating point\n' - 'and imaginary (complex) literals. See section Literals for ' - 'details.\n' - '\n' - 'All literals correspond to immutable data types, and hence ' - 'the\n' - 'object’s identity is less important than its value. ' - 'Multiple\n' - 'evaluations of literals with the same value (either the ' - 'same\n' - 'occurrence in the program text or a different occurrence) ' - 'may obtain\n' - 'the same object or a different object with the same ' - 'value.\n', - 'attribute-access': 'Customizing attribute access\n' - '****************************\n' - '\n' - 'The following methods can be defined to customize the ' - 'meaning of\n' - 'attribute access (use of, assignment to, or deletion of ' - '"x.name") for\n' - 'class instances.\n' - '\n' - 'object.__getattr__(self, name)\n' - '\n' - ' Called when the default attribute access fails with ' - 'an\n' - ' "AttributeError" (either "__getattribute__()" raises ' - 'an\n' - ' "AttributeError" because *name* is not an instance ' - 'attribute or an\n' - ' attribute in the class tree for "self"; or ' - '"__get__()" of a *name*\n' - ' property raises "AttributeError"). This method ' - 'should either\n' - ' return the (computed) attribute value or raise an ' - '"AttributeError"\n' - ' exception.\n' - '\n' - ' Note that if the attribute is found through the ' - 'normal mechanism,\n' - ' "__getattr__()" is not called. (This is an ' - 'intentional asymmetry\n' - ' between "__getattr__()" and "__setattr__()".) This is ' - 'done both for\n' - ' efficiency reasons and because otherwise ' - '"__getattr__()" would have\n' - ' no way to access other attributes of the instance. ' - 'Note that at\n' - ' least for instance variables, you can take total ' - 'control by not\n' - ' inserting any values in the instance attribute ' - 'dictionary (but\n' - ' instead inserting them in another object). See the\n' - ' "__getattribute__()" method below for a way to ' - 'actually get total\n' - ' control over attribute access.\n' - '\n' - 'object.__getattribute__(self, name)\n' - '\n' - ' Called unconditionally to implement attribute ' - 'accesses for\n' - ' instances of the class. If the class also defines ' - '"__getattr__()",\n' - ' the latter will not be called unless ' - '"__getattribute__()" either\n' - ' calls it explicitly or raises an "AttributeError". ' - 'This method\n' - ' should return the (computed) attribute value or raise ' - 'an\n' - ' "AttributeError" exception. In order to avoid ' - 'infinite recursion in\n' - ' this method, its implementation should always call ' - 'the base class\n' - ' method with the same name to access any attributes it ' - 'needs, for\n' - ' example, "object.__getattribute__(self, name)".\n' - '\n' - ' Note:\n' - '\n' - ' This method may still be bypassed when looking up ' - 'special methods\n' - ' as the result of implicit invocation via language ' - 'syntax or\n' - ' built-in functions. See Special method lookup.\n' - '\n' - ' For certain sensitive attribute accesses, raises an ' - 'auditing event\n' - ' "object.__getattr__" with arguments "obj" and ' - '"name".\n' - '\n' - 'object.__setattr__(self, name, value)\n' - '\n' - ' Called when an attribute assignment is attempted. ' - 'This is called\n' - ' instead of the normal mechanism (i.e. store the value ' - 'in the\n' - ' instance dictionary). *name* is the attribute name, ' - '*value* is the\n' - ' value to be assigned to it.\n' - '\n' - ' If "__setattr__()" wants to assign to an instance ' - 'attribute, it\n' - ' should call the base class method with the same name, ' - 'for example,\n' - ' "object.__setattr__(self, name, value)".\n' - '\n' - ' For certain sensitive attribute assignments, raises ' - 'an auditing\n' - ' event "object.__setattr__" with arguments "obj", ' - '"name", "value".\n' - '\n' - 'object.__delattr__(self, name)\n' - '\n' - ' Like "__setattr__()" but for attribute deletion ' - 'instead of\n' - ' assignment. This should only be implemented if "del ' - 'obj.name" is\n' - ' meaningful for the object.\n' - '\n' - ' For certain sensitive attribute deletions, raises an ' - 'auditing event\n' - ' "object.__delattr__" with arguments "obj" and ' - '"name".\n' - '\n' - 'object.__dir__(self)\n' - '\n' - ' Called when "dir()" is called on the object. An ' - 'iterable must be\n' - ' returned. "dir()" converts the returned iterable to a ' - 'list and\n' - ' sorts it.\n' - '\n' - '\n' - 'Customizing module attribute access\n' - '===================================\n' - '\n' - 'Special names "__getattr__" and "__dir__" can be also ' - 'used to\n' - 'customize access to module attributes. The "__getattr__" ' - 'function at\n' - 'the module level should accept one argument which is the ' - 'name of an\n' - 'attribute and return the computed value or raise an ' - '"AttributeError".\n' - 'If an attribute is not found on a module object through ' - 'the normal\n' - 'lookup, i.e. "object.__getattribute__()", then ' - '"__getattr__" is\n' - 'searched in the module "__dict__" before raising an ' - '"AttributeError".\n' - 'If found, it is called with the attribute name and the ' - 'result is\n' - 'returned.\n' - '\n' - 'The "__dir__" function should accept no arguments, and ' - 'return an\n' - 'iterable of strings that represents the names accessible ' - 'on module. If\n' - 'present, this function overrides the standard "dir()" ' - 'search on a\n' - 'module.\n' - '\n' - 'For a more fine grained customization of the module ' - 'behavior (setting\n' - 'attributes, properties, etc.), one can set the ' - '"__class__" attribute\n' - 'of a module object to a subclass of "types.ModuleType". ' - 'For example:\n' - '\n' - ' import sys\n' - ' from types import ModuleType\n' - '\n' - ' class VerboseModule(ModuleType):\n' - ' def __repr__(self):\n' - " return f'Verbose {self.__name__}'\n" - '\n' - ' def __setattr__(self, attr, value):\n' - " print(f'Setting {attr}...')\n" - ' super().__setattr__(attr, value)\n' - '\n' - ' sys.modules[__name__].__class__ = VerboseModule\n' - '\n' - 'Note:\n' - '\n' - ' Defining module "__getattr__" and setting module ' - '"__class__" only\n' - ' affect lookups made using the attribute access syntax ' - '– directly\n' - ' accessing the module globals (whether by code within ' - 'the module, or\n' - ' via a reference to the module’s globals dictionary) is ' - 'unaffected.\n' - '\n' - 'Changed in version 3.5: "__class__" module attribute is ' - 'now writable.\n' - '\n' - 'New in version 3.7: "__getattr__" and "__dir__" module ' - 'attributes.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 562** - Module __getattr__ and __dir__\n' - ' Describes the "__getattr__" and "__dir__" functions ' - 'on modules.\n' - '\n' - '\n' - 'Implementing Descriptors\n' - '========================\n' - '\n' - 'The following methods only apply when an instance of the ' - 'class\n' - 'containing the method (a so-called *descriptor* class) ' - 'appears in an\n' - '*owner* class (the descriptor must be in either the ' - 'owner’s class\n' - 'dictionary or in the class dictionary for one of its ' - 'parents). In the\n' - 'examples below, “the attribute” refers to the attribute ' - 'whose name is\n' - 'the key of the property in the owner class’ "__dict__".\n' - '\n' - 'object.__get__(self, instance, owner=None)\n' - '\n' - ' Called to get the attribute of the owner class (class ' - 'attribute\n' - ' access) or of an instance of that class (instance ' - 'attribute\n' - ' access). The optional *owner* argument is the owner ' - 'class, while\n' - ' *instance* is the instance that the attribute was ' - 'accessed through,\n' - ' or "None" when the attribute is accessed through the ' - '*owner*.\n' - '\n' - ' This method should return the computed attribute ' - 'value or raise an\n' - ' "AttributeError" exception.\n' - '\n' - ' **PEP 252** specifies that "__get__()" is callable ' - 'with one or two\n' - ' arguments. Python’s own built-in descriptors support ' - 'this\n' - ' specification; however, it is likely that some ' - 'third-party tools\n' - ' have descriptors that require both arguments. ' - 'Python’s own\n' - ' "__getattribute__()" implementation always passes in ' - 'both arguments\n' - ' whether they are required or not.\n' - '\n' - 'object.__set__(self, instance, value)\n' - '\n' - ' Called to set the attribute on an instance *instance* ' - 'of the owner\n' - ' class to a new value, *value*.\n' - '\n' - ' Note, adding "__set__()" or "__delete__()" changes ' - 'the kind of\n' - ' descriptor to a “data descriptor”. See Invoking ' - 'Descriptors for\n' - ' more details.\n' - '\n' - 'object.__delete__(self, instance)\n' - '\n' - ' Called to delete the attribute on an instance ' - '*instance* of the\n' - ' owner class.\n' - '\n' - 'Instances of descriptors may also have the ' - '"__objclass__" attribute\n' - 'present:\n' - '\n' - 'object.__objclass__\n' - '\n' - ' The attribute "__objclass__" is interpreted by the ' - '"inspect" module\n' - ' as specifying the class where this object was defined ' - '(setting this\n' - ' appropriately can assist in runtime introspection of ' - 'dynamic class\n' - ' attributes). For callables, it may indicate that an ' - 'instance of the\n' - ' given type (or a subclass) is expected or required as ' - 'the first\n' - ' positional argument (for example, CPython sets this ' - 'attribute for\n' - ' unbound methods that are implemented in C).\n' - '\n' - '\n' - 'Invoking Descriptors\n' - '====================\n' - '\n' - 'In general, a descriptor is an object attribute with ' - '“binding\n' - 'behavior”, one whose attribute access has been ' - 'overridden by methods\n' - 'in the descriptor protocol: "__get__()", "__set__()", ' - 'and\n' - '"__delete__()". If any of those methods are defined for ' - 'an object, it\n' - 'is said to be a descriptor.\n' - '\n' - 'The default behavior for attribute access is to get, ' - 'set, or delete\n' - 'the attribute from an object’s dictionary. For instance, ' - '"a.x" has a\n' - 'lookup chain starting with "a.__dict__[\'x\']", then\n' - '"type(a).__dict__[\'x\']", and continuing through the ' - 'base classes of\n' - '"type(a)" excluding metaclasses.\n' - '\n' - 'However, if the looked-up value is an object defining ' - 'one of the\n' - 'descriptor methods, then Python may override the default ' - 'behavior and\n' - 'invoke the descriptor method instead. Where this occurs ' - 'in the\n' - 'precedence chain depends on which descriptor methods ' - 'were defined and\n' - 'how they were called.\n' - '\n' - 'The starting point for descriptor invocation is a ' - 'binding, "a.x". How\n' - 'the arguments are assembled depends on "a":\n' - '\n' - 'Direct Call\n' - ' The simplest and least common call is when user code ' - 'directly\n' - ' invokes a descriptor method: "x.__get__(a)".\n' - '\n' - 'Instance Binding\n' - ' If binding to an object instance, "a.x" is ' - 'transformed into the\n' - ' call: "type(a).__dict__[\'x\'].__get__(a, type(a))".\n' - '\n' - 'Class Binding\n' - ' If binding to a class, "A.x" is transformed into the ' - 'call:\n' - ' "A.__dict__[\'x\'].__get__(None, A)".\n' - '\n' - 'Super Binding\n' - ' A dotted lookup such as "super(A, a).x" searches\n' - ' "a.__class__.__mro__" for a base class "B" following ' - '"A" and then\n' - ' returns "B.__dict__[\'x\'].__get__(a, A)". If not a ' - 'descriptor, "x"\n' - ' is returned unchanged.\n' - '\n' - 'For instance bindings, the precedence of descriptor ' - 'invocation depends\n' - 'on which descriptor methods are defined. A descriptor ' - 'can define any\n' - 'combination of "__get__()", "__set__()" and ' - '"__delete__()". If it\n' - 'does not define "__get__()", then accessing the ' - 'attribute will return\n' - 'the descriptor object itself unless there is a value in ' - 'the object’s\n' - 'instance dictionary. If the descriptor defines ' - '"__set__()" and/or\n' - '"__delete__()", it is a data descriptor; if it defines ' - 'neither, it is\n' - 'a non-data descriptor. Normally, data descriptors ' - 'define both\n' - '"__get__()" and "__set__()", while non-data descriptors ' - 'have just the\n' - '"__get__()" method. Data descriptors with "__get__()" ' - 'and "__set__()"\n' - '(and/or "__delete__()") defined always override a ' - 'redefinition in an\n' - 'instance dictionary. In contrast, non-data descriptors ' - 'can be\n' - 'overridden by instances.\n' - '\n' - 'Python methods (including those decorated with ' - '"@staticmethod" and\n' - '"@classmethod") are implemented as non-data ' - 'descriptors. Accordingly,\n' - 'instances can redefine and override methods. This ' - 'allows individual\n' - 'instances to acquire behaviors that differ from other ' - 'instances of the\n' - 'same class.\n' - '\n' - 'The "property()" function is implemented as a data ' - 'descriptor.\n' - 'Accordingly, instances cannot override the behavior of a ' - 'property.\n' - '\n' - '\n' - '__slots__\n' - '=========\n' - '\n' - '*__slots__* allow us to explicitly declare data members ' - '(like\n' - 'properties) and deny the creation of "__dict__" and ' - '*__weakref__*\n' - '(unless explicitly declared in *__slots__* or available ' - 'in a parent.)\n' - '\n' - 'The space saved over using "__dict__" can be ' - 'significant. Attribute\n' - 'lookup speed can be significantly improved as well.\n' - '\n' - 'object.__slots__\n' - '\n' - ' This class variable can be assigned a string, ' - 'iterable, or sequence\n' - ' of strings with variable names used by instances. ' - '*__slots__*\n' - ' reserves space for the declared variables and ' - 'prevents the\n' - ' automatic creation of "__dict__" and *__weakref__* ' - 'for each\n' - ' instance.\n' - '\n' - 'Notes on using *__slots__*:\n' - '\n' - '* When inheriting from a class without *__slots__*, the ' - '"__dict__" and\n' - ' *__weakref__* attribute of the instances will always ' - 'be accessible.\n' - '\n' - '* Without a "__dict__" variable, instances cannot be ' - 'assigned new\n' - ' variables not listed in the *__slots__* definition. ' - 'Attempts to\n' - ' assign to an unlisted variable name raises ' - '"AttributeError". If\n' - ' dynamic assignment of new variables is desired, then ' - 'add\n' - ' "\'__dict__\'" to the sequence of strings in the ' - '*__slots__*\n' - ' declaration.\n' - '\n' - '* Without a *__weakref__* variable for each instance, ' - 'classes defining\n' - ' *__slots__* do not support "weak references" to its ' - 'instances. If\n' - ' weak reference support is needed, then add ' - '"\'__weakref__\'" to the\n' - ' sequence of strings in the *__slots__* declaration.\n' - '\n' - '* *__slots__* are implemented at the class level by ' - 'creating\n' - ' descriptors for each variable name. As a result, ' - 'class attributes\n' - ' cannot be used to set default values for instance ' - 'variables defined\n' - ' by *__slots__*; otherwise, the class attribute would ' - 'overwrite the\n' - ' descriptor assignment.\n' - '\n' - '* The action of a *__slots__* declaration is not limited ' - 'to the class\n' - ' where it is defined. *__slots__* declared in parents ' - 'are available\n' - ' in child classes. However, child subclasses will get a ' - '"__dict__"\n' - ' and *__weakref__* unless they also define *__slots__* ' - '(which should\n' - ' only contain names of any *additional* slots).\n' - '\n' - '* If a class defines a slot also defined in a base ' - 'class, the instance\n' - ' variable defined by the base class slot is ' - 'inaccessible (except by\n' - ' retrieving its descriptor directly from the base ' - 'class). This\n' - ' renders the meaning of the program undefined. In the ' - 'future, a\n' - ' check may be added to prevent this.\n' - '\n' - '* "TypeError" will be raised if nonempty *__slots__* are ' - 'defined for a\n' - ' class derived from a ""variable-length" built-in type" ' - 'such as\n' - ' "int", "bytes", and "tuple".\n' - '\n' - '* Any non-string *iterable* may be assigned to ' - '*__slots__*.\n' - '\n' - '* If a "dictionary" is used to assign *__slots__*, the ' - 'dictionary keys\n' - ' will be used as the slot names. The values of the ' - 'dictionary can be\n' - ' used to provide per-attribute docstrings that will be ' - 'recognised by\n' - ' "inspect.getdoc()" and displayed in the output of ' - '"help()".\n' - '\n' - '* "__class__" assignment works only if both classes have ' - 'the same\n' - ' *__slots__*.\n' - '\n' - '* Multiple inheritance with multiple slotted parent ' - 'classes can be\n' - ' used, but only one parent is allowed to have ' - 'attributes created by\n' - ' slots (the other bases must have empty slot layouts) - ' - 'violations\n' - ' raise "TypeError".\n' - '\n' - '* If an *iterator* is used for *__slots__* then a ' - '*descriptor* is\n' - ' created for each of the iterator’s values. However, ' - 'the *__slots__*\n' - ' attribute will be an empty iterator.\n', - 'attribute-references': 'Attribute references\n' - '********************\n' - '\n' - 'An attribute reference is a primary followed by a ' - 'period and a name:\n' - '\n' - ' attributeref ::= primary "." identifier\n' - '\n' - 'The primary must evaluate to an object of a type ' - 'that supports\n' - 'attribute references, which most objects do. This ' - 'object is then\n' - 'asked to produce the attribute whose name is the ' - 'identifier. The type\n' - 'and value produced is determined by the object. ' - 'Multiple evaluations\n' - 'of the same attribute reference may yield different ' - 'objects.\n' - '\n' - 'This production can be customized by overriding the\n' - '"__getattribute__()" method or the "__getattr__()" ' - 'method. The\n' - '"__getattribute__()" method is called first and ' - 'either returns a value\n' - 'or raises "AttributeError" if the attribute is not ' - 'available.\n' - '\n' - 'If an "AttributeError" is raised and the object has ' - 'a "__getattr__()"\n' - 'method, that method is called as a fallback.\n', - 'augassign': 'Augmented assignment statements\n' - '*******************************\n' - '\n' - 'Augmented assignment is the combination, in a single statement, ' - 'of a\n' - 'binary operation and an assignment statement:\n' - '\n' - ' augmented_assignment_stmt ::= augtarget augop ' - '(expression_list | yield_expression)\n' - ' augtarget ::= identifier | attributeref | ' - 'subscription | slicing\n' - ' augop ::= "+=" | "-=" | "*=" | "@=" | ' - '"/=" | "//=" | "%=" | "**="\n' - ' | ">>=" | "<<=" | "&=" | "^=" | "|="\n' - '\n' - '(See section Primaries for the syntax definitions of the last ' - 'three\n' - 'symbols.)\n' - '\n' - 'An augmented assignment evaluates the target (which, unlike ' - 'normal\n' - 'assignment statements, cannot be an unpacking) and the ' - 'expression\n' - 'list, performs the binary operation specific to the type of ' - 'assignment\n' - 'on the two operands, and assigns the result to the original ' - 'target.\n' - 'The target is only evaluated once.\n' - '\n' - 'An augmented assignment expression like "x += 1" can be ' - 'rewritten as\n' - '"x = x + 1" to achieve a similar, but not exactly equal effect. ' - 'In the\n' - 'augmented version, "x" is only evaluated once. Also, when ' - 'possible,\n' - 'the actual operation is performed *in-place*, meaning that ' - 'rather than\n' - 'creating a new object and assigning that to the target, the old ' - 'object\n' - 'is modified instead.\n' - '\n' - 'Unlike normal assignments, augmented assignments evaluate the ' - 'left-\n' - 'hand side *before* evaluating the right-hand side. For ' - 'example, "a[i]\n' - '+= f(x)" first looks-up "a[i]", then it evaluates "f(x)" and ' - 'performs\n' - 'the addition, and lastly, it writes the result back to "a[i]".\n' - '\n' - 'With the exception of assigning to tuples and multiple targets ' - 'in a\n' - 'single statement, the assignment done by augmented assignment\n' - 'statements is handled the same way as normal assignments. ' - 'Similarly,\n' - 'with the exception of the possible *in-place* behavior, the ' - 'binary\n' - 'operation performed by augmented assignment is the same as the ' - 'normal\n' - 'binary operations.\n' - '\n' - 'For targets which are attribute references, the same caveat ' - 'about\n' - 'class and instance attributes applies as for regular ' - 'assignments.\n', - 'await': 'Await expression\n' - '****************\n' - '\n' - 'Suspend the execution of *coroutine* on an *awaitable* object. Can\n' - 'only be used inside a *coroutine function*.\n' - '\n' - ' await_expr ::= "await" primary\n' - '\n' - 'New in version 3.5.\n', - 'binary': 'Binary arithmetic operations\n' - '****************************\n' - '\n' - 'The binary arithmetic operations have the conventional priority\n' - 'levels. Note that some of these operations also apply to certain ' - 'non-\n' - 'numeric types. Apart from the power operator, there are only two\n' - 'levels, one for multiplicative operators and one for additive\n' - 'operators:\n' - '\n' - ' m_expr ::= u_expr | m_expr "*" u_expr | m_expr "@" m_expr |\n' - ' m_expr "//" u_expr | m_expr "/" u_expr |\n' - ' m_expr "%" u_expr\n' - ' a_expr ::= m_expr | a_expr "+" m_expr | a_expr "-" m_expr\n' - '\n' - 'The "*" (multiplication) operator yields the product of its ' - 'arguments.\n' - 'The arguments must either both be numbers, or one argument must be ' - 'an\n' - 'integer and the other must be a sequence. In the former case, the\n' - 'numbers are converted to a common type and then multiplied ' - 'together.\n' - 'In the latter case, sequence repetition is performed; a negative\n' - 'repetition factor yields an empty sequence.\n' - '\n' - 'This operation can be customized using the special "__mul__()" ' - 'and\n' - '"__rmul__()" methods.\n' - '\n' - 'The "@" (at) operator is intended to be used for matrix\n' - 'multiplication. No builtin Python types implement this operator.\n' - '\n' - 'New in version 3.5.\n' - '\n' - 'The "/" (division) and "//" (floor division) operators yield the\n' - 'quotient of their arguments. The numeric arguments are first\n' - 'converted to a common type. Division of integers yields a float, ' - 'while\n' - 'floor division of integers results in an integer; the result is ' - 'that\n' - 'of mathematical division with the ‘floor’ function applied to the\n' - 'result. Division by zero raises the "ZeroDivisionError" ' - 'exception.\n' - '\n' - 'This operation can be customized using the special "__truediv__()" ' - 'and\n' - '"__floordiv__()" methods.\n' - '\n' - 'The "%" (modulo) operator yields the remainder from the division ' - 'of\n' - 'the first argument by the second. The numeric arguments are ' - 'first\n' - 'converted to a common type. A zero right argument raises the\n' - '"ZeroDivisionError" exception. The arguments may be floating ' - 'point\n' - 'numbers, e.g., "3.14%0.7" equals "0.34" (since "3.14" equals ' - '"4*0.7 +\n' - '0.34".) The modulo operator always yields a result with the same ' - 'sign\n' - 'as its second operand (or zero); the absolute value of the result ' - 'is\n' - 'strictly smaller than the absolute value of the second operand ' - '[1].\n' - '\n' - 'The floor division and modulo operators are connected by the ' - 'following\n' - 'identity: "x == (x//y)*y + (x%y)". Floor division and modulo are ' - 'also\n' - 'connected with the built-in function "divmod()": "divmod(x, y) ==\n' - '(x//y, x%y)". [2].\n' - '\n' - 'In addition to performing the modulo operation on numbers, the ' - '"%"\n' - 'operator is also overloaded by string objects to perform ' - 'old-style\n' - 'string formatting (also known as interpolation). The syntax for\n' - 'string formatting is described in the Python Library Reference,\n' - 'section printf-style String Formatting.\n' - '\n' - 'The *modulo* operation can be customized using the special ' - '"__mod__()"\n' - 'method.\n' - '\n' - 'The floor division operator, the modulo operator, and the ' - '"divmod()"\n' - 'function are not defined for complex numbers. Instead, convert to ' - 'a\n' - 'floating point number using the "abs()" function if appropriate.\n' - '\n' - 'The "+" (addition) operator yields the sum of its arguments. The\n' - 'arguments must either both be numbers or both be sequences of the ' - 'same\n' - 'type. In the former case, the numbers are converted to a common ' - 'type\n' - 'and then added together. In the latter case, the sequences are\n' - 'concatenated.\n' - '\n' - 'This operation can be customized using the special "__add__()" ' - 'and\n' - '"__radd__()" methods.\n' - '\n' - 'The "-" (subtraction) operator yields the difference of its ' - 'arguments.\n' - 'The numeric arguments are first converted to a common type.\n' - '\n' - 'This operation can be customized using the special "__sub__()" ' - 'method.\n', - 'bitwise': 'Binary bitwise operations\n' - '*************************\n' - '\n' - 'Each of the three bitwise operations has a different priority ' - 'level:\n' - '\n' - ' and_expr ::= shift_expr | and_expr "&" shift_expr\n' - ' xor_expr ::= and_expr | xor_expr "^" and_expr\n' - ' or_expr ::= xor_expr | or_expr "|" xor_expr\n' - '\n' - 'The "&" operator yields the bitwise AND of its arguments, which ' - 'must\n' - 'be integers or one of them must be a custom object overriding\n' - '"__and__()" or "__rand__()" special methods.\n' - '\n' - 'The "^" operator yields the bitwise XOR (exclusive OR) of its\n' - 'arguments, which must be integers or one of them must be a ' - 'custom\n' - 'object overriding "__xor__()" or "__rxor__()" special methods.\n' - '\n' - 'The "|" operator yields the bitwise (inclusive) OR of its ' - 'arguments,\n' - 'which must be integers or one of them must be a custom object\n' - 'overriding "__or__()" or "__ror__()" special methods.\n', - 'bltin-code-objects': 'Code Objects\n' - '************\n' - '\n' - 'Code objects are used by the implementation to ' - 'represent “pseudo-\n' - 'compiled” executable Python code such as a function ' - 'body. They differ\n' - 'from function objects because they don’t contain a ' - 'reference to their\n' - 'global execution environment. Code objects are ' - 'returned by the built-\n' - 'in "compile()" function and can be extracted from ' - 'function objects\n' - 'through their "__code__" attribute. See also the ' - '"code" module.\n' - '\n' - 'Accessing "__code__" raises an auditing event ' - '"object.__getattr__"\n' - 'with arguments "obj" and ""__code__"".\n' - '\n' - 'A code object can be executed or evaluated by passing ' - 'it (instead of a\n' - 'source string) to the "exec()" or "eval()" built-in ' - 'functions.\n' - '\n' - 'See The standard type hierarchy for more ' - 'information.\n', - 'bltin-ellipsis-object': 'The Ellipsis Object\n' - '*******************\n' - '\n' - 'This object is commonly used by slicing (see ' - 'Slicings). It supports\n' - 'no special operations. There is exactly one ' - 'ellipsis object, named\n' - '"Ellipsis" (a built-in name). "type(Ellipsis)()" ' - 'produces the\n' - '"Ellipsis" singleton.\n' - '\n' - 'It is written as "Ellipsis" or "...".\n', - 'bltin-null-object': 'The Null Object\n' - '***************\n' - '\n' - 'This object is returned by functions that don’t ' - 'explicitly return a\n' - 'value. It supports no special operations. There is ' - 'exactly one null\n' - 'object, named "None" (a built-in name). "type(None)()" ' - 'produces the\n' - 'same singleton.\n' - '\n' - 'It is written as "None".\n', - 'bltin-type-objects': 'Type Objects\n' - '************\n' - '\n' - 'Type objects represent the various object types. An ' - 'object’s type is\n' - 'accessed by the built-in function "type()". There are ' - 'no special\n' - 'operations on types. The standard module "types" ' - 'defines names for\n' - 'all standard built-in types.\n' - '\n' - 'Types are written like this: "".\n', - 'booleans': 'Boolean operations\n' - '******************\n' - '\n' - ' or_test ::= and_test | or_test "or" and_test\n' - ' and_test ::= not_test | and_test "and" not_test\n' - ' not_test ::= comparison | "not" not_test\n' - '\n' - 'In the context of Boolean operations, and also when expressions ' - 'are\n' - 'used by control flow statements, the following values are ' - 'interpreted\n' - 'as false: "False", "None", numeric zero of all types, and empty\n' - 'strings and containers (including strings, tuples, lists,\n' - 'dictionaries, sets and frozensets). All other values are ' - 'interpreted\n' - 'as true. User-defined objects can customize their truth value ' - 'by\n' - 'providing a "__bool__()" method.\n' - '\n' - 'The operator "not" yields "True" if its argument is false, ' - '"False"\n' - 'otherwise.\n' - '\n' - 'The expression "x and y" first evaluates *x*; if *x* is false, ' - 'its\n' - 'value is returned; otherwise, *y* is evaluated and the resulting ' - 'value\n' - 'is returned.\n' - '\n' - 'The expression "x or y" first evaluates *x*; if *x* is true, its ' - 'value\n' - 'is returned; otherwise, *y* is evaluated and the resulting value ' - 'is\n' - 'returned.\n' - '\n' - 'Note that neither "and" nor "or" restrict the value and type ' - 'they\n' - 'return to "False" and "True", but rather return the last ' - 'evaluated\n' - 'argument. This is sometimes useful, e.g., if "s" is a string ' - 'that\n' - 'should be replaced by a default value if it is empty, the ' - 'expression\n' - '"s or \'foo\'" yields the desired value. Because "not" has to ' - 'create a\n' - 'new value, it returns a boolean value regardless of the type of ' - 'its\n' - 'argument (for example, "not \'foo\'" produces "False" rather ' - 'than "\'\'".)\n', - 'break': 'The "break" statement\n' - '*********************\n' - '\n' - ' break_stmt ::= "break"\n' - '\n' - '"break" may only occur syntactically nested in a "for" or "while"\n' - 'loop, but not nested in a function or class definition within that\n' - 'loop.\n' - '\n' - 'It terminates the nearest enclosing loop, skipping the optional ' - '"else"\n' - 'clause if the loop has one.\n' - '\n' - 'If a "for" loop is terminated by "break", the loop control target\n' - 'keeps its current value.\n' - '\n' - 'When "break" passes control out of a "try" statement with a ' - '"finally"\n' - 'clause, that "finally" clause is executed before really leaving ' - 'the\n' - 'loop.\n', - 'callable-types': 'Emulating callable objects\n' - '**************************\n' - '\n' - 'object.__call__(self[, args...])\n' - '\n' - ' Called when the instance is “called” as a function; if ' - 'this method\n' - ' is defined, "x(arg1, arg2, ...)" roughly translates to\n' - ' "type(x).__call__(x, arg1, ...)".\n', - 'calls': 'Calls\n' - '*****\n' - '\n' - 'A call calls a callable object (e.g., a *function*) with a ' - 'possibly\n' - 'empty series of *arguments*:\n' - '\n' - ' call ::= primary "(" [argument_list [","] | ' - 'comprehension] ")"\n' - ' argument_list ::= positional_arguments ["," ' - 'starred_and_keywords]\n' - ' ["," keywords_arguments]\n' - ' | starred_and_keywords ["," ' - 'keywords_arguments]\n' - ' | keywords_arguments\n' - ' positional_arguments ::= positional_item ("," positional_item)*\n' - ' positional_item ::= assignment_expression | "*" expression\n' - ' starred_and_keywords ::= ("*" expression | keyword_item)\n' - ' ("," "*" expression | "," ' - 'keyword_item)*\n' - ' keywords_arguments ::= (keyword_item | "**" expression)\n' - ' ("," keyword_item | "," "**" ' - 'expression)*\n' - ' keyword_item ::= identifier "=" expression\n' - '\n' - 'An optional trailing comma may be present after the positional and\n' - 'keyword arguments but does not affect the semantics.\n' - '\n' - 'The primary must evaluate to a callable object (user-defined\n' - 'functions, built-in functions, methods of built-in objects, class\n' - 'objects, methods of class instances, and all objects having a\n' - '"__call__()" method are callable). All argument expressions are\n' - 'evaluated before the call is attempted. Please refer to section\n' - 'Function definitions for the syntax of formal *parameter* lists.\n' - '\n' - 'If keyword arguments are present, they are first converted to\n' - 'positional arguments, as follows. First, a list of unfilled slots ' - 'is\n' - 'created for the formal parameters. If there are N positional\n' - 'arguments, they are placed in the first N slots. Next, for each\n' - 'keyword argument, the identifier is used to determine the\n' - 'corresponding slot (if the identifier is the same as the first ' - 'formal\n' - 'parameter name, the first slot is used, and so on). If the slot ' - 'is\n' - 'already filled, a "TypeError" exception is raised. Otherwise, the\n' - 'argument is placed in the slot, filling it (even if the expression ' - 'is\n' - '"None", it fills the slot). When all arguments have been ' - 'processed,\n' - 'the slots that are still unfilled are filled with the ' - 'corresponding\n' - 'default value from the function definition. (Default values are\n' - 'calculated, once, when the function is defined; thus, a mutable ' - 'object\n' - 'such as a list or dictionary used as default value will be shared ' - 'by\n' - 'all calls that don’t specify an argument value for the ' - 'corresponding\n' - 'slot; this should usually be avoided.) If there are any unfilled\n' - 'slots for which no default value is specified, a "TypeError" ' - 'exception\n' - 'is raised. Otherwise, the list of filled slots is used as the\n' - 'argument list for the call.\n' - '\n' - '**CPython implementation detail:** An implementation may provide\n' - 'built-in functions whose positional parameters do not have names, ' - 'even\n' - 'if they are ‘named’ for the purpose of documentation, and which\n' - 'therefore cannot be supplied by keyword. In CPython, this is the ' - 'case\n' - 'for functions implemented in C that use "PyArg_ParseTuple()" to ' - 'parse\n' - 'their arguments.\n' - '\n' - 'If there are more positional arguments than there are formal ' - 'parameter\n' - 'slots, a "TypeError" exception is raised, unless a formal ' - 'parameter\n' - 'using the syntax "*identifier" is present; in this case, that ' - 'formal\n' - 'parameter receives a tuple containing the excess positional ' - 'arguments\n' - '(or an empty tuple if there were no excess positional arguments).\n' - '\n' - 'If any keyword argument does not correspond to a formal parameter\n' - 'name, a "TypeError" exception is raised, unless a formal parameter\n' - 'using the syntax "**identifier" is present; in this case, that ' - 'formal\n' - 'parameter receives a dictionary containing the excess keyword\n' - 'arguments (using the keywords as keys and the argument values as\n' - 'corresponding values), or a (new) empty dictionary if there were ' - 'no\n' - 'excess keyword arguments.\n' - '\n' - 'If the syntax "*expression" appears in the function call, ' - '"expression"\n' - 'must evaluate to an *iterable*. Elements from these iterables are\n' - 'treated as if they were additional positional arguments. For the ' - 'call\n' - '"f(x1, x2, *y, x3, x4)", if *y* evaluates to a sequence *y1*, …, ' - '*yM*,\n' - 'this is equivalent to a call with M+4 positional arguments *x1*, ' - '*x2*,\n' - '*y1*, …, *yM*, *x3*, *x4*.\n' - '\n' - 'A consequence of this is that although the "*expression" syntax ' - 'may\n' - 'appear *after* explicit keyword arguments, it is processed ' - '*before*\n' - 'the keyword arguments (and any "**expression" arguments – see ' - 'below).\n' - 'So:\n' - '\n' - ' >>> def f(a, b):\n' - ' ... print(a, b)\n' - ' ...\n' - ' >>> f(b=1, *(2,))\n' - ' 2 1\n' - ' >>> f(a=1, *(2,))\n' - ' Traceback (most recent call last):\n' - ' File "", line 1, in \n' - " TypeError: f() got multiple values for keyword argument 'a'\n" - ' >>> f(1, *(2,))\n' - ' 1 2\n' - '\n' - 'It is unusual for both keyword arguments and the "*expression" ' - 'syntax\n' - 'to be used in the same call, so in practice this confusion does ' - 'not\n' - 'often arise.\n' - '\n' - 'If the syntax "**expression" appears in the function call,\n' - '"expression" must evaluate to a *mapping*, the contents of which ' - 'are\n' - 'treated as additional keyword arguments. If a parameter matching a ' - 'key\n' - 'has already been given a value (by an explicit keyword argument, ' - 'or\n' - 'from another unpacking), a "TypeError" exception is raised.\n' - '\n' - 'When "**expression" is used, each key in this mapping must be a\n' - 'string. Each value from the mapping is assigned to the first ' - 'formal\n' - 'parameter eligible for keyword assignment whose name is equal to ' - 'the\n' - 'key. A key need not be a Python identifier (e.g. ""max-temp °F"" ' - 'is\n' - 'acceptable, although it will not match any formal parameter that ' - 'could\n' - 'be declared). If there is no match to a formal parameter the ' - 'key-value\n' - 'pair is collected by the "**" parameter, if there is one, or if ' - 'there\n' - 'is not, a "TypeError" exception is raised.\n' - '\n' - 'Formal parameters using the syntax "*identifier" or "**identifier"\n' - 'cannot be used as positional argument slots or as keyword argument\n' - 'names.\n' - '\n' - 'Changed in version 3.5: Function calls accept any number of "*" ' - 'and\n' - '"**" unpackings, positional arguments may follow iterable ' - 'unpackings\n' - '("*"), and keyword arguments may follow dictionary unpackings ' - '("**").\n' - 'Originally proposed by **PEP 448**.\n' - '\n' - 'A call always returns some value, possibly "None", unless it raises ' - 'an\n' - 'exception. How this value is computed depends on the type of the\n' - 'callable object.\n' - '\n' - 'If it is—\n' - '\n' - 'a user-defined function:\n' - ' The code block for the function is executed, passing it the\n' - ' argument list. The first thing the code block will do is bind ' - 'the\n' - ' formal parameters to the arguments; this is described in ' - 'section\n' - ' Function definitions. When the code block executes a "return"\n' - ' statement, this specifies the return value of the function ' - 'call.\n' - '\n' - 'a built-in function or method:\n' - ' The result is up to the interpreter; see Built-in Functions for ' - 'the\n' - ' descriptions of built-in functions and methods.\n' - '\n' - 'a class object:\n' - ' A new instance of that class is returned.\n' - '\n' - 'a class instance method:\n' - ' The corresponding user-defined function is called, with an ' - 'argument\n' - ' list that is one longer than the argument list of the call: the\n' - ' instance becomes the first argument.\n' - '\n' - 'a class instance:\n' - ' The class must define a "__call__()" method; the effect is then ' - 'the\n' - ' same as if that method was called.\n', - 'class': 'Class definitions\n' - '*****************\n' - '\n' - 'A class definition defines a class object (see section The ' - 'standard\n' - 'type hierarchy):\n' - '\n' - ' classdef ::= [decorators] "class" classname [type_params] ' - '[inheritance] ":" suite\n' - ' inheritance ::= "(" [argument_list] ")"\n' - ' classname ::= identifier\n' - '\n' - 'A class definition is an executable statement. The inheritance ' - 'list\n' - 'usually gives a list of base classes (see Metaclasses for more\n' - 'advanced uses), so each item in the list should evaluate to a ' - 'class\n' - 'object which allows subclassing. Classes without an inheritance ' - 'list\n' - 'inherit, by default, from the base class "object"; hence,\n' - '\n' - ' class Foo:\n' - ' pass\n' - '\n' - 'is equivalent to\n' - '\n' - ' class Foo(object):\n' - ' pass\n' - '\n' - 'The class’s suite is then executed in a new execution frame (see\n' - 'Naming and binding), using a newly created local namespace and the\n' - 'original global namespace. (Usually, the suite contains mostly\n' - 'function definitions.) When the class’s suite finishes execution, ' - 'its\n' - 'execution frame is discarded but its local namespace is saved. [5] ' - 'A\n' - 'class object is then created using the inheritance list for the ' - 'base\n' - 'classes and the saved local namespace for the attribute ' - 'dictionary.\n' - 'The class name is bound to this class object in the original local\n' - 'namespace.\n' - '\n' - 'The order in which attributes are defined in the class body is\n' - 'preserved in the new class’s "__dict__". Note that this is ' - 'reliable\n' - 'only right after the class is created and only for classes that ' - 'were\n' - 'defined using the definition syntax.\n' - '\n' - 'Class creation can be customized heavily using metaclasses.\n' - '\n' - 'Classes can also be decorated: just like when decorating ' - 'functions,\n' - '\n' - ' @f1(arg)\n' - ' @f2\n' - ' class Foo: pass\n' - '\n' - 'is roughly equivalent to\n' - '\n' - ' class Foo: pass\n' - ' Foo = f1(arg)(f2(Foo))\n' - '\n' - 'The evaluation rules for the decorator expressions are the same as ' - 'for\n' - 'function decorators. The result is then bound to the class name.\n' - '\n' - 'Changed in version 3.9: Classes may be decorated with any valid\n' - '"assignment_expression". Previously, the grammar was much more\n' - 'restrictive; see **PEP 614** for details.\n' - '\n' - 'A list of type parameters may be given in square brackets ' - 'immediately\n' - 'after the class’s name. This indicates to static type checkers ' - 'that\n' - 'the class is generic. At runtime, the type parameters can be ' - 'retrieved\n' - 'from the class’s "__type_params__" attribute. See Generic classes ' - 'for\n' - 'more.\n' - '\n' - 'Changed in version 3.12: Type parameter lists are new in Python ' - '3.12.\n' - '\n' - '**Programmer’s note:** Variables defined in the class definition ' - 'are\n' - 'class attributes; they are shared by instances. Instance ' - 'attributes\n' - 'can be set in a method with "self.name = value". Both class and\n' - 'instance attributes are accessible through the notation ' - '“"self.name"”,\n' - 'and an instance attribute hides a class attribute with the same ' - 'name\n' - 'when accessed in this way. Class attributes can be used as ' - 'defaults\n' - 'for instance attributes, but using mutable values there can lead ' - 'to\n' - 'unexpected results. Descriptors can be used to create instance\n' - 'variables with different implementation details.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3115** - Metaclasses in Python 3000\n' - ' The proposal that changed the declaration of metaclasses to ' - 'the\n' - ' current syntax, and the semantics for how classes with\n' - ' metaclasses are constructed.\n' - '\n' - ' **PEP 3129** - Class Decorators\n' - ' The proposal that added class decorators. Function and ' - 'method\n' - ' decorators were introduced in **PEP 318**.\n', - 'comparisons': 'Comparisons\n' - '***********\n' - '\n' - 'Unlike C, all comparison operations in Python have the same ' - 'priority,\n' - 'which is lower than that of any arithmetic, shifting or ' - 'bitwise\n' - 'operation. Also unlike C, expressions like "a < b < c" have ' - 'the\n' - 'interpretation that is conventional in mathematics:\n' - '\n' - ' comparison ::= or_expr (comp_operator or_expr)*\n' - ' comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "!="\n' - ' | "is" ["not"] | ["not"] "in"\n' - '\n' - 'Comparisons yield boolean values: "True" or "False". Custom ' - '*rich\n' - 'comparison methods* may return non-boolean values. In this ' - 'case Python\n' - 'will call "bool()" on such value in boolean contexts.\n' - '\n' - 'Comparisons can be chained arbitrarily, e.g., "x < y <= z" ' - 'is\n' - 'equivalent to "x < y and y <= z", except that "y" is ' - 'evaluated only\n' - 'once (but in both cases "z" is not evaluated at all when "x < ' - 'y" is\n' - 'found to be false).\n' - '\n' - 'Formally, if *a*, *b*, *c*, …, *y*, *z* are expressions and ' - '*op1*,\n' - '*op2*, …, *opN* are comparison operators, then "a op1 b op2 c ' - '... y\n' - 'opN z" is equivalent to "a op1 b and b op2 c and ... y opN ' - 'z", except\n' - 'that each expression is evaluated at most once.\n' - '\n' - 'Note that "a op1 b op2 c" doesn’t imply any kind of ' - 'comparison between\n' - '*a* and *c*, so that, e.g., "x < y > z" is perfectly legal ' - '(though\n' - 'perhaps not pretty).\n' - '\n' - '\n' - 'Value comparisons\n' - '=================\n' - '\n' - 'The operators "<", ">", "==", ">=", "<=", and "!=" compare ' - 'the values\n' - 'of two objects. The objects do not need to have the same ' - 'type.\n' - '\n' - 'Chapter Objects, values and types states that objects have a ' - 'value (in\n' - 'addition to type and identity). The value of an object is a ' - 'rather\n' - 'abstract notion in Python: For example, there is no canonical ' - 'access\n' - 'method for an object’s value. Also, there is no requirement ' - 'that the\n' - 'value of an object should be constructed in a particular way, ' - 'e.g.\n' - 'comprised of all its data attributes. Comparison operators ' - 'implement a\n' - 'particular notion of what the value of an object is. One can ' - 'think of\n' - 'them as defining the value of an object indirectly, by means ' - 'of their\n' - 'comparison implementation.\n' - '\n' - 'Because all types are (direct or indirect) subtypes of ' - '"object", they\n' - 'inherit the default comparison behavior from "object". Types ' - 'can\n' - 'customize their comparison behavior by implementing *rich ' - 'comparison\n' - 'methods* like "__lt__()", described in Basic customization.\n' - '\n' - 'The default behavior for equality comparison ("==" and "!=") ' - 'is based\n' - 'on the identity of the objects. Hence, equality comparison ' - 'of\n' - 'instances with the same identity results in equality, and ' - 'equality\n' - 'comparison of instances with different identities results in\n' - 'inequality. A motivation for this default behavior is the ' - 'desire that\n' - 'all objects should be reflexive (i.e. "x is y" implies "x == ' - 'y").\n' - '\n' - 'A default order comparison ("<", ">", "<=", and ">=") is not ' - 'provided;\n' - 'an attempt raises "TypeError". A motivation for this default ' - 'behavior\n' - 'is the lack of a similar invariant as for equality.\n' - '\n' - 'The behavior of the default equality comparison, that ' - 'instances with\n' - 'different identities are always unequal, may be in contrast ' - 'to what\n' - 'types will need that have a sensible definition of object ' - 'value and\n' - 'value-based equality. Such types will need to customize ' - 'their\n' - 'comparison behavior, and in fact, a number of built-in types ' - 'have done\n' - 'that.\n' - '\n' - 'The following list describes the comparison behavior of the ' - 'most\n' - 'important built-in types.\n' - '\n' - '* Numbers of built-in numeric types (Numeric Types — int, ' - 'float,\n' - ' complex) and of the standard library types ' - '"fractions.Fraction" and\n' - ' "decimal.Decimal" can be compared within and across their ' - 'types,\n' - ' with the restriction that complex numbers do not support ' - 'order\n' - ' comparison. Within the limits of the types involved, they ' - 'compare\n' - ' mathematically (algorithmically) correct without loss of ' - 'precision.\n' - '\n' - ' The not-a-number values "float(\'NaN\')" and ' - '"decimal.Decimal(\'NaN\')"\n' - ' are special. Any ordered comparison of a number to a ' - 'not-a-number\n' - ' value is false. A counter-intuitive implication is that ' - 'not-a-number\n' - ' values are not equal to themselves. For example, if "x =\n' - ' float(\'NaN\')", "3 < x", "x < 3" and "x == x" are all ' - 'false, while "x\n' - ' != x" is true. This behavior is compliant with IEEE 754.\n' - '\n' - '* "None" and "NotImplemented" are singletons. **PEP 8** ' - 'advises that\n' - ' comparisons for singletons should always be done with "is" ' - 'or "is\n' - ' not", never the equality operators.\n' - '\n' - '* Binary sequences (instances of "bytes" or "bytearray") can ' - 'be\n' - ' compared within and across their types. They compare\n' - ' lexicographically using the numeric values of their ' - 'elements.\n' - '\n' - '* Strings (instances of "str") compare lexicographically ' - 'using the\n' - ' numerical Unicode code points (the result of the built-in ' - 'function\n' - ' "ord()") of their characters. [3]\n' - '\n' - ' Strings and binary sequences cannot be directly compared.\n' - '\n' - '* Sequences (instances of "tuple", "list", or "range") can be ' - 'compared\n' - ' only within each of their types, with the restriction that ' - 'ranges do\n' - ' not support order comparison. Equality comparison across ' - 'these\n' - ' types results in inequality, and ordering comparison across ' - 'these\n' - ' types raises "TypeError".\n' - '\n' - ' Sequences compare lexicographically using comparison of\n' - ' corresponding elements. The built-in containers typically ' - 'assume\n' - ' identical objects are equal to themselves. That lets them ' - 'bypass\n' - ' equality tests for identical objects to improve performance ' - 'and to\n' - ' maintain their internal invariants.\n' - '\n' - ' Lexicographical comparison between built-in collections ' - 'works as\n' - ' follows:\n' - '\n' - ' * For two collections to compare equal, they must be of the ' - 'same\n' - ' type, have the same length, and each pair of ' - 'corresponding\n' - ' elements must compare equal (for example, "[1,2] == ' - '(1,2)" is\n' - ' false because the type is not the same).\n' - '\n' - ' * Collections that support order comparison are ordered the ' - 'same as\n' - ' their first unequal elements (for example, "[1,2,x] <= ' - '[1,2,y]"\n' - ' has the same value as "x <= y"). If a corresponding ' - 'element does\n' - ' not exist, the shorter collection is ordered first (for ' - 'example,\n' - ' "[1,2] < [1,2,3]" is true).\n' - '\n' - '* Mappings (instances of "dict") compare equal if and only if ' - 'they\n' - ' have equal "(key, value)" pairs. Equality comparison of the ' - 'keys and\n' - ' values enforces reflexivity.\n' - '\n' - ' Order comparisons ("<", ">", "<=", and ">=") raise ' - '"TypeError".\n' - '\n' - '* Sets (instances of "set" or "frozenset") can be compared ' - 'within and\n' - ' across their types.\n' - '\n' - ' They define order comparison operators to mean subset and ' - 'superset\n' - ' tests. Those relations do not define total orderings (for ' - 'example,\n' - ' the two sets "{1,2}" and "{2,3}" are not equal, nor subsets ' - 'of one\n' - ' another, nor supersets of one another). Accordingly, sets ' - 'are not\n' - ' appropriate arguments for functions which depend on total ' - 'ordering\n' - ' (for example, "min()", "max()", and "sorted()" produce ' - 'undefined\n' - ' results given a list of sets as inputs).\n' - '\n' - ' Comparison of sets enforces reflexivity of its elements.\n' - '\n' - '* Most other built-in types have no comparison methods ' - 'implemented, so\n' - ' they inherit the default comparison behavior.\n' - '\n' - 'User-defined classes that customize their comparison behavior ' - 'should\n' - 'follow some consistency rules, if possible:\n' - '\n' - '* Equality comparison should be reflexive. In other words, ' - 'identical\n' - ' objects should compare equal:\n' - '\n' - ' "x is y" implies "x == y"\n' - '\n' - '* Comparison should be symmetric. In other words, the ' - 'following\n' - ' expressions should have the same result:\n' - '\n' - ' "x == y" and "y == x"\n' - '\n' - ' "x != y" and "y != x"\n' - '\n' - ' "x < y" and "y > x"\n' - '\n' - ' "x <= y" and "y >= x"\n' - '\n' - '* Comparison should be transitive. The following ' - '(non-exhaustive)\n' - ' examples illustrate that:\n' - '\n' - ' "x > y and y > z" implies "x > z"\n' - '\n' - ' "x < y and y <= z" implies "x < z"\n' - '\n' - '* Inverse comparison should result in the boolean negation. ' - 'In other\n' - ' words, the following expressions should have the same ' - 'result:\n' - '\n' - ' "x == y" and "not x != y"\n' - '\n' - ' "x < y" and "not x >= y" (for total ordering)\n' - '\n' - ' "x > y" and "not x <= y" (for total ordering)\n' - '\n' - ' The last two expressions apply to totally ordered ' - 'collections (e.g.\n' - ' to sequences, but not to sets or mappings). See also the\n' - ' "total_ordering()" decorator.\n' - '\n' - '* The "hash()" result should be consistent with equality. ' - 'Objects that\n' - ' are equal should either have the same hash value, or be ' - 'marked as\n' - ' unhashable.\n' - '\n' - 'Python does not enforce these consistency rules. In fact, ' - 'the\n' - 'not-a-number values are an example for not following these ' - 'rules.\n' - '\n' - '\n' - 'Membership test operations\n' - '==========================\n' - '\n' - 'The operators "in" and "not in" test for membership. "x in ' - 's"\n' - 'evaluates to "True" if *x* is a member of *s*, and "False" ' - 'otherwise.\n' - '"x not in s" returns the negation of "x in s". All built-in ' - 'sequences\n' - 'and set types support this as well as dictionary, for which ' - '"in" tests\n' - 'whether the dictionary has a given key. For container types ' - 'such as\n' - 'list, tuple, set, frozenset, dict, or collections.deque, the\n' - 'expression "x in y" is equivalent to "any(x is e or x == e ' - 'for e in\n' - 'y)".\n' - '\n' - 'For the string and bytes types, "x in y" is "True" if and ' - 'only if *x*\n' - 'is a substring of *y*. An equivalent test is "y.find(x) != ' - '-1".\n' - 'Empty strings are always considered to be a substring of any ' - 'other\n' - 'string, so """ in "abc"" will return "True".\n' - '\n' - 'For user-defined classes which define the "__contains__()" ' - 'method, "x\n' - 'in y" returns "True" if "y.__contains__(x)" returns a true ' - 'value, and\n' - '"False" otherwise.\n' - '\n' - 'For user-defined classes which do not define "__contains__()" ' - 'but do\n' - 'define "__iter__()", "x in y" is "True" if some value "z", ' - 'for which\n' - 'the expression "x is z or x == z" is true, is produced while ' - 'iterating\n' - 'over "y". If an exception is raised during the iteration, it ' - 'is as if\n' - '"in" raised that exception.\n' - '\n' - 'Lastly, the old-style iteration protocol is tried: if a class ' - 'defines\n' - '"__getitem__()", "x in y" is "True" if and only if there is a ' - 'non-\n' - 'negative integer index *i* such that "x is y[i] or x == ' - 'y[i]", and no\n' - 'lower integer index raises the "IndexError" exception. (If ' - 'any other\n' - 'exception is raised, it is as if "in" raised that ' - 'exception).\n' - '\n' - 'The operator "not in" is defined to have the inverse truth ' - 'value of\n' - '"in".\n' - '\n' - '\n' - 'Identity comparisons\n' - '====================\n' - '\n' - 'The operators "is" and "is not" test for an object’s ' - 'identity: "x is\n' - 'y" is true if and only if *x* and *y* are the same object. ' - 'An\n' - 'Object’s identity is determined using the "id()" function. ' - '"x is not\n' - 'y" yields the inverse truth value. [4]\n', - 'compound': 'Compound statements\n' - '*******************\n' - '\n' - 'Compound statements contain (groups of) other statements; they ' - 'affect\n' - 'or control the execution of those other statements in some way. ' - 'In\n' - 'general, compound statements span multiple lines, although in ' - 'simple\n' - 'incarnations a whole compound statement may be contained in one ' - 'line.\n' - '\n' - 'The "if", "while" and "for" statements implement traditional ' - 'control\n' - 'flow constructs. "try" specifies exception handlers and/or ' - 'cleanup\n' - 'code for a group of statements, while the "with" statement ' - 'allows the\n' - 'execution of initialization and finalization code around a block ' - 'of\n' - 'code. Function and class definitions are also syntactically ' - 'compound\n' - 'statements.\n' - '\n' - 'A compound statement consists of one or more ‘clauses.’ A ' - 'clause\n' - 'consists of a header and a ‘suite.’ The clause headers of a\n' - 'particular compound statement are all at the same indentation ' - 'level.\n' - 'Each clause header begins with a uniquely identifying keyword ' - 'and ends\n' - 'with a colon. A suite is a group of statements controlled by a\n' - 'clause. A suite can be one or more semicolon-separated simple\n' - 'statements on the same line as the header, following the ' - 'header’s\n' - 'colon, or it can be one or more indented statements on ' - 'subsequent\n' - 'lines. Only the latter form of a suite can contain nested ' - 'compound\n' - 'statements; the following is illegal, mostly because it wouldn’t ' - 'be\n' - 'clear to which "if" clause a following "else" clause would ' - 'belong:\n' - '\n' - ' if test1: if test2: print(x)\n' - '\n' - 'Also note that the semicolon binds tighter than the colon in ' - 'this\n' - 'context, so that in the following example, either all or none of ' - 'the\n' - '"print()" calls are executed:\n' - '\n' - ' if x < y < z: print(x); print(y); print(z)\n' - '\n' - 'Summarizing:\n' - '\n' - ' compound_stmt ::= if_stmt\n' - ' | while_stmt\n' - ' | for_stmt\n' - ' | try_stmt\n' - ' | with_stmt\n' - ' | match_stmt\n' - ' | funcdef\n' - ' | classdef\n' - ' | async_with_stmt\n' - ' | async_for_stmt\n' - ' | async_funcdef\n' - ' suite ::= stmt_list NEWLINE | NEWLINE INDENT ' - 'statement+ DEDENT\n' - ' statement ::= stmt_list NEWLINE | compound_stmt\n' - ' stmt_list ::= simple_stmt (";" simple_stmt)* [";"]\n' - '\n' - 'Note that statements always end in a "NEWLINE" possibly followed ' - 'by a\n' - '"DEDENT". Also note that optional continuation clauses always ' - 'begin\n' - 'with a keyword that cannot start a statement, thus there are no\n' - 'ambiguities (the ‘dangling "else"’ problem is solved in Python ' - 'by\n' - 'requiring nested "if" statements to be indented).\n' - '\n' - 'The formatting of the grammar rules in the following sections ' - 'places\n' - 'each clause on a separate line for clarity.\n' - '\n' - '\n' - 'The "if" statement\n' - '==================\n' - '\n' - 'The "if" statement is used for conditional execution:\n' - '\n' - ' if_stmt ::= "if" assignment_expression ":" suite\n' - ' ("elif" assignment_expression ":" suite)*\n' - ' ["else" ":" suite]\n' - '\n' - 'It selects exactly one of the suites by evaluating the ' - 'expressions one\n' - 'by one until one is found to be true (see section Boolean ' - 'operations\n' - 'for the definition of true and false); then that suite is ' - 'executed\n' - '(and no other part of the "if" statement is executed or ' - 'evaluated).\n' - 'If all expressions are false, the suite of the "else" clause, ' - 'if\n' - 'present, is executed.\n' - '\n' - '\n' - 'The "while" statement\n' - '=====================\n' - '\n' - 'The "while" statement is used for repeated execution as long as ' - 'an\n' - 'expression is true:\n' - '\n' - ' while_stmt ::= "while" assignment_expression ":" suite\n' - ' ["else" ":" suite]\n' - '\n' - 'This repeatedly tests the expression and, if it is true, ' - 'executes the\n' - 'first suite; if the expression is false (which may be the first ' - 'time\n' - 'it is tested) the suite of the "else" clause, if present, is ' - 'executed\n' - 'and the loop terminates.\n' - '\n' - 'A "break" statement executed in the first suite terminates the ' - 'loop\n' - 'without executing the "else" clause’s suite. A "continue" ' - 'statement\n' - 'executed in the first suite skips the rest of the suite and goes ' - 'back\n' - 'to testing the expression.\n' - '\n' - '\n' - 'The "for" statement\n' - '===================\n' - '\n' - 'The "for" statement is used to iterate over the elements of a ' - 'sequence\n' - '(such as a string, tuple or list) or other iterable object:\n' - '\n' - ' for_stmt ::= "for" target_list "in" starred_list ":" suite\n' - ' ["else" ":" suite]\n' - '\n' - 'The "starred_list" expression is evaluated once; it should yield ' - 'an\n' - '*iterable* object. An *iterator* is created for that iterable. ' - 'The\n' - 'first item provided by the iterator is then assigned to the ' - 'target\n' - 'list using the standard rules for assignments (see Assignment\n' - 'statements), and the suite is executed. This repeats for each ' - 'item\n' - 'provided by the iterator. When the iterator is exhausted, the ' - 'suite\n' - 'in the "else" clause, if present, is executed, and the loop\n' - 'terminates.\n' - '\n' - 'A "break" statement executed in the first suite terminates the ' - 'loop\n' - 'without executing the "else" clause’s suite. A "continue" ' - 'statement\n' - 'executed in the first suite skips the rest of the suite and ' - 'continues\n' - 'with the next item, or with the "else" clause if there is no ' - 'next\n' - 'item.\n' - '\n' - 'The for-loop makes assignments to the variables in the target ' - 'list.\n' - 'This overwrites all previous assignments to those variables ' - 'including\n' - 'those made in the suite of the for-loop:\n' - '\n' - ' for i in range(10):\n' - ' print(i)\n' - ' i = 5 # this will not affect the for-loop\n' - ' # because i will be overwritten with ' - 'the next\n' - ' # index in the range\n' - '\n' - 'Names in the target list are not deleted when the loop is ' - 'finished,\n' - 'but if the sequence is empty, they will not have been assigned ' - 'to at\n' - 'all by the loop. Hint: the built-in type "range()" represents\n' - 'immutable arithmetic sequences of integers. For instance, ' - 'iterating\n' - '"range(3)" successively yields 0, 1, and then 2.\n' - '\n' - 'Changed in version 3.11: Starred elements are now allowed in ' - 'the\n' - 'expression list.\n' - '\n' - '\n' - 'The "try" statement\n' - '===================\n' - '\n' - 'The "try" statement specifies exception handlers and/or cleanup ' - 'code\n' - 'for a group of statements:\n' - '\n' - ' try_stmt ::= try1_stmt | try2_stmt | try3_stmt\n' - ' try1_stmt ::= "try" ":" suite\n' - ' ("except" [expression ["as" identifier]] ":" ' - 'suite)+\n' - ' ["else" ":" suite]\n' - ' ["finally" ":" suite]\n' - ' try2_stmt ::= "try" ":" suite\n' - ' ("except" "*" expression ["as" identifier] ":" ' - 'suite)+\n' - ' ["else" ":" suite]\n' - ' ["finally" ":" suite]\n' - ' try3_stmt ::= "try" ":" suite\n' - ' "finally" ":" suite\n' - '\n' - 'Additional information on exceptions can be found in section\n' - 'Exceptions, and information on using the "raise" statement to ' - 'generate\n' - 'exceptions may be found in section The raise statement.\n' - '\n' - '\n' - '"except" clause\n' - '---------------\n' - '\n' - 'The "except" clause(s) specify one or more exception handlers. ' - 'When no\n' - 'exception occurs in the "try" clause, no exception handler is\n' - 'executed. When an exception occurs in the "try" suite, a search ' - 'for an\n' - 'exception handler is started. This search inspects the "except"\n' - 'clauses in turn until one is found that matches the exception. ' - 'An\n' - 'expression-less "except" clause, if present, must be last; it ' - 'matches\n' - 'any exception. For an "except" clause with an expression, that\n' - 'expression is evaluated, and the clause matches the exception if ' - 'the\n' - 'resulting object is “compatible” with the exception. An object ' - 'is\n' - 'compatible with an exception if the object is the class or a ' - '*non-\n' - 'virtual base class* of the exception object, or a tuple ' - 'containing an\n' - 'item that is the class or a non-virtual base class of the ' - 'exception\n' - 'object.\n' - '\n' - 'If no "except" clause matches the exception, the search for an\n' - 'exception handler continues in the surrounding code and on the\n' - 'invocation stack. [1]\n' - '\n' - 'If the evaluation of an expression in the header of an "except" ' - 'clause\n' - 'raises an exception, the original search for a handler is ' - 'canceled and\n' - 'a search starts for the new exception in the surrounding code ' - 'and on\n' - 'the call stack (it is treated as if the entire "try" statement ' - 'raised\n' - 'the exception).\n' - '\n' - 'When a matching "except" clause is found, the exception is ' - 'assigned to\n' - 'the target specified after the "as" keyword in that "except" ' - 'clause,\n' - 'if present, and the "except" clause’s suite is executed. All ' - '"except"\n' - 'clauses must have an executable block. When the end of this ' - 'block is\n' - 'reached, execution continues normally after the entire "try"\n' - 'statement. (This means that if two nested handlers exist for the ' - 'same\n' - 'exception, and the exception occurs in the "try" clause of the ' - 'inner\n' - 'handler, the outer handler will not handle the exception.)\n' - '\n' - 'When an exception has been assigned using "as target", it is ' - 'cleared\n' - 'at the end of the "except" clause. This is as if\n' - '\n' - ' except E as N:\n' - ' foo\n' - '\n' - 'was translated to\n' - '\n' - ' except E as N:\n' - ' try:\n' - ' foo\n' - ' finally:\n' - ' del N\n' - '\n' - 'This means the exception must be assigned to a different name to ' - 'be\n' - 'able to refer to it after the "except" clause. Exceptions are ' - 'cleared\n' - 'because with the traceback attached to them, they form a ' - 'reference\n' - 'cycle with the stack frame, keeping all locals in that frame ' - 'alive\n' - 'until the next garbage collection occurs.\n' - '\n' - 'Before an "except" clause’s suite is executed, the exception is ' - 'stored\n' - 'in the "sys" module, where it can be accessed from within the ' - 'body of\n' - 'the "except" clause by calling "sys.exception()". When leaving ' - 'an\n' - 'exception handler, the exception stored in the "sys" module is ' - 'reset\n' - 'to its previous value:\n' - '\n' - ' >>> print(sys.exception())\n' - ' None\n' - ' >>> try:\n' - ' ... raise TypeError\n' - ' ... except:\n' - ' ... print(repr(sys.exception()))\n' - ' ... try:\n' - ' ... raise ValueError\n' - ' ... except:\n' - ' ... print(repr(sys.exception()))\n' - ' ... print(repr(sys.exception()))\n' - ' ...\n' - ' TypeError()\n' - ' ValueError()\n' - ' TypeError()\n' - ' >>> print(sys.exception())\n' - ' None\n' - '\n' - '\n' - '"except*" clause\n' - '----------------\n' - '\n' - 'The "except*" clause(s) are used for handling "ExceptionGroup"s. ' - 'The\n' - 'exception type for matching is interpreted as in the case of ' - '"except",\n' - 'but in the case of exception groups we can have partial matches ' - 'when\n' - 'the type matches some of the exceptions in the group. This means ' - 'that\n' - 'multiple "except*" clauses can execute, each handling part of ' - 'the\n' - 'exception group. Each clause executes at most once and handles ' - 'an\n' - 'exception group of all matching exceptions. Each exception in ' - 'the\n' - 'group is handled by at most one "except*" clause, the first ' - 'that\n' - 'matches it.\n' - '\n' - ' >>> try:\n' - ' ... raise ExceptionGroup("eg",\n' - ' ... [ValueError(1), TypeError(2), OSError(3), ' - 'OSError(4)])\n' - ' ... except* TypeError as e:\n' - " ... print(f'caught {type(e)} with nested " - "{e.exceptions}')\n" - ' ... except* OSError as e:\n' - " ... print(f'caught {type(e)} with nested " - "{e.exceptions}')\n" - ' ...\n' - " caught with nested (TypeError(2),)\n" - " caught with nested (OSError(3), " - 'OSError(4))\n' - ' + Exception Group Traceback (most recent call last):\n' - ' | File "", line 2, in \n' - ' | ExceptionGroup: eg\n' - ' +-+---------------- 1 ----------------\n' - ' | ValueError: 1\n' - ' +------------------------------------\n' - '\n' - 'Any remaining exceptions that were not handled by any "except*" ' - 'clause\n' - 'are re-raised at the end, along with all exceptions that were ' - 'raised\n' - 'from within the "except*" clauses. If this list contains more ' - 'than one\n' - 'exception to reraise, they are combined into an exception ' - 'group.\n' - '\n' - 'If the raised exception is not an exception group and its type ' - 'matches\n' - 'one of the "except*" clauses, it is caught and wrapped by an ' - 'exception\n' - 'group with an empty message string.\n' - '\n' - ' >>> try:\n' - ' ... raise BlockingIOError\n' - ' ... except* BlockingIOError as e:\n' - ' ... print(repr(e))\n' - ' ...\n' - " ExceptionGroup('', (BlockingIOError()))\n" - '\n' - 'An "except*" clause must have a matching type, and this type ' - 'cannot be\n' - 'a subclass of "BaseExceptionGroup". It is not possible to mix ' - '"except"\n' - 'and "except*" in the same "try". "break", "continue" and ' - '"return"\n' - 'cannot appear in an "except*" clause.\n' - '\n' - '\n' - '"else" clause\n' - '-------------\n' - '\n' - 'The optional "else" clause is executed if the control flow ' - 'leaves the\n' - '"try" suite, no exception was raised, and no "return", ' - '"continue", or\n' - '"break" statement was executed. Exceptions in the "else" clause ' - 'are\n' - 'not handled by the preceding "except" clauses.\n' - '\n' - '\n' - '"finally" clause\n' - '----------------\n' - '\n' - 'If "finally" is present, it specifies a ‘cleanup’ handler. The ' - '"try"\n' - 'clause is executed, including any "except" and "else" clauses. ' - 'If an\n' - 'exception occurs in any of the clauses and is not handled, the\n' - 'exception is temporarily saved. The "finally" clause is ' - 'executed. If\n' - 'there is a saved exception it is re-raised at the end of the ' - '"finally"\n' - 'clause. If the "finally" clause raises another exception, the ' - 'saved\n' - 'exception is set as the context of the new exception. If the ' - '"finally"\n' - 'clause executes a "return", "break" or "continue" statement, the ' - 'saved\n' - 'exception is discarded:\n' - '\n' - ' >>> def f():\n' - ' ... try:\n' - ' ... 1/0\n' - ' ... finally:\n' - ' ... return 42\n' - ' ...\n' - ' >>> f()\n' - ' 42\n' - '\n' - 'The exception information is not available to the program ' - 'during\n' - 'execution of the "finally" clause.\n' - '\n' - 'When a "return", "break" or "continue" statement is executed in ' - 'the\n' - '"try" suite of a "try"…"finally" statement, the "finally" clause ' - 'is\n' - 'also executed ‘on the way out.’\n' - '\n' - 'The return value of a function is determined by the last ' - '"return"\n' - 'statement executed. Since the "finally" clause always executes, ' - 'a\n' - '"return" statement executed in the "finally" clause will always ' - 'be the\n' - 'last one executed:\n' - '\n' - ' >>> def foo():\n' - ' ... try:\n' - " ... return 'try'\n" - ' ... finally:\n' - " ... return 'finally'\n" - ' ...\n' - ' >>> foo()\n' - " 'finally'\n" - '\n' - 'Changed in version 3.8: Prior to Python 3.8, a "continue" ' - 'statement\n' - 'was illegal in the "finally" clause due to a problem with the\n' - 'implementation.\n' - '\n' - '\n' - 'The "with" statement\n' - '====================\n' - '\n' - 'The "with" statement is used to wrap the execution of a block ' - 'with\n' - 'methods defined by a context manager (see section With ' - 'Statement\n' - 'Context Managers). This allows common "try"…"except"…"finally" ' - 'usage\n' - 'patterns to be encapsulated for convenient reuse.\n' - '\n' - ' with_stmt ::= "with" ( "(" with_stmt_contents ","? ' - '")" | with_stmt_contents ) ":" suite\n' - ' with_stmt_contents ::= with_item ("," with_item)*\n' - ' with_item ::= expression ["as" target]\n' - '\n' - 'The execution of the "with" statement with one “item” proceeds ' - 'as\n' - 'follows:\n' - '\n' - '1. The context expression (the expression given in the ' - '"with_item") is\n' - ' evaluated to obtain a context manager.\n' - '\n' - '2. The context manager’s "__enter__()" is loaded for later use.\n' - '\n' - '3. The context manager’s "__exit__()" is loaded for later use.\n' - '\n' - '4. The context manager’s "__enter__()" method is invoked.\n' - '\n' - '5. If a target was included in the "with" statement, the return ' - 'value\n' - ' from "__enter__()" is assigned to it.\n' - '\n' - ' Note:\n' - '\n' - ' The "with" statement guarantees that if the "__enter__()" ' - 'method\n' - ' returns without an error, then "__exit__()" will always be\n' - ' called. Thus, if an error occurs during the assignment to ' - 'the\n' - ' target list, it will be treated the same as an error ' - 'occurring\n' - ' within the suite would be. See step 7 below.\n' - '\n' - '6. The suite is executed.\n' - '\n' - '7. The context manager’s "__exit__()" method is invoked. If an\n' - ' exception caused the suite to be exited, its type, value, ' - 'and\n' - ' traceback are passed as arguments to "__exit__()". Otherwise, ' - 'three\n' - ' "None" arguments are supplied.\n' - '\n' - ' If the suite was exited due to an exception, and the return ' - 'value\n' - ' from the "__exit__()" method was false, the exception is ' - 'reraised.\n' - ' If the return value was true, the exception is suppressed, ' - 'and\n' - ' execution continues with the statement following the "with"\n' - ' statement.\n' - '\n' - ' If the suite was exited for any reason other than an ' - 'exception, the\n' - ' return value from "__exit__()" is ignored, and execution ' - 'proceeds\n' - ' at the normal location for the kind of exit that was taken.\n' - '\n' - 'The following code:\n' - '\n' - ' with EXPRESSION as TARGET:\n' - ' SUITE\n' - '\n' - 'is semantically equivalent to:\n' - '\n' - ' manager = (EXPRESSION)\n' - ' enter = type(manager).__enter__\n' - ' exit = type(manager).__exit__\n' - ' value = enter(manager)\n' - ' hit_except = False\n' - '\n' - ' try:\n' - ' TARGET = value\n' - ' SUITE\n' - ' except:\n' - ' hit_except = True\n' - ' if not exit(manager, *sys.exc_info()):\n' - ' raise\n' - ' finally:\n' - ' if not hit_except:\n' - ' exit(manager, None, None, None)\n' - '\n' - 'With more than one item, the context managers are processed as ' - 'if\n' - 'multiple "with" statements were nested:\n' - '\n' - ' with A() as a, B() as b:\n' - ' SUITE\n' - '\n' - 'is semantically equivalent to:\n' - '\n' - ' with A() as a:\n' - ' with B() as b:\n' - ' SUITE\n' - '\n' - 'You can also write multi-item context managers in multiple lines ' - 'if\n' - 'the items are surrounded by parentheses. For example:\n' - '\n' - ' with (\n' - ' A() as a,\n' - ' B() as b,\n' - ' ):\n' - ' SUITE\n' - '\n' - 'Changed in version 3.1: Support for multiple context ' - 'expressions.\n' - '\n' - 'Changed in version 3.10: Support for using grouping parentheses ' - 'to\n' - 'break the statement in multiple lines.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 343** - The “with” statement\n' - ' The specification, background, and examples for the Python ' - '"with"\n' - ' statement.\n' - '\n' - '\n' - 'The "match" statement\n' - '=====================\n' - '\n' - 'New in version 3.10.\n' - '\n' - 'The match statement is used for pattern matching. Syntax:\n' - '\n' - ' match_stmt ::= \'match\' subject_expr ":" NEWLINE INDENT ' - 'case_block+ DEDENT\n' - ' subject_expr ::= star_named_expression "," ' - 'star_named_expressions?\n' - ' | named_expression\n' - ' case_block ::= \'case\' patterns [guard] ":" block\n' - '\n' - 'Note:\n' - '\n' - ' This section uses single quotes to denote soft keywords.\n' - '\n' - 'Pattern matching takes a pattern as input (following "case") and ' - 'a\n' - 'subject value (following "match"). The pattern (which may ' - 'contain\n' - 'subpatterns) is matched against the subject value. The outcomes ' - 'are:\n' - '\n' - '* A match success or failure (also termed a pattern success or\n' - ' failure).\n' - '\n' - '* Possible binding of matched values to a name. The ' - 'prerequisites for\n' - ' this are further discussed below.\n' - '\n' - 'The "match" and "case" keywords are soft keywords.\n' - '\n' - 'See also:\n' - '\n' - ' * **PEP 634** – Structural Pattern Matching: Specification\n' - '\n' - ' * **PEP 636** – Structural Pattern Matching: Tutorial\n' - '\n' - '\n' - 'Overview\n' - '--------\n' - '\n' - 'Here’s an overview of the logical flow of a match statement:\n' - '\n' - '1. The subject expression "subject_expr" is evaluated and a ' - 'resulting\n' - ' subject value obtained. If the subject expression contains a ' - 'comma,\n' - ' a tuple is constructed using the standard rules.\n' - '\n' - '2. Each pattern in a "case_block" is attempted to match with ' - 'the\n' - ' subject value. The specific rules for success or failure are\n' - ' described below. The match attempt can also bind some or all ' - 'of the\n' - ' standalone names within the pattern. The precise pattern ' - 'binding\n' - ' rules vary per pattern type and are specified below. **Name\n' - ' bindings made during a successful pattern match outlive the\n' - ' executed block and can be used after the match statement**.\n' - '\n' - ' Note:\n' - '\n' - ' During failed pattern matches, some subpatterns may ' - 'succeed. Do\n' - ' not rely on bindings being made for a failed match. ' - 'Conversely,\n' - ' do not rely on variables remaining unchanged after a ' - 'failed\n' - ' match. The exact behavior is dependent on implementation ' - 'and may\n' - ' vary. This is an intentional decision made to allow ' - 'different\n' - ' implementations to add optimizations.\n' - '\n' - '3. If the pattern succeeds, the corresponding guard (if present) ' - 'is\n' - ' evaluated. In this case all name bindings are guaranteed to ' - 'have\n' - ' happened.\n' - '\n' - ' * If the guard evaluates as true or is missing, the "block" ' - 'inside\n' - ' "case_block" is executed.\n' - '\n' - ' * Otherwise, the next "case_block" is attempted as described ' - 'above.\n' - '\n' - ' * If there are no further case blocks, the match statement ' - 'is\n' - ' completed.\n' - '\n' - 'Note:\n' - '\n' - ' Users should generally never rely on a pattern being ' - 'evaluated.\n' - ' Depending on implementation, the interpreter may cache values ' - 'or use\n' - ' other optimizations which skip repeated evaluations.\n' - '\n' - 'A sample match statement:\n' - '\n' - ' >>> flag = False\n' - ' >>> match (100, 200):\n' - ' ... case (100, 300): # Mismatch: 200 != 300\n' - " ... print('Case 1')\n" - ' ... case (100, 200) if flag: # Successful match, but ' - 'guard fails\n' - " ... print('Case 2')\n" - ' ... case (100, y): # Matches and binds y to 200\n' - " ... print(f'Case 3, y: {y}')\n" - ' ... case _: # Pattern not attempted\n' - " ... print('Case 4, I match anything!')\n" - ' ...\n' - ' Case 3, y: 200\n' - '\n' - 'In this case, "if flag" is a guard. Read more about that in the ' - 'next\n' - 'section.\n' - '\n' - '\n' - 'Guards\n' - '------\n' - '\n' - ' guard ::= "if" named_expression\n' - '\n' - 'A "guard" (which is part of the "case") must succeed for code ' - 'inside\n' - 'the "case" block to execute. It takes the form: "if" followed ' - 'by an\n' - 'expression.\n' - '\n' - 'The logical flow of a "case" block with a "guard" follows:\n' - '\n' - '1. Check that the pattern in the "case" block succeeded. If ' - 'the\n' - ' pattern failed, the "guard" is not evaluated and the next ' - '"case"\n' - ' block is checked.\n' - '\n' - '2. If the pattern succeeded, evaluate the "guard".\n' - '\n' - ' * If the "guard" condition evaluates as true, the case block ' - 'is\n' - ' selected.\n' - '\n' - ' * If the "guard" condition evaluates as false, the case block ' - 'is\n' - ' not selected.\n' - '\n' - ' * If the "guard" raises an exception during evaluation, the\n' - ' exception bubbles up.\n' - '\n' - 'Guards are allowed to have side effects as they are ' - 'expressions.\n' - 'Guard evaluation must proceed from the first to the last case ' - 'block,\n' - 'one at a time, skipping case blocks whose pattern(s) don’t all\n' - 'succeed. (I.e., guard evaluation must happen in order.) Guard\n' - 'evaluation must stop once a case block is selected.\n' - '\n' - '\n' - 'Irrefutable Case Blocks\n' - '-----------------------\n' - '\n' - 'An irrefutable case block is a match-all case block. A match\n' - 'statement may have at most one irrefutable case block, and it ' - 'must be\n' - 'last.\n' - '\n' - 'A case block is considered irrefutable if it has no guard and ' - 'its\n' - 'pattern is irrefutable. A pattern is considered irrefutable if ' - 'we can\n' - 'prove from its syntax alone that it will always succeed. Only ' - 'the\n' - 'following patterns are irrefutable:\n' - '\n' - '* AS Patterns whose left-hand side is irrefutable\n' - '\n' - '* OR Patterns containing at least one irrefutable pattern\n' - '\n' - '* Capture Patterns\n' - '\n' - '* Wildcard Patterns\n' - '\n' - '* parenthesized irrefutable patterns\n' - '\n' - '\n' - 'Patterns\n' - '--------\n' - '\n' - 'Note:\n' - '\n' - ' This section uses grammar notations beyond standard EBNF:\n' - '\n' - ' * the notation "SEP.RULE+" is shorthand for "RULE (SEP ' - 'RULE)*"\n' - '\n' - ' * the notation "!RULE" is shorthand for a negative lookahead\n' - ' assertion\n' - '\n' - 'The top-level syntax for "patterns" is:\n' - '\n' - ' patterns ::= open_sequence_pattern | pattern\n' - ' pattern ::= as_pattern | or_pattern\n' - ' closed_pattern ::= | literal_pattern\n' - ' | capture_pattern\n' - ' | wildcard_pattern\n' - ' | value_pattern\n' - ' | group_pattern\n' - ' | sequence_pattern\n' - ' | mapping_pattern\n' - ' | class_pattern\n' - '\n' - 'The descriptions below will include a description “in simple ' - 'terms” of\n' - 'what a pattern does for illustration purposes (credits to ' - 'Raymond\n' - 'Hettinger for a document that inspired most of the ' - 'descriptions). Note\n' - 'that these descriptions are purely for illustration purposes and ' - '**may\n' - 'not** reflect the underlying implementation. Furthermore, they ' - 'do not\n' - 'cover all valid forms.\n' - '\n' - '\n' - 'OR Patterns\n' - '~~~~~~~~~~~\n' - '\n' - 'An OR pattern is two or more patterns separated by vertical bars ' - '"|".\n' - 'Syntax:\n' - '\n' - ' or_pattern ::= "|".closed_pattern+\n' - '\n' - 'Only the final subpattern may be irrefutable, and each ' - 'subpattern must\n' - 'bind the same set of names to avoid ambiguity.\n' - '\n' - 'An OR pattern matches each of its subpatterns in turn to the ' - 'subject\n' - 'value, until one succeeds. The OR pattern is then considered\n' - 'successful. Otherwise, if none of the subpatterns succeed, the ' - 'OR\n' - 'pattern fails.\n' - '\n' - 'In simple terms, "P1 | P2 | ..." will try to match "P1", if it ' - 'fails\n' - 'it will try to match "P2", succeeding immediately if any ' - 'succeeds,\n' - 'failing otherwise.\n' - '\n' - '\n' - 'AS Patterns\n' - '~~~~~~~~~~~\n' - '\n' - 'An AS pattern matches an OR pattern on the left of the "as" ' - 'keyword\n' - 'against a subject. Syntax:\n' - '\n' - ' as_pattern ::= or_pattern "as" capture_pattern\n' - '\n' - 'If the OR pattern fails, the AS pattern fails. Otherwise, the ' - 'AS\n' - 'pattern binds the subject to the name on the right of the as ' - 'keyword\n' - 'and succeeds. "capture_pattern" cannot be a "_".\n' - '\n' - 'In simple terms "P as NAME" will match with "P", and on success ' - 'it\n' - 'will set "NAME = ".\n' - '\n' - '\n' - 'Literal Patterns\n' - '~~~~~~~~~~~~~~~~\n' - '\n' - 'A literal pattern corresponds to most literals in Python. ' - 'Syntax:\n' - '\n' - ' literal_pattern ::= signed_number\n' - ' | signed_number "+" NUMBER\n' - ' | signed_number "-" NUMBER\n' - ' | strings\n' - ' | "None"\n' - ' | "True"\n' - ' | "False"\n' - ' | signed_number: NUMBER | "-" NUMBER\n' - '\n' - 'The rule "strings" and the token "NUMBER" are defined in the ' - 'standard\n' - 'Python grammar. Triple-quoted strings are supported. Raw ' - 'strings and\n' - 'byte strings are supported. f-strings are not supported.\n' - '\n' - 'The forms "signed_number \'+\' NUMBER" and "signed_number \'-\' ' - 'NUMBER"\n' - 'are for expressing complex numbers; they require a real number ' - 'on the\n' - 'left and an imaginary number on the right. E.g. "3 + 4j".\n' - '\n' - 'In simple terms, "LITERAL" will succeed only if " ==\n' - 'LITERAL". For the singletons "None", "True" and "False", the ' - '"is"\n' - 'operator is used.\n' - '\n' - '\n' - 'Capture Patterns\n' - '~~~~~~~~~~~~~~~~\n' - '\n' - 'A capture pattern binds the subject value to a name. Syntax:\n' - '\n' - " capture_pattern ::= !'_' NAME\n" - '\n' - 'A single underscore "_" is not a capture pattern (this is what ' - '"!\'_\'"\n' - 'expresses). It is instead treated as a "wildcard_pattern".\n' - '\n' - 'In a given pattern, a given name can only be bound once. E.g. ' - '"case\n' - 'x, x: ..." is invalid while "case [x] | x: ..." is allowed.\n' - '\n' - 'Capture patterns always succeed. The binding follows scoping ' - 'rules\n' - 'established by the assignment expression operator in **PEP ' - '572**; the\n' - 'name becomes a local variable in the closest containing function ' - 'scope\n' - 'unless there’s an applicable "global" or "nonlocal" statement.\n' - '\n' - 'In simple terms "NAME" will always succeed and it will set "NAME ' - '=\n' - '".\n' - '\n' - '\n' - 'Wildcard Patterns\n' - '~~~~~~~~~~~~~~~~~\n' - '\n' - 'A wildcard pattern always succeeds (matches anything) and binds ' - 'no\n' - 'name. Syntax:\n' - '\n' - " wildcard_pattern ::= '_'\n" - '\n' - '"_" is a soft keyword within any pattern, but only within ' - 'patterns.\n' - 'It is an identifier, as usual, even within "match" subject\n' - 'expressions, "guard"s, and "case" blocks.\n' - '\n' - 'In simple terms, "_" will always succeed.\n' - '\n' - '\n' - 'Value Patterns\n' - '~~~~~~~~~~~~~~\n' - '\n' - 'A value pattern represents a named value in Python. Syntax:\n' - '\n' - ' value_pattern ::= attr\n' - ' attr ::= name_or_attr "." NAME\n' - ' name_or_attr ::= attr | NAME\n' - '\n' - 'The dotted name in the pattern is looked up using standard ' - 'Python name\n' - 'resolution rules. The pattern succeeds if the value found ' - 'compares\n' - 'equal to the subject value (using the "==" equality operator).\n' - '\n' - 'In simple terms "NAME1.NAME2" will succeed only if " ' - '==\n' - 'NAME1.NAME2"\n' - '\n' - 'Note:\n' - '\n' - ' If the same value occurs multiple times in the same match ' - 'statement,\n' - ' the interpreter may cache the first value found and reuse it ' - 'rather\n' - ' than repeat the same lookup. This cache is strictly tied to a ' - 'given\n' - ' execution of a given match statement.\n' - '\n' - '\n' - 'Group Patterns\n' - '~~~~~~~~~~~~~~\n' - '\n' - 'A group pattern allows users to add parentheses around patterns ' - 'to\n' - 'emphasize the intended grouping. Otherwise, it has no ' - 'additional\n' - 'syntax. Syntax:\n' - '\n' - ' group_pattern ::= "(" pattern ")"\n' - '\n' - 'In simple terms "(P)" has the same effect as "P".\n' - '\n' - '\n' - 'Sequence Patterns\n' - '~~~~~~~~~~~~~~~~~\n' - '\n' - 'A sequence pattern contains several subpatterns to be matched ' - 'against\n' - 'sequence elements. The syntax is similar to the unpacking of a ' - 'list or\n' - 'tuple.\n' - '\n' - ' sequence_pattern ::= "[" [maybe_sequence_pattern] "]"\n' - ' | "(" [open_sequence_pattern] ")"\n' - ' open_sequence_pattern ::= maybe_star_pattern "," ' - '[maybe_sequence_pattern]\n' - ' maybe_sequence_pattern ::= ",".maybe_star_pattern+ ","?\n' - ' maybe_star_pattern ::= star_pattern | pattern\n' - ' star_pattern ::= "*" (capture_pattern | ' - 'wildcard_pattern)\n' - '\n' - 'There is no difference if parentheses or square brackets are ' - 'used for\n' - 'sequence patterns (i.e. "(...)" vs "[...]" ).\n' - '\n' - 'Note:\n' - '\n' - ' A single pattern enclosed in parentheses without a trailing ' - 'comma\n' - ' (e.g. "(3 | 4)") is a group pattern. While a single pattern ' - 'enclosed\n' - ' in square brackets (e.g. "[3 | 4]") is still a sequence ' - 'pattern.\n' - '\n' - 'At most one star subpattern may be in a sequence pattern. The ' - 'star\n' - 'subpattern may occur in any position. If no star subpattern is\n' - 'present, the sequence pattern is a fixed-length sequence ' - 'pattern;\n' - 'otherwise it is a variable-length sequence pattern.\n' - '\n' - 'The following is the logical flow for matching a sequence ' - 'pattern\n' - 'against a subject value:\n' - '\n' - '1. If the subject value is not a sequence [2], the sequence ' - 'pattern\n' - ' fails.\n' - '\n' - '2. If the subject value is an instance of "str", "bytes" or\n' - ' "bytearray" the sequence pattern fails.\n' - '\n' - '3. The subsequent steps depend on whether the sequence pattern ' - 'is\n' - ' fixed or variable-length.\n' - '\n' - ' If the sequence pattern is fixed-length:\n' - '\n' - ' 1. If the length of the subject sequence is not equal to the ' - 'number\n' - ' of subpatterns, the sequence pattern fails\n' - '\n' - ' 2. Subpatterns in the sequence pattern are matched to their\n' - ' corresponding items in the subject sequence from left to ' - 'right.\n' - ' Matching stops as soon as a subpattern fails. If all\n' - ' subpatterns succeed in matching their corresponding item, ' - 'the\n' - ' sequence pattern succeeds.\n' - '\n' - ' Otherwise, if the sequence pattern is variable-length:\n' - '\n' - ' 1. If the length of the subject sequence is less than the ' - 'number of\n' - ' non-star subpatterns, the sequence pattern fails.\n' - '\n' - ' 2. The leading non-star subpatterns are matched to their\n' - ' corresponding items as for fixed-length sequences.\n' - '\n' - ' 3. If the previous step succeeds, the star subpattern matches ' - 'a\n' - ' list formed of the remaining subject items, excluding the\n' - ' remaining items corresponding to non-star subpatterns ' - 'following\n' - ' the star subpattern.\n' - '\n' - ' 4. Remaining non-star subpatterns are matched to their\n' - ' corresponding subject items, as for a fixed-length ' - 'sequence.\n' - '\n' - ' Note:\n' - '\n' - ' The length of the subject sequence is obtained via "len()" ' - '(i.e.\n' - ' via the "__len__()" protocol). This length may be cached ' - 'by the\n' - ' interpreter in a similar manner as value patterns.\n' - '\n' - 'In simple terms "[P1, P2, P3," … ", P]" matches only if all ' - 'the\n' - 'following happens:\n' - '\n' - '* check "" is a sequence\n' - '\n' - '* "len(subject) == "\n' - '\n' - '* "P1" matches "[0]" (note that this match can also ' - 'bind\n' - ' names)\n' - '\n' - '* "P2" matches "[1]" (note that this match can also ' - 'bind\n' - ' names)\n' - '\n' - '* … and so on for the corresponding pattern/element.\n' - '\n' - '\n' - 'Mapping Patterns\n' - '~~~~~~~~~~~~~~~~\n' - '\n' - 'A mapping pattern contains one or more key-value patterns. The ' - 'syntax\n' - 'is similar to the construction of a dictionary. Syntax:\n' - '\n' - ' mapping_pattern ::= "{" [items_pattern] "}"\n' - ' items_pattern ::= ",".key_value_pattern+ ","?\n' - ' key_value_pattern ::= (literal_pattern | value_pattern) ":" ' - 'pattern\n' - ' | double_star_pattern\n' - ' double_star_pattern ::= "**" capture_pattern\n' - '\n' - 'At most one double star pattern may be in a mapping pattern. ' - 'The\n' - 'double star pattern must be the last subpattern in the mapping\n' - 'pattern.\n' - '\n' - 'Duplicate keys in mapping patterns are disallowed. Duplicate ' - 'literal\n' - 'keys will raise a "SyntaxError". Two keys that otherwise have ' - 'the same\n' - 'value will raise a "ValueError" at runtime.\n' - '\n' - 'The following is the logical flow for matching a mapping ' - 'pattern\n' - 'against a subject value:\n' - '\n' - '1. If the subject value is not a mapping [3],the mapping ' - 'pattern\n' - ' fails.\n' - '\n' - '2. If every key given in the mapping pattern is present in the ' - 'subject\n' - ' mapping, and the pattern for each key matches the ' - 'corresponding\n' - ' item of the subject mapping, the mapping pattern succeeds.\n' - '\n' - '3. If duplicate keys are detected in the mapping pattern, the ' - 'pattern\n' - ' is considered invalid. A "SyntaxError" is raised for ' - 'duplicate\n' - ' literal values; or a "ValueError" for named keys of the same ' - 'value.\n' - '\n' - 'Note:\n' - '\n' - ' Key-value pairs are matched using the two-argument form of ' - 'the\n' - ' mapping subject’s "get()" method. Matched key-value pairs ' - 'must\n' - ' already be present in the mapping, and not created on-the-fly ' - 'via\n' - ' "__missing__()" or "__getitem__()".\n' - '\n' - 'In simple terms "{KEY1: P1, KEY2: P2, ... }" matches only if all ' - 'the\n' - 'following happens:\n' - '\n' - '* check "" is a mapping\n' - '\n' - '* "KEY1 in "\n' - '\n' - '* "P1" matches "[KEY1]"\n' - '\n' - '* … and so on for the corresponding KEY/pattern pair.\n' - '\n' - '\n' - 'Class Patterns\n' - '~~~~~~~~~~~~~~\n' - '\n' - 'A class pattern represents a class and its positional and ' - 'keyword\n' - 'arguments (if any). Syntax:\n' - '\n' - ' class_pattern ::= name_or_attr "(" [pattern_arguments ' - '","?] ")"\n' - ' pattern_arguments ::= positional_patterns ["," ' - 'keyword_patterns]\n' - ' | keyword_patterns\n' - ' positional_patterns ::= ",".pattern+\n' - ' keyword_patterns ::= ",".keyword_pattern+\n' - ' keyword_pattern ::= NAME "=" pattern\n' - '\n' - 'The same keyword should not be repeated in class patterns.\n' - '\n' - 'The following is the logical flow for matching a class pattern ' - 'against\n' - 'a subject value:\n' - '\n' - '1. If "name_or_attr" is not an instance of the builtin "type" , ' - 'raise\n' - ' "TypeError".\n' - '\n' - '2. If the subject value is not an instance of "name_or_attr" ' - '(tested\n' - ' via "isinstance()"), the class pattern fails.\n' - '\n' - '3. If no pattern arguments are present, the pattern succeeds.\n' - ' Otherwise, the subsequent steps depend on whether keyword or\n' - ' positional argument patterns are present.\n' - '\n' - ' For a number of built-in types (specified below), a single\n' - ' positional subpattern is accepted which will match the ' - 'entire\n' - ' subject; for these types keyword patterns also work as for ' - 'other\n' - ' types.\n' - '\n' - ' If only keyword patterns are present, they are processed as\n' - ' follows, one by one:\n' - '\n' - ' I. The keyword is looked up as an attribute on the subject.\n' - '\n' - ' * If this raises an exception other than "AttributeError", ' - 'the\n' - ' exception bubbles up.\n' - '\n' - ' * If this raises "AttributeError", the class pattern has ' - 'failed.\n' - '\n' - ' * Else, the subpattern associated with the keyword pattern ' - 'is\n' - ' matched against the subject’s attribute value. If this ' - 'fails,\n' - ' the class pattern fails; if this succeeds, the match ' - 'proceeds\n' - ' to the next keyword.\n' - '\n' - ' II. If all keyword patterns succeed, the class pattern ' - 'succeeds.\n' - '\n' - ' If any positional patterns are present, they are converted ' - 'to\n' - ' keyword patterns using the "__match_args__" attribute on the ' - 'class\n' - ' "name_or_attr" before matching:\n' - '\n' - ' I. The equivalent of "getattr(cls, "__match_args__", ())" is\n' - ' called.\n' - '\n' - ' * If this raises an exception, the exception bubbles up.\n' - '\n' - ' * If the returned value is not a tuple, the conversion ' - 'fails and\n' - ' "TypeError" is raised.\n' - '\n' - ' * If there are more positional patterns than\n' - ' "len(cls.__match_args__)", "TypeError" is raised.\n' - '\n' - ' * Otherwise, positional pattern "i" is converted to a ' - 'keyword\n' - ' pattern using "__match_args__[i]" as the keyword.\n' - ' "__match_args__[i]" must be a string; if not "TypeError" ' - 'is\n' - ' raised.\n' - '\n' - ' * If there are duplicate keywords, "TypeError" is raised.\n' - '\n' - ' See also:\n' - '\n' - ' Customizing positional arguments in class pattern ' - 'matching\n' - '\n' - ' II. Once all positional patterns have been converted to ' - 'keyword\n' - ' patterns,\n' - ' the match proceeds as if there were only keyword ' - 'patterns.\n' - '\n' - ' For the following built-in types the handling of positional\n' - ' subpatterns is different:\n' - '\n' - ' * "bool"\n' - '\n' - ' * "bytearray"\n' - '\n' - ' * "bytes"\n' - '\n' - ' * "dict"\n' - '\n' - ' * "float"\n' - '\n' - ' * "frozenset"\n' - '\n' - ' * "int"\n' - '\n' - ' * "list"\n' - '\n' - ' * "set"\n' - '\n' - ' * "str"\n' - '\n' - ' * "tuple"\n' - '\n' - ' These classes accept a single positional argument, and the ' - 'pattern\n' - ' there is matched against the whole object rather than an ' - 'attribute.\n' - ' For example "int(0|1)" matches the value "0", but not the ' - 'value\n' - ' "0.0".\n' - '\n' - 'In simple terms "CLS(P1, attr=P2)" matches only if the ' - 'following\n' - 'happens:\n' - '\n' - '* "isinstance(, CLS)"\n' - '\n' - '* convert "P1" to a keyword pattern using "CLS.__match_args__"\n' - '\n' - '* For each keyword argument "attr=P2":\n' - '\n' - ' * "hasattr(, "attr")"\n' - '\n' - ' * "P2" matches ".attr"\n' - '\n' - '* … and so on for the corresponding keyword argument/pattern ' - 'pair.\n' - '\n' - 'See also:\n' - '\n' - ' * **PEP 634** – Structural Pattern Matching: Specification\n' - '\n' - ' * **PEP 636** – Structural Pattern Matching: Tutorial\n' - '\n' - '\n' - 'Function definitions\n' - '====================\n' - '\n' - 'A function definition defines a user-defined function object ' - '(see\n' - 'section The standard type hierarchy):\n' - '\n' - ' funcdef ::= [decorators] "def" funcname ' - '[type_params] "(" [parameter_list] ")"\n' - ' ["->" expression] ":" suite\n' - ' decorators ::= decorator+\n' - ' decorator ::= "@" assignment_expression ' - 'NEWLINE\n' - ' parameter_list ::= defparameter ("," ' - 'defparameter)* "," "/" ["," [parameter_list_no_posonly]]\n' - ' | parameter_list_no_posonly\n' - ' parameter_list_no_posonly ::= defparameter ("," ' - 'defparameter)* ["," [parameter_list_starargs]]\n' - ' | parameter_list_starargs\n' - ' parameter_list_starargs ::= "*" [parameter] ("," ' - 'defparameter)* ["," ["**" parameter [","]]]\n' - ' | "**" parameter [","]\n' - ' parameter ::= identifier [":" expression]\n' - ' defparameter ::= parameter ["=" expression]\n' - ' funcname ::= identifier\n' - '\n' - 'A function definition is an executable statement. Its execution ' - 'binds\n' - 'the function name in the current local namespace to a function ' - 'object\n' - '(a wrapper around the executable code for the function). This\n' - 'function object contains a reference to the current global ' - 'namespace\n' - 'as the global namespace to be used when the function is called.\n' - '\n' - 'The function definition does not execute the function body; this ' - 'gets\n' - 'executed only when the function is called. [4]\n' - '\n' - 'A function definition may be wrapped by one or more *decorator*\n' - 'expressions. Decorator expressions are evaluated when the ' - 'function is\n' - 'defined, in the scope that contains the function definition. ' - 'The\n' - 'result must be a callable, which is invoked with the function ' - 'object\n' - 'as the only argument. The returned value is bound to the ' - 'function name\n' - 'instead of the function object. Multiple decorators are applied ' - 'in\n' - 'nested fashion. For example, the following code\n' - '\n' - ' @f1(arg)\n' - ' @f2\n' - ' def func(): pass\n' - '\n' - 'is roughly equivalent to\n' - '\n' - ' def func(): pass\n' - ' func = f1(arg)(f2(func))\n' - '\n' - 'except that the original function is not temporarily bound to ' - 'the name\n' - '"func".\n' - '\n' - 'Changed in version 3.9: Functions may be decorated with any ' - 'valid\n' - '"assignment_expression". Previously, the grammar was much more\n' - 'restrictive; see **PEP 614** for details.\n' - '\n' - 'A list of type parameters may be given in square brackets ' - 'between the\n' - 'function’s name and the opening parenthesis for its parameter ' - 'list.\n' - 'This indicates to static type checkers that the function is ' - 'generic.\n' - 'At runtime, the type parameters can be retrieved from the ' - 'function’s\n' - '"__type_params__" attribute. See Generic functions for more.\n' - '\n' - 'Changed in version 3.12: Type parameter lists are new in Python ' - '3.12.\n' - '\n' - 'When one or more *parameters* have the form *parameter* "="\n' - '*expression*, the function is said to have “default parameter ' - 'values.”\n' - 'For a parameter with a default value, the corresponding ' - '*argument* may\n' - 'be omitted from a call, in which case the parameter’s default ' - 'value is\n' - 'substituted. If a parameter has a default value, all following\n' - 'parameters up until the “"*"” must also have a default value — ' - 'this is\n' - 'a syntactic restriction that is not expressed by the grammar.\n' - '\n' - '**Default parameter values are evaluated from left to right when ' - 'the\n' - 'function definition is executed.** This means that the ' - 'expression is\n' - 'evaluated once, when the function is defined, and that the same ' - '“pre-\n' - 'computed” value is used for each call. This is especially ' - 'important\n' - 'to understand when a default parameter value is a mutable ' - 'object, such\n' - 'as a list or a dictionary: if the function modifies the object ' - '(e.g.\n' - 'by appending an item to a list), the default parameter value is ' - 'in\n' - 'effect modified. This is generally not what was intended. A ' - 'way\n' - 'around this is to use "None" as the default, and explicitly test ' - 'for\n' - 'it in the body of the function, e.g.:\n' - '\n' - ' def whats_on_the_telly(penguin=None):\n' - ' if penguin is None:\n' - ' penguin = []\n' - ' penguin.append("property of the zoo")\n' - ' return penguin\n' - '\n' - 'Function call semantics are described in more detail in section ' - 'Calls.\n' - 'A function call always assigns values to all parameters ' - 'mentioned in\n' - 'the parameter list, either from positional arguments, from ' - 'keyword\n' - 'arguments, or from default values. If the form “"*identifier"” ' - 'is\n' - 'present, it is initialized to a tuple receiving any excess ' - 'positional\n' - 'parameters, defaulting to the empty tuple. If the form\n' - '“"**identifier"” is present, it is initialized to a new ordered\n' - 'mapping receiving any excess keyword arguments, defaulting to a ' - 'new\n' - 'empty mapping of the same type. Parameters after “"*"” or\n' - '“"*identifier"” are keyword-only parameters and may only be ' - 'passed by\n' - 'keyword arguments. Parameters before “"/"” are positional-only\n' - 'parameters and may only be passed by positional arguments.\n' - '\n' - 'Changed in version 3.8: The "/" function parameter syntax may be ' - 'used\n' - 'to indicate positional-only parameters. See **PEP 570** for ' - 'details.\n' - '\n' - 'Parameters may have an *annotation* of the form “": ' - 'expression"”\n' - 'following the parameter name. Any parameter may have an ' - 'annotation,\n' - 'even those of the form "*identifier" or "**identifier". ' - 'Functions may\n' - 'have “return” annotation of the form “"-> expression"” after ' - 'the\n' - 'parameter list. These annotations can be any valid Python ' - 'expression.\n' - 'The presence of annotations does not change the semantics of a\n' - 'function. The annotation values are available as values of a\n' - 'dictionary keyed by the parameters’ names in the ' - '"__annotations__"\n' - 'attribute of the function object. If the "annotations" import ' - 'from\n' - '"__future__" is used, annotations are preserved as strings at ' - 'runtime\n' - 'which enables postponed evaluation. Otherwise, they are ' - 'evaluated\n' - 'when the function definition is executed. In this case ' - 'annotations\n' - 'may be evaluated in a different order than they appear in the ' - 'source\n' - 'code.\n' - '\n' - 'It is also possible to create anonymous functions (functions not ' - 'bound\n' - 'to a name), for immediate use in expressions. This uses lambda\n' - 'expressions, described in section Lambdas. Note that the ' - 'lambda\n' - 'expression is merely a shorthand for a simplified function ' - 'definition;\n' - 'a function defined in a “"def"” statement can be passed around ' - 'or\n' - 'assigned to another name just like a function defined by a ' - 'lambda\n' - 'expression. The “"def"” form is actually more powerful since ' - 'it\n' - 'allows the execution of multiple statements and annotations.\n' - '\n' - '**Programmer’s note:** Functions are first-class objects. A ' - '“"def"”\n' - 'statement executed inside a function definition defines a local\n' - 'function that can be returned or passed around. Free variables ' - 'used\n' - 'in the nested function can access the local variables of the ' - 'function\n' - 'containing the def. See section Naming and binding for ' - 'details.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3107** - Function Annotations\n' - ' The original specification for function annotations.\n' - '\n' - ' **PEP 484** - Type Hints\n' - ' Definition of a standard meaning for annotations: type ' - 'hints.\n' - '\n' - ' **PEP 526** - Syntax for Variable Annotations\n' - ' Ability to type hint variable declarations, including ' - 'class\n' - ' variables and instance variables.\n' - '\n' - ' **PEP 563** - Postponed Evaluation of Annotations\n' - ' Support for forward references within annotations by ' - 'preserving\n' - ' annotations in a string form at runtime instead of eager\n' - ' evaluation.\n' - '\n' - ' **PEP 318** - Decorators for Functions and Methods\n' - ' Function and method decorators were introduced. Class ' - 'decorators\n' - ' were introduced in **PEP 3129**.\n' - '\n' - '\n' - 'Class definitions\n' - '=================\n' - '\n' - 'A class definition defines a class object (see section The ' - 'standard\n' - 'type hierarchy):\n' - '\n' - ' classdef ::= [decorators] "class" classname [type_params] ' - '[inheritance] ":" suite\n' - ' inheritance ::= "(" [argument_list] ")"\n' - ' classname ::= identifier\n' - '\n' - 'A class definition is an executable statement. The inheritance ' - 'list\n' - 'usually gives a list of base classes (see Metaclasses for more\n' - 'advanced uses), so each item in the list should evaluate to a ' - 'class\n' - 'object which allows subclassing. Classes without an inheritance ' - 'list\n' - 'inherit, by default, from the base class "object"; hence,\n' - '\n' - ' class Foo:\n' - ' pass\n' - '\n' - 'is equivalent to\n' - '\n' - ' class Foo(object):\n' - ' pass\n' - '\n' - 'The class’s suite is then executed in a new execution frame ' - '(see\n' - 'Naming and binding), using a newly created local namespace and ' - 'the\n' - 'original global namespace. (Usually, the suite contains mostly\n' - 'function definitions.) When the class’s suite finishes ' - 'execution, its\n' - 'execution frame is discarded but its local namespace is saved. ' - '[5] A\n' - 'class object is then created using the inheritance list for the ' - 'base\n' - 'classes and the saved local namespace for the attribute ' - 'dictionary.\n' - 'The class name is bound to this class object in the original ' - 'local\n' - 'namespace.\n' - '\n' - 'The order in which attributes are defined in the class body is\n' - 'preserved in the new class’s "__dict__". Note that this is ' - 'reliable\n' - 'only right after the class is created and only for classes that ' - 'were\n' - 'defined using the definition syntax.\n' - '\n' - 'Class creation can be customized heavily using metaclasses.\n' - '\n' - 'Classes can also be decorated: just like when decorating ' - 'functions,\n' - '\n' - ' @f1(arg)\n' - ' @f2\n' - ' class Foo: pass\n' - '\n' - 'is roughly equivalent to\n' - '\n' - ' class Foo: pass\n' - ' Foo = f1(arg)(f2(Foo))\n' - '\n' - 'The evaluation rules for the decorator expressions are the same ' - 'as for\n' - 'function decorators. The result is then bound to the class ' - 'name.\n' - '\n' - 'Changed in version 3.9: Classes may be decorated with any valid\n' - '"assignment_expression". Previously, the grammar was much more\n' - 'restrictive; see **PEP 614** for details.\n' - '\n' - 'A list of type parameters may be given in square brackets ' - 'immediately\n' - 'after the class’s name. This indicates to static type checkers ' - 'that\n' - 'the class is generic. At runtime, the type parameters can be ' - 'retrieved\n' - 'from the class’s "__type_params__" attribute. See Generic ' - 'classes for\n' - 'more.\n' - '\n' - 'Changed in version 3.12: Type parameter lists are new in Python ' - '3.12.\n' - '\n' - '**Programmer’s note:** Variables defined in the class definition ' - 'are\n' - 'class attributes; they are shared by instances. Instance ' - 'attributes\n' - 'can be set in a method with "self.name = value". Both class ' - 'and\n' - 'instance attributes are accessible through the notation ' - '“"self.name"”,\n' - 'and an instance attribute hides a class attribute with the same ' - 'name\n' - 'when accessed in this way. Class attributes can be used as ' - 'defaults\n' - 'for instance attributes, but using mutable values there can lead ' - 'to\n' - 'unexpected results. Descriptors can be used to create instance\n' - 'variables with different implementation details.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3115** - Metaclasses in Python 3000\n' - ' The proposal that changed the declaration of metaclasses to ' - 'the\n' - ' current syntax, and the semantics for how classes with\n' - ' metaclasses are constructed.\n' - '\n' - ' **PEP 3129** - Class Decorators\n' - ' The proposal that added class decorators. Function and ' - 'method\n' - ' decorators were introduced in **PEP 318**.\n' - '\n' - '\n' - 'Coroutines\n' - '==========\n' - '\n' - 'New in version 3.5.\n' - '\n' - '\n' - 'Coroutine function definition\n' - '-----------------------------\n' - '\n' - ' async_funcdef ::= [decorators] "async" "def" funcname "(" ' - '[parameter_list] ")"\n' - ' ["->" expression] ":" suite\n' - '\n' - 'Execution of Python coroutines can be suspended and resumed at ' - 'many\n' - 'points (see *coroutine*). "await" expressions, "async for" and ' - '"async\n' - 'with" can only be used in the body of a coroutine function.\n' - '\n' - 'Functions defined with "async def" syntax are always coroutine\n' - 'functions, even if they do not contain "await" or "async" ' - 'keywords.\n' - '\n' - 'It is a "SyntaxError" to use a "yield from" expression inside ' - 'the body\n' - 'of a coroutine function.\n' - '\n' - 'An example of a coroutine function:\n' - '\n' - ' async def func(param1, param2):\n' - ' do_stuff()\n' - ' await some_coroutine()\n' - '\n' - 'Changed in version 3.7: "await" and "async" are now keywords;\n' - 'previously they were only treated as such inside the body of a\n' - 'coroutine function.\n' - '\n' - '\n' - 'The "async for" statement\n' - '-------------------------\n' - '\n' - ' async_for_stmt ::= "async" for_stmt\n' - '\n' - 'An *asynchronous iterable* provides an "__aiter__" method that\n' - 'directly returns an *asynchronous iterator*, which can call\n' - 'asynchronous code in its "__anext__" method.\n' - '\n' - 'The "async for" statement allows convenient iteration over\n' - 'asynchronous iterables.\n' - '\n' - 'The following code:\n' - '\n' - ' async for TARGET in ITER:\n' - ' SUITE\n' - ' else:\n' - ' SUITE2\n' - '\n' - 'Is semantically equivalent to:\n' - '\n' - ' iter = (ITER)\n' - ' iter = type(iter).__aiter__(iter)\n' - ' running = True\n' - '\n' - ' while running:\n' - ' try:\n' - ' TARGET = await type(iter).__anext__(iter)\n' - ' except StopAsyncIteration:\n' - ' running = False\n' - ' else:\n' - ' SUITE\n' - ' else:\n' - ' SUITE2\n' - '\n' - 'See also "__aiter__()" and "__anext__()" for details.\n' - '\n' - 'It is a "SyntaxError" to use an "async for" statement outside ' - 'the body\n' - 'of a coroutine function.\n' - '\n' - '\n' - 'The "async with" statement\n' - '--------------------------\n' - '\n' - ' async_with_stmt ::= "async" with_stmt\n' - '\n' - 'An *asynchronous context manager* is a *context manager* that is ' - 'able\n' - 'to suspend execution in its *enter* and *exit* methods.\n' - '\n' - 'The following code:\n' - '\n' - ' async with EXPRESSION as TARGET:\n' - ' SUITE\n' - '\n' - 'is semantically equivalent to:\n' - '\n' - ' manager = (EXPRESSION)\n' - ' aenter = type(manager).__aenter__\n' - ' aexit = type(manager).__aexit__\n' - ' value = await aenter(manager)\n' - ' hit_except = False\n' - '\n' - ' try:\n' - ' TARGET = value\n' - ' SUITE\n' - ' except:\n' - ' hit_except = True\n' - ' if not await aexit(manager, *sys.exc_info()):\n' - ' raise\n' - ' finally:\n' - ' if not hit_except:\n' - ' await aexit(manager, None, None, None)\n' - '\n' - 'See also "__aenter__()" and "__aexit__()" for details.\n' - '\n' - 'It is a "SyntaxError" to use an "async with" statement outside ' - 'the\n' - 'body of a coroutine function.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 492** - Coroutines with async and await syntax\n' - ' The proposal that made coroutines a proper standalone ' - 'concept in\n' - ' Python, and added supporting syntax.\n' - '\n' - '\n' - 'Type parameter lists\n' - '====================\n' - '\n' - 'New in version 3.12.\n' - '\n' - ' type_params ::= "[" type_param ("," type_param)* "]"\n' - ' type_param ::= typevar | typevartuple | paramspec\n' - ' typevar ::= identifier (":" expression)?\n' - ' typevartuple ::= "*" identifier\n' - ' paramspec ::= "**" identifier\n' - '\n' - 'Functions (including coroutines), classes and type aliases may ' - 'contain\n' - 'a type parameter list:\n' - '\n' - ' def max[T](args: list[T]) -> T:\n' - ' ...\n' - '\n' - ' async def amax[T](args: list[T]) -> T:\n' - ' ...\n' - '\n' - ' class Bag[T]:\n' - ' def __iter__(self) -> Iterator[T]:\n' - ' ...\n' - '\n' - ' def add(self, arg: T) -> None:\n' - ' ...\n' - '\n' - ' type ListOrSet[T] = list[T] | set[T]\n' - '\n' - 'Semantically, this indicates that the function, class, or type ' - 'alias\n' - 'is generic over a type variable. This information is primarily ' - 'used by\n' - 'static type checkers, and at runtime, generic objects behave ' - 'much like\n' - 'their non-generic counterparts.\n' - '\n' - 'Type parameters are declared in square brackets ("[]") ' - 'immediately\n' - 'after the name of the function, class, or type alias. The type\n' - 'parameters are accessible within the scope of the generic ' - 'object, but\n' - 'not elsewhere. Thus, after a declaration "def func[T](): pass", ' - 'the\n' - 'name "T" is not available in the module scope. Below, the ' - 'semantics of\n' - 'generic objects are described with more precision. The scope of ' - 'type\n' - 'parameters is modeled with a special function (technically, an\n' - 'annotation scope) that wraps the creation of the generic ' - 'object.\n' - '\n' - 'Generic functions, classes, and type aliases have a ' - '"__type_params__"\n' - 'attribute listing their type parameters.\n' - '\n' - 'Type parameters come in three kinds:\n' - '\n' - '* "typing.TypeVar", introduced by a plain name (e.g., "T").\n' - ' Semantically, this represents a single type to a type ' - 'checker.\n' - '\n' - '* "typing.TypeVarTuple", introduced by a name prefixed with a ' - 'single\n' - ' asterisk (e.g., "*Ts"). Semantically, this stands for a tuple ' - 'of any\n' - ' number of types.\n' - '\n' - '* "typing.ParamSpec", introduced by a name prefixed with two ' - 'asterisks\n' - ' (e.g., "**P"). Semantically, this stands for the parameters of ' - 'a\n' - ' callable.\n' - '\n' - '"typing.TypeVar" declarations can define *bounds* and ' - '*constraints*\n' - 'with a colon (":") followed by an expression. A single ' - 'expression\n' - 'after the colon indicates a bound (e.g. "T: int"). Semantically, ' - 'this\n' - 'means that the "typing.TypeVar" can only represent types that ' - 'are a\n' - 'subtype of this bound. A parenthesized tuple of expressions ' - 'after the\n' - 'colon indicates a set of constraints (e.g. "T: (str, bytes)"). ' - 'Each\n' - 'member of the tuple should be a type (again, this is not ' - 'enforced at\n' - 'runtime). Constrained type variables can only take on one of the ' - 'types\n' - 'in the list of constraints.\n' - '\n' - 'For "typing.TypeVar"s declared using the type parameter list ' - 'syntax,\n' - 'the bound and constraints are not evaluated when the generic ' - 'object is\n' - 'created, but only when the value is explicitly accessed through ' - 'the\n' - 'attributes "__bound__" and "__constraints__". To accomplish ' - 'this, the\n' - 'bounds or constraints are evaluated in a separate annotation ' - 'scope.\n' - '\n' - '"typing.TypeVarTuple"s and "typing.ParamSpec"s cannot have ' - 'bounds or\n' - 'constraints.\n' - '\n' - 'The following example indicates the full set of allowed type ' - 'parameter\n' - 'declarations:\n' - '\n' - ' def overly_generic[\n' - ' SimpleTypeVar,\n' - ' TypeVarWithBound: int,\n' - ' TypeVarWithConstraints: (str, bytes),\n' - ' *SimpleTypeVarTuple,\n' - ' **SimpleParamSpec,\n' - ' ](\n' - ' a: SimpleTypeVar,\n' - ' b: TypeVarWithBound,\n' - ' c: Callable[SimpleParamSpec, TypeVarWithConstraints],\n' - ' *d: SimpleTypeVarTuple,\n' - ' ): ...\n' - '\n' - '\n' - 'Generic functions\n' - '-----------------\n' - '\n' - 'Generic functions are declared as follows:\n' - '\n' - ' def func[T](arg: T): ...\n' - '\n' - 'This syntax is equivalent to:\n' - '\n' - ' annotation-def TYPE_PARAMS_OF_func():\n' - ' T = typing.TypeVar("T")\n' - ' def func(arg: T): ...\n' - ' func.__type_params__ = (T,)\n' - ' return func\n' - ' func = TYPE_PARAMS_OF_func()\n' - '\n' - 'Here "annotation-def" indicates an annotation scope, which is ' - 'not\n' - 'actually bound to any name at runtime. (One other liberty is ' - 'taken in\n' - 'the translation: the syntax does not go through attribute access ' - 'on\n' - 'the "typing" module, but creates an instance of ' - '"typing.TypeVar"\n' - 'directly.)\n' - '\n' - 'The annotations of generic functions are evaluated within the\n' - 'annotation scope used for declaring the type parameters, but ' - 'the\n' - 'function’s defaults and decorators are not.\n' - '\n' - 'The following example illustrates the scoping rules for these ' - 'cases,\n' - 'as well as for additional flavors of type parameters:\n' - '\n' - ' @decorator\n' - ' def func[T: int, *Ts, **P](*args: *Ts, arg: Callable[P, T] = ' - 'some_default):\n' - ' ...\n' - '\n' - 'Except for the lazy evaluation of the "TypeVar" bound, this is\n' - 'equivalent to:\n' - '\n' - ' DEFAULT_OF_arg = some_default\n' - '\n' - ' annotation-def TYPE_PARAMS_OF_func():\n' - '\n' - ' annotation-def BOUND_OF_T():\n' - ' return int\n' - ' # In reality, BOUND_OF_T() is evaluated only on demand.\n' - ' T = typing.TypeVar("T", bound=BOUND_OF_T())\n' - '\n' - ' Ts = typing.TypeVarTuple("Ts")\n' - ' P = typing.ParamSpec("P")\n' - '\n' - ' def func(*args: *Ts, arg: Callable[P, T] = ' - 'DEFAULT_OF_arg):\n' - ' ...\n' - '\n' - ' func.__type_params__ = (T, Ts, P)\n' - ' return func\n' - ' func = decorator(TYPE_PARAMS_OF_func())\n' - '\n' - 'The capitalized names like "DEFAULT_OF_arg" are not actually ' - 'bound at\n' - 'runtime.\n' - '\n' - '\n' - 'Generic classes\n' - '---------------\n' - '\n' - 'Generic classes are declared as follows:\n' - '\n' - ' class Bag[T]: ...\n' - '\n' - 'This syntax is equivalent to:\n' - '\n' - ' annotation-def TYPE_PARAMS_OF_Bag():\n' - ' T = typing.TypeVar("T")\n' - ' class Bag(typing.Generic[T]):\n' - ' __type_params__ = (T,)\n' - ' ...\n' - ' return Bag\n' - ' Bag = TYPE_PARAMS_OF_Bag()\n' - '\n' - 'Here again "annotation-def" (not a real keyword) indicates an\n' - 'annotation scope, and the name "TYPE_PARAMS_OF_Bag" is not ' - 'actually\n' - 'bound at runtime.\n' - '\n' - 'Generic classes implicitly inherit from "typing.Generic". The ' - 'base\n' - 'classes and keyword arguments of generic classes are evaluated ' - 'within\n' - 'the type scope for the type parameters, and decorators are ' - 'evaluated\n' - 'outside that scope. This is illustrated by this example:\n' - '\n' - ' @decorator\n' - ' class Bag(Base[T], arg=T): ...\n' - '\n' - 'This is equivalent to:\n' - '\n' - ' annotation-def TYPE_PARAMS_OF_Bag():\n' - ' T = typing.TypeVar("T")\n' - ' class Bag(Base[T], typing.Generic[T], arg=T):\n' - ' __type_params__ = (T,)\n' - ' ...\n' - ' return Bag\n' - ' Bag = decorator(TYPE_PARAMS_OF_Bag())\n' - '\n' - '\n' - 'Generic type aliases\n' - '--------------------\n' - '\n' - 'The "type" statement can also be used to create a generic type ' - 'alias:\n' - '\n' - ' type ListOrSet[T] = list[T] | set[T]\n' - '\n' - 'Except for the lazy evaluation of the value, this is equivalent ' - 'to:\n' - '\n' - ' annotation-def TYPE_PARAMS_OF_ListOrSet():\n' - ' T = typing.TypeVar("T")\n' - '\n' - ' annotation-def VALUE_OF_ListOrSet():\n' - ' return list[T] | set[T]\n' - ' # In reality, the value is lazily evaluated\n' - ' return typing.TypeAliasType("ListOrSet", ' - 'VALUE_OF_ListOrSet(), type_params=(T,))\n' - ' ListOrSet = TYPE_PARAMS_OF_ListOrSet()\n' - '\n' - 'Here, "annotation-def" (not a real keyword) indicates an ' - 'annotation\n' - 'scope. The capitalized names like "TYPE_PARAMS_OF_ListOrSet" are ' - 'not\n' - 'actually bound at runtime.\n' - '\n' - '-[ Footnotes ]-\n' - '\n' - '[1] The exception is propagated to the invocation stack unless ' - 'there\n' - ' is a "finally" clause which happens to raise another ' - 'exception.\n' - ' That new exception causes the old one to be lost.\n' - '\n' - '[2] In pattern matching, a sequence is defined as one of the\n' - ' following:\n' - '\n' - ' * a class that inherits from "collections.abc.Sequence"\n' - '\n' - ' * a Python class that has been registered as\n' - ' "collections.abc.Sequence"\n' - '\n' - ' * a builtin class that has its (CPython) ' - '"Py_TPFLAGS_SEQUENCE" bit\n' - ' set\n' - '\n' - ' * a class that inherits from any of the above\n' - '\n' - ' The following standard library classes are sequences:\n' - '\n' - ' * "array.array"\n' - '\n' - ' * "collections.deque"\n' - '\n' - ' * "list"\n' - '\n' - ' * "memoryview"\n' - '\n' - ' * "range"\n' - '\n' - ' * "tuple"\n' - '\n' - ' Note:\n' - '\n' - ' Subject values of type "str", "bytes", and "bytearray" do ' - 'not\n' - ' match sequence patterns.\n' - '\n' - '[3] In pattern matching, a mapping is defined as one of the ' - 'following:\n' - '\n' - ' * a class that inherits from "collections.abc.Mapping"\n' - '\n' - ' * a Python class that has been registered as\n' - ' "collections.abc.Mapping"\n' - '\n' - ' * a builtin class that has its (CPython) ' - '"Py_TPFLAGS_MAPPING" bit\n' - ' set\n' - '\n' - ' * a class that inherits from any of the above\n' - '\n' - ' The standard library classes "dict" and ' - '"types.MappingProxyType"\n' - ' are mappings.\n' - '\n' - '[4] A string literal appearing as the first statement in the ' - 'function\n' - ' body is transformed into the function’s "__doc__" attribute ' - 'and\n' - ' therefore the function’s *docstring*.\n' - '\n' - '[5] A string literal appearing as the first statement in the ' - 'class\n' - ' body is transformed into the namespace’s "__doc__" item and\n' - ' therefore the class’s *docstring*.\n', - 'context-managers': 'With Statement Context Managers\n' - '*******************************\n' - '\n' - 'A *context manager* is an object that defines the ' - 'runtime context to\n' - 'be established when executing a "with" statement. The ' - 'context manager\n' - 'handles the entry into, and the exit from, the desired ' - 'runtime context\n' - 'for the execution of the block of code. Context ' - 'managers are normally\n' - 'invoked using the "with" statement (described in section ' - 'The with\n' - 'statement), but can also be used by directly invoking ' - 'their methods.\n' - '\n' - 'Typical uses of context managers include saving and ' - 'restoring various\n' - 'kinds of global state, locking and unlocking resources, ' - 'closing opened\n' - 'files, etc.\n' - '\n' - 'For more information on context managers, see Context ' - 'Manager Types.\n' - '\n' - 'object.__enter__(self)\n' - '\n' - ' Enter the runtime context related to this object. The ' - '"with"\n' - ' statement will bind this method’s return value to the ' - 'target(s)\n' - ' specified in the "as" clause of the statement, if ' - 'any.\n' - '\n' - 'object.__exit__(self, exc_type, exc_value, traceback)\n' - '\n' - ' Exit the runtime context related to this object. The ' - 'parameters\n' - ' describe the exception that caused the context to be ' - 'exited. If the\n' - ' context was exited without an exception, all three ' - 'arguments will\n' - ' be "None".\n' - '\n' - ' If an exception is supplied, and the method wishes to ' - 'suppress the\n' - ' exception (i.e., prevent it from being propagated), ' - 'it should\n' - ' return a true value. Otherwise, the exception will be ' - 'processed\n' - ' normally upon exit from this method.\n' - '\n' - ' Note that "__exit__()" methods should not reraise the ' - 'passed-in\n' - ' exception; this is the caller’s responsibility.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 343** - The “with” statement\n' - ' The specification, background, and examples for the ' - 'Python "with"\n' - ' statement.\n', - 'continue': 'The "continue" statement\n' - '************************\n' - '\n' - ' continue_stmt ::= "continue"\n' - '\n' - '"continue" may only occur syntactically nested in a "for" or ' - '"while"\n' - 'loop, but not nested in a function or class definition within ' - 'that\n' - 'loop. It continues with the next cycle of the nearest enclosing ' - 'loop.\n' - '\n' - 'When "continue" passes control out of a "try" statement with a\n' - '"finally" clause, that "finally" clause is executed before ' - 'really\n' - 'starting the next loop cycle.\n', - 'conversions': 'Arithmetic conversions\n' - '**********************\n' - '\n' - 'When a description of an arithmetic operator below uses the ' - 'phrase\n' - '“the numeric arguments are converted to a common type”, this ' - 'means\n' - 'that the operator implementation for built-in types works as ' - 'follows:\n' - '\n' - '* If either argument is a complex number, the other is ' - 'converted to\n' - ' complex;\n' - '\n' - '* otherwise, if either argument is a floating point number, ' - 'the other\n' - ' is converted to floating point;\n' - '\n' - '* otherwise, both must be integers and no conversion is ' - 'necessary.\n' - '\n' - 'Some additional rules apply for certain operators (e.g., a ' - 'string as a\n' - 'left argument to the ‘%’ operator). Extensions must define ' - 'their own\n' - 'conversion behavior.\n', - 'customization': 'Basic customization\n' - '*******************\n' - '\n' - 'object.__new__(cls[, ...])\n' - '\n' - ' Called to create a new instance of class *cls*. ' - '"__new__()" is a\n' - ' static method (special-cased so you need not declare it ' - 'as such)\n' - ' that takes the class of which an instance was requested ' - 'as its\n' - ' first argument. The remaining arguments are those ' - 'passed to the\n' - ' object constructor expression (the call to the class). ' - 'The return\n' - ' value of "__new__()" should be the new object instance ' - '(usually an\n' - ' instance of *cls*).\n' - '\n' - ' Typical implementations create a new instance of the ' - 'class by\n' - ' invoking the superclass’s "__new__()" method using\n' - ' "super().__new__(cls[, ...])" with appropriate arguments ' - 'and then\n' - ' modifying the newly created instance as necessary before ' - 'returning\n' - ' it.\n' - '\n' - ' If "__new__()" is invoked during object construction and ' - 'it returns\n' - ' an instance of *cls*, then the new instance’s ' - '"__init__()" method\n' - ' will be invoked like "__init__(self[, ...])", where ' - '*self* is the\n' - ' new instance and the remaining arguments are the same as ' - 'were\n' - ' passed to the object constructor.\n' - '\n' - ' If "__new__()" does not return an instance of *cls*, ' - 'then the new\n' - ' instance’s "__init__()" method will not be invoked.\n' - '\n' - ' "__new__()" is intended mainly to allow subclasses of ' - 'immutable\n' - ' types (like int, str, or tuple) to customize instance ' - 'creation. It\n' - ' is also commonly overridden in custom metaclasses in ' - 'order to\n' - ' customize class creation.\n' - '\n' - 'object.__init__(self[, ...])\n' - '\n' - ' Called after the instance has been created (by ' - '"__new__()"), but\n' - ' before it is returned to the caller. The arguments are ' - 'those\n' - ' passed to the class constructor expression. If a base ' - 'class has an\n' - ' "__init__()" method, the derived class’s "__init__()" ' - 'method, if\n' - ' any, must explicitly call it to ensure proper ' - 'initialization of the\n' - ' base class part of the instance; for example:\n' - ' "super().__init__([args...])".\n' - '\n' - ' Because "__new__()" and "__init__()" work together in ' - 'constructing\n' - ' objects ("__new__()" to create it, and "__init__()" to ' - 'customize\n' - ' it), no non-"None" value may be returned by ' - '"__init__()"; doing so\n' - ' will cause a "TypeError" to be raised at runtime.\n' - '\n' - 'object.__del__(self)\n' - '\n' - ' Called when the instance is about to be destroyed. This ' - 'is also\n' - ' called a finalizer or (improperly) a destructor. If a ' - 'base class\n' - ' has a "__del__()" method, the derived class’s ' - '"__del__()" method,\n' - ' if any, must explicitly call it to ensure proper ' - 'deletion of the\n' - ' base class part of the instance.\n' - '\n' - ' It is possible (though not recommended!) for the ' - '"__del__()" method\n' - ' to postpone destruction of the instance by creating a ' - 'new reference\n' - ' to it. This is called object *resurrection*. It is\n' - ' implementation-dependent whether "__del__()" is called a ' - 'second\n' - ' time when a resurrected object is about to be destroyed; ' - 'the\n' - ' current *CPython* implementation only calls it once.\n' - '\n' - ' It is not guaranteed that "__del__()" methods are called ' - 'for\n' - ' objects that still exist when the interpreter exits.\n' - '\n' - ' Note:\n' - '\n' - ' "del x" doesn’t directly call "x.__del__()" — the ' - 'former\n' - ' decrements the reference count for "x" by one, and the ' - 'latter is\n' - ' only called when "x"’s reference count reaches zero.\n' - '\n' - ' **CPython implementation detail:** It is possible for a ' - 'reference\n' - ' cycle to prevent the reference count of an object from ' - 'going to\n' - ' zero. In this case, the cycle will be later detected ' - 'and deleted\n' - ' by the *cyclic garbage collector*. A common cause of ' - 'reference\n' - ' cycles is when an exception has been caught in a local ' - 'variable.\n' - ' The frame’s locals then reference the exception, which ' - 'references\n' - ' its own traceback, which references the locals of all ' - 'frames caught\n' - ' in the traceback.\n' - '\n' - ' See also: Documentation for the "gc" module.\n' - '\n' - ' Warning:\n' - '\n' - ' Due to the precarious circumstances under which ' - '"__del__()"\n' - ' methods are invoked, exceptions that occur during ' - 'their execution\n' - ' are ignored, and a warning is printed to "sys.stderr" ' - 'instead.\n' - ' In particular:\n' - '\n' - ' * "__del__()" can be invoked when arbitrary code is ' - 'being\n' - ' executed, including from any arbitrary thread. If ' - '"__del__()"\n' - ' needs to take a lock or invoke any other blocking ' - 'resource, it\n' - ' may deadlock as the resource may already be taken by ' - 'the code\n' - ' that gets interrupted to execute "__del__()".\n' - '\n' - ' * "__del__()" can be executed during interpreter ' - 'shutdown. As a\n' - ' consequence, the global variables it needs to access ' - '(including\n' - ' other modules) may already have been deleted or set ' - 'to "None".\n' - ' Python guarantees that globals whose name begins ' - 'with a single\n' - ' underscore are deleted from their module before ' - 'other globals\n' - ' are deleted; if no other references to such globals ' - 'exist, this\n' - ' may help in assuring that imported modules are still ' - 'available\n' - ' at the time when the "__del__()" method is called.\n' - '\n' - 'object.__repr__(self)\n' - '\n' - ' Called by the "repr()" built-in function to compute the ' - '“official”\n' - ' string representation of an object. If at all possible, ' - 'this\n' - ' should look like a valid Python expression that could be ' - 'used to\n' - ' recreate an object with the same value (given an ' - 'appropriate\n' - ' environment). If this is not possible, a string of the ' - 'form\n' - ' "<...some useful description...>" should be returned. ' - 'The return\n' - ' value must be a string object. If a class defines ' - '"__repr__()" but\n' - ' not "__str__()", then "__repr__()" is also used when an ' - '“informal”\n' - ' string representation of instances of that class is ' - 'required.\n' - '\n' - ' This is typically used for debugging, so it is important ' - 'that the\n' - ' representation is information-rich and unambiguous.\n' - '\n' - 'object.__str__(self)\n' - '\n' - ' Called by "str(object)" and the built-in functions ' - '"format()" and\n' - ' "print()" to compute the “informal” or nicely printable ' - 'string\n' - ' representation of an object. The return value must be a ' - 'string\n' - ' object.\n' - '\n' - ' This method differs from "object.__repr__()" in that ' - 'there is no\n' - ' expectation that "__str__()" return a valid Python ' - 'expression: a\n' - ' more convenient or concise representation can be used.\n' - '\n' - ' The default implementation defined by the built-in type ' - '"object"\n' - ' calls "object.__repr__()".\n' - '\n' - 'object.__bytes__(self)\n' - '\n' - ' Called by bytes to compute a byte-string representation ' - 'of an\n' - ' object. This should return a "bytes" object.\n' - '\n' - 'object.__format__(self, format_spec)\n' - '\n' - ' Called by the "format()" built-in function, and by ' - 'extension,\n' - ' evaluation of formatted string literals and the ' - '"str.format()"\n' - ' method, to produce a “formatted” string representation ' - 'of an\n' - ' object. The *format_spec* argument is a string that ' - 'contains a\n' - ' description of the formatting options desired. The ' - 'interpretation\n' - ' of the *format_spec* argument is up to the type ' - 'implementing\n' - ' "__format__()", however most classes will either ' - 'delegate\n' - ' formatting to one of the built-in types, or use a ' - 'similar\n' - ' formatting option syntax.\n' - '\n' - ' See Format Specification Mini-Language for a description ' - 'of the\n' - ' standard formatting syntax.\n' - '\n' - ' The return value must be a string object.\n' - '\n' - ' Changed in version 3.4: The __format__ method of ' - '"object" itself\n' - ' raises a "TypeError" if passed any non-empty string.\n' - '\n' - ' Changed in version 3.7: "object.__format__(x, \'\')" is ' - 'now\n' - ' equivalent to "str(x)" rather than "format(str(x), ' - '\'\')".\n' - '\n' - 'object.__lt__(self, other)\n' - 'object.__le__(self, other)\n' - 'object.__eq__(self, other)\n' - 'object.__ne__(self, other)\n' - 'object.__gt__(self, other)\n' - 'object.__ge__(self, other)\n' - '\n' - ' These are the so-called “rich comparison” methods. The\n' - ' correspondence between operator symbols and method names ' - 'is as\n' - ' follows: "xy" calls\n' - ' "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)".\n' - '\n' - ' A rich comparison method may return the singleton ' - '"NotImplemented"\n' - ' if it does not implement the operation for a given pair ' - 'of\n' - ' arguments. By convention, "False" and "True" are ' - 'returned for a\n' - ' successful comparison. However, these methods can return ' - 'any value,\n' - ' so if the comparison operator is used in a Boolean ' - 'context (e.g.,\n' - ' in the condition of an "if" statement), Python will call ' - '"bool()"\n' - ' on the value to determine if the result is true or ' - 'false.\n' - '\n' - ' By default, "object" implements "__eq__()" by using ' - '"is", returning\n' - ' "NotImplemented" in the case of a false comparison: ' - '"True if x is y\n' - ' else NotImplemented". For "__ne__()", by default it ' - 'delegates to\n' - ' "__eq__()" and inverts the result unless it is ' - '"NotImplemented".\n' - ' There are no other implied relationships among the ' - 'comparison\n' - ' operators or default implementations; for example, the ' - 'truth of\n' - ' "(x.__hash__".\n' - '\n' - ' If a class that does not override "__eq__()" wishes to ' - 'suppress\n' - ' hash support, it should include "__hash__ = None" in the ' - 'class\n' - ' definition. A class which defines its own "__hash__()" ' - 'that\n' - ' explicitly raises a "TypeError" would be incorrectly ' - 'identified as\n' - ' hashable by an "isinstance(obj, ' - 'collections.abc.Hashable)" call.\n' - '\n' - ' Note:\n' - '\n' - ' By default, the "__hash__()" values of str and bytes ' - 'objects are\n' - ' “salted” with an unpredictable random value. Although ' - 'they\n' - ' remain constant within an individual Python process, ' - 'they are not\n' - ' predictable between repeated invocations of ' - 'Python.This is\n' - ' intended to provide protection against a ' - 'denial-of-service caused\n' - ' by carefully chosen inputs that exploit the worst ' - 'case\n' - ' performance of a dict insertion, *O*(*n*^2) ' - 'complexity. See\n' - ' http://ocert.org/advisories/ocert-2011-003.html for\n' - ' details.Changing hash values affects the iteration ' - 'order of sets.\n' - ' Python has never made guarantees about this ordering ' - '(and it\n' - ' typically varies between 32-bit and 64-bit builds).See ' - 'also\n' - ' "PYTHONHASHSEED".\n' - '\n' - ' Changed in version 3.3: Hash randomization is enabled by ' - 'default.\n' - '\n' - 'object.__bool__(self)\n' - '\n' - ' Called to implement truth value testing and the built-in ' - 'operation\n' - ' "bool()"; should return "False" or "True". When this ' - 'method is not\n' - ' defined, "__len__()" is called, if it is defined, and ' - 'the object is\n' - ' considered true if its result is nonzero. If a class ' - 'defines\n' - ' neither "__len__()" nor "__bool__()", all its instances ' - 'are\n' - ' considered true.\n', - 'debugger': '"pdb" — The Python Debugger\n' - '***************************\n' - '\n' - '**Source code:** Lib/pdb.py\n' - '\n' - '======================================================================\n' - '\n' - 'The module "pdb" defines an interactive source code debugger ' - 'for\n' - 'Python programs. It supports setting (conditional) breakpoints ' - 'and\n' - 'single stepping at the source line level, inspection of stack ' - 'frames,\n' - 'source code listing, and evaluation of arbitrary Python code in ' - 'the\n' - 'context of any stack frame. It also supports post-mortem ' - 'debugging\n' - 'and can be called under program control.\n' - '\n' - 'The debugger is extensible – it is actually defined as the ' - 'class\n' - '"Pdb". This is currently undocumented but easily understood by ' - 'reading\n' - 'the source. The extension interface uses the modules "bdb" and ' - '"cmd".\n' - '\n' - 'See also:\n' - '\n' - ' Module "faulthandler"\n' - ' Used to dump Python tracebacks explicitly, on a fault, ' - 'after a\n' - ' timeout, or on a user signal.\n' - '\n' - ' Module "traceback"\n' - ' Standard interface to extract, format and print stack ' - 'traces of\n' - ' Python programs.\n' - '\n' - 'The typical usage to break into the debugger is to insert:\n' - '\n' - ' import pdb; pdb.set_trace()\n' - '\n' - 'Or:\n' - '\n' - ' breakpoint()\n' - '\n' - 'at the location you want to break into the debugger, and then ' - 'run the\n' - 'program. You can then step through the code following this ' - 'statement,\n' - 'and continue running without the debugger using the "continue"\n' - 'command.\n' - '\n' - 'Changed in version 3.7: The built-in "breakpoint()", when called ' - 'with\n' - 'defaults, can be used instead of "import pdb; pdb.set_trace()".\n' - '\n' - ' def double(x):\n' - ' breakpoint()\n' - ' return x * 2\n' - ' val = 3\n' - ' print(f"{val} * 2 is {double(val)}")\n' - '\n' - 'The debugger’s prompt is "(Pdb)", which is the indicator that ' - 'you are\n' - 'in debug mode:\n' - '\n' - ' > ...(3)double()\n' - ' -> return x * 2\n' - ' (Pdb) p x\n' - ' 3\n' - ' (Pdb) continue\n' - ' 3 * 2 is 6\n' - '\n' - 'Changed in version 3.3: Tab-completion via the "readline" module ' - 'is\n' - 'available for commands and command arguments, e.g. the current ' - 'global\n' - 'and local names are offered as arguments of the "p" command.\n' - '\n' - 'You can also invoke "pdb" from the command line to debug other\n' - 'scripts. For example:\n' - '\n' - ' python -m pdb myscript.py\n' - '\n' - 'When invoked as a module, pdb will automatically enter ' - 'post-mortem\n' - 'debugging if the program being debugged exits abnormally. After ' - 'post-\n' - 'mortem debugging (or after normal exit of the program), pdb ' - 'will\n' - 'restart the program. Automatic restarting preserves pdb’s state ' - '(such\n' - 'as breakpoints) and in most cases is more useful than quitting ' - 'the\n' - 'debugger upon program’s exit.\n' - '\n' - 'Changed in version 3.2: Added the "-c" option to execute ' - 'commands as\n' - 'if given in a ".pdbrc" file; see Debugger Commands.\n' - '\n' - 'Changed in version 3.7: Added the "-m" option to execute ' - 'modules\n' - 'similar to the way "python -m" does. As with a script, the ' - 'debugger\n' - 'will pause execution just before the first line of the module.\n' - '\n' - 'Typical usage to execute a statement under control of the ' - 'debugger is:\n' - '\n' - ' >>> import pdb\n' - ' >>> def f(x):\n' - ' ... print(1 / x)\n' - ' >>> pdb.run("f(2)")\n' - ' > (1)()\n' - ' (Pdb) continue\n' - ' 0.5\n' - ' >>>\n' - '\n' - 'The typical usage to inspect a crashed program is:\n' - '\n' - ' >>> import pdb\n' - ' >>> def f(x):\n' - ' ... print(1 / x)\n' - ' ...\n' - ' >>> f(0)\n' - ' Traceback (most recent call last):\n' - ' File "", line 1, in \n' - ' File "", line 2, in f\n' - ' ZeroDivisionError: division by zero\n' - ' >>> pdb.pm()\n' - ' > (2)f()\n' - ' (Pdb) p x\n' - ' 0\n' - ' (Pdb)\n' - '\n' - 'The module defines the following functions; each enters the ' - 'debugger\n' - 'in a slightly different way:\n' - '\n' - 'pdb.run(statement, globals=None, locals=None)\n' - '\n' - ' Execute the *statement* (given as a string or a code object) ' - 'under\n' - ' debugger control. The debugger prompt appears before any ' - 'code is\n' - ' executed; you can set breakpoints and type "continue", or you ' - 'can\n' - ' step through the statement using "step" or "next" (all these\n' - ' commands are explained below). The optional *globals* and ' - '*locals*\n' - ' arguments specify the environment in which the code is ' - 'executed; by\n' - ' default the dictionary of the module "__main__" is used. ' - '(See the\n' - ' explanation of the built-in "exec()" or "eval()" functions.)\n' - '\n' - 'pdb.runeval(expression, globals=None, locals=None)\n' - '\n' - ' Evaluate the *expression* (given as a string or a code ' - 'object)\n' - ' under debugger control. When "runeval()" returns, it returns ' - 'the\n' - ' value of the *expression*. Otherwise this function is ' - 'similar to\n' - ' "run()".\n' - '\n' - 'pdb.runcall(function, *args, **kwds)\n' - '\n' - ' Call the *function* (a function or method object, not a ' - 'string)\n' - ' with the given arguments. When "runcall()" returns, it ' - 'returns\n' - ' whatever the function call returned. The debugger prompt ' - 'appears\n' - ' as soon as the function is entered.\n' - '\n' - 'pdb.set_trace(*, header=None)\n' - '\n' - ' Enter the debugger at the calling stack frame. This is ' - 'useful to\n' - ' hard-code a breakpoint at a given point in a program, even if ' - 'the\n' - ' code is not otherwise being debugged (e.g. when an assertion\n' - ' fails). If given, *header* is printed to the console just ' - 'before\n' - ' debugging begins.\n' - '\n' - ' Changed in version 3.7: The keyword-only argument *header*.\n' - '\n' - 'pdb.post_mortem(traceback=None)\n' - '\n' - ' Enter post-mortem debugging of the given *traceback* object. ' - 'If no\n' - ' *traceback* is given, it uses the one of the exception that ' - 'is\n' - ' currently being handled (an exception must be being handled ' - 'if the\n' - ' default is to be used).\n' - '\n' - 'pdb.pm()\n' - '\n' - ' Enter post-mortem debugging of the exception found in\n' - ' "sys.last_exc".\n' - '\n' - 'The "run*" functions and "set_trace()" are aliases for ' - 'instantiating\n' - 'the "Pdb" class and calling the method of the same name. If you ' - 'want\n' - 'to access further features, you have to do this yourself:\n' - '\n' - "class pdb.Pdb(completekey='tab', stdin=None, stdout=None, " - 'skip=None, nosigint=False, readrc=True)\n' - '\n' - ' "Pdb" is the debugger class.\n' - '\n' - ' The *completekey*, *stdin* and *stdout* arguments are passed ' - 'to the\n' - ' underlying "cmd.Cmd" class; see the description there.\n' - '\n' - ' The *skip* argument, if given, must be an iterable of ' - 'glob-style\n' - ' module name patterns. The debugger will not step into frames ' - 'that\n' - ' originate in a module that matches one of these patterns. ' - '[1]\n' - '\n' - ' By default, Pdb sets a handler for the SIGINT signal (which ' - 'is sent\n' - ' when the user presses "Ctrl-C" on the console) when you give ' - 'a\n' - ' "continue" command. This allows you to break into the ' - 'debugger\n' - ' again by pressing "Ctrl-C". If you want Pdb not to touch ' - 'the\n' - ' SIGINT handler, set *nosigint* to true.\n' - '\n' - ' The *readrc* argument defaults to true and controls whether ' - 'Pdb\n' - ' will load .pdbrc files from the filesystem.\n' - '\n' - ' Example call to enable tracing with *skip*:\n' - '\n' - " import pdb; pdb.Pdb(skip=['django.*']).set_trace()\n" - '\n' - ' Raises an auditing event "pdb.Pdb" with no arguments.\n' - '\n' - ' Changed in version 3.1: Added the *skip* parameter.\n' - '\n' - ' Changed in version 3.2: Added the *nosigint* parameter. ' - 'Previously,\n' - ' a SIGINT handler was never set by Pdb.\n' - '\n' - ' Changed in version 3.6: The *readrc* argument.\n' - '\n' - ' run(statement, globals=None, locals=None)\n' - ' runeval(expression, globals=None, locals=None)\n' - ' runcall(function, *args, **kwds)\n' - ' set_trace()\n' - '\n' - ' See the documentation for the functions explained above.\n' - '\n' - '\n' - 'Debugger Commands\n' - '=================\n' - '\n' - 'The commands recognized by the debugger are listed below. Most\n' - 'commands can be abbreviated to one or two letters as indicated; ' - 'e.g.\n' - '"h(elp)" means that either "h" or "help" can be used to enter ' - 'the help\n' - 'command (but not "he" or "hel", nor "H" or "Help" or "HELP").\n' - 'Arguments to commands must be separated by whitespace (spaces ' - 'or\n' - 'tabs). Optional arguments are enclosed in square brackets ' - '("[]") in\n' - 'the command syntax; the square brackets must not be typed.\n' - 'Alternatives in the command syntax are separated by a vertical ' - 'bar\n' - '("|").\n' - '\n' - 'Entering a blank line repeats the last command entered. ' - 'Exception: if\n' - 'the last command was a "list" command, the next 11 lines are ' - 'listed.\n' - '\n' - 'Commands that the debugger doesn’t recognize are assumed to be ' - 'Python\n' - 'statements and are executed in the context of the program being\n' - 'debugged. Python statements can also be prefixed with an ' - 'exclamation\n' - 'point ("!"). This is a powerful way to inspect the program ' - 'being\n' - 'debugged; it is even possible to change a variable or call a ' - 'function.\n' - 'When an exception occurs in such a statement, the exception name ' - 'is\n' - 'printed but the debugger’s state is not changed.\n' - '\n' - 'Changed in version 3.13: Expressions/Statements whose prefix is ' - 'a pdb\n' - 'command are now correctly identified and executed.\n' - '\n' - 'The debugger supports aliases. Aliases can have parameters ' - 'which\n' - 'allows one a certain level of adaptability to the context under\n' - 'examination.\n' - '\n' - 'Multiple commands may be entered on a single line, separated by ' - '";;".\n' - '(A single ";" is not used as it is the separator for multiple ' - 'commands\n' - 'in a line that is passed to the Python parser.) No intelligence ' - 'is\n' - 'applied to separating the commands; the input is split at the ' - 'first\n' - '";;" pair, even if it is in the middle of a quoted string. A\n' - 'workaround for strings with double semicolons is to use ' - 'implicit\n' - 'string concatenation "\';\'\';\'" or "";"";"".\n' - '\n' - 'To set a temporary global variable, use a *convenience ' - 'variable*. A\n' - '*convenience variable* is a variable whose name starts with ' - '"$". For\n' - 'example, "$foo = 1" sets a global variable "$foo" which you can ' - 'use in\n' - 'the debugger session. The *convenience variables* are cleared ' - 'when\n' - 'the program resumes execution so it’s less likely to interfere ' - 'with\n' - 'your program compared to using normal variables like "foo = 1".\n' - '\n' - 'There are three preset *convenience variables*:\n' - '\n' - '* "$_frame": the current frame you are debugging\n' - '\n' - '* "$_retval": the return value if the frame is returning\n' - '\n' - '* "$_exception": the exception if the frame is raising an ' - 'exception\n' - '\n' - 'New in version 3.12.\n' - '\n' - 'If a file ".pdbrc" exists in the user’s home directory or in ' - 'the\n' - 'current directory, it is read with "\'utf-8\'" encoding and ' - 'executed as\n' - 'if it had been typed at the debugger prompt. This is ' - 'particularly\n' - 'useful for aliases. If both files exist, the one in the home\n' - 'directory is read first and aliases defined there can be ' - 'overridden by\n' - 'the local file.\n' - '\n' - 'Changed in version 3.11: ".pdbrc" is now read with "\'utf-8\'" ' - 'encoding.\n' - 'Previously, it was read with the system locale encoding.\n' - '\n' - 'Changed in version 3.2: ".pdbrc" can now contain commands that\n' - 'continue debugging, such as "continue" or "next". Previously, ' - 'these\n' - 'commands had no effect.\n' - '\n' - 'h(elp) [command]\n' - '\n' - ' Without argument, print the list of available commands. With ' - 'a\n' - ' *command* as argument, print help about that command. "help ' - 'pdb"\n' - ' displays the full documentation (the docstring of the "pdb"\n' - ' module). Since the *command* argument must be an identifier, ' - '"help\n' - ' exec" must be entered to get help on the "!" command.\n' - '\n' - 'w(here)\n' - '\n' - ' Print a stack trace, with the most recent frame at the ' - 'bottom. An\n' - ' arrow (">") indicates the current frame, which determines ' - 'the\n' - ' context of most commands.\n' - '\n' - 'd(own) [count]\n' - '\n' - ' Move the current frame *count* (default one) levels down in ' - 'the\n' - ' stack trace (to a newer frame).\n' - '\n' - 'u(p) [count]\n' - '\n' - ' Move the current frame *count* (default one) levels up in the ' - 'stack\n' - ' trace (to an older frame).\n' - '\n' - 'b(reak) [([filename:]lineno | function) [, condition]]\n' - '\n' - ' With a *lineno* argument, set a break there in the current ' - 'file.\n' - ' With a *function* argument, set a break at the first ' - 'executable\n' - ' statement within that function. The line number may be ' - 'prefixed\n' - ' with a filename and a colon, to specify a breakpoint in ' - 'another\n' - ' file (probably one that hasn’t been loaded yet). The file ' - 'is\n' - ' searched on "sys.path". Note that each breakpoint is ' - 'assigned a\n' - ' number to which all the other breakpoint commands refer.\n' - '\n' - ' If a second argument is present, it is an expression which ' - 'must\n' - ' evaluate to true before the breakpoint is honored.\n' - '\n' - ' Without argument, list all breaks, including for each ' - 'breakpoint,\n' - ' the number of times that breakpoint has been hit, the ' - 'current\n' - ' ignore count, and the associated condition if any.\n' - '\n' - 'tbreak [([filename:]lineno | function) [, condition]]\n' - '\n' - ' Temporary breakpoint, which is removed automatically when it ' - 'is\n' - ' first hit. The arguments are the same as for "break".\n' - '\n' - 'cl(ear) [filename:lineno | bpnumber ...]\n' - '\n' - ' With a *filename:lineno* argument, clear all the breakpoints ' - 'at\n' - ' this line. With a space separated list of breakpoint numbers, ' - 'clear\n' - ' those breakpoints. Without argument, clear all breaks (but ' - 'first\n' - ' ask confirmation).\n' - '\n' - 'disable bpnumber [bpnumber ...]\n' - '\n' - ' Disable the breakpoints given as a space separated list of\n' - ' breakpoint numbers. Disabling a breakpoint means it cannot ' - 'cause\n' - ' the program to stop execution, but unlike clearing a ' - 'breakpoint, it\n' - ' remains in the list of breakpoints and can be (re-)enabled.\n' - '\n' - 'enable bpnumber [bpnumber ...]\n' - '\n' - ' Enable the breakpoints specified.\n' - '\n' - 'ignore bpnumber [count]\n' - '\n' - ' Set the ignore count for the given breakpoint number. If ' - '*count*\n' - ' is omitted, the ignore count is set to 0. A breakpoint ' - 'becomes\n' - ' active when the ignore count is zero. When non-zero, the ' - '*count*\n' - ' is decremented each time the breakpoint is reached and the\n' - ' breakpoint is not disabled and any associated condition ' - 'evaluates\n' - ' to true.\n' - '\n' - 'condition bpnumber [condition]\n' - '\n' - ' Set a new *condition* for the breakpoint, an expression which ' - 'must\n' - ' evaluate to true before the breakpoint is honored. If ' - '*condition*\n' - ' is absent, any existing condition is removed; i.e., the ' - 'breakpoint\n' - ' is made unconditional.\n' - '\n' - 'commands [bpnumber]\n' - '\n' - ' Specify a list of commands for breakpoint number *bpnumber*. ' - 'The\n' - ' commands themselves appear on the following lines. Type a ' - 'line\n' - ' containing just "end" to terminate the commands. An example:\n' - '\n' - ' (Pdb) commands 1\n' - ' (com) p some_variable\n' - ' (com) end\n' - ' (Pdb)\n' - '\n' - ' To remove all commands from a breakpoint, type "commands" ' - 'and\n' - ' follow it immediately with "end"; that is, give no commands.\n' - '\n' - ' With no *bpnumber* argument, "commands" refers to the last\n' - ' breakpoint set.\n' - '\n' - ' You can use breakpoint commands to start your program up ' - 'again.\n' - ' Simply use the "continue" command, or "step", or any other ' - 'command\n' - ' that resumes execution.\n' - '\n' - ' Specifying any command resuming execution (currently ' - '"continue",\n' - ' "step", "next", "return", "jump", "quit" and their ' - 'abbreviations)\n' - ' terminates the command list (as if that command was ' - 'immediately\n' - ' followed by end). This is because any time you resume ' - 'execution\n' - ' (even with a simple next or step), you may encounter another\n' - ' breakpoint—which could have its own command list, leading to\n' - ' ambiguities about which list to execute.\n' - '\n' - ' If you use the "silent" command in the command list, the ' - 'usual\n' - ' message about stopping at a breakpoint is not printed. This ' - 'may be\n' - ' desirable for breakpoints that are to print a specific ' - 'message and\n' - ' then continue. If none of the other commands print anything, ' - 'you\n' - ' see no sign that the breakpoint was reached.\n' - '\n' - 's(tep)\n' - '\n' - ' Execute the current line, stop at the first possible ' - 'occasion\n' - ' (either in a function that is called or on the next line in ' - 'the\n' - ' current function).\n' - '\n' - 'n(ext)\n' - '\n' - ' Continue execution until the next line in the current ' - 'function is\n' - ' reached or it returns. (The difference between "next" and ' - '"step"\n' - ' is that "step" stops inside a called function, while "next"\n' - ' executes called functions at (nearly) full speed, only ' - 'stopping at\n' - ' the next line in the current function.)\n' - '\n' - 'unt(il) [lineno]\n' - '\n' - ' Without argument, continue execution until the line with a ' - 'number\n' - ' greater than the current one is reached.\n' - '\n' - ' With *lineno*, continue execution until a line with a number\n' - ' greater or equal to *lineno* is reached. In both cases, also ' - 'stop\n' - ' when the current frame returns.\n' - '\n' - ' Changed in version 3.2: Allow giving an explicit line ' - 'number.\n' - '\n' - 'r(eturn)\n' - '\n' - ' Continue execution until the current function returns.\n' - '\n' - 'c(ont(inue))\n' - '\n' - ' Continue execution, only stop when a breakpoint is ' - 'encountered.\n' - '\n' - 'j(ump) lineno\n' - '\n' - ' Set the next line that will be executed. Only available in ' - 'the\n' - ' bottom-most frame. This lets you jump back and execute code ' - 'again,\n' - ' or jump forward to skip code that you don’t want to run.\n' - '\n' - ' It should be noted that not all jumps are allowed – for ' - 'instance it\n' - ' is not possible to jump into the middle of a "for" loop or ' - 'out of a\n' - ' "finally" clause.\n' - '\n' - 'l(ist) [first[, last]]\n' - '\n' - ' List source code for the current file. Without arguments, ' - 'list 11\n' - ' lines around the current line or continue the previous ' - 'listing.\n' - ' With "." as argument, list 11 lines around the current line. ' - 'With\n' - ' one argument, list 11 lines around at that line. With two\n' - ' arguments, list the given range; if the second argument is ' - 'less\n' - ' than the first, it is interpreted as a count.\n' - '\n' - ' The current line in the current frame is indicated by "->". ' - 'If an\n' - ' exception is being debugged, the line where the exception ' - 'was\n' - ' originally raised or propagated is indicated by ">>", if it ' - 'differs\n' - ' from the current line.\n' - '\n' - ' Changed in version 3.2: Added the ">>" marker.\n' - '\n' - 'll | longlist\n' - '\n' - ' List all source code for the current function or frame.\n' - ' Interesting lines are marked as for "list".\n' - '\n' - ' New in version 3.2.\n' - '\n' - 'a(rgs)\n' - '\n' - ' Print the arguments of the current function and their ' - 'current\n' - ' values.\n' - '\n' - 'p expression\n' - '\n' - ' Evaluate *expression* in the current context and print its ' - 'value.\n' - '\n' - ' Note:\n' - '\n' - ' "print()" can also be used, but is not a debugger command — ' - 'this\n' - ' executes the Python "print()" function.\n' - '\n' - 'pp expression\n' - '\n' - ' Like the "p" command, except the value of *expression* is ' - 'pretty-\n' - ' printed using the "pprint" module.\n' - '\n' - 'whatis expression\n' - '\n' - ' Print the type of *expression*.\n' - '\n' - 'source expression\n' - '\n' - ' Try to get source code of *expression* and display it.\n' - '\n' - ' New in version 3.2.\n' - '\n' - 'display [expression]\n' - '\n' - ' Display the value of *expression* if it changed, each time\n' - ' execution stops in the current frame.\n' - '\n' - ' Without *expression*, list all display expressions for the ' - 'current\n' - ' frame.\n' - '\n' - ' Note:\n' - '\n' - ' Display evaluates *expression* and compares to the result ' - 'of the\n' - ' previous evaluation of *expression*, so when the result is\n' - ' mutable, display may not be able to pick up the changes.\n' - '\n' - ' Example:\n' - '\n' - ' lst = []\n' - ' breakpoint()\n' - ' pass\n' - ' lst.append(1)\n' - ' print(lst)\n' - '\n' - ' Display won’t realize "lst" has been changed because the ' - 'result of\n' - ' evaluation is modified in place by "lst.append(1)" before ' - 'being\n' - ' compared:\n' - '\n' - ' > example.py(3)()\n' - ' -> pass\n' - ' (Pdb) display lst\n' - ' display lst: []\n' - ' (Pdb) n\n' - ' > example.py(4)()\n' - ' -> lst.append(1)\n' - ' (Pdb) n\n' - ' > example.py(5)()\n' - ' -> print(lst)\n' - ' (Pdb)\n' - '\n' - ' You can do some tricks with copy mechanism to make it work:\n' - '\n' - ' > example.py(3)()\n' - ' -> pass\n' - ' (Pdb) display lst[:]\n' - ' display lst[:]: []\n' - ' (Pdb) n\n' - ' > example.py(4)()\n' - ' -> lst.append(1)\n' - ' (Pdb) n\n' - ' > example.py(5)()\n' - ' -> print(lst)\n' - ' display lst[:]: [1] [old: []]\n' - ' (Pdb)\n' - '\n' - ' New in version 3.2.\n' - '\n' - 'undisplay [expression]\n' - '\n' - ' Do not display *expression* anymore in the current frame. ' - 'Without\n' - ' *expression*, clear all display expressions for the current ' - 'frame.\n' - '\n' - ' New in version 3.2.\n' - '\n' - 'interact\n' - '\n' - ' Start an interactive interpreter (using the "code" module) ' - 'whose\n' - ' global namespace contains all the (global and local) names ' - 'found in\n' - ' the current scope. Use "exit()" or "quit()" to exit the ' - 'interpreter\n' - ' and return to the debugger.\n' - '\n' - ' Note:\n' - '\n' - ' Because interact creates a new global namespace with the ' - 'current\n' - ' global and local namespace for execution, assignment to ' - 'variables\n' - ' will not affect the original namespaces. However, ' - 'modification to\n' - ' the mutable objects will be reflected in the original ' - 'namespaces.\n' - '\n' - ' New in version 3.2.\n' - '\n' - ' Changed in version 3.13: "exit()" and "quit()" can be used to ' - 'exit\n' - ' the "interact" command.\n' - '\n' - ' Changed in version 3.13: "interact" directs its output to ' - 'the\n' - ' debugger’s output channel rather than "sys.stderr".\n' - '\n' - 'alias [name [command]]\n' - '\n' - ' Create an alias called *name* that executes *command*. The\n' - ' *command* must *not* be enclosed in quotes. Replaceable ' - 'parameters\n' - ' can be indicated by "%1", "%2", … and "%9", while "%*" is ' - 'replaced\n' - ' by all the parameters. If *command* is omitted, the current ' - 'alias\n' - ' for *name* is shown. If no arguments are given, all aliases ' - 'are\n' - ' listed.\n' - '\n' - ' Aliases may be nested and can contain anything that can be ' - 'legally\n' - ' typed at the pdb prompt. Note that internal pdb commands ' - '*can* be\n' - ' overridden by aliases. Such a command is then hidden until ' - 'the\n' - ' alias is removed. Aliasing is recursively applied to the ' - 'first\n' - ' word of the command line; all other words in the line are ' - 'left\n' - ' alone.\n' - '\n' - ' As an example, here are two useful aliases (especially when ' - 'placed\n' - ' in the ".pdbrc" file):\n' - '\n' - ' # Print instance variables (usage "pi classInst")\n' - ' alias pi for k in %1.__dict__.keys(): print(f"%1.{k} = ' - '{%1.__dict__[k]}")\n' - ' # Print instance variables in self\n' - ' alias ps pi self\n' - '\n' - 'unalias name\n' - '\n' - ' Delete the specified alias *name*.\n' - '\n' - '! statement\n' - '\n' - ' Execute the (one-line) *statement* in the context of the ' - 'current\n' - ' stack frame. The exclamation point can be omitted unless the ' - 'first\n' - ' word of the statement resembles a debugger command, e.g.:\n' - '\n' - ' (Pdb) ! n=42\n' - ' (Pdb)\n' - '\n' - ' To set a global variable, you can prefix the assignment ' - 'command\n' - ' with a "global" statement on the same line, e.g.:\n' - '\n' - " (Pdb) global list_options; list_options = ['-l']\n" - ' (Pdb)\n' - '\n' - 'run [args ...]\n' - 'restart [args ...]\n' - '\n' - ' Restart the debugged Python program. If *args* is supplied, ' - 'it is\n' - ' split with "shlex" and the result is used as the new ' - '"sys.argv".\n' - ' History, breakpoints, actions and debugger options are ' - 'preserved.\n' - ' "restart" is an alias for "run".\n' - '\n' - 'q(uit)\n' - '\n' - ' Quit from the debugger. The program being executed is ' - 'aborted.\n' - '\n' - 'debug code\n' - '\n' - ' Enter a recursive debugger that steps through *code* (which ' - 'is an\n' - ' arbitrary expression or statement to be executed in the ' - 'current\n' - ' environment).\n' - '\n' - 'retval\n' - '\n' - ' Print the return value for the last return of the current ' - 'function.\n' - '\n' - 'exceptions [excnumber]\n' - '\n' - ' List or jump between chained exceptions.\n' - '\n' - ' When using "pdb.pm()" or "Pdb.post_mortem(...)" with a ' - 'chained\n' - ' exception instead of a traceback, it allows the user to move\n' - ' between the chained exceptions using "exceptions" command to ' - 'list\n' - ' exceptions, and "exception " to switch to that ' - 'exception.\n' - '\n' - ' Example:\n' - '\n' - ' def out():\n' - ' try:\n' - ' middle()\n' - ' except Exception as e:\n' - ' raise ValueError("reraise middle() error") from e\n' - '\n' - ' def middle():\n' - ' try:\n' - ' return inner(0)\n' - ' except Exception as e:\n' - ' raise ValueError("Middle fail")\n' - '\n' - ' def inner(x):\n' - ' 1 / x\n' - '\n' - ' out()\n' - '\n' - ' calling "pdb.pm()" will allow to move between exceptions:\n' - '\n' - ' > example.py(5)out()\n' - ' -> raise ValueError("reraise middle() error") from e\n' - '\n' - ' (Pdb) exceptions\n' - " 0 ZeroDivisionError('division by zero')\n" - " 1 ValueError('Middle fail')\n" - " > 2 ValueError('reraise middle() error')\n" - '\n' - ' (Pdb) exceptions 0\n' - ' > example.py(16)inner()\n' - ' -> 1 / x\n' - '\n' - ' (Pdb) up\n' - ' > example.py(10)middle()\n' - ' -> return inner(0)\n' - '\n' - ' New in version 3.13.\n' - '\n' - '-[ Footnotes ]-\n' - '\n' - '[1] Whether a frame is considered to originate in a certain ' - 'module is\n' - ' determined by the "__name__" in the frame globals.\n', - 'del': 'The "del" statement\n' - '*******************\n' - '\n' - ' del_stmt ::= "del" target_list\n' - '\n' - 'Deletion is recursively defined very similar to the way assignment ' - 'is\n' - 'defined. Rather than spelling it out in full details, here are some\n' - 'hints.\n' - '\n' - 'Deletion of a target list recursively deletes each target, from left\n' - 'to right.\n' - '\n' - 'Deletion of a name removes the binding of that name from the local ' - 'or\n' - 'global namespace, depending on whether the name occurs in a "global"\n' - 'statement in the same code block. If the name is unbound, a\n' - '"NameError" exception will be raised.\n' - '\n' - 'Deletion of attribute references, subscriptions and slicings is ' - 'passed\n' - 'to the primary object involved; deletion of a slicing is in general\n' - 'equivalent to assignment of an empty slice of the right type (but ' - 'even\n' - 'this is determined by the sliced object).\n' - '\n' - 'Changed in version 3.2: Previously it was illegal to delete a name\n' - 'from the local namespace if it occurs as a free variable in a nested\n' - 'block.\n', - 'dict': 'Dictionary displays\n' - '*******************\n' - '\n' - 'A dictionary display is a possibly empty series of dict items\n' - '(key/value pairs) enclosed in curly braces:\n' - '\n' - ' dict_display ::= "{" [dict_item_list | dict_comprehension] ' - '"}"\n' - ' dict_item_list ::= dict_item ("," dict_item)* [","]\n' - ' dict_item ::= expression ":" expression | "**" or_expr\n' - ' dict_comprehension ::= expression ":" expression comp_for\n' - '\n' - 'A dictionary display yields a new dictionary object.\n' - '\n' - 'If a comma-separated sequence of dict items is given, they are\n' - 'evaluated from left to right to define the entries of the ' - 'dictionary:\n' - 'each key object is used as a key into the dictionary to store the\n' - 'corresponding value. This means that you can specify the same key\n' - 'multiple times in the dict item list, and the final dictionary’s ' - 'value\n' - 'for that key will be the last one given.\n' - '\n' - 'A double asterisk "**" denotes *dictionary unpacking*. Its operand\n' - 'must be a *mapping*. Each mapping item is added to the new\n' - 'dictionary. Later values replace values already set by earlier ' - 'dict\n' - 'items and earlier dictionary unpackings.\n' - '\n' - 'New in version 3.5: Unpacking into dictionary displays, originally\n' - 'proposed by **PEP 448**.\n' - '\n' - 'A dict comprehension, in contrast to list and set comprehensions,\n' - 'needs two expressions separated with a colon followed by the usual\n' - '“for” and “if” clauses. When the comprehension is run, the ' - 'resulting\n' - 'key and value elements are inserted in the new dictionary in the ' - 'order\n' - 'they are produced.\n' - '\n' - 'Restrictions on the types of the key values are listed earlier in\n' - 'section The standard type hierarchy. (To summarize, the key type\n' - 'should be *hashable*, which excludes all mutable objects.) Clashes\n' - 'between duplicate keys are not detected; the last value (textually\n' - 'rightmost in the display) stored for a given key value prevails.\n' - '\n' - 'Changed in version 3.8: Prior to Python 3.8, in dict ' - 'comprehensions,\n' - 'the evaluation order of key and value was not well-defined. In\n' - 'CPython, the value was evaluated before the key. Starting with ' - '3.8,\n' - 'the key is evaluated before the value, as proposed by **PEP 572**.\n', - 'dynamic-features': 'Interaction with dynamic features\n' - '*********************************\n' - '\n' - 'Name resolution of free variables occurs at runtime, not ' - 'at compile\n' - 'time. This means that the following code will print 42:\n' - '\n' - ' i = 10\n' - ' def f():\n' - ' print(i)\n' - ' i = 42\n' - ' f()\n' - '\n' - 'The "eval()" and "exec()" functions do not have access ' - 'to the full\n' - 'environment for resolving names. Names may be resolved ' - 'in the local\n' - 'and global namespaces of the caller. Free variables are ' - 'not resolved\n' - 'in the nearest enclosing namespace, but in the global ' - 'namespace. [1]\n' - 'The "exec()" and "eval()" functions have optional ' - 'arguments to\n' - 'override the global and local namespace. If only one ' - 'namespace is\n' - 'specified, it is used for both.\n', - 'else': 'The "if" statement\n' - '******************\n' - '\n' - 'The "if" statement is used for conditional execution:\n' - '\n' - ' if_stmt ::= "if" assignment_expression ":" suite\n' - ' ("elif" assignment_expression ":" suite)*\n' - ' ["else" ":" suite]\n' - '\n' - 'It selects exactly one of the suites by evaluating the expressions ' - 'one\n' - 'by one until one is found to be true (see section Boolean ' - 'operations\n' - 'for the definition of true and false); then that suite is executed\n' - '(and no other part of the "if" statement is executed or evaluated).\n' - 'If all expressions are false, the suite of the "else" clause, if\n' - 'present, is executed.\n', - 'exceptions': 'Exceptions\n' - '**********\n' - '\n' - 'Exceptions are a means of breaking out of the normal flow of ' - 'control\n' - 'of a code block in order to handle errors or other ' - 'exceptional\n' - 'conditions. An exception is *raised* at the point where the ' - 'error is\n' - 'detected; it may be *handled* by the surrounding code block or ' - 'by any\n' - 'code block that directly or indirectly invoked the code block ' - 'where\n' - 'the error occurred.\n' - '\n' - 'The Python interpreter raises an exception when it detects a ' - 'run-time\n' - 'error (such as division by zero). A Python program can also\n' - 'explicitly raise an exception with the "raise" statement. ' - 'Exception\n' - 'handlers are specified with the "try" … "except" statement. ' - 'The\n' - '"finally" clause of such a statement can be used to specify ' - 'cleanup\n' - 'code which does not handle the exception, but is executed ' - 'whether an\n' - 'exception occurred or not in the preceding code.\n' - '\n' - 'Python uses the “termination” model of error handling: an ' - 'exception\n' - 'handler can find out what happened and continue execution at ' - 'an outer\n' - 'level, but it cannot repair the cause of the error and retry ' - 'the\n' - 'failing operation (except by re-entering the offending piece ' - 'of code\n' - 'from the top).\n' - '\n' - 'When an exception is not handled at all, the interpreter ' - 'terminates\n' - 'execution of the program, or returns to its interactive main ' - 'loop. In\n' - 'either case, it prints a stack traceback, except when the ' - 'exception is\n' - '"SystemExit".\n' - '\n' - 'Exceptions are identified by class instances. The "except" ' - 'clause is\n' - 'selected depending on the class of the instance: it must ' - 'reference the\n' - 'class of the instance or a *non-virtual base class* thereof. ' - 'The\n' - 'instance can be received by the handler and can carry ' - 'additional\n' - 'information about the exceptional condition.\n' - '\n' - 'Note:\n' - '\n' - ' Exception messages are not part of the Python API. Their ' - 'contents\n' - ' may change from one version of Python to the next without ' - 'warning\n' - ' and should not be relied on by code which will run under ' - 'multiple\n' - ' versions of the interpreter.\n' - '\n' - 'See also the description of the "try" statement in section The ' - 'try\n' - 'statement and "raise" statement in section The raise ' - 'statement.\n' - '\n' - '-[ Footnotes ]-\n' - '\n' - '[1] This limitation occurs because the code that is executed ' - 'by these\n' - ' operations is not available at the time the module is ' - 'compiled.\n', - 'execmodel': 'Execution model\n' - '***************\n' - '\n' - '\n' - 'Structure of a program\n' - '======================\n' - '\n' - 'A Python program is constructed from code blocks. A *block* is ' - 'a piece\n' - 'of Python program text that is executed as a unit. The ' - 'following are\n' - 'blocks: a module, a function body, and a class definition. ' - 'Each\n' - 'command typed interactively is a block. A script file (a file ' - 'given\n' - 'as standard input to the interpreter or specified as a command ' - 'line\n' - 'argument to the interpreter) is a code block. A script command ' - '(a\n' - 'command specified on the interpreter command line with the ' - '"-c"\n' - 'option) is a code block. A module run as a top level script (as ' - 'module\n' - '"__main__") from the command line using a "-m" argument is also ' - 'a code\n' - 'block. The string argument passed to the built-in functions ' - '"eval()"\n' - 'and "exec()" is a code block.\n' - '\n' - 'A code block is executed in an *execution frame*. A frame ' - 'contains\n' - 'some administrative information (used for debugging) and ' - 'determines\n' - 'where and how execution continues after the code block’s ' - 'execution has\n' - 'completed.\n' - '\n' - '\n' - 'Naming and binding\n' - '==================\n' - '\n' - '\n' - 'Binding of names\n' - '----------------\n' - '\n' - '*Names* refer to objects. Names are introduced by name ' - 'binding\n' - 'operations.\n' - '\n' - 'The following constructs bind names:\n' - '\n' - '* formal parameters to functions,\n' - '\n' - '* class definitions,\n' - '\n' - '* function definitions,\n' - '\n' - '* assignment expressions,\n' - '\n' - '* targets that are identifiers if occurring in an assignment:\n' - '\n' - ' * "for" loop header,\n' - '\n' - ' * after "as" in a "with" statement, "except" clause, ' - '"except*"\n' - ' clause, or in the as-pattern in structural pattern ' - 'matching,\n' - '\n' - ' * in a capture pattern in structural pattern matching\n' - '\n' - '* "import" statements.\n' - '\n' - '* "type" statements.\n' - '\n' - '* type parameter lists.\n' - '\n' - 'The "import" statement of the form "from ... import *" binds ' - 'all names\n' - 'defined in the imported module, except those beginning with an\n' - 'underscore. This form may only be used at the module level.\n' - '\n' - 'A target occurring in a "del" statement is also considered ' - 'bound for\n' - 'this purpose (though the actual semantics are to unbind the ' - 'name).\n' - '\n' - 'Each assignment or import statement occurs within a block ' - 'defined by a\n' - 'class or function definition or at the module level (the ' - 'top-level\n' - 'code block).\n' - '\n' - 'If a name is bound in a block, it is a local variable of that ' - 'block,\n' - 'unless declared as "nonlocal" or "global". If a name is bound ' - 'at the\n' - 'module level, it is a global variable. (The variables of the ' - 'module\n' - 'code block are local and global.) If a variable is used in a ' - 'code\n' - 'block but not defined there, it is a *free variable*.\n' - '\n' - 'Each occurrence of a name in the program text refers to the ' - '*binding*\n' - 'of that name established by the following name resolution ' - 'rules.\n' - '\n' - '\n' - 'Resolution of names\n' - '-------------------\n' - '\n' - 'A *scope* defines the visibility of a name within a block. If ' - 'a local\n' - 'variable is defined in a block, its scope includes that block. ' - 'If the\n' - 'definition occurs in a function block, the scope extends to any ' - 'blocks\n' - 'contained within the defining one, unless a contained block ' - 'introduces\n' - 'a different binding for the name.\n' - '\n' - 'When a name is used in a code block, it is resolved using the ' - 'nearest\n' - 'enclosing scope. The set of all such scopes visible to a code ' - 'block\n' - 'is called the block’s *environment*.\n' - '\n' - 'When a name is not found at all, a "NameError" exception is ' - 'raised. If\n' - 'the current scope is a function scope, and the name refers to a ' - 'local\n' - 'variable that has not yet been bound to a value at the point ' - 'where the\n' - 'name is used, an "UnboundLocalError" exception is raised.\n' - '"UnboundLocalError" is a subclass of "NameError".\n' - '\n' - 'If a name binding operation occurs anywhere within a code ' - 'block, all\n' - 'uses of the name within the block are treated as references to ' - 'the\n' - 'current block. This can lead to errors when a name is used ' - 'within a\n' - 'block before it is bound. This rule is subtle. Python lacks\n' - 'declarations and allows name binding operations to occur ' - 'anywhere\n' - 'within a code block. The local variables of a code block can ' - 'be\n' - 'determined by scanning the entire text of the block for name ' - 'binding\n' - 'operations. See the FAQ entry on UnboundLocalError for ' - 'examples.\n' - '\n' - 'If the "global" statement occurs within a block, all uses of ' - 'the names\n' - 'specified in the statement refer to the bindings of those names ' - 'in the\n' - 'top-level namespace. Names are resolved in the top-level ' - 'namespace by\n' - 'searching the global namespace, i.e. the namespace of the ' - 'module\n' - 'containing the code block, and the builtins namespace, the ' - 'namespace\n' - 'of the module "builtins". The global namespace is searched ' - 'first. If\n' - 'the names are not found there, the builtins namespace is ' - 'searched.\n' - 'The "global" statement must precede all uses of the listed ' - 'names.\n' - '\n' - 'The "global" statement has the same scope as a name binding ' - 'operation\n' - 'in the same block. If the nearest enclosing scope for a free ' - 'variable\n' - 'contains a global statement, the free variable is treated as a ' - 'global.\n' - '\n' - 'The "nonlocal" statement causes corresponding names to refer ' - 'to\n' - 'previously bound variables in the nearest enclosing function ' - 'scope.\n' - '"SyntaxError" is raised at compile time if the given name does ' - 'not\n' - 'exist in any enclosing function scope. Type parameters cannot ' - 'be\n' - 'rebound with the "nonlocal" statement.\n' - '\n' - 'The namespace for a module is automatically created the first ' - 'time a\n' - 'module is imported. The main module for a script is always ' - 'called\n' - '"__main__".\n' - '\n' - 'Class definition blocks and arguments to "exec()" and "eval()" ' - 'are\n' - 'special in the context of name resolution. A class definition ' - 'is an\n' - 'executable statement that may use and define names. These ' - 'references\n' - 'follow the normal rules for name resolution with an exception ' - 'that\n' - 'unbound local variables are looked up in the global namespace. ' - 'The\n' - 'namespace of the class definition becomes the attribute ' - 'dictionary of\n' - 'the class. The scope of names defined in a class block is ' - 'limited to\n' - 'the class block; it does not extend to the code blocks of ' - 'methods.\n' - 'This includes comprehensions and generator expressions, but it ' - 'does\n' - 'not include annotation scopes, which have access to their ' - 'enclosing\n' - 'class scopes. This means that the following will fail:\n' - '\n' - ' class A:\n' - ' a = 42\n' - ' b = list(a + i for i in range(10))\n' - '\n' - 'However, the following will succeed:\n' - '\n' - ' class A:\n' - ' type Alias = Nested\n' - ' class Nested: pass\n' - '\n' - " print(A.Alias.__value__) # \n" - '\n' - '\n' - 'Annotation scopes\n' - '-----------------\n' - '\n' - 'Type parameter lists and "type" statements introduce ' - '*annotation\n' - 'scopes*, which behave mostly like function scopes, but with ' - 'some\n' - 'exceptions discussed below. *Annotations* currently do not use\n' - 'annotation scopes, but they are expected to use annotation ' - 'scopes in\n' - 'Python 3.13 when **PEP 649** is implemented.\n' - '\n' - 'Annotation scopes are used in the following contexts:\n' - '\n' - '* Type parameter lists for generic type aliases.\n' - '\n' - '* Type parameter lists for generic functions. A generic ' - 'function’s\n' - ' annotations are executed within the annotation scope, but ' - 'its\n' - ' defaults and decorators are not.\n' - '\n' - '* Type parameter lists for generic classes. A generic class’s ' - 'base\n' - ' classes and keyword arguments are executed within the ' - 'annotation\n' - ' scope, but its decorators are not.\n' - '\n' - '* The bounds and constraints for type variables (lazily ' - 'evaluated).\n' - '\n' - '* The value of type aliases (lazily evaluated).\n' - '\n' - 'Annotation scopes differ from function scopes in the following ' - 'ways:\n' - '\n' - '* Annotation scopes have access to their enclosing class ' - 'namespace. If\n' - ' an annotation scope is immediately within a class scope, or ' - 'within\n' - ' another annotation scope that is immediately within a class ' - 'scope,\n' - ' the code in the annotation scope can use names defined in the ' - 'class\n' - ' scope as if it were executed directly within the class body. ' - 'This\n' - ' contrasts with regular functions defined within classes, ' - 'which\n' - ' cannot access names defined in the class scope.\n' - '\n' - '* Expressions in annotation scopes cannot contain "yield", ' - '"yield\n' - ' from", "await", or ":=" expressions. (These expressions are ' - 'allowed\n' - ' in other scopes contained within the annotation scope.)\n' - '\n' - '* Names defined in annotation scopes cannot be rebound with ' - '"nonlocal"\n' - ' statements in inner scopes. This includes only type ' - 'parameters, as\n' - ' no other syntactic elements that can appear within annotation ' - 'scopes\n' - ' can introduce new names.\n' - '\n' - '* While annotation scopes have an internal name, that name is ' - 'not\n' - ' reflected in the *__qualname__* of objects defined within the ' - 'scope.\n' - ' Instead, the "__qualname__" of such objects is as if the ' - 'object were\n' - ' defined in the enclosing scope.\n' - '\n' - 'New in version 3.12: Annotation scopes were introduced in ' - 'Python 3.12\n' - 'as part of **PEP 695**.\n' - '\n' - '\n' - 'Lazy evaluation\n' - '---------------\n' - '\n' - 'The values of type aliases created through the "type" statement ' - 'are\n' - '*lazily evaluated*. The same applies to the bounds and ' - 'constraints of\n' - 'type variables created through the type parameter syntax. This ' - 'means\n' - 'that they are not evaluated when the type alias or type ' - 'variable is\n' - 'created. Instead, they are only evaluated when doing so is ' - 'necessary\n' - 'to resolve an attribute access.\n' - '\n' - 'Example:\n' - '\n' - ' >>> type Alias = 1/0\n' - ' >>> Alias.__value__\n' - ' Traceback (most recent call last):\n' - ' ...\n' - ' ZeroDivisionError: division by zero\n' - ' >>> def func[T: 1/0](): pass\n' - ' >>> T = func.__type_params__[0]\n' - ' >>> T.__bound__\n' - ' Traceback (most recent call last):\n' - ' ...\n' - ' ZeroDivisionError: division by zero\n' - '\n' - 'Here the exception is raised only when the "__value__" ' - 'attribute of\n' - 'the type alias or the "__bound__" attribute of the type ' - 'variable is\n' - 'accessed.\n' - '\n' - 'This behavior is primarily useful for references to types that ' - 'have\n' - 'not yet been defined when the type alias or type variable is ' - 'created.\n' - 'For example, lazy evaluation enables creation of mutually ' - 'recursive\n' - 'type aliases:\n' - '\n' - ' from typing import Literal\n' - '\n' - ' type SimpleExpr = int | Parenthesized\n' - ' type Parenthesized = tuple[Literal["("], Expr, ' - 'Literal[")"]]\n' - ' type Expr = SimpleExpr | tuple[SimpleExpr, Literal["+", ' - '"-"], Expr]\n' - '\n' - 'Lazily evaluated values are evaluated in annotation scope, ' - 'which means\n' - 'that names that appear inside the lazily evaluated value are ' - 'looked up\n' - 'as if they were used in the immediately enclosing scope.\n' - '\n' - 'New in version 3.12.\n' - '\n' - '\n' - 'Builtins and restricted execution\n' - '---------------------------------\n' - '\n' - '**CPython implementation detail:** Users should not touch\n' - '"__builtins__"; it is strictly an implementation detail. ' - 'Users\n' - 'wanting to override values in the builtins namespace should ' - '"import"\n' - 'the "builtins" module and modify its attributes appropriately.\n' - '\n' - 'The builtins namespace associated with the execution of a code ' - 'block\n' - 'is actually found by looking up the name "__builtins__" in its ' - 'global\n' - 'namespace; this should be a dictionary or a module (in the ' - 'latter case\n' - 'the module’s dictionary is used). By default, when in the ' - '"__main__"\n' - 'module, "__builtins__" is the built-in module "builtins"; when ' - 'in any\n' - 'other module, "__builtins__" is an alias for the dictionary of ' - 'the\n' - '"builtins" module itself.\n' - '\n' - '\n' - 'Interaction with dynamic features\n' - '---------------------------------\n' - '\n' - 'Name resolution of free variables occurs at runtime, not at ' - 'compile\n' - 'time. This means that the following code will print 42:\n' - '\n' - ' i = 10\n' - ' def f():\n' - ' print(i)\n' - ' i = 42\n' - ' f()\n' - '\n' - 'The "eval()" and "exec()" functions do not have access to the ' - 'full\n' - 'environment for resolving names. Names may be resolved in the ' - 'local\n' - 'and global namespaces of the caller. Free variables are not ' - 'resolved\n' - 'in the nearest enclosing namespace, but in the global ' - 'namespace. [1]\n' - 'The "exec()" and "eval()" functions have optional arguments to\n' - 'override the global and local namespace. If only one namespace ' - 'is\n' - 'specified, it is used for both.\n' - '\n' - '\n' - 'Exceptions\n' - '==========\n' - '\n' - 'Exceptions are a means of breaking out of the normal flow of ' - 'control\n' - 'of a code block in order to handle errors or other exceptional\n' - 'conditions. An exception is *raised* at the point where the ' - 'error is\n' - 'detected; it may be *handled* by the surrounding code block or ' - 'by any\n' - 'code block that directly or indirectly invoked the code block ' - 'where\n' - 'the error occurred.\n' - '\n' - 'The Python interpreter raises an exception when it detects a ' - 'run-time\n' - 'error (such as division by zero). A Python program can also\n' - 'explicitly raise an exception with the "raise" statement. ' - 'Exception\n' - 'handlers are specified with the "try" … "except" statement. ' - 'The\n' - '"finally" clause of such a statement can be used to specify ' - 'cleanup\n' - 'code which does not handle the exception, but is executed ' - 'whether an\n' - 'exception occurred or not in the preceding code.\n' - '\n' - 'Python uses the “termination” model of error handling: an ' - 'exception\n' - 'handler can find out what happened and continue execution at an ' - 'outer\n' - 'level, but it cannot repair the cause of the error and retry ' - 'the\n' - 'failing operation (except by re-entering the offending piece of ' - 'code\n' - 'from the top).\n' - '\n' - 'When an exception is not handled at all, the interpreter ' - 'terminates\n' - 'execution of the program, or returns to its interactive main ' - 'loop. In\n' - 'either case, it prints a stack traceback, except when the ' - 'exception is\n' - '"SystemExit".\n' - '\n' - 'Exceptions are identified by class instances. The "except" ' - 'clause is\n' - 'selected depending on the class of the instance: it must ' - 'reference the\n' - 'class of the instance or a *non-virtual base class* thereof. ' - 'The\n' - 'instance can be received by the handler and can carry ' - 'additional\n' - 'information about the exceptional condition.\n' - '\n' - 'Note:\n' - '\n' - ' Exception messages are not part of the Python API. Their ' - 'contents\n' - ' may change from one version of Python to the next without ' - 'warning\n' - ' and should not be relied on by code which will run under ' - 'multiple\n' - ' versions of the interpreter.\n' - '\n' - 'See also the description of the "try" statement in section The ' - 'try\n' - 'statement and "raise" statement in section The raise ' - 'statement.\n' - '\n' - '-[ Footnotes ]-\n' - '\n' - '[1] This limitation occurs because the code that is executed by ' - 'these\n' - ' operations is not available at the time the module is ' - 'compiled.\n', - 'exprlists': 'Expression lists\n' - '****************\n' - '\n' - ' expression_list ::= expression ("," expression)* [","]\n' - ' starred_list ::= starred_item ("," starred_item)* ' - '[","]\n' - ' starred_expression ::= expression | (starred_item ",")* ' - '[starred_item]\n' - ' starred_item ::= assignment_expression | "*" or_expr\n' - '\n' - 'Except when part of a list or set display, an expression list\n' - 'containing at least one comma yields a tuple. The length of ' - 'the tuple\n' - 'is the number of expressions in the list. The expressions are\n' - 'evaluated from left to right.\n' - '\n' - 'An asterisk "*" denotes *iterable unpacking*. Its operand must ' - 'be an\n' - '*iterable*. The iterable is expanded into a sequence of items, ' - 'which\n' - 'are included in the new tuple, list, or set, at the site of ' - 'the\n' - 'unpacking.\n' - '\n' - 'New in version 3.5: Iterable unpacking in expression lists, ' - 'originally\n' - 'proposed by **PEP 448**.\n' - '\n' - 'A trailing comma is required only to create a one-item tuple, ' - 'such as\n' - '"1,"; it is optional in all other cases. A single expression ' - 'without a\n' - 'trailing comma doesn’t create a tuple, but rather yields the ' - 'value of\n' - 'that expression. (To create an empty tuple, use an empty pair ' - 'of\n' - 'parentheses: "()".)\n', - 'floating': 'Floating point literals\n' - '***********************\n' - '\n' - 'Floating point literals are described by the following lexical\n' - 'definitions:\n' - '\n' - ' floatnumber ::= pointfloat | exponentfloat\n' - ' pointfloat ::= [digitpart] fraction | digitpart "."\n' - ' exponentfloat ::= (digitpart | pointfloat) exponent\n' - ' digitpart ::= digit (["_"] digit)*\n' - ' fraction ::= "." digitpart\n' - ' exponent ::= ("e" | "E") ["+" | "-"] digitpart\n' - '\n' - 'Note that the integer and exponent parts are always interpreted ' - 'using\n' - 'radix 10. For example, "077e010" is legal, and denotes the same ' - 'number\n' - 'as "77e10". The allowed range of floating point literals is\n' - 'implementation-dependent. As in integer literals, underscores ' - 'are\n' - 'supported for digit grouping.\n' - '\n' - 'Some examples of floating point literals:\n' - '\n' - ' 3.14 10. .001 1e100 3.14e-10 0e0 ' - '3.14_15_93\n' - '\n' - 'Changed in version 3.6: Underscores are now allowed for ' - 'grouping\n' - 'purposes in literals.\n', - 'for': 'The "for" statement\n' - '*******************\n' - '\n' - 'The "for" statement is used to iterate over the elements of a ' - 'sequence\n' - '(such as a string, tuple or list) or other iterable object:\n' - '\n' - ' for_stmt ::= "for" target_list "in" starred_list ":" suite\n' - ' ["else" ":" suite]\n' - '\n' - 'The "starred_list" expression is evaluated once; it should yield an\n' - '*iterable* object. An *iterator* is created for that iterable. The\n' - 'first item provided by the iterator is then assigned to the target\n' - 'list using the standard rules for assignments (see Assignment\n' - 'statements), and the suite is executed. This repeats for each item\n' - 'provided by the iterator. When the iterator is exhausted, the suite\n' - 'in the "else" clause, if present, is executed, and the loop\n' - 'terminates.\n' - '\n' - 'A "break" statement executed in the first suite terminates the loop\n' - 'without executing the "else" clause’s suite. A "continue" statement\n' - 'executed in the first suite skips the rest of the suite and ' - 'continues\n' - 'with the next item, or with the "else" clause if there is no next\n' - 'item.\n' - '\n' - 'The for-loop makes assignments to the variables in the target list.\n' - 'This overwrites all previous assignments to those variables ' - 'including\n' - 'those made in the suite of the for-loop:\n' - '\n' - ' for i in range(10):\n' - ' print(i)\n' - ' i = 5 # this will not affect the for-loop\n' - ' # because i will be overwritten with the ' - 'next\n' - ' # index in the range\n' - '\n' - 'Names in the target list are not deleted when the loop is finished,\n' - 'but if the sequence is empty, they will not have been assigned to at\n' - 'all by the loop. Hint: the built-in type "range()" represents\n' - 'immutable arithmetic sequences of integers. For instance, iterating\n' - '"range(3)" successively yields 0, 1, and then 2.\n' - '\n' - 'Changed in version 3.11: Starred elements are now allowed in the\n' - 'expression list.\n', - 'formatstrings': 'Format String Syntax\n' - '********************\n' - '\n' - 'The "str.format()" method and the "Formatter" class share ' - 'the same\n' - 'syntax for format strings (although in the case of ' - '"Formatter",\n' - 'subclasses can define their own format string syntax). The ' - 'syntax is\n' - 'related to that of formatted string literals, but it is ' - 'less\n' - 'sophisticated and, in particular, does not support ' - 'arbitrary\n' - 'expressions.\n' - '\n' - 'Format strings contain “replacement fields” surrounded by ' - 'curly braces\n' - '"{}". Anything that is not contained in braces is ' - 'considered literal\n' - 'text, which is copied unchanged to the output. If you need ' - 'to include\n' - 'a brace character in the literal text, it can be escaped by ' - 'doubling:\n' - '"{{" and "}}".\n' - '\n' - 'The grammar for a replacement field is as follows:\n' - '\n' - ' replacement_field ::= "{" [field_name] ["!" conversion] ' - '[":" format_spec] "}"\n' - ' field_name ::= arg_name ("." attribute_name | "[" ' - 'element_index "]")*\n' - ' arg_name ::= [identifier | digit+]\n' - ' attribute_name ::= identifier\n' - ' element_index ::= digit+ | index_string\n' - ' index_string ::= ' - '+\n' - ' conversion ::= "r" | "s" | "a"\n' - ' format_spec ::= format-spec:format_spec\n' - '\n' - 'In less formal terms, the replacement field can start with ' - 'a\n' - '*field_name* that specifies the object whose value is to be ' - 'formatted\n' - 'and inserted into the output instead of the replacement ' - 'field. The\n' - '*field_name* is optionally followed by a *conversion* ' - 'field, which is\n' - 'preceded by an exclamation point "\'!\'", and a ' - '*format_spec*, which is\n' - 'preceded by a colon "\':\'". These specify a non-default ' - 'format for the\n' - 'replacement value.\n' - '\n' - 'See also the Format Specification Mini-Language section.\n' - '\n' - 'The *field_name* itself begins with an *arg_name* that is ' - 'either a\n' - 'number or a keyword. If it’s a number, it refers to a ' - 'positional\n' - 'argument, and if it’s a keyword, it refers to a named ' - 'keyword\n' - 'argument. An *arg_name* is treated as a number if a call ' - 'to\n' - '"str.isdecimal()" on the string would return true. If the ' - 'numerical\n' - 'arg_names in a format string are 0, 1, 2, … in sequence, ' - 'they can all\n' - 'be omitted (not just some) and the numbers 0, 1, 2, … will ' - 'be\n' - 'automatically inserted in that order. Because *arg_name* is ' - 'not quote-\n' - 'delimited, it is not possible to specify arbitrary ' - 'dictionary keys\n' - '(e.g., the strings "\'10\'" or "\':-]\'") within a format ' - 'string. The\n' - '*arg_name* can be followed by any number of index or ' - 'attribute\n' - 'expressions. An expression of the form "\'.name\'" selects ' - 'the named\n' - 'attribute using "getattr()", while an expression of the ' - 'form\n' - '"\'[index]\'" does an index lookup using "__getitem__()".\n' - '\n' - 'Changed in version 3.1: The positional argument specifiers ' - 'can be\n' - 'omitted for "str.format()", so "\'{} {}\'.format(a, b)" is ' - 'equivalent to\n' - '"\'{0} {1}\'.format(a, b)".\n' - '\n' - 'Changed in version 3.4: The positional argument specifiers ' - 'can be\n' - 'omitted for "Formatter".\n' - '\n' - 'Some simple format string examples:\n' - '\n' - ' "First, thou shalt count to {0}" # References first ' - 'positional argument\n' - ' "Bring me a {}" # Implicitly ' - 'references the first positional argument\n' - ' "From {} to {}" # Same as "From {0} to ' - '{1}"\n' - ' "My quest is {name}" # References keyword ' - "argument 'name'\n" - ' "Weight in tons {0.weight}" # \'weight\' attribute ' - 'of first positional arg\n' - ' "Units destroyed: {players[0]}" # First element of ' - "keyword argument 'players'.\n" - '\n' - 'The *conversion* field causes a type coercion before ' - 'formatting.\n' - 'Normally, the job of formatting a value is done by the ' - '"__format__()"\n' - 'method of the value itself. However, in some cases it is ' - 'desirable to\n' - 'force a type to be formatted as a string, overriding its ' - 'own\n' - 'definition of formatting. By converting the value to a ' - 'string before\n' - 'calling "__format__()", the normal formatting logic is ' - 'bypassed.\n' - '\n' - 'Three conversion flags are currently supported: "\'!s\'" ' - 'which calls\n' - '"str()" on the value, "\'!r\'" which calls "repr()" and ' - '"\'!a\'" which\n' - 'calls "ascii()".\n' - '\n' - 'Some examples:\n' - '\n' - ' "Harold\'s a clever {0!s}" # Calls str() on the ' - 'argument first\n' - ' "Bring out the holy {name!r}" # Calls repr() on the ' - 'argument first\n' - ' "More {!a}" # Calls ascii() on the ' - 'argument first\n' - '\n' - 'The *format_spec* field contains a specification of how the ' - 'value\n' - 'should be presented, including such details as field width, ' - 'alignment,\n' - 'padding, decimal precision and so on. Each value type can ' - 'define its\n' - 'own “formatting mini-language” or interpretation of the ' - '*format_spec*.\n' - '\n' - 'Most built-in types support a common formatting ' - 'mini-language, which\n' - 'is described in the next section.\n' - '\n' - 'A *format_spec* field can also include nested replacement ' - 'fields\n' - 'within it. These nested replacement fields may contain a ' - 'field name,\n' - 'conversion flag and format specification, but deeper ' - 'nesting is not\n' - 'allowed. The replacement fields within the format_spec ' - 'are\n' - 'substituted before the *format_spec* string is interpreted. ' - 'This\n' - 'allows the formatting of a value to be dynamically ' - 'specified.\n' - '\n' - 'See the Format examples section for some examples.\n' - '\n' - '\n' - 'Format Specification Mini-Language\n' - '==================================\n' - '\n' - '“Format specifications” are used within replacement fields ' - 'contained\n' - 'within a format string to define how individual values are ' - 'presented\n' - '(see Format String Syntax and f-strings). They can also be ' - 'passed\n' - 'directly to the built-in "format()" function. Each ' - 'formattable type\n' - 'may define how the format specification is to be ' - 'interpreted.\n' - '\n' - 'Most built-in types implement the following options for ' - 'format\n' - 'specifications, although some of the formatting options are ' - 'only\n' - 'supported by the numeric types.\n' - '\n' - 'A general convention is that an empty format specification ' - 'produces\n' - 'the same result as if you had called "str()" on the value. ' - 'A non-empty\n' - 'format specification typically modifies the result.\n' - '\n' - 'The general form of a *standard format specifier* is:\n' - '\n' - ' format_spec ::= ' - '[[fill]align][sign]["z"]["#"]["0"][width][grouping_option]["." ' - 'precision][type]\n' - ' fill ::= \n' - ' align ::= "<" | ">" | "=" | "^"\n' - ' sign ::= "+" | "-" | " "\n' - ' width ::= digit+\n' - ' grouping_option ::= "_" | ","\n' - ' precision ::= digit+\n' - ' type ::= "b" | "c" | "d" | "e" | "E" | "f" | ' - '"F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n' - '\n' - 'If a valid *align* value is specified, it can be preceded ' - 'by a *fill*\n' - 'character that can be any character and defaults to a space ' - 'if\n' - 'omitted. It is not possible to use a literal curly brace ' - '(”"{"” or\n' - '“"}"”) as the *fill* character in a formatted string ' - 'literal or when\n' - 'using the "str.format()" method. However, it is possible ' - 'to insert a\n' - 'curly brace with a nested replacement field. This ' - 'limitation doesn’t\n' - 'affect the "format()" function.\n' - '\n' - 'The meaning of the various alignment options is as ' - 'follows:\n' - '\n' - '+-----------+------------------------------------------------------------+\n' - '| Option | ' - 'Meaning ' - '|\n' - '|===========|============================================================|\n' - '| "\'<\'" | Forces the field to be left-aligned within ' - 'the available |\n' - '| | space (this is the default for most ' - 'objects). |\n' - '+-----------+------------------------------------------------------------+\n' - '| "\'>\'" | Forces the field to be right-aligned within ' - 'the available |\n' - '| | space (this is the default for ' - 'numbers). |\n' - '+-----------+------------------------------------------------------------+\n' - '| "\'=\'" | Forces the padding to be placed after the ' - 'sign (if any) |\n' - '| | but before the digits. This is used for ' - 'printing fields |\n' - '| | in the form ‘+000000120’. This alignment ' - 'option is only |\n' - '| | valid for numeric types. It becomes the ' - 'default for |\n' - '| | numbers when ‘0’ immediately precedes the ' - 'field width. |\n' - '+-----------+------------------------------------------------------------+\n' - '| "\'^\'" | Forces the field to be centered within the ' - 'available |\n' - '| | ' - 'space. ' - '|\n' - '+-----------+------------------------------------------------------------+\n' - '\n' - 'Note that unless a minimum field width is defined, the ' - 'field width\n' - 'will always be the same size as the data to fill it, so ' - 'that the\n' - 'alignment option has no meaning in this case.\n' - '\n' - 'The *sign* option is only valid for number types, and can ' - 'be one of\n' - 'the following:\n' - '\n' - '+-----------+------------------------------------------------------------+\n' - '| Option | ' - 'Meaning ' - '|\n' - '|===========|============================================================|\n' - '| "\'+\'" | indicates that a sign should be used for ' - 'both positive as |\n' - '| | well as negative ' - 'numbers. |\n' - '+-----------+------------------------------------------------------------+\n' - '| "\'-\'" | indicates that a sign should be used only ' - 'for negative |\n' - '| | numbers (this is the default ' - 'behavior). |\n' - '+-----------+------------------------------------------------------------+\n' - '| space | indicates that a leading space should be used ' - 'on positive |\n' - '| | numbers, and a minus sign on negative ' - 'numbers. |\n' - '+-----------+------------------------------------------------------------+\n' - '\n' - 'The "\'z\'" option coerces negative zero floating-point ' - 'values to\n' - 'positive zero after rounding to the format precision. This ' - 'option is\n' - 'only valid for floating-point presentation types.\n' - '\n' - 'Changed in version 3.11: Added the "\'z\'" option (see also ' - '**PEP\n' - '682**).\n' - '\n' - 'The "\'#\'" option causes the “alternate form” to be used ' - 'for the\n' - 'conversion. The alternate form is defined differently for ' - 'different\n' - 'types. This option is only valid for integer, float and ' - 'complex\n' - 'types. For integers, when binary, octal, or hexadecimal ' - 'output is\n' - 'used, this option adds the respective prefix "\'0b\'", ' - '"\'0o\'", "\'0x\'",\n' - 'or "\'0X\'" to the output value. For float and complex the ' - 'alternate\n' - 'form causes the result of the conversion to always contain ' - 'a decimal-\n' - 'point character, even if no digits follow it. Normally, a ' - 'decimal-\n' - 'point character appears in the result of these conversions ' - 'only if a\n' - 'digit follows it. In addition, for "\'g\'" and "\'G\'" ' - 'conversions,\n' - 'trailing zeros are not removed from the result.\n' - '\n' - 'The "\',\'" option signals the use of a comma for a ' - 'thousands separator.\n' - 'For a locale aware separator, use the "\'n\'" integer ' - 'presentation type\n' - 'instead.\n' - '\n' - 'Changed in version 3.1: Added the "\',\'" option (see also ' - '**PEP 378**).\n' - '\n' - 'The "\'_\'" option signals the use of an underscore for a ' - 'thousands\n' - 'separator for floating point presentation types and for ' - 'integer\n' - 'presentation type "\'d\'". For integer presentation types ' - '"\'b\'", "\'o\'",\n' - '"\'x\'", and "\'X\'", underscores will be inserted every 4 ' - 'digits. For\n' - 'other presentation types, specifying this option is an ' - 'error.\n' - '\n' - 'Changed in version 3.6: Added the "\'_\'" option (see also ' - '**PEP 515**).\n' - '\n' - '*width* is a decimal integer defining the minimum total ' - 'field width,\n' - 'including any prefixes, separators, and other formatting ' - 'characters.\n' - 'If not specified, then the field width will be determined ' - 'by the\n' - 'content.\n' - '\n' - 'When no explicit alignment is given, preceding the *width* ' - 'field by a\n' - 'zero ("\'0\'") character enables sign-aware zero-padding ' - 'for numeric\n' - 'types. This is equivalent to a *fill* character of "\'0\'" ' - 'with an\n' - '*alignment* type of "\'=\'".\n' - '\n' - 'Changed in version 3.10: Preceding the *width* field by ' - '"\'0\'" no\n' - 'longer affects the default alignment for strings.\n' - '\n' - 'The *precision* is a decimal integer indicating how many ' - 'digits should\n' - 'be displayed after the decimal point for presentation types ' - '"\'f\'" and\n' - '"\'F\'", or before and after the decimal point for ' - 'presentation types\n' - '"\'g\'" or "\'G\'". For string presentation types the ' - 'field indicates the\n' - 'maximum field size - in other words, how many characters ' - 'will be used\n' - 'from the field content. The *precision* is not allowed for ' - 'integer\n' - 'presentation types.\n' - '\n' - 'Finally, the *type* determines how the data should be ' - 'presented.\n' - '\n' - 'The available string presentation types are:\n' - '\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | Type | ' - 'Meaning ' - '|\n' - ' ' - '|===========|============================================================|\n' - ' | "\'s\'" | String format. This is the default type ' - 'for strings and |\n' - ' | | may be ' - 'omitted. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | None | The same as ' - '"\'s\'". |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - '\n' - 'The available integer presentation types are:\n' - '\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | Type | ' - 'Meaning ' - '|\n' - ' ' - '|===========|============================================================|\n' - ' | "\'b\'" | Binary format. Outputs the number in ' - 'base 2. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'c\'" | Character. Converts the integer to the ' - 'corresponding |\n' - ' | | unicode character before ' - 'printing. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'d\'" | Decimal Integer. Outputs the number in ' - 'base 10. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'o\'" | Octal format. Outputs the number in base ' - '8. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'x\'" | Hex format. Outputs the number in base ' - '16, using lower- |\n' - ' | | case letters for the digits above ' - '9. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'X\'" | Hex format. Outputs the number in base ' - '16, using upper- |\n' - ' | | case letters for the digits above 9. In ' - 'case "\'#\'" is |\n' - ' | | specified, the prefix "\'0x\'" will be ' - 'upper-cased to "\'0X\'" |\n' - ' | | as ' - 'well. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'n\'" | Number. This is the same as "\'d\'", ' - 'except that it uses the |\n' - ' | | current locale setting to insert the ' - 'appropriate number |\n' - ' | | separator ' - 'characters. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | None | The same as ' - '"\'d\'". |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - '\n' - 'In addition to the above presentation types, integers can ' - 'be formatted\n' - 'with the floating point presentation types listed below ' - '(except "\'n\'"\n' - 'and "None"). When doing so, "float()" is used to convert ' - 'the integer\n' - 'to a floating point number before formatting.\n' - '\n' - 'The available presentation types for "float" and "Decimal" ' - 'values are:\n' - '\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | Type | ' - 'Meaning ' - '|\n' - ' ' - '|===========|============================================================|\n' - ' | "\'e\'" | Scientific notation. For a given ' - 'precision "p", formats |\n' - ' | | the number in scientific notation with the ' - 'letter ‘e’ |\n' - ' | | separating the coefficient from the ' - 'exponent. The |\n' - ' | | coefficient has one digit before and "p" ' - 'digits after the |\n' - ' | | decimal point, for a total of "p + 1" ' - 'significant digits. |\n' - ' | | With no precision given, uses a precision ' - 'of "6" digits |\n' - ' | | after the decimal point for "float", and ' - 'shows all |\n' - ' | | coefficient digits for "Decimal". If no ' - 'digits follow the |\n' - ' | | decimal point, the decimal point is also ' - 'removed unless |\n' - ' | | the "#" option is ' - 'used. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'E\'" | Scientific notation. Same as "\'e\'" ' - 'except it uses an upper |\n' - ' | | case ‘E’ as the separator ' - 'character. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'f\'" | Fixed-point notation. For a given ' - 'precision "p", formats |\n' - ' | | the number as a decimal number with ' - 'exactly "p" digits |\n' - ' | | following the decimal point. With no ' - 'precision given, uses |\n' - ' | | a precision of "6" digits after the ' - 'decimal point for |\n' - ' | | "float", and uses a precision large enough ' - 'to show all |\n' - ' | | coefficient digits for "Decimal". If no ' - 'digits follow the |\n' - ' | | decimal point, the decimal point is also ' - 'removed unless |\n' - ' | | the "#" option is ' - 'used. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'F\'" | Fixed-point notation. Same as "\'f\'", ' - 'but converts "nan" to |\n' - ' | | "NAN" and "inf" to ' - '"INF". |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'g\'" | General format. For a given precision ' - '"p >= 1", this |\n' - ' | | rounds the number to "p" significant ' - 'digits and then |\n' - ' | | formats the result in either fixed-point ' - 'format or in |\n' - ' | | scientific notation, depending on its ' - 'magnitude. A |\n' - ' | | precision of "0" is treated as equivalent ' - 'to a precision |\n' - ' | | of "1". The precise rules are as follows: ' - 'suppose that |\n' - ' | | the result formatted with presentation ' - 'type "\'e\'" and |\n' - ' | | precision "p-1" would have exponent ' - '"exp". Then, if "m <= |\n' - ' | | exp < p", where "m" is -4 for floats and ' - '-6 for |\n' - ' | | "Decimals", the number is formatted with ' - 'presentation type |\n' - ' | | "\'f\'" and precision "p-1-exp". ' - 'Otherwise, the number is |\n' - ' | | formatted with presentation type "\'e\'" ' - 'and precision |\n' - ' | | "p-1". In both cases insignificant ' - 'trailing zeros are |\n' - ' | | removed from the significand, and the ' - 'decimal point is |\n' - ' | | also removed if there are no remaining ' - 'digits following |\n' - ' | | it, unless the "\'#\'" option is used. ' - 'With no precision |\n' - ' | | given, uses a precision of "6" significant ' - 'digits for |\n' - ' | | "float". For "Decimal", the coefficient of ' - 'the result is |\n' - ' | | formed from the coefficient digits of the ' - 'value; |\n' - ' | | scientific notation is used for values ' - 'smaller than "1e-6" |\n' - ' | | in absolute value and values where the ' - 'place value of the |\n' - ' | | least significant digit is larger than 1, ' - 'and fixed-point |\n' - ' | | notation is used otherwise. Positive and ' - 'negative |\n' - ' | | infinity, positive and negative zero, and ' - 'nans, are |\n' - ' | | formatted as "inf", "-inf", "0", "-0" and ' - '"nan" |\n' - ' | | respectively, regardless of the ' - 'precision. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'G\'" | General format. Same as "\'g\'" except ' - 'switches to "\'E\'" if |\n' - ' | | the number gets too large. The ' - 'representations of infinity |\n' - ' | | and NaN are uppercased, ' - 'too. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'n\'" | Number. This is the same as "\'g\'", ' - 'except that it uses the |\n' - ' | | current locale setting to insert the ' - 'appropriate number |\n' - ' | | separator ' - 'characters. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'%\'" | Percentage. Multiplies the number by 100 ' - 'and displays in |\n' - ' | | fixed ("\'f\'") format, followed by a ' - 'percent sign. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | None | For "float" this is the same as "\'g\'", ' - 'except that when |\n' - ' | | fixed-point notation is used to format the ' - 'result, it |\n' - ' | | always includes at least one digit past ' - 'the decimal point. |\n' - ' | | The precision used is as large as needed ' - 'to represent the |\n' - ' | | given value faithfully. For "Decimal", ' - 'this is the same |\n' - ' | | as either "\'g\'" or "\'G\'" depending on ' - 'the value of |\n' - ' | | "context.capitals" for the current decimal ' - 'context. The |\n' - ' | | overall effect is to match the output of ' - '"str()" as |\n' - ' | | altered by the other format ' - 'modifiers. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - '\n' - '\n' - 'Format examples\n' - '===============\n' - '\n' - 'This section contains examples of the "str.format()" syntax ' - 'and\n' - 'comparison with the old "%"-formatting.\n' - '\n' - 'In most of the cases the syntax is similar to the old ' - '"%"-formatting,\n' - 'with the addition of the "{}" and with ":" used instead of ' - '"%". For\n' - 'example, "\'%03.2f\'" can be translated to "\'{:03.2f}\'".\n' - '\n' - 'The new format syntax also supports new and different ' - 'options, shown\n' - 'in the following examples.\n' - '\n' - 'Accessing arguments by position:\n' - '\n' - " >>> '{0}, {1}, {2}'.format('a', 'b', 'c')\n" - " 'a, b, c'\n" - " >>> '{}, {}, {}'.format('a', 'b', 'c') # 3.1+ only\n" - " 'a, b, c'\n" - " >>> '{2}, {1}, {0}'.format('a', 'b', 'c')\n" - " 'c, b, a'\n" - " >>> '{2}, {1}, {0}'.format(*'abc') # unpacking " - 'argument sequence\n' - " 'c, b, a'\n" - " >>> '{0}{1}{0}'.format('abra', 'cad') # arguments' " - 'indices can be repeated\n' - " 'abracadabra'\n" - '\n' - 'Accessing arguments by name:\n' - '\n' - " >>> 'Coordinates: {latitude}, " - "{longitude}'.format(latitude='37.24N', " - "longitude='-115.81W')\n" - " 'Coordinates: 37.24N, -115.81W'\n" - " >>> coord = {'latitude': '37.24N', 'longitude': " - "'-115.81W'}\n" - " >>> 'Coordinates: {latitude}, " - "{longitude}'.format(**coord)\n" - " 'Coordinates: 37.24N, -115.81W'\n" - '\n' - 'Accessing arguments’ attributes:\n' - '\n' - ' >>> c = 3-5j\n' - " >>> ('The complex number {0} is formed from the real " - "part {0.real} '\n" - " ... 'and the imaginary part {0.imag}.').format(c)\n" - " 'The complex number (3-5j) is formed from the real part " - "3.0 and the imaginary part -5.0.'\n" - ' >>> class Point:\n' - ' ... def __init__(self, x, y):\n' - ' ... self.x, self.y = x, y\n' - ' ... def __str__(self):\n' - " ... return 'Point({self.x}, " - "{self.y})'.format(self=self)\n" - ' ...\n' - ' >>> str(Point(4, 2))\n' - " 'Point(4, 2)'\n" - '\n' - 'Accessing arguments’ items:\n' - '\n' - ' >>> coord = (3, 5)\n' - " >>> 'X: {0[0]}; Y: {0[1]}'.format(coord)\n" - " 'X: 3; Y: 5'\n" - '\n' - 'Replacing "%s" and "%r":\n' - '\n' - ' >>> "repr() shows quotes: {!r}; str() doesn\'t: ' - '{!s}".format(\'test1\', \'test2\')\n' - ' "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n' - '\n' - 'Aligning the text and specifying a width:\n' - '\n' - " >>> '{:<30}'.format('left aligned')\n" - " 'left aligned '\n" - " >>> '{:>30}'.format('right aligned')\n" - " ' right aligned'\n" - " >>> '{:^30}'.format('centered')\n" - " ' centered '\n" - " >>> '{:*^30}'.format('centered') # use '*' as a fill " - 'char\n' - " '***********centered***********'\n" - '\n' - 'Replacing "%+f", "%-f", and "% f" and specifying a sign:\n' - '\n' - " >>> '{:+f}; {:+f}'.format(3.14, -3.14) # show it " - 'always\n' - " '+3.140000; -3.140000'\n" - " >>> '{: f}; {: f}'.format(3.14, -3.14) # show a space " - 'for positive numbers\n' - " ' 3.140000; -3.140000'\n" - " >>> '{:-f}; {:-f}'.format(3.14, -3.14) # show only the " - "minus -- same as '{:f}; {:f}'\n" - " '3.140000; -3.140000'\n" - '\n' - 'Replacing "%x" and "%o" and converting the value to ' - 'different bases:\n' - '\n' - ' >>> # format also supports binary numbers\n' - ' >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: ' - '{0:b}".format(42)\n' - " 'int: 42; hex: 2a; oct: 52; bin: 101010'\n" - ' >>> # with 0x, 0o, or 0b as prefix:\n' - ' >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: ' - '{0:#b}".format(42)\n' - " 'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010'\n" - '\n' - 'Using the comma as a thousands separator:\n' - '\n' - " >>> '{:,}'.format(1234567890)\n" - " '1,234,567,890'\n" - '\n' - 'Expressing a percentage:\n' - '\n' - ' >>> points = 19\n' - ' >>> total = 22\n' - " >>> 'Correct answers: {:.2%}'.format(points/total)\n" - " 'Correct answers: 86.36%'\n" - '\n' - 'Using type-specific formatting:\n' - '\n' - ' >>> import datetime\n' - ' >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n' - " >>> '{:%Y-%m-%d %H:%M:%S}'.format(d)\n" - " '2010-07-04 12:15:58'\n" - '\n' - 'Nesting arguments and more complex examples:\n' - '\n' - " >>> for align, text in zip('<^>', ['left', 'center', " - "'right']):\n" - " ... '{0:{fill}{align}16}'.format(text, fill=align, " - 'align=align)\n' - ' ...\n' - " 'left<<<<<<<<<<<<'\n" - " '^^^^^center^^^^^'\n" - " '>>>>>>>>>>>right'\n" - ' >>>\n' - ' >>> octets = [192, 168, 0, 1]\n' - " >>> '{:02X}{:02X}{:02X}{:02X}'.format(*octets)\n" - " 'C0A80001'\n" - ' >>> int(_, 16)\n' - ' 3232235521\n' - ' >>>\n' - ' >>> width = 5\n' - ' >>> for num in range(5,12): \n' - " ... for base in 'dXob':\n" - " ... print('{0:{width}{base}}'.format(num, " - "base=base, width=width), end=' ')\n" - ' ... print()\n' - ' ...\n' - ' 5 5 5 101\n' - ' 6 6 6 110\n' - ' 7 7 7 111\n' - ' 8 8 10 1000\n' - ' 9 9 11 1001\n' - ' 10 A 12 1010\n' - ' 11 B 13 1011\n', - 'function': 'Function definitions\n' - '********************\n' - '\n' - 'A function definition defines a user-defined function object ' - '(see\n' - 'section The standard type hierarchy):\n' - '\n' - ' funcdef ::= [decorators] "def" funcname ' - '[type_params] "(" [parameter_list] ")"\n' - ' ["->" expression] ":" suite\n' - ' decorators ::= decorator+\n' - ' decorator ::= "@" assignment_expression ' - 'NEWLINE\n' - ' parameter_list ::= defparameter ("," ' - 'defparameter)* "," "/" ["," [parameter_list_no_posonly]]\n' - ' | parameter_list_no_posonly\n' - ' parameter_list_no_posonly ::= defparameter ("," ' - 'defparameter)* ["," [parameter_list_starargs]]\n' - ' | parameter_list_starargs\n' - ' parameter_list_starargs ::= "*" [parameter] ("," ' - 'defparameter)* ["," ["**" parameter [","]]]\n' - ' | "**" parameter [","]\n' - ' parameter ::= identifier [":" expression]\n' - ' defparameter ::= parameter ["=" expression]\n' - ' funcname ::= identifier\n' - '\n' - 'A function definition is an executable statement. Its execution ' - 'binds\n' - 'the function name in the current local namespace to a function ' - 'object\n' - '(a wrapper around the executable code for the function). This\n' - 'function object contains a reference to the current global ' - 'namespace\n' - 'as the global namespace to be used when the function is called.\n' - '\n' - 'The function definition does not execute the function body; this ' - 'gets\n' - 'executed only when the function is called. [4]\n' - '\n' - 'A function definition may be wrapped by one or more *decorator*\n' - 'expressions. Decorator expressions are evaluated when the ' - 'function is\n' - 'defined, in the scope that contains the function definition. ' - 'The\n' - 'result must be a callable, which is invoked with the function ' - 'object\n' - 'as the only argument. The returned value is bound to the ' - 'function name\n' - 'instead of the function object. Multiple decorators are applied ' - 'in\n' - 'nested fashion. For example, the following code\n' - '\n' - ' @f1(arg)\n' - ' @f2\n' - ' def func(): pass\n' - '\n' - 'is roughly equivalent to\n' - '\n' - ' def func(): pass\n' - ' func = f1(arg)(f2(func))\n' - '\n' - 'except that the original function is not temporarily bound to ' - 'the name\n' - '"func".\n' - '\n' - 'Changed in version 3.9: Functions may be decorated with any ' - 'valid\n' - '"assignment_expression". Previously, the grammar was much more\n' - 'restrictive; see **PEP 614** for details.\n' - '\n' - 'A list of type parameters may be given in square brackets ' - 'between the\n' - 'function’s name and the opening parenthesis for its parameter ' - 'list.\n' - 'This indicates to static type checkers that the function is ' - 'generic.\n' - 'At runtime, the type parameters can be retrieved from the ' - 'function’s\n' - '"__type_params__" attribute. See Generic functions for more.\n' - '\n' - 'Changed in version 3.12: Type parameter lists are new in Python ' - '3.12.\n' - '\n' - 'When one or more *parameters* have the form *parameter* "="\n' - '*expression*, the function is said to have “default parameter ' - 'values.”\n' - 'For a parameter with a default value, the corresponding ' - '*argument* may\n' - 'be omitted from a call, in which case the parameter’s default ' - 'value is\n' - 'substituted. If a parameter has a default value, all following\n' - 'parameters up until the “"*"” must also have a default value — ' - 'this is\n' - 'a syntactic restriction that is not expressed by the grammar.\n' - '\n' - '**Default parameter values are evaluated from left to right when ' - 'the\n' - 'function definition is executed.** This means that the ' - 'expression is\n' - 'evaluated once, when the function is defined, and that the same ' - '“pre-\n' - 'computed” value is used for each call. This is especially ' - 'important\n' - 'to understand when a default parameter value is a mutable ' - 'object, such\n' - 'as a list or a dictionary: if the function modifies the object ' - '(e.g.\n' - 'by appending an item to a list), the default parameter value is ' - 'in\n' - 'effect modified. This is generally not what was intended. A ' - 'way\n' - 'around this is to use "None" as the default, and explicitly test ' - 'for\n' - 'it in the body of the function, e.g.:\n' - '\n' - ' def whats_on_the_telly(penguin=None):\n' - ' if penguin is None:\n' - ' penguin = []\n' - ' penguin.append("property of the zoo")\n' - ' return penguin\n' - '\n' - 'Function call semantics are described in more detail in section ' - 'Calls.\n' - 'A function call always assigns values to all parameters ' - 'mentioned in\n' - 'the parameter list, either from positional arguments, from ' - 'keyword\n' - 'arguments, or from default values. If the form “"*identifier"” ' - 'is\n' - 'present, it is initialized to a tuple receiving any excess ' - 'positional\n' - 'parameters, defaulting to the empty tuple. If the form\n' - '“"**identifier"” is present, it is initialized to a new ordered\n' - 'mapping receiving any excess keyword arguments, defaulting to a ' - 'new\n' - 'empty mapping of the same type. Parameters after “"*"” or\n' - '“"*identifier"” are keyword-only parameters and may only be ' - 'passed by\n' - 'keyword arguments. Parameters before “"/"” are positional-only\n' - 'parameters and may only be passed by positional arguments.\n' - '\n' - 'Changed in version 3.8: The "/" function parameter syntax may be ' - 'used\n' - 'to indicate positional-only parameters. See **PEP 570** for ' - 'details.\n' - '\n' - 'Parameters may have an *annotation* of the form “": ' - 'expression"”\n' - 'following the parameter name. Any parameter may have an ' - 'annotation,\n' - 'even those of the form "*identifier" or "**identifier". ' - 'Functions may\n' - 'have “return” annotation of the form “"-> expression"” after ' - 'the\n' - 'parameter list. These annotations can be any valid Python ' - 'expression.\n' - 'The presence of annotations does not change the semantics of a\n' - 'function. The annotation values are available as values of a\n' - 'dictionary keyed by the parameters’ names in the ' - '"__annotations__"\n' - 'attribute of the function object. If the "annotations" import ' - 'from\n' - '"__future__" is used, annotations are preserved as strings at ' - 'runtime\n' - 'which enables postponed evaluation. Otherwise, they are ' - 'evaluated\n' - 'when the function definition is executed. In this case ' - 'annotations\n' - 'may be evaluated in a different order than they appear in the ' - 'source\n' - 'code.\n' - '\n' - 'It is also possible to create anonymous functions (functions not ' - 'bound\n' - 'to a name), for immediate use in expressions. This uses lambda\n' - 'expressions, described in section Lambdas. Note that the ' - 'lambda\n' - 'expression is merely a shorthand for a simplified function ' - 'definition;\n' - 'a function defined in a “"def"” statement can be passed around ' - 'or\n' - 'assigned to another name just like a function defined by a ' - 'lambda\n' - 'expression. The “"def"” form is actually more powerful since ' - 'it\n' - 'allows the execution of multiple statements and annotations.\n' - '\n' - '**Programmer’s note:** Functions are first-class objects. A ' - '“"def"”\n' - 'statement executed inside a function definition defines a local\n' - 'function that can be returned or passed around. Free variables ' - 'used\n' - 'in the nested function can access the local variables of the ' - 'function\n' - 'containing the def. See section Naming and binding for ' - 'details.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3107** - Function Annotations\n' - ' The original specification for function annotations.\n' - '\n' - ' **PEP 484** - Type Hints\n' - ' Definition of a standard meaning for annotations: type ' - 'hints.\n' - '\n' - ' **PEP 526** - Syntax for Variable Annotations\n' - ' Ability to type hint variable declarations, including ' - 'class\n' - ' variables and instance variables.\n' - '\n' - ' **PEP 563** - Postponed Evaluation of Annotations\n' - ' Support for forward references within annotations by ' - 'preserving\n' - ' annotations in a string form at runtime instead of eager\n' - ' evaluation.\n' - '\n' - ' **PEP 318** - Decorators for Functions and Methods\n' - ' Function and method decorators were introduced. Class ' - 'decorators\n' - ' were introduced in **PEP 3129**.\n', - 'global': 'The "global" statement\n' - '**********************\n' - '\n' - ' global_stmt ::= "global" identifier ("," identifier)*\n' - '\n' - 'The "global" statement is a declaration which holds for the ' - 'entire\n' - 'current code block. It means that the listed identifiers are to ' - 'be\n' - 'interpreted as globals. It would be impossible to assign to a ' - 'global\n' - 'variable without "global", although free variables may refer to\n' - 'globals without being declared global.\n' - '\n' - 'Names listed in a "global" statement must not be used in the same ' - 'code\n' - 'block textually preceding that "global" statement.\n' - '\n' - 'Names listed in a "global" statement must not be defined as ' - 'formal\n' - 'parameters, or as targets in "with" statements or "except" ' - 'clauses, or\n' - 'in a "for" target list, "class" definition, function definition,\n' - '"import" statement, or variable annotation.\n' - '\n' - '**CPython implementation detail:** The current implementation does ' - 'not\n' - 'enforce some of these restrictions, but programs should not abuse ' - 'this\n' - 'freedom, as future implementations may enforce them or silently ' - 'change\n' - 'the meaning of the program.\n' - '\n' - '**Programmer’s note:** "global" is a directive to the parser. It\n' - 'applies only to code parsed at the same time as the "global"\n' - 'statement. In particular, a "global" statement contained in a ' - 'string\n' - 'or code object supplied to the built-in "exec()" function does ' - 'not\n' - 'affect the code block *containing* the function call, and code\n' - 'contained in such a string is unaffected by "global" statements in ' - 'the\n' - 'code containing the function call. The same applies to the ' - '"eval()"\n' - 'and "compile()" functions.\n', - 'id-classes': 'Reserved classes of identifiers\n' - '*******************************\n' - '\n' - 'Certain classes of identifiers (besides keywords) have ' - 'special\n' - 'meanings. These classes are identified by the patterns of ' - 'leading and\n' - 'trailing underscore characters:\n' - '\n' - '"_*"\n' - ' Not imported by "from module import *".\n' - '\n' - '"_"\n' - ' In a "case" pattern within a "match" statement, "_" is a ' - 'soft\n' - ' keyword that denotes a wildcard.\n' - '\n' - ' Separately, the interactive interpreter makes the result of ' - 'the\n' - ' last evaluation available in the variable "_". (It is ' - 'stored in the\n' - ' "builtins" module, alongside built-in functions like ' - '"print".)\n' - '\n' - ' Elsewhere, "_" is a regular identifier. It is often used to ' - 'name\n' - ' “special” items, but it is not special to Python itself.\n' - '\n' - ' Note:\n' - '\n' - ' The name "_" is often used in conjunction with\n' - ' internationalization; refer to the documentation for the\n' - ' "gettext" module for more information on this ' - 'convention.It is\n' - ' also commonly used for unused variables.\n' - '\n' - '"__*__"\n' - ' System-defined names, informally known as “dunder” names. ' - 'These\n' - ' names are defined by the interpreter and its ' - 'implementation\n' - ' (including the standard library). Current system names are\n' - ' discussed in the Special method names section and ' - 'elsewhere. More\n' - ' will likely be defined in future versions of Python. *Any* ' - 'use of\n' - ' "__*__" names, in any context, that does not follow ' - 'explicitly\n' - ' documented use, is subject to breakage without warning.\n' - '\n' - '"__*"\n' - ' Class-private names. Names in this category, when used ' - 'within the\n' - ' context of a class definition, are re-written to use a ' - 'mangled form\n' - ' to help avoid name clashes between “private” attributes of ' - 'base and\n' - ' derived classes. See section Identifiers (Names).\n', - 'identifiers': 'Identifiers and keywords\n' - '************************\n' - '\n' - 'Identifiers (also referred to as *names*) are described by ' - 'the\n' - 'following lexical definitions.\n' - '\n' - 'The syntax of identifiers in Python is based on the Unicode ' - 'standard\n' - 'annex UAX-31, with elaboration and changes as defined below; ' - 'see also\n' - '**PEP 3131** for further details.\n' - '\n' - 'Within the ASCII range (U+0001..U+007F), the valid characters ' - 'for\n' - 'identifiers are the same as in Python 2.x: the uppercase and ' - 'lowercase\n' - 'letters "A" through "Z", the underscore "_" and, except for ' - 'the first\n' - 'character, the digits "0" through "9".\n' - '\n' - 'Python 3.0 introduces additional characters from outside the ' - 'ASCII\n' - 'range (see **PEP 3131**). For these characters, the ' - 'classification\n' - 'uses the version of the Unicode Character Database as ' - 'included in the\n' - '"unicodedata" module.\n' - '\n' - 'Identifiers are unlimited in length. Case is significant.\n' - '\n' - ' identifier ::= xid_start xid_continue*\n' - ' id_start ::= \n' - ' id_continue ::= \n' - ' xid_start ::= \n' - ' xid_continue ::= \n' - '\n' - 'The Unicode category codes mentioned above stand for:\n' - '\n' - '* *Lu* - uppercase letters\n' - '\n' - '* *Ll* - lowercase letters\n' - '\n' - '* *Lt* - titlecase letters\n' - '\n' - '* *Lm* - modifier letters\n' - '\n' - '* *Lo* - other letters\n' - '\n' - '* *Nl* - letter numbers\n' - '\n' - '* *Mn* - nonspacing marks\n' - '\n' - '* *Mc* - spacing combining marks\n' - '\n' - '* *Nd* - decimal numbers\n' - '\n' - '* *Pc* - connector punctuations\n' - '\n' - '* *Other_ID_Start* - explicit list of characters in ' - 'PropList.txt to\n' - ' support backwards compatibility\n' - '\n' - '* *Other_ID_Continue* - likewise\n' - '\n' - 'All identifiers are converted into the normal form NFKC while ' - 'parsing;\n' - 'comparison of identifiers is based on NFKC.\n' - '\n' - 'A non-normative HTML file listing all valid identifier ' - 'characters for\n' - 'Unicode 15.1.0 can be found at\n' - 'https://www.unicode.org/Public/15.1.0/ucd/DerivedCoreProperties.txt\n' - '\n' - '\n' - 'Keywords\n' - '========\n' - '\n' - 'The following identifiers are used as reserved words, or ' - '*keywords* of\n' - 'the language, and cannot be used as ordinary identifiers. ' - 'They must\n' - 'be spelled exactly as written here:\n' - '\n' - ' False await else import pass\n' - ' None break except in raise\n' - ' True class finally is return\n' - ' and continue for lambda try\n' - ' as def from nonlocal while\n' - ' assert del global not with\n' - ' async elif if or yield\n' - '\n' - '\n' - 'Soft Keywords\n' - '=============\n' - '\n' - 'New in version 3.10.\n' - '\n' - 'Some identifiers are only reserved under specific contexts. ' - 'These are\n' - 'known as *soft keywords*. The identifiers "match", "case", ' - '"type" and\n' - '"_" can syntactically act as keywords in certain contexts, ' - 'but this\n' - 'distinction is done at the parser level, not when ' - 'tokenizing.\n' - '\n' - 'As soft keywords, their use in the grammar is possible while ' - 'still\n' - 'preserving compatibility with existing code that uses these ' - 'names as\n' - 'identifier names.\n' - '\n' - '"match", "case", and "_" are used in the "match" statement. ' - '"type" is\n' - 'used in the "type" statement.\n' - '\n' - 'Changed in version 3.12: "type" is now a soft keyword.\n' - '\n' - '\n' - 'Reserved classes of identifiers\n' - '===============================\n' - '\n' - 'Certain classes of identifiers (besides keywords) have ' - 'special\n' - 'meanings. These classes are identified by the patterns of ' - 'leading and\n' - 'trailing underscore characters:\n' - '\n' - '"_*"\n' - ' Not imported by "from module import *".\n' - '\n' - '"_"\n' - ' In a "case" pattern within a "match" statement, "_" is a ' - 'soft\n' - ' keyword that denotes a wildcard.\n' - '\n' - ' Separately, the interactive interpreter makes the result ' - 'of the\n' - ' last evaluation available in the variable "_". (It is ' - 'stored in the\n' - ' "builtins" module, alongside built-in functions like ' - '"print".)\n' - '\n' - ' Elsewhere, "_" is a regular identifier. It is often used ' - 'to name\n' - ' “special” items, but it is not special to Python itself.\n' - '\n' - ' Note:\n' - '\n' - ' The name "_" is often used in conjunction with\n' - ' internationalization; refer to the documentation for ' - 'the\n' - ' "gettext" module for more information on this ' - 'convention.It is\n' - ' also commonly used for unused variables.\n' - '\n' - '"__*__"\n' - ' System-defined names, informally known as “dunder” names. ' - 'These\n' - ' names are defined by the interpreter and its ' - 'implementation\n' - ' (including the standard library). Current system names ' - 'are\n' - ' discussed in the Special method names section and ' - 'elsewhere. More\n' - ' will likely be defined in future versions of Python. ' - '*Any* use of\n' - ' "__*__" names, in any context, that does not follow ' - 'explicitly\n' - ' documented use, is subject to breakage without warning.\n' - '\n' - '"__*"\n' - ' Class-private names. Names in this category, when used ' - 'within the\n' - ' context of a class definition, are re-written to use a ' - 'mangled form\n' - ' to help avoid name clashes between “private” attributes of ' - 'base and\n' - ' derived classes. See section Identifiers (Names).\n', - 'if': 'The "if" statement\n' - '******************\n' - '\n' - 'The "if" statement is used for conditional execution:\n' - '\n' - ' if_stmt ::= "if" assignment_expression ":" suite\n' - ' ("elif" assignment_expression ":" suite)*\n' - ' ["else" ":" suite]\n' - '\n' - 'It selects exactly one of the suites by evaluating the expressions ' - 'one\n' - 'by one until one is found to be true (see section Boolean operations\n' - 'for the definition of true and false); then that suite is executed\n' - '(and no other part of the "if" statement is executed or evaluated).\n' - 'If all expressions are false, the suite of the "else" clause, if\n' - 'present, is executed.\n', - 'imaginary': 'Imaginary literals\n' - '******************\n' - '\n' - 'Imaginary literals are described by the following lexical ' - 'definitions:\n' - '\n' - ' imagnumber ::= (floatnumber | digitpart) ("j" | "J")\n' - '\n' - 'An imaginary literal yields a complex number with a real part ' - 'of 0.0.\n' - 'Complex numbers are represented as a pair of floating point ' - 'numbers\n' - 'and have the same restrictions on their range. To create a ' - 'complex\n' - 'number with a nonzero real part, add a floating point number to ' - 'it,\n' - 'e.g., "(3+4j)". Some examples of imaginary literals:\n' - '\n' - ' 3.14j 10.j 10j .001j 1e100j 3.14e-10j ' - '3.14_15_93j\n', - 'import': 'The "import" statement\n' - '**********************\n' - '\n' - ' import_stmt ::= "import" module ["as" identifier] ("," ' - 'module ["as" identifier])*\n' - ' | "from" relative_module "import" identifier ' - '["as" identifier]\n' - ' ("," identifier ["as" identifier])*\n' - ' | "from" relative_module "import" "(" ' - 'identifier ["as" identifier]\n' - ' ("," identifier ["as" identifier])* [","] ")"\n' - ' | "from" relative_module "import" "*"\n' - ' module ::= (identifier ".")* identifier\n' - ' relative_module ::= "."* module | "."+\n' - '\n' - 'The basic import statement (no "from" clause) is executed in two\n' - 'steps:\n' - '\n' - '1. find a module, loading and initializing it if necessary\n' - '\n' - '2. define a name or names in the local namespace for the scope ' - 'where\n' - ' the "import" statement occurs.\n' - '\n' - 'When the statement contains multiple clauses (separated by commas) ' - 'the\n' - 'two steps are carried out separately for each clause, just as ' - 'though\n' - 'the clauses had been separated out into individual import ' - 'statements.\n' - '\n' - 'The details of the first step, finding and loading modules, are\n' - 'described in greater detail in the section on the import system, ' - 'which\n' - 'also describes the various types of packages and modules that can ' - 'be\n' - 'imported, as well as all the hooks that can be used to customize ' - 'the\n' - 'import system. Note that failures in this step may indicate ' - 'either\n' - 'that the module could not be located, *or* that an error occurred\n' - 'while initializing the module, which includes execution of the\n' - 'module’s code.\n' - '\n' - 'If the requested module is retrieved successfully, it will be ' - 'made\n' - 'available in the local namespace in one of three ways:\n' - '\n' - '* If the module name is followed by "as", then the name following ' - '"as"\n' - ' is bound directly to the imported module.\n' - '\n' - '* If no other name is specified, and the module being imported is ' - 'a\n' - ' top level module, the module’s name is bound in the local ' - 'namespace\n' - ' as a reference to the imported module\n' - '\n' - '* If the module being imported is *not* a top level module, then ' - 'the\n' - ' name of the top level package that contains the module is bound ' - 'in\n' - ' the local namespace as a reference to the top level package. ' - 'The\n' - ' imported module must be accessed using its full qualified name\n' - ' rather than directly\n' - '\n' - 'The "from" form uses a slightly more complex process:\n' - '\n' - '1. find the module specified in the "from" clause, loading and\n' - ' initializing it if necessary;\n' - '\n' - '2. for each of the identifiers specified in the "import" clauses:\n' - '\n' - ' 1. check if the imported module has an attribute by that name\n' - '\n' - ' 2. if not, attempt to import a submodule with that name and ' - 'then\n' - ' check the imported module again for that attribute\n' - '\n' - ' 3. if the attribute is not found, "ImportError" is raised.\n' - '\n' - ' 4. otherwise, a reference to that value is stored in the local\n' - ' namespace, using the name in the "as" clause if it is ' - 'present,\n' - ' otherwise using the attribute name\n' - '\n' - 'Examples:\n' - '\n' - ' import foo # foo imported and bound locally\n' - ' import foo.bar.baz # foo, foo.bar, and foo.bar.baz ' - 'imported, foo bound locally\n' - ' import foo.bar.baz as fbb # foo, foo.bar, and foo.bar.baz ' - 'imported, foo.bar.baz bound as fbb\n' - ' from foo.bar import baz # foo, foo.bar, and foo.bar.baz ' - 'imported, foo.bar.baz bound as baz\n' - ' from foo import attr # foo imported and foo.attr bound as ' - 'attr\n' - '\n' - 'If the list of identifiers is replaced by a star ("\'*\'"), all ' - 'public\n' - 'names defined in the module are bound in the local namespace for ' - 'the\n' - 'scope where the "import" statement occurs.\n' - '\n' - 'The *public names* defined by a module are determined by checking ' - 'the\n' - 'module’s namespace for a variable named "__all__"; if defined, it ' - 'must\n' - 'be a sequence of strings which are names defined or imported by ' - 'that\n' - 'module. The names given in "__all__" are all considered public ' - 'and\n' - 'are required to exist. If "__all__" is not defined, the set of ' - 'public\n' - 'names includes all names found in the module’s namespace which do ' - 'not\n' - 'begin with an underscore character ("\'_\'"). "__all__" should ' - 'contain\n' - 'the entire public API. It is intended to avoid accidentally ' - 'exporting\n' - 'items that are not part of the API (such as library modules which ' - 'were\n' - 'imported and used within the module).\n' - '\n' - 'The wild card form of import — "from module import *" — is only\n' - 'allowed at the module level. Attempting to use it in class or\n' - 'function definitions will raise a "SyntaxError".\n' - '\n' - 'When specifying what module to import you do not have to specify ' - 'the\n' - 'absolute name of the module. When a module or package is ' - 'contained\n' - 'within another package it is possible to make a relative import ' - 'within\n' - 'the same top package without having to mention the package name. ' - 'By\n' - 'using leading dots in the specified module or package after "from" ' - 'you\n' - 'can specify how high to traverse up the current package hierarchy\n' - 'without specifying exact names. One leading dot means the current\n' - 'package where the module making the import exists. Two dots means ' - 'up\n' - 'one package level. Three dots is up two levels, etc. So if you ' - 'execute\n' - '"from . import mod" from a module in the "pkg" package then you ' - 'will\n' - 'end up importing "pkg.mod". If you execute "from ..subpkg2 import ' - 'mod"\n' - 'from within "pkg.subpkg1" you will import "pkg.subpkg2.mod". The\n' - 'specification for relative imports is contained in the Package\n' - 'Relative Imports section.\n' - '\n' - '"importlib.import_module()" is provided to support applications ' - 'that\n' - 'determine dynamically the modules to be loaded.\n' - '\n' - 'Raises an auditing event "import" with arguments "module", ' - '"filename",\n' - '"sys.path", "sys.meta_path", "sys.path_hooks".\n' - '\n' - '\n' - 'Future statements\n' - '=================\n' - '\n' - 'A *future statement* is a directive to the compiler that a ' - 'particular\n' - 'module should be compiled using syntax or semantics that will be\n' - 'available in a specified future release of Python where the ' - 'feature\n' - 'becomes standard.\n' - '\n' - 'The future statement is intended to ease migration to future ' - 'versions\n' - 'of Python that introduce incompatible changes to the language. ' - 'It\n' - 'allows use of the new features on a per-module basis before the\n' - 'release in which the feature becomes standard.\n' - '\n' - ' future_stmt ::= "from" "__future__" "import" feature ["as" ' - 'identifier]\n' - ' ("," feature ["as" identifier])*\n' - ' | "from" "__future__" "import" "(" feature ' - '["as" identifier]\n' - ' ("," feature ["as" identifier])* [","] ")"\n' - ' feature ::= identifier\n' - '\n' - 'A future statement must appear near the top of the module. The ' - 'only\n' - 'lines that can appear before a future statement are:\n' - '\n' - '* the module docstring (if any),\n' - '\n' - '* comments,\n' - '\n' - '* blank lines, and\n' - '\n' - '* other future statements.\n' - '\n' - 'The only feature that requires using the future statement is\n' - '"annotations" (see **PEP 563**).\n' - '\n' - 'All historical features enabled by the future statement are still\n' - 'recognized by Python 3. The list includes "absolute_import",\n' - '"division", "generators", "generator_stop", "unicode_literals",\n' - '"print_function", "nested_scopes" and "with_statement". They are ' - 'all\n' - 'redundant because they are always enabled, and only kept for ' - 'backwards\n' - 'compatibility.\n' - '\n' - 'A future statement is recognized and treated specially at compile\n' - 'time: Changes to the semantics of core constructs are often\n' - 'implemented by generating different code. It may even be the ' - 'case\n' - 'that a new feature introduces new incompatible syntax (such as a ' - 'new\n' - 'reserved word), in which case the compiler may need to parse the\n' - 'module differently. Such decisions cannot be pushed off until\n' - 'runtime.\n' - '\n' - 'For any given release, the compiler knows which feature names ' - 'have\n' - 'been defined, and raises a compile-time error if a future ' - 'statement\n' - 'contains a feature not known to it.\n' - '\n' - 'The direct runtime semantics are the same as for any import ' - 'statement:\n' - 'there is a standard module "__future__", described later, and it ' - 'will\n' - 'be imported in the usual way at the time the future statement is\n' - 'executed.\n' - '\n' - 'The interesting runtime semantics depend on the specific feature\n' - 'enabled by the future statement.\n' - '\n' - 'Note that there is nothing special about the statement:\n' - '\n' - ' import __future__ [as name]\n' - '\n' - 'That is not a future statement; it’s an ordinary import statement ' - 'with\n' - 'no special semantics or syntax restrictions.\n' - '\n' - 'Code compiled by calls to the built-in functions "exec()" and\n' - '"compile()" that occur in a module "M" containing a future ' - 'statement\n' - 'will, by default, use the new syntax or semantics associated with ' - 'the\n' - 'future statement. This can be controlled by optional arguments ' - 'to\n' - '"compile()" — see the documentation of that function for details.\n' - '\n' - 'A future statement typed at an interactive interpreter prompt ' - 'will\n' - 'take effect for the rest of the interpreter session. If an\n' - 'interpreter is started with the "-i" option, is passed a script ' - 'name\n' - 'to execute, and the script includes a future statement, it will be ' - 'in\n' - 'effect in the interactive session started after the script is\n' - 'executed.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 236** - Back to the __future__\n' - ' The original proposal for the __future__ mechanism.\n', - 'in': 'Membership test operations\n' - '**************************\n' - '\n' - 'The operators "in" and "not in" test for membership. "x in s"\n' - 'evaluates to "True" if *x* is a member of *s*, and "False" otherwise.\n' - '"x not in s" returns the negation of "x in s". All built-in ' - 'sequences\n' - 'and set types support this as well as dictionary, for which "in" ' - 'tests\n' - 'whether the dictionary has a given key. For container types such as\n' - 'list, tuple, set, frozenset, dict, or collections.deque, the\n' - 'expression "x in y" is equivalent to "any(x is e or x == e for e in\n' - 'y)".\n' - '\n' - 'For the string and bytes types, "x in y" is "True" if and only if *x*\n' - 'is a substring of *y*. An equivalent test is "y.find(x) != -1".\n' - 'Empty strings are always considered to be a substring of any other\n' - 'string, so """ in "abc"" will return "True".\n' - '\n' - 'For user-defined classes which define the "__contains__()" method, "x\n' - 'in y" returns "True" if "y.__contains__(x)" returns a true value, and\n' - '"False" otherwise.\n' - '\n' - 'For user-defined classes which do not define "__contains__()" but do\n' - 'define "__iter__()", "x in y" is "True" if some value "z", for which\n' - 'the expression "x is z or x == z" is true, is produced while ' - 'iterating\n' - 'over "y". If an exception is raised during the iteration, it is as if\n' - '"in" raised that exception.\n' - '\n' - 'Lastly, the old-style iteration protocol is tried: if a class defines\n' - '"__getitem__()", "x in y" is "True" if and only if there is a non-\n' - 'negative integer index *i* such that "x is y[i] or x == y[i]", and no\n' - 'lower integer index raises the "IndexError" exception. (If any other\n' - 'exception is raised, it is as if "in" raised that exception).\n' - '\n' - 'The operator "not in" is defined to have the inverse truth value of\n' - '"in".\n', - 'integers': 'Integer literals\n' - '****************\n' - '\n' - 'Integer literals are described by the following lexical ' - 'definitions:\n' - '\n' - ' integer ::= decinteger | bininteger | octinteger | ' - 'hexinteger\n' - ' decinteger ::= nonzerodigit (["_"] digit)* | "0"+ (["_"] ' - '"0")*\n' - ' bininteger ::= "0" ("b" | "B") (["_"] bindigit)+\n' - ' octinteger ::= "0" ("o" | "O") (["_"] octdigit)+\n' - ' hexinteger ::= "0" ("x" | "X") (["_"] hexdigit)+\n' - ' nonzerodigit ::= "1"..."9"\n' - ' digit ::= "0"..."9"\n' - ' bindigit ::= "0" | "1"\n' - ' octdigit ::= "0"..."7"\n' - ' hexdigit ::= digit | "a"..."f" | "A"..."F"\n' - '\n' - 'There is no limit for the length of integer literals apart from ' - 'what\n' - 'can be stored in available memory.\n' - '\n' - 'Underscores are ignored for determining the numeric value of ' - 'the\n' - 'literal. They can be used to group digits for enhanced ' - 'readability.\n' - 'One underscore can occur between digits, and after base ' - 'specifiers\n' - 'like "0x".\n' - '\n' - 'Note that leading zeros in a non-zero decimal number are not ' - 'allowed.\n' - 'This is for disambiguation with C-style octal literals, which ' - 'Python\n' - 'used before version 3.0.\n' - '\n' - 'Some examples of integer literals:\n' - '\n' - ' 7 2147483647 0o177 0b100110111\n' - ' 3 79228162514264337593543950336 0o377 0xdeadbeef\n' - ' 100_000_000_000 0b_1110_0101\n' - '\n' - 'Changed in version 3.6: Underscores are now allowed for ' - 'grouping\n' - 'purposes in literals.\n', - 'lambda': 'Lambdas\n' - '*******\n' - '\n' - ' lambda_expr ::= "lambda" [parameter_list] ":" expression\n' - '\n' - 'Lambda expressions (sometimes called lambda forms) are used to ' - 'create\n' - 'anonymous functions. The expression "lambda parameters: ' - 'expression"\n' - 'yields a function object. The unnamed object behaves like a ' - 'function\n' - 'object defined with:\n' - '\n' - ' def (parameters):\n' - ' return expression\n' - '\n' - 'See section Function definitions for the syntax of parameter ' - 'lists.\n' - 'Note that functions created with lambda expressions cannot ' - 'contain\n' - 'statements or annotations.\n', - 'lists': 'List displays\n' - '*************\n' - '\n' - 'A list display is a possibly empty series of expressions enclosed ' - 'in\n' - 'square brackets:\n' - '\n' - ' list_display ::= "[" [starred_list | comprehension] "]"\n' - '\n' - 'A list display yields a new list object, the contents being ' - 'specified\n' - 'by either a list of expressions or a comprehension. When a comma-\n' - 'separated list of expressions is supplied, its elements are ' - 'evaluated\n' - 'from left to right and placed into the list object in that order.\n' - 'When a comprehension is supplied, the list is constructed from the\n' - 'elements resulting from the comprehension.\n', - 'naming': 'Naming and binding\n' - '******************\n' - '\n' - '\n' - 'Binding of names\n' - '================\n' - '\n' - '*Names* refer to objects. Names are introduced by name binding\n' - 'operations.\n' - '\n' - 'The following constructs bind names:\n' - '\n' - '* formal parameters to functions,\n' - '\n' - '* class definitions,\n' - '\n' - '* function definitions,\n' - '\n' - '* assignment expressions,\n' - '\n' - '* targets that are identifiers if occurring in an assignment:\n' - '\n' - ' * "for" loop header,\n' - '\n' - ' * after "as" in a "with" statement, "except" clause, "except*"\n' - ' clause, or in the as-pattern in structural pattern matching,\n' - '\n' - ' * in a capture pattern in structural pattern matching\n' - '\n' - '* "import" statements.\n' - '\n' - '* "type" statements.\n' - '\n' - '* type parameter lists.\n' - '\n' - 'The "import" statement of the form "from ... import *" binds all ' - 'names\n' - 'defined in the imported module, except those beginning with an\n' - 'underscore. This form may only be used at the module level.\n' - '\n' - 'A target occurring in a "del" statement is also considered bound ' - 'for\n' - 'this purpose (though the actual semantics are to unbind the ' - 'name).\n' - '\n' - 'Each assignment or import statement occurs within a block defined ' - 'by a\n' - 'class or function definition or at the module level (the ' - 'top-level\n' - 'code block).\n' - '\n' - 'If a name is bound in a block, it is a local variable of that ' - 'block,\n' - 'unless declared as "nonlocal" or "global". If a name is bound at ' - 'the\n' - 'module level, it is a global variable. (The variables of the ' - 'module\n' - 'code block are local and global.) If a variable is used in a ' - 'code\n' - 'block but not defined there, it is a *free variable*.\n' - '\n' - 'Each occurrence of a name in the program text refers to the ' - '*binding*\n' - 'of that name established by the following name resolution rules.\n' - '\n' - '\n' - 'Resolution of names\n' - '===================\n' - '\n' - 'A *scope* defines the visibility of a name within a block. If a ' - 'local\n' - 'variable is defined in a block, its scope includes that block. If ' - 'the\n' - 'definition occurs in a function block, the scope extends to any ' - 'blocks\n' - 'contained within the defining one, unless a contained block ' - 'introduces\n' - 'a different binding for the name.\n' - '\n' - 'When a name is used in a code block, it is resolved using the ' - 'nearest\n' - 'enclosing scope. The set of all such scopes visible to a code ' - 'block\n' - 'is called the block’s *environment*.\n' - '\n' - 'When a name is not found at all, a "NameError" exception is ' - 'raised. If\n' - 'the current scope is a function scope, and the name refers to a ' - 'local\n' - 'variable that has not yet been bound to a value at the point where ' - 'the\n' - 'name is used, an "UnboundLocalError" exception is raised.\n' - '"UnboundLocalError" is a subclass of "NameError".\n' - '\n' - 'If a name binding operation occurs anywhere within a code block, ' - 'all\n' - 'uses of the name within the block are treated as references to ' - 'the\n' - 'current block. This can lead to errors when a name is used within ' - 'a\n' - 'block before it is bound. This rule is subtle. Python lacks\n' - 'declarations and allows name binding operations to occur anywhere\n' - 'within a code block. The local variables of a code block can be\n' - 'determined by scanning the entire text of the block for name ' - 'binding\n' - 'operations. See the FAQ entry on UnboundLocalError for examples.\n' - '\n' - 'If the "global" statement occurs within a block, all uses of the ' - 'names\n' - 'specified in the statement refer to the bindings of those names in ' - 'the\n' - 'top-level namespace. Names are resolved in the top-level ' - 'namespace by\n' - 'searching the global namespace, i.e. the namespace of the module\n' - 'containing the code block, and the builtins namespace, the ' - 'namespace\n' - 'of the module "builtins". The global namespace is searched ' - 'first. If\n' - 'the names are not found there, the builtins namespace is ' - 'searched.\n' - 'The "global" statement must precede all uses of the listed names.\n' - '\n' - 'The "global" statement has the same scope as a name binding ' - 'operation\n' - 'in the same block. If the nearest enclosing scope for a free ' - 'variable\n' - 'contains a global statement, the free variable is treated as a ' - 'global.\n' - '\n' - 'The "nonlocal" statement causes corresponding names to refer to\n' - 'previously bound variables in the nearest enclosing function ' - 'scope.\n' - '"SyntaxError" is raised at compile time if the given name does ' - 'not\n' - 'exist in any enclosing function scope. Type parameters cannot be\n' - 'rebound with the "nonlocal" statement.\n' - '\n' - 'The namespace for a module is automatically created the first time ' - 'a\n' - 'module is imported. The main module for a script is always ' - 'called\n' - '"__main__".\n' - '\n' - 'Class definition blocks and arguments to "exec()" and "eval()" ' - 'are\n' - 'special in the context of name resolution. A class definition is ' - 'an\n' - 'executable statement that may use and define names. These ' - 'references\n' - 'follow the normal rules for name resolution with an exception ' - 'that\n' - 'unbound local variables are looked up in the global namespace. ' - 'The\n' - 'namespace of the class definition becomes the attribute dictionary ' - 'of\n' - 'the class. The scope of names defined in a class block is limited ' - 'to\n' - 'the class block; it does not extend to the code blocks of ' - 'methods.\n' - 'This includes comprehensions and generator expressions, but it ' - 'does\n' - 'not include annotation scopes, which have access to their ' - 'enclosing\n' - 'class scopes. This means that the following will fail:\n' - '\n' - ' class A:\n' - ' a = 42\n' - ' b = list(a + i for i in range(10))\n' - '\n' - 'However, the following will succeed:\n' - '\n' - ' class A:\n' - ' type Alias = Nested\n' - ' class Nested: pass\n' - '\n' - " print(A.Alias.__value__) # \n" - '\n' - '\n' - 'Annotation scopes\n' - '=================\n' - '\n' - 'Type parameter lists and "type" statements introduce *annotation\n' - 'scopes*, which behave mostly like function scopes, but with some\n' - 'exceptions discussed below. *Annotations* currently do not use\n' - 'annotation scopes, but they are expected to use annotation scopes ' - 'in\n' - 'Python 3.13 when **PEP 649** is implemented.\n' - '\n' - 'Annotation scopes are used in the following contexts:\n' - '\n' - '* Type parameter lists for generic type aliases.\n' - '\n' - '* Type parameter lists for generic functions. A generic ' - 'function’s\n' - ' annotations are executed within the annotation scope, but its\n' - ' defaults and decorators are not.\n' - '\n' - '* Type parameter lists for generic classes. A generic class’s ' - 'base\n' - ' classes and keyword arguments are executed within the ' - 'annotation\n' - ' scope, but its decorators are not.\n' - '\n' - '* The bounds and constraints for type variables (lazily ' - 'evaluated).\n' - '\n' - '* The value of type aliases (lazily evaluated).\n' - '\n' - 'Annotation scopes differ from function scopes in the following ' - 'ways:\n' - '\n' - '* Annotation scopes have access to their enclosing class ' - 'namespace. If\n' - ' an annotation scope is immediately within a class scope, or ' - 'within\n' - ' another annotation scope that is immediately within a class ' - 'scope,\n' - ' the code in the annotation scope can use names defined in the ' - 'class\n' - ' scope as if it were executed directly within the class body. ' - 'This\n' - ' contrasts with regular functions defined within classes, which\n' - ' cannot access names defined in the class scope.\n' - '\n' - '* Expressions in annotation scopes cannot contain "yield", "yield\n' - ' from", "await", or ":=" expressions. (These expressions are ' - 'allowed\n' - ' in other scopes contained within the annotation scope.)\n' - '\n' - '* Names defined in annotation scopes cannot be rebound with ' - '"nonlocal"\n' - ' statements in inner scopes. This includes only type parameters, ' - 'as\n' - ' no other syntactic elements that can appear within annotation ' - 'scopes\n' - ' can introduce new names.\n' - '\n' - '* While annotation scopes have an internal name, that name is not\n' - ' reflected in the *__qualname__* of objects defined within the ' - 'scope.\n' - ' Instead, the "__qualname__" of such objects is as if the object ' - 'were\n' - ' defined in the enclosing scope.\n' - '\n' - 'New in version 3.12: Annotation scopes were introduced in Python ' - '3.12\n' - 'as part of **PEP 695**.\n' - '\n' - '\n' - 'Lazy evaluation\n' - '===============\n' - '\n' - 'The values of type aliases created through the "type" statement ' - 'are\n' - '*lazily evaluated*. The same applies to the bounds and constraints ' - 'of\n' - 'type variables created through the type parameter syntax. This ' - 'means\n' - 'that they are not evaluated when the type alias or type variable ' - 'is\n' - 'created. Instead, they are only evaluated when doing so is ' - 'necessary\n' - 'to resolve an attribute access.\n' - '\n' - 'Example:\n' - '\n' - ' >>> type Alias = 1/0\n' - ' >>> Alias.__value__\n' - ' Traceback (most recent call last):\n' - ' ...\n' - ' ZeroDivisionError: division by zero\n' - ' >>> def func[T: 1/0](): pass\n' - ' >>> T = func.__type_params__[0]\n' - ' >>> T.__bound__\n' - ' Traceback (most recent call last):\n' - ' ...\n' - ' ZeroDivisionError: division by zero\n' - '\n' - 'Here the exception is raised only when the "__value__" attribute ' - 'of\n' - 'the type alias or the "__bound__" attribute of the type variable ' - 'is\n' - 'accessed.\n' - '\n' - 'This behavior is primarily useful for references to types that ' - 'have\n' - 'not yet been defined when the type alias or type variable is ' - 'created.\n' - 'For example, lazy evaluation enables creation of mutually ' - 'recursive\n' - 'type aliases:\n' - '\n' - ' from typing import Literal\n' - '\n' - ' type SimpleExpr = int | Parenthesized\n' - ' type Parenthesized = tuple[Literal["("], Expr, Literal[")"]]\n' - ' type Expr = SimpleExpr | tuple[SimpleExpr, Literal["+", "-"], ' - 'Expr]\n' - '\n' - 'Lazily evaluated values are evaluated in annotation scope, which ' - 'means\n' - 'that names that appear inside the lazily evaluated value are ' - 'looked up\n' - 'as if they were used in the immediately enclosing scope.\n' - '\n' - 'New in version 3.12.\n' - '\n' - '\n' - 'Builtins and restricted execution\n' - '=================================\n' - '\n' - '**CPython implementation detail:** Users should not touch\n' - '"__builtins__"; it is strictly an implementation detail. Users\n' - 'wanting to override values in the builtins namespace should ' - '"import"\n' - 'the "builtins" module and modify its attributes appropriately.\n' - '\n' - 'The builtins namespace associated with the execution of a code ' - 'block\n' - 'is actually found by looking up the name "__builtins__" in its ' - 'global\n' - 'namespace; this should be a dictionary or a module (in the latter ' - 'case\n' - 'the module’s dictionary is used). By default, when in the ' - '"__main__"\n' - 'module, "__builtins__" is the built-in module "builtins"; when in ' - 'any\n' - 'other module, "__builtins__" is an alias for the dictionary of ' - 'the\n' - '"builtins" module itself.\n' - '\n' - '\n' - 'Interaction with dynamic features\n' - '=================================\n' - '\n' - 'Name resolution of free variables occurs at runtime, not at ' - 'compile\n' - 'time. This means that the following code will print 42:\n' - '\n' - ' i = 10\n' - ' def f():\n' - ' print(i)\n' - ' i = 42\n' - ' f()\n' - '\n' - 'The "eval()" and "exec()" functions do not have access to the ' - 'full\n' - 'environment for resolving names. Names may be resolved in the ' - 'local\n' - 'and global namespaces of the caller. Free variables are not ' - 'resolved\n' - 'in the nearest enclosing namespace, but in the global namespace. ' - '[1]\n' - 'The "exec()" and "eval()" functions have optional arguments to\n' - 'override the global and local namespace. If only one namespace ' - 'is\n' - 'specified, it is used for both.\n', - 'nonlocal': 'The "nonlocal" statement\n' - '************************\n' - '\n' - ' nonlocal_stmt ::= "nonlocal" identifier ("," identifier)*\n' - '\n' - 'The "nonlocal" statement causes the listed identifiers to refer ' - 'to\n' - 'previously bound variables in the nearest enclosing scope ' - 'excluding\n' - 'globals. This is important because the default behavior for ' - 'binding is\n' - 'to search the local namespace first. The statement allows\n' - 'encapsulated code to rebind variables outside of the local ' - 'scope\n' - 'besides the global (module) scope.\n' - '\n' - 'Names listed in a "nonlocal" statement, unlike those listed in ' - 'a\n' - '"global" statement, must refer to pre-existing bindings in an\n' - 'enclosing scope (the scope in which a new binding should be ' - 'created\n' - 'cannot be determined unambiguously).\n' - '\n' - 'Names listed in a "nonlocal" statement must not collide with ' - 'pre-\n' - 'existing bindings in the local scope.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3104** - Access to Names in Outer Scopes\n' - ' The specification for the "nonlocal" statement.\n', - 'numbers': 'Numeric literals\n' - '****************\n' - '\n' - 'There are three types of numeric literals: integers, floating ' - 'point\n' - 'numbers, and imaginary numbers. There are no complex literals\n' - '(complex numbers can be formed by adding a real number and an\n' - 'imaginary number).\n' - '\n' - 'Note that numeric literals do not include a sign; a phrase like ' - '"-1"\n' - 'is actually an expression composed of the unary operator ‘"-"’ ' - 'and the\n' - 'literal "1".\n', - 'numeric-types': 'Emulating numeric types\n' - '***********************\n' - '\n' - 'The following methods can be defined to emulate numeric ' - 'objects.\n' - 'Methods corresponding to operations that are not supported ' - 'by the\n' - 'particular kind of number implemented (e.g., bitwise ' - 'operations for\n' - 'non-integral numbers) should be left undefined.\n' - '\n' - 'object.__add__(self, other)\n' - 'object.__sub__(self, other)\n' - 'object.__mul__(self, other)\n' - 'object.__matmul__(self, other)\n' - 'object.__truediv__(self, other)\n' - 'object.__floordiv__(self, other)\n' - 'object.__mod__(self, other)\n' - 'object.__divmod__(self, other)\n' - 'object.__pow__(self, other[, modulo])\n' - 'object.__lshift__(self, other)\n' - 'object.__rshift__(self, other)\n' - 'object.__and__(self, other)\n' - 'object.__xor__(self, other)\n' - 'object.__or__(self, other)\n' - '\n' - ' These methods are called to implement the binary ' - 'arithmetic\n' - ' operations ("+", "-", "*", "@", "/", "//", "%", ' - '"divmod()",\n' - ' "pow()", "**", "<<", ">>", "&", "^", "|"). For ' - 'instance, to\n' - ' evaluate the expression "x + y", where *x* is an ' - 'instance of a\n' - ' class that has an "__add__()" method, ' - '"type(x).__add__(x, y)" is\n' - ' called. The "__divmod__()" method should be the ' - 'equivalent to\n' - ' using "__floordiv__()" and "__mod__()"; it should not be ' - 'related to\n' - ' "__truediv__()". Note that "__pow__()" should be ' - 'defined to accept\n' - ' an optional third argument if the ternary version of the ' - 'built-in\n' - ' "pow()" function is to be supported.\n' - '\n' - ' If one of those methods does not support the operation ' - 'with the\n' - ' supplied arguments, it should return "NotImplemented".\n' - '\n' - 'object.__radd__(self, other)\n' - 'object.__rsub__(self, other)\n' - 'object.__rmul__(self, other)\n' - 'object.__rmatmul__(self, other)\n' - 'object.__rtruediv__(self, other)\n' - 'object.__rfloordiv__(self, other)\n' - 'object.__rmod__(self, other)\n' - 'object.__rdivmod__(self, other)\n' - 'object.__rpow__(self, other[, modulo])\n' - 'object.__rlshift__(self, other)\n' - 'object.__rrshift__(self, other)\n' - 'object.__rand__(self, other)\n' - 'object.__rxor__(self, other)\n' - 'object.__ror__(self, other)\n' - '\n' - ' These methods are called to implement the binary ' - 'arithmetic\n' - ' operations ("+", "-", "*", "@", "/", "//", "%", ' - '"divmod()",\n' - ' "pow()", "**", "<<", ">>", "&", "^", "|") with reflected ' - '(swapped)\n' - ' operands. These functions are only called if the left ' - 'operand does\n' - ' not support the corresponding operation [3] and the ' - 'operands are of\n' - ' different types. [4] For instance, to evaluate the ' - 'expression "x -\n' - ' y", where *y* is an instance of a class that has an ' - '"__rsub__()"\n' - ' method, "type(y).__rsub__(y, x)" is called if ' - '"type(x).__sub__(x,\n' - ' y)" returns *NotImplemented*.\n' - '\n' - ' Note that ternary "pow()" will not try calling ' - '"__rpow__()" (the\n' - ' coercion rules would become too complicated).\n' - '\n' - ' Note:\n' - '\n' - ' If the right operand’s type is a subclass of the left ' - 'operand’s\n' - ' type and that subclass provides a different ' - 'implementation of the\n' - ' reflected method for the operation, this method will ' - 'be called\n' - ' before the left operand’s non-reflected method. This ' - 'behavior\n' - ' allows subclasses to override their ancestors’ ' - 'operations.\n' - '\n' - 'object.__iadd__(self, other)\n' - 'object.__isub__(self, other)\n' - 'object.__imul__(self, other)\n' - 'object.__imatmul__(self, other)\n' - 'object.__itruediv__(self, other)\n' - 'object.__ifloordiv__(self, other)\n' - 'object.__imod__(self, other)\n' - 'object.__ipow__(self, other[, modulo])\n' - 'object.__ilshift__(self, other)\n' - 'object.__irshift__(self, other)\n' - 'object.__iand__(self, other)\n' - 'object.__ixor__(self, other)\n' - 'object.__ior__(self, other)\n' - '\n' - ' These methods are called to implement the augmented ' - 'arithmetic\n' - ' assignments ("+=", "-=", "*=", "@=", "/=", "//=", "%=", ' - '"**=",\n' - ' "<<=", ">>=", "&=", "^=", "|="). These methods should ' - 'attempt to\n' - ' do the operation in-place (modifying *self*) and return ' - 'the result\n' - ' (which could be, but does not have to be, *self*). If a ' - 'specific\n' - ' method is not defined, the augmented assignment falls ' - 'back to the\n' - ' normal methods. For instance, if *x* is an instance of ' - 'a class\n' - ' with an "__iadd__()" method, "x += y" is equivalent to ' - '"x =\n' - ' x.__iadd__(y)" . Otherwise, "x.__add__(y)" and ' - '"y.__radd__(x)" are\n' - ' considered, as with the evaluation of "x + y". In ' - 'certain\n' - ' situations, augmented assignment can result in ' - 'unexpected errors\n' - ' (see Why does a_tuple[i] += [‘item’] raise an exception ' - 'when the\n' - ' addition works?), but this behavior is in fact part of ' - 'the data\n' - ' model.\n' - '\n' - 'object.__neg__(self)\n' - 'object.__pos__(self)\n' - 'object.__abs__(self)\n' - 'object.__invert__(self)\n' - '\n' - ' Called to implement the unary arithmetic operations ' - '("-", "+",\n' - ' "abs()" and "~").\n' - '\n' - 'object.__complex__(self)\n' - 'object.__int__(self)\n' - 'object.__float__(self)\n' - '\n' - ' Called to implement the built-in functions "complex()", ' - '"int()" and\n' - ' "float()". Should return a value of the appropriate ' - 'type.\n' - '\n' - 'object.__index__(self)\n' - '\n' - ' Called to implement "operator.index()", and whenever ' - 'Python needs\n' - ' to losslessly convert the numeric object to an integer ' - 'object (such\n' - ' as in slicing, or in the built-in "bin()", "hex()" and ' - '"oct()"\n' - ' functions). Presence of this method indicates that the ' - 'numeric\n' - ' object is an integer type. Must return an integer.\n' - '\n' - ' If "__int__()", "__float__()" and "__complex__()" are ' - 'not defined\n' - ' then corresponding built-in functions "int()", "float()" ' - 'and\n' - ' "complex()" fall back to "__index__()".\n' - '\n' - 'object.__round__(self[, ndigits])\n' - 'object.__trunc__(self)\n' - 'object.__floor__(self)\n' - 'object.__ceil__(self)\n' - '\n' - ' Called to implement the built-in function "round()" and ' - '"math"\n' - ' functions "trunc()", "floor()" and "ceil()". Unless ' - '*ndigits* is\n' - ' passed to "__round__()" all these methods should return ' - 'the value\n' - ' of the object truncated to an "Integral" (typically an ' - '"int").\n' - '\n' - ' The built-in function "int()" falls back to ' - '"__trunc__()" if\n' - ' neither "__int__()" nor "__index__()" is defined.\n' - '\n' - ' Changed in version 3.11: The delegation of "int()" to ' - '"__trunc__()"\n' - ' is deprecated.\n', - 'objects': 'Objects, values and types\n' - '*************************\n' - '\n' - '*Objects* are Python’s abstraction for data. All data in a ' - 'Python\n' - 'program is represented by objects or by relations between ' - 'objects. (In\n' - 'a sense, and in conformance to Von Neumann’s model of a “stored\n' - 'program computer”, code is also represented by objects.)\n' - '\n' - 'Every object has an identity, a type and a value. An object’s\n' - '*identity* never changes once it has been created; you may think ' - 'of it\n' - 'as the object’s address in memory. The ‘"is"’ operator compares ' - 'the\n' - 'identity of two objects; the "id()" function returns an integer\n' - 'representing its identity.\n' - '\n' - '**CPython implementation detail:** For CPython, "id(x)" is the ' - 'memory\n' - 'address where "x" is stored.\n' - '\n' - 'An object’s type determines the operations that the object ' - 'supports\n' - '(e.g., “does it have a length?”) and also defines the possible ' - 'values\n' - 'for objects of that type. The "type()" function returns an ' - 'object’s\n' - 'type (which is an object itself). Like its identity, an ' - 'object’s\n' - '*type* is also unchangeable. [1]\n' - '\n' - 'The *value* of some objects can change. Objects whose value can\n' - 'change are said to be *mutable*; objects whose value is ' - 'unchangeable\n' - 'once they are created are called *immutable*. (The value of an\n' - 'immutable container object that contains a reference to a ' - 'mutable\n' - 'object can change when the latter’s value is changed; however ' - 'the\n' - 'container is still considered immutable, because the collection ' - 'of\n' - 'objects it contains cannot be changed. So, immutability is not\n' - 'strictly the same as having an unchangeable value, it is more ' - 'subtle.)\n' - 'An object’s mutability is determined by its type; for instance,\n' - 'numbers, strings and tuples are immutable, while dictionaries ' - 'and\n' - 'lists are mutable.\n' - '\n' - 'Objects are never explicitly destroyed; however, when they ' - 'become\n' - 'unreachable they may be garbage-collected. An implementation is\n' - 'allowed to postpone garbage collection or omit it altogether — it ' - 'is a\n' - 'matter of implementation quality how garbage collection is\n' - 'implemented, as long as no objects are collected that are still\n' - 'reachable.\n' - '\n' - '**CPython implementation detail:** CPython currently uses a ' - 'reference-\n' - 'counting scheme with (optional) delayed detection of cyclically ' - 'linked\n' - 'garbage, which collects most objects as soon as they become\n' - 'unreachable, but is not guaranteed to collect garbage containing\n' - 'circular references. See the documentation of the "gc" module ' - 'for\n' - 'information on controlling the collection of cyclic garbage. ' - 'Other\n' - 'implementations act differently and CPython may change. Do not ' - 'depend\n' - 'on immediate finalization of objects when they become unreachable ' - '(so\n' - 'you should always close files explicitly).\n' - '\n' - 'Note that the use of the implementation’s tracing or debugging\n' - 'facilities may keep objects alive that would normally be ' - 'collectable.\n' - 'Also note that catching an exception with a ‘"try"…"except"’ ' - 'statement\n' - 'may keep objects alive.\n' - '\n' - 'Some objects contain references to “external” resources such as ' - 'open\n' - 'files or windows. It is understood that these resources are ' - 'freed\n' - 'when the object is garbage-collected, but since garbage ' - 'collection is\n' - 'not guaranteed to happen, such objects also provide an explicit ' - 'way to\n' - 'release the external resource, usually a "close()" method. ' - 'Programs\n' - 'are strongly recommended to explicitly close such objects. The\n' - '‘"try"…"finally"’ statement and the ‘"with"’ statement provide\n' - 'convenient ways to do this.\n' - '\n' - 'Some objects contain references to other objects; these are ' - 'called\n' - '*containers*. Examples of containers are tuples, lists and\n' - 'dictionaries. The references are part of a container’s value. ' - 'In\n' - 'most cases, when we talk about the value of a container, we imply ' - 'the\n' - 'values, not the identities of the contained objects; however, ' - 'when we\n' - 'talk about the mutability of a container, only the identities of ' - 'the\n' - 'immediately contained objects are implied. So, if an immutable\n' - 'container (like a tuple) contains a reference to a mutable ' - 'object, its\n' - 'value changes if that mutable object is changed.\n' - '\n' - 'Types affect almost all aspects of object behavior. Even the\n' - 'importance of object identity is affected in some sense: for ' - 'immutable\n' - 'types, operations that compute new values may actually return a\n' - 'reference to any existing object with the same type and value, ' - 'while\n' - 'for mutable objects this is not allowed. E.g., after "a = 1; b = ' - '1",\n' - '"a" and "b" may or may not refer to the same object with the ' - 'value\n' - 'one, depending on the implementation, but after "c = []; d = []", ' - '"c"\n' - 'and "d" are guaranteed to refer to two different, unique, newly\n' - 'created empty lists. (Note that "c = d = []" assigns the same ' - 'object\n' - 'to both "c" and "d".)\n', - 'operator-summary': 'Operator precedence\n' - '*******************\n' - '\n' - 'The following table summarizes the operator precedence ' - 'in Python, from\n' - 'highest precedence (most binding) to lowest precedence ' - '(least\n' - 'binding). Operators in the same box have the same ' - 'precedence. Unless\n' - 'the syntax is explicitly given, operators are binary. ' - 'Operators in\n' - 'the same box group left to right (except for ' - 'exponentiation and\n' - 'conditional expressions, which group from right to ' - 'left).\n' - '\n' - 'Note that comparisons, membership tests, and identity ' - 'tests, all have\n' - 'the same precedence and have a left-to-right chaining ' - 'feature as\n' - 'described in the Comparisons section.\n' - '\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| Operator | ' - 'Description |\n' - '|=================================================|=======================================|\n' - '| "(expressions...)", "[expressions...]", "{key: | ' - 'Binding or parenthesized expression, |\n' - '| value...}", "{expressions...}" | list ' - 'display, dictionary display, set |\n' - '| | ' - 'display |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "x[index]", "x[index:index]", | ' - 'Subscription, slicing, call, |\n' - '| "x(arguments...)", "x.attribute" | ' - 'attribute reference |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "await x" | ' - 'Await expression |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "**" | ' - 'Exponentiation [5] |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "+x", "-x", "~x" | ' - 'Positive, negative, bitwise NOT |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "*", "@", "/", "//", "%" | ' - 'Multiplication, matrix |\n' - '| | ' - 'multiplication, division, floor |\n' - '| | ' - 'division, remainder [6] |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "+", "-" | ' - 'Addition and subtraction |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "<<", ">>" | ' - 'Shifts |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "&" | ' - 'Bitwise AND |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "^" | ' - 'Bitwise XOR |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "|" | ' - 'Bitwise OR |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "in", "not in", "is", "is not", "<", "<=", ">", | ' - 'Comparisons, including membership |\n' - '| ">=", "!=", "==" | ' - 'tests and identity tests |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "not x" | ' - 'Boolean NOT |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "and" | ' - 'Boolean AND |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "or" | ' - 'Boolean OR |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "if" – "else" | ' - 'Conditional expression |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "lambda" | ' - 'Lambda expression |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| ":=" | ' - 'Assignment expression |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '\n' - '-[ Footnotes ]-\n' - '\n' - '[1] While "abs(x%y) < abs(y)" is true mathematically, ' - 'for floats it\n' - ' may not be true numerically due to roundoff. For ' - 'example, and\n' - ' assuming a platform on which a Python float is an ' - 'IEEE 754 double-\n' - ' precision number, in order that "-1e-100 % 1e100" ' - 'have the same\n' - ' sign as "1e100", the computed result is "-1e-100 + ' - '1e100", which\n' - ' is numerically exactly equal to "1e100". The ' - 'function\n' - ' "math.fmod()" returns a result whose sign matches ' - 'the sign of the\n' - ' first argument instead, and so returns "-1e-100" in ' - 'this case.\n' - ' Which approach is more appropriate depends on the ' - 'application.\n' - '\n' - '[2] If x is very close to an exact integer multiple of ' - 'y, it’s\n' - ' possible for "x//y" to be one larger than ' - '"(x-x%y)//y" due to\n' - ' rounding. In such cases, Python returns the latter ' - 'result, in\n' - ' order to preserve that "divmod(x,y)[0] * y + x % y" ' - 'be very close\n' - ' to "x".\n' - '\n' - '[3] The Unicode standard distinguishes between *code ' - 'points* (e.g.\n' - ' U+0041) and *abstract characters* (e.g. “LATIN ' - 'CAPITAL LETTER A”).\n' - ' While most abstract characters in Unicode are only ' - 'represented\n' - ' using one code point, there is a number of abstract ' - 'characters\n' - ' that can in addition be represented using a sequence ' - 'of more than\n' - ' one code point. For example, the abstract character ' - '“LATIN\n' - ' CAPITAL LETTER C WITH CEDILLA” can be represented as ' - 'a single\n' - ' *precomposed character* at code position U+00C7, or ' - 'as a sequence\n' - ' of a *base character* at code position U+0043 (LATIN ' - 'CAPITAL\n' - ' LETTER C), followed by a *combining character* at ' - 'code position\n' - ' U+0327 (COMBINING CEDILLA).\n' - '\n' - ' The comparison operators on strings compare at the ' - 'level of\n' - ' Unicode code points. This may be counter-intuitive ' - 'to humans. For\n' - ' example, ""\\u00C7" == "\\u0043\\u0327"" is "False", ' - 'even though both\n' - ' strings represent the same abstract character “LATIN ' - 'CAPITAL\n' - ' LETTER C WITH CEDILLA”.\n' - '\n' - ' To compare strings at the level of abstract ' - 'characters (that is,\n' - ' in a way intuitive to humans), use ' - '"unicodedata.normalize()".\n' - '\n' - '[4] Due to automatic garbage-collection, free lists, and ' - 'the dynamic\n' - ' nature of descriptors, you may notice seemingly ' - 'unusual behaviour\n' - ' in certain uses of the "is" operator, like those ' - 'involving\n' - ' comparisons between instance methods, or constants. ' - 'Check their\n' - ' documentation for more info.\n' - '\n' - '[5] The power operator "**" binds less tightly than an ' - 'arithmetic or\n' - ' bitwise unary operator on its right, that is, ' - '"2**-1" is "0.5".\n' - '\n' - '[6] The "%" operator is also used for string formatting; ' - 'the same\n' - ' precedence applies.\n', - 'pass': 'The "pass" statement\n' - '********************\n' - '\n' - ' pass_stmt ::= "pass"\n' - '\n' - '"pass" is a null operation — when it is executed, nothing happens. ' - 'It\n' - 'is useful as a placeholder when a statement is required ' - 'syntactically,\n' - 'but no code needs to be executed, for example:\n' - '\n' - ' def f(arg): pass # a function that does nothing (yet)\n' - '\n' - ' class C: pass # a class with no methods (yet)\n', - 'power': 'The power operator\n' - '******************\n' - '\n' - 'The power operator binds more tightly than unary operators on its\n' - 'left; it binds less tightly than unary operators on its right. ' - 'The\n' - 'syntax is:\n' - '\n' - ' power ::= (await_expr | primary) ["**" u_expr]\n' - '\n' - 'Thus, in an unparenthesized sequence of power and unary operators, ' - 'the\n' - 'operators are evaluated from right to left (this does not ' - 'constrain\n' - 'the evaluation order for the operands): "-1**2" results in "-1".\n' - '\n' - 'The power operator has the same semantics as the built-in "pow()"\n' - 'function, when called with two arguments: it yields its left ' - 'argument\n' - 'raised to the power of its right argument. The numeric arguments ' - 'are\n' - 'first converted to a common type, and the result is of that type.\n' - '\n' - 'For int operands, the result has the same type as the operands ' - 'unless\n' - 'the second argument is negative; in that case, all arguments are\n' - 'converted to float and a float result is delivered. For example,\n' - '"10**2" returns "100", but "10**-2" returns "0.01".\n' - '\n' - 'Raising "0.0" to a negative power results in a ' - '"ZeroDivisionError".\n' - 'Raising a negative number to a fractional power results in a ' - '"complex"\n' - 'number. (In earlier versions it raised a "ValueError".)\n' - '\n' - 'This operation can be customized using the special "__pow__()" ' - 'method.\n', - 'raise': 'The "raise" statement\n' - '*********************\n' - '\n' - ' raise_stmt ::= "raise" [expression ["from" expression]]\n' - '\n' - 'If no expressions are present, "raise" re-raises the exception that ' - 'is\n' - 'currently being handled, which is also known as the *active\n' - 'exception*. If there isn’t currently an active exception, a\n' - '"RuntimeError" exception is raised indicating that this is an ' - 'error.\n' - '\n' - 'Otherwise, "raise" evaluates the first expression as the exception\n' - 'object. It must be either a subclass or an instance of\n' - '"BaseException". If it is a class, the exception instance will be\n' - 'obtained when needed by instantiating the class with no arguments.\n' - '\n' - 'The *type* of the exception is the exception instance’s class, the\n' - '*value* is the instance itself.\n' - '\n' - 'A traceback object is normally created automatically when an ' - 'exception\n' - 'is raised and attached to it as the "__traceback__" attribute. You ' - 'can\n' - 'create an exception and set your own traceback in one step using ' - 'the\n' - '"with_traceback()" exception method (which returns the same ' - 'exception\n' - 'instance, with its traceback set to its argument), like so:\n' - '\n' - ' raise Exception("foo occurred").with_traceback(tracebackobj)\n' - '\n' - 'The "from" clause is used for exception chaining: if given, the ' - 'second\n' - '*expression* must be another exception class or instance. If the\n' - 'second expression is an exception instance, it will be attached to ' - 'the\n' - 'raised exception as the "__cause__" attribute (which is writable). ' - 'If\n' - 'the expression is an exception class, the class will be ' - 'instantiated\n' - 'and the resulting exception instance will be attached to the ' - 'raised\n' - 'exception as the "__cause__" attribute. If the raised exception is ' - 'not\n' - 'handled, both exceptions will be printed:\n' - '\n' - ' >>> try:\n' - ' ... print(1 / 0)\n' - ' ... except Exception as exc:\n' - ' ... raise RuntimeError("Something bad happened") from exc\n' - ' ...\n' - ' Traceback (most recent call last):\n' - ' File "", line 2, in \n' - ' print(1 / 0)\n' - ' ~~^~~\n' - ' ZeroDivisionError: division by zero\n' - '\n' - ' The above exception was the direct cause of the following ' - 'exception:\n' - '\n' - ' Traceback (most recent call last):\n' - ' File "", line 4, in \n' - ' raise RuntimeError("Something bad happened") from exc\n' - ' RuntimeError: Something bad happened\n' - '\n' - 'A similar mechanism works implicitly if a new exception is raised ' - 'when\n' - 'an exception is already being handled. An exception may be ' - 'handled\n' - 'when an "except" or "finally" clause, or a "with" statement, is ' - 'used.\n' - 'The previous exception is then attached as the new exception’s\n' - '"__context__" attribute:\n' - '\n' - ' >>> try:\n' - ' ... print(1 / 0)\n' - ' ... except:\n' - ' ... raise RuntimeError("Something bad happened")\n' - ' ...\n' - ' Traceback (most recent call last):\n' - ' File "", line 2, in \n' - ' print(1 / 0)\n' - ' ~~^~~\n' - ' ZeroDivisionError: division by zero\n' - '\n' - ' During handling of the above exception, another exception ' - 'occurred:\n' - '\n' - ' Traceback (most recent call last):\n' - ' File "", line 4, in \n' - ' raise RuntimeError("Something bad happened")\n' - ' RuntimeError: Something bad happened\n' - '\n' - 'Exception chaining can be explicitly suppressed by specifying ' - '"None"\n' - 'in the "from" clause:\n' - '\n' - ' >>> try:\n' - ' ... print(1 / 0)\n' - ' ... except:\n' - ' ... raise RuntimeError("Something bad happened") from None\n' - ' ...\n' - ' Traceback (most recent call last):\n' - ' File "", line 4, in \n' - ' RuntimeError: Something bad happened\n' - '\n' - 'Additional information on exceptions can be found in section\n' - 'Exceptions, and information about handling exceptions is in ' - 'section\n' - 'The try statement.\n' - '\n' - 'Changed in version 3.3: "None" is now permitted as "Y" in "raise X\n' - 'from Y".\n' - '\n' - 'New in version 3.3: The "__suppress_context__" attribute to ' - 'suppress\n' - 'automatic display of the exception context.\n' - '\n' - 'Changed in version 3.11: If the traceback of the active exception ' - 'is\n' - 'modified in an "except" clause, a subsequent "raise" statement re-\n' - 'raises the exception with the modified traceback. Previously, the\n' - 'exception was re-raised with the traceback it had when it was ' - 'caught.\n', - 'return': 'The "return" statement\n' - '**********************\n' - '\n' - ' return_stmt ::= "return" [expression_list]\n' - '\n' - '"return" may only occur syntactically nested in a function ' - 'definition,\n' - 'not within a nested class definition.\n' - '\n' - 'If an expression list is present, it is evaluated, else "None" is\n' - 'substituted.\n' - '\n' - '"return" leaves the current function call with the expression list ' - '(or\n' - '"None") as return value.\n' - '\n' - 'When "return" passes control out of a "try" statement with a ' - '"finally"\n' - 'clause, that "finally" clause is executed before really leaving ' - 'the\n' - 'function.\n' - '\n' - 'In a generator function, the "return" statement indicates that ' - 'the\n' - 'generator is done and will cause "StopIteration" to be raised. ' - 'The\n' - 'returned value (if any) is used as an argument to construct\n' - '"StopIteration" and becomes the "StopIteration.value" attribute.\n' - '\n' - 'In an asynchronous generator function, an empty "return" ' - 'statement\n' - 'indicates that the asynchronous generator is done and will cause\n' - '"StopAsyncIteration" to be raised. A non-empty "return" statement ' - 'is\n' - 'a syntax error in an asynchronous generator function.\n', - 'sequence-types': 'Emulating container types\n' - '*************************\n' - '\n' - 'The following methods can be defined to implement ' - 'container objects.\n' - 'Containers usually are *sequences* (such as "lists" or ' - '"tuples") or\n' - '*mappings* (like "dictionaries"), but can represent other ' - 'containers\n' - 'as well. The first set of methods is used either to ' - 'emulate a\n' - 'sequence or to emulate a mapping; the difference is that ' - 'for a\n' - 'sequence, the allowable keys should be the integers *k* ' - 'for which "0\n' - '<= k < N" where *N* is the length of the sequence, or ' - '"slice" objects,\n' - 'which define a range of items. It is also recommended ' - 'that mappings\n' - 'provide the methods "keys()", "values()", "items()", ' - '"get()",\n' - '"clear()", "setdefault()", "pop()", "popitem()", "copy()", ' - 'and\n' - '"update()" behaving similar to those for Python’s ' - 'standard\n' - '"dictionary" objects. The "collections.abc" module ' - 'provides a\n' - '"MutableMapping" *abstract base class* to help create ' - 'those methods\n' - 'from a base set of "__getitem__()", "__setitem__()", ' - '"__delitem__()",\n' - 'and "keys()". Mutable sequences should provide methods ' - '"append()",\n' - '"count()", "index()", "extend()", "insert()", "pop()", ' - '"remove()",\n' - '"reverse()" and "sort()", like Python standard "list" ' - 'objects.\n' - 'Finally, sequence types should implement addition ' - '(meaning\n' - 'concatenation) and multiplication (meaning repetition) by ' - 'defining the\n' - 'methods "__add__()", "__radd__()", "__iadd__()", ' - '"__mul__()",\n' - '"__rmul__()" and "__imul__()" described below; they should ' - 'not define\n' - 'other numerical operators. It is recommended that both ' - 'mappings and\n' - 'sequences implement the "__contains__()" method to allow ' - 'efficient use\n' - 'of the "in" operator; for mappings, "in" should search the ' - 'mapping’s\n' - 'keys; for sequences, it should search through the values. ' - 'It is\n' - 'further recommended that both mappings and sequences ' - 'implement the\n' - '"__iter__()" method to allow efficient iteration through ' - 'the\n' - 'container; for mappings, "__iter__()" should iterate ' - 'through the\n' - 'object’s keys; for sequences, it should iterate through ' - 'the values.\n' - '\n' - 'object.__len__(self)\n' - '\n' - ' Called to implement the built-in function "len()". ' - 'Should return\n' - ' the length of the object, an integer ">=" 0. Also, an ' - 'object that\n' - ' doesn’t define a "__bool__()" method and whose ' - '"__len__()" method\n' - ' returns zero is considered to be false in a Boolean ' - 'context.\n' - '\n' - ' **CPython implementation detail:** In CPython, the ' - 'length is\n' - ' required to be at most "sys.maxsize". If the length is ' - 'larger than\n' - ' "sys.maxsize" some features (such as "len()") may ' - 'raise\n' - ' "OverflowError". To prevent raising "OverflowError" by ' - 'truth value\n' - ' testing, an object must define a "__bool__()" method.\n' - '\n' - 'object.__length_hint__(self)\n' - '\n' - ' Called to implement "operator.length_hint()". Should ' - 'return an\n' - ' estimated length for the object (which may be greater ' - 'or less than\n' - ' the actual length). The length must be an integer ">=" ' - '0. The\n' - ' return value may also be "NotImplemented", which is ' - 'treated the\n' - ' same as if the "__length_hint__" method didn’t exist at ' - 'all. This\n' - ' method is purely an optimization and is never required ' - 'for\n' - ' correctness.\n' - '\n' - ' New in version 3.4.\n' - '\n' - 'Note:\n' - '\n' - ' Slicing is done exclusively with the following three ' - 'methods. A\n' - ' call like\n' - '\n' - ' a[1:2] = b\n' - '\n' - ' is translated to\n' - '\n' - ' a[slice(1, 2, None)] = b\n' - '\n' - ' and so forth. Missing slice items are always filled in ' - 'with "None".\n' - '\n' - 'object.__getitem__(self, key)\n' - '\n' - ' Called to implement evaluation of "self[key]". For ' - '*sequence*\n' - ' types, the accepted keys should be integers. ' - 'Optionally, they may\n' - ' support "slice" objects as well. Negative index ' - 'support is also\n' - ' optional. If *key* is of an inappropriate type, ' - '"TypeError" may be\n' - ' raised; if *key* is a value outside the set of indexes ' - 'for the\n' - ' sequence (after any special interpretation of negative ' - 'values),\n' - ' "IndexError" should be raised. For *mapping* types, if ' - '*key* is\n' - ' missing (not in the container), "KeyError" should be ' - 'raised.\n' - '\n' - ' Note:\n' - '\n' - ' "for" loops expect that an "IndexError" will be ' - 'raised for\n' - ' illegal indexes to allow proper detection of the end ' - 'of the\n' - ' sequence.\n' - '\n' - ' Note:\n' - '\n' - ' When subscripting a *class*, the special class ' - 'method\n' - ' "__class_getitem__()" may be called instead of ' - '"__getitem__()".\n' - ' See __class_getitem__ versus __getitem__ for more ' - 'details.\n' - '\n' - 'object.__setitem__(self, key, value)\n' - '\n' - ' Called to implement assignment to "self[key]". Same ' - 'note as for\n' - ' "__getitem__()". This should only be implemented for ' - 'mappings if\n' - ' the objects support changes to the values for keys, or ' - 'if new keys\n' - ' can be added, or for sequences if elements can be ' - 'replaced. The\n' - ' same exceptions should be raised for improper *key* ' - 'values as for\n' - ' the "__getitem__()" method.\n' - '\n' - 'object.__delitem__(self, key)\n' - '\n' - ' Called to implement deletion of "self[key]". Same note ' - 'as for\n' - ' "__getitem__()". This should only be implemented for ' - 'mappings if\n' - ' the objects support removal of keys, or for sequences ' - 'if elements\n' - ' can be removed from the sequence. The same exceptions ' - 'should be\n' - ' raised for improper *key* values as for the ' - '"__getitem__()" method.\n' - '\n' - 'object.__missing__(self, key)\n' - '\n' - ' Called by "dict"."__getitem__()" to implement ' - '"self[key]" for dict\n' - ' subclasses when key is not in the dictionary.\n' - '\n' - 'object.__iter__(self)\n' - '\n' - ' This method is called when an *iterator* is required ' - 'for a\n' - ' container. This method should return a new iterator ' - 'object that can\n' - ' iterate over all the objects in the container. For ' - 'mappings, it\n' - ' should iterate over the keys of the container.\n' - '\n' - 'object.__reversed__(self)\n' - '\n' - ' Called (if present) by the "reversed()" built-in to ' - 'implement\n' - ' reverse iteration. It should return a new iterator ' - 'object that\n' - ' iterates over all the objects in the container in ' - 'reverse order.\n' - '\n' - ' If the "__reversed__()" method is not provided, the ' - '"reversed()"\n' - ' built-in will fall back to using the sequence protocol ' - '("__len__()"\n' - ' and "__getitem__()"). Objects that support the ' - 'sequence protocol\n' - ' should only provide "__reversed__()" if they can ' - 'provide an\n' - ' implementation that is more efficient than the one ' - 'provided by\n' - ' "reversed()".\n' - '\n' - 'The membership test operators ("in" and "not in") are ' - 'normally\n' - 'implemented as an iteration through a container. However, ' - 'container\n' - 'objects can supply the following special method with a ' - 'more efficient\n' - 'implementation, which also does not require the object be ' - 'iterable.\n' - '\n' - 'object.__contains__(self, item)\n' - '\n' - ' Called to implement membership test operators. Should ' - 'return true\n' - ' if *item* is in *self*, false otherwise. For mapping ' - 'objects, this\n' - ' should consider the keys of the mapping rather than the ' - 'values or\n' - ' the key-item pairs.\n' - '\n' - ' For objects that don’t define "__contains__()", the ' - 'membership test\n' - ' first tries iteration via "__iter__()", then the old ' - 'sequence\n' - ' iteration protocol via "__getitem__()", see this ' - 'section in the\n' - ' language reference.\n', - 'shifting': 'Shifting operations\n' - '*******************\n' - '\n' - 'The shifting operations have lower priority than the arithmetic\n' - 'operations:\n' - '\n' - ' shift_expr ::= a_expr | shift_expr ("<<" | ">>") a_expr\n' - '\n' - 'These operators accept integers as arguments. They shift the ' - 'first\n' - 'argument to the left or right by the number of bits given by ' - 'the\n' - 'second argument.\n' - '\n' - 'This operation can be customized using the special ' - '"__lshift__()" and\n' - '"__rshift__()" methods.\n' - '\n' - 'A right shift by *n* bits is defined as floor division by ' - '"pow(2,n)".\n' - 'A left shift by *n* bits is defined as multiplication with ' - '"pow(2,n)".\n', - 'slicings': 'Slicings\n' - '********\n' - '\n' - 'A slicing selects a range of items in a sequence object (e.g., ' - 'a\n' - 'string, tuple or list). Slicings may be used as expressions or ' - 'as\n' - 'targets in assignment or "del" statements. The syntax for a ' - 'slicing:\n' - '\n' - ' slicing ::= primary "[" slice_list "]"\n' - ' slice_list ::= slice_item ("," slice_item)* [","]\n' - ' slice_item ::= expression | proper_slice\n' - ' proper_slice ::= [lower_bound] ":" [upper_bound] [ ":" ' - '[stride] ]\n' - ' lower_bound ::= expression\n' - ' upper_bound ::= expression\n' - ' stride ::= expression\n' - '\n' - 'There is ambiguity in the formal syntax here: anything that ' - 'looks like\n' - 'an expression list also looks like a slice list, so any ' - 'subscription\n' - 'can be interpreted as a slicing. Rather than further ' - 'complicating the\n' - 'syntax, this is disambiguated by defining that in this case the\n' - 'interpretation as a subscription takes priority over the\n' - 'interpretation as a slicing (this is the case if the slice list\n' - 'contains no proper slice).\n' - '\n' - 'The semantics for a slicing are as follows. The primary is ' - 'indexed\n' - '(using the same "__getitem__()" method as normal subscription) ' - 'with a\n' - 'key that is constructed from the slice list, as follows. If the ' - 'slice\n' - 'list contains at least one comma, the key is a tuple containing ' - 'the\n' - 'conversion of the slice items; otherwise, the conversion of the ' - 'lone\n' - 'slice item is the key. The conversion of a slice item that is ' - 'an\n' - 'expression is that expression. The conversion of a proper slice ' - 'is a\n' - 'slice object (see section The standard type hierarchy) whose ' - '"start",\n' - '"stop" and "step" attributes are the values of the expressions ' - 'given\n' - 'as lower bound, upper bound and stride, respectively, ' - 'substituting\n' - '"None" for missing expressions.\n', - 'specialattrs': 'Special Attributes\n' - '******************\n' - '\n' - 'The implementation adds a few special read-only attributes ' - 'to several\n' - 'object types, where they are relevant. Some of these are ' - 'not reported\n' - 'by the "dir()" built-in function.\n' - '\n' - 'object.__dict__\n' - '\n' - ' A dictionary or other mapping object used to store an ' - 'object’s\n' - ' (writable) attributes.\n' - '\n' - 'instance.__class__\n' - '\n' - ' The class to which a class instance belongs.\n' - '\n' - 'class.__bases__\n' - '\n' - ' The tuple of base classes of a class object.\n' - '\n' - 'definition.__name__\n' - '\n' - ' The name of the class, function, method, descriptor, or ' - 'generator\n' - ' instance.\n' - '\n' - 'definition.__qualname__\n' - '\n' - ' The *qualified name* of the class, function, method, ' - 'descriptor, or\n' - ' generator instance.\n' - '\n' - ' New in version 3.3.\n' - '\n' - 'definition.__type_params__\n' - '\n' - ' The type parameters of generic classes, functions, and ' - 'type\n' - ' aliases.\n' - '\n' - ' New in version 3.12.\n' - '\n' - 'class.__mro__\n' - '\n' - ' This attribute is a tuple of classes that are considered ' - 'when\n' - ' looking for base classes during method resolution.\n' - '\n' - 'class.mro()\n' - '\n' - ' This method can be overridden by a metaclass to customize ' - 'the\n' - ' method resolution order for its instances. It is called ' - 'at class\n' - ' instantiation, and its result is stored in "__mro__".\n' - '\n' - 'class.__subclasses__()\n' - '\n' - ' Each class keeps a list of weak references to its ' - 'immediate\n' - ' subclasses. This method returns a list of all those ' - 'references\n' - ' still alive. The list is in definition order. Example:\n' - '\n' - ' >>> int.__subclasses__()\n' - " [, , , " - "]\n", - 'specialnames': 'Special method names\n' - '********************\n' - '\n' - 'A class can implement certain operations that are invoked by ' - 'special\n' - 'syntax (such as arithmetic operations or subscripting and ' - 'slicing) by\n' - 'defining methods with special names. This is Python’s ' - 'approach to\n' - '*operator overloading*, allowing classes to define their own ' - 'behavior\n' - 'with respect to language operators. For instance, if a ' - 'class defines\n' - 'a method named "__getitem__()", and "x" is an instance of ' - 'this class,\n' - 'then "x[i]" is roughly equivalent to "type(x).__getitem__(x, ' - 'i)".\n' - 'Except where mentioned, attempts to execute an operation ' - 'raise an\n' - 'exception when no appropriate method is defined (typically\n' - '"AttributeError" or "TypeError").\n' - '\n' - 'Setting a special method to "None" indicates that the ' - 'corresponding\n' - 'operation is not available. For example, if a class sets ' - '"__iter__()"\n' - 'to "None", the class is not iterable, so calling "iter()" on ' - 'its\n' - 'instances will raise a "TypeError" (without falling back to\n' - '"__getitem__()"). [2]\n' - '\n' - 'When implementing a class that emulates any built-in type, ' - 'it is\n' - 'important that the emulation only be implemented to the ' - 'degree that it\n' - 'makes sense for the object being modelled. For example, ' - 'some\n' - 'sequences may work well with retrieval of individual ' - 'elements, but\n' - 'extracting a slice may not make sense. (One example of this ' - 'is the\n' - '"NodeList" interface in the W3C’s Document Object Model.)\n' - '\n' - '\n' - 'Basic customization\n' - '===================\n' - '\n' - 'object.__new__(cls[, ...])\n' - '\n' - ' Called to create a new instance of class *cls*. ' - '"__new__()" is a\n' - ' static method (special-cased so you need not declare it ' - 'as such)\n' - ' that takes the class of which an instance was requested ' - 'as its\n' - ' first argument. The remaining arguments are those passed ' - 'to the\n' - ' object constructor expression (the call to the class). ' - 'The return\n' - ' value of "__new__()" should be the new object instance ' - '(usually an\n' - ' instance of *cls*).\n' - '\n' - ' Typical implementations create a new instance of the ' - 'class by\n' - ' invoking the superclass’s "__new__()" method using\n' - ' "super().__new__(cls[, ...])" with appropriate arguments ' - 'and then\n' - ' modifying the newly created instance as necessary before ' - 'returning\n' - ' it.\n' - '\n' - ' If "__new__()" is invoked during object construction and ' - 'it returns\n' - ' an instance of *cls*, then the new instance’s ' - '"__init__()" method\n' - ' will be invoked like "__init__(self[, ...])", where ' - '*self* is the\n' - ' new instance and the remaining arguments are the same as ' - 'were\n' - ' passed to the object constructor.\n' - '\n' - ' If "__new__()" does not return an instance of *cls*, then ' - 'the new\n' - ' instance’s "__init__()" method will not be invoked.\n' - '\n' - ' "__new__()" is intended mainly to allow subclasses of ' - 'immutable\n' - ' types (like int, str, or tuple) to customize instance ' - 'creation. It\n' - ' is also commonly overridden in custom metaclasses in ' - 'order to\n' - ' customize class creation.\n' - '\n' - 'object.__init__(self[, ...])\n' - '\n' - ' Called after the instance has been created (by ' - '"__new__()"), but\n' - ' before it is returned to the caller. The arguments are ' - 'those\n' - ' passed to the class constructor expression. If a base ' - 'class has an\n' - ' "__init__()" method, the derived class’s "__init__()" ' - 'method, if\n' - ' any, must explicitly call it to ensure proper ' - 'initialization of the\n' - ' base class part of the instance; for example:\n' - ' "super().__init__([args...])".\n' - '\n' - ' Because "__new__()" and "__init__()" work together in ' - 'constructing\n' - ' objects ("__new__()" to create it, and "__init__()" to ' - 'customize\n' - ' it), no non-"None" value may be returned by "__init__()"; ' - 'doing so\n' - ' will cause a "TypeError" to be raised at runtime.\n' - '\n' - 'object.__del__(self)\n' - '\n' - ' Called when the instance is about to be destroyed. This ' - 'is also\n' - ' called a finalizer or (improperly) a destructor. If a ' - 'base class\n' - ' has a "__del__()" method, the derived class’s "__del__()" ' - 'method,\n' - ' if any, must explicitly call it to ensure proper deletion ' - 'of the\n' - ' base class part of the instance.\n' - '\n' - ' It is possible (though not recommended!) for the ' - '"__del__()" method\n' - ' to postpone destruction of the instance by creating a new ' - 'reference\n' - ' to it. This is called object *resurrection*. It is\n' - ' implementation-dependent whether "__del__()" is called a ' - 'second\n' - ' time when a resurrected object is about to be destroyed; ' - 'the\n' - ' current *CPython* implementation only calls it once.\n' - '\n' - ' It is not guaranteed that "__del__()" methods are called ' - 'for\n' - ' objects that still exist when the interpreter exits.\n' - '\n' - ' Note:\n' - '\n' - ' "del x" doesn’t directly call "x.__del__()" — the ' - 'former\n' - ' decrements the reference count for "x" by one, and the ' - 'latter is\n' - ' only called when "x"’s reference count reaches zero.\n' - '\n' - ' **CPython implementation detail:** It is possible for a ' - 'reference\n' - ' cycle to prevent the reference count of an object from ' - 'going to\n' - ' zero. In this case, the cycle will be later detected and ' - 'deleted\n' - ' by the *cyclic garbage collector*. A common cause of ' - 'reference\n' - ' cycles is when an exception has been caught in a local ' - 'variable.\n' - ' The frame’s locals then reference the exception, which ' - 'references\n' - ' its own traceback, which references the locals of all ' - 'frames caught\n' - ' in the traceback.\n' - '\n' - ' See also: Documentation for the "gc" module.\n' - '\n' - ' Warning:\n' - '\n' - ' Due to the precarious circumstances under which ' - '"__del__()"\n' - ' methods are invoked, exceptions that occur during their ' - 'execution\n' - ' are ignored, and a warning is printed to "sys.stderr" ' - 'instead.\n' - ' In particular:\n' - '\n' - ' * "__del__()" can be invoked when arbitrary code is ' - 'being\n' - ' executed, including from any arbitrary thread. If ' - '"__del__()"\n' - ' needs to take a lock or invoke any other blocking ' - 'resource, it\n' - ' may deadlock as the resource may already be taken by ' - 'the code\n' - ' that gets interrupted to execute "__del__()".\n' - '\n' - ' * "__del__()" can be executed during interpreter ' - 'shutdown. As a\n' - ' consequence, the global variables it needs to access ' - '(including\n' - ' other modules) may already have been deleted or set ' - 'to "None".\n' - ' Python guarantees that globals whose name begins with ' - 'a single\n' - ' underscore are deleted from their module before other ' - 'globals\n' - ' are deleted; if no other references to such globals ' - 'exist, this\n' - ' may help in assuring that imported modules are still ' - 'available\n' - ' at the time when the "__del__()" method is called.\n' - '\n' - 'object.__repr__(self)\n' - '\n' - ' Called by the "repr()" built-in function to compute the ' - '“official”\n' - ' string representation of an object. If at all possible, ' - 'this\n' - ' should look like a valid Python expression that could be ' - 'used to\n' - ' recreate an object with the same value (given an ' - 'appropriate\n' - ' environment). If this is not possible, a string of the ' - 'form\n' - ' "<...some useful description...>" should be returned. The ' - 'return\n' - ' value must be a string object. If a class defines ' - '"__repr__()" but\n' - ' not "__str__()", then "__repr__()" is also used when an ' - '“informal”\n' - ' string representation of instances of that class is ' - 'required.\n' - '\n' - ' This is typically used for debugging, so it is important ' - 'that the\n' - ' representation is information-rich and unambiguous.\n' - '\n' - 'object.__str__(self)\n' - '\n' - ' Called by "str(object)" and the built-in functions ' - '"format()" and\n' - ' "print()" to compute the “informal” or nicely printable ' - 'string\n' - ' representation of an object. The return value must be a ' - 'string\n' - ' object.\n' - '\n' - ' This method differs from "object.__repr__()" in that ' - 'there is no\n' - ' expectation that "__str__()" return a valid Python ' - 'expression: a\n' - ' more convenient or concise representation can be used.\n' - '\n' - ' The default implementation defined by the built-in type ' - '"object"\n' - ' calls "object.__repr__()".\n' - '\n' - 'object.__bytes__(self)\n' - '\n' - ' Called by bytes to compute a byte-string representation ' - 'of an\n' - ' object. This should return a "bytes" object.\n' - '\n' - 'object.__format__(self, format_spec)\n' - '\n' - ' Called by the "format()" built-in function, and by ' - 'extension,\n' - ' evaluation of formatted string literals and the ' - '"str.format()"\n' - ' method, to produce a “formatted” string representation of ' - 'an\n' - ' object. The *format_spec* argument is a string that ' - 'contains a\n' - ' description of the formatting options desired. The ' - 'interpretation\n' - ' of the *format_spec* argument is up to the type ' - 'implementing\n' - ' "__format__()", however most classes will either ' - 'delegate\n' - ' formatting to one of the built-in types, or use a ' - 'similar\n' - ' formatting option syntax.\n' - '\n' - ' See Format Specification Mini-Language for a description ' - 'of the\n' - ' standard formatting syntax.\n' - '\n' - ' The return value must be a string object.\n' - '\n' - ' Changed in version 3.4: The __format__ method of "object" ' - 'itself\n' - ' raises a "TypeError" if passed any non-empty string.\n' - '\n' - ' Changed in version 3.7: "object.__format__(x, \'\')" is ' - 'now\n' - ' equivalent to "str(x)" rather than "format(str(x), ' - '\'\')".\n' - '\n' - 'object.__lt__(self, other)\n' - 'object.__le__(self, other)\n' - 'object.__eq__(self, other)\n' - 'object.__ne__(self, other)\n' - 'object.__gt__(self, other)\n' - 'object.__ge__(self, other)\n' - '\n' - ' These are the so-called “rich comparison” methods. The\n' - ' correspondence between operator symbols and method names ' - 'is as\n' - ' follows: "xy" calls\n' - ' "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)".\n' - '\n' - ' A rich comparison method may return the singleton ' - '"NotImplemented"\n' - ' if it does not implement the operation for a given pair ' - 'of\n' - ' arguments. By convention, "False" and "True" are returned ' - 'for a\n' - ' successful comparison. However, these methods can return ' - 'any value,\n' - ' so if the comparison operator is used in a Boolean ' - 'context (e.g.,\n' - ' in the condition of an "if" statement), Python will call ' - '"bool()"\n' - ' on the value to determine if the result is true or ' - 'false.\n' - '\n' - ' By default, "object" implements "__eq__()" by using "is", ' - 'returning\n' - ' "NotImplemented" in the case of a false comparison: "True ' - 'if x is y\n' - ' else NotImplemented". For "__ne__()", by default it ' - 'delegates to\n' - ' "__eq__()" and inverts the result unless it is ' - '"NotImplemented".\n' - ' There are no other implied relationships among the ' - 'comparison\n' - ' operators or default implementations; for example, the ' - 'truth of\n' - ' "(x.__hash__".\n' - '\n' - ' If a class that does not override "__eq__()" wishes to ' - 'suppress\n' - ' hash support, it should include "__hash__ = None" in the ' - 'class\n' - ' definition. A class which defines its own "__hash__()" ' - 'that\n' - ' explicitly raises a "TypeError" would be incorrectly ' - 'identified as\n' - ' hashable by an "isinstance(obj, ' - 'collections.abc.Hashable)" call.\n' - '\n' - ' Note:\n' - '\n' - ' By default, the "__hash__()" values of str and bytes ' - 'objects are\n' - ' “salted” with an unpredictable random value. Although ' - 'they\n' - ' remain constant within an individual Python process, ' - 'they are not\n' - ' predictable between repeated invocations of Python.This ' - 'is\n' - ' intended to provide protection against a ' - 'denial-of-service caused\n' - ' by carefully chosen inputs that exploit the worst case\n' - ' performance of a dict insertion, *O*(*n*^2) ' - 'complexity. See\n' - ' http://ocert.org/advisories/ocert-2011-003.html for\n' - ' details.Changing hash values affects the iteration ' - 'order of sets.\n' - ' Python has never made guarantees about this ordering ' - '(and it\n' - ' typically varies between 32-bit and 64-bit builds).See ' - 'also\n' - ' "PYTHONHASHSEED".\n' - '\n' - ' Changed in version 3.3: Hash randomization is enabled by ' - 'default.\n' - '\n' - 'object.__bool__(self)\n' - '\n' - ' Called to implement truth value testing and the built-in ' - 'operation\n' - ' "bool()"; should return "False" or "True". When this ' - 'method is not\n' - ' defined, "__len__()" is called, if it is defined, and the ' - 'object is\n' - ' considered true if its result is nonzero. If a class ' - 'defines\n' - ' neither "__len__()" nor "__bool__()", all its instances ' - 'are\n' - ' considered true.\n' - '\n' - '\n' - 'Customizing attribute access\n' - '============================\n' - '\n' - 'The following methods can be defined to customize the ' - 'meaning of\n' - 'attribute access (use of, assignment to, or deletion of ' - '"x.name") for\n' - 'class instances.\n' - '\n' - 'object.__getattr__(self, name)\n' - '\n' - ' Called when the default attribute access fails with an\n' - ' "AttributeError" (either "__getattribute__()" raises an\n' - ' "AttributeError" because *name* is not an instance ' - 'attribute or an\n' - ' attribute in the class tree for "self"; or "__get__()" of ' - 'a *name*\n' - ' property raises "AttributeError"). This method should ' - 'either\n' - ' return the (computed) attribute value or raise an ' - '"AttributeError"\n' - ' exception.\n' - '\n' - ' Note that if the attribute is found through the normal ' - 'mechanism,\n' - ' "__getattr__()" is not called. (This is an intentional ' - 'asymmetry\n' - ' between "__getattr__()" and "__setattr__()".) This is ' - 'done both for\n' - ' efficiency reasons and because otherwise "__getattr__()" ' - 'would have\n' - ' no way to access other attributes of the instance. Note ' - 'that at\n' - ' least for instance variables, you can take total control ' - 'by not\n' - ' inserting any values in the instance attribute dictionary ' - '(but\n' - ' instead inserting them in another object). See the\n' - ' "__getattribute__()" method below for a way to actually ' - 'get total\n' - ' control over attribute access.\n' - '\n' - 'object.__getattribute__(self, name)\n' - '\n' - ' Called unconditionally to implement attribute accesses ' - 'for\n' - ' instances of the class. If the class also defines ' - '"__getattr__()",\n' - ' the latter will not be called unless "__getattribute__()" ' - 'either\n' - ' calls it explicitly or raises an "AttributeError". This ' - 'method\n' - ' should return the (computed) attribute value or raise an\n' - ' "AttributeError" exception. In order to avoid infinite ' - 'recursion in\n' - ' this method, its implementation should always call the ' - 'base class\n' - ' method with the same name to access any attributes it ' - 'needs, for\n' - ' example, "object.__getattribute__(self, name)".\n' - '\n' - ' Note:\n' - '\n' - ' This method may still be bypassed when looking up ' - 'special methods\n' - ' as the result of implicit invocation via language ' - 'syntax or\n' - ' built-in functions. See Special method lookup.\n' - '\n' - ' For certain sensitive attribute accesses, raises an ' - 'auditing event\n' - ' "object.__getattr__" with arguments "obj" and "name".\n' - '\n' - 'object.__setattr__(self, name, value)\n' - '\n' - ' Called when an attribute assignment is attempted. This ' - 'is called\n' - ' instead of the normal mechanism (i.e. store the value in ' - 'the\n' - ' instance dictionary). *name* is the attribute name, ' - '*value* is the\n' - ' value to be assigned to it.\n' - '\n' - ' If "__setattr__()" wants to assign to an instance ' - 'attribute, it\n' - ' should call the base class method with the same name, for ' - 'example,\n' - ' "object.__setattr__(self, name, value)".\n' - '\n' - ' For certain sensitive attribute assignments, raises an ' - 'auditing\n' - ' event "object.__setattr__" with arguments "obj", "name", ' - '"value".\n' - '\n' - 'object.__delattr__(self, name)\n' - '\n' - ' Like "__setattr__()" but for attribute deletion instead ' - 'of\n' - ' assignment. This should only be implemented if "del ' - 'obj.name" is\n' - ' meaningful for the object.\n' - '\n' - ' For certain sensitive attribute deletions, raises an ' - 'auditing event\n' - ' "object.__delattr__" with arguments "obj" and "name".\n' - '\n' - 'object.__dir__(self)\n' - '\n' - ' Called when "dir()" is called on the object. An iterable ' - 'must be\n' - ' returned. "dir()" converts the returned iterable to a ' - 'list and\n' - ' sorts it.\n' - '\n' - '\n' - 'Customizing module attribute access\n' - '-----------------------------------\n' - '\n' - 'Special names "__getattr__" and "__dir__" can be also used ' - 'to\n' - 'customize access to module attributes. The "__getattr__" ' - 'function at\n' - 'the module level should accept one argument which is the ' - 'name of an\n' - 'attribute and return the computed value or raise an ' - '"AttributeError".\n' - 'If an attribute is not found on a module object through the ' - 'normal\n' - 'lookup, i.e. "object.__getattribute__()", then "__getattr__" ' - 'is\n' - 'searched in the module "__dict__" before raising an ' - '"AttributeError".\n' - 'If found, it is called with the attribute name and the ' - 'result is\n' - 'returned.\n' - '\n' - 'The "__dir__" function should accept no arguments, and ' - 'return an\n' - 'iterable of strings that represents the names accessible on ' - 'module. If\n' - 'present, this function overrides the standard "dir()" search ' - 'on a\n' - 'module.\n' - '\n' - 'For a more fine grained customization of the module behavior ' - '(setting\n' - 'attributes, properties, etc.), one can set the "__class__" ' - 'attribute\n' - 'of a module object to a subclass of "types.ModuleType". For ' - 'example:\n' - '\n' - ' import sys\n' - ' from types import ModuleType\n' - '\n' - ' class VerboseModule(ModuleType):\n' - ' def __repr__(self):\n' - " return f'Verbose {self.__name__}'\n" - '\n' - ' def __setattr__(self, attr, value):\n' - " print(f'Setting {attr}...')\n" - ' super().__setattr__(attr, value)\n' - '\n' - ' sys.modules[__name__].__class__ = VerboseModule\n' - '\n' - 'Note:\n' - '\n' - ' Defining module "__getattr__" and setting module ' - '"__class__" only\n' - ' affect lookups made using the attribute access syntax – ' - 'directly\n' - ' accessing the module globals (whether by code within the ' - 'module, or\n' - ' via a reference to the module’s globals dictionary) is ' - 'unaffected.\n' - '\n' - 'Changed in version 3.5: "__class__" module attribute is now ' - 'writable.\n' - '\n' - 'New in version 3.7: "__getattr__" and "__dir__" module ' - 'attributes.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 562** - Module __getattr__ and __dir__\n' - ' Describes the "__getattr__" and "__dir__" functions on ' - 'modules.\n' - '\n' - '\n' - 'Implementing Descriptors\n' - '------------------------\n' - '\n' - 'The following methods only apply when an instance of the ' - 'class\n' - 'containing the method (a so-called *descriptor* class) ' - 'appears in an\n' - '*owner* class (the descriptor must be in either the owner’s ' - 'class\n' - 'dictionary or in the class dictionary for one of its ' - 'parents). In the\n' - 'examples below, “the attribute” refers to the attribute ' - 'whose name is\n' - 'the key of the property in the owner class’ "__dict__".\n' - '\n' - 'object.__get__(self, instance, owner=None)\n' - '\n' - ' Called to get the attribute of the owner class (class ' - 'attribute\n' - ' access) or of an instance of that class (instance ' - 'attribute\n' - ' access). The optional *owner* argument is the owner ' - 'class, while\n' - ' *instance* is the instance that the attribute was ' - 'accessed through,\n' - ' or "None" when the attribute is accessed through the ' - '*owner*.\n' - '\n' - ' This method should return the computed attribute value or ' - 'raise an\n' - ' "AttributeError" exception.\n' - '\n' - ' **PEP 252** specifies that "__get__()" is callable with ' - 'one or two\n' - ' arguments. Python’s own built-in descriptors support ' - 'this\n' - ' specification; however, it is likely that some ' - 'third-party tools\n' - ' have descriptors that require both arguments. Python’s ' - 'own\n' - ' "__getattribute__()" implementation always passes in both ' - 'arguments\n' - ' whether they are required or not.\n' - '\n' - 'object.__set__(self, instance, value)\n' - '\n' - ' Called to set the attribute on an instance *instance* of ' - 'the owner\n' - ' class to a new value, *value*.\n' - '\n' - ' Note, adding "__set__()" or "__delete__()" changes the ' - 'kind of\n' - ' descriptor to a “data descriptor”. See Invoking ' - 'Descriptors for\n' - ' more details.\n' - '\n' - 'object.__delete__(self, instance)\n' - '\n' - ' Called to delete the attribute on an instance *instance* ' - 'of the\n' - ' owner class.\n' - '\n' - 'Instances of descriptors may also have the "__objclass__" ' - 'attribute\n' - 'present:\n' - '\n' - 'object.__objclass__\n' - '\n' - ' The attribute "__objclass__" is interpreted by the ' - '"inspect" module\n' - ' as specifying the class where this object was defined ' - '(setting this\n' - ' appropriately can assist in runtime introspection of ' - 'dynamic class\n' - ' attributes). For callables, it may indicate that an ' - 'instance of the\n' - ' given type (or a subclass) is expected or required as the ' - 'first\n' - ' positional argument (for example, CPython sets this ' - 'attribute for\n' - ' unbound methods that are implemented in C).\n' - '\n' - '\n' - 'Invoking Descriptors\n' - '--------------------\n' - '\n' - 'In general, a descriptor is an object attribute with ' - '“binding\n' - 'behavior”, one whose attribute access has been overridden by ' - 'methods\n' - 'in the descriptor protocol: "__get__()", "__set__()", and\n' - '"__delete__()". If any of those methods are defined for an ' - 'object, it\n' - 'is said to be a descriptor.\n' - '\n' - 'The default behavior for attribute access is to get, set, or ' - 'delete\n' - 'the attribute from an object’s dictionary. For instance, ' - '"a.x" has a\n' - 'lookup chain starting with "a.__dict__[\'x\']", then\n' - '"type(a).__dict__[\'x\']", and continuing through the base ' - 'classes of\n' - '"type(a)" excluding metaclasses.\n' - '\n' - 'However, if the looked-up value is an object defining one of ' - 'the\n' - 'descriptor methods, then Python may override the default ' - 'behavior and\n' - 'invoke the descriptor method instead. Where this occurs in ' - 'the\n' - 'precedence chain depends on which descriptor methods were ' - 'defined and\n' - 'how they were called.\n' - '\n' - 'The starting point for descriptor invocation is a binding, ' - '"a.x". How\n' - 'the arguments are assembled depends on "a":\n' - '\n' - 'Direct Call\n' - ' The simplest and least common call is when user code ' - 'directly\n' - ' invokes a descriptor method: "x.__get__(a)".\n' - '\n' - 'Instance Binding\n' - ' If binding to an object instance, "a.x" is transformed ' - 'into the\n' - ' call: "type(a).__dict__[\'x\'].__get__(a, type(a))".\n' - '\n' - 'Class Binding\n' - ' If binding to a class, "A.x" is transformed into the ' - 'call:\n' - ' "A.__dict__[\'x\'].__get__(None, A)".\n' - '\n' - 'Super Binding\n' - ' A dotted lookup such as "super(A, a).x" searches\n' - ' "a.__class__.__mro__" for a base class "B" following "A" ' - 'and then\n' - ' returns "B.__dict__[\'x\'].__get__(a, A)". If not a ' - 'descriptor, "x"\n' - ' is returned unchanged.\n' - '\n' - 'For instance bindings, the precedence of descriptor ' - 'invocation depends\n' - 'on which descriptor methods are defined. A descriptor can ' - 'define any\n' - 'combination of "__get__()", "__set__()" and "__delete__()". ' - 'If it\n' - 'does not define "__get__()", then accessing the attribute ' - 'will return\n' - 'the descriptor object itself unless there is a value in the ' - 'object’s\n' - 'instance dictionary. If the descriptor defines "__set__()" ' - 'and/or\n' - '"__delete__()", it is a data descriptor; if it defines ' - 'neither, it is\n' - 'a non-data descriptor. Normally, data descriptors define ' - 'both\n' - '"__get__()" and "__set__()", while non-data descriptors have ' - 'just the\n' - '"__get__()" method. Data descriptors with "__get__()" and ' - '"__set__()"\n' - '(and/or "__delete__()") defined always override a ' - 'redefinition in an\n' - 'instance dictionary. In contrast, non-data descriptors can ' - 'be\n' - 'overridden by instances.\n' - '\n' - 'Python methods (including those decorated with ' - '"@staticmethod" and\n' - '"@classmethod") are implemented as non-data descriptors. ' - 'Accordingly,\n' - 'instances can redefine and override methods. This allows ' - 'individual\n' - 'instances to acquire behaviors that differ from other ' - 'instances of the\n' - 'same class.\n' - '\n' - 'The "property()" function is implemented as a data ' - 'descriptor.\n' - 'Accordingly, instances cannot override the behavior of a ' - 'property.\n' - '\n' - '\n' - '__slots__\n' - '---------\n' - '\n' - '*__slots__* allow us to explicitly declare data members ' - '(like\n' - 'properties) and deny the creation of "__dict__" and ' - '*__weakref__*\n' - '(unless explicitly declared in *__slots__* or available in a ' - 'parent.)\n' - '\n' - 'The space saved over using "__dict__" can be significant. ' - 'Attribute\n' - 'lookup speed can be significantly improved as well.\n' - '\n' - 'object.__slots__\n' - '\n' - ' This class variable can be assigned a string, iterable, ' - 'or sequence\n' - ' of strings with variable names used by instances. ' - '*__slots__*\n' - ' reserves space for the declared variables and prevents ' - 'the\n' - ' automatic creation of "__dict__" and *__weakref__* for ' - 'each\n' - ' instance.\n' - '\n' - 'Notes on using *__slots__*:\n' - '\n' - '* When inheriting from a class without *__slots__*, the ' - '"__dict__" and\n' - ' *__weakref__* attribute of the instances will always be ' - 'accessible.\n' - '\n' - '* Without a "__dict__" variable, instances cannot be ' - 'assigned new\n' - ' variables not listed in the *__slots__* definition. ' - 'Attempts to\n' - ' assign to an unlisted variable name raises ' - '"AttributeError". If\n' - ' dynamic assignment of new variables is desired, then add\n' - ' "\'__dict__\'" to the sequence of strings in the ' - '*__slots__*\n' - ' declaration.\n' - '\n' - '* Without a *__weakref__* variable for each instance, ' - 'classes defining\n' - ' *__slots__* do not support "weak references" to its ' - 'instances. If\n' - ' weak reference support is needed, then add ' - '"\'__weakref__\'" to the\n' - ' sequence of strings in the *__slots__* declaration.\n' - '\n' - '* *__slots__* are implemented at the class level by ' - 'creating\n' - ' descriptors for each variable name. As a result, class ' - 'attributes\n' - ' cannot be used to set default values for instance ' - 'variables defined\n' - ' by *__slots__*; otherwise, the class attribute would ' - 'overwrite the\n' - ' descriptor assignment.\n' - '\n' - '* The action of a *__slots__* declaration is not limited to ' - 'the class\n' - ' where it is defined. *__slots__* declared in parents are ' - 'available\n' - ' in child classes. However, child subclasses will get a ' - '"__dict__"\n' - ' and *__weakref__* unless they also define *__slots__* ' - '(which should\n' - ' only contain names of any *additional* slots).\n' - '\n' - '* If a class defines a slot also defined in a base class, ' - 'the instance\n' - ' variable defined by the base class slot is inaccessible ' - '(except by\n' - ' retrieving its descriptor directly from the base class). ' - 'This\n' - ' renders the meaning of the program undefined. In the ' - 'future, a\n' - ' check may be added to prevent this.\n' - '\n' - '* "TypeError" will be raised if nonempty *__slots__* are ' - 'defined for a\n' - ' class derived from a ""variable-length" built-in type" ' - 'such as\n' - ' "int", "bytes", and "tuple".\n' - '\n' - '* Any non-string *iterable* may be assigned to *__slots__*.\n' - '\n' - '* If a "dictionary" is used to assign *__slots__*, the ' - 'dictionary keys\n' - ' will be used as the slot names. The values of the ' - 'dictionary can be\n' - ' used to provide per-attribute docstrings that will be ' - 'recognised by\n' - ' "inspect.getdoc()" and displayed in the output of ' - '"help()".\n' - '\n' - '* "__class__" assignment works only if both classes have the ' - 'same\n' - ' *__slots__*.\n' - '\n' - '* Multiple inheritance with multiple slotted parent classes ' - 'can be\n' - ' used, but only one parent is allowed to have attributes ' - 'created by\n' - ' slots (the other bases must have empty slot layouts) - ' - 'violations\n' - ' raise "TypeError".\n' - '\n' - '* If an *iterator* is used for *__slots__* then a ' - '*descriptor* is\n' - ' created for each of the iterator’s values. However, the ' - '*__slots__*\n' - ' attribute will be an empty iterator.\n' - '\n' - '\n' - 'Customizing class creation\n' - '==========================\n' - '\n' - 'Whenever a class inherits from another class, ' - '"__init_subclass__()" is\n' - 'called on the parent class. This way, it is possible to ' - 'write classes\n' - 'which change the behavior of subclasses. This is closely ' - 'related to\n' - 'class decorators, but where class decorators only affect the ' - 'specific\n' - 'class they’re applied to, "__init_subclass__" solely applies ' - 'to future\n' - 'subclasses of the class defining the method.\n' - '\n' - 'classmethod object.__init_subclass__(cls)\n' - '\n' - ' This method is called whenever the containing class is ' - 'subclassed.\n' - ' *cls* is then the new subclass. If defined as a normal ' - 'instance\n' - ' method, this method is implicitly converted to a class ' - 'method.\n' - '\n' - ' Keyword arguments which are given to a new class are ' - 'passed to the\n' - ' parent class’s "__init_subclass__". For compatibility ' - 'with other\n' - ' classes using "__init_subclass__", one should take out ' - 'the needed\n' - ' keyword arguments and pass the others over to the base ' - 'class, as\n' - ' in:\n' - '\n' - ' class Philosopher:\n' - ' def __init_subclass__(cls, /, default_name, ' - '**kwargs):\n' - ' super().__init_subclass__(**kwargs)\n' - ' cls.default_name = default_name\n' - '\n' - ' class AustralianPhilosopher(Philosopher, ' - 'default_name="Bruce"):\n' - ' pass\n' - '\n' - ' The default implementation "object.__init_subclass__" ' - 'does nothing,\n' - ' but raises an error if it is called with any arguments.\n' - '\n' - ' Note:\n' - '\n' - ' The metaclass hint "metaclass" is consumed by the rest ' - 'of the\n' - ' type machinery, and is never passed to ' - '"__init_subclass__"\n' - ' implementations. The actual metaclass (rather than the ' - 'explicit\n' - ' hint) can be accessed as "type(cls)".\n' - '\n' - ' New in version 3.6.\n' - '\n' - 'When a class is created, "type.__new__()" scans the class ' - 'variables\n' - 'and makes callbacks to those with a "__set_name__()" hook.\n' - '\n' - 'object.__set_name__(self, owner, name)\n' - '\n' - ' Automatically called at the time the owning class *owner* ' - 'is\n' - ' created. The object has been assigned to *name* in that ' - 'class:\n' - '\n' - ' class A:\n' - ' x = C() # Automatically calls: x.__set_name__(A, ' - "'x')\n" - '\n' - ' If the class variable is assigned after the class is ' - 'created,\n' - ' "__set_name__()" will not be called automatically. If ' - 'needed,\n' - ' "__set_name__()" can be called directly:\n' - '\n' - ' class A:\n' - ' pass\n' - '\n' - ' c = C()\n' - ' A.x = c # The hook is not called\n' - " c.__set_name__(A, 'x') # Manually invoke the hook\n" - '\n' - ' See Creating the class object for more details.\n' - '\n' - ' New in version 3.6.\n' - '\n' - '\n' - 'Metaclasses\n' - '-----------\n' - '\n' - 'By default, classes are constructed using "type()". The ' - 'class body is\n' - 'executed in a new namespace and the class name is bound ' - 'locally to the\n' - 'result of "type(name, bases, namespace)".\n' - '\n' - 'The class creation process can be customized by passing the\n' - '"metaclass" keyword argument in the class definition line, ' - 'or by\n' - 'inheriting from an existing class that included such an ' - 'argument. In\n' - 'the following example, both "MyClass" and "MySubclass" are ' - 'instances\n' - 'of "Meta":\n' - '\n' - ' class Meta(type):\n' - ' pass\n' - '\n' - ' class MyClass(metaclass=Meta):\n' - ' pass\n' - '\n' - ' class MySubclass(MyClass):\n' - ' pass\n' - '\n' - 'Any other keyword arguments that are specified in the class ' - 'definition\n' - 'are passed through to all metaclass operations described ' - 'below.\n' - '\n' - 'When a class definition is executed, the following steps ' - 'occur:\n' - '\n' - '* MRO entries are resolved;\n' - '\n' - '* the appropriate metaclass is determined;\n' - '\n' - '* the class namespace is prepared;\n' - '\n' - '* the class body is executed;\n' - '\n' - '* the class object is created.\n' - '\n' - '\n' - 'Resolving MRO entries\n' - '---------------------\n' - '\n' - 'object.__mro_entries__(self, bases)\n' - '\n' - ' If a base that appears in a class definition is not an ' - 'instance of\n' - ' "type", then an "__mro_entries__()" method is searched on ' - 'the base.\n' - ' If an "__mro_entries__()" method is found, the base is ' - 'substituted\n' - ' with the result of a call to "__mro_entries__()" when ' - 'creating the\n' - ' class. The method is called with the original bases tuple ' - 'passed to\n' - ' the *bases* parameter, and must return a tuple of classes ' - 'that will\n' - ' be used instead of the base. The returned tuple may be ' - 'empty: in\n' - ' these cases, the original base is ignored.\n' - '\n' - 'See also:\n' - '\n' - ' "types.resolve_bases()"\n' - ' Dynamically resolve bases that are not instances of ' - '"type".\n' - '\n' - ' "types.get_original_bases()"\n' - ' Retrieve a class’s “original bases” prior to ' - 'modifications by\n' - ' "__mro_entries__()".\n' - '\n' - ' **PEP 560**\n' - ' Core support for typing module and generic types.\n' - '\n' - '\n' - 'Determining the appropriate metaclass\n' - '-------------------------------------\n' - '\n' - 'The appropriate metaclass for a class definition is ' - 'determined as\n' - 'follows:\n' - '\n' - '* if no bases and no explicit metaclass are given, then ' - '"type()" is\n' - ' used;\n' - '\n' - '* if an explicit metaclass is given and it is *not* an ' - 'instance of\n' - ' "type()", then it is used directly as the metaclass;\n' - '\n' - '* if an instance of "type()" is given as the explicit ' - 'metaclass, or\n' - ' bases are defined, then the most derived metaclass is ' - 'used.\n' - '\n' - 'The most derived metaclass is selected from the explicitly ' - 'specified\n' - 'metaclass (if any) and the metaclasses (i.e. "type(cls)") of ' - 'all\n' - 'specified base classes. The most derived metaclass is one ' - 'which is a\n' - 'subtype of *all* of these candidate metaclasses. If none of ' - 'the\n' - 'candidate metaclasses meets that criterion, then the class ' - 'definition\n' - 'will fail with "TypeError".\n' - '\n' - '\n' - 'Preparing the class namespace\n' - '-----------------------------\n' - '\n' - 'Once the appropriate metaclass has been identified, then the ' - 'class\n' - 'namespace is prepared. If the metaclass has a "__prepare__" ' - 'attribute,\n' - 'it is called as "namespace = metaclass.__prepare__(name, ' - 'bases,\n' - '**kwds)" (where the additional keyword arguments, if any, ' - 'come from\n' - 'the class definition). The "__prepare__" method should be ' - 'implemented\n' - 'as a "classmethod". The namespace returned by "__prepare__" ' - 'is passed\n' - 'in to "__new__", but when the final class object is created ' - 'the\n' - 'namespace is copied into a new "dict".\n' - '\n' - 'If the metaclass has no "__prepare__" attribute, then the ' - 'class\n' - 'namespace is initialised as an empty ordered mapping.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3115** - Metaclasses in Python 3000\n' - ' Introduced the "__prepare__" namespace hook\n' - '\n' - '\n' - 'Executing the class body\n' - '------------------------\n' - '\n' - 'The class body is executed (approximately) as "exec(body, ' - 'globals(),\n' - 'namespace)". The key difference from a normal call to ' - '"exec()" is that\n' - 'lexical scoping allows the class body (including any ' - 'methods) to\n' - 'reference names from the current and outer scopes when the ' - 'class\n' - 'definition occurs inside a function.\n' - '\n' - 'However, even when the class definition occurs inside the ' - 'function,\n' - 'methods defined inside the class still cannot see names ' - 'defined at the\n' - 'class scope. Class variables must be accessed through the ' - 'first\n' - 'parameter of instance or class methods, or through the ' - 'implicit\n' - 'lexically scoped "__class__" reference described in the next ' - 'section.\n' - '\n' - '\n' - 'Creating the class object\n' - '-------------------------\n' - '\n' - 'Once the class namespace has been populated by executing the ' - 'class\n' - 'body, the class object is created by calling ' - '"metaclass(name, bases,\n' - 'namespace, **kwds)" (the additional keywords passed here are ' - 'the same\n' - 'as those passed to "__prepare__").\n' - '\n' - 'This class object is the one that will be referenced by the ' - 'zero-\n' - 'argument form of "super()". "__class__" is an implicit ' - 'closure\n' - 'reference created by the compiler if any methods in a class ' - 'body refer\n' - 'to either "__class__" or "super". This allows the zero ' - 'argument form\n' - 'of "super()" to correctly identify the class being defined ' - 'based on\n' - 'lexical scoping, while the class or instance that was used ' - 'to make the\n' - 'current call is identified based on the first argument ' - 'passed to the\n' - 'method.\n' - '\n' - '**CPython implementation detail:** In CPython 3.6 and later, ' - 'the\n' - '"__class__" cell is passed to the metaclass as a ' - '"__classcell__" entry\n' - 'in the class namespace. If present, this must be propagated ' - 'up to the\n' - '"type.__new__" call in order for the class to be ' - 'initialised\n' - 'correctly. Failing to do so will result in a "RuntimeError" ' - 'in Python\n' - '3.8.\n' - '\n' - 'When using the default metaclass "type", or any metaclass ' - 'that\n' - 'ultimately calls "type.__new__", the following additional\n' - 'customization steps are invoked after creating the class ' - 'object:\n' - '\n' - '1. The "type.__new__" method collects all of the attributes ' - 'in the\n' - ' class namespace that define a "__set_name__()" method;\n' - '\n' - '2. Those "__set_name__" methods are called with the class ' - 'being\n' - ' defined and the assigned name of that particular ' - 'attribute;\n' - '\n' - '3. The "__init_subclass__()" hook is called on the immediate ' - 'parent of\n' - ' the new class in its method resolution order.\n' - '\n' - 'After the class object is created, it is passed to the ' - 'class\n' - 'decorators included in the class definition (if any) and the ' - 'resulting\n' - 'object is bound in the local namespace as the defined ' - 'class.\n' - '\n' - 'When a new class is created by "type.__new__", the object ' - 'provided as\n' - 'the namespace parameter is copied to a new ordered mapping ' - 'and the\n' - 'original object is discarded. The new copy is wrapped in a ' - 'read-only\n' - 'proxy, which becomes the "__dict__" attribute of the class ' - 'object.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3135** - New super\n' - ' Describes the implicit "__class__" closure reference\n' - '\n' - '\n' - 'Uses for metaclasses\n' - '--------------------\n' - '\n' - 'The potential uses for metaclasses are boundless. Some ideas ' - 'that have\n' - 'been explored include enum, logging, interface checking, ' - 'automatic\n' - 'delegation, automatic property creation, proxies, ' - 'frameworks, and\n' - 'automatic resource locking/synchronization.\n' - '\n' - '\n' - 'Customizing instance and subclass checks\n' - '========================================\n' - '\n' - 'The following methods are used to override the default ' - 'behavior of the\n' - '"isinstance()" and "issubclass()" built-in functions.\n' - '\n' - 'In particular, the metaclass "abc.ABCMeta" implements these ' - 'methods in\n' - 'order to allow the addition of Abstract Base Classes (ABCs) ' - 'as\n' - '“virtual base classes” to any class or type (including ' - 'built-in\n' - 'types), including other ABCs.\n' - '\n' - 'class.__instancecheck__(self, instance)\n' - '\n' - ' Return true if *instance* should be considered a (direct ' - 'or\n' - ' indirect) instance of *class*. If defined, called to ' - 'implement\n' - ' "isinstance(instance, class)".\n' - '\n' - 'class.__subclasscheck__(self, subclass)\n' - '\n' - ' Return true if *subclass* should be considered a (direct ' - 'or\n' - ' indirect) subclass of *class*. If defined, called to ' - 'implement\n' - ' "issubclass(subclass, class)".\n' - '\n' - 'Note that these methods are looked up on the type ' - '(metaclass) of a\n' - 'class. They cannot be defined as class methods in the ' - 'actual class.\n' - 'This is consistent with the lookup of special methods that ' - 'are called\n' - 'on instances, only in this case the instance is itself a ' - 'class.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3119** - Introducing Abstract Base Classes\n' - ' Includes the specification for customizing ' - '"isinstance()" and\n' - ' "issubclass()" behavior through "__instancecheck__()" ' - 'and\n' - ' "__subclasscheck__()", with motivation for this ' - 'functionality in\n' - ' the context of adding Abstract Base Classes (see the ' - '"abc"\n' - ' module) to the language.\n' - '\n' - '\n' - 'Emulating generic types\n' - '=======================\n' - '\n' - 'When using *type annotations*, it is often useful to ' - '*parameterize* a\n' - '*generic type* using Python’s square-brackets notation. For ' - 'example,\n' - 'the annotation "list[int]" might be used to signify a "list" ' - 'in which\n' - 'all the elements are of type "int".\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 484** - Type Hints\n' - ' Introducing Python’s framework for type annotations\n' - '\n' - ' Generic Alias Types\n' - ' Documentation for objects representing parameterized ' - 'generic\n' - ' classes\n' - '\n' - ' Generics, user-defined generics and "typing.Generic"\n' - ' Documentation on how to implement generic classes that ' - 'can be\n' - ' parameterized at runtime and understood by static ' - 'type-checkers.\n' - '\n' - 'A class can *generally* only be parameterized if it defines ' - 'the\n' - 'special class method "__class_getitem__()".\n' - '\n' - 'classmethod object.__class_getitem__(cls, key)\n' - '\n' - ' Return an object representing the specialization of a ' - 'generic class\n' - ' by type arguments found in *key*.\n' - '\n' - ' When defined on a class, "__class_getitem__()" is ' - 'automatically a\n' - ' class method. As such, there is no need for it to be ' - 'decorated with\n' - ' "@classmethod" when it is defined.\n' - '\n' - '\n' - 'The purpose of *__class_getitem__*\n' - '----------------------------------\n' - '\n' - 'The purpose of "__class_getitem__()" is to allow runtime\n' - 'parameterization of standard-library generic classes in ' - 'order to more\n' - 'easily apply *type hints* to these classes.\n' - '\n' - 'To implement custom generic classes that can be ' - 'parameterized at\n' - 'runtime and understood by static type-checkers, users should ' - 'either\n' - 'inherit from a standard library class that already ' - 'implements\n' - '"__class_getitem__()", or inherit from "typing.Generic", ' - 'which has its\n' - 'own implementation of "__class_getitem__()".\n' - '\n' - 'Custom implementations of "__class_getitem__()" on classes ' - 'defined\n' - 'outside of the standard library may not be understood by ' - 'third-party\n' - 'type-checkers such as mypy. Using "__class_getitem__()" on ' - 'any class\n' - 'for purposes other than type hinting is discouraged.\n' - '\n' - '\n' - '*__class_getitem__* versus *__getitem__*\n' - '----------------------------------------\n' - '\n' - 'Usually, the subscription of an object using square brackets ' - 'will call\n' - 'the "__getitem__()" instance method defined on the object’s ' - 'class.\n' - 'However, if the object being subscribed is itself a class, ' - 'the class\n' - 'method "__class_getitem__()" may be called instead.\n' - '"__class_getitem__()" should return a GenericAlias object if ' - 'it is\n' - 'properly defined.\n' - '\n' - 'Presented with the *expression* "obj[x]", the Python ' - 'interpreter\n' - 'follows something like the following process to decide ' - 'whether\n' - '"__getitem__()" or "__class_getitem__()" should be called:\n' - '\n' - ' from inspect import isclass\n' - '\n' - ' def subscribe(obj, x):\n' - ' """Return the result of the expression \'obj[x]\'"""\n' - '\n' - ' class_of_obj = type(obj)\n' - '\n' - ' # If the class of obj defines __getitem__,\n' - ' # call class_of_obj.__getitem__(obj, x)\n' - " if hasattr(class_of_obj, '__getitem__'):\n" - ' return class_of_obj.__getitem__(obj, x)\n' - '\n' - ' # Else, if obj is a class and defines ' - '__class_getitem__,\n' - ' # call obj.__class_getitem__(x)\n' - ' elif isclass(obj) and hasattr(obj, ' - "'__class_getitem__'):\n" - ' return obj.__class_getitem__(x)\n' - '\n' - ' # Else, raise an exception\n' - ' else:\n' - ' raise TypeError(\n' - ' f"\'{class_of_obj.__name__}\' object is not ' - 'subscriptable"\n' - ' )\n' - '\n' - 'In Python, all classes are themselves instances of other ' - 'classes. The\n' - 'class of a class is known as that class’s *metaclass*, and ' - 'most\n' - 'classes have the "type" class as their metaclass. "type" ' - 'does not\n' - 'define "__getitem__()", meaning that expressions such as ' - '"list[int]",\n' - '"dict[str, float]" and "tuple[str, bytes]" all result in\n' - '"__class_getitem__()" being called:\n' - '\n' - ' >>> # list has class "type" as its metaclass, like most ' - 'classes:\n' - ' >>> type(list)\n' - " \n" - ' >>> type(dict) == type(list) == type(tuple) == type(str) ' - '== type(bytes)\n' - ' True\n' - ' >>> # "list[int]" calls "list.__class_getitem__(int)"\n' - ' >>> list[int]\n' - ' list[int]\n' - ' >>> # list.__class_getitem__ returns a GenericAlias ' - 'object:\n' - ' >>> type(list[int])\n' - " \n" - '\n' - 'However, if a class has a custom metaclass that defines\n' - '"__getitem__()", subscribing the class may result in ' - 'different\n' - 'behaviour. An example of this can be found in the "enum" ' - 'module:\n' - '\n' - ' >>> from enum import Enum\n' - ' >>> class Menu(Enum):\n' - ' ... """A breakfast menu"""\n' - " ... SPAM = 'spam'\n" - " ... BACON = 'bacon'\n" - ' ...\n' - ' >>> # Enum classes have a custom metaclass:\n' - ' >>> type(Menu)\n' - " \n" - ' >>> # EnumMeta defines __getitem__,\n' - ' >>> # so __class_getitem__ is not called,\n' - ' >>> # and the result is not a GenericAlias object:\n' - " >>> Menu['SPAM']\n" - " \n" - " >>> type(Menu['SPAM'])\n" - " \n" - '\n' - 'See also:\n' - '\n' - ' **PEP 560** - Core Support for typing module and generic ' - 'types\n' - ' Introducing "__class_getitem__()", and outlining when ' - 'a\n' - ' subscription results in "__class_getitem__()" being ' - 'called\n' - ' instead of "__getitem__()"\n' - '\n' - '\n' - 'Emulating callable objects\n' - '==========================\n' - '\n' - 'object.__call__(self[, args...])\n' - '\n' - ' Called when the instance is “called” as a function; if ' - 'this method\n' - ' is defined, "x(arg1, arg2, ...)" roughly translates to\n' - ' "type(x).__call__(x, arg1, ...)".\n' - '\n' - '\n' - 'Emulating container types\n' - '=========================\n' - '\n' - 'The following methods can be defined to implement container ' - 'objects.\n' - 'Containers usually are *sequences* (such as "lists" or ' - '"tuples") or\n' - '*mappings* (like "dictionaries"), but can represent other ' - 'containers\n' - 'as well. The first set of methods is used either to emulate ' - 'a\n' - 'sequence or to emulate a mapping; the difference is that for ' - 'a\n' - 'sequence, the allowable keys should be the integers *k* for ' - 'which "0\n' - '<= k < N" where *N* is the length of the sequence, or ' - '"slice" objects,\n' - 'which define a range of items. It is also recommended that ' - 'mappings\n' - 'provide the methods "keys()", "values()", "items()", ' - '"get()",\n' - '"clear()", "setdefault()", "pop()", "popitem()", "copy()", ' - 'and\n' - '"update()" behaving similar to those for Python’s standard\n' - '"dictionary" objects. The "collections.abc" module provides ' - 'a\n' - '"MutableMapping" *abstract base class* to help create those ' - 'methods\n' - 'from a base set of "__getitem__()", "__setitem__()", ' - '"__delitem__()",\n' - 'and "keys()". Mutable sequences should provide methods ' - '"append()",\n' - '"count()", "index()", "extend()", "insert()", "pop()", ' - '"remove()",\n' - '"reverse()" and "sort()", like Python standard "list" ' - 'objects.\n' - 'Finally, sequence types should implement addition (meaning\n' - 'concatenation) and multiplication (meaning repetition) by ' - 'defining the\n' - 'methods "__add__()", "__radd__()", "__iadd__()", ' - '"__mul__()",\n' - '"__rmul__()" and "__imul__()" described below; they should ' - 'not define\n' - 'other numerical operators. It is recommended that both ' - 'mappings and\n' - 'sequences implement the "__contains__()" method to allow ' - 'efficient use\n' - 'of the "in" operator; for mappings, "in" should search the ' - 'mapping’s\n' - 'keys; for sequences, it should search through the values. ' - 'It is\n' - 'further recommended that both mappings and sequences ' - 'implement the\n' - '"__iter__()" method to allow efficient iteration through ' - 'the\n' - 'container; for mappings, "__iter__()" should iterate through ' - 'the\n' - 'object’s keys; for sequences, it should iterate through the ' - 'values.\n' - '\n' - 'object.__len__(self)\n' - '\n' - ' Called to implement the built-in function "len()". ' - 'Should return\n' - ' the length of the object, an integer ">=" 0. Also, an ' - 'object that\n' - ' doesn’t define a "__bool__()" method and whose ' - '"__len__()" method\n' - ' returns zero is considered to be false in a Boolean ' - 'context.\n' - '\n' - ' **CPython implementation detail:** In CPython, the length ' - 'is\n' - ' required to be at most "sys.maxsize". If the length is ' - 'larger than\n' - ' "sys.maxsize" some features (such as "len()") may raise\n' - ' "OverflowError". To prevent raising "OverflowError" by ' - 'truth value\n' - ' testing, an object must define a "__bool__()" method.\n' - '\n' - 'object.__length_hint__(self)\n' - '\n' - ' Called to implement "operator.length_hint()". Should ' - 'return an\n' - ' estimated length for the object (which may be greater or ' - 'less than\n' - ' the actual length). The length must be an integer ">=" 0. ' - 'The\n' - ' return value may also be "NotImplemented", which is ' - 'treated the\n' - ' same as if the "__length_hint__" method didn’t exist at ' - 'all. This\n' - ' method is purely an optimization and is never required ' - 'for\n' - ' correctness.\n' - '\n' - ' New in version 3.4.\n' - '\n' - 'Note:\n' - '\n' - ' Slicing is done exclusively with the following three ' - 'methods. A\n' - ' call like\n' - '\n' - ' a[1:2] = b\n' - '\n' - ' is translated to\n' - '\n' - ' a[slice(1, 2, None)] = b\n' - '\n' - ' and so forth. Missing slice items are always filled in ' - 'with "None".\n' - '\n' - 'object.__getitem__(self, key)\n' - '\n' - ' Called to implement evaluation of "self[key]". For ' - '*sequence*\n' - ' types, the accepted keys should be integers. Optionally, ' - 'they may\n' - ' support "slice" objects as well. Negative index support ' - 'is also\n' - ' optional. If *key* is of an inappropriate type, ' - '"TypeError" may be\n' - ' raised; if *key* is a value outside the set of indexes ' - 'for the\n' - ' sequence (after any special interpretation of negative ' - 'values),\n' - ' "IndexError" should be raised. For *mapping* types, if ' - '*key* is\n' - ' missing (not in the container), "KeyError" should be ' - 'raised.\n' - '\n' - ' Note:\n' - '\n' - ' "for" loops expect that an "IndexError" will be raised ' - 'for\n' - ' illegal indexes to allow proper detection of the end of ' - 'the\n' - ' sequence.\n' - '\n' - ' Note:\n' - '\n' - ' When subscripting a *class*, the special class method\n' - ' "__class_getitem__()" may be called instead of ' - '"__getitem__()".\n' - ' See __class_getitem__ versus __getitem__ for more ' - 'details.\n' - '\n' - 'object.__setitem__(self, key, value)\n' - '\n' - ' Called to implement assignment to "self[key]". Same note ' - 'as for\n' - ' "__getitem__()". This should only be implemented for ' - 'mappings if\n' - ' the objects support changes to the values for keys, or if ' - 'new keys\n' - ' can be added, or for sequences if elements can be ' - 'replaced. The\n' - ' same exceptions should be raised for improper *key* ' - 'values as for\n' - ' the "__getitem__()" method.\n' - '\n' - 'object.__delitem__(self, key)\n' - '\n' - ' Called to implement deletion of "self[key]". Same note ' - 'as for\n' - ' "__getitem__()". This should only be implemented for ' - 'mappings if\n' - ' the objects support removal of keys, or for sequences if ' - 'elements\n' - ' can be removed from the sequence. The same exceptions ' - 'should be\n' - ' raised for improper *key* values as for the ' - '"__getitem__()" method.\n' - '\n' - 'object.__missing__(self, key)\n' - '\n' - ' Called by "dict"."__getitem__()" to implement "self[key]" ' - 'for dict\n' - ' subclasses when key is not in the dictionary.\n' - '\n' - 'object.__iter__(self)\n' - '\n' - ' This method is called when an *iterator* is required for ' - 'a\n' - ' container. This method should return a new iterator ' - 'object that can\n' - ' iterate over all the objects in the container. For ' - 'mappings, it\n' - ' should iterate over the keys of the container.\n' - '\n' - 'object.__reversed__(self)\n' - '\n' - ' Called (if present) by the "reversed()" built-in to ' - 'implement\n' - ' reverse iteration. It should return a new iterator ' - 'object that\n' - ' iterates over all the objects in the container in reverse ' - 'order.\n' - '\n' - ' If the "__reversed__()" method is not provided, the ' - '"reversed()"\n' - ' built-in will fall back to using the sequence protocol ' - '("__len__()"\n' - ' and "__getitem__()"). Objects that support the sequence ' - 'protocol\n' - ' should only provide "__reversed__()" if they can provide ' - 'an\n' - ' implementation that is more efficient than the one ' - 'provided by\n' - ' "reversed()".\n' - '\n' - 'The membership test operators ("in" and "not in") are ' - 'normally\n' - 'implemented as an iteration through a container. However, ' - 'container\n' - 'objects can supply the following special method with a more ' - 'efficient\n' - 'implementation, which also does not require the object be ' - 'iterable.\n' - '\n' - 'object.__contains__(self, item)\n' - '\n' - ' Called to implement membership test operators. Should ' - 'return true\n' - ' if *item* is in *self*, false otherwise. For mapping ' - 'objects, this\n' - ' should consider the keys of the mapping rather than the ' - 'values or\n' - ' the key-item pairs.\n' - '\n' - ' For objects that don’t define "__contains__()", the ' - 'membership test\n' - ' first tries iteration via "__iter__()", then the old ' - 'sequence\n' - ' iteration protocol via "__getitem__()", see this section ' - 'in the\n' - ' language reference.\n' - '\n' - '\n' - 'Emulating numeric types\n' - '=======================\n' - '\n' - 'The following methods can be defined to emulate numeric ' - 'objects.\n' - 'Methods corresponding to operations that are not supported ' - 'by the\n' - 'particular kind of number implemented (e.g., bitwise ' - 'operations for\n' - 'non-integral numbers) should be left undefined.\n' - '\n' - 'object.__add__(self, other)\n' - 'object.__sub__(self, other)\n' - 'object.__mul__(self, other)\n' - 'object.__matmul__(self, other)\n' - 'object.__truediv__(self, other)\n' - 'object.__floordiv__(self, other)\n' - 'object.__mod__(self, other)\n' - 'object.__divmod__(self, other)\n' - 'object.__pow__(self, other[, modulo])\n' - 'object.__lshift__(self, other)\n' - 'object.__rshift__(self, other)\n' - 'object.__and__(self, other)\n' - 'object.__xor__(self, other)\n' - 'object.__or__(self, other)\n' - '\n' - ' These methods are called to implement the binary ' - 'arithmetic\n' - ' operations ("+", "-", "*", "@", "/", "//", "%", ' - '"divmod()",\n' - ' "pow()", "**", "<<", ">>", "&", "^", "|"). For instance, ' - 'to\n' - ' evaluate the expression "x + y", where *x* is an instance ' - 'of a\n' - ' class that has an "__add__()" method, "type(x).__add__(x, ' - 'y)" is\n' - ' called. The "__divmod__()" method should be the ' - 'equivalent to\n' - ' using "__floordiv__()" and "__mod__()"; it should not be ' - 'related to\n' - ' "__truediv__()". Note that "__pow__()" should be defined ' - 'to accept\n' - ' an optional third argument if the ternary version of the ' - 'built-in\n' - ' "pow()" function is to be supported.\n' - '\n' - ' If one of those methods does not support the operation ' - 'with the\n' - ' supplied arguments, it should return "NotImplemented".\n' - '\n' - 'object.__radd__(self, other)\n' - 'object.__rsub__(self, other)\n' - 'object.__rmul__(self, other)\n' - 'object.__rmatmul__(self, other)\n' - 'object.__rtruediv__(self, other)\n' - 'object.__rfloordiv__(self, other)\n' - 'object.__rmod__(self, other)\n' - 'object.__rdivmod__(self, other)\n' - 'object.__rpow__(self, other[, modulo])\n' - 'object.__rlshift__(self, other)\n' - 'object.__rrshift__(self, other)\n' - 'object.__rand__(self, other)\n' - 'object.__rxor__(self, other)\n' - 'object.__ror__(self, other)\n' - '\n' - ' These methods are called to implement the binary ' - 'arithmetic\n' - ' operations ("+", "-", "*", "@", "/", "//", "%", ' - '"divmod()",\n' - ' "pow()", "**", "<<", ">>", "&", "^", "|") with reflected ' - '(swapped)\n' - ' operands. These functions are only called if the left ' - 'operand does\n' - ' not support the corresponding operation [3] and the ' - 'operands are of\n' - ' different types. [4] For instance, to evaluate the ' - 'expression "x -\n' - ' y", where *y* is an instance of a class that has an ' - '"__rsub__()"\n' - ' method, "type(y).__rsub__(y, x)" is called if ' - '"type(x).__sub__(x,\n' - ' y)" returns *NotImplemented*.\n' - '\n' - ' Note that ternary "pow()" will not try calling ' - '"__rpow__()" (the\n' - ' coercion rules would become too complicated).\n' - '\n' - ' Note:\n' - '\n' - ' If the right operand’s type is a subclass of the left ' - 'operand’s\n' - ' type and that subclass provides a different ' - 'implementation of the\n' - ' reflected method for the operation, this method will be ' - 'called\n' - ' before the left operand’s non-reflected method. This ' - 'behavior\n' - ' allows subclasses to override their ancestors’ ' - 'operations.\n' - '\n' - 'object.__iadd__(self, other)\n' - 'object.__isub__(self, other)\n' - 'object.__imul__(self, other)\n' - 'object.__imatmul__(self, other)\n' - 'object.__itruediv__(self, other)\n' - 'object.__ifloordiv__(self, other)\n' - 'object.__imod__(self, other)\n' - 'object.__ipow__(self, other[, modulo])\n' - 'object.__ilshift__(self, other)\n' - 'object.__irshift__(self, other)\n' - 'object.__iand__(self, other)\n' - 'object.__ixor__(self, other)\n' - 'object.__ior__(self, other)\n' - '\n' - ' These methods are called to implement the augmented ' - 'arithmetic\n' - ' assignments ("+=", "-=", "*=", "@=", "/=", "//=", "%=", ' - '"**=",\n' - ' "<<=", ">>=", "&=", "^=", "|="). These methods should ' - 'attempt to\n' - ' do the operation in-place (modifying *self*) and return ' - 'the result\n' - ' (which could be, but does not have to be, *self*). If a ' - 'specific\n' - ' method is not defined, the augmented assignment falls ' - 'back to the\n' - ' normal methods. For instance, if *x* is an instance of a ' - 'class\n' - ' with an "__iadd__()" method, "x += y" is equivalent to "x ' - '=\n' - ' x.__iadd__(y)" . Otherwise, "x.__add__(y)" and ' - '"y.__radd__(x)" are\n' - ' considered, as with the evaluation of "x + y". In ' - 'certain\n' - ' situations, augmented assignment can result in unexpected ' - 'errors\n' - ' (see Why does a_tuple[i] += [‘item’] raise an exception ' - 'when the\n' - ' addition works?), but this behavior is in fact part of ' - 'the data\n' - ' model.\n' - '\n' - 'object.__neg__(self)\n' - 'object.__pos__(self)\n' - 'object.__abs__(self)\n' - 'object.__invert__(self)\n' - '\n' - ' Called to implement the unary arithmetic operations ("-", ' - '"+",\n' - ' "abs()" and "~").\n' - '\n' - 'object.__complex__(self)\n' - 'object.__int__(self)\n' - 'object.__float__(self)\n' - '\n' - ' Called to implement the built-in functions "complex()", ' - '"int()" and\n' - ' "float()". Should return a value of the appropriate ' - 'type.\n' - '\n' - 'object.__index__(self)\n' - '\n' - ' Called to implement "operator.index()", and whenever ' - 'Python needs\n' - ' to losslessly convert the numeric object to an integer ' - 'object (such\n' - ' as in slicing, or in the built-in "bin()", "hex()" and ' - '"oct()"\n' - ' functions). Presence of this method indicates that the ' - 'numeric\n' - ' object is an integer type. Must return an integer.\n' - '\n' - ' If "__int__()", "__float__()" and "__complex__()" are not ' - 'defined\n' - ' then corresponding built-in functions "int()", "float()" ' - 'and\n' - ' "complex()" fall back to "__index__()".\n' - '\n' - 'object.__round__(self[, ndigits])\n' - 'object.__trunc__(self)\n' - 'object.__floor__(self)\n' - 'object.__ceil__(self)\n' - '\n' - ' Called to implement the built-in function "round()" and ' - '"math"\n' - ' functions "trunc()", "floor()" and "ceil()". Unless ' - '*ndigits* is\n' - ' passed to "__round__()" all these methods should return ' - 'the value\n' - ' of the object truncated to an "Integral" (typically an ' - '"int").\n' - '\n' - ' The built-in function "int()" falls back to "__trunc__()" ' - 'if\n' - ' neither "__int__()" nor "__index__()" is defined.\n' - '\n' - ' Changed in version 3.11: The delegation of "int()" to ' - '"__trunc__()"\n' - ' is deprecated.\n' - '\n' - '\n' - 'With Statement Context Managers\n' - '===============================\n' - '\n' - 'A *context manager* is an object that defines the runtime ' - 'context to\n' - 'be established when executing a "with" statement. The ' - 'context manager\n' - 'handles the entry into, and the exit from, the desired ' - 'runtime context\n' - 'for the execution of the block of code. Context managers ' - 'are normally\n' - 'invoked using the "with" statement (described in section The ' - 'with\n' - 'statement), but can also be used by directly invoking their ' - 'methods.\n' - '\n' - 'Typical uses of context managers include saving and ' - 'restoring various\n' - 'kinds of global state, locking and unlocking resources, ' - 'closing opened\n' - 'files, etc.\n' - '\n' - 'For more information on context managers, see Context ' - 'Manager Types.\n' - '\n' - 'object.__enter__(self)\n' - '\n' - ' Enter the runtime context related to this object. The ' - '"with"\n' - ' statement will bind this method’s return value to the ' - 'target(s)\n' - ' specified in the "as" clause of the statement, if any.\n' - '\n' - 'object.__exit__(self, exc_type, exc_value, traceback)\n' - '\n' - ' Exit the runtime context related to this object. The ' - 'parameters\n' - ' describe the exception that caused the context to be ' - 'exited. If the\n' - ' context was exited without an exception, all three ' - 'arguments will\n' - ' be "None".\n' - '\n' - ' If an exception is supplied, and the method wishes to ' - 'suppress the\n' - ' exception (i.e., prevent it from being propagated), it ' - 'should\n' - ' return a true value. Otherwise, the exception will be ' - 'processed\n' - ' normally upon exit from this method.\n' - '\n' - ' Note that "__exit__()" methods should not reraise the ' - 'passed-in\n' - ' exception; this is the caller’s responsibility.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 343** - The “with” statement\n' - ' The specification, background, and examples for the ' - 'Python "with"\n' - ' statement.\n' - '\n' - '\n' - 'Customizing positional arguments in class pattern matching\n' - '==========================================================\n' - '\n' - 'When using a class name in a pattern, positional arguments ' - 'in the\n' - 'pattern are not allowed by default, i.e. "case MyClass(x, ' - 'y)" is\n' - 'typically invalid without special support in "MyClass". To ' - 'be able to\n' - 'use that kind of pattern, the class needs to define a ' - '*__match_args__*\n' - 'attribute.\n' - '\n' - 'object.__match_args__\n' - '\n' - ' This class variable can be assigned a tuple of strings. ' - 'When this\n' - ' class is used in a class pattern with positional ' - 'arguments, each\n' - ' positional argument will be converted into a keyword ' - 'argument,\n' - ' using the corresponding value in *__match_args__* as the ' - 'keyword.\n' - ' The absence of this attribute is equivalent to setting it ' - 'to "()".\n' - '\n' - 'For example, if "MyClass.__match_args__" is "("left", ' - '"center",\n' - '"right")" that means that "case MyClass(x, y)" is equivalent ' - 'to "case\n' - 'MyClass(left=x, center=y)". Note that the number of ' - 'arguments in the\n' - 'pattern must be smaller than or equal to the number of ' - 'elements in\n' - '*__match_args__*; if it is larger, the pattern match attempt ' - 'will\n' - 'raise a "TypeError".\n' - '\n' - 'New in version 3.10.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 634** - Structural Pattern Matching\n' - ' The specification for the Python "match" statement.\n' - '\n' - '\n' - 'Emulating buffer types\n' - '======================\n' - '\n' - 'The buffer protocol provides a way for Python objects to ' - 'expose\n' - 'efficient access to a low-level memory array. This protocol ' - 'is\n' - 'implemented by builtin types such as "bytes" and ' - '"memoryview", and\n' - 'third-party libraries may define additional buffer types.\n' - '\n' - 'While buffer types are usually implemented in C, it is also ' - 'possible\n' - 'to implement the protocol in Python.\n' - '\n' - 'object.__buffer__(self, flags)\n' - '\n' - ' Called when a buffer is requested from *self* (for ' - 'example, by the\n' - ' "memoryview" constructor). The *flags* argument is an ' - 'integer\n' - ' representing the kind of buffer requested, affecting for ' - 'example\n' - ' whether the returned buffer is read-only or writable.\n' - ' "inspect.BufferFlags" provides a convenient way to ' - 'interpret the\n' - ' flags. The method must return a "memoryview" object.\n' - '\n' - 'object.__release_buffer__(self, buffer)\n' - '\n' - ' Called when a buffer is no longer needed. The *buffer* ' - 'argument is\n' - ' a "memoryview" object that was previously returned by\n' - ' "__buffer__()". The method must release any resources ' - 'associated\n' - ' with the buffer. This method should return "None". Buffer ' - 'objects\n' - ' that do not need to perform any cleanup are not required ' - 'to\n' - ' implement this method.\n' - '\n' - 'New in version 3.12.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 688** - Making the buffer protocol accessible in ' - 'Python\n' - ' Introduces the Python "__buffer__" and ' - '"__release_buffer__"\n' - ' methods.\n' - '\n' - ' "collections.abc.Buffer"\n' - ' ABC for buffer types.\n' - '\n' - '\n' - 'Special method lookup\n' - '=====================\n' - '\n' - 'For custom classes, implicit invocations of special methods ' - 'are only\n' - 'guaranteed to work correctly if defined on an object’s type, ' - 'not in\n' - 'the object’s instance dictionary. That behaviour is the ' - 'reason why\n' - 'the following code raises an exception:\n' - '\n' - ' >>> class C:\n' - ' ... pass\n' - ' ...\n' - ' >>> c = C()\n' - ' >>> c.__len__ = lambda: 5\n' - ' >>> len(c)\n' - ' Traceback (most recent call last):\n' - ' File "", line 1, in \n' - " TypeError: object of type 'C' has no len()\n" - '\n' - 'The rationale behind this behaviour lies with a number of ' - 'special\n' - 'methods such as "__hash__()" and "__repr__()" that are ' - 'implemented by\n' - 'all objects, including type objects. If the implicit lookup ' - 'of these\n' - 'methods used the conventional lookup process, they would ' - 'fail when\n' - 'invoked on the type object itself:\n' - '\n' - ' >>> 1 .__hash__() == hash(1)\n' - ' True\n' - ' >>> int.__hash__() == hash(int)\n' - ' Traceback (most recent call last):\n' - ' File "", line 1, in \n' - " TypeError: descriptor '__hash__' of 'int' object needs an " - 'argument\n' - '\n' - 'Incorrectly attempting to invoke an unbound method of a ' - 'class in this\n' - 'way is sometimes referred to as ‘metaclass confusion’, and ' - 'is avoided\n' - 'by bypassing the instance when looking up special methods:\n' - '\n' - ' >>> type(1).__hash__(1) == hash(1)\n' - ' True\n' - ' >>> type(int).__hash__(int) == hash(int)\n' - ' True\n' - '\n' - 'In addition to bypassing any instance attributes in the ' - 'interest of\n' - 'correctness, implicit special method lookup generally also ' - 'bypasses\n' - 'the "__getattribute__()" method even of the object’s ' - 'metaclass:\n' - '\n' - ' >>> class Meta(type):\n' - ' ... def __getattribute__(*args):\n' - ' ... print("Metaclass getattribute invoked")\n' - ' ... return type.__getattribute__(*args)\n' - ' ...\n' - ' >>> class C(object, metaclass=Meta):\n' - ' ... def __len__(self):\n' - ' ... return 10\n' - ' ... def __getattribute__(*args):\n' - ' ... print("Class getattribute invoked")\n' - ' ... return object.__getattribute__(*args)\n' - ' ...\n' - ' >>> c = C()\n' - ' >>> c.__len__() # Explicit lookup via ' - 'instance\n' - ' Class getattribute invoked\n' - ' 10\n' - ' >>> type(c).__len__(c) # Explicit lookup via ' - 'type\n' - ' Metaclass getattribute invoked\n' - ' 10\n' - ' >>> len(c) # Implicit lookup\n' - ' 10\n' - '\n' - 'Bypassing the "__getattribute__()" machinery in this fashion ' - 'provides\n' - 'significant scope for speed optimisations within the ' - 'interpreter, at\n' - 'the cost of some flexibility in the handling of special ' - 'methods (the\n' - 'special method *must* be set on the class object itself in ' - 'order to be\n' - 'consistently invoked by the interpreter).\n', - 'string-methods': 'String Methods\n' - '**************\n' - '\n' - 'Strings implement all of the common sequence operations, ' - 'along with\n' - 'the additional methods described below.\n' - '\n' - 'Strings also support two styles of string formatting, one ' - 'providing a\n' - 'large degree of flexibility and customization (see ' - '"str.format()",\n' - 'Format String Syntax and Custom String Formatting) and the ' - 'other based\n' - 'on C "printf" style formatting that handles a narrower ' - 'range of types\n' - 'and is slightly harder to use correctly, but is often ' - 'faster for the\n' - 'cases it can handle (printf-style String Formatting).\n' - '\n' - 'The Text Processing Services section of the standard ' - 'library covers a\n' - 'number of other modules that provide various text related ' - 'utilities\n' - '(including regular expression support in the "re" ' - 'module).\n' - '\n' - 'str.capitalize()\n' - '\n' - ' Return a copy of the string with its first character ' - 'capitalized\n' - ' and the rest lowercased.\n' - '\n' - ' Changed in version 3.8: The first character is now put ' - 'into\n' - ' titlecase rather than uppercase. This means that ' - 'characters like\n' - ' digraphs will only have their first letter capitalized, ' - 'instead of\n' - ' the full character.\n' - '\n' - 'str.casefold()\n' - '\n' - ' Return a casefolded copy of the string. Casefolded ' - 'strings may be\n' - ' used for caseless matching.\n' - '\n' - ' Casefolding is similar to lowercasing but more ' - 'aggressive because\n' - ' it is intended to remove all case distinctions in a ' - 'string. For\n' - ' example, the German lowercase letter "\'ß\'" is ' - 'equivalent to ""ss"".\n' - ' Since it is already lowercase, "lower()" would do ' - 'nothing to "\'ß\'";\n' - ' "casefold()" converts it to ""ss"".\n' - '\n' - ' The casefolding algorithm is described in section 3.13 ' - '‘Default\n' - ' Case Folding’ of the Unicode Standard.\n' - '\n' - ' New in version 3.3.\n' - '\n' - 'str.center(width[, fillchar])\n' - '\n' - ' Return centered in a string of length *width*. Padding ' - 'is done\n' - ' using the specified *fillchar* (default is an ASCII ' - 'space). The\n' - ' original string is returned if *width* is less than or ' - 'equal to\n' - ' "len(s)".\n' - '\n' - 'str.count(sub[, start[, end]])\n' - '\n' - ' Return the number of non-overlapping occurrences of ' - 'substring *sub*\n' - ' in the range [*start*, *end*]. Optional arguments ' - '*start* and\n' - ' *end* are interpreted as in slice notation.\n' - '\n' - ' If *sub* is empty, returns the number of empty strings ' - 'between\n' - ' characters which is the length of the string plus one.\n' - '\n' - "str.encode(encoding='utf-8', errors='strict')\n" - '\n' - ' Return the string encoded to "bytes".\n' - '\n' - ' *encoding* defaults to "\'utf-8\'"; see Standard ' - 'Encodings for\n' - ' possible values.\n' - '\n' - ' *errors* controls how encoding errors are handled. If ' - '"\'strict\'"\n' - ' (the default), a "UnicodeError" exception is raised. ' - 'Other possible\n' - ' values are "\'ignore\'", "\'replace\'", ' - '"\'xmlcharrefreplace\'",\n' - ' "\'backslashreplace\'" and any other name registered ' - 'via\n' - ' "codecs.register_error()". See Error Handlers for ' - 'details.\n' - '\n' - ' For performance reasons, the value of *errors* is not ' - 'checked for\n' - ' validity unless an encoding error actually occurs, ' - 'Python\n' - ' Development Mode is enabled or a debug build is used.\n' - '\n' - ' Changed in version 3.1: Added support for keyword ' - 'arguments.\n' - '\n' - ' Changed in version 3.9: The value of the *errors* ' - 'argument is now\n' - ' checked in Python Development Mode and in debug mode.\n' - '\n' - 'str.endswith(suffix[, start[, end]])\n' - '\n' - ' Return "True" if the string ends with the specified ' - '*suffix*,\n' - ' otherwise return "False". *suffix* can also be a tuple ' - 'of suffixes\n' - ' to look for. With optional *start*, test beginning at ' - 'that\n' - ' position. With optional *end*, stop comparing at that ' - 'position.\n' - '\n' - 'str.expandtabs(tabsize=8)\n' - '\n' - ' Return a copy of the string where all tab characters ' - 'are replaced\n' - ' by one or more spaces, depending on the current column ' - 'and the\n' - ' given tab size. Tab positions occur every *tabsize* ' - 'characters\n' - ' (default is 8, giving tab positions at columns 0, 8, 16 ' - 'and so on).\n' - ' To expand the string, the current column is set to zero ' - 'and the\n' - ' string is examined character by character. If the ' - 'character is a\n' - ' tab ("\\t"), one or more space characters are inserted ' - 'in the result\n' - ' until the current column is equal to the next tab ' - 'position. (The\n' - ' tab character itself is not copied.) If the character ' - 'is a newline\n' - ' ("\\n") or return ("\\r"), it is copied and the current ' - 'column is\n' - ' reset to zero. Any other character is copied unchanged ' - 'and the\n' - ' current column is incremented by one regardless of how ' - 'the\n' - ' character is represented when printed.\n' - '\n' - " >>> '01\\t012\\t0123\\t01234'.expandtabs()\n" - " '01 012 0123 01234'\n" - " >>> '01\\t012\\t0123\\t01234'.expandtabs(4)\n" - " '01 012 0123 01234'\n" - '\n' - 'str.find(sub[, start[, end]])\n' - '\n' - ' Return the lowest index in the string where substring ' - '*sub* is\n' - ' found within the slice "s[start:end]". Optional ' - 'arguments *start*\n' - ' and *end* are interpreted as in slice notation. Return ' - '"-1" if\n' - ' *sub* is not found.\n' - '\n' - ' Note:\n' - '\n' - ' The "find()" method should be used only if you need ' - 'to know the\n' - ' position of *sub*. To check if *sub* is a substring ' - 'or not, use\n' - ' the "in" operator:\n' - '\n' - " >>> 'Py' in 'Python'\n" - ' True\n' - '\n' - 'str.format(*args, **kwargs)\n' - '\n' - ' Perform a string formatting operation. The string on ' - 'which this\n' - ' method is called can contain literal text or ' - 'replacement fields\n' - ' delimited by braces "{}". Each replacement field ' - 'contains either\n' - ' the numeric index of a positional argument, or the name ' - 'of a\n' - ' keyword argument. Returns a copy of the string where ' - 'each\n' - ' replacement field is replaced with the string value of ' - 'the\n' - ' corresponding argument.\n' - '\n' - ' >>> "The sum of 1 + 2 is {0}".format(1+2)\n' - " 'The sum of 1 + 2 is 3'\n" - '\n' - ' See Format String Syntax for a description of the ' - 'various\n' - ' formatting options that can be specified in format ' - 'strings.\n' - '\n' - ' Note:\n' - '\n' - ' When formatting a number ("int", "float", "complex",\n' - ' "decimal.Decimal" and subclasses) with the "n" type ' - '(ex:\n' - ' "\'{:n}\'.format(1234)"), the function temporarily ' - 'sets the\n' - ' "LC_CTYPE" locale to the "LC_NUMERIC" locale to ' - 'decode\n' - ' "decimal_point" and "thousands_sep" fields of ' - '"localeconv()" if\n' - ' they are non-ASCII or longer than 1 byte, and the ' - '"LC_NUMERIC"\n' - ' locale is different than the "LC_CTYPE" locale. This ' - 'temporary\n' - ' change affects other threads.\n' - '\n' - ' Changed in version 3.7: When formatting a number with ' - 'the "n" type,\n' - ' the function sets temporarily the "LC_CTYPE" locale to ' - 'the\n' - ' "LC_NUMERIC" locale in some cases.\n' - '\n' - 'str.format_map(mapping)\n' - '\n' - ' Similar to "str.format(**mapping)", except that ' - '"mapping" is used\n' - ' directly and not copied to a "dict". This is useful if ' - 'for example\n' - ' "mapping" is a dict subclass:\n' - '\n' - ' >>> class Default(dict):\n' - ' ... def __missing__(self, key):\n' - ' ... return key\n' - ' ...\n' - " >>> '{name} was born in " - "{country}'.format_map(Default(name='Guido'))\n" - " 'Guido was born in country'\n" - '\n' - ' New in version 3.2.\n' - '\n' - 'str.index(sub[, start[, end]])\n' - '\n' - ' Like "find()", but raise "ValueError" when the ' - 'substring is not\n' - ' found.\n' - '\n' - 'str.isalnum()\n' - '\n' - ' Return "True" if all characters in the string are ' - 'alphanumeric and\n' - ' there is at least one character, "False" otherwise. A ' - 'character\n' - ' "c" is alphanumeric if one of the following returns ' - '"True":\n' - ' "c.isalpha()", "c.isdecimal()", "c.isdigit()", or ' - '"c.isnumeric()".\n' - '\n' - 'str.isalpha()\n' - '\n' - ' Return "True" if all characters in the string are ' - 'alphabetic and\n' - ' there is at least one character, "False" otherwise. ' - 'Alphabetic\n' - ' characters are those characters defined in the Unicode ' - 'character\n' - ' database as “Letter”, i.e., those with general category ' - 'property\n' - ' being one of “Lm”, “Lt”, “Lu”, “Ll”, or “Lo”. Note ' - 'that this is\n' - ' different from the Alphabetic property defined in the ' - 'section 4.10\n' - ' ‘Letters, Alphabetic, and Ideographic’ of the Unicode ' - 'Standard.\n' - '\n' - 'str.isascii()\n' - '\n' - ' Return "True" if the string is empty or all characters ' - 'in the\n' - ' string are ASCII, "False" otherwise. ASCII characters ' - 'have code\n' - ' points in the range U+0000-U+007F.\n' - '\n' - ' New in version 3.7.\n' - '\n' - 'str.isdecimal()\n' - '\n' - ' Return "True" if all characters in the string are ' - 'decimal\n' - ' characters and there is at least one character, "False" ' - 'otherwise.\n' - ' Decimal characters are those that can be used to form ' - 'numbers in\n' - ' base 10, e.g. U+0660, ARABIC-INDIC DIGIT ZERO. ' - 'Formally a decimal\n' - ' character is a character in the Unicode General ' - 'Category “Nd”.\n' - '\n' - 'str.isdigit()\n' - '\n' - ' Return "True" if all characters in the string are ' - 'digits and there\n' - ' is at least one character, "False" otherwise. Digits ' - 'include\n' - ' decimal characters and digits that need special ' - 'handling, such as\n' - ' the compatibility superscript digits. This covers ' - 'digits which\n' - ' cannot be used to form numbers in base 10, like the ' - 'Kharosthi\n' - ' numbers. Formally, a digit is a character that has the ' - 'property\n' - ' value Numeric_Type=Digit or Numeric_Type=Decimal.\n' - '\n' - 'str.isidentifier()\n' - '\n' - ' Return "True" if the string is a valid identifier ' - 'according to the\n' - ' language definition, section Identifiers and keywords.\n' - '\n' - ' "keyword.iskeyword()" can be used to test whether ' - 'string "s" is a\n' - ' reserved identifier, such as "def" and "class".\n' - '\n' - ' Example:\n' - '\n' - ' >>> from keyword import iskeyword\n' - '\n' - " >>> 'hello'.isidentifier(), iskeyword('hello')\n" - ' (True, False)\n' - " >>> 'def'.isidentifier(), iskeyword('def')\n" - ' (True, True)\n' - '\n' - 'str.islower()\n' - '\n' - ' Return "True" if all cased characters [4] in the string ' - 'are\n' - ' lowercase and there is at least one cased character, ' - '"False"\n' - ' otherwise.\n' - '\n' - 'str.isnumeric()\n' - '\n' - ' Return "True" if all characters in the string are ' - 'numeric\n' - ' characters, and there is at least one character, ' - '"False" otherwise.\n' - ' Numeric characters include digit characters, and all ' - 'characters\n' - ' that have the Unicode numeric value property, e.g. ' - 'U+2155, VULGAR\n' - ' FRACTION ONE FIFTH. Formally, numeric characters are ' - 'those with\n' - ' the property value Numeric_Type=Digit, ' - 'Numeric_Type=Decimal or\n' - ' Numeric_Type=Numeric.\n' - '\n' - 'str.isprintable()\n' - '\n' - ' Return "True" if all characters in the string are ' - 'printable or the\n' - ' string is empty, "False" otherwise. Nonprintable ' - 'characters are\n' - ' those characters defined in the Unicode character ' - 'database as\n' - ' “Other” or “Separator”, excepting the ASCII space ' - '(0x20) which is\n' - ' considered printable. (Note that printable characters ' - 'in this\n' - ' context are those which should not be escaped when ' - '"repr()" is\n' - ' invoked on a string. It has no bearing on the handling ' - 'of strings\n' - ' written to "sys.stdout" or "sys.stderr".)\n' - '\n' - 'str.isspace()\n' - '\n' - ' Return "True" if there are only whitespace characters ' - 'in the string\n' - ' and there is at least one character, "False" ' - 'otherwise.\n' - '\n' - ' A character is *whitespace* if in the Unicode character ' - 'database\n' - ' (see "unicodedata"), either its general category is ' - '"Zs"\n' - ' (“Separator, space”), or its bidirectional class is one ' - 'of "WS",\n' - ' "B", or "S".\n' - '\n' - 'str.istitle()\n' - '\n' - ' Return "True" if the string is a titlecased string and ' - 'there is at\n' - ' least one character, for example uppercase characters ' - 'may only\n' - ' follow uncased characters and lowercase characters only ' - 'cased ones.\n' - ' Return "False" otherwise.\n' - '\n' - 'str.isupper()\n' - '\n' - ' Return "True" if all cased characters [4] in the string ' - 'are\n' - ' uppercase and there is at least one cased character, ' - '"False"\n' - ' otherwise.\n' - '\n' - " >>> 'BANANA'.isupper()\n" - ' True\n' - " >>> 'banana'.isupper()\n" - ' False\n' - " >>> 'baNana'.isupper()\n" - ' False\n' - " >>> ' '.isupper()\n" - ' False\n' - '\n' - 'str.join(iterable)\n' - '\n' - ' Return a string which is the concatenation of the ' - 'strings in\n' - ' *iterable*. A "TypeError" will be raised if there are ' - 'any non-\n' - ' string values in *iterable*, including "bytes" ' - 'objects. The\n' - ' separator between elements is the string providing this ' - 'method.\n' - '\n' - 'str.ljust(width[, fillchar])\n' - '\n' - ' Return the string left justified in a string of length ' - '*width*.\n' - ' Padding is done using the specified *fillchar* (default ' - 'is an ASCII\n' - ' space). The original string is returned if *width* is ' - 'less than or\n' - ' equal to "len(s)".\n' - '\n' - 'str.lower()\n' - '\n' - ' Return a copy of the string with all the cased ' - 'characters [4]\n' - ' converted to lowercase.\n' - '\n' - ' The lowercasing algorithm used is described in section ' - '3.13\n' - ' ‘Default Case Folding’ of the Unicode Standard.\n' - '\n' - 'str.lstrip([chars])\n' - '\n' - ' Return a copy of the string with leading characters ' - 'removed. The\n' - ' *chars* argument is a string specifying the set of ' - 'characters to be\n' - ' removed. If omitted or "None", the *chars* argument ' - 'defaults to\n' - ' removing whitespace. The *chars* argument is not a ' - 'prefix; rather,\n' - ' all combinations of its values are stripped:\n' - '\n' - " >>> ' spacious '.lstrip()\n" - " 'spacious '\n" - " >>> 'www.example.com'.lstrip('cmowz.')\n" - " 'example.com'\n" - '\n' - ' See "str.removeprefix()" for a method that will remove ' - 'a single\n' - ' prefix string rather than all of a set of characters. ' - 'For example:\n' - '\n' - " >>> 'Arthur: three!'.lstrip('Arthur: ')\n" - " 'ee!'\n" - " >>> 'Arthur: three!'.removeprefix('Arthur: ')\n" - " 'three!'\n" - '\n' - 'static str.maketrans(x[, y[, z]])\n' - '\n' - ' This static method returns a translation table usable ' - 'for\n' - ' "str.translate()".\n' - '\n' - ' If there is only one argument, it must be a dictionary ' - 'mapping\n' - ' Unicode ordinals (integers) or characters (strings of ' - 'length 1) to\n' - ' Unicode ordinals, strings (of arbitrary lengths) or ' - '"None".\n' - ' Character keys will then be converted to ordinals.\n' - '\n' - ' If there are two arguments, they must be strings of ' - 'equal length,\n' - ' and in the resulting dictionary, each character in x ' - 'will be mapped\n' - ' to the character at the same position in y. If there ' - 'is a third\n' - ' argument, it must be a string, whose characters will be ' - 'mapped to\n' - ' "None" in the result.\n' - '\n' - 'str.partition(sep)\n' - '\n' - ' Split the string at the first occurrence of *sep*, and ' - 'return a\n' - ' 3-tuple containing the part before the separator, the ' - 'separator\n' - ' itself, and the part after the separator. If the ' - 'separator is not\n' - ' found, return a 3-tuple containing the string itself, ' - 'followed by\n' - ' two empty strings.\n' - '\n' - 'str.removeprefix(prefix, /)\n' - '\n' - ' If the string starts with the *prefix* string, return\n' - ' "string[len(prefix):]". Otherwise, return a copy of the ' - 'original\n' - ' string:\n' - '\n' - " >>> 'TestHook'.removeprefix('Test')\n" - " 'Hook'\n" - " >>> 'BaseTestCase'.removeprefix('Test')\n" - " 'BaseTestCase'\n" - '\n' - ' New in version 3.9.\n' - '\n' - 'str.removesuffix(suffix, /)\n' - '\n' - ' If the string ends with the *suffix* string and that ' - '*suffix* is\n' - ' not empty, return "string[:-len(suffix)]". Otherwise, ' - 'return a copy\n' - ' of the original string:\n' - '\n' - " >>> 'MiscTests'.removesuffix('Tests')\n" - " 'Misc'\n" - " >>> 'TmpDirMixin'.removesuffix('Tests')\n" - " 'TmpDirMixin'\n" - '\n' - ' New in version 3.9.\n' - '\n' - 'str.replace(old, new, count=-1)\n' - '\n' - ' Return a copy of the string with all occurrences of ' - 'substring *old*\n' - ' replaced by *new*. If *count* is given, only the first ' - '*count*\n' - ' occurrences are replaced. If *count* is not specified ' - 'or "-1", then\n' - ' all occurrences are replaced.\n' - '\n' - ' Changed in version 3.13: *count* is now supported as a ' - 'keyword\n' - ' argument.\n' - '\n' - 'str.rfind(sub[, start[, end]])\n' - '\n' - ' Return the highest index in the string where substring ' - '*sub* is\n' - ' found, such that *sub* is contained within ' - '"s[start:end]".\n' - ' Optional arguments *start* and *end* are interpreted as ' - 'in slice\n' - ' notation. Return "-1" on failure.\n' - '\n' - 'str.rindex(sub[, start[, end]])\n' - '\n' - ' Like "rfind()" but raises "ValueError" when the ' - 'substring *sub* is\n' - ' not found.\n' - '\n' - 'str.rjust(width[, fillchar])\n' - '\n' - ' Return the string right justified in a string of length ' - '*width*.\n' - ' Padding is done using the specified *fillchar* (default ' - 'is an ASCII\n' - ' space). The original string is returned if *width* is ' - 'less than or\n' - ' equal to "len(s)".\n' - '\n' - 'str.rpartition(sep)\n' - '\n' - ' Split the string at the last occurrence of *sep*, and ' - 'return a\n' - ' 3-tuple containing the part before the separator, the ' - 'separator\n' - ' itself, and the part after the separator. If the ' - 'separator is not\n' - ' found, return a 3-tuple containing two empty strings, ' - 'followed by\n' - ' the string itself.\n' - '\n' - 'str.rsplit(sep=None, maxsplit=-1)\n' - '\n' - ' Return a list of the words in the string, using *sep* ' - 'as the\n' - ' delimiter string. If *maxsplit* is given, at most ' - '*maxsplit* splits\n' - ' are done, the *rightmost* ones. If *sep* is not ' - 'specified or\n' - ' "None", any whitespace string is a separator. Except ' - 'for splitting\n' - ' from the right, "rsplit()" behaves like "split()" which ' - 'is\n' - ' described in detail below.\n' - '\n' - 'str.rstrip([chars])\n' - '\n' - ' Return a copy of the string with trailing characters ' - 'removed. The\n' - ' *chars* argument is a string specifying the set of ' - 'characters to be\n' - ' removed. If omitted or "None", the *chars* argument ' - 'defaults to\n' - ' removing whitespace. The *chars* argument is not a ' - 'suffix; rather,\n' - ' all combinations of its values are stripped:\n' - '\n' - " >>> ' spacious '.rstrip()\n" - " ' spacious'\n" - " >>> 'mississippi'.rstrip('ipz')\n" - " 'mississ'\n" - '\n' - ' See "str.removesuffix()" for a method that will remove ' - 'a single\n' - ' suffix string rather than all of a set of characters. ' - 'For example:\n' - '\n' - " >>> 'Monty Python'.rstrip(' Python')\n" - " 'M'\n" - " >>> 'Monty Python'.removesuffix(' Python')\n" - " 'Monty'\n" - '\n' - 'str.split(sep=None, maxsplit=-1)\n' - '\n' - ' Return a list of the words in the string, using *sep* ' - 'as the\n' - ' delimiter string. If *maxsplit* is given, at most ' - '*maxsplit*\n' - ' splits are done (thus, the list will have at most ' - '"maxsplit+1"\n' - ' elements). If *maxsplit* is not specified or "-1", ' - 'then there is\n' - ' no limit on the number of splits (all possible splits ' - 'are made).\n' - '\n' - ' If *sep* is given, consecutive delimiters are not ' - 'grouped together\n' - ' and are deemed to delimit empty strings (for example,\n' - ' "\'1,,2\'.split(\',\')" returns "[\'1\', \'\', ' - '\'2\']"). The *sep* argument\n' - ' may consist of multiple characters (for example,\n' - ' "\'1<>2<>3\'.split(\'<>\')" returns "[\'1\', \'2\', ' - '\'3\']"). Splitting an\n' - ' empty string with a specified separator returns ' - '"[\'\']".\n' - '\n' - ' For example:\n' - '\n' - " >>> '1,2,3'.split(',')\n" - " ['1', '2', '3']\n" - " >>> '1,2,3'.split(',', maxsplit=1)\n" - " ['1', '2,3']\n" - " >>> '1,2,,3,'.split(',')\n" - " ['1', '2', '', '3', '']\n" - '\n' - ' If *sep* is not specified or is "None", a different ' - 'splitting\n' - ' algorithm is applied: runs of consecutive whitespace ' - 'are regarded\n' - ' as a single separator, and the result will contain no ' - 'empty strings\n' - ' at the start or end if the string has leading or ' - 'trailing\n' - ' whitespace. Consequently, splitting an empty string or ' - 'a string\n' - ' consisting of just whitespace with a "None" separator ' - 'returns "[]".\n' - '\n' - ' For example:\n' - '\n' - " >>> '1 2 3'.split()\n" - " ['1', '2', '3']\n" - " >>> '1 2 3'.split(maxsplit=1)\n" - " ['1', '2 3']\n" - " >>> ' 1 2 3 '.split()\n" - " ['1', '2', '3']\n" - '\n' - 'str.splitlines(keepends=False)\n' - '\n' - ' Return a list of the lines in the string, breaking at ' - 'line\n' - ' boundaries. Line breaks are not included in the ' - 'resulting list\n' - ' unless *keepends* is given and true.\n' - '\n' - ' This method splits on the following line boundaries. ' - 'In\n' - ' particular, the boundaries are a superset of *universal ' - 'newlines*.\n' - '\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | Representation | ' - 'Description |\n' - ' ' - '|=========================|===============================|\n' - ' | "\\n" | Line ' - 'Feed |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\r" | Carriage ' - 'Return |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\r\\n" | Carriage Return + Line ' - 'Feed |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\v" or "\\x0b" | Line ' - 'Tabulation |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\f" or "\\x0c" | Form ' - 'Feed |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\x1c" | File ' - 'Separator |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\x1d" | Group ' - 'Separator |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\x1e" | Record ' - 'Separator |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\x85" | Next Line (C1 Control ' - 'Code) |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\u2028" | Line ' - 'Separator |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\u2029" | Paragraph ' - 'Separator |\n' - ' ' - '+-------------------------+-------------------------------+\n' - '\n' - ' Changed in version 3.2: "\\v" and "\\f" added to list ' - 'of line\n' - ' boundaries.\n' - '\n' - ' For example:\n' - '\n' - " >>> 'ab c\\n\\nde fg\\rkl\\r\\n'.splitlines()\n" - " ['ab c', '', 'de fg', 'kl']\n" - " >>> 'ab c\\n\\nde " - "fg\\rkl\\r\\n'.splitlines(keepends=True)\n" - " ['ab c\\n', '\\n', 'de fg\\r', 'kl\\r\\n']\n" - '\n' - ' Unlike "split()" when a delimiter string *sep* is ' - 'given, this\n' - ' method returns an empty list for the empty string, and ' - 'a terminal\n' - ' line break does not result in an extra line:\n' - '\n' - ' >>> "".splitlines()\n' - ' []\n' - ' >>> "One line\\n".splitlines()\n' - " ['One line']\n" - '\n' - ' For comparison, "split(\'\\n\')" gives:\n' - '\n' - " >>> ''.split('\\n')\n" - " ['']\n" - " >>> 'Two lines\\n'.split('\\n')\n" - " ['Two lines', '']\n" - '\n' - 'str.startswith(prefix[, start[, end]])\n' - '\n' - ' Return "True" if string starts with the *prefix*, ' - 'otherwise return\n' - ' "False". *prefix* can also be a tuple of prefixes to ' - 'look for.\n' - ' With optional *start*, test string beginning at that ' - 'position.\n' - ' With optional *end*, stop comparing string at that ' - 'position.\n' - '\n' - 'str.strip([chars])\n' - '\n' - ' Return a copy of the string with the leading and ' - 'trailing\n' - ' characters removed. The *chars* argument is a string ' - 'specifying the\n' - ' set of characters to be removed. If omitted or "None", ' - 'the *chars*\n' - ' argument defaults to removing whitespace. The *chars* ' - 'argument is\n' - ' not a prefix or suffix; rather, all combinations of its ' - 'values are\n' - ' stripped:\n' - '\n' - " >>> ' spacious '.strip()\n" - " 'spacious'\n" - " >>> 'www.example.com'.strip('cmowz.')\n" - " 'example'\n" - '\n' - ' The outermost leading and trailing *chars* argument ' - 'values are\n' - ' stripped from the string. Characters are removed from ' - 'the leading\n' - ' end until reaching a string character that is not ' - 'contained in the\n' - ' set of characters in *chars*. A similar action takes ' - 'place on the\n' - ' trailing end. For example:\n' - '\n' - " >>> comment_string = '#....... Section 3.2.1 Issue " - "#32 .......'\n" - " >>> comment_string.strip('.#! ')\n" - " 'Section 3.2.1 Issue #32'\n" - '\n' - 'str.swapcase()\n' - '\n' - ' Return a copy of the string with uppercase characters ' - 'converted to\n' - ' lowercase and vice versa. Note that it is not ' - 'necessarily true that\n' - ' "s.swapcase().swapcase() == s".\n' - '\n' - 'str.title()\n' - '\n' - ' Return a titlecased version of the string where words ' - 'start with an\n' - ' uppercase character and the remaining characters are ' - 'lowercase.\n' - '\n' - ' For example:\n' - '\n' - " >>> 'Hello world'.title()\n" - " 'Hello World'\n" - '\n' - ' The algorithm uses a simple language-independent ' - 'definition of a\n' - ' word as groups of consecutive letters. The definition ' - 'works in\n' - ' many contexts but it means that apostrophes in ' - 'contractions and\n' - ' possessives form word boundaries, which may not be the ' - 'desired\n' - ' result:\n' - '\n' - ' >>> "they\'re bill\'s friends from the UK".title()\n' - ' "They\'Re Bill\'S Friends From The Uk"\n' - '\n' - ' The "string.capwords()" function does not have this ' - 'problem, as it\n' - ' splits words on spaces only.\n' - '\n' - ' Alternatively, a workaround for apostrophes can be ' - 'constructed\n' - ' using regular expressions:\n' - '\n' - ' >>> import re\n' - ' >>> def titlecase(s):\n' - ' ... return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n' - ' ... lambda mo: ' - 'mo.group(0).capitalize(),\n' - ' ... s)\n' - ' ...\n' - ' >>> titlecase("they\'re bill\'s friends.")\n' - ' "They\'re Bill\'s Friends."\n' - '\n' - 'str.translate(table)\n' - '\n' - ' Return a copy of the string in which each character has ' - 'been mapped\n' - ' through the given translation table. The table must be ' - 'an object\n' - ' that implements indexing via "__getitem__()", typically ' - 'a *mapping*\n' - ' or *sequence*. When indexed by a Unicode ordinal (an ' - 'integer), the\n' - ' table object can do any of the following: return a ' - 'Unicode ordinal\n' - ' or a string, to map the character to one or more other ' - 'characters;\n' - ' return "None", to delete the character from the return ' - 'string; or\n' - ' raise a "LookupError" exception, to map the character ' - 'to itself.\n' - '\n' - ' You can use "str.maketrans()" to create a translation ' - 'map from\n' - ' character-to-character mappings in different formats.\n' - '\n' - ' See also the "codecs" module for a more flexible ' - 'approach to custom\n' - ' character mappings.\n' - '\n' - 'str.upper()\n' - '\n' - ' Return a copy of the string with all the cased ' - 'characters [4]\n' - ' converted to uppercase. Note that ' - '"s.upper().isupper()" might be\n' - ' "False" if "s" contains uncased characters or if the ' - 'Unicode\n' - ' category of the resulting character(s) is not “Lu” ' - '(Letter,\n' - ' uppercase), but e.g. “Lt” (Letter, titlecase).\n' - '\n' - ' The uppercasing algorithm used is described in section ' - '3.13\n' - ' ‘Default Case Folding’ of the Unicode Standard.\n' - '\n' - 'str.zfill(width)\n' - '\n' - ' Return a copy of the string left filled with ASCII ' - '"\'0\'" digits to\n' - ' make a string of length *width*. A leading sign prefix\n' - ' ("\'+\'"/"\'-\'") is handled by inserting the padding ' - '*after* the sign\n' - ' character rather than before. The original string is ' - 'returned if\n' - ' *width* is less than or equal to "len(s)".\n' - '\n' - ' For example:\n' - '\n' - ' >>> "42".zfill(5)\n' - " '00042'\n" - ' >>> "-42".zfill(5)\n' - " '-0042'\n", - 'strings': 'String and Bytes literals\n' - '*************************\n' - '\n' - 'String literals are described by the following lexical ' - 'definitions:\n' - '\n' - ' stringliteral ::= [stringprefix](shortstring | longstring)\n' - ' stringprefix ::= "r" | "u" | "R" | "U" | "f" | "F"\n' - ' | "fr" | "Fr" | "fR" | "FR" | "rf" | "rF" | ' - '"Rf" | "RF"\n' - ' shortstring ::= "\'" shortstringitem* "\'" | \'"\' ' - 'shortstringitem* \'"\'\n' - ' longstring ::= "\'\'\'" longstringitem* "\'\'\'" | ' - '\'"""\' longstringitem* \'"""\'\n' - ' shortstringitem ::= shortstringchar | stringescapeseq\n' - ' longstringitem ::= longstringchar | stringescapeseq\n' - ' shortstringchar ::= \n' - ' longstringchar ::= \n' - ' stringescapeseq ::= "\\" \n' - '\n' - ' bytesliteral ::= bytesprefix(shortbytes | longbytes)\n' - ' bytesprefix ::= "b" | "B" | "br" | "Br" | "bR" | "BR" | ' - '"rb" | "rB" | "Rb" | "RB"\n' - ' shortbytes ::= "\'" shortbytesitem* "\'" | \'"\' ' - 'shortbytesitem* \'"\'\n' - ' longbytes ::= "\'\'\'" longbytesitem* "\'\'\'" | \'"""\' ' - 'longbytesitem* \'"""\'\n' - ' shortbytesitem ::= shortbyteschar | bytesescapeseq\n' - ' longbytesitem ::= longbyteschar | bytesescapeseq\n' - ' shortbyteschar ::= \n' - ' longbyteschar ::= \n' - ' bytesescapeseq ::= "\\" \n' - '\n' - 'One syntactic restriction not indicated by these productions is ' - 'that\n' - 'whitespace is not allowed between the "stringprefix" or ' - '"bytesprefix"\n' - 'and the rest of the literal. The source character set is defined ' - 'by\n' - 'the encoding declaration; it is UTF-8 if no encoding declaration ' - 'is\n' - 'given in the source file; see section Encoding declarations.\n' - '\n' - 'In plain English: Both types of literals can be enclosed in ' - 'matching\n' - 'single quotes ("\'") or double quotes ("""). They can also be ' - 'enclosed\n' - 'in matching groups of three single or double quotes (these are\n' - 'generally referred to as *triple-quoted strings*). The backslash ' - '("\\")\n' - 'character is used to give special meaning to otherwise ordinary\n' - 'characters like "n", which means ‘newline’ when escaped ("\\n"). ' - 'It can\n' - 'also be used to escape characters that otherwise have a special\n' - 'meaning, such as newline, backslash itself, or the quote ' - 'character.\n' - 'See escape sequences below for examples.\n' - '\n' - 'Bytes literals are always prefixed with "\'b\'" or "\'B\'"; they ' - 'produce\n' - 'an instance of the "bytes" type instead of the "str" type. They ' - 'may\n' - 'only contain ASCII characters; bytes with a numeric value of 128 ' - 'or\n' - 'greater must be expressed with escapes.\n' - '\n' - 'Both string and bytes literals may optionally be prefixed with a\n' - 'letter "\'r\'" or "\'R\'"; such strings are called *raw strings* ' - 'and treat\n' - 'backslashes as literal characters. As a result, in string ' - 'literals,\n' - '"\'\\U\'" and "\'\\u\'" escapes in raw strings are not treated ' - 'specially.\n' - 'Given that Python 2.x’s raw unicode literals behave differently ' - 'than\n' - 'Python 3.x’s the "\'ur\'" syntax is not supported.\n' - '\n' - 'New in version 3.3: The "\'rb\'" prefix of raw bytes literals has ' - 'been\n' - 'added as a synonym of "\'br\'".\n' - '\n' - 'New in version 3.3: Support for the unicode legacy literal\n' - '("u\'value\'") was reintroduced to simplify the maintenance of ' - 'dual\n' - 'Python 2.x and 3.x codebases. See **PEP 414** for more ' - 'information.\n' - '\n' - 'A string literal with "\'f\'" or "\'F\'" in its prefix is a ' - '*formatted\n' - 'string literal*; see f-strings. The "\'f\'" may be combined with ' - '"\'r\'",\n' - 'but not with "\'b\'" or "\'u\'", therefore raw formatted strings ' - 'are\n' - 'possible, but formatted bytes literals are not.\n' - '\n' - 'In triple-quoted literals, unescaped newlines and quotes are ' - 'allowed\n' - '(and are retained), except that three unescaped quotes in a row\n' - 'terminate the literal. (A “quote” is the character used to open ' - 'the\n' - 'literal, i.e. either "\'" or """.)\n' - '\n' - '\n' - 'Escape sequences\n' - '================\n' - '\n' - 'Unless an "\'r\'" or "\'R\'" prefix is present, escape sequences ' - 'in string\n' - 'and bytes literals are interpreted according to rules similar to ' - 'those\n' - 'used by Standard C. The recognized escape sequences are:\n' - '\n' - '+---------------------------+-----------------------------------+---------+\n' - '| Escape Sequence | Meaning | ' - 'Notes |\n' - '|===========================|===================================|=========|\n' - '| "\\" | Backslash and newline ignored ' - '| (1) |\n' - '+---------------------------+-----------------------------------+---------+\n' - '| "\\\\" | Backslash ' - '("\\") | |\n' - '+---------------------------+-----------------------------------+---------+\n' - '| "\\\'" | Single quote ' - '("\'") | |\n' - '+---------------------------+-----------------------------------+---------+\n' - '| "\\"" | Double quote (""") ' - '| |\n' - '+---------------------------+-----------------------------------+---------+\n' - '| "\\a" | ASCII Bell (BEL) ' - '| |\n' - '+---------------------------+-----------------------------------+---------+\n' - '| "\\b" | ASCII Backspace (BS) ' - '| |\n' - '+---------------------------+-----------------------------------+---------+\n' - '| "\\f" | ASCII Formfeed (FF) ' - '| |\n' - '+---------------------------+-----------------------------------+---------+\n' - '| "\\n" | ASCII Linefeed (LF) ' - '| |\n' - '+---------------------------+-----------------------------------+---------+\n' - '| "\\r" | ASCII Carriage Return (CR) ' - '| |\n' - '+---------------------------+-----------------------------------+---------+\n' - '| "\\t" | ASCII Horizontal Tab (TAB) ' - '| |\n' - '+---------------------------+-----------------------------------+---------+\n' - '| "\\v" | ASCII Vertical Tab (VT) ' - '| |\n' - '+---------------------------+-----------------------------------+---------+\n' - '| "\\*ooo*" | Character with octal value *ooo* ' - '| (2,4) |\n' - '+---------------------------+-----------------------------------+---------+\n' - '| "\\x*hh*" | Character with hex value *hh* ' - '| (3,4) |\n' - '+---------------------------+-----------------------------------+---------+\n' - '\n' - 'Escape sequences only recognized in string literals are:\n' - '\n' - '+---------------------------+-----------------------------------+---------+\n' - '| Escape Sequence | Meaning | ' - 'Notes |\n' - '|===========================|===================================|=========|\n' - '| "\\N{*name*}" | Character named *name* in the ' - '| (5) |\n' - '| | Unicode database ' - '| |\n' - '+---------------------------+-----------------------------------+---------+\n' - '| "\\u*xxxx*" | Character with 16-bit hex value ' - '| (6) |\n' - '| | *xxxx* ' - '| |\n' - '+---------------------------+-----------------------------------+---------+\n' - '| "\\U*xxxxxxxx*" | Character with 32-bit hex value ' - '| (7) |\n' - '| | *xxxxxxxx* ' - '| |\n' - '+---------------------------+-----------------------------------+---------+\n' - '\n' - 'Notes:\n' - '\n' - '1. A backslash can be added at the end of a line to ignore the\n' - ' newline:\n' - '\n' - " >>> 'This string will not include \\\n" - " ... backslashes or newline characters.'\n" - " 'This string will not include backslashes or newline " - "characters.'\n" - '\n' - ' The same result can be achieved using triple-quoted strings, ' - 'or\n' - ' parentheses and string literal concatenation.\n' - '\n' - '2. As in Standard C, up to three octal digits are accepted.\n' - '\n' - ' Changed in version 3.11: Octal escapes with value larger than\n' - ' "0o377" produce a "DeprecationWarning".\n' - '\n' - ' Changed in version 3.12: Octal escapes with value larger than\n' - ' "0o377" produce a "SyntaxWarning". In a future Python version ' - 'they\n' - ' will be eventually a "SyntaxError".\n' - '\n' - '3. Unlike in Standard C, exactly two hex digits are required.\n' - '\n' - '4. In a bytes literal, hexadecimal and octal escapes denote the ' - 'byte\n' - ' with the given value. In a string literal, these escapes ' - 'denote a\n' - ' Unicode character with the given value.\n' - '\n' - '5. Changed in version 3.3: Support for name aliases [1] has been\n' - ' added.\n' - '\n' - '6. Exactly four hex digits are required.\n' - '\n' - '7. Any Unicode character can be encoded this way. Exactly eight ' - 'hex\n' - ' digits are required.\n' - '\n' - 'Unlike Standard C, all unrecognized escape sequences are left in ' - 'the\n' - 'string unchanged, i.e., *the backslash is left in the result*. ' - '(This\n' - 'behavior is useful when debugging: if an escape sequence is ' - 'mistyped,\n' - 'the resulting output is more easily recognized as broken.) It is ' - 'also\n' - 'important to note that the escape sequences only recognized in ' - 'string\n' - 'literals fall into the category of unrecognized escapes for ' - 'bytes\n' - 'literals.\n' - '\n' - 'Changed in version 3.6: Unrecognized escape sequences produce a\n' - '"DeprecationWarning".\n' - '\n' - 'Changed in version 3.12: Unrecognized escape sequences produce a\n' - '"SyntaxWarning". In a future Python version they will be ' - 'eventually a\n' - '"SyntaxError".\n' - '\n' - 'Even in a raw literal, quotes can be escaped with a backslash, ' - 'but the\n' - 'backslash remains in the result; for example, "r"\\""" is a ' - 'valid\n' - 'string literal consisting of two characters: a backslash and a ' - 'double\n' - 'quote; "r"\\"" is not a valid string literal (even a raw string ' - 'cannot\n' - 'end in an odd number of backslashes). Specifically, *a raw ' - 'literal\n' - 'cannot end in a single backslash* (since the backslash would ' - 'escape\n' - 'the following quote character). Note also that a single ' - 'backslash\n' - 'followed by a newline is interpreted as those two characters as ' - 'part\n' - 'of the literal, *not* as a line continuation.\n', - 'subscriptions': 'Subscriptions\n' - '*************\n' - '\n' - 'The subscription of an instance of a container class will ' - 'generally\n' - 'select an element from the container. The subscription of a ' - '*generic\n' - 'class* will generally return a GenericAlias object.\n' - '\n' - ' subscription ::= primary "[" expression_list "]"\n' - '\n' - 'When an object is subscripted, the interpreter will ' - 'evaluate the\n' - 'primary and the expression list.\n' - '\n' - 'The primary must evaluate to an object that supports ' - 'subscription. An\n' - 'object may support subscription through defining one or ' - 'both of\n' - '"__getitem__()" and "__class_getitem__()". When the primary ' - 'is\n' - 'subscripted, the evaluated result of the expression list ' - 'will be\n' - 'passed to one of these methods. For more details on when\n' - '"__class_getitem__" is called instead of "__getitem__", ' - 'see\n' - '__class_getitem__ versus __getitem__.\n' - '\n' - 'If the expression list contains at least one comma, it will ' - 'evaluate\n' - 'to a "tuple" containing the items of the expression list. ' - 'Otherwise,\n' - 'the expression list will evaluate to the value of the ' - 'list’s sole\n' - 'member.\n' - '\n' - 'For built-in objects, there are two types of objects that ' - 'support\n' - 'subscription via "__getitem__()":\n' - '\n' - '1. Mappings. If the primary is a *mapping*, the expression ' - 'list must\n' - ' evaluate to an object whose value is one of the keys of ' - 'the\n' - ' mapping, and the subscription selects the value in the ' - 'mapping that\n' - ' corresponds to that key. An example of a builtin mapping ' - 'class is\n' - ' the "dict" class.\n' - '\n' - '2. Sequences. If the primary is a *sequence*, the ' - 'expression list must\n' - ' evaluate to an "int" or a "slice" (as discussed in the ' - 'following\n' - ' section). Examples of builtin sequence classes include ' - 'the "str",\n' - ' "list" and "tuple" classes.\n' - '\n' - 'The formal syntax makes no special provision for negative ' - 'indices in\n' - '*sequences*. However, built-in sequences all provide a ' - '"__getitem__()"\n' - 'method that interprets negative indices by adding the ' - 'length of the\n' - 'sequence to the index so that, for example, "x[-1]" selects ' - 'the last\n' - 'item of "x". The resulting value must be a nonnegative ' - 'integer less\n' - 'than the number of items in the sequence, and the ' - 'subscription selects\n' - 'the item whose index is that value (counting from zero). ' - 'Since the\n' - 'support for negative indices and slicing occurs in the ' - 'object’s\n' - '"__getitem__()" method, subclasses overriding this method ' - 'will need to\n' - 'explicitly add that support.\n' - '\n' - 'A "string" is a special kind of sequence whose items are ' - '*characters*.\n' - 'A character is not a separate data type but a string of ' - 'exactly one\n' - 'character.\n', - 'truth': 'Truth Value Testing\n' - '*******************\n' - '\n' - 'Any object can be tested for truth value, for use in an "if" or\n' - '"while" condition or as operand of the Boolean operations below.\n' - '\n' - 'By default, an object is considered true unless its class defines\n' - 'either a "__bool__()" method that returns "False" or a "__len__()"\n' - 'method that returns zero, when called with the object. [1] Here ' - 'are\n' - 'most of the built-in objects considered false:\n' - '\n' - '* constants defined to be false: "None" and "False"\n' - '\n' - '* zero of any numeric type: "0", "0.0", "0j", "Decimal(0)",\n' - ' "Fraction(0, 1)"\n' - '\n' - '* empty sequences and collections: "\'\'", "()", "[]", "{}", ' - '"set()",\n' - ' "range(0)"\n' - '\n' - 'Operations and built-in functions that have a Boolean result ' - 'always\n' - 'return "0" or "False" for false and "1" or "True" for true, unless\n' - 'otherwise stated. (Important exception: the Boolean operations ' - '"or"\n' - 'and "and" always return one of their operands.)\n', - 'try': 'The "try" statement\n' - '*******************\n' - '\n' - 'The "try" statement specifies exception handlers and/or cleanup code\n' - 'for a group of statements:\n' - '\n' - ' try_stmt ::= try1_stmt | try2_stmt | try3_stmt\n' - ' try1_stmt ::= "try" ":" suite\n' - ' ("except" [expression ["as" identifier]] ":" ' - 'suite)+\n' - ' ["else" ":" suite]\n' - ' ["finally" ":" suite]\n' - ' try2_stmt ::= "try" ":" suite\n' - ' ("except" "*" expression ["as" identifier] ":" ' - 'suite)+\n' - ' ["else" ":" suite]\n' - ' ["finally" ":" suite]\n' - ' try3_stmt ::= "try" ":" suite\n' - ' "finally" ":" suite\n' - '\n' - 'Additional information on exceptions can be found in section\n' - 'Exceptions, and information on using the "raise" statement to ' - 'generate\n' - 'exceptions may be found in section The raise statement.\n' - '\n' - '\n' - '"except" clause\n' - '===============\n' - '\n' - 'The "except" clause(s) specify one or more exception handlers. When ' - 'no\n' - 'exception occurs in the "try" clause, no exception handler is\n' - 'executed. When an exception occurs in the "try" suite, a search for ' - 'an\n' - 'exception handler is started. This search inspects the "except"\n' - 'clauses in turn until one is found that matches the exception. An\n' - 'expression-less "except" clause, if present, must be last; it ' - 'matches\n' - 'any exception. For an "except" clause with an expression, that\n' - 'expression is evaluated, and the clause matches the exception if the\n' - 'resulting object is “compatible” with the exception. An object is\n' - 'compatible with an exception if the object is the class or a *non-\n' - 'virtual base class* of the exception object, or a tuple containing ' - 'an\n' - 'item that is the class or a non-virtual base class of the exception\n' - 'object.\n' - '\n' - 'If no "except" clause matches the exception, the search for an\n' - 'exception handler continues in the surrounding code and on the\n' - 'invocation stack. [1]\n' - '\n' - 'If the evaluation of an expression in the header of an "except" ' - 'clause\n' - 'raises an exception, the original search for a handler is canceled ' - 'and\n' - 'a search starts for the new exception in the surrounding code and on\n' - 'the call stack (it is treated as if the entire "try" statement ' - 'raised\n' - 'the exception).\n' - '\n' - 'When a matching "except" clause is found, the exception is assigned ' - 'to\n' - 'the target specified after the "as" keyword in that "except" clause,\n' - 'if present, and the "except" clause’s suite is executed. All ' - '"except"\n' - 'clauses must have an executable block. When the end of this block is\n' - 'reached, execution continues normally after the entire "try"\n' - 'statement. (This means that if two nested handlers exist for the ' - 'same\n' - 'exception, and the exception occurs in the "try" clause of the inner\n' - 'handler, the outer handler will not handle the exception.)\n' - '\n' - 'When an exception has been assigned using "as target", it is cleared\n' - 'at the end of the "except" clause. This is as if\n' - '\n' - ' except E as N:\n' - ' foo\n' - '\n' - 'was translated to\n' - '\n' - ' except E as N:\n' - ' try:\n' - ' foo\n' - ' finally:\n' - ' del N\n' - '\n' - 'This means the exception must be assigned to a different name to be\n' - 'able to refer to it after the "except" clause. Exceptions are ' - 'cleared\n' - 'because with the traceback attached to them, they form a reference\n' - 'cycle with the stack frame, keeping all locals in that frame alive\n' - 'until the next garbage collection occurs.\n' - '\n' - 'Before an "except" clause’s suite is executed, the exception is ' - 'stored\n' - 'in the "sys" module, where it can be accessed from within the body ' - 'of\n' - 'the "except" clause by calling "sys.exception()". When leaving an\n' - 'exception handler, the exception stored in the "sys" module is reset\n' - 'to its previous value:\n' - '\n' - ' >>> print(sys.exception())\n' - ' None\n' - ' >>> try:\n' - ' ... raise TypeError\n' - ' ... except:\n' - ' ... print(repr(sys.exception()))\n' - ' ... try:\n' - ' ... raise ValueError\n' - ' ... except:\n' - ' ... print(repr(sys.exception()))\n' - ' ... print(repr(sys.exception()))\n' - ' ...\n' - ' TypeError()\n' - ' ValueError()\n' - ' TypeError()\n' - ' >>> print(sys.exception())\n' - ' None\n' - '\n' - '\n' - '"except*" clause\n' - '================\n' - '\n' - 'The "except*" clause(s) are used for handling "ExceptionGroup"s. The\n' - 'exception type for matching is interpreted as in the case of ' - '"except",\n' - 'but in the case of exception groups we can have partial matches when\n' - 'the type matches some of the exceptions in the group. This means ' - 'that\n' - 'multiple "except*" clauses can execute, each handling part of the\n' - 'exception group. Each clause executes at most once and handles an\n' - 'exception group of all matching exceptions. Each exception in the\n' - 'group is handled by at most one "except*" clause, the first that\n' - 'matches it.\n' - '\n' - ' >>> try:\n' - ' ... raise ExceptionGroup("eg",\n' - ' ... [ValueError(1), TypeError(2), OSError(3), ' - 'OSError(4)])\n' - ' ... except* TypeError as e:\n' - " ... print(f'caught {type(e)} with nested {e.exceptions}')\n" - ' ... except* OSError as e:\n' - " ... print(f'caught {type(e)} with nested {e.exceptions}')\n" - ' ...\n' - " caught with nested (TypeError(2),)\n" - " caught with nested (OSError(3), " - 'OSError(4))\n' - ' + Exception Group Traceback (most recent call last):\n' - ' | File "", line 2, in \n' - ' | ExceptionGroup: eg\n' - ' +-+---------------- 1 ----------------\n' - ' | ValueError: 1\n' - ' +------------------------------------\n' - '\n' - 'Any remaining exceptions that were not handled by any "except*" ' - 'clause\n' - 'are re-raised at the end, along with all exceptions that were raised\n' - 'from within the "except*" clauses. If this list contains more than ' - 'one\n' - 'exception to reraise, they are combined into an exception group.\n' - '\n' - 'If the raised exception is not an exception group and its type ' - 'matches\n' - 'one of the "except*" clauses, it is caught and wrapped by an ' - 'exception\n' - 'group with an empty message string.\n' - '\n' - ' >>> try:\n' - ' ... raise BlockingIOError\n' - ' ... except* BlockingIOError as e:\n' - ' ... print(repr(e))\n' - ' ...\n' - " ExceptionGroup('', (BlockingIOError()))\n" - '\n' - 'An "except*" clause must have a matching type, and this type cannot ' - 'be\n' - 'a subclass of "BaseExceptionGroup". It is not possible to mix ' - '"except"\n' - 'and "except*" in the same "try". "break", "continue" and "return"\n' - 'cannot appear in an "except*" clause.\n' - '\n' - '\n' - '"else" clause\n' - '=============\n' - '\n' - 'The optional "else" clause is executed if the control flow leaves ' - 'the\n' - '"try" suite, no exception was raised, and no "return", "continue", ' - 'or\n' - '"break" statement was executed. Exceptions in the "else" clause are\n' - 'not handled by the preceding "except" clauses.\n' - '\n' - '\n' - '"finally" clause\n' - '================\n' - '\n' - 'If "finally" is present, it specifies a ‘cleanup’ handler. The ' - '"try"\n' - 'clause is executed, including any "except" and "else" clauses. If ' - 'an\n' - 'exception occurs in any of the clauses and is not handled, the\n' - 'exception is temporarily saved. The "finally" clause is executed. ' - 'If\n' - 'there is a saved exception it is re-raised at the end of the ' - '"finally"\n' - 'clause. If the "finally" clause raises another exception, the saved\n' - 'exception is set as the context of the new exception. If the ' - '"finally"\n' - 'clause executes a "return", "break" or "continue" statement, the ' - 'saved\n' - 'exception is discarded:\n' - '\n' - ' >>> def f():\n' - ' ... try:\n' - ' ... 1/0\n' - ' ... finally:\n' - ' ... return 42\n' - ' ...\n' - ' >>> f()\n' - ' 42\n' - '\n' - 'The exception information is not available to the program during\n' - 'execution of the "finally" clause.\n' - '\n' - 'When a "return", "break" or "continue" statement is executed in the\n' - '"try" suite of a "try"…"finally" statement, the "finally" clause is\n' - 'also executed ‘on the way out.’\n' - '\n' - 'The return value of a function is determined by the last "return"\n' - 'statement executed. Since the "finally" clause always executes, a\n' - '"return" statement executed in the "finally" clause will always be ' - 'the\n' - 'last one executed:\n' - '\n' - ' >>> def foo():\n' - ' ... try:\n' - " ... return 'try'\n" - ' ... finally:\n' - " ... return 'finally'\n" - ' ...\n' - ' >>> foo()\n' - " 'finally'\n" - '\n' - 'Changed in version 3.8: Prior to Python 3.8, a "continue" statement\n' - 'was illegal in the "finally" clause due to a problem with the\n' - 'implementation.\n', - 'types': 'The standard type hierarchy\n' - '***************************\n' - '\n' - 'Below is a list of the types that are built into Python. ' - 'Extension\n' - 'modules (written in C, Java, or other languages, depending on the\n' - 'implementation) can define additional types. Future versions of\n' - 'Python may add types to the type hierarchy (e.g., rational ' - 'numbers,\n' - 'efficiently stored arrays of integers, etc.), although such ' - 'additions\n' - 'will often be provided via the standard library instead.\n' - '\n' - 'Some of the type descriptions below contain a paragraph listing\n' - '‘special attributes.’ These are attributes that provide access to ' - 'the\n' - 'implementation and are not intended for general use. Their ' - 'definition\n' - 'may change in the future.\n' - '\n' - '\n' - 'None\n' - '====\n' - '\n' - 'This type has a single value. There is a single object with this\n' - 'value. This object is accessed through the built-in name "None". It ' - 'is\n' - 'used to signify the absence of a value in many situations, e.g., it ' - 'is\n' - 'returned from functions that don’t explicitly return anything. Its\n' - 'truth value is false.\n' - '\n' - '\n' - 'NotImplemented\n' - '==============\n' - '\n' - 'This type has a single value. There is a single object with this\n' - 'value. This object is accessed through the built-in name\n' - '"NotImplemented". Numeric methods and rich comparison methods ' - 'should\n' - 'return this value if they do not implement the operation for the\n' - 'operands provided. (The interpreter will then try the reflected\n' - 'operation, or some other fallback, depending on the operator.) It\n' - 'should not be evaluated in a boolean context.\n' - '\n' - 'See Implementing the arithmetic operations for more details.\n' - '\n' - 'Changed in version 3.9: Evaluating "NotImplemented" in a boolean\n' - 'context is deprecated. While it currently evaluates as true, it ' - 'will\n' - 'emit a "DeprecationWarning". It will raise a "TypeError" in a ' - 'future\n' - 'version of Python.\n' - '\n' - '\n' - 'Ellipsis\n' - '========\n' - '\n' - 'This type has a single value. There is a single object with this\n' - 'value. This object is accessed through the literal "..." or the ' - 'built-\n' - 'in name "Ellipsis". Its truth value is true.\n' - '\n' - '\n' - '"numbers.Number"\n' - '================\n' - '\n' - 'These are created by numeric literals and returned as results by\n' - 'arithmetic operators and arithmetic built-in functions. Numeric\n' - 'objects are immutable; once created their value never changes. ' - 'Python\n' - 'numbers are of course strongly related to mathematical numbers, ' - 'but\n' - 'subject to the limitations of numerical representation in ' - 'computers.\n' - '\n' - 'The string representations of the numeric classes, computed by\n' - '"__repr__()" and "__str__()", have the following properties:\n' - '\n' - '* They are valid numeric literals which, when passed to their ' - 'class\n' - ' constructor, produce an object having the value of the original\n' - ' numeric.\n' - '\n' - '* The representation is in base 10, when possible.\n' - '\n' - '* Leading zeros, possibly excepting a single zero before a decimal\n' - ' point, are not shown.\n' - '\n' - '* Trailing zeros, possibly excepting a single zero after a decimal\n' - ' point, are not shown.\n' - '\n' - '* A sign is shown only when the number is negative.\n' - '\n' - 'Python distinguishes between integers, floating point numbers, and\n' - 'complex numbers:\n' - '\n' - '\n' - '"numbers.Integral"\n' - '------------------\n' - '\n' - 'These represent elements from the mathematical set of integers\n' - '(positive and negative).\n' - '\n' - 'Note:\n' - '\n' - ' The rules for integer representation are intended to give the ' - 'most\n' - ' meaningful interpretation of shift and mask operations involving\n' - ' negative integers.\n' - '\n' - 'There are two types of integers:\n' - '\n' - 'Integers ("int")\n' - ' These represent numbers in an unlimited range, subject to ' - 'available\n' - ' (virtual) memory only. For the purpose of shift and mask\n' - ' operations, a binary representation is assumed, and negative\n' - ' numbers are represented in a variant of 2’s complement which ' - 'gives\n' - ' the illusion of an infinite string of sign bits extending to ' - 'the\n' - ' left.\n' - '\n' - 'Booleans ("bool")\n' - ' These represent the truth values False and True. The two ' - 'objects\n' - ' representing the values "False" and "True" are the only Boolean\n' - ' objects. The Boolean type is a subtype of the integer type, and\n' - ' Boolean values behave like the values 0 and 1, respectively, in\n' - ' almost all contexts, the exception being that when converted to ' - 'a\n' - ' string, the strings ""False"" or ""True"" are returned,\n' - ' respectively.\n' - '\n' - '\n' - '"numbers.Real" ("float")\n' - '------------------------\n' - '\n' - 'These represent machine-level double precision floating point ' - 'numbers.\n' - 'You are at the mercy of the underlying machine architecture (and C ' - 'or\n' - 'Java implementation) for the accepted range and handling of ' - 'overflow.\n' - 'Python does not support single-precision floating point numbers; ' - 'the\n' - 'savings in processor and memory usage that are usually the reason ' - 'for\n' - 'using these are dwarfed by the overhead of using objects in Python, ' - 'so\n' - 'there is no reason to complicate the language with two kinds of\n' - 'floating point numbers.\n' - '\n' - '\n' - '"numbers.Complex" ("complex")\n' - '-----------------------------\n' - '\n' - 'These represent complex numbers as a pair of machine-level double\n' - 'precision floating point numbers. The same caveats apply as for\n' - 'floating point numbers. The real and imaginary parts of a complex\n' - 'number "z" can be retrieved through the read-only attributes ' - '"z.real"\n' - 'and "z.imag".\n' - '\n' - '\n' - 'Sequences\n' - '=========\n' - '\n' - 'These represent finite ordered sets indexed by non-negative ' - 'numbers.\n' - 'The built-in function "len()" returns the number of items of a\n' - 'sequence. When the length of a sequence is *n*, the index set ' - 'contains\n' - 'the numbers 0, 1, …, *n*-1. Item *i* of sequence *a* is selected ' - 'by\n' - '"a[i]".\n' - '\n' - 'Sequences also support slicing: "a[i:j]" selects all items with ' - 'index\n' - '*k* such that *i* "<=" *k* "<" *j*. When used as an expression, a\n' - 'slice is a sequence of the same type. This implies that the index ' - 'set\n' - 'is renumbered so that it starts at 0.\n' - '\n' - 'Some sequences also support “extended slicing” with a third “step”\n' - 'parameter: "a[i:j:k]" selects all items of *a* with index *x* where ' - '"x\n' - '= i + n*k", *n* ">=" "0" and *i* "<=" *x* "<" *j*.\n' - '\n' - 'Sequences are distinguished according to their mutability:\n' - '\n' - '\n' - 'Immutable sequences\n' - '-------------------\n' - '\n' - 'An object of an immutable sequence type cannot change once it is\n' - 'created. (If the object contains references to other objects, ' - 'these\n' - 'other objects may be mutable and may be changed; however, the\n' - 'collection of objects directly referenced by an immutable object\n' - 'cannot change.)\n' - '\n' - 'The following types are immutable sequences:\n' - '\n' - 'Strings\n' - ' A string is a sequence of values that represent Unicode code\n' - ' points. All the code points in the range "U+0000 - U+10FFFF" can ' - 'be\n' - ' represented in a string. Python doesn’t have a char type; ' - 'instead,\n' - ' every code point in the string is represented as a string ' - 'object\n' - ' with length "1". The built-in function "ord()" converts a code\n' - ' point from its string form to an integer in the range "0 - ' - '10FFFF";\n' - ' "chr()" converts an integer in the range "0 - 10FFFF" to the\n' - ' corresponding length "1" string object. "str.encode()" can be ' - 'used\n' - ' to convert a "str" to "bytes" using the given text encoding, ' - 'and\n' - ' "bytes.decode()" can be used to achieve the opposite.\n' - '\n' - 'Tuples\n' - ' The items of a tuple are arbitrary Python objects. Tuples of two ' - 'or\n' - ' more items are formed by comma-separated lists of expressions. ' - 'A\n' - ' tuple of one item (a ‘singleton’) can be formed by affixing a ' - 'comma\n' - ' to an expression (an expression by itself does not create a ' - 'tuple,\n' - ' since parentheses must be usable for grouping of expressions). ' - 'An\n' - ' empty tuple can be formed by an empty pair of parentheses.\n' - '\n' - 'Bytes\n' - ' A bytes object is an immutable array. The items are 8-bit ' - 'bytes,\n' - ' represented by integers in the range 0 <= x < 256. Bytes ' - 'literals\n' - ' (like "b\'abc\'") and the built-in "bytes()" constructor can be ' - 'used\n' - ' to create bytes objects. Also, bytes objects can be decoded to\n' - ' strings via the "decode()" method.\n' - '\n' - '\n' - 'Mutable sequences\n' - '-----------------\n' - '\n' - 'Mutable sequences can be changed after they are created. The\n' - 'subscription and slicing notations can be used as the target of\n' - 'assignment and "del" (delete) statements.\n' - '\n' - 'Note:\n' - '\n' - ' The "collections" and "array" module provide additional examples ' - 'of\n' - ' mutable sequence types.\n' - '\n' - 'There are currently two intrinsic mutable sequence types:\n' - '\n' - 'Lists\n' - ' The items of a list are arbitrary Python objects. Lists are ' - 'formed\n' - ' by placing a comma-separated list of expressions in square\n' - ' brackets. (Note that there are no special cases needed to form\n' - ' lists of length 0 or 1.)\n' - '\n' - 'Byte Arrays\n' - ' A bytearray object is a mutable array. They are created by the\n' - ' built-in "bytearray()" constructor. Aside from being mutable ' - '(and\n' - ' hence unhashable), byte arrays otherwise provide the same ' - 'interface\n' - ' and functionality as immutable "bytes" objects.\n' - '\n' - '\n' - 'Set types\n' - '=========\n' - '\n' - 'These represent unordered, finite sets of unique, immutable ' - 'objects.\n' - 'As such, they cannot be indexed by any subscript. However, they can ' - 'be\n' - 'iterated over, and the built-in function "len()" returns the number ' - 'of\n' - 'items in a set. Common uses for sets are fast membership testing,\n' - 'removing duplicates from a sequence, and computing mathematical\n' - 'operations such as intersection, union, difference, and symmetric\n' - 'difference.\n' - '\n' - 'For set elements, the same immutability rules apply as for ' - 'dictionary\n' - 'keys. Note that numeric types obey the normal rules for numeric\n' - 'comparison: if two numbers compare equal (e.g., "1" and "1.0"), ' - 'only\n' - 'one of them can be contained in a set.\n' - '\n' - 'There are currently two intrinsic set types:\n' - '\n' - 'Sets\n' - ' These represent a mutable set. They are created by the built-in\n' - ' "set()" constructor and can be modified afterwards by several\n' - ' methods, such as "add()".\n' - '\n' - 'Frozen sets\n' - ' These represent an immutable set. They are created by the ' - 'built-in\n' - ' "frozenset()" constructor. As a frozenset is immutable and\n' - ' *hashable*, it can be used again as an element of another set, ' - 'or\n' - ' as a dictionary key.\n' - '\n' - '\n' - 'Mappings\n' - '========\n' - '\n' - 'These represent finite sets of objects indexed by arbitrary index\n' - 'sets. The subscript notation "a[k]" selects the item indexed by ' - '"k"\n' - 'from the mapping "a"; this can be used in expressions and as the\n' - 'target of assignments or "del" statements. The built-in function\n' - '"len()" returns the number of items in a mapping.\n' - '\n' - 'There is currently a single intrinsic mapping type:\n' - '\n' - '\n' - 'Dictionaries\n' - '------------\n' - '\n' - 'These represent finite sets of objects indexed by nearly arbitrary\n' - 'values. The only types of values not acceptable as keys are ' - 'values\n' - 'containing lists or dictionaries or other mutable types that are\n' - 'compared by value rather than by object identity, the reason being\n' - 'that the efficient implementation of dictionaries requires a key’s\n' - 'hash value to remain constant. Numeric types used for keys obey ' - 'the\n' - 'normal rules for numeric comparison: if two numbers compare equal\n' - '(e.g., "1" and "1.0") then they can be used interchangeably to ' - 'index\n' - 'the same dictionary entry.\n' - '\n' - 'Dictionaries preserve insertion order, meaning that keys will be\n' - 'produced in the same order they were added sequentially over the\n' - 'dictionary. Replacing an existing key does not change the order,\n' - 'however removing a key and re-inserting it will add it to the end\n' - 'instead of keeping its old place.\n' - '\n' - 'Dictionaries are mutable; they can be created by the "{...}" ' - 'notation\n' - '(see section Dictionary displays).\n' - '\n' - 'The extension modules "dbm.ndbm" and "dbm.gnu" provide additional\n' - 'examples of mapping types, as does the "collections" module.\n' - '\n' - 'Changed in version 3.7: Dictionaries did not preserve insertion ' - 'order\n' - 'in versions of Python before 3.6. In CPython 3.6, insertion order ' - 'was\n' - 'preserved, but it was considered an implementation detail at that ' - 'time\n' - 'rather than a language guarantee.\n' - '\n' - '\n' - 'Callable types\n' - '==============\n' - '\n' - 'These are the types to which the function call operation (see ' - 'section\n' - 'Calls) can be applied:\n' - '\n' - '\n' - 'User-defined functions\n' - '----------------------\n' - '\n' - 'A user-defined function object is created by a function definition\n' - '(see section Function definitions). It should be called with an\n' - 'argument list containing the same number of items as the ' - 'function’s\n' - 'formal parameter list.\n' - '\n' - '\n' - 'Special read-only attributes\n' - '~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' - '\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| Attribute | ' - 'Meaning |\n' - '|====================================================|====================================================|\n' - '| function.__globals__ | A reference ' - 'to the "dictionary" that holds the |\n' - '| | function’s ' - 'global variables – the global namespace |\n' - '| | of the ' - 'module in which the function was defined. |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| function.__closure__ | "None" or a ' - '"tuple" of cells that contain bindings |\n' - '| | for the ' - 'function’s free variables. A cell object |\n' - '| | has the ' - 'attribute "cell_contents". This can be |\n' - '| | used to get ' - 'the value of the cell, as well as set |\n' - '| | the ' - 'value. |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '\n' - '\n' - 'Special writable attributes\n' - '~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' - '\n' - 'Most of these attributes check the type of the assigned value:\n' - '\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| Attribute | ' - 'Meaning |\n' - '|====================================================|====================================================|\n' - '| function.__doc__ | The ' - 'function’s documentation string, or "None" if |\n' - '| | unavailable. ' - 'Not inherited by subclasses. |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| function.__name__ | The ' - 'function’s name. See also: "__name__ |\n' - '| | ' - 'attributes". |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| function.__qualname__ | The ' - 'function’s *qualified name*. See also: |\n' - '| | ' - '"__qualname__ attributes". New in version 3.3. |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| function.__module__ | The name of ' - 'the module the function was defined |\n' - '| | in, or ' - '"None" if unavailable. |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| function.__defaults__ | A "tuple" ' - 'containing default *parameter* values |\n' - '| | for those ' - 'parameters that have defaults, or "None" |\n' - '| | if no ' - 'parameters have a default value. |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| function.__code__ | The code ' - 'object representing the compiled function |\n' - '| | ' - 'body. |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| function.__dict__ | The ' - 'namespace supporting arbitrary function |\n' - '| | attributes. ' - 'See also: "__dict__ attributes". |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| function.__annotations__ | A ' - '"dictionary" containing annotations of |\n' - '| | ' - '*parameters*. The keys of the dictionary are the |\n' - '| | parameter ' - 'names, and "\'return\'" for the return |\n' - '| | annotation, ' - 'if provided. See also: Annotations |\n' - '| | Best ' - 'Practices. |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| function.__kwdefaults__ | A ' - '"dictionary" containing defaults for keyword- |\n' - '| | only ' - '*parameters*. |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| function.__type_params__ | A "tuple" ' - 'containing the type parameters of a |\n' - '| | generic ' - 'function. New in version 3.12. |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '\n' - 'Function objects also support getting and setting arbitrary\n' - 'attributes, which can be used, for example, to attach metadata to\n' - 'functions. Regular attribute dot-notation is used to get and set ' - 'such\n' - 'attributes.\n' - '\n' - '**CPython implementation detail:** CPython’s current ' - 'implementation\n' - 'only supports function attributes on user-defined functions. ' - 'Function\n' - 'attributes on built-in functions may be supported in the future.\n' - '\n' - 'Additional information about a function’s definition can be ' - 'retrieved\n' - 'from its code object (accessible via the "__code__" attribute).\n' - '\n' - '\n' - 'Instance methods\n' - '----------------\n' - '\n' - 'An instance method object combines a class, a class instance and ' - 'any\n' - 'callable object (normally a user-defined function).\n' - '\n' - 'Special read-only attributes:\n' - '\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| method.__self__ | Refers to ' - 'the class instance object to which the |\n' - '| | method is ' - 'bound |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| method.__func__ | Refers to ' - 'the original function object |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| method.__doc__ | The method’s ' - 'documentation (same as |\n' - '| | ' - '"method.__func__.__doc__"). A "string" if the |\n' - '| | original ' - 'function had a docstring, else "None". |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| method.__name__ | The name of ' - 'the method (same as |\n' - '| | ' - '"method.__func__.__name__") |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| method.__module__ | The name of ' - 'the module the method was defined in, |\n' - '| | or "None" if ' - 'unavailable. |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '\n' - 'Methods also support accessing (but not setting) the arbitrary\n' - 'function attributes on the underlying function object.\n' - '\n' - 'User-defined method objects may be created when getting an ' - 'attribute\n' - 'of a class (perhaps via an instance of that class), if that ' - 'attribute\n' - 'is a user-defined function object or a "classmethod" object.\n' - '\n' - 'When an instance method object is created by retrieving a ' - 'user-defined\n' - 'function object from a class via one of its instances, its ' - '"__self__"\n' - 'attribute is the instance, and the method object is said to be\n' - '*bound*. The new method’s "__func__" attribute is the original\n' - 'function object.\n' - '\n' - 'When an instance method object is created by retrieving a\n' - '"classmethod" object from a class or instance, its "__self__"\n' - 'attribute is the class itself, and its "__func__" attribute is the\n' - 'function object underlying the class method.\n' - '\n' - 'When an instance method object is called, the underlying function\n' - '("__func__") is called, inserting the class instance ("__self__") ' - 'in\n' - 'front of the argument list. For instance, when "C" is a class ' - 'which\n' - 'contains a definition for a function "f()", and "x" is an instance ' - 'of\n' - '"C", calling "x.f(1)" is equivalent to calling "C.f(x, 1)".\n' - '\n' - 'When an instance method object is derived from a "classmethod" ' - 'object,\n' - 'the “class instance” stored in "__self__" will actually be the ' - 'class\n' - 'itself, so that calling either "x.f(1)" or "C.f(1)" is equivalent ' - 'to\n' - 'calling "f(C,1)" where "f" is the underlying function.\n' - '\n' - 'Note that the transformation from function object to instance ' - 'method\n' - 'object happens each time the attribute is retrieved from the ' - 'instance.\n' - 'In some cases, a fruitful optimization is to assign the attribute ' - 'to a\n' - 'local variable and call that local variable. Also notice that this\n' - 'transformation only happens for user-defined functions; other ' - 'callable\n' - 'objects (and all non-callable objects) are retrieved without\n' - 'transformation. It is also important to note that user-defined\n' - 'functions which are attributes of a class instance are not ' - 'converted\n' - 'to bound methods; this *only* happens when the function is an\n' - 'attribute of the class.\n' - '\n' - '\n' - 'Generator functions\n' - '-------------------\n' - '\n' - 'A function or method which uses the "yield" statement (see section ' - 'The\n' - 'yield statement) is called a *generator function*. Such a ' - 'function,\n' - 'when called, always returns an *iterator* object which can be used ' - 'to\n' - 'execute the body of the function: calling the iterator’s\n' - '"iterator.__next__()" method will cause the function to execute ' - 'until\n' - 'it provides a value using the "yield" statement. When the ' - 'function\n' - 'executes a "return" statement or falls off the end, a ' - '"StopIteration"\n' - 'exception is raised and the iterator will have reached the end of ' - 'the\n' - 'set of values to be returned.\n' - '\n' - '\n' - 'Coroutine functions\n' - '-------------------\n' - '\n' - 'A function or method which is defined using "async def" is called ' - 'a\n' - '*coroutine function*. Such a function, when called, returns a\n' - '*coroutine* object. It may contain "await" expressions, as well ' - 'as\n' - '"async with" and "async for" statements. See also the Coroutine\n' - 'Objects section.\n' - '\n' - '\n' - 'Asynchronous generator functions\n' - '--------------------------------\n' - '\n' - 'A function or method which is defined using "async def" and which ' - 'uses\n' - 'the "yield" statement is called a *asynchronous generator ' - 'function*.\n' - 'Such a function, when called, returns an *asynchronous iterator*\n' - 'object which can be used in an "async for" statement to execute ' - 'the\n' - 'body of the function.\n' - '\n' - 'Calling the asynchronous iterator’s "aiterator.__anext__" method ' - 'will\n' - 'return an *awaitable* which when awaited will execute until it\n' - 'provides a value using the "yield" expression. When the function\n' - 'executes an empty "return" statement or falls off the end, a\n' - '"StopAsyncIteration" exception is raised and the asynchronous ' - 'iterator\n' - 'will have reached the end of the set of values to be yielded.\n' - '\n' - '\n' - 'Built-in functions\n' - '------------------\n' - '\n' - 'A built-in function object is a wrapper around a C function. ' - 'Examples\n' - 'of built-in functions are "len()" and "math.sin()" ("math" is a\n' - 'standard built-in module). The number and type of the arguments ' - 'are\n' - 'determined by the C function. Special read-only attributes:\n' - '\n' - '* "__doc__" is the function’s documentation string, or "None" if\n' - ' unavailable. See "function.__doc__".\n' - '\n' - '* "__name__" is the function’s name. See "function.__name__".\n' - '\n' - '* "__self__" is set to "None" (but see the next item).\n' - '\n' - '* "__module__" is the name of the module the function was defined ' - 'in\n' - ' or "None" if unavailable. See "function.__module__".\n' - '\n' - '\n' - 'Built-in methods\n' - '----------------\n' - '\n' - 'This is really a different disguise of a built-in function, this ' - 'time\n' - 'containing an object passed to the C function as an implicit extra\n' - 'argument. An example of a built-in method is "alist.append()",\n' - 'assuming *alist* is a list object. In this case, the special ' - 'read-only\n' - 'attribute "__self__" is set to the object denoted by *alist*. (The\n' - 'attribute has the same semantics as it does with "other instance\n' - 'methods".)\n' - '\n' - '\n' - 'Classes\n' - '-------\n' - '\n' - 'Classes are callable. These objects normally act as factories for ' - 'new\n' - 'instances of themselves, but variations are possible for class ' - 'types\n' - 'that override "__new__()". The arguments of the call are passed ' - 'to\n' - '"__new__()" and, in the typical case, to "__init__()" to ' - 'initialize\n' - 'the new instance.\n' - '\n' - '\n' - 'Class Instances\n' - '---------------\n' - '\n' - 'Instances of arbitrary classes can be made callable by defining a\n' - '"__call__()" method in their class.\n' - '\n' - '\n' - 'Modules\n' - '=======\n' - '\n' - 'Modules are a basic organizational unit of Python code, and are\n' - 'created by the import system as invoked either by the "import"\n' - 'statement, or by calling functions such as ' - '"importlib.import_module()"\n' - 'and built-in "__import__()". A module object has a namespace\n' - 'implemented by a "dictionary" object (this is the dictionary\n' - 'referenced by the "__globals__" attribute of functions defined in ' - 'the\n' - 'module). Attribute references are translated to lookups in this\n' - 'dictionary, e.g., "m.x" is equivalent to "m.__dict__["x"]". A ' - 'module\n' - 'object does not contain the code object used to initialize the ' - 'module\n' - '(since it isn’t needed once the initialization is done).\n' - '\n' - 'Attribute assignment updates the module’s namespace dictionary, ' - 'e.g.,\n' - '"m.x = 1" is equivalent to "m.__dict__["x"] = 1".\n' - '\n' - 'Predefined (writable) attributes:\n' - '\n' - ' "__name__"\n' - ' The module’s name.\n' - '\n' - ' "__doc__"\n' - ' The module’s documentation string, or "None" if unavailable.\n' - '\n' - ' "__file__"\n' - ' The pathname of the file from which the module was loaded, if ' - 'it\n' - ' was loaded from a file. The "__file__" attribute may be ' - 'missing\n' - ' for certain types of modules, such as C modules that are\n' - ' statically linked into the interpreter. For extension ' - 'modules\n' - ' loaded dynamically from a shared library, it’s the pathname ' - 'of\n' - ' the shared library file.\n' - '\n' - ' "__annotations__"\n' - ' A dictionary containing *variable annotations* collected ' - 'during\n' - ' module body execution. For best practices on working with\n' - ' "__annotations__", please see Annotations Best Practices.\n' - '\n' - 'Special read-only attribute: "__dict__" is the module’s namespace ' - 'as a\n' - 'dictionary object.\n' - '\n' - '**CPython implementation detail:** Because of the way CPython ' - 'clears\n' - 'module dictionaries, the module dictionary will be cleared when ' - 'the\n' - 'module falls out of scope even if the dictionary still has live\n' - 'references. To avoid this, copy the dictionary or keep the module\n' - 'around while using its dictionary directly.\n' - '\n' - '\n' - 'Custom classes\n' - '==============\n' - '\n' - 'Custom class types are typically created by class definitions (see\n' - 'section Class definitions). A class has a namespace implemented by ' - 'a\n' - 'dictionary object. Class attribute references are translated to\n' - 'lookups in this dictionary, e.g., "C.x" is translated to\n' - '"C.__dict__["x"]" (although there are a number of hooks which ' - 'allow\n' - 'for other means of locating attributes). When the attribute name ' - 'is\n' - 'not found there, the attribute search continues in the base ' - 'classes.\n' - 'This search of the base classes uses the C3 method resolution ' - 'order\n' - 'which behaves correctly even in the presence of ‘diamond’ ' - 'inheritance\n' - 'structures where there are multiple inheritance paths leading back ' - 'to\n' - 'a common ancestor. Additional details on the C3 MRO used by Python ' - 'can\n' - 'be found in the documentation accompanying the 2.3 release at\n' - 'https://www.python.org/download/releases/2.3/mro/.\n' - '\n' - 'When a class attribute reference (for class "C", say) would yield ' - 'a\n' - 'class method object, it is transformed into an instance method ' - 'object\n' - 'whose "__self__" attribute is "C". When it would yield a\n' - '"staticmethod" object, it is transformed into the object wrapped ' - 'by\n' - 'the static method object. See section Implementing Descriptors for\n' - 'another way in which attributes retrieved from a class may differ ' - 'from\n' - 'those actually contained in its "__dict__".\n' - '\n' - 'Class attribute assignments update the class’s dictionary, never ' - 'the\n' - 'dictionary of a base class.\n' - '\n' - 'A class object can be called (see above) to yield a class instance\n' - '(see below).\n' - '\n' - 'Special attributes:\n' - '\n' - ' "__name__"\n' - ' The class name.\n' - '\n' - ' "__module__"\n' - ' The name of the module in which the class was defined.\n' - '\n' - ' "__dict__"\n' - ' The dictionary containing the class’s namespace.\n' - '\n' - ' "__bases__"\n' - ' A tuple containing the base classes, in the order of their\n' - ' occurrence in the base class list.\n' - '\n' - ' "__doc__"\n' - ' The class’s documentation string, or "None" if undefined.\n' - '\n' - ' "__annotations__"\n' - ' A dictionary containing *variable annotations* collected ' - 'during\n' - ' class body execution. For best practices on working with\n' - ' "__annotations__", please see Annotations Best Practices.\n' - '\n' - ' "__type_params__"\n' - ' A tuple containing the type parameters of a generic class.\n' - '\n' - '\n' - 'Class instances\n' - '===============\n' - '\n' - 'A class instance is created by calling a class object (see above). ' - 'A\n' - 'class instance has a namespace implemented as a dictionary which ' - 'is\n' - 'the first place in which attribute references are searched. When ' - 'an\n' - 'attribute is not found there, and the instance’s class has an\n' - 'attribute by that name, the search continues with the class\n' - 'attributes. If a class attribute is found that is a user-defined\n' - 'function object, it is transformed into an instance method object\n' - 'whose "__self__" attribute is the instance. Static method and ' - 'class\n' - 'method objects are also transformed; see above under “Classes”. ' - 'See\n' - 'section Implementing Descriptors for another way in which ' - 'attributes\n' - 'of a class retrieved via its instances may differ from the objects\n' - 'actually stored in the class’s "__dict__". If no class attribute ' - 'is\n' - 'found, and the object’s class has a "__getattr__()" method, that ' - 'is\n' - 'called to satisfy the lookup.\n' - '\n' - 'Attribute assignments and deletions update the instance’s ' - 'dictionary,\n' - 'never a class’s dictionary. If the class has a "__setattr__()" or\n' - '"__delattr__()" method, this is called instead of updating the\n' - 'instance dictionary directly.\n' - '\n' - 'Class instances can pretend to be numbers, sequences, or mappings ' - 'if\n' - 'they have methods with certain special names. See section Special\n' - 'method names.\n' - '\n' - 'Special attributes: "__dict__" is the attribute dictionary;\n' - '"__class__" is the instance’s class.\n' - '\n' - '\n' - 'I/O objects (also known as file objects)\n' - '========================================\n' - '\n' - 'A *file object* represents an open file. Various shortcuts are\n' - 'available to create file objects: the "open()" built-in function, ' - 'and\n' - 'also "os.popen()", "os.fdopen()", and the "makefile()" method of\n' - 'socket objects (and perhaps by other functions or methods provided ' - 'by\n' - 'extension modules).\n' - '\n' - 'The objects "sys.stdin", "sys.stdout" and "sys.stderr" are ' - 'initialized\n' - 'to file objects corresponding to the interpreter’s standard input,\n' - 'output and error streams; they are all open in text mode and ' - 'therefore\n' - 'follow the interface defined by the "io.TextIOBase" abstract ' - 'class.\n' - '\n' - '\n' - 'Internal types\n' - '==============\n' - '\n' - 'A few types used internally by the interpreter are exposed to the\n' - 'user. Their definitions may change with future versions of the\n' - 'interpreter, but they are mentioned here for completeness.\n' - '\n' - '\n' - 'Code objects\n' - '------------\n' - '\n' - 'Code objects represent *byte-compiled* executable Python code, or\n' - '*bytecode*. The difference between a code object and a function ' - 'object\n' - 'is that the function object contains an explicit reference to the\n' - 'function’s globals (the module in which it was defined), while a ' - 'code\n' - 'object contains no context; also the default argument values are\n' - 'stored in the function object, not in the code object (because ' - 'they\n' - 'represent values calculated at run-time). Unlike function ' - 'objects,\n' - 'code objects are immutable and contain no references (directly or\n' - 'indirectly) to mutable objects.\n' - '\n' - '\n' - 'Special read-only attributes\n' - '~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' - '\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| codeobject.co_name | The function ' - 'name |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| codeobject.co_qualname | The fully ' - 'qualified function name New in version |\n' - '| | ' - '3.11. |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| codeobject.co_argcount | The total ' - 'number of positional *parameters* |\n' - '| | (including ' - 'positional-only parameters and |\n' - '| | parameters ' - 'with default values) that the function |\n' - '| | ' - 'has |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| codeobject.co_posonlyargcount | The number ' - 'of positional-only *parameters* |\n' - '| | (including ' - 'arguments with default values) that the |\n' - '| | function ' - 'has |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| codeobject.co_kwonlyargcount | The number ' - 'of keyword-only *parameters* (including |\n' - '| | arguments ' - 'with default values) that the function |\n' - '| | ' - 'has |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| codeobject.co_nlocals | The number ' - 'of local variables used by the function |\n' - '| | (including ' - 'parameters) |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| codeobject.co_varnames | A "tuple" ' - 'containing the names of the local |\n' - '| | variables in ' - 'the function (starting with the |\n' - '| | parameter ' - 'names) |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| codeobject.co_cellvars | A "tuple" ' - 'containing the names of local variables |\n' - '| | that are ' - 'referenced by nested functions inside the |\n' - '| | ' - 'function |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| codeobject.co_freevars | A "tuple" ' - 'containing the names of free variables |\n' - '| | in the ' - 'function |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| codeobject.co_code | A string ' - 'representing the sequence of *bytecode* |\n' - '| | instructions ' - 'in the function |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| codeobject.co_consts | A "tuple" ' - 'containing the literals used by the |\n' - '| | *bytecode* ' - 'in the function |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| codeobject.co_names | A "tuple" ' - 'containing the names used by the |\n' - '| | *bytecode* ' - 'in the function |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| codeobject.co_filename | The name of ' - 'the file from which the code was |\n' - '| | ' - 'compiled |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| codeobject.co_firstlineno | The line ' - 'number of the first line of the function |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| codeobject.co_lnotab | A string ' - 'encoding the mapping from *bytecode* |\n' - '| | offsets to ' - 'line numbers. For details, see the |\n' - '| | source code ' - 'of the interpreter. Deprecated since |\n' - '| | version ' - '3.12: This attribute of code objects is |\n' - '| | deprecated, ' - 'and may be removed in Python 3.14. |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| codeobject.co_stacksize | The required ' - 'stack size of the code object |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| codeobject.co_flags | An "integer" ' - 'encoding a number of flags for the |\n' - '| | ' - 'interpreter. |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '\n' - 'The following flag bits are defined for "co_flags": bit "0x04" is ' - 'set\n' - 'if the function uses the "*arguments" syntax to accept an ' - 'arbitrary\n' - 'number of positional arguments; bit "0x08" is set if the function ' - 'uses\n' - 'the "**keywords" syntax to accept arbitrary keyword arguments; bit\n' - '"0x20" is set if the function is a generator. See Code Objects Bit\n' - 'Flags for details on the semantics of each flags that might be\n' - 'present.\n' - '\n' - 'Future feature declarations ("from __future__ import division") ' - 'also\n' - 'use bits in "co_flags" to indicate whether a code object was ' - 'compiled\n' - 'with a particular feature enabled: bit "0x2000" is set if the ' - 'function\n' - 'was compiled with future division enabled; bits "0x10" and ' - '"0x1000"\n' - 'were used in earlier versions of Python.\n' - '\n' - 'Other bits in "co_flags" are reserved for internal use.\n' - '\n' - 'If a code object represents a function, the first item in ' - '"co_consts"\n' - 'is the documentation string of the function, or "None" if ' - 'undefined.\n' - '\n' - '\n' - 'Methods on code objects\n' - '~~~~~~~~~~~~~~~~~~~~~~~\n' - '\n' - 'codeobject.co_positions()\n' - '\n' - ' Returns an iterable over the source code positions of each\n' - ' *bytecode* instruction in the code object.\n' - '\n' - ' The iterator returns "tuple"s containing the "(start_line,\n' - ' end_line, start_column, end_column)". The *i-th* tuple ' - 'corresponds\n' - ' to the position of the source code that compiled to the *i-th*\n' - ' instruction. Column information is 0-indexed utf-8 byte offsets ' - 'on\n' - ' the given source line.\n' - '\n' - ' This positional information can be missing. A non-exhaustive ' - 'lists\n' - ' of cases where this may happen:\n' - '\n' - ' * Running the interpreter with "-X" "no_debug_ranges".\n' - '\n' - ' * Loading a pyc file compiled while using "-X" ' - '"no_debug_ranges".\n' - '\n' - ' * Position tuples corresponding to artificial instructions.\n' - '\n' - ' * Line and column numbers that can’t be represented due to\n' - ' implementation specific limitations.\n' - '\n' - ' When this occurs, some or all of the tuple elements can be ' - '"None".\n' - '\n' - ' New in version 3.11.\n' - '\n' - ' Note:\n' - '\n' - ' This feature requires storing column positions in code ' - 'objects\n' - ' which may result in a small increase of disk usage of ' - 'compiled\n' - ' Python files or interpreter memory usage. To avoid storing ' - 'the\n' - ' extra information and/or deactivate printing the extra ' - 'traceback\n' - ' information, the "-X" "no_debug_ranges" command line flag or ' - 'the\n' - ' "PYTHONNODEBUGRANGES" environment variable can be used.\n' - '\n' - 'codeobject.co_lines()\n' - '\n' - ' Returns an iterator that yields information about successive ' - 'ranges\n' - ' of *bytecode*s. Each item yielded is a "(start, end, lineno)"\n' - ' "tuple":\n' - '\n' - ' * "start" (an "int") represents the offset (inclusive) of the ' - 'start\n' - ' of the *bytecode* range\n' - '\n' - ' * "end" (an "int") represents the offset (exclusive) of the end ' - 'of\n' - ' the *bytecode* range\n' - '\n' - ' * "lineno" is an "int" representing the line number of the\n' - ' *bytecode* range, or "None" if the bytecodes in the given ' - 'range\n' - ' have no line number\n' - '\n' - ' The items yielded will have the following properties:\n' - '\n' - ' * The first range yielded will have a "start" of 0.\n' - '\n' - ' * The "(start, end)" ranges will be non-decreasing and ' - 'consecutive.\n' - ' That is, for any pair of "tuple"s, the "start" of the second ' - 'will\n' - ' be equal to the "end" of the first.\n' - '\n' - ' * No range will be backwards: "end >= start" for all triples.\n' - '\n' - ' * The last "tuple" yielded will have "end" equal to the size of ' - 'the\n' - ' *bytecode*.\n' - '\n' - ' Zero-width ranges, where "start == end", are allowed. ' - 'Zero-width\n' - ' ranges are used for lines that are present in the source code, ' - 'but\n' - ' have been eliminated by the *bytecode* compiler.\n' - '\n' - ' New in version 3.10.\n' - '\n' - ' See also:\n' - '\n' - ' **PEP 626** - Precise line numbers for debugging and other ' - 'tools.\n' - ' The PEP that introduced the "co_lines()" method.\n' - '\n' - '\n' - 'Frame objects\n' - '-------------\n' - '\n' - 'Frame objects represent execution frames. They may occur in ' - 'traceback\n' - 'objects, and are also passed to registered trace functions.\n' - '\n' - '\n' - 'Special read-only attributes\n' - '~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' - '\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| frame.f_back | Points to ' - 'the previous stack frame (towards the |\n' - '| | caller), or ' - '"None" if this is the bottom stack |\n' - '| | ' - 'frame |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| frame.f_code | The code ' - 'object being executed in this frame. |\n' - '| | Accessing ' - 'this attribute raises an auditing event |\n' - '| | ' - '"object.__getattr__" with arguments "obj" and |\n' - '| | ' - '""f_code"". |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| frame.f_locals | The ' - 'dictionary used by the frame to look up local |\n' - '| | ' - 'variables |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| frame.f_globals | The ' - 'dictionary used by the frame to look up global |\n' - '| | ' - 'variables |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| frame.f_builtins | The ' - 'dictionary used by the frame to look up built- |\n' - '| | in ' - '(intrinsic) names |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| frame.f_lasti | The “precise ' - 'instruction” of the frame object |\n' - '| | (this is an ' - 'index into the *bytecode* string of |\n' - '| | the code ' - 'object) |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '\n' - '\n' - 'Special writable attributes\n' - '~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' - '\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| frame.f_trace | If not ' - '"None", this is a function called for |\n' - '| | various ' - 'events during code execution (this is used |\n' - '| | by ' - 'debuggers). Normally an event is triggered for |\n' - '| | each new ' - 'source line (see "f_trace_lines"). |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| frame.f_trace_lines | Set this ' - 'attribute to "False" to disable |\n' - '| | triggering a ' - 'tracing event for each source line. |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| frame.f_trace_opcodes | Set this ' - 'attribute to "True" to allow per-opcode |\n' - '| | events to be ' - 'requested. Note that this may lead to |\n' - '| | undefined ' - 'interpreter behaviour if exceptions |\n' - '| | raised by ' - 'the trace function escape to the |\n' - '| | function ' - 'being traced. |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| frame.f_lineno | The current ' - 'line number of the frame – writing to |\n' - '| | this from ' - 'within a trace function jumps to the |\n' - '| | given line ' - '(only for the bottom-most frame). A |\n' - '| | debugger can ' - 'implement a Jump command (aka Set |\n' - '| | Next ' - 'Statement) by writing to this attribute. |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '\n' - '\n' - 'Frame object methods\n' - '~~~~~~~~~~~~~~~~~~~~\n' - '\n' - 'Frame objects support one method:\n' - '\n' - 'frame.clear()\n' - '\n' - ' This method clears all references to local variables held by ' - 'the\n' - ' frame. Also, if the frame belonged to a *generator*, the ' - 'generator\n' - ' is finalized. This helps break reference cycles involving ' - 'frame\n' - ' objects (for example when catching an exception and storing its\n' - ' traceback for later use).\n' - '\n' - ' "RuntimeError" is raised if the frame is currently executing or\n' - ' suspended.\n' - '\n' - ' New in version 3.4.\n' - '\n' - ' Changed in version 3.13: Attempting to clear a suspended frame\n' - ' raises "RuntimeError" (as has always been the case for ' - 'executing\n' - ' frames).\n' - '\n' - '\n' - 'Traceback objects\n' - '-----------------\n' - '\n' - 'Traceback objects represent the stack trace of an exception. A\n' - 'traceback object is implicitly created when an exception occurs, ' - 'and\n' - 'may also be explicitly created by calling "types.TracebackType".\n' - '\n' - 'Changed in version 3.7: Traceback objects can now be explicitly\n' - 'instantiated from Python code.\n' - '\n' - 'For implicitly created tracebacks, when the search for an ' - 'exception\n' - 'handler unwinds the execution stack, at each unwound level a ' - 'traceback\n' - 'object is inserted in front of the current traceback. When an\n' - 'exception handler is entered, the stack trace is made available to ' - 'the\n' - 'program. (See section The try statement.) It is accessible as the\n' - 'third item of the tuple returned by "sys.exc_info()", and as the\n' - '"__traceback__" attribute of the caught exception.\n' - '\n' - 'When the program contains no suitable handler, the stack trace is\n' - 'written (nicely formatted) to the standard error stream; if the\n' - 'interpreter is interactive, it is also made available to the user ' - 'as\n' - '"sys.last_traceback".\n' - '\n' - 'For explicitly created tracebacks, it is up to the creator of the\n' - 'traceback to determine how the "tb_next" attributes should be ' - 'linked\n' - 'to form a full stack trace.\n' - '\n' - 'Special read-only attributes:\n' - '\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| traceback.tb_frame | Points to ' - 'the execution frame of the current |\n' - '| | level. ' - 'Accessing this attribute raises an |\n' - '| | auditing ' - 'event "object.__getattr__" with arguments |\n' - '| | "obj" and ' - '""tb_frame"". |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| traceback.tb_lineno | Gives the ' - 'line number where the exception occurred |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '| traceback.tb_lasti | Indicates ' - 'the “precise instruction”. |\n' - '+----------------------------------------------------+----------------------------------------------------+\n' - '\n' - 'The line number and last instruction in the traceback may differ ' - 'from\n' - 'the line number of its frame object if the exception occurred in a\n' - '"try" statement with no matching except clause or with a "finally"\n' - 'clause.\n' - '\n' - 'traceback.tb_next\n' - '\n' - ' The special writable attribute "tb_next" is the next level in ' - 'the\n' - ' stack trace (towards the frame where the exception occurred), ' - 'or\n' - ' "None" if there is no next level.\n' - '\n' - ' Changed in version 3.7: This attribute is now writable\n' - '\n' - '\n' - 'Slice objects\n' - '-------------\n' - '\n' - 'Slice objects are used to represent slices for "__getitem__()"\n' - 'methods. They are also created by the built-in "slice()" ' - 'function.\n' - '\n' - 'Special read-only attributes: "start" is the lower bound; "stop" ' - 'is\n' - 'the upper bound; "step" is the step value; each is "None" if ' - 'omitted.\n' - 'These attributes can have any type.\n' - '\n' - 'Slice objects support one method:\n' - '\n' - 'slice.indices(self, length)\n' - '\n' - ' This method takes a single integer argument *length* and ' - 'computes\n' - ' information about the slice that the slice object would describe ' - 'if\n' - ' applied to a sequence of *length* items. It returns a tuple of\n' - ' three integers; respectively these are the *start* and *stop*\n' - ' indices and the *step* or stride length of the slice. Missing ' - 'or\n' - ' out-of-bounds indices are handled in a manner consistent with\n' - ' regular slices.\n' - '\n' - '\n' - 'Static method objects\n' - '---------------------\n' - '\n' - 'Static method objects provide a way of defeating the transformation ' - 'of\n' - 'function objects to method objects described above. A static ' - 'method\n' - 'object is a wrapper around any other object, usually a ' - 'user-defined\n' - 'method object. When a static method object is retrieved from a ' - 'class\n' - 'or a class instance, the object actually returned is the wrapped\n' - 'object, which is not subject to any further transformation. Static\n' - 'method objects are also callable. Static method objects are created ' - 'by\n' - 'the built-in "staticmethod()" constructor.\n' - '\n' - '\n' - 'Class method objects\n' - '--------------------\n' - '\n' - 'A class method object, like a static method object, is a wrapper\n' - 'around another object that alters the way in which that object is\n' - 'retrieved from classes and class instances. The behaviour of class\n' - 'method objects upon such retrieval is described above, under ' - '“instance\n' - 'methods”. Class method objects are created by the built-in\n' - '"classmethod()" constructor.\n', - 'typesfunctions': 'Functions\n' - '*********\n' - '\n' - 'Function objects are created by function definitions. The ' - 'only\n' - 'operation on a function object is to call it: ' - '"func(argument-list)".\n' - '\n' - 'There are really two flavors of function objects: built-in ' - 'functions\n' - 'and user-defined functions. Both support the same ' - 'operation (to call\n' - 'the function), but the implementation is different, hence ' - 'the\n' - 'different object types.\n' - '\n' - 'See Function definitions for more information.\n', - 'typesmapping': 'Mapping Types — "dict"\n' - '**********************\n' - '\n' - 'A *mapping* object maps *hashable* values to arbitrary ' - 'objects.\n' - 'Mappings are mutable objects. There is currently only one ' - 'standard\n' - 'mapping type, the *dictionary*. (For other containers see ' - 'the built-\n' - 'in "list", "set", and "tuple" classes, and the "collections" ' - 'module.)\n' - '\n' - 'A dictionary’s keys are *almost* arbitrary values. Values ' - 'that are\n' - 'not *hashable*, that is, values containing lists, ' - 'dictionaries or\n' - 'other mutable types (that are compared by value rather than ' - 'by object\n' - 'identity) may not be used as keys. Values that compare equal ' - '(such as\n' - '"1", "1.0", and "True") can be used interchangeably to index ' - 'the same\n' - 'dictionary entry.\n' - '\n' - 'class dict(**kwargs)\n' - 'class dict(mapping, **kwargs)\n' - 'class dict(iterable, **kwargs)\n' - '\n' - ' Return a new dictionary initialized from an optional ' - 'positional\n' - ' argument and a possibly empty set of keyword arguments.\n' - '\n' - ' Dictionaries can be created by several means:\n' - '\n' - ' * Use a comma-separated list of "key: value" pairs within ' - 'braces:\n' - ' "{\'jack\': 4098, \'sjoerd\': 4127}" or "{4098: ' - "'jack', 4127:\n" - ' \'sjoerd\'}"\n' - '\n' - ' * Use a dict comprehension: "{}", "{x: x ** 2 for x in ' - 'range(10)}"\n' - '\n' - ' * Use the type constructor: "dict()", "dict([(\'foo\', ' - "100), ('bar',\n" - ' 200)])", "dict(foo=100, bar=200)"\n' - '\n' - ' If no positional argument is given, an empty dictionary ' - 'is created.\n' - ' If a positional argument is given and it is a mapping ' - 'object, a\n' - ' dictionary is created with the same key-value pairs as ' - 'the mapping\n' - ' object. Otherwise, the positional argument must be an ' - '*iterable*\n' - ' object. Each item in the iterable must itself be an ' - 'iterable with\n' - ' exactly two objects. The first object of each item ' - 'becomes a key\n' - ' in the new dictionary, and the second object the ' - 'corresponding\n' - ' value. If a key occurs more than once, the last value ' - 'for that key\n' - ' becomes the corresponding value in the new dictionary.\n' - '\n' - ' If keyword arguments are given, the keyword arguments and ' - 'their\n' - ' values are added to the dictionary created from the ' - 'positional\n' - ' argument. If a key being added is already present, the ' - 'value from\n' - ' the keyword argument replaces the value from the ' - 'positional\n' - ' argument.\n' - '\n' - ' To illustrate, the following examples all return a ' - 'dictionary equal\n' - ' to "{"one": 1, "two": 2, "three": 3}":\n' - '\n' - ' >>> a = dict(one=1, two=2, three=3)\n' - " >>> b = {'one': 1, 'two': 2, 'three': 3}\n" - " >>> c = dict(zip(['one', 'two', 'three'], [1, 2, 3]))\n" - " >>> d = dict([('two', 2), ('one', 1), ('three', 3)])\n" - " >>> e = dict({'three': 3, 'one': 1, 'two': 2})\n" - " >>> f = dict({'one': 1, 'three': 3}, two=2)\n" - ' >>> a == b == c == d == e == f\n' - ' True\n' - '\n' - ' Providing keyword arguments as in the first example only ' - 'works for\n' - ' keys that are valid Python identifiers. Otherwise, any ' - 'valid keys\n' - ' can be used.\n' - '\n' - ' These are the operations that dictionaries support (and ' - 'therefore,\n' - ' custom mapping types should support too):\n' - '\n' - ' list(d)\n' - '\n' - ' Return a list of all the keys used in the dictionary ' - '*d*.\n' - '\n' - ' len(d)\n' - '\n' - ' Return the number of items in the dictionary *d*.\n' - '\n' - ' d[key]\n' - '\n' - ' Return the item of *d* with key *key*. Raises a ' - '"KeyError" if\n' - ' *key* is not in the map.\n' - '\n' - ' If a subclass of dict defines a method "__missing__()" ' - 'and *key*\n' - ' is not present, the "d[key]" operation calls that ' - 'method with\n' - ' the key *key* as argument. The "d[key]" operation ' - 'then returns\n' - ' or raises whatever is returned or raised by the\n' - ' "__missing__(key)" call. No other operations or ' - 'methods invoke\n' - ' "__missing__()". If "__missing__()" is not defined, ' - '"KeyError"\n' - ' is raised. "__missing__()" must be a method; it cannot ' - 'be an\n' - ' instance variable:\n' - '\n' - ' >>> class Counter(dict):\n' - ' ... def __missing__(self, key):\n' - ' ... return 0\n' - ' ...\n' - ' >>> c = Counter()\n' - " >>> c['red']\n" - ' 0\n' - " >>> c['red'] += 1\n" - " >>> c['red']\n" - ' 1\n' - '\n' - ' The example above shows part of the implementation of\n' - ' "collections.Counter". A different "__missing__" ' - 'method is used\n' - ' by "collections.defaultdict".\n' - '\n' - ' d[key] = value\n' - '\n' - ' Set "d[key]" to *value*.\n' - '\n' - ' del d[key]\n' - '\n' - ' Remove "d[key]" from *d*. Raises a "KeyError" if ' - '*key* is not\n' - ' in the map.\n' - '\n' - ' key in d\n' - '\n' - ' Return "True" if *d* has a key *key*, else "False".\n' - '\n' - ' key not in d\n' - '\n' - ' Equivalent to "not key in d".\n' - '\n' - ' iter(d)\n' - '\n' - ' Return an iterator over the keys of the dictionary. ' - 'This is a\n' - ' shortcut for "iter(d.keys())".\n' - '\n' - ' clear()\n' - '\n' - ' Remove all items from the dictionary.\n' - '\n' - ' copy()\n' - '\n' - ' Return a shallow copy of the dictionary.\n' - '\n' - ' classmethod fromkeys(iterable[, value])\n' - '\n' - ' Create a new dictionary with keys from *iterable* and ' - 'values set\n' - ' to *value*.\n' - '\n' - ' "fromkeys()" is a class method that returns a new ' - 'dictionary.\n' - ' *value* defaults to "None". All of the values refer ' - 'to just a\n' - ' single instance, so it generally doesn’t make sense ' - 'for *value*\n' - ' to be a mutable object such as an empty list. To get ' - 'distinct\n' - ' values, use a dict comprehension instead.\n' - '\n' - ' get(key[, default])\n' - '\n' - ' Return the value for *key* if *key* is in the ' - 'dictionary, else\n' - ' *default*. If *default* is not given, it defaults to ' - '"None", so\n' - ' that this method never raises a "KeyError".\n' - '\n' - ' items()\n' - '\n' - ' Return a new view of the dictionary’s items ("(key, ' - 'value)"\n' - ' pairs). See the documentation of view objects.\n' - '\n' - ' keys()\n' - '\n' - ' Return a new view of the dictionary’s keys. See the\n' - ' documentation of view objects.\n' - '\n' - ' pop(key[, default])\n' - '\n' - ' If *key* is in the dictionary, remove it and return ' - 'its value,\n' - ' else return *default*. If *default* is not given and ' - '*key* is\n' - ' not in the dictionary, a "KeyError" is raised.\n' - '\n' - ' popitem()\n' - '\n' - ' Remove and return a "(key, value)" pair from the ' - 'dictionary.\n' - ' Pairs are returned in LIFO (last-in, first-out) ' - 'order.\n' - '\n' - ' "popitem()" is useful to destructively iterate over a\n' - ' dictionary, as often used in set algorithms. If the ' - 'dictionary\n' - ' is empty, calling "popitem()" raises a "KeyError".\n' - '\n' - ' Changed in version 3.7: LIFO order is now guaranteed. ' - 'In prior\n' - ' versions, "popitem()" would return an arbitrary ' - 'key/value pair.\n' - '\n' - ' reversed(d)\n' - '\n' - ' Return a reverse iterator over the keys of the ' - 'dictionary. This\n' - ' is a shortcut for "reversed(d.keys())".\n' - '\n' - ' New in version 3.8.\n' - '\n' - ' setdefault(key[, default])\n' - '\n' - ' If *key* is in the dictionary, return its value. If ' - 'not, insert\n' - ' *key* with a value of *default* and return *default*. ' - '*default*\n' - ' defaults to "None".\n' - '\n' - ' update([other])\n' - '\n' - ' Update the dictionary with the key/value pairs from ' - '*other*,\n' - ' overwriting existing keys. Return "None".\n' - '\n' - ' "update()" accepts either another dictionary object or ' - 'an\n' - ' iterable of key/value pairs (as tuples or other ' - 'iterables of\n' - ' length two). If keyword arguments are specified, the ' - 'dictionary\n' - ' is then updated with those key/value pairs: ' - '"d.update(red=1,\n' - ' blue=2)".\n' - '\n' - ' values()\n' - '\n' - ' Return a new view of the dictionary’s values. See ' - 'the\n' - ' documentation of view objects.\n' - '\n' - ' An equality comparison between one "dict.values()" ' - 'view and\n' - ' another will always return "False". This also applies ' - 'when\n' - ' comparing "dict.values()" to itself:\n' - '\n' - " >>> d = {'a': 1}\n" - ' >>> d.values() == d.values()\n' - ' False\n' - '\n' - ' d | other\n' - '\n' - ' Create a new dictionary with the merged keys and ' - 'values of *d*\n' - ' and *other*, which must both be dictionaries. The ' - 'values of\n' - ' *other* take priority when *d* and *other* share ' - 'keys.\n' - '\n' - ' New in version 3.9.\n' - '\n' - ' d |= other\n' - '\n' - ' Update the dictionary *d* with keys and values from ' - '*other*,\n' - ' which may be either a *mapping* or an *iterable* of ' - 'key/value\n' - ' pairs. The values of *other* take priority when *d* ' - 'and *other*\n' - ' share keys.\n' - '\n' - ' New in version 3.9.\n' - '\n' - ' Dictionaries compare equal if and only if they have the ' - 'same "(key,\n' - ' value)" pairs (regardless of ordering). Order comparisons ' - '(‘<’,\n' - ' ‘<=’, ‘>=’, ‘>’) raise "TypeError".\n' - '\n' - ' Dictionaries preserve insertion order. Note that ' - 'updating a key\n' - ' does not affect the order. Keys added after deletion are ' - 'inserted\n' - ' at the end.\n' - '\n' - ' >>> d = {"one": 1, "two": 2, "three": 3, "four": 4}\n' - ' >>> d\n' - " {'one': 1, 'two': 2, 'three': 3, 'four': 4}\n" - ' >>> list(d)\n' - " ['one', 'two', 'three', 'four']\n" - ' >>> list(d.values())\n' - ' [1, 2, 3, 4]\n' - ' >>> d["one"] = 42\n' - ' >>> d\n' - " {'one': 42, 'two': 2, 'three': 3, 'four': 4}\n" - ' >>> del d["two"]\n' - ' >>> d["two"] = None\n' - ' >>> d\n' - " {'one': 42, 'three': 3, 'four': 4, 'two': None}\n" - '\n' - ' Changed in version 3.7: Dictionary order is guaranteed to ' - 'be\n' - ' insertion order. This behavior was an implementation ' - 'detail of\n' - ' CPython from 3.6.\n' - '\n' - ' Dictionaries and dictionary views are reversible.\n' - '\n' - ' >>> d = {"one": 1, "two": 2, "three": 3, "four": 4}\n' - ' >>> d\n' - " {'one': 1, 'two': 2, 'three': 3, 'four': 4}\n" - ' >>> list(reversed(d))\n' - " ['four', 'three', 'two', 'one']\n" - ' >>> list(reversed(d.values()))\n' - ' [4, 3, 2, 1]\n' - ' >>> list(reversed(d.items()))\n' - " [('four', 4), ('three', 3), ('two', 2), ('one', 1)]\n" - '\n' - ' Changed in version 3.8: Dictionaries are now reversible.\n' - '\n' - 'See also:\n' - '\n' - ' "types.MappingProxyType" can be used to create a read-only ' - 'view of a\n' - ' "dict".\n' - '\n' - '\n' - 'Dictionary view objects\n' - '=======================\n' - '\n' - 'The objects returned by "dict.keys()", "dict.values()" and\n' - '"dict.items()" are *view objects*. They provide a dynamic ' - 'view on the\n' - 'dictionary’s entries, which means that when the dictionary ' - 'changes,\n' - 'the view reflects these changes.\n' - '\n' - 'Dictionary views can be iterated over to yield their ' - 'respective data,\n' - 'and support membership tests:\n' - '\n' - 'len(dictview)\n' - '\n' - ' Return the number of entries in the dictionary.\n' - '\n' - 'iter(dictview)\n' - '\n' - ' Return an iterator over the keys, values or items ' - '(represented as\n' - ' tuples of "(key, value)") in the dictionary.\n' - '\n' - ' Keys and values are iterated over in insertion order. ' - 'This allows\n' - ' the creation of "(value, key)" pairs using "zip()": ' - '"pairs =\n' - ' zip(d.values(), d.keys())". Another way to create the ' - 'same list is\n' - ' "pairs = [(v, k) for (k, v) in d.items()]".\n' - '\n' - ' Iterating views while adding or deleting entries in the ' - 'dictionary\n' - ' may raise a "RuntimeError" or fail to iterate over all ' - 'entries.\n' - '\n' - ' Changed in version 3.7: Dictionary order is guaranteed to ' - 'be\n' - ' insertion order.\n' - '\n' - 'x in dictview\n' - '\n' - ' Return "True" if *x* is in the underlying dictionary’s ' - 'keys, values\n' - ' or items (in the latter case, *x* should be a "(key, ' - 'value)"\n' - ' tuple).\n' - '\n' - 'reversed(dictview)\n' - '\n' - ' Return a reverse iterator over the keys, values or items ' - 'of the\n' - ' dictionary. The view will be iterated in reverse order of ' - 'the\n' - ' insertion.\n' - '\n' - ' Changed in version 3.8: Dictionary views are now ' - 'reversible.\n' - '\n' - 'dictview.mapping\n' - '\n' - ' Return a "types.MappingProxyType" that wraps the ' - 'original\n' - ' dictionary to which the view refers.\n' - '\n' - ' New in version 3.10.\n' - '\n' - 'Keys views are set-like since their entries are unique and ' - '*hashable*.\n' - 'Items views also have set-like operations since the (key, ' - 'value) pairs\n' - 'are unique and the keys are hashable. If all values in an ' - 'items view\n' - 'are hashable as well, then the items view can interoperate ' - 'with other\n' - 'sets. (Values views are not treated as set-like since the ' - 'entries are\n' - 'generally not unique.) For set-like views, all of the ' - 'operations\n' - 'defined for the abstract base class "collections.abc.Set" ' - 'are\n' - 'available (for example, "==", "<", or "^"). While using ' - 'set\n' - 'operators, set-like views accept any iterable as the other ' - 'operand,\n' - 'unlike sets which only accept sets as the input.\n' - '\n' - 'An example of dictionary view usage:\n' - '\n' - " >>> dishes = {'eggs': 2, 'sausage': 1, 'bacon': 1, " - "'spam': 500}\n" - ' >>> keys = dishes.keys()\n' - ' >>> values = dishes.values()\n' - '\n' - ' >>> # iteration\n' - ' >>> n = 0\n' - ' >>> for val in values:\n' - ' ... n += val\n' - ' ...\n' - ' >>> print(n)\n' - ' 504\n' - '\n' - ' >>> # keys and values are iterated over in the same order ' - '(insertion order)\n' - ' >>> list(keys)\n' - " ['eggs', 'sausage', 'bacon', 'spam']\n" - ' >>> list(values)\n' - ' [2, 1, 1, 500]\n' - '\n' - ' >>> # view objects are dynamic and reflect dict changes\n' - " >>> del dishes['eggs']\n" - " >>> del dishes['sausage']\n" - ' >>> list(keys)\n' - " ['bacon', 'spam']\n" - '\n' - ' >>> # set operations\n' - " >>> keys & {'eggs', 'bacon', 'salad'}\n" - " {'bacon'}\n" - " >>> keys ^ {'sausage', 'juice'} == {'juice', 'sausage', " - "'bacon', 'spam'}\n" - ' True\n' - " >>> keys | ['juice', 'juice', 'juice'] == {'bacon', " - "'spam', 'juice'}\n" - ' True\n' - '\n' - ' >>> # get back a read-only proxy for the original ' - 'dictionary\n' - ' >>> values.mapping\n' - " mappingproxy({'bacon': 1, 'spam': 500})\n" - " >>> values.mapping['spam']\n" - ' 500\n', - 'typesmethods': 'Methods\n' - '*******\n' - '\n' - 'Methods are functions that are called using the attribute ' - 'notation.\n' - 'There are two flavors: built-in methods (such as "append()" ' - 'on lists)\n' - 'and class instance method. Built-in methods are described ' - 'with the\n' - 'types that support them.\n' - '\n' - 'If you access a method (a function defined in a class ' - 'namespace)\n' - 'through an instance, you get a special object: a *bound ' - 'method* (also\n' - 'called instance method) object. When called, it will add the ' - '"self"\n' - 'argument to the argument list. Bound methods have two ' - 'special read-\n' - 'only attributes: "m.__self__" is the object on which the ' - 'method\n' - 'operates, and "m.__func__" is the function implementing the ' - 'method.\n' - 'Calling "m(arg-1, arg-2, ..., arg-n)" is completely ' - 'equivalent to\n' - 'calling "m.__func__(m.__self__, arg-1, arg-2, ..., arg-n)".\n' - '\n' - 'Like function objects, bound method objects support getting ' - 'arbitrary\n' - 'attributes. However, since method attributes are actually ' - 'stored on\n' - 'the underlying function object ("method.__func__"), setting ' - 'method\n' - 'attributes on bound methods is disallowed. Attempting to ' - 'set an\n' - 'attribute on a method results in an "AttributeError" being ' - 'raised. In\n' - 'order to set a method attribute, you need to explicitly set ' - 'it on the\n' - 'underlying function object:\n' - '\n' - ' >>> class C:\n' - ' ... def method(self):\n' - ' ... pass\n' - ' ...\n' - ' >>> c = C()\n' - " >>> c.method.whoami = 'my name is method' # can't set on " - 'the method\n' - ' Traceback (most recent call last):\n' - ' File "", line 1, in \n' - " AttributeError: 'method' object has no attribute " - "'whoami'\n" - " >>> c.method.__func__.whoami = 'my name is method'\n" - ' >>> c.method.whoami\n' - " 'my name is method'\n" - '\n' - 'See Instance methods for more information.\n', - 'typesmodules': 'Modules\n' - '*******\n' - '\n' - 'The only special operation on a module is attribute access: ' - '"m.name",\n' - 'where *m* is a module and *name* accesses a name defined in ' - '*m*’s\n' - 'symbol table. Module attributes can be assigned to. (Note ' - 'that the\n' - '"import" statement is not, strictly speaking, an operation ' - 'on a module\n' - 'object; "import foo" does not require a module object named ' - '*foo* to\n' - 'exist, rather it requires an (external) *definition* for a ' - 'module\n' - 'named *foo* somewhere.)\n' - '\n' - 'A special attribute of every module is "__dict__". This is ' - 'the\n' - 'dictionary containing the module’s symbol table. Modifying ' - 'this\n' - 'dictionary will actually change the module’s symbol table, ' - 'but direct\n' - 'assignment to the "__dict__" attribute is not possible (you ' - 'can write\n' - '"m.__dict__[\'a\'] = 1", which defines "m.a" to be "1", but ' - 'you can’t\n' - 'write "m.__dict__ = {}"). Modifying "__dict__" directly is ' - 'not\n' - 'recommended.\n' - '\n' - 'Modules built into the interpreter are written like this: ' - '"". If loaded from a file, they are ' - 'written as\n' - '"".\n', - 'typesseq': 'Sequence Types — "list", "tuple", "range"\n' - '*****************************************\n' - '\n' - 'There are three basic sequence types: lists, tuples, and range\n' - 'objects. Additional sequence types tailored for processing of ' - 'binary\n' - 'data and text strings are described in dedicated sections.\n' - '\n' - '\n' - 'Common Sequence Operations\n' - '==========================\n' - '\n' - 'The operations in the following table are supported by most ' - 'sequence\n' - 'types, both mutable and immutable. The ' - '"collections.abc.Sequence" ABC\n' - 'is provided to make it easier to correctly implement these ' - 'operations\n' - 'on custom sequence types.\n' - '\n' - 'This table lists the sequence operations sorted in ascending ' - 'priority.\n' - 'In the table, *s* and *t* are sequences of the same type, *n*, ' - '*i*,\n' - '*j* and *k* are integers and *x* is an arbitrary object that ' - 'meets any\n' - 'type and value restrictions imposed by *s*.\n' - '\n' - 'The "in" and "not in" operations have the same priorities as ' - 'the\n' - 'comparison operations. The "+" (concatenation) and "*" ' - '(repetition)\n' - 'operations have the same priority as the corresponding numeric\n' - 'operations. [3]\n' - '\n' - '+----------------------------+----------------------------------+------------+\n' - '| Operation | Result ' - '| Notes |\n' - '|============================|==================================|============|\n' - '| "x in s" | "True" if an item of *s* is ' - '| (1) |\n' - '| | equal to *x*, else "False" ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "x not in s" | "False" if an item of *s* is ' - '| (1) |\n' - '| | equal to *x*, else "True" ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "s + t" | the concatenation of *s* and *t* ' - '| (6)(7) |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "s * n" or "n * s" | equivalent to adding *s* to ' - '| (2)(7) |\n' - '| | itself *n* times ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "s[i]" | *i*th item of *s*, origin 0 ' - '| (3) |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "s[i:j]" | slice of *s* from *i* to *j* ' - '| (3)(4) |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "s[i:j:k]" | slice of *s* from *i* to *j* ' - '| (3)(5) |\n' - '| | with step *k* ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "len(s)" | length of *s* ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "min(s)" | smallest item of *s* ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "max(s)" | largest item of *s* ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "s.index(x[, i[, j]])" | index of the first occurrence of ' - '| (8) |\n' - '| | *x* in *s* (at or after index ' - '| |\n' - '| | *i* and before index *j*) ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "s.count(x)" | total number of occurrences of ' - '| |\n' - '| | *x* in *s* ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '\n' - 'Sequences of the same type also support comparisons. In ' - 'particular,\n' - 'tuples and lists are compared lexicographically by comparing\n' - 'corresponding elements. This means that to compare equal, every\n' - 'element must compare equal and the two sequences must be of the ' - 'same\n' - 'type and have the same length. (For full details see ' - 'Comparisons in\n' - 'the language reference.)\n' - '\n' - 'Forward and reversed iterators over mutable sequences access ' - 'values\n' - 'using an index. That index will continue to march forward (or\n' - 'backward) even if the underlying sequence is mutated. The ' - 'iterator\n' - 'terminates only when an "IndexError" or a "StopIteration" is\n' - 'encountered (or when the index drops below zero).\n' - '\n' - 'Notes:\n' - '\n' - '1. While the "in" and "not in" operations are used only for ' - 'simple\n' - ' containment testing in the general case, some specialised ' - 'sequences\n' - ' (such as "str", "bytes" and "bytearray") also use them for\n' - ' subsequence testing:\n' - '\n' - ' >>> "gg" in "eggs"\n' - ' True\n' - '\n' - '2. Values of *n* less than "0" are treated as "0" (which yields ' - 'an\n' - ' empty sequence of the same type as *s*). Note that items in ' - 'the\n' - ' sequence *s* are not copied; they are referenced multiple ' - 'times.\n' - ' This often haunts new Python programmers; consider:\n' - '\n' - ' >>> lists = [[]] * 3\n' - ' >>> lists\n' - ' [[], [], []]\n' - ' >>> lists[0].append(3)\n' - ' >>> lists\n' - ' [[3], [3], [3]]\n' - '\n' - ' What has happened is that "[[]]" is a one-element list ' - 'containing\n' - ' an empty list, so all three elements of "[[]] * 3" are ' - 'references\n' - ' to this single empty list. Modifying any of the elements of\n' - ' "lists" modifies this single list. You can create a list of\n' - ' different lists this way:\n' - '\n' - ' >>> lists = [[] for i in range(3)]\n' - ' >>> lists[0].append(3)\n' - ' >>> lists[1].append(5)\n' - ' >>> lists[2].append(7)\n' - ' >>> lists\n' - ' [[3], [5], [7]]\n' - '\n' - ' Further explanation is available in the FAQ entry How do I ' - 'create a\n' - ' multidimensional list?.\n' - '\n' - '3. If *i* or *j* is negative, the index is relative to the end ' - 'of\n' - ' sequence *s*: "len(s) + i" or "len(s) + j" is substituted. ' - 'But\n' - ' note that "-0" is still "0".\n' - '\n' - '4. The slice of *s* from *i* to *j* is defined as the sequence ' - 'of\n' - ' items with index *k* such that "i <= k < j". If *i* or *j* ' - 'is\n' - ' greater than "len(s)", use "len(s)". If *i* is omitted or ' - '"None",\n' - ' use "0". If *j* is omitted or "None", use "len(s)". If *i* ' - 'is\n' - ' greater than or equal to *j*, the slice is empty.\n' - '\n' - '5. The slice of *s* from *i* to *j* with step *k* is defined as ' - 'the\n' - ' sequence of items with index "x = i + n*k" such that "0 <= n ' - '<\n' - ' (j-i)/k". In other words, the indices are "i", "i+k", ' - '"i+2*k",\n' - ' "i+3*k" and so on, stopping when *j* is reached (but never\n' - ' including *j*). When *k* is positive, *i* and *j* are ' - 'reduced to\n' - ' "len(s)" if they are greater. When *k* is negative, *i* and ' - '*j* are\n' - ' reduced to "len(s) - 1" if they are greater. If *i* or *j* ' - 'are\n' - ' omitted or "None", they become “end” values (which end ' - 'depends on\n' - ' the sign of *k*). Note, *k* cannot be zero. If *k* is ' - '"None", it\n' - ' is treated like "1".\n' - '\n' - '6. Concatenating immutable sequences always results in a new ' - 'object.\n' - ' This means that building up a sequence by repeated ' - 'concatenation\n' - ' will have a quadratic runtime cost in the total sequence ' - 'length.\n' - ' To get a linear runtime cost, you must switch to one of the\n' - ' alternatives below:\n' - '\n' - ' * if concatenating "str" objects, you can build a list and ' - 'use\n' - ' "str.join()" at the end or else write to an "io.StringIO"\n' - ' instance and retrieve its value when complete\n' - '\n' - ' * if concatenating "bytes" objects, you can similarly use\n' - ' "bytes.join()" or "io.BytesIO", or you can do in-place\n' - ' concatenation with a "bytearray" object. "bytearray" ' - 'objects are\n' - ' mutable and have an efficient overallocation mechanism\n' - '\n' - ' * if concatenating "tuple" objects, extend a "list" instead\n' - '\n' - ' * for other types, investigate the relevant class ' - 'documentation\n' - '\n' - '7. Some sequence types (such as "range") only support item ' - 'sequences\n' - ' that follow specific patterns, and hence don’t support ' - 'sequence\n' - ' concatenation or repetition.\n' - '\n' - '8. "index" raises "ValueError" when *x* is not found in *s*. Not ' - 'all\n' - ' implementations support passing the additional arguments *i* ' - 'and\n' - ' *j*. These arguments allow efficient searching of subsections ' - 'of\n' - ' the sequence. Passing the extra arguments is roughly ' - 'equivalent to\n' - ' using "s[i:j].index(x)", only without copying any data and ' - 'with the\n' - ' returned index being relative to the start of the sequence ' - 'rather\n' - ' than the start of the slice.\n' - '\n' - '\n' - 'Immutable Sequence Types\n' - '========================\n' - '\n' - 'The only operation that immutable sequence types generally ' - 'implement\n' - 'that is not also implemented by mutable sequence types is ' - 'support for\n' - 'the "hash()" built-in.\n' - '\n' - 'This support allows immutable sequences, such as "tuple" ' - 'instances, to\n' - 'be used as "dict" keys and stored in "set" and "frozenset" ' - 'instances.\n' - '\n' - 'Attempting to hash an immutable sequence that contains ' - 'unhashable\n' - 'values will result in "TypeError".\n' - '\n' - '\n' - 'Mutable Sequence Types\n' - '======================\n' - '\n' - 'The operations in the following table are defined on mutable ' - 'sequence\n' - 'types. The "collections.abc.MutableSequence" ABC is provided to ' - 'make\n' - 'it easier to correctly implement these operations on custom ' - 'sequence\n' - 'types.\n' - '\n' - 'In the table *s* is an instance of a mutable sequence type, *t* ' - 'is any\n' - 'iterable object and *x* is an arbitrary object that meets any ' - 'type and\n' - 'value restrictions imposed by *s* (for example, "bytearray" ' - 'only\n' - 'accepts integers that meet the value restriction "0 <= x <= ' - '255").\n' - '\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| Operation | ' - 'Result | Notes |\n' - '|================================|==================================|=======================|\n' - '| "s[i] = x" | item *i* of *s* is replaced ' - 'by | |\n' - '| | ' - '*x* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s[i:j] = t" | slice of *s* from *i* to *j* ' - 'is | |\n' - '| | replaced by the contents of ' - 'the | |\n' - '| | iterable ' - '*t* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "del s[i:j]" | same as "s[i:j] = ' - '[]" | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s[i:j:k] = t" | the elements of "s[i:j:k]" ' - 'are | (1) |\n' - '| | replaced by those of ' - '*t* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "del s[i:j:k]" | removes the elements ' - 'of | |\n' - '| | "s[i:j:k]" from the ' - 'list | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.append(x)" | appends *x* to the end of ' - 'the | |\n' - '| | sequence (same ' - 'as | |\n' - '| | "s[len(s):len(s)] = ' - '[x]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.clear()" | removes all items from *s* ' - '(same | (5) |\n' - '| | as "del ' - 's[:]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.copy()" | creates a shallow copy of ' - '*s* | (5) |\n' - '| | (same as ' - '"s[:]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.extend(t)" or "s += t" | extends *s* with the contents ' - 'of | |\n' - '| | *t* (for the most part the ' - 'same | |\n' - '| | as "s[len(s):len(s)] = ' - 't") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s *= n" | updates *s* with its ' - 'contents | (6) |\n' - '| | repeated *n* ' - 'times | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.insert(i, x)" | inserts *x* into *s* at ' - 'the | |\n' - '| | index given by *i* (same ' - 'as | |\n' - '| | "s[i:i] = ' - '[x]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.pop()" or "s.pop(i)" | retrieves the item at *i* ' - 'and | (2) |\n' - '| | also removes it from ' - '*s* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.remove(x)" | remove the first item from ' - '*s* | (3) |\n' - '| | where "s[i]" is equal to ' - '*x* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.reverse()" | reverses the items of *s* ' - 'in | (4) |\n' - '| | ' - 'place | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '\n' - 'Notes:\n' - '\n' - '1. *t* must have the same length as the slice it is replacing.\n' - '\n' - '2. The optional argument *i* defaults to "-1", so that by ' - 'default the\n' - ' last item is removed and returned.\n' - '\n' - '3. "remove()" raises "ValueError" when *x* is not found in *s*.\n' - '\n' - '4. The "reverse()" method modifies the sequence in place for ' - 'economy\n' - ' of space when reversing a large sequence. To remind users ' - 'that it\n' - ' operates by side effect, it does not return the reversed ' - 'sequence.\n' - '\n' - '5. "clear()" and "copy()" are included for consistency with the\n' - ' interfaces of mutable containers that don’t support slicing\n' - ' operations (such as "dict" and "set"). "copy()" is not part ' - 'of the\n' - ' "collections.abc.MutableSequence" ABC, but most concrete ' - 'mutable\n' - ' sequence classes provide it.\n' - '\n' - ' New in version 3.3: "clear()" and "copy()" methods.\n' - '\n' - '6. The value *n* is an integer, or an object implementing\n' - ' "__index__()". Zero and negative values of *n* clear the ' - 'sequence.\n' - ' Items in the sequence are not copied; they are referenced ' - 'multiple\n' - ' times, as explained for "s * n" under Common Sequence ' - 'Operations.\n' - '\n' - '\n' - 'Lists\n' - '=====\n' - '\n' - 'Lists are mutable sequences, typically used to store collections ' - 'of\n' - 'homogeneous items (where the precise degree of similarity will ' - 'vary by\n' - 'application).\n' - '\n' - 'class list([iterable])\n' - '\n' - ' Lists may be constructed in several ways:\n' - '\n' - ' * Using a pair of square brackets to denote the empty list: ' - '"[]"\n' - '\n' - ' * Using square brackets, separating items with commas: "[a]", ' - '"[a,\n' - ' b, c]"\n' - '\n' - ' * Using a list comprehension: "[x for x in iterable]"\n' - '\n' - ' * Using the type constructor: "list()" or "list(iterable)"\n' - '\n' - ' The constructor builds a list whose items are the same and in ' - 'the\n' - ' same order as *iterable*’s items. *iterable* may be either ' - 'a\n' - ' sequence, a container that supports iteration, or an ' - 'iterator\n' - ' object. If *iterable* is already a list, a copy is made and\n' - ' returned, similar to "iterable[:]". For example, ' - '"list(\'abc\')"\n' - ' returns "[\'a\', \'b\', \'c\']" and "list( (1, 2, 3) )" ' - 'returns "[1, 2,\n' - ' 3]". If no argument is given, the constructor creates a new ' - 'empty\n' - ' list, "[]".\n' - '\n' - ' Many other operations also produce lists, including the ' - '"sorted()"\n' - ' built-in.\n' - '\n' - ' Lists implement all of the common and mutable sequence ' - 'operations.\n' - ' Lists also provide the following additional method:\n' - '\n' - ' sort(*, key=None, reverse=False)\n' - '\n' - ' This method sorts the list in place, using only "<" ' - 'comparisons\n' - ' between items. Exceptions are not suppressed - if any ' - 'comparison\n' - ' operations fail, the entire sort operation will fail (and ' - 'the\n' - ' list will likely be left in a partially modified state).\n' - '\n' - ' "sort()" accepts two arguments that can only be passed by\n' - ' keyword (keyword-only arguments):\n' - '\n' - ' *key* specifies a function of one argument that is used ' - 'to\n' - ' extract a comparison key from each list element (for ' - 'example,\n' - ' "key=str.lower"). The key corresponding to each item in ' - 'the list\n' - ' is calculated once and then used for the entire sorting ' - 'process.\n' - ' The default value of "None" means that list items are ' - 'sorted\n' - ' directly without calculating a separate key value.\n' - '\n' - ' The "functools.cmp_to_key()" utility is available to ' - 'convert a\n' - ' 2.x style *cmp* function to a *key* function.\n' - '\n' - ' *reverse* is a boolean value. If set to "True", then the ' - 'list\n' - ' elements are sorted as if each comparison were reversed.\n' - '\n' - ' This method modifies the sequence in place for economy of ' - 'space\n' - ' when sorting a large sequence. To remind users that it ' - 'operates\n' - ' by side effect, it does not return the sorted sequence ' - '(use\n' - ' "sorted()" to explicitly request a new sorted list ' - 'instance).\n' - '\n' - ' The "sort()" method is guaranteed to be stable. A sort ' - 'is\n' - ' stable if it guarantees not to change the relative order ' - 'of\n' - ' elements that compare equal — this is helpful for sorting ' - 'in\n' - ' multiple passes (for example, sort by department, then by ' - 'salary\n' - ' grade).\n' - '\n' - ' For sorting examples and a brief sorting tutorial, see ' - 'Sorting\n' - ' HOW TO.\n' - '\n' - ' **CPython implementation detail:** While a list is being ' - 'sorted,\n' - ' the effect of attempting to mutate, or even inspect, the ' - 'list is\n' - ' undefined. The C implementation of Python makes the list ' - 'appear\n' - ' empty for the duration, and raises "ValueError" if it can ' - 'detect\n' - ' that the list has been mutated during a sort.\n' - '\n' - '\n' - 'Tuples\n' - '======\n' - '\n' - 'Tuples are immutable sequences, typically used to store ' - 'collections of\n' - 'heterogeneous data (such as the 2-tuples produced by the ' - '"enumerate()"\n' - 'built-in). Tuples are also used for cases where an immutable ' - 'sequence\n' - 'of homogeneous data is needed (such as allowing storage in a ' - '"set" or\n' - '"dict" instance).\n' - '\n' - 'class tuple([iterable])\n' - '\n' - ' Tuples may be constructed in a number of ways:\n' - '\n' - ' * Using a pair of parentheses to denote the empty tuple: ' - '"()"\n' - '\n' - ' * Using a trailing comma for a singleton tuple: "a," or ' - '"(a,)"\n' - '\n' - ' * Separating items with commas: "a, b, c" or "(a, b, c)"\n' - '\n' - ' * Using the "tuple()" built-in: "tuple()" or ' - '"tuple(iterable)"\n' - '\n' - ' The constructor builds a tuple whose items are the same and ' - 'in the\n' - ' same order as *iterable*’s items. *iterable* may be either ' - 'a\n' - ' sequence, a container that supports iteration, or an ' - 'iterator\n' - ' object. If *iterable* is already a tuple, it is returned\n' - ' unchanged. For example, "tuple(\'abc\')" returns "(\'a\', ' - '\'b\', \'c\')"\n' - ' and "tuple( [1, 2, 3] )" returns "(1, 2, 3)". If no argument ' - 'is\n' - ' given, the constructor creates a new empty tuple, "()".\n' - '\n' - ' Note that it is actually the comma which makes a tuple, not ' - 'the\n' - ' parentheses. The parentheses are optional, except in the ' - 'empty\n' - ' tuple case, or when they are needed to avoid syntactic ' - 'ambiguity.\n' - ' For example, "f(a, b, c)" is a function call with three ' - 'arguments,\n' - ' while "f((a, b, c))" is a function call with a 3-tuple as the ' - 'sole\n' - ' argument.\n' - '\n' - ' Tuples implement all of the common sequence operations.\n' - '\n' - 'For heterogeneous collections of data where access by name is ' - 'clearer\n' - 'than access by index, "collections.namedtuple()" may be a more\n' - 'appropriate choice than a simple tuple object.\n' - '\n' - '\n' - 'Ranges\n' - '======\n' - '\n' - 'The "range" type represents an immutable sequence of numbers and ' - 'is\n' - 'commonly used for looping a specific number of times in "for" ' - 'loops.\n' - '\n' - 'class range(stop)\n' - 'class range(start, stop[, step])\n' - '\n' - ' The arguments to the range constructor must be integers ' - '(either\n' - ' built-in "int" or any object that implements the ' - '"__index__()"\n' - ' special method). If the *step* argument is omitted, it ' - 'defaults to\n' - ' "1". If the *start* argument is omitted, it defaults to "0". ' - 'If\n' - ' *step* is zero, "ValueError" is raised.\n' - '\n' - ' For a positive *step*, the contents of a range "r" are ' - 'determined\n' - ' by the formula "r[i] = start + step*i" where "i >= 0" and ' - '"r[i] <\n' - ' stop".\n' - '\n' - ' For a negative *step*, the contents of the range are still\n' - ' determined by the formula "r[i] = start + step*i", but the\n' - ' constraints are "i >= 0" and "r[i] > stop".\n' - '\n' - ' A range object will be empty if "r[0]" does not meet the ' - 'value\n' - ' constraint. Ranges do support negative indices, but these ' - 'are\n' - ' interpreted as indexing from the end of the sequence ' - 'determined by\n' - ' the positive indices.\n' - '\n' - ' Ranges containing absolute values larger than "sys.maxsize" ' - 'are\n' - ' permitted but some features (such as "len()") may raise\n' - ' "OverflowError".\n' - '\n' - ' Range examples:\n' - '\n' - ' >>> list(range(10))\n' - ' [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n' - ' >>> list(range(1, 11))\n' - ' [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n' - ' >>> list(range(0, 30, 5))\n' - ' [0, 5, 10, 15, 20, 25]\n' - ' >>> list(range(0, 10, 3))\n' - ' [0, 3, 6, 9]\n' - ' >>> list(range(0, -10, -1))\n' - ' [0, -1, -2, -3, -4, -5, -6, -7, -8, -9]\n' - ' >>> list(range(0))\n' - ' []\n' - ' >>> list(range(1, 0))\n' - ' []\n' - '\n' - ' Ranges implement all of the common sequence operations ' - 'except\n' - ' concatenation and repetition (due to the fact that range ' - 'objects\n' - ' can only represent sequences that follow a strict pattern ' - 'and\n' - ' repetition and concatenation will usually violate that ' - 'pattern).\n' - '\n' - ' start\n' - '\n' - ' The value of the *start* parameter (or "0" if the ' - 'parameter was\n' - ' not supplied)\n' - '\n' - ' stop\n' - '\n' - ' The value of the *stop* parameter\n' - '\n' - ' step\n' - '\n' - ' The value of the *step* parameter (or "1" if the parameter ' - 'was\n' - ' not supplied)\n' - '\n' - 'The advantage of the "range" type over a regular "list" or ' - '"tuple" is\n' - 'that a "range" object will always take the same (small) amount ' - 'of\n' - 'memory, no matter the size of the range it represents (as it ' - 'only\n' - 'stores the "start", "stop" and "step" values, calculating ' - 'individual\n' - 'items and subranges as needed).\n' - '\n' - 'Range objects implement the "collections.abc.Sequence" ABC, and\n' - 'provide features such as containment tests, element index ' - 'lookup,\n' - 'slicing and support for negative indices (see Sequence Types — ' - 'list,\n' - 'tuple, range):\n' - '\n' - '>>> r = range(0, 20, 2)\n' - '>>> r\n' - 'range(0, 20, 2)\n' - '>>> 11 in r\n' - 'False\n' - '>>> 10 in r\n' - 'True\n' - '>>> r.index(10)\n' - '5\n' - '>>> r[5]\n' - '10\n' - '>>> r[:5]\n' - 'range(0, 10, 2)\n' - '>>> r[-1]\n' - '18\n' - '\n' - 'Testing range objects for equality with "==" and "!=" compares ' - 'them as\n' - 'sequences. That is, two range objects are considered equal if ' - 'they\n' - 'represent the same sequence of values. (Note that two range ' - 'objects\n' - 'that compare equal might have different "start", "stop" and ' - '"step"\n' - 'attributes, for example "range(0) == range(2, 1, 3)" or ' - '"range(0, 3,\n' - '2) == range(0, 4, 2)".)\n' - '\n' - 'Changed in version 3.2: Implement the Sequence ABC. Support ' - 'slicing\n' - 'and negative indices. Test "int" objects for membership in ' - 'constant\n' - 'time instead of iterating through all items.\n' - '\n' - 'Changed in version 3.3: Define ‘==’ and ‘!=’ to compare range ' - 'objects\n' - 'based on the sequence of values they define (instead of ' - 'comparing\n' - 'based on object identity).\n' - '\n' - 'New in version 3.3: The "start", "stop" and "step" attributes.\n' - '\n' - 'See also:\n' - '\n' - ' * The linspace recipe shows how to implement a lazy version of ' - 'range\n' - ' suitable for floating point applications.\n', - 'typesseq-mutable': 'Mutable Sequence Types\n' - '**********************\n' - '\n' - 'The operations in the following table are defined on ' - 'mutable sequence\n' - 'types. The "collections.abc.MutableSequence" ABC is ' - 'provided to make\n' - 'it easier to correctly implement these operations on ' - 'custom sequence\n' - 'types.\n' - '\n' - 'In the table *s* is an instance of a mutable sequence ' - 'type, *t* is any\n' - 'iterable object and *x* is an arbitrary object that ' - 'meets any type and\n' - 'value restrictions imposed by *s* (for example, ' - '"bytearray" only\n' - 'accepts integers that meet the value restriction "0 <= x ' - '<= 255").\n' - '\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| Operation | ' - 'Result | Notes ' - '|\n' - '|================================|==================================|=======================|\n' - '| "s[i] = x" | item *i* of *s* is ' - 'replaced by | |\n' - '| | ' - '*x* | ' - '|\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s[i:j] = t" | slice of *s* from *i* ' - 'to *j* is | |\n' - '| | replaced by the ' - 'contents of the | |\n' - '| | iterable ' - '*t* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "del s[i:j]" | same as "s[i:j] = ' - '[]" | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s[i:j:k] = t" | the elements of ' - '"s[i:j:k]" are | (1) |\n' - '| | replaced by those of ' - '*t* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "del s[i:j:k]" | removes the elements ' - 'of | |\n' - '| | "s[i:j:k]" from the ' - 'list | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.append(x)" | appends *x* to the ' - 'end of the | |\n' - '| | sequence (same ' - 'as | |\n' - '| | "s[len(s):len(s)] = ' - '[x]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.clear()" | removes all items ' - 'from *s* (same | (5) |\n' - '| | as "del ' - 's[:]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.copy()" | creates a shallow ' - 'copy of *s* | (5) |\n' - '| | (same as ' - '"s[:]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.extend(t)" or "s += t" | extends *s* with the ' - 'contents of | |\n' - '| | *t* (for the most ' - 'part the same | |\n' - '| | as "s[len(s):len(s)] ' - '= t") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s *= n" | updates *s* with its ' - 'contents | (6) |\n' - '| | repeated *n* ' - 'times | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.insert(i, x)" | inserts *x* into *s* ' - 'at the | |\n' - '| | index given by *i* ' - '(same as | |\n' - '| | "s[i:i] = ' - '[x]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.pop()" or "s.pop(i)" | retrieves the item at ' - '*i* and | (2) |\n' - '| | also removes it from ' - '*s* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.remove(x)" | remove the first item ' - 'from *s* | (3) |\n' - '| | where "s[i]" is equal ' - 'to *x* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.reverse()" | reverses the items of ' - '*s* in | (4) |\n' - '| | ' - 'place | ' - '|\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '\n' - 'Notes:\n' - '\n' - '1. *t* must have the same length as the slice it is ' - 'replacing.\n' - '\n' - '2. The optional argument *i* defaults to "-1", so that ' - 'by default the\n' - ' last item is removed and returned.\n' - '\n' - '3. "remove()" raises "ValueError" when *x* is not found ' - 'in *s*.\n' - '\n' - '4. The "reverse()" method modifies the sequence in place ' - 'for economy\n' - ' of space when reversing a large sequence. To remind ' - 'users that it\n' - ' operates by side effect, it does not return the ' - 'reversed sequence.\n' - '\n' - '5. "clear()" and "copy()" are included for consistency ' - 'with the\n' - ' interfaces of mutable containers that don’t support ' - 'slicing\n' - ' operations (such as "dict" and "set"). "copy()" is ' - 'not part of the\n' - ' "collections.abc.MutableSequence" ABC, but most ' - 'concrete mutable\n' - ' sequence classes provide it.\n' - '\n' - ' New in version 3.3: "clear()" and "copy()" methods.\n' - '\n' - '6. The value *n* is an integer, or an object ' - 'implementing\n' - ' "__index__()". Zero and negative values of *n* clear ' - 'the sequence.\n' - ' Items in the sequence are not copied; they are ' - 'referenced multiple\n' - ' times, as explained for "s * n" under Common Sequence ' - 'Operations.\n', - 'unary': 'Unary arithmetic and bitwise operations\n' - '***************************************\n' - '\n' - 'All unary arithmetic and bitwise operations have the same ' - 'priority:\n' - '\n' - ' u_expr ::= power | "-" u_expr | "+" u_expr | "~" u_expr\n' - '\n' - 'The unary "-" (minus) operator yields the negation of its numeric\n' - 'argument; the operation can be overridden with the "__neg__()" ' - 'special\n' - 'method.\n' - '\n' - 'The unary "+" (plus) operator yields its numeric argument ' - 'unchanged;\n' - 'the operation can be overridden with the "__pos__()" special ' - 'method.\n' - '\n' - 'The unary "~" (invert) operator yields the bitwise inversion of ' - 'its\n' - 'integer argument. The bitwise inversion of "x" is defined as\n' - '"-(x+1)". It only applies to integral numbers or to custom ' - 'objects\n' - 'that override the "__invert__()" special method.\n' - '\n' - 'In all three cases, if the argument does not have the proper type, ' - 'a\n' - '"TypeError" exception is raised.\n', - 'while': 'The "while" statement\n' - '*********************\n' - '\n' - 'The "while" statement is used for repeated execution as long as an\n' - 'expression is true:\n' - '\n' - ' while_stmt ::= "while" assignment_expression ":" suite\n' - ' ["else" ":" suite]\n' - '\n' - 'This repeatedly tests the expression and, if it is true, executes ' - 'the\n' - 'first suite; if the expression is false (which may be the first ' - 'time\n' - 'it is tested) the suite of the "else" clause, if present, is ' - 'executed\n' - 'and the loop terminates.\n' - '\n' - 'A "break" statement executed in the first suite terminates the ' - 'loop\n' - 'without executing the "else" clause’s suite. A "continue" ' - 'statement\n' - 'executed in the first suite skips the rest of the suite and goes ' - 'back\n' - 'to testing the expression.\n', - 'with': 'The "with" statement\n' - '********************\n' - '\n' - 'The "with" statement is used to wrap the execution of a block with\n' - 'methods defined by a context manager (see section With Statement\n' - 'Context Managers). This allows common "try"…"except"…"finally" ' - 'usage\n' - 'patterns to be encapsulated for convenient reuse.\n' - '\n' - ' with_stmt ::= "with" ( "(" with_stmt_contents ","? ")" | ' - 'with_stmt_contents ) ":" suite\n' - ' with_stmt_contents ::= with_item ("," with_item)*\n' - ' with_item ::= expression ["as" target]\n' - '\n' - 'The execution of the "with" statement with one “item” proceeds as\n' - 'follows:\n' - '\n' - '1. The context expression (the expression given in the "with_item") ' - 'is\n' - ' evaluated to obtain a context manager.\n' - '\n' - '2. The context manager’s "__enter__()" is loaded for later use.\n' - '\n' - '3. The context manager’s "__exit__()" is loaded for later use.\n' - '\n' - '4. The context manager’s "__enter__()" method is invoked.\n' - '\n' - '5. If a target was included in the "with" statement, the return ' - 'value\n' - ' from "__enter__()" is assigned to it.\n' - '\n' - ' Note:\n' - '\n' - ' The "with" statement guarantees that if the "__enter__()" ' - 'method\n' - ' returns without an error, then "__exit__()" will always be\n' - ' called. Thus, if an error occurs during the assignment to the\n' - ' target list, it will be treated the same as an error occurring\n' - ' within the suite would be. See step 7 below.\n' - '\n' - '6. The suite is executed.\n' - '\n' - '7. The context manager’s "__exit__()" method is invoked. If an\n' - ' exception caused the suite to be exited, its type, value, and\n' - ' traceback are passed as arguments to "__exit__()". Otherwise, ' - 'three\n' - ' "None" arguments are supplied.\n' - '\n' - ' If the suite was exited due to an exception, and the return ' - 'value\n' - ' from the "__exit__()" method was false, the exception is ' - 'reraised.\n' - ' If the return value was true, the exception is suppressed, and\n' - ' execution continues with the statement following the "with"\n' - ' statement.\n' - '\n' - ' If the suite was exited for any reason other than an exception, ' - 'the\n' - ' return value from "__exit__()" is ignored, and execution ' - 'proceeds\n' - ' at the normal location for the kind of exit that was taken.\n' - '\n' - 'The following code:\n' - '\n' - ' with EXPRESSION as TARGET:\n' - ' SUITE\n' - '\n' - 'is semantically equivalent to:\n' - '\n' - ' manager = (EXPRESSION)\n' - ' enter = type(manager).__enter__\n' - ' exit = type(manager).__exit__\n' - ' value = enter(manager)\n' - ' hit_except = False\n' - '\n' - ' try:\n' - ' TARGET = value\n' - ' SUITE\n' - ' except:\n' - ' hit_except = True\n' - ' if not exit(manager, *sys.exc_info()):\n' - ' raise\n' - ' finally:\n' - ' if not hit_except:\n' - ' exit(manager, None, None, None)\n' - '\n' - 'With more than one item, the context managers are processed as if\n' - 'multiple "with" statements were nested:\n' - '\n' - ' with A() as a, B() as b:\n' - ' SUITE\n' - '\n' - 'is semantically equivalent to:\n' - '\n' - ' with A() as a:\n' - ' with B() as b:\n' - ' SUITE\n' - '\n' - 'You can also write multi-item context managers in multiple lines if\n' - 'the items are surrounded by parentheses. For example:\n' - '\n' - ' with (\n' - ' A() as a,\n' - ' B() as b,\n' - ' ):\n' - ' SUITE\n' - '\n' - 'Changed in version 3.1: Support for multiple context expressions.\n' - '\n' - 'Changed in version 3.10: Support for using grouping parentheses to\n' - 'break the statement in multiple lines.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 343** - The “with” statement\n' - ' The specification, background, and examples for the Python ' - '"with"\n' - ' statement.\n', - 'yield': 'The "yield" statement\n' - '*********************\n' - '\n' - ' yield_stmt ::= yield_expression\n' - '\n' - 'A "yield" statement is semantically equivalent to a yield ' - 'expression.\n' - 'The yield statement can be used to omit the parentheses that would\n' - 'otherwise be required in the equivalent yield expression ' - 'statement.\n' - 'For example, the yield statements\n' - '\n' - ' yield \n' - ' yield from \n' - '\n' - 'are equivalent to the yield expression statements\n' - '\n' - ' (yield )\n' - ' (yield from )\n' - '\n' - 'Yield expressions and statements are only used when defining a\n' - '*generator* function, and are only used in the body of the ' - 'generator\n' - 'function. Using yield in a function definition is sufficient to ' - 'cause\n' - 'that definition to create a generator function instead of a normal\n' - 'function.\n' - '\n' - 'For full details of "yield" semantics, refer to the Yield ' - 'expressions\n' - 'section.\n'} + +topics = { + 'assert': r'''The "assert" statement +********************** + +Assert statements are a convenient way to insert debugging assertions +into a program: + + assert_stmt ::= "assert" expression ["," expression] + +The simple form, "assert expression", is equivalent to + + if __debug__: + if not expression: raise AssertionError + +The extended form, "assert expression1, expression2", is equivalent to + + if __debug__: + if not expression1: raise AssertionError(expression2) + +These equivalences assume that "__debug__" and "AssertionError" refer +to the built-in variables with those names. In the current +implementation, the built-in variable "__debug__" is "True" under +normal circumstances, "False" when optimization is requested (command +line option "-O"). The current code generator emits no code for an +"assert" statement when optimization is requested at compile time. +Note that it is unnecessary to include the source code for the +expression that failed in the error message; it will be displayed as +part of the stack trace. + +Assignments to "__debug__" are illegal. The value for the built-in +variable is determined when the interpreter starts. +''', + 'assignment': r'''Assignment statements +********************* + +Assignment statements are used to (re)bind names to values and to +modify attributes or items of mutable objects: + + assignment_stmt ::= (target_list "=")+ (starred_expression | yield_expression) + target_list ::= target ("," target)* [","] + target ::= identifier + | "(" [target_list] ")" + | "[" [target_list] "]" + | attributeref + | subscription + | slicing + | "*" target + +(See section Primaries for the syntax definitions for *attributeref*, +*subscription*, and *slicing*.) + +An assignment statement evaluates the expression list (remember that +this can be a single expression or a comma-separated list, the latter +yielding a tuple) and assigns the single resulting object to each of +the target lists, from left to right. + +Assignment is defined recursively depending on the form of the target +(list). When a target is part of a mutable object (an attribute +reference, subscription or slicing), the mutable object must +ultimately perform the assignment and decide about its validity, and +may raise an exception if the assignment is unacceptable. The rules +observed by various types and the exceptions raised are given with the +definition of the object types (see section The standard type +hierarchy). + +Assignment of an object to a target list, optionally enclosed in +parentheses or square brackets, is recursively defined as follows. + +* If the target list is a single target with no trailing comma, + optionally in parentheses, the object is assigned to that target. + +* Else: + + * If the target list contains one target prefixed with an asterisk, + called a “starred” target: The object must be an iterable with at + least as many items as there are targets in the target list, minus + one. The first items of the iterable are assigned, from left to + right, to the targets before the starred target. The final items + of the iterable are assigned to the targets after the starred + target. A list of the remaining items in the iterable is then + assigned to the starred target (the list can be empty). + + * Else: The object must be an iterable with the same number of items + as there are targets in the target list, and the items are + assigned, from left to right, to the corresponding targets. + +Assignment of an object to a single target is recursively defined as +follows. + +* If the target is an identifier (name): + + * If the name does not occur in a "global" or "nonlocal" statement + in the current code block: the name is bound to the object in the + current local namespace. + + * Otherwise: the name is bound to the object in the global namespace + or the outer namespace determined by "nonlocal", respectively. + + The name is rebound if it was already bound. This may cause the + reference count for the object previously bound to the name to reach + zero, causing the object to be deallocated and its destructor (if it + has one) to be called. + +* If the target is an attribute reference: The primary expression in + the reference is evaluated. It should yield an object with + assignable attributes; if this is not the case, "TypeError" is + raised. That object is then asked to assign the assigned object to + the given attribute; if it cannot perform the assignment, it raises + an exception (usually but not necessarily "AttributeError"). + + Note: If the object is a class instance and the attribute reference + occurs on both sides of the assignment operator, the right-hand side + expression, "a.x" can access either an instance attribute or (if no + instance attribute exists) a class attribute. The left-hand side + target "a.x" is always set as an instance attribute, creating it if + necessary. Thus, the two occurrences of "a.x" do not necessarily + refer to the same attribute: if the right-hand side expression + refers to a class attribute, the left-hand side creates a new + instance attribute as the target of the assignment: + + class Cls: + x = 3 # class variable + inst = Cls() + inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x as 3 + + This description does not necessarily apply to descriptor + attributes, such as properties created with "property()". + +* If the target is a subscription: The primary expression in the + reference is evaluated. It should yield either a mutable sequence + object (such as a list) or a mapping object (such as a dictionary). + Next, the subscript expression is evaluated. + + If the primary is a mutable sequence object (such as a list), the + subscript must yield an integer. If it is negative, the sequence’s + length is added to it. The resulting value must be a nonnegative + integer less than the sequence’s length, and the sequence is asked + to assign the assigned object to its item with that index. If the + index is out of range, "IndexError" is raised (assignment to a + subscripted sequence cannot add new items to a list). + + If the primary is a mapping object (such as a dictionary), the + subscript must have a type compatible with the mapping’s key type, + and the mapping is then asked to create a key/value pair which maps + the subscript to the assigned object. This can either replace an + existing key/value pair with the same key value, or insert a new + key/value pair (if no key with the same value existed). + + For user-defined objects, the "__setitem__()" method is called with + appropriate arguments. + +* If the target is a slicing: The primary expression in the reference + is evaluated. It should yield a mutable sequence object (such as a + list). The assigned object should be a sequence object of the same + type. Next, the lower and upper bound expressions are evaluated, + insofar they are present; defaults are zero and the sequence’s + length. The bounds should evaluate to integers. If either bound is + negative, the sequence’s length is added to it. The resulting + bounds are clipped to lie between zero and the sequence’s length, + inclusive. Finally, the sequence object is asked to replace the + slice with the items of the assigned sequence. The length of the + slice may be different from the length of the assigned sequence, + thus changing the length of the target sequence, if the target + sequence allows it. + +**CPython implementation detail:** In the current implementation, the +syntax for targets is taken to be the same as for expressions, and +invalid syntax is rejected during the code generation phase, causing +less detailed error messages. + +Although the definition of assignment implies that overlaps between +the left-hand side and the right-hand side are ‘simultaneous’ (for +example "a, b = b, a" swaps two variables), overlaps *within* the +collection of assigned-to variables occur left-to-right, sometimes +resulting in confusion. For instance, the following program prints +"[0, 2]": + + x = [0, 1] + i = 0 + i, x[i] = 1, 2 # i is updated, then x[i] is updated + print(x) + +See also: + + **PEP 3132** - Extended Iterable Unpacking + The specification for the "*target" feature. + + +Augmented assignment statements +=============================== + +Augmented assignment is the combination, in a single statement, of a +binary operation and an assignment statement: + + augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression) + augtarget ::= identifier | attributeref | subscription | slicing + augop ::= "+=" | "-=" | "*=" | "@=" | "/=" | "//=" | "%=" | "**=" + | ">>=" | "<<=" | "&=" | "^=" | "|=" + +(See section Primaries for the syntax definitions of the last three +symbols.) + +An augmented assignment evaluates the target (which, unlike normal +assignment statements, cannot be an unpacking) and the expression +list, performs the binary operation specific to the type of assignment +on the two operands, and assigns the result to the original target. +The target is only evaluated once. + +An augmented assignment statement like "x += 1" can be rewritten as "x += x + 1" to achieve a similar, but not exactly equal effect. In the +augmented version, "x" is only evaluated once. Also, when possible, +the actual operation is performed *in-place*, meaning that rather than +creating a new object and assigning that to the target, the old object +is modified instead. + +Unlike normal assignments, augmented assignments evaluate the left- +hand side *before* evaluating the right-hand side. For example, "a[i] ++= f(x)" first looks-up "a[i]", then it evaluates "f(x)" and performs +the addition, and lastly, it writes the result back to "a[i]". + +With the exception of assigning to tuples and multiple targets in a +single statement, the assignment done by augmented assignment +statements is handled the same way as normal assignments. Similarly, +with the exception of the possible *in-place* behavior, the binary +operation performed by augmented assignment is the same as the normal +binary operations. + +For targets which are attribute references, the same caveat about +class and instance attributes applies as for regular assignments. + + +Annotated assignment statements +=============================== + +*Annotation* assignment is the combination, in a single statement, of +a variable or attribute annotation and an optional assignment +statement: + + annotated_assignment_stmt ::= augtarget ":" expression + ["=" (starred_expression | yield_expression)] + +The difference from normal Assignment statements is that only a single +target is allowed. + +The assignment target is considered “simple” if it consists of a +single name that is not enclosed in parentheses. For simple assignment +targets, if in class or module scope, the annotations are gathered in +a lazily evaluated annotation scope. The annotations can be evaluated +using the "__annotations__" attribute of a class or module, or using +the facilities in the "annotationlib" module. + +If the assignment target is not simple (an attribute, subscript node, +or parenthesized name), the annotation is never evaluated. + +If a name is annotated in a function scope, then this name is local +for that scope. Annotations are never evaluated and stored in function +scopes. + +If the right hand side is present, an annotated assignment performs +the actual assignment as if there was no annotation present. If the +right hand side is not present for an expression target, then the +interpreter evaluates the target except for the last "__setitem__()" +or "__setattr__()" call. + +See also: + + **PEP 526** - Syntax for Variable Annotations + The proposal that added syntax for annotating the types of + variables (including class variables and instance variables), + instead of expressing them through comments. + + **PEP 484** - Type hints + The proposal that added the "typing" module to provide a standard + syntax for type annotations that can be used in static analysis + tools and IDEs. + +Changed in version 3.8: Now annotated assignments allow the same +expressions in the right hand side as regular assignments. Previously, +some expressions (like un-parenthesized tuple expressions) caused a +syntax error. + +Changed in version 3.14: Annotations are now lazily evaluated in a +separate annotation scope. If the assignment target is not simple, +annotations are never evaluated. +''', + 'assignment-expressions': r'''Assignment expressions +********************** + + assignment_expression ::= [identifier ":="] expression + +An assignment expression (sometimes also called a “named expression” +or “walrus”) assigns an "expression" to an "identifier", while also +returning the value of the "expression". + +One common use case is when handling matched regular expressions: + + if matching := pattern.search(data): + do_something(matching) + +Or, when processing a file stream in chunks: + + while chunk := file.read(9000): + process(chunk) + +Assignment expressions must be surrounded by parentheses when used as +expression statements and when used as sub-expressions in slicing, +conditional, lambda, keyword-argument, and comprehension-if +expressions and in "assert", "with", and "assignment" statements. In +all other places where they can be used, parentheses are not required, +including in "if" and "while" statements. + +Added in version 3.8: See **PEP 572** for more details about +assignment expressions. +''', + 'async': r'''Coroutines +********** + +Added in version 3.5. + + +Coroutine function definition +============================= + + async_funcdef ::= [decorators] "async" "def" funcname "(" [parameter_list] ")" + ["->" expression] ":" suite + +Execution of Python coroutines can be suspended and resumed at many +points (see *coroutine*). "await" expressions, "async for" and "async +with" can only be used in the body of a coroutine function. + +Functions defined with "async def" syntax are always coroutine +functions, even if they do not contain "await" or "async" keywords. + +It is a "SyntaxError" to use a "yield from" expression inside the body +of a coroutine function. + +An example of a coroutine function: + + async def func(param1, param2): + do_stuff() + await some_coroutine() + +Changed in version 3.7: "await" and "async" are now keywords; +previously they were only treated as such inside the body of a +coroutine function. + + +The "async for" statement +========================= + + async_for_stmt ::= "async" for_stmt + +An *asynchronous iterable* provides an "__aiter__" method that +directly returns an *asynchronous iterator*, which can call +asynchronous code in its "__anext__" method. + +The "async for" statement allows convenient iteration over +asynchronous iterables. + +The following code: + + async for TARGET in ITER: + SUITE + else: + SUITE2 + +Is semantically equivalent to: + + iter = (ITER) + iter = type(iter).__aiter__(iter) + running = True + + while running: + try: + TARGET = await type(iter).__anext__(iter) + except StopAsyncIteration: + running = False + else: + SUITE + else: + SUITE2 + +See also "__aiter__()" and "__anext__()" for details. + +It is a "SyntaxError" to use an "async for" statement outside the body +of a coroutine function. + + +The "async with" statement +========================== + + async_with_stmt ::= "async" with_stmt + +An *asynchronous context manager* is a *context manager* that is able +to suspend execution in its *enter* and *exit* methods. + +The following code: + + async with EXPRESSION as TARGET: + SUITE + +is semantically equivalent to: + + manager = (EXPRESSION) + aenter = type(manager).__aenter__ + aexit = type(manager).__aexit__ + value = await aenter(manager) + hit_except = False + + try: + TARGET = value + SUITE + except: + hit_except = True + if not await aexit(manager, *sys.exc_info()): + raise + finally: + if not hit_except: + await aexit(manager, None, None, None) + +See also "__aenter__()" and "__aexit__()" for details. + +It is a "SyntaxError" to use an "async with" statement outside the +body of a coroutine function. + +See also: + + **PEP 492** - Coroutines with async and await syntax + The proposal that made coroutines a proper standalone concept in + Python, and added supporting syntax. +''', + 'atom-identifiers': r'''Identifiers (Names) +******************* + +An identifier occurring as an atom is a name. See section Identifiers +and keywords for lexical definition and section Naming and binding for +documentation of naming and binding. + +When the name is bound to an object, evaluation of the atom yields +that object. When a name is not bound, an attempt to evaluate it +raises a "NameError" exception. + + +Private name mangling +===================== + +When an identifier that textually occurs in a class definition begins +with two or more underscore characters and does not end in two or more +underscores, it is considered a *private name* of that class. + +See also: The class specifications. + +More precisely, private names are transformed to a longer form before +code is generated for them. If the transformed name is longer than +255 characters, implementation-defined truncation may happen. + +The transformation is independent of the syntactical context in which +the identifier is used but only the following private identifiers are +mangled: + +* Any name used as the name of a variable that is assigned or read or + any name of an attribute being accessed. + + The "__name__" attribute of nested functions, classes, and type + aliases is however not mangled. + +* The name of imported modules, e.g., "__spam" in "import __spam". If + the module is part of a package (i.e., its name contains a dot), the + name is *not* mangled, e.g., the "__foo" in "import __foo.bar" is + not mangled. + +* The name of an imported member, e.g., "__f" in "from spam import + __f". + +The transformation rule is defined as follows: + +* The class name, with leading underscores removed and a single + leading underscore inserted, is inserted in front of the identifier, + e.g., the identifier "__spam" occurring in a class named "Foo", + "_Foo" or "__Foo" is transformed to "_Foo__spam". + +* If the class name consists only of underscores, the transformation + is the identity, e.g., the identifier "__spam" occurring in a class + named "_" or "__" is left as is. +''', + 'atom-literals': r'''Literals +******** + +Python supports string and bytes literals and various numeric +literals: + + literal ::= stringliteral | bytesliteral + | integer | floatnumber | imagnumber + +Evaluation of a literal yields an object of the given type (string, +bytes, integer, floating-point number, complex number) with the given +value. The value may be approximated in the case of floating-point +and imaginary (complex) literals. See section Literals for details. + +All literals correspond to immutable data types, and hence the +object’s identity is less important than its value. Multiple +evaluations of literals with the same value (either the same +occurrence in the program text or a different occurrence) may obtain +the same object or a different object with the same value. +''', + 'attribute-access': r'''Customizing attribute access +**************************** + +The following methods can be defined to customize the meaning of +attribute access (use of, assignment to, or deletion of "x.name") for +class instances. + +object.__getattr__(self, name) + + Called when the default attribute access fails with an + "AttributeError" (either "__getattribute__()" raises an + "AttributeError" because *name* is not an instance attribute or an + attribute in the class tree for "self"; or "__get__()" of a *name* + property raises "AttributeError"). This method should either + return the (computed) attribute value or raise an "AttributeError" + exception. The "object" class itself does not provide this method. + + Note that if the attribute is found through the normal mechanism, + "__getattr__()" is not called. (This is an intentional asymmetry + between "__getattr__()" and "__setattr__()".) This is done both for + efficiency reasons and because otherwise "__getattr__()" would have + no way to access other attributes of the instance. Note that at + least for instance variables, you can take total control by not + inserting any values in the instance attribute dictionary (but + instead inserting them in another object). See the + "__getattribute__()" method below for a way to actually get total + control over attribute access. + +object.__getattribute__(self, name) + + Called unconditionally to implement attribute accesses for + instances of the class. If the class also defines "__getattr__()", + the latter will not be called unless "__getattribute__()" either + calls it explicitly or raises an "AttributeError". This method + should return the (computed) attribute value or raise an + "AttributeError" exception. In order to avoid infinite recursion in + this method, its implementation should always call the base class + method with the same name to access any attributes it needs, for + example, "object.__getattribute__(self, name)". + + Note: + + This method may still be bypassed when looking up special methods + as the result of implicit invocation via language syntax or + built-in functions. See Special method lookup. + + For certain sensitive attribute accesses, raises an auditing event + "object.__getattr__" with arguments "obj" and "name". + +object.__setattr__(self, name, value) + + Called when an attribute assignment is attempted. This is called + instead of the normal mechanism (i.e. store the value in the + instance dictionary). *name* is the attribute name, *value* is the + value to be assigned to it. + + If "__setattr__()" wants to assign to an instance attribute, it + should call the base class method with the same name, for example, + "object.__setattr__(self, name, value)". + + For certain sensitive attribute assignments, raises an auditing + event "object.__setattr__" with arguments "obj", "name", "value". + +object.__delattr__(self, name) + + Like "__setattr__()" but for attribute deletion instead of + assignment. This should only be implemented if "del obj.name" is + meaningful for the object. + + For certain sensitive attribute deletions, raises an auditing event + "object.__delattr__" with arguments "obj" and "name". + +object.__dir__(self) + + Called when "dir()" is called on the object. An iterable must be + returned. "dir()" converts the returned iterable to a list and + sorts it. + + +Customizing module attribute access +=================================== + +Special names "__getattr__" and "__dir__" can be also used to +customize access to module attributes. The "__getattr__" function at +the module level should accept one argument which is the name of an +attribute and return the computed value or raise an "AttributeError". +If an attribute is not found on a module object through the normal +lookup, i.e. "object.__getattribute__()", then "__getattr__" is +searched in the module "__dict__" before raising an "AttributeError". +If found, it is called with the attribute name and the result is +returned. + +The "__dir__" function should accept no arguments, and return an +iterable of strings that represents the names accessible on module. If +present, this function overrides the standard "dir()" search on a +module. + +For a more fine grained customization of the module behavior (setting +attributes, properties, etc.), one can set the "__class__" attribute +of a module object to a subclass of "types.ModuleType". For example: + + import sys + from types import ModuleType + + class VerboseModule(ModuleType): + def __repr__(self): + return f'Verbose {self.__name__}' + + def __setattr__(self, attr, value): + print(f'Setting {attr}...') + super().__setattr__(attr, value) + + sys.modules[__name__].__class__ = VerboseModule + +Note: + + Defining module "__getattr__" and setting module "__class__" only + affect lookups made using the attribute access syntax – directly + accessing the module globals (whether by code within the module, or + via a reference to the module’s globals dictionary) is unaffected. + +Changed in version 3.5: "__class__" module attribute is now writable. + +Added in version 3.7: "__getattr__" and "__dir__" module attributes. + +See also: + + **PEP 562** - Module __getattr__ and __dir__ + Describes the "__getattr__" and "__dir__" functions on modules. + + +Implementing Descriptors +======================== + +The following methods only apply when an instance of the class +containing the method (a so-called *descriptor* class) appears in an +*owner* class (the descriptor must be in either the owner’s class +dictionary or in the class dictionary for one of its parents). In the +examples below, “the attribute” refers to the attribute whose name is +the key of the property in the owner class’ "__dict__". The "object" +class itself does not implement any of these protocols. + +object.__get__(self, instance, owner=None) + + Called to get the attribute of the owner class (class attribute + access) or of an instance of that class (instance attribute + access). The optional *owner* argument is the owner class, while + *instance* is the instance that the attribute was accessed through, + or "None" when the attribute is accessed through the *owner*. + + This method should return the computed attribute value or raise an + "AttributeError" exception. + + **PEP 252** specifies that "__get__()" is callable with one or two + arguments. Python’s own built-in descriptors support this + specification; however, it is likely that some third-party tools + have descriptors that require both arguments. Python’s own + "__getattribute__()" implementation always passes in both arguments + whether they are required or not. + +object.__set__(self, instance, value) + + Called to set the attribute on an instance *instance* of the owner + class to a new value, *value*. + + Note, adding "__set__()" or "__delete__()" changes the kind of + descriptor to a “data descriptor”. See Invoking Descriptors for + more details. + +object.__delete__(self, instance) + + Called to delete the attribute on an instance *instance* of the + owner class. + +Instances of descriptors may also have the "__objclass__" attribute +present: + +object.__objclass__ + + The attribute "__objclass__" is interpreted by the "inspect" module + as specifying the class where this object was defined (setting this + appropriately can assist in runtime introspection of dynamic class + attributes). For callables, it may indicate that an instance of the + given type (or a subclass) is expected or required as the first + positional argument (for example, CPython sets this attribute for + unbound methods that are implemented in C). + + +Invoking Descriptors +==================== + +In general, a descriptor is an object attribute with “binding +behavior”, one whose attribute access has been overridden by methods +in the descriptor protocol: "__get__()", "__set__()", and +"__delete__()". If any of those methods are defined for an object, it +is said to be a descriptor. + +The default behavior for attribute access is to get, set, or delete +the attribute from an object’s dictionary. For instance, "a.x" has a +lookup chain starting with "a.__dict__['x']", then +"type(a).__dict__['x']", and continuing through the base classes of +"type(a)" excluding metaclasses. + +However, if the looked-up value is an object defining one of the +descriptor methods, then Python may override the default behavior and +invoke the descriptor method instead. Where this occurs in the +precedence chain depends on which descriptor methods were defined and +how they were called. + +The starting point for descriptor invocation is a binding, "a.x". How +the arguments are assembled depends on "a": + +Direct Call + The simplest and least common call is when user code directly + invokes a descriptor method: "x.__get__(a)". + +Instance Binding + If binding to an object instance, "a.x" is transformed into the + call: "type(a).__dict__['x'].__get__(a, type(a))". + +Class Binding + If binding to a class, "A.x" is transformed into the call: + "A.__dict__['x'].__get__(None, A)". + +Super Binding + A dotted lookup such as "super(A, a).x" searches + "a.__class__.__mro__" for a base class "B" following "A" and then + returns "B.__dict__['x'].__get__(a, A)". If not a descriptor, "x" + is returned unchanged. + +For instance bindings, the precedence of descriptor invocation depends +on which descriptor methods are defined. A descriptor can define any +combination of "__get__()", "__set__()" and "__delete__()". If it +does not define "__get__()", then accessing the attribute will return +the descriptor object itself unless there is a value in the object’s +instance dictionary. If the descriptor defines "__set__()" and/or +"__delete__()", it is a data descriptor; if it defines neither, it is +a non-data descriptor. Normally, data descriptors define both +"__get__()" and "__set__()", while non-data descriptors have just the +"__get__()" method. Data descriptors with "__get__()" and "__set__()" +(and/or "__delete__()") defined always override a redefinition in an +instance dictionary. In contrast, non-data descriptors can be +overridden by instances. + +Python methods (including those decorated with "@staticmethod" and +"@classmethod") are implemented as non-data descriptors. Accordingly, +instances can redefine and override methods. This allows individual +instances to acquire behaviors that differ from other instances of the +same class. + +The "property()" function is implemented as a data descriptor. +Accordingly, instances cannot override the behavior of a property. + + +__slots__ +========= + +*__slots__* allow us to explicitly declare data members (like +properties) and deny the creation of "__dict__" and *__weakref__* +(unless explicitly declared in *__slots__* or available in a parent.) + +The space saved over using "__dict__" can be significant. Attribute +lookup speed can be significantly improved as well. + +object.__slots__ + + This class variable can be assigned a string, iterable, or sequence + of strings with variable names used by instances. *__slots__* + reserves space for the declared variables and prevents the + automatic creation of "__dict__" and *__weakref__* for each + instance. + +Notes on using *__slots__*: + +* When inheriting from a class without *__slots__*, the "__dict__" and + *__weakref__* attribute of the instances will always be accessible. + +* Without a "__dict__" variable, instances cannot be assigned new + variables not listed in the *__slots__* definition. Attempts to + assign to an unlisted variable name raises "AttributeError". If + dynamic assignment of new variables is desired, then add + "'__dict__'" to the sequence of strings in the *__slots__* + declaration. + +* Without a *__weakref__* variable for each instance, classes defining + *__slots__* do not support "weak references" to its instances. If + weak reference support is needed, then add "'__weakref__'" to the + sequence of strings in the *__slots__* declaration. + +* *__slots__* are implemented at the class level by creating + descriptors for each variable name. As a result, class attributes + cannot be used to set default values for instance variables defined + by *__slots__*; otherwise, the class attribute would overwrite the + descriptor assignment. + +* The action of a *__slots__* declaration is not limited to the class + where it is defined. *__slots__* declared in parents are available + in child classes. However, instances of a child subclass will get a + "__dict__" and *__weakref__* unless the subclass also defines + *__slots__* (which should only contain names of any *additional* + slots). + +* If a class defines a slot also defined in a base class, the instance + variable defined by the base class slot is inaccessible (except by + retrieving its descriptor directly from the base class). This + renders the meaning of the program undefined. In the future, a + check may be added to prevent this. + +* "TypeError" will be raised if nonempty *__slots__* are defined for a + class derived from a ""variable-length" built-in type" such as + "int", "bytes", and "tuple". + +* Any non-string *iterable* may be assigned to *__slots__*. + +* If a "dictionary" is used to assign *__slots__*, the dictionary keys + will be used as the slot names. The values of the dictionary can be + used to provide per-attribute docstrings that will be recognised by + "inspect.getdoc()" and displayed in the output of "help()". + +* "__class__" assignment works only if both classes have the same + *__slots__*. + +* Multiple inheritance with multiple slotted parent classes can be + used, but only one parent is allowed to have attributes created by + slots (the other bases must have empty slot layouts) - violations + raise "TypeError". + +* If an *iterator* is used for *__slots__* then a *descriptor* is + created for each of the iterator’s values. However, the *__slots__* + attribute will be an empty iterator. +''', + 'attribute-references': r'''Attribute references +******************** + +An attribute reference is a primary followed by a period and a name: + + attributeref ::= primary "." identifier + +The primary must evaluate to an object of a type that supports +attribute references, which most objects do. This object is then +asked to produce the attribute whose name is the identifier. The type +and value produced is determined by the object. Multiple evaluations +of the same attribute reference may yield different objects. + +This production can be customized by overriding the +"__getattribute__()" method or the "__getattr__()" method. The +"__getattribute__()" method is called first and either returns a value +or raises "AttributeError" if the attribute is not available. + +If an "AttributeError" is raised and the object has a "__getattr__()" +method, that method is called as a fallback. +''', + 'augassign': r'''Augmented assignment statements +******************************* + +Augmented assignment is the combination, in a single statement, of a +binary operation and an assignment statement: + + augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression) + augtarget ::= identifier | attributeref | subscription | slicing + augop ::= "+=" | "-=" | "*=" | "@=" | "/=" | "//=" | "%=" | "**=" + | ">>=" | "<<=" | "&=" | "^=" | "|=" + +(See section Primaries for the syntax definitions of the last three +symbols.) + +An augmented assignment evaluates the target (which, unlike normal +assignment statements, cannot be an unpacking) and the expression +list, performs the binary operation specific to the type of assignment +on the two operands, and assigns the result to the original target. +The target is only evaluated once. + +An augmented assignment statement like "x += 1" can be rewritten as "x += x + 1" to achieve a similar, but not exactly equal effect. In the +augmented version, "x" is only evaluated once. Also, when possible, +the actual operation is performed *in-place*, meaning that rather than +creating a new object and assigning that to the target, the old object +is modified instead. + +Unlike normal assignments, augmented assignments evaluate the left- +hand side *before* evaluating the right-hand side. For example, "a[i] ++= f(x)" first looks-up "a[i]", then it evaluates "f(x)" and performs +the addition, and lastly, it writes the result back to "a[i]". + +With the exception of assigning to tuples and multiple targets in a +single statement, the assignment done by augmented assignment +statements is handled the same way as normal assignments. Similarly, +with the exception of the possible *in-place* behavior, the binary +operation performed by augmented assignment is the same as the normal +binary operations. + +For targets which are attribute references, the same caveat about +class and instance attributes applies as for regular assignments. +''', + 'await': r'''Await expression +**************** + +Suspend the execution of *coroutine* on an *awaitable* object. Can +only be used inside a *coroutine function*. + + await_expr ::= "await" primary + +Added in version 3.5. +''', + 'binary': r'''Binary arithmetic operations +**************************** + +The binary arithmetic operations have the conventional priority +levels. Note that some of these operations also apply to certain non- +numeric types. Apart from the power operator, there are only two +levels, one for multiplicative operators and one for additive +operators: + + m_expr ::= u_expr | m_expr "*" u_expr | m_expr "@" m_expr | + m_expr "//" u_expr | m_expr "/" u_expr | + m_expr "%" u_expr + a_expr ::= m_expr | a_expr "+" m_expr | a_expr "-" m_expr + +The "*" (multiplication) operator yields the product of its arguments. +The arguments must either both be numbers, or one argument must be an +integer and the other must be a sequence. In the former case, the +numbers are converted to a common real type and then multiplied +together. In the latter case, sequence repetition is performed; a +negative repetition factor yields an empty sequence. + +This operation can be customized using the special "__mul__()" and +"__rmul__()" methods. + +Changed in version 3.14: If only one operand is a complex number, the +other operand is converted to a floating-point number. + +The "@" (at) operator is intended to be used for matrix +multiplication. No builtin Python types implement this operator. + +This operation can be customized using the special "__matmul__()" and +"__rmatmul__()" methods. + +Added in version 3.5. + +The "/" (division) and "//" (floor division) operators yield the +quotient of their arguments. The numeric arguments are first +converted to a common type. Division of integers yields a float, while +floor division of integers results in an integer; the result is that +of mathematical division with the ‘floor’ function applied to the +result. Division by zero raises the "ZeroDivisionError" exception. + +The division operation can be customized using the special +"__truediv__()" and "__rtruediv__()" methods. The floor division +operation can be customized using the special "__floordiv__()" and +"__rfloordiv__()" methods. + +The "%" (modulo) operator yields the remainder from the division of +the first argument by the second. The numeric arguments are first +converted to a common type. A zero right argument raises the +"ZeroDivisionError" exception. The arguments may be floating-point +numbers, e.g., "3.14%0.7" equals "0.34" (since "3.14" equals "4*0.7 + +0.34".) The modulo operator always yields a result with the same sign +as its second operand (or zero); the absolute value of the result is +strictly smaller than the absolute value of the second operand [1]. + +The floor division and modulo operators are connected by the following +identity: "x == (x//y)*y + (x%y)". Floor division and modulo are also +connected with the built-in function "divmod()": "divmod(x, y) == +(x//y, x%y)". [2]. + +In addition to performing the modulo operation on numbers, the "%" +operator is also overloaded by string objects to perform old-style +string formatting (also known as interpolation). The syntax for +string formatting is described in the Python Library Reference, +section printf-style String Formatting. + +The *modulo* operation can be customized using the special "__mod__()" +and "__rmod__()" methods. + +The floor division operator, the modulo operator, and the "divmod()" +function are not defined for complex numbers. Instead, convert to a +floating-point number using the "abs()" function if appropriate. + +The "+" (addition) operator yields the sum of its arguments. The +arguments must either both be numbers or both be sequences of the same +type. In the former case, the numbers are converted to a common real +type and then added together. In the latter case, the sequences are +concatenated. + +This operation can be customized using the special "__add__()" and +"__radd__()" methods. + +Changed in version 3.14: If only one operand is a complex number, the +other operand is converted to a floating-point number. + +The "-" (subtraction) operator yields the difference of its arguments. +The numeric arguments are first converted to a common real type. + +This operation can be customized using the special "__sub__()" and +"__rsub__()" methods. + +Changed in version 3.14: If only one operand is a complex number, the +other operand is converted to a floating-point number. +''', + 'bitwise': r'''Binary bitwise operations +************************* + +Each of the three bitwise operations has a different priority level: + + and_expr ::= shift_expr | and_expr "&" shift_expr + xor_expr ::= and_expr | xor_expr "^" and_expr + or_expr ::= xor_expr | or_expr "|" xor_expr + +The "&" operator yields the bitwise AND of its arguments, which must +be integers or one of them must be a custom object overriding +"__and__()" or "__rand__()" special methods. + +The "^" operator yields the bitwise XOR (exclusive OR) of its +arguments, which must be integers or one of them must be a custom +object overriding "__xor__()" or "__rxor__()" special methods. + +The "|" operator yields the bitwise (inclusive) OR of its arguments, +which must be integers or one of them must be a custom object +overriding "__or__()" or "__ror__()" special methods. +''', + 'bltin-code-objects': r'''Code Objects +************ + +Code objects are used by the implementation to represent “pseudo- +compiled” executable Python code such as a function body. They differ +from function objects because they don’t contain a reference to their +global execution environment. Code objects are returned by the built- +in "compile()" function and can be extracted from function objects +through their "__code__" attribute. See also the "code" module. + +Accessing "__code__" raises an auditing event "object.__getattr__" +with arguments "obj" and ""__code__"". + +A code object can be executed or evaluated by passing it (instead of a +source string) to the "exec()" or "eval()" built-in functions. + +See The standard type hierarchy for more information. +''', + 'bltin-ellipsis-object': r'''The Ellipsis Object +******************* + +This object is commonly used by slicing (see Slicings). It supports +no special operations. There is exactly one ellipsis object, named +"Ellipsis" (a built-in name). "type(Ellipsis)()" produces the +"Ellipsis" singleton. + +It is written as "Ellipsis" or "...". +''', + 'bltin-null-object': r'''The Null Object +*************** + +This object is returned by functions that don’t explicitly return a +value. It supports no special operations. There is exactly one null +object, named "None" (a built-in name). "type(None)()" produces the +same singleton. + +It is written as "None". +''', + 'bltin-type-objects': r'''Type Objects +************ + +Type objects represent the various object types. An object’s type is +accessed by the built-in function "type()". There are no special +operations on types. The standard module "types" defines names for +all standard built-in types. + +Types are written like this: "". +''', + 'booleans': r'''Boolean operations +****************** + + or_test ::= and_test | or_test "or" and_test + and_test ::= not_test | and_test "and" not_test + not_test ::= comparison | "not" not_test + +In the context of Boolean operations, and also when expressions are +used by control flow statements, the following values are interpreted +as false: "False", "None", numeric zero of all types, and empty +strings and containers (including strings, tuples, lists, +dictionaries, sets and frozensets). All other values are interpreted +as true. User-defined objects can customize their truth value by +providing a "__bool__()" method. + +The operator "not" yields "True" if its argument is false, "False" +otherwise. + +The expression "x and y" first evaluates *x*; if *x* is false, its +value is returned; otherwise, *y* is evaluated and the resulting value +is returned. + +The expression "x or y" first evaluates *x*; if *x* is true, its value +is returned; otherwise, *y* is evaluated and the resulting value is +returned. + +Note that neither "and" nor "or" restrict the value and type they +return to "False" and "True", but rather return the last evaluated +argument. This is sometimes useful, e.g., if "s" is a string that +should be replaced by a default value if it is empty, the expression +"s or 'foo'" yields the desired value. Because "not" has to create a +new value, it returns a boolean value regardless of the type of its +argument (for example, "not 'foo'" produces "False" rather than "''".) +''', + 'break': r'''The "break" statement +********************* + + break_stmt ::= "break" + +"break" may only occur syntactically nested in a "for" or "while" +loop, but not nested in a function or class definition within that +loop. + +It terminates the nearest enclosing loop, skipping the optional "else" +clause if the loop has one. + +If a "for" loop is terminated by "break", the loop control target +keeps its current value. + +When "break" passes control out of a "try" statement with a "finally" +clause, that "finally" clause is executed before really leaving the +loop. +''', + 'callable-types': r'''Emulating callable objects +************************** + +object.__call__(self[, args...]) + + Called when the instance is “called” as a function; if this method + is defined, "x(arg1, arg2, ...)" roughly translates to + "type(x).__call__(x, arg1, ...)". The "object" class itself does + not provide this method. +''', + 'calls': r'''Calls +***** + +A call calls a callable object (e.g., a *function*) with a possibly +empty series of *arguments*: + + call ::= primary "(" [argument_list [","] | comprehension] ")" + argument_list ::= positional_arguments ["," starred_and_keywords] + ["," keywords_arguments] + | starred_and_keywords ["," keywords_arguments] + | keywords_arguments + positional_arguments ::= positional_item ("," positional_item)* + positional_item ::= assignment_expression | "*" expression + starred_and_keywords ::= ("*" expression | keyword_item) + ("," "*" expression | "," keyword_item)* + keywords_arguments ::= (keyword_item | "**" expression) + ("," keyword_item | "," "**" expression)* + keyword_item ::= identifier "=" expression + +An optional trailing comma may be present after the positional and +keyword arguments but does not affect the semantics. + +The primary must evaluate to a callable object (user-defined +functions, built-in functions, methods of built-in objects, class +objects, methods of class instances, and all objects having a +"__call__()" method are callable). All argument expressions are +evaluated before the call is attempted. Please refer to section +Function definitions for the syntax of formal *parameter* lists. + +If keyword arguments are present, they are first converted to +positional arguments, as follows. First, a list of unfilled slots is +created for the formal parameters. If there are N positional +arguments, they are placed in the first N slots. Next, for each +keyword argument, the identifier is used to determine the +corresponding slot (if the identifier is the same as the first formal +parameter name, the first slot is used, and so on). If the slot is +already filled, a "TypeError" exception is raised. Otherwise, the +argument is placed in the slot, filling it (even if the expression is +"None", it fills the slot). When all arguments have been processed, +the slots that are still unfilled are filled with the corresponding +default value from the function definition. (Default values are +calculated, once, when the function is defined; thus, a mutable object +such as a list or dictionary used as default value will be shared by +all calls that don’t specify an argument value for the corresponding +slot; this should usually be avoided.) If there are any unfilled +slots for which no default value is specified, a "TypeError" exception +is raised. Otherwise, the list of filled slots is used as the +argument list for the call. + +**CPython implementation detail:** An implementation may provide +built-in functions whose positional parameters do not have names, even +if they are ‘named’ for the purpose of documentation, and which +therefore cannot be supplied by keyword. In CPython, this is the case +for functions implemented in C that use "PyArg_ParseTuple()" to parse +their arguments. + +If there are more positional arguments than there are formal parameter +slots, a "TypeError" exception is raised, unless a formal parameter +using the syntax "*identifier" is present; in this case, that formal +parameter receives a tuple containing the excess positional arguments +(or an empty tuple if there were no excess positional arguments). + +If any keyword argument does not correspond to a formal parameter +name, a "TypeError" exception is raised, unless a formal parameter +using the syntax "**identifier" is present; in this case, that formal +parameter receives a dictionary containing the excess keyword +arguments (using the keywords as keys and the argument values as +corresponding values), or a (new) empty dictionary if there were no +excess keyword arguments. + +If the syntax "*expression" appears in the function call, "expression" +must evaluate to an *iterable*. Elements from these iterables are +treated as if they were additional positional arguments. For the call +"f(x1, x2, *y, x3, x4)", if *y* evaluates to a sequence *y1*, …, *yM*, +this is equivalent to a call with M+4 positional arguments *x1*, *x2*, +*y1*, …, *yM*, *x3*, *x4*. + +A consequence of this is that although the "*expression" syntax may +appear *after* explicit keyword arguments, it is processed *before* +the keyword arguments (and any "**expression" arguments – see below). +So: + + >>> def f(a, b): + ... print(a, b) + ... + >>> f(b=1, *(2,)) + 2 1 + >>> f(a=1, *(2,)) + Traceback (most recent call last): + File "", line 1, in + TypeError: f() got multiple values for keyword argument 'a' + >>> f(1, *(2,)) + 1 2 + +It is unusual for both keyword arguments and the "*expression" syntax +to be used in the same call, so in practice this confusion does not +often arise. + +If the syntax "**expression" appears in the function call, +"expression" must evaluate to a *mapping*, the contents of which are +treated as additional keyword arguments. If a parameter matching a key +has already been given a value (by an explicit keyword argument, or +from another unpacking), a "TypeError" exception is raised. + +When "**expression" is used, each key in this mapping must be a +string. Each value from the mapping is assigned to the first formal +parameter eligible for keyword assignment whose name is equal to the +key. A key need not be a Python identifier (e.g. ""max-temp °F"" is +acceptable, although it will not match any formal parameter that could +be declared). If there is no match to a formal parameter the key-value +pair is collected by the "**" parameter, if there is one, or if there +is not, a "TypeError" exception is raised. + +Formal parameters using the syntax "*identifier" or "**identifier" +cannot be used as positional argument slots or as keyword argument +names. + +Changed in version 3.5: Function calls accept any number of "*" and +"**" unpackings, positional arguments may follow iterable unpackings +("*"), and keyword arguments may follow dictionary unpackings ("**"). +Originally proposed by **PEP 448**. + +A call always returns some value, possibly "None", unless it raises an +exception. How this value is computed depends on the type of the +callable object. + +If it is— + +a user-defined function: + The code block for the function is executed, passing it the + argument list. The first thing the code block will do is bind the + formal parameters to the arguments; this is described in section + Function definitions. When the code block executes a "return" + statement, this specifies the return value of the function call. + If execution reaches the end of the code block without executing a + "return" statement, the return value is "None". + +a built-in function or method: + The result is up to the interpreter; see Built-in Functions for the + descriptions of built-in functions and methods. + +a class object: + A new instance of that class is returned. + +a class instance method: + The corresponding user-defined function is called, with an argument + list that is one longer than the argument list of the call: the + instance becomes the first argument. + +a class instance: + The class must define a "__call__()" method; the effect is then the + same as if that method was called. +''', + 'class': r'''Class definitions +***************** + +A class definition defines a class object (see section The standard +type hierarchy): + + classdef ::= [decorators] "class" classname [type_params] [inheritance] ":" suite + inheritance ::= "(" [argument_list] ")" + classname ::= identifier + +A class definition is an executable statement. The inheritance list +usually gives a list of base classes (see Metaclasses for more +advanced uses), so each item in the list should evaluate to a class +object which allows subclassing. Classes without an inheritance list +inherit, by default, from the base class "object"; hence, + + class Foo: + pass + +is equivalent to + + class Foo(object): + pass + +The class’s suite is then executed in a new execution frame (see +Naming and binding), using a newly created local namespace and the +original global namespace. (Usually, the suite contains mostly +function definitions.) When the class’s suite finishes execution, its +execution frame is discarded but its local namespace is saved. [5] A +class object is then created using the inheritance list for the base +classes and the saved local namespace for the attribute dictionary. +The class name is bound to this class object in the original local +namespace. + +The order in which attributes are defined in the class body is +preserved in the new class’s "__dict__". Note that this is reliable +only right after the class is created and only for classes that were +defined using the definition syntax. + +Class creation can be customized heavily using metaclasses. + +Classes can also be decorated: just like when decorating functions, + + @f1(arg) + @f2 + class Foo: pass + +is roughly equivalent to + + class Foo: pass + Foo = f1(arg)(f2(Foo)) + +The evaluation rules for the decorator expressions are the same as for +function decorators. The result is then bound to the class name. + +Changed in version 3.9: Classes may be decorated with any valid +"assignment_expression". Previously, the grammar was much more +restrictive; see **PEP 614** for details. + +A list of type parameters may be given in square brackets immediately +after the class’s name. This indicates to static type checkers that +the class is generic. At runtime, the type parameters can be retrieved +from the class’s "__type_params__" attribute. See Generic classes for +more. + +Changed in version 3.12: Type parameter lists are new in Python 3.12. + +**Programmer’s note:** Variables defined in the class definition are +class attributes; they are shared by instances. Instance attributes +can be set in a method with "self.name = value". Both class and +instance attributes are accessible through the notation “"self.name"”, +and an instance attribute hides a class attribute with the same name +when accessed in this way. Class attributes can be used as defaults +for instance attributes, but using mutable values there can lead to +unexpected results. Descriptors can be used to create instance +variables with different implementation details. + +See also: + + **PEP 3115** - Metaclasses in Python 3000 + The proposal that changed the declaration of metaclasses to the + current syntax, and the semantics for how classes with + metaclasses are constructed. + + **PEP 3129** - Class Decorators + The proposal that added class decorators. Function and method + decorators were introduced in **PEP 318**. +''', + 'comparisons': r'''Comparisons +*********** + +Unlike C, all comparison operations in Python have the same priority, +which is lower than that of any arithmetic, shifting or bitwise +operation. Also unlike C, expressions like "a < b < c" have the +interpretation that is conventional in mathematics: + + comparison ::= or_expr (comp_operator or_expr)* + comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "!=" + | "is" ["not"] | ["not"] "in" + +Comparisons yield boolean values: "True" or "False". Custom *rich +comparison methods* may return non-boolean values. In this case Python +will call "bool()" on such value in boolean contexts. + +Comparisons can be chained arbitrarily, e.g., "x < y <= z" is +equivalent to "x < y and y <= z", except that "y" is evaluated only +once (but in both cases "z" is not evaluated at all when "x < y" is +found to be false). + +Formally, if *a*, *b*, *c*, …, *y*, *z* are expressions and *op1*, +*op2*, …, *opN* are comparison operators, then "a op1 b op2 c ... y +opN z" is equivalent to "a op1 b and b op2 c and ... y opN z", except +that each expression is evaluated at most once. + +Note that "a op1 b op2 c" doesn’t imply any kind of comparison between +*a* and *c*, so that, e.g., "x < y > z" is perfectly legal (though +perhaps not pretty). + + +Value comparisons +================= + +The operators "<", ">", "==", ">=", "<=", and "!=" compare the values +of two objects. The objects do not need to have the same type. + +Chapter Objects, values and types states that objects have a value (in +addition to type and identity). The value of an object is a rather +abstract notion in Python: For example, there is no canonical access +method for an object’s value. Also, there is no requirement that the +value of an object should be constructed in a particular way, e.g. +comprised of all its data attributes. Comparison operators implement a +particular notion of what the value of an object is. One can think of +them as defining the value of an object indirectly, by means of their +comparison implementation. + +Because all types are (direct or indirect) subtypes of "object", they +inherit the default comparison behavior from "object". Types can +customize their comparison behavior by implementing *rich comparison +methods* like "__lt__()", described in Basic customization. + +The default behavior for equality comparison ("==" and "!=") is based +on the identity of the objects. Hence, equality comparison of +instances with the same identity results in equality, and equality +comparison of instances with different identities results in +inequality. A motivation for this default behavior is the desire that +all objects should be reflexive (i.e. "x is y" implies "x == y"). + +A default order comparison ("<", ">", "<=", and ">=") is not provided; +an attempt raises "TypeError". A motivation for this default behavior +is the lack of a similar invariant as for equality. + +The behavior of the default equality comparison, that instances with +different identities are always unequal, may be in contrast to what +types will need that have a sensible definition of object value and +value-based equality. Such types will need to customize their +comparison behavior, and in fact, a number of built-in types have done +that. + +The following list describes the comparison behavior of the most +important built-in types. + +* Numbers of built-in numeric types (Numeric Types — int, float, + complex) and of the standard library types "fractions.Fraction" and + "decimal.Decimal" can be compared within and across their types, + with the restriction that complex numbers do not support order + comparison. Within the limits of the types involved, they compare + mathematically (algorithmically) correct without loss of precision. + + The not-a-number values "float('NaN')" and "decimal.Decimal('NaN')" + are special. Any ordered comparison of a number to a not-a-number + value is false. A counter-intuitive implication is that not-a-number + values are not equal to themselves. For example, if "x = + float('NaN')", "3 < x", "x < 3" and "x == x" are all false, while "x + != x" is true. This behavior is compliant with IEEE 754. + +* "None" and "NotImplemented" are singletons. **PEP 8** advises that + comparisons for singletons should always be done with "is" or "is + not", never the equality operators. + +* Binary sequences (instances of "bytes" or "bytearray") can be + compared within and across their types. They compare + lexicographically using the numeric values of their elements. + +* Strings (instances of "str") compare lexicographically using the + numerical Unicode code points (the result of the built-in function + "ord()") of their characters. [3] + + Strings and binary sequences cannot be directly compared. + +* Sequences (instances of "tuple", "list", or "range") can be compared + only within each of their types, with the restriction that ranges do + not support order comparison. Equality comparison across these + types results in inequality, and ordering comparison across these + types raises "TypeError". + + Sequences compare lexicographically using comparison of + corresponding elements. The built-in containers typically assume + identical objects are equal to themselves. That lets them bypass + equality tests for identical objects to improve performance and to + maintain their internal invariants. + + Lexicographical comparison between built-in collections works as + follows: + + * For two collections to compare equal, they must be of the same + type, have the same length, and each pair of corresponding + elements must compare equal (for example, "[1,2] == (1,2)" is + false because the type is not the same). + + * Collections that support order comparison are ordered the same as + their first unequal elements (for example, "[1,2,x] <= [1,2,y]" + has the same value as "x <= y"). If a corresponding element does + not exist, the shorter collection is ordered first (for example, + "[1,2] < [1,2,3]" is true). + +* Mappings (instances of "dict") compare equal if and only if they + have equal "(key, value)" pairs. Equality comparison of the keys and + values enforces reflexivity. + + Order comparisons ("<", ">", "<=", and ">=") raise "TypeError". + +* Sets (instances of "set" or "frozenset") can be compared within and + across their types. + + They define order comparison operators to mean subset and superset + tests. Those relations do not define total orderings (for example, + the two sets "{1,2}" and "{2,3}" are not equal, nor subsets of one + another, nor supersets of one another). Accordingly, sets are not + appropriate arguments for functions which depend on total ordering + (for example, "min()", "max()", and "sorted()" produce undefined + results given a list of sets as inputs). + + Comparison of sets enforces reflexivity of its elements. + +* Most other built-in types have no comparison methods implemented, so + they inherit the default comparison behavior. + +User-defined classes that customize their comparison behavior should +follow some consistency rules, if possible: + +* Equality comparison should be reflexive. In other words, identical + objects should compare equal: + + "x is y" implies "x == y" + +* Comparison should be symmetric. In other words, the following + expressions should have the same result: + + "x == y" and "y == x" + + "x != y" and "y != x" + + "x < y" and "y > x" + + "x <= y" and "y >= x" + +* Comparison should be transitive. The following (non-exhaustive) + examples illustrate that: + + "x > y and y > z" implies "x > z" + + "x < y and y <= z" implies "x < z" + +* Inverse comparison should result in the boolean negation. In other + words, the following expressions should have the same result: + + "x == y" and "not x != y" + + "x < y" and "not x >= y" (for total ordering) + + "x > y" and "not x <= y" (for total ordering) + + The last two expressions apply to totally ordered collections (e.g. + to sequences, but not to sets or mappings). See also the + "total_ordering()" decorator. + +* The "hash()" result should be consistent with equality. Objects that + are equal should either have the same hash value, or be marked as + unhashable. + +Python does not enforce these consistency rules. In fact, the +not-a-number values are an example for not following these rules. + + +Membership test operations +========================== + +The operators "in" and "not in" test for membership. "x in s" +evaluates to "True" if *x* is a member of *s*, and "False" otherwise. +"x not in s" returns the negation of "x in s". All built-in sequences +and set types support this as well as dictionary, for which "in" tests +whether the dictionary has a given key. For container types such as +list, tuple, set, frozenset, dict, or collections.deque, the +expression "x in y" is equivalent to "any(x is e or x == e for e in +y)". + +For the string and bytes types, "x in y" is "True" if and only if *x* +is a substring of *y*. An equivalent test is "y.find(x) != -1". +Empty strings are always considered to be a substring of any other +string, so """ in "abc"" will return "True". + +For user-defined classes which define the "__contains__()" method, "x +in y" returns "True" if "y.__contains__(x)" returns a true value, and +"False" otherwise. + +For user-defined classes which do not define "__contains__()" but do +define "__iter__()", "x in y" is "True" if some value "z", for which +the expression "x is z or x == z" is true, is produced while iterating +over "y". If an exception is raised during the iteration, it is as if +"in" raised that exception. + +Lastly, the old-style iteration protocol is tried: if a class defines +"__getitem__()", "x in y" is "True" if and only if there is a non- +negative integer index *i* such that "x is y[i] or x == y[i]", and no +lower integer index raises the "IndexError" exception. (If any other +exception is raised, it is as if "in" raised that exception). + +The operator "not in" is defined to have the inverse truth value of +"in". + + +Identity comparisons +==================== + +The operators "is" and "is not" test for an object’s identity: "x is +y" is true if and only if *x* and *y* are the same object. An +Object’s identity is determined using the "id()" function. "x is not +y" yields the inverse truth value. [4] +''', + 'compound': r'''Compound statements +******************* + +Compound statements contain (groups of) other statements; they affect +or control the execution of those other statements in some way. In +general, compound statements span multiple lines, although in simple +incarnations a whole compound statement may be contained in one line. + +The "if", "while" and "for" statements implement traditional control +flow constructs. "try" specifies exception handlers and/or cleanup +code for a group of statements, while the "with" statement allows the +execution of initialization and finalization code around a block of +code. Function and class definitions are also syntactically compound +statements. + +A compound statement consists of one or more ‘clauses.’ A clause +consists of a header and a ‘suite.’ The clause headers of a +particular compound statement are all at the same indentation level. +Each clause header begins with a uniquely identifying keyword and ends +with a colon. A suite is a group of statements controlled by a +clause. A suite can be one or more semicolon-separated simple +statements on the same line as the header, following the header’s +colon, or it can be one or more indented statements on subsequent +lines. Only the latter form of a suite can contain nested compound +statements; the following is illegal, mostly because it wouldn’t be +clear to which "if" clause a following "else" clause would belong: + + if test1: if test2: print(x) + +Also note that the semicolon binds tighter than the colon in this +context, so that in the following example, either all or none of the +"print()" calls are executed: + + if x < y < z: print(x); print(y); print(z) + +Summarizing: + + compound_stmt ::= if_stmt + | while_stmt + | for_stmt + | try_stmt + | with_stmt + | match_stmt + | funcdef + | classdef + | async_with_stmt + | async_for_stmt + | async_funcdef + suite ::= stmt_list NEWLINE | NEWLINE INDENT statement+ DEDENT + statement ::= stmt_list NEWLINE | compound_stmt + stmt_list ::= simple_stmt (";" simple_stmt)* [";"] + +Note that statements always end in a "NEWLINE" possibly followed by a +"DEDENT". Also note that optional continuation clauses always begin +with a keyword that cannot start a statement, thus there are no +ambiguities (the ‘dangling "else"’ problem is solved in Python by +requiring nested "if" statements to be indented). + +The formatting of the grammar rules in the following sections places +each clause on a separate line for clarity. + + +The "if" statement +================== + +The "if" statement is used for conditional execution: + + if_stmt ::= "if" assignment_expression ":" suite + ("elif" assignment_expression ":" suite)* + ["else" ":" suite] + +It selects exactly one of the suites by evaluating the expressions one +by one until one is found to be true (see section Boolean operations +for the definition of true and false); then that suite is executed +(and no other part of the "if" statement is executed or evaluated). +If all expressions are false, the suite of the "else" clause, if +present, is executed. + + +The "while" statement +===================== + +The "while" statement is used for repeated execution as long as an +expression is true: + + while_stmt ::= "while" assignment_expression ":" suite + ["else" ":" suite] + +This repeatedly tests the expression and, if it is true, executes the +first suite; if the expression is false (which may be the first time +it is tested) the suite of the "else" clause, if present, is executed +and the loop terminates. + +A "break" statement executed in the first suite terminates the loop +without executing the "else" clause’s suite. A "continue" statement +executed in the first suite skips the rest of the suite and goes back +to testing the expression. + + +The "for" statement +=================== + +The "for" statement is used to iterate over the elements of a sequence +(such as a string, tuple or list) or other iterable object: + + for_stmt ::= "for" target_list "in" starred_list ":" suite + ["else" ":" suite] + +The "starred_list" expression is evaluated once; it should yield an +*iterable* object. An *iterator* is created for that iterable. The +first item provided by the iterator is then assigned to the target +list using the standard rules for assignments (see Assignment +statements), and the suite is executed. This repeats for each item +provided by the iterator. When the iterator is exhausted, the suite +in the "else" clause, if present, is executed, and the loop +terminates. + +A "break" statement executed in the first suite terminates the loop +without executing the "else" clause’s suite. A "continue" statement +executed in the first suite skips the rest of the suite and continues +with the next item, or with the "else" clause if there is no next +item. + +The for-loop makes assignments to the variables in the target list. +This overwrites all previous assignments to those variables including +those made in the suite of the for-loop: + + for i in range(10): + print(i) + i = 5 # this will not affect the for-loop + # because i will be overwritten with the next + # index in the range + +Names in the target list are not deleted when the loop is finished, +but if the sequence is empty, they will not have been assigned to at +all by the loop. Hint: the built-in type "range()" represents +immutable arithmetic sequences of integers. For instance, iterating +"range(3)" successively yields 0, 1, and then 2. + +Changed in version 3.11: Starred elements are now allowed in the +expression list. + + +The "try" statement +=================== + +The "try" statement specifies exception handlers and/or cleanup code +for a group of statements: + + try_stmt ::= try1_stmt | try2_stmt | try3_stmt + try1_stmt ::= "try" ":" suite + ("except" [expression ["as" identifier]] ":" suite)+ + ["else" ":" suite] + ["finally" ":" suite] + try2_stmt ::= "try" ":" suite + ("except" "*" expression ["as" identifier] ":" suite)+ + ["else" ":" suite] + ["finally" ":" suite] + try3_stmt ::= "try" ":" suite + "finally" ":" suite + +Additional information on exceptions can be found in section +Exceptions, and information on using the "raise" statement to generate +exceptions may be found in section The raise statement. + + +"except" clause +--------------- + +The "except" clause(s) specify one or more exception handlers. When no +exception occurs in the "try" clause, no exception handler is +executed. When an exception occurs in the "try" suite, a search for an +exception handler is started. This search inspects the "except" +clauses in turn until one is found that matches the exception. An +expression-less "except" clause, if present, must be last; it matches +any exception. + +For an "except" clause with an expression, the expression must +evaluate to an exception type or a tuple of exception types. The +raised exception matches an "except" clause whose expression evaluates +to the class or a *non-virtual base class* of the exception object, or +to a tuple that contains such a class. + +If no "except" clause matches the exception, the search for an +exception handler continues in the surrounding code and on the +invocation stack. [1] + +If the evaluation of an expression in the header of an "except" clause +raises an exception, the original search for a handler is canceled and +a search starts for the new exception in the surrounding code and on +the call stack (it is treated as if the entire "try" statement raised +the exception). + +When a matching "except" clause is found, the exception is assigned to +the target specified after the "as" keyword in that "except" clause, +if present, and the "except" clause’s suite is executed. All "except" +clauses must have an executable block. When the end of this block is +reached, execution continues normally after the entire "try" +statement. (This means that if two nested handlers exist for the same +exception, and the exception occurs in the "try" clause of the inner +handler, the outer handler will not handle the exception.) + +When an exception has been assigned using "as target", it is cleared +at the end of the "except" clause. This is as if + + except E as N: + foo + +was translated to + + except E as N: + try: + foo + finally: + del N + +This means the exception must be assigned to a different name to be +able to refer to it after the "except" clause. Exceptions are cleared +because with the traceback attached to them, they form a reference +cycle with the stack frame, keeping all locals in that frame alive +until the next garbage collection occurs. + +Before an "except" clause’s suite is executed, the exception is stored +in the "sys" module, where it can be accessed from within the body of +the "except" clause by calling "sys.exception()". When leaving an +exception handler, the exception stored in the "sys" module is reset +to its previous value: + + >>> print(sys.exception()) + None + >>> try: + ... raise TypeError + ... except: + ... print(repr(sys.exception())) + ... try: + ... raise ValueError + ... except: + ... print(repr(sys.exception())) + ... print(repr(sys.exception())) + ... + TypeError() + ValueError() + TypeError() + >>> print(sys.exception()) + None + + +"except*" clause +---------------- + +The "except*" clause(s) are used for handling "ExceptionGroup"s. The +exception type for matching is interpreted as in the case of "except", +but in the case of exception groups we can have partial matches when +the type matches some of the exceptions in the group. This means that +multiple "except*" clauses can execute, each handling part of the +exception group. Each clause executes at most once and handles an +exception group of all matching exceptions. Each exception in the +group is handled by at most one "except*" clause, the first that +matches it. + + >>> try: + ... raise ExceptionGroup("eg", + ... [ValueError(1), TypeError(2), OSError(3), OSError(4)]) + ... except* TypeError as e: + ... print(f'caught {type(e)} with nested {e.exceptions}') + ... except* OSError as e: + ... print(f'caught {type(e)} with nested {e.exceptions}') + ... + caught with nested (TypeError(2),) + caught with nested (OSError(3), OSError(4)) + + Exception Group Traceback (most recent call last): + | File "", line 2, in + | ExceptionGroup: eg + +-+---------------- 1 ---------------- + | ValueError: 1 + +------------------------------------ + +Any remaining exceptions that were not handled by any "except*" clause +are re-raised at the end, along with all exceptions that were raised +from within the "except*" clauses. If this list contains more than one +exception to reraise, they are combined into an exception group. + +If the raised exception is not an exception group and its type matches +one of the "except*" clauses, it is caught and wrapped by an exception +group with an empty message string. + + >>> try: + ... raise BlockingIOError + ... except* BlockingIOError as e: + ... print(repr(e)) + ... + ExceptionGroup('', (BlockingIOError())) + +An "except*" clause must have a matching expression; it cannot be +"except*:". Furthermore, this expression cannot contain exception +group types, because that would have ambiguous semantics. + +It is not possible to mix "except" and "except*" in the same "try". +"break", "continue" and "return" cannot appear in an "except*" clause. + + +"else" clause +------------- + +The optional "else" clause is executed if the control flow leaves the +"try" suite, no exception was raised, and no "return", "continue", or +"break" statement was executed. Exceptions in the "else" clause are +not handled by the preceding "except" clauses. + + +"finally" clause +---------------- + +If "finally" is present, it specifies a ‘cleanup’ handler. The "try" +clause is executed, including any "except" and "else" clauses. If an +exception occurs in any of the clauses and is not handled, the +exception is temporarily saved. The "finally" clause is executed. If +there is a saved exception it is re-raised at the end of the "finally" +clause. If the "finally" clause raises another exception, the saved +exception is set as the context of the new exception. If the "finally" +clause executes a "return", "break" or "continue" statement, the saved +exception is discarded: + + >>> def f(): + ... try: + ... 1/0 + ... finally: + ... return 42 + ... + >>> f() + 42 + +The exception information is not available to the program during +execution of the "finally" clause. + +When a "return", "break" or "continue" statement is executed in the +"try" suite of a "try"…"finally" statement, the "finally" clause is +also executed ‘on the way out.’ + +The return value of a function is determined by the last "return" +statement executed. Since the "finally" clause always executes, a +"return" statement executed in the "finally" clause will always be the +last one executed: + + >>> def foo(): + ... try: + ... return 'try' + ... finally: + ... return 'finally' + ... + >>> foo() + 'finally' + +Changed in version 3.8: Prior to Python 3.8, a "continue" statement +was illegal in the "finally" clause due to a problem with the +implementation. + + +The "with" statement +==================== + +The "with" statement is used to wrap the execution of a block with +methods defined by a context manager (see section With Statement +Context Managers). This allows common "try"…"except"…"finally" usage +patterns to be encapsulated for convenient reuse. + + with_stmt ::= "with" ( "(" with_stmt_contents ","? ")" | with_stmt_contents ) ":" suite + with_stmt_contents ::= with_item ("," with_item)* + with_item ::= expression ["as" target] + +The execution of the "with" statement with one “item” proceeds as +follows: + +1. The context expression (the expression given in the "with_item") is + evaluated to obtain a context manager. + +2. The context manager’s "__enter__()" is loaded for later use. + +3. The context manager’s "__exit__()" is loaded for later use. + +4. The context manager’s "__enter__()" method is invoked. + +5. If a target was included in the "with" statement, the return value + from "__enter__()" is assigned to it. + + Note: + + The "with" statement guarantees that if the "__enter__()" method + returns without an error, then "__exit__()" will always be + called. Thus, if an error occurs during the assignment to the + target list, it will be treated the same as an error occurring + within the suite would be. See step 7 below. + +6. The suite is executed. + +7. The context manager’s "__exit__()" method is invoked. If an + exception caused the suite to be exited, its type, value, and + traceback are passed as arguments to "__exit__()". Otherwise, three + "None" arguments are supplied. + + If the suite was exited due to an exception, and the return value + from the "__exit__()" method was false, the exception is reraised. + If the return value was true, the exception is suppressed, and + execution continues with the statement following the "with" + statement. + + If the suite was exited for any reason other than an exception, the + return value from "__exit__()" is ignored, and execution proceeds + at the normal location for the kind of exit that was taken. + +The following code: + + with EXPRESSION as TARGET: + SUITE + +is semantically equivalent to: + + manager = (EXPRESSION) + enter = type(manager).__enter__ + exit = type(manager).__exit__ + value = enter(manager) + hit_except = False + + try: + TARGET = value + SUITE + except: + hit_except = True + if not exit(manager, *sys.exc_info()): + raise + finally: + if not hit_except: + exit(manager, None, None, None) + +With more than one item, the context managers are processed as if +multiple "with" statements were nested: + + with A() as a, B() as b: + SUITE + +is semantically equivalent to: + + with A() as a: + with B() as b: + SUITE + +You can also write multi-item context managers in multiple lines if +the items are surrounded by parentheses. For example: + + with ( + A() as a, + B() as b, + ): + SUITE + +Changed in version 3.1: Support for multiple context expressions. + +Changed in version 3.10: Support for using grouping parentheses to +break the statement in multiple lines. + +See also: + + **PEP 343** - The “with” statement + The specification, background, and examples for the Python "with" + statement. + + +The "match" statement +===================== + +Added in version 3.10. + +The match statement is used for pattern matching. Syntax: + + match_stmt ::= 'match' subject_expr ":" NEWLINE INDENT case_block+ DEDENT + subject_expr ::= star_named_expression "," star_named_expressions? + | named_expression + case_block ::= 'case' patterns [guard] ":" block + +Note: + + This section uses single quotes to denote soft keywords. + +Pattern matching takes a pattern as input (following "case") and a +subject value (following "match"). The pattern (which may contain +subpatterns) is matched against the subject value. The outcomes are: + +* A match success or failure (also termed a pattern success or + failure). + +* Possible binding of matched values to a name. The prerequisites for + this are further discussed below. + +The "match" and "case" keywords are soft keywords. + +See also: + + * **PEP 634** – Structural Pattern Matching: Specification + + * **PEP 636** – Structural Pattern Matching: Tutorial + + +Overview +-------- + +Here’s an overview of the logical flow of a match statement: + +1. The subject expression "subject_expr" is evaluated and a resulting + subject value obtained. If the subject expression contains a comma, + a tuple is constructed using the standard rules. + +2. Each pattern in a "case_block" is attempted to match with the + subject value. The specific rules for success or failure are + described below. The match attempt can also bind some or all of the + standalone names within the pattern. The precise pattern binding + rules vary per pattern type and are specified below. **Name + bindings made during a successful pattern match outlive the + executed block and can be used after the match statement**. + + Note: + + During failed pattern matches, some subpatterns may succeed. Do + not rely on bindings being made for a failed match. Conversely, + do not rely on variables remaining unchanged after a failed + match. The exact behavior is dependent on implementation and may + vary. This is an intentional decision made to allow different + implementations to add optimizations. + +3. If the pattern succeeds, the corresponding guard (if present) is + evaluated. In this case all name bindings are guaranteed to have + happened. + + * If the guard evaluates as true or is missing, the "block" inside + "case_block" is executed. + + * Otherwise, the next "case_block" is attempted as described above. + + * If there are no further case blocks, the match statement is + completed. + +Note: + + Users should generally never rely on a pattern being evaluated. + Depending on implementation, the interpreter may cache values or use + other optimizations which skip repeated evaluations. + +A sample match statement: + + >>> flag = False + >>> match (100, 200): + ... case (100, 300): # Mismatch: 200 != 300 + ... print('Case 1') + ... case (100, 200) if flag: # Successful match, but guard fails + ... print('Case 2') + ... case (100, y): # Matches and binds y to 200 + ... print(f'Case 3, y: {y}') + ... case _: # Pattern not attempted + ... print('Case 4, I match anything!') + ... + Case 3, y: 200 + +In this case, "if flag" is a guard. Read more about that in the next +section. + + +Guards +------ + + guard ::= "if" named_expression + +A "guard" (which is part of the "case") must succeed for code inside +the "case" block to execute. It takes the form: "if" followed by an +expression. + +The logical flow of a "case" block with a "guard" follows: + +1. Check that the pattern in the "case" block succeeded. If the + pattern failed, the "guard" is not evaluated and the next "case" + block is checked. + +2. If the pattern succeeded, evaluate the "guard". + + * If the "guard" condition evaluates as true, the case block is + selected. + + * If the "guard" condition evaluates as false, the case block is + not selected. + + * If the "guard" raises an exception during evaluation, the + exception bubbles up. + +Guards are allowed to have side effects as they are expressions. +Guard evaluation must proceed from the first to the last case block, +one at a time, skipping case blocks whose pattern(s) don’t all +succeed. (I.e., guard evaluation must happen in order.) Guard +evaluation must stop once a case block is selected. + + +Irrefutable Case Blocks +----------------------- + +An irrefutable case block is a match-all case block. A match +statement may have at most one irrefutable case block, and it must be +last. + +A case block is considered irrefutable if it has no guard and its +pattern is irrefutable. A pattern is considered irrefutable if we can +prove from its syntax alone that it will always succeed. Only the +following patterns are irrefutable: + +* AS Patterns whose left-hand side is irrefutable + +* OR Patterns containing at least one irrefutable pattern + +* Capture Patterns + +* Wildcard Patterns + +* parenthesized irrefutable patterns + + +Patterns +-------- + +Note: + + This section uses grammar notations beyond standard EBNF: + + * the notation "SEP.RULE+" is shorthand for "RULE (SEP RULE)*" + + * the notation "!RULE" is shorthand for a negative lookahead + assertion + +The top-level syntax for "patterns" is: + + patterns ::= open_sequence_pattern | pattern + pattern ::= as_pattern | or_pattern + closed_pattern ::= | literal_pattern + | capture_pattern + | wildcard_pattern + | value_pattern + | group_pattern + | sequence_pattern + | mapping_pattern + | class_pattern + +The descriptions below will include a description “in simple terms” of +what a pattern does for illustration purposes (credits to Raymond +Hettinger for a document that inspired most of the descriptions). Note +that these descriptions are purely for illustration purposes and **may +not** reflect the underlying implementation. Furthermore, they do not +cover all valid forms. + + +OR Patterns +~~~~~~~~~~~ + +An OR pattern is two or more patterns separated by vertical bars "|". +Syntax: + + or_pattern ::= "|".closed_pattern+ + +Only the final subpattern may be irrefutable, and each subpattern must +bind the same set of names to avoid ambiguity. + +An OR pattern matches each of its subpatterns in turn to the subject +value, until one succeeds. The OR pattern is then considered +successful. Otherwise, if none of the subpatterns succeed, the OR +pattern fails. + +In simple terms, "P1 | P2 | ..." will try to match "P1", if it fails +it will try to match "P2", succeeding immediately if any succeeds, +failing otherwise. + + +AS Patterns +~~~~~~~~~~~ + +An AS pattern matches an OR pattern on the left of the "as" keyword +against a subject. Syntax: + + as_pattern ::= or_pattern "as" capture_pattern + +If the OR pattern fails, the AS pattern fails. Otherwise, the AS +pattern binds the subject to the name on the right of the as keyword +and succeeds. "capture_pattern" cannot be a "_". + +In simple terms "P as NAME" will match with "P", and on success it +will set "NAME = ". + + +Literal Patterns +~~~~~~~~~~~~~~~~ + +A literal pattern corresponds to most literals in Python. Syntax: + + literal_pattern ::= signed_number + | signed_number "+" NUMBER + | signed_number "-" NUMBER + | strings + | "None" + | "True" + | "False" + signed_number ::= ["-"] NUMBER + +The rule "strings" and the token "NUMBER" are defined in the standard +Python grammar. Triple-quoted strings are supported. Raw strings and +byte strings are supported. f-strings are not supported. + +The forms "signed_number '+' NUMBER" and "signed_number '-' NUMBER" +are for expressing complex numbers; they require a real number on the +left and an imaginary number on the right. E.g. "3 + 4j". + +In simple terms, "LITERAL" will succeed only if " == +LITERAL". For the singletons "None", "True" and "False", the "is" +operator is used. + + +Capture Patterns +~~~~~~~~~~~~~~~~ + +A capture pattern binds the subject value to a name. Syntax: + + capture_pattern ::= !'_' NAME + +A single underscore "_" is not a capture pattern (this is what "!'_'" +expresses). It is instead treated as a "wildcard_pattern". + +In a given pattern, a given name can only be bound once. E.g. "case +x, x: ..." is invalid while "case [x] | x: ..." is allowed. + +Capture patterns always succeed. The binding follows scoping rules +established by the assignment expression operator in **PEP 572**; the +name becomes a local variable in the closest containing function scope +unless there’s an applicable "global" or "nonlocal" statement. + +In simple terms "NAME" will always succeed and it will set "NAME = +". + + +Wildcard Patterns +~~~~~~~~~~~~~~~~~ + +A wildcard pattern always succeeds (matches anything) and binds no +name. Syntax: + + wildcard_pattern ::= '_' + +"_" is a soft keyword within any pattern, but only within patterns. +It is an identifier, as usual, even within "match" subject +expressions, "guard"s, and "case" blocks. + +In simple terms, "_" will always succeed. + + +Value Patterns +~~~~~~~~~~~~~~ + +A value pattern represents a named value in Python. Syntax: + + value_pattern ::= attr + attr ::= name_or_attr "." NAME + name_or_attr ::= attr | NAME + +The dotted name in the pattern is looked up using standard Python name +resolution rules. The pattern succeeds if the value found compares +equal to the subject value (using the "==" equality operator). + +In simple terms "NAME1.NAME2" will succeed only if " == +NAME1.NAME2" + +Note: + + If the same value occurs multiple times in the same match statement, + the interpreter may cache the first value found and reuse it rather + than repeat the same lookup. This cache is strictly tied to a given + execution of a given match statement. + + +Group Patterns +~~~~~~~~~~~~~~ + +A group pattern allows users to add parentheses around patterns to +emphasize the intended grouping. Otherwise, it has no additional +syntax. Syntax: + + group_pattern ::= "(" pattern ")" + +In simple terms "(P)" has the same effect as "P". + + +Sequence Patterns +~~~~~~~~~~~~~~~~~ + +A sequence pattern contains several subpatterns to be matched against +sequence elements. The syntax is similar to the unpacking of a list or +tuple. + + sequence_pattern ::= "[" [maybe_sequence_pattern] "]" + | "(" [open_sequence_pattern] ")" + open_sequence_pattern ::= maybe_star_pattern "," [maybe_sequence_pattern] + maybe_sequence_pattern ::= ",".maybe_star_pattern+ ","? + maybe_star_pattern ::= star_pattern | pattern + star_pattern ::= "*" (capture_pattern | wildcard_pattern) + +There is no difference if parentheses or square brackets are used for +sequence patterns (i.e. "(...)" vs "[...]" ). + +Note: + + A single pattern enclosed in parentheses without a trailing comma + (e.g. "(3 | 4)") is a group pattern. While a single pattern enclosed + in square brackets (e.g. "[3 | 4]") is still a sequence pattern. + +At most one star subpattern may be in a sequence pattern. The star +subpattern may occur in any position. If no star subpattern is +present, the sequence pattern is a fixed-length sequence pattern; +otherwise it is a variable-length sequence pattern. + +The following is the logical flow for matching a sequence pattern +against a subject value: + +1. If the subject value is not a sequence [2], the sequence pattern + fails. + +2. If the subject value is an instance of "str", "bytes" or + "bytearray" the sequence pattern fails. + +3. The subsequent steps depend on whether the sequence pattern is + fixed or variable-length. + + If the sequence pattern is fixed-length: + + 1. If the length of the subject sequence is not equal to the number + of subpatterns, the sequence pattern fails + + 2. Subpatterns in the sequence pattern are matched to their + corresponding items in the subject sequence from left to right. + Matching stops as soon as a subpattern fails. If all + subpatterns succeed in matching their corresponding item, the + sequence pattern succeeds. + + Otherwise, if the sequence pattern is variable-length: + + 1. If the length of the subject sequence is less than the number of + non-star subpatterns, the sequence pattern fails. + + 2. The leading non-star subpatterns are matched to their + corresponding items as for fixed-length sequences. + + 3. If the previous step succeeds, the star subpattern matches a + list formed of the remaining subject items, excluding the + remaining items corresponding to non-star subpatterns following + the star subpattern. + + 4. Remaining non-star subpatterns are matched to their + corresponding subject items, as for a fixed-length sequence. + + Note: + + The length of the subject sequence is obtained via "len()" (i.e. + via the "__len__()" protocol). This length may be cached by the + interpreter in a similar manner as value patterns. + +In simple terms "[P1, P2, P3," … ", P]" matches only if all the +following happens: + +* check "" is a sequence + +* "len(subject) == " + +* "P1" matches "[0]" (note that this match can also bind + names) + +* "P2" matches "[1]" (note that this match can also bind + names) + +* … and so on for the corresponding pattern/element. + + +Mapping Patterns +~~~~~~~~~~~~~~~~ + +A mapping pattern contains one or more key-value patterns. The syntax +is similar to the construction of a dictionary. Syntax: + + mapping_pattern ::= "{" [items_pattern] "}" + items_pattern ::= ",".key_value_pattern+ ","? + key_value_pattern ::= (literal_pattern | value_pattern) ":" pattern + | double_star_pattern + double_star_pattern ::= "**" capture_pattern + +At most one double star pattern may be in a mapping pattern. The +double star pattern must be the last subpattern in the mapping +pattern. + +Duplicate keys in mapping patterns are disallowed. Duplicate literal +keys will raise a "SyntaxError". Two keys that otherwise have the same +value will raise a "ValueError" at runtime. + +The following is the logical flow for matching a mapping pattern +against a subject value: + +1. If the subject value is not a mapping [3],the mapping pattern + fails. + +2. If every key given in the mapping pattern is present in the subject + mapping, and the pattern for each key matches the corresponding + item of the subject mapping, the mapping pattern succeeds. + +3. If duplicate keys are detected in the mapping pattern, the pattern + is considered invalid. A "SyntaxError" is raised for duplicate + literal values; or a "ValueError" for named keys of the same value. + +Note: + + Key-value pairs are matched using the two-argument form of the + mapping subject’s "get()" method. Matched key-value pairs must + already be present in the mapping, and not created on-the-fly via + "__missing__()" or "__getitem__()". + +In simple terms "{KEY1: P1, KEY2: P2, ... }" matches only if all the +following happens: + +* check "" is a mapping + +* "KEY1 in " + +* "P1" matches "[KEY1]" + +* … and so on for the corresponding KEY/pattern pair. + + +Class Patterns +~~~~~~~~~~~~~~ + +A class pattern represents a class and its positional and keyword +arguments (if any). Syntax: + + class_pattern ::= name_or_attr "(" [pattern_arguments ","?] ")" + pattern_arguments ::= positional_patterns ["," keyword_patterns] + | keyword_patterns + positional_patterns ::= ",".pattern+ + keyword_patterns ::= ",".keyword_pattern+ + keyword_pattern ::= NAME "=" pattern + +The same keyword should not be repeated in class patterns. + +The following is the logical flow for matching a class pattern against +a subject value: + +1. If "name_or_attr" is not an instance of the builtin "type" , raise + "TypeError". + +2. If the subject value is not an instance of "name_or_attr" (tested + via "isinstance()"), the class pattern fails. + +3. If no pattern arguments are present, the pattern succeeds. + Otherwise, the subsequent steps depend on whether keyword or + positional argument patterns are present. + + For a number of built-in types (specified below), a single + positional subpattern is accepted which will match the entire + subject; for these types keyword patterns also work as for other + types. + + If only keyword patterns are present, they are processed as + follows, one by one: + + I. The keyword is looked up as an attribute on the subject. + + * If this raises an exception other than "AttributeError", the + exception bubbles up. + + * If this raises "AttributeError", the class pattern has failed. + + * Else, the subpattern associated with the keyword pattern is + matched against the subject’s attribute value. If this fails, + the class pattern fails; if this succeeds, the match proceeds + to the next keyword. + + II. If all keyword patterns succeed, the class pattern succeeds. + + If any positional patterns are present, they are converted to + keyword patterns using the "__match_args__" attribute on the class + "name_or_attr" before matching: + + I. The equivalent of "getattr(cls, "__match_args__", ())" is + called. + + * If this raises an exception, the exception bubbles up. + + * If the returned value is not a tuple, the conversion fails and + "TypeError" is raised. + + * If there are more positional patterns than + "len(cls.__match_args__)", "TypeError" is raised. + + * Otherwise, positional pattern "i" is converted to a keyword + pattern using "__match_args__[i]" as the keyword. + "__match_args__[i]" must be a string; if not "TypeError" is + raised. + + * If there are duplicate keywords, "TypeError" is raised. + + See also: + + Customizing positional arguments in class pattern matching + + II. Once all positional patterns have been converted to keyword + patterns, + the match proceeds as if there were only keyword patterns. + + For the following built-in types the handling of positional + subpatterns is different: + + * "bool" + + * "bytearray" + + * "bytes" + + * "dict" + + * "float" + + * "frozenset" + + * "int" + + * "list" + + * "set" + + * "str" + + * "tuple" + + These classes accept a single positional argument, and the pattern + there is matched against the whole object rather than an attribute. + For example "int(0|1)" matches the value "0", but not the value + "0.0". + +In simple terms "CLS(P1, attr=P2)" matches only if the following +happens: + +* "isinstance(, CLS)" + +* convert "P1" to a keyword pattern using "CLS.__match_args__" + +* For each keyword argument "attr=P2": + + * "hasattr(, "attr")" + + * "P2" matches ".attr" + +* … and so on for the corresponding keyword argument/pattern pair. + +See also: + + * **PEP 634** – Structural Pattern Matching: Specification + + * **PEP 636** – Structural Pattern Matching: Tutorial + + +Function definitions +==================== + +A function definition defines a user-defined function object (see +section The standard type hierarchy): + + funcdef ::= [decorators] "def" funcname [type_params] "(" [parameter_list] ")" + ["->" expression] ":" suite + decorators ::= decorator+ + decorator ::= "@" assignment_expression NEWLINE + parameter_list ::= defparameter ("," defparameter)* "," "/" ["," [parameter_list_no_posonly]] + | parameter_list_no_posonly + parameter_list_no_posonly ::= defparameter ("," defparameter)* ["," [parameter_list_starargs]] + | parameter_list_starargs + parameter_list_starargs ::= "*" [star_parameter] ("," defparameter)* ["," ["**" parameter [","]]] + | "**" parameter [","] + parameter ::= identifier [":" expression] + star_parameter ::= identifier [":" ["*"] expression] + defparameter ::= parameter ["=" expression] + funcname ::= identifier + +A function definition is an executable statement. Its execution binds +the function name in the current local namespace to a function object +(a wrapper around the executable code for the function). This +function object contains a reference to the current global namespace +as the global namespace to be used when the function is called. + +The function definition does not execute the function body; this gets +executed only when the function is called. [4] + +A function definition may be wrapped by one or more *decorator* +expressions. Decorator expressions are evaluated when the function is +defined, in the scope that contains the function definition. The +result must be a callable, which is invoked with the function object +as the only argument. The returned value is bound to the function name +instead of the function object. Multiple decorators are applied in +nested fashion. For example, the following code + + @f1(arg) + @f2 + def func(): pass + +is roughly equivalent to + + def func(): pass + func = f1(arg)(f2(func)) + +except that the original function is not temporarily bound to the name +"func". + +Changed in version 3.9: Functions may be decorated with any valid +"assignment_expression". Previously, the grammar was much more +restrictive; see **PEP 614** for details. + +A list of type parameters may be given in square brackets between the +function’s name and the opening parenthesis for its parameter list. +This indicates to static type checkers that the function is generic. +At runtime, the type parameters can be retrieved from the function’s +"__type_params__" attribute. See Generic functions for more. + +Changed in version 3.12: Type parameter lists are new in Python 3.12. + +When one or more *parameters* have the form *parameter* "=" +*expression*, the function is said to have “default parameter values.” +For a parameter with a default value, the corresponding *argument* may +be omitted from a call, in which case the parameter’s default value is +substituted. If a parameter has a default value, all following +parameters up until the “"*"” must also have a default value — this is +a syntactic restriction that is not expressed by the grammar. + +**Default parameter values are evaluated from left to right when the +function definition is executed.** This means that the expression is +evaluated once, when the function is defined, and that the same “pre- +computed” value is used for each call. This is especially important +to understand when a default parameter value is a mutable object, such +as a list or a dictionary: if the function modifies the object (e.g. +by appending an item to a list), the default parameter value is in +effect modified. This is generally not what was intended. A way +around this is to use "None" as the default, and explicitly test for +it in the body of the function, e.g.: + + def whats_on_the_telly(penguin=None): + if penguin is None: + penguin = [] + penguin.append("property of the zoo") + return penguin + +Function call semantics are described in more detail in section Calls. +A function call always assigns values to all parameters mentioned in +the parameter list, either from positional arguments, from keyword +arguments, or from default values. If the form “"*identifier"” is +present, it is initialized to a tuple receiving any excess positional +parameters, defaulting to the empty tuple. If the form +“"**identifier"” is present, it is initialized to a new ordered +mapping receiving any excess keyword arguments, defaulting to a new +empty mapping of the same type. Parameters after “"*"” or +“"*identifier"” are keyword-only parameters and may only be passed by +keyword arguments. Parameters before “"/"” are positional-only +parameters and may only be passed by positional arguments. + +Changed in version 3.8: The "/" function parameter syntax may be used +to indicate positional-only parameters. See **PEP 570** for details. + +Parameters may have an *annotation* of the form “": expression"” +following the parameter name. Any parameter may have an annotation, +even those of the form "*identifier" or "**identifier". (As a special +case, parameters of the form "*identifier" may have an annotation “": +*expression"”.) Functions may have “return” annotation of the form +“"-> expression"” after the parameter list. These annotations can be +any valid Python expression. The presence of annotations does not +change the semantics of a function. See Annotations for more +information on annotations. + +Changed in version 3.11: Parameters of the form “"*identifier"” may +have an annotation “": *expression"”. See **PEP 646**. + +It is also possible to create anonymous functions (functions not bound +to a name), for immediate use in expressions. This uses lambda +expressions, described in section Lambdas. Note that the lambda +expression is merely a shorthand for a simplified function definition; +a function defined in a “"def"” statement can be passed around or +assigned to another name just like a function defined by a lambda +expression. The “"def"” form is actually more powerful since it +allows the execution of multiple statements and annotations. + +**Programmer’s note:** Functions are first-class objects. A “"def"” +statement executed inside a function definition defines a local +function that can be returned or passed around. Free variables used +in the nested function can access the local variables of the function +containing the def. See section Naming and binding for details. + +See also: + + **PEP 3107** - Function Annotations + The original specification for function annotations. + + **PEP 484** - Type Hints + Definition of a standard meaning for annotations: type hints. + + **PEP 526** - Syntax for Variable Annotations + Ability to type hint variable declarations, including class + variables and instance variables. + + **PEP 563** - Postponed Evaluation of Annotations + Support for forward references within annotations by preserving + annotations in a string form at runtime instead of eager + evaluation. + + **PEP 318** - Decorators for Functions and Methods + Function and method decorators were introduced. Class decorators + were introduced in **PEP 3129**. + + +Class definitions +================= + +A class definition defines a class object (see section The standard +type hierarchy): + + classdef ::= [decorators] "class" classname [type_params] [inheritance] ":" suite + inheritance ::= "(" [argument_list] ")" + classname ::= identifier + +A class definition is an executable statement. The inheritance list +usually gives a list of base classes (see Metaclasses for more +advanced uses), so each item in the list should evaluate to a class +object which allows subclassing. Classes without an inheritance list +inherit, by default, from the base class "object"; hence, + + class Foo: + pass + +is equivalent to + + class Foo(object): + pass + +The class’s suite is then executed in a new execution frame (see +Naming and binding), using a newly created local namespace and the +original global namespace. (Usually, the suite contains mostly +function definitions.) When the class’s suite finishes execution, its +execution frame is discarded but its local namespace is saved. [5] A +class object is then created using the inheritance list for the base +classes and the saved local namespace for the attribute dictionary. +The class name is bound to this class object in the original local +namespace. + +The order in which attributes are defined in the class body is +preserved in the new class’s "__dict__". Note that this is reliable +only right after the class is created and only for classes that were +defined using the definition syntax. + +Class creation can be customized heavily using metaclasses. + +Classes can also be decorated: just like when decorating functions, + + @f1(arg) + @f2 + class Foo: pass + +is roughly equivalent to + + class Foo: pass + Foo = f1(arg)(f2(Foo)) + +The evaluation rules for the decorator expressions are the same as for +function decorators. The result is then bound to the class name. + +Changed in version 3.9: Classes may be decorated with any valid +"assignment_expression". Previously, the grammar was much more +restrictive; see **PEP 614** for details. + +A list of type parameters may be given in square brackets immediately +after the class’s name. This indicates to static type checkers that +the class is generic. At runtime, the type parameters can be retrieved +from the class’s "__type_params__" attribute. See Generic classes for +more. + +Changed in version 3.12: Type parameter lists are new in Python 3.12. + +**Programmer’s note:** Variables defined in the class definition are +class attributes; they are shared by instances. Instance attributes +can be set in a method with "self.name = value". Both class and +instance attributes are accessible through the notation “"self.name"”, +and an instance attribute hides a class attribute with the same name +when accessed in this way. Class attributes can be used as defaults +for instance attributes, but using mutable values there can lead to +unexpected results. Descriptors can be used to create instance +variables with different implementation details. + +See also: + + **PEP 3115** - Metaclasses in Python 3000 + The proposal that changed the declaration of metaclasses to the + current syntax, and the semantics for how classes with + metaclasses are constructed. + + **PEP 3129** - Class Decorators + The proposal that added class decorators. Function and method + decorators were introduced in **PEP 318**. + + +Coroutines +========== + +Added in version 3.5. + + +Coroutine function definition +----------------------------- + + async_funcdef ::= [decorators] "async" "def" funcname "(" [parameter_list] ")" + ["->" expression] ":" suite + +Execution of Python coroutines can be suspended and resumed at many +points (see *coroutine*). "await" expressions, "async for" and "async +with" can only be used in the body of a coroutine function. + +Functions defined with "async def" syntax are always coroutine +functions, even if they do not contain "await" or "async" keywords. + +It is a "SyntaxError" to use a "yield from" expression inside the body +of a coroutine function. + +An example of a coroutine function: + + async def func(param1, param2): + do_stuff() + await some_coroutine() + +Changed in version 3.7: "await" and "async" are now keywords; +previously they were only treated as such inside the body of a +coroutine function. + + +The "async for" statement +------------------------- + + async_for_stmt ::= "async" for_stmt + +An *asynchronous iterable* provides an "__aiter__" method that +directly returns an *asynchronous iterator*, which can call +asynchronous code in its "__anext__" method. + +The "async for" statement allows convenient iteration over +asynchronous iterables. + +The following code: + + async for TARGET in ITER: + SUITE + else: + SUITE2 + +Is semantically equivalent to: + + iter = (ITER) + iter = type(iter).__aiter__(iter) + running = True + + while running: + try: + TARGET = await type(iter).__anext__(iter) + except StopAsyncIteration: + running = False + else: + SUITE + else: + SUITE2 + +See also "__aiter__()" and "__anext__()" for details. + +It is a "SyntaxError" to use an "async for" statement outside the body +of a coroutine function. + + +The "async with" statement +-------------------------- + + async_with_stmt ::= "async" with_stmt + +An *asynchronous context manager* is a *context manager* that is able +to suspend execution in its *enter* and *exit* methods. + +The following code: + + async with EXPRESSION as TARGET: + SUITE + +is semantically equivalent to: + + manager = (EXPRESSION) + aenter = type(manager).__aenter__ + aexit = type(manager).__aexit__ + value = await aenter(manager) + hit_except = False + + try: + TARGET = value + SUITE + except: + hit_except = True + if not await aexit(manager, *sys.exc_info()): + raise + finally: + if not hit_except: + await aexit(manager, None, None, None) + +See also "__aenter__()" and "__aexit__()" for details. + +It is a "SyntaxError" to use an "async with" statement outside the +body of a coroutine function. + +See also: + + **PEP 492** - Coroutines with async and await syntax + The proposal that made coroutines a proper standalone concept in + Python, and added supporting syntax. + + +Type parameter lists +==================== + +Added in version 3.12. + +Changed in version 3.13: Support for default values was added (see +**PEP 696**). + + type_params ::= "[" type_param ("," type_param)* "]" + type_param ::= typevar | typevartuple | paramspec + typevar ::= identifier (":" expression)? ("=" expression)? + typevartuple ::= "*" identifier ("=" expression)? + paramspec ::= "**" identifier ("=" expression)? + +Functions (including coroutines), classes and type aliases may contain +a type parameter list: + + def max[T](args: list[T]) -> T: + ... + + async def amax[T](args: list[T]) -> T: + ... + + class Bag[T]: + def __iter__(self) -> Iterator[T]: + ... + + def add(self, arg: T) -> None: + ... + + type ListOrSet[T] = list[T] | set[T] + +Semantically, this indicates that the function, class, or type alias +is generic over a type variable. This information is primarily used by +static type checkers, and at runtime, generic objects behave much like +their non-generic counterparts. + +Type parameters are declared in square brackets ("[]") immediately +after the name of the function, class, or type alias. The type +parameters are accessible within the scope of the generic object, but +not elsewhere. Thus, after a declaration "def func[T](): pass", the +name "T" is not available in the module scope. Below, the semantics of +generic objects are described with more precision. The scope of type +parameters is modeled with a special function (technically, an +annotation scope) that wraps the creation of the generic object. + +Generic functions, classes, and type aliases have a "__type_params__" +attribute listing their type parameters. + +Type parameters come in three kinds: + +* "typing.TypeVar", introduced by a plain name (e.g., "T"). + Semantically, this represents a single type to a type checker. + +* "typing.TypeVarTuple", introduced by a name prefixed with a single + asterisk (e.g., "*Ts"). Semantically, this stands for a tuple of any + number of types. + +* "typing.ParamSpec", introduced by a name prefixed with two asterisks + (e.g., "**P"). Semantically, this stands for the parameters of a + callable. + +"typing.TypeVar" declarations can define *bounds* and *constraints* +with a colon (":") followed by an expression. A single expression +after the colon indicates a bound (e.g. "T: int"). Semantically, this +means that the "typing.TypeVar" can only represent types that are a +subtype of this bound. A parenthesized tuple of expressions after the +colon indicates a set of constraints (e.g. "T: (str, bytes)"). Each +member of the tuple should be a type (again, this is not enforced at +runtime). Constrained type variables can only take on one of the types +in the list of constraints. + +For "typing.TypeVar"s declared using the type parameter list syntax, +the bound and constraints are not evaluated when the generic object is +created, but only when the value is explicitly accessed through the +attributes "__bound__" and "__constraints__". To accomplish this, the +bounds or constraints are evaluated in a separate annotation scope. + +"typing.TypeVarTuple"s and "typing.ParamSpec"s cannot have bounds or +constraints. + +All three flavors of type parameters can also have a *default value*, +which is used when the type parameter is not explicitly provided. This +is added by appending a single equals sign ("=") followed by an +expression. Like the bounds and constraints of type variables, the +default value is not evaluated when the object is created, but only +when the type parameter’s "__default__" attribute is accessed. To this +end, the default value is evaluated in a separate annotation scope. If +no default value is specified for a type parameter, the "__default__" +attribute is set to the special sentinel object "typing.NoDefault". + +The following example indicates the full set of allowed type parameter +declarations: + + def overly_generic[ + SimpleTypeVar, + TypeVarWithDefault = int, + TypeVarWithBound: int, + TypeVarWithConstraints: (str, bytes), + *SimpleTypeVarTuple = (int, float), + **SimpleParamSpec = (str, bytearray), + ]( + a: SimpleTypeVar, + b: TypeVarWithDefault, + c: TypeVarWithBound, + d: Callable[SimpleParamSpec, TypeVarWithConstraints], + *e: SimpleTypeVarTuple, + ): ... + + +Generic functions +----------------- + +Generic functions are declared as follows: + + def func[T](arg: T): ... + +This syntax is equivalent to: + + annotation-def TYPE_PARAMS_OF_func(): + T = typing.TypeVar("T") + def func(arg: T): ... + func.__type_params__ = (T,) + return func + func = TYPE_PARAMS_OF_func() + +Here "annotation-def" indicates an annotation scope, which is not +actually bound to any name at runtime. (One other liberty is taken in +the translation: the syntax does not go through attribute access on +the "typing" module, but creates an instance of "typing.TypeVar" +directly.) + +The annotations of generic functions are evaluated within the +annotation scope used for declaring the type parameters, but the +function’s defaults and decorators are not. + +The following example illustrates the scoping rules for these cases, +as well as for additional flavors of type parameters: + + @decorator + def func[T: int, *Ts, **P](*args: *Ts, arg: Callable[P, T] = some_default): + ... + +Except for the lazy evaluation of the "TypeVar" bound, this is +equivalent to: + + DEFAULT_OF_arg = some_default + + annotation-def TYPE_PARAMS_OF_func(): + + annotation-def BOUND_OF_T(): + return int + # In reality, BOUND_OF_T() is evaluated only on demand. + T = typing.TypeVar("T", bound=BOUND_OF_T()) + + Ts = typing.TypeVarTuple("Ts") + P = typing.ParamSpec("P") + + def func(*args: *Ts, arg: Callable[P, T] = DEFAULT_OF_arg): + ... + + func.__type_params__ = (T, Ts, P) + return func + func = decorator(TYPE_PARAMS_OF_func()) + +The capitalized names like "DEFAULT_OF_arg" are not actually bound at +runtime. + + +Generic classes +--------------- + +Generic classes are declared as follows: + + class Bag[T]: ... + +This syntax is equivalent to: + + annotation-def TYPE_PARAMS_OF_Bag(): + T = typing.TypeVar("T") + class Bag(typing.Generic[T]): + __type_params__ = (T,) + ... + return Bag + Bag = TYPE_PARAMS_OF_Bag() + +Here again "annotation-def" (not a real keyword) indicates an +annotation scope, and the name "TYPE_PARAMS_OF_Bag" is not actually +bound at runtime. + +Generic classes implicitly inherit from "typing.Generic". The base +classes and keyword arguments of generic classes are evaluated within +the type scope for the type parameters, and decorators are evaluated +outside that scope. This is illustrated by this example: + + @decorator + class Bag(Base[T], arg=T): ... + +This is equivalent to: + + annotation-def TYPE_PARAMS_OF_Bag(): + T = typing.TypeVar("T") + class Bag(Base[T], typing.Generic[T], arg=T): + __type_params__ = (T,) + ... + return Bag + Bag = decorator(TYPE_PARAMS_OF_Bag()) + + +Generic type aliases +-------------------- + +The "type" statement can also be used to create a generic type alias: + + type ListOrSet[T] = list[T] | set[T] + +Except for the lazy evaluation of the value, this is equivalent to: + + annotation-def TYPE_PARAMS_OF_ListOrSet(): + T = typing.TypeVar("T") + + annotation-def VALUE_OF_ListOrSet(): + return list[T] | set[T] + # In reality, the value is lazily evaluated + return typing.TypeAliasType("ListOrSet", VALUE_OF_ListOrSet(), type_params=(T,)) + ListOrSet = TYPE_PARAMS_OF_ListOrSet() + +Here, "annotation-def" (not a real keyword) indicates an annotation +scope. The capitalized names like "TYPE_PARAMS_OF_ListOrSet" are not +actually bound at runtime. + + +Annotations +=========== + +Changed in version 3.14: Annotations are now lazily evaluated by +default. + +Variables and function parameters may carry *annotations*, created by +adding a colon after the name, followed by an expression: + + x: annotation = 1 + def f(param: annotation): ... + +Functions may also carry a return annotation following an arrow: + + def f() -> annotation: ... + +Annotations are conventionally used for *type hints*, but this is not +enforced by the language, and in general annotations may contain +arbitrary expressions. The presence of annotations does not change the +runtime semantics of the code, except if some mechanism is used that +introspects and uses the annotations (such as "dataclasses" or +"functools.singledispatch()"). + +By default, annotations are lazily evaluated in a annotation scope. +This means that they are not evaluated when the code containing the +annotation is evaluated. Instead, the interpreter saves information +that can be used to evaluate the annotation later if requested. The +"annotationlib" module provides tools for evaluating annotations. + +If the future statement "from __future__ import annotations" is +present, all annotations are instead stored as strings: + + >>> from __future__ import annotations + >>> def f(param: annotation): ... + >>> f.__annotations__ + {'param': 'annotation'} + +-[ Footnotes ]- + +[1] The exception is propagated to the invocation stack unless there + is a "finally" clause which happens to raise another exception. + That new exception causes the old one to be lost. + +[2] In pattern matching, a sequence is defined as one of the + following: + + * a class that inherits from "collections.abc.Sequence" + + * a Python class that has been registered as + "collections.abc.Sequence" + + * a builtin class that has its (CPython) "Py_TPFLAGS_SEQUENCE" bit + set + + * a class that inherits from any of the above + + The following standard library classes are sequences: + + * "array.array" + + * "collections.deque" + + * "list" + + * "memoryview" + + * "range" + + * "tuple" + + Note: + + Subject values of type "str", "bytes", and "bytearray" do not + match sequence patterns. + +[3] In pattern matching, a mapping is defined as one of the following: + + * a class that inherits from "collections.abc.Mapping" + + * a Python class that has been registered as + "collections.abc.Mapping" + + * a builtin class that has its (CPython) "Py_TPFLAGS_MAPPING" bit + set + + * a class that inherits from any of the above + + The standard library classes "dict" and "types.MappingProxyType" + are mappings. + +[4] A string literal appearing as the first statement in the function + body is transformed into the function’s "__doc__" attribute and + therefore the function’s *docstring*. + +[5] A string literal appearing as the first statement in the class + body is transformed into the namespace’s "__doc__" item and + therefore the class’s *docstring*. +''', + 'context-managers': r'''With Statement Context Managers +******************************* + +A *context manager* is an object that defines the runtime context to +be established when executing a "with" statement. The context manager +handles the entry into, and the exit from, the desired runtime context +for the execution of the block of code. Context managers are normally +invoked using the "with" statement (described in section The with +statement), but can also be used by directly invoking their methods. + +Typical uses of context managers include saving and restoring various +kinds of global state, locking and unlocking resources, closing opened +files, etc. + +For more information on context managers, see Context Manager Types. +The "object" class itself does not provide the context manager +methods. + +object.__enter__(self) + + Enter the runtime context related to this object. The "with" + statement will bind this method’s return value to the target(s) + specified in the "as" clause of the statement, if any. + +object.__exit__(self, exc_type, exc_value, traceback) + + Exit the runtime context related to this object. The parameters + describe the exception that caused the context to be exited. If the + context was exited without an exception, all three arguments will + be "None". + + If an exception is supplied, and the method wishes to suppress the + exception (i.e., prevent it from being propagated), it should + return a true value. Otherwise, the exception will be processed + normally upon exit from this method. + + Note that "__exit__()" methods should not reraise the passed-in + exception; this is the caller’s responsibility. + +See also: + + **PEP 343** - The “with” statement + The specification, background, and examples for the Python "with" + statement. +''', + 'continue': r'''The "continue" statement +************************ + + continue_stmt ::= "continue" + +"continue" may only occur syntactically nested in a "for" or "while" +loop, but not nested in a function or class definition within that +loop. It continues with the next cycle of the nearest enclosing loop. + +When "continue" passes control out of a "try" statement with a +"finally" clause, that "finally" clause is executed before really +starting the next loop cycle. +''', + 'conversions': r'''Arithmetic conversions +********************** + +When a description of an arithmetic operator below uses the phrase +“the numeric arguments are converted to a common real type”, this +means that the operator implementation for built-in types works as +follows: + +* If both arguments are complex numbers, no conversion is performed; + +* if either argument is a complex or a floating-point number, the + other is converted to a floating-point number; + +* otherwise, both must be integers and no conversion is necessary. + +Some additional rules apply for certain operators (e.g., a string as a +left argument to the ‘%’ operator). Extensions must define their own +conversion behavior. +''', + 'customization': r'''Basic customization +******************* + +object.__new__(cls[, ...]) + + Called to create a new instance of class *cls*. "__new__()" is a + static method (special-cased so you need not declare it as such) + that takes the class of which an instance was requested as its + first argument. The remaining arguments are those passed to the + object constructor expression (the call to the class). The return + value of "__new__()" should be the new object instance (usually an + instance of *cls*). + + Typical implementations create a new instance of the class by + invoking the superclass’s "__new__()" method using + "super().__new__(cls[, ...])" with appropriate arguments and then + modifying the newly created instance as necessary before returning + it. + + If "__new__()" is invoked during object construction and it returns + an instance of *cls*, then the new instance’s "__init__()" method + will be invoked like "__init__(self[, ...])", where *self* is the + new instance and the remaining arguments are the same as were + passed to the object constructor. + + If "__new__()" does not return an instance of *cls*, then the new + instance’s "__init__()" method will not be invoked. + + "__new__()" is intended mainly to allow subclasses of immutable + types (like int, str, or tuple) to customize instance creation. It + is also commonly overridden in custom metaclasses in order to + customize class creation. + +object.__init__(self[, ...]) + + Called after the instance has been created (by "__new__()"), but + before it is returned to the caller. The arguments are those + passed to the class constructor expression. If a base class has an + "__init__()" method, the derived class’s "__init__()" method, if + any, must explicitly call it to ensure proper initialization of the + base class part of the instance; for example: + "super().__init__([args...])". + + Because "__new__()" and "__init__()" work together in constructing + objects ("__new__()" to create it, and "__init__()" to customize + it), no non-"None" value may be returned by "__init__()"; doing so + will cause a "TypeError" to be raised at runtime. + +object.__del__(self) + + Called when the instance is about to be destroyed. This is also + called a finalizer or (improperly) a destructor. If a base class + has a "__del__()" method, the derived class’s "__del__()" method, + if any, must explicitly call it to ensure proper deletion of the + base class part of the instance. + + It is possible (though not recommended!) for the "__del__()" method + to postpone destruction of the instance by creating a new reference + to it. This is called object *resurrection*. It is + implementation-dependent whether "__del__()" is called a second + time when a resurrected object is about to be destroyed; the + current *CPython* implementation only calls it once. + + It is not guaranteed that "__del__()" methods are called for + objects that still exist when the interpreter exits. + "weakref.finalize" provides a straightforward way to register a + cleanup function to be called when an object is garbage collected. + + Note: + + "del x" doesn’t directly call "x.__del__()" — the former + decrements the reference count for "x" by one, and the latter is + only called when "x"’s reference count reaches zero. + + **CPython implementation detail:** It is possible for a reference + cycle to prevent the reference count of an object from going to + zero. In this case, the cycle will be later detected and deleted + by the *cyclic garbage collector*. A common cause of reference + cycles is when an exception has been caught in a local variable. + The frame’s locals then reference the exception, which references + its own traceback, which references the locals of all frames caught + in the traceback. + + See also: Documentation for the "gc" module. + + Warning: + + Due to the precarious circumstances under which "__del__()" + methods are invoked, exceptions that occur during their execution + are ignored, and a warning is printed to "sys.stderr" instead. + In particular: + + * "__del__()" can be invoked when arbitrary code is being + executed, including from any arbitrary thread. If "__del__()" + needs to take a lock or invoke any other blocking resource, it + may deadlock as the resource may already be taken by the code + that gets interrupted to execute "__del__()". + + * "__del__()" can be executed during interpreter shutdown. As a + consequence, the global variables it needs to access (including + other modules) may already have been deleted or set to "None". + Python guarantees that globals whose name begins with a single + underscore are deleted from their module before other globals + are deleted; if no other references to such globals exist, this + may help in assuring that imported modules are still available + at the time when the "__del__()" method is called. + +object.__repr__(self) + + Called by the "repr()" built-in function to compute the “official” + string representation of an object. If at all possible, this + should look like a valid Python expression that could be used to + recreate an object with the same value (given an appropriate + environment). If this is not possible, a string of the form + "<...some useful description...>" should be returned. The return + value must be a string object. If a class defines "__repr__()" but + not "__str__()", then "__repr__()" is also used when an “informal” + string representation of instances of that class is required. + + This is typically used for debugging, so it is important that the + representation is information-rich and unambiguous. A default + implementation is provided by the "object" class itself. + +object.__str__(self) + + Called by "str(object)", the default "__format__()" implementation, + and the built-in function "print()", to compute the “informal” or + nicely printable string representation of an object. The return + value must be a str object. + + This method differs from "object.__repr__()" in that there is no + expectation that "__str__()" return a valid Python expression: a + more convenient or concise representation can be used. + + The default implementation defined by the built-in type "object" + calls "object.__repr__()". + +object.__bytes__(self) + + Called by bytes to compute a byte-string representation of an + object. This should return a "bytes" object. The "object" class + itself does not provide this method. + +object.__format__(self, format_spec) + + Called by the "format()" built-in function, and by extension, + evaluation of formatted string literals and the "str.format()" + method, to produce a “formatted” string representation of an + object. The *format_spec* argument is a string that contains a + description of the formatting options desired. The interpretation + of the *format_spec* argument is up to the type implementing + "__format__()", however most classes will either delegate + formatting to one of the built-in types, or use a similar + formatting option syntax. + + See Format Specification Mini-Language for a description of the + standard formatting syntax. + + The return value must be a string object. + + The default implementation by the "object" class should be given an + empty *format_spec* string. It delegates to "__str__()". + + Changed in version 3.4: The __format__ method of "object" itself + raises a "TypeError" if passed any non-empty string. + + Changed in version 3.7: "object.__format__(x, '')" is now + equivalent to "str(x)" rather than "format(str(x), '')". + +object.__lt__(self, other) +object.__le__(self, other) +object.__eq__(self, other) +object.__ne__(self, other) +object.__gt__(self, other) +object.__ge__(self, other) + + These are the so-called “rich comparison” methods. The + correspondence between operator symbols and method names is as + follows: "xy" calls + "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)". + + A rich comparison method may return the singleton "NotImplemented" + if it does not implement the operation for a given pair of + arguments. By convention, "False" and "True" are returned for a + successful comparison. However, these methods can return any value, + so if the comparison operator is used in a Boolean context (e.g., + in the condition of an "if" statement), Python will call "bool()" + on the value to determine if the result is true or false. + + By default, "object" implements "__eq__()" by using "is", returning + "NotImplemented" in the case of a false comparison: "True if x is y + else NotImplemented". For "__ne__()", by default it delegates to + "__eq__()" and inverts the result unless it is "NotImplemented". + There are no other implied relationships among the comparison + operators or default implementations; for example, the truth of + "(x.__hash__". + + If a class that does not override "__eq__()" wishes to suppress + hash support, it should include "__hash__ = None" in the class + definition. A class which defines its own "__hash__()" that + explicitly raises a "TypeError" would be incorrectly identified as + hashable by an "isinstance(obj, collections.abc.Hashable)" call. + + Note: + + By default, the "__hash__()" values of str and bytes objects are + “salted” with an unpredictable random value. Although they + remain constant within an individual Python process, they are not + predictable between repeated invocations of Python.This is + intended to provide protection against a denial-of-service caused + by carefully chosen inputs that exploit the worst case + performance of a dict insertion, *O*(*n*^2) complexity. See + http://ocert.org/advisories/ocert-2011-003.html for + details.Changing hash values affects the iteration order of sets. + Python has never made guarantees about this ordering (and it + typically varies between 32-bit and 64-bit builds).See also + "PYTHONHASHSEED". + + Changed in version 3.3: Hash randomization is enabled by default. + +object.__bool__(self) + + Called to implement truth value testing and the built-in operation + "bool()"; should return "False" or "True". When this method is not + defined, "__len__()" is called, if it is defined, and the object is + considered true if its result is nonzero. If a class defines + neither "__len__()" nor "__bool__()" (which is true of the "object" + class itself), all its instances are considered true. +''', + 'debugger': r'''"pdb" — The Python Debugger +*************************** + +**Source code:** Lib/pdb.py + +====================================================================== + +The module "pdb" defines an interactive source code debugger for +Python programs. It supports setting (conditional) breakpoints and +single stepping at the source line level, inspection of stack frames, +source code listing, and evaluation of arbitrary Python code in the +context of any stack frame. It also supports post-mortem debugging +and can be called under program control. + +The debugger is extensible – it is actually defined as the class +"Pdb". This is currently undocumented but easily understood by reading +the source. The extension interface uses the modules "bdb" and "cmd". + +See also: + + Module "faulthandler" + Used to dump Python tracebacks explicitly, on a fault, after a + timeout, or on a user signal. + + Module "traceback" + Standard interface to extract, format and print stack traces of + Python programs. + +The typical usage to break into the debugger is to insert: + + import pdb; pdb.set_trace() + +Or: + + breakpoint() + +at the location you want to break into the debugger, and then run the +program. You can then step through the code following this statement, +and continue running without the debugger using the "continue" +command. + +Changed in version 3.7: The built-in "breakpoint()", when called with +defaults, can be used instead of "import pdb; pdb.set_trace()". + + def double(x): + breakpoint() + return x * 2 + val = 3 + print(f"{val} * 2 is {double(val)}") + +The debugger’s prompt is "(Pdb)", which is the indicator that you are +in debug mode: + + > ...(2)double() + -> breakpoint() + (Pdb) p x + 3 + (Pdb) continue + 3 * 2 is 6 + +Changed in version 3.3: Tab-completion via the "readline" module is +available for commands and command arguments, e.g. the current global +and local names are offered as arguments of the "p" command. + +You can also invoke "pdb" from the command line to debug other +scripts. For example: + + python -m pdb myscript.py + +When invoked as a module, pdb will automatically enter post-mortem +debugging if the program being debugged exits abnormally. After post- +mortem debugging (or after normal exit of the program), pdb will +restart the program. Automatic restarting preserves pdb’s state (such +as breakpoints) and in most cases is more useful than quitting the +debugger upon program’s exit. + +Changed in version 3.2: Added the "-c" option to execute commands as +if given in a ".pdbrc" file; see Debugger Commands. + +Changed in version 3.7: Added the "-m" option to execute modules +similar to the way "python -m" does. As with a script, the debugger +will pause execution just before the first line of the module. + +Typical usage to execute a statement under control of the debugger is: + + >>> import pdb + >>> def f(x): + ... print(1 / x) + >>> pdb.run("f(2)") + > (1)() + (Pdb) continue + 0.5 + >>> + +The typical usage to inspect a crashed program is: + + >>> import pdb + >>> def f(x): + ... print(1 / x) + ... + >>> f(0) + Traceback (most recent call last): + File "", line 1, in + File "", line 2, in f + ZeroDivisionError: division by zero + >>> pdb.pm() + > (2)f() + (Pdb) p x + 0 + (Pdb) + +Changed in version 3.13: The implementation of **PEP 667** means that +name assignments made via "pdb" will immediately affect the active +scope, even when running inside an *optimized scope*. + +The module defines the following functions; each enters the debugger +in a slightly different way: + +pdb.run(statement, globals=None, locals=None) + + Execute the *statement* (given as a string or a code object) under + debugger control. The debugger prompt appears before any code is + executed; you can set breakpoints and type "continue", or you can + step through the statement using "step" or "next" (all these + commands are explained below). The optional *globals* and *locals* + arguments specify the environment in which the code is executed; by + default the dictionary of the module "__main__" is used. (See the + explanation of the built-in "exec()" or "eval()" functions.) + +pdb.runeval(expression, globals=None, locals=None) + + Evaluate the *expression* (given as a string or a code object) + under debugger control. When "runeval()" returns, it returns the + value of the *expression*. Otherwise this function is similar to + "run()". + +pdb.runcall(function, *args, **kwds) + + Call the *function* (a function or method object, not a string) + with the given arguments. When "runcall()" returns, it returns + whatever the function call returned. The debugger prompt appears + as soon as the function is entered. + +pdb.set_trace(*, header=None, commands=None) + + Enter the debugger at the calling stack frame. This is useful to + hard-code a breakpoint at a given point in a program, even if the + code is not otherwise being debugged (e.g. when an assertion + fails). If given, *header* is printed to the console just before + debugging begins. The *commands* argument, if given, is a list of + commands to execute when the debugger starts. + + Changed in version 3.7: The keyword-only argument *header*. + + Changed in version 3.13: "set_trace()" will enter the debugger + immediately, rather than on the next line of code to be executed. + + Added in version 3.14: The *commands* argument. + +pdb.post_mortem(t=None) + + Enter post-mortem debugging of the given exception or traceback + object. If no value is given, it uses the exception that is + currently being handled, or raises "ValueError" if there isn’t one. + + Changed in version 3.13: Support for exception objects was added. + +pdb.pm() + + Enter post-mortem debugging of the exception found in + "sys.last_exc". + +The "run*" functions and "set_trace()" are aliases for instantiating +the "Pdb" class and calling the method of the same name. If you want +to access further features, you have to do this yourself: + +class pdb.Pdb(completekey='tab', stdin=None, stdout=None, skip=None, nosigint=False, readrc=True, mode=None) + + "Pdb" is the debugger class. + + The *completekey*, *stdin* and *stdout* arguments are passed to the + underlying "cmd.Cmd" class; see the description there. + + The *skip* argument, if given, must be an iterable of glob-style + module name patterns. The debugger will not step into frames that + originate in a module that matches one of these patterns. [1] + + By default, Pdb sets a handler for the SIGINT signal (which is sent + when the user presses "Ctrl"-"C" on the console) when you give a + "continue" command. This allows you to break into the debugger + again by pressing "Ctrl"-"C". If you want Pdb not to touch the + SIGINT handler, set *nosigint* to true. + + The *readrc* argument defaults to true and controls whether Pdb + will load .pdbrc files from the filesystem. + + The *mode* argument specifies how the debugger was invoked. It + impacts the workings of some debugger commands. Valid values are + "'inline'" (used by the breakpoint() builtin), "'cli'" (used by the + command line invocation) or "None" (for backwards compatible + behaviour, as before the *mode* argument was added). + + Example call to enable tracing with *skip*: + + import pdb; pdb.Pdb(skip=['django.*']).set_trace() + + Raises an auditing event "pdb.Pdb" with no arguments. + + Changed in version 3.1: Added the *skip* parameter. + + Changed in version 3.2: Added the *nosigint* parameter. Previously, + a SIGINT handler was never set by Pdb. + + Changed in version 3.6: The *readrc* argument. + + Added in version 3.14: Added the *mode* argument. + + run(statement, globals=None, locals=None) + runeval(expression, globals=None, locals=None) + runcall(function, *args, **kwds) + set_trace() + + See the documentation for the functions explained above. + + +Debugger Commands +================= + +The commands recognized by the debugger are listed below. Most +commands can be abbreviated to one or two letters as indicated; e.g. +"h(elp)" means that either "h" or "help" can be used to enter the help +command (but not "he" or "hel", nor "H" or "Help" or "HELP"). +Arguments to commands must be separated by whitespace (spaces or +tabs). Optional arguments are enclosed in square brackets ("[]") in +the command syntax; the square brackets must not be typed. +Alternatives in the command syntax are separated by a vertical bar +("|"). + +Entering a blank line repeats the last command entered. Exception: if +the last command was a "list" command, the next 11 lines are listed. + +Commands that the debugger doesn’t recognize are assumed to be Python +statements and are executed in the context of the program being +debugged. Python statements can also be prefixed with an exclamation +point ("!"). This is a powerful way to inspect the program being +debugged; it is even possible to change a variable or call a function. +When an exception occurs in such a statement, the exception name is +printed but the debugger’s state is not changed. + +Changed in version 3.13: Expressions/Statements whose prefix is a pdb +command are now correctly identified and executed. + +The debugger supports aliases. Aliases can have parameters which +allows one a certain level of adaptability to the context under +examination. + +Multiple commands may be entered on a single line, separated by ";;". +(A single ";" is not used as it is the separator for multiple commands +in a line that is passed to the Python parser.) No intelligence is +applied to separating the commands; the input is split at the first +";;" pair, even if it is in the middle of a quoted string. A +workaround for strings with double semicolons is to use implicit +string concatenation "';'';'" or "";"";"". + +To set a temporary global variable, use a *convenience variable*. A +*convenience variable* is a variable whose name starts with "$". For +example, "$foo = 1" sets a global variable "$foo" which you can use in +the debugger session. The *convenience variables* are cleared when +the program resumes execution so it’s less likely to interfere with +your program compared to using normal variables like "foo = 1". + +There are three preset *convenience variables*: + +* "$_frame": the current frame you are debugging + +* "$_retval": the return value if the frame is returning + +* "$_exception": the exception if the frame is raising an exception + +Added in version 3.12: Added the *convenience variable* feature. + +If a file ".pdbrc" exists in the user’s home directory or in the +current directory, it is read with "'utf-8'" encoding and executed as +if it had been typed at the debugger prompt, with the exception that +empty lines and lines starting with "#" are ignored. This is +particularly useful for aliases. If both files exist, the one in the +home directory is read first and aliases defined there can be +overridden by the local file. + +Changed in version 3.2: ".pdbrc" can now contain commands that +continue debugging, such as "continue" or "next". Previously, these +commands had no effect. + +Changed in version 3.11: ".pdbrc" is now read with "'utf-8'" encoding. +Previously, it was read with the system locale encoding. + +h(elp) [command] + + Without argument, print the list of available commands. With a + *command* as argument, print help about that command. "help pdb" + displays the full documentation (the docstring of the "pdb" + module). Since the *command* argument must be an identifier, "help + exec" must be entered to get help on the "!" command. + +w(here) [count] + + Print a stack trace, with the most recent frame at the bottom. if + *count* is 0, print the current frame entry. If *count* is + negative, print the least recent - *count* frames. If *count* is + positive, print the most recent *count* frames. An arrow (">") + indicates the current frame, which determines the context of most + commands. + + Changed in version 3.14: *count* argument is added. + +d(own) [count] + + Move the current frame *count* (default one) levels down in the + stack trace (to a newer frame). + +u(p) [count] + + Move the current frame *count* (default one) levels up in the stack + trace (to an older frame). + +b(reak) [([filename:]lineno | function) [, condition]] + + With a *lineno* argument, set a break at line *lineno* in the + current file. The line number may be prefixed with a *filename* and + a colon, to specify a breakpoint in another file (possibly one that + hasn’t been loaded yet). The file is searched on "sys.path". + Acceptable forms of *filename* are "/abspath/to/file.py", + "relpath/file.py", "module" and "package.module". + + With a *function* argument, set a break at the first executable + statement within that function. *function* can be any expression + that evaluates to a function in the current namespace. + + If a second argument is present, it is an expression which must + evaluate to true before the breakpoint is honored. + + Without argument, list all breaks, including for each breakpoint, + the number of times that breakpoint has been hit, the current + ignore count, and the associated condition if any. + + Each breakpoint is assigned a number to which all the other + breakpoint commands refer. + +tbreak [([filename:]lineno | function) [, condition]] + + Temporary breakpoint, which is removed automatically when it is + first hit. The arguments are the same as for "break". + +cl(ear) [filename:lineno | bpnumber ...] + + With a *filename:lineno* argument, clear all the breakpoints at + this line. With a space separated list of breakpoint numbers, clear + those breakpoints. Without argument, clear all breaks (but first + ask confirmation). + +disable bpnumber [bpnumber ...] + + Disable the breakpoints given as a space separated list of + breakpoint numbers. Disabling a breakpoint means it cannot cause + the program to stop execution, but unlike clearing a breakpoint, it + remains in the list of breakpoints and can be (re-)enabled. + +enable bpnumber [bpnumber ...] + + Enable the breakpoints specified. + +ignore bpnumber [count] + + Set the ignore count for the given breakpoint number. If *count* + is omitted, the ignore count is set to 0. A breakpoint becomes + active when the ignore count is zero. When non-zero, the *count* + is decremented each time the breakpoint is reached and the + breakpoint is not disabled and any associated condition evaluates + to true. + +condition bpnumber [condition] + + Set a new *condition* for the breakpoint, an expression which must + evaluate to true before the breakpoint is honored. If *condition* + is absent, any existing condition is removed; i.e., the breakpoint + is made unconditional. + +commands [bpnumber] + + Specify a list of commands for breakpoint number *bpnumber*. The + commands themselves appear on the following lines. Type a line + containing just "end" to terminate the commands. An example: + + (Pdb) commands 1 + (com) p some_variable + (com) end + (Pdb) + + To remove all commands from a breakpoint, type "commands" and + follow it immediately with "end"; that is, give no commands. + + With no *bpnumber* argument, "commands" refers to the last + breakpoint set. + + You can use breakpoint commands to start your program up again. + Simply use the "continue" command, or "step", or any other command + that resumes execution. + + Specifying any command resuming execution (currently "continue", + "step", "next", "return", "until", "jump", "quit" and their + abbreviations) terminates the command list (as if that command was + immediately followed by end). This is because any time you resume + execution (even with a simple next or step), you may encounter + another breakpoint—which could have its own command list, leading + to ambiguities about which list to execute. + + If the list of commands contains the "silent" command, or a command + that resumes execution, then the breakpoint message containing + information about the frame is not displayed. + + Changed in version 3.14: Frame information will not be displayed if + a command that resumes execution is present in the command list. + +s(tep) + + Execute the current line, stop at the first possible occasion + (either in a function that is called or on the next line in the + current function). + +n(ext) + + Continue execution until the next line in the current function is + reached or it returns. (The difference between "next" and "step" + is that "step" stops inside a called function, while "next" + executes called functions at (nearly) full speed, only stopping at + the next line in the current function.) + +unt(il) [lineno] + + Without argument, continue execution until the line with a number + greater than the current one is reached. + + With *lineno*, continue execution until a line with a number + greater or equal to *lineno* is reached. In both cases, also stop + when the current frame returns. + + Changed in version 3.2: Allow giving an explicit line number. + +r(eturn) + + Continue execution until the current function returns. + +c(ont(inue)) + + Continue execution, only stop when a breakpoint is encountered. + +j(ump) lineno + + Set the next line that will be executed. Only available in the + bottom-most frame. This lets you jump back and execute code again, + or jump forward to skip code that you don’t want to run. + + It should be noted that not all jumps are allowed – for instance it + is not possible to jump into the middle of a "for" loop or out of a + "finally" clause. + +l(ist) [first[, last]] + + List source code for the current file. Without arguments, list 11 + lines around the current line or continue the previous listing. + With "." as argument, list 11 lines around the current line. With + one argument, list 11 lines around at that line. With two + arguments, list the given range; if the second argument is less + than the first, it is interpreted as a count. + + The current line in the current frame is indicated by "->". If an + exception is being debugged, the line where the exception was + originally raised or propagated is indicated by ">>", if it differs + from the current line. + + Changed in version 3.2: Added the ">>" marker. + +ll | longlist + + List all source code for the current function or frame. + Interesting lines are marked as for "list". + + Added in version 3.2. + +a(rgs) + + Print the arguments of the current function and their current + values. + +p expression + + Evaluate *expression* in the current context and print its value. + + Note: + + "print()" can also be used, but is not a debugger command — this + executes the Python "print()" function. + +pp expression + + Like the "p" command, except the value of *expression* is pretty- + printed using the "pprint" module. + +whatis expression + + Print the type of *expression*. + +source expression + + Try to get source code of *expression* and display it. + + Added in version 3.2. + +display [expression] + + Display the value of *expression* if it changed, each time + execution stops in the current frame. + + Without *expression*, list all display expressions for the current + frame. + + Note: + + Display evaluates *expression* and compares to the result of the + previous evaluation of *expression*, so when the result is + mutable, display may not be able to pick up the changes. + + Example: + + lst = [] + breakpoint() + pass + lst.append(1) + print(lst) + + Display won’t realize "lst" has been changed because the result of + evaluation is modified in place by "lst.append(1)" before being + compared: + + > example.py(3)() + -> pass + (Pdb) display lst + display lst: [] + (Pdb) n + > example.py(4)() + -> lst.append(1) + (Pdb) n + > example.py(5)() + -> print(lst) + (Pdb) + + You can do some tricks with copy mechanism to make it work: + + > example.py(3)() + -> pass + (Pdb) display lst[:] + display lst[:]: [] + (Pdb) n + > example.py(4)() + -> lst.append(1) + (Pdb) n + > example.py(5)() + -> print(lst) + display lst[:]: [1] [old: []] + (Pdb) + + Added in version 3.2. + +undisplay [expression] + + Do not display *expression* anymore in the current frame. Without + *expression*, clear all display expressions for the current frame. + + Added in version 3.2. + +interact + + Start an interactive interpreter (using the "code" module) in a new + global namespace initialised from the local and global namespaces + for the current scope. Use "exit()" or "quit()" to exit the + interpreter and return to the debugger. + + Note: + + As "interact" creates a new dedicated namespace for code + execution, assignments to variables will not affect the original + namespaces. However, modifications to any referenced mutable + objects will be reflected in the original namespaces as usual. + + Added in version 3.2. + + Changed in version 3.13: "exit()" and "quit()" can be used to exit + the "interact" command. + + Changed in version 3.13: "interact" directs its output to the + debugger’s output channel rather than "sys.stderr". + +alias [name [command]] + + Create an alias called *name* that executes *command*. The + *command* must *not* be enclosed in quotes. Replaceable parameters + can be indicated by "%1", "%2", … and "%9", while "%*" is replaced + by all the parameters. If *command* is omitted, the current alias + for *name* is shown. If no arguments are given, all aliases are + listed. + + Aliases may be nested and can contain anything that can be legally + typed at the pdb prompt. Note that internal pdb commands *can* be + overridden by aliases. Such a command is then hidden until the + alias is removed. Aliasing is recursively applied to the first + word of the command line; all other words in the line are left + alone. + + As an example, here are two useful aliases (especially when placed + in the ".pdbrc" file): + + # Print instance variables (usage "pi classInst") + alias pi for k in %1.__dict__.keys(): print(f"%1.{k} = {%1.__dict__[k]}") + # Print instance variables in self + alias ps pi self + +unalias name + + Delete the specified alias *name*. + +! statement + + Execute the (one-line) *statement* in the context of the current + stack frame. The exclamation point can be omitted unless the first + word of the statement resembles a debugger command, e.g.: + + (Pdb) ! n=42 + (Pdb) + + To set a global variable, you can prefix the assignment command + with a "global" statement on the same line, e.g.: + + (Pdb) global list_options; list_options = ['-l'] + (Pdb) + +run [args ...] +restart [args ...] + + Restart the debugged Python program. If *args* is supplied, it is + split with "shlex" and the result is used as the new "sys.argv". + History, breakpoints, actions and debugger options are preserved. + "restart" is an alias for "run". + + Changed in version 3.14: "run" and "restart" commands are disabled + when the debugger is invoked in "'inline'" mode. + +q(uit) + + Quit from the debugger. The program being executed is aborted. + +debug code + + Enter a recursive debugger that steps through *code* (which is an + arbitrary expression or statement to be executed in the current + environment). + +retval + + Print the return value for the last return of the current function. + +exceptions [excnumber] + + List or jump between chained exceptions. + + When using "pdb.pm()" or "Pdb.post_mortem(...)" with a chained + exception instead of a traceback, it allows the user to move + between the chained exceptions using "exceptions" command to list + exceptions, and "exception " to switch to that exception. + + Example: + + def out(): + try: + middle() + except Exception as e: + raise ValueError("reraise middle() error") from e + + def middle(): + try: + return inner(0) + except Exception as e: + raise ValueError("Middle fail") + + def inner(x): + 1 / x + + out() + + calling "pdb.pm()" will allow to move between exceptions: + + > example.py(5)out() + -> raise ValueError("reraise middle() error") from e + + (Pdb) exceptions + 0 ZeroDivisionError('division by zero') + 1 ValueError('Middle fail') + > 2 ValueError('reraise middle() error') + + (Pdb) exceptions 0 + > example.py(16)inner() + -> 1 / x + + (Pdb) up + > example.py(10)middle() + -> return inner(0) + + Added in version 3.13. + +-[ Footnotes ]- + +[1] Whether a frame is considered to originate in a certain module is + determined by the "__name__" in the frame globals. +''', + 'del': r'''The "del" statement +******************* + + del_stmt ::= "del" target_list + +Deletion is recursively defined very similar to the way assignment is +defined. Rather than spelling it out in full details, here are some +hints. + +Deletion of a target list recursively deletes each target, from left +to right. + +Deletion of a name removes the binding of that name from the local or +global namespace, depending on whether the name occurs in a "global" +statement in the same code block. If the name is unbound, a +"NameError" exception will be raised. + +Deletion of attribute references, subscriptions and slicings is passed +to the primary object involved; deletion of a slicing is in general +equivalent to assignment of an empty slice of the right type (but even +this is determined by the sliced object). + +Changed in version 3.2: Previously it was illegal to delete a name +from the local namespace if it occurs as a free variable in a nested +block. +''', + 'dict': r'''Dictionary displays +******************* + +A dictionary display is a possibly empty series of dict items +(key/value pairs) enclosed in curly braces: + + dict_display ::= "{" [dict_item_list | dict_comprehension] "}" + dict_item_list ::= dict_item ("," dict_item)* [","] + dict_item ::= expression ":" expression | "**" or_expr + dict_comprehension ::= expression ":" expression comp_for + +A dictionary display yields a new dictionary object. + +If a comma-separated sequence of dict items is given, they are +evaluated from left to right to define the entries of the dictionary: +each key object is used as a key into the dictionary to store the +corresponding value. This means that you can specify the same key +multiple times in the dict item list, and the final dictionary’s value +for that key will be the last one given. + +A double asterisk "**" denotes *dictionary unpacking*. Its operand +must be a *mapping*. Each mapping item is added to the new +dictionary. Later values replace values already set by earlier dict +items and earlier dictionary unpackings. + +Added in version 3.5: Unpacking into dictionary displays, originally +proposed by **PEP 448**. + +A dict comprehension, in contrast to list and set comprehensions, +needs two expressions separated with a colon followed by the usual +“for” and “if” clauses. When the comprehension is run, the resulting +key and value elements are inserted in the new dictionary in the order +they are produced. + +Restrictions on the types of the key values are listed earlier in +section The standard type hierarchy. (To summarize, the key type +should be *hashable*, which excludes all mutable objects.) Clashes +between duplicate keys are not detected; the last value (textually +rightmost in the display) stored for a given key value prevails. + +Changed in version 3.8: Prior to Python 3.8, in dict comprehensions, +the evaluation order of key and value was not well-defined. In +CPython, the value was evaluated before the key. Starting with 3.8, +the key is evaluated before the value, as proposed by **PEP 572**. +''', + 'dynamic-features': r'''Interaction with dynamic features +********************************* + +Name resolution of free variables occurs at runtime, not at compile +time. This means that the following code will print 42: + + i = 10 + def f(): + print(i) + i = 42 + f() + +The "eval()" and "exec()" functions do not have access to the full +environment for resolving names. Names may be resolved in the local +and global namespaces of the caller. Free variables are not resolved +in the nearest enclosing namespace, but in the global namespace. [1] +The "exec()" and "eval()" functions have optional arguments to +override the global and local namespace. If only one namespace is +specified, it is used for both. +''', + 'else': r'''The "if" statement +****************** + +The "if" statement is used for conditional execution: + + if_stmt ::= "if" assignment_expression ":" suite + ("elif" assignment_expression ":" suite)* + ["else" ":" suite] + +It selects exactly one of the suites by evaluating the expressions one +by one until one is found to be true (see section Boolean operations +for the definition of true and false); then that suite is executed +(and no other part of the "if" statement is executed or evaluated). +If all expressions are false, the suite of the "else" clause, if +present, is executed. +''', + 'exceptions': r'''Exceptions +********** + +Exceptions are a means of breaking out of the normal flow of control +of a code block in order to handle errors or other exceptional +conditions. An exception is *raised* at the point where the error is +detected; it may be *handled* by the surrounding code block or by any +code block that directly or indirectly invoked the code block where +the error occurred. + +The Python interpreter raises an exception when it detects a run-time +error (such as division by zero). A Python program can also +explicitly raise an exception with the "raise" statement. Exception +handlers are specified with the "try" … "except" statement. The +"finally" clause of such a statement can be used to specify cleanup +code which does not handle the exception, but is executed whether an +exception occurred or not in the preceding code. + +Python uses the “termination” model of error handling: an exception +handler can find out what happened and continue execution at an outer +level, but it cannot repair the cause of the error and retry the +failing operation (except by re-entering the offending piece of code +from the top). + +When an exception is not handled at all, the interpreter terminates +execution of the program, or returns to its interactive main loop. In +either case, it prints a stack traceback, except when the exception is +"SystemExit". + +Exceptions are identified by class instances. The "except" clause is +selected depending on the class of the instance: it must reference the +class of the instance or a *non-virtual base class* thereof. The +instance can be received by the handler and can carry additional +information about the exceptional condition. + +Note: + + Exception messages are not part of the Python API. Their contents + may change from one version of Python to the next without warning + and should not be relied on by code which will run under multiple + versions of the interpreter. + +See also the description of the "try" statement in section The try +statement and "raise" statement in section The raise statement. + +-[ Footnotes ]- + +[1] This limitation occurs because the code that is executed by these + operations is not available at the time the module is compiled. +''', + 'execmodel': r'''Execution model +*************** + + +Structure of a program +====================== + +A Python program is constructed from code blocks. A *block* is a piece +of Python program text that is executed as a unit. The following are +blocks: a module, a function body, and a class definition. Each +command typed interactively is a block. A script file (a file given +as standard input to the interpreter or specified as a command line +argument to the interpreter) is a code block. A script command (a +command specified on the interpreter command line with the "-c" +option) is a code block. A module run as a top level script (as module +"__main__") from the command line using a "-m" argument is also a code +block. The string argument passed to the built-in functions "eval()" +and "exec()" is a code block. + +A code block is executed in an *execution frame*. A frame contains +some administrative information (used for debugging) and determines +where and how execution continues after the code block’s execution has +completed. + + +Naming and binding +================== + + +Binding of names +---------------- + +*Names* refer to objects. Names are introduced by name binding +operations. + +The following constructs bind names: + +* formal parameters to functions, + +* class definitions, + +* function definitions, + +* assignment expressions, + +* targets that are identifiers if occurring in an assignment: + + * "for" loop header, + + * after "as" in a "with" statement, "except" clause, "except*" + clause, or in the as-pattern in structural pattern matching, + + * in a capture pattern in structural pattern matching + +* "import" statements. + +* "type" statements. + +* type parameter lists. + +The "import" statement of the form "from ... import *" binds all names +defined in the imported module, except those beginning with an +underscore. This form may only be used at the module level. + +A target occurring in a "del" statement is also considered bound for +this purpose (though the actual semantics are to unbind the name). + +Each assignment or import statement occurs within a block defined by a +class or function definition or at the module level (the top-level +code block). + +If a name is bound in a block, it is a local variable of that block, +unless declared as "nonlocal" or "global". If a name is bound at the +module level, it is a global variable. (The variables of the module +code block are local and global.) If a variable is used in a code +block but not defined there, it is a *free variable*. + +Each occurrence of a name in the program text refers to the *binding* +of that name established by the following name resolution rules. + + +Resolution of names +------------------- + +A *scope* defines the visibility of a name within a block. If a local +variable is defined in a block, its scope includes that block. If the +definition occurs in a function block, the scope extends to any blocks +contained within the defining one, unless a contained block introduces +a different binding for the name. + +When a name is used in a code block, it is resolved using the nearest +enclosing scope. The set of all such scopes visible to a code block +is called the block’s *environment*. + +When a name is not found at all, a "NameError" exception is raised. If +the current scope is a function scope, and the name refers to a local +variable that has not yet been bound to a value at the point where the +name is used, an "UnboundLocalError" exception is raised. +"UnboundLocalError" is a subclass of "NameError". + +If a name binding operation occurs anywhere within a code block, all +uses of the name within the block are treated as references to the +current block. This can lead to errors when a name is used within a +block before it is bound. This rule is subtle. Python lacks +declarations and allows name binding operations to occur anywhere +within a code block. The local variables of a code block can be +determined by scanning the entire text of the block for name binding +operations. See the FAQ entry on UnboundLocalError for examples. + +If the "global" statement occurs within a block, all uses of the names +specified in the statement refer to the bindings of those names in the +top-level namespace. Names are resolved in the top-level namespace by +searching the global namespace, i.e. the namespace of the module +containing the code block, and the builtins namespace, the namespace +of the module "builtins". The global namespace is searched first. If +the names are not found there, the builtins namespace is searched +next. If the names are also not found in the builtins namespace, new +variables are created in the global namespace. The global statement +must precede all uses of the listed names. + +The "global" statement has the same scope as a name binding operation +in the same block. If the nearest enclosing scope for a free variable +contains a global statement, the free variable is treated as a global. + +The "nonlocal" statement causes corresponding names to refer to +previously bound variables in the nearest enclosing function scope. +"SyntaxError" is raised at compile time if the given name does not +exist in any enclosing function scope. Type parameters cannot be +rebound with the "nonlocal" statement. + +The namespace for a module is automatically created the first time a +module is imported. The main module for a script is always called +"__main__". + +Class definition blocks and arguments to "exec()" and "eval()" are +special in the context of name resolution. A class definition is an +executable statement that may use and define names. These references +follow the normal rules for name resolution with an exception that +unbound local variables are looked up in the global namespace. The +namespace of the class definition becomes the attribute dictionary of +the class. The scope of names defined in a class block is limited to +the class block; it does not extend to the code blocks of methods. +This includes comprehensions and generator expressions, but it does +not include annotation scopes, which have access to their enclosing +class scopes. This means that the following will fail: + + class A: + a = 42 + b = list(a + i for i in range(10)) + +However, the following will succeed: + + class A: + type Alias = Nested + class Nested: pass + + print(A.Alias.__value__) # + + +Annotation scopes +----------------- + +*Annotations*, type parameter lists and "type" statements introduce +*annotation scopes*, which behave mostly like function scopes, but +with some exceptions discussed below. + +Annotation scopes are used in the following contexts: + +* *Function annotations*. + +* *Variable annotations*. + +* Type parameter lists for generic type aliases. + +* Type parameter lists for generic functions. A generic function’s + annotations are executed within the annotation scope, but its + defaults and decorators are not. + +* Type parameter lists for generic classes. A generic class’s base + classes and keyword arguments are executed within the annotation + scope, but its decorators are not. + +* The bounds, constraints, and default values for type parameters + (lazily evaluated). + +* The value of type aliases (lazily evaluated). + +Annotation scopes differ from function scopes in the following ways: + +* Annotation scopes have access to their enclosing class namespace. If + an annotation scope is immediately within a class scope, or within + another annotation scope that is immediately within a class scope, + the code in the annotation scope can use names defined in the class + scope as if it were executed directly within the class body. This + contrasts with regular functions defined within classes, which + cannot access names defined in the class scope. + +* Expressions in annotation scopes cannot contain "yield", "yield + from", "await", or ":=" expressions. (These expressions are allowed + in other scopes contained within the annotation scope.) + +* Names defined in annotation scopes cannot be rebound with "nonlocal" + statements in inner scopes. This includes only type parameters, as + no other syntactic elements that can appear within annotation scopes + can introduce new names. + +* While annotation scopes have an internal name, that name is not + reflected in the *qualified name* of objects defined within the + scope. Instead, the "__qualname__" of such objects is as if the + object were defined in the enclosing scope. + +Added in version 3.12: Annotation scopes were introduced in Python +3.12 as part of **PEP 695**. + +Changed in version 3.13: Annotation scopes are also used for type +parameter defaults, as introduced by **PEP 696**. + +Changed in version 3.14: Annotation scopes are now also used for +annotations, as specified in **PEP 649** and **PEP 749**. + + +Lazy evaluation +--------------- + +Most annotation scopes are *lazily evaluated*. This includes +annotations, the values of type aliases created through the "type" +statement, and the bounds, constraints, and default values of type +variables created through the type parameter syntax. This means that +they are not evaluated when the type alias or type variable is +created, or when the object carrying annotations is created. Instead, +they are only evaluated when necessary, for example when the +"__value__" attribute on a type alias is accessed. + +Example: + + >>> type Alias = 1/0 + >>> Alias.__value__ + Traceback (most recent call last): + ... + ZeroDivisionError: division by zero + >>> def func[T: 1/0](): pass + >>> T = func.__type_params__[0] + >>> T.__bound__ + Traceback (most recent call last): + ... + ZeroDivisionError: division by zero + +Here the exception is raised only when the "__value__" attribute of +the type alias or the "__bound__" attribute of the type variable is +accessed. + +This behavior is primarily useful for references to types that have +not yet been defined when the type alias or type variable is created. +For example, lazy evaluation enables creation of mutually recursive +type aliases: + + from typing import Literal + + type SimpleExpr = int | Parenthesized + type Parenthesized = tuple[Literal["("], Expr, Literal[")"]] + type Expr = SimpleExpr | tuple[SimpleExpr, Literal["+", "-"], Expr] + +Lazily evaluated values are evaluated in annotation scope, which means +that names that appear inside the lazily evaluated value are looked up +as if they were used in the immediately enclosing scope. + +Added in version 3.12. + + +Builtins and restricted execution +--------------------------------- + +**CPython implementation detail:** Users should not touch +"__builtins__"; it is strictly an implementation detail. Users +wanting to override values in the builtins namespace should "import" +the "builtins" module and modify its attributes appropriately. + +The builtins namespace associated with the execution of a code block +is actually found by looking up the name "__builtins__" in its global +namespace; this should be a dictionary or a module (in the latter case +the module’s dictionary is used). By default, when in the "__main__" +module, "__builtins__" is the built-in module "builtins"; when in any +other module, "__builtins__" is an alias for the dictionary of the +"builtins" module itself. + + +Interaction with dynamic features +--------------------------------- + +Name resolution of free variables occurs at runtime, not at compile +time. This means that the following code will print 42: + + i = 10 + def f(): + print(i) + i = 42 + f() + +The "eval()" and "exec()" functions do not have access to the full +environment for resolving names. Names may be resolved in the local +and global namespaces of the caller. Free variables are not resolved +in the nearest enclosing namespace, but in the global namespace. [1] +The "exec()" and "eval()" functions have optional arguments to +override the global and local namespace. If only one namespace is +specified, it is used for both. + + +Exceptions +========== + +Exceptions are a means of breaking out of the normal flow of control +of a code block in order to handle errors or other exceptional +conditions. An exception is *raised* at the point where the error is +detected; it may be *handled* by the surrounding code block or by any +code block that directly or indirectly invoked the code block where +the error occurred. + +The Python interpreter raises an exception when it detects a run-time +error (such as division by zero). A Python program can also +explicitly raise an exception with the "raise" statement. Exception +handlers are specified with the "try" … "except" statement. The +"finally" clause of such a statement can be used to specify cleanup +code which does not handle the exception, but is executed whether an +exception occurred or not in the preceding code. + +Python uses the “termination” model of error handling: an exception +handler can find out what happened and continue execution at an outer +level, but it cannot repair the cause of the error and retry the +failing operation (except by re-entering the offending piece of code +from the top). + +When an exception is not handled at all, the interpreter terminates +execution of the program, or returns to its interactive main loop. In +either case, it prints a stack traceback, except when the exception is +"SystemExit". + +Exceptions are identified by class instances. The "except" clause is +selected depending on the class of the instance: it must reference the +class of the instance or a *non-virtual base class* thereof. The +instance can be received by the handler and can carry additional +information about the exceptional condition. + +Note: + + Exception messages are not part of the Python API. Their contents + may change from one version of Python to the next without warning + and should not be relied on by code which will run under multiple + versions of the interpreter. + +See also the description of the "try" statement in section The try +statement and "raise" statement in section The raise statement. + +-[ Footnotes ]- + +[1] This limitation occurs because the code that is executed by these + operations is not available at the time the module is compiled. +''', + 'exprlists': r'''Expression lists +**************** + + starred_expression ::= ["*"] or_expr + flexible_expression ::= assignment_expression | starred_expression + flexible_expression_list ::= flexible_expression ("," flexible_expression)* [","] + starred_expression_list ::= starred_expression ("," starred_expression)* [","] + expression_list ::= expression ("," expression)* [","] + yield_list ::= expression_list | starred_expression "," [starred_expression_list] + +Except when part of a list or set display, an expression list +containing at least one comma yields a tuple. The length of the tuple +is the number of expressions in the list. The expressions are +evaluated from left to right. + +An asterisk "*" denotes *iterable unpacking*. Its operand must be an +*iterable*. The iterable is expanded into a sequence of items, which +are included in the new tuple, list, or set, at the site of the +unpacking. + +Added in version 3.5: Iterable unpacking in expression lists, +originally proposed by **PEP 448**. + +Added in version 3.11: Any item in an expression list may be starred. +See **PEP 646**. + +A trailing comma is required only to create a one-item tuple, such as +"1,"; it is optional in all other cases. A single expression without a +trailing comma doesn’t create a tuple, but rather yields the value of +that expression. (To create an empty tuple, use an empty pair of +parentheses: "()".) +''', + 'floating': r'''Floating-point literals +*********************** + +Floating-point literals are described by the following lexical +definitions: + + floatnumber ::= pointfloat | exponentfloat + pointfloat ::= [digitpart] fraction | digitpart "." + exponentfloat ::= (digitpart | pointfloat) exponent + digitpart ::= digit (["_"] digit)* + fraction ::= "." digitpart + exponent ::= ("e" | "E") ["+" | "-"] digitpart + +Note that the integer and exponent parts are always interpreted using +radix 10. For example, "077e010" is legal, and denotes the same number +as "77e10". The allowed range of floating-point literals is +implementation-dependent. As in integer literals, underscores are +supported for digit grouping. + +Some examples of floating-point literals: + + 3.14 10. .001 1e100 3.14e-10 0e0 3.14_15_93 + +Changed in version 3.6: Underscores are now allowed for grouping +purposes in literals. +''', + 'for': r'''The "for" statement +******************* + +The "for" statement is used to iterate over the elements of a sequence +(such as a string, tuple or list) or other iterable object: + + for_stmt ::= "for" target_list "in" starred_list ":" suite + ["else" ":" suite] + +The "starred_list" expression is evaluated once; it should yield an +*iterable* object. An *iterator* is created for that iterable. The +first item provided by the iterator is then assigned to the target +list using the standard rules for assignments (see Assignment +statements), and the suite is executed. This repeats for each item +provided by the iterator. When the iterator is exhausted, the suite +in the "else" clause, if present, is executed, and the loop +terminates. + +A "break" statement executed in the first suite terminates the loop +without executing the "else" clause’s suite. A "continue" statement +executed in the first suite skips the rest of the suite and continues +with the next item, or with the "else" clause if there is no next +item. + +The for-loop makes assignments to the variables in the target list. +This overwrites all previous assignments to those variables including +those made in the suite of the for-loop: + + for i in range(10): + print(i) + i = 5 # this will not affect the for-loop + # because i will be overwritten with the next + # index in the range + +Names in the target list are not deleted when the loop is finished, +but if the sequence is empty, they will not have been assigned to at +all by the loop. Hint: the built-in type "range()" represents +immutable arithmetic sequences of integers. For instance, iterating +"range(3)" successively yields 0, 1, and then 2. + +Changed in version 3.11: Starred elements are now allowed in the +expression list. +''', + 'formatstrings': r'''Format String Syntax +******************** + +The "str.format()" method and the "Formatter" class share the same +syntax for format strings (although in the case of "Formatter", +subclasses can define their own format string syntax). The syntax is +related to that of formatted string literals, but it is less +sophisticated and, in particular, does not support arbitrary +expressions. + +Format strings contain “replacement fields” surrounded by curly braces +"{}". Anything that is not contained in braces is considered literal +text, which is copied unchanged to the output. If you need to include +a brace character in the literal text, it can be escaped by doubling: +"{{" and "}}". + +The grammar for a replacement field is as follows: + + replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}" + field_name ::= arg_name ("." attribute_name | "[" element_index "]")* + arg_name ::= [identifier | digit+] + attribute_name ::= identifier + element_index ::= digit+ | index_string + index_string ::= + + conversion ::= "r" | "s" | "a" + format_spec ::= format-spec:format_spec + +In less formal terms, the replacement field can start with a +*field_name* that specifies the object whose value is to be formatted +and inserted into the output instead of the replacement field. The +*field_name* is optionally followed by a *conversion* field, which is +preceded by an exclamation point "'!'", and a *format_spec*, which is +preceded by a colon "':'". These specify a non-default format for the +replacement value. + +See also the Format Specification Mini-Language section. + +The *field_name* itself begins with an *arg_name* that is either a +number or a keyword. If it’s a number, it refers to a positional +argument, and if it’s a keyword, it refers to a named keyword +argument. An *arg_name* is treated as a number if a call to +"str.isdecimal()" on the string would return true. If the numerical +arg_names in a format string are 0, 1, 2, … in sequence, they can all +be omitted (not just some) and the numbers 0, 1, 2, … will be +automatically inserted in that order. Because *arg_name* is not quote- +delimited, it is not possible to specify arbitrary dictionary keys +(e.g., the strings "'10'" or "':-]'") within a format string. The +*arg_name* can be followed by any number of index or attribute +expressions. An expression of the form "'.name'" selects the named +attribute using "getattr()", while an expression of the form +"'[index]'" does an index lookup using "__getitem__()". + +Changed in version 3.1: The positional argument specifiers can be +omitted for "str.format()", so "'{} {}'.format(a, b)" is equivalent to +"'{0} {1}'.format(a, b)". + +Changed in version 3.4: The positional argument specifiers can be +omitted for "Formatter". + +Some simple format string examples: + + "First, thou shalt count to {0}" # References first positional argument + "Bring me a {}" # Implicitly references the first positional argument + "From {} to {}" # Same as "From {0} to {1}" + "My quest is {name}" # References keyword argument 'name' + "Weight in tons {0.weight}" # 'weight' attribute of first positional arg + "Units destroyed: {players[0]}" # First element of keyword argument 'players'. + +The *conversion* field causes a type coercion before formatting. +Normally, the job of formatting a value is done by the "__format__()" +method of the value itself. However, in some cases it is desirable to +force a type to be formatted as a string, overriding its own +definition of formatting. By converting the value to a string before +calling "__format__()", the normal formatting logic is bypassed. + +Three conversion flags are currently supported: "'!s'" which calls +"str()" on the value, "'!r'" which calls "repr()" and "'!a'" which +calls "ascii()". + +Some examples: + + "Harold's a clever {0!s}" # Calls str() on the argument first + "Bring out the holy {name!r}" # Calls repr() on the argument first + "More {!a}" # Calls ascii() on the argument first + +The *format_spec* field contains a specification of how the value +should be presented, including such details as field width, alignment, +padding, decimal precision and so on. Each value type can define its +own “formatting mini-language” or interpretation of the *format_spec*. + +Most built-in types support a common formatting mini-language, which +is described in the next section. + +A *format_spec* field can also include nested replacement fields +within it. These nested replacement fields may contain a field name, +conversion flag and format specification, but deeper nesting is not +allowed. The replacement fields within the format_spec are +substituted before the *format_spec* string is interpreted. This +allows the formatting of a value to be dynamically specified. + +See the Format examples section for some examples. + + +Format Specification Mini-Language +================================== + +“Format specifications” are used within replacement fields contained +within a format string to define how individual values are presented +(see Format String Syntax and f-strings). They can also be passed +directly to the built-in "format()" function. Each formattable type +may define how the format specification is to be interpreted. + +Most built-in types implement the following options for format +specifications, although some of the formatting options are only +supported by the numeric types. + +A general convention is that an empty format specification produces +the same result as if you had called "str()" on the value. A non-empty +format specification typically modifies the result. + +The general form of a *standard format specifier* is: + + format_spec ::= [[fill]align][sign]["z"]["#"]["0"][width][grouping_option]["." precision][type] + fill ::= + align ::= "<" | ">" | "=" | "^" + sign ::= "+" | "-" | " " + width ::= digit+ + grouping_option ::= "_" | "," + precision ::= digit+ + type ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%" + +If a valid *align* value is specified, it can be preceded by a *fill* +character that can be any character and defaults to a space if +omitted. It is not possible to use a literal curly brace (”"{"” or +“"}"”) as the *fill* character in a formatted string literal or when +using the "str.format()" method. However, it is possible to insert a +curly brace with a nested replacement field. This limitation doesn’t +affect the "format()" function. + +The meaning of the various alignment options is as follows: + ++-----------+------------------------------------------------------------+ +| Option | Meaning | +|===========|============================================================| +| "'<'" | Forces the field to be left-aligned within the available | +| | space (this is the default for most objects). | ++-----------+------------------------------------------------------------+ +| "'>'" | Forces the field to be right-aligned within the available | +| | space (this is the default for numbers). | ++-----------+------------------------------------------------------------+ +| "'='" | Forces the padding to be placed after the sign (if any) | +| | but before the digits. This is used for printing fields | +| | in the form ‘+000000120’. This alignment option is only | +| | valid for numeric types, excluding "complex". It becomes | +| | the default for numbers when ‘0’ immediately precedes the | +| | field width. | ++-----------+------------------------------------------------------------+ +| "'^'" | Forces the field to be centered within the available | +| | space. | ++-----------+------------------------------------------------------------+ + +Note that unless a minimum field width is defined, the field width +will always be the same size as the data to fill it, so that the +alignment option has no meaning in this case. + +The *sign* option is only valid for number types, and can be one of +the following: + ++-----------+------------------------------------------------------------+ +| Option | Meaning | +|===========|============================================================| +| "'+'" | indicates that a sign should be used for both positive as | +| | well as negative numbers. | ++-----------+------------------------------------------------------------+ +| "'-'" | indicates that a sign should be used only for negative | +| | numbers (this is the default behavior). | ++-----------+------------------------------------------------------------+ +| space | indicates that a leading space should be used on positive | +| | numbers, and a minus sign on negative numbers. | ++-----------+------------------------------------------------------------+ + +The "'z'" option coerces negative zero floating-point values to +positive zero after rounding to the format precision. This option is +only valid for floating-point presentation types. + +Changed in version 3.11: Added the "'z'" option (see also **PEP +682**). + +The "'#'" option causes the “alternate form” to be used for the +conversion. The alternate form is defined differently for different +types. This option is only valid for integer, float and complex +types. For integers, when binary, octal, or hexadecimal output is +used, this option adds the respective prefix "'0b'", "'0o'", "'0x'", +or "'0X'" to the output value. For float and complex the alternate +form causes the result of the conversion to always contain a decimal- +point character, even if no digits follow it. Normally, a decimal- +point character appears in the result of these conversions only if a +digit follows it. In addition, for "'g'" and "'G'" conversions, +trailing zeros are not removed from the result. + +The "','" option signals the use of a comma for a thousands separator +for floating-point presentation types and for integer presentation +type "'d'". For other presentation types, this option is an error. For +a locale aware separator, use the "'n'" integer presentation type +instead. + +Changed in version 3.1: Added the "','" option (see also **PEP 378**). + +The "'_'" option signals the use of an underscore for a thousands +separator for floating-point presentation types and for integer +presentation type "'d'". For integer presentation types "'b'", "'o'", +"'x'", and "'X'", underscores will be inserted every 4 digits. For +other presentation types, specifying this option is an error. + +Changed in version 3.6: Added the "'_'" option (see also **PEP 515**). + +*width* is a decimal integer defining the minimum total field width, +including any prefixes, separators, and other formatting characters. +If not specified, then the field width will be determined by the +content. + +When no explicit alignment is given, preceding the *width* field by a +zero ("'0'") character enables sign-aware zero-padding for numeric +types, excluding "complex". This is equivalent to a *fill* character +of "'0'" with an *alignment* type of "'='". + +Changed in version 3.10: Preceding the *width* field by "'0'" no +longer affects the default alignment for strings. + +The *precision* is a decimal integer indicating how many digits should +be displayed after the decimal point for presentation types "'f'" and +"'F'", or before and after the decimal point for presentation types +"'g'" or "'G'". For string presentation types the field indicates the +maximum field size - in other words, how many characters will be used +from the field content. The *precision* is not allowed for integer +presentation types. + +Finally, the *type* determines how the data should be presented. + +The available string presentation types are: + + +-----------+------------------------------------------------------------+ + | Type | Meaning | + |===========|============================================================| + | "'s'" | String format. This is the default type for strings and | + | | may be omitted. | + +-----------+------------------------------------------------------------+ + | None | The same as "'s'". | + +-----------+------------------------------------------------------------+ + +The available integer presentation types are: + + +-----------+------------------------------------------------------------+ + | Type | Meaning | + |===========|============================================================| + | "'b'" | Binary format. Outputs the number in base 2. | + +-----------+------------------------------------------------------------+ + | "'c'" | Character. Converts the integer to the corresponding | + | | unicode character before printing. | + +-----------+------------------------------------------------------------+ + | "'d'" | Decimal Integer. Outputs the number in base 10. | + +-----------+------------------------------------------------------------+ + | "'o'" | Octal format. Outputs the number in base 8. | + +-----------+------------------------------------------------------------+ + | "'x'" | Hex format. Outputs the number in base 16, using lower- | + | | case letters for the digits above 9. | + +-----------+------------------------------------------------------------+ + | "'X'" | Hex format. Outputs the number in base 16, using upper- | + | | case letters for the digits above 9. In case "'#'" is | + | | specified, the prefix "'0x'" will be upper-cased to "'0X'" | + | | as well. | + +-----------+------------------------------------------------------------+ + | "'n'" | Number. This is the same as "'d'", except that it uses the | + | | current locale setting to insert the appropriate number | + | | separator characters. | + +-----------+------------------------------------------------------------+ + | None | The same as "'d'". | + +-----------+------------------------------------------------------------+ + +In addition to the above presentation types, integers can be formatted +with the floating-point presentation types listed below (except "'n'" +and "None"). When doing so, "float()" is used to convert the integer +to a floating-point number before formatting. + +The available presentation types for "float" and "Decimal" values are: + + +-----------+------------------------------------------------------------+ + | Type | Meaning | + |===========|============================================================| + | "'e'" | Scientific notation. For a given precision "p", formats | + | | the number in scientific notation with the letter ‘e’ | + | | separating the coefficient from the exponent. The | + | | coefficient has one digit before and "p" digits after the | + | | decimal point, for a total of "p + 1" significant digits. | + | | With no precision given, uses a precision of "6" digits | + | | after the decimal point for "float", and shows all | + | | coefficient digits for "Decimal". If "p=0", the decimal | + | | point is omitted unless the "#" option is used. | + +-----------+------------------------------------------------------------+ + | "'E'" | Scientific notation. Same as "'e'" except it uses an upper | + | | case ‘E’ as the separator character. | + +-----------+------------------------------------------------------------+ + | "'f'" | Fixed-point notation. For a given precision "p", formats | + | | the number as a decimal number with exactly "p" digits | + | | following the decimal point. With no precision given, uses | + | | a precision of "6" digits after the decimal point for | + | | "float", and uses a precision large enough to show all | + | | coefficient digits for "Decimal". If "p=0", the decimal | + | | point is omitted unless the "#" option is used. | + +-----------+------------------------------------------------------------+ + | "'F'" | Fixed-point notation. Same as "'f'", but converts "nan" to | + | | "NAN" and "inf" to "INF". | + +-----------+------------------------------------------------------------+ + | "'g'" | General format. For a given precision "p >= 1", this | + | | rounds the number to "p" significant digits and then | + | | formats the result in either fixed-point format or in | + | | scientific notation, depending on its magnitude. A | + | | precision of "0" is treated as equivalent to a precision | + | | of "1". The precise rules are as follows: suppose that | + | | the result formatted with presentation type "'e'" and | + | | precision "p-1" would have exponent "exp". Then, if "m <= | + | | exp < p", where "m" is -4 for floats and -6 for | + | | "Decimals", the number is formatted with presentation type | + | | "'f'" and precision "p-1-exp". Otherwise, the number is | + | | formatted with presentation type "'e'" and precision | + | | "p-1". In both cases insignificant trailing zeros are | + | | removed from the significand, and the decimal point is | + | | also removed if there are no remaining digits following | + | | it, unless the "'#'" option is used. With no precision | + | | given, uses a precision of "6" significant digits for | + | | "float". For "Decimal", the coefficient of the result is | + | | formed from the coefficient digits of the value; | + | | scientific notation is used for values smaller than "1e-6" | + | | in absolute value and values where the place value of the | + | | least significant digit is larger than 1, and fixed-point | + | | notation is used otherwise. Positive and negative | + | | infinity, positive and negative zero, and nans, are | + | | formatted as "inf", "-inf", "0", "-0" and "nan" | + | | respectively, regardless of the precision. | + +-----------+------------------------------------------------------------+ + | "'G'" | General format. Same as "'g'" except switches to "'E'" if | + | | the number gets too large. The representations of infinity | + | | and NaN are uppercased, too. | + +-----------+------------------------------------------------------------+ + | "'n'" | Number. This is the same as "'g'", except that it uses the | + | | current locale setting to insert the appropriate number | + | | separator characters. | + +-----------+------------------------------------------------------------+ + | "'%'" | Percentage. Multiplies the number by 100 and displays in | + | | fixed ("'f'") format, followed by a percent sign. | + +-----------+------------------------------------------------------------+ + | None | For "float" this is like the "'g'" type, except that when | + | | fixed- point notation is used to format the result, it | + | | always includes at least one digit past the decimal point, | + | | and switches to the scientific notation when "exp >= p - | + | | 1". When the precision is not specified, the latter will | + | | be as large as needed to represent the given value | + | | faithfully. For "Decimal", this is the same as either | + | | "'g'" or "'G'" depending on the value of | + | | "context.capitals" for the current decimal context. The | + | | overall effect is to match the output of "str()" as | + | | altered by the other format modifiers. | + +-----------+------------------------------------------------------------+ + +The result should be correctly rounded to a given precision "p" of +digits after the decimal point. The rounding mode for "float" matches +that of the "round()" builtin. For "Decimal", the rounding mode of +the current context will be used. + +The available presentation types for "complex" are the same as those +for "float" ("'%'" is not allowed). Both the real and imaginary +components of a complex number are formatted as floating-point +numbers, according to the specified presentation type. They are +separated by the mandatory sign of the imaginary part, the latter +being terminated by a "j" suffix. If the presentation type is +missing, the result will match the output of "str()" (complex numbers +with a non-zero real part are also surrounded by parentheses), +possibly altered by other format modifiers. + + +Format examples +=============== + +This section contains examples of the "str.format()" syntax and +comparison with the old "%"-formatting. + +In most of the cases the syntax is similar to the old "%"-formatting, +with the addition of the "{}" and with ":" used instead of "%". For +example, "'%03.2f'" can be translated to "'{:03.2f}'". + +The new format syntax also supports new and different options, shown +in the following examples. + +Accessing arguments by position: + + >>> '{0}, {1}, {2}'.format('a', 'b', 'c') + 'a, b, c' + >>> '{}, {}, {}'.format('a', 'b', 'c') # 3.1+ only + 'a, b, c' + >>> '{2}, {1}, {0}'.format('a', 'b', 'c') + 'c, b, a' + >>> '{2}, {1}, {0}'.format(*'abc') # unpacking argument sequence + 'c, b, a' + >>> '{0}{1}{0}'.format('abra', 'cad') # arguments' indices can be repeated + 'abracadabra' + +Accessing arguments by name: + + >>> 'Coordinates: {latitude}, {longitude}'.format(latitude='37.24N', longitude='-115.81W') + 'Coordinates: 37.24N, -115.81W' + >>> coord = {'latitude': '37.24N', 'longitude': '-115.81W'} + >>> 'Coordinates: {latitude}, {longitude}'.format(**coord) + 'Coordinates: 37.24N, -115.81W' + +Accessing arguments’ attributes: + + >>> c = 3-5j + >>> ('The complex number {0} is formed from the real part {0.real} ' + ... 'and the imaginary part {0.imag}.').format(c) + 'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.' + >>> class Point: + ... def __init__(self, x, y): + ... self.x, self.y = x, y + ... def __str__(self): + ... return 'Point({self.x}, {self.y})'.format(self=self) + ... + >>> str(Point(4, 2)) + 'Point(4, 2)' + +Accessing arguments’ items: + + >>> coord = (3, 5) + >>> 'X: {0[0]}; Y: {0[1]}'.format(coord) + 'X: 3; Y: 5' + +Replacing "%s" and "%r": + + >>> "repr() shows quotes: {!r}; str() doesn't: {!s}".format('test1', 'test2') + "repr() shows quotes: 'test1'; str() doesn't: test2" + +Aligning the text and specifying a width: + + >>> '{:<30}'.format('left aligned') + 'left aligned ' + >>> '{:>30}'.format('right aligned') + ' right aligned' + >>> '{:^30}'.format('centered') + ' centered ' + >>> '{:*^30}'.format('centered') # use '*' as a fill char + '***********centered***********' + +Replacing "%+f", "%-f", and "% f" and specifying a sign: + + >>> '{:+f}; {:+f}'.format(3.14, -3.14) # show it always + '+3.140000; -3.140000' + >>> '{: f}; {: f}'.format(3.14, -3.14) # show a space for positive numbers + ' 3.140000; -3.140000' + >>> '{:-f}; {:-f}'.format(3.14, -3.14) # show only the minus -- same as '{:f}; {:f}' + '3.140000; -3.140000' + +Replacing "%x" and "%o" and converting the value to different bases: + + >>> # format also supports binary numbers + >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}".format(42) + 'int: 42; hex: 2a; oct: 52; bin: 101010' + >>> # with 0x, 0o, or 0b as prefix: + >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}".format(42) + 'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010' + +Using the comma as a thousands separator: + + >>> '{:,}'.format(1234567890) + '1,234,567,890' + +Expressing a percentage: + + >>> points = 19 + >>> total = 22 + >>> 'Correct answers: {:.2%}'.format(points/total) + 'Correct answers: 86.36%' + +Using type-specific formatting: + + >>> import datetime + >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58) + >>> '{:%Y-%m-%d %H:%M:%S}'.format(d) + '2010-07-04 12:15:58' + +Nesting arguments and more complex examples: + + >>> for align, text in zip('<^>', ['left', 'center', 'right']): + ... '{0:{fill}{align}16}'.format(text, fill=align, align=align) + ... + 'left<<<<<<<<<<<<' + '^^^^^center^^^^^' + '>>>>>>>>>>>right' + >>> + >>> octets = [192, 168, 0, 1] + >>> '{:02X}{:02X}{:02X}{:02X}'.format(*octets) + 'C0A80001' + >>> int(_, 16) + 3232235521 + >>> + >>> width = 5 + >>> for num in range(5,12): + ... for base in 'dXob': + ... print('{0:{width}{base}}'.format(num, base=base, width=width), end=' ') + ... print() + ... + 5 5 5 101 + 6 6 6 110 + 7 7 7 111 + 8 8 10 1000 + 9 9 11 1001 + 10 A 12 1010 + 11 B 13 1011 +''', + 'function': r'''Function definitions +******************** + +A function definition defines a user-defined function object (see +section The standard type hierarchy): + + funcdef ::= [decorators] "def" funcname [type_params] "(" [parameter_list] ")" + ["->" expression] ":" suite + decorators ::= decorator+ + decorator ::= "@" assignment_expression NEWLINE + parameter_list ::= defparameter ("," defparameter)* "," "/" ["," [parameter_list_no_posonly]] + | parameter_list_no_posonly + parameter_list_no_posonly ::= defparameter ("," defparameter)* ["," [parameter_list_starargs]] + | parameter_list_starargs + parameter_list_starargs ::= "*" [star_parameter] ("," defparameter)* ["," ["**" parameter [","]]] + | "**" parameter [","] + parameter ::= identifier [":" expression] + star_parameter ::= identifier [":" ["*"] expression] + defparameter ::= parameter ["=" expression] + funcname ::= identifier + +A function definition is an executable statement. Its execution binds +the function name in the current local namespace to a function object +(a wrapper around the executable code for the function). This +function object contains a reference to the current global namespace +as the global namespace to be used when the function is called. + +The function definition does not execute the function body; this gets +executed only when the function is called. [4] + +A function definition may be wrapped by one or more *decorator* +expressions. Decorator expressions are evaluated when the function is +defined, in the scope that contains the function definition. The +result must be a callable, which is invoked with the function object +as the only argument. The returned value is bound to the function name +instead of the function object. Multiple decorators are applied in +nested fashion. For example, the following code + + @f1(arg) + @f2 + def func(): pass + +is roughly equivalent to + + def func(): pass + func = f1(arg)(f2(func)) + +except that the original function is not temporarily bound to the name +"func". + +Changed in version 3.9: Functions may be decorated with any valid +"assignment_expression". Previously, the grammar was much more +restrictive; see **PEP 614** for details. + +A list of type parameters may be given in square brackets between the +function’s name and the opening parenthesis for its parameter list. +This indicates to static type checkers that the function is generic. +At runtime, the type parameters can be retrieved from the function’s +"__type_params__" attribute. See Generic functions for more. + +Changed in version 3.12: Type parameter lists are new in Python 3.12. + +When one or more *parameters* have the form *parameter* "=" +*expression*, the function is said to have “default parameter values.” +For a parameter with a default value, the corresponding *argument* may +be omitted from a call, in which case the parameter’s default value is +substituted. If a parameter has a default value, all following +parameters up until the “"*"” must also have a default value — this is +a syntactic restriction that is not expressed by the grammar. + +**Default parameter values are evaluated from left to right when the +function definition is executed.** This means that the expression is +evaluated once, when the function is defined, and that the same “pre- +computed” value is used for each call. This is especially important +to understand when a default parameter value is a mutable object, such +as a list or a dictionary: if the function modifies the object (e.g. +by appending an item to a list), the default parameter value is in +effect modified. This is generally not what was intended. A way +around this is to use "None" as the default, and explicitly test for +it in the body of the function, e.g.: + + def whats_on_the_telly(penguin=None): + if penguin is None: + penguin = [] + penguin.append("property of the zoo") + return penguin + +Function call semantics are described in more detail in section Calls. +A function call always assigns values to all parameters mentioned in +the parameter list, either from positional arguments, from keyword +arguments, or from default values. If the form “"*identifier"” is +present, it is initialized to a tuple receiving any excess positional +parameters, defaulting to the empty tuple. If the form +“"**identifier"” is present, it is initialized to a new ordered +mapping receiving any excess keyword arguments, defaulting to a new +empty mapping of the same type. Parameters after “"*"” or +“"*identifier"” are keyword-only parameters and may only be passed by +keyword arguments. Parameters before “"/"” are positional-only +parameters and may only be passed by positional arguments. + +Changed in version 3.8: The "/" function parameter syntax may be used +to indicate positional-only parameters. See **PEP 570** for details. + +Parameters may have an *annotation* of the form “": expression"” +following the parameter name. Any parameter may have an annotation, +even those of the form "*identifier" or "**identifier". (As a special +case, parameters of the form "*identifier" may have an annotation “": +*expression"”.) Functions may have “return” annotation of the form +“"-> expression"” after the parameter list. These annotations can be +any valid Python expression. The presence of annotations does not +change the semantics of a function. See Annotations for more +information on annotations. + +Changed in version 3.11: Parameters of the form “"*identifier"” may +have an annotation “": *expression"”. See **PEP 646**. + +It is also possible to create anonymous functions (functions not bound +to a name), for immediate use in expressions. This uses lambda +expressions, described in section Lambdas. Note that the lambda +expression is merely a shorthand for a simplified function definition; +a function defined in a “"def"” statement can be passed around or +assigned to another name just like a function defined by a lambda +expression. The “"def"” form is actually more powerful since it +allows the execution of multiple statements and annotations. + +**Programmer’s note:** Functions are first-class objects. A “"def"” +statement executed inside a function definition defines a local +function that can be returned or passed around. Free variables used +in the nested function can access the local variables of the function +containing the def. See section Naming and binding for details. + +See also: + + **PEP 3107** - Function Annotations + The original specification for function annotations. + + **PEP 484** - Type Hints + Definition of a standard meaning for annotations: type hints. + + **PEP 526** - Syntax for Variable Annotations + Ability to type hint variable declarations, including class + variables and instance variables. + + **PEP 563** - Postponed Evaluation of Annotations + Support for forward references within annotations by preserving + annotations in a string form at runtime instead of eager + evaluation. + + **PEP 318** - Decorators for Functions and Methods + Function and method decorators were introduced. Class decorators + were introduced in **PEP 3129**. +''', + 'global': r'''The "global" statement +********************** + + global_stmt ::= "global" identifier ("," identifier)* + +The "global" statement causes the listed identifiers to be interpreted +as globals. It would be impossible to assign to a global variable +without "global", although free variables may refer to globals without +being declared global. + +The "global" statement applies to the entire scope of a function or +class body. A "SyntaxError" is raised if a variable is used or +assigned to prior to its global declaration in the scope. + +**Programmer’s note:** "global" is a directive to the parser. It +applies only to code parsed at the same time as the "global" +statement. In particular, a "global" statement contained in a string +or code object supplied to the built-in "exec()" function does not +affect the code block *containing* the function call, and code +contained in such a string is unaffected by "global" statements in the +code containing the function call. The same applies to the "eval()" +and "compile()" functions. +''', + 'id-classes': r'''Reserved classes of identifiers +******************************* + +Certain classes of identifiers (besides keywords) have special +meanings. These classes are identified by the patterns of leading and +trailing underscore characters: + +"_*" + Not imported by "from module import *". + +"_" + In a "case" pattern within a "match" statement, "_" is a soft + keyword that denotes a wildcard. + + Separately, the interactive interpreter makes the result of the + last evaluation available in the variable "_". (It is stored in the + "builtins" module, alongside built-in functions like "print".) + + Elsewhere, "_" is a regular identifier. It is often used to name + “special” items, but it is not special to Python itself. + + Note: + + The name "_" is often used in conjunction with + internationalization; refer to the documentation for the + "gettext" module for more information on this convention.It is + also commonly used for unused variables. + +"__*__" + System-defined names, informally known as “dunder” names. These + names are defined by the interpreter and its implementation + (including the standard library). Current system names are + discussed in the Special method names section and elsewhere. More + will likely be defined in future versions of Python. *Any* use of + "__*__" names, in any context, that does not follow explicitly + documented use, is subject to breakage without warning. + +"__*" + Class-private names. Names in this category, when used within the + context of a class definition, are re-written to use a mangled form + to help avoid name clashes between “private” attributes of base and + derived classes. See section Identifiers (Names). +''', + 'identifiers': r'''Identifiers and keywords +************************ + +Identifiers (also referred to as *names*) are described by the +following lexical definitions. + +The syntax of identifiers in Python is based on the Unicode standard +annex UAX-31, with elaboration and changes as defined below; see also +**PEP 3131** for further details. + +Within the ASCII range (U+0001..U+007F), the valid characters for +identifiers include the uppercase and lowercase letters "A" through +"Z", the underscore "_" and, except for the first character, the +digits "0" through "9". Python 3.0 introduced additional characters +from outside the ASCII range (see **PEP 3131**). For these +characters, the classification uses the version of the Unicode +Character Database as included in the "unicodedata" module. + +Identifiers are unlimited in length. Case is significant. + + identifier ::= xid_start xid_continue* + id_start ::= + id_continue ::= + xid_start ::= + xid_continue ::= + +The Unicode category codes mentioned above stand for: + +* *Lu* - uppercase letters + +* *Ll* - lowercase letters + +* *Lt* - titlecase letters + +* *Lm* - modifier letters + +* *Lo* - other letters + +* *Nl* - letter numbers + +* *Mn* - nonspacing marks + +* *Mc* - spacing combining marks + +* *Nd* - decimal numbers + +* *Pc* - connector punctuations + +* *Other_ID_Start* - explicit list of characters in PropList.txt to + support backwards compatibility + +* *Other_ID_Continue* - likewise + +All identifiers are converted into the normal form NFKC while parsing; +comparison of identifiers is based on NFKC. + +A non-normative HTML file listing all valid identifier characters for +Unicode 16.0.0 can be found at +https://www.unicode.org/Public/16.0.0/ucd/DerivedCoreProperties.txt + + +Keywords +======== + +The following identifiers are used as reserved words, or *keywords* of +the language, and cannot be used as ordinary identifiers. They must +be spelled exactly as written here: + + False await else import pass + None break except in raise + True class finally is return + and continue for lambda try + as def from nonlocal while + assert del global not with + async elif if or yield + + +Soft Keywords +============= + +Added in version 3.10. + +Some identifiers are only reserved under specific contexts. These are +known as *soft keywords*. The identifiers "match", "case", "type" and +"_" can syntactically act as keywords in certain contexts, but this +distinction is done at the parser level, not when tokenizing. + +As soft keywords, their use in the grammar is possible while still +preserving compatibility with existing code that uses these names as +identifier names. + +"match", "case", and "_" are used in the "match" statement. "type" is +used in the "type" statement. + +Changed in version 3.12: "type" is now a soft keyword. + + +Reserved classes of identifiers +=============================== + +Certain classes of identifiers (besides keywords) have special +meanings. These classes are identified by the patterns of leading and +trailing underscore characters: + +"_*" + Not imported by "from module import *". + +"_" + In a "case" pattern within a "match" statement, "_" is a soft + keyword that denotes a wildcard. + + Separately, the interactive interpreter makes the result of the + last evaluation available in the variable "_". (It is stored in the + "builtins" module, alongside built-in functions like "print".) + + Elsewhere, "_" is a regular identifier. It is often used to name + “special” items, but it is not special to Python itself. + + Note: + + The name "_" is often used in conjunction with + internationalization; refer to the documentation for the + "gettext" module for more information on this convention.It is + also commonly used for unused variables. + +"__*__" + System-defined names, informally known as “dunder” names. These + names are defined by the interpreter and its implementation + (including the standard library). Current system names are + discussed in the Special method names section and elsewhere. More + will likely be defined in future versions of Python. *Any* use of + "__*__" names, in any context, that does not follow explicitly + documented use, is subject to breakage without warning. + +"__*" + Class-private names. Names in this category, when used within the + context of a class definition, are re-written to use a mangled form + to help avoid name clashes between “private” attributes of base and + derived classes. See section Identifiers (Names). +''', + 'if': r'''The "if" statement +****************** + +The "if" statement is used for conditional execution: + + if_stmt ::= "if" assignment_expression ":" suite + ("elif" assignment_expression ":" suite)* + ["else" ":" suite] + +It selects exactly one of the suites by evaluating the expressions one +by one until one is found to be true (see section Boolean operations +for the definition of true and false); then that suite is executed +(and no other part of the "if" statement is executed or evaluated). +If all expressions are false, the suite of the "else" clause, if +present, is executed. +''', + 'imaginary': r'''Imaginary literals +****************** + +Imaginary literals are described by the following lexical definitions: + + imagnumber ::= (floatnumber | digitpart) ("j" | "J") + +An imaginary literal yields a complex number with a real part of 0.0. +Complex numbers are represented as a pair of floating-point numbers +and have the same restrictions on their range. To create a complex +number with a nonzero real part, add a floating-point number to it, +e.g., "(3+4j)". Some examples of imaginary literals: + + 3.14j 10.j 10j .001j 1e100j 3.14e-10j 3.14_15_93j +''', + 'import': r'''The "import" statement +********************** + + import_stmt ::= "import" module ["as" identifier] ("," module ["as" identifier])* + | "from" relative_module "import" identifier ["as" identifier] + ("," identifier ["as" identifier])* + | "from" relative_module "import" "(" identifier ["as" identifier] + ("," identifier ["as" identifier])* [","] ")" + | "from" relative_module "import" "*" + module ::= (identifier ".")* identifier + relative_module ::= "."* module | "."+ + +The basic import statement (no "from" clause) is executed in two +steps: + +1. find a module, loading and initializing it if necessary + +2. define a name or names in the local namespace for the scope where + the "import" statement occurs. + +When the statement contains multiple clauses (separated by commas) the +two steps are carried out separately for each clause, just as though +the clauses had been separated out into individual import statements. + +The details of the first step, finding and loading modules, are +described in greater detail in the section on the import system, which +also describes the various types of packages and modules that can be +imported, as well as all the hooks that can be used to customize the +import system. Note that failures in this step may indicate either +that the module could not be located, *or* that an error occurred +while initializing the module, which includes execution of the +module’s code. + +If the requested module is retrieved successfully, it will be made +available in the local namespace in one of three ways: + +* If the module name is followed by "as", then the name following "as" + is bound directly to the imported module. + +* If no other name is specified, and the module being imported is a + top level module, the module’s name is bound in the local namespace + as a reference to the imported module + +* If the module being imported is *not* a top level module, then the + name of the top level package that contains the module is bound in + the local namespace as a reference to the top level package. The + imported module must be accessed using its full qualified name + rather than directly + +The "from" form uses a slightly more complex process: + +1. find the module specified in the "from" clause, loading and + initializing it if necessary; + +2. for each of the identifiers specified in the "import" clauses: + + 1. check if the imported module has an attribute by that name + + 2. if not, attempt to import a submodule with that name and then + check the imported module again for that attribute + + 3. if the attribute is not found, "ImportError" is raised. + + 4. otherwise, a reference to that value is stored in the local + namespace, using the name in the "as" clause if it is present, + otherwise using the attribute name + +Examples: + + import foo # foo imported and bound locally + import foo.bar.baz # foo, foo.bar, and foo.bar.baz imported, foo bound locally + import foo.bar.baz as fbb # foo, foo.bar, and foo.bar.baz imported, foo.bar.baz bound as fbb + from foo.bar import baz # foo, foo.bar, and foo.bar.baz imported, foo.bar.baz bound as baz + from foo import attr # foo imported and foo.attr bound as attr + +If the list of identifiers is replaced by a star ("'*'"), all public +names defined in the module are bound in the local namespace for the +scope where the "import" statement occurs. + +The *public names* defined by a module are determined by checking the +module’s namespace for a variable named "__all__"; if defined, it must +be a sequence of strings which are names defined or imported by that +module. The names given in "__all__" are all considered public and +are required to exist. If "__all__" is not defined, the set of public +names includes all names found in the module’s namespace which do not +begin with an underscore character ("'_'"). "__all__" should contain +the entire public API. It is intended to avoid accidentally exporting +items that are not part of the API (such as library modules which were +imported and used within the module). + +The wild card form of import — "from module import *" — is only +allowed at the module level. Attempting to use it in class or +function definitions will raise a "SyntaxError". + +When specifying what module to import you do not have to specify the +absolute name of the module. When a module or package is contained +within another package it is possible to make a relative import within +the same top package without having to mention the package name. By +using leading dots in the specified module or package after "from" you +can specify how high to traverse up the current package hierarchy +without specifying exact names. One leading dot means the current +package where the module making the import exists. Two dots means up +one package level. Three dots is up two levels, etc. So if you execute +"from . import mod" from a module in the "pkg" package then you will +end up importing "pkg.mod". If you execute "from ..subpkg2 import mod" +from within "pkg.subpkg1" you will import "pkg.subpkg2.mod". The +specification for relative imports is contained in the Package +Relative Imports section. + +"importlib.import_module()" is provided to support applications that +determine dynamically the modules to be loaded. + +Raises an auditing event "import" with arguments "module", "filename", +"sys.path", "sys.meta_path", "sys.path_hooks". + + +Future statements +================= + +A *future statement* is a directive to the compiler that a particular +module should be compiled using syntax or semantics that will be +available in a specified future release of Python where the feature +becomes standard. + +The future statement is intended to ease migration to future versions +of Python that introduce incompatible changes to the language. It +allows use of the new features on a per-module basis before the +release in which the feature becomes standard. + + future_stmt ::= "from" "__future__" "import" feature ["as" identifier] + ("," feature ["as" identifier])* + | "from" "__future__" "import" "(" feature ["as" identifier] + ("," feature ["as" identifier])* [","] ")" + feature ::= identifier + +A future statement must appear near the top of the module. The only +lines that can appear before a future statement are: + +* the module docstring (if any), + +* comments, + +* blank lines, and + +* other future statements. + +The only feature that requires using the future statement is +"annotations" (see **PEP 563**). + +All historical features enabled by the future statement are still +recognized by Python 3. The list includes "absolute_import", +"division", "generators", "generator_stop", "unicode_literals", +"print_function", "nested_scopes" and "with_statement". They are all +redundant because they are always enabled, and only kept for backwards +compatibility. + +A future statement is recognized and treated specially at compile +time: Changes to the semantics of core constructs are often +implemented by generating different code. It may even be the case +that a new feature introduces new incompatible syntax (such as a new +reserved word), in which case the compiler may need to parse the +module differently. Such decisions cannot be pushed off until +runtime. + +For any given release, the compiler knows which feature names have +been defined, and raises a compile-time error if a future statement +contains a feature not known to it. + +The direct runtime semantics are the same as for any import statement: +there is a standard module "__future__", described later, and it will +be imported in the usual way at the time the future statement is +executed. + +The interesting runtime semantics depend on the specific feature +enabled by the future statement. + +Note that there is nothing special about the statement: + + import __future__ [as name] + +That is not a future statement; it’s an ordinary import statement with +no special semantics or syntax restrictions. + +Code compiled by calls to the built-in functions "exec()" and +"compile()" that occur in a module "M" containing a future statement +will, by default, use the new syntax or semantics associated with the +future statement. This can be controlled by optional arguments to +"compile()" — see the documentation of that function for details. + +A future statement typed at an interactive interpreter prompt will +take effect for the rest of the interpreter session. If an +interpreter is started with the "-i" option, is passed a script name +to execute, and the script includes a future statement, it will be in +effect in the interactive session started after the script is +executed. + +See also: + + **PEP 236** - Back to the __future__ + The original proposal for the __future__ mechanism. +''', + 'in': r'''Membership test operations +************************** + +The operators "in" and "not in" test for membership. "x in s" +evaluates to "True" if *x* is a member of *s*, and "False" otherwise. +"x not in s" returns the negation of "x in s". All built-in sequences +and set types support this as well as dictionary, for which "in" tests +whether the dictionary has a given key. For container types such as +list, tuple, set, frozenset, dict, or collections.deque, the +expression "x in y" is equivalent to "any(x is e or x == e for e in +y)". + +For the string and bytes types, "x in y" is "True" if and only if *x* +is a substring of *y*. An equivalent test is "y.find(x) != -1". +Empty strings are always considered to be a substring of any other +string, so """ in "abc"" will return "True". + +For user-defined classes which define the "__contains__()" method, "x +in y" returns "True" if "y.__contains__(x)" returns a true value, and +"False" otherwise. + +For user-defined classes which do not define "__contains__()" but do +define "__iter__()", "x in y" is "True" if some value "z", for which +the expression "x is z or x == z" is true, is produced while iterating +over "y". If an exception is raised during the iteration, it is as if +"in" raised that exception. + +Lastly, the old-style iteration protocol is tried: if a class defines +"__getitem__()", "x in y" is "True" if and only if there is a non- +negative integer index *i* such that "x is y[i] or x == y[i]", and no +lower integer index raises the "IndexError" exception. (If any other +exception is raised, it is as if "in" raised that exception). + +The operator "not in" is defined to have the inverse truth value of +"in". +''', + 'integers': r'''Integer literals +**************** + +Integer literals are described by the following lexical definitions: + + integer ::= decinteger | bininteger | octinteger | hexinteger + decinteger ::= nonzerodigit (["_"] digit)* | "0"+ (["_"] "0")* + bininteger ::= "0" ("b" | "B") (["_"] bindigit)+ + octinteger ::= "0" ("o" | "O") (["_"] octdigit)+ + hexinteger ::= "0" ("x" | "X") (["_"] hexdigit)+ + nonzerodigit ::= "1"..."9" + digit ::= "0"..."9" + bindigit ::= "0" | "1" + octdigit ::= "0"..."7" + hexdigit ::= digit | "a"..."f" | "A"..."F" + +There is no limit for the length of integer literals apart from what +can be stored in available memory. + +Underscores are ignored for determining the numeric value of the +literal. They can be used to group digits for enhanced readability. +One underscore can occur between digits, and after base specifiers +like "0x". + +Note that leading zeros in a non-zero decimal number are not allowed. +This is for disambiguation with C-style octal literals, which Python +used before version 3.0. + +Some examples of integer literals: + + 7 2147483647 0o177 0b100110111 + 3 79228162514264337593543950336 0o377 0xdeadbeef + 100_000_000_000 0b_1110_0101 + +Changed in version 3.6: Underscores are now allowed for grouping +purposes in literals. +''', + 'lambda': r'''Lambdas +******* + + lambda_expr ::= "lambda" [parameter_list] ":" expression + +Lambda expressions (sometimes called lambda forms) are used to create +anonymous functions. The expression "lambda parameters: expression" +yields a function object. The unnamed object behaves like a function +object defined with: + + def (parameters): + return expression + +See section Function definitions for the syntax of parameter lists. +Note that functions created with lambda expressions cannot contain +statements or annotations. +''', + 'lists': r'''List displays +************* + +A list display is a possibly empty series of expressions enclosed in +square brackets: + + list_display ::= "[" [flexible_expression_list | comprehension] "]" + +A list display yields a new list object, the contents being specified +by either a list of expressions or a comprehension. When a comma- +separated list of expressions is supplied, its elements are evaluated +from left to right and placed into the list object in that order. +When a comprehension is supplied, the list is constructed from the +elements resulting from the comprehension. +''', + 'naming': r'''Naming and binding +****************** + + +Binding of names +================ + +*Names* refer to objects. Names are introduced by name binding +operations. + +The following constructs bind names: + +* formal parameters to functions, + +* class definitions, + +* function definitions, + +* assignment expressions, + +* targets that are identifiers if occurring in an assignment: + + * "for" loop header, + + * after "as" in a "with" statement, "except" clause, "except*" + clause, or in the as-pattern in structural pattern matching, + + * in a capture pattern in structural pattern matching + +* "import" statements. + +* "type" statements. + +* type parameter lists. + +The "import" statement of the form "from ... import *" binds all names +defined in the imported module, except those beginning with an +underscore. This form may only be used at the module level. + +A target occurring in a "del" statement is also considered bound for +this purpose (though the actual semantics are to unbind the name). + +Each assignment or import statement occurs within a block defined by a +class or function definition or at the module level (the top-level +code block). + +If a name is bound in a block, it is a local variable of that block, +unless declared as "nonlocal" or "global". If a name is bound at the +module level, it is a global variable. (The variables of the module +code block are local and global.) If a variable is used in a code +block but not defined there, it is a *free variable*. + +Each occurrence of a name in the program text refers to the *binding* +of that name established by the following name resolution rules. + + +Resolution of names +=================== + +A *scope* defines the visibility of a name within a block. If a local +variable is defined in a block, its scope includes that block. If the +definition occurs in a function block, the scope extends to any blocks +contained within the defining one, unless a contained block introduces +a different binding for the name. + +When a name is used in a code block, it is resolved using the nearest +enclosing scope. The set of all such scopes visible to a code block +is called the block’s *environment*. + +When a name is not found at all, a "NameError" exception is raised. If +the current scope is a function scope, and the name refers to a local +variable that has not yet been bound to a value at the point where the +name is used, an "UnboundLocalError" exception is raised. +"UnboundLocalError" is a subclass of "NameError". + +If a name binding operation occurs anywhere within a code block, all +uses of the name within the block are treated as references to the +current block. This can lead to errors when a name is used within a +block before it is bound. This rule is subtle. Python lacks +declarations and allows name binding operations to occur anywhere +within a code block. The local variables of a code block can be +determined by scanning the entire text of the block for name binding +operations. See the FAQ entry on UnboundLocalError for examples. + +If the "global" statement occurs within a block, all uses of the names +specified in the statement refer to the bindings of those names in the +top-level namespace. Names are resolved in the top-level namespace by +searching the global namespace, i.e. the namespace of the module +containing the code block, and the builtins namespace, the namespace +of the module "builtins". The global namespace is searched first. If +the names are not found there, the builtins namespace is searched +next. If the names are also not found in the builtins namespace, new +variables are created in the global namespace. The global statement +must precede all uses of the listed names. + +The "global" statement has the same scope as a name binding operation +in the same block. If the nearest enclosing scope for a free variable +contains a global statement, the free variable is treated as a global. + +The "nonlocal" statement causes corresponding names to refer to +previously bound variables in the nearest enclosing function scope. +"SyntaxError" is raised at compile time if the given name does not +exist in any enclosing function scope. Type parameters cannot be +rebound with the "nonlocal" statement. + +The namespace for a module is automatically created the first time a +module is imported. The main module for a script is always called +"__main__". + +Class definition blocks and arguments to "exec()" and "eval()" are +special in the context of name resolution. A class definition is an +executable statement that may use and define names. These references +follow the normal rules for name resolution with an exception that +unbound local variables are looked up in the global namespace. The +namespace of the class definition becomes the attribute dictionary of +the class. The scope of names defined in a class block is limited to +the class block; it does not extend to the code blocks of methods. +This includes comprehensions and generator expressions, but it does +not include annotation scopes, which have access to their enclosing +class scopes. This means that the following will fail: + + class A: + a = 42 + b = list(a + i for i in range(10)) + +However, the following will succeed: + + class A: + type Alias = Nested + class Nested: pass + + print(A.Alias.__value__) # + + +Annotation scopes +================= + +*Annotations*, type parameter lists and "type" statements introduce +*annotation scopes*, which behave mostly like function scopes, but +with some exceptions discussed below. + +Annotation scopes are used in the following contexts: + +* *Function annotations*. + +* *Variable annotations*. + +* Type parameter lists for generic type aliases. + +* Type parameter lists for generic functions. A generic function’s + annotations are executed within the annotation scope, but its + defaults and decorators are not. + +* Type parameter lists for generic classes. A generic class’s base + classes and keyword arguments are executed within the annotation + scope, but its decorators are not. + +* The bounds, constraints, and default values for type parameters + (lazily evaluated). + +* The value of type aliases (lazily evaluated). + +Annotation scopes differ from function scopes in the following ways: + +* Annotation scopes have access to their enclosing class namespace. If + an annotation scope is immediately within a class scope, or within + another annotation scope that is immediately within a class scope, + the code in the annotation scope can use names defined in the class + scope as if it were executed directly within the class body. This + contrasts with regular functions defined within classes, which + cannot access names defined in the class scope. + +* Expressions in annotation scopes cannot contain "yield", "yield + from", "await", or ":=" expressions. (These expressions are allowed + in other scopes contained within the annotation scope.) + +* Names defined in annotation scopes cannot be rebound with "nonlocal" + statements in inner scopes. This includes only type parameters, as + no other syntactic elements that can appear within annotation scopes + can introduce new names. + +* While annotation scopes have an internal name, that name is not + reflected in the *qualified name* of objects defined within the + scope. Instead, the "__qualname__" of such objects is as if the + object were defined in the enclosing scope. + +Added in version 3.12: Annotation scopes were introduced in Python +3.12 as part of **PEP 695**. + +Changed in version 3.13: Annotation scopes are also used for type +parameter defaults, as introduced by **PEP 696**. + +Changed in version 3.14: Annotation scopes are now also used for +annotations, as specified in **PEP 649** and **PEP 749**. + + +Lazy evaluation +=============== + +Most annotation scopes are *lazily evaluated*. This includes +annotations, the values of type aliases created through the "type" +statement, and the bounds, constraints, and default values of type +variables created through the type parameter syntax. This means that +they are not evaluated when the type alias or type variable is +created, or when the object carrying annotations is created. Instead, +they are only evaluated when necessary, for example when the +"__value__" attribute on a type alias is accessed. + +Example: + + >>> type Alias = 1/0 + >>> Alias.__value__ + Traceback (most recent call last): + ... + ZeroDivisionError: division by zero + >>> def func[T: 1/0](): pass + >>> T = func.__type_params__[0] + >>> T.__bound__ + Traceback (most recent call last): + ... + ZeroDivisionError: division by zero + +Here the exception is raised only when the "__value__" attribute of +the type alias or the "__bound__" attribute of the type variable is +accessed. + +This behavior is primarily useful for references to types that have +not yet been defined when the type alias or type variable is created. +For example, lazy evaluation enables creation of mutually recursive +type aliases: + + from typing import Literal + + type SimpleExpr = int | Parenthesized + type Parenthesized = tuple[Literal["("], Expr, Literal[")"]] + type Expr = SimpleExpr | tuple[SimpleExpr, Literal["+", "-"], Expr] + +Lazily evaluated values are evaluated in annotation scope, which means +that names that appear inside the lazily evaluated value are looked up +as if they were used in the immediately enclosing scope. + +Added in version 3.12. + + +Builtins and restricted execution +================================= + +**CPython implementation detail:** Users should not touch +"__builtins__"; it is strictly an implementation detail. Users +wanting to override values in the builtins namespace should "import" +the "builtins" module and modify its attributes appropriately. + +The builtins namespace associated with the execution of a code block +is actually found by looking up the name "__builtins__" in its global +namespace; this should be a dictionary or a module (in the latter case +the module’s dictionary is used). By default, when in the "__main__" +module, "__builtins__" is the built-in module "builtins"; when in any +other module, "__builtins__" is an alias for the dictionary of the +"builtins" module itself. + + +Interaction with dynamic features +================================= + +Name resolution of free variables occurs at runtime, not at compile +time. This means that the following code will print 42: + + i = 10 + def f(): + print(i) + i = 42 + f() + +The "eval()" and "exec()" functions do not have access to the full +environment for resolving names. Names may be resolved in the local +and global namespaces of the caller. Free variables are not resolved +in the nearest enclosing namespace, but in the global namespace. [1] +The "exec()" and "eval()" functions have optional arguments to +override the global and local namespace. If only one namespace is +specified, it is used for both. +''', + 'nonlocal': r'''The "nonlocal" statement +************************ + + nonlocal_stmt ::= "nonlocal" identifier ("," identifier)* + +When the definition of a function or class is nested (enclosed) within +the definitions of other functions, its nonlocal scopes are the local +scopes of the enclosing functions. The "nonlocal" statement causes the +listed identifiers to refer to names previously bound in nonlocal +scopes. It allows encapsulated code to rebind such nonlocal +identifiers. If a name is bound in more than one nonlocal scope, the +nearest binding is used. If a name is not bound in any nonlocal scope, +or if there is no nonlocal scope, a "SyntaxError" is raised. + +The "nonlocal" statement applies to the entire scope of a function or +class body. A "SyntaxError" is raised if a variable is used or +assigned to prior to its nonlocal declaration in the scope. + +See also: + + **PEP 3104** - Access to Names in Outer Scopes + The specification for the "nonlocal" statement. + +**Programmer’s note:** "nonlocal" is a directive to the parser and +applies only to code parsed along with it. See the note for the +"global" statement. +''', + 'numbers': r'''Numeric literals +**************** + +There are three types of numeric literals: integers, floating-point +numbers, and imaginary numbers. There are no complex literals +(complex numbers can be formed by adding a real number and an +imaginary number). + +Note that numeric literals do not include a sign; a phrase like "-1" +is actually an expression composed of the unary operator ‘"-"’ and the +literal "1". +''', + 'numeric-types': r'''Emulating numeric types +*********************** + +The following methods can be defined to emulate numeric objects. +Methods corresponding to operations that are not supported by the +particular kind of number implemented (e.g., bitwise operations for +non-integral numbers) should be left undefined. + +object.__add__(self, other) +object.__sub__(self, other) +object.__mul__(self, other) +object.__matmul__(self, other) +object.__truediv__(self, other) +object.__floordiv__(self, other) +object.__mod__(self, other) +object.__divmod__(self, other) +object.__pow__(self, other[, modulo]) +object.__lshift__(self, other) +object.__rshift__(self, other) +object.__and__(self, other) +object.__xor__(self, other) +object.__or__(self, other) + + These methods are called to implement the binary arithmetic + operations ("+", "-", "*", "@", "/", "//", "%", "divmod()", + "pow()", "**", "<<", ">>", "&", "^", "|"). For instance, to + evaluate the expression "x + y", where *x* is an instance of a + class that has an "__add__()" method, "type(x).__add__(x, y)" is + called. The "__divmod__()" method should be the equivalent to + using "__floordiv__()" and "__mod__()"; it should not be related to + "__truediv__()". Note that "__pow__()" should be defined to accept + an optional third argument if the ternary version of the built-in + "pow()" function is to be supported. + + If one of those methods does not support the operation with the + supplied arguments, it should return "NotImplemented". + +object.__radd__(self, other) +object.__rsub__(self, other) +object.__rmul__(self, other) +object.__rmatmul__(self, other) +object.__rtruediv__(self, other) +object.__rfloordiv__(self, other) +object.__rmod__(self, other) +object.__rdivmod__(self, other) +object.__rpow__(self, other[, modulo]) +object.__rlshift__(self, other) +object.__rrshift__(self, other) +object.__rand__(self, other) +object.__rxor__(self, other) +object.__ror__(self, other) + + These methods are called to implement the binary arithmetic + operations ("+", "-", "*", "@", "/", "//", "%", "divmod()", + "pow()", "**", "<<", ">>", "&", "^", "|") with reflected (swapped) + operands. These functions are only called if the operands are of + different types, when the left operand does not support the + corresponding operation [3], or the right operand’s class is + derived from the left operand’s class. [4] For instance, to + evaluate the expression "x - y", where *y* is an instance of a + class that has an "__rsub__()" method, "type(y).__rsub__(y, x)" is + called if "type(x).__sub__(x, y)" returns "NotImplemented" or + "type(y)" is a subclass of "type(x)". [5] + + Note that ternary "pow()" will not try calling "__rpow__()" (the + coercion rules would become too complicated). + + Note: + + If the right operand’s type is a subclass of the left operand’s + type and that subclass provides a different implementation of the + reflected method for the operation, this method will be called + before the left operand’s non-reflected method. This behavior + allows subclasses to override their ancestors’ operations. + +object.__iadd__(self, other) +object.__isub__(self, other) +object.__imul__(self, other) +object.__imatmul__(self, other) +object.__itruediv__(self, other) +object.__ifloordiv__(self, other) +object.__imod__(self, other) +object.__ipow__(self, other[, modulo]) +object.__ilshift__(self, other) +object.__irshift__(self, other) +object.__iand__(self, other) +object.__ixor__(self, other) +object.__ior__(self, other) + + These methods are called to implement the augmented arithmetic + assignments ("+=", "-=", "*=", "@=", "/=", "//=", "%=", "**=", + "<<=", ">>=", "&=", "^=", "|="). These methods should attempt to + do the operation in-place (modifying *self*) and return the result + (which could be, but does not have to be, *self*). If a specific + method is not defined, or if that method returns "NotImplemented", + the augmented assignment falls back to the normal methods. For + instance, if *x* is an instance of a class with an "__iadd__()" + method, "x += y" is equivalent to "x = x.__iadd__(y)" . If + "__iadd__()" does not exist, or if "x.__iadd__(y)" returns + "NotImplemented", "x.__add__(y)" and "y.__radd__(x)" are + considered, as with the evaluation of "x + y". In certain + situations, augmented assignment can result in unexpected errors + (see Why does a_tuple[i] += [‘item’] raise an exception when the + addition works?), but this behavior is in fact part of the data + model. + +object.__neg__(self) +object.__pos__(self) +object.__abs__(self) +object.__invert__(self) + + Called to implement the unary arithmetic operations ("-", "+", + "abs()" and "~"). + +object.__complex__(self) +object.__int__(self) +object.__float__(self) + + Called to implement the built-in functions "complex()", "int()" and + "float()". Should return a value of the appropriate type. + +object.__index__(self) + + Called to implement "operator.index()", and whenever Python needs + to losslessly convert the numeric object to an integer object (such + as in slicing, or in the built-in "bin()", "hex()" and "oct()" + functions). Presence of this method indicates that the numeric + object is an integer type. Must return an integer. + + If "__int__()", "__float__()" and "__complex__()" are not defined + then corresponding built-in functions "int()", "float()" and + "complex()" fall back to "__index__()". + +object.__round__(self[, ndigits]) +object.__trunc__(self) +object.__floor__(self) +object.__ceil__(self) + + Called to implement the built-in function "round()" and "math" + functions "trunc()", "floor()" and "ceil()". Unless *ndigits* is + passed to "__round__()" all these methods should return the value + of the object truncated to an "Integral" (typically an "int"). + + Changed in version 3.14: "int()" no longer delegates to the + "__trunc__()" method. +''', + 'objects': r'''Objects, values and types +************************* + +*Objects* are Python’s abstraction for data. All data in a Python +program is represented by objects or by relations between objects. (In +a sense, and in conformance to Von Neumann’s model of a “stored +program computer”, code is also represented by objects.) + +Every object has an identity, a type and a value. An object’s +*identity* never changes once it has been created; you may think of it +as the object’s address in memory. The "is" operator compares the +identity of two objects; the "id()" function returns an integer +representing its identity. + +**CPython implementation detail:** For CPython, "id(x)" is the memory +address where "x" is stored. + +An object’s type determines the operations that the object supports +(e.g., “does it have a length?”) and also defines the possible values +for objects of that type. The "type()" function returns an object’s +type (which is an object itself). Like its identity, an object’s +*type* is also unchangeable. [1] + +The *value* of some objects can change. Objects whose value can +change are said to be *mutable*; objects whose value is unchangeable +once they are created are called *immutable*. (The value of an +immutable container object that contains a reference to a mutable +object can change when the latter’s value is changed; however the +container is still considered immutable, because the collection of +objects it contains cannot be changed. So, immutability is not +strictly the same as having an unchangeable value, it is more subtle.) +An object’s mutability is determined by its type; for instance, +numbers, strings and tuples are immutable, while dictionaries and +lists are mutable. + +Objects are never explicitly destroyed; however, when they become +unreachable they may be garbage-collected. An implementation is +allowed to postpone garbage collection or omit it altogether — it is a +matter of implementation quality how garbage collection is +implemented, as long as no objects are collected that are still +reachable. + +**CPython implementation detail:** CPython currently uses a reference- +counting scheme with (optional) delayed detection of cyclically linked +garbage, which collects most objects as soon as they become +unreachable, but is not guaranteed to collect garbage containing +circular references. See the documentation of the "gc" module for +information on controlling the collection of cyclic garbage. Other +implementations act differently and CPython may change. Do not depend +on immediate finalization of objects when they become unreachable (so +you should always close files explicitly). + +Note that the use of the implementation’s tracing or debugging +facilities may keep objects alive that would normally be collectable. +Also note that catching an exception with a "try"…"except" statement +may keep objects alive. + +Some objects contain references to “external” resources such as open +files or windows. It is understood that these resources are freed +when the object is garbage-collected, but since garbage collection is +not guaranteed to happen, such objects also provide an explicit way to +release the external resource, usually a "close()" method. Programs +are strongly recommended to explicitly close such objects. The +"try"…"finally" statement and the "with" statement provide convenient +ways to do this. + +Some objects contain references to other objects; these are called +*containers*. Examples of containers are tuples, lists and +dictionaries. The references are part of a container’s value. In +most cases, when we talk about the value of a container, we imply the +values, not the identities of the contained objects; however, when we +talk about the mutability of a container, only the identities of the +immediately contained objects are implied. So, if an immutable +container (like a tuple) contains a reference to a mutable object, its +value changes if that mutable object is changed. + +Types affect almost all aspects of object behavior. Even the +importance of object identity is affected in some sense: for immutable +types, operations that compute new values may actually return a +reference to any existing object with the same type and value, while +for mutable objects this is not allowed. For example, after "a = 1; b += 1", *a* and *b* may or may not refer to the same object with the +value one, depending on the implementation. This is because "int" is +an immutable type, so the reference to "1" can be reused. This +behaviour depends on the implementation used, so should not be relied +upon, but is something to be aware of when making use of object +identity tests. However, after "c = []; d = []", *c* and *d* are +guaranteed to refer to two different, unique, newly created empty +lists. (Note that "e = f = []" assigns the *same* object to both *e* +and *f*.) +''', + 'operator-summary': r'''Operator precedence +******************* + +The following table summarizes the operator precedence in Python, from +highest precedence (most binding) to lowest precedence (least +binding). Operators in the same box have the same precedence. Unless +the syntax is explicitly given, operators are binary. Operators in +the same box group left to right (except for exponentiation and +conditional expressions, which group from right to left). + +Note that comparisons, membership tests, and identity tests, all have +the same precedence and have a left-to-right chaining feature as +described in the Comparisons section. + ++-------------------------------------------------+---------------------------------------+ +| Operator | Description | +|=================================================|=======================================| +| "(expressions...)", "[expressions...]", "{key: | Binding or parenthesized expression, | +| value...}", "{expressions...}" | list display, dictionary display, set | +| | display | ++-------------------------------------------------+---------------------------------------+ +| "x[index]", "x[index:index]", | Subscription, slicing, call, | +| "x(arguments...)", "x.attribute" | attribute reference | ++-------------------------------------------------+---------------------------------------+ +| "await x" | Await expression | ++-------------------------------------------------+---------------------------------------+ +| "**" | Exponentiation [5] | ++-------------------------------------------------+---------------------------------------+ +| "+x", "-x", "~x" | Positive, negative, bitwise NOT | ++-------------------------------------------------+---------------------------------------+ +| "*", "@", "/", "//", "%" | Multiplication, matrix | +| | multiplication, division, floor | +| | division, remainder [6] | ++-------------------------------------------------+---------------------------------------+ +| "+", "-" | Addition and subtraction | ++-------------------------------------------------+---------------------------------------+ +| "<<", ">>" | Shifts | ++-------------------------------------------------+---------------------------------------+ +| "&" | Bitwise AND | ++-------------------------------------------------+---------------------------------------+ +| "^" | Bitwise XOR | ++-------------------------------------------------+---------------------------------------+ +| "|" | Bitwise OR | ++-------------------------------------------------+---------------------------------------+ +| "in", "not in", "is", "is not", "<", "<=", ">", | Comparisons, including membership | +| ">=", "!=", "==" | tests and identity tests | ++-------------------------------------------------+---------------------------------------+ +| "not x" | Boolean NOT | ++-------------------------------------------------+---------------------------------------+ +| "and" | Boolean AND | ++-------------------------------------------------+---------------------------------------+ +| "or" | Boolean OR | ++-------------------------------------------------+---------------------------------------+ +| "if" – "else" | Conditional expression | ++-------------------------------------------------+---------------------------------------+ +| "lambda" | Lambda expression | ++-------------------------------------------------+---------------------------------------+ +| ":=" | Assignment expression | ++-------------------------------------------------+---------------------------------------+ + +-[ Footnotes ]- + +[1] While "abs(x%y) < abs(y)" is true mathematically, for floats it + may not be true numerically due to roundoff. For example, and + assuming a platform on which a Python float is an IEEE 754 double- + precision number, in order that "-1e-100 % 1e100" have the same + sign as "1e100", the computed result is "-1e-100 + 1e100", which + is numerically exactly equal to "1e100". The function + "math.fmod()" returns a result whose sign matches the sign of the + first argument instead, and so returns "-1e-100" in this case. + Which approach is more appropriate depends on the application. + +[2] If x is very close to an exact integer multiple of y, it’s + possible for "x//y" to be one larger than "(x-x%y)//y" due to + rounding. In such cases, Python returns the latter result, in + order to preserve that "divmod(x,y)[0] * y + x % y" be very close + to "x". + +[3] The Unicode standard distinguishes between *code points* (e.g. + U+0041) and *abstract characters* (e.g. “LATIN CAPITAL LETTER A”). + While most abstract characters in Unicode are only represented + using one code point, there is a number of abstract characters + that can in addition be represented using a sequence of more than + one code point. For example, the abstract character “LATIN + CAPITAL LETTER C WITH CEDILLA” can be represented as a single + *precomposed character* at code position U+00C7, or as a sequence + of a *base character* at code position U+0043 (LATIN CAPITAL + LETTER C), followed by a *combining character* at code position + U+0327 (COMBINING CEDILLA). + + The comparison operators on strings compare at the level of + Unicode code points. This may be counter-intuitive to humans. For + example, ""\u00C7" == "\u0043\u0327"" is "False", even though both + strings represent the same abstract character “LATIN CAPITAL + LETTER C WITH CEDILLA”. + + To compare strings at the level of abstract characters (that is, + in a way intuitive to humans), use "unicodedata.normalize()". + +[4] Due to automatic garbage-collection, free lists, and the dynamic + nature of descriptors, you may notice seemingly unusual behaviour + in certain uses of the "is" operator, like those involving + comparisons between instance methods, or constants. Check their + documentation for more info. + +[5] The power operator "**" binds less tightly than an arithmetic or + bitwise unary operator on its right, that is, "2**-1" is "0.5". + +[6] The "%" operator is also used for string formatting; the same + precedence applies. +''', + 'pass': r'''The "pass" statement +******************** + + pass_stmt ::= "pass" + +"pass" is a null operation — when it is executed, nothing happens. It +is useful as a placeholder when a statement is required syntactically, +but no code needs to be executed, for example: + + def f(arg): pass # a function that does nothing (yet) + + class C: pass # a class with no methods (yet) +''', + 'power': r'''The power operator +****************** + +The power operator binds more tightly than unary operators on its +left; it binds less tightly than unary operators on its right. The +syntax is: + + power ::= (await_expr | primary) ["**" u_expr] + +Thus, in an unparenthesized sequence of power and unary operators, the +operators are evaluated from right to left (this does not constrain +the evaluation order for the operands): "-1**2" results in "-1". + +The power operator has the same semantics as the built-in "pow()" +function, when called with two arguments: it yields its left argument +raised to the power of its right argument. The numeric arguments are +first converted to a common type, and the result is of that type. + +For int operands, the result has the same type as the operands unless +the second argument is negative; in that case, all arguments are +converted to float and a float result is delivered. For example, +"10**2" returns "100", but "10**-2" returns "0.01". + +Raising "0.0" to a negative power results in a "ZeroDivisionError". +Raising a negative number to a fractional power results in a "complex" +number. (In earlier versions it raised a "ValueError".) + +This operation can be customized using the special "__pow__()" and +"__rpow__()" methods. +''', + 'raise': r'''The "raise" statement +********************* + + raise_stmt ::= "raise" [expression ["from" expression]] + +If no expressions are present, "raise" re-raises the exception that is +currently being handled, which is also known as the *active +exception*. If there isn’t currently an active exception, a +"RuntimeError" exception is raised indicating that this is an error. + +Otherwise, "raise" evaluates the first expression as the exception +object. It must be either a subclass or an instance of +"BaseException". If it is a class, the exception instance will be +obtained when needed by instantiating the class with no arguments. + +The *type* of the exception is the exception instance’s class, the +*value* is the instance itself. + +A traceback object is normally created automatically when an exception +is raised and attached to it as the "__traceback__" attribute. You can +create an exception and set your own traceback in one step using the +"with_traceback()" exception method (which returns the same exception +instance, with its traceback set to its argument), like so: + + raise Exception("foo occurred").with_traceback(tracebackobj) + +The "from" clause is used for exception chaining: if given, the second +*expression* must be another exception class or instance. If the +second expression is an exception instance, it will be attached to the +raised exception as the "__cause__" attribute (which is writable). If +the expression is an exception class, the class will be instantiated +and the resulting exception instance will be attached to the raised +exception as the "__cause__" attribute. If the raised exception is not +handled, both exceptions will be printed: + + >>> try: + ... print(1 / 0) + ... except Exception as exc: + ... raise RuntimeError("Something bad happened") from exc + ... + Traceback (most recent call last): + File "", line 2, in + print(1 / 0) + ~~^~~ + ZeroDivisionError: division by zero + + The above exception was the direct cause of the following exception: + + Traceback (most recent call last): + File "", line 4, in + raise RuntimeError("Something bad happened") from exc + RuntimeError: Something bad happened + +A similar mechanism works implicitly if a new exception is raised when +an exception is already being handled. An exception may be handled +when an "except" or "finally" clause, or a "with" statement, is used. +The previous exception is then attached as the new exception’s +"__context__" attribute: + + >>> try: + ... print(1 / 0) + ... except: + ... raise RuntimeError("Something bad happened") + ... + Traceback (most recent call last): + File "", line 2, in + print(1 / 0) + ~~^~~ + ZeroDivisionError: division by zero + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "", line 4, in + raise RuntimeError("Something bad happened") + RuntimeError: Something bad happened + +Exception chaining can be explicitly suppressed by specifying "None" +in the "from" clause: + + >>> try: + ... print(1 / 0) + ... except: + ... raise RuntimeError("Something bad happened") from None + ... + Traceback (most recent call last): + File "", line 4, in + RuntimeError: Something bad happened + +Additional information on exceptions can be found in section +Exceptions, and information about handling exceptions is in section +The try statement. + +Changed in version 3.3: "None" is now permitted as "Y" in "raise X +from Y".Added the "__suppress_context__" attribute to suppress +automatic display of the exception context. + +Changed in version 3.11: If the traceback of the active exception is +modified in an "except" clause, a subsequent "raise" statement re- +raises the exception with the modified traceback. Previously, the +exception was re-raised with the traceback it had when it was caught. +''', + 'return': r'''The "return" statement +********************** + + return_stmt ::= "return" [expression_list] + +"return" may only occur syntactically nested in a function definition, +not within a nested class definition. + +If an expression list is present, it is evaluated, else "None" is +substituted. + +"return" leaves the current function call with the expression list (or +"None") as return value. + +When "return" passes control out of a "try" statement with a "finally" +clause, that "finally" clause is executed before really leaving the +function. + +In a generator function, the "return" statement indicates that the +generator is done and will cause "StopIteration" to be raised. The +returned value (if any) is used as an argument to construct +"StopIteration" and becomes the "StopIteration.value" attribute. + +In an asynchronous generator function, an empty "return" statement +indicates that the asynchronous generator is done and will cause +"StopAsyncIteration" to be raised. A non-empty "return" statement is +a syntax error in an asynchronous generator function. +''', + 'sequence-types': r'''Emulating container types +************************* + +The following methods can be defined to implement container objects. +None of them are provided by the "object" class itself. Containers +usually are *sequences* (such as "lists" or "tuples") or *mappings* +(like *dictionaries*), but can represent other containers as well. +The first set of methods is used either to emulate a sequence or to +emulate a mapping; the difference is that for a sequence, the +allowable keys should be the integers *k* for which "0 <= k < N" where +*N* is the length of the sequence, or "slice" objects, which define a +range of items. It is also recommended that mappings provide the +methods "keys()", "values()", "items()", "get()", "clear()", +"setdefault()", "pop()", "popitem()", "copy()", and "update()" +behaving similar to those for Python’s standard "dictionary" objects. +The "collections.abc" module provides a "MutableMapping" *abstract +base class* to help create those methods from a base set of +"__getitem__()", "__setitem__()", "__delitem__()", and "keys()". +Mutable sequences should provide methods "append()", "count()", +"index()", "extend()", "insert()", "pop()", "remove()", "reverse()" +and "sort()", like Python standard "list" objects. Finally, sequence +types should implement addition (meaning concatenation) and +multiplication (meaning repetition) by defining the methods +"__add__()", "__radd__()", "__iadd__()", "__mul__()", "__rmul__()" and +"__imul__()" described below; they should not define other numerical +operators. It is recommended that both mappings and sequences +implement the "__contains__()" method to allow efficient use of the +"in" operator; for mappings, "in" should search the mapping’s keys; +for sequences, it should search through the values. It is further +recommended that both mappings and sequences implement the +"__iter__()" method to allow efficient iteration through the +container; for mappings, "__iter__()" should iterate through the +object’s keys; for sequences, it should iterate through the values. + +object.__len__(self) + + Called to implement the built-in function "len()". Should return + the length of the object, an integer ">=" 0. Also, an object that + doesn’t define a "__bool__()" method and whose "__len__()" method + returns zero is considered to be false in a Boolean context. + + **CPython implementation detail:** In CPython, the length is + required to be at most "sys.maxsize". If the length is larger than + "sys.maxsize" some features (such as "len()") may raise + "OverflowError". To prevent raising "OverflowError" by truth value + testing, an object must define a "__bool__()" method. + +object.__length_hint__(self) + + Called to implement "operator.length_hint()". Should return an + estimated length for the object (which may be greater or less than + the actual length). The length must be an integer ">=" 0. The + return value may also be "NotImplemented", which is treated the + same as if the "__length_hint__" method didn’t exist at all. This + method is purely an optimization and is never required for + correctness. + + Added in version 3.4. + +Note: + + Slicing is done exclusively with the following three methods. A + call like + + a[1:2] = b + + is translated to + + a[slice(1, 2, None)] = b + + and so forth. Missing slice items are always filled in with "None". + +object.__getitem__(self, key) + + Called to implement evaluation of "self[key]". For *sequence* + types, the accepted keys should be integers. Optionally, they may + support "slice" objects as well. Negative index support is also + optional. If *key* is of an inappropriate type, "TypeError" may be + raised; if *key* is a value outside the set of indexes for the + sequence (after any special interpretation of negative values), + "IndexError" should be raised. For *mapping* types, if *key* is + missing (not in the container), "KeyError" should be raised. + + Note: + + "for" loops expect that an "IndexError" will be raised for + illegal indexes to allow proper detection of the end of the + sequence. + + Note: + + When subscripting a *class*, the special class method + "__class_getitem__()" may be called instead of "__getitem__()". + See __class_getitem__ versus __getitem__ for more details. + +object.__setitem__(self, key, value) + + Called to implement assignment to "self[key]". Same note as for + "__getitem__()". This should only be implemented for mappings if + the objects support changes to the values for keys, or if new keys + can be added, or for sequences if elements can be replaced. The + same exceptions should be raised for improper *key* values as for + the "__getitem__()" method. + +object.__delitem__(self, key) + + Called to implement deletion of "self[key]". Same note as for + "__getitem__()". This should only be implemented for mappings if + the objects support removal of keys, or for sequences if elements + can be removed from the sequence. The same exceptions should be + raised for improper *key* values as for the "__getitem__()" method. + +object.__missing__(self, key) + + Called by "dict"."__getitem__()" to implement "self[key]" for dict + subclasses when key is not in the dictionary. + +object.__iter__(self) + + This method is called when an *iterator* is required for a + container. This method should return a new iterator object that can + iterate over all the objects in the container. For mappings, it + should iterate over the keys of the container. + +object.__reversed__(self) + + Called (if present) by the "reversed()" built-in to implement + reverse iteration. It should return a new iterator object that + iterates over all the objects in the container in reverse order. + + If the "__reversed__()" method is not provided, the "reversed()" + built-in will fall back to using the sequence protocol ("__len__()" + and "__getitem__()"). Objects that support the sequence protocol + should only provide "__reversed__()" if they can provide an + implementation that is more efficient than the one provided by + "reversed()". + +The membership test operators ("in" and "not in") are normally +implemented as an iteration through a container. However, container +objects can supply the following special method with a more efficient +implementation, which also does not require the object be iterable. + +object.__contains__(self, item) + + Called to implement membership test operators. Should return true + if *item* is in *self*, false otherwise. For mapping objects, this + should consider the keys of the mapping rather than the values or + the key-item pairs. + + For objects that don’t define "__contains__()", the membership test + first tries iteration via "__iter__()", then the old sequence + iteration protocol via "__getitem__()", see this section in the + language reference. +''', + 'shifting': r'''Shifting operations +******************* + +The shifting operations have lower priority than the arithmetic +operations: + + shift_expr ::= a_expr | shift_expr ("<<" | ">>") a_expr + +These operators accept integers as arguments. They shift the first +argument to the left or right by the number of bits given by the +second argument. + +The left shift operation can be customized using the special +"__lshift__()" and "__rlshift__()" methods. The right shift operation +can be customized using the special "__rshift__()" and "__rrshift__()" +methods. + +A right shift by *n* bits is defined as floor division by "pow(2,n)". +A left shift by *n* bits is defined as multiplication with "pow(2,n)". +''', + 'slicings': r'''Slicings +******** + +A slicing selects a range of items in a sequence object (e.g., a +string, tuple or list). Slicings may be used as expressions or as +targets in assignment or "del" statements. The syntax for a slicing: + + slicing ::= primary "[" slice_list "]" + slice_list ::= slice_item ("," slice_item)* [","] + slice_item ::= expression | proper_slice + proper_slice ::= [lower_bound] ":" [upper_bound] [ ":" [stride] ] + lower_bound ::= expression + upper_bound ::= expression + stride ::= expression + +There is ambiguity in the formal syntax here: anything that looks like +an expression list also looks like a slice list, so any subscription +can be interpreted as a slicing. Rather than further complicating the +syntax, this is disambiguated by defining that in this case the +interpretation as a subscription takes priority over the +interpretation as a slicing (this is the case if the slice list +contains no proper slice). + +The semantics for a slicing are as follows. The primary is indexed +(using the same "__getitem__()" method as normal subscription) with a +key that is constructed from the slice list, as follows. If the slice +list contains at least one comma, the key is a tuple containing the +conversion of the slice items; otherwise, the conversion of the lone +slice item is the key. The conversion of a slice item that is an +expression is that expression. The conversion of a proper slice is a +slice object (see section The standard type hierarchy) whose "start", +"stop" and "step" attributes are the values of the expressions given +as lower bound, upper bound and stride, respectively, substituting +"None" for missing expressions. +''', + 'specialattrs': r'''Special Attributes +****************** + +The implementation adds a few special read-only attributes to several +object types, where they are relevant. Some of these are not reported +by the "dir()" built-in function. + +definition.__name__ + + The name of the class, function, method, descriptor, or generator + instance. + +definition.__qualname__ + + The *qualified name* of the class, function, method, descriptor, or + generator instance. + + Added in version 3.3. + +definition.__module__ + + The name of the module in which a class or function was defined. + +definition.__doc__ + + The documentation string of a class or function, or "None" if + undefined. + +definition.__type_params__ + + The type parameters of generic classes, functions, and type + aliases. For classes and functions that are not generic, this will + be an empty tuple. + + Added in version 3.12. +''', + 'specialnames': r'''Special method names +******************** + +A class can implement certain operations that are invoked by special +syntax (such as arithmetic operations or subscripting and slicing) by +defining methods with special names. This is Python’s approach to +*operator overloading*, allowing classes to define their own behavior +with respect to language operators. For instance, if a class defines +a method named "__getitem__()", and "x" is an instance of this class, +then "x[i]" is roughly equivalent to "type(x).__getitem__(x, i)". +Except where mentioned, attempts to execute an operation raise an +exception when no appropriate method is defined (typically +"AttributeError" or "TypeError"). + +Setting a special method to "None" indicates that the corresponding +operation is not available. For example, if a class sets "__iter__()" +to "None", the class is not iterable, so calling "iter()" on its +instances will raise a "TypeError" (without falling back to +"__getitem__()"). [2] + +When implementing a class that emulates any built-in type, it is +important that the emulation only be implemented to the degree that it +makes sense for the object being modelled. For example, some +sequences may work well with retrieval of individual elements, but +extracting a slice may not make sense. (One example of this is the +"NodeList" interface in the W3C’s Document Object Model.) + + +Basic customization +=================== + +object.__new__(cls[, ...]) + + Called to create a new instance of class *cls*. "__new__()" is a + static method (special-cased so you need not declare it as such) + that takes the class of which an instance was requested as its + first argument. The remaining arguments are those passed to the + object constructor expression (the call to the class). The return + value of "__new__()" should be the new object instance (usually an + instance of *cls*). + + Typical implementations create a new instance of the class by + invoking the superclass’s "__new__()" method using + "super().__new__(cls[, ...])" with appropriate arguments and then + modifying the newly created instance as necessary before returning + it. + + If "__new__()" is invoked during object construction and it returns + an instance of *cls*, then the new instance’s "__init__()" method + will be invoked like "__init__(self[, ...])", where *self* is the + new instance and the remaining arguments are the same as were + passed to the object constructor. + + If "__new__()" does not return an instance of *cls*, then the new + instance’s "__init__()" method will not be invoked. + + "__new__()" is intended mainly to allow subclasses of immutable + types (like int, str, or tuple) to customize instance creation. It + is also commonly overridden in custom metaclasses in order to + customize class creation. + +object.__init__(self[, ...]) + + Called after the instance has been created (by "__new__()"), but + before it is returned to the caller. The arguments are those + passed to the class constructor expression. If a base class has an + "__init__()" method, the derived class’s "__init__()" method, if + any, must explicitly call it to ensure proper initialization of the + base class part of the instance; for example: + "super().__init__([args...])". + + Because "__new__()" and "__init__()" work together in constructing + objects ("__new__()" to create it, and "__init__()" to customize + it), no non-"None" value may be returned by "__init__()"; doing so + will cause a "TypeError" to be raised at runtime. + +object.__del__(self) + + Called when the instance is about to be destroyed. This is also + called a finalizer or (improperly) a destructor. If a base class + has a "__del__()" method, the derived class’s "__del__()" method, + if any, must explicitly call it to ensure proper deletion of the + base class part of the instance. + + It is possible (though not recommended!) for the "__del__()" method + to postpone destruction of the instance by creating a new reference + to it. This is called object *resurrection*. It is + implementation-dependent whether "__del__()" is called a second + time when a resurrected object is about to be destroyed; the + current *CPython* implementation only calls it once. + + It is not guaranteed that "__del__()" methods are called for + objects that still exist when the interpreter exits. + "weakref.finalize" provides a straightforward way to register a + cleanup function to be called when an object is garbage collected. + + Note: + + "del x" doesn’t directly call "x.__del__()" — the former + decrements the reference count for "x" by one, and the latter is + only called when "x"’s reference count reaches zero. + + **CPython implementation detail:** It is possible for a reference + cycle to prevent the reference count of an object from going to + zero. In this case, the cycle will be later detected and deleted + by the *cyclic garbage collector*. A common cause of reference + cycles is when an exception has been caught in a local variable. + The frame’s locals then reference the exception, which references + its own traceback, which references the locals of all frames caught + in the traceback. + + See also: Documentation for the "gc" module. + + Warning: + + Due to the precarious circumstances under which "__del__()" + methods are invoked, exceptions that occur during their execution + are ignored, and a warning is printed to "sys.stderr" instead. + In particular: + + * "__del__()" can be invoked when arbitrary code is being + executed, including from any arbitrary thread. If "__del__()" + needs to take a lock or invoke any other blocking resource, it + may deadlock as the resource may already be taken by the code + that gets interrupted to execute "__del__()". + + * "__del__()" can be executed during interpreter shutdown. As a + consequence, the global variables it needs to access (including + other modules) may already have been deleted or set to "None". + Python guarantees that globals whose name begins with a single + underscore are deleted from their module before other globals + are deleted; if no other references to such globals exist, this + may help in assuring that imported modules are still available + at the time when the "__del__()" method is called. + +object.__repr__(self) + + Called by the "repr()" built-in function to compute the “official” + string representation of an object. If at all possible, this + should look like a valid Python expression that could be used to + recreate an object with the same value (given an appropriate + environment). If this is not possible, a string of the form + "<...some useful description...>" should be returned. The return + value must be a string object. If a class defines "__repr__()" but + not "__str__()", then "__repr__()" is also used when an “informal” + string representation of instances of that class is required. + + This is typically used for debugging, so it is important that the + representation is information-rich and unambiguous. A default + implementation is provided by the "object" class itself. + +object.__str__(self) + + Called by "str(object)", the default "__format__()" implementation, + and the built-in function "print()", to compute the “informal” or + nicely printable string representation of an object. The return + value must be a str object. + + This method differs from "object.__repr__()" in that there is no + expectation that "__str__()" return a valid Python expression: a + more convenient or concise representation can be used. + + The default implementation defined by the built-in type "object" + calls "object.__repr__()". + +object.__bytes__(self) + + Called by bytes to compute a byte-string representation of an + object. This should return a "bytes" object. The "object" class + itself does not provide this method. + +object.__format__(self, format_spec) + + Called by the "format()" built-in function, and by extension, + evaluation of formatted string literals and the "str.format()" + method, to produce a “formatted” string representation of an + object. The *format_spec* argument is a string that contains a + description of the formatting options desired. The interpretation + of the *format_spec* argument is up to the type implementing + "__format__()", however most classes will either delegate + formatting to one of the built-in types, or use a similar + formatting option syntax. + + See Format Specification Mini-Language for a description of the + standard formatting syntax. + + The return value must be a string object. + + The default implementation by the "object" class should be given an + empty *format_spec* string. It delegates to "__str__()". + + Changed in version 3.4: The __format__ method of "object" itself + raises a "TypeError" if passed any non-empty string. + + Changed in version 3.7: "object.__format__(x, '')" is now + equivalent to "str(x)" rather than "format(str(x), '')". + +object.__lt__(self, other) +object.__le__(self, other) +object.__eq__(self, other) +object.__ne__(self, other) +object.__gt__(self, other) +object.__ge__(self, other) + + These are the so-called “rich comparison” methods. The + correspondence between operator symbols and method names is as + follows: "xy" calls + "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)". + + A rich comparison method may return the singleton "NotImplemented" + if it does not implement the operation for a given pair of + arguments. By convention, "False" and "True" are returned for a + successful comparison. However, these methods can return any value, + so if the comparison operator is used in a Boolean context (e.g., + in the condition of an "if" statement), Python will call "bool()" + on the value to determine if the result is true or false. + + By default, "object" implements "__eq__()" by using "is", returning + "NotImplemented" in the case of a false comparison: "True if x is y + else NotImplemented". For "__ne__()", by default it delegates to + "__eq__()" and inverts the result unless it is "NotImplemented". + There are no other implied relationships among the comparison + operators or default implementations; for example, the truth of + "(x.__hash__". + + If a class that does not override "__eq__()" wishes to suppress + hash support, it should include "__hash__ = None" in the class + definition. A class which defines its own "__hash__()" that + explicitly raises a "TypeError" would be incorrectly identified as + hashable by an "isinstance(obj, collections.abc.Hashable)" call. + + Note: + + By default, the "__hash__()" values of str and bytes objects are + “salted” with an unpredictable random value. Although they + remain constant within an individual Python process, they are not + predictable between repeated invocations of Python.This is + intended to provide protection against a denial-of-service caused + by carefully chosen inputs that exploit the worst case + performance of a dict insertion, *O*(*n*^2) complexity. See + http://ocert.org/advisories/ocert-2011-003.html for + details.Changing hash values affects the iteration order of sets. + Python has never made guarantees about this ordering (and it + typically varies between 32-bit and 64-bit builds).See also + "PYTHONHASHSEED". + + Changed in version 3.3: Hash randomization is enabled by default. + +object.__bool__(self) + + Called to implement truth value testing and the built-in operation + "bool()"; should return "False" or "True". When this method is not + defined, "__len__()" is called, if it is defined, and the object is + considered true if its result is nonzero. If a class defines + neither "__len__()" nor "__bool__()" (which is true of the "object" + class itself), all its instances are considered true. + + +Customizing attribute access +============================ + +The following methods can be defined to customize the meaning of +attribute access (use of, assignment to, or deletion of "x.name") for +class instances. + +object.__getattr__(self, name) + + Called when the default attribute access fails with an + "AttributeError" (either "__getattribute__()" raises an + "AttributeError" because *name* is not an instance attribute or an + attribute in the class tree for "self"; or "__get__()" of a *name* + property raises "AttributeError"). This method should either + return the (computed) attribute value or raise an "AttributeError" + exception. The "object" class itself does not provide this method. + + Note that if the attribute is found through the normal mechanism, + "__getattr__()" is not called. (This is an intentional asymmetry + between "__getattr__()" and "__setattr__()".) This is done both for + efficiency reasons and because otherwise "__getattr__()" would have + no way to access other attributes of the instance. Note that at + least for instance variables, you can take total control by not + inserting any values in the instance attribute dictionary (but + instead inserting them in another object). See the + "__getattribute__()" method below for a way to actually get total + control over attribute access. + +object.__getattribute__(self, name) + + Called unconditionally to implement attribute accesses for + instances of the class. If the class also defines "__getattr__()", + the latter will not be called unless "__getattribute__()" either + calls it explicitly or raises an "AttributeError". This method + should return the (computed) attribute value or raise an + "AttributeError" exception. In order to avoid infinite recursion in + this method, its implementation should always call the base class + method with the same name to access any attributes it needs, for + example, "object.__getattribute__(self, name)". + + Note: + + This method may still be bypassed when looking up special methods + as the result of implicit invocation via language syntax or + built-in functions. See Special method lookup. + + For certain sensitive attribute accesses, raises an auditing event + "object.__getattr__" with arguments "obj" and "name". + +object.__setattr__(self, name, value) + + Called when an attribute assignment is attempted. This is called + instead of the normal mechanism (i.e. store the value in the + instance dictionary). *name* is the attribute name, *value* is the + value to be assigned to it. + + If "__setattr__()" wants to assign to an instance attribute, it + should call the base class method with the same name, for example, + "object.__setattr__(self, name, value)". + + For certain sensitive attribute assignments, raises an auditing + event "object.__setattr__" with arguments "obj", "name", "value". + +object.__delattr__(self, name) + + Like "__setattr__()" but for attribute deletion instead of + assignment. This should only be implemented if "del obj.name" is + meaningful for the object. + + For certain sensitive attribute deletions, raises an auditing event + "object.__delattr__" with arguments "obj" and "name". + +object.__dir__(self) + + Called when "dir()" is called on the object. An iterable must be + returned. "dir()" converts the returned iterable to a list and + sorts it. + + +Customizing module attribute access +----------------------------------- + +Special names "__getattr__" and "__dir__" can be also used to +customize access to module attributes. The "__getattr__" function at +the module level should accept one argument which is the name of an +attribute and return the computed value or raise an "AttributeError". +If an attribute is not found on a module object through the normal +lookup, i.e. "object.__getattribute__()", then "__getattr__" is +searched in the module "__dict__" before raising an "AttributeError". +If found, it is called with the attribute name and the result is +returned. + +The "__dir__" function should accept no arguments, and return an +iterable of strings that represents the names accessible on module. If +present, this function overrides the standard "dir()" search on a +module. + +For a more fine grained customization of the module behavior (setting +attributes, properties, etc.), one can set the "__class__" attribute +of a module object to a subclass of "types.ModuleType". For example: + + import sys + from types import ModuleType + + class VerboseModule(ModuleType): + def __repr__(self): + return f'Verbose {self.__name__}' + + def __setattr__(self, attr, value): + print(f'Setting {attr}...') + super().__setattr__(attr, value) + + sys.modules[__name__].__class__ = VerboseModule + +Note: + + Defining module "__getattr__" and setting module "__class__" only + affect lookups made using the attribute access syntax – directly + accessing the module globals (whether by code within the module, or + via a reference to the module’s globals dictionary) is unaffected. + +Changed in version 3.5: "__class__" module attribute is now writable. + +Added in version 3.7: "__getattr__" and "__dir__" module attributes. + +See also: + + **PEP 562** - Module __getattr__ and __dir__ + Describes the "__getattr__" and "__dir__" functions on modules. + + +Implementing Descriptors +------------------------ + +The following methods only apply when an instance of the class +containing the method (a so-called *descriptor* class) appears in an +*owner* class (the descriptor must be in either the owner’s class +dictionary or in the class dictionary for one of its parents). In the +examples below, “the attribute” refers to the attribute whose name is +the key of the property in the owner class’ "__dict__". The "object" +class itself does not implement any of these protocols. + +object.__get__(self, instance, owner=None) + + Called to get the attribute of the owner class (class attribute + access) or of an instance of that class (instance attribute + access). The optional *owner* argument is the owner class, while + *instance* is the instance that the attribute was accessed through, + or "None" when the attribute is accessed through the *owner*. + + This method should return the computed attribute value or raise an + "AttributeError" exception. + + **PEP 252** specifies that "__get__()" is callable with one or two + arguments. Python’s own built-in descriptors support this + specification; however, it is likely that some third-party tools + have descriptors that require both arguments. Python’s own + "__getattribute__()" implementation always passes in both arguments + whether they are required or not. + +object.__set__(self, instance, value) + + Called to set the attribute on an instance *instance* of the owner + class to a new value, *value*. + + Note, adding "__set__()" or "__delete__()" changes the kind of + descriptor to a “data descriptor”. See Invoking Descriptors for + more details. + +object.__delete__(self, instance) + + Called to delete the attribute on an instance *instance* of the + owner class. + +Instances of descriptors may also have the "__objclass__" attribute +present: + +object.__objclass__ + + The attribute "__objclass__" is interpreted by the "inspect" module + as specifying the class where this object was defined (setting this + appropriately can assist in runtime introspection of dynamic class + attributes). For callables, it may indicate that an instance of the + given type (or a subclass) is expected or required as the first + positional argument (for example, CPython sets this attribute for + unbound methods that are implemented in C). + + +Invoking Descriptors +-------------------- + +In general, a descriptor is an object attribute with “binding +behavior”, one whose attribute access has been overridden by methods +in the descriptor protocol: "__get__()", "__set__()", and +"__delete__()". If any of those methods are defined for an object, it +is said to be a descriptor. + +The default behavior for attribute access is to get, set, or delete +the attribute from an object’s dictionary. For instance, "a.x" has a +lookup chain starting with "a.__dict__['x']", then +"type(a).__dict__['x']", and continuing through the base classes of +"type(a)" excluding metaclasses. + +However, if the looked-up value is an object defining one of the +descriptor methods, then Python may override the default behavior and +invoke the descriptor method instead. Where this occurs in the +precedence chain depends on which descriptor methods were defined and +how they were called. + +The starting point for descriptor invocation is a binding, "a.x". How +the arguments are assembled depends on "a": + +Direct Call + The simplest and least common call is when user code directly + invokes a descriptor method: "x.__get__(a)". + +Instance Binding + If binding to an object instance, "a.x" is transformed into the + call: "type(a).__dict__['x'].__get__(a, type(a))". + +Class Binding + If binding to a class, "A.x" is transformed into the call: + "A.__dict__['x'].__get__(None, A)". + +Super Binding + A dotted lookup such as "super(A, a).x" searches + "a.__class__.__mro__" for a base class "B" following "A" and then + returns "B.__dict__['x'].__get__(a, A)". If not a descriptor, "x" + is returned unchanged. + +For instance bindings, the precedence of descriptor invocation depends +on which descriptor methods are defined. A descriptor can define any +combination of "__get__()", "__set__()" and "__delete__()". If it +does not define "__get__()", then accessing the attribute will return +the descriptor object itself unless there is a value in the object’s +instance dictionary. If the descriptor defines "__set__()" and/or +"__delete__()", it is a data descriptor; if it defines neither, it is +a non-data descriptor. Normally, data descriptors define both +"__get__()" and "__set__()", while non-data descriptors have just the +"__get__()" method. Data descriptors with "__get__()" and "__set__()" +(and/or "__delete__()") defined always override a redefinition in an +instance dictionary. In contrast, non-data descriptors can be +overridden by instances. + +Python methods (including those decorated with "@staticmethod" and +"@classmethod") are implemented as non-data descriptors. Accordingly, +instances can redefine and override methods. This allows individual +instances to acquire behaviors that differ from other instances of the +same class. + +The "property()" function is implemented as a data descriptor. +Accordingly, instances cannot override the behavior of a property. + + +__slots__ +--------- + +*__slots__* allow us to explicitly declare data members (like +properties) and deny the creation of "__dict__" and *__weakref__* +(unless explicitly declared in *__slots__* or available in a parent.) + +The space saved over using "__dict__" can be significant. Attribute +lookup speed can be significantly improved as well. + +object.__slots__ + + This class variable can be assigned a string, iterable, or sequence + of strings with variable names used by instances. *__slots__* + reserves space for the declared variables and prevents the + automatic creation of "__dict__" and *__weakref__* for each + instance. + +Notes on using *__slots__*: + +* When inheriting from a class without *__slots__*, the "__dict__" and + *__weakref__* attribute of the instances will always be accessible. + +* Without a "__dict__" variable, instances cannot be assigned new + variables not listed in the *__slots__* definition. Attempts to + assign to an unlisted variable name raises "AttributeError". If + dynamic assignment of new variables is desired, then add + "'__dict__'" to the sequence of strings in the *__slots__* + declaration. + +* Without a *__weakref__* variable for each instance, classes defining + *__slots__* do not support "weak references" to its instances. If + weak reference support is needed, then add "'__weakref__'" to the + sequence of strings in the *__slots__* declaration. + +* *__slots__* are implemented at the class level by creating + descriptors for each variable name. As a result, class attributes + cannot be used to set default values for instance variables defined + by *__slots__*; otherwise, the class attribute would overwrite the + descriptor assignment. + +* The action of a *__slots__* declaration is not limited to the class + where it is defined. *__slots__* declared in parents are available + in child classes. However, instances of a child subclass will get a + "__dict__" and *__weakref__* unless the subclass also defines + *__slots__* (which should only contain names of any *additional* + slots). + +* If a class defines a slot also defined in a base class, the instance + variable defined by the base class slot is inaccessible (except by + retrieving its descriptor directly from the base class). This + renders the meaning of the program undefined. In the future, a + check may be added to prevent this. + +* "TypeError" will be raised if nonempty *__slots__* are defined for a + class derived from a ""variable-length" built-in type" such as + "int", "bytes", and "tuple". + +* Any non-string *iterable* may be assigned to *__slots__*. + +* If a "dictionary" is used to assign *__slots__*, the dictionary keys + will be used as the slot names. The values of the dictionary can be + used to provide per-attribute docstrings that will be recognised by + "inspect.getdoc()" and displayed in the output of "help()". + +* "__class__" assignment works only if both classes have the same + *__slots__*. + +* Multiple inheritance with multiple slotted parent classes can be + used, but only one parent is allowed to have attributes created by + slots (the other bases must have empty slot layouts) - violations + raise "TypeError". + +* If an *iterator* is used for *__slots__* then a *descriptor* is + created for each of the iterator’s values. However, the *__slots__* + attribute will be an empty iterator. + + +Customizing class creation +========================== + +Whenever a class inherits from another class, "__init_subclass__()" is +called on the parent class. This way, it is possible to write classes +which change the behavior of subclasses. This is closely related to +class decorators, but where class decorators only affect the specific +class they’re applied to, "__init_subclass__" solely applies to future +subclasses of the class defining the method. + +classmethod object.__init_subclass__(cls) + + This method is called whenever the containing class is subclassed. + *cls* is then the new subclass. If defined as a normal instance + method, this method is implicitly converted to a class method. + + Keyword arguments which are given to a new class are passed to the + parent class’s "__init_subclass__". For compatibility with other + classes using "__init_subclass__", one should take out the needed + keyword arguments and pass the others over to the base class, as + in: + + class Philosopher: + def __init_subclass__(cls, /, default_name, **kwargs): + super().__init_subclass__(**kwargs) + cls.default_name = default_name + + class AustralianPhilosopher(Philosopher, default_name="Bruce"): + pass + + The default implementation "object.__init_subclass__" does nothing, + but raises an error if it is called with any arguments. + + Note: + + The metaclass hint "metaclass" is consumed by the rest of the + type machinery, and is never passed to "__init_subclass__" + implementations. The actual metaclass (rather than the explicit + hint) can be accessed as "type(cls)". + + Added in version 3.6. + +When a class is created, "type.__new__()" scans the class variables +and makes callbacks to those with a "__set_name__()" hook. + +object.__set_name__(self, owner, name) + + Automatically called at the time the owning class *owner* is + created. The object has been assigned to *name* in that class: + + class A: + x = C() # Automatically calls: x.__set_name__(A, 'x') + + If the class variable is assigned after the class is created, + "__set_name__()" will not be called automatically. If needed, + "__set_name__()" can be called directly: + + class A: + pass + + c = C() + A.x = c # The hook is not called + c.__set_name__(A, 'x') # Manually invoke the hook + + See Creating the class object for more details. + + Added in version 3.6. + + +Metaclasses +----------- + +By default, classes are constructed using "type()". The class body is +executed in a new namespace and the class name is bound locally to the +result of "type(name, bases, namespace)". + +The class creation process can be customized by passing the +"metaclass" keyword argument in the class definition line, or by +inheriting from an existing class that included such an argument. In +the following example, both "MyClass" and "MySubclass" are instances +of "Meta": + + class Meta(type): + pass + + class MyClass(metaclass=Meta): + pass + + class MySubclass(MyClass): + pass + +Any other keyword arguments that are specified in the class definition +are passed through to all metaclass operations described below. + +When a class definition is executed, the following steps occur: + +* MRO entries are resolved; + +* the appropriate metaclass is determined; + +* the class namespace is prepared; + +* the class body is executed; + +* the class object is created. + + +Resolving MRO entries +--------------------- + +object.__mro_entries__(self, bases) + + If a base that appears in a class definition is not an instance of + "type", then an "__mro_entries__()" method is searched on the base. + If an "__mro_entries__()" method is found, the base is substituted + with the result of a call to "__mro_entries__()" when creating the + class. The method is called with the original bases tuple passed to + the *bases* parameter, and must return a tuple of classes that will + be used instead of the base. The returned tuple may be empty: in + these cases, the original base is ignored. + +See also: + + "types.resolve_bases()" + Dynamically resolve bases that are not instances of "type". + + "types.get_original_bases()" + Retrieve a class’s “original bases” prior to modifications by + "__mro_entries__()". + + **PEP 560** + Core support for typing module and generic types. + + +Determining the appropriate metaclass +------------------------------------- + +The appropriate metaclass for a class definition is determined as +follows: + +* if no bases and no explicit metaclass are given, then "type()" is + used; + +* if an explicit metaclass is given and it is *not* an instance of + "type()", then it is used directly as the metaclass; + +* if an instance of "type()" is given as the explicit metaclass, or + bases are defined, then the most derived metaclass is used. + +The most derived metaclass is selected from the explicitly specified +metaclass (if any) and the metaclasses (i.e. "type(cls)") of all +specified base classes. The most derived metaclass is one which is a +subtype of *all* of these candidate metaclasses. If none of the +candidate metaclasses meets that criterion, then the class definition +will fail with "TypeError". + + +Preparing the class namespace +----------------------------- + +Once the appropriate metaclass has been identified, then the class +namespace is prepared. If the metaclass has a "__prepare__" attribute, +it is called as "namespace = metaclass.__prepare__(name, bases, +**kwds)" (where the additional keyword arguments, if any, come from +the class definition). The "__prepare__" method should be implemented +as a "classmethod". The namespace returned by "__prepare__" is passed +in to "__new__", but when the final class object is created the +namespace is copied into a new "dict". + +If the metaclass has no "__prepare__" attribute, then the class +namespace is initialised as an empty ordered mapping. + +See also: + + **PEP 3115** - Metaclasses in Python 3000 + Introduced the "__prepare__" namespace hook + + +Executing the class body +------------------------ + +The class body is executed (approximately) as "exec(body, globals(), +namespace)". The key difference from a normal call to "exec()" is that +lexical scoping allows the class body (including any methods) to +reference names from the current and outer scopes when the class +definition occurs inside a function. + +However, even when the class definition occurs inside the function, +methods defined inside the class still cannot see names defined at the +class scope. Class variables must be accessed through the first +parameter of instance or class methods, or through the implicit +lexically scoped "__class__" reference described in the next section. + + +Creating the class object +------------------------- + +Once the class namespace has been populated by executing the class +body, the class object is created by calling "metaclass(name, bases, +namespace, **kwds)" (the additional keywords passed here are the same +as those passed to "__prepare__"). + +This class object is the one that will be referenced by the zero- +argument form of "super()". "__class__" is an implicit closure +reference created by the compiler if any methods in a class body refer +to either "__class__" or "super". This allows the zero argument form +of "super()" to correctly identify the class being defined based on +lexical scoping, while the class or instance that was used to make the +current call is identified based on the first argument passed to the +method. + +**CPython implementation detail:** In CPython 3.6 and later, the +"__class__" cell is passed to the metaclass as a "__classcell__" entry +in the class namespace. If present, this must be propagated up to the +"type.__new__" call in order for the class to be initialised +correctly. Failing to do so will result in a "RuntimeError" in Python +3.8. + +When using the default metaclass "type", or any metaclass that +ultimately calls "type.__new__", the following additional +customization steps are invoked after creating the class object: + +1. The "type.__new__" method collects all of the attributes in the + class namespace that define a "__set_name__()" method; + +2. Those "__set_name__" methods are called with the class being + defined and the assigned name of that particular attribute; + +3. The "__init_subclass__()" hook is called on the immediate parent of + the new class in its method resolution order. + +After the class object is created, it is passed to the class +decorators included in the class definition (if any) and the resulting +object is bound in the local namespace as the defined class. + +When a new class is created by "type.__new__", the object provided as +the namespace parameter is copied to a new ordered mapping and the +original object is discarded. The new copy is wrapped in a read-only +proxy, which becomes the "__dict__" attribute of the class object. + +See also: + + **PEP 3135** - New super + Describes the implicit "__class__" closure reference + + +Uses for metaclasses +-------------------- + +The potential uses for metaclasses are boundless. Some ideas that have +been explored include enum, logging, interface checking, automatic +delegation, automatic property creation, proxies, frameworks, and +automatic resource locking/synchronization. + + +Customizing instance and subclass checks +======================================== + +The following methods are used to override the default behavior of the +"isinstance()" and "issubclass()" built-in functions. + +In particular, the metaclass "abc.ABCMeta" implements these methods in +order to allow the addition of Abstract Base Classes (ABCs) as +“virtual base classes” to any class or type (including built-in +types), including other ABCs. + +type.__instancecheck__(self, instance) + + Return true if *instance* should be considered a (direct or + indirect) instance of *class*. If defined, called to implement + "isinstance(instance, class)". + +type.__subclasscheck__(self, subclass) + + Return true if *subclass* should be considered a (direct or + indirect) subclass of *class*. If defined, called to implement + "issubclass(subclass, class)". + +Note that these methods are looked up on the type (metaclass) of a +class. They cannot be defined as class methods in the actual class. +This is consistent with the lookup of special methods that are called +on instances, only in this case the instance is itself a class. + +See also: + + **PEP 3119** - Introducing Abstract Base Classes + Includes the specification for customizing "isinstance()" and + "issubclass()" behavior through "__instancecheck__()" and + "__subclasscheck__()", with motivation for this functionality in + the context of adding Abstract Base Classes (see the "abc" + module) to the language. + + +Emulating generic types +======================= + +When using *type annotations*, it is often useful to *parameterize* a +*generic type* using Python’s square-brackets notation. For example, +the annotation "list[int]" might be used to signify a "list" in which +all the elements are of type "int". + +See also: + + **PEP 484** - Type Hints + Introducing Python’s framework for type annotations + + Generic Alias Types + Documentation for objects representing parameterized generic + classes + + Generics, user-defined generics and "typing.Generic" + Documentation on how to implement generic classes that can be + parameterized at runtime and understood by static type-checkers. + +A class can *generally* only be parameterized if it defines the +special class method "__class_getitem__()". + +classmethod object.__class_getitem__(cls, key) + + Return an object representing the specialization of a generic class + by type arguments found in *key*. + + When defined on a class, "__class_getitem__()" is automatically a + class method. As such, there is no need for it to be decorated with + "@classmethod" when it is defined. + + +The purpose of *__class_getitem__* +---------------------------------- + +The purpose of "__class_getitem__()" is to allow runtime +parameterization of standard-library generic classes in order to more +easily apply *type hints* to these classes. + +To implement custom generic classes that can be parameterized at +runtime and understood by static type-checkers, users should either +inherit from a standard library class that already implements +"__class_getitem__()", or inherit from "typing.Generic", which has its +own implementation of "__class_getitem__()". + +Custom implementations of "__class_getitem__()" on classes defined +outside of the standard library may not be understood by third-party +type-checkers such as mypy. Using "__class_getitem__()" on any class +for purposes other than type hinting is discouraged. + + +*__class_getitem__* versus *__getitem__* +---------------------------------------- + +Usually, the subscription of an object using square brackets will call +the "__getitem__()" instance method defined on the object’s class. +However, if the object being subscribed is itself a class, the class +method "__class_getitem__()" may be called instead. +"__class_getitem__()" should return a GenericAlias object if it is +properly defined. + +Presented with the *expression* "obj[x]", the Python interpreter +follows something like the following process to decide whether +"__getitem__()" or "__class_getitem__()" should be called: + + from inspect import isclass + + def subscribe(obj, x): + """Return the result of the expression 'obj[x]'""" + + class_of_obj = type(obj) + + # If the class of obj defines __getitem__, + # call class_of_obj.__getitem__(obj, x) + if hasattr(class_of_obj, '__getitem__'): + return class_of_obj.__getitem__(obj, x) + + # Else, if obj is a class and defines __class_getitem__, + # call obj.__class_getitem__(x) + elif isclass(obj) and hasattr(obj, '__class_getitem__'): + return obj.__class_getitem__(x) + + # Else, raise an exception + else: + raise TypeError( + f"'{class_of_obj.__name__}' object is not subscriptable" + ) + +In Python, all classes are themselves instances of other classes. The +class of a class is known as that class’s *metaclass*, and most +classes have the "type" class as their metaclass. "type" does not +define "__getitem__()", meaning that expressions such as "list[int]", +"dict[str, float]" and "tuple[str, bytes]" all result in +"__class_getitem__()" being called: + + >>> # list has class "type" as its metaclass, like most classes: + >>> type(list) + + >>> type(dict) == type(list) == type(tuple) == type(str) == type(bytes) + True + >>> # "list[int]" calls "list.__class_getitem__(int)" + >>> list[int] + list[int] + >>> # list.__class_getitem__ returns a GenericAlias object: + >>> type(list[int]) + + +However, if a class has a custom metaclass that defines +"__getitem__()", subscribing the class may result in different +behaviour. An example of this can be found in the "enum" module: + + >>> from enum import Enum + >>> class Menu(Enum): + ... """A breakfast menu""" + ... SPAM = 'spam' + ... BACON = 'bacon' + ... + >>> # Enum classes have a custom metaclass: + >>> type(Menu) + + >>> # EnumMeta defines __getitem__, + >>> # so __class_getitem__ is not called, + >>> # and the result is not a GenericAlias object: + >>> Menu['SPAM'] + + >>> type(Menu['SPAM']) + + +See also: + + **PEP 560** - Core Support for typing module and generic types + Introducing "__class_getitem__()", and outlining when a + subscription results in "__class_getitem__()" being called + instead of "__getitem__()" + + +Emulating callable objects +========================== + +object.__call__(self[, args...]) + + Called when the instance is “called” as a function; if this method + is defined, "x(arg1, arg2, ...)" roughly translates to + "type(x).__call__(x, arg1, ...)". The "object" class itself does + not provide this method. + + +Emulating container types +========================= + +The following methods can be defined to implement container objects. +None of them are provided by the "object" class itself. Containers +usually are *sequences* (such as "lists" or "tuples") or *mappings* +(like *dictionaries*), but can represent other containers as well. +The first set of methods is used either to emulate a sequence or to +emulate a mapping; the difference is that for a sequence, the +allowable keys should be the integers *k* for which "0 <= k < N" where +*N* is the length of the sequence, or "slice" objects, which define a +range of items. It is also recommended that mappings provide the +methods "keys()", "values()", "items()", "get()", "clear()", +"setdefault()", "pop()", "popitem()", "copy()", and "update()" +behaving similar to those for Python’s standard "dictionary" objects. +The "collections.abc" module provides a "MutableMapping" *abstract +base class* to help create those methods from a base set of +"__getitem__()", "__setitem__()", "__delitem__()", and "keys()". +Mutable sequences should provide methods "append()", "count()", +"index()", "extend()", "insert()", "pop()", "remove()", "reverse()" +and "sort()", like Python standard "list" objects. Finally, sequence +types should implement addition (meaning concatenation) and +multiplication (meaning repetition) by defining the methods +"__add__()", "__radd__()", "__iadd__()", "__mul__()", "__rmul__()" and +"__imul__()" described below; they should not define other numerical +operators. It is recommended that both mappings and sequences +implement the "__contains__()" method to allow efficient use of the +"in" operator; for mappings, "in" should search the mapping’s keys; +for sequences, it should search through the values. It is further +recommended that both mappings and sequences implement the +"__iter__()" method to allow efficient iteration through the +container; for mappings, "__iter__()" should iterate through the +object’s keys; for sequences, it should iterate through the values. + +object.__len__(self) + + Called to implement the built-in function "len()". Should return + the length of the object, an integer ">=" 0. Also, an object that + doesn’t define a "__bool__()" method and whose "__len__()" method + returns zero is considered to be false in a Boolean context. + + **CPython implementation detail:** In CPython, the length is + required to be at most "sys.maxsize". If the length is larger than + "sys.maxsize" some features (such as "len()") may raise + "OverflowError". To prevent raising "OverflowError" by truth value + testing, an object must define a "__bool__()" method. + +object.__length_hint__(self) + + Called to implement "operator.length_hint()". Should return an + estimated length for the object (which may be greater or less than + the actual length). The length must be an integer ">=" 0. The + return value may also be "NotImplemented", which is treated the + same as if the "__length_hint__" method didn’t exist at all. This + method is purely an optimization and is never required for + correctness. + + Added in version 3.4. + +Note: + + Slicing is done exclusively with the following three methods. A + call like + + a[1:2] = b + + is translated to + + a[slice(1, 2, None)] = b + + and so forth. Missing slice items are always filled in with "None". + +object.__getitem__(self, key) + + Called to implement evaluation of "self[key]". For *sequence* + types, the accepted keys should be integers. Optionally, they may + support "slice" objects as well. Negative index support is also + optional. If *key* is of an inappropriate type, "TypeError" may be + raised; if *key* is a value outside the set of indexes for the + sequence (after any special interpretation of negative values), + "IndexError" should be raised. For *mapping* types, if *key* is + missing (not in the container), "KeyError" should be raised. + + Note: + + "for" loops expect that an "IndexError" will be raised for + illegal indexes to allow proper detection of the end of the + sequence. + + Note: + + When subscripting a *class*, the special class method + "__class_getitem__()" may be called instead of "__getitem__()". + See __class_getitem__ versus __getitem__ for more details. + +object.__setitem__(self, key, value) + + Called to implement assignment to "self[key]". Same note as for + "__getitem__()". This should only be implemented for mappings if + the objects support changes to the values for keys, or if new keys + can be added, or for sequences if elements can be replaced. The + same exceptions should be raised for improper *key* values as for + the "__getitem__()" method. + +object.__delitem__(self, key) + + Called to implement deletion of "self[key]". Same note as for + "__getitem__()". This should only be implemented for mappings if + the objects support removal of keys, or for sequences if elements + can be removed from the sequence. The same exceptions should be + raised for improper *key* values as for the "__getitem__()" method. + +object.__missing__(self, key) + + Called by "dict"."__getitem__()" to implement "self[key]" for dict + subclasses when key is not in the dictionary. + +object.__iter__(self) + + This method is called when an *iterator* is required for a + container. This method should return a new iterator object that can + iterate over all the objects in the container. For mappings, it + should iterate over the keys of the container. + +object.__reversed__(self) + + Called (if present) by the "reversed()" built-in to implement + reverse iteration. It should return a new iterator object that + iterates over all the objects in the container in reverse order. + + If the "__reversed__()" method is not provided, the "reversed()" + built-in will fall back to using the sequence protocol ("__len__()" + and "__getitem__()"). Objects that support the sequence protocol + should only provide "__reversed__()" if they can provide an + implementation that is more efficient than the one provided by + "reversed()". + +The membership test operators ("in" and "not in") are normally +implemented as an iteration through a container. However, container +objects can supply the following special method with a more efficient +implementation, which also does not require the object be iterable. + +object.__contains__(self, item) + + Called to implement membership test operators. Should return true + if *item* is in *self*, false otherwise. For mapping objects, this + should consider the keys of the mapping rather than the values or + the key-item pairs. + + For objects that don’t define "__contains__()", the membership test + first tries iteration via "__iter__()", then the old sequence + iteration protocol via "__getitem__()", see this section in the + language reference. + + +Emulating numeric types +======================= + +The following methods can be defined to emulate numeric objects. +Methods corresponding to operations that are not supported by the +particular kind of number implemented (e.g., bitwise operations for +non-integral numbers) should be left undefined. + +object.__add__(self, other) +object.__sub__(self, other) +object.__mul__(self, other) +object.__matmul__(self, other) +object.__truediv__(self, other) +object.__floordiv__(self, other) +object.__mod__(self, other) +object.__divmod__(self, other) +object.__pow__(self, other[, modulo]) +object.__lshift__(self, other) +object.__rshift__(self, other) +object.__and__(self, other) +object.__xor__(self, other) +object.__or__(self, other) + + These methods are called to implement the binary arithmetic + operations ("+", "-", "*", "@", "/", "//", "%", "divmod()", + "pow()", "**", "<<", ">>", "&", "^", "|"). For instance, to + evaluate the expression "x + y", where *x* is an instance of a + class that has an "__add__()" method, "type(x).__add__(x, y)" is + called. The "__divmod__()" method should be the equivalent to + using "__floordiv__()" and "__mod__()"; it should not be related to + "__truediv__()". Note that "__pow__()" should be defined to accept + an optional third argument if the ternary version of the built-in + "pow()" function is to be supported. + + If one of those methods does not support the operation with the + supplied arguments, it should return "NotImplemented". + +object.__radd__(self, other) +object.__rsub__(self, other) +object.__rmul__(self, other) +object.__rmatmul__(self, other) +object.__rtruediv__(self, other) +object.__rfloordiv__(self, other) +object.__rmod__(self, other) +object.__rdivmod__(self, other) +object.__rpow__(self, other[, modulo]) +object.__rlshift__(self, other) +object.__rrshift__(self, other) +object.__rand__(self, other) +object.__rxor__(self, other) +object.__ror__(self, other) + + These methods are called to implement the binary arithmetic + operations ("+", "-", "*", "@", "/", "//", "%", "divmod()", + "pow()", "**", "<<", ">>", "&", "^", "|") with reflected (swapped) + operands. These functions are only called if the operands are of + different types, when the left operand does not support the + corresponding operation [3], or the right operand’s class is + derived from the left operand’s class. [4] For instance, to + evaluate the expression "x - y", where *y* is an instance of a + class that has an "__rsub__()" method, "type(y).__rsub__(y, x)" is + called if "type(x).__sub__(x, y)" returns "NotImplemented" or + "type(y)" is a subclass of "type(x)". [5] + + Note that ternary "pow()" will not try calling "__rpow__()" (the + coercion rules would become too complicated). + + Note: + + If the right operand’s type is a subclass of the left operand’s + type and that subclass provides a different implementation of the + reflected method for the operation, this method will be called + before the left operand’s non-reflected method. This behavior + allows subclasses to override their ancestors’ operations. + +object.__iadd__(self, other) +object.__isub__(self, other) +object.__imul__(self, other) +object.__imatmul__(self, other) +object.__itruediv__(self, other) +object.__ifloordiv__(self, other) +object.__imod__(self, other) +object.__ipow__(self, other[, modulo]) +object.__ilshift__(self, other) +object.__irshift__(self, other) +object.__iand__(self, other) +object.__ixor__(self, other) +object.__ior__(self, other) + + These methods are called to implement the augmented arithmetic + assignments ("+=", "-=", "*=", "@=", "/=", "//=", "%=", "**=", + "<<=", ">>=", "&=", "^=", "|="). These methods should attempt to + do the operation in-place (modifying *self*) and return the result + (which could be, but does not have to be, *self*). If a specific + method is not defined, or if that method returns "NotImplemented", + the augmented assignment falls back to the normal methods. For + instance, if *x* is an instance of a class with an "__iadd__()" + method, "x += y" is equivalent to "x = x.__iadd__(y)" . If + "__iadd__()" does not exist, or if "x.__iadd__(y)" returns + "NotImplemented", "x.__add__(y)" and "y.__radd__(x)" are + considered, as with the evaluation of "x + y". In certain + situations, augmented assignment can result in unexpected errors + (see Why does a_tuple[i] += [‘item’] raise an exception when the + addition works?), but this behavior is in fact part of the data + model. + +object.__neg__(self) +object.__pos__(self) +object.__abs__(self) +object.__invert__(self) + + Called to implement the unary arithmetic operations ("-", "+", + "abs()" and "~"). + +object.__complex__(self) +object.__int__(self) +object.__float__(self) + + Called to implement the built-in functions "complex()", "int()" and + "float()". Should return a value of the appropriate type. + +object.__index__(self) + + Called to implement "operator.index()", and whenever Python needs + to losslessly convert the numeric object to an integer object (such + as in slicing, or in the built-in "bin()", "hex()" and "oct()" + functions). Presence of this method indicates that the numeric + object is an integer type. Must return an integer. + + If "__int__()", "__float__()" and "__complex__()" are not defined + then corresponding built-in functions "int()", "float()" and + "complex()" fall back to "__index__()". + +object.__round__(self[, ndigits]) +object.__trunc__(self) +object.__floor__(self) +object.__ceil__(self) + + Called to implement the built-in function "round()" and "math" + functions "trunc()", "floor()" and "ceil()". Unless *ndigits* is + passed to "__round__()" all these methods should return the value + of the object truncated to an "Integral" (typically an "int"). + + Changed in version 3.14: "int()" no longer delegates to the + "__trunc__()" method. + + +With Statement Context Managers +=============================== + +A *context manager* is an object that defines the runtime context to +be established when executing a "with" statement. The context manager +handles the entry into, and the exit from, the desired runtime context +for the execution of the block of code. Context managers are normally +invoked using the "with" statement (described in section The with +statement), but can also be used by directly invoking their methods. + +Typical uses of context managers include saving and restoring various +kinds of global state, locking and unlocking resources, closing opened +files, etc. + +For more information on context managers, see Context Manager Types. +The "object" class itself does not provide the context manager +methods. + +object.__enter__(self) + + Enter the runtime context related to this object. The "with" + statement will bind this method’s return value to the target(s) + specified in the "as" clause of the statement, if any. + +object.__exit__(self, exc_type, exc_value, traceback) + + Exit the runtime context related to this object. The parameters + describe the exception that caused the context to be exited. If the + context was exited without an exception, all three arguments will + be "None". + + If an exception is supplied, and the method wishes to suppress the + exception (i.e., prevent it from being propagated), it should + return a true value. Otherwise, the exception will be processed + normally upon exit from this method. + + Note that "__exit__()" methods should not reraise the passed-in + exception; this is the caller’s responsibility. + +See also: + + **PEP 343** - The “with” statement + The specification, background, and examples for the Python "with" + statement. + + +Customizing positional arguments in class pattern matching +========================================================== + +When using a class name in a pattern, positional arguments in the +pattern are not allowed by default, i.e. "case MyClass(x, y)" is +typically invalid without special support in "MyClass". To be able to +use that kind of pattern, the class needs to define a *__match_args__* +attribute. + +object.__match_args__ + + This class variable can be assigned a tuple of strings. When this + class is used in a class pattern with positional arguments, each + positional argument will be converted into a keyword argument, + using the corresponding value in *__match_args__* as the keyword. + The absence of this attribute is equivalent to setting it to "()". + +For example, if "MyClass.__match_args__" is "("left", "center", +"right")" that means that "case MyClass(x, y)" is equivalent to "case +MyClass(left=x, center=y)". Note that the number of arguments in the +pattern must be smaller than or equal to the number of elements in +*__match_args__*; if it is larger, the pattern match attempt will +raise a "TypeError". + +Added in version 3.10. + +See also: + + **PEP 634** - Structural Pattern Matching + The specification for the Python "match" statement. + + +Emulating buffer types +====================== + +The buffer protocol provides a way for Python objects to expose +efficient access to a low-level memory array. This protocol is +implemented by builtin types such as "bytes" and "memoryview", and +third-party libraries may define additional buffer types. + +While buffer types are usually implemented in C, it is also possible +to implement the protocol in Python. + +object.__buffer__(self, flags) + + Called when a buffer is requested from *self* (for example, by the + "memoryview" constructor). The *flags* argument is an integer + representing the kind of buffer requested, affecting for example + whether the returned buffer is read-only or writable. + "inspect.BufferFlags" provides a convenient way to interpret the + flags. The method must return a "memoryview" object. + +object.__release_buffer__(self, buffer) + + Called when a buffer is no longer needed. The *buffer* argument is + a "memoryview" object that was previously returned by + "__buffer__()". The method must release any resources associated + with the buffer. This method should return "None". Buffer objects + that do not need to perform any cleanup are not required to + implement this method. + +Added in version 3.12. + +See also: + + **PEP 688** - Making the buffer protocol accessible in Python + Introduces the Python "__buffer__" and "__release_buffer__" + methods. + + "collections.abc.Buffer" + ABC for buffer types. + + +Annotations +=========== + +Functions, classes, and modules may contain *annotations*, which are a +way to associate information (usually *type hints*) with a symbol. + +object.__annotations__ + + This attribute contains the annotations for an object. It is lazily + evaluated, so accessing the attribute may execute arbitrary code + and raise exceptions. If evaluation is successful, the attribute is + set to a dictionary mapping from variable names to annotations. + + Changed in version 3.14: Annotations are now lazily evaluated. + +object.__annotate__(format) + + An *annotate function*. Returns a new dictionary object mapping + attribute/parameter names to their annotation values. + + Takes a format parameter specifying the format in which annotations + values should be provided. It must be a member of the + "annotationlib.Format" enum, or an integer with a value + corresponding to a member of the enum. + + If an annotate function doesn’t support the requested format, it + must raise "NotImplementedError". Annotate functions must always + support "VALUE" format; they must not raise "NotImplementedError()" + when called with this format. + + When called with "VALUE" format, an annotate function may raise + "NameError"; it must not raise "NameError" when called requesting + any other format. + + If an object does not have any annotations, "__annotate__" should + preferably be set to "None" (it can’t be deleted), rather than set + to a function that returns an empty dict. + + Added in version 3.14. + +See also: + + **PEP 649** — Deferred evaluation of annotation using descriptors + Introduces lazy evaluation of annotations and the "__annotate__" + function. + + +Special method lookup +===================== + +For custom classes, implicit invocations of special methods are only +guaranteed to work correctly if defined on an object’s type, not in +the object’s instance dictionary. That behaviour is the reason why +the following code raises an exception: + + >>> class C: + ... pass + ... + >>> c = C() + >>> c.__len__ = lambda: 5 + >>> len(c) + Traceback (most recent call last): + File "", line 1, in + TypeError: object of type 'C' has no len() + +The rationale behind this behaviour lies with a number of special +methods such as "__hash__()" and "__repr__()" that are implemented by +all objects, including type objects. If the implicit lookup of these +methods used the conventional lookup process, they would fail when +invoked on the type object itself: + + >>> 1 .__hash__() == hash(1) + True + >>> int.__hash__() == hash(int) + Traceback (most recent call last): + File "", line 1, in + TypeError: descriptor '__hash__' of 'int' object needs an argument + +Incorrectly attempting to invoke an unbound method of a class in this +way is sometimes referred to as ‘metaclass confusion’, and is avoided +by bypassing the instance when looking up special methods: + + >>> type(1).__hash__(1) == hash(1) + True + >>> type(int).__hash__(int) == hash(int) + True + +In addition to bypassing any instance attributes in the interest of +correctness, implicit special method lookup generally also bypasses +the "__getattribute__()" method even of the object’s metaclass: + + >>> class Meta(type): + ... def __getattribute__(*args): + ... print("Metaclass getattribute invoked") + ... return type.__getattribute__(*args) + ... + >>> class C(object, metaclass=Meta): + ... def __len__(self): + ... return 10 + ... def __getattribute__(*args): + ... print("Class getattribute invoked") + ... return object.__getattribute__(*args) + ... + >>> c = C() + >>> c.__len__() # Explicit lookup via instance + Class getattribute invoked + 10 + >>> type(c).__len__(c) # Explicit lookup via type + Metaclass getattribute invoked + 10 + >>> len(c) # Implicit lookup + 10 + +Bypassing the "__getattribute__()" machinery in this fashion provides +significant scope for speed optimisations within the interpreter, at +the cost of some flexibility in the handling of special methods (the +special method *must* be set on the class object itself in order to be +consistently invoked by the interpreter). +''', + 'string-methods': r'''String Methods +************** + +Strings implement all of the common sequence operations, along with +the additional methods described below. + +Strings also support two styles of string formatting, one providing a +large degree of flexibility and customization (see "str.format()", +Format String Syntax and Custom String Formatting) and the other based +on C "printf" style formatting that handles a narrower range of types +and is slightly harder to use correctly, but is often faster for the +cases it can handle (printf-style String Formatting). + +The Text Processing Services section of the standard library covers a +number of other modules that provide various text related utilities +(including regular expression support in the "re" module). + +str.capitalize() + + Return a copy of the string with its first character capitalized + and the rest lowercased. + + Changed in version 3.8: The first character is now put into + titlecase rather than uppercase. This means that characters like + digraphs will only have their first letter capitalized, instead of + the full character. + +str.casefold() + + Return a casefolded copy of the string. Casefolded strings may be + used for caseless matching. + + Casefolding is similar to lowercasing but more aggressive because + it is intended to remove all case distinctions in a string. For + example, the German lowercase letter "'ß'" is equivalent to ""ss"". + Since it is already lowercase, "lower()" would do nothing to "'ß'"; + "casefold()" converts it to ""ss"". + + The casefolding algorithm is described in section 3.13 ‘Default + Case Folding’ of the Unicode Standard. + + Added in version 3.3. + +str.center(width[, fillchar]) + + Return centered in a string of length *width*. Padding is done + using the specified *fillchar* (default is an ASCII space). The + original string is returned if *width* is less than or equal to + "len(s)". + +str.count(sub[, start[, end]]) + + Return the number of non-overlapping occurrences of substring *sub* + in the range [*start*, *end*]. Optional arguments *start* and + *end* are interpreted as in slice notation. + + If *sub* is empty, returns the number of empty strings between + characters which is the length of the string plus one. + +str.encode(encoding='utf-8', errors='strict') + + Return the string encoded to "bytes". + + *encoding* defaults to "'utf-8'"; see Standard Encodings for + possible values. + + *errors* controls how encoding errors are handled. If "'strict'" + (the default), a "UnicodeError" exception is raised. Other possible + values are "'ignore'", "'replace'", "'xmlcharrefreplace'", + "'backslashreplace'" and any other name registered via + "codecs.register_error()". See Error Handlers for details. + + For performance reasons, the value of *errors* is not checked for + validity unless an encoding error actually occurs, Python + Development Mode is enabled or a debug build is used. + + Changed in version 3.1: Added support for keyword arguments. + + Changed in version 3.9: The value of the *errors* argument is now + checked in Python Development Mode and in debug mode. + +str.endswith(suffix[, start[, end]]) + + Return "True" if the string ends with the specified *suffix*, + otherwise return "False". *suffix* can also be a tuple of suffixes + to look for. With optional *start*, test beginning at that + position. With optional *end*, stop comparing at that position. + +str.expandtabs(tabsize=8) + + Return a copy of the string where all tab characters are replaced + by one or more spaces, depending on the current column and the + given tab size. Tab positions occur every *tabsize* characters + (default is 8, giving tab positions at columns 0, 8, 16 and so on). + To expand the string, the current column is set to zero and the + string is examined character by character. If the character is a + tab ("\t"), one or more space characters are inserted in the result + until the current column is equal to the next tab position. (The + tab character itself is not copied.) If the character is a newline + ("\n") or return ("\r"), it is copied and the current column is + reset to zero. Any other character is copied unchanged and the + current column is incremented by one regardless of how the + character is represented when printed. + + >>> '01\t012\t0123\t01234'.expandtabs() + '01 012 0123 01234' + >>> '01\t012\t0123\t01234'.expandtabs(4) + '01 012 0123 01234' + +str.find(sub[, start[, end]]) + + Return the lowest index in the string where substring *sub* is + found within the slice "s[start:end]". Optional arguments *start* + and *end* are interpreted as in slice notation. Return "-1" if + *sub* is not found. + + Note: + + The "find()" method should be used only if you need to know the + position of *sub*. To check if *sub* is a substring or not, use + the "in" operator: + + >>> 'Py' in 'Python' + True + +str.format(*args, **kwargs) + + Perform a string formatting operation. The string on which this + method is called can contain literal text or replacement fields + delimited by braces "{}". Each replacement field contains either + the numeric index of a positional argument, or the name of a + keyword argument. Returns a copy of the string where each + replacement field is replaced with the string value of the + corresponding argument. + + >>> "The sum of 1 + 2 is {0}".format(1+2) + 'The sum of 1 + 2 is 3' + + See Format String Syntax for a description of the various + formatting options that can be specified in format strings. + + Note: + + When formatting a number ("int", "float", "complex", + "decimal.Decimal" and subclasses) with the "n" type (ex: + "'{:n}'.format(1234)"), the function temporarily sets the + "LC_CTYPE" locale to the "LC_NUMERIC" locale to decode + "decimal_point" and "thousands_sep" fields of "localeconv()" if + they are non-ASCII or longer than 1 byte, and the "LC_NUMERIC" + locale is different than the "LC_CTYPE" locale. This temporary + change affects other threads. + + Changed in version 3.7: When formatting a number with the "n" type, + the function sets temporarily the "LC_CTYPE" locale to the + "LC_NUMERIC" locale in some cases. + +str.format_map(mapping, /) + + Similar to "str.format(**mapping)", except that "mapping" is used + directly and not copied to a "dict". This is useful if for example + "mapping" is a dict subclass: + + >>> class Default(dict): + ... def __missing__(self, key): + ... return key + ... + >>> '{name} was born in {country}'.format_map(Default(name='Guido')) + 'Guido was born in country' + + Added in version 3.2. + +str.index(sub[, start[, end]]) + + Like "find()", but raise "ValueError" when the substring is not + found. + +str.isalnum() + + Return "True" if all characters in the string are alphanumeric and + there is at least one character, "False" otherwise. A character + "c" is alphanumeric if one of the following returns "True": + "c.isalpha()", "c.isdecimal()", "c.isdigit()", or "c.isnumeric()". + +str.isalpha() + + Return "True" if all characters in the string are alphabetic and + there is at least one character, "False" otherwise. Alphabetic + characters are those characters defined in the Unicode character + database as “Letter”, i.e., those with general category property + being one of “Lm”, “Lt”, “Lu”, “Ll”, or “Lo”. Note that this is + different from the Alphabetic property defined in the section 4.10 + ‘Letters, Alphabetic, and Ideographic’ of the Unicode Standard. + +str.isascii() + + Return "True" if the string is empty or all characters in the + string are ASCII, "False" otherwise. ASCII characters have code + points in the range U+0000-U+007F. + + Added in version 3.7. + +str.isdecimal() + + Return "True" if all characters in the string are decimal + characters and there is at least one character, "False" otherwise. + Decimal characters are those that can be used to form numbers in + base 10, e.g. U+0660, ARABIC-INDIC DIGIT ZERO. Formally a decimal + character is a character in the Unicode General Category “Nd”. + +str.isdigit() + + Return "True" if all characters in the string are digits and there + is at least one character, "False" otherwise. Digits include + decimal characters and digits that need special handling, such as + the compatibility superscript digits. This covers digits which + cannot be used to form numbers in base 10, like the Kharosthi + numbers. Formally, a digit is a character that has the property + value Numeric_Type=Digit or Numeric_Type=Decimal. + +str.isidentifier() + + Return "True" if the string is a valid identifier according to the + language definition, section Identifiers and keywords. + + "keyword.iskeyword()" can be used to test whether string "s" is a + reserved identifier, such as "def" and "class". + + Example: + + >>> from keyword import iskeyword + + >>> 'hello'.isidentifier(), iskeyword('hello') + (True, False) + >>> 'def'.isidentifier(), iskeyword('def') + (True, True) + +str.islower() + + Return "True" if all cased characters [4] in the string are + lowercase and there is at least one cased character, "False" + otherwise. + +str.isnumeric() + + Return "True" if all characters in the string are numeric + characters, and there is at least one character, "False" otherwise. + Numeric characters include digit characters, and all characters + that have the Unicode numeric value property, e.g. U+2155, VULGAR + FRACTION ONE FIFTH. Formally, numeric characters are those with + the property value Numeric_Type=Digit, Numeric_Type=Decimal or + Numeric_Type=Numeric. + +str.isprintable() + + Return "True" if all characters in the string are printable or the + string is empty, "False" otherwise. Nonprintable characters are + those characters defined in the Unicode character database as + “Other” or “Separator”, excepting the ASCII space (0x20) which is + considered printable. (Note that printable characters in this + context are those which should not be escaped when "repr()" is + invoked on a string. It has no bearing on the handling of strings + written to "sys.stdout" or "sys.stderr".) + +str.isspace() + + Return "True" if there are only whitespace characters in the string + and there is at least one character, "False" otherwise. + + A character is *whitespace* if in the Unicode character database + (see "unicodedata"), either its general category is "Zs" + (“Separator, space”), or its bidirectional class is one of "WS", + "B", or "S". + +str.istitle() + + Return "True" if the string is a titlecased string and there is at + least one character, for example uppercase characters may only + follow uncased characters and lowercase characters only cased ones. + Return "False" otherwise. + +str.isupper() + + Return "True" if all cased characters [4] in the string are + uppercase and there is at least one cased character, "False" + otherwise. + + >>> 'BANANA'.isupper() + True + >>> 'banana'.isupper() + False + >>> 'baNana'.isupper() + False + >>> ' '.isupper() + False + +str.join(iterable) + + Return a string which is the concatenation of the strings in + *iterable*. A "TypeError" will be raised if there are any non- + string values in *iterable*, including "bytes" objects. The + separator between elements is the string providing this method. + +str.ljust(width[, fillchar]) + + Return the string left justified in a string of length *width*. + Padding is done using the specified *fillchar* (default is an ASCII + space). The original string is returned if *width* is less than or + equal to "len(s)". + +str.lower() + + Return a copy of the string with all the cased characters [4] + converted to lowercase. + + The lowercasing algorithm used is described in section 3.13 + ‘Default Case Folding’ of the Unicode Standard. + +str.lstrip([chars]) + + Return a copy of the string with leading characters removed. The + *chars* argument is a string specifying the set of characters to be + removed. If omitted or "None", the *chars* argument defaults to + removing whitespace. The *chars* argument is not a prefix; rather, + all combinations of its values are stripped: + + >>> ' spacious '.lstrip() + 'spacious ' + >>> 'www.example.com'.lstrip('cmowz.') + 'example.com' + + See "str.removeprefix()" for a method that will remove a single + prefix string rather than all of a set of characters. For example: + + >>> 'Arthur: three!'.lstrip('Arthur: ') + 'ee!' + >>> 'Arthur: three!'.removeprefix('Arthur: ') + 'three!' + +static str.maketrans(x[, y[, z]]) + + This static method returns a translation table usable for + "str.translate()". + + If there is only one argument, it must be a dictionary mapping + Unicode ordinals (integers) or characters (strings of length 1) to + Unicode ordinals, strings (of arbitrary lengths) or "None". + Character keys will then be converted to ordinals. + + If there are two arguments, they must be strings of equal length, + and in the resulting dictionary, each character in x will be mapped + to the character at the same position in y. If there is a third + argument, it must be a string, whose characters will be mapped to + "None" in the result. + +str.partition(sep) + + Split the string at the first occurrence of *sep*, and return a + 3-tuple containing the part before the separator, the separator + itself, and the part after the separator. If the separator is not + found, return a 3-tuple containing the string itself, followed by + two empty strings. + +str.removeprefix(prefix, /) + + If the string starts with the *prefix* string, return + "string[len(prefix):]". Otherwise, return a copy of the original + string: + + >>> 'TestHook'.removeprefix('Test') + 'Hook' + >>> 'BaseTestCase'.removeprefix('Test') + 'BaseTestCase' + + Added in version 3.9. + +str.removesuffix(suffix, /) + + If the string ends with the *suffix* string and that *suffix* is + not empty, return "string[:-len(suffix)]". Otherwise, return a copy + of the original string: + + >>> 'MiscTests'.removesuffix('Tests') + 'Misc' + >>> 'TmpDirMixin'.removesuffix('Tests') + 'TmpDirMixin' + + Added in version 3.9. + +str.replace(old, new, count=-1) + + Return a copy of the string with all occurrences of substring *old* + replaced by *new*. If *count* is given, only the first *count* + occurrences are replaced. If *count* is not specified or "-1", then + all occurrences are replaced. + + Changed in version 3.13: *count* is now supported as a keyword + argument. + +str.rfind(sub[, start[, end]]) + + Return the highest index in the string where substring *sub* is + found, such that *sub* is contained within "s[start:end]". + Optional arguments *start* and *end* are interpreted as in slice + notation. Return "-1" on failure. + +str.rindex(sub[, start[, end]]) + + Like "rfind()" but raises "ValueError" when the substring *sub* is + not found. + +str.rjust(width[, fillchar]) + + Return the string right justified in a string of length *width*. + Padding is done using the specified *fillchar* (default is an ASCII + space). The original string is returned if *width* is less than or + equal to "len(s)". + +str.rpartition(sep) + + Split the string at the last occurrence of *sep*, and return a + 3-tuple containing the part before the separator, the separator + itself, and the part after the separator. If the separator is not + found, return a 3-tuple containing two empty strings, followed by + the string itself. + +str.rsplit(sep=None, maxsplit=-1) + + Return a list of the words in the string, using *sep* as the + delimiter string. If *maxsplit* is given, at most *maxsplit* splits + are done, the *rightmost* ones. If *sep* is not specified or + "None", any whitespace string is a separator. Except for splitting + from the right, "rsplit()" behaves like "split()" which is + described in detail below. + +str.rstrip([chars]) + + Return a copy of the string with trailing characters removed. The + *chars* argument is a string specifying the set of characters to be + removed. If omitted or "None", the *chars* argument defaults to + removing whitespace. The *chars* argument is not a suffix; rather, + all combinations of its values are stripped: + + >>> ' spacious '.rstrip() + ' spacious' + >>> 'mississippi'.rstrip('ipz') + 'mississ' + + See "str.removesuffix()" for a method that will remove a single + suffix string rather than all of a set of characters. For example: + + >>> 'Monty Python'.rstrip(' Python') + 'M' + >>> 'Monty Python'.removesuffix(' Python') + 'Monty' + +str.split(sep=None, maxsplit=-1) + + Return a list of the words in the string, using *sep* as the + delimiter string. If *maxsplit* is given, at most *maxsplit* + splits are done (thus, the list will have at most "maxsplit+1" + elements). If *maxsplit* is not specified or "-1", then there is + no limit on the number of splits (all possible splits are made). + + If *sep* is given, consecutive delimiters are not grouped together + and are deemed to delimit empty strings (for example, + "'1,,2'.split(',')" returns "['1', '', '2']"). The *sep* argument + may consist of multiple characters as a single delimiter (to split + with multiple delimiters, use "re.split()"). Splitting an empty + string with a specified separator returns "['']". + + For example: + + >>> '1,2,3'.split(',') + ['1', '2', '3'] + >>> '1,2,3'.split(',', maxsplit=1) + ['1', '2,3'] + >>> '1,2,,3,'.split(',') + ['1', '2', '', '3', ''] + >>> '1<>2<>3<4'.split('<>') + ['1', '2', '3<4'] + + If *sep* is not specified or is "None", a different splitting + algorithm is applied: runs of consecutive whitespace are regarded + as a single separator, and the result will contain no empty strings + at the start or end if the string has leading or trailing + whitespace. Consequently, splitting an empty string or a string + consisting of just whitespace with a "None" separator returns "[]". + + For example: + + >>> '1 2 3'.split() + ['1', '2', '3'] + >>> '1 2 3'.split(maxsplit=1) + ['1', '2 3'] + >>> ' 1 2 3 '.split() + ['1', '2', '3'] + +str.splitlines(keepends=False) + + Return a list of the lines in the string, breaking at line + boundaries. Line breaks are not included in the resulting list + unless *keepends* is given and true. + + This method splits on the following line boundaries. In + particular, the boundaries are a superset of *universal newlines*. + + +-------------------------+-------------------------------+ + | Representation | Description | + |=========================|===============================| + | "\n" | Line Feed | + +-------------------------+-------------------------------+ + | "\r" | Carriage Return | + +-------------------------+-------------------------------+ + | "\r\n" | Carriage Return + Line Feed | + +-------------------------+-------------------------------+ + | "\v" or "\x0b" | Line Tabulation | + +-------------------------+-------------------------------+ + | "\f" or "\x0c" | Form Feed | + +-------------------------+-------------------------------+ + | "\x1c" | File Separator | + +-------------------------+-------------------------------+ + | "\x1d" | Group Separator | + +-------------------------+-------------------------------+ + | "\x1e" | Record Separator | + +-------------------------+-------------------------------+ + | "\x85" | Next Line (C1 Control Code) | + +-------------------------+-------------------------------+ + | "\u2028" | Line Separator | + +-------------------------+-------------------------------+ + | "\u2029" | Paragraph Separator | + +-------------------------+-------------------------------+ + + Changed in version 3.2: "\v" and "\f" added to list of line + boundaries. + + For example: + + >>> 'ab c\n\nde fg\rkl\r\n'.splitlines() + ['ab c', '', 'de fg', 'kl'] + >>> 'ab c\n\nde fg\rkl\r\n'.splitlines(keepends=True) + ['ab c\n', '\n', 'de fg\r', 'kl\r\n'] + + Unlike "split()" when a delimiter string *sep* is given, this + method returns an empty list for the empty string, and a terminal + line break does not result in an extra line: + + >>> "".splitlines() + [] + >>> "One line\n".splitlines() + ['One line'] + + For comparison, "split('\n')" gives: + + >>> ''.split('\n') + [''] + >>> 'Two lines\n'.split('\n') + ['Two lines', ''] + +str.startswith(prefix[, start[, end]]) + + Return "True" if string starts with the *prefix*, otherwise return + "False". *prefix* can also be a tuple of prefixes to look for. + With optional *start*, test string beginning at that position. + With optional *end*, stop comparing string at that position. + +str.strip([chars]) + + Return a copy of the string with the leading and trailing + characters removed. The *chars* argument is a string specifying the + set of characters to be removed. If omitted or "None", the *chars* + argument defaults to removing whitespace. The *chars* argument is + not a prefix or suffix; rather, all combinations of its values are + stripped: + + >>> ' spacious '.strip() + 'spacious' + >>> 'www.example.com'.strip('cmowz.') + 'example' + + The outermost leading and trailing *chars* argument values are + stripped from the string. Characters are removed from the leading + end until reaching a string character that is not contained in the + set of characters in *chars*. A similar action takes place on the + trailing end. For example: + + >>> comment_string = '#....... Section 3.2.1 Issue #32 .......' + >>> comment_string.strip('.#! ') + 'Section 3.2.1 Issue #32' + +str.swapcase() + + Return a copy of the string with uppercase characters converted to + lowercase and vice versa. Note that it is not necessarily true that + "s.swapcase().swapcase() == s". + +str.title() + + Return a titlecased version of the string where words start with an + uppercase character and the remaining characters are lowercase. + + For example: + + >>> 'Hello world'.title() + 'Hello World' + + The algorithm uses a simple language-independent definition of a + word as groups of consecutive letters. The definition works in + many contexts but it means that apostrophes in contractions and + possessives form word boundaries, which may not be the desired + result: + + >>> "they're bill's friends from the UK".title() + "They'Re Bill'S Friends From The Uk" + + The "string.capwords()" function does not have this problem, as it + splits words on spaces only. + + Alternatively, a workaround for apostrophes can be constructed + using regular expressions: + + >>> import re + >>> def titlecase(s): + ... return re.sub(r"[A-Za-z]+('[A-Za-z]+)?", + ... lambda mo: mo.group(0).capitalize(), + ... s) + ... + >>> titlecase("they're bill's friends.") + "They're Bill's Friends." + +str.translate(table) + + Return a copy of the string in which each character has been mapped + through the given translation table. The table must be an object + that implements indexing via "__getitem__()", typically a *mapping* + or *sequence*. When indexed by a Unicode ordinal (an integer), the + table object can do any of the following: return a Unicode ordinal + or a string, to map the character to one or more other characters; + return "None", to delete the character from the return string; or + raise a "LookupError" exception, to map the character to itself. + + You can use "str.maketrans()" to create a translation map from + character-to-character mappings in different formats. + + See also the "codecs" module for a more flexible approach to custom + character mappings. + +str.upper() + + Return a copy of the string with all the cased characters [4] + converted to uppercase. Note that "s.upper().isupper()" might be + "False" if "s" contains uncased characters or if the Unicode + category of the resulting character(s) is not “Lu” (Letter, + uppercase), but e.g. “Lt” (Letter, titlecase). + + The uppercasing algorithm used is described in section 3.13 + ‘Default Case Folding’ of the Unicode Standard. + +str.zfill(width) + + Return a copy of the string left filled with ASCII "'0'" digits to + make a string of length *width*. A leading sign prefix + ("'+'"/"'-'") is handled by inserting the padding *after* the sign + character rather than before. The original string is returned if + *width* is less than or equal to "len(s)". + + For example: + + >>> "42".zfill(5) + '00042' + >>> "-42".zfill(5) + '-0042' +''', + 'strings': '''String and Bytes literals +************************* + +String literals are described by the following lexical definitions: + + stringliteral ::= [stringprefix](shortstring | longstring) + stringprefix ::= "r" | "u" | "R" | "U" | "f" | "F" + | "fr" | "Fr" | "fR" | "FR" | "rf" | "rF" | "Rf" | "RF" + shortstring ::= "'" shortstringitem* "'" | '"' shortstringitem* '"' + longstring ::= "\'\'\'" longstringitem* "\'\'\'" | '"""' longstringitem* '"""' + shortstringitem ::= shortstringchar | stringescapeseq + longstringitem ::= longstringchar | stringescapeseq + shortstringchar ::= + longstringchar ::= + stringescapeseq ::= "\\" + + bytesliteral ::= bytesprefix(shortbytes | longbytes) + bytesprefix ::= "b" | "B" | "br" | "Br" | "bR" | "BR" | "rb" | "rB" | "Rb" | "RB" + shortbytes ::= "'" shortbytesitem* "'" | '"' shortbytesitem* '"' + longbytes ::= "\'\'\'" longbytesitem* "\'\'\'" | '"""' longbytesitem* '"""' + shortbytesitem ::= shortbyteschar | bytesescapeseq + longbytesitem ::= longbyteschar | bytesescapeseq + shortbyteschar ::= + longbyteschar ::= + bytesescapeseq ::= "\\" + +One syntactic restriction not indicated by these productions is that +whitespace is not allowed between the "stringprefix" or "bytesprefix" +and the rest of the literal. The source character set is defined by +the encoding declaration; it is UTF-8 if no encoding declaration is +given in the source file; see section Encoding declarations. + +In plain English: Both types of literals can be enclosed in matching +single quotes ("'") or double quotes ("""). They can also be enclosed +in matching groups of three single or double quotes (these are +generally referred to as *triple-quoted strings*). The backslash ("\\") +character is used to give special meaning to otherwise ordinary +characters like "n", which means ‘newline’ when escaped ("\\n"). It can +also be used to escape characters that otherwise have a special +meaning, such as newline, backslash itself, or the quote character. +See escape sequences below for examples. + +Bytes literals are always prefixed with "'b'" or "'B'"; they produce +an instance of the "bytes" type instead of the "str" type. They may +only contain ASCII characters; bytes with a numeric value of 128 or +greater must be expressed with escapes. + +Both string and bytes literals may optionally be prefixed with a +letter "'r'" or "'R'"; such constructs are called *raw string +literals* and *raw bytes literals* respectively and treat backslashes +as literal characters. As a result, in raw string literals, "'\\U'" +and "'\\u'" escapes are not treated specially. + +Added in version 3.3: The "'rb'" prefix of raw bytes literals has been +added as a synonym of "'br'".Support for the unicode legacy literal +("u'value'") was reintroduced to simplify the maintenance of dual +Python 2.x and 3.x codebases. See **PEP 414** for more information. + +A string literal with "'f'" or "'F'" in its prefix is a *formatted +string literal*; see f-strings. The "'f'" may be combined with "'r'", +but not with "'b'" or "'u'", therefore raw formatted strings are +possible, but formatted bytes literals are not. + +In triple-quoted literals, unescaped newlines and quotes are allowed +(and are retained), except that three unescaped quotes in a row +terminate the literal. (A “quote” is the character used to open the +literal, i.e. either "'" or """.) + + +Escape sequences +================ + +Unless an "'r'" or "'R'" prefix is present, escape sequences in string +and bytes literals are interpreted according to rules similar to those +used by Standard C. The recognized escape sequences are: + ++---------------------------+-----------------------------------+---------+ +| Escape Sequence | Meaning | Notes | +|===========================|===================================|=========| +| "\\" | Backslash and newline ignored | (1) | ++---------------------------+-----------------------------------+---------+ +| "\\\\" | Backslash ("\\") | | ++---------------------------+-----------------------------------+---------+ +| "\\'" | Single quote ("'") | | ++---------------------------+-----------------------------------+---------+ +| "\\"" | Double quote (""") | | ++---------------------------+-----------------------------------+---------+ +| "\\a" | ASCII Bell (BEL) | | ++---------------------------+-----------------------------------+---------+ +| "\\b" | ASCII Backspace (BS) | | ++---------------------------+-----------------------------------+---------+ +| "\\f" | ASCII Formfeed (FF) | | ++---------------------------+-----------------------------------+---------+ +| "\\n" | ASCII Linefeed (LF) | | ++---------------------------+-----------------------------------+---------+ +| "\\r" | ASCII Carriage Return (CR) | | ++---------------------------+-----------------------------------+---------+ +| "\\t" | ASCII Horizontal Tab (TAB) | | ++---------------------------+-----------------------------------+---------+ +| "\\v" | ASCII Vertical Tab (VT) | | ++---------------------------+-----------------------------------+---------+ +| "\\*ooo*" | Character with octal value *ooo* | (2,4) | ++---------------------------+-----------------------------------+---------+ +| "\\x*hh*" | Character with hex value *hh* | (3,4) | ++---------------------------+-----------------------------------+---------+ + +Escape sequences only recognized in string literals are: + ++---------------------------+-----------------------------------+---------+ +| Escape Sequence | Meaning | Notes | +|===========================|===================================|=========| +| "\\N{*name*}" | Character named *name* in the | (5) | +| | Unicode database | | ++---------------------------+-----------------------------------+---------+ +| "\\u*xxxx*" | Character with 16-bit hex value | (6) | +| | *xxxx* | | ++---------------------------+-----------------------------------+---------+ +| "\\U*xxxxxxxx*" | Character with 32-bit hex value | (7) | +| | *xxxxxxxx* | | ++---------------------------+-----------------------------------+---------+ + +Notes: + +1. A backslash can be added at the end of a line to ignore the + newline: + + >>> 'This string will not include \\ + ... backslashes or newline characters.' + 'This string will not include backslashes or newline characters.' + + The same result can be achieved using triple-quoted strings, or + parentheses and string literal concatenation. + +2. As in Standard C, up to three octal digits are accepted. + + Changed in version 3.11: Octal escapes with value larger than + "0o377" produce a "DeprecationWarning". + + Changed in version 3.12: Octal escapes with value larger than + "0o377" produce a "SyntaxWarning". In a future Python version they + will be eventually a "SyntaxError". + +3. Unlike in Standard C, exactly two hex digits are required. + +4. In a bytes literal, hexadecimal and octal escapes denote the byte + with the given value. In a string literal, these escapes denote a + Unicode character with the given value. + +5. Changed in version 3.3: Support for name aliases [1] has been + added. + +6. Exactly four hex digits are required. + +7. Any Unicode character can be encoded this way. Exactly eight hex + digits are required. + +Unlike Standard C, all unrecognized escape sequences are left in the +string unchanged, i.e., *the backslash is left in the result*. (This +behavior is useful when debugging: if an escape sequence is mistyped, +the resulting output is more easily recognized as broken.) It is also +important to note that the escape sequences only recognized in string +literals fall into the category of unrecognized escapes for bytes +literals. + +Changed in version 3.6: Unrecognized escape sequences produce a +"DeprecationWarning". + +Changed in version 3.12: Unrecognized escape sequences produce a +"SyntaxWarning". In a future Python version they will be eventually a +"SyntaxError". + +Even in a raw literal, quotes can be escaped with a backslash, but the +backslash remains in the result; for example, "r"\\""" is a valid +string literal consisting of two characters: a backslash and a double +quote; "r"\\"" is not a valid string literal (even a raw string cannot +end in an odd number of backslashes). Specifically, *a raw literal +cannot end in a single backslash* (since the backslash would escape +the following quote character). Note also that a single backslash +followed by a newline is interpreted as those two characters as part +of the literal, *not* as a line continuation. +''', + 'subscriptions': r'''Subscriptions +************* + +The subscription of an instance of a container class will generally +select an element from the container. The subscription of a *generic +class* will generally return a GenericAlias object. + + subscription ::= primary "[" flexible_expression_list "]" + +When an object is subscripted, the interpreter will evaluate the +primary and the expression list. + +The primary must evaluate to an object that supports subscription. An +object may support subscription through defining one or both of +"__getitem__()" and "__class_getitem__()". When the primary is +subscripted, the evaluated result of the expression list will be +passed to one of these methods. For more details on when +"__class_getitem__" is called instead of "__getitem__", see +__class_getitem__ versus __getitem__. + +If the expression list contains at least one comma, or if any of the +expressions are starred, the expression list will evaluate to a +"tuple" containing the items of the expression list. Otherwise, the +expression list will evaluate to the value of the list’s sole member. + +Changed in version 3.11: Expressions in an expression list may be +starred. See **PEP 646**. + +For built-in objects, there are two types of objects that support +subscription via "__getitem__()": + +1. Mappings. If the primary is a *mapping*, the expression list must + evaluate to an object whose value is one of the keys of the + mapping, and the subscription selects the value in the mapping that + corresponds to that key. An example of a builtin mapping class is + the "dict" class. + +2. Sequences. If the primary is a *sequence*, the expression list must + evaluate to an "int" or a "slice" (as discussed in the following + section). Examples of builtin sequence classes include the "str", + "list" and "tuple" classes. + +The formal syntax makes no special provision for negative indices in +*sequences*. However, built-in sequences all provide a "__getitem__()" +method that interprets negative indices by adding the length of the +sequence to the index so that, for example, "x[-1]" selects the last +item of "x". The resulting value must be a nonnegative integer less +than the number of items in the sequence, and the subscription selects +the item whose index is that value (counting from zero). Since the +support for negative indices and slicing occurs in the object’s +"__getitem__()" method, subclasses overriding this method will need to +explicitly add that support. + +A "string" is a special kind of sequence whose items are *characters*. +A character is not a separate data type but a string of exactly one +character. +''', + 'truth': r'''Truth Value Testing +******************* + +Any object can be tested for truth value, for use in an "if" or +"while" condition or as operand of the Boolean operations below. + +By default, an object is considered true unless its class defines +either a "__bool__()" method that returns "False" or a "__len__()" +method that returns zero, when called with the object. [1] Here are +most of the built-in objects considered false: + +* constants defined to be false: "None" and "False" + +* zero of any numeric type: "0", "0.0", "0j", "Decimal(0)", + "Fraction(0, 1)" + +* empty sequences and collections: "''", "()", "[]", "{}", "set()", + "range(0)" + +Operations and built-in functions that have a Boolean result always +return "0" or "False" for false and "1" or "True" for true, unless +otherwise stated. (Important exception: the Boolean operations "or" +and "and" always return one of their operands.) +''', + 'try': r'''The "try" statement +******************* + +The "try" statement specifies exception handlers and/or cleanup code +for a group of statements: + + try_stmt ::= try1_stmt | try2_stmt | try3_stmt + try1_stmt ::= "try" ":" suite + ("except" [expression ["as" identifier]] ":" suite)+ + ["else" ":" suite] + ["finally" ":" suite] + try2_stmt ::= "try" ":" suite + ("except" "*" expression ["as" identifier] ":" suite)+ + ["else" ":" suite] + ["finally" ":" suite] + try3_stmt ::= "try" ":" suite + "finally" ":" suite + +Additional information on exceptions can be found in section +Exceptions, and information on using the "raise" statement to generate +exceptions may be found in section The raise statement. + + +"except" clause +=============== + +The "except" clause(s) specify one or more exception handlers. When no +exception occurs in the "try" clause, no exception handler is +executed. When an exception occurs in the "try" suite, a search for an +exception handler is started. This search inspects the "except" +clauses in turn until one is found that matches the exception. An +expression-less "except" clause, if present, must be last; it matches +any exception. + +For an "except" clause with an expression, the expression must +evaluate to an exception type or a tuple of exception types. The +raised exception matches an "except" clause whose expression evaluates +to the class or a *non-virtual base class* of the exception object, or +to a tuple that contains such a class. + +If no "except" clause matches the exception, the search for an +exception handler continues in the surrounding code and on the +invocation stack. [1] + +If the evaluation of an expression in the header of an "except" clause +raises an exception, the original search for a handler is canceled and +a search starts for the new exception in the surrounding code and on +the call stack (it is treated as if the entire "try" statement raised +the exception). + +When a matching "except" clause is found, the exception is assigned to +the target specified after the "as" keyword in that "except" clause, +if present, and the "except" clause’s suite is executed. All "except" +clauses must have an executable block. When the end of this block is +reached, execution continues normally after the entire "try" +statement. (This means that if two nested handlers exist for the same +exception, and the exception occurs in the "try" clause of the inner +handler, the outer handler will not handle the exception.) + +When an exception has been assigned using "as target", it is cleared +at the end of the "except" clause. This is as if + + except E as N: + foo + +was translated to + + except E as N: + try: + foo + finally: + del N + +This means the exception must be assigned to a different name to be +able to refer to it after the "except" clause. Exceptions are cleared +because with the traceback attached to them, they form a reference +cycle with the stack frame, keeping all locals in that frame alive +until the next garbage collection occurs. + +Before an "except" clause’s suite is executed, the exception is stored +in the "sys" module, where it can be accessed from within the body of +the "except" clause by calling "sys.exception()". When leaving an +exception handler, the exception stored in the "sys" module is reset +to its previous value: + + >>> print(sys.exception()) + None + >>> try: + ... raise TypeError + ... except: + ... print(repr(sys.exception())) + ... try: + ... raise ValueError + ... except: + ... print(repr(sys.exception())) + ... print(repr(sys.exception())) + ... + TypeError() + ValueError() + TypeError() + >>> print(sys.exception()) + None + + +"except*" clause +================ + +The "except*" clause(s) are used for handling "ExceptionGroup"s. The +exception type for matching is interpreted as in the case of "except", +but in the case of exception groups we can have partial matches when +the type matches some of the exceptions in the group. This means that +multiple "except*" clauses can execute, each handling part of the +exception group. Each clause executes at most once and handles an +exception group of all matching exceptions. Each exception in the +group is handled by at most one "except*" clause, the first that +matches it. + + >>> try: + ... raise ExceptionGroup("eg", + ... [ValueError(1), TypeError(2), OSError(3), OSError(4)]) + ... except* TypeError as e: + ... print(f'caught {type(e)} with nested {e.exceptions}') + ... except* OSError as e: + ... print(f'caught {type(e)} with nested {e.exceptions}') + ... + caught with nested (TypeError(2),) + caught with nested (OSError(3), OSError(4)) + + Exception Group Traceback (most recent call last): + | File "", line 2, in + | ExceptionGroup: eg + +-+---------------- 1 ---------------- + | ValueError: 1 + +------------------------------------ + +Any remaining exceptions that were not handled by any "except*" clause +are re-raised at the end, along with all exceptions that were raised +from within the "except*" clauses. If this list contains more than one +exception to reraise, they are combined into an exception group. + +If the raised exception is not an exception group and its type matches +one of the "except*" clauses, it is caught and wrapped by an exception +group with an empty message string. + + >>> try: + ... raise BlockingIOError + ... except* BlockingIOError as e: + ... print(repr(e)) + ... + ExceptionGroup('', (BlockingIOError())) + +An "except*" clause must have a matching expression; it cannot be +"except*:". Furthermore, this expression cannot contain exception +group types, because that would have ambiguous semantics. + +It is not possible to mix "except" and "except*" in the same "try". +"break", "continue" and "return" cannot appear in an "except*" clause. + + +"else" clause +============= + +The optional "else" clause is executed if the control flow leaves the +"try" suite, no exception was raised, and no "return", "continue", or +"break" statement was executed. Exceptions in the "else" clause are +not handled by the preceding "except" clauses. + + +"finally" clause +================ + +If "finally" is present, it specifies a ‘cleanup’ handler. The "try" +clause is executed, including any "except" and "else" clauses. If an +exception occurs in any of the clauses and is not handled, the +exception is temporarily saved. The "finally" clause is executed. If +there is a saved exception it is re-raised at the end of the "finally" +clause. If the "finally" clause raises another exception, the saved +exception is set as the context of the new exception. If the "finally" +clause executes a "return", "break" or "continue" statement, the saved +exception is discarded: + + >>> def f(): + ... try: + ... 1/0 + ... finally: + ... return 42 + ... + >>> f() + 42 + +The exception information is not available to the program during +execution of the "finally" clause. + +When a "return", "break" or "continue" statement is executed in the +"try" suite of a "try"…"finally" statement, the "finally" clause is +also executed ‘on the way out.’ + +The return value of a function is determined by the last "return" +statement executed. Since the "finally" clause always executes, a +"return" statement executed in the "finally" clause will always be the +last one executed: + + >>> def foo(): + ... try: + ... return 'try' + ... finally: + ... return 'finally' + ... + >>> foo() + 'finally' + +Changed in version 3.8: Prior to Python 3.8, a "continue" statement +was illegal in the "finally" clause due to a problem with the +implementation. +''', + 'types': r'''The standard type hierarchy +*************************** + +Below is a list of the types that are built into Python. Extension +modules (written in C, Java, or other languages, depending on the +implementation) can define additional types. Future versions of +Python may add types to the type hierarchy (e.g., rational numbers, +efficiently stored arrays of integers, etc.), although such additions +will often be provided via the standard library instead. + +Some of the type descriptions below contain a paragraph listing +‘special attributes.’ These are attributes that provide access to the +implementation and are not intended for general use. Their definition +may change in the future. + + +None +==== + +This type has a single value. There is a single object with this +value. This object is accessed through the built-in name "None". It is +used to signify the absence of a value in many situations, e.g., it is +returned from functions that don’t explicitly return anything. Its +truth value is false. + + +NotImplemented +============== + +This type has a single value. There is a single object with this +value. This object is accessed through the built-in name +"NotImplemented". Numeric methods and rich comparison methods should +return this value if they do not implement the operation for the +operands provided. (The interpreter will then try the reflected +operation, or some other fallback, depending on the operator.) It +should not be evaluated in a boolean context. + +See Implementing the arithmetic operations for more details. + +Changed in version 3.9: Evaluating "NotImplemented" in a boolean +context was deprecated. + +Changed in version 3.14: Evaluating "NotImplemented" in a boolean +context now raises a "TypeError". It previously evaluated to "True" +and emitted a "DeprecationWarning" since Python 3.9. + + +Ellipsis +======== + +This type has a single value. There is a single object with this +value. This object is accessed through the literal "..." or the built- +in name "Ellipsis". Its truth value is true. + + +"numbers.Number" +================ + +These are created by numeric literals and returned as results by +arithmetic operators and arithmetic built-in functions. Numeric +objects are immutable; once created their value never changes. Python +numbers are of course strongly related to mathematical numbers, but +subject to the limitations of numerical representation in computers. + +The string representations of the numeric classes, computed by +"__repr__()" and "__str__()", have the following properties: + +* They are valid numeric literals which, when passed to their class + constructor, produce an object having the value of the original + numeric. + +* The representation is in base 10, when possible. + +* Leading zeros, possibly excepting a single zero before a decimal + point, are not shown. + +* Trailing zeros, possibly excepting a single zero after a decimal + point, are not shown. + +* A sign is shown only when the number is negative. + +Python distinguishes between integers, floating-point numbers, and +complex numbers: + + +"numbers.Integral" +------------------ + +These represent elements from the mathematical set of integers +(positive and negative). + +Note: + + The rules for integer representation are intended to give the most + meaningful interpretation of shift and mask operations involving + negative integers. + +There are two types of integers: + +Integers ("int") + These represent numbers in an unlimited range, subject to available + (virtual) memory only. For the purpose of shift and mask + operations, a binary representation is assumed, and negative + numbers are represented in a variant of 2’s complement which gives + the illusion of an infinite string of sign bits extending to the + left. + +Booleans ("bool") + These represent the truth values False and True. The two objects + representing the values "False" and "True" are the only Boolean + objects. The Boolean type is a subtype of the integer type, and + Boolean values behave like the values 0 and 1, respectively, in + almost all contexts, the exception being that when converted to a + string, the strings ""False"" or ""True"" are returned, + respectively. + + +"numbers.Real" ("float") +------------------------ + +These represent machine-level double precision floating-point numbers. +You are at the mercy of the underlying machine architecture (and C or +Java implementation) for the accepted range and handling of overflow. +Python does not support single-precision floating-point numbers; the +savings in processor and memory usage that are usually the reason for +using these are dwarfed by the overhead of using objects in Python, so +there is no reason to complicate the language with two kinds of +floating-point numbers. + + +"numbers.Complex" ("complex") +----------------------------- + +These represent complex numbers as a pair of machine-level double +precision floating-point numbers. The same caveats apply as for +floating-point numbers. The real and imaginary parts of a complex +number "z" can be retrieved through the read-only attributes "z.real" +and "z.imag". + + +Sequences +========= + +These represent finite ordered sets indexed by non-negative numbers. +The built-in function "len()" returns the number of items of a +sequence. When the length of a sequence is *n*, the index set contains +the numbers 0, 1, …, *n*-1. Item *i* of sequence *a* is selected by +"a[i]". Some sequences, including built-in sequences, interpret +negative subscripts by adding the sequence length. For example, +"a[-2]" equals "a[n-2]", the second to last item of sequence a with +length "n". + +Sequences also support slicing: "a[i:j]" selects all items with index +*k* such that *i* "<=" *k* "<" *j*. When used as an expression, a +slice is a sequence of the same type. The comment above about negative +indexes also applies to negative slice positions. + +Some sequences also support “extended slicing” with a third “step” +parameter: "a[i:j:k]" selects all items of *a* with index *x* where "x += i + n*k", *n* ">=" "0" and *i* "<=" *x* "<" *j*. + +Sequences are distinguished according to their mutability: + + +Immutable sequences +------------------- + +An object of an immutable sequence type cannot change once it is +created. (If the object contains references to other objects, these +other objects may be mutable and may be changed; however, the +collection of objects directly referenced by an immutable object +cannot change.) + +The following types are immutable sequences: + +Strings + A string is a sequence of values that represent Unicode code + points. All the code points in the range "U+0000 - U+10FFFF" can be + represented in a string. Python doesn’t have a char type; instead, + every code point in the string is represented as a string object + with length "1". The built-in function "ord()" converts a code + point from its string form to an integer in the range "0 - 10FFFF"; + "chr()" converts an integer in the range "0 - 10FFFF" to the + corresponding length "1" string object. "str.encode()" can be used + to convert a "str" to "bytes" using the given text encoding, and + "bytes.decode()" can be used to achieve the opposite. + +Tuples + The items of a tuple are arbitrary Python objects. Tuples of two or + more items are formed by comma-separated lists of expressions. A + tuple of one item (a ‘singleton’) can be formed by affixing a comma + to an expression (an expression by itself does not create a tuple, + since parentheses must be usable for grouping of expressions). An + empty tuple can be formed by an empty pair of parentheses. + +Bytes + A bytes object is an immutable array. The items are 8-bit bytes, + represented by integers in the range 0 <= x < 256. Bytes literals + (like "b'abc'") and the built-in "bytes()" constructor can be used + to create bytes objects. Also, bytes objects can be decoded to + strings via the "decode()" method. + + +Mutable sequences +----------------- + +Mutable sequences can be changed after they are created. The +subscription and slicing notations can be used as the target of +assignment and "del" (delete) statements. + +Note: + + The "collections" and "array" module provide additional examples of + mutable sequence types. + +There are currently two intrinsic mutable sequence types: + +Lists + The items of a list are arbitrary Python objects. Lists are formed + by placing a comma-separated list of expressions in square + brackets. (Note that there are no special cases needed to form + lists of length 0 or 1.) + +Byte Arrays + A bytearray object is a mutable array. They are created by the + built-in "bytearray()" constructor. Aside from being mutable (and + hence unhashable), byte arrays otherwise provide the same interface + and functionality as immutable "bytes" objects. + + +Set types +========= + +These represent unordered, finite sets of unique, immutable objects. +As such, they cannot be indexed by any subscript. However, they can be +iterated over, and the built-in function "len()" returns the number of +items in a set. Common uses for sets are fast membership testing, +removing duplicates from a sequence, and computing mathematical +operations such as intersection, union, difference, and symmetric +difference. + +For set elements, the same immutability rules apply as for dictionary +keys. Note that numeric types obey the normal rules for numeric +comparison: if two numbers compare equal (e.g., "1" and "1.0"), only +one of them can be contained in a set. + +There are currently two intrinsic set types: + +Sets + These represent a mutable set. They are created by the built-in + "set()" constructor and can be modified afterwards by several + methods, such as "add()". + +Frozen sets + These represent an immutable set. They are created by the built-in + "frozenset()" constructor. As a frozenset is immutable and + *hashable*, it can be used again as an element of another set, or + as a dictionary key. + + +Mappings +======== + +These represent finite sets of objects indexed by arbitrary index +sets. The subscript notation "a[k]" selects the item indexed by "k" +from the mapping "a"; this can be used in expressions and as the +target of assignments or "del" statements. The built-in function +"len()" returns the number of items in a mapping. + +There is currently a single intrinsic mapping type: + + +Dictionaries +------------ + +These represent finite sets of objects indexed by nearly arbitrary +values. The only types of values not acceptable as keys are values +containing lists or dictionaries or other mutable types that are +compared by value rather than by object identity, the reason being +that the efficient implementation of dictionaries requires a key’s +hash value to remain constant. Numeric types used for keys obey the +normal rules for numeric comparison: if two numbers compare equal +(e.g., "1" and "1.0") then they can be used interchangeably to index +the same dictionary entry. + +Dictionaries preserve insertion order, meaning that keys will be +produced in the same order they were added sequentially over the +dictionary. Replacing an existing key does not change the order, +however removing a key and re-inserting it will add it to the end +instead of keeping its old place. + +Dictionaries are mutable; they can be created by the "{}" notation +(see section Dictionary displays). + +The extension modules "dbm.ndbm" and "dbm.gnu" provide additional +examples of mapping types, as does the "collections" module. + +Changed in version 3.7: Dictionaries did not preserve insertion order +in versions of Python before 3.6. In CPython 3.6, insertion order was +preserved, but it was considered an implementation detail at that time +rather than a language guarantee. + + +Callable types +============== + +These are the types to which the function call operation (see section +Calls) can be applied: + + +User-defined functions +---------------------- + +A user-defined function object is created by a function definition +(see section Function definitions). It should be called with an +argument list containing the same number of items as the function’s +formal parameter list. + + +Special read-only attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++----------------------------------------------------+----------------------------------------------------+ +| Attribute | Meaning | +|====================================================|====================================================| +| function.__globals__ | A reference to the "dictionary" that holds the | +| | function’s global variables – the global namespace | +| | of the module in which the function was defined. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__closure__ | "None" or a "tuple" of cells that contain bindings | +| | for the names specified in the "co_freevars" | +| | attribute of the function’s "code object". A cell | +| | object has the attribute "cell_contents". This can | +| | be used to get the value of the cell, as well as | +| | set the value. | ++----------------------------------------------------+----------------------------------------------------+ + + +Special writable attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Most of these attributes check the type of the assigned value: + ++----------------------------------------------------+----------------------------------------------------+ +| Attribute | Meaning | +|====================================================|====================================================| +| function.__doc__ | The function’s documentation string, or "None" if | +| | unavailable. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__name__ | The function’s name. See also: "__name__ | +| | attributes". | ++----------------------------------------------------+----------------------------------------------------+ +| function.__qualname__ | The function’s *qualified name*. See also: | +| | "__qualname__ attributes". Added in version 3.3. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__module__ | The name of the module the function was defined | +| | in, or "None" if unavailable. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__defaults__ | A "tuple" containing default *parameter* values | +| | for those parameters that have defaults, or "None" | +| | if no parameters have a default value. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__code__ | The code object representing the compiled function | +| | body. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__dict__ | The namespace supporting arbitrary function | +| | attributes. See also: "__dict__ attributes". | ++----------------------------------------------------+----------------------------------------------------+ +| function.__annotations__ | A "dictionary" containing annotations of | +| | *parameters*. The keys of the dictionary are the | +| | parameter names, and "'return'" for the return | +| | annotation, if provided. See also: | +| | "object.__annotations__". Changed in version | +| | 3.14: Annotations are now lazily evaluated. See | +| | **PEP 649**. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__annotate__ | The *annotate function* for this function, or | +| | "None" if the function has no annotations. See | +| | "object.__annotate__". Added in version 3.14. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__kwdefaults__ | A "dictionary" containing defaults for keyword- | +| | only *parameters*. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__type_params__ | A "tuple" containing the type parameters of a | +| | generic function. Added in version 3.12. | ++----------------------------------------------------+----------------------------------------------------+ + +Function objects also support getting and setting arbitrary +attributes, which can be used, for example, to attach metadata to +functions. Regular attribute dot-notation is used to get and set such +attributes. + +**CPython implementation detail:** CPython’s current implementation +only supports function attributes on user-defined functions. Function +attributes on built-in functions may be supported in the future. + +Additional information about a function’s definition can be retrieved +from its code object (accessible via the "__code__" attribute). + + +Instance methods +---------------- + +An instance method object combines a class, a class instance and any +callable object (normally a user-defined function). + +Special read-only attributes: + ++----------------------------------------------------+----------------------------------------------------+ +| method.__self__ | Refers to the class instance object to which the | +| | method is bound | ++----------------------------------------------------+----------------------------------------------------+ +| method.__func__ | Refers to the original function object | ++----------------------------------------------------+----------------------------------------------------+ +| method.__doc__ | The method’s documentation (same as | +| | "method.__func__.__doc__"). A "string" if the | +| | original function had a docstring, else "None". | ++----------------------------------------------------+----------------------------------------------------+ +| method.__name__ | The name of the method (same as | +| | "method.__func__.__name__") | ++----------------------------------------------------+----------------------------------------------------+ +| method.__module__ | The name of the module the method was defined in, | +| | or "None" if unavailable. | ++----------------------------------------------------+----------------------------------------------------+ + +Methods also support accessing (but not setting) the arbitrary +function attributes on the underlying function object. + +User-defined method objects may be created when getting an attribute +of a class (perhaps via an instance of that class), if that attribute +is a user-defined function object or a "classmethod" object. + +When an instance method object is created by retrieving a user-defined +function object from a class via one of its instances, its "__self__" +attribute is the instance, and the method object is said to be +*bound*. The new method’s "__func__" attribute is the original +function object. + +When an instance method object is created by retrieving a +"classmethod" object from a class or instance, its "__self__" +attribute is the class itself, and its "__func__" attribute is the +function object underlying the class method. + +When an instance method object is called, the underlying function +("__func__") is called, inserting the class instance ("__self__") in +front of the argument list. For instance, when "C" is a class which +contains a definition for a function "f()", and "x" is an instance of +"C", calling "x.f(1)" is equivalent to calling "C.f(x, 1)". + +When an instance method object is derived from a "classmethod" object, +the “class instance” stored in "__self__" will actually be the class +itself, so that calling either "x.f(1)" or "C.f(1)" is equivalent to +calling "f(C,1)" where "f" is the underlying function. + +It is important to note that user-defined functions which are +attributes of a class instance are not converted to bound methods; +this *only* happens when the function is an attribute of the class. + + +Generator functions +------------------- + +A function or method which uses the "yield" statement (see section The +yield statement) is called a *generator function*. Such a function, +when called, always returns an *iterator* object which can be used to +execute the body of the function: calling the iterator’s +"iterator.__next__()" method will cause the function to execute until +it provides a value using the "yield" statement. When the function +executes a "return" statement or falls off the end, a "StopIteration" +exception is raised and the iterator will have reached the end of the +set of values to be returned. + + +Coroutine functions +------------------- + +A function or method which is defined using "async def" is called a +*coroutine function*. Such a function, when called, returns a +*coroutine* object. It may contain "await" expressions, as well as +"async with" and "async for" statements. See also the Coroutine +Objects section. + + +Asynchronous generator functions +-------------------------------- + +A function or method which is defined using "async def" and which uses +the "yield" statement is called a *asynchronous generator function*. +Such a function, when called, returns an *asynchronous iterator* +object which can be used in an "async for" statement to execute the +body of the function. + +Calling the asynchronous iterator’s "aiterator.__anext__" method will +return an *awaitable* which when awaited will execute until it +provides a value using the "yield" expression. When the function +executes an empty "return" statement or falls off the end, a +"StopAsyncIteration" exception is raised and the asynchronous iterator +will have reached the end of the set of values to be yielded. + + +Built-in functions +------------------ + +A built-in function object is a wrapper around a C function. Examples +of built-in functions are "len()" and "math.sin()" ("math" is a +standard built-in module). The number and type of the arguments are +determined by the C function. Special read-only attributes: + +* "__doc__" is the function’s documentation string, or "None" if + unavailable. See "function.__doc__". + +* "__name__" is the function’s name. See "function.__name__". + +* "__self__" is set to "None" (but see the next item). + +* "__module__" is the name of the module the function was defined in + or "None" if unavailable. See "function.__module__". + + +Built-in methods +---------------- + +This is really a different disguise of a built-in function, this time +containing an object passed to the C function as an implicit extra +argument. An example of a built-in method is "alist.append()", +assuming *alist* is a list object. In this case, the special read-only +attribute "__self__" is set to the object denoted by *alist*. (The +attribute has the same semantics as it does with "other instance +methods".) + + +Classes +------- + +Classes are callable. These objects normally act as factories for new +instances of themselves, but variations are possible for class types +that override "__new__()". The arguments of the call are passed to +"__new__()" and, in the typical case, to "__init__()" to initialize +the new instance. + + +Class Instances +--------------- + +Instances of arbitrary classes can be made callable by defining a +"__call__()" method in their class. + + +Modules +======= + +Modules are a basic organizational unit of Python code, and are +created by the import system as invoked either by the "import" +statement, or by calling functions such as "importlib.import_module()" +and built-in "__import__()". A module object has a namespace +implemented by a "dictionary" object (this is the dictionary +referenced by the "__globals__" attribute of functions defined in the +module). Attribute references are translated to lookups in this +dictionary, e.g., "m.x" is equivalent to "m.__dict__["x"]". A module +object does not contain the code object used to initialize the module +(since it isn’t needed once the initialization is done). + +Attribute assignment updates the module’s namespace dictionary, e.g., +"m.x = 1" is equivalent to "m.__dict__["x"] = 1". + + +Import-related attributes on module objects +------------------------------------------- + +Module objects have the following attributes that relate to the import +system. When a module is created using the machinery associated with +the import system, these attributes are filled in based on the +module’s *spec*, before the *loader* executes and loads the module. + +To create a module dynamically rather than using the import system, +it’s recommended to use "importlib.util.module_from_spec()", which +will set the various import-controlled attributes to appropriate +values. It’s also possible to use the "types.ModuleType" constructor +to create modules directly, but this technique is more error-prone, as +most attributes must be manually set on the module object after it has +been created when using this approach. + +Caution: + + With the exception of "__name__", it is **strongly** recommended + that you rely on "__spec__" and its attributes instead of any of the + other individual attributes listed in this subsection. Note that + updating an attribute on "__spec__" will not update the + corresponding attribute on the module itself: + + >>> import typing + >>> typing.__name__, typing.__spec__.name + ('typing', 'typing') + >>> typing.__spec__.name = 'spelling' + >>> typing.__name__, typing.__spec__.name + ('typing', 'spelling') + >>> typing.__name__ = 'keyboard_smashing' + >>> typing.__name__, typing.__spec__.name + ('keyboard_smashing', 'spelling') + +module.__name__ + + The name used to uniquely identify the module in the import system. + For a directly executed module, this will be set to ""__main__"". + + This attribute must be set to the fully qualified name of the + module. It is expected to match the value of + "module.__spec__.name". + +module.__spec__ + + A record of the module’s import-system-related state. + + Set to the "module spec" that was used when importing the module. + See Module specs for more details. + + Added in version 3.4. + +module.__package__ + + The *package* a module belongs to. + + If the module is top-level (that is, not a part of any specific + package) then the attribute should be set to "''" (the empty + string). Otherwise, it should be set to the name of the module’s + package (which can be equal to "module.__name__" if the module + itself is a package). See **PEP 366** for further details. + + This attribute is used instead of "__name__" to calculate explicit + relative imports for main modules. It defaults to "None" for + modules created dynamically using the "types.ModuleType" + constructor; use "importlib.util.module_from_spec()" instead to + ensure the attribute is set to a "str". + + It is **strongly** recommended that you use + "module.__spec__.parent" instead of "module.__package__". + "__package__" is now only used as a fallback if "__spec__.parent" + is not set, and this fallback path is deprecated. + + Changed in version 3.4: This attribute now defaults to "None" for + modules created dynamically using the "types.ModuleType" + constructor. Previously the attribute was optional. + + Changed in version 3.6: The value of "__package__" is expected to + be the same as "__spec__.parent". "__package__" is now only used as + a fallback during import resolution if "__spec__.parent" is not + defined. + + Changed in version 3.10: "ImportWarning" is raised if an import + resolution falls back to "__package__" instead of + "__spec__.parent". + + Changed in version 3.12: Raise "DeprecationWarning" instead of + "ImportWarning" when falling back to "__package__" during import + resolution. + + Deprecated since version 3.13, will be removed in version 3.15: + "__package__" will cease to be set or taken into consideration by + the import system or standard library. + +module.__loader__ + + The *loader* object that the import machinery used to load the + module. + + This attribute is mostly useful for introspection, but can be used + for additional loader-specific functionality, for example getting + data associated with a loader. + + "__loader__" defaults to "None" for modules created dynamically + using the "types.ModuleType" constructor; use + "importlib.util.module_from_spec()" instead to ensure the attribute + is set to a *loader* object. + + It is **strongly** recommended that you use + "module.__spec__.loader" instead of "module.__loader__". + + Changed in version 3.4: This attribute now defaults to "None" for + modules created dynamically using the "types.ModuleType" + constructor. Previously the attribute was optional. + + Deprecated since version 3.12, will be removed in version 3.16: + Setting "__loader__" on a module while failing to set + "__spec__.loader" is deprecated. In Python 3.16, "__loader__" will + cease to be set or taken into consideration by the import system or + the standard library. + +module.__path__ + + A (possibly empty) *sequence* of strings enumerating the locations + where the package’s submodules will be found. Non-package modules + should not have a "__path__" attribute. See __path__ attributes on + modules for more details. + + It is **strongly** recommended that you use + "module.__spec__.submodule_search_locations" instead of + "module.__path__". + +module.__file__ + +module.__cached__ + + "__file__" and "__cached__" are both optional attributes that may + or may not be set. Both attributes should be a "str" when they are + available. + + "__file__" indicates the pathname of the file from which the module + was loaded (if loaded from a file), or the pathname of the shared + library file for extension modules loaded dynamically from a shared + library. It might be missing for certain types of modules, such as + C modules that are statically linked into the interpreter, and the + import system may opt to leave it unset if it has no semantic + meaning (for example, a module loaded from a database). + + If "__file__" is set then the "__cached__" attribute might also be + set, which is the path to any compiled version of the code (for + example, a byte-compiled file). The file does not need to exist to + set this attribute; the path can simply point to where the compiled + file *would* exist (see **PEP 3147**). + + Note that "__cached__" may be set even if "__file__" is not set. + However, that scenario is quite atypical. Ultimately, the *loader* + is what makes use of the module spec provided by the *finder* (from + which "__file__" and "__cached__" are derived). So if a loader can + load from a cached module but otherwise does not load from a file, + that atypical scenario may be appropriate. + + It is **strongly** recommended that you use + "module.__spec__.cached" instead of "module.__cached__". + + Deprecated since version 3.13, will be removed in version 3.15: + Setting "__cached__" on a module while failing to set + "__spec__.cached" is deprecated. In Python 3.15, "__cached__" will + cease to be set or taken into consideration by the import system or + standard library. + + +Other writable attributes on module objects +------------------------------------------- + +As well as the import-related attributes listed above, module objects +also have the following writable attributes: + +module.__doc__ + + The module’s documentation string, or "None" if unavailable. See + also: "__doc__ attributes". + +module.__annotations__ + + A dictionary containing *variable annotations* collected during + module body execution. For best practices on working with + "__annotations__", see "annotationlib". + + Changed in version 3.14: Annotations are now lazily evaluated. See + **PEP 649**. + +module.__annotate__ + + The *annotate function* for this module, or "None" if the module + has no annotations. See also: "__annotate__" attributes. + + Added in version 3.14. + + +Module dictionaries +------------------- + +Module objects also have the following special read-only attribute: + +module.__dict__ + + The module’s namespace as a dictionary object. Uniquely among the + attributes listed here, "__dict__" cannot be accessed as a global + variable from within a module; it can only be accessed as an + attribute on module objects. + + **CPython implementation detail:** Because of the way CPython + clears module dictionaries, the module dictionary will be cleared + when the module falls out of scope even if the dictionary still has + live references. To avoid this, copy the dictionary or keep the + module around while using its dictionary directly. + + +Custom classes +============== + +Custom class types are typically created by class definitions (see +section Class definitions). A class has a namespace implemented by a +dictionary object. Class attribute references are translated to +lookups in this dictionary, e.g., "C.x" is translated to +"C.__dict__["x"]" (although there are a number of hooks which allow +for other means of locating attributes). When the attribute name is +not found there, the attribute search continues in the base classes. +This search of the base classes uses the C3 method resolution order +which behaves correctly even in the presence of ‘diamond’ inheritance +structures where there are multiple inheritance paths leading back to +a common ancestor. Additional details on the C3 MRO used by Python can +be found at The Python 2.3 Method Resolution Order. + +When a class attribute reference (for class "C", say) would yield a +class method object, it is transformed into an instance method object +whose "__self__" attribute is "C". When it would yield a +"staticmethod" object, it is transformed into the object wrapped by +the static method object. See section Implementing Descriptors for +another way in which attributes retrieved from a class may differ from +those actually contained in its "__dict__". + +Class attribute assignments update the class’s dictionary, never the +dictionary of a base class. + +A class object can be called (see above) to yield a class instance +(see below). + + +Special attributes +------------------ + ++----------------------------------------------------+----------------------------------------------------+ +| Attribute | Meaning | +|====================================================|====================================================| +| type.__name__ | The class’s name. See also: "__name__ attributes". | ++----------------------------------------------------+----------------------------------------------------+ +| type.__qualname__ | The class’s *qualified name*. See also: | +| | "__qualname__ attributes". | ++----------------------------------------------------+----------------------------------------------------+ +| type.__module__ | The name of the module in which the class was | +| | defined. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__dict__ | A "mapping proxy" providing a read-only view of | +| | the class’s namespace. See also: "__dict__ | +| | attributes". | ++----------------------------------------------------+----------------------------------------------------+ +| type.__bases__ | A "tuple" containing the class’s bases. In most | +| | cases, for a class defined as "class X(A, B, C)", | +| | "X.__bases__" will be exactly equal to "(A, B, | +| | C)". | ++----------------------------------------------------+----------------------------------------------------+ +| type.__doc__ | The class’s documentation string, or "None" if | +| | undefined. Not inherited by subclasses. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__annotations__ | A dictionary containing *variable annotations* | +| | collected during class body execution. See also: | +| | "__annotations__ attributes". For best practices | +| | on working with "__annotations__", please see | +| | "annotationlib". Caution: Accessing the | +| | "__annotations__" attribute of a class object | +| | directly may yield incorrect results in the | +| | presence of metaclasses. In addition, the | +| | attribute may not exist for some classes. Use | +| | "annotationlib.get_annotations()" to retrieve | +| | class annotations safely. Changed in version | +| | 3.14: Annotations are now lazily evaluated. See | +| | **PEP 649**. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__annotate__() | The *annotate function* for this class, or "None" | +| | if the class has no annotations. See also: | +| | "__annotate__ attributes". Caution: Accessing | +| | the "__annotate__" attribute of a class object | +| | directly may yield incorrect results in the | +| | presence of metaclasses. Use | +| | "annotationlib.get_annotate_function()" to | +| | retrieve the annotate function safely. Added in | +| | version 3.14. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__type_params__ | A "tuple" containing the type parameters of a | +| | generic class. Added in version 3.12. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__static_attributes__ | A "tuple" containing names of attributes of this | +| | class which are assigned through "self.X" from any | +| | function in its body. Added in version 3.13. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__firstlineno__ | The line number of the first line of the class | +| | definition, including decorators. Setting the | +| | "__module__" attribute removes the | +| | "__firstlineno__" item from the type’s dictionary. | +| | Added in version 3.13. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__mro__ | The "tuple" of classes that are considered when | +| | looking for base classes during method resolution. | ++----------------------------------------------------+----------------------------------------------------+ + + +Special methods +--------------- + +In addition to the special attributes described above, all Python +classes also have the following two methods available: + +type.mro() + + This method can be overridden by a metaclass to customize the + method resolution order for its instances. It is called at class + instantiation, and its result is stored in "__mro__". + +type.__subclasses__() + + Each class keeps a list of weak references to its immediate + subclasses. This method returns a list of all those references + still alive. The list is in definition order. Example: + + >>> class A: pass + >>> class B(A): pass + >>> A.__subclasses__() + [] + + +Class instances +=============== + +A class instance is created by calling a class object (see above). A +class instance has a namespace implemented as a dictionary which is +the first place in which attribute references are searched. When an +attribute is not found there, and the instance’s class has an +attribute by that name, the search continues with the class +attributes. If a class attribute is found that is a user-defined +function object, it is transformed into an instance method object +whose "__self__" attribute is the instance. Static method and class +method objects are also transformed; see above under “Classes”. See +section Implementing Descriptors for another way in which attributes +of a class retrieved via its instances may differ from the objects +actually stored in the class’s "__dict__". If no class attribute is +found, and the object’s class has a "__getattr__()" method, that is +called to satisfy the lookup. + +Attribute assignments and deletions update the instance’s dictionary, +never a class’s dictionary. If the class has a "__setattr__()" or +"__delattr__()" method, this is called instead of updating the +instance dictionary directly. + +Class instances can pretend to be numbers, sequences, or mappings if +they have methods with certain special names. See section Special +method names. + + +Special attributes +------------------ + +object.__class__ + + The class to which a class instance belongs. + +object.__dict__ + + A dictionary or other mapping object used to store an object’s + (writable) attributes. Not all instances have a "__dict__" + attribute; see the section on __slots__ for more details. + + +I/O objects (also known as file objects) +======================================== + +A *file object* represents an open file. Various shortcuts are +available to create file objects: the "open()" built-in function, and +also "os.popen()", "os.fdopen()", and the "makefile()" method of +socket objects (and perhaps by other functions or methods provided by +extension modules). + +The objects "sys.stdin", "sys.stdout" and "sys.stderr" are initialized +to file objects corresponding to the interpreter’s standard input, +output and error streams; they are all open in text mode and therefore +follow the interface defined by the "io.TextIOBase" abstract class. + + +Internal types +============== + +A few types used internally by the interpreter are exposed to the +user. Their definitions may change with future versions of the +interpreter, but they are mentioned here for completeness. + + +Code objects +------------ + +Code objects represent *byte-compiled* executable Python code, or +*bytecode*. The difference between a code object and a function object +is that the function object contains an explicit reference to the +function’s globals (the module in which it was defined), while a code +object contains no context; also the default argument values are +stored in the function object, not in the code object (because they +represent values calculated at run-time). Unlike function objects, +code objects are immutable and contain no references (directly or +indirectly) to mutable objects. + + +Special read-only attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_name | The function name | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_qualname | The fully qualified function name Added in | +| | version 3.11. | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_argcount | The total number of positional *parameters* | +| | (including positional-only parameters and | +| | parameters with default values) that the function | +| | has | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_posonlyargcount | The number of positional-only *parameters* | +| | (including arguments with default values) that the | +| | function has | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_kwonlyargcount | The number of keyword-only *parameters* (including | +| | arguments with default values) that the function | +| | has | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_nlocals | The number of local variables used by the function | +| | (including parameters) | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_varnames | A "tuple" containing the names of the local | +| | variables in the function (starting with the | +| | parameter names) | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_cellvars | A "tuple" containing the names of local variables | +| | that are referenced from at least one *nested | +| | scope* inside the function | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_freevars | A "tuple" containing the names of *free (closure) | +| | variables* that a *nested scope* references in an | +| | outer scope. See also "function.__closure__". | +| | Note: references to global and builtin names are | +| | *not* included. | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_code | A string representing the sequence of *bytecode* | +| | instructions in the function | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_consts | A "tuple" containing the literals used by the | +| | *bytecode* in the function | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_names | A "tuple" containing the names used by the | +| | *bytecode* in the function | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_filename | The name of the file from which the code was | +| | compiled | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_firstlineno | The line number of the first line of the function | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_lnotab | A string encoding the mapping from *bytecode* | +| | offsets to line numbers. For details, see the | +| | source code of the interpreter. Deprecated since | +| | version 3.12: This attribute of code objects is | +| | deprecated, and may be removed in Python 3.15. | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_stacksize | The required stack size of the code object | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_flags | An "integer" encoding a number of flags for the | +| | interpreter. | ++----------------------------------------------------+----------------------------------------------------+ + +The following flag bits are defined for "co_flags": bit "0x04" is set +if the function uses the "*arguments" syntax to accept an arbitrary +number of positional arguments; bit "0x08" is set if the function uses +the "**keywords" syntax to accept arbitrary keyword arguments; bit +"0x20" is set if the function is a generator. See Code Objects Bit +Flags for details on the semantics of each flags that might be +present. + +Future feature declarations ("from __future__ import division") also +use bits in "co_flags" to indicate whether a code object was compiled +with a particular feature enabled: bit "0x2000" is set if the function +was compiled with future division enabled; bits "0x10" and "0x1000" +were used in earlier versions of Python. + +Other bits in "co_flags" are reserved for internal use. + +If a code object represents a function and has a docstring, the first +item in "co_consts" is the docstring of the function. + + +Methods on code objects +~~~~~~~~~~~~~~~~~~~~~~~ + +codeobject.co_positions() + + Returns an iterable over the source code positions of each + *bytecode* instruction in the code object. + + The iterator returns "tuple"s containing the "(start_line, + end_line, start_column, end_column)". The *i-th* tuple corresponds + to the position of the source code that compiled to the *i-th* code + unit. Column information is 0-indexed utf-8 byte offsets on the + given source line. + + This positional information can be missing. A non-exhaustive lists + of cases where this may happen: + + * Running the interpreter with "-X" "no_debug_ranges". + + * Loading a pyc file compiled while using "-X" "no_debug_ranges". + + * Position tuples corresponding to artificial instructions. + + * Line and column numbers that can’t be represented due to + implementation specific limitations. + + When this occurs, some or all of the tuple elements can be "None". + + Added in version 3.11. + + Note: + + This feature requires storing column positions in code objects + which may result in a small increase of disk usage of compiled + Python files or interpreter memory usage. To avoid storing the + extra information and/or deactivate printing the extra traceback + information, the "-X" "no_debug_ranges" command line flag or the + "PYTHONNODEBUGRANGES" environment variable can be used. + +codeobject.co_lines() + + Returns an iterator that yields information about successive ranges + of *bytecode*s. Each item yielded is a "(start, end, lineno)" + "tuple": + + * "start" (an "int") represents the offset (inclusive) of the start + of the *bytecode* range + + * "end" (an "int") represents the offset (exclusive) of the end of + the *bytecode* range + + * "lineno" is an "int" representing the line number of the + *bytecode* range, or "None" if the bytecodes in the given range + have no line number + + The items yielded will have the following properties: + + * The first range yielded will have a "start" of 0. + + * The "(start, end)" ranges will be non-decreasing and consecutive. + That is, for any pair of "tuple"s, the "start" of the second will + be equal to the "end" of the first. + + * No range will be backwards: "end >= start" for all triples. + + * The last "tuple" yielded will have "end" equal to the size of the + *bytecode*. + + Zero-width ranges, where "start == end", are allowed. Zero-width + ranges are used for lines that are present in the source code, but + have been eliminated by the *bytecode* compiler. + + Added in version 3.10. + + See also: + + **PEP 626** - Precise line numbers for debugging and other tools. + The PEP that introduced the "co_lines()" method. + +codeobject.replace(**kwargs) + + Return a copy of the code object with new values for the specified + fields. + + Code objects are also supported by the generic function + "copy.replace()". + + Added in version 3.8. + + +Frame objects +------------- + +Frame objects represent execution frames. They may occur in traceback +objects, and are also passed to registered trace functions. + + +Special read-only attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_back | Points to the previous stack frame (towards the | +| | caller), or "None" if this is the bottom stack | +| | frame | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_code | The code object being executed in this frame. | +| | Accessing this attribute raises an auditing event | +| | "object.__getattr__" with arguments "obj" and | +| | ""f_code"". | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_locals | The mapping used by the frame to look up local | +| | variables. If the frame refers to an *optimized | +| | scope*, this may return a write-through proxy | +| | object. Changed in version 3.13: Return a proxy | +| | for optimized scopes. | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_globals | The dictionary used by the frame to look up global | +| | variables | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_builtins | The dictionary used by the frame to look up built- | +| | in (intrinsic) names | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_lasti | The “precise instruction” of the frame object | +| | (this is an index into the *bytecode* string of | +| | the code object) | ++----------------------------------------------------+----------------------------------------------------+ + + +Special writable attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_trace | If not "None", this is a function called for | +| | various events during code execution (this is used | +| | by debuggers). Normally an event is triggered for | +| | each new source line (see "f_trace_lines"). | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_trace_lines | Set this attribute to "False" to disable | +| | triggering a tracing event for each source line. | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_trace_opcodes | Set this attribute to "True" to allow per-opcode | +| | events to be requested. Note that this may lead to | +| | undefined interpreter behaviour if exceptions | +| | raised by the trace function escape to the | +| | function being traced. | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_lineno | The current line number of the frame – writing to | +| | this from within a trace function jumps to the | +| | given line (only for the bottom-most frame). A | +| | debugger can implement a Jump command (aka Set | +| | Next Statement) by writing to this attribute. | ++----------------------------------------------------+----------------------------------------------------+ + + +Frame object methods +~~~~~~~~~~~~~~~~~~~~ + +Frame objects support one method: + +frame.clear() + + This method clears all references to local variables held by the + frame. Also, if the frame belonged to a *generator*, the generator + is finalized. This helps break reference cycles involving frame + objects (for example when catching an exception and storing its + traceback for later use). + + "RuntimeError" is raised if the frame is currently executing or + suspended. + + Added in version 3.4. + + Changed in version 3.13: Attempting to clear a suspended frame + raises "RuntimeError" (as has always been the case for executing + frames). + + +Traceback objects +----------------- + +Traceback objects represent the stack trace of an exception. A +traceback object is implicitly created when an exception occurs, and +may also be explicitly created by calling "types.TracebackType". + +Changed in version 3.7: Traceback objects can now be explicitly +instantiated from Python code. + +For implicitly created tracebacks, when the search for an exception +handler unwinds the execution stack, at each unwound level a traceback +object is inserted in front of the current traceback. When an +exception handler is entered, the stack trace is made available to the +program. (See section The try statement.) It is accessible as the +third item of the tuple returned by "sys.exc_info()", and as the +"__traceback__" attribute of the caught exception. + +When the program contains no suitable handler, the stack trace is +written (nicely formatted) to the standard error stream; if the +interpreter is interactive, it is also made available to the user as +"sys.last_traceback". + +For explicitly created tracebacks, it is up to the creator of the +traceback to determine how the "tb_next" attributes should be linked +to form a full stack trace. + +Special read-only attributes: + ++----------------------------------------------------+----------------------------------------------------+ +| traceback.tb_frame | Points to the execution frame of the current | +| | level. Accessing this attribute raises an | +| | auditing event "object.__getattr__" with arguments | +| | "obj" and ""tb_frame"". | ++----------------------------------------------------+----------------------------------------------------+ +| traceback.tb_lineno | Gives the line number where the exception occurred | ++----------------------------------------------------+----------------------------------------------------+ +| traceback.tb_lasti | Indicates the “precise instruction”. | ++----------------------------------------------------+----------------------------------------------------+ + +The line number and last instruction in the traceback may differ from +the line number of its frame object if the exception occurred in a +"try" statement with no matching except clause or with a "finally" +clause. + +traceback.tb_next + + The special writable attribute "tb_next" is the next level in the + stack trace (towards the frame where the exception occurred), or + "None" if there is no next level. + + Changed in version 3.7: This attribute is now writable + + +Slice objects +------------- + +Slice objects are used to represent slices for "__getitem__()" +methods. They are also created by the built-in "slice()" function. + +Special read-only attributes: "start" is the lower bound; "stop" is +the upper bound; "step" is the step value; each is "None" if omitted. +These attributes can have any type. + +Slice objects support one method: + +slice.indices(self, length) + + This method takes a single integer argument *length* and computes + information about the slice that the slice object would describe if + applied to a sequence of *length* items. It returns a tuple of + three integers; respectively these are the *start* and *stop* + indices and the *step* or stride length of the slice. Missing or + out-of-bounds indices are handled in a manner consistent with + regular slices. + + +Static method objects +--------------------- + +Static method objects provide a way of defeating the transformation of +function objects to method objects described above. A static method +object is a wrapper around any other object, usually a user-defined +method object. When a static method object is retrieved from a class +or a class instance, the object actually returned is the wrapped +object, which is not subject to any further transformation. Static +method objects are also callable. Static method objects are created by +the built-in "staticmethod()" constructor. + + +Class method objects +-------------------- + +A class method object, like a static method object, is a wrapper +around another object that alters the way in which that object is +retrieved from classes and class instances. The behaviour of class +method objects upon such retrieval is described above, under “instance +methods”. Class method objects are created by the built-in +"classmethod()" constructor. +''', + 'typesfunctions': r'''Functions +********* + +Function objects are created by function definitions. The only +operation on a function object is to call it: "func(argument-list)". + +There are really two flavors of function objects: built-in functions +and user-defined functions. Both support the same operation (to call +the function), but the implementation is different, hence the +different object types. + +See Function definitions for more information. +''', + 'typesmapping': r'''Mapping Types — "dict" +********************** + +A *mapping* object maps *hashable* values to arbitrary objects. +Mappings are mutable objects. There is currently only one standard +mapping type, the *dictionary*. (For other containers see the built- +in "list", "set", and "tuple" classes, and the "collections" module.) + +A dictionary’s keys are *almost* arbitrary values. Values that are +not *hashable*, that is, values containing lists, dictionaries or +other mutable types (that are compared by value rather than by object +identity) may not be used as keys. Values that compare equal (such as +"1", "1.0", and "True") can be used interchangeably to index the same +dictionary entry. + +class dict(**kwargs) +class dict(mapping, **kwargs) +class dict(iterable, **kwargs) + + Return a new dictionary initialized from an optional positional + argument and a possibly empty set of keyword arguments. + + Dictionaries can be created by several means: + + * Use a comma-separated list of "key: value" pairs within braces: + "{'jack': 4098, 'sjoerd': 4127}" or "{4098: 'jack', 4127: + 'sjoerd'}" + + * Use a dict comprehension: "{}", "{x: x ** 2 for x in range(10)}" + + * Use the type constructor: "dict()", "dict([('foo', 100), ('bar', + 200)])", "dict(foo=100, bar=200)" + + If no positional argument is given, an empty dictionary is created. + If a positional argument is given and it defines a "keys()" method, + a dictionary is created by calling "__getitem__()" on the argument + with each returned key from the method. Otherwise, the positional + argument must be an *iterable* object. Each item in the iterable + must itself be an iterable with exactly two elements. The first + element of each item becomes a key in the new dictionary, and the + second element the corresponding value. If a key occurs more than + once, the last value for that key becomes the corresponding value + in the new dictionary. + + If keyword arguments are given, the keyword arguments and their + values are added to the dictionary created from the positional + argument. If a key being added is already present, the value from + the keyword argument replaces the value from the positional + argument. + + To illustrate, the following examples all return a dictionary equal + to "{"one": 1, "two": 2, "three": 3}": + + >>> a = dict(one=1, two=2, three=3) + >>> b = {'one': 1, 'two': 2, 'three': 3} + >>> c = dict(zip(['one', 'two', 'three'], [1, 2, 3])) + >>> d = dict([('two', 2), ('one', 1), ('three', 3)]) + >>> e = dict({'three': 3, 'one': 1, 'two': 2}) + >>> f = dict({'one': 1, 'three': 3}, two=2) + >>> a == b == c == d == e == f + True + + Providing keyword arguments as in the first example only works for + keys that are valid Python identifiers. Otherwise, any valid keys + can be used. + + These are the operations that dictionaries support (and therefore, + custom mapping types should support too): + + list(d) + + Return a list of all the keys used in the dictionary *d*. + + len(d) + + Return the number of items in the dictionary *d*. + + d[key] + + Return the item of *d* with key *key*. Raises a "KeyError" if + *key* is not in the map. + + If a subclass of dict defines a method "__missing__()" and *key* + is not present, the "d[key]" operation calls that method with + the key *key* as argument. The "d[key]" operation then returns + or raises whatever is returned or raised by the + "__missing__(key)" call. No other operations or methods invoke + "__missing__()". If "__missing__()" is not defined, "KeyError" + is raised. "__missing__()" must be a method; it cannot be an + instance variable: + + >>> class Counter(dict): + ... def __missing__(self, key): + ... return 0 + ... + >>> c = Counter() + >>> c['red'] + 0 + >>> c['red'] += 1 + >>> c['red'] + 1 + + The example above shows part of the implementation of + "collections.Counter". A different "__missing__" method is used + by "collections.defaultdict". + + d[key] = value + + Set "d[key]" to *value*. + + del d[key] + + Remove "d[key]" from *d*. Raises a "KeyError" if *key* is not + in the map. + + key in d + + Return "True" if *d* has a key *key*, else "False". + + key not in d + + Equivalent to "not key in d". + + iter(d) + + Return an iterator over the keys of the dictionary. This is a + shortcut for "iter(d.keys())". + + clear() + + Remove all items from the dictionary. + + copy() + + Return a shallow copy of the dictionary. + + classmethod fromkeys(iterable, value=None, /) + + Create a new dictionary with keys from *iterable* and values set + to *value*. + + "fromkeys()" is a class method that returns a new dictionary. + *value* defaults to "None". All of the values refer to just a + single instance, so it generally doesn’t make sense for *value* + to be a mutable object such as an empty list. To get distinct + values, use a dict comprehension instead. + + get(key, default=None) + + Return the value for *key* if *key* is in the dictionary, else + *default*. If *default* is not given, it defaults to "None", so + that this method never raises a "KeyError". + + items() + + Return a new view of the dictionary’s items ("(key, value)" + pairs). See the documentation of view objects. + + keys() + + Return a new view of the dictionary’s keys. See the + documentation of view objects. + + pop(key[, default]) + + If *key* is in the dictionary, remove it and return its value, + else return *default*. If *default* is not given and *key* is + not in the dictionary, a "KeyError" is raised. + + popitem() + + Remove and return a "(key, value)" pair from the dictionary. + Pairs are returned in LIFO (last-in, first-out) order. + + "popitem()" is useful to destructively iterate over a + dictionary, as often used in set algorithms. If the dictionary + is empty, calling "popitem()" raises a "KeyError". + + Changed in version 3.7: LIFO order is now guaranteed. In prior + versions, "popitem()" would return an arbitrary key/value pair. + + reversed(d) + + Return a reverse iterator over the keys of the dictionary. This + is a shortcut for "reversed(d.keys())". + + Added in version 3.8. + + setdefault(key, default=None) + + If *key* is in the dictionary, return its value. If not, insert + *key* with a value of *default* and return *default*. *default* + defaults to "None". + + update([other]) + + Update the dictionary with the key/value pairs from *other*, + overwriting existing keys. Return "None". + + "update()" accepts either another object with a "keys()" method + (in which case "__getitem__()" is called with every key returned + from the method) or an iterable of key/value pairs (as tuples or + other iterables of length two). If keyword arguments are + specified, the dictionary is then updated with those key/value + pairs: "d.update(red=1, blue=2)". + + values() + + Return a new view of the dictionary’s values. See the + documentation of view objects. + + An equality comparison between one "dict.values()" view and + another will always return "False". This also applies when + comparing "dict.values()" to itself: + + >>> d = {'a': 1} + >>> d.values() == d.values() + False + + d | other + + Create a new dictionary with the merged keys and values of *d* + and *other*, which must both be dictionaries. The values of + *other* take priority when *d* and *other* share keys. + + Added in version 3.9. + + d |= other + + Update the dictionary *d* with keys and values from *other*, + which may be either a *mapping* or an *iterable* of key/value + pairs. The values of *other* take priority when *d* and *other* + share keys. + + Added in version 3.9. + + Dictionaries compare equal if and only if they have the same "(key, + value)" pairs (regardless of ordering). Order comparisons (‘<’, + ‘<=’, ‘>=’, ‘>’) raise "TypeError". + + Dictionaries preserve insertion order. Note that updating a key + does not affect the order. Keys added after deletion are inserted + at the end. + + >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} + >>> d + {'one': 1, 'two': 2, 'three': 3, 'four': 4} + >>> list(d) + ['one', 'two', 'three', 'four'] + >>> list(d.values()) + [1, 2, 3, 4] + >>> d["one"] = 42 + >>> d + {'one': 42, 'two': 2, 'three': 3, 'four': 4} + >>> del d["two"] + >>> d["two"] = None + >>> d + {'one': 42, 'three': 3, 'four': 4, 'two': None} + + Changed in version 3.7: Dictionary order is guaranteed to be + insertion order. This behavior was an implementation detail of + CPython from 3.6. + + Dictionaries and dictionary views are reversible. + + >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} + >>> d + {'one': 1, 'two': 2, 'three': 3, 'four': 4} + >>> list(reversed(d)) + ['four', 'three', 'two', 'one'] + >>> list(reversed(d.values())) + [4, 3, 2, 1] + >>> list(reversed(d.items())) + [('four', 4), ('three', 3), ('two', 2), ('one', 1)] + + Changed in version 3.8: Dictionaries are now reversible. + +See also: + + "types.MappingProxyType" can be used to create a read-only view of a + "dict". + + +Dictionary view objects +======================= + +The objects returned by "dict.keys()", "dict.values()" and +"dict.items()" are *view objects*. They provide a dynamic view on the +dictionary’s entries, which means that when the dictionary changes, +the view reflects these changes. + +Dictionary views can be iterated over to yield their respective data, +and support membership tests: + +len(dictview) + + Return the number of entries in the dictionary. + +iter(dictview) + + Return an iterator over the keys, values or items (represented as + tuples of "(key, value)") in the dictionary. + + Keys and values are iterated over in insertion order. This allows + the creation of "(value, key)" pairs using "zip()": "pairs = + zip(d.values(), d.keys())". Another way to create the same list is + "pairs = [(v, k) for (k, v) in d.items()]". + + Iterating views while adding or deleting entries in the dictionary + may raise a "RuntimeError" or fail to iterate over all entries. + + Changed in version 3.7: Dictionary order is guaranteed to be + insertion order. + +x in dictview + + Return "True" if *x* is in the underlying dictionary’s keys, values + or items (in the latter case, *x* should be a "(key, value)" + tuple). + +reversed(dictview) + + Return a reverse iterator over the keys, values or items of the + dictionary. The view will be iterated in reverse order of the + insertion. + + Changed in version 3.8: Dictionary views are now reversible. + +dictview.mapping + + Return a "types.MappingProxyType" that wraps the original + dictionary to which the view refers. + + Added in version 3.10. + +Keys views are set-like since their entries are unique and *hashable*. +Items views also have set-like operations since the (key, value) pairs +are unique and the keys are hashable. If all values in an items view +are hashable as well, then the items view can interoperate with other +sets. (Values views are not treated as set-like since the entries are +generally not unique.) For set-like views, all of the operations +defined for the abstract base class "collections.abc.Set" are +available (for example, "==", "<", or "^"). While using set +operators, set-like views accept any iterable as the other operand, +unlike sets which only accept sets as the input. + +An example of dictionary view usage: + + >>> dishes = {'eggs': 2, 'sausage': 1, 'bacon': 1, 'spam': 500} + >>> keys = dishes.keys() + >>> values = dishes.values() + + >>> # iteration + >>> n = 0 + >>> for val in values: + ... n += val + ... + >>> print(n) + 504 + + >>> # keys and values are iterated over in the same order (insertion order) + >>> list(keys) + ['eggs', 'sausage', 'bacon', 'spam'] + >>> list(values) + [2, 1, 1, 500] + + >>> # view objects are dynamic and reflect dict changes + >>> del dishes['eggs'] + >>> del dishes['sausage'] + >>> list(keys) + ['bacon', 'spam'] + + >>> # set operations + >>> keys & {'eggs', 'bacon', 'salad'} + {'bacon'} + >>> keys ^ {'sausage', 'juice'} == {'juice', 'sausage', 'bacon', 'spam'} + True + >>> keys | ['juice', 'juice', 'juice'] == {'bacon', 'spam', 'juice'} + True + + >>> # get back a read-only proxy for the original dictionary + >>> values.mapping + mappingproxy({'bacon': 1, 'spam': 500}) + >>> values.mapping['spam'] + 500 +''', + 'typesmethods': r'''Methods +******* + +Methods are functions that are called using the attribute notation. +There are two flavors: built-in methods (such as "append()" on lists) +and class instance method. Built-in methods are described with the +types that support them. + +If you access a method (a function defined in a class namespace) +through an instance, you get a special object: a *bound method* (also +called instance method) object. When called, it will add the "self" +argument to the argument list. Bound methods have two special read- +only attributes: "m.__self__" is the object on which the method +operates, and "m.__func__" is the function implementing the method. +Calling "m(arg-1, arg-2, ..., arg-n)" is completely equivalent to +calling "m.__func__(m.__self__, arg-1, arg-2, ..., arg-n)". + +Like function objects, bound method objects support getting arbitrary +attributes. However, since method attributes are actually stored on +the underlying function object ("method.__func__"), setting method +attributes on bound methods is disallowed. Attempting to set an +attribute on a method results in an "AttributeError" being raised. In +order to set a method attribute, you need to explicitly set it on the +underlying function object: + + >>> class C: + ... def method(self): + ... pass + ... + >>> c = C() + >>> c.method.whoami = 'my name is method' # can't set on the method + Traceback (most recent call last): + File "", line 1, in + AttributeError: 'method' object has no attribute 'whoami' + >>> c.method.__func__.whoami = 'my name is method' + >>> c.method.whoami + 'my name is method' + +See Instance methods for more information. +''', + 'typesmodules': r'''Modules +******* + +The only special operation on a module is attribute access: "m.name", +where *m* is a module and *name* accesses a name defined in *m*’s +symbol table. Module attributes can be assigned to. (Note that the +"import" statement is not, strictly speaking, an operation on a module +object; "import foo" does not require a module object named *foo* to +exist, rather it requires an (external) *definition* for a module +named *foo* somewhere.) + +A special attribute of every module is "__dict__". This is the +dictionary containing the module’s symbol table. Modifying this +dictionary will actually change the module’s symbol table, but direct +assignment to the "__dict__" attribute is not possible (you can write +"m.__dict__['a'] = 1", which defines "m.a" to be "1", but you can’t +write "m.__dict__ = {}"). Modifying "__dict__" directly is not +recommended. + +Modules built into the interpreter are written like this: "". If loaded from a file, they are written as +"". +''', + 'typesseq': r'''Sequence Types — "list", "tuple", "range" +***************************************** + +There are three basic sequence types: lists, tuples, and range +objects. Additional sequence types tailored for processing of binary +data and text strings are described in dedicated sections. + + +Common Sequence Operations +========================== + +The operations in the following table are supported by most sequence +types, both mutable and immutable. The "collections.abc.Sequence" ABC +is provided to make it easier to correctly implement these operations +on custom sequence types. + +This table lists the sequence operations sorted in ascending priority. +In the table, *s* and *t* are sequences of the same type, *n*, *i*, +*j* and *k* are integers and *x* is an arbitrary object that meets any +type and value restrictions imposed by *s*. + +The "in" and "not in" operations have the same priorities as the +comparison operations. The "+" (concatenation) and "*" (repetition) +operations have the same priority as the corresponding numeric +operations. [3] + ++----------------------------+----------------------------------+------------+ +| Operation | Result | Notes | +|============================|==================================|============| +| "x in s" | "True" if an item of *s* is | (1) | +| | equal to *x*, else "False" | | ++----------------------------+----------------------------------+------------+ +| "x not in s" | "False" if an item of *s* is | (1) | +| | equal to *x*, else "True" | | ++----------------------------+----------------------------------+------------+ +| "s + t" | the concatenation of *s* and *t* | (6)(7) | ++----------------------------+----------------------------------+------------+ +| "s * n" or "n * s" | equivalent to adding *s* to | (2)(7) | +| | itself *n* times | | ++----------------------------+----------------------------------+------------+ +| "s[i]" | *i*th item of *s*, origin 0 | (3) | ++----------------------------+----------------------------------+------------+ +| "s[i:j]" | slice of *s* from *i* to *j* | (3)(4) | ++----------------------------+----------------------------------+------------+ +| "s[i:j:k]" | slice of *s* from *i* to *j* | (3)(5) | +| | with step *k* | | ++----------------------------+----------------------------------+------------+ +| "len(s)" | length of *s* | | ++----------------------------+----------------------------------+------------+ +| "min(s)" | smallest item of *s* | | ++----------------------------+----------------------------------+------------+ +| "max(s)" | largest item of *s* | | ++----------------------------+----------------------------------+------------+ +| "s.index(x[, i[, j]])" | index of the first occurrence of | (8) | +| | *x* in *s* (at or after index | | +| | *i* and before index *j*) | | ++----------------------------+----------------------------------+------------+ +| "s.count(x)" | total number of occurrences of | | +| | *x* in *s* | | ++----------------------------+----------------------------------+------------+ + +Sequences of the same type also support comparisons. In particular, +tuples and lists are compared lexicographically by comparing +corresponding elements. This means that to compare equal, every +element must compare equal and the two sequences must be of the same +type and have the same length. (For full details see Comparisons in +the language reference.) + +Forward and reversed iterators over mutable sequences access values +using an index. That index will continue to march forward (or +backward) even if the underlying sequence is mutated. The iterator +terminates only when an "IndexError" or a "StopIteration" is +encountered (or when the index drops below zero). + +Notes: + +1. While the "in" and "not in" operations are used only for simple + containment testing in the general case, some specialised sequences + (such as "str", "bytes" and "bytearray") also use them for + subsequence testing: + + >>> "gg" in "eggs" + True + +2. Values of *n* less than "0" are treated as "0" (which yields an + empty sequence of the same type as *s*). Note that items in the + sequence *s* are not copied; they are referenced multiple times. + This often haunts new Python programmers; consider: + + >>> lists = [[]] * 3 + >>> lists + [[], [], []] + >>> lists[0].append(3) + >>> lists + [[3], [3], [3]] + + What has happened is that "[[]]" is a one-element list containing + an empty list, so all three elements of "[[]] * 3" are references + to this single empty list. Modifying any of the elements of + "lists" modifies this single list. You can create a list of + different lists this way: + + >>> lists = [[] for i in range(3)] + >>> lists[0].append(3) + >>> lists[1].append(5) + >>> lists[2].append(7) + >>> lists + [[3], [5], [7]] + + Further explanation is available in the FAQ entry How do I create a + multidimensional list?. + +3. If *i* or *j* is negative, the index is relative to the end of + sequence *s*: "len(s) + i" or "len(s) + j" is substituted. But + note that "-0" is still "0". + +4. The slice of *s* from *i* to *j* is defined as the sequence of + items with index *k* such that "i <= k < j". If *i* or *j* is + greater than "len(s)", use "len(s)". If *i* is omitted or "None", + use "0". If *j* is omitted or "None", use "len(s)". If *i* is + greater than or equal to *j*, the slice is empty. + +5. The slice of *s* from *i* to *j* with step *k* is defined as the + sequence of items with index "x = i + n*k" such that "0 <= n < + (j-i)/k". In other words, the indices are "i", "i+k", "i+2*k", + "i+3*k" and so on, stopping when *j* is reached (but never + including *j*). When *k* is positive, *i* and *j* are reduced to + "len(s)" if they are greater. When *k* is negative, *i* and *j* are + reduced to "len(s) - 1" if they are greater. If *i* or *j* are + omitted or "None", they become “end” values (which end depends on + the sign of *k*). Note, *k* cannot be zero. If *k* is "None", it + is treated like "1". + +6. Concatenating immutable sequences always results in a new object. + This means that building up a sequence by repeated concatenation + will have a quadratic runtime cost in the total sequence length. + To get a linear runtime cost, you must switch to one of the + alternatives below: + + * if concatenating "str" objects, you can build a list and use + "str.join()" at the end or else write to an "io.StringIO" + instance and retrieve its value when complete + + * if concatenating "bytes" objects, you can similarly use + "bytes.join()" or "io.BytesIO", or you can do in-place + concatenation with a "bytearray" object. "bytearray" objects are + mutable and have an efficient overallocation mechanism + + * if concatenating "tuple" objects, extend a "list" instead + + * for other types, investigate the relevant class documentation + +7. Some sequence types (such as "range") only support item sequences + that follow specific patterns, and hence don’t support sequence + concatenation or repetition. + +8. "index" raises "ValueError" when *x* is not found in *s*. Not all + implementations support passing the additional arguments *i* and + *j*. These arguments allow efficient searching of subsections of + the sequence. Passing the extra arguments is roughly equivalent to + using "s[i:j].index(x)", only without copying any data and with the + returned index being relative to the start of the sequence rather + than the start of the slice. + + +Immutable Sequence Types +======================== + +The only operation that immutable sequence types generally implement +that is not also implemented by mutable sequence types is support for +the "hash()" built-in. + +This support allows immutable sequences, such as "tuple" instances, to +be used as "dict" keys and stored in "set" and "frozenset" instances. + +Attempting to hash an immutable sequence that contains unhashable +values will result in "TypeError". + + +Mutable Sequence Types +====================== + +The operations in the following table are defined on mutable sequence +types. The "collections.abc.MutableSequence" ABC is provided to make +it easier to correctly implement these operations on custom sequence +types. + +In the table *s* is an instance of a mutable sequence type, *t* is any +iterable object and *x* is an arbitrary object that meets any type and +value restrictions imposed by *s* (for example, "bytearray" only +accepts integers that meet the value restriction "0 <= x <= 255"). + ++--------------------------------+----------------------------------+-----------------------+ +| Operation | Result | Notes | +|================================|==================================|=======================| +| "s[i] = x" | item *i* of *s* is replaced by | | +| | *x* | | ++--------------------------------+----------------------------------+-----------------------+ +| "s[i:j] = t" | slice of *s* from *i* to *j* is | | +| | replaced by the contents of the | | +| | iterable *t* | | ++--------------------------------+----------------------------------+-----------------------+ +| "del s[i:j]" | same as "s[i:j] = []" | | ++--------------------------------+----------------------------------+-----------------------+ +| "s[i:j:k] = t" | the elements of "s[i:j:k]" are | (1) | +| | replaced by those of *t* | | ++--------------------------------+----------------------------------+-----------------------+ +| "del s[i:j:k]" | removes the elements of | | +| | "s[i:j:k]" from the list | | ++--------------------------------+----------------------------------+-----------------------+ +| "s.append(x)" | appends *x* to the end of the | | +| | sequence (same as | | +| | "s[len(s):len(s)] = [x]") | | ++--------------------------------+----------------------------------+-----------------------+ +| "s.clear()" | removes all items from *s* (same | (5) | +| | as "del s[:]") | | ++--------------------------------+----------------------------------+-----------------------+ +| "s.copy()" | creates a shallow copy of *s* | (5) | +| | (same as "s[:]") | | ++--------------------------------+----------------------------------+-----------------------+ +| "s.extend(t)" or "s += t" | extends *s* with the contents of | | +| | *t* (for the most part the same | | +| | as "s[len(s):len(s)] = t") | | ++--------------------------------+----------------------------------+-----------------------+ +| "s *= n" | updates *s* with its contents | (6) | +| | repeated *n* times | | ++--------------------------------+----------------------------------+-----------------------+ +| "s.insert(i, x)" | inserts *x* into *s* at the | | +| | index given by *i* (same as | | +| | "s[i:i] = [x]") | | ++--------------------------------+----------------------------------+-----------------------+ +| "s.pop()" or "s.pop(i)" | retrieves the item at *i* and | (2) | +| | also removes it from *s* | | ++--------------------------------+----------------------------------+-----------------------+ +| "s.remove(x)" | removes the first item from *s* | (3) | +| | where "s[i]" is equal to *x* | | ++--------------------------------+----------------------------------+-----------------------+ +| "s.reverse()" | reverses the items of *s* in | (4) | +| | place | | ++--------------------------------+----------------------------------+-----------------------+ + +Notes: + +1. If *k* is not equal to "1", *t* must have the same length as the + slice it is replacing. + +2. The optional argument *i* defaults to "-1", so that by default the + last item is removed and returned. + +3. "remove()" raises "ValueError" when *x* is not found in *s*. + +4. The "reverse()" method modifies the sequence in place for economy + of space when reversing a large sequence. To remind users that it + operates by side effect, it does not return the reversed sequence. + +5. "clear()" and "copy()" are included for consistency with the + interfaces of mutable containers that don’t support slicing + operations (such as "dict" and "set"). "copy()" is not part of the + "collections.abc.MutableSequence" ABC, but most concrete mutable + sequence classes provide it. + + Added in version 3.3: "clear()" and "copy()" methods. + +6. The value *n* is an integer, or an object implementing + "__index__()". Zero and negative values of *n* clear the sequence. + Items in the sequence are not copied; they are referenced multiple + times, as explained for "s * n" under Common Sequence Operations. + + +Lists +===== + +Lists are mutable sequences, typically used to store collections of +homogeneous items (where the precise degree of similarity will vary by +application). + +class list([iterable]) + + Lists may be constructed in several ways: + + * Using a pair of square brackets to denote the empty list: "[]" + + * Using square brackets, separating items with commas: "[a]", "[a, + b, c]" + + * Using a list comprehension: "[x for x in iterable]" + + * Using the type constructor: "list()" or "list(iterable)" + + The constructor builds a list whose items are the same and in the + same order as *iterable*’s items. *iterable* may be either a + sequence, a container that supports iteration, or an iterator + object. If *iterable* is already a list, a copy is made and + returned, similar to "iterable[:]". For example, "list('abc')" + returns "['a', 'b', 'c']" and "list( (1, 2, 3) )" returns "[1, 2, + 3]". If no argument is given, the constructor creates a new empty + list, "[]". + + Many other operations also produce lists, including the "sorted()" + built-in. + + Lists implement all of the common and mutable sequence operations. + Lists also provide the following additional method: + + sort(*, key=None, reverse=False) + + This method sorts the list in place, using only "<" comparisons + between items. Exceptions are not suppressed - if any comparison + operations fail, the entire sort operation will fail (and the + list will likely be left in a partially modified state). + + "sort()" accepts two arguments that can only be passed by + keyword (keyword-only arguments): + + *key* specifies a function of one argument that is used to + extract a comparison key from each list element (for example, + "key=str.lower"). The key corresponding to each item in the list + is calculated once and then used for the entire sorting process. + The default value of "None" means that list items are sorted + directly without calculating a separate key value. + + The "functools.cmp_to_key()" utility is available to convert a + 2.x style *cmp* function to a *key* function. + + *reverse* is a boolean value. If set to "True", then the list + elements are sorted as if each comparison were reversed. + + This method modifies the sequence in place for economy of space + when sorting a large sequence. To remind users that it operates + by side effect, it does not return the sorted sequence (use + "sorted()" to explicitly request a new sorted list instance). + + The "sort()" method is guaranteed to be stable. A sort is + stable if it guarantees not to change the relative order of + elements that compare equal — this is helpful for sorting in + multiple passes (for example, sort by department, then by salary + grade). + + For sorting examples and a brief sorting tutorial, see Sorting + Techniques. + + **CPython implementation detail:** While a list is being sorted, + the effect of attempting to mutate, or even inspect, the list is + undefined. The C implementation of Python makes the list appear + empty for the duration, and raises "ValueError" if it can detect + that the list has been mutated during a sort. + + +Tuples +====== + +Tuples are immutable sequences, typically used to store collections of +heterogeneous data (such as the 2-tuples produced by the "enumerate()" +built-in). Tuples are also used for cases where an immutable sequence +of homogeneous data is needed (such as allowing storage in a "set" or +"dict" instance). + +class tuple([iterable]) + + Tuples may be constructed in a number of ways: + + * Using a pair of parentheses to denote the empty tuple: "()" + + * Using a trailing comma for a singleton tuple: "a," or "(a,)" + + * Separating items with commas: "a, b, c" or "(a, b, c)" + + * Using the "tuple()" built-in: "tuple()" or "tuple(iterable)" + + The constructor builds a tuple whose items are the same and in the + same order as *iterable*’s items. *iterable* may be either a + sequence, a container that supports iteration, or an iterator + object. If *iterable* is already a tuple, it is returned + unchanged. For example, "tuple('abc')" returns "('a', 'b', 'c')" + and "tuple( [1, 2, 3] )" returns "(1, 2, 3)". If no argument is + given, the constructor creates a new empty tuple, "()". + + Note that it is actually the comma which makes a tuple, not the + parentheses. The parentheses are optional, except in the empty + tuple case, or when they are needed to avoid syntactic ambiguity. + For example, "f(a, b, c)" is a function call with three arguments, + while "f((a, b, c))" is a function call with a 3-tuple as the sole + argument. + + Tuples implement all of the common sequence operations. + +For heterogeneous collections of data where access by name is clearer +than access by index, "collections.namedtuple()" may be a more +appropriate choice than a simple tuple object. + + +Ranges +====== + +The "range" type represents an immutable sequence of numbers and is +commonly used for looping a specific number of times in "for" loops. + +class range(stop) +class range(start, stop[, step]) + + The arguments to the range constructor must be integers (either + built-in "int" or any object that implements the "__index__()" + special method). If the *step* argument is omitted, it defaults to + "1". If the *start* argument is omitted, it defaults to "0". If + *step* is zero, "ValueError" is raised. + + For a positive *step*, the contents of a range "r" are determined + by the formula "r[i] = start + step*i" where "i >= 0" and "r[i] < + stop". + + For a negative *step*, the contents of the range are still + determined by the formula "r[i] = start + step*i", but the + constraints are "i >= 0" and "r[i] > stop". + + A range object will be empty if "r[0]" does not meet the value + constraint. Ranges do support negative indices, but these are + interpreted as indexing from the end of the sequence determined by + the positive indices. + + Ranges containing absolute values larger than "sys.maxsize" are + permitted but some features (such as "len()") may raise + "OverflowError". + + Range examples: + + >>> list(range(10)) + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + >>> list(range(1, 11)) + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + >>> list(range(0, 30, 5)) + [0, 5, 10, 15, 20, 25] + >>> list(range(0, 10, 3)) + [0, 3, 6, 9] + >>> list(range(0, -10, -1)) + [0, -1, -2, -3, -4, -5, -6, -7, -8, -9] + >>> list(range(0)) + [] + >>> list(range(1, 0)) + [] + + Ranges implement all of the common sequence operations except + concatenation and repetition (due to the fact that range objects + can only represent sequences that follow a strict pattern and + repetition and concatenation will usually violate that pattern). + + start + + The value of the *start* parameter (or "0" if the parameter was + not supplied) + + stop + + The value of the *stop* parameter + + step + + The value of the *step* parameter (or "1" if the parameter was + not supplied) + +The advantage of the "range" type over a regular "list" or "tuple" is +that a "range" object will always take the same (small) amount of +memory, no matter the size of the range it represents (as it only +stores the "start", "stop" and "step" values, calculating individual +items and subranges as needed). + +Range objects implement the "collections.abc.Sequence" ABC, and +provide features such as containment tests, element index lookup, +slicing and support for negative indices (see Sequence Types — list, +tuple, range): + +>>> r = range(0, 20, 2) +>>> r +range(0, 20, 2) +>>> 11 in r +False +>>> 10 in r +True +>>> r.index(10) +5 +>>> r[5] +10 +>>> r[:5] +range(0, 10, 2) +>>> r[-1] +18 + +Testing range objects for equality with "==" and "!=" compares them as +sequences. That is, two range objects are considered equal if they +represent the same sequence of values. (Note that two range objects +that compare equal might have different "start", "stop" and "step" +attributes, for example "range(0) == range(2, 1, 3)" or "range(0, 3, +2) == range(0, 4, 2)".) + +Changed in version 3.2: Implement the Sequence ABC. Support slicing +and negative indices. Test "int" objects for membership in constant +time instead of iterating through all items. + +Changed in version 3.3: Define ‘==’ and ‘!=’ to compare range objects +based on the sequence of values they define (instead of comparing +based on object identity).Added the "start", "stop" and "step" +attributes. + +See also: + + * The linspace recipe shows how to implement a lazy version of range + suitable for floating-point applications. +''', + 'typesseq-mutable': r'''Mutable Sequence Types +********************** + +The operations in the following table are defined on mutable sequence +types. The "collections.abc.MutableSequence" ABC is provided to make +it easier to correctly implement these operations on custom sequence +types. + +In the table *s* is an instance of a mutable sequence type, *t* is any +iterable object and *x* is an arbitrary object that meets any type and +value restrictions imposed by *s* (for example, "bytearray" only +accepts integers that meet the value restriction "0 <= x <= 255"). + ++--------------------------------+----------------------------------+-----------------------+ +| Operation | Result | Notes | +|================================|==================================|=======================| +| "s[i] = x" | item *i* of *s* is replaced by | | +| | *x* | | ++--------------------------------+----------------------------------+-----------------------+ +| "s[i:j] = t" | slice of *s* from *i* to *j* is | | +| | replaced by the contents of the | | +| | iterable *t* | | ++--------------------------------+----------------------------------+-----------------------+ +| "del s[i:j]" | same as "s[i:j] = []" | | ++--------------------------------+----------------------------------+-----------------------+ +| "s[i:j:k] = t" | the elements of "s[i:j:k]" are | (1) | +| | replaced by those of *t* | | ++--------------------------------+----------------------------------+-----------------------+ +| "del s[i:j:k]" | removes the elements of | | +| | "s[i:j:k]" from the list | | ++--------------------------------+----------------------------------+-----------------------+ +| "s.append(x)" | appends *x* to the end of the | | +| | sequence (same as | | +| | "s[len(s):len(s)] = [x]") | | ++--------------------------------+----------------------------------+-----------------------+ +| "s.clear()" | removes all items from *s* (same | (5) | +| | as "del s[:]") | | ++--------------------------------+----------------------------------+-----------------------+ +| "s.copy()" | creates a shallow copy of *s* | (5) | +| | (same as "s[:]") | | ++--------------------------------+----------------------------------+-----------------------+ +| "s.extend(t)" or "s += t" | extends *s* with the contents of | | +| | *t* (for the most part the same | | +| | as "s[len(s):len(s)] = t") | | ++--------------------------------+----------------------------------+-----------------------+ +| "s *= n" | updates *s* with its contents | (6) | +| | repeated *n* times | | ++--------------------------------+----------------------------------+-----------------------+ +| "s.insert(i, x)" | inserts *x* into *s* at the | | +| | index given by *i* (same as | | +| | "s[i:i] = [x]") | | ++--------------------------------+----------------------------------+-----------------------+ +| "s.pop()" or "s.pop(i)" | retrieves the item at *i* and | (2) | +| | also removes it from *s* | | ++--------------------------------+----------------------------------+-----------------------+ +| "s.remove(x)" | removes the first item from *s* | (3) | +| | where "s[i]" is equal to *x* | | ++--------------------------------+----------------------------------+-----------------------+ +| "s.reverse()" | reverses the items of *s* in | (4) | +| | place | | ++--------------------------------+----------------------------------+-----------------------+ + +Notes: + +1. If *k* is not equal to "1", *t* must have the same length as the + slice it is replacing. + +2. The optional argument *i* defaults to "-1", so that by default the + last item is removed and returned. + +3. "remove()" raises "ValueError" when *x* is not found in *s*. + +4. The "reverse()" method modifies the sequence in place for economy + of space when reversing a large sequence. To remind users that it + operates by side effect, it does not return the reversed sequence. + +5. "clear()" and "copy()" are included for consistency with the + interfaces of mutable containers that don’t support slicing + operations (such as "dict" and "set"). "copy()" is not part of the + "collections.abc.MutableSequence" ABC, but most concrete mutable + sequence classes provide it. + + Added in version 3.3: "clear()" and "copy()" methods. + +6. The value *n* is an integer, or an object implementing + "__index__()". Zero and negative values of *n* clear the sequence. + Items in the sequence are not copied; they are referenced multiple + times, as explained for "s * n" under Common Sequence Operations. +''', + 'unary': r'''Unary arithmetic and bitwise operations +*************************************** + +All unary arithmetic and bitwise operations have the same priority: + + u_expr ::= power | "-" u_expr | "+" u_expr | "~" u_expr + +The unary "-" (minus) operator yields the negation of its numeric +argument; the operation can be overridden with the "__neg__()" special +method. + +The unary "+" (plus) operator yields its numeric argument unchanged; +the operation can be overridden with the "__pos__()" special method. + +The unary "~" (invert) operator yields the bitwise inversion of its +integer argument. The bitwise inversion of "x" is defined as +"-(x+1)". It only applies to integral numbers or to custom objects +that override the "__invert__()" special method. + +In all three cases, if the argument does not have the proper type, a +"TypeError" exception is raised. +''', + 'while': r'''The "while" statement +********************* + +The "while" statement is used for repeated execution as long as an +expression is true: + + while_stmt ::= "while" assignment_expression ":" suite + ["else" ":" suite] + +This repeatedly tests the expression and, if it is true, executes the +first suite; if the expression is false (which may be the first time +it is tested) the suite of the "else" clause, if present, is executed +and the loop terminates. + +A "break" statement executed in the first suite terminates the loop +without executing the "else" clause’s suite. A "continue" statement +executed in the first suite skips the rest of the suite and goes back +to testing the expression. +''', + 'with': r'''The "with" statement +******************** + +The "with" statement is used to wrap the execution of a block with +methods defined by a context manager (see section With Statement +Context Managers). This allows common "try"…"except"…"finally" usage +patterns to be encapsulated for convenient reuse. + + with_stmt ::= "with" ( "(" with_stmt_contents ","? ")" | with_stmt_contents ) ":" suite + with_stmt_contents ::= with_item ("," with_item)* + with_item ::= expression ["as" target] + +The execution of the "with" statement with one “item” proceeds as +follows: + +1. The context expression (the expression given in the "with_item") is + evaluated to obtain a context manager. + +2. The context manager’s "__enter__()" is loaded for later use. + +3. The context manager’s "__exit__()" is loaded for later use. + +4. The context manager’s "__enter__()" method is invoked. + +5. If a target was included in the "with" statement, the return value + from "__enter__()" is assigned to it. + + Note: + + The "with" statement guarantees that if the "__enter__()" method + returns without an error, then "__exit__()" will always be + called. Thus, if an error occurs during the assignment to the + target list, it will be treated the same as an error occurring + within the suite would be. See step 7 below. + +6. The suite is executed. + +7. The context manager’s "__exit__()" method is invoked. If an + exception caused the suite to be exited, its type, value, and + traceback are passed as arguments to "__exit__()". Otherwise, three + "None" arguments are supplied. + + If the suite was exited due to an exception, and the return value + from the "__exit__()" method was false, the exception is reraised. + If the return value was true, the exception is suppressed, and + execution continues with the statement following the "with" + statement. + + If the suite was exited for any reason other than an exception, the + return value from "__exit__()" is ignored, and execution proceeds + at the normal location for the kind of exit that was taken. + +The following code: + + with EXPRESSION as TARGET: + SUITE + +is semantically equivalent to: + + manager = (EXPRESSION) + enter = type(manager).__enter__ + exit = type(manager).__exit__ + value = enter(manager) + hit_except = False + + try: + TARGET = value + SUITE + except: + hit_except = True + if not exit(manager, *sys.exc_info()): + raise + finally: + if not hit_except: + exit(manager, None, None, None) + +With more than one item, the context managers are processed as if +multiple "with" statements were nested: + + with A() as a, B() as b: + SUITE + +is semantically equivalent to: + + with A() as a: + with B() as b: + SUITE + +You can also write multi-item context managers in multiple lines if +the items are surrounded by parentheses. For example: + + with ( + A() as a, + B() as b, + ): + SUITE + +Changed in version 3.1: Support for multiple context expressions. + +Changed in version 3.10: Support for using grouping parentheses to +break the statement in multiple lines. + +See also: + + **PEP 343** - The “with” statement + The specification, background, and examples for the Python "with" + statement. +''', + 'yield': r'''The "yield" statement +********************* + + yield_stmt ::= yield_expression + +A "yield" statement is semantically equivalent to a yield expression. +The "yield" statement can be used to omit the parentheses that would +otherwise be required in the equivalent yield expression statement. +For example, the yield statements + + yield + yield from + +are equivalent to the yield expression statements + + (yield ) + (yield from ) + +Yield expressions and statements are only used when defining a +*generator* function, and are only used in the body of the generator +function. Using "yield" in a function definition is sufficient to +cause that definition to create a generator function instead of a +normal function. + +For full details of "yield" semantics, refer to the Yield expressions +section. +''', +} diff --git a/Lib/queue.py b/Lib/queue.py index 18fad8df08e469..25beb46e30d6bd 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -10,7 +10,15 @@ except ImportError: SimpleQueue = None -__all__ = ['Empty', 'Full', 'Queue', 'PriorityQueue', 'LifoQueue', 'SimpleQueue'] +__all__ = [ + 'Empty', + 'Full', + 'ShutDown', + 'Queue', + 'PriorityQueue', + 'LifoQueue', + 'SimpleQueue', +] try: @@ -231,8 +239,9 @@ def shutdown(self, immediate=False): By default, gets will only raise once the queue is empty. Set 'immediate' to True to make gets raise immediately instead. - All blocked callers of put() will be unblocked, and also get() - and join() if 'immediate'. + All blocked callers of put() and get() will be unblocked. If + 'immediate', a task is marked as done for each item remaining in + the queue, which may unblock callers of join(). ''' with self.mutex: self.is_shutdown = True @@ -241,9 +250,10 @@ def shutdown(self, immediate=False): self._get() if self.unfinished_tasks > 0: self.unfinished_tasks -= 1 - self.not_empty.notify_all() # release all blocked threads in `join()` self.all_tasks_done.notify_all() + # All getters need to re-check queue-empty to raise ShutDown + self.not_empty.notify_all() self.not_full.notify_all() # Override these methods to implement other queue organizations diff --git a/Lib/quopri.py b/Lib/quopri.py old mode 100755 new mode 100644 index f36cf7b3951cda..129fd2f5c7c28a --- a/Lib/quopri.py +++ b/Lib/quopri.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """Conversions to/from quoted-printable transport encoding as per RFC 1521.""" # (Dec 1991 version). diff --git a/Lib/random.py b/Lib/random.py index 875beb2f8cf41c..8b9a270c429e4a 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -996,5 +996,75 @@ def _test(N=10_000): _os.register_at_fork(after_in_child=_inst.seed) +# ------------------------------------------------------ +# -------------- command-line interface ---------------- + + +def _parse_args(arg_list: list[str] | None): + import argparse + parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter) + group = parser.add_mutually_exclusive_group() + group.add_argument( + "-c", "--choice", nargs="+", + help="print a random choice") + group.add_argument( + "-i", "--integer", type=int, metavar="N", + help="print a random integer between 1 and N inclusive") + group.add_argument( + "-f", "--float", type=float, metavar="N", + help="print a random floating-point number between 0 and N inclusive") + group.add_argument( + "--test", type=int, const=10_000, nargs="?", + help=argparse.SUPPRESS) + parser.add_argument("input", nargs="*", + help="""\ +if no options given, output depends on the input + string or multiple: same as --choice + integer: same as --integer + float: same as --float""") + args = parser.parse_args(arg_list) + return args, parser.format_help() + + +def main(arg_list: list[str] | None = None) -> int | str: + args, help_text = _parse_args(arg_list) + + # Explicit arguments + if args.choice: + return choice(args.choice) + + if args.integer is not None: + return randint(1, args.integer) + + if args.float is not None: + return uniform(0, args.float) + + if args.test: + _test(args.test) + return "" + + # No explicit argument, select based on input + if len(args.input) == 1: + val = args.input[0] + try: + # Is it an integer? + val = int(val) + return randint(1, val) + except ValueError: + try: + # Is it a float? + val = float(val) + return uniform(0, val) + except ValueError: + # Split in case of space-separated string: "a b c" + return choice(val.split()) + + if len(args.input) >= 2: + return choice(args.input) + + return help_text + + if __name__ == '__main__': - _test() + print(main()) diff --git a/Lib/re/_casefix.py b/Lib/re/_casefix.py index 06507d08bee02b..fed2d84fc01473 100644 --- a/Lib/re/_casefix.py +++ b/Lib/re/_casefix.py @@ -1,4 +1,4 @@ -# Auto-generated by Tools/scripts/generate_re_casefix.py. +# Auto-generated by Tools/build/generate_re_casefix.py. # Maps the code of lowercased character to codes of different lowercased # characters which have the same uppercase. diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index 7b888f877eb3dc..20dd561d1c1520 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -28,6 +28,8 @@ POSSESSIVE_REPEAT: (POSSESSIVE_REPEAT, SUCCESS, POSSESSIVE_REPEAT_ONE), } +_CHARSET_ALL = [(NEGATE, None)] + def _combine_flags(flags, add_flags, del_flags, TYPE_FLAGS=_parser.TYPE_FLAGS): if add_flags & TYPE_FLAGS: @@ -84,17 +86,22 @@ def _compile(code, pattern, flags): code[skip] = _len(code) - skip elif op is IN: charset, hascased = _optimize_charset(av, iscased, tolower, fixes) - if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE: - emit(IN_LOC_IGNORE) - elif not hascased: - emit(IN) - elif not fixes: # ascii - emit(IN_IGNORE) + if not charset: + emit(FAILURE) + elif charset == _CHARSET_ALL: + emit(ANY_ALL) else: - emit(IN_UNI_IGNORE) - skip = _len(code); emit(0) - _compile_charset(charset, flags, code) - code[skip] = _len(code) - skip + if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE: + emit(IN_LOC_IGNORE) + elif not hascased: + emit(IN) + elif not fixes: # ascii + emit(IN_IGNORE) + else: + emit(IN_UNI_IGNORE) + skip = _len(code); emit(0) + _compile_charset(charset, flags, code) + code[skip] = _len(code) - skip elif op is ANY: if flags & SRE_FLAG_DOTALL: emit(ANY_ALL) @@ -248,11 +255,11 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): while True: try: if op is LITERAL: - if fixup: - lo = fixup(av) - charmap[lo] = 1 - if fixes and lo in fixes: - for k in fixes[lo]: + if fixup: # IGNORECASE and not LOCALE + av = fixup(av) + charmap[av] = 1 + if fixes and av in fixes: + for k in fixes[av]: charmap[k] = 1 if not hascased and iscased(av): hascased = True @@ -260,7 +267,7 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): charmap[av] = 1 elif op is RANGE: r = range(av[0], av[1]+1) - if fixup: + if fixup: # IGNORECASE and not LOCALE if fixes: for i in map(fixup, r): charmap[i] = 1 @@ -277,6 +284,10 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): charmap[i] = 1 elif op is NEGATE: out.append((op, av)) + elif op is CATEGORY and tail and (CATEGORY, CH_NEGATE[av]) in tail: + # Optimize [\s\S] etc. + out = [] if out else _CHARSET_ALL + return out, False else: tail.append((op, av)) except IndexError: @@ -287,8 +298,7 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): # Character set contains non-BMP character codes. # For range, all BMP characters in the range are already # proceeded. - if fixup: - hascased = True + if fixup: # IGNORECASE and not LOCALE # For now, IN_UNI_IGNORE+LITERAL and # IN_UNI_IGNORE+RANGE_UNI_IGNORE work for all non-BMP # characters, because two characters (at least one of @@ -299,7 +309,13 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): # Also, both c.lower() and c.lower().upper() are single # characters for every non-BMP character. if op is RANGE: - op = RANGE_UNI_IGNORE + if fixes: # not ASCII + op = RANGE_UNI_IGNORE + hascased = True + else: + assert op is LITERAL + if not hascased and iscased(av): + hascased = True tail.append((op, av)) break @@ -519,13 +535,18 @@ def _compile_info(code, pattern, flags): # look for a literal prefix prefix = [] prefix_skip = 0 - charset = [] # not used + charset = None # not used if not (flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE): # look for literal prefix prefix, prefix_skip, got_all = _get_literal_prefix(pattern, flags) # if no prefix, look for charset prefix if not prefix: charset = _get_charset_prefix(pattern, flags) + if charset: + charset, hascased = _optimize_charset(charset) + assert not hascased + if charset == _CHARSET_ALL: + charset = None ## if prefix: ## print("*** PREFIX", prefix, prefix_skip) ## if charset: @@ -560,8 +581,6 @@ def _compile_info(code, pattern, flags): # generate overlap table code.extend(_generate_overlap_table(prefix)) elif charset: - charset, hascased = _optimize_charset(charset) - assert not hascased _compile_charset(charset, flags, code) code[skip] = len(code) - skip diff --git a/Lib/re/_constants.py b/Lib/re/_constants.py index 9c3c294ba448b4..d6f32302d37b2d 100644 --- a/Lib/re/_constants.py +++ b/Lib/re/_constants.py @@ -15,7 +15,7 @@ MAGIC = 20230612 -from _sre import MAXREPEAT, MAXGROUPS +from _sre import MAXREPEAT, MAXGROUPS # noqa: F401 # SRE standard exception (access as sre.error) # should this really be here? @@ -206,6 +206,8 @@ def _makecodes(*names): CATEGORY_NOT_LINEBREAK: CATEGORY_UNI_NOT_LINEBREAK } +CH_NEGATE = dict(zip(CHCODES[::2] + CHCODES[1::2], CHCODES[1::2] + CHCODES[::2])) + # flags SRE_FLAG_IGNORECASE = 2 # case insensitive SRE_FLAG_LOCALE = 4 # honour system locale diff --git a/Lib/re/_parser.py b/Lib/re/_parser.py index f3c779340fe230..0990255b22c219 100644 --- a/Lib/re/_parser.py +++ b/Lib/re/_parser.py @@ -807,14 +807,6 @@ def _parse(source, state, verbose, nested, first=False): state.grouprefpos[condgroup] = ( source.tell() - len(condname) - 1 ) - if not (condname.isdecimal() and condname.isascii()): - import warnings - warnings.warn( - "bad character in group name %s at position %d" % - (repr(condname) if source.istext else ascii(condname), - source.tell() - len(condname) - 1), - DeprecationWarning, stacklevel=nested + 6 - ) state.checklookbehindgroup(condgroup, source) item_yes = _parse(source, state, verbose, nested + 1) if source.match("|"): @@ -1038,14 +1030,6 @@ def addgroup(index, pos): if index >= MAXGROUPS: raise s.error("invalid group reference %d" % index, len(name) + 1) - if not (name.isdecimal() and name.isascii()): - import warnings - warnings.warn( - "bad character in group name %s at position %d" % - (repr(name) if s.istext else ascii(name), - s.tell() - len(name) - 1), - DeprecationWarning, stacklevel=5 - ) addgroup(index, len(name) + 1) elif c == "0": if s.next in OCTDIGITS: diff --git a/Lib/reprlib.py b/Lib/reprlib.py index 05bb1a0eb01795..19dbe3a07eb618 100644 --- a/Lib/reprlib.py +++ b/Lib/reprlib.py @@ -36,6 +36,17 @@ def wrapper(self): return decorating_function class Repr: + _lookup = { + 'tuple': 'builtins', + 'list': 'builtins', + 'array': 'array', + 'set': 'builtins', + 'frozenset': 'builtins', + 'deque': 'collections', + 'dict': 'builtins', + 'str': 'builtins', + 'int': 'builtins' + } def __init__( self, *, maxlevel=6, maxtuple=6, maxlist=6, maxarray=5, maxdict=4, @@ -60,14 +71,24 @@ def repr(self, x): return self.repr1(x, self.maxlevel) def repr1(self, x, level): - typename = type(x).__name__ + cls = type(x) + typename = cls.__name__ + if ' ' in typename: parts = typename.split() typename = '_'.join(parts) - if hasattr(self, 'repr_' + typename): - return getattr(self, 'repr_' + typename)(x, level) - else: - return self.repr_instance(x, level) + + method = getattr(self, 'repr_' + typename, None) + if method: + # not defined in this class + if typename not in self._lookup: + return method(x, level) + module = getattr(cls, '__module__', None) + # defined in this class and is the module intended + if module == self._lookup[typename]: + return method(x, level) + + return self.repr_instance(x, level) def _join(self, pieces, level): if self.indent is None: diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index 206d6fb511cdf6..23eb0020f42e8a 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -35,6 +35,7 @@ import keyword import re import __main__ +import warnings __all__ = ["Completer"] @@ -88,10 +89,11 @@ def complete(self, text, state): return None if state == 0: - if "." in text: - self.matches = self.attr_matches(text) - else: - self.matches = self.global_matches(text) + with warnings.catch_warnings(action="ignore"): + if "." in text: + self.matches = self.attr_matches(text) + else: + self.matches = self.global_matches(text) try: return self.matches[state] except IndexError: diff --git a/Lib/sched.py b/Lib/sched.py index 14613cf29874da..fb20639d459967 100644 --- a/Lib/sched.py +++ b/Lib/sched.py @@ -11,7 +11,7 @@ implement simulated time by writing your own functions. This can also be used to integrate scheduling with STDWIN events; the delay function is allowed to modify the queue. Time can be expressed as -integers or floating point numbers, as long as it is consistent. +integers or floating-point numbers, as long as it is consistent. Events are specified by tuples (time, priority, action, argument, kwargs). As in UNIX, lower priority numbers mean higher priority; in this diff --git a/Lib/shutil.py b/Lib/shutil.py index c19ea0607208af..171489ca41f2a7 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -44,10 +44,11 @@ else: _winapi = None -COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024 +COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 256 * 1024 # This should never be removed, see rationale in: # https://bugs.python.org/issue43743#msg393429 -_USE_CP_SENDFILE = hasattr(os, "sendfile") and sys.platform.startswith("linux") +_USE_CP_SENDFILE = (hasattr(os, "sendfile") + and sys.platform.startswith(("linux", "android", "sunos"))) _HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile") # macOS # CMD defaults in Windows 10 @@ -55,7 +56,7 @@ __all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2", "copytree", "move", "rmtree", "Error", "SpecialFileError", - "ExecError", "make_archive", "get_archive_formats", + "make_archive", "get_archive_formats", "register_archive_format", "unregister_archive_format", "get_unpack_formats", "register_unpack_format", "unregister_unpack_format", "unpack_archive", @@ -73,8 +74,6 @@ class SpecialFileError(OSError): """Raised when trying to do a kind of operation (e.g. copying) which is not supported on a special file (e.g. a named pipe)""" -class ExecError(OSError): - """Raised when a command could not be executed""" class ReadError(OSError): """Raised when an archive cannot be read""" @@ -111,7 +110,7 @@ def _fastcopy_fcopyfile(fsrc, fdst, flags): def _fastcopy_sendfile(fsrc, fdst): """Copy data from one regular mmap-like fd to another by using high-performance sendfile(2) syscall. - This should work on Linux >= 2.6.33 only. + This should work on Linux >= 2.6.33, Android and Solaris. """ # Note: copyfileobj() is left alone in order to not introduce any # unexpected breakage. Possible risks by using zero-copy calls @@ -148,7 +147,7 @@ def _fastcopy_sendfile(fsrc, fdst): try: sent = os.sendfile(outfd, infd, offset, blocksize) except OSError as err: - # ...in oder to have a more informative exception. + # ...in order to have a more informative exception. err.filename = fsrc.name err.filename2 = fdst.name @@ -266,7 +265,7 @@ def copyfile(src, dst, *, follow_symlinks=True): return dst except _GiveupOnFastCopy: pass - # Linux + # Linux / Android / Solaris elif _USE_CP_SENDFILE: try: _fastcopy_sendfile(fsrc, fdst) @@ -555,7 +554,7 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, If the optional symlinks flag is true, symbolic links in the source tree result in symbolic links in the destination tree; if it is false, the contents of the files pointed to by symbolic - links are copied. If the file pointed by the symlink doesn't + links are copied. If the file pointed to by the symlink doesn't exist, an exception will be added in the list of errors raised in an Error exception at the end of the copy process. @@ -604,38 +603,37 @@ def _rmtree_islink(st): return stat.S_ISLNK(st.st_mode) # version vulnerable to race conditions -def _rmtree_unsafe(path, onexc): +def _rmtree_unsafe(path, dir_fd, onexc): + if dir_fd is not None: + raise NotImplementedError("dir_fd unavailable on this platform") try: - with os.scandir(path) as scandir_it: - entries = list(scandir_it) - except FileNotFoundError: + st = os.lstat(path) + except OSError as err: + onexc(os.lstat, path, err) return + try: + if _rmtree_islink(st): + # symlinks to directories are forbidden, see bug #1669 + raise OSError("Cannot call rmtree on a symbolic link") except OSError as err: - onexc(os.scandir, path, err) - entries = [] - for entry in entries: - fullname = entry.path - try: - is_dir = entry.is_dir(follow_symlinks=False) - except FileNotFoundError: - continue - except OSError: - is_dir = False - - if is_dir and not entry.is_junction(): + onexc(os.path.islink, path, err) + # can't continue even if onexc hook returns + return + def onerror(err): + if not isinstance(err, FileNotFoundError): + onexc(os.scandir, err.filename, err) + results = os.walk(path, topdown=False, onerror=onerror, followlinks=os._walk_symlinks_as_files) + for dirpath, dirnames, filenames in results: + for name in dirnames: + fullname = os.path.join(dirpath, name) try: - if entry.is_symlink(): - # This can only happen if someone replaces - # a directory with a symlink after the call to - # os.scandir or entry.is_dir above. - raise OSError("Cannot call rmtree on a symbolic link") + os.rmdir(fullname) except FileNotFoundError: continue except OSError as err: - onexc(os.path.islink, fullname, err) - continue - _rmtree_unsafe(fullname, onexc) - else: + onexc(os.rmdir, fullname, err) + for name in filenames: + fullname = os.path.join(dirpath, name) try: os.unlink(fullname) except FileNotFoundError: @@ -650,86 +648,101 @@ def _rmtree_unsafe(path, onexc): onexc(os.rmdir, path, err) # Version using fd-based APIs to protect against races -def _rmtree_safe_fd(topfd, path, onexc): +def _rmtree_safe_fd(path, dir_fd, onexc): + # While the unsafe rmtree works fine on bytes, the fd based does not. + if isinstance(path, bytes): + path = os.fsdecode(path) + stack = [(os.lstat, dir_fd, path, None)] + try: + while stack: + _rmtree_safe_fd_step(stack, onexc) + finally: + # Close any file descriptors still on the stack. + while stack: + func, fd, path, entry = stack.pop() + if func is not os.close: + continue + try: + os.close(fd) + except OSError as err: + onexc(os.close, path, err) + +def _rmtree_safe_fd_step(stack, onexc): + # Each stack item has four elements: + # * func: The first operation to perform: os.lstat, os.close or os.rmdir. + # Walking a directory starts with an os.lstat() to detect symlinks; in + # this case, func is updated before subsequent operations and passed to + # onexc() if an error occurs. + # * dirfd: Open file descriptor, or None if we're processing the top-level + # directory given to rmtree() and the user didn't supply dir_fd. + # * path: Path of file to operate upon. This is passed to onexc() if an + # error occurs. + # * orig_entry: os.DirEntry, or None if we're processing the top-level + # directory given to rmtree(). We used the cached stat() of the entry to + # save a call to os.lstat() when walking subdirectories. + func, dirfd, path, orig_entry = stack.pop() + name = path if orig_entry is None else orig_entry.name try: + if func is os.close: + os.close(dirfd) + return + if func is os.rmdir: + os.rmdir(name, dir_fd=dirfd) + return + + # Note: To guard against symlink races, we use the standard + # lstat()/open()/fstat() trick. + assert func is os.lstat + if orig_entry is None: + orig_st = os.lstat(name, dir_fd=dirfd) + else: + orig_st = orig_entry.stat(follow_symlinks=False) + + func = os.open # For error reporting. + topfd = os.open(name, os.O_RDONLY | os.O_NONBLOCK, dir_fd=dirfd) + + func = os.path.islink # For error reporting. + try: + if not os.path.samestat(orig_st, os.fstat(topfd)): + # Symlinks to directories are forbidden, see GH-46010. + raise OSError("Cannot call rmtree on a symbolic link") + stack.append((os.rmdir, dirfd, path, orig_entry)) + finally: + stack.append((os.close, topfd, path, orig_entry)) + + func = os.scandir # For error reporting. with os.scandir(topfd) as scandir_it: entries = list(scandir_it) - except FileNotFoundError: - return - except OSError as err: - err.filename = path - onexc(os.scandir, path, err) - return - for entry in entries: - fullname = os.path.join(path, entry.name) - try: - is_dir = entry.is_dir(follow_symlinks=False) - except FileNotFoundError: - continue - except OSError: - is_dir = False - else: - if is_dir: - try: - orig_st = entry.stat(follow_symlinks=False) - is_dir = stat.S_ISDIR(orig_st.st_mode) - except FileNotFoundError: - continue - except OSError as err: - onexc(os.lstat, fullname, err) - continue - if is_dir: + for entry in entries: + fullname = os.path.join(path, entry.name) try: - dirfd = os.open(entry.name, os.O_RDONLY, dir_fd=topfd) - dirfd_closed = False + if entry.is_dir(follow_symlinks=False): + # Traverse into sub-directory. + stack.append((os.lstat, topfd, fullname, entry)) + continue except FileNotFoundError: continue - except OSError as err: - onexc(os.open, fullname, err) - else: - try: - if os.path.samestat(orig_st, os.fstat(dirfd)): - _rmtree_safe_fd(dirfd, fullname, onexc) - try: - os.close(dirfd) - except OSError as err: - # close() should not be retried after an error. - dirfd_closed = True - onexc(os.close, fullname, err) - dirfd_closed = True - try: - os.rmdir(entry.name, dir_fd=topfd) - except FileNotFoundError: - continue - except OSError as err: - onexc(os.rmdir, fullname, err) - else: - try: - # This can only happen if someone replaces - # a directory with a symlink after the call to - # os.scandir or stat.S_ISDIR above. - raise OSError("Cannot call rmtree on a symbolic " - "link") - except OSError as err: - onexc(os.path.islink, fullname, err) - finally: - if not dirfd_closed: - try: - os.close(dirfd) - except OSError as err: - onexc(os.close, fullname, err) - else: + except OSError: + pass try: os.unlink(entry.name, dir_fd=topfd) except FileNotFoundError: continue except OSError as err: onexc(os.unlink, fullname, err) + except FileNotFoundError as err: + if orig_entry is None or func is os.close: + err.filename = path + onexc(func, path, err) + except OSError as err: + err.filename = path + onexc(func, path, err) _use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <= os.supports_dir_fd and os.scandir in os.supports_fd and os.stat in os.supports_follow_symlinks) +_rmtree_impl = _rmtree_safe_fd if _use_fd_functions else _rmtree_unsafe def rmtree(path, ignore_errors=False, onerror=None, *, onexc=None, dir_fd=None): """Recursively delete a directory tree. @@ -773,66 +786,7 @@ def onexc(*args): exc_info = type(exc), exc, exc.__traceback__ return onerror(func, path, exc_info) - if _use_fd_functions: - # While the unsafe rmtree works fine on bytes, the fd based does not. - if isinstance(path, bytes): - path = os.fsdecode(path) - # Note: To guard against symlink races, we use the standard - # lstat()/open()/fstat() trick. - try: - orig_st = os.lstat(path, dir_fd=dir_fd) - except OSError as err: - onexc(os.lstat, path, err) - return - try: - fd = os.open(path, os.O_RDONLY, dir_fd=dir_fd) - fd_closed = False - except OSError as err: - onexc(os.open, path, err) - return - try: - if os.path.samestat(orig_st, os.fstat(fd)): - _rmtree_safe_fd(fd, path, onexc) - try: - os.close(fd) - except OSError as err: - # close() should not be retried after an error. - fd_closed = True - onexc(os.close, path, err) - fd_closed = True - try: - os.rmdir(path, dir_fd=dir_fd) - except OSError as err: - onexc(os.rmdir, path, err) - else: - try: - # symlinks to directories are forbidden, see bug #1669 - raise OSError("Cannot call rmtree on a symbolic link") - except OSError as err: - onexc(os.path.islink, path, err) - finally: - if not fd_closed: - try: - os.close(fd) - except OSError as err: - onexc(os.close, path, err) - else: - if dir_fd is not None: - raise NotImplementedError("dir_fd unavailable on this platform") - try: - st = os.lstat(path) - except OSError as err: - onexc(os.lstat, path, err) - return - try: - if _rmtree_islink(st): - # symlinks to directories are forbidden, see bug #1669 - raise OSError("Cannot call rmtree on a symbolic link") - except OSError as err: - onexc(os.path.islink, path, err) - # can't continue even if onexc hook returns - return - return _rmtree_unsafe(path, onexc) + _rmtree_impl(path, dir_fd, onexc) # Allow introspection of whether or not the hardening against symlink # attacks is supported on the current platform @@ -1441,11 +1395,18 @@ def disk_usage(path): return _ntuple_diskusage(total, used, free) -def chown(path, user=None, group=None): +def chown(path, user=None, group=None, *, dir_fd=None, follow_symlinks=True): """Change owner user and group of the given path. user and group can be the uid/gid or the user/group names, and in that case, they are converted to their respective uid/gid. + + If dir_fd is set, it should be an open file descriptor to the directory to + be used as the root of *path* if it is relative. + + If follow_symlinks is set to False and the last element of the path is a + symbolic link, chown will modify the link itself and not the file being + referenced by the link. """ sys.audit('shutil.chown', path, user, group) @@ -1471,7 +1432,8 @@ def chown(path, user=None, group=None): if _group is None: raise LookupError("no such group: {!r}".format(group)) - os.chown(path, _user, _group) + os.chown(path, _user, _group, dir_fd=dir_fd, + follow_symlinks=follow_symlinks) def get_terminal_size(fallback=(80, 24)): """Get the size of the terminal window. @@ -1588,21 +1550,21 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): if sys.platform == "win32": # PATHEXT is necessary to check on Windows. pathext_source = os.getenv("PATHEXT") or _WIN_DEFAULT_PATHEXT - pathext = [ext for ext in pathext_source.split(os.pathsep) if ext] + pathext = pathext_source.split(os.pathsep) + pathext = [ext.rstrip('.') for ext in pathext if ext] if use_bytes: pathext = [os.fsencode(ext) for ext in pathext] - files = ([cmd] + [cmd + ext for ext in pathext]) + files = [cmd + ext for ext in pathext] - # gh-109590. If we are looking for an executable, we need to look - # for a PATHEXT match. The first cmd is the direct match - # (e.g. python.exe instead of python) - # Check that direct match first if and only if the extension is in PATHEXT - # Otherwise check it last - suffix = os.path.splitext(files[0])[1].upper() - if mode & os.X_OK and not any(suffix == ext.upper() for ext in pathext): - files.append(files.pop(0)) + # If X_OK in mode, simulate the cmd.exe behavior: look at direct + # match if and only if the extension is in PATHEXT. + # If X_OK not in mode, simulate the first result of where.exe: + # always look at direct match before a PATHEXT match. + normcmd = cmd.upper() + if not (mode & os.X_OK) or any(normcmd.endswith(ext.upper()) for ext in pathext): + files.insert(0, cmd) else: # On other platforms you don't have things like PATHEXT to tell you # what file suffixes are executable, so just pass on cmd as-is. @@ -1611,10 +1573,22 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): seen = set() for dir in path: normdir = os.path.normcase(dir) - if not normdir in seen: + if normdir not in seen: seen.add(normdir) for thefile in files: name = os.path.join(dir, thefile) if _access_check(name, mode): return name return None + +def __getattr__(name): + if name == "ExecError": + import warnings + warnings._deprecated( + "shutil.ExecError", + f"{warnings._DEPRECATED_MSG}; it " + "isn't raised by any shutil function.", + remove=(3, 16) + ) + return RuntimeError + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/site.py b/Lib/site.py index 2aee63e24ca52b..9da8b6724e1cec 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -94,6 +94,12 @@ def _trace(message): print(message, file=sys.stderr) +def _warn(*args, **kwargs): + import warnings + + warnings.warn(*args, **kwargs) + + def makepath(*paths): dir = os.path.join(*paths) try: @@ -179,35 +185,46 @@ def addpackage(sitedir, name, known_paths): return _trace(f"Processing .pth file: {fullname!r}") try: - # locale encoding is not ideal especially on Windows. But we have used - # it for a long time. setuptools uses the locale encoding too. - f = io.TextIOWrapper(io.open_code(fullname), encoding="locale") + with io.open_code(fullname) as f: + pth_content = f.read() except OSError: return - with f: - for n, line in enumerate(f): - if line.startswith("#"): - continue - if line.strip() == "": + + try: + # Accept BOM markers in .pth files as we do in source files + # (Windows PowerShell 5.1 makes it hard to emit UTF-8 files without a BOM) + pth_content = pth_content.decode("utf-8-sig") + except UnicodeDecodeError: + # Fallback to locale encoding for backward compatibility. + # We will deprecate this fallback in the future. + import locale + pth_content = pth_content.decode(locale.getencoding()) + _trace(f"Cannot read {fullname!r} as UTF-8. " + f"Using fallback encoding {locale.getencoding()!r}") + + for n, line in enumerate(pth_content.splitlines(), 1): + if line.startswith("#"): + continue + if line.strip() == "": + continue + try: + if line.startswith(("import ", "import\t")): + exec(line) continue - try: - if line.startswith(("import ", "import\t")): - exec(line) - continue - line = line.rstrip() - dir, dircase = makepath(sitedir, line) - if not dircase in known_paths and os.path.exists(dir): - sys.path.append(dir) - known_paths.add(dircase) - except Exception as exc: - print("Error processing line {:d} of {}:\n".format(n+1, fullname), - file=sys.stderr) - import traceback - for record in traceback.format_exception(exc): - for line in record.splitlines(): - print(' '+line, file=sys.stderr) - print("\nRemainder of file ignored", file=sys.stderr) - break + line = line.rstrip() + dir, dircase = makepath(sitedir, line) + if dircase not in known_paths and os.path.exists(dir): + sys.path.append(dir) + known_paths.add(dircase) + except Exception as exc: + print(f"Error processing line {n:d} of {fullname}:\n", + file=sys.stderr) + import traceback + for record in traceback.format_exception(exc): + for line in record.splitlines(): + print(' '+line, file=sys.stderr) + print("\nRemainder of file ignored", file=sys.stderr) + break if reset: known_paths = None return known_paths @@ -280,8 +297,8 @@ def _getuserbase(): if env_base: return env_base - # Emscripten, VxWorks, and WASI have no home directories - if sys.platform in {"emscripten", "vxworks", "wasi"}: + # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories + if sys.platform in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: return None def joinuser(*args): @@ -301,6 +318,10 @@ def joinuser(*args): # Same to sysconfig.get_path('purelib', os.name+'_user') def _get_path(userbase): version = sys.version_info + if hasattr(sys, 'abiflags') and 't' in sys.abiflags: + abi_thread = 't' + else: + abi_thread = '' implementation = _get_implementation() implementation_lower = implementation.lower() @@ -311,7 +332,7 @@ def _get_path(userbase): if sys.platform == 'darwin' and sys._framework: return f'{userbase}/lib/{implementation_lower}/site-packages' - return f'{userbase}/lib/python{version[0]}.{version[1]}/site-packages' + return f'{userbase}/lib/python{version[0]}.{version[1]}{abi_thread}/site-packages' def getuserbase(): @@ -379,6 +400,10 @@ def getsitepackages(prefixes=None): implementation = _get_implementation().lower() ver = sys.version_info + if hasattr(sys, 'abiflags') and 't' in sys.abiflags: + abi_thread = 't' + else: + abi_thread = '' if os.sep == '/': libdirs = [sys.platlibdir] if sys.platlibdir != "lib": @@ -386,7 +411,7 @@ def getsitepackages(prefixes=None): for libdir in libdirs: path = os.path.join(prefix, libdir, - f"{implementation}{ver[0]}.{ver[1]}", + f"{implementation}{ver[0]}.{ver[1]}{abi_thread}", "site-packages") sitepackages.append(path) else: @@ -423,8 +448,9 @@ def setcopyright(): """Set 'copyright' and 'credits' in builtins""" builtins.copyright = _sitebuiltins._Printer("copyright", sys.copyright) builtins.credits = _sitebuiltins._Printer("credits", """\ - Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands - for supporting Python development. See www.python.org for more information.""") + Thanks to CWI, CNRI, BeOpen, Zope Corporation, the Python Software + Foundation, and a cast of thousands for supporting Python + development. See www.python.org for more information.""") files, dirs = [], [] # Not all modules are required to have a __file__ attribute. See # PEP 420 for more details. @@ -472,30 +498,61 @@ def register_readline(): This can be overridden in the sitecustomize or usercustomize module, or in a PYTHONSTARTUP file. """ + if not sys.flags.ignore_environment: + PYTHON_BASIC_REPL = os.getenv("PYTHON_BASIC_REPL") + else: + PYTHON_BASIC_REPL = False + import atexit + + try: + try: + import readline + except ImportError: + readline = None + else: + import rlcompleter # noqa: F401 + except ImportError: + return + try: - import readline - import rlcompleter + if PYTHON_BASIC_REPL: + CAN_USE_PYREPL = False + else: + original_path = sys.path + sys.path = [p for p in original_path if p != ''] + try: + import _pyrepl.readline + if os.name == "nt": + import _pyrepl.windows_console + console_errors = (_pyrepl.windows_console._error,) + else: + import _pyrepl.unix_console + console_errors = _pyrepl.unix_console._error + from _pyrepl.main import CAN_USE_PYREPL + finally: + sys.path = original_path except ImportError: return - # Reading the initialization (config) file may not be enough to set a - # completion key, so we set one first and then read the file. - if readline.backend == 'editline': - readline.parse_and_bind('bind ^I rl_complete') - else: - readline.parse_and_bind('tab: complete') + if readline is not None: + # Reading the initialization (config) file may not be enough to set a + # completion key, so we set one first and then read the file. + if readline.backend == 'editline': + readline.parse_and_bind('bind ^I rl_complete') + else: + readline.parse_and_bind('tab: complete') - try: - readline.read_init_file() - except OSError: - # An OSError here could have many causes, but the most likely one - # is that there's no .inputrc file (or .editrc file in the case of - # Mac OS X + libedit) in the expected location. In that case, we - # want to ignore the exception. - pass + try: + readline.read_init_file() + except OSError: + # An OSError here could have many causes, but the most likely one + # is that there's no .inputrc file (or .editrc file in the case of + # Mac OS X + libedit) in the expected location. In that case, we + # want to ignore the exception. + pass - if readline.get_current_history_length() == 0: + if readline is None or readline.get_current_history_length() == 0: # If no history was loaded, default to .python_history, # or PYTHON_HISTORY. # The guard is necessary to avoid doubling history size at @@ -503,14 +560,24 @@ def register_readline(): # through a PYTHONSTARTUP hook, see: # http://bugs.python.org/issue5845#msg198636 history = gethistoryfile() + + if CAN_USE_PYREPL: + readline_module = _pyrepl.readline + exceptions = (OSError, *console_errors) + else: + if readline is None: + return + readline_module = readline + exceptions = OSError + try: - readline.read_history_file(history) - except OSError: + readline_module.read_history_file(history) + except exceptions: pass def write_history(): try: - readline.write_history_file(history) + readline_module.write_history_file(history) except (FileNotFoundError, PermissionError): # home directory does not exist or is not writable # https://bugs.python.org/issue19891 @@ -558,17 +625,17 @@ def venv(known_paths): elif key == 'home': sys._home = value - sys.prefix = sys.exec_prefix = site_prefix + if sys.prefix != site_prefix: + _warn(f'Unexpected value in sys.prefix, expected {site_prefix}, got {sys.prefix}', RuntimeWarning) + if sys.exec_prefix != site_prefix: + _warn(f'Unexpected value in sys.exec_prefix, expected {site_prefix}, got {sys.exec_prefix}', RuntimeWarning) # Doing this here ensures venv takes precedence over user-site addsitepackages(known_paths, [sys.prefix]) - # addsitepackages will process site_prefix again if its in PREFIXES, - # but that's ok; known_paths will prevent anything being added twice if system_site == "true": - PREFIXES.insert(0, sys.prefix) + PREFIXES += [sys.base_prefix, sys.base_exec_prefix] else: - PREFIXES = [sys.prefix] ENABLE_USER_SITE = False return known_paths @@ -578,7 +645,7 @@ def execsitecustomize(): """Run custom site specific code, if available.""" try: try: - import sitecustomize + import sitecustomize # noqa: F401 except ImportError as exc: if exc.name == 'sitecustomize': pass @@ -598,7 +665,7 @@ def execusercustomize(): """Run custom user specific code, if available.""" try: try: - import usercustomize + import usercustomize # noqa: F401 except ImportError as exc: if exc.name == 'usercustomize': pass diff --git a/Lib/smtplib.py b/Lib/smtplib.py old mode 100755 new mode 100644 index b3cc68a789a7d8..84d6d858e7dec1 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - '''SMTP/ESMTP client class. This should follow RFC 821 (SMTP), RFC 1869 (ESMTP), RFC 2554 (SMTP @@ -105,7 +103,7 @@ class SMTPSenderRefused(SMTPResponseException): """Sender address refused. In addition to the attributes set by on all SMTPResponseException - exceptions, this sets `sender' to the string that the SMTP refused. + exceptions, this sets 'sender' to the string that the SMTP refused. """ def __init__(self, code, msg, sender): @@ -315,7 +313,7 @@ def _get_socket(self, host, port, timeout): def connect(self, host='localhost', port=0, source_address=None): """Connect to a host on a given port. - If the hostname ends with a colon (`:') followed by a number, and + If the hostname ends with a colon (':') followed by a number, and there is no port specified, that suffix will be stripped off and the number interpreted as the port number to use. @@ -346,7 +344,7 @@ def connect(self, host='localhost', port=0, source_address=None): return (code, msg) def send(self, s): - """Send `s' to the server.""" + """Send 's' to the server.""" if self.debuglevel > 0: self._print_debug('send:', repr(s)) if self.sock: diff --git a/Lib/socket.py b/Lib/socket.py index 77986fc2e48099..727b0e75f03595 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -52,7 +52,9 @@ import _socket from _socket import * -import os, sys, io, selectors +import io +import os +import sys from enum import IntEnum, IntFlag try: @@ -110,102 +112,103 @@ def _intenum_converter(value, enum_klass): # WSA error codes if sys.platform.lower().startswith("win"): - errorTab = {} - errorTab[6] = "Specified event object handle is invalid." - errorTab[8] = "Insufficient memory available." - errorTab[87] = "One or more parameters are invalid." - errorTab[995] = "Overlapped operation aborted." - errorTab[996] = "Overlapped I/O event object not in signaled state." - errorTab[997] = "Overlapped operation will complete later." - errorTab[10004] = "The operation was interrupted." - errorTab[10009] = "A bad file handle was passed." - errorTab[10013] = "Permission denied." - errorTab[10014] = "A fault occurred on the network??" # WSAEFAULT - errorTab[10022] = "An invalid operation was attempted." - errorTab[10024] = "Too many open files." - errorTab[10035] = "The socket operation would block." - errorTab[10036] = "A blocking operation is already in progress." - errorTab[10037] = "Operation already in progress." - errorTab[10038] = "Socket operation on nonsocket." - errorTab[10039] = "Destination address required." - errorTab[10040] = "Message too long." - errorTab[10041] = "Protocol wrong type for socket." - errorTab[10042] = "Bad protocol option." - errorTab[10043] = "Protocol not supported." - errorTab[10044] = "Socket type not supported." - errorTab[10045] = "Operation not supported." - errorTab[10046] = "Protocol family not supported." - errorTab[10047] = "Address family not supported by protocol family." - errorTab[10048] = "The network address is in use." - errorTab[10049] = "Cannot assign requested address." - errorTab[10050] = "Network is down." - errorTab[10051] = "Network is unreachable." - errorTab[10052] = "Network dropped connection on reset." - errorTab[10053] = "Software caused connection abort." - errorTab[10054] = "The connection has been reset." - errorTab[10055] = "No buffer space available." - errorTab[10056] = "Socket is already connected." - errorTab[10057] = "Socket is not connected." - errorTab[10058] = "The network has been shut down." - errorTab[10059] = "Too many references." - errorTab[10060] = "The operation timed out." - errorTab[10061] = "Connection refused." - errorTab[10062] = "Cannot translate name." - errorTab[10063] = "The name is too long." - errorTab[10064] = "The host is down." - errorTab[10065] = "The host is unreachable." - errorTab[10066] = "Directory not empty." - errorTab[10067] = "Too many processes." - errorTab[10068] = "User quota exceeded." - errorTab[10069] = "Disk quota exceeded." - errorTab[10070] = "Stale file handle reference." - errorTab[10071] = "Item is remote." - errorTab[10091] = "Network subsystem is unavailable." - errorTab[10092] = "Winsock.dll version out of range." - errorTab[10093] = "Successful WSAStartup not yet performed." - errorTab[10101] = "Graceful shutdown in progress." - errorTab[10102] = "No more results from WSALookupServiceNext." - errorTab[10103] = "Call has been canceled." - errorTab[10104] = "Procedure call table is invalid." - errorTab[10105] = "Service provider is invalid." - errorTab[10106] = "Service provider failed to initialize." - errorTab[10107] = "System call failure." - errorTab[10108] = "Service not found." - errorTab[10109] = "Class type not found." - errorTab[10110] = "No more results from WSALookupServiceNext." - errorTab[10111] = "Call was canceled." - errorTab[10112] = "Database query was refused." - errorTab[11001] = "Host not found." - errorTab[11002] = "Nonauthoritative host not found." - errorTab[11003] = "This is a nonrecoverable error." - errorTab[11004] = "Valid name, no data record requested type." - errorTab[11005] = "QoS receivers." - errorTab[11006] = "QoS senders." - errorTab[11007] = "No QoS senders." - errorTab[11008] = "QoS no receivers." - errorTab[11009] = "QoS request confirmed." - errorTab[11010] = "QoS admission error." - errorTab[11011] = "QoS policy failure." - errorTab[11012] = "QoS bad style." - errorTab[11013] = "QoS bad object." - errorTab[11014] = "QoS traffic control error." - errorTab[11015] = "QoS generic error." - errorTab[11016] = "QoS service type error." - errorTab[11017] = "QoS flowspec error." - errorTab[11018] = "Invalid QoS provider buffer." - errorTab[11019] = "Invalid QoS filter style." - errorTab[11020] = "Invalid QoS filter style." - errorTab[11021] = "Incorrect QoS filter count." - errorTab[11022] = "Invalid QoS object length." - errorTab[11023] = "Incorrect QoS flow count." - errorTab[11024] = "Unrecognized QoS object." - errorTab[11025] = "Invalid QoS policy object." - errorTab[11026] = "Invalid QoS flow descriptor." - errorTab[11027] = "Invalid QoS provider-specific flowspec." - errorTab[11028] = "Invalid QoS provider-specific filterspec." - errorTab[11029] = "Invalid QoS shape discard mode object." - errorTab[11030] = "Invalid QoS shaping rate object." - errorTab[11031] = "Reserved policy QoS element type." + errorTab = { + 6: "Specified event object handle is invalid.", + 8: "Insufficient memory available.", + 87: "One or more parameters are invalid.", + 995: "Overlapped operation aborted.", + 996: "Overlapped I/O event object not in signaled state.", + 997: "Overlapped operation will complete later.", + 10004: "The operation was interrupted.", + 10009: "A bad file handle was passed.", + 10013: "Permission denied.", + 10014: "A fault occurred on the network??", + 10022: "An invalid operation was attempted.", + 10024: "Too many open files.", + 10035: "The socket operation would block.", + 10036: "A blocking operation is already in progress.", + 10037: "Operation already in progress.", + 10038: "Socket operation on nonsocket.", + 10039: "Destination address required.", + 10040: "Message too long.", + 10041: "Protocol wrong type for socket.", + 10042: "Bad protocol option.", + 10043: "Protocol not supported.", + 10044: "Socket type not supported.", + 10045: "Operation not supported.", + 10046: "Protocol family not supported.", + 10047: "Address family not supported by protocol family.", + 10048: "The network address is in use.", + 10049: "Cannot assign requested address.", + 10050: "Network is down.", + 10051: "Network is unreachable.", + 10052: "Network dropped connection on reset.", + 10053: "Software caused connection abort.", + 10054: "The connection has been reset.", + 10055: "No buffer space available.", + 10056: "Socket is already connected.", + 10057: "Socket is not connected.", + 10058: "The network has been shut down.", + 10059: "Too many references.", + 10060: "The operation timed out.", + 10061: "Connection refused.", + 10062: "Cannot translate name.", + 10063: "The name is too long.", + 10064: "The host is down.", + 10065: "The host is unreachable.", + 10066: "Directory not empty.", + 10067: "Too many processes.", + 10068: "User quota exceeded.", + 10069: "Disk quota exceeded.", + 10070: "Stale file handle reference.", + 10071: "Item is remote.", + 10091: "Network subsystem is unavailable.", + 10092: "Winsock.dll version out of range.", + 10093: "Successful WSAStartup not yet performed.", + 10101: "Graceful shutdown in progress.", + 10102: "No more results from WSALookupServiceNext.", + 10103: "Call has been canceled.", + 10104: "Procedure call table is invalid.", + 10105: "Service provider is invalid.", + 10106: "Service provider failed to initialize.", + 10107: "System call failure.", + 10108: "Service not found.", + 10109: "Class type not found.", + 10110: "No more results from WSALookupServiceNext.", + 10111: "Call was canceled.", + 10112: "Database query was refused.", + 11001: "Host not found.", + 11002: "Nonauthoritative host not found.", + 11003: "This is a nonrecoverable error.", + 11004: "Valid name, no data record requested type.", + 11005: "QoS receivers.", + 11006: "QoS senders.", + 11007: "No QoS senders.", + 11008: "QoS no receivers.", + 11009: "QoS request confirmed.", + 11010: "QoS admission error.", + 11011: "QoS policy failure.", + 11012: "QoS bad style.", + 11013: "QoS bad object.", + 11014: "QoS traffic control error.", + 11015: "QoS generic error.", + 11016: "QoS service type error.", + 11017: "QoS flowspec error.", + 11018: "Invalid QoS provider buffer.", + 11019: "Invalid QoS filter style.", + 11020: "Invalid QoS filter style.", + 11021: "Incorrect QoS filter count.", + 11022: "Invalid QoS object length.", + 11023: "Incorrect QoS flow count.", + 11024: "Unrecognized QoS object.", + 11025: "Invalid QoS policy object.", + 11026: "Invalid QoS flow descriptor.", + 11027: "Invalid QoS provider-specific flowspec.", + 11028: "Invalid QoS provider-specific filterspec.", + 11029: "Invalid QoS shape discard mode object.", + 11030: "Invalid QoS shaping rate object.", + 11031: "Reserved policy QoS element type." + } __all__.append("errorTab") @@ -306,7 +309,8 @@ def makefile(self, mode="r", buffering=None, *, """makefile(...) -> an I/O stream connected to the socket The arguments are as for io.open() after the filename, except the only - supported mode values are 'r' (default), 'w' and 'b'. + supported mode values are 'r' (default), 'w', 'b', or a combination of + those. """ # XXX refactor to share code? if not set(mode) <= {"r", "w", "b"}: @@ -347,6 +351,9 @@ def makefile(self, mode="r", buffering=None, *, if hasattr(os, 'sendfile'): def _sendfile_use_sendfile(self, file, offset=0, count=None): + # Lazy import to improve module import time + import selectors + self._check_sendfile_params(file, offset, count) sockno = self.fileno() try: @@ -548,20 +555,18 @@ def fromfd(fd, family, type, proto=0): return socket(family, type, proto, nfd) if hasattr(_socket.socket, "sendmsg"): - import array - def send_fds(sock, buffers, fds, flags=0, address=None): """ send_fds(sock, buffers, fds[, flags[, address]]) -> integer Send the list of file descriptors fds over an AF_UNIX socket. """ + import array + return sock.sendmsg(buffers, [(_socket.SOL_SOCKET, _socket.SCM_RIGHTS, array.array("i", fds))]) __all__.append("send_fds") if hasattr(_socket.socket, "recvmsg"): - import array - def recv_fds(sock, bufsize, maxfds, flags=0): """ recv_fds(sock, bufsize, maxfds[, flags]) -> (data, list of file descriptors, msg_flags, address) @@ -569,6 +574,8 @@ def recv_fds(sock, bufsize, maxfds, flags=0): Receive up to maxfds file descriptors returning the message data and a list containing the descriptors. """ + import array + # Array of ints fds = array.array("i") msg, ancdata, flags, addr = sock.recvmsg(bufsize, @@ -591,16 +598,65 @@ def fromshare(info): return socket(0, 0, 0, info) __all__.append("fromshare") -if hasattr(_socket, "socketpair"): +# Origin: https://gist.github.com/4325783, by Geert Jansen. Public domain. +# This is used if _socket doesn't natively provide socketpair. It's +# always defined so that it can be patched in for testing purposes. +def _fallback_socketpair(family=AF_INET, type=SOCK_STREAM, proto=0): + if family == AF_INET: + host = _LOCALHOST + elif family == AF_INET6: + host = _LOCALHOST_V6 + else: + raise ValueError("Only AF_INET and AF_INET6 socket address families " + "are supported") + if type != SOCK_STREAM: + raise ValueError("Only SOCK_STREAM socket type is supported") + if proto != 0: + raise ValueError("Only protocol zero is supported") + + # We create a connected TCP socket. Note the trick with + # setblocking(False) that prevents us from having to create a thread. + lsock = socket(family, type, proto) + try: + lsock.bind((host, 0)) + lsock.listen() + # On IPv6, ignore flow_info and scope_id + addr, port = lsock.getsockname()[:2] + csock = socket(family, type, proto) + try: + csock.setblocking(False) + try: + csock.connect((addr, port)) + except (BlockingIOError, InterruptedError): + pass + csock.setblocking(True) + ssock, _ = lsock.accept() + except: + csock.close() + raise + finally: + lsock.close() - def socketpair(family=None, type=SOCK_STREAM, proto=0): - """socketpair([family[, type[, proto]]]) -> (socket object, socket object) + # Authenticating avoids using a connection from something else + # able to connect to {host}:{port} instead of us. + # We expect only AF_INET and AF_INET6 families. + try: + if ( + ssock.getsockname() != csock.getpeername() + or csock.getsockname() != ssock.getpeername() + ): + raise ConnectionError("Unexpected peer connection") + except: + # getsockname() and getpeername() can fail + # if either socket isn't connected. + ssock.close() + csock.close() + raise - Create a pair of socket objects from the sockets returned by the platform - socketpair() function. - The arguments are the same as for socket() except the default family is - AF_UNIX if defined on the platform; otherwise, the default is AF_INET. - """ + return (ssock, csock) + +if hasattr(_socket, "socketpair"): + def socketpair(family=None, type=SOCK_STREAM, proto=0): if family is None: try: family = AF_UNIX @@ -612,44 +668,7 @@ def socketpair(family=None, type=SOCK_STREAM, proto=0): return a, b else: - - # Origin: https://gist.github.com/4325783, by Geert Jansen. Public domain. - def socketpair(family=AF_INET, type=SOCK_STREAM, proto=0): - if family == AF_INET: - host = _LOCALHOST - elif family == AF_INET6: - host = _LOCALHOST_V6 - else: - raise ValueError("Only AF_INET and AF_INET6 socket address families " - "are supported") - if type != SOCK_STREAM: - raise ValueError("Only SOCK_STREAM socket type is supported") - if proto != 0: - raise ValueError("Only protocol zero is supported") - - # We create a connected TCP socket. Note the trick with - # setblocking(False) that prevents us from having to create a thread. - lsock = socket(family, type, proto) - try: - lsock.bind((host, 0)) - lsock.listen() - # On IPv6, ignore flow_info and scope_id - addr, port = lsock.getsockname()[:2] - csock = socket(family, type, proto) - try: - csock.setblocking(False) - try: - csock.connect((addr, port)) - except (BlockingIOError, InterruptedError): - pass - csock.setblocking(True) - ssock, _ = lsock.accept() - except: - csock.close() - raise - finally: - lsock.close() - return (ssock, csock) + socketpair = _fallback_socketpair __all__.append("socketpair") socketpair.__doc__ = """socketpair([family[, type[, proto]]]) -> (socket object, socket object) @@ -918,7 +937,9 @@ def create_server(address, *, family=AF_INET, backlog=None, reuse_port=False, # Fail later on bind(), for platforms which may not # support this option. pass - if reuse_port: + # Since Linux 6.12.9, SO_REUSEPORT is not allowed + # on other address families than AF_INET/AF_INET6. + if reuse_port and family in (AF_INET, AF_INET6): sock.setsockopt(SOL_SOCKET, SO_REUSEPORT, 1) if has_ipv6 and family == AF_INET6: if dualstack_ipv6: diff --git a/Lib/socketserver.py b/Lib/socketserver.py index cd028ef1c63b85..35b2723de3babe 100644 --- a/Lib/socketserver.py +++ b/Lib/socketserver.py @@ -468,7 +468,12 @@ def server_bind(self): """ if self.allow_reuse_address and hasattr(socket, "SO_REUSEADDR"): self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - if self.allow_reuse_port and hasattr(socket, "SO_REUSEPORT"): + # Since Linux 6.12.9, SO_REUSEPORT is not allowed + # on other address families than AF_INET/AF_INET6. + if ( + self.allow_reuse_port and hasattr(socket, "SO_REUSEPORT") + and self.address_family in (socket.AF_INET, socket.AF_INET6) + ): self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) self.socket.bind(self.server_address) self.server_address = self.socket.getsockname() diff --git a/Lib/sqlite3/__init__.py b/Lib/sqlite3/__init__.py index 927267cf0b92ff..34a9c047dd607c 100644 --- a/Lib/sqlite3/__init__.py +++ b/Lib/sqlite3/__init__.py @@ -55,16 +55,3 @@ """ from sqlite3.dbapi2 import * -from sqlite3.dbapi2 import (_deprecated_names, - _deprecated_version_info, - _deprecated_version) - - -def __getattr__(name): - if name in _deprecated_names: - from warnings import warn - - warn(f"{name} is deprecated and will be removed in Python 3.14", - DeprecationWarning, stacklevel=2) - return globals()[f"_deprecated_{name}"] - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/sqlite3/__main__.py b/Lib/sqlite3/__main__.py index b93b84384a0925..cfdee61403d1fa 100644 --- a/Lib/sqlite3/__main__.py +++ b/Lib/sqlite3/__main__.py @@ -65,7 +65,6 @@ def runsource(self, source, filename="", symbol="single"): def main(*args): parser = ArgumentParser( description="Python sqlite3 CLI", - prog="python -m sqlite3", ) parser.add_argument( "filename", type=str, default=":memory:", nargs="?", @@ -117,7 +116,7 @@ def main(*args): # No SQL provided; start the REPL. console = SqliteInteractiveConsole(con) try: - import readline + import readline # noqa: F401 except ImportError: pass console.interact(banner, exitmsg="") diff --git a/Lib/sqlite3/dbapi2.py b/Lib/sqlite3/dbapi2.py index 56fc0461e6c922..0315760516edf8 100644 --- a/Lib/sqlite3/dbapi2.py +++ b/Lib/sqlite3/dbapi2.py @@ -25,9 +25,6 @@ import collections.abc from _sqlite3 import * -from _sqlite3 import _deprecated_version - -_deprecated_names = frozenset({"version", "version_info"}) paramstyle = "qmark" @@ -48,7 +45,7 @@ def TimeFromTicks(ticks): def TimestampFromTicks(ticks): return Timestamp(*time.localtime(ticks)[:6]) -_deprecated_version_info = tuple(map(int, _deprecated_version.split("."))) + sqlite_version_info = tuple([int(x) for x in sqlite_version.split(".")]) Binary = memoryview @@ -97,12 +94,3 @@ def convert_timestamp(val): # Clean up namespace del(register_adapters_and_converters) - -def __getattr__(name): - if name in _deprecated_names: - from warnings import warn - - warn(f"{name} is deprecated and will be removed in Python 3.14", - DeprecationWarning, stacklevel=2) - return globals()[f"_deprecated_{name}"] - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/sqlite3/dump.py b/Lib/sqlite3/dump.py index 9dcce7dc76ced4..57e6a3b4f1e6eb 100644 --- a/Lib/sqlite3/dump.py +++ b/Lib/sqlite3/dump.py @@ -26,6 +26,7 @@ def _iterdump(connection, *, filter=None): writeable_schema = False cu = connection.cursor() + cu.row_factory = None # Make sure we get predictable results. # Disable foreign key constraints, if there is any foreign key violation. violations = cu.execute("PRAGMA foreign_key_check").fetchall() if violations: diff --git a/Lib/ssl.py b/Lib/ssl.py index 03d0121891ff4c..05df4ad7f0f05c 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -116,7 +116,7 @@ from _ssl import ( HAS_SNI, HAS_ECDH, HAS_NPN, HAS_ALPN, HAS_SSLv2, HAS_SSLv3, HAS_TLSv1, - HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3, HAS_PSK + HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3, HAS_PSK, HAS_PHA ) from _ssl import _DEFAULT_CIPHERS, _OPENSSL_API_VERSION @@ -513,18 +513,17 @@ def set_alpn_protocols(self, alpn_protocols): self._set_alpn_protocols(protos) def _load_windows_store_certs(self, storename, purpose): - certs = bytearray() try: for cert, encoding, trust in enum_certificates(storename): # CA certs are never PKCS#7 encoded if encoding == "x509_asn": if trust is True or purpose.oid in trust: - certs.extend(cert) + try: + self.load_verify_locations(cadata=cert) + except SSLError as exc: + warnings.warn(f"Bad certificate in Windows certificate store: {exc!s}") except PermissionError: warnings.warn("unable to enumerate Windows certificate store") - if certs: - self.load_verify_locations(cadata=certs) - return certs def load_default_certs(self, purpose=Purpose.SERVER_AUTH): if not isinstance(purpose, _ASN1Object): @@ -704,6 +703,16 @@ def create_default_context(purpose=Purpose.SERVER_AUTH, *, cafile=None, else: raise ValueError(purpose) + # `VERIFY_X509_PARTIAL_CHAIN` makes OpenSSL's chain building behave more + # like RFC 3280 and 5280, which specify that chain building stops with the + # first trust anchor, even if that anchor is not self-signed. + # + # `VERIFY_X509_STRICT` makes OpenSSL more conservative about the + # certificates it accepts, including "disabling workarounds for + # some broken certificates." + context.verify_flags |= (_ssl.VERIFY_X509_PARTIAL_CHAIN | + _ssl.VERIFY_X509_STRICT) + if cafile or capath or cadata: context.load_verify_locations(cafile, capath, cadata) elif context.verify_mode != CERT_NONE: @@ -1155,11 +1164,21 @@ def getpeercert(self, binary_form=False): @_sslcopydoc def get_verified_chain(self): - return self._sslobj.get_verified_chain() + chain = self._sslobj.get_verified_chain() + + if chain is None: + return [] + + return [cert.public_bytes(_ssl.ENCODING_DER) for cert in chain] @_sslcopydoc def get_unverified_chain(self): - return self._sslobj.get_unverified_chain() + chain = self._sslobj.get_unverified_chain() + + if chain is None: + return [] + + return [cert.public_bytes(_ssl.ENCODING_DER) for cert in chain] @_sslcopydoc def selected_npn_protocol(self): diff --git a/Lib/stat.py b/Lib/stat.py index 9167ab185944fb..1b4ed1ebc940ef 100644 --- a/Lib/stat.py +++ b/Lib/stat.py @@ -2,7 +2,6 @@ Suggested usage: from stat import * """ -import sys # Indices for stat struct members in the tuple returned by os.stat() diff --git a/Lib/statistics.py b/Lib/statistics.py index 7924123c05b8c3..e2b59267f04f68 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -11,7 +11,7 @@ Function Description ================== ================================================== mean Arithmetic mean (average) of data. -fmean Fast, floating point arithmetic mean. +fmean Fast, floating-point arithmetic mean. geometric_mean Geometric mean of data. harmonic_mean Harmonic mean of data. median Median (middle value) of data. @@ -113,6 +113,7 @@ 'geometric_mean', 'harmonic_mean', 'kde', + 'kde_random', 'linear_regression', 'mean', 'median', @@ -138,454 +139,157 @@ from itertools import count, groupby, repeat from bisect import bisect_left, bisect_right from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum, sumprod -from math import isfinite, isinf, pi, cos, cosh +from math import isfinite, isinf, pi, cos, sin, tan, cosh, asin, atan, acos from functools import reduce from operator import itemgetter from collections import Counter, namedtuple, defaultdict _SQRT2 = sqrt(2.0) +_random = random -# === Exceptions === +## Exceptions ############################################################## class StatisticsError(ValueError): pass -# === Private utilities === +## Measures of central tendency (averages) ################################# -def _sum(data): - """_sum(data) -> (type, sum, count) - - Return a high-precision sum of the given numeric data as a fraction, - together with the type to be converted to and the count of items. - - Examples - -------- - - >>> _sum([3, 2.25, 4.5, -0.5, 0.25]) - (, Fraction(19, 2), 5) - - Some sources of round-off error will be avoided: - - # Built-in sum returns zero. - >>> _sum([1e50, 1, -1e50] * 1000) - (, Fraction(1000, 1), 3000) +def mean(data): + """Return the sample arithmetic mean of data. - Fractions and Decimals are also supported: + >>> mean([1, 2, 3, 4, 4]) + 2.8 >>> from fractions import Fraction as F - >>> _sum([F(2, 3), F(7, 5), F(1, 4), F(5, 6)]) - (, Fraction(63, 20), 4) + >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) + Fraction(13, 21) >>> from decimal import Decimal as D - >>> data = [D("0.1375"), D("0.2108"), D("0.3061"), D("0.0419")] - >>> _sum(data) - (, Fraction(6963, 10000), 4) + >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) + Decimal('0.5625') + + If ``data`` is empty, StatisticsError will be raised. - Mixed types are currently treated as an error, except that int is - allowed. """ - count = 0 - types = set() - types_add = types.add - partials = {} - partials_get = partials.get - for typ, values in groupby(data, type): - types_add(typ) - for n, d in map(_exact_ratio, values): - count += 1 - partials[d] = partials_get(d, 0) + n - if None in partials: - # The sum will be a NAN or INF. We can ignore all the finite - # partials, and just look at this special one. - total = partials[None] - assert not _isfinite(total) - else: - # Sum all the partial sums using builtin sum. - total = sum(Fraction(n, d) for d, n in partials.items()) - T = reduce(_coerce, types, int) # or raise TypeError - return (T, total, count) + T, total, n = _sum(data) + if n < 1: + raise StatisticsError('mean requires at least one data point') + return _convert(total / n, T) -def _ss(data, c=None): - """Return the exact mean and sum of square deviations of sequence data. +def fmean(data, weights=None): + """Convert data to floats and compute the arithmetic mean. - Calculations are done in a single pass, allowing the input to be an iterator. + This runs faster than the mean() function and it always returns a float. + If the input dataset is empty, it raises a StatisticsError. - If given *c* is used the mean; otherwise, it is calculated from the data. - Use the *c* argument with care, as it can lead to garbage results. + >>> fmean([3.5, 4.0, 5.25]) + 4.25 """ - if c is not None: - T, ssd, count = _sum((d := x - c) * d for x in data) - return (T, ssd, c, count) - count = 0 - types = set() - types_add = types.add - sx_partials = defaultdict(int) - sxx_partials = defaultdict(int) - for typ, values in groupby(data, type): - types_add(typ) - for n, d in map(_exact_ratio, values): - count += 1 - sx_partials[d] += n - sxx_partials[d] += n * n - if not count: - ssd = c = Fraction(0) - elif None in sx_partials: - # The sum will be a NAN or INF. We can ignore all the finite - # partials, and just look at this special one. - ssd = c = sx_partials[None] - assert not _isfinite(ssd) - else: - sx = sum(Fraction(n, d) for d, n in sx_partials.items()) - sxx = sum(Fraction(n, d*d) for d, n in sxx_partials.items()) - # This formula has poor numeric properties for floats, - # but with fractions it is exact. - ssd = (count * sxx - sx * sx) / count - c = sx / count - T = reduce(_coerce, types, int) # or raise TypeError - return (T, ssd, c, count) + if weights is None: + try: + n = len(data) + except TypeError: + # Handle iterators that do not define __len__(). + counter = count() + total = fsum(map(itemgetter(0), zip(data, counter))) + n = next(counter) + else: + total = fsum(data) -def _isfinite(x): - try: - return x.is_finite() # Likely a Decimal. - except AttributeError: - return math.isfinite(x) # Coerces to float first. + if not n: + raise StatisticsError('fmean requires at least one data point') + return total / n -def _coerce(T, S): - """Coerce types T and S to a common type, or raise TypeError. + if not isinstance(weights, (list, tuple)): + weights = list(weights) - Coercion rules are currently an implementation detail. See the CoerceTest - test class in test_statistics for details. - """ - # See http://bugs.python.org/issue24068. - assert T is not bool, "initial type T is bool" - # If the types are the same, no need to coerce anything. Put this - # first, so that the usual case (no coercion needed) happens as soon - # as possible. - if T is S: return T - # Mixed int & other coerce to the other type. - if S is int or S is bool: return T - if T is int: return S - # If one is a (strict) subclass of the other, coerce to the subclass. - if issubclass(S, T): return S - if issubclass(T, S): return T - # Ints coerce to the other type. - if issubclass(T, int): return S - if issubclass(S, int): return T - # Mixed fraction & float coerces to float (or float subclass). - if issubclass(T, Fraction) and issubclass(S, float): - return S - if issubclass(T, float) and issubclass(S, Fraction): - return T - # Any other combination is disallowed. - msg = "don't know how to coerce %s and %s" - raise TypeError(msg % (T.__name__, S.__name__)) + try: + num = sumprod(data, weights) + except ValueError: + raise StatisticsError('data and weights must be the same length') + den = fsum(weights) -def _exact_ratio(x): - """Return Real number x to exact (numerator, denominator) pair. + if not den: + raise StatisticsError('sum of weights must be non-zero') - >>> _exact_ratio(0.25) - (1, 4) + return num / den - x is expected to be an int, Fraction, Decimal or float. - """ - # XXX We should revisit whether using fractions to accumulate exact - # ratios is the right way to go. +def geometric_mean(data): + """Convert data to floats and compute the geometric mean. - # The integer ratios for binary floats can have numerators or - # denominators with over 300 decimal digits. The problem is more - # acute with decimal floats where the default decimal context - # supports a huge range of exponents from Emin=-999999 to - # Emax=999999. When expanded with as_integer_ratio(), numbers like - # Decimal('3.14E+5000') and Decimal('3.14E-5000') have large - # numerators or denominators that will slow computation. + Raises a StatisticsError if the input dataset is empty + or if it contains a negative value. - # When the integer ratios are accumulated as fractions, the size - # grows to cover the full range from the smallest magnitude to the - # largest. For example, Fraction(3.14E+300) + Fraction(3.14E-300), - # has a 616 digit numerator. Likewise, - # Fraction(Decimal('3.14E+5000')) + Fraction(Decimal('3.14E-5000')) - # has 10,003 digit numerator. + Returns zero if the product of inputs is zero. - # This doesn't seem to have been problem in practice, but it is a - # potential pitfall. + No special efforts are made to achieve exact results. + (However, this may change in the future.) - try: - return x.as_integer_ratio() - except AttributeError: - pass - except (OverflowError, ValueError): - # float NAN or INF. - assert not _isfinite(x) - return (x, None) - try: - # x may be an Integral ABC. - return (x.numerator, x.denominator) - except AttributeError: - msg = f"can't convert type '{type(x).__name__}' to numerator/denominator" - raise TypeError(msg) + >>> round(geometric_mean([54, 24, 36]), 9) + 36.0 + """ + n = 0 + found_zero = False -def _convert(value, T): - """Convert value to given numeric type T.""" - if type(value) is T: - # This covers the cases where T is Fraction, or where value is - # a NAN or INF (Decimal or float). - return value - if issubclass(T, int) and value.denominator != 1: - T = float - try: - # FIXME: what do we do if this overflows? - return T(value) - except TypeError: - if issubclass(T, Decimal): - return T(value.numerator) / T(value.denominator) - else: - raise + def count_positive(iterable): + nonlocal n, found_zero + for n, x in enumerate(iterable, start=1): + if x > 0.0 or math.isnan(x): + yield x + elif x == 0.0: + found_zero = True + else: + raise StatisticsError('No negative inputs allowed', x) + total = fsum(map(log, count_positive(data))) -def _fail_neg(values, errmsg='negative value'): - """Iterate over values, failing if any are less than zero.""" - for x in values: - if x < 0: - raise StatisticsError(errmsg) - yield x + if not n: + raise StatisticsError('Must have a non-empty dataset') + if math.isnan(total): + return math.nan + if found_zero: + return math.nan if total == math.inf else 0.0 + return exp(total / n) -def _rank(data, /, *, key=None, reverse=False, ties='average', start=1) -> list[float]: - """Rank order a dataset. The lowest value has rank 1. - Ties are averaged so that equal values receive the same rank: +def harmonic_mean(data, weights=None): + """Return the harmonic mean of data. - >>> data = [31, 56, 31, 25, 75, 18] - >>> _rank(data) - [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + The harmonic mean is the reciprocal of the arithmetic mean of the + reciprocals of the data. It can be used for averaging ratios or + rates, for example speeds. - The operation is idempotent: + Suppose a car travels 40 km/hr for 5 km and then speeds-up to + 60 km/hr for another 5 km. What is the average speed? - >>> _rank([3.5, 5.0, 3.5, 2.0, 6.0, 1.0]) - [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + >>> harmonic_mean([40, 60]) + 48.0 - It is possible to rank the data in reverse order so that the - highest value has rank 1. Also, a key-function can extract - the field to be ranked: + Suppose a car travels 40 km/hr for 5 km, and when traffic clears, + speeds-up to 60 km/hr for the remaining 30 km of the journey. What + is the average speed? - >>> goals = [('eagles', 45), ('bears', 48), ('lions', 44)] - >>> _rank(goals, key=itemgetter(1), reverse=True) - [2.0, 1.0, 3.0] + >>> harmonic_mean([40, 60], weights=[5, 30]) + 56.0 - Ranks are conventionally numbered starting from one; however, - setting *start* to zero allows the ranks to be used as array indices: + If ``data`` is empty, or any element is less than zero, + ``harmonic_mean`` will raise ``StatisticsError``. - >>> prize = ['Gold', 'Silver', 'Bronze', 'Certificate'] - >>> scores = [8.1, 7.3, 9.4, 8.3] - >>> [prize[int(i)] for i in _rank(scores, start=0, reverse=True)] - ['Bronze', 'Certificate', 'Gold', 'Silver'] - - """ - # If this function becomes public at some point, more thought - # needs to be given to the signature. A list of ints is - # plausible when ties is "min" or "max". When ties is "average", - # either list[float] or list[Fraction] is plausible. - - # Default handling of ties matches scipy.stats.mstats.spearmanr. - if ties != 'average': - raise ValueError(f'Unknown tie resolution method: {ties!r}') - if key is not None: - data = map(key, data) - val_pos = sorted(zip(data, count()), reverse=reverse) - i = start - 1 - result = [0] * len(val_pos) - for _, g in groupby(val_pos, key=itemgetter(0)): - group = list(g) - size = len(group) - rank = i + (size + 1) / 2 - for value, orig_pos in group: - result[orig_pos] = rank - i += size - return result - - -def _integer_sqrt_of_frac_rto(n: int, m: int) -> int: - """Square root of n/m, rounded to the nearest integer using round-to-odd.""" - # Reference: https://www.lri.fr/~melquion/doc/05-imacs17_1-expose.pdf - a = math.isqrt(n // m) - return a | (a*a*m != n) - - -# For 53 bit precision floats, the bit width used in -# _float_sqrt_of_frac() is 109. -_sqrt_bit_width: int = 2 * sys.float_info.mant_dig + 3 - - -def _float_sqrt_of_frac(n: int, m: int) -> float: - """Square root of n/m as a float, correctly rounded.""" - # See principle and proof sketch at: https://bugs.python.org/msg407078 - q = (n.bit_length() - m.bit_length() - _sqrt_bit_width) // 2 - if q >= 0: - numerator = _integer_sqrt_of_frac_rto(n, m << 2 * q) << q - denominator = 1 - else: - numerator = _integer_sqrt_of_frac_rto(n << -2 * q, m) - denominator = 1 << -q - return numerator / denominator # Convert to float - - -def _decimal_sqrt_of_frac(n: int, m: int) -> Decimal: - """Square root of n/m as a Decimal, correctly rounded.""" - # Premise: For decimal, computing (n/m).sqrt() can be off - # by 1 ulp from the correctly rounded result. - # Method: Check the result, moving up or down a step if needed. - if n <= 0: - if not n: - return Decimal('0.0') - n, m = -n, -m - - root = (Decimal(n) / Decimal(m)).sqrt() - nr, dr = root.as_integer_ratio() - - plus = root.next_plus() - np, dp = plus.as_integer_ratio() - # test: n / m > ((root + plus) / 2) ** 2 - if 4 * n * (dr*dp)**2 > m * (dr*np + dp*nr)**2: - return plus - - minus = root.next_minus() - nm, dm = minus.as_integer_ratio() - # test: n / m < ((root + minus) / 2) ** 2 - if 4 * n * (dr*dm)**2 < m * (dr*nm + dm*nr)**2: - return minus - - return root - - -# === Measures of central tendency (averages) === - -def mean(data): - """Return the sample arithmetic mean of data. - - >>> mean([1, 2, 3, 4, 4]) - 2.8 - - >>> from fractions import Fraction as F - >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) - Fraction(13, 21) - - >>> from decimal import Decimal as D - >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) - Decimal('0.5625') - - If ``data`` is empty, StatisticsError will be raised. - """ - T, total, n = _sum(data) - if n < 1: - raise StatisticsError('mean requires at least one data point') - return _convert(total / n, T) - - -def fmean(data, weights=None): - """Convert data to floats and compute the arithmetic mean. - - This runs faster than the mean() function and it always returns a float. - If the input dataset is empty, it raises a StatisticsError. - - >>> fmean([3.5, 4.0, 5.25]) - 4.25 - """ - if weights is None: - try: - n = len(data) - except TypeError: - # Handle iterators that do not define __len__(). - n = 0 - def count(iterable): - nonlocal n - for n, x in enumerate(iterable, start=1): - yield x - data = count(data) - total = fsum(data) - if not n: - raise StatisticsError('fmean requires at least one data point') - return total / n - if not isinstance(weights, (list, tuple)): - weights = list(weights) - try: - num = sumprod(data, weights) - except ValueError: - raise StatisticsError('data and weights must be the same length') - den = fsum(weights) - if not den: - raise StatisticsError('sum of weights must be non-zero') - return num / den - - -def geometric_mean(data): - """Convert data to floats and compute the geometric mean. - - Raises a StatisticsError if the input dataset is empty - or if it contains a negative value. - - Returns zero if the product of inputs is zero. - - No special efforts are made to achieve exact results. - (However, this may change in the future.) - - >>> round(geometric_mean([54, 24, 36]), 9) - 36.0 - """ - n = 0 - found_zero = False - def count_positive(iterable): - nonlocal n, found_zero - for n, x in enumerate(iterable, start=1): - if x > 0.0 or math.isnan(x): - yield x - elif x == 0.0: - found_zero = True - else: - raise StatisticsError('No negative inputs allowed', x) - total = fsum(map(log, count_positive(data))) - if not n: - raise StatisticsError('Must have a non-empty dataset') - if math.isnan(total): - return math.nan - if found_zero: - return math.nan if total == math.inf else 0.0 - return exp(total / n) - - -def harmonic_mean(data, weights=None): - """Return the harmonic mean of data. - - The harmonic mean is the reciprocal of the arithmetic mean of the - reciprocals of the data. It can be used for averaging ratios or - rates, for example speeds. - - Suppose a car travels 40 km/hr for 5 km and then speeds-up to - 60 km/hr for another 5 km. What is the average speed? - - >>> harmonic_mean([40, 60]) - 48.0 - - Suppose a car travels 40 km/hr for 5 km, and when traffic clears, - speeds-up to 60 km/hr for the remaining 30 km of the journey. What - is the average speed? - - >>> harmonic_mean([40, 60], weights=[5, 30]) - 56.0 - - If ``data`` is empty, or any element is less than zero, - ``harmonic_mean`` will raise ``StatisticsError``. """ if iter(data) is data: data = list(data) + errmsg = 'harmonic mean does not support negative values' + n = len(data) if n < 1: raise StatisticsError('harmonic_mean requires at least one data point') @@ -597,6 +301,7 @@ def harmonic_mean(data, weights=None): return x else: raise TypeError('unsupported type') + if weights is None: weights = repeat(1, n) sum_weights = n @@ -606,16 +311,19 @@ def harmonic_mean(data, weights=None): if len(weights) != n: raise StatisticsError('Number of weights does not match data size') _, sum_weights, _ = _sum(w for w in _fail_neg(weights, errmsg)) + try: data = _fail_neg(data, errmsg) T, total, count = _sum(w / x if w else 0 for w, x in zip(weights, data)) except ZeroDivisionError: return 0 + if total <= 0: raise StatisticsError('Weighted sum must be positive') + return _convert(sum_weights / total, T) -# FIXME: investigate ways to calculate medians without sorting? Quickselect? + def median(data): """Return the median (middle value) of numeric data. @@ -652,6 +360,9 @@ def median_low(data): 3 """ + # Potentially the sorting step could be replaced with a quickselect. + # However, it would require an excellent implementation to beat our + # highly optimized builtin sort. data = sorted(data) n = len(data) if n == 0: @@ -795,6 +506,7 @@ def multimode(data): ['b', 'd', 'f'] >>> multimode('') [] + """ counts = Counter(iter(data)) if not counts: @@ -803,301 +515,52 @@ def multimode(data): return [value for value, count in counts.items() if count == maxcount] -def kde(data, h, kernel='normal'): - """Kernel Density Estimation: Create a continuous probability - density function from discrete samples. +## Measures of spread ###################################################### - The basic idea is to smooth the data using a kernel function - to help draw inferences about a population from a sample. +def variance(data, xbar=None): + """Return the sample variance of data. - The degree of smoothing is controlled by the scaling parameter h - which is called the bandwidth. Smaller values emphasize local - features while larger values give smoother results. + data should be an iterable of Real-valued numbers, with at least two + values. The optional argument xbar, if given, should be the mean of + the data. If it is missing or None, the mean is automatically calculated. - The kernel determines the relative weights of the sample data - points. Generally, the choice of kernel shape does not matter - as much as the more influential bandwidth smoothing parameter. + Use this function when your data is a sample from a population. To + calculate the variance from the entire population, see ``pvariance``. - Kernels that give some weight to every sample point: + Examples: - normal or gauss - logistic - sigmoid + >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] + >>> variance(data) + 1.3720238095238095 - Kernels that only give weight to sample points within - the bandwidth: + If you have already calculated the mean of your data, you can pass it as + the optional second argument ``xbar`` to avoid recalculating it: - rectangular or uniform - triangular - parabolic or epanechnikov - quartic or biweight - triweight - cosine + >>> m = mean(data) + >>> variance(data, m) + 1.3720238095238095 - A StatisticsError will be raised if the data sequence is empty. + This function does not check that ``xbar`` is actually the mean of + ``data``. Giving arbitrary values for ``xbar`` may lead to invalid or + impossible results. - Example - ------- + Decimals and Fractions are supported: - Given a sample of six data points, construct a continuous - function that estimates the underlying probability density: + >>> from decimal import Decimal as D + >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) + Decimal('31.01875') - >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] - >>> f_hat = kde(sample, h=1.5) + >>> from fractions import Fraction as F + >>> variance([F(1, 6), F(1, 2), F(5, 3)]) + Fraction(67, 108) - Compute the area under the curve: + """ + # http://mathworld.wolfram.com/SampleVariance.html - >>> sum(f_hat(x) for x in range(-20, 20)) - 1.0 - - Plot the estimated probability density function at - evenly spaced points from -6 to 10: - - >>> for x in range(-6, 11): - ... density = f_hat(x) - ... plot = ' ' * int(density * 400) + 'x' - ... print(f'{x:2}: {density:.3f} {plot}') - ... - -6: 0.002 x - -5: 0.009 x - -4: 0.031 x - -3: 0.070 x - -2: 0.111 x - -1: 0.125 x - 0: 0.110 x - 1: 0.086 x - 2: 0.068 x - 3: 0.059 x - 4: 0.066 x - 5: 0.082 x - 6: 0.082 x - 7: 0.058 x - 8: 0.028 x - 9: 0.009 x - 10: 0.002 x - - References - ---------- - - Kernel density estimation and its application: - https://www.itm-conferences.org/articles/itmconf/pdf/2018/08/itmconf_sam2018_00037.pdf - - Kernel functions in common use: - https://en.wikipedia.org/wiki/Kernel_(statistics)#kernel_functions_in_common_use - - Interactive graphical demonstration and exploration: - https://demonstrations.wolfram.com/KernelDensityEstimation/ - - """ - - n = len(data) - if not n: - raise StatisticsError('Empty data sequence') - - if not isinstance(data[0], (int, float)): - raise TypeError('Data sequence must contain ints or floats') - - if h <= 0.0: - raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') - - match kernel: - - case 'normal' | 'gauss': - c = 1 / sqrt(2 * pi) - K = lambda t: c * exp(-1/2 * t * t) - support = None - - case 'logistic': - # 1.0 / (exp(t) + 2.0 + exp(-t)) - K = lambda t: 1/2 / (1.0 + cosh(t)) - support = None - - case 'sigmoid': - # (2/pi) / (exp(t) + exp(-t)) - c = 1 / pi - K = lambda t: c / cosh(t) - support = None - - case 'rectangular' | 'uniform': - K = lambda t: 1/2 - support = 1.0 - - case 'triangular': - K = lambda t: 1.0 - abs(t) - support = 1.0 - - case 'parabolic' | 'epanechnikov': - K = lambda t: 3/4 * (1.0 - t * t) - support = 1.0 - - case 'quartic' | 'biweight': - K = lambda t: 15/16 * (1.0 - t * t) ** 2 - support = 1.0 - - case 'triweight': - K = lambda t: 35/32 * (1.0 - t * t) ** 3 - support = 1.0 - - case 'cosine': - c1 = pi / 4 - c2 = pi / 2 - K = lambda t: c1 * cos(c2 * t) - support = 1.0 - - case _: - raise StatisticsError(f'Unknown kernel name: {kernel!r}') - - if support is None: - - def pdf(x): - return sum(K((x - x_i) / h) for x_i in data) / (n * h) - - else: - - sample = sorted(data) - bandwidth = h * support - - def pdf(x): - i = bisect_left(sample, x - bandwidth) - j = bisect_right(sample, x + bandwidth) - supported = sample[i : j] - return sum(K((x - x_i) / h) for x_i in supported) / (n * h) - - pdf.__doc__ = f'PDF estimate with {kernel=!r} and {h=!r}' - - return pdf - - -# Notes on methods for computing quantiles -# ---------------------------------------- -# -# There is no one perfect way to compute quantiles. Here we offer -# two methods that serve common needs. Most other packages -# surveyed offered at least one or both of these two, making them -# "standard" in the sense of "widely-adopted and reproducible". -# They are also easy to explain, easy to compute manually, and have -# straight-forward interpretations that aren't surprising. - -# The default method is known as "R6", "PERCENTILE.EXC", or "expected -# value of rank order statistics". The alternative method is known as -# "R7", "PERCENTILE.INC", or "mode of rank order statistics". - -# For sample data where there is a positive probability for values -# beyond the range of the data, the R6 exclusive method is a -# reasonable choice. Consider a random sample of nine values from a -# population with a uniform distribution from 0.0 to 1.0. The -# distribution of the third ranked sample point is described by -# betavariate(alpha=3, beta=7) which has mode=0.250, median=0.286, and -# mean=0.300. Only the latter (which corresponds with R6) gives the -# desired cut point with 30% of the population falling below that -# value, making it comparable to a result from an inv_cdf() function. -# The R6 exclusive method is also idempotent. - -# For describing population data where the end points are known to -# be included in the data, the R7 inclusive method is a reasonable -# choice. Instead of the mean, it uses the mode of the beta -# distribution for the interior points. Per Hyndman & Fan, "One nice -# property is that the vertices of Q7(p) divide the range into n - 1 -# intervals, and exactly 100p% of the intervals lie to the left of -# Q7(p) and 100(1 - p)% of the intervals lie to the right of Q7(p)." - -# If needed, other methods could be added. However, for now, the -# position is that fewer options make for easier choices and that -# external packages can be used for anything more advanced. - -def quantiles(data, *, n=4, method='exclusive'): - """Divide *data* into *n* continuous intervals with equal probability. - - Returns a list of (n - 1) cut points separating the intervals. - - Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. - Set *n* to 100 for percentiles which gives the 99 cuts points that - separate *data* in to 100 equal sized groups. - - The *data* can be any iterable containing sample. - The cut points are linearly interpolated between data points. - - If *method* is set to *inclusive*, *data* is treated as population - data. The minimum value is treated as the 0th percentile and the - maximum value is treated as the 100th percentile. - """ - if n < 1: - raise StatisticsError('n must be at least 1') - data = sorted(data) - ld = len(data) - if ld < 2: - if ld == 1: - return data * (n - 1) - raise StatisticsError('must have at least one data point') - if method == 'inclusive': - m = ld - 1 - result = [] - for i in range(1, n): - j, delta = divmod(i * m, n) - interpolated = (data[j] * (n - delta) + data[j + 1] * delta) / n - result.append(interpolated) - return result - if method == 'exclusive': - m = ld + 1 - result = [] - for i in range(1, n): - j = i * m // n # rescale i to m/n - j = 1 if j < 1 else ld-1 if j > ld-1 else j # clamp to 1 .. ld-1 - delta = i*m - j*n # exact integer math - interpolated = (data[j - 1] * (n - delta) + data[j] * delta) / n - result.append(interpolated) - return result - raise ValueError(f'Unknown method: {method!r}') - - -# === Measures of spread === - -# See http://mathworld.wolfram.com/Variance.html -# http://mathworld.wolfram.com/SampleVariance.html - - -def variance(data, xbar=None): - """Return the sample variance of data. - - data should be an iterable of Real-valued numbers, with at least two - values. The optional argument xbar, if given, should be the mean of - the data. If it is missing or None, the mean is automatically calculated. - - Use this function when your data is a sample from a population. To - calculate the variance from the entire population, see ``pvariance``. - - Examples: - - >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] - >>> variance(data) - 1.3720238095238095 - - If you have already calculated the mean of your data, you can pass it as - the optional second argument ``xbar`` to avoid recalculating it: - - >>> m = mean(data) - >>> variance(data, m) - 1.3720238095238095 - - This function does not check that ``xbar`` is actually the mean of - ``data``. Giving arbitrary values for ``xbar`` may lead to invalid or - impossible results. - - Decimals and Fractions are supported: - - >>> from decimal import Decimal as D - >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) - Decimal('31.01875') - - >>> from fractions import Fraction as F - >>> variance([F(1, 6), F(1, 2), F(5, 3)]) - Fraction(67, 108) - - """ - T, ss, c, n = _ss(data, xbar) - if n < 2: - raise StatisticsError('variance requires at least two data points') - return _convert(ss / (n - 1), T) + T, ss, c, n = _ss(data, xbar) + if n < 2: + raise StatisticsError('variance requires at least two data points') + return _convert(ss / (n - 1), T) def pvariance(data, mu=None): @@ -1135,6 +598,8 @@ def pvariance(data, mu=None): Fraction(13, 72) """ + # http://mathworld.wolfram.com/Variance.html + T, ss, c, n = _ss(data, mu) if n < 1: raise StatisticsError('pvariance requires at least one data point') @@ -1177,46 +642,7 @@ def pstdev(data, mu=None): return _float_sqrt_of_frac(mss.numerator, mss.denominator) -def _mean_stdev(data): - """In one pass, compute the mean and sample standard deviation as floats.""" - T, ss, xbar, n = _ss(data) - if n < 2: - raise StatisticsError('stdev requires at least two data points') - mss = ss / (n - 1) - try: - return float(xbar), _float_sqrt_of_frac(mss.numerator, mss.denominator) - except AttributeError: - # Handle Nans and Infs gracefully - return float(xbar), float(xbar) / float(ss) - -def _sqrtprod(x: float, y: float) -> float: - "Return sqrt(x * y) computed with improved accuracy and without overflow/underflow." - h = sqrt(x * y) - if not isfinite(h): - if isinf(h) and not isinf(x) and not isinf(y): - # Finite inputs overflowed, so scale down, and recompute. - scale = 2.0 ** -512 # sqrt(1 / sys.float_info.max) - return _sqrtprod(scale * x, scale * y) / scale - return h - if not h: - if x and y: - # Non-zero inputs underflowed, so scale up, and recompute. - # Scale: 1 / sqrt(sys.float_info.min * sys.float_info.epsilon) - scale = 2.0 ** 537 - return _sqrtprod(scale * x, scale * y) / scale - return h - # Improve accuracy with a differential correction. - # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 - d = sumprod((x, h), (y, -h)) - return h + d / (2.0 * h) - - -# === Statistics for relations between two inputs === - -# See https://en.wikipedia.org/wiki/Covariance -# https://en.wikipedia.org/wiki/Pearson_correlation_coefficient -# https://en.wikipedia.org/wiki/Simple_linear_regression - +## Statistics for relations between two inputs ############################# def covariance(x, y, /): """Covariance @@ -1235,6 +661,7 @@ def covariance(x, y, /): -7.5 """ + # https://en.wikipedia.org/wiki/Covariance n = len(x) if len(y) != n: raise StatisticsError('covariance requires that both inputs have same number of data points') @@ -1268,7 +695,10 @@ def correlation(x, y, /, *, method='linear'): Spearman's rank correlation coefficient is appropriate for ordinal data or for continuous data that doesn't meet the linear proportion requirement for Pearson's correlation coefficient. + """ + # https://en.wikipedia.org/wiki/Pearson_correlation_coefficient + # https://en.wikipedia.org/wiki/Spearman%27s_rank_correlation_coefficient n = len(x) if len(y) != n: raise StatisticsError('correlation requires that both inputs have same number of data points') @@ -1276,18 +706,22 @@ def correlation(x, y, /, *, method='linear'): raise StatisticsError('correlation requires at least two data points') if method not in {'linear', 'ranked'}: raise ValueError(f'Unknown method: {method!r}') + if method == 'ranked': start = (n - 1) / -2 # Center rankings around zero x = _rank(x, start=start) y = _rank(y, start=start) + else: xbar = fsum(x) / n ybar = fsum(y) / n x = [xi - xbar for xi in x] y = [yi - ybar for yi in y] + sxy = sumprod(x, y) sxx = sumprod(x, x) syy = sumprod(y, y) + try: return sxy / _sqrtprod(sxx, syy) except ZeroDivisionError: @@ -1335,27 +769,1019 @@ def linear_regression(x, y, /, *, proportional=False): LinearRegression(slope=2.90475..., intercept=0.0) """ + # https://en.wikipedia.org/wiki/Simple_linear_regression n = len(x) if len(y) != n: raise StatisticsError('linear regression requires that both inputs have same number of data points') if n < 2: raise StatisticsError('linear regression requires at least two data points') + if not proportional: xbar = fsum(x) / n ybar = fsum(y) / n x = [xi - xbar for xi in x] # List because used three times below y = (yi - ybar for yi in y) # Generator because only used once below + sxy = sumprod(x, y) + 0.0 # Add zero to coerce result to a float sxx = sumprod(x, x) + try: slope = sxy / sxx # equivalent to: covariance(x, y) / variance(x) except ZeroDivisionError: raise StatisticsError('x is constant') + intercept = 0.0 if proportional else ybar - slope * xbar return LinearRegression(slope=slope, intercept=intercept) -## Normal Distribution ##################################################### +## Kernel Density Estimation ############################################### + +_kernel_specs = {} + +def register(*kernels): + "Load the kernel's pdf, cdf, invcdf, and support into _kernel_specs." + def deco(builder): + spec = dict(zip(('pdf', 'cdf', 'invcdf', 'support'), builder())) + for kernel in kernels: + _kernel_specs[kernel] = spec + return builder + return deco + +@register('normal', 'gauss') +def normal_kernel(): + sqrt2pi = sqrt(2 * pi) + sqrt2 = sqrt(2) + pdf = lambda t: exp(-1/2 * t * t) / sqrt2pi + cdf = lambda t: 1/2 * (1.0 + erf(t / sqrt2)) + invcdf = lambda t: _normal_dist_inv_cdf(t, 0.0, 1.0) + support = None + return pdf, cdf, invcdf, support + +@register('logistic') +def logistic_kernel(): + # 1.0 / (exp(t) + 2.0 + exp(-t)) + pdf = lambda t: 1/2 / (1.0 + cosh(t)) + cdf = lambda t: 1.0 - 1.0 / (exp(t) + 1.0) + invcdf = lambda p: log(p / (1.0 - p)) + support = None + return pdf, cdf, invcdf, support + +@register('sigmoid') +def sigmoid_kernel(): + # (2/pi) / (exp(t) + exp(-t)) + c1 = 1 / pi + c2 = 2 / pi + c3 = pi / 2 + pdf = lambda t: c1 / cosh(t) + cdf = lambda t: c2 * atan(exp(t)) + invcdf = lambda p: log(tan(p * c3)) + support = None + return pdf, cdf, invcdf, support + +@register('rectangular', 'uniform') +def rectangular_kernel(): + pdf = lambda t: 1/2 + cdf = lambda t: 1/2 * t + 1/2 + invcdf = lambda p: 2.0 * p - 1.0 + support = 1.0 + return pdf, cdf, invcdf, support + +@register('triangular') +def triangular_kernel(): + pdf = lambda t: 1.0 - abs(t) + cdf = lambda t: t*t * (1/2 if t < 0.0 else -1/2) + t + 1/2 + invcdf = lambda p: sqrt(2.0*p) - 1.0 if p < 1/2 else 1.0 - sqrt(2.0 - 2.0*p) + support = 1.0 + return pdf, cdf, invcdf, support + +@register('parabolic', 'epanechnikov') +def parabolic_kernel(): + pdf = lambda t: 3/4 * (1.0 - t * t) + cdf = lambda t: sumprod((-1/4, 3/4, 1/2), (t**3, t, 1.0)) + invcdf = lambda p: 2.0 * cos((acos(2.0*p - 1.0) + pi) / 3.0) + support = 1.0 + return pdf, cdf, invcdf, support + +def _newton_raphson(f_inv_estimate, f, f_prime, tolerance=1e-12): + def f_inv(y): + "Return x such that f(x) ≈ y within the specified tolerance." + x = f_inv_estimate(y) + while abs(diff := f(x) - y) > tolerance: + x -= diff / f_prime(x) + return x + return f_inv + +def _quartic_invcdf_estimate(p): + # A handrolled piecewise approximation. There is no magic here. + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + if p < 0.0106: + return ((2.0 * p) ** 0.3838 - 1.0) * sign + x = (2.0 * p) ** 0.4258865685331 - 1.0 + if p < 0.499: + x += 0.026818732 * sin(7.101753784 * p + 2.73230839482953) + return x * sign + +@register('quartic', 'biweight') +def quartic_kernel(): + pdf = lambda t: 15/16 * (1.0 - t * t) ** 2 + cdf = lambda t: sumprod((3/16, -5/8, 15/16, 1/2), + (t**5, t**3, t, 1.0)) + invcdf = _newton_raphson(_quartic_invcdf_estimate, f=cdf, f_prime=pdf) + support = 1.0 + return pdf, cdf, invcdf, support + +def _triweight_invcdf_estimate(p): + # A handrolled piecewise approximation. There is no magic here. + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + x = (2.0 * p) ** 0.3400218741872791 - 1.0 + if 0.00001 < p < 0.499: + x -= 0.033 * sin(1.07 * tau * (p - 0.035)) + return x * sign + +@register('triweight') +def triweight_kernel(): + pdf = lambda t: 35/32 * (1.0 - t * t) ** 3 + cdf = lambda t: sumprod((-5/32, 21/32, -35/32, 35/32, 1/2), + (t**7, t**5, t**3, t, 1.0)) + invcdf = _newton_raphson(_triweight_invcdf_estimate, f=cdf, f_prime=pdf) + support = 1.0 + return pdf, cdf, invcdf, support + +@register('cosine') +def cosine_kernel(): + c1 = pi / 4 + c2 = pi / 2 + pdf = lambda t: c1 * cos(c2 * t) + cdf = lambda t: 1/2 * sin(c2 * t) + 1/2 + invcdf = lambda p: 2.0 * asin(2.0 * p - 1.0) / pi + support = 1.0 + return pdf, cdf, invcdf, support + +del register, normal_kernel, logistic_kernel, sigmoid_kernel +del rectangular_kernel, triangular_kernel, parabolic_kernel +del quartic_kernel, triweight_kernel, cosine_kernel + + +def kde(data, h, kernel='normal', *, cumulative=False): + """Kernel Density Estimation: Create a continuous probability density + function or cumulative distribution function from discrete samples. + + The basic idea is to smooth the data using a kernel function + to help draw inferences about a population from a sample. + + The degree of smoothing is controlled by the scaling parameter h + which is called the bandwidth. Smaller values emphasize local + features while larger values give smoother results. + + The kernel determines the relative weights of the sample data + points. Generally, the choice of kernel shape does not matter + as much as the more influential bandwidth smoothing parameter. + + Kernels that give some weight to every sample point: + + normal (gauss) + logistic + sigmoid + + Kernels that only give weight to sample points within + the bandwidth: + + rectangular (uniform) + triangular + parabolic (epanechnikov) + quartic (biweight) + triweight + cosine + + If *cumulative* is true, will return a cumulative distribution function. + + A StatisticsError will be raised if the data sequence is empty. + + Example + ------- + + Given a sample of six data points, construct a continuous + function that estimates the underlying probability density: + + >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> f_hat = kde(sample, h=1.5) + + Compute the area under the curve: + + >>> area = sum(f_hat(x) for x in range(-20, 20)) + >>> round(area, 4) + 1.0 + + Plot the estimated probability density function at + evenly spaced points from -6 to 10: + + >>> for x in range(-6, 11): + ... density = f_hat(x) + ... plot = ' ' * int(density * 400) + 'x' + ... print(f'{x:2}: {density:.3f} {plot}') + ... + -6: 0.002 x + -5: 0.009 x + -4: 0.031 x + -3: 0.070 x + -2: 0.111 x + -1: 0.125 x + 0: 0.110 x + 1: 0.086 x + 2: 0.068 x + 3: 0.059 x + 4: 0.066 x + 5: 0.082 x + 6: 0.082 x + 7: 0.058 x + 8: 0.028 x + 9: 0.009 x + 10: 0.002 x + + Estimate P(4.5 < X <= 7.5), the probability that a new sample value + will be between 4.5 and 7.5: + + >>> cdf = kde(sample, h=1.5, cumulative=True) + >>> round(cdf(7.5) - cdf(4.5), 2) + 0.22 + + References + ---------- + + Kernel density estimation and its application: + https://www.itm-conferences.org/articles/itmconf/pdf/2018/08/itmconf_sam2018_00037.pdf + + Kernel functions in common use: + https://en.wikipedia.org/wiki/Kernel_(statistics)#kernel_functions_in_common_use + + Interactive graphical demonstration and exploration: + https://demonstrations.wolfram.com/KernelDensityEstimation/ + + Kernel estimation of cumulative distribution function of a random variable with bounded support + https://www.econstor.eu/bitstream/10419/207829/1/10.21307_stattrans-2016-037.pdf + + """ + + n = len(data) + if not n: + raise StatisticsError('Empty data sequence') + + if not isinstance(data[0], (int, float)): + raise TypeError('Data sequence must contain ints or floats') + + if h <= 0.0: + raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') + + kernel_spec = _kernel_specs.get(kernel) + if kernel_spec is None: + raise StatisticsError(f'Unknown kernel name: {kernel!r}') + K = kernel_spec['pdf'] + W = kernel_spec['cdf'] + support = kernel_spec['support'] + + if support is None: + + def pdf(x): + return sum(K((x - x_i) / h) for x_i in data) / (len(data) * h) + + def cdf(x): + return sum(W((x - x_i) / h) for x_i in data) / len(data) + + else: + + sample = sorted(data) + bandwidth = h * support + + def pdf(x): + nonlocal n, sample + if len(data) != n: + sample = sorted(data) + n = len(data) + i = bisect_left(sample, x - bandwidth) + j = bisect_right(sample, x + bandwidth) + supported = sample[i : j] + return sum(K((x - x_i) / h) for x_i in supported) / (n * h) + + def cdf(x): + nonlocal n, sample + if len(data) != n: + sample = sorted(data) + n = len(data) + i = bisect_left(sample, x - bandwidth) + j = bisect_right(sample, x + bandwidth) + supported = sample[i : j] + return sum((W((x - x_i) / h) for x_i in supported), i) / n + + if cumulative: + cdf.__doc__ = f'CDF estimate with {h=!r} and {kernel=!r}' + return cdf + + else: + pdf.__doc__ = f'PDF estimate with {h=!r} and {kernel=!r}' + return pdf + + +def kde_random(data, h, kernel='normal', *, seed=None): + """Return a function that makes a random selection from the estimated + probability density function created by kde(data, h, kernel). + + Providing a *seed* allows reproducible selections within a single + thread. The seed may be an integer, float, str, or bytes. + + A StatisticsError will be raised if the *data* sequence is empty. + + Example: + + >>> data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> rand = kde_random(data, h=1.5, seed=8675309) + >>> new_selections = [rand() for i in range(10)] + >>> [round(x, 1) for x in new_selections] + [0.7, 6.2, 1.2, 6.9, 7.0, 1.8, 2.5, -0.5, -1.8, 5.6] + + """ + n = len(data) + if not n: + raise StatisticsError('Empty data sequence') + + if not isinstance(data[0], (int, float)): + raise TypeError('Data sequence must contain ints or floats') + + if h <= 0.0: + raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') + + kernel_spec = _kernel_specs.get(kernel) + if kernel_spec is None: + raise StatisticsError(f'Unknown kernel name: {kernel!r}') + invcdf = kernel_spec['invcdf'] + + prng = _random.Random(seed) + random = prng.random + choice = prng.choice + + def rand(): + return choice(data) + h * invcdf(random()) + + rand.__doc__ = f'Random KDE selection with {h=!r} and {kernel=!r}' + + return rand + + +## Quantiles ############################################################### + +# There is no one perfect way to compute quantiles. Here we offer +# two methods that serve common needs. Most other packages +# surveyed offered at least one or both of these two, making them +# "standard" in the sense of "widely-adopted and reproducible". +# They are also easy to explain, easy to compute manually, and have +# straight-forward interpretations that aren't surprising. + +# The default method is known as "R6", "PERCENTILE.EXC", or "expected +# value of rank order statistics". The alternative method is known as +# "R7", "PERCENTILE.INC", or "mode of rank order statistics". + +# For sample data where there is a positive probability for values +# beyond the range of the data, the R6 exclusive method is a +# reasonable choice. Consider a random sample of nine values from a +# population with a uniform distribution from 0.0 to 1.0. The +# distribution of the third ranked sample point is described by +# betavariate(alpha=3, beta=7) which has mode=0.250, median=0.286, and +# mean=0.300. Only the latter (which corresponds with R6) gives the +# desired cut point with 30% of the population falling below that +# value, making it comparable to a result from an inv_cdf() function. +# The R6 exclusive method is also idempotent. + +# For describing population data where the end points are known to +# be included in the data, the R7 inclusive method is a reasonable +# choice. Instead of the mean, it uses the mode of the beta +# distribution for the interior points. Per Hyndman & Fan, "One nice +# property is that the vertices of Q7(p) divide the range into n - 1 +# intervals, and exactly 100p% of the intervals lie to the left of +# Q7(p) and 100(1 - p)% of the intervals lie to the right of Q7(p)." + +# If needed, other methods could be added. However, for now, the +# position is that fewer options make for easier choices and that +# external packages can be used for anything more advanced. + +def quantiles(data, *, n=4, method='exclusive'): + """Divide *data* into *n* continuous intervals with equal probability. + + Returns a list of (n - 1) cut points separating the intervals. + + Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. + Set *n* to 100 for percentiles which gives the 99 cuts points that + separate *data* in to 100 equal sized groups. + + The *data* can be any iterable containing sample. + The cut points are linearly interpolated between data points. + + If *method* is set to *inclusive*, *data* is treated as population + data. The minimum value is treated as the 0th percentile and the + maximum value is treated as the 100th percentile. + + """ + if n < 1: + raise StatisticsError('n must be at least 1') + + data = sorted(data) + + ld = len(data) + if ld < 2: + if ld == 1: + return data * (n - 1) + raise StatisticsError('must have at least one data point') + + if method == 'inclusive': + m = ld - 1 + result = [] + for i in range(1, n): + j, delta = divmod(i * m, n) + interpolated = (data[j] * (n - delta) + data[j + 1] * delta) / n + result.append(interpolated) + return result + + if method == 'exclusive': + m = ld + 1 + result = [] + for i in range(1, n): + j = i * m // n # rescale i to m/n + j = 1 if j < 1 else ld-1 if j > ld-1 else j # clamp to 1 .. ld-1 + delta = i*m - j*n # exact integer math + interpolated = (data[j - 1] * (n - delta) + data[j] * delta) / n + result.append(interpolated) + return result + + raise ValueError(f'Unknown method: {method!r}') + + +## Normal Distribution ##################################################### + +class NormalDist: + "Normal distribution of a random variable" + # https://en.wikipedia.org/wiki/Normal_distribution + # https://en.wikipedia.org/wiki/Variance#Properties + + __slots__ = { + '_mu': 'Arithmetic mean of a normal distribution', + '_sigma': 'Standard deviation of a normal distribution', + } + + def __init__(self, mu=0.0, sigma=1.0): + "NormalDist where mu is the mean and sigma is the standard deviation." + if sigma < 0.0: + raise StatisticsError('sigma must be non-negative') + self._mu = float(mu) + self._sigma = float(sigma) + + @classmethod + def from_samples(cls, data): + "Make a normal distribution instance from sample data." + return cls(*_mean_stdev(data)) + + def samples(self, n, *, seed=None): + "Generate *n* samples for a given mean and standard deviation." + rnd = random.random if seed is None else random.Random(seed).random + inv_cdf = _normal_dist_inv_cdf + mu = self._mu + sigma = self._sigma + return [inv_cdf(rnd(), mu, sigma) for _ in repeat(None, n)] + + def pdf(self, x): + "Probability density function. P(x <= X < x+dx) / dx" + variance = self._sigma * self._sigma + if not variance: + raise StatisticsError('pdf() not defined when sigma is zero') + diff = x - self._mu + return exp(diff * diff / (-2.0 * variance)) / sqrt(tau * variance) + + def cdf(self, x): + "Cumulative distribution function. P(X <= x)" + if not self._sigma: + raise StatisticsError('cdf() not defined when sigma is zero') + return 0.5 * (1.0 + erf((x - self._mu) / (self._sigma * _SQRT2))) + + def inv_cdf(self, p): + """Inverse cumulative distribution function. x : P(X <= x) = p + + Finds the value of the random variable such that the probability of + the variable being less than or equal to that value equals the given + probability. + + This function is also called the percent point function or quantile + function. + """ + if p <= 0.0 or p >= 1.0: + raise StatisticsError('p must be in the range 0.0 < p < 1.0') + return _normal_dist_inv_cdf(p, self._mu, self._sigma) + + def quantiles(self, n=4): + """Divide into *n* continuous intervals with equal probability. + + Returns a list of (n - 1) cut points separating the intervals. + + Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. + Set *n* to 100 for percentiles which gives the 99 cuts points that + separate the normal distribution in to 100 equal sized groups. + """ + return [self.inv_cdf(i / n) for i in range(1, n)] + + def overlap(self, other): + """Compute the overlapping coefficient (OVL) between two normal distributions. + + Measures the agreement between two normal probability distributions. + Returns a value between 0.0 and 1.0 giving the overlapping area in + the two underlying probability density functions. + + >>> N1 = NormalDist(2.4, 1.6) + >>> N2 = NormalDist(3.2, 2.0) + >>> N1.overlap(N2) + 0.8035050657330205 + """ + # See: "The overlapping coefficient as a measure of agreement between + # probability distributions and point estimation of the overlap of two + # normal densities" -- Henry F. Inman and Edwin L. Bradley Jr + # http://dx.doi.org/10.1080/03610928908830127 + if not isinstance(other, NormalDist): + raise TypeError('Expected another NormalDist instance') + X, Y = self, other + if (Y._sigma, Y._mu) < (X._sigma, X._mu): # sort to assure commutativity + X, Y = Y, X + X_var, Y_var = X.variance, Y.variance + if not X_var or not Y_var: + raise StatisticsError('overlap() not defined when sigma is zero') + dv = Y_var - X_var + dm = fabs(Y._mu - X._mu) + if not dv: + return 1.0 - erf(dm / (2.0 * X._sigma * _SQRT2)) + a = X._mu * Y_var - Y._mu * X_var + b = X._sigma * Y._sigma * sqrt(dm * dm + dv * log(Y_var / X_var)) + x1 = (a + b) / dv + x2 = (a - b) / dv + return 1.0 - (fabs(Y.cdf(x1) - X.cdf(x1)) + fabs(Y.cdf(x2) - X.cdf(x2))) + + def zscore(self, x): + """Compute the Standard Score. (x - mean) / stdev + + Describes *x* in terms of the number of standard deviations + above or below the mean of the normal distribution. + """ + # https://www.statisticshowto.com/probability-and-statistics/z-score/ + if not self._sigma: + raise StatisticsError('zscore() not defined when sigma is zero') + return (x - self._mu) / self._sigma + + @property + def mean(self): + "Arithmetic mean of the normal distribution." + return self._mu + + @property + def median(self): + "Return the median of the normal distribution" + return self._mu + + @property + def mode(self): + """Return the mode of the normal distribution + + The mode is the value x where which the probability density + function (pdf) takes its maximum value. + """ + return self._mu + + @property + def stdev(self): + "Standard deviation of the normal distribution." + return self._sigma + + @property + def variance(self): + "Square of the standard deviation." + return self._sigma * self._sigma + + def __add__(x1, x2): + """Add a constant or another NormalDist instance. + + If *other* is a constant, translate mu by the constant, + leaving sigma unchanged. + + If *other* is a NormalDist, add both the means and the variances. + Mathematically, this works only if the two distributions are + independent or if they are jointly normally distributed. + """ + if isinstance(x2, NormalDist): + return NormalDist(x1._mu + x2._mu, hypot(x1._sigma, x2._sigma)) + return NormalDist(x1._mu + x2, x1._sigma) + + def __sub__(x1, x2): + """Subtract a constant or another NormalDist instance. + + If *other* is a constant, translate by the constant mu, + leaving sigma unchanged. + + If *other* is a NormalDist, subtract the means and add the variances. + Mathematically, this works only if the two distributions are + independent or if they are jointly normally distributed. + """ + if isinstance(x2, NormalDist): + return NormalDist(x1._mu - x2._mu, hypot(x1._sigma, x2._sigma)) + return NormalDist(x1._mu - x2, x1._sigma) + + def __mul__(x1, x2): + """Multiply both mu and sigma by a constant. + + Used for rescaling, perhaps to change measurement units. + Sigma is scaled with the absolute value of the constant. + """ + return NormalDist(x1._mu * x2, x1._sigma * fabs(x2)) + + def __truediv__(x1, x2): + """Divide both mu and sigma by a constant. + + Used for rescaling, perhaps to change measurement units. + Sigma is scaled with the absolute value of the constant. + """ + return NormalDist(x1._mu / x2, x1._sigma / fabs(x2)) + + def __pos__(x1): + "Return a copy of the instance." + return NormalDist(x1._mu, x1._sigma) + + def __neg__(x1): + "Negates mu while keeping sigma the same." + return NormalDist(-x1._mu, x1._sigma) + + __radd__ = __add__ + + def __rsub__(x1, x2): + "Subtract a NormalDist from a constant or another NormalDist." + return -(x1 - x2) + + __rmul__ = __mul__ + + def __eq__(x1, x2): + "Two NormalDist objects are equal if their mu and sigma are both equal." + if not isinstance(x2, NormalDist): + return NotImplemented + return x1._mu == x2._mu and x1._sigma == x2._sigma + + def __hash__(self): + "NormalDist objects hash equal if their mu and sigma are both equal." + return hash((self._mu, self._sigma)) + + def __repr__(self): + return f'{type(self).__name__}(mu={self._mu!r}, sigma={self._sigma!r})' + + def __getstate__(self): + return self._mu, self._sigma + + def __setstate__(self, state): + self._mu, self._sigma = state + + +## Private utilities ####################################################### + +def _sum(data): + """_sum(data) -> (type, sum, count) + + Return a high-precision sum of the given numeric data as a fraction, + together with the type to be converted to and the count of items. + + Examples + -------- + + >>> _sum([3, 2.25, 4.5, -0.5, 0.25]) + (, Fraction(19, 2), 5) + + Some sources of round-off error will be avoided: + + # Built-in sum returns zero. + >>> _sum([1e50, 1, -1e50] * 1000) + (, Fraction(1000, 1), 3000) + + Fractions and Decimals are also supported: + + >>> from fractions import Fraction as F + >>> _sum([F(2, 3), F(7, 5), F(1, 4), F(5, 6)]) + (, Fraction(63, 20), 4) + + >>> from decimal import Decimal as D + >>> data = [D("0.1375"), D("0.2108"), D("0.3061"), D("0.0419")] + >>> _sum(data) + (, Fraction(6963, 10000), 4) + + Mixed types are currently treated as an error, except that int is + allowed. + + """ + count = 0 + types = set() + types_add = types.add + partials = {} + partials_get = partials.get + + for typ, values in groupby(data, type): + types_add(typ) + for n, d in map(_exact_ratio, values): + count += 1 + partials[d] = partials_get(d, 0) + n + + if None in partials: + # The sum will be a NAN or INF. We can ignore all the finite + # partials, and just look at this special one. + total = partials[None] + assert not _isfinite(total) + else: + # Sum all the partial sums using builtin sum. + total = sum(Fraction(n, d) for d, n in partials.items()) + + T = reduce(_coerce, types, int) # or raise TypeError + return (T, total, count) + + +def _ss(data, c=None): + """Return the exact mean and sum of square deviations of sequence data. + + Calculations are done in a single pass, allowing the input to be an iterator. + + If given *c* is used the mean; otherwise, it is calculated from the data. + Use the *c* argument with care, as it can lead to garbage results. + + """ + if c is not None: + T, ssd, count = _sum((d := x - c) * d for x in data) + return (T, ssd, c, count) + + count = 0 + types = set() + types_add = types.add + sx_partials = defaultdict(int) + sxx_partials = defaultdict(int) + + for typ, values in groupby(data, type): + types_add(typ) + for n, d in map(_exact_ratio, values): + count += 1 + sx_partials[d] += n + sxx_partials[d] += n * n + + if not count: + ssd = c = Fraction(0) + + elif None in sx_partials: + # The sum will be a NAN or INF. We can ignore all the finite + # partials, and just look at this special one. + ssd = c = sx_partials[None] + assert not _isfinite(ssd) + + else: + sx = sum(Fraction(n, d) for d, n in sx_partials.items()) + sxx = sum(Fraction(n, d*d) for d, n in sxx_partials.items()) + # This formula has poor numeric properties for floats, + # but with fractions it is exact. + ssd = (count * sxx - sx * sx) / count + c = sx / count + + T = reduce(_coerce, types, int) # or raise TypeError + return (T, ssd, c, count) + + +def _isfinite(x): + try: + return x.is_finite() # Likely a Decimal. + except AttributeError: + return math.isfinite(x) # Coerces to float first. + + +def _coerce(T, S): + """Coerce types T and S to a common type, or raise TypeError. + + Coercion rules are currently an implementation detail. See the CoerceTest + test class in test_statistics for details. + + """ + # See http://bugs.python.org/issue24068. + assert T is not bool, "initial type T is bool" + # If the types are the same, no need to coerce anything. Put this + # first, so that the usual case (no coercion needed) happens as soon + # as possible. + if T is S: return T + # Mixed int & other coerce to the other type. + if S is int or S is bool: return T + if T is int: return S + # If one is a (strict) subclass of the other, coerce to the subclass. + if issubclass(S, T): return S + if issubclass(T, S): return T + # Ints coerce to the other type. + if issubclass(T, int): return S + if issubclass(S, int): return T + # Mixed fraction & float coerces to float (or float subclass). + if issubclass(T, Fraction) and issubclass(S, float): + return S + if issubclass(T, float) and issubclass(S, Fraction): + return T + # Any other combination is disallowed. + msg = "don't know how to coerce %s and %s" + raise TypeError(msg % (T.__name__, S.__name__)) + + +def _exact_ratio(x): + """Return Real number x to exact (numerator, denominator) pair. + + >>> _exact_ratio(0.25) + (1, 4) + + x is expected to be an int, Fraction, Decimal or float. + + """ + try: + return x.as_integer_ratio() + except AttributeError: + pass + except (OverflowError, ValueError): + # float NAN or INF. + assert not _isfinite(x) + return (x, None) + + try: + # x may be an Integral ABC. + return (x.numerator, x.denominator) + except AttributeError: + msg = f"can't convert type '{type(x).__name__}' to numerator/denominator" + raise TypeError(msg) + + +def _convert(value, T): + """Convert value to given numeric type T.""" + if type(value) is T: + # This covers the cases where T is Fraction, or where value is + # a NAN or INF (Decimal or float). + return value + + if issubclass(T, int) and value.denominator != 1: + T = float + + try: + # FIXME: what do we do if this overflows? + return T(value) + except TypeError: + if issubclass(T, Decimal): + return T(value.numerator) / T(value.denominator) + else: + raise + + +def _fail_neg(values, errmsg='negative value'): + """Iterate over values, failing if any are less than zero.""" + for x in values: + if x < 0: + raise StatisticsError(errmsg) + yield x + + +def _rank(data, /, *, key=None, reverse=False, ties='average', start=1) -> list[float]: + """Rank order a dataset. The lowest value has rank 1. + + Ties are averaged so that equal values receive the same rank: + + >>> data = [31, 56, 31, 25, 75, 18] + >>> _rank(data) + [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + + The operation is idempotent: + + >>> _rank([3.5, 5.0, 3.5, 2.0, 6.0, 1.0]) + [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + + It is possible to rank the data in reverse order so that the + highest value has rank 1. Also, a key-function can extract + the field to be ranked: + + >>> goals = [('eagles', 45), ('bears', 48), ('lions', 44)] + >>> _rank(goals, key=itemgetter(1), reverse=True) + [2.0, 1.0, 3.0] + + Ranks are conventionally numbered starting from one; however, + setting *start* to zero allows the ranks to be used as array indices: + + >>> prize = ['Gold', 'Silver', 'Bronze', 'Certificate'] + >>> scores = [8.1, 7.3, 9.4, 8.3] + >>> [prize[int(i)] for i in _rank(scores, start=0, reverse=True)] + ['Bronze', 'Certificate', 'Gold', 'Silver'] + + """ + # If this function becomes public at some point, more thought + # needs to be given to the signature. A list of ints is + # plausible when ties is "min" or "max". When ties is "average", + # either list[float] or list[Fraction] is plausible. + + # Default handling of ties matches scipy.stats.mstats.spearmanr. + if ties != 'average': + raise ValueError(f'Unknown tie resolution method: {ties!r}') + if key is not None: + data = map(key, data) + val_pos = sorted(zip(data, count()), reverse=reverse) + i = start - 1 + result = [0] * len(val_pos) + for _, g in groupby(val_pos, key=itemgetter(0)): + group = list(g) + size = len(group) + rank = i + (size + 1) / 2 + for value, orig_pos in group: + result[orig_pos] = rank + i += size + return result + + +def _integer_sqrt_of_frac_rto(n: int, m: int) -> int: + """Square root of n/m, rounded to the nearest integer using round-to-odd.""" + # Reference: https://www.lri.fr/~melquion/doc/05-imacs17_1-expose.pdf + a = math.isqrt(n // m) + return a | (a*a*m != n) + + +# For 53 bit precision floats, the bit width used in +# _float_sqrt_of_frac() is 109. +_sqrt_bit_width: int = 2 * sys.float_info.mant_dig + 3 + + +def _float_sqrt_of_frac(n: int, m: int) -> float: + """Square root of n/m as a float, correctly rounded.""" + # See principle and proof sketch at: https://bugs.python.org/msg407078 + q = (n.bit_length() - m.bit_length() - _sqrt_bit_width) // 2 + if q >= 0: + numerator = _integer_sqrt_of_frac_rto(n, m << 2 * q) << q + denominator = 1 + else: + numerator = _integer_sqrt_of_frac_rto(n << -2 * q, m) + denominator = 1 << -q + return numerator / denominator # Convert to float + + +def _decimal_sqrt_of_frac(n: int, m: int) -> Decimal: + """Square root of n/m as a Decimal, correctly rounded.""" + # Premise: For decimal, computing (n/m).sqrt() can be off + # by 1 ulp from the correctly rounded result. + # Method: Check the result, moving up or down a step if needed. + if n <= 0: + if not n: + return Decimal('0.0') + n, m = -n, -m + + root = (Decimal(n) / Decimal(m)).sqrt() + nr, dr = root.as_integer_ratio() + + plus = root.next_plus() + np, dp = plus.as_integer_ratio() + # test: n / m > ((root + plus) / 2) ** 2 + if 4 * n * (dr*dp)**2 > m * (dr*np + dp*nr)**2: + return plus + + minus = root.next_minus() + nm, dm = minus.as_integer_ratio() + # test: n / m < ((root + minus) / 2) ** 2 + if 4 * n * (dr*dm)**2 < m * (dr*nm + dm*nr)**2: + return minus + + return root + + +def _mean_stdev(data): + """In one pass, compute the mean and sample standard deviation as floats.""" + T, ss, xbar, n = _ss(data) + if n < 2: + raise StatisticsError('stdev requires at least two data points') + mss = ss / (n - 1) + try: + return float(xbar), _float_sqrt_of_frac(mss.numerator, mss.denominator) + except AttributeError: + # Handle Nans and Infs gracefully + return float(xbar), float(xbar) / float(ss) + + +def _sqrtprod(x: float, y: float) -> float: + "Return sqrt(x * y) computed with improved accuracy and without overflow/underflow." + + h = sqrt(x * y) + + if not isfinite(h): + if isinf(h) and not isinf(x) and not isinf(y): + # Finite inputs overflowed, so scale down, and recompute. + scale = 2.0 ** -512 # sqrt(1 / sys.float_info.max) + return _sqrtprod(scale * x, scale * y) / scale + return h + + if not h: + if x and y: + # Non-zero inputs underflowed, so scale up, and recompute. + # Scale: 1 / sqrt(sys.float_info.min * sys.float_info.epsilon) + scale = 2.0 ** 537 + return _sqrtprod(scale * x, scale * y) / scale + return h + + # Improve accuracy with a differential correction. + # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 + d = sumprod((x, h), (y, -h)) + return h + d / (2.0 * h) def _normal_dist_inv_cdf(p, mu, sigma): @@ -1365,6 +1791,7 @@ def _normal_dist_inv_cdf(p, mu, sigma): # Normal Distribution". Applied Statistics. Blackwell Publishing. 37 # (3): 477–484. doi:10.2307/2347330. JSTOR 2347330. q = p - 0.5 + if fabs(q) <= 0.425: r = 0.180625 - q * q # Hash sum: 55.88319_28806_14901_4439 @@ -1386,6 +1813,7 @@ def _normal_dist_inv_cdf(p, mu, sigma): 1.0) x = num / den return mu + (x * sigma) + r = p if q <= 0.0 else 1.0 - p r = sqrt(-log(r)) if r <= 5.0: @@ -1426,9 +1854,11 @@ def _normal_dist_inv_cdf(p, mu, sigma): 1.36929_88092_27358_05310e-1) * r + 5.99832_20655_58879_37690e-1) * r + 1.0) + x = num / den if q < 0.0: x = -x + return mu + (x * sigma) @@ -1437,226 +1867,3 @@ def _normal_dist_inv_cdf(p, mu, sigma): from _statistics import _normal_dist_inv_cdf except ImportError: pass - - -class NormalDist: - "Normal distribution of a random variable" - # https://en.wikipedia.org/wiki/Normal_distribution - # https://en.wikipedia.org/wiki/Variance#Properties - - __slots__ = { - '_mu': 'Arithmetic mean of a normal distribution', - '_sigma': 'Standard deviation of a normal distribution', - } - - def __init__(self, mu=0.0, sigma=1.0): - "NormalDist where mu is the mean and sigma is the standard deviation." - if sigma < 0.0: - raise StatisticsError('sigma must be non-negative') - self._mu = float(mu) - self._sigma = float(sigma) - - @classmethod - def from_samples(cls, data): - "Make a normal distribution instance from sample data." - return cls(*_mean_stdev(data)) - - def samples(self, n, *, seed=None): - "Generate *n* samples for a given mean and standard deviation." - rnd = random.random if seed is None else random.Random(seed).random - inv_cdf = _normal_dist_inv_cdf - mu = self._mu - sigma = self._sigma - return [inv_cdf(rnd(), mu, sigma) for _ in repeat(None, n)] - - def pdf(self, x): - "Probability density function. P(x <= X < x+dx) / dx" - variance = self._sigma * self._sigma - if not variance: - raise StatisticsError('pdf() not defined when sigma is zero') - diff = x - self._mu - return exp(diff * diff / (-2.0 * variance)) / sqrt(tau * variance) - - def cdf(self, x): - "Cumulative distribution function. P(X <= x)" - if not self._sigma: - raise StatisticsError('cdf() not defined when sigma is zero') - return 0.5 * (1.0 + erf((x - self._mu) / (self._sigma * _SQRT2))) - - def inv_cdf(self, p): - """Inverse cumulative distribution function. x : P(X <= x) = p - - Finds the value of the random variable such that the probability of - the variable being less than or equal to that value equals the given - probability. - - This function is also called the percent point function or quantile - function. - """ - if p <= 0.0 or p >= 1.0: - raise StatisticsError('p must be in the range 0.0 < p < 1.0') - return _normal_dist_inv_cdf(p, self._mu, self._sigma) - - def quantiles(self, n=4): - """Divide into *n* continuous intervals with equal probability. - - Returns a list of (n - 1) cut points separating the intervals. - - Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. - Set *n* to 100 for percentiles which gives the 99 cuts points that - separate the normal distribution in to 100 equal sized groups. - """ - return [self.inv_cdf(i / n) for i in range(1, n)] - - def overlap(self, other): - """Compute the overlapping coefficient (OVL) between two normal distributions. - - Measures the agreement between two normal probability distributions. - Returns a value between 0.0 and 1.0 giving the overlapping area in - the two underlying probability density functions. - - >>> N1 = NormalDist(2.4, 1.6) - >>> N2 = NormalDist(3.2, 2.0) - >>> N1.overlap(N2) - 0.8035050657330205 - """ - # See: "The overlapping coefficient as a measure of agreement between - # probability distributions and point estimation of the overlap of two - # normal densities" -- Henry F. Inman and Edwin L. Bradley Jr - # http://dx.doi.org/10.1080/03610928908830127 - if not isinstance(other, NormalDist): - raise TypeError('Expected another NormalDist instance') - X, Y = self, other - if (Y._sigma, Y._mu) < (X._sigma, X._mu): # sort to assure commutativity - X, Y = Y, X - X_var, Y_var = X.variance, Y.variance - if not X_var or not Y_var: - raise StatisticsError('overlap() not defined when sigma is zero') - dv = Y_var - X_var - dm = fabs(Y._mu - X._mu) - if not dv: - return 1.0 - erf(dm / (2.0 * X._sigma * _SQRT2)) - a = X._mu * Y_var - Y._mu * X_var - b = X._sigma * Y._sigma * sqrt(dm * dm + dv * log(Y_var / X_var)) - x1 = (a + b) / dv - x2 = (a - b) / dv - return 1.0 - (fabs(Y.cdf(x1) - X.cdf(x1)) + fabs(Y.cdf(x2) - X.cdf(x2))) - - def zscore(self, x): - """Compute the Standard Score. (x - mean) / stdev - - Describes *x* in terms of the number of standard deviations - above or below the mean of the normal distribution. - """ - # https://www.statisticshowto.com/probability-and-statistics/z-score/ - if not self._sigma: - raise StatisticsError('zscore() not defined when sigma is zero') - return (x - self._mu) / self._sigma - - @property - def mean(self): - "Arithmetic mean of the normal distribution." - return self._mu - - @property - def median(self): - "Return the median of the normal distribution" - return self._mu - - @property - def mode(self): - """Return the mode of the normal distribution - - The mode is the value x where which the probability density - function (pdf) takes its maximum value. - """ - return self._mu - - @property - def stdev(self): - "Standard deviation of the normal distribution." - return self._sigma - - @property - def variance(self): - "Square of the standard deviation." - return self._sigma * self._sigma - - def __add__(x1, x2): - """Add a constant or another NormalDist instance. - - If *other* is a constant, translate mu by the constant, - leaving sigma unchanged. - - If *other* is a NormalDist, add both the means and the variances. - Mathematically, this works only if the two distributions are - independent or if they are jointly normally distributed. - """ - if isinstance(x2, NormalDist): - return NormalDist(x1._mu + x2._mu, hypot(x1._sigma, x2._sigma)) - return NormalDist(x1._mu + x2, x1._sigma) - - def __sub__(x1, x2): - """Subtract a constant or another NormalDist instance. - - If *other* is a constant, translate by the constant mu, - leaving sigma unchanged. - - If *other* is a NormalDist, subtract the means and add the variances. - Mathematically, this works only if the two distributions are - independent or if they are jointly normally distributed. - """ - if isinstance(x2, NormalDist): - return NormalDist(x1._mu - x2._mu, hypot(x1._sigma, x2._sigma)) - return NormalDist(x1._mu - x2, x1._sigma) - - def __mul__(x1, x2): - """Multiply both mu and sigma by a constant. - - Used for rescaling, perhaps to change measurement units. - Sigma is scaled with the absolute value of the constant. - """ - return NormalDist(x1._mu * x2, x1._sigma * fabs(x2)) - - def __truediv__(x1, x2): - """Divide both mu and sigma by a constant. - - Used for rescaling, perhaps to change measurement units. - Sigma is scaled with the absolute value of the constant. - """ - return NormalDist(x1._mu / x2, x1._sigma / fabs(x2)) - - def __pos__(x1): - "Return a copy of the instance." - return NormalDist(x1._mu, x1._sigma) - - def __neg__(x1): - "Negates mu while keeping sigma the same." - return NormalDist(-x1._mu, x1._sigma) - - __radd__ = __add__ - - def __rsub__(x1, x2): - "Subtract a NormalDist from a constant or another NormalDist." - return -(x1 - x2) - - __rmul__ = __mul__ - - def __eq__(x1, x2): - "Two NormalDist objects are equal if their mu and sigma are both equal." - if not isinstance(x2, NormalDist): - return NotImplemented - return x1._mu == x2._mu and x1._sigma == x2._sigma - - def __hash__(self): - "NormalDist objects hash equal if their mu and sigma are both equal." - return hash((self._mu, self._sigma)) - - def __repr__(self): - return f'{type(self).__name__}(mu={self._mu!r}, sigma={self._sigma!r})' - - def __getstate__(self): - return self._mu, self._sigma - - def __setstate__(self, state): - self._mu, self._sigma = state diff --git a/Lib/struct.py b/Lib/struct.py index d6bba588636498..ff98e8c4cb3f1d 100644 --- a/Lib/struct.py +++ b/Lib/struct.py @@ -11,5 +11,5 @@ ] from _struct import * -from _struct import _clearcache -from _struct import __doc__ +from _struct import _clearcache # noqa: F401 +from _struct import __doc__ # noqa: F401 diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 20db7747d5db13..2044d2a42897e9 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -43,10 +43,8 @@ import builtins import errno import io -import locale import os import time -import signal import sys import threading import warnings @@ -79,10 +77,11 @@ if _mswindows: import _winapi - from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, + from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, # noqa: F401 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, STD_ERROR_HANDLE, SW_HIDE, STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW, + STARTF_FORCEONFEEDBACK, STARTF_FORCEOFFFEEDBACK, ABOVE_NORMAL_PRIORITY_CLASS, BELOW_NORMAL_PRIORITY_CLASS, HIGH_PRIORITY_CLASS, IDLE_PRIORITY_CLASS, NORMAL_PRIORITY_CLASS, REALTIME_PRIORITY_CLASS, @@ -93,6 +92,7 @@ "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", "STD_ERROR_HANDLE", "SW_HIDE", "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW", + "STARTF_FORCEONFEEDBACK", "STARTF_FORCEOFFFEEDBACK", "STARTUPINFO", "ABOVE_NORMAL_PRIORITY_CLASS", "BELOW_NORMAL_PRIORITY_CLASS", "HIGH_PRIORITY_CLASS", "IDLE_PRIORITY_CLASS", @@ -142,6 +142,8 @@ def __init__(self, returncode, cmd, output=None, stderr=None): def __str__(self): if self.returncode and self.returncode < 0: + # Lazy import to improve module import time + import signal try: return "Command '%s' died with %r." % ( self.cmd, signal.Signals(-self.returncode)) @@ -350,7 +352,7 @@ def _args_from_interpreter_flags(): if dev_mode: args.extend(('-X', 'dev')) for opt in ('faulthandler', 'tracemalloc', 'importtime', - 'frozen_modules', 'showrefcount', 'utf8'): + 'frozen_modules', 'showrefcount', 'utf8', 'gil'): if opt in xoptions: value = xoptions[opt] if value is True: @@ -379,12 +381,14 @@ def _text_encoding(): if sys.flags.utf8_mode: return "utf-8" else: + # Lazy import to improve module import time + import locale return locale.getencoding() def call(*popenargs, timeout=None, **kwargs): """Run command with arguments. Wait for command to complete or - timeout, then return the returncode attribute. + for timeout seconds, then return the returncode attribute. The arguments are the same as for the Popen constructor. Example: @@ -521,8 +525,8 @@ def run(*popenargs, in the returncode attribute, and output & stderr attributes if those streams were captured. - If timeout is given, and the process takes too long, a TimeoutExpired - exception will be raised. + If timeout (seconds) is given and the process takes too long, + a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument @@ -747,7 +751,6 @@ def _use_posix_spawn(): # These are primarily fail-safe knobs for negatives. A True value does not # guarantee the given libc/syscall API will be used. _USE_POSIX_SPAWN = _use_posix_spawn() -_USE_VFORK = True _HAVE_POSIX_SPAWN_CLOSEFROM = hasattr(os, 'POSIX_SPAWN_CLOSEFROM') @@ -839,6 +842,9 @@ def __init__(self, args, bufsize=-1, executable=None, if not isinstance(bufsize, int): raise TypeError("bufsize must be an integer") + if stdout is STDOUT: + raise ValueError("STDOUT can only be used for stderr") + if pipesize is None: pipesize = -1 # Restore default if not isinstance(pipesize, int): @@ -1586,6 +1592,8 @@ def _wait(self, timeout): """Internal implementation of wait() on Windows.""" if timeout is None: timeout_millis = _winapi.INFINITE + elif timeout <= 0: + timeout_millis = 0 else: timeout_millis = int(timeout * 1000) if self.returncode is None: @@ -1658,6 +1666,9 @@ def send_signal(self, sig): # Don't signal a process that we know has already died. if self.returncode is not None: return + + # Lazy import to improve module import time + import signal if sig == signal.SIGTERM: self.terminate() elif sig == signal.CTRL_C_EVENT: @@ -1759,6 +1770,9 @@ def _posix_spawn(self, args, executable, env, restore_signals, close_fds, """Execute program using os.posix_spawn().""" kwargs = {} if restore_signals: + # Lazy import to improve module import time + import signal + # See _Py_RestoreSignals() in Python/pylifecycle.c sigset = [] for signame in ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ'): @@ -1895,7 +1909,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, errpipe_read, errpipe_write, restore_signals, start_new_session, process_group, gid, gids, uid, umask, - preexec_fn, _USE_VFORK) + preexec_fn) self._child_created = True finally: # be sure the FD is closed no matter what @@ -2208,9 +2222,13 @@ def send_signal(self, sig): def terminate(self): """Terminate the process with SIGTERM """ + # Lazy import to improve module import time + import signal self.send_signal(signal.SIGTERM) def kill(self): """Kill the process with SIGKILL """ + # Lazy import to improve module import time + import signal self.send_signal(signal.SIGKILL) diff --git a/Lib/symtable.py b/Lib/symtable.py index 17f820abd56660..7a30e1ac4ca378 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -1,13 +1,21 @@ """Interface to the compiler's internal symbol tables""" import _symtable -from _symtable import (USE, DEF_GLOBAL, DEF_NONLOCAL, DEF_LOCAL, DEF_PARAM, - DEF_IMPORT, DEF_BOUND, DEF_ANNOT, SCOPE_OFF, SCOPE_MASK, FREE, - LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL) +from _symtable import ( + USE, + DEF_GLOBAL, # noqa: F401 + DEF_NONLOCAL, DEF_LOCAL, + DEF_PARAM, DEF_TYPE_PARAM, DEF_FREE_CLASS, + DEF_IMPORT, DEF_BOUND, DEF_ANNOT, + DEF_COMP_ITER, DEF_COMP_CELL, + SCOPE_OFF, SCOPE_MASK, + FREE, LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL +) import weakref +from enum import StrEnum -__all__ = ["symtable", "SymbolTable", "Class", "Function", "Symbol"] +__all__ = ["symtable", "SymbolTableType", "SymbolTable", "Class", "Function", "Symbol"] def symtable(code, filename, compile_type): """ Return the toplevel *SymbolTable* for the source code. @@ -39,6 +47,16 @@ def __call__(self, table, filename): _newSymbolTable = SymbolTableFactory() +class SymbolTableType(StrEnum): + MODULE = "module" + FUNCTION = "function" + CLASS = "class" + ANNOTATION = "annotation" + TYPE_ALIAS = "type alias" + TYPE_PARAMETERS = "type parameters" + TYPE_VARIABLE = "type variable" + + class SymbolTable: def __init__(self, raw_table, filename): @@ -62,23 +80,23 @@ def __repr__(self): def get_type(self): """Return the type of the symbol table. - The values returned are 'class', 'module', 'function', - 'annotation', 'TypeVar bound', 'type alias', and 'type parameter'. + The value returned is one of the values in + the ``SymbolTableType`` enumeration. """ if self._table.type == _symtable.TYPE_MODULE: - return "module" + return SymbolTableType.MODULE if self._table.type == _symtable.TYPE_FUNCTION: - return "function" + return SymbolTableType.FUNCTION if self._table.type == _symtable.TYPE_CLASS: - return "class" + return SymbolTableType.CLASS if self._table.type == _symtable.TYPE_ANNOTATION: - return "annotation" - if self._table.type == _symtable.TYPE_TYPE_VAR_BOUND: - return "TypeVar bound" + return SymbolTableType.ANNOTATION if self._table.type == _symtable.TYPE_TYPE_ALIAS: - return "type alias" - if self._table.type == _symtable.TYPE_TYPE_PARAM: - return "type parameter" + return SymbolTableType.TYPE_ALIAS + if self._table.type == _symtable.TYPE_TYPE_PARAMETERS: + return SymbolTableType.TYPE_PARAMETERS + if self._table.type == _symtable.TYPE_TYPE_VARIABLE: + return SymbolTableType.TYPE_VARIABLE assert False, f"unexpected type: {self._table.type}" def get_id(self): @@ -154,6 +172,10 @@ def get_children(self): for st in self._table.children] +def _get_scope(flags): # like _PyST_GetScope() + return (flags >> SCOPE_OFF) & SCOPE_MASK + + class Function(SymbolTable): # Default values for instance variables @@ -179,7 +201,7 @@ def get_locals(self): """ if self.__locals is None: locs = (LOCAL, CELL) - test = lambda x: ((x >> SCOPE_OFF) & SCOPE_MASK) in locs + test = lambda x: _get_scope(x) in locs self.__locals = self.__idents_matching(test) return self.__locals @@ -188,7 +210,7 @@ def get_globals(self): """ if self.__globals is None: glob = (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT) - test = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) in glob + test = lambda x: _get_scope(x) in glob self.__globals = self.__idents_matching(test) return self.__globals @@ -203,7 +225,7 @@ def get_frees(self): """Return a tuple of free variables in the function. """ if self.__frees is None: - is_free = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) == FREE + is_free = lambda x: _get_scope(x) == FREE self.__frees = self.__idents_matching(is_free) return self.__frees @@ -215,10 +237,45 @@ class Class(SymbolTable): def get_methods(self): """Return a tuple of methods declared in the class. """ + import warnings + typename = f'{self.__class__.__module__}.{self.__class__.__name__}' + warnings.warn(f'{typename}.get_methods() is deprecated ' + f'and will be removed in Python 3.16.', + DeprecationWarning, stacklevel=2) + if self.__methods is None: d = {} + + def is_local_symbol(ident): + flags = self._table.symbols.get(ident, 0) + return ((flags >> SCOPE_OFF) & SCOPE_MASK) == LOCAL + for st in self._table.children: - d[st.name] = 1 + # pick the function-like symbols that are local identifiers + if is_local_symbol(st.name): + match st.type: + case _symtable.TYPE_FUNCTION: + # generators are of type TYPE_FUNCTION with a ".0" + # parameter as a first parameter (which makes them + # distinguishable from a function named 'genexpr') + if st.name == 'genexpr' and '.0' in st.varnames: + continue + d[st.name] = 1 + case _symtable.TYPE_TYPE_PARAMETERS: + # Get the function-def block in the annotation + # scope 'st' with the same identifier, if any. + scope_name = st.name + for c in st.children: + if c.name == scope_name and c.type == _symtable.TYPE_FUNCTION: + # A generic generator of type TYPE_FUNCTION + # cannot be a direct child of 'st' (but it + # can be a descendant), e.g.: + # + # class A: + # type genexpr[genexpr] = (x for x in []) + assert scope_name != 'genexpr' or '.0' not in c.varnames + d[scope_name] = 1 + break self.__methods = tuple(d) return self.__methods @@ -228,7 +285,7 @@ class Symbol: def __init__(self, name, flags, namespaces=None, *, module_scope=False): self.__name = name self.__flags = flags - self.__scope = (flags >> SCOPE_OFF) & SCOPE_MASK # like PyST_GetScope() + self.__scope = _get_scope(flags) self.__namespaces = namespaces or () self.__module_scope = module_scope @@ -253,13 +310,18 @@ def is_referenced(self): """Return *True* if the symbol is used in its block. """ - return bool(self.__flags & _symtable.USE) + return bool(self.__flags & USE) def is_parameter(self): """Return *True* if the symbol is a parameter. """ return bool(self.__flags & DEF_PARAM) + def is_type_parameter(self): + """Return *True* if the symbol is a type parameter. + """ + return bool(self.__flags & DEF_TYPE_PARAM) + def is_global(self): """Return *True* if the symbol is global. """ @@ -292,6 +354,11 @@ def is_free(self): """ return bool(self.__scope == FREE) + def is_free_class(self): + """Return *True* if a class-scoped symbol is free from + the perspective of a method.""" + return bool(self.__flags & DEF_FREE_CLASS) + def is_imported(self): """Return *True* if the symbol is created from an import statement. @@ -302,6 +369,16 @@ def is_assigned(self): """Return *True* if a symbol is assigned to.""" return bool(self.__flags & DEF_LOCAL) + def is_comp_iter(self): + """Return *True* if the symbol is a comprehension iteration variable. + """ + return bool(self.__flags & DEF_COMP_ITER) + + def is_comp_cell(self): + """Return *True* if the symbol is a cell in an inlined comprehension. + """ + return bool(self.__flags & DEF_COMP_CELL) + def is_namespace(self): """Returns *True* if name binding introduces new namespace. diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 07ab27c7fb0c35..69f72452c4069a 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -21,15 +21,16 @@ # Keys for get_config_var() that are never converted to Python integers. _ALWAYS_STR = { + 'IPHONEOS_DEPLOYMENT_TARGET', 'MACOSX_DEPLOYMENT_TARGET', } _INSTALL_SCHEMES = { 'posix_prefix': { - 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}', - 'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}/site-packages', + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages', 'include': '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', 'platinclude': @@ -57,6 +58,7 @@ 'scripts': '{base}/Scripts', 'data': '{base}', }, + # Downstream distributors can overwrite the default install scheme. # This is done to support downstream modifications where distributors change # the installation layout (eg. different site-packages directory). @@ -75,10 +77,10 @@ # Downstream distributors who patch posix_prefix/nt scheme are encouraged to # leave the following schemes unchanged 'posix_venv': { - 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}', - 'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}/site-packages', + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages', 'include': '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', 'platinclude': @@ -114,8 +116,10 @@ def _getuserbase(): if env_base: return env_base - # Emscripten, VxWorks, and WASI have no home directories - if sys.platform in {"emscripten", "vxworks", "wasi"}: + # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories. + # Use _PYTHON_HOST_PLATFORM to get the correct platform when cross-compiling. + system_name = os.environ.get('_PYTHON_HOST_PLATFORM', sys.platform).split('-')[0] + if system_name in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: return None def joinuser(*args): @@ -146,11 +150,11 @@ def joinuser(*args): 'data': '{userbase}', }, 'posix_user': { - 'stdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}', - 'platstdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}', - 'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages', - 'include': '{userbase}/include/{implementation_lower}{py_version_short}', + 'stdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'include': '{userbase}/include/{implementation_lower}{py_version_short}{abi_thread}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, @@ -171,9 +175,7 @@ def joinuser(*args): _PY_VERSION = sys.version.split()[0] _PY_VERSION_SHORT = f'{sys.version_info[0]}.{sys.version_info[1]}' _PY_VERSION_SHORT_NO_DOT = f'{sys.version_info[0]}{sys.version_info[1]}' -_PREFIX = os.path.normpath(sys.prefix) _BASE_PREFIX = os.path.normpath(sys.base_prefix) -_EXEC_PREFIX = os.path.normpath(sys.exec_prefix) _BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) # Mutex guarding initialization of _CONFIG_VARS. _CONFIG_VARS_LOCK = threading.RLock() @@ -220,8 +222,15 @@ def _safe_realpath(path): def is_python_build(check_home=None): if check_home is not None: import warnings - warnings.warn("check_home argument is deprecated and ignored.", - DeprecationWarning, stacklevel=2) + warnings.warn( + ( + 'The check_home argument of sysconfig.is_python_build is ' + 'deprecated and its value is ignored. ' + 'It will be removed in Python 3.15.' + ), + DeprecationWarning, + stacklevel=2, + ) for fn in ("Setup", "Setup.local"): if os.path.isfile(os.path.join(_PROJECT_BASE, "Modules", fn)): return True @@ -290,6 +299,7 @@ def _get_preferred_schemes(): 'home': 'posix_home', 'user': 'osx_framework_user', } + return { 'prefix': 'posix_prefix', 'home': 'posix_home', @@ -315,17 +325,37 @@ def get_default_scheme(): def get_makefile_filename(): """Return the path of the Makefile.""" + + # GH-127429: When cross-compiling, use the Makefile from the target, instead of the host Python. + if cross_base := os.environ.get('_PYTHON_PROJECT_BASE'): + return os.path.join(cross_base, 'Makefile') + if _PYTHON_BUILD: return os.path.join(_PROJECT_BASE, "Makefile") + if hasattr(sys, 'abiflags'): config_dir_name = f'config-{_PY_VERSION_SHORT}{sys.abiflags}' else: config_dir_name = 'config' + if hasattr(sys.implementation, '_multiarch'): config_dir_name += f'-{sys.implementation._multiarch}' + return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile') +def _import_from_directory(path, name): + if name not in sys.modules: + import importlib.machinery + import importlib.util + + spec = importlib.machinery.PathFinder.find_spec(name, [path]) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + sys.modules[name] = module + return sys.modules[name] + + def _get_sysconfigdata_name(): multiarch = getattr(sys.implementation, '_multiarch', '') return os.environ.get( @@ -333,13 +363,34 @@ def _get_sysconfigdata_name(): f'_sysconfigdata_{sys.abiflags}_{sys.platform}_{multiarch}', ) + +def _get_sysconfigdata(): + import importlib + + name = _get_sysconfigdata_name() + path = os.environ.get('_PYTHON_SYSCONFIGDATA_PATH') + module = _import_from_directory(path, name) if path else importlib.import_module(name) + + return module.build_time_vars + + +def _installation_is_relocated(): + """Is the Python installation running from a different prefix than what was targetted when building?""" + if os.name != 'posix': + raise NotImplementedError('sysconfig._installation_is_relocated() is currently only supported on POSIX') + + data = _get_sysconfigdata() + return ( + data['prefix'] != getattr(sys, 'base_prefix', '') + or data['exec_prefix'] != getattr(sys, 'base_exec_prefix', '') + ) + + def _init_posix(vars): """Initialize the module as appropriate for POSIX systems.""" - # _sysconfigdata is generated at build time, see _generate_posix_vars() - name = _get_sysconfigdata_name() - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) - build_time_vars = _temp.build_time_vars - vars.update(build_time_vars) + # GH-126920: Make sure we don't overwrite any of the keys already set + vars.update(_get_sysconfigdata() | vars) + def _init_non_posix(vars): """Initialize the module as appropriate for NT""" @@ -447,27 +498,44 @@ def get_path(name, scheme=get_default_scheme(), vars=None, expand=True): def _init_config_vars(): global _CONFIG_VARS _CONFIG_VARS = {} + + prefix = os.path.normpath(sys.prefix) + exec_prefix = os.path.normpath(sys.exec_prefix) + base_prefix = _BASE_PREFIX + base_exec_prefix = _BASE_EXEC_PREFIX + + try: + abiflags = sys.abiflags + except AttributeError: + abiflags = '' + + if os.name == 'posix': + _init_posix(_CONFIG_VARS) + # If we are cross-compiling, load the prefixes from the Makefile instead. + if '_PYTHON_PROJECT_BASE' in os.environ: + prefix = _CONFIG_VARS['host_prefix'] + exec_prefix = _CONFIG_VARS['host_exec_prefix'] + base_prefix = _CONFIG_VARS['host_prefix'] + base_exec_prefix = _CONFIG_VARS['host_exec_prefix'] + abiflags = _CONFIG_VARS['ABIFLAGS'] + # Normalized versions of prefix and exec_prefix are handy to have; # in fact, these are the standard versions used most places in the # Distutils. - _CONFIG_VARS['prefix'] = _PREFIX - _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX + _CONFIG_VARS['prefix'] = prefix + _CONFIG_VARS['exec_prefix'] = exec_prefix _CONFIG_VARS['py_version'] = _PY_VERSION _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT - _CONFIG_VARS['installed_base'] = _BASE_PREFIX - _CONFIG_VARS['base'] = _PREFIX - _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX - _CONFIG_VARS['platbase'] = _EXEC_PREFIX + _CONFIG_VARS['installed_base'] = base_prefix + _CONFIG_VARS['base'] = prefix + _CONFIG_VARS['installed_platbase'] = base_exec_prefix + _CONFIG_VARS['platbase'] = exec_prefix _CONFIG_VARS['projectbase'] = _PROJECT_BASE _CONFIG_VARS['platlibdir'] = sys.platlibdir _CONFIG_VARS['implementation'] = _get_implementation() _CONFIG_VARS['implementation_lower'] = _get_implementation().lower() - try: - _CONFIG_VARS['abiflags'] = sys.abiflags - except AttributeError: - # sys.abiflags may not be defined on all platforms. - _CONFIG_VARS['abiflags'] = '' + _CONFIG_VARS['abiflags'] = abiflags try: _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '') except AttributeError: @@ -476,14 +544,15 @@ def _init_config_vars(): if os.name == 'nt': _init_non_posix(_CONFIG_VARS) _CONFIG_VARS['VPATH'] = sys._vpath - if os.name == 'posix': - _init_posix(_CONFIG_VARS) if _HAS_USER_BASE: # Setting 'userbase' is done below the call to the # init function to enable using 'get_config_var' in # the init-function. _CONFIG_VARS['userbase'] = _getuserbase() + # e.g., 't' for free-threaded or '' for default build + _CONFIG_VARS['abi_thread'] = 't' if _CONFIG_VARS.get('Py_GIL_DISABLED') else '' + # Always convert srcdir to an absolute path srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE) if os.name == 'posix': @@ -521,9 +590,19 @@ def get_config_vars(*args): With arguments, return a list of values that result from looking up each argument in the configuration variable dictionary. """ + global _CONFIG_VARS_INITIALIZED # Avoid claiming the lock once initialization is complete. - if not _CONFIG_VARS_INITIALIZED: + if _CONFIG_VARS_INITIALIZED: + # GH-126789: If sys.prefix or sys.exec_prefix were updated, invalidate the cache. + prefix = os.path.normpath(sys.prefix) + exec_prefix = os.path.normpath(sys.exec_prefix) + if _CONFIG_VARS['prefix'] != prefix or _CONFIG_VARS['exec_prefix'] != exec_prefix: + with _CONFIG_VARS_LOCK: + _CONFIG_VARS_INITIALIZED = False + _init_config_vars() + else: + # Initialize the config_vars cache. with _CONFIG_VARS_LOCK: # Test again with the lock held to avoid races. Note that # we test _CONFIG_VARS here, not _CONFIG_VARS_INITIALIZED, @@ -565,7 +644,8 @@ def get_platform(): solaris-2.6-sun4u Windows will return one of: - win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) + win-amd64 (64-bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) + win-arm64 (64-bit Windows on ARM64 (aka AArch64) win32 (all others - specifically, sys.platform is returned) For other non-POSIX platforms, currently just returns 'sys.platform'. @@ -598,10 +678,22 @@ def get_platform(): machine = machine.replace('/', '-') if osname[:5] == "linux": - # At least on Linux/Intel, 'machine' is the processor -- - # i386, etc. - # XXX what about Alpha, SPARC, etc? - return f"{osname}-{machine}" + if sys.platform == "android": + osname = "android" + release = get_config_var("ANDROID_API_LEVEL") + + # Wheel tags use the ABI names from Android's own tools. + machine = { + "x86_64": "x86_64", + "i686": "x86", + "aarch64": "arm64_v8a", + "armv7l": "armeabi_v7a", + }[machine] + else: + # At least on Linux/Intel, 'machine' is the processor -- + # i386, etc. + # XXX what about Alpha, SPARC, etc? + return f"{osname}-{machine}" elif osname[:5] == "sunos": if release[0] >= "5": # SunOS 5 == Solaris 2 osname = "solaris" @@ -623,10 +715,15 @@ def get_platform(): if m: release = m.group() elif osname[:6] == "darwin": - import _osx_support - osname, release, machine = _osx_support.get_platform_osx( - get_config_vars(), - osname, release, machine) + if sys.platform == "ios": + release = get_config_vars().get("IPHONEOS_DEPLOYMENT_TARGET", "13.0") + osname = sys.platform + machine = sys.implementation._multiarch + else: + import _osx_support + osname, release, machine = _osx_support.get_platform_osx( + get_config_vars(), + osname, release, machine) return f"{osname}-{release}-{machine}" @@ -635,6 +732,10 @@ def get_python_version(): return _PY_VERSION_SHORT +def _get_python_version_abi(): + return _PY_VERSION_SHORT + get_config_var("abi_thread") + + def expand_makefile_vars(s, vars): """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in 'string' according to 'vars' (a dictionary mapping variable names to @@ -643,8 +744,20 @@ def expand_makefile_vars(s, vars): variable expansions; if 'vars' is the output of 'parse_makefile()', you're fine. Returns a variable-expanded version of 's'. """ + + import warnings + warnings.warn( + 'sysconfig.expand_makefile_vars is deprecated and will be removed in ' + 'Python 3.16. Use sysconfig.get_paths(vars=...) instead.', + DeprecationWarning, + stacklevel=2, + ) + import re + _findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)" + _findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}" + # This algorithm does multiple expansion, so if vars['foo'] contains # "${bar}", it will expand ${foo} to ${bar}, and then expand # ${bar}... and so forth. This is fine as long as 'vars' comes from diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index d7257b9d2d00db..bc2197cfe79402 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -1,10 +1,13 @@ +import json import os import sys +import types from sysconfig import ( _ALWAYS_STR, _PYTHON_BUILD, _get_sysconfigdata_name, get_config_h_filename, + get_config_var, get_config_vars, get_default_scheme, get_makefile_filename, @@ -157,6 +160,19 @@ def _print_config_dict(d, stream): print ("}", file=stream) +def _get_pybuilddir(): + pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' + if get_config_var('Py_DEBUG') == '1': + pybuilddir += '-pydebug' + return pybuilddir + + +def _get_json_data_name(): + name = _get_sysconfigdata_name() + assert name.startswith('_sysconfigdata') + return name.replace('_sysconfigdata', '_sysconfig_vars') + '.json' + + def _generate_posix_vars(): """Generate the Python module containing build-time variables.""" vars = {} @@ -185,6 +201,8 @@ def _generate_posix_vars(): if _PYTHON_BUILD: vars['BLDSHARED'] = vars['LDSHARED'] + name = _get_sysconfigdata_name() + # There's a chicken-and-egg situation on OS X with regards to the # _sysconfigdata module after the changes introduced by #15298: # get_config_vars() is called by get_platform() as part of the @@ -196,16 +214,13 @@ def _generate_posix_vars(): # _sysconfigdata module manually and populate it with the build vars. # This is more than sufficient for ensuring the subsequent call to # get_platform() succeeds. - name = _get_sysconfigdata_name() - if 'darwin' in sys.platform: - import types - module = types.ModuleType(name) - module.build_time_vars = vars - sys.modules[name] = module + # GH-127178: Since we started generating a .json file, we also need this to + # be able to run sysconfig.get_config_vars(). + module = types.ModuleType(name) + module.build_time_vars = vars + sys.modules[name] = module - pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' - if hasattr(sys, "gettotalrefcount"): - pybuilddir += '-pydebug' + pybuilddir = _get_pybuilddir() os.makedirs(pybuilddir, exist_ok=True) destfile = os.path.join(pybuilddir, name + '.py') @@ -215,6 +230,19 @@ def _generate_posix_vars(): f.write('build_time_vars = ') _print_config_dict(vars, stream=f) + print(f'Written {destfile}') + + install_vars = get_config_vars() + # Fix config vars to match the values after install (of the default environment) + install_vars['projectbase'] = install_vars['BINDIR'] + install_vars['srcdir'] = install_vars['LIBPL'] + # Write a JSON file with the output of sysconfig.get_config_vars + jsonfile = os.path.join(pybuilddir, _get_json_data_name()) + with open(jsonfile, 'w') as f: + json.dump(install_vars, f, indent=2) + + print(f'Written {jsonfile}') + # Create file used for sys.path fixup -- see Modules/getpath.c with open('pybuilddir.txt', 'w', encoding='utf8') as f: f.write(pybuilddir) diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py old mode 100755 new mode 100644 index e2ac6837f157d5..c0097351b269f2 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """The Tab Nanny despises ambiguous indentation. She knows no mercy. tabnanny -- Detection of ambiguous indentation @@ -107,14 +105,14 @@ def check(file): errprint("%r: Token Error: %s" % (file, msg)) return - except SyntaxError as msg: - errprint("%r: Token Error: %s" % (file, msg)) - return - except IndentationError as msg: errprint("%r: Indentation Error: %s" % (file, msg)) return + except SyntaxError as msg: + errprint("%r: Syntax Error: %s" % (file, msg)) + return + except NannyNag as nag: badline = nag.get_lineno() line = nag.get_line() diff --git a/Lib/tarfile.py b/Lib/tarfile.py old mode 100755 new mode 100644 index 6f315a6408f185..a0fab46b24e249 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 #------------------------------------------------------------------- # tarfile.py #------------------------------------------------------------------- @@ -636,6 +635,10 @@ def __init__(self, fileobj, offset, size, name, blockinfo=None): def flush(self): pass + @property + def mode(self): + return 'rb' + def readable(self): return True @@ -842,6 +845,9 @@ def data_filter(member, dest_path): # Sentinel for replace() defaults, meaning "don't change the attribute" _KEEP = object() +# Header length is digits followed by a space. +_header_length_prefix_re = re.compile(br"([0-9]{1,20}) ") + class TarInfo(object): """Informational class which holds the details about an archive member given by a tar header block. @@ -1213,7 +1219,7 @@ def _create_pax_generic_header(cls, pax_headers, type, encoding): for keyword, value in pax_headers.items(): keyword = keyword.encode("utf-8") if binary: - # Try to restore the original byte representation of `value'. + # Try to restore the original byte representation of 'value'. # Needless to say, that the encoding must match the string. value = value.encode(encoding, "surrogateescape") else: @@ -1429,37 +1435,59 @@ def _proc_pax(self, tarfile): else: pax_headers = tarfile.pax_headers.copy() - # Check if the pax header contains a hdrcharset field. This tells us - # the encoding of the path, linkpath, uname and gname fields. Normally, - # these fields are UTF-8 encoded but since POSIX.1-2008 tar - # implementations are allowed to store them as raw binary strings if - # the translation to UTF-8 fails. - match = re.search(br"\d+ hdrcharset=([^\n]+)\n", buf) - if match is not None: - pax_headers["hdrcharset"] = match.group(1).decode("utf-8") - - # For the time being, we don't care about anything other than "BINARY". - # The only other value that is currently allowed by the standard is - # "ISO-IR 10646 2000 UTF-8" in other words UTF-8. - hdrcharset = pax_headers.get("hdrcharset") - if hdrcharset == "BINARY": - encoding = tarfile.encoding - else: - encoding = "utf-8" - # Parse pax header information. A record looks like that: # "%d %s=%s\n" % (length, keyword, value). length is the size # of the complete record including the length field itself and - # the newline. keyword and value are both UTF-8 encoded strings. - regex = re.compile(br"(\d+) ([^=]+)=") + # the newline. pos = 0 - while match := regex.match(buf, pos): - length, keyword = match.groups() - length = int(length) - if length == 0: + encoding = None + raw_headers = [] + while len(buf) > pos and buf[pos] != 0x00: + if not (match := _header_length_prefix_re.match(buf, pos)): + raise InvalidHeaderError("invalid header") + try: + length = int(match.group(1)) + except ValueError: + raise InvalidHeaderError("invalid header") + # Headers must be at least 5 bytes, shortest being '5 x=\n'. + # Value is allowed to be empty. + if length < 5: raise InvalidHeaderError("invalid header") - value = buf[match.end(2) + 1:match.start(1) + length - 1] + if pos + length > len(buf): + raise InvalidHeaderError("invalid header") + + header_value_end_offset = match.start(1) + length - 1 # Last byte of the header + keyword_and_value = buf[match.end(1) + 1:header_value_end_offset] + raw_keyword, equals, raw_value = keyword_and_value.partition(b"=") + # Check the framing of the header. The last character must be '\n' (0x0A) + if not raw_keyword or equals != b"=" or buf[header_value_end_offset] != 0x0A: + raise InvalidHeaderError("invalid header") + raw_headers.append((length, raw_keyword, raw_value)) + + # Check if the pax header contains a hdrcharset field. This tells us + # the encoding of the path, linkpath, uname and gname fields. Normally, + # these fields are UTF-8 encoded but since POSIX.1-2008 tar + # implementations are allowed to store them as raw binary strings if + # the translation to UTF-8 fails. For the time being, we don't care about + # anything other than "BINARY". The only other value that is currently + # allowed by the standard is "ISO-IR 10646 2000 UTF-8" in other words UTF-8. + # Note that we only follow the initial 'hdrcharset' setting to preserve + # the initial behavior of the 'tarfile' module. + if raw_keyword == b"hdrcharset" and encoding is None: + if raw_value == b"BINARY": + encoding = tarfile.encoding + else: # This branch ensures only the first 'hdrcharset' header is used. + encoding = "utf-8" + + pos += length + + # If no explicit hdrcharset is set, we use UTF-8 as a default. + if encoding is None: + encoding = "utf-8" + + # After parsing the raw headers we can decode them to text. + for length, raw_keyword, raw_value in raw_headers: # Normally, we could just use "utf-8" as the encoding and "strict" # as the error handler, but we better not take the risk. For # example, GNU tar <= 1.23 is known to store filenames it cannot @@ -1467,17 +1495,16 @@ def _proc_pax(self, tarfile): # hdrcharset=BINARY header). # We first try the strict standard encoding, and if that fails we # fall back on the user's encoding and error handler. - keyword = self._decode_pax_field(keyword, "utf-8", "utf-8", + keyword = self._decode_pax_field(raw_keyword, "utf-8", "utf-8", tarfile.errors) if keyword in PAX_NAME_FIELDS: - value = self._decode_pax_field(value, encoding, tarfile.encoding, + value = self._decode_pax_field(raw_value, encoding, tarfile.encoding, tarfile.errors) else: - value = self._decode_pax_field(value, "utf-8", "utf-8", + value = self._decode_pax_field(raw_value, "utf-8", "utf-8", tarfile.errors) pax_headers[keyword] = value - pos += length # Fetch the next header. try: @@ -1492,7 +1519,7 @@ def _proc_pax(self, tarfile): elif "GNU.sparse.size" in pax_headers: # GNU extended sparse format version 0.0. - self._proc_gnusparse_00(next, pax_headers, buf) + self._proc_gnusparse_00(next, raw_headers) elif pax_headers.get("GNU.sparse.major") == "1" and pax_headers.get("GNU.sparse.minor") == "0": # GNU extended sparse format version 1.0. @@ -1514,15 +1541,24 @@ def _proc_pax(self, tarfile): return next - def _proc_gnusparse_00(self, next, pax_headers, buf): + def _proc_gnusparse_00(self, next, raw_headers): """Process a GNU tar extended sparse header, version 0.0. """ offsets = [] - for match in re.finditer(br"\d+ GNU.sparse.offset=(\d+)\n", buf): - offsets.append(int(match.group(1))) numbytes = [] - for match in re.finditer(br"\d+ GNU.sparse.numbytes=(\d+)\n", buf): - numbytes.append(int(match.group(1))) + for _, keyword, value in raw_headers: + if keyword == b"GNU.sparse.offset": + try: + offsets.append(int(value.decode())) + except ValueError: + raise InvalidHeaderError("invalid header") + + elif keyword == b"GNU.sparse.numbytes": + try: + numbytes.append(int(value.decode())) + except ValueError: + raise InvalidHeaderError("invalid header") + next.sparse = list(zip(offsets, numbytes)) def _proc_gnusparse_01(self, next, pax_headers): @@ -1659,13 +1695,13 @@ def __init__(self, name=None, mode="r", fileobj=None, format=None, tarinfo=None, dereference=None, ignore_zeros=None, encoding=None, errors="surrogateescape", pax_headers=None, debug=None, errorlevel=None, copybufsize=None, stream=False): - """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to + """Open an (uncompressed) tar archive 'name'. 'mode' is either 'r' to read from an existing archive, 'a' to append data to an existing - file or 'w' to create a new file overwriting an existing one. `mode' + file or 'w' to create a new file overwriting an existing one. 'mode' defaults to 'r'. - If `fileobj' is given, it is used for reading or writing data. If it - can be determined, `mode' is overridden by `fileobj's mode. - `fileobj' is not closed, when TarFile is closed. + If 'fileobj' is given, it is used for reading or writing data. If it + can be determined, 'mode' is overridden by 'fileobj's mode. + 'fileobj' is not closed, when TarFile is closed. """ modes = {"r": "rb", "a": "r+b", "w": "wb", "x": "xb"} if mode not in modes: @@ -1724,6 +1760,8 @@ def __init__(self, name=None, mode="r", fileobj=None, format=None, # current position in the archive file self.inodes = {} # dictionary caching the inodes of # archive members already added + self._unames = {} # Cached mappings of uid -> uname + self._gnames = {} # Cached mappings of gid -> gname try: if self.mode == "r": @@ -1994,7 +2032,7 @@ def close(self): self.fileobj.close() def getmember(self, name): - """Return a TarInfo object for member `name'. If `name' can not be + """Return a TarInfo object for member 'name'. If 'name' can not be found in the archive, KeyError is raised. If a member occurs more than once in the archive, its last occurrence is assumed to be the most up-to-date version. @@ -2022,9 +2060,9 @@ def getnames(self): def gettarinfo(self, name=None, arcname=None, fileobj=None): """Create a TarInfo object from the result of os.stat or equivalent - on an existing file. The file is either named by `name', or - specified as a file object `fileobj' with a file descriptor. If - given, `arcname' specifies an alternative name for the file in the + on an existing file. The file is either named by 'name', or + specified as a file object 'fileobj' with a file descriptor. If + given, 'arcname' specifies an alternative name for the file in the archive, otherwise, the name is taken from the 'name' attribute of 'fileobj', or the 'name' argument. The name should be a text string. @@ -2102,16 +2140,23 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): tarinfo.mtime = statres.st_mtime tarinfo.type = type tarinfo.linkname = linkname + + # Calls to pwd.getpwuid() and grp.getgrgid() tend to be expensive. To + # speed things up, cache the resolved usernames and group names. if pwd: - try: - tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0] - except KeyError: - pass + if tarinfo.uid not in self._unames: + try: + self._unames[tarinfo.uid] = pwd.getpwuid(tarinfo.uid)[0] + except KeyError: + self._unames[tarinfo.uid] = '' + tarinfo.uname = self._unames[tarinfo.uid] if grp: - try: - tarinfo.gname = grp.getgrgid(tarinfo.gid)[0] - except KeyError: - pass + if tarinfo.gid not in self._gnames: + try: + self._gnames[tarinfo.gid] = grp.getgrgid(tarinfo.gid)[0] + except KeyError: + self._gnames[tarinfo.gid] = '' + tarinfo.gname = self._gnames[tarinfo.gid] if type in (CHRTYPE, BLKTYPE): if hasattr(os, "major") and hasattr(os, "minor"): @@ -2120,9 +2165,9 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): return tarinfo def list(self, verbose=True, *, members=None): - """Print a table of contents to sys.stdout. If `verbose' is False, only - the names of the members are printed. If it is True, an `ls -l'-like - output is produced. `members' is optional and must be a subset of the + """Print a table of contents to sys.stdout. If 'verbose' is False, only + the names of the members are printed. If it is True, an 'ls -l'-like + output is produced. 'members' is optional and must be a subset of the list returned by getmembers(). """ # Convert tarinfo type to stat type. @@ -2163,11 +2208,11 @@ def list(self, verbose=True, *, members=None): print() def add(self, name, arcname=None, recursive=True, *, filter=None): - """Add the file `name' to the archive. `name' may be any type of file - (directory, fifo, symbolic link, etc.). If given, `arcname' + """Add the file 'name' to the archive. 'name' may be any type of file + (directory, fifo, symbolic link, etc.). If given, 'arcname' specifies an alternative name for the file in the archive. Directories are added recursively by default. This can be avoided by - setting `recursive' to False. `filter' is a function + setting 'recursive' to False. 'filter' is a function that expects a TarInfo object argument and returns the changed TarInfo object, if it returns None the TarInfo object will be excluded from the archive. @@ -2214,13 +2259,16 @@ def add(self, name, arcname=None, recursive=True, *, filter=None): self.addfile(tarinfo) def addfile(self, tarinfo, fileobj=None): - """Add the TarInfo object `tarinfo' to the archive. If `fileobj' is - given, it should be a binary file, and tarinfo.size bytes are read - from it and added to the archive. You can create TarInfo objects - directly, or by using gettarinfo(). + """Add the TarInfo object 'tarinfo' to the archive. If 'tarinfo' represents + a non zero-size regular file, the 'fileobj' argument should be a binary file, + and tarinfo.size bytes are read from it and added to the archive. + You can create TarInfo objects directly, or by using gettarinfo(). """ self._check("awx") + if fileobj is None and tarinfo.isreg() and tarinfo.size != 0: + raise ValueError("fileobj not provided for non zero-size regular file") + tarinfo = copy.copy(tarinfo) buf = tarinfo.tobuf(self.format, self.encoding, self.errors) @@ -2242,13 +2290,7 @@ def _get_filter_function(self, filter): if filter is None: filter = self.extraction_filter if filter is None: - import warnings - warnings.warn( - 'Python 3.14 will, by default, filter extracted tar ' - + 'archives and reject files or modify their metadata. ' - + 'Use the filter argument to control this behavior.', - DeprecationWarning) - return fully_trusted_filter + return data_filter if isinstance(filter, str): raise TypeError( 'String names are not supported for ' @@ -2266,12 +2308,12 @@ def extractall(self, path=".", members=None, *, numeric_owner=False, filter=None): """Extract all members from the archive to the current working directory and set owner, modification time and permissions on - directories afterwards. `path' specifies a different directory - to extract to. `members' is optional and must be a subset of the - list returned by getmembers(). If `numeric_owner` is True, only + directories afterwards. 'path' specifies a different directory + to extract to. 'members' is optional and must be a subset of the + list returned by getmembers(). If 'numeric_owner' is True, only the numbers for user/group names are used and not the names. - The `filter` function will be called on each member just + The 'filter' function will be called on each member just before extraction. It can return a changed TarInfo or None to skip the member. String names of common filters are accepted. @@ -2311,13 +2353,13 @@ def extract(self, member, path="", set_attrs=True, *, numeric_owner=False, filter=None): """Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately - as possible. `member' may be a filename or a TarInfo object. You can - specify a different directory using `path'. File attributes (owner, - mtime, mode) are set unless `set_attrs' is False. If `numeric_owner` + as possible. 'member' may be a filename or a TarInfo object. You can + specify a different directory using 'path'. File attributes (owner, + mtime, mode) are set unless 'set_attrs' is False. If 'numeric_owner' is True, only the numbers for user/group names are used and not the names. - The `filter` function will be called before extraction. + The 'filter' function will be called before extraction. It can return a changed TarInfo or None to skip the member. String names of common filters are accepted. """ @@ -2382,10 +2424,10 @@ def _handle_fatal_error(self, e): self._dbg(1, "tarfile: %s %s" % (type(e).__name__, e)) def extractfile(self, member): - """Extract a member from the archive as a file object. `member' may be - a filename or a TarInfo object. If `member' is a regular file or + """Extract a member from the archive as a file object. 'member' may be + a filename or a TarInfo object. If 'member' is a regular file or a link, an io.BufferedReader object is returned. For all other - existing members, None is returned. If `member' does not appear + existing members, None is returned. If 'member' does not appear in the archive, KeyError is raised. """ self._check("r") @@ -2583,7 +2625,7 @@ def chown(self, tarinfo, targetpath, numeric_owner): else: os.chown(targetpath, u, g) except (OSError, OverflowError) as e: - # OverflowError can be raised if an ID doesn't fit in `id_t` + # OverflowError can be raised if an ID doesn't fit in 'id_t' raise ExtractError("could not change owner") from e def chmod(self, tarinfo, targetpath): diff --git a/Lib/tempfile.py b/Lib/tempfile.py index b5a15f7b72c872..0eb9ddeb6ac377 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -437,11 +437,19 @@ class _TemporaryFileCloser: cleanup_called = False close_called = False - def __init__(self, file, name, delete=True, delete_on_close=True): + def __init__( + self, + file, + name, + delete=True, + delete_on_close=True, + warn_message="Implicitly cleaning up unknown file", + ): self.file = file self.name = name self.delete = delete self.delete_on_close = delete_on_close + self.warn_message = warn_message def cleanup(self, windows=(_os.name == 'nt'), unlink=_os.unlink): if not self.cleanup_called: @@ -469,7 +477,10 @@ def close(self): self.cleanup() def __del__(self): + close_called = self.close_called self.cleanup() + if not close_called: + _warnings.warn(self.warn_message, ResourceWarning) class _TemporaryFileWrapper: @@ -483,8 +494,17 @@ class _TemporaryFileWrapper: def __init__(self, file, name, delete=True, delete_on_close=True): self.file = file self.name = name - self._closer = _TemporaryFileCloser(file, name, delete, - delete_on_close) + self._closer = _TemporaryFileCloser( + file, + name, + delete, + delete_on_close, + warn_message=f"Implicitly cleaning up {self!r}", + ) + + def __repr__(self): + file = self.__dict__['file'] + return f"<{type(self).__name__} {file=}>" def __getattr__(self, name): # Attribute lookups are delegated to the underlying file diff --git a/Lib/test/_test_eintr.py b/Lib/test/_test_eintr.py index 15586f15dfab30..0ce42276bfe3d6 100644 --- a/Lib/test/_test_eintr.py +++ b/Lib/test/_test_eintr.py @@ -18,6 +18,7 @@ import socket import subprocess import sys +import textwrap import time import unittest @@ -90,7 +91,7 @@ class OSEINTRTest(EINTRBaseTest): """ EINTR tests for the os module. """ def new_sleep_process(self): - code = 'import time; time.sleep(%r)' % self.sleep_time + code = f'import time; time.sleep({self.sleep_time!r})' return self.subprocess(code) def _test_wait_multiple(self, wait_func): @@ -122,35 +123,46 @@ def test_waitpid(self): def test_wait4(self): self._test_wait_single(lambda pid: os.wait4(pid, 0)) - def test_read(self): + def _interrupted_reads(self): + """Make a fd which will force block on read of expected bytes.""" rd, wr = os.pipe() self.addCleanup(os.close, rd) # wr closed explicitly by parent # the payload below are smaller than PIPE_BUF, hence the writes will be # atomic - datas = [b"hello", b"world", b"spam"] + data = [b"hello", b"world", b"spam"] code = '\n'.join(( 'import os, sys, time', '', 'wr = int(sys.argv[1])', - 'datas = %r' % datas, - 'sleep_time = %r' % self.sleep_time, + f'data = {data!r}', + f'sleep_time = {self.sleep_time!r}', '', - 'for data in datas:', + 'for item in data:', ' # let the parent block on read()', ' time.sleep(sleep_time)', - ' os.write(wr, data)', + ' os.write(wr, item)', )) proc = self.subprocess(code, str(wr), pass_fds=[wr]) with kill_on_error(proc): os.close(wr) - for data in datas: - self.assertEqual(data, os.read(rd, len(data))) + for datum in data: + yield rd, datum self.assertEqual(proc.wait(), 0) + def test_read(self): + for fd, expected in self._interrupted_reads(): + self.assertEqual(expected, os.read(fd, len(expected))) + + def test_readinto(self): + for fd, expected in self._interrupted_reads(): + buffer = bytearray(len(expected)) + self.assertEqual(os.readinto(fd, buffer), len(expected)) + self.assertEqual(buffer, expected) + def test_write(self): rd, wr = os.pipe() self.addCleanup(os.close, wr) @@ -163,8 +175,8 @@ def test_write(self): 'import io, os, sys, time', '', 'rd = int(sys.argv[1])', - 'sleep_time = %r' % self.sleep_time, - 'data = b"x" * %s' % support.PIPE_MAX_SIZE, + f'sleep_time = {self.sleep_time!r}', + f'data = b"x" * {support.PIPE_MAX_SIZE}', 'data_len = len(data)', '', '# let the parent block on write()', @@ -177,8 +189,8 @@ def test_write(self): '', 'value = read_data.getvalue()', 'if value != data:', - ' raise Exception("read error: %s vs %s bytes"', - ' % (len(value), data_len))', + ' raise Exception(f"read error: {len(value)}' + ' vs {data_len} bytes")', )) proc = self.subprocess(code, str(rd), pass_fds=[rd]) @@ -201,33 +213,33 @@ def _test_recv(self, recv_func): # wr closed explicitly by parent # single-byte payload guard us against partial recv - datas = [b"x", b"y", b"z"] + data = [b"x", b"y", b"z"] code = '\n'.join(( 'import os, socket, sys, time', '', 'fd = int(sys.argv[1])', - 'family = %s' % int(wr.family), - 'sock_type = %s' % int(wr.type), - 'datas = %r' % datas, - 'sleep_time = %r' % self.sleep_time, + f'family = {int(wr.family)}', + f'sock_type = {int(wr.type)}', + f'data = {data!r}', + f'sleep_time = {self.sleep_time!r}', '', 'wr = socket.fromfd(fd, family, sock_type)', 'os.close(fd)', '', 'with wr:', - ' for data in datas:', + ' for item in data:', ' # let the parent block on recv()', ' time.sleep(sleep_time)', - ' wr.sendall(data)', + ' wr.sendall(item)', )) fd = wr.fileno() proc = self.subprocess(code, str(fd), pass_fds=[fd]) with kill_on_error(proc): wr.close() - for data in datas: - self.assertEqual(data, recv_func(rd, len(data))) + for item in data: + self.assertEqual(item, recv_func(rd, len(item))) self.assertEqual(proc.wait(), 0) def test_recv(self): @@ -249,10 +261,10 @@ def _test_send(self, send_func): 'import os, socket, sys, time', '', 'fd = int(sys.argv[1])', - 'family = %s' % int(rd.family), - 'sock_type = %s' % int(rd.type), - 'sleep_time = %r' % self.sleep_time, - 'data = b"xyz" * %s' % (support.SOCK_MAX_SIZE // 3), + f'family = {int(rd.family)}', + f'sock_type = {int(rd.type)}', + f'sleep_time = {self.sleep_time!r}', + f'data = b"xyz" * {support.SOCK_MAX_SIZE // 3}', 'data_len = len(data)', '', 'rd = socket.fromfd(fd, family, sock_type)', @@ -268,8 +280,8 @@ def _test_send(self, send_func): ' n += rd.recv_into(memoryview(received_data)[n:])', '', 'if received_data != data:', - ' raise Exception("recv error: %s vs %s bytes"', - ' % (len(received_data), data_len))', + ' raise Exception(f"recv error: {len(received_data)}' + ' vs {data_len} bytes")', )) fd = rd.fileno() @@ -301,9 +313,9 @@ def test_accept(self): code = '\n'.join(( 'import socket, time', '', - 'host = %r' % socket_helper.HOST, - 'port = %s' % port, - 'sleep_time = %r' % self.sleep_time, + f'host = {socket_helper.HOST!r}', + f'port = {port}', + f'sleep_time = {self.sleep_time!r}', '', '# let parent block on accept()', 'time.sleep(sleep_time)', @@ -331,15 +343,15 @@ def _test_open(self, do_open_close_reader, do_open_close_writer): os_helper.unlink(filename) try: os.mkfifo(filename) - except PermissionError as e: - self.skipTest('os.mkfifo(): %s' % e) + except PermissionError as exc: + self.skipTest(f'os.mkfifo(): {exc!r}') self.addCleanup(os_helper.unlink, filename) code = '\n'.join(( 'import os, time', '', - 'path = %a' % filename, - 'sleep_time = %r' % self.sleep_time, + f'path = {filename!a}', + f'sleep_time = {self.sleep_time!r}', '', '# let the parent block', 'time.sleep(sleep_time)', @@ -395,21 +407,20 @@ class SignalEINTRTest(EINTRBaseTest): def check_sigwait(self, wait_func): signum = signal.SIGUSR1 - pid = os.getpid() old_handler = signal.signal(signum, lambda *args: None) self.addCleanup(signal.signal, signum, old_handler) code = '\n'.join(( 'import os, time', - 'pid = %s' % os.getpid(), - 'signum = %s' % int(signum), - 'sleep_time = %r' % self.sleep_time, + f'pid = {os.getpid()}', + f'signum = {int(signum)}', + f'sleep_time = {self.sleep_time!r}', 'time.sleep(sleep_time)', 'os.kill(pid, signum)', )) - old_mask = signal.pthread_sigmask(signal.SIG_BLOCK, [signum]) + signal.pthread_sigmask(signal.SIG_BLOCK, [signum]) self.addCleanup(signal.pthread_sigmask, signal.SIG_UNBLOCK, [signum]) proc = self.subprocess(code) @@ -492,29 +503,31 @@ def test_devpoll(self): self.check_elapsed_time(dt) -class FNTLEINTRTest(EINTRBaseTest): +class FCNTLEINTRTest(EINTRBaseTest): def _lock(self, lock_func, lock_name): self.addCleanup(os_helper.unlink, os_helper.TESTFN) - code = '\n'.join(( - "import fcntl, time", - "with open('%s', 'wb') as f:" % os_helper.TESTFN, - " fcntl.%s(f, fcntl.LOCK_EX)" % lock_name, - " time.sleep(%s)" % self.sleep_time)) - start_time = time.monotonic() - proc = self.subprocess(code) + rd1, wr1 = os.pipe() + rd2, wr2 = os.pipe() + for fd in (rd1, wr1, rd2, wr2): + self.addCleanup(os.close, fd) + code = textwrap.dedent(f""" + import fcntl, os, time + with open('{os_helper.TESTFN}', 'wb') as f: + fcntl.{lock_name}(f, fcntl.LOCK_EX) + os.write({wr1}, b"ok") + _ = os.read({rd2}, 2) # wait for parent process + time.sleep({self.sleep_time}) + """) + proc = self.subprocess(code, pass_fds=[wr1, rd2]) with kill_on_error(proc): with open(os_helper.TESTFN, 'wb') as f: # synchronize the subprocess + ok = os.read(rd1, 2) + self.assertEqual(ok, b"ok") + + # notify the child that the parent is ready start_time = time.monotonic() - for _ in support.sleeping_retry(support.LONG_TIMEOUT, error=False): - try: - lock_func(f, fcntl.LOCK_EX | fcntl.LOCK_NB) - lock_func(f, fcntl.LOCK_UN) - except BlockingIOError: - break - else: - dt = time.monotonic() - start_time - raise Exception("failed to sync child in %.1f sec" % dt) + os.write(wr2, b"go") # the child locked the file just a moment ago for 'sleep_time' seconds # that means that the lock below will block for 'sleep_time' minus some diff --git a/Lib/test/_test_embed_set_config.py b/Lib/test/_test_embed_set_config.py deleted file mode 100644 index a2ddd133cf47c8..00000000000000 --- a/Lib/test/_test_embed_set_config.py +++ /dev/null @@ -1,263 +0,0 @@ -# bpo-42260: Test _PyInterpreterState_GetConfigCopy() -# and _PyInterpreterState_SetConfig(). -# -# Test run in a subprocess since set_config(get_config()) -# does reset sys attributes to their state of the Python startup -# (before the site module is run). - -import _testinternalcapi -import os -import sys -import unittest -from test.support import MS_WINDOWS - - -MAX_HASH_SEED = 4294967295 - -class SetConfigTests(unittest.TestCase): - def setUp(self): - self.old_config = _testinternalcapi.get_config() - self.sys_copy = dict(sys.__dict__) - - def tearDown(self): - _testinternalcapi.reset_path_config() - _testinternalcapi.set_config(self.old_config) - sys.__dict__.clear() - sys.__dict__.update(self.sys_copy) - - def set_config(self, **kwargs): - _testinternalcapi.set_config(self.old_config | kwargs) - - def check(self, **kwargs): - self.set_config(**kwargs) - for key, value in kwargs.items(): - self.assertEqual(getattr(sys, key), value, - (key, value)) - - def test_set_invalid(self): - invalid_uint = -1 - NULL = None - invalid_wstr = NULL - # PyWideStringList strings must be non-NULL - invalid_wstrlist = ["abc", NULL, "def"] - - type_tests = [] - value_tests = [ - # enum - ('_config_init', 0), - ('_config_init', 4), - # unsigned long - ("hash_seed", -1), - ("hash_seed", MAX_HASH_SEED + 1), - ] - - # int (unsigned) - options = [ - '_config_init', - 'isolated', - 'use_environment', - 'dev_mode', - 'install_signal_handlers', - 'use_hash_seed', - 'faulthandler', - 'tracemalloc', - 'import_time', - 'code_debug_ranges', - 'show_ref_count', - 'dump_refs', - 'malloc_stats', - 'parse_argv', - 'site_import', - 'bytes_warning', - 'inspect', - 'interactive', - 'optimization_level', - 'parser_debug', - 'write_bytecode', - 'verbose', - 'quiet', - 'user_site_directory', - 'configure_c_stdio', - 'buffered_stdio', - 'pathconfig_warnings', - 'module_search_paths_set', - 'skip_source_first_line', - '_install_importlib', - '_init_main', - ] - if MS_WINDOWS: - options.append('legacy_windows_stdio') - for key in options: - value_tests.append((key, invalid_uint)) - type_tests.append((key, "abc")) - type_tests.append((key, 2.0)) - - # wchar_t* - for key in ( - 'filesystem_encoding', - 'filesystem_errors', - 'stdio_encoding', - 'stdio_errors', - 'check_hash_pycs_mode', - 'program_name', - 'platlibdir', - # optional wstr: - # 'pythonpath_env' - # 'home' - # 'pycache_prefix' - # 'run_command' - # 'run_module' - # 'run_filename' - # 'executable' - # 'prefix' - # 'exec_prefix' - # 'base_executable' - # 'base_prefix' - # 'base_exec_prefix' - ): - value_tests.append((key, invalid_wstr)) - type_tests.append((key, b'bytes')) - type_tests.append((key, 123)) - - # PyWideStringList - for key in ( - 'orig_argv', - 'argv', - 'xoptions', - 'warnoptions', - 'module_search_paths', - ): - value_tests.append((key, invalid_wstrlist)) - type_tests.append((key, 123)) - type_tests.append((key, "abc")) - type_tests.append((key, [123])) - type_tests.append((key, [b"bytes"])) - - - if MS_WINDOWS: - value_tests.append(('legacy_windows_stdio', invalid_uint)) - - for exc_type, tests in ( - (ValueError, value_tests), - (TypeError, type_tests), - ): - for key, value in tests: - config = self.old_config | {key: value} - with self.subTest(key=key, value=value, exc_type=exc_type): - with self.assertRaises(exc_type): - _testinternalcapi.set_config(config) - - def test_flags(self): - for sys_attr, key, value in ( - ("debug", "parser_debug", 1), - ("inspect", "inspect", 2), - ("interactive", "interactive", 3), - ("optimize", "optimization_level", 4), - ("verbose", "verbose", 1), - ("bytes_warning", "bytes_warning", 10), - ("quiet", "quiet", 11), - ("isolated", "isolated", 12), - ): - with self.subTest(sys=sys_attr, key=key, value=value): - self.set_config(**{key: value, 'parse_argv': 0}) - self.assertEqual(getattr(sys.flags, sys_attr), value) - - self.set_config(write_bytecode=0) - self.assertEqual(sys.flags.dont_write_bytecode, True) - self.assertEqual(sys.dont_write_bytecode, True) - - self.set_config(write_bytecode=1) - self.assertEqual(sys.flags.dont_write_bytecode, False) - self.assertEqual(sys.dont_write_bytecode, False) - - self.set_config(user_site_directory=0, isolated=0) - self.assertEqual(sys.flags.no_user_site, 1) - self.set_config(user_site_directory=1, isolated=0) - self.assertEqual(sys.flags.no_user_site, 0) - - self.set_config(site_import=0) - self.assertEqual(sys.flags.no_site, 1) - self.set_config(site_import=1) - self.assertEqual(sys.flags.no_site, 0) - - self.set_config(dev_mode=0) - self.assertEqual(sys.flags.dev_mode, False) - self.set_config(dev_mode=1) - self.assertEqual(sys.flags.dev_mode, True) - - self.set_config(use_environment=0, isolated=0) - self.assertEqual(sys.flags.ignore_environment, 1) - self.set_config(use_environment=1, isolated=0) - self.assertEqual(sys.flags.ignore_environment, 0) - - self.set_config(use_hash_seed=1, hash_seed=0) - self.assertEqual(sys.flags.hash_randomization, 0) - self.set_config(use_hash_seed=0, hash_seed=0) - self.assertEqual(sys.flags.hash_randomization, 1) - self.set_config(use_hash_seed=1, hash_seed=123) - self.assertEqual(sys.flags.hash_randomization, 1) - - def test_options(self): - self.check(warnoptions=[]) - self.check(warnoptions=["default", "ignore"]) - - self.set_config(xoptions=[]) - self.assertEqual(sys._xoptions, {}) - self.set_config(xoptions=["dev", "tracemalloc=5"]) - self.assertEqual(sys._xoptions, {"dev": True, "tracemalloc": "5"}) - - def test_pathconfig(self): - self.check( - executable='executable', - prefix="prefix", - base_prefix="base_prefix", - exec_prefix="exec_prefix", - base_exec_prefix="base_exec_prefix", - platlibdir="platlibdir") - - self.set_config(base_executable="base_executable") - self.assertEqual(sys._base_executable, "base_executable") - - # When base_xxx is NULL, value is copied from xxxx - self.set_config( - executable='executable', - prefix="prefix", - exec_prefix="exec_prefix", - base_executable=None, - base_prefix=None, - base_exec_prefix=None) - self.assertEqual(sys._base_executable, "executable") - self.assertEqual(sys.base_prefix, "prefix") - self.assertEqual(sys.base_exec_prefix, "exec_prefix") - - def test_path(self): - self.set_config(module_search_paths_set=1, - module_search_paths=['a', 'b', 'c']) - self.assertEqual(sys.path, ['a', 'b', 'c']) - - # sys.path is reset if module_search_paths_set=0 - self.set_config(module_search_paths_set=0, - module_search_paths=['new_path']) - self.assertNotEqual(sys.path, ['a', 'b', 'c']) - self.assertNotEqual(sys.path, ['new_path']) - - def test_argv(self): - self.set_config(parse_argv=0, - argv=['python_program', 'args'], - orig_argv=['orig', 'orig_args']) - self.assertEqual(sys.argv, ['python_program', 'args']) - self.assertEqual(sys.orig_argv, ['orig', 'orig_args']) - - self.set_config(parse_argv=0, - argv=[], - orig_argv=[]) - self.assertEqual(sys.argv, ['']) - self.assertEqual(sys.orig_argv, []) - - def test_pycache_prefix(self): - self.check(pycache_prefix=None) - self.check(pycache_prefix="pycache_prefix") - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/_test_embed_structseq.py b/Lib/test/_test_embed_structseq.py index 834daa4df55fec..154662efce9412 100644 --- a/Lib/test/_test_embed_structseq.py +++ b/Lib/test/_test_embed_structseq.py @@ -1,31 +1,27 @@ import sys import types +import unittest -# Note: This test file can't import `unittest` since the runtime can't -# currently guarantee that it will not leak memory. Doing so will mark -# the test as passing but with reference leaks. This can safely import -# the `unittest` library once there's a strict guarantee of no leaks -# during runtime shutdown. # bpo-46417: Test that structseq types used by the sys module are still # valid when Py_Finalize()/Py_Initialize() are called multiple times. -class TestStructSeq: +class TestStructSeq(unittest.TestCase): # test PyTypeObject members - def _check_structseq(self, obj_type): + def check_structseq(self, obj_type): # ob_refcnt - assert sys.getrefcount(obj_type) > 1 + self.assertGreaterEqual(sys.getrefcount(obj_type), 1) # tp_base - assert issubclass(obj_type, tuple) + self.assertTrue(issubclass(obj_type, tuple)) # tp_bases - assert obj_type.__bases__ == (tuple,) + self.assertEqual(obj_type.__bases__, (tuple,)) # tp_dict - assert isinstance(obj_type.__dict__, types.MappingProxyType) + self.assertIsInstance(obj_type.__dict__, types.MappingProxyType) # tp_mro - assert obj_type.__mro__ == (obj_type, tuple, object) + self.assertEqual(obj_type.__mro__, (obj_type, tuple, object)) # tp_name - assert isinstance(type.__name__, str) + self.assertIsInstance(type.__name__, str) # tp_subclasses - assert obj_type.__subclasses__() == [] + self.assertEqual(obj_type.__subclasses__(), []) def test_sys_attrs(self): for attr_name in ( @@ -36,23 +32,30 @@ def test_sys_attrs(self): 'thread_info', # ThreadInfoType 'version_info', # VersionInfoType ): - attr = getattr(sys, attr_name) - self._check_structseq(type(attr)) + with self.subTest(attr=attr_name): + attr = getattr(sys, attr_name) + self.check_structseq(type(attr)) def test_sys_funcs(self): func_names = ['get_asyncgen_hooks'] # AsyncGenHooksType if hasattr(sys, 'getwindowsversion'): func_names.append('getwindowsversion') # WindowsVersionType for func_name in func_names: - func = getattr(sys, func_name) - obj = func() - self._check_structseq(type(obj)) + with self.subTest(func=func_name): + func = getattr(sys, func_name) + obj = func() + self.check_structseq(type(obj)) try: - tests = TestStructSeq() - tests.test_sys_attrs() - tests.test_sys_funcs() + unittest.main( + module=( + '__main__' + if __name__ == '__main__' + # Avoiding a circular import: + else sys.modules['test._test_embed_structseq'] + ) + ) except SystemExit as exc: if exc.args[0] != 0: raise diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 058537bab5af26..4b7c3e7fa8bdd7 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -12,17 +12,20 @@ import sys import os import gc +import importlib import errno import functools import signal import array +import collections.abc import socket import random import logging +import shutil import subprocess import struct +import tempfile import operator -import pathlib import pickle import weakref import warnings @@ -255,6 +258,9 @@ def __call__(self, *args, **kwds): class BaseTestCase(object): ALLOWED_TYPES = ('processes', 'manager', 'threads') + # If not empty, limit which start method suites run this class. + START_METHODS: set[str] = set() + start_method = None # set by install_tests_in_module_dict() def assertTimingAlmostEqual(self, a, b): if CHECK_TIMINGS: @@ -313,7 +319,7 @@ def test_current(self): authkey = current.authkey self.assertTrue(current.is_alive()) - self.assertTrue(not current.daemon) + self.assertFalse(current.daemon) self.assertIsInstance(authkey, bytes) self.assertTrue(len(authkey) > 0) self.assertEqual(current.ident, os.getpid()) @@ -324,8 +330,9 @@ def test_set_executable(self): self.skipTest(f'test not appropriate for {self.TYPE}') paths = [ sys.executable, # str - sys.executable.encode(), # bytes - pathlib.Path(sys.executable) # os.PathLike + os.fsencode(sys.executable), # bytes + os_helper.FakePath(sys.executable), # os.PathLike + os_helper.FakePath(os.fsencode(sys.executable)), # os.PathLike bytes ] for path in paths: self.set_executable(path) @@ -456,7 +463,7 @@ def test_process(self): self.assertEqual(p.is_alive(), False) self.assertEqual(p.daemon, True) self.assertNotIn(p, self.active_children()) - self.assertTrue(type(self.active_children()) is list) + self.assertIs(type(self.active_children()), list) self.assertEqual(p.exitcode, None) p.start() @@ -576,8 +583,8 @@ def test_cpu_count(self): cpus = multiprocessing.cpu_count() except NotImplementedError: cpus = 1 - self.assertTrue(type(cpus) is int) - self.assertTrue(cpus >= 1) + self.assertIsInstance(cpus, int) + self.assertGreaterEqual(cpus, 1) def test_active_children(self): self.assertEqual(type(self.active_children()), list) @@ -839,8 +846,8 @@ def test_error_on_stdio_flush_2(self): finally: setattr(sys, stream_name, old_stream) - @classmethod - def _sleep_and_set_event(self, evt, delay=0.0): + @staticmethod + def _sleep_and_set_event(evt, delay=0.0): time.sleep(delay) evt.set() @@ -891,6 +898,56 @@ def test_forkserver_sigkill(self): if os.name != 'nt': self.check_forkserver_death(signal.SIGKILL) + def test_forkserver_auth_is_enabled(self): + if self.TYPE == "threads": + self.skipTest(f"test not appropriate for {self.TYPE}") + if multiprocessing.get_start_method() != "forkserver": + self.skipTest("forkserver start method specific") + + forkserver = multiprocessing.forkserver._forkserver + forkserver.ensure_running() + self.assertTrue(forkserver._forkserver_pid) + authkey = forkserver._forkserver_authkey + self.assertTrue(authkey) + self.assertGreater(len(authkey), 15) + addr = forkserver._forkserver_address + self.assertTrue(addr) + + # Demonstrate that a raw auth handshake, as Client performs, does not + # raise an error. + client = multiprocessing.connection.Client(addr, authkey=authkey) + client.close() + + # That worked, now launch a quick process. + proc = self.Process(target=sys.exit) + proc.start() + proc.join() + self.assertEqual(proc.exitcode, 0) + + def test_forkserver_without_auth_fails(self): + if self.TYPE == "threads": + self.skipTest(f"test not appropriate for {self.TYPE}") + if multiprocessing.get_start_method() != "forkserver": + self.skipTest("forkserver start method specific") + + forkserver = multiprocessing.forkserver._forkserver + forkserver.ensure_running() + self.assertTrue(forkserver._forkserver_pid) + authkey_len = len(forkserver._forkserver_authkey) + with unittest.mock.patch.object( + forkserver, '_forkserver_authkey', None): + # With an incorrect authkey we should get an auth rejection + # rather than the above protocol error. + forkserver._forkserver_authkey = b'T' * authkey_len + proc = self.Process(target=sys.exit) + with self.assertRaises(multiprocessing.AuthenticationError): + proc.start() + del proc + + # authkey restored, launching processes should work again. + proc = self.Process(target=sys.exit) + proc.start() + proc.join() # # @@ -1332,6 +1389,23 @@ def _on_queue_feeder_error(e, obj): self.assertTrue(not_serializable_obj.reduce_was_called) self.assertTrue(not_serializable_obj.on_queue_feeder_error_was_called) + def test_closed_queue_empty_exceptions(self): + # Assert that checking the emptiness of an unused closed queue + # does not raise an OSError. The rationale is that q.close() is + # a no-op upon construction and becomes effective once the queue + # has been used (e.g., by calling q.put()). + for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): + q.close() # this is a no-op since the feeder thread is None + q.join_thread() # this is also a no-op + self.assertTrue(q.empty()) + + for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): + q.put('foo') # make sure that the queue is 'used' + q.close() # close the feeder thread + q.join_thread() # make sure to join the feeder thread + with self.assertRaisesRegex(OSError, 'is closed'): + q.empty() + def test_closed_queue_put_get_exceptions(self): for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): q.close() @@ -1345,6 +1419,66 @@ def test_closed_queue_put_get_exceptions(self): class _TestLock(BaseTestCase): + @staticmethod + def _acquire(lock, l=None): + lock.acquire() + if l is not None: + l.append(repr(lock)) + + @staticmethod + def _acquire_event(lock, event): + lock.acquire() + event.set() + time.sleep(1.0) + + def test_repr_lock(self): + if self.TYPE != 'processes': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + lock = self.Lock() + self.assertEqual(f'', repr(lock)) + + lock.acquire() + self.assertEqual(f'', repr(lock)) + lock.release() + + tname = 'T1' + l = [] + t = threading.Thread(target=self._acquire, + args=(lock, l), + name=tname) + t.start() + time.sleep(0.1) + self.assertEqual(f'', l[0]) + lock.release() + + t = threading.Thread(target=self._acquire, + args=(lock,), + name=tname) + t.start() + time.sleep(0.1) + self.assertEqual('', repr(lock)) + lock.release() + + pname = 'P1' + l = multiprocessing.Manager().list() + p = self.Process(target=self._acquire, + args=(lock, l), + name=pname) + p.start() + p.join() + self.assertEqual(f'', l[0]) + + lock = self.Lock() + event = self.Event() + p = self.Process(target=self._acquire_event, + args=(lock, event), + name='P2') + p.start() + event.wait() + self.assertEqual(f'', repr(lock)) + p.terminate() + def test_lock(self): lock = self.Lock() self.assertEqual(lock.acquire(), True) @@ -1352,6 +1486,68 @@ def test_lock(self): self.assertEqual(lock.release(), None) self.assertRaises((ValueError, threading.ThreadError), lock.release) + @staticmethod + def _acquire_release(lock, timeout, l=None, n=1): + for _ in range(n): + lock.acquire() + if l is not None: + l.append(repr(lock)) + time.sleep(timeout) + for _ in range(n): + lock.release() + + def test_repr_rlock(self): + if self.TYPE != 'processes': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + lock = self.RLock() + self.assertEqual('', repr(lock)) + + n = 3 + for _ in range(n): + lock.acquire() + self.assertEqual(f'', repr(lock)) + for _ in range(n): + lock.release() + + t, l = [], [] + for i in range(n): + t.append(threading.Thread(target=self._acquire_release, + args=(lock, 0.1, l, i+1), + name=f'T{i+1}')) + t[-1].start() + for t_ in t: + t_.join() + for i in range(n): + self.assertIn(f'', l) + + + t = threading.Thread(target=self._acquire_release, + args=(lock, 0.2), + name=f'T1') + t.start() + time.sleep(0.1) + self.assertEqual('', repr(lock)) + time.sleep(0.2) + + pname = 'P1' + l = multiprocessing.Manager().list() + p = self.Process(target=self._acquire_release, + args=(lock, 0.1, l), + name=pname) + p.start() + p.join() + self.assertEqual(f'', l[0]) + + event = self.Event() + lock = self.RLock() + p = self.Process(target=self._acquire_event, + args=(lock, event)) + p.start() + event.wait() + self.assertEqual('', repr(lock)) + p.join() + def test_rlock(self): lock = self.RLock() self.assertEqual(lock.acquire(), True) @@ -2186,14 +2382,14 @@ def test_getobj_getlock(self): self.assertEqual(lock, lock3) arr4 = self.Value('i', 5, lock=False) - self.assertFalse(hasattr(arr4, 'get_lock')) - self.assertFalse(hasattr(arr4, 'get_obj')) + self.assertNotHasAttr(arr4, 'get_lock') + self.assertNotHasAttr(arr4, 'get_obj') self.assertRaises(AttributeError, self.Value, 'i', 5, lock='navalue') arr5 = self.RawValue('i', 5) - self.assertFalse(hasattr(arr5, 'get_lock')) - self.assertFalse(hasattr(arr5, 'get_obj')) + self.assertNotHasAttr(arr5, 'get_lock') + self.assertNotHasAttr(arr5, 'get_obj') class _TestArray(BaseTestCase): @@ -2266,14 +2462,14 @@ def test_getobj_getlock_obj(self): self.assertEqual(lock, lock3) arr4 = self.Array('i', range(10), lock=False) - self.assertFalse(hasattr(arr4, 'get_lock')) - self.assertFalse(hasattr(arr4, 'get_obj')) + self.assertNotHasAttr(arr4, 'get_lock') + self.assertNotHasAttr(arr4, 'get_obj') self.assertRaises(AttributeError, self.Array, 'i', range(10), lock='notalock') arr5 = self.RawArray('i', range(10)) - self.assertFalse(hasattr(arr5, 'get_lock')) - self.assertFalse(hasattr(arr5, 'get_obj')) + self.assertNotHasAttr(arr5, 'get_lock') + self.assertNotHasAttr(arr5, 'get_obj') # # @@ -2314,6 +2510,23 @@ def test_list(self): a.append('hello') self.assertEqual(f[0][:], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 'hello']) + def test_list_isinstance(self): + a = self.list() + self.assertIsInstance(a, collections.abc.MutableSequence) + + # MutableSequence also has __iter__, but we can iterate over + # ListProxy using __getitem__ instead. Adding __iter__ to ListProxy + # would change the behavior of a list modified during iteration. + mutable_sequence_methods = ( + '__contains__', '__delitem__', '__getitem__', '__iadd__', + '__len__', '__reversed__', '__setitem__', 'append', + 'clear', 'count', 'extend', 'index', 'insert', 'pop', 'remove', + 'reverse', + ) + for name in mutable_sequence_methods: + with self.subTest(name=name): + self.assertTrue(callable(getattr(a, name))) + def test_list_iter(self): a = self.list(list(range(10))) it = iter(a) @@ -2354,6 +2567,19 @@ def test_dict(self): self.assertEqual(sorted(d.values()), [chr(i) for i in indices]) self.assertEqual(sorted(d.items()), [(i, chr(i)) for i in indices]) + def test_dict_isinstance(self): + a = self.dict() + self.assertIsInstance(a, collections.abc.MutableMapping) + + mutable_mapping_methods = ( + '__contains__', '__delitem__', '__eq__', '__getitem__', '__iter__', + '__len__', '__ne__', '__setitem__', 'clear', 'get', 'items', + 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values', + ) + for name in mutable_mapping_methods: + with self.subTest(name=name): + self.assertTrue(callable(getattr(a, name))) + def test_dict_iter(self): d = self.dict() indices = list(range(65, 70)) @@ -2431,8 +2657,8 @@ def test_namespace(self): self.assertEqual((n.name, n.job), ('Bob', 'Builder')) del n.job self.assertEqual(str(n), "Namespace(name='Bob')") - self.assertTrue(hasattr(n, 'name')) - self.assertTrue(not hasattr(n, 'job')) + self.assertHasAttr(n, 'name') + self.assertNotHasAttr(n, 'job') # # @@ -2812,8 +3038,8 @@ def test_release_task_refs(self): self.pool.map(identity, objs) del objs - gc.collect() # For PyPy or other GCs. time.sleep(DELTA) # let threaded cleanup code run + support.gc_collect() # For PyPy or other GCs. self.assertEqual(set(wr() for wr in refs), {None}) # With a process pool, copies of the objects are returned, check # they were released too. @@ -3513,15 +3739,20 @@ def run(addr, authkey): client = self.connection.Client(addr, authkey=authkey) client.send(1729) - key = b"" + key = b'' with self.connection.Listener(authkey=key) as listener: - threading.Thread(target=run, args=(listener.address, key)).start() - with listener.accept() as d: - self.assertEqual(d.recv(), 1729) + thread = threading.Thread(target=run, args=(listener.address, key)) + thread.start() + try: + with listener.accept() as d: + self.assertEqual(d.recv(), 1729) + finally: + thread.join() if self.TYPE == 'processes': - self.assertRaises(OSError, listener.accept) + with self.assertRaises(OSError): + listener.accept() @unittest.skipUnless(util.abstract_sockets_supported, "test needs abstract socket support") @@ -4657,7 +4888,7 @@ def make_finalizers(): old_interval = sys.getswitchinterval() old_threshold = gc.get_threshold() try: - sys.setswitchinterval(1e-6) + support.setswitchinterval(1e-6) gc.set_threshold(5, 5, 5) threads = [threading.Thread(target=run_finalizers), threading.Thread(target=make_finalizers)] @@ -4707,13 +4938,9 @@ def test_import(self): for name in modules: __import__(name) mod = sys.modules[name] - self.assertTrue(hasattr(mod, '__all__'), name) - + self.assertHasAttr(mod, '__all__', name) for attr in mod.__all__: - self.assertTrue( - hasattr(mod, attr), - '%r does not have attribute %r' % (mod, attr) - ) + self.assertHasAttr(mod, attr) # # Quick test that logging works -- does not test logging output @@ -4726,7 +4953,7 @@ class _TestLogging(BaseTestCase): def test_enable_logging(self): logger = multiprocessing.get_logger() logger.setLevel(util.SUBWARNING) - self.assertTrue(logger is not None) + self.assertIsNotNone(logger) logger.debug('this will not be printed') logger.info('nor will this') logger.setLevel(LOG_LEVEL) @@ -5522,24 +5749,37 @@ def test_set_get(self): self.assertEqual(multiprocessing.get_start_method(), method) ctx = multiprocessing.get_context() self.assertEqual(ctx.get_start_method(), method) - self.assertTrue(type(ctx).__name__.lower().startswith(method)) - self.assertTrue( - ctx.Process.__name__.lower().startswith(method)) + self.assertStartsWith(type(ctx).__name__.lower(), method) + self.assertStartsWith(ctx.Process.__name__.lower(), method) self.check_context(multiprocessing) count += 1 finally: multiprocessing.set_start_method(old_method, force=True) self.assertGreaterEqual(count, 1) - def test_get_all(self): + def test_get_all_start_methods(self): methods = multiprocessing.get_all_start_methods() + self.assertIn('spawn', methods) if sys.platform == 'win32': self.assertEqual(methods, ['spawn']) + elif sys.platform == 'darwin': + self.assertEqual(methods[0], 'spawn') # The default is first. + # Whether these work or not, they remain available on macOS. + self.assertIn('fork', methods) + self.assertIn('forkserver', methods) else: - self.assertTrue(methods == ['fork', 'spawn'] or - methods == ['spawn', 'fork'] or - methods == ['fork', 'spawn', 'forkserver'] or - methods == ['spawn', 'fork', 'forkserver']) + # POSIX + self.assertIn('fork', methods) + if other_methods := set(methods) - {'fork', 'spawn'}: + # If there are more than those two, forkserver must be one. + self.assertEqual({'forkserver'}, other_methods) + # The default is the first method in the list. + self.assertIn(methods[0], {'forkserver', 'spawn'}, + msg='3.14+ default must not be fork') + if methods[0] == 'spawn': + # Confirm that the current default selection logic prefers + # forkserver vs spawn when available. + self.assertNotIn('forkserver', methods) def test_preload_resources(self): if multiprocessing.get_start_method() != 'forkserver': @@ -5711,9 +5951,9 @@ def check_resource_tracker_death(self, signum, should_die): if should_die: self.assertEqual(len(all_warn), 1) the_warn = all_warn[0] - self.assertTrue(issubclass(the_warn.category, UserWarning)) - self.assertTrue("resource_tracker: process died" - in str(the_warn.message)) + self.assertIsSubclass(the_warn.category, UserWarning) + self.assertIn("resource_tracker: process died", + str(the_warn.message)) else: self.assertEqual(len(all_warn), 0) @@ -5725,6 +5965,8 @@ def test_resource_tracker_sigterm(self): # Catchable signal (ignored by semaphore tracker) self.check_resource_tracker_death(signal.SIGTERM, False) + @unittest.skipIf(sys.platform.startswith("netbsd"), + "gh-125620: Skip on NetBSD due to long wait for SIGKILL process termination.") def test_resource_tracker_sigkill(self): # Uncatchable signal. self.check_resource_tracker_death(signal.SIGKILL, True) @@ -5798,6 +6040,27 @@ def test_resource_tracker_exit_code(self): cleanup=cleanup, ) + @unittest.skipUnless(hasattr(signal, "pthread_sigmask"), "pthread_sigmask is not available") + def test_resource_tracker_blocked_signals(self): + # + # gh-127586: Check that resource_tracker does not override blocked signals of caller. + # + from multiprocessing.resource_tracker import ResourceTracker + orig_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, set()) + signals = {signal.SIGTERM, signal.SIGINT, signal.SIGUSR1} + + try: + for sig in signals: + signal.pthread_sigmask(signal.SIG_SETMASK, {sig}) + self.assertEqual(signal.pthread_sigmask(signal.SIG_BLOCK, set()), {sig}) + tracker = ResourceTracker() + tracker.ensure_running() + self.assertEqual(signal.pthread_sigmask(signal.SIG_BLOCK, set()), {sig}) + tracker._stop() + finally: + # restore sigmask to what it was before executing test + signal.pthread_sigmask(signal.SIG_SETMASK, orig_sigmask) + class TestSimpleQueue(unittest.TestCase): @classmethod @@ -5810,6 +6073,15 @@ def _test_empty(cls, queue, child_can_start, parent_can_continue): finally: parent_can_continue.set() + def test_empty_exceptions(self): + # Assert that checking emptiness of a closed queue raises + # an OSError, independently of whether the queue was used + # or not. This differs from Queue and JoinableQueue. + q = multiprocessing.SimpleQueue() + q.close() # close the pipe + with self.assertRaisesRegex(OSError, 'is closed'): + q.empty() + def test_empty(self): queue = multiprocessing.SimpleQueue() child_can_start = multiprocessing.Event() @@ -5886,8 +6158,8 @@ def is_alive(self): Process=FailingForkProcess)) p.close() p.join() - self.assertFalse( - any(process.is_alive() for process in forked_processes)) + for process in forked_processes: + self.assertFalse(process.is_alive(), process) @hashlib_helper.requires_hashdigest('sha256') @@ -6059,12 +6331,30 @@ def _test_list(cls, obj): case.assertEqual(obj[0], 5) case.assertEqual(obj.count(5), 1) case.assertEqual(obj.index(5), 0) + obj += [7] + case.assertIsInstance(obj, multiprocessing.managers.ListProxy) + case.assertListEqual(list(obj), [5, 7]) + obj *= 2 + case.assertIsInstance(obj, multiprocessing.managers.ListProxy) + case.assertListEqual(list(obj), [5, 7, 5, 7]) + double_obj = obj * 2 + case.assertIsInstance(double_obj, list) + case.assertListEqual(list(double_obj), [5, 7, 5, 7, 5, 7, 5, 7]) + double_obj = 2 * obj + case.assertIsInstance(double_obj, list) + case.assertListEqual(list(double_obj), [5, 7, 5, 7, 5, 7, 5, 7]) + copied_obj = obj.copy() + case.assertIsInstance(copied_obj, list) + case.assertListEqual(list(copied_obj), [5, 7, 5, 7]) + obj.extend(double_obj + copied_obj) obj.sort() obj.reverse() for x in obj: pass - case.assertEqual(len(obj), 1) - case.assertEqual(obj.pop(0), 5) + case.assertEqual(len(obj), 16) + case.assertEqual(obj.pop(0), 7) + obj.clear() + case.assertEqual(len(obj), 0) def test_list(self): o = self.manager.list() @@ -6083,7 +6373,29 @@ def _test_dict(cls, obj): case.assertListEqual(list(obj.keys()), ['foo']) case.assertListEqual(list(obj.values()), [5]) case.assertDictEqual(obj.copy(), {'foo': 5}) - case.assertTupleEqual(obj.popitem(), ('foo', 5)) + obj |= {'bar': 6} + case.assertIsInstance(obj, multiprocessing.managers.DictProxy) + case.assertDictEqual(dict(obj), {'foo': 5, 'bar': 6}) + x = reversed(obj) + case.assertIsInstance(x, type(iter([]))) + case.assertListEqual(list(x), ['bar', 'foo']) + x = {'bar': 7, 'baz': 7} | obj + case.assertIsInstance(x, dict) + case.assertDictEqual(dict(x), {'foo': 5, 'bar': 6, 'baz': 7}) + x = obj | {'bar': 7, 'baz': 7} + case.assertIsInstance(x, dict) + case.assertDictEqual(dict(x), {'foo': 5, 'bar': 7, 'baz': 7}) + x = obj.fromkeys(['bar'], 6) + case.assertIsInstance(x, dict) + case.assertDictEqual(x, {'bar': 6}) + x = obj.popitem() + case.assertIsInstance(x, tuple) + case.assertTupleEqual(x, ('bar', 6)) + obj.setdefault('bar', 0) + obj.update({'bar': 7}) + case.assertEqual(obj.pop('bar'), 7) + obj.clear() + case.assertEqual(len(obj), 0) def test_dict(self): o = self.manager.dict() @@ -6156,6 +6468,99 @@ def submain(): pass self.assertFalse(err, msg=err.decode('utf-8')) +class _TestAtExit(BaseTestCase): + + ALLOWED_TYPES = ('processes',) + + @classmethod + def _write_file_at_exit(self, output_path): + import atexit + def exit_handler(): + with open(output_path, 'w') as f: + f.write("deadbeef") + atexit.register(exit_handler) + + def test_atexit(self): + # gh-83856 + with os_helper.temp_dir() as temp_dir: + output_path = os.path.join(temp_dir, 'output.txt') + p = self.Process(target=self._write_file_at_exit, args=(output_path,)) + p.start() + p.join() + with open(output_path) as f: + self.assertEqual(f.read(), 'deadbeef') + + +class _TestSpawnedSysPath(BaseTestCase): + """Test that sys.path is setup in forkserver and spawn processes.""" + + ALLOWED_TYPES = {'processes'} + # Not applicable to fork which inherits everything from the process as is. + START_METHODS = {"forkserver", "spawn"} + + def setUp(self): + self._orig_sys_path = list(sys.path) + self._temp_dir = tempfile.mkdtemp(prefix="test_sys_path-") + self._mod_name = "unique_test_mod" + module_path = os.path.join(self._temp_dir, f"{self._mod_name}.py") + with open(module_path, "w", encoding="utf-8") as mod: + mod.write("# A simple test module\n") + sys.path[:] = [p for p in sys.path if p] # remove any existing ""s + sys.path.insert(0, self._temp_dir) + sys.path.insert(0, "") # Replaced with an abspath in child. + self.assertIn(self.start_method, self.START_METHODS) + self._ctx = multiprocessing.get_context(self.start_method) + + def tearDown(self): + sys.path[:] = self._orig_sys_path + shutil.rmtree(self._temp_dir, ignore_errors=True) + + @staticmethod + def enq_imported_module_names(queue): + queue.put(tuple(sys.modules)) + + def test_forkserver_preload_imports_sys_path(self): + if self._ctx.get_start_method() != "forkserver": + self.skipTest("forkserver specific test.") + self.assertNotIn(self._mod_name, sys.modules) + multiprocessing.forkserver._forkserver._stop() # Must be fresh. + self._ctx.set_forkserver_preload( + ["test.test_multiprocessing_forkserver", self._mod_name]) + q = self._ctx.Queue() + proc = self._ctx.Process( + target=self.enq_imported_module_names, args=(q,)) + proc.start() + proc.join() + child_imported_modules = q.get() + q.close() + self.assertIn(self._mod_name, child_imported_modules) + + @staticmethod + def enq_sys_path_and_import(queue, mod_name): + queue.put(sys.path) + try: + importlib.import_module(mod_name) + except ImportError as exc: + queue.put(exc) + else: + queue.put(None) + + def test_child_sys_path(self): + q = self._ctx.Queue() + proc = self._ctx.Process( + target=self.enq_sys_path_and_import, args=(q, self._mod_name)) + proc.start() + proc.join() + child_sys_path = q.get() + import_error = q.get() + q.close() + self.assertNotIn("", child_sys_path) # replaced by an abspath + self.assertIn(self._temp_dir, child_sys_path) # our addition + # ignore the first element, it is the absolute "" replacement + self.assertEqual(child_sys_path[1:], sys.path[1:]) + self.assertIsNone(import_error, msg=f"child could not import {self._mod_name}") + + class MiscTestCase(unittest.TestCase): def test__all__(self): # Just make sure names in not_exported are excluded @@ -6350,6 +6755,8 @@ def install_tests_in_module_dict(remote_globs, start_method, if base is BaseTestCase: continue assert set(base.ALLOWED_TYPES) <= ALL_TYPES, base.ALLOWED_TYPES + if base.START_METHODS and start_method not in base.START_METHODS: + continue # class not intended for this start method. for type_ in base.ALLOWED_TYPES: if only_type and type_ != only_type: continue @@ -6363,6 +6770,7 @@ class Temp(base, Mixin, unittest.TestCase): Temp = hashlib_helper.requires_hashdigest('sha256')(Temp) Temp.__name__ = Temp.__qualname__ = newname Temp.__module__ = __module__ + Temp.start_method = start_method remote_globs[newname] = Temp elif issubclass(base, unittest.TestCase): if only_type: diff --git a/Lib/test/archivetestdata/zipdir_backslash.zip b/Lib/test/archivetestdata/zipdir_backslash.zip new file mode 100644 index 00000000000000..979126ef5e37eb Binary files /dev/null and b/Lib/test/archivetestdata/zipdir_backslash.zip differ diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index de7d0da560a1c7..6b9b21cf7f6a3c 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -187,7 +187,7 @@ class C(A): def test_open(testfn): - # SSLContext.load_dh_params uses _Py_fopen_obj rather than normal open() + # SSLContext.load_dh_params uses Py_fopen() rather than normal open() try: import ssl @@ -556,6 +556,28 @@ def hook(event, args): sys.monitoring.register_callback(1, 1, None) +def test_winapi_createnamedpipe(pipe_name): + import _winapi + + def hook(event, args): + if event == "_winapi.CreateNamedPipe": + print(event, args) + + sys.addaudithook(hook) + _winapi.CreateNamedPipe(pipe_name, _winapi.PIPE_ACCESS_DUPLEX, 8, 2, 0, 0, 0, 0) + + +def test_assert_unicode(): + import sys + sys.addaudithook(lambda *args: None) + try: + sys.audit(9) + except TypeError: + pass + else: + raise RuntimeError("Expected sys.audit(9) to fail.") + + if __name__ == "__main__": from test.support import suppress_msvcrt_asserts diff --git a/Lib/test/certdata/allsans.pem b/Lib/test/certdata/allsans.pem index e400e178a1f599..f6a63b9219d246 100644 --- a/Lib/test/certdata/allsans.pem +++ b/Lib/test/certdata/allsans.pem @@ -1,42 +1,42 @@ -----BEGIN PRIVATE KEY----- -MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDBGvj+Uy/VUyTR -mmIA1UEENThh0+pWODcvvUlkeIo+XTJ3FhF4/RVjImDHjozl28Xf2TzKnvQJa1KC -pqa7fr8cL9QMwk4pH+S4ulxOu02Bl3Yafx2oJVUML37vciJg+zkzPx1k3tXFjXkr -LGjZwOoufBC3AmPuq2xHFBzHrvp5/DIRH2slQFM9fpVZzN77gYyzxba0wCfCPpCf -eJFRyYKW8c7MXrwnM82YtE7Rlnf227EkCdMNaSeZLUIxeVpcnScqZl0SIbR3YEiV -0LPFkx0wJFm8qUEFU/h+0jamgy/ON+11nqmMlp3BjNi/JTVsa7N7A3dvdHC7VVlr -WnUgU6MoSniyL6ijpucyHtZzK2mJy0sHR8PadHKow0O423/5N8GKTSOvaGMXTjAe -OGs+9/P1ZYo3IjjQPz/NV3QlhK8zRqxF3cW0ekHHkT+/jZjCvSKm6mdbMQunKE1W -+dokAc815pb48Mzf1eWKd/7UyUf7CXussyAaJ3clpaK1sbbn9m0CAwEAAQKCAYAe -BaCCgdJk+xk1USg9cuo5ykBqzTSYlQLXdDlN2oO7sGehJhgvVEGX+QdM3ze+oM2B -wNd3tQDB2iKo11oCunDh4/m2xhq6wA+iPK8POoWRSUf+VJb6xlsTmurENV1s8IHz -GrPqM87OePFGqg/fEuQVuAotObzppVMfNdxHm0er4W6zRMw2rWqDnAOCQ5zDQ1/p -ryp5rYpA49M+R9NoAMlByHRbR7s+6Qnk3NuIMDmUcpF2xeQ/KIMUiHnLEU/gKDpi -bsk+VtyjlibR4zhh9/cJrLTApAIA+4eC176EJvKXCh5UIjd92JC7741HTNQXJpvG -9PXbzhyUCmncr04U+46snGHdwD+lG4LS7oBGACTLMtpcMrlgAm6XCg4T8gRVE/9n -FvCkqPHBR+vnhOxm+0x0yUY/DstJby6IPYPsfGK/s2n//j/vJrAZE1Pxlm9EPU13 -MRLcHstwjAc/NXRPnUN1DfcQvPLx6Tt6rqw3Wm1KO75kM+HZ56BX9/Bi1TgkiI0C -gcEA5JTlXssJ3W8Cz6w1ZtGsThHQBDbvHF2D5AdqO7y6/eqzCQgBQl9BTfXOzsvP -I1gf2CLEFBtGK09UjAuJQg90/NlKur7i7xt7HpAzEfGsDAL4P5BW5JnMNrzpJjjL -0uUDsPJlA75Wi29N2SFiaIslY0sZ6nckInat5GRe4O1AMSHoJ5suY9yTZTU3XB4O -A+XyddutI1GsFZgl8/8LyyNMcyNjxG3T5sr7IKf5/nIv6oMDjC2zLVZa8QS/MEnL -Kaa7AoHBANhEsxfcjw2MaPkrsqAsOP0dDf7g2rdz6wKT5BzZu9e+/E76NmvVDpns -e+kCjql9Os3/wonOMINvn1bTCQGTgk8+dw1fMyqg+zQCvH4ImcE6LSqhzblVHsIB -zZ7rW86trri1U9+olNHG4nwkus0i4LV8eeORns+j8DgXr6/eOvjX3ZW5TyU7/Qgm -SiSdBapzJbom3xJrbo9KQsrN5PVCOwuwrgY0o+2BeKyKhnt4uGv0bR+ii06EOJUA -WvjD7gLI9wKBwGVRXk3jH29IOm3EvjLh80bzfEmx89CV3tUfOEZcRGIyOsNhCfXa -dP7SWqWtDxZyhELwPgtPf43I7wfYQTHH2ioNQqN94ubrPmpwrkJg5cq5MkIyf2F6 -jlsg5xMrD6VeH4G6H25GWuQZJN9+fbkrHBpj+ovD3X9tLWzT1H5Miyx8BAQyM6DN -74Nn0C8Dn2C49vyor5i9JdK4ivIY9ahH8CYE5L73k3p0NFXoPtY61ORUyCjFROtu -oIa+fOQxgVzn6wKBwQC3DD7BnY7/Gq7m51ODOqrpoaPs7Qhyagyp298hhDD3hNEt -T56sWmLHaV/fcqipUDNrlGRmGzz4ooutA2YGDYIn7Gj7ym4WULcN6Jr92e25nLIJ -+XWUvjUQZFJThkXogxz1fZSGI7wCamHcTYJGipTDR54rPV+7w7hY4cN0CZbEdIE6 -buRMUZ/zO+VZZAYdpORz0N7SSlgDtAkgenCmHe64EEzbN8bgCcvHzl/RNfZyeSm7 -supSBJuXkfttvvg/JzUCgcEAlx0Pep9qCLvpk0WqzijBVHc3zK4wYxjhN2MBkF42 -SLWfogKpiPfIqxX6YF94roIA0VlW6Pj50v+sbPwq8nwsgFNhml80A4ODKr3O3Y3M -fXDBJW5W5ZRb/vhIKRjXyCSckSRfj7N8HUYjCLkxQansNWimrldmSet0H2mYJN0Y -JpBXdqpa76zoHzWpKFwD0fSVzvnMelPHSDCNOdIEHmR8e1x2F1/ufR/9/dBzPULY -HMj0OhQHoi8kJyMIj3+bQkbC +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQCYEY+lIwS/w+gA +U5PfxSeL4qMF5TtFkrifL0oQaCjWFb8jeX2Rb1tKaObyrYmTURVWbMkI1LJOQWYt +GlANo2mvf4VD1wZZ/x/Tmylxe3A7itSTZgZuGXLHtFP9wk7yS46BlVcNq3ZoOi7k +k8NxTiUqgsmLg3Nj+Ifk8OI7c1BcP2pa1wZO/l3PTWqtFSLsFEvOK5N6ZPqILAUi +5vEkOiFefxVHHvD4N3IWSFgSwiDi9KhntdRwa+axsZxjq/o70FjQO5q5Quym9aDq +qo47PMykEb1gdTBQK1mPliMKsLi5zycI/ypm1PEcH53+9/F+U2B1bmvA/9PBx8T7 +hUyclBbQHCmdcXIYCgFr5AbGS3jFeVPiF7hCPj1k/jcpayXZAtWmqJTlkXKGbM68 +AgM/DHO0fGCYGRUw2vhV3ZIRCm0BqwforeFh+2oMyZjDjvo/uZjZNazdpcwvOy0c +lFkORwVI0zTIixXYSV4xN1BX821JCuhYubro89m+MSjxO14qgs0CAwEAAQKCAYAZ +txqF7+qNLSlN6NRASnw6BQzRYebIiJus25fYN2z0awAEFTbdQan75sprLlpt7Y/A +qivC0QkH/7oyFVmFPOWR1mwoQTPjQyfUJlB3Tsr3Xq488MtUkfwddkqfxlyIT6ud +ES6D8sNWs8QbRjuOLQtO6YgAji2UscH1PqDbMdBckSLAks0Pzab6d9p6w3DA4FvD +VQ4e6/WL0nnZ4ZjUqfnbm3zzJnHUX7fsubYfEfHyvzG9O/vdOPntgZ3zIvFxbPVc +/Dp8MRNgfQoRgAsc7VBQQifFNvva9SjvRsXwE7zQLQh9JszFpHWqIMevGwjdH8dO +1hHESqkRHL0sYwsqoGqbaHFrJVEA2Ljuxs/UsH6G2U6NuDVaYkqM8UwH0+EqGFtF +9qP8oh6Yvp+szI5yET4brW0Rlk8r2UqvQ5iSSl8SSWsNKR2wRs0PM0aNm13TDJmt +NOuxd/0ranG+NvBMdtLCwsoEplYDaaczYv9gbKHT64OlFAdGBkOytbS7eqOojmcC +gcEA0uQb9bPKbiMYSEJMAalOlruzvMiHBRW420IEuGyHJtocn/RK804Jj6gzu+WD +GT81rB9NXAt/3s6WDMSdrGrrBh/CQZJs50ok9hej0f6gsq1JNsWJ/c10rU/mqrTn +s++pdaRqbd+bgM/N6h79tJi3j4SgB9rJqjC+7qWlDJFWZOoDR59elc10xDA0AE0a +K6UvsJJGk4/v715K1Uh2qU5HYupuijNc0m3z2OMZ6zxTNNUdbSL4XF5X9e2+BwMQ +YrRzAoHBALiYeYWsmHGRdg++/p/9SE86YwyEYr+zrUiWenIPC1W5WQlkdFQqALmg +kjcXAXFFouC/DnKG65YOfGq772ekFuw06Lm4DEotQP02pESfAP9ulVufK2yJtbxi +m0PeN3JRrs/H72VW9hUSxCafNRyBeZ6nsfzEfEkPOdc+lRGeMHT9C7r7Y1asNk8A +Zl7wKZgv+Yvn8xhHWQj5z/T6K2z5CXFn6FERc0qV8+DggE9m3CjaaNxft6IodfHD +h1iakNtbvwKBwQCpWr/NRy131r0IQh0xdFoFGAUVxF8RSUli4hhSVe0O2TcFiLOr +wW5SK/wnlv75hlY+vABuu1lbfsDmzfnk3RORnm1sJP9JmbQm4AMRfw5jjl7uGiJf +a9+X0kNlsNMlH4ARVhCV3WzOO5KbwXlxzvYRzaqJxDwQbQbXNLRfbFNZxMcPfD8D +w7NSXXdVCpXKmOO8QytkEsHWkv07W+7WtWMEX0iXuPmAjwW0lWNaEd6r3by8yMlz +u9udRedFUEOXUFsCgcAX0g00c75EQXoTtBjVenC/UJCBh//aLwx4Znqsh0Z2LHHR +5XWher4XNiJIG57jCBJpoB30J3b1KS9i8peFL0aJ+pXhiV+EnuxZAJkYBdCyJYn+ +hb6rxeV+xta0XlOXW/UL+QfqcttUgtRvC3JmGEsibw9nx88l+mIDZZ8E4/3qytCd +s1zxTU3AyhNrwuALNH2mUSssgeB6aQot2a6K5GQUj00KUQ8om8sZxL6qAGL+npiT +f4KJ2WDG7u1jQKbat68CgcBONTGp3M3pdwc3OV/PMFq1+h1/BuyW/5qUtbsvXkrD +DZyXGY2SxJFMipakzOppusaQOYmO+VXKVDRJm8UG6fa2cdXjwm2kDxHB5E+v59VI ++uM8ErX6L52I0+rdoOU6AwpXyiVW5AXzMInOGHF4B7zJ1SA25RwVG44LbDw1cehW +MALXUdVCaegmvCCFq5TQlrlaEktxWk6Th92mXWbNbwUHlIGpcjEkASFvt90aBXWZ +w0YCQFVxx/K95nzKyRTjHTQ= -----END PRIVATE KEY----- Certificate: Data: @@ -47,42 +47,42 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=allsans Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (3072 bit) + Public-Key: (3072 bit) Modulus: - 00:c1:1a:f8:fe:53:2f:d5:53:24:d1:9a:62:00:d5: - 41:04:35:38:61:d3:ea:56:38:37:2f:bd:49:64:78: - 8a:3e:5d:32:77:16:11:78:fd:15:63:22:60:c7:8e: - 8c:e5:db:c5:df:d9:3c:ca:9e:f4:09:6b:52:82:a6: - a6:bb:7e:bf:1c:2f:d4:0c:c2:4e:29:1f:e4:b8:ba: - 5c:4e:bb:4d:81:97:76:1a:7f:1d:a8:25:55:0c:2f: - 7e:ef:72:22:60:fb:39:33:3f:1d:64:de:d5:c5:8d: - 79:2b:2c:68:d9:c0:ea:2e:7c:10:b7:02:63:ee:ab: - 6c:47:14:1c:c7:ae:fa:79:fc:32:11:1f:6b:25:40: - 53:3d:7e:95:59:cc:de:fb:81:8c:b3:c5:b6:b4:c0: - 27:c2:3e:90:9f:78:91:51:c9:82:96:f1:ce:cc:5e: - bc:27:33:cd:98:b4:4e:d1:96:77:f6:db:b1:24:09: - d3:0d:69:27:99:2d:42:31:79:5a:5c:9d:27:2a:66: - 5d:12:21:b4:77:60:48:95:d0:b3:c5:93:1d:30:24: - 59:bc:a9:41:05:53:f8:7e:d2:36:a6:83:2f:ce:37: - ed:75:9e:a9:8c:96:9d:c1:8c:d8:bf:25:35:6c:6b: - b3:7b:03:77:6f:74:70:bb:55:59:6b:5a:75:20:53: - a3:28:4a:78:b2:2f:a8:a3:a6:e7:32:1e:d6:73:2b: - 69:89:cb:4b:07:47:c3:da:74:72:a8:c3:43:b8:db: - 7f:f9:37:c1:8a:4d:23:af:68:63:17:4e:30:1e:38: - 6b:3e:f7:f3:f5:65:8a:37:22:38:d0:3f:3f:cd:57: - 74:25:84:af:33:46:ac:45:dd:c5:b4:7a:41:c7:91: - 3f:bf:8d:98:c2:bd:22:a6:ea:67:5b:31:0b:a7:28: - 4d:56:f9:da:24:01:cf:35:e6:96:f8:f0:cc:df:d5: - e5:8a:77:fe:d4:c9:47:fb:09:7b:ac:b3:20:1a:27: - 77:25:a5:a2:b5:b1:b6:e7:f6:6d + 00:98:11:8f:a5:23:04:bf:c3:e8:00:53:93:df:c5: + 27:8b:e2:a3:05:e5:3b:45:92:b8:9f:2f:4a:10:68: + 28:d6:15:bf:23:79:7d:91:6f:5b:4a:68:e6:f2:ad: + 89:93:51:15:56:6c:c9:08:d4:b2:4e:41:66:2d:1a: + 50:0d:a3:69:af:7f:85:43:d7:06:59:ff:1f:d3:9b: + 29:71:7b:70:3b:8a:d4:93:66:06:6e:19:72:c7:b4: + 53:fd:c2:4e:f2:4b:8e:81:95:57:0d:ab:76:68:3a: + 2e:e4:93:c3:71:4e:25:2a:82:c9:8b:83:73:63:f8: + 87:e4:f0:e2:3b:73:50:5c:3f:6a:5a:d7:06:4e:fe: + 5d:cf:4d:6a:ad:15:22:ec:14:4b:ce:2b:93:7a:64: + fa:88:2c:05:22:e6:f1:24:3a:21:5e:7f:15:47:1e: + f0:f8:37:72:16:48:58:12:c2:20:e2:f4:a8:67:b5: + d4:70:6b:e6:b1:b1:9c:63:ab:fa:3b:d0:58:d0:3b: + 9a:b9:42:ec:a6:f5:a0:ea:aa:8e:3b:3c:cc:a4:11: + bd:60:75:30:50:2b:59:8f:96:23:0a:b0:b8:b9:cf: + 27:08:ff:2a:66:d4:f1:1c:1f:9d:fe:f7:f1:7e:53: + 60:75:6e:6b:c0:ff:d3:c1:c7:c4:fb:85:4c:9c:94: + 16:d0:1c:29:9d:71:72:18:0a:01:6b:e4:06:c6:4b: + 78:c5:79:53:e2:17:b8:42:3e:3d:64:fe:37:29:6b: + 25:d9:02:d5:a6:a8:94:e5:91:72:86:6c:ce:bc:02: + 03:3f:0c:73:b4:7c:60:98:19:15:30:da:f8:55:dd: + 92:11:0a:6d:01:ab:07:e8:ad:e1:61:fb:6a:0c:c9: + 98:c3:8e:fa:3f:b9:98:d9:35:ac:dd:a5:cc:2f:3b: + 2d:1c:94:59:0e:47:05:48:d3:34:c8:8b:15:d8:49: + 5e:31:37:50:57:f3:6d:49:0a:e8:58:b9:ba:e8:f3: + d9:be:31:28:f1:3b:5e:2a:82:cd Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Alternative Name: - DNS:allsans, othername:, othername:, email:user@example.org, DNS:www.example.org, DirName:/C=XY/L=Castle Anthrax/O=Python Software Foundation/CN=dirname example, URI:https://www.python.org/, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1, Registered ID:1.2.3.4.5 + DNS:allsans, othername: 1.2.3.4::some other identifier, othername: 1.3.6.1.5.2.2::, email:user@example.org, DNS:www.example.org, DirName:/C=XY/L=Castle Anthrax/O=Python Software Foundation/CN=dirname example, URI:https://www.python.org/, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1, Registered ID:1.2.3.4.5 X509v3 Key Usage: critical Digital Signature, Key Encipherment X509v3 Extended Key Usage: @@ -90,81 +90,78 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - D4:F1:D8:23:E0:A7:E9:CA:12:45:A0:0D:03:C2:25:A6:E8:65:BC:EE + 73:0D:4E:1F:4A:EC:F2:53:0F:53:FC:85:6F:CF:82:CD:A3:E8:11:8D X509v3 Authority Key Identifier: - keyid:B3:8A:A0:A2:BA:71:F1:A8:24:79:D4:A4:5B:25:36:15:1E:49:C8:CD + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B - Authority Information Access: CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer OCSP - URI:http://testca.pythontest.net/testca/ocsp/ - X509v3 CRL Distribution Points: - Full Name: URI:http://testca.pythontest.net/testca/revocation.crl - Signature Algorithm: sha256WithRSAEncryption - 70:77:d8:82:b0:f4:ab:de:84:ce:88:32:63:5e:23:0f:b6:58: - a2:b1:65:ff:12:22:0b:88:a6:fa:06:40:9a:e7:63:a7:5d:ae: - 94:c5:68:3c:4b:e9:95:34:01:75:24:df:9d:6e:9b:e4:ff:3f: - 61:97:29:7b:ab:34:2c:14:d3:01:d2:eb:fb:84:40:db:12:54: - 7e:7a:44:bc:08:eb:9f:e2:15:0b:11:4f:25:d2:56:51:95:ad: - 6d:ad:07:aa:6a:61:f9:39:d5:82:8c:45:31:9f:2a:ff:18:98: - 49:0c:bb:17:ad:d5:24:d3:d1:c7:c4:10:3e:c4:79:26:58:f4: - c5:de:82:16:c4:c3:c4:a7:a3:62:22:41:90:36:0f:bc:4c:fd: - 6a:18:22:f2:87:e9:07:db:b4:3d:65:00:e4:70:f9:d6:e5:a8: - a1:b9:c9:9d:e7:5d:78:aa:98:d5:f8:f4:fd:5c:d9:4c:d0:6d: - bf:87:71:d3:5b:ec:f4:bf:46:f9:c8:f8:10:c5:72:af:c3:15: - b9:c4:06:67:0b:3f:f6:f4:64:c5:27:74:c1:6b:00:37:da:ea: - 18:36:77:36:a7:3e:80:2e:5d:54:0f:01:df:ce:9e:97:dd:c9: - f2:8b:59:82:c5:65:31:c8:73:20:fd:24:23:25:d8:00:df:90: - 93:26:76:08:0a:06:a9:0e:d3:d3:4c:6f:ef:a7:fb:de:eb:2a: - 40:b9:e4:b1:44:0c:37:ca:c6:9e:44:4a:b4:7c:2c:40:52:35: - bb:b3:71:28:3d:35:fd:be:c9:4f:54:b3:99:c5:5f:84:38:fb: - 2b:fb:ea:dd:88:e8:9d:c1:9b:67:87:3d:79:7b:3d:7e:61:1f: - 70:3c:b7:c8:4c:17:a5:0c:a3:28:c7:ab:48:11:14:f7:98:7a: - da:4e:fb:91:76:89:0a:a6:c6:72:e0:96:d9:f1:80:ea:68:90: - 37:5c:c6:69:c7:d7:bc:c7:d1:ae:5b:a9:12:59:c6:e4:6c:61: - a9:8b:ba:51:b3:13 + Signature Value: + a5:3d:12:87:b9:3b:3e:9c:1c:59:fb:d5:38:22:49:61:f3:c3: + 11:53:4b:4e:63:af:f2:3d:ef:24:67:45:bc:74:5c:4a:65:c5: + b4:bb:fe:84:b8:b6:ca:7d:fc:aa:ff:80:ae:67:1f:cb:c3:cd: + 8f:f9:75:8c:f9:d3:3f:db:f6:81:d8:06:42:c3:5d:a9:1e:a3: + 81:7d:57:ac:97:d9:bd:c8:ce:1e:ec:74:d7:94:d5:df:b1:ad: + ce:84:14:5c:8c:45:a4:a8:eb:67:ab:16:57:61:15:86:ae:11: + 1e:b5:10:42:de:84:76:88:9b:37:12:aa:a6:77:42:75:b4:c0: + 04:b3:75:45:e0:d7:aa:34:e3:07:c5:ed:f8:4e:f0:39:99:1f: + 5b:d8:4e:0c:ad:64:6d:09:07:3f:e3:e1:9f:1b:65:07:96:59: + 9a:b5:f1:4d:c3:ec:f7:32:a1:05:94:d1:0b:18:54:3c:67:cf: + 38:f5:2b:ec:cb:bd:79:be:f7:1b:b7:71:3f:c6:44:80:7f:00: + dc:3d:a0:07:c0:b5:1c:fb:52:f6:a0:f8:92:c6:c6:73:07:c5: + ca:0b:04:7c:55:51:e3:ba:93:32:17:bd:61:ae:cf:13:e4:5e: + 03:b2:51:11:c7:68:f5:08:b6:0e:57:49:11:3c:e3:f4:0e:e1: + 96:20:44:28:31:94:11:44:50:cf:17:70:8d:9c:14:c5:ed:94: + 4d:ba:94:9b:db:8b:9f:55:a1:5e:0a:90:bb:a0:0e:0d:3b:a0: + dd:4d:47:d1:cf:d0:47:6b:ff:6f:af:e4:83:40:73:e6:3a:59: + 40:bd:4f:3a:21:22:63:27:5d:02:26:67:89:1d:2f:19:c5:77: + e6:b5:67:4a:dd:b5:0e:de:46:34:57:c7:03:84:5d:cd:34:08: + 8f:47:c9:4d:7f:04:c0:4f:ff:68:52:bb:ae:84:0e:54:ce:5c: + 27:71:0f:a2:3f:9f:7a:49:a8:fa:0a:45:cf:96:42:a7:65:23: + b9:3e:40:eb:46:7b -----BEGIN CERTIFICATE----- -MIIHDTCCBXWgAwIBAgIJAMstgJlaaVJfMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIHDzCCBXegAwIBAgIJAMstgJlaaVJfMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMF0xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2Fs -bHNhbnMwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDBGvj+Uy/VUyTR -mmIA1UEENThh0+pWODcvvUlkeIo+XTJ3FhF4/RVjImDHjozl28Xf2TzKnvQJa1KC -pqa7fr8cL9QMwk4pH+S4ulxOu02Bl3Yafx2oJVUML37vciJg+zkzPx1k3tXFjXkr -LGjZwOoufBC3AmPuq2xHFBzHrvp5/DIRH2slQFM9fpVZzN77gYyzxba0wCfCPpCf -eJFRyYKW8c7MXrwnM82YtE7Rlnf227EkCdMNaSeZLUIxeVpcnScqZl0SIbR3YEiV -0LPFkx0wJFm8qUEFU/h+0jamgy/ON+11nqmMlp3BjNi/JTVsa7N7A3dvdHC7VVlr -WnUgU6MoSniyL6ijpucyHtZzK2mJy0sHR8PadHKow0O423/5N8GKTSOvaGMXTjAe -OGs+9/P1ZYo3IjjQPz/NV3QlhK8zRqxF3cW0ekHHkT+/jZjCvSKm6mdbMQunKE1W -+dokAc815pb48Mzf1eWKd/7UyUf7CXussyAaJ3clpaK1sbbn9m0CAwEAAaOCAt4w -ggLaMIIBMAYDVR0RBIIBJzCCASOCB2FsbHNhbnOgHgYDKgMEoBcMFXNvbWUgb3Ro -ZXIgaWRlbnRpZmllcqA1BgYrBgEFAgKgKzApoBAbDktFUkJFUk9TLlJFQUxNoRUw -E6ADAgEBoQwwChsIdXNlcm5hbWWBEHVzZXJAZXhhbXBsZS5vcmeCD3d3dy5leGFt -cGxlLm9yZ6RnMGUxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJh -eDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xGDAWBgNVBAMM -D2Rpcm5hbWUgZXhhbXBsZYYXaHR0cHM6Ly93d3cucHl0aG9uLm9yZy+HBH8AAAGH -EAAAAAAAAAAAAAAAAAAAAAGIBCoDBAUwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQW -MBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBTU -8dgj4KfpyhJFoA0DwiWm6GW87jB9BgNVHSMEdjB0gBSziqCiunHxqCR51KRbJTYV -HknIzaFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy -ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyggkAyy2AmVpp -UlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rlc3RjYS5w -eXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUFBzABhilo -dHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBDBgNVHR8E -PDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9y -ZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAcHfYgrD0q96EzogyY14j -D7ZYorFl/xIiC4im+gZAmudjp12ulMVoPEvplTQBdSTfnW6b5P8/YZcpe6s0LBTT -AdLr+4RA2xJUfnpEvAjrn+IVCxFPJdJWUZWtba0Hqmph+TnVgoxFMZ8q/xiYSQy7 -F63VJNPRx8QQPsR5Jlj0xd6CFsTDxKejYiJBkDYPvEz9ahgi8ofpB9u0PWUA5HD5 -1uWoobnJneddeKqY1fj0/VzZTNBtv4dx01vs9L9G+cj4EMVyr8MVucQGZws/9vRk -xSd0wWsAN9rqGDZ3Nqc+gC5dVA8B386el93J8otZgsVlMchzIP0kIyXYAN+QkyZ2 -CAoGqQ7T00xv76f73usqQLnksUQMN8rGnkRKtHwsQFI1u7NxKD01/b7JT1SzmcVf -hDj7K/vq3YjoncGbZ4c9eXs9fmEfcDy3yEwXpQyjKMerSBEU95h62k77kXaJCqbG -cuCW2fGA6miQN1zGacfXvMfRrlupElnG5GxhqYu6UbMT +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowXTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEQMA4GA1UEAwwH +YWxsc2FuczCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAJgRj6UjBL/D +6ABTk9/FJ4viowXlO0WSuJ8vShBoKNYVvyN5fZFvW0po5vKtiZNRFVZsyQjUsk5B +Zi0aUA2jaa9/hUPXBln/H9ObKXF7cDuK1JNmBm4Zcse0U/3CTvJLjoGVVw2rdmg6 +LuSTw3FOJSqCyYuDc2P4h+Tw4jtzUFw/alrXBk7+Xc9Naq0VIuwUS84rk3pk+ogs +BSLm8SQ6IV5/FUce8Pg3chZIWBLCIOL0qGe11HBr5rGxnGOr+jvQWNA7mrlC7Kb1 +oOqqjjs8zKQRvWB1MFArWY+WIwqwuLnPJwj/KmbU8Rwfnf738X5TYHVua8D/08HH +xPuFTJyUFtAcKZ1xchgKAWvkBsZLeMV5U+IXuEI+PWT+NylrJdkC1aaolOWRcoZs +zrwCAz8Mc7R8YJgZFTDa+FXdkhEKbQGrB+it4WH7agzJmMOO+j+5mNk1rN2lzC87 +LRyUWQ5HBUjTNMiLFdhJXjE3UFfzbUkK6Fi5uujz2b4xKPE7XiqCzQIDAQABo4IC +3jCCAtowggEwBgNVHREEggEnMIIBI4IHYWxsc2Fuc6AeBgMqAwSgFwwVc29tZSBv +dGhlciBpZGVudGlmaWVyoDUGBisGAQUCAqArMCmgEBsOS0VSQkVST1MuUkVBTE2h +FTAToAMCAQGhDDAKGwh1c2VybmFtZYEQdXNlckBleGFtcGxlLm9yZ4IPd3d3LmV4 +YW1wbGUub3JnpGcwZTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRo +cmF4MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEYMBYGA1UE +AwwPZGlybmFtZSBleGFtcGxlhhdodHRwczovL3d3dy5weXRob24ub3JnL4cEfwAA +AYcQAAAAAAAAAAAAAAAAAAAAAYgEKgMEBTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0l +BBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYE +FHMNTh9K7PJTD1P8hW/Pgs2j6BGNMH0GA1UdIwR2MHSAFMAKKyhD3l/JfUflR5s2 +8mWMZzvioVGkTzBNMQswCQYDVQQGEwJYWTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3 +YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91ci1jYS1zZXJ2ZXKCCQDLLYCZ +WmlSWzCBgwYIKwYBBQUHAQEEdzB1MDwGCCsGAQUFBzAChjBodHRwOi8vdGVzdGNh +LnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9weWNhY2VydC5jZXIwNQYIKwYBBQUHMAGG +KWh0dHA6Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL29jc3AvMEMGA1Ud +HwQ8MDowOKA2oDSGMmh0dHA6Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNh +L3Jldm9jYXRpb24uY3JsMA0GCSqGSIb3DQEBCwUAA4IBgQClPRKHuTs+nBxZ+9U4 +Iklh88MRU0tOY6/yPe8kZ0W8dFxKZcW0u/6EuLbKffyq/4CuZx/Lw82P+XWM+dM/ +2/aB2AZCw12pHqOBfVesl9m9yM4e7HTXlNXfsa3OhBRcjEWkqOtnqxZXYRWGrhEe +tRBC3oR2iJs3Eqqmd0J1tMAEs3VF4NeqNOMHxe34TvA5mR9b2E4MrWRtCQc/4+Gf +G2UHllmatfFNw+z3MqEFlNELGFQ8Z8849Svsy715vvcbt3E/xkSAfwDcPaAHwLUc ++1L2oPiSxsZzB8XKCwR8VVHjupMyF71hrs8T5F4DslERx2j1CLYOV0kRPOP0DuGW +IEQoMZQRRFDPF3CNnBTF7ZRNupSb24ufVaFeCpC7oA4NO6DdTUfRz9BHa/9vr+SD +QHPmOllAvU86ISJjJ10CJmeJHS8ZxXfmtWdK3bUO3kY0V8cDhF3NNAiPR8lNfwTA +T/9oUruuhA5UzlwncQ+iP596Saj6CkXPlkKnZSO5PkDrRns= -----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/b1930218.0 b/Lib/test/certdata/capath/b1930218.0 index 941d7919f8033e..6d773ee5b32432 100644 --- a/Lib/test/certdata/capath/b1930218.0 +++ b/Lib/test/certdata/capath/b1930218.0 @@ -1,26 +1,27 @@ -----BEGIN CERTIFICATE----- -MIIEbTCCAtWgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIEgjCCAuqgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg -Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCAaIwDQYJKoZI -hvcNAQEBBQADggGPADCCAYoCggGBALGE009cBICRT4JJujAL9+jL+RTvPZ8LPwpi -/BsgpSDRYF+HWh8W0e2XcKbaGwMsfqBbPE4vFn4OiSmJ4RANONpqd183E7Moj3tc -dq2e6NP1nvWDqhAHjeZRmPB8DVLyDCEe2LmZJqklAye7XKsuMyei1iOog4dEKZ+X -tSRv17kK/Sjuu/tBWOodmd1EhquYvhzcy6mJHTZcqehHtfRSSKq1pGfvPtfi0zPe -mCnYerBZXOexDsz9n+v21ToOC8/+Cz2iv0UYzpTnqVVgiNTYhFB5BS5BA3SuZyb2 -WxIImM4Kl+0BD4lPF1z6Ph01JEeSMr/3pBgrPNBImeGizaPMUFMgtcbjZoV7VxDs -M0/Bd+cbfoHGxPNFIMCR3RN2ewOv9naOooNjV91jvLtaHBdSitYGSMwPx9NP6Noi -bIb5TlymKQc72FZMWbMgSQd7lITPK8McGk6HZJK6QuHmrX0d9lSQbyvps8xLKzMm -I/1lwDzwea3JwYHvNwTgJz6w7hW+UQIDAQABo1AwTjAdBgNVHQ4EFgQUs4qgorpx -8agkedSkWyU2FR5JyM0wHwYDVR0jBBgwFoAUs4qgorpx8agkedSkWyU2FR5JyM0w -DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAYEAazIv5wUY6lzJlfTgwgxB -XxoKlcnHfQXuilYpNVBAt/6fe1scw2kvoMvSuJEvUBli9ycYbZV7UxYVolrcFOP7 -sTKpadumM0c8ux/S3HD5ai4M2Ixt5V0dQzxOkd6gyNqgSw6dXrYPSknwe7ZTnv01 -FFvjTbQYpjZh6I8zm9QF+VRm3+DLGKNO3BeooLPBqPTWncp/aFMa15Xa6NOeSABx -lZkRB8+WwH3OfTDoT+GDFjOh/1mbPkznOjgBnw9nTP0ti0rUAUY3M+gTaxWpHWh2 -RaKCM2kmMGAFyI+9tHWrvnqLSGhwQLQbUcXmeq1rT9sXwGBnLmNhmyxImbh2RaCe -zO8zHlBOq3LDZciyebM1gyF404tsOhjoZTI5uMCdcS81NorAF2LYiz7hIhgrTGOm -Dp0K+qtbNfuIkXdMjYydqc/8q8LmWgV7fgRuOc+Tzmc7esuvtjbh+3FkRdSm8M7v -dQSZaZrliAoQAnSJ7HWERIBI38H36TfOzpKSXIkiCHMf +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyMIIBojANBgkq +hkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA5rri5MHCDBw+Yti5XFcuUriDxYg65pp6 +9WQWM+s3bi9782gDRWVHXXEQWcorGwBsgRRh9IZZP+r9eDcWnUPxxPZpjMUpBoie +JiIErATYhzRIOetr8guSqsNuY1FRa8Kt/1zIL7IbnCCKQD6iL2rqyNk3Q1zc7ZLi +2UDSYZ9xivXtObqoXj61IWMQ2G+04hEBCxDou/ti70hVvPXSnKtorpUlGfKXfRrc +ZuqIXobky8tpTV6wo/tsMeQoYF6Q8dQuEFDhhfANXL3dRSQIGT4ck2aPK9pTfQQc +DkLEaF6mzakY7afNatDRhrqQ/7dM3sdDJG3HHGucgefhG1clkKkOyVbz9mteLbQu +QFCbQmPS1pkcONzPKyyncvHHXmM0dkj0PogTnoYWUy90+4cBjSKkaDPuE2x6BhRU +VhdXV5g00AtmCeOICfilFRwQc9CIUJleGGU7/zEnG17GqkH9LS8Yp8Dyq8citQtp +0nPRu9AcPfqkNWLNM4bHoCMPuWrV0m2NAgMBAAGjYzBhMB0GA1UdDgQWBBTACiso +Q95fyX1H5UebNvJljGc74jAfBgNVHSMEGDAWgBTACisoQ95fyX1H5UebNvJljGc7 +4jAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsF +AAOCAYEAvONWIgPkXLln/5S8dZwAhbDVnMTDKWZei7KppjCGcRpr8gDFgqtfUAQq +++24TLkAG0lXkhHNorzLD6i0YfgUyqDsQBe6VaHEvKayWu/0IBB3R9CgxVi5bLUQ +e4VKQ6P7LAG5dxewvqDurq5NZ4ZIiVeGeOo87fBBNY1xaFX58umsMtTGou/sVObE +jiz9vapgVmUzleoQxnQE6ypumxH2YQCq/ezyC7FLEc2T69+Yrky0BwRK5e//Ulh1 +9T6kceFKclypj9SqiPBqcbTDAF+ZbteRr2yYDWTCJMeeBRFoXiRi4y5F7KM08qOd +TeUyGC90/BHxNlBPoEApaFxDTCNsXXLE7FJ269yyvB+mxAZmm1zHzMry0SVP3qUf +jeQMSbbPhUChuR/GxxkVB2M0k9BXoFpw7K9KHHIXHXSjbDFFCzN6obhG28cOZExv +t5kEgkMf4FnWmSEnKAlArvzEI6qgDAgFKpIc2yOe0dVjrjkToxKIWkM8Sm4y8ISf ++QkMkee4 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/ceff1710.0 b/Lib/test/certdata/capath/ceff1710.0 index 941d7919f8033e..6d773ee5b32432 100644 --- a/Lib/test/certdata/capath/ceff1710.0 +++ b/Lib/test/certdata/capath/ceff1710.0 @@ -1,26 +1,27 @@ -----BEGIN CERTIFICATE----- -MIIEbTCCAtWgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIEgjCCAuqgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg -Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCAaIwDQYJKoZI -hvcNAQEBBQADggGPADCCAYoCggGBALGE009cBICRT4JJujAL9+jL+RTvPZ8LPwpi -/BsgpSDRYF+HWh8W0e2XcKbaGwMsfqBbPE4vFn4OiSmJ4RANONpqd183E7Moj3tc -dq2e6NP1nvWDqhAHjeZRmPB8DVLyDCEe2LmZJqklAye7XKsuMyei1iOog4dEKZ+X -tSRv17kK/Sjuu/tBWOodmd1EhquYvhzcy6mJHTZcqehHtfRSSKq1pGfvPtfi0zPe -mCnYerBZXOexDsz9n+v21ToOC8/+Cz2iv0UYzpTnqVVgiNTYhFB5BS5BA3SuZyb2 -WxIImM4Kl+0BD4lPF1z6Ph01JEeSMr/3pBgrPNBImeGizaPMUFMgtcbjZoV7VxDs -M0/Bd+cbfoHGxPNFIMCR3RN2ewOv9naOooNjV91jvLtaHBdSitYGSMwPx9NP6Noi -bIb5TlymKQc72FZMWbMgSQd7lITPK8McGk6HZJK6QuHmrX0d9lSQbyvps8xLKzMm -I/1lwDzwea3JwYHvNwTgJz6w7hW+UQIDAQABo1AwTjAdBgNVHQ4EFgQUs4qgorpx -8agkedSkWyU2FR5JyM0wHwYDVR0jBBgwFoAUs4qgorpx8agkedSkWyU2FR5JyM0w -DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAYEAazIv5wUY6lzJlfTgwgxB -XxoKlcnHfQXuilYpNVBAt/6fe1scw2kvoMvSuJEvUBli9ycYbZV7UxYVolrcFOP7 -sTKpadumM0c8ux/S3HD5ai4M2Ixt5V0dQzxOkd6gyNqgSw6dXrYPSknwe7ZTnv01 -FFvjTbQYpjZh6I8zm9QF+VRm3+DLGKNO3BeooLPBqPTWncp/aFMa15Xa6NOeSABx -lZkRB8+WwH3OfTDoT+GDFjOh/1mbPkznOjgBnw9nTP0ti0rUAUY3M+gTaxWpHWh2 -RaKCM2kmMGAFyI+9tHWrvnqLSGhwQLQbUcXmeq1rT9sXwGBnLmNhmyxImbh2RaCe -zO8zHlBOq3LDZciyebM1gyF404tsOhjoZTI5uMCdcS81NorAF2LYiz7hIhgrTGOm -Dp0K+qtbNfuIkXdMjYydqc/8q8LmWgV7fgRuOc+Tzmc7esuvtjbh+3FkRdSm8M7v -dQSZaZrliAoQAnSJ7HWERIBI38H36TfOzpKSXIkiCHMf +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyMIIBojANBgkq +hkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA5rri5MHCDBw+Yti5XFcuUriDxYg65pp6 +9WQWM+s3bi9782gDRWVHXXEQWcorGwBsgRRh9IZZP+r9eDcWnUPxxPZpjMUpBoie +JiIErATYhzRIOetr8guSqsNuY1FRa8Kt/1zIL7IbnCCKQD6iL2rqyNk3Q1zc7ZLi +2UDSYZ9xivXtObqoXj61IWMQ2G+04hEBCxDou/ti70hVvPXSnKtorpUlGfKXfRrc +ZuqIXobky8tpTV6wo/tsMeQoYF6Q8dQuEFDhhfANXL3dRSQIGT4ck2aPK9pTfQQc +DkLEaF6mzakY7afNatDRhrqQ/7dM3sdDJG3HHGucgefhG1clkKkOyVbz9mteLbQu +QFCbQmPS1pkcONzPKyyncvHHXmM0dkj0PogTnoYWUy90+4cBjSKkaDPuE2x6BhRU +VhdXV5g00AtmCeOICfilFRwQc9CIUJleGGU7/zEnG17GqkH9LS8Yp8Dyq8citQtp +0nPRu9AcPfqkNWLNM4bHoCMPuWrV0m2NAgMBAAGjYzBhMB0GA1UdDgQWBBTACiso +Q95fyX1H5UebNvJljGc74jAfBgNVHSMEGDAWgBTACisoQ95fyX1H5UebNvJljGc7 +4jAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsF +AAOCAYEAvONWIgPkXLln/5S8dZwAhbDVnMTDKWZei7KppjCGcRpr8gDFgqtfUAQq +++24TLkAG0lXkhHNorzLD6i0YfgUyqDsQBe6VaHEvKayWu/0IBB3R9CgxVi5bLUQ +e4VKQ6P7LAG5dxewvqDurq5NZ4ZIiVeGeOo87fBBNY1xaFX58umsMtTGou/sVObE +jiz9vapgVmUzleoQxnQE6ypumxH2YQCq/ezyC7FLEc2T69+Yrky0BwRK5e//Ulh1 +9T6kceFKclypj9SqiPBqcbTDAF+ZbteRr2yYDWTCJMeeBRFoXiRi4y5F7KM08qOd +TeUyGC90/BHxNlBPoEApaFxDTCNsXXLE7FJ269yyvB+mxAZmm1zHzMry0SVP3qUf +jeQMSbbPhUChuR/GxxkVB2M0k9BXoFpw7K9KHHIXHXSjbDFFCzN6obhG28cOZExv +t5kEgkMf4FnWmSEnKAlArvzEI6qgDAgFKpIc2yOe0dVjrjkToxKIWkM8Sm4y8ISf ++QkMkee4 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/cert3.pem b/Lib/test/certdata/cert3.pem new file mode 100644 index 00000000000000..a11dc614657d2f --- /dev/null +++ b/Lib/test/certdata/cert3.pem @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIF8zCCBFugAwIBAgIJAMstgJlaaVJcMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJ +bG9jYWxob3N0MIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA7jZwqmFq +ILsyiAdz8DrKEcyPMIDXccd0YiCUiLRSEpQIAeF5GMzwaYuZxUfBgqquyFEEPomM +HWWt8h+l9dSHZawseIA3UBUhTnoJxuNaKr+xsARBb6usZaMKaGhsPuf/P7CV/1VO +OKy/f34jFU23oTITEv8+Z00mEgAle7EV58FuE+pdjne+xczwY52hRQza+RiKIg+J +jUid+bdObZYhnM9CMhOUxkepCBBTSB+bYXh6CSeCQuLi8licHiacQ8ddJ41kcCjf +7V5vBZx0DzEQFJdsDNO0GRCNcn81K9NP6BtnaT5z8jYfuqdpXfCUtINvz3dqUC/D +rZjNnA3DeRPqghFtVFSCef/2nfKVHKEMMkSAUTiW2pKr+hXFU3YE6IKKuVbvk+k1 +RS0iEr1b6bFdDLU/x/f/U7Qp6jsJYhPLPJG9zY0E/Hu9lRzXeN21TxOA3kPl5WzK +Cs1fhjpkh0n80jmQfZEnEphneWA/O/N02y/P+zZ9REUHVqmosRiN+vgRAgMBAAGj +ggHAMIIBvDAUBgNVHREEDTALgglsb2NhbGhvc3QwDgYDVR0PAQH/BAQDAgWgMB0G +A1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1Ud +DgQWBBQWIsmqINT0ju2cprsj9fIpRO3yHjB9BgNVHSMEdjB0gBTACisoQ95fyX1H +5UebNvJljGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyggkA +yy2AmVppUlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rl +c3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUF +BzABhilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBD +BgNVHR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rl +c3RjYS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAQ4IfGLTLerdO +rMNlLrdXOvB4s7IgPr17JPfnF8xiwLhj8C4wDFS+yZR8VNRABm6SnXIsRPXjwUo/ +JuQhyrrvT6NQVu6JXNxbmLwM6dsWmPBMP2W52eAuvYOexxv3T4dfdf9nXQr/zvoL +8dWLWCMrkie4Ff9mwvlo4u1koErgQousNWpnZPXLqQA3IFbdOgJu2A+0Xf+Sow1l +/C6rTje8ftZbHFV4oG6pLlUxz2HwG0z+/mB1dujZofUU8EMzTVIFvjP/2jGUvQ3l +Taju0fOSNMI2kTc6bewg37Oeol3Q8KHi/7eFzgnjyEpqk6Su7MFnQveOL2TK13Zy +vz/vZP8Q3aI+LfWqAs8x8G2Ta1ZMsIiVVNzUrNzBiCeL2ZxOZpP43V50QSaa7+jI +RlzV9PzNzGfHM2IucJvROd40/a2duUhh54lTYmLwQGxoL+HaQGEqUK/JQW2YFq/L +YwPsBJngOZhgrqpqV5slcwMWv3jI1y/r/GR/x3iMNBVbZkCYhuYK +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/idnsans.pem b/Lib/test/certdata/idnsans.pem index cbcac7818ddc67..ebd7250cb4eae9 100644 --- a/Lib/test/certdata/idnsans.pem +++ b/Lib/test/certdata/idnsans.pem @@ -1,42 +1,42 @@ -----BEGIN PRIVATE KEY----- -MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQC8sqplTuHuLjbW -TL5SL2D1fw9U6WQzLVAF5gsyhd5lr2FpfYwjrob5Mav91aOLbJRTvoNyXsJ26FPS -0RycRGXbomcIEJxXGy9aI+0MLYBt1G5mgqCH+HcVCwPzCNlhVnTwvpgA7y8zs3+6 -ezZAPWkF0yWOMYLtTcq9A5GWeavt5VMgm1KZF3gO4k58oPyk3Ae9D0LAaYsX6DFi -BYx41eUR5UbSb5IYXaDd8d6jqW/jnYhgc6Cxkv1gTJFn87V5lrG0vYMSRUtWDQ9Y -Jh/EKAxjGw7AeY429p6TE4UoJhDmoFYR2NLvawhNIplxol/v0fs0veFQjI/UsTD8 -2tRfnYL4IX8szhLsE5/5Iq8aiLHjVbIMwmDYAa0P63Ap2kf1biSn9mpDL8lQazSo -yr8xzIq2QS5HMvGbeMAmS0ih10Zx84uVmkWlavgvtSflw8K/ZXT9c70rZp/TdBGY -95cOFsbg5U/20M/Llpis9tcBCaoVaYSFupatrP+p8y19qP2nebsCAwEAAQKCAYEA -uaYWWwHW6pzxOrnabcVLYX0WunW9LVShbIw97AElI2n/LuhkXh6xkK48BsqP0vaK -oDHJ5VYxgQdmoP03Zs8sX4BSWe7twg1u8wJxkA+cUXI1BAn0opHjpwJlalEEfe2v -s8PwjMrF59nsCq56W42PrDlms5UmuQ5WLsw6Co++hZmfxW7LPu+GIS6qBZfluNT5 -kBpZlDDCtkyteUD4SVI3wvmOSi+Wzv4e7P2wC9kByjENIcfhC5QQURRD4sA1hWCp -2SThYWqJOCEc2SvGgoqgTRaJuQ2aVG9qrntXt0N4V+WdJWXBK0jedkB2flLve1fR -KmDYuc9k/c1svmS3Y+iZohBha9H8jpuJmXYBxxg1iNg9m7qkfg8F8wxCYLQKB+U6 -tjRS7by+jSE08On7mpDDhJORnlh+rfEuWPPwAKQpLpdp76KDTvR++GvfOMUiOrFM -e9s5aXp+vcgkSSqYvigE+sFpCjQWwkGBkMdT16Pf9CzhQaM08YuLnzfLEYgLFw6R -AoHBAN5NQINBmlq/cptGSru66kfecqHfI7xHnnGWKAkto/B1x7Crrgs4Tk5b4vaA -JmAqatt5P1e7zco7uAXXebY5VURuH/30TlkuaB+oGFp0OMw6165n8RVPT2ZaDViK -ssJ9LT8fJ+23TWCCT2Z1zUlM/NnHAMjKOVsJK3/KEkVvlc7ROC7uVooc78AsQehg -zpL3GBYEeBukT8aNUMqUlesCsIs/dQHW7DzQL2xGkQagm5/PDsxaCsT7ynA8eL3X -TW+IXwKBwQDZTV3TaG6wqtL8y2DR0lN5jY/eYayX4e18iZ+XEZVTntPdVVyJIE4d -0A5ZfcILb9WE8R21iptROYSjcH/05j+3fQMJ1WAK0sNfGTUNNT3jYU8YzLvos+wW -G8E+mNMpFPWNvLV5Qrl4VvoifGh8AMvplUEz8uAzGJbXbRxUPcmjth2ph8zULEDn -/+o4OcT3gh1bp+HCqch0OuiJRn9qNUpsJG5GMm5FtjBjZM97ucZ1/0DaWl3JUxUN -/pueo3J9vCUCgcBg2Fjdlcvv8u2z1aijJmgATVm1SWfhE3ZkV50zem2sSTNotTJK -cwoyOveimeueA3ywBp9g0lFx5Bhkex3sFAggmrVXRoKHeZ8lA28woOdJmezybxfp -R7b4iQy9YRdFgZEfqawUdMHB5KNAqNt5LpANNBQUZX0dOt53eooBM/6Yri8CyxRq -cPbFysIfwWTdQ8Z7eRD2Qdv7TP9AcgDp9C8DSu7nkUEzsSKn0gpGT9vcgDEbN7Lv -ZB4qTT3wvoZeq5MCgcBIG18eDtJkN1sp3Yb0OTnP5QSvg3PVNngq0jQt2fzWMacW -FARP0HN7exW35n4kc2jD44q7OhJOAqsb3PHo3xqXlZkTg0WKceO4w9GR32/46spn -bVCRaFrX/z/BuM6hHD5bWRpS8aw/3YTFOsklFNKVYRyw01BIREmRlLhIz/QAKidv -oQt8AG9NTON44tqUUw3Q40WL5fEJeJ6/JrCTGrnmZrRdANEMuucVpFchNEVB1IC9 -tCzY6IPdD/atzojoZi0CgcB2x9oWLjJ0XJIp2pMAb8nCMVjkKrznKFjZbDm8EQBs -ou7pM2zkO3VRcWT1BXQocinJsjQqjQiTawP6IN2FQgT0d89V+pwd+jdvpdildQhP -1/6SErVRZV//oopKTsC6TIBL/EmW1TkP3ulQIZs8YklFgybeHdDyNFi+VgPXkVGe -IHp0nEzrui9q0YJsjHfFHBeGyzDSfbiBYiF7Auk66gYZbXufebP/LZNG/FIamPP3 -rwYIeeV1IVwk9tPBw6fGwrs= +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDeGGysqNqsz64+ +xsN+CUZDfCgPPhzUK0lDUlkEkp0ZeFeQm3+po2Y82gvJIValO5ChiXjPxSgJwoto +SHe4QZ8fWAlkX1d19A8xjjetx1K3dqZstqe3mPP7HQvfJlRkmAtVhFaUj1+SUx9K +s4HB4Lk2hqKUoq3TlRMpeSfJTXaHThVP7KZPWBzfM7pdpGYnznBpxgfymNrpTcXN +CjeR6lUlj/TExHiYLj4E0aODR/SUz0tBNdCQRcRDw+OyREe6Rk32Krl34Pr1dNKe +ER0tulhM2wRFsqw0a8vNs13I30BH4aXAr5bGgLCF2tXbP3KoeoR/GkQSWV14KrhT +xhKD3QEJ8gHjoHYKr4IdcFYcTakbI9E3THsQjQTUMJ/Xygza7OOr6pNPKY10tYfp +7WhyyY7cB/C/rHnUY0f58IZlRdMYgcLCUl7wzUxSVEMyr9Ary8XZZlxuO629k2ud +AIGwIi2TVq+UK0kNPZyGicGHMeEhoNc3YDMB/zYeqMr4KryI+mkCAwEAAQKCAYA4 +EENSnHdC+1P5bdRIew/dFjjIjD3bwyeF0oI9GMOGe+3ix5YE3QYAY2xpM7y7Dhu2 +40x3akXunMjzJKPwA8SmtWL9juG1mUvCjyt39yJmxJFDTSJuQrKIF694/6R7FjR6 +PGNcsgqGlewGv+SH6/HlFTxyN9SYXf/NztMfyimbAzd3Cv56dfwnzdeELu1IrCCN +WtuDvlk4XpUJasRXVadzyXCYwR3OEJJARik4CRBxBhjxl6OT38Co+IiAZiMTHw6z +jXeRuvyxXyYB76BQs9uCt05c5JV1I2OT1aKdpC0EbHlJEzJhmxrpdfBeoVdJUVVg +4b0QoocZE89KXOVmjmIKExxmjOjxH6o6qfxvDQWLspADD4D+69Zqjj6hFAfn3EUW +hcq3RHwuUheCE6i2xRtWPnA1ygqWN9mdNojX1+EXIv0Dh29kFWHpMUC29WWm9JA6 +Arx2UrtvoWGYotKxTDxZ0SV5ws/LzGjvMaT/dbXZHjywnB7sPfpcE9b91p8Q6VcC +gcEA+Wr44ZzyMirF9FYYA+KEBL91khmffhXzCDkRXqd1j2BCQWETWqvrUuAx+kwh +FZ5AABdip96w8Eaj8XBWb1c81mJUbOimbcqiV6B4GokCI/7u3FkOMPFRNBq41R/M +43JH6QLHNxhXI4OlZsxv5WHmR5N88kXEHhofBoSn14EHtdkK/Hu8ly1Wux4/da83 +2wanwlysmQkk1CnpJVT0st7HrKHP8YWDbcBvH+xysHOIOs22q3gvAF2rsdORidpR +fZnHAoHBAOP03brYpLU1xa5LHQz/B7uWfgJm3l7wRQ627YOfUBe4IjnH2y/kNKjv +ROVjaooB2gtya6Z6xVuY1m4LzUfYuRN1FdX1Np4NjvfgPNFViX+0HoxyAytXn++V +Iy1tCpL6X43Iqrj/VxBcX09vIpbsprW2d3dcIo/45abkAO6Uoa6HUmZoqrqrBA7b +LnuhEfBAyyN42OiXbHDqif7XAWC30yY6k0FfDaOSLiBkAOKZ9lkf7ei3eNr4SdOa +Tf5a/RVKTwKBwB7YfOkiCM3tfkfGcffhBqSzrO2hn5jvS/wjWqOTIDXYGLmPMN6Q +zmyUb3nd+mV7Cb05JylNoCJHCjVsyDPC3TJCPOCvMQ349nTR0qitcwdSmuXDWb7x +yTIhb+Rjp2olkwEdJ9gHeZdZy5XYCKqcnecSNWyc9jEm19lthHhha7uwmOw6vUsQ +/13q0rxSLB05SHwADBRtDhHzEPNd+1k3tggChv3+ng9vsg6HpnNuBlYHZOT12xI3 +g2ldme0rg9J9twKBwA4R1gGrT3czy3C3iCJ+Ny732e0yBjWb5NdEqSI/mgTsw4gH +ctrg3fMzWXBDE5dTB+8+77AF0dqWc121csUldj7iMifTi7xzn8hi2b4d5m+wYVZP +zyxEq0VxUguCuG1b8Lvij879S5Vh7iwL8vmXv65lhbgjQqraNOp5FimjmNsZ1Rcn +DKqa1ZRJKPROe7n1ddRJqDGq7vGFOGE3Sgl7Lxgj82TMhh37bsdnBLr3v8G+e8Oq +V1ZEjuH1myzA1vASdwKBwQDcMHbUKeoJUxyIlABFsWxhm0MKwL5ZRgo7rhH01rF2 +TF3mbAEHHsAEfkHgBVRooifbxglxUDy1olIyRk+kqs1gmXZA0UmE9lUqoX5+n/j8 +pgbCc3sV53x7JJ4BCnAn49XTOnTi2ILeQ3MvTqcj+sDKo+0T4Zq0r5d8ZrMikKbO +HJ4MMBGth645QKhgpb0XgltvSn8aceS4uTxIKrbNpZVnWj/VzuOYoXQmKnZko9p1 +dbyjt6PMeVXWj0tz8FB2DGE= -----END PRIVATE KEY----- Certificate: Data: @@ -47,38 +47,38 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=idnsans Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (3072 bit) + Public-Key: (3072 bit) Modulus: - 00:bc:b2:aa:65:4e:e1:ee:2e:36:d6:4c:be:52:2f: - 60:f5:7f:0f:54:e9:64:33:2d:50:05:e6:0b:32:85: - de:65:af:61:69:7d:8c:23:ae:86:f9:31:ab:fd:d5: - a3:8b:6c:94:53:be:83:72:5e:c2:76:e8:53:d2:d1: - 1c:9c:44:65:db:a2:67:08:10:9c:57:1b:2f:5a:23: - ed:0c:2d:80:6d:d4:6e:66:82:a0:87:f8:77:15:0b: - 03:f3:08:d9:61:56:74:f0:be:98:00:ef:2f:33:b3: - 7f:ba:7b:36:40:3d:69:05:d3:25:8e:31:82:ed:4d: - ca:bd:03:91:96:79:ab:ed:e5:53:20:9b:52:99:17: - 78:0e:e2:4e:7c:a0:fc:a4:dc:07:bd:0f:42:c0:69: - 8b:17:e8:31:62:05:8c:78:d5:e5:11:e5:46:d2:6f: - 92:18:5d:a0:dd:f1:de:a3:a9:6f:e3:9d:88:60:73: - a0:b1:92:fd:60:4c:91:67:f3:b5:79:96:b1:b4:bd: - 83:12:45:4b:56:0d:0f:58:26:1f:c4:28:0c:63:1b: - 0e:c0:79:8e:36:f6:9e:93:13:85:28:26:10:e6:a0: - 56:11:d8:d2:ef:6b:08:4d:22:99:71:a2:5f:ef:d1: - fb:34:bd:e1:50:8c:8f:d4:b1:30:fc:da:d4:5f:9d: - 82:f8:21:7f:2c:ce:12:ec:13:9f:f9:22:af:1a:88: - b1:e3:55:b2:0c:c2:60:d8:01:ad:0f:eb:70:29:da: - 47:f5:6e:24:a7:f6:6a:43:2f:c9:50:6b:34:a8:ca: - bf:31:cc:8a:b6:41:2e:47:32:f1:9b:78:c0:26:4b: - 48:a1:d7:46:71:f3:8b:95:9a:45:a5:6a:f8:2f:b5: - 27:e5:c3:c2:bf:65:74:fd:73:bd:2b:66:9f:d3:74: - 11:98:f7:97:0e:16:c6:e0:e5:4f:f6:d0:cf:cb:96: - 98:ac:f6:d7:01:09:aa:15:69:84:85:ba:96:ad:ac: - ff:a9:f3:2d:7d:a8:fd:a7:79:bb + 00:de:18:6c:ac:a8:da:ac:cf:ae:3e:c6:c3:7e:09: + 46:43:7c:28:0f:3e:1c:d4:2b:49:43:52:59:04:92: + 9d:19:78:57:90:9b:7f:a9:a3:66:3c:da:0b:c9:21: + 56:a5:3b:90:a1:89:78:cf:c5:28:09:c2:8b:68:48: + 77:b8:41:9f:1f:58:09:64:5f:57:75:f4:0f:31:8e: + 37:ad:c7:52:b7:76:a6:6c:b6:a7:b7:98:f3:fb:1d: + 0b:df:26:54:64:98:0b:55:84:56:94:8f:5f:92:53: + 1f:4a:b3:81:c1:e0:b9:36:86:a2:94:a2:ad:d3:95: + 13:29:79:27:c9:4d:76:87:4e:15:4f:ec:a6:4f:58: + 1c:df:33:ba:5d:a4:66:27:ce:70:69:c6:07:f2:98: + da:e9:4d:c5:cd:0a:37:91:ea:55:25:8f:f4:c4:c4: + 78:98:2e:3e:04:d1:a3:83:47:f4:94:cf:4b:41:35: + d0:90:45:c4:43:c3:e3:b2:44:47:ba:46:4d:f6:2a: + b9:77:e0:fa:f5:74:d2:9e:11:1d:2d:ba:58:4c:db: + 04:45:b2:ac:34:6b:cb:cd:b3:5d:c8:df:40:47:e1: + a5:c0:af:96:c6:80:b0:85:da:d5:db:3f:72:a8:7a: + 84:7f:1a:44:12:59:5d:78:2a:b8:53:c6:12:83:dd: + 01:09:f2:01:e3:a0:76:0a:af:82:1d:70:56:1c:4d: + a9:1b:23:d1:37:4c:7b:10:8d:04:d4:30:9f:d7:ca: + 0c:da:ec:e3:ab:ea:93:4f:29:8d:74:b5:87:e9:ed: + 68:72:c9:8e:dc:07:f0:bf:ac:79:d4:63:47:f9:f0: + 86:65:45:d3:18:81:c2:c2:52:5e:f0:cd:4c:52:54: + 43:32:af:d0:2b:cb:c5:d9:66:5c:6e:3b:ad:bd:93: + 6b:9d:00:81:b0:22:2d:93:56:af:94:2b:49:0d:3d: + 9c:86:89:c1:87:31:e1:21:a0:d7:37:60:33:01:ff: + 36:1e:a8:ca:f8:2a:bc:88:fa:69 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Alternative Name: @@ -90,80 +90,77 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - 5C:BE:18:7F:7B:3F:CE:99:66:80:79:53:4B:DD:33:1B:42:A5:7E:00 + C8:06:99:B7:E8:8F:EC:4F:3D:5C:89:6A:06:F5:77:2E:B0:E0:6A:9E X509v3 Authority Key Identifier: - keyid:B3:8A:A0:A2:BA:71:F1:A8:24:79:D4:A4:5B:25:36:15:1E:49:C8:CD + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B - Authority Information Access: CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer OCSP - URI:http://testca.pythontest.net/testca/ocsp/ - X509v3 CRL Distribution Points: - Full Name: URI:http://testca.pythontest.net/testca/revocation.crl - Signature Algorithm: sha256WithRSAEncryption - 5d:7a:f8:81:e0:a7:c1:3f:39:eb:d3:52:2c:e1:cb:4d:29:b3: - 77:18:17:18:9e:12:fc:11:cc:3c:49:cb:6b:f4:4d:6c:b8:d2: - f4:e9:37:f8:6b:ed:f5:d7:f1:eb:5a:41:04:c7:f3:8c:da:e1: - 05:8e:ae:58:71:d9:01:8a:32:46:b2:dd:95:46:e1:ce:82:04: - fa:0b:1c:29:75:07:85:ce:cd:59:d4:cc:f3:56:b3:72:4d:cb: - 90:0f:ce:02:21:ce:5d:17:84:96:7f:6a:00:57:42:b7:24:5b: - 07:25:1e:77:a8:9d:da:41:09:8e:29:79:b4:b0:a1:45:c8:70: - ae:2c:86:24:ae:3d:9a:74:a7:04:78:d6:1f:1b:17:c5:c1:6d: - b1:1a:fd:f4:50:2e:61:16:84:89:d0:42:3f:b6:bf:bd:52:bd: - c8:3e:8e:87:b4:f0:bd:ad:c7:51:65:2f:77:e8:69:79:0e:03: - 63:89:e7:70:ad:c8:d1:2f:1a:a5:06:d2:90:db:7c:07:35:9a: - 0b:0e:85:87:d1:70:17:a7:88:0f:c6:b5:9c:88:00:fa:f9:b2: - 0a:19:5a:4b:8d:91:12:51:5e:0e:c1:d8:9e:02:78:d0:2d:24: - 09:fe:d4:97:3c:cb:a0:1f:9a:ab:f7:0f:e2:fa:64:23:4e:53: - 0a:15:3e:f5:04:01:86:29:8b:8e:24:40:2f:b1:90:87:5c:3b: - 7b:a7:4c:06:af:c3:90:7f:e9:c6:56:42:61:15:2c:83:f1:7c: - 4f:89:17:f3:a0:11:34:3f:8d:af:75:34:60:1e:e0:f2:f3:02: - e7:aa:b3:f7:9f:1c:f8:69:f4:fe:da:57:6e:1b:95:53:70:cd: - ed:b6:bb:2a:84:eb:ab:c3:a9:b4:d5:15:a0:b2:cc:81:2d:f1: - 56:c1:54:9b:5f:14:4c:5f:ad:5f:f5:06:ee:22:60:45:e4:50: - 35:64:ac:ac:ca:4a:bf:86:78:f8:53:2d:17:d8:e8:84:c8:07: - a4:c2:29:76:c7:1f + Signature Value: + 40:d1:6d:8e:a2:0b:91:4b:a8:c4:08:d0:f3:f9:8b:d0:a3:0b: + dc:00:22:8c:f1:2e:2b:e5:e6:b4:6e:ce:9d:cf:59:32:66:6c: + bb:0e:3b:1d:9c:05:d2:eb:a6:29:f8:74:4f:dc:83:3b:32:5a: + 2c:67:86:61:25:bc:bd:19:eb:20:c6:30:69:0e:4c:b2:e3:18: + ca:9e:fe:40:bc:1c:ad:8b:03:f5:04:be:90:ce:27:27:2f:83: + 14:57:8d:4f:a0:db:46:ce:e0:7d:e2:cf:7d:ea:0c:fd:8d:00: + 27:0a:db:0d:5f:e7:1e:52:25:1f:64:b9:30:5f:07:1a:10:a3: + 69:35:0e:dc:f8:23:f7:34:07:ce:c8:92:94:39:4d:d5:c3:ab: + 33:aa:f9:67:be:66:18:ac:67:14:5f:93:5f:68:48:04:ed:1e: + c9:74:28:b2:47:34:49:11:e4:7b:38:32:e5:dc:40:13:b4:69: + 75:39:43:db:7c:4a:f0:2b:94:cd:01:ba:4d:9b:9e:68:b3:ee: + 03:9e:7f:9c:0c:cf:9c:5c:cb:d4:33:d5:f0:e3:21:54:9a:13: + 6f:eb:1a:0f:f3:8b:e8:ef:eb:34:ba:09:77:39:2a:8a:4b:e1: + 7e:9f:b5:05:be:95:b6:92:5d:4c:35:47:38:64:38:5e:27:b8: + f9:34:94:2f:57:16:b0:f5:6a:21:3f:09:34:b9:dd:f8:d1:47: + 2c:c7:5e:7f:63:49:f4:5b:f4:d9:ea:66:fc:aa:64:27:f0:72: + d7:94:6f:86:0f:e7:3b:b3:d4:d9:30:67:b8:a2:c3:f7:4d:07: + 44:b3:70:67:dd:b1:21:ac:7c:2a:04:7b:2c:1d:df:0b:82:a9: + fb:df:88:72:47:1c:f5:5d:a3:f7:52:22:2d:ea:f4:2a:45:4f: + 9b:9d:63:95:59:f3:79:05:2b:f1:5b:3b:62:71:69:90:30:d7: + 7a:b2:c8:ec:68:e5:94:bb:97:00:d0:95:a7:fd:04:6c:f7:8b: + 28:c1:96:9b:6a:94 -----BEGIN CERTIFICATE----- -MIIGvTCCBSWgAwIBAgIJAMstgJlaaVJgMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIGvzCCBSegAwIBAgIJAMstgJlaaVJgMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMF0xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2lk -bnNhbnMwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQC8sqplTuHuLjbW -TL5SL2D1fw9U6WQzLVAF5gsyhd5lr2FpfYwjrob5Mav91aOLbJRTvoNyXsJ26FPS -0RycRGXbomcIEJxXGy9aI+0MLYBt1G5mgqCH+HcVCwPzCNlhVnTwvpgA7y8zs3+6 -ezZAPWkF0yWOMYLtTcq9A5GWeavt5VMgm1KZF3gO4k58oPyk3Ae9D0LAaYsX6DFi -BYx41eUR5UbSb5IYXaDd8d6jqW/jnYhgc6Cxkv1gTJFn87V5lrG0vYMSRUtWDQ9Y -Jh/EKAxjGw7AeY429p6TE4UoJhDmoFYR2NLvawhNIplxol/v0fs0veFQjI/UsTD8 -2tRfnYL4IX8szhLsE5/5Iq8aiLHjVbIMwmDYAa0P63Ap2kf1biSn9mpDL8lQazSo -yr8xzIq2QS5HMvGbeMAmS0ih10Zx84uVmkWlavgvtSflw8K/ZXT9c70rZp/TdBGY -95cOFsbg5U/20M/Llpis9tcBCaoVaYSFupatrP+p8y19qP2nebsCAwEAAaOCAo4w -ggKKMIHhBgNVHREEgdkwgdaCB2lkbnNhbnOCH3huLS1rbmlnLTVxYS5pZG4ucHl0 -aG9udGVzdC5uZXSCLnhuLS1rbmlnc2dzc2NoZW4tbGNiMHcuaWRuYTIwMDMucHl0 -aG9udGVzdC5uZXSCLnhuLS1rbmlnc2djaGVuLWI0YTNkdW4uaWRuYTIwMDgucHl0 -aG9udGVzdC5uZXSCJHhuLS1ueGFzbXE2Yi5pZG5hMjAwMy5weXRob250ZXN0Lm5l -dIIkeG4tLW54YXNtbTFjLmlkbmEyMDA4LnB5dGhvbnRlc3QubmV0MA4GA1UdDwEB -/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/ -BAIwADAdBgNVHQ4EFgQUXL4Yf3s/zplmgHlTS90zG0KlfgAwfQYDVR0jBHYwdIAU -s4qgorpx8agkedSkWyU2FR5JyM2hUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQK -DB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNh -LXNlcnZlcoIJAMstgJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKG -MGh0dHA6Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNl -cjA1BggrBgEFBQcwAYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0 -Y2Evb2NzcC8wQwYDVR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250 -ZXN0Lm5ldC90ZXN0Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGB -AF16+IHgp8E/OevTUizhy00ps3cYFxieEvwRzDxJy2v0TWy40vTpN/hr7fXX8eta -QQTH84za4QWOrlhx2QGKMkay3ZVG4c6CBPoLHCl1B4XOzVnUzPNWs3JNy5APzgIh -zl0XhJZ/agBXQrckWwclHneondpBCY4pebSwoUXIcK4shiSuPZp0pwR41h8bF8XB -bbEa/fRQLmEWhInQQj+2v71Svcg+joe08L2tx1FlL3foaXkOA2OJ53CtyNEvGqUG -0pDbfAc1mgsOhYfRcBeniA/GtZyIAPr5sgoZWkuNkRJRXg7B2J4CeNAtJAn+1Jc8 -y6Afmqv3D+L6ZCNOUwoVPvUEAYYpi44kQC+xkIdcO3unTAavw5B/6cZWQmEVLIPx -fE+JF/OgETQ/ja91NGAe4PLzAueqs/efHPhp9P7aV24blVNwze22uyqE66vDqbTV -FaCyzIEt8VbBVJtfFExfrV/1Bu4iYEXkUDVkrKzKSr+GePhTLRfY6ITIB6TCKXbH -Hw== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowXTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEQMA4GA1UEAwwH +aWRuc2FuczCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAN4YbKyo2qzP +rj7Gw34JRkN8KA8+HNQrSUNSWQSSnRl4V5Cbf6mjZjzaC8khVqU7kKGJeM/FKAnC +i2hId7hBnx9YCWRfV3X0DzGON63HUrd2pmy2p7eY8/sdC98mVGSYC1WEVpSPX5JT +H0qzgcHguTaGopSirdOVEyl5J8lNdodOFU/spk9YHN8zul2kZifOcGnGB/KY2ulN +xc0KN5HqVSWP9MTEeJguPgTRo4NH9JTPS0E10JBFxEPD47JER7pGTfYquXfg+vV0 +0p4RHS26WEzbBEWyrDRry82zXcjfQEfhpcCvlsaAsIXa1ds/cqh6hH8aRBJZXXgq +uFPGEoPdAQnyAeOgdgqvgh1wVhxNqRsj0TdMexCNBNQwn9fKDNrs46vqk08pjXS1 +h+ntaHLJjtwH8L+sedRjR/nwhmVF0xiBwsJSXvDNTFJUQzKv0CvLxdlmXG47rb2T +a50AgbAiLZNWr5QrSQ09nIaJwYcx4SGg1zdgMwH/Nh6oyvgqvIj6aQIDAQABo4IC +jjCCAoowgeEGA1UdEQSB2TCB1oIHaWRuc2Fuc4IfeG4tLWtuaWctNXFhLmlkbi5w +eXRob250ZXN0Lm5ldIIueG4tLWtuaWdzZ3NzY2hlbi1sY2Iwdy5pZG5hMjAwMy5w +eXRob250ZXN0Lm5ldIIueG4tLWtuaWdzZ2NoZW4tYjRhM2R1bi5pZG5hMjAwOC5w +eXRob250ZXN0Lm5ldIIkeG4tLW54YXNtcTZiLmlkbmEyMDAzLnB5dGhvbnRlc3Qu +bmV0giR4bi0tbnhhc21tMWMuaWRuYTIwMDgucHl0aG9udGVzdC5uZXQwDgYDVR0P +AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB +Af8EAjAAMB0GA1UdDgQWBBTIBpm36I/sTz1ciWoG9XcusOBqnjB9BgNVHSMEdjB0 +gBTACisoQ95fyX1H5UebNvJljGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNV +BAoMHVB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXIt +Y2Etc2VydmVyggkAyy2AmVppUlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcw +AoYwaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQu +Y2VyMDUGCCsGAQUFBzABhilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rl +c3RjYS9vY3NwLzBDBgNVHR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhv +bnRlc3QubmV0L3Rlc3RjYS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOC +AYEAQNFtjqILkUuoxAjQ8/mL0KML3AAijPEuK+XmtG7Onc9ZMmZsuw47HZwF0uum +Kfh0T9yDOzJaLGeGYSW8vRnrIMYwaQ5MsuMYyp7+QLwcrYsD9QS+kM4nJy+DFFeN +T6DbRs7gfeLPfeoM/Y0AJwrbDV/nHlIlH2S5MF8HGhCjaTUO3Pgj9zQHzsiSlDlN +1cOrM6r5Z75mGKxnFF+TX2hIBO0eyXQoskc0SRHkezgy5dxAE7RpdTlD23xK8CuU +zQG6TZueaLPuA55/nAzPnFzL1DPV8OMhVJoTb+saD/OL6O/rNLoJdzkqikvhfp+1 +Bb6VtpJdTDVHOGQ4Xie4+TSUL1cWsPVqIT8JNLnd+NFHLMdef2NJ9Fv02epm/Kpk +J/By15Rvhg/nO7PU2TBnuKLD900HRLNwZ92xIax8KgR7LB3fC4Kp+9+Ickcc9V2j +91IiLer0KkVPm51jlVnzeQUr8Vs7YnFpkDDXerLI7GjllLuXANCVp/0EbPeLKMGW +m2qU -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert.passwd.pem b/Lib/test/certdata/keycert.passwd.pem index c330c36d8f9fde..1739a3525fee2b 100644 --- a/Lib/test/certdata/keycert.passwd.pem +++ b/Lib/test/certdata/keycert.passwd.pem @@ -1,69 +1,69 @@ -----BEGIN ENCRYPTED PRIVATE KEY----- -MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQIhD+rJdxqb6ECAggA -MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBDTdyjCP3riOSUfxix4aXEvBIIH -ECGkbsFabrcFMZcplw5jHMaOlG7rYjUzwDJ80JM8uzbv2Jb8SvNlns2+xmnEvH/M -mNvRmnXmplbVjH3XBMK8o2Psnr2V/a0j7/pgqpRxHykG+koOY4gzdt3MAg8JPbS2 -hymSl+Y5EpciO3xLfz4aFL1ZNqspQbO/TD13Ij7DUIy7xIRBMp4taoZCrP0cEBAZ -+wgu9m23I4dh3E8RUBzWyFFNic2MVVHrui6JbHc4dIHfyKLtXJDhUcS0vIC9PvcV -jhorh3UZC4lM+/jjXV5AhzQ0VrJ2tXAUX2dA144XHzkSH2QmwfnajPsci7BL2CGC -rjyTy4NfB/lDwU+55dqJZQSKXMxAapJMrtgw7LD5CKQcN6zmfhXGssJ7HQUXKkaX -I1YOFzuUD7oo56BVCnVswv0jX9RxrE5QYNreMlOP9cS+kIYH65N+PAhlURuQC14K -PgDkHn5knSa2UQA5tc5f7zdHOZhGRUfcjLP+KAWA3nh+/2OKw/X3zuPx75YT/FKe -tACPw5hjEpl62m9Xa0eWepZXwqkIOkzHMmCyNCsbC0mmRoEjmvfnslfsmnh4Dg/c -4YsTYMOLLIeCa+WIc38aA5W2lNO9lW0LwLhX1rP+GRVPv+TVHXlfoyaI+jp0iXrJ -t3xxT0gaiIR/VznyS7Py68QV/zB7VdqbsNzS7LdquHK1k8+7OYiWjY3gqyU40Iu2 -d1eSnIoDvQJwyYp7XYXbOlXNLY+s1Qb7yxcW3vXm0Bg3gKT8r1XHWJ9rj+CxAn5r -ysfkPs1JsesxzzQjwTiDNvHnBnZnwxuxfBr26ektEHmuAXSl8V6dzLN/aaPjpTj4 -CkE7KyqX3U9bLkp+ztl4xWKEmW44nskzm0+iqrtrxMyTfvvID4QrABjZL4zmWIqc -e3ZfA3AYk9VDIegk/YKGC5VZ8YS7ZXQ0ASK652XqJ7QlMKTxxV7zda6Fp4uW6/qN -ezt5wgbGGhZQXj2wDQmWNQYyG/juIgYTpCUA54U5XBIjuR6pg+Ytm0UrvNjsUoAC -wGelyqaLDq8U8jdIFYVTJy9aJjQOYXjsUJ0dZN2aGHSlju0ZGIZc49cTIVQ9BTC5 -Yc0Vlwzpl+LuA25DzKZNSb/ci0lO/cQGJ2uXQQgaNgdsHlu8nukENGJhnIzx4fzK -wEh3yHxhTRCzPPwDfXmx0IHXrPqJhSpAgaXBVIm8OjvmMxO+W75W4uLfNY/B7e2H -3cjklGuvkofOf7sEOrGUYf4cb6Obg8FpvHgpKo5Twwmoh/qvEKckBFqNhZXDDl88 -GbGlSEgyaAV1Ig8s1NJKBolWFa0juyPAwJ8vT1T4iwW7kQ7KXKt2UNn96K/HxkLu -pikvukz8oRHMlfVHa0R48UB1fFHwZLzPmwkpu6ancIxk3uO3yfhf6iDk3bmnyMlz -g3k/b6MrLYaOVByRxay85jH3Vvgqfgn6wa6BJ7xQ81eZ8B45gFuTH0J5JtLL7SH8 -darRPLCYfA+Ums9/H6pU5EXfd3yfjMIbvhCXHkJrrljkZ+th3p8dyto6wmYqIY6I -qR9sU+o6DhRaiP8tCICuhHxQpXylUM6WeJkJwduTJ8KWIvzsj4mReIKOl/oC2jSd -gIdKhb9Q3zj9ce4N5m6v66tyvjxGZ+xf3BvUPDD+LwZeXgf7OBsNVbXzQbzto594 -nbCzPocFi3gERE50ru4K70eQCy08TPG5NpOz+DDdO5vpAuMLYEuI7O3L+3GjW40Q -G5bu7H5/i7o/RWR67qhG/7p9kPw3nkUtYgnvnWaPMIuTfb4c2d069kjlfgWjIbbI -tpSKmm5DHlqTE4/ECAbIEDtSaw9dXHCdL3nh5+n428xDdGbjN4lT86tfu17EYKzl -ydH1RJ1LX3o3TEj9UkmDPt7LnftvwybMFEcP7hM2xD4lC++wKQs7Alg6dTkBnJV4 -5xU78WRntJkJTU7kFkpPKA0QfyCuSF1fAMoukDBkqUdOj6jE0BlJQlHk5iwgnJlt -uEdkTjHZEjIUxWC6llPcAzaPNlmnD45AgfEW+Jn21IvutmJiQAz5lm9Z9PXaR0C8 -hXB6owRY67C0YKQwXhoNf6xQun2xGBGYy5rPEEezX1S1tUH5GR/KW1Lh+FzFqHXI -ZEb5avfDqHKehGAjPON+Br7akuQ125M9LLjKuSyPaQzeeCAy356Xd7XzVwbPddbm -9S9WSPqzaPgh10chIHoNoC8HMd33dB5j9/Q6jrbU/oPlptu/GlorWblvJdcTuBGI -IVn45RFnkG8hCz0GJSNzW7+70YdESQbfJW79vssWMaiSjFE0pMyFXrFR5lBywBTx -PiGEUWtvrKG94X1TMlGUzDzDJOQNZ9dT94bonNe9pVmP5BP4/DzwwiWh6qrzWk6p -j8OE4cfCSh2WvHnhJbH7/N0v+JKjtxeIeJ16jx/K2oK5 +MIIHdTBfBgkqhkiG9w0BBQ0wUjAxBgkqhkiG9w0BBQwwJAQQEKGM7Z40p3TpyCvl +LTPLsQICCAAwDAYIKoZIhvcNAgkFADAdBglghkgBZQMEASoEEJAwQy7HESbbXMoF +KXadaz8EggcQBlxqFXmWQ3YlUuWTTc+dUcmikIUmI8gUSB2doBxQWLkTzaNuQ/hj +uV/eU7VkNFswxrU1j+EEkoCAtClWrQ2aKTyQSM4YfhgN1vQJdOD8DypHtd0TaRS3 +VCSkJODqcWaCEc/Ypgb1KnB6UWhit1waPq4NVCZdiSF/ueRK9/iMlIRaejg61Qn+ +LVLcegWw47zTtCZKuZqFAO3PcvfC0cresGc6hrQrb9tjUyI9qCR6MeZSrHUJQ/Pb +T9e7OREQxCLxWEUb0/tMJH5k9HoG7waU5lELOyjGKXsflEIs+uBaWIssWp69KGLi +yas7BHZc8kxJNtigFQlmECZjoKvRxJWN4fOknxI8T+ttBovOtsIlVvocBfmH+Z2m +7a5VMBxD8Phk6zU4Pk0L2S6EIUCIYGS+kgHibgxZ58/QcoDZBrKOaJ/Md+Lhk2Rn +2JCuPe8CBTkB3V+xAJdaz7JAcgxz4TqwXFXA249lfYb8qY9XXcr67O83Cxwb95lu +skmxnhAKOwKIrhS1nKiefuwN5qOiA+nuPbJFadCOO5cdOSriG3s80ugDDvlPN8k6 +4b7XLulB38R2PdH/OHuF05QGQ3kOUbJWatMig+/09LNo9FiMaCczdIXQ6LGxcHc1 +G1Y3BqK17z3yULqHvxU4tIbj5Elt+X1mfKIfifmXBUujbz5rR1pBTfpnk+DFGcHZ +B/cXWFP0tDJAE7mvna3HimNDQMP58NbhZwFtFdU3H9R5n4R5MsYQ6+a+amQ0pZ1Y +XAaQ/iEg0WY8CEFO8zEvB9R05zMm6vycNtN8CbfBq9CJ0OaR1ymMW8pRVagsLDuh +D0T5ZtWZctE2+ImlwlpGDs91CX3zDxvW/3Bwf40PF4x7LJMt/tFzqQEovokvzhUw +0jX/kf42QhvydnDoxqczdVZOfbjHELA21U+JAeAAj9jhcCEYd50c2BVt4jhST6dP +pzNAqxc1RGwTU3K3nZkWKt4hMXPDGjb0pVqiOuPz/718Nsd6ck+Ko90gFX13mPLC +6FLPGjNUmE6f3TKjbRrQz3+IWMncCyo/JSWA85rldxD0XAem64h+2XppFWJsxJD5 +nH2prLckQUWBLljWNIQQyVuAWo0TVq3NCXDFGaP6GHNrZo9zZxDhxcp6v2Me3Mir +few9aQNb59Q1/A0qwiCf26Oe8JYTD14UCez41imxD+w721SO8jzLR80+ReemdwzE +B9fXh3T/lrCIDNB+F+GWA9wQUz8mzArKbhqQhl5gk39dnLahc4aN7GrkwL4hjhKW +O3X5bBdhhRlcub9SbXFC+tx/w6G/0roGMSvD7h8G/mw3QcUUwwDvOuI7qAqQ8FC+ +xcGWeHUiTGxp2t2gz8DSaeL2TbBMbHe+tRBmKlmmIrAq3AEllWzyze9ta5ZYt46P +njZzS+vbbA0mfcFAId7tQOjQ2/ygZePs5hg91revLzNMDRJ1IeBHnrVUuDA5D1HR +iFW+hhj1Lx1s0XAJBJcDos8xy/LZlRDLxJSHrMktZOELTYh7UZw4S2PNkOPms4kP +V4G6D/0wZh0t+T9zpH57ivLdyBQ2fsFwx3X3JfQkYmoFlwCz+uUb368TunQ/yZZY +fVh9Q9ndk6y6aor0wkORsBefVv3eN+2BDt6ZUpYu3Liim+W4C/SiKVfQ+5yDb/qe +KmX7kcmr/f13BWvEOUN4fzCMiw/N0aAgEejZQTwMgivvGSMcNaBiLfdvM6hmy7NE +uvgpLtK5v+YRcOLwiaVH7UL6Q61CJ+pcDNRvbJT/iVwidYUO16zTVR6NrtPhM62a +Ziq3Q77HD4D2KCRPEo3Xivc5RwITHeDgKHwpLRB00jiZ3iR0Wm5lI5VbyWLnmQ4x +CDPAZetJZGBkbhTmUVbL0m3HNLHfF9g+LWA23L9mRNCKRemL0c0UJRRJPaNDIx8p +8h27al4xI8nFav0TZLxmSw7mqnzyE7YAoe4EsmedOSXktHWN06dnczHu8AWrgNjY +Izv9XUmy9V5vsRG/lg09FyG/eZSgfs9rdmL/qeFXbbl2J33YTqrt6O3ZaEsgg42r +piZztung8Xro3VlFSAXZGrl7Z0AWwzaOvwreCxbAG1WtBmGgLrs98z627XgfBFYA +BPvJwn9f2GZzixiN6M8c2M5XueDE/Vpn4A/GKLJ8RXXxKtbGRXFfdF/M6mEgcpa1 +9pxAhzNTPaOp+SFbm/cFa43HZYZgDg1D9zth2ZII0ZITd5OpEaSNOsrOrhqXeVQU +isBpybgPqVQ50xUuRUyoHYAZClVe6+PBYVbvlXBlTWhSPY3leUm4AYt+fjZ4VFlb +p4I2KRmd68zXXwl2spkEsC96e9k28kHta55dO32gSwaXUUHRKnFmKB24jsleF6n3 +BOKxzsO9jr6vi0jUqZXvaCnmYWqS84wZH/1S1pgotaWdMQ/KT27xwHk= -----END ENCRYPTED PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIEWTCCAsGgAwIBAgIJAJinz4jHSjLtMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xODA4 -MjkxNDIzMTVaFw0yODA4MjYxNDIzMTVaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH -DA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5k -YXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDCCAaIwDQYJKoZIhvcNAQEBBQADggGP -ADCCAYoCggGBALKUqUtopT6E68kN+uJNEt34i2EbmG/bwjcD8IaMsgJPSsMO2Bpd -3S6qWgkCeOyCfmAwBxK2kNbxGb63ouysEv7l8GCTJTWv3hG/HQcejJpnAEGi6K1U -fDbyE/db6yZ12SoHVTGkadN4vYGCPd1Wj9ZO1F877SHQ8rDWX3xgTWkxN2ojBw44 -T8RHSDiG8D/CvG4uEy+VUszL+Uvny5y2poNSqvI3J56sptWSrh8nIIbkPZPBdUne -LYMOHTFK3ZjXSmhlXgziTxK71nnzM3Y9K9gxPnRqoXbvu/wFo55hQCkETiRkYgmm -jXcBMZ0TClQVnQWuLjMthRnWFZs4Lfmwqjs7FZD/61581R2BYehvpWbLvvuOJhwv -DFzexL2sXcAl7SsxbzeQKRHqGbIDfbnQTXfs3/VC6Ye5P82P2ucj+XC32N9piRmO -gCBP8L3ub+YzzdxikZN2gZXXE2jsb3QyE/R2LkWdWyshpKe+RsZP1SBRbHShUyOh -yJ90baoiEwj2mwIDAQABoxgwFjAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJKoZI -hvcNAQELBQADggGBAHRUO/UIHl3jXQENewYayHxkIx8t7nu40iO2DXbicSijz5bo -5//xAB6RxhBAlsDBehgQP1uoZg+WJW+nHu3CIVOU3qZNZRaozxiCl2UFKcNqLOmx -R3NKpo1jYf4REQIeG8Yw9+hSWLRbshNteP6bKUUf+vanhg9+axyOEOH/iOQvgk/m -b8wA8wNa4ujWljPbTQnj7ry8RqhTM0GcAN5LSdSvcKcpzLcs3aYwh+Z8e30sQWna -F40sa5u7izgBTOrwpcDm/w5kC46vpRQ5fnbshVw6pne2by0mdMECASid/p25N103 -jMqTFlmO7kpf/jpCSmamp3/JSEE1BJKHwQ6Ql4nzRA2N1mnvWH7Zxcv043gkHeAu -0x8evpvwuhdIyproejNFlBpKmW8OX7yKTCPPMC/VkX8Q1rVkxU0DQ6hmvwZlhoKa -9Wc2uXpw9xF8itV4Uvcdr3dwqByvIqn7iI/gB+4l41e0u8OmH2MKOx4Nxlly5TNW -HcVKQHyOeyvnINuBAQ== +MIIEhTCCAu2gAwIBAgIUTxhwRX6zBQc3+l3imDklEbqcDpIwDQYJKoZIhvcNAQEL +BQAwXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD +VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxo +b3N0MCAXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWjBfMQswCQYDVQQG +EwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDDAlsb2NhbGhvc3QwggGiMA0GCSqG +SIb3DQEBAQUAA4IBjwAwggGKAoIBgQDwcIAYm12nmQTGB3caFn7alDe3LSliEfNC +2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3zmrI/FAiQI/RrUHyBZiEt +nFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVwg9cR0khTqD5cg2jvTB05 +yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYliGx5qqCQmsXeTsxCpvQD7 +u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAaF48xEnokxenzBIqZ82BR +kFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLYsFXUp8L7ZMVE27Bej+Sq +4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11tgtctsYbwgDSUYGA3w+DC +KD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3RPJKV+Db8E1V2mXmNE0M +Lg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAaM3MDUwFAYDVR0RBA0wC4IJbG9j +YWxob3N0MB0GA1UdDgQWBBR459BlAel5MqCtG0DrvVtMZlQfuTANBgkqhkiG9w0B +AQsFAAOCAYEAaTFWjK/LFzOo+0TWqTTj4WC4N3I8JFrHnlqFJlpchYTW2z92SU1G +iEzFjWzuDNjp5KM9BqlmGtzXZvy6MItGkYsjPRdPVU0rbCmyTho6y77kTyiEG12V +UAJ1in3FOQfDwLPcp7wQRgCQq3iZlv7pwXp2Lm5fzu8kZPnxmTVdiKQun9Ps7uKq +BoM0fM2K14MxVO4Wc0SERnaPszE7xAhkIcs+NRT/gHYdTBlPhao83S3LOOdtCqCP +pNUOEaShlwI5bVsDPUXNX/eS0MYFNlsYTb5rCxK8jf3W3KNjKTOzN94pHiQOhpkg +xMPPi3m03/9oKTVXBtHI2A+u3ukheKE6sBXCLdv/GEs9zYI49zmpQxNWz5EOumWL +k+W/vPv7cD6LeHxxp+nCbEJi1gZtYb3gMY1sLkMNxcOu0QHTqHfme+k7VKWm8anO +3ogGdGtPuPAD/qjMwg3ChSDLl5Ur/E9UPlD4yM/7KtUD7mLv+jbddA62EiA9MxVB +t+yt7pOwOA66 -----END CERTIFICATE----- - diff --git a/Lib/test/certdata/keycert.pem b/Lib/test/certdata/keycert.pem index 0d398633739a51..43ad52c2af507b 100644 --- a/Lib/test/certdata/keycert.pem +++ b/Lib/test/certdata/keycert.pem @@ -1,66 +1,67 @@ -----BEGIN PRIVATE KEY----- -MIIG/wIBADANBgkqhkiG9w0BAQEFAASCBukwggblAgEAAoIBgQCylKlLaKU+hOvJ -DfriTRLd+IthG5hv28I3A/CGjLICT0rDDtgaXd0uqloJAnjsgn5gMAcStpDW8Rm+ -t6LsrBL+5fBgkyU1r94Rvx0HHoyaZwBBouitVHw28hP3W+smddkqB1UxpGnTeL2B -gj3dVo/WTtRfO+0h0PKw1l98YE1pMTdqIwcOOE/ER0g4hvA/wrxuLhMvlVLMy/lL -58uctqaDUqryNyeerKbVkq4fJyCG5D2TwXVJ3i2DDh0xSt2Y10poZV4M4k8Su9Z5 -8zN2PSvYMT50aqF277v8BaOeYUApBE4kZGIJpo13ATGdEwpUFZ0Fri4zLYUZ1hWb -OC35sKo7OxWQ/+tefNUdgWHob6Vmy777jiYcLwxc3sS9rF3AJe0rMW83kCkR6hmy -A3250E137N/1QumHuT/Nj9rnI/lwt9jfaYkZjoAgT/C97m/mM83cYpGTdoGV1xNo -7G90MhP0di5FnVsrIaSnvkbGT9UgUWx0oVMjocifdG2qIhMI9psCAwEAAQKCAYBT -sHmaPmNaZj59jZCqp0YVQlpHWwBYQ5vD3pPE6oCttm0p9nXt/VkfenQRTthOtmT1 -POzDp00/feP7zeGLmqSYUjgRekPw4gdnN7Ip2PY5kdW77NWwDSzdLxuOS8Rq1MW9 -/Yu+ZPe3RBlDbT8C0IM+Atlh/BqIQ3zIxN4g0pzUlF0M33d6AYfYSzOcUhibOO7H -j84r+YXBNkIRgYKZYbutRXuZYaGuqejRpBj3voVu0d3Ntdb6lCWuClpB9HzfGN0c -RTv8g6UYO4sK3qyFn90ibIR/1GB9watvtoWVZqggiWeBzSWVWRsGEf9O+Cx4oJw1 -IphglhmhbgNksbj7bD24on/icldSOiVkoUemUOFmHWhCm4PnB1GmbD8YMfEdSbks -qDr1Ps1zg4mGOinVD/4cY7vuPFO/HCH07wfeaUGzRt4g0/yLr+XjVofOA3oowyxv -JAzr+niHA3lg5ecj4r7M68efwzN1OCyjMrVJw2RAzwvGxE+rm5NiT08SWlKQZnkC -gcEA4wvyLpIur/UB84nV3XVJ89UMNBLm++aTFzld047BLJtMaOhvNqx6Cl5c8VuW -l261KHjiVzpfNM3/A2LBQJcYkhX7avkqEXlj57cl+dCWAVwUzKmLJTPjfaTTZnYJ -xeN3dMYjJz2z2WtgvfvDoJLukVwIMmhTY8wtqqYyQBJ/l06pBsfw5TNvmVIOQHds -8ASOiFt+WRLk2bl9xrGGayqt3VV93KVRzF27cpjOgEcG74F3c0ZW9snERN7vIYwB -JfrlAoHBAMlahPwMP2TYylG8OzHe7EiehTekSO26LGh0Cq3wTGXYsK/q8hQCzL14 -kWW638vpwXL6L9ntvrd7hjzWRO3vX/VxnYEA6f0bpqHq1tZi6lzix5CTUN5McpDg -QnjenSJNrNjS1zEF8WeY9iLEuDI/M/iUW4y9R6s3WpgQhPDXpSvd2g3gMGRUYhxQ -Xna8auiJeYFq0oNaOxvJj+VeOfJ3ZMJttd+Y7gTOYZcbg3SdRb/kdxYki0RMD2hF -4ZvjJ6CTfwKBwQDiMqiZFTJGQwYqp4vWEmAW+I4r4xkUpWatoI2Fk5eI5T9+1PLX -uYXsho56NxEU1UrOg4Cb/p+TcBc8PErkGqR0BkpxDMOInTOXSrQe6lxIBoECVXc3 -HTbrmiay0a5y5GfCgxPKqIJhfcToAceoVjovv0y7S4yoxGZKuUEe7E8JY2iqRNAO -yOvKCCICv/hcN235E44RF+2/rDlOltagNej5tY6rIFkaDdgOF4bD7f9O5eEni1Bg -litfoesDtQP/3rECgcEAkQfvQ7D6tIPmbqsbJBfCr6fmoqZllT4FIJN84b50+OL0 -mTGsfjdqC4tdhx3sdu7/VPbaIqm5NmX10bowWgWSY7MbVME4yQPyqSwC5NbIonEC -d6N0mzoLR0kQ+Ai4u+2g82gicgAq2oj1uSNi3WZi48jQjHYFulCbo246o1NgeFFK -77WshYe2R1ioQfQDOU1URKCR0uTaMHClgfu112yiGd12JAD+aF3TM0kxDXz+sXI5 -SKy311DFxECZeXRLpcC3AoHBAJkNMJWTyPYbeVu+CTQkec8Uun233EkXa2kUNZc/ -5DuXDaK+A3DMgYRufTKSPpDHGaCZ1SYPInX1Uoe2dgVjWssRL2uitR4ENabDoAOA -ICVYXYYNagqQu5wwirF0QeaMXo1fjhuuHQh8GsMdXZvYEaAITZ9/NG5x/oY08+8H -kr78SMBOPy3XQn964uKG+e3JwpOG14GKABdAlrHKFXNWchu/6dgcYXB87mrC/GhO -zNwzC+QhFTZoOomFoqMgFWujng== +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDwcIAYm12nmQTG +B3caFn7alDe3LSliEfNC2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3z +mrI/FAiQI/RrUHyBZiEtnFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVw +g9cR0khTqD5cg2jvTB05yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYli +Gx5qqCQmsXeTsxCpvQD7u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAa +F48xEnokxenzBIqZ82BRkFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLY +sFXUp8L7ZMVE27Bej+Sq4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11t +gtctsYbwgDSUYGA3w+DCKD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3 +RPJKV+Db8E1V2mXmNE0MLg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAQKCAYAK +Ap0KqTlCd4fv2LK4NtSMNByHt00gRKAMmfNstJ12UKoxBLFjXOXaHjWv5PYkh4bz +vjo7pBHMCWnDuR6Pqr1ahuyvpRex6XcbJ4VebsaOKYO6+gphlm2C2ZqCQ1Vh6Akd +aZ40Wb1gfgK/zvVezBLvzLLf9iahw9j5pWZ2iDci5zdUuvd9Sn+qUB3+nyRf/NW5 +MXgBsp07zIcOOxPOm/Z5V+0jDJL2hiRq1pbmUKClTShcgqtfJKU//niF+6ZQAuiJ +LBPaKIdPXyxLYnkyq2IgjPU0ZwxzdP0a2k72kvImd25Daj7elhGr3++IR+nFzt6h +vqflOfmKDF3zZPyUVI3YXjxo/FrOwGbLMHuuHBgE9txH/mOl1gByrxP+Ax18i3Bf +spSLeUvtaf/w/MopyspPoJBRbAM06PUHQI2v9xq3BZL/gHe2CdJPds2WzpaaVFG4 +oJWNrE3s6CowLqUkqzB7LqJ4ReZ6xe6SpkRotdmVknlIKgDenTFeEUEEVyBiFQEC +gcEA/F1GAaBG0e9vB+AOHZ96SLlZVzObSBYq2kVwUhhGItNnyU9c3fWPIrGREKQa +lw5dsvjl58ij5uEtJoPZf5BsJ0q6xHAs/kKxfpNfZAeoKAV96Z6MVcY+6WOyGjPF +aQo+GgSrCPIciW//WXZrWI1t0M2G0vZ5CFNohnKod+cSgV03PAActlyM2H+r7dtm +MpAD3EPWeeA75saKj/x0SOzuL/wzXKR8BZ6CINZ6r61Tcbk2mDwOHPhUrHeCwjoU +nhy5AoHBAPPnP2FSXFCPXD1Z1hFInCFgm41j7LEyBTVLlnqUrRk7i18fi/WcwwLH ++XvM5DcONY/V3sh7a3tZeHN1P70tRxLE0oO51D4tP5im/oZ6L+hszSYXX7lCbJSR +tni6nU1dssW3nmswfUn01Oh+B0rBGon3RQB6x4beTAW0piVxg9Ic2HYucS1Scrqw +afiFQ5KWklnMYJKInPFzlCwMdgBCuue1dZoJstU9nLQALNNSpGXB2X0+7j9D/qkz +Caw5MfgQwQKBwQDzdCvP78XCSuBq0XvsmefG9n+4fwGDFld6v9gualpmyFjsPJKT +UYwm5PPUAOvh46sCt9hatRVg6sO6zyFoTXP4p7/rN2hAVSiTuiog/r369elVEW3C +ZYBVeKbdXipIPehRA0XYWHCtKY1Fydae07kn4M37AGkcXhKM+VmKajFQ+RMK3/TS +/A+n3+qFiM1bY9FFkW/7nRVMeSY850dq/p59TihibA91AEf6084BYg0IvatsSys2 +SV6uDpDnPE6dhYkCgcBECtAwq1RbmRLnfqdsnPAJk7Txhd3jNQwk6RhqzA1aS7U+ +7UMTWw9AOF+OPQOxpEInBUgob931RGmI9D263eXFA6mi2/Ws/tyODpBVHcM9uRSm +OsEWosQ90kSwe4ckrS4RYH9OcfGR7z5yOa55GVP5B0V1s8r0AhH9SX9MVNWsiSWO +GriyJx0gndSCY1MNkvnzGUQbvQbjiRXeD//fZL5Vo9bSCUCdopmT0bSvo49/X8v3 +19WJSsPBmh5psG8TQEECgcEA64CqZpPux35LeLQsKe0fYeNfAncqiaIoRbAvxKCi +SQf27SD8HK+sfvhvYY7bP7TMEeM7B/O2/AqBQQP0UARIGJg2AknBQT0A7//yJu+o +v4FHy2XKh+RMAx7QrdvnQ4CfrjvjQIaAcN1HrdTKWwgQZZImRf57nUCMm82ktZ2k +vYEJTXMkT8CY0DSeGtPmX5ynk7cauHTdZrkPGhZ3Hr6GAFomOammnnytv2wc+5FA +Ap+d65UgF4KjGY4rtsS+jOHn -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIEWTCCAsGgAwIBAgIJAJinz4jHSjLtMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xODA4 -MjkxNDIzMTVaFw0yODA4MjYxNDIzMTVaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH -DA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5k -YXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDCCAaIwDQYJKoZIhvcNAQEBBQADggGP -ADCCAYoCggGBALKUqUtopT6E68kN+uJNEt34i2EbmG/bwjcD8IaMsgJPSsMO2Bpd -3S6qWgkCeOyCfmAwBxK2kNbxGb63ouysEv7l8GCTJTWv3hG/HQcejJpnAEGi6K1U -fDbyE/db6yZ12SoHVTGkadN4vYGCPd1Wj9ZO1F877SHQ8rDWX3xgTWkxN2ojBw44 -T8RHSDiG8D/CvG4uEy+VUszL+Uvny5y2poNSqvI3J56sptWSrh8nIIbkPZPBdUne -LYMOHTFK3ZjXSmhlXgziTxK71nnzM3Y9K9gxPnRqoXbvu/wFo55hQCkETiRkYgmm -jXcBMZ0TClQVnQWuLjMthRnWFZs4Lfmwqjs7FZD/61581R2BYehvpWbLvvuOJhwv -DFzexL2sXcAl7SsxbzeQKRHqGbIDfbnQTXfs3/VC6Ye5P82P2ucj+XC32N9piRmO -gCBP8L3ub+YzzdxikZN2gZXXE2jsb3QyE/R2LkWdWyshpKe+RsZP1SBRbHShUyOh -yJ90baoiEwj2mwIDAQABoxgwFjAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJKoZI -hvcNAQELBQADggGBAHRUO/UIHl3jXQENewYayHxkIx8t7nu40iO2DXbicSijz5bo -5//xAB6RxhBAlsDBehgQP1uoZg+WJW+nHu3CIVOU3qZNZRaozxiCl2UFKcNqLOmx -R3NKpo1jYf4REQIeG8Yw9+hSWLRbshNteP6bKUUf+vanhg9+axyOEOH/iOQvgk/m -b8wA8wNa4ujWljPbTQnj7ry8RqhTM0GcAN5LSdSvcKcpzLcs3aYwh+Z8e30sQWna -F40sa5u7izgBTOrwpcDm/w5kC46vpRQ5fnbshVw6pne2by0mdMECASid/p25N103 -jMqTFlmO7kpf/jpCSmamp3/JSEE1BJKHwQ6Ql4nzRA2N1mnvWH7Zxcv043gkHeAu -0x8evpvwuhdIyproejNFlBpKmW8OX7yKTCPPMC/VkX8Q1rVkxU0DQ6hmvwZlhoKa -9Wc2uXpw9xF8itV4Uvcdr3dwqByvIqn7iI/gB+4l41e0u8OmH2MKOx4Nxlly5TNW -HcVKQHyOeyvnINuBAQ== +MIIEhTCCAu2gAwIBAgIUTxhwRX6zBQc3+l3imDklEbqcDpIwDQYJKoZIhvcNAQEL +BQAwXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD +VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxo +b3N0MCAXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWjBfMQswCQYDVQQG +EwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDDAlsb2NhbGhvc3QwggGiMA0GCSqG +SIb3DQEBAQUAA4IBjwAwggGKAoIBgQDwcIAYm12nmQTGB3caFn7alDe3LSliEfNC +2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3zmrI/FAiQI/RrUHyBZiEt +nFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVwg9cR0khTqD5cg2jvTB05 +yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYliGx5qqCQmsXeTsxCpvQD7 +u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAaF48xEnokxenzBIqZ82BR +kFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLYsFXUp8L7ZMVE27Bej+Sq +4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11tgtctsYbwgDSUYGA3w+DC +KD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3RPJKV+Db8E1V2mXmNE0M +Lg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAaM3MDUwFAYDVR0RBA0wC4IJbG9j +YWxob3N0MB0GA1UdDgQWBBR459BlAel5MqCtG0DrvVtMZlQfuTANBgkqhkiG9w0B +AQsFAAOCAYEAaTFWjK/LFzOo+0TWqTTj4WC4N3I8JFrHnlqFJlpchYTW2z92SU1G +iEzFjWzuDNjp5KM9BqlmGtzXZvy6MItGkYsjPRdPVU0rbCmyTho6y77kTyiEG12V +UAJ1in3FOQfDwLPcp7wQRgCQq3iZlv7pwXp2Lm5fzu8kZPnxmTVdiKQun9Ps7uKq +BoM0fM2K14MxVO4Wc0SERnaPszE7xAhkIcs+NRT/gHYdTBlPhao83S3LOOdtCqCP +pNUOEaShlwI5bVsDPUXNX/eS0MYFNlsYTb5rCxK8jf3W3KNjKTOzN94pHiQOhpkg +xMPPi3m03/9oKTVXBtHI2A+u3ukheKE6sBXCLdv/GEs9zYI49zmpQxNWz5EOumWL +k+W/vPv7cD6LeHxxp+nCbEJi1gZtYb3gMY1sLkMNxcOu0QHTqHfme+k7VKWm8anO +3ogGdGtPuPAD/qjMwg3ChSDLl5Ur/E9UPlD4yM/7KtUD7mLv+jbddA62EiA9MxVB +t+yt7pOwOA66 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert.pem.reference b/Lib/test/certdata/keycert.pem.reference new file mode 100644 index 00000000000000..5528b98fff9377 --- /dev/null +++ b/Lib/test/certdata/keycert.pem.reference @@ -0,0 +1,13 @@ +{'issuer': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'notAfter': 'Jan 29 11:51:12 2408 GMT', + 'notBefore': 'Oct 8 11:51:12 2024 GMT', + 'serialNumber': '4F1870457EB3050737FA5DE298392511BA9C0E92', + 'subject': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'subjectAltName': (('DNS', 'localhost'),), + 'version': 3} diff --git a/Lib/test/certdata/keycert2.pem b/Lib/test/certdata/keycert2.pem index e59d45439d4b6d..63e2181ed83e78 100644 --- a/Lib/test/certdata/keycert2.pem +++ b/Lib/test/certdata/keycert2.pem @@ -1,66 +1,67 @@ -----BEGIN PRIVATE KEY----- -MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQCf8FWxi4oVlDVx -e8NDFgb+IYAGr/hZWuY1Zq7d7g57yPoxJrgt+bN89+U7qTduqyB2Hy8G0TqeACOr -IdpPZ8P7V5E5YiASwfJ72nbVo7qR9DAKA5FE8PU0bJFmFLjDDihc970zc4ilRDfR -WylUpj68nefOY4CzFzeiqVOLX2wezs7Z0hflkSXGBmC0j1FbQU2I3YJg3CKCabhT -tU6OyKItzjJ2vVaOoQ+B0Kv8leaRQ6ANZBAFQF2LepSy5F2+oSD+QHjPr+012V5D -mrsdIc9We8YyonS1u/3HI7lLohf3W+qFroQWjn0DJI56ScV1uEr/B0+hn2jBRTM5 -d1F9BeVWm1u8BOJu50CvOeuxiVLsxJpa4T41DJznJk5V+hE4hKvDKmlrwulsRp8o -jUEyUi8dzWOBRfAijIWv3qAPjGA/J33n6+PllCczC2BsVZhVmLqSMCwp1g2JTCM/ -KC7T4vOl/EGkm76fcmLeA1Ef8oUdRg+3T77VP+HqZ2JP06J8O8MCAwEAAQKCAYAw -YvJZ82BEJQGCIrIxMpHNAm+MFmKpDdIFp9oRdDrXgjcG9bLU3e1KSmkEgq4tggIh -GlAM3PHB6ULhPC2ixj7JZHWgCaqwYhKtG6vF+HGyRFDgRrIFTGyyfoICgxReloLp -lV2dGj/l19yXLuAzJtRmFdOSYhIGnGiNgnKvAKBiNajoxyHJpv7piPZqyc0QMZJ2 -bKVMDm02TSuhz4FDuzktaGtl9uQf5GQfnvTZRrRpkC70vigGnrFuSBiCgopF6NLq -6AXl8YS3Jcu2oGWrZDfS/GlG1QmvGGsmr9wndJSGG43jcpcRZt0g1nJNu4Fioq3e -7y6Gap9TEsciuQOv/6RD457XkNARmTQxFpEwmSgOPQn2pFcDspo71Ej7azzL/Z+3 -jvnVo3wxgxBcrpyh+vhBtJARp4pT4anW4PcD6IcPSOWbnI8Ldoj1XN5QkJcBcykK -6LmsAUqsmEQDNsmnGZWyYSCns4P2vUJi0hwQz8UiQwgAta3xnq4v5On7l3cq35kC -gcEA0+joOFbZBeGlCb27tDW4VCW0cQuczzuNEoBUKnsNSqy0nx1O7hgHm/f/NQDD -cpxiD15bRQ0KM9QbQC4dGaVoLsM07hUGk97dCxQPs2zot4CodCKGohs7E154tEDP -zVg3YS5mubUmqdqtn8ZCKeeZye/Tv2ageyF300sEgj2Cd7EZ8S4sB0PxZ2tqT3jy -cBL5cDruLEWuHIQjN7WwSjxnXocpb1OU7dJ+v4zFPCkSCOoa0DTTw4jFhPEOBdqV -T619AoHBAME3QyW4QVtU2Ct9u0B1XThhqSEyOpUrcH9nOoefggwP4WF3phVx16BG -aDKUIGQ62klRa5fi2eooxcjQRLv1sWO0UzssnO6ABMnGkUiRdrowo6xukNak0RTp -0gvNoJ0SZxGF0yWSCw1Rq3qP2Koj7XDumFChAzLMyUsnoOl29SA7GfXcZp1pZTiq -kOfFMWt0CIHu/EK03YWcd4vfQEq6lus39RCSXuL++Jva3yiEl5s069RFZvP1bNrD -emkfetDSPwKBwQClk+8fVnzs44sZOW9ZOEB3P57mVbSJGHb6Zdtd9hhEqP3Y9gWe -dJg9fmGjAJ23CAp3B7s5ER9PsAQ6+c0zJNNq9ox9G2CwWgtNhLdf81FDUPxPAktA -jxZx4/dcoOe+A5gCD0elA67aOUxA86DvLVA1QXeqrn3muBfwuUUknvs6mt8yXGl6 -o9QUgxHmVxLYD3tn/iPr4+ZP0c/Sz9yXpOsAKYxuuFg+G6N9+HiEsXKuFH4vAZgV -yODNJ61VVZ4lS+ECgcAqFqOl39E81+qO7sCPdgFsermg5ZQlUmUbG52AVZq6jesG -lE21disGWs/v1JyJuNg8CGRrnZriiycqa1PNreOKWImY5kr5GSHx4jNbn3RBcr70 -nNEoMJbq+1QqBgzqqkuRYZlxIbMOn6++7v6/cTwT0aWUSr6rnjhrCqLeuG8FKlqp -V+1ydLb79QvDsQzm30vLIggJb+ShakgQS/1xSdv+OR5FEd1hjTESokbiSJ/Ny2Vj -xAp9MgUYUmSj6ZuTSXkCgcAggshdRQLom/EK2pYwffIpKfBiyLbi+KIjKxkiPEsb -jrrQbvh9ZN6iAG3StVAYB5c6vewfeIlcDT0YJDyy1hGRLRG7vf9ubPf+n7Xp1y0W -oo9L9qfCHu0jmWwtinkFYjpTDkXlxXCG2v3TllNsNX/5afYo8sb9oxXHLTpBlwZB -fw6IgNZblWQevdgmUMTP9W2W7AZUxEz4gOM6lQkOwC3U59Dx2yO6rD3An6G1tlZF -2MClyf8o5d5ePObH8rkxrpY= +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQCtFmmyQ+HjjeBX +UE61fTenp2C9cvHzZSTFvKNeKZ2ZVZzH1HFnMBlTXGlXybSi2PhfxNYo5JrNGW4v +ZADh/eIFOONvPebQBONi4B1188Mb1RHgyIMtl/gOLF+L9DkStFb2eJ3fFInjNh5b +yhhpcOH1AmPCsiLeCAO3qc9qeChmJ4hAy1xmx74yQkOGzu0hKZ9d5QNNemCpPKTh +J+X8+PCY+FRObAdKJ56er2BjkdaUB/Mp6BBLokYs917VdnH6TiicqfrUsjh36XpO +G35/bW9jsQRpbeGkMIW6vn9GEYT79d4+FWdl9SxTfcSy7tn35/SvtIYDByWLc4IJ +izTuFV91pGmsXSav4YVF1hLC5cy8iEfKrdAl1kzamHfHKuY+P+sbwSQHPn5Z5xyU +Vr+odU0t7+YExBKLG8zHkQ+h2bwEeFy6H9uYQZFhBLOHrkkWSPYLKjzch6DyNkTp ++5cGWLOkcs8Xt0mjG2gzUlgwIYWHkq7wE9jxz3/tJ3MGf2CNDxsCAwEAAQKCAYAU +x+HCxFb6LLjS4tJrBHCf0VRSX9rf/7Ri2oYRXRUN1B3cTrZFeZyLdJyKggNiSy3E +GcF4lrxl2Xgaz8/cp3rxE5EYHv93hCQPZUbu4r2kZCLJsRPcGn3D/dyEKhCGyAi/ +fgsTkxyxnyK/9HXqhYyrZNu4rrh0SH63GKJ4GZsIFhZME95b0f6/e/3YJpSJFxKK +LF+xd/rjoWNOS62s1JqA/czpaXDqnJ0frNX5aQixLHnWUy/5qgRGNnY3/LOB0MuU +EjFizRSJp6gQQ64dIFgiMyUMi+Z6fw6xkTpN5EbmgFCwmaja1/hp9UZ+atlVpJsO +SB+OhF2wxB+bX2phGHJ9DD4RY/di1YKPvmH0+LITq4JiJWwJLgoCm7krzfVxZFZN +lgZPvdNw/Yt/Y3bN31FMw8DwRCThnL+MH83E58iCWQq2zKS1RssBBwFacI6Wf1ed +442NPIH47oi5qRKUxYHze6kFssSs4TkbR++VYHXLbTLjqfePLP92ZFxdOstx5jEC +gcEA48mx39dwr1oSk4uR+q8aDZIjPT+9D0IxcdAmL968QgG98hF2Tx7s5alR4y+9 +2FueR5fSqRweJfaDn77j5DVNXLD5jBz73pdwIkfqJHLfcsN7NnzCz3mxU9WP2F7R +W8KD3C2MtzGwQwOI3GT20+wJaA7p5n5zKR7AC0RxhC3U4zwH5nSjtY70A3dWXkn3 +F0j+B6ak0P0oN9VQUuvk5zw0IZ32uxef7poiHvFZpl1DQSgX/edT1CIT1s0nmVFl +w1ojAoHBAMKGX+TobGZfZFj/I7oIVQ7023pi516fZx/R0Y0FTmZzIKsdt3wrPmar +3K5dZGasNd0wzUsxX+vSjMtI1oERuPd4Rs6YqQHpX892uJ7L41/oBcFCc+Fk7jBY +TiRg/8/gOTaQe1nDR52K/R0sqBatRqInNMBDcBsVguBC4dTqP8CQ0CnEjAOYsk1s +1nYPKcfM36BUXBmSlnvUPWMm8KCT9Kko9ylHVq+d+ziDebfYpy6fTvti2NM31wTM +1prasEUaqQKBwQDQX14+5MapMd1SaVelmW5cwbVIvzjEb4npkj6MhdVzEELg4IZ4 +hFKzGDvXdoHVHKJi3YiQuC8ADUyE4kt4JCZbx2zQdmcVTTT/tweCRi8PvbDFvEU9 +JBZKGU+X38zmgr66uFRD9MlH1EDrU9TTMdW9Af+HoV7ZW87Tv82T25UmNXEIqORl +HpsrXIx+fmzxOQ1glFmq8BpNUO5EnJPtz43kvqrIpSjhTNAvvBqFbEUsom+oDWgK +4w2A7nTt9J8BoD8CgcBkYKa3HmBhazQC4JV097u8nglrXAH8R9EVEFZLqMNOBnaD +FjCKeF4Y6PJVX4fhm1eoLfihpnbS37EbbRiTPavutzgCf7AmdmCkU6Ts/FT2NmpR +0ZKuakCm3cpk51DZ2eBsEZ41MZmQ6Bm4pkSOfxeFsSl9VM9SioUgaCLUlZQUMCXa +h7ugV3kajuETxrtOiJ+UwjNMVuIkP971fTCKDA8iAyuXN2K5+JGcFewHPFr4qeg9 +vEIarCPeLD1JZzOyVRECgcEAhqeXfmFezNKGvr7qCflJe92LF9MStgJ7yE0OfADy +B+RZKeOwoqOO3VFR1piAn/DzrC2K9q+Of61gw+KWQOgMnsigqZ2mFLGChRjWhb7S +3G0DGOb4+DD9RR6wlFPFXSwVxSWGKrqNhOJik/IzVWvYxOOFjVt1adXZ3GftnYsv +nZCsS94H4kMiXr6UkbkjjxnZ1WkE9DaQJU27Mw1dtwb1ECzO0rty9B9L6b80MYW/ +nMhJV6sanCUex4nAChHD+VR7 -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIEbTCCAtWgAwIBAgIUF15VKdwjiTzzKgs6PnNpEekV9QQwDQYJKoZIhvcNAQEL +MIIEjjCCAvagAwIBAgIUCP/QP57z62jCsjf6l0ft/yzDxOkwDQYJKoZIhvcNAQEL BQAwYjELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEVMBMGA1UEAwwMZmFrZWhv -c3RuYW1lMB4XDTIxMDMxNzA4NDgyMFoXDTQwMDUxNjA4NDgyMFowYjELMAkGA1UE -BhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRob24g -U29mdHdhcmUgRm91bmRhdGlvbjEVMBMGA1UEAwwMZmFrZWhvc3RuYW1lMIIBojAN -BgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAn/BVsYuKFZQ1cXvDQxYG/iGABq/4 -WVrmNWau3e4Oe8j6MSa4LfmzfPflO6k3bqsgdh8vBtE6ngAjqyHaT2fD+1eROWIg -EsHye9p21aO6kfQwCgORRPD1NGyRZhS4ww4oXPe9M3OIpUQ30VspVKY+vJ3nzmOA -sxc3oqlTi19sHs7O2dIX5ZElxgZgtI9RW0FNiN2CYNwigmm4U7VOjsiiLc4ydr1W -jqEPgdCr/JXmkUOgDWQQBUBdi3qUsuRdvqEg/kB4z6/tNdleQ5q7HSHPVnvGMqJ0 -tbv9xyO5S6IX91vqha6EFo59AySOeknFdbhK/wdPoZ9owUUzOXdRfQXlVptbvATi -budArznrsYlS7MSaWuE+NQyc5yZOVfoROISrwyppa8LpbEafKI1BMlIvHc1jgUXw -IoyFr96gD4xgPyd95+vj5ZQnMwtgbFWYVZi6kjAsKdYNiUwjPygu0+LzpfxBpJu+ -n3Ji3gNRH/KFHUYPt0++1T/h6mdiT9OifDvDAgMBAAGjGzAZMBcGA1UdEQQQMA6C -DGZha2Vob3N0bmFtZTANBgkqhkiG9w0BAQsFAAOCAYEARzdkuqa0Hexi/saMkdi3 -bubpQkc7X0RYKWnjy/PgcmbvQXLiWRMZOH9rMWvd5v+ZfkgAtsbOQuP8ycioNIFY -Il5SEmxHEN81z5UNSPLOib6ky13gzrnXRAxnnO7cICG7AaMu1dHv57fqjevcx/n/ -nxPNKwKL+TDpMw7ATVZw7Py7JciKyFAfwtkvt17j/ldvaQvuwmWHzyFVrQniQcQq -QEa4jy/Y/pXHAgCKq1qbe0ush17j1ChyH7l4SkF2xJKcYYQF5ipw8zg6WeOL2NFE -G1KDJN0SsMmM3PMN1e0lLQP3G+UaatervrKXu51QleKL32Xlby+pp1w9KKs39/Tb -RT8EMe9A6cecod6TL0ZUQHow6ykNYBkfSKDLTKWnL9ifZ0C/DvgmS7DpJg3oAa1e -GhIglMrgqJflTHAI/PvEsCKM1O0Un2dVGWsUCzPfhj1cKmagyb0Zd+2Tk9xGSRs9 -2ceXMxRCjOJwEHUCFuTYeqowabdlpi0nyPbSn7JIwCpT +c3RuYW1lMCAXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWjBiMQswCQYD +VQQGEwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhv +biBTb2Z0d2FyZSBGb3VuZGF0aW9uMRUwEwYDVQQDDAxmYWtlaG9zdG5hbWUwggGi +MA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQCtFmmyQ+HjjeBXUE61fTenp2C9 +cvHzZSTFvKNeKZ2ZVZzH1HFnMBlTXGlXybSi2PhfxNYo5JrNGW4vZADh/eIFOONv +PebQBONi4B1188Mb1RHgyIMtl/gOLF+L9DkStFb2eJ3fFInjNh5byhhpcOH1AmPC +siLeCAO3qc9qeChmJ4hAy1xmx74yQkOGzu0hKZ9d5QNNemCpPKThJ+X8+PCY+FRO +bAdKJ56er2BjkdaUB/Mp6BBLokYs917VdnH6TiicqfrUsjh36XpOG35/bW9jsQRp +beGkMIW6vn9GEYT79d4+FWdl9SxTfcSy7tn35/SvtIYDByWLc4IJizTuFV91pGms +XSav4YVF1hLC5cy8iEfKrdAl1kzamHfHKuY+P+sbwSQHPn5Z5xyUVr+odU0t7+YE +xBKLG8zHkQ+h2bwEeFy6H9uYQZFhBLOHrkkWSPYLKjzch6DyNkTp+5cGWLOkcs8X +t0mjG2gzUlgwIYWHkq7wE9jxz3/tJ3MGf2CNDxsCAwEAAaM6MDgwFwYDVR0RBBAw +DoIMZmFrZWhvc3RuYW1lMB0GA1UdDgQWBBSAf/Z+TUHMOcr020NGG4geSnlNhTAN +BgkqhkiG9w0BAQsFAAOCAYEAIia0ULQmC510p78HZbiOb2BV8q+0AY0l6Cn/FRIL +Voqy7uB5oEYdDisla8epmBTU35+JmfrRHbP6IuSqKdcGj8VqsjZljhntSrB7rw4O +IqjbnxnfEuREjY2w+WGLvvdtVGXCBgfRmIItORFKpoOvLzLIi1lXeDq8QL97K4nM +tPsZpzILCBCoXmhh0MweCCNe1HSD8q6EbSDjVoxraBl0YK/l1ID08Fi5fwdddRX7 +txfx5NtoWWbsLZY0GdBtQmRCifs7P9lyhjUd87bJGd8WCBE4IzLwJiHEyNPI4WHe +8jmvKD/zO7mKxT9/jF7IBqwTHgQQ1uaRGytiXjRlfPu4Ez8ASR6rP7Hlua7h0Ba7 +OcDPcgM8rzN4zcfsvZ+8Cd84HXFjHhV6ZdRPbBWFWd8TlDufrWNuY4SyOtkyfE5E +lWcoRnkBGBaooZNcd72ZwZVO0gAS31bgs8ju+k02GIDLkuZSRNPeFsRQbcMu1Mhc +cFteFsqniLhzpkWJVQQgeGlV -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert3.pem b/Lib/test/certdata/keycert3.pem index f6887ba7a84e1a..097dc9f8e88091 100644 --- a/Lib/test/certdata/keycert3.pem +++ b/Lib/test/certdata/keycert3.pem @@ -1,42 +1,42 @@ -----BEGIN PRIVATE KEY----- -MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDFtLOteQlQojN7 -ztkux7m0hmGKkP1hh0hbKqTcD87jkLAqAwZWenjZMjCbbZ3vP+AObCIkYIKzPXY7 -Yi+H5M3O2mXIDxoHGjL/GWtoEyDNXvm9UC+MRuSOq2MaLHHQG0Rx2TxcYrMVUM7b -93rpN1LGRrCv1gISXM4EvEJooAR7Aadj0pG/o0fqDAdFjH6QZbhn1iZle+eGbjcf -dgH/H0F8dn1PPGoViHXicbsQ4kB6002Pf+aXP4b2QKAbflyNHEKHPHEOOTXrFjMd -c+bqKW24epEsMZI59qx9hU/4Rvp3/v+vEwTL7Nm7ilptzZn2cvGCW39LC0nNYLOz -kO3H8xwA75h6uykdB+WO/v2CKIK9M/ZO+9QNrmaokfKDamCk39b8hlCwNL6LsVpv -d3XTS5Wn4YWn92EqiltUJJoPo7pc7VTdWCg4zVFn4Q8Zh4NFNn/qTB8lEMgrsNTV -5cyZ7zhoBiUMSO45bmo2NsnE7ce/JUhlqe5uh0PT1MIBgTV+oDMCAwEAAQKCAYEA -udsy4gwblqK0tVnxz0lQqYV+os3EdO/BNHr1Oi7eNg2pngTz603812mYSjUVOHma -vtQmkH3twGQyBoc52Y1dcGzdK+IOfMjDUg7qao840ffL3I1J9ZwbdodlhZBsec94 -W3J1jP/4DDzICf8vm5g3h0+i/9m2Xt7BibAU2dg7/grC+lNUUoxDqaEfIOF/hW0q -muq1c8e0EisAROIh5FzUqhWVnWxU6eM7tuFlkuyu4whLLHB3LI466Lo+CTqT9M+v -jJYlvS5+AZW3qMBp6WOI8C+VIiBL178mo+Igkyyy5AYXcWeNkjp6ygRWvtWXIhCv -CI29mf+BP/54jAY0rQRXJ2UcSHXmM6PTDkE/L2OKeiY1Ou8gLOwun3yBVdbkXJMb -PWmUW4N8qSIJQ+vE2TDqmkqAT6m+ilzOXl1O+LLTvGyMnOiiSLXK9mC4ND3tqaQu -hvKivnI1doErcWUaIf1DHiJmLrGxrTCUKjCEoefqVq2/dDdtCfx7CqUvjl3DYKMB -AoHBAP+Vdi6D07gZFepEGCaJ+YH6cxEyO73CNnea/F1whVAzOv91kHS32jC9PAI3 -/wYlX+DLcN9mVF/q62V4SLZYfOxTPW4vWO0A45URe9s9Z795fdAcQ5jt3QFOVSnk -3XSaCkIOwckuwabGJi4+foiUEOnLLzQi1/g7x12dwejxVNhqhz5KFkOQPv8fQRed -sb5LVLYDeprsB2Vsx0fHwg4z9FvTIxLBeI7+sJD30lNpYZrCl/T9x4e1SV2Rwn2W -bghxgQKBwQDGBx07biZK9RB5g4qPl+G6vz0M+/KBfpwQbMYxSyct7u6gfGD9mWBO -qocIIr39Unac3kUL237Cn3HbgiGCRe7Mwd7XqnSSGWM5oWSlVQxEKTXYUlTbd9O9 -DKuyQGOl/AMEwD4ZbEOfQNmnd1U4nh1AV052FQY8Ry/atGFT9fApA/5X/bbenOwQ -YGDsokLzPf2BIDncpE+VNevUMoMI7EnySgjjfpL+cRld0qpLqBMo2h5VddeJ/5YM -1YcNfMQiw7MCgcEAwXqXuKa7A8aZvHpH/gS9CRRbP01TxFbdfLWrDeE8SnY9111c -Ob9kQTk/0D4rpK9uYXIgxD1m6iWghXQFN2TNTOnGuz7EhsYBgrt1k4Zsn5qND5oV -4hNPFsoB1nEW5EooMdGSCYaHuoSOKrvMdgAAvbu+xC0MaTJ3vfrK7Fik7h/WueTD -7emohuFWGVabU38bZZ5EljrPboxmX4Rs9uuFtG2lQ3GKnlVXvKaeZd6EsO9WsXPc -NHOcUmUhYokaSvIBAoHAGCxGJTsM8Zl4qVylTWH87A7sJOmccLJD2r1sdBf4cGL6 -PhzwugQ+/VtToGqdRo8Ka5u2Ufw5PQi5nVIFRSHERLpluW3VTQBMXHyXDJeVJ7zg -Fcf3E9NMxYcGbnvtrhVVSP8ulWvh1U7VQtwOSxsB9xixOzjVygXmkYvzVYxwBJG4 -OoV+DS6aomUhb8Fe6tJmX5zPc1+bV1t9ril8VVqCrFDdROfuiaDEt+8/Wnzp2dLG -YShBZ1cLugVWtw7D4nqBAoHAF29k64iAxY5Y4OOibVkqjUCPyqG2oxiXqgO7CxZp -FGUat5UtV2mIBlSENs1o5AZ1nPlgWtPtg0xVCaG2t/Rq7ugvUfAnAhUK6zX8FS+T -gCXE+7iKuuIJiCo13/iAwF/CLfuXvj4CZ71ta0wX9w99f1FcPEk0x+ytiyuWJK8K -tyubL34JwNrnkh/8e3LcV3L88Sk9ZmxeTz31f3cA3Fy2ZJOAUMD9dKXeKtY7azzt -MkhXedRsdLSKqMh0VGeGHoLS +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDuNnCqYWoguzKI +B3PwOsoRzI8wgNdxx3RiIJSItFISlAgB4XkYzPBpi5nFR8GCqq7IUQQ+iYwdZa3y +H6X11IdlrCx4gDdQFSFOegnG41oqv7GwBEFvq6xlowpoaGw+5/8/sJX/VU44rL9/ +fiMVTbehMhMS/z5nTSYSACV7sRXnwW4T6l2Od77FzPBjnaFFDNr5GIoiD4mNSJ35 +t05tliGcz0IyE5TGR6kIEFNIH5theHoJJ4JC4uLyWJweJpxDx10njWRwKN/tXm8F +nHQPMRAUl2wM07QZEI1yfzUr00/oG2dpPnPyNh+6p2ld8JS0g2/Pd2pQL8OtmM2c +DcN5E+qCEW1UVIJ5//ad8pUcoQwyRIBROJbakqv6FcVTdgTogoq5Vu+T6TVFLSIS +vVvpsV0MtT/H9/9TtCnqOwliE8s8kb3NjQT8e72VHNd43bVPE4DeQ+XlbMoKzV+G +OmSHSfzSOZB9kScSmGd5YD8783TbL8/7Nn1ERQdWqaixGI36+BECAwEAAQKCAYBV +Ubhevg9V89ZwdELpSxUu9NZgZ/VCck7UCplIsVUoBE8t5UULRfPhybdkuoOrulhp +tOLRR1ChAtcffohhmSJ5nwY6jqnBDCBmzD0OOEYGQ6xvv8Z0KcfQi2nh5WzHxy5b +8HJ5BmPC1tSr5FDKg5B+ssG0Lyl5tF8rWVQTjmSrIlO73Fhv+6GflPyQoVeutKEF +UO3Ar1H0AYtbcnUruPcHBBDQgMTrk6UOF1LM5U0wxwbmmnkEXeEtLeGUxv13JUMe +QTrSw0P+hn5uiDMwY5lI212ayorbwyNuzU0hGW7j98qa/S9MVSeRwv4bsbtBqwtT +ZVGL2TogjeJ+8+qDlv5tf3crD1pV0yu7uVUeIGrJZJdiiMM6N7/x3cIJtrLDaF1w +9kUqdQGWDWwRV3w0FaeXDVlUTJ3VFTZIinLo1Vmd3AXLn97eVy+68DWD6ELlMkfF +ro3bQbQu59DGswDF9+mMauhgvljnxY/wUnkyp8LGZt8w/RU/op8LcvPKFjQfX6UC +gcEA/RClguU/JTWQSF4Vjy5+DmrzSCXabrO7986lkOwPXEaXLsF2pGOU+sbxffXZ +PCQ4zU6aNYBMtNWLrVqBrPnI78XZQbmSG7zUDxWwmr1ttuSHGthdaZUVTOsFsnpE +R+1lj20G2T2ankzv0ICFz7BWyaYwyzDrABwuP189b3juuX6xGLlCO25/AnK+iTXQ +WF4ZURZiwDg4WpxvPsE2FFglN09TtDDCeQw/zv7Jh6jWtHZfivfv/wR8ILu72gde +LdinAoHBAPD5slojp2St6iWTaxcYYBc+4x+Rv7oyxeAyM/00iIAnf19oaAw3jki1 +yz6zeNnhaZuwa6ruvhJAESPc+pXkmoPpQ1yIRjKZc3jPVoaxYuSPgUEbHV6BQbjD +clIoGkC6ISsjanMq8Z3pK7e2bxY4mxGRQIFSFnTeK8m8nzq5dYisVtxxJFnV2fDG +NWDMdDYBh52vpkPjnm5OoOuilkEH2ygtZlT6bKAbnhi/FVjYCHk46inaYx8PakMu +LnvZI0OIhwKBwBGzYGBPeKM5o+Xr7sYdEmQfxvR88VJc8ADdS2dfm5NwvJJgpdPJ +w1nnIG0XDSLPxclWfiLP3o2ngiWV9wwKTKu4wwF94WJfStXjRn8MUOhCA9E04RPJ +gbvnlHZvZudBC6GElr4LOQ1phDypQLLOOsPQBAmyWj2fuvxjxQBPDSOcYPbBvog5 +qliZfgpK4U/NBShO0IlxZT+xQXa6PPYfVDsSKWCpKHEfEjeASshaXuowfW5S+U51 +GdmQSAtwCH5ccQKBwQCOLgW5gYfms2aPvSdWfR9VF9nSaqCBMCvoWDasky5mzucs +V+HsM2tUI09EM4h+pa02GyWruSmUgxCZ5GxFvJgedKc2FYG1oSysf0lCN69tw+4z +h9gQRpuMdGUjbF3xCuE/HqpUQWZGEamlv5JTvhpghx9ULibp1Zxob05Ty9E5TtYB +QxB7oN3yXkBoWLnIk6Z8t4KWU9rKosH3xfp5bDU2w3K5ePhWj3T8jOH/hZeaTqZ7 +A0uwq9u6v6jVkgxocEkCgcEAhyMPyiMqqq55qD6JlPY4zf4mWQJS4MnmWexH/5N1 +a6uogerwZnIjuH0/wNc2qzQSOwVJRqjlxRZbEkWST0JK/meXZ+e0nqaROWICutO6 +ciIf2wVjTzn1f85DHwPhUf3P2zIfpVFh+XuvSb32J+pzNwK40gMhirYMDSbsm/RQ +JrJHQwEX9BvI6IP5kaMOMlSwiG3Soo3MxeJWmozkz09+cs7BwMFsUKOdqruJHSaw +/Q06DQ9u9UyjmLyKv3IkN/cK -----END PRIVATE KEY----- Certificate: Data: @@ -47,38 +47,38 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (3072 bit) + Public-Key: (3072 bit) Modulus: - 00:c5:b4:b3:ad:79:09:50:a2:33:7b:ce:d9:2e:c7: - b9:b4:86:61:8a:90:fd:61:87:48:5b:2a:a4:dc:0f: - ce:e3:90:b0:2a:03:06:56:7a:78:d9:32:30:9b:6d: - 9d:ef:3f:e0:0e:6c:22:24:60:82:b3:3d:76:3b:62: - 2f:87:e4:cd:ce:da:65:c8:0f:1a:07:1a:32:ff:19: - 6b:68:13:20:cd:5e:f9:bd:50:2f:8c:46:e4:8e:ab: - 63:1a:2c:71:d0:1b:44:71:d9:3c:5c:62:b3:15:50: - ce:db:f7:7a:e9:37:52:c6:46:b0:af:d6:02:12:5c: - ce:04:bc:42:68:a0:04:7b:01:a7:63:d2:91:bf:a3: - 47:ea:0c:07:45:8c:7e:90:65:b8:67:d6:26:65:7b: - e7:86:6e:37:1f:76:01:ff:1f:41:7c:76:7d:4f:3c: - 6a:15:88:75:e2:71:bb:10:e2:40:7a:d3:4d:8f:7f: - e6:97:3f:86:f6:40:a0:1b:7e:5c:8d:1c:42:87:3c: - 71:0e:39:35:eb:16:33:1d:73:e6:ea:29:6d:b8:7a: - 91:2c:31:92:39:f6:ac:7d:85:4f:f8:46:fa:77:fe: - ff:af:13:04:cb:ec:d9:bb:8a:5a:6d:cd:99:f6:72: - f1:82:5b:7f:4b:0b:49:cd:60:b3:b3:90:ed:c7:f3: - 1c:00:ef:98:7a:bb:29:1d:07:e5:8e:fe:fd:82:28: - 82:bd:33:f6:4e:fb:d4:0d:ae:66:a8:91:f2:83:6a: - 60:a4:df:d6:fc:86:50:b0:34:be:8b:b1:5a:6f:77: - 75:d3:4b:95:a7:e1:85:a7:f7:61:2a:8a:5b:54:24: - 9a:0f:a3:ba:5c:ed:54:dd:58:28:38:cd:51:67:e1: - 0f:19:87:83:45:36:7f:ea:4c:1f:25:10:c8:2b:b0: - d4:d5:e5:cc:99:ef:38:68:06:25:0c:48:ee:39:6e: - 6a:36:36:c9:c4:ed:c7:bf:25:48:65:a9:ee:6e:87: - 43:d3:d4:c2:01:81:35:7e:a0:33 + 00:ee:36:70:aa:61:6a:20:bb:32:88:07:73:f0:3a: + ca:11:cc:8f:30:80:d7:71:c7:74:62:20:94:88:b4: + 52:12:94:08:01:e1:79:18:cc:f0:69:8b:99:c5:47: + c1:82:aa:ae:c8:51:04:3e:89:8c:1d:65:ad:f2:1f: + a5:f5:d4:87:65:ac:2c:78:80:37:50:15:21:4e:7a: + 09:c6:e3:5a:2a:bf:b1:b0:04:41:6f:ab:ac:65:a3: + 0a:68:68:6c:3e:e7:ff:3f:b0:95:ff:55:4e:38:ac: + bf:7f:7e:23:15:4d:b7:a1:32:13:12:ff:3e:67:4d: + 26:12:00:25:7b:b1:15:e7:c1:6e:13:ea:5d:8e:77: + be:c5:cc:f0:63:9d:a1:45:0c:da:f9:18:8a:22:0f: + 89:8d:48:9d:f9:b7:4e:6d:96:21:9c:cf:42:32:13: + 94:c6:47:a9:08:10:53:48:1f:9b:61:78:7a:09:27: + 82:42:e2:e2:f2:58:9c:1e:26:9c:43:c7:5d:27:8d: + 64:70:28:df:ed:5e:6f:05:9c:74:0f:31:10:14:97: + 6c:0c:d3:b4:19:10:8d:72:7f:35:2b:d3:4f:e8:1b: + 67:69:3e:73:f2:36:1f:ba:a7:69:5d:f0:94:b4:83: + 6f:cf:77:6a:50:2f:c3:ad:98:cd:9c:0d:c3:79:13: + ea:82:11:6d:54:54:82:79:ff:f6:9d:f2:95:1c:a1: + 0c:32:44:80:51:38:96:da:92:ab:fa:15:c5:53:76: + 04:e8:82:8a:b9:56:ef:93:e9:35:45:2d:22:12:bd: + 5b:e9:b1:5d:0c:b5:3f:c7:f7:ff:53:b4:29:ea:3b: + 09:62:13:cb:3c:91:bd:cd:8d:04:fc:7b:bd:95:1c: + d7:78:dd:b5:4f:13:80:de:43:e5:e5:6c:ca:0a:cd: + 5f:86:3a:64:87:49:fc:d2:39:90:7d:91:27:12:98: + 67:79:60:3f:3b:f3:74:db:2f:cf:fb:36:7d:44:45: + 07:56:a9:a8:b1:18:8d:fa:f8:11 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Alternative Name: @@ -90,75 +90,72 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - 85:75:10:25:D0:2C:80:50:24:1A:5B:57:70:DE:B5:CB:71:A9:3B:7B + 16:22:C9:AA:20:D4:F4:8E:ED:9C:A6:BB:23:F5:F2:29:44:ED:F2:1E X509v3 Authority Key Identifier: - keyid:B3:8A:A0:A2:BA:71:F1:A8:24:79:D4:A4:5B:25:36:15:1E:49:C8:CD + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B - Authority Information Access: CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer OCSP - URI:http://testca.pythontest.net/testca/ocsp/ - X509v3 CRL Distribution Points: - Full Name: URI:http://testca.pythontest.net/testca/revocation.crl - Signature Algorithm: sha256WithRSAEncryption - 95:f3:56:bb:d5:8c:70:bd:d1:de:da:63:b0:29:d7:db:60:27: - d6:59:fd:61:1b:30:c6:d0:5d:73:7d:34:e1:68:e3:28:a6:89: - e6:60:bd:89:d3:0e:f4:72:ad:72:76:f8:86:21:fd:75:3c:f8: - 6d:be:9c:04:e1:82:03:69:6c:ae:d0:55:ba:5e:f2:ca:f5:0f: - 8e:d6:d9:8d:c8:56:46:f4:f8:ac:74:2a:19:7b:8e:47:70:1f: - fb:fb:bd:69:02:a1:a5:4a:6e:21:1c:04:14:15:55:bf:bf:24: - 43:c8:17:03:be:3e:2c:ea:db:c8:af:1d:fd:52:df:d6:15:49: - 9e:c2:44:69:ef:f1:45:43:83:b2:1e:cf:14:1c:13:3f:fe:9c: - 71:cb:e7:1b:18:56:36:a7:af:44:f1:0b:a1:79:44:46:f9:43: - 46:29:d8:b0:ca:49:4d:65:60:d3:f6:8e:74:bc:62:9e:1e:8d: - 4b:29:9a:b4:0d:f0:a2:77:5b:34:e4:11:2f:a7:25:c5:e5:07: - 76:12:ae:be:75:73:15:e4:0a:7d:53:38:56:3f:79:6d:6e:ca: - ed:80:ab:56:ed:7e:8b:1c:e7:e3:d4:62:30:22:70:e7:29:b2: - 03:3c:fe:fa:3d:f0:36:c0:4d:11:a2:99:d3:29:31:27:b8:c5: - b8:15:a3:3c:4f:9b:73:5e:2b:b2:fb:cb:fd:75:47:b8:17:bd: - 21:d8:e6:c1:b9:ff:73:81:d8:25:08:6d:08:5e:1c:a5:83:50: - de:67:e6:da:d0:8e:5a:d3:f2:2a:b1:3f:b8:80:21:07:6a:71: - 15:6d:05:eb:51:b3:59:8d:d4:15:46:7e:02:a8:13:01:16:99: - bd:03:cc:70:71:2a:23:16:78:af:d1:d5:01:9d:04:b4:63:93: - 9a:04:3a:92:2e:e6:7e:73:93:a5:fe:50:9b:bd:0e:ea:54:86: - 6f:7c:e5:14:77:fe:c2:28:5a:4a:0e:d7:2d:8c:e9:ed:61:29: - b2:53:ff:6c:04:bc + Signature Value: + 43:82:1f:18:b4:cb:7a:b7:4e:ac:c3:65:2e:b7:57:3a:f0:78: + b3:b2:20:3e:bd:7b:24:f7:e7:17:cc:62:c0:b8:63:f0:2e:30: + 0c:54:be:c9:94:7c:54:d4:40:06:6e:92:9d:72:2c:44:f5:e3: + c1:4a:3f:26:e4:21:ca:ba:ef:4f:a3:50:56:ee:89:5c:dc:5b: + 98:bc:0c:e9:db:16:98:f0:4c:3f:65:b9:d9:e0:2e:bd:83:9e: + c7:1b:f7:4f:87:5f:75:ff:67:5d:0a:ff:ce:fa:0b:f1:d5:8b: + 58:23:2b:92:27:b8:15:ff:66:c2:f9:68:e2:ed:64:a0:4a:e0: + 42:8b:ac:35:6a:67:64:f5:cb:a9:00:37:20:56:dd:3a:02:6e: + d8:0f:b4:5d:ff:92:a3:0d:65:fc:2e:ab:4e:37:bc:7e:d6:5b: + 1c:55:78:a0:6e:a9:2e:55:31:cf:61:f0:1b:4c:fe:fe:60:75: + 76:e8:d9:a1:f5:14:f0:43:33:4d:52:05:be:33:ff:da:31:94: + bd:0d:e5:4d:a8:ee:d1:f3:92:34:c2:36:91:37:3a:6d:ec:20: + df:b3:9e:a2:5d:d0:f0:a1:e2:ff:b7:85:ce:09:e3:c8:4a:6a: + 93:a4:ae:ec:c1:67:42:f7:8e:2f:64:ca:d7:76:72:bf:3f:ef: + 64:ff:10:dd:a2:3e:2d:f5:aa:02:cf:31:f0:6d:93:6b:56:4c: + b0:88:95:54:dc:d4:ac:dc:c1:88:27:8b:d9:9c:4e:66:93:f8: + dd:5e:74:41:26:9a:ef:e8:c8:46:5c:d5:f4:fc:cd:cc:67:c7: + 33:62:2e:70:9b:d1:39:de:34:fd:ad:9d:b9:48:61:e7:89:53: + 62:62:f0:40:6c:68:2f:e1:da:40:61:2a:50:af:c9:41:6d:98: + 16:af:cb:63:03:ec:04:99:e0:39:98:60:ae:aa:6a:57:9b:25: + 73:03:16:bf:78:c8:d7:2f:eb:fc:64:7f:c7:78:8c:34:15:5b: + 66:40:98:86:e6:0a -----BEGIN CERTIFICATE----- -MIIF8TCCBFmgAwIBAgIJAMstgJlaaVJcMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIF8zCCBFugAwIBAgIJAMstgJlaaVJcMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxv -Y2FsaG9zdDCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAMW0s615CVCi -M3vO2S7HubSGYYqQ/WGHSFsqpNwPzuOQsCoDBlZ6eNkyMJttne8/4A5sIiRggrM9 -djtiL4fkzc7aZcgPGgcaMv8Za2gTIM1e+b1QL4xG5I6rYxoscdAbRHHZPFxisxVQ -ztv3euk3UsZGsK/WAhJczgS8QmigBHsBp2PSkb+jR+oMB0WMfpBluGfWJmV754Zu -Nx92Af8fQXx2fU88ahWIdeJxuxDiQHrTTY9/5pc/hvZAoBt+XI0cQoc8cQ45NesW -Mx1z5uopbbh6kSwxkjn2rH2FT/hG+nf+/68TBMvs2buKWm3NmfZy8YJbf0sLSc1g -s7OQ7cfzHADvmHq7KR0H5Y7+/YIogr0z9k771A2uZqiR8oNqYKTf1vyGULA0voux -Wm93ddNLlafhhaf3YSqKW1Qkmg+julztVN1YKDjNUWfhDxmHg0U2f+pMHyUQyCuw -1NXlzJnvOGgGJQxI7jluajY2ycTtx78lSGWp7m6HQ9PUwgGBNX6gMwIDAQABo4IB -wDCCAbwwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQEAwIFoDAdBgNV -HSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4E -FgQUhXUQJdAsgFAkGltXcN61y3GpO3swfQYDVR0jBHYwdIAUs4qgorpx8agkedSk -WyU2FR5JyM2hUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29m -dHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcoIJAMst -gJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6Ly90ZXN0 -Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1BggrBgEFBQcw -AYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2NzcC8wQwYD -VR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0 -Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAJXzVrvVjHC90d7a -Y7Ap19tgJ9ZZ/WEbMMbQXXN9NOFo4yimieZgvYnTDvRyrXJ2+IYh/XU8+G2+nATh -ggNpbK7QVbpe8sr1D47W2Y3IVkb0+Kx0Khl7jkdwH/v7vWkCoaVKbiEcBBQVVb+/ -JEPIFwO+Pizq28ivHf1S39YVSZ7CRGnv8UVDg7IezxQcEz/+nHHL5xsYVjanr0Tx -C6F5REb5Q0Yp2LDKSU1lYNP2jnS8Yp4ejUspmrQN8KJ3WzTkES+nJcXlB3YSrr51 -cxXkCn1TOFY/eW1uyu2Aq1btfosc5+PUYjAicOcpsgM8/vo98DbATRGimdMpMSe4 -xbgVozxPm3NeK7L7y/11R7gXvSHY5sG5/3OB2CUIbQheHKWDUN5n5trQjlrT8iqx -P7iAIQdqcRVtBetRs1mN1BVGfgKoEwEWmb0DzHBxKiMWeK/R1QGdBLRjk5oEOpIu -5n5zk6X+UJu9DupUhm985RR3/sIoWkoO1y2M6e1hKbJT/2wEvA== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJ +bG9jYWxob3N0MIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA7jZwqmFq +ILsyiAdz8DrKEcyPMIDXccd0YiCUiLRSEpQIAeF5GMzwaYuZxUfBgqquyFEEPomM +HWWt8h+l9dSHZawseIA3UBUhTnoJxuNaKr+xsARBb6usZaMKaGhsPuf/P7CV/1VO +OKy/f34jFU23oTITEv8+Z00mEgAle7EV58FuE+pdjne+xczwY52hRQza+RiKIg+J +jUid+bdObZYhnM9CMhOUxkepCBBTSB+bYXh6CSeCQuLi8licHiacQ8ddJ41kcCjf +7V5vBZx0DzEQFJdsDNO0GRCNcn81K9NP6BtnaT5z8jYfuqdpXfCUtINvz3dqUC/D +rZjNnA3DeRPqghFtVFSCef/2nfKVHKEMMkSAUTiW2pKr+hXFU3YE6IKKuVbvk+k1 +RS0iEr1b6bFdDLU/x/f/U7Qp6jsJYhPLPJG9zY0E/Hu9lRzXeN21TxOA3kPl5WzK +Cs1fhjpkh0n80jmQfZEnEphneWA/O/N02y/P+zZ9REUHVqmosRiN+vgRAgMBAAGj +ggHAMIIBvDAUBgNVHREEDTALgglsb2NhbGhvc3QwDgYDVR0PAQH/BAQDAgWgMB0G +A1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1Ud +DgQWBBQWIsmqINT0ju2cprsj9fIpRO3yHjB9BgNVHSMEdjB0gBTACisoQ95fyX1H +5UebNvJljGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyggkA +yy2AmVppUlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rl +c3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUF +BzABhilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBD +BgNVHR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rl +c3RjYS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAQ4IfGLTLerdO +rMNlLrdXOvB4s7IgPr17JPfnF8xiwLhj8C4wDFS+yZR8VNRABm6SnXIsRPXjwUo/ +JuQhyrrvT6NQVu6JXNxbmLwM6dsWmPBMP2W52eAuvYOexxv3T4dfdf9nXQr/zvoL +8dWLWCMrkie4Ff9mwvlo4u1koErgQousNWpnZPXLqQA3IFbdOgJu2A+0Xf+Sow1l +/C6rTje8ftZbHFV4oG6pLlUxz2HwG0z+/mB1dujZofUU8EMzTVIFvjP/2jGUvQ3l +Taju0fOSNMI2kTc6bewg37Oeol3Q8KHi/7eFzgnjyEpqk6Su7MFnQveOL2TK13Zy +vz/vZP8Q3aI+LfWqAs8x8G2Ta1ZMsIiVVNzUrNzBiCeL2ZxOZpP43V50QSaa7+jI +RlzV9PzNzGfHM2IucJvROd40/a2duUhh54lTYmLwQGxoL+HaQGEqUK/JQW2YFq/L +YwPsBJngOZhgrqpqV5slcwMWv3jI1y/r/GR/x3iMNBVbZkCYhuYK -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert3.pem.reference b/Lib/test/certdata/keycert3.pem.reference new file mode 100644 index 00000000000000..84d2ca29953e46 --- /dev/null +++ b/Lib/test/certdata/keycert3.pem.reference @@ -0,0 +1,15 @@ +{'OCSP': ('http://testca.pythontest.net/testca/ocsp/',), + 'caIssuers': ('http://testca.pythontest.net/testca/pycacert.cer',), + 'crlDistributionPoints': ('http://testca.pythontest.net/testca/revocation.crl',), + 'issuer': ((('countryName', 'XY'),), + (('organizationName', 'Python Software Foundation CA'),), + (('commonName', 'our-ca-server'),)), + 'notAfter': 'Oct 28 14:23:16 2525 GMT', + 'notBefore': 'Aug 29 14:23:16 2018 GMT', + 'serialNumber': 'CB2D80995A69525C', + 'subject': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'subjectAltName': (('DNS', 'localhost'),), + 'version': 3} diff --git a/Lib/test/certdata/keycert4.pem b/Lib/test/certdata/keycert4.pem index 1003d67fd075ec..fcaa3a1656d1a5 100644 --- a/Lib/test/certdata/keycert4.pem +++ b/Lib/test/certdata/keycert4.pem @@ -1,42 +1,42 @@ -----BEGIN PRIVATE KEY----- -MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQC34y3S6iXdmdvd -M/2aFBe6CvRvZwhh1huGl7IQRtdoakPqMLlEdNHJtNeF5M27xLei+p4wt7N1Jyi0 -2keHQb1m9TqH5AruOkE2ti+15zEoKoU9aWydTiH+epKTT0yjg2NcKQjRUaWcbhzB -H4EMKuCIlzIIz8/EIKkOqhCDwq6+Fv3Ays+z7Bz+yR80ixivKu/l7SjxQ7z7R/kC -I7OViRcIO5QBQPj7VLvCTz4VA6u/LdXngK2HNuau6WXm5yNNQbqrB11AEJcYZf/c -VrneV4F+ZjLloAKgSn9GB8eWOyilTQ18TcKd+H2icipRaP/+QR/KPx5GK/SXU3my -qm62QOGI7t/5ktVdjGhs6tHZxw1SRiipiLYWbtVRrSxa4wYlgpgoUwvrvvtC5kAN -nTw1VGWsxcs+6a7+PocYnJiq7k4b5OAUb3Ryvl9DLAMy8NqpRWo4cHD/XQ3FCYwF -HlOSgx/dL5Se0i3dW1KzbP6OvaNg6nl/1EXPUsJ1ATS8nzvzhccCAwEAAQKCAYEA -nD3GvaJ9MeB802JNZBEWZ9jO/6jHknldQeq6POI0PF+t/NoRUH0BkyS4yucxdw0a -CrxulG5BaJUxHRkqFV5iE4zhgnzcXLXamyYJO8GIHtyiASAGTVIJyDNVPxztvTDx -x2iGOXPqBxP4Eo82EqSLywLMXHhVzAsEGZWeGpXb61+Vk62+9Nz1dfZlMTvOaWdO -Fkp/sx8e/1KT3KGBANlOXIxioP4Xj1Tbg6nY0fogf3vud5j52B1pu8xL7PkPIaFq -DEGz3XvWhBF/+Cs5iDeYz8eQpfQig7HdHVn2D8dZmzQgpLw1yGbPAnqrgopWfm7R -MqiyFe82p2t+vfSoG5jz28XxPtzBJV3ljxKxlbnclqu/CAYSjzaYohDzyhjdZOZI -r9DOfWOqu01Ha3EEsApn95fusHHGTH2FOy0u61FSTrfLfqsLw9WRJPWleirKikhf -SZzi223QrmzZMtuCF7VgTx3ghDhBmFD8uzVVQ1SwPZ8CgftRkFcn1llXIAfJ3iHB -AoHBAOg3DOIdtUVgpjMKhpAyuH54fYvGl7afIMNbKRle0kCiP45wtGJ43RPMqiR8 -1rxZB3+iapICI/lnhk3O7vVRkR64yiqQBcl/hXZ1BhyD6iDXWYmm5mcnymcoqfwc -p9TfzEPyGPb3SM2YlI0cSPRqM/jDvGvnDeKIpzEKvUlwJ59WoN2HOHTIXf+XbN5n -unpuTt6YKJvc48DrXsPnUzkCmUfbOmgHfeb9/qBs/8kY4YJMsZEjqf88o7mCJCIy -BtDxTwKBwQDKuOwE8e0GIA01ZHd6RfR+ZCvmp2oauxal4EJsBx+ZZnhEWGaSm1fE -Bf/ih074ghcSKoSrdYpD1xGZ6fGVWMx3jcL11yLDOUiiPDJsm8hUBZ0IW1qXyfCP -l7xy1bUkWwPXdmFuGp1exrcjooKrFNuTdYiK4nQZSKuCfXQRADrmEJmM+gYwhqI7 -4XsYo848B9A4hbY6RLEox4uvo/RmafY0iR0PMhVEc+ydNLKB/4LpahZqBQ4kTpMv -o4+rEvYt1gkCgcB08gx177ozx1nMCLf99N0/LBUmCIytNvR8DfPjyAIg9NUHOjFO -CkpkR0VEfO50Cm4hVD1RbOyLFRzpIJbtSvfHvg5qYv/XG3auUn8Sa0jE408/aKNO -PhbL3wnEYvYO2ep4KXtzHNQ4XmgprJ39IWMtG/5PZRx0ApgYtazgSDBcKXd4OTow -bhwQtUTpuNmMAPONXJnO7O5yYNbn2B7sbiedrYV7kJJSe4X5awtiTjp7sX4XdxuM -5BAcQ7NI2WLfZTcCgcBp/X9hIoATmMRvKwUQx+yJ/KO7Z8KhETpJJdR0mNDbqmit -Cy8t7cxYb+6WqLoQUivv0o0k/EJ7L8JDH76woAnfZB4P3RiOy69/K0wN3vFBhOHS -kbju7aU53lKoE7YuuOtsRrewEng/KlRsbDY3bqNTGLt4KegbpBQQGLmLffxNd1Zh -EAQWcP33ou9yNYrJdihWtQpOssWRlash/O32ceZJF3s7C6t068tFclz2fPocQdxQ -OC5pqy9nU/P0tOhDlMkCgcEAosaBJLIeAYlOU0+2uSx5g5mIqOOTyrDEmqqad6T/ -wkB7vW2QaoDvLL22Yrzdn9vQ0V0rqzhVtan7sq5pn/BQJAueZYN8rFxS3uuW+UQk -Nsc4GLJzU8Az/2DvqEIrnE7zRc5E1FOI9gKLrBlpJB2o0hVcBznDe05Gax6Kjqbm -jHqzyU73SpxpEy3OesClCeCQIMr47HaL9aSqaEX4U9bMpgHi0HgTTHqvJ5pch0hY -dYl+WAE9LAyF1DF29BirEXVw +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQC/uCI/4eVSCIrx +//GcCCxeQP16Mtf0wAuINBR9uTySF8sEVoS6rNBwLeudZXaHmcp3jM7N2e9GjI8d +dQccPApKBwjAxhdT4XeuUTep4VraKWjLTWDvABHfCVFB1tX3HusEWaS8n/pH+N4L +UfnxS3HnFqGC5ib2DSgFTCnLQSj82iUgACdf87iZTons+kRvqYsdwpsgBUnO+w+o +c5o7Nj1bJJlTREsY6kQGY94as4pCkmqhoBTC7/le9wFGmXipRYi3pfSTOqAQJfqW +1VlQ4Bbc3lN5L+ONTmFXOgTJ5qhaS8cglnYElJI1QQTagyWQMHuBDIMpr3Iv/5qU +UzdIMzoMrpX6ien3Gan/jyjjzvP95jJvGujQudw/myXK8VOUThVtstRDFA3cIoV4 +UqMODz8w6qYU0GPYFBq//1UdpDrk/fBPBe3MLJT2+sB/kTzwxxYsKqcdjPG2Bt3Y +2nk7hCzE0KuueqCQPk/3fomY1TDyTlWOriAS40EYue5Rhx9qr8kCAwEAAQKCAYAL +X7EoeQ5Rv4/+q2B6EUIQlWp0RW/qZTpJ6k0M6GBfimnQ6BEXwgjbnt3LiKlvggAw +93mNXNCFLvGOndK+KhGyMpiiVx4rK8Ud4lObEHODXdGJvh1yEF7/DF51uXkYIA1x +RKAxUIxYmLsTkNlzJzaqrv0F9wF4t28YYVxZYpQ76/Un4Np1JtBcx/wGwxIsTbKj +IVhynd2zGdHj/He5643YSmOOPQ73e25tsp8KlnwtVuJclFKm/fWdXKoGtjQiVq2b +bViG7WTqb+a+R3IrO8CE2vAQ1ZqklP4Pg33AU1XFogRweqSkXvBGy1GHSSXCO4uX +gORj/bo/gAZ/eeufwqxBxJMS4Kbmxes/oYEgJidoaeA6DZjcKr+J3rFUt9j6rpxt +abeKBiK6wTHkKAdQwoccDdKFROvL4ZEu4zpjkmiqDrtIZEqtB8YCvoMpteIqrBhr +stLCDrftsysBTNGNn0THb4H5wx8nkZPCVyvaBD4N4Skp5rapP2bbrPhigcOoR8EC +gcEA4ToVg2b09tLWpmCg6XXssT1SvvFcSaJ3jjRKMfpmP5epWFiZ21qAOCbYhMNu +2G21AyNd1W7oVi6epXsB2YBOlSI0Fr6RuN0fDwg5jZLxEkEqDsL2QKsgoKHtGfBn +gX9jLQ5Lsy5/0CRXYCfPjdccbr2J5Gm2fU/jhhCi9ji/6b2RbgrRkIkr2I/J6KbS +sO+p1d93CFNagLMbdDspugDlIgj52OBMYO60nDlNU8O2pkAcLQTpWVqEjmoo4Q4K +oyOXAoHBANnqB93sLCndEiZNth7cOOEhNKCWRDkpGwDBbn2Dj3kBaXg8eCTG7tiF +FPyNVydYg+u0vcPIjj9URsOfpHc9MLSYtKAgCL3rSk+OQnROeg9hCjmB5Ucg7NTR +gSh/flfWp/q1EDeOVAu8B9DuBX1209br4xi78LnBP5/UzRjhuFEmvCvOqVCMCI7Y +I1k3l4NhOk87fKq5dbtxvNTU2eFTtt1+YjYwZFY+AUmx3WBz+aFwygks3S4SvN1J +LtLVYhOznwKBwQDC047WixIuDLX3WDD5orOroeNZHsn5PFv1HBBuaS9XpSatMH9u ++ztc12WGetQAze2+GDLMNNMv8cX0WZKBBfd0FBFA93pwkn6Sb0fxyoFUjCAIguen +iyB/M3M5c/blUz+EMxCSoA+aCkW2/NkS1lhXBwgoGLXuclPbnbqKCQ8h74TEzwD2 +6WGPRNqgsOYifj7IrjR2dDwehlCiW6c9qhaLOX5+94+6beK4HO1iHzN5Xo3A97Wv +QJjX5McV3yKeemMCgcA4N3reEo52IlULUqL4JSH7WkCkaP+iq2sO79fcQ3Ng6S9X +WGo6OqPlcbevS5s/SEOILDGEb5na1pgG4YlhRYTqIjb+1CTNMgUSrwWP0asFiqhD +m7IVfnX6lS23z+Q9LuBY+hr76hjeihyOFsmNy3jtCh+lAt8gXK1YQ2LB14FgVhjX +SFI/uFCA4VuFKaVJvGx5gkQwGvY3bCkl0t9+lMUpMPCPQD6yTP6yD1OoDWNJ9bn5 +UfyhZS4Z/EY7F9dcc8sCgcEAi8yW4gNCZSsGQRLh+fyDCSdak4ahATAeiI24231u +jK5+ncK9ZoZvzO/h+V5bw7ha59rD8aGmrLqzBVKEMi0PAT38psNH9ZtRt9y29YQ8 +WpyHtODMbphfbH4SFH3nj3PldRtBcOt0EDdp/4s64A9qdOWRx/vlJhdg0vtWTy5s +koyyOHhTcplpyPPq0BetYKl9UlXPVjus/RT1XV/lz2YrE8E9UAq+/r7ECoMExaZt +8j8J8O3lTWPxNTaQuiPiHA7E -----END PRIVATE KEY----- Certificate: Data: @@ -47,38 +47,38 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=fakehostname Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (3072 bit) + Public-Key: (3072 bit) Modulus: - 00:b7:e3:2d:d2:ea:25:dd:99:db:dd:33:fd:9a:14: - 17:ba:0a:f4:6f:67:08:61:d6:1b:86:97:b2:10:46: - d7:68:6a:43:ea:30:b9:44:74:d1:c9:b4:d7:85:e4: - cd:bb:c4:b7:a2:fa:9e:30:b7:b3:75:27:28:b4:da: - 47:87:41:bd:66:f5:3a:87:e4:0a:ee:3a:41:36:b6: - 2f:b5:e7:31:28:2a:85:3d:69:6c:9d:4e:21:fe:7a: - 92:93:4f:4c:a3:83:63:5c:29:08:d1:51:a5:9c:6e: - 1c:c1:1f:81:0c:2a:e0:88:97:32:08:cf:cf:c4:20: - a9:0e:aa:10:83:c2:ae:be:16:fd:c0:ca:cf:b3:ec: - 1c:fe:c9:1f:34:8b:18:af:2a:ef:e5:ed:28:f1:43: - bc:fb:47:f9:02:23:b3:95:89:17:08:3b:94:01:40: - f8:fb:54:bb:c2:4f:3e:15:03:ab:bf:2d:d5:e7:80: - ad:87:36:e6:ae:e9:65:e6:e7:23:4d:41:ba:ab:07: - 5d:40:10:97:18:65:ff:dc:56:b9:de:57:81:7e:66: - 32:e5:a0:02:a0:4a:7f:46:07:c7:96:3b:28:a5:4d: - 0d:7c:4d:c2:9d:f8:7d:a2:72:2a:51:68:ff:fe:41: - 1f:ca:3f:1e:46:2b:f4:97:53:79:b2:aa:6e:b6:40: - e1:88:ee:df:f9:92:d5:5d:8c:68:6c:ea:d1:d9:c7: - 0d:52:46:28:a9:88:b6:16:6e:d5:51:ad:2c:5a:e3: - 06:25:82:98:28:53:0b:eb:be:fb:42:e6:40:0d:9d: - 3c:35:54:65:ac:c5:cb:3e:e9:ae:fe:3e:87:18:9c: - 98:aa:ee:4e:1b:e4:e0:14:6f:74:72:be:5f:43:2c: - 03:32:f0:da:a9:45:6a:38:70:70:ff:5d:0d:c5:09: - 8c:05:1e:53:92:83:1f:dd:2f:94:9e:d2:2d:dd:5b: - 52:b3:6c:fe:8e:bd:a3:60:ea:79:7f:d4:45:cf:52: - c2:75:01:34:bc:9f:3b:f3:85:c7 + 00:bf:b8:22:3f:e1:e5:52:08:8a:f1:ff:f1:9c:08: + 2c:5e:40:fd:7a:32:d7:f4:c0:0b:88:34:14:7d:b9: + 3c:92:17:cb:04:56:84:ba:ac:d0:70:2d:eb:9d:65: + 76:87:99:ca:77:8c:ce:cd:d9:ef:46:8c:8f:1d:75: + 07:1c:3c:0a:4a:07:08:c0:c6:17:53:e1:77:ae:51: + 37:a9:e1:5a:da:29:68:cb:4d:60:ef:00:11:df:09: + 51:41:d6:d5:f7:1e:eb:04:59:a4:bc:9f:fa:47:f8: + de:0b:51:f9:f1:4b:71:e7:16:a1:82:e6:26:f6:0d: + 28:05:4c:29:cb:41:28:fc:da:25:20:00:27:5f:f3: + b8:99:4e:89:ec:fa:44:6f:a9:8b:1d:c2:9b:20:05: + 49:ce:fb:0f:a8:73:9a:3b:36:3d:5b:24:99:53:44: + 4b:18:ea:44:06:63:de:1a:b3:8a:42:92:6a:a1:a0: + 14:c2:ef:f9:5e:f7:01:46:99:78:a9:45:88:b7:a5: + f4:93:3a:a0:10:25:fa:96:d5:59:50:e0:16:dc:de: + 53:79:2f:e3:8d:4e:61:57:3a:04:c9:e6:a8:5a:4b: + c7:20:96:76:04:94:92:35:41:04:da:83:25:90:30: + 7b:81:0c:83:29:af:72:2f:ff:9a:94:53:37:48:33: + 3a:0c:ae:95:fa:89:e9:f7:19:a9:ff:8f:28:e3:ce: + f3:fd:e6:32:6f:1a:e8:d0:b9:dc:3f:9b:25:ca:f1: + 53:94:4e:15:6d:b2:d4:43:14:0d:dc:22:85:78:52: + a3:0e:0f:3f:30:ea:a6:14:d0:63:d8:14:1a:bf:ff: + 55:1d:a4:3a:e4:fd:f0:4f:05:ed:cc:2c:94:f6:fa: + c0:7f:91:3c:f0:c7:16:2c:2a:a7:1d:8c:f1:b6:06: + dd:d8:da:79:3b:84:2c:c4:d0:ab:ae:7a:a0:90:3e: + 4f:f7:7e:89:98:d5:30:f2:4e:55:8e:ae:20:12:e3: + 41:18:b9:ee:51:87:1f:6a:af:c9 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Alternative Name: @@ -90,75 +90,72 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - C8:BD:A8:B4:C0:F2:32:10:73:47:9C:48:81:32:F8:BA:BB:26:84:97 + E4:3D:4C:DB:96:85:CC:90:ED:0F:58:1E:8D:4D:7D:34:6C:4E:11:10 X509v3 Authority Key Identifier: - keyid:B3:8A:A0:A2:BA:71:F1:A8:24:79:D4:A4:5B:25:36:15:1E:49:C8:CD + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B - Authority Information Access: CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer OCSP - URI:http://testca.pythontest.net/testca/ocsp/ - X509v3 CRL Distribution Points: - Full Name: URI:http://testca.pythontest.net/testca/revocation.crl - Signature Algorithm: sha256WithRSAEncryption - 76:87:76:4d:e4:0f:88:bf:2c:f3:58:67:c0:97:6c:cd:59:18: - 82:83:4c:04:19:a5:6d:aa:fa:64:3d:49:32:3e:e1:56:95:b2: - 13:f7:cf:d3:11:b0:72:b7:5b:e7:d7:85:69:51:3c:b6:54:80: - 45:2f:28:10:21:20:b9:ba:e9:27:5a:b7:3f:82:b7:69:f5:46: - f5:bf:a2:8b:17:7f:f2:14:d1:46:97:b5:8b:47:fb:9f:e8:5c: - 05:0e:9d:11:bd:7c:9a:03:84:0b:ca:29:66:4a:ca:0d:6f:09: - 1e:7a:27:c1:7f:03:96:70:8d:18:a5:2f:a4:98:a5:19:aa:8c: - 5d:1e:8c:3e:bb:6d:3b:c0:33:c0:15:e1:bd:09:3d:9f:e8:dc: - 12:d4:cb:44:1d:06:f5:e8:d6:4e:a1:2d:5c:9f:5d:1f:5b:2a: - c3:4d:40:8d:da:d1:78:80:d0:c6:31:72:10:48:8a:e9:10:7a: - 13:30:11:b2:9e:67:0e:ed:a1:aa:ec:73:2d:f0:b8:8a:22:75: - 0f:30:69:5c:50:7e:91:ce:da:91:c7:70:8c:65:ff:f6:58:fb: - 00:bd:45:cc:e2:e4:e3:e5:16:36:7d:f3:a2:4a:9c:45:ff:d9: - a5:16:e0:2f:b5:5b:6c:e6:8a:13:15:48:73:bd:7c:80:33:c3: - d4:3b:3a:1d:85:0e:a4:f7:f7:fb:48:0c:e9:a0:4b:5e:8a:5c: - 67:f8:25:02:6f:cd:72:c1:aa:5a:93:64:7c:14:20:43:e0:13: - 7f:0d:e1:0d:61:5e:2e:2c:cd:7a:2e:2a:ae:b6:75:6a:5f:a0: - 1a:9b:b6:67:2d:b0:a5:1c:54:bc:8c:70:7e:15:2b:c0:50:e3: - 03:bb:a4:a5:fc:45:01:c9:3f:a7:b8:18:dc:3e:08:07:a1:9b: - f5:bd:95:bd:49:e8:10:7c:91:7d:2d:c4:c2:98:b6:b7:51:69: - d7:0a:68:40:b5:0f:85:a0:a9:67:77:c6:68:cb:0e:58:34:b3: - 58:e7:c8:7c:09:67 + Signature Value: + 63:e9:cd:0f:ec:dd:27:a0:fc:76:16:a1:f2:5f:d3:07:9a:3f: + 97:f4:11:e8:20:e9:3c:dd:16:4f:20:62:71:73:a9:46:3e:85: + 6e:44:be:9d:08:4b:eb:80:08:d5:1b:93:5f:4a:0f:df:b7:1b: + 38:e2:2b:5c:68:48:ea:0a:58:5c:36:6c:79:a4:1e:b5:7e:ef: + cb:8d:5d:bd:7d:4a:e6:4e:dd:0b:87:10:ff:01:0e:9b:8b:bd: + de:1c:9a:25:fb:a2:e1:52:7b:8a:aa:08:37:b2:87:f7:45:9d: + 0b:ab:11:6b:0c:7f:db:ed:de:cc:1e:86:f0:be:30:25:6b:4b: + ff:f5:f2:99:ed:b0:b1:68:c1:44:0b:79:7b:81:95:b5:36:37: + 12:c4:99:9e:85:e8:2f:2d:cc:cd:d8:c6:f2:20:ec:6a:06:fd: + b8:fc:ff:02:11:95:4d:38:0d:42:8b:b2:43:eb:c2:b9:a4:e0: + 33:f1:da:25:2f:11:cf:57:b9:25:e5:ab:92:f2:b2:b5:11:2f: + bd:31:f7:55:eb:96:94:78:5a:ad:9c:8f:56:ec:34:a0:73:9c: + 01:c2:76:24:e0:f8:b3:c9:23:b9:ea:ab:c3:0a:d3:f8:53:44: + c7:12:37:76:44:d7:ee:25:93:f4:1c:1d:7c:fe:06:2c:fa:c5: + bb:c6:90:a4:57:fd:09:2e:da:af:f3:ac:e6:6d:7f:03:a1:26: + 36:9a:51:7d:8d:28:a7:5d:5d:37:57:cc:6a:11:2a:98:1d:57: + 71:32:6e:98:7c:12:28:ef:5c:2f:26:29:d4:56:0a:b1:23:d9: + 9d:35:20:1a:ec:cc:51:53:6e:ef:1c:e7:bc:3c:21:e7:64:31: + b1:4c:f3:55:b5:c4:93:17:d8:72:5a:05:2e:4a:e5:33:07:0e: + 7d:cf:22:73:0b:67:9f:b4:82:60:cd:71:f8:76:0c:c4:dc:98: + ee:49:f9:03:7f:0d:d8:c6:76:79:3c:28:ed:77:77:d3:7e:d5: + aa:1e:6e:2e:df:a5 -----BEGIN CERTIFICATE----- -MIIF9zCCBF+gAwIBAgIJAMstgJlaaVJdMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIF+TCCBGGgAwIBAgIJAMstgJlaaVJdMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMGIxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xFTATBgNVBAMMDGZh -a2Vob3N0bmFtZTCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBALfjLdLq -Jd2Z290z/ZoUF7oK9G9nCGHWG4aXshBG12hqQ+owuUR00cm014XkzbvEt6L6njC3 -s3UnKLTaR4dBvWb1OofkCu46QTa2L7XnMSgqhT1pbJ1OIf56kpNPTKODY1wpCNFR -pZxuHMEfgQwq4IiXMgjPz8QgqQ6qEIPCrr4W/cDKz7PsHP7JHzSLGK8q7+XtKPFD -vPtH+QIjs5WJFwg7lAFA+PtUu8JPPhUDq78t1eeArYc25q7pZebnI01BuqsHXUAQ -lxhl/9xWud5XgX5mMuWgAqBKf0YHx5Y7KKVNDXxNwp34faJyKlFo//5BH8o/HkYr -9JdTebKqbrZA4Yju3/mS1V2MaGzq0dnHDVJGKKmIthZu1VGtLFrjBiWCmChTC+u+ -+0LmQA2dPDVUZazFyz7prv4+hxicmKruThvk4BRvdHK+X0MsAzLw2qlFajhwcP9d -DcUJjAUeU5KDH90vlJ7SLd1bUrNs/o69o2DqeX/URc9SwnUBNLyfO/OFxwIDAQAB -o4IBwzCCAb8wFwYDVR0RBBAwDoIMZmFrZWhvc3RuYW1lMA4GA1UdDwEB/wQEAwIF -oDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAd -BgNVHQ4EFgQUyL2otMDyMhBzR5xIgTL4ursmhJcwfQYDVR0jBHYwdIAUs4qgorpx -8agkedSkWyU2FR5JyM2hUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRo -b24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZl -coIJAMstgJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6 -Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1Bggr -BgEFBQcwAYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2Nz -cC8wQwYDVR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5l -dC90ZXN0Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAHaHdk3k -D4i/LPNYZ8CXbM1ZGIKDTAQZpW2q+mQ9STI+4VaVshP3z9MRsHK3W+fXhWlRPLZU -gEUvKBAhILm66Sdatz+Ct2n1RvW/oosXf/IU0UaXtYtH+5/oXAUOnRG9fJoDhAvK -KWZKyg1vCR56J8F/A5ZwjRilL6SYpRmqjF0ejD67bTvAM8AV4b0JPZ/o3BLUy0Qd -BvXo1k6hLVyfXR9bKsNNQI3a0XiA0MYxchBIiukQehMwEbKeZw7toarscy3wuIoi -dQ8waVxQfpHO2pHHcIxl//ZY+wC9Rczi5OPlFjZ986JKnEX/2aUW4C+1W2zmihMV -SHO9fIAzw9Q7Oh2FDqT39/tIDOmgS16KXGf4JQJvzXLBqlqTZHwUIEPgE38N4Q1h -Xi4szXouKq62dWpfoBqbtmctsKUcVLyMcH4VK8BQ4wO7pKX8RQHJP6e4GNw+CAeh -m/W9lb1J6BB8kX0txMKYtrdRadcKaEC1D4WgqWd3xmjLDlg0s1jnyHwJZw== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowYjELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEVMBMGA1UEAwwM +ZmFrZWhvc3RuYW1lMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAv7gi +P+HlUgiK8f/xnAgsXkD9ejLX9MALiDQUfbk8khfLBFaEuqzQcC3rnWV2h5nKd4zO +zdnvRoyPHXUHHDwKSgcIwMYXU+F3rlE3qeFa2iloy01g7wAR3wlRQdbV9x7rBFmk +vJ/6R/jeC1H58Utx5xahguYm9g0oBUwpy0Eo/NolIAAnX/O4mU6J7PpEb6mLHcKb +IAVJzvsPqHOaOzY9WySZU0RLGOpEBmPeGrOKQpJqoaAUwu/5XvcBRpl4qUWIt6X0 +kzqgECX6ltVZUOAW3N5TeS/jjU5hVzoEyeaoWkvHIJZ2BJSSNUEE2oMlkDB7gQyD +Ka9yL/+alFM3SDM6DK6V+onp9xmp/48o487z/eYybxro0LncP5slyvFTlE4VbbLU +QxQN3CKFeFKjDg8/MOqmFNBj2BQav/9VHaQ65P3wTwXtzCyU9vrAf5E88McWLCqn +HYzxtgbd2Np5O4QsxNCrrnqgkD5P936JmNUw8k5Vjq4gEuNBGLnuUYcfaq/JAgMB +AAGjggHDMIIBvzAXBgNVHREEEDAOggxmYWtlaG9zdG5hbWUwDgYDVR0PAQH/BAQD +AgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAA +MB0GA1UdDgQWBBTkPUzbloXMkO0PWB6NTX00bE4REDB9BgNVHSMEdjB0gBTACiso +Q95fyX1H5UebNvJljGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5 +dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2Vy +dmVyggkAyy2AmVppUlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0 +cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUG +CCsGAQUFBzABhilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9v +Y3NwLzBDBgNVHR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3Qu +bmV0L3Rlc3RjYS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAY+nN +D+zdJ6D8dhah8l/TB5o/l/QR6CDpPN0WTyBicXOpRj6FbkS+nQhL64AI1RuTX0oP +37cbOOIrXGhI6gpYXDZseaQetX7vy41dvX1K5k7dC4cQ/wEOm4u93hyaJfui4VJ7 +iqoIN7KH90WdC6sRawx/2+3ezB6G8L4wJWtL//Xyme2wsWjBRAt5e4GVtTY3EsSZ +noXoLy3MzdjG8iDsagb9uPz/AhGVTTgNQouyQ+vCuaTgM/HaJS8Rz1e5JeWrkvKy +tREvvTH3VeuWlHharZyPVuw0oHOcAcJ2JOD4s8kjueqrwwrT+FNExxI3dkTX7iWT +9BwdfP4GLPrFu8aQpFf9CS7ar/Os5m1/A6EmNppRfY0op11dN1fMahEqmB1XcTJu +mHwSKO9cLyYp1FYKsSPZnTUgGuzMUVNu7xznvDwh52QxsUzzVbXEkxfYcloFLkrl +MwcOfc8icwtnn7SCYM1x+HYMxNyY7kn5A38N2MZ2eTwo7Xd3037Vqh5uLt+l -----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycertecc.pem b/Lib/test/certdata/keycertecc.pem index 81daa4ccb94217..d503d49dc85f15 100644 --- a/Lib/test/certdata/keycertecc.pem +++ b/Lib/test/certdata/keycertecc.pem @@ -1,8 +1,8 @@ -----BEGIN PRIVATE KEY----- -MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDBcNwE+cm17mmr7Yg6d -0DNCnheGFOjkYH4tYzTyCkcZGShkmF/tKhIqb3imKz0Kx9+hZANiAATyp8ws6CuN -OI2/3MC4jZVSkmoDzm/X/ZrkEm4TVHKPSZ6kzZRpmmUlLS9l7SQZSLYyDAFBFzoG -JJYHhZNQXEO7HFszn6KnvLjhwS6ddzlaHPziEknrSr0OKhJmdJHrQAQ= +MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDBuhPMBtySJ8hFhn5hU +1yvdDujmB8ajjXNlkQ9GhGdghQIAfKmP9vWtSY+1+BRjpAahZANiAAQKLvQoetPb +S5tbgExB5vSDUtgc7hK1j6AN++En6AkW8KkK+O/31jkItqBoSQz8ts+VCU5cloTf +H6i5AS9zUkHr4DEyn+XxfiutzcNQPdQIBmygWAsUpa1XlZf8ExMFf/A= -----END PRIVATE KEY----- Certificate: Data: @@ -13,19 +13,19 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost-ecc Subject Public Key Info: Public Key Algorithm: id-ecPublicKey Public-Key: (384 bit) pub: - 04:f2:a7:cc:2c:e8:2b:8d:38:8d:bf:dc:c0:b8:8d: - 95:52:92:6a:03:ce:6f:d7:fd:9a:e4:12:6e:13:54: - 72:8f:49:9e:a4:cd:94:69:9a:65:25:2d:2f:65:ed: - 24:19:48:b6:32:0c:01:41:17:3a:06:24:96:07:85: - 93:50:5c:43:bb:1c:5b:33:9f:a2:a7:bc:b8:e1:c1: - 2e:9d:77:39:5a:1c:fc:e2:12:49:eb:4a:bd:0e:2a: - 12:66:74:91:eb:40:04 + 04:0a:2e:f4:28:7a:d3:db:4b:9b:5b:80:4c:41:e6: + f4:83:52:d8:1c:ee:12:b5:8f:a0:0d:fb:e1:27:e8: + 09:16:f0:a9:0a:f8:ef:f7:d6:39:08:b6:a0:68:49: + 0c:fc:b6:cf:95:09:4e:5c:96:84:df:1f:a8:b9:01: + 2f:73:52:41:eb:e0:31:32:9f:e5:f1:7e:2b:ad:cd: + c3:50:3d:d4:08:06:6c:a0:58:0b:14:a5:ad:57:95: + 97:fc:13:13:05:7f:f0 ASN1 OID: secp384r1 NIST CURVE: P-384 X509v3 extensions: @@ -38,69 +38,66 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - 79:11:98:86:15:4F:48:F4:31:0B:D2:CC:C8:26:3A:09:07:5D:96:40 + F3:39:E5:90:3F:58:6F:55:9D:91:D4:86:B8:1B:14:35:09:AC:06:97 X509v3 Authority Key Identifier: - keyid:B3:8A:A0:A2:BA:71:F1:A8:24:79:D4:A4:5B:25:36:15:1E:49:C8:CD + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server serial:CB:2D:80:99:5A:69:52:5B - Authority Information Access: CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer OCSP - URI:http://testca.pythontest.net/testca/ocsp/ - X509v3 CRL Distribution Points: - Full Name: URI:http://testca.pythontest.net/testca/revocation.crl - Signature Algorithm: sha256WithRSAEncryption - 6e:42:e8:a2:2d:28:14:e3:25:5c:c1:7e:54:e9:3a:ff:30:db: - 94:ba:b2:f6:5f:ae:9a:c1:90:b3:4f:ce:65:1d:84:64:c0:71: - 2c:44:8e:7e:00:79:f5:8c:4a:1d:34:13:44:de:99:2e:db:53: - ee:ec:74:97:4d:59:1a:09:82:4f:98:75:91:a7:a0:b9:da:5e: - 68:f5:32:85:be:36:3d:83:d4:ee:f9:87:67:31:85:41:53:9a: - e7:05:96:13:1c:88:2e:7f:33:b1:ee:bd:f9:50:52:24:ed:3d: - 92:95:6e:30:c3:af:74:a9:ee:15:bb:da:7c:14:50:8e:e3:99: - ea:ba:b4:37:8a:50:61:26:de:01:93:b8:a2:6b:d9:c7:38:5e: - b2:f8:96:3d:a8:9f:7d:0c:71:d4:7e:cc:a0:57:af:7e:ce:3f: - a7:a7:27:68:c1:28:d7:4f:44:c1:b4:93:c3:c7:35:2b:50:c3: - 8e:2c:d0:46:c1:3f:e1:67:d3:f0:81:ae:f3:5c:3e:4f:d5:a8: - 07:8f:e0:eb:ef:d8:dc:47:e0:3d:58:eb:de:0e:7f:b2:58:cb: - 5c:f1:2f:65:7e:0f:0d:cc:ca:ba:83:53:63:bc:dd:18:0c:ee: - ed:ec:96:88:d0:38:c5:d7:ab:e7:55:79:7b:6d:ba:c0:a0:e9: - 5c:ca:7c:fb:f8:70:c7:fb:f5:b2:b5:74:cb:f7:c0:0d:20:9f: - 1d:b7:4c:bf:8a:8d:cd:e3:bc:4e:30:78:02:12:a0:9b:d5:8f: - 49:3c:95:91:76:6e:7c:54:dc:61:7a:2e:20:ed:35:25:e0:c5: - 17:50:02:83:00:74:8f:f0:1c:97:96:08:fc:2e:63:a4:f7:97: - 87:43:2a:32:04:2d:4c:f9:1a:07:bf:68:91:fc:50:21:a1:3c: - 8d:8f:fb:83:57:83:1f:b6:55:5c:55:2f:58:64:ad:f3:27:ba: - d0:e3:cd:58:01:a3:c9:ba:1d:95:dc:30:d5:af:b9:20:ad:d9: - 48:ba:8d:9a:66:ee + Signature Value: + b8:d9:bf:b8:88:0a:01:2b:aa:64:32:e8:97:e1:0f:ee:34:40: + ef:71:fc:e5:f4:a2:3b:26:00:e2:19:3b:3e:cb:8e:2b:51:4c: + 30:ff:ab:44:9d:28:8d:d2:9c:32:e3:6b:96:73:1c:9d:55:76: + b2:bf:af:b8:51:ed:fd:04:e7:0d:fa:8c:2a:68:01:cb:92:90: + 7d:d1:31:d8:6c:f4:b8:4b:ea:62:59:1e:31:4e:f7:17:ae:3f: + f0:b6:ff:f8:6e:64:e3:6e:e9:19:4f:d0:1e:84:1e:df:56:49: + ae:90:a9:e0:7b:70:e6:97:7f:08:f2:03:49:82:7d:58:8e:a5: + a0:88:59:f3:a1:ab:b7:dd:6b:9a:2b:79:64:9c:d3:07:7c:ec: + a3:8d:56:77:28:10:c9:42:a5:fa:81:8e:35:ad:f7:28:da:58: + cc:a0:61:fa:0d:75:fd:26:ab:07:88:c5:64:21:f1:98:1c:e3: + 13:51:38:a7:59:a5:59:88:b9:10:4d:c1:79:70:3f:36:9e:08: + 75:53:7f:77:a3:3d:5d:16:b7:3b:a2:f6:eb:41:9b:56:16:80: + 34:96:57:f0:3a:3d:91:43:17:51:2b:64:2c:d7:e2:25:45:85: + f3:5f:73:f8:7d:aa:a3:6d:61:e8:ba:c5:90:5e:16:50:bc:79: + b9:4e:df:66:db:ae:95:80:15:da:4e:bf:8f:4e:9d:e3:7d:b0: + ab:8b:72:93:3c:56:1b:92:d3:67:07:d5:5a:ad:98:16:69:ea: + 46:fe:e0:d0:f4:ae:95:d3:b5:80:7d:f2:64:4d:14:c0:97:d2: + f3:91:bb:e6:43:d0:7e:39:14:0b:95:ab:c1:56:fd:26:79:f8: + 8c:69:eb:bc:74:0e:ea:cd:94:62:7e:30:58:01:7f:84:ee:9d: + 58:9c:fc:8f:75:1e:d9:86:cb:f4:fd:4c:af:b2:f7:69:ea:58: + cb:7c:93:ce:3d:86:f0:46:01:df:86:02:a1:6c:0f:fb:13:84: + d6:75:23:f9:17:21 -----BEGIN CERTIFICATE----- -MIIEyzCCAzOgAwIBAgIJAMstgJlaaVJeMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIEzTCCAzWgAwIBAgIJAMstgJlaaVJeMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaMGMxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xFjAUBgNVBAMMDWxv -Y2FsaG9zdC1lY2MwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAATyp8ws6CuNOI2/3MC4 -jZVSkmoDzm/X/ZrkEm4TVHKPSZ6kzZRpmmUlLS9l7SQZSLYyDAFBFzoGJJYHhZNQ -XEO7HFszn6KnvLjhwS6ddzlaHPziEknrSr0OKhJmdJHrQASjggHEMIIBwDAYBgNV -HREEETAPgg1sb2NhbGhvc3QtZWNjMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAU -BggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUeRGY -hhVPSPQxC9LMyCY6CQddlkAwfQYDVR0jBHYwdIAUs4qgorpx8agkedSkWyU2FR5J -yM2hUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg -Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcoIJAMstgJlaaVJb -MIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6Ly90ZXN0Y2EucHl0 -aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1BggrBgEFBQcwAYYpaHR0 -cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2NzcC8wQwYDVR0fBDww -OjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcmV2 -b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAG5C6KItKBTjJVzBflTpOv8w -25S6svZfrprBkLNPzmUdhGTAcSxEjn4AefWMSh00E0TemS7bU+7sdJdNWRoJgk+Y -dZGnoLnaXmj1MoW+Nj2D1O75h2cxhUFTmucFlhMciC5/M7HuvflQUiTtPZKVbjDD -r3Sp7hW72nwUUI7jmeq6tDeKUGEm3gGTuKJr2cc4XrL4lj2on30McdR+zKBXr37O -P6enJ2jBKNdPRMG0k8PHNStQw44s0EbBP+Fn0/CBrvNcPk/VqAeP4Ovv2NxH4D1Y -694Of7JYy1zxL2V+Dw3MyrqDU2O83RgM7u3slojQOMXXq+dVeXttusCg6VzKfPv4 -cMf79bK1dMv3wA0gnx23TL+Kjc3jvE4weAISoJvVj0k8lZF2bnxU3GF6LiDtNSXg -xRdQAoMAdI/wHJeWCPwuY6T3l4dDKjIELUz5Gge/aJH8UCGhPI2P+4NXgx+2VVxV -L1hkrfMnutDjzVgBo8m6HZXcMNWvuSCt2Ui6jZpm7g== +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowYzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEWMBQGA1UEAwwN +bG9jYWxob3N0LWVjYzB2MBAGByqGSM49AgEGBSuBBAAiA2IABAou9Ch609tLm1uA +TEHm9INS2BzuErWPoA374SfoCRbwqQr47/fWOQi2oGhJDPy2z5UJTlyWhN8fqLkB +L3NSQevgMTKf5fF+K63Nw1A91AgGbKBYCxSlrVeVl/wTEwV/8KOCAcQwggHAMBgG +A1UdEQQRMA+CDWxvY2FsaG9zdC1lY2MwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQW +MBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBTz +OeWQP1hvVZ2R1Ia4GxQ1CawGlzB9BgNVHSMEdjB0gBTACisoQ95fyX1H5UebNvJl +jGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyggkAyy2AmVpp +UlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rlc3RjYS5w +eXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUFBzABhilo +dHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBDBgNVHR8E +PDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9y +ZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAuNm/uIgKASuqZDLol+EP +7jRA73H85fSiOyYA4hk7PsuOK1FMMP+rRJ0ojdKcMuNrlnMcnVV2sr+vuFHt/QTn +DfqMKmgBy5KQfdEx2Gz0uEvqYlkeMU73F64/8Lb/+G5k427pGU/QHoQe31ZJrpCp +4Htw5pd/CPIDSYJ9WI6loIhZ86Grt91rmit5ZJzTB3zso41WdygQyUKl+oGONa33 +KNpYzKBh+g11/SarB4jFZCHxmBzjE1E4p1mlWYi5EE3BeXA/Np4IdVN/d6M9XRa3 +O6L260GbVhaANJZX8Do9kUMXUStkLNfiJUWF819z+H2qo21h6LrFkF4WULx5uU7f +ZtuulYAV2k6/j06d432wq4tykzxWG5LTZwfVWq2YFmnqRv7g0PSuldO1gH3yZE0U +wJfS85G75kPQfjkUC5WrwVb9Jnn4jGnrvHQO6s2UYn4wWAF/hO6dWJz8j3Ue2YbL +9P1Mr7L3aepYy3yTzj2G8EYB34YCoWwP+xOE1nUj+Rch -----END CERTIFICATE----- diff --git a/Lib/test/certdata/leaf-missing-aki.ca.pem b/Lib/test/certdata/leaf-missing-aki.ca.pem new file mode 100644 index 00000000000000..36b202ae02ee54 --- /dev/null +++ b/Lib/test/certdata/leaf-missing-aki.ca.pem @@ -0,0 +1,13 @@ +# Taken from x509-limbo's `rfc5280::aki::leaf-missing-aki` testcase. +# See: https://x509-limbo.com/testcases/rfc5280/#rfc5280akileaf-missing-aki +-----BEGIN CERTIFICATE----- +MIIBkDCCATWgAwIBAgIUGjIb/aYm9u9fBh2o4GAYRJwk5XIwCgYIKoZIzj0EAwIw +GjEYMBYGA1UEAwwPeDUwOS1saW1iby1yb290MCAXDTcwMDEwMTAwMDAwMVoYDzI5 +NjkwNTAzMDAwMDAxWjAaMRgwFgYDVQQDDA94NTA5LWxpbWJvLXJvb3QwWTATBgcq +hkjOPQIBBggqhkjOPQMBBwNCAARUzBhjMOkO911U65Fvs4YmL1YPNj63P9Fa+g9U +KrUqiIy8WjaDXdIe8g8Zj0TalpbU1gYCs3atteMxgIp6qxwHo1cwVTAPBgNVHRMB +Af8EBTADAQH/MAsGA1UdDwQEAwICBDAWBgNVHREEDzANggtleGFtcGxlLmNvbTAd +BgNVHQ4EFgQUcv1fyqgezMGzmo+lhmUkdUuAbIowCgYIKoZIzj0EAwIDSQAwRgIh +AIOErPSRlWpnyMub9UgtPF/lSzdvnD4Q8KjLQppHx6oPAiEA373p4L/HvUbs0xg8 +6/pLyn0RT02toKKJcMV3ChohLtM= +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/leaf-missing-aki.keycert.pem b/Lib/test/certdata/leaf-missing-aki.keycert.pem new file mode 100644 index 00000000000000..0fd2ab39bf2c70 --- /dev/null +++ b/Lib/test/certdata/leaf-missing-aki.keycert.pem @@ -0,0 +1,18 @@ +# Taken from x509-limbo's `rfc5280::aki::leaf-missing-aki` testcase. +# See: https://x509-limbo.com/testcases/rfc5280/#rfc5280akileaf-missing-aki +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIF5Re+/FP3rg+7c1odKEQPXhb9V65kXnlZIWHDG9gKrLoAoGCCqGSM49 +AwEHoUQDQgAE1WAQMdC7ims7T9lpK9uzaCuKqHb/oNMbGjh1f10pOHv3Z+oAvsqF +Sv3hGzreu69YLy01afA6sUCf1AA/95dKkg== +-----END EC PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIBjjCCATWgAwIBAgIUVlBgclml+OXlrWzZfcgYCiNm96UwCgYIKoZIzj0EAwIw +GjEYMBYGA1UEAwwPeDUwOS1saW1iby1yb290MCAXDTcwMDEwMTAwMDAwMVoYDzI5 +NjkwNTAzMDAwMDAxWjAWMRQwEgYDVQQDDAtleGFtcGxlLmNvbTBZMBMGByqGSM49 +AgEGCCqGSM49AwEHA0IABNVgEDHQu4prO0/ZaSvbs2griqh2/6DTGxo4dX9dKTh7 +92fqAL7KhUr94Rs63ruvWC8tNWnwOrFAn9QAP/eXSpKjWzBZMB0GA1UdDgQWBBS3 +yYRQQwo3syjGVQ8Yw7/XRZHbpzALBgNVHQ8EBAMCB4AwEwYDVR0lBAwwCgYIKwYB +BQUHAwEwFgYDVR0RBA8wDYILZXhhbXBsZS5jb20wCgYIKoZIzj0EAwIDRwAwRAIg +BVq7lw4Y5MPEyisPhowMWd4KnERupdM5qeImDO+dD7ICIE/ksd6Wz1b8rMAfllNV +yiYst9lfwTd2SkFgdDNUDFud +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/make_ssl_certs.py b/Lib/test/certdata/make_ssl_certs.py index 94a35a64ab1abe..18e614496385fd 100644 --- a/Lib/test/certdata/make_ssl_certs.py +++ b/Lib/test/certdata/make_ssl_certs.py @@ -1,6 +1,7 @@ """Make the custom certificate and private key files used by test_ssl and friends.""" +import argparse import os import pprint import shutil @@ -8,7 +9,8 @@ from subprocess import * startdate = "20180829142316Z" -enddate = "20371028142316Z" +enddate_default = "25251028142316Z" +days_default = "140000" req_template = """ [ default ] @@ -79,8 +81,8 @@ default_startdate = {startdate} enddate = {enddate} default_enddate = {enddate} - default_days = 7000 - default_crl_days = 7000 + default_days = {days} + default_crl_days = {days} certificate = pycacert.pem private_key = pycakey.pem serial = $dir/serial @@ -109,14 +111,15 @@ subjectKeyIdentifier=hash authorityKeyIdentifier=keyid:always,issuer - basicConstraints = CA:true + basicConstraints = critical, CA:true + keyUsage = critical, digitalSignature, keyCertSign, cRLSign """ here = os.path.abspath(os.path.dirname(__file__)) -def make_cert_key(hostname, sign=False, extra_san='', +def make_cert_key(cmdlineargs, hostname, sign=False, extra_san='', ext='req_x509_extensions_full', key='rsa:3072'): print("creating cert for " + hostname) tempnames = [] @@ -129,13 +132,13 @@ def make_cert_key(hostname, sign=False, extra_san='', hostname=hostname, extra_san=extra_san, startdate=startdate, - enddate=enddate + enddate=cmdlineargs.enddate, + days=cmdlineargs.days ) with open(req_file, 'w') as f: f.write(req) - args = ['req', '-new', '-nodes', '-days', '7000', + args = ['req', '-new', '-nodes', '-days', cmdlineargs.days, '-newkey', key, '-keyout', key_file, - '-extensions', ext, '-config', req_file] if sign: with tempfile.NamedTemporaryFile(delete=False) as f: @@ -144,7 +147,7 @@ def make_cert_key(hostname, sign=False, extra_san='', args += ['-out', reqfile ] else: - args += ['-x509', '-out', cert_file ] + args += ['-extensions', ext, '-x509', '-out', cert_file ] check_call(['openssl'] + args) if sign: @@ -174,7 +177,7 @@ def make_cert_key(hostname, sign=False, extra_san='', def unmake_ca(): shutil.rmtree(TMP_CADIR) -def make_ca(): +def make_ca(cmdlineargs): os.mkdir(TMP_CADIR) with open(os.path.join('cadir','index.txt'),'a+') as f: pass # empty file @@ -191,7 +194,8 @@ def make_ca(): hostname='our-ca-server', extra_san='', startdate=startdate, - enddate=enddate + enddate=cmdlineargs.enddate, + days=cmdlineargs.days ) t.write(req) t.flush() @@ -218,14 +222,22 @@ def make_ca(): shutil.copy('capath/ceff1710.0', 'capath/b1930218.0') -def print_cert(path): +def write_cert_reference(path): import _ssl - pprint.pprint(_ssl._test_decode_cert(path)) + refdata = pprint.pformat(_ssl._test_decode_cert(path)) + print(refdata) + with open(path + '.reference', 'w') as f: + print(refdata, file=f) if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Make the custom certificate and private key files used by test_ssl and friends.') + parser.add_argument('--days', default=days_default) + parser.add_argument('--enddate', default=enddate_default) + cmdlineargs = parser.parse_args() + os.chdir(here) - cert, key = make_cert_key('localhost', ext='req_x509_extensions_simple') + cert, key = make_cert_key(cmdlineargs, 'localhost', ext='req_x509_extensions_simple') with open('ssl_cert.pem', 'w') as f: f.write(cert) with open('ssl_key.pem', 'w') as f: @@ -242,24 +254,26 @@ def print_cert(path): f.write(cert) # For certificate matching tests - make_ca() - cert, key = make_cert_key('fakehostname', ext='req_x509_extensions_simple') + make_ca(cmdlineargs) + cert, key = make_cert_key(cmdlineargs, 'fakehostname', ext='req_x509_extensions_simple') with open('keycert2.pem', 'w') as f: f.write(key) f.write(cert) - cert, key = make_cert_key('localhost', sign=True) + cert, key = make_cert_key(cmdlineargs, 'localhost', sign=True) with open('keycert3.pem', 'w') as f: f.write(key) f.write(cert) - cert, key = make_cert_key('fakehostname', sign=True) + check_call(['openssl', 'x509', '-outform', 'pem', '-in', 'keycert3.pem', '-out', 'cert3.pem']) + + cert, key = make_cert_key(cmdlineargs, 'fakehostname', sign=True) with open('keycert4.pem', 'w') as f: f.write(key) f.write(cert) cert, key = make_cert_key( - 'localhost-ecc', sign=True, key='param:secp384r1.pem' + cmdlineargs, 'localhost-ecc', sign=True, key='param:secp384r1.pem' ) with open('keycertecc.pem', 'w') as f: f.write(key) @@ -279,7 +293,7 @@ def print_cert(path): 'RID.1 = 1.2.3.4.5', ] - cert, key = make_cert_key('allsans', sign=True, extra_san='\n'.join(extra_san)) + cert, key = make_cert_key(cmdlineargs, 'allsans', sign=True, extra_san='\n'.join(extra_san)) with open('allsans.pem', 'w') as f: f.write(key) f.write(cert) @@ -296,17 +310,17 @@ def print_cert(path): ] # IDN SANS, signed - cert, key = make_cert_key('idnsans', sign=True, extra_san='\n'.join(extra_san)) + cert, key = make_cert_key(cmdlineargs, 'idnsans', sign=True, extra_san='\n'.join(extra_san)) with open('idnsans.pem', 'w') as f: f.write(key) f.write(cert) - cert, key = make_cert_key('nosan', sign=True, ext='req_x509_extensions_nosan') + cert, key = make_cert_key(cmdlineargs, 'nosan', sign=True, ext='req_x509_extensions_nosan') with open('nosan.pem', 'w') as f: f.write(key) f.write(cert) unmake_ca() - print("update Lib/test/test_ssl.py and Lib/test/test_asyncio/utils.py") - print_cert('keycert.pem') - print_cert('keycert3.pem') + print("Writing out reference data for Lib/test/test_ssl.py and Lib/test/test_asyncio/utils.py") + write_cert_reference('keycert.pem') + write_cert_reference('keycert3.pem') diff --git a/Lib/test/certdata/nosan.pem b/Lib/test/certdata/nosan.pem index ec10cdcabb9e35..c397342fa86f6e 100644 --- a/Lib/test/certdata/nosan.pem +++ b/Lib/test/certdata/nosan.pem @@ -1,130 +1,137 @@ -----BEGIN PRIVATE KEY----- -MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQCv3sUoOE4F7Pye -AT2Q6XpXrGUOu1fYgdnItLLLhvn7ACuHMj7TA5UKXxsepJn5m2Ji9LvAbksr1IWd -LZAvNgjwsUR+E4HbY108BhVt9sk3HFkvE0OOFbAa14ICtYPe18P/4Hv6Zfu/GJDU -rwXHNCUu0p6i/mospZ5O3sx5MgVaShknGAEC3Kp7zOgusMmE8XSbkNQa3ARMkW4o -kTqWKAeAHDjVFVyyhzZQmo+gaLzhWfJVSZhlJsuiLoZGGrVTq85EiXsE4l8rPaI+ -mKkVzWP13IZW+Fx1tiIktumdHWb1OQWrvm8AiT9b8PcFCUUrvhQFcLDSCZjKlQ0t -RWrSSKrrVsSldOreqRLtpjGzFJpGnTcvslL7rP5pg5DjBsYmVcDjrmRuJuhGq52X -/6HEC97GouVK8tT1LVMv1wufVPn+i9TzwxOuRWeUvVqLAJgWQ9N3yKdymH+VrpZk -/oB9ScyDakGezZBW5CeOQbNJ8WoX58jNxefGjtqKxmyztu43r3ECAwEAAQKCAYBQ -fVoqYCqFV8L95X9x1QljGsldhqxbsIIl811o/KtoDtndFEfgd2E8z+4vhhHaRR0w -QOW02kWZF7jXCMVWdhp9XgQE15S0/bLsB7TDERFiIZ1HiD+AxbhFcKBV8REbahCQ -CQN0xDwFZ47RaBDy7JCf71EfM+UP7fSYECvww83jVspQNBIyZx+3bT5OMCbqqz88 -+3m3mT52dJDADEeN9WAJZ+Ey1IYKRwu6tCJLvePEF1BrbDVNBgZogXZ+mzalxpjr -4RpGPMMa+VWc8HmDVd+LtpwKJcQD00GvUP4fNywn+5jvNWl54FdQiTLPrieTWxas -XUQ2crxP7Aqr2/vsU5Ruru5uF7H+ssMHp9YQDhpJ2+SVhQ9P+/loXCuKGt+BrB2Z -MlitO3f+vfRtzATmJ8G0qFrOqZK1A/qsiyIze240C1hAl3oy2xpZqTDGp4gRWwoi -OIN0HmH9UbP7bbNQY1x/zstTbza4/7rGb1+DZKeZIMu7QjBCU0rtsJpGtUvcQGEC -gcEA42GMYSL/HljZMF1LsDhTX/cmP8FDNgONhWYxT+w0Csnj1usLNBaT63dYnEiW -QKydRR4casAR1Kdy4Yfcy2lCy1kCfwqkQYk8fxSsOSHRjUfwC1SnfdYlwKFMxw4a -oZF0R4oVCBYrfP+8kqrj+5gs/gXblsw72XkYtbCdIriKKdmUzTx7MegzSqh2PVRi -rJzuwCZQ/O0NfhwdOHxLQDo0dgD+vv9e+KOSoJ9FDv8HH1tnolpRMdkSA8AJR/Nk -DXt1AoHBAMYBfTKQZ2jqLKybe4tP+YKjvjVp8vJx0iNUXFN/P6hBaSBOgq85uxXL -X3s7N/pkOCjyE95B8QusIkbnbfdyEP89O4bTbUHPXyAkHyRkR7Vny49HYuaR/aXQ -mXC0J2z5bXVpCQ514l/R/Io3wBph+hbG3To7pp9pMOV4qzvibUZaTZFwH+q+xDwf -SKSFy3fcomgH4/K5/QuKVj0jOUQsYjQQWb8GukS2KZK3zYJIAG1bBcsCVpSuBdW0 -eCZgqjnwjQKBwCUyUwWc9QEg5b68tGIKhNEhHDe3xOf0ItWcxxpc+JJ/Pm9tGfMW -cnJFntBKK5I+6qdg6qMn8oLINcnhMORxvsSHNhpUQlSaP7RGTHo4JxCmoQUpfxDd -1GUzvdyeWQrvQYdmdlRRVCHpsA6KOCtzVIDlsmtz06Ka5cjrMHl6mNeJyYbdiwW6 -B5ICBv23bUDxlzkFy5/ko51qufkAlErYeraHKSVTn1SrZZQzGdf/LkoZ6NUtUzUF -XqYQZzRHA6oU9QKBwDslzLljC5D6ivfQxln6POV6dmJMUOd9erFVDPNgSqq/R2EA -MueXDjzXcKFGMlWYxHHuxmKZPiEnfWHC1kWZjFxCdVq0I6oKATd/stHTJtyYseUO -BQwtRiDXLE7PcguKgtkU1EC+lC3dc1vyhW8cH3HYW9N+aCqsaI/TuQr9e3kNlqhA -XzhnXgU7rx5+XSZkARukZ8JlLqLY4yQGNqAXxgoZbEW1A8VsyQRr5XbqfT4td5CK -FUT6qwGIlG+aZp9CLQKBwQCQkwdW9A/Q4Ffq8+XTL1hJ24m/q11OLAPODUypOhWw -OCbX2fkv59pSBe6niZDBls1NpHB9mzalBrJCfU+yKC667gKcKULOnWULIoOQvmcg -Ka3hkkW28gTnCjfDIYm3IdsLjc67zJplOixaKgxhO8NtJZGtg0oLIrofG8EYRInv -OmtGw+XE+s4TVs6WgXnEg9zWQ5ZYtqQVn6PT5jsz+Nrvipi61HWHVBd7g+78ojps -3suWxl0FvgzTW5HD16WRXeI= +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDD5QDHBBTqG5g/ +OW6kVb6oTHLM4q6wX42YofPGBjWFlIeMX78A1PzRvgKLo9RXbBOb/Q89rbtsL9dd ++FcUaNP7GNLnX+MpeOnMOL6M7LzihFi0OyRmvCytWIUMYkH8lqHuydwdUROsbqGk +2r7Sbe8XzDbf9dNnCYZnoOJMiGvp08D5FFGJL+K7N9xCEHCpHJ5mFyywzunoR3us +EgV+0b8nt287RcLoBNaORCLt6L1FSsSb6g3NS/AUNNDb1VLoa96HFLL3rBCqSt3y +SCGtz3lncEG5HXBRNEWoka4SKqHDOP8zDZZxtmNwWp6QtHaFj4rY/XnMGiekJgIG +crzeB8jAgwGC9u23Yo5/IX5OhgSIbqj0mtYVRwZwJbszXVQn7BNbQzNFNgnpJPpJ +xA4xDRr415IsAQHm7ugt3G0aX2PbbXYxOpF/Ud+3JQu5vYGbcMUxqOPaHi3C8QUT +rcvIRBm4W2L2fd3xANpC6RafRVXjfeGFfk0pyArlvIyYCGvxAeUCAwEAAQKCAYAE +A1FQwmMxHOhUbuAIRoUInwhMeLs7ZKJx8OsGPDSCTGRGGnAFD/rSP63cNy71T+AF +gggpiGQegz9KkeGfhQwM1GssLGk8WnCp2fnqbAwyhkZzpOs+WEJSCsKMFDc3S0GU +hChIYAJFtgRLJnAFM+b2V0uIfHCz26hGqKOdDb/Wumzd8zPhH+/SNI5hdM7xStez +2d/dlqVZ21c0ghiMLZ/M2qrz8EEvWccc4BEuYtqJYNpdgAauLIGF0SxByPt/oZMf +4fa0zOVfAL6F5cPFkycRzuz97Lx+ZdtLuUBx/3y8Sk24TR1QutwaFhbwpRSQSUrg +xT7ZmS24rQhaP8i7+ROLGy0CPlLMc/D3WvvtdLDWOUF30azURvAFcqKA87Bp05bR +HI/ewXaSB26J34MUmBHy5dBjld+HX6hQInPhCTZu7gNDOrPaEc8Ug50c1r7HkYRl +VBv/4BQo3YtNdoKhL08MVuNbcxrzge3kI0mbQu7qDSyrve02WnQ1zijJ916NbgEC +gcEA6oFNNtWzVw/7/MBxUZ1tE5VW5U2SjRvQZlcPTJ9Ub16byhOCdNvE17uyYLt5 +diqoVAvSGWiJ+OKsmw3XTLdhxzWeZ1a9ZiCozsmIfHMQmL9t3MxzTnMEZCfcvxz0 +DPvAN0REUtwRkMbt7vC+g01Eom9UpAH7hKXkc3pTgtkoyFeebC/YneGtdOTVATKU +WMfsY6Biye7P1D8V7btjLI8LTnv90ygoP0uSrX53hp7k4EFAeXyT9VN5wMXLw81a +5K69AoHBANXZsvSSIwe7fjspNUU9yELnEokDIdBn4ARX6x5PonL2KB0iVUzWgjHb +vs23GAzAgKKY+S8VLYptBLUBlsZ/pSLfZLWd1ywxeHmk/nkzuqlAw1jwuUlMq6Vr +BJmfhpOoXeSrQwvkL/uOejdrDSmLAFiIV3l9LtkmIJzfQSSy2QB8mfWwni0vxOJT +mIqZE01rfRTNLIqxO3+0d7pDqArVvWxYqF4C8kihUhNjkB5NaURUTzDQ3Y33aAvT +zAm697nGSQKBwQDEQ8ed9ykL2sLpfR7aUclytHBvpYbcNsUqgf66ADeopiP48m8i +4rRSYjMepok3jugmv2XuAgJHnV8cvm7NNEXPdl7G2l/V08u0lhN3JM5lKQIH4801 +gSnRsVMdWFwhaaosFySfvLOu2e9VJYQtXEPvNwI96bLaCAW1aFHwl1N8qWhb34eK +S9DinopvYCesTlbX4uoLW6XxW4M83rJYHrg1zaxYR6m3n8Z5EflzYBTqY3JUuyES +F/U0k9bAX2SNNHkCgcALk8mYbADxfjkLQuPbZ8jbtl7OhBjki3sZQRk9ftowlxr8 +2Mr9ae+Ke3cM9AidSB6urtFutxrMD7LdicR74pUyGh39pxnrDpKTI1eTgDVuzE7H +FeEyErCIOA77siM7AzZyFsN+dVATslbzgRwpT5kpMdhqf1h18RZ656tDLVuKJzS+ +lF073QYvqo7rkfX1jwgqhCERMR8jfsWsk9UZIREsOHCFBmvPesxSuGUo/s/gHyBa +aDRWZzp+yWyWakTXDeECgcBaPIxLK0ky89Mv8jTSyIJIMhQj8+z8wFLJFQG5vF/Z +1taLIDY0stU8HzKflTJ5v72r2jpIvu5YgDEgl+kKjyqLMWSu4nBZ2OoJet49S6d3 +X0YL2VzsIiSG+8cmVvBBf7iZCHDKiCM20dzzhZq8BCLiaPq+Zu2No2xEJyE0m8rc +I66z6fV1BWQXLe6vPC8EVvBWz3Ybje0czUdZnZ44qV6qyZzREXuqbC0qS5RanE4Z ++Ps9b3TezE7gDII6SbUwhZo= -----END PRIVATE KEY----- Certificate: Data: - Version: 1 (0x0) + Version: 3 (0x2) Serial Number: cb:2d:80:99:5a:69:52:61 Signature Algorithm: sha256WithRSAEncryption Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=nosan Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (3072 bit) + Public-Key: (3072 bit) Modulus: - 00:af:de:c5:28:38:4e:05:ec:fc:9e:01:3d:90:e9: - 7a:57:ac:65:0e:bb:57:d8:81:d9:c8:b4:b2:cb:86: - f9:fb:00:2b:87:32:3e:d3:03:95:0a:5f:1b:1e:a4: - 99:f9:9b:62:62:f4:bb:c0:6e:4b:2b:d4:85:9d:2d: - 90:2f:36:08:f0:b1:44:7e:13:81:db:63:5d:3c:06: - 15:6d:f6:c9:37:1c:59:2f:13:43:8e:15:b0:1a:d7: - 82:02:b5:83:de:d7:c3:ff:e0:7b:fa:65:fb:bf:18: - 90:d4:af:05:c7:34:25:2e:d2:9e:a2:fe:6a:2c:a5: - 9e:4e:de:cc:79:32:05:5a:4a:19:27:18:01:02:dc: - aa:7b:cc:e8:2e:b0:c9:84:f1:74:9b:90:d4:1a:dc: - 04:4c:91:6e:28:91:3a:96:28:07:80:1c:38:d5:15: - 5c:b2:87:36:50:9a:8f:a0:68:bc:e1:59:f2:55:49: - 98:65:26:cb:a2:2e:86:46:1a:b5:53:ab:ce:44:89: - 7b:04:e2:5f:2b:3d:a2:3e:98:a9:15:cd:63:f5:dc: - 86:56:f8:5c:75:b6:22:24:b6:e9:9d:1d:66:f5:39: - 05:ab:be:6f:00:89:3f:5b:f0:f7:05:09:45:2b:be: - 14:05:70:b0:d2:09:98:ca:95:0d:2d:45:6a:d2:48: - aa:eb:56:c4:a5:74:ea:de:a9:12:ed:a6:31:b3:14: - 9a:46:9d:37:2f:b2:52:fb:ac:fe:69:83:90:e3:06: - c6:26:55:c0:e3:ae:64:6e:26:e8:46:ab:9d:97:ff: - a1:c4:0b:de:c6:a2:e5:4a:f2:d4:f5:2d:53:2f:d7: - 0b:9f:54:f9:fe:8b:d4:f3:c3:13:ae:45:67:94:bd: - 5a:8b:00:98:16:43:d3:77:c8:a7:72:98:7f:95:ae: - 96:64:fe:80:7d:49:cc:83:6a:41:9e:cd:90:56:e4: - 27:8e:41:b3:49:f1:6a:17:e7:c8:cd:c5:e7:c6:8e: - da:8a:c6:6c:b3:b6:ee:37:af:71 + 00:c3:e5:00:c7:04:14:ea:1b:98:3f:39:6e:a4:55: + be:a8:4c:72:cc:e2:ae:b0:5f:8d:98:a1:f3:c6:06: + 35:85:94:87:8c:5f:bf:00:d4:fc:d1:be:02:8b:a3: + d4:57:6c:13:9b:fd:0f:3d:ad:bb:6c:2f:d7:5d:f8: + 57:14:68:d3:fb:18:d2:e7:5f:e3:29:78:e9:cc:38: + be:8c:ec:bc:e2:84:58:b4:3b:24:66:bc:2c:ad:58: + 85:0c:62:41:fc:96:a1:ee:c9:dc:1d:51:13:ac:6e: + a1:a4:da:be:d2:6d:ef:17:cc:36:df:f5:d3:67:09: + 86:67:a0:e2:4c:88:6b:e9:d3:c0:f9:14:51:89:2f: + e2:bb:37:dc:42:10:70:a9:1c:9e:66:17:2c:b0:ce: + e9:e8:47:7b:ac:12:05:7e:d1:bf:27:b7:6f:3b:45: + c2:e8:04:d6:8e:44:22:ed:e8:bd:45:4a:c4:9b:ea: + 0d:cd:4b:f0:14:34:d0:db:d5:52:e8:6b:de:87:14: + b2:f7:ac:10:aa:4a:dd:f2:48:21:ad:cf:79:67:70: + 41:b9:1d:70:51:34:45:a8:91:ae:12:2a:a1:c3:38: + ff:33:0d:96:71:b6:63:70:5a:9e:90:b4:76:85:8f: + 8a:d8:fd:79:cc:1a:27:a4:26:02:06:72:bc:de:07: + c8:c0:83:01:82:f6:ed:b7:62:8e:7f:21:7e:4e:86: + 04:88:6e:a8:f4:9a:d6:15:47:06:70:25:bb:33:5d: + 54:27:ec:13:5b:43:33:45:36:09:e9:24:fa:49:c4: + 0e:31:0d:1a:f8:d7:92:2c:01:01:e6:ee:e8:2d:dc: + 6d:1a:5f:63:db:6d:76:31:3a:91:7f:51:df:b7:25: + 0b:b9:bd:81:9b:70:c5:31:a8:e3:da:1e:2d:c2:f1: + 05:13:ad:cb:c8:44:19:b8:5b:62:f6:7d:dd:f1:00: + da:42:e9:16:9f:45:55:e3:7d:e1:85:7e:4d:29:c8: + 0a:e5:bc:8c:98:08:6b:f1:01:e5 Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 27:04:5B:37:51:0D:92:FD:2C:60:61:FE:4F:9C:66:20:41:2C:08:45 + X509v3 Authority Key Identifier: + C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 Signature Algorithm: sha256WithRSAEncryption - 91:42:c2:15:57:42:47:77:e7:0f:c5:55:26:b1:5b:c3:5e:ba: - 81:db:e1:a4:9f:b8:42:5a:21:c9:8c:18:ae:0f:90:ab:9a:24: - e7:d2:78:fc:bd:97:29:b1:5c:46:1f:5b:b8:d2:a7:87:f1:50: - 53:5b:d3:be:57:74:bd:e5:75:db:50:81:f7:37:95:0b:69:ef: - 39:8c:5c:82:d5:64:62:d5:8b:e9:e0:31:e1:73:d2:5a:2c:de: - 43:5a:06:e5:d3:4d:d0:35:e0:9f:c2:73:31:bc:35:69:d4:fb: - 7d:f0:1a:33:f7:f6:25:72:9c:a6:84:05:08:f6:b5:e8:04:10: - f1:1f:f2:95:ad:a1:f8:d8:80:a5:eb:75:43:99:33:90:0c:79: - fc:c0:87:08:95:20:aa:c2:81:0b:22:6f:56:f4:8f:2a:23:f8: - 40:47:1c:03:a5:b1:04:0a:04:4a:df:d0:88:a8:bc:31:f2:42: - 9b:d8:11:14:9e:e3:68:ea:07:2c:15:de:d2:36:5a:15:38:ed: - d2:af:0e:b4:b6:1d:a0:57:94:ea:c3:c7:4c:14:57:81:00:57: - 94:d3:b0:27:69:d7:48:02:6c:e5:97:f7:be:22:7c:38:24:af: - b2:b0:7b:08:75:1e:ca:2e:c7:41:ef:8b:74:cf:c9:c3:6f:39: - b9:52:41:18:c6:70:24:54:51:04:fe:5f:88:70:35:e5:1c:8e: - d6:67:69:44:44:33:9b:8c:fe:a5:b9:95:48:66:84:f3:1a:04: - ab:a3:57:c1:b6:b4:2f:28:12:45:2b:cb:42:d3:f4:a5:ce:7b: - 6c:1f:e4:c8:a9:e7:d4:6d:c8:27:2d:69:26:c5:e8:73:10:54: - 1f:c3:bf:fd:aa:f5:95:6f:f6:ca:d5:06:8f:1b:79:93:e3:86: - ba:8d:fe:a8:10:8f:95:3e:14:09:bf:ca:88:59:e2:93:b6:ec: - 03:a9:7e:dd:1f:5f:13:d3:29:b3:a6:f3:6a:df:30:53:44:c8: - cd:e5:82:57:bc:9c + Signature Value: + a0:b7:48:56:c6:b3:92:26:3f:d8:92:3c:ed:72:b5:89:ea:fd: + c9:66:da:ba:6a:8e:0d:04:ea:b2:fd:1f:e4:29:da:1e:c7:8f: + 5a:f0:88:74:dd:b3:f0:5e:a7:c4:77:13:cf:a8:19:fb:f2:2d: + ee:47:b4:0c:7c:5b:d3:dc:2f:2a:5c:bf:43:22:1c:91:d8:03: + 7d:44:90:c0:2d:fe:9e:7c:8b:ef:39:4e:b3:87:99:c8:eb:c2: + b7:cf:86:65:05:52:8c:15:b9:6a:8d:cd:e3:2a:29:d1:f5:87: + 42:11:c3:2e:42:ec:ed:26:55:8d:f3:ad:66:f4:79:72:f7:9e: + ed:bc:0c:5a:a7:74:ab:dc:57:e8:5c:99:b6:32:8f:7e:58:6e: + 70:48:ea:5d:a7:fa:b1:fc:c0:e9:50:3a:a4:53:21:e7:8b:77: + 54:2d:bb:64:1e:fc:88:86:90:c1:03:90:17:bb:3c:cf:ce:e8: + 48:32:c2:07:e5:8e:4f:93:a9:1a:e2:f5:93:a8:01:9f:30:26: + 1e:ed:b5:62:e9:25:c5:b0:32:e1:fc:bd:d6:48:b4:70:a9:e2: + cd:f6:a9:42:cb:bb:24:39:b9:34:fc:b9:cb:09:01:f0:5e:7e: + ef:b5:59:d6:88:31:a9:4c:be:7d:5b:de:4d:ec:84:1b:a1:1b: + d8:7d:83:cb:f1:04:c9:f1:f3:a4:08:05:3c:b5:96:13:1d:37: + 8a:23:83:22:86:72:17:13:5e:e8:89:06:58:cd:89:42:71:12: + e5:47:fc:f7:6e:96:28:8f:19:b9:d7:86:5b:c5:62:14:e1:5b: + 06:e7:e0:66:7e:fc:b7:9e:a9:99:14:e5:0a:d6:df:8f:b5:a2: + 1a:74:54:30:f6:f4:bf:1b:43:1d:be:4f:38:92:55:10:7b:d8: + 4f:e0:33:0f:40:2e:58:ec:9c:78:1b:43:17:b3:cb:0b:f5:34: + e2:7e:11:a1:90:b6:3c:79:6a:0b:91:ce:0b:8d:d5:60:e4:6d: + c8:2a:3d:40:6d:17 -----BEGIN CERTIFICATE----- -MIIEJDCCAowCCQDLLYCZWmlSYTANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQGEwJY -WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV -BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTgwODI5MTQyMzE2WhcNMzcxMDI4MTQyMzE2 -WjBbMQswCQYDVQQGEwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNV -BAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQ4wDAYDVQQDDAVub3NhbjCC -AaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAK/exSg4TgXs/J4BPZDpeles -ZQ67V9iB2ci0ssuG+fsAK4cyPtMDlQpfGx6kmfmbYmL0u8BuSyvUhZ0tkC82CPCx -RH4TgdtjXTwGFW32yTccWS8TQ44VsBrXggK1g97Xw//ge/pl+78YkNSvBcc0JS7S -nqL+aiylnk7ezHkyBVpKGScYAQLcqnvM6C6wyYTxdJuQ1BrcBEyRbiiROpYoB4Ac -ONUVXLKHNlCaj6BovOFZ8lVJmGUmy6IuhkYatVOrzkSJewTiXys9oj6YqRXNY/Xc -hlb4XHW2IiS26Z0dZvU5Bau+bwCJP1vw9wUJRSu+FAVwsNIJmMqVDS1FatJIqutW -xKV06t6pEu2mMbMUmkadNy+yUvus/mmDkOMGxiZVwOOuZG4m6EarnZf/ocQL3sai -5Ury1PUtUy/XC59U+f6L1PPDE65FZ5S9WosAmBZD03fIp3KYf5WulmT+gH1JzINq -QZ7NkFbkJ45Bs0nxahfnyM3F58aO2orGbLO27jevcQIDAQABMA0GCSqGSIb3DQEB -CwUAA4IBgQCRQsIVV0JHd+cPxVUmsVvDXrqB2+Gkn7hCWiHJjBiuD5CrmiTn0nj8 -vZcpsVxGH1u40qeH8VBTW9O+V3S95XXbUIH3N5ULae85jFyC1WRi1Yvp4DHhc9Ja -LN5DWgbl003QNeCfwnMxvDVp1Pt98Boz9/YlcpymhAUI9rXoBBDxH/KVraH42ICl -63VDmTOQDHn8wIcIlSCqwoELIm9W9I8qI/hARxwDpbEECgRK39CIqLwx8kKb2BEU -nuNo6gcsFd7SNloVOO3Srw60th2gV5Tqw8dMFFeBAFeU07AnaddIAmzll/e+Inw4 -JK+ysHsIdR7KLsdB74t0z8nDbzm5UkEYxnAkVFEE/l+IcDXlHI7WZ2lERDObjP6l -uZVIZoTzGgSro1fBtrQvKBJFK8tC0/SlzntsH+TIqefUbcgnLWkmxehzEFQfw7/9 -qvWVb/bK1QaPG3mT44a6jf6oEI+VPhQJv8qIWeKTtuwDqX7dH18T0ymzpvNq3zBT -RMjN5YJXvJw= +MIIEbzCCAtegAwIBAgIJAMstgJlaaVJhMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowWzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEOMAwGA1UEAwwF +bm9zYW4wggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDD5QDHBBTqG5g/ +OW6kVb6oTHLM4q6wX42YofPGBjWFlIeMX78A1PzRvgKLo9RXbBOb/Q89rbtsL9dd ++FcUaNP7GNLnX+MpeOnMOL6M7LzihFi0OyRmvCytWIUMYkH8lqHuydwdUROsbqGk +2r7Sbe8XzDbf9dNnCYZnoOJMiGvp08D5FFGJL+K7N9xCEHCpHJ5mFyywzunoR3us +EgV+0b8nt287RcLoBNaORCLt6L1FSsSb6g3NS/AUNNDb1VLoa96HFLL3rBCqSt3y +SCGtz3lncEG5HXBRNEWoka4SKqHDOP8zDZZxtmNwWp6QtHaFj4rY/XnMGiekJgIG +crzeB8jAgwGC9u23Yo5/IX5OhgSIbqj0mtYVRwZwJbszXVQn7BNbQzNFNgnpJPpJ +xA4xDRr415IsAQHm7ugt3G0aX2PbbXYxOpF/Ud+3JQu5vYGbcMUxqOPaHi3C8QUT +rcvIRBm4W2L2fd3xANpC6RafRVXjfeGFfk0pyArlvIyYCGvxAeUCAwEAAaNCMEAw +HQYDVR0OBBYEFCcEWzdRDZL9LGBh/k+cZiBBLAhFMB8GA1UdIwQYMBaAFMAKKyhD +3l/JfUflR5s28mWMZzviMA0GCSqGSIb3DQEBCwUAA4IBgQCgt0hWxrOSJj/Ykjzt +crWJ6v3JZtq6ao4NBOqy/R/kKdoex49a8Ih03bPwXqfEdxPPqBn78i3uR7QMfFvT +3C8qXL9DIhyR2AN9RJDALf6efIvvOU6zh5nI68K3z4ZlBVKMFblqjc3jKinR9YdC +EcMuQuztJlWN861m9Hly957tvAxap3Sr3FfoXJm2Mo9+WG5wSOpdp/qx/MDpUDqk +UyHni3dULbtkHvyIhpDBA5AXuzzPzuhIMsIH5Y5Pk6ka4vWTqAGfMCYe7bVi6SXF +sDLh/L3WSLRwqeLN9qlCy7skObk0/LnLCQHwXn7vtVnWiDGpTL59W95N7IQboRvY +fYPL8QTJ8fOkCAU8tZYTHTeKI4MihnIXE17oiQZYzYlCcRLlR/z3bpYojxm514Zb +xWIU4VsG5+Bmfvy3nqmZFOUK1t+PtaIadFQw9vS/G0Mdvk84klUQe9hP4DMPQC5Y +7Jx4G0MXs8sL9TTifhGhkLY8eWoLkc4LjdVg5G3IKj1AbRc= -----END CERTIFICATE----- diff --git a/Lib/test/certdata/pycacert.pem b/Lib/test/certdata/pycacert.pem index 360cd57426a5a8..c2c8b1ecdcfc97 100644 --- a/Lib/test/certdata/pycacert.pem +++ b/Lib/test/certdata/pycacert.pem @@ -7,93 +7,96 @@ Certificate: Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity Not Before: Aug 29 14:23:16 2018 GMT - Not After : Oct 28 14:23:16 2037 GMT + Not After : Oct 28 14:23:16 2525 GMT Subject: C=XY, O=Python Software Foundation CA, CN=our-ca-server Subject Public Key Info: Public Key Algorithm: rsaEncryption - RSA Public-Key: (3072 bit) + Public-Key: (3072 bit) Modulus: - 00:b1:84:d3:4f:5c:04:80:91:4f:82:49:ba:30:0b: - f7:e8:cb:f9:14:ef:3d:9f:0b:3f:0a:62:fc:1b:20: - a5:20:d1:60:5f:87:5a:1f:16:d1:ed:97:70:a6:da: - 1b:03:2c:7e:a0:5b:3c:4e:2f:16:7e:0e:89:29:89: - e1:10:0d:38:da:6a:77:5f:37:13:b3:28:8f:7b:5c: - 76:ad:9e:e8:d3:f5:9e:f5:83:aa:10:07:8d:e6:51: - 98:f0:7c:0d:52:f2:0c:21:1e:d8:b9:99:26:a9:25: - 03:27:bb:5c:ab:2e:33:27:a2:d6:23:a8:83:87:44: - 29:9f:97:b5:24:6f:d7:b9:0a:fd:28:ee:bb:fb:41: - 58:ea:1d:99:dd:44:86:ab:98:be:1c:dc:cb:a9:89: - 1d:36:5c:a9:e8:47:b5:f4:52:48:aa:b5:a4:67:ef: - 3e:d7:e2:d3:33:de:98:29:d8:7a:b0:59:5c:e7:b1: - 0e:cc:fd:9f:eb:f6:d5:3a:0e:0b:cf:fe:0b:3d:a2: - bf:45:18:ce:94:e7:a9:55:60:88:d4:d8:84:50:79: - 05:2e:41:03:74:ae:67:26:f6:5b:12:08:98:ce:0a: - 97:ed:01:0f:89:4f:17:5c:fa:3e:1d:35:24:47:92: - 32:bf:f7:a4:18:2b:3c:d0:48:99:e1:a2:cd:a3:cc: - 50:53:20:b5:c6:e3:66:85:7b:57:10:ec:33:4f:c1: - 77:e7:1b:7e:81:c6:c4:f3:45:20:c0:91:dd:13:76: - 7b:03:af:f6:76:8e:a2:83:63:57:dd:63:bc:bb:5a: - 1c:17:52:8a:d6:06:48:cc:0f:c7:d3:4f:e8:da:22: - 6c:86:f9:4e:5c:a6:29:07:3b:d8:56:4c:59:b3:20: - 49:07:7b:94:84:cf:2b:c3:1c:1a:4e:87:64:92:ba: - 42:e1:e6:ad:7d:1d:f6:54:90:6f:2b:e9:b3:cc:4b: - 2b:33:26:23:fd:65:c0:3c:f0:79:ad:c9:c1:81:ef: - 37:04:e0:27:3e:b0:ee:15:be:51 + 00:e6:ba:e2:e4:c1:c2:0c:1c:3e:62:d8:b9:5c:57: + 2e:52:b8:83:c5:88:3a:e6:9a:7a:f5:64:16:33:eb: + 37:6e:2f:7b:f3:68:03:45:65:47:5d:71:10:59:ca: + 2b:1b:00:6c:81:14:61:f4:86:59:3f:ea:fd:78:37: + 16:9d:43:f1:c4:f6:69:8c:c5:29:06:88:9e:26:22: + 04:ac:04:d8:87:34:48:39:eb:6b:f2:0b:92:aa:c3: + 6e:63:51:51:6b:c2:ad:ff:5c:c8:2f:b2:1b:9c:20: + 8a:40:3e:a2:2f:6a:ea:c8:d9:37:43:5c:dc:ed:92: + e2:d9:40:d2:61:9f:71:8a:f5:ed:39:ba:a8:5e:3e: + b5:21:63:10:d8:6f:b4:e2:11:01:0b:10:e8:bb:fb: + 62:ef:48:55:bc:f5:d2:9c:ab:68:ae:95:25:19:f2: + 97:7d:1a:dc:66:ea:88:5e:86:e4:cb:cb:69:4d:5e: + b0:a3:fb:6c:31:e4:28:60:5e:90:f1:d4:2e:10:50: + e1:85:f0:0d:5c:bd:dd:45:24:08:19:3e:1c:93:66: + 8f:2b:da:53:7d:04:1c:0e:42:c4:68:5e:a6:cd:a9: + 18:ed:a7:cd:6a:d0:d1:86:ba:90:ff:b7:4c:de:c7: + 43:24:6d:c7:1c:6b:9c:81:e7:e1:1b:57:25:90:a9: + 0e:c9:56:f3:f6:6b:5e:2d:b4:2e:40:50:9b:42:63: + d2:d6:99:1c:38:dc:cf:2b:2c:a7:72:f1:c7:5e:63: + 34:76:48:f4:3e:88:13:9e:86:16:53:2f:74:fb:87: + 01:8d:22:a4:68:33:ee:13:6c:7a:06:14:54:56:17: + 57:57:98:34:d0:0b:66:09:e3:88:09:f8:a5:15:1c: + 10:73:d0:88:50:99:5e:18:65:3b:ff:31:27:1b:5e: + c6:aa:41:fd:2d:2f:18:a7:c0:f2:ab:c7:22:b5:0b: + 69:d2:73:d1:bb:d0:1c:3d:fa:a4:35:62:cd:33:86: + c7:a0:23:0f:b9:6a:d5:d2:6d:8d Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: - B3:8A:A0:A2:BA:71:F1:A8:24:79:D4:A4:5B:25:36:15:1E:49:C8:CD + C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 X509v3 Authority Key Identifier: - keyid:B3:8A:A0:A2:BA:71:F1:A8:24:79:D4:A4:5B:25:36:15:1E:49:C8:CD - - X509v3 Basic Constraints: + C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 + X509v3 Basic Constraints: critical CA:TRUE + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign Signature Algorithm: sha256WithRSAEncryption - 6b:32:2f:e7:05:18:ea:5c:c9:95:f4:e0:c2:0c:41:5f:1a:0a: - 95:c9:c7:7d:05:ee:8a:56:29:35:50:40:b7:fe:9f:7b:5b:1c: - c3:69:2f:a0:cb:d2:b8:91:2f:50:19:62:f7:27:18:6d:95:7b: - 53:16:15:a2:5a:dc:14:e3:fb:b1:32:a9:69:db:a6:33:47:3c: - bb:1f:d2:dc:70:f9:6a:2e:0c:d8:8c:6d:e5:5d:1d:43:3c:4e: - 91:de:a0:c8:da:a0:4b:0e:9d:5e:b6:0f:4a:49:f0:7b:b6:53: - 9e:fd:35:14:5b:e3:4d:b4:18:a6:36:61:e8:8f:33:9b:d4:05: - f9:54:66:df:e0:cb:18:a3:4e:dc:17:a8:a0:b3:c1:a8:f4:d6: - 9d:ca:7f:68:53:1a:d7:95:da:e8:d3:9e:48:00:71:95:99:11: - 07:cf:96:c0:7d:ce:7d:30:e8:4f:e1:83:16:33:a1:ff:59:9b: - 3e:4c:e7:3a:38:01:9f:0f:67:4c:fd:2d:8b:4a:d4:01:46:37: - 33:e8:13:6b:15:a9:1d:68:76:45:a2:82:33:69:26:30:60:05: - c8:8f:bd:b4:75:ab:be:7a:8b:48:68:70:40:b4:1b:51:c5:e6: - 7a:ad:6b:4f:db:17:c0:60:67:2e:63:61:9b:2c:48:99:b8:76: - 45:a0:9e:cc:ef:33:1e:50:4e:ab:72:c3:65:c8:b2:79:b3:35: - 83:21:78:d3:8b:6c:3a:18:e8:65:32:39:b8:c0:9d:71:2f:35: - 36:8a:c0:17:62:d8:8b:3e:e1:22:18:2b:4c:63:a6:0e:9d:0a: - fa:ab:5b:35:fb:88:91:77:4c:8d:8c:9d:a9:cf:fc:ab:c2:e6: - 5a:05:7b:7e:04:6e:39:cf:93:ce:67:3b:7a:cb:af:b6:36:e1: - fb:71:64:45:d4:a6:f0:ce:ef:75:04:99:69:9a:e5:88:0a:10: - 02:74:89:ec:75:84:44:80:48:df:c1:f7:e9:37:ce:ce:92:92: - 5c:89:22:08:73:1f + Signature Value: + bc:e3:56:22:03:e4:5c:b9:67:ff:94:bc:75:9c:00:85:b0:d5: + 9c:c4:c3:29:66:5e:8b:b2:a9:a6:30:86:71:1a:6b:f2:00:c5: + 82:ab:5f:50:04:2a:fb:ed:b8:4c:b9:00:1b:49:57:92:11:cd: + a2:bc:cb:0f:a8:b4:61:f8:14:ca:a0:ec:40:17:ba:55:a1:c4: + bc:a6:b2:5a:ef:f4:20:10:77:47:d0:a0:c5:58:b9:6c:b5:10: + 7b:85:4a:43:a3:fb:2c:01:b9:77:17:b0:be:a0:ee:ae:ae:4d: + 67:86:48:89:57:86:78:ea:3c:ed:f0:41:35:8d:71:68:55:f9: + f2:e9:ac:32:d4:c6:a2:ef:ec:54:e6:c4:8e:2c:fd:bd:aa:60: + 56:65:33:95:ea:10:c6:74:04:eb:2a:6e:9b:11:f6:61:00:aa: + fd:ec:f2:0b:b1:4b:11:cd:93:eb:df:98:ae:4c:b4:07:04:4a: + e5:ef:ff:52:58:75:f5:3e:a4:71:e1:4a:72:5c:a9:8f:d4:aa: + 88:f0:6a:71:b4:c3:00:5f:99:6e:d7:91:af:6c:98:0d:64:c2: + 24:c7:9e:05:11:68:5e:24:62:e3:2e:45:ec:a3:34:f2:a3:9d: + 4d:e5:32:18:2f:74:fc:11:f1:36:50:4f:a0:40:29:68:5c:43: + 4c:23:6c:5d:72:c4:ec:52:76:eb:dc:b2:bc:1f:a6:c4:06:66: + 9b:5c:c7:cc:ca:f2:d1:25:4f:de:a5:1f:8d:e4:0c:49:b6:cf: + 85:40:a1:b9:1f:c6:c7:19:15:07:63:34:93:d0:57:a0:5a:70: + ec:af:4a:1c:72:17:1d:74:a3:6c:31:45:0b:33:7a:a1:b8:46: + db:c7:0e:64:4c:6f:b7:99:04:82:43:1f:e0:59:d6:99:21:27: + 28:09:40:ae:fc:c4:23:aa:a0:0c:08:05:2a:92:1c:db:23:9e: + d1:d5:63:ae:39:13:a3:12:88:5a:43:3c:4a:6e:32:f0:84:9f: + f9:09:0c:91:e7:b8 -----BEGIN CERTIFICATE----- -MIIEbTCCAtWgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +MIIEgjCCAuqgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx -NDIzMTZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg -Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCAaIwDQYJKoZI -hvcNAQEBBQADggGPADCCAYoCggGBALGE009cBICRT4JJujAL9+jL+RTvPZ8LPwpi -/BsgpSDRYF+HWh8W0e2XcKbaGwMsfqBbPE4vFn4OiSmJ4RANONpqd183E7Moj3tc -dq2e6NP1nvWDqhAHjeZRmPB8DVLyDCEe2LmZJqklAye7XKsuMyei1iOog4dEKZ+X -tSRv17kK/Sjuu/tBWOodmd1EhquYvhzcy6mJHTZcqehHtfRSSKq1pGfvPtfi0zPe -mCnYerBZXOexDsz9n+v21ToOC8/+Cz2iv0UYzpTnqVVgiNTYhFB5BS5BA3SuZyb2 -WxIImM4Kl+0BD4lPF1z6Ph01JEeSMr/3pBgrPNBImeGizaPMUFMgtcbjZoV7VxDs -M0/Bd+cbfoHGxPNFIMCR3RN2ewOv9naOooNjV91jvLtaHBdSitYGSMwPx9NP6Noi -bIb5TlymKQc72FZMWbMgSQd7lITPK8McGk6HZJK6QuHmrX0d9lSQbyvps8xLKzMm -I/1lwDzwea3JwYHvNwTgJz6w7hW+UQIDAQABo1AwTjAdBgNVHQ4EFgQUs4qgorpx -8agkedSkWyU2FR5JyM0wHwYDVR0jBBgwFoAUs4qgorpx8agkedSkWyU2FR5JyM0w -DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAYEAazIv5wUY6lzJlfTgwgxB -XxoKlcnHfQXuilYpNVBAt/6fe1scw2kvoMvSuJEvUBli9ycYbZV7UxYVolrcFOP7 -sTKpadumM0c8ux/S3HD5ai4M2Ixt5V0dQzxOkd6gyNqgSw6dXrYPSknwe7ZTnv01 -FFvjTbQYpjZh6I8zm9QF+VRm3+DLGKNO3BeooLPBqPTWncp/aFMa15Xa6NOeSABx -lZkRB8+WwH3OfTDoT+GDFjOh/1mbPkznOjgBnw9nTP0ti0rUAUY3M+gTaxWpHWh2 -RaKCM2kmMGAFyI+9tHWrvnqLSGhwQLQbUcXmeq1rT9sXwGBnLmNhmyxImbh2RaCe -zO8zHlBOq3LDZciyebM1gyF404tsOhjoZTI5uMCdcS81NorAF2LYiz7hIhgrTGOm -Dp0K+qtbNfuIkXdMjYydqc/8q8LmWgV7fgRuOc+Tzmc7esuvtjbh+3FkRdSm8M7v -dQSZaZrliAoQAnSJ7HWERIBI38H36TfOzpKSXIkiCHMf +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyMIIBojANBgkq +hkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA5rri5MHCDBw+Yti5XFcuUriDxYg65pp6 +9WQWM+s3bi9782gDRWVHXXEQWcorGwBsgRRh9IZZP+r9eDcWnUPxxPZpjMUpBoie +JiIErATYhzRIOetr8guSqsNuY1FRa8Kt/1zIL7IbnCCKQD6iL2rqyNk3Q1zc7ZLi +2UDSYZ9xivXtObqoXj61IWMQ2G+04hEBCxDou/ti70hVvPXSnKtorpUlGfKXfRrc +ZuqIXobky8tpTV6wo/tsMeQoYF6Q8dQuEFDhhfANXL3dRSQIGT4ck2aPK9pTfQQc +DkLEaF6mzakY7afNatDRhrqQ/7dM3sdDJG3HHGucgefhG1clkKkOyVbz9mteLbQu +QFCbQmPS1pkcONzPKyyncvHHXmM0dkj0PogTnoYWUy90+4cBjSKkaDPuE2x6BhRU +VhdXV5g00AtmCeOICfilFRwQc9CIUJleGGU7/zEnG17GqkH9LS8Yp8Dyq8citQtp +0nPRu9AcPfqkNWLNM4bHoCMPuWrV0m2NAgMBAAGjYzBhMB0GA1UdDgQWBBTACiso +Q95fyX1H5UebNvJljGc74jAfBgNVHSMEGDAWgBTACisoQ95fyX1H5UebNvJljGc7 +4jAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsF +AAOCAYEAvONWIgPkXLln/5S8dZwAhbDVnMTDKWZei7KppjCGcRpr8gDFgqtfUAQq +++24TLkAG0lXkhHNorzLD6i0YfgUyqDsQBe6VaHEvKayWu/0IBB3R9CgxVi5bLUQ +e4VKQ6P7LAG5dxewvqDurq5NZ4ZIiVeGeOo87fBBNY1xaFX58umsMtTGou/sVObE +jiz9vapgVmUzleoQxnQE6ypumxH2YQCq/ezyC7FLEc2T69+Yrky0BwRK5e//Ulh1 +9T6kceFKclypj9SqiPBqcbTDAF+ZbteRr2yYDWTCJMeeBRFoXiRi4y5F7KM08qOd +TeUyGC90/BHxNlBPoEApaFxDTCNsXXLE7FJ269yyvB+mxAZmm1zHzMry0SVP3qUf +jeQMSbbPhUChuR/GxxkVB2M0k9BXoFpw7K9KHHIXHXSjbDFFCzN6obhG28cOZExv +t5kEgkMf4FnWmSEnKAlArvzEI6qgDAgFKpIc2yOe0dVjrjkToxKIWkM8Sm4y8ISf ++QkMkee4 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/pycakey.pem b/Lib/test/certdata/pycakey.pem index 819bdef1ff9bfc..0248f985545e20 100644 --- a/Lib/test/certdata/pycakey.pem +++ b/Lib/test/certdata/pycakey.pem @@ -1,40 +1,40 @@ -----BEGIN PRIVATE KEY----- -MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQCxhNNPXASAkU+C -SbowC/foy/kU7z2fCz8KYvwbIKUg0WBfh1ofFtHtl3Cm2hsDLH6gWzxOLxZ+Dokp -ieEQDTjaandfNxOzKI97XHatnujT9Z71g6oQB43mUZjwfA1S8gwhHti5mSapJQMn -u1yrLjMnotYjqIOHRCmfl7Ukb9e5Cv0o7rv7QVjqHZndRIarmL4c3MupiR02XKno -R7X0UkiqtaRn7z7X4tMz3pgp2HqwWVznsQ7M/Z/r9tU6DgvP/gs9or9FGM6U56lV -YIjU2IRQeQUuQQN0rmcm9lsSCJjOCpftAQ+JTxdc+j4dNSRHkjK/96QYKzzQSJnh -os2jzFBTILXG42aFe1cQ7DNPwXfnG36BxsTzRSDAkd0TdnsDr/Z2jqKDY1fdY7y7 -WhwXUorWBkjMD8fTT+jaImyG+U5cpikHO9hWTFmzIEkHe5SEzyvDHBpOh2SSukLh -5q19HfZUkG8r6bPMSyszJiP9ZcA88HmtycGB7zcE4Cc+sO4VvlECAwEAAQKCAYB7 -gUnzALYxLOgAYYMkQm9si9zz768TpCNr+ooj5YZ9Wq6OSAEveBT+FErQCxaYErDW -qCNA0gn4Eezj9YWcQVa4vzHmEM+n6iRJU39ONC0Qqua5Ma10EY1sHIEnb2dlufku -YeOu3RrEu3eCgRxsDGySuvv5OxinV4kN++KPQzD3EOopPE+U81YFLCsMgsyfPlmm -gwc/IKIuXDHp5Vp2bXkZK98CYLV8RddjUw7SrkZNwx6cI9eET0CgTs7y4SrevoOy -jCdnA0j1HvL8AbLQuYoXo9fdGYDeq55hyYlxSMYLaEToZG3DJ0UAldrT+r7x52D8 -2QMnJUo2XHzVYPlXPJIAkFJisZZ36TkBvywCgXZMMLibPo9U6V0nfkybTtXKoory -nmgBv+XSGSNrVWMiygpDPqpX1G6bBgqUX3CiTlxtSkYYz1M4Vgj2cux5XEPTnVCq -CLVzvNIXZt1RyzXPxGWpPidCjOaiWBRT4u1Dol9fs3PmVvDaRxcKo9nspiUHCfEC -gcEA4GgxZ+IJwpAMHkdYId0oxjKgTqIg+Ua+EwfUoQT10ERl/k/V4cDwJRHT8lML -rKhTNQJMEE040jq+6mPJDl1KqMb/v05Q7fF22ToGw1HkZwK52O6CeEiJW4/J6bR1 -pZGN0irsa6GvzV65Y6gZVFEUl0JPRf8wPvQHXsWAw8/2LuXkXjV0ieIMq4pbWJf4 -kaid7dYLHnobiP9RVk7BGr7ifmCshoPjWp4TRMwYf6iIZrqMxUSX0QY8Xsqx6bch -LLx/AoHBAMqCvvwUKTrF4gKh5jyl6T6DTZ/Dujaz7BuAJdsSSHvuTa/Y1EfsQHZN -jABn89ZqHYDiyyCuVFO3dqhLtsPjhyFMSXj+98JYcL3FGKnqQqRTwtzzx2P2lV5X -U0WhrNRb3iLu79Tr8pE/2EPnvTr+J5b0DHEeRyM72LWs43zrDYHorH0/Aa5Qd37F -gDLCTBEl8jO5irRuAIq/KV9ZFnn8JDjNGVpXgHPW3354ON1YaMLnPASk7FQizSOQ -QZAsyxtdLwKBwGUosvTYYXvygXP4x1LkpmfKFJe94E1exXpAsmovmTvcSXn9tTXC -Sr77LWb0ZrPbYT7pHS7QEMg8MSnp941hIrG4mzs666KHkgLUdI4B0YtaIDsZMXlV -gY3j4KpYbhxH4/2U2eSfC2fxxnKVKW3n6vdQrfmo0q/eQ6BGOgiLK7fybCLHyBQL -8Zg2k3z5bNUEhMTdE0AW3WjBZ4IXmFcdK26616r/szJ7RcZilrydVXexqpmWlTVl -sTst9kucAPlwswKBwQCwf7my/GNezR8Jik+fZj7edBQQfcdra+8JnOvhfpLcKLte -2s1RjjA0q6usou1bYAsszP2bEzV97XWmgq7dFg4tUE7s/NO1d91zGDhBx2Gj1TkN -2A5dKonOuq9iDeITB6qYqcUvvyEfxRRZQr2jj+WzZCr/4BLCO6PJ29A9jKOuKLtF -QcfWRF2RiNMN6lffzkHFIR4p2YHxa2DEsGGtmbt8Ig3Jtl/HFmydzmxJRoev71dY -+ODdB6PhLhZmcRPoWpMCgcEAhGArwL68GwwRMqAX79gMv8tVT0CJnDyGk5mD/ZIB -Nzt0yQFO7rTEa1l1vAtOiVJ9IpAak2lgbEwodOfGnQst7lujNYDFzTRPTFt/lID1 -u6JBxmqawOSlqa00bt4l2YsTZV+BfSznBP6XO1PK4iR3o5G3NkoKJjZWm3e3asHk -6eTeMLcsIJ+Fp7gG0ve2EdQwhVSVMFEu4Q4C2FcJeU++L4kYpY7sTnAjUtiLvtHn -yp3jllEn3CBD8Uhs4B+sL/6p +MIIG/AIBADANBgkqhkiG9w0BAQEFAASCBuYwggbiAgEAAoIBgQDmuuLkwcIMHD5i +2LlcVy5SuIPFiDrmmnr1ZBYz6zduL3vzaANFZUddcRBZyisbAGyBFGH0hlk/6v14 +NxadQ/HE9mmMxSkGiJ4mIgSsBNiHNEg562vyC5Kqw25jUVFrwq3/XMgvshucIIpA +PqIvaurI2TdDXNztkuLZQNJhn3GK9e05uqhePrUhYxDYb7TiEQELEOi7+2LvSFW8 +9dKcq2iulSUZ8pd9Gtxm6ohehuTLy2lNXrCj+2wx5ChgXpDx1C4QUOGF8A1cvd1F +JAgZPhyTZo8r2lN9BBwOQsRoXqbNqRjtp81q0NGGupD/t0zex0Mkbccca5yB5+Eb +VyWQqQ7JVvP2a14ttC5AUJtCY9LWmRw43M8rLKdy8cdeYzR2SPQ+iBOehhZTL3T7 +hwGNIqRoM+4TbHoGFFRWF1dXmDTQC2YJ44gJ+KUVHBBz0IhQmV4YZTv/MScbXsaq +Qf0tLxinwPKrxyK1C2nSc9G70Bw9+qQ1Ys0zhsegIw+5atXSbY0CAwEAAQKCAYBf +4VWcPjBHHA2Iwgr1Jn1nfqmzklL3tUZXZwoa9SoJrc3Sbmy9j8LCP9PNnEehZuGw +GipClPnNp/dA15OcMrnrYYKnLt9Hico+inBqk3Dvbnh9KSmoYcrHD4N13jr5juMD +dSjzOQ5kKNmKrPx0u/dpE2r1oUdlql5+bYN/ceSbHGtCTCDfWSun/iTn7DO8pdhL +IvGz/Fk2mlaWuYiV9lz//5Z1W+w73sesNNYKgf/d+F9/+VNqMXbanLdypJmTBNp9 +eoS3eLk3ycoiGHCnVNm28IjeElFkUOJKVXY39BoMbS/x1MUHjuZDxdOcEaCV9iOr +adg7d91srwJlHGGX3IkC+08J5OJZtfTEMcB3i9g2omOxLAFfapeRxZdD6zMdxWUG +PUNIiy7UKaHe7JJ+fftnk0QBu5DzyrdP0fryFNz9nySot+gzVlU9idTMnIyE73UJ +foPowKpmxDOYNw0ZGXBIP/lMRNXkQM765P1NJpCN0Ud+kBl7GdFMkI++GFjiH3kC +gcEA+VIsOliXn5xfU2zXZb2q/bPPlCBDw/EzIelRTFVqUC2kL3oh4NlDNWAFgcfX +rQUCCoY9rEKYdTQI0w47K4AkJYajnDG2zHj6fkaPcLweyEDa9Hw2ezDDiwz2uwH2 +4HW+JBZ0I7rRuSRXTSJrzFYYVU6tsr54Kl85l2bmL8V2XVYj4ko9uddsyH0F1Ejt +BYPBghr01uEHsO3nIcNMl10fS9eXdhMTr/haXdDxpFXjR5AO/ouQEyGrXk6LPJpn +rUE7AoHBAOzpN8lQDQlf0JcYFBvFd7JdrhpbaIdATmT0NM/Mh+4B8qy5qnIJBdeQ +9T46sHmHzvhtGHmnP+dckKJi1Y/8Lt6noWNYDRnmrDol046ce2Nj8gxDGBT1kG4n +GBoKkA1Gj5N5yWMB8cGTVzQ+tViRMkn9+jmxRaNsW1Y0j7lMtVz0hJOUscbUALV1 +fYGH7zPiqfHoKoz7UuB2OlCKOH0+V96lAO4EgKMQNcEKMs9+Tg13fHTAyodxy/CY +tjUcKpef1wKBwBE24EDjDw0BMf/Dmxe2QdEkkieLFsK3q60iu+9GUoHYtOZmS2KH +/cD4sUilsLmMh/iMDkQPkRE+l4FjESjOvzAsHK3TLOjvTXRckNja1FFFURjiXqyg +0E+QhJSi7RXQa2F4f2pcItDitnhn8QN5ylJRjWKzDf729jYC78/KlYKaSP393Ecx +nZw2LanboynnT/wYumD/xpUrx/Kn1mj5EAkfiKCpbomO30Zs/9I17+xoAPEIV9lK +UNfBGpIDozbuMwKBwFlZmAWf4FrRvSzPEv5qWjt2I2yjXufrs+VVSPm6LOXx7CGC +oKsDhiWH8UZ4AgjD1KZTFvECyBItEgt8dQkp1k95L1/1XHORURFZJNHbaJnSnv5K +67Ez8DXrHqbrpuqq2wmG3BIwMIqOVExK/kAZ+rp3REEv/5CkFEqN5kq/iIM3YSz7 +3pSbbm0Bk8UfjHKoIOowYqPrQZWQYWvwxV9O/PrmhlQ+dHmLaoqUmxcwjqV7k//A +mmG85GqoXcfoCJRI3wKBwBdnxBzg17TMFufuvX2Bc/M9MqL3+vlwH6SDdr+2yYKA +hiD8Ur2OwtDGHnV4m3NeA/Guyz32H4CzvFAnpzlvMow/dvfp9JUcpdeidhIBZy8V +D7VODIiCyyTAb3g1LK0+HTEHAVRFihbNXhub/P6NckFXw0MJJQOvNNsYQnJTUngY +oxqdt1HeAujEwBrRSfrOGE2K8FVJ/MYf4PmxTIocICBk1/BmNHsUeJ5yFUDBweh5 +UJN6yp5PiGwvW8WFl4waXw== -----END PRIVATE KEY----- diff --git a/Lib/test/certdata/revocation.crl b/Lib/test/certdata/revocation.crl index 621675eb5c183d..a10586d1f79345 100644 --- a/Lib/test/certdata/revocation.crl +++ b/Lib/test/certdata/revocation.crl @@ -1,14 +1,14 @@ -----BEGIN X509 CRL----- -MIICJjCBjwIBATANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQGEwJYWTEmMCQGA1UE +MIICKDCBkQIBATANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQGEwJYWTEmMCQGA1UE CgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91ci1j -YS1zZXJ2ZXIXDTIxMDMxNzA4NDgyMFoXDTQwMDUxNjA4NDgyMFqgDjAMMAoGA1Ud -FAQDAgEAMA0GCSqGSIb3DQEBCwUAA4IBgQCd2GrHb4zr2R8eK7YMHwlkgICxbWP1 -4nuEi55yzUcmMcCZJ6ZQV3yYqTlAULGQ9qWAUdhsyH+yu3hRKFKHQv0DAdKKxgow -66YasAQQ99DskXOPxmRoIA7qtIWZbLtBwHQJWh+uUFlTdUXitGIX5Xie74xu5YIr -moa3QeuZyG5+gigSTUyst5T/J/cHfBzlAJLc2k3Ty4EPYXKHCVnrZWJbRmxq199l -A7S+eBb9qWXSYXCn6v+EZ76pUS3u/66kZ86PO3h9294BzdhxbCJ27dQXNHw6owe2 -Iyiv0aWx+TNSGSf4yCqaYTH6RtEoviI3h/inVFHNGgjlMzdaGw/0I3bkB0rt2WSR -Vck37HnXvQvVEkgO/39C0WKZus6m4gmOgZcbJbXaR8uIR5Hmw3SEyGEPEIBu6tXV -BLJOSOSu2vVUH5GUIrpvK9FTySKYa+MGryoPasuqZNfwpaXK+ON2G6QsmcXPWZY0 -Dry6t0w2geW6UYVGmb831i8ZP3JVVVwcwi0= +YS1zZXJ2ZXIXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWqAOMAwwCgYD +VR0UBAMCAQAwDQYJKoZIhvcNAQELBQADggGBAKN4S2g4wCeU1fO5TSckAwxgdzdh +pY28f4musnQt7l37MzB2gmJVDSCZfQyrUnfSEST15WEY7CZVyTlbsu6gYKK53Yej +j3ORBfGUgzaz62Hs8di7SrHDzWUlNCFa47YFWDmtj96KTX1AItnpkCCE58Wfpivp +Hu+YINFpi/2vI2nvP/xcfvgT3dXek9kyz+2jHmadxcn2VerSBZZ9fiZk/k4NzgoI +JdiSswtN1c5GelHQfftwRXbWqsp6TvgHC5MagDuHh5Bj7/DftI7nCy0IT5GnP8lS +ZqmXUMpa8zbtSNSTIk0XepmypNW8HHMQbfJp0y7yOQ4pPyXICrjYTg7wKpODRcm3 +BRN89vvNfCszMU41glVfQG+2Po5uAMTl1hX8WYSj0+Xxrdg+wgJead4S5Sq3CgMT +bKsH2Dqh43L8BTxuxzLQyduK0gKSl8vlN7a9Bzm3IXYlyk+kKSyo4jP8XK79pj1k +1JglMFM9jpoMF2VmNjiROtVEl2tbDGwlvpjWYQ== -----END X509 CRL----- diff --git a/Lib/test/certdata/ssl_cert.pem b/Lib/test/certdata/ssl_cert.pem index de596717bd855f..6db52404942210 100644 --- a/Lib/test/certdata/ssl_cert.pem +++ b/Lib/test/certdata/ssl_cert.pem @@ -1,26 +1,27 @@ -----BEGIN CERTIFICATE----- -MIIEWTCCAsGgAwIBAgIJAJinz4jHSjLtMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xODA4 -MjkxNDIzMTVaFw0yODA4MjYxNDIzMTVaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH -DA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5k -YXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDCCAaIwDQYJKoZIhvcNAQEBBQADggGP -ADCCAYoCggGBALKUqUtopT6E68kN+uJNEt34i2EbmG/bwjcD8IaMsgJPSsMO2Bpd -3S6qWgkCeOyCfmAwBxK2kNbxGb63ouysEv7l8GCTJTWv3hG/HQcejJpnAEGi6K1U -fDbyE/db6yZ12SoHVTGkadN4vYGCPd1Wj9ZO1F877SHQ8rDWX3xgTWkxN2ojBw44 -T8RHSDiG8D/CvG4uEy+VUszL+Uvny5y2poNSqvI3J56sptWSrh8nIIbkPZPBdUne -LYMOHTFK3ZjXSmhlXgziTxK71nnzM3Y9K9gxPnRqoXbvu/wFo55hQCkETiRkYgmm -jXcBMZ0TClQVnQWuLjMthRnWFZs4Lfmwqjs7FZD/61581R2BYehvpWbLvvuOJhwv -DFzexL2sXcAl7SsxbzeQKRHqGbIDfbnQTXfs3/VC6Ye5P82P2ucj+XC32N9piRmO -gCBP8L3ub+YzzdxikZN2gZXXE2jsb3QyE/R2LkWdWyshpKe+RsZP1SBRbHShUyOh -yJ90baoiEwj2mwIDAQABoxgwFjAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJKoZI -hvcNAQELBQADggGBAHRUO/UIHl3jXQENewYayHxkIx8t7nu40iO2DXbicSijz5bo -5//xAB6RxhBAlsDBehgQP1uoZg+WJW+nHu3CIVOU3qZNZRaozxiCl2UFKcNqLOmx -R3NKpo1jYf4REQIeG8Yw9+hSWLRbshNteP6bKUUf+vanhg9+axyOEOH/iOQvgk/m -b8wA8wNa4ujWljPbTQnj7ry8RqhTM0GcAN5LSdSvcKcpzLcs3aYwh+Z8e30sQWna -F40sa5u7izgBTOrwpcDm/w5kC46vpRQ5fnbshVw6pne2by0mdMECASid/p25N103 -jMqTFlmO7kpf/jpCSmamp3/JSEE1BJKHwQ6Ql4nzRA2N1mnvWH7Zxcv043gkHeAu -0x8evpvwuhdIyproejNFlBpKmW8OX7yKTCPPMC/VkX8Q1rVkxU0DQ6hmvwZlhoKa -9Wc2uXpw9xF8itV4Uvcdr3dwqByvIqn7iI/gB+4l41e0u8OmH2MKOx4Nxlly5TNW -HcVKQHyOeyvnINuBAQ== +MIIEhTCCAu2gAwIBAgIUTxhwRX6zBQc3+l3imDklEbqcDpIwDQYJKoZIhvcNAQEL +BQAwXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD +VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxo +b3N0MCAXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWjBfMQswCQYDVQQG +EwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDDAlsb2NhbGhvc3QwggGiMA0GCSqG +SIb3DQEBAQUAA4IBjwAwggGKAoIBgQDwcIAYm12nmQTGB3caFn7alDe3LSliEfNC +2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3zmrI/FAiQI/RrUHyBZiEt +nFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVwg9cR0khTqD5cg2jvTB05 +yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYliGx5qqCQmsXeTsxCpvQD7 +u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAaF48xEnokxenzBIqZ82BR +kFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLYsFXUp8L7ZMVE27Bej+Sq +4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11tgtctsYbwgDSUYGA3w+DC +KD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3RPJKV+Db8E1V2mXmNE0M +Lg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAaM3MDUwFAYDVR0RBA0wC4IJbG9j +YWxob3N0MB0GA1UdDgQWBBR459BlAel5MqCtG0DrvVtMZlQfuTANBgkqhkiG9w0B +AQsFAAOCAYEAaTFWjK/LFzOo+0TWqTTj4WC4N3I8JFrHnlqFJlpchYTW2z92SU1G +iEzFjWzuDNjp5KM9BqlmGtzXZvy6MItGkYsjPRdPVU0rbCmyTho6y77kTyiEG12V +UAJ1in3FOQfDwLPcp7wQRgCQq3iZlv7pwXp2Lm5fzu8kZPnxmTVdiKQun9Ps7uKq +BoM0fM2K14MxVO4Wc0SERnaPszE7xAhkIcs+NRT/gHYdTBlPhao83S3LOOdtCqCP +pNUOEaShlwI5bVsDPUXNX/eS0MYFNlsYTb5rCxK8jf3W3KNjKTOzN94pHiQOhpkg +xMPPi3m03/9oKTVXBtHI2A+u3ukheKE6sBXCLdv/GEs9zYI49zmpQxNWz5EOumWL +k+W/vPv7cD6LeHxxp+nCbEJi1gZtYb3gMY1sLkMNxcOu0QHTqHfme+k7VKWm8anO +3ogGdGtPuPAD/qjMwg3ChSDLl5Ur/E9UPlD4yM/7KtUD7mLv+jbddA62EiA9MxVB +t+yt7pOwOA66 -----END CERTIFICATE----- diff --git a/Lib/test/certdata/ssl_key.passwd.pem b/Lib/test/certdata/ssl_key.passwd.pem index 46de61ab85b136..e60a66c2798784 100644 --- a/Lib/test/certdata/ssl_key.passwd.pem +++ b/Lib/test/certdata/ssl_key.passwd.pem @@ -1,42 +1,42 @@ -----BEGIN ENCRYPTED PRIVATE KEY----- -MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQI072N7W+PDDMCAggA -MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBA/AuaRNi4vE4KGqI4In+70BIIH -ENGS5Vex5NID873frmd1UZEHZ+O/Bd0wDb+NUpIqesHkRYf7kKi6Gnr+nKQ/oVVn -Lm3JjE7c8ECP0OkOOXmiXuWL1SkzBBWqCI4stSGUPvBiHsGwNnvJAaGjUffgMlcC -aJOA2+dnejLkzblq4CB2LQdm06N3Xoe9tyqtQaUHxfzJAf5Ydd8uj7vpKN2MMhY7 -icIPJwSyh0N7S6XWVtHEokr9Kp4y2hS5a+BgCWV1/1z0aF7agnSVndmT1VR+nWmc -lM14k+lethmHMB+fsNSjnqeJ7XOPlOTHqhiZ9bBSTgF/xr5Bck/NiKRzHjdovBox -TKg+xchaBhpRh7wBPBIlNJeHmIjv+8obOKjKU98Ig/7R9+IryZaNcKAH0PuOT+Sw -QHXiCGQbOiYHB9UyhDTWiB7YVjd8KHefOFxfHzOQb/iBhbv1x3bTl3DgepvRN6VO -dIsPLoIZe42sdf9GeMsk8mGJyZUQ6AzsfhWk3grb/XscizPSvrNsJ2VL1R7YTyT3 -3WA4ZXR1EqvXnWL7N/raemQjy62iOG6t7fcF5IdP9CMbWP+Plpsz4cQW7FtesCTq -a5ZXraochQz361ODFNIeBEGU+0qqXUtZDlmos/EySkZykSeU/L0bImS62VGE3afo -YXBmznTTT9kkFkqv7H0MerfJsrE/wF8puP3GM01DW2JRgXRpSWlvbPV/2LnMtRuD -II7iH4rWDtTjCN6BWKAgDOnPkc9sZ4XulqT32lcUeV6LTdMBfq8kMEc8eDij1vUT -maVCRpuwaq8EIT3lVgNLufHiG96ojlyYtj3orzw22IjkgC/9ee8UDik9CqbMVmFf -fVHhsw8LNSg8Q4bmwm5Eg2w2it2gtI68+mwr75oCxuJ/8OMjW21Prj8XDh5reie2 -c0lDKQOFZ9UnLU1bXR/6qUM+JFKR4DMq+fOCuoQSVoyVUEOsJpvBOYnYZN9cxsZm -vh9dKafMEcKZ8flsbr+gOmOw7+Py2ifSlf25E/Frb1W4gtbTb0LQVHb6+drutrZj -8HEu4CnHYFCD4ZnOJb26XlZCb8GFBddW86yJYyUqMMV6Q1aJfAOAglsTo1LjIMOZ -byo0BTAmwUevU/iuOXQ4qRBXXcoidDcTCrxfUSPG9wdt9l+m5SdQpWqfQ+fx5O7m -SLlrHyZCiPSFMtC9DxqjIklHjf5W3wslGLgaD30YXa4VDYkRihf3CNsxGQ+tVvef -l0ZjoAitF7Gaua06IESmKnpHe23dkr1cjYq+u2IV+xGH8LeExdwsQ9kpuTeXPnQs -JOA99SsFx1ct32RrwjxnDDsiNkaViTKo9GDkV3jQTfoFgAVqfSgg9wGXpqUqhNG7 -TiSIHCowllLny2zn4XrXCy2niD3VDt0skb3l/PaegHE2z7S5YY85nQtYwpLiwB9M -SQ08DYKxPBZYKtS2iZ/fsA1gjSRQDPg/SIxMhUC3M3qH8iWny1Lzl25F2Uq7VVEX -LdTUtaby49jRTT3CQGr5n6z7bMbUegiY7h8WmOekuThGDH+4xZp6+rDP4GFk4FeK -JcF70vMQYIjQZhadic6olv+9VtUP42ltGG/yP9a3eWRkzfAf2eCh6B1rYdgEWwE8 -rlcZzwM+y6eUmeNF2FVWB8iWtTMQHy+dYNPM+Jtus1KQKxiiq/yCRs7nWvzWRFWA -HRyqV0J6/lqgm4FvfktFt1T0W+mDoLJOR2/zIwMy2lgL5zeHuR3SaMJnCikJbqKS -HB3UvrhAWUcZqdH29+FhVWeM7ybyF1Wccmf+IIC/ePLa6gjtqPV8lG/5kbpcpnB6 -UQY8WWaKMxyr3jJ9bAX5QKshchp04cDecOLZrpFGNNQngR8RxSEkiIgAqNxWunIu -KrdBDrupv/XAgEOclmgToY3iywLJSV5gHAyHWDUhRH4cFCLiGPl4XIcnXOuTze3H -3j+EYSiS3v3DhHjp33YU2pXlJDjiYsKzAXejEh66++Y8qaQdCAad3ruWRCzW3kgk -Md0A1VGzntTnQsewvExQEMZH2LtYIsPv3KCYGeSAuLabX4tbGk79PswjnjLLEOr0 -Ghf6RF6qf5/iFyJoG4vrbKT8kx6ywh0InILCdjUunuDskIBxX6tEcr9XwajoIvb2 -kcmGdjam5kKLS7QOWQTl8/r/cuFes0dj34cX5Qpq+Gd7tRq/D+b0207926Cxvftv -qQ1cVn8HiLxKkZzd3tpf2xnoV1zkTL0oHrNg+qzxoxXUTUcwtIf1d/HRbYEAhi/d -bBBoFeftEHWNq+sJgS9bH+XNzo/yK4u04B5miOq8v4CSkJdzu+ZdF22d4cjiGmtQ -8BTmcn0Unzm+u5H0+QSZe54QBHJGNXXOIKMTkgnOdW27g4DbI1y7fCqJiSMbRW6L -oHmMfbdB3GWqGbsUkhY8i6h9op0MU6WOX7ea2Rxyt4t6 +MIIHdTBfBgkqhkiG9w0BBQ0wUjAxBgkqhkiG9w0BBQwwJAQQMMUN+qWFiwTbIjjb +cLEtYQICCAAwDAYIKoZIhvcNAgkFADAdBglghkgBZQMEASoEEGGzvsc6e5Zn2M9S +sdoMbaIEggcQ0mQYuMcujHtFfRKLmfclrf/dHOYgnVYc2c9MSR3jfoOwnTdBR1Op +MUuO4ukqrxGkrJBy+MLb1oPEGrnvn39pdBMZx4CBmdwX0jDq36UL9EU5/m071mT+ +Xoe/z/RHtrtmbpA9XKu7n/5gLi9jg57qorss8r5MaPHghm2ZOqaQZIOOMCF+4gNU +Cmp/NHL9TUM/sfQpaw9YM4VuTRyHNO3OiDKaCCz2romFrgM1TYz/I5/n5MMR543J +dsyIFzef3jHYfaXTl97v9ibGVpIUCKfHgYlQf41H+Me890AP4HaS0e2y+73/UwjB +4YSzPVm+lCiBWhFDbMuRmPY3YDQOA+TV06oCO8/aMamPRsryl8g7iOQAIz4hQdoP +ZTXtSZ+F68YmfPjKAj3NNQosvLzfuSfxZ9HqyvZ/0w1eead7pdSs7uvCEvN2Zcvk +a9EWy1bM895zD3DwrxqGnSYbitXouIOf9zHsl2lRUK44XmvgTHoNaJFIVyzVNgQr +e1YJM7LbEvErspa6cZTz7DCR44g9cYBi393XEt5yl4NCtq2PFq762pNqpsQqt5/Y +pLX5w12npf+OZ1WqJla89FabqLaJ1blmDEgfko0XriofW5gWLByvhYINtt+u5x/5 +QdTya2gWBxI2K+sAeGIt913RBAus8hLEDnVGDX/FEthz1OOKLXp0G6HQmqUom5PU +/O36GszhWIz1Q/SpMfZkdHfEfbJFCx0mpaJZTlC01hg+OpnwWxUvO2dxrK46xAzd +/obsVQRLOuyTxjIrQd+vVZRIcOroTHVUbh7qV2NV1sdQbsKnc4mYK8jjHsb/v/O3 +lcOFJX2V55z7FdEOi7qKKn3l5j6SNmdMGlGde16692VCdYo4Wjs2ufkYryJoJQ7L +O3AeoXVrpc2JwKj6GFKN3Bw6A3XEXTShWdHS8DAuHdbLBhHESeb/qA76qNInw/od +PXtsRNBWaS87rh7GGWrO2ULHBk0k5hLa9puXgAZVh5NcrVuujWM2spGPHGJWhvDt +ffOJAe0Kra8b1CHOC9aRUuHnapuu/mAD+XZTAt/UUNq0hdTDO1VTxL4lHpZjmWJ3 +OvmTp1HpA8FpSmzSBw6H0beVT9LXDbkSax86E2YQJPcC9IjQARoaL+7ba1XdXeSQ +k796zrq4YD1SB09qRVVar8nBSHUYAzG2gbAWgAQgOjwtC5nT1Que3kI7weG2nFZ4 +B1MfBmpEqPwMs8O5hurdYmfpBRoudivKrK5si+LB7og1IwVD2IS8mm55/Kmnugf4 +YCEiGu5s9F+njqEZBV7UPAabaFph6Iw3Jfmcg0YUZV60hY6O3EieonfwVBBKWBQp +K0SUs24Ld80B3oYWuqYI+MQlSprskRKFBfx7hp2PsQU9jAVPc8NohmfM5IJAFBwo +rCqAOsRsrRZnVoaRP89X50VuBg6ROiu/cmI5sqou3u30Pndzmlbcg8ekWj+1qrur +prva6uPc0oiCgFTMksFnhKvlHErIm+ceGvNSib+rWNomaL5cjpgnqAG6lZMP7N+p +QvH+ABchP+fJyyl/XuaOLmcXDSkIcadfjJSHACwmGRas3unfOYEyBg29oPu6PNlW +jXFJb7OzWaQfFNcCnHzvYGTvmrbg7VqBVqmVRYX3r6WNDJeaSwDJpxYNNQlTugbm +FOlD9JTeHyZ17rrqiCitbQwBVBpWOhBEIkFD+3JL1ZdGjyc3rcvbERLr/UN649vS +FcMOLiEvFjFdirq6Pe9fx/VD1GrIzXaEhvyCePSeoV6eILm7SSFTVCm+JMtiBlBi +ZjDxlUhQGuMg3IwZRqjfUR90wo3QWe5XOgM9mJa5qY/Yaa9YpFEJcd4mYnXcPLIz +eY8lqGAhROwJhGRhQoTQXucsSEhAkUBpRiE7UCN8OjezvAeFn0+6XyHeST+QZenX +ixAKJ27lFn/njvL4sDocd7ZvXpb3P5ZxCRakMnjunQQyjtUlJmPrNjs+ZlPenLuh +UA3Oj5d96dDqzgZNLxbDHKr6B+CMApBrwUDcum09PYgJ5xZ/Hrct4iSa57Gi528L +l0dcVgPHd80oIn/vyhjVYkkXNMrRhTJIgLuh0KsddZ7/8Xvxma9W8hNQ1sYwE0Yp +RqLgMRdFpTSN8hKUTfRvjzbrW4nQ2XVqAdyM6PEkdrPCBZczdGisp9oMF6woI/pA +ZKNxdUr0DG3tVBJ7z6qfdp2j+yHvhqQt+ohmk3YldPKpXfYz20ZJ52URE8vbCj2K +NZ/MVEACbg6FDgNQ1bIKD61pKL72+SjyKW1wQfIkeqMfOBI8BGclp7BL6dGRrVHN +PLg2j9gsgZ3XsiJOFtJ6Q3UABkeUrbRvAFQXPM8+keWU4VP/99dCJTvnEQTM6O8U +gz2NA4j1ZEOmB5L+hmf2gW+xgW6eeMjylj5JdeAliNaUqIhOju30vo4= -----END ENCRYPTED PRIVATE KEY----- diff --git a/Lib/test/certdata/ssl_key.pem b/Lib/test/certdata/ssl_key.pem index 1ea4578d81ecc4..1ad9e5e2b832cf 100644 --- a/Lib/test/certdata/ssl_key.pem +++ b/Lib/test/certdata/ssl_key.pem @@ -1,40 +1,40 @@ -----BEGIN PRIVATE KEY----- -MIIG/wIBADANBgkqhkiG9w0BAQEFAASCBukwggblAgEAAoIBgQCylKlLaKU+hOvJ -DfriTRLd+IthG5hv28I3A/CGjLICT0rDDtgaXd0uqloJAnjsgn5gMAcStpDW8Rm+ -t6LsrBL+5fBgkyU1r94Rvx0HHoyaZwBBouitVHw28hP3W+smddkqB1UxpGnTeL2B -gj3dVo/WTtRfO+0h0PKw1l98YE1pMTdqIwcOOE/ER0g4hvA/wrxuLhMvlVLMy/lL -58uctqaDUqryNyeerKbVkq4fJyCG5D2TwXVJ3i2DDh0xSt2Y10poZV4M4k8Su9Z5 -8zN2PSvYMT50aqF277v8BaOeYUApBE4kZGIJpo13ATGdEwpUFZ0Fri4zLYUZ1hWb -OC35sKo7OxWQ/+tefNUdgWHob6Vmy777jiYcLwxc3sS9rF3AJe0rMW83kCkR6hmy -A3250E137N/1QumHuT/Nj9rnI/lwt9jfaYkZjoAgT/C97m/mM83cYpGTdoGV1xNo -7G90MhP0di5FnVsrIaSnvkbGT9UgUWx0oVMjocifdG2qIhMI9psCAwEAAQKCAYBT -sHmaPmNaZj59jZCqp0YVQlpHWwBYQ5vD3pPE6oCttm0p9nXt/VkfenQRTthOtmT1 -POzDp00/feP7zeGLmqSYUjgRekPw4gdnN7Ip2PY5kdW77NWwDSzdLxuOS8Rq1MW9 -/Yu+ZPe3RBlDbT8C0IM+Atlh/BqIQ3zIxN4g0pzUlF0M33d6AYfYSzOcUhibOO7H -j84r+YXBNkIRgYKZYbutRXuZYaGuqejRpBj3voVu0d3Ntdb6lCWuClpB9HzfGN0c -RTv8g6UYO4sK3qyFn90ibIR/1GB9watvtoWVZqggiWeBzSWVWRsGEf9O+Cx4oJw1 -IphglhmhbgNksbj7bD24on/icldSOiVkoUemUOFmHWhCm4PnB1GmbD8YMfEdSbks -qDr1Ps1zg4mGOinVD/4cY7vuPFO/HCH07wfeaUGzRt4g0/yLr+XjVofOA3oowyxv -JAzr+niHA3lg5ecj4r7M68efwzN1OCyjMrVJw2RAzwvGxE+rm5NiT08SWlKQZnkC -gcEA4wvyLpIur/UB84nV3XVJ89UMNBLm++aTFzld047BLJtMaOhvNqx6Cl5c8VuW -l261KHjiVzpfNM3/A2LBQJcYkhX7avkqEXlj57cl+dCWAVwUzKmLJTPjfaTTZnYJ -xeN3dMYjJz2z2WtgvfvDoJLukVwIMmhTY8wtqqYyQBJ/l06pBsfw5TNvmVIOQHds -8ASOiFt+WRLk2bl9xrGGayqt3VV93KVRzF27cpjOgEcG74F3c0ZW9snERN7vIYwB -JfrlAoHBAMlahPwMP2TYylG8OzHe7EiehTekSO26LGh0Cq3wTGXYsK/q8hQCzL14 -kWW638vpwXL6L9ntvrd7hjzWRO3vX/VxnYEA6f0bpqHq1tZi6lzix5CTUN5McpDg -QnjenSJNrNjS1zEF8WeY9iLEuDI/M/iUW4y9R6s3WpgQhPDXpSvd2g3gMGRUYhxQ -Xna8auiJeYFq0oNaOxvJj+VeOfJ3ZMJttd+Y7gTOYZcbg3SdRb/kdxYki0RMD2hF -4ZvjJ6CTfwKBwQDiMqiZFTJGQwYqp4vWEmAW+I4r4xkUpWatoI2Fk5eI5T9+1PLX -uYXsho56NxEU1UrOg4Cb/p+TcBc8PErkGqR0BkpxDMOInTOXSrQe6lxIBoECVXc3 -HTbrmiay0a5y5GfCgxPKqIJhfcToAceoVjovv0y7S4yoxGZKuUEe7E8JY2iqRNAO -yOvKCCICv/hcN235E44RF+2/rDlOltagNej5tY6rIFkaDdgOF4bD7f9O5eEni1Bg -litfoesDtQP/3rECgcEAkQfvQ7D6tIPmbqsbJBfCr6fmoqZllT4FIJN84b50+OL0 -mTGsfjdqC4tdhx3sdu7/VPbaIqm5NmX10bowWgWSY7MbVME4yQPyqSwC5NbIonEC -d6N0mzoLR0kQ+Ai4u+2g82gicgAq2oj1uSNi3WZi48jQjHYFulCbo246o1NgeFFK -77WshYe2R1ioQfQDOU1URKCR0uTaMHClgfu112yiGd12JAD+aF3TM0kxDXz+sXI5 -SKy311DFxECZeXRLpcC3AoHBAJkNMJWTyPYbeVu+CTQkec8Uun233EkXa2kUNZc/ -5DuXDaK+A3DMgYRufTKSPpDHGaCZ1SYPInX1Uoe2dgVjWssRL2uitR4ENabDoAOA -ICVYXYYNagqQu5wwirF0QeaMXo1fjhuuHQh8GsMdXZvYEaAITZ9/NG5x/oY08+8H -kr78SMBOPy3XQn964uKG+e3JwpOG14GKABdAlrHKFXNWchu/6dgcYXB87mrC/GhO -zNwzC+QhFTZoOomFoqMgFWujng== +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDwcIAYm12nmQTG +B3caFn7alDe3LSliEfNC2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3z +mrI/FAiQI/RrUHyBZiEtnFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVw +g9cR0khTqD5cg2jvTB05yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYli +Gx5qqCQmsXeTsxCpvQD7u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAa +F48xEnokxenzBIqZ82BRkFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLY +sFXUp8L7ZMVE27Bej+Sq4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11t +gtctsYbwgDSUYGA3w+DCKD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3 +RPJKV+Db8E1V2mXmNE0MLg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAQKCAYAK +Ap0KqTlCd4fv2LK4NtSMNByHt00gRKAMmfNstJ12UKoxBLFjXOXaHjWv5PYkh4bz +vjo7pBHMCWnDuR6Pqr1ahuyvpRex6XcbJ4VebsaOKYO6+gphlm2C2ZqCQ1Vh6Akd +aZ40Wb1gfgK/zvVezBLvzLLf9iahw9j5pWZ2iDci5zdUuvd9Sn+qUB3+nyRf/NW5 +MXgBsp07zIcOOxPOm/Z5V+0jDJL2hiRq1pbmUKClTShcgqtfJKU//niF+6ZQAuiJ +LBPaKIdPXyxLYnkyq2IgjPU0ZwxzdP0a2k72kvImd25Daj7elhGr3++IR+nFzt6h +vqflOfmKDF3zZPyUVI3YXjxo/FrOwGbLMHuuHBgE9txH/mOl1gByrxP+Ax18i3Bf +spSLeUvtaf/w/MopyspPoJBRbAM06PUHQI2v9xq3BZL/gHe2CdJPds2WzpaaVFG4 +oJWNrE3s6CowLqUkqzB7LqJ4ReZ6xe6SpkRotdmVknlIKgDenTFeEUEEVyBiFQEC +gcEA/F1GAaBG0e9vB+AOHZ96SLlZVzObSBYq2kVwUhhGItNnyU9c3fWPIrGREKQa +lw5dsvjl58ij5uEtJoPZf5BsJ0q6xHAs/kKxfpNfZAeoKAV96Z6MVcY+6WOyGjPF +aQo+GgSrCPIciW//WXZrWI1t0M2G0vZ5CFNohnKod+cSgV03PAActlyM2H+r7dtm +MpAD3EPWeeA75saKj/x0SOzuL/wzXKR8BZ6CINZ6r61Tcbk2mDwOHPhUrHeCwjoU +nhy5AoHBAPPnP2FSXFCPXD1Z1hFInCFgm41j7LEyBTVLlnqUrRk7i18fi/WcwwLH ++XvM5DcONY/V3sh7a3tZeHN1P70tRxLE0oO51D4tP5im/oZ6L+hszSYXX7lCbJSR +tni6nU1dssW3nmswfUn01Oh+B0rBGon3RQB6x4beTAW0piVxg9Ic2HYucS1Scrqw +afiFQ5KWklnMYJKInPFzlCwMdgBCuue1dZoJstU9nLQALNNSpGXB2X0+7j9D/qkz +Caw5MfgQwQKBwQDzdCvP78XCSuBq0XvsmefG9n+4fwGDFld6v9gualpmyFjsPJKT +UYwm5PPUAOvh46sCt9hatRVg6sO6zyFoTXP4p7/rN2hAVSiTuiog/r369elVEW3C +ZYBVeKbdXipIPehRA0XYWHCtKY1Fydae07kn4M37AGkcXhKM+VmKajFQ+RMK3/TS +/A+n3+qFiM1bY9FFkW/7nRVMeSY850dq/p59TihibA91AEf6084BYg0IvatsSys2 +SV6uDpDnPE6dhYkCgcBECtAwq1RbmRLnfqdsnPAJk7Txhd3jNQwk6RhqzA1aS7U+ +7UMTWw9AOF+OPQOxpEInBUgob931RGmI9D263eXFA6mi2/Ws/tyODpBVHcM9uRSm +OsEWosQ90kSwe4ckrS4RYH9OcfGR7z5yOa55GVP5B0V1s8r0AhH9SX9MVNWsiSWO +GriyJx0gndSCY1MNkvnzGUQbvQbjiRXeD//fZL5Vo9bSCUCdopmT0bSvo49/X8v3 +19WJSsPBmh5psG8TQEECgcEA64CqZpPux35LeLQsKe0fYeNfAncqiaIoRbAvxKCi +SQf27SD8HK+sfvhvYY7bP7TMEeM7B/O2/AqBQQP0UARIGJg2AknBQT0A7//yJu+o +v4FHy2XKh+RMAx7QrdvnQ4CfrjvjQIaAcN1HrdTKWwgQZZImRf57nUCMm82ktZ2k +vYEJTXMkT8CY0DSeGtPmX5ynk7cauHTdZrkPGhZ3Hr6GAFomOammnnytv2wc+5FA +Ap+d65UgF4KjGY4rtsS+jOHn -----END PRIVATE KEY----- diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 58ffc0ad4ab88b..0dfcc281985100 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -550,10 +550,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - if (PyBytes_Check(args[0]) && PyBytes_GET_SIZE(args[0]) == 1) { + if (PyBytes_Check(args[0])) { + if (PyBytes_GET_SIZE(args[0]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 1 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[0])); + goto exit; + } a = PyBytes_AS_STRING(args[0])[0]; } - else if (PyByteArray_Check(args[0]) && PyByteArray_GET_SIZE(args[0]) == 1) { + else if (PyByteArray_Check(args[0])) { + if (PyByteArray_GET_SIZE(args[0]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 1 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[0])); + goto exit; + } a = PyByteArray_AS_STRING(args[0])[0]; } else { @@ -563,10 +577,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + if (PyBytes_Check(args[1])) { + if (PyBytes_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 2 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[1])); + goto exit; + } b = PyBytes_AS_STRING(args[1])[0]; } - else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + else if (PyByteArray_Check(args[1])) { + if (PyByteArray_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 2 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[1])); + goto exit; + } b = PyByteArray_AS_STRING(args[1])[0]; } else { @@ -576,10 +604,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - if (PyBytes_Check(args[2]) && PyBytes_GET_SIZE(args[2]) == 1) { + if (PyBytes_Check(args[2])) { + if (PyBytes_GET_SIZE(args[2]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 3 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[2])); + goto exit; + } c = PyBytes_AS_STRING(args[2])[0]; } - else if (PyByteArray_Check(args[2]) && PyByteArray_GET_SIZE(args[2]) == 1) { + else if (PyByteArray_Check(args[2])) { + if (PyByteArray_GET_SIZE(args[2]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 3 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[2])); + goto exit; + } c = PyByteArray_AS_STRING(args[2])[0]; } else { @@ -589,10 +631,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 4) { goto skip_optional; } - if (PyBytes_Check(args[3]) && PyBytes_GET_SIZE(args[3]) == 1) { + if (PyBytes_Check(args[3])) { + if (PyBytes_GET_SIZE(args[3]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 4 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[3])); + goto exit; + } d = PyBytes_AS_STRING(args[3])[0]; } - else if (PyByteArray_Check(args[3]) && PyByteArray_GET_SIZE(args[3]) == 1) { + else if (PyByteArray_Check(args[3])) { + if (PyByteArray_GET_SIZE(args[3]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 4 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[3])); + goto exit; + } d = PyByteArray_AS_STRING(args[3])[0]; } else { @@ -602,10 +658,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 5) { goto skip_optional; } - if (PyBytes_Check(args[4]) && PyBytes_GET_SIZE(args[4]) == 1) { + if (PyBytes_Check(args[4])) { + if (PyBytes_GET_SIZE(args[4]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 5 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[4])); + goto exit; + } e = PyBytes_AS_STRING(args[4])[0]; } - else if (PyByteArray_Check(args[4]) && PyByteArray_GET_SIZE(args[4]) == 1) { + else if (PyByteArray_Check(args[4])) { + if (PyByteArray_GET_SIZE(args[4]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 5 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[4])); + goto exit; + } e = PyByteArray_AS_STRING(args[4])[0]; } else { @@ -615,10 +685,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 6) { goto skip_optional; } - if (PyBytes_Check(args[5]) && PyBytes_GET_SIZE(args[5]) == 1) { + if (PyBytes_Check(args[5])) { + if (PyBytes_GET_SIZE(args[5]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 6 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[5])); + goto exit; + } f = PyBytes_AS_STRING(args[5])[0]; } - else if (PyByteArray_Check(args[5]) && PyByteArray_GET_SIZE(args[5]) == 1) { + else if (PyByteArray_Check(args[5])) { + if (PyByteArray_GET_SIZE(args[5]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 6 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[5])); + goto exit; + } f = PyByteArray_AS_STRING(args[5])[0]; } else { @@ -628,10 +712,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 7) { goto skip_optional; } - if (PyBytes_Check(args[6]) && PyBytes_GET_SIZE(args[6]) == 1) { + if (PyBytes_Check(args[6])) { + if (PyBytes_GET_SIZE(args[6]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 7 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[6])); + goto exit; + } g = PyBytes_AS_STRING(args[6])[0]; } - else if (PyByteArray_Check(args[6]) && PyByteArray_GET_SIZE(args[6]) == 1) { + else if (PyByteArray_Check(args[6])) { + if (PyByteArray_GET_SIZE(args[6]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 7 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[6])); + goto exit; + } g = PyByteArray_AS_STRING(args[6])[0]; } else { @@ -641,10 +739,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 8) { goto skip_optional; } - if (PyBytes_Check(args[7]) && PyBytes_GET_SIZE(args[7]) == 1) { + if (PyBytes_Check(args[7])) { + if (PyBytes_GET_SIZE(args[7]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 8 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[7])); + goto exit; + } h = PyBytes_AS_STRING(args[7])[0]; } - else if (PyByteArray_Check(args[7]) && PyByteArray_GET_SIZE(args[7]) == 1) { + else if (PyByteArray_Check(args[7])) { + if (PyByteArray_GET_SIZE(args[7]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 8 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[7])); + goto exit; + } h = PyByteArray_AS_STRING(args[7])[0]; } else { @@ -654,10 +766,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 9) { goto skip_optional; } - if (PyBytes_Check(args[8]) && PyBytes_GET_SIZE(args[8]) == 1) { + if (PyBytes_Check(args[8])) { + if (PyBytes_GET_SIZE(args[8]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 9 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[8])); + goto exit; + } i = PyBytes_AS_STRING(args[8])[0]; } - else if (PyByteArray_Check(args[8]) && PyByteArray_GET_SIZE(args[8]) == 1) { + else if (PyByteArray_Check(args[8])) { + if (PyByteArray_GET_SIZE(args[8]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 9 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[8])); + goto exit; + } i = PyByteArray_AS_STRING(args[8])[0]; } else { @@ -667,10 +793,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 10) { goto skip_optional; } - if (PyBytes_Check(args[9]) && PyBytes_GET_SIZE(args[9]) == 1) { + if (PyBytes_Check(args[9])) { + if (PyBytes_GET_SIZE(args[9]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 10 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[9])); + goto exit; + } j = PyBytes_AS_STRING(args[9])[0]; } - else if (PyByteArray_Check(args[9]) && PyByteArray_GET_SIZE(args[9]) == 1) { + else if (PyByteArray_Check(args[9])) { + if (PyByteArray_GET_SIZE(args[9]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 10 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[9])); + goto exit; + } j = PyByteArray_AS_STRING(args[9])[0]; } else { @@ -680,10 +820,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 11) { goto skip_optional; } - if (PyBytes_Check(args[10]) && PyBytes_GET_SIZE(args[10]) == 1) { + if (PyBytes_Check(args[10])) { + if (PyBytes_GET_SIZE(args[10]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 11 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[10])); + goto exit; + } k = PyBytes_AS_STRING(args[10])[0]; } - else if (PyByteArray_Check(args[10]) && PyByteArray_GET_SIZE(args[10]) == 1) { + else if (PyByteArray_Check(args[10])) { + if (PyByteArray_GET_SIZE(args[10]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 11 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[10])); + goto exit; + } k = PyByteArray_AS_STRING(args[10])[0]; } else { @@ -693,10 +847,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 12) { goto skip_optional; } - if (PyBytes_Check(args[11]) && PyBytes_GET_SIZE(args[11]) == 1) { + if (PyBytes_Check(args[11])) { + if (PyBytes_GET_SIZE(args[11]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 12 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[11])); + goto exit; + } l = PyBytes_AS_STRING(args[11])[0]; } - else if (PyByteArray_Check(args[11]) && PyByteArray_GET_SIZE(args[11]) == 1) { + else if (PyByteArray_Check(args[11])) { + if (PyByteArray_GET_SIZE(args[11]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 12 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[11])); + goto exit; + } l = PyByteArray_AS_STRING(args[11])[0]; } else { @@ -706,10 +874,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 13) { goto skip_optional; } - if (PyBytes_Check(args[12]) && PyBytes_GET_SIZE(args[12]) == 1) { + if (PyBytes_Check(args[12])) { + if (PyBytes_GET_SIZE(args[12]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 13 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[12])); + goto exit; + } m = PyBytes_AS_STRING(args[12])[0]; } - else if (PyByteArray_Check(args[12]) && PyByteArray_GET_SIZE(args[12]) == 1) { + else if (PyByteArray_Check(args[12])) { + if (PyByteArray_GET_SIZE(args[12]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 13 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[12])); + goto exit; + } m = PyByteArray_AS_STRING(args[12])[0]; } else { @@ -719,10 +901,24 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 14) { goto skip_optional; } - if (PyBytes_Check(args[13]) && PyBytes_GET_SIZE(args[13]) == 1) { + if (PyBytes_Check(args[13])) { + if (PyBytes_GET_SIZE(args[13]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 14 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[13])); + goto exit; + } n = PyBytes_AS_STRING(args[13])[0]; } - else if (PyByteArray_Check(args[13]) && PyByteArray_GET_SIZE(args[13]) == 1) { + else if (PyByteArray_Check(args[13])) { + if (PyByteArray_GET_SIZE(args[13]) != 1) { + PyErr_Format(PyExc_TypeError, + "test_char_converter(): argument 14 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[13])); + goto exit; + } n = PyByteArray_AS_STRING(args[13])[0]; } else { @@ -740,7 +936,7 @@ static PyObject * test_char_converter_impl(PyObject *module, char a, char b, char c, char d, char e, char f, char g, char h, char i, char j, char k, char l, char m, char n) -/*[clinic end generated code: output=98589f02422fe6b1 input=e42330417a44feac]*/ +/*[clinic end generated code: output=ff11e203248582df input=e42330417a44feac]*/ /*[clinic input] @@ -1028,7 +1224,10 @@ test_int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[2]) != 1) { - _PyArg_BadArgument("test_int_converter", "argument 3", "a unicode character", args[2]); + PyErr_Format(PyExc_TypeError, + "test_int_converter(): argument 3 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[2])); goto exit; } c = PyUnicode_READ_CHAR(args[2], 0); @@ -1048,7 +1247,7 @@ test_int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) static PyObject * test_int_converter_impl(PyObject *module, int a, int b, int c, myenum d) -/*[clinic end generated code: output=5aed87a7589eefb2 input=d20541fc1ca0553e]*/ +/*[clinic end generated code: output=fbcfb7554688663d input=d20541fc1ca0553e]*/ /*[clinic input] @@ -1980,7 +2179,7 @@ test_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2000,7 +2199,8 @@ test_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec PyObject *a; PyObject *b; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2014,7 +2214,7 @@ test_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec static PyObject * test_keywords_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=73d46a9ae3320f96 input=0d3484844749c05b]*/ +/*[clinic end generated code: output=5411fdee0a85d60d input=0d3484844749c05b]*/ /*[clinic input] @@ -2050,7 +2250,7 @@ test_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2070,7 +2270,8 @@ test_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *a; PyObject *b; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2084,7 +2285,7 @@ test_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, static PyObject * test_keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=c9f02a41f425897d input=384adc78bfa0bff7]*/ +/*[clinic end generated code: output=38b11b4dd0a94399 input=384adc78bfa0bff7]*/ /*[clinic input] @@ -2121,7 +2322,7 @@ test_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2143,7 +2344,8 @@ test_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO PyObject *b = Py_None; PyObject *c = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2168,7 +2370,7 @@ test_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO static PyObject * test_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=b35d4e66f7283e46 input=eda7964f784f4607]*/ +/*[clinic end generated code: output=c59ecb8eca9880cf input=eda7964f784f4607]*/ /*[clinic input] @@ -2207,7 +2409,7 @@ test_keywords_opt_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2230,7 +2432,8 @@ test_keywords_opt_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2265,7 +2468,7 @@ test_keywords_opt_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nar static PyObject * test_keywords_opt_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=ede7e6e65106bf2b input=209387a4815e5082]*/ +/*[clinic end generated code: output=b9572975a3e5d857 input=209387a4815e5082]*/ /*[clinic input] @@ -2303,7 +2506,7 @@ test_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2325,7 +2528,8 @@ test_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *b = Py_None; PyObject *c = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2350,7 +2554,7 @@ test_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nar static PyObject * test_keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=36d4df939a4c3eef input=18393cc64fa000f4]*/ +/*[clinic end generated code: output=94114334841ff55a input=18393cc64fa000f4]*/ /*[clinic input] @@ -2386,7 +2590,7 @@ test_posonly_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2406,7 +2610,8 @@ test_posonly_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *a; PyObject *b; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2420,7 +2625,7 @@ test_posonly_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, static PyObject * test_posonly_keywords_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=4835f4b6cf386c28 input=1767b0ebdf06060e]*/ +/*[clinic end generated code: output=4abfbb43b77fd29a input=1767b0ebdf06060e]*/ /*[clinic input] @@ -2457,7 +2662,7 @@ test_posonly_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2477,7 +2682,8 @@ test_posonly_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *a; PyObject *c; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2491,7 +2697,7 @@ test_posonly_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P static PyObject * test_posonly_kwonly_impl(PyObject *module, PyObject *a, PyObject *c) -/*[clinic end generated code: output=2570ea156a8d3cb5 input=9042f2818f664839]*/ +/*[clinic end generated code: output=9fe05c76d80d63a9 input=9042f2818f664839]*/ /*[clinic input] @@ -2530,7 +2736,7 @@ test_posonly_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2551,7 +2757,8 @@ test_posonly_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *b; PyObject *c; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2567,7 +2774,7 @@ test_posonly_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t static PyObject * test_posonly_keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=aaa0e6b5ce02900d input=29546ebdca492fea]*/ +/*[clinic end generated code: output=de2b8c631dc1b5d3 input=29546ebdca492fea]*/ /*[clinic input] @@ -2606,7 +2813,7 @@ test_posonly_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2629,7 +2836,8 @@ test_posonly_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2655,7 +2863,7 @@ test_posonly_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t na static PyObject * test_posonly_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=1d9f2d8420d0a85f input=cdf5a9625e554e9b]*/ +/*[clinic end generated code: output=70002e6beef31864 input=cdf5a9625e554e9b]*/ /*[clinic input] @@ -2693,7 +2901,7 @@ test_posonly_keywords_opt2(PyObject *module, PyObject *const *args, Py_ssize_t n PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2715,7 +2923,8 @@ test_posonly_keywords_opt2(PyObject *module, PyObject *const *args, Py_ssize_t n PyObject *b = Py_None; PyObject *c = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2740,7 +2949,7 @@ test_posonly_keywords_opt2(PyObject *module, PyObject *const *args, Py_ssize_t n static PyObject * test_posonly_keywords_opt2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=a83caa0505b296cf input=1581299d21d16f14]*/ +/*[clinic end generated code: output=b0f78abf242565cf input=1581299d21d16f14]*/ /*[clinic input] @@ -2779,7 +2988,7 @@ test_posonly_opt_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_ PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2802,7 +3011,8 @@ test_posonly_opt_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_ PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2833,7 +3043,7 @@ test_posonly_opt_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_ static PyObject * test_posonly_opt_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=0b24fba3dc04d26b input=408798ec3d42949f]*/ +/*[clinic end generated code: output=75708b03d2837419 input=408798ec3d42949f]*/ /*[clinic input] @@ -2873,7 +3083,7 @@ test_posonly_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2896,7 +3106,8 @@ test_posonly_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2922,7 +3133,7 @@ test_posonly_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t narg static PyObject * test_posonly_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=592b217bca2f7bcc input=8d8e5643bbbc2309]*/ +/*[clinic end generated code: output=3aab93ed7b9cc713 input=8d8e5643bbbc2309]*/ /*[clinic input] @@ -2961,7 +3172,7 @@ test_posonly_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2983,7 +3194,8 @@ test_posonly_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *b = Py_None; PyObject *c = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3008,7 +3220,7 @@ test_posonly_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ssize_t nar static PyObject * test_posonly_kwonly_opt2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=b8b00420826bc11f input=f7e5eed94f75fff0]*/ +/*[clinic end generated code: output=ac73a9a84a83e0c7 input=f7e5eed94f75fff0]*/ /*[clinic input] @@ -3048,7 +3260,7 @@ test_posonly_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -3071,7 +3283,8 @@ test_posonly_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3102,7 +3315,7 @@ test_posonly_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t static PyObject * test_posonly_opt_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=3b9ee879ebee285a input=1e557dc979d120fd]*/ +/*[clinic end generated code: output=21a1bfee0c50ec98 input=1e557dc979d120fd]*/ /*[clinic input] @@ -3144,7 +3357,7 @@ test_posonly_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssi PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -3168,7 +3381,8 @@ test_posonly_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssi PyObject *d = Py_None; PyObject *e = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3196,7 +3410,7 @@ static PyObject * test_posonly_keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e) -/*[clinic end generated code: output=d380f84f81cc0e45 input=c3884a4f956fdc89]*/ +/*[clinic end generated code: output=60ac8589c5331fc9 input=c3884a4f956fdc89]*/ /*[clinic input] @@ -3236,7 +3450,7 @@ test_posonly_keywords_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ss PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -3259,7 +3473,8 @@ test_posonly_keywords_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ss PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3285,7 +3500,7 @@ test_posonly_keywords_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ss static PyObject * test_posonly_keywords_kwonly_opt2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=ee629e962cb06992 input=68d01d7c0f6dafb0]*/ +/*[clinic end generated code: output=349e1a19c507956a input=68d01d7c0f6dafb0]*/ /*[clinic input] @@ -3328,7 +3543,7 @@ test_posonly_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -3352,7 +3567,8 @@ test_posonly_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py PyObject *d = Py_None; PyObject *e = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3389,7 +3605,7 @@ static PyObject * test_posonly_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e) -/*[clinic end generated code: output=a2721babb42ecfd1 input=d0883d45876f186c]*/ +/*[clinic end generated code: output=94787c9e20861a56 input=d0883d45876f186c]*/ /*[clinic input] @@ -3432,7 +3648,7 @@ test_posonly_keywords_opt2_kwonly_opt(PyObject *module, PyObject *const *args, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -3456,7 +3672,8 @@ test_posonly_keywords_opt2_kwonly_opt(PyObject *module, PyObject *const *args, P PyObject *d = Py_None; PyObject *e = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3498,7 +3715,7 @@ static PyObject * test_posonly_keywords_opt2_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e) -/*[clinic end generated code: output=0626203eedb6e7e8 input=c95e2e1ec93035ad]*/ +/*[clinic end generated code: output=ea8664ae9fdb4769 input=c95e2e1ec93035ad]*/ /*[clinic input] @@ -3543,7 +3760,7 @@ test_posonly_opt_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), }, + .ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), _Py_LATIN1_CHR('f'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -3568,7 +3785,8 @@ test_posonly_opt_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args PyObject *e = Py_None; PyObject *f = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3617,7 +3835,7 @@ test_posonly_opt_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e, PyObject *f) -/*[clinic end generated code: output=07d8acc04558a5a0 input=9914857713c5bbf8]*/ +/*[clinic end generated code: output=6e0ee12cf458acb1 input=9914857713c5bbf8]*/ /*[clinic input] test_keyword_only_parameter @@ -3672,7 +3890,8 @@ test_keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyBytesObject *co_lnotab = (PyBytesObject *)self->co_lnotab; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3693,7 +3912,7 @@ test_keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t static PyObject * test_keyword_only_parameter_impl(PyObject *module, PyBytesObject *co_lnotab) -/*[clinic end generated code: output=b12fe2e515a62603 input=303df5046c7e37a3]*/ +/*[clinic end generated code: output=18b5b3c80412bc4a input=303df5046c7e37a3]*/ /*[clinic input] @@ -3935,8 +4154,8 @@ test_vararg_and_posonly a: object - *args: object / + *args: tuple [clinic start generated code]*/ @@ -3962,30 +4181,67 @@ test_vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t narg goto exit; } a = args[0]; - __clinic_args = PyTuple_New(nargs - 1); - if (!__clinic_args) { + __clinic_args = _PyTuple_FromArray(args + 1, nargs - 1); + if (__clinic_args == NULL) { goto exit; } - for (Py_ssize_t i = 0; i < nargs - 1; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[1 + i])); - } return_value = test_vararg_and_posonly_impl(module, a, __clinic_args); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } static PyObject * test_vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=79b75dc07decc8d6 input=08dc2bf7afbf1613]*/ +/*[clinic end generated code: output=0c11c475e240869e input=2c49a482f68545c0]*/ + +PyDoc_STRVAR(test_vararg_and_posonly__doc__, +"test_vararg_and_posonly($module, a, /, *args)\n" +"--\n" +"\n"); + +#define TEST_VARARG_AND_POSONLY_METHODDEF \ + {"test_vararg_and_posonly", _PyCFunction_CAST(test_vararg_and_posonly), METH_FASTCALL, test_vararg_and_posonly__doc__}, + +static PyObject * +test_vararg_and_posonly_impl(PyObject *module, PyObject *a, Py_ssize_t nargs, + PyObject *const *args); + +static PyObject * +test_vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t nvararg = nargs - 1; + PyObject *a; + PyObject *const *__clinic_args = NULL; + + if (!_PyArg_CheckPositional("test_vararg_and_posonly", nargs, 1, PY_SSIZE_T_MAX)) { + goto exit; + } + a = args[0]; + __clinic_args = _PyTuple_FromArray(args + 1, nargs - 1); + if (__clinic_args == NULL) { + goto exit; + } + return_value = test_vararg_and_posonly_impl(module, a, nvararg, __clinic_args); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + /*[clinic input] test_vararg a: object - *args: object + *args: tuple [clinic start generated code]*/ @@ -4013,7 +4269,7 @@ test_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -4029,33 +4285,42 @@ test_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[1]; + PyObject * const *fastargs; PyObject *a; PyObject *__clinic_args = NULL; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + __clinic_args = nargs > 1 + ? _PyTuple_FromArray(args + 1, nargs - 1) + : PyTuple_New(0); + if (__clinic_args == NULL) { goto exit; } - a = args[0]; - __clinic_args = args[1]; return_value = test_vararg_impl(module, a, __clinic_args); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } static PyObject * test_vararg_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=880365c61ae205d7 input=81d33815ad1bae6e]*/ +/*[clinic end generated code: output=5fa7dc68243a5211 input=7448995636d9186a]*/ /*[clinic input] test_vararg_with_default a: object - *args: object + *args: tuple b: bool = False [clinic start generated code]*/ @@ -4085,7 +4350,7 @@ test_vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -4101,43 +4366,52 @@ test_vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nar .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[2]; + PyObject * const *fastargs; Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *a; PyObject *__clinic_args = NULL; int b = 0; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { goto exit; } - a = args[0]; - __clinic_args = args[1]; + a = fastargs[0]; if (!noptargs) { goto skip_optional_kwonly; } - b = PyObject_IsTrue(args[2]); + b = PyObject_IsTrue(fastargs[1]); if (b < 0) { goto exit; } skip_optional_kwonly: + __clinic_args = nargs > 1 + ? _PyTuple_FromArray(args + 1, nargs - 1) + : PyTuple_New(0); + if (__clinic_args == NULL) { + goto exit; + } return_value = test_vararg_with_default_impl(module, a, __clinic_args, b); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } static PyObject * test_vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, int b) -/*[clinic end generated code: output=291e9a5a09831128 input=6e110b54acd9b22d]*/ +/*[clinic end generated code: output=603ae52bfa307337 input=3a0f9f557ce1f712]*/ /*[clinic input] test_vararg_with_only_defaults - *args: object + *args: tuple b: bool = False c: object = ' ' @@ -4168,7 +4442,7 @@ test_vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -4184,22 +4458,23 @@ test_vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[2]; + PyObject * const *fastargs; Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *__clinic_args = NULL; int b = 0; PyObject *c = " "; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { goto exit; } - __clinic_args = args[0]; if (!noptargs) { goto skip_optional_kwonly; } - if (args[1]) { - b = PyObject_IsTrue(args[1]); + if (fastargs[0]) { + b = PyObject_IsTrue(fastargs[0]); if (b < 0) { goto exit; } @@ -4207,19 +4482,25 @@ test_vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize goto skip_optional_kwonly; } } - c = args[2]; + c = fastargs[1]; skip_optional_kwonly: + __clinic_args = _PyTuple_FromArray(args, nargs); + if (__clinic_args == NULL) { + goto exit; + } return_value = test_vararg_with_only_defaults_impl(module, __clinic_args, b, c); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } static PyObject * test_vararg_with_only_defaults_impl(PyObject *module, PyObject *args, int b, PyObject *c) -/*[clinic end generated code: output=dd21b28f0db26a4b input=fa56a709a035666e]*/ +/*[clinic end generated code: output=195fe286176f4ee8 input=6983e66817f82924]*/ /*[clinic input] test_paramname_module @@ -4270,7 +4551,8 @@ test_paramname_module(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *argsbuf[1]; PyObject *mod; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -4283,7 +4565,7 @@ test_paramname_module(PyObject *module, PyObject *const *args, Py_ssize_t nargs, static PyObject * test_paramname_module_impl(PyObject *module, PyObject *mod) -/*[clinic end generated code: output=4a2a849ecbcc8b53 input=afefe259667f13ba]*/ +/*[clinic end generated code: output=7ab63a2adaea2873 input=afefe259667f13ba]*/ /*[clinic input] mangle1 @@ -4355,7 +4637,8 @@ mangle1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn PyObject *__clinic_nargs; PyObject *__clinic_noptargs; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 9, 9, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 9, /*maxpos*/ 9, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -4379,7 +4662,7 @@ mangle1_impl(PyObject *module, PyObject *args, PyObject *kwnames, PyObject *return_value, PyObject *_keywords, PyObject *_parser, PyObject *argsbuf, PyObject *fastargs, PyObject *nargs, PyObject *noptargs) -/*[clinic end generated code: output=083e5076be9987c3 input=a3ed51bdedf8a3c7]*/ +/*[clinic end generated code: output=f9127cea2f508da3 input=a3ed51bdedf8a3c7]*/ /*[clinic input] mangle2 @@ -4436,7 +4719,8 @@ mangle2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn PyObject *__clinic_kwargs; PyObject *__clinic_return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -4452,7 +4736,7 @@ mangle2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn static PyObject * mangle2_impl(PyObject *module, PyObject *args, PyObject *kwargs, PyObject *return_value) -/*[clinic end generated code: output=2ebb62aaefe7590a input=391766fee51bad7a]*/ +/*[clinic end generated code: output=3afbc3260971e5d3 input=391766fee51bad7a]*/ /*[clinic input] @@ -4474,7 +4758,7 @@ static PyObject * Test_cls_with_param_impl(TestObj *self, PyTypeObject *cls, int a); static PyObject * -Test_cls_with_param(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +Test_cls_with_param(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -4486,7 +4770,7 @@ Test_cls_with_param(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -4505,7 +4789,8 @@ Test_cls_with_param(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ PyObject *argsbuf[1]; int a; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -4513,7 +4798,7 @@ Test_cls_with_param(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ if (a == -1 && PyErr_Occurred()) { goto exit; } - return_value = Test_cls_with_param_impl(self, cls, a); + return_value = Test_cls_with_param_impl((TestObj *)self, cls, a); exit: return return_value; @@ -4521,7 +4806,7 @@ Test_cls_with_param(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ static PyObject * Test_cls_with_param_impl(TestObj *self, PyTypeObject *cls, int a) -/*[clinic end generated code: output=d89b99e83d442be0 input=af158077bd237ef9]*/ +/*[clinic end generated code: output=7e893134a81fef92 input=af158077bd237ef9]*/ /*[clinic input] @@ -4623,18 +4908,18 @@ static PyObject * Test_cls_no_params_impl(TestObj *self, PyTypeObject *cls); static PyObject * -Test_cls_no_params(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +Test_cls_no_params(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "cls_no_params() takes no arguments"); return NULL; } - return Test_cls_no_params_impl(self, cls); + return Test_cls_no_params_impl((TestObj *)self, cls); } static PyObject * Test_cls_no_params_impl(TestObj *self, PyTypeObject *cls) -/*[clinic end generated code: output=4d68b4652c144af3 input=e7e2e4e344e96a11]*/ +/*[clinic end generated code: output=8845de054449f40a input=e7e2e4e344e96a11]*/ /*[clinic input] @@ -4660,7 +4945,7 @@ Test_metho_not_default_return_converter(TestObj *self, PyObject *a) PyObject *return_value = NULL; int _return_value; - _return_value = Test_metho_not_default_return_converter_impl(self, a); + _return_value = Test_metho_not_default_return_converter_impl((TestObj *)self, a); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -4672,7 +4957,7 @@ Test_metho_not_default_return_converter(TestObj *self, PyObject *a) static int Test_metho_not_default_return_converter_impl(TestObj *self, PyObject *a) -/*[clinic end generated code: output=3350de11bd538007 input=428657129b521177]*/ +/*[clinic end generated code: output=b2cce75a7af2e6ce input=428657129b521177]*/ /*[clinic input] @@ -4698,7 +4983,7 @@ static PyObject * Test_an_metho_arg_named_arg_impl(TestObj *self, int arg); static PyObject * -Test_an_metho_arg_named_arg(TestObj *self, PyObject *arg_) +Test_an_metho_arg_named_arg(PyObject *self, PyObject *arg_) { PyObject *return_value = NULL; int arg; @@ -4707,7 +4992,7 @@ Test_an_metho_arg_named_arg(TestObj *self, PyObject *arg_) if (arg == -1 && PyErr_Occurred()) { goto exit; } - return_value = Test_an_metho_arg_named_arg_impl(self, arg); + return_value = Test_an_metho_arg_named_arg_impl((TestObj *)self, arg); exit: return return_value; @@ -4715,13 +5000,13 @@ Test_an_metho_arg_named_arg(TestObj *self, PyObject *arg_) static PyObject * Test_an_metho_arg_named_arg_impl(TestObj *self, int arg) -/*[clinic end generated code: output=9f04de4a62287e28 input=2a53a57cf5624f95]*/ +/*[clinic end generated code: output=38554f09950d07e7 input=2a53a57cf5624f95]*/ /*[clinic input] Test.__init__ - *args: object - / + *args: tuple + Varargs init method. For example, nargs is translated to PyTuple_GET_SIZE. [clinic start generated code]*/ @@ -4746,27 +5031,58 @@ Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) !_PyArg_NoKeywords("Test", kwargs)) { goto exit; } - if (!_PyArg_CheckPositional("Test", PyTuple_GET_SIZE(args), 0, PY_SSIZE_T_MAX)) { - goto exit; - } - __clinic_args = PyTuple_GetSlice(0, -1); + __clinic_args = Py_NewRef(args); return_value = Test___init___impl((TestObj *)self, __clinic_args); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } static int Test___init___impl(TestObj *self, PyObject *args) -/*[clinic end generated code: output=0ed1009fe0dcf98d input=96c3ddc0cd38fc0c]*/ +/*[clinic end generated code: output=f172425cec373cd6 input=4b8388c4e6baab6f]*/ + +PyDoc_STRVAR(Test___init____doc__, +"Test(*args)\n" +"--\n" +"\n" +"Varargs init method. For example, nargs is translated to PyTuple_GET_SIZE."); + +static int +Test___init___impl(TestObj *self, Py_ssize_t nargs, PyObject *const *args); + +static int +Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + PyTypeObject *base_tp = TestType; + PyObject *const *__clinic_args = NULL; + + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoKeywords("Test", kwargs)) { + goto exit; + } + __clinic_args = Py_NewRef(args); + return_value = Test___init___impl((TestObj *)self, nvararg, __clinic_args); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + /*[clinic input] @classmethod Test.__new__ - *args: object - / + *args: tuple + Varargs new method. For example, nargs is translated to PyTuple_GET_SIZE. [clinic start generated code]*/ @@ -4790,20 +5106,50 @@ Test(PyTypeObject *type, PyObject *args, PyObject *kwargs) !_PyArg_NoKeywords("Test", kwargs)) { goto exit; } - if (!_PyArg_CheckPositional("Test", PyTuple_GET_SIZE(args), 0, PY_SSIZE_T_MAX)) { - goto exit; - } - __clinic_args = PyTuple_GetSlice(0, -1); + __clinic_args = Py_NewRef(args); return_value = Test_impl(type, __clinic_args); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } static PyObject * Test_impl(PyTypeObject *type, PyObject *args) -/*[clinic end generated code: output=8b219f6633e2a2e9 input=26a672e2e9750120]*/ +/*[clinic end generated code: output=ee1e8892a67abd4a input=a8259521129cad20]*/ + +PyDoc_STRVAR(Test__doc__, +"Test(*args)\n" +"--\n" +"\n" +"Varargs new method. For example, nargs is translated to PyTuple_GET_SIZE."); + +static PyObject * +Test_impl(PyTypeObject *type, Py_ssize_t nargs, PyObject *const *args); + +static PyObject * +Test(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = TestType; + PyObject *const *__clinic_args = NULL; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("Test", kwargs)) { + goto exit; + } + __clinic_args = Py_NewRef(args); + return_value = Test_impl(type, nvararg, __clinic_args); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + /*[clinic input] @@ -4834,7 +5180,7 @@ Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -4855,7 +5201,8 @@ Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) Py_ssize_t nargs = PyTuple_GET_SIZE(args); PyObject *a; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -4868,7 +5215,7 @@ Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) static int Test___init___impl(TestObj *self, PyObject *a) -/*[clinic end generated code: output=0b9ca79638ab3ecb input=a8f9222a6ab35c59]*/ +/*[clinic end generated code: output=aa1f3d73e611e438 input=a8f9222a6ab35c59]*/ /*[clinic input] @@ -4942,23 +5289,21 @@ static PyObject * Test_meth_coexist_impl(TestObj *self); static PyObject * -Test_meth_coexist(TestObj *self, PyObject *Py_UNUSED(ignored)) +Test_meth_coexist(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return Test_meth_coexist_impl(self); + return Test_meth_coexist_impl((TestObj *)self); } static PyObject * Test_meth_coexist_impl(TestObj *self) -/*[clinic end generated code: output=808a293d0cd27439 input=2a1d75b5e6fec6dd]*/ +/*[clinic end generated code: output=7edf4e95b29f06fa input=2a1d75b5e6fec6dd]*/ /*[clinic input] @getter Test.property [clinic start generated code]*/ -#if defined(Test_property_HAS_DOCSTR) -# define Test_property_DOCSTR Test_property__doc__ -#else +#if !defined(Test_property_DOCSTR) # define Test_property_DOCSTR NULL #endif #if defined(TEST_PROPERTY_GETSETDEF) @@ -4972,23 +5317,21 @@ static PyObject * Test_property_get_impl(TestObj *self); static PyObject * -Test_property_get(TestObj *self, void *Py_UNUSED(context)) +Test_property_get(PyObject *self, void *Py_UNUSED(context)) { - return Test_property_get_impl(self); + return Test_property_get_impl((TestObj *)self); } static PyObject * Test_property_get_impl(TestObj *self) -/*[clinic end generated code: output=27b519719d992e03 input=2d92b3449fbc7d2b]*/ +/*[clinic end generated code: output=b38d68abd3466a6e input=2d92b3449fbc7d2b]*/ /*[clinic input] @setter Test.property [clinic start generated code]*/ -#if defined(TEST_PROPERTY_HAS_DOCSTR) -# define Test_property_DOCSTR Test_property__doc__ -#else +#if !defined(Test_property_DOCSTR) # define Test_property_DOCSTR NULL #endif #if defined(TEST_PROPERTY_GETSETDEF) @@ -5002,18 +5345,87 @@ static int Test_property_set_impl(TestObj *self, PyObject *value); static int -Test_property_set(TestObj *self, PyObject *value, void *Py_UNUSED(context)) +Test_property_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) { int return_value; - return_value = Test_property_set_impl(self, value); + return_value = Test_property_set_impl((TestObj *)self, value); return return_value; } static int Test_property_set_impl(TestObj *self, PyObject *value) -/*[clinic end generated code: output=d51023f17c4ac3a1 input=3bc3f46a23c83a88]*/ +/*[clinic end generated code: output=49f925ab2a33b637 input=3bc3f46a23c83a88]*/ + +/*[clinic input] +@setter +Test.setter_first_with_docstr +[clinic start generated code]*/ + +#if !defined(Test_setter_first_with_docstr_DOCSTR) +# define Test_setter_first_with_docstr_DOCSTR NULL +#endif +#if defined(TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF) +# undef TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF +# define TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF {"setter_first_with_docstr", (getter)Test_setter_first_with_docstr_get, (setter)Test_setter_first_with_docstr_set, Test_setter_first_with_docstr_DOCSTR}, +#else +# define TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF {"setter_first_with_docstr", NULL, (setter)Test_setter_first_with_docstr_set, NULL}, +#endif + +static int +Test_setter_first_with_docstr_set_impl(TestObj *self, PyObject *value); + +static int +Test_setter_first_with_docstr_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + return_value = Test_setter_first_with_docstr_set_impl((TestObj *)self, value); + + return return_value; +} + +static int +Test_setter_first_with_docstr_set_impl(TestObj *self, PyObject *value) +/*[clinic end generated code: output=5aaf44373c0af545 input=31a045ce11bbe961]*/ + +/*[clinic input] +@getter +Test.setter_first_with_docstr + +my silly docstring +[clinic start generated code]*/ + +PyDoc_STRVAR(Test_setter_first_with_docstr__doc__, +"my silly docstring"); +#if defined(Test_setter_first_with_docstr_DOCSTR) +# undef Test_setter_first_with_docstr_DOCSTR +#endif +#define Test_setter_first_with_docstr_DOCSTR Test_setter_first_with_docstr__doc__ + +#if !defined(Test_setter_first_with_docstr_DOCSTR) +# define Test_setter_first_with_docstr_DOCSTR NULL +#endif +#if defined(TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF) +# undef TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF +# define TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF {"setter_first_with_docstr", (getter)Test_setter_first_with_docstr_get, (setter)Test_setter_first_with_docstr_set, Test_setter_first_with_docstr_DOCSTR}, +#else +# define TEST_SETTER_FIRST_WITH_DOCSTR_GETSETDEF {"setter_first_with_docstr", (getter)Test_setter_first_with_docstr_get, NULL, Test_setter_first_with_docstr_DOCSTR}, +#endif + +static PyObject * +Test_setter_first_with_docstr_get_impl(TestObj *self); + +static PyObject * +Test_setter_first_with_docstr_get(PyObject *self, void *Py_UNUSED(context)) +{ + return Test_setter_first_with_docstr_get_impl((TestObj *)self); +} + +static PyObject * +Test_setter_first_with_docstr_get_impl(TestObj *self) +/*[clinic end generated code: output=fe6e3aa844a24920 input=10af4e43b3cb34dc]*/ /*[clinic input] output push @@ -5102,7 +5514,7 @@ mangled_c_keyword_identifier(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(i), }, + .ob_item = { _Py_LATIN1_CHR('i'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -5121,7 +5533,8 @@ mangled_c_keyword_identifier(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *argsbuf[1]; int int_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -5137,7 +5550,7 @@ mangled_c_keyword_identifier(PyObject *module, PyObject *const *args, Py_ssize_t static PyObject * mangled_c_keyword_identifier_impl(PyObject *module, int int_value) -/*[clinic end generated code: output=f24b37e0368e0eb8 input=060876448ab567a2]*/ +/*[clinic end generated code: output=19ec1a02f0819cf7 input=060876448ab567a2]*/ /*[clinic input] @@ -5295,7 +5708,7 @@ Test__pyarg_parsestackandkeywords_impl(TestObj *self, PyTypeObject *cls, Py_ssize_t key_length); static PyObject * -Test__pyarg_parsestackandkeywords(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +Test__pyarg_parsestackandkeywords(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -5318,7 +5731,7 @@ Test__pyarg_parsestackandkeywords(TestObj *self, PyTypeObject *cls, PyObject *co &key, &key_length)) { goto exit; } - return_value = Test__pyarg_parsestackandkeywords_impl(self, cls, key, key_length); + return_value = Test__pyarg_parsestackandkeywords_impl((TestObj *)self, cls, key, key_length); exit: return return_value; @@ -5328,7 +5741,7 @@ static PyObject * Test__pyarg_parsestackandkeywords_impl(TestObj *self, PyTypeObject *cls, const char *key, Py_ssize_t key_length) -/*[clinic end generated code: output=4fda8a7f2547137c input=fc72ef4b4cfafabc]*/ +/*[clinic end generated code: output=7060c213d7b8200e input=fc72ef4b4cfafabc]*/ /*[clinic input] @@ -5380,7 +5793,8 @@ fn_with_default_binop_expr(PyObject *module, PyObject *const *args, Py_ssize_t n Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *arg = CONST_A + CONST_B; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -5397,7 +5811,7 @@ fn_with_default_binop_expr(PyObject *module, PyObject *const *args, Py_ssize_t n static PyObject * fn_with_default_binop_expr_impl(PyObject *module, PyObject *arg) -/*[clinic end generated code: output=018672772e4092ff input=1b55c8ae68d89453]*/ +/*[clinic end generated code: output=e15a6f5d9f7bea0c input=1b55c8ae68d89453]*/ /*[python input] @@ -5449,7 +5863,7 @@ docstr_fallback_to_converter_default(PyObject *module, PyObject *const *args, Py PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -5468,7 +5882,8 @@ docstr_fallback_to_converter_default(PyObject *module, PyObject *const *args, Py PyObject *argsbuf[1]; str a; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -5483,7 +5898,7 @@ docstr_fallback_to_converter_default(PyObject *module, PyObject *const *args, Py static PyObject * docstr_fallback_to_converter_default_impl(PyObject *module, str a) -/*[clinic end generated code: output=ae24a9c6f60ee8a6 input=0cbe6a4d24bc2274]*/ +/*[clinic end generated code: output=15bc9f32945ed0da input=0cbe6a4d24bc2274]*/ /*[clinic input] diff --git a/Lib/test/crashers/README b/Lib/test/crashers/README index 0259a0688cbc67..7111946b93b280 100644 --- a/Lib/test/crashers/README +++ b/Lib/test/crashers/README @@ -8,13 +8,10 @@ Each test should fail when run from the command line: ./python Lib/test/crashers/weakref_in_del.py Put as much info into a docstring or comments to help determine the cause of the -failure, as well as a bugs.python.org issue number if it exists. Particularly -note if the cause is system or environment dependent and what the variables are. +failure, as well as an issue number or link if it exists. +Particularly note if the cause is system or environment dependent and +what the variables are. Once the crash is fixed, the test case should be moved into an appropriate test (even if it was originally from the test suite). This ensures the regression doesn't happen again. And if it does, it should be easier to track down. - -Also see Lib/test_crashers.py which exercises the crashers in this directory. -In particular, make sure to add any new infinite loop crashers to the black -list so it doesn't try to run them. diff --git a/Lib/test/crashers/bogus_code_obj.py b/Lib/test/crashers/bogus_code_obj.py index e71b3582cf2d76..b3ff07995c95ed 100644 --- a/Lib/test/crashers/bogus_code_obj.py +++ b/Lib/test/crashers/bogus_code_obj.py @@ -12,8 +12,8 @@ """ -import types +def f(): + pass -co = types.CodeType(0, 0, 0, 0, 0, 0, b'\x04\x00\x71\x00', - (), (), (), '', '', 1, b'') -exec(co) +f.__code__ = f.__code__.replace(co_code=b"") +f() diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 31fc383e29707a..25a3015c4e19ce 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1,7 +1,4 @@ -"""Test date/time type. - -See https://www.zope.dev/Members/fdrake/DateTimeWiki/TestCases -""" +"""Test the datetime module.""" import bisect import copy import decimal @@ -13,6 +10,7 @@ import re import struct import sys +import textwrap import unittest import warnings @@ -22,6 +20,7 @@ from test import support from test.support import is_resource_enabled, ALWAYS_EQ, LARGEST, SMALLEST +from test.support import script_helper, warnings_helper import datetime as datetime_module from datetime import MINYEAR, MAXYEAR @@ -37,6 +36,10 @@ import _testcapi except ImportError: _testcapi = None +try: + import _interpreters +except ModuleNotFoundError: + _interpreters = None # Needed by test_datetime import _strptime @@ -507,6 +510,7 @@ class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase): def test_constructor(self): eq = self.assertEqual + ra = self.assertRaises td = timedelta # Check keyword args to constructor @@ -530,6 +534,15 @@ def test_constructor(self): eq(td(seconds=0.001), td(milliseconds=1)) eq(td(milliseconds=0.001), td(microseconds=1)) + # Check type of args to constructor + ra(TypeError, lambda: td(weeks='1')) + ra(TypeError, lambda: td(days='1')) + ra(TypeError, lambda: td(hours='1')) + ra(TypeError, lambda: td(minutes='1')) + ra(TypeError, lambda: td(seconds='1')) + ra(TypeError, lambda: td(milliseconds='1')) + ra(TypeError, lambda: td(microseconds='1')) + def test_computations(self): eq = self.assertEqual td = timedelta @@ -1103,6 +1116,85 @@ def test_delta_non_days_ignored(self): dt2 = dt - delta self.assertEqual(dt2, dt - days) + def test_strptime(self): + inputs = [ + # Basic valid cases + (date(1998, 2, 3), '1998-02-03', '%Y-%m-%d'), + (date(2004, 12, 2), '2004-12-02', '%Y-%m-%d'), + + # Edge cases: Leap year + (date(2020, 2, 29), '2020-02-29', '%Y-%m-%d'), # Valid leap year date + + # bpo-34482: Handle surrogate pairs + (date(2004, 12, 2), '2004-12\ud80002', '%Y-%m\ud800%d'), + (date(2004, 12, 2), '2004\ud80012-02', '%Y\ud800%m-%d'), + + # Month/day variations + (date(2004, 2, 1), '2004-02', '%Y-%m'), # No day provided + (date(2004, 2, 1), '02-2004', '%m-%Y'), # Month and year swapped + + # Different day-month-year formats + (date(2004, 12, 2), '02/12/2004', '%d/%m/%Y'), # Day/Month/Year + (date(2004, 12, 2), '12/02/2004', '%m/%d/%Y'), # Month/Day/Year + + # Different separators + (date(2023, 9, 24), '24.09.2023', '%d.%m.%Y'), # Dots as separators + (date(2023, 9, 24), '24-09-2023', '%d-%m-%Y'), # Dashes + (date(2023, 9, 24), '2023/09/24', '%Y/%m/%d'), # Slashes + + # Handling years with fewer digits + (date(127, 2, 3), '0127-02-03', '%Y-%m-%d'), + (date(99, 2, 3), '0099-02-03', '%Y-%m-%d'), + (date(5, 2, 3), '0005-02-03', '%Y-%m-%d'), + + # Variations on ISO 8601 format + (date(2023, 9, 25), '2023-W39-1', '%G-W%V-%u'), # ISO week date (Week 39, Monday) + (date(2023, 9, 25), '2023-268', '%Y-%j'), # Year and day of the year (Julian) + ] + for expected, string, format in inputs: + with self.subTest(string=string, format=format): + got = date.strptime(string, format) + self.assertEqual(expected, got) + self.assertIs(type(got), date) + + def test_strptime_single_digit(self): + # bpo-34903: Check that single digit dates are allowed. + strptime = date.strptime + with self.assertRaises(ValueError): + # %y does require two digits. + newdate = strptime('01/02/3', '%d/%m/%y') + + d1 = date(2003, 2, 1) + d2 = date(2003, 1, 2) + d3 = date(2003, 1, 25) + inputs = [ + ('%d', '1/02/03', '%d/%m/%y', d1), + ('%m', '01/2/03', '%d/%m/%y', d1), + ('%j', '2/03', '%j/%y', d2), + ('%w', '6/04/03', '%w/%U/%y', d1), + # %u requires a single digit. + ('%W', '6/4/2003', '%u/%W/%Y', d1), + ('%V', '6/4/2003', '%u/%V/%G', d3), + ] + for reason, string, format, target in inputs: + reason = 'test single digit ' + reason + with self.subTest(reason=reason, + string=string, + format=format, + target=target): + newdate = strptime(string, format) + self.assertEqual(newdate, target, msg=reason) + + @warnings_helper.ignore_warnings(category=DeprecationWarning) + def test_strptime_leap_year(self): + # GH-70647: warns if parsing a format with a day and no year. + with self.assertRaises(ValueError): + # The existing behavior that GH-70647 seeks to change. + date.strptime('02-29', '%m-%d') + with self._assertNotWarns(DeprecationWarning): + date.strptime('20-03-14', '%y-%m-%d') + date.strptime('02-29,2024', '%m-%d,%Y') + class SubclassDate(date): sub_var = 1 @@ -1335,6 +1427,11 @@ def test_insane_fromtimestamp(self): self.assertRaises(OverflowError, self.theclass.fromtimestamp, insane) + def test_fromtimestamp_with_none_arg(self): + # See gh-120268 for more details + with self.assertRaises(TypeError): + self.theclass.fromtimestamp(None) + def test_today(self): import time @@ -1686,18 +1783,35 @@ def test_bool(self): self.assertTrue(self.theclass.max) def test_strftime_y2k(self): - for y in (1, 49, 70, 99, 100, 999, 1000, 1970): - d = self.theclass(y, 1, 1) - # Issue 13305: For years < 1000, the value is not always - # padded to 4 digits across platforms. The C standard - # assumes year >= 1900, so it does not specify the number - # of digits. - if d.strftime("%Y") != '%04d' % y: - # Year 42 returns '42', not padded - self.assertEqual(d.strftime("%Y"), '%d' % y) - # '0042' is obtained anyway - if support.has_strftime_extensions: - self.assertEqual(d.strftime("%4Y"), '%04d' % y) + # Test that years less than 1000 are 0-padded; note that the beginning + # of an ISO 8601 year may fall in an ISO week of the year before, and + # therefore needs an offset of -1 when formatting with '%G'. + dataset = ( + (1, 0), + (49, -1), + (70, 0), + (99, 0), + (100, -1), + (999, 0), + (1000, 0), + (1970, 0), + ) + specifiers = 'YG' + if _time.strftime('%F', (1900, 1, 1, 0, 0, 0, 0, 1, 0)) == '1900-01-01': + specifiers += 'FC' + for year, g_offset in dataset: + for specifier in specifiers: + with self.subTest(year=year, specifier=specifier): + d = self.theclass(year, 1, 1) + if specifier == 'G': + year += g_offset + if specifier == 'C': + expected = f"{year // 100:02d}" + else: + expected = f"{year:04d}" + if specifier == 'F': + expected += f"-01-01" + self.assertEqual(d.strftime(f"%{specifier}"), expected) def test_replace(self): cls = self.theclass @@ -1927,6 +2041,10 @@ def test_fromisoformat_fails(self): '2009-02-29', # Invalid leap day '2019-W53-1', # No week 53 in 2019 '2020-W54-1', # No week 54 + '0000-W25-1', # Invalid year + '10000-W25-1', # Invalid year + '2020-W25-0', # Invalid day-of-week + '2020-W25-8', # Invalid day-of-week '2009\ud80002\ud80028', # Separators are surrogate codepoints ] @@ -2703,7 +2821,8 @@ def test_utcnow(self): def test_strptime(self): string = '2004-12-01 13:02:47.197' format = '%Y-%m-%d %H:%M:%S.%f' - expected = _strptime._strptime_datetime(self.theclass, string, format) + expected = _strptime._strptime_datetime_datetime(self.theclass, string, + format) got = self.theclass.strptime(string, format) self.assertEqual(expected, got) self.assertIs(type(expected), self.theclass) @@ -2717,8 +2836,8 @@ def test_strptime(self): ] for string, format in inputs: with self.subTest(string=string, format=format): - expected = _strptime._strptime_datetime(self.theclass, string, - format) + expected = _strptime._strptime_datetime_datetime(self.theclass, + string, format) got = self.theclass.strptime(string, format) self.assertEqual(expected, got) @@ -2793,6 +2912,20 @@ def test_strptime_single_digit(self): newdate = strptime(string, format) self.assertEqual(newdate, target, msg=reason) + @warnings_helper.ignore_warnings(category=DeprecationWarning) + def test_strptime_leap_year(self): + # GH-70647: warns if parsing a format with a day and no year. + with self.assertRaises(ValueError): + # The existing behavior that GH-70647 seeks to change. + self.theclass.strptime('02-29', '%m-%d') + with self.assertWarnsRegex(DeprecationWarning, + r'.*day of month without a year.*'): + self.theclass.strptime('03-14.159265', '%m-%d.%f') + with self._assertNotWarns(DeprecationWarning): + self.theclass.strptime('20-03-14.159265', '%y-%m-%d.%f') + with self._assertNotWarns(DeprecationWarning): + self.theclass.strptime('02-29,2024', '%m-%d,%Y') + def test_more_timetuple(self): # This tests fields beyond those tested by the TestDate.test_timetuple. t = self.theclass(2004, 12, 31, 6, 22, 33) @@ -2826,11 +2959,32 @@ def test_more_strftime(self): self.assertEqual(t.strftime("%z"), "-0200" + z) self.assertEqual(t.strftime("%:z"), "-02:00:" + z) - # bpo-34482: Check that surrogates don't cause a crash. - try: - t.strftime('%y\ud800%m %H\ud800%M') - except UnicodeEncodeError: - pass + def test_strftime_special(self): + t = self.theclass(2004, 12, 31, 6, 22, 33, 47) + s1 = t.strftime('%c') + s2 = t.strftime('%B') + # gh-52551, gh-78662: Unicode strings should pass through strftime, + # independently from locale. + self.assertEqual(t.strftime('\U0001f40d'), '\U0001f40d') + self.assertEqual(t.strftime('\U0001f4bb%c\U0001f40d%B'), f'\U0001f4bb{s1}\U0001f40d{s2}') + self.assertEqual(t.strftime('%c\U0001f4bb%B\U0001f40d'), f'{s1}\U0001f4bb{s2}\U0001f40d') + # Lone surrogates should pass through. + self.assertEqual(t.strftime('\ud83d'), '\ud83d') + self.assertEqual(t.strftime('\udc0d'), '\udc0d') + self.assertEqual(t.strftime('\ud83d%c\udc0d%B'), f'\ud83d{s1}\udc0d{s2}') + self.assertEqual(t.strftime('%c\ud83d%B\udc0d'), f'{s1}\ud83d{s2}\udc0d') + self.assertEqual(t.strftime('%c\udc0d%B\ud83d'), f'{s1}\udc0d{s2}\ud83d') + # Surrogate pairs should not recombine. + self.assertEqual(t.strftime('\ud83d\udc0d'), '\ud83d\udc0d') + self.assertEqual(t.strftime('%c\ud83d\udc0d%B'), f'{s1}\ud83d\udc0d{s2}') + # Surrogate-escaped bytes should not recombine. + self.assertEqual(t.strftime('\udcf0\udc9f\udc90\udc8d'), '\udcf0\udc9f\udc90\udc8d') + self.assertEqual(t.strftime('%c\udcf0\udc9f\udc90\udc8d%B'), f'{s1}\udcf0\udc9f\udc90\udc8d{s2}') + # gh-124531: The null character should not terminate the format string. + self.assertEqual(t.strftime('\0'), '\0') + self.assertEqual(t.strftime('\0'*1000), '\0'*1000) + self.assertEqual(t.strftime('\0%c\0%B'), f'\0{s1}\0{s2}') + self.assertEqual(t.strftime('%c\0%B\0'), f'{s1}\0{s2}\0') def test_extract(self): dt = self.theclass(2002, 3, 4, 18, 45, 3, 1234) @@ -3299,6 +3453,9 @@ def test_fromisoformat_datetime_examples(self): ('2025-01-02T03:04:05,678+00:00:10', self.theclass(2025, 1, 2, 3, 4, 5, 678000, tzinfo=timezone(timedelta(seconds=10)))), + ('2025-01-02T24:00:00', self.theclass(2025, 1, 3, 0, 0, 0)), + ('2025-01-31T24:00:00', self.theclass(2025, 2, 1, 0, 0, 0)), + ('2025-12-31T24:00:00', self.theclass(2026, 1, 1, 0, 0, 0)) ] for input_str, expected in examples: @@ -3330,11 +3487,17 @@ def test_fromisoformat_fails_datetime(self): '2009-04-19T12:', # Ends with time separator '2009-04-19T12:30:', # Ends with time separator '2009-04-19T12:30:45.', # Ends with time separator - '2009-04-19T12:30:45.123456+', # Ends with timzone separator - '2009-04-19T12:30:45.123456-', # Ends with timzone separator + '2009-04-19T12:30:45.123456+', # Ends with timezone separator + '2009-04-19T12:30:45.123456-', # Ends with timezone separator '2009-04-19T12:30:45.123456-05:00a', # Extra text '2009-04-19T12:30:45.123-05:00a', # Extra text '2009-04-19T12:30:45-05:00a', # Extra text + '2009-04-19T24:00:00.000001', # Has non-zero microseconds on 24:00 + '2009-04-19T24:00:01.000000', # Has non-zero seconds on 24:00 + '2009-04-19T24:01:00.000000', # Has non-zero minutes on 24:00 + '2009-04-32T24:00:00.000000', # Day is invalid before wrapping due to 24:00 + '2009-13-01T24:00:00.000000', # Month is invalid before wrapping due to 24:00 + '9999-12-31T24:00:00.000000', # Year is invalid after wrapping due to 24:00 ] for bad_str in bad_strs: @@ -3604,6 +3767,33 @@ def test_strftime(self): # gh-85432: The parameter was named "fmt" in the pure-Python impl. t.strftime(format="%f") + def test_strftime_special(self): + t = self.theclass(1, 2, 3, 4) + s1 = t.strftime('%I%p%Z') + s2 = t.strftime('%X') + # gh-52551, gh-78662: Unicode strings should pass through strftime, + # independently from locale. + self.assertEqual(t.strftime('\U0001f40d'), '\U0001f40d') + self.assertEqual(t.strftime('\U0001f4bb%I%p%Z\U0001f40d%X'), f'\U0001f4bb{s1}\U0001f40d{s2}') + self.assertEqual(t.strftime('%I%p%Z\U0001f4bb%X\U0001f40d'), f'{s1}\U0001f4bb{s2}\U0001f40d') + # Lone surrogates should pass through. + self.assertEqual(t.strftime('\ud83d'), '\ud83d') + self.assertEqual(t.strftime('\udc0d'), '\udc0d') + self.assertEqual(t.strftime('\ud83d%I%p%Z\udc0d%X'), f'\ud83d{s1}\udc0d{s2}') + self.assertEqual(t.strftime('%I%p%Z\ud83d%X\udc0d'), f'{s1}\ud83d{s2}\udc0d') + self.assertEqual(t.strftime('%I%p%Z\udc0d%X\ud83d'), f'{s1}\udc0d{s2}\ud83d') + # Surrogate pairs should not recombine. + self.assertEqual(t.strftime('\ud83d\udc0d'), '\ud83d\udc0d') + self.assertEqual(t.strftime('%I%p%Z\ud83d\udc0d%X'), f'{s1}\ud83d\udc0d{s2}') + # Surrogate-escaped bytes should not recombine. + self.assertEqual(t.strftime('\udcf0\udc9f\udc90\udc8d'), '\udcf0\udc9f\udc90\udc8d') + self.assertEqual(t.strftime('%I%p%Z\udcf0\udc9f\udc90\udc8d%X'), f'{s1}\udcf0\udc9f\udc90\udc8d{s2}') + # gh-124531: The null character should not terminate the format string. + self.assertEqual(t.strftime('\0'), '\0') + self.assertEqual(t.strftime('\0'*1000), '\0'*1000) + self.assertEqual(t.strftime('\0%I%p%Z\0%X'), f'\0{s1}\0{s2}') + self.assertEqual(t.strftime('%I%p%Z\0%X\0'), f'{s1}\0{s2}\0') + def test_format(self): t = self.theclass(1, 2, 3, 4) self.assertEqual(t.__format__(''), str(t)) @@ -3697,6 +3887,78 @@ def test_compat_unpickle(self): derived = loads(data, encoding='latin1') self.assertEqual(derived, expected) + def test_strptime(self): + # bpo-34482: Check that surrogates are handled properly. + inputs = [ + (self.theclass(13, 2, 47, 197000), '13:02:47.197', '%H:%M:%S.%f'), + (self.theclass(13, 2, 47, 197000), '13:02\ud80047.197', '%H:%M\ud800%S.%f'), + (self.theclass(13, 2, 47, 197000), '13\ud80002:47.197', '%H\ud800%M:%S.%f'), + ] + for expected, string, format in inputs: + with self.subTest(string=string, format=format): + got = self.theclass.strptime(string, format) + self.assertEqual(expected, got) + self.assertIs(type(got), self.theclass) + + def test_strptime_tz(self): + strptime = self.theclass.strptime + self.assertEqual(strptime("+0002", "%z").utcoffset(), 2 * MINUTE) + self.assertEqual(strptime("-0002", "%z").utcoffset(), -2 * MINUTE) + self.assertEqual( + strptime("-00:02:01.000003", "%z").utcoffset(), + -timedelta(minutes=2, seconds=1, microseconds=3) + ) + # Only local timezone and UTC are supported + for tzseconds, tzname in ((0, 'UTC'), (0, 'GMT'), + (-_time.timezone, _time.tzname[0])): + if tzseconds < 0: + sign = '-' + seconds = -tzseconds + else: + sign ='+' + seconds = tzseconds + hours, minutes = divmod(seconds//60, 60) + tstr = "{}{:02d}{:02d} {}".format(sign, hours, minutes, tzname) + with self.subTest(tstr=tstr): + t = strptime(tstr, "%z %Z") + self.assertEqual(t.utcoffset(), timedelta(seconds=tzseconds)) + self.assertEqual(t.tzname(), tzname) + self.assertIs(type(t), self.theclass) + + # Can produce inconsistent time + tstr, fmt = "+1234 UTC", "%z %Z" + t = strptime(tstr, fmt) + self.assertEqual(t.utcoffset(), 12 * HOUR + 34 * MINUTE) + self.assertEqual(t.tzname(), 'UTC') + # yet will roundtrip + self.assertEqual(t.strftime(fmt), tstr) + + # Produce naive time if no %z is provided + self.assertEqual(strptime("UTC", "%Z").tzinfo, None) + + def test_strptime_errors(self): + for tzstr in ("-2400", "-000", "z"): + with self.assertRaises(ValueError): + self.theclass.strptime(tzstr, "%z") + + def test_strptime_single_digit(self): + # bpo-34903: Check that single digit times are allowed. + t = self.theclass(4, 5, 6) + inputs = [ + ('%H', '4:05:06', '%H:%M:%S', t), + ('%M', '04:5:06', '%H:%M:%S', t), + ('%S', '04:05:6', '%H:%M:%S', t), + ('%I', '4am:05:06', '%I%p:%M:%S', t), + ] + for reason, string, format, target in inputs: + reason = 'test single digit ' + reason + with self.subTest(reason=reason, + string=string, + format=format, + target=target): + newdate = self.theclass.strptime(string, format) + self.assertEqual(newdate, target, msg=reason) + def test_bool(self): # time is always True. cls = self.theclass @@ -4055,9 +4317,8 @@ def tzname(self, dt): return self.tz self.assertRaises(TypeError, t.strftime, "%Z") # Issue #6697: - if '_Fast' in self.__class__.__name__: - Badtzname.tz = '\ud800' - self.assertRaises(ValueError, t.strftime, "%Z") + Badtzname.tz = '\ud800' + self.assertEqual(t.strftime("%Z"), '\ud800') def test_hash_edge_cases(self): # Offsets that overflow a basic time. @@ -4269,7 +4530,7 @@ def test_fromisoformat_timezone(self): with self.subTest(tstr=tstr): t_rt = self.theclass.fromisoformat(tstr) - assert t == t_rt, t_rt + assert t == t_rt def test_fromisoformat_timespecs(self): time_bases = [ @@ -4393,6 +4654,8 @@ def test_fromisoformat_fails(self): '12:30:45.123456-', # Extra at end of microsecond time '12:30:45.123456+', # Extra at end of microsecond time '12:30:45.123456+12:00:30a', # Extra at end of full time + '12.5', # Decimal mark at end of hour + '12:30,5', # Decimal mark at end of minute ] for bad_str in bad_strs: @@ -6754,6 +7017,123 @@ def test_datetime_from_timestamp(self): self.assertEqual(dt_orig, dt_rt) + def test_type_check_in_subinterp(self): + # iOS requires the use of the custom framework loader, + # not the ExtensionFileLoader. + if sys.platform == "ios": + extension_loader = "AppleFrameworkLoader" + else: + extension_loader = "ExtensionFileLoader" + + script = textwrap.dedent(f""" + if {_interpreters is None}: + import _testcapi as module + module.test_datetime_capi() + else: + import importlib.machinery + import importlib.util + fullname = '_testcapi_datetime' + origin = importlib.util.find_spec('_testcapi').origin + loader = importlib.machinery.{extension_loader}(fullname, origin) + spec = importlib.util.spec_from_loader(fullname, loader) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + def run(type_checker, obj): + if not type_checker(obj, True): + raise TypeError(f'{{type(obj)}} is not C API type') + + import _datetime + run(module.datetime_check_date, _datetime.date.today()) + run(module.datetime_check_datetime, _datetime.datetime.now()) + run(module.datetime_check_time, _datetime.time(12, 30)) + run(module.datetime_check_delta, _datetime.timedelta(1)) + run(module.datetime_check_tzinfo, _datetime.tzinfo()) + """) + if _interpreters is None: + ret = support.run_in_subinterp(script) + self.assertEqual(ret, 0) + else: + for name in ('isolated', 'legacy'): + with self.subTest(name): + config = _interpreters.new_config(name).__dict__ + ret = support.run_in_subinterp_with_config(script, **config) + self.assertEqual(ret, 0) + + +class ExtensionModuleTests(unittest.TestCase): + + def setUp(self): + if self.__class__.__name__.endswith('Pure'): + self.skipTest('Not relevant in pure Python') + + @support.cpython_only + def test_gh_120161(self): + with self.subTest('simple'): + script = textwrap.dedent(""" + import datetime + from _ast import Tuple + f = lambda: None + Tuple.dims = property(f, f) + + class tzutc(datetime.tzinfo): + pass + """) + script_helper.assert_python_ok('-c', script) + + with self.subTest('complex'): + script = textwrap.dedent(""" + import asyncio + import datetime + from typing import Type + + class tzutc(datetime.tzinfo): + pass + _EPOCHTZ = datetime.datetime(1970, 1, 1, tzinfo=tzutc()) + + class FakeDateMeta(type): + def __instancecheck__(self, obj): + return True + class FakeDate(datetime.date, metaclass=FakeDateMeta): + pass + def pickle_fake_date(datetime_) -> Type[FakeDate]: + # A pickle function for FakeDate + return FakeDate + """) + script_helper.assert_python_ok('-c', script) + + def test_update_type_cache(self): + # gh-120782 + script = textwrap.dedent(""" + import sys + for i in range(5): + import _datetime + assert _datetime.date.max > _datetime.date.min + assert _datetime.time.max > _datetime.time.min + assert _datetime.datetime.max > _datetime.datetime.min + assert _datetime.timedelta.max > _datetime.timedelta.min + assert _datetime.date.__dict__["min"] is _datetime.date.min + assert _datetime.date.__dict__["max"] is _datetime.date.max + assert _datetime.date.__dict__["resolution"] is _datetime.date.resolution + assert _datetime.time.__dict__["min"] is _datetime.time.min + assert _datetime.time.__dict__["max"] is _datetime.time.max + assert _datetime.time.__dict__["resolution"] is _datetime.time.resolution + assert _datetime.datetime.__dict__["min"] is _datetime.datetime.min + assert _datetime.datetime.__dict__["max"] is _datetime.datetime.max + assert _datetime.datetime.__dict__["resolution"] is _datetime.datetime.resolution + assert _datetime.timedelta.__dict__["min"] is _datetime.timedelta.min + assert _datetime.timedelta.__dict__["max"] is _datetime.timedelta.max + assert _datetime.timedelta.__dict__["resolution"] is _datetime.timedelta.resolution + assert _datetime.timezone.__dict__["min"] is _datetime.timezone.min + assert _datetime.timezone.__dict__["max"] is _datetime.timezone.max + assert _datetime.timezone.__dict__["utc"] is _datetime.timezone.utc + assert isinstance(_datetime.timezone.min, _datetime.tzinfo) + assert isinstance(_datetime.timezone.max, _datetime.tzinfo) + assert isinstance(_datetime.timezone.utc, _datetime.tzinfo) + del sys.modules['_datetime'] + """) + script_helper.assert_python_ok('-c', script) + def load_tests(loader, standard_tests, pattern): standard_tests.addTest(ZoneInfoCompleteTest()) diff --git a/Lib/test/decimaltestdata/ddFMA.decTest b/Lib/test/decimaltestdata/ddFMA.decTest index 9094fc015bde18..7f2e52303747f8 100644 --- a/Lib/test/decimaltestdata/ddFMA.decTest +++ b/Lib/test/decimaltestdata/ddFMA.decTest @@ -1663,7 +1663,7 @@ ddfma375087 fma 1 12345678 1E-33 -> 12345678.00000001 Inexac ddfma375088 fma 1 12345678 1E-34 -> 12345678.00000001 Inexact Rounded ddfma375089 fma 1 12345678 1E-35 -> 12345678.00000001 Inexact Rounded --- desctructive subtraction (from remainder tests) +-- destructive subtraction (from remainder tests) -- +++ some of these will be off-by-one remainder vs remainderNear diff --git a/Lib/test/decimaltestdata/ddQuantize.decTest b/Lib/test/decimaltestdata/ddQuantize.decTest index 91776201694dd6..e1c5674d9ac042 100644 --- a/Lib/test/decimaltestdata/ddQuantize.decTest +++ b/Lib/test/decimaltestdata/ddQuantize.decTest @@ -462,7 +462,7 @@ ddqua520 quantize 1.234 1e359 -> 0E+359 Inexact Rounded ddqua521 quantize 123.456 1e359 -> 0E+359 Inexact Rounded ddqua522 quantize 1.234 1e359 -> 0E+359 Inexact Rounded ddqua523 quantize 123.456 1e359 -> 0E+359 Inexact Rounded --- next four are "won't fit" overfl +-- next four are "won't fit" overflow ddqua526 quantize 1.234 1e-299 -> NaN Invalid_operation ddqua527 quantize 123.456 1e-299 -> NaN Invalid_operation ddqua528 quantize 1.234 1e-299 -> NaN Invalid_operation diff --git a/Lib/test/decimaltestdata/ddRemainder.decTest b/Lib/test/decimaltestdata/ddRemainder.decTest index 5bd1e32d01ef03..b1866d39a2868c 100644 --- a/Lib/test/decimaltestdata/ddRemainder.decTest +++ b/Lib/test/decimaltestdata/ddRemainder.decTest @@ -422,7 +422,7 @@ ddrem757 remainder 1 sNaN -> NaN Invalid_operation ddrem758 remainder 1000 sNaN -> NaN Invalid_operation ddrem759 remainder Inf -sNaN -> -NaN Invalid_operation --- propaging NaNs +-- propagating NaNs ddrem760 remainder NaN1 NaN7 -> NaN1 ddrem761 remainder sNaN2 NaN8 -> NaN2 Invalid_operation ddrem762 remainder NaN3 sNaN9 -> NaN9 Invalid_operation diff --git a/Lib/test/decimaltestdata/ddRemainderNear.decTest b/Lib/test/decimaltestdata/ddRemainderNear.decTest index 6ba64ebafe9a18..bbe82ea3747d16 100644 --- a/Lib/test/decimaltestdata/ddRemainderNear.decTest +++ b/Lib/test/decimaltestdata/ddRemainderNear.decTest @@ -450,7 +450,7 @@ ddrmn757 remaindernear 1 sNaN -> NaN Invalid_operation ddrmn758 remaindernear 1000 sNaN -> NaN Invalid_operation ddrmn759 remaindernear Inf -sNaN -> -NaN Invalid_operation --- propaging NaNs +-- propagating NaNs ddrmn760 remaindernear NaN1 NaN7 -> NaN1 ddrmn761 remaindernear sNaN2 NaN8 -> NaN2 Invalid_operation ddrmn762 remaindernear NaN3 sNaN9 -> NaN9 Invalid_operation diff --git a/Lib/test/decimaltestdata/dqRemainder.decTest b/Lib/test/decimaltestdata/dqRemainder.decTest index bae8eae5269d43..e0aaca3747e229 100644 --- a/Lib/test/decimaltestdata/dqRemainder.decTest +++ b/Lib/test/decimaltestdata/dqRemainder.decTest @@ -418,7 +418,7 @@ dqrem757 remainder 1 sNaN -> NaN Invalid_operation dqrem758 remainder 1000 sNaN -> NaN Invalid_operation dqrem759 remainder Inf -sNaN -> -NaN Invalid_operation --- propaging NaNs +-- propagating NaNs dqrem760 remainder NaN1 NaN7 -> NaN1 dqrem761 remainder sNaN2 NaN8 -> NaN2 Invalid_operation dqrem762 remainder NaN3 sNaN9 -> NaN9 Invalid_operation diff --git a/Lib/test/decimaltestdata/dqRemainderNear.decTest b/Lib/test/decimaltestdata/dqRemainderNear.decTest index b850626fe4e04d..2c5c3f5074ea23 100644 --- a/Lib/test/decimaltestdata/dqRemainderNear.decTest +++ b/Lib/test/decimaltestdata/dqRemainderNear.decTest @@ -450,7 +450,7 @@ dqrmn757 remaindernear 1 sNaN -> NaN Invalid_operation dqrmn758 remaindernear 1000 sNaN -> NaN Invalid_operation dqrmn759 remaindernear Inf -sNaN -> -NaN Invalid_operation --- propaging NaNs +-- propagating NaNs dqrmn760 remaindernear NaN1 NaN7 -> NaN1 dqrmn761 remaindernear sNaN2 NaN8 -> NaN2 Invalid_operation dqrmn762 remaindernear NaN3 sNaN9 -> NaN9 Invalid_operation diff --git a/Lib/test/decimaltestdata/exp.decTest b/Lib/test/decimaltestdata/exp.decTest index 6a7af23b625530..e01d7a8f92db1f 100644 --- a/Lib/test/decimaltestdata/exp.decTest +++ b/Lib/test/decimaltestdata/exp.decTest @@ -28,7 +28,7 @@ rounding: half_even maxExponent: 384 minexponent: -383 --- basics (examples in specificiation, etc.) +-- basics (examples in specification, etc.) expx001 exp -Infinity -> 0 expx002 exp -10 -> 0.0000453999298 Inexact Rounded expx003 exp -1 -> 0.367879441 Inexact Rounded diff --git a/Lib/test/decimaltestdata/remainder.decTest b/Lib/test/decimaltestdata/remainder.decTest index 7a1061b1e6f0f8..4f59b332877afe 100644 --- a/Lib/test/decimaltestdata/remainder.decTest +++ b/Lib/test/decimaltestdata/remainder.decTest @@ -435,7 +435,7 @@ remx757 remainder 1 sNaN -> NaN Invalid_operation remx758 remainder 1000 sNaN -> NaN Invalid_operation remx759 remainder Inf -sNaN -> -NaN Invalid_operation --- propaging NaNs +-- propagating NaNs remx760 remainder NaN1 NaN7 -> NaN1 remx761 remainder sNaN2 NaN8 -> NaN2 Invalid_operation remx762 remainder NaN3 sNaN9 -> NaN9 Invalid_operation diff --git a/Lib/test/decimaltestdata/remainderNear.decTest b/Lib/test/decimaltestdata/remainderNear.decTest index b768b9e0cf6370..000b1424d8ab48 100644 --- a/Lib/test/decimaltestdata/remainderNear.decTest +++ b/Lib/test/decimaltestdata/remainderNear.decTest @@ -498,7 +498,7 @@ rmnx758 remaindernear 1000 sNaN -> NaN Invalid_operation rmnx759 remaindernear Inf sNaN -> NaN Invalid_operation rmnx760 remaindernear NaN sNaN -> NaN Invalid_operation --- propaging NaNs +-- propagating NaNs rmnx761 remaindernear NaN1 NaN7 -> NaN1 rmnx762 remaindernear sNaN2 NaN8 -> NaN2 Invalid_operation rmnx763 remaindernear NaN3 -sNaN9 -> -NaN9 Invalid_operation diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt index 65f54859e2a21d..f2649aa2d41fef 100644 --- a/Lib/test/exception_hierarchy.txt +++ b/Lib/test/exception_hierarchy.txt @@ -45,7 +45,6 @@ BaseException ├── StopAsyncIteration ├── StopIteration ├── SyntaxError - │ └── IncompleteInputError │ └── IndentationError │ └── TabError ├── SystemError diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 608b12bb6f2a38..bf9a71efbdbff9 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -87,38 +87,40 @@ The argument is a comma-separated list of words indicating the resources to test. Currently only the following are defined: - all - Enable all special resources. + all - Enable all special resources. - none - Disable all special resources (this is the default). + none - Disable all special resources (this is the default). - audio - Tests that use the audio device. (There are known - cases of broken audio drivers that can crash Python or - even the Linux kernel.) + audio - Tests that use the audio device. (There are known + cases of broken audio drivers that can crash Python or + even the Linux kernel.) - curses - Tests that use curses and will modify the terminal's - state and output modes. + curses - Tests that use curses and will modify the terminal's + state and output modes. - largefile - It is okay to run some test that may create huge - files. These tests can take a long time and may - consume >2 GiB of disk space temporarily. + largefile - It is okay to run some test that may create huge + files. These tests can take a long time and may + consume >2 GiB of disk space temporarily. - network - It is okay to run tests that use external network - resource, e.g. testing SSL support for sockets. + extralargefile - Like 'largefile', but even larger (and slower). - decimal - Test the decimal module against a large suite that - verifies compliance with standards. + network - It is okay to run tests that use external network + resource, e.g. testing SSL support for sockets. - cpu - Used for certain CPU-heavy tests. + decimal - Test the decimal module against a large suite that + verifies compliance with standards. - walltime - Long running but not CPU-bound tests. + cpu - Used for certain CPU-heavy tests. - subprocess Run all tests for the subprocess module. + walltime - Long running but not CPU-bound tests. - urlfetch - It is okay to download files required on testing. + subprocess Run all tests for the subprocess module. - gui - Run tests that require a running GUI. + urlfetch - It is okay to download files required on testing. - tzdata - Run tests that require timezone data. + gui - Run tests that require a running GUI. + + tzdata - Run tests that require timezone data. To enable all resources except one, use '-uall,-'. For example, to run all the tests except for the gui tests, give the @@ -148,7 +150,7 @@ def __init__(self, **kwargs) -> None: self.randomize = False self.fromfile = None self.fail_env_changed = False - self.use_resources = None + self.use_resources: list[str] = [] self.trace = False self.coverdir = 'coverage' self.runleaks = False @@ -164,6 +166,7 @@ def __init__(self, **kwargs) -> None: self.match_tests: TestFilter = [] self.pgo = False self.pgo_extended = False + self.tsan = False self.worker_json = None self.start = None self.timeout = None @@ -172,6 +175,8 @@ def __init__(self, **kwargs) -> None: self.fail_rerun = False self.tempdir = None self._add_python_opts = True + self.xmlpath = None + self.single_process = False super().__init__(**kwargs) @@ -305,6 +310,12 @@ def _create_parser(): group.add_argument('-j', '--multiprocess', metavar='PROCESSES', dest='use_mp', type=int, help='run PROCESSES processes at once') + group.add_argument('--single-process', action='store_true', + dest='single_process', + help='always run all tests sequentially in ' + 'a single process, ignore -jN option, ' + 'and failed tests are also rerun sequentially ' + 'in the same process') group.add_argument('-T', '--coverage', action='store_true', dest='trace', help='turn on code coverage tracing using the trace ' @@ -333,6 +344,8 @@ def _create_parser(): help='enable Profile Guided Optimization (PGO) training') group.add_argument('--pgo-extended', action='store_true', help='enable extended PGO training (slower training)') + group.add_argument('--tsan', dest='tsan', action='store_true', + help='run a subset of test cases that are proper for the TSAN test') group.add_argument('--fail-env-changed', action='store_true', help='if a test file alters the environment, mark ' 'the test as failed') @@ -392,8 +405,6 @@ def _parse_args(args, **kwargs): raise TypeError('%r is an invalid keyword argument ' 'for this function' % k) setattr(ns, k, v) - if ns.use_resources is None: - ns.use_resources = [] parser = _create_parser() # Issue #14191: argparse doesn't support "intermixed" positional and @@ -417,9 +428,7 @@ def _parse_args(args, **kwargs): # Continuous Integration (CI): common options for fast/slow CI modes if ns.slow_ci or ns.fast_ci: # Similar to options: - # - # -j0 --randomize --fail-env-changed --fail-rerun --rerun - # --slowest --verbose3 + # -j0 --randomize --fail-env-changed --rerun --slowest --verbose3 if ns.use_mp is None: ns.use_mp = 0 ns.randomize = True @@ -431,6 +440,10 @@ def _parse_args(args, **kwargs): else: ns._add_python_opts = False + # --singleprocess overrides -jN option + if ns.single_process: + ns.use_mp = None + # When both --slow-ci and --fast-ci options are present, # --slow-ci has the priority if ns.slow_ci: @@ -503,17 +516,19 @@ def _parse_args(args, **kwargs): ns.randomize = True if ns.verbose: ns.header = True + # When -jN option is used, a worker process does not use --verbose3 # and so -R 3:3 -jN --verbose3 just works as expected: there is no false # alarm about memory leak. if ns.huntrleaks and ns.verbose3 and ns.use_mp is None: - ns.verbose3 = False # run_single_test() replaces sys.stdout with io.StringIO if verbose3 # is true. In this case, huntrleaks sees an write into StringIO as # a memory leak, whereas it is not (gh-71290). + ns.verbose3 = False print("WARNING: Disable --verbose3 because it's incompatible with " "--huntrleaks without -jN option", file=sys.stderr) + if ns.forever: # --forever implies --failfast ns.failfast = True diff --git a/Lib/test/libregrtest/filter.py b/Lib/test/libregrtest/filter.py index 817624d79e9263..41372e427ffd03 100644 --- a/Lib/test/libregrtest/filter.py +++ b/Lib/test/libregrtest/filter.py @@ -27,6 +27,11 @@ def _is_full_match_test(pattern): return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) +def get_match_tests(): + global _test_patterns + return _test_patterns + + def set_match_tests(patterns): global _test_matchers, _test_patterns diff --git a/Lib/test/libregrtest/findtests.py b/Lib/test/libregrtest/findtests.py index 4ac95e23a56b8f..f01c1240774707 100644 --- a/Lib/test/libregrtest/findtests.py +++ b/Lib/test/libregrtest/findtests.py @@ -1,6 +1,7 @@ import os import sys import unittest +from collections.abc import Container from test import support @@ -34,7 +35,7 @@ def findtestdir(path: StrPath | None = None) -> StrPath: return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir -def findtests(*, testdir: StrPath | None = None, exclude=(), +def findtests(*, testdir: StrPath | None = None, exclude: Container[str] = (), split_test_dirs: set[TestName] = SPLITTESTDIRS, base_mod: str = "") -> TestList: """Return a list of all applicable test modules.""" @@ -60,8 +61,9 @@ def findtests(*, testdir: StrPath | None = None, exclude=(), return sorted(tests) -def split_test_packages(tests, *, testdir: StrPath | None = None, exclude=(), - split_test_dirs=SPLITTESTDIRS): +def split_test_packages(tests, *, testdir: StrPath | None = None, + exclude: Container[str] = (), + split_test_dirs=SPLITTESTDIRS) -> list[TestName]: testdir = findtestdir(testdir) splitted = [] for name in tests: @@ -75,9 +77,9 @@ def split_test_packages(tests, *, testdir: StrPath | None = None, exclude=(), return splitted -def _list_cases(suite): +def _list_cases(suite: unittest.TestSuite) -> None: for test in suite: - if isinstance(test, unittest.loader._FailedTest): + if isinstance(test, unittest.loader._FailedTest): # type: ignore[attr-defined] continue if isinstance(test, unittest.TestSuite): _list_cases(test) @@ -87,7 +89,7 @@ def _list_cases(suite): def list_cases(tests: TestTuple, *, match_tests: TestFilter | None = None, - test_dir: StrPath | None = None): + test_dir: StrPath | None = None) -> None: support.verbose = False set_match_tests(match_tests) diff --git a/Lib/test/libregrtest/logger.py b/Lib/test/libregrtest/logger.py index a125706927393c..fa1d4d575c8fd4 100644 --- a/Lib/test/libregrtest/logger.py +++ b/Lib/test/libregrtest/logger.py @@ -43,7 +43,10 @@ def log(self, line: str = '') -> None: def get_load_avg(self) -> float | None: if hasattr(os, 'getloadavg'): - return os.getloadavg()[0] + try: + return os.getloadavg()[0] + except OSError: + pass if self.win_load_tracker is not None: return self.win_load_tracker.getloadavg() return None diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 126daca388fd7f..dcbcc6790c68d8 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -6,6 +6,8 @@ import sysconfig import time import trace +from _colorize import get_colors # type: ignore[import-not-found] +from typing import NoReturn from test.support import os_helper, MS_WINDOWS, flush_std_streams @@ -18,6 +20,7 @@ from .runtests import RunTests, HuntRefleak from .setup import setup_process, setup_test_dir from .single import run_single_test, PROGRESS_MIN_TIME +from .tsan import setup_tsan_tests from .utils import ( StrPath, StrJSON, TestName, TestList, TestTuple, TestFilter, strip_py_suffix, count, format_duration, @@ -56,6 +59,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.quiet: bool = ns.quiet self.pgo: bool = ns.pgo self.pgo_extended: bool = ns.pgo_extended + self.tsan: bool = ns.tsan # Test results self.results: TestResults = TestResults() @@ -86,12 +90,13 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.cmdline_args: TestList = ns.args # Workers - if ns.use_mp is None: - num_workers = 0 # run sequentially + self.single_process: bool = ns.single_process + if self.single_process or ns.use_mp is None: + num_workers = 0 # run sequentially in a single process elif ns.use_mp <= 0: - num_workers = -1 # use the number of CPUs + num_workers = -1 # run in parallel, use the number of CPUs else: - num_workers = ns.use_mp + num_workers = ns.use_mp # run in parallel self.num_workers: int = num_workers self.worker_json: StrJSON | None = ns.worker_json @@ -119,7 +124,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.python_cmd = None self.coverage: bool = ns.trace self.coverage_dir: StrPath | None = ns.coverdir - self.tmp_dir: StrPath | None = ns.tempdir + self._tmp_dir: StrPath | None = ns.tempdir # Randomize self.randomize: bool = ns.randomize @@ -151,10 +156,12 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.next_single_test: TestName | None = None self.next_single_filename: StrPath | None = None - def log(self, line=''): + def log(self, line: str = '') -> None: self.logger.log(line) def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList | None]: + if tests is None: + tests = [] if self.single_test_run: self.next_single_filename = os.path.join(self.tmp_dir, 'pynexttest') try: @@ -183,6 +190,9 @@ def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList # add default PGO tests if no tests are specified setup_pgo_tests(self.cmdline_args, self.pgo_extended) + if self.tsan: + setup_tsan_tests(self.cmdline_args) + exclude_tests = set() if self.exclude: for arg in self.cmdline_args: @@ -224,13 +234,13 @@ def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList return (tuple(selected), tests) @staticmethod - def list_tests(tests: TestTuple): + def list_tests(tests: TestTuple) -> None: for name in tests: print(name) - def _rerun_failed_tests(self, runtests: RunTests): + def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: # Configure the runner to re-run tests - if self.num_workers == 0: + if self.num_workers == 0 and not self.single_process: # Always run tests in fresh processes to have more deterministic # initial state. Don't re-run tests in parallel but limit to a # single worker process to have side effects (on the system load @@ -240,7 +250,6 @@ def _rerun_failed_tests(self, runtests: RunTests): tests, match_tests_dict = self.results.prepare_rerun() # Re-run failed tests - self.log(f"Re-running {len(tests)} failed tests in verbose mode in subprocesses") runtests = runtests.copy( tests=tests, rerun=True, @@ -250,10 +259,21 @@ def _rerun_failed_tests(self, runtests: RunTests): match_tests_dict=match_tests_dict, output_on_failure=False) self.logger.set_tests(runtests) - self._run_tests_mp(runtests, self.num_workers) + + msg = f"Re-running {len(tests)} failed tests in verbose mode" + if not self.single_process: + msg = f"{msg} in subprocesses" + self.log(msg) + self._run_tests_mp(runtests, self.num_workers) + else: + self.log(msg) + self.run_tests_sequentially(runtests) return runtests - def rerun_failed_tests(self, runtests: RunTests): + def rerun_failed_tests(self, runtests: RunTests) -> None: + ansi = get_colors() + red, reset = ansi.BOLD_RED, ansi.RESET + if self.python_cmd: # Temp patch for https://github.com/python/cpython/issues/94052 self.log( @@ -268,7 +288,10 @@ def rerun_failed_tests(self, runtests: RunTests): rerun_runtests = self._rerun_failed_tests(runtests) if self.results.bad: - print(count(len(self.results.bad), 'test'), "failed again:") + print( + f"{red}{count(len(self.results.bad), 'test')} " + f"failed again:{reset}" + ) printlist(self.results.bad) self.display_result(rerun_runtests) @@ -322,7 +345,7 @@ def run_bisect(self, runtests: RunTests) -> None: if not self._run_bisect(runtests, name, progress): return - def display_result(self, runtests): + def display_result(self, runtests: RunTests) -> None: # If running the test suite for PGO then no one cares about results. if runtests.pgo: return @@ -344,9 +367,7 @@ def run_test( namespace = dict(locals()) tracer.runctx(cmd, globals=globals(), locals=namespace) result = namespace['result'] - # Mypy doesn't know about this attribute yet, - # but it will do soon: https://github.com/python/typeshed/pull/11091 - result.covered_lines = list(tracer.counts) # type: ignore[attr-defined] + result.covered_lines = list(tracer.counts) else: result = run_single_test(test_name, runtests) @@ -354,7 +375,7 @@ def run_test( return result - def run_tests_sequentially(self, runtests) -> None: + def run_tests_sequentially(self, runtests: RunTests) -> None: if self.coverage: tracer = trace.Trace(trace=False, count=True) else: @@ -367,7 +388,7 @@ def run_tests_sequentially(self, runtests) -> None: tests = count(jobs, 'test') else: tests = 'tests' - msg = f"Run {tests} sequentially" + msg = f"Run {tests} sequentially in a single process" if runtests.timeout: msg += " (timeout: %s)" % format_duration(runtests.timeout) self.log(msg) @@ -411,7 +432,7 @@ def run_tests_sequentially(self, runtests) -> None: if previous_test: print(previous_test) - def get_state(self): + def get_state(self) -> str: state = self.results.get_state(self.fail_env_changed) if self.first_state: state = f'{self.first_state} then {state}' @@ -441,7 +462,12 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: if self.junit_filename: self.results.write_junit(self.junit_filename) - def display_summary(self): + def display_summary(self) -> None: + if self.first_runtests is None: + raise ValueError( + "Should never call `display_summary()` before calling `_run_test()`" + ) + duration = time.perf_counter() - self.logger.start_time filtered = bool(self.match_tests) @@ -455,7 +481,7 @@ def display_summary(self): state = self.get_state() print(f"Result: {state}") - def create_run_tests(self, tests: TestTuple): + def create_run_tests(self, tests: TestTuple) -> RunTests: return RunTests( tests, fail_fast=self.fail_fast, @@ -511,7 +537,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: setup_process() if (runtests.hunt_refleak is not None) and (not self.num_workers): - # gh-109739: WindowsLoadTracker thread interfers with refleak check + # gh-109739: WindowsLoadTracker thread interferes with refleak check use_load_tracker = False else: # WindowsLoadTracker is only needed on Windows @@ -579,6 +605,7 @@ def _add_cross_compile_opts(self, regrtest_opts): '_PYTHON_PROJECT_BASE', '_PYTHON_HOST_PLATFORM', '_PYTHON_SYSCONFIGDATA_NAME', + "_PYTHON_SYSCONFIGDATA_PATH", 'PYTHONPATH' } old_environ = os.environ @@ -592,7 +619,7 @@ def _add_cross_compile_opts(self, regrtest_opts): keep_environ = True if cross_compile and hostrunner: - if self.num_workers == 0: + if self.num_workers == 0 and not self.single_process: # For now use only two cores for cross-compiled builds; # hostrunner can be expensive. regrtest_opts.extend(['-j', '2']) @@ -662,9 +689,9 @@ def _execute_python(self, cmd, environ): f"Command: {cmd_text}") # continue executing main() - def _add_python_opts(self): - python_opts = [] - regrtest_opts = [] + def _add_python_opts(self) -> None: + python_opts: list[str] = [] + regrtest_opts: list[str] = [] environ, keep_environ = self._add_cross_compile_opts(regrtest_opts) if self.ci_mode: @@ -695,9 +722,17 @@ def _init(self): strip_py_suffix(self.cmdline_args) - self.tmp_dir = get_temp_dir(self.tmp_dir) + self._tmp_dir = get_temp_dir(self._tmp_dir) + + @property + def tmp_dir(self) -> StrPath: + if self._tmp_dir is None: + raise ValueError( + "Should never use `.tmp_dir` before calling `.main()`" + ) + return self._tmp_dir - def main(self, tests: TestList | None = None): + def main(self, tests: TestList | None = None) -> NoReturn: if self.want_add_python_opts: self._add_python_opts() @@ -726,7 +761,7 @@ def main(self, tests: TestList | None = None): sys.exit(exitcode) -def main(tests=None, _add_python_opts=False, **kwargs): +def main(tests=None, _add_python_opts=False, **kwargs) -> NoReturn: """Run the Python suite.""" ns = _parse_args(sys.argv[1:], **kwargs) Regrtest(ns, _add_python_opts=_add_python_opts).main(tests=tests) diff --git a/Lib/test/libregrtest/mypy.ini b/Lib/test/libregrtest/mypy.ini index 22c7c7a9acef14..3fa9afcb7a4a8c 100644 --- a/Lib/test/libregrtest/mypy.ini +++ b/Lib/test/libregrtest/mypy.ini @@ -15,19 +15,12 @@ strict = True # Various stricter settings that we can't yet enable # Try to enable these in the following order: -disallow_any_generics = False disallow_incomplete_defs = False disallow_untyped_calls = False disallow_untyped_defs = False check_untyped_defs = False warn_return_any = False -disable_error_code = return - -# Enable --strict-optional for these ASAP: -[mypy-Lib.test.libregrtest.main.*,Lib.test.libregrtest.run_workers.*] -strict_optional = False - # Various internal modules that typeshed deliberately doesn't have stubs for: [mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*] ignore_missing_imports = True diff --git a/Lib/test/libregrtest/pgo.py b/Lib/test/libregrtest/pgo.py index e3a6927be5db1d..04803ddf64453c 100644 --- a/Lib/test/libregrtest/pgo.py +++ b/Lib/test/libregrtest/pgo.py @@ -19,7 +19,6 @@ 'test_datetime', 'test_decimal', 'test_difflib', - 'test_embed', 'test_float', 'test_fstring', 'test_functools', @@ -50,7 +49,7 @@ 'test_xml_etree_c', ] -def setup_pgo_tests(cmdline_args, pgo_extended: bool): +def setup_pgo_tests(cmdline_args, pgo_extended: bool) -> None: if not cmdline_args and not pgo_extended: # run default set of tests for PGO training cmdline_args[:] = PGO_TESTS[:] diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index f582c0d3e7ff13..d0d1c8cdc9a11b 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -1,3 +1,4 @@ +import os import sys import warnings from inspect import isabstract @@ -23,6 +24,30 @@ def _get_dump(cls): cls._abc_negative_cache, cls._abc_negative_cache_version) +def save_support_xml(filename): + if support.junit_xml_list is None: + return + + import pickle + with open(filename, 'xb') as fp: + pickle.dump(support.junit_xml_list, fp) + support.junit_xml_list = None + + +def restore_support_xml(filename): + try: + fp = open(filename, 'rb') + except FileNotFoundError: + return + + import pickle + with fp: + xml_list = pickle.load(fp) + os.unlink(filename) + + support.junit_xml_list = xml_list + + def runtest_refleak(test_name, test_func, hunt_refleak: HuntRefleak, quiet: bool): @@ -85,7 +110,7 @@ def get_pooled_int(value): getunicodeinternedsize = sys.getunicodeinternedsize fd_count = os_helper.fd_count # initialize variables to make pyflakes quiet - rc_before = alloc_before = fd_before = interned_before = 0 + rc_before = alloc_before = fd_before = interned_immortal_before = 0 if not quiet: print("beginning", repcount, "repetitions. Showing number of leaks " @@ -95,18 +120,20 @@ def get_pooled_int(value): numbers = numbers[:warmups] + ':' + numbers[warmups:] print(numbers, file=sys.stderr, flush=True) - results = None + xml_filename = 'refleak-xml.tmp' + result = None dash_R_cleanup(fs, ps, pic, zdc, abcs) - support.gc_collect() for i in rep_range: + support.gc_collect() current = refleak_helper._hunting_for_refleaks refleak_helper._hunting_for_refleaks = True try: - results = test_func() + result = test_func() finally: refleak_helper._hunting_for_refleaks = current + save_support_xml(xml_filename) dash_R_cleanup(fs, ps, pic, zdc, abcs) support.gc_collect() @@ -114,9 +141,11 @@ def get_pooled_int(value): # Also, readjust the reference counts and alloc blocks by ignoring # any strings that might have been interned during test_func. These # strings will be deallocated at runtime shutdown - interned_after = getunicodeinternedsize() - alloc_after = getallocatedblocks() - interned_after - rc_after = gettotalrefcount() - interned_after * 2 + interned_immortal_after = getunicodeinternedsize( + # Use an internal-only keyword argument that mypy doesn't know yet + _only_immortal=True) # type: ignore[call-arg] + alloc_after = getallocatedblocks() - interned_immortal_after + rc_after = gettotalrefcount() fd_after = fd_count() rc_deltas[i] = get_pooled_int(rc_after - rc_before) @@ -143,7 +172,9 @@ def get_pooled_int(value): alloc_before = alloc_after rc_before = rc_after fd_before = fd_after - interned_before = interned_after + interned_immortal_before = interned_immortal_after + + restore_support_xml(xml_filename) if not quiet: print(file=sys.stderr) @@ -189,7 +220,7 @@ def check_fd_deltas(deltas): failed = True else: print(' (this is fine)', file=sys.stderr, flush=True) - return (failed, results) + return (failed, result) def dash_R_cleanup(fs, ps, pic, zdc, abcs): @@ -211,14 +242,17 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): zipimport._zip_directory_cache.update(zdc) # Clear ABC registries, restoring previously saved ABC registries. - # ignore deprecation warning for collections.abc.ByteString abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] abs_classes = filter(isabstract, abs_classes) for abc in abs_classes: for obj in abc.__subclasses__() + [abc]: - for ref in abcs.get(obj, set()): - if ref() is not None: - obj.register(ref()) + refs = abcs.get(obj, None) + if refs is not None: + obj._abc_registry_clear() + for ref in refs: + subclass = ref() + if subclass is not None: + obj.register(subclass) obj._abc_caches_clear() # Clear caches @@ -228,7 +262,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): sys._clear_internal_caches() -def warm_caches(): +def warm_caches() -> None: # char cache s = bytes(range(256)) for i in range(256): diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 74eae40440435d..daf7624366ee20 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,5 +1,6 @@ import dataclasses import json +from _colorize import get_colors # type: ignore[import-not-found] from typing import Any from .utils import ( @@ -105,53 +106,71 @@ def is_failed(self, fail_env_changed: bool) -> bool: return State.is_failed(self.state) def _format_failed(self): + ansi = get_colors() + red, reset = ansi.BOLD_RED, ansi.RESET if self.errors and self.failures: le = len(self.errors) lf = len(self.failures) error_s = "error" + ("s" if le > 1 else "") failure_s = "failure" + ("s" if lf > 1 else "") - return f"{self.test_name} failed ({le} {error_s}, {lf} {failure_s})" + return ( + f"{red}{self.test_name} failed " + f"({le} {error_s}, {lf} {failure_s}){reset}" + ) if self.errors: le = len(self.errors) error_s = "error" + ("s" if le > 1 else "") - return f"{self.test_name} failed ({le} {error_s})" + return f"{red}{self.test_name} failed ({le} {error_s}){reset}" if self.failures: lf = len(self.failures) failure_s = "failure" + ("s" if lf > 1 else "") - return f"{self.test_name} failed ({lf} {failure_s})" + return f"{red}{self.test_name} failed ({lf} {failure_s}){reset}" - return f"{self.test_name} failed" + return f"{red}{self.test_name} failed{reset}" def __str__(self) -> str: + ansi = get_colors() + green = ansi.GREEN + red = ansi.BOLD_RED + reset = ansi.RESET + yellow = ansi.YELLOW + match self.state: case State.PASSED: - return f"{self.test_name} passed" + return f"{green}{self.test_name} passed{reset}" case State.FAILED: - return self._format_failed() + return f"{red}{self._format_failed()}{reset}" case State.SKIPPED: - return f"{self.test_name} skipped" + return f"{yellow}{self.test_name} skipped{reset}" case State.UNCAUGHT_EXC: - return f"{self.test_name} failed (uncaught exception)" + return ( + f"{red}{self.test_name} failed (uncaught exception){reset}" + ) case State.REFLEAK: - return f"{self.test_name} failed (reference leak)" + return f"{red}{self.test_name} failed (reference leak){reset}" case State.ENV_CHANGED: - return f"{self.test_name} failed (env changed)" + return f"{red}{self.test_name} failed (env changed){reset}" case State.RESOURCE_DENIED: - return f"{self.test_name} skipped (resource denied)" + return f"{yellow}{self.test_name} skipped (resource denied){reset}" case State.INTERRUPTED: - return f"{self.test_name} interrupted" + return f"{yellow}{self.test_name} interrupted{reset}" case State.WORKER_FAILED: - return f"{self.test_name} worker non-zero exit code" + return ( + f"{red}{self.test_name} worker non-zero exit code{reset}" + ) case State.WORKER_BUG: - return f"{self.test_name} worker bug" + return f"{red}{self.test_name} worker bug{reset}" case State.DID_NOT_RUN: - return f"{self.test_name} ran no tests" + return f"{yellow}{self.test_name} ran no tests{reset}" case State.TIMEOUT: + assert self.duration is not None, "self.duration is None" return f"{self.test_name} timed out ({format_duration(self.duration)})" case _: - raise ValueError("unknown result state: {state!r}") + raise ValueError( + f"{red}unknown result state: {{state!r}}{reset}" + ) def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 85c82052eae19b..a35934fc2c9ca8 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -1,5 +1,7 @@ import sys import trace +from _colorize import get_colors # type: ignore[import-not-found] +from typing import TYPE_CHECKING from .runtests import RunTests from .result import State, TestResult, TestStats, Location @@ -7,6 +9,9 @@ StrPath, TestName, TestTuple, TestList, FilterDict, printlist, count, format_duration) +if TYPE_CHECKING: + from xml.etree.ElementTree import Element + # Python uses exit code 1 when an exception is not caught # argparse.ArgumentParser.error() uses exit code 2 @@ -18,7 +23,7 @@ class TestResults: - def __init__(self): + def __init__(self) -> None: self.bad: TestList = [] self.good: TestList = [] self.rerun_bad: TestList = [] @@ -34,44 +39,49 @@ def __init__(self): self.test_times: list[tuple[float, TestName]] = [] self.stats = TestStats() # used by --junit-xml - self.testsuite_xml: list = [] + self.testsuite_xml: list['Element'] = [] # used by -T with -j self.covered_lines: set[Location] = set() - def is_all_good(self): + def is_all_good(self) -> bool: return (not self.bad and not self.skipped and not self.interrupted and not self.worker_bug) - def get_executed(self): + def get_executed(self) -> set[TestName]: return (set(self.good) | set(self.bad) | set(self.skipped) | set(self.resource_denied) | set(self.env_changed) | set(self.run_no_tests)) - def no_tests_run(self): + def no_tests_run(self) -> bool: return not any((self.good, self.bad, self.skipped, self.interrupted, self.env_changed)) - def get_state(self, fail_env_changed): + def get_state(self, fail_env_changed: bool) -> str: state = [] + ansi = get_colors() + green = ansi.GREEN + red = ansi.BOLD_RED + reset = ansi.RESET + yellow = ansi.YELLOW if self.bad: - state.append("FAILURE") + state.append(f"{red}FAILURE{reset}") elif fail_env_changed and self.env_changed: - state.append("ENV CHANGED") + state.append(f"{yellow}ENV CHANGED{reset}") elif self.no_tests_run(): - state.append("NO TESTS RAN") + state.append(f"{yellow}NO TESTS RAN{reset}") if self.interrupted: - state.append("INTERRUPTED") + state.append(f"{yellow}INTERRUPTED{reset}") if self.worker_bug: - state.append("WORKER BUG") + state.append(f"{red}WORKER BUG{reset}") if not state: - state.append("SUCCESS") + state.append(f"{green}SUCCESS{reset}") return ', '.join(state) - def get_exitcode(self, fail_env_changed, fail_rerun): + def get_exitcode(self, fail_env_changed: bool, fail_rerun: bool) -> int: exitcode = 0 if self.bad: exitcode = EXITCODE_BAD_TEST @@ -87,7 +97,7 @@ def get_exitcode(self, fail_env_changed, fail_rerun): exitcode = EXITCODE_BAD_TEST return exitcode - def accumulate_result(self, result: TestResult, runtests: RunTests): + def accumulate_result(self, result: TestResult, runtests: RunTests) -> None: test_name = result.test_name rerun = runtests.rerun fail_env_changed = runtests.fail_env_changed @@ -135,7 +145,7 @@ def get_coverage_results(self) -> trace.CoverageResults: counts = {loc: 1 for loc in self.covered_lines} return trace.CoverageResults(counts=counts) - def need_rerun(self): + def need_rerun(self) -> bool: return bool(self.rerun_results) def prepare_rerun(self, *, clear: bool = True) -> tuple[TestTuple, FilterDict]: @@ -158,7 +168,7 @@ def prepare_rerun(self, *, clear: bool = True) -> tuple[TestTuple, FilterDict]: return (tuple(tests), match_tests_dict) - def add_junit(self, xml_data: list[str]): + def add_junit(self, xml_data: list[str]) -> None: import xml.etree.ElementTree as ET for e in xml_data: try: @@ -167,7 +177,7 @@ def add_junit(self, xml_data: list[str]): print(xml_data, file=sys.__stderr__) raise - def write_junit(self, filename: StrPath): + def write_junit(self, filename: StrPath) -> None: if not self.testsuite_xml: # Don't create empty XML file return @@ -192,28 +202,52 @@ def write_junit(self, filename: StrPath): for s in ET.tostringlist(root): f.write(s) - def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool): + def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool) -> None: + ansi = get_colors() + green = ansi.GREEN + red = ansi.BOLD_RED + reset = ansi.RESET + yellow = ansi.YELLOW + if print_slowest: self.test_times.sort(reverse=True) print() - print("10 slowest tests:") + print(f"{yellow}10 slowest tests:{reset}") for test_time, test in self.test_times[:10]: - print("- %s: %s" % (test, format_duration(test_time))) + print(f"- {test}: {format_duration(test_time)}") all_tests = [] omitted = set(tests) - self.get_executed() # less important - all_tests.append((omitted, "test", "{} omitted:")) + all_tests.append( + (sorted(omitted), "test", f"{yellow}{{}} omitted:{reset}") + ) if not quiet: - all_tests.append((self.skipped, "test", "{} skipped:")) - all_tests.append((self.resource_denied, "test", "{} skipped (resource denied):")) - all_tests.append((self.run_no_tests, "test", "{} run no tests:")) + all_tests.append( + (self.skipped, "test", f"{yellow}{{}} skipped:{reset}") + ) + all_tests.append( + ( + self.resource_denied, + "test", + f"{yellow}{{}} skipped (resource denied):{reset}", + ) + ) + all_tests.append( + (self.run_no_tests, "test", f"{yellow}{{}} run no tests:{reset}") + ) # more important - all_tests.append((self.env_changed, "test", "{} altered the execution environment (env changed):")) - all_tests.append((self.rerun, "re-run test", "{}:")) - all_tests.append((self.bad, "test", "{} failed:")) + all_tests.append( + ( + self.env_changed, + "test", + f"{yellow}{{}} altered the execution environment (env changed):{reset}", + ) + ) + all_tests.append((self.rerun, "re-run test", f"{yellow}{{}}:{reset}")) + all_tests.append((self.bad, "test", f"{red}{{}} failed:{reset}")) for tests_list, count_text, title_format in all_tests: if tests_list: @@ -225,26 +259,29 @@ def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool): if self.good and not quiet: print() text = count(len(self.good), "test") - text = f"{text} OK." - if (self.is_all_good() and len(self.good) > 1): + text = f"{green}{text} OK.{reset}" + if self.is_all_good() and len(self.good) > 1: text = f"All {text}" print(text) if self.interrupted: print() - print("Test suite interrupted by signal SIGINT.") + print(f"{yellow}Test suite interrupted by signal SIGINT.{reset}") - def display_summary(self, first_runtests: RunTests, filtered: bool): + def display_summary(self, first_runtests: RunTests, filtered: bool) -> None: # Total tests + ansi = get_colors() + red, reset, yellow = ansi.RED, ansi.RESET, ansi.YELLOW + stats = self.stats text = f'run={stats.tests_run:,}' if filtered: text = f"{text} (filtered)" report = [text] if stats.failures: - report.append(f'failures={stats.failures:,}') + report.append(f'{red}failures={stats.failures:,}{reset}') if stats.skipped: - report.append(f'skipped={stats.skipped:,}') + report.append(f'{yellow}skipped={stats.skipped:,}{reset}') print(f"Total tests: {' '.join(report)}") # Total test files @@ -259,14 +296,14 @@ def display_summary(self, first_runtests: RunTests, filtered: bool): if filtered: text = f"{text} (filtered)" report = [text] - for name, tests in ( - ('failed', self.bad), - ('env_changed', self.env_changed), - ('skipped', self.skipped), - ('resource_denied', self.resource_denied), - ('rerun', self.rerun), - ('run_no_tests', self.run_no_tests), + for name, tests, color in ( + ('failed', self.bad, red), + ('env_changed', self.env_changed, yellow), + ('skipped', self.skipped, yellow), + ('resource_denied', self.resource_denied, yellow), + ('rerun', self.rerun, yellow), + ('run_no_tests', self.run_no_tests, yellow), ): if tests: - report.append(f'{name}={len(tests)}') + report.append(f'{color}{name}={len(tests)}{reset}') print(f"Total test files: {' '.join(report)}") diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 18a0342f0611cf..424085a0050eb5 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -22,7 +22,7 @@ from .single import PROGRESS_MIN_TIME from .utils import ( StrPath, TestName, - format_duration, print_warning, count, plural, get_signal_name) + format_duration, print_warning, count, plural) from .worker import create_worker_process, USE_PROCESS_GROUP if MS_WINDOWS: @@ -79,8 +79,12 @@ class MultiprocessResult: err_msg: str | None = None +class WorkerThreadExited: + """Indicates that a worker thread has exited""" + ExcStr = str QueueOutput = tuple[Literal[False], MultiprocessResult] | tuple[Literal[True], ExcStr] +QueueContent = QueueOutput | WorkerThreadExited class ExitThread(Exception): @@ -98,6 +102,9 @@ def __init__(self, super().__init__() +_NOT_RUNNING = "" + + class WorkerThread(threading.Thread): def __init__(self, worker_id: int, runner: "RunWorkers") -> None: super().__init__() @@ -107,8 +114,8 @@ def __init__(self, worker_id: int, runner: "RunWorkers") -> None: self.output = runner.output self.timeout = runner.worker_timeout self.log = runner.log - self.test_name: TestName | None = None - self.start_time: float | None = None + self.test_name = _NOT_RUNNING + self.start_time = time.monotonic() self._popen: subprocess.Popen[str] | None = None self._killed = False self._stopped = False @@ -125,7 +132,7 @@ def __repr__(self) -> str: popen = self._popen if popen is not None: dt = time.monotonic() - self.start_time - info.extend((f'pid={self._popen.pid}', + info.extend((f'pid={popen.pid}', f'time={format_duration(dt)}')) return '<%s>' % ' '.join(info) @@ -138,14 +145,20 @@ def _kill(self) -> None: return self._killed = True - if USE_PROCESS_GROUP: + use_killpg = USE_PROCESS_GROUP + if use_killpg: + parent_sid = os.getsid(0) + sid = os.getsid(popen.pid) + use_killpg = (sid != parent_sid) + + if use_killpg: what = f"{self} process group" else: what = f"{self} process" print(f"Kill {what}", file=sys.stderr, flush=True) try: - if USE_PROCESS_GROUP: + if use_killpg: os.killpg(popen.pid, signal.SIGKILL) else: popen.kill() @@ -201,6 +214,7 @@ def _run_process(self, runtests: WorkerRunTests, output_fd: int, # on reading closed stdout raise ExitThread raise + return None except: self._kill() raise @@ -209,7 +223,7 @@ def _run_process(self, runtests: WorkerRunTests, output_fd: int, self._popen = None def create_stdout(self, stack: contextlib.ExitStack) -> TextIO: - """Create stdout temporay file (file descriptor).""" + """Create stdout temporary file (file descriptor).""" if MS_WINDOWS: # gh-95027: When stdout is not a TTY, Python uses the ANSI code @@ -356,7 +370,7 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: err_msg=None, state=State.TIMEOUT) if retcode != 0: - name = get_signal_name(retcode) + name = support.get_signal_name(retcode) if name: retcode = f"{retcode} ({name})" raise WorkerError(self.test_name, f"Exit code {retcode}", stdout, @@ -376,8 +390,8 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: def run(self) -> None: fail_fast = self.runtests.fail_fast fail_env_changed = self.runtests.fail_env_changed - while not self._stopped: - try: + try: + while not self._stopped: try: test_name = next(self.pending) except StopIteration: @@ -390,20 +404,24 @@ def run(self) -> None: except WorkerError as exc: mp_result = exc.mp_result finally: - self.test_name = None + self.test_name = _NOT_RUNNING mp_result.result.duration = time.monotonic() - self.start_time self.output.put((False, mp_result)) if mp_result.result.must_stop(fail_fast, fail_env_changed): break - except ExitThread: - break - except BaseException: - self.output.put((True, traceback.format_exc())) - break + except ExitThread: + pass + except BaseException: + self.output.put((True, traceback.format_exc())) + finally: + self.output.put(WorkerThreadExited()) def _wait_completed(self) -> None: popen = self._popen + # only needed for mypy: + if popen is None: + raise ValueError("Should never access `._popen` before calling `.run()`") try: popen.wait(WAIT_COMPLETED_TIMEOUT) @@ -439,7 +457,7 @@ def get_running(workers: list[WorkerThread]) -> str | None: running: list[str] = [] for worker in workers: test_name = worker.test_name - if not test_name: + if test_name == _NOT_RUNNING: continue dt = time.monotonic() - worker.start_time if dt >= PROGRESS_MIN_TIME: @@ -458,8 +476,9 @@ def __init__(self, num_workers: int, runtests: RunTests, self.log = logger.log self.display_progress = logger.display_progress self.results: TestResults = results + self.live_worker_count = 0 - self.output: queue.Queue[QueueOutput] = queue.Queue() + self.output: queue.Queue[QueueContent] = queue.Queue() tests_iter = runtests.iter_tests() self.pending = MultiprocessIterator(tests_iter) self.timeout = runtests.timeout @@ -470,7 +489,7 @@ def __init__(self, num_workers: int, runtests: RunTests, self.worker_timeout: float | None = min(self.timeout * 1.5, self.timeout + 5 * 60) else: self.worker_timeout = None - self.workers: list[WorkerThread] | None = None + self.workers: list[WorkerThread] = [] jobs = self.runtests.get_jobs() if jobs is not None: @@ -490,13 +509,14 @@ def start_workers(self) -> None: processes = plural(nworkers, "process", "processes") msg = (f"Run {tests} in parallel using " f"{nworkers} worker {processes}") - if self.timeout: + if self.timeout and self.worker_timeout is not None: msg += (" (timeout: %s, worker timeout: %s)" % (format_duration(self.timeout), format_duration(self.worker_timeout))) self.log(msg) for worker in self.workers: worker.start() + self.live_worker_count += 1 def stop_workers(self) -> None: start_time = time.monotonic() @@ -511,14 +531,18 @@ def _get_result(self) -> QueueOutput | None: # bpo-46205: check the status of workers every iteration to avoid # waiting forever on an empty queue. - while any(worker.is_alive() for worker in self.workers): + while self.live_worker_count > 0: if use_faulthandler: faulthandler.dump_traceback_later(MAIN_PROCESS_TIMEOUT, exit=True) # wait for a thread try: - return self.output.get(timeout=PROGRESS_UPDATE) + result = self.output.get(timeout=PROGRESS_UPDATE) + if isinstance(result, WorkerThreadExited): + self.live_worker_count -= 1 + continue + return result except queue.Empty: pass @@ -527,12 +551,7 @@ def _get_result(self) -> QueueOutput | None: running = get_running(self.workers) if running: self.log(running) - - # all worker threads are done: consume pending results - try: - return self.output.get(timeout=0) - except queue.Empty: - return None + return None def display_result(self, mp_result: MultiprocessResult) -> None: result = mp_result.result @@ -542,7 +561,7 @@ def display_result(self, mp_result: MultiprocessResult) -> None: if mp_result.err_msg: # WORKER_BUG text += ' (%s)' % mp_result.err_msg - elif (result.duration >= PROGRESS_MIN_TIME and not pgo): + elif (result.duration and result.duration >= PROGRESS_MIN_TIME and not pgo): text += ' (%s)' % format_duration(result.duration) if not pgo: running = get_running(self.workers) diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 8e9779524c56a2..130c036a62eefb 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -5,12 +5,12 @@ import shlex import subprocess import sys -from typing import Any +from typing import Any, Iterator from test import support from .utils import ( - StrPath, StrJSON, TestTuple, TestFilter, FilterTuple, FilterDict) + StrPath, StrJSON, TestTuple, TestName, TestFilter, FilterTuple, FilterDict) class JsonFileType: @@ -28,7 +28,7 @@ class JsonFile: file: int | None file_type: str - def configure_subprocess(self, popen_kwargs: dict) -> None: + def configure_subprocess(self, popen_kwargs: dict[str, Any]) -> None: match self.file_type: case JsonFileType.UNIX_FD: # Unix file descriptor @@ -41,8 +41,8 @@ def configure_subprocess(self, popen_kwargs: dict) -> None: popen_kwargs['startupinfo'] = startupinfo @contextlib.contextmanager - def inherit_subprocess(self): - if self.file_type == JsonFileType.WINDOWS_HANDLE: + def inherit_subprocess(self) -> Iterator[None]: + if sys.platform == 'win32' and self.file_type == JsonFileType.WINDOWS_HANDLE: os.set_handle_inheritable(self.file, True) try: yield @@ -106,25 +106,25 @@ def copy(self, **override) -> 'RunTests': state.update(override) return RunTests(**state) - def create_worker_runtests(self, **override): + def create_worker_runtests(self, **override) -> WorkerRunTests: state = dataclasses.asdict(self) state.update(override) return WorkerRunTests(**state) - def get_match_tests(self, test_name) -> FilterTuple | None: + def get_match_tests(self, test_name: TestName) -> FilterTuple | None: if self.match_tests_dict is not None: return self.match_tests_dict.get(test_name, None) else: return None - def get_jobs(self): + def get_jobs(self) -> int | None: # Number of run_single_test() calls needed to run all tests. # None means that there is not bound limit (--forever option). if self.forever: return None return len(self.tests) - def iter_tests(self): + def iter_tests(self) -> Iterator[TestName]: if self.forever: while True: yield from self.tests diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index b2cc381344b2ef..ffc29fa8dc686a 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -97,7 +97,7 @@ def get_asyncio_events__event_loop_policy(self): return support.maybe_get_event_loop_policy() def restore_asyncio_events__event_loop_policy(self, policy): asyncio = self.get_module('asyncio') - asyncio.set_event_loop_policy(policy) + asyncio._set_event_loop_policy(policy) def get_sys_argv(self): return id(sys.argv), sys.argv, sys.argv[:] diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 9e9741493e9a5b..c0346aa934d394 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -11,7 +11,7 @@ from .filter import set_match_tests from .runtests import RunTests from .utils import ( - setup_unraisable_hook, setup_threading_excepthook, fix_umask, + setup_unraisable_hook, setup_threading_excepthook, adjust_rlimit_nofile) @@ -25,9 +25,8 @@ def setup_test_dir(testdir: str | None) -> None: sys.path.insert(0, os.path.abspath(testdir)) -def setup_process(): - fix_umask() - +def setup_process() -> None: + assert sys.__stderr__ is not None, "sys.__stderr__ is None" try: stderr_fd = sys.__stderr__.fileno() except (ValueError, AttributeError): @@ -35,7 +34,7 @@ def setup_process(): # and ValueError on a closed stream. # # Catch AttributeError for stderr being None. - stderr_fd = None + pass else: # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True, file=stderr_fd) @@ -68,7 +67,7 @@ def setup_process(): for index, path in enumerate(module.__path__): module.__path__[index] = os.path.abspath(path) if getattr(module, '__file__', None): - module.__file__ = os.path.abspath(module.__file__) + module.__file__ = os.path.abspath(module.__file__) # type: ignore[type-var] if hasattr(sys, 'addaudithook'): # Add an auditing hook for all tests to ensure PySys_Audit is tested @@ -87,7 +86,7 @@ def _test_audit_hook(name, args): os.environ.setdefault(UNICODE_GUARD_ENV, FS_NONASCII) -def setup_tests(runtests: RunTests): +def setup_tests(runtests: RunTests) -> None: support.verbose = runtests.verbose support.failfast = runtests.fail_fast support.PGO = runtests.pgo diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index 235029d8620ff5..54df688bbc470e 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -7,6 +7,7 @@ import traceback import unittest +from _colorize import get_colors # type: ignore[import-not-found] from test import support from test.support import threading_helper @@ -57,7 +58,10 @@ def _run_suite(suite): result = runner.run(suite) if support.junit_xml_list is not None: - support.junit_xml_list.append(result.get_xml_element()) + import xml.etree.ElementTree as ET + xml_elem = result.get_xml_element() + xml_str = ET.tostring(xml_elem).decode('ascii') + support.junit_xml_list.append(xml_str) if not result.testsRun and not result.skipped and not result.errors: raise support.TestDidNotRun @@ -158,6 +162,8 @@ def test_func(): def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, display_failure: bool = True) -> None: # Handle exceptions, detect environment changes. + stdout = get_colors(file=sys.stdout) + stderr = get_colors(file=sys.stderr) # Reset the environment_altered flag to detect if a test altered # the environment @@ -178,18 +184,24 @@ def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, _load_run_test(result, runtests) except support.ResourceDenied as exc: if not quiet and not pgo: - print(f"{test_name} skipped -- {exc}", flush=True) + print( + f"{stdout.YELLOW}{test_name} skipped -- {exc}{stdout.RESET}", + flush=True, + ) result.state = State.RESOURCE_DENIED return except unittest.SkipTest as exc: if not quiet and not pgo: - print(f"{test_name} skipped -- {exc}", flush=True) + print( + f"{stdout.YELLOW}{test_name} skipped -- {exc}{stdout.RESET}", + flush=True, + ) result.state = State.SKIPPED return except support.TestFailedWithDetails as exc: - msg = f"test {test_name} failed" + msg = f"{stderr.RED}test {test_name} failed{stderr.RESET}" if display_failure: - msg = f"{msg} -- {exc}" + msg = f"{stderr.RED}{msg} -- {exc}{stderr.RESET}" print(msg, file=sys.stderr, flush=True) result.state = State.FAILED result.errors = exc.errors @@ -197,9 +209,9 @@ def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, result.stats = exc.stats return except support.TestFailed as exc: - msg = f"test {test_name} failed" + msg = f"{stderr.RED}test {test_name} failed{stderr.RESET}" if display_failure: - msg = f"{msg} -- {exc}" + msg = f"{stderr.RED}{msg} -- {exc}{stderr.RESET}" print(msg, file=sys.stderr, flush=True) result.state = State.FAILED result.stats = exc.stats @@ -214,8 +226,11 @@ def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, except: if not pgo: msg = traceback.format_exc() - print(f"test {test_name} crashed -- {msg}", - file=sys.stderr, flush=True) + print( + f"{stderr.RED}test {test_name} crashed -- {msg}{stderr.RESET}", + file=sys.stderr, + flush=True, + ) result.state = State.UNCAUGHT_EXC return @@ -280,9 +295,7 @@ def _runtest(result: TestResult, runtests: RunTests) -> None: xml_list = support.junit_xml_list if xml_list: - import xml.etree.ElementTree as ET - result.xml_data = [ET.tostring(x).decode('us-ascii') - for x in xml_list] + result.xml_data = xml_list finally: if use_timeout: faulthandler.cancel_dump_traceback_later() @@ -299,6 +312,9 @@ def run_single_test(test_name: TestName, runtests: RunTests) -> TestResult: If runtests.use_junit, xml_data is a list containing each generated testsuite element. """ + ansi = get_colors(file=sys.stderr) + red, reset, yellow = ansi.BOLD_RED, ansi.RESET, ansi.YELLOW + start_time = time.perf_counter() result = TestResult(test_name) pgo = runtests.pgo @@ -307,7 +323,7 @@ def run_single_test(test_name: TestName, runtests: RunTests) -> TestResult: except: if not pgo: msg = traceback.format_exc() - print(f"test {test_name} crashed -- {msg}", + print(f"{red}test {test_name} crashed -- {msg}{reset}", file=sys.stderr, flush=True) result.state = State.UNCAUGHT_EXC diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index de23fdd59ded95..1820f354572521 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -9,6 +9,7 @@ import traceback import unittest from test import support +from test.libregrtest.utils import sanitize_xml class RegressionTestResult(unittest.TextTestResult): USE_XML = False @@ -65,23 +66,24 @@ def _add_result(self, test, capture=False, **args): if capture: if self._stdout_buffer is not None: stdout = self._stdout_buffer.getvalue().rstrip() - ET.SubElement(e, 'system-out').text = stdout + ET.SubElement(e, 'system-out').text = sanitize_xml(stdout) if self._stderr_buffer is not None: stderr = self._stderr_buffer.getvalue().rstrip() - ET.SubElement(e, 'system-err').text = stderr + ET.SubElement(e, 'system-err').text = sanitize_xml(stderr) for k, v in args.items(): if not k or not v: continue + e2 = ET.SubElement(e, k) if hasattr(v, 'items'): for k2, v2 in v.items(): if k2: - e2.set(k2, str(v2)) + e2.set(k2, sanitize_xml(str(v2))) else: - e2.text = str(v2) + e2.text = sanitize_xml(str(v2)) else: - e2.text = str(v) + e2.text = sanitize_xml(str(v)) @classmethod def __makeErrorDict(cls, err_type, err_value, err_tb): diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py new file mode 100644 index 00000000000000..00d5779d950e72 --- /dev/null +++ b/Lib/test/libregrtest/tsan.py @@ -0,0 +1,34 @@ +# Set of tests run by default if --tsan is specified. The tests below were +# chosen because they use threads and run in a reasonable amount of time. + +TSAN_TESTS = [ + # TODO: enable more of test_capi once bugs are fixed (GH-116908, GH-116909). + 'test_capi.test_mem', + 'test_capi.test_pyatomic', + 'test_code', + 'test_enum', + 'test_functools', + 'test_httpservers', + 'test_imaplib', + 'test_importlib', + 'test_io', + 'test_logging', + 'test_queue', + 'test_signal', + 'test_socket', + 'test_sqlite3', + 'test_ssl', + 'test_syslog', + 'test_thread', + 'test_threadedtempfile', + 'test_threading', + 'test_threading_local', + 'test_threadsignals', + 'test_weakref', + 'test_free_threading.test_slots', +] + + +def setup_tsan_tests(cmdline_args) -> None: + if not cmdline_args: + cmdline_args[:] = TSAN_TESTS[:] diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index b30025d962413c..3eff9e753b6d84 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -5,6 +5,7 @@ import os.path import platform import random +import re import shlex import signal import subprocess @@ -57,7 +58,7 @@ FilterDict = dict[TestName, FilterTuple] -def format_duration(seconds): +def format_duration(seconds: float) -> str: ms = math.ceil(seconds * 1e3) seconds, ms = divmod(ms, 1000) minutes, seconds = divmod(seconds, 60) @@ -91,7 +92,7 @@ def strip_py_suffix(names: list[str] | None) -> None: names[idx] = basename -def plural(n, singular, plural=None): +def plural(n: int, singular: str, plural: str | None = None) -> str: if n == 1: return singular elif plural is not None: @@ -100,7 +101,7 @@ def plural(n, singular, plural=None): return singular + 's' -def count(n, word): +def count(n: int, word: str) -> str: if n == 1: return f"{n} {word}" else: @@ -122,14 +123,14 @@ def printlist(x, width=70, indent=4, file=None): file=file) -def print_warning(msg): +def print_warning(msg: str) -> None: support.print_warning(msg) -orig_unraisablehook = None +orig_unraisablehook: Callable[..., None] | None = None -def regrtest_unraisable_hook(unraisable): +def regrtest_unraisable_hook(unraisable) -> None: global orig_unraisablehook support.environment_altered = True support.print_warning("Unraisable exception") @@ -137,22 +138,23 @@ def regrtest_unraisable_hook(unraisable): try: support.flush_std_streams() sys.stderr = support.print_warning.orig_stderr + assert orig_unraisablehook is not None, "orig_unraisablehook not set" orig_unraisablehook(unraisable) sys.stderr.flush() finally: sys.stderr = old_stderr -def setup_unraisable_hook(): +def setup_unraisable_hook() -> None: global orig_unraisablehook orig_unraisablehook = sys.unraisablehook sys.unraisablehook = regrtest_unraisable_hook -orig_threading_excepthook = None +orig_threading_excepthook: Callable[..., None] | None = None -def regrtest_threading_excepthook(args): +def regrtest_threading_excepthook(args) -> None: global orig_threading_excepthook support.environment_altered = True support.print_warning(f"Uncaught thread exception: {args.exc_type.__name__}") @@ -160,13 +162,14 @@ def regrtest_threading_excepthook(args): try: support.flush_std_streams() sys.stderr = support.print_warning.orig_stderr + assert orig_threading_excepthook is not None, "orig_threading_excepthook not set" orig_threading_excepthook(args) sys.stderr.flush() finally: sys.stderr = old_stderr -def setup_threading_excepthook(): +def setup_threading_excepthook() -> None: global orig_threading_excepthook import threading orig_threading_excepthook = threading.excepthook @@ -263,6 +266,12 @@ def clear_caches(): for f in typing._cleanups: f() + import inspect + abs_classes = filter(inspect.isabstract, typing.__dict__.values()) + for abc in abs_classes: + for obj in abc.__subclasses__() + [abc]: + obj._abc_caches_clear() + try: fractions = sys.modules['fractions'] except KeyError: @@ -275,7 +284,16 @@ def clear_caches(): except KeyError: pass else: - inspect._shadowed_dict_from_mro_tuple.cache_clear() + inspect._shadowed_dict_from_weakref_mro_tuple.cache_clear() + inspect._filesbymodname.clear() + inspect.modulesbyfile.clear() + + try: + importlib_metadata = sys.modules['importlib.metadata'] + except KeyError: + pass + else: + importlib_metadata.FastPath.__new__.cache_clear() def get_build_info(): @@ -284,29 +302,78 @@ def get_build_info(): config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' cflags = sysconfig.get_config_var('PY_CFLAGS') or '' - cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST') or '' + cflags += ' ' + (sysconfig.get_config_var('PY_CFLAGS_NODIST') or '') ldflags_nodist = sysconfig.get_config_var('PY_LDFLAGS_NODIST') or '' build = [] # --disable-gil if sysconfig.get_config_var('Py_GIL_DISABLED'): - build.append("free_threading") + if not sys.flags.ignore_environment: + PYTHON_GIL = os.environ.get('PYTHON_GIL', None) + if PYTHON_GIL: + PYTHON_GIL = (PYTHON_GIL == '1') + else: + PYTHON_GIL = None + + free_threading = "free_threading" + if PYTHON_GIL is not None: + free_threading = f"{free_threading} GIL={int(PYTHON_GIL)}" + build.append(free_threading) if hasattr(sys, 'gettotalrefcount'): # --with-pydebug build.append('debug') - if '-DNDEBUG' in (cflags + cflags_nodist): + if '-DNDEBUG' in cflags: build.append('without_assert') else: build.append('release') if '--with-assertions' in config_args: build.append('with_assert') - elif '-DNDEBUG' not in (cflags + cflags_nodist): + elif '-DNDEBUG' not in cflags: build.append('with_assert') + # --enable-experimental-jit + tier2 = re.search('-D_Py_TIER2=([0-9]+)', cflags) + if tier2: + tier2 = int(tier2.group(1)) + + if not sys.flags.ignore_environment: + PYTHON_JIT = os.environ.get('PYTHON_JIT', None) + if PYTHON_JIT: + PYTHON_JIT = (PYTHON_JIT != '0') + else: + PYTHON_JIT = None + + if tier2 == 1: # =yes + if PYTHON_JIT == False: + jit = 'JIT=off' + else: + jit = 'JIT' + elif tier2 == 3: # =yes-off + if PYTHON_JIT: + jit = 'JIT' + else: + jit = 'JIT=off' + elif tier2 == 4: # =interpreter + if PYTHON_JIT == False: + jit = 'JIT-interpreter=off' + else: + jit = 'JIT-interpreter' + elif tier2 == 6: # =interpreter-off (Secret option!) + if PYTHON_JIT: + jit = 'JIT-interpreter' + else: + jit = 'JIT-interpreter=off' + elif '-D_Py_JIT' in cflags: + jit = 'JIT' + else: + jit = None + if jit: + build.append(jit) + # --enable-framework=name framework = sysconfig.get_config_var('PYTHONFRAMEWORK') if framework: @@ -327,6 +394,11 @@ def get_build_info(): if support.check_cflags_pgo(): # PGO (--enable-optimizations) optimizations.append('PGO') + + if support.check_bolt_optimized(): + # BOLT (--enable-bolt) + optimizations.append('BOLT') + if optimizations: build.append('+'.join(optimizations)) @@ -406,23 +478,12 @@ def get_temp_dir(tmp_dir: StrPath | None = None) -> StrPath: return os.path.abspath(tmp_dir) -def fix_umask(): - if support.is_emscripten: - # Emscripten has default umask 0o777, which breaks some tests. - # see https://github.com/emscripten-core/emscripten/issues/17269 - old_mask = os.umask(0) - if old_mask == 0o777: - os.umask(0o027) - else: - os.umask(old_mask) - - def get_work_dir(parent_dir: StrPath, worker: bool = False) -> StrPath: # Define a writable temp dir that will be used as cwd while running # the tests. The name of the dir includes the pid to allow parallel # testing (see the -j option). # Emscripten and WASI have stubbed getpid(), Emscripten has only - # milisecond clock resolution. Use randint() instead. + # millisecond clock resolution. Use randint() instead. if support.is_emscripten or support.is_wasi: nounce = random.randint(0, 1_000_000) else: @@ -502,7 +563,8 @@ def abs_module_name(test_name: TestName, test_dir: StrPath | None) -> TestName: 'setUpModule', 'tearDownModule', )) -def normalize_test_name(test_full_name, *, is_error=False): +def normalize_test_name(test_full_name: str, *, + is_error: bool = False) -> str | None: short_name = test_full_name.split(" ")[0] if is_error and short_name in _TEST_LIFECYCLE_HOOKS: if test_full_name.startswith(('setUpModule (', 'tearDownModule (')): @@ -523,7 +585,7 @@ def normalize_test_name(test_full_name, *, is_error=False): return short_name -def adjust_rlimit_nofile(): +def adjust_rlimit_nofile() -> None: """ On macOS the default fd limit (RLIMIT_NOFILE) is sometimes too low (256) for our test suite to succeed. Raise it to something more reasonable. 1024 @@ -549,17 +611,17 @@ def adjust_rlimit_nofile(): f"{new_fd_limit}: {err}.") -def get_host_runner(): +def get_host_runner() -> str: if (hostrunner := os.environ.get("_PYTHON_HOSTRUNNER")) is None: hostrunner = sysconfig.get_config_var("HOSTRUNNER") return hostrunner -def is_cross_compiled(): +def is_cross_compiled() -> bool: return ('_PYTHON_HOST_PLATFORM' in os.environ) -def format_resources(use_resources: Iterable[str]): +def format_resources(use_resources: Iterable[str]) -> str: use_resources = set(use_resources) all_resources = set(ALL_RESOURCES) @@ -584,7 +646,7 @@ def format_resources(use_resources: Iterable[str]): def display_header(use_resources: tuple[str, ...], - python_cmd: tuple[str, ...] | None): + python_cmd: tuple[str, ...] | None) -> None: # Print basic platform information print("==", platform.python_implementation(), *sys.version.split()) print("==", platform.platform(aliased=True), @@ -662,7 +724,7 @@ def display_header(use_resources: tuple[str, ...], print(flush=True) -def cleanup_temp_dir(tmp_dir: StrPath): +def cleanup_temp_dir(tmp_dir: StrPath) -> None: import glob path = os.path.join(glob.escape(tmp_dir), TMP_PREFIX + '*') @@ -675,23 +737,23 @@ def cleanup_temp_dir(tmp_dir: StrPath): print("Remove file: %s" % name) os_helper.unlink(name) -WINDOWS_STATUS = { - 0xC0000005: "STATUS_ACCESS_VIOLATION", - 0xC00000FD: "STATUS_STACK_OVERFLOW", - 0xC000013A: "STATUS_CONTROL_C_EXIT", -} - -def get_signal_name(exitcode): - if exitcode < 0: - signum = -exitcode - try: - return signal.Signals(signum).name - except ValueError: - pass - - try: - return WINDOWS_STATUS[exitcode] - except KeyError: - pass - return None +ILLEGAL_XML_CHARS_RE = re.compile( + '[' + # Control characters; newline (\x0A and \x0D) and TAB (\x09) are legal + '\x00-\x08\x0B\x0C\x0E-\x1F' + # Surrogate characters + '\uD800-\uDFFF' + # Special Unicode characters + '\uFFFE' + '\uFFFF' + # Match multiple sequential invalid characters for better efficiency + ']+') + +def _sanitize_xml_replace(regs): + text = regs[0] + return ''.join(f'\\x{ord(ch):02x}' if ch <= '\xff' else ascii(ch)[1:-1] + for ch in text) + +def sanitize_xml(text: str) -> str: + return ILLEGAL_XML_CHARS_RE.sub(_sanitize_xml_replace, text) diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index f8b8e45eca3276..0c9f5bd6e42f11 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -14,10 +14,13 @@ USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) +NEED_TTY = { + 'test_ioctl', +} def create_worker_process(runtests: WorkerRunTests, output_fd: int, - tmp_dir: StrPath | None = None) -> subprocess.Popen: + tmp_dir: StrPath | None = None) -> subprocess.Popen[str]: worker_json = runtests.as_json() cmd = runtests.create_python_cmd() @@ -47,7 +50,10 @@ def create_worker_process(runtests: WorkerRunTests, output_fd: int, close_fds=True, cwd=work_dir, ) - if USE_PROCESS_GROUP: + + # Don't use setsid() in tests using TTY + test_name = runtests.tests[0] + if USE_PROCESS_GROUP and test_name not in NEED_TTY: kwargs['start_new_session'] = True # Pass json_file to the worker process @@ -98,7 +104,7 @@ def worker_process(worker_json: StrJSON) -> NoReturn: sys.exit(0) -def main(): +def main() -> NoReturn: if len(sys.argv) != 2: print("usage: python -m test.libregrtest.worker JSON") sys.exit(1) diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index 26118e14bb97e0..dbd9f27872962d 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -6,7 +6,7 @@ from functools import cmp_to_key from test import seq_tests -from test.support import ALWAYS_EQ, NEVER_EQ, Py_C_RECURSION_LIMIT +from test.support import ALWAYS_EQ, NEVER_EQ, get_c_recursion_limit, skip_emscripten_stack_overflow class CommonTest(seq_tests.CommonTest): @@ -59,9 +59,10 @@ def test_repr(self): self.assertEqual(str(a2), "[0, 1, 2, [...], 3]") self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]") + @skip_emscripten_stack_overflow() def test_repr_deep(self): a = self.type2test([]) - for i in range(Py_C_RECURSION_LIMIT + 1): + for i in range(get_c_recursion_limit() + 1): a = self.type2test([a]) self.assertRaises(RecursionError, repr, a) @@ -191,6 +192,14 @@ def test_setslice(self): self.assertRaises(TypeError, a.__setitem__) + def test_slice_assign_iterator(self): + x = self.type2test(range(5)) + x[0:3] = reversed(range(3)) + self.assertEqual(x, self.type2test([2, 1, 0, 3, 4])) + + x[:] = reversed(range(3)) + self.assertEqual(x, self.type2test([2, 1, 0])) + def test_delslice(self): a = self.type2test([0, 1]) del a[1:2] diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index 024c6debcd4a54..8c8f8901f00178 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -1013,6 +1013,10 @@ def multipass(self, results, n): self.assertEqual(self.barrier.n_waiting, 0) self.assertFalse(self.barrier.broken) + def test_constructor(self): + self.assertRaises(ValueError, self.barriertype, parties=0) + self.assertRaises(ValueError, self.barriertype, parties=-1) + def test_barrier(self, passes=1): """ Test that a barrier is passed in lockstep diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index b4cfce19a7174e..f249f0021e9c1c 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -1,7 +1,7 @@ # tests common to dict and UserDict import unittest import collections -from test.support import Py_C_RECURSION_LIMIT +from test.support import get_c_recursion_limit, skip_emscripten_stack_overflow class BasicTestMappingProtocol(unittest.TestCase): @@ -622,9 +622,10 @@ def __repr__(self): d = self._full_mapping({1: BadRepr()}) self.assertRaises(Exc, repr, d) + @skip_emscripten_stack_overflow() def test_repr_deep(self): d = self._empty_mapping() - for i in range(Py_C_RECURSION_LIMIT + 1): + for i in range(get_c_recursion_limit() + 1): d0 = d d = self._empty_mapping() d[1] = d0 diff --git a/Lib/test/mathdata/cmath_testcases.txt b/Lib/test/mathdata/cmath_testcases.txt index 0165e17634f41c..7b98b5a2998413 100644 --- a/Lib/test/mathdata/cmath_testcases.txt +++ b/Lib/test/mathdata/cmath_testcases.txt @@ -371,9 +371,9 @@ acosh1002 acosh 0.0 inf -> inf 1.5707963267948966 acosh1003 acosh 2.3 inf -> inf 1.5707963267948966 acosh1004 acosh -0.0 inf -> inf 1.5707963267948966 acosh1005 acosh -2.3 inf -> inf 1.5707963267948966 -acosh1006 acosh 0.0 nan -> nan nan +acosh1006 acosh 0.0 nan -> nan 1.5707963267948966 ignore-imag-sign acosh1007 acosh 2.3 nan -> nan nan -acosh1008 acosh -0.0 nan -> nan nan +acosh1008 acosh -0.0 nan -> nan 1.5707963267948966 ignore-imag-sign acosh1009 acosh -2.3 nan -> nan nan acosh1010 acosh -inf 0.0 -> inf 3.1415926535897931 acosh1011 acosh -inf 2.3 -> inf 3.1415926535897931 @@ -1992,9 +1992,9 @@ tanh0065 tanh 1.797e+308 0.0 -> 1.0 0.0 --special values tanh1000 tanh 0.0 0.0 -> 0.0 0.0 -tanh1001 tanh 0.0 inf -> nan nan invalid +tanh1001 tanh 0.0 inf -> 0.0 nan invalid tanh1002 tanh 2.3 inf -> nan nan invalid -tanh1003 tanh 0.0 nan -> nan nan +tanh1003 tanh 0.0 nan -> 0.0 nan tanh1004 tanh 2.3 nan -> nan nan tanh1005 tanh inf 0.0 -> 1.0 0.0 tanh1006 tanh inf 0.7 -> 1.0 0.0 @@ -2009,7 +2009,7 @@ tanh1014 tanh nan 2.3 -> nan nan tanh1015 tanh nan inf -> nan nan tanh1016 tanh nan nan -> nan nan tanh1017 tanh 0.0 -0.0 -> 0.0 -0.0 -tanh1018 tanh 0.0 -inf -> nan nan invalid +tanh1018 tanh 0.0 -inf -> 0.0 nan invalid tanh1019 tanh 2.3 -inf -> nan nan invalid tanh1020 tanh inf -0.0 -> 1.0 -0.0 tanh1021 tanh inf -0.7 -> 1.0 -0.0 @@ -2022,9 +2022,9 @@ tanh1027 tanh nan -0.0 -> nan -0.0 tanh1028 tanh nan -2.3 -> nan nan tanh1029 tanh nan -inf -> nan nan tanh1030 tanh -0.0 -0.0 -> -0.0 -0.0 -tanh1031 tanh -0.0 -inf -> nan nan invalid +tanh1031 tanh -0.0 -inf -> -0.0 nan invalid tanh1032 tanh -2.3 -inf -> nan nan invalid -tanh1033 tanh -0.0 nan -> nan nan +tanh1033 tanh -0.0 nan -> -0.0 nan tanh1034 tanh -2.3 nan -> nan nan tanh1035 tanh -inf -0.0 -> -1.0 -0.0 tanh1036 tanh -inf -0.7 -> -1.0 -0.0 @@ -2035,7 +2035,7 @@ tanh1040 tanh -inf -3.5 -> -1.0 -0.0 tanh1041 tanh -inf -inf -> -1.0 0.0 ignore-imag-sign tanh1042 tanh -inf nan -> -1.0 0.0 ignore-imag-sign tanh1043 tanh -0.0 0.0 -> -0.0 0.0 -tanh1044 tanh -0.0 inf -> nan nan invalid +tanh1044 tanh -0.0 inf -> -0.0 nan invalid tanh1045 tanh -2.3 inf -> nan nan invalid tanh1046 tanh -inf 0.0 -> -1.0 0.0 tanh1047 tanh -inf 0.7 -> -1.0 0.0 @@ -2307,9 +2307,9 @@ tan0066 tan -8.79645943005142 0.0 -> 0.7265425280053614098 0.0 -- special values tan1000 tan -0.0 0.0 -> -0.0 0.0 -tan1001 tan -inf 0.0 -> nan nan invalid +tan1001 tan -inf 0.0 -> nan 0.0 invalid tan1002 tan -inf 2.2999999999999998 -> nan nan invalid -tan1003 tan nan 0.0 -> nan nan +tan1003 tan nan 0.0 -> nan 0.0 tan1004 tan nan 2.2999999999999998 -> nan nan tan1005 tan -0.0 inf -> -0.0 1.0 tan1006 tan -0.69999999999999996 inf -> -0.0 1.0 @@ -2324,7 +2324,7 @@ tan1014 tan -2.2999999999999998 nan -> nan nan tan1015 tan -inf nan -> nan nan tan1016 tan nan nan -> nan nan tan1017 tan 0.0 0.0 -> 0.0 0.0 -tan1018 tan inf 0.0 -> nan nan invalid +tan1018 tan inf 0.0 -> nan 0.0 invalid tan1019 tan inf 2.2999999999999998 -> nan nan invalid tan1020 tan 0.0 inf -> 0.0 1.0 tan1021 tan 0.69999999999999996 inf -> 0.0 1.0 @@ -2337,9 +2337,9 @@ tan1027 tan 0.0 nan -> 0.0 nan tan1028 tan 2.2999999999999998 nan -> nan nan tan1029 tan inf nan -> nan nan tan1030 tan 0.0 -0.0 -> 0.0 -0.0 -tan1031 tan inf -0.0 -> nan nan invalid +tan1031 tan inf -0.0 -> nan -0.0 invalid tan1032 tan inf -2.2999999999999998 -> nan nan invalid -tan1033 tan nan -0.0 -> nan nan +tan1033 tan nan -0.0 -> nan -0.0 tan1034 tan nan -2.2999999999999998 -> nan nan tan1035 tan 0.0 -inf -> 0.0 -1.0 tan1036 tan 0.69999999999999996 -inf -> 0.0 -1.0 @@ -2350,7 +2350,7 @@ tan1040 tan 3.5 -inf -> 0.0 -1.0 tan1041 tan inf -inf -> -0.0 -1.0 ignore-real-sign tan1042 tan nan -inf -> -0.0 -1.0 ignore-real-sign tan1043 tan -0.0 -0.0 -> -0.0 -0.0 -tan1044 tan -inf -0.0 -> nan nan invalid +tan1044 tan -inf -0.0 -> nan -0.0 invalid tan1045 tan -inf -2.2999999999999998 -> nan nan invalid tan1046 tan -0.0 -inf -> -0.0 -1.0 tan1047 tan -0.69999999999999996 -inf -> -0.0 -1.0 diff --git a/Lib/test/mathdata/ieee754.txt b/Lib/test/mathdata/ieee754.txt index a8b8a0a2148f00..0dac69040ad011 100644 --- a/Lib/test/mathdata/ieee754.txt +++ b/Lib/test/mathdata/ieee754.txt @@ -51,7 +51,7 @@ nan >>> INF / INF nan -However unambigous operations with inf return inf: +However unambiguous operations with inf return inf: >>> INF * INF inf >>> 1.5 * INF @@ -116,7 +116,7 @@ inf >>> 0 ** -1 Traceback (most recent call last): ... -ZeroDivisionError: 0.0 cannot be raised to a negative power +ZeroDivisionError: zero to a negative power >>> pow(0, NAN) nan diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 93e7dbbd103934..bdc7ef62943a28 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -26,7 +26,7 @@ from test import support from test.support import os_helper from test.support import ( - TestFailed, run_with_locale, no_tracing, + TestFailed, run_with_locales, no_tracing, _2G, _4G, bigmemtest ) from test.support.import_helper import forget @@ -144,6 +144,14 @@ class E(C): def __getinitargs__(self): return () +import __main__ +__main__.C = C +C.__module__ = "__main__" +__main__.D = D +D.__module__ = "__main__" +__main__.E = E +E.__module__ = "__main__" + # Simple mutable object. class Object: pass @@ -157,14 +165,6 @@ def __reduce__(self): # Shouldn't support the recursion itself return K, (self.value,) -import __main__ -__main__.C = C -C.__module__ = "__main__" -__main__.D = D -D.__module__ = "__main__" -__main__.E = E -E.__module__ = "__main__" - class myint(int): def __init__(self, x): self.str = str(x) @@ -1012,6 +1012,26 @@ def test_constants(self): self.assertIs(self.loads(b'I01\n.'), True) self.assertIs(self.loads(b'I00\n.'), False) + def test_zero_padded_integers(self): + self.assertEqual(self.loads(b'I010\n.'), 10) + self.assertEqual(self.loads(b'I-010\n.'), -10) + self.assertEqual(self.loads(b'I0010\n.'), 10) + self.assertEqual(self.loads(b'I-0010\n.'), -10) + self.assertEqual(self.loads(b'L010\n.'), 10) + self.assertEqual(self.loads(b'L-010\n.'), -10) + self.assertEqual(self.loads(b'L0010\n.'), 10) + self.assertEqual(self.loads(b'L-0010\n.'), -10) + self.assertEqual(self.loads(b'L010L\n.'), 10) + self.assertEqual(self.loads(b'L-010L\n.'), -10) + + def test_nondecimal_integers(self): + self.assertRaises(ValueError, self.loads, b'I0b10\n.') + self.assertRaises(ValueError, self.loads, b'I0o10\n.') + self.assertRaises(ValueError, self.loads, b'I0x10\n.') + self.assertRaises(ValueError, self.loads, b'L0b10L\n.') + self.assertRaises(ValueError, self.loads, b'L0o10L\n.') + self.assertRaises(ValueError, self.loads, b'L0x10L\n.') + def test_empty_bytestring(self): # issue 11286 empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r') @@ -1179,6 +1199,166 @@ def test_compat_unpickle(self): self.assertIs(type(unpickled), collections.UserDict) self.assertEqual(unpickled, collections.UserDict({1: 2})) + def test_load_global(self): + self.assertIs(self.loads(b'cbuiltins\nstr\n.'), str) + self.assertIs(self.loads(b'cmath\nlog\n.'), math.log) + self.assertIs(self.loads(b'cos.path\njoin\n.'), os.path.join) + self.assertIs(self.loads(b'\x80\x04cbuiltins\nstr.upper\n.'), str.upper) + with support.swap_item(sys.modules, 'mödule', types.SimpleNamespace(glöbal=42)): + self.assertEqual(self.loads(b'\x80\x04cm\xc3\xb6dule\ngl\xc3\xb6bal\n.'), 42) + + self.assertRaises(UnicodeDecodeError, self.loads, b'c\xff\nlog\n.') + self.assertRaises(UnicodeDecodeError, self.loads, b'cmath\n\xff\n.') + self.assertRaises(self.truncated_errors, self.loads, b'c\nlog\n.') + self.assertRaises(self.truncated_errors, self.loads, b'cmath\n\n.') + self.assertRaises(self.truncated_errors, self.loads, b'\x80\x04cmath\n\n.') + + def test_load_stack_global(self): + self.assertIs(self.loads(b'\x8c\x08builtins\x8c\x03str\x93.'), str) + self.assertIs(self.loads(b'\x8c\x04math\x8c\x03log\x93.'), math.log) + self.assertIs(self.loads(b'\x8c\x07os.path\x8c\x04join\x93.'), + os.path.join) + self.assertIs(self.loads(b'\x80\x04\x8c\x08builtins\x8c\x09str.upper\x93.'), + str.upper) + with support.swap_item(sys.modules, 'mödule', types.SimpleNamespace(glöbal=42)): + self.assertEqual(self.loads(b'\x80\x04\x8c\x07m\xc3\xb6dule\x8c\x07gl\xc3\xb6bal\x93.'), 42) + + self.assertRaises(UnicodeDecodeError, self.loads, b'\x8c\x01\xff\x8c\x03log\x93.') + self.assertRaises(UnicodeDecodeError, self.loads, b'\x8c\x04math\x8c\x01\xff\x93.') + self.assertRaises(ValueError, self.loads, b'\x8c\x00\x8c\x03log\x93.') + self.assertRaises(AttributeError, self.loads, b'\x8c\x04math\x8c\x00\x93.') + self.assertRaises(AttributeError, self.loads, b'\x80\x04\x8c\x04math\x8c\x00\x93.') + + self.assertRaises(pickle.UnpicklingError, self.loads, b'N\x8c\x03log\x93.') + self.assertRaises(pickle.UnpicklingError, self.loads, b'\x8c\x04mathN\x93.') + self.assertRaises(pickle.UnpicklingError, self.loads, b'\x80\x04\x8c\x04mathN\x93.') + + def test_find_class(self): + unpickler = self.unpickler(io.BytesIO()) + unpickler_nofix = self.unpickler(io.BytesIO(), fix_imports=False) + unpickler4 = self.unpickler(io.BytesIO(b'\x80\x04N.')) + unpickler4.load() + + self.assertIs(unpickler.find_class('__builtin__', 'str'), str) + self.assertRaises(ModuleNotFoundError, + unpickler_nofix.find_class, '__builtin__', 'str') + self.assertIs(unpickler.find_class('builtins', 'str'), str) + self.assertIs(unpickler_nofix.find_class('builtins', 'str'), str) + self.assertIs(unpickler.find_class('math', 'log'), math.log) + self.assertIs(unpickler.find_class('os.path', 'join'), os.path.join) + self.assertIs(unpickler.find_class('os.path', 'join'), os.path.join) + + self.assertIs(unpickler4.find_class('builtins', 'str.upper'), str.upper) + with self.assertRaisesRegex(AttributeError, + r"module 'builtins' has no attribute 'str\.upper'"): + unpickler.find_class('builtins', 'str.upper') + + with self.assertRaisesRegex(AttributeError, + "module 'math' has no attribute 'spam'"): + unpickler.find_class('math', 'spam') + with self.assertRaisesRegex(AttributeError, + "module 'math' has no attribute 'spam'"): + unpickler4.find_class('math', 'spam') + with self.assertRaisesRegex(AttributeError, + r"module 'math' has no attribute 'log\.spam'"): + unpickler.find_class('math', 'log.spam') + with self.assertRaisesRegex(AttributeError, + r"Can't resolve path 'log\.spam' on module 'math'") as cm: + unpickler4.find_class('math', 'log.spam') + self.assertEqual(str(cm.exception.__context__), + "'builtin_function_or_method' object has no attribute 'spam'") + with self.assertRaisesRegex(AttributeError, + r"module 'math' has no attribute 'log\.\.spam'"): + unpickler.find_class('math', 'log..spam') + with self.assertRaisesRegex(AttributeError, + r"Can't resolve path 'log\.\.spam' on module 'math'") as cm: + unpickler4.find_class('math', 'log..spam') + self.assertEqual(str(cm.exception.__context__), + "'builtin_function_or_method' object has no attribute ''") + with self.assertRaisesRegex(AttributeError, + "module 'math' has no attribute ''"): + unpickler.find_class('math', '') + with self.assertRaisesRegex(AttributeError, + "module 'math' has no attribute ''"): + unpickler4.find_class('math', '') + self.assertRaises(ModuleNotFoundError, unpickler.find_class, 'spam', 'log') + self.assertRaises(ValueError, unpickler.find_class, '', 'log') + + self.assertRaises(TypeError, unpickler.find_class, None, 'log') + self.assertRaises(TypeError, unpickler.find_class, 'math', None) + self.assertRaises((TypeError, AttributeError), unpickler4.find_class, 'math', None) + + def test_custom_find_class(self): + def loads(data): + class Unpickler(self.unpickler): + def find_class(self, module_name, global_name): + return (module_name, global_name) + return Unpickler(io.BytesIO(data)).load() + + self.assertEqual(loads(b'cmath\nlog\n.'), ('math', 'log')) + self.assertEqual(loads(b'\x8c\x04math\x8c\x03log\x93.'), ('math', 'log')) + + def loads(data): + class Unpickler(self.unpickler): + @staticmethod + def find_class(module_name, global_name): + return (module_name, global_name) + return Unpickler(io.BytesIO(data)).load() + + self.assertEqual(loads(b'cmath\nlog\n.'), ('math', 'log')) + self.assertEqual(loads(b'\x8c\x04math\x8c\x03log\x93.'), ('math', 'log')) + + def loads(data): + class Unpickler(self.unpickler): + @classmethod + def find_class(cls, module_name, global_name): + return (module_name, global_name) + return Unpickler(io.BytesIO(data)).load() + + self.assertEqual(loads(b'cmath\nlog\n.'), ('math', 'log')) + self.assertEqual(loads(b'\x8c\x04math\x8c\x03log\x93.'), ('math', 'log')) + + def loads(data): + class Unpickler(self.unpickler): + pass + def find_class(module_name, global_name): + return (module_name, global_name) + unpickler = Unpickler(io.BytesIO(data)) + unpickler.find_class = find_class + return unpickler.load() + + self.assertEqual(loads(b'cmath\nlog\n.'), ('math', 'log')) + self.assertEqual(loads(b'\x8c\x04math\x8c\x03log\x93.'), ('math', 'log')) + + def test_bad_ext_code(self): + # unregistered extension code + self.check_unpickling_error(ValueError, b'\x82\x01.') + self.check_unpickling_error(ValueError, b'\x82\xff.') + self.check_unpickling_error(ValueError, b'\x83\x01\x00.') + self.check_unpickling_error(ValueError, b'\x83\xff\xff.') + self.check_unpickling_error(ValueError, b'\x84\x01\x00\x00\x00.') + self.check_unpickling_error(ValueError, b'\x84\xff\xff\xff\x7f.') + # EXT specifies code <= 0 + self.check_unpickling_error(pickle.UnpicklingError, b'\x82\x00.') + self.check_unpickling_error(pickle.UnpicklingError, b'\x83\x00\x00.') + self.check_unpickling_error(pickle.UnpicklingError, b'\x84\x00\x00\x00\x00.') + self.check_unpickling_error(pickle.UnpicklingError, b'\x84\x00\x00\x00\x80.') + self.check_unpickling_error(pickle.UnpicklingError, b'\x84\xff\xff\xff\xff.') + + @support.cpython_only + def test_bad_ext_inverted_registry(self): + code = 1 + def check(key, exc): + with support.swap_item(copyreg._inverted_registry, code, key): + with self.assertRaises(exc): + self.loads(b'\x82\x01.') + check(None, ValueError) + check((), ValueError) + check((__name__,), (TypeError, ValueError)) + check((__name__, "MyList", "x"), (TypeError, ValueError)) + check((__name__, None), (TypeError, ValueError)) + check((None, "MyList"), (TypeError, ValueError)) + def test_bad_reduce(self): self.assertEqual(self.loads(b'cbuiltins\nint\n)R.'), 0) self.check_unpickling_error(TypeError, b'N)R.') @@ -1197,6 +1377,41 @@ def test_bad_newobj_ex(self): self.check_unpickling_error(error, b'cbuiltins\nint\nN}\x92.') self.check_unpickling_error(error, b'cbuiltins\nint\n)N\x92.') + def test_bad_state(self): + c = C() + c.x = None + base = b'c__main__\nC\n)\x81' + self.assertEqual(self.loads(base + b'}X\x01\x00\x00\x00xNsb.'), c) + self.assertEqual(self.loads(base + b'N}X\x01\x00\x00\x00xNs\x86b.'), c) + # non-hashable dict key + self.check_unpickling_error(TypeError, base + b'}]Nsb.') + # state = list + error = (pickle.UnpicklingError, AttributeError) + self.check_unpickling_error(error, base + b'](}}eb.') + # state = 1-tuple + self.check_unpickling_error(error, base + b'}\x85b.') + # state = 3-tuple + self.check_unpickling_error(error, base + b'}}}\x87b.') + # non-hashable slot name + self.check_unpickling_error(TypeError, base + b'}}]Ns\x86b.') + # non-string slot name + self.check_unpickling_error(TypeError, base + b'}}NNs\x86b.') + # dict = True + self.check_unpickling_error(error, base + b'\x88}\x86b.') + # slots dict = True + self.check_unpickling_error(error, base + b'}\x88\x86b.') + + class BadKey1: + count = 1 + def __hash__(self): + if not self.count: + raise CustomError + self.count -= 1 + return 42 + __main__.BadKey1 = BadKey1 + # bad hashable dict key + self.check_unpickling_error(CustomError, base + b'}c__main__\nBadKey1\n)\x81Nsb.') + def test_bad_stack(self): badpickles = [ b'.', # STOP @@ -1443,6 +1658,789 @@ def t(): [ToBeUnpickled] * 2) +class AbstractPicklingErrorTests: + # Subclass must define self.dumps, self.pickler. + + def test_bad_reduce_result(self): + obj = REX([print, ()]) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + '__reduce__ must return a string or tuple, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + obj = REX((print,)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'tuple returned by __reduce__ must contain 2 through 6 elements') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + obj = REX((print, (), None, None, None, None, None)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'tuple returned by __reduce__ must contain 2 through 6 elements') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_bad_reconstructor(self): + obj = REX((42, ())) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'first item of the tuple returned by __reduce__ ' + 'must be callable, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_reconstructor(self): + obj = REX((UnpickleableCallable(), ())) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX reconstructor', + 'when serializing test.pickletester.REX object']) + + def test_bad_reconstructor_args(self): + obj = REX((print, [])) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'second item of the tuple returned by __reduce__ ' + 'must be a tuple, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_reconstructor_args(self): + obj = REX((print, (1, 2, UNPICKLEABLE))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing test.pickletester.REX reconstructor arguments', + 'when serializing test.pickletester.REX object']) + + def test_bad_newobj_args(self): + obj = REX((copyreg.__newobj__, ())) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises((IndexError, pickle.PicklingError)) as cm: + self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'tuple index out of range', + '__newobj__ expected at least 1 argument, got 0'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + obj = REX((copyreg.__newobj__, [REX])) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'second item of the tuple returned by __reduce__ ' + 'must be a tuple, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_bad_newobj_class(self): + obj = REX((copyreg.__newobj__, (NoNew(),))) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'first argument to __newobj__() has no __new__', + f'first argument to __newobj__() must be a class, not {__name__}.NoNew'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_wrong_newobj_class(self): + obj = REX((copyreg.__newobj__, (str,))) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f'first argument to __newobj__() must be {REX!r}, not {str!r}') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_newobj_class(self): + class LocalREX(REX): pass + obj = LocalREX((copyreg.__newobj__, (LocalREX,))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + if proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} class', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 0', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} reconstructor arguments', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) + + def test_unpickleable_newobj_args(self): + obj = REX((copyreg.__newobj__, (REX, 1, 2, UNPICKLEABLE))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + if proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing test.pickletester.REX __new__ arguments', + 'when serializing test.pickletester.REX object']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 3', + 'when serializing test.pickletester.REX reconstructor arguments', + 'when serializing test.pickletester.REX object']) + + def test_bad_newobj_ex_args(self): + obj = REX((copyreg.__newobj_ex__, ())) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises((ValueError, pickle.PicklingError)) as cm: + self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'not enough values to unpack (expected 3, got 0)', + '__newobj_ex__ expected 3 arguments, got 0'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + obj = REX((copyreg.__newobj_ex__, 42)) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'second item of the tuple returned by __reduce__ ' + 'must be a tuple, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + obj = REX((copyreg.__newobj_ex__, (REX, 42, {}))) + if self.pickler is pickle._Pickler: + for proto in protocols[2:4]: + with self.subTest(proto=proto): + with self.assertRaises(TypeError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'Value after * must be an iterable, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + else: + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'second argument to __newobj_ex__() must be a tuple, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + obj = REX((copyreg.__newobj_ex__, (REX, (), []))) + if self.pickler is pickle._Pickler: + for proto in protocols[2:4]: + with self.subTest(proto=proto): + with self.assertRaises(TypeError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'functools.partial() argument after ** must be a mapping, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + else: + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'third argument to __newobj_ex__() must be a dict, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_bad_newobj_ex__class(self): + obj = REX((copyreg.__newobj_ex__, (NoNew(), (), {}))) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'first argument to __newobj_ex__() has no __new__', + f'first argument to __newobj_ex__() must be a class, not {__name__}.NoNew'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_wrong_newobj_ex_class(self): + if self.pickler is not pickle._Pickler: + self.skipTest('only verified in the Python implementation') + obj = REX((copyreg.__newobj_ex__, (str, (), {}))) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f'first argument to __newobj_ex__() must be {REX}, not {str}') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_newobj_ex_class(self): + class LocalREX(REX): pass + obj = LocalREX((copyreg.__newobj_ex__, (LocalREX, (), {}))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} class', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) + elif proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 0', + 'when serializing tuple item 1', + 'when serializing functools.partial state', + 'when serializing functools.partial object', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} reconstructor', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 0', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} reconstructor arguments', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) + + def test_unpickleable_newobj_ex_args(self): + obj = REX((copyreg.__newobj_ex__, (REX, (1, 2, UNPICKLEABLE), {}))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing test.pickletester.REX __new__ arguments', + 'when serializing test.pickletester.REX object']) + elif proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 3', + 'when serializing tuple item 1', + 'when serializing functools.partial state', + 'when serializing functools.partial object', + 'when serializing test.pickletester.REX reconstructor', + 'when serializing test.pickletester.REX object']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing tuple item 1', + 'when serializing test.pickletester.REX reconstructor arguments', + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_newobj_ex_kwargs(self): + obj = REX((copyreg.__newobj_ex__, (REX, (), {'a': UNPICKLEABLE}))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'a'", + 'when serializing test.pickletester.REX __new__ arguments', + 'when serializing test.pickletester.REX object']) + elif proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'a'", + 'when serializing tuple item 2', + 'when serializing functools.partial state', + 'when serializing functools.partial object', + 'when serializing test.pickletester.REX reconstructor', + 'when serializing test.pickletester.REX object']) + else: + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'a'", + 'when serializing tuple item 2', + 'when serializing test.pickletester.REX reconstructor arguments', + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_state(self): + obj = REX_state(UNPICKLEABLE) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX_state state', + 'when serializing test.pickletester.REX_state object']) + + def test_bad_state_setter(self): + if self.pickler is pickle._Pickler: + self.skipTest('only verified in the C implementation') + obj = REX((print, (), 'state', None, None, 42)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'sixth item of the tuple returned by __reduce__ ' + 'must be callable, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_state_setter(self): + obj = REX((print, (), 'state', None, None, UnpickleableCallable())) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX state setter', + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_state_with_state_setter(self): + obj = REX((print, (), UNPICKLEABLE, None, None, print)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX state', + 'when serializing test.pickletester.REX object']) + + def test_bad_object_list_items(self): + # Issue4176: crash when 4th and 5th items of __reduce__() + # are not iterators + obj = REX((list, (), None, 42)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises((TypeError, pickle.PicklingError)) as cm: + self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + "'int' object is not iterable", + 'fourth item of the tuple returned by __reduce__ ' + 'must be an iterator, not int'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + if self.pickler is not pickle._Pickler: + # Python implementation is less strict and also accepts iterables. + obj = REX((list, (), None, [])) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'fourth item of the tuple returned by __reduce__ ' + 'must be an iterator, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_object_list_items(self): + obj = REX_six([1, 2, UNPICKLEABLE]) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX_six item 2', + 'when serializing test.pickletester.REX_six object']) + + def test_bad_object_dict_items(self): + # Issue4176: crash when 4th and 5th items of __reduce__() + # are not iterators + obj = REX((dict, (), None, None, 42)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises((TypeError, pickle.PicklingError)) as cm: + self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + "'int' object is not iterable", + 'fifth item of the tuple returned by __reduce__ ' + 'must be an iterator, not int'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + for proto in protocols: + obj = REX((dict, (), None, None, iter([('a',)]))) + with self.subTest(proto=proto): + with self.assertRaises((ValueError, TypeError)) as cm: + self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'not enough values to unpack (expected 2, got 1)', + 'dict items iterator must return 2-tuples'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + if self.pickler is not pickle._Pickler: + # Python implementation is less strict and also accepts iterables. + obj = REX((dict, (), None, None, [])) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'dict items iterator must return 2-tuples') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_object_dict_items(self): + obj = REX_seven({'a': UNPICKLEABLE}) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + "when serializing test.pickletester.REX_seven item 'a'", + 'when serializing test.pickletester.REX_seven object']) + + def test_unpickleable_list_items(self): + obj = [1, [2, 3, UNPICKLEABLE]] + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing list item 2', + 'when serializing list item 1']) + for n in [0, 1, 1000, 1005]: + obj = [*range(n), UNPICKLEABLE] + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + f'when serializing list item {n}']) + + def test_unpickleable_tuple_items(self): + obj = (1, (2, 3, UNPICKLEABLE)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing tuple item 1']) + obj = (*range(10), UNPICKLEABLE) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 10']) + + def test_unpickleable_dict_items(self): + obj = {'a': {'b': UNPICKLEABLE}} + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'b'", + "when serializing dict item 'a'"]) + for n in [0, 1, 1000, 1005]: + obj = dict.fromkeys(range(n)) + obj['a'] = UNPICKLEABLE + for proto in protocols: + with self.subTest(proto=proto, n=n): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'a'"]) + + def test_unpickleable_set_items(self): + obj = {UNPICKLEABLE} + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing set element']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing list item 0', + 'when serializing tuple item 0', + 'when serializing set reconstructor arguments']) + + def test_unpickleable_frozenset_items(self): + obj = frozenset({frozenset({UNPICKLEABLE})}) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing frozenset element', + 'when serializing frozenset element']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing list item 0', + 'when serializing tuple item 0', + 'when serializing frozenset reconstructor arguments', + 'when serializing list item 0', + 'when serializing tuple item 0', + 'when serializing frozenset reconstructor arguments']) + + def test_global_lookup_error(self): + # Global name does not exist + obj = REX('spam') + obj.__module__ = __name__ + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: it's not found as {__name__}.spam") + self.assertEqual(str(cm.exception.__context__), + f"module '{__name__}' has no attribute 'spam'") + + obj.__module__ = 'nonexisting' + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: No module named 'nonexisting'") + self.assertEqual(str(cm.exception.__context__), + "No module named 'nonexisting'") + + obj.__module__ = '' + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: Empty module name") + self.assertEqual(str(cm.exception.__context__), + "Empty module name") + + obj.__module__ = None + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: it's not found as __main__.spam") + self.assertEqual(str(cm.exception.__context__), + "module '__main__' has no attribute 'spam'") + + def test_nonencodable_global_name_error(self): + for proto in protocols[:4]: + with self.subTest(proto=proto): + name = 'nonascii\xff' if proto < 3 else 'nonencodable\udbff' + obj = REX(name) + obj.__module__ = __name__ + with support.swap_item(globals(), name, obj): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"can't pickle global identifier {name!r} using pickle protocol {proto}") + self.assertIsInstance(cm.exception.__context__, UnicodeEncodeError) + + def test_nonencodable_module_name_error(self): + for proto in protocols[:4]: + with self.subTest(proto=proto): + name = 'nonascii\xff' if proto < 3 else 'nonencodable\udbff' + obj = REX('test') + obj.__module__ = name + mod = types.SimpleNamespace(test=obj) + with support.swap_item(sys.modules, name, mod): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"can't pickle module identifier {name!r} using pickle protocol {proto}") + self.assertIsInstance(cm.exception.__context__, UnicodeEncodeError) + + def test_nested_lookup_error(self): + # Nested name does not exist + obj = REX('AbstractPickleTests.spam') + obj.__module__ = __name__ + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: " + f"it's not found as {__name__}.AbstractPickleTests.spam") + self.assertEqual(str(cm.exception.__context__), + "type object 'AbstractPickleTests' has no attribute 'spam'") + + obj.__module__ = None + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: it's not found as __main__.AbstractPickleTests.spam") + self.assertEqual(str(cm.exception.__context__), + "module '__main__' has no attribute 'AbstractPickleTests'") + + def test_wrong_object_lookup_error(self): + # Name is bound to different object + obj = REX('AbstractPickleTests') + obj.__module__ = __name__ + AbstractPickleTests.ham = [] + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: it's not the same object as {__name__}.AbstractPickleTests") + self.assertIsNone(cm.exception.__context__) + + obj.__module__ = None + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: it's not found as __main__.AbstractPickleTests") + self.assertEqual(str(cm.exception.__context__), + "module '__main__' has no attribute 'AbstractPickleTests'") + + def test_local_lookup_error(self): + # Test that whichmodule() errors out cleanly when looking up + # an assumed globally-reachable object fails. + def f(): + pass + # Since the function is local, lookup will fail + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(f, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle local object {f!r}") + # Same without a __module__ attribute (exercises a different path + # in _pickle.c). + del f.__module__ + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(f, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle local object {f!r}") + # Yet a different path. + f.__name__ = f.__qualname__ + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(f, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle local object {f!r}") + + def test_reduce_ex_None(self): + c = REX_None() + with self.assertRaises(TypeError): + self.dumps(c) + + def test_reduce_None(self): + c = R_None() + with self.assertRaises(TypeError): + self.dumps(c) + + @no_tracing + def test_bad_getattr(self): + # Issue #3514: crash when there is an infinite loop in __getattr__ + x = BadGetattr() + for proto in range(2): + with support.infinite_recursion(25): + self.assertRaises(RuntimeError, self.dumps, x, proto) + for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): + s = self.dumps(x, proto) + + def test_picklebuffer_error(self): + # PickleBuffer forbidden with protocol < 5 + pb = pickle.PickleBuffer(b"foobar") + for proto in range(0, 5): + with self.subTest(proto=proto): + with self.assertRaises(pickle.PickleError) as cm: + self.dumps(pb, proto) + self.assertEqual(str(cm.exception), + 'PickleBuffer can only be pickled with protocol >= 5') + + def test_non_continuous_buffer(self): + for proto in protocols[5:]: + with self.subTest(proto=proto): + pb = pickle.PickleBuffer(memoryview(b"foobar")[::2]) + with self.assertRaises((pickle.PicklingError, BufferError)): + self.dumps(pb, proto) + + def test_buffer_callback_error(self): + def buffer_callback(buffers): + raise CustomError + pb = pickle.PickleBuffer(b"foobar") + with self.assertRaises(CustomError): + self.dumps(pb, 5, buffer_callback=buffer_callback) + + def test_evil_pickler_mutating_collection(self): + # https://github.com/python/cpython/issues/92930 + global Clearer + class Clearer: + pass + + def check(collection): + class EvilPickler(self.pickler): + def persistent_id(self, obj): + if isinstance(obj, Clearer): + collection.clear() + return None + pickler = EvilPickler(io.BytesIO(), proto) + try: + pickler.dump(collection) + except RuntimeError as e: + expected = "changed size during iteration" + self.assertIn(expected, str(e)) + + for proto in protocols: + check([Clearer()]) + check([Clearer(), Clearer()]) + check({Clearer()}) + check({Clearer(), Clearer()}) + check({Clearer(): 1}) + check({Clearer(): 1, Clearer(): 2}) + check({1: Clearer(), 2: Clearer()}) + + @support.cpython_only + def test_bad_ext_code(self): + # This should never happen in normal circumstances, because the type + # and the value of the extension code is checked in copyreg.add_extension(). + key = (__name__, 'MyList') + def check(code, exc): + assert key not in copyreg._extension_registry + assert code not in copyreg._inverted_registry + with (support.swap_item(copyreg._extension_registry, key, code), + support.swap_item(copyreg._inverted_registry, code, key)): + for proto in protocols[2:]: + with self.assertRaises(exc): + self.dumps(MyList, proto) + + check(object(), TypeError) + check(None, TypeError) + check(-1, (RuntimeError, struct.error)) + check(0, RuntimeError) + check(2**31, (RuntimeError, OverflowError, struct.error)) + check(2**1000, (OverflowError, struct.error)) + check(-2**1000, (OverflowError, struct.error)) + class AbstractPickleTests: # Subclass must define self.dumps, self.loads. @@ -1845,6 +2843,25 @@ def test_bytes(self): p = self.dumps(s, proto) self.assert_is_copy(s, self.loads(p)) + def test_bytes_memoization(self): + for proto in protocols: + for array_type in [bytes, ZeroCopyBytes]: + for s in b'', b'xyz', b'xyz'*100: + with self.subTest(proto=proto, array_type=array_type, s=s, independent=False): + b = array_type(s) + p = self.dumps((b, b), proto) + x, y = self.loads(p) + self.assertIs(x, y) + self.assert_is_copy((b, b), (x, y)) + + with self.subTest(proto=proto, array_type=array_type, s=s, independent=True): + b1, b2 = array_type(s), array_type(s) + p = self.dumps((b1, b2), proto) + # Note that (b1, b2) = self.loads(p) might have identical + # components, i.e., b1 is b2, but this is not always the + # case if the content is large (equality still holds). + self.assert_is_copy((b1, b2), self.loads(p)) + def test_bytearray(self): for proto in protocols: for s in b'', b'xyz', b'xyz'*100: @@ -1864,13 +2881,31 @@ def test_bytearray(self): self.assertNotIn(b'bytearray', p) self.assertTrue(opcode_in_pickle(pickle.BYTEARRAY8, p)) - def test_bytearray_memoization_bug(self): + def test_bytearray_memoization(self): for proto in protocols: - for s in b'', b'xyz', b'xyz'*100: - b = bytearray(s) - p = self.dumps((b, b), proto) - b1, b2 = self.loads(p) - self.assertIs(b1, b2) + for array_type in [bytearray, ZeroCopyBytearray]: + for s in b'', b'xyz', b'xyz'*100: + with self.subTest(proto=proto, array_type=array_type, s=s, independent=False): + b = array_type(s) + p = self.dumps((b, b), proto) + b1, b2 = self.loads(p) + self.assertIs(b1, b2) + + with self.subTest(proto=proto, array_type=array_type, s=s, independent=True): + b1a, b2a = array_type(s), array_type(s) + # Unlike bytes, equal but independent bytearray objects are + # never identical. + self.assertIsNot(b1a, b2a) + + p = self.dumps((b1a, b2a), proto) + b1b, b2b = self.loads(p) + self.assertIsNot(b1b, b2b) + + self.assertIsNot(b1a, b1b) + self.assert_is_copy(b1a, b1b) + + self.assertIsNot(b2a, b2b) + self.assert_is_copy(b2a, b2b) def test_ints(self): for proto in protocols: @@ -1915,7 +2950,7 @@ def test_float(self): got = self.loads(pickle) self.assert_is_copy(value, got) - @run_with_locale('LC_ALL', 'de_DE', 'fr_FR') + @run_with_locales('LC_ALL', 'de_DE', 'fr_FR', '') def test_float_format(self): # make sure that floats are formatted locale independent with proto 0 self.assertEqual(self.dumps(1.2, 0)[0:3], b'F1.') @@ -2416,55 +3451,12 @@ def test_reduce_calls_base(self): y = self.loads(s) self.assertEqual(y._reduce_called, 1) - def test_reduce_ex_None(self): - c = REX_None() - with self.assertRaises(TypeError): - self.dumps(c) - - def test_reduce_None(self): - c = R_None() - with self.assertRaises(TypeError): - self.dumps(c) - def test_pickle_setstate_None(self): c = C_None_setstate() p = self.dumps(c) with self.assertRaises(TypeError): self.loads(p) - @no_tracing - def test_bad_getattr(self): - # Issue #3514: crash when there is an infinite loop in __getattr__ - x = BadGetattr() - for proto in range(2): - with support.infinite_recursion(25): - self.assertRaises(RuntimeError, self.dumps, x, proto) - for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): - s = self.dumps(x, proto) - - def test_reduce_bad_iterator(self): - # Issue4176: crash when 4th and 5th items of __reduce__() - # are not iterators - class C(object): - def __reduce__(self): - # 4th item is not an iterator - return list, (), None, [], None - class D(object): - def __reduce__(self): - # 5th item is not an iterator - return dict, (), None, None, [] - - # Python implementation is less strict and also accepts iterables. - for proto in protocols: - try: - self.dumps(C(), proto) - except pickle.PicklingError: - pass - try: - self.dumps(D(), proto) - except pickle.PicklingError: - pass - def test_many_puts_and_gets(self): # Test that internal data structures correctly deal with lots of # puts/gets. @@ -2781,6 +3773,18 @@ class Recursive: self.assertIs(unpickled, Recursive) del Recursive.mod # break reference loop + def test_recursive_nested_names2(self): + global Recursive + class Recursive: + pass + Recursive.ref = Recursive + Recursive.__qualname__ = 'Recursive.ref' + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + unpickled = self.loads(self.dumps(Recursive, proto)) + self.assertIs(unpickled, Recursive) + del Recursive.ref # break reference loop + def test_py_methods(self): global PyMethodsTest class PyMethodsTest: @@ -2901,27 +3905,6 @@ def test_compat_pickle(self): self.assertIn(('c%s\n%s' % (mod, name)).encode(), pickled) self.assertIs(type(self.loads(pickled)), type(val)) - def test_local_lookup_error(self): - # Test that whichmodule() errors out cleanly when looking up - # an assumed globally-reachable object fails. - def f(): - pass - # Since the function is local, lookup will fail - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises((AttributeError, pickle.PicklingError)): - pickletools.dis(self.dumps(f, proto)) - # Same without a __module__ attribute (exercises a different path - # in _pickle.c). - del f.__module__ - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises((AttributeError, pickle.PicklingError)): - pickletools.dis(self.dumps(f, proto)) - # Yet a different path. - f.__name__ = f.__qualname__ - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises((AttributeError, pickle.PicklingError)): - pickletools.dis(self.dumps(f, proto)) - # # PEP 574 tests below # @@ -3032,20 +4015,6 @@ def test_oob_buffers_writable_to_readonly(self): self.assertIs(type(new), type(obj)) self.assertEqual(new, obj) - def test_picklebuffer_error(self): - # PickleBuffer forbidden with protocol < 5 - pb = pickle.PickleBuffer(b"foobar") - for proto in range(0, 5): - with self.assertRaises(pickle.PickleError): - self.dumps(pb, proto) - - def test_buffer_callback_error(self): - def buffer_callback(buffers): - 1/0 - pb = pickle.PickleBuffer(b"foobar") - with self.assertRaises(ZeroDivisionError): - self.dumps(pb, 5, buffer_callback=buffer_callback) - def test_buffers_error(self): pb = pickle.PickleBuffer(b"foobar") for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): @@ -3137,37 +4106,6 @@ def __reduce__(self): expected = "changed size during iteration" self.assertIn(expected, str(e)) - def test_evil_pickler_mutating_collection(self): - # https://github.com/python/cpython/issues/92930 - if not hasattr(self, "pickler"): - raise self.skipTest(f"{type(self)} has no associated pickler type") - - global Clearer - class Clearer: - pass - - def check(collection): - class EvilPickler(self.pickler): - def persistent_id(self, obj): - if isinstance(obj, Clearer): - collection.clear() - return None - pickler = EvilPickler(io.BytesIO(), proto) - try: - pickler.dump(collection) - except RuntimeError as e: - expected = "changed size during iteration" - self.assertIn(expected, str(e)) - - for proto in protocols: - check([Clearer()]) - check([Clearer(), Clearer()]) - check({Clearer()}) - check({Clearer(), Clearer()}) - check({Clearer(): 1}) - check({Clearer(): 1, Clearer(): 2}) - check({1: Clearer(), 2: Clearer()}) - class BigmemPickleTests: @@ -3298,6 +4236,18 @@ def test_huge_str_64b(self, size): # Test classes for reduce_ex +class R: + def __init__(self, reduce=None): + self.reduce = reduce + def __reduce__(self, proto): + return self.reduce + +class REX: + def __init__(self, reduce_ex=None): + self.reduce_ex = reduce_ex + def __reduce_ex__(self, proto): + return self.reduce_ex + class REX_one(object): """No __reduce_ex__ here, but inheriting it from object""" _reduce_called = 0 @@ -3388,6 +4338,19 @@ def __getstate__(self): __setstate__ = None +class CustomError(Exception): + pass + +class Unpickleable: + def __reduce__(self): + raise CustomError + +UNPICKLEABLE = Unpickleable() + +class UnpickleableCallable(Unpickleable): + def __call__(self, *args, **kwargs): + pass + # Test classes for newobj @@ -3437,7 +4400,9 @@ class MyIntWithNew2(MyIntWithNew): class SlotList(MyList): __slots__ = ["foo"] -class SimpleNewObj(int): +# Ruff "redefined while unused" false positive here due to `global` variables +# being assigned (and then restored) from within test methods earlier in the file +class SimpleNewObj(int): # noqa: F811 def __init__(self, *args, **kwargs): # raise an error, to make sure this isn't called raise TypeError("SimpleNewObj.__init__() didn't expect to get called") @@ -3456,6 +4421,12 @@ class BadGetattr: def __getattr__(self, key): self.foo +class NoNew: + def __getattribute__(self, name): + if name == '__new__': + raise AttributeError + return super().__getattribute__(name) + class AbstractPickleModuleTests: @@ -3528,7 +4499,7 @@ def raises_oserror(self, *args, **kwargs): raise OSError @property def bad_property(self): - 1/0 + raise CustomError # File without read and readline class F: @@ -3549,23 +4520,23 @@ class F: class F: read = bad_property readline = raises_oserror - self.assertRaises(ZeroDivisionError, self.Unpickler, F()) + self.assertRaises(CustomError, self.Unpickler, F()) # File with bad readline class F: readline = bad_property read = raises_oserror - self.assertRaises(ZeroDivisionError, self.Unpickler, F()) + self.assertRaises(CustomError, self.Unpickler, F()) # File with bad readline, no read class F: readline = bad_property - self.assertRaises(ZeroDivisionError, self.Unpickler, F()) + self.assertRaises(CustomError, self.Unpickler, F()) # File with bad read, no readline class F: read = bad_property - self.assertRaises((AttributeError, ZeroDivisionError), self.Unpickler, F()) + self.assertRaises((AttributeError, CustomError), self.Unpickler, F()) # File with bad peek class F: @@ -3574,7 +4545,7 @@ class F: readline = raises_oserror try: self.Unpickler(F()) - except ZeroDivisionError: + except CustomError: pass # File with bad readinto @@ -3584,7 +4555,7 @@ class F: readline = raises_oserror try: self.Unpickler(F()) - except ZeroDivisionError: + except CustomError: pass def test_pickler_bad_file(self): @@ -3597,8 +4568,8 @@ class F: class F: @property def write(self): - 1/0 - self.assertRaises(ZeroDivisionError, self.Pickler, F()) + raise CustomError + self.assertRaises(CustomError, self.Pickler, F()) def check_dumps_loads_oob_buffers(self, dumps, loads): # No need to do the full gamut of tests here, just enough to @@ -3706,9 +4677,15 @@ def test_return_correct_type(self): def test_protocol0_is_ascii_only(self): non_ascii_str = "\N{EMPTY SET}" - self.assertRaises(pickle.PicklingError, self.dumps, non_ascii_str, 0) + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(non_ascii_str, 0) + self.assertEqual(str(cm.exception), + 'persistent IDs in protocol 0 must be ASCII strings') pickled = pickle.PERSID + non_ascii_str.encode('utf-8') + b'\n.' - self.assertRaises(pickle.UnpicklingError, self.loads, pickled) + with self.assertRaises(pickle.UnpicklingError) as cm: + self.loads(pickled) + self.assertEqual(str(cm.exception), + 'persistent IDs in protocol 0 must be ASCII strings') class AbstractPicklerUnpicklerObjectTests: @@ -3978,8 +4955,11 @@ class MyClass: # NotImplemented self.assertIs(math_log, math.log) - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: p.dump(g) + self.assertRegex(str(cm.exception), + r'(__reduce__|)' + r' must return (a )?string or tuple') with self.assertRaisesRegex( ValueError, 'The reducer just failed'): diff --git a/Lib/test/pyclbr_input.py b/Lib/test/pyclbr_input.py index 19ccd62dead8ee..5535edbfa7795b 100644 --- a/Lib/test/pyclbr_input.py +++ b/Lib/test/pyclbr_input.py @@ -12,17 +12,19 @@ class B (object): def bm(self): pass class C (B): - foo = Other().foo - om = Other.om - d = 10 - # XXX: This causes test_pyclbr.py to fail, but only because the - # introspection-based is_method() code in the test can't - # distinguish between this and a genuine method function like m(). - # The pyclbr.py module gets this right as it parses the text. + # This one is correctly considered by both test_pyclbr.py and pyclbr.py + # as a non-method of C. + foo = Other().foo + + # This causes test_pyclbr.py to fail, but only because the + # introspection-based is_method() code in the test can't + # distinguish between this and a genuine method function like m(). # - #f = f + # The pyclbr.py module gets this right as it parses the text. + om = Other.om + f = f def m(self): pass @@ -31,3 +33,53 @@ def sm(self): pass @classmethod def cm(self): pass + +# Check that mangling is correctly handled + +class a: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class _: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class __: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class ___: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class _a: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class __a: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 814358746d6d8a..0b2e4b1c1988c4 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -105,9 +105,13 @@ def collect_sys(info_add): ) copy_attributes(info_add, sys, 'sys.%s', attributes) - call_func(info_add, 'sys.androidapilevel', sys, 'getandroidapilevel') - call_func(info_add, 'sys.windowsversion', sys, 'getwindowsversion') - call_func(info_add, 'sys.getrecursionlimit', sys, 'getrecursionlimit') + for func in ( + '_is_gil_enabled', + 'getandroidapilevel', + 'getrecursionlimit', + 'getwindowsversion', + ): + call_func(info_add, f'sys.{func}', sys, func) encoding = sys.getfilesystemencoding() if hasattr(sys, 'getfilesystemencodeerrors'): @@ -179,6 +183,9 @@ def collect_platform(info_add): info_add(f'platform.freedesktop_os_release[{key}]', os_release[key]) + if sys.platform == 'android': + call_func(info_add, 'platform.android_ver', platform, 'android_ver') + def collect_locale(info_add): import locale @@ -287,6 +294,7 @@ def format_groups(groups): "HOMEDRIVE", "HOMEPATH", "IDLESTARTUP", + "IPHONEOS_DEPLOYMENT_TARGET", "LANG", "LDFLAGS", "LDSHARED", @@ -326,6 +334,7 @@ def format_groups(groups): "_PYTHON_HOST_PLATFORM", "_PYTHON_PROJECT_BASE", "_PYTHON_SYSCONFIGDATA_NAME", + "_PYTHON_SYSCONFIGDATA_PATH", "__PYVENV_LAUNCHER__", # Sanitizer options @@ -509,6 +518,7 @@ def collect_sysconfig(info_add): 'MACHDEP', 'MULTIARCH', 'OPT', + 'PGO_PROF_USE_FLAG', 'PY_CFLAGS', 'PY_CFLAGS_NODIST', 'PY_CORE_LDFLAGS', @@ -520,6 +530,7 @@ def collect_sysconfig(info_add): 'Py_GIL_DISABLED', 'SHELL', 'SOABI', + 'TEST_MODULES', 'abs_builddir', 'abs_srcdir', 'prefix', @@ -543,7 +554,6 @@ def collect_sysconfig(info_add): for name in ( 'WITH_DOC_STRINGS', 'WITH_DTRACE', - 'WITH_FREELISTS', 'WITH_MIMALLOC', 'WITH_PYMALLOC', 'WITH_VALGRIND', diff --git a/Lib/test/reperf.py b/Lib/test/reperf.py deleted file mode 100644 index e93bacdb5843c6..00000000000000 --- a/Lib/test/reperf.py +++ /dev/null @@ -1,23 +0,0 @@ -import re -import time - -def main(): - s = "\13hello\14 \13world\14 " * 1000 - p = re.compile(r"([\13\14])") - timefunc(10, p.sub, "", s) - timefunc(10, p.split, s) - timefunc(10, p.findall, s) - -def timefunc(n, func, *args, **kw): - t0 = time.perf_counter() - try: - for i in range(n): - result = func(*args, **kw) - return result - finally: - t1 = time.perf_counter() - if n > 1: - print(n, "times", end=' ') - print(func.__name__, "%.3f" % (t1-t0), "CPU seconds") - -main() diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index cecf309dca9194..4b82d51b4508ac 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -1132,8 +1132,8 @@ def test_capitalize_nonascii(self): self.checkequal('\u2160\u2171\u2172', '\u2170\u2171\u2172', 'capitalize') # check with Ll chars with no upper - nothing changes here - self.checkequal('\u019b\u1d00\u1d86\u0221\u1fb7', - '\u019b\u1d00\u1d86\u0221\u1fb7', 'capitalize') + self.checkequal('\u1d00\u1d86\u0221\u1fb7', + '\u1d00\u1d86\u0221\u1fb7', 'capitalize') def test_startswith(self): self.checkequal(True, 'hello', 'startswith', 'he') @@ -1503,19 +1503,19 @@ def test_find_etc_raise_correct_error_messages(self): # issue 11828 s = 'hello' x = 'x' - self.assertRaisesRegex(TypeError, r'^find\(', s.find, + self.assertRaisesRegex(TypeError, r'^find\b', s.find, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^rfind\(', s.rfind, + self.assertRaisesRegex(TypeError, r'^rfind\b', s.rfind, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^index\(', s.index, + self.assertRaisesRegex(TypeError, r'^index\b', s.index, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^rindex\(', s.rindex, + self.assertRaisesRegex(TypeError, r'^rindex\b', s.rindex, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^count\(', s.count, + self.assertRaisesRegex(TypeError, r'^count\b', s.count, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^startswith\(', s.startswith, + self.assertRaisesRegex(TypeError, r'^startswith\b', s.startswith, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^endswith\(', s.endswith, + self.assertRaisesRegex(TypeError, r'^endswith\b', s.endswith, x, None, None, None) # issue #15534 diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 14e3766a34377b..6436753f998a16 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -4,8 +4,8 @@ raise ImportError('support must be imported from the test package') import contextlib -import dataclasses import functools +import inspect import _opcode import os import re @@ -26,10 +26,10 @@ "Error", "TestFailed", "TestDidNotRun", "ResourceDenied", # io "record_original_stdout", "get_original_stdout", "captured_stdout", - "captured_stdin", "captured_stderr", + "captured_stdin", "captured_stderr", "captured_output", # unittest "is_resource_enabled", "requires", "requires_freebsd_version", - "requires_linux_version", "requires_mac_ver", + "requires_gil_enabled", "requires_linux_version", "requires_mac_ver", "check_syntax_error", "requires_gzip", "requires_bz2", "requires_lzma", "bigmemtest", "bigaddrspacetest", "cpython_only", "get_attribute", @@ -56,9 +56,16 @@ "run_with_tz", "PGO", "missing_compiler_executable", "ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST", "LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT", - "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", "Py_C_RECURSION_LIMIT", + "Py_DEBUG", "exceeds_recursion_limit", "get_c_recursion_limit", "skip_on_s390x", - "without_optimizer", + "requires_jit_enabled", + "requires_jit_disabled", + "force_not_colorized", + "force_not_colorized_test_class", + "make_clean_env", + "BrokenIter", + "in_systemd_nspawn_sync_suppressed", + "run_no_yield_async_fn", "run_yielding_async_fn", "async_yield", ] @@ -250,22 +257,16 @@ class USEROBJECTFLAGS(ctypes.Structure): # process not running under the same user id as the current console # user. To avoid that, raise an exception if the window manager # connection is not available. - from ctypes import cdll, c_int, pointer, Structure - from ctypes.util import find_library - - app_services = cdll.LoadLibrary(find_library("ApplicationServices")) - - if app_services.CGMainDisplayID() == 0: - reason = "gui tests cannot run without OS X window manager" + import subprocess + try: + rc = subprocess.run(["launchctl", "managername"], + capture_output=True, check=True) + managername = rc.stdout.decode("utf-8").strip() + except subprocess.CalledProcessError: + reason = "unable to detect macOS launchd job manager" else: - class ProcessSerialNumber(Structure): - _fields_ = [("highLongOfPSN", c_int), - ("lowLongOfPSN", c_int)] - psn = ProcessSerialNumber() - psn_p = pointer(psn) - if ( (app_services.GetCurrentProcess(psn_p) < 0) or - (app_services.SetFrontProcess(psn_p) < 0) ): - reason = "cannot run without OS X gui process" + if managername != "Aqua": + reason = f"{managername=} -- can only run in a macOS GUI session" # check on every platform whether tkinter can actually do anything if not reason: @@ -387,7 +388,7 @@ def skip_if_buildbot(reason=None): reason = 'not suitable for buildbots' try: isbuildbot = getpass.getuser().lower() == 'buildbot' - except (KeyError, EnvironmentError) as err: + except (KeyError, OSError) as err: warnings.warn(f'getpass.getuser() failed {err}.', RuntimeWarning) isbuildbot = False return unittest.skipIf(isbuildbot, reason) @@ -506,21 +507,27 @@ def requires_lzma(reason='requires lzma'): def has_no_debug_ranges(): try: - import _testinternalcapi + import _testcapi except ImportError: raise unittest.SkipTest("_testinternalcapi required") - config = _testinternalcapi.get_config() + return not _testcapi.config_get('code_debug_ranges') return not bool(config['code_debug_ranges']) def requires_debug_ranges(reason='requires co_positions / debug_ranges'): return unittest.skipIf(has_no_debug_ranges(), reason) + MS_WINDOWS = (sys.platform == 'win32') # Is not actually used in tests, but is kept for compatibility. is_jython = sys.platform.startswith('java') -is_android = hasattr(sys, 'getandroidapilevel') +is_android = sys.platform == "android" + +def skip_android_selinux(name): + return unittest.skipIf( + sys.platform == "android", f"Android blocks {name} with SELinux" + ) if sys.platform not in {"win32", "vxworks", "ios", "tvos", "watchos"}: unix_shell = '/system/bin/sh' if is_android else '/bin/sh' @@ -532,6 +539,9 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'): is_emscripten = sys.platform == "emscripten" is_wasi = sys.platform == "wasi" +def skip_emscripten_stack_overflow(): + return unittest.skipIf(is_emscripten, "Exhausts limited stack on Emscripten") + is_apple_mobile = sys.platform in {"ios", "tvos", "watchos"} is_apple = is_apple_mobile or sys.platform == "darwin" @@ -797,6 +807,16 @@ def disable_gc(): if have_gc: gc.enable() +@contextlib.contextmanager +def gc_threshold(*args): + import gc + old_threshold = gc.get_threshold() + gc.set_threshold(*args) + try: + yield + finally: + gc.set_threshold(*old_threshold) + def python_is_optimized(): """Find if Python was built with optimizations.""" @@ -826,7 +846,27 @@ def check_cflags_pgo(): return any(option in cflags_nodist for option in pgo_options) +def check_bolt_optimized(): + # Always return false, if the platform is WASI, + # because BOLT optimization does not support WASM binary. + if is_wasi: + return False + config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' + return '--enable-bolt' in config_args + + Py_GIL_DISABLED = bool(sysconfig.get_config_var('Py_GIL_DISABLED')) + +def requires_gil_enabled(msg="needs the GIL enabled"): + """Decorator for skipping tests on the free-threaded build.""" + return unittest.skipIf(Py_GIL_DISABLED, msg) + +def expected_failure_if_gil_disabled(): + """Expect test failure if the GIL is disabled.""" + if Py_GIL_DISABLED: + return unittest.expectedFailure + return lambda test_case: test_case + if Py_GIL_DISABLED: _header = 'PHBBInP' else: @@ -843,8 +883,16 @@ def calcvobjsize(fmt): return struct.calcsize(_vheader + fmt + _align) -_TPFLAGS_HAVE_GC = 1<<14 +_TPFLAGS_STATIC_BUILTIN = 1<<1 +_TPFLAGS_DISALLOW_INSTANTIATION = 1<<7 +_TPFLAGS_IMMUTABLETYPE = 1<<8 _TPFLAGS_HEAPTYPE = 1<<9 +_TPFLAGS_BASETYPE = 1<<10 +_TPFLAGS_READY = 1<<12 +_TPFLAGS_READYING = 1<<13 +_TPFLAGS_HAVE_GC = 1<<14 +_TPFLAGS_BASE_EXC_SUBCLASS = 1<<30 +_TPFLAGS_TYPE_SUBCLASS = 1<<31 def check_sizeof(test, o, size): try: @@ -861,8 +909,8 @@ def check_sizeof(test, o, size): test.assertEqual(result, size, msg) #======================================================================= -# Decorator for running a function in a different locale, correctly resetting -# it afterwards. +# Decorator/context manager for running a code in a different locale, +# correctly resetting it afterwards. @contextlib.contextmanager def run_with_locale(catstr, *locales): @@ -873,16 +921,21 @@ def run_with_locale(catstr, *locales): except AttributeError: # if the test author gives us an invalid category string raise - except: + except Exception: # cannot retrieve original locale, so do nothing locale = orig_locale = None + if '' not in locales: + raise unittest.SkipTest('no locales') else: for loc in locales: try: locale.setlocale(category, loc) break - except: + except locale.Error: pass + else: + if '' not in locales: + raise unittest.SkipTest(f'no locales {locales}') try: yield @@ -890,6 +943,46 @@ def run_with_locale(catstr, *locales): if locale and orig_locale: locale.setlocale(category, orig_locale) +#======================================================================= +# Decorator for running a function in multiple locales (if they are +# availasble) and resetting the original locale afterwards. + +def run_with_locales(catstr, *locales): + def deco(func): + @functools.wraps(func) + def wrapper(self, /, *args, **kwargs): + dry_run = '' in locales + try: + import locale + category = getattr(locale, catstr) + orig_locale = locale.setlocale(category) + except AttributeError: + # if the test author gives us an invalid category string + raise + except Exception: + # cannot retrieve original locale, so do nothing + pass + else: + try: + for loc in locales: + with self.subTest(locale=loc): + try: + locale.setlocale(category, loc) + except locale.Error: + self.skipTest(f'no locale {loc!r}') + else: + dry_run = False + func(self, *args, **kwargs) + finally: + locale.setlocale(category, orig_locale) + if dry_run: + # no locales available, so just run the test + # with the current locale + with self.subTest(locale=None): + func(self, *args, **kwargs) + return wrapper + return deco + #======================================================================= # Decorator for running a function in a specific timezone, correctly # resetting it afterwards. @@ -1139,6 +1232,26 @@ def coverage_wrapper(*args, **kwargs): return coverage_wrapper +def no_rerun(reason): + """Skip rerunning for a particular test. + + WARNING: Use this decorator with care; skipping rerunning makes it + impossible to find reference leaks. Provide a clear reason for skipping the + test using the 'reason' parameter. + """ + def deco(func): + assert not isinstance(func, type), func + _has_run = False + def wrapper(self): + nonlocal _has_run + if _has_run: + self.skipTest(reason) + func(self) + _has_run = True + return wrapper + return deco + + def refcount_test(test): """Decorator for tests which involve reference counting. @@ -1152,16 +1265,27 @@ def refcount_test(test): def requires_limited_api(test): try: - import _testcapi + import _testcapi # noqa: F401 + import _testlimitedcapi # noqa: F401 except ImportError: - return unittest.skip('needs _testcapi module')(test) + return unittest.skip('needs _testcapi and _testlimitedcapi modules')(test) return test + +# Windows build doesn't support --disable-test-modules feature, so there's no +# 'TEST_MODULES' var in config +TEST_MODULES_ENABLED = (sysconfig.get_config_var('TEST_MODULES') or 'yes') == 'yes' + def requires_specialization(test): return unittest.skipUnless( _opcode.ENABLE_SPECIALIZATION, "requires specialization")(test) +def requires_specialization_ft(test): + return unittest.skipUnless( + _opcode.ENABLE_SPECIALIZATION_FT, "requires specialization")(test) + + #======================================================================= # Check for the presence of docstrings. @@ -1704,7 +1828,10 @@ def run_in_subinterp(code): module is enabled. """ _check_tracemalloc() - import _testcapi + try: + import _testcapi + except ImportError: + raise unittest.SkipTest("requires _testcapi") return _testcapi.run_in_subinterp(code) @@ -1714,11 +1841,25 @@ def run_in_subinterp_with_config(code, *, own_gil=None, **config): module is enabled. """ _check_tracemalloc() - import _testinternalcapi + try: + import _testinternalcapi + except ImportError: + raise unittest.SkipTest("requires _testinternalcapi") if own_gil is not None: assert 'gil' not in config, (own_gil, config) - config['gil'] = 2 if own_gil else 1 - return _testinternalcapi.run_in_subinterp_with_config(code, **config) + config['gil'] = 'own' if own_gil else 'shared' + else: + gil = config['gil'] + if gil == 0: + config['gil'] = 'default' + elif gil == 1: + config['gil'] = 'shared' + elif gil == 2: + config['gil'] = 'own' + elif not isinstance(gil, str): + raise NotImplementedError(gil) + config = types.SimpleNamespace(**config) + return _testinternalcapi.run_in_subinterp_with_config(code, config) def _check_tracemalloc(): @@ -1790,18 +1931,18 @@ def missing_compiler_executable(cmd_names=[]): return cmd[0] -_is_android_emulator = None +_old_android_emulator = None def setswitchinterval(interval): # Setting a very low gil interval on the Android emulator causes python # to hang (issue #26939). - minimum_interval = 1e-5 + minimum_interval = 1e-4 # 100 us if is_android and interval < minimum_interval: - global _is_android_emulator - if _is_android_emulator is None: - import subprocess - _is_android_emulator = (subprocess.check_output( - ['getprop', 'ro.kernel.qemu']).strip() == b'1') - if _is_android_emulator: + global _old_android_emulator + if _old_android_emulator is None: + import platform + av = platform.android_ver() + _old_android_emulator = av.is_emulator and av.api_level < 24 + if _old_android_emulator: interval = minimum_interval return sys.setswitchinterval(interval) @@ -1876,12 +2017,18 @@ def restore(self): def with_pymalloc(): - import _testcapi + try: + import _testcapi + except ImportError: + raise unittest.SkipTest("requires _testcapi") return _testcapi.WITH_PYMALLOC and not Py_GIL_DISABLED def with_mimalloc(): - import _testcapi + try: + import _testcapi + except ImportError: + raise unittest.SkipTest("requires _testcapi") return _testcapi.WITH_MIMALLOC @@ -2091,7 +2238,15 @@ def skip_if_broken_multiprocessing_synchronize(): # bpo-38377: On Linux, creating a semaphore fails with OSError # if the current user does not have the permission to create # a file in /dev/shm/ directory. - synchronize.Lock(ctx=None) + import multiprocessing + synchronize.Lock(ctx=multiprocessing.get_context('fork')) + # The explicit fork mp context is required in order for + # TestResourceTracker.test_resource_tracker_reused to work. + # synchronize creates a new multiprocessing.resource_tracker + # process at module import time via the above call in that + # scenario. Awkward. This enables gh-84559. No code involved + # should have threads at that point so fork() should be safe. + except OSError as exc: raise unittest.SkipTest(f"broken multiprocessing SemLock: {exc!r}") @@ -2201,7 +2356,7 @@ def clear_ignored_deprecations(*tokens: object) -> None: def requires_venv_with_pip(): # ensurepip requires zlib to open ZIP archives (.whl binary wheel packages) try: - import zlib + import zlib # noqa: F401 except ImportError: return unittest.skipIf(True, "venv: ensurepip requires zlib") @@ -2240,16 +2395,25 @@ def _findwheel(pkgname): # and returns the path to the venv directory and the path to the python executable @contextlib.contextmanager def setup_venv_with_pip_setuptools_wheel(venv_dir): + import shlex import subprocess from .os_helper import temp_cwd + def run_command(cmd): + if verbose: + print() + print('Run:', ' '.join(map(shlex.quote, cmd))) + subprocess.run(cmd, check=True) + else: + subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True) + with temp_cwd() as temp_dir: # Create virtual environment to get setuptools cmd = [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir] - if verbose: - print() - print('Run:', ' '.join(cmd)) - subprocess.run(cmd, check=True) + run_command(cmd) venv = os.path.join(temp_dir, venv_dir) @@ -2264,10 +2428,7 @@ def setup_venv_with_pip_setuptools_wheel(venv_dir): '-m', 'pip', 'install', _findwheel('setuptools'), _findwheel('wheel')] - if verbose: - print() - print('Run:', ' '.join(cmd)) - subprocess.run(cmd, check=True) + run_command(cmd) yield python @@ -2441,44 +2602,32 @@ def adjust_int_max_str_digits(max_digits): sys.set_int_max_str_digits(current) -def _get_c_recursion_limit(): +def get_c_recursion_limit(): try: import _testcapi return _testcapi.Py_C_RECURSION_LIMIT - except (ImportError, AttributeError): - # Originally taken from Include/cpython/pystate.h . - if sys.platform == 'win32': - return 4000 - else: - return 10000 + except ImportError: + raise unittest.SkipTest('requires _testcapi') + -# The default C recursion limit. -Py_C_RECURSION_LIMIT = _get_c_recursion_limit() +def exceeds_recursion_limit(): + """For recursion tests, easily exceeds default recursion limit.""" + return get_c_recursion_limit() * 3 -#For recursion tests, easily exceeds default recursion limit -EXCEEDS_RECURSION_LIMIT = Py_C_RECURSION_LIMIT * 3 -#Windows doesn't have os.uname() but it doesn't support s390x. -skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', - 'skipped on s390x') +# Windows doesn't have os.uname() but it doesn't support s390x. +is_s390x = hasattr(os, 'uname') and os.uname().machine == 's390x' +skip_on_s390x = unittest.skipIf(is_s390x, 'skipped on s390x') Py_TRACE_REFS = hasattr(sys, 'getobjects') -# Decorator to disable optimizer while a function run -def without_optimizer(func): - try: - import _testinternalcapi - except ImportError: - return func - @functools.wraps(func) - def wrapper(*args, **kwargs): - save_opt = _testinternalcapi.get_optimizer() - try: - _testinternalcapi.set_optimizer(None) - return func(*args, **kwargs) - finally: - _testinternalcapi.set_optimizer(save_opt) - return wrapper +try: + from _testinternalcapi import jit_enabled +except ImportError: + requires_jit_enabled = requires_jit_disabled = unittest.skip("requires _testinternalcapi") +else: + requires_jit_enabled = unittest.skipUnless(jit_enabled(), "requires JIT enabled") + requires_jit_disabled = unittest.skipIf(jit_enabled(), "requires JIT disabled") _BASE_COPY_SRC_DIR_IGNORED_NAMES = frozenset({ @@ -2506,3 +2655,344 @@ def copy_python_src_ignore(path, names): 'build', } return ignored + + +# XXX Move this to the inspect module? +def walk_class_hierarchy(top, *, topdown=True): + # This is based on the logic in os.walk(). + assert isinstance(top, type), repr(top) + stack = [top] + while stack: + top = stack.pop() + if isinstance(top, tuple): + yield top + continue + + subs = type(top).__subclasses__(top) + if topdown: + # Yield before subclass traversal if going top down. + yield top, subs + # Traverse into subclasses. + for sub in reversed(subs): + stack.append(sub) + else: + # Yield after subclass traversal if going bottom up. + stack.append((top, subs)) + # Traverse into subclasses. + for sub in reversed(subs): + stack.append(sub) + + +def iter_builtin_types(): + # First try the explicit route. + try: + import _testinternalcapi + except ImportError: + _testinternalcapi = None + if _testinternalcapi is not None: + yield from _testinternalcapi.get_static_builtin_types() + return + + # Fall back to making a best-effort guess. + if hasattr(object, '__flags__'): + # Look for any type object with the Py_TPFLAGS_STATIC_BUILTIN flag set. + import datetime + seen = set() + for cls, subs in walk_class_hierarchy(object): + if cls in seen: + continue + seen.add(cls) + if not (cls.__flags__ & _TPFLAGS_STATIC_BUILTIN): + # Do not walk its subclasses. + subs[:] = [] + continue + yield cls + else: + # Fall back to a naive approach. + seen = set() + for obj in __builtins__.values(): + if not isinstance(obj, type): + continue + cls = obj + # XXX? + if cls.__module__ != 'builtins': + continue + if cls == ExceptionGroup: + # It's a heap type. + continue + if cls in seen: + continue + seen.add(cls) + yield cls + + +# XXX Move this to the inspect module? +def iter_name_in_mro(cls, name): + """Yield matching items found in base.__dict__ across the MRO. + + The descriptor protocol is not invoked. + + list(iter_name_in_mro(cls, name))[0] is roughly equivalent to + find_name_in_mro() in Objects/typeobject.c (AKA PyType_Lookup()). + + inspect.getattr_static() is similar. + """ + # This can fail if "cls" is weird. + for base in inspect._static_getmro(cls): + # This can fail if "base" is weird. + ns = inspect._get_dunder_dict_of_class(base) + try: + obj = ns[name] + except KeyError: + continue + yield obj, base + + +# XXX Move this to the inspect module? +def find_name_in_mro(cls, name, default=inspect._sentinel): + for res in iter_name_in_mro(cls, name): + # Return the first one. + return res + if default is not inspect._sentinel: + return default, None + raise AttributeError(name) + + +# XXX The return value should always be exactly the same... +def identify_type_slot_wrappers(): + try: + import _testinternalcapi + except ImportError: + _testinternalcapi = None + if _testinternalcapi is not None: + names = {n: None for n in _testinternalcapi.identify_type_slot_wrappers()} + return list(names) + else: + raise NotImplementedError + + +def iter_slot_wrappers(cls): + def is_slot_wrapper(name, value): + if not isinstance(value, types.WrapperDescriptorType): + assert not repr(value).startswith(' dict[str, str]: + clean_env = os.environ.copy() + for k in clean_env.copy(): + if k.startswith("PYTHON"): + clean_env.pop(k) + clean_env.pop("FORCE_COLOR", None) + clean_env.pop("NO_COLOR", None) + return clean_env + + +def initialized_with_pyrepl(): + """Detect whether PyREPL was used during Python initialization.""" + # If the main module has a __file__ attribute it's a Python module, which means PyREPL. + return hasattr(sys.modules["__main__"], "__file__") + + +WINDOWS_STATUS = { + 0xC0000005: "STATUS_ACCESS_VIOLATION", + 0xC00000FD: "STATUS_STACK_OVERFLOW", + 0xC000013A: "STATUS_CONTROL_C_EXIT", +} + +def get_signal_name(exitcode): + import signal + + if exitcode < 0: + signum = -exitcode + try: + return signal.Signals(signum).name + except ValueError: + pass + + # Shell exit code (ex: WASI build) + if 128 < exitcode < 256: + signum = exitcode - 128 + try: + return signal.Signals(signum).name + except ValueError: + pass + + try: + return WINDOWS_STATUS[exitcode] + except KeyError: + pass + + return None + +class BrokenIter: + def __init__(self, init_raises=False, next_raises=False, iter_raises=False): + if init_raises: + 1/0 + self.next_raises = next_raises + self.iter_raises = iter_raises + + def __next__(self): + if self.next_raises: + 1/0 + + def __iter__(self): + if self.iter_raises: + 1/0 + return self + + +def in_systemd_nspawn_sync_suppressed() -> bool: + """ + Test whether the test suite is runing in systemd-nspawn + with ``--suppress-sync=true``. + + This can be used to skip tests that rely on ``fsync()`` calls + and similar not being intercepted. + """ + + if not hasattr(os, "O_SYNC"): + return False + + try: + with open("/run/systemd/container", "rb") as fp: + if fp.read().rstrip() != b"systemd-nspawn": + return False + except FileNotFoundError: + return False + + # If systemd-nspawn is used, O_SYNC flag will immediately + # trigger EINVAL. Otherwise, ENOENT will be given instead. + import errno + try: + fd = os.open(__file__, os.O_RDONLY | os.O_SYNC) + except OSError as err: + if err.errno == errno.EINVAL: + return True + else: + os.close(fd) + + return False + +def run_no_yield_async_fn(async_fn, /, *args, **kwargs): + coro = async_fn(*args, **kwargs) + try: + coro.send(None) + except StopIteration as e: + return e.value + else: + raise AssertionError("coroutine did not complete") + finally: + coro.close() + + +@types.coroutine +def async_yield(v): + return (yield v) + + +def run_yielding_async_fn(async_fn, /, *args, **kwargs): + coro = async_fn(*args, **kwargs) + try: + while True: + try: + coro.send(None) + except StopIteration as e: + return e.value + finally: + coro.close() + + +def is_libssl_fips_mode(): + try: + from _hashlib import get_fips_mode # ask _hashopenssl.c + except ImportError: + return False # more of a maybe, unless we add this to the _ssl module. + return get_fips_mode() != 0 diff --git a/Lib/test/support/asynchat.py b/Lib/test/support/asynchat.py index 38c47a1fda683e..a8c6b28a9e1a91 100644 --- a/Lib/test/support/asynchat.py +++ b/Lib/test/support/asynchat.py @@ -1,5 +1,5 @@ # TODO: This module was deprecated and removed from CPython 3.12 -# Now it is a test-only helper. Any attempts to rewrite exising tests that +# Now it is a test-only helper. Any attempts to rewrite existing tests that # are using this module and remove it completely are appreciated! # See: https://github.com/python/cpython/issues/72719 diff --git a/Lib/test/support/asyncore.py b/Lib/test/support/asyncore.py index b397aca5568079..870e42837640de 100644 --- a/Lib/test/support/asyncore.py +++ b/Lib/test/support/asyncore.py @@ -1,5 +1,5 @@ # TODO: This module was deprecated and removed from CPython 3.12 -# Now it is a test-only helper. Any attempts to rewrite exising tests that +# Now it is a test-only helper. Any attempts to rewrite existing tests that # are using this module and remove it completely are appreciated! # See: https://github.com/python/cpython/issues/72719 diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index a4845065a5322e..f6426c3e285b2d 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -3,7 +3,11 @@ import unittest import dis import io -from _testinternalcapi import compiler_codegen, optimize_cfg, assemble_code_object +import opcode +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None _UNSPECIFIED = object() @@ -65,16 +69,14 @@ class CompilationStepTestCase(unittest.TestCase): class Label: pass - def assertInstructionsMatch(self, actual_, expected_): - # get two lists where each entry is a label or - # an instruction tuple. Normalize the labels to the - # instruction count of the target, and compare the lists. + def assertInstructionsMatch(self, actual_seq, expected): + # get an InstructionSequence and an expected list, where each + # entry is a label or an instruction tuple. Construct an expected + # instruction sequence and compare with the one given. - self.assertIsInstance(actual_, list) - self.assertIsInstance(expected_, list) - - actual = self.normalize_insts(actual_) - expected = self.normalize_insts(expected_) + self.assertIsInstance(expected, list) + actual = actual_seq.get_instructions() + expected = self.seq_from_insts(expected).get_instructions() self.assertEqual(len(actual), len(expected)) # compare instructions @@ -84,10 +86,8 @@ def assertInstructionsMatch(self, actual_, expected_): continue self.assertIsInstance(exp, tuple) self.assertIsInstance(act, tuple) - # crop comparison to the provided expected values - if len(act) > len(exp): - act = act[:len(exp)] - self.assertEqual(exp, act) + idx = max([p[0] for p in enumerate(exp) if p[1] != -1]) + self.assertEqual(exp[:idx], act[:idx]) def resolveAndRemoveLabels(self, insts): idx = 0 @@ -102,54 +102,57 @@ def resolveAndRemoveLabels(self, insts): return res - def normalize_insts(self, insts): - """ Map labels to instruction index. - Map opcodes to opnames. - """ - insts = self.resolveAndRemoveLabels(insts) - res = [] - for item in insts: - assert isinstance(item, tuple) - opcode, oparg, *loc = item - opcode = dis.opmap.get(opcode, opcode) - if isinstance(oparg, self.Label): - arg = oparg.value - else: - arg = oparg if opcode in self.HAS_ARG else None - opcode = dis.opname[opcode] - res.append((opcode, arg, *loc)) - return res + def seq_from_insts(self, insts): + labels = {item for item in insts if isinstance(item, self.Label)} + for i, lbl in enumerate(labels): + lbl.value = i - def complete_insts_info(self, insts): - # fill in omitted fields in location, and oparg 0 for ops with no arg. - res = [] + seq = _testinternalcapi.new_instruction_sequence() for item in insts: - assert isinstance(item, tuple) - inst = list(item) - opcode = dis.opmap[inst[0]] - oparg = inst[1] - loc = inst[2:] + [-1] * (6 - len(inst)) - res.append((opcode, oparg, *loc)) - return res + if isinstance(item, self.Label): + seq.use_label(item.value) + else: + op = item[0] + if isinstance(op, str): + op = opcode.opmap[op] + arg, *loc = item[1:] + if isinstance(arg, self.Label): + arg = arg.value + loc = loc + [-1] * (4 - len(loc)) + seq.addop(op, arg or 0, *loc) + return seq + + def check_instructions(self, insts): + for inst in insts: + if isinstance(inst, self.Label): + continue + op, arg, *loc = inst + if isinstance(op, str): + op = opcode.opmap[op] + self.assertEqual(op in opcode.hasarg, + arg is not None, + f"{opcode.opname[op]=} {arg=}") + self.assertTrue(all(isinstance(l, int) for l in loc)) +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class CodegenTestCase(CompilationStepTestCase): def generate_code(self, ast): - insts, _ = compiler_codegen(ast, "my_file.py", 0) + insts, _ = _testinternalcapi.compiler_codegen(ast, "my_file.py", 0) return insts +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class CfgOptimizationTestCase(CompilationStepTestCase): - def get_optimized(self, insts, consts, nlocals=0): - insts = self.normalize_insts(insts) - insts = self.complete_insts_info(insts) - insts = optimize_cfg(insts, consts, nlocals) + def get_optimized(self, seq, consts, nlocals=0): + insts = _testinternalcapi.optimize_cfg(seq, consts, nlocals) return insts, consts +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class AssemblerTestCase(CompilationStepTestCase): def get_code_object(self, filename, insts, metadata): - co = assemble_code_object(filename, insts, metadata) + co = _testinternalcapi.assemble_code_object(filename, insts, metadata) return co diff --git a/Lib/test/support/hypothesis_helper.py b/Lib/test/support/hypothesis_helper.py index db93eea5e912e0..40f58a2f59c6c3 100644 --- a/Lib/test/support/hypothesis_helper.py +++ b/Lib/test/support/hypothesis_helper.py @@ -5,6 +5,13 @@ except ImportError: from . import _hypothesis_stubs as hypothesis else: + # Regrtest changes to use a tempdir as the working directory, so we have + # to tell Hypothesis to use the original in order to persist the database. + from .os_helper import SAVEDCWD + from hypothesis.configuration import set_hypothesis_home_dir + + set_hypothesis_home_dir(os.path.join(SAVEDCWD, ".hypothesis")) + # When using the real Hypothesis, we'll configure it to ignore occasional # slow tests (avoiding flakiness from random VM slowness in CI). hypothesis.settings.register_profile( diff --git a/Lib/test/support/i18n_helper.py b/Lib/test/support/i18n_helper.py new file mode 100644 index 00000000000000..2e304f29e8ba7f --- /dev/null +++ b/Lib/test/support/i18n_helper.py @@ -0,0 +1,63 @@ +import re +import subprocess +import sys +import unittest +from pathlib import Path +from test.support import REPO_ROOT, TEST_HOME_DIR, requires_subprocess +from test.test_tools import skip_if_missing + + +pygettext = Path(REPO_ROOT) / 'Tools' / 'i18n' / 'pygettext.py' + +msgid_pattern = re.compile(r'msgid(.*?)(?:msgid_plural|msgctxt|msgstr)', + re.DOTALL) +msgid_string_pattern = re.compile(r'"((?:\\"|[^"])*)"') + + +def _generate_po_file(path, *, stdout_only=True): + res = subprocess.run([sys.executable, pygettext, + '--no-location', '-o', '-', path], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True) + if stdout_only: + return res.stdout + return res + + +def _extract_msgids(po): + msgids = [] + for msgid in msgid_pattern.findall(po): + msgid_string = ''.join(msgid_string_pattern.findall(msgid)) + msgid_string = msgid_string.replace(r'\"', '"') + if msgid_string: + msgids.append(msgid_string) + return sorted(msgids) + + +def _get_snapshot_path(module_name): + return Path(TEST_HOME_DIR) / 'translationdata' / module_name / 'msgids.txt' + + +@requires_subprocess() +class TestTranslationsBase(unittest.TestCase): + + def assertMsgidsEqual(self, module): + '''Assert that msgids extracted from a given module match a + snapshot. + + ''' + skip_if_missing('i18n') + res = _generate_po_file(module.__file__, stdout_only=False) + self.assertEqual(res.returncode, 0) + self.assertEqual(res.stderr, '') + msgids = _extract_msgids(res.stdout) + snapshot_path = _get_snapshot_path(module.__name__) + snapshot = snapshot_path.read_text().splitlines() + self.assertListEqual(msgids, snapshot) + + +def update_translation_snapshots(module): + contents = _generate_po_file(module.__file__) + msgids = _extract_msgids(contents) + snapshot_path = _get_snapshot_path(module.__name__) + snapshot_path.write_text('\n'.join(msgids)) diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 3d804f2b590108..2b91bdcf9cd859 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -58,8 +58,8 @@ def make_legacy_pyc(source): :return: The file system path to the legacy pyc file. """ pyc_file = importlib.util.cache_from_source(source) - up_one = os.path.dirname(os.path.abspath(source)) - legacy_pyc = os.path.join(up_one, source + 'c') + assert source.endswith('.py') + legacy_pyc = source + 'c' shutil.move(pyc_file, legacy_pyc) return legacy_pyc @@ -114,7 +114,7 @@ def multi_interp_extensions_check(enabled=True): This only applies to modules that haven't been imported yet. It overrides the PyInterpreterConfig.check_multi_interp_extensions setting (see support.run_in_subinterp_with_config() and - _xxsubinterpreters.create()). + _interpreters.create()). Also see importlib.utils.allowing_all_extensions(). """ @@ -268,6 +268,18 @@ def modules_cleanup(oldmodules): sys.modules.update(oldmodules) +@contextlib.contextmanager +def isolated_modules(): + """ + Save modules on entry and cleanup on exit. + """ + (saved,) = modules_setup() + try: + yield + finally: + modules_cleanup(saved) + + def mock_register_at_fork(func): # bpo-30599: Mock os.register_at_fork() when importing the random module, # since this function doesn't allow to unregister callbacks and would leak diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index d02ffbae1113c0..e067f259364d2a 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -2,10 +2,10 @@ import threading import weakref -import _xxsubinterpreters as _interpreters +import _interpreters # aliases: -from _xxsubinterpreters import ( +from _interpreters import ( InterpreterError, InterpreterNotFoundError, NotShareableError, is_shareable, ) @@ -43,7 +43,7 @@ def __getattr__(name): {formatted} """.strip() -class ExecutionFailed(RuntimeError): +class ExecutionFailed(InterpreterError): """An unhandled exception happened during execution. This is raised from Interpreter.exec() and Interpreter.call(). @@ -73,51 +73,78 @@ def __str__(self): def create(): """Return a new (idle) Python interpreter.""" - id = _interpreters.create(isolated=True) - return Interpreter(id) + id = _interpreters.create(reqrefs=True) + return Interpreter(id, _ownsref=True) def list_all(): """Return all existing interpreters.""" - return [Interpreter(id) for id in _interpreters.list_all()] + return [Interpreter(id, _whence=whence) + for id, whence in _interpreters.list_all(require_ready=True)] def get_current(): """Return the currently running interpreter.""" - id = _interpreters.get_current() - return Interpreter(id) + id, whence = _interpreters.get_current() + return Interpreter(id, _whence=whence) def get_main(): """Return the main interpreter.""" - id = _interpreters.get_main() - return Interpreter(id) + id, whence = _interpreters.get_main() + assert whence == _interpreters.WHENCE_RUNTIME, repr(whence) + return Interpreter(id, _whence=whence) _known = weakref.WeakValueDictionary() class Interpreter: - """A single Python interpreter.""" + """A single Python interpreter. - def __new__(cls, id, /): + Attributes: + + "id" - the unique process-global ID number for the interpreter + "whence" - indicates where the interpreter was created + + If the interpreter wasn't created by this module + then any method that modifies the interpreter will fail, + i.e. .close(), .prepare_main(), .exec(), and .call() + """ + + _WHENCE_TO_STR = { + _interpreters.WHENCE_UNKNOWN: 'unknown', + _interpreters.WHENCE_RUNTIME: 'runtime init', + _interpreters.WHENCE_LEGACY_CAPI: 'legacy C-API', + _interpreters.WHENCE_CAPI: 'C-API', + _interpreters.WHENCE_XI: 'cross-interpreter C-API', + _interpreters.WHENCE_STDLIB: '_interpreters module', + } + + def __new__(cls, id, /, _whence=None, _ownsref=None): # There is only one instance for any given ID. if not isinstance(id, int): raise TypeError(f'id must be an int, got {id!r}') id = int(id) + if _whence is None: + if _ownsref: + _whence = _interpreters.WHENCE_STDLIB + else: + _whence = _interpreters.whence(id) + assert _whence in cls._WHENCE_TO_STR, repr(_whence) + if _ownsref is None: + _ownsref = (_whence == _interpreters.WHENCE_STDLIB) try: self = _known[id] assert hasattr(self, '_ownsref') except KeyError: - # This may raise InterpreterNotFoundError: - _interpreters._incref(id) - try: - self = super().__new__(cls) - self._id = id - self._ownsref = True - except BaseException: - _interpreters._deccref(id) - raise + self = super().__new__(cls) _known[id] = self + self._id = id + self._whence = _whence + self._ownsref = _ownsref + if _ownsref: + # This may raise InterpreterNotFoundError: + _interpreters.incref(id) return self def __repr__(self): @@ -129,12 +156,20 @@ def __hash__(self): def __del__(self): self._decref() + # for pickling: + def __getnewargs__(self): + return (self._id,) + + # for pickling: + def __getstate__(self): + return None + def _decref(self): if not self._ownsref: return self._ownsref = False try: - _interpreters._decref(self.id) + _interpreters.decref(self._id) except InterpreterNotFoundError: pass @@ -142,17 +177,24 @@ def _decref(self): def id(self): return self._id + @property + def whence(self): + return self._WHENCE_TO_STR[self._whence] + def is_running(self): """Return whether or not the identified interpreter is running.""" return _interpreters.is_running(self._id) + # Everything past here is available only to interpreters created by + # interpreters.create(). + def close(self): """Finalize and destroy the interpreter. Attempting to destroy the current interpreter results - in a RuntimeError. + in an InterpreterError. """ - return _interpreters.destroy(self._id) + return _interpreters.destroy(self._id, restrict=True) def prepare_main(self, ns=None, /, **kwargs): """Bind the given values into the interpreter's __main__. @@ -160,7 +202,7 @@ def prepare_main(self, ns=None, /, **kwargs): The values must be shareable. """ ns = dict(ns, **kwargs) if ns is not None else kwargs - _interpreters.set___main___attrs(self._id, ns) + _interpreters.set___main___attrs(self._id, ns, restrict=True) def exec(self, code, /): """Run the given source code in the interpreter. @@ -180,7 +222,7 @@ def exec(self, code, /): that time, the previous interpreter is allowed to run in other threads. """ - excinfo = _interpreters.exec(self._id, code) + excinfo = _interpreters.exec(self._id, code, restrict=True) if excinfo is not None: raise ExecutionFailed(excinfo) @@ -200,7 +242,7 @@ def call(self, callable, /): # XXX Support args and kwargs. # XXX Support arbitrary callables. # XXX Support returning the return value (e.g. via pickle). - excinfo = _interpreters.call(self._id, callable) + excinfo = _interpreters.call(self._id, callable, restrict=True) if excinfo is not None: raise ExecutionFailed(excinfo) diff --git a/Lib/test/support/interpreters/_crossinterp.py b/Lib/test/support/interpreters/_crossinterp.py new file mode 100644 index 00000000000000..544e197ba4c028 --- /dev/null +++ b/Lib/test/support/interpreters/_crossinterp.py @@ -0,0 +1,102 @@ +"""Common code between queues and channels.""" + + +class ItemInterpreterDestroyed(Exception): + """Raised when trying to get an item whose interpreter was destroyed.""" + + +class classonly: + """A non-data descriptor that makes a value only visible on the class. + + This is like the "classmethod" builtin, but does not show up on + instances of the class. It may be used as a decorator. + """ + + def __init__(self, value): + self.value = value + self.getter = classmethod(value).__get__ + self.name = None + + def __set_name__(self, cls, name): + if self.name is not None: + raise TypeError('already used') + self.name = name + + def __get__(self, obj, cls): + if obj is not None: + raise AttributeError(self.name) + # called on the class + return self.getter(None, cls) + + +class UnboundItem: + """Represents a cross-interpreter item no longer bound to an interpreter. + + An item is unbound when the interpreter that added it to the + cross-interpreter container is destroyed. + """ + + __slots__ = () + + @classonly + def singleton(cls, kind, module, name='UNBOUND'): + doc = cls.__doc__.replace('cross-interpreter container', kind) + doc = doc.replace('cross-interpreter', kind) + subclass = type( + f'Unbound{kind.capitalize()}Item', + (cls,), + dict( + _MODULE=module, + _NAME=name, + __doc__=doc, + ), + ) + return object.__new__(subclass) + + _MODULE = __name__ + _NAME = 'UNBOUND' + + def __new__(cls): + raise Exception(f'use {cls._MODULE}.{cls._NAME}') + + def __repr__(self): + return f'{self._MODULE}.{self._NAME}' +# return f'interpreters.queues.UNBOUND' + + +UNBOUND = object.__new__(UnboundItem) +UNBOUND_ERROR = object() +UNBOUND_REMOVE = object() + +_UNBOUND_CONSTANT_TO_FLAG = { + UNBOUND_REMOVE: 1, + UNBOUND_ERROR: 2, + UNBOUND: 3, +} +_UNBOUND_FLAG_TO_CONSTANT = {v: k + for k, v in _UNBOUND_CONSTANT_TO_FLAG.items()} + + +def serialize_unbound(unbound): + op = unbound + try: + flag = _UNBOUND_CONSTANT_TO_FLAG[op] + except KeyError: + raise NotImplementedError(f'unsupported unbound replacement op {op!r}') + return flag, + + +def resolve_unbound(flag, exctype_destroyed): + try: + op = _UNBOUND_FLAG_TO_CONSTANT[flag] + except KeyError: + raise NotImplementedError(f'unsupported unbound replacement op {flag!r}') + if op is UNBOUND_REMOVE: + # "remove" not possible here + raise NotImplementedError + elif op is UNBOUND_ERROR: + raise exctype_destroyed("item's original interpreter destroyed") + elif op is UNBOUND: + return UNBOUND + else: + raise NotImplementedError(repr(op)) diff --git a/Lib/test/support/interpreters/channels.py b/Lib/test/support/interpreters/channels.py index 75a5a60f54f926..d2bd93d77f7169 100644 --- a/Lib/test/support/interpreters/channels.py +++ b/Lib/test/support/interpreters/channels.py @@ -1,36 +1,69 @@ """Cross-interpreter Channels High Level Module.""" import time -import _xxinterpchannels as _channels +import _interpchannels as _channels +from . import _crossinterp # aliases: -from _xxinterpchannels import ( +from _interpchannels import ( ChannelError, ChannelNotFoundError, ChannelClosedError, ChannelEmptyError, ChannelNotEmptyError, ) +from ._crossinterp import ( + UNBOUND_ERROR, UNBOUND_REMOVE, +) __all__ = [ + 'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE', 'create', 'list_all', 'SendChannel', 'RecvChannel', 'ChannelError', 'ChannelNotFoundError', 'ChannelEmptyError', + 'ItemInterpreterDestroyed', ] -def create(): +class ItemInterpreterDestroyed(ChannelError, + _crossinterp.ItemInterpreterDestroyed): + """Raised from get() and get_nowait().""" + + +UNBOUND = _crossinterp.UnboundItem.singleton('queue', __name__) + + +def _serialize_unbound(unbound): + if unbound is UNBOUND: + unbound = _crossinterp.UNBOUND + return _crossinterp.serialize_unbound(unbound) + + +def _resolve_unbound(flag): + resolved = _crossinterp.resolve_unbound(flag, ItemInterpreterDestroyed) + if resolved is _crossinterp.UNBOUND: + resolved = UNBOUND + return resolved + + +def create(*, unbounditems=UNBOUND): """Return (recv, send) for a new cross-interpreter channel. The channel may be used to pass data safely between interpreters. + + "unbounditems" sets the default for the send end of the channel. + See SendChannel.send() for supported values. The default value + is UNBOUND, which replaces the unbound item when received. """ - cid = _channels.create() - recv, send = RecvChannel(cid), SendChannel(cid) + unbound = _serialize_unbound(unbounditems) + unboundop, = unbound + cid = _channels.create(unboundop) + recv, send = RecvChannel(cid), SendChannel(cid, _unbound=unbound) return recv, send def list_all(): """Return a list of (recv, send) for all open channels.""" - return [(RecvChannel(cid), SendChannel(cid)) - for cid in _channels.list_all()] + return [(RecvChannel(cid), SendChannel(cid, _unbound=unbound)) + for cid, unbound in _channels.list_all()] class _ChannelEnd: @@ -38,7 +71,8 @@ class _ChannelEnd: _end = None - def __init__(self, cid): + def __new__(cls, cid): + self = super().__new__(cls) if self._end == 'send': cid = _channels._channel_id(cid, send=True, force=True) elif self._end == 'recv': @@ -46,6 +80,7 @@ def __init__(self, cid): else: raise NotImplementedError(self._end) self._id = cid + return self def __repr__(self): return f'{type(self).__name__}(id={int(self._id)})' @@ -61,6 +96,14 @@ def __eq__(self, other): return NotImplemented return other._id == self._id + # for pickling: + def __getnewargs__(self): + return (int(self._id),) + + # for pickling: + def __getstate__(self): + return None + @property def id(self): return self._id @@ -96,12 +139,15 @@ def recv(self, timeout=None, *, if timeout < 0: raise ValueError(f'timeout value must be non-negative') end = time.time() + timeout - obj = _channels.recv(self._id, _sentinel) + obj, unboundop = _channels.recv(self._id, _sentinel) while obj is _sentinel: time.sleep(_delay) if timeout is not None and time.time() >= end: raise TimeoutError - obj = _channels.recv(self._id, _sentinel) + obj, unboundop = _channels.recv(self._id, _sentinel) + if unboundop is not None: + assert obj is None, repr(obj) + return _resolve_unbound(unboundop) return obj def recv_nowait(self, default=_NOT_SET): @@ -112,9 +158,13 @@ def recv_nowait(self, default=_NOT_SET): is the same as recv(). """ if default is _NOT_SET: - return _channels.recv(self._id) + obj, unboundop = _channels.recv(self._id) else: - return _channels.recv(self._id, default) + obj, unboundop = _channels.recv(self._id, default) + if unboundop is not None: + assert obj is None, repr(obj) + return _resolve_unbound(unboundop) + return obj def close(self): _channels.close(self._id, recv=True) @@ -125,43 +175,79 @@ class SendChannel(_ChannelEnd): _end = 'send' + def __new__(cls, cid, *, _unbound=None): + if _unbound is None: + try: + op = _channels.get_channel_defaults(cid) + _unbound = (op,) + except ChannelNotFoundError: + _unbound = _serialize_unbound(UNBOUND) + self = super().__new__(cls, cid) + self._unbound = _unbound + return self + @property def is_closed(self): info = self._info return info.closed or info.closing - def send(self, obj, timeout=None): + def send(self, obj, timeout=None, *, + unbound=None, + ): """Send the object (i.e. its data) to the channel's receiving end. This blocks until the object is received. """ - _channels.send(self._id, obj, timeout=timeout, blocking=True) + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) + _channels.send(self._id, obj, unboundop, timeout=timeout, blocking=True) - def send_nowait(self, obj): + def send_nowait(self, obj, *, + unbound=None, + ): """Send the object to the channel's receiving end. If the object is immediately received then return True (else False). Otherwise this is the same as send(). """ + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) # XXX Note that at the moment channel_send() only ever returns # None. This should be fixed when channel_send_wait() is added. # See bpo-32604 and gh-19829. - return _channels.send(self._id, obj, blocking=False) + return _channels.send(self._id, obj, unboundop, blocking=False) - def send_buffer(self, obj, timeout=None): + def send_buffer(self, obj, timeout=None, *, + unbound=None, + ): """Send the object's buffer to the channel's receiving end. This blocks until the object is received. """ - _channels.send_buffer(self._id, obj, timeout=timeout, blocking=True) + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) + _channels.send_buffer(self._id, obj, unboundop, + timeout=timeout, blocking=True) - def send_buffer_nowait(self, obj): + def send_buffer_nowait(self, obj, *, + unbound=None, + ): """Send the object's buffer to the channel's receiving end. If the object is immediately received then return True (else False). Otherwise this is the same as send(). """ - return _channels.send_buffer(self._id, obj, blocking=False) + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) + return _channels.send_buffer(self._id, obj, unboundop, blocking=False) def close(self): _channels.close(self._id, send=True) diff --git a/Lib/test/support/interpreters/queues.py b/Lib/test/support/interpreters/queues.py index f9978f0bec5a62..deb8e8613af731 100644 --- a/Lib/test/support/interpreters/queues.py +++ b/Lib/test/support/interpreters/queues.py @@ -4,54 +4,88 @@ import queue import time import weakref -import _xxinterpqueues as _queues +import _interpqueues as _queues +from . import _crossinterp # aliases: -from _xxinterpqueues import ( +from _interpqueues import ( QueueError, QueueNotFoundError, ) +from ._crossinterp import ( + UNBOUND_ERROR, UNBOUND_REMOVE, +) __all__ = [ + 'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE', 'create', 'list_all', 'Queue', 'QueueError', 'QueueNotFoundError', 'QueueEmpty', 'QueueFull', + 'ItemInterpreterDestroyed', ] -class QueueEmpty(_queues.QueueEmpty, queue.Empty): +class QueueEmpty(QueueError, queue.Empty): """Raised from get_nowait() when the queue is empty. It is also raised from get() if it times out. """ -class QueueFull(_queues.QueueFull, queue.Full): +class QueueFull(QueueError, queue.Full): """Raised from put_nowait() when the queue is full. It is also raised from put() if it times out. """ +class ItemInterpreterDestroyed(QueueError, + _crossinterp.ItemInterpreterDestroyed): + """Raised from get() and get_nowait().""" + + _SHARED_ONLY = 0 _PICKLED = 1 -def create(maxsize=0, *, syncobj=False): + +UNBOUND = _crossinterp.UnboundItem.singleton('queue', __name__) + + +def _serialize_unbound(unbound): + if unbound is UNBOUND: + unbound = _crossinterp.UNBOUND + return _crossinterp.serialize_unbound(unbound) + + +def _resolve_unbound(flag): + resolved = _crossinterp.resolve_unbound(flag, ItemInterpreterDestroyed) + if resolved is _crossinterp.UNBOUND: + resolved = UNBOUND + return resolved + + +def create(maxsize=0, *, syncobj=False, unbounditems=UNBOUND): """Return a new cross-interpreter queue. The queue may be used to pass data safely between interpreters. "syncobj" sets the default for Queue.put() and Queue.put_nowait(). + + "unbounditems" likewise sets the default. See Queue.put() for + supported values. The default value is UNBOUND, which replaces + the unbound item. """ fmt = _SHARED_ONLY if syncobj else _PICKLED - qid = _queues.create(maxsize, fmt) - return Queue(qid, _fmt=fmt) + unbound = _serialize_unbound(unbounditems) + unboundop, = unbound + qid = _queues.create(maxsize, fmt, unboundop) + return Queue(qid, _fmt=fmt, _unbound=unbound) def list_all(): """Return a list of all open queues.""" - return [Queue(qid, _fmt=fmt) - for qid, fmt in _queues.list_all()] + return [Queue(qid, _fmt=fmt, _unbound=(unboundop,)) + for qid, fmt, unboundop in _queues.list_all()] _known_queues = weakref.WeakValueDictionary() @@ -59,20 +93,28 @@ def list_all(): class Queue: """A cross-interpreter queue.""" - def __new__(cls, id, /, *, _fmt=None): + def __new__(cls, id, /, *, _fmt=None, _unbound=None): # There is only one instance for any given ID. if isinstance(id, int): id = int(id) else: raise TypeError(f'id must be an int, got {id!r}') if _fmt is None: - _fmt = _queues.get_default_fmt(id) + if _unbound is None: + _fmt, op = _queues.get_queue_defaults(id) + _unbound = (op,) + else: + _fmt, _ = _queues.get_queue_defaults(id) + elif _unbound is None: + _, op = _queues.get_queue_defaults(id) + _unbound = (op,) try: self = _known_queues[id] except KeyError: self = super().__new__(cls) self._id = id self._fmt = _fmt + self._unbound = _unbound _known_queues[id] = self _queues.bind(id) return self @@ -93,6 +135,14 @@ def __repr__(self): def __hash__(self): return hash(self._id) + # for pickling: + def __getnewargs__(self): + return (self._id,) + + # for pickling: + def __getstate__(self): + return None + @property def id(self): return self._id @@ -116,6 +166,7 @@ def qsize(self): def put(self, obj, timeout=None, *, syncobj=None, + unbound=None, _delay=10 / 1000, # 10 milliseconds ): """Add the object to the queue. @@ -123,7 +174,7 @@ def put(self, obj, timeout=None, *, This blocks while the queue is full. If "syncobj" is None (the default) then it uses the - queue's default, set with create_queue().. + queue's default, set with create_queue(). If "syncobj" is false then all objects are supported, at the expense of worse performance. @@ -144,11 +195,37 @@ def put(self, obj, timeout=None, *, actually is. That's a slightly different and stronger promise than just (initial) equality, which is all "syncobj=False" can promise. + + "unbound" controls the behavior of Queue.get() for the given + object if the current interpreter (calling put()) is later + destroyed. + + If "unbound" is None (the default) then it uses the + queue's default, set with create_queue(), + which is usually UNBOUND. + + If "unbound" is UNBOUND_ERROR then get() will raise an + ItemInterpreterDestroyed exception if the original interpreter + has been destroyed. This does not otherwise affect the queue; + the next call to put() will work like normal, returning the next + item in the queue. + + If "unbound" is UNBOUND_REMOVE then the item will be removed + from the queue as soon as the original interpreter is destroyed. + Be aware that this will introduce an imbalance between put() + and get() calls. + + If "unbound" is UNBOUND then it is returned by get() in place + of the unbound item. """ if syncobj is None: fmt = self._fmt else: fmt = _SHARED_ONLY if syncobj else _PICKLED + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) if timeout is not None: timeout = int(timeout) if timeout < 0: @@ -158,27 +235,26 @@ def put(self, obj, timeout=None, *, obj = pickle.dumps(obj) while True: try: - _queues.put(self._id, obj, fmt) - except _queues.QueueFull as exc: + _queues.put(self._id, obj, fmt, unboundop) + except QueueFull as exc: if timeout is not None and time.time() >= end: - exc.__class__ = QueueFull raise # re-raise time.sleep(_delay) else: break - def put_nowait(self, obj, *, syncobj=None): + def put_nowait(self, obj, *, syncobj=None, unbound=None): if syncobj is None: fmt = self._fmt else: fmt = _SHARED_ONLY if syncobj else _PICKLED + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) if fmt is _PICKLED: obj = pickle.dumps(obj) - try: - _queues.put(self._id, obj, fmt) - except _queues.QueueFull as exc: - exc.__class__ = QueueFull - raise # re-raise + _queues.put(self._id, obj, fmt, unboundop) def get(self, timeout=None, *, _delay=10 / 1000, # 10 milliseconds @@ -186,6 +262,10 @@ def get(self, timeout=None, *, """Return the next object from the queue. This blocks while the queue is empty. + + If the next item's original interpreter has been destroyed + then the "next object" is determined by the value of the + "unbound" argument to put(). """ if timeout is not None: timeout = int(timeout) @@ -194,14 +274,16 @@ def get(self, timeout=None, *, end = time.time() + timeout while True: try: - obj, fmt = _queues.get(self._id) - except _queues.QueueEmpty as exc: + obj, fmt, unboundop = _queues.get(self._id) + except QueueEmpty as exc: if timeout is not None and time.time() >= end: - exc.__class__ = QueueEmpty raise # re-raise time.sleep(_delay) else: break + if unboundop is not None: + assert obj is None, repr(obj) + return _resolve_unbound(unboundop) if fmt == _PICKLED: obj = pickle.loads(obj) else: @@ -215,10 +297,12 @@ def get_nowait(self): is the same as get(). """ try: - obj, fmt = _queues.get(self._id) - except _queues.QueueEmpty as exc: - exc.__class__ = QueueEmpty + obj, fmt, unboundop = _queues.get(self._id) + except QueueEmpty as exc: raise # re-raise + if unboundop is not None: + assert obj is None, repr(obj) + return _resolve_unbound(unboundop) if fmt == _PICKLED: obj = pickle.loads(obj) else: @@ -226,4 +310,4 @@ def get_nowait(self): return obj -_queues._register_queue_type(Queue) +_queues._register_heap_types(Queue, QueueEmpty, QueueFull) diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index ffa5fc52e9b2bd..15dcdc9b1fddfb 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -294,6 +294,33 @@ def skip_unless_working_chmod(test): return test if ok else unittest.skip(msg)(test) +@contextlib.contextmanager +def save_mode(path, *, quiet=False): + """Context manager that restores the mode (permissions) of *path* on exit. + + Arguments: + + path: Path of the file to restore the mode of. + + quiet: if False (the default), the context manager raises an exception + on error. Otherwise, it issues only a warning and keeps the current + working directory the same. + + """ + saved_mode = os.stat(path) + try: + yield + finally: + try: + os.chmod(path, saved_mode.st_mode) + except OSError as exc: + if not quiet: + raise + warnings.warn(f'tests may fail, unable to restore the mode of ' + f'{path!r} to {saved_mode.st_mode}: {exc}', + RuntimeWarning, stacklevel=3) + + # Check whether the current effective user has the capability to override # DAC (discretionary access control). Typically user root is able to # bypass file read, write, and execute permission checks. The capability @@ -612,7 +639,7 @@ def __fspath__(self): def fd_count(): """Count the number of open file descriptors. """ - if sys.platform.startswith(('linux', 'freebsd', 'emscripten')): + if sys.platform.startswith(('linux', 'android', 'freebsd', 'emscripten')): fd_path = "/proc/self/fd" elif sys.platform == "darwin": fd_path = "/dev/fd" diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py index 759020c33aa700..46ce950433ddf6 100644 --- a/Lib/test/support/script_helper.py +++ b/Lib/test/support/script_helper.py @@ -63,30 +63,32 @@ class _PythonRunResult(collections.namedtuple("_PythonRunResult", """Helper for reporting Python subprocess run results""" def fail(self, cmd_line): """Provide helpful details about failed subcommand runs""" - # Limit to 80 lines to ASCII characters - maxlen = 80 * 100 + # Limit to 300 lines of ASCII characters + maxlen = 300 * 100 out, err = self.out, self.err if len(out) > maxlen: out = b'(... truncated stdout ...)' + out[-maxlen:] if len(err) > maxlen: err = b'(... truncated stderr ...)' + err[-maxlen:] - out = out.decode('ascii', 'replace').rstrip() - err = err.decode('ascii', 'replace').rstrip() - raise AssertionError("Process return code is %d\n" - "command line: %r\n" - "\n" - "stdout:\n" - "---\n" - "%s\n" - "---\n" - "\n" - "stderr:\n" - "---\n" - "%s\n" - "---" - % (self.rc, cmd_line, - out, - err)) + out = out.decode('utf8', 'replace').rstrip() + err = err.decode('utf8', 'replace').rstrip() + + exitcode = self.rc + signame = support.get_signal_name(exitcode) + if signame: + exitcode = f"{exitcode} ({signame})" + raise AssertionError(f"Process return code is {exitcode}\n" + f"command line: {cmd_line!r}\n" + f"\n" + f"stdout:\n" + f"---\n" + f"{out}\n" + f"---\n" + f"\n" + f"stderr:\n" + f"---\n" + f"{err}\n" + f"---") # Executing the interpreter in a subprocess @@ -232,9 +234,13 @@ def make_script(script_dir, script_basename, source, omit_suffix=False): if not omit_suffix: script_filename += os.extsep + 'py' script_name = os.path.join(script_dir, script_filename) - # The script should be encoded to UTF-8, the default string encoding - with open(script_name, 'w', encoding='utf-8') as script_file: - script_file.write(source) + if isinstance(source, str): + # The script should be encoded to UTF-8, the default string encoding + with open(script_name, 'w', encoding='utf-8') as script_file: + script_file.write(source) + else: + with open(script_name, 'wb') as script_file: + script_file.write(source) importlib.invalidate_caches() return script_name diff --git a/Lib/test/support/smtpd.py b/Lib/test/support/smtpd.py index 6052232ec2b585..6537679db9ad24 100755 --- a/Lib/test/support/smtpd.py +++ b/Lib/test/support/smtpd.py @@ -7,7 +7,7 @@ --nosetuid -n - This program generally tries to setuid `nobody', unless this flag is + This program generally tries to setuid 'nobody', unless this flag is set. The setuid call will fail if this program is not run as root (in which case, use this flag). @@ -17,7 +17,7 @@ --class classname -c classname - Use `classname' as the concrete SMTP proxy class. Uses `PureProxy' by + Use 'classname' as the concrete SMTP proxy class. Uses 'PureProxy' by default. --size limit @@ -39,8 +39,8 @@ Version: %(__version__)s -If localhost is not given then `localhost' is used, and if localport is not -given then 8025 is used. If remotehost is not given then `localhost' is used, +If localhost is not given then 'localhost' is used, and if localport is not +given then 8025 is used. If remotehost is not given then 'localhost' is used, and if remoteport is not given, then 25 is used. """ @@ -633,7 +633,8 @@ def __init__(self, localaddr, remoteaddr, " be set to True at the same time") asyncore.dispatcher.__init__(self, map=map) try: - gai_results = socket.getaddrinfo(*localaddr, + family = 0 if socket.has_ipv6 else socket.AF_INET + gai_results = socket.getaddrinfo(*localaddr, family=family, type=socket.SOCK_STREAM) self.create_socket(gai_results[0][0], gai_results[0][1]) # try to re-use a server port if possible @@ -672,9 +673,9 @@ def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): message to. data is a string containing the entire full text of the message, - headers (if supplied) and all. It has been `de-transparencied' + headers (if supplied) and all. It has been 'de-transparencied' according to RFC 821, Section 4.5.2. In other words, a line - containing a `.' followed by other text has had the leading dot + containing a '.' followed by other text has had the leading dot removed. kwargs is a dictionary containing additional information. It is @@ -685,7 +686,7 @@ def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): ['BODY=8BITMIME', 'SMTPUTF8']. 'rcpt_options': same, for the rcpt command. - This function should return None for a normal `250 Ok' response; + This function should return None for a normal '250 Ok' response; otherwise, it should return the desired response string in RFC 821 format. diff --git a/Lib/test/support/strace_helper.py b/Lib/test/support/strace_helper.py new file mode 100644 index 00000000000000..798d6c6886962f --- /dev/null +++ b/Lib/test/support/strace_helper.py @@ -0,0 +1,206 @@ +import re +import sys +import textwrap +import os +import unittest +from dataclasses import dataclass +from functools import cache +from test import support +from test.support.script_helper import run_python_until_end + +_strace_binary = "/usr/bin/strace" +_syscall_regex = re.compile( + r"(?P[^(]*)\((?P[^)]*)\)\s*[=]\s*(?P.+)") +_returncode_regex = re.compile( + br"\+\+\+ exited with (?P\d+) \+\+\+") + + +@dataclass +class StraceEvent: + syscall: str + args: list[str] + returncode: str + + +@dataclass +class StraceResult: + strace_returncode: int + python_returncode: int + + """The event messages generated by strace. This is very similar to the + stderr strace produces with returncode marker section removed.""" + event_bytes: bytes + stdout: bytes + stderr: bytes + + def events(self): + """Parse event_bytes data into system calls for easier processing. + + This assumes the program under inspection doesn't print any non-utf8 + strings which would mix into the strace output.""" + decoded_events = self.event_bytes.decode('utf-8') + matches = [ + _syscall_regex.match(event) + for event in decoded_events.splitlines() + ] + return [ + StraceEvent(match["syscall"], + [arg.strip() for arg in (match["args"].split(","))], + match["returncode"]) for match in matches if match + ] + + def sections(self): + """Find all "MARK " writes and use them to make groups of events. + + This is useful to avoid variable / overhead events, like those at + interpreter startup or when opening a file so a test can verify just + the small case under study.""" + current_section = "__startup" + sections = {current_section: []} + for event in self.events(): + if event.syscall == 'write' and len( + event.args) > 2 and event.args[1].startswith("\"MARK "): + # Found a new section, don't include the write in the section + # but all events until next mark should be in that section + current_section = event.args[1].split( + " ", 1)[1].removesuffix('\\n"') + if current_section not in sections: + sections[current_section] = list() + else: + sections[current_section].append(event) + + return sections + +def _filter_memory_call(call): + # mmap can operate on a fd or "MAP_ANONYMOUS" which gives a block of memory. + # Ignore "MAP_ANONYMOUS + the "MAP_ANON" alias. + if call.syscall == "mmap" and "MAP_ANON" in call.args[3]: + return True + + if call.syscall in ("munmap", "mprotect"): + return True + + return False + + +def filter_memory(syscalls): + """Filter out memory allocation calls from File I/O calls. + + Some calls (mmap, munmap, etc) can be used on files or to just get a block + of memory. Use this function to filter out the memory related calls from + other calls.""" + + return [call for call in syscalls if not _filter_memory_call(call)] + + +@support.requires_subprocess() +def strace_python(code, strace_flags, check=True): + """Run strace and return the trace. + + Sets strace_returncode and python_returncode to `-1` on error.""" + res = None + + def _make_error(reason, details): + return StraceResult( + strace_returncode=-1, + python_returncode=-1, + event_bytes= f"error({reason},details={details!r}) = -1".encode('utf-8'), + stdout=res.out if res else b"", + stderr=res.err if res else b"") + + # Run strace, and get out the raw text + try: + res, cmd_line = run_python_until_end( + "-c", + textwrap.dedent(code), + __run_using_command=[_strace_binary] + strace_flags, + ) + except OSError as err: + return _make_error("Caught OSError", err) + + if check and res.rc: + res.fail(cmd_line) + + # Get out program returncode + stripped = res.err.strip() + output = stripped.rsplit(b"\n", 1) + if len(output) != 2: + return _make_error("Expected strace events and exit code line", + stripped[-50:]) + + returncode_match = _returncode_regex.match(output[1]) + if not returncode_match: + return _make_error("Expected to find returncode in last line.", + output[1][:50]) + + python_returncode = int(returncode_match["returncode"]) + if check and python_returncode: + res.fail(cmd_line) + + return StraceResult(strace_returncode=res.rc, + python_returncode=python_returncode, + event_bytes=output[0], + stdout=res.out, + stderr=res.err) + + +def get_events(code, strace_flags, prelude, cleanup): + # NOTE: The flush is currently required to prevent the prints from getting + # buffered and done all at once at exit + prelude = textwrap.dedent(prelude) + code = textwrap.dedent(code) + cleanup = textwrap.dedent(cleanup) + to_run = f""" +print("MARK prelude", flush=True) +{prelude} +print("MARK code", flush=True) +{code} +print("MARK cleanup", flush=True) +{cleanup} +print("MARK __shutdown", flush=True) + """ + trace = strace_python(to_run, strace_flags) + all_sections = trace.sections() + return all_sections['code'] + + +def get_syscalls(code, strace_flags, prelude="", cleanup="", + ignore_memory=True): + """Get the syscalls which a given chunk of python code generates""" + events = get_events(code, strace_flags, prelude=prelude, cleanup=cleanup) + + if ignore_memory: + events = filter_memory(events) + + return [ev.syscall for ev in events] + + +# Moderately expensive (spawns a subprocess), so share results when possible. +@cache +def _can_strace(): + res = strace_python("import sys; sys.exit(0)", [], check=False) + if res.strace_returncode == 0 and res.python_returncode == 0: + assert res.events(), "Should have parsed multiple calls" + return True + return False + + +def requires_strace(): + if sys.platform != "linux": + return unittest.skip("Linux only, requires strace.") + + if "LD_PRELOAD" in os.environ: + # Distribution packaging (ex. Debian `fakeroot` and Gentoo `sandbox`) + # use LD_PRELOAD to intercept system calls, which changes the overall + # set of system calls which breaks tests expecting a specific set of + # system calls). + return unittest.skip("Not supported when LD_PRELOAD is intercepting system calls.") + + if support.check_sanitizer(address=True, memory=True): + return unittest.skip("LeakSanitizer does not work under ptrace (strace, gdb, etc)") + + return unittest.skipUnless(_can_strace(), "Requires working strace") + + +__all__ = ["filter_memory", "get_events", "get_syscalls", "requires_strace", + "strace_python", "StraceEvent", "StraceResult"] diff --git a/Lib/test/support/testcase.py b/Lib/test/support/testcase.py index 1e4363b15783eb..fad1e4cb3499c0 100644 --- a/Lib/test/support/testcase.py +++ b/Lib/test/support/testcase.py @@ -1,3 +1,6 @@ +from math import copysign, isnan + + class ExceptionIsLikeMixin: def assertExceptionIsLike(self, exc, template): """ @@ -23,3 +26,40 @@ def assertExceptionIsLike(self, exc, template): self.assertEqual(len(exc.exceptions), len(template.exceptions)) for e, t in zip(exc.exceptions, template.exceptions): self.assertExceptionIsLike(e, t) + + +class FloatsAreIdenticalMixin: + def assertFloatsAreIdentical(self, x, y): + """Fail unless floats x and y are identical, in the sense that: + (1) both x and y are nans, or + (2) both x and y are infinities, with the same sign, or + (3) both x and y are zeros, with the same sign, or + (4) x and y are both finite and nonzero, and x == y + + """ + msg = 'floats {!r} and {!r} are not identical' + + if isnan(x) or isnan(y): + if isnan(x) and isnan(y): + return + elif x == y: + if x != 0.0: + return + # both zero; check that signs match + elif copysign(1.0, x) == copysign(1.0, y): + return + else: + msg += ': zeros have different signs' + self.fail(msg.format(x, y)) + + +class ComplexesAreIdenticalMixin(FloatsAreIdenticalMixin): + def assertComplexesAreIdentical(self, x, y): + """Fail unless complex numbers x and y have equal values and signs. + + In particular, if x and y both have real (or imaginary) part + zero, but the zeros have different signs, this test will fail. + + """ + self.assertFloatsAreIdentical(x.real, y.real) + self.assertFloatsAreIdentical(x.imag, y.imag) diff --git a/Lib/test/support/venv.py b/Lib/test/support/venv.py new file mode 100644 index 00000000000000..7bfb9e4f3c479f --- /dev/null +++ b/Lib/test/support/venv.py @@ -0,0 +1,82 @@ +import contextlib +import logging +import os +import subprocess +import shlex +import sys +import sysconfig +import tempfile +import unittest +import venv + + +class VirtualEnvironment: + def __init__(self, prefix, **venv_create_args): + self._logger = logging.getLogger(self.__class__.__name__) + venv.create(prefix, **venv_create_args) + self._prefix = prefix + self._paths = sysconfig.get_paths( + scheme='venv', + vars={'base': self.prefix}, + expand=True, + ) + + @classmethod + @contextlib.contextmanager + def from_tmpdir(cls, *, prefix=None, dir=None, **venv_create_args): + delete = not bool(os.environ.get('PYTHON_TESTS_KEEP_VENV')) + with tempfile.TemporaryDirectory(prefix=prefix, dir=dir, delete=delete) as tmpdir: + yield cls(tmpdir, **venv_create_args) + + @property + def prefix(self): + return self._prefix + + @property + def paths(self): + return self._paths + + @property + def interpreter(self): + return os.path.join(self.paths['scripts'], os.path.basename(sys.executable)) + + def _format_output(self, name, data, indent='\t'): + if not data: + return indent + f'{name}: (none)' + if len(data.splitlines()) == 1: + return indent + f'{name}: {data}' + else: + prefixed_lines = '\n'.join(indent + '> ' + line for line in data.splitlines()) + return indent + f'{name}:\n' + prefixed_lines + + def run(self, *args, **subprocess_args): + if subprocess_args.get('shell'): + raise ValueError('Running the subprocess in shell mode is not supported.') + default_args = { + 'capture_output': True, + 'check': True, + } + try: + result = subprocess.run([self.interpreter, *args], **default_args | subprocess_args) + except subprocess.CalledProcessError as e: + if e.returncode != 0: + self._logger.error( + f'Interpreter returned non-zero exit status {e.returncode}.\n' + + self._format_output('COMMAND', shlex.join(e.cmd)) + '\n' + + self._format_output('STDOUT', e.stdout.decode()) + '\n' + + self._format_output('STDERR', e.stderr.decode()) + '\n' + ) + raise + else: + return result + + +class VirtualEnvironmentMixin: + def venv(self, name=None, **venv_create_args): + venv_name = self.id() + if name: + venv_name += f'-{name}' + return VirtualEnvironment.from_tmpdir( + prefix=f'{venv_name}-venv-', + **venv_create_args, + ) diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index 19dcbb207e914a..e405056c8ffcb5 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -3,7 +3,6 @@ from test.support import warnings_helper import os import sys -import types if support.check_sanitizer(address=True, memory=True): @@ -104,7 +103,7 @@ def test_all(self): # In case _socket fails to build, make this test fail more gracefully # than an AttributeError somewhere deep in concurrent.futures, email # or unittest. - import _socket + import _socket # noqa: F401 ignored = [] failed_imports = [] diff --git a/Lib/test/test__colorize.py b/Lib/test/test__colorize.py new file mode 100644 index 00000000000000..056a5306ced183 --- /dev/null +++ b/Lib/test/test__colorize.py @@ -0,0 +1,135 @@ +import contextlib +import io +import sys +import unittest +import unittest.mock +import _colorize +from test.support.os_helper import EnvironmentVarGuard + + +@contextlib.contextmanager +def clear_env(): + with EnvironmentVarGuard() as mock_env: + for var in "FORCE_COLOR", "NO_COLOR", "PYTHON_COLORS": + mock_env.unset(var) + yield mock_env + + +def supports_virtual_terminal(): + if sys.platform == "win32": + return unittest.mock.patch("nt._supports_virtual_terminal", return_value=True) + else: + return contextlib.nullcontext() + + +class TestColorizeFunction(unittest.TestCase): + def test_colorized_detection_checks_for_environment_variables(self): + def check(env, fallback, expected): + with (self.subTest(env=env, fallback=fallback), + clear_env() as mock_env): + mock_env.update(env) + isatty_mock.return_value = fallback + stdout_mock.isatty.return_value = fallback + self.assertEqual(_colorize.can_colorize(), expected) + + with (unittest.mock.patch("os.isatty") as isatty_mock, + unittest.mock.patch("sys.stdout") as stdout_mock, + supports_virtual_terminal()): + stdout_mock.fileno.return_value = 1 + + for fallback in False, True: + check({}, fallback, fallback) + check({'TERM': 'dumb'}, fallback, False) + check({'TERM': 'xterm'}, fallback, fallback) + check({'TERM': ''}, fallback, fallback) + check({'FORCE_COLOR': '1'}, fallback, True) + check({'FORCE_COLOR': '0'}, fallback, True) + check({'FORCE_COLOR': ''}, fallback, fallback) + check({'NO_COLOR': '1'}, fallback, False) + check({'NO_COLOR': '0'}, fallback, False) + check({'NO_COLOR': ''}, fallback, fallback) + + check({'TERM': 'dumb', 'FORCE_COLOR': '1'}, False, True) + check({'FORCE_COLOR': '1', 'NO_COLOR': '1'}, True, False) + + for ignore_environment in False, True: + # Simulate running with or without `-E`. + flags = unittest.mock.MagicMock(ignore_environment=ignore_environment) + with unittest.mock.patch("sys.flags", flags): + check({'PYTHON_COLORS': '1'}, True, True) + check({'PYTHON_COLORS': '1'}, False, not ignore_environment) + check({'PYTHON_COLORS': '0'}, True, ignore_environment) + check({'PYTHON_COLORS': '0'}, False, False) + for fallback in False, True: + check({'PYTHON_COLORS': 'x'}, fallback, fallback) + check({'PYTHON_COLORS': ''}, fallback, fallback) + + check({'TERM': 'dumb', 'PYTHON_COLORS': '1'}, False, not ignore_environment) + check({'NO_COLOR': '1', 'PYTHON_COLORS': '1'}, False, not ignore_environment) + check({'FORCE_COLOR': '1', 'PYTHON_COLORS': '0'}, True, ignore_environment) + + @unittest.skipUnless(sys.platform == "win32", "requires Windows") + def test_colorized_detection_checks_on_windows(self): + with (clear_env(), + unittest.mock.patch("os.isatty") as isatty_mock, + unittest.mock.patch("sys.stdout") as stdout_mock, + supports_virtual_terminal() as vt_mock): + stdout_mock.fileno.return_value = 1 + isatty_mock.return_value = True + stdout_mock.isatty.return_value = True + + vt_mock.return_value = True + self.assertEqual(_colorize.can_colorize(), True) + vt_mock.return_value = False + self.assertEqual(_colorize.can_colorize(), False) + import nt + del nt._supports_virtual_terminal + self.assertEqual(_colorize.can_colorize(), False) + + def test_colorized_detection_checks_for_std_streams(self): + with (clear_env(), + unittest.mock.patch("os.isatty") as isatty_mock, + unittest.mock.patch("sys.stdout") as stdout_mock, + unittest.mock.patch("sys.stderr") as stderr_mock, + supports_virtual_terminal()): + stdout_mock.fileno.return_value = 1 + stderr_mock.fileno.side_effect = ZeroDivisionError + stderr_mock.isatty.side_effect = ZeroDivisionError + + isatty_mock.return_value = True + stdout_mock.isatty.return_value = True + self.assertEqual(_colorize.can_colorize(), True) + + isatty_mock.return_value = False + stdout_mock.isatty.return_value = False + self.assertEqual(_colorize.can_colorize(), False) + + def test_colorized_detection_checks_for_file(self): + with clear_env(), supports_virtual_terminal(): + + with unittest.mock.patch("os.isatty") as isatty_mock: + file = unittest.mock.MagicMock() + file.fileno.return_value = 1 + isatty_mock.return_value = True + self.assertEqual(_colorize.can_colorize(file=file), True) + isatty_mock.return_value = False + self.assertEqual(_colorize.can_colorize(file=file), False) + + # No file.fileno. + with unittest.mock.patch("os.isatty", side_effect=ZeroDivisionError): + file = unittest.mock.MagicMock(spec=['isatty']) + file.isatty.return_value = True + self.assertEqual(_colorize.can_colorize(file=file), False) + + # file.fileno() raises io.UnsupportedOperation. + with unittest.mock.patch("os.isatty", side_effect=ZeroDivisionError): + file = unittest.mock.MagicMock() + file.fileno.side_effect = io.UnsupportedOperation + file.isatty.return_value = True + self.assertEqual(_colorize.can_colorize(file=file), True) + file.isatty.return_value = False + self.assertEqual(_colorize.can_colorize(file=file), False) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test__xxinterpchannels.py b/Lib/test/test__interpchannels.py similarity index 62% rename from Lib/test/test__xxinterpchannels.py rename to Lib/test/test__interpchannels.py index c5d29bd2dd911f..4a7f04b9df9843 100644 --- a/Lib/test/test__xxinterpchannels.py +++ b/Lib/test/test__interpchannels.py @@ -8,14 +8,16 @@ from test.support import import_helper -from test.test__xxsubinterpreters import ( - interpreters, +_channels = import_helper.import_module('_interpchannels') +from test.support.interpreters import _crossinterp +from test.test__interpreters import ( + _interpreters, _run_output, clean_up_interpreters, ) -channels = import_helper.import_module('_xxinterpchannels') +REPLACE = _crossinterp._UNBOUND_CONSTANT_TO_FLAG[_crossinterp.UNBOUND] # Additional tests are found in Lib/test/test_interpreters/test_channels.py. @@ -29,9 +31,19 @@ def recv_wait(cid): while True: try: - return channels.recv(cid) - except channels.ChannelEmptyError: + obj, unboundop = _channels.recv(cid) + except _channels.ChannelEmptyError: time.sleep(0.1) + else: + assert unboundop is None, repr(unboundop) + return obj + + +def recv_nowait(cid, *args, unbound=False): + obj, unboundop = _channels.recv(cid, *args) + assert (unboundop is None) != unbound, repr(unboundop) + return obj + #@contextmanager #def run_threaded(id, source, **shared): @@ -49,14 +61,15 @@ def run_interp(id, source, **shared): def _run_interp(id, source, shared, _mainns={}): source = dedent(source) - main = interpreters.get_main() + main, *_ = _interpreters.get_main() if main == id: - if interpreters.get_current() != main: + cur, *_ = _interpreters.get_current() + if cur != main: raise RuntimeError # XXX Run a func? exec(source, _mainns) else: - interpreters.run_string(id, source, shared) + _interpreters.run_string(id, source, shared) class Interpreter(namedtuple('Interpreter', 'name id')): @@ -71,7 +84,7 @@ def from_raw(cls, raw): raise NotImplementedError def __new__(cls, name=None, id=None): - main = interpreters.get_main() + main, *_ = _interpreters.get_main() if id == main: if not name: name = 'main' @@ -89,7 +102,7 @@ def __new__(cls, name=None, id=None): name = 'main' id = main else: - id = interpreters.create() + id = _interpreters.create() self = super().__new__(cls, name, id) return self @@ -100,7 +113,7 @@ def __new__(cls, name=None, id=None): def expect_channel_closed(): try: yield - except channels.ChannelClosedError: + except _channels.ChannelClosedError: pass else: assert False, 'channel not closed' @@ -187,7 +200,7 @@ def run_action(cid, action, end, state, *, hideclosed=True): try: result = _run_action(cid, action, end, state) - except channels.ChannelClosedError: + except _channels.ChannelClosedError: if not hideclosed and not expectfail: raise result = state.close() @@ -200,18 +213,18 @@ def run_action(cid, action, end, state, *, hideclosed=True): def _run_action(cid, action, end, state): if action == 'use': if end == 'send': - channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'spam', blocking=False) return state.incr() elif end == 'recv': if not state.pending: try: - channels.recv(cid) - except channels.ChannelEmptyError: + _channels.recv(cid) + except _channels.ChannelEmptyError: return state else: raise Exception('expected ChannelEmptyError') else: - channels.recv(cid) + recv_nowait(cid) return state.decr() else: raise ValueError(end) @@ -219,7 +232,7 @@ def _run_action(cid, action, end, state): kwargs = {} if end in ('recv', 'send'): kwargs[end] = True - channels.close(cid, **kwargs) + _channels.close(cid, **kwargs) return state.close() elif action == 'force-close': kwargs = { @@ -227,17 +240,17 @@ def _run_action(cid, action, end, state): } if end in ('recv', 'send'): kwargs[end] = True - channels.close(cid, **kwargs) + _channels.close(cid, **kwargs) return state.close(force=True) else: raise ValueError(action) def clean_up_channels(): - for cid in channels.list_all(): + for cid, _ in _channels.list_all(): try: - channels.destroy(cid) - except channels.ChannelNotFoundError: + _channels.destroy(cid) + except _channels.ChannelNotFoundError: pass # already destroyed @@ -254,25 +267,25 @@ def tearDown(self): class ChannelIDTests(TestBase): def test_default_kwargs(self): - cid = channels._channel_id(10, force=True) + cid = _channels._channel_id(10, force=True) self.assertEqual(int(cid), 10) self.assertEqual(cid.end, 'both') def test_with_kwargs(self): - cid = channels._channel_id(10, send=True, force=True) + cid = _channels._channel_id(10, send=True, force=True) self.assertEqual(cid.end, 'send') - cid = channels._channel_id(10, send=True, recv=False, force=True) + cid = _channels._channel_id(10, send=True, recv=False, force=True) self.assertEqual(cid.end, 'send') - cid = channels._channel_id(10, recv=True, force=True) + cid = _channels._channel_id(10, recv=True, force=True) self.assertEqual(cid.end, 'recv') - cid = channels._channel_id(10, recv=True, send=False, force=True) + cid = _channels._channel_id(10, recv=True, send=False, force=True) self.assertEqual(cid.end, 'recv') - cid = channels._channel_id(10, send=True, recv=True, force=True) + cid = _channels._channel_id(10, send=True, recv=True, force=True) self.assertEqual(cid.end, 'both') def test_coerce_id(self): @@ -280,47 +293,47 @@ class Int(str): def __index__(self): return 10 - cid = channels._channel_id(Int(), force=True) + cid = _channels._channel_id(Int(), force=True) self.assertEqual(int(cid), 10) def test_bad_id(self): - self.assertRaises(TypeError, channels._channel_id, object()) - self.assertRaises(TypeError, channels._channel_id, 10.0) - self.assertRaises(TypeError, channels._channel_id, '10') - self.assertRaises(TypeError, channels._channel_id, b'10') - self.assertRaises(ValueError, channels._channel_id, -1) - self.assertRaises(OverflowError, channels._channel_id, 2**64) + self.assertRaises(TypeError, _channels._channel_id, object()) + self.assertRaises(TypeError, _channels._channel_id, 10.0) + self.assertRaises(TypeError, _channels._channel_id, '10') + self.assertRaises(TypeError, _channels._channel_id, b'10') + self.assertRaises(ValueError, _channels._channel_id, -1) + self.assertRaises(OverflowError, _channels._channel_id, 2**64) def test_bad_kwargs(self): with self.assertRaises(ValueError): - channels._channel_id(10, send=False, recv=False) + _channels._channel_id(10, send=False, recv=False) def test_does_not_exist(self): - cid = channels.create() - with self.assertRaises(channels.ChannelNotFoundError): - channels._channel_id(int(cid) + 1) # unforced + cid = _channels.create(REPLACE) + with self.assertRaises(_channels.ChannelNotFoundError): + _channels._channel_id(int(cid) + 1) # unforced def test_str(self): - cid = channels._channel_id(10, force=True) + cid = _channels._channel_id(10, force=True) self.assertEqual(str(cid), '10') def test_repr(self): - cid = channels._channel_id(10, force=True) + cid = _channels._channel_id(10, force=True) self.assertEqual(repr(cid), 'ChannelID(10)') - cid = channels._channel_id(10, send=True, force=True) + cid = _channels._channel_id(10, send=True, force=True) self.assertEqual(repr(cid), 'ChannelID(10, send=True)') - cid = channels._channel_id(10, recv=True, force=True) + cid = _channels._channel_id(10, recv=True, force=True) self.assertEqual(repr(cid), 'ChannelID(10, recv=True)') - cid = channels._channel_id(10, send=True, recv=True, force=True) + cid = _channels._channel_id(10, send=True, recv=True, force=True) self.assertEqual(repr(cid), 'ChannelID(10)') def test_equality(self): - cid1 = channels.create() - cid2 = channels._channel_id(int(cid1)) - cid3 = channels.create() + cid1 = _channels.create(REPLACE) + cid2 = _channels._channel_id(int(cid1)) + cid3 = _channels.create(REPLACE) self.assertTrue(cid1 == cid1) self.assertTrue(cid1 == cid2) @@ -340,11 +353,11 @@ def test_equality(self): self.assertTrue(cid1 != cid3) def test_shareable(self): - chan = channels.create() + chan = _channels.create(REPLACE) - obj = channels.create() - channels.send(chan, obj, blocking=False) - got = channels.recv(chan) + obj = _channels.create(REPLACE) + _channels.send(chan, obj, blocking=False) + got = recv_nowait(chan) self.assertEqual(got, obj) self.assertIs(type(got), type(obj)) @@ -355,33 +368,33 @@ def test_shareable(self): class ChannelTests(TestBase): def test_create_cid(self): - cid = channels.create() - self.assertIsInstance(cid, channels.ChannelID) + cid = _channels.create(REPLACE) + self.assertIsInstance(cid, _channels.ChannelID) def test_sequential_ids(self): - before = channels.list_all() - id1 = channels.create() - id2 = channels.create() - id3 = channels.create() - after = channels.list_all() + before = [cid for cid, _ in _channels.list_all()] + id1 = _channels.create(REPLACE) + id2 = _channels.create(REPLACE) + id3 = _channels.create(REPLACE) + after = [cid for cid, _ in _channels.list_all()] self.assertEqual(id2, int(id1) + 1) self.assertEqual(id3, int(id2) + 1) self.assertEqual(set(after) - set(before), {id1, id2, id3}) def test_ids_global(self): - id1 = interpreters.create() + id1 = _interpreters.create() out = _run_output(id1, dedent(""" - import _xxinterpchannels as _channels - cid = _channels.create() + import _interpchannels as _channels + cid = _channels.create(3) print(cid) """)) cid1 = int(out.strip()) - id2 = interpreters.create() + id2 = _interpreters.create() out = _run_output(id2, dedent(""" - import _xxinterpchannels as _channels - cid = _channels.create() + import _interpchannels as _channels + cid = _channels.create(3) print(cid) """)) cid2 = int(out.strip()) @@ -390,81 +403,81 @@ def test_ids_global(self): def test_channel_list_interpreters_none(self): """Test listing interpreters for a channel with no associations.""" - # Test for channel with no associated interpreters. - cid = channels.create() - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + # Test for channel with no associated _interpreters. + cid = _channels.create(REPLACE) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(send_interps, []) self.assertEqual(recv_interps, []) def test_channel_list_interpreters_basic(self): - """Test basic listing channel interpreters.""" - interp0 = interpreters.get_main() - cid = channels.create() - channels.send(cid, "send", blocking=False) + """Test basic listing channel _interpreters.""" + interp0, *_ = _interpreters.get_main() + cid = _channels.create(REPLACE) + _channels.send(cid, "send", blocking=False) # Test for a channel that has one end associated to an interpreter. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(send_interps, [interp0]) self.assertEqual(recv_interps, []) - interp1 = interpreters.create() + interp1 = _interpreters.create() _run_output(interp1, dedent(f""" - import _xxinterpchannels as _channels - obj = _channels.recv({cid}) + import _interpchannels as _channels + _channels.recv({cid}) """)) # Test for channel that has both ends associated to an interpreter. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(send_interps, [interp0]) self.assertEqual(recv_interps, [interp1]) def test_channel_list_interpreters_multiple(self): """Test listing interpreters for a channel with many associations.""" - interp0 = interpreters.get_main() - interp1 = interpreters.create() - interp2 = interpreters.create() - interp3 = interpreters.create() - cid = channels.create() + interp0, *_ = _interpreters.get_main() + interp1 = _interpreters.create() + interp2 = _interpreters.create() + interp3 = _interpreters.create() + cid = _channels.create(REPLACE) - channels.send(cid, "send", blocking=False) + _channels.send(cid, "send", blocking=False) _run_output(interp1, dedent(f""" - import _xxinterpchannels as _channels + import _interpchannels as _channels _channels.send({cid}, "send", blocking=False) """)) _run_output(interp2, dedent(f""" - import _xxinterpchannels as _channels - obj = _channels.recv({cid}) + import _interpchannels as _channels + _channels.recv({cid}) """)) _run_output(interp3, dedent(f""" - import _xxinterpchannels as _channels - obj = _channels.recv({cid}) + import _interpchannels as _channels + _channels.recv({cid}) """)) - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(set(send_interps), {interp0, interp1}) self.assertEqual(set(recv_interps), {interp2, interp3}) def test_channel_list_interpreters_destroyed(self): """Test listing channel interpreters with a destroyed interpreter.""" - interp0 = interpreters.get_main() - interp1 = interpreters.create() - cid = channels.create() - channels.send(cid, "send", blocking=False) + interp0, *_ = _interpreters.get_main() + interp1 = _interpreters.create() + cid = _channels.create(REPLACE) + _channels.send(cid, "send", blocking=False) _run_output(interp1, dedent(f""" - import _xxinterpchannels as _channels - obj = _channels.recv({cid}) + import _interpchannels as _channels + _channels.recv({cid}) """)) # Should be one interpreter associated with each end. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(send_interps, [interp0]) self.assertEqual(recv_interps, [interp1]) - interpreters.destroy(interp1) + _interpreters.destroy(interp1) # Destroyed interpreter should not be listed. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(send_interps, [interp0]) self.assertEqual(recv_interps, []) @@ -472,104 +485,104 @@ def test_channel_list_interpreters_released(self): """Test listing channel interpreters with a released channel.""" # Set up one channel with main interpreter on the send end and two # subinterpreters on the receive end. - interp0 = interpreters.get_main() - interp1 = interpreters.create() - interp2 = interpreters.create() - cid = channels.create() - channels.send(cid, "data", blocking=False) + interp0, *_ = _interpreters.get_main() + interp1 = _interpreters.create() + interp2 = _interpreters.create() + cid = _channels.create(REPLACE) + _channels.send(cid, "data", blocking=False) _run_output(interp1, dedent(f""" - import _xxinterpchannels as _channels - obj = _channels.recv({cid}) + import _interpchannels as _channels + _channels.recv({cid}) """)) - channels.send(cid, "data", blocking=False) + _channels.send(cid, "data", blocking=False) _run_output(interp2, dedent(f""" - import _xxinterpchannels as _channels - obj = _channels.recv({cid}) + import _interpchannels as _channels + _channels.recv({cid}) """)) # Check the setup. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(len(send_interps), 1) self.assertEqual(len(recv_interps), 2) # Release the main interpreter from the send end. - channels.release(cid, send=True) - # Send end should have no associated interpreters. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + _channels.release(cid, send=True) + # Send end should have no associated _interpreters. + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(len(send_interps), 0) self.assertEqual(len(recv_interps), 2) # Release one of the subinterpreters from the receive end. _run_output(interp2, dedent(f""" - import _xxinterpchannels as _channels + import _interpchannels as _channels _channels.release({cid}) """)) # Receive end should have the released interpreter removed. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(len(send_interps), 0) self.assertEqual(recv_interps, [interp1]) def test_channel_list_interpreters_closed(self): """Test listing channel interpreters with a closed channel.""" - interp0 = interpreters.get_main() - interp1 = interpreters.create() - cid = channels.create() + interp0, *_ = _interpreters.get_main() + interp1 = _interpreters.create() + cid = _channels.create(REPLACE) # Put something in the channel so that it's not empty. - channels.send(cid, "send", blocking=False) + _channels.send(cid, "send", blocking=False) # Check initial state. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(len(send_interps), 1) self.assertEqual(len(recv_interps), 0) # Force close the channel. - channels.close(cid, force=True) + _channels.close(cid, force=True) # Both ends should raise an error. - with self.assertRaises(channels.ChannelClosedError): - channels.list_interpreters(cid, send=True) - with self.assertRaises(channels.ChannelClosedError): - channels.list_interpreters(cid, send=False) + with self.assertRaises(_channels.ChannelClosedError): + _channels.list_interpreters(cid, send=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.list_interpreters(cid, send=False) def test_channel_list_interpreters_closed_send_end(self): """Test listing channel interpreters with a channel's send end closed.""" - interp0 = interpreters.get_main() - interp1 = interpreters.create() - cid = channels.create() + interp0, *_ = _interpreters.get_main() + interp1 = _interpreters.create() + cid = _channels.create(REPLACE) # Put something in the channel so that it's not empty. - channels.send(cid, "send", blocking=False) + _channels.send(cid, "send", blocking=False) # Check initial state. - send_interps = channels.list_interpreters(cid, send=True) - recv_interps = channels.list_interpreters(cid, send=False) + send_interps = _channels.list_interpreters(cid, send=True) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(len(send_interps), 1) self.assertEqual(len(recv_interps), 0) # Close the send end of the channel. - channels.close(cid, send=True) + _channels.close(cid, send=True) # Send end should raise an error. - with self.assertRaises(channels.ChannelClosedError): - channels.list_interpreters(cid, send=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.list_interpreters(cid, send=True) # Receive end should not be closed (since channel is not empty). - recv_interps = channels.list_interpreters(cid, send=False) + recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(len(recv_interps), 0) # Close the receive end of the channel from a subinterpreter. _run_output(interp1, dedent(f""" - import _xxinterpchannels as _channels + import _interpchannels as _channels _channels.close({cid}, force=True) """)) return # Both ends should raise an error. - with self.assertRaises(channels.ChannelClosedError): - channels.list_interpreters(cid, send=True) - with self.assertRaises(channels.ChannelClosedError): - channels.list_interpreters(cid, send=False) + with self.assertRaises(_channels.ChannelClosedError): + _channels.list_interpreters(cid, send=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.list_interpreters(cid, send=False) def test_allowed_types(self): - cid = channels.create() + cid = _channels.create(REPLACE) objects = [ None, 'spam', @@ -578,8 +591,8 @@ def test_allowed_types(self): ] for obj in objects: with self.subTest(obj): - channels.send(cid, obj, blocking=False) - got = channels.recv(cid) + _channels.send(cid, obj, blocking=False) + got = recv_nowait(cid) self.assertEqual(got, obj) self.assertIs(type(got), type(obj)) @@ -588,16 +601,16 @@ def test_allowed_types(self): # XXX What about between interpreters? def test_run_string_arg_unresolved(self): - cid = channels.create() - interp = interpreters.create() + cid = _channels.create(REPLACE) + interp = _interpreters.create() - interpreters.set___main___attrs(interp, dict(cid=cid.send)) + _interpreters.set___main___attrs(interp, dict(cid=cid.send)) out = _run_output(interp, dedent(""" - import _xxinterpchannels as _channels + import _interpchannels as _channels print(cid.end) _channels.send(cid, b'spam', blocking=False) """)) - obj = channels.recv(cid) + obj = recv_nowait(cid) self.assertEqual(obj, b'spam') self.assertEqual(out.strip(), 'send') @@ -607,17 +620,17 @@ def test_run_string_arg_unresolved(self): # Note: this test caused crashes on some buildbots (bpo-33615). @unittest.skip('disabled until high-level channels exist') def test_run_string_arg_resolved(self): - cid = channels.create() - cid = channels._channel_id(cid, _resolve=True) - interp = interpreters.create() + cid = _channels.create(REPLACE) + cid = _channels._channel_id(cid, _resolve=True) + interp = _interpreters.create() out = _run_output(interp, dedent(""" - import _xxinterpchannels as _channels + import _interpchannels as _channels print(chan.id.end) _channels.send(chan.id, b'spam', blocking=False) """), dict(chan=cid.send)) - obj = channels.recv(cid) + obj = recv_nowait(cid) self.assertEqual(obj, b'spam') self.assertEqual(out.strip(), 'send') @@ -626,65 +639,65 @@ def test_run_string_arg_resolved(self): # send/recv def test_send_recv_main(self): - cid = channels.create() + cid = _channels.create(REPLACE) orig = b'spam' - channels.send(cid, orig, blocking=False) - obj = channels.recv(cid) + _channels.send(cid, orig, blocking=False) + obj = recv_nowait(cid) self.assertEqual(obj, orig) self.assertIsNot(obj, orig) def test_send_recv_same_interpreter(self): - id1 = interpreters.create() + id1 = _interpreters.create() out = _run_output(id1, dedent(""" - import _xxinterpchannels as _channels - cid = _channels.create() + import _interpchannels as _channels + cid = _channels.create(REPLACE) orig = b'spam' _channels.send(cid, orig, blocking=False) - obj = _channels.recv(cid) + obj, _ = _channels.recv(cid) assert obj is not orig assert obj == orig """)) def test_send_recv_different_interpreters(self): - cid = channels.create() - id1 = interpreters.create() + cid = _channels.create(REPLACE) + id1 = _interpreters.create() out = _run_output(id1, dedent(f""" - import _xxinterpchannels as _channels + import _interpchannels as _channels _channels.send({cid}, b'spam', blocking=False) """)) - obj = channels.recv(cid) + obj = recv_nowait(cid) self.assertEqual(obj, b'spam') def test_send_recv_different_threads(self): - cid = channels.create() + cid = _channels.create(REPLACE) def f(): obj = recv_wait(cid) - channels.send(cid, obj) + _channels.send(cid, obj) t = threading.Thread(target=f) t.start() - channels.send(cid, b'spam') + _channels.send(cid, b'spam') obj = recv_wait(cid) t.join() self.assertEqual(obj, b'spam') def test_send_recv_different_interpreters_and_threads(self): - cid = channels.create() - id1 = interpreters.create() + cid = _channels.create(REPLACE) + id1 = _interpreters.create() out = None def f(): nonlocal out out = _run_output(id1, dedent(f""" import time - import _xxinterpchannels as _channels + import _interpchannels as _channels while True: try: - obj = _channels.recv({cid}) + obj, _ = _channels.recv({cid}) break except _channels.ChannelEmptyError: time.sleep(0.1) @@ -694,38 +707,38 @@ def f(): t = threading.Thread(target=f) t.start() - channels.send(cid, b'spam') + _channels.send(cid, b'spam') obj = recv_wait(cid) t.join() self.assertEqual(obj, b'eggs') def test_send_not_found(self): - with self.assertRaises(channels.ChannelNotFoundError): - channels.send(10, b'spam') + with self.assertRaises(_channels.ChannelNotFoundError): + _channels.send(10, b'spam') def test_recv_not_found(self): - with self.assertRaises(channels.ChannelNotFoundError): - channels.recv(10) + with self.assertRaises(_channels.ChannelNotFoundError): + _channels.recv(10) def test_recv_empty(self): - cid = channels.create() - with self.assertRaises(channels.ChannelEmptyError): - channels.recv(cid) + cid = _channels.create(REPLACE) + with self.assertRaises(_channels.ChannelEmptyError): + _channels.recv(cid) def test_recv_default(self): default = object() - cid = channels.create() - obj1 = channels.recv(cid, default) - channels.send(cid, None, blocking=False) - channels.send(cid, 1, blocking=False) - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'eggs', blocking=False) - obj2 = channels.recv(cid, default) - obj3 = channels.recv(cid, default) - obj4 = channels.recv(cid) - obj5 = channels.recv(cid, default) - obj6 = channels.recv(cid, default) + cid = _channels.create(REPLACE) + obj1 = recv_nowait(cid, default) + _channels.send(cid, None, blocking=False) + _channels.send(cid, 1, blocking=False) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'eggs', blocking=False) + obj2 = recv_nowait(cid, default) + obj3 = recv_nowait(cid, default) + obj4 = recv_nowait(cid) + obj5 = recv_nowait(cid, default) + obj6 = recv_nowait(cid, default) self.assertIs(obj1, default) self.assertIs(obj2, None) @@ -736,32 +749,33 @@ def test_recv_default(self): def test_recv_sending_interp_destroyed(self): with self.subTest('closed'): - cid1 = channels.create() - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxinterpchannels as _channels + cid1 = _channels.create(REPLACE) + interp = _interpreters.create() + _interpreters.run_string(interp, dedent(f""" + import _interpchannels as _channels _channels.send({cid1}, b'spam', blocking=False) """)) - interpreters.destroy(interp) + _interpreters.destroy(interp) with self.assertRaisesRegex(RuntimeError, f'channel {cid1} is closed'): - channels.recv(cid1) + _channels.recv(cid1) del cid1 with self.subTest('still open'): - cid2 = channels.create() - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxinterpchannels as _channels + cid2 = _channels.create(REPLACE) + interp = _interpreters.create() + _interpreters.run_string(interp, dedent(f""" + import _interpchannels as _channels _channels.send({cid2}, b'spam', blocking=False) """)) - channels.send(cid2, b'eggs', blocking=False) - interpreters.destroy(interp) + _channels.send(cid2, b'eggs', blocking=False) + _interpreters.destroy(interp) - channels.recv(cid2) + recv_nowait(cid2, unbound=True) + recv_nowait(cid2, unbound=False) with self.assertRaisesRegex(RuntimeError, f'channel {cid2} is empty'): - channels.recv(cid2) + _channels.recv(cid2) del cid2 #------------------- @@ -769,9 +783,9 @@ def test_recv_sending_interp_destroyed(self): def test_send_buffer(self): buf = bytearray(b'spamspamspam') - cid = channels.create() - channels.send_buffer(cid, buf, blocking=False) - obj = channels.recv(cid) + cid = _channels.create(REPLACE) + _channels.send_buffer(cid, buf, blocking=False) + obj = recv_nowait(cid) self.assertIsNot(obj, buf) self.assertIsInstance(obj, memoryview) @@ -789,18 +803,18 @@ def build_send_waiter(self, obj, *, buffer=False): # We want a long enough sleep that send() actually has to wait. if buffer: - send = channels.send_buffer + send = _channels.send_buffer else: - send = channels.send + send = _channels.send - cid = channels.create() + cid = _channels.create(REPLACE) try: started = time.monotonic() send(cid, obj, blocking=False) stopped = time.monotonic() - channels.recv(cid) + recv_nowait(cid) finally: - channels.destroy(cid) + _channels.destroy(cid) delay = stopped - started # seconds delay *= 3 @@ -812,14 +826,14 @@ def test_send_blocking_waiting(self): received = None obj = b'spam' wait = self.build_send_waiter(obj) - cid = channels.create() + cid = _channels.create(REPLACE) def f(): nonlocal received wait() received = recv_wait(cid) t = threading.Thread(target=f) t.start() - channels.send(cid, obj, blocking=True) + _channels.send(cid, obj, blocking=True) t.join() self.assertEqual(received, obj) @@ -828,14 +842,14 @@ def test_send_buffer_blocking_waiting(self): received = None obj = bytearray(b'spam') wait = self.build_send_waiter(obj, buffer=True) - cid = channels.create() + cid = _channels.create(REPLACE) def f(): nonlocal received wait() received = recv_wait(cid) t = threading.Thread(target=f) t.start() - channels.send_buffer(cid, obj, blocking=True) + _channels.send_buffer(cid, obj, blocking=True) t.join() self.assertEqual(received, obj) @@ -843,13 +857,13 @@ def f(): def test_send_blocking_no_wait(self): received = None obj = b'spam' - cid = channels.create() + cid = _channels.create(REPLACE) def f(): nonlocal received received = recv_wait(cid) t = threading.Thread(target=f) t.start() - channels.send(cid, obj, blocking=True) + _channels.send(cid, obj, blocking=True) t.join() self.assertEqual(received, obj) @@ -857,13 +871,13 @@ def f(): def test_send_buffer_blocking_no_wait(self): received = None obj = bytearray(b'spam') - cid = channels.create() + cid = _channels.create(REPLACE) def f(): nonlocal received received = recv_wait(cid) t = threading.Thread(target=f) t.start() - channels.send_buffer(cid, obj, blocking=True) + _channels.send_buffer(cid, obj, blocking=True) t.join() self.assertEqual(received, obj) @@ -872,25 +886,25 @@ def test_send_timeout(self): obj = b'spam' with self.subTest('non-blocking with timeout'): - cid = channels.create() + cid = _channels.create(REPLACE) with self.assertRaises(ValueError): - channels.send(cid, obj, blocking=False, timeout=0.1) + _channels.send(cid, obj, blocking=False, timeout=0.1) with self.subTest('timeout hit'): - cid = channels.create() + cid = _channels.create(REPLACE) with self.assertRaises(TimeoutError): - channels.send(cid, obj, blocking=True, timeout=0.1) - with self.assertRaises(channels.ChannelEmptyError): - received = channels.recv(cid) + _channels.send(cid, obj, blocking=True, timeout=0.1) + with self.assertRaises(_channels.ChannelEmptyError): + received = recv_nowait(cid) print(repr(received)) with self.subTest('timeout not hit'): - cid = channels.create() + cid = _channels.create(REPLACE) def f(): recv_wait(cid) t = threading.Thread(target=f) t.start() - channels.send(cid, obj, blocking=True, timeout=10) + _channels.send(cid, obj, blocking=True, timeout=10) t.join() def test_send_buffer_timeout(self): @@ -909,25 +923,25 @@ def test_send_buffer_timeout(self): obj = bytearray(b'spam') with self.subTest('non-blocking with timeout'): - cid = channels.create() + cid = _channels.create(REPLACE) with self.assertRaises(ValueError): - channels.send_buffer(cid, obj, blocking=False, timeout=0.1) + _channels.send_buffer(cid, obj, blocking=False, timeout=0.1) with self.subTest('timeout hit'): - cid = channels.create() + cid = _channels.create(REPLACE) with self.assertRaises(TimeoutError): - channels.send_buffer(cid, obj, blocking=True, timeout=0.1) - with self.assertRaises(channels.ChannelEmptyError): - received = channels.recv(cid) + _channels.send_buffer(cid, obj, blocking=True, timeout=0.1) + with self.assertRaises(_channels.ChannelEmptyError): + received = recv_nowait(cid) print(repr(received)) with self.subTest('timeout not hit'): - cid = channels.create() + cid = _channels.create(REPLACE) def f(): recv_wait(cid) t = threading.Thread(target=f) t.start() - channels.send_buffer(cid, obj, blocking=True, timeout=10) + _channels.send_buffer(cid, obj, blocking=True, timeout=10) t.join() def test_send_closed_while_waiting(self): @@ -935,25 +949,25 @@ def test_send_closed_while_waiting(self): wait = self.build_send_waiter(obj) with self.subTest('without timeout'): - cid = channels.create() + cid = _channels.create(REPLACE) def f(): wait() - channels.close(cid, force=True) + _channels.close(cid, force=True) t = threading.Thread(target=f) t.start() - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, obj, blocking=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, obj, blocking=True) t.join() with self.subTest('with timeout'): - cid = channels.create() + cid = _channels.create(REPLACE) def f(): wait() - channels.close(cid, force=True) + _channels.close(cid, force=True) t = threading.Thread(target=f) t.start() - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, obj, blocking=True, timeout=30) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, obj, blocking=True, timeout=30) t.join() def test_send_buffer_closed_while_waiting(self): @@ -973,73 +987,73 @@ def test_send_buffer_closed_while_waiting(self): wait = self.build_send_waiter(obj, buffer=True) with self.subTest('without timeout'): - cid = channels.create() + cid = _channels.create(REPLACE) def f(): wait() - channels.close(cid, force=True) + _channels.close(cid, force=True) t = threading.Thread(target=f) t.start() - with self.assertRaises(channels.ChannelClosedError): - channels.send_buffer(cid, obj, blocking=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send_buffer(cid, obj, blocking=True) t.join() with self.subTest('with timeout'): - cid = channels.create() + cid = _channels.create(REPLACE) def f(): wait() - channels.close(cid, force=True) + _channels.close(cid, force=True) t = threading.Thread(target=f) t.start() - with self.assertRaises(channels.ChannelClosedError): - channels.send_buffer(cid, obj, blocking=True, timeout=30) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send_buffer(cid, obj, blocking=True, timeout=30) t.join() #------------------- # close def test_close_single_user(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.close(cid) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + recv_nowait(cid) + _channels.close(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_close_multiple_users(self): - cid = channels.create() - id1 = interpreters.create() - id2 = interpreters.create() - interpreters.run_string(id1, dedent(f""" - import _xxinterpchannels as _channels + cid = _channels.create(REPLACE) + id1 = _interpreters.create() + id2 = _interpreters.create() + _interpreters.run_string(id1, dedent(f""" + import _interpchannels as _channels _channels.send({cid}, b'spam', blocking=False) """)) - interpreters.run_string(id2, dedent(f""" - import _xxinterpchannels as _channels + _interpreters.run_string(id2, dedent(f""" + import _interpchannels as _channels _channels.recv({cid}) """)) - channels.close(cid) + _channels.close(cid) - excsnap = interpreters.run_string(id1, dedent(f""" + excsnap = _interpreters.run_string(id1, dedent(f""" _channels.send({cid}, b'spam') """)) self.assertEqual(excsnap.type.__name__, 'ChannelClosedError') - excsnap = interpreters.run_string(id2, dedent(f""" + excsnap = _interpreters.run_string(id2, dedent(f""" _channels.send({cid}, b'spam') """)) self.assertEqual(excsnap.type.__name__, 'ChannelClosedError') def test_close_multiple_times(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.close(cid) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + recv_nowait(cid) + _channels.close(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.close(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.close(cid) def test_close_empty(self): tests = [ @@ -1050,149 +1064,149 @@ def test_close_empty(self): ] for send, recv in tests: with self.subTest((send, recv)): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.close(cid, send=send, recv=recv) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + recv_nowait(cid) + _channels.close(cid, send=send, recv=recv) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_close_defaults_with_unused_items(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) - with self.assertRaises(channels.ChannelNotEmptyError): - channels.close(cid) - channels.recv(cid) - channels.send(cid, b'eggs', blocking=False) + with self.assertRaises(_channels.ChannelNotEmptyError): + _channels.close(cid) + recv_nowait(cid) + _channels.send(cid, b'eggs', blocking=False) def test_close_recv_with_unused_items_unforced(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) - - with self.assertRaises(channels.ChannelNotEmptyError): - channels.close(cid, recv=True) - channels.recv(cid) - channels.send(cid, b'eggs', blocking=False) - channels.recv(cid) - channels.recv(cid) - channels.close(cid, recv=True) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) + + with self.assertRaises(_channels.ChannelNotEmptyError): + _channels.close(cid, recv=True) + recv_nowait(cid) + _channels.send(cid, b'eggs', blocking=False) + recv_nowait(cid) + recv_nowait(cid) + _channels.close(cid, recv=True) def test_close_send_with_unused_items_unforced(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) - channels.close(cid, send=True) - - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - channels.recv(cid) - channels.recv(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) + _channels.close(cid, send=True) + + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + recv_nowait(cid) + recv_nowait(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_close_both_with_unused_items_unforced(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) - - with self.assertRaises(channels.ChannelNotEmptyError): - channels.close(cid, recv=True, send=True) - channels.recv(cid) - channels.send(cid, b'eggs', blocking=False) - channels.recv(cid) - channels.recv(cid) - channels.close(cid, recv=True) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) + + with self.assertRaises(_channels.ChannelNotEmptyError): + _channels.close(cid, recv=True, send=True) + recv_nowait(cid) + _channels.send(cid, b'eggs', blocking=False) + recv_nowait(cid) + recv_nowait(cid) + _channels.close(cid, recv=True) def test_close_recv_with_unused_items_forced(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) - channels.close(cid, recv=True, force=True) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) + _channels.close(cid, recv=True, force=True) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_close_send_with_unused_items_forced(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) - channels.close(cid, send=True, force=True) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) + _channels.close(cid, send=True, force=True) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_close_both_with_unused_items_forced(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) - channels.close(cid, send=True, recv=True, force=True) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) + _channels.close(cid, send=True, recv=True, force=True) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_close_never_used(self): - cid = channels.create() - channels.close(cid) + cid = _channels.create(REPLACE) + _channels.close(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'spam') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'spam') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_close_by_unassociated_interp(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxinterpchannels as _channels + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + interp = _interpreters.create() + _interpreters.run_string(interp, dedent(f""" + import _interpchannels as _channels _channels.close({cid}, force=True) """)) - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.close(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.close(cid) def test_close_used_multiple_times_by_single_user(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.close(cid, force=True) - - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'spam', blocking=False) + recv_nowait(cid) + _channels.close(cid, force=True) + + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_channel_list_interpreters_invalid_channel(self): - cid = channels.create() + cid = _channels.create(REPLACE) # Test for invalid channel ID. - with self.assertRaises(channels.ChannelNotFoundError): - channels.list_interpreters(1000, send=True) + with self.assertRaises(_channels.ChannelNotFoundError): + _channels.list_interpreters(1000, send=True) - channels.close(cid) + _channels.close(cid) # Test for a channel that has been closed. - with self.assertRaises(channels.ChannelClosedError): - channels.list_interpreters(cid, send=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.list_interpreters(cid, send=True) def test_channel_list_interpreters_invalid_args(self): # Tests for invalid arguments passed to the API. - cid = channels.create() + cid = _channels.create(REPLACE) with self.assertRaises(TypeError): - channels.list_interpreters(cid) + _channels.list_interpreters(cid) class ChannelReleaseTests(TestBase): @@ -1239,125 +1253,125 @@ class ChannelReleaseTests(TestBase): """ def test_single_user(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.release(cid, send=True, recv=True) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + recv_nowait(cid) + _channels.release(cid, send=True, recv=True) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_multiple_users(self): - cid = channels.create() - id1 = interpreters.create() - id2 = interpreters.create() - interpreters.run_string(id1, dedent(f""" - import _xxinterpchannels as _channels + cid = _channels.create(REPLACE) + id1 = _interpreters.create() + id2 = _interpreters.create() + _interpreters.run_string(id1, dedent(f""" + import _interpchannels as _channels _channels.send({cid}, b'spam', blocking=False) """)) out = _run_output(id2, dedent(f""" - import _xxinterpchannels as _channels - obj = _channels.recv({cid}) + import _interpchannels as _channels + obj, _ = _channels.recv({cid}) _channels.release({cid}) print(repr(obj)) """)) - interpreters.run_string(id1, dedent(f""" + _interpreters.run_string(id1, dedent(f""" _channels.release({cid}) """)) self.assertEqual(out.strip(), "b'spam'") def test_no_kwargs(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.release(cid) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + recv_nowait(cid) + _channels.release(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_multiple_times(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.release(cid, send=True, recv=True) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + recv_nowait(cid) + _channels.release(cid, send=True, recv=True) - with self.assertRaises(channels.ChannelClosedError): - channels.release(cid, send=True, recv=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.release(cid, send=True, recv=True) def test_with_unused_items(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'ham', blocking=False) - channels.release(cid, send=True, recv=True) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'ham', blocking=False) + _channels.release(cid, send=True, recv=True) - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_never_used(self): - cid = channels.create() - channels.release(cid) + cid = _channels.create(REPLACE) + _channels.release(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'spam') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'spam') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_by_unassociated_interp(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxinterpchannels as _channels + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + interp = _interpreters.create() + _interpreters.run_string(interp, dedent(f""" + import _interpchannels as _channels _channels.release({cid}) """)) - obj = channels.recv(cid) - channels.release(cid) + obj = recv_nowait(cid) + _channels.release(cid) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') self.assertEqual(obj, b'spam') def test_close_if_unassociated(self): # XXX Something's not right with this test... - cid = channels.create() - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxinterpchannels as _channels + cid = _channels.create(REPLACE) + interp = _interpreters.create() + _interpreters.run_string(interp, dedent(f""" + import _interpchannels as _channels obj = _channels.send({cid}, b'spam', blocking=False) _channels.release({cid}) """)) - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) def test_partially(self): # XXX Is partial close too weird/confusing? - cid = channels.create() - channels.send(cid, None, blocking=False) - channels.recv(cid) - channels.send(cid, b'spam', blocking=False) - channels.release(cid, send=True) - obj = channels.recv(cid) + cid = _channels.create(REPLACE) + _channels.send(cid, None, blocking=False) + recv_nowait(cid) + _channels.send(cid, b'spam', blocking=False) + _channels.release(cid, send=True) + obj = recv_nowait(cid) self.assertEqual(obj, b'spam') def test_used_multiple_times_by_single_user(self): - cid = channels.create() - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'spam', blocking=False) - channels.send(cid, b'spam', blocking=False) - channels.recv(cid) - channels.release(cid, send=True, recv=True) + cid = _channels.create(REPLACE) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'spam', blocking=False) + recv_nowait(cid) + _channels.release(cid, send=True, recv=True) - with self.assertRaises(channels.ChannelClosedError): - channels.send(cid, b'eggs') - with self.assertRaises(channels.ChannelClosedError): - channels.recv(cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(cid, b'eggs') + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(cid) class ChannelCloseFixture(namedtuple('ChannelCloseFixture', @@ -1427,18 +1441,18 @@ def clean_up(self): def _new_channel(self, creator): if creator.name == 'main': - return channels.create() + return _channels.create(REPLACE) else: - ch = channels.create() + ch = _channels.create(REPLACE) run_interp(creator.id, f""" - import _xxsubinterpreters + import _interpreters cid = _xxsubchannels.create() # We purposefully send back an int to avoid tying the # channel to the other interpreter. _xxsubchannels.send({ch}, int(cid), blocking=False) - del _xxsubinterpreters + del _interpreters """) - self._cid = channels.recv(ch) + self._cid = recv_nowait(ch) return self._cid def _get_interpreter(self, interp): @@ -1463,13 +1477,13 @@ def _prep_interpreter(self, interp): if interp.name == 'main': return run_interp(interp.id, f""" - import _xxinterpchannels as channels - import test.test__xxinterpchannels as helpers + import _interpchannels as channels + import test.test__interpchannels as helpers ChannelState = helpers.ChannelState try: cid except NameError: - cid = channels._channel_id({self.cid}) + cid = _channels._channel_id({self.cid}) """) @@ -1656,7 +1670,7 @@ def run_action(self, fix, action, *, hideclosed=True): ) fix.record_action(action, result) else: - _cid = channels.create() + _cid = _channels.create(REPLACE) run_interp(interp.id, f""" result = helpers.run_action( {fix.cid}, @@ -1665,12 +1679,12 @@ def run_action(self, fix, action, *, hideclosed=True): {repr(fix.state)}, hideclosed={hideclosed}, ) - channels.send({_cid}, result.pending.to_bytes(1, 'little'), blocking=False) - channels.send({_cid}, b'X' if result.closed else b'', blocking=False) + _channels.send({_cid}, result.pending.to_bytes(1, 'little'), blocking=False) + _channels.send({_cid}, b'X' if result.closed else b'', blocking=False) """) result = ChannelState( - pending=int.from_bytes(channels.recv(_cid), 'little'), - closed=bool(channels.recv(_cid)), + pending=int.from_bytes(recv_nowait(_cid), 'little'), + closed=bool(recv_nowait(_cid)), ) fix.record_action(action, result) @@ -1693,42 +1707,42 @@ def _close(self, fix, *, force): if not fix.expect_closed_error(): self.run_action(fix, close, hideclosed=False) else: - with self.assertRaises(channels.ChannelClosedError): + with self.assertRaises(_channels.ChannelClosedError): self.run_action(fix, close, hideclosed=False) def _assert_closed_in_interp(self, fix, interp=None): if interp is None or interp.name == 'main': - with self.assertRaises(channels.ChannelClosedError): - channels.recv(fix.cid) - with self.assertRaises(channels.ChannelClosedError): - channels.send(fix.cid, b'spam') - with self.assertRaises(channels.ChannelClosedError): - channels.close(fix.cid) - with self.assertRaises(channels.ChannelClosedError): - channels.close(fix.cid, force=True) + with self.assertRaises(_channels.ChannelClosedError): + _channels.recv(fix.cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.send(fix.cid, b'spam') + with self.assertRaises(_channels.ChannelClosedError): + _channels.close(fix.cid) + with self.assertRaises(_channels.ChannelClosedError): + _channels.close(fix.cid, force=True) else: run_interp(interp.id, """ with helpers.expect_channel_closed(): - channels.recv(cid) + _channels.recv(cid) """) run_interp(interp.id, """ with helpers.expect_channel_closed(): - channels.send(cid, b'spam', blocking=False) + _channels.send(cid, b'spam', blocking=False) """) run_interp(interp.id, """ with helpers.expect_channel_closed(): - channels.close(cid) + _channels.close(cid) """) run_interp(interp.id, """ with helpers.expect_channel_closed(): - channels.close(cid, force=True) + _channels.close(cid, force=True) """) def _assert_closed(self, fix): self.assertTrue(fix.state.closed) for _ in range(fix.state.pending): - channels.recv(fix.cid) + recv_nowait(fix.cid) self._assert_closed_in_interp(fix) for interp in ('same', 'other'): diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__interpreters.py similarity index 71% rename from Lib/test/test__xxsubinterpreters.py rename to Lib/test/test__interpreters.py index a76e4d0ade5b8a..fd444f1f06ce48 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__interpreters.py @@ -7,15 +7,15 @@ import threading import unittest -import _testinternalcapi from test import support from test.support import import_helper from test.support import os_helper from test.support import script_helper -interpreters = import_helper.import_module('_xxsubinterpreters') -from _xxsubinterpreters import InterpreterNotFoundError +_interpreters = import_helper.import_module('_interpreters') +_testinternalcapi = import_helper.import_module('_testinternalcapi') +from _interpreters import InterpreterNotFoundError ################################## @@ -36,18 +36,18 @@ def _captured_script(script): def _run_output(interp, request): script, rpipe = _captured_script(request) with rpipe: - interpreters.run_string(interp, script) + _interpreters.run_string(interp, script) return rpipe.read() def _wait_for_interp_to_run(interp, timeout=None): # bpo-37224: Running this test file in multiprocesses will fail randomly. # The failure reason is that the thread can't acquire the cpu to - # run subinterpreter eariler than the main thread in multiprocess. + # run subinterpreter earlier than the main thread in multiprocess. if timeout is None: timeout = support.SHORT_TIMEOUT for _ in support.sleeping_retry(timeout, error=False): - if interpreters.is_running(interp): + if _interpreters.is_running(interp): break else: raise RuntimeError('interp is not running') @@ -57,7 +57,7 @@ def _wait_for_interp_to_run(interp, timeout=None): def _running(interp): r, w = os.pipe() def run(): - interpreters.run_string(interp, dedent(f""" + _interpreters.run_string(interp, dedent(f""" # wait for "signal" with open({r}, encoding="utf-8") as rpipe: rpipe.read() @@ -75,12 +75,12 @@ def run(): def clean_up_interpreters(): - for id in interpreters.list_all(): + for id, *_ in _interpreters.list_all(): if id == 0: # main continue try: - interpreters.destroy(id) - except RuntimeError: + _interpreters.destroy(id) + except _interpreters.InterpreterError: pass # already destroyed @@ -112,7 +112,7 @@ def test_default_shareables(self): for obj in shareables: with self.subTest(obj): self.assertTrue( - interpreters.is_shareable(obj)) + _interpreters.is_shareable(obj)) def test_not_shareable(self): class Cheese: @@ -141,7 +141,7 @@ class SubBytes(bytes): for obj in not_shareables: with self.subTest(repr(obj)): self.assertFalse( - interpreters.is_shareable(obj)) + _interpreters.is_shareable(obj)) class ShareableTypeTests(unittest.TestCase): @@ -230,8 +230,8 @@ class ModuleTests(TestBase): def test_import_in_interpreter(self): _run_output( - interpreters.create(), - 'import _xxsubinterpreters as _interpreters', + _interpreters.create(), + 'import _interpreters', ) @@ -241,45 +241,45 @@ def test_import_in_interpreter(self): class ListAllTests(TestBase): def test_initial(self): - main = interpreters.get_main() - ids = interpreters.list_all() + main, *_ = _interpreters.get_main() + ids = [id for id, *_ in _interpreters.list_all()] self.assertEqual(ids, [main]) def test_after_creating(self): - main = interpreters.get_main() - first = interpreters.create() - second = interpreters.create() - ids = interpreters.list_all() + main, *_ = _interpreters.get_main() + first = _interpreters.create() + second = _interpreters.create() + ids = [id for id, *_ in _interpreters.list_all()] self.assertEqual(ids, [main, first, second]) def test_after_destroying(self): - main = interpreters.get_main() - first = interpreters.create() - second = interpreters.create() - interpreters.destroy(first) - ids = interpreters.list_all() + main, *_ = _interpreters.get_main() + first = _interpreters.create() + second = _interpreters.create() + _interpreters.destroy(first) + ids = [id for id, *_ in _interpreters.list_all()] self.assertEqual(ids, [main, second]) class GetCurrentTests(TestBase): def test_main(self): - main = interpreters.get_main() - cur = interpreters.get_current() + main, *_ = _interpreters.get_main() + cur, *_ = _interpreters.get_current() self.assertEqual(cur, main) self.assertIsInstance(cur, int) def test_subinterpreter(self): - main = interpreters.get_main() - interp = interpreters.create() + main, *_ = _interpreters.get_main() + interp = _interpreters.create() out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters - cur = _interpreters.get_current() + import _interpreters + cur, *_ = _interpreters.get_current() print(cur) assert isinstance(cur, int) """)) cur = int(out.strip()) - _, expected = interpreters.list_all() + _, expected = [id for id, *_ in _interpreters.list_all()] self.assertEqual(cur, expected) self.assertNotEqual(cur, main) @@ -287,17 +287,17 @@ def test_subinterpreter(self): class GetMainTests(TestBase): def test_from_main(self): - [expected] = interpreters.list_all() - main = interpreters.get_main() + [expected] = [id for id, *_ in _interpreters.list_all()] + main, *_ = _interpreters.get_main() self.assertEqual(main, expected) self.assertIsInstance(main, int) def test_from_subinterpreter(self): - [expected] = interpreters.list_all() - interp = interpreters.create() + [expected] = [id for id, *_ in _interpreters.list_all()] + interp = _interpreters.create() out = _run_output(interp, dedent(""" - import _xxsubinterpreters as _interpreters - main = _interpreters.get_main() + import _interpreters + main, *_ = _interpreters.get_main() print(main) assert isinstance(main, int) """)) @@ -308,22 +308,22 @@ def test_from_subinterpreter(self): class IsRunningTests(TestBase): def test_main(self): - main = interpreters.get_main() - self.assertTrue(interpreters.is_running(main)) + main, *_ = _interpreters.get_main() + self.assertTrue(_interpreters.is_running(main)) @unittest.skip('Fails on FreeBSD') def test_subinterpreter(self): - interp = interpreters.create() - self.assertFalse(interpreters.is_running(interp)) + interp = _interpreters.create() + self.assertFalse(_interpreters.is_running(interp)) with _running(interp): - self.assertTrue(interpreters.is_running(interp)) - self.assertFalse(interpreters.is_running(interp)) + self.assertTrue(_interpreters.is_running(interp)) + self.assertFalse(_interpreters.is_running(interp)) def test_from_subinterpreter(self): - interp = interpreters.create() + interp = _interpreters.create() out = _run_output(interp, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters if _interpreters.is_running({interp}): print(True) else: @@ -332,34 +332,35 @@ def test_from_subinterpreter(self): self.assertEqual(out.strip(), 'True') def test_already_destroyed(self): - interp = interpreters.create() - interpreters.destroy(interp) + interp = _interpreters.create() + _interpreters.destroy(interp) with self.assertRaises(InterpreterNotFoundError): - interpreters.is_running(interp) + _interpreters.is_running(interp) def test_does_not_exist(self): with self.assertRaises(InterpreterNotFoundError): - interpreters.is_running(1_000_000) + _interpreters.is_running(1_000_000) def test_bad_id(self): with self.assertRaises(ValueError): - interpreters.is_running(-1) + _interpreters.is_running(-1) class CreateTests(TestBase): def test_in_main(self): - id = interpreters.create() + id = _interpreters.create() self.assertIsInstance(id, int) - self.assertIn(id, interpreters.list_all()) + after = [id for id, *_ in _interpreters.list_all()] + self.assertIn(id, after) @unittest.skip('enable this test when working on pystate.c') def test_unique_id(self): seen = set() for _ in range(100): - id = interpreters.create() - interpreters.destroy(id) + id = _interpreters.create() + _interpreters.destroy(id) seen.add(id) self.assertEqual(len(seen), 100) @@ -369,7 +370,7 @@ def test_in_thread(self): id = None def f(): nonlocal id - id = interpreters.create() + id = _interpreters.create() lock.acquire() lock.release() @@ -377,29 +378,31 @@ def f(): with lock: t.start() t.join() - self.assertIn(id, interpreters.list_all()) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertIn(id, after) def test_in_subinterpreter(self): - main, = interpreters.list_all() - id1 = interpreters.create() + main, = [id for id, *_ in _interpreters.list_all()] + id1 = _interpreters.create() out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters id = _interpreters.create() print(id) assert isinstance(id, int) """)) id2 = int(out.strip()) - self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, {main, id1, id2}) def test_in_threaded_subinterpreter(self): - main, = interpreters.list_all() - id1 = interpreters.create() + main, = [id for id, *_ in _interpreters.list_all()] + id1 = _interpreters.create() id2 = None def f(): nonlocal id2 out = _run_output(id1, dedent(""" - import _xxsubinterpreters as _interpreters + import _interpreters id = _interpreters.create() print(id) """)) @@ -409,144 +412,184 @@ def f(): t.start() t.join() - self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, {main, id1, id2}) def test_after_destroy_all(self): - before = set(interpreters.list_all()) + before = set(id for id, *_ in _interpreters.list_all()) # Create 3 subinterpreters. ids = [] for _ in range(3): - id = interpreters.create() + id = _interpreters.create() ids.append(id) # Now destroy them. for id in ids: - interpreters.destroy(id) + _interpreters.destroy(id) # Finally, create another. - id = interpreters.create() - self.assertEqual(set(interpreters.list_all()), before | {id}) + id = _interpreters.create() + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, before | {id}) def test_after_destroy_some(self): - before = set(interpreters.list_all()) + before = set(id for id, *_ in _interpreters.list_all()) # Create 3 subinterpreters. - id1 = interpreters.create() - id2 = interpreters.create() - id3 = interpreters.create() + id1 = _interpreters.create() + id2 = _interpreters.create() + id3 = _interpreters.create() # Now destroy 2 of them. - interpreters.destroy(id1) - interpreters.destroy(id3) + _interpreters.destroy(id1) + _interpreters.destroy(id3) # Finally, create another. - id = interpreters.create() - self.assertEqual(set(interpreters.list_all()), before | {id, id2}) + id = _interpreters.create() + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, before | {id, id2}) class DestroyTests(TestBase): def test_one(self): - id1 = interpreters.create() - id2 = interpreters.create() - id3 = interpreters.create() - self.assertIn(id2, interpreters.list_all()) - interpreters.destroy(id2) - self.assertNotIn(id2, interpreters.list_all()) - self.assertIn(id1, interpreters.list_all()) - self.assertIn(id3, interpreters.list_all()) + id1 = _interpreters.create() + id2 = _interpreters.create() + id3 = _interpreters.create() + before = set(id for id, *_ in _interpreters.list_all()) + self.assertIn(id2, before) + + _interpreters.destroy(id2) + + after = set(id for id, *_ in _interpreters.list_all()) + self.assertNotIn(id2, after) + self.assertIn(id1, after) + self.assertIn(id3, after) def test_all(self): - before = set(interpreters.list_all()) + initial = set(id for id, *_ in _interpreters.list_all()) ids = set() for _ in range(3): - id = interpreters.create() + id = _interpreters.create() ids.add(id) - self.assertEqual(set(interpreters.list_all()), before | ids) + before = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(before, initial | ids) for id in ids: - interpreters.destroy(id) - self.assertEqual(set(interpreters.list_all()), before) + _interpreters.destroy(id) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, initial) def test_main(self): - main, = interpreters.list_all() - with self.assertRaises(RuntimeError): - interpreters.destroy(main) + main, = [id for id, *_ in _interpreters.list_all()] + with self.assertRaises(_interpreters.InterpreterError): + _interpreters.destroy(main) def f(): - with self.assertRaises(RuntimeError): - interpreters.destroy(main) + with self.assertRaises(_interpreters.InterpreterError): + _interpreters.destroy(main) t = threading.Thread(target=f) t.start() t.join() def test_already_destroyed(self): - id = interpreters.create() - interpreters.destroy(id) + id = _interpreters.create() + _interpreters.destroy(id) with self.assertRaises(InterpreterNotFoundError): - interpreters.destroy(id) + _interpreters.destroy(id) def test_does_not_exist(self): with self.assertRaises(InterpreterNotFoundError): - interpreters.destroy(1_000_000) + _interpreters.destroy(1_000_000) def test_bad_id(self): with self.assertRaises(ValueError): - interpreters.destroy(-1) + _interpreters.destroy(-1) def test_from_current(self): - main, = interpreters.list_all() - id = interpreters.create() + main, = [id for id, *_ in _interpreters.list_all()] + id = _interpreters.create() script = dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters try: _interpreters.destroy({id}) - except RuntimeError: + except _interpreters.InterpreterError: pass """) - interpreters.run_string(id, script) - self.assertEqual(set(interpreters.list_all()), {main, id}) + _interpreters.run_string(id, script) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, {main, id}) def test_from_sibling(self): - main, = interpreters.list_all() - id1 = interpreters.create() - id2 = interpreters.create() + main, = [id for id, *_ in _interpreters.list_all()] + id1 = _interpreters.create() + id2 = _interpreters.create() script = dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters _interpreters.destroy({id2}) """) - interpreters.run_string(id1, script) + _interpreters.run_string(id1, script) - self.assertEqual(set(interpreters.list_all()), {main, id1}) + after = set(id for id, *_ in _interpreters.list_all()) + self.assertEqual(after, {main, id1}) def test_from_other_thread(self): - id = interpreters.create() + id = _interpreters.create() def f(): - interpreters.destroy(id) + _interpreters.destroy(id) t = threading.Thread(target=f) t.start() t.join() def test_still_running(self): - main, = interpreters.list_all() - interp = interpreters.create() + main, = [id for id, *_ in _interpreters.list_all()] + interp = _interpreters.create() with _running(interp): - self.assertTrue(interpreters.is_running(interp), + self.assertTrue(_interpreters.is_running(interp), msg=f"Interp {interp} should be running before destruction.") - with self.assertRaises(RuntimeError, + with self.assertRaises(_interpreters.InterpreterError, msg=f"Should not be able to destroy interp {interp} while it's still running."): - interpreters.destroy(interp) - self.assertTrue(interpreters.is_running(interp)) + _interpreters.destroy(interp) + self.assertTrue(_interpreters.is_running(interp)) + + +class CommonTests(TestBase): + def setUp(self): + super().setUp() + self.id = _interpreters.create() + + def test_signatures(self): + # See https://github.com/python/cpython/issues/126654 + msg = "expected 'shared' to be a dict" + with self.assertRaisesRegex(TypeError, msg): + _interpreters.exec(self.id, 'a', 1) + with self.assertRaisesRegex(TypeError, msg): + _interpreters.exec(self.id, 'a', shared=1) + with self.assertRaisesRegex(TypeError, msg): + _interpreters.run_string(self.id, 'a', shared=1) + with self.assertRaisesRegex(TypeError, msg): + _interpreters.run_func(self.id, lambda: None, shared=1) + + def test_invalid_shared_encoding(self): + # See https://github.com/python/cpython/issues/127196 + bad_shared = {"\uD82A": 0} + msg = 'surrogates not allowed' + with self.assertRaisesRegex(UnicodeEncodeError, msg): + _interpreters.exec(self.id, 'a', shared=bad_shared) + with self.assertRaisesRegex(UnicodeEncodeError, msg): + _interpreters.run_string(self.id, 'a', shared=bad_shared) + with self.assertRaisesRegex(UnicodeEncodeError, msg): + _interpreters.run_func(self.id, lambda: None, shared=bad_shared) class RunStringTests(TestBase): def setUp(self): super().setUp() - self.id = interpreters.create() + self.id = _interpreters.create() def test_success(self): script, file = _captured_script('print("it worked!", end="")') with file: - interpreters.run_string(self.id, script) + _interpreters.run_string(self.id, script) out = file.read() self.assertEqual(out, 'it worked!') @@ -555,7 +598,7 @@ def test_in_thread(self): script, file = _captured_script('print("it worked!", end="")') with file: def f(): - interpreters.run_string(self.id, script) + _interpreters.run_string(self.id, script) t = threading.Thread(target=f) t.start() @@ -565,7 +608,7 @@ def f(): self.assertEqual(out, 'it worked!') def test_create_thread(self): - subinterp = interpreters.create() + subinterp = _interpreters.create() script, file = _captured_script(""" import threading def f(): @@ -576,7 +619,7 @@ def f(): t.join() """) with file: - interpreters.run_string(subinterp, script) + _interpreters.run_string(subinterp, script) out = file.read() self.assertEqual(out, 'it worked!') @@ -584,7 +627,7 @@ def f(): def test_create_daemon_thread(self): with self.subTest('isolated'): expected = 'spam spam spam spam spam' - subinterp = interpreters.create(isolated=True) + subinterp = _interpreters.create('isolated') script, file = _captured_script(f""" import threading def f(): @@ -598,13 +641,13 @@ def f(): print('{expected}', end='') """) with file: - interpreters.run_string(subinterp, script) + _interpreters.run_string(subinterp, script) out = file.read() self.assertEqual(out, expected) with self.subTest('not isolated'): - subinterp = interpreters.create(isolated=False) + subinterp = _interpreters.create('legacy') script, file = _captured_script(""" import threading def f(): @@ -615,13 +658,13 @@ def f(): t.join() """) with file: - interpreters.run_string(subinterp, script) + _interpreters.run_string(subinterp, script) out = file.read() self.assertEqual(out, 'it worked!') def test_shareable_types(self): - interp = interpreters.create() + interp = _interpreters.create() objects = [ None, 'spam', @@ -630,15 +673,15 @@ def test_shareable_types(self): ] for obj in objects: with self.subTest(obj): - interpreters.set___main___attrs(interp, dict(obj=obj)) - interpreters.run_string( + _interpreters.set___main___attrs(interp, dict(obj=obj)) + _interpreters.run_string( interp, f'assert(obj == {obj!r})', ) def test_os_exec(self): expected = 'spam spam spam spam spam' - subinterp = interpreters.create() + subinterp = _interpreters.create() script, file = _captured_script(f""" import os, sys try: @@ -647,7 +690,7 @@ def test_os_exec(self): print('{expected}', end='') """) with file: - interpreters.run_string(subinterp, script) + _interpreters.run_string(subinterp, script) out = file.read() self.assertEqual(out, expected) @@ -668,7 +711,7 @@ def test_fork(self): with open('{file.name}', 'w', encoding='utf-8') as out: out.write('{expected}') """) - interpreters.run_string(self.id, script) + _interpreters.run_string(self.id, script) file.seek(0) content = file.read() @@ -676,31 +719,31 @@ def test_fork(self): def test_already_running(self): with _running(self.id): - with self.assertRaises(RuntimeError): - interpreters.run_string(self.id, 'print("spam")') + with self.assertRaises(_interpreters.InterpreterError): + _interpreters.run_string(self.id, 'print("spam")') def test_does_not_exist(self): id = 0 - while id in interpreters.list_all(): + while id in set(id for id, *_ in _interpreters.list_all()): id += 1 with self.assertRaises(InterpreterNotFoundError): - interpreters.run_string(id, 'print("spam")') + _interpreters.run_string(id, 'print("spam")') def test_error_id(self): with self.assertRaises(ValueError): - interpreters.run_string(-1, 'print("spam")') + _interpreters.run_string(-1, 'print("spam")') def test_bad_id(self): with self.assertRaises(TypeError): - interpreters.run_string('spam', 'print("spam")') + _interpreters.run_string('spam', 'print("spam")') def test_bad_script(self): with self.assertRaises(TypeError): - interpreters.run_string(self.id, 10) + _interpreters.run_string(self.id, 10) def test_bytes_for_script(self): with self.assertRaises(TypeError): - interpreters.run_string(self.id, b'print("spam")') + _interpreters.run_string(self.id, b'print("spam")') def test_with_shared(self): r, w = os.pipe() @@ -721,8 +764,8 @@ def test_with_shared(self): with open({w}, 'wb') as chan: pickle.dump(ns, chan) """) - interpreters.set___main___attrs(self.id, shared) - interpreters.run_string(self.id, script) + _interpreters.set___main___attrs(self.id, shared) + _interpreters.run_string(self.id, script) with open(r, 'rb') as chan: ns = pickle.load(chan) @@ -732,7 +775,7 @@ def test_with_shared(self): self.assertIsNone(ns['cheddar']) def test_shared_overwrites(self): - interpreters.run_string(self.id, dedent(""" + _interpreters.run_string(self.id, dedent(""" spam = 'eggs' ns1 = dict(vars()) del ns1['__builtins__'] @@ -743,8 +786,8 @@ def test_shared_overwrites(self): ns2 = dict(vars()) del ns2['__builtins__'] """) - interpreters.set___main___attrs(self.id, shared) - interpreters.run_string(self.id, script) + _interpreters.set___main___attrs(self.id, shared) + _interpreters.run_string(self.id, script) r, w = os.pipe() script = dedent(f""" @@ -754,7 +797,7 @@ def test_shared_overwrites(self): with open({w}, 'wb') as chan: pickle.dump(ns, chan) """) - interpreters.run_string(self.id, script) + _interpreters.run_string(self.id, script) with open(r, 'rb') as chan: ns = pickle.load(chan) @@ -775,8 +818,8 @@ def test_shared_overwrites_default_vars(self): with open({w}, 'wb') as chan: pickle.dump(ns, chan) """) - interpreters.set___main___attrs(self.id, shared) - interpreters.run_string(self.id, script) + _interpreters.set___main___attrs(self.id, shared) + _interpreters.run_string(self.id, script) with open(r, 'rb') as chan: ns = pickle.load(chan) @@ -784,7 +827,7 @@ def test_shared_overwrites_default_vars(self): def test_main_reused(self): r, w = os.pipe() - interpreters.run_string(self.id, dedent(f""" + _interpreters.run_string(self.id, dedent(f""" spam = True ns = dict(vars()) @@ -798,7 +841,7 @@ def test_main_reused(self): ns1 = pickle.load(chan) r, w = os.pipe() - interpreters.run_string(self.id, dedent(f""" + _interpreters.run_string(self.id, dedent(f""" eggs = False ns = dict(vars()) @@ -827,7 +870,7 @@ def test_execution_namespace_is_main(self): with open({w}, 'wb') as chan: pickle.dump(ns, chan) """) - interpreters.run_string(self.id, script) + _interpreters.run_string(self.id, script) with open(r, 'rb') as chan: ns = pickle.load(chan) @@ -835,7 +878,6 @@ def test_execution_namespace_is_main(self): ns.pop('__loader__') self.assertEqual(ns, { '__name__': '__main__', - '__annotations__': {}, '__doc__': None, '__package__': None, '__spec__': None, @@ -848,7 +890,7 @@ def test_still_running_at_exit(self): script = dedent(""" from textwrap import dedent import threading - import _xxsubinterpreters as _interpreters + import _interpreters id = _interpreters.create() def f(): _interpreters.run_string(id, dedent(''' @@ -872,13 +914,13 @@ class RunFailedTests(TestBase): def setUp(self): super().setUp() - self.id = interpreters.create() + self.id = _interpreters.create() def add_module(self, modname, text): import tempfile tempdir = tempfile.mkdtemp() self.addCleanup(lambda: os_helper.rmtree(tempdir)) - interpreters.run_string(self.id, dedent(f""" + _interpreters.run_string(self.id, dedent(f""" import sys sys.path.insert(0, {tempdir!r}) """)) @@ -900,11 +942,11 @@ class NeverError(Exception): pass raise NeverError # never raised """).format(dedent(text)) if fails: - err = interpreters.run_string(self.id, script) + err = _interpreters.run_string(self.id, script) self.assertIsNot(err, None) return err else: - err = interpreters.run_string(self.id, script) + err = _interpreters.run_string(self.id, script) self.assertIs(err, None) return None except: @@ -1029,7 +1071,7 @@ class RunFuncTests(TestBase): def setUp(self): super().setUp() - self.id = interpreters.create() + self.id = _interpreters.create() def test_success(self): r, w = os.pipe() @@ -1039,8 +1081,8 @@ def script(): with open(w, 'w', encoding="utf-8") as spipe: with contextlib.redirect_stdout(spipe): print('it worked!', end='') - interpreters.set___main___attrs(self.id, dict(w=w)) - interpreters.run_func(self.id, script) + _interpreters.set___main___attrs(self.id, dict(w=w)) + _interpreters.run_func(self.id, script) with open(r, encoding="utf-8") as outfile: out = outfile.read() @@ -1056,8 +1098,8 @@ def script(): with contextlib.redirect_stdout(spipe): print('it worked!', end='') def f(): - interpreters.set___main___attrs(self.id, dict(w=w)) - interpreters.run_func(self.id, script) + _interpreters.set___main___attrs(self.id, dict(w=w)) + _interpreters.run_func(self.id, script) t = threading.Thread(target=f) t.start() t.join() @@ -1077,8 +1119,8 @@ def script(): with contextlib.redirect_stdout(spipe): print('it worked!', end='') code = script.__code__ - interpreters.set___main___attrs(self.id, dict(w=w)) - interpreters.run_func(self.id, code) + _interpreters.set___main___attrs(self.id, dict(w=w)) + _interpreters.run_func(self.id, code) with open(r, encoding="utf-8") as outfile: out = outfile.read() @@ -1091,7 +1133,7 @@ def script(): assert spam with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) # XXX This hasn't been fixed yet. @unittest.expectedFailure @@ -1099,38 +1141,38 @@ def test_return_value(self): def script(): return 'spam' with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) def test_args(self): with self.subTest('args'): def script(a, b=0): assert a == b with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) with self.subTest('*args'): def script(*args): assert not args with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) with self.subTest('**kwargs'): def script(**kwargs): assert not kwargs with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) with self.subTest('kwonly'): def script(*, spam=True): assert spam with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) with self.subTest('posonly'): def script(spam, /): assert spam with self.assertRaises(ValueError): - interpreters.run_func(self.id, script) + _interpreters.run_func(self.id, script) if __name__ == '__main__': diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py index 0947464bb8c04e..cef84fd9580c37 100644 --- a/Lib/test/test__locale.py +++ b/Lib/test/test__locale.py @@ -1,4 +1,4 @@ -from _locale import (setlocale, LC_ALL, LC_CTYPE, LC_NUMERIC, localeconv, Error) +from _locale import (setlocale, LC_ALL, LC_CTYPE, LC_NUMERIC, LC_TIME, localeconv, Error) try: from _locale import (RADIXCHAR, THOUSEP, nl_langinfo) except ImportError: @@ -26,7 +26,10 @@ 'bs_BA', 'fr_LU', 'kl_GL', 'fa_IR', 'de_BE', 'sv_SE', 'it_CH', 'uk_UA', 'eu_ES', 'vi_VN', 'af_ZA', 'nb_NO', 'en_DK', 'tg_TJ', 'ps_AF', 'en_US', 'fr_FR.ISO8859-1', 'fr_FR.UTF-8', 'fr_FR.ISO8859-15@euro', - 'ru_RU.KOI8-R', 'ko_KR.eucKR'] + 'ru_RU.KOI8-R', 'ko_KR.eucKR', + 'ja_JP.UTF-8', 'lzh_TW.UTF-8', 'my_MM.UTF-8', 'or_IN.UTF-8', 'shn_MM.UTF-8', + 'ar_AE.UTF-8', 'bn_IN.UTF-8', 'mr_IN.UTF-8', 'th_TH.TIS620', +] def setUpModule(): global candidate_locales @@ -74,10 +77,36 @@ def accept(loc): 'ps_AF': ('\u066b', '\u066c'), } +known_alt_digits = { + 'C': (0, {}), + 'en_US': (0, {}), + 'fa_IR': (100, {0: '\u06f0\u06f0', 10: '\u06f1\u06f0', 99: '\u06f9\u06f9'}), + 'ja_JP': (100, {1: '\u4e00', 10: '\u5341', 99: '\u4e5d\u5341\u4e5d'}), + 'lzh_TW': (32, {0: '\u3007', 10: '\u5341', 31: '\u5345\u4e00'}), + 'my_MM': (100, {0: '\u1040\u1040', 10: '\u1041\u1040', 99: '\u1049\u1049'}), + 'or_IN': (100, {0: '\u0b66', 10: '\u0b67\u0b66', 99: '\u0b6f\u0b6f'}), + 'shn_MM': (100, {0: '\u1090\u1090', 10: '\u1091\u1090', 99: '\u1099\u1099'}), + 'ar_AE': (100, {0: '\u0660', 10: '\u0661\u0660', 99: '\u0669\u0669'}), + 'bn_IN': (100, {0: '\u09e6', 10: '\u09e7\u09e6', 99: '\u09ef\u09ef'}), +} + +known_era = { + 'C': (0, ''), + 'en_US': (0, ''), + 'ja_JP': (11, '+:1:2019/05/01:2019/12/31:令和:%EC元年'), + 'zh_TW': (3, '+:1:1912/01/01:1912/12/31:民國:%EC元年'), + 'th_TW': (1, '+:1:-543/01/01:+*:พ.ศ.:%EC %Ey'), +} + if sys.platform == 'win32': # ps_AF doesn't work on Windows: see bpo-38324 (msg361830) del known_numerics['ps_AF'] +if sys.platform == 'sunos5': + # On Solaris, Japanese ERAs start with the year 1927, + # and thus there's less of them. + known_era['ja_JP'] = (5, '+:1:2019/05/01:2019/12/31:令和:%EC元年') + class _LocaleTests(unittest.TestCase): def setUp(self): @@ -115,16 +144,17 @@ def numeric_tester(self, calc_type, calc_value, data_type, used_locale): def test_lc_numeric_nl_langinfo(self): # Test nl_langinfo against known values tested = False + oldloc = setlocale(LC_CTYPE) for loc in candidate_locales: try: setlocale(LC_NUMERIC, loc) - setlocale(LC_CTYPE, loc) except Error: continue for li, lc in ((RADIXCHAR, "decimal_point"), (THOUSEP, "thousands_sep")): if self.numeric_tester('nl_langinfo', nl_langinfo(li), lc, loc): tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) if not tested: self.skipTest('no suitable locales') @@ -135,10 +165,10 @@ def test_lc_numeric_nl_langinfo(self): def test_lc_numeric_localeconv(self): # Test localeconv against known values tested = False + oldloc = setlocale(LC_CTYPE) for loc in candidate_locales: try: setlocale(LC_NUMERIC, loc) - setlocale(LC_CTYPE, loc) except Error: continue formatting = localeconv() @@ -146,6 +176,7 @@ def test_lc_numeric_localeconv(self): "thousands_sep"): if self.numeric_tester('localeconv', formatting[lc], lc, loc): tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) if not tested: self.skipTest('no suitable locales') @@ -153,10 +184,10 @@ def test_lc_numeric_localeconv(self): def test_lc_numeric_basic(self): # Test nl_langinfo against localeconv tested = False + oldloc = setlocale(LC_CTYPE) for loc in candidate_locales: try: setlocale(LC_NUMERIC, loc) - setlocale(LC_CTYPE, loc) except Error: continue for li, lc in ((RADIXCHAR, "decimal_point"), @@ -173,6 +204,79 @@ def test_lc_numeric_basic(self): nl_radixchar, li_radixchar, loc, set_locale)) tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) + if not tested: + self.skipTest('no suitable locales') + + @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available") + @unittest.skipUnless(hasattr(locale, 'ALT_DIGITS'), "requires locale.ALT_DIGITS") + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "musl libc issue on Emscripten, bpo-46390" + ) + def test_alt_digits_nl_langinfo(self): + # Test nl_langinfo(ALT_DIGITS) + tested = False + for loc in candidate_locales: + with self.subTest(locale=loc): + try: + setlocale(LC_TIME, loc) + except Error: + self.skipTest(f'no locale {loc!r}') + continue + + with self.subTest(locale=loc): + alt_digits = nl_langinfo(locale.ALT_DIGITS) + self.assertIsInstance(alt_digits, str) + alt_digits = alt_digits.split(';') if alt_digits else [] + if alt_digits: + self.assertGreaterEqual(len(alt_digits), 10, alt_digits) + loc1 = loc.split('.', 1)[0] + if loc1 in known_alt_digits: + count, samples = known_alt_digits[loc1] + if count and not alt_digits: + self.skipTest(f'ALT_DIGITS is not set for locale {loc!r} on this platform') + self.assertEqual(len(alt_digits), count, alt_digits) + for i in samples: + self.assertEqual(alt_digits[i], samples[i]) + tested = True + if not tested: + self.skipTest('no suitable locales') + + @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available") + @unittest.skipUnless(hasattr(locale, 'ERA'), "requires locale.ERA") + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "musl libc issue on Emscripten, bpo-46390" + ) + def test_era_nl_langinfo(self): + # Test nl_langinfo(ERA) + tested = False + for loc in candidate_locales: + with self.subTest(locale=loc): + try: + setlocale(LC_TIME, loc) + except Error: + self.skipTest(f'no locale {loc!r}') + continue + + with self.subTest(locale=loc): + era = nl_langinfo(locale.ERA) + self.assertIsInstance(era, str) + if era: + self.assertEqual(era.count(':'), (era.count(';') + 1) * 5, era) + + loc1 = loc.split('.', 1)[0] + if loc1 in known_era: + count, sample = known_era[loc1] + if count: + if not era: + self.skipTest(f'ERA is not set for locale {loc!r} on this platform') + self.assertGreaterEqual(era.count(';') + 1, count) + self.assertIn(sample, era) + else: + self.assertEqual(era, '') + tested = True if not tested: self.skipTest('no suitable locales') @@ -180,10 +284,10 @@ def test_float_parsing(self): # Bug #1391872: Test whether float parsing is okay on European # locales. tested = False + oldloc = setlocale(LC_CTYPE) for loc in candidate_locales: try: setlocale(LC_NUMERIC, loc) - setlocale(LC_CTYPE, loc) except Error: continue @@ -199,6 +303,7 @@ def test_float_parsing(self): self.assertRaises(ValueError, float, localeconv()['decimal_point'].join(['1', '23'])) tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) if not tested: self.skipTest('no suitable locales') diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index 10f04b64dda40e..4b11e83ae59a78 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -17,7 +17,7 @@ def check_bool_function_result(self, func, ops, expected): self.assertEqual(func(op), expected) def test_invalid_opcodes(self): - invalid = [-100, -1, 255, 512, 513, 1000] + invalid = [-100, -1, 512, 513, 1000] self.check_bool_function_result(_opcode.is_valid, invalid, False) self.check_bool_function_result(_opcode.has_arg, invalid, False) self.check_bool_function_result(_opcode.has_const, invalid, False) @@ -38,6 +38,13 @@ def test_is_valid(self): opcodes = [dis.opmap[opname] for opname in names] self.check_bool_function_result(_opcode.is_valid, opcodes, True) + def test_opmaps(self): + def check_roundtrip(name, map): + return self.assertEqual(opcode.opname[map[name]], name) + + check_roundtrip('BINARY_OP', opcode.opmap) + check_roundtrip('BINARY_OP_ADD_INT', opcode._specialized_opmap) + def test_oplists(self): def check_function(self, func, expected): for op in [-10, 520]: @@ -58,8 +65,7 @@ def check_function(self, func, expected): class StackEffectTests(unittest.TestCase): def test_stack_effect(self): self.assertEqual(stack_effect(dis.opmap['POP_TOP']), -1) - self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 0), -1) - self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 1), -1) + self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 2), -1) self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 3), -2) self.assertRaises(ValueError, stack_effect, 30000) # All defined opcodes diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index 5ce57cc209ea85..e90a8dc617c094 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -20,7 +20,7 @@ def test_abstractproperty_basics(self): def foo(self): pass self.assertTrue(foo.__isabstractmethod__) def bar(self): pass - self.assertFalse(hasattr(bar, "__isabstractmethod__")) + self.assertNotHasAttr(bar, "__isabstractmethod__") class C(metaclass=abc_ABCMeta): @abc.abstractproperty @@ -89,7 +89,7 @@ def test_abstractmethod_basics(self): def foo(self): pass self.assertTrue(foo.__isabstractmethod__) def bar(self): pass - self.assertFalse(hasattr(bar, "__isabstractmethod__")) + self.assertNotHasAttr(bar, "__isabstractmethod__") def test_abstractproperty_basics(self): @property @@ -276,21 +276,21 @@ class A(metaclass=abc_ABCMeta): class B(object): pass b = B() - self.assertFalse(issubclass(B, A)) - self.assertFalse(issubclass(B, (A,))) + self.assertNotIsSubclass(B, A) + self.assertNotIsSubclass(B, (A,)) self.assertNotIsInstance(b, A) self.assertNotIsInstance(b, (A,)) B1 = A.register(B) - self.assertTrue(issubclass(B, A)) - self.assertTrue(issubclass(B, (A,))) + self.assertIsSubclass(B, A) + self.assertIsSubclass(B, (A,)) self.assertIsInstance(b, A) self.assertIsInstance(b, (A,)) self.assertIs(B1, B) class C(B): pass c = C() - self.assertTrue(issubclass(C, A)) - self.assertTrue(issubclass(C, (A,))) + self.assertIsSubclass(C, A) + self.assertIsSubclass(C, (A,)) self.assertIsInstance(c, A) self.assertIsInstance(c, (A,)) @@ -301,16 +301,16 @@ class A(metaclass=abc_ABCMeta): class B(object): pass b = B() - self.assertTrue(issubclass(B, A)) - self.assertTrue(issubclass(B, (A,))) + self.assertIsSubclass(B, A) + self.assertIsSubclass(B, (A,)) self.assertIsInstance(b, A) self.assertIsInstance(b, (A,)) @A.register class C(B): pass c = C() - self.assertTrue(issubclass(C, A)) - self.assertTrue(issubclass(C, (A,))) + self.assertIsSubclass(C, A) + self.assertIsSubclass(C, (A,)) self.assertIsInstance(c, A) self.assertIsInstance(c, (A,)) self.assertIs(C, A.register(C)) @@ -321,14 +321,14 @@ class A(metaclass=abc_ABCMeta): class B: pass b = B() - self.assertFalse(isinstance(b, A)) - self.assertFalse(isinstance(b, (A,))) + self.assertNotIsInstance(b, A) + self.assertNotIsInstance(b, (A,)) token_old = abc_get_cache_token() A.register(B) token_new = abc_get_cache_token() self.assertGreater(token_new, token_old) - self.assertTrue(isinstance(b, A)) - self.assertTrue(isinstance(b, (A,))) + self.assertIsInstance(b, A) + self.assertIsInstance(b, (A,)) def test_registration_builtins(self): class A(metaclass=abc_ABCMeta): @@ -336,18 +336,18 @@ class A(metaclass=abc_ABCMeta): A.register(int) self.assertIsInstance(42, A) self.assertIsInstance(42, (A,)) - self.assertTrue(issubclass(int, A)) - self.assertTrue(issubclass(int, (A,))) + self.assertIsSubclass(int, A) + self.assertIsSubclass(int, (A,)) class B(A): pass B.register(str) class C(str): pass self.assertIsInstance("", A) self.assertIsInstance("", (A,)) - self.assertTrue(issubclass(str, A)) - self.assertTrue(issubclass(str, (A,))) - self.assertTrue(issubclass(C, A)) - self.assertTrue(issubclass(C, (A,))) + self.assertIsSubclass(str, A) + self.assertIsSubclass(str, (A,)) + self.assertIsSubclass(C, A) + self.assertIsSubclass(C, (A,)) def test_registration_edge_cases(self): class A(metaclass=abc_ABCMeta): @@ -375,39 +375,39 @@ class A(metaclass=abc_ABCMeta): def test_registration_transitiveness(self): class A(metaclass=abc_ABCMeta): pass - self.assertTrue(issubclass(A, A)) - self.assertTrue(issubclass(A, (A,))) + self.assertIsSubclass(A, A) + self.assertIsSubclass(A, (A,)) class B(metaclass=abc_ABCMeta): pass - self.assertFalse(issubclass(A, B)) - self.assertFalse(issubclass(A, (B,))) - self.assertFalse(issubclass(B, A)) - self.assertFalse(issubclass(B, (A,))) + self.assertNotIsSubclass(A, B) + self.assertNotIsSubclass(A, (B,)) + self.assertNotIsSubclass(B, A) + self.assertNotIsSubclass(B, (A,)) class C(metaclass=abc_ABCMeta): pass A.register(B) class B1(B): pass - self.assertTrue(issubclass(B1, A)) - self.assertTrue(issubclass(B1, (A,))) + self.assertIsSubclass(B1, A) + self.assertIsSubclass(B1, (A,)) class C1(C): pass B1.register(C1) - self.assertFalse(issubclass(C, B)) - self.assertFalse(issubclass(C, (B,))) - self.assertFalse(issubclass(C, B1)) - self.assertFalse(issubclass(C, (B1,))) - self.assertTrue(issubclass(C1, A)) - self.assertTrue(issubclass(C1, (A,))) - self.assertTrue(issubclass(C1, B)) - self.assertTrue(issubclass(C1, (B,))) - self.assertTrue(issubclass(C1, B1)) - self.assertTrue(issubclass(C1, (B1,))) + self.assertNotIsSubclass(C, B) + self.assertNotIsSubclass(C, (B,)) + self.assertNotIsSubclass(C, B1) + self.assertNotIsSubclass(C, (B1,)) + self.assertIsSubclass(C1, A) + self.assertIsSubclass(C1, (A,)) + self.assertIsSubclass(C1, B) + self.assertIsSubclass(C1, (B,)) + self.assertIsSubclass(C1, B1) + self.assertIsSubclass(C1, (B1,)) C1.register(int) class MyInt(int): pass - self.assertTrue(issubclass(MyInt, A)) - self.assertTrue(issubclass(MyInt, (A,))) + self.assertIsSubclass(MyInt, A) + self.assertIsSubclass(MyInt, (A,)) self.assertIsInstance(42, A) self.assertIsInstance(42, (A,)) @@ -467,16 +467,16 @@ def __subclasshook__(cls, C): if cls is A: return 'foo' in C.__dict__ return NotImplemented - self.assertFalse(issubclass(A, A)) - self.assertFalse(issubclass(A, (A,))) + self.assertNotIsSubclass(A, A) + self.assertNotIsSubclass(A, (A,)) class B: foo = 42 - self.assertTrue(issubclass(B, A)) - self.assertTrue(issubclass(B, (A,))) + self.assertIsSubclass(B, A) + self.assertIsSubclass(B, (A,)) class C: spam = 42 - self.assertFalse(issubclass(C, A)) - self.assertFalse(issubclass(C, (A,))) + self.assertNotIsSubclass(C, A) + self.assertNotIsSubclass(C, (A,)) def test_all_new_methods_are_called(self): class A(metaclass=abc_ABCMeta): @@ -493,7 +493,7 @@ class C(A, B): self.assertEqual(B.counter, 1) def test_ABC_has___slots__(self): - self.assertTrue(hasattr(abc.ABC, '__slots__')) + self.assertHasAttr(abc.ABC, '__slots__') def test_tricky_new_works(self): def with_metaclass(meta, *bases): @@ -515,7 +515,7 @@ def foo(self): del A.foo self.assertEqual(A.__abstractmethods__, {'foo'}) - self.assertFalse(hasattr(A, 'foo')) + self.assertNotHasAttr(A, 'foo') abc.update_abstractmethods(A) @@ -588,7 +588,7 @@ def updated_foo(self): A.foo = updated_foo abc.update_abstractmethods(A) A() - self.assertFalse(hasattr(A, '__abstractmethods__')) + self.assertNotHasAttr(A, '__abstractmethods__') def test_update_del_implementation(self): class A(metaclass=abc_ABCMeta): diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py new file mode 100644 index 00000000000000..a9abd61b9a8d92 --- /dev/null +++ b/Lib/test/test_android.py @@ -0,0 +1,444 @@ +import io +import platform +import queue +import re +import subprocess +import sys +import unittest +from _android_support import TextLogStream +from array import array +from contextlib import ExitStack, contextmanager +from threading import Thread +from test.support import LOOPBACK_TIMEOUT +from time import time +from unittest.mock import patch + + +if sys.platform != "android": + raise unittest.SkipTest("Android-specific") + +api_level = platform.android_ver().api_level + +# (name, level, fileno) +STREAM_INFO = [("stdout", "I", 1), ("stderr", "W", 2)] + + +# Test redirection of stdout and stderr to the Android log. +class TestAndroidOutput(unittest.TestCase): + maxDiff = None + + def setUp(self): + self.logcat_process = subprocess.Popen( + ["logcat", "-v", "tag"], stdout=subprocess.PIPE, + errors="backslashreplace" + ) + self.logcat_queue = queue.Queue() + + def logcat_thread(): + for line in self.logcat_process.stdout: + self.logcat_queue.put(line.rstrip("\n")) + self.logcat_process.stdout.close() + self.logcat_thread = Thread(target=logcat_thread) + self.logcat_thread.start() + + from ctypes import CDLL, c_char_p, c_int + android_log_write = getattr(CDLL("liblog.so"), "__android_log_write") + android_log_write.argtypes = (c_int, c_char_p, c_char_p) + ANDROID_LOG_INFO = 4 + + # Separate tests using a marker line with a different tag. + tag, message = "python.test", f"{self.id()} {time()}" + android_log_write( + ANDROID_LOG_INFO, tag.encode("UTF-8"), message.encode("UTF-8")) + self.assert_log("I", tag, message, skip=True, timeout=5) + + def assert_logs(self, level, tag, expected, **kwargs): + for line in expected: + self.assert_log(level, tag, line, **kwargs) + + def assert_log(self, level, tag, expected, *, skip=False, timeout=0.5): + deadline = time() + timeout + while True: + try: + line = self.logcat_queue.get(timeout=(deadline - time())) + except queue.Empty: + self.fail(f"line not found: {expected!r}") + if match := re.fullmatch(fr"(.)/{tag}: (.*)", line): + try: + self.assertEqual(level, match[1]) + self.assertEqual(expected, match[2]) + break + except AssertionError: + if not skip: + raise + + def tearDown(self): + self.logcat_process.terminate() + self.logcat_process.wait(LOOPBACK_TIMEOUT) + self.logcat_thread.join(LOOPBACK_TIMEOUT) + + @contextmanager + def unbuffered(self, stream): + stream.reconfigure(write_through=True) + try: + yield + finally: + stream.reconfigure(write_through=False) + + # In --verbose3 mode, sys.stdout and sys.stderr are captured, so we can't + # test them directly. Detect this mode and use some temporary streams with + # the same properties. + def stream_context(self, stream_name, level): + # https://developer.android.com/ndk/reference/group/logging + prio = {"I": 4, "W": 5}[level] + + stack = ExitStack() + stack.enter_context(self.subTest(stream_name)) + stream = getattr(sys, stream_name) + native_stream = getattr(sys, f"__{stream_name}__") + if isinstance(stream, io.StringIO): + stack.enter_context( + patch( + f"sys.{stream_name}", + TextLogStream( + prio, f"python.{stream_name}", native_stream.fileno(), + errors="backslashreplace" + ), + ) + ) + return stack + + def test_str(self): + for stream_name, level, fileno in STREAM_INFO: + with self.stream_context(stream_name, level): + stream = getattr(sys, stream_name) + tag = f"python.{stream_name}" + self.assertEqual(f"", repr(stream)) + + self.assertIs(stream.writable(), True) + self.assertIs(stream.readable(), False) + self.assertEqual(stream.fileno(), fileno) + self.assertEqual("UTF-8", stream.encoding) + self.assertEqual("backslashreplace", stream.errors) + self.assertIs(stream.line_buffering, True) + self.assertIs(stream.write_through, False) + + def write(s, lines=None, *, write_len=None): + if write_len is None: + write_len = len(s) + self.assertEqual(write_len, stream.write(s)) + if lines is None: + lines = [s] + self.assert_logs(level, tag, lines) + + # Single-line messages, + with self.unbuffered(stream): + write("", []) + + write("a") + write("Hello") + write("Hello world") + write(" ") + write(" ") + + # Non-ASCII text + write("ol\u00e9") # Spanish + write("\u4e2d\u6587") # Chinese + + # Non-BMP emoji + write("\U0001f600") + + # Non-encodable surrogates + write("\ud800\udc00", [r"\ud800\udc00"]) + + # Code used by surrogateescape (which isn't enabled here) + write("\udc80", [r"\udc80"]) + + # Null characters are logged using "modified UTF-8". + write("\u0000", [r"\xc0\x80"]) + write("a\u0000", [r"a\xc0\x80"]) + write("\u0000b", [r"\xc0\x80b"]) + write("a\u0000b", [r"a\xc0\x80b"]) + + # Multi-line messages. Avoid identical consecutive lines, as + # they may activate "chatty" filtering and break the tests. + write("\nx", [""]) + write("\na\n", ["x", "a"]) + write("\n", [""]) + write("b\n", ["b"]) + write("c\n\n", ["c", ""]) + write("d\ne", ["d"]) + write("xx", []) + write("f\n\ng", ["exxf", ""]) + write("\n", ["g"]) + + # Since this is a line-based logging system, line buffering + # cannot be turned off, i.e. a newline always causes a flush. + stream.reconfigure(line_buffering=False) + self.assertIs(stream.line_buffering, True) + + # However, buffering can be turned off completely if you want a + # flush after every write. + with self.unbuffered(stream): + write("\nx", ["", "x"]) + write("\na\n", ["", "a"]) + write("\n", [""]) + write("b\n", ["b"]) + write("c\n\n", ["c", ""]) + write("d\ne", ["d", "e"]) + write("xx", ["xx"]) + write("f\n\ng", ["f", "", "g"]) + write("\n", [""]) + + # "\r\n" should be translated into "\n". + write("hello\r\n", ["hello"]) + write("hello\r\nworld\r\n", ["hello", "world"]) + write("\r\n", [""]) + + # Non-standard line separators should be preserved. + write("before form feed\x0cafter form feed\n", + ["before form feed\x0cafter form feed"]) + write("before line separator\u2028after line separator\n", + ["before line separator\u2028after line separator"]) + + # String subclasses are accepted, but they should be converted + # to a standard str without calling any of their methods. + class CustomStr(str): + def splitlines(self, *args, **kwargs): + raise AssertionError() + + def __len__(self): + raise AssertionError() + + def __str__(self): + raise AssertionError() + + write(CustomStr("custom\n"), ["custom"], write_len=7) + + # Non-string classes are not accepted. + for obj in [b"", b"hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be str, not " + fr"{type(obj).__name__}" + ): + stream.write(obj) + + # Manual flushing is supported. + write("hello", []) + stream.flush() + self.assert_log(level, tag, "hello") + write("hello", []) + write("world", []) + stream.flush() + self.assert_log(level, tag, "helloworld") + + # Long lines are split into blocks of 1000 characters + # (MAX_CHARS_PER_WRITE in _android_support.py), but + # TextIOWrapper should then join them back together as much as + # possible without exceeding 4000 UTF-8 bytes + # (MAX_BYTES_PER_WRITE). + # + # ASCII (1 byte per character) + write(("foobar" * 700) + "\n", # 4200 bytes in + [("foobar" * 666) + "foob", # 4000 bytes out + "ar" + ("foobar" * 33)]) # 200 bytes out + + # "Full-width" digits 0-9 (3 bytes per character) + s = "\uff10\uff11\uff12\uff13\uff14\uff15\uff16\uff17\uff18\uff19" + write((s * 150) + "\n", # 4500 bytes in + [s * 100, # 3000 bytes out + s * 50]) # 1500 bytes out + + s = "0123456789" + write(s * 200, []) # 2000 bytes in + write(s * 150, []) # 1500 bytes in + write(s * 51, [s * 350]) # 510 bytes in, 3500 bytes out + write("\n", [s * 51]) # 0 bytes in, 510 bytes out + + def test_bytes(self): + for stream_name, level, fileno in STREAM_INFO: + with self.stream_context(stream_name, level): + stream = getattr(sys, stream_name).buffer + tag = f"python.{stream_name}" + self.assertEqual(f"", repr(stream)) + self.assertIs(stream.writable(), True) + self.assertIs(stream.readable(), False) + self.assertEqual(stream.fileno(), fileno) + + def write(b, lines=None, *, write_len=None): + if write_len is None: + write_len = len(b) + self.assertEqual(write_len, stream.write(b)) + if lines is None: + lines = [b.decode()] + self.assert_logs(level, tag, lines) + + # Single-line messages, + write(b"", []) + + write(b"a") + write(b"Hello") + write(b"Hello world") + write(b" ") + write(b" ") + + # Non-ASCII text + write(b"ol\xc3\xa9") # Spanish + write(b"\xe4\xb8\xad\xe6\x96\x87") # Chinese + + # Non-BMP emoji + write(b"\xf0\x9f\x98\x80") + + # Null bytes are logged using "modified UTF-8". + write(b"\x00", [r"\xc0\x80"]) + write(b"a\x00", [r"a\xc0\x80"]) + write(b"\x00b", [r"\xc0\x80b"]) + write(b"a\x00b", [r"a\xc0\x80b"]) + + # Invalid UTF-8 + write(b"\xff", [r"\xff"]) + write(b"a\xff", [r"a\xff"]) + write(b"\xffb", [r"\xffb"]) + write(b"a\xffb", [r"a\xffb"]) + + # Log entries containing newlines are shown differently by + # `logcat -v tag`, `logcat -v long`, and Android Studio. We + # currently use `logcat -v tag`, which shows each line as if it + # was a separate log entry, but strips a single trailing + # newline. + # + # On newer versions of Android, all three of the above tools (or + # maybe Logcat itself) will also strip any number of leading + # newlines. + write(b"\nx", ["", "x"] if api_level < 30 else ["x"]) + write(b"\na\n", ["", "a"] if api_level < 30 else ["a"]) + write(b"\n", [""]) + write(b"b\n", ["b"]) + write(b"c\n\n", ["c", ""]) + write(b"d\ne", ["d", "e"]) + write(b"xx", ["xx"]) + write(b"f\n\ng", ["f", "", "g"]) + write(b"\n", [""]) + + # "\r\n" should be translated into "\n". + write(b"hello\r\n", ["hello"]) + write(b"hello\r\nworld\r\n", ["hello", "world"]) + write(b"\r\n", [""]) + + # Other bytes-like objects are accepted. + write(bytearray(b"bytearray")) + + mv = memoryview(b"memoryview") + write(mv, ["memoryview"]) # Continuous + write(mv[::2], ["mmrve"]) # Discontinuous + + write( + # Android only supports little-endian architectures, so the + # bytes representation is as follows: + array("H", [ + 0, # 00 00 + 1, # 01 00 + 65534, # FE FF + 65535, # FF FF + ]), + + # After encoding null bytes with modified UTF-8, the only + # valid UTF-8 sequence is \x01. All other bytes are handled + # by backslashreplace. + ["\\xc0\\x80\\xc0\\x80" + "\x01\\xc0\\x80" + "\\xfe\\xff" + "\\xff\\xff"], + write_len=8, + ) + + # Non-bytes-like classes are not accepted. + for obj in ["", "hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be bytes-like, not " + fr"{type(obj).__name__}" + ): + stream.write(obj) + + +class TestAndroidRateLimit(unittest.TestCase): + def test_rate_limit(self): + # https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39 + PER_MESSAGE_OVERHEAD = 28 + + # https://developer.android.com/ndk/reference/group/logging + ANDROID_LOG_DEBUG = 3 + + # To avoid flooding the test script output, use a different tag rather + # than stdout or stderr. + tag = "python.rate_limit" + stream = TextLogStream(ANDROID_LOG_DEBUG, tag) + + # Make a test message which consumes 1 KB of the logcat buffer. + message = "Line {:03d} " + message += "." * ( + 1024 - PER_MESSAGE_OVERHEAD - len(tag) - len(message.format(0)) + ) + "\n" + + # To avoid depending on the performance of the test device, we mock the + # passage of time. + mock_now = time() + + def mock_time(): + # Avoid division by zero by simulating a small delay. + mock_sleep(0.0001) + return mock_now + + def mock_sleep(duration): + nonlocal mock_now + mock_now += duration + + # See _android_support.py. The default values of these parameters work + # well across a wide range of devices, but we'll use smaller values to + # ensure a quick and reliable test that doesn't flood the log too much. + MAX_KB_PER_SECOND = 100 + BUCKET_KB = 10 + with ( + patch("_android_support.MAX_BYTES_PER_SECOND", MAX_KB_PER_SECOND * 1024), + patch("_android_support.BUCKET_SIZE", BUCKET_KB * 1024), + patch("_android_support.sleep", mock_sleep), + patch("_android_support.time", mock_time), + ): + # Make sure the token bucket is full. + stream.write("Initial message to reset _prev_write_time") + mock_sleep(BUCKET_KB / MAX_KB_PER_SECOND) + line_num = 0 + + # Write BUCKET_KB messages, and return the rate at which they were + # accepted in KB per second. + def write_bucketful(): + nonlocal line_num + start = mock_time() + max_line_num = line_num + BUCKET_KB + while line_num < max_line_num: + stream.write(message.format(line_num)) + line_num += 1 + return BUCKET_KB / (mock_time() - start) + + # The first bucketful should be written with minimal delay. The + # factor of 2 here is not arbitrary: it verifies that the system can + # write fast enough to empty the bucket within two bucketfuls, which + # the next part of the test depends on. + self.assertGreater(write_bucketful(), MAX_KB_PER_SECOND * 2) + + # Write another bucketful to empty the token bucket completely. + write_bucketful() + + # The next bucketful should be written at the rate limit. + self.assertAlmostEqual( + write_bucketful(), MAX_KB_PER_SECOND, + delta=MAX_KB_PER_SECOND * 0.1 + ) + + # Once the token bucket refills, we should go back to full speed. + mock_sleep(BUCKET_KB / MAX_KB_PER_SECOND) + self.assertGreater(write_bucketful(), MAX_KB_PER_SECOND * 2) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py new file mode 100644 index 00000000000000..20f74b4ed0aadb --- /dev/null +++ b/Lib/test/test_annotationlib.py @@ -0,0 +1,1155 @@ +"""Tests for the annotations module.""" + +import annotationlib +import builtins +import collections +import functools +import itertools +import pickle +import unittest +from annotationlib import ( + Format, + ForwardRef, + get_annotations, + get_annotate_function, + annotations_to_string, + value_to_string, +) +from typing import Unpack + +from test import support +from test.test_inspect import inspect_stock_annotations +from test.test_inspect import inspect_stringized_annotations +from test.test_inspect import inspect_stringized_annotations_2 +from test.test_inspect import inspect_stringized_annotations_pep695 + + +def times_three(fn): + @functools.wraps(fn) + def wrapper(a, b): + return fn(a * 3, b * 3) + + return wrapper + + +class MyClass: + def __repr__(self): + return "my repr" + + +class TestFormat(unittest.TestCase): + def test_enum(self): + self.assertEqual(Format.VALUE.value, 1) + self.assertEqual(Format.VALUE, 1) + + self.assertEqual(Format.VALUE_WITH_FAKE_GLOBALS.value, 2) + self.assertEqual(Format.VALUE_WITH_FAKE_GLOBALS, 2) + + self.assertEqual(Format.FORWARDREF.value, 3) + self.assertEqual(Format.FORWARDREF, 3) + + self.assertEqual(Format.STRING.value, 4) + self.assertEqual(Format.STRING, 4) + + +class TestForwardRefFormat(unittest.TestCase): + def test_closure(self): + def inner(arg: x): + pass + + anno = annotationlib.get_annotations(inner, format=Format.FORWARDREF) + fwdref = anno["arg"] + self.assertIsInstance(fwdref, annotationlib.ForwardRef) + self.assertEqual(fwdref.__forward_arg__, "x") + with self.assertRaises(NameError): + fwdref.evaluate() + + x = 1 + self.assertEqual(fwdref.evaluate(), x) + + anno = annotationlib.get_annotations(inner, format=Format.FORWARDREF) + self.assertEqual(anno["arg"], x) + + def test_function(self): + def f(x: int, y: doesntexist): + pass + + anno = annotationlib.get_annotations(f, format=Format.FORWARDREF) + self.assertIs(anno["x"], int) + fwdref = anno["y"] + self.assertIsInstance(fwdref, annotationlib.ForwardRef) + self.assertEqual(fwdref.__forward_arg__, "doesntexist") + with self.assertRaises(NameError): + fwdref.evaluate() + self.assertEqual(fwdref.evaluate(globals={"doesntexist": 1}), 1) + + def test_nonexistent_attribute(self): + def f( + x: some.module, + y: some[module], + z: some(module), + alpha: some | obj, + beta: +some, + gamma: some < obj, + ): + pass + + anno = annotationlib.get_annotations(f, format=Format.FORWARDREF) + x_anno = anno["x"] + self.assertIsInstance(x_anno, ForwardRef) + self.assertEqual(x_anno, ForwardRef("some.module")) + + y_anno = anno["y"] + self.assertIsInstance(y_anno, ForwardRef) + self.assertEqual(y_anno, ForwardRef("some[module]")) + + z_anno = anno["z"] + self.assertIsInstance(z_anno, ForwardRef) + self.assertEqual(z_anno, ForwardRef("some(module)")) + + alpha_anno = anno["alpha"] + self.assertIsInstance(alpha_anno, ForwardRef) + self.assertEqual(alpha_anno, ForwardRef("some | obj")) + + beta_anno = anno["beta"] + self.assertIsInstance(beta_anno, ForwardRef) + self.assertEqual(beta_anno, ForwardRef("+some")) + + gamma_anno = anno["gamma"] + self.assertIsInstance(gamma_anno, ForwardRef) + self.assertEqual(gamma_anno, ForwardRef("some < obj")) + + +class TestSourceFormat(unittest.TestCase): + def test_closure(self): + x = 0 + + def inner(arg: x): + pass + + anno = annotationlib.get_annotations(inner, format=Format.STRING) + self.assertEqual(anno, {"arg": "x"}) + + def test_closure_undefined(self): + if False: + x = 0 + + def inner(arg: x): + pass + + anno = annotationlib.get_annotations(inner, format=Format.STRING) + self.assertEqual(anno, {"arg": "x"}) + + def test_function(self): + def f(x: int, y: doesntexist): + pass + + anno = annotationlib.get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "int", "y": "doesntexist"}) + + def test_expressions(self): + def f( + add: a + b, + sub: a - b, + mul: a * b, + matmul: a @ b, + truediv: a / b, + mod: a % b, + lshift: a << b, + rshift: a >> b, + or_: a | b, + xor: a ^ b, + and_: a & b, + floordiv: a // b, + pow_: a**b, + lt: a < b, + le: a <= b, + eq: a == b, + ne: a != b, + gt: a > b, + ge: a >= b, + invert: ~a, + neg: -a, + pos: +a, + getitem: a[b], + getattr: a.b, + call: a(b, *c, d=e), # **kwargs are not supported + *args: *a, + ): + pass + + anno = annotationlib.get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "add": "a + b", + "sub": "a - b", + "mul": "a * b", + "matmul": "a @ b", + "truediv": "a / b", + "mod": "a % b", + "lshift": "a << b", + "rshift": "a >> b", + "or_": "a | b", + "xor": "a ^ b", + "and_": "a & b", + "floordiv": "a // b", + "pow_": "a ** b", + "lt": "a < b", + "le": "a <= b", + "eq": "a == b", + "ne": "a != b", + "gt": "a > b", + "ge": "a >= b", + "invert": "~a", + "neg": "-a", + "pos": "+a", + "getitem": "a[b]", + "getattr": "a.b", + "call": "a(b, *c, d=e)", + "args": "*a", + }, + ) + + def test_reverse_ops(self): + def f( + radd: 1 + a, + rsub: 1 - a, + rmul: 1 * a, + rmatmul: 1 @ a, + rtruediv: 1 / a, + rmod: 1 % a, + rlshift: 1 << a, + rrshift: 1 >> a, + ror: 1 | a, + rxor: 1 ^ a, + rand: 1 & a, + rfloordiv: 1 // a, + rpow: 1**a, + ): + pass + + anno = annotationlib.get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "radd": "1 + a", + "rsub": "1 - a", + "rmul": "1 * a", + "rmatmul": "1 @ a", + "rtruediv": "1 / a", + "rmod": "1 % a", + "rlshift": "1 << a", + "rrshift": "1 >> a", + "ror": "1 | a", + "rxor": "1 ^ a", + "rand": "1 & a", + "rfloordiv": "1 // a", + "rpow": "1 ** a", + }, + ) + + def test_nested_expressions(self): + def f( + nested: list[Annotated[set[int], "set of ints", 4j]], + set: {a + b}, # single element because order is not guaranteed + dict: {a + b: c + d, "key": e + g}, + list: [a, b, c], + tuple: (a, b, c), + slice: (a[b:c], a[b:c:d], a[:c], a[b:], a[:], a[::d], a[b::d]), + extended_slice: a[:, :, c:d], + unpack1: [*a], + unpack2: [*a, b, c], + ): + pass + + anno = annotationlib.get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "nested": "list[Annotated[set[int], 'set of ints', 4j]]", + "set": "{a + b}", + "dict": "{a + b: c + d, 'key': e + g}", + "list": "[a, b, c]", + "tuple": "(a, b, c)", + "slice": "(a[b:c], a[b:c:d], a[:c], a[b:], a[:], a[::d], a[b::d])", + "extended_slice": "a[:, :, c:d]", + "unpack1": "[*a]", + "unpack2": "[*a, b, c]", + }, + ) + + def test_unsupported_operations(self): + format_msg = "Cannot stringify annotation containing string formatting" + + def f(fstring: f"{a}"): + pass + + with self.assertRaisesRegex(TypeError, format_msg): + annotationlib.get_annotations(f, format=Format.STRING) + + def f(fstring_format: f"{a:02d}"): + pass + + with self.assertRaisesRegex(TypeError, format_msg): + annotationlib.get_annotations(f, format=Format.STRING) + + +class TestForwardRefClass(unittest.TestCase): + def test_special_attrs(self): + # Forward refs provide a different introspection API. __name__ and + # __qualname__ make little sense for forward refs as they can store + # complex typing expressions. + fr = annotationlib.ForwardRef("set[Any]") + self.assertFalse(hasattr(fr, "__name__")) + self.assertFalse(hasattr(fr, "__qualname__")) + self.assertEqual(fr.__module__, "annotationlib") + # Forward refs are currently unpicklable once they contain a code object. + fr.__forward_code__ # fill the cache + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.assertRaises(TypeError): + pickle.dumps(fr, proto) + + def test_evaluate_with_type_params(self): + class Gen[T]: + alias = int + + with self.assertRaises(NameError): + ForwardRef("T").evaluate() + with self.assertRaises(NameError): + ForwardRef("T").evaluate(type_params=()) + with self.assertRaises(NameError): + ForwardRef("T").evaluate(owner=int) + + (T,) = Gen.__type_params__ + self.assertIs(ForwardRef("T").evaluate(type_params=Gen.__type_params__), T) + self.assertIs(ForwardRef("T").evaluate(owner=Gen), T) + + with self.assertRaises(NameError): + ForwardRef("alias").evaluate(type_params=Gen.__type_params__) + self.assertIs(ForwardRef("alias").evaluate(owner=Gen), int) + # If you pass custom locals, we don't look at the owner's locals + with self.assertRaises(NameError): + ForwardRef("alias").evaluate(owner=Gen, locals={}) + # But if the name exists in the locals, it works + self.assertIs( + ForwardRef("alias").evaluate(owner=Gen, locals={"alias": str}), str + ) + + def test_fwdref_with_module(self): + self.assertIs(ForwardRef("Format", module="annotationlib").evaluate(), Format) + self.assertIs( + ForwardRef("Counter", module="collections").evaluate(), collections.Counter + ) + self.assertEqual( + ForwardRef("Counter[int]", module="collections").evaluate(), + collections.Counter[int], + ) + + with self.assertRaises(NameError): + # If globals are passed explicitly, we don't look at the module dict + ForwardRef("Format", module="annotationlib").evaluate(globals={}) + + def test_fwdref_to_builtin(self): + self.assertIs(ForwardRef("int").evaluate(), int) + self.assertIs(ForwardRef("int", module="collections").evaluate(), int) + self.assertIs(ForwardRef("int", owner=str).evaluate(), int) + + # builtins are still searched with explicit globals + self.assertIs(ForwardRef("int").evaluate(globals={}), int) + + # explicit values in globals have precedence + obj = object() + self.assertIs(ForwardRef("int").evaluate(globals={"int": obj}), obj) + + def test_fwdref_value_is_cached(self): + fr = ForwardRef("hello") + with self.assertRaises(NameError): + fr.evaluate() + self.assertIs(fr.evaluate(globals={"hello": str}), str) + self.assertIs(fr.evaluate(), str) + + def test_fwdref_with_owner(self): + self.assertEqual( + ForwardRef("Counter[int]", owner=collections).evaluate(), + collections.Counter[int], + ) + + def test_name_lookup_without_eval(self): + # test the codepath where we look up simple names directly in the + # namespaces without going through eval() + self.assertIs(ForwardRef("int").evaluate(), int) + self.assertIs(ForwardRef("int").evaluate(locals={"int": str}), str) + self.assertIs( + ForwardRef("int").evaluate(locals={"int": float}, globals={"int": str}), + float, + ) + self.assertIs(ForwardRef("int").evaluate(globals={"int": str}), str) + with support.swap_attr(builtins, "int", dict): + self.assertIs(ForwardRef("int").evaluate(), dict) + + with self.assertRaises(NameError): + ForwardRef("doesntexist").evaluate() + + def test_fwdref_invalid_syntax(self): + fr = ForwardRef("if") + with self.assertRaises(SyntaxError): + fr.evaluate() + fr = ForwardRef("1+") + with self.assertRaises(SyntaxError): + fr.evaluate() + + +class TestGetAnnotations(unittest.TestCase): + def test_builtin_type(self): + self.assertEqual(annotationlib.get_annotations(int), {}) + self.assertEqual(annotationlib.get_annotations(object), {}) + + def test_custom_metaclass(self): + class Meta(type): + pass + + class C(metaclass=Meta): + x: int + + self.assertEqual(annotationlib.get_annotations(C), {"x": int}) + + def test_missing_dunder_dict(self): + class NoDict(type): + @property + def __dict__(cls): + raise AttributeError + + b: str + + class C1(metaclass=NoDict): + a: int + + self.assertEqual(annotationlib.get_annotations(C1), {"a": int}) + self.assertEqual( + annotationlib.get_annotations(C1, format=Format.FORWARDREF), + {"a": int}, + ) + self.assertEqual( + annotationlib.get_annotations(C1, format=Format.STRING), + {"a": "int"}, + ) + self.assertEqual(annotationlib.get_annotations(NoDict), {"b": str}) + self.assertEqual( + annotationlib.get_annotations(NoDict, format=Format.FORWARDREF), + {"b": str}, + ) + self.assertEqual( + annotationlib.get_annotations(NoDict, format=Format.STRING), + {"b": "str"}, + ) + + def test_format(self): + def f1(a: int): + pass + + def f2(a: undefined): + pass + + self.assertEqual( + annotationlib.get_annotations(f1, format=Format.VALUE), + {"a": int}, + ) + self.assertEqual(annotationlib.get_annotations(f1, format=1), {"a": int}) + + fwd = annotationlib.ForwardRef("undefined") + self.assertEqual( + annotationlib.get_annotations(f2, format=Format.FORWARDREF), + {"a": fwd}, + ) + self.assertEqual(annotationlib.get_annotations(f2, format=3), {"a": fwd}) + + self.assertEqual( + annotationlib.get_annotations(f1, format=Format.STRING), + {"a": "int"}, + ) + self.assertEqual(annotationlib.get_annotations(f1, format=4), {"a": "int"}) + + with self.assertRaises(ValueError): + annotationlib.get_annotations(f1, format=42) + + with self.assertRaisesRegex( + ValueError, + r"The VALUE_WITH_FAKE_GLOBALS format is for internal use only", + ): + annotationlib.get_annotations(f1, format=Format.VALUE_WITH_FAKE_GLOBALS) + + with self.assertRaisesRegex( + ValueError, + r"The VALUE_WITH_FAKE_GLOBALS format is for internal use only", + ): + annotationlib.get_annotations(f1, format=2) + + def test_custom_object_with_annotations(self): + class C: + def __init__(self): + self.__annotations__ = {"x": int, "y": str} + + self.assertEqual(annotationlib.get_annotations(C()), {"x": int, "y": str}) + + def test_custom_format_eval_str(self): + def foo(): + pass + + with self.assertRaises(ValueError): + annotationlib.get_annotations(foo, format=Format.FORWARDREF, eval_str=True) + annotationlib.get_annotations(foo, format=Format.STRING, eval_str=True) + + def test_stock_annotations(self): + def foo(a: int, b: str): + pass + + for format in (Format.VALUE, Format.FORWARDREF): + with self.subTest(format=format): + self.assertEqual( + annotationlib.get_annotations(foo, format=format), + {"a": int, "b": str}, + ) + self.assertEqual( + annotationlib.get_annotations(foo, format=Format.STRING), + {"a": "int", "b": "str"}, + ) + + foo.__annotations__ = {"a": "foo", "b": "str"} + for format in Format: + if format is Format.VALUE_WITH_FAKE_GLOBALS: + continue + with self.subTest(format=format): + self.assertEqual( + annotationlib.get_annotations(foo, format=format), + {"a": "foo", "b": "str"}, + ) + + self.assertEqual( + annotationlib.get_annotations(foo, eval_str=True, locals=locals()), + {"a": foo, "b": str}, + ) + self.assertEqual( + annotationlib.get_annotations(foo, eval_str=True, globals=locals()), + {"a": foo, "b": str}, + ) + + def test_stock_annotations_in_module(self): + isa = inspect_stock_annotations + + for kwargs in [ + {}, + {"eval_str": False}, + {"format": Format.VALUE}, + {"format": Format.FORWARDREF}, + {"format": Format.VALUE, "eval_str": False}, + {"format": Format.FORWARDREF, "eval_str": False}, + ]: + with self.subTest(**kwargs): + self.assertEqual( + annotationlib.get_annotations(isa, **kwargs), {"a": int, "b": str} + ) + self.assertEqual( + annotationlib.get_annotations(isa.MyClass, **kwargs), + {"a": int, "b": str}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function, **kwargs), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function2, **kwargs), + {"a": int, "b": "str", "c": isa.MyClass, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function3, **kwargs), + {"a": "int", "b": "str", "c": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(annotationlib, **kwargs), {} + ) # annotations module has no annotations + self.assertEqual( + annotationlib.get_annotations(isa.UnannotatedClass, **kwargs), {} + ) + self.assertEqual( + annotationlib.get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + for kwargs in [ + {"eval_str": True}, + {"format": Format.VALUE, "eval_str": True}, + ]: + with self.subTest(**kwargs): + self.assertEqual( + annotationlib.get_annotations(isa, **kwargs), {"a": int, "b": str} + ) + self.assertEqual( + annotationlib.get_annotations(isa.MyClass, **kwargs), + {"a": int, "b": str}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function, **kwargs), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function2, **kwargs), + {"a": int, "b": str, "c": isa.MyClass, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function3, **kwargs), + {"a": int, "b": str, "c": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(annotationlib, **kwargs), {} + ) + self.assertEqual( + annotationlib.get_annotations(isa.UnannotatedClass, **kwargs), {} + ) + self.assertEqual( + annotationlib.get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + self.assertEqual( + annotationlib.get_annotations(isa, format=Format.STRING), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.MyClass, format=Format.STRING), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function, format=Format.STRING), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function2, format=Format.STRING), + {"a": "int", "b": "str", "c": "MyClass", "return": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function3, format=Format.STRING), + {"a": "int", "b": "str", "c": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(annotationlib, format=Format.STRING), + {}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.UnannotatedClass, format=Format.STRING), + {}, + ) + self.assertEqual( + annotationlib.get_annotations( + isa.unannotated_function, format=Format.STRING + ), + {}, + ) + + def test_stock_annotations_on_wrapper(self): + isa = inspect_stock_annotations + + wrapped = times_three(isa.function) + self.assertEqual(wrapped(1, "x"), isa.MyClass(3, "xxx")) + self.assertIsNot(wrapped.__globals__, isa.function.__globals__) + self.assertEqual( + annotationlib.get_annotations(wrapped), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(wrapped, format=Format.FORWARDREF), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(wrapped, format=Format.STRING), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(wrapped, eval_str=True), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(wrapped, eval_str=False), + {"a": int, "b": str, "return": isa.MyClass}, + ) + + def test_stringized_annotations_in_module(self): + isa = inspect_stringized_annotations + for kwargs in [ + {}, + {"eval_str": False}, + {"format": Format.VALUE}, + {"format": Format.FORWARDREF}, + {"format": Format.STRING}, + {"format": Format.VALUE, "eval_str": False}, + {"format": Format.FORWARDREF, "eval_str": False}, + {"format": Format.STRING, "eval_str": False}, + ]: + with self.subTest(**kwargs): + self.assertEqual( + annotationlib.get_annotations(isa, **kwargs), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.MyClass, **kwargs), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function, **kwargs), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function2, **kwargs), + {"a": "int", "b": "'str'", "c": "MyClass", "return": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function3, **kwargs), + {"a": "'int'", "b": "'str'", "c": "'MyClass'"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.UnannotatedClass, **kwargs), {} + ) + self.assertEqual( + annotationlib.get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + for kwargs in [ + {"eval_str": True}, + {"format": Format.VALUE, "eval_str": True}, + ]: + with self.subTest(**kwargs): + self.assertEqual( + annotationlib.get_annotations(isa, **kwargs), {"a": int, "b": str} + ) + self.assertEqual( + annotationlib.get_annotations(isa.MyClass, **kwargs), + {"a": int, "b": str}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function, **kwargs), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function2, **kwargs), + {"a": int, "b": "str", "c": isa.MyClass, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function3, **kwargs), + {"a": "int", "b": "str", "c": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.UnannotatedClass, **kwargs), {} + ) + self.assertEqual( + annotationlib.get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + def test_stringized_annotations_in_empty_module(self): + isa2 = inspect_stringized_annotations_2 + self.assertEqual(annotationlib.get_annotations(isa2), {}) + self.assertEqual(annotationlib.get_annotations(isa2, eval_str=True), {}) + self.assertEqual(annotationlib.get_annotations(isa2, eval_str=False), {}) + + def test_stringized_annotations_on_wrapper(self): + isa = inspect_stringized_annotations + wrapped = times_three(isa.function) + self.assertEqual(wrapped(1, "x"), isa.MyClass(3, "xxx")) + self.assertIsNot(wrapped.__globals__, isa.function.__globals__) + self.assertEqual( + annotationlib.get_annotations(wrapped), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(wrapped, eval_str=True), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(wrapped, eval_str=False), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + + def test_stringized_annotations_on_class(self): + isa = inspect_stringized_annotations + # test that local namespace lookups work + self.assertEqual( + annotationlib.get_annotations(isa.MyClassWithLocalAnnotations), + {"x": "mytype"}, + ) + self.assertEqual( + annotationlib.get_annotations( + isa.MyClassWithLocalAnnotations, eval_str=True + ), + {"x": int}, + ) + + def test_modify_annotations(self): + def f(x: int): + pass + + self.assertEqual(annotationlib.get_annotations(f), {"x": int}) + self.assertEqual( + annotationlib.get_annotations(f, format=Format.FORWARDREF), + {"x": int}, + ) + + f.__annotations__["x"] = str + # The modification is reflected in VALUE (the default) + self.assertEqual(annotationlib.get_annotations(f), {"x": str}) + # ... and also in FORWARDREF, which tries __annotations__ if available + self.assertEqual( + annotationlib.get_annotations(f, format=Format.FORWARDREF), + {"x": str}, + ) + # ... but not in STRING which always uses __annotate__ + self.assertEqual( + annotationlib.get_annotations(f, format=Format.STRING), + {"x": "int"}, + ) + + def test_non_dict_annotations(self): + class WeirdAnnotations: + @property + def __annotations__(self): + return "not a dict" + + wa = WeirdAnnotations() + for format in Format: + if format is Format.VALUE_WITH_FAKE_GLOBALS: + continue + with ( + self.subTest(format=format), + self.assertRaisesRegex( + ValueError, r".*__annotations__ is neither a dict nor None" + ), + ): + annotationlib.get_annotations(wa, format=format) + + def test_annotations_on_custom_object(self): + class HasAnnotations: + @property + def __annotations__(self): + return {"x": int} + + ha = HasAnnotations() + self.assertEqual( + annotationlib.get_annotations(ha, format=Format.VALUE), {"x": int} + ) + self.assertEqual( + annotationlib.get_annotations(ha, format=Format.FORWARDREF), {"x": int} + ) + + self.assertEqual( + annotationlib.get_annotations(ha, format=Format.STRING), {"x": "int"} + ) + + def test_raising_annotations_on_custom_object(self): + class HasRaisingAnnotations: + @property + def __annotations__(self): + return {"x": undefined} + + hra = HasRaisingAnnotations() + + with self.assertRaises(NameError): + annotationlib.get_annotations(hra, format=Format.VALUE) + + with self.assertRaises(NameError): + annotationlib.get_annotations(hra, format=Format.FORWARDREF) + + undefined = float + self.assertEqual( + annotationlib.get_annotations(hra, format=Format.VALUE), {"x": float} + ) + + def test_forwardref_prefers_annotations(self): + class HasBoth: + @property + def __annotations__(self): + return {"x": int} + + @property + def __annotate__(self): + return lambda format: {"x": str} + + hb = HasBoth() + self.assertEqual( + annotationlib.get_annotations(hb, format=Format.VALUE), {"x": int} + ) + self.assertEqual( + annotationlib.get_annotations(hb, format=Format.FORWARDREF), {"x": int} + ) + self.assertEqual( + annotationlib.get_annotations(hb, format=Format.STRING), {"x": str} + ) + + def test_pep695_generic_class_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + A_annotations = annotationlib.get_annotations(ann_module695.A, eval_str=True) + A_type_params = ann_module695.A.__type_params__ + self.assertIs(A_annotations["x"], A_type_params[0]) + self.assertEqual(A_annotations["y"].__args__[0], Unpack[A_type_params[1]]) + self.assertIs(A_annotations["z"].__args__[0], A_type_params[2]) + + def test_pep695_generic_class_with_future_annotations_and_local_shadowing(self): + B_annotations = annotationlib.get_annotations( + inspect_stringized_annotations_pep695.B, eval_str=True + ) + self.assertEqual(B_annotations, {"x": int, "y": str, "z": bytes}) + + def test_pep695_generic_class_with_future_annotations_name_clash_with_global_vars( + self, + ): + ann_module695 = inspect_stringized_annotations_pep695 + C_annotations = annotationlib.get_annotations(ann_module695.C, eval_str=True) + self.assertEqual( + set(C_annotations.values()), set(ann_module695.C.__type_params__) + ) + + def test_pep_695_generic_function_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + generic_func_annotations = annotationlib.get_annotations( + ann_module695.generic_function, eval_str=True + ) + func_t_params = ann_module695.generic_function.__type_params__ + self.assertEqual( + generic_func_annotations.keys(), {"x", "y", "z", "zz", "return"} + ) + self.assertIs(generic_func_annotations["x"], func_t_params[0]) + self.assertEqual(generic_func_annotations["y"], Unpack[func_t_params[1]]) + self.assertIs(generic_func_annotations["z"].__origin__, func_t_params[2]) + self.assertIs(generic_func_annotations["zz"].__origin__, func_t_params[2]) + + def test_pep_695_generic_function_with_future_annotations_name_clash_with_global_vars( + self, + ): + self.assertEqual( + set( + annotationlib.get_annotations( + inspect_stringized_annotations_pep695.generic_function_2, + eval_str=True, + ).values() + ), + set( + inspect_stringized_annotations_pep695.generic_function_2.__type_params__ + ), + ) + + def test_pep_695_generic_method_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + generic_method_annotations = annotationlib.get_annotations( + ann_module695.D.generic_method, eval_str=True + ) + params = { + param.__name__: param + for param in ann_module695.D.generic_method.__type_params__ + } + self.assertEqual( + generic_method_annotations, + {"x": params["Foo"], "y": params["Bar"], "return": None}, + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_vars( + self, + ): + self.assertEqual( + set( + annotationlib.get_annotations( + inspect_stringized_annotations_pep695.D.generic_method_2, + eval_str=True, + ).values() + ), + set( + inspect_stringized_annotations_pep695.D.generic_method_2.__type_params__ + ), + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_and_local_vars( + self, + ): + self.assertEqual( + annotationlib.get_annotations( + inspect_stringized_annotations_pep695.E, eval_str=True + ), + {"x": str}, + ) + + def test_pep_695_generics_with_future_annotations_nested_in_function(self): + results = inspect_stringized_annotations_pep695.nested() + + self.assertEqual( + set(results.F_annotations.values()), set(results.F.__type_params__) + ) + self.assertEqual( + set(results.F_meth_annotations.values()), + set(results.F.generic_method.__type_params__), + ) + self.assertNotEqual( + set(results.F_meth_annotations.values()), set(results.F.__type_params__) + ) + self.assertEqual( + set(results.F_meth_annotations.values()).intersection( + results.F.__type_params__ + ), + set(), + ) + + self.assertEqual(results.G_annotations, {"x": str}) + + self.assertEqual( + set(results.generic_func_annotations.values()), + set(results.generic_func.__type_params__), + ) + + +class TestCallEvaluateFunction(unittest.TestCase): + def test_evaluation(self): + def evaluate(format, exc=NotImplementedError): + if format > 2: + raise exc + return undefined + + with self.assertRaises(NameError): + annotationlib.call_evaluate_function(evaluate, Format.VALUE) + self.assertEqual( + annotationlib.call_evaluate_function(evaluate, Format.FORWARDREF), + annotationlib.ForwardRef("undefined"), + ) + self.assertEqual( + annotationlib.call_evaluate_function(evaluate, Format.STRING), + "undefined", + ) + + +class MetaclassTests(unittest.TestCase): + def test_annotated_meta(self): + class Meta(type): + a: int + + class X(metaclass=Meta): + pass + + class Y(metaclass=Meta): + b: float + + self.assertEqual(get_annotations(Meta), {"a": int}) + self.assertEqual(get_annotate_function(Meta)(Format.VALUE), {"a": int}) + + self.assertEqual(get_annotations(X), {}) + self.assertIs(get_annotate_function(X), None) + + self.assertEqual(get_annotations(Y), {"b": float}) + self.assertEqual(get_annotate_function(Y)(Format.VALUE), {"b": float}) + + def test_unannotated_meta(self): + class Meta(type): + pass + + class X(metaclass=Meta): + a: str + + class Y(X): + pass + + self.assertEqual(get_annotations(Meta), {}) + self.assertIs(get_annotate_function(Meta), None) + + self.assertEqual(get_annotations(Y), {}) + self.assertIs(get_annotate_function(Y), None) + + self.assertEqual(get_annotations(X), {"a": str}) + self.assertEqual(get_annotate_function(X)(Format.VALUE), {"a": str}) + + def test_ordering(self): + # Based on a sample by David Ellis + # https://discuss.python.org/t/pep-749-implementing-pep-649/54974/38 + + def make_classes(): + class Meta(type): + a: int + expected_annotations = {"a": int} + + class A(type, metaclass=Meta): + b: float + expected_annotations = {"b": float} + + class B(metaclass=A): + c: str + expected_annotations = {"c": str} + + class C(B): + expected_annotations = {} + + class D(metaclass=Meta): + expected_annotations = {} + + return Meta, A, B, C, D + + classes = make_classes() + class_count = len(classes) + for order in itertools.permutations(range(class_count), class_count): + names = ", ".join(classes[i].__name__ for i in order) + with self.subTest(names=names): + classes = make_classes() # Regenerate classes + for i in order: + get_annotations(classes[i]) + for c in classes: + with self.subTest(c=c): + self.assertEqual(get_annotations(c), c.expected_annotations) + annotate_func = get_annotate_function(c) + if c.expected_annotations: + self.assertEqual( + annotate_func(Format.VALUE), c.expected_annotations + ) + else: + self.assertIs(annotate_func, None) + + +class TestGetAnnotateFunction(unittest.TestCase): + def test_static_class(self): + self.assertIsNone(get_annotate_function(object)) + self.assertIsNone(get_annotate_function(int)) + + def test_unannotated_class(self): + class C: + pass + + self.assertIsNone(get_annotate_function(C)) + + D = type("D", (), {}) + self.assertIsNone(get_annotate_function(D)) + + def test_annotated_class(self): + class C: + a: int + + self.assertEqual(get_annotate_function(C)(Format.VALUE), {"a": int}) + + +class TestToSource(unittest.TestCase): + def test_value_to_string(self): + self.assertEqual(value_to_string(int), "int") + self.assertEqual(value_to_string(MyClass), "test.test_annotationlib.MyClass") + self.assertEqual(value_to_string(len), "len") + self.assertEqual(value_to_string(value_to_string), "value_to_string") + self.assertEqual(value_to_string(times_three), "times_three") + self.assertEqual(value_to_string(...), "...") + self.assertEqual(value_to_string(None), "None") + self.assertEqual(value_to_string(1), "1") + self.assertEqual(value_to_string("1"), "'1'") + self.assertEqual(value_to_string(Format.VALUE), repr(Format.VALUE)) + self.assertEqual(value_to_string(MyClass()), "my repr") + + def test_annotations_to_string(self): + self.assertEqual(annotations_to_string({}), {}) + self.assertEqual(annotations_to_string({"x": int}), {"x": "int"}) + self.assertEqual(annotations_to_string({"x": "int"}), {"x": "int"}) + self.assertEqual( + annotations_to_string({"x": int, "y": str}), {"x": "int", "y": "str"} + ) + + +class TestAnnotationLib(unittest.TestCase): + def test__all__(self): + support.check__all__(self, annotationlib) diff --git a/Lib/test/test_apple.py b/Lib/test/test_apple.py new file mode 100644 index 00000000000000..ab5296afad1d3f --- /dev/null +++ b/Lib/test/test_apple.py @@ -0,0 +1,155 @@ +import unittest +from _apple_support import SystemLog +from test.support import is_apple +from unittest.mock import Mock, call + +if not is_apple: + raise unittest.SkipTest("Apple-specific") + + +# Test redirection of stdout and stderr to the Apple system log. +class TestAppleSystemLogOutput(unittest.TestCase): + maxDiff = None + + def assert_writes(self, output): + self.assertEqual( + self.log_write.mock_calls, + [ + call(self.log_level, line) + for line in output + ] + ) + + self.log_write.reset_mock() + + def setUp(self): + self.log_write = Mock() + self.log_level = 42 + self.log = SystemLog(self.log_write, self.log_level, errors="replace") + + def test_repr(self): + self.assertEqual(repr(self.log), "") + self.assertEqual(repr(self.log.buffer), "") + + def test_log_config(self): + self.assertIs(self.log.writable(), True) + self.assertIs(self.log.readable(), False) + + self.assertEqual("UTF-8", self.log.encoding) + self.assertEqual("replace", self.log.errors) + + self.assertIs(self.log.line_buffering, True) + self.assertIs(self.log.write_through, False) + + def test_empty_str(self): + self.log.write("") + self.log.flush() + + self.assert_writes([]) + + def test_simple_str(self): + self.log.write("hello world\n") + + self.assert_writes([b"hello world\n"]) + + def test_buffered_str(self): + self.log.write("h") + self.log.write("ello") + self.log.write(" ") + self.log.write("world\n") + self.log.write("goodbye.") + self.log.flush() + + self.assert_writes([b"hello world\n", b"goodbye."]) + + def test_manual_flush(self): + self.log.write("Hello") + + self.assert_writes([]) + + self.log.write(" world\nHere for a while...\nGoodbye") + self.assert_writes([b"Hello world\n", b"Here for a while...\n"]) + + self.log.write(" world\nHello again") + self.assert_writes([b"Goodbye world\n"]) + + self.log.flush() + self.assert_writes([b"Hello again"]) + + def test_non_ascii(self): + # Spanish + self.log.write("ol\u00e9\n") + self.assert_writes([b"ol\xc3\xa9\n"]) + + # Chinese + self.log.write("\u4e2d\u6587\n") + self.assert_writes([b"\xe4\xb8\xad\xe6\x96\x87\n"]) + + # Printing Non-BMP emoji + self.log.write("\U0001f600\n") + self.assert_writes([b"\xf0\x9f\x98\x80\n"]) + + # Non-encodable surrogates are replaced + self.log.write("\ud800\udc00\n") + self.assert_writes([b"??\n"]) + + def test_modified_null(self): + # Null characters are logged using "modified UTF-8". + self.log.write("\u0000\n") + self.assert_writes([b"\xc0\x80\n"]) + self.log.write("a\u0000\n") + self.assert_writes([b"a\xc0\x80\n"]) + self.log.write("\u0000b\n") + self.assert_writes([b"\xc0\x80b\n"]) + self.log.write("a\u0000b\n") + self.assert_writes([b"a\xc0\x80b\n"]) + + def test_nonstandard_str(self): + # String subclasses are accepted, but they should be converted + # to a standard str without calling any of their methods. + class CustomStr(str): + def splitlines(self, *args, **kwargs): + raise AssertionError() + + def __len__(self): + raise AssertionError() + + def __str__(self): + raise AssertionError() + + self.log.write(CustomStr("custom\n")) + self.assert_writes([b"custom\n"]) + + def test_non_str(self): + # Non-string classes are not accepted. + for obj in [b"", b"hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be str, not " + fr"{type(obj).__name__}" + ): + self.log.write(obj) + + def test_byteslike_in_buffer(self): + # The underlying buffer *can* accept bytes-like objects + self.log.buffer.write(bytearray(b"hello")) + self.log.flush() + + self.log.buffer.write(b"") + self.log.flush() + + self.log.buffer.write(b"goodbye") + self.log.flush() + + self.assert_writes([b"hello", b"goodbye"]) + + def test_non_byteslike_in_buffer(self): + for obj in ["hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be bytes-like, not " + fr"{type(obj).__name__}" + ): + self.log.buffer.write(obj) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 617b1721f3dbb1..488a3a4ed20fac 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -6,6 +6,7 @@ import io import operator import os +import py_compile import shutil import stat import sys @@ -15,10 +16,18 @@ import argparse import warnings -from test.support import os_helper, captured_stderr +from enum import StrEnum +from test.support import captured_stderr +from test.support import import_helper +from test.support import os_helper +from test.support import script_helper +from test.support.i18n_helper import TestTranslationsBase, update_translation_snapshots from unittest import mock +py = os.path.basename(sys.executable) + + class StdIOBuffer(io.TextIOWrapper): '''Replacement for writable io.StringIO that behaves more like real file @@ -280,16 +289,18 @@ def test_failures(self, tester): parser = self._get_parser(tester) for args_str in tester.failures: args = args_str.split() - with tester.assertRaises(ArgumentParserError, msg=args): - parser.parse_args(args) + with tester.subTest(args=args): + with tester.assertRaises(ArgumentParserError, msg=args): + parser.parse_args(args) def test_successes(self, tester): parser = self._get_parser(tester) for args, expected_ns in tester.successes: if isinstance(args, str): args = args.split() - result_ns = self._parse_args(parser, args) - tester.assertEqual(expected_ns, result_ns) + with tester.subTest(args=args): + result_ns = self._parse_args(parser, args) + tester.assertEqual(expected_ns, result_ns) # add tests for each combination of an optionals adding method # and an arg parsing method @@ -378,15 +389,22 @@ class TestOptionalsSingleDashAmbiguous(ParserTestCase): """Test Optionals that partially match but are not subsets""" argument_signatures = [Sig('-foobar'), Sig('-foorab')] - failures = ['-f', '-f a', '-fa', '-foa', '-foo', '-fo', '-foo b'] + failures = ['-f', '-f a', '-fa', '-foa', '-foo', '-fo', '-foo b', + '-f=a', '-foo=b'] successes = [ ('', NS(foobar=None, foorab=None)), ('-foob a', NS(foobar='a', foorab=None)), + ('-foob=a', NS(foobar='a', foorab=None)), ('-foor a', NS(foobar=None, foorab='a')), + ('-foor=a', NS(foobar=None, foorab='a')), ('-fooba a', NS(foobar='a', foorab=None)), + ('-fooba=a', NS(foobar='a', foorab=None)), ('-foora a', NS(foobar=None, foorab='a')), + ('-foora=a', NS(foobar=None, foorab='a')), ('-foobar a', NS(foobar='a', foorab=None)), + ('-foobar=a', NS(foobar='a', foorab=None)), ('-foorab a', NS(foobar=None, foorab='a')), + ('-foorab=a', NS(foobar=None, foorab='a')), ] @@ -619,9 +637,9 @@ class TestOptionalsNargsOptional(ParserTestCase): Sig('-w', nargs='?'), Sig('-x', nargs='?', const=42), Sig('-y', nargs='?', default='spam'), - Sig('-z', nargs='?', type=int, const='42', default='84'), + Sig('-z', nargs='?', type=int, const='42', default='84', choices=[1, 2]), ] - failures = ['2'] + failures = ['2', '-z a', '-z 42', '-z 84'] successes = [ ('', NS(w=None, x=None, y='spam', z=84)), ('-w', NS(w=None, x=None, y='spam', z=84)), @@ -677,7 +695,7 @@ class TestOptionalsChoices(ParserTestCase): argument_signatures = [ Sig('-f', choices='abc'), Sig('-g', type=int, choices=range(5))] - failures = ['a', '-f d', '-fad', '-ga', '-g 6'] + failures = ['a', '-f d', '-f ab', '-fad', '-ga', '-g 6'] successes = [ ('', NS(f=None, g=None)), ('-f a', NS(f='a', g=None)), @@ -765,48 +783,12 @@ def test_const(self): self.assertIn("got an unexpected keyword argument 'const'", str(cm.exception)) - def test_deprecated_init_kw(self): - # See gh-92248 + def test_invalid_name(self): parser = argparse.ArgumentParser() - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-a', - action=argparse.BooleanOptionalAction, - type=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-b', - action=argparse.BooleanOptionalAction, - type=bool, - ) - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-c', - action=argparse.BooleanOptionalAction, - metavar=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-d', - action=argparse.BooleanOptionalAction, - metavar='d', - ) - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-e', - action=argparse.BooleanOptionalAction, - choices=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-f', - action=argparse.BooleanOptionalAction, - choices=(), - ) + with self.assertRaises(ValueError) as cm: + parser.add_argument('--no-foo', action=argparse.BooleanOptionalAction) + self.assertEqual(str(cm.exception), + "invalid option name '--no-foo' for BooleanOptionalAction") class TestBooleanOptionalActionRequired(ParserTestCase): """Tests BooleanOptionalAction required""" @@ -916,7 +898,9 @@ class TestOptionalsAllowLongAbbreviation(ParserTestCase): successes = [ ('', NS(foo=None, foobaz=None, fooble=False)), ('--foo 7', NS(foo='7', foobaz=None, fooble=False)), + ('--foo=7', NS(foo='7', foobaz=None, fooble=False)), ('--fooba a', NS(foo=None, foobaz='a', fooble=False)), + ('--fooba=a', NS(foo=None, foobaz='a', fooble=False)), ('--foobl --foo g', NS(foo='g', foobaz=None, fooble=True)), ] @@ -955,6 +939,23 @@ class TestOptionalsDisallowLongAbbreviationPrefixChars(ParserTestCase): ] +class TestOptionalsDisallowSingleDashLongAbbreviation(ParserTestCase): + """Do not allow abbreviations of long options at all""" + + parser_signature = Sig(allow_abbrev=False) + argument_signatures = [ + Sig('-foo'), + Sig('-foodle', action='store_true'), + Sig('-foonly'), + ] + failures = ['-foon 3', '-food', '-food -foo 2'] + successes = [ + ('', NS(foo=None, foodle=False, foonly=None)), + ('-foo 3', NS(foo='3', foodle=False, foonly=None)), + ('-foonly 7 -foodle -foo 2', NS(foo='2', foodle=True, foonly='7')), + ] + + class TestDisallowLongAbbreviationAllowsShortGrouping(ParserTestCase): """Do not allow abbreviations of long options at all""" @@ -993,6 +994,34 @@ class TestDisallowLongAbbreviationAllowsShortGroupingPrefix(ParserTestCase): ] +class TestStrEnumChoices(TestCase): + class Color(StrEnum): + RED = "red" + GREEN = "green" + BLUE = "blue" + + def test_parse_enum_value(self): + parser = argparse.ArgumentParser() + parser.add_argument('--color', choices=self.Color) + args = parser.parse_args(['--color', 'red']) + self.assertEqual(args.color, self.Color.RED) + + def test_help_message_contains_enum_choices(self): + parser = argparse.ArgumentParser() + parser.add_argument('--color', choices=self.Color, help='Choose a color') + self.assertIn('[--color {red,green,blue}]', parser.format_usage()) + self.assertIn(' --color {red,green,blue}', parser.format_help()) + + def test_invalid_enum_value_raises_error(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('--color', choices=self.Color) + self.assertRaisesRegex( + argparse.ArgumentError, + r"invalid choice: 'yellow' \(choose from red, green, blue\)", + parser.parse_args, + ['--color', 'yellow'], + ) + # ================ # Positional tests # ================ @@ -1042,8 +1071,8 @@ class TestPositionalsNargsZeroOrMore(ParserTestCase): class TestPositionalsNargsZeroOrMoreDefault(ParserTestCase): """Test a Positional that specifies unlimited nargs and a default""" - argument_signatures = [Sig('foo', nargs='*', default='bar')] - failures = ['-x'] + argument_signatures = [Sig('foo', nargs='*', default='bar', choices=['a', 'b'])] + failures = ['-x', 'bar', 'a c'] successes = [ ('', NS(foo='bar')), ('a', NS(foo=['a'])), @@ -1076,8 +1105,8 @@ class TestPositionalsNargsOptional(ParserTestCase): class TestPositionalsNargsOptionalDefault(ParserTestCase): """Tests an Optional Positional with a default value""" - argument_signatures = [Sig('foo', nargs='?', default=42)] - failures = ['-x', 'a b'] + argument_signatures = [Sig('foo', nargs='?', default=42, choices=['a', 'b'])] + failures = ['-x', 'a b', '42'] successes = [ ('', NS(foo=42)), ('a', NS(foo='a')), @@ -1090,9 +1119,9 @@ class TestPositionalsNargsOptionalConvertedDefault(ParserTestCase): """ argument_signatures = [ - Sig('foo', nargs='?', type=int, default='42'), + Sig('foo', nargs='?', type=int, default='42', choices=[1, 2]), ] - failures = ['-x', 'a b', '1 2'] + failures = ['-x', 'a b', '1 2', '42'] successes = [ ('', NS(foo=42)), ('1', NS(foo=1)), @@ -1132,57 +1161,87 @@ class TestPositionalsNargs2None(ParserTestCase): class TestPositionalsNargsNoneZeroOrMore(ParserTestCase): """Test a Positional with no nargs followed by one with unlimited""" - argument_signatures = [Sig('foo'), Sig('bar', nargs='*')] - failures = ['', '--foo'] + argument_signatures = [Sig('-x'), Sig('foo'), Sig('bar', nargs='*')] + failures = ['', '--foo', 'a b -x X c'] successes = [ - ('a', NS(foo='a', bar=[])), - ('a b', NS(foo='a', bar=['b'])), - ('a b c', NS(foo='a', bar=['b', 'c'])), + ('a', NS(x=None, foo='a', bar=[])), + ('a b', NS(x=None, foo='a', bar=['b'])), + ('a b c', NS(x=None, foo='a', bar=['b', 'c'])), + ('-x X a', NS(x='X', foo='a', bar=[])), + ('a -x X', NS(x='X', foo='a', bar=[])), + ('-x X a b', NS(x='X', foo='a', bar=['b'])), + ('a -x X b', NS(x='X', foo='a', bar=['b'])), + ('a b -x X', NS(x='X', foo='a', bar=['b'])), + ('-x X a b c', NS(x='X', foo='a', bar=['b', 'c'])), + ('a -x X b c', NS(x='X', foo='a', bar=['b', 'c'])), + ('a b c -x X', NS(x='X', foo='a', bar=['b', 'c'])), ] class TestPositionalsNargsNoneOneOrMore(ParserTestCase): """Test a Positional with no nargs followed by one with one or more""" - argument_signatures = [Sig('foo'), Sig('bar', nargs='+')] - failures = ['', '--foo', 'a'] + argument_signatures = [Sig('-x'), Sig('foo'), Sig('bar', nargs='+')] + failures = ['', '--foo', 'a', 'a b -x X c'] successes = [ - ('a b', NS(foo='a', bar=['b'])), - ('a b c', NS(foo='a', bar=['b', 'c'])), + ('a b', NS(x=None, foo='a', bar=['b'])), + ('a b c', NS(x=None, foo='a', bar=['b', 'c'])), + ('-x X a b', NS(x='X', foo='a', bar=['b'])), + ('a -x X b', NS(x='X', foo='a', bar=['b'])), + ('a b -x X', NS(x='X', foo='a', bar=['b'])), + ('-x X a b c', NS(x='X', foo='a', bar=['b', 'c'])), + ('a -x X b c', NS(x='X', foo='a', bar=['b', 'c'])), + ('a b c -x X', NS(x='X', foo='a', bar=['b', 'c'])), ] class TestPositionalsNargsNoneOptional(ParserTestCase): """Test a Positional with no nargs followed by one with an Optional""" - argument_signatures = [Sig('foo'), Sig('bar', nargs='?')] + argument_signatures = [Sig('-x'), Sig('foo'), Sig('bar', nargs='?')] failures = ['', '--foo', 'a b c'] successes = [ - ('a', NS(foo='a', bar=None)), - ('a b', NS(foo='a', bar='b')), + ('a', NS(x=None, foo='a', bar=None)), + ('a b', NS(x=None, foo='a', bar='b')), + ('-x X a', NS(x='X', foo='a', bar=None)), + ('a -x X', NS(x='X', foo='a', bar=None)), + ('-x X a b', NS(x='X', foo='a', bar='b')), + ('a -x X b', NS(x='X', foo='a', bar='b')), + ('a b -x X', NS(x='X', foo='a', bar='b')), ] class TestPositionalsNargsZeroOrMoreNone(ParserTestCase): """Test a Positional with unlimited nargs followed by one with none""" - argument_signatures = [Sig('foo', nargs='*'), Sig('bar')] - failures = ['', '--foo'] + argument_signatures = [Sig('-x'), Sig('foo', nargs='*'), Sig('bar')] + failures = ['', '--foo', 'a -x X b', 'a -x X b c', 'a b -x X c'] successes = [ - ('a', NS(foo=[], bar='a')), - ('a b', NS(foo=['a'], bar='b')), - ('a b c', NS(foo=['a', 'b'], bar='c')), + ('a', NS(x=None, foo=[], bar='a')), + ('a b', NS(x=None, foo=['a'], bar='b')), + ('a b c', NS(x=None, foo=['a', 'b'], bar='c')), + ('-x X a', NS(x='X', foo=[], bar='a')), + ('a -x X', NS(x='X', foo=[], bar='a')), + ('-x X a b', NS(x='X', foo=['a'], bar='b')), + ('a b -x X', NS(x='X', foo=['a'], bar='b')), + ('-x X a b c', NS(x='X', foo=['a', 'b'], bar='c')), + ('a b c -x X', NS(x='X', foo=['a', 'b'], bar='c')), ] class TestPositionalsNargsOneOrMoreNone(ParserTestCase): """Test a Positional with one or more nargs followed by one with none""" - argument_signatures = [Sig('foo', nargs='+'), Sig('bar')] - failures = ['', '--foo', 'a'] + argument_signatures = [Sig('-x'), Sig('foo', nargs='+'), Sig('bar')] + failures = ['', '--foo', 'a', 'a -x X b c', 'a b -x X c'] successes = [ - ('a b', NS(foo=['a'], bar='b')), - ('a b c', NS(foo=['a', 'b'], bar='c')), + ('a b', NS(x=None, foo=['a'], bar='b')), + ('a b c', NS(x=None, foo=['a', 'b'], bar='c')), + ('-x X a b', NS(x='X', foo=['a'], bar='b')), + ('a -x X b', NS(x='X', foo=['a'], bar='b')), + ('a b -x X', NS(x='X', foo=['a'], bar='b')), + ('-x X a b c', NS(x='X', foo=['a', 'b'], bar='c')), + ('a b c -x X', NS(x='X', foo=['a', 'b'], bar='c')), ] @@ -1267,14 +1326,21 @@ class TestPositionalsNargsNoneZeroOrMore1(ParserTestCase): """Test three Positionals: no nargs, unlimited nargs and 1 nargs""" argument_signatures = [ + Sig('-x'), Sig('foo'), Sig('bar', nargs='*'), Sig('baz', nargs=1), ] - failures = ['', '--foo', 'a'] + failures = ['', '--foo', 'a', 'a b -x X c'] successes = [ - ('a b', NS(foo='a', bar=[], baz=['b'])), - ('a b c', NS(foo='a', bar=['b'], baz=['c'])), + ('a b', NS(x=None, foo='a', bar=[], baz=['b'])), + ('a b c', NS(x=None, foo='a', bar=['b'], baz=['c'])), + ('-x X a b', NS(x='X', foo='a', bar=[], baz=['b'])), + ('a -x X b', NS(x='X', foo='a', bar=[], baz=['b'])), + ('a b -x X', NS(x='X', foo='a', bar=[], baz=['b'])), + ('-x X a b c', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('a -x X b c', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('a b c -x X', NS(x='X', foo='a', bar=['b'], baz=['c'])), ] @@ -1282,14 +1348,22 @@ class TestPositionalsNargsNoneOneOrMore1(ParserTestCase): """Test three Positionals: no nargs, one or more nargs and 1 nargs""" argument_signatures = [ + Sig('-x'), Sig('foo'), Sig('bar', nargs='+'), Sig('baz', nargs=1), ] - failures = ['', '--foo', 'a', 'b'] + failures = ['', '--foo', 'a', 'b', 'a b -x X c d', 'a b c -x X d'] successes = [ - ('a b c', NS(foo='a', bar=['b'], baz=['c'])), - ('a b c d', NS(foo='a', bar=['b', 'c'], baz=['d'])), + ('a b c', NS(x=None, foo='a', bar=['b'], baz=['c'])), + ('a b c d', NS(x=None, foo='a', bar=['b', 'c'], baz=['d'])), + ('-x X a b c', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('a -x X b c', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('a b -x X c', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('a b c -x X', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('-x X a b c d', NS(x='X', foo='a', bar=['b', 'c'], baz=['d'])), + ('a -x X b c d', NS(x='X', foo='a', bar=['b', 'c'], baz=['d'])), + ('a b c d -x X', NS(x='X', foo='a', bar=['b', 'c'], baz=['d'])), ] @@ -1297,14 +1371,21 @@ class TestPositionalsNargsNoneOptional1(ParserTestCase): """Test three Positionals: no nargs, optional narg and 1 nargs""" argument_signatures = [ + Sig('-x'), Sig('foo'), Sig('bar', nargs='?', default=0.625), Sig('baz', nargs=1), ] - failures = ['', '--foo', 'a'] + failures = ['', '--foo', 'a', 'a b -x X c'] successes = [ - ('a b', NS(foo='a', bar=0.625, baz=['b'])), - ('a b c', NS(foo='a', bar='b', baz=['c'])), + ('a b', NS(x=None, foo='a', bar=0.625, baz=['b'])), + ('a b c', NS(x=None, foo='a', bar='b', baz=['c'])), + ('-x X a b', NS(x='X', foo='a', bar=0.625, baz=['b'])), + ('a -x X b', NS(x='X', foo='a', bar=0.625, baz=['b'])), + ('a b -x X', NS(x='X', foo='a', bar=0.625, baz=['b'])), + ('-x X a b c', NS(x='X', foo='a', bar='b', baz=['c'])), + ('a -x X b c', NS(x='X', foo='a', bar='b', baz=['c'])), + ('a b c -x X', NS(x='X', foo='a', bar='b', baz=['c'])), ] @@ -1520,6 +1601,9 @@ class TestNargsRemainder(ParserTestCase): successes = [ ('X', NS(x='X', y=[], z=None)), ('-z Z X', NS(x='X', y=[], z='Z')), + ('-z Z X A B', NS(x='X', y=['A', 'B'], z='Z')), + ('X -z Z A B', NS(x='X', y=['-z', 'Z', 'A', 'B'], z=None)), + ('X A -z Z B', NS(x='X', y=['A', '-z', 'Z', 'B'], z=None)), ('X A B -z Z', NS(x='X', y=['A', 'B', '-z', 'Z'], z=None)), ('X Y --foo', NS(x='X', y=['Y', '--foo'], z=None)), ] @@ -1556,18 +1640,24 @@ class TestDefaultSuppress(ParserTestCase): """Test actions with suppressed defaults""" argument_signatures = [ - Sig('foo', nargs='?', default=argparse.SUPPRESS), - Sig('bar', nargs='*', default=argparse.SUPPRESS), + Sig('foo', nargs='?', type=int, default=argparse.SUPPRESS), + Sig('bar', nargs='*', type=int, default=argparse.SUPPRESS), Sig('--baz', action='store_true', default=argparse.SUPPRESS), + Sig('--qux', nargs='?', type=int, default=argparse.SUPPRESS), + Sig('--quux', nargs='*', type=int, default=argparse.SUPPRESS), ] - failures = ['-x'] + failures = ['-x', 'a', '1 a'] successes = [ ('', NS()), - ('a', NS(foo='a')), - ('a b', NS(foo='a', bar=['b'])), + ('1', NS(foo=1)), + ('1 2', NS(foo=1, bar=[2])), ('--baz', NS(baz=True)), - ('a --baz', NS(foo='a', baz=True)), - ('--baz a b', NS(foo='a', bar=['b'], baz=True)), + ('1 --baz', NS(foo=1, baz=True)), + ('--baz 1 2', NS(foo=1, bar=[2], baz=True)), + ('--qux', NS(qux=None)), + ('--qux 1', NS(qux=1)), + ('--quux', NS(quux=[])), + ('--quux 1 2', NS(quux=[1, 2])), ] @@ -1691,27 +1781,43 @@ def convert_arg_line_to_args(self, arg_line): # Type conversion tests # ===================== +def FileType(*args, **kwargs): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'FileType is deprecated', + PendingDeprecationWarning, __name__) + return argparse.FileType(*args, **kwargs) + + +class TestFileTypeDeprecation(TestCase): + + def test(self): + with self.assertWarns(PendingDeprecationWarning) as cm: + argparse.FileType() + self.assertIn('FileType is deprecated', str(cm.warning)) + self.assertEqual(cm.filename, __file__) + + class TestFileTypeRepr(TestCase): def test_r(self): - type = argparse.FileType('r') + type = FileType('r') self.assertEqual("FileType('r')", repr(type)) def test_wb_1(self): - type = argparse.FileType('wb', 1) + type = FileType('wb', 1) self.assertEqual("FileType('wb', 1)", repr(type)) def test_r_latin(self): - type = argparse.FileType('r', encoding='latin_1') + type = FileType('r', encoding='latin_1') self.assertEqual("FileType('r', encoding='latin_1')", repr(type)) def test_w_big5_ignore(self): - type = argparse.FileType('w', encoding='big5', errors='ignore') + type = FileType('w', encoding='big5', errors='ignore') self.assertEqual("FileType('w', encoding='big5', errors='ignore')", repr(type)) def test_r_1_replace(self): - type = argparse.FileType('r', 1, errors='replace') + type = FileType('r', 1, errors='replace') self.assertEqual("FileType('r', 1, errors='replace')", repr(type)) @@ -1765,7 +1871,6 @@ def __eq__(self, other): text = text.decode('ascii') return self.name == other.name == text - class TestFileTypeR(TempDirMixin, ParserTestCase): """Test the FileType option/argument type for reading files""" @@ -1778,8 +1883,8 @@ def setUp(self): self.create_readonly_file('readonly') argument_signatures = [ - Sig('-x', type=argparse.FileType()), - Sig('spam', type=argparse.FileType('r')), + Sig('-x', type=FileType()), + Sig('spam', type=FileType('r')), ] failures = ['-x', '', 'non-existent-file.txt'] successes = [ @@ -1799,7 +1904,7 @@ def setUp(self): file.close() argument_signatures = [ - Sig('-c', type=argparse.FileType('r'), default='no-file.txt'), + Sig('-c', type=FileType('r'), default='no-file.txt'), ] # should provoke no such file error failures = [''] @@ -1818,8 +1923,8 @@ def setUp(self): file.write(file_name) argument_signatures = [ - Sig('-x', type=argparse.FileType('rb')), - Sig('spam', type=argparse.FileType('rb')), + Sig('-x', type=FileType('rb')), + Sig('spam', type=FileType('rb')), ] failures = ['-x', ''] successes = [ @@ -1857,8 +1962,8 @@ def setUp(self): self.create_writable_file('writable') argument_signatures = [ - Sig('-x', type=argparse.FileType('w')), - Sig('spam', type=argparse.FileType('w')), + Sig('-x', type=FileType('w')), + Sig('spam', type=FileType('w')), ] failures = ['-x', '', 'readonly'] successes = [ @@ -1880,8 +1985,8 @@ def setUp(self): self.create_writable_file('writable') argument_signatures = [ - Sig('-x', type=argparse.FileType('x')), - Sig('spam', type=argparse.FileType('x')), + Sig('-x', type=FileType('x')), + Sig('spam', type=FileType('x')), ] failures = ['-x', '', 'readonly', 'writable'] successes = [ @@ -1895,8 +2000,8 @@ class TestFileTypeWB(TempDirMixin, ParserTestCase): """Test the FileType option/argument type for writing binary files""" argument_signatures = [ - Sig('-x', type=argparse.FileType('wb')), - Sig('spam', type=argparse.FileType('wb')), + Sig('-x', type=FileType('wb')), + Sig('spam', type=FileType('wb')), ] failures = ['-x', ''] successes = [ @@ -1912,8 +2017,8 @@ class TestFileTypeXB(TestFileTypeX): "Test the FileType option/argument type for writing new binary files only" argument_signatures = [ - Sig('-x', type=argparse.FileType('xb')), - Sig('spam', type=argparse.FileType('xb')), + Sig('-x', type=FileType('xb')), + Sig('spam', type=FileType('xb')), ] successes = [ ('-x foo bar', NS(x=WFile('foo'), spam=WFile('bar'))), @@ -1925,7 +2030,7 @@ class TestFileTypeOpenArgs(TestCase): """Test that open (the builtin) is correctly called""" def test_open_args(self): - FT = argparse.FileType + FT = FileType cases = [ (FT('rb'), ('rb', -1, None, None)), (FT('w', 1), ('w', 1, None, None)), @@ -1940,7 +2045,7 @@ def test_open_args(self): def test_invalid_file_type(self): with self.assertRaises(ValueError): - argparse.FileType('b')('-test') + FileType('b')('-test') class TestFileTypeMissingInitialization(TestCase): @@ -1951,7 +2056,7 @@ class TestFileTypeMissingInitialization(TestCase): def test(self): parser = argparse.ArgumentParser() - with self.assertRaises(ValueError) as cm: + with self.assertRaises(TypeError) as cm: parser.add_argument('-x', type=argparse.FileType) self.assertEqual( @@ -2136,6 +2241,130 @@ class TestActionExtend(ParserTestCase): ] +class TestNegativeNumber(ParserTestCase): + """Test parsing negative numbers""" + + argument_signatures = [ + Sig('--int', type=int), + Sig('--float', type=float), + Sig('--complex', type=complex), + ] + failures = [ + '--float -_.45', + '--float -1__000.0', + '--float -1.0.0', + '--int -1__000', + '--int -1.0', + '--complex -1__000.0j', + '--complex -1.0jj', + '--complex -_.45j', + ] + successes = [ + ('--int -1000 --float -1000.0', NS(int=-1000, float=-1000.0, complex=None)), + ('--int -1_000 --float -1_000.0', NS(int=-1000, float=-1000.0, complex=None)), + ('--int -1_000_000 --float -1_000_000.0', NS(int=-1000000, float=-1000000.0, complex=None)), + ('--float -1_000.0', NS(int=None, float=-1000.0, complex=None)), + ('--float -1_000_000.0_0', NS(int=None, float=-1000000.0, complex=None)), + ('--float -.5', NS(int=None, float=-0.5, complex=None)), + ('--float -.5_000', NS(int=None, float=-0.5, complex=None)), + ('--float -1e3', NS(int=None, float=-1000, complex=None)), + ('--float -1e-3', NS(int=None, float=-0.001, complex=None)), + ('--complex -1j', NS(int=None, float=None, complex=-1j)), + ('--complex -1_000j', NS(int=None, float=None, complex=-1000j)), + ('--complex -1_000.0j', NS(int=None, float=None, complex=-1000.0j)), + ('--complex -1e3j', NS(int=None, float=None, complex=-1000j)), + ('--complex -1e-3j', NS(int=None, float=None, complex=-0.001j)), + ] + +class TestArgumentAndSubparserSuggestions(TestCase): + """Test error handling and suggestion when a user makes a typo""" + + def test_wrong_argument_error_with_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=['bar', 'baz']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz', maybe you meant 'baz'? (choose from bar, baz)", + excinfo.exception.stderr + ) + + def test_wrong_argument_error_no_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=False) + parser.add_argument('foo', choices=['bar', 'baz']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz' (choose from bar, baz)", + excinfo.exception.stderr, + ) + + def test_wrong_argument_subparsers_with_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + subparsers = parser.add_subparsers(required=True) + subparsers.add_parser('foo') + subparsers.add_parser('bar') + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('baz',)) + self.assertIn( + "error: argument {foo,bar}: invalid choice: 'baz', maybe you meant" + " 'bar'? (choose from foo, bar)", + excinfo.exception.stderr, + ) + + def test_wrong_argument_subparsers_no_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=False) + subparsers = parser.add_subparsers(required=True) + subparsers.add_parser('foo') + subparsers.add_parser('bar') + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('baz',)) + self.assertIn( + "error: argument {foo,bar}: invalid choice: 'baz' (choose from foo, bar)", + excinfo.exception.stderr, + ) + + def test_wrong_argument_no_suggestion_implicit(self): + parser = ErrorRaisingArgumentParser() + parser.add_argument('foo', choices=['bar', 'baz']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz' (choose from bar, baz)", + excinfo.exception.stderr, + ) + + def test_suggestions_choices_empty(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=[]) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz' (choose from )", + excinfo.exception.stderr, + ) + + def test_suggestions_choices_int(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=[1, 2]) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('3',)) + self.assertIn( + "error: argument foo: invalid choice: '3' (choose from 1, 2)", + excinfo.exception.stderr, + ) + + def test_suggestions_choices_mixed_types(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=[1, '2']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('3',)) + self.assertIn( + "error: argument foo: invalid choice: '3' (choose from 1, 2)", + excinfo.exception.stderr, + ) + + class TestInvalidAction(TestCase): """Test invalid user defined Action""" @@ -2149,11 +2378,24 @@ def test_invalid_type(self): self.assertRaises(NotImplementedError, parser.parse_args, ['--foo', 'bar']) def test_modified_invalid_action(self): - parser = ErrorRaisingArgumentParser() + parser = argparse.ArgumentParser(exit_on_error=False) action = parser.add_argument('--foo') # Someone got crazy and did this action.type = 1 - self.assertRaises(ArgumentParserError, parser.parse_args, ['--foo', 'bar']) + self.assertRaisesRegex(TypeError, '1 is not callable', + parser.parse_args, ['--foo', 'bar']) + action.type = () + self.assertRaisesRegex(TypeError, r'\(\) is not callable', + parser.parse_args, ['--foo', 'bar']) + # It is impossible to distinguish a TypeError raised due to a mismatch + # of the required function arguments from a TypeError raised for an incorrect + # argument value, and using the heavy inspection machinery is not worthwhile + # as it does not reliably work in all cases. + # Therefore, a generic ArgumentError is raised to handle this logical error. + action.type = pow + self.assertRaisesRegex(argparse.ArgumentError, + "argument --foo: invalid pow value: 'bar'", + parser.parse_args, ['--foo', 'bar']) # ================ @@ -2167,16 +2409,17 @@ def assertArgumentParserError(self, *args, **kwargs): self.assertRaises(ArgumentParserError, *args, **kwargs) def _get_parser(self, subparser_help=False, prefix_chars=None, - aliases=False): + aliases=False, usage=None): # create a parser with a subparsers argument if prefix_chars: parser = ErrorRaisingArgumentParser( - prog='PROG', description='main description', prefix_chars=prefix_chars) + prog='PROG', description='main description', usage=usage, + prefix_chars=prefix_chars) parser.add_argument( prefix_chars[0] * 2 + 'foo', action='store_true', help='foo help') else: parser = ErrorRaisingArgumentParser( - prog='PROG', description='main description') + prog='PROG', description='main description', usage=usage) parser.add_argument( '--foo', action='store_true', help='foo help') parser.add_argument( @@ -2190,7 +2433,9 @@ def _get_parser(self, subparser_help=False, prefix_chars=None, else: subparsers_kwargs['help'] = 'command help' subparsers = parser.add_subparsers(**subparsers_kwargs) - self.assertArgumentParserError(parser.add_subparsers) + self.assertRaisesRegex(ValueError, + 'cannot have multiple subparser arguments', + parser.add_subparsers) # add first sub-parser parser1_kwargs = dict(description='1 description') @@ -2200,18 +2445,19 @@ def _get_parser(self, subparser_help=False, prefix_chars=None, parser1_kwargs['aliases'] = ['1alias1', '1alias2'] parser1 = subparsers.add_parser('1', **parser1_kwargs) parser1.add_argument('-w', type=int, help='w help') - parser1.add_argument('x', choices='abc', help='x help') + parser1.add_argument('x', choices=['a', 'b', 'c'], help='x help') # add second sub-parser parser2_kwargs = dict(description='2 description') if subparser_help: parser2_kwargs['help'] = '2 help' parser2 = subparsers.add_parser('2', **parser2_kwargs) - parser2.add_argument('-y', choices='123', help='y help') + parser2.add_argument('-y', choices=['1', '2', '3'], help='y help') parser2.add_argument('z', type=complex, nargs='*', help='z help') # add third sub-parser - parser3_kwargs = dict(description='3 description') + parser3_kwargs = dict(description='3 description', + usage='PROG --foo bar 3 t ...') if subparser_help: parser3_kwargs['help'] = '3 help' parser3 = subparsers.add_parser('3', **parser3_kwargs) @@ -2233,6 +2479,47 @@ def test_parse_args_failures(self): args = args_str.split() self.assertArgumentParserError(self.parser.parse_args, args) + def test_parse_args_failures_details(self): + for args_str, usage_str, error_str in [ + ('', + 'usage: PROG [-h] [--foo] bar {1,2,3} ...', + 'PROG: error: the following arguments are required: bar'), + ('0.5 1 -y', + 'usage: PROG bar 1 [-h] [-w W] {a,b,c}', + 'PROG bar 1: error: the following arguments are required: x'), + ('0.5 3', + 'usage: PROG --foo bar 3 t ...', + 'PROG bar 3: error: the following arguments are required: t'), + ]: + with self.subTest(args_str): + args = args_str.split() + with self.assertRaises(ArgumentParserError) as cm: + self.parser.parse_args(args) + self.assertEqual(cm.exception.args[0], 'SystemExit') + self.assertEqual(cm.exception.args[2], f'{usage_str}\n{error_str}\n') + + def test_parse_args_failures_details_custom_usage(self): + parser = self._get_parser(usage='PROG [--foo] bar 1 [-w W] {a,b,c}\n' + ' PROG --foo bar 3 t ...') + for args_str, usage_str, error_str in [ + ('', + 'usage: PROG [--foo] bar 1 [-w W] {a,b,c}\n' + ' PROG --foo bar 3 t ...', + 'PROG: error: the following arguments are required: bar'), + ('0.5 1 -y', + 'usage: PROG bar 1 [-h] [-w W] {a,b,c}', + 'PROG bar 1: error: the following arguments are required: x'), + ('0.5 3', + 'usage: PROG --foo bar 3 t ...', + 'PROG bar 3: error: the following arguments are required: t'), + ]: + with self.subTest(args_str): + args = args_str.split() + with self.assertRaises(ArgumentParserError) as cm: + parser.parse_args(args) + self.assertEqual(cm.exception.args[0], 'SystemExit') + self.assertEqual(cm.exception.args[2], f'{usage_str}\n{error_str}\n') + def test_parse_args(self): # check some non-failure cases: self.assertEqual( @@ -2274,6 +2561,40 @@ def test_parse_known_args(self): (NS(foo=False, bar=0.5, w=7, x='b'), ['-W', '-X', 'Y', 'Z']), ) + def test_parse_known_args_to_class_namespace(self): + class C: + pass + self.assertEqual( + self.parser.parse_known_args('0.5 1 b -w 7 -p'.split(), namespace=C), + (C, ['-p']), + ) + self.assertIs(C.foo, False) + self.assertEqual(C.bar, 0.5) + self.assertEqual(C.w, 7) + self.assertEqual(C.x, 'b') + + def test_abbreviation(self): + parser = ErrorRaisingArgumentParser() + parser.add_argument('--foodle') + parser.add_argument('--foonly') + subparsers = parser.add_subparsers() + parser1 = subparsers.add_parser('bar') + parser1.add_argument('--fo') + parser1.add_argument('--foonew') + + self.assertEqual(parser.parse_args(['--food', 'baz', 'bar']), + NS(foodle='baz', foonly=None, fo=None, foonew=None)) + self.assertEqual(parser.parse_args(['--foon', 'baz', 'bar']), + NS(foodle=None, foonly='baz', fo=None, foonew=None)) + self.assertArgumentParserError(parser.parse_args, ['--fo', 'baz', 'bar']) + self.assertEqual(parser.parse_args(['bar', '--fo', 'baz']), + NS(foodle=None, foonly=None, fo='baz', foonew=None)) + self.assertEqual(parser.parse_args(['bar', '--foo', 'baz']), + NS(foodle=None, foonly=None, fo=None, foonew='baz')) + self.assertEqual(parser.parse_args(['bar', '--foon', 'baz']), + NS(foodle=None, foonly=None, fo=None, foonew='baz')) + self.assertArgumentParserError(parser.parse_args, ['bar', '--food', 'baz']) + def test_parse_known_args_with_single_dash_option(self): parser = ErrorRaisingArgumentParser() parser.add_argument('-k', '--known', action='count', default=0) @@ -2352,18 +2673,6 @@ def test_required_subparsers_no_destination_error(self): 'error: the following arguments are required: {foo,bar}\n$' ) - def test_wrong_argument_subparsers_no_destination_error(self): - parser = ErrorRaisingArgumentParser() - subparsers = parser.add_subparsers(required=True) - subparsers.add_parser('foo') - subparsers.add_parser('bar') - with self.assertRaises(ArgumentParserError) as excinfo: - parser.parse_args(('baz',)) - self.assertRegex( - excinfo.exception.stderr, - r"error: argument {foo,bar}: invalid choice: 'baz' \(choose from 'foo', 'bar'\)\n$" - ) - def test_optional_subparsers(self): parser = ErrorRaisingArgumentParser() subparsers = parser.add_subparsers(dest='command', required=False) @@ -2499,6 +2808,29 @@ def test_parser_command_help(self): --foo foo help ''')) + def assert_bad_help(self, context_type, func, *args, **kwargs): + with self.assertRaisesRegex(ValueError, 'badly formed help string') as cm: + func(*args, **kwargs) + self.assertIsInstance(cm.exception.__context__, context_type) + + def test_invalid_subparsers_help(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assert_bad_help(ValueError, parser.add_subparsers, help='%Y-%m-%d') + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assert_bad_help(KeyError, parser.add_subparsers, help='%(spam)s') + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assert_bad_help(TypeError, parser.add_subparsers, help='%(prog)d') + + def test_invalid_subparser_help(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + subparsers = parser.add_subparsers() + self.assert_bad_help(ValueError, subparsers.add_parser, '1', + help='%Y-%m-%d') + self.assert_bad_help(KeyError, subparsers.add_parser, '1', + help='%(spam)s') + self.assert_bad_help(TypeError, subparsers.add_parser, '1', + help='%(prog)d') + def test_subparser_title_help(self): parser = ErrorRaisingArgumentParser(prog='PROG', description='main description') @@ -2640,6 +2972,38 @@ def test_interleaved_groups(self): result = parser.parse_args('1 2 3 4'.split()) self.assertEqual(expected, result) +class TestGroupConstructor(TestCase): + def test_group_prefix_chars(self): + parser = ErrorRaisingArgumentParser() + msg = ( + "The use of the undocumented 'prefix_chars' parameter in " + "ArgumentParser.add_argument_group() is deprecated." + ) + with self.assertWarns(DeprecationWarning) as cm: + parser.add_argument_group(prefix_chars='-+') + self.assertEqual(msg, str(cm.warning)) + self.assertEqual(cm.filename, __file__) + + def test_group_prefix_chars_default(self): + # "default" isn't quite the right word here, but it's the same as + # the parser's default prefix so it's a good test + parser = ErrorRaisingArgumentParser() + msg = ( + "The use of the undocumented 'prefix_chars' parameter in " + "ArgumentParser.add_argument_group() is deprecated." + ) + with self.assertWarns(DeprecationWarning) as cm: + parser.add_argument_group(prefix_chars='-') + self.assertEqual(msg, str(cm.warning)) + self.assertEqual(cm.filename, __file__) + + def test_nested_argument_group(self): + parser = argparse.ArgumentParser() + g = parser.add_argument_group() + self.assertRaisesRegex(ValueError, + 'argument groups cannot be nested', + g.add_argument_group) + # =================== # Parent parser tests # =================== @@ -2676,8 +3040,6 @@ def setUp(self): group.add_argument('-a', action='store_true') group.add_argument('-b', action='store_true') - self.main_program = os.path.basename(sys.argv[0]) - def test_single_parent(self): parser = ErrorRaisingArgumentParser(parents=[self.wxyz_parent]) self.assertEqual(parser.parse_args('-y 1 2 --w 3'.split()), @@ -2688,7 +3050,7 @@ def test_single_parent_mutex(self): parser = ErrorRaisingArgumentParser(parents=[self.ab_mutex_parent]) self._test_mutex_ab(parser.parse_args) - def test_single_granparent_mutex(self): + def test_single_grandparent_mutex(self): parents = [self.ab_mutex_parent] parser = ErrorRaisingArgumentParser(add_help=False, parents=parents) parser = ErrorRaisingArgumentParser(parents=[parser]) @@ -2767,11 +3129,10 @@ def test_subparser_parents_mutex(self): def test_parent_help(self): parents = [self.abcd_parent, self.wxyz_parent] - parser = ErrorRaisingArgumentParser(parents=parents) + parser = ErrorRaisingArgumentParser(prog='PROG', parents=parents) parser_help = parser.format_help() - progname = self.main_program self.assertEqual(parser_help, textwrap.dedent('''\ - usage: {}{}[-h] [-b B] [--d D] [--w W] [-y Y] a z + usage: PROG [-h] [-b B] [--d D] [--w W] [-y Y] a z positional arguments: a @@ -2787,7 +3148,7 @@ def test_parent_help(self): x: -y Y - '''.format(progname, ' ' if progname else '' ))) + ''')) def test_groups_parents(self): parent = ErrorRaisingArgumentParser(add_help=False) @@ -2797,15 +3158,14 @@ def test_groups_parents(self): m = parent.add_mutually_exclusive_group() m.add_argument('-y') m.add_argument('-z') - parser = ErrorRaisingArgumentParser(parents=[parent]) + parser = ErrorRaisingArgumentParser(prog='PROG', parents=[parent]) self.assertRaises(ArgumentParserError, parser.parse_args, ['-y', 'Y', '-z', 'Z']) parser_help = parser.format_help() - progname = self.main_program self.assertEqual(parser_help, textwrap.dedent('''\ - usage: {}{}[-h] [-w W] [-x X] [-y Y | -z Z] + usage: PROG [-h] [-w W] [-x X] [-y Y | -z Z] options: -h, --help show this help message and exit @@ -2817,11 +3177,40 @@ def test_groups_parents(self): -w W -x X - '''.format(progname, ' ' if progname else '' ))) + ''')) def test_wrong_type_parents(self): self.assertRaises(TypeError, ErrorRaisingArgumentParser, parents=[1]) + def test_mutex_groups_parents(self): + parent = ErrorRaisingArgumentParser(add_help=False) + g = parent.add_argument_group(title='g', description='gd') + g.add_argument('-w') + g.add_argument('-x') + m = g.add_mutually_exclusive_group() + m.add_argument('-y') + m.add_argument('-z') + parser = ErrorRaisingArgumentParser(prog='PROG', parents=[parent]) + + self.assertRaises(ArgumentParserError, parser.parse_args, + ['-y', 'Y', '-z', 'Z']) + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent('''\ + usage: PROG [-h] [-w W] [-x X] [-y Y | -z Z] + + options: + -h, --help show this help message and exit + + g: + gd + + -w W + -x X + -y Y + -z Z + ''')) + # ============================== # Mutually exclusive group tests # ============================== @@ -2864,6 +3253,29 @@ def test_help(self): ''' self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + def test_optional_order(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--foo') + group.add_argument('bar', nargs='?') + expected = '''\ + usage: PROG [-h] (--foo FOO | bar) + + positional arguments: + bar + + options: + -h, --help show this help message and exit + --foo FOO + ''' + self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('bar', nargs='?') + group.add_argument('--foo') + self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + def test_help_subparser_all_mutually_exclusive_group_members_suppressed(self): self.maxDiff = None parser = ErrorRaisingArgumentParser(prog='PROG') @@ -2892,32 +3304,44 @@ def test_empty_group(self): with self.assertRaises(ValueError): parser.parse_args(['-h']) + def test_nested_mutex_groups(self): + parser = argparse.ArgumentParser(prog='PROG') + g = parser.add_mutually_exclusive_group() + g.add_argument("--spam") + self.assertRaisesRegex(ValueError, + 'mutually exclusive groups cannot be nested', + g.add_mutually_exclusive_group) + class MEMixin(object): def test_failures_when_not_required(self): parse_args = self.get_parser(required=False).parse_args error = ArgumentParserError for args_string in self.failures: - self.assertRaises(error, parse_args, args_string.split()) + with self.subTest(args=args_string): + self.assertRaises(error, parse_args, args_string.split()) def test_failures_when_required(self): parse_args = self.get_parser(required=True).parse_args error = ArgumentParserError for args_string in self.failures + ['']: - self.assertRaises(error, parse_args, args_string.split()) + with self.subTest(args=args_string): + self.assertRaises(error, parse_args, args_string.split()) def test_successes_when_not_required(self): parse_args = self.get_parser(required=False).parse_args successes = self.successes + self.successes_when_not_required for args_string, expected_ns in successes: - actual_ns = parse_args(args_string.split()) - self.assertEqual(actual_ns, expected_ns) + with self.subTest(args=args_string): + actual_ns = parse_args(args_string.split()) + self.assertEqual(actual_ns, expected_ns) def test_successes_when_required(self): parse_args = self.get_parser(required=True).parse_args for args_string, expected_ns in self.successes: - actual_ns = parse_args(args_string.split()) - self.assertEqual(actual_ns, expected_ns) + with self.subTest(args=args_string): + actual_ns = parse_args(args_string.split()) + self.assertEqual(actual_ns, expected_ns) def test_usage_when_not_required(self): format_usage = self.get_parser(required=False).format_usage @@ -3000,12 +3424,12 @@ def get_parser(self, required=None): ] usage_when_not_required = '''\ - usage: PROG [-h] [--abcde ABCDE] [--fghij FGHIJ] - [--klmno KLMNO | --pqrst PQRST] + usage: PROG [-h] [--abcde ABCDE] [--fghij FGHIJ] [--klmno KLMNO | + --pqrst PQRST] ''' usage_when_required = '''\ - usage: PROG [-h] [--abcde ABCDE] [--fghij FGHIJ] - (--klmno KLMNO | --pqrst PQRST) + usage: PROG [-h] [--abcde ABCDE] [--fghij FGHIJ] (--klmno KLMNO | + --pqrst PQRST) ''' help = '''\ @@ -3094,7 +3518,7 @@ def get_parser(self, required): group = parser.add_mutually_exclusive_group(required=required) group.add_argument('--foo', action='store_true', help='FOO') group.add_argument('--spam', help='SPAM') - group.add_argument('badger', nargs='*', default='X', help='BADGER') + group.add_argument('badger', nargs='*', help='BADGER') return parser failures = [ @@ -3105,13 +3529,13 @@ def get_parser(self, required): '--foo X Y', ] successes = [ - ('--foo', NS(foo=True, spam=None, badger='X')), - ('--spam S', NS(foo=False, spam='S', badger='X')), + ('--foo', NS(foo=True, spam=None, badger=[])), + ('--spam S', NS(foo=False, spam='S', badger=[])), ('X', NS(foo=False, spam=None, badger=['X'])), ('X Y Z', NS(foo=False, spam=None, badger=['X', 'Y', 'Z'])), ] successes_when_not_required = [ - ('', NS(foo=False, spam=None, badger='X')), + ('', NS(foo=False, spam=None, badger=[])), ] usage_when_not_required = '''\ @@ -3255,54 +3679,110 @@ def get_parser(self, required): -c c help ''' -class TestMutuallyExclusiveNested(MEMixin, TestCase): - - # Nesting mutually exclusive groups is an undocumented feature - # that came about by accident through inheritance and has been - # the source of many bugs. It is deprecated and this test should - # eventually be removed along with it. - def get_parser(self, required): +class TestMutuallyExclusiveOptionalOptional(MEMixin, TestCase): + def get_parser(self, required=None): parser = ErrorRaisingArgumentParser(prog='PROG') group = parser.add_mutually_exclusive_group(required=required) - group.add_argument('-a') - group.add_argument('-b') - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - group2 = group.add_mutually_exclusive_group(required=required) - group2.add_argument('-c') - group2.add_argument('-d') - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - group3 = group2.add_mutually_exclusive_group(required=required) - group3.add_argument('-e') - group3.add_argument('-f') + group.add_argument('--foo') + group.add_argument('--bar', nargs='?') return parser + failures = [ + '--foo X --bar Y', + '--foo X --bar', + ] + successes = [ + ('--foo X', NS(foo='X', bar=None)), + ('--bar X', NS(foo=None, bar='X')), + ('--bar', NS(foo=None, bar=None)), + ] + successes_when_not_required = [ + ('', NS(foo=None, bar=None)), + ] + usage_when_required = '''\ + usage: PROG [-h] (--foo FOO | --bar [BAR]) + ''' usage_when_not_required = '''\ - usage: PROG [-h] [-a A | -b B | [-c C | -d D | [-e E | -f F]]] + usage: PROG [-h] [--foo FOO | --bar [BAR]] ''' - usage_when_required = '''\ - usage: PROG [-h] (-a A | -b B | (-c C | -d D | (-e E | -f F))) + help = '''\ + + options: + -h, --help show this help message and exit + --foo FOO + --bar [BAR] ''' + +class TestMutuallyExclusiveOptionalWithDefault(MEMixin, TestCase): + def get_parser(self, required=None): + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=required) + group.add_argument('--foo') + group.add_argument('--bar', type=bool, default=True) + return parser + + failures = [ + '--foo X --bar Y', + '--foo X --bar=', + ] + successes = [ + ('--foo X', NS(foo='X', bar=True)), + ('--bar X', NS(foo=None, bar=True)), + ('--bar=', NS(foo=None, bar=False)), + ] + successes_when_not_required = [ + ('', NS(foo=None, bar=True)), + ] + usage_when_required = '''\ + usage: PROG [-h] (--foo FOO | --bar BAR) + ''' + usage_when_not_required = '''\ + usage: PROG [-h] [--foo FOO | --bar BAR] + ''' help = '''\ options: -h, --help show this help message and exit - -a A - -b B - -c C - -d D - -e E - -f F + --foo FOO + --bar BAR ''' - # We are only interested in testing the behavior of format_usage(). - test_failures_when_not_required = None - test_failures_when_required = None - test_successes_when_not_required = None - test_successes_when_required = None + +class TestMutuallyExclusivePositionalWithDefault(MEMixin, TestCase): + def get_parser(self, required=None): + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=required) + group.add_argument('--foo') + group.add_argument('bar', nargs='?', type=bool, default=True) + return parser + + failures = [ + '--foo X Y', + ] + successes = [ + ('--foo X', NS(foo='X', bar=True)), + ('X', NS(foo=None, bar=True)), + ] + successes_when_not_required = [ + ('', NS(foo=None, bar=True)), + ] + usage_when_required = '''\ + usage: PROG [-h] (--foo FOO | bar) + ''' + usage_when_not_required = '''\ + usage: PROG [-h] [--foo FOO | bar] + ''' + help = '''\ + + positional arguments: + bar + + options: + -h, --help show this help message and exit + --foo FOO + ''' # ================================================= # Mutually exclusive group in parent parser tests @@ -4255,6 +4735,139 @@ class TestHelpUsagePositionalsOnlyWrap(HelpTestCase): version = '' +class TestHelpUsageMetavarsSpacesParentheses(HelpTestCase): + # https://github.com/python/cpython/issues/62549 + # https://github.com/python/cpython/issues/89743 + parser_signature = Sig(prog='PROG') + argument_signatures = [ + Sig('-n1', metavar='()', help='n1'), + Sig('-o1', metavar='(1, 2)', help='o1'), + Sig('-u1', metavar=' (uu) ', help='u1'), + Sig('-v1', metavar='( vv )', help='v1'), + Sig('-w1', metavar='(w)w', help='w1'), + Sig('-x1', metavar='x(x)', help='x1'), + Sig('-y1', metavar='yy)', help='y1'), + Sig('-z1', metavar='(zz', help='z1'), + Sig('-n2', metavar='[]', help='n2'), + Sig('-o2', metavar='[1, 2]', help='o2'), + Sig('-u2', metavar=' [uu] ', help='u2'), + Sig('-v2', metavar='[ vv ]', help='v2'), + Sig('-w2', metavar='[w]w', help='w2'), + Sig('-x2', metavar='x[x]', help='x2'), + Sig('-y2', metavar='yy]', help='y2'), + Sig('-z2', metavar='[zz', help='z2'), + ] + + usage = '''\ + usage: PROG [-h] [-n1 ()] [-o1 (1, 2)] [-u1 (uu) ] [-v1 ( vv )] [-w1 (w)w] + [-x1 x(x)] [-y1 yy)] [-z1 (zz] [-n2 []] [-o2 [1, 2]] [-u2 [uu] ] + [-v2 [ vv ]] [-w2 [w]w] [-x2 x[x]] [-y2 yy]] [-z2 [zz] + ''' + help = usage + '''\ + + options: + -h, --help show this help message and exit + -n1 () n1 + -o1 (1, 2) o1 + -u1 (uu) u1 + -v1 ( vv ) v1 + -w1 (w)w w1 + -x1 x(x) x1 + -y1 yy) y1 + -z1 (zz z1 + -n2 [] n2 + -o2 [1, 2] o2 + -u2 [uu] u2 + -v2 [ vv ] v2 + -w2 [w]w w2 + -x2 x[x] x2 + -y2 yy] y2 + -z2 [zz z2 + ''' + version = '' + + +class TestHelpUsageNoWhitespaceCrash(TestCase): + + def test_all_suppressed_mutex_followed_by_long_arg(self): + # https://github.com/python/cpython/issues/62090 + # https://github.com/python/cpython/issues/96310 + parser = argparse.ArgumentParser(prog='PROG') + mutex = parser.add_mutually_exclusive_group() + mutex.add_argument('--spam', help=argparse.SUPPRESS) + parser.add_argument('--eggs-eggs-eggs-eggs-eggs-eggs') + usage = textwrap.dedent('''\ + usage: PROG [-h] + [--eggs-eggs-eggs-eggs-eggs-eggs EGGS_EGGS_EGGS_EGGS_EGGS_EGGS] + ''') + self.assertEqual(parser.format_usage(), usage) + + def test_newline_in_metavar(self): + # https://github.com/python/cpython/issues/77048 + mapping = ['123456', '12345', '12345', '123'] + parser = argparse.ArgumentParser('11111111111111') + parser.add_argument('-v', '--verbose', + help='verbose mode', action='store_true') + parser.add_argument('targets', + help='installation targets', + nargs='+', + metavar='\n'.join(mapping)) + usage = textwrap.dedent('''\ + usage: 11111111111111 [-h] [-v] + 123456 + 12345 + 12345 + 123 [123456 + 12345 + 12345 + 123 ...] + ''') + self.assertEqual(parser.format_usage(), usage) + + def test_empty_metavar_required_arg(self): + # https://github.com/python/cpython/issues/82091 + parser = argparse.ArgumentParser(prog='PROG') + parser.add_argument('--nil', metavar='', required=True) + parser.add_argument('--a', metavar='A' * 70) + usage = ( + 'usage: PROG [-h] --nil \n' + ' [--a AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' + 'AAAAAAAAAAAAAAAAAAAAAAA]\n' + ) + self.assertEqual(parser.format_usage(), usage) + + def test_all_suppressed_mutex_with_optional_nargs(self): + # https://github.com/python/cpython/issues/98666 + parser = argparse.ArgumentParser(prog='PROG') + mutex = parser.add_mutually_exclusive_group() + mutex.add_argument( + '--param1', + nargs='?', const='default', metavar='NAME', help=argparse.SUPPRESS) + mutex.add_argument( + '--param2', + nargs='?', const='default', metavar='NAME', help=argparse.SUPPRESS) + usage = 'usage: PROG [-h]\n' + self.assertEqual(parser.format_usage(), usage) + + def test_long_mutex_groups_wrap(self): + parser = argparse.ArgumentParser(prog='PROG') + g = parser.add_mutually_exclusive_group() + g.add_argument('--op1', metavar='MET', nargs='?') + g.add_argument('--op2', metavar=('MET1', 'MET2'), nargs='*') + g.add_argument('--op3', nargs='*') + g.add_argument('--op4', metavar=('MET1', 'MET2'), nargs='+') + g.add_argument('--op5', nargs='+') + g.add_argument('--op6', nargs=3) + g.add_argument('--op7', metavar=('MET1', 'MET2', 'MET3'), nargs=3) + + usage = textwrap.dedent('''\ + usage: PROG [-h] [--op1 [MET] | --op2 [MET1 [MET2 ...]] | --op3 [OP3 ...] | + --op4 MET1 [MET2 ...] | --op5 OP5 [OP5 ...] | --op6 OP6 OP6 OP6 | + --op7 MET1 MET2 MET3] + ''') + self.assertEqual(parser.format_usage(), usage) + + class TestHelpVariableExpansion(HelpTestCase): """Test that variables are expanded properly in help messages""" @@ -4264,7 +4877,7 @@ class TestHelpVariableExpansion(HelpTestCase): help='x %(prog)s %(default)s %(type)s %%'), Sig('-y', action='store_const', default=42, const='XXX', help='y %(prog)s %(default)s %(const)s'), - Sig('--foo', choices='abc', + Sig('--foo', choices=['a', 'b', 'c'], help='foo %(prog)s %(default)s %(choices)s'), Sig('--bar', default='baz', choices=[1, 2], metavar='BBB', help='bar %(prog)s %(default)s %(dest)s'), @@ -4507,7 +5120,7 @@ class TestHelpNone(HelpTestCase): version = '' -class TestHelpTupleMetavar(HelpTestCase): +class TestHelpTupleMetavarOptional(HelpTestCase): """Test specifying metavar as a tuple""" parser_signature = Sig(prog='PROG') @@ -4534,6 +5147,34 @@ class TestHelpTupleMetavar(HelpTestCase): version = '' +class TestHelpTupleMetavarPositional(HelpTestCase): + """Test specifying metavar on a Positional as a tuple""" + + parser_signature = Sig(prog='PROG') + argument_signatures = [ + Sig('w', help='w help', nargs='+', metavar=('W1', 'W2')), + Sig('x', help='x help', nargs='*', metavar=('X1', 'X2')), + Sig('y', help='y help', nargs=3, metavar=('Y1', 'Y2', 'Y3')), + Sig('z', help='z help', nargs='?', metavar=('Z1',)), + ] + argument_group_signatures = [] + usage = '''\ + usage: PROG [-h] W1 [W2 ...] [X1 [X2 ...]] Y1 Y2 Y3 [Z1] + ''' + help = usage + '''\ + + positional arguments: + W1 W2 w help + X1 X2 x help + Y1 Y2 Y3 y help + Z1 z help + + options: + -h, --help show this help message and exit + ''' + version = '' + + class TestHelpRawText(HelpTestCase): """Test the RawTextHelpFormatter""" @@ -4827,6 +5468,46 @@ def custom_type(string): version = '' +class TestHelpUsageLongSubparserCommand(TestCase): + """Test that subparser commands are formatted correctly in help""" + maxDiff = None + + def test_parent_help(self): + def custom_formatter(prog): + return argparse.RawTextHelpFormatter(prog, max_help_position=50) + + parent_parser = argparse.ArgumentParser( + prog='PROG', + formatter_class=custom_formatter + ) + + cmd_subparsers = parent_parser.add_subparsers(title="commands", + metavar='CMD', + help='command to use') + cmd_subparsers.add_parser("add", + help="add something") + + cmd_subparsers.add_parser("remove", + help="remove something") + + cmd_subparsers.add_parser("a-very-long-command", + help="command that does something") + + parser_help = parent_parser.format_help() + self.assertEqual(parser_help, textwrap.dedent('''\ + usage: PROG [-h] CMD ... + + options: + -h, --help show this help message and exit + + commands: + CMD command to use + add add something + remove remove something + a-very-long-command command that does something + ''')) + + # ===================================== # Optional/Positional constructor tests # ===================================== @@ -4834,15 +5515,15 @@ def custom_type(string): class TestInvalidArgumentConstructors(TestCase): """Test a bunch of invalid Argument constructors""" - def assertTypeError(self, *args, **kwargs): + def assertTypeError(self, *args, errmsg=None, **kwargs): parser = argparse.ArgumentParser() - self.assertRaises(TypeError, parser.add_argument, - *args, **kwargs) + self.assertRaisesRegex(TypeError, errmsg, parser.add_argument, + *args, **kwargs) - def assertValueError(self, *args, **kwargs): + def assertValueError(self, *args, errmsg=None, **kwargs): parser = argparse.ArgumentParser() - self.assertRaises(ValueError, parser.add_argument, - *args, **kwargs) + self.assertRaisesRegex(ValueError, errmsg, parser.add_argument, + *args, **kwargs) def test_invalid_keyword_arguments(self): self.assertTypeError('-x', bar=None) @@ -4852,73 +5533,99 @@ def test_invalid_keyword_arguments(self): def test_missing_destination(self): self.assertTypeError() - for action in ['append', 'store']: - self.assertTypeError(action=action) + for action in ['store', 'append', 'extend']: + with self.subTest(action=action): + self.assertTypeError(action=action) def test_invalid_option_strings(self): - self.assertValueError('--') - self.assertValueError('---') + self.assertTypeError('-', errmsg='dest= is required') + self.assertTypeError('--', errmsg='dest= is required') + self.assertTypeError('---', errmsg='dest= is required') def test_invalid_prefix(self): - self.assertValueError('--foo', '+foo') + self.assertValueError('--foo', '+foo', + errmsg='must start with a character') def test_invalid_type(self): - self.assertValueError('--foo', type='int') - self.assertValueError('--foo', type=(int, float)) + self.assertTypeError('--foo', type='int', + errmsg="'int' is not callable") + self.assertTypeError('--foo', type=(int, float), + errmsg='is not callable') def test_invalid_action(self): - self.assertValueError('-x', action='foo') - self.assertValueError('foo', action='baz') - self.assertValueError('--foo', action=('store', 'append')) - parser = argparse.ArgumentParser() - with self.assertRaises(ValueError) as cm: - parser.add_argument("--foo", action="store-true") - self.assertIn('unknown action', str(cm.exception)) + self.assertValueError('-x', action='foo', + errmsg='unknown action') + self.assertValueError('foo', action='baz', + errmsg='unknown action') + self.assertValueError('--foo', action=('store', 'append'), + errmsg='unknown action') + self.assertValueError('--foo', action="store-true", + errmsg='unknown action') + + def test_invalid_help(self): + self.assertValueError('--foo', help='%Y-%m-%d', + errmsg='badly formed help string') + self.assertValueError('--foo', help='%(spam)s', + errmsg='badly formed help string') + self.assertValueError('--foo', help='%(prog)d', + errmsg='badly formed help string') def test_multiple_dest(self): parser = argparse.ArgumentParser() parser.add_argument(dest='foo') - with self.assertRaises(ValueError) as cm: + with self.assertRaises(TypeError) as cm: parser.add_argument('bar', dest='baz') - self.assertIn('dest supplied twice for positional argument', + self.assertIn('dest supplied twice for positional argument,' + ' did you mean metavar?', str(cm.exception)) def test_no_argument_actions(self): for action in ['store_const', 'store_true', 'store_false', 'append_const', 'count']: - for attrs in [dict(type=int), dict(nargs='+'), - dict(choices='ab')]: - self.assertTypeError('-x', action=action, **attrs) + with self.subTest(action=action): + for attrs in [dict(type=int), dict(nargs='+'), + dict(choices=['a', 'b'])]: + with self.subTest(attrs=attrs): + self.assertTypeError('-x', action=action, **attrs) + self.assertTypeError('x', action=action, **attrs) + self.assertValueError('x', action=action, + errmsg=f"action '{action}' is not valid for positional arguments") + self.assertTypeError('-x', action=action, nargs=0) + self.assertValueError('x', action=action, nargs=0, + errmsg='nargs for positionals must be != 0') def test_no_argument_no_const_actions(self): # options with zero arguments for action in ['store_true', 'store_false', 'count']: + with self.subTest(action=action): + # const is always disallowed + self.assertTypeError('-x', const='foo', action=action) - # const is always disallowed - self.assertTypeError('-x', const='foo', action=action) - - # nargs is always disallowed - self.assertTypeError('-x', nargs='*', action=action) + # nargs is always disallowed + self.assertTypeError('-x', nargs='*', action=action) def test_more_than_one_argument_actions(self): - for action in ['store', 'append']: - - # nargs=0 is disallowed - self.assertValueError('-x', nargs=0, action=action) - self.assertValueError('spam', nargs=0, action=action) - - # const is disallowed with non-optional arguments - for nargs in [1, '*', '+']: - self.assertValueError('-x', const='foo', - nargs=nargs, action=action) - self.assertValueError('spam', const='foo', - nargs=nargs, action=action) + for action in ['store', 'append', 'extend']: + with self.subTest(action=action): + # nargs=0 is disallowed + action_name = 'append' if action == 'extend' else action + self.assertValueError('-x', nargs=0, action=action, + errmsg=f'nargs for {action_name} actions must be != 0') + self.assertValueError('spam', nargs=0, action=action, + errmsg='nargs for positionals must be != 0') + + # const is disallowed with non-optional arguments + for nargs in [1, '*', '+']: + self.assertValueError('-x', const='foo', + nargs=nargs, action=action) + self.assertValueError('spam', const='foo', + nargs=nargs, action=action) def test_required_const_actions(self): for action in ['store_const', 'append_const']: - - # nargs is always disallowed - self.assertTypeError('-x', nargs='+', action=action) + with self.subTest(action=action): + # nargs is always disallowed + self.assertTypeError('-x', nargs='+', action=action) def test_parsers_action_missing_params(self): self.assertTypeError('command', action='parsers') @@ -5037,14 +5744,18 @@ def test_subparser_conflict(self): parser = argparse.ArgumentParser() sp = parser.add_subparsers() sp.add_parser('fullname', aliases=['alias']) - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'fullname') - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'alias') - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'other', aliases=['fullname']) - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'other', aliases=['alias']) + self.assertRaisesRegex(ValueError, + 'conflicting subparser: fullname', + sp.add_parser, 'fullname') + self.assertRaisesRegex(ValueError, + 'conflicting subparser: alias', + sp.add_parser, 'alias') + self.assertRaisesRegex(ValueError, + 'conflicting subparser alias: fullname', + sp.add_parser, 'other', aliases=['fullname']) + self.assertRaisesRegex(ValueError, + 'conflicting subparser alias: alias', + sp.add_parser, 'other', aliases=['alias']) # ============================= @@ -5589,8 +6300,34 @@ def test_zero_or_more_optional(self): args = parser.parse_args([]) self.assertEqual(NS(x=[]), args) - def test_double_dash(self): - parser = argparse.ArgumentParser() + +class TestDoubleDash(TestCase): + def test_single_argument_option(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('-f', '--foo') + parser.add_argument('bar', nargs='*') + + args = parser.parse_args(['--foo=--']) + self.assertEqual(NS(foo='--', bar=[]), args) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument -f/--foo: expected one argument', + parser.parse_args, ['--foo', '--']) + args = parser.parse_args(['-f--']) + self.assertEqual(NS(foo='--', bar=[]), args) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument -f/--foo: expected one argument', + parser.parse_args, ['-f', '--']) + args = parser.parse_args(['--foo', 'a', '--', 'b', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', 'b', '--foo', 'c']) + self.assertEqual(NS(foo='c', bar=['a', 'b']), args) + args = parser.parse_args(['a', '--', 'b', '--foo', 'c']) + self.assertEqual(NS(foo=None, bar=['a', 'b', '--foo', 'c']), args) + args = parser.parse_args(['a', '--', 'b', '--', 'c', '--foo', 'd']) + self.assertEqual(NS(foo=None, bar=['a', 'b', '--', 'c', '--foo', 'd']), args) + + def test_multiple_argument_option(self): + parser = argparse.ArgumentParser(exit_on_error=False) parser.add_argument('-f', '--foo', nargs='*') parser.add_argument('bar', nargs='*') @@ -5604,6 +6341,91 @@ def test_double_dash(self): self.assertEqual(NS(foo=[], bar=[]), args) args = parser.parse_args(['--foo', 'a', 'b', '--', 'c', 'd']) self.assertEqual(NS(foo=['a', 'b'], bar=['c', 'd']), args) + args = parser.parse_args(['a', 'b', '--foo', 'c', 'd']) + self.assertEqual(NS(foo=['c', 'd'], bar=['a', 'b']), args) + args = parser.parse_args(['a', '--', 'b', '--foo', 'c', 'd']) + self.assertEqual(NS(foo=None, bar=['a', 'b', '--foo', 'c', 'd']), args) + args, argv = parser.parse_known_args(['a', 'b', '--foo', 'c', '--', 'd']) + self.assertEqual(NS(foo=['c'], bar=['a', 'b']), args) + self.assertEqual(argv, ['--', 'd']) + + def test_multiple_double_dashes(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('foo') + parser.add_argument('bar', nargs='*') + + args = parser.parse_args(['--', 'a', 'b', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', '--', 'b', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', 'b', '--', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', '--', 'b', '--', 'c']) + self.assertEqual(NS(foo='a', bar=['b', '--', 'c']), args) + args = parser.parse_args(['--', '--', 'a', '--', 'b', 'c']) + self.assertEqual(NS(foo='--', bar=['a', '--', 'b', 'c']), args) + + def test_remainder(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('foo') + parser.add_argument('bar', nargs='...') + + args = parser.parse_args(['--', 'a', 'b', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', '--', 'b', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', 'b', '--', 'c']) + self.assertEqual(NS(foo='a', bar=['b', '--', 'c']), args) + args = parser.parse_args(['a', '--', 'b', '--', 'c']) + self.assertEqual(NS(foo='a', bar=['b', '--', 'c']), args) + + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('--foo') + parser.add_argument('bar', nargs='...') + args = parser.parse_args(['--foo', 'a', '--', 'b', '--', 'c']) + self.assertEqual(NS(foo='a', bar=['--', 'b', '--', 'c']), args) + + def test_subparser(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('foo') + subparsers = parser.add_subparsers() + parser1 = subparsers.add_parser('run') + parser1.add_argument('-f') + parser1.add_argument('bar', nargs='*') + + args = parser.parse_args(['x', 'run', 'a', 'b', '-f', 'c']) + self.assertEqual(NS(foo='x', f='c', bar=['a', 'b']), args) + args = parser.parse_args(['x', 'run', 'a', 'b', '--', '-f', 'c']) + self.assertEqual(NS(foo='x', f=None, bar=['a', 'b', '-f', 'c']), args) + args = parser.parse_args(['x', 'run', 'a', '--', 'b', '-f', 'c']) + self.assertEqual(NS(foo='x', f=None, bar=['a', 'b', '-f', 'c']), args) + args = parser.parse_args(['x', 'run', '--', 'a', 'b', '-f', 'c']) + self.assertEqual(NS(foo='x', f=None, bar=['a', 'b', '-f', 'c']), args) + args = parser.parse_args(['x', '--', 'run', 'a', 'b', '-f', 'c']) + self.assertEqual(NS(foo='x', f='c', bar=['a', 'b']), args) + args = parser.parse_args(['--', 'x', 'run', 'a', 'b', '-f', 'c']) + self.assertEqual(NS(foo='x', f='c', bar=['a', 'b']), args) + args = parser.parse_args(['x', 'run', '--', 'a', '--', 'b']) + self.assertEqual(NS(foo='x', f=None, bar=['a', '--', 'b']), args) + args = parser.parse_args(['x', '--', 'run', '--', 'a', '--', 'b']) + self.assertEqual(NS(foo='x', f=None, bar=['a', '--', 'b']), args) + self.assertRaisesRegex(argparse.ArgumentError, + "invalid choice: '--'", + parser.parse_args, ['--', 'x', '--', 'run', 'a', 'b']) + + def test_subparser_after_multiple_argument_option(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('--foo', nargs='*') + subparsers = parser.add_subparsers() + parser1 = subparsers.add_parser('run') + parser1.add_argument('-f') + parser1.add_argument('bar', nargs='*') + + args = parser.parse_args(['--foo', 'x', 'y', '--', 'run', 'a', 'b', '-f', 'c']) + self.assertEqual(NS(foo=['x', 'y'], f='c', bar=['a', 'b']), args) + self.assertRaisesRegex(argparse.ArgumentError, + "invalid choice: '--'", + parser.parse_args, ['--foo', 'x', '--', '--', 'run', 'a', 'b']) # =========================== @@ -5625,14 +6447,25 @@ def test_basic(self): args, extras = parser.parse_known_args(argv) # cannot parse the '1,2,3' - self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[]), args) - self.assertEqual(["1", "2", "3"], extras) + self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1]), args) + self.assertEqual(["2", "3"], extras) + args, extras = parser.parse_known_intermixed_args(argv) + self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1, 2, 3]), args) + self.assertEqual([], extras) + # unknown optionals go into extras + argv = 'cmd --foo x --error 1 2 --bar y 3'.split() + args, extras = parser.parse_known_intermixed_args(argv) + self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1, 2, 3]), args) + self.assertEqual(['--error'], extras) argv = 'cmd --foo x 1 --error 2 --bar y 3'.split() args, extras = parser.parse_known_intermixed_args(argv) - # unknown optionals go into extras - self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1]), args) - self.assertEqual(['--error', '2', '3'], extras) + self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1, 2, 3]), args) + self.assertEqual(['--error'], extras) + argv = 'cmd --foo x 1 2 --error --bar y 3'.split() + args, extras = parser.parse_known_intermixed_args(argv) + self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1, 2, 3]), args) + self.assertEqual(['--error'], extras) # restores attributes that were temporarily changed self.assertIsNone(parser.usage) @@ -5651,38 +6484,48 @@ def test_remainder(self): parser.parse_intermixed_args(argv) self.assertRegex(str(cm.exception), r'\.\.\.') - def test_exclusive(self): - # mutually exclusive group; intermixed works fine - parser = ErrorRaisingArgumentParser(prog='PROG') + def test_required_exclusive(self): + # required mutually exclusive group; intermixed works fine + parser = argparse.ArgumentParser(prog='PROG', exit_on_error=False) group = parser.add_mutually_exclusive_group(required=True) group.add_argument('--foo', action='store_true', help='FOO') group.add_argument('--spam', help='SPAM') parser.add_argument('badger', nargs='*', default='X', help='BADGER') + args = parser.parse_intermixed_args('--foo 1 2'.split()) + self.assertEqual(NS(badger=['1', '2'], foo=True, spam=None), args) args = parser.parse_intermixed_args('1 --foo 2'.split()) self.assertEqual(NS(badger=['1', '2'], foo=True, spam=None), args) - self.assertRaises(ArgumentParserError, parser.parse_intermixed_args, '1 2'.split()) + self.assertRaisesRegex(argparse.ArgumentError, + 'one of the arguments --foo --spam is required', + parser.parse_intermixed_args, '1 2'.split()) self.assertEqual(group.required, True) - def test_exclusive_incompatible(self): - # mutually exclusive group including positional - fail - parser = ErrorRaisingArgumentParser(prog='PROG') + def test_required_exclusive_with_positional(self): + # required mutually exclusive group with positional argument + parser = argparse.ArgumentParser(prog='PROG', exit_on_error=False) group = parser.add_mutually_exclusive_group(required=True) group.add_argument('--foo', action='store_true', help='FOO') group.add_argument('--spam', help='SPAM') group.add_argument('badger', nargs='*', default='X', help='BADGER') - self.assertRaises(TypeError, parser.parse_intermixed_args, []) + args = parser.parse_intermixed_args(['--foo']) + self.assertEqual(NS(foo=True, spam=None, badger='X'), args) + args = parser.parse_intermixed_args(['a', 'b']) + self.assertEqual(NS(foo=False, spam=None, badger=['a', 'b']), args) + self.assertRaisesRegex(argparse.ArgumentError, + 'one of the arguments --foo --spam badger is required', + parser.parse_intermixed_args, []) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument badger: not allowed with argument --foo', + parser.parse_intermixed_args, ['--foo', 'a', 'b']) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument badger: not allowed with argument --foo', + parser.parse_intermixed_args, ['a', '--foo', 'b']) self.assertEqual(group.required, True) def test_invalid_args(self): parser = ErrorRaisingArgumentParser(prog='PROG') self.assertRaises(ArgumentParserError, parser.parse_intermixed_args, ['a']) - parser = ErrorRaisingArgumentParser(prog='PROG') - parser.add_argument('--foo', nargs="*") - parser.add_argument('foo') - with captured_stderr() as stderr: - parser.parse_intermixed_args(['hello', '--foo']) - self.assertIn("UserWarning", stderr.getvalue()) class TestIntermixedMessageContentError(TestCase): # case where Intermixed gives different error message @@ -5701,7 +6544,7 @@ def test_missing_argument_name_in_message(self): with self.assertRaises(ArgumentParserError) as cm: parser.parse_intermixed_args([]) msg = str(cm.exception) - self.assertNotRegex(msg, 'req_pos') + self.assertRegex(msg, 'req_pos') self.assertRegex(msg, 'req_opt') # ========================== @@ -5951,7 +6794,8 @@ def test_help_with_metavar(self): class TestExitOnError(TestCase): def setUp(self): - self.parser = argparse.ArgumentParser(exit_on_error=False) + self.parser = argparse.ArgumentParser(exit_on_error=False, + fromfile_prefix_chars='@') self.parser.add_argument('--integers', metavar='N', type=int) def test_exit_on_error_with_good_args(self): @@ -5962,6 +6806,245 @@ def test_exit_on_error_with_bad_args(self): with self.assertRaises(argparse.ArgumentError): self.parser.parse_args('--integers a'.split()) + def test_unrecognized_args(self): + self.assertRaisesRegex(argparse.ArgumentError, + 'unrecognized arguments: --foo bar', + self.parser.parse_args, '--foo bar'.split()) + + def test_unrecognized_intermixed_args(self): + self.assertRaisesRegex(argparse.ArgumentError, + 'unrecognized arguments: --foo bar', + self.parser.parse_intermixed_args, '--foo bar'.split()) + + def test_required_args(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, baz$', + self.parser.parse_args, []) + + def test_required_args_with_metavar(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', metavar='BaZ') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, BaZ$', + self.parser.parse_args, []) + + def test_required_args_n(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs=3) + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, baz$', + self.parser.parse_args, []) + + def test_required_args_n_with_metavar(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs=3, metavar=('B', 'A', 'Z')) + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, B, A, Z$', + self.parser.parse_args, []) + + def test_required_args_optional(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs='?') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar$', + self.parser.parse_args, []) + + def test_required_args_zero_or_more(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs='*') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar$', + self.parser.parse_args, []) + + def test_required_args_one_or_more(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs='+') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, baz$', + self.parser.parse_args, []) + + def test_required_args_one_or_more_with_metavar(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs='+', metavar=('BaZ1', 'BaZ2')) + self.assertRaisesRegex(argparse.ArgumentError, + r'the following arguments are required: bar, BaZ1\[, BaZ2]$', + self.parser.parse_args, []) + + def test_required_args_remainder(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs='...') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar$', + self.parser.parse_args, []) + + def test_required_mutually_exclusive_args(self): + group = self.parser.add_mutually_exclusive_group(required=True) + group.add_argument('--bar') + group.add_argument('--baz') + self.assertRaisesRegex(argparse.ArgumentError, + 'one of the arguments --bar --baz is required', + self.parser.parse_args, []) + + def test_conflicting_mutually_exclusive_args_optional_with_metavar(self): + group = self.parser.add_mutually_exclusive_group() + group.add_argument('--bar') + group.add_argument('baz', nargs='?', metavar='BaZ') + self.assertRaisesRegex(argparse.ArgumentError, + 'argument BaZ: not allowed with argument --bar$', + self.parser.parse_args, ['--bar', 'a', 'b']) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument --bar: not allowed with argument BaZ$', + self.parser.parse_args, ['a', '--bar', 'b']) + + def test_conflicting_mutually_exclusive_args_zero_or_more_with_metavar1(self): + group = self.parser.add_mutually_exclusive_group() + group.add_argument('--bar') + group.add_argument('baz', nargs='*', metavar=('BAZ1',)) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument BAZ1: not allowed with argument --bar$', + self.parser.parse_args, ['--bar', 'a', 'b']) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument --bar: not allowed with argument BAZ1$', + self.parser.parse_args, ['a', '--bar', 'b']) + + def test_conflicting_mutually_exclusive_args_zero_or_more_with_metavar2(self): + group = self.parser.add_mutually_exclusive_group() + group.add_argument('--bar') + group.add_argument('baz', nargs='*', metavar=('BAZ1', 'BAZ2')) + self.assertRaisesRegex(argparse.ArgumentError, + r'argument BAZ1\[, BAZ2]: not allowed with argument --bar$', + self.parser.parse_args, ['--bar', 'a', 'b']) + self.assertRaisesRegex(argparse.ArgumentError, + r'argument --bar: not allowed with argument BAZ1\[, BAZ2]$', + self.parser.parse_args, ['a', '--bar', 'b']) + + def test_ambiguous_option(self): + self.parser.add_argument('--foobaz') + self.parser.add_argument('--fooble', action='store_true') + self.parser.add_argument('--foogle') + self.assertRaisesRegex(argparse.ArgumentError, + "ambiguous option: --foob could match --foobaz, --fooble", + self.parser.parse_args, ['--foob']) + self.assertRaisesRegex(argparse.ArgumentError, + "ambiguous option: --foob=1 could match --foobaz, --fooble$", + self.parser.parse_args, ['--foob=1']) + self.assertRaisesRegex(argparse.ArgumentError, + "ambiguous option: --foob could match --foobaz, --fooble$", + self.parser.parse_args, ['--foob', '1', '--foogle', '2']) + self.assertRaisesRegex(argparse.ArgumentError, + "ambiguous option: --foob=1 could match --foobaz, --fooble$", + self.parser.parse_args, ['--foob=1', '--foogle', '2']) + + def test_os_error(self): + self.parser.add_argument('file') + self.assertRaisesRegex(argparse.ArgumentError, + "No such file or directory: 'no-such-file'", + self.parser.parse_args, ['@no-such-file']) + + +class TestProgName(TestCase): + source = textwrap.dedent('''\ + import argparse + parser = argparse.ArgumentParser() + parser.parse_args() + ''') + + def setUp(self): + self.dirname = 'package' + os_helper.FS_NONASCII + self.addCleanup(os_helper.rmtree, self.dirname) + os.mkdir(self.dirname) + + def make_script(self, dirname, basename, *, compiled=False): + script_name = script_helper.make_script(dirname, basename, self.source) + if not compiled: + return script_name + py_compile.compile(script_name, doraise=True) + os.remove(script_name) + pyc_file = import_helper.make_legacy_pyc(script_name) + return pyc_file + + def make_zip_script(self, script_name, name_in_zip=None): + zip_name, _ = script_helper.make_zip_script(self.dirname, 'test_zip', + script_name, name_in_zip) + return zip_name + + def check_usage(self, expected, *args, **kwargs): + res = script_helper.assert_python_ok('-Xutf8', *args, '-h', **kwargs) + self.assertEqual(res.out.splitlines()[0].decode(), + f'usage: {expected} [-h]') + + def test_script(self, compiled=False): + basename = os_helper.TESTFN + script_name = self.make_script(self.dirname, basename, compiled=compiled) + self.check_usage(os.path.basename(script_name), script_name, '-h') + + def test_script_compiled(self): + self.test_script(compiled=True) + + def test_directory(self, compiled=False): + dirname = os.path.join(self.dirname, os_helper.TESTFN) + os.mkdir(dirname) + self.make_script(dirname, '__main__', compiled=compiled) + self.check_usage(f'{py} {dirname}', dirname) + dirname2 = os.path.join(os.curdir, dirname) + self.check_usage(f'{py} {dirname2}', dirname2) + + def test_directory_compiled(self): + self.test_directory(compiled=True) + + def test_module(self, compiled=False): + basename = 'module' + os_helper.FS_NONASCII + modulename = f'{self.dirname}.{basename}' + self.make_script(self.dirname, basename, compiled=compiled) + self.check_usage(f'{py} -m {modulename}', + '-m', modulename, PYTHONPATH=os.curdir) + + def test_module_compiled(self): + self.test_module(compiled=True) + + def test_package(self, compiled=False): + basename = 'subpackage' + os_helper.FS_NONASCII + packagename = f'{self.dirname}.{basename}' + subdirname = os.path.join(self.dirname, basename) + os.mkdir(subdirname) + self.make_script(subdirname, '__main__', compiled=compiled) + self.check_usage(f'{py} -m {packagename}', + '-m', packagename, PYTHONPATH=os.curdir) + self.check_usage(f'{py} -m {packagename}', + '-m', packagename + '.__main__', PYTHONPATH=os.curdir) + + def test_package_compiled(self): + self.test_package(compiled=True) + + def test_zipfile(self, compiled=False): + script_name = self.make_script(self.dirname, '__main__', compiled=compiled) + zip_name = self.make_zip_script(script_name) + self.check_usage(f'{py} {zip_name}', zip_name) + + def test_zipfile_compiled(self): + self.test_zipfile(compiled=True) + + def test_directory_in_zipfile(self, compiled=False): + script_name = self.make_script(self.dirname, '__main__', compiled=compiled) + name_in_zip = 'package/subpackage/__main__' + ('.py', '.pyc')[compiled] + zip_name = self.make_zip_script(script_name, name_in_zip) + dirname = os.path.join(zip_name, 'package', 'subpackage') + self.check_usage(f'{py} {dirname}', dirname) + + def test_directory_in_zipfile_compiled(self): + self.test_directory_in_zipfile(compiled=True) + +# ================= +# Translation tests +# ================= + +class TestTranslations(TestTranslationsBase): + + def test_translations(self): + self.assertMsgidsEqual(argparse) + def tearDownModule(): # Remove global references to avoid looking like we have refleaks. @@ -5970,4 +7053,8 @@ def tearDownModule(): if __name__ == '__main__': + # To regenerate translation snapshots + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + update_translation_snapshots(argparse) + sys.exit(0) unittest.main() diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 95383be9659eb9..58ea89c4fac833 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1492,8 +1492,8 @@ def test_byteswap(self): if a.itemsize==1: self.assertEqual(a, b) else: - # On alphas treating the byte swapped bit patters as - # floats/doubles results in floating point exceptions + # On alphas treating the byte swapped bit patterns as + # floats/doubles results in floating-point exceptions # => compare the 8bit string values instead self.assertNotEqual(a.tobytes(), b.tobytes()) b.byteswap() @@ -1665,5 +1665,13 @@ def test_tolist(self, size): self.assertEqual(ls[:8], list(example[:8])) self.assertEqual(ls[-8:], list(example[-8:])) + def test_gh_128961(self): + a = array.array('i') + it = iter(a) + list(it) + it.__setstate__(0) + self.assertRaises(StopIteration, next, it) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ast/__init__.py b/Lib/test/test_ast/__init__.py new file mode 100644 index 00000000000000..9a89d27ba9f979 --- /dev/null +++ b/Lib/test/test_ast/__init__.py @@ -0,0 +1,7 @@ +import os + +from test import support + + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_ast/data/ast_repr.txt b/Lib/test/test_ast/data/ast_repr.txt new file mode 100644 index 00000000000000..3778b9e70a4605 --- /dev/null +++ b/Lib/test/test_ast/data/ast_repr.txt @@ -0,0 +1,209 @@ +Module(body=[Expr(value=Constant(value='module docstring', kind=None))], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=Constant(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=arg(...), defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[arg(...), ..., arg(...)], vararg=arg(...), kwonlyargs=[arg(...)], kw_defaults=[Constant(...)], kwarg=arg(...), defaults=[Constant(...), ..., Dict(...)]), body=[Expr(value=Constant(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[], keywords=[], body=[Expr(value=Constant(...))], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[Name(id='object', ctx=Load(...))], keywords=[], body=[Pass()], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[Name(id='A', ctx=Load(...)), Name(id='B', ctx=Load(...))], keywords=[], body=[Pass()], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Return(value=Constant(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Return(value=None)], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[Delete(targets=[Name(id='v', ctx=Del(...))])], type_ignores=[]) +Module(body=[Assign(targets=[Name(id='v', ctx=Store(...))], value=Constant(value=1, kind=None), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[Tuple(elts=[Name(...), Name(...)], ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[Tuple(elts=[Name(...), Name(...)], ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[List(elts=[Name(...), Name(...)], ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[Subscript(value=Name(...), slice=Name(...), ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[AnnAssign(target=Name(id='x', ctx=Store(...)), annotation=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), value=None, simple=1)], type_ignores=[]) +Module(body=[AnnAssign(target=Name(id='x', ctx=Store(...)), annotation=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), value=None, simple=1)], type_ignores=[]) +Module(body=[AnnAssign(target=Name(id='x', ctx=Store(...)), annotation=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), value=None, simple=1)], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Add(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Sub(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Mult(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=MatMult(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Div(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Mod(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Pow(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=LShift(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=RShift(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=BitOr(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=BitXor(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=BitAnd(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=FloorDiv(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[While(test=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[While(test=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[Pass()])], type_ignores=[]) +Module(body=[If(test=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[If(test=Name(...), body=[Pass(...)], orelse=[])])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[Pass()])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[If(test=Name(...), body=[Pass(...)], orelse=[Pass(...)])])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[If(test=Name(...), body=[Pass(...)], orelse=[If(...)])])], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=None)], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=None), withitem(context_expr=Name(...), optional_vars=None)], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=Name(...))], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=Name(...)), withitem(context_expr=Name(...), optional_vars=Name(...))], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=Name(...))], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=None), withitem(context_expr=Name(...), optional_vars=None)], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[Raise(exc=None, cause=None)], type_ignores=[]) +Module(body=[Raise(exc=Call(func=Name(...), args=[Constant(...)], keywords=[]), cause=None)], type_ignores=[]) +Module(body=[Raise(exc=Name(id='Exception', ctx=Load(...)), cause=None)], type_ignores=[]) +Module(body=[Raise(exc=Call(func=Name(...), args=[Constant(...)], keywords=[]), cause=Constant(value=None, kind=None))], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name=None, body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[], orelse=[], finalbody=[Pass()])], type_ignores=[]) +Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name=None, body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name=None, body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[]) +Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[]) +Module(body=[Assert(test=Name(id='v', ctx=Load(...)), msg=None)], type_ignores=[]) +Module(body=[Assert(test=Name(id='v', ctx=Load(...)), msg=Constant(value='message', kind=None))], type_ignores=[]) +Module(body=[Import(names=[alias(name='sys', asname=None)])], type_ignores=[]) +Module(body=[Import(names=[alias(name='foo', asname='bar')])], type_ignores=[]) +Module(body=[ImportFrom(module='sys', names=[alias(name='x', asname='y')], level=0)], type_ignores=[]) +Module(body=[ImportFrom(module='sys', names=[alias(name='v', asname=None)], level=0)], type_ignores=[]) +Module(body=[Global(names=['v'])], type_ignores=[]) +Module(body=[Expr(value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[Pass()], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Break()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Continue()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Tuple(elts=[Name(...), Name(...)], ctx=Store(...)), iter=Name(id='c', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Tuple(elts=[Name(...), Name(...)], ctx=Store(...)), iter=Name(id='c', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=List(elts=[Name(...), Name(...)], ctx=Store(...)), iter=Name(id='c', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=DictComp(key=Name(...), value=Name(...), generators=[comprehension(...), comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=DictComp(key=Name(...), value=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=Constant(...)), Expr(value=Await(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[AsyncFor(target=Name(...), iter=Name(...), body=[Expr(...)], orelse=[Expr(...)], type_comment=None)], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[AsyncWith(items=[withitem(...)], body=[Expr(...)], type_comment=None)], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[None, Constant(...)], values=[Dict(...), Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Set(elts=[Starred(...), Constant(...)]))], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=Yield(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=YieldFrom(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=ListComp(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Name(id='deco1', ctx=Load(...)), ..., Call(func=Name(...), args=[Constant(...)], keywords=[])], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Name(id='deco1', ctx=Load(...)), ..., Call(func=Name(...), args=[Constant(...)], keywords=[])], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[], keywords=[], body=[Pass()], decorator_list=[Name(id='deco1', ctx=Load(...)), ..., Call(func=Name(...), args=[Constant(...)], keywords=[])], type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Call(func=Name(...), args=[GeneratorExp(...)], keywords=[])], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Attribute(value=Attribute(...), attr='c', ctx=Load(...))], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[Expr(value=NamedExpr(target=Name(...), value=Constant(...)))], type_ignores=[]) +Module(body=[If(test=NamedExpr(target=Name(...), value=Call(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[While(test=NamedExpr(target=Name(...), value=Call(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...), ..., arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...), arg(...)], kw_defaults=[None, None], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...), arg(...)], kw_defaults=[None, None], kwarg=arg(...), defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...), arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[Constant(...), ..., Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[Constant(...)], kwarg=None, defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[None], kwarg=None, defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[Constant(...)], kwarg=arg(...), defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[None], kwarg=arg(...), defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[], value=Name(id='int', ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=None, default_value=None)], value=Name(id='int', ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=None, default_value=None), ..., ParamSpec(name='P', default_value=None)], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=Name(...), default_value=None), ..., ParamSpec(name='P', default_value=None)], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=Tuple(...), default_value=None), ..., ParamSpec(name='P', default_value=None)], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=Name(...), default_value=Constant(...)), ..., ParamSpec(name='P', default_value=Constant(...))], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=None, default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=None, default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=Name(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=Tuple(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=Name(...), default_value=Constant(...)), ..., ParamSpec(name='P', default_value=Constant(...))])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=None, default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=None, default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=Name(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=Tuple(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=Name(...), default_value=Constant(...)), ..., ParamSpec(name='P', default_value=Constant(...))])], type_ignores=[]) +Module(body=[Match(subject=Name(id='x', ctx=Load(...)), cases=[match_case(pattern=MatchValue(...), guard=None, body=[Pass(...)])])], type_ignores=[]) +Module(body=[Match(subject=Name(id='x', ctx=Load(...)), cases=[match_case(pattern=MatchValue(...), guard=None, body=[Pass(...)]), match_case(pattern=MatchAs(...), guard=None, body=[Pass(...)])])], type_ignores=[]) +Module(body=[Expr(value=Constant(value=None, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=True, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=False, kind=None))], type_ignores=[]) +Module(body=[Expr(value=BoolOp(op=And(...), values=[Name(...), Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=BoolOp(op=Or(...), values=[Name(...), Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Add(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Sub(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Mult(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Div(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=MatMult(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=FloorDiv(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Pow(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Mod(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=RShift(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=LShift(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=BitXor(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=BitOr(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=BitAnd(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=Not(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=UAdd(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=USub(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=Invert(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=Lambda(args=arguments(...), body=Constant(...)))], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[Constant(...)], values=[Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[], values=[]))], type_ignores=[]) +Module(body=[Expr(value=Set(elts=[Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[Constant(...)], values=[Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=List(elts=[Constant(...), Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Set(elts=[Constant(...), Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=DictComp(key=Name(...), value=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Constant(...), ops=[Lt(...), Lt(...)], comparators=[Constant(...), Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[Eq(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[LtE(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[GtE(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[NotEq(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[Is(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[IsNot(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[In(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[NotIn(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[Constant(...), ..., Starred(...)], keywords=[keyword(...), keyword(...)]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[Starred(...)], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[GeneratorExp(...)], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=10, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=1j, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value='string', kind=None))], type_ignores=[]) +Module(body=[Expr(value=Attribute(value=Name(...), attr='b', ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=Name(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Name(id='v', ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=List(elts=[Constant(...), ..., Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=List(elts=[], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[Constant(...), ..., Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[Constant(...), ..., Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Attribute(...), args=[Subscript(...)], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=IfExp(test=Name(...), body=Call(...), orelse=Call(...)))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[FormattedValue(...)]))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[FormattedValue(...)]))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[FormattedValue(...)]))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[Constant(...), ..., Constant(...)]))], type_ignores=[]) \ No newline at end of file diff --git a/Lib/test/test_ast/snippets.py b/Lib/test/test_ast/snippets.py new file mode 100644 index 00000000000000..28d32b2941f30d --- /dev/null +++ b/Lib/test/test_ast/snippets.py @@ -0,0 +1,601 @@ +import ast +import sys + +from test.test_ast.utils import to_tuple + + +# These tests are compiled through "exec" +# There should be at least one test per statement +exec_tests = [ + # Module docstring + "'module docstring'", + # FunctionDef + "def f(): pass", + # FunctionDef with docstring + "def f(): 'function docstring'", + # FunctionDef with arg + "def f(a): pass", + # FunctionDef with arg and default value + "def f(a=0): pass", + # FunctionDef with varargs + "def f(*args): pass", + # FunctionDef with varargs as TypeVarTuple + "def f(*args: *Ts): pass", + # FunctionDef with varargs as unpacked Tuple + "def f(*args: *tuple[int, ...]): pass", + # FunctionDef with varargs as unpacked Tuple *and* TypeVarTuple + "def f(*args: *tuple[int, *Ts]): pass", + # FunctionDef with kwargs + "def f(**kwargs): pass", + # FunctionDef with all kind of args and docstring + "def f(a, b=1, c=None, d=[], e={}, *args, f=42, **kwargs): 'doc for f()'", + # FunctionDef with type annotation on return involving unpacking + "def f() -> tuple[*Ts]: pass", + "def f() -> tuple[int, *Ts]: pass", + "def f() -> tuple[int, *tuple[int, ...]]: pass", + # ClassDef + "class C:pass", + # ClassDef with docstring + "class C: 'docstring for class C'", + # ClassDef, new style class + "class C(object): pass", + # Classdef with multiple bases + "class C(A, B): pass", + # Return + "def f():return 1", + "def f():return", + # Delete + "del v", + # Assign + "v = 1", + "a,b = c", + "(a,b) = c", + "[a,b] = c", + "a[b] = c", + # AnnAssign with unpacked types + "x: tuple[*Ts]", + "x: tuple[int, *Ts]", + "x: tuple[int, *tuple[str, ...]]", + # AugAssign + "v += 1", + "v -= 1", + "v *= 1", + "v @= 1", + "v /= 1", + "v %= 1", + "v **= 1", + "v <<= 1", + "v >>= 1", + "v |= 1", + "v ^= 1", + "v &= 1", + "v //= 1", + # For + "for v in v:pass", + # For-Else + "for v in v:\n pass\nelse:\n pass", + # While + "while v:pass", + # While-Else + "while v:\n pass\nelse:\n pass", + # If-Elif-Else + "if v:pass", + "if a:\n pass\nelif b:\n pass", + "if a:\n pass\nelse:\n pass", + "if a:\n pass\nelif b:\n pass\nelse:\n pass", + "if a:\n pass\nelif b:\n pass\nelif b:\n pass\nelif b:\n pass\nelse:\n pass", + # With + "with x: pass", + "with x, y: pass", + "with x as y: pass", + "with x as y, z as q: pass", + "with (x as y): pass", + "with (x, y): pass", + # Raise + "raise", + "raise Exception('string')", + "raise Exception", + "raise Exception('string') from None", + # TryExcept + "try:\n pass\nexcept Exception:\n pass", + "try:\n pass\nexcept Exception as exc:\n pass", + # TryFinally + "try:\n pass\nfinally:\n pass", + # TryStarExcept + "try:\n pass\nexcept* Exception:\n pass", + "try:\n pass\nexcept* Exception as exc:\n pass", + # TryExceptFinallyElse + "try:\n pass\nexcept Exception:\n pass\nelse: pass\nfinally:\n pass", + "try:\n pass\nexcept Exception as exc:\n pass\nelse: pass\nfinally:\n pass", + "try:\n pass\nexcept* Exception as exc:\n pass\nelse: pass\nfinally:\n pass", + # Assert + "assert v", + # Assert with message + "assert v, 'message'", + # Import + "import sys", + "import foo as bar", + # ImportFrom + "from sys import x as y", + "from sys import v", + # Global + "global v", + # Expr + "1", + # Pass, + "pass", + # Break + "for v in v:break", + # Continue + "for v in v:continue", + # for statements with naked tuples (see http://bugs.python.org/issue6704) + "for a,b in c: pass", + "for (a,b) in c: pass", + "for [a,b] in c: pass", + # Multiline generator expression (test for .lineno & .col_offset) + """( + ( + Aa + , + Bb + ) + for + Aa + , + Bb in Cc + )""", + # dictcomp + "{a : b for w in x for m in p if g}", + # dictcomp with naked tuple + "{a : b for v,w in x}", + # setcomp + "{r for l in x if g}", + # setcomp with naked tuple + "{r for l,m in x}", + # AsyncFunctionDef + "async def f():\n 'async function'\n await something()", + # AsyncFor + "async def f():\n async for e in i: 1\n else: 2", + # AsyncWith + "async def f():\n async with a as b: 1", + # PEP 448: Additional Unpacking Generalizations + "{**{1:2}, 2:3}", + "{*{1, 2}, 3}", + # Function with yield (from) + "def f(): yield 1", + "def f(): yield from []", + # Asynchronous comprehensions + "async def f():\n [i async for b in c]", + # Decorated FunctionDef + "@deco1\n@deco2()\n@deco3(1)\ndef f(): pass", + # Decorated AsyncFunctionDef + "@deco1\n@deco2()\n@deco3(1)\nasync def f(): pass", + # Decorated ClassDef + "@deco1\n@deco2()\n@deco3(1)\nclass C: pass", + # Decorator with generator argument + "@deco(a for a in b)\ndef f(): pass", + # Decorator with attribute + "@a.b.c\ndef f(): pass", + # Simple assignment expression + "(a := 1)", + # Assignment expression in if statement + "if a := foo(): pass", + # Assignment expression in while + "while a := foo(): pass", + # Positional-only arguments + "def f(a, /,): pass", + "def f(a, /, c, d, e): pass", + "def f(a, /, c, *, d, e): pass", + "def f(a, /, c, *, d, e, **kwargs): pass", + # Positional-only arguments with defaults + "def f(a=1, /,): pass", + "def f(a=1, /, b=2, c=4): pass", + "def f(a=1, /, b=2, *, c=4): pass", + "def f(a=1, /, b=2, *, c): pass", + "def f(a=1, /, b=2, *, c=4, **kwargs): pass", + "def f(a=1, /, b=2, *, c, **kwargs): pass", + # Type aliases + "type X = int", + "type X[T] = int", + "type X[T, *Ts, **P] = (T, Ts, P)", + "type X[T: int, *Ts, **P] = (T, Ts, P)", + "type X[T: (int, str), *Ts, **P] = (T, Ts, P)", + "type X[T: int = 1, *Ts = 2, **P =3] = (T, Ts, P)", + # Generic classes + "class X[T]: pass", + "class X[T, *Ts, **P]: pass", + "class X[T: int, *Ts, **P]: pass", + "class X[T: (int, str), *Ts, **P]: pass", + "class X[T: int = 1, *Ts = 2, **P = 3]: pass", + # Generic functions + "def f[T](): pass", + "def f[T, *Ts, **P](): pass", + "def f[T: int, *Ts, **P](): pass", + "def f[T: (int, str), *Ts, **P](): pass", + "def f[T: int = 1, *Ts = 2, **P = 3](): pass", + # Match + "match x:\n\tcase 1:\n\t\tpass", + # Match with _ + "match x:\n\tcase 1:\n\t\tpass\n\tcase _:\n\t\tpass", +] + +# These are compiled through "single" +# because of overlap with "eval", it just tests what +# can't be tested with "eval" +single_tests = [ + "1+2" +] + +# These are compiled through "eval" +# It should test all expressions +eval_tests = [ + # Constant(value=None) + "None", + # True + "True", + # False + "False", + # BoolOp + "a and b", + "a or b", + # BinOp + "a + b", + "a - b", + "a * b", + "a / b", + "a @ b", + "a // b", + "a ** b", + "a % b", + "a >> b", + "a << b", + "a ^ b", + "a | b", + "a & b", + # UnaryOp + "not v", + "+v", + "-v", + "~v", + # Lambda + "lambda:None", + # Dict + "{ 1:2 }", + # Empty dict + "{}", + # Set + "{None,}", + # Multiline dict (test for .lineno & .col_offset) + """{ + 1 + : + 2 + }""", + # Multiline list + """[ + 1 + , + 1 + ]""", + # Multiline tuple + """( + 1 + , + )""", + # Multiline set + """{ + 1 + , + 1 + }""", + # ListComp + "[a for b in c if d]", + # GeneratorExp + "(a for b in c if d)", + # SetComp + "{a for b in c if d}", + # DictComp + "{k: v for k, v in c if d}", + # Comprehensions with multiple for targets + "[(a,b) for a,b in c]", + "[(a,b) for (a,b) in c]", + "[(a,b) for [a,b] in c]", + "{(a,b) for a,b in c}", + "{(a,b) for (a,b) in c}", + "{(a,b) for [a,b] in c}", + "((a,b) for a,b in c)", + "((a,b) for (a,b) in c)", + "((a,b) for [a,b] in c)", + # Async comprehensions - async comprehensions can't work outside an asynchronous function + # + # Yield - yield expressions can't work outside a function + # + # Compare + "1 < 2 < 3", + "a == b", + "a <= b", + "a >= b", + "a != b", + "a is b", + "a is not b", + "a in b", + "a not in b", + # Call without argument + "f()", + # Call + "f(1,2,c=3,*d,**e)", + # Call with multi-character starred + "f(*[0, 1])", + # Call with a generator argument + "f(a for a in b)", + # Constant(value=int()) + "10", + # Complex num + "1j", + # Constant(value=str()) + "'string'", + # Attribute + "a.b", + # Subscript + "a[b:c]", + # Name + "v", + # List + "[1,2,3]", + # Empty list + "[]", + # Tuple + "1,2,3", + # Tuple + "(1,2,3)", + # Empty tuple + "()", + # Combination + "a.b.c.d(a.b[1:2])", + # Slice + "[5][1:]", + "[5][:1]", + "[5][::1]", + "[5][1:1:1]", + # IfExp + "foo() if x else bar()", + # JoinedStr and FormattedValue + "f'{a}'", + "f'{a:.2f}'", + "f'{a!r}'", + "f'foo({a})'", +] + + +def main(): + if __name__ != '__main__': + return + if sys.argv[1:] == ['-g']: + for statements, kind in ((exec_tests, "exec"), (single_tests, "single"), + (eval_tests, "eval")): + print(kind+"_results = [") + for statement in statements: + tree = ast.parse(statement, "?", kind) + print("%r," % (to_tuple(tree),)) + print("]") + print("main()") + raise SystemExit + +#### EVERYTHING BELOW IS GENERATED BY python Lib/test/test_ast/snippets.py -g ##### +exec_results = [ +('Module', [('Expr', (1, 0, 1, 18), ('Constant', (1, 0, 1, 18), 'module docstring', None))], []), +('Module', [('FunctionDef', (1, 0, 1, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 9, 1, 13))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 29), ('Constant', (1, 9, 1, 29), 'function docstring', None))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, []), [('Pass', (1, 10, 1, 14))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 0, None)]), [('Pass', (1, 12, 1, 16))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 11), 'args', None, None), [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 23), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 16), 'args', ('Starred', (1, 13, 1, 16), ('Name', (1, 14, 1, 16), 'Ts', ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 19, 1, 23))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Constant', (1, 25, 1, 28), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Starred', (1, 25, 1, 28), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 21), 'f', ('arguments', [], [], None, [], [], ('arg', (1, 8, 1, 14), 'kwargs', None, None), []), [('Pass', (1, 17, 1, 21))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 71), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None), ('arg', (1, 9, 1, 10), 'b', None, None), ('arg', (1, 14, 1, 15), 'c', None, None), ('arg', (1, 22, 1, 23), 'd', None, None), ('arg', (1, 28, 1, 29), 'e', None, None)], ('arg', (1, 35, 1, 39), 'args', None, None), [('arg', (1, 41, 1, 42), 'f', None, None)], [('Constant', (1, 43, 1, 45), 42, None)], ('arg', (1, 49, 1, 55), 'kwargs', None, None), [('Constant', (1, 11, 1, 12), 1, None), ('Constant', (1, 16, 1, 20), None, None), ('List', (1, 24, 1, 26), [], ('Load',)), ('Dict', (1, 30, 1, 32), [], [])]), [('Expr', (1, 58, 1, 71), ('Constant', (1, 58, 1, 71), 'doc for f()', None))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 27), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 23, 1, 27))], [], ('Subscript', (1, 11, 1, 21), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 20), [('Starred', (1, 17, 1, 20), ('Name', (1, 18, 1, 20), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 28, 1, 32))], [], ('Subscript', (1, 11, 1, 26), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 25), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 25), ('Name', (1, 23, 1, 25), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 45), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 41, 1, 45))], [], ('Subscript', (1, 11, 1, 39), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 38), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 38), ('Subscript', (1, 23, 1, 38), ('Name', (1, 23, 1, 28), 'tuple', ('Load',)), ('Tuple', (1, 29, 1, 37), [('Name', (1, 29, 1, 32), 'int', ('Load',)), ('Constant', (1, 34, 1, 37), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), +('Module', [('ClassDef', (1, 0, 1, 12), 'C', [], [], [('Pass', (1, 8, 1, 12))], [], [])], []), +('Module', [('ClassDef', (1, 0, 1, 32), 'C', [], [], [('Expr', (1, 9, 1, 32), ('Constant', (1, 9, 1, 32), 'docstring for class C', None))], [], [])], []), +('Module', [('ClassDef', (1, 0, 1, 21), 'C', [('Name', (1, 8, 1, 14), 'object', ('Load',))], [], [('Pass', (1, 17, 1, 21))], [], [])], []), +('Module', [('ClassDef', (1, 0, 1, 19), 'C', [('Name', (1, 8, 1, 9), 'A', ('Load',)), ('Name', (1, 11, 1, 12), 'B', ('Load',))], [], [('Pass', (1, 15, 1, 19))], [], [])], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Return', (1, 8, 1, 16), ('Constant', (1, 15, 1, 16), 1, None))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [], None, [], [], None, []), [('Return', (1, 8, 1, 14), None)], [], None, None, [])], []), +('Module', [('Delete', (1, 0, 1, 5), [('Name', (1, 4, 1, 5), 'v', ('Del',))])], []), +('Module', [('Assign', (1, 0, 1, 5), [('Name', (1, 0, 1, 1), 'v', ('Store',))], ('Constant', (1, 4, 1, 5), 1, None), None)], []), +('Module', [('Assign', (1, 0, 1, 7), [('Tuple', (1, 0, 1, 3), [('Name', (1, 0, 1, 1), 'a', ('Store',)), ('Name', (1, 2, 1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 6, 1, 7), 'c', ('Load',)), None)], []), +('Module', [('Assign', (1, 0, 1, 9), [('Tuple', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), +('Module', [('Assign', (1, 0, 1, 9), [('List', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), +('Module', [('Assign', (1, 0, 1, 8), [('Subscript', (1, 0, 1, 4), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Store',))], ('Name', (1, 7, 1, 8), 'c', ('Load',)), None)], []), +('Module', [('AnnAssign', (1, 0, 1, 13), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 13), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 12), [('Starred', (1, 9, 1, 12), ('Name', (1, 10, 1, 12), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), +('Module', [('AnnAssign', (1, 0, 1, 18), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 18), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 17), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 17), ('Name', (1, 15, 1, 17), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), +('Module', [('AnnAssign', (1, 0, 1, 31), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 31), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 30), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 30), ('Subscript', (1, 15, 1, 30), ('Name', (1, 15, 1, 20), 'tuple', ('Load',)), ('Tuple', (1, 21, 1, 29), [('Name', (1, 21, 1, 24), 'str', ('Load',)), ('Constant', (1, 26, 1, 29), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Add',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Sub',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Mult',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('MatMult',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Div',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Mod',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Pow',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('LShift',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('RShift',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('BitOr',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('BitXor',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('BitAnd',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('FloorDiv',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('For', (1, 0, 1, 15), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (1, 11, 1, 15))], [], None)], []), +('Module', [('For', (1, 0, 4, 6), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (2, 2, 2, 6))], [('Pass', (4, 2, 4, 6))], None)], []), +('Module', [('While', (1, 0, 1, 12), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (1, 8, 1, 12))], [])], []), +('Module', [('While', (1, 0, 4, 6), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (2, 2, 2, 6))], [('Pass', (4, 2, 4, 6))])], []), +('Module', [('If', (1, 0, 1, 9), ('Name', (1, 3, 1, 4), 'v', ('Load',)), [('Pass', (1, 5, 1, 9))], [])], []), +('Module', [('If', (1, 0, 4, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 4, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [])])], []), +('Module', [('If', (1, 0, 4, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('Pass', (4, 2, 4, 6))])], []), +('Module', [('If', (1, 0, 6, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 6, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [('Pass', (6, 2, 6, 6))])])], []), +('Module', [('If', (1, 0, 10, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 10, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [('If', (5, 0, 10, 6), ('Name', (5, 5, 5, 6), 'b', ('Load',)), [('Pass', (6, 2, 6, 6))], [('If', (7, 0, 10, 6), ('Name', (7, 5, 7, 6), 'b', ('Load',)), [('Pass', (8, 2, 8, 6))], [('Pass', (10, 2, 10, 6))])])])])], []), +('Module', [('With', (1, 0, 1, 12), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), None)], [('Pass', (1, 8, 1, 12))], None)], []), +('Module', [('With', (1, 0, 1, 15), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), None), ('withitem', ('Name', (1, 8, 1, 9), 'y', ('Load',)), None)], [('Pass', (1, 11, 1, 15))], None)], []), +('Module', [('With', (1, 0, 1, 17), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',)))], [('Pass', (1, 13, 1, 17))], None)], []), +('Module', [('With', (1, 0, 1, 25), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',))), ('withitem', ('Name', (1, 13, 1, 14), 'z', ('Load',)), ('Name', (1, 18, 1, 19), 'q', ('Store',)))], [('Pass', (1, 21, 1, 25))], None)], []), +('Module', [('With', (1, 0, 1, 19), [('withitem', ('Name', (1, 6, 1, 7), 'x', ('Load',)), ('Name', (1, 11, 1, 12), 'y', ('Store',)))], [('Pass', (1, 15, 1, 19))], None)], []), +('Module', [('With', (1, 0, 1, 17), [('withitem', ('Name', (1, 6, 1, 7), 'x', ('Load',)), None), ('withitem', ('Name', (1, 9, 1, 10), 'y', ('Load',)), None)], [('Pass', (1, 13, 1, 17))], None)], []), +('Module', [('Raise', (1, 0, 1, 5), None, None)], []), +('Module', [('Raise', (1, 0, 1, 25), ('Call', (1, 6, 1, 25), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), [('Constant', (1, 16, 1, 24), 'string', None)], []), None)], []), +('Module', [('Raise', (1, 0, 1, 15), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), None)], []), +('Module', [('Raise', (1, 0, 1, 35), ('Call', (1, 6, 1, 25), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), [('Constant', (1, 16, 1, 24), 'string', None)], []), ('Constant', (1, 31, 1, 35), None, None))], []), +('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [], [], [('Pass', (4, 2, 4, 6))])], []), +('Module', [('TryStar', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('TryStar', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('Try', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), +('Module', [('Try', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), +('Module', [('TryStar', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), +('Module', [('Assert', (1, 0, 1, 8), ('Name', (1, 7, 1, 8), 'v', ('Load',)), None)], []), +('Module', [('Assert', (1, 0, 1, 19), ('Name', (1, 7, 1, 8), 'v', ('Load',)), ('Constant', (1, 10, 1, 19), 'message', None))], []), +('Module', [('Import', (1, 0, 1, 10), [('alias', (1, 7, 1, 10), 'sys', None)])], []), +('Module', [('Import', (1, 0, 1, 17), [('alias', (1, 7, 1, 17), 'foo', 'bar')])], []), +('Module', [('ImportFrom', (1, 0, 1, 22), 'sys', [('alias', (1, 16, 1, 22), 'x', 'y')], 0)], []), +('Module', [('ImportFrom', (1, 0, 1, 17), 'sys', [('alias', (1, 16, 1, 17), 'v', None)], 0)], []), +('Module', [('Global', (1, 0, 1, 8), ['v'])], []), +('Module', [('Expr', (1, 0, 1, 1), ('Constant', (1, 0, 1, 1), 1, None))], []), +('Module', [('Pass', (1, 0, 1, 4))], []), +('Module', [('For', (1, 0, 1, 16), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Break', (1, 11, 1, 16))], [], None)], []), +('Module', [('For', (1, 0, 1, 19), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Continue', (1, 11, 1, 19))], [], None)], []), +('Module', [('For', (1, 0, 1, 18), ('Tuple', (1, 4, 1, 7), [('Name', (1, 4, 1, 5), 'a', ('Store',)), ('Name', (1, 6, 1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 11, 1, 12), 'c', ('Load',)), [('Pass', (1, 14, 1, 18))], [], None)], []), +('Module', [('For', (1, 0, 1, 20), ('Tuple', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), +('Module', [('For', (1, 0, 1, 20), ('List', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), +('Module', [('Expr', (1, 0, 11, 5), ('GeneratorExp', (1, 0, 11, 5), ('Tuple', (2, 4, 6, 5), [('Name', (3, 4, 3, 6), 'Aa', ('Load',)), ('Name', (5, 7, 5, 9), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4, 10, 6), [('Name', (8, 4, 8, 6), 'Aa', ('Store',)), ('Name', (10, 4, 10, 6), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10, 10, 12), 'Cc', ('Load',)), [], 0)]))], []), +('Module', [('Expr', (1, 0, 1, 34), ('DictComp', (1, 0, 1, 34), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Name', (1, 11, 1, 12), 'w', ('Store',)), ('Name', (1, 16, 1, 17), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22, 1, 23), 'm', ('Store',)), ('Name', (1, 27, 1, 28), 'p', ('Load',)), [('Name', (1, 32, 1, 33), 'g', ('Load',))], 0)]))], []), +('Module', [('Expr', (1, 0, 1, 20), ('DictComp', (1, 0, 1, 20), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'v', ('Store',)), ('Name', (1, 13, 1, 14), 'w', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'x', ('Load',)), [], 0)]))], []), +('Module', [('Expr', (1, 0, 1, 19), ('SetComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 12, 1, 13), 'x', ('Load',)), [('Name', (1, 17, 1, 18), 'g', ('Load',))], 0)]))], []), +('Module', [('Expr', (1, 0, 1, 16), ('SetComp', (1, 0, 1, 16), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7, 1, 10), [('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 9, 1, 10), 'm', ('Store',))], ('Store',)), ('Name', (1, 14, 1, 15), 'x', ('Load',)), [], 0)]))], []), +('Module', [('AsyncFunctionDef', (1, 0, 3, 18), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 17), ('Constant', (2, 1, 2, 17), 'async function', None)), ('Expr', (3, 1, 3, 18), ('Await', (3, 1, 3, 18), ('Call', (3, 7, 3, 18), ('Name', (3, 7, 3, 16), 'something', ('Load',)), [], [])))], [], None, None, [])], []), +('Module', [('AsyncFunctionDef', (1, 0, 3, 8), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncFor', (2, 1, 3, 8), ('Name', (2, 11, 2, 12), 'e', ('Store',)), ('Name', (2, 16, 2, 17), 'i', ('Load',)), [('Expr', (2, 19, 2, 20), ('Constant', (2, 19, 2, 20), 1, None))], [('Expr', (3, 7, 3, 8), ('Constant', (3, 7, 3, 8), 2, None))], None)], [], None, None, [])], []), +('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncWith', (2, 1, 2, 21), [('withitem', ('Name', (2, 12, 2, 13), 'a', ('Load',)), ('Name', (2, 17, 2, 18), 'b', ('Store',)))], [('Expr', (2, 20, 2, 21), ('Constant', (2, 20, 2, 21), 1, None))], None)], [], None, None, [])], []), +('Module', [('Expr', (1, 0, 1, 14), ('Dict', (1, 0, 1, 14), [None, ('Constant', (1, 10, 1, 11), 2, None)], [('Dict', (1, 3, 1, 8), [('Constant', (1, 4, 1, 5), 1, None)], [('Constant', (1, 6, 1, 7), 2, None)]), ('Constant', (1, 12, 1, 13), 3, None)]))], []), +('Module', [('Expr', (1, 0, 1, 12), ('Set', (1, 0, 1, 12), [('Starred', (1, 1, 1, 8), ('Set', (1, 2, 1, 8), [('Constant', (1, 3, 1, 4), 1, None), ('Constant', (1, 6, 1, 7), 2, None)]), ('Load',)), ('Constant', (1, 10, 1, 11), 3, None)]))], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 16), ('Yield', (1, 9, 1, 16), ('Constant', (1, 15, 1, 16), 1, None)))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 22), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 22), ('YieldFrom', (1, 9, 1, 22), ('List', (1, 20, 1, 22), [], ('Load',))))], [], None, None, [])], []), +('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 21), ('ListComp', (2, 1, 2, 21), ('Name', (2, 2, 2, 3), 'i', ('Load',)), [('comprehension', ('Name', (2, 14, 2, 15), 'b', ('Store',)), ('Name', (2, 19, 2, 20), 'c', ('Load',)), [], 1)]))], [], None, None, [])], []), +('Module', [('FunctionDef', (4, 0, 4, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), +('Module', [('AsyncFunctionDef', (4, 0, 4, 19), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 15, 4, 19))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), +('Module', [('ClassDef', (4, 0, 4, 13), 'C', [], [], [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], [])], []), +('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Call', (1, 1, 1, 19), ('Name', (1, 1, 1, 5), 'deco', ('Load',)), [('GeneratorExp', (1, 5, 1, 19), ('Name', (1, 6, 1, 7), 'a', ('Load',)), [('comprehension', ('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 17, 1, 18), 'b', ('Load',)), [], 0)])], [])], None, None, [])], []), +('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Attribute', (1, 1, 1, 6), ('Attribute', (1, 1, 1, 4), ('Name', (1, 1, 1, 2), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',))], None, None, [])], []), +('Module', [('Expr', (1, 0, 1, 8), ('NamedExpr', (1, 1, 1, 7), ('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Constant', (1, 6, 1, 7), 1, None)))], []), +('Module', [('If', (1, 0, 1, 19), ('NamedExpr', (1, 3, 1, 13), ('Name', (1, 3, 1, 4), 'a', ('Store',)), ('Call', (1, 8, 1, 13), ('Name', (1, 8, 1, 11), 'foo', ('Load',)), [], [])), [('Pass', (1, 15, 1, 19))], [])], []), +('Module', [('While', (1, 0, 1, 22), ('NamedExpr', (1, 6, 1, 16), ('Name', (1, 6, 1, 7), 'a', ('Store',)), ('Call', (1, 11, 1, 16), ('Name', (1, 11, 1, 14), 'foo', ('Load',)), [], [])), [('Pass', (1, 18, 1, 22))], [])], []), +('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None), ('arg', (1, 15, 1, 16), 'd', None, None), ('arg', (1, 18, 1, 19), 'e', None, None)], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], None, []), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 39), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], ('arg', (1, 26, 1, 32), 'kwargs', None, None), []), [('Pass', (1, 35, 1, 39))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 20), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None)]), [('Pass', (1, 16, 1, 20))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None), ('arg', (1, 19, 1, 20), 'c', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None), ('Constant', (1, 21, 1, 22), 4, None)]), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 28, 1, 32))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 30), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 26, 1, 30))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 42), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], ('arg', (1, 29, 1, 35), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 38, 1, 42))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 40), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], ('arg', (1, 27, 1, 33), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 36, 1, 40))], [], None, None, [])], []), +('Module', [('TypeAlias', (1, 0, 1, 12), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [], ('Name', (1, 9, 1, 12), 'int', ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 15), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None, None)], ('Name', (1, 12, 1, 15), 'int', ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 32), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None, None), ('TypeVarTuple', (1, 10, 1, 13), 'Ts', None), ('ParamSpec', (1, 15, 1, 18), 'P', None)], ('Tuple', (1, 22, 1, 32), [('Name', (1, 23, 1, 24), 'T', ('Load',)), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Name', (1, 30, 1, 31), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 37), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 13), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',)), None), ('TypeVarTuple', (1, 15, 1, 18), 'Ts', None), ('ParamSpec', (1, 20, 1, 23), 'P', None)], ('Tuple', (1, 27, 1, 37), [('Name', (1, 28, 1, 29), 'T', ('Load',)), ('Name', (1, 31, 1, 33), 'Ts', ('Load',)), ('Name', (1, 35, 1, 36), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 44), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 20), 'T', ('Tuple', (1, 10, 1, 20), [('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Name', (1, 16, 1, 19), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 22, 1, 25), 'Ts', None), ('ParamSpec', (1, 27, 1, 30), 'P', None)], ('Tuple', (1, 34, 1, 44), [('Name', (1, 35, 1, 36), 'T', ('Load',)), ('Name', (1, 38, 1, 40), 'Ts', ('Load',)), ('Name', (1, 42, 1, 43), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 48), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 17), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Constant', (1, 16, 1, 17), 1, None)), ('TypeVarTuple', (1, 19, 1, 26), 'Ts', ('Constant', (1, 25, 1, 26), 2, None)), ('ParamSpec', (1, 28, 1, 34), 'P', ('Constant', (1, 33, 1, 34), 3, None))], ('Tuple', (1, 38, 1, 48), [('Name', (1, 39, 1, 40), 'T', ('Load',)), ('Name', (1, 42, 1, 44), 'Ts', ('Load',)), ('Name', (1, 46, 1, 47), 'P', ('Load',))], ('Load',)))], []), +('Module', [('ClassDef', (1, 0, 1, 16), 'X', [], [], [('Pass', (1, 12, 1, 16))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None)])], []), +('Module', [('ClassDef', (1, 0, 1, 26), 'X', [], [], [('Pass', (1, 22, 1, 26))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None), ('TypeVarTuple', (1, 11, 1, 14), 'Ts', None), ('ParamSpec', (1, 16, 1, 19), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 31), 'X', [], [], [('Pass', (1, 27, 1, 31))], [], [('TypeVar', (1, 8, 1, 14), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), None), ('TypeVarTuple', (1, 16, 1, 19), 'Ts', None), ('ParamSpec', (1, 21, 1, 24), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 38), 'X', [], [], [('Pass', (1, 34, 1, 38))], [], [('TypeVar', (1, 8, 1, 21), 'T', ('Tuple', (1, 11, 1, 21), [('Name', (1, 12, 1, 15), 'int', ('Load',)), ('Name', (1, 17, 1, 20), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 23, 1, 26), 'Ts', None), ('ParamSpec', (1, 28, 1, 31), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 43), 'X', [], [], [('Pass', (1, 39, 1, 43))], [], [('TypeVar', (1, 8, 1, 18), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Constant', (1, 17, 1, 18), 1, None)), ('TypeVarTuple', (1, 20, 1, 27), 'Ts', ('Constant', (1, 26, 1, 27), 2, None)), ('ParamSpec', (1, 29, 1, 36), 'P', ('Constant', (1, 35, 1, 36), 3, None))])], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 12, 1, 16))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None, None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None, None), ('TypeVarTuple', (1, 9, 1, 12), 'Ts', None), ('ParamSpec', (1, 14, 1, 17), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 31), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 27, 1, 31))], [], None, None, [('TypeVar', (1, 6, 1, 12), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',)), None), ('TypeVarTuple', (1, 14, 1, 17), 'Ts', None), ('ParamSpec', (1, 19, 1, 22), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 38), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 34, 1, 38))], [], None, None, [('TypeVar', (1, 6, 1, 19), 'T', ('Tuple', (1, 9, 1, 19), [('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Name', (1, 15, 1, 18), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 21, 1, 24), 'Ts', None), ('ParamSpec', (1, 26, 1, 29), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 43), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 39, 1, 43))], [], None, None, [('TypeVar', (1, 6, 1, 16), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Constant', (1, 15, 1, 16), 1, None)), ('TypeVarTuple', (1, 18, 1, 25), 'Ts', ('Constant', (1, 24, 1, 25), 2, None)), ('ParamSpec', (1, 27, 1, 34), 'P', ('Constant', (1, 33, 1, 34), 3, None))])], []), +('Module', [('Match', (1, 0, 3, 6), ('Name', (1, 6, 1, 7), 'x', ('Load',)), [('match_case', ('MatchValue', (2, 6, 2, 7), ('Constant', (2, 6, 2, 7), 1, None)), None, [('Pass', (3, 2, 3, 6))])])], []), +('Module', [('Match', (1, 0, 5, 6), ('Name', (1, 6, 1, 7), 'x', ('Load',)), [('match_case', ('MatchValue', (2, 6, 2, 7), ('Constant', (2, 6, 2, 7), 1, None)), None, [('Pass', (3, 2, 3, 6))]), ('match_case', ('MatchAs', (4, 6, 4, 7), None, None), None, [('Pass', (5, 2, 5, 6))])])], []), +] +single_results = [ +('Interactive', [('Expr', (1, 0, 1, 3), ('BinOp', (1, 0, 1, 3), ('Constant', (1, 0, 1, 1), 1, None), ('Add',), ('Constant', (1, 2, 1, 3), 2, None)))]), +] +eval_results = [ +('Expression', ('Constant', (1, 0, 1, 4), None, None)), +('Expression', ('Constant', (1, 0, 1, 4), True, None)), +('Expression', ('Constant', (1, 0, 1, 5), False, None)), +('Expression', ('BoolOp', (1, 0, 1, 7), ('And',), [('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 6, 1, 7), 'b', ('Load',))])), +('Expression', ('BoolOp', (1, 0, 1, 6), ('Or',), [('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Add',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Sub',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Mult',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Div',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('MatMult',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('FloorDiv',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Pow',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Mod',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('RShift',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('LShift',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('BitXor',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('BitOr',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('BitAnd',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 5), ('Not',), ('Name', (1, 4, 1, 5), 'v', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 2), ('UAdd',), ('Name', (1, 1, 1, 2), 'v', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 2), ('USub',), ('Name', (1, 1, 1, 2), 'v', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 2), ('Invert',), ('Name', (1, 1, 1, 2), 'v', ('Load',)))), +('Expression', ('Lambda', (1, 0, 1, 11), ('arguments', [], [], None, [], [], None, []), ('Constant', (1, 7, 1, 11), None, None))), +('Expression', ('Dict', (1, 0, 1, 7), [('Constant', (1, 2, 1, 3), 1, None)], [('Constant', (1, 4, 1, 5), 2, None)])), +('Expression', ('Dict', (1, 0, 1, 2), [], [])), +('Expression', ('Set', (1, 0, 1, 7), [('Constant', (1, 1, 1, 5), None, None)])), +('Expression', ('Dict', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None)], [('Constant', (4, 10, 4, 11), 2, None)])), +('Expression', ('List', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None), ('Constant', (4, 8, 4, 9), 1, None)], ('Load',))), +('Expression', ('Tuple', (1, 0, 4, 6), [('Constant', (2, 6, 2, 7), 1, None)], ('Load',))), +('Expression', ('Set', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None), ('Constant', (4, 8, 4, 9), 1, None)])), +('Expression', ('ListComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), +('Expression', ('GeneratorExp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), +('Expression', ('SetComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), +('Expression', ('DictComp', (1, 0, 1, 25), ('Name', (1, 1, 1, 2), 'k', ('Load',)), ('Name', (1, 4, 1, 5), 'v', ('Load',)), [('comprehension', ('Tuple', (1, 10, 1, 14), [('Name', (1, 10, 1, 11), 'k', ('Store',)), ('Name', (1, 13, 1, 14), 'v', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [('Name', (1, 23, 1, 24), 'd', ('Load',))], 0)])), +('Expression', ('ListComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), +('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('SetComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), +('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('GeneratorExp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), +('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('Compare', (1, 0, 1, 9), ('Constant', (1, 0, 1, 1), 1, None), [('Lt',), ('Lt',)], [('Constant', (1, 4, 1, 5), 2, None), ('Constant', (1, 8, 1, 9), 3, None)])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('Eq',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('LtE',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('GtE',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('NotEq',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('Is',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('IsNot',)], [('Name', (1, 9, 1, 10), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('In',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('NotIn',)], [('Name', (1, 9, 1, 10), 'b', ('Load',))])), +('Expression', ('Call', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [], [])), +('Expression', ('Call', (1, 0, 1, 17), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Constant', (1, 2, 1, 3), 1, None), ('Constant', (1, 4, 1, 5), 2, None), ('Starred', (1, 10, 1, 12), ('Name', (1, 11, 1, 12), 'd', ('Load',)), ('Load',))], [('keyword', (1, 6, 1, 9), 'c', ('Constant', (1, 8, 1, 9), 3, None)), ('keyword', (1, 13, 1, 16), None, ('Name', (1, 15, 1, 16), 'e', ('Load',)))])), +('Expression', ('Call', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Starred', (1, 2, 1, 9), ('List', (1, 3, 1, 9), [('Constant', (1, 4, 1, 5), 0, None), ('Constant', (1, 7, 1, 8), 1, None)], ('Load',)), ('Load',))], [])), +('Expression', ('Call', (1, 0, 1, 15), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('GeneratorExp', (1, 1, 1, 15), ('Name', (1, 2, 1, 3), 'a', ('Load',)), [('comprehension', ('Name', (1, 8, 1, 9), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Load',)), [], 0)])], [])), +('Expression', ('Constant', (1, 0, 1, 2), 10, None)), +('Expression', ('Constant', (1, 0, 1, 2), 1j, None)), +('Expression', ('Constant', (1, 0, 1, 8), 'string', None)), +('Expression', ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Slice', (1, 2, 1, 5), ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Name', (1, 4, 1, 5), 'c', ('Load',)), None), ('Load',))), +('Expression', ('Name', (1, 0, 1, 1), 'v', ('Load',))), +('Expression', ('List', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), +('Expression', ('List', (1, 0, 1, 2), [], ('Load',))), +('Expression', ('Tuple', (1, 0, 1, 5), [('Constant', (1, 0, 1, 1), 1, None), ('Constant', (1, 2, 1, 3), 2, None), ('Constant', (1, 4, 1, 5), 3, None)], ('Load',))), +('Expression', ('Tuple', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), +('Expression', ('Tuple', (1, 0, 1, 2), [], ('Load',))), +('Expression', ('Call', (1, 0, 1, 17), ('Attribute', (1, 0, 1, 7), ('Attribute', (1, 0, 1, 5), ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8, 1, 16), ('Attribute', (1, 8, 1, 11), ('Name', (1, 8, 1, 9), 'a', ('Load',)), 'b', ('Load',)), ('Slice', (1, 12, 1, 15), ('Constant', (1, 12, 1, 13), 1, None), ('Constant', (1, 14, 1, 15), 2, None), None), ('Load',))], [])), +('Expression', ('Subscript', (1, 0, 1, 7), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 6), ('Constant', (1, 4, 1, 5), 1, None), None, None), ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 7), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 6), None, ('Constant', (1, 5, 1, 6), 1, None), None), ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 8), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 7), None, None, ('Constant', (1, 6, 1, 7), 1, None)), ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 10), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 9), ('Constant', (1, 4, 1, 5), 1, None), ('Constant', (1, 6, 1, 7), 1, None), ('Constant', (1, 8, 1, 9), 1, None)), ('Load',))), +('Expression', ('IfExp', (1, 0, 1, 21), ('Name', (1, 9, 1, 10), 'x', ('Load',)), ('Call', (1, 0, 1, 5), ('Name', (1, 0, 1, 3), 'foo', ('Load',)), [], []), ('Call', (1, 16, 1, 21), ('Name', (1, 16, 1, 19), 'bar', ('Load',)), [], []))), +('Expression', ('JoinedStr', (1, 0, 1, 6), [('FormattedValue', (1, 2, 1, 5), ('Name', (1, 3, 1, 4), 'a', ('Load',)), -1, None)])), +('Expression', ('JoinedStr', (1, 0, 1, 10), [('FormattedValue', (1, 2, 1, 9), ('Name', (1, 3, 1, 4), 'a', ('Load',)), -1, ('JoinedStr', (1, 4, 1, 8), [('Constant', (1, 5, 1, 8), '.2f', None)]))])), +('Expression', ('JoinedStr', (1, 0, 1, 8), [('FormattedValue', (1, 2, 1, 7), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 114, None)])), +('Expression', ('JoinedStr', (1, 0, 1, 11), [('Constant', (1, 2, 1, 6), 'foo(', None), ('FormattedValue', (1, 6, 1, 9), ('Name', (1, 7, 1, 8), 'a', ('Load',)), -1, None), ('Constant', (1, 9, 1, 10), ')', None)])), +] +main() diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast/test_ast.py similarity index 60% rename from Lib/test/test_ast.py rename to Lib/test/test_ast/test_ast.py index 7cecf319e3638f..c268a1f00f938e 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -1,5 +1,6 @@ import ast import builtins +import copy import dis import enum import os @@ -8,9 +9,8 @@ import textwrap import types import unittest -import warnings import weakref -from functools import partial +from pathlib import Path from textwrap import dedent try: import _testinternalcapi @@ -18,277 +18,28 @@ _testinternalcapi = None from test import support -from test.support.import_helper import import_fresh_module -from test.support import os_helper, script_helper +from test.support import os_helper, script_helper, skip_emscripten_stack_overflow from test.support.ast_helper import ASTTestMixin +from test.test_ast.utils import to_tuple +from test.test_ast.snippets import ( + eval_tests, eval_results, exec_tests, exec_results, single_tests, single_results +) + + +STDLIB = os.path.dirname(ast.__file__) +STDLIB_FILES = [fn for fn in os.listdir(STDLIB) if fn.endswith(".py")] +STDLIB_FILES.extend(["test/test_grammar.py", "test/test_unpack_ex.py"]) + +AST_REPR_DATA_FILE = Path(__file__).parent / "data" / "ast_repr.txt" + +def ast_repr_get_test_cases() -> list[str]: + return exec_tests + eval_tests + + +def ast_repr_update_snapshots() -> None: + data = [repr(ast.parse(test)) for test in ast_repr_get_test_cases()] + AST_REPR_DATA_FILE.write_text("\n".join(data)) -def to_tuple(t): - if t is None or isinstance(t, (str, int, complex)) or t is Ellipsis: - return t - elif isinstance(t, list): - return [to_tuple(e) for e in t] - result = [t.__class__.__name__] - if hasattr(t, 'lineno') and hasattr(t, 'col_offset'): - result.append((t.lineno, t.col_offset)) - if hasattr(t, 'end_lineno') and hasattr(t, 'end_col_offset'): - result[-1] += (t.end_lineno, t.end_col_offset) - if t._fields is None: - return tuple(result) - for f in t._fields: - result.append(to_tuple(getattr(t, f))) - return tuple(result) - - -# These tests are compiled through "exec" -# There should be at least one test per statement -exec_tests = [ - # None - "None", - # Module docstring - "'module docstring'", - # FunctionDef - "def f(): pass", - # FunctionDef with docstring - "def f(): 'function docstring'", - # FunctionDef with arg - "def f(a): pass", - # FunctionDef with arg and default value - "def f(a=0): pass", - # FunctionDef with varargs - "def f(*args): pass", - # FunctionDef with varargs as TypeVarTuple - "def f(*args: *Ts): pass", - # FunctionDef with varargs as unpacked Tuple - "def f(*args: *tuple[int, ...]): pass", - # FunctionDef with varargs as unpacked Tuple *and* TypeVarTuple - "def f(*args: *tuple[int, *Ts]): pass", - # FunctionDef with kwargs - "def f(**kwargs): pass", - # FunctionDef with all kind of args and docstring - "def f(a, b=1, c=None, d=[], e={}, *args, f=42, **kwargs): 'doc for f()'", - # FunctionDef with type annotation on return involving unpacking - "def f() -> tuple[*Ts]: pass", - "def f() -> tuple[int, *Ts]: pass", - "def f() -> tuple[int, *tuple[int, ...]]: pass", - # ClassDef - "class C:pass", - # ClassDef with docstring - "class C: 'docstring for class C'", - # ClassDef, new style class - "class C(object): pass", - # Return - "def f():return 1", - # Delete - "del v", - # Assign - "v = 1", - "a,b = c", - "(a,b) = c", - "[a,b] = c", - # AnnAssign with unpacked types - "x: tuple[*Ts]", - "x: tuple[int, *Ts]", - "x: tuple[int, *tuple[str, ...]]", - # AugAssign - "v += 1", - # For - "for v in v:pass", - # While - "while v:pass", - # If - "if v:pass", - # If-Elif - "if a:\n pass\nelif b:\n pass", - # If-Elif-Else - "if a:\n pass\nelif b:\n pass\nelse:\n pass", - # With - "with x as y: pass", - "with x as y, z as q: pass", - "with (x as y): pass", - "with (x, y): pass", - # Raise - "raise Exception('string')", - # TryExcept - "try:\n pass\nexcept Exception:\n pass", - # TryFinally - "try:\n pass\nfinally:\n pass", - # TryStarExcept - "try:\n pass\nexcept* Exception:\n pass", - # Assert - "assert v", - # Import - "import sys", - # ImportFrom - "from sys import v", - # Global - "global v", - # Expr - "1", - # Pass, - "pass", - # Break - "for v in v:break", - # Continue - "for v in v:continue", - # for statements with naked tuples (see http://bugs.python.org/issue6704) - "for a,b in c: pass", - "for (a,b) in c: pass", - "for [a,b] in c: pass", - # Multiline generator expression (test for .lineno & .col_offset) - """( - ( - Aa - , - Bb - ) - for - Aa - , - Bb in Cc - )""", - # dictcomp - "{a : b for w in x for m in p if g}", - # dictcomp with naked tuple - "{a : b for v,w in x}", - # setcomp - "{r for l in x if g}", - # setcomp with naked tuple - "{r for l,m in x}", - # AsyncFunctionDef - "async def f():\n 'async function'\n await something()", - # AsyncFor - "async def f():\n async for e in i: 1\n else: 2", - # AsyncWith - "async def f():\n async with a as b: 1", - # PEP 448: Additional Unpacking Generalizations - "{**{1:2}, 2:3}", - "{*{1, 2}, 3}", - # Asynchronous comprehensions - "async def f():\n [i async for b in c]", - # Decorated FunctionDef - "@deco1\n@deco2()\n@deco3(1)\ndef f(): pass", - # Decorated AsyncFunctionDef - "@deco1\n@deco2()\n@deco3(1)\nasync def f(): pass", - # Decorated ClassDef - "@deco1\n@deco2()\n@deco3(1)\nclass C: pass", - # Decorator with generator argument - "@deco(a for a in b)\ndef f(): pass", - # Decorator with attribute - "@a.b.c\ndef f(): pass", - # Simple assignment expression - "(a := 1)", - # Positional-only arguments - "def f(a, /,): pass", - "def f(a, /, c, d, e): pass", - "def f(a, /, c, *, d, e): pass", - "def f(a, /, c, *, d, e, **kwargs): pass", - # Positional-only arguments with defaults - "def f(a=1, /,): pass", - "def f(a=1, /, b=2, c=4): pass", - "def f(a=1, /, b=2, *, c=4): pass", - "def f(a=1, /, b=2, *, c): pass", - "def f(a=1, /, b=2, *, c=4, **kwargs): pass", - "def f(a=1, /, b=2, *, c, **kwargs): pass", - # Type aliases - "type X = int", - "type X[T] = int", - "type X[T, *Ts, **P] = (T, Ts, P)", - "type X[T: int, *Ts, **P] = (T, Ts, P)", - "type X[T: (int, str), *Ts, **P] = (T, Ts, P)", - # Generic classes - "class X[T]: pass", - "class X[T, *Ts, **P]: pass", - "class X[T: int, *Ts, **P]: pass", - "class X[T: (int, str), *Ts, **P]: pass", - # Generic functions - "def f[T](): pass", - "def f[T, *Ts, **P](): pass", - "def f[T: int, *Ts, **P](): pass", - "def f[T: (int, str), *Ts, **P](): pass", -] - -# These are compiled through "single" -# because of overlap with "eval", it just tests what -# can't be tested with "eval" -single_tests = [ - "1+2" -] - -# These are compiled through "eval" -# It should test all expressions -eval_tests = [ - # None - "None", - # BoolOp - "a and b", - # BinOp - "a + b", - # UnaryOp - "not v", - # Lambda - "lambda:None", - # Dict - "{ 1:2 }", - # Empty dict - "{}", - # Set - "{None,}", - # Multiline dict (test for .lineno & .col_offset) - """{ - 1 - : - 2 - }""", - # ListComp - "[a for b in c if d]", - # GeneratorExp - "(a for b in c if d)", - # Comprehensions with multiple for targets - "[(a,b) for a,b in c]", - "[(a,b) for (a,b) in c]", - "[(a,b) for [a,b] in c]", - "{(a,b) for a,b in c}", - "{(a,b) for (a,b) in c}", - "{(a,b) for [a,b] in c}", - "((a,b) for a,b in c)", - "((a,b) for (a,b) in c)", - "((a,b) for [a,b] in c)", - # Yield - yield expressions can't work outside a function - # - # Compare - "1 < 2 < 3", - # Call - "f(1,2,c=3,*d,**e)", - # Call with multi-character starred - "f(*[0, 1])", - # Call with a generator argument - "f(a for a in b)", - # Num - "10", - # Str - "'string'", - # Attribute - "a.b", - # Subscript - "a[b:c]", - # Name - "v", - # List - "[1,2,3]", - # Empty list - "[]", - # Tuple - "1,2,3", - # Tuple - "(1,2,3)", - # Empty tuple - "()", - # Combination - "a.b.c.d(a.b[1:2])", -] - -# TODO: expr_context, slice, boolop, operator, unaryop, cmpop, comprehension -# excepthandler, arguments, keywords, alias class AST_Tests(unittest.TestCase): maxDiff = None @@ -333,6 +84,23 @@ def test_AST_objects(self): # "ast.AST constructor takes 0 positional arguments" ast.AST(2) + def test_AST_fields_NULL_check(self): + # See: https://github.com/python/cpython/issues/126105 + old_value = ast.AST._fields + + def cleanup(): + ast.AST._fields = old_value + self.addCleanup(cleanup) + + del ast.AST._fields + + msg = "type object 'ast.AST' has no attribute '_fields'" + # Both examples used to crash: + with self.assertRaisesRegex(AttributeError, msg): + ast.AST(arg1=123) + with self.assertRaisesRegex(AttributeError, msg): + ast.AST() + def test_AST_garbage_collection(self): class X: pass @@ -492,35 +260,8 @@ def test_base_classes(self): self.assertTrue(issubclass(ast.comprehension, ast.AST)) self.assertTrue(issubclass(ast.Gt, ast.AST)) - def test_import_deprecated(self): - ast = import_fresh_module('ast') - depr_regex = ( - r'ast\.{} is deprecated and will be removed in Python 3.14; ' - r'use ast\.Constant instead' - ) - for name in 'Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis': - with self.assertWarnsRegex(DeprecationWarning, depr_regex.format(name)): - getattr(ast, name) - - def test_field_attr_existence_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis'): - item = getattr(ast, name) - if self._is_ast_node(name, item): - with self.subTest(item): - with self.assertWarns(DeprecationWarning): - x = item() - if isinstance(x, ast.AST): - self.assertIs(type(x._fields), tuple) - def test_field_attr_existence(self): for name, item in ast.__dict__.items(): - # These emit DeprecationWarnings - if name in {'Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis'}: - continue # constructor has a different signature if name == 'Index': continue @@ -563,106 +304,12 @@ def test_arguments(self): self.assertEqual(x.args, 2) self.assertEqual(x.vararg, 3) - def test_field_attr_writable_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - x = ast.Num() - # We can assign to _fields - x._fields = 666 - self.assertEqual(x._fields, 666) - def test_field_attr_writable(self): x = ast.Constant(1) # We can assign to _fields x._fields = 666 self.assertEqual(x._fields, 666) - def test_classattrs_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - x = ast.Num() - self.assertEqual(x._fields, ('value', 'kind')) - - with self.assertRaises(AttributeError): - x.value - - with self.assertRaises(AttributeError): - x.n - - x = ast.Num(42) - self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - - with self.assertRaises(AttributeError): - x.lineno - - with self.assertRaises(AttributeError): - x.foobar - - x = ast.Num(lineno=2) - self.assertEqual(x.lineno, 2) - - x = ast.Num(42, lineno=0) - self.assertEqual(x.lineno, 0) - self.assertEqual(x._fields, ('value', 'kind')) - self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - - self.assertRaises(TypeError, ast.Num, 1, None, 2) - self.assertRaises(TypeError, ast.Num, 1, None, 2, lineno=0) - - # Arbitrary keyword arguments are supported - self.assertEqual(ast.Num(1, foo='bar').foo, 'bar') - - with self.assertRaisesRegex(TypeError, "Num got multiple values for argument 'n'"): - ast.Num(1, n=2) - - self.assertEqual(ast.Num(42).n, 42) - self.assertEqual(ast.Num(4.25).n, 4.25) - self.assertEqual(ast.Num(4.25j).n, 4.25j) - self.assertEqual(ast.Str('42').s, '42') - self.assertEqual(ast.Bytes(b'42').s, b'42') - self.assertIs(ast.NameConstant(True).value, True) - self.assertIs(ast.NameConstant(False).value, False) - self.assertIs(ast.NameConstant(None).value, None) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - "Constant.__init__ missing 1 required positional argument: 'value'. This will become " - 'an error in Python 3.15.', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - "Constant.__init__ missing 1 required positional argument: 'value'. This will become " - 'an error in Python 3.15.', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - "Constant.__init__ got an unexpected keyword argument 'foo'. Support for " - 'arbitrary keyword arguments is deprecated and will be removed in Python ' - '3.15.', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - def test_classattrs(self): with self.assertWarns(DeprecationWarning): x = ast.Constant() @@ -708,190 +355,6 @@ def test_classattrs(self): self.assertIs(ast.Constant(None).value, None) self.assertIs(ast.Constant(...).value, ...) - def test_realtype(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - self.assertIs(type(ast.Num(42)), ast.Constant) - self.assertIs(type(ast.Num(4.25)), ast.Constant) - self.assertIs(type(ast.Num(4.25j)), ast.Constant) - self.assertIs(type(ast.Str('42')), ast.Constant) - self.assertIs(type(ast.Bytes(b'42')), ast.Constant) - self.assertIs(type(ast.NameConstant(True)), ast.Constant) - self.assertIs(type(ast.NameConstant(False)), ast.Constant) - self.assertIs(type(ast.NameConstant(None)), ast.Constant) - self.assertIs(type(ast.Ellipsis()), ast.Constant) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Ellipsis is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - - def test_isinstance(self): - from ast import Constant - - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - cls_depr_msg = ( - 'ast.{} is deprecated and will be removed in Python 3.14; ' - 'use ast.Constant instead' - ) - - assertNumDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Num") - ) - assertStrDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Str") - ) - assertBytesDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Bytes") - ) - assertNameConstantDeprecated = partial( - self.assertWarnsRegex, - DeprecationWarning, - cls_depr_msg.format("NameConstant") - ) - assertEllipsisDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Ellipsis") - ) - - for arg in 42, 4.2, 4.2j: - with self.subTest(arg=arg): - with assertNumDeprecated(): - n = Num(arg) - with assertNumDeprecated(): - self.assertIsInstance(n, Num) - - with assertStrDeprecated(): - s = Str('42') - with assertStrDeprecated(): - self.assertIsInstance(s, Str) - - with assertBytesDeprecated(): - b = Bytes(b'42') - with assertBytesDeprecated(): - self.assertIsInstance(b, Bytes) - - for arg in True, False, None: - with self.subTest(arg=arg): - with assertNameConstantDeprecated(): - n = NameConstant(arg) - with assertNameConstantDeprecated(): - self.assertIsInstance(n, NameConstant) - - with assertEllipsisDeprecated(): - e = Ellipsis() - with assertEllipsisDeprecated(): - self.assertIsInstance(e, Ellipsis) - - for arg in 42, 4.2, 4.2j: - with self.subTest(arg=arg): - with assertNumDeprecated(): - self.assertIsInstance(Constant(arg), Num) - - with assertStrDeprecated(): - self.assertIsInstance(Constant('42'), Str) - - with assertBytesDeprecated(): - self.assertIsInstance(Constant(b'42'), Bytes) - - for arg in True, False, None: - with self.subTest(arg=arg): - with assertNameConstantDeprecated(): - self.assertIsInstance(Constant(arg), NameConstant) - - with assertEllipsisDeprecated(): - self.assertIsInstance(Constant(...), Ellipsis) - - with assertStrDeprecated(): - s = Str('42') - assertNumDeprecated(self.assertNotIsInstance, s, Num) - assertBytesDeprecated(self.assertNotIsInstance, s, Bytes) - - with assertNumDeprecated(): - n = Num(42) - assertStrDeprecated(self.assertNotIsInstance, n, Str) - assertNameConstantDeprecated(self.assertNotIsInstance, n, NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, n, Ellipsis) - - with assertNameConstantDeprecated(): - n = NameConstant(True) - with assertNumDeprecated(): - self.assertNotIsInstance(n, Num) - - with assertNameConstantDeprecated(): - n = NameConstant(False) - with assertNumDeprecated(): - self.assertNotIsInstance(n, Num) - - for arg in '42', True, False: - with self.subTest(arg=arg): - with assertNumDeprecated(): - self.assertNotIsInstance(Constant(arg), Num) - - assertStrDeprecated(self.assertNotIsInstance, Constant(42), Str) - assertBytesDeprecated(self.assertNotIsInstance, Constant('42'), Bytes) - assertNameConstantDeprecated(self.assertNotIsInstance, Constant(42), NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, Constant(42), Ellipsis) - assertNumDeprecated(self.assertNotIsInstance, Constant(None), Num) - assertStrDeprecated(self.assertNotIsInstance, Constant(None), Str) - assertBytesDeprecated(self.assertNotIsInstance, Constant(None), Bytes) - assertNameConstantDeprecated(self.assertNotIsInstance, Constant(1), NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, Constant(None), Ellipsis) - - class S(str): pass - with assertStrDeprecated(): - self.assertIsInstance(Constant(S('42')), Str) - with assertNumDeprecated(): - self.assertNotIsInstance(Constant(S('42')), Num) - - def test_constant_subclasses_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - class N(ast.Num): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.z = 'spam' - class N2(ast.Num): - pass - - n = N(42) - self.assertEqual(n.n, 42) - self.assertEqual(n.z, 'spam') - self.assertIs(type(n), N) - self.assertIsInstance(n, N) - self.assertIsInstance(n, ast.Num) - self.assertNotIsInstance(n, N2) - self.assertNotIsInstance(ast.Num(42), N) - n = N(n=42) - self.assertEqual(n.n, 42) - self.assertIs(type(n), N) - - self.assertEqual([str(w.message) for w in wlog], [ - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - ]) - def test_constant_subclasses(self): class N(ast.Constant): def __init__(self, *args, **kwargs): @@ -968,21 +431,12 @@ def test_no_fields(self): x = ast.Sub() self.assertEqual(x._fields, ()) - def test_pickling(self): - import pickle - - for protocol in range(pickle.HIGHEST_PROTOCOL + 1): - for ast in (compile(i, "?", "exec", 0x400) for i in exec_tests): - with self.subTest(ast=ast, protocol=protocol): - ast2 = pickle.loads(pickle.dumps(ast, protocol)) - self.assertEqual(to_tuple(ast2), to_tuple(ast)) - def test_invalid_sum(self): pos = dict(lineno=2, col_offset=3) m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], []) with self.assertRaises(TypeError) as cm: compile(m, "", "exec") - self.assertIn("but got None: for node, attr, source in tests: self.assert_none_check(node, attr, source) + def test_repr(self) -> None: + snapshots = AST_REPR_DATA_FILE.read_text().split("\n") + for test, snapshot in zip(ast_repr_get_test_cases(), snapshots, strict=True): + with self.subTest(test_input=test): + self.assertEqual(repr(ast.parse(test)), snapshot) + + def test_repr_large_input_crash(self): + # gh-125010: Fix use-after-free in ast repr() + source = "0x0" + "e" * 10_000 + with self.assertRaisesRegex(ValueError, + r"Exceeds the limit \(\d+ digits\)"): + repr(ast.Constant(value=eval(source))) + + +class CopyTests(unittest.TestCase): + """Test copying and pickling AST nodes.""" + + @staticmethod + def iter_ast_classes(): + """Iterate over the (native) subclasses of ast.AST recursively. + + This excludes the special class ast.Index since its constructor + returns an integer. + """ + def do(cls): + if cls.__module__ != 'ast': + return + if cls is ast.Index: + return + + yield cls + for sub in cls.__subclasses__(): + yield from do(sub) + + yield from do(ast.AST) + + def test_pickling(self): + import pickle + + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + for code in exec_tests: + with self.subTest(code=code, protocol=protocol): + tree = compile(code, "?", "exec", 0x400) + ast2 = pickle.loads(pickle.dumps(tree, protocol)) + self.assertEqual(to_tuple(ast2), to_tuple(tree)) + + def test_copy_with_parents(self): + # gh-120108 + code = """ + ('',) + while i < n: + if ch == '': + ch = format[i] + if ch == '': + if freplace is None: + '' % getattr(object) + elif ch == '': + if zreplace is None: + if hasattr: + offset = object.utcoffset() + if offset is not None: + if offset.days < 0: + offset = -offset + h = divmod(timedelta(hours=0)) + if u: + zreplace = '' % (sign,) + elif s: + zreplace = '' % (sign,) + else: + zreplace = '' % (sign,) + elif ch == '': + if Zreplace is None: + Zreplace = '' + if hasattr(object): + s = object.tzname() + if s is not None: + Zreplace = s.replace('') + newformat.append(Zreplace) + else: + push('') + else: + push(ch) + + """ + tree = ast.parse(textwrap.dedent(code)) + for node in ast.walk(tree): + for child in ast.iter_child_nodes(node): + child.parent = node + try: + with support.infinite_recursion(200): + tree2 = copy.deepcopy(tree) + finally: + # Singletons like ast.Load() are shared; make sure we don't + # leave them mutated after this test. + for node in ast.walk(tree): + if hasattr(node, "parent"): + del node.parent + + for node in ast.walk(tree2): + for child in ast.iter_child_nodes(node): + if hasattr(child, "parent") and not isinstance(child, ( + ast.expr_context, ast.boolop, ast.unaryop, ast.cmpop, ast.operator, + )): + self.assertEqual(to_tuple(child.parent), to_tuple(node)) + + def test_replace_interface(self): + for klass in self.iter_ast_classes(): + with self.subTest(klass=klass): + self.assertTrue(hasattr(klass, '__replace__')) + + fields = set(klass._fields) + with self.subTest(klass=klass, fields=fields): + node = klass(**dict.fromkeys(fields)) + # forbid positional arguments in replace() + self.assertRaises(TypeError, copy.replace, node, 1) + self.assertRaises(TypeError, node.__replace__, 1) + + def test_replace_native(self): + for klass in self.iter_ast_classes(): + fields = set(klass._fields) + attributes = set(klass._attributes) + + with self.subTest(klass=klass, fields=fields, attributes=attributes): + # use of object() to ensure that '==' and 'is' + # behave similarly in ast.compare(node, repl) + old_fields = {field: object() for field in fields} + old_attrs = {attr: object() for attr in attributes} + + # check shallow copy + node = klass(**old_fields) + repl = copy.replace(node) + self.assertTrue(ast.compare(node, repl, compare_attributes=True)) + # check when passing using attributes (they may be optional!) + node = klass(**old_fields, **old_attrs) + repl = copy.replace(node) + self.assertTrue(ast.compare(node, repl, compare_attributes=True)) + + for field in fields: + # check when we sometimes have attributes and sometimes not + for init_attrs in [{}, old_attrs]: + node = klass(**old_fields, **init_attrs) + # only change a single field (do not change attributes) + new_value = object() + repl = copy.replace(node, **{field: new_value}) + for f in fields: + old_value = old_fields[f] + # assert that there is no side-effect + self.assertIs(getattr(node, f), old_value) + # check the changes + if f != field: + self.assertIs(getattr(repl, f), old_value) + else: + self.assertIs(getattr(repl, f), new_value) + self.assertFalse(ast.compare(node, repl, compare_attributes=True)) + + for attribute in attributes: + node = klass(**old_fields, **old_attrs) + # only change a single attribute (do not change fields) + new_attr = object() + repl = copy.replace(node, **{attribute: new_attr}) + for a in attributes: + old_attr = old_attrs[a] + # assert that there is no side-effect + self.assertIs(getattr(node, a), old_attr) + # check the changes + if a != attribute: + self.assertIs(getattr(repl, a), old_attr) + else: + self.assertIs(getattr(repl, a), new_attr) + self.assertFalse(ast.compare(node, repl, compare_attributes=True)) + + def test_replace_accept_known_class_fields(self): + nid, ctx = object(), object() + + node = ast.Name(id=nid, ctx=ctx) + self.assertIs(node.id, nid) + self.assertIs(node.ctx, ctx) + + new_nid = object() + repl = copy.replace(node, id=new_nid) + # assert that there is no side-effect + self.assertIs(node.id, nid) + self.assertIs(node.ctx, ctx) + # check the changes + self.assertIs(repl.id, new_nid) + self.assertIs(repl.ctx, node.ctx) # no changes + + def test_replace_accept_known_class_attributes(self): + node = ast.parse('x').body[0].value + self.assertEqual(node.id, 'x') + self.assertEqual(node.lineno, 1) + + # constructor allows any type so replace() should do the same + lineno = object() + repl = copy.replace(node, lineno=lineno) + # assert that there is no side-effect + self.assertEqual(node.lineno, 1) + # check the changes + self.assertEqual(repl.id, node.id) + self.assertEqual(repl.ctx, node.ctx) + self.assertEqual(repl.lineno, lineno) + + _, _, state = node.__reduce__() + self.assertEqual(state['id'], 'x') + self.assertEqual(state['ctx'], node.ctx) + self.assertEqual(state['lineno'], 1) + + _, _, state = repl.__reduce__() + self.assertEqual(state['id'], 'x') + self.assertEqual(state['ctx'], node.ctx) + self.assertEqual(state['lineno'], lineno) + + def test_replace_accept_known_custom_class_fields(self): + class MyNode(ast.AST): + _fields = ('name', 'data') + __annotations__ = {'name': str, 'data': object} + __match_args__ = ('name', 'data') + + name, data = 'name', object() + + node = MyNode(name, data) + self.assertIs(node.name, name) + self.assertIs(node.data, data) + # check shallow copy + repl = copy.replace(node) + # assert that there is no side-effect + self.assertIs(node.name, name) + self.assertIs(node.data, data) + # check the shallow copy + self.assertIs(repl.name, name) + self.assertIs(repl.data, data) + + node = MyNode(name, data) + repl_data = object() + # replace custom but known field + repl = copy.replace(node, data=repl_data) + # assert that there is no side-effect + self.assertIs(node.name, name) + self.assertIs(node.data, data) + # check the changes + self.assertIs(repl.name, node.name) + self.assertIs(repl.data, repl_data) + + def test_replace_accept_known_custom_class_attributes(self): + class MyNode(ast.AST): + x = 0 + y = 1 + _attributes = ('x', 'y') + + node = MyNode() + self.assertEqual(node.x, 0) + self.assertEqual(node.y, 1) + + y = object() + repl = copy.replace(node, y=y) + # assert that there is no side-effect + self.assertEqual(node.x, 0) + self.assertEqual(node.y, 1) + # check the changes + self.assertEqual(repl.x, 0) + self.assertEqual(repl.y, y) + + def test_replace_ignore_known_custom_instance_fields(self): + node = ast.parse('x').body[0].value + node.extra = extra = object() # add instance 'extra' field + context = node.ctx + + # assert initial values + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + # shallow copy, but drops extra fields + repl = copy.replace(node) + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + # verify that the 'extra' field is not kept + self.assertIs(repl.id, 'x') + self.assertIs(repl.ctx, context) + self.assertRaises(AttributeError, getattr, repl, 'extra') + + # change known native field + repl = copy.replace(node, id='y') + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + # verify that the 'extra' field is not kept + self.assertIs(repl.id, 'y') + self.assertIs(repl.ctx, context) + self.assertRaises(AttributeError, getattr, repl, 'extra') + + def test_replace_reject_missing_field(self): + # case: warn if deleted field is not replaced + node = ast.parse('x').body[0].value + context = node.ctx + del node.id + + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + msg = "Name.__replace__ missing 1 keyword argument: 'id'." + with self.assertRaisesRegex(TypeError, re.escape(msg)): + copy.replace(node) + # assert that there is no side-effect + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + + # case: do not raise if deleted field is replaced + node = ast.parse('x').body[0].value + context = node.ctx + del node.id + + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + repl = copy.replace(node, id='y') + # assert that there is no side-effect + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + self.assertIs(repl.id, 'y') + self.assertIs(repl.ctx, context) + + def test_replace_reject_known_custom_instance_fields_commits(self): + node = ast.parse('x').body[0].value + node.extra = extra = object() # add instance 'extra' field + context = node.ctx + + # explicit rejection of known instance fields + self.assertTrue(hasattr(node, 'extra')) + msg = "Name.__replace__ got an unexpected keyword argument 'extra'." + with self.assertRaisesRegex(TypeError, re.escape(msg)): + copy.replace(node, extra=1) + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + + def test_replace_reject_unknown_instance_fields(self): + node = ast.parse('x').body[0].value + context = node.ctx + + # explicit rejection of unknown extra fields + self.assertRaises(AttributeError, getattr, node, 'unknown') + msg = "Name.__replace__ got an unexpected keyword argument 'unknown'." + with self.assertRaisesRegex(TypeError, re.escape(msg)): + copy.replace(node, unknown=1) + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertRaises(AttributeError, getattr, node, 'unknown') + class ASTHelpers_Test(unittest.TestCase): maxDiff = None @@ -1227,21 +1172,20 @@ def test_dump(self): node = ast.parse('spam(eggs, "and cheese")') self.assertEqual(ast.dump(node), "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), " - "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')], " - "keywords=[]))], type_ignores=[])" + "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')]))])" ) self.assertEqual(ast.dump(node, annotate_fields=False), "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), " - "Constant('and cheese')], []))], [])" + "Constant('and cheese')]))])" ) self.assertEqual(ast.dump(node, include_attributes=True), "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=4), " "args=[Name(id='eggs', ctx=Load(), lineno=1, col_offset=5, " "end_lineno=1, end_col_offset=9), Constant(value='and cheese', " - "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], keywords=[], " + "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24), " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)], type_ignores=[])" + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)])" ) def test_dump_indent(self): @@ -1254,9 +1198,7 @@ def test_dump_indent(self): func=Name(id='spam', ctx=Load()), args=[ Name(id='eggs', ctx=Load()), - Constant(value='and cheese')], - keywords=[]))], - type_ignores=[])""") + Constant(value='and cheese')]))])""") self.assertEqual(ast.dump(node, annotate_fields=False, indent='\t'), """\ Module( \t[ @@ -1265,9 +1207,7 @@ def test_dump_indent(self): \t\t\t\tName('spam', Load()), \t\t\t\t[ \t\t\t\t\tName('eggs', Load()), -\t\t\t\t\tConstant('and cheese')], -\t\t\t\t[]))], -\t[])""") +\t\t\t\t\tConstant('and cheese')]))])""") self.assertEqual(ast.dump(node, include_attributes=True, indent=3), """\ Module( body=[ @@ -1294,7 +1234,6 @@ def test_dump_indent(self): col_offset=11, end_lineno=1, end_col_offset=23)], - keywords=[], lineno=1, col_offset=0, end_lineno=1, @@ -1302,8 +1241,7 @@ def test_dump_indent(self): lineno=1, col_offset=0, end_lineno=1, - end_col_offset=24)], - type_ignores=[])""") + end_col_offset=24)])""") def test_dump_incomplete(self): node = ast.Raise(lineno=3, col_offset=4) @@ -1333,6 +1271,119 @@ def test_dump_incomplete(self): self.assertEqual(ast.dump(node, annotate_fields=False), "Raise(cause=Name('e', Load()))" ) + # Arguments: + node = ast.arguments(args=[ast.arg("x")]) + self.assertEqual(ast.dump(node, annotate_fields=False), + "arguments([], [arg('x')])", + ) + node = ast.arguments(posonlyargs=[ast.arg("x")]) + self.assertEqual(ast.dump(node, annotate_fields=False), + "arguments([arg('x')])", + ) + node = ast.arguments(posonlyargs=[ast.arg("x")], kwonlyargs=[ast.arg('y')]) + self.assertEqual(ast.dump(node, annotate_fields=False), + "arguments([arg('x')], kwonlyargs=[arg('y')])", + ) + node = ast.arguments(args=[ast.arg("x")], kwonlyargs=[ast.arg('y')]) + self.assertEqual(ast.dump(node, annotate_fields=False), + "arguments([], [arg('x')], kwonlyargs=[arg('y')])", + ) + node = ast.arguments() + self.assertEqual(ast.dump(node, annotate_fields=False), + "arguments()", + ) + # Classes: + node = ast.ClassDef( + 'T', + [], + [ast.keyword('a', ast.Constant(None))], + [], + [ast.Name('dataclass', ctx=ast.Load())], + ) + self.assertEqual(ast.dump(node), + "ClassDef(name='T', keywords=[keyword(arg='a', value=Constant(value=None))], decorator_list=[Name(id='dataclass', ctx=Load())])", + ) + self.assertEqual(ast.dump(node, annotate_fields=False), + "ClassDef('T', [], [keyword('a', Constant(None))], [], [Name('dataclass', Load())])", + ) + + def test_dump_show_empty(self): + def check_node(node, empty, full, **kwargs): + with self.subTest(show_empty=False): + self.assertEqual( + ast.dump(node, show_empty=False, **kwargs), + empty, + ) + with self.subTest(show_empty=True): + self.assertEqual( + ast.dump(node, show_empty=True, **kwargs), + full, + ) + + def check_text(code, empty, full, **kwargs): + check_node(ast.parse(code), empty, full, **kwargs) + + check_node( + ast.arguments(), + empty="arguments()", + full="arguments(posonlyargs=[], args=[], kwonlyargs=[], kw_defaults=[], defaults=[])", + ) + + check_node( + # Corner case: there are no real `Name` instances with `id=''`: + ast.Name(id='', ctx=ast.Load()), + empty="Name(id='', ctx=Load())", + full="Name(id='', ctx=Load())", + ) + + check_node( + ast.MatchSingleton(value=None), + empty="MatchSingleton(value=None)", + full="MatchSingleton(value=None)", + ) + + check_node( + ast.Constant(value=None), + empty="Constant(value=None)", + full="Constant(value=None)", + ) + + check_node( + ast.Constant(value=''), + empty="Constant(value='')", + full="Constant(value='')", + ) + + check_text( + "def a(b: int = 0, *, c): ...", + empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', ctx=Load()))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))])])", + full="Module(body=[FunctionDef(name='a', args=arguments(posonlyargs=[], args=[arg(arg='b', annotation=Name(id='int', ctx=Load()))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))], decorator_list=[], type_params=[])], type_ignores=[])", + ) + + check_text( + "def a(b: int = 0, *, c): ...", + empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', ctx=Load(), lineno=1, col_offset=9, end_lineno=1, end_col_offset=12), lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], kwonlyargs=[arg(arg='c', lineno=1, col_offset=21, end_lineno=1, end_col_offset=22)], kw_defaults=[None], defaults=[Constant(value=0, lineno=1, col_offset=15, end_lineno=1, end_col_offset=16)]), body=[Expr(value=Constant(value=Ellipsis, lineno=1, col_offset=25, end_lineno=1, end_col_offset=28), lineno=1, col_offset=25, end_lineno=1, end_col_offset=28)], lineno=1, col_offset=0, end_lineno=1, end_col_offset=28)])", + full="Module(body=[FunctionDef(name='a', args=arguments(posonlyargs=[], args=[arg(arg='b', annotation=Name(id='int', ctx=Load(), lineno=1, col_offset=9, end_lineno=1, end_col_offset=12), lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], kwonlyargs=[arg(arg='c', lineno=1, col_offset=21, end_lineno=1, end_col_offset=22)], kw_defaults=[None], defaults=[Constant(value=0, lineno=1, col_offset=15, end_lineno=1, end_col_offset=16)]), body=[Expr(value=Constant(value=Ellipsis, lineno=1, col_offset=25, end_lineno=1, end_col_offset=28), lineno=1, col_offset=25, end_lineno=1, end_col_offset=28)], decorator_list=[], type_params=[], lineno=1, col_offset=0, end_lineno=1, end_col_offset=28)], type_ignores=[])", + include_attributes=True, + ) + + check_text( + 'spam(eggs, "and cheese")', + empty="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')]))])", + full="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')], keywords=[]))], type_ignores=[])", + ) + + check_text( + 'spam(eggs, text="and cheese")', + empty="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load())], keywords=[keyword(arg='text', value=Constant(value='and cheese'))]))])", + full="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load())], keywords=[keyword(arg='text', value=Constant(value='and cheese'))]))], type_ignores=[])", + ) + + check_text( + "import _ast as ast; from module import sub", + empty="Module(body=[Import(names=[alias(name='_ast', asname='ast')]), ImportFrom(module='module', names=[alias(name='sub')], level=0)])", + full="Module(body=[Import(names=[alias(name='_ast', asname='ast')]), ImportFrom(module='module', names=[alias(name='sub')], level=0)], type_ignores=[])", + ) def test_copy_location(self): src = ast.parse('1 + 1', mode='eval') @@ -1361,14 +1412,13 @@ def test_fix_missing_locations(self): "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=5), " "args=[Constant(value='spam', lineno=1, col_offset=6, end_lineno=1, " - "end_col_offset=12)], keywords=[], lineno=1, col_offset=0, end_lineno=1, " + "end_col_offset=12)], lineno=1, col_offset=0, end_lineno=1, " "end_col_offset=13), lineno=1, col_offset=0, end_lineno=1, " "end_col_offset=13), Expr(value=Call(func=Name(id='spam', ctx=Load(), " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), " "args=[Constant(value='eggs', lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=0)], keywords=[], lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)], " - "type_ignores=[])" + "end_col_offset=0)], lineno=1, col_offset=0, end_lineno=1, " + "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)])" ) def test_increment_lineno(self): @@ -1440,6 +1490,12 @@ def test_get_docstring(self): node = ast.parse('async def foo():\n """spam\n ham"""') self.assertEqual(ast.get_docstring(node.body[0]), 'spam\nham') + node = ast.parse('async def foo():\n """spam\n ham"""') + self.assertEqual(ast.get_docstring(node.body[0], clean=False), 'spam\n ham') + + node = ast.parse('x') + self.assertRaises(TypeError, ast.get_docstring, node.body[0]) + def test_get_docstring_none(self): self.assertIsNone(ast.get_docstring(ast.parse(''))) node = ast.parse('x = "not docstring"') @@ -1464,6 +1520,9 @@ def test_get_docstring_none(self): node = ast.parse('async def foo():\n x = "not docstring"') self.assertIsNone(ast.get_docstring(node.body[0])) + node = ast.parse('async def foo():\n 42') + self.assertIsNone(ast.get_docstring(node.body[0])) + def test_multi_line_docstring_col_offset_and_lineno_issue16806(self): node = ast.parse( '"""line one\nline two"""\n\n' @@ -1603,6 +1662,7 @@ def test_level_as_none(self): exec(code, ns) self.assertIn('sleep', ns) + @skip_emscripten_stack_overflow() def test_recursion_direct(self): e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) e.operand = e @@ -1610,6 +1670,7 @@ def test_recursion_direct(self): with support.infinite_recursion(): compile(ast.Expression(e), "", "eval") + @skip_emscripten_stack_overflow() def test_recursion_indirect(self): e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) @@ -1991,32 +2052,6 @@ def test_call(self): call = ast.Call(func, args, bad_keywords) self.expr(call, "must have Load context") - def test_num(self): - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - class subint(int): - pass - class subfloat(float): - pass - class subcomplex(complex): - pass - for obj in "0", "hello": - self.expr(ast.Num(obj)) - for obj in subint(), subfloat(), subcomplex(): - self.expr(ast.Num(obj), "invalid type", exc=TypeError) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - def test_attribute(self): attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()) self.expr(attr, "must have Load context") @@ -2056,31 +2091,17 @@ def test_list(self): def test_tuple(self): self._sequence(ast.Tuple) - def test_nameconstant(self): - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import NameConstant - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - self.expr(ast.NameConstant(4)) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - @support.requires_resource('cpu') def test_stdlib_validates(self): - stdlib = os.path.dirname(ast.__file__) - tests = [fn for fn in os.listdir(stdlib) if fn.endswith(".py")] - tests.extend(["test/test_grammar.py", "test/test_unpack_ex.py"]) - for module in tests: + for module in STDLIB_FILES: with self.subTest(module): - fn = os.path.join(stdlib, module) + fn = os.path.join(STDLIB, module) with open(fn, "r", encoding="utf-8") as fp: source = fp.read() mod = ast.parse(source, fn) compile(mod, fn, "exec") + mod2 = ast.parse(source, fn) + self.assertTrue(ast.compare(mod, mod2)) constant_1 = ast.Constant(1) pattern_1 = ast.MatchValue(constant_1) @@ -2258,7 +2279,7 @@ def test_validation(self): "got an invalid type in Constant: list") def test_singletons(self): - for const in (None, False, True, Ellipsis, b'', frozenset()): + for const in (None, False, True, Ellipsis, b''): with self.subTest(const=const): value = self.compile_constant(const) self.assertIs(value, const) @@ -2302,7 +2323,7 @@ def get_load_const(self, tree): co = compile(tree, '', 'exec') consts = [] for instr in dis.get_instructions(co): - if instr.opname == 'LOAD_CONST' or instr.opname == 'RETURN_CONST': + if instr.opcode in dis.hasconst: consts.append(instr.argval) return consts @@ -2310,7 +2331,7 @@ def get_load_const(self, tree): def test_load_const(self): consts = [None, True, False, - 124, + 1000, 2.0, 3j, "unicode", @@ -2722,76 +2743,15 @@ def test_source_segment_missing_info(self): self.assertIsNone(ast.get_source_segment(s, x)) self.assertIsNone(ast.get_source_segment(s, y)) -class BaseNodeVisitorCases: - # Both `NodeVisitor` and `NodeTranformer` must raise these warnings: - def test_old_constant_nodes(self): - class Visitor(self.visitor_class): - def visit_Num(self, node): - log.append((node.lineno, 'Num', node.n)) - def visit_Str(self, node): - log.append((node.lineno, 'Str', node.s)) - def visit_Bytes(self, node): - log.append((node.lineno, 'Bytes', node.s)) - def visit_NameConstant(self, node): - log.append((node.lineno, 'NameConstant', node.value)) - def visit_Ellipsis(self, node): - log.append((node.lineno, 'Ellipsis', ...)) - mod = ast.parse(dedent('''\ - i = 42 - f = 4.25 - c = 4.25j - s = 'string' - b = b'bytes' - t = True - n = None - e = ... - ''')) - visitor = Visitor() - log = [] - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - visitor.visit(mod) - self.assertEqual(log, [ - (1, 'Num', 42), - (2, 'Num', 4.25), - (3, 'Num', 4.25j), - (4, 'Str', 'string'), - (5, 'Bytes', b'bytes'), - (6, 'NameConstant', True), - (7, 'NameConstant', None), - (8, 'Ellipsis', ...), - ]) - self.assertEqual([str(w.message) for w in wlog], [ - 'visit_Num is deprecated; add visit_Constant', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Num is deprecated; add visit_Constant', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Num is deprecated; add visit_Constant', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Str is deprecated; add visit_Constant', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Bytes is deprecated; add visit_Constant', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'visit_NameConstant is deprecated; add visit_Constant', - 'visit_NameConstant is deprecated; add visit_Constant', - 'visit_Ellipsis is deprecated; add visit_Constant', - ]) - - -class NodeVisitorTests(BaseNodeVisitorCases, unittest.TestCase): - visitor_class = ast.NodeVisitor - - -class NodeTransformerTests(ASTTestMixin, BaseNodeVisitorCases, unittest.TestCase): - visitor_class = ast.NodeTransformer - - def assertASTTransformation(self, tranformer_class, + +class NodeTransformerTests(ASTTestMixin, unittest.TestCase): + def assertASTTransformation(self, transformer_class, initial_code, expected_code): initial_ast = ast.parse(dedent(initial_code)) expected_ast = ast.parse(dedent(expected_code)) - tranformer = tranformer_class() - result_ast = ast.fix_missing_locations(tranformer.visit(initial_ast)) + transformer = transformer_class() + result_ast = ast.fix_missing_locations(transformer.visit(initial_ast)) self.assertASTEqual(result_ast, expected_ast) @@ -2916,6 +2876,109 @@ def test_FunctionDef(self): self.assertEqual(node.name, 'foo') self.assertEqual(node.decorator_list, []) + def test_expr_context(self): + name = ast.Name("x") + self.assertEqual(name.id, "x") + self.assertIsInstance(name.ctx, ast.Load) + + name2 = ast.Name("x", ast.Store()) + self.assertEqual(name2.id, "x") + self.assertIsInstance(name2.ctx, ast.Store) + + name3 = ast.Name("x", ctx=ast.Del()) + self.assertEqual(name3.id, "x") + self.assertIsInstance(name3.ctx, ast.Del) + + with self.assertWarnsRegex(DeprecationWarning, + r"Name\.__init__ missing 1 required positional argument: 'id'"): + name3 = ast.Name() + + def test_custom_subclass_with_no_fields(self): + class NoInit(ast.AST): + pass + + obj = NoInit() + self.assertIsInstance(obj, NoInit) + self.assertEqual(obj.__dict__, {}) + + def test_fields_but_no_field_types(self): + class Fields(ast.AST): + _fields = ('a',) + + obj = Fields() + with self.assertRaises(AttributeError): + obj.a + obj = Fields(a=1) + self.assertEqual(obj.a, 1) + + def test_fields_and_types(self): + class FieldsAndTypes(ast.AST): + _fields = ('a',) + _field_types = {'a': int | None} + a: int | None = None + + obj = FieldsAndTypes() + self.assertIs(obj.a, None) + obj = FieldsAndTypes(a=1) + self.assertEqual(obj.a, 1) + + def test_custom_attributes(self): + class MyAttrs(ast.AST): + _attributes = ("a", "b") + + obj = MyAttrs(a=1, b=2) + self.assertEqual(obj.a, 1) + self.assertEqual(obj.b, 2) + + with self.assertWarnsRegex(DeprecationWarning, + r"MyAttrs.__init__ got an unexpected keyword argument 'c'."): + obj = MyAttrs(c=3) + + def test_fields_and_types_no_default(self): + class FieldsAndTypesNoDefault(ast.AST): + _fields = ('a',) + _field_types = {'a': int} + + with self.assertWarnsRegex(DeprecationWarning, + r"FieldsAndTypesNoDefault\.__init__ missing 1 required positional argument: 'a'\."): + obj = FieldsAndTypesNoDefault() + with self.assertRaises(AttributeError): + obj.a + obj = FieldsAndTypesNoDefault(a=1) + self.assertEqual(obj.a, 1) + + def test_incomplete_field_types(self): + class MoreFieldsThanTypes(ast.AST): + _fields = ('a', 'b') + _field_types = {'a': int | None} + a: int | None = None + b: int | None = None + + with self.assertWarnsRegex( + DeprecationWarning, + r"Field 'b' is missing from MoreFieldsThanTypes\._field_types" + ): + obj = MoreFieldsThanTypes() + self.assertIs(obj.a, None) + self.assertIs(obj.b, None) + + obj = MoreFieldsThanTypes(a=1, b=2) + self.assertEqual(obj.a, 1) + self.assertEqual(obj.b, 2) + + def test_complete_field_types(self): + class _AllFieldTypes(ast.AST): + _fields = ('a', 'b') + _field_types = {'a': int | None, 'b': list[str]} + # This must be set explicitly + a: int | None = None + # This will add an implicit empty list default + b: list[str] + + obj = _AllFieldTypes() + self.assertIs(obj.a, None) + self.assertEqual(obj.b, []) + @support.cpython_only class ModuleStateTests(unittest.TestCase): @@ -3018,154 +3081,303 @@ def test_cli_file_input(self): self.assertEqual(res.rc, 0) -def main(): - if __name__ != '__main__': - return - if sys.argv[1:] == ['-g']: - for statements, kind in ((exec_tests, "exec"), (single_tests, "single"), - (eval_tests, "eval")): - print(kind+"_results = [") - for statement in statements: - tree = ast.parse(statement, "?", kind) - print("%r," % (to_tuple(tree),)) - print("]") - print("main()") - raise SystemExit - unittest.main() +class ASTOptimiziationTests(unittest.TestCase): + binop = { + "+": ast.Add(), + "-": ast.Sub(), + "*": ast.Mult(), + "/": ast.Div(), + "%": ast.Mod(), + "<<": ast.LShift(), + ">>": ast.RShift(), + "|": ast.BitOr(), + "^": ast.BitXor(), + "&": ast.BitAnd(), + "//": ast.FloorDiv(), + "**": ast.Pow(), + } + + unaryop = { + "~": ast.Invert(), + "+": ast.UAdd(), + "-": ast.USub(), + } + + def wrap_expr(self, expr): + return ast.Module(body=[ast.Expr(value=expr)]) + + def wrap_statement(self, statement): + return ast.Module(body=[statement]) + + def assert_ast(self, code, non_optimized_target, optimized_target): + non_optimized_tree = ast.parse(code, optimize=-1) + optimized_tree = ast.parse(code, optimize=1) + + # Is a non-optimized tree equal to a non-optimized target? + self.assertTrue( + ast.compare(non_optimized_tree, non_optimized_target), + f"{ast.dump(non_optimized_target)} must equal " + f"{ast.dump(non_optimized_tree)}", + ) + + # Is a optimized tree equal to a non-optimized target? + self.assertFalse( + ast.compare(optimized_tree, non_optimized_target), + f"{ast.dump(non_optimized_target)} must not equal " + f"{ast.dump(non_optimized_tree)}" + ) + + # Is a optimized tree is equal to an optimized target? + self.assertTrue( + ast.compare(optimized_tree, optimized_target), + f"{ast.dump(optimized_target)} must equal " + f"{ast.dump(optimized_tree)}", + ) + + def create_binop(self, operand, left=ast.Constant(1), right=ast.Constant(1)): + return ast.BinOp(left=left, op=self.binop[operand], right=right) + + def test_folding_binop(self): + code = "1 %s 1" + operators = self.binop.keys() + + for op in operators: + result_code = code % op + non_optimized_target = self.wrap_expr(self.create_binop(op)) + optimized_target = self.wrap_expr(ast.Constant(value=eval(result_code))) -#### EVERYTHING BELOW IS GENERATED BY python Lib/test/test_ast.py -g ##### -exec_results = [ -('Module', [('Expr', (1, 0, 1, 4), ('Constant', (1, 0, 1, 4), None, None))], []), -('Module', [('Expr', (1, 0, 1, 18), ('Constant', (1, 0, 1, 18), 'module docstring', None))], []), -('Module', [('FunctionDef', (1, 0, 1, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 9, 1, 13))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 29), ('Constant', (1, 9, 1, 29), 'function docstring', None))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, []), [('Pass', (1, 10, 1, 14))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 0, None)]), [('Pass', (1, 12, 1, 16))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 11), 'args', None, None), [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 23), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 16), 'args', ('Starred', (1, 13, 1, 16), ('Name', (1, 14, 1, 16), 'Ts', ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 19, 1, 23))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Constant', (1, 25, 1, 28), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Starred', (1, 25, 1, 28), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 21), 'f', ('arguments', [], [], None, [], [], ('arg', (1, 8, 1, 14), 'kwargs', None, None), []), [('Pass', (1, 17, 1, 21))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 71), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None), ('arg', (1, 9, 1, 10), 'b', None, None), ('arg', (1, 14, 1, 15), 'c', None, None), ('arg', (1, 22, 1, 23), 'd', None, None), ('arg', (1, 28, 1, 29), 'e', None, None)], ('arg', (1, 35, 1, 39), 'args', None, None), [('arg', (1, 41, 1, 42), 'f', None, None)], [('Constant', (1, 43, 1, 45), 42, None)], ('arg', (1, 49, 1, 55), 'kwargs', None, None), [('Constant', (1, 11, 1, 12), 1, None), ('Constant', (1, 16, 1, 20), None, None), ('List', (1, 24, 1, 26), [], ('Load',)), ('Dict', (1, 30, 1, 32), [], [])]), [('Expr', (1, 58, 1, 71), ('Constant', (1, 58, 1, 71), 'doc for f()', None))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 27), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 23, 1, 27))], [], ('Subscript', (1, 11, 1, 21), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 20), [('Starred', (1, 17, 1, 20), ('Name', (1, 18, 1, 20), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 28, 1, 32))], [], ('Subscript', (1, 11, 1, 26), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 25), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 25), ('Name', (1, 23, 1, 25), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 45), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 41, 1, 45))], [], ('Subscript', (1, 11, 1, 39), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 38), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 38), ('Subscript', (1, 23, 1, 38), ('Name', (1, 23, 1, 28), 'tuple', ('Load',)), ('Tuple', (1, 29, 1, 37), [('Name', (1, 29, 1, 32), 'int', ('Load',)), ('Constant', (1, 34, 1, 37), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), -('Module', [('ClassDef', (1, 0, 1, 12), 'C', [], [], [('Pass', (1, 8, 1, 12))], [], [])], []), -('Module', [('ClassDef', (1, 0, 1, 32), 'C', [], [], [('Expr', (1, 9, 1, 32), ('Constant', (1, 9, 1, 32), 'docstring for class C', None))], [], [])], []), -('Module', [('ClassDef', (1, 0, 1, 21), 'C', [('Name', (1, 8, 1, 14), 'object', ('Load',))], [], [('Pass', (1, 17, 1, 21))], [], [])], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Return', (1, 8, 1, 16), ('Constant', (1, 15, 1, 16), 1, None))], [], None, None, [])], []), -('Module', [('Delete', (1, 0, 1, 5), [('Name', (1, 4, 1, 5), 'v', ('Del',))])], []), -('Module', [('Assign', (1, 0, 1, 5), [('Name', (1, 0, 1, 1), 'v', ('Store',))], ('Constant', (1, 4, 1, 5), 1, None), None)], []), -('Module', [('Assign', (1, 0, 1, 7), [('Tuple', (1, 0, 1, 3), [('Name', (1, 0, 1, 1), 'a', ('Store',)), ('Name', (1, 2, 1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 6, 1, 7), 'c', ('Load',)), None)], []), -('Module', [('Assign', (1, 0, 1, 9), [('Tuple', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), -('Module', [('Assign', (1, 0, 1, 9), [('List', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), -('Module', [('AnnAssign', (1, 0, 1, 13), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 13), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 12), [('Starred', (1, 9, 1, 12), ('Name', (1, 10, 1, 12), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), -('Module', [('AnnAssign', (1, 0, 1, 18), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 18), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 17), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 17), ('Name', (1, 15, 1, 17), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), -('Module', [('AnnAssign', (1, 0, 1, 31), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 31), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 30), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 30), ('Subscript', (1, 15, 1, 30), ('Name', (1, 15, 1, 20), 'tuple', ('Load',)), ('Tuple', (1, 21, 1, 29), [('Name', (1, 21, 1, 24), 'str', ('Load',)), ('Constant', (1, 26, 1, 29), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), -('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Add',), ('Constant', (1, 5, 1, 6), 1, None))], []), -('Module', [('For', (1, 0, 1, 15), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (1, 11, 1, 15))], [], None)], []), -('Module', [('While', (1, 0, 1, 12), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (1, 8, 1, 12))], [])], []), -('Module', [('If', (1, 0, 1, 9), ('Name', (1, 3, 1, 4), 'v', ('Load',)), [('Pass', (1, 5, 1, 9))], [])], []), -('Module', [('If', (1, 0, 4, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 4, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [])])], []), -('Module', [('If', (1, 0, 6, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 6, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [('Pass', (6, 2, 6, 6))])])], []), -('Module', [('With', (1, 0, 1, 17), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',)))], [('Pass', (1, 13, 1, 17))], None)], []), -('Module', [('With', (1, 0, 1, 25), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',))), ('withitem', ('Name', (1, 13, 1, 14), 'z', ('Load',)), ('Name', (1, 18, 1, 19), 'q', ('Store',)))], [('Pass', (1, 21, 1, 25))], None)], []), -('Module', [('With', (1, 0, 1, 19), [('withitem', ('Name', (1, 6, 1, 7), 'x', ('Load',)), ('Name', (1, 11, 1, 12), 'y', ('Store',)))], [('Pass', (1, 15, 1, 19))], None)], []), -('Module', [('With', (1, 0, 1, 17), [('withitem', ('Name', (1, 6, 1, 7), 'x', ('Load',)), None), ('withitem', ('Name', (1, 9, 1, 10), 'y', ('Load',)), None)], [('Pass', (1, 13, 1, 17))], None)], []), -('Module', [('Raise', (1, 0, 1, 25), ('Call', (1, 6, 1, 25), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), [('Constant', (1, 16, 1, 24), 'string', None)], []), None)], []), -('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), -('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [], [], [('Pass', (4, 2, 4, 6))])], []), -('Module', [('TryStar', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), -('Module', [('Assert', (1, 0, 1, 8), ('Name', (1, 7, 1, 8), 'v', ('Load',)), None)], []), -('Module', [('Import', (1, 0, 1, 10), [('alias', (1, 7, 1, 10), 'sys', None)])], []), -('Module', [('ImportFrom', (1, 0, 1, 17), 'sys', [('alias', (1, 16, 1, 17), 'v', None)], 0)], []), -('Module', [('Global', (1, 0, 1, 8), ['v'])], []), -('Module', [('Expr', (1, 0, 1, 1), ('Constant', (1, 0, 1, 1), 1, None))], []), -('Module', [('Pass', (1, 0, 1, 4))], []), -('Module', [('For', (1, 0, 1, 16), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Break', (1, 11, 1, 16))], [], None)], []), -('Module', [('For', (1, 0, 1, 19), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Continue', (1, 11, 1, 19))], [], None)], []), -('Module', [('For', (1, 0, 1, 18), ('Tuple', (1, 4, 1, 7), [('Name', (1, 4, 1, 5), 'a', ('Store',)), ('Name', (1, 6, 1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 11, 1, 12), 'c', ('Load',)), [('Pass', (1, 14, 1, 18))], [], None)], []), -('Module', [('For', (1, 0, 1, 20), ('Tuple', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), -('Module', [('For', (1, 0, 1, 20), ('List', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), -('Module', [('Expr', (1, 0, 11, 5), ('GeneratorExp', (1, 0, 11, 5), ('Tuple', (2, 4, 6, 5), [('Name', (3, 4, 3, 6), 'Aa', ('Load',)), ('Name', (5, 7, 5, 9), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4, 10, 6), [('Name', (8, 4, 8, 6), 'Aa', ('Store',)), ('Name', (10, 4, 10, 6), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10, 10, 12), 'Cc', ('Load',)), [], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 34), ('DictComp', (1, 0, 1, 34), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Name', (1, 11, 1, 12), 'w', ('Store',)), ('Name', (1, 16, 1, 17), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22, 1, 23), 'm', ('Store',)), ('Name', (1, 27, 1, 28), 'p', ('Load',)), [('Name', (1, 32, 1, 33), 'g', ('Load',))], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 20), ('DictComp', (1, 0, 1, 20), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'v', ('Store',)), ('Name', (1, 13, 1, 14), 'w', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'x', ('Load',)), [], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 19), ('SetComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 12, 1, 13), 'x', ('Load',)), [('Name', (1, 17, 1, 18), 'g', ('Load',))], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 16), ('SetComp', (1, 0, 1, 16), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7, 1, 10), [('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 9, 1, 10), 'm', ('Store',))], ('Store',)), ('Name', (1, 14, 1, 15), 'x', ('Load',)), [], 0)]))], []), -('Module', [('AsyncFunctionDef', (1, 0, 3, 18), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 17), ('Constant', (2, 1, 2, 17), 'async function', None)), ('Expr', (3, 1, 3, 18), ('Await', (3, 1, 3, 18), ('Call', (3, 7, 3, 18), ('Name', (3, 7, 3, 16), 'something', ('Load',)), [], [])))], [], None, None, [])], []), -('Module', [('AsyncFunctionDef', (1, 0, 3, 8), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncFor', (2, 1, 3, 8), ('Name', (2, 11, 2, 12), 'e', ('Store',)), ('Name', (2, 16, 2, 17), 'i', ('Load',)), [('Expr', (2, 19, 2, 20), ('Constant', (2, 19, 2, 20), 1, None))], [('Expr', (3, 7, 3, 8), ('Constant', (3, 7, 3, 8), 2, None))], None)], [], None, None, [])], []), -('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncWith', (2, 1, 2, 21), [('withitem', ('Name', (2, 12, 2, 13), 'a', ('Load',)), ('Name', (2, 17, 2, 18), 'b', ('Store',)))], [('Expr', (2, 20, 2, 21), ('Constant', (2, 20, 2, 21), 1, None))], None)], [], None, None, [])], []), -('Module', [('Expr', (1, 0, 1, 14), ('Dict', (1, 0, 1, 14), [None, ('Constant', (1, 10, 1, 11), 2, None)], [('Dict', (1, 3, 1, 8), [('Constant', (1, 4, 1, 5), 1, None)], [('Constant', (1, 6, 1, 7), 2, None)]), ('Constant', (1, 12, 1, 13), 3, None)]))], []), -('Module', [('Expr', (1, 0, 1, 12), ('Set', (1, 0, 1, 12), [('Starred', (1, 1, 1, 8), ('Set', (1, 2, 1, 8), [('Constant', (1, 3, 1, 4), 1, None), ('Constant', (1, 6, 1, 7), 2, None)]), ('Load',)), ('Constant', (1, 10, 1, 11), 3, None)]))], []), -('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 21), ('ListComp', (2, 1, 2, 21), ('Name', (2, 2, 2, 3), 'i', ('Load',)), [('comprehension', ('Name', (2, 14, 2, 15), 'b', ('Store',)), ('Name', (2, 19, 2, 20), 'c', ('Load',)), [], 1)]))], [], None, None, [])], []), -('Module', [('FunctionDef', (4, 0, 4, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), -('Module', [('AsyncFunctionDef', (4, 0, 4, 19), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 15, 4, 19))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), -('Module', [('ClassDef', (4, 0, 4, 13), 'C', [], [], [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], [])], []), -('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Call', (1, 1, 1, 19), ('Name', (1, 1, 1, 5), 'deco', ('Load',)), [('GeneratorExp', (1, 5, 1, 19), ('Name', (1, 6, 1, 7), 'a', ('Load',)), [('comprehension', ('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 17, 1, 18), 'b', ('Load',)), [], 0)])], [])], None, None, [])], []), -('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Attribute', (1, 1, 1, 6), ('Attribute', (1, 1, 1, 4), ('Name', (1, 1, 1, 2), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',))], None, None, [])], []), -('Module', [('Expr', (1, 0, 1, 8), ('NamedExpr', (1, 1, 1, 7), ('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Constant', (1, 6, 1, 7), 1, None)))], []), -('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None), ('arg', (1, 15, 1, 16), 'd', None, None), ('arg', (1, 18, 1, 19), 'e', None, None)], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], None, []), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 39), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], ('arg', (1, 26, 1, 32), 'kwargs', None, None), []), [('Pass', (1, 35, 1, 39))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 20), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None)]), [('Pass', (1, 16, 1, 20))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None), ('arg', (1, 19, 1, 20), 'c', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None), ('Constant', (1, 21, 1, 22), 4, None)]), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 28, 1, 32))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 30), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 26, 1, 30))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 42), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], ('arg', (1, 29, 1, 35), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 38, 1, 42))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 40), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], ('arg', (1, 27, 1, 33), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 36, 1, 40))], [], None, None, [])], []), -('Module', [('TypeAlias', (1, 0, 1, 12), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [], ('Name', (1, 9, 1, 12), 'int', ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 15), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None)], ('Name', (1, 12, 1, 15), 'int', ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 32), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None), ('TypeVarTuple', (1, 10, 1, 13), 'Ts'), ('ParamSpec', (1, 15, 1, 18), 'P')], ('Tuple', (1, 22, 1, 32), [('Name', (1, 23, 1, 24), 'T', ('Load',)), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Name', (1, 30, 1, 31), 'P', ('Load',))], ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 37), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 13), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',))), ('TypeVarTuple', (1, 15, 1, 18), 'Ts'), ('ParamSpec', (1, 20, 1, 23), 'P')], ('Tuple', (1, 27, 1, 37), [('Name', (1, 28, 1, 29), 'T', ('Load',)), ('Name', (1, 31, 1, 33), 'Ts', ('Load',)), ('Name', (1, 35, 1, 36), 'P', ('Load',))], ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 44), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 20), 'T', ('Tuple', (1, 10, 1, 20), [('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Name', (1, 16, 1, 19), 'str', ('Load',))], ('Load',))), ('TypeVarTuple', (1, 22, 1, 25), 'Ts'), ('ParamSpec', (1, 27, 1, 30), 'P')], ('Tuple', (1, 34, 1, 44), [('Name', (1, 35, 1, 36), 'T', ('Load',)), ('Name', (1, 38, 1, 40), 'Ts', ('Load',)), ('Name', (1, 42, 1, 43), 'P', ('Load',))], ('Load',)))], []), -('Module', [('ClassDef', (1, 0, 1, 16), 'X', [], [], [('Pass', (1, 12, 1, 16))], [], [('TypeVar', (1, 8, 1, 9), 'T', None)])], []), -('Module', [('ClassDef', (1, 0, 1, 26), 'X', [], [], [('Pass', (1, 22, 1, 26))], [], [('TypeVar', (1, 8, 1, 9), 'T', None), ('TypeVarTuple', (1, 11, 1, 14), 'Ts'), ('ParamSpec', (1, 16, 1, 19), 'P')])], []), -('Module', [('ClassDef', (1, 0, 1, 31), 'X', [], [], [('Pass', (1, 27, 1, 31))], [], [('TypeVar', (1, 8, 1, 14), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',))), ('TypeVarTuple', (1, 16, 1, 19), 'Ts'), ('ParamSpec', (1, 21, 1, 24), 'P')])], []), -('Module', [('ClassDef', (1, 0, 1, 38), 'X', [], [], [('Pass', (1, 34, 1, 38))], [], [('TypeVar', (1, 8, 1, 21), 'T', ('Tuple', (1, 11, 1, 21), [('Name', (1, 12, 1, 15), 'int', ('Load',)), ('Name', (1, 17, 1, 20), 'str', ('Load',))], ('Load',))), ('TypeVarTuple', (1, 23, 1, 26), 'Ts'), ('ParamSpec', (1, 28, 1, 31), 'P')])], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 12, 1, 16))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None)])], []), -('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None), ('TypeVarTuple', (1, 9, 1, 12), 'Ts'), ('ParamSpec', (1, 14, 1, 17), 'P')])], []), -('Module', [('FunctionDef', (1, 0, 1, 31), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 27, 1, 31))], [], None, None, [('TypeVar', (1, 6, 1, 12), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',))), ('TypeVarTuple', (1, 14, 1, 17), 'Ts'), ('ParamSpec', (1, 19, 1, 22), 'P')])], []), -('Module', [('FunctionDef', (1, 0, 1, 38), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 34, 1, 38))], [], None, None, [('TypeVar', (1, 6, 1, 19), 'T', ('Tuple', (1, 9, 1, 19), [('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Name', (1, 15, 1, 18), 'str', ('Load',))], ('Load',))), ('TypeVarTuple', (1, 21, 1, 24), 'Ts'), ('ParamSpec', (1, 26, 1, 29), 'P')])], []), -] -single_results = [ -('Interactive', [('Expr', (1, 0, 1, 3), ('BinOp', (1, 0, 1, 3), ('Constant', (1, 0, 1, 1), 1, None), ('Add',), ('Constant', (1, 2, 1, 3), 2, None)))]), -] -eval_results = [ -('Expression', ('Constant', (1, 0, 1, 4), None, None)), -('Expression', ('BoolOp', (1, 0, 1, 7), ('And',), [('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 6, 1, 7), 'b', ('Load',))])), -('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Add',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), -('Expression', ('UnaryOp', (1, 0, 1, 5), ('Not',), ('Name', (1, 4, 1, 5), 'v', ('Load',)))), -('Expression', ('Lambda', (1, 0, 1, 11), ('arguments', [], [], None, [], [], None, []), ('Constant', (1, 7, 1, 11), None, None))), -('Expression', ('Dict', (1, 0, 1, 7), [('Constant', (1, 2, 1, 3), 1, None)], [('Constant', (1, 4, 1, 5), 2, None)])), -('Expression', ('Dict', (1, 0, 1, 2), [], [])), -('Expression', ('Set', (1, 0, 1, 7), [('Constant', (1, 1, 1, 5), None, None)])), -('Expression', ('Dict', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None)], [('Constant', (4, 10, 4, 11), 2, None)])), -('Expression', ('ListComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), -('Expression', ('ListComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), -('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('SetComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), -('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('Compare', (1, 0, 1, 9), ('Constant', (1, 0, 1, 1), 1, None), [('Lt',), ('Lt',)], [('Constant', (1, 4, 1, 5), 2, None), ('Constant', (1, 8, 1, 9), 3, None)])), -('Expression', ('Call', (1, 0, 1, 17), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Constant', (1, 2, 1, 3), 1, None), ('Constant', (1, 4, 1, 5), 2, None), ('Starred', (1, 10, 1, 12), ('Name', (1, 11, 1, 12), 'd', ('Load',)), ('Load',))], [('keyword', (1, 6, 1, 9), 'c', ('Constant', (1, 8, 1, 9), 3, None)), ('keyword', (1, 13, 1, 16), None, ('Name', (1, 15, 1, 16), 'e', ('Load',)))])), -('Expression', ('Call', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Starred', (1, 2, 1, 9), ('List', (1, 3, 1, 9), [('Constant', (1, 4, 1, 5), 0, None), ('Constant', (1, 7, 1, 8), 1, None)], ('Load',)), ('Load',))], [])), -('Expression', ('Call', (1, 0, 1, 15), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('GeneratorExp', (1, 1, 1, 15), ('Name', (1, 2, 1, 3), 'a', ('Load',)), [('comprehension', ('Name', (1, 8, 1, 9), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Load',)), [], 0)])], [])), -('Expression', ('Constant', (1, 0, 1, 2), 10, None)), -('Expression', ('Constant', (1, 0, 1, 8), 'string', None)), -('Expression', ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',))), -('Expression', ('Subscript', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Slice', (1, 2, 1, 5), ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Name', (1, 4, 1, 5), 'c', ('Load',)), None), ('Load',))), -('Expression', ('Name', (1, 0, 1, 1), 'v', ('Load',))), -('Expression', ('List', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), -('Expression', ('List', (1, 0, 1, 2), [], ('Load',))), -('Expression', ('Tuple', (1, 0, 1, 5), [('Constant', (1, 0, 1, 1), 1, None), ('Constant', (1, 2, 1, 3), 2, None), ('Constant', (1, 4, 1, 5), 3, None)], ('Load',))), -('Expression', ('Tuple', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), -('Expression', ('Tuple', (1, 0, 1, 2), [], ('Load',))), -('Expression', ('Call', (1, 0, 1, 17), ('Attribute', (1, 0, 1, 7), ('Attribute', (1, 0, 1, 5), ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8, 1, 16), ('Attribute', (1, 8, 1, 11), ('Name', (1, 8, 1, 9), 'a', ('Load',)), 'b', ('Load',)), ('Slice', (1, 12, 1, 15), ('Constant', (1, 12, 1, 13), 1, None), ('Constant', (1, 14, 1, 15), 2, None), None), ('Load',))], [])), -] -main() + with self.subTest( + result_code=result_code, + non_optimized_target=non_optimized_target, + optimized_target=optimized_target + ): + self.assert_ast(result_code, non_optimized_target, optimized_target) + + # Multiplication of constant tuples must be folded + code = "(1,) * 3" + non_optimized_target = self.wrap_expr(self.create_binop("*", ast.Tuple(elts=[ast.Constant(value=1)]), ast.Constant(value=3))) + optimized_target = self.wrap_expr(ast.Constant(eval(code))) + + self.assert_ast(code, non_optimized_target, optimized_target) + + def test_folding_unaryop(self): + code = "%s1" + operators = self.unaryop.keys() + + def create_unaryop(operand): + return ast.UnaryOp(op=self.unaryop[operand], operand=ast.Constant(1)) + + for op in operators: + result_code = code % op + non_optimized_target = self.wrap_expr(create_unaryop(op)) + optimized_target = self.wrap_expr(ast.Constant(eval(result_code))) + + with self.subTest( + result_code=result_code, + non_optimized_target=non_optimized_target, + optimized_target=optimized_target + ): + self.assert_ast(result_code, non_optimized_target, optimized_target) + + def test_folding_not(self): + code = "not (1 %s (1,))" + operators = { + "in": ast.In(), + "is": ast.Is(), + } + opt_operators = { + "is": ast.IsNot(), + "in": ast.NotIn(), + } + + def create_notop(operand): + return ast.UnaryOp(op=ast.Not(), operand=ast.Compare( + left=ast.Constant(value=1), + ops=[operators[operand]], + comparators=[ast.Tuple(elts=[ast.Constant(value=1)])] + )) + + for op in operators.keys(): + result_code = code % op + non_optimized_target = self.wrap_expr(create_notop(op)) + optimized_target = self.wrap_expr( + ast.Compare(left=ast.Constant(1), ops=[opt_operators[op]], comparators=[ast.Constant(value=(1,))]) + ) + + with self.subTest( + result_code=result_code, + non_optimized_target=non_optimized_target, + optimized_target=optimized_target + ): + self.assert_ast(result_code, non_optimized_target, optimized_target) + + def test_folding_format(self): + code = "'%s' % (a,)" + + non_optimized_target = self.wrap_expr( + ast.BinOp( + left=ast.Constant(value="%s"), + op=ast.Mod(), + right=ast.Tuple(elts=[ast.Name(id='a')])) + ) + optimized_target = self.wrap_expr( + ast.JoinedStr( + values=[ + ast.FormattedValue(value=ast.Name(id='a'), conversion=115) + ] + ) + ) + + self.assert_ast(code, non_optimized_target, optimized_target) + + + def test_folding_tuple(self): + code = "(1,)" + + non_optimized_target = self.wrap_expr(ast.Tuple(elts=[ast.Constant(1)])) + optimized_target = self.wrap_expr(ast.Constant(value=(1,))) + + self.assert_ast(code, non_optimized_target, optimized_target) + + def test_folding_comparator(self): + code = "1 %s %s1%s" + operators = [("in", ast.In()), ("not in", ast.NotIn())] + braces = [ + ("[", "]", ast.List, (1,)), + ("{", "}", ast.Set, frozenset({1})), + ] + for left, right, non_optimized_comparator, optimized_comparator in braces: + for op, node in operators: + non_optimized_target = self.wrap_expr(ast.Compare( + left=ast.Constant(1), ops=[node], + comparators=[non_optimized_comparator(elts=[ast.Constant(1)])] + )) + optimized_target = self.wrap_expr(ast.Compare( + left=ast.Constant(1), ops=[node], + comparators=[ast.Constant(value=optimized_comparator)] + )) + self.assert_ast(code % (op, left, right), non_optimized_target, optimized_target) + + def test_folding_iter(self): + code = "for _ in %s1%s: pass" + braces = [ + ("[", "]", ast.List, (1,)), + ("{", "}", ast.Set, frozenset({1})), + ] + + for left, right, ast_cls, optimized_iter in braces: + non_optimized_target = self.wrap_statement(ast.For( + target=ast.Name(id="_", ctx=ast.Store()), + iter=ast_cls(elts=[ast.Constant(1)]), + body=[ast.Pass()] + )) + optimized_target = self.wrap_statement(ast.For( + target=ast.Name(id="_", ctx=ast.Store()), + iter=ast.Constant(value=optimized_iter), + body=[ast.Pass()] + )) + + self.assert_ast(code % (left, right), non_optimized_target, optimized_target) + + def test_folding_subscript(self): + code = "(1,)[0]" + + non_optimized_target = self.wrap_expr( + ast.Subscript(value=ast.Tuple(elts=[ast.Constant(value=1)]), slice=ast.Constant(value=0)) + ) + optimized_target = self.wrap_expr(ast.Constant(value=1)) + + self.assert_ast(code, non_optimized_target, optimized_target) + + def test_folding_type_param_in_function_def(self): + code = "def foo[%s = 1 + 1](): pass" + + unoptimized_binop = self.create_binop("+") + unoptimized_type_params = [ + ("T", "T", ast.TypeVar), + ("**P", "P", ast.ParamSpec), + ("*Ts", "Ts", ast.TypeVarTuple), + ] + + for type, name, type_param in unoptimized_type_params: + result_code = code % type + optimized_target = self.wrap_statement( + ast.FunctionDef( + name='foo', + args=ast.arguments(), + body=[ast.Pass()], + type_params=[type_param(name=name, default_value=ast.Constant(2))] + ) + ) + non_optimized_target = self.wrap_statement( + ast.FunctionDef( + name='foo', + args=ast.arguments(), + body=[ast.Pass()], + type_params=[type_param(name=name, default_value=unoptimized_binop)] + ) + ) + self.assert_ast(result_code, non_optimized_target, optimized_target) + + def test_folding_type_param_in_class_def(self): + code = "class foo[%s = 1 + 1]: pass" + + unoptimized_binop = self.create_binop("+") + unoptimized_type_params = [ + ("T", "T", ast.TypeVar), + ("**P", "P", ast.ParamSpec), + ("*Ts", "Ts", ast.TypeVarTuple), + ] + + for type, name, type_param in unoptimized_type_params: + result_code = code % type + optimized_target = self.wrap_statement( + ast.ClassDef( + name='foo', + body=[ast.Pass()], + type_params=[type_param(name=name, default_value=ast.Constant(2))] + ) + ) + non_optimized_target = self.wrap_statement( + ast.ClassDef( + name='foo', + body=[ast.Pass()], + type_params=[type_param(name=name, default_value=unoptimized_binop)] + ) + ) + self.assert_ast(result_code, non_optimized_target, optimized_target) + + def test_folding_type_param_in_type_alias(self): + code = "type foo[%s = 1 + 1] = 1" + + unoptimized_binop = self.create_binop("+") + unoptimized_type_params = [ + ("T", "T", ast.TypeVar), + ("**P", "P", ast.ParamSpec), + ("*Ts", "Ts", ast.TypeVarTuple), + ] + + for type, name, type_param in unoptimized_type_params: + result_code = code % type + optimized_target = self.wrap_statement( + ast.TypeAlias( + name=ast.Name(id='foo', ctx=ast.Store()), + type_params=[type_param(name=name, default_value=ast.Constant(2))], + value=ast.Constant(value=1), + ) + ) + non_optimized_target = self.wrap_statement( + ast.TypeAlias( + name=ast.Name(id='foo', ctx=ast.Store()), + type_params=[type_param(name=name, default_value=unoptimized_binop)], + value=ast.Constant(value=1), + ) + ) + self.assert_ast(result_code, non_optimized_target, optimized_target) + + +if __name__ == '__main__': + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + ast_repr_update_snapshots() + sys.exit(0) + unittest.main() diff --git a/Lib/test/test_ast/utils.py b/Lib/test/test_ast/utils.py new file mode 100644 index 00000000000000..145e89ee94e935 --- /dev/null +++ b/Lib/test/test_ast/utils.py @@ -0,0 +1,15 @@ +def to_tuple(t): + if t is None or isinstance(t, (str, int, complex, float, bytes)) or t is Ellipsis: + return t + elif isinstance(t, list): + return [to_tuple(e) for e in t] + result = [t.__class__.__name__] + if hasattr(t, 'lineno') and hasattr(t, 'col_offset'): + result.append((t.lineno, t.col_offset)) + if hasattr(t, 'end_lineno') and hasattr(t, 'end_col_offset'): + result[-1] += (t.end_lineno, t.end_col_offset) + if t._fields is None: + return tuple(result) + for f in t._fields: + result.append(to_tuple(getattr(t, f))) + return tuple(result) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 39605dca3886c8..b81187871753b9 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -393,15 +393,159 @@ async def gen(): r'anext\(\): asynchronous generator is already running'): an.__next__() + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + an.send(None) + + def test_async_gen_asend_throw_concurrent_with_send(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + while True: + try: + await _async_yield(None) + except MyExc: + pass + return + yield + + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.asend(None) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + gen2.send(None) + + def test_async_gen_athrow_throw_concurrent_with_send(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + while True: + try: + await _async_yield(None) + except MyExc: + pass + return + yield + + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.athrow(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)"): + gen2.send(None) + + def test_async_gen_asend_throw_concurrent_with_throw(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + pass + while True: + try: + await _async_yield(None) + except MyExc: + pass + + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + + gen = agen.athrow(MyExc) + gen.throw(MyExc) + gen2 = agen.asend(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + gen2.send(None) + + def test_async_gen_athrow_throw_concurrent_with_throw(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + pass + while True: + try: + await _async_yield(None) + except MyExc: + pass + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + + gen = agen.athrow(MyExc) + gen.throw(MyExc) + gen2 = agen.athrow(None) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)"): + gen2.send(None) + def test_async_gen_3_arg_deprecation_warning(self): async def gen(): yield 123 with self.assertWarns(DeprecationWarning): x = gen().athrow(GeneratorExit, GeneratorExit(), None) - with self.assertWarnsRegex(RuntimeWarning, - f"coroutine method 'athrow' of '{gen.__qualname__}' " - f"was never awaited"): + with self.assertRaises(GeneratorExit): + x.send(None) del x gc_collect() @@ -427,17 +571,65 @@ async def gen(): self.assertTrue(inspect.isawaitable(aclose)) aclose.close() + def test_async_gen_asend_close_runtime_error(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + try: + await _async_yield(None) + except GeneratorExit: + await _async_yield(None) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + with self.assertRaisesRegex(RuntimeError, "coroutine ignored GeneratorExit"): + gen.close() + + def test_async_gen_athrow_close_runtime_error(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + try: + await _async_yield(None) + except GeneratorExit: + await _async_yield(None) + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + gen = agen.athrow(MyExc) + gen.send(None) + with self.assertRaisesRegex(RuntimeError, "coroutine ignored GeneratorExit"): + gen.close() + class AsyncGenAsyncioTest(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) def tearDown(self): self.loop.close() self.loop = None - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) def check_async_iterator_anext(self, ait_class): with self.subTest(anext="pure-Python"): @@ -960,6 +1152,23 @@ async def run(): self.loop.run_until_complete(run()) + def test_async_gen_asyncio_anext_tuple_no_exceptions(self): + # StopAsyncIteration exceptions should be cleared. + # See: https://github.com/python/cpython/issues/128078. + + async def foo(): + if False: + yield (1, 2) + + async def run(): + it = foo().__aiter__() + with self.assertRaises(StopAsyncIteration): + await it.__anext__() + res = await anext(it, ('a', 'b')) + self.assertTupleEqual(res, ('a', 'b')) + + self.loop.run_until_complete(run()) + def test_async_gen_asyncio_anext_stopiteration(self): async def foo(): try: @@ -1572,11 +1781,8 @@ async def main(): self.assertIsInstance(message['exception'], ZeroDivisionError) self.assertIn('unhandled exception during asyncio.run() shutdown', message['message']) - with self.assertWarnsRegex(RuntimeWarning, - f"coroutine method 'aclose' of '{async_iterate.__qualname__}' " - f"was never awaited"): - del message, messages - gc_collect() + del message, messages + gc_collect() def test_async_gen_expression_01(self): async def arange(n): @@ -1630,10 +1836,7 @@ async def main(): asyncio.run(main()) self.assertEqual([], messages) - with self.assertWarnsRegex(RuntimeWarning, - f"coroutine method 'aclose' of '{async_iterate.__qualname__}' " - f"was never awaited"): - gc_collect() + gc_collect() def test_async_gen_await_same_anext_coro_twice(self): async def async_iterate(): @@ -1671,6 +1874,62 @@ async def run(): self.loop.run_until_complete(run()) + def test_async_gen_throw_same_aclose_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + it = async_iterate() + nxt = it.aclose() + with self.assertRaises(StopIteration): + nxt.throw(GeneratorExit) + + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + nxt.throw(GeneratorExit) + + def test_async_gen_throw_custom_same_aclose_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + it = async_iterate() + + class MyException(Exception): + pass + + nxt = it.aclose() + with self.assertRaises(MyException): + nxt.throw(MyException) + + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + nxt.throw(MyException) + + def test_async_gen_throw_custom_same_athrow_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + it = async_iterate() + + class MyException(Exception): + pass + + nxt = it.athrow(MyException) + with self.assertRaises(MyException): + nxt.throw(MyException) + + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + nxt.throw(MyException) + def test_async_gen_aclose_twice_with_different_coros(self): # Regression test for https://bugs.python.org/issue39606 async def async_iterate(): @@ -1752,7 +2011,67 @@ async def gen(): g.aclose() gc_collect() + def test_aclose_throw(self): + async def gen(): + return + yield + + class MyException(Exception): + pass + + g = gen() + with self.assertRaises(MyException): + g.aclose().throw(MyException) + + del g + gc_collect() # does not warn unawaited + + def test_asend_send_already_running(self): + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + while True: + await _async_yield(1) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.asend(None) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.send(None) + + del gen2 + gc_collect() # does not warn unawaited + + + def test_athrow_send_already_running(self): + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + while True: + await _async_yield(1) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.athrow(Exception) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.send(None) + del gen2 + gc_collect() # does not warn unawaited if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_asyncio/functional.py b/Lib/test/test_asyncio/functional.py index d19c7a612ccf86..2934325b6dfbc7 100644 --- a/Lib/test/test_asyncio/functional.py +++ b/Lib/test/test_asyncio/functional.py @@ -24,7 +24,7 @@ def loop_exception_handler(self, loop, context): def setUp(self): self.loop = self.new_loop() - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) self.loop.set_exception_handler(self.loop_exception_handler) self.__unhandled_exceptions = [] @@ -39,7 +39,7 @@ def tearDown(self): self.fail('unexpected calls to loop.call_exception_handler()') finally: - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) self.loop = None def tcp_server(self, server_prog, *, diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 4cd872d3a5b2d8..8cf1f6891faf97 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -3,6 +3,7 @@ import concurrent.futures import errno import math +import platform import socket import sys import threading @@ -24,7 +25,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) def mock_socket_module(): @@ -330,10 +331,10 @@ def check_in_thread(loop, event, debug, create_loop, fut): if create_loop: loop2 = base_events.BaseEventLoop() try: - asyncio.set_event_loop(loop2) + asyncio._set_event_loop(loop2) self.check_thread(loop, debug) finally: - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) loop2.close() else: self.check_thread(loop, debug) @@ -689,7 +690,7 @@ def default_exception_handler(self, context): loop = Loop() self.addCleanup(loop.close) - asyncio.set_event_loop(loop) + asyncio._set_event_loop(loop) def run_loop(): def zero_error(): @@ -832,8 +833,8 @@ async def test(): loop.close() def test_create_named_task_with_custom_factory(self): - def task_factory(loop, coro): - return asyncio.Task(coro, loop=loop) + def task_factory(loop, coro, **kwargs): + return asyncio.Task(coro, loop=loop, **kwargs) async def test(): pass @@ -1344,7 +1345,7 @@ def getaddrinfo_task(*args, **kwds): with self.assertRaises(OSError) as cm: self.loop.run_until_complete(coro) - self.assertTrue(str(cm.exception).startswith('Multiple exceptions: ')) + self.assertStartsWith(str(cm.exception), 'Multiple exceptions: ') self.assertTrue(m_socket.socket.return_value.close.called) coro = self.loop.create_connection( @@ -1430,6 +1431,10 @@ def test_create_connection_no_inet_pton(self, m_socket): self._test_create_connection_ip_addr(m_socket, False) @patch_socket + @unittest.skipIf( + support.is_android and platform.android_ver().api_level < 23, + "Issue gh-71123: this fails on Android before API level 23" + ) def test_create_connection_service_name(self, m_socket): m_socket.getaddrinfo = socket.getaddrinfo sock = m_socket.socket.return_value @@ -1978,7 +1983,7 @@ def stop_loop_cb(loop): async def stop_loop_coro(loop): loop.stop() - asyncio.set_event_loop(self.loop) + asyncio._set_event_loop(self.loop) self.loop.set_debug(True) self.loop.slow_callback_duration = 0.0 diff --git a/Lib/test/test_asyncio/test_buffered_proto.py b/Lib/test/test_asyncio/test_buffered_proto.py index f24e363ebfcfa3..9c386dd2e63815 100644 --- a/Lib/test/test_asyncio/test_buffered_proto.py +++ b/Lib/test/test_asyncio/test_buffered_proto.py @@ -5,7 +5,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class ReceiveStuffProto(asyncio.BufferedProtocol): diff --git a/Lib/test/test_asyncio/test_context.py b/Lib/test/test_asyncio/test_context.py index 6b80721873d95c..ad394f44e7e5f6 100644 --- a/Lib/test/test_asyncio/test_context.py +++ b/Lib/test/test_asyncio/test_context.py @@ -4,7 +4,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) @unittest.skipUnless(decimal.HAVE_CONTEXTVAR, "decimal is built with a thread-local context") diff --git a/Lib/test/test_asyncio/test_eager_task_factory.py b/Lib/test/test_asyncio/test_eager_task_factory.py index 0f8212dbec47be..10450c11b68279 100644 --- a/Lib/test/test_asyncio/test_eager_task_factory.py +++ b/Lib/test/test_asyncio/test_eager_task_factory.py @@ -13,7 +13,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class EagerTaskFactoryLoopTests: @@ -213,6 +213,56 @@ async def run(): self.run_coro(run()) + def test_staggered_race_with_eager_tasks(self): + # See https://github.com/python/cpython/issues/124309 + + async def fail(): + await asyncio.sleep(0) + raise ValueError("no good") + + async def blocked(): + fut = asyncio.Future() + await fut + + async def run(): + winner, index, excs = await asyncio.staggered.staggered_race( + [ + lambda: blocked(), + lambda: asyncio.sleep(1, result="sleep1"), + lambda: fail() + ], + delay=0.25 + ) + self.assertEqual(winner, 'sleep1') + self.assertEqual(index, 1) + self.assertIsNone(excs[index]) + self.assertIsInstance(excs[0], asyncio.CancelledError) + self.assertIsInstance(excs[2], ValueError) + + self.run_coro(run()) + + def test_staggered_race_with_eager_tasks_no_delay(self): + # See https://github.com/python/cpython/issues/124309 + async def fail(): + raise ValueError("no good") + + async def run(): + winner, index, excs = await asyncio.staggered.staggered_race( + [ + lambda: fail(), + lambda: asyncio.sleep(1, result="sleep1"), + lambda: asyncio.sleep(0, result="sleep0"), + ], + delay=None + ) + self.assertEqual(winner, 'sleep1') + self.assertEqual(index, 1) + self.assertIsNone(excs[index]) + self.assertIsInstance(excs[0], ValueError) + self.assertEqual(len(excs), 2) + + self.run_coro(run()) + class PyEagerTaskFactoryLoopTests(EagerTaskFactoryLoopTests, test_utils.TestCase): Task = tasks._PyTask @@ -241,6 +291,30 @@ class DummyLoop: _, out, err = assert_python_ok("-c", code) self.assertFalse(err) + def test_issue122332(self): + async def coro(): + pass + + async def run(): + task = self.loop.create_task(coro()) + await task + self.assertIsNone(task.get_coro()) + + self.run_coro(run()) + + def test_name(self): + name = None + async def coro(): + nonlocal name + name = asyncio.current_task().get_name() + + async def main(): + task = self.loop.create_task(coro(), name="test name") + self.assertEqual(name, "test name") + await task + + self.run_coro(coro()) + class AsyncTaskCounter: def __init__(self, loop, *, task_class, eager): self.suspense_count = 0 diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 5b9c871e1d1b5a..ada049e9c7d387 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -36,10 +36,10 @@ from test.support import socket_helper from test.support import threading_helper from test.support import ALWAYS_EQ, LARGEST, SMALLEST - +from test.support import warnings_helper def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) def broken_unix_getsockname(): @@ -58,7 +58,7 @@ async def doit(): return 'hello' loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + asyncio._set_event_loop(loop) return loop.run_until_complete(doit()) @@ -353,6 +353,124 @@ def run_in_thread(): t.join() self.assertEqual(results, ['hello', 'world']) + def test_call_soon_threadsafe_handle_block_check_cancelled(self): + results = [] + + callback_started = threading.Event() + callback_finished = threading.Event() + def callback(arg): + callback_started.set() + results.append(arg) + time.sleep(1) + callback_finished.set() + + def run_in_thread(): + handle = self.loop.call_soon_threadsafe(callback, 'hello') + self.assertIsInstance(handle, events._ThreadSafeHandle) + callback_started.wait() + # callback started so it should block checking for cancellation + # until it finishes + self.assertFalse(handle.cancelled()) + self.assertTrue(callback_finished.is_set()) + self.loop.call_soon_threadsafe(self.loop.stop) + + t = threading.Thread(target=run_in_thread) + t.start() + + self.loop.run_forever() + t.join() + self.assertEqual(results, ['hello']) + + def test_call_soon_threadsafe_handle_block_cancellation(self): + results = [] + + callback_started = threading.Event() + callback_finished = threading.Event() + def callback(arg): + callback_started.set() + results.append(arg) + time.sleep(1) + callback_finished.set() + + def run_in_thread(): + handle = self.loop.call_soon_threadsafe(callback, 'hello') + self.assertIsInstance(handle, events._ThreadSafeHandle) + callback_started.wait() + # callback started so it cannot be cancelled from other thread until + # it finishes + handle.cancel() + self.assertTrue(callback_finished.is_set()) + self.loop.call_soon_threadsafe(self.loop.stop) + + t = threading.Thread(target=run_in_thread) + t.start() + + self.loop.run_forever() + t.join() + self.assertEqual(results, ['hello']) + + def test_call_soon_threadsafe_handle_cancel_same_thread(self): + results = [] + callback_started = threading.Event() + callback_finished = threading.Event() + + fut = concurrent.futures.Future() + def callback(arg): + callback_started.set() + handle = fut.result() + handle.cancel() + results.append(arg) + callback_finished.set() + self.loop.stop() + + def run_in_thread(): + handle = self.loop.call_soon_threadsafe(callback, 'hello') + fut.set_result(handle) + self.assertIsInstance(handle, events._ThreadSafeHandle) + callback_started.wait() + # callback cancels itself from same thread so it has no effect + # it runs to completion + self.assertTrue(handle.cancelled()) + self.assertTrue(callback_finished.is_set()) + self.loop.call_soon_threadsafe(self.loop.stop) + + t = threading.Thread(target=run_in_thread) + t.start() + + self.loop.run_forever() + t.join() + self.assertEqual(results, ['hello']) + + def test_call_soon_threadsafe_handle_cancel_other_thread(self): + results = [] + ev = threading.Event() + + callback_finished = threading.Event() + def callback(arg): + results.append(arg) + callback_finished.set() + self.loop.stop() + + def run_in_thread(): + handle = self.loop.call_soon_threadsafe(callback, 'hello') + # handle can be cancelled from other thread if not started yet + self.assertIsInstance(handle, events._ThreadSafeHandle) + handle.cancel() + self.assertTrue(handle.cancelled()) + self.assertFalse(callback_finished.is_set()) + ev.set() + self.loop.call_soon_threadsafe(self.loop.stop) + + # block the main loop until the callback is added and cancelled in the + # other thread + self.loop.call_soon(ev.wait) + t = threading.Thread(target=run_in_thread) + t.start() + self.loop.run_forever() + t.join() + self.assertEqual(results, []) + self.assertFalse(callback_finished.is_set()) + def test_call_soon_threadsafe_same_thread(self): results = [] @@ -1125,12 +1243,16 @@ def test_create_server_ssl_match_failed(self): # incorrect server_hostname f_c = self.loop.create_connection(MyProto, host, port, ssl=sslcontext_client) + + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + IP address mismatch, certificate is not valid for '127.0.0.1' # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) with mock.patch.object(self.loop, 'call_exception_handler'): with test_utils.disable_logger(): - with self.assertRaisesRegex( - ssl.CertificateError, - "IP address mismatch, certificate is not valid for " - "'127.0.0.1'"): + with self.assertRaisesRegex(ssl.CertificateError, regex): self.loop.run_until_complete(f_c) # close connection @@ -1374,6 +1496,80 @@ def test_create_datagram_endpoint_sock(self): tr.close() self.loop.run_until_complete(pr.done) + def test_datagram_send_to_non_listening_address(self): + # see: + # https://github.com/python/cpython/issues/91227 + # https://github.com/python/cpython/issues/88906 + # https://bugs.python.org/issue47071 + # https://bugs.python.org/issue44743 + # The Proactor event loop would fail to receive datagram messages after + # sending a message to an address that wasn't listening. + loop = self.loop + + class Protocol(asyncio.DatagramProtocol): + + _received_datagram = None + + def datagram_received(self, data, addr): + self._received_datagram.set_result(data) + + async def wait_for_datagram_received(self): + self._received_datagram = loop.create_future() + result = await asyncio.wait_for(self._received_datagram, 10) + self._received_datagram = None + return result + + def create_socket(): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setblocking(False) + sock.bind(('127.0.0.1', 0)) + return sock + + socket_1 = create_socket() + transport_1, protocol_1 = loop.run_until_complete( + loop.create_datagram_endpoint(Protocol, sock=socket_1) + ) + addr_1 = socket_1.getsockname() + + socket_2 = create_socket() + transport_2, protocol_2 = loop.run_until_complete( + loop.create_datagram_endpoint(Protocol, sock=socket_2) + ) + addr_2 = socket_2.getsockname() + + # creating and immediately closing this to try to get an address that + # is not listening + socket_3 = create_socket() + transport_3, protocol_3 = loop.run_until_complete( + loop.create_datagram_endpoint(Protocol, sock=socket_3) + ) + addr_3 = socket_3.getsockname() + transport_3.abort() + + transport_1.sendto(b'a', addr=addr_2) + self.assertEqual(loop.run_until_complete( + protocol_2.wait_for_datagram_received() + ), b'a') + + transport_2.sendto(b'b', addr=addr_1) + self.assertEqual(loop.run_until_complete( + protocol_1.wait_for_datagram_received() + ), b'b') + + # this should send to an address that isn't listening + transport_1.sendto(b'c', addr=addr_3) + loop.run_until_complete(asyncio.sleep(0)) + + # transport 1 should still be able to receive messages after sending to + # an address that wasn't listening + transport_2.sendto(b'd', addr=addr_1) + self.assertEqual(loop.run_until_complete( + protocol_1.wait_for_datagram_received() + ), b'd') + + transport_1.close() + transport_2.close() + def test_internal_fds(self): loop = self.create_event_loop() if not isinstance(loop, selector_events.BaseSelectorEventLoop): @@ -1988,7 +2184,7 @@ def test_subprocess_stderr(self): transp.close() self.assertEqual(b'OUT:test', proto.data[1]) - self.assertTrue(proto.data[2].startswith(b'ERR:test'), proto.data[2]) + self.assertStartsWith(proto.data[2], b'ERR:test') self.assertEqual(0, proto.returncode) @support.requires_subprocess() @@ -2010,8 +2206,7 @@ def test_subprocess_stderr_redirect_to_stdout(self): stdin.write(b'test') self.loop.run_until_complete(proto.completed) - self.assertTrue(proto.data[1].startswith(b'OUT:testERR:test'), - proto.data[1]) + self.assertStartsWith(proto.data[1], b'OUT:testERR:test') self.assertEqual(b'', proto.data[2]) transp.close() @@ -2131,24 +2326,8 @@ def test_remove_fds_after_closing(self): else: import selectors - class UnixEventLoopTestsMixin(EventLoopTestsMixin): - def setUp(self): - super().setUp() - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = asyncio.SafeChildWatcher() - watcher.attach_loop(self.loop) - asyncio.set_child_watcher(watcher) - - def tearDown(self): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - asyncio.set_child_watcher(None) - super().tearDown() - - if hasattr(selectors, 'KqueueSelector'): - class KqueueEventLoopTests(UnixEventLoopTestsMixin, + class KqueueEventLoopTests(EventLoopTestsMixin, SubprocessTestsMixin, test_utils.TestCase): @@ -2173,7 +2352,7 @@ def test_write_pty(self): super().test_write_pty() if hasattr(selectors, 'EpollSelector'): - class EPollEventLoopTests(UnixEventLoopTestsMixin, + class EPollEventLoopTests(EventLoopTestsMixin, SubprocessTestsMixin, test_utils.TestCase): @@ -2181,7 +2360,7 @@ def create_event_loop(self): return asyncio.SelectorEventLoop(selectors.EpollSelector()) if hasattr(selectors, 'PollSelector'): - class PollEventLoopTests(UnixEventLoopTestsMixin, + class PollEventLoopTests(EventLoopTestsMixin, SubprocessTestsMixin, test_utils.TestCase): @@ -2189,7 +2368,7 @@ def create_event_loop(self): return asyncio.SelectorEventLoop(selectors.PollSelector()) # Should always exist. - class SelectEventLoopTests(UnixEventLoopTestsMixin, + class SelectEventLoopTests(EventLoopTestsMixin, SubprocessTestsMixin, test_utils.TestCase): @@ -2286,7 +2465,7 @@ def test_handle_repr(self): h = asyncio.Handle(cb, (), self.loop) cb_regex = r'' - cb_regex = fr'functools.partialmethod\({cb_regex}, , \)\(\)' + cb_regex = fr'functools.partialmethod\({cb_regex}\)\(\)' regex = fr'^$' self.assertRegex(repr(h), regex) @@ -2335,7 +2514,7 @@ def test_handle_repr_debug(self): self.assertRegex(repr(h), regex) def test_handle_source_traceback(self): - loop = asyncio.get_event_loop_policy().new_event_loop() + loop = asyncio.new_event_loop() loop.set_debug(True) self.set_event_loop(loop) @@ -2633,48 +2812,54 @@ async def inner(): class PolicyTests(unittest.TestCase): + def test_asyncio_set_event_loop_deprecation(self): + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.set_event_loop' is deprecated"): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + self.assertIs(loop, asyncio.get_event_loop()) + loop.close() + + def test_abstract_event_loop_policy_deprecation(self): + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.AbstractEventLoopPolicy' is deprecated"): + policy = asyncio.AbstractEventLoopPolicy() + self.assertIsInstance(policy, asyncio.AbstractEventLoopPolicy) + + def test_default_event_loop_policy_deprecation(self): + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.DefaultEventLoopPolicy' is deprecated"): + policy = asyncio.DefaultEventLoopPolicy() + self.assertIsInstance(policy, asyncio.DefaultEventLoopPolicy) + def test_event_loop_policy(self): - policy = asyncio.AbstractEventLoopPolicy() + policy = asyncio._AbstractEventLoopPolicy() self.assertRaises(NotImplementedError, policy.get_event_loop) self.assertRaises(NotImplementedError, policy.set_event_loop, object()) self.assertRaises(NotImplementedError, policy.new_event_loop) - self.assertRaises(NotImplementedError, policy.get_child_watcher) - self.assertRaises(NotImplementedError, policy.set_child_watcher, - object()) def test_get_event_loop(self): - policy = asyncio.DefaultEventLoopPolicy() + policy = asyncio._DefaultEventLoopPolicy() self.assertIsNone(policy._local._loop) - with self.assertWarns(DeprecationWarning) as cm: - loop = policy.get_event_loop() - self.assertEqual(cm.filename, __file__) - self.assertIsInstance(loop, asyncio.AbstractEventLoop) - self.assertIs(policy._local._loop, loop) - self.assertIs(loop, policy.get_event_loop()) - loop.close() + with self.assertRaises(RuntimeError): + loop = policy.get_event_loop() + self.assertIsNone(policy._local._loop) - def test_get_event_loop_calls_set_event_loop(self): - policy = asyncio.DefaultEventLoopPolicy() + def test_get_event_loop_does_not_call_set_event_loop(self): + policy = asyncio._DefaultEventLoopPolicy() with mock.patch.object( policy, "set_event_loop", wraps=policy.set_event_loop) as m_set_event_loop: - with self.assertWarns(DeprecationWarning) as cm: + with self.assertRaises(RuntimeError): loop = policy.get_event_loop() - self.addCleanup(loop.close) - self.assertEqual(cm.filename, __file__) - - # policy._local._loop must be set through .set_event_loop() - # (the unix DefaultEventLoopPolicy needs this call to attach - # the child watcher correctly) - m_set_event_loop.assert_called_with(loop) - loop.close() + m_set_event_loop.assert_not_called() def test_get_event_loop_after_set_none(self): - policy = asyncio.DefaultEventLoopPolicy() + policy = asyncio._DefaultEventLoopPolicy() policy.set_event_loop(None) self.assertRaises(RuntimeError, policy.get_event_loop) @@ -2682,7 +2867,7 @@ def test_get_event_loop_after_set_none(self): def test_get_event_loop_thread(self, m_current_thread): def f(): - policy = asyncio.DefaultEventLoopPolicy() + policy = asyncio._DefaultEventLoopPolicy() self.assertRaises(RuntimeError, policy.get_event_loop) th = threading.Thread(target=f) @@ -2690,14 +2875,14 @@ def f(): th.join() def test_new_event_loop(self): - policy = asyncio.DefaultEventLoopPolicy() + policy = asyncio._DefaultEventLoopPolicy() loop = policy.new_event_loop() self.assertIsInstance(loop, asyncio.AbstractEventLoop) loop.close() def test_set_event_loop(self): - policy = asyncio.DefaultEventLoopPolicy() + policy = asyncio._DefaultEventLoopPolicy() old_loop = policy.new_event_loop() policy.set_event_loop(old_loop) @@ -2711,20 +2896,31 @@ def test_set_event_loop(self): old_loop.close() def test_get_event_loop_policy(self): - policy = asyncio.get_event_loop_policy() - self.assertIsInstance(policy, asyncio.AbstractEventLoopPolicy) - self.assertIs(policy, asyncio.get_event_loop_policy()) + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): + policy = asyncio.get_event_loop_policy() + self.assertIsInstance(policy, asyncio._AbstractEventLoopPolicy) + self.assertIs(policy, asyncio.get_event_loop_policy()) def test_set_event_loop_policy(self): - self.assertRaises( - TypeError, asyncio.set_event_loop_policy, object()) + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.set_event_loop_policy' is deprecated"): + self.assertRaises( + TypeError, asyncio.set_event_loop_policy, object()) + + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): + old_policy = asyncio.get_event_loop_policy() - old_policy = asyncio.get_event_loop_policy() + policy = asyncio._DefaultEventLoopPolicy() + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.set_event_loop_policy' is deprecated"): + asyncio.set_event_loop_policy(policy) - policy = asyncio.DefaultEventLoopPolicy() - asyncio.set_event_loop_policy(policy) - self.assertIs(policy, asyncio.get_event_loop_policy()) - self.assertIsNot(policy, old_policy) + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): + self.assertIs(policy, asyncio.get_event_loop_policy()) + self.assertIsNot(policy, old_policy) class GetEventLoopTestsMixin: @@ -2753,26 +2949,14 @@ def setUp(self): super().setUp() self.loop = asyncio.new_event_loop() - asyncio.set_event_loop(self.loop) - - if sys.platform != 'win32': - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = asyncio.SafeChildWatcher() - watcher.attach_loop(self.loop) - asyncio.set_child_watcher(watcher) + asyncio._set_event_loop(self.loop) def tearDown(self): try: - if sys.platform != 'win32': - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - asyncio.set_child_watcher(None) - super().tearDown() finally: self.loop.close() - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) events._get_running_loop = self._get_running_loop_saved events._set_running_loop = self._set_running_loop_saved @@ -2815,18 +2999,18 @@ def test_get_event_loop_returns_running_loop(self): class TestError(Exception): pass - class Policy(asyncio.DefaultEventLoopPolicy): + class Policy(asyncio._DefaultEventLoopPolicy): def get_event_loop(self): raise TestError - old_policy = asyncio.get_event_loop_policy() + old_policy = asyncio._get_event_loop_policy() try: - asyncio.set_event_loop_policy(Policy()) + asyncio._set_event_loop_policy(Policy()) loop = asyncio.new_event_loop() with self.assertRaises(TestError): asyncio.get_event_loop() - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) with self.assertRaises(TestError): asyncio.get_event_loop() @@ -2841,15 +3025,15 @@ async def func(): loop.run_until_complete(func()) - asyncio.set_event_loop(loop) + asyncio._set_event_loop(loop) with self.assertRaises(TestError): asyncio.get_event_loop() - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) with self.assertRaises(TestError): asyncio.get_event_loop() finally: - asyncio.set_event_loop_policy(old_policy) + asyncio._set_event_loop_policy(old_policy) if loop is not None: loop.close() @@ -2859,23 +3043,18 @@ async def func(): self.assertIs(asyncio._get_running_loop(), None) def test_get_event_loop_returns_running_loop2(self): - old_policy = asyncio.get_event_loop_policy() + old_policy = asyncio._get_event_loop_policy() try: - asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy()) + asyncio._set_event_loop_policy(asyncio._DefaultEventLoopPolicy()) loop = asyncio.new_event_loop() self.addCleanup(loop.close) - with self.assertWarns(DeprecationWarning) as cm: - loop2 = asyncio.get_event_loop() - self.addCleanup(loop2.close) - self.assertEqual(cm.filename, __file__) - asyncio.set_event_loop(None) with self.assertRaisesRegex(RuntimeError, 'no current'): asyncio.get_event_loop() - with self.assertRaisesRegex(RuntimeError, 'no running'): - asyncio.get_running_loop() - self.assertIs(asyncio._get_running_loop(), None) + asyncio._set_event_loop(None) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() async def func(): self.assertIs(asyncio.get_event_loop(), loop) @@ -2884,15 +3063,15 @@ async def func(): loop.run_until_complete(func()) - asyncio.set_event_loop(loop) + asyncio._set_event_loop(loop) self.assertIs(asyncio.get_event_loop(), loop) - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) with self.assertRaisesRegex(RuntimeError, 'no current'): asyncio.get_event_loop() finally: - asyncio.set_event_loop_policy(old_policy) + asyncio._set_event_loop_policy(old_policy) if loop is not None: loop.close() diff --git a/Lib/test/test_asyncio/test_free_threading.py b/Lib/test/test_asyncio/test_free_threading.py new file mode 100644 index 00000000000000..6da398e77e7797 --- /dev/null +++ b/Lib/test/test_asyncio/test_free_threading.py @@ -0,0 +1,173 @@ +import asyncio +import threading +import unittest +from threading import Thread +from unittest import TestCase + +from test.support import threading_helper + +threading_helper.requires_working_threading(module=True) + + +class MyException(Exception): + pass + + +def tearDownModule(): + asyncio._set_event_loop_policy(None) + + +class TestFreeThreading: + def test_all_tasks_race(self) -> None: + async def main(): + loop = asyncio.get_running_loop() + future = loop.create_future() + + async def coro(): + await future + + tasks = set() + + async with asyncio.TaskGroup() as tg: + for _ in range(100): + tasks.add(tg.create_task(coro())) + + all_tasks = self.all_tasks(loop) + self.assertEqual(len(all_tasks), 101) + + for task in all_tasks: + self.assertEqual(task.get_loop(), loop) + self.assertFalse(task.done()) + + current = self.current_task() + self.assertEqual(current.get_loop(), loop) + self.assertSetEqual(all_tasks, tasks | {current}) + future.set_result(None) + + def runner(): + with asyncio.Runner() as runner: + loop = runner.get_loop() + loop.set_task_factory(self.factory) + runner.run(main()) + + threads = [] + + for _ in range(10): + thread = Thread(target=runner) + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + def test_all_tasks_different_thread(self) -> None: + loop = None + started = threading.Event() + done = threading.Event() # used for main task not finishing early + async def coro(): + await asyncio.Future() + + lock = threading.Lock() + tasks = set() + + async def main(): + nonlocal tasks, loop + loop = asyncio.get_running_loop() + started.set() + for i in range(1000): + with lock: + asyncio.create_task(coro()) + tasks = self.all_tasks(loop) + done.wait() + + runner = threading.Thread(target=lambda: asyncio.run(main())) + + def check(): + started.wait() + with lock: + self.assertSetEqual(tasks & self.all_tasks(loop), tasks) + + threads = [threading.Thread(target=check) for _ in range(10)] + runner.start() + + with threading_helper.start_threads(threads): + pass + + done.set() + runner.join() + + def test_run_coroutine_threadsafe(self) -> None: + results = [] + + def in_thread(loop: asyncio.AbstractEventLoop): + coro = asyncio.sleep(0.1, result=42) + fut = asyncio.run_coroutine_threadsafe(coro, loop) + result = fut.result() + self.assertEqual(result, 42) + results.append(result) + + async def main(): + loop = asyncio.get_running_loop() + async with asyncio.TaskGroup() as tg: + for _ in range(10): + tg.create_task(asyncio.to_thread(in_thread, loop)) + self.assertEqual(results, [42] * 10) + + with asyncio.Runner() as r: + loop = r.get_loop() + loop.set_task_factory(self.factory) + r.run(main()) + + def test_run_coroutine_threadsafe_exception(self) -> None: + async def coro(): + await asyncio.sleep(0) + raise MyException("test") + + def in_thread(loop: asyncio.AbstractEventLoop): + fut = asyncio.run_coroutine_threadsafe(coro(), loop) + return fut.result() + + async def main(): + loop = asyncio.get_running_loop() + tasks = [] + for _ in range(10): + task = loop.create_task(asyncio.to_thread(in_thread, loop)) + tasks.append(task) + results = await asyncio.gather(*tasks, return_exceptions=True) + + self.assertEqual(len(results), 10) + for result in results: + self.assertIsInstance(result, MyException) + self.assertEqual(str(result), "test") + + with asyncio.Runner() as r: + loop = r.get_loop() + loop.set_task_factory(self.factory) + r.run(main()) + + +class TestPyFreeThreading(TestFreeThreading, TestCase): + all_tasks = staticmethod(asyncio.tasks._py_all_tasks) + current_task = staticmethod(asyncio.tasks._py_current_task) + + def factory(self, loop, coro, **kwargs): + return asyncio.tasks._PyTask(coro, loop=loop, **kwargs) + + +@unittest.skipUnless(hasattr(asyncio.tasks, "_c_all_tasks"), "requires _asyncio") +class TestCFreeThreading(TestFreeThreading, TestCase): + all_tasks = staticmethod(getattr(asyncio.tasks, "_c_all_tasks", None)) + current_task = staticmethod(getattr(asyncio.tasks, "_c_current_task", None)) + + def factory(self, loop, coro, **kwargs): + return asyncio.tasks._CTask(coro, loop=loop, **kwargs) + + +class TestEagerPyFreeThreading(TestPyFreeThreading): + def factory(self, loop, coro, eager_start=True, **kwargs): + return asyncio.tasks._PyTask(coro, loop=loop, **kwargs, eager_start=eager_start) + + +@unittest.skipUnless(hasattr(asyncio.tasks, "_c_all_tasks"), "requires _asyncio") +class TestEagerCFreeThreading(TestCFreeThreading, TestCase): + def factory(self, loop, coro, eager_start=True, **kwargs): + return asyncio.tasks._CTask(coro, loop=loop, **kwargs, eager_start=eager_start) diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index d3e8efec1c04c2..01d6230e6dd9a3 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -5,6 +5,7 @@ import re import sys import threading +import traceback import unittest from unittest import mock from types import GenericAlias @@ -16,7 +17,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) def _fakefunc(f): @@ -31,6 +32,25 @@ def last_cb(): pass +class ReachableCode(Exception): + """Exception to raise to indicate that some code was reached. + + Use this exception if using mocks is not a good alternative. + """ + + +class SimpleEvilEventLoop(asyncio.base_events.BaseEventLoop): + """Base class for UAF and other evil stuff requiring an evil event loop.""" + + def get_debug(self): # to suppress tracebacks + return False + + def __del__(self): + # Automatically close the evil event loop to avoid warnings. + if not self.is_closed() and not self.is_running(): + self.close() + + class DuckFuture: # Class that does not inherit from Future but aims to be duck-type # compatible with it. @@ -158,8 +178,8 @@ async def test(): def test_constructor_use_global_loop(self): # Deprecated in 3.10, undeprecated in 3.12 - asyncio.set_event_loop(self.loop) - self.addCleanup(asyncio.set_event_loop, None) + asyncio._set_event_loop(self.loop) + self.addCleanup(asyncio._set_event_loop, None) f = self._new_future() self.assertIs(f._loop, self.loop) self.assertIs(f.get_loop(), self.loop) @@ -222,7 +242,7 @@ def test_uninitialized(self): def test_future_cancel_message_getter(self): f = self._new_future(loop=self.loop) - self.assertTrue(hasattr(f, '_cancel_message')) + self.assertHasAttr(f, '_cancel_message') self.assertEqual(f._cancel_message, None) f.cancel('my message') @@ -416,6 +436,24 @@ def test_copy_state(self): _copy_future_state(f_cancelled, newf_cancelled) self.assertTrue(newf_cancelled.cancelled()) + try: + raise concurrent.futures.InvalidStateError + except BaseException as e: + f_exc = e + + f_conexc = self._new_future(loop=self.loop) + f_conexc.set_exception(f_exc) + + newf_conexc = self._new_future(loop=self.loop) + _copy_future_state(f_conexc, newf_conexc) + self.assertTrue(newf_conexc.done()) + try: + newf_conexc.result() + except BaseException as e: + newf_exc = e # assertRaises context manager drops the traceback + newf_tb = ''.join(traceback.format_tb(newf_exc.__traceback__)) + self.assertEqual(newf_tb.count('raise concurrent.futures.InvalidStateError'), 1) + def test_iter(self): fut = self._new_future(loop=self.loop) @@ -528,8 +566,8 @@ async def test(): def test_wrap_future_use_global_loop(self): # Deprecated in 3.10, undeprecated in 3.12 - asyncio.set_event_loop(self.loop) - self.addCleanup(asyncio.set_event_loop, None) + asyncio._set_event_loop(self.loop) + self.addCleanup(asyncio._set_event_loop, None) def run(arg): return (arg, threading.get_ident()) ex = concurrent.futures.ThreadPoolExecutor(1) @@ -640,6 +678,28 @@ def __del__(self): fut = self._new_future(loop=self.loop) fut.set_result(Evil()) + def test_future_cancelled_result_refcycles(self): + f = self._new_future(loop=self.loop) + f.cancel() + exc = None + try: + f.result() + except asyncio.CancelledError as e: + exc = e + self.assertIsNotNone(exc) + self.assertListEqual(gc.get_referrers(exc), []) + + def test_future_cancelled_exception_refcycles(self): + f = self._new_future(loop=self.loop) + f.cancel() + exc = None + try: + f.exception() + except asyncio.CancelledError as e: + exc = e + self.assertIsNotNone(exc) + self.assertListEqual(gc.get_referrers(exc), []) + @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') @@ -656,6 +716,24 @@ def test_future_del_segfault(self): with self.assertRaises(AttributeError): del fut._log_traceback + def test_callbacks_copy(self): + # See https://github.com/python/cpython/issues/125789 + # In C implementation, the `_callbacks` attribute + # always returns a new list to avoid mutations of internal state + + fut = self._new_future(loop=self.loop) + f1 = lambda _: 1 + f2 = lambda _: 2 + fut.add_done_callback(f1) + fut.add_done_callback(f2) + callbacks = fut._callbacks + self.assertIsNot(callbacks, fut._callbacks) + fut.remove_done_callback(f1) + callbacks = fut._callbacks + self.assertIsNot(callbacks, fut._callbacks) + fut.remove_done_callback(f2) + self.assertIsNone(fut._callbacks) + @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') @@ -888,6 +966,107 @@ def __eq__(self, other): fut.remove_done_callback(evil()) + def test_evil_call_soon_list_mutation(self): + # see: https://github.com/python/cpython/issues/125969 + called_on_fut_callback0 = False + + pad = lambda: ... + + def evil_call_soon(*args, **kwargs): + nonlocal called_on_fut_callback0 + if called_on_fut_callback0: + # Called when handling fut->fut_callbacks[0] + # and mutates the length fut->fut_callbacks. + fut.remove_done_callback(int) + fut.remove_done_callback(pad) + else: + called_on_fut_callback0 = True + + fake_event_loop = SimpleEvilEventLoop() + fake_event_loop.call_soon = evil_call_soon + + with mock.patch.object(self, 'loop', fake_event_loop): + fut = self._new_future() + self.assertIs(fut.get_loop(), fake_event_loop) + + fut.add_done_callback(str) # sets fut->fut_callback0 + fut.add_done_callback(int) # sets fut->fut_callbacks[0] + fut.add_done_callback(pad) # sets fut->fut_callbacks[1] + fut.add_done_callback(pad) # sets fut->fut_callbacks[2] + fut.set_result("boom") + + # When there are no more callbacks, the Python implementation + # returns an empty list but the C implementation returns None. + self.assertIn(fut._callbacks, (None, [])) + + def test_use_after_free_on_fut_callback_0_with_evil__eq__(self): + # Special thanks to Nico-Posada for the original PoC. + # See https://github.com/python/cpython/issues/125966. + + fut = self._new_future() + + class cb_pad: + def __eq__(self, other): + return True + + class evil(cb_pad): + def __eq__(self, other): + fut.remove_done_callback(None) + return NotImplemented + + fut.add_done_callback(cb_pad()) + fut.remove_done_callback(evil()) + + def test_use_after_free_on_fut_callback_0_with_evil__getattribute__(self): + # see: https://github.com/python/cpython/issues/125984 + + class EvilEventLoop(SimpleEvilEventLoop): + def call_soon(self, *args, **kwargs): + super().call_soon(*args, **kwargs) + raise ReachableCode + + def __getattribute__(self, name): + nonlocal fut_callback_0 + if name == 'call_soon': + fut.remove_done_callback(fut_callback_0) + del fut_callback_0 + return object.__getattribute__(self, name) + + evil_loop = EvilEventLoop() + with mock.patch.object(self, 'loop', evil_loop): + fut = self._new_future() + self.assertIs(fut.get_loop(), evil_loop) + + fut_callback_0 = lambda: ... + fut.add_done_callback(fut_callback_0) + self.assertRaises(ReachableCode, fut.set_result, "boom") + + def test_use_after_free_on_fut_context_0_with_evil__getattribute__(self): + # see: https://github.com/python/cpython/issues/125984 + + class EvilEventLoop(SimpleEvilEventLoop): + def call_soon(self, *args, **kwargs): + super().call_soon(*args, **kwargs) + raise ReachableCode + + def __getattribute__(self, name): + if name == 'call_soon': + # resets the future's event loop + fut.__init__(loop=SimpleEvilEventLoop()) + return object.__getattribute__(self, name) + + evil_loop = EvilEventLoop() + with mock.patch.object(self, 'loop', evil_loop): + fut = self._new_future() + self.assertIs(fut.get_loop(), evil_loop) + + fut_callback_0 = mock.Mock() + fut_context_0 = mock.Mock() + fut.add_done_callback(fut_callback_0, context=fut_context_0) + del fut_context_0 + del fut_callback_0 + self.assertRaises(ReachableCode, fut.set_result, "boom") + @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index b7cfffb76bd8f1..e2cddea01ecd93 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -7,7 +7,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class FutureTests: diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py new file mode 100644 index 00000000000000..fd2160d4ca3137 --- /dev/null +++ b/Lib/test/test_asyncio/test_graph.py @@ -0,0 +1,436 @@ +import asyncio +import io +import unittest + + +# To prevent a warning "test altered the execution environment" +def tearDownModule(): + asyncio._set_event_loop_policy(None) + + +def capture_test_stack(*, fut=None, depth=1): + + def walk(s): + ret = [ + (f"T<{n}>" if '-' not in (n := s.future.get_name()) else 'T') + if isinstance(s.future, asyncio.Task) else 'F' + ] + + ret.append( + [ + ( + f"s {entry.frame.f_code.co_name}" + if entry.frame.f_generator is None else + ( + f"a {entry.frame.f_generator.cr_code.co_name}" + if hasattr(entry.frame.f_generator, 'cr_code') else + f"ag {entry.frame.f_generator.ag_code.co_name}" + ) + ) for entry in s.call_stack + ] + ) + + ret.append( + sorted([ + walk(ab) for ab in s.awaited_by + ], key=lambda entry: entry[0]) + ) + + return ret + + buf = io.StringIO() + asyncio.print_call_graph(fut, file=buf, depth=depth+1) + + stack = asyncio.capture_call_graph(fut, depth=depth) + return walk(stack), buf.getvalue() + + +class CallStackTestBase: + + async def test_stack_tgroup(self): + + stack_for_c5 = None + + def c5(): + nonlocal stack_for_c5 + stack_for_c5 = capture_test_stack(depth=2) + + async def c4(): + await asyncio.sleep(0) + c5() + + async def c3(): + await c4() + + async def c2(): + await c3() + + async def c1(task): + await task + + async def main(): + async with asyncio.TaskGroup() as tg: + task = tg.create_task(c2(), name="c2_root") + tg.create_task(c1(task), name="sub_main_1") + tg.create_task(c1(task), name="sub_main_2") + + await main() + + self.assertEqual(stack_for_c5[0], [ + # task name + 'T', + # call stack + ['s c5', 'a c4', 'a c3', 'a c2'], + # awaited by + [ + ['T', + ['a _aexit', 'a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ], + ['T', + ['a c1'], + [ + ['T', + ['a _aexit', 'a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ] + ] + ], + ['T', + ['a c1'], + [ + ['T', + ['a _aexit', 'a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ] + ] + ] + ] + ]) + + self.assertIn( + ' async CallStackTestBase.test_stack_tgroup()', + stack_for_c5[1]) + + + async def test_stack_async_gen(self): + + stack_for_gen_nested_call = None + + async def gen_nested_call(): + nonlocal stack_for_gen_nested_call + stack_for_gen_nested_call = capture_test_stack() + + async def gen(): + for num in range(2): + yield num + if num == 1: + await gen_nested_call() + + async def main(): + async for el in gen(): + pass + + await main() + + self.assertEqual(stack_for_gen_nested_call[0], [ + 'T', + [ + 's capture_test_stack', + 'a gen_nested_call', + 'ag gen', + 'a main', + 'a test_stack_async_gen' + ], + [] + ]) + + self.assertIn( + 'async generator CallStackTestBase.test_stack_async_gen..gen()', + stack_for_gen_nested_call[1]) + + async def test_stack_gather(self): + + stack_for_deep = None + + async def deep(): + await asyncio.sleep(0) + nonlocal stack_for_deep + stack_for_deep = capture_test_stack() + + async def c1(): + await asyncio.sleep(0) + await deep() + + async def c2(): + await asyncio.sleep(0) + + async def main(): + await asyncio.gather(c1(), c2()) + + await main() + + self.assertEqual(stack_for_deep[0], [ + 'T', + ['s capture_test_stack', 'a deep', 'a c1'], + [ + ['T', ['a main', 'a test_stack_gather'], []] + ] + ]) + + async def test_stack_shield(self): + + stack_for_shield = None + + async def deep(): + await asyncio.sleep(0) + nonlocal stack_for_shield + stack_for_shield = capture_test_stack() + + async def c1(): + await asyncio.sleep(0) + await deep() + + async def main(): + await asyncio.shield(c1()) + + await main() + + self.assertEqual(stack_for_shield[0], [ + 'T', + ['s capture_test_stack', 'a deep', 'a c1'], + [ + ['T', ['a main', 'a test_stack_shield'], []] + ] + ]) + + async def test_stack_timeout(self): + + stack_for_inner = None + + async def inner(): + await asyncio.sleep(0) + nonlocal stack_for_inner + stack_for_inner = capture_test_stack() + + async def c1(): + async with asyncio.timeout(1): + await asyncio.sleep(0) + await inner() + + async def main(): + await asyncio.shield(c1()) + + await main() + + self.assertEqual(stack_for_inner[0], [ + 'T', + ['s capture_test_stack', 'a inner', 'a c1'], + [ + ['T', ['a main', 'a test_stack_timeout'], []] + ] + ]) + + async def test_stack_wait(self): + + stack_for_inner = None + + async def inner(): + await asyncio.sleep(0) + nonlocal stack_for_inner + stack_for_inner = capture_test_stack() + + async def c1(): + async with asyncio.timeout(1): + await asyncio.sleep(0) + await inner() + + async def c2(): + for i in range(3): + await asyncio.sleep(0) + + async def main(t1, t2): + while True: + _, pending = await asyncio.wait([t1, t2]) + if not pending: + break + + t1 = asyncio.create_task(c1()) + t2 = asyncio.create_task(c2()) + try: + await main(t1, t2) + finally: + await t1 + await t2 + + self.assertEqual(stack_for_inner[0], [ + 'T', + ['s capture_test_stack', 'a inner', 'a c1'], + [ + ['T', + ['a _wait', 'a wait', 'a main', 'a test_stack_wait'], + [] + ] + ] + ]) + + async def test_stack_task(self): + + stack_for_inner = None + + async def inner(): + await asyncio.sleep(0) + nonlocal stack_for_inner + stack_for_inner = capture_test_stack() + + async def c1(): + await inner() + + async def c2(): + await asyncio.create_task(c1(), name='there there') + + async def main(): + await c2() + + await main() + + self.assertEqual(stack_for_inner[0], [ + 'T', + ['s capture_test_stack', 'a inner', 'a c1'], + [['T', ['a c2', 'a main', 'a test_stack_task'], []]] + ]) + + async def test_stack_future(self): + + stack_for_fut = None + + async def a2(fut): + await fut + + async def a1(fut): + await a2(fut) + + async def b1(fut): + await fut + + async def main(): + nonlocal stack_for_fut + + fut = asyncio.Future() + async with asyncio.TaskGroup() as g: + g.create_task(a1(fut), name="task A") + g.create_task(b1(fut), name='task B') + + for _ in range(5): + # Do a few iterations to ensure that both a1 and b1 + # await on the future + await asyncio.sleep(0) + + stack_for_fut = capture_test_stack(fut=fut) + fut.set_result(None) + + await main() + + self.assertEqual(stack_for_fut[0], + ['F', + [], + [ + ['T', + ['a a2', 'a a1'], + [['T', ['a test_stack_future'], []]] + ], + ['T', + ['a b1'], + [['T', ['a test_stack_future'], []]] + ], + ]] + ) + + self.assertTrue(stack_for_fut[1].startswith('* Future(id=')) + + +@unittest.skipIf( + not hasattr(asyncio.futures, "_c_future_add_to_awaited_by"), + "C-accelerated asyncio call graph backend missing", +) +class TestCallStackC(CallStackTestBase, unittest.IsolatedAsyncioTestCase): + def setUp(self): + futures = asyncio.futures + tasks = asyncio.tasks + + self._Future = asyncio.Future + asyncio.Future = futures.Future = futures._CFuture + + self._Task = asyncio.Task + asyncio.Task = tasks.Task = tasks._CTask + + self._future_add_to_awaited_by = asyncio.future_add_to_awaited_by + futures.future_add_to_awaited_by = futures._c_future_add_to_awaited_by + asyncio.future_add_to_awaited_by = futures.future_add_to_awaited_by + + self._future_discard_from_awaited_by = asyncio.future_discard_from_awaited_by + futures.future_discard_from_awaited_by = futures._c_future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = futures.future_discard_from_awaited_by + + + def tearDown(self): + futures = asyncio.futures + tasks = asyncio.tasks + + futures.future_discard_from_awaited_by = self._future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = self._future_discard_from_awaited_by + del self._future_discard_from_awaited_by + + futures.future_add_to_awaited_by = self._future_add_to_awaited_by + asyncio.future_add_to_awaited_by = self._future_add_to_awaited_by + del self._future_add_to_awaited_by + + asyncio.Task = self._Task + tasks.Task = self._Task + del self._Task + + asyncio.Future = self._Future + futures.Future = self._Future + del self._Future + + +@unittest.skipIf( + not hasattr(asyncio.futures, "_py_future_add_to_awaited_by"), + "Pure Python asyncio call graph backend missing", +) +class TestCallStackPy(CallStackTestBase, unittest.IsolatedAsyncioTestCase): + def setUp(self): + futures = asyncio.futures + tasks = asyncio.tasks + + self._Future = asyncio.Future + asyncio.Future = futures.Future = futures._PyFuture + + self._Task = asyncio.Task + asyncio.Task = tasks.Task = tasks._PyTask + + self._future_add_to_awaited_by = asyncio.future_add_to_awaited_by + futures.future_add_to_awaited_by = futures._py_future_add_to_awaited_by + asyncio.future_add_to_awaited_by = futures.future_add_to_awaited_by + + self._future_discard_from_awaited_by = asyncio.future_discard_from_awaited_by + futures.future_discard_from_awaited_by = futures._py_future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = futures.future_discard_from_awaited_by + + + def tearDown(self): + futures = asyncio.futures + tasks = asyncio.tasks + + futures.future_discard_from_awaited_by = self._future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = self._future_discard_from_awaited_by + del self._future_discard_from_awaited_by + + futures.future_add_to_awaited_by = self._future_add_to_awaited_by + asyncio.future_add_to_awaited_by = self._future_add_to_awaited_by + del self._future_add_to_awaited_by + + asyncio.Task = self._Task + tasks.Task = self._Task + del self._Task + + asyncio.Future = self._Future + futures.Future = self._Future + del self._Future diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index a0884bffe6b0de..3bb3e5c4ca0658 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -20,18 +20,18 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class LockTests(unittest.IsolatedAsyncioTestCase): async def test_repr(self): lock = asyncio.Lock() - self.assertTrue(repr(lock).endswith('[unlocked]>')) + self.assertEndsWith(repr(lock), '[unlocked]>') self.assertTrue(RGX_REPR.match(repr(lock))) await lock.acquire() - self.assertTrue(repr(lock).endswith('[locked]>')) + self.assertEndsWith(repr(lock), '[locked]>') self.assertTrue(RGX_REPR.match(repr(lock))) async def test_lock(self): @@ -39,7 +39,7 @@ async def test_lock(self): with self.assertRaisesRegex( TypeError, - "object Lock can't be used in 'await' expression" + "'Lock' object can't be awaited" ): await lock @@ -77,7 +77,7 @@ async def test_lock_by_with_statement(self): self.assertFalse(lock.locked()) with self.assertRaisesRegex( TypeError, - r"object \w+ can't be used in 'await' expression" + r"'\w+' object can't be awaited" ): with await lock: pass @@ -286,12 +286,12 @@ class EventTests(unittest.IsolatedAsyncioTestCase): def test_repr(self): ev = asyncio.Event() - self.assertTrue(repr(ev).endswith('[unset]>')) + self.assertEndsWith(repr(ev), '[unset]>') match = RGX_REPR.match(repr(ev)) self.assertEqual(match.group('extras'), 'unset') ev.set() - self.assertTrue(repr(ev).endswith('[set]>')) + self.assertEndsWith(repr(ev), '[set]>') self.assertTrue(RGX_REPR.match(repr(ev))) ev._waiters.append(mock.Mock()) @@ -916,11 +916,11 @@ def test_initial_value_zero(self): async def test_repr(self): sem = asyncio.Semaphore() - self.assertTrue(repr(sem).endswith('[unlocked, value:1]>')) + self.assertEndsWith(repr(sem), '[unlocked, value:1]>') self.assertTrue(RGX_REPR.match(repr(sem))) await sem.acquire() - self.assertTrue(repr(sem).endswith('[locked]>')) + self.assertEndsWith(repr(sem), '[locked]>') self.assertTrue('waiters' not in repr(sem)) self.assertTrue(RGX_REPR.match(repr(sem))) @@ -941,7 +941,7 @@ async def test_semaphore(self): with self.assertRaisesRegex( TypeError, - "object Semaphore can't be used in 'await' expression", + "'Semaphore' object can't be awaited", ): await sem @@ -1194,14 +1194,14 @@ async def c3(result): self.assertEqual([2, 3], result) async def test_acquire_fifo_order_4(self): - # Test that a successfule `acquire()` will wake up multiple Tasks + # Test that a successful `acquire()` will wake up multiple Tasks # that were waiting in the Semaphore queue due to FIFO rules. sem = asyncio.Semaphore(0) result = [] count = 0 async def c1(result): - # First task immediatlly waits for semaphore. It will be awoken by c2. + # First task immediately waits for semaphore. It will be awoken by c2. self.assertEqual(sem._value, 0) await sem.acquire() # We should have woken up all waiting tasks now. @@ -1270,7 +1270,7 @@ async def test_barrier(self): self.assertIn("filling", repr(barrier)) with self.assertRaisesRegex( TypeError, - "object Barrier can't be used in 'await' expression", + "'Barrier' object can't be awaited", ): await barrier @@ -1475,13 +1475,13 @@ async def coro(): # first time waiting await barrier.wait() - # after wainting once for all tasks + # after waiting once for all tasks if rewait_n > 0: rewait_n -= 1 # wait again only for rewait tasks await barrier.wait() else: - # wait for end of draining state` + # wait for end of draining state await barrier_nowaiting.wait() # wait for other waiting tasks await barrier.wait() @@ -1780,7 +1780,7 @@ async def coro(): self.assertEqual(barrier.n_waiting, 0) async def test_abort_barrier_when_exception_then_resetting(self): - # test from threading.Barrier: see `lock_tests.test_abort_and_reset`` + # test from threading.Barrier: see `lock_tests.test_abort_and_reset` barrier1 = asyncio.Barrier(self.N) barrier2 = asyncio.Barrier(self.N) results1 = [] diff --git a/Lib/test/test_asyncio/test_pep492.py b/Lib/test/test_asyncio/test_pep492.py index dc25a46985e349..48f4a75e0fd56c 100644 --- a/Lib/test/test_asyncio/test_pep492.py +++ b/Lib/test/test_asyncio/test_pep492.py @@ -11,7 +11,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) # Test that asyncio.iscoroutine() uses collections.abc.Coroutine @@ -77,7 +77,7 @@ async def test(lock): self.assertFalse(lock.locked()) with self.assertRaisesRegex( TypeError, - "can't be used in 'await' expression" + "can't be awaited" ): with await lock: pass @@ -124,10 +124,10 @@ def foo(): yield self.assertFalse(asyncio.iscoroutine(foo())) - def test_iscoroutinefunction(self): async def foo(): pass - self.assertTrue(asyncio.iscoroutinefunction(foo)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(asyncio.iscoroutinefunction(foo)) def test_async_def_coroutines(self): async def bar(): diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py index fcaa2f6ade2b76..24c4e8546b17aa 100644 --- a/Lib/test/test_asyncio/test_proactor_events.py +++ b/Lib/test/test_asyncio/test_proactor_events.py @@ -18,7 +18,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) def close_transport(transport): @@ -1018,9 +1018,9 @@ def setUp(self): self.addCleanup(self.file.close) super().setUp() - def make_socket(self, cleanup=True): + def make_socket(self, cleanup=True, blocking=False): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.setblocking(False) + sock.setblocking(blocking) sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1024) sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1024) if cleanup: @@ -1082,6 +1082,11 @@ def test_sock_sendfile_not_regular_file(self): 0, None)) self.assertEqual(self.file.tell(), 0) + def test_blocking_socket(self): + self.loop.set_debug(True) + sock = self.make_socket(blocking=True) + with self.assertRaisesRegex(ValueError, "must be non-blocking"): + self.run_loop(self.loop.sock_sendfile(sock, self.file)) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_protocols.py b/Lib/test/test_asyncio/test_protocols.py index 0f232631867db5..4484a031988533 100644 --- a/Lib/test/test_asyncio/test_protocols.py +++ b/Lib/test/test_asyncio/test_protocols.py @@ -7,7 +7,7 @@ def tearDownModule(): # not needed for the test file but added for uniformness with all other # asyncio test files for the sake of unified cleanup - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class ProtocolsAbsTests(unittest.TestCase): @@ -19,7 +19,7 @@ def test_base_protocol(self): self.assertIsNone(p.connection_lost(f)) self.assertIsNone(p.pause_writing()) self.assertIsNone(p.resume_writing()) - self.assertFalse(hasattr(p, '__dict__')) + self.assertNotHasAttr(p, '__dict__') def test_protocol(self): f = mock.Mock() @@ -30,7 +30,7 @@ def test_protocol(self): self.assertIsNone(p.eof_received()) self.assertIsNone(p.pause_writing()) self.assertIsNone(p.resume_writing()) - self.assertFalse(hasattr(p, '__dict__')) + self.assertNotHasAttr(p, '__dict__') def test_buffered_protocol(self): f = mock.Mock() @@ -41,7 +41,7 @@ def test_buffered_protocol(self): self.assertIsNone(p.buffer_updated(150)) self.assertIsNone(p.pause_writing()) self.assertIsNone(p.resume_writing()) - self.assertFalse(hasattr(p, '__dict__')) + self.assertNotHasAttr(p, '__dict__') def test_datagram_protocol(self): f = mock.Mock() @@ -50,7 +50,7 @@ def test_datagram_protocol(self): self.assertIsNone(dp.connection_lost(f)) self.assertIsNone(dp.error_received(f)) self.assertIsNone(dp.datagram_received(f, f)) - self.assertFalse(hasattr(dp, '__dict__')) + self.assertNotHasAttr(dp, '__dict__') def test_subprocess_protocol(self): f = mock.Mock() @@ -60,7 +60,7 @@ def test_subprocess_protocol(self): self.assertIsNone(sp.pipe_data_received(1, f)) self.assertIsNone(sp.pipe_connection_lost(1, f)) self.assertIsNone(sp.process_exited()) - self.assertFalse(hasattr(sp, '__dict__')) + self.assertNotHasAttr(sp, '__dict__') if __name__ == '__main__': diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index 2d058ccf6a8c72..090b9774c2289f 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -6,7 +6,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class QueueBasicTests(unittest.IsolatedAsyncioTestCase): @@ -18,7 +18,7 @@ async def _test_repr_or_str(self, fn, expect_id): appear in fn(Queue()). """ q = asyncio.Queue() - self.assertTrue(fn(q).startswith(' 0 + + self.assertEqual(q.qsize(), 0) + + # Ensure join() task successfully finishes + await q.join() + + # Ensure get() task is finished, and raised ShutDown + await asyncio.sleep(0) + self.assertTrue(get_task.done()) + with self.assertRaisesShutdown(): + await get_task + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + async def test_shutdown_nonempty(self): + # Test shutting down a non-empty queue + + # Setup full queue with 1 item, and join() and put() tasks + q = self.q_class(maxsize=1) + loop = asyncio.get_running_loop() + + q.put_nowait("data") + join_task = loop.create_task(q.join()) + put_task = loop.create_task(q.put("data2")) + + # Ensure put() task is not finished + await asyncio.sleep(0) + self.assertFalse(put_task.done()) + + # Perform shut-down + q.shutdown(immediate=False) # unfinished tasks: 1 -> 1 + + self.assertEqual(q.qsize(), 1) + + # Ensure put() task is finished, and raised ShutDown + await asyncio.sleep(0) + self.assertTrue(put_task.done()) + with self.assertRaisesShutdown(): + await put_task + + # Ensure get() succeeds on enqueued item + self.assertEqual(await q.get(), "data") + + # Ensure join() task is not finished + await asyncio.sleep(0) + self.assertFalse(join_task.done()) + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + # Ensure there is 1 unfinished task, and join() task succeeds + q.task_done() + + await asyncio.sleep(0) + self.assertTrue(join_task.done()) + await join_task + + with self.assertRaises( + ValueError, msg="Didn't appear to mark all tasks done" + ): + q.task_done() + + async def test_shutdown_immediate(self): + # Test immediately shutting down a queue + + # Setup queue with 1 item, and a join() task + q = self.q_class() + loop = asyncio.get_running_loop() + q.put_nowait("data") + join_task = loop.create_task(q.join()) + + # Perform shut-down + q.shutdown(immediate=True) # unfinished tasks: 1 -> 0 + + self.assertEqual(q.qsize(), 0) + + # Ensure join() task has successfully finished + await asyncio.sleep(0) + self.assertTrue(join_task.done()) + await join_task + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + # Ensure there are no unfinished tasks + with self.assertRaises( + ValueError, msg="Didn't appear to mark all tasks done" + ): + q.task_done() + + async def test_shutdown_immediate_with_unfinished(self): + # Test immediately shutting down a queue with unfinished tasks + + # Setup queue with 2 items (1 retrieved), and a join() task + q = self.q_class() + loop = asyncio.get_running_loop() + q.put_nowait("data") + q.put_nowait("data") + join_task = loop.create_task(q.join()) + self.assertEqual(await q.get(), "data") + + # Perform shut-down + q.shutdown(immediate=True) # unfinished tasks: 2 -> 1 + + self.assertEqual(q.qsize(), 0) + + # Ensure join() task is not finished + await asyncio.sleep(0) + self.assertFalse(join_task.done()) + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + # Ensure there is 1 unfinished task + q.task_done() + with self.assertRaises( + ValueError, msg="Didn't appear to mark all tasks done" + ): + q.task_done() + + # Ensure join() task has successfully finished + await asyncio.sleep(0) + self.assertTrue(join_task.done()) + await join_task + + +class QueueShutdownTests( + _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase +): + q_class = asyncio.Queue + + +class LifoQueueShutdownTests( + _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase +): + q_class = asyncio.LifoQueue + + +class PriorityQueueShutdownTests( + _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase +): + q_class = asyncio.PriorityQueue + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index 266f057f0776c3..21f277bc2d8d5f 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -12,14 +12,14 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) def interrupt_self(): _thread.interrupt_main() -class TestPolicy(asyncio.AbstractEventLoopPolicy): +class TestPolicy(asyncio._AbstractEventLoopPolicy): def __init__(self, loop_factory): self.loop_factory = loop_factory @@ -61,15 +61,15 @@ def setUp(self): super().setUp() policy = TestPolicy(self.new_loop) - asyncio.set_event_loop_policy(policy) + asyncio._set_event_loop_policy(policy) def tearDown(self): - policy = asyncio.get_event_loop_policy() + policy = asyncio._get_event_loop_policy() if policy.loop is not None: self.assertTrue(policy.loop.is_closed()) self.assertTrue(policy.loop.shutdown_ag_run) - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) super().tearDown() @@ -93,8 +93,8 @@ async def main(): def test_asyncio_run_only_coro(self): for o in {1, lambda: None}: with self.subTest(obj=o), \ - self.assertRaisesRegex(ValueError, - 'a coroutine was expected'): + self.assertRaisesRegex(TypeError, + 'an awaitable is required'): asyncio.run(o) def test_asyncio_run_debug(self): @@ -208,7 +208,7 @@ async def main(): await asyncio.sleep(0) return 42 - policy = asyncio.get_event_loop_policy() + policy = asyncio._get_event_loop_policy() policy.set_event_loop = mock.Mock() asyncio.run(main()) self.assertTrue(policy.set_event_loop.called) @@ -259,7 +259,7 @@ def new_event_loop(): loop.set_task_factory(Task) return loop - asyncio.set_event_loop_policy(TestPolicy(new_event_loop)) + asyncio._set_event_loop_policy(TestPolicy(new_event_loop)) with self.assertRaises(asyncio.CancelledError): asyncio.run(main()) @@ -319,19 +319,28 @@ async def f(): def test_run_non_coro(self): with asyncio.Runner() as runner: with self.assertRaisesRegex( - ValueError, - "a coroutine was expected" + TypeError, + "an awaitable is required" ): runner.run(123) def test_run_future(self): with asyncio.Runner() as runner: - with self.assertRaisesRegex( - ValueError, - "a coroutine was expected" - ): - fut = runner.get_loop().create_future() - runner.run(fut) + fut = runner.get_loop().create_future() + fut.set_result('done') + self.assertEqual('done', runner.run(fut)) + + def test_run_awaitable(self): + class MyAwaitable: + def __await__(self): + return self.run().__await__() + + @staticmethod + async def run(): + return 'done' + + with asyncio.Runner() as runner: + self.assertEqual('done', runner.run(MyAwaitable())) def test_explicit_close(self): runner = asyncio.Runner() @@ -486,7 +495,7 @@ def test_set_event_loop_called_once(self): async def coro(): pass - policy = asyncio.get_event_loop_policy() + policy = asyncio._get_event_loop_policy() policy.set_event_loop = mock.Mock() runner = asyncio.Runner() runner.run(coro()) diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index aaeda33dd0c677..c9217d04bcd322 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -24,7 +24,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class TestBaseSelectorEventLoop(BaseSelectorEventLoop): @@ -364,6 +364,31 @@ def test_accept_connection_multiple(self): self.loop.run_until_complete(asyncio.sleep(0)) self.assertEqual(sock.accept.call_count, backlog) + def test_accept_connection_skip_connectionabortederror(self): + sock = mock.Mock() + + def mock_sock_accept(): + # mock accept(2) returning -ECONNABORTED every-other + # time that it's called. This applies most to OpenBSD + # whose sockets generate this errno more reproducibly than + # Linux and other OS. + if sock.accept.call_count % 2 == 0: + raise ConnectionAbortedError + return (mock.Mock(), mock.Mock()) + + sock.accept.side_effect = mock_sock_accept + backlog = 100 + # test that _accept_connection's loop calls sock.accept + # all 100 times, continuing past ConnectionAbortedError + # instead of unnecessarily returning early + mock_obj = mock.patch.object + with mock_obj(self.loop, '_accept_connection2') as accept2_mock: + self.loop._accept_connection( + mock.Mock(), sock, backlog=backlog) + # as in test_accept_connection_multiple avoid task pending + # warnings by using asyncio.sleep(0) + self.loop.run_until_complete(asyncio.sleep(0)) + self.assertEqual(sock.accept.call_count, backlog) class SelectorTransportTests(test_utils.TestCase): @@ -805,6 +830,18 @@ def test_writelines_send_partial(self): self.assertTrue(self.sock.send.called) self.assertTrue(self.loop.writers) + def test_writelines_pauses_protocol(self): + data = memoryview(b'data') + self.sock.send.return_value = 2 + self.sock.send.fileno.return_value = 7 + + transport = self.socket_transport() + transport._high_water = 1 + transport.writelines([data]) + self.assertTrue(self.protocol.pause_writing.called) + self.assertTrue(self.sock.send.called) + self.assertTrue(self.loop.writers) + @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') def test_write_sendmsg_full(self): data = memoryview(b'data') diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py index d33ff197bbfa1d..e1b766d06cbe1e 100644 --- a/Lib/test/test_asyncio/test_sendfile.py +++ b/Lib/test/test_asyncio/test_sendfile.py @@ -22,7 +22,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class MySendfileProto(asyncio.Protocol): @@ -93,13 +93,10 @@ async def wait_closed(self): class SendfileBase: - # 256 KiB plus small unaligned to buffer chunk - # Newer versions of Windows seems to have increased its internal - # buffer and tries to send as much of the data as it can as it - # has some form of buffering for this which is less than 256KiB - # on newer server versions and Windows 11. - # So DATA should be larger than 256 KiB to make this test reliable. - DATA = b"x" * (1024 * 256 + 1) + # Linux >= 6.10 seems buffering up to 17 pages of data. + # So DATA should be large enough to make this test reliable even with a + # 64 KiB page configuration. + DATA = b"x" * (1024 * 17 * 64 + 1) # Reduce socket buffer size to test on relative small data sets. BUF_SIZE = 4 * 1024 # 4 KiB diff --git a/Lib/test/test_asyncio/test_server.py b/Lib/test/test_asyncio/test_server.py index 918faac909b9bf..32211f4cba32cb 100644 --- a/Lib/test/test_asyncio/test_server.py +++ b/Lib/test/test_asyncio/test_server.py @@ -11,7 +11,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class BaseStartServer(func_tests.FunctionalTestCaseMixin): @@ -125,8 +125,12 @@ async def main(srv): class TestServer2(unittest.IsolatedAsyncioTestCase): async def test_wait_closed_basic(self): - async def serve(*args): - pass + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) self.addCleanup(srv.close) @@ -137,7 +141,8 @@ async def serve(*args): self.assertFalse(task1.done()) # active count != 0, not closed: should block - srv._attach() + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) task2 = asyncio.create_task(srv.wait_closed()) await asyncio.sleep(0) self.assertFalse(task1.done()) @@ -152,7 +157,8 @@ async def serve(*args): self.assertFalse(task2.done()) self.assertFalse(task3.done()) - srv._detach() + wr.close() + await wr.wait_closed() # active count == 0, closed: should unblock await task1 await task2 @@ -161,8 +167,12 @@ async def serve(*args): async def test_wait_closed_race(self): # Test a regression in 3.12.0, should be fixed in 3.12.1 - async def serve(*args): - pass + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) self.addCleanup(srv.close) @@ -170,13 +180,91 @@ async def serve(*args): task = asyncio.create_task(srv.wait_closed()) await asyncio.sleep(0) self.assertFalse(task.done()) - srv._attach() + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) loop = asyncio.get_running_loop() loop.call_soon(srv.close) - loop.call_soon(srv._detach) + loop.call_soon(wr.close) await srv.wait_closed() + async def test_close_clients(self): + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() + + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) + self.addCleanup(wr.close) + + task = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task.done()) + + srv.close() + srv.close_clients() + await asyncio.sleep(0) + await asyncio.sleep(0) + self.assertTrue(task.done()) + + async def test_abort_clients(self): + async def serve(rd, wr): + fut.set_result((rd, wr)) + await wr.wait_closed() + + fut = asyncio.Future() + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + addr = srv.sockets[0].getsockname() + (c_rd, c_wr) = await asyncio.open_connection(addr[0], addr[1], limit=4096) + self.addCleanup(c_wr.close) + + (s_rd, s_wr) = await fut + + # Limit the socket buffers so we can more reliably overfill them + s_sock = s_wr.get_extra_info('socket') + s_sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 65536) + c_sock = c_wr.get_extra_info('socket') + c_sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 65536) + + # Get the reader in to a paused state by sending more than twice + # the configured limit + s_wr.write(b'a' * 4096) + s_wr.write(b'a' * 4096) + s_wr.write(b'a' * 4096) + while c_wr.transport.is_reading(): + await asyncio.sleep(0) + + # Get the writer in a waiting state by sending data until the + # kernel stops accepting more data in the send buffer. + # gh-122136: getsockopt() does not reliably report the buffer size + # available for message content. + # We loop until we start filling up the asyncio buffer. + # To avoid an infinite loop we cap at 10 times the expected value + c_bufsize = c_sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF) + s_bufsize = s_sock.getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF) + for i in range(10): + s_wr.write(b'a' * c_bufsize) + s_wr.write(b'a' * s_bufsize) + if s_wr.transport.get_write_buffer_size() > 0: + break + self.assertNotEqual(s_wr.transport.get_write_buffer_size(), 0) + + task = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task.done()) + srv.close() + srv.abort_clients() + await asyncio.sleep(0) + await asyncio.sleep(0) + self.assertTrue(task.done()) # Test the various corner cases of Unix server socket removal diff --git a/Lib/test/test_asyncio/test_sock_lowlevel.py b/Lib/test/test_asyncio/test_sock_lowlevel.py index 075113cbe8e4a6..4f7b9a1dda6b78 100644 --- a/Lib/test/test_asyncio/test_sock_lowlevel.py +++ b/Lib/test/test_asyncio/test_sock_lowlevel.py @@ -15,7 +15,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class MyProto(asyncio.Protocol): @@ -110,7 +110,7 @@ def _basetest_sock_client_ops(self, httpd, sock): self.loop.run_until_complete( self.loop.sock_recv(sock, 1024)) sock.close() - self.assertTrue(data.startswith(b'HTTP/1.0 200 OK')) + self.assertStartsWith(data, b'HTTP/1.0 200 OK') def _basetest_sock_recv_into(self, httpd, sock): # same as _basetest_sock_client_ops, but using sock_recv_into @@ -127,7 +127,7 @@ def _basetest_sock_recv_into(self, httpd, sock): self.loop.run_until_complete( self.loop.sock_recv_into(sock, buf[nbytes:])) sock.close() - self.assertTrue(data.startswith(b'HTTP/1.0 200 OK')) + self.assertStartsWith(data, b'HTTP/1.0 200 OK') def test_sock_client_ops(self): with test_utils.run_test_server() as httpd: @@ -150,7 +150,7 @@ async def _basetest_sock_recv_racing(self, httpd, sock): # consume data await self.loop.sock_recv(sock, 1024) - self.assertTrue(data.startswith(b'HTTP/1.0 200 OK')) + self.assertStartsWith(data, b'HTTP/1.0 200 OK') async def _basetest_sock_recv_into_racing(self, httpd, sock): sock.setblocking(False) @@ -168,7 +168,7 @@ async def _basetest_sock_recv_into_racing(self, httpd, sock): nbytes = await self.loop.sock_recv_into(sock, buf[:1024]) # consume data await self.loop.sock_recv_into(sock, buf[nbytes:]) - self.assertTrue(data.startswith(b'HTTP/1.0 200 OK')) + self.assertStartsWith(data, b'HTTP/1.0 200 OK') await task @@ -217,7 +217,7 @@ async def recv_all(): sock.shutdown(socket.SHUT_WR) data = await task # ProactorEventLoop could deliver hello, so endswith is necessary - self.assertTrue(data.endswith(b'world')) + self.assertEndsWith(data, b'world') # After the first connect attempt before the listener is ready, # the socket needs time to "recover" to make the next connect call. @@ -298,7 +298,7 @@ async def _basetest_huge_content(self, address): data = await self.loop.sock_recv(sock, DATA_SIZE) # HTTP headers size is less than MTU, # they are sent by the first packet always - self.assertTrue(data.startswith(b'HTTP/1.0 200 OK')) + self.assertStartsWith(data, b'HTTP/1.0 200 OK') while data.find(b'\r\n\r\n') == -1: data += await self.loop.sock_recv(sock, DATA_SIZE) # Strip headers @@ -351,7 +351,7 @@ async def _basetest_huge_content_recvinto(self, address): data = bytes(buf[:nbytes]) # HTTP headers size is less than MTU, # they are sent by the first packet always - self.assertTrue(data.startswith(b'HTTP/1.0 200 OK')) + self.assertStartsWith(data, b'HTTP/1.0 200 OK') while data.find(b'\r\n\r\n') == -1: nbytes = await self.loop.sock_recv_into(sock, buf) data = bytes(buf[:nbytes]) @@ -555,12 +555,93 @@ class SelectEventLoopTests(BaseSockTestsMixin, def create_event_loop(self): return asyncio.SelectorEventLoop() + class ProactorEventLoopTests(BaseSockTestsMixin, test_utils.TestCase): def create_event_loop(self): return asyncio.ProactorEventLoop() + + async def _basetest_datagram_send_to_non_listening_address(self, + recvfrom): + # see: + # https://github.com/python/cpython/issues/91227 + # https://github.com/python/cpython/issues/88906 + # https://bugs.python.org/issue47071 + # https://bugs.python.org/issue44743 + # The Proactor event loop would fail to receive datagram messages + # after sending a message to an address that wasn't listening. + + def create_socket(): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setblocking(False) + sock.bind(('127.0.0.1', 0)) + return sock + + socket_1 = create_socket() + addr_1 = socket_1.getsockname() + + socket_2 = create_socket() + addr_2 = socket_2.getsockname() + + # creating and immediately closing this to try to get an address + # that is not listening + socket_3 = create_socket() + addr_3 = socket_3.getsockname() + socket_3.shutdown(socket.SHUT_RDWR) + socket_3.close() + + socket_1_recv_task = self.loop.create_task(recvfrom(socket_1)) + socket_2_recv_task = self.loop.create_task(recvfrom(socket_2)) + await asyncio.sleep(0) + + await self.loop.sock_sendto(socket_1, b'a', addr_2) + self.assertEqual(await socket_2_recv_task, b'a') + + await self.loop.sock_sendto(socket_2, b'b', addr_1) + self.assertEqual(await socket_1_recv_task, b'b') + socket_1_recv_task = self.loop.create_task(recvfrom(socket_1)) + await asyncio.sleep(0) + + # this should send to an address that isn't listening + await self.loop.sock_sendto(socket_1, b'c', addr_3) + self.assertEqual(await socket_1_recv_task, b'') + socket_1_recv_task = self.loop.create_task(recvfrom(socket_1)) + await asyncio.sleep(0) + + # socket 1 should still be able to receive messages after sending + # to an address that wasn't listening + socket_2.sendto(b'd', addr_1) + self.assertEqual(await socket_1_recv_task, b'd') + + socket_1.shutdown(socket.SHUT_RDWR) + socket_1.close() + socket_2.shutdown(socket.SHUT_RDWR) + socket_2.close() + + + def test_datagram_send_to_non_listening_address_recvfrom(self): + async def recvfrom(socket): + data, _ = await self.loop.sock_recvfrom(socket, 4096) + return data + + self.loop.run_until_complete( + self._basetest_datagram_send_to_non_listening_address( + recvfrom)) + + + def test_datagram_send_to_non_listening_address_recvfrom_into(self): + async def recvfrom_into(socket): + buf = bytearray(4096) + length, _ = await self.loop.sock_recvfrom_into(socket, buf, + 4096) + return buf[:length] + + self.loop.run_until_complete( + self._basetest_datagram_send_to_non_listening_address( + recvfrom_into)) + else: import selectors diff --git a/Lib/test/test_asyncio/test_ssl.py b/Lib/test/test_asyncio/test_ssl.py index e072ede29ee3c7..125a6c35793c44 100644 --- a/Lib/test/test_asyncio/test_ssl.py +++ b/Lib/test/test_asyncio/test_ssl.py @@ -29,7 +29,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class MyBaseProto(asyncio.Protocol): diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index f5f0afeab51c9e..aa248c5786f634 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -21,7 +21,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) @unittest.skipIf(ssl is None, 'No ssl module') @@ -109,6 +109,54 @@ def test_connection_lost(self): test_utils.run_briefly(self.loop) self.assertIsInstance(waiter.exception(), ConnectionAbortedError) + def test_connection_lost_when_busy(self): + # gh-118950: SSLProtocol.connection_lost not being called when OSError + # is thrown on asyncio.write. + sock = mock.Mock() + sock.fileno = mock.Mock(return_value=12345) + sock.send = mock.Mock(side_effect=BrokenPipeError) + + # construct StreamWriter chain that contains loop dependant logic this emulates + # what _make_ssl_transport() does in BaseSelectorEventLoop + reader = asyncio.StreamReader(limit=2 ** 16, loop=self.loop) + protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop) + ssl_proto = self.ssl_protocol(proto=protocol) + + # emulate reading decompressed data + sslobj = mock.Mock() + sslobj.read.side_effect = ssl.SSLWantReadError + sslobj.write.side_effect = ssl.SSLWantReadError + ssl_proto._sslobj = sslobj + + # emulate outgoing data + data = b'An interesting message' + + outgoing = mock.Mock() + outgoing.read = mock.Mock(return_value=data) + outgoing.pending = len(data) + ssl_proto._outgoing = outgoing + + # use correct socket transport to initialize the SSLProtocol + self.loop._make_socket_transport(sock, ssl_proto) + + transport = ssl_proto._app_transport + writer = asyncio.StreamWriter(transport, protocol, reader, self.loop) + + async def main(): + # writes data to transport + async def write(): + writer.write(data) + await writer.drain() + + # try to write for the first time + await write() + # try to write for the second time, this raises as the connection_lost + # callback should be done with error + with self.assertRaises(ConnectionResetError): + await write() + + self.loop.run_until_complete(main()) + def test_close_during_handshake(self): # bpo-29743 Closing transport during handshake process leaks socket waiter = self.loop.create_future() diff --git a/Lib/test/test_asyncio/test_staggered.py b/Lib/test/test_asyncio/test_staggered.py new file mode 100644 index 00000000000000..ad34aa6da01f54 --- /dev/null +++ b/Lib/test/test_asyncio/test_staggered.py @@ -0,0 +1,151 @@ +import asyncio +import unittest +from asyncio.staggered import staggered_race + +from test import support + +support.requires_working_socket(module=True) + + +def tearDownModule(): + asyncio._set_event_loop_policy(None) + + +class StaggeredTests(unittest.IsolatedAsyncioTestCase): + async def test_empty(self): + winner, index, excs = await staggered_race( + [], + delay=None, + ) + + self.assertIs(winner, None) + self.assertIs(index, None) + self.assertEqual(excs, []) + + async def test_one_successful(self): + async def coro(index): + return f'Res: {index}' + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=None, + ) + + self.assertEqual(winner, 'Res: 0') + self.assertEqual(index, 0) + self.assertEqual(excs, [None]) + + async def test_first_error_second_successful(self): + async def coro(index): + if index == 0: + raise ValueError(index) + return f'Res: {index}' + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=None, + ) + + self.assertEqual(winner, 'Res: 1') + self.assertEqual(index, 1) + self.assertEqual(len(excs), 2) + self.assertIsInstance(excs[0], ValueError) + self.assertIs(excs[1], None) + + async def test_first_timeout_second_successful(self): + async def coro(index): + if index == 0: + await asyncio.sleep(10) # much bigger than delay + return f'Res: {index}' + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=0.1, + ) + + self.assertEqual(winner, 'Res: 1') + self.assertEqual(index, 1) + self.assertEqual(len(excs), 2) + self.assertIsInstance(excs[0], asyncio.CancelledError) + self.assertIs(excs[1], None) + + async def test_none_successful(self): + async def coro(index): + raise ValueError(index) + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=None, + ) + + self.assertIs(winner, None) + self.assertIs(index, None) + self.assertEqual(len(excs), 2) + self.assertIsInstance(excs[0], ValueError) + self.assertIsInstance(excs[1], ValueError) + + + async def test_multiple_winners(self): + event = asyncio.Event() + + async def coro(index): + await event.wait() + return index + + async def do_set(): + event.set() + await asyncio.Event().wait() + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + do_set, + ], + delay=0.1, + ) + self.assertIs(winner, 0) + self.assertIs(index, 0) + self.assertEqual(len(excs), 3) + self.assertIsNone(excs[0], None) + self.assertIsInstance(excs[1], asyncio.CancelledError) + self.assertIsInstance(excs[2], asyncio.CancelledError) + + + async def test_cancelled(self): + log = [] + with self.assertRaises(TimeoutError): + async with asyncio.timeout(None) as cs_outer, asyncio.timeout(None) as cs_inner: + async def coro_fn(): + cs_inner.reschedule(-1) + await asyncio.sleep(0) + try: + await asyncio.sleep(0) + except asyncio.CancelledError: + log.append("cancelled 1") + + cs_outer.reschedule(-1) + await asyncio.sleep(0) + try: + await asyncio.sleep(0) + except asyncio.CancelledError: + log.append("cancelled 2") + try: + await staggered_race([coro_fn], delay=None) + except asyncio.CancelledError: + log.append("cancelled 3") + raise + + self.assertListEqual(log, ["cancelled 1", "cancelled 2", "cancelled 3"]) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index bf123ebf9bd158..673c6b46c647f3 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -21,7 +21,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class StreamTests(test_utils.TestCase): @@ -50,7 +50,7 @@ def _basetest_open_connection(self, open_connection_fut): self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') f = reader.read() data = self.loop.run_until_complete(f) - self.assertTrue(data.endswith(b'\r\n\r\nTest message')) + self.assertEndsWith(data, b'\r\n\r\nTest message') writer.close() self.assertEqual(messages, []) @@ -71,11 +71,11 @@ def _basetest_open_connection_no_loop_ssl(self, open_connection_fut): try: reader, writer = self.loop.run_until_complete(open_connection_fut) finally: - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) writer.write(b'GET / HTTP/1.0\r\n\r\n') f = reader.read() data = self.loop.run_until_complete(f) - self.assertTrue(data.endswith(b'\r\n\r\nTest message')) + self.assertEndsWith(data, b'\r\n\r\nTest message') writer.close() self.assertEqual(messages, []) @@ -383,6 +383,10 @@ def test_readuntil_separator(self): stream = asyncio.StreamReader(loop=self.loop) with self.assertRaisesRegex(ValueError, 'Separator should be'): self.loop.run_until_complete(stream.readuntil(separator=b'')) + with self.assertRaisesRegex(ValueError, 'Separator should be'): + self.loop.run_until_complete(stream.readuntil(separator=(b'',))) + with self.assertRaisesRegex(ValueError, 'Separator should contain'): + self.loop.run_until_complete(stream.readuntil(separator=())) def test_readuntil_multi_chunks(self): stream = asyncio.StreamReader(loop=self.loop) @@ -466,6 +470,55 @@ def test_readuntil_limit_found_sep(self): self.assertEqual(b'some dataAAA', stream._buffer) + def test_readuntil_multi_separator(self): + stream = asyncio.StreamReader(loop=self.loop) + + # Simple case + stream.feed_data(b'line 1\nline 2\r') + data = self.loop.run_until_complete(stream.readuntil((b'\r', b'\n'))) + self.assertEqual(b'line 1\n', data) + data = self.loop.run_until_complete(stream.readuntil((b'\r', b'\n'))) + self.assertEqual(b'line 2\r', data) + self.assertEqual(b'', stream._buffer) + + # First end position matches, even if that's a longer match + stream.feed_data(b'ABCDEFG') + data = self.loop.run_until_complete(stream.readuntil((b'DEF', b'BCDE'))) + self.assertEqual(b'ABCDE', data) + self.assertEqual(b'FG', stream._buffer) + + def test_readuntil_multi_separator_limit(self): + stream = asyncio.StreamReader(loop=self.loop, limit=3) + stream.feed_data(b'some dataA') + + with self.assertRaisesRegex(asyncio.LimitOverrunError, + 'is found') as cm: + self.loop.run_until_complete(stream.readuntil((b'A', b'ome dataA'))) + + self.assertEqual(b'some dataA', stream._buffer) + + def test_readuntil_multi_separator_negative_offset(self): + # If the buffer is big enough for the smallest separator (but does + # not contain it) but too small for the largest, `offset` must not + # become negative. + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'data') + + readuntil_task = self.loop.create_task(stream.readuntil((b'A', b'long sep'))) + self.loop.call_soon(stream.feed_data, b'Z') + self.loop.call_soon(stream.feed_data, b'Aaaa') + + data = self.loop.run_until_complete(readuntil_task) + self.assertEqual(b'dataZA', data) + self.assertEqual(b'aaa', stream._buffer) + + def test_readuntil_bytearray(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'some data\r\n') + data = self.loop.run_until_complete(stream.readuntil(bytearray(b'\r\n'))) + self.assertEqual(b'some data\r\n', data) + self.assertEqual(b'', stream._buffer) + def test_readexactly_zero_or_less(self): # Read exact number of bytes (zero or less). stream = asyncio.StreamReader(loop=self.loop) @@ -769,52 +822,6 @@ async def client(addr): self.assertEqual(msg1, b"hello world 1!\n") self.assertEqual(msg2, b"hello world 2!\n") - @unittest.skipIf(sys.platform == 'win32', "Don't have pipes") - @requires_subprocess() - def test_read_all_from_pipe_reader(self): - # See asyncio issue 168. This test is derived from the example - # subprocess_attach_read_pipe.py, but we configure the - # StreamReader's limit so that twice it is less than the size - # of the data writer. Also we must explicitly attach a child - # watcher to the event loop. - - code = """\ -import os, sys -fd = int(sys.argv[1]) -os.write(fd, b'data') -os.close(fd) -""" - rfd, wfd = os.pipe() - args = [sys.executable, '-c', code, str(wfd)] - - pipe = open(rfd, 'rb', 0) - reader = asyncio.StreamReader(loop=self.loop, limit=1) - protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop) - transport, _ = self.loop.run_until_complete( - self.loop.connect_read_pipe(lambda: protocol, pipe)) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = asyncio.SafeChildWatcher() - watcher.attach_loop(self.loop) - try: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - asyncio.set_child_watcher(watcher) - create = asyncio.create_subprocess_exec( - *args, - pass_fds={wfd}, - ) - proc = self.loop.run_until_complete(create) - self.loop.run_until_complete(proc.wait()) - finally: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - asyncio.set_child_watcher(None) - - os.close(wfd) - data = self.loop.run_until_complete(reader.read(-1)) - self.assertEqual(data, b'data') - def test_streamreader_constructor_without_loop(self): with self.assertRaisesRegex(RuntimeError, 'no current event loop'): asyncio.StreamReader() @@ -832,8 +839,8 @@ def test_streamreader_constructor_use_global_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor # retrieves the current loop if the loop parameter is not set # Deprecated in 3.10, undeprecated in 3.12 - self.addCleanup(asyncio.set_event_loop, None) - asyncio.set_event_loop(self.loop) + self.addCleanup(asyncio._set_event_loop, None) + asyncio._set_event_loop(self.loop) reader = asyncio.StreamReader() self.assertIs(reader._loop, self.loop) @@ -856,8 +863,8 @@ def test_streamreaderprotocol_constructor_use_global_loop(self): # asyncio issue #184: Ensure that StreamReaderProtocol constructor # retrieves the current loop if the loop parameter is not set # Deprecated in 3.10, undeprecated in 3.12 - self.addCleanup(asyncio.set_event_loop, None) - asyncio.set_event_loop(self.loop) + self.addCleanup(asyncio._set_event_loop, None) + asyncio._set_event_loop(self.loop) reader = mock.Mock() protocol = asyncio.StreamReaderProtocol(reader) self.assertIs(protocol._loop, self.loop) @@ -995,7 +1002,7 @@ def test_wait_closed_on_close(self): self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') f = rd.read() data = self.loop.run_until_complete(f) - self.assertTrue(data.endswith(b'\r\n\r\nTest message')) + self.assertEndsWith(data, b'\r\n\r\nTest message') self.assertFalse(wr.is_closing()) wr.close() self.assertTrue(wr.is_closing()) @@ -1021,7 +1028,7 @@ async def inner(httpd): data = await rd.readline() self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') data = await rd.read() - self.assertTrue(data.endswith(b'\r\n\r\nTest message')) + self.assertEndsWith(data, b'\r\n\r\nTest message') wr.close() await wr.wait_closed() @@ -1041,7 +1048,7 @@ async def inner(httpd): data = await rd.readline() self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') data = await rd.read() - self.assertTrue(data.endswith(b'\r\n\r\nTest message')) + self.assertEndsWith(data, b'\r\n\r\nTest message') wr.close() with self.assertRaises(ConnectionResetError): wr.write(b'data') @@ -1082,12 +1089,12 @@ async def inner(httpd): data = await rd.readline() self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') data = await rd.read() - self.assertTrue(data.endswith(b'\r\n\r\nTest message')) + self.assertEndsWith(data, b'\r\n\r\nTest message') with self.assertWarns(ResourceWarning) as cm: del wr gc.collect() self.assertEqual(len(cm.warnings), 1) - self.assertTrue(str(cm.warnings[0].message).startswith("unclosed None: def test_subprocess_protocol_events(self): # gh-108973: Test that all subprocess protocol methods are called. - # The protocol methods are not called in a determistic order. + # The protocol methods are not called in a deterministic order. # The order depends on the event loop and the operating system. events = [] fds = [1, 2] @@ -872,60 +864,51 @@ async def main(): self.loop.run_until_complete(main()) + @unittest.skipIf(sys.platform != 'linux', "Linux only") + def test_subprocess_send_signal_race(self): + # See https://github.com/python/cpython/issues/87744 + async def main(): + for _ in range(10): + proc = await asyncio.create_subprocess_exec('sleep', '0.1') + await asyncio.sleep(0.1) + try: + proc.send_signal(signal.SIGUSR1) + except ProcessLookupError: + pass + self.assertNotEqual(await proc.wait(), 255) + + self.loop.run_until_complete(main()) + if sys.platform != 'win32': # Unix class SubprocessWatcherMixin(SubprocessMixin): - Watcher = None - def setUp(self): super().setUp() - policy = asyncio.get_event_loop_policy() - self.loop = policy.new_event_loop() + self.loop = asyncio.new_event_loop() self.set_event_loop(self.loop) - watcher = self._get_watcher() - watcher.attach_loop(self.loop) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - policy.set_child_watcher(watcher) + def test_watcher_implementation(self): + loop = self.loop + watcher = loop._watcher + if unix_events.can_use_pidfd(): + self.assertIsInstance(watcher, unix_events._PidfdChildWatcher) + else: + self.assertIsInstance(watcher, unix_events._ThreadedChildWatcher) - def tearDown(self): - super().tearDown() - policy = asyncio.get_event_loop_policy() - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = policy.get_child_watcher() - policy.set_child_watcher(None) - watcher.attach_loop(None) - watcher.close() class SubprocessThreadedWatcherTests(SubprocessWatcherMixin, test_utils.TestCase): + def setUp(self): + self._original_can_use_pidfd = unix_events.can_use_pidfd + # Force the use of the threaded child watcher + unix_events.can_use_pidfd = mock.Mock(return_value=False) + super().setUp() - def _get_watcher(self): - return unix_events.ThreadedChildWatcher() - - class SubprocessSafeWatcherTests(SubprocessWatcherMixin, - test_utils.TestCase): - - def _get_watcher(self): - with self.assertWarns(DeprecationWarning): - return unix_events.SafeChildWatcher() - - class MultiLoopChildWatcherTests(test_utils.TestCase): - - def test_warns(self): - with self.assertWarns(DeprecationWarning): - unix_events.MultiLoopChildWatcher() - - class SubprocessFastWatcherTests(SubprocessWatcherMixin, - test_utils.TestCase): - - def _get_watcher(self): - with self.assertWarns(DeprecationWarning): - return unix_events.FastChildWatcher() + def tearDown(self): + unix_events.can_use_pidfd = self._original_can_use_pidfd + return super().tearDown() @unittest.skipUnless( unix_events.can_use_pidfd(), @@ -934,70 +917,8 @@ def _get_watcher(self): class SubprocessPidfdWatcherTests(SubprocessWatcherMixin, test_utils.TestCase): - def _get_watcher(self): - return unix_events.PidfdChildWatcher() - - - class GenericWatcherTests(test_utils.TestCase): - - def test_create_subprocess_fails_with_inactive_watcher(self): - watcher = mock.create_autospec(asyncio.AbstractChildWatcher) - watcher.is_active.return_value = False - - async def execute(): - asyncio.set_child_watcher(watcher) - - with self.assertRaises(RuntimeError): - await subprocess.create_subprocess_exec( - os_helper.FakePath(sys.executable), '-c', 'pass') + pass - watcher.add_child_handler.assert_not_called() - - with asyncio.Runner(loop_factory=asyncio.new_event_loop) as runner: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - self.assertIsNone(runner.run(execute())) - self.assertListEqual(watcher.mock_calls, [ - mock.call.__enter__(), - mock.call.is_active(), - mock.call.__exit__(RuntimeError, mock.ANY, mock.ANY), - ], watcher.mock_calls) - - - @unittest.skipUnless( - unix_events.can_use_pidfd(), - "operating system does not support pidfds", - ) - def test_create_subprocess_with_pidfd(self): - async def in_thread(): - proc = await asyncio.create_subprocess_exec( - *PROGRAM_CAT, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - stdout, stderr = await proc.communicate(b"some data") - return proc.returncode, stdout - - async def main(): - # asyncio.Runner did not call asyncio.set_event_loop() - with warnings.catch_warnings(): - warnings.simplefilter('error', DeprecationWarning) - # get_event_loop() raises DeprecationWarning if - # set_event_loop() was never called and RuntimeError if - # it was called at least once. - with self.assertRaises((RuntimeError, DeprecationWarning)): - asyncio.get_event_loop_policy().get_event_loop() - return await asyncio.to_thread(asyncio.run, in_thread()) - with self.assertWarns(DeprecationWarning): - asyncio.set_child_watcher(asyncio.PidfdChildWatcher()) - try: - with asyncio.Runner(loop_factory=asyncio.new_event_loop) as runner: - returncode, stdout = runner.run(main()) - self.assertEqual(returncode, 0) - self.assertEqual(stdout, b'some data') - finally: - with self.assertWarns(DeprecationWarning): - asyncio.set_child_watcher(None) else: # Windows class SubprocessProactorTests(SubprocessMixin, test_utils.TestCase): diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py index 7a18362b54e469..0d69a436fdb840 100644 --- a/Lib/test/test_asyncio/test_taskgroups.py +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -1,19 +1,21 @@ # Adapted with permission from the EdgeDB project; # license: PSFL. - +import weakref +import sys +import gc import asyncio import contextvars import contextlib from asyncio import taskgroups import unittest +import warnings from test.test_asyncio.utils import await_without_task - # To prevent a warning "test altered the execution environment" def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class MyExc(Exception): @@ -28,7 +30,34 @@ def get_error_types(eg): return {type(exc) for exc in eg.exceptions} -class TestTaskGroup(unittest.IsolatedAsyncioTestCase): +def no_other_refs(): + # due to gh-124392 coroutines now refer to their locals + coro = asyncio.current_task().get_coro() + frame = sys._getframe(1) + while coro.cr_frame != frame: + coro = coro.cr_await + return [coro] + + +def set_gc_state(enabled): + was_enabled = gc.isenabled() + if enabled: + gc.enable() + else: + gc.disable() + return was_enabled + + +@contextlib.contextmanager +def disable_gc(): + was_enabled = set_gc_state(enabled=False) + try: + yield + finally: + set_gc_state(enabled=was_enabled) + + +class BaseTestTaskGroup: async def test_taskgroup_01(self): @@ -738,10 +767,7 @@ async def coro2(g): await asyncio.sleep(1) except asyncio.CancelledError: with self.assertRaises(RuntimeError): - g.create_task(c1 := coro1()) - # We still have to await c1 to avoid a warning - with self.assertRaises(ZeroDivisionError): - await c1 + g.create_task(coro1()) with self.assertRaises(ExceptionGroup) as cm: async with taskgroups.TaskGroup() as g: @@ -797,22 +823,25 @@ async def test_taskgroup_double_enter(self): pass async def test_taskgroup_finished(self): - tg = taskgroups.TaskGroup() - async with tg: - pass - coro = asyncio.sleep(0) - with self.assertRaisesRegex(RuntimeError, "is finished"): - tg.create_task(coro) - # We still have to await coro to avoid a warning - await coro + async def create_task_after_tg_finish(): + tg = taskgroups.TaskGroup() + async with tg: + pass + coro = asyncio.sleep(0) + with self.assertRaisesRegex(RuntimeError, "is finished"): + tg.create_task(coro) + + # Make sure the coroutine was closed when submitted to the inactive tg + # (if not closed, a RuntimeWarning should have been raised) + with warnings.catch_warnings(record=True) as w: + await create_task_after_tg_finish() + self.assertEqual(len(w), 0) async def test_taskgroup_not_entered(self): tg = taskgroups.TaskGroup() coro = asyncio.sleep(0) with self.assertRaisesRegex(RuntimeError, "has not been entered"): tg.create_task(coro) - # We still have to await coro to avoid a warning - await coro async def test_taskgroup_without_parent_task(self): tg = taskgroups.TaskGroup() @@ -821,8 +850,268 @@ async def test_taskgroup_without_parent_task(self): coro = asyncio.sleep(0) with self.assertRaisesRegex(RuntimeError, "has not been entered"): tg.create_task(coro) - # We still have to await coro to avoid a warning - await coro + + async def test_coro_closed_when_tg_closed(self): + async def run_coro_after_tg_closes(): + async with taskgroups.TaskGroup() as tg: + pass + coro = asyncio.sleep(0) + with self.assertRaisesRegex(RuntimeError, "is finished"): + tg.create_task(coro) + + await run_coro_after_tg_closes() + + async def test_cancelling_level_preserved(self): + async def raise_after(t, e): + await asyncio.sleep(t) + raise e() + + try: + async with asyncio.TaskGroup() as tg: + tg.create_task(raise_after(0.0, RuntimeError)) + except* RuntimeError: + pass + self.assertEqual(asyncio.current_task().cancelling(), 0) + + async def test_nested_groups_both_cancelled(self): + async def raise_after(t, e): + await asyncio.sleep(t) + raise e() + + try: + async with asyncio.TaskGroup() as outer_tg: + try: + async with asyncio.TaskGroup() as inner_tg: + inner_tg.create_task(raise_after(0, RuntimeError)) + outer_tg.create_task(raise_after(0, ValueError)) + except* RuntimeError: + pass + else: + self.fail("RuntimeError not raised") + self.assertEqual(asyncio.current_task().cancelling(), 1) + except* ValueError: + pass + else: + self.fail("ValueError not raised") + self.assertEqual(asyncio.current_task().cancelling(), 0) + + async def test_error_and_cancel(self): + event = asyncio.Event() + + async def raise_error(): + event.set() + await asyncio.sleep(0) + raise RuntimeError() + + async def inner(): + try: + async with taskgroups.TaskGroup() as tg: + tg.create_task(raise_error()) + await asyncio.sleep(1) + self.fail("Sleep in group should have been cancelled") + except* RuntimeError: + self.assertEqual(asyncio.current_task().cancelling(), 1) + self.assertEqual(asyncio.current_task().cancelling(), 1) + await asyncio.sleep(1) + self.fail("Sleep after group should have been cancelled") + + async def outer(): + t = asyncio.create_task(inner()) + await event.wait() + self.assertEqual(t.cancelling(), 0) + t.cancel() + self.assertEqual(t.cancelling(), 1) + with self.assertRaises(asyncio.CancelledError): + await t + self.assertTrue(t.cancelled()) + + await outer() + + async def test_exception_refcycles_direct(self): + """Test that TaskGroup doesn't keep a reference to the raised ExceptionGroup""" + tg = asyncio.TaskGroup() + exc = None + + class _Done(Exception): + pass + + try: + async with tg: + raise _Done + except ExceptionGroup as e: + exc = e + + self.assertIsNotNone(exc) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + + async def test_exception_refcycles_errors(self): + """Test that TaskGroup deletes self._errors, and __aexit__ args""" + tg = asyncio.TaskGroup() + exc = None + + class _Done(Exception): + pass + + try: + async with tg: + raise _Done + except* _Done as excs: + exc = excs.exceptions[0] + + self.assertIsInstance(exc, _Done) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + + async def test_exception_refcycles_parent_task(self): + """Test that TaskGroup deletes self._parent_task""" + tg = asyncio.TaskGroup() + exc = None + + class _Done(Exception): + pass + + async def coro_fn(): + async with tg: + raise _Done + + try: + async with asyncio.TaskGroup() as tg2: + tg2.create_task(coro_fn()) + except* _Done as excs: + exc = excs.exceptions[0].exceptions[0] + + self.assertIsInstance(exc, _Done) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + + async def test_exception_refcycles_parent_task_wr(self): + """Test that TaskGroup deletes self._parent_task and create_task() deletes task""" + tg = asyncio.TaskGroup() + exc = None + + class _Done(Exception): + pass + + async def coro_fn(): + async with tg: + raise _Done + + with disable_gc(): + try: + async with asyncio.TaskGroup() as tg2: + task_wr = weakref.ref(tg2.create_task(coro_fn())) + except* _Done as excs: + exc = excs.exceptions[0].exceptions[0] + + self.assertIsNone(task_wr()) + self.assertIsInstance(exc, _Done) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + async def test_exception_refcycles_propagate_cancellation_error(self): + """Test that TaskGroup deletes propagate_cancellation_error""" + tg = asyncio.TaskGroup() + exc = None + + try: + async with asyncio.timeout(-1): + async with tg: + await asyncio.sleep(0) + except TimeoutError as e: + exc = e.__cause__ + + self.assertIsInstance(exc, asyncio.CancelledError) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + async def test_exception_refcycles_base_error(self): + """Test that TaskGroup deletes self._base_error""" + class MyKeyboardInterrupt(KeyboardInterrupt): + pass + + tg = asyncio.TaskGroup() + exc = None + + try: + async with tg: + raise MyKeyboardInterrupt + except MyKeyboardInterrupt as e: + exc = e + + self.assertIsNotNone(exc) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + async def test_name(self): + name = None + + async def asyncfn(): + nonlocal name + name = asyncio.current_task().get_name() + + async with asyncio.TaskGroup() as tg: + tg.create_task(asyncfn(), name="example name") + + self.assertEqual(name, "example name") + + + async def test_cancels_task_if_created_during_creation(self): + # regression test for gh-128550 + ran = False + class MyError(Exception): + pass + + exc = None + try: + async with asyncio.TaskGroup() as tg: + async def third_task(): + raise MyError("third task failed") + + async def second_task(): + nonlocal ran + tg.create_task(third_task()) + with self.assertRaises(asyncio.CancelledError): + await asyncio.sleep(0) # eager tasks cancel here + await asyncio.sleep(0) # lazy tasks cancel here + ran = True + + tg.create_task(second_task()) + except* MyError as excs: + exc = excs.exceptions[0] + + self.assertTrue(ran) + self.assertIsInstance(exc, MyError) + + + async def test_cancellation_does_not_leak_out_of_tg(self): + class MyError(Exception): + pass + + async def throw_error(): + raise MyError + + try: + async with asyncio.TaskGroup() as tg: + tg.create_task(throw_error()) + except* MyError: + pass + else: + self.fail("should have raised one MyError in group") + + # if this test fails this current task will be cancelled + # outside the task group and inside unittest internals + # we yield to the event loop with sleep(0) so that + # cancellation happens here and error is more understandable + await asyncio.sleep(0) + + +class TestTaskGroup(BaseTestTaskGroup, unittest.IsolatedAsyncioTestCase): + loop_factory = asyncio.EventLoop + +class TestEagerTaskTaskGroup(BaseTestTaskGroup, unittest.IsolatedAsyncioTestCase): + @staticmethod + def loop_factory(): + loop = asyncio.EventLoop() + loop.set_task_factory(asyncio.eager_task_factory) + return loop if __name__ == "__main__": diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 4dfaff847edb90..7a052817766a07 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -1,6 +1,7 @@ """Tests for tasks.py.""" import collections +import contextlib import contextvars import gc import io @@ -19,10 +20,11 @@ from test.test_asyncio import utils as test_utils from test import support from test.support.script_helper import assert_python_ok +from test.support.warnings_helper import ignore_warnings def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) async def coroutine_function(): @@ -85,6 +87,7 @@ class BaseTaskTests: Task = None Future = None + all_tasks = None def new_task(self, loop, coro, name='TestTask', context=None): return self.__class__.Task(coro, loop=loop, name=name, context=context) @@ -107,7 +110,7 @@ def test_task_cancel_message_getter(self): async def coro(): pass t = self.new_task(self.loop, coro()) - self.assertTrue(hasattr(t, '_cancel_message')) + self.assertHasAttr(t, '_cancel_message') self.assertEqual(t._cancel_message, None) t.cancel('my message') @@ -209,8 +212,8 @@ async def test(): self.assertEqual(t.result(), 'ok') # Deprecated in 3.10, undeprecated in 3.12 - asyncio.set_event_loop(self.loop) - self.addCleanup(asyncio.set_event_loop, None) + asyncio._set_event_loop(self.loop) + self.addCleanup(asyncio._set_event_loop, None) t = asyncio.ensure_future(notmuch()) self.assertIs(t._loop, self.loop) self.loop.run_until_complete(t) @@ -683,6 +686,30 @@ def on_timeout(): finally: loop.close() + def test_uncancel_resets_must_cancel(self): + + async def coro(): + await fut + return 42 + + loop = asyncio.new_event_loop() + fut = asyncio.Future(loop=loop) + task = self.new_task(loop, coro()) + loop.run_until_complete(asyncio.sleep(0)) # Get task waiting for fut + fut.set_result(None) # Make task runnable + try: + task.cancel() # Enter cancelled state + self.assertEqual(task.cancelling(), 1) + self.assertTrue(task._must_cancel) + + task.uncancel() # Undo cancellation + self.assertEqual(task.cancelling(), 0) + self.assertFalse(task._must_cancel) + finally: + res = loop.run_until_complete(task) + self.assertEqual(res, 42) + loop.close() + def test_cancel(self): def gen(): @@ -1409,12 +1436,6 @@ def gen(): yield 0.01 yield 0 - loop = self.new_test_loop(gen) - # disable "slow callback" warning - loop.slow_callback_duration = 1.0 - completed = set() - time_shifted = False - async def sleeper(dt, x): nonlocal time_shifted await asyncio.sleep(dt) @@ -1424,21 +1445,78 @@ async def sleeper(dt, x): loop.advance_time(0.14) return x - a = sleeper(0.01, 'a') - b = sleeper(0.01, 'b') - c = sleeper(0.15, 'c') + async def try_iterator(awaitables): + values = [] + for f in asyncio.as_completed(awaitables): + values.append(await f) + return values - async def foo(): + async def try_async_iterator(awaitables): values = [] - for f in asyncio.as_completed([b, c, a]): + async for f in asyncio.as_completed(awaitables): values.append(await f) return values - res = loop.run_until_complete(self.new_task(loop, foo())) - self.assertAlmostEqual(0.15, loop.time()) - self.assertTrue('a' in res[:2]) - self.assertTrue('b' in res[:2]) - self.assertEqual(res[2], 'c') + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + loop = self.new_test_loop(gen) + # disable "slow callback" warning + loop.slow_callback_duration = 1.0 + + completed = set() + time_shifted = False + + a = sleeper(0.01, 'a') + b = sleeper(0.01, 'b') + c = sleeper(0.15, 'c') + + res = loop.run_until_complete(self.new_task(loop, foo([b, c, a]))) + self.assertAlmostEqual(0.15, loop.time()) + self.assertTrue('a' in res[:2]) + self.assertTrue('b' in res[:2]) + self.assertEqual(res[2], 'c') + + def test_as_completed_same_tasks_in_as_out(self): + # Ensures that asynchronously iterating as_completed's iterator + # yields awaitables are the same awaitables that were passed in when + # those awaitables are futures. + async def try_async_iterator(awaitables): + awaitables_out = set() + async for out_aw in asyncio.as_completed(awaitables): + awaitables_out.add(out_aw) + return awaitables_out + + async def coro(i): + return i + + with contextlib.closing(asyncio.new_event_loop()) as loop: + # Coroutines shouldn't be yielded back as finished coroutines + # can't be re-used. + awaitables_in = frozenset( + (coro(0), coro(1), coro(2), coro(3)) + ) + awaitables_out = loop.run_until_complete( + try_async_iterator(awaitables_in) + ) + if awaitables_in - awaitables_out != awaitables_in: + raise self.failureException('Got original coroutines ' + 'out of as_completed iterator.') + + # Tasks should be yielded back. + coro_obj_a = coro('a') + task_b = loop.create_task(coro('b')) + coro_obj_c = coro('c') + task_d = loop.create_task(coro('d')) + awaitables_in = frozenset( + (coro_obj_a, task_b, coro_obj_c, task_d) + ) + awaitables_out = loop.run_until_complete( + try_async_iterator(awaitables_in) + ) + if awaitables_in & awaitables_out != {task_b, task_d}: + raise self.failureException('Only tasks should be yielded ' + 'from as_completed iterator ' + 'as-is.') def test_as_completed_with_timeout(self): @@ -1448,12 +1526,7 @@ def gen(): yield 0 yield 0.1 - loop = self.new_test_loop(gen) - - a = loop.create_task(asyncio.sleep(0.1, 'a')) - b = loop.create_task(asyncio.sleep(0.15, 'b')) - - async def foo(): + async def try_iterator(): values = [] for f in asyncio.as_completed([a, b], timeout=0.12): if values: @@ -1465,16 +1538,33 @@ async def foo(): values.append((2, exc)) return values - res = loop.run_until_complete(self.new_task(loop, foo())) - self.assertEqual(len(res), 2, res) - self.assertEqual(res[0], (1, 'a')) - self.assertEqual(res[1][0], 2) - self.assertIsInstance(res[1][1], asyncio.TimeoutError) - self.assertAlmostEqual(0.12, loop.time()) + async def try_async_iterator(): + values = [] + try: + async for f in asyncio.as_completed([a, b], timeout=0.12): + v = await f + values.append((1, v)) + loop.advance_time(0.02) + except asyncio.TimeoutError as exc: + values.append((2, exc)) + return values - # move forward to close generator - loop.advance_time(10) - loop.run_until_complete(asyncio.wait([a, b])) + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + loop = self.new_test_loop(gen) + a = loop.create_task(asyncio.sleep(0.1, 'a')) + b = loop.create_task(asyncio.sleep(0.15, 'b')) + + res = loop.run_until_complete(self.new_task(loop, foo())) + self.assertEqual(len(res), 2, res) + self.assertEqual(res[0], (1, 'a')) + self.assertEqual(res[1][0], 2) + self.assertIsInstance(res[1][1], asyncio.TimeoutError) + self.assertAlmostEqual(0.12, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b])) def test_as_completed_with_unused_timeout(self): @@ -1483,19 +1573,75 @@ def gen(): yield 0 yield 0.01 - loop = self.new_test_loop(gen) - - a = asyncio.sleep(0.01, 'a') - - async def foo(): + async def try_iterator(): for f in asyncio.as_completed([a], timeout=1): v = await f self.assertEqual(v, 'a') - loop.run_until_complete(self.new_task(loop, foo())) + async def try_async_iterator(): + async for f in asyncio.as_completed([a], timeout=1): + v = await f + self.assertEqual(v, 'a') - def test_as_completed_reverse_wait(self): + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + a = asyncio.sleep(0.01, 'a') + loop = self.new_test_loop(gen) + loop.run_until_complete(self.new_task(loop, foo())) + loop.close() + + def test_as_completed_resume_iterator(self): + # Test that as_completed returns an iterator that can be resumed + # the next time iteration is performed (i.e. if __iter__ is called + # again) + async def try_iterator(awaitables): + iterations = 0 + iterator = asyncio.as_completed(awaitables) + collected = [] + for f in iterator: + collected.append(await f) + iterations += 1 + if iterations == 2: + break + self.assertEqual(len(collected), 2) + + # Resume same iterator: + for f in iterator: + collected.append(await f) + return collected + + async def try_async_iterator(awaitables): + iterations = 0 + iterator = asyncio.as_completed(awaitables) + collected = [] + async for f in iterator: + collected.append(await f) + iterations += 1 + if iterations == 2: + break + self.assertEqual(len(collected), 2) + + # Resume same iterator: + async for f in iterator: + collected.append(await f) + return collected + + async def coro(i): + return i + + with contextlib.closing(asyncio.new_event_loop()) as loop: + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + results = loop.run_until_complete( + foo((coro(0), coro(1), coro(2), coro(3))) + ) + self.assertCountEqual(results, (0, 1, 2, 3)) + def test_as_completed_reverse_wait(self): + # Tests the plain iterator style of as_completed iteration to + # ensure that the first future awaited resolves to the first + # completed awaitable from the set we passed in, even if it wasn't + # the first future generated by as_completed. def gen(): yield 0 yield 0.05 @@ -1522,7 +1668,8 @@ async def test(): loop.run_until_complete(test()) def test_as_completed_concurrent(self): - + # Ensure that more than one future or coroutine yielded from + # as_completed can be awaited concurrently. def gen(): when = yield self.assertAlmostEqual(0.05, when) @@ -1530,38 +1677,55 @@ def gen(): self.assertAlmostEqual(0.05, when) yield 0.05 - a = asyncio.sleep(0.05, 'a') - b = asyncio.sleep(0.05, 'b') - fs = {a, b} + async def try_iterator(fs): + return list(asyncio.as_completed(fs)) - async def test(): - futs = list(asyncio.as_completed(fs)) - self.assertEqual(len(futs), 2) - done, pending = await asyncio.wait( - [asyncio.ensure_future(fut) for fut in futs] - ) - self.assertEqual(set(f.result() for f in done), {'a', 'b'}) + async def try_async_iterator(fs): + return [f async for f in asyncio.as_completed(fs)] - loop = self.new_test_loop(gen) - loop.run_until_complete(test()) + for runner in try_iterator, try_async_iterator: + with self.subTest(method=runner.__name__): + a = asyncio.sleep(0.05, 'a') + b = asyncio.sleep(0.05, 'b') + fs = {a, b} + + async def test(): + futs = await runner(fs) + self.assertEqual(len(futs), 2) + done, pending = await asyncio.wait( + [asyncio.ensure_future(fut) for fut in futs] + ) + self.assertEqual(set(f.result() for f in done), {'a', 'b'}) + + loop = self.new_test_loop(gen) + loop.run_until_complete(test()) def test_as_completed_duplicate_coroutines(self): async def coro(s): return s - async def runner(): + async def try_iterator(): result = [] c = coro('ham') for f in asyncio.as_completed([c, c, coro('spam')]): result.append(await f) return result - fut = self.new_task(self.loop, runner()) - self.loop.run_until_complete(fut) - result = fut.result() - self.assertEqual(set(result), {'ham', 'spam'}) - self.assertEqual(len(result), 2) + async def try_async_iterator(): + result = [] + c = coro('ham') + async for f in asyncio.as_completed([c, c, coro('spam')]): + result.append(await f) + return result + + for runner in try_iterator, try_async_iterator: + with self.subTest(method=runner.__name__): + fut = self.new_task(self.loop, runner()) + self.loop.run_until_complete(fut) + result = fut.result() + self.assertEqual(set(result), {'ham', 'spam'}) + self.assertEqual(len(result), 2) def test_as_completed_coroutine_without_loop(self): async def coro(): @@ -1570,8 +1734,8 @@ async def coro(): a = coro() self.addCleanup(a.close) - futs = asyncio.as_completed([a]) with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + futs = asyncio.as_completed([a]) list(futs) def test_as_completed_coroutine_use_running_loop(self): @@ -1776,6 +1940,7 @@ async def notmutch(): self.assertFalse(task.cancelled()) self.assertIs(task.exception(), base_exc) + @ignore_warnings(category=DeprecationWarning) def test_iscoroutinefunction(self): def fn(): pass @@ -1793,6 +1958,7 @@ async def fn2(): self.assertFalse(asyncio.iscoroutinefunction(mock.Mock())) self.assertTrue(asyncio.iscoroutinefunction(mock.AsyncMock())) + @ignore_warnings(category=DeprecationWarning) def test_coroutine_non_gen_function(self): async def func(): return 'test' @@ -2036,22 +2202,40 @@ def test_shield_coroutine_use_global_loop(self): async def coro(): return 42 - asyncio.set_event_loop(self.loop) - self.addCleanup(asyncio.set_event_loop, None) + asyncio._set_event_loop(self.loop) + self.addCleanup(asyncio._set_event_loop, None) outer = asyncio.shield(coro()) self.assertEqual(outer._loop, self.loop) res = self.loop.run_until_complete(outer) self.assertEqual(res, 42) def test_as_completed_invalid_args(self): + # as_completed() expects a list of futures, not a future instance + # TypeError should be raised either on iterator construction or first + # iteration + + # Plain iterator fut = self.new_future(self.loop) + with self.assertRaises(TypeError): + iterator = asyncio.as_completed(fut) + next(iterator) + coro = coroutine_function() + with self.assertRaises(TypeError): + iterator = asyncio.as_completed(coro) + next(iterator) + coro.close() - # as_completed() expects a list of futures, not a future instance - self.assertRaises(TypeError, self.loop.run_until_complete, - asyncio.as_completed(fut)) + # Async iterator + async def try_async_iterator(aw): + async for f in asyncio.as_completed(aw): + break + + fut = self.new_future(self.loop) + with self.assertRaises(TypeError): + self.loop.run_until_complete(try_async_iterator(fut)) coro = coroutine_function() - self.assertRaises(TypeError, self.loop.run_until_complete, - asyncio.as_completed(coro)) + with self.assertRaises(TypeError): + self.loop.run_until_complete(try_async_iterator(coro)) coro.close() def test_wait_invalid_args(self): @@ -2087,9 +2271,9 @@ async def kill_me(loop): coro = kill_me(self.loop) task = asyncio.ensure_future(coro, loop=self.loop) - self.assertEqual(asyncio.all_tasks(loop=self.loop), {task}) + self.assertEqual(self.all_tasks(loop=self.loop), {task}) - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) # execute the task so it waits for future self.loop._run_once() @@ -2102,7 +2286,7 @@ async def kill_me(loop): # no more reference to kill_me() task: the task is destroyed by the GC support.gc_collect() - self.assertEqual(asyncio.all_tasks(loop=self.loop), set()) + self.assertEqual(self.all_tasks(loop=self.loop), set()) mock_handler.assert_called_with(self.loop, { 'message': 'Task was destroyed but it is pending!', @@ -2251,7 +2435,7 @@ async def coro(): message = m_log.error.call_args[0][0] self.assertIn('Task was destroyed but it is pending', message) - self.assertEqual(asyncio.all_tasks(self.loop), set()) + self.assertEqual(self.all_tasks(self.loop), set()) def test_create_task_with_noncoroutine(self): with self.assertRaisesRegex(TypeError, @@ -2504,6 +2688,28 @@ def test_get_context(self): finally: loop.close() + def test_proper_refcounts(self): + # see: https://github.com/python/cpython/issues/126083 + class Break: + def __str__(self): + raise RuntimeError("break") + + obj = object() + initial_refcount = sys.getrefcount(obj) + + coro = coroutine_function() + with contextlib.closing(asyncio.EventLoop()) as loop: + task = asyncio.Task.__new__(asyncio.Task) + + for _ in range(5): + with self.assertRaisesRegex(RuntimeError, 'break'): + task.__init__(coro, loop=loop, context=obj, name=Break()) + + coro.close() + del task + + self.assertEqual(sys.getrefcount(obj), initial_refcount) + def add_subclass_tests(cls): BaseTask = cls.Task @@ -2551,6 +2757,7 @@ async def func(): # Add patched Task & Future back to the test case cls.Task = Task cls.Future = Future + cls.all_tasks = tasks.all_tasks # Add an extra unit-test cls.test_subclasses_ctask_cfuture = test_subclasses_ctask_cfuture @@ -2624,6 +2831,7 @@ class CTask_CFuture_Tests(BaseTaskTests, SetMethodsTest, Task = getattr(tasks, '_CTask', None) Future = getattr(futures, '_CFuture', None) + all_tasks = getattr(tasks, '_c_all_tasks', None) @support.refcount_test def test_refleaks_in_task___init__(self): @@ -2655,6 +2863,7 @@ class CTask_CFuture_SubclassTests(BaseTaskTests, test_utils.TestCase): Task = getattr(tasks, '_CTask', None) Future = getattr(futures, '_CFuture', None) + all_tasks = getattr(tasks, '_c_all_tasks', None) @unittest.skipUnless(hasattr(tasks, '_CTask'), @@ -2664,6 +2873,7 @@ class CTaskSubclass_PyFuture_Tests(BaseTaskTests, test_utils.TestCase): Task = getattr(tasks, '_CTask', None) Future = futures._PyFuture + all_tasks = getattr(tasks, '_c_all_tasks', None) @unittest.skipUnless(hasattr(futures, '_CFuture'), @@ -2673,6 +2883,7 @@ class PyTask_CFutureSubclass_Tests(BaseTaskTests, test_utils.TestCase): Future = getattr(futures, '_CFuture', None) Task = tasks._PyTask + all_tasks = tasks._py_all_tasks @unittest.skipUnless(hasattr(tasks, '_CTask'), @@ -2681,6 +2892,7 @@ class CTask_PyFuture_Tests(BaseTaskTests, test_utils.TestCase): Task = getattr(tasks, '_CTask', None) Future = futures._PyFuture + all_tasks = getattr(tasks, '_c_all_tasks', None) @unittest.skipUnless(hasattr(futures, '_CFuture'), @@ -2689,6 +2901,7 @@ class PyTask_CFuture_Tests(BaseTaskTests, test_utils.TestCase): Task = tasks._PyTask Future = getattr(futures, '_CFuture', None) + all_tasks = staticmethod(tasks._py_all_tasks) class PyTask_PyFuture_Tests(BaseTaskTests, SetMethodsTest, @@ -2696,6 +2909,7 @@ class PyTask_PyFuture_Tests(BaseTaskTests, SetMethodsTest, Task = tasks._PyTask Future = futures._PyFuture + all_tasks = staticmethod(tasks._py_all_tasks) @add_subclass_tests @@ -2735,6 +2949,7 @@ class BaseTaskIntrospectionTests: _unregister_task = None _enter_task = None _leave_task = None + all_tasks = None def test__register_task_1(self): class TaskLike: @@ -2748,9 +2963,9 @@ def done(self): task = TaskLike() loop = mock.Mock() - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) self._register_task(task) - self.assertEqual(asyncio.all_tasks(loop), {task}) + self.assertEqual(self.all_tasks(loop), {task}) self._unregister_task(task) def test__register_task_2(self): @@ -2764,9 +2979,9 @@ def done(self): task = TaskLike() loop = mock.Mock() - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) self._register_task(task) - self.assertEqual(asyncio.all_tasks(loop), {task}) + self.assertEqual(self.all_tasks(loop), {task}) self._unregister_task(task) def test__register_task_3(self): @@ -2780,9 +2995,9 @@ def done(self): task = TaskLike() loop = mock.Mock() - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) self._register_task(task) - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) self._unregister_task(task) def test__enter_task(self): @@ -2833,13 +3048,13 @@ def test__unregister_task(self): task.get_loop = lambda: loop self._register_task(task) self._unregister_task(task) - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) def test__unregister_task_not_registered(self): task = mock.Mock() loop = mock.Mock() self._unregister_task(task) - self.assertEqual(asyncio.all_tasks(loop), set()) + self.assertEqual(self.all_tasks(loop), set()) class PyIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests): @@ -2847,6 +3062,7 @@ class PyIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests): _unregister_task = staticmethod(tasks._py_unregister_task) _enter_task = staticmethod(tasks._py_enter_task) _leave_task = staticmethod(tasks._py_leave_task) + all_tasks = staticmethod(tasks._py_all_tasks) @unittest.skipUnless(hasattr(tasks, '_c_register_task'), @@ -2857,6 +3073,7 @@ class CIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests): _unregister_task = staticmethod(tasks._c_unregister_task) _enter_task = staticmethod(tasks._c_enter_task) _leave_task = staticmethod(tasks._c_leave_task) + all_tasks = staticmethod(tasks._c_all_tasks) else: _register_task = _unregister_task = _enter_task = _leave_task = None @@ -2914,7 +3131,7 @@ def new_task(self, coro): class GenericTaskTests(test_utils.TestCase): def test_future_subclass(self): - self.assertTrue(issubclass(asyncio.Task, asyncio.Future)) + self.assertIsSubclass(asyncio.Task, asyncio.Future) @support.cpython_only def test_asyncio_module_compiled(self): @@ -2923,14 +3140,14 @@ def test_asyncio_module_compiled(self): # fail on systems where C modules were successfully compiled # (hence the test for _functools etc), but _asyncio somehow didn't. try: - import _functools - import _json - import _pickle + import _functools # noqa: F401 + import _json # noqa: F401 + import _pickle # noqa: F401 except ImportError: self.skipTest('C modules are not available') else: try: - import _asyncio + import _asyncio # noqa: F401 except ImportError: self.fail('_asyncio module is missing') @@ -3061,8 +3278,8 @@ async def gather(): def test_constructor_empty_sequence_use_global_loop(self): # Deprecated in 3.10, undeprecated in 3.12 - asyncio.set_event_loop(self.one_loop) - self.addCleanup(asyncio.set_event_loop, None) + asyncio._set_event_loop(self.one_loop) + self.addCleanup(asyncio._set_event_loop, None) fut = asyncio.gather() self.assertIsInstance(fut, asyncio.Future) self.assertIs(fut._loop, self.one_loop) @@ -3169,8 +3386,8 @@ def test_constructor_use_global_loop(self): # Deprecated in 3.10, undeprecated in 3.12 async def coro(): return 'abc' - asyncio.set_event_loop(self.other_loop) - self.addCleanup(asyncio.set_event_loop, None) + asyncio._set_event_loop(self.other_loop) + self.addCleanup(asyncio._set_event_loop, None) gen1 = coro() gen2 = coro() fut = asyncio.gather(gen1, gen2) diff --git a/Lib/test/test_asyncio/test_threads.py b/Lib/test/test_asyncio/test_threads.py index 1138a93e0f78ec..c98c9a9b395ff9 100644 --- a/Lib/test/test_asyncio/test_threads.py +++ b/Lib/test/test_asyncio/test_threads.py @@ -8,7 +8,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class ToThreadTests(unittest.IsolatedAsyncioTestCase): @@ -30,7 +30,9 @@ async def test_to_thread_once(self): func.assert_called_once() async def test_to_thread_concurrent(self): - func = mock.Mock() + calls = [] + def func(): + calls.append(1) futs = [] for _ in range(10): @@ -38,7 +40,7 @@ async def test_to_thread_concurrent(self): futs.append(fut) await asyncio.gather(*futs) - self.assertEqual(func.call_count, 10) + self.assertEqual(sum(calls), 10) async def test_to_thread_args_kwargs(self): # Unlike run_in_executor(), to_thread() should directly accept kwargs. diff --git a/Lib/test/test_asyncio/test_timeouts.py b/Lib/test/test_asyncio/test_timeouts.py index 1f7f9ee696a525..3ba84d63b2ca5f 100644 --- a/Lib/test/test_asyncio/test_timeouts.py +++ b/Lib/test/test_asyncio/test_timeouts.py @@ -9,7 +9,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class TimeoutTests(unittest.IsolatedAsyncioTestCase): @@ -220,7 +220,7 @@ async def test_nested_timeouts_loop_busy(self): # Pretend the loop is busy for a while. time.sleep(0.1) await asyncio.sleep(1) - # TimeoutError was cought by (2) + # TimeoutError was caught by (2) await asyncio.sleep(10) # This sleep should be interrupted by (1) t1 = loop.time() self.assertTrue(t0 <= t1 <= t0 + 1) diff --git a/Lib/test/test_asyncio/test_transports.py b/Lib/test/test_asyncio/test_transports.py index bbdb218efaa3b6..af10d3dc2a80df 100644 --- a/Lib/test/test_asyncio/test_transports.py +++ b/Lib/test/test_asyncio/test_transports.py @@ -10,7 +10,7 @@ def tearDownModule(): # not needed for the test file but added for uniformness with all other # asyncio test files for the sake of unified cleanup - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class TransportTests(unittest.TestCase): diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 59ef9f5f58cabc..ebb4cc0f7b64fd 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -6,7 +6,6 @@ import multiprocessing from multiprocessing.util import _cleanup_tests as multiprocessing_cleanup_tests import os -import pathlib import signal import socket import stat @@ -34,7 +33,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) MOCK_ANY = mock.ANY @@ -304,20 +303,20 @@ def test_create_unix_server_existing_path_sock(self): self.loop.run_until_complete(srv.wait_closed()) @socket_helper.skip_unless_bind_unix_socket - def test_create_unix_server_pathlib(self): + def test_create_unix_server_pathlike(self): with test_utils.unix_socket_path() as path: - path = pathlib.Path(path) + path = os_helper.FakePath(path) srv_coro = self.loop.create_unix_server(lambda: None, path) srv = self.loop.run_until_complete(srv_coro) srv.close() self.loop.run_until_complete(srv.wait_closed()) - def test_create_unix_connection_pathlib(self): + def test_create_unix_connection_pathlike(self): with test_utils.unix_socket_path() as path: - path = pathlib.Path(path) + path = os_helper.FakePath(path) coro = self.loop.create_unix_connection(lambda: None, path) with self.assertRaises(FileNotFoundError): - # If pathlib.Path wasn't supported, the exception would be + # If path-like object weren't supported, the exception would be # different. self.loop.run_until_complete(coro) @@ -1113,706 +1112,15 @@ def test_write_eof_pending(self): self.assertFalse(self.protocol.connection_lost.called) -class AbstractChildWatcherTests(unittest.TestCase): - - def test_warns_on_subclassing(self): - with self.assertWarns(DeprecationWarning): - class MyWatcher(asyncio.AbstractChildWatcher): - pass - - def test_not_implemented(self): - f = mock.Mock() - watcher = asyncio.AbstractChildWatcher() - self.assertRaises( - NotImplementedError, watcher.add_child_handler, f, f) - self.assertRaises( - NotImplementedError, watcher.remove_child_handler, f) - self.assertRaises( - NotImplementedError, watcher.attach_loop, f) - self.assertRaises( - NotImplementedError, watcher.close) - self.assertRaises( - NotImplementedError, watcher.is_active) - self.assertRaises( - NotImplementedError, watcher.__enter__) - self.assertRaises( - NotImplementedError, watcher.__exit__, f, f, f) - - -class BaseChildWatcherTests(unittest.TestCase): - - def test_not_implemented(self): - f = mock.Mock() - watcher = unix_events.BaseChildWatcher() - self.assertRaises( - NotImplementedError, watcher._do_waitpid, f) - - -class ChildWatcherTestsMixin: - - ignore_warnings = mock.patch.object(log.logger, "warning") - - def setUp(self): - super().setUp() - self.loop = self.new_test_loop() - self.running = False - self.zombies = {} - - with mock.patch.object( - self.loop, "add_signal_handler") as self.m_add_signal_handler: - self.watcher = self.create_watcher() - self.watcher.attach_loop(self.loop) - - def waitpid(self, pid, flags): - if isinstance(self.watcher, asyncio.SafeChildWatcher) or pid != -1: - self.assertGreater(pid, 0) - try: - if pid < 0: - return self.zombies.popitem() - else: - return pid, self.zombies.pop(pid) - except KeyError: - pass - if self.running: - return 0, 0 - else: - raise ChildProcessError() - - def add_zombie(self, pid, status): - self.zombies[pid] = status - - def waitstatus_to_exitcode(self, status): - if status > 32768: - return status - 32768 - elif 32700 < status < 32768: - return status - 32768 - else: - return status - - def test_create_watcher(self): - self.m_add_signal_handler.assert_called_once_with( - signal.SIGCHLD, self.watcher._sig_chld) - - def waitpid_mocks(func): - def wrapped_func(self): - def patch(target, wrapper): - return mock.patch(target, wraps=wrapper, - new_callable=mock.Mock) - - with patch('asyncio.unix_events.waitstatus_to_exitcode', self.waitstatus_to_exitcode), \ - patch('os.waitpid', self.waitpid) as m_waitpid: - func(self, m_waitpid) - return wrapped_func - - @waitpid_mocks - def test_sigchld(self, m_waitpid): - # register a child - callback = mock.Mock() - - with self.watcher: - self.running = True - self.watcher.add_child_handler(42, callback, 9, 10, 14) - - self.assertFalse(callback.called) - - # child is running - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - # child terminates (returncode 12) - self.running = False - self.add_zombie(42, EXITCODE(12)) - self.watcher._sig_chld() - - callback.assert_called_once_with(42, 12, 9, 10, 14) - - callback.reset_mock() - - # ensure that the child is effectively reaped - self.add_zombie(42, EXITCODE(13)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - # sigchld called again - self.zombies.clear() - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - @waitpid_mocks - def test_sigchld_two_children(self, m_waitpid): - callback1 = mock.Mock() - callback2 = mock.Mock() - - # register child 1 - with self.watcher: - self.running = True - self.watcher.add_child_handler(43, callback1, 7, 8) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # register child 2 - with self.watcher: - self.watcher.add_child_handler(44, callback2, 147, 18) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # children are running - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # child 1 terminates (signal 3) - self.add_zombie(43, SIGNAL(3)) - self.watcher._sig_chld() - - callback1.assert_called_once_with(43, -3, 7, 8) - self.assertFalse(callback2.called) - - callback1.reset_mock() - - # child 2 still running - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # child 2 terminates (code 108) - self.add_zombie(44, EXITCODE(108)) - self.running = False - self.watcher._sig_chld() - - callback2.assert_called_once_with(44, 108, 147, 18) - self.assertFalse(callback1.called) - - callback2.reset_mock() - - # ensure that the children are effectively reaped - self.add_zombie(43, EXITCODE(14)) - self.add_zombie(44, EXITCODE(15)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # sigchld called again - self.zombies.clear() - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - @waitpid_mocks - def test_sigchld_two_children_terminating_together(self, m_waitpid): - callback1 = mock.Mock() - callback2 = mock.Mock() - - # register child 1 - with self.watcher: - self.running = True - self.watcher.add_child_handler(45, callback1, 17, 8) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # register child 2 - with self.watcher: - self.watcher.add_child_handler(46, callback2, 1147, 18) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # children are running - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # child 1 terminates (code 78) - # child 2 terminates (signal 5) - self.add_zombie(45, EXITCODE(78)) - self.add_zombie(46, SIGNAL(5)) - self.running = False - self.watcher._sig_chld() - - callback1.assert_called_once_with(45, 78, 17, 8) - callback2.assert_called_once_with(46, -5, 1147, 18) - - callback1.reset_mock() - callback2.reset_mock() - - # ensure that the children are effectively reaped - self.add_zombie(45, EXITCODE(14)) - self.add_zombie(46, EXITCODE(15)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - @waitpid_mocks - def test_sigchld_race_condition(self, m_waitpid): - # register a child - callback = mock.Mock() - - with self.watcher: - # child terminates before being registered - self.add_zombie(50, EXITCODE(4)) - self.watcher._sig_chld() - - self.watcher.add_child_handler(50, callback, 1, 12) - - callback.assert_called_once_with(50, 4, 1, 12) - callback.reset_mock() - - # ensure that the child is effectively reaped - self.add_zombie(50, SIGNAL(1)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - @waitpid_mocks - def test_sigchld_replace_handler(self, m_waitpid): - callback1 = mock.Mock() - callback2 = mock.Mock() - - # register a child - with self.watcher: - self.running = True - self.watcher.add_child_handler(51, callback1, 19) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # register the same child again - with self.watcher: - self.watcher.add_child_handler(51, callback2, 21) - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - # child terminates (signal 8) - self.running = False - self.add_zombie(51, SIGNAL(8)) - self.watcher._sig_chld() - - callback2.assert_called_once_with(51, -8, 21) - self.assertFalse(callback1.called) - - callback2.reset_mock() - - # ensure that the child is effectively reaped - self.add_zombie(51, EXITCODE(13)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - - @waitpid_mocks - def test_sigchld_remove_handler(self, m_waitpid): - callback = mock.Mock() - - # register a child - with self.watcher: - self.running = True - self.watcher.add_child_handler(52, callback, 1984) - - self.assertFalse(callback.called) - - # unregister the child - self.watcher.remove_child_handler(52) - - self.assertFalse(callback.called) - - # child terminates (code 99) - self.running = False - self.add_zombie(52, EXITCODE(99)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - @waitpid_mocks - def test_sigchld_unknown_status(self, m_waitpid): - callback = mock.Mock() - - # register a child - with self.watcher: - self.running = True - self.watcher.add_child_handler(53, callback, -19) - - self.assertFalse(callback.called) - - # terminate with unknown status - self.zombies[53] = 1178 - self.running = False - self.watcher._sig_chld() - - callback.assert_called_once_with(53, 1178, -19) - - callback.reset_mock() - - # ensure that the child is effectively reaped - self.add_zombie(53, EXITCODE(101)) - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback.called) - - @waitpid_mocks - def test_remove_child_handler(self, m_waitpid): - callback1 = mock.Mock() - callback2 = mock.Mock() - callback3 = mock.Mock() - - # register children - with self.watcher: - self.running = True - self.watcher.add_child_handler(54, callback1, 1) - self.watcher.add_child_handler(55, callback2, 2) - self.watcher.add_child_handler(56, callback3, 3) - - # remove child handler 1 - self.assertTrue(self.watcher.remove_child_handler(54)) - - # remove child handler 2 multiple times - self.assertTrue(self.watcher.remove_child_handler(55)) - self.assertFalse(self.watcher.remove_child_handler(55)) - self.assertFalse(self.watcher.remove_child_handler(55)) - - # all children terminate - self.add_zombie(54, EXITCODE(0)) - self.add_zombie(55, EXITCODE(1)) - self.add_zombie(56, EXITCODE(2)) - self.running = False - with self.ignore_warnings: - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - callback3.assert_called_once_with(56, 2, 3) - - @waitpid_mocks - def test_sigchld_unhandled_exception(self, m_waitpid): - callback = mock.Mock() - - # register a child - with self.watcher: - self.running = True - self.watcher.add_child_handler(57, callback) - - # raise an exception - m_waitpid.side_effect = ValueError - - with mock.patch.object(log.logger, - 'error') as m_error: - - self.assertEqual(self.watcher._sig_chld(), None) - self.assertTrue(m_error.called) - - @waitpid_mocks - def test_sigchld_child_reaped_elsewhere(self, m_waitpid): - # register a child - callback = mock.Mock() - - with self.watcher: - self.running = True - self.watcher.add_child_handler(58, callback) - - self.assertFalse(callback.called) - - # child terminates - self.running = False - self.add_zombie(58, EXITCODE(4)) - - # waitpid is called elsewhere - os.waitpid(58, os.WNOHANG) - - m_waitpid.reset_mock() - - # sigchld - with self.ignore_warnings: - self.watcher._sig_chld() - - if isinstance(self.watcher, asyncio.FastChildWatcher): - # here the FastChildWatcher enters a deadlock - # (there is no way to prevent it) - self.assertFalse(callback.called) - else: - callback.assert_called_once_with(58, 255) - - @waitpid_mocks - def test_sigchld_unknown_pid_during_registration(self, m_waitpid): - # register two children - callback1 = mock.Mock() - callback2 = mock.Mock() - - with self.ignore_warnings, self.watcher: - self.running = True - # child 1 terminates - self.add_zombie(591, EXITCODE(7)) - # an unknown child terminates - self.add_zombie(593, EXITCODE(17)) - - self.watcher._sig_chld() - - self.watcher.add_child_handler(591, callback1) - self.watcher.add_child_handler(592, callback2) - - callback1.assert_called_once_with(591, 7) - self.assertFalse(callback2.called) - - @waitpid_mocks - def test_set_loop(self, m_waitpid): - # register a child - callback = mock.Mock() - - with self.watcher: - self.running = True - self.watcher.add_child_handler(60, callback) - - # attach a new loop - old_loop = self.loop - self.loop = self.new_test_loop() - patch = mock.patch.object - - with patch(old_loop, "remove_signal_handler") as m_old_remove, \ - patch(self.loop, "add_signal_handler") as m_new_add: - - self.watcher.attach_loop(self.loop) - - m_old_remove.assert_called_once_with( - signal.SIGCHLD) - m_new_add.assert_called_once_with( - signal.SIGCHLD, self.watcher._sig_chld) - - # child terminates - self.running = False - self.add_zombie(60, EXITCODE(9)) - self.watcher._sig_chld() - - callback.assert_called_once_with(60, 9) - - @waitpid_mocks - def test_set_loop_race_condition(self, m_waitpid): - # register 3 children - callback1 = mock.Mock() - callback2 = mock.Mock() - callback3 = mock.Mock() - - with self.watcher: - self.running = True - self.watcher.add_child_handler(61, callback1) - self.watcher.add_child_handler(62, callback2) - self.watcher.add_child_handler(622, callback3) - - # detach the loop - old_loop = self.loop - self.loop = None - - with mock.patch.object( - old_loop, "remove_signal_handler") as m_remove_signal_handler: - - with self.assertWarnsRegex( - RuntimeWarning, 'A loop is being detached'): - self.watcher.attach_loop(None) - - m_remove_signal_handler.assert_called_once_with( - signal.SIGCHLD) - - # child 1 & 2 terminate - self.add_zombie(61, EXITCODE(11)) - self.add_zombie(62, SIGNAL(5)) - - # SIGCHLD was not caught - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - self.assertFalse(callback3.called) - - # attach a new loop - self.loop = self.new_test_loop() - - with mock.patch.object( - self.loop, "add_signal_handler") as m_add_signal_handler: - - self.watcher.attach_loop(self.loop) - - m_add_signal_handler.assert_called_once_with( - signal.SIGCHLD, self.watcher._sig_chld) - callback1.assert_called_once_with(61, 11) # race condition! - callback2.assert_called_once_with(62, -5) # race condition! - self.assertFalse(callback3.called) - - callback1.reset_mock() - callback2.reset_mock() - - # child 3 terminates - self.running = False - self.add_zombie(622, EXITCODE(19)) - self.watcher._sig_chld() - - self.assertFalse(callback1.called) - self.assertFalse(callback2.called) - callback3.assert_called_once_with(622, 19) - - @waitpid_mocks - def test_close(self, m_waitpid): - # register two children - callback1 = mock.Mock() - - with self.watcher: - self.running = True - # child 1 terminates - self.add_zombie(63, EXITCODE(9)) - # other child terminates - self.add_zombie(65, EXITCODE(18)) - self.watcher._sig_chld() - - self.watcher.add_child_handler(63, callback1) - self.watcher.add_child_handler(64, callback1) - - self.assertEqual(len(self.watcher._callbacks), 1) - if isinstance(self.watcher, asyncio.FastChildWatcher): - self.assertEqual(len(self.watcher._zombies), 1) - - with mock.patch.object( - self.loop, - "remove_signal_handler") as m_remove_signal_handler: - - self.watcher.close() - - m_remove_signal_handler.assert_called_once_with( - signal.SIGCHLD) - self.assertFalse(self.watcher._callbacks) - if isinstance(self.watcher, asyncio.FastChildWatcher): - self.assertFalse(self.watcher._zombies) - - -class SafeChildWatcherTests (ChildWatcherTestsMixin, test_utils.TestCase): - def create_watcher(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return asyncio.SafeChildWatcher() - - -class FastChildWatcherTests (ChildWatcherTestsMixin, test_utils.TestCase): - def create_watcher(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return asyncio.FastChildWatcher() - - -class PolicyTests(unittest.TestCase): - - def create_policy(self): - return asyncio.DefaultEventLoopPolicy() - - @mock.patch('asyncio.unix_events.can_use_pidfd') - def test_get_default_child_watcher(self, m_can_use_pidfd): - m_can_use_pidfd.return_value = False - policy = self.create_policy() - self.assertIsNone(policy._watcher) - with self.assertWarns(DeprecationWarning): - watcher = policy.get_child_watcher() - self.assertIsInstance(watcher, asyncio.ThreadedChildWatcher) - - self.assertIs(policy._watcher, watcher) - with self.assertWarns(DeprecationWarning): - self.assertIs(watcher, policy.get_child_watcher()) - - m_can_use_pidfd.return_value = True - policy = self.create_policy() - self.assertIsNone(policy._watcher) - with self.assertWarns(DeprecationWarning): - watcher = policy.get_child_watcher() - self.assertIsInstance(watcher, asyncio.PidfdChildWatcher) - - self.assertIs(policy._watcher, watcher) - with self.assertWarns(DeprecationWarning): - self.assertIs(watcher, policy.get_child_watcher()) - - def test_get_child_watcher_after_set(self): - policy = self.create_policy() - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - watcher = asyncio.FastChildWatcher() - policy.set_child_watcher(watcher) - - self.assertIs(policy._watcher, watcher) - with self.assertWarns(DeprecationWarning): - self.assertIs(watcher, policy.get_child_watcher()) - - def test_get_child_watcher_thread(self): - - def f(): - policy.set_event_loop(policy.new_event_loop()) - - self.assertIsInstance(policy.get_event_loop(), - asyncio.AbstractEventLoop) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - watcher = policy.get_child_watcher() - - self.assertIsInstance(watcher, asyncio.SafeChildWatcher) - self.assertIsNone(watcher._loop) - - policy.get_event_loop().close() - - policy = self.create_policy() - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - policy.set_child_watcher(asyncio.SafeChildWatcher()) - - th = threading.Thread(target=f) - th.start() - th.join() - - def test_child_watcher_replace_mainloop_existing(self): - policy = self.create_policy() - loop = policy.new_event_loop() - policy.set_event_loop(loop) - - # Explicitly setup SafeChildWatcher, - # default ThreadedChildWatcher has no _loop property - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - watcher = asyncio.SafeChildWatcher() - policy.set_child_watcher(watcher) - watcher.attach_loop(loop) - - self.assertIs(watcher._loop, loop) - - new_loop = policy.new_event_loop() - policy.set_event_loop(new_loop) - - self.assertIs(watcher._loop, new_loop) - - policy.set_event_loop(None) - - self.assertIs(watcher._loop, None) - - loop.close() - new_loop.close() - - class TestFunctional(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() - asyncio.set_event_loop(self.loop) + asyncio._set_event_loop(self.loop) def tearDown(self): self.loop.close() - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) def test_add_reader_invalid_argument(self): def assert_raises(): @@ -1887,8 +1195,7 @@ async def test_fork_not_share_event_loop(self): if pid == 0: # child try: - with self.assertWarns(DeprecationWarning): - loop = asyncio.get_event_loop_policy().get_event_loop() + loop = asyncio.get_event_loop() os.write(w, b'LOOP:' + str(id(loop)).encode()) except RuntimeError: os.write(w, b'NO LOOP') @@ -1899,11 +1206,11 @@ async def test_fork_not_share_event_loop(self): else: # parent result = os.read(r, 100) - self.assertEqual(result[:5], b'LOOP:', result) - self.assertNotEqual(int(result[5:]), id(loop)) + self.assertEqual(result, b'NO LOOP') wait_process(pid, exitcode=0) @hashlib_helper.requires_hashdigest('md5') + @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) def test_fork_signal_handling(self): self.addCleanup(multiprocessing_cleanup_tests) @@ -1950,6 +1257,7 @@ async def func(): self.assertTrue(child_handled.is_set()) @hashlib_helper.requires_hashdigest('md5') + @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) def test_fork_asyncio_run(self): self.addCleanup(multiprocessing_cleanup_tests) @@ -1969,6 +1277,7 @@ async def child_main(): self.assertEqual(result.value, 42) @hashlib_helper.requires_hashdigest('md5') + @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) def test_fork_asyncio_subprocess(self): self.addCleanup(multiprocessing_cleanup_tests) diff --git a/Lib/test/test_asyncio/test_waitfor.py b/Lib/test/test_asyncio/test_waitfor.py index d52f32534a0cfe..d083f6b4d2a535 100644 --- a/Lib/test/test_asyncio/test_waitfor.py +++ b/Lib/test/test_asyncio/test_waitfor.py @@ -5,7 +5,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) # The following value can be used as a very small timeout: @@ -249,8 +249,8 @@ async def test_cancel_wait_for(self): await self._test_cancel_wait_for(60.0) async def test_wait_for_cancel_suppressed(self): - # GH-86296: Supressing CancelledError is discouraged - # but if a task subpresses CancelledError and returns a value, + # GH-86296: Suppressing CancelledError is discouraged + # but if a task suppresses CancelledError and returns a value, # `wait_for` should return the value instead of raising CancelledError. # This is the same behavior as `asyncio.timeout`. diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py index 6e6c90a247b291..69e9905205eee0 100644 --- a/Lib/test/test_asyncio/test_windows_events.py +++ b/Lib/test/test_asyncio/test_windows_events.py @@ -19,7 +19,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class UpperProto(asyncio.Protocol): @@ -36,7 +36,23 @@ def data_received(self, data): self.trans.close() -class ProactorLoopCtrlC(test_utils.TestCase): +class WindowsEventsTestCase(test_utils.TestCase): + def _unraisablehook(self, unraisable): + # Storing unraisable.object can resurrect an object which is being + # finalized. Storing unraisable.exc_value creates a reference cycle. + self._unraisable = unraisable + print(unraisable) + + def setUp(self): + self._prev_unraisablehook = sys.unraisablehook + self._unraisable = None + sys.unraisablehook = self._unraisablehook + + def tearDown(self): + sys.unraisablehook = self._prev_unraisablehook + self.assertIsNone(self._unraisable) + +class ProactorLoopCtrlC(WindowsEventsTestCase): def test_ctrl_c(self): @@ -58,7 +74,7 @@ def SIGINT_after_delay(): thread.join() -class ProactorMultithreading(test_utils.TestCase): +class ProactorMultithreading(WindowsEventsTestCase): def test_run_from_nonmain_thread(self): finished = False @@ -79,7 +95,7 @@ def func(): self.assertTrue(finished) -class ProactorTests(test_utils.TestCase): +class ProactorTests(WindowsEventsTestCase): def setUp(self): super().setUp() @@ -283,22 +299,47 @@ async def probe(): return "done" - -class WinPolicyTests(test_utils.TestCase): + def test_loop_restart(self): + # We're fishing for the "RuntimeError: <_overlapped.Overlapped object at XXX> + # still has pending operation at deallocation, the process may crash" error + stop = threading.Event() + def threadMain(): + while not stop.is_set(): + self.loop.call_soon_threadsafe(lambda: None) + time.sleep(0.01) + thr = threading.Thread(target=threadMain) + + # In 10 60-second runs of this test prior to the fix: + # time in seconds until failure: (none), 15.0, 6.4, (none), 7.6, 8.3, 1.7, 22.2, 23.5, 8.3 + # 10 seconds had a 50% failure rate but longer would be more costly + end_time = time.time() + 10 # Run for 10 seconds + self.loop.call_soon(thr.start) + while not self._unraisable: # Stop if we got an unraisable exc + self.loop.stop() + self.loop.run_forever() + if time.time() >= end_time: + break + + stop.set() + thr.join() + + +class WinPolicyTests(WindowsEventsTestCase): def test_selector_win_policy(self): async def main(): - self.assertIsInstance( - asyncio.get_running_loop(), - asyncio.SelectorEventLoop) + self.assertIsInstance(asyncio.get_running_loop(), asyncio.SelectorEventLoop) - old_policy = asyncio.get_event_loop_policy() + old_policy = asyncio._get_event_loop_policy() try: - asyncio.set_event_loop_policy( - asyncio.WindowsSelectorEventLoopPolicy()) + with self.assertWarnsRegex( + DeprecationWarning, + "'asyncio.WindowsSelectorEventLoopPolicy' is deprecated", + ): + asyncio._set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncio.run(main()) finally: - asyncio.set_event_loop_policy(old_policy) + asyncio._set_event_loop_policy(old_policy) def test_proactor_win_policy(self): async def main(): @@ -306,13 +347,16 @@ async def main(): asyncio.get_running_loop(), asyncio.ProactorEventLoop) - old_policy = asyncio.get_event_loop_policy() + old_policy = asyncio._get_event_loop_policy() try: - asyncio.set_event_loop_policy( - asyncio.WindowsProactorEventLoopPolicy()) + with self.assertWarnsRegex( + DeprecationWarning, + "'asyncio.WindowsProactorEventLoopPolicy' is deprecated", + ): + asyncio._set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) asyncio.run(main()) finally: - asyncio.set_event_loop_policy(old_policy) + asyncio._set_event_loop_policy(old_policy) if __name__ == '__main__': diff --git a/Lib/test/test_asyncio/test_windows_utils.py b/Lib/test/test_asyncio/test_windows_utils.py index eafa5be3829682..a6b207567c4f00 100644 --- a/Lib/test/test_asyncio/test_windows_utils.py +++ b/Lib/test/test_asyncio/test_windows_utils.py @@ -16,7 +16,7 @@ def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class PipeTests(unittest.TestCase): @@ -121,8 +121,8 @@ def test_popen(self): self.assertGreater(len(out), 0) self.assertGreater(len(err), 0) # allow for partial reads... - self.assertTrue(msg.upper().rstrip().startswith(out)) - self.assertTrue(b"stderr".startswith(err)) + self.assertStartsWith(msg.upper().rstrip(), out) + self.assertStartsWith(b"stderr", err) # The context manager calls wait() and closes resources with p: diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index 44943e1fa7bc4e..35ce13896da08f 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -15,6 +15,7 @@ import unittest import weakref import warnings +from ast import literal_eval from unittest import mock from http.server import HTTPServer @@ -56,24 +57,8 @@ def data_file(*filename): ONLYKEY = data_file('certdata', 'ssl_key.pem') SIGNED_CERTFILE = data_file('certdata', 'keycert3.pem') SIGNING_CA = data_file('certdata', 'pycacert.pem') -PEERCERT = { - 'OCSP': ('http://testca.pythontest.net/testca/ocsp/',), - 'caIssuers': ('http://testca.pythontest.net/testca/pycacert.cer',), - 'crlDistributionPoints': ('http://testca.pythontest.net/testca/revocation.crl',), - 'issuer': ((('countryName', 'XY'),), - (('organizationName', 'Python Software Foundation CA'),), - (('commonName', 'our-ca-server'),)), - 'notAfter': 'Oct 28 14:23:16 2037 GMT', - 'notBefore': 'Aug 29 14:23:16 2018 GMT', - 'serialNumber': 'CB2D80995A69525C', - 'subject': ((('countryName', 'XY'),), - (('localityName', 'Castle Anthrax'),), - (('organizationName', 'Python Software Foundation'),), - (('commonName', 'localhost'),)), - 'subjectAltName': (('DNS', 'localhost'),), - 'version': 3 -} - +with open(data_file('certdata', 'keycert3.pem.reference')) as file: + PEERCERT = literal_eval(file.read()) def simple_server_sslcontext(): server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) @@ -301,12 +286,17 @@ def run_udp_echo_server(*, host='127.0.0.1', port=0): family, type, proto, _, sockaddr = addr_info[0] sock = socket.socket(family, type, proto) sock.bind((host, port)) + sockname = sock.getsockname() thread = threading.Thread(target=lambda: echo_datagrams(sock)) thread.start() try: - yield sock.getsockname() + yield sockname finally: - sock.sendto(b'STOP', sock.getsockname()) + # gh-122187: use a separate socket to send the stop message to avoid + # TSan reported race on the same socket. + sock2 = socket.socket(family, type, proto) + sock2.sendto(b'STOP', sockname) + sock2.close() thread.join() @@ -547,30 +537,11 @@ def close_loop(loop): loop._default_executor.shutdown(wait=True) loop.close() - policy = support.maybe_get_event_loop_policy() - if policy is not None: - try: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = policy.get_child_watcher() - except NotImplementedError: - # watcher is not implemented by EventLoopPolicy, e.g. Windows - pass - else: - if isinstance(watcher, asyncio.ThreadedChildWatcher): - # Wait for subprocess to finish, but not forever - for thread in list(watcher._threads.values()): - thread.join(timeout=support.SHORT_TIMEOUT) - if thread.is_alive(): - raise RuntimeError(f"thread {thread} still alive: " - "subprocess still running") - - def set_event_loop(self, loop, *, cleanup=True): if loop is None: raise AssertionError('loop is None') # ensure that the event loop is passed explicitly in asyncio - events.set_event_loop(None) + events._set_event_loop(None) if cleanup: self.addCleanup(self.close_loop, loop) @@ -583,7 +554,7 @@ def setUp(self): self._thread_cleanup = threading_helper.threading_setup() def tearDown(self): - events.set_event_loop(None) + events._set_event_loop(None) # Detect CPython bug #23353: ensure that yield/yield-from is not used # in an except block of a generator diff --git a/Lib/test/test_atexit.py b/Lib/test/test_atexit.py index 913b7556be83af..eb01da6e88a8bc 100644 --- a/Lib/test/test_atexit.py +++ b/Lib/test/test_atexit.py @@ -4,7 +4,7 @@ import unittest from test import support from test.support import script_helper - +from test.support import threading_helper class GeneralTest(unittest.TestCase): def test_general(self): @@ -46,6 +46,39 @@ def test_atexit_instances(self): self.assertEqual(res.out.decode().splitlines(), ["atexit2", "atexit1"]) self.assertFalse(res.err) + @threading_helper.requires_working_threading() + @support.requires_resource("cpu") + @unittest.skipUnless(support.Py_GIL_DISABLED, "only meaningful without the GIL") + def test_atexit_thread_safety(self): + # GH-126907: atexit was not thread safe on the free-threaded build + source = """ + from threading import Thread + + def dummy(): + pass + + + def thready(): + for _ in range(100): + atexit.register(dummy) + atexit._clear() + atexit.register(dummy) + atexit.unregister(dummy) + atexit._run_exitfuncs() + + + threads = [Thread(target=thready) for _ in range(10)] + for thread in threads: + thread.start() + + for thread in threads: + thread.join() + """ + + # atexit._clear() has some evil side effects, and we don't + # want them to affect the rest of the tests. + script_helper.assert_python_ok("-c", textwrap.dedent(source)) + @support.cpython_only class SubinterpreterTest(unittest.TestCase): diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index cd0a4e2264865d..ddd9f951143df7 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -89,6 +89,7 @@ def test_excepthook(self): ) def test_unraisablehook(self): + import_helper.import_module("_testcapi") returncode, events, stderr = self.run_python("test_unraisablehook") if returncode: self.fail(stderr) @@ -139,6 +140,7 @@ def test_gc(self): ) + @support.requires_resource('network') def test_http(self): import_helper.import_module("http.client") returncode, events, stderr = self.run_python("test_http_client") @@ -209,7 +211,7 @@ def test_threading(self): expected = [ ("_thread.start_new_thread", "(, (), None)"), ("test.test_func", "()"), - ("_thread.start_joinable_thread", "(,)"), + ("_thread.start_joinable_thread", "(, 1, None)"), ("test.test_func", "()"), ] @@ -290,6 +292,27 @@ def test_sys_monitoring_register_callback(self): self.assertEqual(actual, expected) + def test_winapi_createnamedpipe(self): + winapi = import_helper.import_module("_winapi") + + pipe_name = r"\\.\pipe\LOCAL\test_winapi_createnamed_pipe" + returncode, events, stderr = self.run_python("test_winapi_createnamedpipe", pipe_name) + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + actual = [(ev[0], ev[2]) for ev in events] + expected = [("_winapi.CreateNamedPipe", f"({pipe_name!r}, 3, 8)")] + + self.assertEqual(actual, expected) + + def test_assert_unicode(self): + # See gh-126018 + returncode, _, stderr = self.run_python("test_assert_unicode") + if returncode: + self.fail(stderr) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_baseexception.py b/Lib/test/test_baseexception.py index 4c3cf0b964ae56..e599b02c17d9c0 100644 --- a/Lib/test/test_baseexception.py +++ b/Lib/test/test_baseexception.py @@ -78,6 +78,9 @@ def test_inheritance(self): last_depth = depth finally: inheritance_tree.close() + + # Underscore-prefixed (private) exceptions don't need to be documented + exc_set = set(e for e in exc_set if not e.startswith('_')) self.assertEqual(len(exc_set), 0, "%s not accounted for" % exc_set) interface_tests = ("length", "args", "str", "repr") @@ -129,7 +132,7 @@ class Value(str): d[HashThisKeyWillClearTheDict()] = Value() # refcount of Value() is 1 now - # Exception.__setstate__ should aquire a strong reference of key and + # Exception.__setstate__ should acquire a strong reference of key and # value in the dict. Otherwise, Value()'s refcount would go below # zero in the tp_hash call in PyObject_SetAttr(), and it would cause # crash in GC. diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py index 568c88e326c087..f15dae13eb384e 100644 --- a/Lib/test/test_bdb.py +++ b/Lib/test/test_bdb.py @@ -228,6 +228,10 @@ def user_exception(self, frame, exc_info): self.process_event('exception', frame) self.next_set_method() + def user_opcode(self, frame): + self.process_event('opcode', frame) + self.next_set_method() + def do_clear(self, arg): # The temporary breakpoints are deleted in user_line(). bp_list = [self.currentbp] @@ -366,7 +370,7 @@ def next_set_method(self): set_method = getattr(self, 'set_' + set_type) # The following set methods give back control to the tracer. - if set_type in ('step', 'continue', 'quit'): + if set_type in ('step', 'stepinstr', 'continue', 'quit'): set_method() return elif set_type in ('next', 'return'): @@ -610,6 +614,15 @@ def test_step_next_on_last_statement(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) + def test_stepinstr(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('stepinstr', ), + ('opcode', 2, 'tfunc_main'), ('next', ), + ('line', 3, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + def test_next(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -1033,8 +1046,9 @@ def main(): ('return', 1, ''), ('quit', ), ] import test_module_for_bdb + ns = {'test_module_for_bdb': test_module_for_bdb} with TracerRun(self) as tracer: - tracer.runeval('test_module_for_bdb.main()', globals(), locals()) + tracer.runeval('test_module_for_bdb.main()', ns, ns) class IssuesTestCase(BaseTestCase): """Test fixed bdb issues.""" @@ -1203,6 +1217,19 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) + def test_next_to_botframe(self): + # gh-125422 + # Check that next command won't go to the bottom frame. + code = """ + lno = 2 + """ + self.expect_set = [ + ('line', 2, ''), ('step', ), + ('return', 2, ''), ('next', ), + ] + with TracerRun(self) as tracer: + tracer.run(compile(textwrap.dedent(code), '', 'exec')) + class TestRegressions(unittest.TestCase): def test_format_stack_entry_no_lineno(self): diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index 82dea8a6d731ea..1f3b6746ce4a62 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -139,13 +139,21 @@ def assertLeadingPadding(data, non_strict_mode_expected_result: bytes): def assertDiscontinuousPadding(data, non_strict_mode_expected_result: bytes): _assertRegexTemplate(r'(?i)Discontinuous padding', data, non_strict_mode_expected_result) + def assertExcessPadding(data, non_strict_mode_expected_result: bytes): + _assertRegexTemplate(r'(?i)Excess padding', data, non_strict_mode_expected_result) + # Test excess data exceptions assertExcessData(b'ab==a', b'i') assertExcessData(b'ab===', b'i') + assertExcessData(b'ab====', b'i') assertExcessData(b'ab==:', b'i') assertExcessData(b'abc=a', b'i\xb7') assertExcessData(b'abc=:', b'i\xb7') assertExcessData(b'ab==\n', b'i') + assertExcessData(b'abc==', b'i\xb7') + assertExcessData(b'abc===', b'i\xb7') + assertExcessData(b'abc====', b'i\xb7') + assertExcessData(b'abc=====', b'i\xb7') # Test non-base64 data exceptions assertNonBase64Data(b'\nab==', b'i') @@ -157,8 +165,15 @@ def assertDiscontinuousPadding(data, non_strict_mode_expected_result: bytes): assertLeadingPadding(b'=', b'') assertLeadingPadding(b'==', b'') assertLeadingPadding(b'===', b'') + assertLeadingPadding(b'====', b'') + assertLeadingPadding(b'=====', b'') assertDiscontinuousPadding(b'ab=c=', b'i\xb7') assertDiscontinuousPadding(b'ab=ab==', b'i\xb6\x9b') + assertExcessPadding(b'abcd=', b'i\xb7\x1d') + assertExcessPadding(b'abcd==', b'i\xb7\x1d') + assertExcessPadding(b'abcd===', b'i\xb7\x1d') + assertExcessPadding(b'abcd====', b'i\xb7\x1d') + assertExcessPadding(b'abcd=====', b'i\xb7\x1d') def test_base64errors(self): diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 5b1b95b9c82064..61921e93e85e63 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -130,10 +130,10 @@ def native_type_range(fmt): for fmt in fmtdict['@']: fmtdict['@'][fmt] = native_type_range(fmt) -# Format codes suppported by the memoryview object +# Format codes supported by the memoryview object MEMORYVIEW = NATIVE.copy() -# Format codes suppported by array.array +# Format codes supported by array.array ARRAY = NATIVE.copy() for k in NATIVE: if not k in "bBhHiIlLfd": @@ -168,7 +168,7 @@ def randrange_fmt(mode, char, obj): if char == 'c': x = bytes([x]) if obj == 'numpy' and x == b'\x00': - # http://projects.scipy.org/numpy/ticket/1925 + # https://github.com/numpy/numpy/issues/2518 x = b'\x01' if char == '?': x = bool(x) @@ -1918,7 +1918,7 @@ def test_ndarray_random(self): if numpy_array: shape = t[3] if 0 in shape: - continue # http://projects.scipy.org/numpy/ticket/1910 + continue # https://github.com/numpy/numpy/issues/2503 z = numpy_array_from_structure(items, fmt, t) self.verify(x, obj=None, itemsize=z.itemsize, fmt=fmt, readonly=False, @@ -1950,7 +1950,7 @@ def test_ndarray_random_invalid(self): except Exception as e: numpy_err = e.__class__ - if 0: # http://projects.scipy.org/numpy/ticket/1910 + if 0: # https://github.com/numpy/numpy/issues/2503 self.assertTrue(numpy_err) def test_ndarray_random_slice_assign(self): @@ -1996,7 +1996,7 @@ def test_ndarray_random_slice_assign(self): if numpy_array: if 0 in lshape or 0 in rshape: - continue # http://projects.scipy.org/numpy/ticket/1910 + continue # https://github.com/numpy/numpy/issues/2503 zl = numpy_array_from_structure(litems, fmt, tl) zr = numpy_array_from_structure(ritems, fmt, tr) @@ -3910,6 +3910,8 @@ def test_memoryview_check_released(self): self.assertRaises(ValueError, memoryview, m) # memoryview.cast() self.assertRaises(ValueError, m.cast, 'c') + # memoryview.__iter__() + self.assertRaises(ValueError, m.__iter__) # getbuffer() self.assertRaises(ValueError, ndarray, m) # memoryview.tolist() @@ -4437,6 +4439,14 @@ def test_issue_7385(self): x = ndarray([1,2,3], shape=[3], flags=ND_GETBUF_FAIL) self.assertRaises(BufferError, memoryview, x) + def test_bytearray_release_buffer_read_flag(self): + # See https://github.com/python/cpython/issues/126980 + obj = bytearray(b'abc') + with self.assertRaises(SystemError): + obj.__buffer__(inspect.BufferFlags.READ) + with self.assertRaises(SystemError): + obj.__buffer__(inspect.BufferFlags.WRITE) + @support.cpython_only def test_pybuffer_size_from_format(self): # basic tests @@ -4444,6 +4454,21 @@ def test_pybuffer_size_from_format(self): self.assertEqual(_testcapi.PyBuffer_SizeFromFormat(format), struct.calcsize(format)) + @support.cpython_only + def test_flags_overflow(self): + # gh-126594: Check for integer overlow on large flags + try: + from _testcapi import INT_MIN, INT_MAX + except ImportError: + INT_MIN = -(2 ** 31) + INT_MAX = 2 ** 31 - 1 + + obj = b'abc' + for flags in (INT_MIN - 1, INT_MAX + 1): + with self.subTest(flags=flags): + with self.assertRaises(OverflowError): + obj.__buffer__(flags) + class TestPythonBufferProtocol(unittest.TestCase): def test_basic(self): diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 9a0bf524e3943f..913d007a126d72 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1,9 +1,9 @@ # Python test set -- built-in functions import ast -import asyncio import builtins import collections +import contextlib import decimal import fractions import gc @@ -16,6 +16,7 @@ import random import re import sys +import textwrap import traceback import types import typing @@ -29,9 +30,12 @@ from types import AsyncGeneratorType, FunctionType, CellType from operator import neg from test import support -from test.support import (cpython_only, swap_attr, maybe_get_event_loop_policy) +from test.support import cpython_only, swap_attr +from test.support import async_yield, run_yielding_async_fn +from test.support.import_helper import import_module from test.support.os_helper import (EnvironmentVarGuard, TESTFN, unlink) from test.support.script_helper import assert_python_ok +from test.support.testcase import ComplexesAreIdenticalMixin from test.support.warnings_helper import check_warnings from test.support import requires_IEEE_754 from unittest.mock import MagicMock, patch @@ -46,6 +50,8 @@ x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4) +# used as proof of globals being used +A_GLOBAL_VALUE = 123 class Squares: @@ -143,7 +149,10 @@ def filter_char(arg): def map_char(arg): return chr(ord(arg)+1) -class BuiltinTest(unittest.TestCase): +def pack(*args): + return args + +class BuiltinTest(ComplexesAreIdenticalMixin, unittest.TestCase): # Helper to check picklability def check_iter_pickle(self, it, seq, proto): itorg = it @@ -405,12 +414,8 @@ def test_compile_top_level_await_no_coro(self): msg=f"source={source} mode={mode}") - @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "socket.accept is broken" - ) def test_compile_top_level_await(self): - """Test whether code some top level await can be compiled. + """Test whether code with top level await can be compiled. Make sure it compiles only with the PyCF_ALLOW_TOP_LEVEL_AWAIT flag set, and make sure the generated code object has the CO_COROUTINE flag @@ -423,12 +428,25 @@ async def arange(n): for i in range(n): yield i + class Lock: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc_info): + pass + + async def sleep(delay, result=None): + assert delay == 0 + await async_yield(None) + return result + modes = ('single', 'exec') + optimizations = (-1, 0, 1, 2) code_samples = [ - '''a = await asyncio.sleep(0, result=1)''', + '''a = await sleep(0, result=1)''', '''async for i in arange(1): a = 1''', - '''async with asyncio.Lock() as l: + '''async with Lock() as l: a = 1''', '''a = [x async for x in arange(2)][1]''', '''a = 1 in {x async for x in arange(2)}''', @@ -436,36 +454,49 @@ async def arange(n): '''a = [x async for x in arange(2) async for x in arange(2)][1]''', '''a = [x async for x in (x async for x in arange(5))][1]''', '''a, = [1 for x in {x async for x in arange(1)}]''', - '''a = [await asyncio.sleep(0, x) async for x in arange(2)][1]''' + '''a = [await sleep(0, x) async for x in arange(2)][1]''', + # gh-121637: Make sure we correctly handle the case where the + # async code is optimized away + '''assert not await sleep(0); a = 1''', + '''assert [x async for x in arange(1)]; a = 1''', + '''assert {x async for x in arange(1)}; a = 1''', + '''assert {x: x async for x in arange(1)}; a = 1''', + ''' + if (a := 1) and __debug__: + async with Lock() as l: + pass + ''', + ''' + if (a := 1) and __debug__: + async for x in arange(2): + pass + ''', ] - policy = maybe_get_event_loop_policy() - try: - for mode, code_sample in product(modes, code_samples): + for mode, code_sample, optimize in product(modes, code_samples, optimizations): + with self.subTest(mode=mode, code_sample=code_sample, optimize=optimize): source = dedent(code_sample) with self.assertRaises( SyntaxError, msg=f"source={source} mode={mode}"): - compile(source, '?', mode) + compile(source, '?', mode, optimize=optimize) co = compile(source, - '?', - mode, - flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) + '?', + mode, + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, + optimize=optimize) self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE, - msg=f"source={source} mode={mode}") + msg=f"source={source} mode={mode}") # test we can create and advance a function type - globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange} - async_f = FunctionType(co, globals_) - asyncio.run(async_f()) + globals_ = {'Lock': Lock, 'a': 0, 'arange': arange, 'sleep': sleep} + run_yielding_async_fn(FunctionType(co, globals_)) self.assertEqual(globals_['a'], 1) # test we can await-eval, - globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange} - asyncio.run(eval(co, globals_)) + globals_ = {'Lock': Lock, 'a': 0, 'arange': arange, 'sleep': sleep} + run_yielding_async_fn(lambda: eval(co, globals_)) self.assertEqual(globals_['a'], 1) - finally: - asyncio.set_event_loop_policy(policy) def test_compile_top_level_await_invalid_cases(self): # helper function just to check we can run top=level async-for @@ -473,6 +504,13 @@ async def arange(n): for i in range(n): yield i + class Lock: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc_info): + pass + modes = ('single', 'exec') code_samples = [ '''def f(): await arange(10)\n''', @@ -483,27 +521,22 @@ async def arange(n): a = 1 ''', '''def f(): - async with asyncio.Lock() as l: + async with Lock() as l: a = 1 ''' ] - policy = maybe_get_event_loop_policy() - try: - for mode, code_sample in product(modes, code_samples): - source = dedent(code_sample) - with self.assertRaises( - SyntaxError, msg=f"source={source} mode={mode}"): - compile(source, '?', mode) - - with self.assertRaises( - SyntaxError, msg=f"source={source} mode={mode}"): - co = compile(source, - '?', - mode, - flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) - finally: - asyncio.set_event_loop_policy(policy) + for mode, code_sample in product(modes, code_samples): + source = dedent(code_sample) + with self.assertRaises( + SyntaxError, msg=f"source={source} mode={mode}"): + compile(source, '?', mode) + with self.assertRaises( + SyntaxError, msg=f"source={source} mode={mode}"): + co = compile(source, + '?', + mode, + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) def test_compile_async_generator(self): """ @@ -514,7 +547,7 @@ def test_compile_async_generator(self): code = dedent("""async def ticker(): for i in range(10): yield i - await asyncio.sleep(0)""") + await sleep(0)""") co = compile(code, '?', 'exec', flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) glob = {} @@ -660,6 +693,16 @@ def test_divmod(self): self.assertAlmostEqual(result[1], exp_result[1]) self.assertRaises(TypeError, divmod) + self.assertRaisesRegex( + ZeroDivisionError, + "division by zero", + divmod, 1, 0, + ) + self.assertRaisesRegex( + ZeroDivisionError, + "division by zero", + divmod, 0.0, 0, + ) def test_eval(self): self.assertEqual(eval('1+1'), 2) @@ -684,6 +727,11 @@ def __getitem__(self, key): raise ValueError self.assertRaises(ValueError, eval, "foo", {}, X()) + def test_eval_kwargs(self): + data = {"A_GLOBAL_VALUE": 456} + self.assertEqual(eval("globals()['A_GLOBAL_VALUE']", globals=data), 456) + self.assertEqual(eval("globals()['A_GLOBAL_VALUE']", locals=data), 123) + def test_general_eval(self): # Tests that general mappings can be used for the locals argument @@ -777,6 +825,19 @@ def test_exec(self): del l['__builtins__'] self.assertEqual((g, l), ({'a': 1}, {'b': 2})) + def test_exec_kwargs(self): + g = {} + exec('global z\nz = 1', globals=g) + if '__builtins__' in g: + del g['__builtins__'] + self.assertEqual(g, {'z': 1}) + + # if we only set locals, the global assignment will not + # reach this locals dictionary + g = {} + exec('global z\nz = 1', locals=g) + self.assertEqual(g, {}) + def test_exec_globals(self): code = compile("print('Hello World!')", "", "exec") # no builtin function @@ -1217,6 +1278,108 @@ def test_map_pickle(self): m2 = map(map_char, "Is this the real life?") self.check_iter_pickle(m1, list(m2), proto) + # strict map tests based on strict zip tests + + def test_map_pickle_strict(self): + a = (1, 2, 3) + b = (4, 5, 6) + t = [(1, 4), (2, 5), (3, 6)] + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + m1 = map(pack, a, b, strict=True) + self.check_iter_pickle(m1, t, proto) + + def test_map_pickle_strict_fail(self): + a = (1, 2, 3) + b = (4, 5, 6, 7) + t = [(1, 4), (2, 5), (3, 6)] + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + m1 = map(pack, a, b, strict=True) + m2 = pickle.loads(pickle.dumps(m1, proto)) + self.assertEqual(self.iter_error(m1, ValueError), t) + self.assertEqual(self.iter_error(m2, ValueError), t) + + def test_map_strict(self): + self.assertEqual(tuple(map(pack, (1, 2, 3), 'abc', strict=True)), + ((1, 'a'), (2, 'b'), (3, 'c'))) + self.assertRaises(ValueError, tuple, + map(pack, (1, 2, 3, 4), 'abc', strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, (1, 2), 'abc', strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, (1, 2), (1, 2), 'abc', strict=True)) + + def test_map_strict_iterators(self): + x = iter(range(5)) + y = [0] + z = iter(range(5)) + self.assertRaises(ValueError, list, + (map(pack, x, y, z, strict=True))) + self.assertEqual(next(x), 2) + self.assertEqual(next(z), 1) + + def test_map_strict_error_handling(self): + + class Error(Exception): + pass + + class Iter: + def __init__(self, size): + self.size = size + def __iter__(self): + return self + def __next__(self): + self.size -= 1 + if self.size < 0: + raise Error + return self.size + + l1 = self.iter_error(map(pack, "AB", Iter(1), strict=True), Error) + self.assertEqual(l1, [("A", 0)]) + l2 = self.iter_error(map(pack, "AB", Iter(2), "A", strict=True), ValueError) + self.assertEqual(l2, [("A", 1, "A")]) + l3 = self.iter_error(map(pack, "AB", Iter(2), "ABC", strict=True), Error) + self.assertEqual(l3, [("A", 1, "A"), ("B", 0, "B")]) + l4 = self.iter_error(map(pack, "AB", Iter(3), strict=True), ValueError) + self.assertEqual(l4, [("A", 2), ("B", 1)]) + l5 = self.iter_error(map(pack, Iter(1), "AB", strict=True), Error) + self.assertEqual(l5, [(0, "A")]) + l6 = self.iter_error(map(pack, Iter(2), "A", strict=True), ValueError) + self.assertEqual(l6, [(1, "A")]) + l7 = self.iter_error(map(pack, Iter(2), "ABC", strict=True), Error) + self.assertEqual(l7, [(1, "A"), (0, "B")]) + l8 = self.iter_error(map(pack, Iter(3), "AB", strict=True), ValueError) + self.assertEqual(l8, [(2, "A"), (1, "B")]) + + def test_map_strict_error_handling_stopiteration(self): + + class Iter: + def __init__(self, size): + self.size = size + def __iter__(self): + return self + def __next__(self): + self.size -= 1 + if self.size < 0: + raise StopIteration + return self.size + + l1 = self.iter_error(map(pack, "AB", Iter(1), strict=True), ValueError) + self.assertEqual(l1, [("A", 0)]) + l2 = self.iter_error(map(pack, "AB", Iter(2), "A", strict=True), ValueError) + self.assertEqual(l2, [("A", 1, "A")]) + l3 = self.iter_error(map(pack, "AB", Iter(2), "ABC", strict=True), ValueError) + self.assertEqual(l3, [("A", 1, "A"), ("B", 0, "B")]) + l4 = self.iter_error(map(pack, "AB", Iter(3), strict=True), ValueError) + self.assertEqual(l4, [("A", 2), ("B", 1)]) + l5 = self.iter_error(map(pack, Iter(1), "AB", strict=True), ValueError) + self.assertEqual(l5, [(0, "A")]) + l6 = self.iter_error(map(pack, Iter(2), "A", strict=True), ValueError) + self.assertEqual(l6, [(1, "A")]) + l7 = self.iter_error(map(pack, Iter(2), "ABC", strict=True), ValueError) + self.assertEqual(l7, [(1, "A"), (0, "B")]) + l8 = self.iter_error(map(pack, Iter(3), "AB", strict=True), ValueError) + self.assertEqual(l8, [(2, "A"), (1, "B")]) + def test_max(self): self.assertEqual(max('123123'), '3') self.assertEqual(max(1, 2, 3), 3) @@ -1404,14 +1567,12 @@ def test_open(self): @unittest.skipIf(sys.flags.utf8_mode, "utf-8 mode is enabled") def test_open_default_encoding(self): - old_environ = dict(os.environ) - try: + with EnvironmentVarGuard() as env: # try to get a user preferred encoding different than the current # locale encoding to check that open() uses the current locale # encoding and not the user preferred encoding for key in ('LC_ALL', 'LANG', 'LC_CTYPE'): - if key in os.environ: - del os.environ[key] + env.unset(key) self.write_testfile() current_locale_encoding = locale.getencoding() @@ -1420,9 +1581,6 @@ def test_open_default_encoding(self): fp = open(TESTFN, 'w') with fp: self.assertEqual(fp.encoding, current_locale_encoding) - finally: - os.environ.clear() - os.environ.update(old_environ) @support.requires_subprocess() def test_open_non_inheritable(self): @@ -1675,7 +1833,10 @@ def test_bug_27936(self): def test_setattr(self): setattr(sys, 'spam', 1) - self.assertEqual(sys.spam, 1) + try: + self.assertEqual(sys.spam, 1) + finally: + del sys.spam self.assertRaises(TypeError, setattr) self.assertRaises(TypeError, setattr, sys) self.assertRaises(TypeError, setattr, sys, 'spam') @@ -1728,6 +1889,8 @@ def test_sum(self): self.assertRaises(TypeError, sum, [], '') self.assertRaises(TypeError, sum, [], b'') self.assertRaises(TypeError, sum, [], bytearray()) + self.assertRaises(OverflowError, sum, [1.0, 10**1000]) + self.assertRaises(OverflowError, sum, [1j, 10**1000]) class BadSeq: def __getitem__(self, index): @@ -1738,6 +1901,22 @@ def __getitem__(self, index): sum(([x] for x in range(10)), empty) self.assertEqual(empty, []) + xs = [complex(random.random() - .5, random.random() - .5) + for _ in range(10000)] + self.assertEqual(sum(xs), complex(sum(z.real for z in xs), + sum(z.imag for z in xs))) + + # test that sum() of complex and real numbers doesn't + # smash sign of imaginary 0 + self.assertComplexesAreIdentical(sum([complex(1, -0.0), 1]), + complex(2, -0.0)) + self.assertComplexesAreIdentical(sum([1, complex(1, -0.0)]), + complex(2, -0.0)) + self.assertComplexesAreIdentical(sum([complex(1, -0.0), 1.0]), + complex(2, -0.0)) + self.assertComplexesAreIdentical(sum([1.0, complex(1, -0.0)]), + complex(2, -0.0)) + @requires_IEEE_754 @unittest.skipIf(HAVE_DOUBLE_ROUNDING, "sum accuracy not guaranteed on machines with double rounding") @@ -1745,6 +1924,13 @@ def __getitem__(self, index): def test_sum_accuracy(self): self.assertEqual(sum([0.1] * 10), 1.0) self.assertEqual(sum([1.0, 10E100, 1.0, -10E100]), 2.0) + self.assertEqual(sum([1.0, 10E100, 1.0, -10E100, 2j]), 2+2j) + self.assertEqual(sum([2+1j, 10E100j, 1j, -10E100j]), 2+2j) + self.assertEqual(sum([1j, 1, 10E100j, 1j, 1.0, -10E100j]), 2+2j) + self.assertEqual(sum([2j, 1., 10E100, 1., -10E100]), 2+2j) + self.assertEqual(sum([1.0, 10**100, 1.0, -10**100]), 2.0) + self.assertEqual(sum([2j, 1.0, 10**100, 1.0, -10**100]), 2+2j) + self.assertEqual(sum([0.1j]*10 + [fractions.Fraction(1, 10)]), 0.1+1j) def test_type(self): self.assertEqual(type(''), type('123')) @@ -2105,19 +2291,36 @@ def test_construct_singletons(self): self.assertRaises(TypeError, tp, 1, 2) self.assertRaises(TypeError, tp, a=1, b=2) - def test_warning_notimplemented(self): - # Issue #35712: NotImplemented is a sentinel value that should never + def test_bool_notimplemented(self): + # GH-79893: NotImplemented is a sentinel value that should never # be evaluated in a boolean context (virtually all such use cases # are a result of accidental misuse implementing rich comparison # operations in terms of one another). - # For the time being, it will continue to evaluate as a true value, but - # issue a deprecation warning (with the eventual intent to make it - # a TypeError). - self.assertWarns(DeprecationWarning, bool, NotImplemented) - with self.assertWarns(DeprecationWarning): - self.assertTrue(NotImplemented) - with self.assertWarns(DeprecationWarning): - self.assertFalse(not NotImplemented) + msg = "NotImplemented should not be used in a boolean context" + self.assertRaisesRegex(TypeError, msg, bool, NotImplemented) + with self.assertRaisesRegex(TypeError, msg): + if NotImplemented: + pass + with self.assertRaisesRegex(TypeError, msg): + not NotImplemented + + def test_singleton_attribute_access(self): + for singleton in (NotImplemented, Ellipsis): + with self.subTest(singleton): + self.assertIs(type(singleton), singleton.__class__) + self.assertIs(type(singleton).__class__, type) + + # Missing instance attributes: + with self.assertRaises(AttributeError): + singleton.prop = 1 + with self.assertRaises(AttributeError): + singleton.prop + + # Missing class attributes: + with self.assertRaises(TypeError): + type(singleton).prop = 1 + with self.assertRaises(AttributeError): + type(singleton).prop class TestBreakpoint(unittest.TestCase): @@ -2331,7 +2534,8 @@ def child(wpipe): print(ascii(input(prompt)), file=wpipe) except BaseException as e: print(ascii(f'{e.__class__.__name__}: {e!s}'), file=wpipe) - lines = self.run_child(child, terminal_input + b"\r\n") + with self.detach_readline(): + lines = self.run_child(child, terminal_input + b"\r\n") # Check we did exercise the GNU readline path self.assertIn(lines[0], {'tty = True', 'tty = False'}) if lines[0] != 'tty = True': @@ -2344,28 +2548,36 @@ def child(wpipe): expected = terminal_input.decode(sys.stdin.encoding) # what else? self.assertEqual(input_result, expected) - def test_input_tty(self): - # Test input() functionality when wired to a tty (the code path - # is different and invokes GNU readline if available). - self.check_input_tty("prompt", b"quux") - - def skip_if_readline(self): + @contextlib.contextmanager + def detach_readline(self): # bpo-13886: When the readline module is loaded, PyOS_Readline() uses # the readline implementation. In some cases, the Python readline # callback rlhandler() is called by readline with a string without - # non-ASCII characters. Skip tests on non-ASCII characters if the - # readline module is loaded, since test_builtin is not intended to test + # non-ASCII characters. + # Unlink readline temporarily from PyOS_Readline() for those tests, + # since test_builtin is not intended to test # the readline module, but the builtins module. - if 'readline' in sys.modules: - self.skipTest("the readline module is loaded") + if "readline" in sys.modules: + c = import_module("ctypes") + fp_api = "PyOS_ReadlineFunctionPointer" + prev_value = c.c_void_p.in_dll(c.pythonapi, fp_api).value + c.c_void_p.in_dll(c.pythonapi, fp_api).value = None + try: + yield + finally: + c.c_void_p.in_dll(c.pythonapi, fp_api).value = prev_value + else: + yield + + def test_input_tty(self): + # Test input() functionality when wired to a tty + self.check_input_tty("prompt", b"quux") def test_input_tty_non_ascii(self): - self.skip_if_readline() # Check stdin/stdout encoding is used when invoking PyOS_Readline() self.check_input_tty("prompté", b"quux\xc3\xa9", "utf-8") def test_input_tty_non_ascii_unicode_errors(self): - self.skip_if_readline() # Check stdin/stdout error handler is used when invoking PyOS_Readline() self.check_input_tty("prompté", b"quux\xe9", "ascii") @@ -2375,14 +2587,12 @@ def test_input_tty_null_in_prompt(self): 'null characters') def test_input_tty_nonencodable_prompt(self): - self.skip_if_readline() self.check_input_tty("prompté", b"quux", "ascii", stdout_errors='strict', expected="UnicodeEncodeError: 'ascii' codec can't encode " "character '\\xe9' in position 6: ordinal not in " "range(128)") def test_input_tty_nondecodable_input(self): - self.skip_if_readline() self.check_input_tty("prompt", b"quux\xe9", "ascii", stdin_errors='strict', expected="UnicodeDecodeError: 'ascii' codec can't decode " "byte 0xe9 in position 4: ordinal not in " @@ -2484,9 +2694,12 @@ def __del__(self): class ImmortalTests(unittest.TestCase): if sys.maxsize < (1 << 32): - IMMORTAL_REFCOUNT = (1 << 30) - 1 + if support.Py_GIL_DISABLED: + IMMORTAL_REFCOUNT = 5 << 28 + else: + IMMORTAL_REFCOUNT = 7 << 28 else: - IMMORTAL_REFCOUNT = (1 << 32) - 1 + IMMORTAL_REFCOUNT = 3 << 30 IMMORTALS = (None, True, False, Ellipsis, NotImplemented, *range(-5, 257)) @@ -2517,6 +2730,7 @@ def test_new_type(self): self.assertEqual(A.__module__, __name__) self.assertEqual(A.__bases__, (object,)) self.assertIs(A.__base__, object) + self.assertNotIn('__firstlineno__', A.__dict__) x = A() self.assertIs(type(x), A) self.assertIs(x.__class__, A) @@ -2595,6 +2809,17 @@ def test_type_qualname(self): A.__qualname__ = b'B' self.assertEqual(A.__qualname__, 'D.E') + def test_type_firstlineno(self): + A = type('A', (), {'__firstlineno__': 42}) + self.assertEqual(A.__name__, 'A') + self.assertEqual(A.__module__, __name__) + self.assertEqual(A.__dict__['__firstlineno__'], 42) + A.__module__ = 'testmodule' + self.assertEqual(A.__module__, 'testmodule') + self.assertNotIn('__firstlineno__', A.__dict__) + A.__firstlineno__ = 43 + self.assertEqual(A.__dict__['__firstlineno__'], 43) + def test_type_typeparams(self): class A[T]: pass diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 71bb1e75c6affd..7bb1ab38aa4fdf 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -459,6 +459,16 @@ def test_fromhex(self): self.assertRaises(ValueError, self.type2test.fromhex, '\x00') self.assertRaises(ValueError, self.type2test.fromhex, '12 \x00 34') + # For odd number of character(s) + for value in ("a", "aaa", "deadbee"): + with self.assertRaises(ValueError) as cm: + self.type2test.fromhex(value) + self.assertIn("fromhex() arg must contain an even number of hexadecimal digits", str(cm.exception)) + for value, position in (("a ", 1), (" aa a ", 5), (" aa a a ", 5)): + with self.assertRaises(ValueError) as cm: + self.type2test.fromhex(value) + self.assertIn(f"non-hexadecimal number found in fromhex() arg at position {position}", str(cm.exception)) + for data, pos in ( # invalid first hexadecimal character ('12 x4 56', 3), @@ -991,13 +1001,13 @@ def test_translate(self): self.assertEqual(c, b'hllo') def test_sq_item(self): - _testcapi = import_helper.import_module('_testcapi') + _testlimitedcapi = import_helper.import_module('_testlimitedcapi') obj = self.type2test((42,)) with self.assertRaises(IndexError): - _testcapi.sequence_getitem(obj, -2) + _testlimitedcapi.sequence_getitem(obj, -2) with self.assertRaises(IndexError): - _testcapi.sequence_getitem(obj, 1) - self.assertEqual(_testcapi.sequence_getitem(obj, 0), 42) + _testlimitedcapi.sequence_getitem(obj, 1) + self.assertEqual(_testlimitedcapi.sequence_getitem(obj, 0), 42) class BytesTest(BaseBytesTest, unittest.TestCase): @@ -1256,7 +1266,7 @@ class SubBytes(bytes): class ByteArrayTest(BaseBytesTest, unittest.TestCase): type2test = bytearray - _testcapi = import_helper.import_module('_testcapi') + _testlimitedcapi = import_helper.import_module('_testlimitedcapi') def test_getitem_error(self): b = bytearray(b'python') @@ -1354,7 +1364,7 @@ def setitem_as_mapping(b, i, val): b[i] = val def setitem_as_sequence(b, i, val): - self._testcapi.sequence_setitem(b, i, val) + self._testlimitedcapi.sequence_setitem(b, i, val) def do_tests(setitem): b = bytearray([1, 2, 3]) @@ -1401,7 +1411,7 @@ def del_as_mapping(b, i): del b[i] def del_as_sequence(b, i): - self._testcapi.sequence_delitem(b, i) + self._testlimitedcapi.sequence_delitem(b, i) def do_tests(delete): b = bytearray(range(10)) @@ -1810,7 +1820,7 @@ def __index__(self): with self.subTest("tp_as_sequence"): b = bytearray(b'Now you see me...') with self.assertRaises(IndexError): - self._testcapi.sequence_setitem(b, 0, Boom()) + self._testlimitedcapi.sequence_setitem(b, 0, Boom()) class AssortedBytesTest(unittest.TestCase): diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 772f0eacce28f5..7d786be1d25b1c 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -476,7 +476,6 @@ def testReadlinesNoNewline(self): self.assertEqual(xlines, [b'Test']) def testContextProtocol(self): - f = None with BZ2File(self.filename, "wb") as f: f.write(b"xxx") f = BZ2File(self.filename, "rb") @@ -540,40 +539,54 @@ def testMultiStreamOrdering(self): def testOpenFilename(self): with BZ2File(self.filename, "wb") as f: f.write(b'content') + self.assertEqual(f.name, self.filename) self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'wb') self.assertIs(f.readable(), False) self.assertIs(f.writable(), True) self.assertIs(f.seekable(), False) self.assertIs(f.closed, False) self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) with BZ2File(self.filename, "ab") as f: f.write(b'appendix') + self.assertEqual(f.name, self.filename) self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'wb') self.assertIs(f.readable(), False) self.assertIs(f.writable(), True) self.assertIs(f.seekable(), False) self.assertIs(f.closed, False) self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) with BZ2File(self.filename, 'rb') as f: self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.name, self.filename) self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'rb') self.assertIs(f.readable(), True) self.assertIs(f.writable(), False) self.assertIs(f.seekable(), True) self.assertIs(f.closed, False) self.assertIs(f.closed, True) with self.assertRaises(ValueError): - f.fileno() + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) @@ -582,13 +595,18 @@ def testOpenFileWithName(self): with open(self.filename, 'wb') as raw: with BZ2File(raw, 'wb') as f: f.write(b'content') + self.assertEqual(f.name, raw.name) self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') self.assertIs(f.readable(), False) self.assertIs(f.writable(), True) self.assertIs(f.seekable(), False) self.assertIs(f.closed, False) self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) @@ -596,13 +614,18 @@ def testOpenFileWithName(self): with open(self.filename, 'ab') as raw: with BZ2File(raw, 'ab') as f: f.write(b'appendix') + self.assertEqual(f.name, raw.name) self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') self.assertIs(f.readable(), False) self.assertIs(f.writable(), True) self.assertIs(f.seekable(), False) self.assertIs(f.closed, False) self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) @@ -610,14 +633,18 @@ def testOpenFileWithName(self): with open(self.filename, 'rb') as raw: with BZ2File(raw, 'rb') as f: self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.name, raw.name) self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') self.assertIs(f.readable(), True) self.assertIs(f.writable(), False) self.assertIs(f.seekable(), True) self.assertIs(f.closed, False) self.assertIs(f.closed, True) with self.assertRaises(ValueError): - f.fileno() + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) @@ -626,61 +653,91 @@ def testOpenFileWithoutName(self): bio = BytesIO() with BZ2File(bio, 'wb') as f: f.write(b'content') + with self.assertRaises(AttributeError): + f.name self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) with BZ2File(bio, 'ab') as f: f.write(b'appendix') + with self.assertRaises(AttributeError): + f.name self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) bio.seek(0) with BZ2File(bio, 'rb') as f: self.assertEqual(f.read(), b'contentappendix') + with self.assertRaises(AttributeError): + f.name self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, 'rb') with self.assertRaises(ValueError): - f.fileno() + f.name + self.assertRaises(ValueError, f.fileno) def testOpenFileWithIntName(self): fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) with open(fd, 'wb') as raw: with BZ2File(raw, 'wb') as f: f.write(b'content') + self.assertEqual(f.name, raw.name) self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_APPEND) with open(fd, 'ab') as raw: with BZ2File(raw, 'ab') as f: f.write(b'appendix') + self.assertEqual(f.name, raw.name) self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) fd = os.open(self.filename, os.O_RDONLY) with open(fd, 'rb') as raw: with BZ2File(raw, 'rb') as f: self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.name, raw.name) self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') with self.assertRaises(ValueError): - f.fileno() + f.name + self.assertRaises(ValueError, f.fileno) def testOpenBytesFilename(self): str_filename = self.filename bytes_filename = os.fsencode(str_filename) with BZ2File(bytes_filename, "wb") as f: f.write(self.DATA) + self.assertEqual(f.name, bytes_filename) with BZ2File(bytes_filename, "rb") as f: self.assertEqual(f.read(), self.DATA) + self.assertEqual(f.name, bytes_filename) # Sanity check that we are actually operating on the right file. with BZ2File(str_filename, "rb") as f: self.assertEqual(f.read(), self.DATA) + self.assertEqual(f.name, str_filename) def testOpenPathLikeFilename(self): filename = FakePath(self.filename) with BZ2File(filename, "wb") as f: f.write(self.DATA) + self.assertEqual(f.name, self.filename) with BZ2File(filename, "rb") as f: self.assertEqual(f.read(), self.DATA) + self.assertEqual(f.name, self.filename) def testDecompressLimited(self): """Decompressed data buffering should be limited""" @@ -701,6 +758,9 @@ def testReadBytesIO(self): with BZ2File(bio) as bz2f: self.assertRaises(TypeError, bz2f.read, float()) self.assertEqual(bz2f.read(), self.TEXT) + with self.assertRaises(AttributeError): + bz2.name + self.assertEqual(bz2f.mode, 'rb') self.assertFalse(bio.closed) def testPeekBytesIO(self): @@ -716,6 +776,9 @@ def testWriteBytesIO(self): with BZ2File(bio, "w") as bz2f: self.assertRaises(TypeError, bz2f.write) bz2f.write(self.TEXT) + with self.assertRaises(AttributeError): + bz2.name + self.assertEqual(bz2f.mode, 'wb') self.assertEqual(ext_decompress(bio.getvalue()), self.TEXT) self.assertFalse(bio.closed) diff --git a/Lib/test/test_c_locale_coercion.py b/Lib/test/test_c_locale_coercion.py index 7334a325ba22f0..e4b0b8c451fd45 100644 --- a/Lib/test/test_c_locale_coercion.py +++ b/Lib/test/test_c_locale_coercion.py @@ -26,17 +26,16 @@ TARGET_LOCALES = ["C.UTF-8", "C.utf8", "UTF-8"] # Apply some platform dependent overrides -if sys.platform.startswith("linux"): - if support.is_android: - # Android defaults to using UTF-8 for all system interfaces - EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8" - EXPECTED_C_LOCALE_FS_ENCODING = "utf-8" - else: - # Linux distros typically alias the POSIX locale directly to the C - # locale. - # TODO: Once https://bugs.python.org/issue30672 is addressed, we'll be - # able to check this case unconditionally - EXPECTED_C_LOCALE_EQUIVALENTS.append("POSIX") +if sys.platform == "android": + # Android defaults to using UTF-8 for all system interfaces + EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8" + EXPECTED_C_LOCALE_FS_ENCODING = "utf-8" +elif sys.platform.startswith("linux"): + # Linux distros typically alias the POSIX locale directly to the C + # locale. + # TODO: Once https://bugs.python.org/issue30672 is addressed, we'll be + # able to check this case unconditionally + EXPECTED_C_LOCALE_EQUIVALENTS.append("POSIX") elif sys.platform.startswith("aix"): # AIX uses iso8859-1 in the C locale, other *nix platforms use ASCII EXPECTED_C_LOCALE_STREAM_ENCODING = "iso8859-1" diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index 1f9ffc5e9a5c33..073df310bb49eb 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -457,6 +457,11 @@ def test_formatmonth(self): calendar.TextCalendar().formatmonth(0, 2), result_0_02_text ) + def test_formatmonth_with_invalid_month(self): + with self.assertRaises(calendar.IllegalMonthError): + calendar.TextCalendar().formatmonth(2017, 13) + with self.assertRaises(calendar.IllegalMonthError): + calendar.TextCalendar().formatmonth(2017, -1) def test_formatmonthname_with_year(self): self.assertEqual( @@ -985,7 +990,7 @@ def assertFailure(self, *args): def test_help(self): stdout = self.run_cmd_ok('-h') self.assertIn(b'usage:', stdout) - self.assertIn(b'calendar.py', stdout) + self.assertIn(b' -m calendar ', stdout) self.assertIn(b'--help', stdout) # special case: stdout but sys.exit() @@ -1121,7 +1126,7 @@ def test__all__(self): not_exported = { 'mdays', 'January', 'February', 'EPOCH', 'different_locale', 'c', 'prweek', 'week', 'format', - 'formatstring', 'main', 'monthlen', 'prevmonth', 'nextmonth'} + 'formatstring', 'main', 'monthlen', 'prevmonth', 'nextmonth', ""} support.check__all__(self, calendar, not_exported=not_exported) @@ -1149,6 +1154,13 @@ def test_formatmonth(self): self.assertIn('class="text-center month"', self.cal.formatmonth(2017, 5)) + def test_formatmonth_with_invalid_month(self): + with self.assertRaises(calendar.IllegalMonthError): + self.cal.formatmonth(2017, 13) + with self.assertRaises(calendar.IllegalMonthError): + self.cal.formatmonth(2017, -1) + + def test_formatweek(self): weeks = self.cal.monthdays2calendar(2017, 5) self.assertIn('class="wed text-nowrap"', self.cal.formatweek(weeks[0])) diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 2a6a5d287b04ee..a4115d54ce5995 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1,16 +1,19 @@ import unittest from test.support import (cpython_only, is_wasi, requires_limited_api, Py_DEBUG, - set_recursion_limit, skip_on_s390x) + set_recursion_limit, skip_on_s390x, skip_emscripten_stack_overflow, import_helper) try: import _testcapi except ImportError: _testcapi = None +try: + import _testlimitedcapi +except ImportError: + _testlimitedcapi = None import struct import collections import itertools import gc import contextlib -import sys import types @@ -42,11 +45,16 @@ def test_frames_are_popped_after_failed_calls(self): # recovering from failed calls: def f(): pass - for _ in range(1000): - try: - f(None) - except TypeError: + class C: + def m(self): pass + callables = [f, C.m, [].__len__] + for c in callables: + for _ in range(1000): + try: + c(None) + except TypeError: + pass # BOOM! @@ -160,7 +168,7 @@ def test_varargs17_kw(self): print, 0, sep=1, end=2, file=3, flush=4, foo=5) def test_varargs18_kw(self): - # _PyArg_UnpackKeywordsWithVararg() + # _PyArg_UnpackKeywords() with varpos msg = r"invalid keyword argument for print\(\)$" with self.assertRaisesRegex(TypeError, msg): print(0, 1, **{BadStr('foo'): ','}) @@ -240,6 +248,7 @@ def test_module_not_callable_suggestion(self): self.assertRaisesRegex(TypeError, msg, mod) +@unittest.skipIf(_testcapi is None, "requires _testcapi") class TestCallingConventions(unittest.TestCase): """Test calling using various C calling conventions (METH_*) from Python @@ -437,6 +446,7 @@ def static_method(): NULL_OR_EMPTY = object() + class FastCallTests(unittest.TestCase): """Test calling using various callables from C """ @@ -480,42 +490,43 @@ class FastCallTests(unittest.TestCase): ] # Add all the calling conventions and variants of C callables - _instance = _testcapi.MethInstance() - for obj, expected_self in ( - (_testcapi, _testcapi), # module-level function - (_instance, _instance), # bound method - (_testcapi.MethClass, _testcapi.MethClass), # class method on class - (_testcapi.MethClass(), _testcapi.MethClass), # class method on inst. - (_testcapi.MethStatic, None), # static method - ): - CALLS_POSARGS.extend([ - (obj.meth_varargs, (1, 2), (expected_self, (1, 2))), - (obj.meth_varargs_keywords, - (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), - (obj.meth_fastcall, (1, 2), (expected_self, (1, 2))), - (obj.meth_fastcall, (), (expected_self, ())), - (obj.meth_fastcall_keywords, - (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), - (obj.meth_fastcall_keywords, - (), (expected_self, (), NULL_OR_EMPTY)), - (obj.meth_noargs, (), expected_self), - (obj.meth_o, (123, ), (expected_self, 123)), - ]) - - CALLS_KWARGS.extend([ - (obj.meth_varargs_keywords, - (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), - (obj.meth_varargs_keywords, - (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), - (obj.meth_varargs_keywords, - (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), - (obj.meth_fastcall_keywords, - (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), - (obj.meth_fastcall_keywords, - (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), - (obj.meth_fastcall_keywords, - (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), - ]) + if _testcapi: + _instance = _testcapi.MethInstance() + for obj, expected_self in ( + (_testcapi, _testcapi), # module-level function + (_instance, _instance), # bound method + (_testcapi.MethClass, _testcapi.MethClass), # class method on class + (_testcapi.MethClass(), _testcapi.MethClass), # class method on inst. + (_testcapi.MethStatic, None), # static method + ): + CALLS_POSARGS.extend([ + (obj.meth_varargs, (1, 2), (expected_self, (1, 2))), + (obj.meth_varargs_keywords, + (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), + (obj.meth_fastcall, (1, 2), (expected_self, (1, 2))), + (obj.meth_fastcall, (), (expected_self, ())), + (obj.meth_fastcall_keywords, + (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), + (obj.meth_fastcall_keywords, + (), (expected_self, (), NULL_OR_EMPTY)), + (obj.meth_noargs, (), expected_self), + (obj.meth_o, (123, ), (expected_self, 123)), + ]) + + CALLS_KWARGS.extend([ + (obj.meth_varargs_keywords, + (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), + (obj.meth_varargs_keywords, + (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), + (obj.meth_varargs_keywords, + (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), + (obj.meth_fastcall_keywords, + (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), + (obj.meth_fastcall_keywords, + (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), + (obj.meth_fastcall_keywords, + (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), + ]) def check_result(self, result, expected): if isinstance(expected, tuple) and expected[-1] is NULL_OR_EMPTY: @@ -523,6 +534,7 @@ def check_result(self, result, expected): expected = (*expected[:-1], result[-1]) self.assertEqual(result, expected) + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_vectorcall_dict(self): # Test PyObject_VectorcallDict() @@ -542,6 +554,7 @@ def test_vectorcall_dict(self): result = _testcapi.pyobject_fastcalldict(func, args, kwargs) self.check_result(result, expected) + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_vectorcall(self): # Test PyObject_Vectorcall() @@ -603,9 +616,7 @@ def testfunction_kw(self, *, kw): return self -ADAPTIVE_WARMUP_DELAY = 2 - - +@unittest.skipIf(_testcapi is None, "requires _testcapi") class TestPEP590(unittest.TestCase): def test_method_descriptor_flag(self): @@ -788,17 +799,18 @@ def __call__(self, *args): def test_setvectorcall(self): from _testcapi import function_setvectorcall + _testinternalcapi = import_helper.import_module("_testinternalcapi") def f(num): return num + 1 assert_equal = self.assertEqual num = 10 assert_equal(11, f(num)) function_setvectorcall(f) - # make sure specializer is triggered by running > 50 times - for _ in range(10 * ADAPTIVE_WARMUP_DELAY): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): assert_equal("overridden", f(num)) def test_setvectorcall_load_attr_specialization_skip(self): from _testcapi import function_setvectorcall + _testinternalcapi = import_helper.import_module("_testinternalcapi") class X: def __getattribute__(self, attr): @@ -810,11 +822,12 @@ def __getattribute__(self, attr): function_setvectorcall(X.__getattribute__) # make sure specialization doesn't trigger # when vectorcall is overridden - for _ in range(ADAPTIVE_WARMUP_DELAY): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): assert_equal("overridden", x.a) def test_setvectorcall_load_attr_specialization_deopt(self): from _testcapi import function_setvectorcall + _testinternalcapi = import_helper.import_module("_testinternalcapi") class X: def __getattribute__(self, attr): @@ -826,23 +839,28 @@ def get_a(x): assert_equal = self.assertEqual x = X() # trigger LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN specialization - for _ in range(ADAPTIVE_WARMUP_DELAY): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): assert_equal("a", get_a(x)) function_setvectorcall(X.__getattribute__) # make sure specialized LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN # gets deopted due to overridden vectorcall - for _ in range(ADAPTIVE_WARMUP_DELAY): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): assert_equal("overridden", get_a(x)) @requires_limited_api def test_vectorcall_limited_incoming(self): from _testcapi import pyobject_vectorcall - obj = _testcapi.LimitedVectorCallClass() - self.assertEqual(pyobject_vectorcall(obj, (), ()), "vectorcall called") + for cls in (_testlimitedcapi.LimitedVectorCallClass, + _testlimitedcapi.LimitedRelativeVectorCallClass): + with self.subTest(cls=cls): + obj = cls() + self.assertEqual( + pyobject_vectorcall(obj, (), ()), + "vectorcall called") @requires_limited_api def test_vectorcall_limited_outgoing(self): - from _testcapi import call_vectorcall + from _testlimitedcapi import call_vectorcall args_captured = [] kwargs_captured = [] @@ -858,7 +876,7 @@ def f(*args, **kwargs): @requires_limited_api def test_vectorcall_limited_outgoing_method(self): - from _testcapi import call_vectorcall_method + from _testlimitedcapi import call_vectorcall_method args_captured = [] kwargs_captured = [] @@ -1018,6 +1036,8 @@ class TestRecursion(unittest.TestCase): @skip_on_s390x @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") + @unittest.skipIf(_testcapi is None, "requires _testcapi") + @skip_emscripten_stack_overflow() def test_super_deep(self): def recurse(n): diff --git a/Lib/test/test_capi/__init__.py b/Lib/test/test_capi/__init__.py index 4b16ecc31156a5..5a8ba6845399e0 100644 --- a/Lib/test/test_capi/__init__.py +++ b/Lib/test/test_capi/__init__.py @@ -1,5 +1,12 @@ import os +import unittest from test.support import load_package_tests +from test.support import TEST_MODULES_ENABLED + + +if not TEST_MODULES_ENABLED: + raise unittest.SkipTest("requires test modules") + def load_tests(*args): return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_capi/check_config.py b/Lib/test/test_capi/check_config.py index eb99ae16f2b69e..bc4f3a0beb9c1e 100644 --- a/Lib/test/test_capi/check_config.py +++ b/Lib/test/test_capi/check_config.py @@ -10,7 +10,7 @@ def import_singlephase(): assert '_testsinglephase' not in sys.modules try: - import _testsinglephase + import _testsinglephase # noqa: F401 except ImportError: sys.modules.pop('_testsinglephase', None) return False diff --git a/Lib/test/test_capi/test_abstract.py b/Lib/test/test_capi/test_abstract.py index 97ed939928c360..3de251bc5c241e 100644 --- a/Lib/test/test_capi/test_abstract.py +++ b/Lib/test/test_capi/test_abstract.py @@ -4,6 +4,7 @@ from test.support import import_helper _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX NULL = None @@ -74,7 +75,7 @@ def assertTypedEqual(self, actual, expected): def test_object_str(self): # Test PyObject_Str() - object_str = _testcapi.object_str + object_str = _testlimitedcapi.object_str self.assertTypedEqual(object_str(''), '') self.assertTypedEqual(object_str('abc'), 'abc') self.assertTypedEqual(object_str('\U0001f40d'), '\U0001f40d') @@ -87,7 +88,7 @@ def test_object_str(self): def test_object_repr(self): # Test PyObject_Repr() - object_repr = _testcapi.object_repr + object_repr = _testlimitedcapi.object_repr self.assertTypedEqual(object_repr(''), "''") self.assertTypedEqual(object_repr('abc'), "'abc'") self.assertTypedEqual(object_repr('\U0001f40d'), "'\U0001f40d'") @@ -100,7 +101,7 @@ def test_object_repr(self): def test_object_ascii(self): # Test PyObject_ASCII() - object_ascii = _testcapi.object_ascii + object_ascii = _testlimitedcapi.object_ascii self.assertTypedEqual(object_ascii(''), "''") self.assertTypedEqual(object_ascii('abc'), "'abc'") self.assertTypedEqual(object_ascii('\U0001f40d'), r"'\U0001f40d'") @@ -113,7 +114,7 @@ def test_object_ascii(self): def test_object_bytes(self): # Test PyObject_Bytes() - object_bytes = _testcapi.object_bytes + object_bytes = _testlimitedcapi.object_bytes self.assertTypedEqual(object_bytes(b''), b'') self.assertTypedEqual(object_bytes(b'abc'), b'abc') self.assertTypedEqual(object_bytes(BytesSubclass(b'abc')), b'abc') @@ -132,7 +133,7 @@ def test_object_bytes(self): self.assertTypedEqual(object_bytes(NULL), b'') def test_object_getattr(self): - xgetattr = _testcapi.object_getattr + xgetattr = _testlimitedcapi.object_getattr obj = TestObject() obj.a = 11 setattr(obj, '\U0001f40d', 22) @@ -146,7 +147,7 @@ def test_object_getattr(self): # CRASHES xgetattr(NULL, 'a') def test_object_getattrstring(self): - getattrstring = _testcapi.object_getattrstring + getattrstring = _testlimitedcapi.object_getattrstring obj = TestObject() obj.a = 11 setattr(obj, '\U0001f40d', 22) @@ -188,7 +189,7 @@ def test_object_getoptionalattrstring(self): # CRASHES getoptionalattrstring(NULL, b'a') def test_object_hasattr(self): - xhasattr = _testcapi.object_hasattr + xhasattr = _testlimitedcapi.object_hasattr obj = TestObject() obj.a = 1 setattr(obj, '\U0001f40d', 2) @@ -212,7 +213,7 @@ def test_object_hasattr(self): # CRASHES xhasattr(NULL, 'a') def test_object_hasattrstring(self): - hasattrstring = _testcapi.object_hasattrstring + hasattrstring = _testlimitedcapi.object_hasattrstring obj = TestObject() obj.a = 1 setattr(obj, '\U0001f40d', 2) @@ -264,7 +265,7 @@ def test_object_hasattrstringwitherror(self): # CRASHES hasattrstring(NULL, b'a') def test_object_setattr(self): - xsetattr = _testcapi.object_setattr + xsetattr = _testlimitedcapi.object_setattr obj = TestObject() xsetattr(obj, 'a', 5) self.assertEqual(obj.a, 5) @@ -273,7 +274,7 @@ def test_object_setattr(self): # PyObject_SetAttr(obj, attr_name, NULL) removes the attribute xsetattr(obj, 'a', NULL) - self.assertFalse(hasattr(obj, 'a')) + self.assertNotHasAttr(obj, 'a') self.assertRaises(AttributeError, xsetattr, obj, 'b', NULL) self.assertRaises(RuntimeError, xsetattr, obj, 'evil', NULL) @@ -284,7 +285,7 @@ def test_object_setattr(self): # CRASHES xsetattr(NULL, 'a', 5) def test_object_setattrstring(self): - setattrstring = _testcapi.object_setattrstring + setattrstring = _testlimitedcapi.object_setattrstring obj = TestObject() setattrstring(obj, b'a', 5) self.assertEqual(obj.a, 5) @@ -293,7 +294,7 @@ def test_object_setattrstring(self): # PyObject_SetAttrString(obj, attr_name, NULL) removes the attribute setattrstring(obj, b'a', NULL) - self.assertFalse(hasattr(obj, 'a')) + self.assertNotHasAttr(obj, 'a') self.assertRaises(AttributeError, setattrstring, obj, b'b', NULL) self.assertRaises(RuntimeError, setattrstring, obj, b'evil', NULL) @@ -305,15 +306,15 @@ def test_object_setattrstring(self): # CRASHES setattrstring(NULL, b'a', 5) def test_object_delattr(self): - xdelattr = _testcapi.object_delattr + xdelattr = _testlimitedcapi.object_delattr obj = TestObject() obj.a = 1 setattr(obj, '\U0001f40d', 2) xdelattr(obj, 'a') - self.assertFalse(hasattr(obj, 'a')) + self.assertNotHasAttr(obj, 'a') self.assertRaises(AttributeError, xdelattr, obj, 'b') xdelattr(obj, '\U0001f40d') - self.assertFalse(hasattr(obj, '\U0001f40d')) + self.assertNotHasAttr(obj, '\U0001f40d') self.assertRaises(AttributeError, xdelattr, 42, 'numerator') self.assertRaises(RuntimeError, xdelattr, obj, 'evil') @@ -322,15 +323,15 @@ def test_object_delattr(self): # CRASHES xdelattr(NULL, 'a') def test_object_delattrstring(self): - delattrstring = _testcapi.object_delattrstring + delattrstring = _testlimitedcapi.object_delattrstring obj = TestObject() obj.a = 1 setattr(obj, '\U0001f40d', 2) delattrstring(obj, b'a') - self.assertFalse(hasattr(obj, 'a')) + self.assertNotHasAttr(obj, 'a') self.assertRaises(AttributeError, delattrstring, obj, b'b') delattrstring(obj, '\U0001f40d'.encode()) - self.assertFalse(hasattr(obj, '\U0001f40d')) + self.assertNotHasAttr(obj, '\U0001f40d') self.assertRaises(AttributeError, delattrstring, 42, b'numerator') self.assertRaises(RuntimeError, delattrstring, obj, b'evil') @@ -340,7 +341,7 @@ def test_object_delattrstring(self): def test_mapping_check(self): - check = _testcapi.mapping_check + check = _testlimitedcapi.mapping_check self.assertTrue(check({1: 2})) self.assertTrue(check([1, 2])) self.assertTrue(check((1, 2))) @@ -351,7 +352,7 @@ def test_mapping_check(self): self.assertFalse(check(NULL)) def test_mapping_size(self): - for size in _testcapi.mapping_size, _testcapi.mapping_length: + for size in _testlimitedcapi.mapping_size, _testlimitedcapi.mapping_length: self.assertEqual(size({1: 2}), 1) self.assertEqual(size([1, 2]), 2) self.assertEqual(size((1, 2)), 2) @@ -363,7 +364,7 @@ def test_mapping_size(self): self.assertRaises(SystemError, size, NULL) def test_object_getitem(self): - getitem = _testcapi.object_getitem + getitem = _testlimitedcapi.object_getitem dct = {'a': 1, '\U0001f40d': 2} self.assertEqual(getitem(dct, 'a'), 1) self.assertRaises(KeyError, getitem, dct, 'b') @@ -383,7 +384,7 @@ def test_object_getitem(self): self.assertRaises(SystemError, getitem, NULL, 'a') def test_mapping_getitemstring(self): - getitemstring = _testcapi.mapping_getitemstring + getitemstring = _testlimitedcapi.mapping_getitemstring dct = {'a': 1, '\U0001f40d': 2} self.assertEqual(getitemstring(dct, b'a'), 1) self.assertRaises(KeyError, getitemstring, dct, b'b') @@ -437,7 +438,7 @@ def test_mapping_getoptionalitemstring(self): # CRASHES getitemstring(NULL, b'a') def test_mapping_haskey(self): - haskey = _testcapi.mapping_haskey + haskey = _testlimitedcapi.mapping_haskey dct = {'a': 1, '\U0001f40d': 2} self.assertTrue(haskey(dct, 'a')) self.assertFalse(haskey(dct, 'b')) @@ -486,7 +487,7 @@ def test_mapping_haskey(self): 'null argument to internal routine') def test_mapping_haskeystring(self): - haskeystring = _testcapi.mapping_haskeystring + haskeystring = _testlimitedcapi.mapping_haskeystring dct = {'a': 1, '\U0001f40d': 2} self.assertTrue(haskeystring(dct, b'a')) self.assertFalse(haskeystring(dct, b'b')) @@ -527,7 +528,7 @@ def test_mapping_haskeystring(self): "null argument to internal routine") def test_mapping_haskeywitherror(self): - haskey = _testcapi.mapping_haskeywitherror + haskey = _testlimitedcapi.mapping_haskeywitherror dct = {'a': 1, '\U0001f40d': 2} self.assertTrue(haskey(dct, 'a')) self.assertFalse(haskey(dct, 'b')) @@ -548,7 +549,7 @@ def test_mapping_haskeywitherror(self): # CRASHES haskey(NULL, 'a')) def test_mapping_haskeystringwitherror(self): - haskeystring = _testcapi.mapping_haskeystringwitherror + haskeystring = _testlimitedcapi.mapping_haskeystringwitherror dct = {'a': 1, '\U0001f40d': 2} self.assertTrue(haskeystring(dct, b'a')) self.assertFalse(haskeystring(dct, b'b')) @@ -565,7 +566,7 @@ def test_mapping_haskeystringwitherror(self): # CRASHES haskeystring(NULL, b'a') def test_object_setitem(self): - setitem = _testcapi.object_setitem + setitem = _testlimitedcapi.object_setitem dct = {} setitem(dct, 'a', 5) self.assertEqual(dct, {'a': 5}) @@ -591,7 +592,7 @@ def test_object_setitem(self): self.assertRaises(SystemError, setitem, NULL, 'a', 5) def test_mapping_setitemstring(self): - setitemstring = _testcapi.mapping_setitemstring + setitemstring = _testlimitedcapi.mapping_setitemstring dct = {} setitemstring(dct, b'a', 5) self.assertEqual(dct, {'a': 5}) @@ -611,7 +612,7 @@ def test_mapping_setitemstring(self): self.assertRaises(SystemError, setitemstring, NULL, b'a', 5) def test_object_delitem(self): - for delitem in _testcapi.object_delitem, _testcapi.mapping_delitem: + for delitem in _testlimitedcapi.object_delitem, _testlimitedcapi.mapping_delitem: dct = {'a': 1, 'c': 2, '\U0001f40d': 3} delitem(dct, 'a') self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) @@ -637,7 +638,7 @@ def test_object_delitem(self): self.assertRaises(SystemError, delitem, NULL, 'a') def test_mapping_delitemstring(self): - delitemstring = _testcapi.mapping_delitemstring + delitemstring = _testlimitedcapi.mapping_delitemstring dct = {'a': 1, 'c': 2, '\U0001f40d': 3} delitemstring(dct, b'a') self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) @@ -677,23 +678,23 @@ def items(self): for mapping in [{}, OrderedDict(), Mapping1(), Mapping2(), dict_obj, OrderedDict(dict_obj), Mapping1(dict_obj), Mapping2(dict_obj)]: - self.assertListEqual(_testcapi.mapping_keys(mapping), + self.assertListEqual(_testlimitedcapi.mapping_keys(mapping), list(mapping.keys())) - self.assertListEqual(_testcapi.mapping_values(mapping), + self.assertListEqual(_testlimitedcapi.mapping_values(mapping), list(mapping.values())) - self.assertListEqual(_testcapi.mapping_items(mapping), + self.assertListEqual(_testlimitedcapi.mapping_items(mapping), list(mapping.items())) def test_mapping_keys_valuesitems_bad_arg(self): - self.assertRaises(AttributeError, _testcapi.mapping_keys, object()) - self.assertRaises(AttributeError, _testcapi.mapping_values, object()) - self.assertRaises(AttributeError, _testcapi.mapping_items, object()) - self.assertRaises(AttributeError, _testcapi.mapping_keys, []) - self.assertRaises(AttributeError, _testcapi.mapping_values, []) - self.assertRaises(AttributeError, _testcapi.mapping_items, []) - self.assertRaises(SystemError, _testcapi.mapping_keys, NULL) - self.assertRaises(SystemError, _testcapi.mapping_values, NULL) - self.assertRaises(SystemError, _testcapi.mapping_items, NULL) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_keys, object()) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_values, object()) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_items, object()) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_keys, []) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_values, []) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_items, []) + self.assertRaises(SystemError, _testlimitedcapi.mapping_keys, NULL) + self.assertRaises(SystemError, _testlimitedcapi.mapping_values, NULL) + self.assertRaises(SystemError, _testlimitedcapi.mapping_items, NULL) class BadMapping: def keys(self): @@ -703,12 +704,12 @@ def values(self): def items(self): return None bad_mapping = BadMapping() - self.assertRaises(TypeError, _testcapi.mapping_keys, bad_mapping) - self.assertRaises(TypeError, _testcapi.mapping_values, bad_mapping) - self.assertRaises(TypeError, _testcapi.mapping_items, bad_mapping) + self.assertRaises(TypeError, _testlimitedcapi.mapping_keys, bad_mapping) + self.assertRaises(TypeError, _testlimitedcapi.mapping_values, bad_mapping) + self.assertRaises(TypeError, _testlimitedcapi.mapping_items, bad_mapping) def test_sequence_check(self): - check = _testcapi.sequence_check + check = _testlimitedcapi.sequence_check self.assertFalse(check({1: 2})) self.assertTrue(check([1, 2])) self.assertTrue(check((1, 2))) @@ -719,7 +720,7 @@ def test_sequence_check(self): # CRASHES check(NULL) def test_sequence_size(self): - for size in _testcapi.sequence_size, _testcapi.sequence_length: + for size in _testlimitedcapi.sequence_size, _testlimitedcapi.sequence_length: self.assertEqual(size([1, 2]), 2) self.assertEqual(size((1, 2)), 2) self.assertEqual(size('abc'), 3) @@ -731,7 +732,7 @@ def test_sequence_size(self): self.assertRaises(SystemError, size, NULL) def test_sequence_getitem(self): - getitem = _testcapi.sequence_getitem + getitem = _testlimitedcapi.sequence_getitem lst = ['a', 'b', 'c'] self.assertEqual(getitem(lst, 1), 'b') self.assertEqual(getitem(lst, -1), 'c') @@ -744,7 +745,7 @@ def test_sequence_getitem(self): self.assertRaises(SystemError, getitem, NULL, 1) def test_sequence_concat(self): - concat = _testcapi.sequence_concat + concat = _testlimitedcapi.sequence_concat self.assertEqual(concat(['a', 'b'], [1, 2]), ['a', 'b', 1, 2]) self.assertEqual(concat(('a', 'b'), (1, 2)), ('a', 'b', 1, 2)) @@ -757,7 +758,7 @@ def test_sequence_concat(self): self.assertRaises(SystemError, concat, NULL, []) def test_sequence_repeat(self): - repeat = _testcapi.sequence_repeat + repeat = _testlimitedcapi.sequence_repeat self.assertEqual(repeat(['a', 'b'], 2), ['a', 'b', 'a', 'b']) self.assertEqual(repeat(('a', 'b'), 2), ('a', 'b', 'a', 'b')) self.assertEqual(repeat(['a', 'b'], 0), []) @@ -771,7 +772,7 @@ def test_sequence_repeat(self): self.assertRaises(SystemError, repeat, NULL, 2) def test_sequence_inplaceconcat(self): - inplaceconcat = _testcapi.sequence_inplaceconcat + inplaceconcat = _testlimitedcapi.sequence_inplaceconcat lst = ['a', 'b'] res = inplaceconcat(lst, [1, 2]) self.assertEqual(res, ['a', 'b', 1, 2]) @@ -790,7 +791,7 @@ def test_sequence_inplaceconcat(self): self.assertRaises(SystemError, inplaceconcat, NULL, []) def test_sequence_inplacerepeat(self): - inplacerepeat = _testcapi.sequence_inplacerepeat + inplacerepeat = _testlimitedcapi.sequence_inplacerepeat lst = ['a', 'b'] res = inplacerepeat(lst, 2) self.assertEqual(res, ['a', 'b', 'a', 'b']) @@ -807,7 +808,7 @@ def test_sequence_inplacerepeat(self): self.assertRaises(SystemError, inplacerepeat, NULL, 2) def test_sequence_setitem(self): - setitem = _testcapi.sequence_setitem + setitem = _testlimitedcapi.sequence_setitem lst = ['a', 'b', 'c'] setitem(lst, 1, 'x') self.assertEqual(lst, ['a', 'x', 'c']) @@ -825,7 +826,7 @@ def test_sequence_setitem(self): self.assertRaises(SystemError, setitem, NULL, 1, 'x') def test_sequence_delitem(self): - delitem = _testcapi.sequence_delitem + delitem = _testlimitedcapi.sequence_delitem lst = ['a', 'b', 'c'] delitem(lst, 1) self.assertEqual(lst, ['a', 'c']) @@ -840,7 +841,7 @@ def test_sequence_delitem(self): self.assertRaises(SystemError, delitem, NULL, 1) def test_sequence_setslice(self): - setslice = _testcapi.sequence_setslice + setslice = _testlimitedcapi.sequence_setslice # Correct case: for start in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: @@ -882,7 +883,7 @@ def __setitem__(self, index, value): self.assertRaises(SystemError, setslice, NULL, 1, 3, 'xy') def test_sequence_delslice(self): - delslice = _testcapi.sequence_delslice + delslice = _testlimitedcapi.sequence_delslice # Correct case: for start in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: @@ -920,7 +921,7 @@ def __delitem__(self, index): self.assertEqual(mapping, {1: 'a', 2: 'b', 3: 'c'}) def test_sequence_count(self): - count = _testcapi.sequence_count + count = _testlimitedcapi.sequence_count lst = ['a', 'b', 'a'] self.assertEqual(count(lst, 'a'), 2) @@ -935,7 +936,7 @@ def test_sequence_count(self): self.assertRaises(SystemError, count, NULL, 'a') def test_sequence_contains(self): - contains = _testcapi.sequence_contains + contains = _testlimitedcapi.sequence_contains lst = ['a', 'b', 'a'] self.assertEqual(contains(lst, 'a'), 1) @@ -954,7 +955,7 @@ def test_sequence_contains(self): # CRASHES contains(NULL, 'a') def test_sequence_index(self): - index = _testcapi.sequence_index + index = _testlimitedcapi.sequence_index lst = ['a', 'b', 'a'] self.assertEqual(index(lst, 'a'), 0) @@ -974,7 +975,7 @@ def test_sequence_index(self): self.assertRaises(SystemError, index, NULL, 'a') def test_sequence_list(self): - xlist = _testcapi.sequence_list + xlist = _testlimitedcapi.sequence_list self.assertEqual(xlist(['a', 'b', 'c']), ['a', 'b', 'c']) self.assertEqual(xlist(('a', 'b', 'c')), ['a', 'b', 'c']) self.assertEqual(xlist(iter(['a', 'b', 'c'])), ['a', 'b', 'c']) @@ -984,7 +985,7 @@ def test_sequence_list(self): self.assertRaises(SystemError, xlist, NULL) def test_sequence_tuple(self): - xtuple = _testcapi.sequence_tuple + xtuple = _testlimitedcapi.sequence_tuple self.assertEqual(xtuple(['a', 'b', 'c']), ('a', 'b', 'c')) self.assertEqual(xtuple(('a', 'b', 'c')), ('a', 'b', 'c')) self.assertEqual(xtuple(iter(['a', 'b', 'c'])), ('a', 'b', 'c')) @@ -993,12 +994,51 @@ def test_sequence_tuple(self): self.assertRaises(TypeError, xtuple, 42) self.assertRaises(SystemError, xtuple, NULL) - def test_number_check(self): - number_check = _testcapi.number_check - self.assertTrue(number_check(1 + 1j)) - self.assertTrue(number_check(1)) - self.assertTrue(number_check(0.5)) - self.assertFalse(number_check("1 + 1j")) + def test_object_generichash(self): + # Test PyObject_GenericHash() + generichash = _testcapi.object_generichash + for obj in object(), 1, 'string', []: + self.assertEqual(generichash(obj), object.__hash__(obj)) + + def run_iter_api_test(self, next_func): + for data in (), [], (1, 2, 3), [1 , 2, 3], "123": + with self.subTest(data=data): + items = [] + it = iter(data) + while (item := next_func(it)) is not None: + items.append(item) + self.assertEqual(items, list(data)) + + class Broken: + def __init__(self): + self.count = 0 + + def __next__(self): + if self.count < 3: + self.count += 1 + return self.count + else: + raise TypeError('bad type') + + it = Broken() + self.assertEqual(next_func(it), 1) + self.assertEqual(next_func(it), 2) + self.assertEqual(next_func(it), 3) + with self.assertRaisesRegex(TypeError, 'bad type'): + next_func(it) + + def test_iter_next(self): + from _testcapi import PyIter_Next + self.run_iter_api_test(PyIter_Next) + # CRASHES PyIter_Next(10) + + def test_iter_nextitem(self): + from _testcapi import PyIter_NextItem + self.run_iter_api_test(PyIter_NextItem) + + regex = "expected.*iterator.*got.*'int'" + with self.assertRaisesRegex(TypeError, regex): + PyIter_NextItem(10) if __name__ == "__main__": diff --git a/Lib/test/test_capi/test_bytearray.py b/Lib/test/test_capi/test_bytearray.py index 833122c4e319d8..39099f6b82240f 100644 --- a/Lib/test/test_capi/test_bytearray.py +++ b/Lib/test/test_capi/test_bytearray.py @@ -1,7 +1,7 @@ import unittest from test.support import import_helper -_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX NULL = None @@ -19,7 +19,7 @@ def __bytes__(self): class CAPITest(unittest.TestCase): def test_check(self): # Test PyByteArray_Check() - check = _testcapi.bytearray_check + check = _testlimitedcapi.bytearray_check self.assertTrue(check(bytearray(b'abc'))) self.assertFalse(check(b'abc')) self.assertTrue(check(ByteArraySubclass(b'abc'))) @@ -32,7 +32,7 @@ def test_check(self): def test_checkexact(self): # Test PyByteArray_CheckExact() - check = _testcapi.bytearray_checkexact + check = _testlimitedcapi.bytearray_checkexact self.assertTrue(check(bytearray(b'abc'))) self.assertFalse(check(b'abc')) self.assertFalse(check(ByteArraySubclass(b'abc'))) @@ -45,7 +45,7 @@ def test_checkexact(self): def test_fromstringandsize(self): # Test PyByteArray_FromStringAndSize() - fromstringandsize = _testcapi.bytearray_fromstringandsize + fromstringandsize = _testlimitedcapi.bytearray_fromstringandsize self.assertEqual(fromstringandsize(b'abc'), bytearray(b'abc')) self.assertEqual(fromstringandsize(b'abc', 2), bytearray(b'ab')) @@ -62,7 +62,7 @@ def test_fromstringandsize(self): def test_fromobject(self): # Test PyByteArray_FromObject() - fromobject = _testcapi.bytearray_fromobject + fromobject = _testlimitedcapi.bytearray_fromobject self.assertEqual(fromobject(b'abc'), bytearray(b'abc')) self.assertEqual(fromobject(bytearray(b'abc')), bytearray(b'abc')) @@ -77,7 +77,7 @@ def test_fromobject(self): def test_size(self): # Test PyByteArray_Size() - size = _testcapi.bytearray_size + size = _testlimitedcapi.bytearray_size self.assertEqual(size(bytearray(b'abc')), 3) self.assertEqual(size(ByteArraySubclass(b'abc')), 3) @@ -88,7 +88,7 @@ def test_size(self): def test_asstring(self): """Test PyByteArray_AsString()""" - asstring = _testcapi.bytearray_asstring + asstring = _testlimitedcapi.bytearray_asstring self.assertEqual(asstring(bytearray(b'abc'), 4), b'abc\0') self.assertEqual(asstring(ByteArraySubclass(b'abc'), 4), b'abc\0') @@ -100,7 +100,7 @@ def test_asstring(self): def test_concat(self): """Test PyByteArray_Concat()""" - concat = _testcapi.bytearray_concat + concat = _testlimitedcapi.bytearray_concat ba = bytearray(b'abc') self.assertEqual(concat(ba, b'def'), bytearray(b'abcdef')) @@ -133,7 +133,7 @@ def test_concat(self): def test_resize(self): """Test PyByteArray_Resize()""" - resize = _testcapi.bytearray_resize + resize = _testlimitedcapi.bytearray_resize ba = bytearray(b'abcdef') self.assertEqual(resize(ba, 3), 0) diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index bb5d724ff187d4..5908d79e14029b 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -1,6 +1,7 @@ import unittest from test.support import import_helper +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') _testcapi = import_helper.import_module('_testcapi') from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX @@ -19,7 +20,7 @@ def __bytes__(self): class CAPITest(unittest.TestCase): def test_check(self): # Test PyBytes_Check() - check = _testcapi.bytes_check + check = _testlimitedcapi.bytes_check self.assertTrue(check(b'abc')) self.assertFalse(check('abc')) self.assertFalse(check(bytearray(b'abc'))) @@ -33,7 +34,7 @@ def test_check(self): def test_checkexact(self): # Test PyBytes_CheckExact() - check = _testcapi.bytes_checkexact + check = _testlimitedcapi.bytes_checkexact self.assertTrue(check(b'abc')) self.assertFalse(check('abc')) self.assertFalse(check(bytearray(b'abc'))) @@ -47,11 +48,13 @@ def test_checkexact(self): def test_fromstringandsize(self): # Test PyBytes_FromStringAndSize() - fromstringandsize = _testcapi.bytes_fromstringandsize + fromstringandsize = _testlimitedcapi.bytes_fromstringandsize self.assertEqual(fromstringandsize(b'abc'), b'abc') self.assertEqual(fromstringandsize(b'abc', 2), b'ab') self.assertEqual(fromstringandsize(b'abc\0def'), b'abc\0def') + self.assertEqual(fromstringandsize(b'a'), b'a') + self.assertEqual(fromstringandsize(b'a', 1), b'a') self.assertEqual(fromstringandsize(b'', 0), b'') self.assertEqual(fromstringandsize(NULL, 0), b'') self.assertEqual(len(fromstringandsize(NULL, 3)), 3) @@ -65,7 +68,7 @@ def test_fromstringandsize(self): def test_fromstring(self): # Test PyBytes_FromString() - fromstring = _testcapi.bytes_fromstring + fromstring = _testlimitedcapi.bytes_fromstring self.assertEqual(fromstring(b'abc\0def'), b'abc') self.assertEqual(fromstring(b''), b'') @@ -74,7 +77,7 @@ def test_fromstring(self): def test_fromobject(self): # Test PyBytes_FromObject() - fromobject = _testcapi.bytes_fromobject + fromobject = _testlimitedcapi.bytes_fromobject self.assertEqual(fromobject(b'abc'), b'abc') self.assertEqual(fromobject(bytearray(b'abc')), b'abc') @@ -88,7 +91,7 @@ def test_fromobject(self): def test_size(self): # Test PyBytes_Size() - size = _testcapi.bytes_size + size = _testlimitedcapi.bytes_size self.assertEqual(size(b'abc'), 3) self.assertEqual(size(BytesSubclass(b'abc')), 3) @@ -100,7 +103,7 @@ def test_size(self): def test_asstring(self): """Test PyBytes_AsString()""" - asstring = _testcapi.bytes_asstring + asstring = _testlimitedcapi.bytes_asstring self.assertEqual(asstring(b'abc', 4), b'abc\0') self.assertEqual(asstring(b'abc\0def', 8), b'abc\0def\0') @@ -111,8 +114,8 @@ def test_asstring(self): def test_asstringandsize(self): """Test PyBytes_AsStringAndSize()""" - asstringandsize = _testcapi.bytes_asstringandsize - asstringandsize_null = _testcapi.bytes_asstringandsize_null + asstringandsize = _testlimitedcapi.bytes_asstringandsize + asstringandsize_null = _testlimitedcapi.bytes_asstringandsize_null self.assertEqual(asstringandsize(b'abc', 4), (b'abc\0', 3)) self.assertEqual(asstringandsize(b'abc\0def', 8), (b'abc\0def\0', 7)) @@ -128,7 +131,7 @@ def test_asstringandsize(self): def test_repr(self): # Test PyBytes_Repr() - bytes_repr = _testcapi.bytes_repr + bytes_repr = _testlimitedcapi.bytes_repr self.assertEqual(bytes_repr(b'''abc''', 0), r"""b'abc'""") self.assertEqual(bytes_repr(b'''abc''', 1), r"""b'abc'""") @@ -149,7 +152,7 @@ def test_repr(self): def test_concat(self, concat=None): """Test PyBytes_Concat()""" if concat is None: - concat = _testcapi.bytes_concat + concat = _testlimitedcapi.bytes_concat self.assertEqual(concat(b'abc', b'def'), b'abcdef') self.assertEqual(concat(b'a\0b', b'c\0d'), b'a\0bc\0d') @@ -182,11 +185,11 @@ def test_concat(self, concat=None): def test_concatanddel(self): """Test PyBytes_ConcatAndDel()""" - self.test_concat(_testcapi.bytes_concatanddel) + self.test_concat(_testlimitedcapi.bytes_concatanddel) def test_decodeescape(self): """Test PyBytes_DecodeEscape()""" - decodeescape = _testcapi.bytes_decodeescape + decodeescape = _testlimitedcapi.bytes_decodeescape self.assertEqual(decodeescape(b'abc'), b'abc') self.assertEqual(decodeescape(br'\t\n\r\x0b\x0c\x00\\\'\"'), @@ -217,6 +220,75 @@ def test_decodeescape(self): # CRASHES decodeescape(b'abc', NULL, -1) # CRASHES decodeescape(NULL, NULL, 1) + def test_resize(self): + """Test _PyBytes_Resize()""" + resize = _testcapi.bytes_resize + + for new in True, False: + self.assertEqual(resize(b'abc', 0, new), b'') + self.assertEqual(resize(b'abc', 1, new), b'a') + self.assertEqual(resize(b'abc', 2, new), b'ab') + self.assertEqual(resize(b'abc', 3, new), b'abc') + b = resize(b'abc', 4, new) + self.assertEqual(len(b), 4) + self.assertEqual(b[:3], b'abc') + + self.assertEqual(resize(b'a', 0, new), b'') + self.assertEqual(resize(b'a', 1, new), b'a') + b = resize(b'a', 2, new) + self.assertEqual(len(b), 2) + self.assertEqual(b[:1], b'a') + + self.assertEqual(resize(b'', 0, new), b'') + self.assertEqual(len(resize(b'', 1, new)), 1) + self.assertEqual(len(resize(b'', 2, new)), 2) + + self.assertRaises(SystemError, resize, b'abc', -1, False) + self.assertRaises(SystemError, resize, bytearray(b'abc'), 3, False) + + # CRASHES resize(NULL, 0, False) + # CRASHES resize(NULL, 3, False) + + def test_join(self): + """Test PyBytes_Join()""" + bytes_join = _testcapi.bytes_join + + self.assertEqual(bytes_join(b'', []), b'') + self.assertEqual(bytes_join(b'sep', []), b'') + + self.assertEqual(bytes_join(b'', [b'a', b'b', b'c']), b'abc') + self.assertEqual(bytes_join(b'-', [b'a', b'b', b'c']), b'a-b-c') + self.assertEqual(bytes_join(b' - ', [b'a', b'b', b'c']), b'a - b - c') + self.assertEqual(bytes_join(b'-', [bytearray(b'abc'), + memoryview(b'def')]), + b'abc-def') + + self.assertEqual(bytes_join(b'-', iter([b'a', b'b', b'c'])), b'a-b-c') + + # invalid 'sep' argument + with self.assertRaises(TypeError): + bytes_join(bytearray(b'sep'), []) + with self.assertRaises(TypeError): + bytes_join(memoryview(b'sep'), []) + with self.assertRaises(TypeError): + bytes_join('', []) # empty Unicode string + with self.assertRaises(TypeError): + bytes_join('unicode', []) + with self.assertRaises(TypeError): + bytes_join(123, []) + with self.assertRaises(SystemError): + self.assertEqual(bytes_join(NULL, [b'a', b'b', b'c']), b'abc') + + # invalid 'iterable' argument + with self.assertRaises(TypeError): + bytes_join(b'', [b'bytes', 'unicode']) + with self.assertRaises(TypeError): + bytes_join(b'', [b'bytes', 123]) + with self.assertRaises(TypeError): + bytes_join(b'', 123) + with self.assertRaises(SystemError): + bytes_join(b'', NULL) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py index 682c56979c6dfa..a0355c7a388c57 100644 --- a/Lib/test/test_capi/test_codecs.py +++ b/Lib/test/test_capi/test_codecs.py @@ -1,13 +1,20 @@ -import unittest +import codecs +import contextlib +import io +import re import sys +import unittest +import unittest.mock as mock +import _testcapi from test.support import import_helper -_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') NULL = None +BAD_ARGUMENT = re.escape('bad argument type for built-in operation') -class CAPITest(unittest.TestCase): +class CAPIUnicodeTest(unittest.TestCase): # TODO: Test the following functions: # # PyUnicode_BuildEncodingMap @@ -27,7 +34,7 @@ class CAPITest(unittest.TestCase): def test_fromencodedobject(self): """Test PyUnicode_FromEncodedObject()""" - fromencodedobject = _testcapi.unicode_fromencodedobject + fromencodedobject = _testlimitedcapi.unicode_fromencodedobject self.assertEqual(fromencodedobject(b'abc', NULL), 'abc') self.assertEqual(fromencodedobject(b'abc', 'ascii'), 'abc') @@ -52,7 +59,7 @@ def test_fromencodedobject(self): def test_decode(self): """Test PyUnicode_Decode()""" - decode = _testcapi.unicode_decode + decode = _testlimitedcapi.unicode_decode self.assertEqual(decode(b'[\xe2\x82\xac]', 'utf-8'), '[\u20ac]') self.assertEqual(decode(b'[\xa4]', 'iso8859-15'), '[\u20ac]') @@ -70,7 +77,7 @@ def test_decode(self): def test_asencodedstring(self): """Test PyUnicode_AsEncodedString()""" - asencodedstring = _testcapi.unicode_asencodedstring + asencodedstring = _testlimitedcapi.unicode_asencodedstring self.assertEqual(asencodedstring('abc', NULL), b'abc') self.assertEqual(asencodedstring('abc', 'ascii'), b'abc') @@ -93,7 +100,7 @@ def test_asencodedstring(self): def test_decodeutf8(self): """Test PyUnicode_DecodeUTF8()""" - decodeutf8 = _testcapi.unicode_decodeutf8 + decodeutf8 = _testlimitedcapi.unicode_decodeutf8 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: b = s.encode('utf-8') @@ -113,7 +120,7 @@ def test_decodeutf8(self): def test_decodeutf8stateful(self): """Test PyUnicode_DecodeUTF8Stateful()""" - decodeutf8stateful = _testcapi.unicode_decodeutf8stateful + decodeutf8stateful = _testlimitedcapi.unicode_decodeutf8stateful for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: b = s.encode('utf-8') @@ -136,7 +143,7 @@ def test_decodeutf8stateful(self): def test_asutf8string(self): """Test PyUnicode_AsUTF8String()""" - asutf8string = _testcapi.unicode_asutf8string + asutf8string = _testlimitedcapi.unicode_asutf8string for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: self.assertEqual(asutf8string(s), s.encode('utf-8')) @@ -148,7 +155,7 @@ def test_asutf8string(self): def test_decodeutf16(self): """Test PyUnicode_DecodeUTF16()""" - decodeutf16 = _testcapi.unicode_decodeutf16 + decodeutf16 = _testlimitedcapi.unicode_decodeutf16 naturalbyteorder = -1 if sys.byteorder == 'little' else 1 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: @@ -192,7 +199,7 @@ def test_decodeutf16(self): def test_decodeutf16stateful(self): """Test PyUnicode_DecodeUTF16Stateful()""" - decodeutf16stateful = _testcapi.unicode_decodeutf16stateful + decodeutf16stateful = _testlimitedcapi.unicode_decodeutf16stateful naturalbyteorder = -1 if sys.byteorder == 'little' else 1 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: @@ -238,7 +245,7 @@ def test_decodeutf16stateful(self): def test_asutf16string(self): """Test PyUnicode_AsUTF16String()""" - asutf16string = _testcapi.unicode_asutf16string + asutf16string = _testlimitedcapi.unicode_asutf16string for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: self.assertEqual(asutf16string(s), s.encode('utf-16')) @@ -250,7 +257,7 @@ def test_asutf16string(self): def test_decodeutf32(self): """Test PyUnicode_DecodeUTF8()""" - decodeutf32 = _testcapi.unicode_decodeutf32 + decodeutf32 = _testlimitedcapi.unicode_decodeutf32 naturalbyteorder = -1 if sys.byteorder == 'little' else 1 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: @@ -290,7 +297,7 @@ def test_decodeutf32(self): def test_decodeutf32stateful(self): """Test PyUnicode_DecodeUTF32Stateful()""" - decodeutf32stateful = _testcapi.unicode_decodeutf32stateful + decodeutf32stateful = _testlimitedcapi.unicode_decodeutf32stateful naturalbyteorder = -1 if sys.byteorder == 'little' else 1 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: @@ -342,7 +349,7 @@ def test_decodeutf32stateful(self): def test_asutf32string(self): """Test PyUnicode_AsUTF32String()""" - asutf32string = _testcapi.unicode_asutf32string + asutf32string = _testlimitedcapi.unicode_asutf32string for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: self.assertEqual(asutf32string(s), s.encode('utf-32')) @@ -354,7 +361,7 @@ def test_asutf32string(self): def test_decodelatin1(self): """Test PyUnicode_DecodeLatin1()""" - decodelatin1 = _testcapi.unicode_decodelatin1 + decodelatin1 = _testlimitedcapi.unicode_decodelatin1 self.assertEqual(decodelatin1(b'abc'), 'abc') self.assertEqual(decodelatin1(b'abc', 'strict'), 'abc') @@ -365,7 +372,7 @@ def test_decodelatin1(self): def test_aslatin1string(self): """Test PyUnicode_AsLatin1String()""" - aslatin1string = _testcapi.unicode_aslatin1string + aslatin1string = _testlimitedcapi.unicode_aslatin1string self.assertEqual(aslatin1string('abc'), b'abc') self.assertEqual(aslatin1string('\xa1\xa2'), b'\xa1\xa2') @@ -377,7 +384,7 @@ def test_aslatin1string(self): def test_decodeascii(self): """Test PyUnicode_DecodeASCII()""" - decodeascii = _testcapi.unicode_decodeascii + decodeascii = _testlimitedcapi.unicode_decodeascii self.assertEqual(decodeascii(b'abc'), 'abc') self.assertEqual(decodeascii(b'abc', 'strict'), 'abc') @@ -392,7 +399,7 @@ def test_decodeascii(self): def test_asasciistring(self): """Test PyUnicode_AsASCIIString()""" - asasciistring = _testcapi.unicode_asasciistring + asasciistring = _testlimitedcapi.unicode_asasciistring self.assertEqual(asasciistring('abc'), b'abc') @@ -403,7 +410,7 @@ def test_asasciistring(self): def test_decodecharmap(self): """Test PyUnicode_DecodeCharmap()""" - decodecharmap = _testcapi.unicode_decodecharmap + decodecharmap = _testlimitedcapi.unicode_decodecharmap self.assertEqual(decodecharmap(b'\3\0\7', {0: 'a', 3: 'b', 7: 'c'}), 'bac') self.assertEqual(decodecharmap(b'\1\0\2', ['a', 'b', 'c']), 'bac') @@ -426,7 +433,7 @@ def test_decodecharmap(self): def test_ascharmapstring(self): """Test PyUnicode_AsCharmapString()""" - ascharmapstring = _testcapi.unicode_ascharmapstring + ascharmapstring = _testlimitedcapi.unicode_ascharmapstring self.assertEqual(ascharmapstring('abc', {97: 3, 98: 0, 99: 7}), b'\3\0\7') self.assertEqual(ascharmapstring('\xa1\xa2\xa3', {0xa1: 3, 0xa2: 0, 0xa3: 7}), b'\3\0\7') @@ -443,7 +450,7 @@ def test_ascharmapstring(self): def test_decodeunicodeescape(self): """Test PyUnicode_DecodeUnicodeEscape()""" - decodeunicodeescape = _testcapi.unicode_decodeunicodeescape + decodeunicodeescape = _testlimitedcapi.unicode_decodeunicodeescape self.assertEqual(decodeunicodeescape(b'abc'), 'abc') self.assertEqual(decodeunicodeescape(br'\t\n\r\x0b\x0c\x00\\'), '\t\n\r\v\f\0\\') @@ -467,7 +474,7 @@ def test_decodeunicodeescape(self): def test_asunicodeescapestring(self): """Test PyUnicode_AsUnicodeEscapeString()""" - asunicodeescapestring = _testcapi.unicode_asunicodeescapestring + asunicodeescapestring = _testlimitedcapi.unicode_asunicodeescapestring self.assertEqual(asunicodeescapestring('abc'), b'abc') self.assertEqual(asunicodeescapestring('\t\n\r\v\f\0\\'), br'\t\n\r\x0b\x0c\x00\\') @@ -481,7 +488,7 @@ def test_asunicodeescapestring(self): def test_decoderawunicodeescape(self): """Test PyUnicode_DecodeRawUnicodeEscape()""" - decoderawunicodeescape = _testcapi.unicode_decoderawunicodeescape + decoderawunicodeescape = _testlimitedcapi.unicode_decoderawunicodeescape self.assertEqual(decoderawunicodeescape(b'abc'), 'abc') self.assertEqual(decoderawunicodeescape(b'\t\n\r\v\f\0\\'), '\t\n\r\v\f\0\\') @@ -503,7 +510,7 @@ def test_decoderawunicodeescape(self): def test_asrawunicodeescapestring(self): """Test PyUnicode_AsRawUnicodeEscapeString()""" - asrawunicodeescapestring = _testcapi.unicode_asrawunicodeescapestring + asrawunicodeescapestring = _testlimitedcapi.unicode_asrawunicodeescapestring self.assertEqual(asrawunicodeescapestring('abc'), b'abc') self.assertEqual(asrawunicodeescapestring('\t\n\r\v\f\0\\'), b'\t\n\r\v\f\0\\') @@ -516,5 +523,357 @@ def test_asrawunicodeescapestring(self): # CRASHES asrawunicodeescapestring(NULL) +class CAPICodecs(unittest.TestCase): + + def setUp(self): + # Encoding names are normalized internally by converting them + # to lowercase and their hyphens are replaced by underscores. + self.encoding_name = 'test.test_capi.test_codecs.codec_reversed' + # Make sure that our custom codec is not already registered (that + # way we know whether we correctly unregistered the custom codec + # after a test or not). + self.assertRaises(LookupError, codecs.lookup, self.encoding_name) + # create the search function without registering yet + self._create_custom_codec() + + def _create_custom_codec(self): + def codec_encoder(m, errors='strict'): + return (type(m)().join(reversed(m)), len(m)) + + def codec_decoder(c, errors='strict'): + return (type(c)().join(reversed(c)), len(c)) + + class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input, final=False): + return codec_encoder(input) + + class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input, final=False): + return codec_decoder(input) + + class StreamReader(codecs.StreamReader): + def encode(self, input, errors='strict'): + return codec_encoder(input, errors=errors) + + def decode(self, input, errors='strict'): + return codec_decoder(input, errors=errors) + + class StreamWriter(codecs.StreamWriter): + def encode(self, input, errors='strict'): + return codec_encoder(input, errors=errors) + + def decode(self, input, errors='strict'): + return codec_decoder(input, errors=errors) + + info = codecs.CodecInfo( + encode=codec_encoder, + decode=codec_decoder, + streamreader=StreamReader, + streamwriter=StreamWriter, + incrementalencoder=IncrementalEncoder, + incrementaldecoder=IncrementalDecoder, + name=self.encoding_name + ) + + def search_function(encoding): + if encoding == self.encoding_name: + return info + return None + + self.codec_info = info + self.search_function = search_function + + @contextlib.contextmanager + def use_custom_encoder(self): + self.assertRaises(LookupError, codecs.lookup, self.encoding_name) + codecs.register(self.search_function) + yield + codecs.unregister(self.search_function) + self.assertRaises(LookupError, codecs.lookup, self.encoding_name) + + def test_codec_register(self): + search_function, encoding = self.search_function, self.encoding_name + # register the search function using the C API + self.assertIsNone(_testcapi.codec_register(search_function)) + # in case the test failed before cleaning up + self.addCleanup(codecs.unregister, self.search_function) + self.assertIs(codecs.lookup(encoding), search_function(encoding)) + self.assertEqual(codecs.encode('123', encoding=encoding), '321') + # unregister the search function using the regular API + codecs.unregister(search_function) + self.assertRaises(LookupError, codecs.lookup, encoding) + + def test_codec_unregister(self): + search_function, encoding = self.search_function, self.encoding_name + self.assertRaises(LookupError, codecs.lookup, encoding) + # register the search function using the regular API + codecs.register(search_function) + # in case the test failed before cleaning up + self.addCleanup(codecs.unregister, self.search_function) + self.assertIsNotNone(codecs.lookup(encoding)) + # unregister the search function using the C API + self.assertIsNone(_testcapi.codec_unregister(search_function)) + self.assertRaises(LookupError, codecs.lookup, encoding) + + def test_codec_known_encoding(self): + self.assertRaises(LookupError, codecs.lookup, 'unknown-codec') + self.assertFalse(_testcapi.codec_known_encoding('unknown-codec')) + self.assertFalse(_testcapi.codec_known_encoding('unknown_codec')) + self.assertFalse(_testcapi.codec_known_encoding('UNKNOWN-codec')) + + encoding_name = self.encoding_name + self.assertRaises(LookupError, codecs.lookup, encoding_name) + + codecs.register(self.search_function) + self.addCleanup(codecs.unregister, self.search_function) + + for name in [ + encoding_name, + encoding_name.upper(), + encoding_name.replace('_', '-'), + ]: + with self.subTest(name): + self.assertTrue(_testcapi.codec_known_encoding(name)) + + def test_codec_encode(self): + encode = _testcapi.codec_encode + self.assertEqual(encode('a', 'utf-8', NULL), b'a') + self.assertEqual(encode('a', 'utf-8', 'strict'), b'a') + self.assertEqual(encode('[é]', 'ascii', 'ignore'), b'[]') + + self.assertRaises(TypeError, encode, NULL, 'ascii', 'strict') + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + encode('a', NULL, 'strict') + + def test_codec_decode(self): + decode = _testcapi.codec_decode + + s = 'a\xa1\u4f60\U0001f600' + b = s.encode() + + self.assertEqual(decode(b, 'utf-8', 'strict'), s) + self.assertEqual(decode(b, 'utf-8', NULL), s) + self.assertEqual(decode(b, 'latin1', 'strict'), b.decode('latin1')) + self.assertRaises(UnicodeDecodeError, decode, b, 'ascii', 'strict') + self.assertRaises(UnicodeDecodeError, decode, b, 'ascii', NULL) + self.assertEqual(decode(b, 'ascii', 'replace'), 'a' + '\ufffd'*9) + + # _codecs.decode() only reports an unknown error handling name when + # the corresponding error handling function is used; this difers + # from PyUnicode_Decode() which checks that both the encoding and + # the error handling name are recognized before even attempting to + # call the decoder. + self.assertEqual(decode(b'', 'utf-8', 'unknown-error-handler'), '') + self.assertEqual(decode(b'a', 'utf-8', 'unknown-error-handler'), 'a') + + self.assertRaises(TypeError, decode, NULL, 'ascii', 'strict') + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + decode(b, NULL, 'strict') + + def test_codec_encoder(self): + codec_encoder = _testcapi.codec_encoder + + with self.use_custom_encoder(): + encoder = codec_encoder(self.encoding_name) + self.assertIs(encoder, self.codec_info.encode) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_encoder(NULL) + + def test_codec_decoder(self): + codec_decoder = _testcapi.codec_decoder + + with self.use_custom_encoder(): + decoder = codec_decoder(self.encoding_name) + self.assertIs(decoder, self.codec_info.decode) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_decoder(NULL) + + def test_codec_incremental_encoder(self): + codec_incremental_encoder = _testcapi.codec_incremental_encoder + + with self.use_custom_encoder(): + encoding = self.encoding_name + + for errors in ['strict', NULL]: + with self.subTest(errors): + encoder = codec_incremental_encoder(encoding, errors) + self.assertIsInstance(encoder, self.codec_info.incrementalencoder) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_incremental_encoder(NULL, 'strict') + + def test_codec_incremental_decoder(self): + codec_incremental_decoder = _testcapi.codec_incremental_decoder + + with self.use_custom_encoder(): + encoding = self.encoding_name + + for errors in ['strict', NULL]: + with self.subTest(errors): + decoder = codec_incremental_decoder(encoding, errors) + self.assertIsInstance(decoder, self.codec_info.incrementaldecoder) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_incremental_decoder(NULL, 'strict') + + def test_codec_stream_reader(self): + codec_stream_reader = _testcapi.codec_stream_reader + + with self.use_custom_encoder(): + encoding, stream = self.encoding_name, io.StringIO() + for errors in ['strict', NULL]: + with self.subTest(errors): + writer = codec_stream_reader(encoding, stream, errors) + self.assertIsInstance(writer, self.codec_info.streamreader) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_stream_reader(NULL, stream, 'strict') + + def test_codec_stream_writer(self): + codec_stream_writer = _testcapi.codec_stream_writer + + with self.use_custom_encoder(): + encoding, stream = self.encoding_name, io.StringIO() + for errors in ['strict', NULL]: + with self.subTest(errors): + writer = codec_stream_writer(encoding, stream, errors) + self.assertIsInstance(writer, self.codec_info.streamwriter) + + with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): + codec_stream_writer(NULL, stream, 'strict') + + +class CAPICodecErrors(unittest.TestCase): + + @classmethod + def _generate_exception_args(cls): + for objlen in range(5): + maxind = 2 * max(2, objlen) + for start in range(-maxind, maxind + 1): + for end in range(-maxind, maxind + 1): + yield objlen, start, end + + @classmethod + def generate_encode_errors(cls): + return tuple( + UnicodeEncodeError('utf-8', '0' * objlen, start, end, 'why') + for objlen, start, end in cls._generate_exception_args() + ) + + @classmethod + def generate_decode_errors(cls): + return tuple( + UnicodeDecodeError('utf-8', b'0' * objlen, start, end, 'why') + for objlen, start, end in cls._generate_exception_args() + ) + + @classmethod + def generate_translate_errors(cls): + return tuple( + UnicodeTranslateError('0' * objlen, start, end, 'why') + for objlen, start, end in cls._generate_exception_args() + ) + + @classmethod + def setUpClass(cls): + cls.unicode_encode_errors = cls.generate_encode_errors() + cls.unicode_decode_errors = cls.generate_decode_errors() + cls.unicode_translate_errors = cls.generate_translate_errors() + cls.all_unicode_errors = ( + cls.unicode_encode_errors + + cls.unicode_decode_errors + + cls.unicode_translate_errors + ) + cls.bad_unicode_errors = ( + ValueError(), + ) + + def test_codec_register_error(self): + # for cleaning up between tests + from _codecs import _unregister_error as _codecs_unregister_error + + self.assertRaises(LookupError, _testcapi.codec_lookup_error, 'custom') + + def custom_error_handler(exc): + raise exc + + error_handler = mock.Mock(wraps=custom_error_handler) + _testcapi.codec_register_error('custom', error_handler) + self.addCleanup(_codecs_unregister_error, 'custom') + + self.assertRaises(UnicodeEncodeError, codecs.encode, + '\xff', 'ascii', errors='custom') + error_handler.assert_called_once() + error_handler.reset_mock() + + self.assertRaises(UnicodeDecodeError, codecs.decode, + b'\xff', 'ascii', errors='custom') + error_handler.assert_called_once() + + # _codecs._unregister_error directly delegates to the internal C + # function so a Python-level function test is sufficient (it is + # tested in test_codeccallbacks). + + def test_codec_lookup_error(self): + codec_lookup_error = _testcapi.codec_lookup_error + self.assertIs(codec_lookup_error(NULL), codecs.strict_errors) + self.assertIs(codec_lookup_error('strict'), codecs.strict_errors) + self.assertIs(codec_lookup_error('ignore'), codecs.ignore_errors) + self.assertIs(codec_lookup_error('replace'), codecs.replace_errors) + self.assertIs(codec_lookup_error('xmlcharrefreplace'), codecs.xmlcharrefreplace_errors) + self.assertIs(codec_lookup_error('backslashreplace'), codecs.backslashreplace_errors) + self.assertIs(codec_lookup_error('namereplace'), codecs.namereplace_errors) + self.assertRaises(LookupError, codec_lookup_error, 'unknown') + + def test_codec_strict_errors_handler(self): + handler = _testcapi.codec_strict_errors + for exc in self.all_unicode_errors + self.bad_unicode_errors: + with self.subTest(handler=handler, exc=exc): + self.assertRaises(type(exc), handler, exc) + + def test_codec_ignore_errors_handler(self): + handler = _testcapi.codec_ignore_errors + self.do_test_codec_errors_handler(handler, self.all_unicode_errors) + + def test_codec_replace_errors_handler(self): + handler = _testcapi.codec_replace_errors + self.do_test_codec_errors_handler(handler, self.all_unicode_errors) + + def test_codec_xmlcharrefreplace_errors_handler(self): + handler = _testcapi.codec_xmlcharrefreplace_errors + self.do_test_codec_errors_handler(handler, self.unicode_encode_errors) + + def test_codec_backslashreplace_errors_handler(self): + handler = _testcapi.codec_backslashreplace_errors + self.do_test_codec_errors_handler(handler, self.all_unicode_errors) + + def test_codec_namereplace_errors_handler(self): + handler = _testlimitedcapi.codec_namereplace_errors + self.do_test_codec_errors_handler(handler, self.unicode_encode_errors) + + def do_test_codec_errors_handler(self, handler, exceptions): + self.assertNotEqual(len(exceptions), 0) + for exc in exceptions: + with self.subTest(handler=handler, exc=exc): + # test that the handler does not crash + res = handler(exc) + self.assertIsInstance(res, tuple) + self.assertEqual(len(res), 2) + replacement, continue_from = res + self.assertIsInstance(replacement, str) + self.assertIsInstance(continue_from, int) + self.assertGreaterEqual(continue_from, 0) + self.assertLessEqual(continue_from, len(exc.object)) + + for bad_exc in ( + self.bad_unicode_errors + + tuple(e for e in self.all_unicode_errors if e not in exceptions) + ): + with self.subTest('bad type', handler=handler, exc=bad_exc): + self.assertRaises(TypeError, handler, bad_exc) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_complex.py b/Lib/test/test_capi/test_complex.py index a5b59558e7f851..97e0eb3f043080 100644 --- a/Lib/test/test_capi/test_complex.py +++ b/Lib/test/test_capi/test_complex.py @@ -7,9 +7,11 @@ FloatSubclass, Float, BadFloat, BadFloat2, ComplexSubclass) from test.support import import_helper +from test.support.testcase import ComplexesAreIdenticalMixin _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') NULL = None INF = float("inf") @@ -22,10 +24,10 @@ def __complex__(self): raise RuntimeError -class CAPIComplexTest(unittest.TestCase): +class CAPIComplexTest(ComplexesAreIdenticalMixin, unittest.TestCase): def test_check(self): # Test PyComplex_Check() - check = _testcapi.complex_check + check = _testlimitedcapi.complex_check self.assertTrue(check(1+2j)) self.assertTrue(check(ComplexSubclass(1+2j))) @@ -38,7 +40,7 @@ def test_check(self): def test_checkexact(self): # PyComplex_CheckExact() - checkexact = _testcapi.complex_checkexact + checkexact = _testlimitedcapi.complex_checkexact self.assertTrue(checkexact(1+2j)) self.assertFalse(checkexact(ComplexSubclass(1+2j))) @@ -57,13 +59,13 @@ def test_fromccomplex(self): def test_fromdoubles(self): # Test PyComplex_FromDoubles() - fromdoubles = _testcapi.complex_fromdoubles + fromdoubles = _testlimitedcapi.complex_fromdoubles self.assertEqual(fromdoubles(1.0, 2.0), 1.0+2.0j) def test_realasdouble(self): # Test PyComplex_RealAsDouble() - realasdouble = _testcapi.complex_realasdouble + realasdouble = _testlimitedcapi.complex_realasdouble self.assertEqual(realasdouble(1+2j), 1.0) self.assertEqual(realasdouble(-1+0j), -1.0) @@ -98,7 +100,7 @@ def test_realasdouble(self): def test_imagasdouble(self): # Test PyComplex_ImagAsDouble() - imagasdouble = _testcapi.complex_imagasdouble + imagasdouble = _testlimitedcapi.complex_imagasdouble self.assertEqual(imagasdouble(1+2j), 2.0) self.assertEqual(imagasdouble(1-1j), -1.0) @@ -170,12 +172,33 @@ def test_py_c_sum(self): self.assertEqual(_py_c_sum(1, 1j), (1+1j, 0)) + def test_py_cr_sum(self): + # Test _Py_cr_sum() + _py_cr_sum = _testcapi._py_cr_sum + + self.assertComplexesAreIdentical(_py_cr_sum(-0j, -0.0)[0], + complex(-0.0, -0.0)) + def test_py_c_diff(self): # Test _Py_c_diff() _py_c_diff = _testcapi._py_c_diff self.assertEqual(_py_c_diff(1, 1j), (1-1j, 0)) + def test_py_cr_diff(self): + # Test _Py_cr_diff() + _py_cr_diff = _testcapi._py_cr_diff + + self.assertComplexesAreIdentical(_py_cr_diff(-0j, 0.0)[0], + complex(-0.0, -0.0)) + + def test_py_rc_diff(self): + # Test _Py_rc_diff() + _py_rc_diff = _testcapi._py_rc_diff + + self.assertComplexesAreIdentical(_py_rc_diff(-0.0, 0j)[0], + complex(-0.0, -0.0)) + def test_py_c_neg(self): # Test _Py_c_neg() _py_c_neg = _testcapi._py_c_neg @@ -188,6 +211,13 @@ def test_py_c_prod(self): self.assertEqual(_py_c_prod(2, 1j), (2j, 0)) + def test_py_cr_prod(self): + # Test _Py_cr_prod() + _py_cr_prod = _testcapi._py_cr_prod + + self.assertComplexesAreIdentical(_py_cr_prod(complex('inf+1j'), INF)[0], + complex('inf+infj')) + def test_py_c_quot(self): # Test _Py_c_quot() _py_c_quot = _testcapi._py_c_quot @@ -210,6 +240,20 @@ def test_py_c_quot(self): self.assertEqual(_py_c_quot(1, 0j)[1], errno.EDOM) + def test_py_cr_quot(self): + # Test _Py_cr_quot() + _py_cr_quot = _testcapi._py_cr_quot + + self.assertComplexesAreIdentical(_py_cr_quot(complex('inf+1j'), 2**1000)[0], + INF + 2**-1000*1j) + + def test_py_rc_quot(self): + # Test _Py_rc_quot() + _py_rc_quot = _testcapi._py_rc_quot + + self.assertComplexesAreIdentical(_py_rc_quot(1.0, complex('nan-infj'))[0], + 0j) + def test_py_c_pow(self): # Test _Py_c_pow() _py_c_pow = _testcapi._py_c_pow @@ -225,7 +269,11 @@ def test_py_c_pow(self): self.assertEqual(_py_c_pow(0j, -1)[1], errno.EDOM) self.assertEqual(_py_c_pow(0j, 1j)[1], errno.EDOM) - self.assertEqual(_py_c_pow(*[DBL_MAX+1j]*2)[0], complex(*[INF]*2)) + max_num = DBL_MAX+1j + self.assertEqual(_py_c_pow(max_num, max_num), + (complex(INF, INF), errno.ERANGE)) + self.assertEqual(_py_c_pow(max_num, 2), + (complex(INF, INF), errno.ERANGE)) def test_py_c_abs(self): diff --git a/Lib/test/test_capi/test_config.py b/Lib/test/test_capi/test_config.py new file mode 100644 index 00000000000000..a3179efe4a8235 --- /dev/null +++ b/Lib/test/test_capi/test_config.py @@ -0,0 +1,384 @@ +""" +Tests PyConfig_Get() and PyConfig_Set() C API (PEP 741). +""" +import os +import sys +import sysconfig +import types +import unittest +from test import support +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') + + +# Is the Py_STATS macro defined? +Py_STATS = hasattr(sys, '_stats_on') + + +class CAPITests(unittest.TestCase): + def test_config_get(self): + # Test PyConfig_Get() + config_get = _testcapi.config_get + config_names = _testcapi.config_names + + TEST_VALUE = { + str: "TEST_MARKER_STR", + str | None: "TEST_MARKER_OPT_STR", + list[str]: ("TEST_MARKER_STR_TUPLE",), + dict[str, str | bool]: {"x": "value", "y": True}, + } + + # read config options and check their type + options = [ + ("allocator", int, None), + ("argv", list[str], "argv"), + ("base_exec_prefix", str | None, "base_exec_prefix"), + ("base_executable", str | None, "_base_executable"), + ("base_prefix", str | None, "base_prefix"), + ("buffered_stdio", bool, None), + ("bytes_warning", int, None), + ("check_hash_pycs_mode", str, None), + ("code_debug_ranges", bool, None), + ("configure_c_stdio", bool, None), + ("coerce_c_locale", bool, None), + ("coerce_c_locale_warn", bool, None), + ("configure_locale", bool, None), + ("cpu_count", int, None), + ("dev_mode", bool, None), + ("dump_refs", bool, None), + ("dump_refs_file", str | None, None), + ("exec_prefix", str | None, "exec_prefix"), + ("executable", str | None, "executable"), + ("faulthandler", bool, None), + ("filesystem_encoding", str, None), + ("filesystem_errors", str, None), + ("hash_seed", int, None), + ("home", str | None, None), + ("import_time", bool, None), + ("inspect", bool, None), + ("install_signal_handlers", bool, None), + ("int_max_str_digits", int, None), + ("interactive", bool, None), + ("isolated", bool, None), + ("malloc_stats", bool, None), + ("module_search_paths", list[str], "path"), + ("optimization_level", int, None), + ("orig_argv", list[str], "orig_argv"), + ("parser_debug", bool, None), + ("parse_argv", bool, None), + ("pathconfig_warnings", bool, None), + ("perf_profiling", int, None), + ("platlibdir", str, "platlibdir"), + ("prefix", str | None, "prefix"), + ("program_name", str, None), + ("pycache_prefix", str | None, "pycache_prefix"), + ("quiet", bool, None), + ("run_command", str | None, None), + ("run_filename", str | None, None), + ("run_module", str | None, None), + ("safe_path", bool, None), + ("show_ref_count", bool, None), + ("site_import", bool, None), + ("skip_source_first_line", bool, None), + ("stdio_encoding", str, None), + ("stdio_errors", str, None), + ("stdlib_dir", str | None, "_stdlib_dir"), + ("tracemalloc", int, None), + ("use_environment", bool, None), + ("use_frozen_modules", bool, None), + ("use_hash_seed", bool, None), + ("user_site_directory", bool, None), + ("utf8_mode", bool, None), + ("verbose", int, None), + ("warn_default_encoding", bool, None), + ("warnoptions", list[str], "warnoptions"), + ("write_bytecode", bool, None), + ("xoptions", dict[str, str | bool], "_xoptions"), + ] + if support.Py_DEBUG: + options.append(("run_presite", str | None, None)) + if sysconfig.get_config_var('Py_GIL_DISABLED'): + options.append(("enable_gil", int, None)) + options.append(("tlbc_enabled", int, None)) + if support.MS_WINDOWS: + options.extend(( + ("legacy_windows_stdio", bool, None), + ("legacy_windows_fs_encoding", bool, None), + )) + if Py_STATS: + options.extend(( + ("_pystats", bool, None), + )) + if support.is_apple: + options.extend(( + ("use_system_logger", bool, None), + )) + + for name, option_type, sys_attr in options: + with self.subTest(name=name, option_type=option_type, + sys_attr=sys_attr): + value = config_get(name) + if isinstance(option_type, types.GenericAlias): + self.assertIsInstance(value, option_type.__origin__) + if option_type.__origin__ == dict: + key_type = option_type.__args__[0] + value_type = option_type.__args__[1] + for item in value.items(): + self.assertIsInstance(item[0], key_type) + self.assertIsInstance(item[1], value_type) + else: + item_type = option_type.__args__[0] + for item in value: + self.assertIsInstance(item, item_type) + else: + self.assertIsInstance(value, option_type) + + if sys_attr is not None: + expected = getattr(sys, sys_attr) + self.assertEqual(expected, value) + + override = TEST_VALUE[option_type] + with support.swap_attr(sys, sys_attr, override): + self.assertEqual(config_get(name), override) + + # check that the test checks all options + self.assertEqual(sorted(name for name, option_type, sys_attr in options), + sorted(config_names())) + + def test_config_get_sys_flags(self): + # Test PyConfig_Get() + config_get = _testcapi.config_get + + # compare config options with sys.flags + for flag, name, negate in ( + ("debug", "parser_debug", False), + ("inspect", "inspect", False), + ("interactive", "interactive", False), + ("optimize", "optimization_level", False), + ("dont_write_bytecode", "write_bytecode", True), + ("no_user_site", "user_site_directory", True), + ("no_site", "site_import", True), + ("ignore_environment", "use_environment", True), + ("verbose", "verbose", False), + ("bytes_warning", "bytes_warning", False), + ("quiet", "quiet", False), + # "hash_randomization" is tested below + ("isolated", "isolated", False), + ("dev_mode", "dev_mode", False), + ("utf8_mode", "utf8_mode", False), + ("warn_default_encoding", "warn_default_encoding", False), + ("safe_path", "safe_path", False), + ("int_max_str_digits", "int_max_str_digits", False), + # "gil" is tested below + ): + with self.subTest(flag=flag, name=name, negate=negate): + value = config_get(name) + if negate: + value = not value + self.assertEqual(getattr(sys.flags, flag), value) + + self.assertEqual(sys.flags.hash_randomization, + config_get('use_hash_seed') == 0 + or config_get('hash_seed') != 0) + + if sysconfig.get_config_var('Py_GIL_DISABLED'): + value = config_get('enable_gil') + expected = (value if value != -1 else None) + self.assertEqual(sys.flags.gil, expected) + + def test_config_get_non_existent(self): + # Test PyConfig_Get() on non-existent option name + config_get = _testcapi.config_get + nonexistent_key = 'NONEXISTENT_KEY' + err_msg = f'unknown config option name: {nonexistent_key}' + with self.assertRaisesRegex(ValueError, err_msg): + config_get(nonexistent_key) + + def test_config_get_write_bytecode(self): + # PyConfig_Get("write_bytecode") gets sys.dont_write_bytecode + # as an integer + config_get = _testcapi.config_get + with support.swap_attr(sys, "dont_write_bytecode", 0): + self.assertEqual(config_get('write_bytecode'), 1) + with support.swap_attr(sys, "dont_write_bytecode", "yes"): + self.assertEqual(config_get('write_bytecode'), 0) + with support.swap_attr(sys, "dont_write_bytecode", []): + self.assertEqual(config_get('write_bytecode'), 1) + + def test_config_getint(self): + # Test PyConfig_GetInt() + config_getint = _testcapi.config_getint + + # PyConfig_MEMBER_INT type + self.assertEqual(config_getint('verbose'), sys.flags.verbose) + + # PyConfig_MEMBER_UINT type + self.assertEqual(config_getint('isolated'), sys.flags.isolated) + + # PyConfig_MEMBER_ULONG type + self.assertIsInstance(config_getint('hash_seed'), int) + + # PyPreConfig member + self.assertIsInstance(config_getint('allocator'), int) + + # platlibdir type is str + with self.assertRaises(TypeError): + config_getint('platlibdir') + + def test_get_config_names(self): + names = _testcapi.config_names() + self.assertIsInstance(names, frozenset) + for name in names: + self.assertIsInstance(name, str) + + def test_config_set_sys_attr(self): + # Test PyConfig_Set() with sys attributes + config_get = _testcapi.config_get + config_set = _testcapi.config_set + + # mutable configuration option mapped to sys attributes + for name, sys_attr, option_type in ( + ('argv', 'argv', list[str]), + ('base_exec_prefix', 'base_exec_prefix', str | None), + ('base_executable', '_base_executable', str | None), + ('base_prefix', 'base_prefix', str | None), + ('exec_prefix', 'exec_prefix', str | None), + ('executable', 'executable', str | None), + ('module_search_paths', 'path', list[str]), + ('platlibdir', 'platlibdir', str), + ('prefix', 'prefix', str | None), + ('pycache_prefix', 'pycache_prefix', str | None), + ('stdlib_dir', '_stdlib_dir', str | None), + ('warnoptions', 'warnoptions', list[str]), + ('xoptions', '_xoptions', dict[str, str | bool]), + ): + with self.subTest(name=name): + if option_type == str: + test_values = ('TEST_REPLACE',) + invalid_types = (1, None) + elif option_type == str | None: + test_values = ('TEST_REPLACE', None) + invalid_types = (123,) + elif option_type == list[str]: + test_values = (['TEST_REPLACE'], []) + invalid_types = ('text', 123, [123]) + else: # option_type == dict[str, str | bool]: + test_values = ({"x": "value", "y": True},) + invalid_types = ('text', 123, ['option'], + {123: 'value'}, + {'key': b'bytes'}) + + old_opt_value = config_get(name) + old_sys_value = getattr(sys, sys_attr) + try: + for value in test_values: + config_set(name, value) + self.assertEqual(config_get(name), value) + self.assertEqual(getattr(sys, sys_attr), value) + + for value in invalid_types: + with self.assertRaises(TypeError): + config_set(name, value) + finally: + setattr(sys, sys_attr, old_sys_value) + config_set(name, old_opt_value) + + def test_config_set_sys_flag(self): + # Test PyConfig_Set() with sys.flags + config_get = _testcapi.config_get + config_set = _testcapi.config_set + + # mutable configuration option mapped to sys.flags + class unsigned_int(int): + pass + + def expect_int(value): + value = int(value) + return (value, value) + + def expect_bool(value): + value = int(bool(value)) + return (value, value) + + def expect_bool_not(value): + value = bool(value) + return (int(value), int(not value)) + + for name, sys_flag, option_type, expect_func in ( + # (some flags cannot be set, see comments below.) + ('parser_debug', 'debug', bool, expect_bool), + ('inspect', 'inspect', bool, expect_bool), + ('interactive', 'interactive', bool, expect_bool), + ('optimization_level', 'optimize', unsigned_int, expect_int), + ('write_bytecode', 'dont_write_bytecode', bool, expect_bool_not), + # user_site_directory + # site_import + ('use_environment', 'ignore_environment', bool, expect_bool_not), + ('verbose', 'verbose', unsigned_int, expect_int), + ('bytes_warning', 'bytes_warning', unsigned_int, expect_int), + ('quiet', 'quiet', bool, expect_bool), + # hash_randomization + # isolated + # dev_mode + # utf8_mode + # warn_default_encoding + # safe_path + ('int_max_str_digits', 'int_max_str_digits', unsigned_int, expect_int), + # gil + ): + if name == "int_max_str_digits": + new_values = (0, 5_000, 999_999) + invalid_values = (-1, 40) # value must 0 or >= 4300 + invalid_types = (1.0, "abc") + elif option_type == int: + new_values = (False, True, 0, 1, 5, -5) + invalid_values = () + invalid_types = (1.0, "abc") + else: + new_values = (False, True, 0, 1, 5) + invalid_values = (-5,) + invalid_types = (1.0, "abc") + + with self.subTest(name=name): + old_value = config_get(name) + try: + for value in new_values: + expected, expect_flag = expect_func(value) + + config_set(name, value) + self.assertEqual(config_get(name), expected) + self.assertEqual(getattr(sys.flags, sys_flag), expect_flag) + if name == "write_bytecode": + self.assertEqual(getattr(sys, "dont_write_bytecode"), + expect_flag) + if name == "int_max_str_digits": + self.assertEqual(sys.get_int_max_str_digits(), + expect_flag) + + for value in invalid_values: + with self.assertRaises(ValueError): + config_set(name, value) + + for value in invalid_types: + with self.assertRaises(TypeError): + config_set(name, value) + finally: + config_set(name, old_value) + + def test_config_set_read_only(self): + # Test PyConfig_Set() on read-only options + config_set = _testcapi.config_set + for name, value in ( + ("allocator", 0), # PyPreConfig member + ("cpu_count", 8), + ("dev_mode", True), + ("filesystem_encoding", "utf-8"), + ): + with self.subTest(name=name, value=value): + with self.assertRaisesRegex(ValueError, r"read-only"): + config_set(name, value) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py index cca6145bc90c04..e726e3d813d888 100644 --- a/Lib/test/test_capi/test_dict.py +++ b/Lib/test/test_capi/test_dict.py @@ -2,7 +2,11 @@ from collections import OrderedDict, UserDict from types import MappingProxyType from test import support -import _testcapi +from test.support import import_helper + + +_testcapi = import_helper.import_module("_testcapi") +_testlimitedcapi = import_helper.import_module("_testlimitedcapi") NULL = None @@ -25,7 +29,7 @@ def gen(): class CAPITest(unittest.TestCase): def test_dict_check(self): - check = _testcapi.dict_check + check = _testlimitedcapi.dict_check self.assertTrue(check({1: 2})) self.assertTrue(check(OrderedDict({1: 2}))) self.assertFalse(check(UserDict({1: 2}))) @@ -34,7 +38,7 @@ def test_dict_check(self): # CRASHES check(NULL) def test_dict_checkexact(self): - check = _testcapi.dict_checkexact + check = _testlimitedcapi.dict_checkexact self.assertTrue(check({1: 2})) self.assertFalse(check(OrderedDict({1: 2}))) self.assertFalse(check(UserDict({1: 2}))) @@ -43,7 +47,7 @@ def test_dict_checkexact(self): # CRASHES check(NULL) def test_dict_new(self): - dict_new = _testcapi.dict_new + dict_new = _testlimitedcapi.dict_new dct = dict_new() self.assertEqual(dct, {}) self.assertIs(type(dct), dict) @@ -51,7 +55,7 @@ def test_dict_new(self): self.assertIsNot(dct2, dct) def test_dictproxy_new(self): - dictproxy_new = _testcapi.dictproxy_new + dictproxy_new = _testlimitedcapi.dictproxy_new for dct in {1: 2}, OrderedDict({1: 2}), UserDict({1: 2}): proxy = dictproxy_new(dct) self.assertIs(type(proxy), MappingProxyType) @@ -67,7 +71,7 @@ def test_dictproxy_new(self): # CRASHES dictproxy_new(NULL) def test_dict_copy(self): - copy = _testcapi.dict_copy + copy = _testlimitedcapi.dict_copy for dct in {1: 2}, OrderedDict({1: 2}): dct_copy = copy(dct) self.assertIs(type(dct_copy), dict) @@ -79,7 +83,7 @@ def test_dict_copy(self): self.assertRaises(SystemError, copy, NULL) def test_dict_clear(self): - clear = _testcapi.dict_clear + clear = _testlimitedcapi.dict_clear dct = {1: 2} clear(dct) self.assertEqual(dct, {}) @@ -98,7 +102,7 @@ def test_dict_clear(self): # CRASHES? clear(NULL) def test_dict_size(self): - size = _testcapi.dict_size + size = _testlimitedcapi.dict_size self.assertEqual(size({1: 2}), 1) self.assertEqual(size(OrderedDict({1: 2})), 1) @@ -109,7 +113,7 @@ def test_dict_size(self): self.assertRaises(SystemError, size, NULL) def test_dict_getitem(self): - getitem = _testcapi.dict_getitem + getitem = _testlimitedcapi.dict_getitem dct = {'a': 1, '\U0001f40d': 2} self.assertEqual(getitem(dct, 'a'), 1) self.assertIs(getitem(dct, 'b'), KeyError) @@ -131,7 +135,7 @@ def test_dict_getitem(self): # CRASHES getitem(NULL, 'a') def test_dict_getitemstring(self): - getitemstring = _testcapi.dict_getitemstring + getitemstring = _testlimitedcapi.dict_getitemstring dct = {'a': 1, '\U0001f40d': 2} self.assertEqual(getitemstring(dct, b'a'), 1) self.assertIs(getitemstring(dct, b'b'), KeyError) @@ -188,7 +192,7 @@ def test_dict_getitemstringref(self): # CRASHES getitemstring(NULL, b'a') def test_dict_getitemwitherror(self): - getitem = _testcapi.dict_getitemwitherror + getitem = _testlimitedcapi.dict_getitemwitherror dct = {'a': 1, '\U0001f40d': 2} self.assertEqual(getitem(dct, 'a'), 1) self.assertIs(getitem(dct, 'b'), KeyError) @@ -206,7 +210,7 @@ def test_dict_getitemwitherror(self): # CRASHES getitem(NULL, 'a') def test_dict_contains(self): - contains = _testcapi.dict_contains + contains = _testlimitedcapi.dict_contains dct = {'a': 1, '\U0001f40d': 2} self.assertTrue(contains(dct, 'a')) self.assertFalse(contains(dct, 'b')) @@ -238,7 +242,7 @@ def test_dict_contains_string(self): # CRASHES contains(NULL, b'a') def test_dict_setitem(self): - setitem = _testcapi.dict_setitem + setitem = _testlimitedcapi.dict_setitem dct = {} setitem(dct, 'a', 5) self.assertEqual(dct, {'a': 5}) @@ -258,7 +262,7 @@ def test_dict_setitem(self): # CRASHES setitem(NULL, 'a', 5) def test_dict_setitemstring(self): - setitemstring = _testcapi.dict_setitemstring + setitemstring = _testlimitedcapi.dict_setitemstring dct = {} setitemstring(dct, b'a', 5) self.assertEqual(dct, {'a': 5}) @@ -277,7 +281,7 @@ def test_dict_setitemstring(self): # CRASHES setitemstring(NULL, b'a', 5) def test_dict_delitem(self): - delitem = _testcapi.dict_delitem + delitem = _testlimitedcapi.dict_delitem dct = {'a': 1, 'c': 2, '\U0001f40d': 3} delitem(dct, 'a') self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) @@ -298,7 +302,7 @@ def test_dict_delitem(self): # CRASHES delitem(NULL, 'a') def test_dict_delitemstring(self): - delitemstring = _testcapi.dict_delitemstring + delitemstring = _testlimitedcapi.dict_delitemstring dct = {'a': 1, 'c': 2, '\U0001f40d': 3} delitemstring(dct, b'a') self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) @@ -371,21 +375,21 @@ def items(self): return None dict_obj = {'foo': 1, 'bar': 2, 'spam': 3} for mapping in [dict_obj, DictSubclass(dict_obj), BadMapping(dict_obj)]: - self.assertListEqual(_testcapi.dict_keys(mapping), + self.assertListEqual(_testlimitedcapi.dict_keys(mapping), list(dict_obj.keys())) - self.assertListEqual(_testcapi.dict_values(mapping), + self.assertListEqual(_testlimitedcapi.dict_values(mapping), list(dict_obj.values())) - self.assertListEqual(_testcapi.dict_items(mapping), + self.assertListEqual(_testlimitedcapi.dict_items(mapping), list(dict_obj.items())) def test_dict_keys_valuesitems_bad_arg(self): for mapping in UserDict(), [], object(): - self.assertRaises(SystemError, _testcapi.dict_keys, mapping) - self.assertRaises(SystemError, _testcapi.dict_values, mapping) - self.assertRaises(SystemError, _testcapi.dict_items, mapping) + self.assertRaises(SystemError, _testlimitedcapi.dict_keys, mapping) + self.assertRaises(SystemError, _testlimitedcapi.dict_values, mapping) + self.assertRaises(SystemError, _testlimitedcapi.dict_items, mapping) def test_dict_next(self): - dict_next = _testcapi.dict_next + dict_next = _testlimitedcapi.dict_next self.assertIsNone(dict_next({}, 0)) dct = {'a': 1, 'b': 2, 'c': 3} pos = 0 @@ -402,7 +406,7 @@ def test_dict_next(self): # CRASHES dict_next(NULL, 0) def test_dict_update(self): - update = _testcapi.dict_update + update = _testlimitedcapi.dict_update for cls1 in dict, DictSubclass: for cls2 in dict, DictSubclass, UserDict: dct = cls1({'a': 1, 'b': 2}) @@ -417,7 +421,7 @@ def test_dict_update(self): self.assertRaises(SystemError, update, NULL, {}) def test_dict_merge(self): - merge = _testcapi.dict_merge + merge = _testlimitedcapi.dict_merge for cls1 in dict, DictSubclass: for cls2 in dict, DictSubclass, UserDict: dct = cls1({'a': 1, 'b': 2}) @@ -435,7 +439,7 @@ def test_dict_merge(self): self.assertRaises(SystemError, merge, NULL, {}, 0) def test_dict_mergefromseq2(self): - mergefromseq2 = _testcapi.dict_mergefromseq2 + mergefromseq2 = _testlimitedcapi.dict_mergefromseq2 for cls1 in dict, DictSubclass: for cls2 in list, iter: dct = cls1({'a': 1, 'b': 2}) diff --git a/Lib/test/test_capi/test_eval.py b/Lib/test/test_capi/test_eval.py new file mode 100644 index 00000000000000..20ef2695ef3e27 --- /dev/null +++ b/Lib/test/test_capi/test_eval.py @@ -0,0 +1,103 @@ +import sys +import unittest +from test.support import import_helper + +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') + + +class Tests(unittest.TestCase): + def test_eval_get_func_name(self): + eval_get_func_name = _testlimitedcapi.eval_get_func_name + + def function_example(): ... + + class A: + def method_example(self): ... + + self.assertEqual(eval_get_func_name(function_example), + "function_example") + self.assertEqual(eval_get_func_name(A.method_example), + "method_example") + self.assertEqual(eval_get_func_name(A().method_example), + "method_example") + self.assertEqual(eval_get_func_name(sum), "sum") # c function + self.assertEqual(eval_get_func_name(A), "type") + + def test_eval_get_func_desc(self): + eval_get_func_desc = _testlimitedcapi.eval_get_func_desc + + def function_example(): ... + + class A: + def method_example(self): ... + + self.assertEqual(eval_get_func_desc(function_example), + "()") + self.assertEqual(eval_get_func_desc(A.method_example), + "()") + self.assertEqual(eval_get_func_desc(A().method_example), + "()") + self.assertEqual(eval_get_func_desc(sum), "()") # c function + self.assertEqual(eval_get_func_desc(A), " object") + + def test_eval_getlocals(self): + # Test PyEval_GetLocals() + x = 1 + self.assertEqual(_testlimitedcapi.eval_getlocals(), + {'self': self, + 'x': 1}) + + y = 2 + self.assertEqual(_testlimitedcapi.eval_getlocals(), + {'self': self, + 'x': 1, + 'y': 2}) + + def test_eval_getglobals(self): + # Test PyEval_GetGlobals() + self.assertEqual(_testlimitedcapi.eval_getglobals(), + globals()) + + def test_eval_getbuiltins(self): + # Test PyEval_GetBuiltins() + self.assertEqual(_testlimitedcapi.eval_getbuiltins(), + globals()['__builtins__']) + + def test_eval_getframe(self): + # Test PyEval_GetFrame() + self.assertEqual(_testlimitedcapi.eval_getframe(), + sys._getframe()) + + def test_eval_getframe_builtins(self): + # Test PyEval_GetFrameBuiltins() + self.assertEqual(_testlimitedcapi.eval_getframe_builtins(), + sys._getframe().f_builtins) + + def test_eval_getframe_globals(self): + # Test PyEval_GetFrameGlobals() + self.assertEqual(_testlimitedcapi.eval_getframe_globals(), + sys._getframe().f_globals) + + def test_eval_getframe_locals(self): + # Test PyEval_GetFrameLocals() + self.assertEqual(_testlimitedcapi.eval_getframe_locals(), + sys._getframe().f_locals) + + def test_eval_get_recursion_limit(self): + # Test Py_GetRecursionLimit() + self.assertEqual(_testlimitedcapi.eval_get_recursion_limit(), + sys.getrecursionlimit()) + + def test_eval_set_recursion_limit(self): + # Test Py_SetRecursionLimit() + old_limit = sys.getrecursionlimit() + try: + limit = old_limit + 123 + _testlimitedcapi.eval_set_recursion_limit(limit) + self.assertEqual(sys.getrecursionlimit(), limit) + finally: + sys.setrecursionlimit(old_limit) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_eval_code_ex.py b/Lib/test/test_capi/test_eval_code_ex.py index 2d28e5289eff94..b298e5007e5e7d 100644 --- a/Lib/test/test_capi/test_eval_code_ex.py +++ b/Lib/test/test_capi/test_eval_code_ex.py @@ -1,11 +1,16 @@ import unittest +import builtins +from collections import UserDict from test.support import import_helper +from test.support import swap_attr # Skip this test if the _testcapi module isn't available. _testcapi = import_helper.import_module('_testcapi') +NULL = None + class PyEval_EvalCodeExTests(unittest.TestCase): @@ -13,43 +18,108 @@ def test_simple(self): def f(): return a - self.assertEqual(_testcapi.eval_code_ex(f.__code__, dict(a=1)), 1) - - # Need to force the compiler to use LOAD_NAME - # def test_custom_locals(self): - # def f(): - # return + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, dict(a=1)), 1) + + self.assertRaises(NameError, eval_code_ex, code, {}) + self.assertRaises(SystemError, eval_code_ex, code, UserDict(a=1)) + self.assertRaises(SystemError, eval_code_ex, code, []) + self.assertRaises(SystemError, eval_code_ex, code, 1) + # CRASHES eval_code_ex(code, NULL) + # CRASHES eval_code_ex(1, {}) + # CRASHES eval_code_ex(NULL, {}) + + def test_custom_locals(self): + # Monkey-patch __build_class__ to get a class code object. + code = None + def build_class(func, name, /, *bases, **kwds): + nonlocal code + code = func.__code__ + + with swap_attr(builtins, '__build_class__', build_class): + class A: + # Uses LOAD_NAME for a + r[:] = [a] + + eval_code_ex = _testcapi.eval_code_ex + results = [] + g = dict(a=1, r=results) + self.assertIsNone(eval_code_ex(code, g)) + self.assertEqual(results, [1]) + self.assertIsNone(eval_code_ex(code, g, dict(a=2))) + self.assertEqual(results, [2]) + self.assertIsNone(eval_code_ex(code, g, UserDict(a=3))) + self.assertEqual(results, [3]) + self.assertIsNone(eval_code_ex(code, g, {})) + self.assertEqual(results, [1]) + self.assertIsNone(eval_code_ex(code, g, NULL)) + self.assertEqual(results, [1]) + + self.assertRaises(TypeError, eval_code_ex, code, g, []) + self.assertRaises(TypeError, eval_code_ex, code, g, 1) + self.assertRaises(NameError, eval_code_ex, code, dict(r=results), {}) + self.assertRaises(NameError, eval_code_ex, code, dict(r=results), NULL) + self.assertRaises(TypeError, eval_code_ex, code, dict(r=results), []) + self.assertRaises(TypeError, eval_code_ex, code, dict(r=results), 1) def test_with_args(self): def f(a, b, c): return a - self.assertEqual(_testcapi.eval_code_ex(f.__code__, {}, {}, (1, 2, 3)), 1) + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, {}, {}, (1, 2, 3)), 1) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (1, 2)) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (1, 2, 3, 4)) def test_with_kwargs(self): def f(a, b, c): return a - self.assertEqual(_testcapi.eval_code_ex(f.__code__, {}, {}, (), dict(a=1, b=2, c=3)), 1) + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, {}, {}, (), dict(a=1, b=2, c=3)), 1) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), dict(a=1, b=2)) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), dict(a=1, b=2)) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), dict(a=1, b=2, c=3, d=4)) def test_with_default(self): def f(a): return a - self.assertEqual(_testcapi.eval_code_ex(f.__code__, {}, {}, (), {}, (1,)), 1) + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, {}, {}, (), {}, (1,)), 1) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), {}, ()) def test_with_kwarg_default(self): def f(*, a): return a - self.assertEqual(_testcapi.eval_code_ex(f.__code__, {}, {}, (), {}, (), dict(a=1)), 1) + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, {}, {}, (), {}, (), dict(a=1)), 1) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), {}, (), {}) + self.assertRaises(TypeError, eval_code_ex, code, {}, {}, (), {}, (), NULL) + self.assertRaises(SystemError, eval_code_ex, code, {}, {}, (), {}, (), UserDict(a=1)) + self.assertRaises(SystemError, eval_code_ex, code, {}, {}, (), {}, (), []) + self.assertRaises(SystemError, eval_code_ex, code, {}, {}, (), {}, (), 1) def test_with_closure(self): a = 1 + b = 2 def f(): + b return a - self.assertEqual(_testcapi.eval_code_ex(f.__code__, {}, {}, (), {}, (), {}, f.__closure__), 1) + eval_code_ex = _testcapi.eval_code_ex + code = f.__code__ + self.assertEqual(eval_code_ex(code, {}, {}, (), {}, (), {}, f.__closure__), 1) + self.assertEqual(eval_code_ex(code, {}, {}, (), {}, (), {}, f.__closure__[::-1]), 2) + + # CRASHES eval_code_ex(code, {}, {}, (), {}, (), {}, ()), 1) + # CRASHES eval_code_ex(code, {}, {}, (), {}, (), {}, NULL), 1) if __name__ == "__main__": diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index 1d158e3586e98d..666e2f2ab09548 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -3,11 +3,12 @@ import re import sys import unittest +import textwrap from test import support from test.support import import_helper from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE -from test.support.script_helper import assert_python_failure +from test.support.script_helper import assert_python_failure, assert_python_ok from test.support.testcase import ExceptionIsLikeMixin from .test_misc import decode_stderr @@ -68,6 +69,47 @@ def test_exc_info(self): else: self.assertTrue(False) + def test_warn_with_stacklevel(self): + code = textwrap.dedent('''\ + import _testcapi + + def foo(): + _testcapi.function_set_warning() + + foo() # line 6 + + + foo() # line 9 + ''') + proc = assert_python_ok("-c", code) + warnings = proc.err.splitlines() + self.assertEqual(warnings, [ + b':6: RuntimeWarning: Testing PyErr_WarnEx', + b' foo() # line 6', + b':9: RuntimeWarning: Testing PyErr_WarnEx', + b' foo() # line 9', + ]) + + def test_warn_during_finalization(self): + code = textwrap.dedent('''\ + import _testcapi + + class Foo: + def foo(self): + _testcapi.function_set_warning() + def __del__(self): + self.foo() + + ref = Foo() + ''') + proc = assert_python_ok("-c", code) + warnings = proc.err.splitlines() + # Due to the finalization of the interpreter, the source will be omitted + # because the ``warnings`` module cannot be imported at this time + self.assertEqual(warnings, [ + b':7: RuntimeWarning: Testing PyErr_WarnEx', + ]) + class Test_FatalError(unittest.TestCase): @@ -373,6 +415,156 @@ def test_err_formatunraisable(self): # CRASHES formatunraisable(NULL, NULL) +class TestUnicodeTranslateError(UnicodeTranslateError): + # UnicodeTranslateError takes 4 arguments instead of 5, + # so we just make a UnicodeTranslateError class that is + # compatible with the UnicodeError.__init__. + def __init__(self, encoding, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class TestUnicodeError(unittest.TestCase): + + def _check_no_crash(self, exc): + # ensure that the __str__() method does not crash + _ = str(exc) + + def test_unicode_encode_error_get_start(self): + get_start = _testcapi.unicode_encode_get_start + self._test_unicode_error_get_start('x', UnicodeEncodeError, get_start) + + def test_unicode_decode_error_get_start(self): + get_start = _testcapi.unicode_decode_get_start + self._test_unicode_error_get_start(b'x', UnicodeDecodeError, get_start) + + def test_unicode_translate_error_get_start(self): + get_start = _testcapi.unicode_translate_get_start + self._test_unicode_error_get_start('x', TestUnicodeTranslateError, get_start) + + def _test_unicode_error_get_start(self, literal, exc_type, get_start): + for obj_len, start, c_start in [ + # normal cases + (5, 0, 0), + (5, 1, 1), + (5, 2, 2), + # out of range start is clamped to max(0, obj_len - 1) + (0, 0, 0), + (0, 1, 0), + (0, 10, 0), + (5, 5, 4), + (5, 10, 4), + # negative values are allowed but clipped in the getter + (0, -1, 0), + (1, -1, 0), + (2, -1, 0), + (2, -2, 0), + ]: + obj = literal * obj_len + with self.subTest(obj, exc_type=exc_type, start=start): + exc = exc_type('utf-8', obj, start, obj_len, 'reason') + self.assertEqual(get_start(exc), c_start) + self._check_no_crash(exc) + + def test_unicode_encode_error_set_start(self): + set_start = _testcapi.unicode_encode_set_start + self._test_unicode_error_set_start('x', UnicodeEncodeError, set_start) + + def test_unicode_decode_error_set_start(self): + set_start = _testcapi.unicode_decode_set_start + self._test_unicode_error_set_start(b'x', UnicodeDecodeError, set_start) + + def test_unicode_translate_error_set_start(self): + set_start = _testcapi.unicode_translate_set_start + self._test_unicode_error_set_start('x', TestUnicodeTranslateError, set_start) + + def _test_unicode_error_set_start(self, literal, exc_type, set_start): + obj_len = 5 + obj = literal * obj_len + for new_start in range(-2 * obj_len, 2 * obj_len): + with self.subTest('C-API', obj=obj, exc_type=exc_type, new_start=new_start): + exc = exc_type('utf-8', obj, 0, obj_len, 'reason') + # arbitrary value is allowed in the C API setter + set_start(exc, new_start) + self.assertEqual(exc.start, new_start) + self._check_no_crash(exc) + + with self.subTest('Py-API', obj=obj, exc_type=exc_type, new_start=new_start): + exc = exc_type('utf-8', obj, 0, obj_len, 'reason') + # arbitrary value is allowed in the attribute setter + exc.start = new_start + self.assertEqual(exc.start, new_start) + self._check_no_crash(exc) + + def test_unicode_encode_error_get_end(self): + get_end = _testcapi.unicode_encode_get_end + self._test_unicode_error_get_end('x', UnicodeEncodeError, get_end) + + def test_unicode_decode_error_get_end(self): + get_end = _testcapi.unicode_decode_get_end + self._test_unicode_error_get_end(b'x', UnicodeDecodeError, get_end) + + def test_unicode_translate_error_get_end(self): + get_end = _testcapi.unicode_translate_get_end + self._test_unicode_error_get_end('x', TestUnicodeTranslateError, get_end) + + def _test_unicode_error_get_end(self, literal, exc_type, get_end): + for obj_len, end, c_end in [ + # normal cases + (5, 0, 1), + (5, 1, 1), + (5, 2, 2), + # out-of-range clipped in [MIN(1, OBJLEN), MAX(MIN(1, OBJLEN), OBJLEN)] + (0, 0, 0), + (0, 1, 0), + (0, 10, 0), + (1, 1, 1), + (1, 2, 1), + (5, 5, 5), + (5, 5, 5), + (5, 10, 5), + # negative values are allowed but clipped in the getter + (0, -1, 0), + (1, -1, 1), + (2, -1, 1), + (2, -2, 1), + ]: + obj = literal * obj_len + with self.subTest(obj, exc_type=exc_type, end=end): + exc = exc_type('utf-8', obj, 0, end, 'reason') + self.assertEqual(get_end(exc), c_end) + self._check_no_crash(exc) + + def test_unicode_encode_error_set_end(self): + set_end = _testcapi.unicode_encode_set_end + self._test_unicode_error_set_end('x', UnicodeEncodeError, set_end) + + def test_unicode_decode_error_set_end(self): + set_end = _testcapi.unicode_decode_set_end + self._test_unicode_error_set_end(b'x', UnicodeDecodeError, set_end) + + def test_unicode_translate_error_set_end(self): + set_end = _testcapi.unicode_translate_set_end + self._test_unicode_error_set_end('x', TestUnicodeTranslateError, set_end) + + def _test_unicode_error_set_end(self, literal, exc_type, set_end): + obj_len = 5 + obj = literal * obj_len + for new_end in range(-2 * obj_len, 2 * obj_len): + with self.subTest('C-API', obj=obj, exc_type=exc_type, new_end=new_end): + exc = exc_type('utf-8', obj, 0, obj_len, 'reason') + # arbitrary value is allowed in the C API setter + set_end(exc, new_end) + self.assertEqual(exc.end, new_end) + self._check_no_crash(exc) + + with self.subTest('Py-API', obj=obj, exc_type=exc_type, new_end=new_end): + exc = exc_type('utf-8', obj, 0, obj_len, 'reason') + # arbitrary value is allowed in the attribute setter + exc.end = new_end + self.assertEqual(exc.end, new_end) + self._check_no_crash(exc) + + class Test_PyUnstable_Exc_PrepReraiseStar(ExceptionIsLikeMixin, unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_capi/test_file.py b/Lib/test/test_capi/test_file.py new file mode 100644 index 00000000000000..a67a5121c4588b --- /dev/null +++ b/Lib/test/test_capi/test_file.py @@ -0,0 +1,84 @@ +import os +import unittest +from test import support +from test.support import import_helper, os_helper + +_testcapi = import_helper.import_module('_testcapi') + +NULL = None + + +class CAPIFileTest(unittest.TestCase): + def test_py_fopen(self): + # Test Py_fopen() and Py_fclose() + + with open(__file__, "rb") as fp: + source = fp.read() + + for filename in (__file__, os.fsencode(__file__)): + with self.subTest(filename=filename): + data = _testcapi.py_fopen(filename, "rb") + self.assertEqual(data, source[:256]) + + data = _testcapi.py_fopen(os_helper.FakePath(filename), "rb") + self.assertEqual(data, source[:256]) + + filenames = [ + os_helper.TESTFN, + os.fsencode(os_helper.TESTFN), + ] + if os_helper.TESTFN_UNDECODABLE is not None: + filenames.append(os_helper.TESTFN_UNDECODABLE) + filenames.append(os.fsdecode(os_helper.TESTFN_UNDECODABLE)) + if os_helper.TESTFN_UNENCODABLE is not None: + filenames.append(os_helper.TESTFN_UNENCODABLE) + for filename in filenames: + with self.subTest(filename=filename): + try: + with open(filename, "wb") as fp: + fp.write(source) + except OSError: + # TESTFN_UNDECODABLE cannot be used to create a file + # on macOS/WASI. + filename = None + continue + try: + data = _testcapi.py_fopen(filename, "rb") + self.assertEqual(data, source[:256]) + finally: + os_helper.unlink(filename) + + # embedded null character/byte in the filename + with self.assertRaises(ValueError): + _testcapi.py_fopen("a\x00b", "rb") + with self.assertRaises(ValueError): + _testcapi.py_fopen(b"a\x00b", "rb") + + # non-ASCII mode failing with "Invalid argument" + with self.assertRaises(OSError): + _testcapi.py_fopen(__file__, b"\xc2\x80") + with self.assertRaises(OSError): + # \x98 is invalid in cp1250, cp1251, cp1257 + # \x9d is invalid in cp1252-cp1255, cp1258 + _testcapi.py_fopen(__file__, b"\xc2\x98\xc2\x9d") + # UnicodeDecodeError can come from the audit hook code + with self.assertRaises((UnicodeDecodeError, OSError)): + _testcapi.py_fopen(__file__, b"\x98\x9d") + + # invalid filename type + for invalid_type in (123, object()): + with self.subTest(filename=invalid_type): + with self.assertRaises(TypeError): + _testcapi.py_fopen(invalid_type, "rb") + + if support.MS_WINDOWS: + with self.assertRaises(OSError): + # On Windows, the file mode is limited to 10 characters + _testcapi.py_fopen(__file__, "rt+, ccs=UTF-8") + + # CRASHES _testcapi.py_fopen(NULL, 'rb') + # CRASHES _testcapi.py_fopen(__file__, NULL) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_float.py b/Lib/test/test_capi/test_float.py index cb94d562645916..92c987794142c9 100644 --- a/Lib/test/test_capi/test_float.py +++ b/Lib/test/test_capi/test_float.py @@ -9,6 +9,7 @@ from test.support import import_helper _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') NULL = None @@ -29,7 +30,7 @@ class CAPIFloatTest(unittest.TestCase): def test_check(self): # Test PyFloat_Check() - check = _testcapi.float_check + check = _testlimitedcapi.float_check self.assertTrue(check(4.25)) self.assertTrue(check(FloatSubclass(4.25))) @@ -41,7 +42,7 @@ def test_check(self): def test_checkexact(self): # Test PyFloat_CheckExact() - checkexact = _testcapi.float_checkexact + checkexact = _testlimitedcapi.float_checkexact self.assertTrue(checkexact(4.25)) self.assertFalse(checkexact(FloatSubclass(4.25))) @@ -53,7 +54,7 @@ def test_checkexact(self): def test_fromstring(self): # Test PyFloat_FromString() - fromstring = _testcapi.float_fromstring + fromstring = _testlimitedcapi.float_fromstring self.assertEqual(fromstring("4.25"), 4.25) self.assertEqual(fromstring(b"4.25"), 4.25) @@ -72,13 +73,13 @@ def test_fromstring(self): def test_fromdouble(self): # Test PyFloat_FromDouble() - fromdouble = _testcapi.float_fromdouble + fromdouble = _testlimitedcapi.float_fromdouble self.assertEqual(fromdouble(4.25), 4.25) def test_asdouble(self): # Test PyFloat_AsDouble() - asdouble = _testcapi.float_asdouble + asdouble = _testlimitedcapi.float_asdouble class BadFloat3: def __float__(self): @@ -109,19 +110,19 @@ def __float__(self): def test_getinfo(self): # Test PyFloat_GetInfo() - getinfo = _testcapi.float_getinfo + getinfo = _testlimitedcapi.float_getinfo self.assertEqual(getinfo(), sys.float_info) def test_getmax(self): # Test PyFloat_GetMax() - getmax = _testcapi.float_getmax + getmax = _testlimitedcapi.float_getmax self.assertEqual(getmax(), sys.float_info.max) def test_getmin(self): # Test PyFloat_GetMax() - getmin = _testcapi.float_getmin + getmin = _testlimitedcapi.float_getmin self.assertEqual(getmin(), sys.float_info.min) diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 12039803ba543e..703d228f92e713 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -4,11 +4,18 @@ import sys from test import support from test.support import import_helper +from test.support import script_helper from test.support import warnings_helper +from test.support.testcase import FloatsAreIdenticalMixin # Skip this test if the _testcapi module isn't available. _testcapi = import_helper.import_module('_testcapi') from _testcapi import getargs_keywords, getargs_keyword_only +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = NULL + # > How about the following counterproposal. This also changes some of # > the other format codes to be a little more regular. # > @@ -430,11 +437,7 @@ def test_K(self): self.assertEqual(VERY_LARGE & ULLONG_MAX, getargs_K(VERY_LARGE)) -class Float_TestCase(unittest.TestCase): - def assertEqualWithSign(self, actual, expected): - self.assertEqual(actual, expected) - self.assertEqual(math.copysign(1, actual), math.copysign(1, expected)) - +class Float_TestCase(unittest.TestCase, FloatsAreIdenticalMixin): def test_f(self): from _testcapi import getargs_f self.assertEqual(getargs_f(4.25), 4.25) @@ -456,10 +459,10 @@ def test_f(self): self.assertEqual(getargs_f(DBL_MAX), INF) self.assertEqual(getargs_f(-DBL_MAX), -INF) if FLT_MIN > DBL_MIN: - self.assertEqualWithSign(getargs_f(DBL_MIN), 0.0) - self.assertEqualWithSign(getargs_f(-DBL_MIN), -0.0) - self.assertEqualWithSign(getargs_f(0.0), 0.0) - self.assertEqualWithSign(getargs_f(-0.0), -0.0) + self.assertFloatsAreIdentical(getargs_f(DBL_MIN), 0.0) + self.assertFloatsAreIdentical(getargs_f(-DBL_MIN), -0.0) + self.assertFloatsAreIdentical(getargs_f(0.0), 0.0) + self.assertFloatsAreIdentical(getargs_f(-0.0), -0.0) r = getargs_f(NAN) self.assertNotEqual(r, r) @@ -488,8 +491,8 @@ def test_d(self): self.assertEqual(getargs_d(x), x) self.assertRaises(OverflowError, getargs_d, 1<', 'exec') + module2 = execute_code(name, code, *args) + self.assertIs(module2, module) + + # Check the function side effects + self.assertEqual(module.attr, 1) + finally: + sys.modules.pop(name, None) + return module.__spec__.origin + + def test_executecodemodule(self): + # Test PyImport_ExecCodeModule() + execcodemodule = _testlimitedcapi.PyImport_ExecCodeModule + self.check_executecodemodule(execcodemodule) + + code = compile('attr = 1', '', 'exec') + self.assertRaises(UnicodeDecodeError, execcodemodule, b'\xff', code) + # CRASHES execcodemodule(NULL, code) + # CRASHES execcodemodule(name, NULL) + + def test_executecodemoduleex(self): + # Test PyImport_ExecCodeModuleEx() + execcodemoduleex = _testlimitedcapi.PyImport_ExecCodeModuleEx + + # Test NULL path (it should not crash) + self.check_executecodemodule(execcodemoduleex, NULL) + + # Test non-NULL path + pathname = b'pathname' + origin = self.check_executecodemodule(execcodemoduleex, pathname) + self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname))) + + pathname = os_helper.TESTFN_UNDECODABLE + if pathname: + origin = self.check_executecodemodule(execcodemoduleex, pathname) + self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname))) + + code = compile('attr = 1', '', 'exec') + self.assertRaises(UnicodeDecodeError, execcodemoduleex, b'\xff', code, NULL) + # CRASHES execcodemoduleex(NULL, code, NULL) + # CRASHES execcodemoduleex(name, NULL, NULL) + + def check_executecode_pathnames(self, execute_code_func, object=False): + # Test non-NULL pathname and NULL cpathname + + # Test NULL paths (it should not crash) + self.check_executecodemodule(execute_code_func, NULL, NULL) + + pathname = 'pathname' + origin = self.check_executecodemodule(execute_code_func, pathname, NULL) + self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname))) + origin = self.check_executecodemodule(execute_code_func, NULL, pathname) + if not object: + self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname))) + + pathname = os_helper.TESTFN_UNDECODABLE + if pathname: + if object: + pathname = os.fsdecode(pathname) + origin = self.check_executecodemodule(execute_code_func, pathname, NULL) + self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname))) + self.check_executecodemodule(execute_code_func, NULL, pathname) + + # Test NULL pathname and non-NULL cpathname + pyc_filename = importlib.util.cache_from_source(__file__) + py_filename = importlib.util.source_from_cache(pyc_filename) + origin = self.check_executecodemodule(execute_code_func, NULL, pyc_filename) + if not object: + self.assertEqual(origin, py_filename) + + def test_executecodemodulewithpathnames(self): + # Test PyImport_ExecCodeModuleWithPathnames() + execute_code_func = _testlimitedcapi.PyImport_ExecCodeModuleWithPathnames + self.check_executecode_pathnames(execute_code_func) + + code = compile('attr = 1', '', 'exec') + self.assertRaises(UnicodeDecodeError, execute_code_func, b'\xff', code, NULL, NULL) + # CRASHES execute_code_func(NULL, code, NULL, NULL) + # CRASHES execute_code_func(name, NULL, NULL, NULL) + + def test_executecodemoduleobject(self): + # Test PyImport_ExecCodeModuleObject() + execute_code_func = _testlimitedcapi.PyImport_ExecCodeModuleObject + self.check_executecode_pathnames(execute_code_func, object=True) + + code = compile('attr = 1', '', 'exec') + self.assertRaises(TypeError, execute_code_func, [], code, NULL, NULL) + nonstring = tuple(['hashable non-string']) + self.assertRaises(AttributeError, execute_code_func, nonstring, code, NULL, NULL) + sys.modules.pop(nonstring, None) + # CRASHES execute_code_func(NULL, code, NULL, NULL) + # CRASHES execute_code_func(name, NULL, NULL, NULL) + + # TODO: test PyImport_GetImporter() + # TODO: test PyImport_ReloadModule() + # TODO: test PyImport_ExtendInittab() + # PyImport_AppendInittab() is tested by test_embed + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_list.py b/Lib/test/test_capi/test_list.py index dceb4fce3c077b..fb3cb3c8769687 100644 --- a/Lib/test/test_capi/test_list.py +++ b/Lib/test/test_capi/test_list.py @@ -1,9 +1,8 @@ -import gc -import weakref import unittest from test.support import import_helper from collections import UserList _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') NULL = None PY_SSIZE_T_MIN = _testcapi.PY_SSIZE_T_MIN @@ -25,7 +24,7 @@ def __del__(self): class CAPITest(unittest.TestCase): def test_check(self): # Test PyList_Check() - check = _testcapi.list_check + check = _testlimitedcapi.list_check self.assertTrue(check([1, 2])) self.assertTrue(check([])) self.assertTrue(check(ListSubclass([1, 2]))) @@ -39,7 +38,7 @@ def test_check(self): def test_list_check_exact(self): # Test PyList_CheckExact() - check = _testcapi.list_check_exact + check = _testlimitedcapi.list_check_exact self.assertTrue(check([1])) self.assertTrue(check([])) self.assertFalse(check(ListSubclass([1]))) @@ -51,7 +50,7 @@ def test_list_check_exact(self): def test_list_new(self): # Test PyList_New() - list_new = _testcapi.list_new + list_new = _testlimitedcapi.list_new lst = list_new(0) self.assertEqual(lst, []) self.assertIs(type(lst), list) @@ -62,7 +61,7 @@ def test_list_new(self): def test_list_size(self): # Test PyList_Size() - size = _testcapi.list_size + size = _testlimitedcapi.list_size self.assertEqual(size([1, 2]), 2) self.assertEqual(size(ListSubclass([1, 2])), 2) self.assertRaises(SystemError, size, UserList()) @@ -98,11 +97,11 @@ def check_list_get_item(self, getitem, exctype): def test_list_getitem(self): # Test PyList_GetItem() - self.check_list_get_item(_testcapi.list_getitem, SystemError) + self.check_list_get_item(_testlimitedcapi.list_getitem, SystemError) def test_list_get_item_ref(self): # Test PyList_GetItemRef() - self.check_list_get_item(_testcapi.list_get_item_ref, TypeError) + self.check_list_get_item(_testlimitedcapi.list_get_item_ref, TypeError) def test_list_get_item(self): # Test PyList_GET_ITEM() @@ -119,7 +118,7 @@ def test_list_get_item(self): def test_list_setitem(self): # Test PyList_SetItem() - setitem = _testcapi.list_setitem + setitem = _testlimitedcapi.list_setitem lst = [1, 2, 3] setitem(lst, 0, 10) self.assertEqual(lst, [10, 2, 3]) @@ -151,7 +150,7 @@ def test_list_set_item(self): def test_list_insert(self): # Test PyList_Insert() - insert = _testcapi.list_insert + insert = _testlimitedcapi.list_insert lst = [1, 2, 3] insert(lst, 0, 23) self.assertEqual(lst, [23, 1, 2, 3]) @@ -173,7 +172,7 @@ def test_list_insert(self): def test_list_append(self): # Test PyList_Append() - append = _testcapi.list_append + append = _testlimitedcapi.list_append lst = [1, 2, 3] append(lst, 10) self.assertEqual(lst, [1, 2, 3, 10]) @@ -186,7 +185,7 @@ def test_list_append(self): def test_list_getslice(self): # Test PyList_GetSlice() - getslice = _testcapi.list_getslice + getslice = _testlimitedcapi.list_getslice lst = [1, 2, 3] # empty @@ -210,7 +209,7 @@ def test_list_getslice(self): def test_list_setslice(self): # Test PyList_SetSlice() - list_setslice = _testcapi.list_setslice + list_setslice = _testlimitedcapi.list_setslice def set_slice(lst, low, high, value): lst = lst.copy() self.assertEqual(list_setslice(lst, low, high, value), 0) @@ -265,7 +264,7 @@ def set_slice(lst, low, high, value): def test_list_sort(self): # Test PyList_Sort() - sort = _testcapi.list_sort + sort = _testlimitedcapi.list_sort lst = [4, 6, 7, 3, 1, 5, 9, 2, 0, 8] sort(lst) self.assertEqual(lst, list(range(10))) @@ -281,7 +280,7 @@ def test_list_sort(self): def test_list_reverse(self): # Test PyList_Reverse() - reverse = _testcapi.list_reverse + reverse = _testlimitedcapi.list_reverse def list_reverse(lst): self.assertEqual(reverse(lst), 0) return lst @@ -295,7 +294,7 @@ def list_reverse(lst): def test_list_astuple(self): # Test PyList_AsTuple() - astuple = _testcapi.list_astuple + astuple = _testlimitedcapi.list_astuple self.assertEqual(astuple([]), ()) self.assertEqual(astuple([2, 5, 10]), (2, 5, 10)) @@ -346,3 +345,7 @@ def test_list_extend(self): # CRASHES list_extend(NULL, []) # CRASHES list_extend([], NULL) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index 9f5ee507a8eb85..d45ac75c822ea9 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -4,11 +4,13 @@ from test.support import import_helper -# Skip this test if the _testcapi module isn't available. +# Skip this test if the _testcapi and _testlimitedcapi modules isn't available. _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') NULL = None + class IntSubclass(int): pass @@ -56,7 +58,7 @@ def test_compact_known(self): def test_long_check(self): # Test PyLong_Check() - check = _testcapi.pylong_check + check = _testlimitedcapi.pylong_check self.assertTrue(check(1)) self.assertTrue(check(123456789012345678901234567890)) self.assertTrue(check(-1)) @@ -68,7 +70,7 @@ def test_long_check(self): def test_long_checkexact(self): # Test PyLong_CheckExact() - check = _testcapi.pylong_checkexact + check = _testlimitedcapi.pylong_checkexact self.assertTrue(check(1)) self.assertTrue(check(123456789012345678901234567890)) self.assertTrue(check(-1)) @@ -80,7 +82,7 @@ def test_long_checkexact(self): def test_long_fromdouble(self): # Test PyLong_FromDouble() - fromdouble = _testcapi.pylong_fromdouble + fromdouble = _testlimitedcapi.pylong_fromdouble float_max = sys.float_info.max for value in (5.0, 5.1, 5.9, -5.1, -5.9, 0.0, -0.0, float_max, -float_max): with self.subTest(value=value): @@ -91,7 +93,7 @@ def test_long_fromdouble(self): def test_long_fromvoidptr(self): # Test PyLong_FromVoidPtr() - fromvoidptr = _testcapi.pylong_fromvoidptr + fromvoidptr = _testlimitedcapi.pylong_fromvoidptr obj = object() x = fromvoidptr(obj) y = fromvoidptr(NULL) @@ -103,7 +105,7 @@ def test_long_fromvoidptr(self): def test_long_fromstring(self): # Test PyLong_FromString() - fromstring = _testcapi.pylong_fromstring + fromstring = _testlimitedcapi.pylong_fromstring self.assertEqual(fromstring(b'123', 10), (123, 3)) self.assertEqual(fromstring(b'cafe', 16), (0xcafe, 4)) self.assertEqual(fromstring(b'xyz', 36), (44027, 3)) @@ -161,228 +163,129 @@ def test_long_fromunicodeobject(self): # CRASHES fromunicodeobject(NULL, 0) # CRASHES fromunicodeobject(NULL, 16) - def test_long_asint(self): - # Test PyLong_AsInt() - PyLong_AsInt = _testcapi.PyLong_AsInt - from _testcapi import INT_MIN, INT_MAX - - # round trip (object -> int -> object) - for value in (INT_MIN, INT_MAX, -1, 0, 1, 123): + def check_long_asint(self, func, min_val, max_val, *, + use_index=True, + mask=False, + negative_value_error=OverflowError): + # round trip (object -> C integer -> object) + values = (0, 1, 1234, max_val) + if min_val < 0: + values += (-1, min_val) + for value in values: with self.subTest(value=value): - self.assertEqual(PyLong_AsInt(value), value) - self.assertEqual(PyLong_AsInt(IntSubclass(42)), 42) - self.assertEqual(PyLong_AsInt(Index(42)), 42) - self.assertEqual(PyLong_AsInt(MyIndexAndInt()), 10) + self.assertEqual(func(value), value) + self.assertEqual(func(IntSubclass(value)), value) + if use_index: + self.assertEqual(func(Index(value)), value) + + if use_index: + self.assertEqual(func(MyIndexAndInt()), 10) + else: + self.assertRaises(TypeError, func, Index(42)) + self.assertRaises(TypeError, func, MyIndexAndInt()) + + if mask: + self.assertEqual(func(min_val - 1), max_val) + self.assertEqual(func(max_val + 1), min_val) + self.assertEqual(func(-1 << 1000), 0) + self.assertEqual(func(1 << 1000), 0) + else: + self.assertRaises(negative_value_error, func, min_val - 1) + self.assertRaises(negative_value_error, func, -1 << 1000) + self.assertRaises(OverflowError, func, max_val + 1) + self.assertRaises(OverflowError, func, 1 << 1000) + self.assertRaises(TypeError, func, 1.0) + self.assertRaises(TypeError, func, b'2') + self.assertRaises(TypeError, func, '3') + self.assertRaises(SystemError, func, NULL) + + def check_long_asintandoverflow(self, func, min_val, max_val): + # round trip (object -> C integer -> object) + for value in (min_val, max_val, -1, 0, 1, 1234): + with self.subTest(value=value): + self.assertEqual(func(value), (value, 0)) + self.assertEqual(func(IntSubclass(value)), (value, 0)) + self.assertEqual(func(Index(value)), (value, 0)) + + self.assertEqual(func(MyIndexAndInt()), (10, 0)) - # bound checking - self.assertRaises(OverflowError, PyLong_AsInt, INT_MIN - 1) - self.assertRaises(OverflowError, PyLong_AsInt, INT_MAX + 1) + self.assertEqual(func(min_val - 1), (-1, -1)) + self.assertEqual(func(max_val + 1), (-1, +1)) - # invalid type - self.assertRaises(TypeError, PyLong_AsInt, 1.0) - self.assertRaises(TypeError, PyLong_AsInt, b'2') - self.assertRaises(TypeError, PyLong_AsInt, '3') - self.assertRaises(SystemError, PyLong_AsInt, NULL) + # CRASHES func(1.0) + # CRASHES func(NULL) + + def test_long_asint(self): + # Test PyLong_AsInt() + PyLong_AsInt = _testlimitedcapi.PyLong_AsInt + from _testcapi import INT_MIN, INT_MAX + self.check_long_asint(PyLong_AsInt, INT_MIN, INT_MAX) def test_long_aslong(self): # Test PyLong_AsLong() and PyLong_FromLong() - aslong = _testcapi.pylong_aslong + aslong = _testlimitedcapi.pylong_aslong from _testcapi import LONG_MIN, LONG_MAX - # round trip (object -> long -> object) - for value in (LONG_MIN, LONG_MAX, -1, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(aslong(value), value) - - self.assertEqual(aslong(IntSubclass(42)), 42) - self.assertEqual(aslong(Index(42)), 42) - self.assertEqual(aslong(MyIndexAndInt()), 10) - - self.assertRaises(OverflowError, aslong, LONG_MIN - 1) - self.assertRaises(OverflowError, aslong, LONG_MAX + 1) - self.assertRaises(TypeError, aslong, 1.0) - self.assertRaises(TypeError, aslong, b'2') - self.assertRaises(TypeError, aslong, '3') - self.assertRaises(SystemError, aslong, NULL) + self.check_long_asint(aslong, LONG_MIN, LONG_MAX) def test_long_aslongandoverflow(self): # Test PyLong_AsLongAndOverflow() - aslongandoverflow = _testcapi.pylong_aslongandoverflow + aslongandoverflow = _testlimitedcapi.pylong_aslongandoverflow from _testcapi import LONG_MIN, LONG_MAX - # round trip (object -> long -> object) - for value in (LONG_MIN, LONG_MAX, -1, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(aslongandoverflow(value), (value, 0)) - - self.assertEqual(aslongandoverflow(IntSubclass(42)), (42, 0)) - self.assertEqual(aslongandoverflow(Index(42)), (42, 0)) - self.assertEqual(aslongandoverflow(MyIndexAndInt()), (10, 0)) - - self.assertEqual(aslongandoverflow(LONG_MIN - 1), (-1, -1)) - self.assertEqual(aslongandoverflow(LONG_MAX + 1), (-1, 1)) - # CRASHES aslongandoverflow(1.0) - # CRASHES aslongandoverflow(NULL) + self.check_long_asintandoverflow(aslongandoverflow, LONG_MIN, LONG_MAX) def test_long_asunsignedlong(self): # Test PyLong_AsUnsignedLong() and PyLong_FromUnsignedLong() - asunsignedlong = _testcapi.pylong_asunsignedlong + asunsignedlong = _testlimitedcapi.pylong_asunsignedlong from _testcapi import ULONG_MAX - # round trip (object -> unsigned long -> object) - for value in (ULONG_MAX, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(asunsignedlong(value), value) - - self.assertEqual(asunsignedlong(IntSubclass(42)), 42) - self.assertRaises(TypeError, asunsignedlong, Index(42)) - self.assertRaises(TypeError, asunsignedlong, MyIndexAndInt()) - - self.assertRaises(OverflowError, asunsignedlong, -1) - self.assertRaises(OverflowError, asunsignedlong, ULONG_MAX + 1) - self.assertRaises(TypeError, asunsignedlong, 1.0) - self.assertRaises(TypeError, asunsignedlong, b'2') - self.assertRaises(TypeError, asunsignedlong, '3') - self.assertRaises(SystemError, asunsignedlong, NULL) + self.check_long_asint(asunsignedlong, 0, ULONG_MAX, + use_index=False) def test_long_asunsignedlongmask(self): # Test PyLong_AsUnsignedLongMask() - asunsignedlongmask = _testcapi.pylong_asunsignedlongmask + asunsignedlongmask = _testlimitedcapi.pylong_asunsignedlongmask from _testcapi import ULONG_MAX - # round trip (object -> unsigned long -> object) - for value in (ULONG_MAX, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(asunsignedlongmask(value), value) - - self.assertEqual(asunsignedlongmask(IntSubclass(42)), 42) - self.assertEqual(asunsignedlongmask(Index(42)), 42) - self.assertEqual(asunsignedlongmask(MyIndexAndInt()), 10) - - self.assertEqual(asunsignedlongmask(-1), ULONG_MAX) - self.assertEqual(asunsignedlongmask(ULONG_MAX + 1), 0) - self.assertRaises(TypeError, asunsignedlongmask, 1.0) - self.assertRaises(TypeError, asunsignedlongmask, b'2') - self.assertRaises(TypeError, asunsignedlongmask, '3') - self.assertRaises(SystemError, asunsignedlongmask, NULL) + self.check_long_asint(asunsignedlongmask, 0, ULONG_MAX, mask=True) def test_long_aslonglong(self): # Test PyLong_AsLongLong() and PyLong_FromLongLong() - aslonglong = _testcapi.pylong_aslonglong + aslonglong = _testlimitedcapi.pylong_aslonglong from _testcapi import LLONG_MIN, LLONG_MAX - # round trip (object -> long long -> object) - for value in (LLONG_MIN, LLONG_MAX, -1, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(aslonglong(value), value) - - self.assertEqual(aslonglong(IntSubclass(42)), 42) - self.assertEqual(aslonglong(Index(42)), 42) - self.assertEqual(aslonglong(MyIndexAndInt()), 10) - - self.assertRaises(OverflowError, aslonglong, LLONG_MIN - 1) - self.assertRaises(OverflowError, aslonglong, LLONG_MAX + 1) - self.assertRaises(TypeError, aslonglong, 1.0) - self.assertRaises(TypeError, aslonglong, b'2') - self.assertRaises(TypeError, aslonglong, '3') - self.assertRaises(SystemError, aslonglong, NULL) + self.check_long_asint(aslonglong, LLONG_MIN, LLONG_MAX) def test_long_aslonglongandoverflow(self): # Test PyLong_AsLongLongAndOverflow() - aslonglongandoverflow = _testcapi.pylong_aslonglongandoverflow + aslonglongandoverflow = _testlimitedcapi.pylong_aslonglongandoverflow from _testcapi import LLONG_MIN, LLONG_MAX - # round trip (object -> long long -> object) - for value in (LLONG_MIN, LLONG_MAX, -1, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(aslonglongandoverflow(value), (value, 0)) - - self.assertEqual(aslonglongandoverflow(IntSubclass(42)), (42, 0)) - self.assertEqual(aslonglongandoverflow(Index(42)), (42, 0)) - self.assertEqual(aslonglongandoverflow(MyIndexAndInt()), (10, 0)) - - self.assertEqual(aslonglongandoverflow(LLONG_MIN - 1), (-1, -1)) - self.assertEqual(aslonglongandoverflow(LLONG_MAX + 1), (-1, 1)) - # CRASHES aslonglongandoverflow(1.0) - # CRASHES aslonglongandoverflow(NULL) + self.check_long_asintandoverflow(aslonglongandoverflow, LLONG_MIN, LLONG_MAX) def test_long_asunsignedlonglong(self): # Test PyLong_AsUnsignedLongLong() and PyLong_FromUnsignedLongLong() - asunsignedlonglong = _testcapi.pylong_asunsignedlonglong + asunsignedlonglong = _testlimitedcapi.pylong_asunsignedlonglong from _testcapi import ULLONG_MAX - # round trip (object -> unsigned long long -> object) - for value in (ULLONG_MAX, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(asunsignedlonglong(value), value) - - self.assertEqual(asunsignedlonglong(IntSubclass(42)), 42) - self.assertRaises(TypeError, asunsignedlonglong, Index(42)) - self.assertRaises(TypeError, asunsignedlonglong, MyIndexAndInt()) - - self.assertRaises(OverflowError, asunsignedlonglong, -1) - self.assertRaises(OverflowError, asunsignedlonglong, ULLONG_MAX + 1) - self.assertRaises(TypeError, asunsignedlonglong, 1.0) - self.assertRaises(TypeError, asunsignedlonglong, b'2') - self.assertRaises(TypeError, asunsignedlonglong, '3') - self.assertRaises(SystemError, asunsignedlonglong, NULL) + self.check_long_asint(asunsignedlonglong, 0, ULLONG_MAX, use_index=False) def test_long_asunsignedlonglongmask(self): # Test PyLong_AsUnsignedLongLongMask() - asunsignedlonglongmask = _testcapi.pylong_asunsignedlonglongmask + asunsignedlonglongmask = _testlimitedcapi.pylong_asunsignedlonglongmask from _testcapi import ULLONG_MAX - # round trip (object -> unsigned long long -> object) - for value in (ULLONG_MAX, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(asunsignedlonglongmask(value), value) - - self.assertEqual(asunsignedlonglongmask(IntSubclass(42)), 42) - self.assertEqual(asunsignedlonglongmask(Index(42)), 42) - self.assertEqual(asunsignedlonglongmask(MyIndexAndInt()), 10) - - self.assertEqual(asunsignedlonglongmask(-1), ULLONG_MAX) - self.assertEqual(asunsignedlonglongmask(ULLONG_MAX + 1), 0) - self.assertRaises(TypeError, asunsignedlonglongmask, 1.0) - self.assertRaises(TypeError, asunsignedlonglongmask, b'2') - self.assertRaises(TypeError, asunsignedlonglongmask, '3') - self.assertRaises(SystemError, asunsignedlonglongmask, NULL) + self.check_long_asint(asunsignedlonglongmask, 0, ULLONG_MAX, mask=True) def test_long_as_ssize_t(self): # Test PyLong_AsSsize_t() and PyLong_FromSsize_t() - as_ssize_t = _testcapi.pylong_as_ssize_t + as_ssize_t = _testlimitedcapi.pylong_as_ssize_t from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX - # round trip (object -> Py_ssize_t -> object) - for value in (PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, -1, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(as_ssize_t(value), value) - - self.assertEqual(as_ssize_t(IntSubclass(42)), 42) - self.assertRaises(TypeError, as_ssize_t, Index(42)) - self.assertRaises(TypeError, as_ssize_t, MyIndexAndInt()) - - self.assertRaises(OverflowError, as_ssize_t, PY_SSIZE_T_MIN - 1) - self.assertRaises(OverflowError, as_ssize_t, PY_SSIZE_T_MAX + 1) - self.assertRaises(TypeError, as_ssize_t, 1.0) - self.assertRaises(TypeError, as_ssize_t, b'2') - self.assertRaises(TypeError, as_ssize_t, '3') - self.assertRaises(SystemError, as_ssize_t, NULL) + self.check_long_asint(as_ssize_t, PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, + use_index=False) def test_long_as_size_t(self): # Test PyLong_AsSize_t() and PyLong_FromSize_t() - as_size_t = _testcapi.pylong_as_size_t + as_size_t = _testlimitedcapi.pylong_as_size_t from _testcapi import SIZE_MAX - # round trip (object -> size_t -> object) - for value in (SIZE_MAX, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(as_size_t(value), value) - - self.assertEqual(as_size_t(IntSubclass(42)), 42) - self.assertRaises(TypeError, as_size_t, Index(42)) - self.assertRaises(TypeError, as_size_t, MyIndexAndInt()) - - self.assertRaises(OverflowError, as_size_t, -1) - self.assertRaises(OverflowError, as_size_t, SIZE_MAX + 1) - self.assertRaises(TypeError, as_size_t, 1.0) - self.assertRaises(TypeError, as_size_t, b'2') - self.assertRaises(TypeError, as_size_t, '3') - self.assertRaises(SystemError, as_size_t, NULL) + self.check_long_asint(as_size_t, 0, SIZE_MAX, use_index=False) def test_long_asdouble(self): # Test PyLong_AsDouble() - asdouble = _testcapi.pylong_asdouble + asdouble = _testlimitedcapi.pylong_asdouble MAX = int(sys.float_info.max) for value in (-MAX, MAX, -1, 0, 1, 1234): with self.subTest(value=value): @@ -402,8 +305,8 @@ def test_long_asdouble(self): def test_long_asvoidptr(self): # Test PyLong_AsVoidPtr() - fromvoidptr = _testcapi.pylong_fromvoidptr - asvoidptr = _testcapi.pylong_asvoidptr + fromvoidptr = _testlimitedcapi.pylong_fromvoidptr + asvoidptr = _testlimitedcapi.pylong_asvoidptr obj = object() x = fromvoidptr(obj) y = fromvoidptr(NULL) @@ -424,6 +327,32 @@ def test_long_asvoidptr(self): self.assertRaises(OverflowError, asvoidptr, -2**1000) # CRASHES asvoidptr(NULL) + def _test_long_aspid(self, aspid): + # Test PyLong_AsPid() + from _testcapi import SIZEOF_PID_T + bits = 8 * SIZEOF_PID_T + PID_T_MIN = -2**(bits-1) + PID_T_MAX = 2**(bits-1) - 1 + self.check_long_asint(aspid, PID_T_MIN, PID_T_MAX) + + def test_long_aspid(self): + self._test_long_aspid(_testcapi.pylong_aspid) + + def test_long_aspid_limited(self): + self._test_long_aspid(_testlimitedcapi.pylong_aspid) + + @support.bigmemtest(2**32, memuse=0.35) + def test_long_asnativebytes_huge(self, size): + asnativebytes = _testcapi.pylong_asnativebytes + v = 1 << size + buffer = bytearray(size * 2 // 15 + 10) + r = asnativebytes(v, buffer, 0, -1) + self.assertEqual(r, size // 8 + 1) + self.assertEqual(buffer.count(0), len(buffer)) + r = asnativebytes(v, buffer, len(buffer), -1) + self.assertEqual(r, size // 8 + 1) + self.assertEqual(buffer.count(0), len(buffer) - 1) + def test_long_asnativebytes(self): import math from _testcapi import ( @@ -454,8 +383,12 @@ def test_long_asnativebytes(self): (-MAX_USIZE, SZ + 1), (2**255-1, 32), (-(2**255-1), 32), + (2**255, 33), + (-(2**255), 33), # if you ask, we'll say 33, but 32 would do (2**256-1, 33), (-(2**256-1), 33), + (2**256, 33), + (-(2**256), 33), ]: with self.subTest(f"sizeof-{v:X}"): buffer = bytearray(b"\x5a") @@ -463,8 +396,9 @@ def test_long_asnativebytes(self): "PyLong_AsNativeBytes(v, , 0, -1)") self.assertEqual(buffer, b"\x5a", "buffer overwritten when it should not have been") - # Also check via the __index__ path - self.assertEqual(expect, asnativebytes(Index(v), buffer, 0, -1), + # Also check via the __index__ path. + # We pass Py_ASNATIVEBYTES_NATIVE_ENDIAN | ALLOW_INDEX + self.assertEqual(expect, asnativebytes(Index(v), buffer, 0, 3 | 16), "PyLong_AsNativeBytes(Index(v), , 0, -1)") self.assertEqual(buffer, b"\x5a", "buffer overwritten when it should not have been") @@ -494,15 +428,17 @@ def test_long_asnativebytes(self): (-1, b'\xff' * 10, min(11, SZ)), (-42, b'\xd6', 1), (-42, b'\xff' * 10 + b'\xd6', min(11, SZ)), - # Extracts 255 into a single byte, but requests sizeof(Py_ssize_t) - (255, b'\xff', SZ), + # Extracts 255 into a single byte, but requests 2 + # (this is currently a special case, and "should" request SZ) + (255, b'\xff', 2), (255, b'\x00\xff', 2), (256, b'\x01\x00', 2), + (0x80, b'\x00' * 7 + b'\x80', min(8, SZ)), # Extracts successfully (unsigned), but requests 9 bytes (2**63, b'\x80' + b'\x00' * 7, 9), - # "Extracts", but requests 9 bytes - (-2**63, b'\x80' + b'\x00' * 7, 9), (2**63, b'\x00\x80' + b'\x00' * 7, 9), + # Extracts into 8 bytes, but if you provide 9 we'll say 9 + (-2**63, b'\x80' + b'\x00' * 7, 8), (-2**63, b'\xff\x80' + b'\x00' * 7, 9), (2**255-1, b'\x7f' + b'\xff' * 31, 32), @@ -519,10 +455,15 @@ def test_long_asnativebytes(self): (-(2**256-1), b'\x00' * 31 + b'\x01', 33), (-(2**256-1), b'\xff' + b'\x00' * 31 + b'\x01', 33), (-(2**256-1), b'\xff\xff' + b'\x00' * 31 + b'\x01', 33), + # However, -2**255 precisely will extract into 32 bytes and return + # success. For bigger buffers, it will still succeed, but will + # return 33 + (-(2**255), b'\x80' + b'\x00' * 31, 32), + (-(2**255), b'\xff\x80' + b'\x00' * 31, 33), # The classic "Windows HRESULT as negative number" case # HRESULT hr; - # PyLong_CopyBits(<-2147467259>, &hr, sizeof(HRESULT)) + # PyLong_AsNativeBytes(<-2147467259>, &hr, sizeof(HRESULT), -1) # assert(hr == E_FAIL) (-2147467259, b'\x80\x00\x40\x05', 4), ]: @@ -540,14 +481,108 @@ def test_long_asnativebytes(self): f"PyLong_AsNativeBytes(v, buffer, {n}, )") self.assertEqual(expect_le, buffer[:n], "") + # Test cases that do not request size for a sign bit when we pass the + # Py_ASNATIVEBYTES_UNSIGNED_BUFFER flag + for v, expect_be, expect_n in [ + (255, b'\xff', 1), + # We pass a 2 byte buffer so it just uses the whole thing + (255, b'\x00\xff', 2), + + (2**63, b'\x80' + b'\x00' * 7, 8), + # We pass a 9 byte buffer so it uses the whole thing + (2**63, b'\x00\x80' + b'\x00' * 7, 9), + + (2**256-1, b'\xff' * 32, 32), + # We pass a 33 byte buffer so it uses the whole thing + (2**256-1, b'\x00' + b'\xff' * 32, 33), + ]: + with self.subTest(f"{v:X}-{len(expect_be)}bytes-unsigned"): + n = len(expect_be) + buffer = bytearray(b"\xa5"*n) + self.assertEqual(expect_n, asnativebytes(v, buffer, n, 4), + f"PyLong_AsNativeBytes(v, buffer, {n}, )") + self.assertEqual(expect_n, asnativebytes(v, buffer, n, 5), + f"PyLong_AsNativeBytes(v, buffer, {n}, )") + + # Ensure Py_ASNATIVEBYTES_REJECT_NEGATIVE raises on negative value + with self.assertRaises(ValueError): + asnativebytes(-1, buffer, 0, 8) + + # Ensure omitting Py_ASNATIVEBYTES_ALLOW_INDEX raises on __index__ value + with self.assertRaises(TypeError): + asnativebytes(Index(1), buffer, 0, -1) + with self.assertRaises(TypeError): + asnativebytes(Index(1), buffer, 0, 3) + # Check a few error conditions. These are validated in code, but are # unspecified in docs, so if we make changes to the implementation, it's # fine to just update these tests rather than preserve the behaviour. - with self.assertRaises(SystemError): - asnativebytes(1, buffer, 0, 2) with self.assertRaises(TypeError): asnativebytes('not a number', buffer, 0, -1) + def test_long_asnativebytes_fuzz(self): + import math + from random import Random + from _testcapi import ( + pylong_asnativebytes as asnativebytes, + SIZE_MAX, + ) + + # Abbreviate sizeof(Py_ssize_t) to SZ because we use it a lot + SZ = int(math.ceil(math.log(SIZE_MAX + 1) / math.log(2)) / 8) + + rng = Random() + # Allocate bigger buffer than actual values are going to be + buffer = bytearray(260) + + for _ in range(1000): + n = rng.randrange(1, 256) + bytes_be = bytes([ + # Ensure the most significant byte is nonzero + rng.randrange(1, 256), + *[rng.randrange(256) for _ in range(n - 1)] + ]) + bytes_le = bytes_be[::-1] + v = int.from_bytes(bytes_le, 'little') + + expect_1 = expect_2 = (SZ, n) + if bytes_be[0] & 0x80: + # All values are positive, so if MSB is set, expect extra bit + # when we request the size or have a large enough buffer + expect_1 = (SZ, n + 1) + # When passing Py_ASNATIVEBYTES_UNSIGNED_BUFFER, we expect the + # return to be exactly the right size. + expect_2 = (n,) + + try: + actual = asnativebytes(v, buffer, 0, -1) + self.assertIn(actual, expect_1) + + actual = asnativebytes(v, buffer, len(buffer), 0) + self.assertIn(actual, expect_1) + self.assertEqual(bytes_be, buffer[-n:]) + + actual = asnativebytes(v, buffer, len(buffer), 1) + self.assertIn(actual, expect_1) + self.assertEqual(bytes_le, buffer[:n]) + + actual = asnativebytes(v, buffer, n, 4) + self.assertIn(actual, expect_2, bytes_be.hex()) + actual = asnativebytes(v, buffer, n, 5) + self.assertIn(actual, expect_2, bytes_be.hex()) + except AssertionError as ex: + value_hex = ''.join(reversed([ + f'{b:02X}{"" if i % 8 else "_"}' + for i, b in enumerate(bytes_le, start=1) + ])).strip('_') + if support.verbose: + print() + print(n, 'bytes') + print('hex =', value_hex) + print('int =', v) + raise + raise AssertionError(f"Value: 0x{value_hex}") from ex + def test_long_fromnativebytes(self): import math from _testcapi import ( @@ -588,6 +623,187 @@ def test_long_fromnativebytes(self): self.assertEqual(expect_u, fromnativebytes(v_be, n, -1, 0), f"PyLong_FromUnsignedNativeBytes(buffer, {n}, )") + # Swap the unsigned request for tests and use the + # Py_ASNATIVEBYTES_UNSIGNED_BUFFER flag instead + self.assertEqual(expect_u, fromnativebytes(v_be, n, 4, 1), + f"PyLong_FromNativeBytes(buffer, {n}, )") + + def test_long_getsign(self): + # Test PyLong_GetSign() + getsign = _testcapi.pylong_getsign + self.assertEqual(getsign(1), 1) + self.assertEqual(getsign(123456), 1) + self.assertEqual(getsign(-2), -1) + self.assertEqual(getsign(0), 0) + self.assertEqual(getsign(True), 1) + self.assertEqual(getsign(IntSubclass(-11)), -1) + self.assertEqual(getsign(False), 0) + + self.assertRaises(TypeError, getsign, 1.0) + self.assertRaises(TypeError, getsign, Index(123)) + + # CRASHES getsign(NULL) + + def test_long_ispositive(self): + # Test PyLong_IsPositive() + ispositive = _testcapi.pylong_ispositive + self.assertEqual(ispositive(1), 1) + self.assertEqual(ispositive(123), 1) + self.assertEqual(ispositive(-1), 0) + self.assertEqual(ispositive(0), 0) + self.assertEqual(ispositive(True), 1) + self.assertEqual(ispositive(False), 0) + self.assertEqual(ispositive(IntSubclass(-1)), 0) + self.assertRaises(TypeError, ispositive, 1.0) + self.assertRaises(TypeError, ispositive, Index(123)) + + # CRASHES ispositive(NULL) + + def test_long_isnegative(self): + # Test PyLong_IsNegative() + isnegative = _testcapi.pylong_isnegative + self.assertEqual(isnegative(1), 0) + self.assertEqual(isnegative(123), 0) + self.assertEqual(isnegative(-1), 1) + self.assertEqual(isnegative(0), 0) + self.assertEqual(isnegative(True), 0) + self.assertEqual(isnegative(False), 0) + self.assertEqual(isnegative(IntSubclass(-1)), 1) + self.assertRaises(TypeError, isnegative, 1.0) + self.assertRaises(TypeError, isnegative, Index(123)) + + # CRASHES isnegative(NULL) + + def test_long_iszero(self): + # Test PyLong_IsZero() + iszero = _testcapi.pylong_iszero + self.assertEqual(iszero(1), 0) + self.assertEqual(iszero(-1), 0) + self.assertEqual(iszero(0), 1) + self.assertEqual(iszero(True), 0) + self.assertEqual(iszero(False), 1) + self.assertEqual(iszero(IntSubclass(-1)), 0) + self.assertEqual(iszero(IntSubclass(0)), 1) + self.assertRaises(TypeError, iszero, 1.0) + self.assertRaises(TypeError, iszero, Index(123)) + + # CRASHES iszero(NULL) + + def test_long_asint32(self): + # Test PyLong_AsInt32() and PyLong_FromInt32() + to_int32 = _testlimitedcapi.pylong_asint32 + from _testcapi import INT32_MIN, INT32_MAX + self.check_long_asint(to_int32, INT32_MIN, INT32_MAX) + + def test_long_asint64(self): + # Test PyLong_AsInt64() and PyLong_FromInt64() + as_int64 = _testlimitedcapi.pylong_asint64 + from _testcapi import INT64_MIN, INT64_MAX + self.check_long_asint(as_int64, INT64_MIN, INT64_MAX) + + def test_long_asuint32(self): + # Test PyLong_AsUInt32() and PyLong_FromUInt32() + as_uint32 = _testlimitedcapi.pylong_asuint32 + from _testcapi import UINT32_MAX + self.check_long_asint(as_uint32, 0, UINT32_MAX, + negative_value_error=ValueError) + + def test_long_asuint64(self): + # Test PyLong_AsUInt64() and PyLong_FromUInt64() + as_uint64 = _testlimitedcapi.pylong_asuint64 + from _testcapi import UINT64_MAX + self.check_long_asint(as_uint64, 0, UINT64_MAX, + negative_value_error=ValueError) + + def test_long_layout(self): + # Test PyLong_GetNativeLayout() + int_info = sys.int_info + layout = _testcapi.get_pylong_layout() + expected = { + 'bits_per_digit': int_info.bits_per_digit, + 'digit_size': int_info.sizeof_digit, + 'digits_order': -1, + 'digit_endianness': -1 if sys.byteorder == 'little' else 1, + } + self.assertEqual(layout, expected) + + def test_long_export(self): + # Test PyLong_Export() + layout = _testcapi.get_pylong_layout() + base = 2 ** layout['bits_per_digit'] + + pylong_export = _testcapi.pylong_export + + # value fits into int64_t + self.assertEqual(pylong_export(0), 0) + self.assertEqual(pylong_export(123), 123) + self.assertEqual(pylong_export(-123), -123) + self.assertEqual(pylong_export(IntSubclass(123)), 123) + + # use an array, doesn't fit into int64_t + self.assertEqual(pylong_export(base**10 * 2 + 1), + (0, [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2])) + self.assertEqual(pylong_export(-(base**10 * 2 + 1)), + (1, [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2])) + self.assertEqual(pylong_export(IntSubclass(base**10 * 2 + 1)), + (0, [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2])) + + self.assertRaises(TypeError, pylong_export, 1.0) + self.assertRaises(TypeError, pylong_export, 0+1j) + self.assertRaises(TypeError, pylong_export, "abc") + + def test_longwriter_create(self): + # Test PyLongWriter_Create() + layout = _testcapi.get_pylong_layout() + base = 2 ** layout['bits_per_digit'] + + pylongwriter_create = _testcapi.pylongwriter_create + self.assertRaises(ValueError, pylongwriter_create, 0, []) + self.assertRaises(ValueError, pylongwriter_create, -123, []) + self.assertEqual(pylongwriter_create(0, [0]), 0) + self.assertEqual(pylongwriter_create(0, [123]), 123) + self.assertEqual(pylongwriter_create(1, [123]), -123) + self.assertEqual(pylongwriter_create(1, [1, 2]), + -(base * 2 + 1)) + self.assertEqual(pylongwriter_create(0, [1, 2, 3]), + base**2 * 3 + base * 2 + 1) + max_digit = base - 1 + self.assertEqual(pylongwriter_create(0, [max_digit, max_digit, max_digit]), + base**2 * max_digit + base * max_digit + max_digit) + + # normalize + self.assertEqual(pylongwriter_create(0, [123, 0, 0]), 123) + + # test singletons + normalize + for num in (-2, 0, 1, 5, 42, 100): + self.assertIs(pylongwriter_create(bool(num < 0), [abs(num), 0]), + num) + + def to_digits(num): + digits = [] + while True: + num, digit = divmod(num, base) + digits.append(digit) + if not num: + break + return digits + + # round trip: Python int -> export -> Python int + pylong_export = _testcapi.pylong_export + numbers = [*range(0, 10), 12345, 0xdeadbeef, 2**100, 2**100-1] + numbers.extend(-num for num in list(numbers)) + for num in numbers: + with self.subTest(num=num): + data = pylong_export(num) + if isinstance(data, tuple): + negative, digits = data + else: + value = data + negative = int(value < 0) + digits = to_digits(abs(value)) + self.assertEqual(pylongwriter_create(negative, digits), num, + (negative, digits)) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py index 04f17a9ec9e72a..5035b2b4829bf6 100644 --- a/Lib/test/test_capi/test_mem.py +++ b/Lib/test/test_capi/test_mem.py @@ -68,8 +68,13 @@ def test_api_misuse(self): def check_malloc_without_gil(self, code): out = self.check(code) - expected = ('Fatal Python error: _PyMem_DebugMalloc: ' - 'Python memory allocator called without holding the GIL') + if not support.Py_GIL_DISABLED: + expected = ('Fatal Python error: _PyMem_DebugMalloc: ' + 'Python memory allocator called without holding the GIL') + else: + expected = ('Fatal Python error: _PyMem_DebugMalloc: ' + 'Python memory allocator called without an active thread state. ' + 'Are you trying to call it inside of a Py_BEGIN_ALLOW_THREADS block?') self.assertIn(expected, out) def test_pymem_malloc_without_gil(self): @@ -148,12 +153,12 @@ class C(): pass self.assertIn(b'MemoryError', out) *_, count = line.split(b' ') count = int(count) - self.assertLessEqual(count, i*5) - self.assertGreaterEqual(count, i*5-2) + self.assertLessEqual(count, i*10) + self.assertGreaterEqual(count, i*10-4) -# Py_GIL_DISABLED requires mimalloc (not malloc) -@unittest.skipIf(support.Py_GIL_DISABLED, 'need malloc') +# free-threading requires mimalloc (not malloc) +@support.requires_gil_enabled() class PyMemMallocDebugTests(PyMemDebugTests): PYTHONMALLOC = 'malloc_debug' diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 67fbef4f269814..1087b38c225085 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -17,7 +17,6 @@ import time import types import unittest -import warnings import weakref import operator from test import support @@ -26,6 +25,8 @@ from test.support import threading_helper from test.support import warnings_helper from test.support import requires_limited_api +from test.support import expected_failure_if_gil_disabled +from test.support import Py_GIL_DISABLED from test.support.script_helper import assert_python_failure, assert_python_ok, run_python_until_end try: import _posixsubprocess @@ -40,13 +41,16 @@ except ImportError: _testsinglephase = None try: - import _xxsubinterpreters as _interpreters + import _interpreters except ModuleNotFoundError: _interpreters = None # Skip this test if the _testcapi module isn't available. _testcapi = import_helper.import_module('_testcapi') +from _testcapi import HeapCTypeSubclass, HeapCTypeSubclassWithFinalizer + +import _testlimitedcapi import _testinternalcapi @@ -71,6 +75,11 @@ class InstanceMethod: id = _testcapi.instancemethod(id) testfunction = _testcapi.instancemethod(testfunction) + +CURRENT_THREAD_REGEX = r'Current thread.*:\n' if not support.Py_GIL_DISABLED else r'Stack .*:\n' + + +@support.force_not_colorized_test_class class CAPITest(unittest.TestCase): def test_instancemethod(self): @@ -98,13 +107,19 @@ def test_no_FatalError_infinite_loop(self): _rc, out, err = run_result self.assertEqual(out, b'') # This used to cause an infinite loop. - msg = ("Fatal Python error: PyThreadState_Get: " - "the function must be called with the GIL held, " - "after Python initialization and before Python finalization, " - "but the GIL is released " - "(the current Python thread state is NULL)").encode() - self.assertTrue(err.rstrip().startswith(msg), - err) + if not support.Py_GIL_DISABLED: + msg = ("Fatal Python error: PyThreadState_Get: " + "the function must be called with the GIL held, " + "after Python initialization and before Python finalization, " + "but the GIL is released " + "(the current Python thread state is NULL)").encode() + else: + msg = ("Fatal Python error: PyThreadState_Get: " + "the function must be called with an active thread state, " + "after Python initialization and before Python finalization, " + "but it was called without an active thread state. " + "Are you trying to call the C API inside of a Py_BEGIN_ALLOW_THREADS block?").encode() + self.assertStartsWith(err.rstrip(), msg) def test_memoryview_from_NULL_pointer(self): self.assertRaises(ValueError, _testcapi.make_memoryview_from_NULL_pointer) @@ -117,7 +132,7 @@ def __len__(self): return 1 with self.assertRaisesRegex(TypeError, 'indexing'): _posixsubprocess.fork_exec( - 1,Z(),True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22,False) + 1,Z(),True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22) # Issue #15736: overflow in _PySequence_BytesToCharpArray() class Z(object): def __len__(self): @@ -125,7 +140,7 @@ def __len__(self): def __getitem__(self, i): return b'x' self.assertRaises(MemoryError, _posixsubprocess.fork_exec, - 1,Z(),True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22,False) + 1,Z(),True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22) @unittest.skipUnless(_posixsubprocess, '_posixsubprocess required for this test.') def test_subprocess_fork_exec(self): @@ -135,7 +150,7 @@ def __len__(self): # Issue #15738: crash in subprocess_fork_exec() self.assertRaises(TypeError, _posixsubprocess.fork_exec, - Z(),[b'1'],True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22,False) + Z(),[b'1'],True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22) @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") @@ -223,8 +238,8 @@ def test_return_null_without_error(self): r'Python runtime state: initialized\n' r'SystemError: ' r'returned NULL without setting an exception\n' - r'\n' - r'Current thread.*:\n' + r'\n' + + CURRENT_THREAD_REGEX + r' File .*", line 6 in \n') else: with self.assertRaises(SystemError) as cm: @@ -257,8 +272,8 @@ def test_return_result_with_error(self): r'SystemError: ' r'returned a result with an exception set\n' - r'\n' - r'Current thread.*:\n' + r'\n' + + CURRENT_THREAD_REGEX + r' File .*, line 6 in \n') else: with self.assertRaises(SystemError) as cm: @@ -287,11 +302,11 @@ def test_getitem_with_error(self): r'with an exception set\n' r'Python runtime state: initialized\n' r'ValueError: bug\n' - r'\n' - r'Current thread .* \(most recent call first\):\n' + r'\n' + + CURRENT_THREAD_REGEX + r' File .*, line 6 in \n' r'\n' - r'Extension modules: _testcapi \(total: 1\)\n') + r'Extension modules: _testcapi, _testinternalcapi \(total: 2\)\n') else: # Python built with NDEBUG macro defined: # test _Py_CheckFunctionResult() instead. @@ -536,14 +551,19 @@ def __del__(self): self.assertEqual(new_type_refcnt, sys.getrefcount(A)) def test_heaptype_with_dict(self): - inst = _testcapi.HeapCTypeWithDict() - inst.foo = 42 - self.assertEqual(inst.foo, 42) - self.assertEqual(inst.dictobj, inst.__dict__) - self.assertEqual(inst.dictobj, {"foo": 42}) + for cls in ( + _testcapi.HeapCTypeWithDict, + _testlimitedcapi.HeapCTypeWithRelativeDict, + ): + with self.subTest(cls=cls): + inst = cls() + inst.foo = 42 + self.assertEqual(inst.foo, 42) + self.assertEqual(inst.dictobj, inst.__dict__) + self.assertEqual(inst.dictobj, {"foo": 42}) - inst = _testcapi.HeapCTypeWithDict() - self.assertEqual({}, inst.__dict__) + inst = cls() + self.assertEqual({}, inst.__dict__) def test_heaptype_with_managed_dict(self): inst = _testcapi.HeapCTypeWithManagedDict() @@ -580,10 +600,15 @@ def test_heaptype_with_negative_dict(self): self.assertEqual({}, inst.__dict__) def test_heaptype_with_weakref(self): - inst = _testcapi.HeapCTypeWithWeakref() - ref = weakref.ref(inst) - self.assertEqual(ref(), inst) - self.assertEqual(inst.weakreflist, ref) + for cls in ( + _testcapi.HeapCTypeWithWeakref, + _testlimitedcapi.HeapCTypeWithRelativeWeakref, + ): + with self.subTest(cls=cls): + inst = cls() + ref = weakref.ref(inst) + self.assertEqual(ref(), inst) + self.assertEqual(inst.weakreflist, ref) def test_heaptype_with_managed_weakref(self): inst = _testcapi.HeapCTypeWithManagedWeakref() @@ -634,9 +659,9 @@ def test_c_subclass_of_heap_ctype_with_tpdealloc_decrefs_once(self): self.assertEqual(type_refcnt - 1, sys.getrefcount(_testcapi.HeapCTypeSubclass)) def test_c_subclass_of_heap_ctype_with_del_modifying_dunder_class_only_decrefs_once(self): - subclass_instance = _testcapi.HeapCTypeSubclassWithFinalizer() - type_refcnt = sys.getrefcount(_testcapi.HeapCTypeSubclassWithFinalizer) - new_type_refcnt = sys.getrefcount(_testcapi.HeapCTypeSubclass) + subclass_instance = HeapCTypeSubclassWithFinalizer() + type_refcnt = sys.getrefcount(HeapCTypeSubclassWithFinalizer) + new_type_refcnt = sys.getrefcount(HeapCTypeSubclass) # Test that subclass instance was fully created self.assertEqual(subclass_instance.value, 10) @@ -646,19 +671,46 @@ def test_c_subclass_of_heap_ctype_with_del_modifying_dunder_class_only_decrefs_o del subclass_instance # Test that setting __class__ modified the reference counts of the types + # + # This is highly sensitive to implementation details and may break in the future. + # + # We expect the refcount on the old type, HeapCTypeSubclassWithFinalizer, to + # remain the same: the finalizer gets a strong reference (+1) when it gets the + # type from the module and setting __class__ decrements the refcount (-1). + # + # We expect the refcount on the new type, HeapCTypeSubclass, to increase by 2: + # the finalizer get a strong reference (+1) when it gets the type from the + # module and setting __class__ increments the refcount (+1). + expected_type_refcnt = type_refcnt + expected_new_type_refcnt = new_type_refcnt + 2 + + if not Py_GIL_DISABLED: + # In default builds the result returned from sys.getrefcount + # includes a temporary reference that is created by the interpreter + # when it pushes its argument on the operand stack. This temporary + # reference is not included in the result returned by Py_REFCNT, which + # is used in the finalizer. + # + # In free-threaded builds the result returned from sys.getrefcount + # does not include the temporary reference. Types use deferred + # refcounting and the interpreter will not create a new reference + # for deferred values on the operand stack. + expected_type_refcnt -= 1 + expected_new_type_refcnt -= 1 + if support.Py_DEBUG: # gh-89373: In debug mode, _Py_Dealloc() keeps a strong reference # to the type while calling tp_dealloc() - self.assertEqual(type_refcnt, _testcapi.HeapCTypeSubclassWithFinalizer.refcnt_in_del) - else: - self.assertEqual(type_refcnt - 1, _testcapi.HeapCTypeSubclassWithFinalizer.refcnt_in_del) - self.assertEqual(new_type_refcnt + 1, _testcapi.HeapCTypeSubclass.refcnt_in_del) + expected_type_refcnt += 1 + + self.assertEqual(expected_type_refcnt, HeapCTypeSubclassWithFinalizer.refcnt_in_del) + self.assertEqual(expected_new_type_refcnt, HeapCTypeSubclass.refcnt_in_del) # Test that the original type already has decreased its refcnt - self.assertEqual(type_refcnt - 1, sys.getrefcount(_testcapi.HeapCTypeSubclassWithFinalizer)) + self.assertEqual(type_refcnt - 1, sys.getrefcount(HeapCTypeSubclassWithFinalizer)) # Test that subtype_dealloc decref the newly assigned __class__ only once - self.assertEqual(new_type_refcnt, sys.getrefcount(_testcapi.HeapCTypeSubclass)) + self.assertEqual(new_type_refcnt, sys.getrefcount(HeapCTypeSubclass)) def test_heaptype_with_setattro(self): obj = _testcapi.HeapCTypeSetattr() @@ -670,7 +722,7 @@ def test_heaptype_with_setattro(self): def test_heaptype_with_custom_metaclass(self): metaclass = _testcapi.HeapCTypeMetaclass - self.assertTrue(issubclass(metaclass, type)) + self.assertIsSubclass(metaclass, type) # Class creation from C t = _testcapi.pytype_fromspec_meta(metaclass) @@ -686,7 +738,7 @@ def test_heaptype_with_custom_metaclass(self): def test_heaptype_with_custom_metaclass_null_new(self): metaclass = _testcapi.HeapCTypeMetaclassNullNew - self.assertTrue(issubclass(metaclass, type)) + self.assertIsSubclass(metaclass, type) # Class creation from C t = _testcapi.pytype_fromspec_meta(metaclass) @@ -701,69 +753,82 @@ def test_heaptype_with_custom_metaclass_null_new(self): def test_heaptype_with_custom_metaclass_custom_new(self): metaclass = _testcapi.HeapCTypeMetaclassCustomNew - self.assertTrue(issubclass(_testcapi.HeapCTypeMetaclassCustomNew, type)) + self.assertIsSubclass(_testcapi.HeapCTypeMetaclassCustomNew, type) msg = "Metaclasses with custom tp_new are not supported." with self.assertRaisesRegex(TypeError, msg): t = _testcapi.pytype_fromspec_meta(metaclass) - def test_heaptype_with_custom_metaclass_deprecation(self): + def test_heaptype_base_with_custom_metaclass(self): metaclass = _testcapi.HeapCTypeMetaclassCustomNew - # gh-103968: a metaclass with custom tp_new is deprecated, but still - # allowed for functions that existed in 3.11 - # (PyType_FromSpecWithBases is used here). class Base(metaclass=metaclass): pass # Class creation from C - with warnings_helper.check_warnings( - ('.* _testcapi.Subclass .* custom tp_new.*in Python 3.14.*', DeprecationWarning), - ): + msg = "Metaclasses with custom tp_new are not supported." + with self.assertRaisesRegex(TypeError, msg): sub = _testcapi.make_type_with_base(Base) - self.assertTrue(issubclass(sub, Base)) - self.assertIsInstance(sub, metaclass) - def test_multiple_inheritance_ctypes_with_weakref_or_dict(self): + def test_heaptype_with_tp_vectorcall(self): + tp = _testcapi.HeapCTypeVectorcall + v0 = tp.__new__(tp) + v0.__init__() + v1 = tp() + self.assertEqual(v0.value, 2) + self.assertEqual(v1.value, 1) - with self.assertRaises(TypeError): - class Both1(_testcapi.HeapCTypeWithWeakref, _testcapi.HeapCTypeWithDict): - pass - with self.assertRaises(TypeError): - class Both2(_testcapi.HeapCTypeWithDict, _testcapi.HeapCTypeWithWeakref): - pass + def test_multiple_inheritance_ctypes_with_weakref_or_dict(self): + for weakref_cls in (_testcapi.HeapCTypeWithWeakref, + _testlimitedcapi.HeapCTypeWithRelativeWeakref): + for dict_cls in (_testcapi.HeapCTypeWithDict, + _testlimitedcapi.HeapCTypeWithRelativeDict): + with self.subTest(weakref_cls=weakref_cls, dict_cls=dict_cls): + + with self.assertRaises(TypeError): + class Both1(weakref_cls, dict_cls): + pass + with self.assertRaises(TypeError): + class Both2(dict_cls, weakref_cls): + pass def test_multiple_inheritance_ctypes_with_weakref_or_dict_and_other_builtin(self): + for dict_cls in (_testcapi.HeapCTypeWithDict, + _testlimitedcapi.HeapCTypeWithRelativeDict): + for weakref_cls in (_testcapi.HeapCTypeWithWeakref, + _testlimitedcapi.HeapCTypeWithRelativeWeakref): + with self.subTest(dict_cls=dict_cls, weakref_cls=weakref_cls): + + with self.assertRaises(TypeError): + class C1(dict_cls, list): + pass - with self.assertRaises(TypeError): - class C1(_testcapi.HeapCTypeWithDict, list): - pass - - with self.assertRaises(TypeError): - class C2(_testcapi.HeapCTypeWithWeakref, list): - pass + with self.assertRaises(TypeError): + class C2(weakref_cls, list): + pass - class C3(_testcapi.HeapCTypeWithManagedDict, list): - pass - class C4(_testcapi.HeapCTypeWithManagedWeakref, list): - pass + class C3(_testcapi.HeapCTypeWithManagedDict, list): + pass + class C4(_testcapi.HeapCTypeWithManagedWeakref, list): + pass - inst = C3() - inst.append(0) - str(inst.__dict__) + inst = C3() + inst.append(0) + str(inst.__dict__) - inst = C4() - inst.append(0) - str(inst.__weakref__) + inst = C4() + inst.append(0) + str(inst.__weakref__) - for cls in (_testcapi.HeapCTypeWithManagedDict, _testcapi.HeapCTypeWithManagedWeakref): - for cls2 in (_testcapi.HeapCTypeWithDict, _testcapi.HeapCTypeWithWeakref): - class S(cls, cls2): - pass - class B1(C3, cls): - pass - class B2(C4, cls): - pass + for cls in (_testcapi.HeapCTypeWithManagedDict, + _testcapi.HeapCTypeWithManagedWeakref): + for cls2 in (dict_cls, weakref_cls): + class S(cls, cls2): + pass + class B1(C3, cls): + pass + class B2(C4, cls): + pass def test_pytype_fromspec_with_repeated_slots(self): for variant in range(2): @@ -771,33 +836,11 @@ def test_pytype_fromspec_with_repeated_slots(self): with self.assertRaises(SystemError): _testcapi.create_type_from_repeated_slots(variant) - @warnings_helper.ignore_warnings(category=DeprecationWarning) def test_immutable_type_with_mutable_base(self): - # Add deprecation warning here so it's removed in 3.14 - warnings._deprecated( - 'creating immutable classes with mutable bases', remove=(3, 14)) - - class MutableBase: - def meth(self): - return 'original' - - with self.assertWarns(DeprecationWarning): - ImmutableSubclass = _testcapi.make_immutable_type_with_base( - MutableBase) - instance = ImmutableSubclass() + class MutableBase: ... - self.assertEqual(instance.meth(), 'original') - - # Cannot override the static type's method - with self.assertRaisesRegex( - TypeError, - "cannot set 'meth' attribute of immutable type"): - ImmutableSubclass.meth = lambda self: 'overridden' - self.assertEqual(instance.meth(), 'original') - - # Can change the method on the mutable base - MutableBase.meth = lambda self: 'changed' - self.assertEqual(instance.meth(), 'changed') + with self.assertRaisesRegex(TypeError, 'Creating immutable type'): + _testcapi.make_immutable_type_with_base(MutableBase) def test_pynumber_tobase(self): from _testcapi import pynumber_tobase @@ -869,8 +912,7 @@ def test_export_symbols(self): names.append('Py_FrozenMain') for name in names: - with self.subTest(name=name): - self.assertTrue(hasattr(ctypes.pythonapi, name)) + self.assertHasAttr(ctypes.pythonapi, name) def test_clear_managed_dict(self): @@ -886,36 +928,6 @@ def __init__(self): _testcapi.clear_managed_dict(c) self.assertEqual(c.__dict__, {}) - def test_eval_get_func_name(self): - def function_example(): ... - - class A: - def method_example(self): ... - - self.assertEqual(_testcapi.eval_get_func_name(function_example), - "function_example") - self.assertEqual(_testcapi.eval_get_func_name(A.method_example), - "method_example") - self.assertEqual(_testcapi.eval_get_func_name(A().method_example), - "method_example") - self.assertEqual(_testcapi.eval_get_func_name(sum), "sum") # c function - self.assertEqual(_testcapi.eval_get_func_name(A), "type") - - def test_eval_get_func_desc(self): - def function_example(): ... - - class A: - def method_example(self): ... - - self.assertEqual(_testcapi.eval_get_func_desc(function_example), - "()") - self.assertEqual(_testcapi.eval_get_func_desc(A.method_example), - "()") - self.assertEqual(_testcapi.eval_get_func_desc(A().method_example), - "()") - self.assertEqual(_testcapi.eval_get_func_desc(sum), "()") # c function - self.assertEqual(_testcapi.eval_get_func_desc(A), " object") - def test_function_get_code(self): import types @@ -1099,21 +1111,152 @@ class Data(_testcapi.ObjExtraData): del d.extra self.assertIsNone(d.extra) - def test_get_type_module_name(self): + def test_get_type_name(self): + class MyType: + pass + + from _testcapi import ( + get_type_name, get_type_qualname, + get_type_fullyqualname, get_type_module_name) + from collections import OrderedDict ht = _testcapi.get_heaptype_for_name() - for cls, expected in { - int: 'builtins', - OrderedDict: 'collections', - ht: '_testcapi', - }.items(): - with self.subTest(repr(cls)): - modname = _testinternalcapi.get_type_module_name(cls) - self.assertEqual(modname, expected) + for cls, fullname, modname, qualname, name in ( + (int, + 'int', + 'builtins', + 'int', + 'int'), + (OrderedDict, + 'collections.OrderedDict', + 'collections', + 'OrderedDict', + 'OrderedDict'), + (ht, + '_testcapi.HeapTypeNameType', + '_testcapi', + 'HeapTypeNameType', + 'HeapTypeNameType'), + (MyType, + f'{__name__}.CAPITest.test_get_type_name..MyType', + __name__, + 'CAPITest.test_get_type_name..MyType', + 'MyType'), + ): + with self.subTest(cls=repr(cls)): + self.assertEqual(get_type_fullyqualname(cls), fullname) + self.assertEqual(get_type_module_name(cls), modname) + self.assertEqual(get_type_qualname(cls), qualname) + self.assertEqual(get_type_name(cls), name) + # override __module__ ht.__module__ = 'test_module' - modname = _testinternalcapi.get_type_module_name(ht) - self.assertEqual(modname, 'test_module') + self.assertEqual(get_type_fullyqualname(ht), 'test_module.HeapTypeNameType') + self.assertEqual(get_type_module_name(ht), 'test_module') + self.assertEqual(get_type_qualname(ht), 'HeapTypeNameType') + self.assertEqual(get_type_name(ht), 'HeapTypeNameType') + + # override __name__ and __qualname__ + MyType.__name__ = 'my_name' + MyType.__qualname__ = 'my_qualname' + self.assertEqual(get_type_fullyqualname(MyType), f'{__name__}.my_qualname') + self.assertEqual(get_type_module_name(MyType), __name__) + self.assertEqual(get_type_qualname(MyType), 'my_qualname') + self.assertEqual(get_type_name(MyType), 'my_name') + + # override also __module__ + MyType.__module__ = 'my_module' + self.assertEqual(get_type_fullyqualname(MyType), 'my_module.my_qualname') + self.assertEqual(get_type_module_name(MyType), 'my_module') + self.assertEqual(get_type_qualname(MyType), 'my_qualname') + self.assertEqual(get_type_name(MyType), 'my_name') + + # PyType_GetFullyQualifiedName() ignores the module if it's "builtins" + # or "__main__" of it is not a string + MyType.__module__ = 'builtins' + self.assertEqual(get_type_fullyqualname(MyType), 'my_qualname') + MyType.__module__ = '__main__' + self.assertEqual(get_type_fullyqualname(MyType), 'my_qualname') + MyType.__module__ = 123 + self.assertEqual(get_type_fullyqualname(MyType), 'my_qualname') + + def test_get_base_by_token(self): + def get_base_by_token(src, key, comparable=True): + def run(use_mro): + find_first = _testcapi.pytype_getbasebytoken + ret1, result = find_first(src, key, use_mro, True) + ret2, no_result = find_first(src, key, use_mro, False) + self.assertIn(ret1, (0, 1)) + self.assertEqual(ret1, result is not None) + self.assertEqual(ret1, ret2) + self.assertIsNone(no_result) + return result + + found_in_mro = run(True) + found_in_bases = run(False) + if comparable: + self.assertIs(found_in_mro, found_in_bases) + return found_in_mro + return found_in_mro, found_in_bases + + create_type = _testcapi.create_type_with_token + get_token = _testcapi.get_tp_token + + Py_TP_USE_SPEC = _testcapi.Py_TP_USE_SPEC + self.assertEqual(Py_TP_USE_SPEC, 0) + + A1 = create_type('_testcapi.A1', Py_TP_USE_SPEC) + self.assertTrue(get_token(A1) != Py_TP_USE_SPEC) + + B1 = create_type('_testcapi.B1', id(self)) + self.assertTrue(get_token(B1) == id(self)) + + tokenA1 = get_token(A1) + # find A1 from A1 + found = get_base_by_token(A1, tokenA1) + self.assertIs(found, A1) + + # no token in static types + STATIC = type(1) + self.assertEqual(get_token(STATIC), 0) + found = get_base_by_token(STATIC, tokenA1) + self.assertIs(found, None) + + # no token in pure subtypes + class A2(A1): pass + self.assertEqual(get_token(A2), 0) + # find A1 + class Z(STATIC, B1, A2): pass + found = get_base_by_token(Z, tokenA1) + self.assertIs(found, A1) + + # searching for NULL token is an error + with self.assertRaises(SystemError): + get_base_by_token(Z, 0) + with self.assertRaises(SystemError): + get_base_by_token(STATIC, 0) + + # share the token with A1 + C1 = create_type('_testcapi.C1', tokenA1) + self.assertTrue(get_token(C1) == tokenA1) + + # find C1 first by shared token + class Z(C1, A2): pass + found = get_base_by_token(Z, tokenA1) + self.assertIs(found, C1) + # B1 not found + found = get_base_by_token(Z, get_token(B1)) + self.assertIs(found, None) + + with self.assertRaises(TypeError): + _testcapi.pytype_getbasebytoken( + 'not a type', id(self), True, False) + + def test_gen_get_code(self): + def genf(): yield + gen = genf() + self.assertEqual(_testcapi.gen_get_code(gen), gen.gi_code) + @requires_limited_api class TestHeapTypeRelative(unittest.TestCase): @@ -1124,7 +1267,7 @@ def test_heaptype_relative_sizes(self): # Test subclassing using "relative" basicsize, see PEP 697 def check(extra_base_size, extra_size): Base, Sub, instance, data_ptr, data_offset, data_size = ( - _testcapi.make_sized_heaptypes( + _testlimitedcapi.make_sized_heaptypes( extra_base_size, -extra_size)) # no alignment shenanigans when inheriting directly @@ -1152,11 +1295,11 @@ def check(extra_base_size, extra_size): # we don't reserve (requested + alignment) or more data self.assertLess(data_size - extra_size, - _testcapi.ALIGNOF_MAX_ALIGN_T) + _testlimitedcapi.ALIGNOF_MAX_ALIGN_T) # The offsets/sizes we calculated should be aligned. - self.assertEqual(data_offset % _testcapi.ALIGNOF_MAX_ALIGN_T, 0) - self.assertEqual(data_size % _testcapi.ALIGNOF_MAX_ALIGN_T, 0) + self.assertEqual(data_offset % _testlimitedcapi.ALIGNOF_MAX_ALIGN_T, 0) + self.assertEqual(data_size % _testlimitedcapi.ALIGNOF_MAX_ALIGN_T, 0) sizes = sorted({0, 1, 2, 3, 4, 7, 8, 123, object.__basicsize__, @@ -1182,7 +1325,7 @@ def test_heaptype_inherit_itemsize(self): object.__basicsize__+1}) for extra_size in sizes: with self.subTest(extra_size=extra_size): - Sub = _testcapi.subclass_var_heaptype( + Sub = _testlimitedcapi.subclass_var_heaptype( _testcapi.HeapCCollection, -extra_size, 0, 0) collection = Sub(1, 2, 3) collection.set_data_to_3s() @@ -1196,7 +1339,7 @@ def test_heaptype_invalid_inheritance(self): with self.assertRaises(SystemError, msg="Cannot extend variable-size class without " + "Py_TPFLAGS_ITEMS_AT_END"): - _testcapi.subclass_heaptype(int, -8, 0) + _testlimitedcapi.subclass_heaptype(int, -8, 0) def test_heaptype_relative_members(self): """Test HeapCCollection subclasses work properly""" @@ -1209,7 +1352,7 @@ def test_heaptype_relative_members(self): for offset in sizes: with self.subTest(extra_base_size=extra_base_size, extra_size=extra_size, offset=offset): if offset < extra_size: - Sub = _testcapi.make_heaptype_with_member( + Sub = _testlimitedcapi.make_heaptype_with_member( extra_base_size, -extra_size, offset, True) Base = Sub.mro()[1] instance = Sub() @@ -1228,29 +1371,29 @@ def test_heaptype_relative_members(self): instance.set_memb_relative(0) else: with self.assertRaises(SystemError): - Sub = _testcapi.make_heaptype_with_member( + Sub = _testlimitedcapi.make_heaptype_with_member( extra_base_size, -extra_size, offset, True) with self.assertRaises(SystemError): - Sub = _testcapi.make_heaptype_with_member( + Sub = _testlimitedcapi.make_heaptype_with_member( extra_base_size, extra_size, offset, True) with self.subTest(extra_base_size=extra_base_size, extra_size=extra_size): with self.assertRaises(SystemError): - Sub = _testcapi.make_heaptype_with_member( + Sub = _testlimitedcapi.make_heaptype_with_member( extra_base_size, -extra_size, -1, True) def test_heaptype_relative_members_errors(self): with self.assertRaisesRegex( SystemError, r"With Py_RELATIVE_OFFSET, basicsize must be negative"): - _testcapi.make_heaptype_with_member(0, 1234, 0, True) + _testlimitedcapi.make_heaptype_with_member(0, 1234, 0, True) with self.assertRaisesRegex( SystemError, r"Member offset out of range \(0\.\.-basicsize\)"): - _testcapi.make_heaptype_with_member(0, -8, 1234, True) + _testlimitedcapi.make_heaptype_with_member(0, -8, 1234, True) with self.assertRaisesRegex( SystemError, r"Member offset out of range \(0\.\.-basicsize\)"): - _testcapi.make_heaptype_with_member(0, -8, -1, True) + _testlimitedcapi.make_heaptype_with_member(0, -8, -1, True) - Sub = _testcapi.make_heaptype_with_member(0, -8, 0, True) + Sub = _testlimitedcapi.make_heaptype_with_member(0, -8, 0, True) instance = Sub() with self.assertRaisesRegex( SystemError, r"PyMember_GetOne used with Py_RELATIVE_OFFSET"): @@ -1259,6 +1402,53 @@ def test_heaptype_relative_members_errors(self): SystemError, r"PyMember_SetOne used with Py_RELATIVE_OFFSET"): instance.set_memb_relative(0) + def test_heaptype_relative_special_members_errors(self): + for member_name in "__vectorcalloffset__", "__dictoffset__", "__weaklistoffset__": + with self.subTest(member_name=member_name): + with self.assertRaisesRegex( + SystemError, + r"With Py_RELATIVE_OFFSET, basicsize must be negative."): + _testlimitedcapi.make_heaptype_with_member( + basicsize=sys.getsizeof(object()) + 100, + add_relative_flag=True, + member_name=member_name, + member_offset=0, + member_type=_testlimitedcapi.Py_T_PYSSIZET, + member_flags=_testlimitedcapi.Py_READONLY, + ) + with self.assertRaisesRegex( + SystemError, + r"Member offset out of range \(0\.\.-basicsize\)"): + _testlimitedcapi.make_heaptype_with_member( + basicsize=-8, + add_relative_flag=True, + member_name=member_name, + member_offset=-1, + member_type=_testlimitedcapi.Py_T_PYSSIZET, + member_flags=_testlimitedcapi.Py_READONLY, + ) + with self.assertRaisesRegex( + SystemError, + r"type of %s must be Py_T_PYSSIZET" % member_name): + _testlimitedcapi.make_heaptype_with_member( + basicsize=-100, + add_relative_flag=True, + member_name=member_name, + member_offset=0, + member_flags=_testlimitedcapi.Py_READONLY, + ) + with self.assertRaisesRegex( + SystemError, + r"flags for %s must be " % member_name): + _testlimitedcapi.make_heaptype_with_member( + basicsize=-100, + add_relative_flag=True, + member_name=member_name, + member_offset=0, + member_type=_testlimitedcapi.Py_T_PYSSIZET, + member_flags=0, + ) + def test_pyobject_getitemdata_error(self): """Test PyObject_GetItemData fails on unsupported types""" with self.assertRaises(TypeError): @@ -1270,13 +1460,134 @@ def test_pyobject_getitemdata_error(self): _testcapi.pyobject_getitemdata(0) + def test_function_get_closure(self): + from types import CellType + + def regular_function(): ... + def unused_one_level(arg1): + def inner(arg2, arg3): ... + return inner + def unused_two_levels(arg1, arg2): + def decorator(arg3, arg4): + def inner(arg5, arg6): ... + return inner + return decorator + def with_one_level(arg1): + def inner(arg2, arg3): + return arg1 + arg2 + arg3 + return inner + def with_two_levels(arg1, arg2): + def decorator(arg3, arg4): + def inner(arg5, arg6): + return arg1 + arg2 + arg3 + arg4 + arg5 + arg6 + return inner + return decorator + + # Functions without closures: + self.assertIsNone(_testcapi.function_get_closure(regular_function)) + self.assertIsNone(regular_function.__closure__) + + func = unused_one_level(1) + closure = _testcapi.function_get_closure(func) + self.assertIsNone(closure) + self.assertIsNone(func.__closure__) + + func = unused_two_levels(1, 2)(3, 4) + closure = _testcapi.function_get_closure(func) + self.assertIsNone(closure) + self.assertIsNone(func.__closure__) + + # Functions with closures: + func = with_one_level(5) + closure = _testcapi.function_get_closure(func) + self.assertEqual(closure, func.__closure__) + self.assertIsInstance(closure, tuple) + self.assertEqual(len(closure), 1) + self.assertEqual(len(closure), len(func.__code__.co_freevars)) + for cell in closure: + self.assertIsInstance(cell, CellType) + self.assertTrue(closure[0].cell_contents, 5) + + func = with_two_levels(1, 2)(3, 4) + closure = _testcapi.function_get_closure(func) + self.assertEqual(closure, func.__closure__) + self.assertIsInstance(closure, tuple) + self.assertEqual(len(closure), 4) + self.assertEqual(len(closure), len(func.__code__.co_freevars)) + for cell in closure: + self.assertIsInstance(cell, CellType) + self.assertEqual([cell.cell_contents for cell in closure], + [1, 2, 3, 4]) + + def test_function_get_closure_error(self): + with self.assertRaises(SystemError): + _testcapi.function_get_closure(1) + with self.assertRaises(SystemError): + _testcapi.function_get_closure(None) + + def test_function_set_closure(self): + from types import CellType + + def function_without_closure(): ... + def function_with_closure(arg): + def inner(): + return arg + return inner + + func = function_without_closure + _testcapi.function_set_closure(func, (CellType(1), CellType(1))) + closure = _testcapi.function_get_closure(func) + self.assertEqual([c.cell_contents for c in closure], [1, 1]) + self.assertEqual([c.cell_contents for c in func.__closure__], [1, 1]) + + func = function_with_closure(1) + _testcapi.function_set_closure(func, + (CellType(1), CellType(2), CellType(3))) + closure = _testcapi.function_get_closure(func) + self.assertEqual([c.cell_contents for c in closure], [1, 2, 3]) + self.assertEqual([c.cell_contents for c in func.__closure__], [1, 2, 3]) + + def test_function_set_closure_none(self): + def function_without_closure(): ... + def function_with_closure(arg): + def inner(): + return arg + return inner + + _testcapi.function_set_closure(function_without_closure, None) + self.assertIsNone( + _testcapi.function_get_closure(function_without_closure)) + self.assertIsNone(function_without_closure.__closure__) + + _testcapi.function_set_closure(function_with_closure, None) + self.assertIsNone( + _testcapi.function_get_closure(function_with_closure)) + self.assertIsNone(function_with_closure.__closure__) + + def test_function_set_closure_errors(self): + def function_without_closure(): ... + + with self.assertRaises(SystemError): + _testcapi.function_set_closure(None, ()) # not a function + + with self.assertRaises(SystemError): + _testcapi.function_set_closure(function_without_closure, 1) + self.assertIsNone(function_without_closure.__closure__) # no change + + # NOTE: this works, but goes against the docs: + _testcapi.function_set_closure(function_without_closure, (1, 2)) + self.assertEqual( + _testcapi.function_get_closure(function_without_closure), (1, 2)) + self.assertEqual(function_without_closure.__closure__, (1, 2)) + + class TestPendingCalls(unittest.TestCase): # See the comment in ceval.c (at the "handle_eval_breaker" label) # about when pending calls get run. This is especially relevant # here for creating deterministic tests. - def pendingcalls_submit(self, l, n): + def main_pendingcalls_submit(self, l, n): def callback(): #this function can be interrupted by thread switching so let's #use an atomic operation @@ -1291,12 +1602,27 @@ def callback(): if _testcapi._pending_threadfunc(callback): break - def pendingcalls_wait(self, l, n, context = None): + def pendingcalls_submit(self, l, n, *, main=True, ensure=False): + def callback(): + #this function can be interrupted by thread switching so let's + #use an atomic operation + l.append(None) + + if main: + return _testcapi._pending_threadfunc(callback, n, + blocking=False, + ensure_added=ensure) + else: + return _testinternalcapi.pending_threadfunc(callback, n, + blocking=False, + ensure_added=ensure) + + def pendingcalls_wait(self, l, numadded, context = None): #now, stick around until l[0] has grown to 10 count = 0 - while len(l) != n: + while len(l) != numadded: #this busy loop is where we expect to be interrupted to - #run our callbacks. Note that callbacks are only run on the + #run our callbacks. Note that some callbacks are only run on the #main thread if False and support.verbose: print("(%i)"%(len(l),),) @@ -1306,12 +1632,12 @@ def pendingcalls_wait(self, l, n, context = None): continue count += 1 self.assertTrue(count < 10000, - "timeout waiting for %i callbacks, got %i"%(n, len(l))) + "timeout waiting for %i callbacks, got %i"%(numadded, len(l))) if False and support.verbose: print("(%i)"%(len(l),)) @threading_helper.requires_working_threading() - def test_pendingcalls_threaded(self): + def test_main_pendingcalls_threaded(self): #do every callback on a separate thread n = 32 #total callbacks @@ -1325,15 +1651,15 @@ class foo(object):pass context.lock = threading.Lock() context.event = threading.Event() - threads = [threading.Thread(target=self.pendingcalls_thread, + threads = [threading.Thread(target=self.main_pendingcalls_thread, args=(context,)) for i in range(context.nThreads)] with threading_helper.start_threads(threads): self.pendingcalls_wait(context.l, n, context) - def pendingcalls_thread(self, context): + def main_pendingcalls_thread(self, context): try: - self.pendingcalls_submit(context.l, context.n) + self.main_pendingcalls_submit(context.l, context.n) finally: with context.lock: context.nFinished += 1 @@ -1343,20 +1669,54 @@ def pendingcalls_thread(self, context): if nFinished == context.nThreads: context.event.set() - def test_pendingcalls_non_threaded(self): + def test_main_pendingcalls_non_threaded(self): #again, just using the main thread, likely they will all be dispatched at #once. It is ok to ask for too many, because we loop until we find a slot. #the loop can be interrupted to dispatch. #there are only 32 dispatch slots, so we go for twice that! l = [] n = 64 - self.pendingcalls_submit(l, n) + self.main_pendingcalls_submit(l, n) self.pendingcalls_wait(l, n) - def test_gen_get_code(self): - def genf(): yield - gen = genf() - self.assertEqual(_testcapi.gen_get_code(gen), gen.gi_code) + def test_max_pending(self): + with self.subTest('main-only'): + maxpending = 32 + + l = [] + added = self.pendingcalls_submit(l, 1, main=True) + self.pendingcalls_wait(l, added) + self.assertEqual(added, 1) + + l = [] + added = self.pendingcalls_submit(l, maxpending, main=True) + self.pendingcalls_wait(l, added) + self.assertEqual(added, maxpending) + + l = [] + added = self.pendingcalls_submit(l, maxpending+1, main=True) + self.pendingcalls_wait(l, added) + self.assertEqual(added, maxpending) + + with self.subTest('not main-only'): + # Per-interpreter pending calls has a much higher limit + # on how many may be pending at a time. + maxpending = 300 + + l = [] + added = self.pendingcalls_submit(l, 1, main=False) + self.pendingcalls_wait(l, added) + self.assertEqual(added, 1) + + l = [] + added = self.pendingcalls_submit(l, maxpending, main=False) + self.pendingcalls_wait(l, added) + self.assertEqual(added, maxpending) + + l = [] + added = self.pendingcalls_submit(l, maxpending+1, main=False) + self.pendingcalls_wait(l, added) + self.assertEqual(added, maxpending) class PendingTask(types.SimpleNamespace): @@ -1784,27 +2144,27 @@ async def foo(arg): return await arg # Py 3.5 self.assertEqual(ret, 0) self.assertEqual(pickle.load(f), {'a': '123x', 'b': '123'}) + # _testcapi cannot be imported in a subinterpreter on a Free Threaded build + @support.requires_gil_enabled() def test_py_config_isoloated_per_interpreter(self): # A config change in one interpreter must not leak to out to others. # # This test could verify ANY config value, it just happens to have been # written around the time of int_max_str_digits. Refactoring is okay. code = """if 1: - import sys, _testinternalcapi + import sys, _testcapi # Any config value would do, this happens to be the one being # double checked at the time this test was written. - config = _testinternalcapi.get_config() - config['int_max_str_digits'] = 55555 - _testinternalcapi.set_config(config) - sub_value = _testinternalcapi.get_config()['int_max_str_digits'] + _testcapi.config_set('int_max_str_digits', 55555) + sub_value = _testcapi.config_get('int_max_str_digits') assert sub_value == 55555, sub_value """ - before_config = _testinternalcapi.get_config() - assert before_config['int_max_str_digits'] != 55555 + before_config = _testcapi.config_get('int_max_str_digits') + assert before_config != 55555 self.assertEqual(support.run_in_subinterp(code), 0, 'subinterp code failure, check stderr.') - after_config = _testinternalcapi.get_config() + after_config = _testcapi.config_get('int_max_str_digits') self.assertIsNot( before_config, after_config, "Expected get_config() to return a new dict on each call") @@ -1817,6 +2177,7 @@ def test_py_config_isoloated_per_interpreter(self): # test fails, assume that the environment in this process may # be altered and suspect. + @requires_subinterpreters @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_configured_settings(self): """ @@ -1848,15 +2209,30 @@ def test_configured_settings(self): kwlist[-2] = 'check_multi_interp_extensions' kwlist[-1] = 'own_gil' - # expected to work - for config, expected in { + expected_to_work = { (True, True, True, True, True, True, True): (ALL_FLAGS, True), (True, False, False, False, False, False, False): (OBMALLOC, False), (False, False, False, True, False, True, False): (THREADS | EXTENSIONS, False), - }.items(): + } + + expected_to_fail = { + (False, False, False, False, False, False, False), + } + + # gh-117649: The free-threaded build does not currently allow + # setting check_multi_interp_extensions to False. + if Py_GIL_DISABLED: + for config in list(expected_to_work.keys()): + kwargs = dict(zip(kwlist, config)) + if not kwargs['check_multi_interp_extensions']: + del expected_to_work[config] + expected_to_fail.add(config) + + # expected to work + for config, expected in expected_to_work.items(): kwargs = dict(zip(kwlist, config)) exp_flags, exp_gil = expected expected = { @@ -1880,9 +2256,7 @@ def test_configured_settings(self): self.assertEqual(settings, expected) # expected to fail - for config in [ - (False, False, False, False, False, False, False), - ]: + for config in expected_to_fail: kwargs = dict(zip(kwlist, config)) with self.subTest(config): script = textwrap.dedent(f''' @@ -1890,11 +2264,14 @@ def test_configured_settings(self): _testinternalcapi.get_interp_settings() raise NotImplementedError('unreachable') ''') - with self.assertRaises(RuntimeError): + with self.assertRaises(_interpreters.InterpreterError): support.run_in_subinterp_with_config(script, **kwargs) @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + # gh-117649: The free-threaded build does not currently allow overriding + # the check_multi_interp_extensions setting. + @expected_failure_if_gil_disabled() def test_overridden_setting_extensions_subinterp_check(self): """ PyInterpreterConfig.check_multi_interp_extensions can be overridden @@ -1946,6 +2323,9 @@ def check(enabled, override): } r, w = os.pipe() + if Py_GIL_DISABLED: + # gh-117649: The test fails before `w` is closed + self.addCleanup(os.close, w) script = textwrap.dedent(f''' from test.test_capi.check_config import run_singlephase_check run_singlephase_check({override}, {w}) @@ -1987,9 +2367,12 @@ def test_mutate_exception(self): support.run_in_subinterp("import binascii; binascii.Error.foobar = 'foobar'") - self.assertFalse(hasattr(binascii.Error, "foobar")) + self.assertNotHasAttr(binascii.Error, "foobar") @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + # gh-117649: The free-threaded build does not currently support sharing + # extension module state between interpreters. + @expected_failure_if_gil_disabled() def test_module_state_shared_in_global(self): """ bpo-44050: Extension module state should be shared between interpreters @@ -1999,6 +2382,13 @@ def test_module_state_shared_in_global(self): self.addCleanup(os.close, r) self.addCleanup(os.close, w) + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if support.is_apple_mobile: + loader = "AppleFrameworkLoader" + else: + loader = "ExtensionFileLoader" + script = textwrap.dedent(f""" import importlib.machinery import importlib.util @@ -2006,7 +2396,7 @@ def test_module_state_shared_in_global(self): fullname = '_test_module_state_shared' origin = importlib.util.find_spec('_testmultiphase').origin - loader = importlib.machinery.ExtensionFileLoader(fullname, origin) + loader = importlib.machinery.{loader}(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) attr_id = str(id(module.Error)).encode() @@ -2023,134 +2413,513 @@ def test_module_state_shared_in_global(self): @requires_subinterpreters -class InterpreterIDTests(unittest.TestCase): +class InterpreterConfigTests(unittest.TestCase): + + supported = { + 'isolated': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + check_multi_interp_extensions=True, + gil='own', + ), + 'legacy': types.SimpleNamespace( + use_main_obmalloc=True, + allow_fork=True, + allow_exec=True, + allow_threads=True, + allow_daemon_threads=True, + check_multi_interp_extensions=bool(Py_GIL_DISABLED), + gil='shared', + ), + 'empty': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=False, + allow_daemon_threads=False, + check_multi_interp_extensions=False, + gil='default', + ), + } + gil_supported = ['default', 'shared', 'own'] + + def iter_all_configs(self): + for use_main_obmalloc in (True, False): + for allow_fork in (True, False): + for allow_exec in (True, False): + for allow_threads in (True, False): + for allow_daemon in (True, False): + for checkext in (True, False): + for gil in ('shared', 'own', 'default'): + yield types.SimpleNamespace( + use_main_obmalloc=use_main_obmalloc, + allow_fork=allow_fork, + allow_exec=allow_exec, + allow_threads=allow_threads, + allow_daemon_threads=allow_daemon, + check_multi_interp_extensions=checkext, + gil=gil, + ) + + def assert_ns_equal(self, ns1, ns2, msg=None): + # This is mostly copied from TestCase.assertDictEqual. + self.assertEqual(type(ns1), type(ns2)) + if ns1 == ns2: + return + + import difflib + import pprint + from unittest.util import _common_shorten_repr + standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2) + diff = ('\n' + '\n'.join(difflib.ndiff( + pprint.pformat(vars(ns1)).splitlines(), + pprint.pformat(vars(ns2)).splitlines()))) + diff = f'namespace({diff})' + standardMsg = self._truncateMessage(standardMsg, diff) + self.fail(self._formatMessage(msg, standardMsg)) + + def test_predefined_config(self): + def check(name, expected): + expected = self.supported[expected] + args = (name,) if name else () + + config1 = _interpreters.new_config(*args) + self.assert_ns_equal(config1, expected) + self.assertIsNot(config1, expected) + + config2 = _interpreters.new_config(*args) + self.assert_ns_equal(config2, expected) + self.assertIsNot(config2, expected) + self.assertIsNot(config2, config1) + + with self.subTest('default'): + check(None, 'isolated') + + for name in self.supported: + with self.subTest(name): + check(name, name) + + def test_update_from_dict(self): + for name, vanilla in self.supported.items(): + with self.subTest(f'noop ({name})'): + expected = vanilla + overrides = vars(vanilla) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest(f'change all ({name})'): + overrides = {k: not v for k, v in vars(vanilla).items()} + for gil in self.gil_supported: + if vanilla.gil == gil: + continue + overrides['gil'] = gil + expected = types.SimpleNamespace(**overrides) + config = _interpreters.new_config( + name, **overrides) + self.assert_ns_equal(config, expected) + + # Override individual fields. + for field, old in vars(vanilla).items(): + if field == 'gil': + values = [v for v in self.gil_supported if v != old] + else: + values = [not old] + for val in values: + with self.subTest(f'{name}.{field} ({old!r} -> {val!r})'): + overrides = {field: val} + expected = types.SimpleNamespace( + **dict(vars(vanilla), **overrides), + ) + config = _interpreters.new_config( + name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest('unsupported field'): + for name in self.supported: + with self.assertRaises(ValueError): + _interpreters.new_config(name, spam=True) + + # Bad values for bool fields. + for field, value in vars(self.supported['empty']).items(): + if field == 'gil': + continue + assert isinstance(value, bool) + for value in [1, '', 'spam', 1.0, None, object()]: + with self.subTest(f'unsupported value ({field}={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(**{field: value}) + + # Bad values for .gil. + for value in [True, 1, 1.0, None, object()]: + with self.subTest(f'unsupported value(gil={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(gil=value) + for value in ['', 'spam']: + with self.subTest(f'unsupported value (gil={value!r})'): + with self.assertRaises(ValueError): + _interpreters.new_config(gil=value) + + def test_interp_init(self): + questionable = [ + # strange + dict( + allow_fork=True, + allow_exec=False, + ), + dict( + gil='shared', + use_main_obmalloc=False, + ), + # risky + dict( + allow_fork=True, + allow_threads=True, + ), + # ought to be invalid? + dict( + allow_threads=False, + allow_daemon_threads=True, + ), + dict( + gil='own', + use_main_obmalloc=True, + ), + ] + invalid = [ + dict( + use_main_obmalloc=False, + check_multi_interp_extensions=False + ), + ] + if Py_GIL_DISABLED: + invalid.append(dict(check_multi_interp_extensions=False)) + def match(config, override_cases): + ns = vars(config) + for overrides in override_cases: + if dict(ns, **overrides) == ns: + return True + return False + + def check(config): + script = 'pass' + rc = _testinternalcapi.run_in_subinterp_with_config(script, config) + self.assertEqual(rc, 0) + + for config in self.iter_all_configs(): + if config.gil == 'default': + continue + if match(config, invalid): + with self.subTest(f'invalid: {config}'): + with self.assertRaises(_interpreters.InterpreterError): + check(config) + elif match(config, questionable): + with self.subTest(f'questionable: {config}'): + check(config) + else: + with self.subTest(f'valid: {config}'): + check(config) - InterpreterID = _testcapi.get_interpreterid_type() + def test_get_config(self): + @contextlib.contextmanager + def new_interp(config): + interpid = _interpreters.create(config, reqrefs=False) + try: + yield interpid + finally: + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass - def new_interpreter(self): - def ensure_destroyed(interpid): + with self.subTest('main'): + expected = _interpreters.new_config('legacy') + expected.gil = 'own' + if Py_GIL_DISABLED: + expected.check_multi_interp_extensions = False + interpid, *_ = _interpreters.get_main() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('isolated'): + expected = _interpreters.new_config('isolated') + with new_interp('isolated') as interpid: + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('legacy'): + expected = _interpreters.new_config('legacy') + with new_interp('legacy') as interpid: + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('custom'): + orig = _interpreters.new_config( + 'empty', + use_main_obmalloc=True, + gil='shared', + check_multi_interp_extensions=bool(Py_GIL_DISABLED), + ) + with new_interp(orig) as interpid: + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, orig) + + +@requires_subinterpreters +class InterpreterIDTests(unittest.TestCase): + + def add_interp_cleanup(self, interpid): + def ensure_destroyed(): try: _interpreters.destroy(interpid) except _interpreters.InterpreterNotFoundError: pass + self.addCleanup(ensure_destroyed) + + def new_interpreter(self): id = _interpreters.create() - self.addCleanup(lambda: ensure_destroyed(id)) + self.add_interp_cleanup(id) return id - def test_with_int(self): - id = self.InterpreterID(10, force=True) - - self.assertEqual(int(id), 10) + def test_conversion_int(self): + convert = _testinternalcapi.normalize_interp_id + interpid = convert(10) + self.assertEqual(interpid, 10) - def test_coerce_id(self): - class Int(str): + def test_conversion_coerced(self): + convert = _testinternalcapi.normalize_interp_id + class MyInt(str): def __index__(self): return 10 + interpid = convert(MyInt()) + self.assertEqual(interpid, 10) + + def test_conversion_from_interpreter(self): + convert = _testinternalcapi.normalize_interp_id + interpid = self.new_interpreter() + converted = convert(interpid) + self.assertEqual(converted, interpid) - id = self.InterpreterID(Int(), force=True) - self.assertEqual(int(id), 10) + def test_conversion_bad(self): + convert = _testinternalcapi.normalize_interp_id - def test_bad_id(self): for badid in [ object(), 10.0, '10', b'10', ]: - with self.subTest(badid): + with self.subTest(f'bad: {badid!r}'): with self.assertRaises(TypeError): - self.InterpreterID(badid) + convert(badid) badid = -1 - with self.subTest(badid): + with self.subTest(f'bad: {badid!r}'): with self.assertRaises(ValueError): - self.InterpreterID(badid) + convert(badid) badid = 2**64 - with self.subTest(badid): + with self.subTest(f'bad: {badid!r}'): with self.assertRaises(OverflowError): - self.InterpreterID(badid) + convert(badid) - def test_exists(self): - id = self.new_interpreter() - with self.assertRaises(_interpreters.InterpreterNotFoundError): - self.InterpreterID(int(id) + 1) # unforced + def test_lookup_exists(self): + interpid = self.new_interpreter() + self.assertTrue( + _testinternalcapi.interpreter_exists(interpid)) - def test_does_not_exist(self): - id = self.new_interpreter() - with self.assertRaises(_interpreters.InterpreterNotFoundError): - self.InterpreterID(int(id) + 1) # unforced + def test_lookup_does_not_exist(self): + interpid = _testinternalcapi.unused_interpreter_id() + self.assertFalse( + _testinternalcapi.interpreter_exists(interpid)) - def test_destroyed(self): - id = _interpreters.create() - _interpreters.destroy(id) - with self.assertRaises(_interpreters.InterpreterNotFoundError): - self.InterpreterID(id) # unforced - - def test_str(self): - id = self.InterpreterID(10, force=True) - self.assertEqual(str(id), '10') - - def test_repr(self): - id = self.InterpreterID(10, force=True) - self.assertEqual(repr(id), 'InterpreterID(10)') - - def test_equality(self): - id1 = self.new_interpreter() - id2 = self.InterpreterID(id1) - id3 = self.InterpreterID( - self.new_interpreter()) - - self.assertTrue(id2 == id2) # identity - self.assertTrue(id2 == id1) # int-equivalent - self.assertTrue(id1 == id2) # reversed - self.assertTrue(id2 == int(id2)) - self.assertTrue(id2 == float(int(id2))) - self.assertTrue(float(int(id2)) == id2) - self.assertFalse(id2 == float(int(id2)) + 0.1) - self.assertFalse(id2 == str(int(id2))) - self.assertFalse(id2 == 2**1000) - self.assertFalse(id2 == float('inf')) - self.assertFalse(id2 == 'spam') - self.assertFalse(id2 == id3) - - self.assertFalse(id2 != id2) - self.assertFalse(id2 != id1) - self.assertFalse(id1 != id2) - self.assertTrue(id2 != id3) - - def test_linked_lifecycle(self): - id1 = _interpreters.create() - _testcapi.unlink_interpreter_refcount(id1) + def test_lookup_destroyed(self): + interpid = _interpreters.create() + _interpreters.destroy(interpid) + self.assertFalse( + _testinternalcapi.interpreter_exists(interpid)) + + def get_refcount_helpers(self): + return ( + _testinternalcapi.get_interpreter_refcount, + (lambda id: _interpreters.incref(id, implieslink=False)), + _interpreters.decref, + ) + + def test_linked_lifecycle_does_not_exist(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + unlink = _testinternalcapi.unlink_interpreter_refcount + get_refcount, incref, decref = self.get_refcount_helpers() + + with self.subTest('never existed'): + interpid = _testinternalcapi.unused_interpreter_id() + self.assertFalse( + exists(interpid)) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + is_linked(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + link(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + unlink(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + get_refcount(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + incref(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + decref(interpid) + + with self.subTest('destroyed'): + interpid = _interpreters.create() + _interpreters.destroy(interpid) + self.assertFalse( + exists(interpid)) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + is_linked(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + link(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + unlink(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + get_refcount(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + incref(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + decref(interpid) + + def test_linked_lifecycle_initial(self): + is_linked = _testinternalcapi.interpreter_refcount_linked + get_refcount, _, _ = self.get_refcount_helpers() + + # A new interpreter will start out not linked, with a refcount of 0. + interpid = self.new_interpreter() + linked = is_linked(interpid) + refcount = get_refcount(interpid) + + self.assertFalse(linked) + self.assertEqual(refcount, 0) + + def test_linked_lifecycle_never_linked(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + get_refcount, incref, decref = self.get_refcount_helpers() + + interpid = self.new_interpreter() + + # Incref will not automatically link it. + incref(interpid) + self.assertFalse( + is_linked(interpid)) self.assertEqual( - _testinternalcapi.get_interpreter_refcount(id1), - 0) + 1, get_refcount(interpid)) + + # It isn't linked so it isn't destroyed. + decref(interpid) + self.assertTrue( + exists(interpid)) + self.assertFalse( + is_linked(interpid)) + self.assertEqual( + 0, get_refcount(interpid)) + + def test_linked_lifecycle_link_unlink(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + unlink = _testinternalcapi.unlink_interpreter_refcount + + interpid = self.new_interpreter() + + # Linking at refcount 0 does not destroy the interpreter. + link(interpid) + self.assertTrue( + exists(interpid)) + self.assertTrue( + is_linked(interpid)) + + # Unlinking at refcount 0 does not destroy the interpreter. + unlink(interpid) + self.assertTrue( + exists(interpid)) + self.assertFalse( + is_linked(interpid)) + + def test_linked_lifecycle_link_incref_decref(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + get_refcount, incref, decref = self.get_refcount_helpers() + + interpid = self.new_interpreter() + + # Linking it will not change the refcount. + link(interpid) + self.assertTrue( + is_linked(interpid)) + self.assertEqual( + 0, get_refcount(interpid)) - id2 = self.InterpreterID(id1) + # Decref with a refcount of 0 is not allowed. + incref(interpid) self.assertEqual( - _testinternalcapi.get_interpreter_refcount(id1), - 1) + 1, get_refcount(interpid)) + + # When linked, decref back to 0 destroys the interpreter. + decref(interpid) + self.assertFalse( + exists(interpid)) + + def test_linked_lifecycle_incref_link(self): + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + get_refcount, incref, _ = self.get_refcount_helpers() + + interpid = self.new_interpreter() - # The interpreter isn't linked to ID objects, so it isn't destroyed. - del id2 + incref(interpid) self.assertEqual( - _testinternalcapi.get_interpreter_refcount(id1), - 0) + 1, get_refcount(interpid)) - _testcapi.link_interpreter_refcount(id1) + # Linking it will not reset the refcount. + link(interpid) + self.assertTrue( + is_linked(interpid)) self.assertEqual( - _testinternalcapi.get_interpreter_refcount(id1), - 0) + 1, get_refcount(interpid)) - id3 = self.InterpreterID(id1) + def test_linked_lifecycle_link_incref_unlink_decref(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + unlink = _testinternalcapi.unlink_interpreter_refcount + get_refcount, incref, decref = self.get_refcount_helpers() + + interpid = self.new_interpreter() + + link(interpid) + self.assertTrue( + is_linked(interpid)) + + incref(interpid) self.assertEqual( - _testinternalcapi.get_interpreter_refcount(id1), - 1) + 1, get_refcount(interpid)) - # The interpreter is linked now so is destroyed. - del id3 - with self.assertRaises(_interpreters.InterpreterNotFoundError): - _testinternalcapi.get_interpreter_refcount(id1) + # Unlinking it will not change the refcount. + unlink(interpid) + self.assertFalse( + is_linked(interpid)) + self.assertEqual( + 1, get_refcount(interpid)) + + # Unlinked: decref back to 0 does not destroys the interpreter. + decref(interpid) + self.assertTrue( + exists(interpid)) + self.assertEqual( + 0, get_refcount(interpid)) class BuiltinStaticTypesTests(unittest.TestCase): @@ -2245,6 +3014,22 @@ def callback(): t.start() t.join() + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_thread_gilstate_in_clear(self): + # See https://github.com/python/cpython/issues/119585 + class C: + def __del__(self): + _testcapi.gilstate_ensure_release() + + # Thread-local variables are destroyed in `PyThreadState_Clear()`. + local_var = threading.local() + + def callback(): + local_var.x = C() + + _testcapi._test_thread_state(callback) + @threading_helper.reap_threads @threading_helper.requires_working_threading() def test_gilstate_ensure_no_deadlock(self): @@ -2264,25 +3049,36 @@ def test_gilstate_matches_current(self): _testcapi.test_current_tstate_matches() +def get_test_funcs(mod, exclude_prefix=None): + funcs = {} + for name in dir(mod): + if not name.startswith('test_'): + continue + if exclude_prefix is not None and name.startswith(exclude_prefix): + continue + funcs[name] = getattr(mod, name) + return funcs + + class Test_testcapi(unittest.TestCase): - locals().update((name, getattr(_testcapi, name)) - for name in dir(_testcapi) - if name.startswith('test_')) + locals().update(get_test_funcs(_testcapi)) # Suppress warning from PyUnicode_FromUnicode(). @warnings_helper.ignore_warnings(category=DeprecationWarning) def test_widechar(self): - _testcapi.test_widechar() + _testlimitedcapi.test_widechar() def test_version_api_data(self): self.assertEqual(_testcapi.Py_Version, sys.hexversion) +class Test_testlimitedcapi(unittest.TestCase): + locals().update(get_test_funcs(_testlimitedcapi)) + + class Test_testinternalcapi(unittest.TestCase): - locals().update((name, getattr(_testinternalcapi, name)) - for name in dir(_testinternalcapi) - if name.startswith('test_') - and not name.startswith('test_lock_')) + locals().update(get_test_funcs(_testinternalcapi, + exclude_prefix='test_lock_')) @threading_helper.requires_working_threading() @@ -2303,7 +3099,12 @@ class Test_ModuleStateAccess(unittest.TestCase): def setUp(self): fullname = '_testmultiphase_meth_state_access' # XXX origin = importlib.util.find_spec('_testmultiphase').origin - loader = importlib.machinery.ExtensionFileLoader(fullname, origin) + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if support.is_apple_mobile: + loader = importlib.machinery.AppleFrameworkLoader(fullname, origin) + else: + loader = importlib.machinery.ExtensionFileLoader(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) @@ -2538,6 +3339,49 @@ def run(self): self.assertEqual(len(set(py_thread_ids)), len(py_thread_ids), py_thread_ids) +class TestVersions(unittest.TestCase): + full_cases = ( + (3, 4, 1, 0xA, 2, 0x030401a2), + (3, 10, 0, 0xF, 0, 0x030a00f0), + (0x103, 0x10B, 0xFF00, -1, 0xF0, 0x030b00f0), # test masking + ) + xy_cases = ( + (3, 4, 0x03040000), + (3, 10, 0x030a0000), + (0x103, 0x10B, 0x030b0000), # test masking + ) + + def test_pack_full_version(self): + for *args, expected in self.full_cases: + with self.subTest(hexversion=hex(expected)): + result = _testlimitedcapi.pack_full_version(*args) + self.assertEqual(result, expected) + + def test_pack_version(self): + for *args, expected in self.xy_cases: + with self.subTest(hexversion=hex(expected)): + result = _testlimitedcapi.pack_version(*args) + self.assertEqual(result, expected) + + def test_pack_full_version_ctypes(self): + ctypes = import_helper.import_module('ctypes') + ctypes_func = ctypes.pythonapi.Py_PACK_FULL_VERSION + ctypes_func.restype = ctypes.c_uint32 + ctypes_func.argtypes = [ctypes.c_int] * 5 + for *args, expected in self.full_cases: + with self.subTest(hexversion=hex(expected)): + result = ctypes_func(*args) + self.assertEqual(result, expected) + + def test_pack_version_ctypes(self): + ctypes = import_helper.import_module('ctypes') + ctypes_func = ctypes.pythonapi.Py_PACK_VERSION + ctypes_func.restype = ctypes.c_uint32 + ctypes_func.argtypes = [ctypes.c_int] * 2 + for *args, expected in self.xy_cases: + with self.subTest(hexversion=hex(expected)): + result = ctypes_func(*args) + self.assertEqual(result, expected) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_number.py b/Lib/test/test_capi/test_number.py new file mode 100644 index 00000000000000..04f85d2395083a --- /dev/null +++ b/Lib/test/test_capi/test_number.py @@ -0,0 +1,320 @@ +import itertools +import operator +import sys +import unittest +import warnings + +from test.support import cpython_only, import_helper + +_testcapi = import_helper.import_module('_testcapi') +from _testcapi import PY_SSIZE_T_MAX, PY_SSIZE_T_MIN + +try: + from _testbuffer import ndarray +except ImportError: + ndarray = None + +NULL = None + +class BadDescr: + def __get__(self, obj, objtype=None): + raise RuntimeError + +class WithDunder: + def _meth(self, *args): + if self.val: + return self.val + if self.exc: + raise self.exc + @classmethod + def with_val(cls, val): + obj = super().__new__(cls) + obj.val = val + obj.exc = None + setattr(cls, cls.methname, cls._meth) + return obj + + @classmethod + def with_exc(cls, exc): + obj = super().__new__(cls) + obj.val = None + obj.exc = exc + setattr(cls, cls.methname, cls._meth) + return obj + +class HasBadAttr: + def __new__(cls): + obj = super().__new__(cls) + setattr(cls, cls.methname, BadDescr()) + return obj + + +class IndexLike(WithDunder): + methname = '__index__' + +class IntLike(WithDunder): + methname = '__int__' + +class FloatLike(WithDunder): + methname = '__float__' + + +def subclassof(base): + return type(base.__name__ + 'Subclass', (base,), {}) + + +class SomeError(Exception): + pass + +class OtherError(Exception): + pass + + +class CAPITest(unittest.TestCase): + def test_check(self): + # Test PyNumber_Check() + check = _testcapi.number_check + + self.assertTrue(check(1)) + self.assertTrue(check(IndexLike.with_val(1))) + self.assertTrue(check(IntLike.with_val(99))) + self.assertTrue(check(0.5)) + self.assertTrue(check(FloatLike.with_val(4.25))) + self.assertTrue(check(1+2j)) + + self.assertFalse(check([])) + self.assertFalse(check("abc")) + self.assertFalse(check(object())) + self.assertFalse(check(NULL)) + + def test_unary_ops(self): + methmap = {'__neg__': _testcapi.number_negative, # PyNumber_Negative() + '__pos__': _testcapi.number_positive, # PyNumber_Positive() + '__abs__': _testcapi.number_absolute, # PyNumber_Absolute() + '__invert__': _testcapi.number_invert} # PyNumber_Invert() + + for name, func in methmap.items(): + # Generic object, has no tp_as_number structure + self.assertRaises(TypeError, func, object()) + + # C-API function accepts NULL + self.assertRaises(SystemError, func, NULL) + + # Behave as corresponding unary operation + op = getattr(operator, name) + for x in [0, 42, -1, 3.14, 1+2j]: + try: + op(x) + except TypeError: + self.assertRaises(TypeError, func, x) + else: + self.assertEqual(func(x), op(x)) + + def test_binary_ops(self): + methmap = {'__add__': _testcapi.number_add, # PyNumber_Add() + '__sub__': _testcapi.number_subtract, # PyNumber_Subtract() + '__mul__': _testcapi.number_multiply, # PyNumber_Multiply() + '__matmul__': _testcapi.number_matrixmultiply, # PyNumber_MatrixMultiply() + '__floordiv__': _testcapi.number_floordivide, # PyNumber_FloorDivide() + '__truediv__': _testcapi.number_truedivide, # PyNumber_TrueDivide() + '__mod__': _testcapi.number_remainder, # PyNumber_Remainder() + '__divmod__': _testcapi.number_divmod, # PyNumber_Divmod() + '__lshift__': _testcapi.number_lshift, # PyNumber_Lshift() + '__rshift__': _testcapi.number_rshift, # PyNumber_Rshift() + '__and__': _testcapi.number_and, # PyNumber_And() + '__xor__': _testcapi.number_xor, # PyNumber_Xor() + '__or__': _testcapi.number_or, # PyNumber_Or() + '__pow__': _testcapi.number_power, # PyNumber_Power() + '__iadd__': _testcapi.number_inplaceadd, # PyNumber_InPlaceAdd() + '__isub__': _testcapi.number_inplacesubtract, # PyNumber_InPlaceSubtract() + '__imul__': _testcapi.number_inplacemultiply, # PyNumber_InPlaceMultiply() + '__imatmul__': _testcapi.number_inplacematrixmultiply, # PyNumber_InPlaceMatrixMultiply() + '__ifloordiv__': _testcapi.number_inplacefloordivide, # PyNumber_InPlaceFloorDivide() + '__itruediv__': _testcapi.number_inplacetruedivide, # PyNumber_InPlaceTrueDivide() + '__imod__': _testcapi.number_inplaceremainder, # PyNumber_InPlaceRemainder() + '__ilshift__': _testcapi.number_inplacelshift, # PyNumber_InPlaceLshift() + '__irshift__': _testcapi.number_inplacershift, # PyNumber_InPlaceRshift() + '__iand__': _testcapi.number_inplaceand, # PyNumber_InPlaceAnd() + '__ixor__': _testcapi.number_inplacexor, # PyNumber_InPlaceXor() + '__ior__': _testcapi.number_inplaceor, # PyNumber_InPlaceOr() + '__ipow__': _testcapi.number_inplacepower, # PyNumber_InPlacePower() + } + + for name, func in methmap.items(): + cases = [0, 42, 3.14, -1, 123, 1+2j] + + # Generic object, has no tp_as_number structure + for x in cases: + self.assertRaises(TypeError, func, object(), x) + self.assertRaises(TypeError, func, x, object()) + + # Behave as corresponding binary operation + op = getattr(operator, name, divmod) + for x, y in itertools.combinations(cases, 2): + try: + op(x, y) + except (TypeError, ValueError, ZeroDivisionError) as exc: + self.assertRaises(exc.__class__, func, x, y) + else: + self.assertEqual(func(x, y), op(x, y)) + + # CRASHES func(NULL, object()) + # CRASHES func(object(), NULL) + + @unittest.skipIf(ndarray is None, "needs _testbuffer") + def test_misc_add(self): + # PyNumber_Add(), PyNumber_InPlaceAdd() + add = _testcapi.number_add + inplaceadd = _testcapi.number_inplaceadd + + # test sq_concat/sq_inplace_concat slots + a, b, r = [1, 2], [3, 4], [1, 2, 3, 4] + self.assertEqual(add(a, b), r) + self.assertEqual(a, [1, 2]) + self.assertRaises(TypeError, add, ndarray([1], (1,)), 2) + a, b, r = [1, 2], [3, 4], [1, 2, 3, 4] + self.assertEqual(inplaceadd(a, b), r) + self.assertEqual(a, r) + self.assertRaises(TypeError, inplaceadd, ndarray([1], (1,)), 2) + + @unittest.skipIf(ndarray is None, "needs _testbuffer") + def test_misc_multiply(self): + # PyNumber_Multiply(), PyNumber_InPlaceMultiply() + multiply = _testcapi.number_multiply + inplacemultiply = _testcapi.number_inplacemultiply + + # test sq_repeat/sq_inplace_repeat slots + a, b, r = [1], 2, [1, 1] + self.assertEqual(multiply(a, b), r) + self.assertEqual((a, b), ([1], 2)) + self.assertEqual(multiply(b, a), r) + self.assertEqual((a, b), ([1], 2)) + self.assertEqual(multiply([1], -1), []) + self.assertRaises(TypeError, multiply, ndarray([1], (1,)), 2) + self.assertRaises(TypeError, multiply, [1], 0.5) + self.assertRaises(OverflowError, multiply, [1], PY_SSIZE_T_MAX + 1) + self.assertRaises(MemoryError, multiply, [1, 2], PY_SSIZE_T_MAX//2 + 1) + a, b, r = [1], 2, [1, 1] + self.assertEqual(inplacemultiply(a, b), r) + self.assertEqual((a, b), (r, 2)) + a = [1] + self.assertEqual(inplacemultiply(b, a), r) + self.assertEqual((a, b), ([1], 2)) + self.assertRaises(TypeError, inplacemultiply, ndarray([1], (1,)), 2) + self.assertRaises(OverflowError, inplacemultiply, [1], PY_SSIZE_T_MAX + 1) + self.assertRaises(MemoryError, inplacemultiply, [1, 2], PY_SSIZE_T_MAX//2 + 1) + + def test_misc_power(self): + # PyNumber_Power() + power = _testcapi.number_power + + class HasPow(WithDunder): + methname = '__pow__' + + # ternary op + self.assertEqual(power(4, 11, 5), pow(4, 11, 5)) + self.assertRaises(TypeError, power, 4, 11, 1.25) + self.assertRaises(TypeError, power, 4, 11, HasPow.with_val(NotImplemented)) + self.assertRaises(TypeError, power, 4, 11, object()) + + def test_long(self): + # Test PyNumber_Long() + long = _testcapi.number_long + + self.assertEqual(long(42), 42) + self.assertEqual(long(1.25), 1) + self.assertEqual(long("42"), 42) + self.assertEqual(long(b"42"), 42) + self.assertEqual(long(bytearray(b"42")), 42) + self.assertEqual(long(memoryview(b"42")), 42) + self.assertEqual(long(IndexLike.with_val(99)), 99) + self.assertEqual(long(IntLike.with_val(99)), 99) + + self.assertRaises(TypeError, long, IntLike.with_val(1.0)) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + self.assertRaises(DeprecationWarning, long, IntLike.with_val(True)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(long(IntLike.with_val(True)), 1) + self.assertRaises(RuntimeError, long, IntLike.with_exc(RuntimeError)) + + self.assertRaises(TypeError, long, 1j) + self.assertRaises(TypeError, long, object()) + self.assertRaises(SystemError, long, NULL) + + def test_float(self): + # Test PyNumber_Float() + float_ = _testcapi.number_float + + self.assertEqual(float_(1.25), 1.25) + self.assertEqual(float_(123), 123.) + self.assertEqual(float_("1.25"), 1.25) + + self.assertEqual(float_(FloatLike.with_val(4.25)), 4.25) + self.assertEqual(float_(IndexLike.with_val(99)), 99.0) + self.assertEqual(float_(IndexLike.with_val(-1)), -1.0) + + self.assertRaises(TypeError, float_, FloatLike.with_val(687)) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + self.assertRaises(DeprecationWarning, float_, FloatLike.with_val(subclassof(float)(4.25))) + with self.assertWarns(DeprecationWarning): + self.assertEqual(float_(FloatLike.with_val(subclassof(float)(4.25))), 4.25) + self.assertRaises(RuntimeError, float_, FloatLike.with_exc(RuntimeError)) + + self.assertRaises(TypeError, float_, IndexLike.with_val(1.25)) + self.assertRaises(OverflowError, float_, IndexLike.with_val(2**2000)) + + self.assertRaises(TypeError, float_, 1j) + self.assertRaises(TypeError, float_, object()) + self.assertRaises(SystemError, float_, NULL) + + def test_index(self): + # Test PyNumber_Index() + index = _testcapi.number_index + + self.assertEqual(index(11), 11) + + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + self.assertRaises(DeprecationWarning, index, IndexLike.with_val(True)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(index(IndexLike.with_val(True)), 1) + self.assertRaises(TypeError, index, IndexLike.with_val(1.0)) + self.assertRaises(RuntimeError, index, IndexLike.with_exc(RuntimeError)) + + self.assertRaises(TypeError, index, 1.25) + self.assertRaises(TypeError, index, "42") + self.assertRaises(TypeError, index, object()) + self.assertRaises(SystemError, index, NULL) + + def test_tobase(self): + # Test PyNumber_ToBase() + tobase = _testcapi.number_tobase + + self.assertEqual(tobase(10, 2), bin(10)) + self.assertEqual(tobase(11, 8), oct(11)) + self.assertEqual(tobase(16, 10), str(16)) + self.assertEqual(tobase(13, 16), hex(13)) + + self.assertRaises(SystemError, tobase, NULL, 2) + self.assertRaises(SystemError, tobase, 2, 3) + self.assertRaises(TypeError, tobase, 1.25, 2) + self.assertRaises(TypeError, tobase, "42", 2) + + def test_asssizet(self): + # Test PyNumber_AsSsize_t() + asssizet = _testcapi.number_asssizet + + for n in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: + self.assertEqual(asssizet(n, OverflowError), n) + self.assertEqual(asssizet(PY_SSIZE_T_MAX+10, NULL), PY_SSIZE_T_MAX) + self.assertEqual(asssizet(PY_SSIZE_T_MIN-10, NULL), PY_SSIZE_T_MIN) + + self.assertRaises(OverflowError, asssizet, PY_SSIZE_T_MAX + 10, OverflowError) + self.assertRaises(RuntimeError, asssizet, PY_SSIZE_T_MAX + 10, RuntimeError) + self.assertRaises(SystemError, asssizet, NULL, TypeError) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py new file mode 100644 index 00000000000000..b0d39937fd865f --- /dev/null +++ b/Lib/test/test_capi/test_object.py @@ -0,0 +1,174 @@ +import enum +import unittest +from test import support +from test.support import import_helper +from test.support import os_helper +from test.support import threading_helper + +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') +_testcapi = import_helper.import_module('_testcapi') +_testinternalcapi = import_helper.import_module('_testinternalcapi') + + +class Constant(enum.IntEnum): + Py_CONSTANT_NONE = 0 + Py_CONSTANT_FALSE = 1 + Py_CONSTANT_TRUE = 2 + Py_CONSTANT_ELLIPSIS = 3 + Py_CONSTANT_NOT_IMPLEMENTED = 4 + Py_CONSTANT_ZERO = 5 + Py_CONSTANT_ONE = 6 + Py_CONSTANT_EMPTY_STR = 7 + Py_CONSTANT_EMPTY_BYTES = 8 + Py_CONSTANT_EMPTY_TUPLE = 9 + + INVALID_CONSTANT = Py_CONSTANT_EMPTY_TUPLE + 1 + + +class GetConstantTest(unittest.TestCase): + def check_get_constant(self, get_constant): + self.assertIs(get_constant(Constant.Py_CONSTANT_NONE), None) + self.assertIs(get_constant(Constant.Py_CONSTANT_FALSE), False) + self.assertIs(get_constant(Constant.Py_CONSTANT_TRUE), True) + self.assertIs(get_constant(Constant.Py_CONSTANT_ELLIPSIS), Ellipsis) + self.assertIs(get_constant(Constant.Py_CONSTANT_NOT_IMPLEMENTED), NotImplemented) + + for constant_id, constant_type, value in ( + (Constant.Py_CONSTANT_ZERO, int, 0), + (Constant.Py_CONSTANT_ONE, int, 1), + (Constant.Py_CONSTANT_EMPTY_STR, str, ""), + (Constant.Py_CONSTANT_EMPTY_BYTES, bytes, b""), + (Constant.Py_CONSTANT_EMPTY_TUPLE, tuple, ()), + ): + with self.subTest(constant_id=constant_id): + obj = get_constant(constant_id) + self.assertEqual(type(obj), constant_type, obj) + self.assertEqual(obj, value) + + with self.assertRaises(SystemError): + get_constant(Constant.INVALID_CONSTANT) + + def test_get_constant(self): + self.check_get_constant(_testlimitedcapi.get_constant) + + def test_get_constant_borrowed(self): + self.check_get_constant(_testlimitedcapi.get_constant_borrowed) + + +class PrintTest(unittest.TestCase): + def testPyObjectPrintObject(self): + + class PrintableObject: + + def __repr__(self): + return "spam spam spam" + + def __str__(self): + return "egg egg egg" + + obj = PrintableObject() + output_filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, output_filename) + + # Test repr printing + _testcapi.call_pyobject_print(obj, output_filename, False) + with open(output_filename, 'r') as output_file: + self.assertEqual(output_file.read(), repr(obj)) + + # Test str printing + _testcapi.call_pyobject_print(obj, output_filename, True) + with open(output_filename, 'r') as output_file: + self.assertEqual(output_file.read(), str(obj)) + + def testPyObjectPrintNULL(self): + output_filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, output_filename) + + # Test repr printing + _testcapi.pyobject_print_null(output_filename) + with open(output_filename, 'r') as output_file: + self.assertEqual(output_file.read(), '') + + def testPyObjectPrintNoRefObject(self): + output_filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, output_filename) + + # Test repr printing + correct_output = _testcapi.pyobject_print_noref_object(output_filename) + with open(output_filename, 'r') as output_file: + self.assertEqual(output_file.read(), correct_output) + + def testPyObjectPrintOSError(self): + output_filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, output_filename) + + open(output_filename, "w+").close() + with self.assertRaises(OSError): + _testcapi.pyobject_print_os_error(output_filename) + + +class ClearWeakRefsNoCallbacksTest(unittest.TestCase): + """Test PyUnstable_Object_ClearWeakRefsNoCallbacks""" + def test_ClearWeakRefsNoCallbacks(self): + """Ensure PyUnstable_Object_ClearWeakRefsNoCallbacks works""" + import weakref + import gc + class C: + pass + obj = C() + messages = [] + ref = weakref.ref(obj, lambda: messages.append("don't add this")) + self.assertIs(ref(), obj) + self.assertFalse(messages) + _testcapi.pyobject_clear_weakrefs_no_callbacks(obj) + self.assertIsNone(ref()) + gc.collect() + self.assertFalse(messages) + + def test_ClearWeakRefsNoCallbacks_no_weakref_support(self): + """Don't fail on objects that don't support weakrefs""" + import weakref + obj = object() + with self.assertRaises(TypeError): + ref = weakref.ref(obj) + _testcapi.pyobject_clear_weakrefs_no_callbacks(obj) + + +@threading_helper.requires_working_threading() +class EnableDeferredRefcountingTest(unittest.TestCase): + """Test PyUnstable_Object_EnableDeferredRefcount""" + @support.requires_resource("cpu") + def test_enable_deferred_refcount(self): + from threading import Thread + + self.assertEqual(_testcapi.pyobject_enable_deferred_refcount("not tracked"), 0) + foo = [] + self.assertEqual(_testcapi.pyobject_enable_deferred_refcount(foo), int(support.Py_GIL_DISABLED)) + + # Make sure reference counting works on foo now + self.assertEqual(foo, []) + if support.Py_GIL_DISABLED: + self.assertTrue(_testinternalcapi.has_deferred_refcount(foo)) + + # Make sure that PyUnstable_Object_EnableDeferredRefcount is thread safe + def silly_func(obj): + self.assertIn( + _testcapi.pyobject_enable_deferred_refcount(obj), + (0, 1) + ) + + silly_list = [1, 2, 3] + threads = [ + Thread(target=silly_func, args=(silly_list,)) for _ in range(4) + ] + + with threading_helper.start_threads(threads): + for i in range(10): + silly_list.append(i) + + if support.Py_GIL_DISABLED: + self.assertTrue(_testinternalcapi.has_deferred_refcount(silly_list)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index a0a19225b79433..d3aea37e094e61 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1,24 +1,19 @@ import contextlib -import opcode +import itertools import sys import textwrap import unittest import gc import os -import _testinternalcapi +import _opcode -from test.support import script_helper, requires_specialization +from test.support import (script_helper, requires_specialization, + import_helper, Py_GIL_DISABLED, requires_jit_enabled) +_testinternalcapi = import_helper.import_module("_testinternalcapi") -@contextlib.contextmanager -def temporary_optimizer(opt): - old_opt = _testinternalcapi.get_optimizer() - _testinternalcapi.set_optimizer(opt) - try: - yield - finally: - _testinternalcapi.set_optimizer(old_opt) +from _testinternalcapi import TIER2_THRESHOLD @contextlib.contextmanager @@ -31,97 +26,14 @@ def clear_executors(func): func.__code__ = func.__code__.replace() -@requires_specialization -class TestOptimizerAPI(unittest.TestCase): - - def test_new_counter_optimizer_dealloc(self): - # See gh-108727 - def f(): - _testinternalcapi.new_counter_optimizer() - - f() - - def test_get_set_optimizer(self): - old = _testinternalcapi.get_optimizer() - opt = _testinternalcapi.new_counter_optimizer() - try: - _testinternalcapi.set_optimizer(opt) - self.assertEqual(_testinternalcapi.get_optimizer(), opt) - _testinternalcapi.set_optimizer(None) - self.assertEqual(_testinternalcapi.get_optimizer(), None) - finally: - _testinternalcapi.set_optimizer(old) - - - def test_counter_optimizer(self): - # Generate a new function at each call - ns = {} - exec(textwrap.dedent(""" - def loop(): - for _ in range(1000): - pass - """), ns, ns) - loop = ns['loop'] - - for repeat in range(5): - opt = _testinternalcapi.new_counter_optimizer() - with temporary_optimizer(opt): - self.assertEqual(opt.get_count(), 0) - with clear_executors(loop): - loop() - self.assertEqual(opt.get_count(), 1000) - - def test_long_loop(self): - "Check that we aren't confused by EXTENDED_ARG" - - # Generate a new function at each call - ns = {} - exec(textwrap.dedent(""" - def nop(): - pass - - def long_loop(): - for _ in range(10): - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - """), ns, ns) - long_loop = ns['long_loop'] - - opt = _testinternalcapi.new_counter_optimizer() - with temporary_optimizer(opt): - self.assertEqual(opt.get_count(), 0) - long_loop() - self.assertEqual(opt.get_count(), 10) - - def test_code_restore_for_ENTER_EXECUTOR(self): - def testfunc(x): - i = 0 - while i < x: - i += 1 - - opt = _testinternalcapi.new_counter_optimizer() - with temporary_optimizer(opt): - testfunc(1000) - code, replace_code = testfunc.__code__, testfunc.__code__.replace() - self.assertEqual(code, replace_code) - self.assertEqual(hash(code), hash(replace_code)) - - def get_first_executor(func): code = func.__code__ co_code = code.co_code - JUMP_BACKWARD = opcode.opmap["JUMP_BACKWARD"] for i in range(0, len(co_code), 2): - if co_code[i] == JUMP_BACKWARD: - try: - return _testinternalcapi.get_executor(code, i) - except ValueError: - pass + try: + return _opcode.get_executor(code, i) + except ValueError: + pass return None @@ -131,27 +43,21 @@ def iter_opnames(ex): def get_opnames(ex): - return set(iter_opnames(ex)) + return list(iter_opnames(ex)) @requires_specialization +@unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") +@requires_jit_enabled class TestExecutorInvalidation(unittest.TestCase): - def setUp(self): - self.old = _testinternalcapi.get_optimizer() - self.opt = _testinternalcapi.new_counter_optimizer() - _testinternalcapi.set_optimizer(self.opt) - - def tearDown(self): - _testinternalcapi.set_optimizer(self.old) - def test_invalidate_object(self): # Generate a new set of functions at each call ns = {} func_src = "\n".join( f""" def f{n}(): - for _ in range(1000): + for _ in range({TIER2_THRESHOLD}): pass """ for n in range(5) ) @@ -171,7 +77,7 @@ def f{n}(): self.assertTrue(exe.is_valid()) # Assert that the correct executors are invalidated # and check that nothing crashes when we invalidate - # an executor mutliple times. + # an executor multiple times. for i in (4,3,2,1,0): _testinternalcapi.invalidate_executors(objects[i]) for exe in executors[i:]: @@ -182,15 +88,13 @@ def f{n}(): def test_uop_optimizer_invalidation(self): # Generate a new function at each call ns = {} - exec(textwrap.dedent(""" + exec(textwrap.dedent(f""" def f(): - for i in range(1000): + for i in range({TIER2_THRESHOLD}): pass """), ns, ns) f = ns['f'] - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - f() + f() exe = get_first_executor(f) self.assertIsNotNone(exe) self.assertTrue(exe.is_valid()) @@ -199,11 +103,9 @@ def f(): def test_sys__clear_internal_caches(self): def f(): - for _ in range(1000): + for _ in range(TIER2_THRESHOLD): pass - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - f() + f() exe = get_first_executor(f) self.assertIsNotNone(exe) self.assertTrue(exe.is_valid()) @@ -214,6 +116,8 @@ def f(): @requires_specialization +@unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") +@requires_jit_enabled @unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") class TestUops(unittest.TestCase): @@ -223,20 +127,18 @@ def testfunc(x): while i < x: i += 1 - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - testfunc(1000) + testfunc(TIER2_THRESHOLD) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_SET_IP", uops) + self.assertIn("_JUMP_TO_TOP", uops) self.assertIn("_LOAD_FAST_0", uops) def test_extended_arg(self): "Check EXTENDED_ARG handling in superblock creation" ns = {} - exec(textwrap.dedent(""" + exec(textwrap.dedent(f""" def many_vars(): # 260 vars, so z9 should have index 259 a0 = a1 = a2 = a3 = a4 = a5 = a6 = a7 = a8 = a9 = 42 @@ -264,17 +166,16 @@ def many_vars(): w0 = w1 = w2 = w3 = w4 = w5 = w6 = w7 = w8 = w9 = 42 x0 = x1 = x2 = x3 = x4 = x5 = x6 = x7 = x8 = x9 = 42 y0 = y1 = y2 = y3 = y4 = y5 = y6 = y7 = y8 = y9 = 42 - z0 = z1 = z2 = z3 = z4 = z5 = z6 = z7 = z8 = z9 = 42 + z0 = z1 = z2 = z3 = z4 = z5 = z6 = z7 = z8 = z9 = {TIER2_THRESHOLD} while z9 > 0: z9 = z9 - 1 + +z9 """), ns, ns) many_vars = ns["many_vars"] - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - ex = get_first_executor(many_vars) - self.assertIsNone(ex) - many_vars() + ex = get_first_executor(many_vars) + self.assertIsNone(ex) + many_vars() ex = get_first_executor(many_vars) self.assertIsNotNone(ex) @@ -293,10 +194,7 @@ def testfunc(x): while i < x: i += 1 - opt = _testinternalcapi.new_uop_optimizer() - - with temporary_optimizer(opt): - testfunc(20) + testfunc(TIER2_THRESHOLD) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -309,9 +207,7 @@ def testfunc(n): while i < n: i += 1 - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) + testfunc(TIER2_THRESHOLD) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -324,14 +220,13 @@ def testfunc(a): if x is None: x = 0 - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - testfunc(range(20)) + testfunc(range(TIER2_THRESHOLD)) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_GUARD_IS_NOT_NONE_POP", uops) + self.assertNotIn("_GUARD_IS_NONE_POP", uops) + self.assertNotIn("_GUARD_IS_NOT_NONE_POP", uops) def test_pop_jump_if_not_none(self): def testfunc(a): @@ -340,14 +235,13 @@ def testfunc(a): if x is not None: x = 0 - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - testfunc(range(20)) + testfunc(range(TIER2_THRESHOLD)) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_GUARD_IS_NONE_POP", uops) + self.assertNotIn("_GUARD_IS_NONE_POP", uops) + self.assertNotIn("_GUARD_IS_NOT_NONE_POP", uops) def test_pop_jump_if_true(self): def testfunc(n): @@ -355,9 +249,7 @@ def testfunc(n): while not i >= n: i += 1 - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) + testfunc(TIER2_THRESHOLD) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -370,9 +262,7 @@ def testfunc(n): while i < n: i += 1 - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) + testfunc(TIER2_THRESHOLD) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -390,9 +280,7 @@ def testfunc(n): a += 1 return a - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) + testfunc(TIER2_THRESHOLD) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -408,10 +296,8 @@ def testfunc(n): total += i return total - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - total = testfunc(20) - self.assertEqual(total, 190) + total = testfunc(TIER2_THRESHOLD) + self.assertEqual(total, sum(range(TIER2_THRESHOLD))) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -429,11 +315,9 @@ def testfunc(a): total += i return total - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - a = list(range(20)) - total = testfunc(a) - self.assertEqual(total, 190) + a = list(range(TIER2_THRESHOLD)) + total = testfunc(a) + self.assertEqual(total, sum(a)) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -451,11 +335,9 @@ def testfunc(a): total += i return total - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - a = tuple(range(20)) - total = testfunc(a) - self.assertEqual(total, 190) + a = tuple(range(TIER2_THRESHOLD)) + total = testfunc(a) + self.assertEqual(total, sum(a)) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -471,14 +353,12 @@ def testfunc(it): for x in it: pass - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - a = [1, 2, 3] - it = iter(a) - testfunc(it) - a.append(4) - with self.assertRaises(StopIteration): - next(it) + a = [1, 2, 3] + it = iter(a) + testfunc(it) + a.append(4) + with self.assertRaises(StopIteration): + next(it) def test_call_py_exact_args(self): def testfunc(n): @@ -487,9 +367,7 @@ def dummy(x): for i in range(n): dummy(i) - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) + testfunc(TIER2_THRESHOLD) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -505,9 +383,7 @@ def testfunc(n): else: i = 1 - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - testfunc(20) + testfunc(TIER2_THRESHOLD) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -533,11 +409,9 @@ def testfunc(n, m): x += 1000*i + j return x - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - x = testfunc(10, 10) + x = testfunc(TIER2_THRESHOLD, TIER2_THRESHOLD) - self.assertEqual(x, sum(range(10)) * 10010) + self.assertEqual(x, sum(range(TIER2_THRESHOLD)) * TIER2_THRESHOLD * 1001) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -558,15 +432,11 @@ def testfunc(n): bits += 1 if i&0x10: bits += 1 - if i&0x20: - bits += 1 return bits - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - x = testfunc(20) + x = testfunc(TIER2_THRESHOLD * 2) - self.assertEqual(x, 40) + self.assertEqual(x, TIER2_THRESHOLD * 5) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) ops = list(iter_opnames(ex)) @@ -576,14 +446,13 @@ def testfunc(n): @requires_specialization +@unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") +@requires_jit_enabled @unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") class TestUopsOptimization(unittest.TestCase): def _run_with_optimizer(self, testfunc, arg): - res = None - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - res = testfunc(arg) + res = testfunc(arg) ex = get_first_executor(testfunc) return res, ex @@ -598,9 +467,9 @@ def testfunc(loops): num += 1 return a - res, ex = self._run_with_optimizer(testfunc, 32) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertIsNotNone(ex) - self.assertEqual(res, 63) + self.assertEqual(res, (TIER2_THRESHOLD - 1) * 2 + 1) binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] self.assertGreaterEqual(len(binop_count), 3) @@ -617,14 +486,11 @@ def testfunc(loops): num += 1 return a - opt = _testinternalcapi.new_uop_optimizer() - res = None - with temporary_optimizer(opt): - res = testfunc(32) + res = testfunc(TIER2_THRESHOLD) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - self.assertEqual(res, 124) + self.assertEqual(res, (TIER2_THRESHOLD - 1) * 4) binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] self.assertGreaterEqual(len(binop_count), 3) @@ -641,14 +507,11 @@ def testfunc(loops): num += 1 return x - opt = _testinternalcapi.new_uop_optimizer() - res = None - with temporary_optimizer(opt): - res = testfunc(32) + res = testfunc(TIER2_THRESHOLD) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - self.assertEqual(res, 124) + self.assertEqual(res, (TIER2_THRESHOLD - 1) * 4) binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] self.assertGreaterEqual(len(binop_count), 3) @@ -665,7 +528,7 @@ def testfunc(loops): num += 1 return a - res, ex = self._run_with_optimizer(testfunc, 64) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertIsNotNone(ex) binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] self.assertGreaterEqual(len(binop_count), 3) @@ -677,7 +540,7 @@ def dummy(x): for i in range(n): dummy(i) - res, ex = self._run_with_optimizer(testfunc, 32) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertIn("_PUSH_FRAME", uops) @@ -691,8 +554,8 @@ def testfunc(n): x = i + i return x - res, ex = self._run_with_optimizer(testfunc, 32) - self.assertEqual(res, 62) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, (TIER2_THRESHOLD - 1) * 2) self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertNotIn("_GUARD_BOTH_INT", uops) @@ -709,7 +572,7 @@ def testfunc(n): res = x + z + a + b return res - res, ex = self._run_with_optimizer(testfunc, 32) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertEqual(res, 4) self.assertIsNotNone(ex) uops = get_opnames(ex) @@ -722,8 +585,8 @@ def testfunc(n): for _ in range(n): return [i for i in range(n)] - res, ex = self._run_with_optimizer(testfunc, 32) - self.assertEqual(res, list(range(32))) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, list(range(TIER2_THRESHOLD))) self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertNotIn("_BINARY_OP_ADD_INT", uops) @@ -736,16 +599,14 @@ def testfunc(n): for i in range(n): dummy(i) - opt = _testinternalcapi.new_uop_optimizer() # Trigger specialization testfunc(8) - with temporary_optimizer(opt): - del dummy - gc.collect() + del dummy + gc.collect() - def dummy(x): - return x + 2 - testfunc(32) + def dummy(x): + return x + 2 + testfunc(32) ex = get_first_executor(testfunc) # Honestly as long as it doesn't crash it's fine. @@ -758,17 +619,16 @@ def test_promote_globals_to_constants(self): result = script_helper.run_python_until_end('-c', textwrap.dedent(""" import _testinternalcapi import opcode + import _opcode def get_first_executor(func): code = func.__code__ co_code = code.co_code - JUMP_BACKWARD = opcode.opmap["JUMP_BACKWARD"] for i in range(0, len(co_code), 2): - if co_code[i] == JUMP_BACKWARD: - try: - return _testinternalcapi.get_executor(code, i) - except ValueError: - pass + try: + return _opcode.get_executor(code, i) + except ValueError: + pass return None def get_opnames(ex): @@ -779,16 +639,14 @@ def testfunc(n): x = range(i) return x - opt = _testinternalcapi.new_uop_optimizer() - _testinternalcapi.set_optimizer(opt) - testfunc(64) + testfunc(_testinternalcapi.TIER2_THRESHOLD) ex = get_first_executor(testfunc) assert ex is not None uops = get_opnames(ex) assert "_LOAD_GLOBAL_BUILTINS" not in uops - assert "_LOAD_CONST_INLINE_BORROW_WITH_NULL" in uops - """)) + assert "_LOAD_CONST_INLINE_BORROW" in uops + """), PYTHON_JIT="1") self.assertEqual(result[0].rc, 0, result) def test_float_add_constant_propagation(self): @@ -801,8 +659,8 @@ def testfunc(n): a = a + 0.25 return a - res, ex = self._run_with_optimizer(testfunc, 32) - self.assertAlmostEqual(res, 33.0) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertAlmostEqual(res, TIER2_THRESHOLD + 1) self.assertIsNotNone(ex) uops = get_opnames(ex) guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] @@ -821,8 +679,8 @@ def testfunc(n): a = a - 0.25 return a - res, ex = self._run_with_optimizer(testfunc, 32) - self.assertAlmostEqual(res, -31.0) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertAlmostEqual(res, -TIER2_THRESHOLD + 1) self.assertIsNotNone(ex) uops = get_opnames(ex) guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] @@ -841,7 +699,7 @@ def testfunc(n): a = a * 1.0 return a - res, ex = self._run_with_optimizer(testfunc, 32) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertAlmostEqual(res, 1.0) self.assertIsNotNone(ex) uops = get_opnames(ex) @@ -861,7 +719,7 @@ def testfunc(n): a + a return a - res, ex = self._run_with_optimizer(testfunc, 32) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertEqual(res, "") self.assertIsNotNone(ex) uops = get_opnames(ex) @@ -879,7 +737,7 @@ def testfunc(n): x = a == a return x - res, ex = self._run_with_optimizer(testfunc, 32) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertTrue(res) self.assertIsNotNone(ex) uops = get_opnames(ex) @@ -897,14 +755,54 @@ def testfunc(n): x = a == a return x - res, ex = self._run_with_optimizer(testfunc, 32) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertTrue(res) self.assertIsNotNone(ex) uops = get_opnames(ex) - guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] - self.assertLessEqual(len(guard_both_float_count), 1) + guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertLessEqual(len(guard_both_int_count), 1) self.assertIn("_COMPARE_OP_INT", uops) + def test_compare_op_type_propagation_int_partial(self): + def testfunc(n): + a = 1 + for _ in range(n): + if a > 2: + x = 0 + if a < 2: + x = 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, 1) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_left_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_INT"] + guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertLessEqual(len(guard_left_int_count), 1) + self.assertEqual(len(guard_both_int_count), 0) + self.assertIn("_COMPARE_OP_INT", uops) + + def test_compare_op_type_propagation_float_partial(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + if a > 2.0: + x = 0 + if a < 2.0: + x = 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, 1) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_left_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_FLOAT"] + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] + self.assertLessEqual(len(guard_left_float_count), 1) + self.assertEqual(len(guard_both_float_count), 0) + self.assertIn("_COMPARE_OP_FLOAT", uops) + def test_compare_op_type_propagation_unicode(self): def testfunc(n): a = "" @@ -915,7 +813,7 @@ def testfunc(n): x = a == a return x - res, ex = self._run_with_optimizer(testfunc, 32) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertTrue(res) self.assertIsNotNone(ex) uops = get_opnames(ex) @@ -933,10 +831,10 @@ def testfunc(n): exec(src, ns, ns) testfunc = ns['testfunc'] ns['_test_global'] = 0 - _, ex = self._run_with_optimizer(testfunc, 16) + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) self.assertIsNone(ex) ns['_test_global'] = 1 - _, ex = self._run_with_optimizer(testfunc, 16) + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertNotIn("_GUARD_BOTH_INT", uops) @@ -947,12 +845,599 @@ def testfunc(n): exec(src, ns, ns) testfunc = ns['testfunc'] ns['_test_global'] = 0 - _, ex = self._run_with_optimizer(testfunc, 16) + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) self.assertIsNone(ex) ns['_test_global'] = 3.14 - _, ex = self._run_with_optimizer(testfunc, 16) + _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) self.assertIsNone(ex) + def test_combine_stack_space_checks_sequential(self): + def dummy12(x): + return x - 1 + def dummy13(y): + z = y + 2 + return y, z + def testfunc(n): + a = 0 + for _ in range(n): + b = dummy12(7) + c, d = dummy13(9) + a += b + c + d + return a + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 26) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 2) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + # sequential calls: max(12, 13) == 13 + largest_stack = _testinternalcapi.get_co_framesize(dummy13.__code__) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_combine_stack_space_checks_nested(self): + def dummy12(x): + return x + 3 + def dummy15(y): + z = dummy12(y) + return y, z + def testfunc(n): + a = 0 + for _ in range(n): + b, c = dummy15(2) + a += b + c + return a + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 7) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 2) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + # nested calls: 15 + 12 == 27 + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy15.__code__) + + _testinternalcapi.get_co_framesize(dummy12.__code__) + ) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_combine_stack_space_checks_several_calls(self): + def dummy12(x): + return x + 3 + def dummy13(y): + z = y + 2 + return y, z + def dummy18(y): + z = dummy12(y) + x, w = dummy13(z) + return z, x, w + def testfunc(n): + a = 0 + for _ in range(n): + b = dummy12(5) + c, d, e = dummy18(2) + a += b + c + d + e + return a + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 25) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 4) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 4) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + # max(12, 18 + max(12, 13)) == 31 + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy18.__code__) + + _testinternalcapi.get_co_framesize(dummy13.__code__) + ) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_combine_stack_space_checks_several_calls_different_order(self): + # same as `several_calls` but with top-level calls reversed + def dummy12(x): + return x + 3 + def dummy13(y): + z = y + 2 + return y, z + def dummy18(y): + z = dummy12(y) + x, w = dummy13(z) + return z, x, w + def testfunc(n): + a = 0 + for _ in range(n): + c, d, e = dummy18(2) + b = dummy12(5) + a += b + c + d + e + return a + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 25) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 4) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 4) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + # max(18 + max(12, 13), 12) == 31 + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy18.__code__) + + _testinternalcapi.get_co_framesize(dummy13.__code__) + ) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_combine_stack_space_complex(self): + def dummy0(x): + return x + def dummy1(x): + return dummy0(x) + def dummy2(x): + return dummy1(x) + def dummy3(x): + return dummy0(x) + def dummy4(x): + y = dummy0(x) + return dummy3(y) + def dummy5(x): + return dummy2(x) + def dummy6(x): + y = dummy5(x) + z = dummy0(y) + return dummy4(z) + def testfunc(n): + a = 0 + for _ in range(n): + b = dummy5(1) + c = dummy0(1) + d = dummy6(1) + a += b + c + d + return a + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * 3) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 15) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 15) + + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy6.__code__) + + _testinternalcapi.get_co_framesize(dummy5.__code__) + + _testinternalcapi.get_co_framesize(dummy2.__code__) + + _testinternalcapi.get_co_framesize(dummy1.__code__) + + _testinternalcapi.get_co_framesize(dummy0.__code__) + ) + self.assertIn( + ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands + ) + + def test_combine_stack_space_checks_large_framesize(self): + # Create a function with a large framesize. This ensures _CHECK_STACK_SPACE is + # actually doing its job. Note that the resulting trace hits + # UOP_MAX_TRACE_LENGTH, but since all _CHECK_STACK_SPACEs happen early, this + # test is still meaningful. + repetitions = 10000 + ns = {} + header = """ + def dummy_large(a0): + """ + body = "".join([f""" + a{n+1} = a{n} + 1 + """ for n in range(repetitions)]) + return_ = f""" + return a{repetitions-1} + """ + exec(textwrap.dedent(header + body + return_), ns, ns) + dummy_large = ns['dummy_large'] + + # this is something like: + # + # def dummy_large(a0): + # a1 = a0 + 1 + # a2 = a1 + 1 + # .... + # a9999 = a9998 + 1 + # return a9999 + + def dummy15(z): + y = dummy_large(z) + return y + 3 + + def testfunc(n): + b = 0 + for _ in range(n): + b += dummy15(7) + return b + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * (repetitions + 9)) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + + # this hits a different case during trace projection in refcount test runs only, + # so we need to account for both possibilities + self.assertIn(uop_names.count("_CHECK_STACK_SPACE"), [0, 1]) + if uop_names.count("_CHECK_STACK_SPACE") == 0: + largest_stack = ( + _testinternalcapi.get_co_framesize(dummy15.__code__) + + _testinternalcapi.get_co_framesize(dummy_large.__code__) + ) + else: + largest_stack = _testinternalcapi.get_co_framesize(dummy15.__code__) + self.assertIn( + ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands + ) + + def test_combine_stack_space_checks_recursion(self): + def dummy15(x): + while x > 0: + return dummy15(x - 1) + return 42 + def testfunc(n): + a = 0 + for _ in range(n): + a += dummy15(n) + return a + + recursion_limit = sys.getrecursionlimit() + try: + sys.setrecursionlimit(TIER2_THRESHOLD + recursion_limit) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + finally: + sys.setrecursionlimit(recursion_limit) + self.assertEqual(res, TIER2_THRESHOLD * 42) + self.assertIsNotNone(ex) + + uops_and_operands = [(opcode, operand) for opcode, _, _, operand in ex] + uop_names = [uop[0] for uop in uops_and_operands] + self.assertEqual(uop_names.count("_PUSH_FRAME"), 2) + self.assertEqual(uop_names.count("_RETURN_VALUE"), 0) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE"), 1) + self.assertEqual(uop_names.count("_CHECK_STACK_SPACE_OPERAND"), 1) + largest_stack = _testinternalcapi.get_co_framesize(dummy15.__code__) + self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + + def test_many_nested(self): + # overflow the trace_stack + def dummy_a(x): + return x + def dummy_b(x): + return dummy_a(x) + def dummy_c(x): + return dummy_b(x) + def dummy_d(x): + return dummy_c(x) + def dummy_e(x): + return dummy_d(x) + def dummy_f(x): + return dummy_e(x) + def dummy_g(x): + return dummy_f(x) + def dummy_h(x): + return dummy_g(x) + def testfunc(n): + a = 0 + for _ in range(n): + a += dummy_h(n) + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, 32 * 32) + self.assertIsNone(ex) + + def test_return_generator(self): + def gen(): + yield None + def testfunc(n): + for i in range(n): + gen() + return i + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD - 1) + self.assertIsNotNone(ex) + self.assertIn("_RETURN_GENERATOR", get_opnames(ex)) + + def test_for_iter_gen(self): + def gen(n): + for i in range(n): + yield i + def testfunc(n): + g = gen(n) + s = 0 + for i in g: + s += i + return s + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, sum(range(TIER2_THRESHOLD))) + self.assertIsNotNone(ex) + self.assertIn("_FOR_ITER_GEN_FRAME", get_opnames(ex)) + + def test_modified_local_is_seen_by_optimized_code(self): + l = sys._getframe().f_locals + a = 1 + s = 0 + for j in range(1 << 10): + a + a + l["xa"[j >> 9]] = 1.0 + s += a + self.assertIs(type(a), float) + self.assertIs(type(s), float) + self.assertEqual(s, 1024.0) + + def test_guard_type_version_removed(self): + def thing(a): + x = 0 + for _ in range(TIER2_THRESHOLD): + x += a.attr + x += a.attr + return x + + class Foo: + attr = 1 + + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + self.assertIsNotNone(ex) + self.assertEqual(res, TIER2_THRESHOLD * 2) + guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") + self.assertEqual(guard_type_version_count, 1) + + def test_guard_type_version_removed_inlined(self): + """ + Verify that the guard type version if we have an inlined function + """ + + def fn(): + pass + + def thing(a): + x = 0 + for _ in range(TIER2_THRESHOLD): + x += a.attr + fn() + x += a.attr + return x + + class Foo: + attr = 1 + + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + self.assertIsNotNone(ex) + self.assertEqual(res, TIER2_THRESHOLD * 2) + guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") + self.assertEqual(guard_type_version_count, 1) + + def test_guard_type_version_removed_invalidation(self): + + def thing(a): + x = 0 + for i in range(TIER2_THRESHOLD * 2 + 1): + x += a.attr + # The first TIER2_THRESHOLD iterations we set the attribute on + # this dummy class, which shouldn't trigger the type watcher. + # Note that the code needs to be in this weird form so it's + # optimized inline without any control flow: + setattr((Bar, Foo)[i == TIER2_THRESHOLD + 1], "attr", 2) + x += a.attr + return x + + class Foo: + attr = 1 + + class Bar: + pass + + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + self.assertIsNotNone(ex) + self.assertEqual(res, TIER2_THRESHOLD * 6 + 1) + call = opnames.index("_CALL_BUILTIN_FAST") + load_attr_top = opnames.index("_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", 0, call) + load_attr_bottom = opnames.index("_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", call) + self.assertEqual(opnames[:load_attr_top].count("_GUARD_TYPE_VERSION"), 1) + self.assertEqual(opnames[call:load_attr_bottom].count("_CHECK_VALIDITY"), 1) + + def test_guard_type_version_removed_escaping(self): + + def thing(a): + x = 0 + for i in range(TIER2_THRESHOLD): + x += a.attr + # eval should be escaping + eval("None") + x += a.attr + return x + + class Foo: + attr = 1 + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + self.assertIsNotNone(ex) + self.assertEqual(res, TIER2_THRESHOLD * 2) + call = opnames.index("_CALL_BUILTIN_FAST_WITH_KEYWORDS") + load_attr_top = opnames.index("_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", 0, call) + load_attr_bottom = opnames.index("_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", call) + self.assertEqual(opnames[:load_attr_top].count("_GUARD_TYPE_VERSION"), 1) + self.assertEqual(opnames[call:load_attr_bottom].count("_CHECK_VALIDITY"), 1) + + def test_guard_type_version_executor_invalidated(self): + """ + Verify that the executor is invalided on a type change. + """ + + def thing(a): + x = 0 + for i in range(TIER2_THRESHOLD): + x += a.attr + x += a.attr + return x + + class Foo: + attr = 1 + + res, ex = self._run_with_optimizer(thing, Foo()) + self.assertEqual(res, TIER2_THRESHOLD * 2) + self.assertIsNotNone(ex) + self.assertEqual(list(iter_opnames(ex)).count("_GUARD_TYPE_VERSION"), 1) + self.assertTrue(ex.is_valid()) + Foo.attr = 0 + self.assertFalse(ex.is_valid()) + + def test_type_version_doesnt_segfault(self): + """ + Tests that setting a type version doesn't cause a segfault when later looking at the stack. + """ + + # Minimized from mdp.py benchmark + + class A: + def __init__(self): + self.attr = {} + + def method(self, arg): + self.attr[arg] = None + + def fn(a): + for _ in range(100): + (_ for _ in []) + (_ for _ in [a.method(None)]) + + fn(A()) + + def test_func_guards_removed_or_reduced(self): + def testfunc(n): + for i in range(n): + # Only works on functions promoted to constants + global_identity(i) + + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_PUSH_FRAME", uops) + # Strength reduced version + self.assertIn("_CHECK_FUNCTION_VERSION_INLINE", uops) + self.assertNotIn("_CHECK_FUNCTION_VERSION", uops) + # Removed guard + self.assertNotIn("_CHECK_FUNCTION_EXACT_ARGS", uops) + + def test_jit_error_pops(self): + """ + Tests that the correct number of pops are inserted into the + exit stub + """ + items = 17 * [None] + [[]] + with self.assertRaises(TypeError): + {item for item in items} + + def test_power_type_depends_on_input_values(self): + template = textwrap.dedent(""" + import _testinternalcapi + + L, R, X, Y = {l}, {r}, {x}, {y} + + def check(actual: complex, expected: complex) -> None: + assert actual == expected, (actual, expected) + assert type(actual) is type(expected), (actual, expected) + + def f(l: complex, r: complex) -> None: + expected_local_local = pow(l, r) + pow(l, r) + expected_const_local = pow(L, r) + pow(L, r) + expected_local_const = pow(l, R) + pow(l, R) + expected_const_const = pow(L, R) + pow(L, R) + for _ in range(_testinternalcapi.TIER2_THRESHOLD): + # Narrow types: + l + l, r + r + # The powers produce results, and the addition is unguarded: + check(l ** r + l ** r, expected_local_local) + check(L ** r + L ** r, expected_const_local) + check(l ** R + l ** R, expected_local_const) + check(L ** R + L ** R, expected_const_const) + + # JIT for one pair of values... + f(L, R) + # ...then run with another: + f(X, Y) + """) + interesting = [ + (1, 1), # int ** int -> int + (1, -1), # int ** int -> float + (1.0, 1), # float ** int -> float + (1, 1.0), # int ** float -> float + (-1, 0.5), # int ** float -> complex + (1.0, 1.0), # float ** float -> float + (-1.0, 0.5), # float ** float -> complex + ] + for (l, r), (x, y) in itertools.product(interesting, repeat=2): + s = template.format(l=l, r=r, x=x, y=y) + with self.subTest(l=l, r=r, x=x, y=y): + script_helper.assert_python_ok("-c", s) + + def test_symbols_flow_through_tuples(self): + def testfunc(n): + for _ in range(n): + a = 1 + b = 2 + t = a, b + x, y = t + r = x + y + return r + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, 3) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_ADD_INT", uops) + self.assertNotIn("_GUARD_BOTH_INT", uops) + self.assertNotIn("_GUARD_NOS_INT", uops) + self.assertNotIn("_GUARD_TOS_INT", uops) + + def test_decref_escapes(self): + class Convert9999ToNone: + def __del__(self): + ns = sys._getframe(1).f_locals + if ns["i"] == _testinternalcapi.TIER2_THRESHOLD: + ns["i"] = None + + def crash_addition(): + try: + for i in range(_testinternalcapi.TIER2_THRESHOLD + 1): + n = Convert9999ToNone() + i + i # Remove guards for i. + n = None # Change i. + i + i # This crashed when we didn't treat DECREF as escaping (gh-124483) + except TypeError: + pass + + crash_addition() + + +def global_identity(x): + return x if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_run.py b/Lib/test/test_capi/test_run.py new file mode 100644 index 00000000000000..894f66b437a39c --- /dev/null +++ b/Lib/test/test_capi/test_run.py @@ -0,0 +1,118 @@ +import os +import unittest +from collections import UserDict +from test.support import import_helper +from test.support.os_helper import unlink, TESTFN, TESTFN_ASCII, TESTFN_UNDECODABLE + +NULL = None +_testcapi = import_helper.import_module('_testcapi') +Py_single_input = _testcapi.Py_single_input +Py_file_input = _testcapi.Py_file_input +Py_eval_input = _testcapi.Py_eval_input + + +class DictSubclass(dict): + pass + + +class CAPITest(unittest.TestCase): + # TODO: Test the following functions: + # + # PyRun_SimpleStringFlags + # PyRun_AnyFileExFlags + # PyRun_SimpleFileExFlags + # PyRun_InteractiveOneFlags + # PyRun_InteractiveOneObject + # PyRun_InteractiveLoopFlags + # PyRun_String (may be a macro) + # PyRun_AnyFile (may be a macro) + # PyRun_AnyFileEx (may be a macro) + # PyRun_AnyFileFlags (may be a macro) + # PyRun_SimpleString (may be a macro) + # PyRun_SimpleFile (may be a macro) + # PyRun_SimpleFileEx (may be a macro) + # PyRun_InteractiveOne (may be a macro) + # PyRun_InteractiveLoop (may be a macro) + # PyRun_File (may be a macro) + # PyRun_FileEx (may be a macro) + # PyRun_FileFlags (may be a macro) + + def test_run_stringflags(self): + # Test PyRun_StringFlags(). + # XXX: fopen() uses different path encoding than Python on Windows. + def run(s, *args): + return _testcapi.run_stringflags(s, Py_file_input, *args) + source = b'a\n' + + self.assertIsNone(run(b'a\n', dict(a=1))) + self.assertIsNone(run(b'a\n', dict(a=1), {})) + self.assertIsNone(run(b'a\n', {}, dict(a=1))) + self.assertIsNone(run(b'a\n', {}, UserDict(a=1))) + + self.assertRaises(NameError, run, b'a\n', {}) + self.assertRaises(NameError, run, b'a\n', {}, {}) + self.assertRaises(TypeError, run, b'a\n', dict(a=1), []) + self.assertRaises(TypeError, run, b'a\n', dict(a=1), 1) + + self.assertIsNone(run(b'a\n', DictSubclass(a=1))) + self.assertIsNone(run(b'a\n', DictSubclass(), dict(a=1))) + self.assertRaises(NameError, run, b'a\n', DictSubclass()) + + self.assertIsNone(run(b'\xc3\xa4\n', {'\xe4': 1})) + self.assertRaises(SyntaxError, run, b'\xe4\n', {}) + + self.assertRaises(SystemError, run, b'a\n', NULL) + self.assertRaises(SystemError, run, b'a\n', NULL, {}) + self.assertRaises(SystemError, run, b'a\n', NULL, dict(a=1)) + self.assertRaises(SystemError, run, b'a\n', UserDict()) + self.assertRaises(SystemError, run, b'a\n', UserDict(), {}) + self.assertRaises(SystemError, run, b'a\n', UserDict(), dict(a=1)) + + # CRASHES run(NULL, {}) + + def test_run_fileexflags(self): + # Test PyRun_FileExFlags(). + filename = os.fsencode(TESTFN if os.name != 'nt' else TESTFN_ASCII) + with open(filename, 'wb') as fp: + fp.write(b'a\n') + self.addCleanup(unlink, filename) + def run(*args): + return _testcapi.run_fileexflags(filename, Py_file_input, *args) + + self.assertIsNone(run(dict(a=1))) + self.assertIsNone(run(dict(a=1), {})) + self.assertIsNone(run({}, dict(a=1))) + self.assertIsNone(run({}, UserDict(a=1))) + self.assertIsNone(run(dict(a=1), {}, 1)) # closeit = True + + self.assertRaises(NameError, run, {}) + self.assertRaises(NameError, run, {}, {}) + self.assertRaises(TypeError, run, dict(a=1), []) + self.assertRaises(TypeError, run, dict(a=1), 1) + + self.assertIsNone(run(DictSubclass(a=1))) + self.assertIsNone(run(DictSubclass(), dict(a=1))) + self.assertRaises(NameError, run, DictSubclass()) + + self.assertRaises(SystemError, run, NULL) + self.assertRaises(SystemError, run, NULL, {}) + self.assertRaises(SystemError, run, NULL, dict(a=1)) + self.assertRaises(SystemError, run, UserDict()) + self.assertRaises(SystemError, run, UserDict(), {}) + self.assertRaises(SystemError, run, UserDict(), dict(a=1)) + + @unittest.skipUnless(TESTFN_UNDECODABLE, 'only works if there are undecodable paths') + @unittest.skipIf(os.name == 'nt', 'does not work on Windows') + def test_run_fileexflags_with_undecodable_filename(self): + run = _testcapi.run_fileexflags + try: + with open(TESTFN_UNDECODABLE, 'wb') as fp: + fp.write(b'a\n') + self.addCleanup(unlink, TESTFN_UNDECODABLE) + except OSError: + self.skipTest('undecodable paths are not supported') + self.assertIsNone(run(TESTFN_UNDECODABLE, Py_file_input, dict(a=1))) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_capi/test_set.py b/Lib/test/test_capi/test_set.py index 5235f81543e0b6..62d90a3f94326d 100644 --- a/Lib/test/test_capi/test_set.py +++ b/Lib/test/test_capi/test_set.py @@ -2,8 +2,10 @@ from test.support import import_helper -# Skip this test if the _testcapi or _testinternalcapi modules aren't available. +# Skip this test if the _testcapi, _testlimitedcapi or _testinternalcapi +# modules aren't available. _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') _testinternalcapi = import_helper.import_module('_testinternalcapi') class set_subclass(set): @@ -23,7 +25,7 @@ def assertImmutable(self, action, *args): class TestSetCAPI(BaseSetTests, unittest.TestCase): def test_set_check(self): - check = _testcapi.set_check + check = _testlimitedcapi.set_check self.assertTrue(check(set())) self.assertTrue(check({1, 2})) self.assertFalse(check(frozenset())) @@ -33,7 +35,7 @@ def test_set_check(self): # CRASHES: check(NULL) def test_set_check_exact(self): - check = _testcapi.set_checkexact + check = _testlimitedcapi.set_checkexact self.assertTrue(check(set())) self.assertTrue(check({1, 2})) self.assertFalse(check(frozenset())) @@ -43,7 +45,7 @@ def test_set_check_exact(self): # CRASHES: check(NULL) def test_frozenset_check(self): - check = _testcapi.frozenset_check + check = _testlimitedcapi.frozenset_check self.assertFalse(check(set())) self.assertTrue(check(frozenset())) self.assertTrue(check(frozenset({1, 2}))) @@ -53,7 +55,7 @@ def test_frozenset_check(self): # CRASHES: check(NULL) def test_frozenset_check_exact(self): - check = _testcapi.frozenset_checkexact + check = _testlimitedcapi.frozenset_checkexact self.assertFalse(check(set())) self.assertTrue(check(frozenset())) self.assertTrue(check(frozenset({1, 2}))) @@ -63,7 +65,7 @@ def test_frozenset_check_exact(self): # CRASHES: check(NULL) def test_anyset_check(self): - check = _testcapi.anyset_check + check = _testlimitedcapi.anyset_check self.assertTrue(check(set())) self.assertTrue(check({1, 2})) self.assertTrue(check(frozenset())) @@ -74,7 +76,7 @@ def test_anyset_check(self): # CRASHES: check(NULL) def test_anyset_check_exact(self): - check = _testcapi.anyset_checkexact + check = _testlimitedcapi.anyset_checkexact self.assertTrue(check(set())) self.assertTrue(check({1, 2})) self.assertTrue(check(frozenset())) @@ -85,7 +87,7 @@ def test_anyset_check_exact(self): # CRASHES: check(NULL) def test_set_new(self): - set_new = _testcapi.set_new + set_new = _testlimitedcapi.set_new self.assertEqual(set_new().__class__, set) self.assertEqual(set_new(), set()) self.assertEqual(set_new((1, 1, 2)), {1, 2}) @@ -98,7 +100,7 @@ def test_set_new(self): set_new((1, {})) def test_frozenset_new(self): - frozenset_new = _testcapi.frozenset_new + frozenset_new = _testlimitedcapi.frozenset_new self.assertEqual(frozenset_new().__class__, frozenset) self.assertEqual(frozenset_new(), frozenset()) self.assertEqual(frozenset_new((1, 1, 2)), frozenset({1, 2})) @@ -111,7 +113,7 @@ def test_frozenset_new(self): frozenset_new((1, {})) def test_set_size(self): - get_size = _testcapi.set_size + get_size = _testlimitedcapi.set_size self.assertEqual(get_size(set()), 0) self.assertEqual(get_size(frozenset()), 0) self.assertEqual(get_size({1, 1, 2}), 2) @@ -134,7 +136,7 @@ def test_set_get_size(self): # CRASHES: get_size(object()) def test_set_contains(self): - contains = _testcapi.set_contains + contains = _testlimitedcapi.set_contains for cls in (set, frozenset, set_subclass, frozenset_subclass): with self.subTest(cls=cls): instance = cls((1, 2)) @@ -147,7 +149,7 @@ def test_set_contains(self): # CRASHES: contains(NULL, NULL) def test_add(self): - add = _testcapi.set_add + add = _testlimitedcapi.set_add for cls in (set, set_subclass): with self.subTest(cls=cls): instance = cls((1, 2)) @@ -165,7 +167,7 @@ def test_add(self): # CRASHES: add(NULL, NULL) def test_discard(self): - discard = _testcapi.set_discard + discard = _testlimitedcapi.set_discard for cls in (set, set_subclass): with self.subTest(cls=cls): instance = cls((1, 2)) @@ -187,7 +189,7 @@ def test_discard(self): # CRASHES: discard(NULL, NULL) def test_pop(self): - pop = _testcapi.set_pop + pop = _testlimitedcapi.set_pop orig = (1, 2) for cls in (set, set_subclass): with self.subTest(cls=cls): @@ -204,7 +206,7 @@ def test_pop(self): # CRASHES: pop(NULL) def test_clear(self): - clear = _testcapi.set_clear + clear = _testlimitedcapi.set_clear for cls in (set, set_subclass): with self.subTest(cls=cls): instance = cls((1, 2)) @@ -263,3 +265,7 @@ def test_set_next_entry(self): with self.assertRaises(SystemError): set_next(object(), 0) # CRASHES: set_next(NULL, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_structmembers.py b/Lib/test/test_capi/test_structmembers.py index 08ca1f828529cf..f14ad9a9a5f512 100644 --- a/Lib/test/test_capi/test_structmembers.py +++ b/Lib/test/test_capi/test_structmembers.py @@ -1,6 +1,5 @@ import unittest from test.support import import_helper -from test.support import warnings_helper # Skip this test if the _testcapi module isn't available. import_helper.import_module('_testcapi') @@ -38,6 +37,9 @@ def _make_test_object(cls): 9.99999,# T_FLOAT 10.1010101010, # T_DOUBLE "hi", # T_STRING_INPLACE + 12, # T_LONGLONG + 13, # T_ULONGLONG + b"c", # T_CHAR ) @@ -163,6 +165,16 @@ def test_inplace_string(self): self.assertRaises(TypeError, setattr, ts, "T_STRING_INPLACE", "s") self.assertRaises(TypeError, delattr, ts, "T_STRING_INPLACE") + def test_char(self): + ts = self.ts + self.assertEqual(ts.T_CHAR, "c") + ts.T_CHAR = "z" + self.assertEqual(ts.T_CHAR, "z") + self.assertRaises(TypeError, setattr, ts, "T_CHAR", "") + self.assertRaises(TypeError, setattr, ts, "T_CHAR", b"a") + self.assertRaises(TypeError, setattr, ts, "T_CHAR", bytearray(b"b")) + self.assertRaises(TypeError, delattr, ts, "T_STRING_INPLACE") + class ReadWriteTests_OldAPI(ReadWriteTests, unittest.TestCase): cls = _test_structmembersType_OldAPI diff --git a/Lib/test/test_capi/test_sys.py b/Lib/test/test_capi/test_sys.py index 08bf0370fc59b5..d3a9b378e7769a 100644 --- a/Lib/test/test_capi/test_sys.py +++ b/Lib/test/test_capi/test_sys.py @@ -5,9 +5,9 @@ from test.support import import_helper try: - import _testcapi + import _testlimitedcapi except ImportError: - _testcapi = None + _testlimitedcapi = None NULL = None @@ -20,10 +20,10 @@ class CAPITest(unittest.TestCase): maxDiff = None @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_sys_getobject(self): # Test PySys_GetObject() - getobject = _testcapi.sys_getobject + getobject = _testlimitedcapi.sys_getobject self.assertIs(getobject(b'stdout'), sys.stdout) with support.swap_attr(sys, '\U0001f40d', 42): @@ -38,10 +38,10 @@ def test_sys_getobject(self): # CRASHES getobject(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_sys_setobject(self): # Test PySys_SetObject() - setobject = _testcapi.sys_setobject + setobject = _testlimitedcapi.sys_setobject value = ['value'] value2 = ['value2'] @@ -51,7 +51,7 @@ def test_sys_setobject(self): self.assertEqual(setobject(b'newattr', value2), 0) self.assertIs(sys.newattr, value2) self.assertEqual(setobject(b'newattr', NULL), 0) - self.assertFalse(hasattr(sys, 'newattr')) + self.assertNotHasAttr(sys, 'newattr') self.assertEqual(setobject(b'newattr', NULL), 0) finally: with contextlib.suppress(AttributeError): @@ -60,7 +60,7 @@ def test_sys_setobject(self): self.assertEqual(setobject('\U0001f40d'.encode(), value), 0) self.assertIs(getattr(sys, '\U0001f40d'), value) self.assertEqual(setobject('\U0001f40d'.encode(), NULL), 0) - self.assertFalse(hasattr(sys, '\U0001f40d')) + self.assertNotHasAttr(sys, '\U0001f40d') finally: with contextlib.suppress(AttributeError): delattr(sys, '\U0001f40d') @@ -70,10 +70,10 @@ def test_sys_setobject(self): # CRASHES setobject(NULL, value) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_sys_getxoptions(self): # Test PySys_GetXOptions() - getxoptions = _testcapi.sys_getxoptions + getxoptions = _testlimitedcapi.sys_getxoptions self.assertIs(getxoptions(), sys._xoptions) diff --git a/Lib/test/test_capi/test_time.py b/Lib/test/test_capi/test_time.py index 10b7fbf2c372a3..989c158818ccf0 100644 --- a/Lib/test/test_capi/test_time.py +++ b/Lib/test/test_capi/test_time.py @@ -18,11 +18,6 @@ def test_min_max(self): self.assertEqual(PyTime_MIN, -2**63) self.assertEqual(PyTime_MAX, 2**63 - 1) - def check_clock(self, c_func, py_func): - t1 = c_func() - t2 = py_func() - self.assertAlmostEqual(t1, t2, delta=CLOCK_RES) - def test_assecondsdouble(self): # Test PyTime_AsSecondsDouble() def ns_to_sec(ns): @@ -58,14 +53,26 @@ def ns_to_sec(ns): self.assertEqual(_testcapi.PyTime_AsSecondsDouble(ns), ns_to_sec(ns)) + def check_clock(self, c_func, py_func): + t1 = c_func() + t2 = py_func() + self.assertAlmostEqual(t1, t2, delta=CLOCK_RES) + def test_monotonic(self): - # Test PyTime_Monotonic() + # Test PyTime_Monotonic() and PyTime_MonotonicRaw() self.check_clock(_testcapi.PyTime_Monotonic, time.monotonic) + self.check_clock(_testcapi.PyTime_MonotonicRaw, time.monotonic) def test_perf_counter(self): - # Test PyTime_PerfCounter() + # Test PyTime_PerfCounter() and PyTime_PerfCounterRaw() self.check_clock(_testcapi.PyTime_PerfCounter, time.perf_counter) + self.check_clock(_testcapi.PyTime_PerfCounterRaw, time.perf_counter) def test_time(self): - # Test PyTime_time() + # Test PyTime_Time() and PyTime_TimeRaw() self.check_clock(_testcapi.PyTime_Time, time.time) + self.check_clock(_testcapi.PyTime_TimeRaw, time.time) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_tuple.py b/Lib/test/test_capi/test_tuple.py new file mode 100644 index 00000000000000..6349467c5d6b70 --- /dev/null +++ b/Lib/test/test_capi/test_tuple.py @@ -0,0 +1,286 @@ +import unittest +import sys +import gc +from collections import namedtuple +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') + +NULL = None +PY_SSIZE_T_MIN = _testcapi.PY_SSIZE_T_MIN +PY_SSIZE_T_MAX = _testcapi.PY_SSIZE_T_MAX + +class TupleSubclass(tuple): + pass + + +class CAPITest(unittest.TestCase): + def test_check(self): + # Test PyTuple_Check() + check = _testlimitedcapi.tuple_check + + self.assertTrue(check((1, 2))) + self.assertTrue(check(())) + self.assertTrue(check(TupleSubclass((1, 2)))) + self.assertFalse(check({1: 2})) + self.assertFalse(check([1, 2])) + self.assertFalse(check(42)) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + def test_tuple_checkexact(self): + # Test PyTuple_CheckExact() + check = _testlimitedcapi.tuple_checkexact + + self.assertTrue(check((1, 2))) + self.assertTrue(check(())) + self.assertFalse(check(TupleSubclass((1, 2)))) + self.assertFalse(check({1: 2})) + self.assertFalse(check([1, 2])) + self.assertFalse(check(42)) + self.assertFalse(check(object())) + + # CRASHES check(NULL) + + def test_tuple_new(self): + # Test PyTuple_New() + tuple_new = _testlimitedcapi.tuple_new + size = _testlimitedcapi.tuple_size + checknull = _testcapi._check_tuple_item_is_NULL + + tup1 = tuple_new(0) + self.assertEqual(tup1, ()) + self.assertEqual(size(tup1), 0) + self.assertIs(type(tup1), tuple) + tup2 = tuple_new(1) + self.assertIs(type(tup2), tuple) + self.assertEqual(size(tup2), 1) + self.assertIsNot(tup2, tup1) + self.assertTrue(checknull(tup2, 0)) + + self.assertRaises(SystemError, tuple_new, -1) + self.assertRaises(SystemError, tuple_new, PY_SSIZE_T_MIN) + self.assertRaises(MemoryError, tuple_new, PY_SSIZE_T_MAX) + + def test_tuple_pack(self): + # Test PyTuple_Pack() + pack = _testlimitedcapi.tuple_pack + + self.assertEqual(pack(0), ()) + self.assertEqual(pack(1, [1]), ([1],)) + self.assertEqual(pack(2, [1], [2]), ([1], [2])) + + self.assertRaises(SystemError, pack, PY_SSIZE_T_MIN) + self.assertRaises(SystemError, pack, -1) + self.assertRaises(MemoryError, pack, PY_SSIZE_T_MAX) + + # CRASHES pack(1, NULL) + # CRASHES pack(2, [1]) + + def test_tuple_size(self): + # Test PyTuple_Size() + size = _testlimitedcapi.tuple_size + + self.assertEqual(size(()), 0) + self.assertEqual(size((1, 2)), 2) + self.assertEqual(size(TupleSubclass((1, 2))), 2) + + self.assertRaises(SystemError, size, []) + self.assertRaises(SystemError, size, 42) + self.assertRaises(SystemError, size, object()) + + # CRASHES size(NULL) + + def test_tuple_get_size(self): + # Test PyTuple_GET_SIZE() + size = _testcapi.tuple_get_size + + self.assertEqual(size(()), 0) + self.assertEqual(size((1, 2)), 2) + self.assertEqual(size(TupleSubclass((1, 2))), 2) + + def test_tuple_getitem(self): + # Test PyTuple_GetItem() + getitem = _testlimitedcapi.tuple_getitem + + tup = ([1], [2], [3]) + self.assertEqual(getitem(tup, 0), [1]) + self.assertEqual(getitem(tup, 2), [3]) + + tup2 = TupleSubclass(([1], [2], [3])) + self.assertEqual(getitem(tup2, 0), [1]) + self.assertEqual(getitem(tup2, 2), [3]) + + self.assertRaises(IndexError, getitem, tup, PY_SSIZE_T_MIN) + self.assertRaises(IndexError, getitem, tup, -1) + self.assertRaises(IndexError, getitem, tup, len(tup)) + self.assertRaises(IndexError, getitem, tup, PY_SSIZE_T_MAX) + self.assertRaises(SystemError, getitem, [1, 2, 3], 1) + self.assertRaises(SystemError, getitem, 42, 1) + + # CRASHES getitem(NULL, 0) + + def test_tuple_get_item(self): + # Test PyTuple_GET_ITEM() + get_item = _testcapi.tuple_get_item + + tup = ([1], [2], [3]) + self.assertEqual(get_item(tup, 0), [1]) + self.assertEqual(get_item(tup, 2), [3]) + + tup2 = TupleSubclass(([1], [2], [3])) + self.assertEqual(get_item(tup2, 0), [1]) + self.assertEqual(get_item(tup2, 2), [3]) + + # CRASHES get_item(NULL, 0) + + def test_tuple_getslice(self): + # Test PyTuple_GetSlice() + getslice = _testlimitedcapi.tuple_getslice + + # empty + tup = ([1], [2], [3]) + self.assertEqual(getslice(tup, PY_SSIZE_T_MIN, 0), ()) + self.assertEqual(getslice(tup, -1, 0), ()) + self.assertEqual(getslice(tup, 3, PY_SSIZE_T_MAX), ()) + self.assertEqual(getslice(tup, 1, 1), ()) + self.assertEqual(getslice(tup, 2, 1), ()) + tup = TupleSubclass(([1], [2], [3])) + self.assertEqual(getslice(tup, PY_SSIZE_T_MIN, 0), ()) + self.assertEqual(getslice(tup, -1, 0), ()) + self.assertEqual(getslice(tup, 3, PY_SSIZE_T_MAX), ()) + self.assertEqual(getslice(tup, 1, 1), ()) + self.assertEqual(getslice(tup, 2, 1), ()) + + # slice + tup = ([1], [2], [3], [4]) + self.assertEqual(getslice(tup, 1, 3), ([2], [3])) + tup = TupleSubclass(([1], [2], [3], [4])) + self.assertEqual(getslice(tup, 1, 3), ([2], [3])) + + # whole + tup = ([1], [2], [3]) + self.assertEqual(getslice(tup, 0, 3), tup) + self.assertEqual(getslice(tup, 0, 100), tup) + self.assertEqual(getslice(tup, -100, 100), tup) + tup = TupleSubclass(([1], [2], [3])) + self.assertEqual(getslice(tup, 0, 3), tup) + self.assertEqual(getslice(tup, 0, 100), tup) + self.assertEqual(getslice(tup, -100, 100), tup) + + self.assertRaises(SystemError, getslice, [[1], [2], [3]], 0, 0) + self.assertRaises(SystemError, getslice, 42, 0, 0) + + # CRASHES getslice(NULL, 0, 0) + + def test_tuple_setitem(self): + # Test PyTuple_SetItem() + setitem = _testlimitedcapi.tuple_setitem + checknull = _testcapi._check_tuple_item_is_NULL + + tup = ([1], [2]) + self.assertEqual(setitem(tup, 0, []), ([], [2])) + self.assertEqual(setitem(tup, 1, []), ([1], [])) + + tup2 = setitem(tup, 1, NULL) + self.assertTrue(checknull(tup2, 1)) + + tup2 = TupleSubclass(([1], [2])) + self.assertRaises(SystemError, setitem, tup2, 0, []) + + self.assertRaises(IndexError, setitem, tup, PY_SSIZE_T_MIN, []) + self.assertRaises(IndexError, setitem, tup, -1, []) + self.assertRaises(IndexError, setitem, tup, len(tup), []) + self.assertRaises(IndexError, setitem, tup, PY_SSIZE_T_MAX, []) + self.assertRaises(SystemError, setitem, [1], 0, []) + self.assertRaises(SystemError, setitem, 42, 0, []) + + # CRASHES setitem(NULL, 0, []) + + def test_tuple_set_item(self): + # Test PyTuple_SET_ITEM() + set_item = _testcapi.tuple_set_item + checknull = _testcapi._check_tuple_item_is_NULL + + tup = ([1], [2]) + self.assertEqual(set_item(tup, 0, []), ([], [2])) + self.assertEqual(set_item(tup, 1, []), ([1], [])) + + tup2 = set_item(tup, 1, NULL) + self.assertTrue(checknull(tup2, 1)) + + tup2 = TupleSubclass(([1], [2])) + self.assertIs(set_item(tup2, 0, []), tup2) + self.assertEqual(tup2, ([], [2])) + + # CRASHES set_item(tup, -1, []) + # CRASHES set_item(tup, len(tup), []) + # CRASHES set_item([1], 0, []) + # CRASHES set_item(NULL, 0, []) + + def test__tuple_resize(self): + # Test _PyTuple_Resize() + resize = _testcapi._tuple_resize + checknull = _testcapi._check_tuple_item_is_NULL + + a = () + b = resize(a, 0, False) + self.assertEqual(len(a), 0) + self.assertEqual(len(b), 0) + b = resize(a, 2, False) + self.assertEqual(len(a), 0) + self.assertEqual(len(b), 2) + self.assertTrue(checknull(b, 0)) + self.assertTrue(checknull(b, 1)) + + a = ([1], [2], [3]) + b = resize(a, 3) + self.assertEqual(b, a) + b = resize(a, 2) + self.assertEqual(b, a[:2]) + b = resize(a, 5) + self.assertEqual(len(b), 5) + self.assertEqual(b[:3], a) + self.assertTrue(checknull(b, 3)) + self.assertTrue(checknull(b, 4)) + + a = () + self.assertRaises(MemoryError, resize, a, PY_SSIZE_T_MAX) + self.assertRaises(SystemError, resize, a, -1) + self.assertRaises(SystemError, resize, a, PY_SSIZE_T_MIN) + # refcount > 1 + a = (1, 2, 3) + self.assertRaises(SystemError, resize, a, 3, False) + self.assertRaises(SystemError, resize, a, 0, False) + # non-tuple + self.assertRaises(SystemError, resize, [1, 2, 3], 0, False) + self.assertRaises(SystemError, resize, NULL, 0, False) + + def test_bug_59313(self): + # Before 3.14, the C-API function PySequence_Tuple + # would create incomplete tuples which were visible to + # the cycle GC, and this test would crash the interpeter. + TAG = object() + tuples = [] + + def referrer_tuples(): + return [x for x in gc.get_referrers(TAG) + if isinstance(x, tuple)] + + def my_iter(): + nonlocal tuples + yield TAG # 'tag' gets stored in the result tuple + tuples += referrer_tuples() + for x in range(10): + tuples += referrer_tuples() + # Prior to 3.13 would raise a SystemError when the tuple needs to be resized + yield x + + self.assertEqual(tuple(my_iter()), (TAG, *range(10))) + self.assertEqual(tuples, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_type.py b/Lib/test/test_capi/test_type.py new file mode 100644 index 00000000000000..ffcaae73bca236 --- /dev/null +++ b/Lib/test/test_capi/test_type.py @@ -0,0 +1,76 @@ +from test.support import import_helper, Py_GIL_DISABLED, refleak_helper +import unittest + +_testcapi = import_helper.import_module('_testcapi') + + +class TypeTests(unittest.TestCase): + def test_freeze(self): + # test PyType_Freeze() + type_freeze = _testcapi.type_freeze + + # simple case, no inherante + class MyType: + pass + MyType.attr = "mutable" + + type_freeze(MyType) + err_msg = "cannot set 'attr' attribute of immutable type 'MyType'" + with self.assertRaisesRegex(TypeError, err_msg): + # the class is now immutable + MyType.attr = "immutable" + + # test MRO: PyType_Freeze() requires base classes to be immutable + class A: pass + class B: pass + class C(B): pass + class D(A, C): pass + + self.assertEqual(D.mro(), [D, A, C, B, object]) + with self.assertRaises(TypeError): + type_freeze(D) + + type_freeze(A) + type_freeze(B) + type_freeze(C) + # all parent classes are now immutable, so D can be made immutable + # as well + type_freeze(D) + + @unittest.skipIf( + Py_GIL_DISABLED and refleak_helper.hunting_for_refleaks(), + "Specialization failure triggers gh-127773") + def test_freeze_meta(self): + """test PyType_Freeze() with overridden MRO""" + type_freeze = _testcapi.type_freeze + + class Base: + value = 1 + + class Meta(type): + def mro(cls): + return (cls, Base, object) + + class FreezeThis(metaclass=Meta): + """This has `Base` in the MRO, but not tp_bases""" + + self.assertEqual(FreezeThis.value, 1) + + with self.assertRaises(TypeError): + type_freeze(FreezeThis) + + Base.value = 2 + self.assertEqual(FreezeThis.value, 2) + + type_freeze(Base) + with self.assertRaises(TypeError): + Base.value = 3 + type_freeze(FreezeThis) + self.assertEqual(FreezeThis.value, 2) + + def test_manual_heap_type(self): + # gh-128923: test that a manually allocated and initailized heap type + # works correctly + ManualHeapType = _testcapi.ManualHeapType + for i in range(100): + self.assertIsInstance(ManualHeapType(), ManualHeapType) diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index bb6161abf4da81..3408c10f426058 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -1,17 +1,25 @@ import unittest import sys from test import support -from test.support import import_helper +from test.support import threading_helper try: import _testcapi from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX except ImportError: _testcapi = None +try: + import _testlimitedcapi +except ImportError: + _testlimitedcapi = None try: import _testinternalcapi except ImportError: _testinternalcapi = None +try: + import ctypes +except ImportError: + ctypes = None NULL = None @@ -84,10 +92,10 @@ def test_fill(self): # TODO: Test PyUnicode_Fill() with non-modifiable unicode. @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_writechar(self): """Test PyUnicode_WriteChar()""" - from _testcapi import unicode_writechar as writechar + from _testlimitedcapi import unicode_writechar as writechar strings = [ # one string for every kind @@ -115,10 +123,10 @@ def test_writechar(self): # unicode. @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_resize(self): """Test PyUnicode_Resize()""" - from _testcapi import unicode_resize as resize + from _testlimitedcapi import unicode_resize as resize strings = [ # all strings have exactly 3 characters @@ -141,10 +149,10 @@ def test_resize(self): # and with NULL as the address. @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_append(self): """Test PyUnicode_Append()""" - from _testcapi import unicode_append as append + from _testlimitedcapi import unicode_append as append strings = [ 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', @@ -169,10 +177,10 @@ def test_append(self): # TODO: Check reference counts. @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_appendanddel(self): """Test PyUnicode_AppendAndDel()""" - from _testcapi import unicode_appendanddel as appendanddel + from _testlimitedcapi import unicode_appendanddel as appendanddel strings = [ 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', @@ -196,10 +204,10 @@ def test_appendanddel(self): # TODO: Check reference counts. @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_fromstringandsize(self): """Test PyUnicode_FromStringAndSize()""" - from _testcapi import unicode_fromstringandsize as fromstringandsize + from _testlimitedcapi import unicode_fromstringandsize as fromstringandsize self.assertEqual(fromstringandsize(b'abc'), 'abc') self.assertEqual(fromstringandsize(b'abc', 2), 'ab') @@ -221,10 +229,10 @@ def test_fromstringandsize(self): self.assertRaises(SystemError, fromstringandsize, NULL, PY_SSIZE_T_MAX) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_fromstring(self): """Test PyUnicode_FromString()""" - from _testcapi import unicode_fromstring as fromstring + from _testlimitedcapi import unicode_fromstring as fromstring self.assertEqual(fromstring(b'abc'), 'abc') self.assertEqual(fromstring(b'\xc2\xa1\xc2\xa2'), '\xa1\xa2') @@ -273,10 +281,10 @@ def test_fromkindanddata(self): # CRASHES fromkindanddata(4, b'\xff\xff\xff\xff') @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_substring(self): """Test PyUnicode_Substring()""" - from _testcapi import unicode_substring as substring + from _testlimitedcapi import unicode_substring as substring strings = [ 'ab', 'ab\xa1\xa2', @@ -297,10 +305,10 @@ def test_substring(self): # CRASHES substring(NULL, 0, 0) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_getlength(self): """Test PyUnicode_GetLength()""" - from _testcapi import unicode_getlength as getlength + from _testlimitedcapi import unicode_getlength as getlength for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', 'a\ud800b\udfffc', '\ud834\udd1e']: @@ -311,10 +319,10 @@ def test_getlength(self): # CRASHES getlength(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_readchar(self): """Test PyUnicode_ReadChar()""" - from _testcapi import unicode_readchar as readchar + from _testlimitedcapi import unicode_readchar as readchar for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', 'a\ud800b\udfffc', '\ud834\udd1e']: @@ -330,10 +338,10 @@ def test_readchar(self): # CRASHES readchar(NULL, 0) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_fromobject(self): """Test PyUnicode_FromObject()""" - from _testcapi import unicode_fromobject as fromobject + from _testlimitedcapi import unicode_fromobject as fromobject for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', 'a\ud800b\udfffc', '\ud834\udd1e']: @@ -348,13 +356,13 @@ def test_fromobject(self): self.assertRaises(TypeError, fromobject, []) # CRASHES fromobject(NULL) + @unittest.skipIf(ctypes is None, 'need ctypes') def test_from_format(self): """Test PyUnicode_FromFormat()""" # Length modifiers "j" and "t" are not tested here because ctypes does # not expose types for intmax_t and ptrdiff_t. - # _testcapi.test_string_from_format() has a wider coverage of all + # _testlimitedcapi.test_string_from_format() has a wider coverage of all # formats. - import_helper.import_module('ctypes') from ctypes import ( c_char_p, pythonapi, py_object, sizeof, @@ -411,8 +419,29 @@ def check_format(expected, format, *args): # truncated string check_format('abc', b'%.3s', b'abcdef') + check_format('abc[', + b'%.6s', 'abc[\u20ac]'.encode('utf8')) + check_format('abc[\u20ac', + b'%.7s', 'abc[\u20ac]'.encode('utf8')) + check_format('abc[\ufffd', + b'%.5s', b'abc[\xff]') + check_format('abc[', + b'%.6s', b'abc[\xe2\x82]') + check_format('abc[\ufffd]', + b'%.7s', b'abc[\xe2\x82]') check_format('abc[\ufffd', - b'%.5s', 'abc[\u20ac]'.encode('utf8')) + b'%.7s', b'abc[\xe2\x82\0') + check_format(' abc[', + b'%10.6s', 'abc[\u20ac]'.encode('utf8')) + check_format(' abc[\u20ac', + b'%10.7s', 'abc[\u20ac]'.encode('utf8')) + check_format(' abc[\ufffd', + b'%10.5s', b'abc[\xff]') + check_format(' abc[', + b'%10.6s', b'abc[\xe2\x82]') + check_format(' abc[\ufffd]', + b'%10.7s', b'abc[\xe2\x82]') + check_format("'\\u20acABC'", b'%A', '\u20acABC') check_format("'\\u20", @@ -425,10 +454,31 @@ def check_format(expected, format, *args): b'%.3S', '\u20acABCDEF') check_format('\u20acAB', b'%.3U', '\u20acABCDEF') + check_format('\u20acAB', b'%.3V', '\u20acABCDEF', None) + check_format('abc[', + b'%.6V', None, 'abc[\u20ac]'.encode('utf8')) + check_format('abc[\u20ac', + b'%.7V', None, 'abc[\u20ac]'.encode('utf8')) check_format('abc[\ufffd', - b'%.5V', None, 'abc[\u20ac]'.encode('utf8')) + b'%.5V', None, b'abc[\xff]') + check_format('abc[', + b'%.6V', None, b'abc[\xe2\x82]') + check_format('abc[\ufffd]', + b'%.7V', None, b'abc[\xe2\x82]') + check_format(' abc[', + b'%10.6V', None, 'abc[\u20ac]'.encode('utf8')) + check_format(' abc[\u20ac', + b'%10.7V', None, 'abc[\u20ac]'.encode('utf8')) + check_format(' abc[\ufffd', + b'%10.5V', None, b'abc[\xff]') + check_format(' abc[', + b'%10.6V', None, b'abc[\xe2\x82]') + check_format(' abc[\ufffd]', + b'%10.7V', None, b'abc[\xe2\x82]') + check_format(' abc[\ufffd', + b'%10.7V', None, b'abc[\xe2\x82\0') # following tests comes from #7330 # test width modifier and precision modifier with %S @@ -646,6 +696,40 @@ def check_format(expected, format, *args): check_format('\U0001f4bb+' if sizeof(c_wchar) > 2 else '\U0001f4bb', b'%.2lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) + # test %T + check_format('type: str', + b'type: %T', py_object("abc")) + check_format(f'type: st', + b'type: %.2T', py_object("abc")) + check_format(f'type: str', + b'type: %10T', py_object("abc")) + + class LocalType: + pass + obj = LocalType() + fullname = f'{__name__}.{LocalType.__qualname__}' + check_format(f'type: {fullname}', + b'type: %T', py_object(obj)) + fullname_alt = f'{__name__}:{LocalType.__qualname__}' + check_format(f'type: {fullname_alt}', + b'type: %#T', py_object(obj)) + + # test %N + check_format('type: str', + b'type: %N', py_object(str)) + check_format(f'type: st', + b'type: %.2N', py_object(str)) + check_format(f'type: str', + b'type: %10N', py_object(str)) + + check_format(f'type: {fullname}', + b'type: %N', py_object(type(obj))) + check_format(f'type: {fullname_alt}', + b'type: %#N', py_object(type(obj))) + with self.assertRaisesRegex(TypeError, "%N argument must be a type"): + check_format('type: str', + b'type: %N', py_object("abc")) + # test variable width and precision check_format(' abc', b'%*s', c_int(5), b'abc') check_format('ab', b'%.*s', c_int(2), b'abc') @@ -707,10 +791,10 @@ def check_format(expected, format, *args): PyUnicode_FromFormat, b'%+i', c_int(10)) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_interninplace(self): """Test PyUnicode_InternInPlace()""" - from _testcapi import unicode_interninplace as interninplace + from _testlimitedcapi import unicode_interninplace as interninplace s = b'abc'.decode() r = interninplace(s) @@ -720,10 +804,10 @@ def test_interninplace(self): # CRASHES interninplace(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_internfromstring(self): """Test PyUnicode_InternFromString()""" - from _testcapi import unicode_internfromstring as internfromstring + from _testlimitedcapi import unicode_internfromstring as internfromstring self.assertEqual(internfromstring(b'abc'), 'abc') self.assertEqual(internfromstring(b'\xf0\x9f\x98\x80'), '\U0001f600') @@ -734,10 +818,10 @@ def test_internfromstring(self): # CRASHES internfromstring(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_fromwidechar(self): """Test PyUnicode_FromWideChar()""" - from _testcapi import unicode_fromwidechar as fromwidechar + from _testlimitedcapi import unicode_fromwidechar as fromwidechar from _testcapi import SIZEOF_WCHAR_T if SIZEOF_WCHAR_T == 2: @@ -769,11 +853,11 @@ def test_fromwidechar(self): self.assertRaises(SystemError, fromwidechar, NULL, PY_SSIZE_T_MIN) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_aswidechar(self): """Test PyUnicode_AsWideChar()""" - from _testcapi import unicode_aswidechar - from _testcapi import unicode_aswidechar_null + from _testlimitedcapi import unicode_aswidechar + from _testlimitedcapi import unicode_aswidechar_null from _testcapi import SIZEOF_WCHAR_T wchar, size = unicode_aswidechar('abcdef', 2) @@ -817,11 +901,11 @@ def test_aswidechar(self): self.assertRaises(SystemError, unicode_aswidechar_null, NULL, 10) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_aswidecharstring(self): """Test PyUnicode_AsWideCharString()""" - from _testcapi import unicode_aswidecharstring - from _testcapi import unicode_aswidecharstring_null + from _testlimitedcapi import unicode_aswidecharstring + from _testlimitedcapi import unicode_aswidecharstring_null from _testcapi import SIZEOF_WCHAR_T wchar, size = unicode_aswidecharstring('abc') @@ -893,10 +977,10 @@ def test_asucs4copy(self): # CRASHES asucs4copy(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_fromordinal(self): """Test PyUnicode_FromOrdinal()""" - from _testcapi import unicode_fromordinal as fromordinal + from _testlimitedcapi import unicode_fromordinal as fromordinal self.assertEqual(fromordinal(0x61), 'a') self.assertEqual(fromordinal(0x20ac), '\u20ac') @@ -921,12 +1005,30 @@ def test_asutf8(self): self.assertRaises(TypeError, unicode_asutf8, [], 0) # CRASHES unicode_asutf8(NULL, 0) - @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @threading_helper.requires_working_threading() + def test_asutf8_race(self): + """Test that there's no race condition in PyUnicode_AsUTF8()""" + unicode_asutf8 = _testcapi.unicode_asutf8 + from threading import Thread + + data = "😊" + + def worker(): + for _ in range(1000): + self.assertEqual(unicode_asutf8(data, 5), b'\xf0\x9f\x98\x8a\0') + + threads = [Thread(target=worker) for _ in range(10)] + with threading_helper.start_threads(threads): + pass + + + @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_asutf8andsize(self): """Test PyUnicode_AsUTF8AndSize()""" - from _testcapi import unicode_asutf8andsize - from _testcapi import unicode_asutf8andsize_null + from _testlimitedcapi import unicode_asutf8andsize + from _testlimitedcapi import unicode_asutf8andsize_null self.assertEqual(unicode_asutf8andsize('abc', 4), (b'abc\0', 3)) self.assertEqual(unicode_asutf8andsize('абв', 7), (b'\xd0\xb0\xd0\xb1\xd0\xb2\0', 6)) @@ -945,10 +1047,10 @@ def test_asutf8andsize(self): # CRASHES unicode_asutf8andsize_null(NULL, 0) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_getdefaultencoding(self): """Test PyUnicode_GetDefaultEncoding()""" - from _testcapi import unicode_getdefaultencoding as getdefaultencoding + from _testlimitedcapi import unicode_getdefaultencoding as getdefaultencoding self.assertEqual(getdefaultencoding(), b'utf-8') @@ -973,10 +1075,10 @@ def test_transform_decimal_and_space(self): # CRASHES transform_decimal(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_concat(self): """Test PyUnicode_Concat()""" - from _testcapi import unicode_concat as concat + from _testlimitedcapi import unicode_concat as concat self.assertEqual(concat('abc', 'def'), 'abcdef') self.assertEqual(concat('abc', 'где'), 'abcгде') @@ -994,10 +1096,10 @@ def test_concat(self): # CRASHES concat('abc', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_split(self): """Test PyUnicode_Split()""" - from _testcapi import unicode_split as split + from _testlimitedcapi import unicode_split as split self.assertEqual(split('a|b|c|d', '|'), ['a', 'b', 'c', 'd']) self.assertEqual(split('a|b|c|d', '|', 2), ['a', 'b', 'c|d']) @@ -1022,10 +1124,10 @@ def test_split(self): # CRASHES split(NULL, '|') @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_rsplit(self): """Test PyUnicode_RSplit()""" - from _testcapi import unicode_rsplit as rsplit + from _testlimitedcapi import unicode_rsplit as rsplit self.assertEqual(rsplit('a|b|c|d', '|'), ['a', 'b', 'c', 'd']) self.assertEqual(rsplit('a|b|c|d', '|', 2), ['a|b', 'c', 'd']) @@ -1051,10 +1153,10 @@ def test_rsplit(self): # CRASHES rsplit(NULL, '|') @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_partition(self): """Test PyUnicode_Partition()""" - from _testcapi import unicode_partition as partition + from _testlimitedcapi import unicode_partition as partition self.assertEqual(partition('a|b|c', '|'), ('a', '|', 'b|c')) self.assertEqual(partition('a||b||c', '||'), ('a', '||', 'b||c')) @@ -1071,10 +1173,10 @@ def test_partition(self): # CRASHES partition('a|b|c', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_rpartition(self): """Test PyUnicode_RPartition()""" - from _testcapi import unicode_rpartition as rpartition + from _testlimitedcapi import unicode_rpartition as rpartition self.assertEqual(rpartition('a|b|c', '|'), ('a|b', '|', 'c')) self.assertEqual(rpartition('a||b||c', '||'), ('a||b', '||', 'c')) @@ -1091,10 +1193,10 @@ def test_rpartition(self): # CRASHES rpartition('a|b|c', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_splitlines(self): """Test PyUnicode_SplitLines()""" - from _testcapi import unicode_splitlines as splitlines + from _testlimitedcapi import unicode_splitlines as splitlines self.assertEqual(splitlines('a\nb\rc\r\nd'), ['a', 'b', 'c', 'd']) self.assertEqual(splitlines('a\nb\rc\r\nd', True), @@ -1109,10 +1211,10 @@ def test_splitlines(self): # CRASHES splitlines(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_translate(self): """Test PyUnicode_Translate()""" - from _testcapi import unicode_translate as translate + from _testlimitedcapi import unicode_translate as translate self.assertEqual(translate('abcd', {ord('a'): 'A', ord('b'): ord('B'), ord('c'): '<>'}), 'AB<>d') self.assertEqual(translate('абвг', {ord('а'): 'А', ord('б'): ord('Б'), ord('в'): '<>'}), 'АБ<>г') @@ -1134,10 +1236,10 @@ def test_translate(self): # CRASHES translate(NULL, []) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_join(self): """Test PyUnicode_Join()""" - from _testcapi import unicode_join as join + from _testlimitedcapi import unicode_join as join self.assertEqual(join('|', ['a', 'b', 'c']), 'a|b|c') self.assertEqual(join('|', ['a', '', 'c']), 'a||c') self.assertEqual(join('', ['a', 'b', 'c']), 'abc') @@ -1152,10 +1254,10 @@ def test_join(self): self.assertRaises(SystemError, join, '|', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_count(self): """Test PyUnicode_Count()""" - from _testcapi import unicode_count + from _testlimitedcapi import unicode_count for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": for i, ch in enumerate(str): @@ -1183,10 +1285,10 @@ def test_count(self): # CRASHES unicode_count(str, NULL, 0, len(str)) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_tailmatch(self): """Test PyUnicode_Tailmatch()""" - from _testcapi import unicode_tailmatch as tailmatch + from _testlimitedcapi import unicode_tailmatch as tailmatch str = 'ababahalamaha' self.assertEqual(tailmatch(str, 'aba', 0, len(str), -1), 1) @@ -1218,10 +1320,10 @@ def test_tailmatch(self): # CRASHES tailmatch(str, NULL, 0, len(str), -1) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_find(self): """Test PyUnicode_Find()""" - from _testcapi import unicode_find as find + from _testlimitedcapi import unicode_find as find for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": for i, ch in enumerate(str): @@ -1259,10 +1361,10 @@ def test_find(self): # CRASHES find(str, NULL, 0, len(str), 1) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_findchar(self): """Test PyUnicode_FindChar()""" - from _testcapi import unicode_findchar + from _testlimitedcapi import unicode_findchar for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": for i, ch in enumerate(str): @@ -1295,10 +1397,10 @@ def test_findchar(self): # CRASHES unicode_findchar(NULL, ord('!'), 0, len(str), 1), 1) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_replace(self): """Test PyUnicode_Replace()""" - from _testcapi import unicode_replace as replace + from _testlimitedcapi import unicode_replace as replace str = 'abracadabra' self.assertEqual(replace(str, 'a', '='), '=br=c=d=br=') @@ -1326,10 +1428,10 @@ def test_replace(self): # CRASHES replace(NULL, 'a', '=') @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_compare(self): """Test PyUnicode_Compare()""" - from _testcapi import unicode_compare as compare + from _testlimitedcapi import unicode_compare as compare self.assertEqual(compare('abc', 'abc'), 0) self.assertEqual(compare('abc', 'def'), -1) @@ -1348,10 +1450,10 @@ def test_compare(self): # CRASHES compare('abc', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_comparewithasciistring(self): """Test PyUnicode_CompareWithASCIIString()""" - from _testcapi import unicode_comparewithasciistring as comparewithasciistring + from _testlimitedcapi import unicode_comparewithasciistring as comparewithasciistring self.assertEqual(comparewithasciistring('abc', b'abc'), 0) self.assertEqual(comparewithasciistring('abc', b'def'), -1) @@ -1365,11 +1467,11 @@ def test_comparewithasciistring(self): # CRASHES comparewithasciistring(NULL, b'abc') @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_equaltoutf8(self): # Test PyUnicode_EqualToUTF8() - from _testcapi import unicode_equaltoutf8 as equaltoutf8 - from _testcapi import unicode_asutf8andsize as asutf8andsize + from _testlimitedcapi import unicode_equaltoutf8 as equaltoutf8 + from _testlimitedcapi import unicode_asutf8andsize as asutf8andsize strings = [ 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', @@ -1411,11 +1513,11 @@ def test_equaltoutf8(self): '\ud801'.encode("utf8", "surrogatepass")), 0) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_equaltoutf8andsize(self): # Test PyUnicode_EqualToUTF8AndSize() - from _testcapi import unicode_equaltoutf8andsize as equaltoutf8andsize - from _testcapi import unicode_asutf8andsize as asutf8andsize + from _testlimitedcapi import unicode_equaltoutf8andsize as equaltoutf8andsize + from _testlimitedcapi import unicode_asutf8andsize as asutf8andsize strings = [ 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', @@ -1480,10 +1582,10 @@ def check_not_equal_encoding(text, encoding): # CRASHES equaltoutf8andsize('abc', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_richcompare(self): """Test PyUnicode_RichCompare()""" - from _testcapi import unicode_richcompare as richcompare + from _testlimitedcapi import unicode_richcompare as richcompare LT, LE, EQ, NE, GT, GE = range(6) strings = ('abc', 'абв', '\U0001f600', 'abc\0') @@ -1508,10 +1610,10 @@ def test_richcompare(self): # CRASHES richcompare('abc', NULL, op) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_format(self): """Test PyUnicode_Format()""" - from _testcapi import unicode_format as format + from _testlimitedcapi import unicode_format as format self.assertEqual(format('x=%d!', 42), 'x=42!') self.assertEqual(format('x=%d!', (42,)), 'x=42!') @@ -1521,10 +1623,10 @@ def test_format(self): self.assertRaises(SystemError, format, NULL, 42) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_contains(self): """Test PyUnicode_Contains()""" - from _testcapi import unicode_contains as contains + from _testlimitedcapi import unicode_contains as contains self.assertEqual(contains('abcd', ''), 1) self.assertEqual(contains('abcd', 'b'), 1) @@ -1543,10 +1645,10 @@ def test_contains(self): # CRASHES contains('abcd', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_isidentifier(self): """Test PyUnicode_IsIdentifier()""" - from _testcapi import unicode_isidentifier as isidentifier + from _testlimitedcapi import unicode_isidentifier as isidentifier self.assertEqual(isidentifier("a"), 1) self.assertEqual(isidentifier("b0"), 1) @@ -1638,5 +1740,220 @@ def test_pep393_utf8_caching_bug(self): self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) +class PyUnicodeWriterTest(unittest.TestCase): + def create_writer(self, size): + return _testcapi.PyUnicodeWriter(size) + + def test_basic(self): + writer = self.create_writer(100) + + # test PyUnicodeWriter_WriteUTF8() + writer.write_utf8(b'var', -1) + + # test PyUnicodeWriter_WriteChar() + writer.write_char('=') + + # test PyUnicodeWriter_WriteSubstring() + writer.write_substring("[long]", 1, 5) + + # test PyUnicodeWriter_WriteStr() + writer.write_str(" value ") + + # test PyUnicodeWriter_WriteRepr() + writer.write_repr("repr") + + self.assertEqual(writer.finish(), + "var=long value 'repr'") + + def test_utf8(self): + writer = self.create_writer(0) + writer.write_utf8(b"ascii", -1) + writer.write_char('-') + writer.write_utf8(b"latin1=\xC3\xA9", -1) + writer.write_char('-') + writer.write_utf8(b"euro=\xE2\x82\xAC", -1) + writer.write_char('.') + self.assertEqual(writer.finish(), + "ascii-latin1=\xE9-euro=\u20AC.") + + def test_invalid_utf8(self): + writer = self.create_writer(0) + with self.assertRaises(UnicodeDecodeError): + writer.write_utf8(b"invalid=\xFF", -1) + + def test_recover_utf8_error(self): + # test recovering from PyUnicodeWriter_WriteUTF8() error + writer = self.create_writer(0) + writer.write_utf8(b"value=", -1) + + # write fails with an invalid string + with self.assertRaises(UnicodeDecodeError): + writer.write_utf8(b"invalid\xFF", -1) + + # retry write with a valid string + writer.write_utf8(b"valid", -1) + + self.assertEqual(writer.finish(), + "value=valid") + + def test_decode_utf8(self): + # test PyUnicodeWriter_DecodeUTF8Stateful() + writer = self.create_writer(0) + writer.decodeutf8stateful(b"ign\xFFore", -1, b"ignore") + writer.write_char('-') + writer.decodeutf8stateful(b"replace\xFF", -1, b"replace") + writer.write_char('-') + + # incomplete trailing UTF-8 sequence + writer.decodeutf8stateful(b"incomplete\xC3", -1, b"replace") + + self.assertEqual(writer.finish(), + "ignore-replace\uFFFD-incomplete\uFFFD") + + def test_decode_utf8_consumed(self): + # test PyUnicodeWriter_DecodeUTF8Stateful() with consumed + writer = self.create_writer(0) + + # valid string + consumed = writer.decodeutf8stateful(b"text", -1, b"strict", True) + self.assertEqual(consumed, 4) + writer.write_char('-') + + # non-ASCII + consumed = writer.decodeutf8stateful(b"\xC3\xA9-\xE2\x82\xAC", 6, b"strict", True) + self.assertEqual(consumed, 6) + writer.write_char('-') + + # invalid UTF-8 (consumed is 0 on error) + with self.assertRaises(UnicodeDecodeError): + writer.decodeutf8stateful(b"invalid\xFF", -1, b"strict", True) + + # ignore error handler + consumed = writer.decodeutf8stateful(b"more\xFF", -1, b"ignore", True) + self.assertEqual(consumed, 5) + writer.write_char('-') + + # incomplete trailing UTF-8 sequence + consumed = writer.decodeutf8stateful(b"incomplete\xC3", -1, b"ignore", True) + self.assertEqual(consumed, 10) + + self.assertEqual(writer.finish(), "text-\xE9-\u20AC-more-incomplete") + + def test_widechar(self): + writer = self.create_writer(0) + writer.write_widechar("latin1=\xE9") + writer.write_widechar("-") + writer.write_widechar("euro=\u20AC") + writer.write_char("-") + writer.write_widechar("max=\U0010ffff") + writer.write_char('.') + self.assertEqual(writer.finish(), + "latin1=\xE9-euro=\u20AC-max=\U0010ffff.") + + def test_ucs4(self): + writer = self.create_writer(0) + writer.write_ucs4("ascii IGNORED", 5) + writer.write_char("-") + writer.write_ucs4("latin1=\xe9", 8) + writer.write_char("-") + writer.write_ucs4("euro=\u20ac", 6) + writer.write_char("-") + writer.write_ucs4("max=\U0010ffff", 5) + writer.write_char(".") + self.assertEqual(writer.finish(), + "ascii-latin1=\xE9-euro=\u20AC-max=\U0010ffff.") + + # Test some special characters + writer = self.create_writer(0) + # Lone surrogate character + writer.write_ucs4("lone\uDC80", 5) + writer.write_char("-") + # Surrogate pair + writer.write_ucs4("pair\uDBFF\uDFFF", 5) + writer.write_char("-") + writer.write_ucs4("null[\0]", 7) + self.assertEqual(writer.finish(), + "lone\udc80-pair\udbff-null[\0]") + + # invalid size + writer = self.create_writer(0) + with self.assertRaises(ValueError): + writer.write_ucs4("text", -1) + + def test_substring_empty(self): + writer = self.create_writer(0) + writer.write_substring("abc", 1, 1) + self.assertEqual(writer.finish(), '') + + +@unittest.skipIf(ctypes is None, 'need ctypes') +class PyUnicodeWriterFormatTest(unittest.TestCase): + def create_writer(self, size): + return _testcapi.PyUnicodeWriter(size) + + def writer_format(self, writer, *args): + from ctypes import c_char_p, pythonapi, c_int, c_void_p + _PyUnicodeWriter_Format = getattr(pythonapi, "PyUnicodeWriter_Format") + _PyUnicodeWriter_Format.argtypes = (c_void_p, c_char_p,) + _PyUnicodeWriter_Format.restype = c_int + + if _PyUnicodeWriter_Format(writer.get_pointer(), *args) < 0: + raise ValueError("PyUnicodeWriter_Format failed") + + def test_format(self): + from ctypes import c_int + writer = self.create_writer(0) + self.writer_format(writer, b'%s %i', b'abc', c_int(123)) + writer.write_char('.') + self.assertEqual(writer.finish(), 'abc 123.') + + def test_recover_error(self): + # test recovering from PyUnicodeWriter_Format() error + writer = self.create_writer(0) + self.writer_format(writer, b"%s ", b"Hello") + + # PyUnicodeWriter_Format() fails with an invalid format string + with self.assertRaises(ValueError): + self.writer_format(writer, b"%s\xff", b"World") + + # Retry PyUnicodeWriter_Format() with a valid format string + self.writer_format(writer, b"%s.", b"World") + + self.assertEqual(writer.finish(), 'Hello World.') + + def test_unicode_equal(self): + unicode_equal = _testlimitedcapi.unicode_equal + + def copy(text): + return text.encode().decode() + + self.assertTrue(unicode_equal("", "")) + self.assertTrue(unicode_equal("abc", "abc")) + self.assertTrue(unicode_equal("abc", copy("abc"))) + self.assertTrue(unicode_equal("\u20ac", copy("\u20ac"))) + self.assertTrue(unicode_equal("\U0010ffff", copy("\U0010ffff"))) + + self.assertFalse(unicode_equal("abc", "abcd")) + self.assertFalse(unicode_equal("\u20ac", "\u20ad")) + self.assertFalse(unicode_equal("\U0010ffff", "\U0010fffe")) + + # str subclass + self.assertTrue(unicode_equal("abc", Str("abc"))) + self.assertTrue(unicode_equal(Str("abc"), "abc")) + self.assertFalse(unicode_equal("abc", Str("abcd"))) + self.assertFalse(unicode_equal(Str("abc"), "abcd")) + + # invalid type + for invalid_type in (b'bytes', 123, ("tuple",)): + with self.subTest(invalid_type=invalid_type): + with self.assertRaises(TypeError): + unicode_equal("abc", invalid_type) + with self.assertRaises(TypeError): + unicode_equal(invalid_type, "abc") + + # CRASHES unicode_equal("abc", NULL) + # CRASHES unicode_equal(NULL, "abc") + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index ae062b1bda26b7..8644479d83d5ed 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -1,7 +1,10 @@ import unittest +import contextvars from contextlib import contextmanager, ExitStack -from test.support import catch_unraisable_exception, import_helper +from test.support import ( + catch_unraisable_exception, import_helper, + gc_collect) # Skip this test if the _testcapi module isn't available. @@ -94,6 +97,23 @@ def test_dealloc(self): del d self.assert_events(["dealloc"]) + def test_object_dict(self): + class MyObj: pass + o = MyObj() + + with self.watcher() as wid: + self.watch(wid, o.__dict__) + o.foo = "bar" + o.foo = "baz" + del o.foo + self.assert_events(["new:foo:bar", "mod:foo:baz", "del:foo"]) + + with self.watcher() as wid: + self.watch(wid, o.__dict__) + for _ in range(100): + o.foo = "bar" + self.assert_events(["new:foo:bar"] + ["mod:foo:bar"] * 99) + def test_unwatch(self): d = {} with self.watcher() as wid: @@ -280,8 +300,10 @@ class C: pass self.watch(wid, C) with catch_unraisable_exception() as cm: C.foo = "bar" - self.assertEqual(cm.unraisable.err_msg, - f"Exception ignored in type watcher callback #0 for {C!r}") + self.assertEqual( + cm.unraisable.err_msg, + f"Exception ignored in type watcher callback #1 for {C!r}", + ) self.assertIs(cm.unraisable.object, None) self.assertEqual(str(cm.unraisable.exc_value), "boom!") self.assert_events([]) @@ -372,6 +394,7 @@ def code_watcher(self, which_watcher): def assert_event_counts(self, exp_created_0, exp_destroyed_0, exp_created_1, exp_destroyed_1): + gc_collect() # code objects are collected by GC in free-threaded build self.assertEqual( exp_created_0, _testcapi.get_code_watcher_num_created_events(0)) self.assertEqual( @@ -432,6 +455,7 @@ def test_dealloc_error(self): with self.code_watcher(2): with catch_unraisable_exception() as cm: del co + gc_collect() self.assertEqual(str(cm.unraisable.exc_value), "boom!") @@ -563,5 +587,92 @@ def test_allocate_too_many_watchers(self): _testcapi.allocate_too_many_func_watchers() +class TestContextObjectWatchers(unittest.TestCase): + @contextmanager + def context_watcher(self, which_watcher): + wid = _testcapi.add_context_watcher(which_watcher) + try: + switches = _testcapi.get_context_switches(which_watcher) + except ValueError: + switches = None + try: + yield switches + finally: + _testcapi.clear_context_watcher(wid) + + def assert_event_counts(self, want_0, want_1): + self.assertEqual(len(_testcapi.get_context_switches(0)), want_0) + self.assertEqual(len(_testcapi.get_context_switches(1)), want_1) + + def test_context_object_events_dispatched(self): + # verify that all counts are zero before any watchers are registered + self.assert_event_counts(0, 0) + + # verify that all counts remain zero when a context object is + # entered and exited with no watchers registered + ctx = contextvars.copy_context() + ctx.run(self.assert_event_counts, 0, 0) + self.assert_event_counts(0, 0) + + # verify counts are as expected when first watcher is registered + with self.context_watcher(0): + self.assert_event_counts(0, 0) + ctx.run(self.assert_event_counts, 1, 0) + self.assert_event_counts(2, 0) + + # again with second watcher registered + with self.context_watcher(1): + self.assert_event_counts(2, 0) + ctx.run(self.assert_event_counts, 3, 1) + self.assert_event_counts(4, 2) + + # verify counts are reset and don't change after both watchers are cleared + ctx.run(self.assert_event_counts, 0, 0) + self.assert_event_counts(0, 0) + + def test_callback_error(self): + ctx_outer = contextvars.copy_context() + ctx_inner = contextvars.copy_context() + unraisables = [] + + def _in_outer(): + with self.context_watcher(2): + with catch_unraisable_exception() as cm: + ctx_inner.run(lambda: unraisables.append(cm.unraisable)) + unraisables.append(cm.unraisable) + + try: + ctx_outer.run(_in_outer) + self.assertEqual([x.err_msg for x in unraisables], + ["Exception ignored in Py_CONTEXT_SWITCHED " + f"watcher callback for {ctx!r}" + for ctx in [ctx_inner, ctx_outer]]) + self.assertEqual([str(x.exc_value) for x in unraisables], + ["boom!", "boom!"]) + finally: + # Break reference cycle + unraisables = None + + def test_clear_out_of_range_watcher_id(self): + with self.assertRaisesRegex(ValueError, r"Invalid context watcher ID -1"): + _testcapi.clear_context_watcher(-1) + with self.assertRaisesRegex(ValueError, r"Invalid context watcher ID 8"): + _testcapi.clear_context_watcher(8) # CONTEXT_MAX_WATCHERS = 8 + + def test_clear_unassigned_watcher_id(self): + with self.assertRaisesRegex(ValueError, r"No context watcher set for ID 1"): + _testcapi.clear_context_watcher(1) + + def test_allocate_too_many_watchers(self): + with self.assertRaisesRegex(RuntimeError, r"no more context watcher IDs available"): + _testcapi.allocate_too_many_context_watchers() + + def test_exit_base_context(self): + ctx = contextvars.Context() + _testcapi.clear_context_stack() + with self.context_watcher(0) as switches: + ctx.run(lambda: None) + self.assertEqual(switches, [ctx, None]) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py new file mode 100644 index 00000000000000..54859f9ff7622d --- /dev/null +++ b/Lib/test/test_cext/__init__.py @@ -0,0 +1,111 @@ +# gh-116869: Build a basic C test extension to check that the Python C API +# does not emit C compiler warnings. +# +# The Python C API must be compatible with building +# with the -Werror=declaration-after-statement compiler flag. + +import os.path +import shlex +import shutil +import subprocess +import unittest +from test import support + + +SOURCE = os.path.join(os.path.dirname(__file__), 'extension.c') +SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') + + +# With MSVC on a debug build, the linker fails with: cannot open file +# 'python311.lib', it should look 'python311_d.lib'. +@unittest.skipIf(support.MS_WINDOWS and support.Py_DEBUG, + 'test fails on Windows debug build') +# Building and running an extension in clang sanitizing mode is not +# straightforward +@support.skip_if_sanitizer('test does not work with analyzing builds', + address=True, memory=True, ub=True, thread=True) +# the test uses venv+pip: skip if it's not available +@support.requires_venv_with_pip() +@support.requires_subprocess() +@support.requires_resource('cpu') +class TestExt(unittest.TestCase): + # Default build with no options + def test_build(self): + self.check_build('_test_cext') + + def test_build_c11(self): + self.check_build('_test_c11_cext', std='c11') + + @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99") + def test_build_c99(self): + self.check_build('_test_c99_cext', std='c99') + + @support.requires_gil_enabled('incompatible with Free Threading') + def test_build_limited(self): + self.check_build('_test_limited_cext', limited=True) + + @support.requires_gil_enabled('broken for now with Free Threading') + def test_build_limited_c11(self): + self.check_build('_test_limited_c11_cext', limited=True, std='c11') + + def check_build(self, extension_name, std=None, limited=False): + venv_dir = 'env' + with support.setup_venv_with_pip_setuptools_wheel(venv_dir) as python_exe: + self._check_build(extension_name, python_exe, + std=std, limited=limited) + + def _check_build(self, extension_name, python_exe, std, limited): + pkg_dir = 'pkg' + os.mkdir(pkg_dir) + shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) + shutil.copy(SOURCE, os.path.join(pkg_dir, os.path.basename(SOURCE))) + + def run_cmd(operation, cmd): + env = os.environ.copy() + if std: + env['CPYTHON_TEST_STD'] = std + if limited: + env['CPYTHON_TEST_LIMITED'] = '1' + env['CPYTHON_TEST_EXT_NAME'] = extension_name + if support.verbose: + print('Run:', ' '.join(map(shlex.quote, cmd))) + subprocess.run(cmd, check=True, env=env) + else: + proc = subprocess.run(cmd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True) + if proc.returncode: + print('Run:', ' '.join(map(shlex.quote, cmd))) + print(proc.stdout, end='') + self.fail( + f"{operation} failed with exit code {proc.returncode}") + + # Build and install the C extension + cmd = [python_exe, '-X', 'dev', + '-m', 'pip', 'install', '--no-build-isolation', + os.path.abspath(pkg_dir)] + if support.verbose: + cmd.append('-v') + run_cmd('Install', cmd) + + # Do a reference run. Until we test that running python + # doesn't leak references (gh-94755), run it so one can manually check + # -X showrefcount results against this baseline. + cmd = [python_exe, + '-X', 'dev', + '-X', 'showrefcount', + '-c', 'pass'] + run_cmd('Reference run', cmd) + + # Import the C extension + cmd = [python_exe, + '-X', 'dev', + '-X', 'showrefcount', + '-c', f"import {extension_name}"] + run_cmd('Import', cmd) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c new file mode 100644 index 00000000000000..b76abe1d74c628 --- /dev/null +++ b/Lib/test/test_cext/extension.c @@ -0,0 +1,89 @@ +// gh-116869: Basic C test extension to check that the Python C API +// does not emit C compiler warnings. + +// Always enable assertions +#undef NDEBUG + +#include "Python.h" + +#ifndef MODULE_NAME +# error "MODULE_NAME macro must be defined" +#endif + +#define _STR(NAME) #NAME +#define STR(NAME) _STR(NAME) + +PyDoc_STRVAR(_testcext_add_doc, +"add(x, y)\n" +"\n" +"Return the sum of two integers: x + y."); + +static PyObject * +_testcext_add(PyObject *Py_UNUSED(module), PyObject *args) +{ + long i, j, res; + if (!PyArg_ParseTuple(args, "ll:foo", &i, &j)) { + return NULL; + } + res = i + j; + return PyLong_FromLong(res); +} + + +static PyMethodDef _testcext_methods[] = { + {"add", _testcext_add, METH_VARARGS, _testcext_add_doc}, + {NULL, NULL, 0, NULL} // sentinel +}; + + +static int +_testcext_exec( +#ifdef __STDC_VERSION__ + PyObject *module +#else + PyObject *Py_UNUSED(module) +#endif + ) +{ +#ifdef __STDC_VERSION__ + if (PyModule_AddIntMacro(module, __STDC_VERSION__) < 0) { + return -1; + } +#endif + + // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR() + Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int)); + assert(Py_BUILD_ASSERT_EXPR(sizeof(int) == sizeof(unsigned int)) == 0); + + return 0; +} + +static PyModuleDef_Slot _testcext_slots[] = { + {Py_mod_exec, (void*)_testcext_exec}, + {0, NULL} +}; + + +PyDoc_STRVAR(_testcext_doc, "C test extension."); + +static struct PyModuleDef _testcext_module = { + PyModuleDef_HEAD_INIT, // m_base + STR(MODULE_NAME), // m_name + _testcext_doc, // m_doc + 0, // m_size + _testcext_methods, // m_methods + _testcext_slots, // m_slots + NULL, // m_traverse + NULL, // m_clear + NULL, // m_free +}; + + +#define _FUNC_NAME(NAME) PyInit_ ## NAME +#define FUNC_NAME(NAME) _FUNC_NAME(NAME) + +PyMODINIT_FUNC +FUNC_NAME(MODULE_NAME)(void) +{ + return PyModuleDef_Init(&_testcext_module); +} diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py new file mode 100644 index 00000000000000..e97749b45ea6f3 --- /dev/null +++ b/Lib/test/test_cext/setup.py @@ -0,0 +1,109 @@ +# gh-91321: Build a basic C test extension to check that the Python C API is +# compatible with C and does not emit C compiler warnings. +import os +import platform +import shlex +import sys +import sysconfig +from test import support + +from setuptools import setup, Extension + + +SOURCE = 'extension.c' + +if not support.MS_WINDOWS: + # C compiler flags for GCC and clang + CFLAGS = [ + # The purpose of test_cext extension is to check that building a C + # extension using the Python C API does not emit C compiler warnings. + '-Werror', + + # gh-120593: Check the 'const' qualifier + '-Wcast-qual', + ] + if not support.Py_GIL_DISABLED: + CFLAGS.append( + # gh-116869: The Python C API must be compatible with building + # with the -Werror=declaration-after-statement compiler flag. + '-Werror=declaration-after-statement', + ) +else: + # MSVC compiler flags + CFLAGS = [ + # Display warnings level 1 to 4 + '/W4', + # Treat all compiler warnings as compiler errors + '/WX', + ] + + +def main(): + std = os.environ.get("CPYTHON_TEST_STD", "") + module_name = os.environ["CPYTHON_TEST_EXT_NAME"] + limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", "")) + + cflags = list(CFLAGS) + cflags.append(f'-DMODULE_NAME={module_name}') + + # Add -std=STD or /std:STD (MSVC) compiler flag + if std: + if support.MS_WINDOWS: + cflags.append(f'/std:{std}') + else: + cflags.append(f'-std={std}') + + # Remove existing -std or /std options from CC command line. + # Python adds -std=c11 option. + cmd = (sysconfig.get_config_var('CC') or '') + if cmd is not None: + if support.MS_WINDOWS: + std_prefix = '/std' + else: + std_prefix = '-std' + cmd = shlex.split(cmd) + cmd = [arg for arg in cmd if not arg.startswith(std_prefix)] + cmd = shlex.join(cmd) + # CC env var overrides sysconfig CC variable in setuptools + os.environ['CC'] = cmd + + # Define Py_LIMITED_API macro + if limited: + version = sys.hexversion + cflags.append(f'-DPy_LIMITED_API={version:#x}') + + # On Windows, add PCbuild\amd64\ to include and library directories + include_dirs = [] + library_dirs = [] + if support.MS_WINDOWS: + srcdir = sysconfig.get_config_var('srcdir') + machine = platform.uname().machine + pcbuild = os.path.join(srcdir, 'PCbuild', machine) + if os.path.exists(pcbuild): + # pyconfig.h is generated in PCbuild\amd64\ + include_dirs.append(pcbuild) + # python313.lib is generated in PCbuild\amd64\ + library_dirs.append(pcbuild) + print(f"Add PCbuild directory: {pcbuild}") + + # Display information to help debugging + for env_name in ('CC', 'CFLAGS'): + if env_name in os.environ: + print(f"{env_name} env var: {os.environ[env_name]!r}") + else: + print(f"{env_name} env var: ") + print(f"extra_compile_args: {cflags!r}") + + ext = Extension( + module_name, + sources=[SOURCE], + extra_compile_args=cflags, + include_dirs=include_dirs, + library_dirs=library_dirs) + setup(name=f'internal_{module_name}', + version='0.0', + ext_modules=[ext]) + + +if __name__ == "__main__": + main() diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index d59271435e9eb0..e20e59944e9ce9 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -1,7 +1,7 @@ "Test the functionality of Python classes implementing operators." import unittest - +from test.support import cpython_only, import_helper, script_helper, skip_emscripten_stack_overflow testmeths = [ @@ -448,15 +448,15 @@ def __delattr__(self, *args): def testHasAttrString(self): import sys from test.support import import_helper - _testcapi = import_helper.import_module('_testcapi') + _testlimitedcapi = import_helper.import_module('_testlimitedcapi') class A: def __init__(self): self.attr = 1 a = A() - self.assertEqual(_testcapi.object_hasattrstring(a, b"attr"), 1) - self.assertEqual(_testcapi.object_hasattrstring(a, b"noattr"), 0) + self.assertEqual(_testlimitedcapi.object_hasattrstring(a, b"attr"), 1) + self.assertEqual(_testlimitedcapi.object_hasattrstring(a, b"noattr"), 0) self.assertIsNone(sys.exception()) def testDel(self): @@ -503,7 +503,58 @@ def __eq__(self, other): return 1 self.assertRaises(TypeError, hash, C2()) + def testPredefinedAttrs(self): + o = object() + + class Custom: + pass + c = Custom() + + methods = ( + '__class__', '__delattr__', '__dir__', '__eq__', '__format__', + '__ge__', '__getattribute__', '__getstate__', '__gt__', '__hash__', + '__init__', '__init_subclass__', '__le__', '__lt__', '__ne__', + '__new__', '__reduce__', '__reduce_ex__', '__repr__', + '__setattr__', '__sizeof__', '__str__', '__subclasshook__' + ) + for name in methods: + with self.subTest(name): + self.assertTrue(callable(getattr(object, name, None))) + self.assertTrue(callable(getattr(o, name, None))) + self.assertTrue(callable(getattr(Custom, name, None))) + self.assertTrue(callable(getattr(c, name, None))) + + not_defined = [ + '__abs__', '__aenter__', '__aexit__', '__aiter__', '__anext__', + '__await__', '__bool__', '__bytes__', '__ceil__', + '__complex__', '__contains__', '__del__', '__delete__', + '__delitem__', '__divmod__', '__enter__', '__exit__', + '__float__', '__floor__', '__get__', '__getattr__', '__getitem__', + '__index__', '__int__', '__invert__', '__iter__', '__len__', + '__length_hint__', '__missing__', '__neg__', '__next__', + '__objclass__', '__pos__', '__rdivmod__', '__reversed__', + '__round__', '__set__', '__setitem__', '__trunc__' + ] + augment = ( + 'add', 'and', 'floordiv', 'lshift', 'matmul', 'mod', 'mul', 'pow', + 'rshift', 'sub', 'truediv', 'xor' + ) + not_defined.extend(map("__{}__".format, augment)) + not_defined.extend(map("__r{}__".format, augment)) + not_defined.extend(map("__i{}__".format, augment)) + for name in not_defined: + with self.subTest(name): + self.assertFalse(hasattr(object, name)) + self.assertFalse(hasattr(o, name)) + self.assertFalse(hasattr(Custom, name)) + self.assertFalse(hasattr(c, name)) + + # __call__() is defined on the metaclass but not the class + self.assertFalse(hasattr(o, "__call__")) + self.assertFalse(hasattr(c, "__call__")) + + @skip_emscripten_stack_overflow() def testSFBug532646(self): # Test for SF bug 532646 @@ -788,5 +839,181 @@ def __init__(self, obj): Type(i) self.assertEqual(calls, 100) + def test_specialization_class_call_doesnt_crash(self): + # gh-123185 + + class Foo: + def __init__(self, arg): + pass + + for _ in range(8): + try: + Foo() + except: + pass + + +from _testinternalcapi import has_inline_values + +Py_TPFLAGS_MANAGED_DICT = (1 << 2) + +class Plain: + pass + + +class WithAttrs: + + def __init__(self): + self.a = 1 + self.b = 2 + self.c = 3 + self.d = 4 + + +class TestInlineValues(unittest.TestCase): + + def test_flags(self): + self.assertEqual(Plain.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + self.assertEqual(WithAttrs.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + + def test_has_inline_values(self): + c = Plain() + self.assertTrue(has_inline_values(c)) + del c.__dict__ + self.assertFalse(has_inline_values(c)) + + def test_instances(self): + self.assertTrue(has_inline_values(Plain())) + self.assertTrue(has_inline_values(WithAttrs())) + + def test_inspect_dict(self): + for cls in (Plain, WithAttrs): + c = cls() + c.__dict__ + self.assertTrue(has_inline_values(c)) + + def test_update_dict(self): + d = { "e": 5, "f": 6 } + for cls in (Plain, WithAttrs): + c = cls() + c.__dict__.update(d) + self.assertTrue(has_inline_values(c)) + + @staticmethod + def set_100(obj): + for i in range(100): + setattr(obj, f"a{i}", i) + + def check_100(self, obj): + for i in range(100): + self.assertEqual(getattr(obj, f"a{i}"), i) + + def test_many_attributes(self): + class C: pass + c = C() + self.assertTrue(has_inline_values(c)) + self.set_100(c) + self.assertFalse(has_inline_values(c)) + self.check_100(c) + c = C() + self.assertTrue(has_inline_values(c)) + + def test_many_attributes_with_dict(self): + class C: pass + c = C() + d = c.__dict__ + self.assertTrue(has_inline_values(c)) + self.set_100(c) + self.assertFalse(has_inline_values(c)) + self.check_100(c) + + def test_bug_117750(self): + "Aborted on 3.13a6" + class C: + def __init__(self): + self.__dict__.clear() + + obj = C() + self.assertEqual(obj.__dict__, {}) + obj.foo = None # Aborted here + self.assertEqual(obj.__dict__, {"foo":None}) + + def test_store_attr_deleted_dict(self): + class Foo: + pass + + f = Foo() + del f.__dict__ + f.a = 3 + self.assertEqual(f.a, 3) + + def test_rematerialize_object_dict(self): + # gh-121860: rematerializing an object's managed dictionary after it + # had been deleted caused a crash. + class Foo: pass + f = Foo() + f.__dict__["attr"] = 1 + del f.__dict__ + + # Using a str subclass is a way to trigger the re-materialization + class StrSubclass(str): pass + self.assertFalse(hasattr(f, StrSubclass("attr"))) + + # Changing the __class__ also triggers the re-materialization + class Bar: pass + f.__class__ = Bar + self.assertIsInstance(f, Bar) + self.assertEqual(f.__dict__, {}) + + def test_store_attr_type_cache(self): + """Verifies that the type cache doesn't provide a value which is + inconsistent from the dict.""" + class X: + def __del__(inner_self): + v = C.a + self.assertEqual(v, C.__dict__['a']) + + class C: + a = X() + + # prime the cache + C.a + C.a + + # destructor shouldn't be able to see inconsistent state + C.a = X() + C.a = X() + + @cpython_only + def test_detach_materialized_dict_no_memory(self): + # Skip test if _testcapi is not available: + import_helper.import_module('_testcapi') + + code = """if 1: + import test.support + import _testcapi + + class A: + def __init__(self): + self.a = 1 + self.b = 2 + a = A() + d = a.__dict__ + with test.support.catch_unraisable_exception() as ex: + _testcapi.set_nomemory(0, 1) + del a + assert ex.unraisable.exc_type is MemoryError + try: + d["a"] + except KeyError: + pass + else: + assert False, "KeyError not raised" + """ + rc, out, err = script_helper.assert_python_ok("-c", code) + self.assertEqual(rc, 0) + self.assertFalse(out, msg=out.decode('utf-8')) + self.assertFalse(err, msg=err.decode('utf-8')) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 02a293e04b2182..b45b9ee89ee3de 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -5,7 +5,7 @@ from functools import partial from test import support, test_tools from test.support import os_helper -from test.support.os_helper import TESTFN, unlink +from test.support.os_helper import TESTFN, unlink, rmtree from textwrap import dedent from unittest import TestCase import inspect @@ -17,14 +17,35 @@ test_tools.skip_if_missing('clinic') with test_tools.imports_under_tool('clinic'): import libclinic + from libclinic import ClinicError, unspecified, NULL, fail + from libclinic.converters import int_converter, str_converter, self_converter + from libclinic.function import ( + Module, Class, Function, FunctionKind, Parameter, + permute_optional_groups, permute_right_option_groups, + permute_left_option_groups) import clinic - from clinic import DSLParser - - -def _make_clinic(*, filename='clinic_tests'): - clang = clinic.CLanguage(filename) - c = clinic.Clinic(clang, filename=filename, limited_capi=False) - c.block_parser = clinic.BlockParser('', clang) + from libclinic.clanguage import CLanguage + from libclinic.converter import converters, legacy_converters + from libclinic.return_converters import return_converters, int_return_converter + from libclinic.block_parser import Block, BlockParser + from libclinic.codegen import BlockPrinter, Destination + from libclinic.dsl_parser import DSLParser + from libclinic.cli import parse_file, Clinic + + +def repeat_fn(*functions): + def wrapper(test): + def wrapped(self): + for fn in functions: + with self.subTest(fn=fn): + test(self, fn) + return wrapped + return wrapper + +def _make_clinic(*, filename='clinic_tests', limited_capi=False): + clang = CLanguage(filename) + c = Clinic(clang, filename=filename, limited_capi=limited_capi) + c.block_parser = BlockParser('', clang) return c @@ -43,7 +64,7 @@ def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None, if strip: code = code.strip() errmsg = re.escape(errmsg) - with tc.assertRaisesRegex(clinic.ClinicError, errmsg) as cm: + with tc.assertRaisesRegex(ClinicError, errmsg) as cm: parser(code) if filename is not None: tc.assertEqual(cm.exception.filename, filename) @@ -52,6 +73,20 @@ def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None, return cm.exception +def restore_dict(converters, old_converters): + converters.clear() + converters.update(old_converters) + + +def save_restore_converters(testcase): + testcase.addCleanup(restore_dict, converters, + converters.copy()) + testcase.addCleanup(restore_dict, legacy_converters, + legacy_converters.copy()) + testcase.addCleanup(restore_dict, return_converters, + return_converters.copy()) + + class ClinicWholeFileTest(TestCase): maxDiff = None @@ -60,6 +95,7 @@ def expect_failure(self, raw, errmsg, *, filename=None, lineno=None): filename=filename, lineno=lineno) def setUp(self): + save_restore_converters(self) self.clinic = _make_clinic(filename="test.c") def test_eol(self): @@ -121,11 +157,11 @@ def test_whitespace_before_stop_line(self): self.expect_failure(raw, err, filename="test.c", lineno=2) def test_parse_with_body_prefix(self): - clang = clinic.CLanguage(None) + clang = CLanguage(None) clang.body_prefix = "//" clang.start_line = "//[{dsl_name} start]" clang.stop_line = "//[{dsl_name} stop]" - cl = clinic.Clinic(clang, filename="test.c", limited_capi=False) + cl = Clinic(clang, filename="test.c", limited_capi=False) raw = dedent(""" //[clinic start] //module test @@ -295,20 +331,34 @@ def __init__(self): """ self.expect_failure(block, err, lineno=8) - def test_multiple_star_in_args(self): + def test_star_after_vararg(self): err = "'my_test_func' uses '*' more than once." block = """ /*[clinic input] my_test_func pos_arg: object - *args: object + *args: tuple * kw_arg: object [clinic start generated code]*/ """ self.expect_failure(block, err, lineno=6) + def test_vararg_after_star(self): + err = "'my_test_func' uses '*' more than once." + block = """ + /*[clinic input] + my_test_func + + pos_arg: object + * + *args: tuple + kw_arg: object + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=6) + def test_module_already_got_one(self): err = "Already defined module 'm'!" block = """ @@ -635,14 +685,79 @@ class C "void *" "" err = "Illegal C basename: '.illegal.'" self.expect_failure(block, err, lineno=7) + def test_cloned_forced_text_signature(self): + block = dedent(""" + /*[clinic input] + @text_signature "($module, a[, b])" + src + a: object + param a + b: object = NULL + / + + docstring + [clinic start generated code]*/ + + /*[clinic input] + dst = src + [clinic start generated code]*/ + """) + self.clinic.parse(block) + self.addCleanup(rmtree, "clinic") + funcs = self.clinic.functions + self.assertEqual(len(funcs), 2) + + src_docstring_lines = funcs[0].docstring.split("\n") + dst_docstring_lines = funcs[1].docstring.split("\n") + + # Signatures are copied. + self.assertEqual(src_docstring_lines[0], "src($module, a[, b])") + self.assertEqual(dst_docstring_lines[0], "dst($module, a[, b])") + + # Param docstrings are copied. + self.assertIn(" param a", src_docstring_lines) + self.assertIn(" param a", dst_docstring_lines) + + # Docstrings are not copied. + self.assertIn("docstring", src_docstring_lines) + self.assertNotIn("docstring", dst_docstring_lines) + + def test_cloned_forced_text_signature_illegal(self): + block = """ + /*[clinic input] + @text_signature "($module, a[, b])" + src + a: object + b: object = NULL + / + [clinic start generated code]*/ + + /*[clinic input] + @text_signature "($module, a_override[, b])" + dst = src + [clinic start generated code]*/ + """ + err = "Cannot use @text_signature when cloning a function" + self.expect_failure(block, err, lineno=11) + + def test_ignore_preprocessor_in_comments(self): + for dsl in "clinic", "python": + raw = dedent(f"""\ + /*[{dsl} input] + # CPP directives, valid or not, should be ignored in C comments. + # + [{dsl} start generated code]*/ + """) + self.clinic.parse(raw) + class ParseFileUnitTest(TestCase): def expect_parsing_failure( self, *, filename, expected_error, verify=True, output=None ): errmsg = re.escape(dedent(expected_error).strip()) - with self.assertRaisesRegex(clinic.ClinicError, errmsg): - clinic.parse_file(filename, limited_capi=False) + with self.assertRaisesRegex(ClinicError, errmsg): + parse_file(filename, limited_capi=False) def test_parse_file_no_extension(self) -> None: self.expect_parsing_failure( @@ -663,7 +778,7 @@ def test_parse_file_strange_extension(self) -> None: class ClinicGroupPermuterTest(TestCase): def _test(self, l, m, r, output): - computed = clinic.permute_optional_groups(l, m, r) + computed = permute_optional_groups(l, m, r) self.assertEqual(output, computed) def test_range(self): @@ -705,7 +820,7 @@ def test_right_only(self): def test_have_left_options_but_required_is_empty(self): def fn(): - clinic.permute_optional_groups(['a'], [], []) + permute_optional_groups(['a'], [], []) self.assertRaises(ValueError, fn) @@ -763,13 +878,13 @@ def test_multiline_substitution(self): def test_text_before_block_marker(self): regex = re.escape("found before '{marker}'") - with self.assertRaisesRegex(clinic.ClinicError, regex): + with self.assertRaisesRegex(ClinicError, regex): libclinic.linear_format("no text before marker for you! {marker}", marker="not allowed!") def test_text_after_block_marker(self): regex = re.escape("found after '{marker}'") - with self.assertRaisesRegex(clinic.ClinicError, regex): + with self.assertRaisesRegex(ClinicError, regex): libclinic.linear_format("{marker} no text after marker for you!", marker="not allowed!") @@ -791,13 +906,12 @@ def parse(self, block): class ClinicBlockParserTest(TestCase): def _test(self, input, output): - language = clinic.CLanguage(None) + language = CLanguage(None) - blocks = list(clinic.BlockParser(input, language)) - writer = clinic.BlockPrinter(language) - c = _make_clinic() + blocks = list(BlockParser(input, language)) + writer = BlockPrinter(language) for block in blocks: - writer.print_block(block, limited_capi=c.limited_capi, header_includes=c.includes) + writer.print_block(block) output = writer.f.getvalue() assert output == input, "output != input!\n\noutput " + repr(output) + "\n\n input " + repr(input) @@ -822,8 +936,8 @@ def test_round_trip_2(self): """) def _test_clinic(self, input, output): - language = clinic.CLanguage(None) - c = clinic.Clinic(language, filename="file", limited_capi=False) + language = CLanguage(None) + c = Clinic(language, filename="file", limited_capi=False) c.parsers['inert'] = InertParser(c) c.parsers['copy'] = CopyParser(c) computed = c.parse(input) @@ -856,7 +970,7 @@ class ClinicParserTest(TestCase): def parse(self, text): c = _make_clinic() parser = DSLParser(c) - block = clinic.Block(text) + block = Block(text) parser.parse(block) return block @@ -864,8 +978,8 @@ def parse_function(self, text, signatures_in_block=2, function_index=1): block = self.parse(text) s = block.signatures self.assertEqual(len(s), signatures_in_block) - assert isinstance(s[0], clinic.Module) - assert isinstance(s[function_index], clinic.Function) + assert isinstance(s[0], Module) + assert isinstance(s[function_index], Function) return s[function_index] def expect_failure(self, block, err, *, @@ -880,7 +994,7 @@ def checkDocstring(self, fn, expected): def test_trivial(self): parser = DSLParser(_make_clinic()) - block = clinic.Block(""" + block = Block(""" module os os.access """) @@ -909,7 +1023,7 @@ def test_param(self): self.assertEqual(2, len(function.parameters)) p = function.parameters['path'] self.assertEqual('path', p.name) - self.assertIsInstance(p.converter, clinic.int_converter) + self.assertIsInstance(p.converter, int_converter) def test_param_default(self): function = self.parse_function(""" @@ -1008,7 +1122,7 @@ def test_param_no_docstring(self): """) self.assertEqual(3, len(function.parameters)) conv = function.parameters['something_else'].converter - self.assertIsInstance(conv, clinic.str_converter) + self.assertIsInstance(conv, str_converter) def test_param_default_parameters_out_of_order(self): err = ( @@ -1169,7 +1283,7 @@ def test_base_invalid_syntax(self): Function 'stat' has an invalid parameter declaration: \s+'invalid syntax: int = 42' """).strip() - with self.assertRaisesRegex(clinic.ClinicError, err): + with self.assertRaisesRegex(ClinicError, err): self.parse_function(block) def test_param_default_invalid_syntax(self): @@ -1201,7 +1315,7 @@ def test_return_converter(self): module os os.stat -> int """) - self.assertIsInstance(function.return_converter, clinic.int_return_converter) + self.assertIsInstance(function.return_converter, int_return_converter) def test_return_converter_invalid_syntax(self): block = """ @@ -1706,13 +1820,43 @@ def test_parameters_required_after_depr_star2(self): ) self.expect_failure(block, err, lineno=4) + def test_parameters_required_after_depr_star3(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + *args: tuple + b: int + Docstring. + """ + err = ( + "Function 'bar' specifies '* [from ...]' without " + "following parameters." + ) + self.expect_failure(block, err, lineno=4) + def test_depr_star_must_come_before_star(self): block = """ module foo foo.bar - this: int + a: int * * [from 3.14] + b: int + Docstring. + """ + err = "Function 'bar': '* [from ...]' must precede '*'" + self.expect_failure(block, err, lineno=4) + + def test_depr_star_must_come_before_vararg(self): + block = """ + module foo + foo.bar + a: int + *args: tuple + * [from 3.14] + b: int Docstring. """ err = "Function 'bar': '* [from ...]' must precede '*'" @@ -1827,7 +1971,7 @@ def test_double_slash(self): err = "Function 'bar' uses '/' more than once." self.expect_failure(block, err) - def test_mix_star_and_slash(self): + def test_slash_after_star(self): block = """ module foo foo.bar @@ -1840,6 +1984,19 @@ def test_mix_star_and_slash(self): err = "Function 'bar': '/' must precede '*'" self.expect_failure(block, err) + def test_slash_after_vararg(self): + block = """ + module foo + foo.bar + x: int + y: int + *args: tuple + z: int + / + """ + err = "Function 'bar': '/' must precede '*'" + self.expect_failure(block, err) + def test_depr_star_must_come_after_slash(self): block = """ module foo @@ -1879,6 +2036,19 @@ def test_star_must_come_after_depr_slash(self): err = "Function 'bar': '/ [from ...]' must precede '*'" self.expect_failure(block, err, lineno=4) + def test_vararg_must_come_after_depr_slash(self): + block = """ + module foo + foo.bar + a: int + *args: tuple + / [from 3.14] + b: int + Docstring. + """ + err = "Function 'bar': '/ [from ...]' must precede '*'" + self.expect_failure(block, err, lineno=4) + def test_depr_slash_must_come_after_slash(self): block = """ module foo @@ -1906,12 +2076,12 @@ def test_parameters_not_permitted_after_slash_for_now(self): self.expect_failure(block, err) def test_parameters_no_more_than_one_vararg(self): - err = "Too many var args" + err = "Function 'bar' uses '*' more than once." block = """ module foo foo.bar - *vararg1: object - *vararg2: object + *vararg1: tuple + *vararg2: tuple """ self.expect_failure(block, err, lineno=3) @@ -1996,11 +2166,11 @@ def test_indent_stack_no_tabs(self): block = """ module foo foo.bar - *vararg1: object - \t*vararg2: object + *vararg1: tuple + \t*vararg2: tuple """ err = ("Tab characters are illegal in the Clinic DSL: " - r"'\t*vararg2: object'") + r"'\t*vararg2: tuple'") self.expect_failure(block, err) def test_indent_stack_illegal_outdent(self): @@ -2017,7 +2187,7 @@ def test_directive(self): parser = DSLParser(_make_clinic()) parser.flag = False parser.directives['setflag'] = lambda : setattr(parser, 'flag', True) - block = clinic.Block("setflag") + block = Block("setflag") parser.parse(block) self.assertTrue(parser.flag) @@ -2025,7 +2195,7 @@ def test_legacy_converters(self): block = self.parse('module os\nos.access\n path: "s"') module, function = block.signatures conv = (function.parameters['path']).converter - self.assertIsInstance(conv, clinic.str_converter) + self.assertIsInstance(conv, str_converter) def test_legacy_converters_non_string_constant_annotation(self): err = "Annotations must be either a name, a function call, or a string" @@ -2282,14 +2452,14 @@ def test_unused_param(self): def test_scaffolding(self): # test repr on special values - self.assertEqual(repr(clinic.unspecified), '') - self.assertEqual(repr(clinic.NULL), '') + self.assertEqual(repr(unspecified), '') + self.assertEqual(repr(NULL), '') # test that fail fails with support.captured_stdout() as stdout: errmsg = 'The igloos are melting' - with self.assertRaisesRegex(clinic.ClinicError, errmsg) as cm: - clinic.fail(errmsg, filename='clown.txt', line_number=69) + with self.assertRaisesRegex(ClinicError, errmsg) as cm: + fail(errmsg, filename='clown.txt', line_number=69) exc = cm.exception self.assertEqual(exc.filename, 'clown.txt') self.assertEqual(exc.lineno, 69) @@ -2345,7 +2515,7 @@ def test_vararg_cannot_take_default_value(self): err = "Vararg can't take a default value!" block = """ fn - *args: object = None + *args: tuple = None """ self.expect_failure(block, err, lineno=1) @@ -2427,10 +2597,32 @@ def test_state_func_docstring_only_one_param_template(self): """ self.expect_failure(block, err, lineno=7) + def test_kind_defining_class(self): + function = self.parse_function(""" + module m + class m.C "PyObject *" "" + m.C.meth + cls: defining_class + """, signatures_in_block=3, function_index=2) + p = function.parameters['cls'] + self.assertEqual(p.kind, inspect.Parameter.POSITIONAL_ONLY) + + def test_disallow_defining_class_at_module_level(self): + err = "A 'defining_class' parameter cannot be defined at module level." + block = """ + module m + m.func + cls: defining_class + """ + self.expect_failure(block, err, lineno=2) + class ClinicExternalTest(TestCase): maxDiff = None + def setUp(self): + save_restore_converters(self) + def run_clinic(self, *args): with ( support.captured_stdout() as out, @@ -2521,7 +2713,7 @@ def test_cli_force(self): # Verify by checking the checksum. checksum = ( "/*[clinic end generated code: " - "output=c16447c01510dfb3 input=9543a8d2da235301]*/\n" + "output=00512eb783a9b748 input=9543a8d2da235301]*/\n" ) with open(fn, encoding='utf-8') as f: generated = f.read() @@ -2657,6 +2849,7 @@ def test_cli_converters(self): float() int() long() + object() Py_ssize_t() size_t() unsigned_int() @@ -3204,44 +3397,111 @@ def test_keyword_only_parameter(self): ac_tester.keyword_only_parameter(1) self.assertEqual(ac_tester.keyword_only_parameter(a=1), (1,)) - def test_posonly_vararg(self): - with self.assertRaises(TypeError): - ac_tester.posonly_vararg() - self.assertEqual(ac_tester.posonly_vararg(1, 2), (1, 2, ())) - self.assertEqual(ac_tester.posonly_vararg(1, b=2), (1, 2, ())) - self.assertEqual(ac_tester.posonly_vararg(1, 2, 3, 4), (1, 2, (3, 4))) - with self.assertRaises(TypeError): - ac_tester.posonly_vararg(b=4) - with self.assertRaises(TypeError): - ac_tester.posonly_vararg(1, 2, 3, b=4) - - def test_vararg_and_posonly(self): - with self.assertRaises(TypeError): - ac_tester.vararg_and_posonly() - with self.assertRaises(TypeError): - ac_tester.vararg_and_posonly(1, b=2) - self.assertEqual(ac_tester.vararg_and_posonly(1, 2, 3, 4), (1, (2, 3, 4))) - - def test_vararg(self): - with self.assertRaises(TypeError): - ac_tester.vararg() - with self.assertRaises(TypeError): - ac_tester.vararg(1, b=2) - self.assertEqual(ac_tester.vararg(1, 2, 3, 4), (1, (2, 3, 4))) - - def test_vararg_with_default(self): - with self.assertRaises(TypeError): - ac_tester.vararg_with_default() - self.assertEqual(ac_tester.vararg_with_default(1, b=False), (1, (), False)) - self.assertEqual(ac_tester.vararg_with_default(1, 2, 3, 4), (1, (2, 3, 4), False)) - self.assertEqual(ac_tester.vararg_with_default(1, 2, 3, 4, b=True), (1, (2, 3, 4), True)) - - def test_vararg_with_only_defaults(self): - self.assertEqual(ac_tester.vararg_with_only_defaults(), ((), None)) - self.assertEqual(ac_tester.vararg_with_only_defaults(b=2), ((), 2)) - self.assertEqual(ac_tester.vararg_with_only_defaults(1, b=2), ((1, ), 2)) - self.assertEqual(ac_tester.vararg_with_only_defaults(1, 2, 3, 4), ((1, 2, 3, 4), None)) - self.assertEqual(ac_tester.vararg_with_only_defaults(1, 2, 3, 4, b=5), ((1, 2, 3, 4), 5)) + if ac_tester is not None: + @repeat_fn(ac_tester.varpos, + ac_tester.varpos_array, + ac_tester.TestClass.varpos_no_fastcall, + ac_tester.TestClass.varpos_array_no_fastcall) + def test_varpos(self, fn): + # fn(*args) + self.assertEqual(fn(), ()) + self.assertEqual(fn(1, 2), (1, 2)) + + @repeat_fn(ac_tester.posonly_varpos, + ac_tester.posonly_varpos_array, + ac_tester.TestClass.posonly_varpos_no_fastcall, + ac_tester.TestClass.posonly_varpos_array_no_fastcall) + def test_posonly_varpos(self, fn): + # fn(a, b, /, *args) + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, 1) + self.assertRaises(TypeError, fn, 1, b=2) + self.assertEqual(fn(1, 2), (1, 2, ())) + self.assertEqual(fn(1, 2, 3, 4), (1, 2, (3, 4))) + + @repeat_fn(ac_tester.posonly_req_opt_varpos, + ac_tester.posonly_req_opt_varpos_array, + ac_tester.TestClass.posonly_req_opt_varpos_no_fastcall, + ac_tester.TestClass.posonly_req_opt_varpos_array_no_fastcall) + def test_posonly_req_opt_varpos(self, fn): + # fn(a, b=False, /, *args) + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, a=1) + self.assertEqual(fn(1), (1, False, ())) + self.assertEqual(fn(1, 2), (1, 2, ())) + self.assertEqual(fn(1, 2, 3, 4), (1, 2, (3, 4))) + + @repeat_fn(ac_tester.posonly_poskw_varpos, + ac_tester.posonly_poskw_varpos_array, + ac_tester.TestClass.posonly_poskw_varpos_no_fastcall, + ac_tester.TestClass.posonly_poskw_varpos_array_no_fastcall) + def test_posonly_poskw_varpos(self, fn): + # fn(a, /, b, *args) + self.assertRaises(TypeError, fn) + self.assertEqual(fn(1, 2), (1, 2, ())) + self.assertEqual(fn(1, b=2), (1, 2, ())) + self.assertEqual(fn(1, 2, 3, 4), (1, 2, (3, 4))) + self.assertRaises(TypeError, fn, b=4) + errmsg = re.escape("given by name ('b') and position (2)") + self.assertRaisesRegex(TypeError, errmsg, fn, 1, 2, 3, b=4) + + def test_poskw_varpos(self): + # fn(a, *args) + fn = ac_tester.poskw_varpos + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, 1, b=2) + self.assertEqual(fn(a=1), (1, ())) + errmsg = re.escape("given by name ('a') and position (1)") + self.assertRaisesRegex(TypeError, errmsg, fn, 1, a=2) + self.assertEqual(fn(1), (1, ())) + self.assertEqual(fn(1, 2, 3, 4), (1, (2, 3, 4))) + + def test_poskw_varpos_kwonly_opt(self): + # fn(a, *args, b=False) + fn = ac_tester.poskw_varpos_kwonly_opt + self.assertRaises(TypeError, fn) + errmsg = re.escape("given by name ('a') and position (1)") + self.assertRaisesRegex(TypeError, errmsg, fn, 1, a=2) + self.assertEqual(fn(1, b=2), (1, (), True)) + self.assertEqual(fn(1, 2, 3, 4), (1, (2, 3, 4), False)) + self.assertEqual(fn(1, 2, 3, 4, b=5), (1, (2, 3, 4), True)) + self.assertEqual(fn(a=1), (1, (), False)) + self.assertEqual(fn(a=1, b=2), (1, (), True)) + + def test_poskw_varpos_kwonly_opt2(self): + # fn(a, *args, b=False, c=False) + fn = ac_tester.poskw_varpos_kwonly_opt2 + self.assertRaises(TypeError, fn) + errmsg = re.escape("given by name ('a') and position (1)") + self.assertRaisesRegex(TypeError, errmsg, fn, 1, a=2) + self.assertEqual(fn(1, b=2), (1, (), 2, False)) + self.assertEqual(fn(1, b=2, c=3), (1, (), 2, 3)) + self.assertEqual(fn(1, 2, 3), (1, (2, 3), False, False)) + self.assertEqual(fn(1, 2, 3, b=4), (1, (2, 3), 4, False)) + self.assertEqual(fn(1, 2, 3, b=4, c=5), (1, (2, 3), 4, 5)) + self.assertEqual(fn(a=1), (1, (), False, False)) + self.assertEqual(fn(a=1, b=2), (1, (), 2, False)) + self.assertEqual(fn(a=1, b=2, c=3), (1, (), 2, 3)) + + def test_varpos_kwonly_opt(self): + # fn(*args, b=False) + fn = ac_tester.varpos_kwonly_opt + self.assertEqual(fn(), ((), False)) + self.assertEqual(fn(b=2), ((), 2)) + self.assertEqual(fn(1, b=2), ((1, ), 2)) + self.assertEqual(fn(1, 2, 3, 4), ((1, 2, 3, 4), False)) + self.assertEqual(fn(1, 2, 3, 4, b=5), ((1, 2, 3, 4), 5)) + + def test_varpos_kwonly_req_opt(self): + fn = ac_tester.varpos_kwonly_req_opt + self.assertRaises(TypeError, fn) + self.assertEqual(fn(a=1), ((), 1, False, False)) + self.assertEqual(fn(a=1, b=2), ((), 1, 2, False)) + self.assertEqual(fn(a=1, b=2, c=3), ((), 1, 2, 3)) + self.assertRaises(TypeError, fn, 1) + self.assertEqual(fn(1, a=2), ((1,), 2, False, False)) + self.assertEqual(fn(1, a=2, b=3), ((1,), 2, 3, False)) + self.assertEqual(fn(1, a=2, b=3, c=4), ((1,), 2, 3, 4)) def test_gh_32092_oob(self): ac_tester.gh_32092_oob(1, 2, 3, 4, kw1=5, kw2=6) @@ -3265,35 +3525,20 @@ def test_gh_99240_double_free(self): ac_tester.gh_99240_double_free('a', '\0b') def test_null_or_tuple_for_varargs(self): + # fn(name, *constraints, covariant=False) + fn = ac_tester.null_or_tuple_for_varargs # All of these should not crash: - valid_args_for_test = [ - (('a',), {}, - ('a', (), False)), - (('a', 1, 2, 3), {'covariant': True}, - ('a', (1, 2, 3), True)), - ((), {'name': 'a'}, - ('a', (), False)), - ((), {'name': 'a', 'covariant': True}, - ('a', (), True)), - ((), {'covariant': True, 'name': 'a'}, - ('a', (), True)), - ] - for args, kwargs, expected in valid_args_for_test: - with self.subTest(args=args, kwargs=kwargs): - self.assertEqual( - ac_tester.null_or_tuple_for_varargs(*args, **kwargs), - expected, - ) - - def test_null_or_tuple_for_varargs_error(self): - with self.assertRaises(TypeError): - ac_tester.null_or_tuple_for_varargs(covariant=True) - with self.assertRaises(TypeError): - ac_tester.null_or_tuple_for_varargs(1, name='a') - with self.assertRaises(TypeError): - ac_tester.null_or_tuple_for_varargs(1, 2, 3, name='a', covariant=True) - with self.assertRaises(TypeError): - ac_tester.null_or_tuple_for_varargs(1, 2, 3, covariant=True, name='a') + self.assertEqual(fn('a'), ('a', (), False)) + self.assertEqual(fn('a', 1, 2, 3, covariant=True), ('a', (1, 2, 3), True)) + self.assertEqual(fn(name='a'), ('a', (), False)) + self.assertEqual(fn(name='a', covariant=True), ('a', (), True)) + self.assertEqual(fn(covariant=True, name='a'), ('a', (), True)) + + self.assertRaises(TypeError, fn, covariant=True) + errmsg = re.escape("given by name ('name') and position (1)") + self.assertRaisesRegex(TypeError, errmsg, fn, 1, name='a') + self.assertRaisesRegex(TypeError, errmsg, fn, 1, 2, 3, name='a', covariant=True) + self.assertRaisesRegex(TypeError, errmsg, fn, 1, 2, 3, covariant=True, name='a') def test_cloned_func_exception_message(self): incorrect_arg = -1 # f1() and f2() accept a single str @@ -3308,26 +3553,79 @@ def test_cloned_func_with_converter_exception_message(self): func = getattr(ac_tester, name) self.assertEqual(func(), name) - def test_meth_method_no_params(self): + def test_get_defining_class(self): obj = ac_tester.TestClass() - meth = obj.meth_method_no_params + meth = obj.get_defining_class + self.assertIs(obj.get_defining_class(), ac_tester.TestClass) + + # 'defining_class' argument is a positional only argument + with self.assertRaises(TypeError): + obj.get_defining_class_arg(cls=ac_tester.TestClass) + check = partial(self.assertRaisesRegex, TypeError, "no arguments") check(meth, 1) check(meth, a=1) - def test_meth_method_no_params_capi(self): + def test_get_defining_class_capi(self): from _testcapi import pyobject_vectorcall obj = ac_tester.TestClass() - meth = obj.meth_method_no_params + meth = obj.get_defining_class pyobject_vectorcall(meth, None, None) pyobject_vectorcall(meth, (), None) pyobject_vectorcall(meth, (), ()) pyobject_vectorcall(meth, None, ()) + self.assertIs(pyobject_vectorcall(meth, (), ()), ac_tester.TestClass) check = partial(self.assertRaisesRegex, TypeError, "no arguments") check(pyobject_vectorcall, meth, (1,), None) check(pyobject_vectorcall, meth, (1,), ("a",)) + def test_get_defining_class_arg(self): + obj = ac_tester.TestClass() + self.assertEqual(obj.get_defining_class_arg("arg"), + (ac_tester.TestClass, "arg")) + self.assertEqual(obj.get_defining_class_arg(arg=123), + (ac_tester.TestClass, 123)) + + # 'defining_class' argument is a positional only argument + with self.assertRaises(TypeError): + obj.get_defining_class_arg(cls=ac_tester.TestClass, arg="arg") + + # wrong number of arguments + with self.assertRaises(TypeError): + obj.get_defining_class_arg() + with self.assertRaises(TypeError): + obj.get_defining_class_arg("arg1", "arg2") + + def test_defclass_varpos(self): + # fn(*args) + cls = ac_tester.TestClass + obj = cls() + fn = obj.defclass_varpos + self.assertEqual(fn(), (cls, ())) + self.assertEqual(fn(1, 2), (cls, (1, 2))) + fn = cls.defclass_varpos + self.assertRaises(TypeError, fn) + self.assertEqual(fn(obj), (cls, ())) + self.assertEqual(fn(obj, 1, 2), (cls, (1, 2))) + + def test_defclass_posonly_varpos(self): + # fn(a, b, /, *args) + cls = ac_tester.TestClass + obj = cls() + fn = obj.defclass_posonly_varpos + errmsg = 'takes at least 2 positional arguments' + self.assertRaisesRegex(TypeError, errmsg, fn) + self.assertRaisesRegex(TypeError, errmsg, fn, 1) + self.assertEqual(fn(1, 2), (cls, 1, 2, ())) + self.assertEqual(fn(1, 2, 3, 4), (cls, 1, 2, (3, 4))) + fn = cls.defclass_posonly_varpos + self.assertRaises(TypeError, fn) + self.assertRaisesRegex(TypeError, errmsg, fn, obj) + self.assertRaisesRegex(TypeError, errmsg, fn, obj, 1) + self.assertEqual(fn(obj, 1, 2), (cls, 1, 2, ())) + self.assertEqual(fn(obj, 1, 2, 3, 4), (cls, 1, 2, (3, 4))) + def test_depr_star_new(self): cls = ac_tester.DeprStarNew cls() @@ -3614,6 +3912,46 @@ def test_depr_multi(self): self.assertRaises(TypeError, fn, a="a", b="b", c="c", d="d", e="e", f="f", g="g") +class LimitedCAPIOutputTests(unittest.TestCase): + + def setUp(self): + self.clinic = _make_clinic(limited_capi=True) + + @staticmethod + def wrap_clinic_input(block): + return dedent(f""" + /*[clinic input] + output everything buffer + {block} + [clinic start generated code]*/ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + + def test_limited_capi_float(self): + block = self.wrap_clinic_input(""" + func + f: float + / + """) + generated = self.clinic.parse(block) + self.assertNotIn("PyFloat_AS_DOUBLE", generated) + self.assertIn("float f;", generated) + self.assertIn("f = (float) PyFloat_AsDouble", generated) + + def test_limited_capi_double(self): + block = self.wrap_clinic_input(""" + func + f: double + / + """) + generated = self.clinic.parse(block) + self.assertNotIn("PyFloat_AS_DOUBLE", generated) + self.assertIn("double f;", generated) + self.assertIn("f = PyFloat_AsDouble", generated) + + try: import _testclinic_limited except ImportError: @@ -3644,6 +3982,53 @@ def test_my_int_sum(self): with self.assertRaises(TypeError): _testclinic_limited.my_int_sum(1, "str") + def test_my_double_sum(self): + for func in ( + _testclinic_limited.my_float_sum, + _testclinic_limited.my_double_sum, + ): + with self.subTest(func=func.__name__): + self.assertEqual(func(1.0, 2.5), 3.5) + with self.assertRaises(TypeError): + func() + with self.assertRaises(TypeError): + func(1) + with self.assertRaises(TypeError): + func(1., "2") + + def test_get_file_descriptor(self): + # test 'file descriptor' converter: call PyObject_AsFileDescriptor() + get_fd = _testclinic_limited.get_file_descriptor + + class MyInt(int): + pass + + class MyFile: + def __init__(self, fd): + self._fd = fd + def fileno(self): + return self._fd + + for fd in (0, 1, 2, 5, 123_456): + self.assertEqual(get_fd(fd), fd) + + myint = MyInt(fd) + self.assertEqual(get_fd(myint), fd) + + myfile = MyFile(fd) + self.assertEqual(get_fd(myfile), fd) + + with self.assertRaises(OverflowError): + get_fd(2**256) + with self.assertWarnsRegex(RuntimeWarning, + "bool is used as a file descriptor"): + get_fd(True) + with self.assertRaises(TypeError): + get_fd(1.0) + with self.assertRaises(TypeError): + get_fd("abc") + with self.assertRaises(TypeError): + get_fd(None) class PermutationTests(unittest.TestCase): @@ -3657,7 +4042,7 @@ def test_permute_left_option_groups(self): (1, 2, 3), ) data = list(zip([1, 2, 3])) # Generate a list of 1-tuples. - actual = tuple(clinic.permute_left_option_groups(data)) + actual = tuple(permute_left_option_groups(data)) self.assertEqual(actual, expected) def test_permute_right_option_groups(self): @@ -3668,7 +4053,7 @@ def test_permute_right_option_groups(self): (1, 2, 3), ) data = list(zip([1, 2, 3])) # Generate a list of 1-tuples. - actual = tuple(clinic.permute_right_option_groups(data)) + actual = tuple(permute_right_option_groups(data)) self.assertEqual(actual, expected) def test_permute_optional_groups(self): @@ -3747,7 +4132,7 @@ def test_permute_optional_groups(self): for params in dataset: with self.subTest(**params): left, required, right, expected = params.values() - permutations = clinic.permute_optional_groups(left, required, right) + permutations = permute_optional_groups(left, required, right) actual = tuple(permutations) self.assertEqual(actual, expected) @@ -3888,15 +4273,15 @@ def test_suffix_all_lines(self): class ClinicReprTests(unittest.TestCase): def test_Block_repr(self): - block = clinic.Block("foo") + block = Block("foo") expected_repr = "" self.assertEqual(repr(block), expected_repr) - block2 = clinic.Block("bar", "baz", [], "eggs", "spam") + block2 = Block("bar", "baz", [], "eggs", "spam") expected_repr_2 = "" self.assertEqual(repr(block2), expected_repr_2) - block3 = clinic.Block( + block3 = Block( input="longboi_" * 100, dsl_name="wow_so_long", signatures=[], @@ -3911,47 +4296,44 @@ def test_Block_repr(self): def test_Destination_repr(self): c = _make_clinic() - destination = clinic.Destination( + destination = Destination( "foo", type="file", clinic=c, args=("eggs",) ) self.assertEqual( repr(destination), "" ) - destination2 = clinic.Destination("bar", type="buffer", clinic=c) + destination2 = Destination("bar", type="buffer", clinic=c) self.assertEqual(repr(destination2), "") def test_Module_repr(self): - module = clinic.Module("foo", _make_clinic()) + module = Module("foo", _make_clinic()) self.assertRegex(repr(module), r"") def test_Class_repr(self): - cls = clinic.Class("foo", _make_clinic(), None, 'some_typedef', 'some_type_object') + cls = Class("foo", _make_clinic(), None, 'some_typedef', 'some_type_object') self.assertRegex(repr(cls), r"") def test_FunctionKind_repr(self): self.assertEqual( - repr(clinic.FunctionKind.INVALID), "" - ) - self.assertEqual( - repr(clinic.FunctionKind.CLASS_METHOD), "" + repr(FunctionKind.CLASS_METHOD), "" ) def test_Function_and_Parameter_reprs(self): - function = clinic.Function( + function = Function( name='foo', module=_make_clinic(), cls=None, c_basename=None, full_name='foofoo', - return_converter=clinic.int_return_converter(), - kind=clinic.FunctionKind.METHOD_INIT, + return_converter=int_return_converter(), + kind=FunctionKind.METHOD_INIT, coexist=False ) self.assertEqual(repr(function), "") - converter = clinic.self_converter('bar', 'bar', function) - parameter = clinic.Parameter( + converter = self_converter('bar', 'bar', function) + parameter = Parameter( "bar", kind=inspect.Parameter.POSITIONAL_OR_KEYWORD, function=function, diff --git a/Lib/test/test_cmath.py b/Lib/test/test_cmath.py index 57f80d5d8cd016..a96a5780b31b6f 100644 --- a/Lib/test/test_cmath.py +++ b/Lib/test/test_cmath.py @@ -1,4 +1,5 @@ from test.support import requires_IEEE_754, cpython_only, import_helper +from test.support.testcase import ComplexesAreIdenticalMixin from test.test_math import parse_testfile, test_file import test.test_math as test_math import unittest @@ -49,7 +50,7 @@ (INF, NAN) ]] -class CMathTests(unittest.TestCase): +class CMathTests(ComplexesAreIdenticalMixin, unittest.TestCase): # list of all functions in cmath test_functions = [getattr(cmath, fname) for fname in [ 'acos', 'acosh', 'asin', 'asinh', 'atan', 'atanh', @@ -65,39 +66,6 @@ def setUp(self): def tearDown(self): self.test_values.close() - def assertFloatIdentical(self, x, y): - """Fail unless floats x and y are identical, in the sense that: - (1) both x and y are nans, or - (2) both x and y are infinities, with the same sign, or - (3) both x and y are zeros, with the same sign, or - (4) x and y are both finite and nonzero, and x == y - - """ - msg = 'floats {!r} and {!r} are not identical' - - if math.isnan(x) or math.isnan(y): - if math.isnan(x) and math.isnan(y): - return - elif x == y: - if x != 0.0: - return - # both zero; check that signs match - elif math.copysign(1.0, x) == math.copysign(1.0, y): - return - else: - msg += ': zeros have different signs' - self.fail(msg.format(x, y)) - - def assertComplexIdentical(self, x, y): - """Fail unless complex numbers x and y have equal values and signs. - - In particular, if x and y both have real (or imaginary) part - zero, but the zeros have different signs, this test will fail. - - """ - self.assertFloatIdentical(x.real, y.real) - self.assertFloatIdentical(x.imag, y.imag) - def rAssertAlmostEqual(self, a, b, rel_err = 2e-15, abs_err = 5e-323, msg=None): """Fail if the two floating-point numbers are not almost equal. @@ -555,7 +523,7 @@ def test_isinf(self): @requires_IEEE_754 def testTanhSign(self): for z in complex_zeros: - self.assertComplexIdentical(cmath.tanh(z), z) + self.assertComplexesAreIdentical(cmath.tanh(z), z) # The algorithm used for atan and atanh makes use of the system # log1p function; If that system function doesn't respect the sign @@ -564,12 +532,12 @@ def testTanhSign(self): @requires_IEEE_754 def testAtanSign(self): for z in complex_zeros: - self.assertComplexIdentical(cmath.atan(z), z) + self.assertComplexesAreIdentical(cmath.atan(z), z) @requires_IEEE_754 def testAtanhSign(self): for z in complex_zeros: - self.assertComplexIdentical(cmath.atanh(z), z) + self.assertComplexesAreIdentical(cmath.atanh(z), z) class IsCloseTests(test_math.IsCloseTests): diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 1fe3b2fe53c0b6..24cf357c581096 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -5,11 +5,14 @@ import os import subprocess import sys +import sysconfig import tempfile import textwrap import unittest from test import support from test.support import os_helper +from test.support import force_not_colorized +from test.support import threading_helper from test.support.script_helper import ( spawn_python, kill_python, assert_python_ok, assert_python_failure, interpreter_requires_environment @@ -38,6 +41,7 @@ def verify_valid_flag(self, cmd_line): self.assertNotIn(b'Traceback', err) return out + @support.cpython_only def test_help(self): self.verify_valid_flag('-h') self.verify_valid_flag('-?') @@ -48,14 +52,17 @@ def test_help(self): self.assertNotIn(b'-X dev', out) self.assertLess(len(lines), 50) + @support.cpython_only def test_help_env(self): out = self.verify_valid_flag('--help-env') self.assertIn(b'PYTHONHOME', out) + @support.cpython_only def test_help_xoptions(self): out = self.verify_valid_flag('--help-xoptions') self.assertIn(b'-X dev', out) + @support.cpython_only def test_help_all(self): out = self.verify_valid_flag('--help-all') lines = out.splitlines() @@ -74,6 +81,7 @@ def test_optimize(self): def test_site_flag(self): self.verify_valid_flag('-S') + @support.cpython_only def test_version(self): version = ('Python %d.%d' % sys.version_info[:2]).encode("ascii") for switch in '-V', '--version', '-VV': @@ -139,6 +147,7 @@ def run_python(*args): else: self.assertEqual(err, b'') + @support.cpython_only def test_xoption_frozen_modules(self): tests = { ('=on', 'FrozenImporter'), @@ -153,6 +162,7 @@ def test_xoption_frozen_modules(self): res = assert_python_ok(*cmd) self.assertRegex(res.out.decode('utf-8'), expected) + @support.cpython_only def test_env_var_frozen_modules(self): tests = { ('on', 'FrozenImporter'), @@ -326,6 +336,8 @@ def test_osx_android_utf8(self): self.assertEqual(stdout, expected) self.assertEqual(p.returncode, 0) + @unittest.skipIf(os.environ.get("PYTHONUNBUFFERED", "0") != "0", + "Python stdio buffering is disabled.") def test_non_interactive_output_buffering(self): code = textwrap.dedent(""" import sys @@ -579,6 +591,7 @@ def test_del___main__(self): print("del sys.modules['__main__']", file=script) assert_python_ok(filename) + @support.cpython_only def test_unknown_options(self): rc, out, err = assert_python_failure('-E', '-z') self.assertIn(b'Unknown option: -z', err) @@ -634,15 +647,13 @@ def test_sys_flags_set(self): PYTHONDONTWRITEBYTECODE=value, PYTHONVERBOSE=value, ) - dont_write_bytecode = int(bool(value)) + expected_bool = int(bool(value)) code = ( "import sys; " "sys.stderr.write(str(sys.flags)); " f"""sys.exit(not ( - sys.flags.debug == sys.flags.optimize == - sys.flags.verbose == - {expected} - and sys.flags.dont_write_bytecode == {dont_write_bytecode} + sys.flags.optimize == sys.flags.verbose == {expected} + and sys.flags.debug == sys.flags.dont_write_bytecode == {expected_bool} ))""" ) with self.subTest(envar_value=value): @@ -693,6 +704,7 @@ def run_xdev(self, *args, check_exitcode=True, xdev=True): self.assertEqual(proc.returncode, 0, proc) return proc.stdout.rstrip() + @support.cpython_only def test_xdev(self): # sys.flags.dev_mode code = "import sys; print(sys.flags.dev_mode)" @@ -729,7 +741,7 @@ def test_xdev(self): # Memory allocator debug hooks try: - import _testinternalcapi + import _testinternalcapi # noqa: F401 except ImportError: pass else: @@ -746,7 +758,7 @@ def test_xdev(self): # Faulthandler try: - import faulthandler + import faulthandler # noqa: F401 except ImportError: pass else: @@ -871,6 +883,118 @@ def test_pythondevmode_env(self): self.assertEqual(proc.stdout.rstrip(), 'True') self.assertEqual(proc.returncode, 0, proc) + def test_python_gil(self): + cases = [ + # (env, opt, expected, msg) + ('1', None, '1', "PYTHON_GIL=1"), + (None, '1', '1', "-X gil=1"), + ] + + if support.Py_GIL_DISABLED: + cases.extend( + [ + (None, None, 'None', "no options set"), + ('0', None, '0', "PYTHON_GIL=0"), + ('1', '0', '0', "-X gil=0 overrides PYTHON_GIL=1"), + (None, '0', '0', "-X gil=0"), + ] + ) + else: + cases.extend( + [ + (None, None, '1', '-X gil=0 (unsupported by this build)'), + ('1', None, '1', 'PYTHON_GIL=0 (unsupported by this build)'), + ] + ) + code = "import sys; print(sys.flags.gil)" + environ = dict(os.environ) + + for env, opt, expected, msg in cases: + with self.subTest(msg, env=env, opt=opt): + environ.pop('PYTHON_GIL', None) + if env is not None: + environ['PYTHON_GIL'] = env + extra_args = [] + if opt is not None: + extra_args = ['-X', f'gil={opt}'] + + proc = subprocess.run([sys.executable, *extra_args, '-c', code], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, env=environ) + self.assertEqual(proc.returncode, 0, proc) + self.assertEqual(proc.stdout.rstrip(), expected) + self.assertEqual(proc.stderr, '') + + def test_python_asyncio_debug(self): + code = "import asyncio; print(asyncio.new_event_loop().get_debug())" + rc, out, err = assert_python_ok('-c', code, PYTHONASYNCIODEBUG='1') + self.assertIn(b'True', out) + + @unittest.skipUnless(sysconfig.get_config_var('Py_TRACE_REFS'), "Requires --with-trace-refs build option") + def test_python_dump_refs(self): + code = 'import sys; sys._clear_type_cache()' + rc, out, err = assert_python_ok('-c', code, PYTHONDUMPREFS='1') + self.assertEqual(rc, 0) + + @unittest.skipUnless(sysconfig.get_config_var('Py_TRACE_REFS'), "Requires --with-trace-refs build option") + def test_python_dump_refs_file(self): + with tempfile.NamedTemporaryFile() as dump_file: + code = 'import sys; sys._clear_type_cache()' + rc, out, err = assert_python_ok('-c', code, PYTHONDUMPREFSFILE=dump_file.name) + self.assertEqual(rc, 0) + with open(dump_file.name, 'r') as file: + contents = file.read() + self.assertIn('Remaining objects', contents) + + @unittest.skipUnless(sys.platform == 'darwin', 'PYTHONEXECUTABLE only works on macOS') + def test_python_executable(self): + code = 'import sys; print(sys.executable)' + expected = "/busr/bbin/bpython" + rc, out, err = assert_python_ok('-c', code, PYTHONEXECUTABLE=expected) + self.assertIn(expected.encode(), out) + + @unittest.skipUnless(support.MS_WINDOWS, 'Test only applicable on Windows') + def test_python_legacy_windows_fs_encoding(self): + code = "import sys; print(sys.getfilesystemencoding())" + expected = 'mbcs' + rc, out, err = assert_python_ok('-c', code, PYTHONLEGACYWINDOWSFSENCODING='1') + self.assertIn(expected.encode(), out) + + @unittest.skipUnless(support.MS_WINDOWS, 'Test only applicable on Windows') + def test_python_legacy_windows_stdio(self): + code = "import sys; print(sys.stdin.encoding, sys.stdout.encoding)" + expected = 'cp' + rc, out, err = assert_python_ok('-c', code, PYTHONLEGACYWINDOWSSTDIO='1') + self.assertIn(expected.encode(), out) + + @unittest.skipIf("-fsanitize" in sysconfig.get_config_vars().get('PY_CFLAGS', ()), + "PYTHONMALLOCSTATS doesn't work with ASAN") + def test_python_malloc_stats(self): + code = "pass" + rc, out, err = assert_python_ok('-c', code, PYTHONMALLOCSTATS='1') + self.assertIn(b'Small block threshold', err) + + def test_python_user_base(self): + code = "import site; print(site.USER_BASE)" + expected = "/custom/userbase" + rc, out, err = assert_python_ok('-c', code, PYTHONUSERBASE=expected) + self.assertIn(expected.encode(), out) + + def test_python_basic_repl(self): + # Currently this only tests that the env var is set. See test_pyrepl.test_python_basic_repl. + code = "import os; print('PYTHON_BASIC_REPL' in os.environ)" + expected = "True" + rc, out, err = assert_python_ok('-c', code, PYTHON_BASIC_REPL='1') + self.assertIn(expected.encode(), out) + + @unittest.skipUnless(sysconfig.get_config_var('HAVE_PERF_TRAMPOLINE'), "Requires HAVE_PERF_TRAMPOLINE support") + def test_python_perf_jit_support(self): + code = "import sys; print(sys.is_stack_trampoline_active())" + expected = "True" + rc, out, err = assert_python_ok('-c', code, PYTHON_PERF_JIT_SUPPORT='1') + self.assertIn(expected.encode(), out) + @unittest.skipUnless(sys.platform == 'win32', 'bpo-32457 only applies on Windows') def test_argv0_normalization(self): @@ -883,13 +1007,14 @@ def test_argv0_normalization(self): self.assertEqual(proc.returncode, 0, proc) self.assertEqual(proc.stdout.strip(), b'0') + @support.cpython_only def test_parsing_error(self): args = [sys.executable, '-I', '--unknown-option'] proc = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - err_msg = "unknown option --unknown-option\nusage: " + err_msg = "Unknown option: --unknown-option\nusage: " self.assertTrue(proc.stderr.startswith(err_msg), proc.stderr) self.assertNotEqual(proc.returncode, 0) @@ -939,13 +1064,64 @@ def test_cpu_count_default(self): self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) res = assert_python_ok('-X', 'cpu_count=default', '-c', code, PYTHON_CPU_COUNT='1234') self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) - es = assert_python_ok('-c', code, PYTHON_CPU_COUNT='default') + res = assert_python_ok('-c', code, PYTHON_CPU_COUNT='default') self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) def res2int(self, res): out = res.out.strip().decode("utf-8") return tuple(int(i) for i in out.split()) + @unittest.skipUnless(support.Py_GIL_DISABLED, + "PYTHON_TLBC and -X tlbc" + " only supported in Py_GIL_DISABLED builds") + @threading_helper.requires_working_threading() + def test_disable_thread_local_bytecode(self): + code = """if 1: + import threading + def test(x, y): + return x + y + t = threading.Thread(target=test, args=(1,2)) + t.start() + t.join()""" + assert_python_ok("-W", "always", "-X", "tlbc=0", "-c", code) + assert_python_ok("-W", "always", "-c", code, PYTHON_TLBC="0") + + @unittest.skipUnless(support.Py_GIL_DISABLED, + "PYTHON_TLBC and -X tlbc" + " only supported in Py_GIL_DISABLED builds") + @threading_helper.requires_working_threading() + def test_enable_thread_local_bytecode(self): + code = """if 1: + import threading + def test(x, y): + return x + y + t = threading.Thread(target=test, args=(1,2)) + t.start() + t.join()""" + # The functionality of thread-local bytecode is tested more extensively + # in test_thread_local_bytecode + assert_python_ok("-W", "always", "-X", "tlbc=1", "-c", code) + assert_python_ok("-W", "always", "-c", code, PYTHON_TLBC="1") + + @unittest.skipUnless(support.Py_GIL_DISABLED, + "PYTHON_TLBC and -X tlbc" + " only supported in Py_GIL_DISABLED builds") + def test_invalid_thread_local_bytecode(self): + rc, out, err = assert_python_failure("-X", "tlbc") + self.assertIn(b"tlbc=n: n is missing or invalid", err) + rc, out, err = assert_python_failure("-X", "tlbc=foo") + self.assertIn(b"tlbc=n: n is missing or invalid", err) + rc, out, err = assert_python_failure("-X", "tlbc=-1") + self.assertIn(b"tlbc=n: n is missing or invalid", err) + rc, out, err = assert_python_failure("-X", "tlbc=2") + self.assertIn(b"tlbc=n: n is missing or invalid", err) + rc, out, err = assert_python_failure(PYTHON_TLBC="foo") + self.assertIn(b"PYTHON_TLBC=N: N is missing or invalid", err) + rc, out, err = assert_python_failure(PYTHON_TLBC="-1") + self.assertIn(b"PYTHON_TLBC=N: N is missing or invalid", err) + rc, out, err = assert_python_failure(PYTHON_TLBC="2") + self.assertIn(b"PYTHON_TLBC=N: N is missing or invalid", err) + @unittest.skipIf(interpreter_requires_environment(), 'Cannot run -I tests when PYTHON env vars are required.') @@ -986,6 +1162,7 @@ def test_sys_flags_not_set(self): class SyntaxErrorTests(unittest.TestCase): + @force_not_colorized def check_string(self, code): proc = subprocess.run([sys.executable, "-"], input=code, stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 3a5a8abf81e43d..e7f3e46c1868f7 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -88,6 +88,8 @@ def _make_test_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, importlib.invalidate_caches() return to_return + +@support.force_not_colorized_test_class class CmdLineTest(unittest.TestCase): def _check_output(self, script_name, exit_code, data, expected_file, expected_argv0, @@ -543,7 +545,7 @@ def test_pep_409_verbiage(self): script = textwrap.dedent("""\ try: raise ValueError - except: + except ValueError: raise NameError from None """) with os_helper.temp_dir() as script_dir: @@ -659,7 +661,8 @@ def test_syntaxerror_invalid_escape_sequence_multi_line(self): stderr.splitlines()[-3:], [ b' foo = """\\q"""', b' ^^^^^^^^', - b'SyntaxError: invalid escape sequence \'\\q\'' + b'SyntaxError: "\\q" is an invalid escape sequence. ' + b'Did you mean "\\\\q"? A raw string is also an option.' ], ) diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 46bebfc7af675b..69c1ee0690d269 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -17,7 +17,7 @@ freevars: () nlocals: 2 flags: 3 -consts: ('None', '') +consts: ('',) >>> dump(f(4).__code__) name: g @@ -86,7 +86,7 @@ cellvars: () freevars: () nlocals: 0 -flags: 3 +flags: 67108867 consts: ("'doc string'", 'None') >>> def keywordonly_args(a,b,*,k1): @@ -123,6 +123,75 @@ flags: 3 consts: ('None',) +>>> def has_docstring(x: str): +... 'This is a one-line doc string' +... x += x +... x += "hello world" +... # co_flags should be 0x4000003 = 67108867 +... return x + +>>> dump(has_docstring.__code__) +name: has_docstring +argcount: 1 +posonlyargcount: 0 +kwonlyargcount: 0 +names: () +varnames: ('x',) +cellvars: () +freevars: () +nlocals: 1 +flags: 67108867 +consts: ("'This is a one-line doc string'", "'hello world'") + +>>> async def async_func_docstring(x: str, y: str): +... "This is a docstring from async function" +... import asyncio +... await asyncio.sleep(1) +... # co_flags should be 0x4000083 = 67108995 +... return x + y + +>>> dump(async_func_docstring.__code__) +name: async_func_docstring +argcount: 2 +posonlyargcount: 0 +kwonlyargcount: 0 +names: ('asyncio', 'sleep') +varnames: ('x', 'y', 'asyncio') +cellvars: () +freevars: () +nlocals: 3 +flags: 67108995 +consts: ("'This is a docstring from async function'", 'None') + +>>> def no_docstring(x, y, z): +... return x + "hello" + y + z + "world" + +>>> dump(no_docstring.__code__) +name: no_docstring +argcount: 3 +posonlyargcount: 0 +kwonlyargcount: 0 +names: () +varnames: ('x', 'y', 'z') +cellvars: () +freevars: () +nlocals: 3 +flags: 3 +consts: ("'hello'", "'world'") + +>>> class class_with_docstring: +... '''This is a docstring for class''' +... '''This line is not docstring''' +... pass + +>>> print(class_with_docstring.__doc__) +This is a docstring for class + +>>> class class_without_docstring: +... pass + +>>> print(class_without_docstring.__doc__) +None """ import copy @@ -141,12 +210,13 @@ ctypes = None from test.support import (cpython_only, check_impl_detail, requires_debug_ranges, - gc_collect) + gc_collect, Py_GIL_DISABLED) from test.support.script_helper import assert_python_ok -from test.support import threading_helper -from test.support.bytecode_helper import (BytecodeTestCase, - instructions_with_positions) +from test.support import threading_helper, import_helper +from test.support.bytecode_helper import instructions_with_positions from opcode import opmap, opname +from _testcapi import code_offset_to_line + COPY_FREE_VARS = opmap['COPY_FREE_VARS'] @@ -176,7 +246,7 @@ class CodeTest(unittest.TestCase): @cpython_only def test_newempty(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") co = _testcapi.code_newempty("filename", "funcname", 15) self.assertEqual(co.co_filename, "filename") self.assertEqual(co.co_name, "funcname") @@ -255,10 +325,11 @@ def func(): return x code = func.__code__ - # different co_name, co_varnames, co_consts + # Different co_name, co_varnames, co_consts. + # Must have the same number of constants and + # variables or we get crashes. def func2(): y = 2 - z = 3 return y code2 = func2.__code__ @@ -358,14 +429,14 @@ def test_invalid_bytecode(self): def foo(): pass - # assert that opcode 229 is invalid - self.assertEqual(opname[229], '<229>') + # assert that opcode 135 is invalid + self.assertEqual(opname[135], '<135>') - # change first opcode to 0xeb (=229) + # change first opcode to 0x87 (=135) foo.__code__ = foo.__code__.replace( - co_code=b'\xe5' + foo.__code__.co_code[1:]) + co_code=b'\x87' + foo.__code__.co_code[1:]) - msg = "unknown opcode 229" + msg = "unknown opcode 135" with self.assertRaisesRegex(SystemError, msg): foo() @@ -570,11 +641,30 @@ def f(a='str_value'): self.assertIsInterned(f()) @cpython_only + @unittest.skipIf(Py_GIL_DISABLED, "free-threaded build interns all string constants") def test_interned_string_with_null(self): co = compile(r'res = "str\0value!"', '?', 'exec') v = self.find_const(co.co_consts, 'str\0value!') self.assertIsNotInterned(v) + @cpython_only + @unittest.skipUnless(Py_GIL_DISABLED, "does not intern all constants") + def test_interned_constants(self): + # compile separately to avoid compile time de-duping + + globals = {} + exec(textwrap.dedent(""" + def func1(): + return (0.0, (1, 2, "hello")) + """), globals) + + exec(textwrap.dedent(""" + def func2(): + return (0.0, (1, 2, "hello")) + """), globals) + + self.assertTrue(globals["func1"]() is globals["func2"]()) + class CodeWeakRefTest(unittest.TestCase): @@ -761,7 +851,7 @@ def f(): co_code=bytes( [ dis.opmap["RESUME"], 0, - dis.opmap["LOAD_ASSERTION_ERROR"], 0, + dis.opmap["LOAD_COMMON_CONSTANT"], 0, dis.opmap["RAISE_VARARGS"], 1, ] ), @@ -780,6 +870,71 @@ def f(): 3 * [(42, 42, None, None)], ) + @cpython_only + def test_docstring_under_o2(self): + code = textwrap.dedent(''' + def has_docstring(x, y): + """This is a first-line doc string""" + """This is a second-line doc string""" + a = x + y + b = x - y + return a, b + + + def no_docstring(x): + def g(y): + return x + y + return g + + + async def async_func(): + """asynf function doc string""" + pass + + + for func in [has_docstring, no_docstring(4), async_func]: + assert(func.__doc__ is None) + ''') + + rc, out, err = assert_python_ok('-OO', '-c', code) + + def test_co_branches(self): + + def get_line_branches(func): + code = func.__code__ + base = code.co_firstlineno + return [ + ( + code_offset_to_line(code, src) - base, + code_offset_to_line(code, left) - base, + code_offset_to_line(code, right) - base + ) for (src, left, right) in + code.co_branches() + ] + + def simple(x): + if x: + A + else: + B + + self.assertEqual( + get_line_branches(simple), + [(1,2,4)]) + + def with_extended_args(x): + if x: + A.x; A.x; A.x; A.x; A.x; A.x; + A.x; A.x; A.x; A.x; A.x; A.x; + A.x; A.x; A.x; A.x; A.x; A.x; + A.x; A.x; A.x; A.x; A.x; A.x; + A.x; A.x; A.x; A.x; A.x; A.x; + else: + B + + self.assertEqual( + get_line_branches(with_extended_args), + [(1,2,8)]) if check_impl_detail(cpython=True) and ctypes is not None: py = ctypes.pythonapi @@ -835,6 +990,7 @@ def test_free_called(self): SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(100)) del f + gc_collect() # For free-threaded build self.assertEqual(LAST_FREED, 100) def test_get_set(self): @@ -866,13 +1022,18 @@ def __init__(self, f, test): def run(self): del self.f gc_collect() - self.test.assertEqual(LAST_FREED, 500) + # gh-117683: In the free-threaded build, the code object's + # destructor may still be running concurrently in the main + # thread. + if not Py_GIL_DISABLED: + self.test.assertEqual(LAST_FREED, 500) SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(500)) tt = ThreadTest(f, self) del f tt.start() tt.join() + gc_collect() # For free-threaded build self.assertEqual(LAST_FREED, 500) diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 259778a5cade98..20b960ce8d1e02 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -1,12 +1,13 @@ "Test InteractiveConsole and InteractiveInterpreter from code module" import sys +import traceback import unittest from textwrap import dedent from contextlib import ExitStack from unittest import mock +from test.support import force_not_colorized_test_class from test.support import import_helper - code = import_helper.import_module('code') @@ -29,7 +30,9 @@ def mock_sys(self): del self.sysmod.ps2 +@force_not_colorized_test_class class TestInteractiveConsole(unittest.TestCase, MockSys): + maxDiff = None def setUp(self): self.console = code.InteractiveConsole() @@ -61,21 +64,151 @@ def test_console_stderr(self): raise AssertionError("no console stdout") def test_syntax_error(self): - self.infunc.side_effect = ["undefined", EOFError('Finished')] + self.infunc.side_effect = ["def f():", + " x = ?", + "", + EOFError('Finished')] self.console.interact() - for call in self.stderr.method_calls: - if 'NameError' in ''.join(call[1]): - break - else: - raise AssertionError("No syntax error from console") + output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) + output = output[output.index('(InteractiveConsole)'):] + output = output[:output.index('\nnow exiting')] + self.assertEqual(output.splitlines()[1:], [ + ' File "", line 2', + ' x = ?', + ' ^', + 'SyntaxError: invalid syntax']) + self.assertIs(self.sysmod.last_type, SyntaxError) + self.assertIs(type(self.sysmod.last_value), SyntaxError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + + def test_indentation_error(self): + self.infunc.side_effect = [" 1", EOFError('Finished')] + self.console.interact() + output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) + output = output[output.index('(InteractiveConsole)'):] + output = output[:output.index('\nnow exiting')] + self.assertEqual(output.splitlines()[1:], [ + ' File "", line 1', + ' 1', + 'IndentationError: unexpected indent']) + self.assertIs(self.sysmod.last_type, IndentationError) + self.assertIs(type(self.sysmod.last_value), IndentationError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + + def test_unicode_error(self): + self.infunc.side_effect = ["'\ud800'", EOFError('Finished')] + self.console.interact() + output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) + output = output[output.index('(InteractiveConsole)'):] + output = output[output.index('\n') + 1:] + self.assertTrue(output.startswith('UnicodeEncodeError: '), output) + self.assertIs(self.sysmod.last_type, UnicodeEncodeError) + self.assertIs(type(self.sysmod.last_value), UnicodeEncodeError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) def test_sysexcepthook(self): - self.infunc.side_effect = ["raise ValueError('')", + self.infunc.side_effect = ["def f():", + " raise ValueError('BOOM!')", + "", + "f()", EOFError('Finished')] hook = mock.Mock() self.sysmod.excepthook = hook self.console.interact() - self.assertTrue(hook.called) + hook.assert_called() + hook.assert_called_with(self.sysmod.last_type, + self.sysmod.last_value, + self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_type, ValueError) + self.assertIs(type(self.sysmod.last_value), ValueError) + self.assertIs(self.sysmod.last_traceback, self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + self.assertEqual(traceback.format_exception(self.sysmod.last_exc), [ + 'Traceback (most recent call last):\n', + ' File "", line 1, in \n', + ' File "", line 2, in f\n', + 'ValueError: BOOM!\n']) + + def test_sysexcepthook_syntax_error(self): + self.infunc.side_effect = ["def f():", + " x = ?", + "", + EOFError('Finished')] + hook = mock.Mock() + self.sysmod.excepthook = hook + self.console.interact() + hook.assert_called() + hook.assert_called_with(self.sysmod.last_type, + self.sysmod.last_value, + self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_type, SyntaxError) + self.assertIs(type(self.sysmod.last_value), SyntaxError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + self.assertEqual(traceback.format_exception(self.sysmod.last_exc), [ + ' File "", line 2\n', + ' x = ?\n', + ' ^\n', + 'SyntaxError: invalid syntax\n']) + + def test_sysexcepthook_indentation_error(self): + self.infunc.side_effect = [" 1", EOFError('Finished')] + hook = mock.Mock() + self.sysmod.excepthook = hook + self.console.interact() + hook.assert_called() + hook.assert_called_with(self.sysmod.last_type, + self.sysmod.last_value, + self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_type, IndentationError) + self.assertIs(type(self.sysmod.last_value), IndentationError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + self.assertEqual(traceback.format_exception(self.sysmod.last_exc), [ + ' File "", line 1\n', + ' 1\n', + 'IndentationError: unexpected indent\n']) + + def test_sysexcepthook_crashing_doesnt_close_repl(self): + self.infunc.side_effect = ["1/0", "a = 123", "print(a)", EOFError('Finished')] + self.sysmod.excepthook = 1 + self.console.interact() + self.assertEqual(['write', ('123', ), {}], self.stdout.method_calls[0]) + error = "".join(call.args[0] for call in self.stderr.method_calls if call[0] == 'write') + self.assertIn("Error in sys.excepthook:", error) + self.assertEqual(error.count("'int' object is not callable"), 1) + self.assertIn("Original exception was:", error) + self.assertIn("division by zero", error) + + def test_sysexcepthook_raising_BaseException(self): + self.infunc.side_effect = ["1/0", "a = 123", "print(a)", EOFError('Finished')] + s = "not so fast" + def raise_base(*args, **kwargs): + raise BaseException(s) + self.sysmod.excepthook = raise_base + self.console.interact() + self.assertEqual(['write', ('123', ), {}], self.stdout.method_calls[0]) + error = "".join(call.args[0] for call in self.stderr.method_calls if call[0] == 'write') + self.assertIn("Error in sys.excepthook:", error) + self.assertEqual(error.count("not so fast"), 1) + self.assertIn("Original exception was:", error) + self.assertIn("division by zero", error) + + def test_sysexcepthook_raising_SystemExit_gets_through(self): + self.infunc.side_effect = ["1/0"] + def raise_base(*args, **kwargs): + raise SystemExit + self.sysmod.excepthook = raise_base + with self.assertRaises(SystemExit): + self.console.interact() def test_banner(self): # with banner @@ -134,6 +267,11 @@ def test_cause_tb(self): ValueError """) self.assertIn(expected, output) + self.assertIs(self.sysmod.last_type, ValueError) + self.assertIs(type(self.sysmod.last_value), ValueError) + self.assertIs(self.sysmod.last_traceback, self.sysmod.last_value.__traceback__) + self.assertIsNotNone(self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) def test_context_tb(self): self.infunc.side_effect = ["try: ham\nexcept: eggs\n", @@ -152,6 +290,11 @@ def test_context_tb(self): NameError: name 'eggs' is not defined """) self.assertIn(expected, output) + self.assertIs(self.sysmod.last_type, NameError) + self.assertIs(type(self.sysmod.last_value), NameError) + self.assertIs(self.sysmod.last_traceback, self.sysmod.last_value.__traceback__) + self.assertIsNotNone(self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) class TestInteractiveConsoleLocalExit(unittest.TestCase, MockSys): diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 4991330489d139..86e5e5c1474674 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -1,3 +1,4 @@ +from _codecs import _unregister_error as _codecs_unregister_error import codecs import html.entities import itertools @@ -1210,7 +1211,6 @@ def replace_with_long(exc): '\ufffd\x00\x00' ) - def test_fake_error_class(self): handlers = [ codecs.strict_errors, @@ -1235,6 +1235,31 @@ class FakeUnicodeError(Exception): with self.assertRaises((TypeError, FakeUnicodeError)): handler(FakeUnicodeError()) + def test_reject_unregister_builtin_error_handler(self): + for name in [ + 'strict', 'ignore', 'replace', 'backslashreplace', 'namereplace', + 'xmlcharrefreplace', 'surrogateescape', 'surrogatepass', + ]: + with self.subTest(name): + self.assertRaises(ValueError, _codecs_unregister_error, name) + + def test_unregister_custom_error_handler(self): + def custom_handler(exc): + raise exc + + custom_name = 'test.test_unregister_custom_error_handler' + self.assertRaises(LookupError, codecs.lookup_error, custom_name) + codecs.register_error(custom_name, custom_handler) + self.assertIs(codecs.lookup_error(custom_name), custom_handler) + self.assertTrue(_codecs_unregister_error(custom_name)) + self.assertRaises(LookupError, codecs.lookup_error, custom_name) + + def test_unregister_custom_unknown_error_handler(self): + unknown_name = 'test.test_unregister_custom_unknown_error_handler' + self.assertRaises(LookupError, codecs.lookup_error, unknown_name) + self.assertFalse(_codecs_unregister_error(unknown_name)) + self.assertRaises(LookupError, codecs.lookup_error, unknown_name) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index ff511a625a0194..e51f7e0ee12b1f 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -2,7 +2,6 @@ import contextlib import copy import io -import locale import pickle import sys import unittest @@ -13,9 +12,9 @@ from test.support import os_helper try: - import _testcapi + import _testlimitedcapi except ImportError: - _testcapi = None + _testlimitedcapi = None try: import _testinternalcapi except ImportError: @@ -482,11 +481,11 @@ def test_only_one_bom(self): def test_badbom(self): s = io.BytesIO(4*b"\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) s = io.BytesIO(8*b"\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) def test_partial(self): self.check_partial( @@ -666,11 +665,11 @@ def test_only_one_bom(self): def test_badbom(self): s = io.BytesIO(b"\xff\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) s = io.BytesIO(b"\xff\xff\xff\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) def test_partial(self): self.check_partial( @@ -1356,13 +1355,29 @@ def test_decode(self): def test_decode_invalid(self): testcases = [ - (b"xn--w&", "strict", UnicodeError()), + (b"xn--w&", "strict", UnicodeDecodeError("punycode", b"", 5, 6, "")), + (b"&egbpdaj6bu4bxfgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 0, 1, "")), + (b"egbpdaj6bu&4bx&fgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 10, 11, "")), + (b"egbpdaj6bu4bxfgehfvwxn&", "strict", UnicodeDecodeError("punycode", b"", 22, 23, "")), + (b"\xFFProprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 0, 1, "")), + (b"Pro\xFFprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 3, 4, "")), + (b"Proprost&nemluvesky-uyb24&dma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")), + (b"Proprostnemluvesky&-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 20, 21, "")), + (b"Proprostnemluvesky-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 19, 20, "")), + (b"Proprostnemluvesky-uyb24d&ma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")), + (b"Proprostnemluvesky-uyb24dma41a&", "strict", UnicodeDecodeError("punycode", b"", 30, 31, "")), (b"xn--w&", "ignore", "xn-"), ] for puny, errors, expected in testcases: with self.subTest(puny=puny, errors=errors): if isinstance(expected, Exception): - self.assertRaises(UnicodeError, puny.decode, "punycode", errors) + with self.assertRaises(UnicodeDecodeError) as cm: + puny.decode("punycode", errors) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, puny) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) else: self.assertEqual(puny.decode("punycode", errors), expected) @@ -1532,7 +1547,7 @@ def test_nameprep(self): orig = str(orig, "utf-8", "surrogatepass") if prepped is None: # Input contains prohibited characters - self.assertRaises(UnicodeError, nameprep, orig) + self.assertRaises(UnicodeEncodeError, nameprep, orig) else: prepped = str(prepped, "utf-8", "surrogatepass") try: @@ -1542,11 +1557,46 @@ def test_nameprep(self): class IDNACodecTest(unittest.TestCase): + + invalid_decode_testcases = [ + (b"\xFFpython.org", UnicodeDecodeError("idna", b"\xFFpython.org", 0, 1, "")), + (b"pyt\xFFhon.org", UnicodeDecodeError("idna", b"pyt\xFFhon.org", 3, 4, "")), + (b"python\xFF.org", UnicodeDecodeError("idna", b"python\xFF.org", 6, 7, "")), + (b"python.\xFForg", UnicodeDecodeError("idna", b"python.\xFForg", 7, 8, "")), + (b"python.o\xFFrg", UnicodeDecodeError("idna", b"python.o\xFFrg", 8, 9, "")), + (b"python.org\xFF", UnicodeDecodeError("idna", b"python.org\xFF", 10, 11, "")), + (b"xn--pythn-&mua.org", UnicodeDecodeError("idna", b"xn--pythn-&mua.org", 10, 11, "")), + (b"xn--pythn-m&ua.org", UnicodeDecodeError("idna", b"xn--pythn-m&ua.org", 11, 12, "")), + (b"xn--pythn-mua&.org", UnicodeDecodeError("idna", b"xn--pythn-mua&.org", 13, 14, "")), + ] + invalid_encode_testcases = [ + (f"foo.{'\xff'*60}", UnicodeEncodeError("idna", f"foo.{'\xff'*60}", 4, 64, "")), + ("あさ.\u034f", UnicodeEncodeError("idna", "あさ.\u034f", 3, 4, "")), + ] + def test_builtin_decode(self): self.assertEqual(str(b"python.org", "idna"), "python.org") self.assertEqual(str(b"python.org.", "idna"), "python.org.") self.assertEqual(str(b"xn--pythn-mua.org", "idna"), "pyth\xf6n.org") self.assertEqual(str(b"xn--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"XN--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"xN--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"Xn--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"bugs.xn--pythn-mua.org.", "idna"), + "bugs.pyth\xf6n.org.") + self.assertEqual(str(b"bugs.XN--pythn-mua.org.", "idna"), + "bugs.pyth\xf6n.org.") + + def test_builtin_decode_invalid(self): + for case, expected in self.invalid_decode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeDecodeError) as cm: + case.decode("idna") + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start, msg=f'reason: {exc.reason}') + self.assertEqual(exc.end, expected.end) def test_builtin_encode(self): self.assertEqual("python.org".encode("idna"), b"python.org") @@ -1554,10 +1604,21 @@ def test_builtin_encode(self): self.assertEqual("pyth\xf6n.org".encode("idna"), b"xn--pythn-mua.org") self.assertEqual("pyth\xf6n.org.".encode("idna"), b"xn--pythn-mua.org.") + def test_builtin_encode_invalid(self): + for case, expected in self.invalid_encode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeEncodeError) as cm: + case.encode("idna") + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + def test_builtin_decode_length_limit(self): - with self.assertRaisesRegex(UnicodeError, "way too long"): + with self.assertRaisesRegex(UnicodeDecodeError, "way too long"): (b"xn--016c"+b"a"*1100).decode("idna") - with self.assertRaisesRegex(UnicodeError, "too long"): + with self.assertRaisesRegex(UnicodeDecodeError, "too long"): (b"xn--016c"+b"a"*70).decode("idna") def test_stream(self): @@ -1595,6 +1656,39 @@ def test_incremental_decode(self): self.assertEqual(decoder.decode(b"rg."), "org.") self.assertEqual(decoder.decode(b"", True), "") + def test_incremental_decode_invalid(self): + iterdecode_testcases = [ + (b"\xFFpython.org", UnicodeDecodeError("idna", b"\xFF", 0, 1, "")), + (b"pyt\xFFhon.org", UnicodeDecodeError("idna", b"pyt\xFF", 3, 4, "")), + (b"python\xFF.org", UnicodeDecodeError("idna", b"python\xFF", 6, 7, "")), + (b"python.\xFForg", UnicodeDecodeError("idna", b"\xFF", 0, 1, "")), + (b"python.o\xFFrg", UnicodeDecodeError("idna", b"o\xFF", 1, 2, "")), + (b"python.org\xFF", UnicodeDecodeError("idna", b"org\xFF", 3, 4, "")), + (b"xn--pythn-&mua.org", UnicodeDecodeError("idna", b"xn--pythn-&mua.", 10, 11, "")), + (b"xn--pythn-m&ua.org", UnicodeDecodeError("idna", b"xn--pythn-m&ua.", 11, 12, "")), + (b"xn--pythn-mua&.org", UnicodeDecodeError("idna", b"xn--pythn-mua&.", 13, 14, "")), + ] + for case, expected in iterdecode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeDecodeError) as cm: + list(codecs.iterdecode((bytes([c]) for c in case), "idna")) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + + decoder = codecs.getincrementaldecoder("idna")() + for case, expected in self.invalid_decode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeDecodeError) as cm: + decoder.decode(case) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + def test_incremental_encode(self): self.assertEqual( b"".join(codecs.iterencode("python.org", "idna")), @@ -1623,6 +1717,23 @@ def test_incremental_encode(self): self.assertEqual(encoder.encode("ample.org."), b"xn--xample-9ta.org.") self.assertEqual(encoder.encode("", True), b"") + def test_incremental_encode_invalid(self): + iterencode_testcases = [ + (f"foo.{'\xff'*60}", UnicodeEncodeError("idna", f"{'\xff'*60}", 0, 60, "")), + ("あさ.\u034f", UnicodeEncodeError("idna", "\u034f", 0, 1, "")), + ] + for case, expected in iterencode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeEncodeError) as cm: + list(codecs.iterencode(case, "idna")) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + + # codecs.getincrementalencoder.encode() does not throw an error + def test_errors(self): """Only supports "strict" error handler""" "python.org".encode("idna", "strict") @@ -1700,16 +1811,10 @@ def test_getwriter(self): self.assertRaises(TypeError, codecs.getwriter) self.assertRaises(LookupError, codecs.getwriter, "__spam__") + @support.run_with_locale('LC_CTYPE', 'tr_TR') def test_lookup_issue1813(self): # Issue #1813: under Turkish locales, lookup of some codecs failed # because 'I' is lowercased as "ı" (dotless i) - oldlocale = locale.setlocale(locale.LC_CTYPE) - self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) - try: - locale.setlocale(locale.LC_CTYPE, 'tr_TR') - except locale.Error: - # Unsupported locale on this system - self.skipTest('test needs Turkish locale') c = codecs.lookup('ASCII') self.assertEqual(c.name, 'ascii') @@ -2112,14 +2217,14 @@ def test_basics(self): "encoding=%r" % encoding) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_basics_capi(self): s = "abc123" # all codecs should be able to encode these for encoding in all_unicode_encodings: if encoding not in broken_unicode_with_stateful: # check incremental decoder/encoder (fetched via the C API) try: - cencoder = _testcapi.codec_incrementalencoder(encoding) + cencoder = _testlimitedcapi.codec_incrementalencoder(encoding) except LookupError: # no IncrementalEncoder pass else: @@ -2128,7 +2233,7 @@ def test_basics_capi(self): for c in s: encodedresult += cencoder.encode(c) encodedresult += cencoder.encode("", True) - cdecoder = _testcapi.codec_incrementaldecoder(encoding) + cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding) decodedresult = "" for c in encodedresult: decodedresult += cdecoder.decode(bytes([c])) @@ -2139,12 +2244,12 @@ def test_basics_capi(self): if encoding not in ("idna", "mbcs"): # check incremental decoder/encoder with errors argument try: - cencoder = _testcapi.codec_incrementalencoder(encoding, "ignore") + cencoder = _testlimitedcapi.codec_incrementalencoder(encoding, "ignore") except LookupError: # no IncrementalEncoder pass else: encodedresult = b"".join(cencoder.encode(c) for c in s) - cdecoder = _testcapi.codec_incrementaldecoder(encoding, "ignore") + cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding, "ignore") decodedresult = "".join(cdecoder.decode(bytes([c])) for c in encodedresult) self.assertEqual(decodedresult, s, @@ -2846,7 +2951,7 @@ def test_seek0(self): bytes_transform_encodings.append("zlib_codec") transform_aliases["zlib_codec"] = ["zip", "zlib"] try: - import bz2 + import bz2 # noqa: F401 except ImportError: pass else: @@ -3151,7 +3256,11 @@ def test_code_page_name(self): codecs.code_page_decode, self.CP_UTF8, b'\xff', 'strict', True) def check_decode(self, cp, tests): - for raw, errors, expected in tests: + for raw, errors, expected, *rest in tests: + if rest: + altexpected, = rest + else: + altexpected = expected if expected is not None: try: decoded = codecs.code_page_decode(cp, raw, errors, True) @@ -3168,8 +3277,21 @@ def check_decode(self, cp, tests): self.assertRaises(UnicodeDecodeError, codecs.code_page_decode, cp, raw, errors, True) + if altexpected is not None: + decoded = raw.decode(f'cp{cp}', errors) + self.assertEqual(decoded, altexpected, + '%a.decode("cp%s", %r)=%a != %a' + % (raw, cp, errors, decoded, altexpected)) + else: + self.assertRaises(UnicodeDecodeError, + raw.decode, f'cp{cp}', errors) + def check_encode(self, cp, tests): - for text, errors, expected in tests: + for text, errors, expected, *rest in tests: + if rest: + altexpected, = rest + else: + altexpected = expected if expected is not None: try: encoded = codecs.code_page_encode(cp, text, errors) @@ -3180,18 +3302,26 @@ def check_encode(self, cp, tests): '%a.encode("cp%s", %r)=%a != %a' % (text, cp, errors, encoded[0], expected)) self.assertEqual(encoded[1], len(text)) + + encoded = text.encode(f'cp{cp}', errors) + self.assertEqual(encoded, altexpected, + '%a.encode("cp%s", %r)=%a != %a' + % (text, cp, errors, encoded, altexpected)) else: self.assertRaises(UnicodeEncodeError, codecs.code_page_encode, cp, text, errors) + self.assertRaises(UnicodeEncodeError, + text.encode, f'cp{cp}', errors) def test_cp932(self): self.check_encode(932, ( ('abc', 'strict', b'abc'), ('\uff44\u9a3e', 'strict', b'\x82\x84\xe9\x80'), + ('\uf8f3', 'strict', b'\xff'), # test error handlers ('\xff', 'strict', None), ('[\xff]', 'ignore', b'[]'), - ('[\xff]', 'replace', b'[y]'), + ('[\xff]', 'replace', b'[y]', b'[?]'), ('[\u20ac]', 'replace', b'[?]'), ('[\xff]', 'backslashreplace', b'[\\xff]'), ('[\xff]', 'namereplace', @@ -3205,12 +3335,12 @@ def test_cp932(self): (b'abc', 'strict', 'abc'), (b'\x82\x84\xe9\x80', 'strict', '\uff44\u9a3e'), # invalid bytes - (b'[\xff]', 'strict', None), - (b'[\xff]', 'ignore', '[]'), - (b'[\xff]', 'replace', '[\ufffd]'), - (b'[\xff]', 'backslashreplace', '[\\xff]'), - (b'[\xff]', 'surrogateescape', '[\udcff]'), - (b'[\xff]', 'surrogatepass', None), + (b'[\xff]', 'strict', None, '[\uf8f3]'), + (b'[\xff]', 'ignore', '[]', '[\uf8f3]'), + (b'[\xff]', 'replace', '[\ufffd]', '[\uf8f3]'), + (b'[\xff]', 'backslashreplace', '[\\xff]', '[\uf8f3]'), + (b'[\xff]', 'surrogateescape', '[\udcff]', '[\uf8f3]'), + (b'[\xff]', 'surrogatepass', None, '[\uf8f3]'), (b'\x81\x00abc', 'strict', None), (b'\x81\x00abc', 'ignore', '\x00abc'), (b'\x81\x00abc', 'replace', '\ufffd\x00abc'), @@ -3225,7 +3355,7 @@ def test_cp1252(self): # test error handlers ('\u0141', 'strict', None), ('\u0141', 'ignore', b''), - ('\u0141', 'replace', b'L'), + ('\u0141', 'replace', b'L', b'?'), ('\udc98', 'surrogateescape', b'\x98'), ('\udc98', 'surrogatepass', None), )) @@ -3235,6 +3365,59 @@ def test_cp1252(self): (b'\xff', 'strict', '\xff'), )) + def test_cp708(self): + self.check_encode(708, ( + ('abc2%', 'strict', b'abc2%'), + ('\u060c\u0621\u064a', 'strict', b'\xac\xc1\xea'), + ('\u2562\xe7\xa0', 'strict', b'\x86\x87\xff'), + ('\x9a\x9f', 'strict', b'\x9a\x9f'), + ('\u256b', 'strict', b'\xc0'), + # test error handlers + ('[\u0662]', 'strict', None), + ('[\u0662]', 'ignore', b'[]'), + ('[\u0662]', 'replace', b'[?]'), + ('\udca0', 'surrogateescape', b'\xa0'), + ('\udca0', 'surrogatepass', None), + )) + self.check_decode(708, ( + (b'abc2%', 'strict', 'abc2%'), + (b'\xac\xc1\xea', 'strict', '\u060c\u0621\u064a'), + (b'\x86\x87\xff', 'strict', '\u2562\xe7\xa0'), + (b'\x9a\x9f', 'strict', '\x9a\x9f'), + (b'\xc0', 'strict', '\u256b'), + # test error handlers + (b'\xa0', 'strict', None), + (b'[\xa0]', 'ignore', '[]'), + (b'[\xa0]', 'replace', '[\ufffd]'), + (b'[\xa0]', 'backslashreplace', '[\\xa0]'), + (b'[\xa0]', 'surrogateescape', '[\udca0]'), + (b'[\xa0]', 'surrogatepass', None), + )) + + def test_cp20106(self): + self.check_encode(20106, ( + ('abc', 'strict', b'abc'), + ('\xa7\xc4\xdf', 'strict', b'@[~'), + # test error handlers + ('@', 'strict', None), + ('@', 'ignore', b''), + ('@', 'replace', b'?'), + ('\udcbf', 'surrogateescape', b'\xbf'), + ('\udcbf', 'surrogatepass', None), + )) + self.check_decode(20106, ( + (b'abc', 'strict', 'abc'), + (b'@[~', 'strict', '\xa7\xc4\xdf'), + (b'\xe1\xfe', 'strict', 'a\xdf'), + # test error handlers + (b'(\xbf)', 'strict', None), + (b'(\xbf)', 'ignore', '()'), + (b'(\xbf)', 'replace', '(\ufffd)'), + (b'(\xbf)', 'backslashreplace', '(\\xbf)'), + (b'(\xbf)', 'surrogateescape', '(\udcbf)'), + (b'(\xbf)', 'surrogatepass', None), + )) + def test_cp_utf7(self): cp = 65000 self.check_encode(cp, ( @@ -3307,17 +3490,15 @@ def test_incremental(self): False) self.assertEqual(decoded, ('abc', 3)) - def test_mbcs_alias(self): - # Check that looking up our 'default' codepage will return - # mbcs when we don't have a more specific one available - code_page = 99_999 - name = f'cp{code_page}' - with mock.patch('_winapi.GetACP', return_value=code_page): - try: - codec = codecs.lookup(name) - self.assertEqual(codec.name, 'mbcs') - finally: - codecs.unregister(name) + def test_mbcs_code_page(self): + # Check that codec for the current Windows (ANSII) code page is + # always available. + try: + from _winapi import GetACP + except ImportError: + self.skipTest('requires _winapi.GetACP') + cp = GetACP() + codecs.lookup(f'cp{cp}') @support.bigmemtest(size=2**31, memuse=7, dry_run=False) def test_large_input(self, size): diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index 787bd1b6a79e20..0eefc22d11bce0 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -282,7 +282,7 @@ def test_warning(self): # Test that the warning is only returned once. with warnings_helper.check_warnings( ('"is" with \'str\' literal', SyntaxWarning), - ("invalid escape sequence", SyntaxWarning), + ('"\\\\e" is an invalid escape sequence', SyntaxWarning), ) as w: compile_command(r"'\e' is 0") self.assertEqual(len(w.warnings), 2) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 1fb492ecebd668..1e93530398be79 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -26,7 +26,7 @@ from collections.abc import Set, MutableSet from collections.abc import Mapping, MutableMapping, KeysView, ItemsView, ValuesView from collections.abc import Sequence, MutableSequence -from collections.abc import ByteString, Buffer +from collections.abc import Buffer class TestUserObjects(unittest.TestCase): @@ -542,7 +542,7 @@ def test_odd_sizes(self): self.assertEqual(Dot(1)._replace(d=999), (999,)) self.assertEqual(Dot(1)._fields, ('d',)) - n = support.EXCEEDS_RECURSION_LIMIT + n = support.exceeds_recursion_limit() names = list(set(''.join([choice(string.ascii_letters) for j in range(10)]) for i in range(n))) n = len(names) @@ -742,11 +742,11 @@ def validate_isinstance(self, abc, name): C = type('C', (object,), {'__hash__': None}) setattr(C, name, stub) self.assertIsInstance(C(), abc) - self.assertTrue(issubclass(C, abc)) + self.assertIsSubclass(C, abc) C = type('C', (object,), {'__hash__': None}) self.assertNotIsInstance(C(), abc) - self.assertFalse(issubclass(C, abc)) + self.assertNotIsSubclass(C, abc) def validate_comparison(self, instance): ops = ['lt', 'gt', 'le', 'ge', 'ne', 'or', 'and', 'xor', 'sub'] @@ -812,12 +812,12 @@ def __await__(self): non_samples = [None, int(), gen(), object()] for x in non_samples: self.assertNotIsInstance(x, Awaitable) - self.assertFalse(issubclass(type(x), Awaitable), repr(type(x))) + self.assertNotIsSubclass(type(x), Awaitable) samples = [Bar(), MinimalCoro()] for x in samples: self.assertIsInstance(x, Awaitable) - self.assertTrue(issubclass(type(x), Awaitable)) + self.assertIsSubclass(type(x), Awaitable) c = coro() # Iterable coroutines (generators with CO_ITERABLE_COROUTINE @@ -831,8 +831,8 @@ def __await__(self): class CoroLike: pass Coroutine.register(CoroLike) - self.assertTrue(isinstance(CoroLike(), Awaitable)) - self.assertTrue(issubclass(CoroLike, Awaitable)) + self.assertIsInstance(CoroLike(), Awaitable) + self.assertIsSubclass(CoroLike, Awaitable) CoroLike = None support.gc_collect() # Kill CoroLike to clean-up ABCMeta cache @@ -864,12 +864,12 @@ def __await__(self): non_samples = [None, int(), gen(), object(), Bar()] for x in non_samples: self.assertNotIsInstance(x, Coroutine) - self.assertFalse(issubclass(type(x), Coroutine), repr(type(x))) + self.assertNotIsSubclass(type(x), Coroutine) samples = [MinimalCoro()] for x in samples: self.assertIsInstance(x, Awaitable) - self.assertTrue(issubclass(type(x), Awaitable)) + self.assertIsSubclass(type(x), Awaitable) c = coro() # Iterable coroutines (generators with CO_ITERABLE_COROUTINE @@ -890,8 +890,8 @@ def close(self): pass def __await__(self): pass - self.assertTrue(isinstance(CoroLike(), Coroutine)) - self.assertTrue(issubclass(CoroLike, Coroutine)) + self.assertIsInstance(CoroLike(), Coroutine) + self.assertIsSubclass(CoroLike, Coroutine) class CoroLike: def send(self, value): @@ -900,15 +900,15 @@ def close(self): pass def __await__(self): pass - self.assertFalse(isinstance(CoroLike(), Coroutine)) - self.assertFalse(issubclass(CoroLike, Coroutine)) + self.assertNotIsInstance(CoroLike(), Coroutine) + self.assertNotIsSubclass(CoroLike, Coroutine) def test_Hashable(self): # Check some non-hashables non_samples = [bytearray(), list(), set(), dict()] for x in non_samples: self.assertNotIsInstance(x, Hashable) - self.assertFalse(issubclass(type(x), Hashable), repr(type(x))) + self.assertNotIsSubclass(type(x), Hashable) # Check some hashables samples = [None, int(), float(), complex(), @@ -918,14 +918,14 @@ def test_Hashable(self): ] for x in samples: self.assertIsInstance(x, Hashable) - self.assertTrue(issubclass(type(x), Hashable), repr(type(x))) + self.assertIsSubclass(type(x), Hashable) self.assertRaises(TypeError, Hashable) # Check direct subclassing class H(Hashable): def __hash__(self): return super().__hash__() self.assertEqual(hash(H()), 0) - self.assertFalse(issubclass(int, H)) + self.assertNotIsSubclass(int, H) self.validate_abstract_methods(Hashable, '__hash__') self.validate_isinstance(Hashable, '__hash__') @@ -933,13 +933,13 @@ def test_AsyncIterable(self): class AI: def __aiter__(self): return self - self.assertTrue(isinstance(AI(), AsyncIterable)) - self.assertTrue(issubclass(AI, AsyncIterable)) + self.assertIsInstance(AI(), AsyncIterable) + self.assertIsSubclass(AI, AsyncIterable) # Check some non-iterables non_samples = [None, object, []] for x in non_samples: self.assertNotIsInstance(x, AsyncIterable) - self.assertFalse(issubclass(type(x), AsyncIterable), repr(type(x))) + self.assertNotIsSubclass(type(x), AsyncIterable) self.validate_abstract_methods(AsyncIterable, '__aiter__') self.validate_isinstance(AsyncIterable, '__aiter__') @@ -949,13 +949,13 @@ def __aiter__(self): return self async def __anext__(self): raise StopAsyncIteration - self.assertTrue(isinstance(AI(), AsyncIterator)) - self.assertTrue(issubclass(AI, AsyncIterator)) + self.assertIsInstance(AI(), AsyncIterator) + self.assertIsSubclass(AI, AsyncIterator) non_samples = [None, object, []] # Check some non-iterables for x in non_samples: self.assertNotIsInstance(x, AsyncIterator) - self.assertFalse(issubclass(type(x), AsyncIterator), repr(type(x))) + self.assertNotIsSubclass(type(x), AsyncIterator) # Similarly to regular iterators (see issue 10565) class AnextOnly: async def __anext__(self): @@ -968,7 +968,7 @@ def test_Iterable(self): non_samples = [None, 42, 3.14, 1j] for x in non_samples: self.assertNotIsInstance(x, Iterable) - self.assertFalse(issubclass(type(x), Iterable), repr(type(x))) + self.assertNotIsSubclass(type(x), Iterable) # Check some iterables samples = [bytes(), str(), tuple(), list(), set(), frozenset(), dict(), @@ -978,13 +978,13 @@ def test_Iterable(self): ] for x in samples: self.assertIsInstance(x, Iterable) - self.assertTrue(issubclass(type(x), Iterable), repr(type(x))) + self.assertIsSubclass(type(x), Iterable) # Check direct subclassing class I(Iterable): def __iter__(self): return super().__iter__() self.assertEqual(list(I()), []) - self.assertFalse(issubclass(str, I)) + self.assertNotIsSubclass(str, I) self.validate_abstract_methods(Iterable, '__iter__') self.validate_isinstance(Iterable, '__iter__') # Check None blocking @@ -992,22 +992,22 @@ class It: def __iter__(self): return iter([]) class ItBlocked(It): __iter__ = None - self.assertTrue(issubclass(It, Iterable)) - self.assertTrue(isinstance(It(), Iterable)) - self.assertFalse(issubclass(ItBlocked, Iterable)) - self.assertFalse(isinstance(ItBlocked(), Iterable)) + self.assertIsSubclass(It, Iterable) + self.assertIsInstance(It(), Iterable) + self.assertNotIsSubclass(ItBlocked, Iterable) + self.assertNotIsInstance(ItBlocked(), Iterable) def test_Reversible(self): # Check some non-reversibles non_samples = [None, 42, 3.14, 1j, set(), frozenset()] for x in non_samples: self.assertNotIsInstance(x, Reversible) - self.assertFalse(issubclass(type(x), Reversible), repr(type(x))) + self.assertNotIsSubclass(type(x), Reversible) # Check some non-reversible iterables non_reversibles = [_test_gen(), (x for x in []), iter([]), reversed([])] for x in non_reversibles: self.assertNotIsInstance(x, Reversible) - self.assertFalse(issubclass(type(x), Reversible), repr(type(x))) + self.assertNotIsSubclass(type(x), Reversible) # Check some reversible iterables samples = [bytes(), str(), tuple(), list(), OrderedDict(), OrderedDict().keys(), OrderedDict().items(), @@ -1016,11 +1016,11 @@ def test_Reversible(self): dict().keys(), dict().items(), dict().values()] for x in samples: self.assertIsInstance(x, Reversible) - self.assertTrue(issubclass(type(x), Reversible), repr(type(x))) + self.assertIsSubclass(type(x), Reversible) # Check also Mapping, MutableMapping, and Sequence - self.assertTrue(issubclass(Sequence, Reversible), repr(Sequence)) - self.assertFalse(issubclass(Mapping, Reversible), repr(Mapping)) - self.assertFalse(issubclass(MutableMapping, Reversible), repr(MutableMapping)) + self.assertIsSubclass(Sequence, Reversible) + self.assertNotIsSubclass(Mapping, Reversible) + self.assertNotIsSubclass(MutableMapping, Reversible) # Check direct subclassing class R(Reversible): def __iter__(self): @@ -1028,17 +1028,17 @@ def __iter__(self): def __reversed__(self): return iter(list()) self.assertEqual(list(reversed(R())), []) - self.assertFalse(issubclass(float, R)) + self.assertNotIsSubclass(float, R) self.validate_abstract_methods(Reversible, '__reversed__', '__iter__') # Check reversible non-iterable (which is not Reversible) class RevNoIter: def __reversed__(self): return reversed([]) class RevPlusIter(RevNoIter): def __iter__(self): return iter([]) - self.assertFalse(issubclass(RevNoIter, Reversible)) - self.assertFalse(isinstance(RevNoIter(), Reversible)) - self.assertTrue(issubclass(RevPlusIter, Reversible)) - self.assertTrue(isinstance(RevPlusIter(), Reversible)) + self.assertNotIsSubclass(RevNoIter, Reversible) + self.assertNotIsInstance(RevNoIter(), Reversible) + self.assertIsSubclass(RevPlusIter, Reversible) + self.assertIsInstance(RevPlusIter(), Reversible) # Check None blocking class Rev: def __iter__(self): return iter([]) @@ -1047,39 +1047,38 @@ class RevItBlocked(Rev): __iter__ = None class RevRevBlocked(Rev): __reversed__ = None - self.assertTrue(issubclass(Rev, Reversible)) - self.assertTrue(isinstance(Rev(), Reversible)) - self.assertFalse(issubclass(RevItBlocked, Reversible)) - self.assertFalse(isinstance(RevItBlocked(), Reversible)) - self.assertFalse(issubclass(RevRevBlocked, Reversible)) - self.assertFalse(isinstance(RevRevBlocked(), Reversible)) + self.assertIsSubclass(Rev, Reversible) + self.assertIsInstance(Rev(), Reversible) + self.assertNotIsSubclass(RevItBlocked, Reversible) + self.assertNotIsInstance(RevItBlocked(), Reversible) + self.assertNotIsSubclass(RevRevBlocked, Reversible) + self.assertNotIsInstance(RevRevBlocked(), Reversible) def test_Collection(self): # Check some non-collections non_collections = [None, 42, 3.14, 1j, lambda x: 2*x] for x in non_collections: self.assertNotIsInstance(x, Collection) - self.assertFalse(issubclass(type(x), Collection), repr(type(x))) + self.assertNotIsSubclass(type(x), Collection) # Check some non-collection iterables non_col_iterables = [_test_gen(), iter(b''), iter(bytearray()), (x for x in [])] for x in non_col_iterables: self.assertNotIsInstance(x, Collection) - self.assertFalse(issubclass(type(x), Collection), repr(type(x))) + self.assertNotIsSubclass(type(x), Collection) # Check some collections samples = [set(), frozenset(), dict(), bytes(), str(), tuple(), list(), dict().keys(), dict().items(), dict().values()] for x in samples: self.assertIsInstance(x, Collection) - self.assertTrue(issubclass(type(x), Collection), repr(type(x))) + self.assertIsSubclass(type(x), Collection) # Check also Mapping, MutableMapping, etc. - self.assertTrue(issubclass(Sequence, Collection), repr(Sequence)) - self.assertTrue(issubclass(Mapping, Collection), repr(Mapping)) - self.assertTrue(issubclass(MutableMapping, Collection), - repr(MutableMapping)) - self.assertTrue(issubclass(Set, Collection), repr(Set)) - self.assertTrue(issubclass(MutableSet, Collection), repr(MutableSet)) - self.assertTrue(issubclass(Sequence, Collection), repr(MutableSet)) + self.assertIsSubclass(Sequence, Collection) + self.assertIsSubclass(Mapping, Collection) + self.assertIsSubclass(MutableMapping, Collection) + self.assertIsSubclass(Set, Collection) + self.assertIsSubclass(MutableSet, Collection) + self.assertIsSubclass(Sequence, Collection) # Check direct subclassing class Col(Collection): def __iter__(self): @@ -1090,13 +1089,13 @@ def __contains__(self, item): return False class DerCol(Col): pass self.assertEqual(list(iter(Col())), []) - self.assertFalse(issubclass(list, Col)) - self.assertFalse(issubclass(set, Col)) - self.assertFalse(issubclass(float, Col)) + self.assertNotIsSubclass(list, Col) + self.assertNotIsSubclass(set, Col) + self.assertNotIsSubclass(float, Col) self.assertEqual(list(iter(DerCol())), []) - self.assertFalse(issubclass(list, DerCol)) - self.assertFalse(issubclass(set, DerCol)) - self.assertFalse(issubclass(float, DerCol)) + self.assertNotIsSubclass(list, DerCol) + self.assertNotIsSubclass(set, DerCol) + self.assertNotIsSubclass(float, DerCol) self.validate_abstract_methods(Collection, '__len__', '__iter__', '__contains__') # Check sized container non-iterable (which is not Collection) etc. @@ -1109,12 +1108,12 @@ def __contains__(self, item): return False class ColNoCont: def __iter__(self): return iter([]) def __len__(self): return 0 - self.assertFalse(issubclass(ColNoIter, Collection)) - self.assertFalse(isinstance(ColNoIter(), Collection)) - self.assertFalse(issubclass(ColNoSize, Collection)) - self.assertFalse(isinstance(ColNoSize(), Collection)) - self.assertFalse(issubclass(ColNoCont, Collection)) - self.assertFalse(isinstance(ColNoCont(), Collection)) + self.assertNotIsSubclass(ColNoIter, Collection) + self.assertNotIsInstance(ColNoIter(), Collection) + self.assertNotIsSubclass(ColNoSize, Collection) + self.assertNotIsInstance(ColNoSize(), Collection) + self.assertNotIsSubclass(ColNoCont, Collection) + self.assertNotIsInstance(ColNoCont(), Collection) # Check None blocking class SizeBlock: def __iter__(self): return iter([]) @@ -1124,10 +1123,10 @@ class IterBlock: def __len__(self): return 0 def __contains__(self): return True __iter__ = None - self.assertFalse(issubclass(SizeBlock, Collection)) - self.assertFalse(isinstance(SizeBlock(), Collection)) - self.assertFalse(issubclass(IterBlock, Collection)) - self.assertFalse(isinstance(IterBlock(), Collection)) + self.assertNotIsSubclass(SizeBlock, Collection) + self.assertNotIsInstance(SizeBlock(), Collection) + self.assertNotIsSubclass(IterBlock, Collection) + self.assertNotIsInstance(IterBlock(), Collection) # Check None blocking in subclass class ColImpl: def __iter__(self): @@ -1138,15 +1137,15 @@ def __contains__(self, item): return False class NonCol(ColImpl): __contains__ = None - self.assertFalse(issubclass(NonCol, Collection)) - self.assertFalse(isinstance(NonCol(), Collection)) + self.assertNotIsSubclass(NonCol, Collection) + self.assertNotIsInstance(NonCol(), Collection) def test_Iterator(self): non_samples = [None, 42, 3.14, 1j, b"", "", (), [], {}, set()] for x in non_samples: self.assertNotIsInstance(x, Iterator) - self.assertFalse(issubclass(type(x), Iterator), repr(type(x))) + self.assertNotIsSubclass(type(x), Iterator) samples = [iter(bytes()), iter(str()), iter(tuple()), iter(list()), iter(dict()), iter(set()), iter(frozenset()), @@ -1157,7 +1156,7 @@ def test_Iterator(self): ] for x in samples: self.assertIsInstance(x, Iterator) - self.assertTrue(issubclass(type(x), Iterator), repr(type(x))) + self.assertIsSubclass(type(x), Iterator) self.validate_abstract_methods(Iterator, '__next__', '__iter__') # Issue 10565 @@ -1190,7 +1189,7 @@ def throw(self, typ, val=None, tb=None): pass iter(()), iter([]), NonGen1(), NonGen2(), NonGen3()] for x in non_samples: self.assertNotIsInstance(x, Generator) - self.assertFalse(issubclass(type(x), Generator), repr(type(x))) + self.assertNotIsSubclass(type(x), Generator) class Gen: def __iter__(self): return self @@ -1212,7 +1211,7 @@ def gen(): for x in samples: self.assertIsInstance(x, Iterator) self.assertIsInstance(x, Generator) - self.assertTrue(issubclass(type(x), Generator), repr(type(x))) + self.assertIsSubclass(type(x), Generator) self.validate_abstract_methods(Generator, 'send', 'throw') # mixin tests @@ -1261,7 +1260,7 @@ def athrow(self, typ, val=None, tb=None): pass iter(()), iter([]), NonAGen1(), NonAGen2(), NonAGen3()] for x in non_samples: self.assertNotIsInstance(x, AsyncGenerator) - self.assertFalse(issubclass(type(x), AsyncGenerator), repr(type(x))) + self.assertNotIsSubclass(type(x), AsyncGenerator) class Gen: def __aiter__(self): return self @@ -1283,7 +1282,7 @@ async def gen(): for x in samples: self.assertIsInstance(x, AsyncIterator) self.assertIsInstance(x, AsyncGenerator) - self.assertTrue(issubclass(type(x), AsyncGenerator), repr(type(x))) + self.assertIsSubclass(type(x), AsyncGenerator) self.validate_abstract_methods(AsyncGenerator, 'asend', 'athrow') def run_async(coro): @@ -1326,14 +1325,14 @@ def test_Sized(self): ] for x in non_samples: self.assertNotIsInstance(x, Sized) - self.assertFalse(issubclass(type(x), Sized), repr(type(x))) + self.assertNotIsSubclass(type(x), Sized) samples = [bytes(), str(), tuple(), list(), set(), frozenset(), dict(), dict().keys(), dict().items(), dict().values(), ] for x in samples: self.assertIsInstance(x, Sized) - self.assertTrue(issubclass(type(x), Sized), repr(type(x))) + self.assertIsSubclass(type(x), Sized) self.validate_abstract_methods(Sized, '__len__') self.validate_isinstance(Sized, '__len__') @@ -1344,14 +1343,14 @@ def test_Container(self): ] for x in non_samples: self.assertNotIsInstance(x, Container) - self.assertFalse(issubclass(type(x), Container), repr(type(x))) + self.assertNotIsSubclass(type(x), Container) samples = [bytes(), str(), tuple(), list(), set(), frozenset(), dict(), dict().keys(), dict().items(), ] for x in samples: self.assertIsInstance(x, Container) - self.assertTrue(issubclass(type(x), Container), repr(type(x))) + self.assertIsSubclass(type(x), Container) self.validate_abstract_methods(Container, '__contains__') self.validate_isinstance(Container, '__contains__') @@ -1363,7 +1362,7 @@ def test_Callable(self): ] for x in non_samples: self.assertNotIsInstance(x, Callable) - self.assertFalse(issubclass(type(x), Callable), repr(type(x))) + self.assertNotIsSubclass(type(x), Callable) samples = [lambda: None, type, int, object, len, @@ -1371,7 +1370,7 @@ def test_Callable(self): ] for x in samples: self.assertIsInstance(x, Callable) - self.assertTrue(issubclass(type(x), Callable), repr(type(x))) + self.assertIsSubclass(type(x), Callable) self.validate_abstract_methods(Callable, '__call__') self.validate_isinstance(Callable, '__call__') @@ -1379,16 +1378,16 @@ def test_direct_subclassing(self): for B in Hashable, Iterable, Iterator, Reversible, Sized, Container, Callable: class C(B): pass - self.assertTrue(issubclass(C, B)) - self.assertFalse(issubclass(int, C)) + self.assertIsSubclass(C, B) + self.assertNotIsSubclass(int, C) def test_registration(self): for B in Hashable, Iterable, Iterator, Reversible, Sized, Container, Callable: class C: __hash__ = None # Make sure it isn't hashable by default - self.assertFalse(issubclass(C, B), B.__name__) + self.assertNotIsSubclass(C, B) B.register(C) - self.assertTrue(issubclass(C, B)) + self.assertIsSubclass(C, B) class WithSet(MutableSet): @@ -1419,7 +1418,7 @@ class TestCollectionABCs(ABCTestCase): def test_Set(self): for sample in [set, frozenset]: self.assertIsInstance(sample(), Set) - self.assertTrue(issubclass(sample, Set)) + self.assertIsSubclass(sample, Set) self.validate_abstract_methods(Set, '__contains__', '__iter__', '__len__') class MySet(Set): def __contains__(self, x): @@ -1500,9 +1499,9 @@ def __len__(self): def test_MutableSet(self): self.assertIsInstance(set(), MutableSet) - self.assertTrue(issubclass(set, MutableSet)) + self.assertIsSubclass(set, MutableSet) self.assertNotIsInstance(frozenset(), MutableSet) - self.assertFalse(issubclass(frozenset, MutableSet)) + self.assertNotIsSubclass(frozenset, MutableSet) self.validate_abstract_methods(MutableSet, '__contains__', '__iter__', '__len__', 'add', 'discard') @@ -1841,7 +1840,7 @@ def test_Set_hash_matches_frozenset(self): def test_Mapping(self): for sample in [dict]: self.assertIsInstance(sample(), Mapping) - self.assertTrue(issubclass(sample, Mapping)) + self.assertIsSubclass(sample, Mapping) self.validate_abstract_methods(Mapping, '__contains__', '__iter__', '__len__', '__getitem__') class MyMapping(Mapping): @@ -1857,7 +1856,7 @@ def __iter__(self): def test_MutableMapping(self): for sample in [dict]: self.assertIsInstance(sample(), MutableMapping) - self.assertTrue(issubclass(sample, MutableMapping)) + self.assertIsSubclass(sample, MutableMapping) self.validate_abstract_methods(MutableMapping, '__contains__', '__iter__', '__len__', '__getitem__', '__setitem__', '__delitem__') @@ -1891,12 +1890,12 @@ def test_MutableMapping_subclass(self): def test_Sequence(self): for sample in [tuple, list, bytes, str]: self.assertIsInstance(sample(), Sequence) - self.assertTrue(issubclass(sample, Sequence)) + self.assertIsSubclass(sample, Sequence) self.assertIsInstance(range(10), Sequence) - self.assertTrue(issubclass(range, Sequence)) + self.assertIsSubclass(range, Sequence) self.assertIsInstance(memoryview(b""), Sequence) - self.assertTrue(issubclass(memoryview, Sequence)) - self.assertTrue(issubclass(str, Sequence)) + self.assertIsSubclass(memoryview, Sequence) + self.assertIsSubclass(str, Sequence) self.validate_abstract_methods(Sequence, '__contains__', '__iter__', '__len__', '__getitem__') @@ -1935,46 +1934,24 @@ def assert_index_same(seq1, seq2, index_args): assert_index_same( nativeseq, seqseq, (letter, start, stop)) - def test_ByteString(self): - for sample in [bytes, bytearray]: - with self.assertWarns(DeprecationWarning): - self.assertIsInstance(sample(), ByteString) - self.assertTrue(issubclass(sample, ByteString)) - for sample in [str, list, tuple]: - with self.assertWarns(DeprecationWarning): - self.assertNotIsInstance(sample(), ByteString) - self.assertFalse(issubclass(sample, ByteString)) - with self.assertWarns(DeprecationWarning): - self.assertNotIsInstance(memoryview(b""), ByteString) - self.assertFalse(issubclass(memoryview, ByteString)) - with self.assertWarns(DeprecationWarning): - self.validate_abstract_methods(ByteString, '__getitem__', '__len__') - - with self.assertWarns(DeprecationWarning): - class X(ByteString): pass - - with self.assertWarns(DeprecationWarning): - # No metaclass conflict - class Z(ByteString, Awaitable): pass - def test_Buffer(self): for sample in [bytes, bytearray, memoryview]: self.assertIsInstance(sample(b"x"), Buffer) - self.assertTrue(issubclass(sample, Buffer)) + self.assertIsSubclass(sample, Buffer) for sample in [str, list, tuple]: self.assertNotIsInstance(sample(), Buffer) - self.assertFalse(issubclass(sample, Buffer)) + self.assertNotIsSubclass(sample, Buffer) self.validate_abstract_methods(Buffer, '__buffer__') def test_MutableSequence(self): for sample in [tuple, str, bytes]: self.assertNotIsInstance(sample(), MutableSequence) - self.assertFalse(issubclass(sample, MutableSequence)) + self.assertNotIsSubclass(sample, MutableSequence) for sample in [list, bytearray, deque]: self.assertIsInstance(sample(), MutableSequence) - self.assertTrue(issubclass(sample, MutableSequence)) - self.assertTrue(issubclass(array.array, MutableSequence)) - self.assertFalse(issubclass(str, MutableSequence)) + self.assertIsSubclass(sample, MutableSequence) + self.assertIsSubclass(array.array, MutableSequence) + self.assertNotIsSubclass(str, MutableSequence) self.validate_abstract_methods(MutableSequence, '__contains__', '__iter__', '__len__', '__getitem__', '__setitem__', '__delitem__', 'insert') @@ -2065,8 +2042,8 @@ def test_basics(self): self.assertEqual(c, Counter(a=3, b=2, c=1)) self.assertIsInstance(c, dict) self.assertIsInstance(c, Mapping) - self.assertTrue(issubclass(Counter, dict)) - self.assertTrue(issubclass(Counter, Mapping)) + self.assertIsSubclass(Counter, dict) + self.assertIsSubclass(Counter, Mapping) self.assertEqual(len(c), 3) self.assertEqual(sum(c.values()), 6) self.assertEqual(list(c.values()), [3, 2, 1]) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index d3e69bfedccd07..b5cf2ad18fe60b 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1,7 +1,10 @@ import contextlib import dis import io +import itertools +import marshal import math +import opcode import os import unittest import sys @@ -11,9 +14,14 @@ import types import textwrap import warnings +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None + from test import support -from test.support import (script_helper, requires_debug_ranges, - requires_specialization, Py_C_RECURSION_LIMIT) +from test.support import (script_helper, requires_debug_ranges, run_code, + requires_specialization, get_c_recursion_limit) from test.support.bytecode_helper import instructions_with_positions from test.support.os_helper import FakePath @@ -113,8 +121,9 @@ def __getitem__(self, key): self.assertEqual(d['z'], 12) @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") + @support.skip_emscripten_stack_overflow() def test_extended_arg(self): - repeat = int(Py_C_RECURSION_LIMIT * 0.9) + repeat = int(get_c_recursion_limit() * 0.9) longexpr = 'x = x or ' + '-x' * repeat g = {} code = textwrap.dedent(''' @@ -334,6 +343,10 @@ def test_lambda_doc(self): l = lambda: "foo" self.assertIsNone(l.__doc__) + def test_lambda_consts(self): + l = lambda: "this is the only const" + self.assertEqual(l.__code__.co_consts, ("this is the only const",)) + def test_encoding(self): code = b'# -*- coding: badencoding -*-\npass\n' self.assertRaises(SyntaxError, compile, code, 'tmp', 'exec') @@ -438,8 +451,8 @@ class A: def f(): __mangled = 1 __not_mangled__ = 2 - import __mangled_mod - import __package__.module + import __mangled_mod # noqa: F401 + import __package__.module # noqa: F401 self.assertIn("_A__mangled", A.f.__code__.co_varnames) self.assertIn("__not_mangled__", A.f.__code__.co_varnames) @@ -472,6 +485,19 @@ def test_dead_code_with_except_handler_compiles(self): x = 2 """), '', 'exec') + def test_try_except_in_while_with_chained_condition_compiles(self): + # see gh-124871 + compile(textwrap.dedent(""" + name_1, name_2, name_3 = 1, 2, 3 + while name_3 <= name_2 > name_1: + try: + raise + except: + pass + finally: + pass + """), '', 'exec') + def test_compile_invalid_namedexpr(self): # gh-109351 m = ast.Module( @@ -499,6 +525,58 @@ def test_compile_invalid_namedexpr(self): with self.assertRaisesRegex(TypeError, "NamedExpr target must be a Name"): compile(ast.fix_missing_locations(m), "", "exec") + def test_compile_redundant_jumps_and_nops_after_moving_cold_blocks(self): + # See gh-120367 + code=textwrap.dedent(""" + try: + pass + except: + pass + else: + match name_2: + case b'': + pass + finally: + something + """) + + tree = ast.parse(code) + + # make all instruction locations the same to create redundancies + for node in ast.walk(tree): + if hasattr(node,"lineno"): + del node.lineno + del node.end_lineno + del node.col_offset + del node.end_col_offset + + compile(ast.fix_missing_locations(tree), "", "exec") + + def test_compile_redundant_jump_after_convert_pseudo_ops(self): + # See gh-120367 + code=textwrap.dedent(""" + if name_2: + pass + else: + try: + pass + except: + pass + ~name_5 + """) + + tree = ast.parse(code) + + # make all instruction locations the same to create redundancies + for node in ast.walk(tree): + if hasattr(node,"lineno"): + del node.lineno + del node.end_lineno + del node.col_offset + del node.end_col_offset + + compile(ast.fix_missing_locations(tree), "", "exec") + def test_compile_ast(self): fname = __file__ if fname.lower().endswith('pyc'): @@ -632,11 +710,13 @@ def test_yet_more_evil_still_undecodable(self): @support.cpython_only @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") + @support.skip_emscripten_stack_overflow() def test_compiler_recursion_limit(self): # Expected limit is Py_C_RECURSION_LIMIT - fail_depth = Py_C_RECURSION_LIMIT + 1 - crash_depth = Py_C_RECURSION_LIMIT * 100 - success_depth = int(Py_C_RECURSION_LIMIT * 0.8) + limit = get_c_recursion_limit() + fail_depth = limit + 1 + crash_depth = limit * 100 + success_depth = int(limit * 0.8) def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth @@ -702,7 +782,6 @@ def check_same_constant(const): self.assertEqual(repr(f1()), repr(const)) check_same_constant(None) - check_same_constant(0) check_same_constant(0.0) check_same_constant(b'abc') check_same_constant('abc') @@ -717,10 +796,10 @@ def check_same_constant(const): # Merge constants in tuple or frozenset f1, f2 = lambda: "not a name", lambda: ("not a name",) f3 = lambda x: x in {("not a name",)} - self.assertIs(f1.__code__.co_consts[1], - f2.__code__.co_consts[1][0]) - self.assertIs(next(iter(f3.__code__.co_consts[1])), - f2.__code__.co_consts[1]) + self.assertIs(f1.__code__.co_consts[0], + f2.__code__.co_consts[0][0]) + self.assertIs(next(iter(f3.__code__.co_consts[0])), + f2.__code__.co_consts[0]) # {0} is converted to a constant frozenset({0}) by the peephole # optimizer @@ -762,7 +841,7 @@ def f(): return "unused" self.assertEqual(f.__code__.co_consts, - (None, "used")) + (True, "used")) @support.cpython_only def test_remove_unused_consts_extended_args(self): @@ -780,9 +859,9 @@ def test_remove_unused_consts_extended_args(self): eval(compile(code, "file.py", "exec"), g) exec(code, g) f = g['f'] - expected = tuple([None, '', 1] + [f't{i}' for i in range(N)]) + expected = tuple([''] + [f't{i}' for i in range(N)]) self.assertEqual(f.__code__.co_consts, expected) - expected = "".join(expected[3:]) + expected = "".join(expected[1:]) self.assertEqual(expected, f()) # Stripping unused constants is not a strict requirement for the @@ -794,7 +873,7 @@ def test_strip_unused_None(self): def f1(): "docstring" return 42 - self.assertEqual(f1.__code__.co_consts, (f1.__doc__, 42)) + self.assertEqual(f1.__code__.co_consts, (f1.__doc__,)) # This is a regression test for a CPython specific peephole optimizer # implementation bug present in a few releases. It's assertion verifies @@ -811,9 +890,67 @@ def unused_code_at_end(): # RETURN_VALUE opcode. This does not always crash an interpreter. # When you build with the clang memory sanitizer it reliably aborts. self.assertEqual( - 'RETURN_CONST', + 'RETURN_VALUE', list(dis.get_instructions(unused_code_at_end))[-1].opname) + @support.cpython_only + def test_docstring(self): + src = textwrap.dedent(""" + def with_docstring(): + "docstring" + + def two_strings(): + "docstring" + "not docstring" + + def with_fstring(): + f"not docstring" + + def with_const_expression(): + "also" + " not docstring" + + def multiple_const_strings(): + "not docstring " * 3 + """) + + for opt in [0, 1, 2]: + with self.subTest(opt=opt): + code = compile(src, "", "exec", optimize=opt) + ns = {} + exec(code, ns) + + if opt < 2: + self.assertEqual(ns['with_docstring'].__doc__, "docstring") + self.assertEqual(ns['two_strings'].__doc__, "docstring") + else: + self.assertIsNone(ns['with_docstring'].__doc__) + self.assertIsNone(ns['two_strings'].__doc__) + self.assertIsNone(ns['with_fstring'].__doc__) + self.assertIsNone(ns['with_const_expression'].__doc__) + self.assertIsNone(ns['multiple_const_strings'].__doc__) + + @support.cpython_only + def test_docstring_interactive_mode(self): + srcs = [ + """def with_docstring(): + "docstring" + """, + """class with_docstring: + "docstring" + """, + ] + + for opt in [0, 1, 2]: + for src in srcs: + with self.subTest(opt=opt, src=src): + code = compile(textwrap.dedent(src), "", "single", optimize=opt) + ns = {} + exec(code, ns) + if opt < 2: + self.assertEqual(ns['with_docstring'].__doc__, "docstring") + else: + self.assertIsNone(ns['with_docstring'].__doc__) + @support.cpython_only def test_docstring_omitted(self): # See gh-115347 @@ -831,12 +968,13 @@ class C: return h """) for opt in [-1, 0, 1, 2]: - with self.subTest(opt=opt): - code = compile(src, "", "exec", optimize=opt) - output = io.StringIO() - with contextlib.redirect_stdout(output): - dis.dis(code) - self.assertNotIn('NOP' , output.getvalue()) + for mode in ["exec", "single"]: + with self.subTest(opt=opt, mode=mode): + code = compile(src, "", mode, optimize=opt) + output = io.StringIO() + with contextlib.redirect_stdout(output): + dis.dis(code) + self.assertNotIn('NOP', output.getvalue()) def test_dont_merge_constants(self): # Issue #25843: compile() must not merge constants which are equal @@ -854,7 +992,6 @@ def check_different_constants(const1, const2): self.assertEqual(repr(f1()), repr(const1)) self.assertEqual(repr(f2()), repr(const2)) - check_different_constants(0, 0.0) check_different_constants(+0.0, -0.0) check_different_constants((0,), (0.0,)) check_different_constants('a', b'a') @@ -917,8 +1054,8 @@ def unused_block_while_else(): for func in funcs: opcodes = list(dis.get_instructions(func)) - self.assertLessEqual(len(opcodes), 3) - self.assertEqual('RETURN_CONST', opcodes[-1].opname) + self.assertLessEqual(len(opcodes), 4) + self.assertEqual('RETURN_VALUE', opcodes[-1].opname) self.assertEqual(None, opcodes[-1].argval) def test_false_while_loop(self): @@ -935,8 +1072,8 @@ def continue_in_while(): # Check that we did not raise but we also don't generate bytecode for func in funcs: opcodes = list(dis.get_instructions(func)) - self.assertEqual(2, len(opcodes)) - self.assertEqual('RETURN_CONST', opcodes[1].opname) + self.assertEqual(3, len(opcodes)) + self.assertEqual('RETURN_VALUE', opcodes[-1].opname) self.assertEqual(None, opcodes[1].argval) def test_consts_in_conditionals(self): @@ -1116,9 +1253,9 @@ def return_genexp(): x in y) - genexp_lines = [0, 2, 0] + genexp_lines = [0, 4, 2, 0, 4] - genexp_code = return_genexp.__code__.co_consts[1] + genexp_code = return_genexp.__code__.co_consts[0] code_lines = self.get_code_lines(genexp_code) self.assertEqual(genexp_lines, code_lines) @@ -1258,30 +1395,92 @@ def check_op_count(func, op, expected): actual += 1 self.assertEqual(actual, expected) + def check_consts(func, typ, expected): + expected = set([repr(x) for x in expected]) + all_consts = set() + consts = func.__code__.co_consts + for instr in dis.Bytecode(func): + if instr.opname == "LOAD_CONST" and isinstance(consts[instr.oparg], typ): + all_consts.add(repr(consts[instr.oparg])) + self.assertEqual(all_consts, expected) + def load(): return x[a:b] + x [a:] + x[:b] + x[:] + check_op_count(load, "BINARY_SLICE", 3) + check_op_count(load, "BUILD_SLICE", 0) + check_consts(load, slice, [slice(None, None, None)]) + check_op_count(load, "BINARY_SUBSCR", 1) + def store(): x[a:b] = y x [a:] = y x[:b] = y x[:] = y + check_op_count(store, "STORE_SLICE", 3) + check_op_count(store, "BUILD_SLICE", 0) + check_op_count(store, "STORE_SUBSCR", 1) + check_consts(store, slice, [slice(None, None, None)]) + def long_slice(): return x[a:b:c] + check_op_count(long_slice, "BUILD_SLICE", 1) + check_op_count(long_slice, "BINARY_SLICE", 0) + check_consts(long_slice, slice, []) + check_op_count(long_slice, "BINARY_SUBSCR", 1) + def aug(): x[a:b] += y - check_op_count(load, "BINARY_SLICE", 4) - check_op_count(load, "BUILD_SLICE", 0) - check_op_count(store, "STORE_SLICE", 4) - check_op_count(store, "BUILD_SLICE", 0) - check_op_count(long_slice, "BUILD_SLICE", 1) - check_op_count(long_slice, "BINARY_SLICE", 0) check_op_count(aug, "BINARY_SLICE", 1) check_op_count(aug, "STORE_SLICE", 1) check_op_count(aug, "BUILD_SLICE", 0) + check_op_count(aug, "BINARY_SUBSCR", 0) + check_op_count(aug, "STORE_SUBSCR", 0) + check_consts(aug, slice, []) + + def aug_const(): + x[1:2] += y + + check_op_count(aug_const, "BINARY_SLICE", 0) + check_op_count(aug_const, "STORE_SLICE", 0) + check_op_count(aug_const, "BINARY_SUBSCR", 1) + check_op_count(aug_const, "STORE_SUBSCR", 1) + check_consts(aug_const, slice, [slice(1, 2)]) + + def compound_const_slice(): + x[1:2:3, 4:5:6] = y + + check_op_count(compound_const_slice, "BINARY_SLICE", 0) + check_op_count(compound_const_slice, "BUILD_SLICE", 0) + check_op_count(compound_const_slice, "STORE_SLICE", 0) + check_op_count(compound_const_slice, "STORE_SUBSCR", 1) + check_consts(compound_const_slice, slice, []) + check_consts(compound_const_slice, tuple, [(slice(1, 2, 3), slice(4, 5, 6))]) + + def mutable_slice(): + x[[]:] = y + + check_consts(mutable_slice, slice, {}) + + def different_but_equal(): + x[:0] = y + x[:0.0] = y + x[:False] = y + x[:None] = y + + check_consts( + different_but_equal, + slice, + [ + slice(None, 0, None), + slice(None, 0.0, None), + slice(None, False, None), + slice(None, None, None) + ] + ) def test_compare_positions(self): for opname_prefix, op in [ @@ -1405,6 +1604,55 @@ def f(): for kw in ("except", "except*"): exec(code % kw, g, l); + def test_regression_gh_120225(self): + async def name_4(): + match b'': + case True: + pass + case name_5 if f'e': + {name_3: name_4 async for name_2 in name_5} + case []: + pass + [[]] + +class TestBooleanExpression(unittest.TestCase): + class Value: + def __init__(self): + self.called = 0 + + def __bool__(self): + self.called += 1 + return self.value + + class Yes(Value): + value = True + + class No(Value): + value = False + + def test_short_circuit_and(self): + v = [self.Yes(), self.No(), self.Yes()] + res = v[0] and v[1] and v[0] + self.assertIs(res, v[1]) + self.assertEqual([e.called for e in v], [1, 1, 0]) + + def test_short_circuit_or(self): + v = [self.No(), self.Yes(), self.No()] + res = v[0] or v[1] or v[0] + self.assertIs(res, v[1]) + self.assertEqual([e.called for e in v], [1, 1, 0]) + + def test_compound(self): + # See gh-124285 + v = [self.No(), self.Yes(), self.Yes(), self.Yes()] + res = v[0] and v[1] or v[2] or v[3] + self.assertIs(res, v[2]) + self.assertEqual([e.called for e in v], [1, 0, 1, 0]) + + v = [self.No(), self.No(), self.Yes(), self.Yes(), self.No()] + res = v[0] or v[1] and v[2] or v[3] or v[4] + self.assertIs(res, v[3]) + self.assertEqual([e.called for e in v], [1, 1, 0, 1, 0]) @requires_debug_ranges() class TestSourcePositions(unittest.TestCase): @@ -1534,11 +1782,11 @@ def test_multiline_assert(self): ccc == 1000000), "error msg" """) compiled_code, _ = self.check_positions_against_ast(snippet) - self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_ASSERTION_ERROR', + self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_COMMON_CONSTANT', line=1, end_line=3, column=0, end_column=36, occurrence=1) # The "error msg": self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_CONST', - line=3, end_line=3, column=25, end_column=36, occurrence=4) + line=3, end_line=3, column=25, end_column=36, occurrence=2) self.assertOpcodeSourcePositionIs(compiled_code, 'CALL', line=1, end_line=3, column=0, end_column=36, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'RAISE_VARARGS', @@ -1560,8 +1808,8 @@ def test_multiline_generator_expression(self): line=1, end_line=2, column=1, end_column=8, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', line=1, end_line=2, column=1, end_column=8, occurrence=1) - self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST', - line=1, end_line=6, column=0, end_column=32, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=4, end_line=4, column=7, end_column=14, occurrence=1) def test_multiline_async_generator_expression(self): snippet = textwrap.dedent("""\ @@ -1577,7 +1825,7 @@ def test_multiline_async_generator_expression(self): self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'YIELD_VALUE', line=1, end_line=2, column=1, end_column=8, occurrence=2) - self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST', + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', line=1, end_line=6, column=0, end_column=32, occurrence=1) def test_multiline_list_comprehension(self): @@ -1615,7 +1863,7 @@ async def f(): line=2, end_line=3, column=5, end_column=12, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', line=2, end_line=3, column=5, end_column=12, occurrence=1) - self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST', + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', line=2, end_line=7, column=4, end_column=36, occurrence=1) def test_multiline_set_comprehension(self): @@ -1653,7 +1901,7 @@ async def f(): line=2, end_line=3, column=5, end_column=12, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', line=2, end_line=3, column=5, end_column=12, occurrence=1) - self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST', + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', line=2, end_line=7, column=4, end_column=36, occurrence=1) def test_multiline_dict_comprehension(self): @@ -1691,7 +1939,7 @@ async def f(): line=2, end_line=3, column=5, end_column=11, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', line=2, end_line=3, column=5, end_column=11, occurrence=1) - self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST', + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', line=2, end_line=7, column=4, end_column=36, occurrence=1) def test_matchcase_sequence(self): @@ -1954,11 +2202,135 @@ def test_column_offset_deduplication(self): def test_load_super_attr(self): source = "class C:\n def __init__(self):\n super().__init__()" - code = compile(source, "", "exec").co_consts[0].co_consts[1] + for const in compile(source, "", "exec").co_consts[0].co_consts: + if isinstance(const, types.CodeType): + code = const + break self.assertOpcodeSourcePositionIs( code, "LOAD_GLOBAL", line=3, end_line=3, column=4, end_column=9 ) + def test_lambda_return_position(self): + snippets = [ + "f = lambda: x", + "f = lambda: 42", + "f = lambda: 1 + 2", + "f = lambda: a + b", + ] + for snippet in snippets: + with self.subTest(snippet=snippet): + lamb = run_code(snippet)["f"] + positions = lamb.__code__.co_positions() + # assert that all positions are within the lambda + for i, pos in enumerate(positions): + with self.subTest(i=i, pos=pos): + start_line, end_line, start_col, end_col = pos + if i == 0 and start_col == end_col == 0: + # ignore the RESUME in the beginning + continue + self.assertEqual(start_line, 1) + self.assertEqual(end_line, 1) + code_start = snippet.find(":") + 2 + code_end = len(snippet) + self.assertGreaterEqual(start_col, code_start) + self.assertLessEqual(end_col, code_end) + self.assertGreaterEqual(end_col, start_col) + self.assertLessEqual(end_col, code_end) + + def test_return_in_with_positions(self): + # See gh-98442 + def f(): + with xyz: + 1 + 2 + 3 + 4 + return R + + # All instructions should have locations on a single line + for instr in dis.get_instructions(f): + start_line, end_line, _, _ = instr.positions + self.assertEqual(start_line, end_line) + + # Expect four `LOAD_CONST None` instructions: + # three for the no-exception __exit__ call, and one for the return. + # They should all have the locations of the context manager ('xyz'). + + load_none = [instr for instr in dis.get_instructions(f) if + instr.opname == 'LOAD_CONST' and instr.argval is None] + return_value = [instr for instr in dis.get_instructions(f) if + instr.opname == 'RETURN_VALUE'] + + self.assertEqual(len(load_none), 4) + self.assertEqual(len(return_value), 2) + for instr in load_none + return_value: + start_line, end_line, start_col, end_col = instr.positions + self.assertEqual(start_line, f.__code__.co_firstlineno + 1) + self.assertEqual(end_line, f.__code__.co_firstlineno + 1) + self.assertEqual(start_col, 17) + self.assertEqual(end_col, 20) + + +class TestStaticAttributes(unittest.TestCase): + + def test_basic(self): + class C: + def f(self): + self.a = self.b = 42 + # read fields are not included + self.f() + self.arr[3] + + self.assertIsInstance(C.__static_attributes__, tuple) + self.assertEqual(sorted(C.__static_attributes__), ['a', 'b']) + + def test_nested_function(self): + class C: + def f(self): + self.x = 1 + self.y = 2 + self.x = 3 # check deduplication + + def g(self, obj): + self.y = 4 + self.z = 5 + + def h(self, a): + self.u = 6 + self.v = 7 + + obj.self = 8 + + self.assertEqual(sorted(C.__static_attributes__), ['u', 'v', 'x', 'y', 'z']) + + def test_nested_class(self): + class C: + def f(self): + self.x = 42 + self.y = 42 + + class D: + def g(self): + self.y = 42 + self.z = 42 + + self.assertEqual(sorted(C.__static_attributes__), ['x', 'y']) + self.assertEqual(sorted(C.D.__static_attributes__), ['y', 'z']) + + def test_subclass(self): + class C: + def f(self): + self.x = 42 + self.y = 42 + + class D(C): + def g(self): + self.y = 42 + self.z = 42 + + self.assertEqual(sorted(C.__static_attributes__), ['x', 'y']) + self.assertEqual(sorted(D.__static_attributes__), ['y', 'z']) + class TestExpressionStackSize(unittest.TestCase): # These tests check that the computed stack size for a code object @@ -2360,6 +2732,67 @@ def test_return_inside_async_with_block(self): """ self.check_stack_size(snippet, async_=True) +@support.cpython_only +@unittest.skipIf(_testinternalcapi is None, 'need _testinternalcapi module') +class TestInstructionSequence(unittest.TestCase): + def compare_instructions(self, seq, expected): + self.assertEqual([(opcode.opname[i[0]],) + i[1:] for i in seq.get_instructions()], + expected) + + def test_basics(self): + seq = _testinternalcapi.new_instruction_sequence() + + def add_op(seq, opname, oparg, bl, bc=0, el=0, ec=0): + seq.addop(opcode.opmap[opname], oparg, bl, bc, el, el) + + add_op(seq, 'LOAD_CONST', 1, 1) + add_op(seq, 'JUMP', lbl1 := seq.new_label(), 2) + add_op(seq, 'LOAD_CONST', 1, 3) + add_op(seq, 'JUMP', lbl2 := seq.new_label(), 4) + seq.use_label(lbl1) + add_op(seq, 'LOAD_CONST', 2, 4) + seq.use_label(lbl2) + add_op(seq, 'RETURN_VALUE', 0, 3) + + expected = [('LOAD_CONST', 1, 1), + ('JUMP', 4, 2), + ('LOAD_CONST', 1, 3), + ('JUMP', 5, 4), + ('LOAD_CONST', 2, 4), + ('RETURN_VALUE', None, 3), + ] + + self.compare_instructions(seq, [ex + (0,0,0) for ex in expected]) + + def test_nested(self): + seq = _testinternalcapi.new_instruction_sequence() + seq.addop(opcode.opmap['LOAD_CONST'], 1, 1, 0, 0, 0) + nested = _testinternalcapi.new_instruction_sequence() + nested.addop(opcode.opmap['LOAD_CONST'], 2, 2, 0, 0, 0) + + self.compare_instructions(seq, [('LOAD_CONST', 1, 1, 0, 0, 0)]) + self.compare_instructions(nested, [('LOAD_CONST', 2, 2, 0, 0, 0)]) + + seq.add_nested(nested) + self.compare_instructions(seq, [('LOAD_CONST', 1, 1, 0, 0, 0)]) + self.compare_instructions(seq.get_nested()[0], [('LOAD_CONST', 2, 2, 0, 0, 0)]) + + def test_static_attributes_are_sorted(self): + code = ( + 'class T:\n' + ' def __init__(self):\n' + ' self.{V1} = 10\n' + ' self.{V2} = 10\n' + ' def foo(self):\n' + ' self.{V3} = 10\n' + ) + attributes = ("a", "b", "c") + for perm in itertools.permutations(attributes): + var_names = {f'V{i + 1}': name for i, name in enumerate(perm)} + ns = run_code(code.format(**var_names)) + t = ns['T'] + self.assertEqual(t.__static_attributes__, attributes) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 14c2af19e3eb28..a580a240d9f474 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -4,7 +4,6 @@ import importlib.util import io import os -import pathlib import py_compile import shutil import struct @@ -19,7 +18,7 @@ # compileall relies on ProcessPoolExecutor if ProcessPoolExecutor exists # and it can function. from multiprocessing.util import _cleanup_tests as multiprocessing_cleanup_tests - from concurrent.futures import ProcessPoolExecutor + from concurrent.futures import ProcessPoolExecutor # noqa: F401 from concurrent.futures.process import _check_system_limits _check_system_limits() _have_multiprocessing = True @@ -31,6 +30,7 @@ from test.support import script_helper from test.test_py_compile import without_source_date_epoch from test.test_py_compile import SourceDateEpochTestMeta +from test.support.os_helper import FakePath def get_pyc(script, opt): @@ -156,28 +156,28 @@ def test_compile_file_pathlike(self): self.assertFalse(os.path.isfile(self.bc_path)) # we should also test the output with support.captured_stdout() as stdout: - self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path))) + self.assertTrue(compileall.compile_file(FakePath(self.source_path))) self.assertRegex(stdout.getvalue(), r'Compiling ([^WindowsPath|PosixPath].*)') self.assertTrue(os.path.isfile(self.bc_path)) def test_compile_file_pathlike_ddir(self): self.assertFalse(os.path.isfile(self.bc_path)) - self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), - ddir=pathlib.Path('ddir_path'), + self.assertTrue(compileall.compile_file(FakePath(self.source_path), + ddir=FakePath('ddir_path'), quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) def test_compile_file_pathlike_stripdir(self): self.assertFalse(os.path.isfile(self.bc_path)) - self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), - stripdir=pathlib.Path('stripdir_path'), + self.assertTrue(compileall.compile_file(FakePath(self.source_path), + stripdir=FakePath('stripdir_path'), quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) def test_compile_file_pathlike_prependdir(self): self.assertFalse(os.path.isfile(self.bc_path)) - self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), - prependdir=pathlib.Path('prependdir_path'), + self.assertTrue(compileall.compile_file(FakePath(self.source_path), + prependdir=FakePath('prependdir_path'), quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) @@ -228,22 +228,22 @@ def test_optimize(self): def test_compile_dir_pathlike(self): self.assertFalse(os.path.isfile(self.bc_path)) with support.captured_stdout() as stdout: - compileall.compile_dir(pathlib.Path(self.directory)) + compileall.compile_dir(FakePath(self.directory)) line = stdout.getvalue().splitlines()[0] self.assertRegex(line, r'Listing ([^WindowsPath|PosixPath].*)') self.assertTrue(os.path.isfile(self.bc_path)) def test_compile_dir_pathlike_stripdir(self): self.assertFalse(os.path.isfile(self.bc_path)) - self.assertTrue(compileall.compile_dir(pathlib.Path(self.directory), - stripdir=pathlib.Path('stripdir_path'), + self.assertTrue(compileall.compile_dir(FakePath(self.directory), + stripdir=FakePath('stripdir_path'), quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) def test_compile_dir_pathlike_prependdir(self): self.assertFalse(os.path.isfile(self.bc_path)) - self.assertTrue(compileall.compile_dir(pathlib.Path(self.directory), - prependdir=pathlib.Path('prependdir_path'), + self.assertTrue(compileall.compile_dir(FakePath(self.directory), + prependdir=FakePath('prependdir_path'), quiet=2)) self.assertTrue(os.path.isfile(self.bc_path)) @@ -502,19 +502,25 @@ def setUp(self): self.directory = tempfile.mkdtemp() self.source_path = os.path.join(self.directory, '_test.py') with open(self.source_path, 'w', encoding='utf-8') as file: - file.write('# -*- coding: utf-8 -*-\n') - file.write('print u"\u20ac"\n') + # Intentional syntax error: bytes can only contain + # ASCII literal characters. + file.write('b"\u20ac"') def tearDown(self): shutil.rmtree(self.directory) def test_error(self): - try: - orig_stdout = sys.stdout - sys.stdout = io.TextIOWrapper(io.BytesIO(),encoding='ascii') - compileall.compile_dir(self.directory) - finally: - sys.stdout = orig_stdout + buffer = io.TextIOWrapper(io.BytesIO(), encoding='ascii') + with contextlib.redirect_stdout(buffer): + compiled = compileall.compile_dir(self.directory) + self.assertFalse(compiled) # should not be successful + buffer.seek(0) + res = buffer.read() + self.assertIn( + 'SyntaxError: bytes can only contain ASCII literal characters', + res, + ) + self.assertNotIn('UnicodeEncodeError', res) class CommandLineTestsBase: @@ -760,6 +766,7 @@ def test_d_compile_error(self): rc, out, err = self.assertRunNotOK('-q', '-d', 'dinsdale', self.pkgdir) self.assertRegex(out, b'File "dinsdale') + @support.force_not_colorized def test_d_runtime_error(self): bazfn = script_helper.make_script(self.pkgdir, 'baz', 'raise Exception') self.assertRunOK('-q', '-d', 'dinsdale', self.pkgdir) diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index ab9f04dd63af20..c4962e3599986e 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -27,8 +27,8 @@ def complete_metadata(self, metadata, filename="myfile.py"): def insts_to_code_object(self, insts, metadata): metadata = self.complete_metadata(metadata) - insts = self.complete_insts_info(insts) - return self.get_code_object(metadata['filename'], insts, metadata) + seq = self.seq_from_insts(insts) + return self.get_code_object(metadata['filename'], seq, metadata) def assemble_test(self, insts, metadata, expected): co = self.insts_to_code_object(insts, metadata) @@ -71,7 +71,7 @@ def test_simple_expr(self): ('BINARY_OP', 0, 1), # '+' ('LOAD_CONST', 0, 1), # 2 ('BINARY_OP', 11, 1), # '/' - ('RETURN_VALUE', 1), + ('RETURN_VALUE', None, 1), ] expected = {(3, 4) : 3.5, (-100, 200) : 50, (10, 18) : 14} self.assemble_test(insts, metadata, expected) @@ -84,7 +84,7 @@ def inner(): return x return inner() % 2 - inner_code = mod_two.__code__.co_consts[1] + inner_code = mod_two.__code__.co_consts[0] assert isinstance(inner_code, types.CodeType) metadata = { @@ -102,13 +102,13 @@ def inner(): ('LOAD_CLOSURE', 0, 1), ('BUILD_TUPLE', 1, 1), ('LOAD_CONST', 1, 1), - ('MAKE_FUNCTION', 0, 2), + ('MAKE_FUNCTION', None, 2), ('SET_FUNCTION_ATTRIBUTE', 8, 2), - ('PUSH_NULL', 0, 1), + ('PUSH_NULL', None, 1), ('CALL', 0, 2), # (lambda: x)() ('LOAD_CONST', 2, 2), # 2 ('BINARY_OP', 6, 2), # % - ('RETURN_VALUE', 0, 2) + ('RETURN_VALUE', None, 2) ] expected = {(0,): 0, (1,): 1, (2,): 0, (120,): 0, (121,): 1} @@ -125,15 +125,17 @@ def test_exception_table(self): # code for "try: pass\n except: pass" insts = [ ('RESUME', 0), - ('SETUP_FINALLY', 3), - ('RETURN_CONST', 0), - ('SETUP_CLEANUP', 8), - ('PUSH_EXC_INFO', 0), - ('POP_TOP', 0), - ('POP_EXCEPT', 0), - ('RETURN_CONST', 0), + ('SETUP_FINALLY', 4), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), + ('SETUP_CLEANUP', 10), + ('PUSH_EXC_INFO', None), + ('POP_TOP', None), + ('POP_EXCEPT', None), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), ('COPY', 3), - ('POP_EXCEPT', 0), + ('POP_EXCEPT', None), ('RERAISE', 1), ] co = self.insts_to_code_object(insts, metadata) diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index 166294a40c1cb7..cf5e2d901db4de 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -1,4 +1,5 @@ +import textwrap from test.support.bytecode_helper import CodegenTestCase # Tests for the code-generation stage of the compiler. @@ -6,11 +7,19 @@ class IsolatedCodeGenTests(CodegenTestCase): + def assertInstructionsMatch_recursive(self, insts, expected_insts): + expected_nested = [i for i in expected_insts if isinstance(i, list)] + expected_insts = [i for i in expected_insts if not isinstance(i, list)] + self.assertInstructionsMatch(insts, expected_insts) + self.assertEqual(len(insts.get_nested()), len(expected_nested)) + for n_insts, n_expected in zip(insts.get_nested(), expected_nested): + self.assertInstructionsMatch_recursive(n_insts, n_expected) + def codegen_test(self, snippet, expected_insts): import ast a = ast.parse(snippet, "my_file.py", "exec") insts = self.generate_code(a) - self.assertInstructionsMatch(insts, expected_insts) + self.assertInstructionsMatch_recursive(insts, expected_insts) def test_if_expression(self): snippet = "42 if True else 24" @@ -20,13 +29,13 @@ def test_if_expression(self): ('LOAD_CONST', 0, 1), ('TO_BOOL', 0, 1), ('POP_JUMP_IF_FALSE', false_lbl := self.Label(), 1), - ('LOAD_CONST', 1, 1), + ('LOAD_SMALL_INT', 42, 1), ('JUMP_NO_INTERRUPT', exit_lbl := self.Label()), false_lbl, - ('LOAD_CONST', 2, 1), + ('LOAD_SMALL_INT', 24, 1), exit_lbl, ('POP_TOP', None), - ('LOAD_CONST', 3), + ('LOAD_CONST', 1), ('RETURN_VALUE', None), ] self.codegen_test(snippet, expected) @@ -40,6 +49,7 @@ def test_for_loop(self): ('GET_ITER', None, 1), loop_lbl := self.Label(), ('FOR_ITER', exit_lbl := self.Label(), 1), + ('NOP', None, 1, 1), ('STORE_NAME', 1, 1), ('LOAD_NAME', 2, 2), ('PUSH_NULL', None, 2), @@ -49,13 +59,101 @@ def test_for_loop(self): ('JUMP', loop_lbl), exit_lbl, ('END_FOR', None), - ('POP_TOP', None), + ('POP_ITER', None), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), + ] + self.codegen_test(snippet, expected) + + def test_function(self): + snippet = textwrap.dedent(""" + def f(x): + return x + 42 + """) + expected = [ + # Function definition + ('RESUME', 0), + ('LOAD_CONST', 0), + ('MAKE_FUNCTION', None), + ('STORE_NAME', 0), + ('LOAD_CONST', 1), + ('RETURN_VALUE', None), + [ + # Function body + ('RESUME', 0), + ('LOAD_FAST', 0), + ('LOAD_SMALL_INT', 42), + ('BINARY_OP', 0), + ('RETURN_VALUE', None), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), + ] + ] + self.codegen_test(snippet, expected) + + def test_nested_functions(self): + snippet = textwrap.dedent(""" + def f(): + def h(): + return 12 + def g(): + x = 1 + y = 2 + z = 3 + u = 4 + return 42 + """) + expected = [ + # Function definition + ('RESUME', 0), ('LOAD_CONST', 0), + ('MAKE_FUNCTION', None), + ('STORE_NAME', 0), + ('LOAD_CONST', 1), ('RETURN_VALUE', None), + [ + # Function body + ('RESUME', 0), + ('LOAD_CONST', 1), + ('MAKE_FUNCTION', None), + ('STORE_FAST', 0), + ('LOAD_CONST', 2), + ('MAKE_FUNCTION', None), + ('STORE_FAST', 1), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), + [ + ('RESUME', 0), + ('NOP', None), + ('LOAD_SMALL_INT', 12), + ('RETURN_VALUE', None), + ('LOAD_CONST', 1), + ('RETURN_VALUE', None), + ], + [ + ('RESUME', 0), + ('LOAD_SMALL_INT', 1), + ('STORE_FAST', 0), + ('LOAD_SMALL_INT', 2), + ('STORE_FAST', 1), + ('LOAD_SMALL_INT', 3), + ('STORE_FAST', 2), + ('LOAD_SMALL_INT', 4), + ('STORE_FAST', 3), + ('NOP', None), + ('LOAD_SMALL_INT', 42), + ('RETURN_VALUE', None), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), + ], + ], ] self.codegen_test(snippet, expected) def test_syntax_error__return_not_in_function(self): snippet = "return 42" - with self.assertRaisesRegex(SyntaxError, "'return' outside function"): + with self.assertRaisesRegex(SyntaxError, "'return' outside function") as cm: self.codegen_test(snippet, None) + self.assertIsNone(cm.exception.text) + self.assertEqual(cm.exception.offset, 1) + self.assertEqual(cm.exception.end_offset, 10) diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index b057121f285dc7..fd002fb00ac338 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -1,15 +1,17 @@ import unittest import sys from test import support +from test.support.testcase import ComplexesAreIdenticalMixin from test.test_grammar import (VALID_UNDERSCORE_LITERALS, INVALID_UNDERSCORE_LITERALS) from random import random -from math import atan2, isnan, copysign +from math import isnan, copysign import operator INF = float("inf") NAN = float("nan") +DBL_MAX = sys.float_info.max # These tests ensure that complex math does the right thing ZERO_DIVISION = ( @@ -20,7 +22,38 @@ (1, 0+0j), ) -class ComplexTest(unittest.TestCase): +class WithIndex: + def __init__(self, value): + self.value = value + def __index__(self): + return self.value + +class WithFloat: + def __init__(self, value): + self.value = value + def __float__(self): + return self.value + +class ComplexSubclass(complex): + pass + +class OtherComplexSubclass(complex): + pass + +class MyInt: + def __init__(self, value): + self.value = value + + def __int__(self): + return self.value + +class WithComplex: + def __init__(self, value): + self.value = value + def __complex__(self): + return self.value + +class ComplexTest(ComplexesAreIdenticalMixin, unittest.TestCase): def assertAlmostEqual(self, a, b): if isinstance(a, complex): @@ -49,29 +82,6 @@ def assertCloseAbs(self, x, y, eps=1e-9): # check that relative difference < eps self.assertTrue(abs((x-y)/y) < eps) - def assertFloatsAreIdentical(self, x, y): - """assert that floats x and y are identical, in the sense that: - (1) both x and y are nans, or - (2) both x and y are infinities, with the same sign, or - (3) both x and y are zeros, with the same sign, or - (4) x and y are both finite and nonzero, and x == y - - """ - msg = 'floats {!r} and {!r} are not identical' - - if isnan(x) or isnan(y): - if isnan(x) and isnan(y): - return - elif x == y: - if x != 0.0: - return - # both zero; check that signs match - elif copysign(1.0, x) == copysign(1.0, y): - return - else: - msg += ': zeros have different signs' - self.fail(msg.format(x, y)) - def assertClose(self, x, y, eps=1e-9): """Return true iff complexes x and y "are close".""" self.assertCloseAbs(x.real, y.real, eps) @@ -116,6 +126,72 @@ def test_truediv(self): z = complex(0, 0) / complex(denom_real, denom_imag) self.assertTrue(isnan(z.real)) self.assertTrue(isnan(z.imag)) + z = float(0) / complex(denom_real, denom_imag) + self.assertTrue(isnan(z.real)) + self.assertTrue(isnan(z.imag)) + + self.assertComplexesAreIdentical(complex(INF, NAN) / 2, + complex(INF, NAN)) + + self.assertComplexesAreIdentical(complex(INF, 1)/(0.0+1j), + complex(NAN, -INF)) + + # test recover of infs if numerator has infs and denominator is finite + self.assertComplexesAreIdentical(complex(INF, -INF)/(1+0j), + complex(INF, -INF)) + self.assertComplexesAreIdentical(complex(INF, INF)/(0.0+1j), + complex(INF, -INF)) + self.assertComplexesAreIdentical(complex(NAN, INF)/complex(2**1000, 2**-1000), + complex(INF, INF)) + self.assertComplexesAreIdentical(complex(INF, NAN)/complex(2**1000, 2**-1000), + complex(INF, -INF)) + + # test recover of zeros if denominator is infinite + self.assertComplexesAreIdentical((1+1j)/complex(INF, INF), (0.0+0j)) + self.assertComplexesAreIdentical((1+1j)/complex(INF, -INF), (0.0+0j)) + self.assertComplexesAreIdentical((1+1j)/complex(-INF, INF), + complex(0.0, -0.0)) + self.assertComplexesAreIdentical((1+1j)/complex(-INF, -INF), + complex(-0.0, 0)) + self.assertComplexesAreIdentical((INF+1j)/complex(INF, INF), + complex(NAN, NAN)) + self.assertComplexesAreIdentical(complex(1, INF)/complex(INF, INF), + complex(NAN, NAN)) + self.assertComplexesAreIdentical(complex(INF, 1)/complex(1, INF), + complex(NAN, NAN)) + + # mixed types + self.assertEqual((1+1j)/float(2), 0.5+0.5j) + self.assertEqual(float(1)/(1+2j), 0.2-0.4j) + self.assertEqual(float(1)/(-1+2j), -0.2-0.4j) + self.assertEqual(float(1)/(1-2j), 0.2+0.4j) + self.assertEqual(float(1)/(2+1j), 0.4-0.2j) + self.assertEqual(float(1)/(-2+1j), -0.4-0.2j) + self.assertEqual(float(1)/(2-1j), 0.4+0.2j) + + self.assertComplexesAreIdentical(INF/(1+0j), + complex(INF, NAN)) + self.assertComplexesAreIdentical(INF/(0.0+1j), + complex(NAN, -INF)) + self.assertComplexesAreIdentical(INF/complex(2**1000, 2**-1000), + complex(INF, NAN)) + self.assertComplexesAreIdentical(INF/complex(NAN, NAN), + complex(NAN, NAN)) + + self.assertComplexesAreIdentical(float(1)/complex(INF, INF), (0.0-0j)) + self.assertComplexesAreIdentical(float(1)/complex(INF, -INF), (0.0+0j)) + self.assertComplexesAreIdentical(float(1)/complex(-INF, INF), + complex(-0.0, -0.0)) + self.assertComplexesAreIdentical(float(1)/complex(-INF, -INF), + complex(-0.0, 0)) + self.assertComplexesAreIdentical(float(1)/complex(INF, NAN), + complex(0.0, -0.0)) + self.assertComplexesAreIdentical(float(1)/complex(-INF, NAN), + complex(-0.0, -0.0)) + self.assertComplexesAreIdentical(float(1)/complex(NAN, INF), + complex(0.0, -0.0)) + self.assertComplexesAreIdentical(float(INF)/complex(NAN, INF), + complex(NAN, NAN)) def test_truediv_zero_division(self): for a, b in ZERO_DIVISION: @@ -187,6 +263,10 @@ def check(n, deltas, is_equal, imag = 0.0): def test_add(self): self.assertEqual(1j + int(+1), complex(+1, 1)) self.assertEqual(1j + int(-1), complex(-1, 1)) + self.assertComplexesAreIdentical(complex(-0.0, -0.0) + (-0.0), + complex(-0.0, -0.0)) + self.assertComplexesAreIdentical((-0.0) + complex(-0.0, -0.0), + complex(-0.0, -0.0)) self.assertRaises(OverflowError, operator.add, 1j, 10**1000) self.assertRaises(TypeError, operator.add, 1j, None) self.assertRaises(TypeError, operator.add, None, 1j) @@ -194,6 +274,14 @@ def test_add(self): def test_sub(self): self.assertEqual(1j - int(+1), complex(-1, 1)) self.assertEqual(1j - int(-1), complex(1, 1)) + self.assertComplexesAreIdentical(complex(-0.0, -0.0) - 0.0, + complex(-0.0, -0.0)) + self.assertComplexesAreIdentical(-0.0 - complex(0.0, 0.0), + complex(-0.0, -0.0)) + self.assertComplexesAreIdentical(complex(1, 2) - complex(2, 1), + complex(-1, 1)) + self.assertComplexesAreIdentical(complex(2, 1) - complex(1, 2), + complex(1, -1)) self.assertRaises(OverflowError, operator.sub, 1j, 10**1000) self.assertRaises(TypeError, operator.sub, 1j, None) self.assertRaises(TypeError, operator.sub, None, 1j) @@ -201,10 +289,32 @@ def test_sub(self): def test_mul(self): self.assertEqual(1j * int(20), complex(0, 20)) self.assertEqual(1j * int(-1), complex(0, -1)) + for c, r in [(2, complex(INF, 2)), (INF, complex(INF, INF)), + (0, complex(NAN, 0)), (-0.0, complex(NAN, -0.0)), + (NAN, complex(NAN, NAN))]: + with self.subTest(c=c, r=r): + self.assertComplexesAreIdentical(complex(INF, 1) * c, r) + self.assertComplexesAreIdentical(c * complex(INF, 1), r) self.assertRaises(OverflowError, operator.mul, 1j, 10**1000) self.assertRaises(TypeError, operator.mul, 1j, None) self.assertRaises(TypeError, operator.mul, None, 1j) + for z, w, r in [(1e300+1j, complex(INF, INF), complex(NAN, INF)), + (1e300+1j, complex(NAN, INF), complex(-INF, INF)), + (1e300+1j, complex(INF, NAN), complex(INF, INF)), + (complex(INF, 1), complex(NAN, INF), complex(NAN, INF)), + (complex(INF, 1), complex(INF, NAN), complex(INF, NAN)), + (complex(NAN, 1), complex(1, INF), complex(-INF, NAN)), + (complex(1, NAN), complex(1, INF), complex(NAN, INF)), + (complex(1e200, NAN), complex(1e200, NAN), complex(INF, NAN)), + (complex(1e200, NAN), complex(NAN, 1e200), complex(NAN, INF)), + (complex(NAN, 1e200), complex(1e200, NAN), complex(NAN, INF)), + (complex(NAN, 1e200), complex(NAN, 1e200), complex(-INF, NAN)), + (complex(NAN, NAN), complex(NAN, NAN), complex(NAN, NAN))]: + with self.subTest(z=z, w=w, r=r): + self.assertComplexesAreIdentical(z * w, r) + self.assertComplexesAreIdentical(w * z, r) + def test_mod(self): # % is no longer supported on complex numbers with self.assertRaises(TypeError): @@ -246,6 +356,7 @@ def test_pow(self): self.assertAlmostEqual(pow(1j, 200), 1) self.assertRaises(ValueError, pow, 1+1j, 1+1j, 1+1j) self.assertRaises(OverflowError, pow, 1e200+1j, 1e200+1j) + self.assertRaises(OverflowError, pow, 1e200+1j, 5) self.assertRaises(TypeError, pow, 1j, None) self.assertRaises(TypeError, pow, None, 1j) self.assertAlmostEqual(pow(1j, 0.5), 0.7071067811865476+0.7071067811865475j) @@ -301,6 +412,11 @@ def test_pow(self): except OverflowError: pass + # gh-113841: possible undefined division by 0 in _Py_c_pow() + x, y = 9j, 33j**3 + with self.assertRaises(OverflowError): + x**y + def test_pow_with_small_integer_exponents(self): # Check that small integer exponents are handled identically # regardless of their type. @@ -339,137 +455,118 @@ def test_conjugate(self): self.assertClose(complex(5.3, 9.8).conjugate(), 5.3-9.8j) def test_constructor(self): - class NS: - def __init__(self, value): self.value = value - def __complex__(self): return self.value - self.assertEqual(complex(NS(1+10j)), 1+10j) - self.assertRaises(TypeError, complex, NS(None)) - self.assertRaises(TypeError, complex, {}) - self.assertRaises(TypeError, complex, NS(1.5)) - self.assertRaises(TypeError, complex, NS(1)) - self.assertRaises(TypeError, complex, object()) - self.assertRaises(TypeError, complex, NS(4.25+0.5j), object()) - - self.assertAlmostEqual(complex("1+10j"), 1+10j) - self.assertAlmostEqual(complex(10), 10+0j) - self.assertAlmostEqual(complex(10.0), 10+0j) - self.assertAlmostEqual(complex(10), 10+0j) - self.assertAlmostEqual(complex(10+0j), 10+0j) - self.assertAlmostEqual(complex(1,10), 1+10j) - self.assertAlmostEqual(complex(1,10), 1+10j) - self.assertAlmostEqual(complex(1,10.0), 1+10j) - self.assertAlmostEqual(complex(1,10), 1+10j) - self.assertAlmostEqual(complex(1,10), 1+10j) - self.assertAlmostEqual(complex(1,10.0), 1+10j) - self.assertAlmostEqual(complex(1.0,10), 1+10j) - self.assertAlmostEqual(complex(1.0,10), 1+10j) - self.assertAlmostEqual(complex(1.0,10.0), 1+10j) - self.assertAlmostEqual(complex(3.14+0j), 3.14+0j) - self.assertAlmostEqual(complex(3.14), 3.14+0j) - self.assertAlmostEqual(complex(314), 314.0+0j) - self.assertAlmostEqual(complex(314), 314.0+0j) - self.assertAlmostEqual(complex(3.14+0j, 0j), 3.14+0j) - self.assertAlmostEqual(complex(3.14, 0.0), 3.14+0j) - self.assertAlmostEqual(complex(314, 0), 314.0+0j) - self.assertAlmostEqual(complex(314, 0), 314.0+0j) - self.assertAlmostEqual(complex(0j, 3.14j), -3.14+0j) - self.assertAlmostEqual(complex(0.0, 3.14j), -3.14+0j) - self.assertAlmostEqual(complex(0j, 3.14), 3.14j) - self.assertAlmostEqual(complex(0.0, 3.14), 3.14j) - self.assertAlmostEqual(complex("1"), 1+0j) - self.assertAlmostEqual(complex("1j"), 1j) - self.assertAlmostEqual(complex(), 0) - self.assertAlmostEqual(complex("-1"), -1) - self.assertAlmostEqual(complex("+1"), +1) - self.assertAlmostEqual(complex("(1+2j)"), 1+2j) - self.assertAlmostEqual(complex("(1.3+2.2j)"), 1.3+2.2j) - self.assertAlmostEqual(complex("3.14+1J"), 3.14+1j) - self.assertAlmostEqual(complex(" ( +3.14-6J )"), 3.14-6j) - self.assertAlmostEqual(complex(" ( +3.14-J )"), 3.14-1j) - self.assertAlmostEqual(complex(" ( +3.14+j )"), 3.14+1j) - self.assertAlmostEqual(complex("J"), 1j) - self.assertAlmostEqual(complex("( j )"), 1j) - self.assertAlmostEqual(complex("+J"), 1j) - self.assertAlmostEqual(complex("( -j)"), -1j) - self.assertAlmostEqual(complex('1e-500'), 0.0 + 0.0j) - self.assertAlmostEqual(complex('-1e-500j'), 0.0 - 0.0j) - self.assertAlmostEqual(complex('-1e-500+1e-500j'), -0.0 + 0.0j) - self.assertEqual(complex('1-1j'), 1.0 - 1j) - self.assertEqual(complex('1J'), 1j) - - class complex2(complex): pass - self.assertAlmostEqual(complex(complex2(1+1j)), 1+1j) - self.assertAlmostEqual(complex(real=17, imag=23), 17+23j) - self.assertAlmostEqual(complex(real=17+23j), 17+23j) - self.assertAlmostEqual(complex(real=17+23j, imag=23), 17+46j) - self.assertAlmostEqual(complex(real=1+2j, imag=3+4j), -3+5j) + def check(z, x, y): + self.assertIs(type(z), complex) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + + check(complex(), 0.0, 0.0) + check(complex(10), 10.0, 0.0) + check(complex(4.25), 4.25, 0.0) + check(complex(4.25+0j), 4.25, 0.0) + check(complex(4.25+0.5j), 4.25, 0.5) + check(complex(ComplexSubclass(4.25+0.5j)), 4.25, 0.5) + check(complex(WithComplex(4.25+0.5j)), 4.25, 0.5) + + check(complex(1, 10), 1.0, 10.0) + check(complex(1, 10.0), 1.0, 10.0) + check(complex(1, 4.25), 1.0, 4.25) + check(complex(1.0, 10), 1.0, 10.0) + check(complex(4.25, 10), 4.25, 10.0) + check(complex(1.0, 10.0), 1.0, 10.0) + check(complex(4.25, 0.5), 4.25, 0.5) + + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(4.25+0j, 0), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not .*ComplexSubclass"): + check(complex(ComplexSubclass(4.25+0j), 0), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not .*WithComplex"): + check(complex(WithComplex(4.25+0j), 0), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(4.25j, 0), 0.0, 4.25) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(0j, 4.25), 0.0, 4.25) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'imag' must be a real number, not complex"): + check(complex(0, 4.25+0j), 0.0, 4.25) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'imag' must be a real number, not .*ComplexSubclass"): + check(complex(0, ComplexSubclass(4.25+0j)), 0.0, 4.25) + with self.assertRaisesRegex(TypeError, + "argument 'imag' must be a real number, not .*WithComplex"): + complex(0, WithComplex(4.25+0j)) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'imag' must be a real number, not complex"): + check(complex(0.0, 4.25j), -4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(4.25+0j, 0j), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(4.25j, 0j), 0.0, 4.25) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(0j, 4.25+0j), 0.0, 4.25) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(0j, 4.25j), -4.25, 0.0) + + check(complex(real=4.25), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(real=4.25+0j), 4.25, 0.0) + with self.assertWarnsRegex(DeprecationWarning, + "argument 'real' must be a real number, not complex"): + check(complex(real=4.25+1.5j), 4.25, 1.5) + check(complex(imag=1.5), 0.0, 1.5) + check(complex(real=4.25, imag=1.5), 4.25, 1.5) + check(complex(4.25, imag=1.5), 4.25, 1.5) # check that the sign of a zero in the real or imaginary part - # is preserved when constructing from two floats. (These checks - # are harmless on systems without support for signed zeros.) - def split_zeros(x): - """Function that produces different results for 0. and -0.""" - return atan2(x, -1.) - - self.assertEqual(split_zeros(complex(1., 0.).imag), split_zeros(0.)) - self.assertEqual(split_zeros(complex(1., -0.).imag), split_zeros(-0.)) - self.assertEqual(split_zeros(complex(0., 1.).real), split_zeros(0.)) - self.assertEqual(split_zeros(complex(-0., 1.).real), split_zeros(-0.)) - - c = 3.14 + 1j - self.assertTrue(complex(c) is c) - del c - - self.assertRaises(TypeError, complex, "1", "1") - self.assertRaises(TypeError, complex, 1, "1") - - # SF bug 543840: complex(string) accepts strings with \0 - # Fixed in 2.3. - self.assertRaises(ValueError, complex, '1+1j\0j') - - self.assertRaises(TypeError, int, 5+3j) - self.assertRaises(TypeError, int, 5+3j) - self.assertRaises(TypeError, float, 5+3j) - self.assertRaises(ValueError, complex, "") - self.assertRaises(TypeError, complex, None) - self.assertRaisesRegex(TypeError, "not 'NoneType'", complex, None) - self.assertRaises(ValueError, complex, "\0") - self.assertRaises(ValueError, complex, "3\09") - self.assertRaises(TypeError, complex, "1", "2") - self.assertRaises(TypeError, complex, "1", 42) - self.assertRaises(TypeError, complex, 1, "2") - self.assertRaises(ValueError, complex, "1+") - self.assertRaises(ValueError, complex, "1+1j+1j") - self.assertRaises(ValueError, complex, "--") - self.assertRaises(ValueError, complex, "(1+2j") - self.assertRaises(ValueError, complex, "1+2j)") - self.assertRaises(ValueError, complex, "1+(2j)") - self.assertRaises(ValueError, complex, "(1+2j)123") - self.assertRaises(ValueError, complex, "x") - self.assertRaises(ValueError, complex, "1j+2") - self.assertRaises(ValueError, complex, "1e1ej") - self.assertRaises(ValueError, complex, "1e++1ej") - self.assertRaises(ValueError, complex, ")1+2j(") - self.assertRaisesRegex( - TypeError, - "first argument must be a string or a number, not 'dict'", - complex, {1:2}, 1) - self.assertRaisesRegex( - TypeError, - "second argument must be a number, not 'dict'", - complex, 1, {1:2}) - # the following three are accepted by Python 2.6 - self.assertRaises(ValueError, complex, "1..1j") - self.assertRaises(ValueError, complex, "1.11.1j") - self.assertRaises(ValueError, complex, "1e1.1j") - - # check that complex accepts long unicode strings - self.assertEqual(type(complex("1"*500)), complex) - # check whitespace processing - self.assertEqual(complex('\N{EM SPACE}(\N{EN SPACE}1+1j ) '), 1+1j) - # Invalid unicode string - # See bpo-34087 - self.assertRaises(ValueError, complex, '\u3053\u3093\u306b\u3061\u306f') + # is preserved when constructing from two floats. + for x in 1.0, -1.0: + for y in 0.0, -0.0: + check(complex(x, y), x, y) + check(complex(y, x), y, x) + + c = complex(4.25, 1.5) + self.assertIs(complex(c), c) + c2 = ComplexSubclass(c) + self.assertEqual(c2, c) + self.assertIs(type(c2), ComplexSubclass) + del c, c2 + + self.assertRaisesRegex(TypeError, + "argument must be a string or a number, not dict", + complex, {}) + self.assertRaisesRegex(TypeError, + "argument must be a string or a number, not NoneType", + complex, None) + self.assertRaisesRegex(TypeError, + "argument 'real' must be a real number, not dict", + complex, {1:2}, 0) + self.assertRaisesRegex(TypeError, + "argument 'real' must be a real number, not str", + complex, '1', 0) + self.assertRaisesRegex(TypeError, + "argument 'imag' must be a real number, not dict", + complex, 0, {1:2}) + self.assertRaisesRegex(TypeError, + "argument 'imag' must be a real number, not str", + complex, 0, '1') + + self.assertRaises(TypeError, complex, WithComplex(1.5)) + self.assertRaises(TypeError, complex, WithComplex(1)) + self.assertRaises(TypeError, complex, WithComplex(None)) + self.assertRaises(TypeError, complex, WithComplex(4.25+0j), object()) + self.assertRaises(TypeError, complex, WithComplex(1.5), object()) + self.assertRaises(TypeError, complex, WithComplex(1), object()) + self.assertRaises(TypeError, complex, WithComplex(None), object()) class EvilExc(Exception): pass @@ -480,33 +577,33 @@ def __complex__(self): self.assertRaises(EvilExc, complex, evilcomplex()) - class float2: - def __init__(self, value): - self.value = value - def __float__(self): - return self.value - - self.assertAlmostEqual(complex(float2(42.)), 42) - self.assertAlmostEqual(complex(real=float2(17.), imag=float2(23.)), 17+23j) - self.assertRaises(TypeError, complex, float2(None)) - - class MyIndex: - def __init__(self, value): - self.value = value - def __index__(self): - return self.value - - self.assertAlmostEqual(complex(MyIndex(42)), 42.0+0.0j) - self.assertAlmostEqual(complex(123, MyIndex(42)), 123.0+42.0j) - self.assertRaises(OverflowError, complex, MyIndex(2**2000)) - self.assertRaises(OverflowError, complex, 123, MyIndex(2**2000)) + check(complex(WithFloat(4.25)), 4.25, 0.0) + check(complex(WithFloat(4.25), 1.5), 4.25, 1.5) + check(complex(1.5, WithFloat(4.25)), 1.5, 4.25) + self.assertRaises(TypeError, complex, WithFloat(42)) + self.assertRaises(TypeError, complex, WithFloat(42), 1.5) + self.assertRaises(TypeError, complex, 1.5, WithFloat(42)) + self.assertRaises(TypeError, complex, WithFloat(None)) + self.assertRaises(TypeError, complex, WithFloat(None), 1.5) + self.assertRaises(TypeError, complex, 1.5, WithFloat(None)) + + check(complex(WithIndex(42)), 42.0, 0.0) + check(complex(WithIndex(42), 1.5), 42.0, 1.5) + check(complex(1.5, WithIndex(42)), 1.5, 42.0) + self.assertRaises(OverflowError, complex, WithIndex(2**2000)) + self.assertRaises(OverflowError, complex, WithIndex(2**2000), 1.5) + self.assertRaises(OverflowError, complex, 1.5, WithIndex(2**2000)) + self.assertRaises(TypeError, complex, WithIndex(None)) + self.assertRaises(TypeError, complex, WithIndex(None), 1.5) + self.assertRaises(TypeError, complex, 1.5, WithIndex(None)) class MyInt: def __int__(self): return 42 self.assertRaises(TypeError, complex, MyInt()) - self.assertRaises(TypeError, complex, 123, MyInt()) + self.assertRaises(TypeError, complex, MyInt(), 1.5) + self.assertRaises(TypeError, complex, 1.5, MyInt()) class complex0(complex): """Test usage of __complex__() when inheriting from 'complex'""" @@ -526,9 +623,9 @@ class complex2(complex): def __complex__(self): return None - self.assertEqual(complex(complex0(1j)), 42j) + check(complex(complex0(1j)), 0.0, 42.0) with self.assertWarns(DeprecationWarning): - self.assertEqual(complex(complex1(1j)), 2j) + check(complex(complex1(1j)), 0.0, 2.0) self.assertRaises(TypeError, complex, complex2(1j)) def test___complex__(self): @@ -536,36 +633,93 @@ def test___complex__(self): self.assertEqual(z.__complex__(), z) self.assertEqual(type(z.__complex__()), complex) - class complex_subclass(complex): - pass - - z = complex_subclass(3 + 4j) + z = ComplexSubclass(3 + 4j) self.assertEqual(z.__complex__(), 3 + 4j) self.assertEqual(type(z.__complex__()), complex) @support.requires_IEEE_754 def test_constructor_special_numbers(self): - class complex2(complex): - pass for x in 0.0, -0.0, INF, -INF, NAN: for y in 0.0, -0.0, INF, -INF, NAN: with self.subTest(x=x, y=y): z = complex(x, y) self.assertFloatsAreIdentical(z.real, x) self.assertFloatsAreIdentical(z.imag, y) - z = complex2(x, y) - self.assertIs(type(z), complex2) + z = ComplexSubclass(x, y) + self.assertIs(type(z), ComplexSubclass) self.assertFloatsAreIdentical(z.real, x) self.assertFloatsAreIdentical(z.imag, y) - z = complex(complex2(x, y)) + z = complex(ComplexSubclass(x, y)) self.assertIs(type(z), complex) self.assertFloatsAreIdentical(z.real, x) self.assertFloatsAreIdentical(z.imag, y) - z = complex2(complex(x, y)) - self.assertIs(type(z), complex2) + z = ComplexSubclass(complex(x, y)) + self.assertIs(type(z), ComplexSubclass) self.assertFloatsAreIdentical(z.real, x) self.assertFloatsAreIdentical(z.imag, y) + def test_constructor_from_string(self): + def check(z, x, y): + self.assertIs(type(z), complex) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + + check(complex("1"), 1.0, 0.0) + check(complex("1j"), 0.0, 1.0) + check(complex("-1"), -1.0, 0.0) + check(complex("+1"), 1.0, 0.0) + check(complex("1+2j"), 1.0, 2.0) + check(complex("(1+2j)"), 1.0, 2.0) + check(complex("(1.5+4.25j)"), 1.5, 4.25) + check(complex("4.25+1J"), 4.25, 1.0) + check(complex(" ( +4.25-6J )"), 4.25, -6.0) + check(complex(" ( +4.25-J )"), 4.25, -1.0) + check(complex(" ( +4.25+j )"), 4.25, 1.0) + check(complex("J"), 0.0, 1.0) + check(complex("( j )"), 0.0, 1.0) + check(complex("+J"), 0.0, 1.0) + check(complex("( -j)"), 0.0, -1.0) + check(complex('1-1j'), 1.0, -1.0) + check(complex('1J'), 0.0, 1.0) + + check(complex('1e-500'), 0.0, 0.0) + check(complex('-1e-500j'), 0.0, -0.0) + check(complex('1e-500+1e-500j'), 0.0, 0.0) + check(complex('-1e-500+1e-500j'), -0.0, 0.0) + check(complex('1e-500-1e-500j'), 0.0, -0.0) + check(complex('-1e-500-1e-500j'), -0.0, -0.0) + + # SF bug 543840: complex(string) accepts strings with \0 + # Fixed in 2.3. + self.assertRaises(ValueError, complex, '1+1j\0j') + self.assertRaises(ValueError, complex, "") + self.assertRaises(ValueError, complex, "\0") + self.assertRaises(ValueError, complex, "3\09") + self.assertRaises(ValueError, complex, "1+") + self.assertRaises(ValueError, complex, "1+1j+1j") + self.assertRaises(ValueError, complex, "--") + self.assertRaises(ValueError, complex, "(1+2j") + self.assertRaises(ValueError, complex, "1+2j)") + self.assertRaises(ValueError, complex, "1+(2j)") + self.assertRaises(ValueError, complex, "(1+2j)123") + self.assertRaises(ValueError, complex, "x") + self.assertRaises(ValueError, complex, "1j+2") + self.assertRaises(ValueError, complex, "1e1ej") + self.assertRaises(ValueError, complex, "1e++1ej") + self.assertRaises(ValueError, complex, ")1+2j(") + # the following three are accepted by Python 2.6 + self.assertRaises(ValueError, complex, "1..1j") + self.assertRaises(ValueError, complex, "1.11.1j") + self.assertRaises(ValueError, complex, "1e1.1j") + + # check that complex accepts long unicode strings + self.assertIs(type(complex("1"*500)), complex) + # check whitespace processing + self.assertEqual(complex('\N{EM SPACE}(\N{EN SPACE}1+1j ) '), 1+1j) + # Invalid unicode string + # See bpo-34087 + self.assertRaises(ValueError, complex, '\u3053\u3093\u306b\u3061\u306f') + def test_constructor_negative_nans_from_string(self): self.assertEqual(copysign(1., complex("-nan").real), -1.) self.assertEqual(copysign(1., complex("-nanj").imag), -1.) @@ -584,10 +738,39 @@ def test_underscores(self): if not any(ch in lit for ch in 'xXoObB'): self.assertRaises(ValueError, complex, lit) + def test_from_number(self, cls=complex): + def eq(actual, expected): + self.assertEqual(actual, expected) + self.assertIs(type(actual), cls) + + eq(cls.from_number(3.14), 3.14+0j) + eq(cls.from_number(3.14j), 3.14j) + eq(cls.from_number(314), 314.0+0j) + eq(cls.from_number(OtherComplexSubclass(3.14, 2.72)), 3.14+2.72j) + eq(cls.from_number(WithComplex(3.14+2.72j)), 3.14+2.72j) + eq(cls.from_number(WithFloat(3.14)), 3.14+0j) + eq(cls.from_number(WithIndex(314)), 314.0+0j) + + cNAN = complex(NAN, NAN) + x = cls.from_number(cNAN) + self.assertTrue(x != x) + self.assertIs(type(x), cls) + if cls is complex: + self.assertIs(cls.from_number(cNAN), cNAN) + + self.assertRaises(TypeError, cls.from_number, '3.14') + self.assertRaises(TypeError, cls.from_number, b'3.14') + self.assertRaises(TypeError, cls.from_number, MyInt(314)) + self.assertRaises(TypeError, cls.from_number, {}) + self.assertRaises(TypeError, cls.from_number) + + def test_from_number_subclass(self): + self.test_from_number(ComplexSubclass) + def test_hash(self): for x in range(-30, 30): self.assertEqual(hash(x), hash(complex(x, 0))) - x /= 3.0 # now check against floating point + x /= 3.0 # now check against floating-point self.assertEqual(hash(x), hash(complex(x, 0.))) self.assertNotEqual(hash(2000005 - 1j), -1) @@ -597,6 +780,8 @@ def test_abs(self): for num in nums: self.assertAlmostEqual((num.real**2 + num.imag**2) ** 0.5, abs(num)) + self.assertRaises(OverflowError, abs, complex(DBL_MAX, DBL_MAX)) + def test_repr_str(self): def test(v, expected, test_fn=self.assertEqual): test_fn(repr(v), expected) @@ -642,9 +827,6 @@ def test(v, expected, test_fn=self.assertEqual): test(complex(-0., -0.), "(-0-0j)") def test_pos(self): - class ComplexSubclass(complex): - pass - self.assertEqual(+(1+6j), 1+6j) self.assertEqual(+ComplexSubclass(1, 6), 1+6j) self.assertIs(type(+ComplexSubclass(1, 6)), complex) @@ -664,8 +846,8 @@ def test_getnewargs(self): def test_plus_minus_0j(self): # test that -0j and 0j literals are not identified z1, z2 = 0j, -0j - self.assertEqual(atan2(z1.imag, -1.), atan2(0., -1.)) - self.assertEqual(atan2(z2.imag, -1.), atan2(-0., -1.)) + self.assertFloatsAreIdentical(z1.imag, 0.0) + self.assertFloatsAreIdentical(z2.imag, -0.0) @support.requires_IEEE_754 def test_negated_imaginary_literal(self): @@ -700,8 +882,7 @@ def test_repr_roundtrip(self): for y in vals: z = complex(x, y) roundtrip = complex(repr(z)) - self.assertFloatsAreIdentical(z.real, roundtrip.real) - self.assertFloatsAreIdentical(z.imag, roundtrip.imag) + self.assertComplexesAreIdentical(z, roundtrip) # if we predefine some constants, then eval(repr(z)) should # also work, except that it might change the sign of zeros diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 6a79fe69ec37cf..b97d9ffd94b1f8 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -1,6 +1,5 @@ import threading import time -import unittest import weakref from concurrent import futures from test import support @@ -24,6 +23,7 @@ def make_dummy_object(_): class ExecutorTest: + # Executor.shutdown() and context manager usage is tested by # ExecutorShutdownTest. def test_submit(self): @@ -53,7 +53,8 @@ def test_map_exception(self): i = self.executor.map(divmod, [1, 1, 1, 1], [2, 3, 0, 5]) self.assertEqual(i.__next__(), (0, 1)) self.assertEqual(i.__next__(), (0, 1)) - self.assertRaises(ZeroDivisionError, i.__next__) + with self.assertRaises(ZeroDivisionError): + i.__next__() @support.requires_resource('walltime') def test_map_timeout(self): @@ -83,24 +84,34 @@ def test_no_stale_references(self): # references. my_object = MyObject() my_object_collected = threading.Event() - my_object_callback = weakref.ref( - my_object, lambda obj: my_object_collected.set()) - fut = self.executor.submit(my_object.my_method) + def set_event(): + if Py_GIL_DISABLED: + # gh-117688 Avoid deadlock by setting the event in a + # background thread. The current thread may be in the middle + # of the my_object_collected.wait() call, which holds locks + # needed by my_object_collected.set(). + threading.Thread(target=my_object_collected.set).start() + else: + my_object_collected.set() + my_object_callback = weakref.ref(my_object, lambda obj: set_event()) + # Deliberately discarding the future. + self.executor.submit(my_object.my_method) del my_object if Py_GIL_DISABLED: # Due to biased reference counting, my_object might only be # deallocated while the thread that created it runs -- if the # thread is paused waiting on an event, it may not merge the - # refcount of the queued object. For that reason, we wait for the - # task to finish (so that it's no longer referenced) and force a - # GC to ensure that it is collected. - fut.result() # Wait for the task to finish. - support.gc_collect() + # refcount of the queued object. For that reason, we alternate + # between running the GC and waiting for the event. + wait_time = 0 + collected = False + while not collected and wait_time <= support.SHORT_TIMEOUT: + support.gc_collect() + collected = my_object_collected.wait(timeout=1.0) + wait_time += 1.0 else: - del fut # Deliberately discard the future. - - collected = my_object_collected.wait(timeout=support.SHORT_TIMEOUT) + collected = my_object_collected.wait(timeout=support.SHORT_TIMEOUT) self.assertTrue(collected, "Stale reference not collected within timeout.") diff --git a/Lib/test/test_concurrent_futures/test_deadlock.py b/Lib/test/test_concurrent_futures/test_deadlock.py index 3c30c4558c0b3e..ee589753a3e456 100644 --- a/Lib/test/test_concurrent_futures/test_deadlock.py +++ b/Lib/test/test_concurrent_futures/test_deadlock.py @@ -145,7 +145,7 @@ def test_exit_at_task_unpickle(self): self._check_crash(BrokenProcessPool, id, ExitAtUnpickle()) def test_error_at_task_unpickle(self): - # gh-109832: Restore stderr overriden by _raise_error_ignore_stderr() + # gh-109832: Restore stderr overridden by _raise_error_ignore_stderr() self.addCleanup(setattr, sys, 'stderr', sys.stderr) # Check problem occurring while unpickling a task on workers @@ -183,7 +183,7 @@ def test_error_during_result_pickle_on_worker(self): self._check_crash(PicklingError, _return_instance, ErrorAtPickle) def test_error_during_result_unpickle_in_result_handler(self): - # gh-109832: Restore stderr overriden by _raise_error_ignore_stderr() + # gh-109832: Restore stderr overridden by _raise_error_ignore_stderr() self.addCleanup(setattr, sys, 'stderr', sys.stderr) # Check problem occurring while unpickling a task in @@ -236,7 +236,7 @@ def test_shutdown_deadlock_pickle(self): executor_manager.join() def test_crash_big_data(self): - # Test that there is a clean exception instad of a deadlock when a + # Test that there is a clean exception instead of a deadlock when a # child process crashes while some data is being written into the # queue. # https://github.com/python/cpython/issues/94777 diff --git a/Lib/test/test_concurrent_futures/test_init.py b/Lib/test/test_concurrent_futures/test_init.py index 113a4d1c54be03..df640929309318 100644 --- a/Lib/test/test_concurrent_futures/test_init.py +++ b/Lib/test/test_concurrent_futures/test_init.py @@ -4,6 +4,7 @@ import time import unittest import sys +import io from concurrent.futures._base import BrokenExecutor from concurrent.futures.process import _check_system_limits @@ -124,7 +125,7 @@ def _test(self, test_class): except NotImplementedError: self.skipTest("ProcessPoolExecutor unavailable on this system") - runner = unittest.TextTestRunner() + runner = unittest.TextTestRunner(stream=io.StringIO()) runner.run(test_class('test_initializer')) # GH-104090: @@ -138,6 +139,7 @@ def _test(self, test_class): def test_spawn(self): self._test(ProcessPoolSpawnFailingInitializerTest) + @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) def test_forkserver(self): self._test(ProcessPoolForkserverFailingInitializerTest) diff --git a/Lib/test/test_concurrent_futures/test_interpreter_pool.py b/Lib/test/test_concurrent_futures/test_interpreter_pool.py new file mode 100644 index 00000000000000..93eec08bfe10d5 --- /dev/null +++ b/Lib/test/test_concurrent_futures/test_interpreter_pool.py @@ -0,0 +1,377 @@ +import asyncio +import contextlib +import io +import os +import pickle +import sys +import time +import unittest +from concurrent.futures.interpreter import ( + ExecutionFailed, BrokenInterpreterPool, +) +import _interpreters +from test import support +import test.test_asyncio.utils as testasyncio_utils +from test.support.interpreters import queues + +from .executor import ExecutorTest, mul +from .util import BaseTestCase, InterpreterPoolMixin, setup_module + + +def noop(): + pass + + +def write_msg(fd, msg): + os.write(fd, msg + b'\0') + + +def read_msg(fd): + msg = b'' + while ch := os.read(fd, 1): + if ch == b'\0': + return msg + msg += ch + + +def get_current_name(): + return __name__ + + +def fail(exctype, msg=None): + raise exctype(msg) + + +def get_current_interpid(*extra): + interpid, _ = _interpreters.get_current() + return (interpid, *extra) + + +class InterpretersMixin(InterpreterPoolMixin): + + def pipe(self): + r, w = os.pipe() + self.addCleanup(lambda: os.close(r)) + self.addCleanup(lambda: os.close(w)) + return r, w + + +class PickleShenanigans: + """Succeeds with pickle.dumps(), but fails with pickle.loads()""" + def __init__(self, value): + if value == 1: + raise RuntimeError("gotcha") + + def __reduce__(self): + return (self.__class__, (1,)) + + +class InterpreterPoolExecutorTest( + InterpretersMixin, ExecutorTest, BaseTestCase): + + @unittest.expectedFailure + def test_init_script(self): + msg1 = b'step: init' + msg2 = b'step: run' + r, w = self.pipe() + initscript = f""" + import os + msg = {msg2!r} + os.write({w}, {msg1!r} + b'\\0') + """ + script = f""" + os.write({w}, msg + b'\\0') + """ + os.write(w, b'\0') + + executor = self.executor_type(initializer=initscript) + before_init = os.read(r, 100) + fut = executor.submit(script) + after_init = read_msg(r) + fut.result() + after_run = read_msg(r) + + self.assertEqual(before_init, b'\0') + self.assertEqual(after_init, msg1) + self.assertEqual(after_run, msg2) + + @unittest.expectedFailure + def test_init_script_args(self): + with self.assertRaises(ValueError): + self.executor_type(initializer='pass', initargs=('spam',)) + + def test_init_func(self): + msg = b'step: init' + r, w = self.pipe() + os.write(w, b'\0') + + executor = self.executor_type( + initializer=write_msg, initargs=(w, msg)) + before = os.read(r, 100) + executor.submit(mul, 10, 10) + after = read_msg(r) + + self.assertEqual(before, b'\0') + self.assertEqual(after, msg) + + def test_init_closure(self): + count = 0 + def init1(): + assert count == 0, count + def init2(): + nonlocal count + count += 1 + + with self.assertRaises(pickle.PicklingError): + self.executor_type(initializer=init1) + with self.assertRaises(pickle.PicklingError): + self.executor_type(initializer=init2) + + def test_init_instance_method(self): + class Spam: + def initializer(self): + raise NotImplementedError + spam = Spam() + + with self.assertRaises(pickle.PicklingError): + self.executor_type(initializer=spam.initializer) + + def test_init_shared(self): + msg = b'eggs' + r, w = self.pipe() + script = f"""if True: + import os + if __name__ != '__main__': + import __main__ + spam = __main__.spam + os.write({w}, spam + b'\\0') + """ + + executor = self.executor_type(shared={'spam': msg}) + fut = executor.submit(exec, script) + fut.result() + after = read_msg(r) + + self.assertEqual(after, msg) + + @unittest.expectedFailure + def test_init_exception_in_script(self): + executor = self.executor_type(initializer='raise Exception("spam")') + with executor: + with contextlib.redirect_stderr(io.StringIO()) as stderr: + fut = executor.submit('pass') + with self.assertRaises(BrokenInterpreterPool): + fut.result() + stderr = stderr.getvalue() + self.assertIn('ExecutionFailed: Exception: spam', stderr) + self.assertIn('Uncaught in the interpreter:', stderr) + self.assertIn('The above exception was the direct cause of the following exception:', + stderr) + + def test_init_exception_in_func(self): + executor = self.executor_type(initializer=fail, + initargs=(Exception, 'spam')) + with executor: + with contextlib.redirect_stderr(io.StringIO()) as stderr: + fut = executor.submit(noop) + with self.assertRaises(BrokenInterpreterPool): + fut.result() + stderr = stderr.getvalue() + self.assertIn('ExecutionFailed: Exception: spam', stderr) + self.assertIn('Uncaught in the interpreter:', stderr) + self.assertIn('The above exception was the direct cause of the following exception:', + stderr) + + @unittest.expectedFailure + def test_submit_script(self): + msg = b'spam' + r, w = self.pipe() + script = f""" + import os + os.write({w}, __name__.encode('utf-8') + b'\\0') + """ + executor = self.executor_type() + + fut = executor.submit(script) + res = fut.result() + after = read_msg(r) + + self.assertEqual(after, b'__main__') + self.assertIs(res, None) + + def test_submit_closure(self): + spam = True + def task1(): + return spam + def task2(): + nonlocal spam + spam += 1 + return spam + + executor = self.executor_type() + with self.assertRaises(pickle.PicklingError): + executor.submit(task1) + with self.assertRaises(pickle.PicklingError): + executor.submit(task2) + + def test_submit_local_instance(self): + class Spam: + def __init__(self): + self.value = True + + executor = self.executor_type() + with self.assertRaises(pickle.PicklingError): + executor.submit(Spam) + + def test_submit_instance_method(self): + class Spam: + def run(self): + return True + spam = Spam() + + executor = self.executor_type() + with self.assertRaises(pickle.PicklingError): + executor.submit(spam.run) + + def test_submit_func_globals(self): + executor = self.executor_type() + fut = executor.submit(get_current_name) + name = fut.result() + + self.assertEqual(name, __name__) + self.assertNotEqual(name, '__main__') + + @unittest.expectedFailure + def test_submit_exception_in_script(self): + fut = self.executor.submit('raise Exception("spam")') + with self.assertRaises(Exception) as captured: + fut.result() + self.assertIs(type(captured.exception), Exception) + self.assertEqual(str(captured.exception), 'spam') + cause = captured.exception.__cause__ + self.assertIs(type(cause), ExecutionFailed) + for attr in ('__name__', '__qualname__', '__module__'): + self.assertEqual(getattr(cause.excinfo.type, attr), + getattr(Exception, attr)) + self.assertEqual(cause.excinfo.msg, 'spam') + + def test_submit_exception_in_func(self): + fut = self.executor.submit(fail, Exception, 'spam') + with self.assertRaises(Exception) as captured: + fut.result() + self.assertIs(type(captured.exception), Exception) + self.assertEqual(str(captured.exception), 'spam') + cause = captured.exception.__cause__ + self.assertIs(type(cause), ExecutionFailed) + for attr in ('__name__', '__qualname__', '__module__'): + self.assertEqual(getattr(cause.excinfo.type, attr), + getattr(Exception, attr)) + self.assertEqual(cause.excinfo.msg, 'spam') + + def test_saturation(self): + blocker = queues.create() + executor = self.executor_type(4, shared=dict(blocker=blocker)) + + for i in range(15 * executor._max_workers): + executor.submit(exec, 'import __main__; __main__.blocker.get()') + #executor.submit('blocker.get()') + self.assertEqual(len(executor._threads), executor._max_workers) + for i in range(15 * executor._max_workers): + blocker.put_nowait(None) + executor.shutdown(wait=True) + + @support.requires_gil_enabled("gh-117344: test is flaky without the GIL") + def test_idle_thread_reuse(self): + executor = self.executor_type() + executor.submit(mul, 21, 2).result() + executor.submit(mul, 6, 7).result() + executor.submit(mul, 3, 14).result() + self.assertEqual(len(executor._threads), 1) + executor.shutdown(wait=True) + + def test_pickle_errors_propagate(self): + # GH-125864: Pickle errors happen before the script tries to execute, so the + # queue used to wait infinitely. + + fut = self.executor.submit(PickleShenanigans(0)) + with self.assertRaisesRegex(RuntimeError, "gotcha"): + fut.result() + + +class AsyncioTest(InterpretersMixin, testasyncio_utils.TestCase): + + @classmethod + def setUpClass(cls): + # Most uses of asyncio will implicitly call set_event_loop_policy() + # with the default policy if a policy hasn't been set already. + # If that happens in a test, like here, we'll end up with a failure + # when --fail-env-changed is used. That's why the other tests that + # use asyncio are careful to set the policy back to None and why + # we're careful to do so here. We also validate that no other + # tests left a policy in place, just in case. + policy = support.maybe_get_event_loop_policy() + assert policy is None, policy + cls.addClassCleanup(lambda: asyncio._set_event_loop_policy(None)) + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + + self.executor = self.executor_type() + self.addCleanup(lambda: self.executor.shutdown()) + + def tearDown(self): + if not self.loop.is_closed(): + testasyncio_utils.run_briefly(self.loop) + + self.doCleanups() + support.gc_collect() + super().tearDown() + + def test_run_in_executor(self): + unexpected, _ = _interpreters.get_current() + + func = get_current_interpid + fut = self.loop.run_in_executor(self.executor, func, 'yo') + interpid, res = self.loop.run_until_complete(fut) + + self.assertEqual(res, 'yo') + self.assertNotEqual(interpid, unexpected) + + def test_run_in_executor_cancel(self): + executor = self.executor_type() + + called = False + + def patched_call_soon(*args): + nonlocal called + called = True + + func = time.sleep + fut = self.loop.run_in_executor(self.executor, func, 0.05) + fut.cancel() + self.loop.run_until_complete( + self.loop.shutdown_default_executor()) + self.loop.close() + self.loop.call_soon = patched_call_soon + self.loop.call_soon_threadsafe = patched_call_soon + time.sleep(0.4) + self.assertFalse(called) + + def test_default_executor(self): + unexpected, _ = _interpreters.get_current() + + self.loop.set_default_executor(self.executor) + fut = self.loop.run_in_executor(None, get_current_interpid) + interpid, = self.loop.run_until_complete(fut) + + self.assertNotEqual(interpid, unexpected) + + +def setUpModule(): + setup_module() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py index 7fc59a05f3deac..8b1bdaa33d8f5c 100644 --- a/Lib/test/test_concurrent_futures/test_process_pool.py +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -116,6 +116,7 @@ def test_saturation(self): for _ in range(job_count): sem.release() + @support.requires_gil_enabled("gh-117344: test is flaky without the GIL") def test_idle_process_reuse_one(self): executor = self.executor assert executor._max_workers >= 4 @@ -201,13 +202,13 @@ def test_python_finalization_error(self): # QueueFeederThread. orig_start_new_thread = threading._start_joinable_thread nthread = 0 - def mock_start_new_thread(func, *args): + def mock_start_new_thread(func, *args, **kwargs): nonlocal nthread if nthread >= 1: raise RuntimeError("can't create new thread at " "interpreter shutdown") nthread += 1 - return orig_start_new_thread(func, *args) + return orig_start_new_thread(func, *args, **kwargs) with support.swap_attr(threading, '_start_joinable_thread', mock_start_new_thread): diff --git a/Lib/test/test_concurrent_futures/test_shutdown.py b/Lib/test/test_concurrent_futures/test_shutdown.py index 45dab7a75fdd50..7a4065afd46fc8 100644 --- a/Lib/test/test_concurrent_futures/test_shutdown.py +++ b/Lib/test/test_concurrent_futures/test_shutdown.py @@ -247,7 +247,9 @@ def test_cancel_futures_wait_false(self): # Errors in atexit hooks don't change the process exit code, check # stderr manually. self.assertFalse(err) - self.assertEqual(out.strip(), b"apple") + # gh-116682: stdout may be empty if shutdown happens before task + # starts executing. + self.assertIn(out.strip(), [b"apple", b""]) class ProcessPoolShutdownTest(ExecutorShutdownTest): diff --git a/Lib/test/test_concurrent_futures/test_thread_pool.py b/Lib/test/test_concurrent_futures/test_thread_pool.py index 5926a632aa4bec..4324241b374967 100644 --- a/Lib/test/test_concurrent_futures/test_thread_pool.py +++ b/Lib/test/test_concurrent_futures/test_thread_pool.py @@ -41,6 +41,7 @@ def acquire_lock(lock): sem.release() executor.shutdown(wait=True) + @support.requires_gil_enabled("gh-117344: test is flaky without the GIL") def test_idle_thread_reuse(self): executor = self.executor_type() executor.submit(mul, 21, 2).result() @@ -49,6 +50,7 @@ def test_idle_thread_reuse(self): self.assertEqual(len(executor._threads), 1) executor.shutdown(wait=True) + @support.requires_fork() @unittest.skipUnless(hasattr(os, 'register_at_fork'), 'need os.register_at_fork') @support.requires_resource('cpu') def test_hang_global_shutdown_lock(self): @@ -64,6 +66,25 @@ def submit(pool): with futures.ProcessPoolExecutor(1, mp_context=mp.get_context('fork')) as workers: workers.submit(tuple) + @support.requires_fork() + @unittest.skipUnless(hasattr(os, 'register_at_fork'), 'need os.register_at_fork') + def test_process_fork_from_a_threadpool(self): + # bpo-43944: clear concurrent.futures.thread._threads_queues after fork, + # otherwise child process will try to join parent thread + def fork_process_and_return_exitcode(): + # Ignore the warning about fork with threads. + with self.assertWarnsRegex(DeprecationWarning, + r"use of fork\(\) may lead to deadlocks in the child"): + p = mp.get_context('fork').Process(target=lambda: 1) + p.start() + p.join() + return p.exitcode + + with futures.ThreadPoolExecutor(1) as pool: + process_exitcode = pool.submit(fork_process_and_return_exitcode).result() + + self.assertEqual(process_exitcode, 0) + def test_executor_map_current_future_cancel(self): stop_event = threading.Event() log = [] diff --git a/Lib/test/test_concurrent_futures/test_wait.py b/Lib/test/test_concurrent_futures/test_wait.py index ff486202092c81..108cf54bf79e6f 100644 --- a/Lib/test/test_concurrent_futures/test_wait.py +++ b/Lib/test/test_concurrent_futures/test_wait.py @@ -142,7 +142,7 @@ def test_pending_calls_race(self): def future_func(): event.wait() oldswitchinterval = sys.getswitchinterval() - sys.setswitchinterval(1e-6) + support.setswitchinterval(1e-6) try: fs = {self.executor.submit(future_func) for i in range(100)} event.set() diff --git a/Lib/test/test_concurrent_futures/util.py b/Lib/test/test_concurrent_futures/util.py index 3e855031913042..52baab51340fc9 100644 --- a/Lib/test/test_concurrent_futures/util.py +++ b/Lib/test/test_concurrent_futures/util.py @@ -74,6 +74,10 @@ class ThreadPoolMixin(ExecutorMixin): executor_type = futures.ThreadPoolExecutor +class InterpreterPoolMixin(ExecutorMixin): + executor_type = futures.InterpreterPoolExecutor + + class ProcessPoolForkMixin(ExecutorMixin): executor_type = futures.ProcessPoolExecutor ctx = "fork" @@ -85,6 +89,8 @@ def get_context(self): self.skipTest("ProcessPoolExecutor unavailable on this system") if sys.platform == "win32": self.skipTest("require unix system") + if support.check_sanitizer(thread=True): + self.skipTest("TSAN doesn't support threads after fork") return super().get_context() @@ -111,11 +117,14 @@ def get_context(self): self.skipTest("ProcessPoolExecutor unavailable on this system") if sys.platform == "win32": self.skipTest("require unix system") + if support.check_sanitizer(thread=True): + self.skipTest("TSAN doesn't support threads after fork") return super().get_context() def create_executor_tests(remote_globals, mixin, bases=(BaseTestCase,), executor_mixins=(ThreadPoolMixin, + InterpreterPoolMixin, ProcessPoolForkMixin, ProcessPoolForkserverMixin, ProcessPoolSpawnMixin)): diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index 2d7dfbde7082ee..bde805eb741c33 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -2,7 +2,6 @@ import configparser import io import os -import pathlib import textwrap import unittest @@ -646,6 +645,21 @@ def test_weird_errors(self): "'opt' in section 'Bar' already exists") self.assertEqual(e.args, ("Bar", "opt", "", None)) + def test_get_after_duplicate_option_error(self): + cf = self.newconfig() + ini = textwrap.dedent("""\ + [Foo] + x{equals}1 + y{equals}2 + y{equals}3 + """.format(equals=self.delimiters[0])) + if self.strict: + with self.assertRaises(configparser.DuplicateOptionError): + cf.read_string(ini) + else: + cf.read_string(ini) + self.assertEqual(cf.get('Foo', 'x'), '1') + def test_write(self): config_string = ( "[Long Line]\n" @@ -730,12 +744,12 @@ def test_read_returns_file_list(self): self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only a Path object: cf = self.newconfig() - parsed_files = cf.read(pathlib.Path(file1), encoding="utf-8") + parsed_files = cf.read(os_helper.FakePath(file1), encoding="utf-8") self.assertEqual(parsed_files, [file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we passed both a filename and a Path object: cf = self.newconfig() - parsed_files = cf.read([pathlib.Path(file1), file1], encoding="utf-8") + parsed_files = cf.read([os_helper.FakePath(file1), file1], encoding="utf-8") self.assertEqual(parsed_files, [file1, file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only missing files: @@ -1555,6 +1569,30 @@ def test_source_as_bytes(self): "'[badbad'" ) + def test_keys_without_value_with_extra_whitespace(self): + lines = [ + '[SECT]\n', + 'KEY1\n', + ' KEY2 = VAL2\n', # note the Space before the key! + ] + parser = configparser.ConfigParser( + comment_prefixes="", + allow_no_value=True, + strict=False, + delimiters=('=',), + interpolation=None, + ) + with self.assertRaises(configparser.MultilineContinuationError) as dse: + parser.read_file(lines) + self.assertEqual( + str(dse.exception), + "Key without value continued with an indented line.\n" + "file: '', line: 3\n" + "' KEY2 = VAL2\\n'" + ) + + + class CoverageOneHundredTestCase(unittest.TestCase): """Covers edge cases in the codebase.""" @@ -2076,6 +2114,76 @@ def test_instance_assignment(self): self.assertEqual(cfg['two'].getlen('one'), 5) +class SectionlessTestCase(unittest.TestCase): + + def fromstring(self, string): + cfg = configparser.ConfigParser(allow_unnamed_section=True) + cfg.read_string(string) + return cfg + + def test_no_first_section(self): + cfg1 = self.fromstring(""" + a = 1 + b = 2 + [sect1] + c = 3 + """) + + self.assertEqual(set([configparser.UNNAMED_SECTION, 'sect1']), set(cfg1.sections())) + self.assertEqual('1', cfg1[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg1[configparser.UNNAMED_SECTION]['b']) + self.assertEqual('3', cfg1['sect1']['c']) + + output = io.StringIO() + cfg1.write(output) + cfg2 = self.fromstring(output.getvalue()) + + #self.assertEqual(set([configparser.UNNAMED_SECTION, 'sect1']), set(cfg2.sections())) + self.assertEqual('1', cfg2[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg2[configparser.UNNAMED_SECTION]['b']) + self.assertEqual('3', cfg2['sect1']['c']) + + def test_no_section(self): + cfg1 = self.fromstring(""" + a = 1 + b = 2 + """) + + self.assertEqual([configparser.UNNAMED_SECTION], cfg1.sections()) + self.assertEqual('1', cfg1[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg1[configparser.UNNAMED_SECTION]['b']) + + output = io.StringIO() + cfg1.write(output) + cfg2 = self.fromstring(output.getvalue()) + + self.assertEqual([configparser.UNNAMED_SECTION], cfg2.sections()) + self.assertEqual('1', cfg2[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg2[configparser.UNNAMED_SECTION]['b']) + + def test_add_section(self): + cfg = configparser.ConfigParser(allow_unnamed_section=True) + cfg.add_section(configparser.UNNAMED_SECTION) + cfg.set(configparser.UNNAMED_SECTION, 'a', '1') + self.assertEqual('1', cfg[configparser.UNNAMED_SECTION]['a']) + + def test_disabled_error(self): + with self.assertRaises(configparser.MissingSectionHeaderError): + configparser.ConfigParser().read_string("a = 1") + + with self.assertRaises(configparser.UnnamedSectionDisabledError): + configparser.ConfigParser().add_section(configparser.UNNAMED_SECTION) + + def test_multiple_configs(self): + cfg = configparser.ConfigParser(allow_unnamed_section=True) + cfg.read_string('a = 1') + cfg.read_string('b = 2') + + self.assertEqual([configparser.UNNAMED_SECTION], cfg.sections()) + self.assertEqual('1', cfg[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg[configparser.UNNAMED_SECTION]['b']) + + class MiscTestCase(unittest.TestCase): def test__all__(self): support.check__all__(self, configparser, not_exported={"Error"}) diff --git a/Lib/test/test_contains.py b/Lib/test/test_contains.py index 471d04a76ca4e4..259c954aa1d616 100644 --- a/Lib/test/test_contains.py +++ b/Lib/test/test_contains.py @@ -24,8 +24,11 @@ def test_common_tests(self): self.assertNotIn(0, b) self.assertIn(1, c) self.assertNotIn(0, c) - self.assertRaises(TypeError, lambda: 1 in a) - self.assertRaises(TypeError, lambda: 1 not in a) + msg = "argument of type 'base_set' is not a container or iterable" + with self.assertRaisesRegex(TypeError, msg): + 1 in a + with self.assertRaisesRegex(TypeError, msg): + 1 not in a # test char in string self.assertIn('c', 'abc') diff --git a/Lib/test/test_context.py b/Lib/test/test_context.py index 255be306156c0b..82d1797ab3b79e 100644 --- a/Lib/test/test_context.py +++ b/Lib/test/test_context.py @@ -1,3 +1,4 @@ +import collections.abc import concurrent.futures import contextvars import functools @@ -60,6 +61,14 @@ def test_context_var_repr_1(self): c.reset(t) self.assertIn(' used ', repr(t)) + @isolated_context + def test_token_repr_1(self): + c = contextvars.ContextVar('a') + tok = c.set(1) + self.assertRegex(repr(tok), + r"^ at 0x[0-9a-fA-F]+>$") + def test_context_subclassing_1(self): with self.assertRaisesRegex(TypeError, 'not an acceptable base type'): class MyContextVar(contextvars.ContextVar): @@ -342,6 +351,19 @@ def ctx2_fun(): ctx1.run(ctx1_fun) + def test_context_isinstance(self): + ctx = contextvars.Context() + self.assertIsInstance(ctx, collections.abc.Mapping) + self.assertTrue(issubclass(contextvars.Context, collections.abc.Mapping)) + + mapping_methods = ( + '__contains__', '__eq__', '__getitem__', '__iter__', '__len__', + '__ne__', 'get', 'items', 'keys', 'values', + ) + for name in mapping_methods: + with self.subTest(name=name): + self.assertTrue(callable(getattr(ctx, name))) + @isolated_context @threading_helper.requires_working_threading() def test_context_threads_1(self): diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 36c3abca80f894..cf6519598037e9 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -444,12 +444,10 @@ class FileContextTestCase(unittest.TestCase): def testWithOpen(self): tfn = tempfile.mktemp() try: - f = None with open(tfn, "w", encoding="utf-8") as f: self.assertFalse(f.closed) f.write("Booh\n") self.assertTrue(f.closed) - f = None with self.assertRaises(ZeroDivisionError): with open(tfn, "r", encoding="utf-8") as f: self.assertFalse(f.closed) diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index ca7315783b9674..7750186e56a5cc 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -1,21 +1,27 @@ -import asyncio +import functools from contextlib import ( asynccontextmanager, AbstractAsyncContextManager, AsyncExitStack, nullcontext, aclosing, contextmanager) from test import support +from test.support import run_no_yield_async_fn as _run_async_fn import unittest import traceback from test.test_contextlib import TestBaseExitStack -support.requires_working_socket(module=True) -def tearDownModule(): - asyncio.set_event_loop_policy(None) +def _async_test(async_fn): + """Decorator to turn an async function into a synchronous function""" + @functools.wraps(async_fn) + def wrapper(*args, **kwargs): + return _run_async_fn(async_fn, *args, **kwargs) + return wrapper -class TestAbstractAsyncContextManager(unittest.IsolatedAsyncioTestCase): +class TestAbstractAsyncContextManager(unittest.TestCase): + + @_async_test async def test_enter(self): class DefaultEnter(AbstractAsyncContextManager): async def __aexit__(self, *args): @@ -27,6 +33,7 @@ async def __aexit__(self, *args): async with manager as context: self.assertIs(manager, context) + @_async_test async def test_slots(self): class DefaultAsyncContextManager(AbstractAsyncContextManager): __slots__ = () @@ -38,6 +45,7 @@ async def __aexit__(self, *args): manager = DefaultAsyncContextManager() manager.var = 42 + @_async_test async def test_async_gen_propagates_generator_exit(self): # A regression test for https://bugs.python.org/issue33786. @@ -88,8 +96,9 @@ class NoneAexit(ManagerFromScratch): self.assertFalse(issubclass(NoneAexit, AbstractAsyncContextManager)) -class AsyncContextManagerTestCase(unittest.IsolatedAsyncioTestCase): +class AsyncContextManagerTestCase(unittest.TestCase): + @_async_test async def test_contextmanager_plain(self): state = [] @asynccontextmanager @@ -103,6 +112,7 @@ async def woohoo(): state.append(x) self.assertEqual(state, [1, 42, 999]) + @_async_test async def test_contextmanager_finally(self): state = [] @asynccontextmanager @@ -120,6 +130,7 @@ async def woohoo(): raise ZeroDivisionError() self.assertEqual(state, [1, 42, 999]) + @_async_test async def test_contextmanager_traceback(self): @asynccontextmanager async def f(): @@ -175,6 +186,7 @@ class StopAsyncIterationSubclass(StopAsyncIteration): self.assertEqual(frames[0].name, 'test_contextmanager_traceback') self.assertEqual(frames[0].line, 'raise stop_exc') + @_async_test async def test_contextmanager_no_reraise(self): @asynccontextmanager async def whee(): @@ -184,6 +196,7 @@ async def whee(): # Calling __aexit__ should not result in an exception self.assertFalse(await ctx.__aexit__(TypeError, TypeError("foo"), None)) + @_async_test async def test_contextmanager_trap_yield_after_throw(self): @asynccontextmanager async def whoo(): @@ -199,6 +212,7 @@ async def whoo(): # The "gen" attribute is an implementation detail. self.assertFalse(ctx.gen.ag_suspended) + @_async_test async def test_contextmanager_trap_no_yield(self): @asynccontextmanager async def whoo(): @@ -208,6 +222,7 @@ async def whoo(): with self.assertRaises(RuntimeError): await ctx.__aenter__() + @_async_test async def test_contextmanager_trap_second_yield(self): @asynccontextmanager async def whoo(): @@ -221,6 +236,7 @@ async def whoo(): # The "gen" attribute is an implementation detail. self.assertFalse(ctx.gen.ag_suspended) + @_async_test async def test_contextmanager_non_normalised(self): @asynccontextmanager async def whoo(): @@ -234,6 +250,7 @@ async def whoo(): with self.assertRaises(SyntaxError): await ctx.__aexit__(RuntimeError, None, None) + @_async_test async def test_contextmanager_except(self): state = [] @asynccontextmanager @@ -251,6 +268,7 @@ async def woohoo(): raise ZeroDivisionError(999) self.assertEqual(state, [1, 42, 999]) + @_async_test async def test_contextmanager_except_stopiter(self): @asynccontextmanager async def woohoo(): @@ -277,6 +295,7 @@ class StopAsyncIterationSubclass(StopAsyncIteration): else: self.fail(f'{stop_exc} was suppressed') + @_async_test async def test_contextmanager_wrap_runtimeerror(self): @asynccontextmanager async def woohoo(): @@ -321,12 +340,14 @@ def test_contextmanager_doc_attrib(self): self.assertEqual(baz.__doc__, "Whee!") @support.requires_docstrings + @_async_test async def test_instance_docstring_given_cm_docstring(self): baz = self._create_contextmanager_attribs()(None) self.assertEqual(baz.__doc__, "Whee!") async with baz: pass # suppress warning + @_async_test async def test_keywords(self): # Ensure no keyword arguments are inhibited @asynccontextmanager @@ -335,6 +356,7 @@ async def woohoo(self, func, args, kwds): async with woohoo(self=11, func=22, args=33, kwds=44) as target: self.assertEqual(target, (11, 22, 33, 44)) + @_async_test async def test_recursive(self): depth = 0 ncols = 0 @@ -361,6 +383,7 @@ async def recursive(): self.assertEqual(ncols, 10) self.assertEqual(depth, 0) + @_async_test async def test_decorator(self): entered = False @@ -379,6 +402,7 @@ async def test(): await test() self.assertFalse(entered) + @_async_test async def test_decorator_with_exception(self): entered = False @@ -401,6 +425,7 @@ async def test(): await test() self.assertFalse(entered) + @_async_test async def test_decorating_method(self): @asynccontextmanager @@ -435,7 +460,7 @@ async def method(self, a, b, c=None): self.assertEqual(test.b, 2) -class AclosingTestCase(unittest.IsolatedAsyncioTestCase): +class AclosingTestCase(unittest.TestCase): @support.requires_docstrings def test_instance_docs(self): @@ -443,6 +468,7 @@ def test_instance_docs(self): obj = aclosing(None) self.assertEqual(obj.__doc__, cm_docstring) + @_async_test async def test_aclosing(self): state = [] class C: @@ -454,6 +480,7 @@ async def aclose(self): self.assertEqual(x, y) self.assertEqual(state, [1]) + @_async_test async def test_aclosing_error(self): state = [] class C: @@ -467,6 +494,7 @@ async def aclose(self): 1 / 0 self.assertEqual(state, [1]) + @_async_test async def test_aclosing_bpo41229(self): state = [] @@ -492,45 +520,27 @@ async def agenfunc(): self.assertEqual(state, [1]) -class TestAsyncExitStack(TestBaseExitStack, unittest.IsolatedAsyncioTestCase): +class TestAsyncExitStack(TestBaseExitStack, unittest.TestCase): class SyncAsyncExitStack(AsyncExitStack): - @staticmethod - def run_coroutine(coro): - loop = asyncio.get_event_loop_policy().get_event_loop() - t = loop.create_task(coro) - t.add_done_callback(lambda f: loop.stop()) - loop.run_forever() - - exc = t.exception() - if not exc: - return t.result() - else: - context = exc.__context__ - - try: - raise exc - except: - exc.__context__ = context - raise exc def close(self): - return self.run_coroutine(self.aclose()) + return _run_async_fn(self.aclose) def __enter__(self): - return self.run_coroutine(self.__aenter__()) + return _run_async_fn(self.__aenter__) def __exit__(self, *exc_details): - return self.run_coroutine(self.__aexit__(*exc_details)) + return _run_async_fn(self.__aexit__, *exc_details) exit_stack = SyncAsyncExitStack callback_error_internal_frames = [ - ('__exit__', 'return self.run_coroutine(self.__aexit__(*exc_details))'), - ('run_coroutine', 'raise exc'), - ('run_coroutine', 'raise exc'), + ('__exit__', 'return _run_async_fn(self.__aexit__, *exc_details)'), + ('run_no_yield_async_fn', 'coro.send(None)'), ('__aexit__', 'raise exc'), ('__aexit__', 'cb_suppress = cb(*exc_details)'), ] + @_async_test async def test_async_callback(self): expected = [ ((), {}), @@ -573,6 +583,7 @@ async def _exit(*args, **kwds): stack.push_async_callback(callback=_exit, arg=3) self.assertEqual(result, []) + @_async_test async def test_async_push(self): exc_raised = ZeroDivisionError async def _expect_exc(exc_type, exc, exc_tb): @@ -608,6 +619,7 @@ async def __aexit__(self, *exc_details): self.assertIs(stack._exit_callbacks[-1][1], _expect_exc) 1/0 + @_async_test async def test_enter_async_context(self): class TestCM(object): async def __aenter__(self): @@ -629,6 +641,7 @@ async def _exit(): self.assertEqual(result, [1, 2, 3, 4]) + @_async_test async def test_enter_async_context_errors(self): class LacksEnterAndExit: pass @@ -648,6 +661,7 @@ async def __aenter__(self): await stack.enter_async_context(LacksExit()) self.assertFalse(stack._exit_callbacks) + @_async_test async def test_async_exit_exception_chaining(self): # Ensure exception chaining matches the reference behaviour async def raise_exc(exc): @@ -679,6 +693,7 @@ async def suppress_exc(*exc_details): self.assertIsInstance(inner_exc, ValueError) self.assertIsInstance(inner_exc.__context__, ZeroDivisionError) + @_async_test async def test_async_exit_exception_explicit_none_context(self): # Ensure AsyncExitStack chaining matches actual nested `with` statements # regarding explicit __context__ = None. @@ -713,6 +728,7 @@ async def my_cm_with_exit_stack(): else: self.fail("Expected IndexError, but no exception was raised") + @_async_test async def test_instance_bypass_async(self): class Example(object): pass cm = Example() @@ -725,7 +741,8 @@ class Example(object): pass self.assertIs(stack._exit_callbacks[-1][1], cm) -class TestAsyncNullcontext(unittest.IsolatedAsyncioTestCase): +class TestAsyncNullcontext(unittest.TestCase): + @_async_test async def test_async_nullcontext(self): class C: pass diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index 89102373759ca0..d76341417e9bef 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -371,6 +371,7 @@ def test_deepcopy_list(self): self.assertIsNot(x, y) self.assertIsNot(x[0], y[0]) + @support.skip_emscripten_stack_overflow() def test_deepcopy_reflexive_list(self): x = [] x.append(x) @@ -398,6 +399,7 @@ def test_deepcopy_tuple_of_immutables(self): y = copy.deepcopy(x) self.assertIs(x, y) + @support.skip_emscripten_stack_overflow() def test_deepcopy_reflexive_tuple(self): x = ([],) x[0].append(x) @@ -415,6 +417,7 @@ def test_deepcopy_dict(self): self.assertIsNot(x, y) self.assertIsNot(x["foo"], y["foo"]) + @support.skip_emscripten_stack_overflow() def test_deepcopy_reflexive_dict(self): x = {} x['foo'] = x @@ -972,6 +975,10 @@ class C: copy.replace(c, x=1, error=2) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + support.check__all__(self, copy, not_exported={"dispatch_table", "error"}) + def global_foo(x, y): return x+y diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index d848bfbd46c83b..840043d5271224 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -11,6 +11,10 @@ from test.support import import_helper from test.support import warnings_helper from test.support.script_helper import assert_python_ok +try: + import _testcapi +except ImportError: + _testcapi = None class AsyncYieldFrom: @@ -970,13 +974,13 @@ def test_await_1(self): async def foo(): await 1 - with self.assertRaisesRegex(TypeError, "object int can.t.*await"): + with self.assertRaisesRegex(TypeError, "'int' object can.t be awaited"): run_async(foo()) def test_await_2(self): async def foo(): await [] - with self.assertRaisesRegex(TypeError, "object list can.t.*await"): + with self.assertRaisesRegex(TypeError, "'list' object can.t be awaited"): run_async(foo()) def test_await_3(self): @@ -1036,7 +1040,7 @@ class Awaitable: async def foo(): return await Awaitable() with self.assertRaisesRegex( - TypeError, "object Awaitable can't be used in 'await' expression"): + TypeError, "'Awaitable' object can't be awaited"): run_async(foo()) @@ -1181,7 +1185,7 @@ async def f(): async def g(): try: raise KeyError - except: + except KeyError: return await f() _, result = run_async(g()) @@ -2283,14 +2287,14 @@ async def f(): buffer.append('unreachable') loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + asyncio._set_event_loop(loop) try: loop.run_until_complete(f()) except MyException: pass finally: loop.close() - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) self.assertEqual(buffer, [1, 2, 'MyException']) @@ -2445,6 +2449,7 @@ def test_unawaited_warning_during_shutdown(self): @support.cpython_only +@unittest.skipIf(_testcapi is None, "requires _testcapi") class CAPITest(unittest.TestCase): def test_tp_await_1(self): diff --git a/Lib/test/test_cppext/__init__.py b/Lib/test/test_cppext/__init__.py index c6039bd17b0662..d5195227308fec 100644 --- a/Lib/test/test_cppext/__init__.py +++ b/Lib/test/test_cppext/__init__.py @@ -1,10 +1,10 @@ # gh-91321: Build a basic C++ test extension to check that the Python C API is # compatible with C++ and does not emit C++ compiler warnings. import os.path +import shlex import shutil -import unittest import subprocess -import sysconfig +import unittest from test import support @@ -12,36 +12,46 @@ SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') -# gh-110119: pip does not currently support 't' in the ABI flag use by -# --disable-gil builds. Once it does, we can remove this skip. -@unittest.skipIf(support.Py_GIL_DISABLED, - 'test does not work with --disable-gil') +# With MSVC on a debug build, the linker fails with: cannot open file +# 'python311.lib', it should look 'python311_d.lib'. +@unittest.skipIf(support.MS_WINDOWS and support.Py_DEBUG, + 'test fails on Windows debug build') +# Building and running an extension in clang sanitizing mode is not +# straightforward +@support.skip_if_sanitizer('test does not work with analyzing builds', + address=True, memory=True, ub=True, thread=True) +# the test uses venv+pip: skip if it's not available +@support.requires_venv_with_pip() @support.requires_subprocess() +@support.requires_resource('cpu') class TestCPPExt(unittest.TestCase): - @support.requires_resource('cpu') - def test_build_cpp11(self): - self.check_build(False, '_testcpp11ext') + def test_build(self): + self.check_build('_testcppext') - @support.requires_resource('cpu') def test_build_cpp03(self): - self.check_build(True, '_testcpp03ext') - - # With MSVC, the linker fails with: cannot open file 'python311.lib' - # https://github.com/python/cpython/pull/32175#issuecomment-1111175897 - @unittest.skipIf(support.MS_WINDOWS, 'test fails on Windows') - # Building and running an extension in clang sanitizing mode is not - # straightforward - @unittest.skipIf( - '-fsanitize' in (sysconfig.get_config_var('PY_CFLAGS') or ''), - 'test does not work with analyzing builds') - # the test uses venv+pip: skip if it's not available - @support.requires_venv_with_pip() - def check_build(self, std_cpp03, extension_name): + self.check_build('_testcpp03ext', std='c++03') + + @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c++11") + def test_build_cpp11(self): + self.check_build('_testcpp11ext', std='c++11') + + # Only test C++14 on MSVC. + # On s390x RHEL7, GCC 4.8.5 doesn't support C++14. + @unittest.skipIf(not support.MS_WINDOWS, "need Windows") + def test_build_cpp14(self): + self.check_build('_testcpp14ext', std='c++14') + + @support.requires_gil_enabled('incompatible with Free Threading') + def test_build_limited(self): + self.check_build('_testcppext_limited', limited=True) + + def check_build(self, extension_name, std=None, limited=False): venv_dir = 'env' with support.setup_venv_with_pip_setuptools_wheel(venv_dir) as python_exe: - self._check_build(std_cpp03, extension_name, python_exe) + self._check_build(extension_name, python_exe, + std=std, limited=limited) - def _check_build(self, std_cpp03, extension_name, python_exe): + def _check_build(self, extension_name, python_exe, std, limited): pkg_dir = 'pkg' os.mkdir(pkg_dir) shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) @@ -49,10 +59,13 @@ def _check_build(self, std_cpp03, extension_name, python_exe): def run_cmd(operation, cmd): env = os.environ.copy() - env['CPYTHON_TEST_CPP_STD'] = 'c++03' if std_cpp03 else 'c++11' + if std: + env['CPYTHON_TEST_CPP_STD'] = std + if limited: + env['CPYTHON_TEST_LIMITED'] = '1' env['CPYTHON_TEST_EXT_NAME'] = extension_name if support.verbose: - print('Run:', ' '.join(cmd)) + print('Run:', ' '.join(map(shlex.quote, cmd))) subprocess.run(cmd, check=True, env=env) else: proc = subprocess.run(cmd, @@ -61,6 +74,7 @@ def run_cmd(operation, cmd): stderr=subprocess.STDOUT, text=True) if proc.returncode: + print('Run:', ' '.join(map(shlex.quote, cmd))) print(proc.stdout, end='') self.fail( f"{operation} failed with exit code {proc.returncode}") @@ -69,6 +83,8 @@ def run_cmd(operation, cmd): cmd = [python_exe, '-X', 'dev', '-m', 'pip', 'install', '--no-build-isolation', os.path.abspath(pkg_dir)] + if support.verbose: + cmd.append('-v') run_cmd('Install', cmd) # Do a reference run. Until we test that running python diff --git a/Lib/test/test_cppext/extension.cpp b/Lib/test/test_cppext/extension.cpp index 90669b10cb2c6d..500d5918145c00 100644 --- a/Lib/test/test_cppext/extension.cpp +++ b/Lib/test/test_cppext/extension.cpp @@ -8,10 +8,8 @@ #include "Python.h" -#if __cplusplus >= 201103 -# define NAME _testcpp11ext -#else -# define NAME _testcpp03ext +#ifndef MODULE_NAME +# error "MODULE_NAME macro must be defined" #endif #define _STR(NAME) #NAME @@ -64,6 +62,7 @@ test_api_casts(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) Py_ssize_t refcnt = Py_REFCNT(obj); assert(refcnt >= 1); +#ifndef Py_LIMITED_API // gh-92138: For backward compatibility, functions of Python C API accepts // "const PyObject*". Check that using it does not emit C++ compiler // warnings. @@ -76,6 +75,7 @@ test_api_casts(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) assert(PyTuple_GET_SIZE(const_obj) == 2); PyObject *one = PyTuple_GET_ITEM(const_obj, 0); assert(PyLong_AsLong(one) == 1); +#endif // gh-92898: StrongRef doesn't inherit from PyObject but has an operator to // cast to PyObject*. @@ -108,6 +108,12 @@ test_unicode(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) } assert(PyUnicode_Check(str)); + + assert(PyUnicode_GetLength(str) == 3); + assert(PyUnicode_ReadChar(str, 0) == 'a'); + assert(PyUnicode_ReadChar(str, 1) == 'b'); + +#ifndef Py_LIMITED_API assert(PyUnicode_GET_LENGTH(str) == 3); // gh-92800: test PyUnicode_READ() @@ -123,6 +129,7 @@ test_unicode(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) assert(PyUnicode_READ(ukind, const_data, 2) == 'c'); assert(PyUnicode_READ_CHAR(str, 1) == 'b'); +#endif Py_DECREF(str); Py_RETURN_NONE; @@ -160,7 +167,7 @@ PyType_Slot VirtualPyObject_Slots[] = { }; PyType_Spec VirtualPyObject_Spec = { - /* .name */ STR(NAME) ".VirtualPyObject", + /* .name */ STR(MODULE_NAME) ".VirtualPyObject", /* .basicsize */ sizeof(VirtualPyObject), /* .itemsize */ 0, /* .flags */ Py_TPFLAGS_DEFAULT, @@ -227,6 +234,10 @@ _testcppext_exec(PyObject *module) if (!result) return -1; Py_DECREF(result); + // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR() + Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int)); + assert(Py_BUILD_ASSERT_EXPR(sizeof(int) == sizeof(unsigned int)) == 0); + return 0; } @@ -240,7 +251,7 @@ PyDoc_STRVAR(_testcppext_doc, "C++ test extension."); static struct PyModuleDef _testcppext_module = { PyModuleDef_HEAD_INIT, // m_base - STR(NAME), // m_name + STR(MODULE_NAME), // m_name _testcppext_doc, // m_doc 0, // m_size _testcppext_methods, // m_methods @@ -254,7 +265,7 @@ static struct PyModuleDef _testcppext_module = { #define FUNC_NAME(NAME) _FUNC_NAME(NAME) PyMODINIT_FUNC -FUNC_NAME(NAME)(void) +FUNC_NAME(MODULE_NAME)(void) { return PyModuleDef_Init(&_testcppext_module); } diff --git a/Lib/test/test_cppext/setup.py b/Lib/test/test_cppext/setup.py index c7ba1efb4dd05a..019ff18446a2eb 100644 --- a/Lib/test/test_cppext/setup.py +++ b/Lib/test/test_cppext/setup.py @@ -1,8 +1,8 @@ # gh-91321: Build a basic C++ test extension to check that the Python C API is # compatible with C++ and does not emit C++ compiler warnings. import os +import platform import shlex -import sys import sysconfig from test import support @@ -10,6 +10,7 @@ SOURCE = 'extension.cpp' + if not support.MS_WINDOWS: # C++ compiler flags for GCC and clang CPPFLAGS = [ @@ -19,34 +20,83 @@ '-Werror', ] else: - # Don't pass any compiler flag to MSVC - CPPFLAGS = [] + # MSVC compiler flags + CPPFLAGS = [ + # Display warnings level 1 to 4 + '/W4', + # Treat all compiler warnings as compiler errors + '/WX', + ] def main(): cppflags = list(CPPFLAGS) - std = os.environ["CPYTHON_TEST_CPP_STD"] - name = os.environ["CPYTHON_TEST_EXT_NAME"] + std = os.environ.get("CPYTHON_TEST_CPP_STD", "") + module_name = os.environ["CPYTHON_TEST_EXT_NAME"] + limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", "")) + + cppflags = list(CPPFLAGS) + cppflags.append(f'-DMODULE_NAME={module_name}') - cppflags = [*CPPFLAGS, f'-std={std}'] + # Add -std=STD or /std:STD (MSVC) compiler flag + if std: + if support.MS_WINDOWS: + cppflags.append(f'/std:{std}') + else: + cppflags.append(f'-std={std}') # gh-105776: When "gcc -std=11" is used as the C++ compiler, -std=c11 # option emits a C++ compiler warning. Remove "-std11" option from the # CC command. cmd = (sysconfig.get_config_var('CC') or '') if cmd is not None: + if support.MS_WINDOWS: + std_prefix = '/std' + else: + std_prefix = '-std' cmd = shlex.split(cmd) - cmd = [arg for arg in cmd if not arg.startswith('-std=')] + cmd = [arg for arg in cmd if not arg.startswith(std_prefix)] cmd = shlex.join(cmd) # CC env var overrides sysconfig CC variable in setuptools os.environ['CC'] = cmd - cpp_ext = Extension( - name, + # Define Py_LIMITED_API macro + if limited: + version = sys.hexversion + cppflags.append(f'-DPy_LIMITED_API={version:#x}') + + # On Windows, add PCbuild\amd64\ to include and library directories + include_dirs = [] + library_dirs = [] + if support.MS_WINDOWS: + srcdir = sysconfig.get_config_var('srcdir') + machine = platform.uname().machine + pcbuild = os.path.join(srcdir, 'PCbuild', machine) + if os.path.exists(pcbuild): + # pyconfig.h is generated in PCbuild\amd64\ + include_dirs.append(pcbuild) + # python313.lib is generated in PCbuild\amd64\ + library_dirs.append(pcbuild) + print(f"Add PCbuild directory: {pcbuild}") + + # Display information to help debugging + for env_name in ('CC', 'CFLAGS', 'CPPFLAGS'): + if env_name in os.environ: + print(f"{env_name} env var: {os.environ[env_name]!r}") + else: + print(f"{env_name} env var: ") + print(f"extra_compile_args: {cppflags!r}") + + ext = Extension( + module_name, sources=[SOURCE], language='c++', - extra_compile_args=cppflags) - setup(name='internal' + name, version='0.0', ext_modules=[cpp_ext]) + extra_compile_args=cppflags, + include_dirs=include_dirs, + library_dirs=library_dirs) + setup(name=f'internal_{module_name}', + version='0.0', + ext_modules=[ext]) if __name__ == "__main__": diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index 27e8a767903777..65720871d5c5f0 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -30,6 +30,59 @@ def test_bad_counter_during_dealloc(self): self.assertEqual(cm.unraisable.exc_type, TypeError) + def test_crash_with_not_enough_args(self): + # gh-126220 + import _lsprof + + for profile in [_lsprof.Profiler(), cProfile.Profile()]: + for method in [ + "_pystart_callback", + "_pyreturn_callback", + "_ccall_callback", + "_creturn_callback", + ]: + with self.subTest(profile=profile, method=method): + method_obj = getattr(profile, method) + with self.assertRaises(TypeError): + method_obj() # should not crash + + def test_evil_external_timer(self): + # gh-120289 + # Disabling profiler in external timer should not crash + import _lsprof + class EvilTimer(): + def __init__(self, disable_count): + self.count = 0 + self.disable_count = disable_count + + def __call__(self): + self.count += 1 + if self.count == self.disable_count: + profiler_with_evil_timer.disable() + return self.count + + # this will trigger external timer to disable profiler at + # call event - in initContext in _lsprof.c + with support.catch_unraisable_exception() as cm: + profiler_with_evil_timer = _lsprof.Profiler(EvilTimer(1)) + profiler_with_evil_timer.enable() + # Make a call to trigger timer + (lambda: None)() + profiler_with_evil_timer.disable() + profiler_with_evil_timer.clear() + self.assertEqual(cm.unraisable.exc_type, RuntimeError) + + # this will trigger external timer to disable profiler at + # return event - in Stop in _lsprof.c + with support.catch_unraisable_exception() as cm: + profiler_with_evil_timer = _lsprof.Profiler(EvilTimer(2)) + profiler_with_evil_timer.enable() + # Make a call to trigger timer + (lambda: None)() + profiler_with_evil_timer.disable() + profiler_with_evil_timer.clear() + self.assertEqual(cm.unraisable.exc_type, RuntimeError) + def test_profile_enable_disable(self): prof = self.profilerclass() # Make sure we clean ourselves up if the test fails for some reason. diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index d74ab7e016f78c..4af8f7f480e759 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001,2002 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # csv package unit tests import copy @@ -454,6 +454,10 @@ def test_read_quoting(self): quoting=csv.QUOTE_STRINGS) self._read_test(['1,@,3,@,5'], [['1', ',3,', '5']], quotechar='@') self._read_test(['1,\0,3,\0,5'], [['1', ',3,', '5']], quotechar='\0') + self._read_test(['1\\.5,\\.5,.5'], [[1.5, 0.5, 0.5]], + quoting=csv.QUOTE_NONNUMERIC, escapechar='\\') + self._read_test(['1\\.5,\\.5,"\\.5"'], [[1.5, 0.5, ".5"]], + quoting=csv.QUOTE_STRINGS, escapechar='\\') def test_read_skipinitialspace(self): self._read_test(['no space, space, spaces,\ttab'], @@ -1099,6 +1103,12 @@ class mydialect(csv.Dialect): mydialect.quoting = None self.assertRaises(csv.Error, mydialect) + mydialect.quoting = 42 + with self.assertRaises(csv.Error) as cm: + mydialect() + self.assertEqual(str(cm.exception), + 'bad "quoting" value') + mydialect.doublequote = True mydialect.quoting = csv.QUOTE_ALL mydialect.quotechar = '"' diff --git a/Lib/test/test_ctypes/test_arrays.py b/Lib/test/test_ctypes/test_arrays.py index 774316e227ff73..7f1f6cf58402c9 100644 --- a/Lib/test/test_ctypes/test_arrays.py +++ b/Lib/test/test_ctypes/test_arrays.py @@ -1,12 +1,11 @@ import ctypes import sys import unittest -import warnings -from ctypes import (Structure, Array, sizeof, addressof, +from ctypes import (Structure, Array, ARRAY, sizeof, addressof, create_string_buffer, create_unicode_buffer, c_char, c_wchar, c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, c_long, c_ulonglong, c_float, c_double, c_longdouble) -from test.support import bigmemtest, _2G +from test.support import bigmemtest, _2G, threading_helper, Py_GIL_DISABLED from ._support import (_CData, PyCArrayType, Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE) @@ -17,13 +16,6 @@ c_long, c_ulonglong, c_float, c_double, c_longdouble -def ARRAY(*args): - # ignore DeprecationWarning in tests - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - return ctypes.ARRAY(*args) - - class ArrayTestCase(unittest.TestCase): def test_inheritance_hierarchy(self): self.assertEqual(Array.mro(), [Array, _CData, object]) @@ -37,6 +29,22 @@ def test_type_flags(self): self.assertTrue(cls.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) self.assertFalse(cls.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + def test_metaclass_details(self): + # Abstract classes (whose metaclass __init__ was not called) can't be + # instantiated directly + NewArray = PyCArrayType.__new__(PyCArrayType, 'NewArray', (Array,), {}) + for cls in Array, NewArray: + with self.subTest(cls=cls): + with self.assertRaisesRegex(TypeError, "abstract class"): + obj = cls() + + # Cannot call the metaclass __init__ more than once + class T(Array): + _type_ = c_int + _length_ = 13 + with self.assertRaisesRegex(SystemError, "already initialized"): + PyCArrayType.__init__(T, 'ptr', (), {}) + def test_simple(self): # create classes holding simple numeric types, and check # various properties. @@ -237,7 +245,7 @@ def test_empty_element_struct(self): class EmptyStruct(Structure): _fields_ = [] - obj = (EmptyStruct * 2)() # bpo37188: Floating point exception + obj = (EmptyStruct * 2)() # bpo37188: Floating-point exception self.assertEqual(sizeof(obj), 0) def test_empty_element_array(self): @@ -245,7 +253,7 @@ class EmptyArray(Array): _type_ = c_int _length_ = 0 - obj = (EmptyArray * 2)() # bpo37188: Floating point exception + obj = (EmptyArray * 2)() # bpo37188: Floating-point exception self.assertEqual(sizeof(obj), 0) def test_bpo36504_signed_int_overflow(self): @@ -259,9 +267,25 @@ def test_bpo36504_signed_int_overflow(self): def test_large_array(self, size): c_char * size - def test_deprecation(self): - with self.assertWarns(DeprecationWarning): - CharArray = ctypes.ARRAY(c_char, 3) + @threading_helper.requires_working_threading() + @unittest.skipUnless(Py_GIL_DISABLED, "only meaningful if the GIL is disabled") + def test_thread_safety(self): + from threading import Thread + + buffer = (ctypes.c_char_p * 10)() + + def run(): + for i in range(100): + buffer.value = b"hello" + buffer[0] = b"j" + + with threading_helper.catch_threading_exception() as cm: + threads = (Thread(target=run) for _ in range(25)) + with threading_helper.start_threads(threads): + pass + + if cm.exc_value: + raise cm.exc_value if __name__ == '__main__': diff --git a/Lib/test/test_ctypes/test_as_parameter.py b/Lib/test/test_ctypes/test_as_parameter.py index ca75e748256083..c5e1840b0eb7af 100644 --- a/Lib/test/test_ctypes/test_as_parameter.py +++ b/Lib/test/test_ctypes/test_as_parameter.py @@ -1,4 +1,3 @@ -import _ctypes_test import ctypes import unittest from ctypes import (Structure, CDLL, CFUNCTYPE, @@ -6,6 +5,8 @@ c_short, c_int, c_long, c_longlong, c_byte, c_wchar, c_float, c_double, ArgumentError) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") dll = CDLL(_ctypes_test.__file__) @@ -197,8 +198,16 @@ class A: a = A() a._as_parameter_ = a - with self.assertRaises(RecursionError): - c_int.from_param(a) + for c_type in ( + ctypes.c_wchar_p, + ctypes.c_char_p, + ctypes.c_void_p, + ctypes.c_int, # PyCSimpleType + POINT, # CDataType + ): + with self.subTest(c_type=c_type): + with self.assertRaises(RecursionError): + c_type.from_param(a) class AsParamWrapper: diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index d43c56ad371fbd..19ba2f4484e7da 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -1,12 +1,16 @@ -import _ctypes_test import os +import sys import unittest from ctypes import (CDLL, Structure, sizeof, POINTER, byref, alignment, LittleEndianStructure, BigEndianStructure, c_byte, c_ubyte, c_char, c_char_p, c_void_p, c_wchar, - c_uint32, c_uint64, - c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong) + c_uint8, c_uint16, c_uint32, c_uint64, + c_short, c_ushort, c_int, c_uint, c_long, c_ulong, + c_longlong, c_ulonglong, + Union) from test import support +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class BITS(Structure): @@ -32,27 +36,88 @@ class BITS(Structure): func.argtypes = POINTER(BITS), c_char +class BITS_msvc(Structure): + _layout_ = "ms" + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9), + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + + +try: + func_msvc = CDLL(_ctypes_test.__file__).unpack_bitfields_msvc +except AttributeError as err: + # The MSVC struct must be available on Windows; it's optional elsewhere + if support.MS_WINDOWS: + raise err + func_msvc = None +else: + func_msvc.argtypes = POINTER(BITS_msvc), c_char + + class C_Test(unittest.TestCase): def test_ints(self): for i in range(512): for name in "ABCDEFGHI": - b = BITS() - setattr(b, name, i) - self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii'))) + with self.subTest(i=i, name=name): + b = BITS() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func(byref(b), (name.encode('ascii')))) - # bpo-46913: _ctypes/cfield.c h_get() has an undefined behavior - @support.skip_if_sanitizer(ub=True) def test_shorts(self): b = BITS() name = "M" + # See Modules/_ctypes/_ctypes_test.c for where the magic 999 comes from. if func(byref(b), name.encode('ascii')) == 999: + # unpack_bitfields and unpack_bitfields_msvc in + # Modules/_ctypes/_ctypes_test.c return 999 to indicate + # an invalid name. 'M' is only valid, if signed short bitfields + # are supported by the C compiler. self.skipTest("Compiler does not support signed short bitfields") for i in range(256): for name in "MNOPQRS": - b = BITS() - setattr(b, name, i) - self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii'))) + with self.subTest(i=i, name=name): + b = BITS() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func(byref(b), (name.encode('ascii')))) + + @unittest.skipUnless(func_msvc, "need MSVC or __attribute__((ms_struct))") + def test_shorts_msvc_mode(self): + b = BITS_msvc() + name = "M" + # See Modules/_ctypes/_ctypes_test.c for where the magic 999 comes from. + if func_msvc(byref(b), name.encode('ascii')) == 999: + # unpack_bitfields and unpack_bitfields_msvc in + # Modules/_ctypes/_ctypes_test.c return 999 to indicate + # an invalid name. 'M' is only valid, if signed short bitfields + # are supported by the C compiler. + self.skipTest("Compiler does not support signed short bitfields") + for i in range(256): + for name in "MNOPQRS": + with self.subTest(i=i, name=name): + b = BITS_msvc() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func_msvc(byref(b), name.encode('ascii'))) signed_int_types = (c_byte, c_short, c_int, c_long, c_longlong) @@ -86,39 +151,47 @@ class X(Structure): def test_signed(self): for c_typ in signed_int_types: - class X(Structure): - _fields_ = [("dummy", c_typ), - ("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)*2) - - x = X() - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) - x.a = -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, -1, 0, 0)) - x.a, x.b = 0, -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, -1, 0)) + with self.subTest(c_typ): + if sizeof(c_typ) != alignment(c_typ): + self.skipTest('assumes size=alignment') + class X(Structure): + _fields_ = [("dummy", c_typ), + ("a", c_typ, 3), + ("b", c_typ, 3), + ("c", c_typ, 1)] + self.assertEqual(sizeof(X), sizeof(c_typ)*2) + + x = X() + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) + x.a = -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, -1, 0, 0)) + x.a, x.b = 0, -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, -1, 0)) def test_unsigned(self): for c_typ in unsigned_int_types: - class X(Structure): - _fields_ = [("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)) - - x = X() - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) - x.a = -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 7, 0, 0)) - x.a, x.b = 0, -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0)) + with self.subTest(c_typ): + if sizeof(c_typ) != alignment(c_typ): + self.skipTest('assumes size=alignment') + class X(Structure): + _fields_ = [("a", c_typ, 3), + ("b", c_typ, 3), + ("c", c_typ, 1)] + self.assertEqual(sizeof(X), sizeof(c_typ)) + + x = X() + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) + x.a = -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 7, 0, 0)) + x.a, x.b = 0, -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0)) def fail_fields(self, *fields): - return self.get_except(type(Structure), "X", (), - {"_fields_": fields}) + for layout in "ms", "gcc-sysv": + with self.subTest(layout=layout): + return self.get_except(type(Structure), "X", (), + {"_fields_": fields, "layout": layout}) def test_nonint_types(self): # bit fields are not allowed on non-integer types. @@ -135,9 +208,15 @@ def test_nonint_types(self): result = self.fail_fields(("a", c_char, 1)) self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char')) - class Dummy(Structure): + class Empty(Structure): _fields_ = [] + result = self.fail_fields(("a", Empty, 1)) + self.assertEqual(result, (ValueError, "number of bits invalid for bit field 'a'")) + + class Dummy(Structure): + _fields_ = [("x", c_int)] + result = self.fail_fields(("a", Dummy, 1)) self.assertEqual(result, (TypeError, 'bit fields not allowed for type Dummy')) @@ -148,22 +227,28 @@ def test_c_wchar(self): def test_single_bitfield_size(self): for c_typ in int_types: - result = self.fail_fields(("a", c_typ, -1)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) + with self.subTest(c_typ): + if sizeof(c_typ) != alignment(c_typ): + self.skipTest('assumes size=alignment') + result = self.fail_fields(("a", c_typ, -1)) + self.assertEqual(result, (ValueError, + "number of bits invalid for bit field 'a'")) - result = self.fail_fields(("a", c_typ, 0)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) + result = self.fail_fields(("a", c_typ, 0)) + self.assertEqual(result, (ValueError, + "number of bits invalid for bit field 'a'")) - class X(Structure): - _fields_ = [("a", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)) + class X(Structure): + _fields_ = [("a", c_typ, 1)] + self.assertEqual(sizeof(X), sizeof(c_typ)) - class X(Structure): - _fields_ = [("a", c_typ, sizeof(c_typ)*8)] - self.assertEqual(sizeof(X), sizeof(c_typ)) + class X(Structure): + _fields_ = [("a", c_typ, sizeof(c_typ)*8)] + self.assertEqual(sizeof(X), sizeof(c_typ)) - result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) + result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) + self.assertEqual(result, (ValueError, + "number of bits invalid for bit field 'a'")) def test_multi_bitfields_size(self): class X(Structure): @@ -235,6 +320,161 @@ class X(Structure): else: self.assertEqual(sizeof(X), sizeof(c_int) * 2) + def test_mixed_5(self): + class X(Structure): + _fields_ = [ + ('A', c_uint, 1), + ('B', c_ushort, 16)] + a = X() + a.A = 0 + a.B = 1 + self.assertEqual(1, a.B) + + def test_mixed_6(self): + class X(Structure): + _fields_ = [ + ('A', c_ulonglong, 1), + ('B', c_uint, 32)] + a = X() + a.A = 0 + a.B = 1 + self.assertEqual(1, a.B) + + @unittest.skipIf(sizeof(c_uint64) != alignment(c_uint64), + 'assumes size=alignment') + def test_mixed_7(self): + class X(Structure): + _fields_ = [ + ("A", c_uint32), + ('B', c_uint32, 20), + ('C', c_uint64, 24)] + self.assertEqual(16, sizeof(X)) + + def test_mixed_8(self): + class Foo(Structure): + _fields_ = [ + ("A", c_uint32), + ("B", c_uint32, 32), + ("C", c_ulonglong, 1), + ] + + class Bar(Structure): + _fields_ = [ + ("A", c_uint32), + ("B", c_uint32), + ("C", c_ulonglong, 1), + ] + self.assertEqual(sizeof(Foo), sizeof(Bar)) + + def test_mixed_9(self): + class X(Structure): + _fields_ = [ + ("A", c_uint8), + ("B", c_uint32, 1), + ] + if sys.platform == 'win32': + self.assertEqual(8, sizeof(X)) + else: + self.assertEqual(4, sizeof(X)) + + @unittest.skipIf(sizeof(c_uint64) != alignment(c_uint64), + 'assumes size=alignment') + def test_mixed_10(self): + class X(Structure): + _fields_ = [ + ("A", c_uint32, 1), + ("B", c_uint64, 1), + ] + if sys.platform == 'win32': + self.assertEqual(8, alignment(X)) + self.assertEqual(16, sizeof(X)) + else: + self.assertEqual(8, alignment(X)) + self.assertEqual(8, sizeof(X)) + + def test_gh_95496(self): + for field_width in range(1, 33): + class TestStruct(Structure): + _fields_ = [ + ("Field1", c_uint32, field_width), + ("Field2", c_uint8, 8) + ] + + cmd = TestStruct() + cmd.Field2 = 1 + self.assertEqual(1, cmd.Field2) + + def test_gh_84039(self): + class Bad(Structure): + _pack_ = 1 + _fields_ = [ + ("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12), + ] + + + class GoodA(Structure): + _pack_ = 1 + _fields_ = [ + ("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1), + ] + + + class Good(Structure): + _pack_ = 1 + _fields_ = [ + ("a", GoodA), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12), + ] + + self.assertEqual(3, sizeof(Bad)) + self.assertEqual(3, sizeof(Good)) + + def test_gh_73939(self): + class MyStructure(Structure): + _pack_ = 1 + _fields_ = [ + ("P", c_uint16), + ("L", c_uint16, 9), + ("Pro", c_uint16, 1), + ("G", c_uint16, 1), + ("IB", c_uint16, 1), + ("IR", c_uint16, 1), + ("R", c_uint16, 3), + ("T", c_uint32, 10), + ("C", c_uint32, 20), + ("R2", c_uint32, 2) + ] + self.assertEqual(8, sizeof(MyStructure)) + + def test_gh_86098(self): + class X(Structure): + _fields_ = [ + ("a", c_uint8, 8), + ("b", c_uint8, 8), + ("c", c_uint32, 16) + ] + if sys.platform == 'win32': + self.assertEqual(8, sizeof(X)) + else: + self.assertEqual(4, sizeof(X)) + def test_anon_bitfields(self): # anonymous bit-fields gave a strange error message class X(Structure): @@ -288,6 +528,21 @@ class Big(BigEndianStructure): x.c = 2 self.assertEqual(b, b'\xab\xcd\xef\x12') + def test_union_bitfield(self): + class BitfieldUnion(Union): + _fields_ = [("a", c_uint32, 1), + ("b", c_uint32, 2), + ("c", c_uint32, 3)] + self.assertEqual(sizeof(BitfieldUnion), 4) + b = bytearray(4) + x = BitfieldUnion.from_buffer(b) + x.a = 1 + self.assertEqual(int.from_bytes(b).bit_count(), 1) + x.b = 3 + self.assertEqual(int.from_bytes(b).bit_count(), 2) + x.c = 7 + self.assertEqual(int.from_bytes(b).bit_count(), 3) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py new file mode 100644 index 00000000000000..b446fd5c77dde2 --- /dev/null +++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py @@ -0,0 +1,170 @@ +import unittest +from test.support import MS_WINDOWS +import ctypes +from ctypes import POINTER, c_void_p + +from ._support import PyCSimpleType + + +class PyCSimpleTypeAsMetaclassTest(unittest.TestCase): + def tearDown(self): + # to not leak references, we must clean _pointer_type_cache + ctypes._reset_cache() + + def test_creating_pointer_in_dunder_new_1(self): + # Test metaclass whose instances are C types; when the type is + # created it automatically creates a pointer type for itself. + # The pointer type is also an instance of the metaclass. + # Such an implementation is used in `IUnknown` of the `comtypes` + # project. See gh-124520. + + class ct_meta(type): + def __new__(cls, name, bases, namespace): + self = super().__new__(cls, name, bases, namespace) + + # Avoid recursion: don't set up a pointer to + # a pointer (to a pointer...) + if bases == (c_void_p,): + # When creating PtrBase itself, the name + # is not yet available + return self + if issubclass(self, PtrBase): + return self + + if bases == (object,): + ptr_bases = (self, PtrBase) + else: + ptr_bases = (self, POINTER(bases[0])) + p = p_meta(f"POINTER({self.__name__})", ptr_bases, {}) + ctypes._pointer_type_cache[self] = p + return self + + class p_meta(PyCSimpleType, ct_meta): + pass + + class PtrBase(c_void_p, metaclass=p_meta): + pass + + class CtBase(object, metaclass=ct_meta): + pass + + class Sub(CtBase): + pass + + class Sub2(Sub): + pass + + self.assertIsInstance(POINTER(Sub2), p_meta) + self.assertIsSubclass(POINTER(Sub2), Sub2) + self.assertIsSubclass(POINTER(Sub2), POINTER(Sub)) + self.assertIsSubclass(POINTER(Sub), POINTER(CtBase)) + + def test_creating_pointer_in_dunder_new_2(self): + # A simpler variant of the above, used in `CoClass` of the `comtypes` + # project. + + class ct_meta(type): + def __new__(cls, name, bases, namespace): + self = super().__new__(cls, name, bases, namespace) + if isinstance(self, p_meta): + return self + p = p_meta(f"POINTER({self.__name__})", (self, c_void_p), {}) + ctypes._pointer_type_cache[self] = p + return self + + class p_meta(PyCSimpleType, ct_meta): + pass + + class Core(object): + pass + + class CtBase(Core, metaclass=ct_meta): + pass + + class Sub(CtBase): + pass + + self.assertIsInstance(POINTER(Sub), p_meta) + self.assertIsSubclass(POINTER(Sub), Sub) + + def test_creating_pointer_in_dunder_init_1(self): + class ct_meta(type): + def __init__(self, name, bases, namespace): + super().__init__(name, bases, namespace) + + # Avoid recursion. + # (See test_creating_pointer_in_dunder_new_1) + if bases == (c_void_p,): + return + if issubclass(self, PtrBase): + return + if bases == (object,): + ptr_bases = (self, PtrBase) + else: + ptr_bases = (self, POINTER(bases[0])) + p = p_meta(f"POINTER({self.__name__})", ptr_bases, {}) + ctypes._pointer_type_cache[self] = p + + class p_meta(PyCSimpleType, ct_meta): + pass + + class PtrBase(c_void_p, metaclass=p_meta): + pass + + class CtBase(object, metaclass=ct_meta): + pass + + class Sub(CtBase): + pass + + class Sub2(Sub): + pass + + self.assertIsInstance(POINTER(Sub2), p_meta) + self.assertIsSubclass(POINTER(Sub2), Sub2) + self.assertIsSubclass(POINTER(Sub2), POINTER(Sub)) + self.assertIsSubclass(POINTER(Sub), POINTER(CtBase)) + + def test_creating_pointer_in_dunder_init_2(self): + class ct_meta(type): + def __init__(self, name, bases, namespace): + super().__init__(name, bases, namespace) + + # Avoid recursion. + # (See test_creating_pointer_in_dunder_new_2) + if isinstance(self, p_meta): + return + p = p_meta(f"POINTER({self.__name__})", (self, c_void_p), {}) + ctypes._pointer_type_cache[self] = p + + class p_meta(PyCSimpleType, ct_meta): + pass + + class Core(object): + pass + + class CtBase(Core, metaclass=ct_meta): + pass + + class Sub(CtBase): + pass + + self.assertIsInstance(POINTER(Sub), p_meta) + self.assertIsSubclass(POINTER(Sub), Sub) + + def test_bad_type_message(self): + """Verify the error message that lists all available type codes""" + # (The string is generated at runtime, so this checks the underlying + # set of types as well as correct construction of the string.) + with self.assertRaises(AttributeError) as cm: + class F(metaclass=PyCSimpleType): + _type_ = "\0" + message = str(cm.exception) + expected_type_chars = list('cbBhHiIlLdCEFfuzZqQPXOv?g') + if not hasattr(ctypes, 'c_float_complex'): + expected_type_chars.remove('C') + expected_type_chars.remove('E') + expected_type_chars.remove('F') + if not MS_WINDOWS: + expected_type_chars.remove('X') + self.assertIn("'" + ''.join(expected_type_chars) + "'", message) diff --git a/Lib/test/test_ctypes/test_callbacks.py b/Lib/test/test_ctypes/test_callbacks.py index 64f92ffdca6a3f..8f483dfe1db801 100644 --- a/Lib/test/test_ctypes/test_callbacks.py +++ b/Lib/test/test_ctypes/test_callbacks.py @@ -1,4 +1,3 @@ -import _ctypes_test import ctypes import functools import gc @@ -14,6 +13,8 @@ c_float, c_double, c_longdouble, py_object) from ctypes.util import find_library from test import support +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class Callbacks(unittest.TestCase): @@ -106,7 +107,7 @@ def test_pyobject(self): def test_unsupported_restype_1(self): # Only "fundamental" result types are supported for callback - # functions, the type must have a non-NULL stgdict->setfunc. + # functions, the type must have a non-NULL stginfo->setfunc. # POINTER(c_double), for example, is not supported. prototype = self.functype.__func__(POINTER(c_double)) diff --git a/Lib/test/test_ctypes/test_cfuncs.py b/Lib/test/test_ctypes/test_cfuncs.py index 6ff0878a35da2f..e0c124607cb2e9 100644 --- a/Lib/test/test_ctypes/test_cfuncs.py +++ b/Lib/test/test_ctypes/test_cfuncs.py @@ -1,4 +1,3 @@ -import _ctypes_test import ctypes import unittest from ctypes import (CDLL, @@ -6,6 +5,9 @@ c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double, c_longdouble) +from test import support +from test.support import import_helper, threading_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class CFunctions(unittest.TestCase): @@ -190,6 +192,23 @@ def test_void(self): self.assertEqual(self._dll.tv_i(-42), None) self.assertEqual(self.S(), -42) + @threading_helper.requires_working_threading() + @support.requires_resource("cpu") + @unittest.skipUnless(support.Py_GIL_DISABLED, "only meaningful on free-threading") + def test_thread_safety(self): + from threading import Thread + + def concurrent(): + for _ in range(100): + self._dll.tf_b.restype = c_byte + self._dll.tf_b.argtypes = (c_byte,) + + with threading_helper.catch_threading_exception() as exc: + with threading_helper.start_threads((Thread(target=concurrent) for _ in range(10))): + pass + + self.assertIsNone(exc.exc_value) + # The following repeats the above tests with stdcall functions (where # they are available) diff --git a/Lib/test/test_ctypes/test_checkretval.py b/Lib/test/test_ctypes/test_checkretval.py index 5dc9e25aa38226..9d6bfdb845e6c7 100644 --- a/Lib/test/test_ctypes/test_checkretval.py +++ b/Lib/test/test_ctypes/test_checkretval.py @@ -1,7 +1,8 @@ -import _ctypes_test import ctypes import unittest from ctypes import CDLL, c_int +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class CHECKED(c_int): diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py new file mode 100644 index 00000000000000..6bf492399cbf95 --- /dev/null +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -0,0 +1,176 @@ +import _ctypes +import os +import platform +import sys +import test.support +import unittest +from ctypes import CDLL, c_int +from ctypes.util import find_library + + +FOO_C = r""" +#include + +/* This is a 'GNU indirect function' (IFUNC) that will be called by + dlsym() to resolve the symbol "foo" to an address. Typically, such + a function would return the address of an actual function, but it + can also just return NULL. For some background on IFUNCs, see + https://willnewton.name/uncategorized/using-gnu-indirect-functions. + + Adapted from Michael Kerrisk's answer: https://stackoverflow.com/a/53590014. +*/ + +asm (".type foo STT_GNU_IFUNC"); + +void *foo(void) +{ + write($DESCRIPTOR, "OK", 2); + return NULL; +} +""" + + +@unittest.skipUnless(sys.platform.startswith('linux'), + 'test requires GNU IFUNC support') +class TestNullDlsym(unittest.TestCase): + """GH-126554: Ensure that we catch NULL dlsym return values + + In rare cases, such as when using GNU IFUNCs, dlsym(), + the C function that ctypes' CDLL uses to get the address + of symbols, can return NULL. + + The objective way of telling if an error during symbol + lookup happened is to call glibc's dlerror() and check + for a non-NULL return value. + + However, there can be cases where dlsym() returns NULL + and dlerror() is also NULL, meaning that glibc did not + encounter any error. + + In the case of ctypes, we subjectively treat that as + an error, and throw a relevant exception. + + This test case ensures that we correctly enforce + this 'dlsym returned NULL -> throw Error' rule. + """ + + def test_null_dlsym(self): + import subprocess + import tempfile + + retcode = subprocess.call(["gcc", "--version"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + if retcode != 0: + self.skipTest("gcc is missing") + + pipe_r, pipe_w = os.pipe() + self.addCleanup(os.close, pipe_r) + self.addCleanup(os.close, pipe_w) + + with tempfile.TemporaryDirectory() as d: + # Create a C file with a GNU Indirect Function (FOO_C) + # and compile it into a shared library. + srcname = os.path.join(d, 'foo.c') + dstname = os.path.join(d, 'libfoo.so') + with open(srcname, 'w') as f: + f.write(FOO_C.replace('$DESCRIPTOR', str(pipe_w))) + args = ['gcc', '-fPIC', '-shared', '-o', dstname, srcname] + p = subprocess.run(args, capture_output=True) + + if p.returncode != 0: + # IFUNC is not supported on all architectures. + if platform.machine() == 'x86_64': + # It should be supported here. Something else went wrong. + p.check_returncode() + else: + # IFUNC might not be supported on this machine. + self.skipTest(f"could not compile indirect function: {p}") + + # Case #1: Test 'PyCFuncPtr_FromDll' from Modules/_ctypes/_ctypes.c + L = CDLL(dstname) + with self.assertRaisesRegex(AttributeError, "function 'foo' not found"): + # Try accessing the 'foo' symbol. + # It should resolve via dlsym() to NULL, + # and since we subjectively treat NULL + # addresses as errors, we should get + # an error. + L.foo + + # Assert that the IFUNC was called + self.assertEqual(os.read(pipe_r, 2), b'OK') + + # Case #2: Test 'CDataType_in_dll_impl' from Modules/_ctypes/_ctypes.c + with self.assertRaisesRegex(ValueError, "symbol 'foo' not found"): + c_int.in_dll(L, "foo") + + # Assert that the IFUNC was called + self.assertEqual(os.read(pipe_r, 2), b'OK') + + # Case #3: Test 'py_dl_sym' from Modules/_ctypes/callproc.c + dlopen = test.support.get_attribute(_ctypes, 'dlopen') + dlsym = test.support.get_attribute(_ctypes, 'dlsym') + L = dlopen(dstname) + with self.assertRaisesRegex(OSError, "symbol 'foo' not found"): + dlsym(L, "foo") + + # Assert that the IFUNC was called + self.assertEqual(os.read(pipe_r, 2), b'OK') + + +@unittest.skipUnless(os.name != 'nt', 'test requires dlerror() calls') +class TestLocalization(unittest.TestCase): + + @staticmethod + def configure_locales(func): + return test.support.run_with_locale( + 'LC_ALL', + 'fr_FR.iso88591', 'ja_JP.sjis', 'zh_CN.gbk', + 'fr_FR.utf8', 'en_US.utf8', + '', + )(func) + + @classmethod + def setUpClass(cls): + cls.libc_filename = find_library("c") + if cls.libc_filename is None: + raise unittest.SkipTest('cannot find libc') + + @configure_locales + def test_localized_error_from_dll(self): + dll = CDLL(self.libc_filename) + with self.assertRaises(AttributeError): + dll.this_name_does_not_exist + + @configure_locales + def test_localized_error_in_dll(self): + dll = CDLL(self.libc_filename) + with self.assertRaises(ValueError): + c_int.in_dll(dll, 'this_name_does_not_exist') + + @unittest.skipUnless(hasattr(_ctypes, 'dlopen'), + 'test requires _ctypes.dlopen()') + @configure_locales + def test_localized_error_dlopen(self): + missing_filename = b'missing\xff.so' + # Depending whether the locale, we may encode '\xff' differently + # but we are only interested in avoiding a UnicodeDecodeError + # when reporting the dlerror() error message which contains + # the localized filename. + filename_pattern = r'missing.*?\.so' + with self.assertRaisesRegex(OSError, filename_pattern): + _ctypes.dlopen(missing_filename, 2) + + @unittest.skipUnless(hasattr(_ctypes, 'dlopen'), + 'test requires _ctypes.dlopen()') + @unittest.skipUnless(hasattr(_ctypes, 'dlsym'), + 'test requires _ctypes.dlsym()') + @configure_locales + def test_localized_error_dlsym(self): + dll = _ctypes.dlopen(self.libc_filename) + with self.assertRaises(OSError): + _ctypes.dlsym(dll, 'this_name_does_not_exist') + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_ctypes/test_find.py b/Lib/test/test_ctypes/test_find.py index 7732ff37308848..85b28617d2d754 100644 --- a/Lib/test/test_ctypes/test_find.py +++ b/Lib/test/test_ctypes/test_find.py @@ -129,5 +129,28 @@ def test_gh114257(self): self.assertIsNone(find_library("libc")) +@unittest.skipUnless(sys.platform == 'android', 'Test only valid for Android') +class FindLibraryAndroid(unittest.TestCase): + def test_find(self): + for name in [ + "c", "m", # POSIX + "z", # Non-POSIX, but present on Linux + "log", # Not present on Linux + ]: + with self.subTest(name=name): + path = find_library(name) + self.assertIsInstance(path, str) + self.assertEqual( + os.path.dirname(path), + "/system/lib64" if "64" in os.uname().machine + else "/system/lib") + self.assertEqual(os.path.basename(path), f"lib{name}.so") + self.assertTrue(os.path.isfile(path), path) + + for name in ["libc", "nonexistent"]: + with self.subTest(name=name): + self.assertIsNone(find_library(name)) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_funcptr.py b/Lib/test/test_ctypes/test_funcptr.py index 0eed39484fb39e..8362fb16d94dcd 100644 --- a/Lib/test/test_ctypes/test_funcptr.py +++ b/Lib/test/test_ctypes/test_funcptr.py @@ -1,8 +1,9 @@ -import _ctypes_test import ctypes import unittest from ctypes import (CDLL, Structure, CFUNCTYPE, sizeof, _CFuncPtr, c_void_p, c_char_p, c_char, c_int, c_uint, c_long) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") from ._support import (_CData, PyCFuncPtrType, Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE) @@ -29,6 +30,12 @@ def test_type_flags(self): self.assertTrue(_CFuncPtr.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) self.assertFalse(_CFuncPtr.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + def test_metaclass_details(self): + # Cannot call the metaclass __init__ more than once + CdeclCallback = CFUNCTYPE(c_int, c_int, c_int) + with self.assertRaisesRegex(SystemError, "already initialized"): + PyCFuncPtrType.__init__(CdeclCallback, 'ptr', (), {}) + def test_basic(self): X = WINFUNCTYPE(c_int, c_int, c_int) diff --git a/Lib/test/test_ctypes/test_functions.py b/Lib/test/test_ctypes/test_functions.py index 04e8582ff1e427..3454b83d43e1e7 100644 --- a/Lib/test/test_ctypes/test_functions.py +++ b/Lib/test/test_ctypes/test_functions.py @@ -1,12 +1,13 @@ -import _ctypes_test import ctypes import sys import unittest from ctypes import (CDLL, Structure, Array, CFUNCTYPE, - byref, POINTER, pointer, ArgumentError, + byref, POINTER, pointer, ArgumentError, sizeof, c_char, c_wchar, c_byte, c_char_p, c_wchar_p, c_short, c_int, c_long, c_longlong, c_void_p, c_float, c_double, c_longdouble) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") from _ctypes import _Pointer, _SimpleCData @@ -71,7 +72,8 @@ def callback(*args): self.assertEqual(str(cm.exception), "argument 1: TypeError: one character bytes, " - "bytearray or integer expected") + "bytearray, or an integer in range(256) expected, " + "not bytes of length 3") def test_wchar_parm(self): f = dll._testfunc_i_bhilfd @@ -83,14 +85,27 @@ def test_wchar_parm(self): with self.assertRaises(ArgumentError) as cm: f(1, 2, 3, 4, 5.0, 6.0) self.assertEqual(str(cm.exception), - "argument 2: TypeError: unicode string expected " - "instead of int instance") + "argument 2: TypeError: a unicode character expected, " + "not instance of int") with self.assertRaises(ArgumentError) as cm: f(1, "abc", 3, 4, 5.0, 6.0) self.assertEqual(str(cm.exception), - "argument 2: TypeError: one character unicode string " - "expected") + "argument 2: TypeError: a unicode character expected, " + "not a string of length 3") + + with self.assertRaises(ArgumentError) as cm: + f(1, "", 3, 4, 5.0, 6.0) + self.assertEqual(str(cm.exception), + "argument 2: TypeError: a unicode character expected, " + "not a string of length 0") + + if sizeof(c_wchar) < 4: + with self.assertRaises(ArgumentError) as cm: + f(1, "\U0001f40d", 3, 4, 5.0, 6.0) + self.assertEqual(str(cm.exception), + "argument 2: TypeError: the string '\\U0001f40d' " + "cannot be converted to a single wchar_t character") def test_c_char_p_parm(self): """Test the error message when converting an incompatible type to c_char_p.""" diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py new file mode 100644 index 00000000000000..1df9f0dc16368f --- /dev/null +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -0,0 +1,732 @@ +"""Test CTypes structs, unions, bitfields against C equivalents. + +The types here are auto-converted to C source at +`Modules/_ctypes/_ctypes_test_generated.c.h`, which is compiled into +_ctypes_test. + +Run this module to regenerate the files: + +./python Lib/test/test_ctypes/test_generated_structs.py > Modules/_ctypes/_ctypes_test_generated.c.h +""" + +import unittest +from test.support import import_helper +import re +from dataclasses import dataclass +from functools import cached_property + +import ctypes +from ctypes import Structure, Union +from ctypes import sizeof, alignment, pointer, string_at +_ctypes_test = import_helper.import_module("_ctypes_test") + + +# ctypes erases the difference between `c_int` and e.g.`c_int16`. +# To keep it, we'll use custom subclasses with the C name stashed in `_c_name`: +class c_bool(ctypes.c_bool): + _c_name = '_Bool' + +# To do it for all the other types, use some metaprogramming: +for c_name, ctypes_name in { + 'signed char': 'c_byte', + 'short': 'c_short', + 'int': 'c_int', + 'long': 'c_long', + 'long long': 'c_longlong', + 'unsigned char': 'c_ubyte', + 'unsigned short': 'c_ushort', + 'unsigned int': 'c_uint', + 'unsigned long': 'c_ulong', + 'unsigned long long': 'c_ulonglong', + **{f'{u}int{n}_t': f'c_{u}int{n}' + for u in ('', 'u') + for n in (8, 16, 32, 64)} +}.items(): + ctype = getattr(ctypes, ctypes_name) + newtype = type(ctypes_name, (ctype,), {'_c_name': c_name}) + globals()[ctypes_name] = newtype + + +# Register structs and unions to test + +TESTCASES = {} +def register(name=None, set_name=False): + def decorator(cls, name=name): + if name is None: + name = cls.__name__ + assert name.isascii() # will be used in _PyUnicode_EqualToASCIIString + assert name.isidentifier() # will be used as a C identifier + assert name not in TESTCASES + TESTCASES[name] = cls + if set_name: + cls.__name__ = name + return cls + return decorator + +@register() +class SingleInt(Structure): + _fields_ = [('a', c_int)] + +@register() +class SingleInt_Union(Union): + _fields_ = [('a', c_int)] + + +@register() +class SingleU32(Structure): + _fields_ = [('a', c_uint32)] + + +@register() +class SimpleStruct(Structure): + _fields_ = [('x', c_int32), ('y', c_int8), ('z', c_uint16)] + + +@register() +class SimpleUnion(Union): + _fields_ = [('x', c_int32), ('y', c_int8), ('z', c_uint16)] + + +@register() +class ManyTypes(Structure): + _fields_ = [ + ('i8', c_int8), ('u8', c_uint8), + ('i16', c_int16), ('u16', c_uint16), + ('i32', c_int32), ('u32', c_uint32), + ('i64', c_int64), ('u64', c_uint64), + ] + + +@register() +class ManyTypesU(Union): + _fields_ = [ + ('i8', c_int8), ('u8', c_uint8), + ('i16', c_int16), ('u16', c_uint16), + ('i32', c_int32), ('u32', c_uint32), + ('i64', c_int64), ('u64', c_uint64), + ] + + +@register() +class Nested(Structure): + _fields_ = [ + ('a', SimpleStruct), ('b', SimpleUnion), ('anon', SimpleStruct), + ] + _anonymous_ = ['anon'] + + +@register() +class Packed1(Structure): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 1 + + +@register() +class Packed2(Structure): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 2 + + +@register() +class Packed3(Structure): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 4 + + +@register() +class Packed4(Structure): + def _maybe_skip(): + # `_pack_` enables MSVC-style packing, but keeps platform-specific + # alignments. + # The C code we generate for GCC/clang currently uses + # `__attribute__((ms_struct))`, which activates MSVC layout *and* + # alignments, that is, sizeof(basic type) == alignment(basic type). + # On a Pentium, int64 is 32-bit aligned, so the two won't match. + # The expected behavior is instead tested in + # StructureTestCase.test_packed, over in test_structures.py. + if sizeof(c_int64) != alignment(c_int64): + raise unittest.SkipTest('cannot test on this platform') + + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 8 + +@register() +class X86_32EdgeCase(Structure): + # On a Pentium, long long (int64) is 32-bit aligned, + # so these are packed tightly. + _fields_ = [('a', c_int32), ('b', c_int64), ('c', c_int32)] + +@register() +class MSBitFieldExample(Structure): + # From https://learn.microsoft.com/en-us/cpp/c-language/c-bit-fields + _fields_ = [ + ('a', c_uint, 4), + ('b', c_uint, 5), + ('c', c_uint, 7)] + +@register() +class MSStraddlingExample(Structure): + # From https://learn.microsoft.com/en-us/cpp/c-language/c-bit-fields + _fields_ = [ + ('first', c_uint, 9), + ('second', c_uint, 7), + ('may_straddle', c_uint, 30), + ('last', c_uint, 18)] + +@register() +class IntBits(Structure): + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9)] + +@register() +class Bits(Structure): + _fields_ = [*IntBits._fields_, + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +@register() +class IntBits_MSVC(Structure): + _layout_ = "ms" + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9)] + +@register() +class Bits_MSVC(Structure): + _layout_ = "ms" + _fields_ = [*IntBits_MSVC._fields_, + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +# Skipped for now -- we don't always match the alignment +#@register() +class IntBits_Union(Union): + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9)] + +# Skipped for now -- we don't always match the alignment +#@register() +class BitsUnion(Union): + _fields_ = [*IntBits_Union._fields_, + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +@register() +class I64Bits(Structure): + _fields_ = [("a", c_int64, 1), + ("b", c_int64, 62), + ("c", c_int64, 1)] + +@register() +class U64Bits(Structure): + _fields_ = [("a", c_uint64, 1), + ("b", c_uint64, 62), + ("c", c_uint64, 1)] + +for n in 8, 16, 32, 64: + for signedness in '', 'u': + ctype = globals()[f'c_{signedness}int{n}'] + + @register(f'Struct331_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 3), + ("b", ctype, 3), + ("c", ctype, 1)] + + @register(f'Struct1x1_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 1), + ("b", ctype, n-2), + ("c", ctype, 1)] + + @register(f'Struct1nx1_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 1), + ("full", ctype), + ("b", ctype, n-2), + ("c", ctype, 1)] + + @register(f'Struct3xx_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 3), + ("b", ctype, n-2), + ("c", ctype, n-2)] + +@register() +class Mixed1(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_int, 4)] + +@register() +class Mixed2(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_int32, 32)] + +@register() +class Mixed3(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_ubyte, 4)] + +@register() +class Mixed4(Structure): + _fields_ = [("a", c_short, 4), + ("b", c_short, 4), + ("c", c_int, 24), + ("d", c_short, 4), + ("e", c_short, 4), + ("f", c_int, 24)] + +@register() +class Mixed5(Structure): + _fields_ = [('A', c_uint, 1), + ('B', c_ushort, 16)] + +@register() +class Mixed6(Structure): + _fields_ = [('A', c_ulonglong, 1), + ('B', c_uint, 32)] + +@register() +class Mixed7(Structure): + _fields_ = [("A", c_uint32), + ('B', c_uint32, 20), + ('C', c_uint64, 24)] + +@register() +class Mixed8_a(Structure): + _fields_ = [("A", c_uint32), + ("B", c_uint32, 32), + ("C", c_ulonglong, 1)] + +@register() +class Mixed8_b(Structure): + _fields_ = [("A", c_uint32), + ("B", c_uint32), + ("C", c_ulonglong, 1)] + +@register() +class Mixed9(Structure): + _fields_ = [("A", c_uint8), + ("B", c_uint32, 1)] + +@register() +class Mixed10(Structure): + _fields_ = [("A", c_uint32, 1), + ("B", c_uint64, 1)] + +@register() +class Example_gh_95496(Structure): + _fields_ = [("A", c_uint32, 1), + ("B", c_uint64, 1)] + +@register() +class Example_gh_84039_bad(Structure): + _pack_ = 1 + _fields_ = [("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12)] + +@register() +class Example_gh_84039_good_a(Structure): + _pack_ = 1 + _fields_ = [("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1)] + +@register() +class Example_gh_84039_good(Structure): + _pack_ = 1 + _fields_ = [("a", Example_gh_84039_good_a), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12)] + +@register() +class Example_gh_73939(Structure): + _pack_ = 1 + _fields_ = [("P", c_uint16), + ("L", c_uint16, 9), + ("Pro", c_uint16, 1), + ("G", c_uint16, 1), + ("IB", c_uint16, 1), + ("IR", c_uint16, 1), + ("R", c_uint16, 3), + ("T", c_uint32, 10), + ("C", c_uint32, 20), + ("R2", c_uint32, 2)] + +@register() +class Example_gh_86098(Structure): + _fields_ = [("a", c_uint8, 8), + ("b", c_uint8, 8), + ("c", c_uint32, 16)] + +@register() +class Example_gh_86098_pack(Structure): + _pack_ = 1 + _fields_ = [("a", c_uint8, 8), + ("b", c_uint8, 8), + ("c", c_uint32, 16)] + +@register() +class AnonBitfields(Structure): + class X(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_ubyte, 4)] + _anonymous_ = ["_"] + _fields_ = [("_", X), ('y', c_byte)] + + +class GeneratedTest(unittest.TestCase): + def test_generated_data(self): + """Check that a ctypes struct/union matches its C equivalent. + + This compares with data from get_generated_test_data(), a list of: + - name (str) + - size (int) + - alignment (int) + - for each field, three snapshots of memory, as bytes: + - memory after the field is set to -1 + - memory after the field is set to 1 + - memory after the field is set to 0 + + or: + - None + - reason to skip the test (str) + + This does depend on the C compiler keeping padding bits unchanged. + Common compilers seem to do so. + """ + for name, cls in TESTCASES.items(): + with self.subTest(name=name): + if _maybe_skip := getattr(cls, '_maybe_skip', None): + _maybe_skip() + expected = iter(_ctypes_test.get_generated_test_data(name)) + expected_name = next(expected) + if expected_name is None: + self.skipTest(next(expected)) + self.assertEqual(name, expected_name) + self.assertEqual(sizeof(cls), next(expected)) + with self.subTest('alignment'): + self.assertEqual(alignment(cls), next(expected)) + obj = cls() + ptr = pointer(obj) + for field in iterfields(cls): + for value in -1, 1, 0: + with self.subTest(field=field.full_name, value=value): + field.set_to(obj, value) + py_mem = string_at(ptr, sizeof(obj)) + c_mem = next(expected) + if py_mem != c_mem: + # Generate a helpful failure message + lines, requires = dump_ctype(cls) + m = "\n".join([str(field), 'in:', *lines]) + self.assertEqual(py_mem.hex(), c_mem.hex(), m) + + +# The rest of this file is generating C code from a ctypes type. +# This is only meant for (and tested with) the known inputs in this file! + +def c_str_repr(string): + """Return a string as a C literal""" + return '"' + re.sub('([\"\'\\\\\n])', r'\\\1', string) + '"' + +def dump_simple_ctype(tp, variable_name='', semi=''): + """Get C type name or declaration of a scalar type + + variable_name: if given, declare the given variable + semi: a semicolon, and/or bitfield specification to tack on to the end + """ + length = getattr(tp, '_length_', None) + if length is not None: + return f'{dump_simple_ctype(tp._type_, variable_name)}[{length}]{semi}' + assert not issubclass(tp, (Structure, Union)) + return f'{tp._c_name}{maybe_space(variable_name)}{semi}' + + +def dump_ctype(tp, struct_or_union_tag='', variable_name='', semi=''): + """Get C type name or declaration of a ctype + + struct_or_union_tag: name of the struct or union + variable_name: if given, declare the given variable + semi: a semicolon, and/or bitfield specification to tack on to the end + """ + requires = set() + if issubclass(tp, (Structure, Union)): + attributes = [] + pushes = [] + pops = [] + pack = getattr(tp, '_pack_', None) + if pack is not None: + pushes.append(f'#pragma pack(push, {pack})') + pops.append(f'#pragma pack(pop)') + layout = getattr(tp, '_layout_', None) + if layout == 'ms' or pack: + # The 'ms_struct' attribute only works on x86 and PowerPC + requires.add( + 'defined(MS_WIN32) || (' + '(defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (' + 'defined(__GNUC__) || defined(__clang__)))' + ) + attributes.append('ms_struct') + if attributes: + a = f' GCC_ATTR({", ".join(attributes)})' + else: + a = '' + lines = [f'{struct_or_union(tp)}{a}{maybe_space(struct_or_union_tag)} ' +'{'] + for fielddesc in tp._fields_: + f_name, f_tp, f_bits = unpack_field_desc(*fielddesc) + if f_name in getattr(tp, '_anonymous_', ()): + f_name = '' + if f_bits is None: + subsemi = ';' + else: + if f_tp not in (c_int, c_uint): + # XLC can reportedly only handle int & unsigned int + # bitfields (the only types required by C spec) + requires.add('!defined(__xlc__)') + subsemi = f' :{f_bits};' + sub_lines, sub_requires = dump_ctype( + f_tp, variable_name=f_name, semi=subsemi) + requires.update(sub_requires) + for line in sub_lines: + lines.append(' ' + line) + lines.append(f'}}{maybe_space(variable_name)}{semi}') + return [*pushes, *lines, *reversed(pops)], requires + else: + return [dump_simple_ctype(tp, variable_name, semi)], requires + +def struct_or_union(cls): + if issubclass(cls, Structure): + return 'struct' + if issubclass(cls, Union): + return 'union' + raise TypeError(cls) + +def maybe_space(string): + if string: + return ' ' + string + return string + +def unpack_field_desc(f_name, f_tp, f_bits=None): + """Unpack a _fields_ entry into a (name, type, bits) triple""" + return f_name, f_tp, f_bits + +@dataclass +class FieldInfo: + """Information about a (possibly nested) struct/union field""" + name: str + tp: type + bits: int | None # number if this is a bit field + parent_type: type + parent: 'FieldInfo' #| None + + @cached_property + def attr_path(self): + """Attribute names to get at the value of this field""" + if self.name in getattr(self.parent_type, '_anonymous_', ()): + selfpath = () + else: + selfpath = (self.name,) + if self.parent: + return (*self.parent.attr_path, *selfpath) + else: + return selfpath + + @cached_property + def full_name(self): + """Attribute names to get at the value of this field""" + return '.'.join(self.attr_path) + + def set_to(self, obj, new): + """Set the field on a given Structure/Union instance""" + for attr_name in self.attr_path[:-1]: + obj = getattr(obj, attr_name) + setattr(obj, self.attr_path[-1], new) + + @cached_property + def root(self): + if self.parent is None: + return self + else: + return self.parent + + @cached_property + def descriptor(self): + return getattr(self.parent_type, self.name) + + def __repr__(self): + qname = f'{self.root.parent_type.__name__}.{self.full_name}' + try: + desc = self.descriptor + except AttributeError: + desc = '???' + return f'<{type(self).__name__} for {qname}: {desc}>' + +def iterfields(tp, parent=None): + """Get *leaf* fields of a structure or union, as FieldInfo""" + try: + fields = tp._fields_ + except AttributeError: + yield parent + else: + for fielddesc in fields: + f_name, f_tp, f_bits = unpack_field_desc(*fielddesc) + sub = FieldInfo(f_name, f_tp, f_bits, tp, parent) + yield from iterfields(f_tp, sub) + + +if __name__ == '__main__': + # Dump C source to stdout + def output(string): + print(re.compile(r'^ +$', re.MULTILINE).sub('', string).lstrip('\n')) + output(""" + /* Generated by Lib/test/test_ctypes/test_generated_structs.py */ + + + // Append VALUE to the result. + #define APPEND(ITEM) { \\ + PyObject *item = ITEM; \\ + if (!item) { \\ + Py_DECREF(result); \\ + return NULL; \\ + } \\ + int rv = PyList_Append(result, item); \\ + Py_DECREF(item); \\ + if (rv < 0) { \\ + Py_DECREF(result); \\ + return NULL; \\ + } \\ + } + + // Set TARGET, and append a snapshot of `value`'s + // memory to the result. + #define SET_AND_APPEND(TYPE, TARGET, VAL) { \\ + TYPE v = VAL; \\ + TARGET = v; \\ + APPEND(PyBytes_FromStringAndSize( \\ + (char*)&value, sizeof(value))); \\ + } + + // Set a field to -1, 1 and 0; append a snapshot of the memory + // after each of the operations. + #define TEST_FIELD(TYPE, TARGET) { \\ + SET_AND_APPEND(TYPE, TARGET, -1) \\ + SET_AND_APPEND(TYPE, TARGET, 1) \\ + SET_AND_APPEND(TYPE, TARGET, 0) \\ + } + + #if defined(__GNUC__) || defined(__clang__) + #define GCC_ATTR(X) __attribute__((X)) + #else + #define GCC_ATTR(X) /* */ + #endif + + static PyObject * + get_generated_test_data(PyObject *self, PyObject *name) + { + if (!PyUnicode_Check(name)) { + PyErr_SetString(PyExc_TypeError, "need a string"); + return NULL; + } + PyObject *result = PyList_New(0); + if (!result) { + return NULL; + } + """) + for name, cls in TESTCASES.items(): + output(""" + if (PyUnicode_CompareWithASCIIString(name, %s) == 0) { + """ % c_str_repr(name)) + lines, requires = dump_ctype(cls, struct_or_union_tag=name, semi=';') + if requires: + output(f""" + #if {" && ".join(f'({r})' for r in sorted(requires))} + """) + for line in lines: + output(' ' + line) + typename = f'{struct_or_union(cls)} {name}' + output(f""" + {typename} value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString({c_str_repr(name)})); + APPEND(PyLong_FromLong(sizeof({typename}))); + APPEND(PyLong_FromLong(_Alignof({typename}))); + """.rstrip()) + for field in iterfields(cls): + f_tp = dump_simple_ctype(field.tp) + output(f"""\ + TEST_FIELD({f_tp}, value.{field.full_name}); + """.rstrip()) + if requires: + output(f""" + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + """) + output(""" + return result; + } + """) + + output(""" + Py_DECREF(result); + PyErr_Format(PyExc_ValueError, "unknown testcase %R", name); + return NULL; + } + + #undef GCC_ATTR + #undef TEST_FIELD + #undef SET_AND_APPEND + #undef APPEND + """) diff --git a/Lib/test/test_ctypes/test_internals.py b/Lib/test/test_ctypes/test_internals.py index 94c9a86c2d06df..778da6573da975 100644 --- a/Lib/test/test_ctypes/test_internals.py +++ b/Lib/test/test_ctypes/test_internals.py @@ -28,7 +28,7 @@ def test_ints(self): self.assertEqual(ci._objects, None) def test_c_char_p(self): - s = b"Hello, World" + s = "Hello, World".encode("ascii") refcnt = sys.getrefcount(s) cs = c_char_p(s) self.assertEqual(refcnt + 1, sys.getrefcount(s)) diff --git a/Lib/test/test_ctypes/test_libc.py b/Lib/test/test_ctypes/test_libc.py index 09c76db0bd0b17..df7dbc0ae264a0 100644 --- a/Lib/test/test_ctypes/test_libc.py +++ b/Lib/test/test_ctypes/test_libc.py @@ -1,8 +1,10 @@ -import _ctypes_test +import ctypes import math import unittest from ctypes import (CDLL, CFUNCTYPE, POINTER, create_string_buffer, sizeof, c_void_p, c_char, c_int, c_double, c_size_t) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") lib = CDLL(_ctypes_test.__file__) @@ -20,6 +22,31 @@ def test_sqrt(self): self.assertEqual(lib.my_sqrt(4.0), 2.0) self.assertEqual(lib.my_sqrt(2.0), math.sqrt(2.0)) + @unittest.skipUnless(hasattr(ctypes, "c_double_complex"), + "requires C11 complex type and libffi >= 3.3.0") + def test_csqrt(self): + lib.my_csqrt.argtypes = ctypes.c_double_complex, + lib.my_csqrt.restype = ctypes.c_double_complex + self.assertEqual(lib.my_csqrt(4), 2+0j) + self.assertAlmostEqual(lib.my_csqrt(-1+0.01j), + 0.004999937502734214+1.0000124996093955j) + self.assertAlmostEqual(lib.my_csqrt(-1-0.01j), + 0.004999937502734214-1.0000124996093955j) + + lib.my_csqrtf.argtypes = ctypes.c_float_complex, + lib.my_csqrtf.restype = ctypes.c_float_complex + self.assertAlmostEqual(lib.my_csqrtf(-1+0.01j), + 0.004999937502734214+1.0000124996093955j) + self.assertAlmostEqual(lib.my_csqrtf(-1-0.01j), + 0.004999937502734214-1.0000124996093955j) + + lib.my_csqrtl.argtypes = ctypes.c_longdouble_complex, + lib.my_csqrtl.restype = ctypes.c_longdouble_complex + self.assertAlmostEqual(lib.my_csqrtl(-1+0.01j), + 0.004999937502734214+1.0000124996093955j) + self.assertAlmostEqual(lib.my_csqrtl(-1-0.01j), + 0.004999937502734214-1.0000124996093955j) + def test_qsort(self): comparefunc = CFUNCTYPE(c_int, POINTER(c_char), POINTER(c_char)) lib.my_qsort.argtypes = c_void_p, c_size_t, c_size_t, comparefunc diff --git a/Lib/test/test_ctypes/test_loading.py b/Lib/test/test_ctypes/test_loading.py index b218e9e7720c79..13ed813ad98c31 100644 --- a/Lib/test/test_ctypes/test_loading.py +++ b/Lib/test/test_ctypes/test_loading.py @@ -1,5 +1,4 @@ import _ctypes -import _ctypes_test import ctypes import os import shutil @@ -10,6 +9,7 @@ from ctypes import CDLL, cdll, addressof, c_void_p, c_char_p from ctypes.util import find_library from test.support import import_helper, os_helper +_ctypes_test = import_helper.import_module("_ctypes_test") libc_name = None @@ -42,10 +42,7 @@ def test_load(self): self.skipTest('could not find library to load') CDLL(test_lib) CDLL(os.path.basename(test_lib)) - class CTypesTestPathLikeCls: - def __fspath__(self): - return test_lib - CDLL(CTypesTestPathLikeCls()) + CDLL(os_helper.FakePath(test_lib)) self.assertRaises(OSError, CDLL, self.unknowndll) def test_load_version(self): @@ -138,7 +135,7 @@ def test_1703286_B(self): 'test specific to Windows') def test_load_hasattr(self): # bpo-34816: shouldn't raise OSError - self.assertFalse(hasattr(ctypes.windll, 'test')) + self.assertNotHasAttr(ctypes.windll, 'test') @unittest.skipUnless(os.name == "nt", 'test specific to Windows') diff --git a/Lib/test/test_ctypes/test_memfunctions.py b/Lib/test/test_ctypes/test_memfunctions.py index 112b27ba48e07e..325487618137f6 100644 --- a/Lib/test/test_ctypes/test_memfunctions.py +++ b/Lib/test/test_ctypes/test_memfunctions.py @@ -5,7 +5,9 @@ create_string_buffer, string_at, create_unicode_buffer, wstring_at, memmove, memset, - c_char_p, c_byte, c_ubyte, c_wchar) + memoryview_at, c_void_p, + c_char_p, c_byte, c_ubyte, c_wchar, + addressof, byref) class MemFunctionsTest(unittest.TestCase): @@ -77,6 +79,62 @@ def test_wstring_at(self): self.assertEqual(wstring_at(a, 16), "Hello, World\0\0\0\0") self.assertEqual(wstring_at(a, 0), "") + def test_memoryview_at(self): + b = (c_byte * 10)() + + size = len(b) + for foreign_ptr in ( + b, + cast(b, c_void_p), + byref(b), + addressof(b), + ): + with self.subTest(foreign_ptr=type(foreign_ptr).__name__): + b[:] = b"initialval" + v = memoryview_at(foreign_ptr, size) + self.assertIsInstance(v, memoryview) + self.assertEqual(bytes(v), b"initialval") + + # test that writes to source buffer get reflected in memoryview + b[:] = b"0123456789" + self.assertEqual(bytes(v), b"0123456789") + + # test that writes to memoryview get reflected in source buffer + v[:] = b"9876543210" + self.assertEqual(bytes(b), b"9876543210") + + with self.assertRaises(ValueError): + memoryview_at(foreign_ptr, -1) + + with self.assertRaises(ValueError): + memoryview_at(foreign_ptr, sys.maxsize + 1) + + v0 = memoryview_at(foreign_ptr, 0) + self.assertEqual(bytes(v0), b'') + + def test_memoryview_at_readonly(self): + b = (c_byte * 10)() + + size = len(b) + for foreign_ptr in ( + b, + cast(b, c_void_p), + byref(b), + addressof(b), + ): + with self.subTest(foreign_ptr=type(foreign_ptr).__name__): + b[:] = b"initialval" + v = memoryview_at(foreign_ptr, size, readonly=True) + self.assertIsInstance(v, memoryview) + self.assertEqual(bytes(v), b"initialval") + + # test that writes to source buffer get reflected in memoryview + b[:] = b"0123456789" + self.assertEqual(bytes(v), b"0123456789") + + # test that writes to the memoryview are blocked + with self.assertRaises(TypeError): + v[:] = b"9876543210" if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_numbers.py b/Lib/test/test_ctypes/test_numbers.py index 29108a28ec16e1..c57c58eb002328 100644 --- a/Lib/test/test_ctypes/test_numbers.py +++ b/Lib/test/test_ctypes/test_numbers.py @@ -1,12 +1,15 @@ import array +import ctypes import struct import sys import unittest +from itertools import combinations from operator import truth from ctypes import (byref, sizeof, alignment, c_char, c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double, c_longdouble, c_bool) +from test.support.testcase import ComplexesAreIdenticalMixin def valid_ranges(*types): @@ -38,8 +41,28 @@ def valid_ranges(*types): signed_ranges = valid_ranges(*signed_types) bool_values = [True, False, 0, 1, -1, 5000, 'test', [], [1]] +class IntLike: + def __int__(self): + return 2 -class NumberTestCase(unittest.TestCase): +class IndexLike: + def __index__(self): + return 2 + +class FloatLike: + def __float__(self): + return 2.0 + +class ComplexLike: + def __complex__(self): + return 1+1j + + +INF = float("inf") +NAN = float("nan") + + +class NumberTestCase(unittest.TestCase, ComplexesAreIdenticalMixin): def test_default_init(self): # default values are set to zero @@ -86,9 +109,6 @@ def test_byref(self): def test_floats(self): # c_float and c_double can be created from # Python int and float - class FloatLike: - def __float__(self): - return 2.0 f = FloatLike() for t in float_types: self.assertEqual(t(2.0).value, 2.0) @@ -96,18 +116,34 @@ def __float__(self): self.assertEqual(t(2).value, 2.0) self.assertEqual(t(f).value, 2.0) + @unittest.skipUnless(hasattr(ctypes, "c_double_complex"), + "requires C11 complex type") + def test_complex(self): + for t in [ctypes.c_double_complex, ctypes.c_float_complex, + ctypes.c_longdouble_complex]: + self.assertEqual(t(1).value, 1+0j) + self.assertEqual(t(1.0).value, 1+0j) + self.assertEqual(t(1+0.125j).value, 1+0.125j) + self.assertEqual(t(IndexLike()).value, 2+0j) + self.assertEqual(t(FloatLike()).value, 2+0j) + self.assertEqual(t(ComplexLike()).value, 1+1j) + + @unittest.skipUnless(hasattr(ctypes, "c_double_complex"), + "requires C11 complex type") + def test_complex_round_trip(self): + # Ensure complexes transformed exactly. The CMPLX macro should + # preserve special components (like inf/nan or signed zero). + values = [complex(*_) for _ in combinations([1, -1, 0.0, -0.0, 2, + -3, INF, -INF, NAN], 2)] + for z in values: + for t in [ctypes.c_double_complex, ctypes.c_float_complex, + ctypes.c_longdouble_complex]: + with self.subTest(z=z, type=t): + self.assertComplexesAreIdentical(z, t(z).value) + def test_integers(self): - class FloatLike: - def __float__(self): - return 2.0 f = FloatLike() - class IntLike: - def __int__(self): - return 2 d = IntLike() - class IndexLike: - def __index__(self): - return 2 i = IndexLike() # integers cannot be constructed from floats, # but from integer-like objects diff --git a/Lib/test/test_ctypes/test_parameters.py b/Lib/test/test_ctypes/test_parameters.py index d1eeee6b0306fe..f89521cf8b3a67 100644 --- a/Lib/test/test_ctypes/test_parameters.py +++ b/Lib/test/test_ctypes/test_parameters.py @@ -1,10 +1,9 @@ -import _ctypes_test import unittest import test.support from ctypes import (CDLL, PyDLL, ArgumentError, Structure, Array, Union, _Pointer, _SimpleCData, _CFuncPtr, - POINTER, pointer, byref, + POINTER, pointer, byref, sizeof, c_void_p, c_char_p, c_wchar_p, py_object, c_bool, c_char, c_wchar, @@ -14,6 +13,8 @@ c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double, c_longdouble) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") class SimpleTypesTestCase(unittest.TestCase): @@ -86,19 +87,33 @@ def test_c_char(self): with self.assertRaises(TypeError) as cm: c_char.from_param(b"abc") self.assertEqual(str(cm.exception), - "one character bytes, bytearray or integer expected") + "one character bytes, bytearray, or an integer " + "in range(256) expected, not bytes of length 3") def test_c_wchar(self): with self.assertRaises(TypeError) as cm: c_wchar.from_param("abc") self.assertEqual(str(cm.exception), - "one character unicode string expected") + "a unicode character expected, not a string of length 3") + with self.assertRaises(TypeError) as cm: + c_wchar.from_param("") + self.assertEqual(str(cm.exception), + "a unicode character expected, not a string of length 0") with self.assertRaises(TypeError) as cm: c_wchar.from_param(123) self.assertEqual(str(cm.exception), - "unicode string expected instead of int instance") + "a unicode character expected, not instance of int") + + if sizeof(c_wchar) < 4: + with self.assertRaises(TypeError) as cm: + c_wchar.from_param('\U0001f40d') + self.assertEqual(str(cm.exception), + "the string '\\U0001f40d' cannot be converted to " + "a single wchar_t character") + + def test_int_pointers(self): LPINT = POINTER(c_int) diff --git a/Lib/test/test_ctypes/test_pickling.py b/Lib/test/test_ctypes/test_pickling.py index 0ca42a68f0675f..9d433fc69de391 100644 --- a/Lib/test/test_ctypes/test_pickling.py +++ b/Lib/test/test_ctypes/test_pickling.py @@ -1,9 +1,10 @@ -import _ctypes_test import pickle import unittest from ctypes import (CDLL, Structure, CFUNCTYPE, pointer, c_void_p, c_char_p, c_wchar_p, c_char, c_wchar, c_int, c_double) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") dll = CDLL(_ctypes_test.__file__) diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index 8cf2114c282cab..fc558e10ba40c5 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -1,4 +1,3 @@ -import _ctypes_test import array import ctypes import sys @@ -10,6 +9,8 @@ c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") from ._support import (_CData, PyCPointerType, Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE) @@ -33,6 +34,11 @@ def test_type_flags(self): self.assertTrue(_Pointer.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) self.assertFalse(_Pointer.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + def test_metaclass_details(self): + # Cannot call the metaclass __init__ more than once + with self.assertRaisesRegex(SystemError, "already initialized"): + PyCPointerType.__init__(POINTER(c_byte), 'ptr', (), {}) + def test_pointer_crash(self): class A(POINTER(c_ulong)): diff --git a/Lib/test/test_ctypes/test_prototypes.py b/Lib/test/test_ctypes/test_prototypes.py index 81eb4562c740fd..63ae799ea86ab2 100644 --- a/Lib/test/test_ctypes/test_prototypes.py +++ b/Lib/test/test_ctypes/test_prototypes.py @@ -18,12 +18,13 @@ # # In this case, there would have to be an additional reference to the argument... -import _ctypes_test import unittest from ctypes import (CDLL, CFUNCTYPE, POINTER, ArgumentError, pointer, byref, sizeof, addressof, create_string_buffer, c_void_p, c_char_p, c_wchar_p, c_char, c_wchar, c_short, c_int, c_long, c_longlong, c_double) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") testdll = CDLL(_ctypes_test.__file__) diff --git a/Lib/test/test_ctypes/test_python_api.py b/Lib/test/test_ctypes/test_python_api.py index 77da35855928a4..1072a109833261 100644 --- a/Lib/test/test_ctypes/test_python_api.py +++ b/Lib/test/test_ctypes/test_python_api.py @@ -47,7 +47,7 @@ def test_PyLong_Long(self): @support.refcount_test def test_PyObj_FromPtr(self): - s = "abc def ghi jkl" + s = object() ref = sys.getrefcount(s) # id(python-object) is the address pyobj = _ctypes.PyObj_FromPtr(id(s)) diff --git a/Lib/test/test_ctypes/test_refcounts.py b/Lib/test/test_ctypes/test_refcounts.py index a90588ca9bb1b6..012722d8486218 100644 --- a/Lib/test/test_ctypes/test_refcounts.py +++ b/Lib/test/test_ctypes/test_refcounts.py @@ -1,9 +1,11 @@ -import _ctypes_test import ctypes import gc import sys import unittest from test import support +from test.support import import_helper +from test.support import script_helper +_ctypes_test = import_helper.import_module("_ctypes_test") MyCallback = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int) @@ -109,5 +111,18 @@ def func(): func() +class ModuleIsolationTest(unittest.TestCase): + def test_finalize(self): + # check if gc_decref() succeeds + script = ( + "import ctypes;" + "import sys;" + "del sys.modules['_ctypes'];" + "import _ctypes;" + "exit()" + ) + script_helper.assert_python_ok("-c", script) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ctypes/test_repr.py b/Lib/test/test_ctypes/test_repr.py index e7587984a92c45..8c85e6cbe70cea 100644 --- a/Lib/test/test_ctypes/test_repr.py +++ b/Lib/test/test_ctypes/test_repr.py @@ -22,12 +22,12 @@ class ReprTest(unittest.TestCase): def test_numbers(self): for typ in subclasses: base = typ.__bases__[0] - self.assertTrue(repr(base(42)).startswith(base.__name__)) - self.assertEqual(".X' object has no attribute 'a'"): X().a # Test string subclass in `__slots__`, see gh-98783 @@ -1593,8 +1587,7 @@ def f(cls, arg): self.fail("classmethod shouldn't accept keyword args") cm = classmethod(f) - cm_dict = {'__annotations__': {}, - '__doc__': ( + cm_dict = {'__doc__': ( "f docstring" if support.HAVE_DOCSTRINGS else None @@ -1610,6 +1603,55 @@ def f(cls, arg): del cm.x self.assertNotHasAttr(cm, "x") + def test_classmethod_staticmethod_annotations(self): + for deco in (classmethod, staticmethod): + @deco + def unannotated(cls): pass + @deco + def annotated(cls) -> int: pass + + for method in (annotated, unannotated): + with self.subTest(deco=deco, method=method): + with self.assertRaises(AttributeError): + del unannotated.__annotations__ + + original_annotations = dict(method.__wrapped__.__annotations__) + self.assertNotIn('__annotations__', method.__dict__) + self.assertEqual(method.__annotations__, original_annotations) + self.assertIn('__annotations__', method.__dict__) + + new_annotations = {"a": "b"} + method.__annotations__ = new_annotations + self.assertEqual(method.__annotations__, new_annotations) + self.assertEqual(method.__wrapped__.__annotations__, original_annotations) + + del method.__annotations__ + self.assertEqual(method.__annotations__, original_annotations) + + original_annotate = method.__wrapped__.__annotate__ + self.assertNotIn('__annotate__', method.__dict__) + self.assertIs(method.__annotate__, original_annotate) + self.assertIn('__annotate__', method.__dict__) + + new_annotate = lambda: {"annotations": 1} + method.__annotate__ = new_annotate + self.assertIs(method.__annotate__, new_annotate) + self.assertIs(method.__wrapped__.__annotate__, original_annotate) + + del method.__annotate__ + self.assertIs(method.__annotate__, original_annotate) + + def test_staticmethod_annotations_without_dict_access(self): + # gh-125017: this used to crash + class Spam: + def __new__(cls, x, y): + pass + + self.assertEqual(Spam.__new__.__annotations__, {}) + obj = Spam.__dict__['__new__'] + self.assertIsInstance(obj, staticmethod) + self.assertEqual(obj.__annotations__, {}) + @support.refcount_test def test_refleaks_in_classmethod___init__(self): gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') @@ -1687,10 +1729,10 @@ class D(C): self.assertEqual(d.foo(1), (d, 1)) self.assertEqual(D.foo(d, 1), (d, 1)) sm = staticmethod(None) - self.assertEqual(sm.__dict__, {'__doc__': None}) + self.assertEqual(sm.__dict__, {'__doc__': None.__doc__}) sm.x = 42 self.assertEqual(sm.x, 42) - self.assertEqual(sm.__dict__, {"x" : 42, '__doc__': None}) + self.assertEqual(sm.__dict__, {"x" : 42, '__doc__': None.__doc__}) del sm.x self.assertNotHasAttr(sm, "x") @@ -3191,8 +3233,6 @@ class classic: class C(base): def __init__(self, value): self.value = int(value) - def __cmp__(self_, other): - self.fail("shouldn't call __cmp__") def __eq__(self, other): if isinstance(other, C): return self.value == other.value @@ -3617,6 +3657,7 @@ def f(a): return a encoding='latin1', errors='replace') self.assertEqual(ba, b'abc\xbd?') + @support.skip_emscripten_stack_overflow() def test_recursive_call(self): # Testing recursive __call__() by setting to instance of class... class A(object): @@ -3896,6 +3937,7 @@ def __del__(self): # it as a leak. del C.__del__ + @unittest.skipIf(support.is_emscripten, "Seems to works in Pyodide?") def test_slots_trash(self): # Testing slot trash... # Deallocating deeply nested slotted trash caused stack overflows @@ -3987,6 +4029,20 @@ def test_ipow_exception_text(self): y = x ** 2 self.assertIn('unsupported operand type(s) for **', str(cm.exception)) + def test_pow_wrapper_error_messages(self): + self.assertRaisesRegex(TypeError, + 'expected 1 or 2 arguments, got 0', + int().__pow__) + self.assertRaisesRegex(TypeError, + 'expected 1 or 2 arguments, got 3', + int().__pow__, 1, 2, 3) + self.assertRaisesRegex(TypeError, + 'expected 1 or 2 arguments, got 0', + int().__rpow__) + self.assertRaisesRegex(TypeError, + 'expected 1 or 2 arguments, got 3', + int().__rpow__, 1, 2, 3) + def test_mutable_bases(self): # Testing mutable bases... @@ -4594,18 +4650,16 @@ def test_special_unbound_method_types(self): def test_not_implemented(self): # Testing NotImplemented... # all binary methods should be able to return a NotImplemented - import operator def specialmethod(self, other): return NotImplemented def check(expr, x, y): - try: - exec(expr, {'x': x, 'y': y, 'operator': operator}) - except TypeError: - pass - else: - self.fail("no TypeError from %r" % (expr,)) + with ( + self.subTest(expr=expr, x=x, y=y), + self.assertRaises(TypeError), + ): + exec(expr, {'x': x, 'y': y}) N1 = sys.maxsize + 1 # might trigger OverflowErrors instead of # TypeErrors @@ -4626,12 +4680,23 @@ def check(expr, x, y): ('__and__', 'x & y', 'x &= y'), ('__or__', 'x | y', 'x |= y'), ('__xor__', 'x ^ y', 'x ^= y')]: - rname = '__r' + name[2:] + # Defines 'left' magic method: A = type('A', (), {name: specialmethod}) a = A() check(expr, a, a) check(expr, a, N1) check(expr, a, N2) + # Defines 'right' magic method: + rname = '__r' + name[2:] + B = type('B', (), {rname: specialmethod}) + b = B() + check(expr, b, b) + check(expr, a, b) + check(expr, b, a) + check(expr, b, N1) + check(expr, b, N2) + check(expr, N1, b) + check(expr, N2, b) if iexpr: check(iexpr, a, a) check(iexpr, a, N1) @@ -4795,6 +4860,7 @@ class Thing: # CALL_METHOD_DESCRIPTOR_O deque.append(thing, thing) + @support.skip_emscripten_stack_overflow() def test_repr_as_str(self): # Issue #11603: crash or infinite loop when rebinding __str__ as # __repr__. @@ -5079,8 +5145,10 @@ def test_iter_keys(self): self.assertNotIsInstance(it, list) keys = list(it) keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__module__', - '__weakref__', 'meth']) + self.assertEqual(keys, ['__dict__', '__doc__', '__firstlineno__', + '__module__', + '__static_attributes__', '__weakref__', + 'meth']) @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __local__') @@ -5089,7 +5157,7 @@ def test_iter_values(self): it = self.C.__dict__.values() self.assertNotIsInstance(it, list) values = list(it) - self.assertEqual(len(values), 5) + self.assertEqual(len(values), 7) @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __local__') @@ -5099,8 +5167,10 @@ def test_iter_items(self): self.assertNotIsInstance(it, list) keys = [item[0] for item in it] keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__module__', - '__weakref__', 'meth']) + self.assertEqual(keys, ['__dict__', '__doc__', '__firstlineno__', + '__module__', + '__static_attributes__', '__weakref__', + 'meth']) def test_dict_type_with_metaclass(self): # Testing type of __dict__ when metaclass set... @@ -5154,6 +5224,7 @@ def test_type_lookup_mro_reference(self): # Issue #14199: _PyType_Lookup() has to keep a strong reference to # the type MRO because it may be modified during the lookup, if # __bases__ is set during the lookup for example. + code = textwrap.dedent(""" class MyKey(object): def __hash__(self): return hash('mykey') @@ -5169,12 +5240,29 @@ class Base2(object): mykey = 'from Base2' mykey2 = 'from Base2' - with self.assertWarnsRegex(RuntimeWarning, 'X'): - X = type('X', (Base,), {MyKey(): 5}) - # mykey is read from Base - self.assertEqual(X.mykey, 'from Base') - # mykey2 is read from Base2 because MyKey.__eq__ has set __bases__ - self.assertEqual(X.mykey2, 'from Base2') + X = type('X', (Base,), {MyKey(): 5}) + + bases_before = ",".join([c.__name__ for c in X.__bases__]) + print(f"before={bases_before}") + + # mykey is initially read from Base, however, the lookup will be perfomed + # again if specialization fails. The second lookup will use the new + # mro set by __eq__. + print(X.mykey) + + bases_after = ",".join([c.__name__ for c in X.__bases__]) + print(f"after={bases_after}") + + # mykey2 is read from Base2 because MyKey.__eq__ has set __bases_ + print(f"mykey2={X.mykey2}") + """) + _, out, err = assert_python_ok("-c", code) + err = err.decode() + self.assertRegex(err, "RuntimeWarning: .*X") + out = out.decode() + self.assertRegex(out, "before=Base") + self.assertRegex(out, "after=Base2") + self.assertRegex(out, "mykey2=from Base2") class PicklingTests(unittest.TestCase): diff --git a/Lib/test/test_descrtut.py b/Lib/test/test_descrtut.py index f097c4e7300baa..828440a993a975 100644 --- a/Lib/test/test_descrtut.py +++ b/Lib/test/test_descrtut.py @@ -8,7 +8,7 @@ # of much interest anymore), and a few were fiddled to make the output # deterministic. -from test.support import sortdict +from test.support import sortdict # noqa: F401 import doctest import unittest diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 620d0ca4f4c2da..86b2f22dee5347 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -8,7 +8,7 @@ import unittest import weakref from test import support -from test.support import import_helper, Py_C_RECURSION_LIMIT +from test.support import import_helper, get_c_recursion_limit class DictTest(unittest.TestCase): @@ -594,9 +594,10 @@ def __repr__(self): d = {1: BadRepr()} self.assertRaises(Exc, repr, d) + @support.skip_emscripten_stack_overflow() def test_repr_deep(self): d = {} - for i in range(Py_C_RECURSION_LIMIT + 1): + for i in range(get_c_recursion_limit() + 1): d = {1: d} self.assertRaises(RecursionError, repr, d) @@ -880,115 +881,6 @@ class C(object): gc.collect() self.assertIs(ref(), None, "Cycle was not collected") - def _not_tracked(self, t): - # Nested containers can take several collections to untrack - gc.collect() - gc.collect() - self.assertFalse(gc.is_tracked(t), t) - - def _tracked(self, t): - self.assertTrue(gc.is_tracked(t), t) - gc.collect() - gc.collect() - self.assertTrue(gc.is_tracked(t), t) - - def test_string_keys_can_track_values(self): - # Test that this doesn't leak. - for i in range(10): - d = {} - for j in range(10): - d[str(j)] = j - d["foo"] = d - - @support.cpython_only - def test_track_literals(self): - # Test GC-optimization of dict literals - x, y, z, w = 1.5, "a", (1, None), [] - - self._not_tracked({}) - self._not_tracked({x:(), y:x, z:1}) - self._not_tracked({1: "a", "b": 2}) - self._not_tracked({1: 2, (None, True, False, ()): int}) - self._not_tracked({1: object()}) - - # Dicts with mutable elements are always tracked, even if those - # elements are not tracked right now. - self._tracked({1: []}) - self._tracked({1: ([],)}) - self._tracked({1: {}}) - self._tracked({1: set()}) - - @support.cpython_only - def test_track_dynamic(self): - # Test GC-optimization of dynamically-created dicts - class MyObject(object): - pass - x, y, z, w, o = 1.5, "a", (1, object()), [], MyObject() - - d = dict() - self._not_tracked(d) - d[1] = "a" - self._not_tracked(d) - d[y] = 2 - self._not_tracked(d) - d[z] = 3 - self._not_tracked(d) - self._not_tracked(d.copy()) - d[4] = w - self._tracked(d) - self._tracked(d.copy()) - d[4] = None - self._not_tracked(d) - self._not_tracked(d.copy()) - - # dd isn't tracked right now, but it may mutate and therefore d - # which contains it must be tracked. - d = dict() - dd = dict() - d[1] = dd - self._not_tracked(dd) - self._tracked(d) - dd[1] = d - self._tracked(dd) - - d = dict.fromkeys([x, y, z]) - self._not_tracked(d) - dd = dict() - dd.update(d) - self._not_tracked(dd) - d = dict.fromkeys([x, y, z, o]) - self._tracked(d) - dd = dict() - dd.update(d) - self._tracked(dd) - - d = dict(x=x, y=y, z=z) - self._not_tracked(d) - d = dict(x=x, y=y, z=z, w=w) - self._tracked(d) - d = dict() - d.update(x=x, y=y, z=z) - self._not_tracked(d) - d.update(w=w) - self._tracked(d) - - d = dict([(x, y), (z, 1)]) - self._not_tracked(d) - d = dict([(x, y), (z, w)]) - self._tracked(d) - d = dict() - d.update([(x, y), (z, 1)]) - self._not_tracked(d) - d.update([(x, y), (z, w)]) - self._tracked(d) - - @support.cpython_only - def test_track_subtypes(self): - # Dict subtypes are always tracked - class MyDict(dict): - pass - self._tracked(MyDict()) - def make_shared_key_dict(self, n): class C: pass @@ -1476,6 +1368,24 @@ def test_dict_items_result_gc_reversed(self): gc.collect() self.assertTrue(gc.is_tracked(next(it))) + def test_store_evilattr(self): + class EvilAttr: + def __init__(self, d): + self.d = d + + def __del__(self): + if 'attr' in self.d: + del self.d['attr'] + gc.collect() + + class Obj: + pass + + obj = Obj() + obj.__dict__ = {} + for _ in range(10): + obj.attr = EvilAttr(obj.__dict__) + def test_str_nonstr(self): # cpython uses a different lookup function if the dict only contains # `str` keys. Make sure the unoptimized path is used when a non-`str` diff --git a/Lib/test/test_dict_version.py b/Lib/test/test_dict_version.py deleted file mode 100644 index 243084c75c42bc..00000000000000 --- a/Lib/test/test_dict_version.py +++ /dev/null @@ -1,191 +0,0 @@ -""" -Test implementation of the PEP 509: dictionary versioning. -""" -import unittest -from test.support import import_helper - -# PEP 509 is implemented in CPython but other Python implementations -# don't require to implement it -_testcapi = import_helper.import_module('_testcapi') - - -class DictVersionTests(unittest.TestCase): - type2test = dict - - def setUp(self): - self.seen_versions = set() - self.dict = None - - def check_version_unique(self, mydict): - version = _testcapi.dict_get_version(mydict) - self.assertNotIn(version, self.seen_versions) - self.seen_versions.add(version) - - def check_version_changed(self, mydict, method, *args, **kw): - result = method(*args, **kw) - self.check_version_unique(mydict) - return result - - def check_version_dont_change(self, mydict, method, *args, **kw): - version1 = _testcapi.dict_get_version(mydict) - self.seen_versions.add(version1) - - result = method(*args, **kw) - - version2 = _testcapi.dict_get_version(mydict) - self.assertEqual(version2, version1, "version changed") - - return result - - def new_dict(self, *args, **kw): - d = self.type2test(*args, **kw) - self.check_version_unique(d) - return d - - def test_constructor(self): - # new empty dictionaries must all have an unique version - empty1 = self.new_dict() - empty2 = self.new_dict() - empty3 = self.new_dict() - - # non-empty dictionaries must also have an unique version - nonempty1 = self.new_dict(x='x') - nonempty2 = self.new_dict(x='x', y='y') - - def test_copy(self): - d = self.new_dict(a=1, b=2) - - d2 = self.check_version_dont_change(d, d.copy) - - # dict.copy() must create a dictionary with a new unique version - self.check_version_unique(d2) - - def test_setitem(self): - d = self.new_dict() - - # creating new keys must change the version - self.check_version_changed(d, d.__setitem__, 'x', 'x') - self.check_version_changed(d, d.__setitem__, 'y', 'y') - - # changing values must change the version - self.check_version_changed(d, d.__setitem__, 'x', 1) - self.check_version_changed(d, d.__setitem__, 'y', 2) - - def test_setitem_same_value(self): - value = object() - d = self.new_dict() - - # setting a key must change the version - self.check_version_changed(d, d.__setitem__, 'key', value) - - # setting a key to the same value with dict.__setitem__ - # must change the version - self.check_version_dont_change(d, d.__setitem__, 'key', value) - - # setting a key to the same value with dict.update - # must change the version - self.check_version_dont_change(d, d.update, key=value) - - d2 = self.new_dict(key=value) - self.check_version_dont_change(d, d.update, d2) - - def test_setitem_equal(self): - class AlwaysEqual: - def __eq__(self, other): - return True - - value1 = AlwaysEqual() - value2 = AlwaysEqual() - self.assertTrue(value1 == value2) - self.assertFalse(value1 != value2) - self.assertIsNot(value1, value2) - - d = self.new_dict() - self.check_version_changed(d, d.__setitem__, 'key', value1) - self.assertIs(d['key'], value1) - - # setting a key to a value equal to the current value - # with dict.__setitem__() must change the version - self.check_version_changed(d, d.__setitem__, 'key', value2) - self.assertIs(d['key'], value2) - - # setting a key to a value equal to the current value - # with dict.update() must change the version - self.check_version_changed(d, d.update, key=value1) - self.assertIs(d['key'], value1) - - d2 = self.new_dict(key=value2) - self.check_version_changed(d, d.update, d2) - self.assertIs(d['key'], value2) - - def test_setdefault(self): - d = self.new_dict() - - # setting a key with dict.setdefault() must change the version - self.check_version_changed(d, d.setdefault, 'key', 'value1') - - # don't change the version if the key already exists - self.check_version_dont_change(d, d.setdefault, 'key', 'value2') - - def test_delitem(self): - d = self.new_dict(key='value') - - # deleting a key with dict.__delitem__() must change the version - self.check_version_changed(d, d.__delitem__, 'key') - - # don't change the version if the key doesn't exist - self.check_version_dont_change(d, self.assertRaises, KeyError, - d.__delitem__, 'key') - - def test_pop(self): - d = self.new_dict(key='value') - - # pop() must change the version if the key exists - self.check_version_changed(d, d.pop, 'key') - - # pop() must not change the version if the key does not exist - self.check_version_dont_change(d, self.assertRaises, KeyError, - d.pop, 'key') - - def test_popitem(self): - d = self.new_dict(key='value') - - # popitem() must change the version if the dict is not empty - self.check_version_changed(d, d.popitem) - - # popitem() must not change the version if the dict is empty - self.check_version_dont_change(d, self.assertRaises, KeyError, - d.popitem) - - def test_update(self): - d = self.new_dict(key='value') - - # update() calling with no argument must not change the version - self.check_version_dont_change(d, d.update) - - # update() must change the version - self.check_version_changed(d, d.update, key='new value') - - d2 = self.new_dict(key='value 3') - self.check_version_changed(d, d.update, d2) - - def test_clear(self): - d = self.new_dict(key='value') - - # clear() must change the version if the dict is not empty - self.check_version_changed(d, d.clear) - - # clear() must not change the version if the dict is empty - self.check_version_dont_change(d, d.clear) - - -class Dict(dict): - pass - - -class DictSubtypeVersionTests(DictVersionTests): - type2test = Dict - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_dictcomps.py b/Lib/test/test_dictcomps.py index 472e3dfa0d8a0a..26b56dac5032fa 100644 --- a/Lib/test/test_dictcomps.py +++ b/Lib/test/test_dictcomps.py @@ -1,5 +1,8 @@ +import traceback import unittest +from test.support import BrokenIter + # For scope testing. g = "Global variable" @@ -127,6 +130,41 @@ def test_star_expression(self): self.assertEqual({i: i*i for i in [*range(4)]}, expected) self.assertEqual({i: i*i for i in (*range(4),)}, expected) + def test_exception_locations(self): + # The location of an exception raised from __init__ or + # __next__ should should be the iterator expression + def init_raises(): + try: + {x:x for x in BrokenIter(init_raises=True)} + except Exception as e: + return e + + def next_raises(): + try: + {x:x for x in BrokenIter(next_raises=True)} + except Exception as e: + return e + + def iter_raises(): + try: + {x:x for x in BrokenIter(iter_raises=True)} + except Exception as e: + return e + + for func, expected in [(init_raises, "BrokenIter(init_raises=True)"), + (next_raises, "BrokenIter(next_raises=True)"), + (iter_raises, "BrokenIter(iter_raises=True)"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index cad568b6ac4c2d..d6bf00eeeb0013 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -2,7 +2,7 @@ import copy import pickle import unittest -from test.support import Py_C_RECURSION_LIMIT +from test.support import get_c_recursion_limit, skip_emscripten_stack_overflow class DictSetTest(unittest.TestCase): @@ -277,9 +277,10 @@ def test_recursive_repr(self): # Again. self.assertIsInstance(r, str) + @skip_emscripten_stack_overflow() def test_deeply_nested_repr(self): d = {} - for i in range(Py_C_RECURSION_LIMIT//2 + 100): + for i in range(get_c_recursion_limit()//2 + 100): d = {42: d.values()} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 6afd90af8442ad..9e217249be7332 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -272,10 +272,30 @@ def test_make_file_usascii_charset_with_nonascii_input(self): self.assertIn('content="text/html; charset=us-ascii"', output) self.assertIn('ımplıcıt', output) +class TestDiffer(unittest.TestCase): + def test_close_matches_aligned(self): + # Of the 4 closely matching pairs, we want 1 to match with 3, + # and 2 with 4, to align with a "top to bottom" mental model. + a = ["cat\n", "dog\n", "close match 1\n", "close match 2\n"] + b = ["close match 3\n", "close match 4\n", "kitten\n", "puppy\n"] + m = difflib.Differ().compare(a, b) + self.assertEqual(list(m), + ['- cat\n', + '- dog\n', + '- close match 1\n', + '? ^\n', + '+ close match 3\n', + '? ^\n', + '- close match 2\n', + '? ^\n', + '+ close match 4\n', + '? ^\n', + '+ kitten\n', + '+ puppy\n']) class TestOutputFormat(unittest.TestCase): def test_tab_delimiter(self): - args = ['one', 'two', 'Original', 'Current', + args = [['one'], ['two'], 'Original', 'Current', '2005-01-26 23:30:50', '2010-04-02 10:20:52'] ud = difflib.unified_diff(*args, lineterm='') self.assertEqual(list(ud)[0:2], [ @@ -287,7 +307,7 @@ def test_tab_delimiter(self): "--- Current\t2010-04-02 10:20:52"]) def test_no_trailing_tab_on_empty_filedate(self): - args = ['one', 'two', 'Original', 'Current'] + args = [['one'], ['two'], 'Original', 'Current'] ud = difflib.unified_diff(*args, lineterm='') self.assertEqual(list(ud)[0:2], ["--- Original", "+++ Current"]) @@ -427,6 +447,28 @@ def assertDiff(expect, actual): lineterm=b'') assertDiff(expect, actual) + +class TestInputTypes(unittest.TestCase): + def _assert_type_error(self, msg, generator, *args): + with self.assertRaises(TypeError) as ctx: + list(generator(*args)) + self.assertEqual(msg, str(ctx.exception)) + + def test_input_type_checks(self): + unified = difflib.unified_diff + context = difflib.context_diff + + expect = "input must be a sequence of strings, not str" + self._assert_type_error(expect, unified, 'a', ['b']) + self._assert_type_error(expect, context, 'a', ['b']) + + self._assert_type_error(expect, unified, ['a'], 'b') + self._assert_type_error(expect, context, ['a'], 'b') + + expect = "lines to compare must be str, not NoneType (None)" + self._assert_type_error(expect, unified, ['a'], [None]) + self._assert_type_error(expect, context, ['a'], [None]) + def test_mixed_types_content(self): # type of input content must be consistent: all str or all bytes a = [b'hello'] @@ -475,10 +517,6 @@ def test_mixed_types_dates(self): b = ['bar\n'] list(difflib.unified_diff(a, b, 'a', 'b', datea, dateb)) - def _assert_type_error(self, msg, generator, *args): - with self.assertRaises(TypeError) as ctx: - list(generator(*args)) - self.assertEqual(msg, str(ctx.exception)) class TestJunkAPIs(unittest.TestCase): def test_is_line_junk_true(self): diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index a93cb509b651c5..bba2ac80aa6769 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1,18 +1,23 @@ # Minimal tests for dis module +import ast import contextlib import dis import functools import io +import itertools +import opcode import re import sys +import tempfile +import textwrap import types import unittest from test.support import (captured_stdout, requires_debug_ranges, - requires_specialization, cpython_only) + requires_specialization, cpython_only, + os_helper, import_helper) from test.support.bytecode_helper import BytecodeTestCase -import opcode CACHE = dis.opmap["CACHE"] @@ -47,42 +52,46 @@ def cm(cls, x): %3d RESUME 0 %3d LOAD_FAST 1 (x) - LOAD_CONST 1 (1) + LOAD_SMALL_INT 1 COMPARE_OP 72 (==) LOAD_FAST 0 (self) STORE_ATTR 0 (x) - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE """ % (_C.__init__.__code__.co_firstlineno, _C.__init__.__code__.co_firstlineno + 1,) dis_c_instance_method_bytes = """\ RESUME 0 LOAD_FAST 1 - LOAD_CONST 1 + LOAD_SMALL_INT 1 COMPARE_OP 72 (==) LOAD_FAST 0 STORE_ATTR 0 - RETURN_CONST 0 + LOAD_CONST 0 + RETURN_VALUE """ dis_c_class_method = """\ %3d RESUME 0 %3d LOAD_FAST 1 (x) - LOAD_CONST 1 (1) + LOAD_SMALL_INT 1 COMPARE_OP 72 (==) LOAD_FAST 0 (cls) STORE_ATTR 0 (x) - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE """ % (_C.cm.__code__.co_firstlineno, _C.cm.__code__.co_firstlineno + 2,) dis_c_static_method = """\ %3d RESUME 0 %3d LOAD_FAST 0 (x) - LOAD_CONST 1 (1) + LOAD_SMALL_INT 1 COMPARE_OP 72 (==) STORE_FAST 0 (x) - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE """ % (_C.sm.__code__.co_firstlineno, _C.sm.__code__.co_firstlineno + 2,) # Class disassembling info has an extra newline at end. @@ -109,7 +118,8 @@ def _f(a): CALL 1 POP_TOP -%3d RETURN_CONST 1 (1) +%3d LOAD_SMALL_INT 1 + RETURN_VALUE """ % (_f.__code__.co_firstlineno, _f.__code__.co_firstlineno + 1, _f.__code__.co_firstlineno + 2) @@ -122,11 +132,23 @@ def _f(a): 14 CALL 1 22 POP_TOP -%3d 24 RETURN_CONST 1 (1) +%3d 24 LOAD_SMALL_INT 1 + 26 RETURN_VALUE """ % (_f.__code__.co_firstlineno, _f.__code__.co_firstlineno + 1, _f.__code__.co_firstlineno + 2) +dis_f_with_positions_format = f"""\ +%-14s RESUME 0 + +%-14s LOAD_GLOBAL 1 (print + NULL) +%-14s LOAD_FAST 0 (a) +%-14s CALL 1 +%-14s POP_TOP + +%-14s LOAD_SMALL_INT 1 +%-14s RETURN_VALUE +""" dis_f_co_code = """\ RESUME 0 @@ -134,7 +156,8 @@ def _f(a): LOAD_FAST 0 CALL 1 POP_TOP - RETURN_CONST 1 + LOAD_SMALL_INT 1 + RETURN_VALUE """ def bug708901(): @@ -146,9 +169,9 @@ def bug708901(): %3d RESUME 0 %3d LOAD_GLOBAL 1 (range + NULL) - LOAD_CONST 1 (1) + LOAD_SMALL_INT 1 -%3d LOAD_CONST 2 (10) +%3d LOAD_SMALL_INT 10 %3d CALL 2 GET_ITER @@ -158,8 +181,9 @@ def bug708901(): %3d JUMP_BACKWARD 5 (to L1) %3d L2: END_FOR - POP_TOP - RETURN_CONST 0 (None) + POP_ITER + LOAD_CONST 0 (None) + RETURN_VALUE """ % (bug708901.__code__.co_firstlineno, bug708901.__code__.co_firstlineno + 1, bug708901.__code__.co_firstlineno + 2, @@ -176,14 +200,14 @@ def bug1333982(x=[]): dis_bug1333982 = """\ %3d RESUME 0 -%3d LOAD_ASSERTION_ERROR - LOAD_CONST 1 ( at 0x..., file "%s", line %d>) +%3d LOAD_COMMON_CONSTANT 0 (AssertionError) + LOAD_CONST 0 ( at 0x..., file "%s", line %d>) MAKE_FUNCTION LOAD_FAST 0 (x) GET_ITER CALL 0 -%3d LOAD_CONST 2 (1) +%3d LOAD_SMALL_INT 1 %3d BINARY_OP 0 (+) CALL 0 @@ -206,16 +230,17 @@ def bug42562(): dis_bug42562 = """\ RESUME 0 - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE """ # Extended arg followed by NOP code_bug_45757 = bytes([ - opcode.opmap['EXTENDED_ARG'], 0x01, # EXTENDED_ARG 0x01 - opcode.opmap['NOP'], 0xFF, # NOP 0xFF - opcode.opmap['EXTENDED_ARG'], 0x01, # EXTENDED_ARG 0x01 - opcode.opmap['LOAD_CONST'], 0x29, # LOAD_CONST 0x29 - opcode.opmap['RETURN_VALUE'], 0x00, # RETURN_VALUE 0x00 + opcode.opmap['EXTENDED_ARG'], 0x01, + opcode.opmap['NOP'], 0xFF, + opcode.opmap['EXTENDED_ARG'], 0x01, + opcode.opmap['LOAD_CONST'], 0x29, + opcode.opmap['RETURN_VALUE'], 0x00, ]) dis_bug_45757 = """\ @@ -252,25 +277,27 @@ def wrap_func_w_kwargs(): %3d RESUME 0 %3d LOAD_GLOBAL 1 (func_w_kwargs + NULL) - LOAD_CONST 1 (1) - LOAD_CONST 2 (2) - LOAD_CONST 3 (5) - LOAD_CONST 4 (('c',)) + LOAD_SMALL_INT 1 + LOAD_SMALL_INT 2 + LOAD_SMALL_INT 5 + LOAD_CONST 0 (('c',)) CALL_KW 3 POP_TOP - RETURN_CONST 0 (None) + LOAD_CONST 1 (None) + RETURN_VALUE """ % (wrap_func_w_kwargs.__code__.co_firstlineno, wrap_func_w_kwargs.__code__.co_firstlineno + 1) dis_intrinsic_1_2 = """\ 0 RESUME 0 - 1 LOAD_CONST 0 (0) - LOAD_CONST 1 (('*',)) + 1 LOAD_SMALL_INT 0 + LOAD_CONST 0 (('*',)) IMPORT_NAME 0 (math) CALL_INTRINSIC_1 2 (INTRINSIC_IMPORT_STAR) POP_TOP - RETURN_CONST 2 (None) + LOAD_CONST 1 (None) + RETURN_VALUE """ dis_intrinsic_1_5 = """\ @@ -296,7 +323,8 @@ def wrap_func_w_kwargs(): %3d LOAD_GLOBAL 0 (spam) POP_TOP - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE """ _BIG_LINENO_FORMAT2 = """\ @@ -304,17 +332,20 @@ def wrap_func_w_kwargs(): %4d LOAD_GLOBAL 0 (spam) POP_TOP - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE """ dis_module_expected_results = """\ Disassembly of f: 4 RESUME 0 - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE Disassembly of g: 5 RESUME 0 - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE """ @@ -324,7 +355,7 @@ def wrap_func_w_kwargs(): 0 RESUME 0 1 LOAD_NAME 0 (x) - LOAD_CONST 0 (1) + LOAD_SMALL_INT 1 BINARY_OP 0 (+) RETURN_VALUE """ @@ -335,10 +366,11 @@ def wrap_func_w_kwargs(): 0 RESUME 0 1 LOAD_NAME 0 (x) - LOAD_CONST 0 (1) + LOAD_SMALL_INT 1 BINARY_OP 0 (+) STORE_NAME 0 (x) - RETURN_CONST 1 (None) + LOAD_CONST 0 (None) + RETURN_VALUE """ annot_stmt_str = """\ @@ -352,32 +384,40 @@ def wrap_func_w_kwargs(): dis_annot_stmt_str = """\ 0 RESUME 0 - 2 SETUP_ANNOTATIONS - LOAD_CONST 0 (1) + 2 LOAD_SMALL_INT 1 STORE_NAME 0 (x) - LOAD_NAME 1 (int) - LOAD_NAME 2 (__annotations__) - LOAD_CONST 1 ('x') - STORE_SUBSCR - 3 LOAD_NAME 3 (fun) + 4 LOAD_SMALL_INT 1 + LOAD_NAME 1 (lst) + LOAD_NAME 2 (fun) PUSH_NULL - LOAD_CONST 0 (1) + LOAD_SMALL_INT 0 CALL 1 - LOAD_NAME 2 (__annotations__) - LOAD_CONST 2 ('y') STORE_SUBSCR - 4 LOAD_CONST 0 (1) - LOAD_NAME 4 (lst) - LOAD_NAME 3 (fun) - PUSH_NULL - LOAD_CONST 3 (0) - CALL 1 - STORE_SUBSCR - LOAD_NAME 1 (int) - POP_TOP - RETURN_CONST 4 (None) + 2 LOAD_CONST 0 (", line 2>) + MAKE_FUNCTION + STORE_NAME 3 (__annotate__) + LOAD_CONST 1 (None) + RETURN_VALUE +""" + +fn_with_annotate_str = """ +def foo(a: int, b: str) -> str: + return a * b +""" + +dis_fn_with_annotate_str = """\ + 0 RESUME 0 + + 2 LOAD_CONST 0 (", line 2>) + MAKE_FUNCTION + LOAD_CONST 1 (", line 2>) + MAKE_FUNCTION + SET_FUNCTION_ATTRIBUTE 16 (annotate) + STORE_NAME 0 (foo) + LOAD_CONST 2 (None) + RETURN_VALUE """ compound_stmt_str = """\ @@ -389,17 +429,16 @@ def wrap_func_w_kwargs(): dis_compound_stmt_str = """\ 0 RESUME 0 - 1 LOAD_CONST 0 (0) + 1 LOAD_SMALL_INT 0 STORE_NAME 0 (x) - 2 NOP + 2 L1: NOP - 3 L1: LOAD_NAME 0 (x) - LOAD_CONST 1 (1) + 3 LOAD_NAME 0 (x) + LOAD_SMALL_INT 1 BINARY_OP 13 (+=) STORE_NAME 0 (x) - - 2 JUMP_BACKWARD 7 (to L1) + JUMP_BACKWARD 12 (to L1) """ dis_traceback = """\ @@ -407,8 +446,8 @@ def wrap_func_w_kwargs(): %4d NOP -%4d L1: LOAD_CONST 1 (1) - LOAD_CONST 2 (0) +%4d L1: LOAD_SMALL_INT 1 + LOAD_SMALL_INT 0 --> BINARY_OP 11 (/) POP_TOP @@ -419,7 +458,8 @@ def wrap_func_w_kwargs(): %4d LOAD_GLOBAL 0 (Exception) CHECK_EXC_MATCH - POP_JUMP_IF_FALSE 23 (to L7) + POP_JUMP_IF_FALSE 24 (to L7) + NOT_TAKEN STORE_FAST 0 (e) %4d L4: LOAD_FAST 0 (e) @@ -465,18 +505,18 @@ def _fstring(a, b, c, d): %3d LOAD_FAST 0 (a) FORMAT_SIMPLE - LOAD_CONST 1 (' ') + LOAD_CONST 0 (' ') LOAD_FAST 1 (b) - LOAD_CONST 2 ('4') + LOAD_CONST 1 ('4') FORMAT_WITH_SPEC - LOAD_CONST 1 (' ') + LOAD_CONST 0 (' ') LOAD_FAST 2 (c) CONVERT_VALUE 2 (repr) FORMAT_SIMPLE - LOAD_CONST 1 (' ') + LOAD_CONST 0 (' ') LOAD_FAST 3 (d) CONVERT_VALUE 2 (repr) - LOAD_CONST 2 ('4') + LOAD_CONST 1 ('4') FORMAT_WITH_SPEC BUILD_STRING 7 RETURN_VALUE @@ -491,42 +531,51 @@ def _with(c): %4d RESUME 0 %4d LOAD_FAST 0 (c) - BEFORE_WITH + COPY 1 + LOAD_SPECIAL 1 (__exit__) + SWAP 2 + SWAP 3 + LOAD_SPECIAL 0 (__enter__) + CALL 0 L1: POP_TOP -%4d LOAD_CONST 1 (1) +%4d LOAD_SMALL_INT 1 STORE_FAST 1 (x) %4d L2: LOAD_CONST 0 (None) LOAD_CONST 0 (None) LOAD_CONST 0 (None) - CALL 2 + CALL 3 POP_TOP -%4d LOAD_CONST 2 (2) +%4d LOAD_SMALL_INT 2 STORE_FAST 2 (y) - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE %4d L3: PUSH_EXC_INFO WITH_EXCEPT_START TO_BOOL - POP_JUMP_IF_TRUE 1 (to L4) + POP_JUMP_IF_TRUE 2 (to L4) + NOT_TAKEN RERAISE 2 L4: POP_TOP L5: POP_EXCEPT POP_TOP POP_TOP + POP_TOP -%4d LOAD_CONST 2 (2) +%4d LOAD_SMALL_INT 2 STORE_FAST 2 (y) - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE -- L6: COPY 3 POP_EXCEPT RERAISE 1 ExceptionTable: - L1 to L2 -> L3 [1] lasti - L3 to L5 -> L6 [3] lasti + L1 to L2 -> L3 [2] lasti + L3 to L5 -> L6 [4] lasti """ % (_with.__code__.co_firstlineno, _with.__code__.co_firstlineno + 1, _with.__code__.co_firstlineno + 2, @@ -547,7 +596,12 @@ async def _asyncwith(c): L1: RESUME 0 %4d LOAD_FAST 0 (c) - BEFORE_ASYNC_WITH + COPY 1 + LOAD_SPECIAL 3 (__aexit__) + SWAP 2 + SWAP 3 + LOAD_SPECIAL 2 (__aenter__) + CALL 0 GET_AWAITABLE 1 LOAD_CONST 0 (None) L2: SEND 3 (to L5) @@ -557,13 +611,13 @@ async def _asyncwith(c): L5: END_SEND L6: POP_TOP -%4d LOAD_CONST 1 (1) +%4d LOAD_SMALL_INT 1 STORE_FAST 1 (x) %4d L7: LOAD_CONST 0 (None) LOAD_CONST 0 (None) LOAD_CONST 0 (None) - CALL 2 + CALL 3 GET_AWAITABLE 2 LOAD_CONST 0 (None) L8: SEND 3 (to L11) @@ -573,14 +627,15 @@ async def _asyncwith(c): L11: END_SEND POP_TOP -%4d LOAD_CONST 2 (2) +%4d LOAD_SMALL_INT 2 STORE_FAST 2 (y) - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE %4d L12: CLEANUP_THROW - L13: JUMP_BACKWARD_NO_INTERRUPT 25 (to L5) + L13: JUMP_BACKWARD_NO_INTERRUPT 26 (to L5) L14: CLEANUP_THROW - L15: JUMP_BACKWARD_NO_INTERRUPT 9 (to L11) + L15: JUMP_BACKWARD_NO_INTERRUPT 10 (to L11) L16: PUSH_EXC_INFO WITH_EXCEPT_START GET_AWAITABLE 2 @@ -592,35 +647,39 @@ async def _asyncwith(c): L20: CLEANUP_THROW L21: END_SEND TO_BOOL - POP_JUMP_IF_TRUE 1 (to L22) - RERAISE 2 - L22: POP_TOP - L23: POP_EXCEPT + POP_JUMP_IF_TRUE 2 (to L24) + L22: NOT_TAKEN + L23: RERAISE 2 + L24: POP_TOP + L25: POP_EXCEPT + POP_TOP POP_TOP POP_TOP -%4d LOAD_CONST 2 (2) +%4d LOAD_SMALL_INT 2 STORE_FAST 2 (y) - RETURN_CONST 0 (None) + LOAD_CONST 0 (None) + RETURN_VALUE - -- L24: COPY 3 + -- L26: COPY 3 POP_EXCEPT RERAISE 1 - L25: CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) + L27: CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) RERAISE 1 ExceptionTable: - L1 to L3 -> L25 [0] lasti - L3 to L4 -> L12 [3] - L4 to L6 -> L25 [0] lasti - L6 to L7 -> L16 [1] lasti - L7 to L9 -> L25 [0] lasti + L1 to L3 -> L27 [0] lasti + L3 to L4 -> L12 [4] + L4 to L6 -> L27 [0] lasti + L6 to L7 -> L16 [2] lasti + L7 to L9 -> L27 [0] lasti L9 to L10 -> L14 [2] - L10 to L13 -> L25 [0] lasti - L14 to L15 -> L25 [0] lasti - L16 to L18 -> L24 [3] lasti - L18 to L19 -> L20 [6] - L19 to L23 -> L24 [3] lasti - L23 to L25 -> L25 [0] lasti + L10 to L13 -> L27 [0] lasti + L14 to L15 -> L27 [0] lasti + L16 to L18 -> L26 [4] lasti + L18 to L19 -> L20 [7] + L19 to L22 -> L26 [4] lasti + L23 to L25 -> L26 [4] lasti + L25 to L27 -> L27 [0] lasti """ % (_asyncwith.__code__.co_firstlineno, _asyncwith.__code__.co_firstlineno + 1, _asyncwith.__code__.co_firstlineno + 2, @@ -688,7 +747,8 @@ def _tryfinallyconst(b): PUSH_NULL CALL 0 POP_TOP - RETURN_CONST 1 (1) + LOAD_SMALL_INT 1 + RETURN_VALUE -- L1: PUSH_EXC_INFO @@ -733,7 +793,7 @@ def foo(x): %4d LOAD_FAST 0 (y) BUILD_TUPLE 1 - LOAD_CONST 1 () + LOAD_CONST 0 () MAKE_FUNCTION SET_FUNCTION_ATTRIBUTE 8 (closure) STORE_FAST 1 (foo) @@ -782,7 +842,8 @@ def foo(x): POP_TOP L1: RESUME 0 LOAD_FAST 0 (.0) - L2: FOR_ITER 10 (to L3) + GET_ITER + L2: FOR_ITER 14 (to L3) STORE_FAST 1 (z) LOAD_DEREF 2 (x) LOAD_FAST 1 (z) @@ -790,10 +851,11 @@ def foo(x): YIELD_VALUE 0 RESUME 5 POP_TOP - JUMP_BACKWARD 12 (to L2) + JUMP_BACKWARD 16 (to L2) L3: END_FOR - POP_TOP - RETURN_CONST 0 (None) + POP_ITER + LOAD_CONST 0 (None) + RETURN_VALUE -- L4: CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) RERAISE 1 @@ -830,9 +892,9 @@ def loop_test(): %3d RESUME_CHECK 0 %3d BUILD_LIST 0 - LOAD_CONST 1 ((1, 2, 3)) + LOAD_CONST_MORTAL 0 ((1, 2, 3)) LIST_EXTEND 1 - LOAD_CONST 2 (3) + LOAD_SMALL_INT 3 BINARY_OP 5 (*) GET_ITER L1: FOR_ITER_LIST 14 (to L2) @@ -840,13 +902,14 @@ def loop_test(): %3d LOAD_GLOBAL_MODULE 1 (load_test + NULL) LOAD_FAST 0 (i) - CALL_PY_WITH_DEFAULTS 1 + CALL_PY_GENERAL 1 POP_TOP - JUMP_BACKWARD 16 (to L1) + JUMP_BACKWARD_{: <6} 16 (to L1) %3d L2: END_FOR - POP_TOP - RETURN_CONST 0 (None) + POP_ITER + LOAD_CONST_IMMORTAL 1 (None) + RETURN_VALUE """ % (loop_test.__code__.co_firstlineno, loop_test.__code__.co_firstlineno + 1, loop_test.__code__.co_firstlineno + 2, @@ -858,17 +921,16 @@ def extended_arg_quick(): dis_extended_arg_quick_code = """\ %3d RESUME 0 -%3d LOAD_CONST 1 (Ellipsis) +%3d LOAD_CONST 0 (Ellipsis) EXTENDED_ARG 1 UNPACK_EX 256 POP_TOP STORE_FAST 0 (_) - RETURN_CONST 0 (None) + LOAD_CONST 1 (None) + RETURN_VALUE """% (extended_arg_quick.__code__.co_firstlineno, extended_arg_quick.__code__.co_firstlineno + 1,) -ADAPTIVE_WARMUP_DELAY = 2 - class DisTestBase(unittest.TestCase): "Common utilities for DisTests and TestDisTraceback" @@ -935,12 +997,14 @@ def test_boundaries(self): def test_widths(self): long_opcodes = set(['JUMP_BACKWARD_NO_INTERRUPT', 'INSTRUMENTED_CALL_FUNCTION_EX']) - for opcode, opname in enumerate(dis.opname): + for op, opname in enumerate(dis.opname): if opname in long_opcodes or opname.startswith("INSTRUMENTED"): continue + if opname in opcode._specialized_opmap: + continue with self.subTest(opname=opname): width = dis._OPNAME_WIDTH - if opcode in dis.hasarg: + if op in dis.hasarg: width += 1 + dis._OPARG_WIDTH self.assertLessEqual(len(opname), width) @@ -950,6 +1014,82 @@ def test_dis(self): def test_dis_with_offsets(self): self.do_disassembly_test(_f, dis_f_with_offsets, show_offsets=True) + @requires_debug_ranges() + def test_dis_with_all_positions(self): + def format_instr_positions(instr): + values = tuple('?' if p is None else p for p in instr.positions) + return '%s:%s-%s:%s' % (values[0], values[2], values[1], values[3]) + + instrs = list(dis.get_instructions(_f)) + for instr in instrs: + with self.subTest(instr=instr): + self.assertTrue(all(p is not None for p in instr.positions)) + positions = tuple(map(format_instr_positions, instrs)) + expected = dis_f_with_positions_format % positions + self.do_disassembly_test(_f, expected, show_positions=True) + + @requires_debug_ranges() + def test_dis_with_some_positions(self): + code = ("def f():\n" + " try: pass\n" + " finally:pass") + f = compile(ast.parse(code), "?", "exec").co_consts[0] + + expect = '\n'.join([ + '1:0-1:0 RESUME 0', + '', + '2:3-3:15 NOP', + '', + '3:11-3:15 LOAD_CONST 0 (None)', + '3:11-3:15 RETURN_VALUE', + '', + ' -- L1: PUSH_EXC_INFO', + '', + '3:11-3:15 RERAISE 0', + '', + ' -- L2: COPY 3', + ' -- POP_EXCEPT', + ' -- RERAISE 1', + 'ExceptionTable:', + ' L1 to L2 -> L2 [1] lasti', + '', + ]) + self.do_disassembly_test(f, expect, show_positions=True) + + @requires_debug_ranges() + def test_dis_with_linenos_but_no_columns(self): + code = "def f():\n\tx = 1" + tree = ast.parse(code) + func = tree.body[0] + ass_x = func.body[0].targets[0] + # remove columns information but keep line information + ass_x.col_offset = ass_x.end_col_offset = -1 + f = compile(tree, "?", "exec").co_consts[0] + + expect = '\n'.join([ + '1:0-1:0 RESUME 0', + '', + '2:5-2:6 LOAD_SMALL_INT 1', + '2:?-2:? STORE_FAST 0 (x)', + '2:?-2:? LOAD_CONST 0 (None)', + '2:?-2:? RETURN_VALUE', + '', + ]) + self.do_disassembly_test(f, expect, show_positions=True) + + def test_dis_with_no_positions(self): + def f(): + pass + + f.__code__ = f.__code__.replace(co_linetable=b'') + expect = '\n'.join([ + ' RESUME 0', + ' LOAD_CONST 0 (None)', + ' RETURN_VALUE', + '', + ]) + self.do_disassembly_test(f, expect, show_positions=True) + def test_bug_708901(self): self.do_disassembly_test(bug708901, dis_bug708901) @@ -1014,6 +1154,7 @@ def test_disassemble_str(self): self.do_disassembly_test(expr_str, dis_expr_str) self.do_disassembly_test(simple_stmt_str, dis_simple_stmt_str) self.do_disassembly_test(annot_stmt_str, dis_annot_stmt_str) + self.do_disassembly_test(fn_with_annotate_str, dis_fn_with_annotate_str) self.do_disassembly_test(compound_stmt_str, dis_compound_stmt_str) def test_disassemble_bytes(self): @@ -1116,8 +1257,9 @@ def test__try_compile_no_context_exc_on_error(self): self.assertIsNone(e.__context__) @staticmethod - def code_quicken(f, times=ADAPTIVE_WARMUP_DELAY): - for _ in range(times): + def code_quicken(f): + _testinternalcapi = import_helper.import_module("_testinternalcapi") + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): f() @cpython_only @@ -1127,52 +1269,13 @@ def test_super_instructions(self): got = self.get_disassembly(load_test, adaptive=True) self.do_disassembly_compare(got, dis_load_test_quickened_code) - @cpython_only - @requires_specialization - def test_binary_specialize(self): - binary_op_quicken = """\ - 0 RESUME_CHECK 0 - - 1 LOAD_NAME 0 (a) - LOAD_NAME 1 (b) - %s - RETURN_VALUE -""" - co_int = compile('a + b', "", "eval") - self.code_quicken(lambda: exec(co_int, {}, {'a': 1, 'b': 2})) - got = self.get_disassembly(co_int, adaptive=True) - self.do_disassembly_compare(got, binary_op_quicken % "BINARY_OP_ADD_INT 0 (+)") - - co_unicode = compile('a + b', "", "eval") - self.code_quicken(lambda: exec(co_unicode, {}, {'a': 'a', 'b': 'b'})) - got = self.get_disassembly(co_unicode, adaptive=True) - self.do_disassembly_compare(got, binary_op_quicken % "BINARY_OP_ADD_UNICODE 0 (+)") - - binary_subscr_quicken = """\ - 0 RESUME_CHECK 0 - - 1 LOAD_NAME 0 (a) - LOAD_CONST 0 (0) - %s - RETURN_VALUE -""" - co_list = compile('a[0]', "", "eval") - self.code_quicken(lambda: exec(co_list, {}, {'a': [0]})) - got = self.get_disassembly(co_list, adaptive=True) - self.do_disassembly_compare(got, binary_subscr_quicken % "BINARY_SUBSCR_LIST_INT") - - co_dict = compile('a[0]', "", "eval") - self.code_quicken(lambda: exec(co_dict, {}, {'a': {0: '1'}})) - got = self.get_disassembly(co_dict, adaptive=True) - self.do_disassembly_compare(got, binary_subscr_quicken % "BINARY_SUBSCR_DICT") - @cpython_only @requires_specialization def test_load_attr_specialize(self): load_attr_quicken = """\ 0 RESUME_CHECK 0 - 1 LOAD_CONST 0 ('a') + 1 LOAD_CONST_IMMORTAL 0 ('a') LOAD_ATTR_SLOT 0 (__class__) RETURN_VALUE """ @@ -1189,7 +1292,7 @@ def test_call_specialize(self): 1 LOAD_NAME 0 (str) PUSH_NULL - LOAD_CONST 0 (1) + LOAD_SMALL_INT 1 CALL_STR_1 1 RETURN_VALUE """ @@ -1201,21 +1304,44 @@ def test_call_specialize(self): @cpython_only @requires_specialization def test_loop_quicken(self): - import _testinternalcapi # Loop can trigger a quicken where the loop is located - self.code_quicken(loop_test, 1) + self.code_quicken(loop_test) got = self.get_disassembly(loop_test, adaptive=True) - expected = dis_loop_test_quickened_code - if _testinternalcapi.get_optimizer(): - # We *may* see ENTER_EXECUTOR in the disassembly. This is a - # temporary hack to keep the test working until dis is able to - # handle the instruction correctly (GH-112383): - got = got.replace( - "ENTER_EXECUTOR 16", - "JUMP_BACKWARD 16 (to L1)", - ) + jit = import_helper.import_module("_testinternalcapi").jit_enabled() + expected = dis_loop_test_quickened_code.format("JIT" if jit else "NO_JIT") self.do_disassembly_compare(got, expected) + @cpython_only + @requires_specialization + def test_loop_with_conditional_at_end_is_quickened(self): + _testinternalcapi = import_helper.import_module("_testinternalcapi") + def for_loop_true(x): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + if x: + pass + + for_loop_true(True) + self.assertIn('FOR_ITER_RANGE', + self.get_disassembly(for_loop_true, adaptive=True)) + + def for_loop_false(x): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + if x: + pass + + for_loop_false(False) + self.assertIn('FOR_ITER_RANGE', + self.get_disassembly(for_loop_false, adaptive=True)) + + def while_loop(): + i = 0 + while i < _testinternalcapi.SPECIALIZATION_THRESHOLD: + i += 1 + + while_loop() + self.assertIn('COMPARE_OP_INT', + self.get_disassembly(while_loop, adaptive=True)) + @cpython_only def test_extended_arg_quick(self): got = self.get_disassembly(extended_arg_quick) @@ -1311,7 +1437,7 @@ def get_disassembly(self, func, lasti=-1, wrapper=True, **kwargs): Kw-only arguments: 0 Number of locals: 1 Stack size: \\d+ -Flags: OPTIMIZED, NEWLOCALS +Flags: OPTIMIZED, NEWLOCALS, HAS_DOCSTRING Constants: {code_info_consts} Names: @@ -1337,8 +1463,8 @@ def f(c=c): Stack size: \\d+ Flags: OPTIMIZED, NEWLOCALS, VARARGS, VARKEYWORDS, GENERATOR Constants: - 0: None - 1: + 0: + 1: None Variable names: 0: a 1: b @@ -1356,10 +1482,12 @@ def f(c=c): 2: [abedfxyz] 3: [abedfxyz] 4: [abedfxyz] - 5: [abedfxyz]""" + 5: [abedfxyz] + 6: [abedfxyz] + 7: [abedfxyz]""" # NOTE: the order of the cell variables above depends on dictionary order! -co_tricky_nested_f = tricky.__func__.__code__.co_consts[1] +co_tricky_nested_f = tricky.__func__.__code__.co_consts[0] code_info_tricky_nested_f = """\ Filename: (.*) @@ -1392,8 +1520,6 @@ def f(c=c): Number of locals: 0 Stack size: \\d+ Flags: 0x0 -Constants: - 0: 1 Names: 0: x""" @@ -1407,8 +1533,7 @@ def f(c=c): Stack size: \\d+ Flags: 0x0 Constants: - 0: 1 - 1: None + 0: None Names: 0: x""" @@ -1422,8 +1547,7 @@ def f(c=c): Stack size: \\d+ Flags: 0x0 Constants: - 0: 0 - 1: 1 + 0: None Names: 0: x""" @@ -1444,7 +1568,6 @@ async def async_def(): Flags: OPTIMIZED, NEWLOCALS, COROUTINE Constants: 0: None - 1: 1 Names: 0: b 1: c @@ -1528,9 +1651,9 @@ def jumpy(): # End fodder for opinfo generation tests expected_outer_line = 1 _line_offset = outer.__code__.co_firstlineno - 1 -code_object_f = outer.__code__.co_consts[1] +code_object_f = outer.__code__.co_consts[0] expected_f_line = code_object_f.co_firstlineno - _line_offset -code_object_inner = code_object_f.co_consts[1] +code_object_inner = code_object_f.co_consts[0] expected_inner_line = code_object_inner.co_firstlineno - _line_offset expected_jumpy_line = 1 @@ -1583,198 +1706,211 @@ def _prepare_test_cases(): Instruction = dis.Instruction expected_opinfo_outer = [ - Instruction(opname='MAKE_CELL', opcode=94, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None), - Instruction(opname='MAKE_CELL', opcode=94, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=4, start_offset=4, starts_line=True, line_number=1, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=5, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='BUILD_TUPLE', opcode=52, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=1, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='MAKE_FUNCTION', opcode=26, arg=None, argval=None, argrepr='', offset=16, start_offset=16, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=106, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=106, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='STORE_FAST', opcode=110, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=False, line_number=2, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=True, line_number=7, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval='', argrepr="''", offset=38, start_offset=38, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=3, argval=1, argrepr='1', offset=40, start_offset=40, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='BUILD_LIST', opcode=47, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='BUILD_MAP', opcode=48, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=7, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8, label=None, positions=None), - Instruction(opname='RETURN_VALUE', opcode=36, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=8, label=None, positions=None), + Instruction(opname='MAKE_CELL', opcode=94, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=94, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=4, start_offset=4, starts_line=True, line_number=1, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_TUPLE', opcode=51, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=0, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_FUNCTION', opcode=24, arg=None, argval=None, argrepr='', offset=16, start_offset=16, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=105, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=105, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='STORE_FAST', opcode=109, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=True, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=82, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=82, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=1, argval='', argrepr="''", offset=38, start_offset=38, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=1, argval=1, argrepr='', offset=40, start_offset=40, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_LIST', opcode=46, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_MAP', opcode=47, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=2, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=36, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), ] expected_opinfo_f = [ - Instruction(opname='COPY_FREE_VARS', opcode=62, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None), - Instruction(opname='MAKE_CELL', opcode=94, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='MAKE_CELL', opcode=94, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=False, line_number=None, label=None, positions=None), - Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=True, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='BUILD_TUPLE', opcode=52, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=1, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='MAKE_FUNCTION', opcode=26, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=106, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=106, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='STORE_FAST', opcode=110, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=5, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=True, line_number=6, label=None, positions=None), - Instruction(opname='RETURN_VALUE', opcode=36, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=6, label=None, positions=None), + Instruction(opname='COPY_FREE_VARS', opcode=60, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=94, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=94, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=1, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=True, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_TUPLE', opcode=51, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=0, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_FUNCTION', opcode=24, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=105, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=105, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='STORE_FAST', opcode=109, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=82, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=82, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=82, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=82, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=True, line_number=6, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=36, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=6, label=None, positions=None, cache_info=None), ] expected_opinfo_inner = [ - Instruction(opname='COPY_FREE_VARS', opcode=62, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None), - Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=True, line_number=4, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='LOAD_DEREF', opcode=84, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=88, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='CALL', opcode=53, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=False, line_number=4, label=None, positions=None), - Instruction(opname='RETURN_CONST', opcode=103, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=False, line_number=4, label=None, positions=None), + Instruction(opname='COPY_FREE_VARS', opcode=60, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=82, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=82, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=82, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=82, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=86, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=36, arg=None, argval=None, argrepr='', offset=36, start_offset=36, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), ] expected_opinfo_jumpy = [ Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=1, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=83, arg=1, argval=10, argrepr='10', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='GET_ITER', opcode=19, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='FOR_ITER', opcode=72, arg=30, argval=88, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='STORE_FAST', opcode=110, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=10, argval=10, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='GET_ITER', opcode=18, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='FOR_ITER', opcode=69, arg=32, argval=92, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=109, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=True, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval=4, argrepr='4', offset=54, start_offset=54, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=68, argrepr='to L2', offset=60, start_offset=60, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=22, argval=24, argrepr='to L1', offset=64, start_offset=64, starts_line=True, line_number=6, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=68, start_offset=68, starts_line=True, line_number=7, label=2, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=3, argval=6, argrepr='6', offset=70, start_offset=70, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=148, argval='>', argrepr='bool(>)', offset=72, start_offset=72, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=2, argval=84, argrepr='to L3', offset=76, start_offset=76, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=30, argval=24, argrepr='to L1', offset=80, start_offset=80, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=True, line_number=8, label=3, positions=None, cache_info=None), - Instruction(opname='JUMP_FORWARD', opcode=79, arg=13, argval=114, argrepr='to L5', offset=86, start_offset=86, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), - Instruction(opname='END_FOR', opcode=11, arg=None, argval=None, argrepr='', offset=88, start_offset=88, starts_line=True, line_number=3, label=4, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=90, start_offset=90, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=92, start_offset=92, starts_line=True, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=83, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=102, start_offset=102, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=104, start_offset=104, starts_line=False, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=112, start_offset=112, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST_CHECK', opcode=87, arg=0, argval='i', argrepr='i', offset=114, start_offset=114, starts_line=True, line_number=11, label=5, positions=None, cache_info=None), - Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=116, start_offset=116, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=40, argval=208, argrepr='to L9', offset=124, start_offset=124, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=128, start_offset=128, starts_line=True, line_number=12, label=6, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=138, start_offset=138, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=140, start_offset=140, starts_line=False, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=148, start_offset=148, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=150, start_offset=150, starts_line=True, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=5, argval=1, argrepr='1', offset=152, start_offset=152, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='BINARY_OP', opcode=45, arg=23, argval=23, argrepr='-=', offset=154, start_offset=154, starts_line=False, line_number=13, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='STORE_FAST', opcode=110, arg=0, argval='i', argrepr='i', offset=158, start_offset=158, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=160, start_offset=160, starts_line=True, line_number=14, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=3, argval=6, argrepr='6', offset=162, start_offset=162, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=148, argval='>', argrepr='bool(>)', offset=164, start_offset=164, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=176, argrepr='to L7', offset=168, start_offset=168, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=31, argval=114, argrepr='to L5', offset=172, start_offset=172, starts_line=True, line_number=15, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=176, start_offset=176, starts_line=True, line_number=16, label=7, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=2, argval=4, argrepr='4', offset=178, start_offset=178, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=58, arg=18, argval='<', argrepr='bool(<)', offset=180, start_offset=180, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=1, argval=190, argrepr='to L8', offset=184, start_offset=184, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_FORWARD', opcode=79, arg=20, argval=230, argrepr='to L10', offset=188, start_offset=188, starts_line=True, line_number=17, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=190, start_offset=190, starts_line=True, line_number=11, label=8, positions=None, cache_info=None), - Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=192, start_offset=192, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=2, argval=208, argrepr='to L9', offset=200, start_offset=200, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=77, arg=40, argval=128, argrepr='to L6', offset=204, start_offset=204, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=208, start_offset=208, starts_line=True, line_number=19, label=9, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=83, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=218, start_offset=218, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=220, start_offset=220, starts_line=False, line_number=19, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=228, start_offset=228, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), - Instruction(opname='NOP', opcode=30, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=True, line_number=20, label=10, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=5, argval=1, argrepr='1', offset=232, start_offset=232, starts_line=True, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=7, argval=0, argrepr='0', offset=234, start_offset=234, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='BINARY_OP', opcode=45, arg=11, argval=11, argrepr='/', offset=236, start_offset=236, starts_line=False, line_number=21, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=240, start_offset=240, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=85, arg=0, argval='i', argrepr='i', offset=242, start_offset=242, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='BEFORE_WITH', opcode=2, arg=None, argval=None, argrepr='', offset=244, start_offset=244, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='STORE_FAST', opcode=110, arg=1, argval='dodgy', argrepr='dodgy', offset=246, start_offset=246, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=248, start_offset=248, starts_line=True, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=83, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=258, start_offset=258, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=260, start_offset=260, starts_line=False, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=268, start_offset=268, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=270, start_offset=270, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=272, start_offset=272, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=83, arg=0, argval=None, argrepr='None', offset=274, start_offset=274, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=2, argval=2, argrepr='', offset=276, start_offset=276, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=284, start_offset=284, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=286, start_offset=286, starts_line=True, line_number=28, label=11, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=83, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=296, start_offset=296, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=298, start_offset=298, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=306, start_offset=306, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='RETURN_CONST', opcode=103, arg=0, argval=None, argrepr='None', offset=308, start_offset=308, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=310, start_offset=310, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='WITH_EXCEPT_START', opcode=44, arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=314, start_offset=314, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=1, argval=328, argrepr='to L12', offset=322, start_offset=322, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='RERAISE', opcode=102, arg=2, argval=2, argrepr='', offset=326, start_offset=326, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=25, label=12, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=332, start_offset=332, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=334, start_offset=334, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=78, arg=26, argval=286, argrepr='to L11', offset=336, start_offset=336, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=338, start_offset=338, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=342, start_offset=342, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=346, start_offset=346, starts_line=True, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='CHECK_EXC_MATCH', opcode=7, arg=None, argval=None, argrepr='', offset=356, start_offset=356, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=14, argval=390, argrepr='to L13', offset=358, start_offset=358, starts_line=False, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=364, start_offset=364, starts_line=True, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=83, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=374, start_offset=374, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=376, start_offset=376, starts_line=False, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=384, start_offset=384, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=386, start_offset=386, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=78, arg=52, argval=286, argrepr='to L11', offset=388, start_offset=388, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=390, start_offset=390, starts_line=True, line_number=22, label=13, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=392, start_offset=392, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=394, start_offset=394, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=396, start_offset=396, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=398, start_offset=398, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=91, arg=3, argval='print', argrepr='print + NULL', offset=400, start_offset=400, starts_line=True, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=83, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=410, start_offset=410, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=53, arg=1, argval=1, argrepr='', offset=412, start_offset=412, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=420, start_offset=420, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=422, start_offset=422, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=61, arg=3, argval=3, argrepr='', offset=424, start_offset=424, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='POP_EXCEPT', opcode=31, arg=None, argval=None, argrepr='', offset=426, start_offset=426, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=428, start_offset=428, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=True, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=4, argval=4, argrepr='', offset=54, start_offset=54, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=56, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=3, argval=70, argrepr='to L2', offset=60, start_offset=60, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=29, arg=None, argval=None, argrepr='', offset=64, start_offset=64, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD', opcode=74, arg=23, argval=24, argrepr='to L1', offset=66, start_offset=66, starts_line=True, line_number=6, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=70, start_offset=70, starts_line=True, line_number=7, label=2, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=6, argval=6, argrepr='', offset=72, start_offset=72, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=56, arg=148, argval='>', argrepr='bool(>)', offset=74, start_offset=74, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=3, argval=88, argrepr='to L3', offset=78, start_offset=78, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=29, arg=None, argval=None, argrepr='', offset=82, start_offset=82, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD', opcode=74, arg=32, argval=24, argrepr='to L1', offset=84, start_offset=84, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=88, start_offset=88, starts_line=True, line_number=8, label=3, positions=None, cache_info=None), + Instruction(opname='JUMP_FORWARD', opcode=76, arg=13, argval=118, argrepr='to L5', offset=90, start_offset=90, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='END_FOR', opcode=10, arg=None, argval=None, argrepr='', offset=92, start_offset=92, starts_line=True, line_number=3, label=4, positions=None, cache_info=None), + Instruction(opname='POP_ITER', opcode=31, arg=None, argval=None, argrepr='', offset=94, start_offset=94, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=96, start_offset=96, starts_line=True, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=0, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=106, start_offset=106, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=1, argval=1, argrepr='', offset=108, start_offset=108, starts_line=False, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=116, start_offset=116, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST_CHECK', opcode=85, arg=0, argval='i', argrepr='i', offset=118, start_offset=118, starts_line=True, line_number=11, label=5, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=120, start_offset=120, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=40, argval=212, argrepr='to L8', offset=128, start_offset=128, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=29, arg=None, argval=None, argrepr='', offset=132, start_offset=132, starts_line=False, line_number=11, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=134, start_offset=134, starts_line=True, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=144, start_offset=144, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=1, argval=1, argrepr='', offset=146, start_offset=146, starts_line=False, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=154, start_offset=154, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=156, start_offset=156, starts_line=True, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=1, argval=1, argrepr='', offset=158, start_offset=158, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='BINARY_OP', opcode=45, arg=23, argval=23, argrepr='-=', offset=160, start_offset=160, starts_line=False, line_number=13, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('descr', 4, b'\x00\x00\x00\x00\x00\x00\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=109, arg=0, argval='i', argrepr='i', offset=172, start_offset=172, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=174, start_offset=174, starts_line=True, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=6, argval=6, argrepr='', offset=176, start_offset=176, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=56, arg=148, argval='>', argrepr='bool(>)', offset=178, start_offset=178, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=3, argval=192, argrepr='to L6', offset=182, start_offset=182, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=29, arg=None, argval=None, argrepr='', offset=186, start_offset=186, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD', opcode=74, arg=37, argval=118, argrepr='to L5', offset=188, start_offset=188, starts_line=True, line_number=15, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=192, start_offset=192, starts_line=True, line_number=16, label=6, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=4, argval=4, argrepr='', offset=194, start_offset=194, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=56, arg=18, argval='<', argrepr='bool(<)', offset=196, start_offset=196, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=3, argval=210, argrepr='to L7', offset=200, start_offset=200, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=29, arg=None, argval=None, argrepr='', offset=204, start_offset=204, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD', opcode=74, arg=46, argval=118, argrepr='to L5', offset=206, start_offset=206, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_FORWARD', opcode=76, arg=11, argval=234, argrepr='to L9', offset=210, start_offset=210, starts_line=True, line_number=17, label=7, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=212, start_offset=212, starts_line=True, line_number=19, label=8, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=1, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=222, start_offset=222, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=1, argval=1, argrepr='', offset=224, start_offset=224, starts_line=False, line_number=19, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=232, start_offset=232, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='NOP', opcode=28, arg=None, argval=None, argrepr='', offset=234, start_offset=234, starts_line=True, line_number=20, label=9, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=1, argval=1, argrepr='', offset=236, start_offset=236, starts_line=True, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SMALL_INT', opcode=91, arg=0, argval=0, argrepr='', offset=238, start_offset=238, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='BINARY_OP', opcode=45, arg=11, argval=11, argrepr='/', offset=240, start_offset=240, starts_line=False, line_number=21, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('descr', 4, b'\x00\x00\x00\x00\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=252, start_offset=252, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=254, start_offset=254, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=59, arg=1, argval=1, argrepr='', offset=256, start_offset=256, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SPECIAL', opcode=92, arg=1, argval=1, argrepr='__exit__', offset=258, start_offset=258, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='SWAP', opcode=114, arg=2, argval=2, argrepr='', offset=260, start_offset=260, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='SWAP', opcode=114, arg=3, argval=3, argrepr='', offset=262, start_offset=262, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SPECIAL', opcode=92, arg=0, argval=0, argrepr='__enter__', offset=264, start_offset=264, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=0, argval=0, argrepr='', offset=266, start_offset=266, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=109, arg=1, argval='dodgy', argrepr='dodgy', offset=274, start_offset=274, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=276, start_offset=276, starts_line=True, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=2, argval='Never reach this', argrepr="'Never reach this'", offset=286, start_offset=286, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=1, argval=1, argrepr='', offset=288, start_offset=288, starts_line=False, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=296, start_offset=296, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=298, start_offset=298, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=300, start_offset=300, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=302, start_offset=302, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=3, argval=3, argrepr='', offset=304, start_offset=304, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=314, start_offset=314, starts_line=True, line_number=28, label=10, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=324, start_offset=324, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=1, argval=1, argrepr='', offset=326, start_offset=326, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=334, start_offset=334, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=None, argrepr='None', offset=336, start_offset=336, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_VALUE', opcode=36, arg=None, argval=None, argrepr='', offset=338, start_offset=338, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='WITH_EXCEPT_START', opcode=44, arg=None, argval=None, argrepr='', offset=342, start_offset=342, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='TO_BOOL', opcode=40, arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=100, arg=2, argval=360, argrepr='to L11', offset=352, start_offset=352, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=29, arg=None, argval=None, argrepr='', offset=356, start_offset=356, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=2, argval=2, argrepr='', offset=358, start_offset=358, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=360, start_offset=360, starts_line=False, line_number=25, label=11, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=30, arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=364, start_offset=364, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=366, start_offset=366, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=368, start_offset=368, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=75, arg=29, argval=314, argrepr='to L10', offset=370, start_offset=370, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=59, arg=3, argval=3, argrepr='', offset=372, start_offset=372, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=30, arg=None, argval=None, argrepr='', offset=374, start_offset=374, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=376, start_offset=376, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=378, start_offset=378, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=380, start_offset=380, starts_line=True, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='CHECK_EXC_MATCH', opcode=6, arg=None, argval=None, argrepr='', offset=390, start_offset=390, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=97, arg=15, argval=426, argrepr='to L12', offset=392, start_offset=392, starts_line=False, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='NOT_TAKEN', opcode=29, arg=None, argval=None, argrepr='', offset=396, start_offset=396, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=398, start_offset=398, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=400, start_offset=400, starts_line=True, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=4, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=410, start_offset=410, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=1, argval=1, argrepr='', offset=412, start_offset=412, starts_line=False, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=420, start_offset=420, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=30, arg=None, argval=None, argrepr='', offset=422, start_offset=422, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=75, arg=56, argval=314, argrepr='to L10', offset=424, start_offset=424, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=426, start_offset=426, starts_line=True, line_number=22, label=12, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=59, arg=3, argval=3, argrepr='', offset=428, start_offset=428, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=30, arg=None, argval=None, argrepr='', offset=430, start_offset=430, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=432, start_offset=432, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='PUSH_EXC_INFO', opcode=33, arg=None, argval=None, argrepr='', offset=434, start_offset=434, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=436, start_offset=436, starts_line=True, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=81, arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=446, start_offset=446, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=52, arg=1, argval=1, argrepr='', offset=448, start_offset=448, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_TOP', opcode=32, arg=None, argval=None, argrepr='', offset=456, start_offset=456, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=0, argval=0, argrepr='', offset=458, start_offset=458, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=59, arg=3, argval=3, argrepr='', offset=460, start_offset=460, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='POP_EXCEPT', opcode=30, arg=None, argval=None, argrepr='', offset=462, start_offset=462, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=102, arg=1, argval=1, argrepr='', offset=464, start_offset=464, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), ] # One last piece of inspect fodder to check the default line number handling def simple(): pass expected_opinfo_simple = [ Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=simple.__code__.co_firstlineno, label=None, positions=None), - Instruction(opname='RETURN_CONST', opcode=103, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=False, line_number=simple.__code__.co_firstlineno, label=None), + Instruction(opname='LOAD_CONST', opcode=81, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=False, line_number=simple.__code__.co_firstlineno, label=None), + Instruction(opname='RETURN_VALUE', opcode=36, arg=None, argval=None, argrepr='', offset=4, start_offset=4, starts_line=False, line_number=simple.__code__.co_firstlineno, label=None), ] @@ -1841,6 +1977,7 @@ def test_co_positions(self): (2, 2, 8, 9), (1, 3, 0, 1), (1, 3, 0, 1), + (1, 3, 0, 1), (1, 3, 0, 1) ] self.assertEqual(positions, expected) @@ -1920,6 +2057,24 @@ def f(x, y, z): dis.dis(f.__code__, file=output, show_caches=True) self.assertIn("L1:", output.getvalue()) + def test_is_op_format(self): + output = io.StringIO() + dis.dis("a is b", file=output, show_caches=True) + self.assertIn("IS_OP 0 (is)", output.getvalue()) + + output = io.StringIO() + dis.dis("a is not b", file=output, show_caches=True) + self.assertIn("IS_OP 1 (is not)", output.getvalue()) + + def test_contains_op_format(self): + output = io.StringIO() + dis.dis("a in b", file=output, show_caches=True) + self.assertIn("CONTAINS_OP 0 (in)", output.getvalue()) + + output = io.StringIO() + dis.dis("a not in b", file=output, show_caches=True) + self.assertIn("CONTAINS_OP 1 (not in)", output.getvalue()) + def test_baseopname_and_baseopcode(self): # Standard instructions for name, code in dis.opmap.items(): @@ -2152,20 +2307,20 @@ def test_bytecode_co_positions(self): class TestBytecodeTestCase(BytecodeTestCase): def test_assert_not_in_with_op_not_in_bytecode(self): code = compile("a = 1", "", "exec") - self.assertInBytecode(code, "LOAD_CONST", 1) + self.assertInBytecode(code, "LOAD_SMALL_INT", 1) self.assertNotInBytecode(code, "LOAD_NAME") self.assertNotInBytecode(code, "LOAD_NAME", "a") def test_assert_not_in_with_arg_not_in_bytecode(self): code = compile("a = 1", "", "exec") - self.assertInBytecode(code, "LOAD_CONST") - self.assertInBytecode(code, "LOAD_CONST", 1) + self.assertInBytecode(code, "LOAD_SMALL_INT") + self.assertInBytecode(code, "LOAD_SMALL_INT", 1) self.assertNotInBytecode(code, "LOAD_CONST", 2) def test_assert_not_in_with_arg_in_bytecode(self): code = compile("a = 1", "", "exec") with self.assertRaises(AssertionError): - self.assertNotInBytecode(code, "LOAD_CONST", 1) + self.assertNotInBytecode(code, "LOAD_SMALL_INT", 1) class TestFinderMethods(unittest.TestCase): def test__find_imports(self): @@ -2289,5 +2444,119 @@ def _unroll_caches_as_Instructions(instrs, show_caches=False): False, None, None, instr.positions) +class TestDisCLI(unittest.TestCase): + + def setUp(self): + self.filename = tempfile.mktemp() + self.addCleanup(os_helper.unlink, self.filename) + + @staticmethod + def text_normalize(string): + """Dedent *string* and strip it from its surrounding whitespaces. + + This method is used by the other utility functions so that any + string to write or to match against can be freely indented. + """ + return textwrap.dedent(string).strip() + + def set_source(self, content): + with open(self.filename, 'w') as fp: + fp.write(self.text_normalize(content)) + + def invoke_dis(self, *flags): + output = io.StringIO() + with contextlib.redirect_stdout(output): + dis.main(args=[*flags, self.filename]) + return self.text_normalize(output.getvalue()) + + def check_output(self, source, expect, *flags): + with self.subTest(source=source, flags=flags): + self.set_source(source) + res = self.invoke_dis(*flags) + expect = self.text_normalize(expect) + self.assertListEqual(res.splitlines(), expect.splitlines()) + + def test_invocation(self): + # test various combinations of parameters + base_flags = [ + ('-C', '--show-caches'), + ('-O', '--show-offsets'), + ('-P', '--show-positions'), + ('-S', '--specialized'), + ] + + self.set_source(''' + def f(): + print(x) + return None + ''') + + for r in range(1, len(base_flags) + 1): + for choices in itertools.combinations(base_flags, r=r): + for args in itertools.product(*choices): + with self.subTest(args=args[1:]): + _ = self.invoke_dis(*args) + + with self.assertRaises(SystemExit): + # suppress argparse error message + with contextlib.redirect_stderr(io.StringIO()): + _ = self.invoke_dis('--unknown') + + def test_show_cache(self): + # test 'python -m dis -C/--show-caches' + source = 'print()' + expect = ''' + 0 RESUME 0 + + 1 LOAD_NAME 0 (print) + PUSH_NULL + CALL 0 + CACHE 0 (counter: 0) + CACHE 0 (func_version: 0) + CACHE 0 + POP_TOP + LOAD_CONST 0 (None) + RETURN_VALUE + ''' + for flag in ['-C', '--show-caches']: + self.check_output(source, expect, flag) + + def test_show_offsets(self): + # test 'python -m dis -O/--show-offsets' + source = 'pass' + expect = ''' + 0 0 RESUME 0 + + 1 2 LOAD_CONST 0 (None) + 4 RETURN_VALUE + ''' + for flag in ['-O', '--show-offsets']: + self.check_output(source, expect, flag) + + def test_show_positions(self): + # test 'python -m dis -P/--show-positions' + source = 'pass' + expect = ''' + 0:0-1:0 RESUME 0 + + 1:0-1:4 LOAD_CONST 0 (None) + 1:0-1:4 RETURN_VALUE + ''' + for flag in ['-P', '--show-positions']: + self.check_output(source, expect, flag) + + def test_specialized_code(self): + # test 'python -m dis -S/--specialized' + source = 'pass' + expect = ''' + 0 RESUME 0 + + 1 LOAD_CONST 0 (None) + RETURN_VALUE + ''' + for flag in ['-S', '--specialized']: + self.check_output(source, expect, flag) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_doctest/sample_doctest_skip.py b/Lib/test/test_doctest/sample_doctest_skip.py new file mode 100644 index 00000000000000..1b83dec1f8c4dc --- /dev/null +++ b/Lib/test/test_doctest/sample_doctest_skip.py @@ -0,0 +1,49 @@ +"""This is a sample module used for testing doctest. + +This module includes various scenarios involving skips. +""" + +def no_skip_pass(): + """ + >>> 2 + 2 + 4 + """ + +def no_skip_fail(): + """ + >>> 2 + 2 + 5 + """ + +def single_skip(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + """ + +def double_skip(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + >>> 3 + 3 # doctest: +SKIP + 6 + """ + +def partial_skip_pass(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + >>> 3 + 3 + 6 + """ + +def partial_skip_fail(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + >>> 2 + 2 + 5 + """ + +def no_examples(): + """A docstring with no examples should not be counted as run or skipped.""" diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index 43be200b983227..a4a49298bab3be 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -4,7 +4,6 @@ from test import support from test.support import import_helper -from test.support.pty_helper import FakeInput # used in doctests import doctest import functools import os @@ -18,8 +17,12 @@ import contextlib -if not support.has_subprocess_support: - raise unittest.SkipTest("test_CLI requires subprocess support.") +def doctest_skip_if(condition): + def decorator(func): + if condition and support.HAVE_DOCSTRINGS: + func.__doc__ = ">>> pass # doctest: +SKIP" + return func + return decorator # NOTE: There are some additional tests relating to interaction with @@ -466,7 +469,7 @@ def basics(): r""" >>> tests = finder.find(sample_func) >>> print(tests) # doctest: +ELLIPSIS - [] + [] The exact name depends on how test_doctest was invoked, so allow for leading path components. @@ -735,7 +738,7 @@ def non_Python_modules(): r""" >>> import builtins >>> tests = doctest.DocTestFinder().find(builtins) - >>> 830 < len(tests) < 860 # approximate number of objects with docstrings + >>> 750 < len(tests) < 800 # approximate number of objects with docstrings True >>> real_tests = [t for t in tests if len(t.examples) > 0] >>> len(real_tests) # objects that actually have doctests @@ -888,6 +891,10 @@ def basics(): r""" DocTestRunner is used to run DocTest test cases, and to accumulate statistics. Here's a simple DocTest case we can use: + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> def f(x): ... ''' ... >>> x = 12 @@ -942,6 +949,8 @@ def basics(): r""" 6 ok TestResults(failed=1, attempted=3) + + >>> _colorize.COLORIZE = save_colorize """ def verbose_flag(): r""" The `verbose` flag makes the test runner generate more detailed @@ -1017,12 +1026,16 @@ def exceptions(): r""" lines between the first line and the type/value may be omitted or replaced with any other string: + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> def f(x): ... ''' ... >>> x = 12 ... >>> print(x//0) ... Traceback (most recent call last): - ... ZeroDivisionError: integer division or modulo by zero + ... ZeroDivisionError: division by zero ... ''' >>> test = doctest.DocTestFinder().find(f)[0] >>> doctest.DocTestRunner(verbose=False).run(test) @@ -1039,7 +1052,7 @@ def exceptions(): r""" ... >>> print('pre-exception output', x//0) ... pre-exception output ... Traceback (most recent call last): - ... ZeroDivisionError: integer division or modulo by zero + ... ZeroDivisionError: division by zero ... ''' >>> test = doctest.DocTestFinder().find(f)[0] >>> doctest.DocTestRunner(verbose=False).run(test) @@ -1050,7 +1063,7 @@ def exceptions(): r""" print('pre-exception output', x//0) Exception raised: ... - ZeroDivisionError: integer division or modulo by zero + ZeroDivisionError: division by zero TestResults(failed=1, attempted=2) Exception messages may contain newlines: @@ -1245,8 +1258,10 @@ def exceptions(): r""" Exception raised: Traceback (most recent call last): ... - ZeroDivisionError: integer division or modulo by zero + ZeroDivisionError: division by zero TestResults(failed=1, attempted=1) + + >>> _colorize.COLORIZE = save_colorize """ def displayhook(): r""" Test that changing sys.displayhook doesn't matter for doctest. @@ -1288,6 +1303,10 @@ def optionflags(): r""" The DONT_ACCEPT_TRUE_FOR_1 flag disables matches between True/False and 1/0: + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> def f(x): ... '>>> True\n1\n' @@ -1707,6 +1726,7 @@ def optionflags(): r""" Clean up. >>> del doctest.OPTIONFLAGS_BY_NAME[unlikely] + >>> _colorize.COLORIZE = save_colorize """ @@ -1717,6 +1737,10 @@ def option_directives(): r""" single example. To turn an option on for an example, follow that example with a comment of the form ``# doctest: +OPTION``: + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> def f(x): r''' ... >>> print(list(range(10))) # should fail: no ellipsis ... [0, 1, ..., 9] @@ -1924,6 +1948,8 @@ def option_directives(): r""" >>> test = doctest.DocTestParser().get_doctest(s, {}, 's', 's.py', 0) Traceback (most recent call last): ValueError: line 0 of the doctest for s has an option directive on a line with no example: '# doctest: +ELLIPSIS' + + >>> _colorize.COLORIZE = save_colorize """ def test_testsource(): r""" @@ -1978,6 +2004,7 @@ def test_debug(): r""" Create some fake stdin input, to feed to the debugger: + >>> from test.support.pty_helper import FakeInput >>> real_stdin = sys.stdin >>> sys.stdin = FakeInput(['next', 'print(x)', 'continue']) @@ -2007,6 +2034,10 @@ def test_pdb_set_trace(): with a version that restores stdout. This is necessary for you to see debugger output. + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> doc = ''' ... >>> x = 42 ... >>> raise Exception('clé') @@ -2021,6 +2052,7 @@ def test_pdb_set_trace(): To demonstrate this, we'll create a fake standard input that captures our debugger input: + >>> from test.support.pty_helper import FakeInput >>> real_stdin = sys.stdin >>> sys.stdin = FakeInput([ ... 'print(x)', # print data defined by the example @@ -2029,8 +2061,7 @@ def test_pdb_set_trace(): >>> try: runner.run(test) ... finally: sys.stdin = real_stdin - --Return-- - > (1)()->None + > (1)() -> import pdb; pdb.set_trace() (Pdb) print(x) 42 @@ -2060,8 +2091,7 @@ def test_pdb_set_trace(): ... runner.run(test) ... finally: ... sys.stdin = real_stdin - --Return-- - > (3)calls_set_trace()->None + > (3)calls_set_trace() -> import pdb; pdb.set_trace() (Pdb) print(y) 2 @@ -2087,6 +2117,7 @@ def test_pdb_set_trace(): >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) >>> real_stdin = sys.stdin >>> sys.stdin = FakeInput([ + ... 'step', # return event of g ... 'list', # list source from example 2 ... 'next', # return from g() ... 'list', # list source from example 1 @@ -2097,6 +2128,9 @@ def test_pdb_set_trace(): >>> try: runner.run(test) ... finally: sys.stdin = real_stdin ... # doctest: +NORMALIZE_WHITESPACE + > (3)g() + -> import pdb; pdb.set_trace() + (Pdb) step --Return-- > (3)g()->None -> import pdb; pdb.set_trace() @@ -2129,6 +2163,8 @@ def test_pdb_set_trace(): Got: 9 TestResults(failed=1, attempted=3) + + >>> _colorize.COLORIZE = save_colorize """ def test_pdb_set_trace_nested(): @@ -2157,8 +2193,10 @@ def test_pdb_set_trace_nested(): >>> parser = doctest.DocTestParser() >>> runner = doctest.DocTestRunner(verbose=False) >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) + >>> from test.support.pty_helper import FakeInput >>> real_stdin = sys.stdin >>> sys.stdin = FakeInput([ + ... 'step', ... 'print(y)', # print data defined in the function ... 'step', 'step', 'step', 'step', 'step', 'step', 'print(z)', ... 'up', 'print(x)', @@ -2172,6 +2210,9 @@ def test_pdb_set_trace_nested(): ... finally: ... sys.stdin = real_stdin ... # doctest: +REPORT_NDIFF + > (4)calls_set_trace() + -> import pdb; pdb.set_trace() + (Pdb) step > (5)calls_set_trace() -> self.f1() (Pdb) print(y) @@ -2247,6 +2288,16 @@ def test_DocTestSuite(): >>> suite.run(unittest.TestResult()) + If all examples in a docstring are skipped, unittest will report it as a + skipped test: + + >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest_skip') + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> len(result.skipped) + 2 + We can use the current module: >>> suite = test.test_doctest.sample_doctest.test_suite() @@ -2359,25 +2410,6 @@ def test_DocFileSuite(): >>> suite.run(unittest.TestResult()) - Support for using a package's __loader__.get_data() is also - provided. - - >>> import unittest, pkgutil, test - >>> added_loader = False - >>> if not hasattr(test, '__loader__'): - ... test.__loader__ = pkgutil.get_loader(test) - ... added_loader = True - >>> try: - ... suite = doctest.DocFileSuite('test_doctest.txt', - ... 'test_doctest2.txt', - ... 'test_doctest4.txt', - ... package='test.test_doctest') - ... suite.run(unittest.TestResult()) - ... finally: - ... if added_loader: - ... del test.__loader__ - - '/' should be used as a path separator. It will be converted to a native separator at run time: @@ -2418,6 +2450,18 @@ def test_DocFileSuite(): Traceback (most recent call last): ValueError: Package may only be specified for module-relative paths. + If all examples in a file are skipped, unittest will report it as a + skipped test: + + >>> suite = doctest.DocFileSuite('test_doctest.txt', + ... 'test_doctest4.txt', + ... 'test_doctest_skip.txt') + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> len(result.skipped) + 1 + You can specify initial global variables: >>> suite = doctest.DocFileSuite('test_doctest.txt', @@ -2519,7 +2563,7 @@ def __call__(self, *args, **kwargs): self.func(*args, **kwargs) @Wrapper -def test_look_in_unwrapped(): +def wrapped(): """ Docstrings in wrapped functions must be detected as well. @@ -2527,6 +2571,35 @@ def test_look_in_unwrapped(): 'one other test' """ +def test_look_in_unwrapped(): + """ + Ensure that wrapped doctests work correctly. + + >>> import doctest + >>> doctest.run_docstring_examples( + ... wrapped, {}, name=wrapped.__name__, verbose=True) + Finding tests in wrapped + Trying: + 'one other test' + Expecting: + 'one other test' + ok + """ + +@doctest_skip_if(support.check_impl_detail(cpython=False)) +def test_wrapped_c_func(): + """ + # https://github.com/python/cpython/issues/117692 + >>> import binascii + >>> from test.test_doctest.decorator_mod import decorator + + >>> c_func_wrapped = decorator(binascii.b2a_hex) + >>> tests = doctest.DocTestFinder(exclude_empty=False).find(c_func_wrapped) + >>> for test in tests: + ... print(test.lineno, test.name) + None b2a_hex + """ + def test_unittest_reportflags(): """Default unittest reporting flags can be set to control reporting @@ -2612,7 +2685,11 @@ def test_testfile(): r""" called with the name of a file, which is taken to be relative to the calling module. The return value is (#failures, #tests). -We don't want `-v` in sys.argv for these tests. +We don't want color or `-v` in sys.argv for these tests. + + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> save_argv = sys.argv >>> if '-v' in sys.argv: @@ -2628,9 +2705,9 @@ def test_testfile(): r""" ... NameError: name 'favorite_color' is not defined ********************************************************************** - 1 items had failures: + 1 item had failures: 1 of 2 in test_doctest.txt - ***Test Failed*** 1 failures. + ***Test Failed*** 1 failure. TestResults(failed=1, attempted=2) >>> doctest.master = None # Reset master. @@ -2657,9 +2734,9 @@ def test_testfile(): r""" Got: 'red' ********************************************************************** - 1 items had failures: + 1 item had failures: 1 of 2 in test_doctest.txt - ***Test Failed*** 1 failures. + ***Test Failed*** 1 failure. TestResults(failed=1, attempted=2) >>> doctest.master = None # Reset master. @@ -2689,10 +2766,10 @@ def test_testfile(): r""" b ok - 1 items passed all tests: + 1 item passed all tests: 2 tests in test_doctest.txt - 2 tests in 1 items. - 2 passed and 0 failed. + 2 tests in 1 item. + 2 passed. Test passed. TestResults(failed=0, attempted=2) >>> doctest.master = None # Reset master. @@ -2749,7 +2826,7 @@ def test_testfile(): r""" ********************************************************************** ... ********************************************************************** - 1 items had failures: + 1 item had failures: 2 of 2 in test_doctest4.txt ***Test Failed*** 2 failures. TestResults(failed=2, attempted=2) @@ -2772,17 +2849,18 @@ def test_testfile(): r""" Expecting: 'b\u0105r' ok - 1 items passed all tests: + 1 item passed all tests: 2 tests in test_doctest4.txt - 2 tests in 1 items. - 2 passed and 0 failed. + 2 tests in 1 item. + 2 passed. Test passed. TestResults(failed=0, attempted=2) >>> doctest.master = None # Reset master. >>> sys.argv = save_argv + >>> _colorize.COLORIZE = save_colorize """ -class TestImporter(importlib.abc.MetaPathFinder, importlib.abc.ResourceLoader): +class TestImporter(importlib.abc.MetaPathFinder): def find_spec(self, fullname, path, target=None): return importlib.util.spec_from_file_location(fullname, path, loader=self) @@ -2791,6 +2869,12 @@ def get_data(self, path): with open(path, mode='rb') as f: return f.read() + def exec_module(self, module): + raise ImportError + + def create_module(self, spec): + return None + class TestHook: def __init__(self, pathdir): @@ -2917,6 +3001,10 @@ def test_testmod(): r""" def test_unicode(): """ Check doctest with a non-ascii filename: + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> doc = ''' ... >>> raise Exception('clé') ... ''' @@ -2942,8 +3030,12 @@ def test_unicode(): """ raise Exception('clé') Exception: clé TestResults(failed=1, attempted=1) + + >>> _colorize.COLORIZE = save_colorize """ + +@doctest_skip_if(not support.has_subprocess_support) def test_CLI(): r""" The doctest module can be used to run doctests against an arbitrary file. These tests test this CLI functionality. @@ -2997,10 +3089,10 @@ def test_CLI(): r""" Expecting: 'a' ok - 1 items passed all tests: + 1 item passed all tests: 2 tests in myfile.doc - 2 tests in 1 items. - 2 passed and 0 failed. + 2 tests in 1 item. + 2 passed. Test passed. Now we'll write a couple files, one with three tests, the other a python module @@ -3074,7 +3166,7 @@ def test_CLI(): r""" Got: 'ajkml' ********************************************************************** - 1 items had failures: + 1 item had failures: 2 of 3 in myfile.doc ***Test Failed*** 2 failures. @@ -3101,9 +3193,9 @@ def test_CLI(): r""" Got: 'abcdef' ********************************************************************** - 1 items had failures: + 1 item had failures: 1 of 2 in myfile.doc - ***Test Failed*** 1 failures. + ***Test Failed*** 1 failure. The fifth test uses verbose with the two options, so we should get verbose success output for the tests in both files: @@ -3126,10 +3218,10 @@ def test_CLI(): r""" Expecting: 'a...l' ok - 1 items passed all tests: + 1 item passed all tests: 3 tests in myfile.doc - 3 tests in 1 items. - 3 passed and 0 failed. + 3 tests in 1 item. + 3 passed. Test passed. Trying: 1 + 1 @@ -3141,12 +3233,12 @@ def test_CLI(): r""" Expecting: 'abc def' ok - 1 items had no tests: + 1 item had no tests: myfile2 - 1 items passed all tests: + 1 item passed all tests: 2 tests in myfile2.test_func 2 tests in 2 items. - 2 passed and 0 failed. + 2 passed. Test passed. We should also check some typical error cases. @@ -3234,6 +3326,10 @@ def test_run_doctestsuite_multiple_times(): def test_exception_with_note(note): """ + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> test_exception_with_note('Note') Traceback (most recent call last): ... @@ -3283,6 +3379,8 @@ def test_exception_with_note(note): ValueError: message note TestResults(failed=1, attempted=...) + + >>> _colorize.COLORIZE = save_colorize """ exc = ValueError('Text') exc.add_note(note) @@ -3363,6 +3461,10 @@ def test_syntax_error_subclass_from_stdlib(): def test_syntax_error_with_incorrect_expected_note(): """ + >>> import _colorize + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + >>> def f(x): ... r''' ... >>> exc = SyntaxError("error", ("x.py", 23, None, "bad syntax")) @@ -3391,6 +3493,8 @@ def test_syntax_error_with_incorrect_expected_note(): note1 note2 TestResults(failed=1, attempted=...) + + >>> _colorize.COLORIZE = save_colorize """ diff --git a/Lib/test/test_doctest/test_doctest_skip.txt b/Lib/test/test_doctest/test_doctest_skip.txt new file mode 100644 index 00000000000000..f340e2b8141253 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest_skip.txt @@ -0,0 +1,4 @@ +This is a sample doctest in a text file, in which all examples are skipped. + + >>> 2 + 2 # doctest: +SKIP + 5 diff --git a/Lib/test/test_dynamic.py b/Lib/test/test_dynamic.py index 3928bbab4423c2..21bece26b893c6 100644 --- a/Lib/test/test_dynamic.py +++ b/Lib/test/test_dynamic.py @@ -4,7 +4,7 @@ import sys import unittest -from test.support import is_wasi, Py_DEBUG, swap_item, swap_attr +from test.support import swap_item, swap_attr, is_wasi, Py_DEBUG class RebindBuiltinsTests(unittest.TestCase): @@ -134,7 +134,7 @@ def test_eval_gives_lambda_custom_globals(self): self.assertEqual(foo(), 7) - @unittest.skipIf(is_wasi and Py_DEBUG, "stack depth too shallow in pydebug WASI") + @unittest.skipIf(is_wasi and Py_DEBUG, "requires too much stack") def test_load_global_specialization_failure_keeps_oparg(self): # https://github.com/python/cpython/issues/91625 class MyGlobals(dict): diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index f7e80749c456f8..d60a7039f9d4c6 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -801,6 +801,10 @@ def test_get_quoted_string_header_ends_in_qcontent(self): self.assertEqual(qs.content, 'bob') self.assertEqual(qs.quoted_value, ' "bob"') + def test_get_quoted_string_cfws_only_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_quoted_string(' (foo) ') + def test_get_quoted_string_no_quoted_string(self): with self.assertRaises(errors.HeaderParseError): parser.get_quoted_string(' (ab) xyz') @@ -1135,6 +1139,10 @@ def test_get_local_part_complex_obsolete_invalid(self): '@python.org') self.assertEqual(local_part.local_part, 'Fred.A.Johnson and dogs') + def test_get_local_part_empty_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_local_part('') + def test_get_local_part_no_part_raises(self): with self.assertRaises(errors.HeaderParseError): parser.get_local_part(' (foo) ') @@ -1387,6 +1395,10 @@ def test_get_domain_obsolete(self): '') self.assertEqual(domain.domain, 'example.com') + def test_get_domain_empty_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_domain("") + def test_get_domain_no_non_cfws_raises(self): with self.assertRaises(errors.HeaderParseError): parser.get_domain(" (foo)\t") @@ -1512,6 +1524,10 @@ def test_get_obs_route_no_route_before_end_raises(self): with self.assertRaises(errors.HeaderParseError): parser.get_obs_route('(foo) @example.com,') + def test_get_obs_route_no_route_before_end_raises2(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_obs_route('(foo) @example.com, (foo) ') + def test_get_obs_route_no_route_before_special_raises(self): with self.assertRaises(errors.HeaderParseError): parser.get_obs_route('(foo) [abc],') @@ -1520,6 +1536,14 @@ def test_get_obs_route_no_route_before_special_raises2(self): with self.assertRaises(errors.HeaderParseError): parser.get_obs_route('(foo) @example.com [abc],') + def test_get_obs_route_no_domain_after_at_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_obs_route('@') + + def test_get_obs_route_no_domain_after_at_raises2(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_obs_route('@example.com, @') + # get_angle_addr def test_get_angle_addr_simple(self): @@ -1646,6 +1670,14 @@ def test_get_angle_addr_ends_at_special(self): self.assertIsNone(angle_addr.route) self.assertEqual(angle_addr.addr_spec, 'dinsdale@example.com') + def test_get_angle_addr_empty_raise(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_angle_addr('') + + def test_get_angle_addr_left_angle_only_raise(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_angle_addr('<') + def test_get_angle_addr_no_angle_raise(self): with self.assertRaises(errors.HeaderParseError): parser.get_angle_addr('(foo) ') @@ -1805,6 +1837,32 @@ def test_get_name_addr_qs_name(self): self.assertIsNone(name_addr.route) self.assertEqual(name_addr.addr_spec, 'dinsdale@example.com') + def test_get_name_addr_ending_with_dot_without_space(self): + name_addr = self._test_get_x(parser.get_name_addr, + 'John X.', + 'John X.', + '"John X."', + [errors.ObsoleteHeaderDefect], + '') + self.assertEqual(name_addr.display_name, 'John X.') + self.assertEqual(name_addr.local_part, 'jxd') + self.assertEqual(name_addr.domain, 'example.com') + self.assertIsNone(name_addr.route) + self.assertEqual(name_addr.addr_spec, 'jxd@example.com') + + def test_get_name_addr_starting_with_dot(self): + name_addr = self._test_get_x(parser.get_name_addr, + '. Doe ', + '. Doe ', + '". Doe" ', + [errors.InvalidHeaderDefect, errors.ObsoleteHeaderDefect], + '') + self.assertEqual(name_addr.display_name, '. Doe') + self.assertEqual(name_addr.local_part, 'jxd') + self.assertEqual(name_addr.domain, 'example.com') + self.assertIsNone(name_addr.route) + self.assertEqual(name_addr.addr_spec, 'jxd@example.com') + def test_get_name_addr_with_route(self): name_addr = self._test_get_x(parser.get_name_addr, '"Roy.A.Bear" <@two.example.com: dinsdale@example.com>', @@ -1831,6 +1889,10 @@ def test_get_name_addr_ends_at_special(self): self.assertIsNone(name_addr.route) self.assertEqual(name_addr.addr_spec, 'dinsdale@example.com') + def test_get_name_addr_empty_raises(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_name_addr('') + def test_get_name_addr_no_content_raises(self): with self.assertRaises(errors.HeaderParseError): parser.get_name_addr(' (foo) ') @@ -2698,6 +2760,35 @@ def test_get_msg_id_no_angle_end(self): ) self.assertEqual(msg_id.token_type, 'msg-id') + def test_get_msg_id_empty_id_left(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_msg_id("<@domain>") + + def test_get_msg_id_empty_id_right(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_msg_id("") + + def test_get_msg_id_no_id_right(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_msg_id("") + + def test_get_msg_id_ws_only_local(self): + msg_id = self._test_get_x( + parser.get_msg_id, + "< @domain>", + "< @domain>", + "< @domain>", + [errors.ObsoleteHeaderDefect], + "" + ) + self.assertEqual(msg_id.token_type, 'msg-id') + @parameterize @@ -2986,9 +3077,44 @@ def test_address_list_with_unicode_names_in_quotes(self): ' =?utf-8?q?bei=C3=9Ft_bei=C3=9Ft?= \n') def test_address_list_with_list_separator_after_fold(self): - to = '0123456789' * 8 + '@foo, ä ' + a = 'x' * 66 + '@example.com' + to = f'{a}, "Hübsch Kaktus" ' + self._test(parser.get_address_list(to)[0], + f'{a},\n =?utf-8?q?H=C3=BCbsch?= Kaktus \n') + + a = '.' * 79 # ('.' is a special, so must be in quoted-string.) + to = f'"{a}" , "Hübsch Kaktus" ' self._test(parser.get_address_list(to)[0], - '0123456789' * 8 + '@foo,\n =?utf-8?q?=C3=A4?= \n') + f'"{a}"\n' + ' , =?utf-8?q?H=C3=BCbsch?= Kaktus ' + '\n') + + def test_address_list_with_specials_in_long_quoted_string(self): + # Regression for gh-80222. + policy = self.policy.clone(max_line_length=40) + cases = [ + # (to, folded) + ('"Exfiltrator (unclosed comment?" ', + '"Exfiltrator (unclosed\n' + ' comment?" \n'), + ('"Escaped \\" chars \\\\ in quoted-string stay escaped" ', + '"Escaped \\" chars \\\\ in quoted-string\n' + ' stay escaped" \n'), + ('This long display name does not need quotes ', + 'This long display name does not need\n' + ' quotes \n'), + ('"Quotes are not required but are retained here" ', + '"Quotes are not required but are\n' + ' retained here" \n'), + ('"A quoted-string, it can be a valid local-part"@example.com', + '"A quoted-string, it can be a valid\n' + ' local-part"@example.com\n'), + ('"local-part-with-specials@but-no-fws.cannot-fold"@example.com', + '"local-part-with-specials@but-no-fws.cannot-fold"@example.com\n'), + ] + for (to, folded) in cases: + with self.subTest(to=to): + self._test(parser.get_address_list(to)[0], folded, policy=policy) # XXX Need tests with comments on various sides of a unicode token, # and with unicode tokens in the comments. Spaces inside the quotes diff --git a/Lib/test/test_email/test_asian_codecs.py b/Lib/test/test_email/test_asian_codecs.py index 1e0caeeaed0810..ca44f54c69b39b 100644 --- a/Lib/test/test_email/test_asian_codecs.py +++ b/Lib/test/test_email/test_asian_codecs.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2006 Python Software Foundation +# Copyright (C) 2002 Python Software Foundation # Contact: email-sig@python.org # email package unit tests for (optional) Asian codecs diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 39d4ace8d4a1d8..2deb35721576b8 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2010 Python Software Foundation +# Copyright (C) 2001 Python Software Foundation # Contact: email-sig@python.org # email package unit tests @@ -337,6 +337,21 @@ def test_nonascii_as_string_without_cte(self): msg = email.message_from_bytes(source) self.assertEqual(msg.as_string(), expected) + def test_nonascii_as_string_with_ascii_charset(self): + m = textwrap.dedent("""\ + MIME-Version: 1.0 + Content-type: text/plain; charset="us-ascii" + Content-Transfer-Encoding: 8bit + + Test if non-ascii messages with no Content-Transfer-Encoding set + can be as_string'd: + Föö bär + """) + source = m.encode('iso-8859-1') + expected = source.decode('ascii', 'replace') + msg = email.message_from_bytes(source) + self.assertEqual(msg.as_string(), expected) + def test_nonascii_as_string_without_content_type_and_cte(self): m = textwrap.dedent("""\ MIME-Version: 1.0 @@ -795,6 +810,16 @@ def test_unicode_body_defaults_to_utf8_encoding(self): w4kgdGVzdGFiYwo= """)) + def test_string_payload_with_base64_cte(self): + msg = email.message_from_string(textwrap.dedent("""\ + Content-Transfer-Encoding: base64 + + SGVsbG8uIFRlc3Rpbmc= + """), policy=email.policy.default) + self.assertEqual(msg.get_payload(decode=True), b"Hello. Testing") + self.assertDefectsEqual(msg['content-transfer-encoding'].defects, []) + + # Test the email.encoders module class TestEncoders(unittest.TestCase): @@ -2337,6 +2362,40 @@ def test_missing_header_body_separator(self): self.assertDefectsEqual(msg.defects, [errors.MissingHeaderBodySeparatorDefect]) + def test_string_payload_with_extra_space_after_cte(self): + # https://github.com/python/cpython/issues/98188 + cte = "base64 " + msg = email.message_from_string(textwrap.dedent(f"""\ + Content-Transfer-Encoding: {cte} + + SGVsbG8uIFRlc3Rpbmc= + """), policy=email.policy.default) + self.assertEqual(msg.get_payload(decode=True), b"Hello. Testing") + self.assertDefectsEqual(msg['content-transfer-encoding'].defects, []) + + def test_string_payload_with_extra_text_after_cte(self): + msg = email.message_from_string(textwrap.dedent("""\ + Content-Transfer-Encoding: base64 some text + + SGVsbG8uIFRlc3Rpbmc= + """), policy=email.policy.default) + self.assertEqual(msg.get_payload(decode=True), b"Hello. Testing") + cte = msg['content-transfer-encoding'] + self.assertDefectsEqual(cte.defects, [email.errors.InvalidHeaderDefect]) + + def test_string_payload_with_extra_space_after_cte_compat32(self): + cte = "base64 " + msg = email.message_from_string(textwrap.dedent(f"""\ + Content-Transfer-Encoding: {cte} + + SGVsbG8uIFRlc3Rpbmc= + """), policy=email.policy.compat32) + pasted_cte = msg['content-transfer-encoding'] + self.assertEqual(pasted_cte, cte) + self.assertEqual(msg.get_payload(decode=True), b"Hello. Testing") + self.assertDefectsEqual(msg.defects, []) + + # Test RFC 2047 header encoding and decoding class TestRFC2047(TestEmailBase): @@ -4166,6 +4225,21 @@ def test_8bit_in_uuencode_body(self): self.assertEqual(msg.get_payload(decode=True), '<,.V', + [errors.ObsoleteHeaderDefect], + '"John X." ', + 'John X.', + 'jxd@example.com', + 'jxd', + 'example.com', + None), + + 'name_starting_with_dot': + ('. Doe ', + [errors.InvalidHeaderDefect, errors.ObsoleteHeaderDefect], + '". Doe" ', + '. Doe', + 'jxd@example.com', + 'jxd', + 'example.com', + None), + } # XXX: Need many more examples, and in particular some with names in @@ -1628,7 +1653,7 @@ def test_address_display_names(self): 'Lôrem ipsum dôlôr sit amet, cônsectetuer adipiscing. ' 'Suspendisse pôtenti. Aliquam nibh. Suspendisse pôtenti.', '=?utf-8?q?L=C3=B4rem_ipsum_d=C3=B4l=C3=B4r_sit_amet=2C_c' - '=C3=B4nsectetuer?=\n =?utf-8?q?adipiscing=2E_Suspendisse' + '=C3=B4nsectetuer?=\n =?utf-8?q?_adipiscing=2E_Suspendisse' '_p=C3=B4tenti=2E_Aliquam_nibh=2E?=\n Suspendisse =?utf-8' '?q?p=C3=B4tenti=2E?=', ), diff --git a/Lib/test/test_email/test_message.py b/Lib/test/test_email/test_message.py index 034f7626c1fc7c..96979db27f3a21 100644 --- a/Lib/test/test_email/test_message.py +++ b/Lib/test/test_email/test_message.py @@ -1,6 +1,6 @@ -import unittest import textwrap -from email import policy, message_from_string +import unittest +from email import message_from_bytes, message_from_string, policy from email.message import EmailMessage, MIMEPart from test.test_email import TestEmailBase, parameterize @@ -958,6 +958,52 @@ def test_folding_with_utf8_encoding_8(self): b'123456789-123456789\n 123456789 Hello ' b'=?utf-8?q?W=C3=B6rld!?= 123456789 123456789\n\n') + def test_folding_with_short_nospace_1(self): + # bpo-36520 + # + # Fold a line that contains a long whitespace after + # the fold point. + + m = EmailMessage(policy.default) + m['Message-ID'] = '123456789' * 3 + parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) + self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + + def test_folding_with_long_nospace_default_policy_1(self): + # Fixed: https://github.com/python/cpython/issues/124452 + # + # When the value is too long, it should be converted back + # to its original form without any modifications. + + m = EmailMessage(policy.default) + message = '123456789' * 10 + m['Message-ID'] = message + self.assertEqual(m.as_bytes(), + f'Message-ID:\n {message}\n\n'.encode()) + parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) + self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + + def test_folding_with_long_nospace_compat32_policy_1(self): + m = EmailMessage(policy.compat32) + message = '123456789' * 10 + m['Message-ID'] = message + parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) + self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + + def test_folding_with_long_nospace_smtp_policy_1(self): + m = EmailMessage(policy.SMTP) + message = '123456789' * 10 + m['Message-ID'] = message + parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) + self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + + def test_folding_with_long_nospace_http_policy_1(self): + m = EmailMessage(policy.HTTP) + message = '123456789' * 10 + m['Message-ID'] = message + parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) + self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + def test_get_body_malformed(self): """test for bpo-42892""" msg = textwrap.dedent("""\ diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py index c6b9c80efe1b54..baa35fd68e49c5 100644 --- a/Lib/test/test_email/test_policy.py +++ b/Lib/test/test_email/test_policy.py @@ -26,6 +26,7 @@ class PolicyAPITests(unittest.TestCase): 'raise_on_defect': False, 'mangle_from_': True, 'message_factory': None, + 'verify_generated_headers': True, } # These default values are the ones set on email.policy.default. # If any of these defaults change, the docs must be updated. @@ -294,6 +295,31 @@ def test_short_maxlen_error(self): with self.assertRaises(email.errors.HeaderParseError): policy.fold("Subject", subject) + def test_verify_generated_headers(self): + """Turning protection off allows header injection""" + policy = email.policy.default.clone(verify_generated_headers=False) + for text in ( + 'Header: Value\r\nBad: Injection\r\n', + 'Header: NoNewLine' + ): + with self.subTest(text=text): + message = email.message_from_string( + "Header: Value\r\n\r\nBody", + policy=policy, + ) + class LiteralHeader(str): + name = 'Header' + def fold(self, **kwargs): + return self + + del message['Header'] + message['Header'] = LiteralHeader(text) + + self.assertEqual( + message.as_string(), + f"{text}\nBody", + ) + # XXX: Need subclassing tests. # For adding subclassed objects, make sure the usual rules apply (subclass # wins), but that the order still works (right overrides left). diff --git a/Lib/test/test_email/test_utils.py b/Lib/test/test_email/test_utils.py index d04b3909efa643..4e6201e13c87f9 100644 --- a/Lib/test/test_email/test_utils.py +++ b/Lib/test/test_email/test_utils.py @@ -3,9 +3,7 @@ import test.support import time import unittest -import sys -import os.path -import zoneinfo + class DateTimeTests(unittest.TestCase): @@ -154,10 +152,6 @@ def test_variable_tzname(self): t1 = utils.localtime(t0) self.assertEqual(t1.tzname(), 'EET') - def test_isdst_deprecation(self): - with self.assertWarns(DeprecationWarning): - t0 = datetime.datetime(1990, 1, 1) - t1 = utils.localtime(t0, isdst=True) # Issue #24836: The timezone files are out of date (pre 2011k) # on Mac OS X Snow Leopard. diff --git a/Lib/test/test_email/torture_test.py b/Lib/test/test_email/torture_test.py index 9cf9362c9b77e0..d15948a38b25dd 100644 --- a/Lib/test/test_email/torture_test.py +++ b/Lib/test/test_email/torture_test.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2004 Python Software Foundation +# Copyright (C) 2002 Python Software Foundation # # A torture test of the email package. This should not be run as part of the # standard Python test suite since it requires several meg of email messages diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 6c60854bbd76cc..f41c45ec4d9cdd 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1,10 +1,12 @@ # Run the tests in Programs/_testembed.c (tests for the CPython embedding APIs) from test import support -from test.support import import_helper, os_helper, MS_WINDOWS +from test.libregrtest.utils import get_build_info +from test.support import import_helper, os_helper, threading_helper, MS_WINDOWS import unittest from collections import namedtuple import contextlib +import io import json import os import os.path @@ -19,6 +21,13 @@ if not support.has_subprocess_support: raise unittest.SkipTest("test module requires subprocess") + +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None + + MACOS = (sys.platform == 'darwin') PYMEM_ALLOCATOR_NOT_SET = 0 PYMEM_ALLOCATOR_DEBUG = 2 @@ -41,6 +50,16 @@ INIT_LOOPS = 4 MAX_HASH_SEED = 4294967295 +ABI_THREAD = 't' if sysconfig.get_config_var('Py_GIL_DISABLED') else '' +# PLATSTDLIB_LANDMARK copied from Modules/getpath.py +if os.name == 'nt': + PLATSTDLIB_LANDMARK = f'{sys.platlibdir}' +else: + VERSION_MAJOR = sys.version_info.major + VERSION_MINOR = sys.version_info.minor + PLATSTDLIB_LANDMARK = (f'{sys.platlibdir}/python{VERSION_MAJOR}.' + f'{VERSION_MINOR}{ABI_THREAD}/lib-dynload') + # If we are running from a build dir, but the stdlib has been installed, # some tests need to expect different results. @@ -158,7 +177,8 @@ def run_repeated_init_and_subinterpreters(self): # Parse the line from the loop. The first line is the main # interpreter and the 3 afterward are subinterpreters. interp = Interp(*match.groups()) - if support.verbose > 1: + if support.verbose > 2: + # 5 lines per pass is super-spammy, so limit that to -vvv print(interp) self.assertTrue(interp.interp) self.assertTrue(interp.tstate) @@ -269,6 +289,10 @@ def test_pre_initialization_api(self): """ env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path)) out, err = self.run_embedded_interpreter("test_pre_initialization_api", env=env) + if support.verbose > 1: + print() + print(out) + print(err) if MS_WINDOWS: expected_path = self.test_exe else: @@ -286,6 +310,10 @@ def test_pre_initialization_sys_options(self): env['PYTHONPATH'] = os.pathsep.join(sys.path) out, err = self.run_embedded_interpreter( "test_pre_initialization_sys_options", env=env) + if support.verbose > 1: + print() + print(out) + print(err) expected_output = ( "sys.warnoptions: ['once', 'module', 'default']\n" "sys._xoptions: {'not_an_option': '1', 'also_not_an_option': '2'}\n" @@ -352,10 +380,13 @@ def test_simple_initialization_api(self): self.assertEqual(out, 'Finalized\n' * INIT_LOOPS) @support.requires_specialization + @unittest.skipUnless(support.TEST_MODULES_ENABLED, "requires test modules") def test_specialized_static_code_gets_unspecialized_at_Py_FINALIZE(self): # https://github.com/python/cpython/issues/92031 - code = textwrap.dedent("""\ + _testinternalcapi = import_helper.import_module("_testinternalcapi") + + code = textwrap.dedent(f"""\ import dis import importlib._bootstrap import opcode @@ -368,6 +399,9 @@ def is_specialized(f): opname in opcode._specialized_opmap # Exclude superinstructions: and "__" not in opname + # LOAD_CONST_IMMORTAL is "specialized", but is + # inserted during quickening. + and opname != "LOAD_CONST_IMMORTAL" ): return True return False @@ -379,7 +413,7 @@ def is_specialized(f): assert not is_specialized(func), "specialized instructions found" - for i in range(test.test_dis.ADAPTIVE_WARMUP_DELAY): + for _ in range({_testinternalcapi.SPECIALIZATION_THRESHOLD}): func(importlib._bootstrap, ["x"], lambda *args: None) assert is_specialized(func), "no specialized instructions found" @@ -396,6 +430,101 @@ def test_ucnhash_capi_reset(self): out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) self.assertEqual(out, '9\n' * INIT_LOOPS) + def test_datetime_reset_strptime(self): + code = ( + "import datetime;" + "d = datetime.datetime.strptime('2000-01-01', '%Y-%m-%d');" + "print(d.strftime('%Y%m%d'))" + ) + out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) + self.assertEqual(out, '20000101\n' * INIT_LOOPS) + + def test_static_types_inherited_slots(self): + script = textwrap.dedent(""" + import test.support + results = [] + for cls in test.support.iter_builtin_types(): + for attr, _ in test.support.iter_slot_wrappers(cls): + wrapper = getattr(cls, attr) + res = (cls, attr, wrapper) + results.append(res) + results = ((repr(c), a, repr(w)) for c, a, w in results) + """) + def collate_results(raw): + results = {} + for cls, attr, wrapper in raw: + key = cls, attr + assert key not in results, (results, key, wrapper) + results[key] = wrapper + return results + + ns = {} + exec(script, ns, ns) + main_results = collate_results(ns['results']) + del ns + + script += textwrap.dedent(""" + import json + import sys + text = json.dumps(list(results)) + print(text, file=sys.stderr) + """) + out, err = self.run_embedded_interpreter( + "test_repeated_init_exec", script, script) + _results = err.split('--- Loop #')[1:] + (_embedded, _reinit, + ) = [json.loads(res.rpartition(' ---\n')[-1]) for res in _results] + embedded_results = collate_results(_embedded) + reinit_results = collate_results(_reinit) + + for key, expected in main_results.items(): + cls, attr = key + for src, results in [ + ('embedded', embedded_results), + ('reinit', reinit_results), + ]: + with self.subTest(src, cls=cls, slotattr=attr): + actual = results.pop(key) + self.assertEqual(actual, expected) + self.maxDiff = None + self.assertEqual(embedded_results, {}) + self.assertEqual(reinit_results, {}) + + self.assertEqual(out, '') + + def test_getargs_reset_static_parser(self): + # Test _PyArg_Parser initializations via _PyArg_UnpackKeywords() + # https://github.com/python/cpython/issues/122334 + code = textwrap.dedent(""" + try: + import _ssl + except ModuleNotFoundError: + _ssl = None + if _ssl is not None: + _ssl.txt2obj(txt='1.3') + print('1') + + import _queue + _queue.SimpleQueue().put_nowait(item=None) + print('2') + + import _zoneinfo + _zoneinfo.ZoneInfo.clear_cache(only_keys=['Foo/Bar']) + print('3') + """) + out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) + self.assertEqual(out, '1\n2\n3\n' * INIT_LOOPS) + + +def config_dev_mode(preconfig, config): + preconfig['allocator'] = PYMEM_ALLOCATOR_DEBUG + preconfig['dev_mode'] = 1 + config['dev_mode'] = 1 + config['warnoptions'] = ['default'] + config['faulthandler'] = 1 + + +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): maxDiff = 4096 UTF8_MODE_ERRORS = ('surrogatepass' if MS_WINDOWS else 'surrogateescape') @@ -409,30 +538,30 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): PRE_CONFIG_COMPAT = { '_config_init': API_COMPAT, 'allocator': PYMEM_ALLOCATOR_NOT_SET, - 'parse_argv': 0, - 'configure_locale': 1, - 'coerce_c_locale': 0, - 'coerce_c_locale_warn': 0, - 'utf8_mode': 0, + 'parse_argv': False, + 'configure_locale': True, + 'coerce_c_locale': False, + 'coerce_c_locale_warn': False, + 'utf8_mode': False, } if MS_WINDOWS: PRE_CONFIG_COMPAT.update({ - 'legacy_windows_fs_encoding': 0, + 'legacy_windows_fs_encoding': False, }) PRE_CONFIG_PYTHON = dict(PRE_CONFIG_COMPAT, _config_init=API_PYTHON, - parse_argv=1, + parse_argv=True, coerce_c_locale=GET_DEFAULT_CONFIG, utf8_mode=GET_DEFAULT_CONFIG, ) PRE_CONFIG_ISOLATED = dict(PRE_CONFIG_COMPAT, _config_init=API_ISOLATED, - configure_locale=0, - isolated=1, - use_environment=0, - utf8_mode=0, - dev_mode=0, - coerce_c_locale=0, + configure_locale=False, + isolated=True, + use_environment=False, + utf8_mode=False, + dev_mode=False, + coerce_c_locale=False, ) COPY_PRE_CONFIG = [ @@ -443,35 +572,35 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): CONFIG_COMPAT = { '_config_init': API_COMPAT, - 'isolated': 0, - 'use_environment': 1, - 'dev_mode': 0, + 'isolated': False, + 'use_environment': True, + 'dev_mode': False, - 'install_signal_handlers': 1, - 'use_hash_seed': 0, + 'install_signal_handlers': True, + 'use_hash_seed': False, 'hash_seed': 0, 'int_max_str_digits': sys.int_info.default_max_str_digits, 'cpu_count': -1, - 'faulthandler': 0, + 'faulthandler': False, 'tracemalloc': 0, 'perf_profiling': 0, - 'import_time': 0, - 'code_debug_ranges': 1, - 'show_ref_count': 0, - 'dump_refs': 0, + 'import_time': False, + 'code_debug_ranges': True, + 'show_ref_count': False, + 'dump_refs': False, 'dump_refs_file': None, - 'malloc_stats': 0, + 'malloc_stats': False, 'filesystem_encoding': GET_DEFAULT_CONFIG, 'filesystem_errors': GET_DEFAULT_CONFIG, 'pycache_prefix': None, 'program_name': GET_DEFAULT_CONFIG, - 'parse_argv': 0, + 'parse_argv': False, 'argv': [""], 'orig_argv': [], - 'xoptions': [], + 'xoptions': {}, 'warnoptions': [], 'pythonpath_env': None, @@ -484,71 +613,76 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'exec_prefix': GET_DEFAULT_CONFIG, 'base_exec_prefix': GET_DEFAULT_CONFIG, 'module_search_paths': GET_DEFAULT_CONFIG, - 'module_search_paths_set': 1, + 'module_search_paths_set': True, 'platlibdir': sys.platlibdir, 'stdlib_dir': GET_DEFAULT_CONFIG, - 'site_import': 1, + 'site_import': True, 'bytes_warning': 0, - 'warn_default_encoding': 0, - 'inspect': 0, - 'interactive': 0, + 'warn_default_encoding': False, + 'inspect': False, + 'interactive': False, 'optimization_level': 0, - 'parser_debug': 0, - 'write_bytecode': 1, + 'parser_debug': False, + 'write_bytecode': True, 'verbose': 0, - 'quiet': 0, - 'user_site_directory': 1, - 'configure_c_stdio': 0, - 'buffered_stdio': 1, + 'quiet': False, + 'user_site_directory': True, + 'configure_c_stdio': False, + 'buffered_stdio': True, 'stdio_encoding': GET_DEFAULT_CONFIG, 'stdio_errors': GET_DEFAULT_CONFIG, - 'skip_source_first_line': 0, + 'skip_source_first_line': False, 'run_command': None, 'run_module': None, 'run_filename': None, 'sys_path_0': None, - '_install_importlib': 1, + '_install_importlib': True, 'check_hash_pycs_mode': 'default', - 'pathconfig_warnings': 1, - '_init_main': 1, + 'pathconfig_warnings': True, + '_init_main': True, 'use_frozen_modules': not support.Py_DEBUG, - 'safe_path': 0, + 'safe_path': False, '_is_python_build': IGNORE_CONFIG, } if Py_STATS: - CONFIG_COMPAT['_pystats'] = 0 + CONFIG_COMPAT['_pystats'] = False if support.Py_DEBUG: CONFIG_COMPAT['run_presite'] = None + if support.Py_GIL_DISABLED: + CONFIG_COMPAT['enable_gil'] = -1 + CONFIG_COMPAT['tlbc_enabled'] = GET_DEFAULT_CONFIG if MS_WINDOWS: CONFIG_COMPAT.update({ - 'legacy_windows_stdio': 0, + 'legacy_windows_stdio': False, }) + if support.is_apple: + CONFIG_COMPAT['use_system_logger'] = False CONFIG_PYTHON = dict(CONFIG_COMPAT, _config_init=API_PYTHON, - configure_c_stdio=1, - parse_argv=2, + configure_c_stdio=True, + parse_argv=True, ) CONFIG_ISOLATED = dict(CONFIG_COMPAT, _config_init=API_ISOLATED, - isolated=1, - use_environment=0, - user_site_directory=0, - safe_path=1, - dev_mode=0, - install_signal_handlers=0, - use_hash_seed=0, - faulthandler=0, - tracemalloc=0, + isolated=True, + use_environment=False, + user_site_directory=False, + safe_path=True, + dev_mode=False, + install_signal_handlers=False, + use_hash_seed=False, + faulthandler=False, + tracemalloc=False, perf_profiling=0, - pathconfig_warnings=0, + pathconfig_warnings=False, ) if MS_WINDOWS: - CONFIG_ISOLATED['legacy_windows_stdio'] = 0 + CONFIG_ISOLATED['legacy_windows_stdio'] = False # global config DEFAULT_GLOBAL_CONFIG = { @@ -735,6 +869,9 @@ def check_config(self, configs, expected): if value is self.IGNORE_CONFIG: config.pop(key, None) del expected[key] + # Resolve bool/int mismatches to reduce noise in diffs + if isinstance(value, (bool, int)) and isinstance(config.get(key), (bool, int)): + expected[key] = type(config[key])(expected[key]) self.assertEqual(config, expected) def check_global_config(self, configs): @@ -813,6 +950,7 @@ def check_all_configs(self, testname, expected_config=None, self.check_global_config(configs) return configs + @unittest.skipIf(support.check_bolt_optimized, "segfaults on BOLT instrumented binaries") def test_init_default_config(self): self.check_all_configs("test_init_initialize_config", api=API_COMPAT) @@ -824,23 +962,23 @@ def test_init_compat_config(self): def test_init_global_config(self): preconfig = { - 'utf8_mode': 1, + 'utf8_mode': True, } config = { 'program_name': './globalvar', - 'site_import': 0, - 'bytes_warning': 1, + 'site_import': False, + 'bytes_warning': True, 'warnoptions': ['default::BytesWarning'], - 'inspect': 1, - 'interactive': 1, + 'inspect': True, + 'interactive': True, 'optimization_level': 2, - 'write_bytecode': 0, - 'verbose': 1, - 'quiet': 1, - 'buffered_stdio': 0, + 'write_bytecode': False, + 'verbose': True, + 'quiet': True, + 'buffered_stdio': False, - 'user_site_directory': 0, - 'pathconfig_warnings': 0, + 'user_site_directory': False, + 'pathconfig_warnings': False, } self.check_all_configs("test_init_global_config", config, preconfig, api=API_COMPAT) @@ -848,18 +986,18 @@ def test_init_global_config(self): def test_init_from_config(self): preconfig = { 'allocator': ALLOCATOR_FOR_CONFIG, - 'utf8_mode': 1, + 'utf8_mode': True, } config = { - 'install_signal_handlers': 0, - 'use_hash_seed': 1, + 'install_signal_handlers': False, + 'use_hash_seed': True, 'hash_seed': 123, 'tracemalloc': 2, 'perf_profiling': 0, - 'import_time': 1, - 'code_debug_ranges': 0, - 'show_ref_count': 1, - 'malloc_stats': 1, + 'import_time': True, + 'code_debug_ranges': False, + 'show_ref_count': True, + 'malloc_stats': True, 'stdio_encoding': 'iso8859-1', 'stdio_errors': 'replace', @@ -872,13 +1010,13 @@ def test_init_from_config(self): '-X', 'cmdline_xoption', '-c', 'pass', 'arg2'], - 'parse_argv': 2, - 'xoptions': [ - 'config_xoption1=3', - 'config_xoption2=', - 'config_xoption3', - 'cmdline_xoption', - ], + 'parse_argv': True, + 'xoptions': { + 'config_xoption1': '3', + 'config_xoption2': '', + 'config_xoption3': True, + 'cmdline_xoption': True, + }, 'warnoptions': [ 'cmdline_warnoption', 'default::BytesWarning', @@ -886,98 +1024,100 @@ def test_init_from_config(self): ], 'run_command': 'pass\n', - 'site_import': 0, + 'site_import': False, 'bytes_warning': 1, - 'inspect': 1, - 'interactive': 1, + 'inspect': True, + 'interactive': True, 'optimization_level': 2, - 'write_bytecode': 0, + 'write_bytecode': False, 'verbose': 1, - 'quiet': 1, - 'configure_c_stdio': 1, - 'buffered_stdio': 0, - 'user_site_directory': 0, - 'faulthandler': 1, + 'quiet': True, + 'configure_c_stdio': True, + 'buffered_stdio': False, + 'user_site_directory': False, + 'faulthandler': True, 'platlibdir': 'my_platlibdir', 'module_search_paths': self.IGNORE_CONFIG, - 'safe_path': 1, + 'safe_path': True, 'int_max_str_digits': 31337, 'cpu_count': 4321, 'check_hash_pycs_mode': 'always', - 'pathconfig_warnings': 0, + 'pathconfig_warnings': False, } if Py_STATS: config['_pystats'] = 1 self.check_all_configs("test_init_from_config", config, preconfig, api=API_COMPAT) + @unittest.skipIf(support.check_bolt_optimized, "segfaults on BOLT instrumented binaries") def test_init_compat_env(self): preconfig = { 'allocator': ALLOCATOR_FOR_CONFIG, } config = { - 'use_hash_seed': 1, + 'use_hash_seed': True, 'hash_seed': 42, 'tracemalloc': 2, - 'perf_profiling': 0, - 'import_time': 1, - 'code_debug_ranges': 0, - 'malloc_stats': 1, - 'inspect': 1, + 'import_time': True, + 'code_debug_ranges': False, + 'malloc_stats': True, + 'inspect': True, 'optimization_level': 2, 'pythonpath_env': '/my/path', 'pycache_prefix': 'env_pycache_prefix', - 'write_bytecode': 0, + 'write_bytecode': False, 'verbose': 1, - 'buffered_stdio': 0, + 'buffered_stdio': False, 'stdio_encoding': 'iso8859-1', 'stdio_errors': 'replace', - 'user_site_directory': 0, - 'faulthandler': 1, + 'user_site_directory': False, + 'faulthandler': True, 'warnoptions': ['EnvVar'], 'platlibdir': 'env_platlibdir', 'module_search_paths': self.IGNORE_CONFIG, - 'safe_path': 1, + 'safe_path': True, 'int_max_str_digits': 4567, + 'perf_profiling': 1, } if Py_STATS: config['_pystats'] = 1 self.check_all_configs("test_init_compat_env", config, preconfig, api=API_COMPAT) + @unittest.skipIf(support.check_bolt_optimized, "segfaults on BOLT instrumented binaries") def test_init_python_env(self): preconfig = { 'allocator': ALLOCATOR_FOR_CONFIG, 'utf8_mode': 1, } config = { - 'use_hash_seed': 1, + 'use_hash_seed': True, 'hash_seed': 42, 'tracemalloc': 2, - 'perf_profiling': 0, - 'import_time': 1, - 'code_debug_ranges': 0, - 'malloc_stats': 1, - 'inspect': 1, + 'import_time': True, + 'code_debug_ranges': False, + 'malloc_stats': True, + 'inspect': True, 'optimization_level': 2, 'pythonpath_env': '/my/path', 'pycache_prefix': 'env_pycache_prefix', - 'write_bytecode': 0, + 'write_bytecode': False, 'verbose': 1, - 'buffered_stdio': 0, + 'buffered_stdio': False, 'stdio_encoding': 'iso8859-1', 'stdio_errors': 'replace', - 'user_site_directory': 0, - 'faulthandler': 1, + 'user_site_directory': False, + 'faulthandler': True, 'warnoptions': ['EnvVar'], 'platlibdir': 'env_platlibdir', 'module_search_paths': self.IGNORE_CONFIG, - 'safe_path': 1, + 'safe_path': True, 'int_max_str_digits': 4567, + 'perf_profiling': 1, } if Py_STATS: - config['_pystats'] = 1 + config['_pystats'] = True self.check_all_configs("test_init_python_env", config, preconfig, api=API_PYTHON) @@ -998,40 +1138,38 @@ def test_init_env_dev_mode_alloc(self): api=API_COMPAT) def test_init_dev_mode(self): - preconfig = { - 'allocator': PYMEM_ALLOCATOR_DEBUG, - } + preconfig = {} config = { - 'faulthandler': 1, - 'dev_mode': 1, + 'faulthandler': True, + 'dev_mode': True, 'warnoptions': ['default'], } + config_dev_mode(preconfig, config) self.check_all_configs("test_init_dev_mode", config, preconfig, api=API_PYTHON) def test_preinit_parse_argv(self): # Pre-initialize implicitly using argv: make sure that -X dev # is used to configure the allocation in preinitialization - preconfig = { - 'allocator': PYMEM_ALLOCATOR_DEBUG, - } + preconfig = {} config = { 'argv': ['script.py'], 'orig_argv': ['python3', '-X', 'dev', '-P', 'script.py'], 'run_filename': os.path.abspath('script.py'), - 'dev_mode': 1, - 'faulthandler': 1, + 'dev_mode': True, + 'faulthandler': True, 'warnoptions': ['default'], - 'xoptions': ['dev'], - 'safe_path': 1, + 'xoptions': {'dev': True}, + 'safe_path': True, } + config_dev_mode(preconfig, config) self.check_all_configs("test_preinit_parse_argv", config, preconfig, api=API_PYTHON) def test_preinit_dont_parse_argv(self): # -X dev must be ignored by isolated preconfiguration preconfig = { - 'isolated': 0, + 'isolated': False, } argv = ["python3", "-E", "-I", "-P", @@ -1041,37 +1179,37 @@ def test_preinit_dont_parse_argv(self): config = { 'argv': argv, 'orig_argv': argv, - 'isolated': 0, + 'isolated': False, } self.check_all_configs("test_preinit_dont_parse_argv", config, preconfig, api=API_ISOLATED) def test_init_isolated_flag(self): config = { - 'isolated': 1, - 'safe_path': 1, - 'use_environment': 0, - 'user_site_directory': 0, + 'isolated': True, + 'safe_path': True, + 'use_environment': False, + 'user_site_directory': False, } self.check_all_configs("test_init_isolated_flag", config, api=API_PYTHON) def test_preinit_isolated1(self): # _PyPreConfig.isolated=1, _PyCoreConfig.isolated not set config = { - 'isolated': 1, - 'safe_path': 1, - 'use_environment': 0, - 'user_site_directory': 0, + 'isolated': True, + 'safe_path': True, + 'use_environment': False, + 'user_site_directory': False, } self.check_all_configs("test_preinit_isolated1", config, api=API_COMPAT) def test_preinit_isolated2(self): # _PyPreConfig.isolated=0, _PyCoreConfig.isolated=1 config = { - 'isolated': 1, - 'safe_path': 1, - 'use_environment': 0, - 'user_site_directory': 0, + 'isolated': True, + 'safe_path': True, + 'use_environment': False, + 'user_site_directory': False, } self.check_all_configs("test_preinit_isolated2", config, api=API_COMPAT) @@ -1114,12 +1252,12 @@ def modify_path(path): def test_init_sys_add(self): config = { 'faulthandler': 1, - 'xoptions': [ - 'config_xoption', - 'cmdline_xoption', - 'sysadd_xoption', - 'faulthandler', - ], + 'xoptions': { + 'config_xoption': True, + 'cmdline_xoption': True, + 'sysadd_xoption': True, + 'faulthandler': True, + }, 'warnoptions': [ 'ignore:::cmdline_warnoption', 'ignore:::sysadd_warnoption', @@ -1139,37 +1277,19 @@ def test_init_run_main(self): 'orig_argv': ['python3', '-c', code, 'arg2'], 'program_name': './python3', 'run_command': code + '\n', - 'parse_argv': 2, + 'parse_argv': True, 'sys_path_0': '', } self.check_all_configs("test_init_run_main", config, api=API_PYTHON) - def test_init_main(self): - code = ('import _testinternalcapi, json; ' - 'print(json.dumps(_testinternalcapi.get_configs()))') - config = { - 'argv': ['-c', 'arg2'], - 'orig_argv': ['python3', - '-c', code, - 'arg2'], - 'program_name': './python3', - 'run_command': code + '\n', - 'parse_argv': 2, - '_init_main': 0, - 'sys_path_0': '', - } - self.check_all_configs("test_init_main", config, - api=API_PYTHON, - stderr="Run Python code before _Py_InitializeMain") - def test_init_parse_argv(self): config = { - 'parse_argv': 2, + 'parse_argv': True, 'argv': ['-c', 'arg1', '-v', 'arg3'], 'orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], 'program_name': './argv0', 'run_command': 'pass\n', - 'use_environment': 0, + 'use_environment': False, } self.check_all_configs("test_init_parse_argv", config, api=API_PYTHON) @@ -1178,7 +1298,7 @@ def test_init_dont_parse_argv(self): 'parse_argv': 0, } config = { - 'parse_argv': 0, + 'parse_argv': False, 'argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], 'orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], 'program_name': './argv0', @@ -1261,11 +1381,11 @@ def module_search_paths(self, prefix=None, exec_prefix=None): ver = sys.version_info return [ os.path.join(prefix, sys.platlibdir, - f'python{ver.major}{ver.minor}.zip'), + f'python{ver.major}{ver.minor}{ABI_THREAD}.zip'), os.path.join(prefix, sys.platlibdir, - f'python{ver.major}.{ver.minor}'), + f'python{ver.major}.{ver.minor}{ABI_THREAD}'), os.path.join(exec_prefix, sys.platlibdir, - f'python{ver.major}.{ver.minor}', 'lib-dynload'), + f'python{ver.major}.{ver.minor}{ABI_THREAD}', 'lib-dynload'), ] @contextlib.contextmanager @@ -1319,7 +1439,7 @@ def test_init_setpythonhome(self): expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib] else: version = f'{sys.version_info.major}.{sys.version_info.minor}' - stdlib = os.path.join(home, sys.platlibdir, f'python{version}') + stdlib = os.path.join(home, sys.platlibdir, f'python{version}{ABI_THREAD}') expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) config = { @@ -1360,7 +1480,7 @@ def test_init_is_python_build_with_home(self): expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib] else: version = f'{sys.version_info.major}.{sys.version_info.minor}' - stdlib = os.path.join(home, sys.platlibdir, f'python{version}') + stdlib = os.path.join(home, sys.platlibdir, f'python{version}{ABI_THREAD}') expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) config = { @@ -1486,12 +1606,18 @@ def test_init_pyvenv_cfg(self): with self.tmpdir_with_python() as tmpdir, \ tempfile.TemporaryDirectory() as pyvenv_home: + ver = sys.version_info + base_prefix = sysconfig.get_config_var("prefix") + + # gh-128690: base_exec_prefix depends if PLATSTDLIB_LANDMARK exists + platstdlib = os.path.join(base_prefix, PLATSTDLIB_LANDMARK) + change_exec_prefix = not os.path.isdir(platstdlib) if not MS_WINDOWS: lib_dynload = os.path.join(pyvenv_home, sys.platlibdir, - f'python{ver.major}.{ver.minor}', + f'python{ver.major}.{ver.minor}{ABI_THREAD}', 'lib-dynload') os.makedirs(lib_dynload) else: @@ -1510,7 +1636,8 @@ def test_init_pyvenv_cfg(self): paths = self.module_search_paths() if not MS_WINDOWS: - paths[-1] = lib_dynload + if change_exec_prefix: + paths[-1] = lib_dynload else: paths = [ os.path.join(tmpdir, os.path.basename(paths[0])), @@ -1520,25 +1647,25 @@ def test_init_pyvenv_cfg(self): executable = self.test_exe base_executable = os.path.join(pyvenv_home, os.path.basename(executable)) - exec_prefix = pyvenv_home config = { - 'base_prefix': sysconfig.get_config_var("prefix"), - 'base_exec_prefix': exec_prefix, - 'exec_prefix': exec_prefix, + 'base_prefix': base_prefix, + 'exec_prefix': tmpdir, + 'prefix': tmpdir, 'base_executable': base_executable, 'executable': executable, 'module_search_paths': paths, } + if change_exec_prefix: + config['base_exec_prefix'] = pyvenv_home if MS_WINDOWS: config['base_prefix'] = pyvenv_home - config['prefix'] = pyvenv_home config['stdlib_dir'] = os.path.join(pyvenv_home, 'Lib') - config['use_frozen_modules'] = int(not support.Py_DEBUG) + config['use_frozen_modules'] = bool(not support.Py_DEBUG) else: # cannot reliably assume stdlib_dir here because it # depends too much on our build. But it ought to be found config['stdlib_dir'] = self.IGNORE_CONFIG - config['use_frozen_modules'] = int(not support.Py_DEBUG) + config['use_frozen_modules'] = bool(not support.Py_DEBUG) env = self.copy_paths_by_env(config) self.check_all_configs("test_init_compat_config", config, @@ -1586,7 +1713,6 @@ def test_global_pathconfig(self): # The global path configuration (_Py_path_config) must be a copy # of the path configuration of PyInterpreter.config (PyConfig). ctypes = import_helper.import_module('ctypes') - _testinternalcapi = import_helper.import_module('_testinternalcapi') def get_func(name): func = getattr(ctypes.pythonapi, name) @@ -1603,7 +1729,7 @@ def get_func(name): config = _testinternalcapi.get_configs()['config'] - self.assertEqual(Py_GetPath().split(os.path.pathsep), + self.assertEqual(tuple(Py_GetPath().split(os.path.pathsep)), config['module_search_paths']) self.assertEqual(Py_GetPrefix(), config['prefix']) self.assertEqual(Py_GetExecPrefix(), config['exec_prefix']) @@ -1625,27 +1751,44 @@ def test_init_warnoptions(self): 'ignore:::PySys_AddWarnOption2', # PySys_AddWarnOption() 'ignore:::PyConfig_BeforeRead', # PyConfig.warnoptions 'ignore:::PyConfig_AfterRead'] # PyWideStringList_Append() - preconfig = dict(allocator=PYMEM_ALLOCATOR_DEBUG) + preconfig = {} config = { - 'dev_mode': 1, - 'faulthandler': 1, 'bytes_warning': 1, - 'warnoptions': warnoptions, 'orig_argv': ['python3', '-Wignore:::cmdline1', '-Wignore:::cmdline2'], } + config_dev_mode(preconfig, config) + config['warnoptions'] = warnoptions self.check_all_configs("test_init_warnoptions", config, preconfig, api=API_PYTHON) - def test_init_set_config(self): + @unittest.skipIf(support.check_bolt_optimized, "segfaults on BOLT instrumented binaries") + def test_initconfig_api(self): + preconfig = { + 'configure_locale': True, + } config = { - '_init_main': 0, - 'bytes_warning': 2, - 'warnoptions': ['error::BytesWarning'], + 'pycache_prefix': 'conf_pycache_prefix', + 'xoptions': {'faulthandler': True}, + 'hash_seed': 10, + 'use_hash_seed': True, + 'perf_profiling': 2, } - self.check_all_configs("test_init_set_config", config, - api=API_ISOLATED) + config_dev_mode(preconfig, config) + # Temporarily enable ignore_stderr=True to ignore warnings on JIT builds + # See gh-126255 for more information + self.check_all_configs("test_initconfig_api", config, preconfig, + api=API_ISOLATED, ignore_stderr=True) + + def test_initconfig_get_api(self): + self.run_embedded_interpreter("test_initconfig_get_api") + + def test_initconfig_exit(self): + self.run_embedded_interpreter("test_initconfig_exit") + + def test_initconfig_module(self): + self.run_embedded_interpreter("test_initconfig_module") def test_get_argc_argv(self): self.run_embedded_interpreter("test_get_argc_argv") @@ -1653,21 +1796,25 @@ def test_get_argc_argv(self): def test_init_use_frozen_modules(self): tests = { - ('=on', 1), - ('=off', 0), - ('=', 1), - ('', 1), + ('=on', True), + ('=off', False), + ('=', True), + ('', True), } for raw, expected in tests: optval = f'frozen_modules{raw}' + if raw.startswith('='): + xoption_value = raw[1:] + else: + xoption_value = True config = { - 'parse_argv': 2, + 'parse_argv': True, 'argv': ['-c'], 'orig_argv': ['./argv0', '-X', optval, '-c', 'pass'], 'program_name': './argv0', 'run_command': 'pass\n', - 'use_environment': 1, - 'xoptions': [optval], + 'use_environment': True, + 'xoptions': {'frozen_modules': xoption_value}, 'use_frozen_modules': expected, } env = {'TESTFROZEN': raw[1:]} if raw else None @@ -1699,21 +1846,12 @@ def test_init_main_interpreter_settings(self): self.assertEqual(out, expected) - -class SetConfigTests(unittest.TestCase): - def test_set_config(self): - # bpo-42260: Test _PyInterpreterState_SetConfig() - import_helper.import_module('_testcapi') - cmd = [sys.executable, '-X', 'utf8', '-I', '-m', 'test._test_embed_set_config'] - proc = subprocess.run(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding='utf-8', errors='backslashreplace') - if proc.returncode and support.verbose: - print(proc.stdout) - print(proc.stderr) - self.assertEqual(proc.returncode, 0, - (proc.returncode, proc.stdout, proc.stderr)) + @threading_helper.requires_working_threading() + def test_init_in_background_thread(self): + # gh-123022: Check that running Py_Initialize() in a background + # thread doesn't crash. + out, err = self.run_embedded_interpreter("test_init_in_background_thread") + self.assertEqual(err, "") class AuditingTests(EmbeddingTestsMixin, unittest.TestCase): @@ -1782,6 +1920,7 @@ def test_unicode_id_init(self): # See bpo-44133 @unittest.skipIf(os.name == 'nt', 'Py_FrozenMain is not exported on Windows') + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") def test_frozenmain(self): env = dict(os.environ) env['PYTHONUNBUFFERED'] = '1' @@ -1792,9 +1931,9 @@ def test_frozenmain(self): sys.argv ['./argv0', '-E', 'arg1', 'arg2'] config program_name: ./argv0 config executable: {executable} - config use_environment: 1 - config configure_c_stdio: 1 - config buffered_stdio: 0 + config use_environment: True + config configure_c_stdio: True + config buffered_stdio: False """).lstrip() self.assertEqual(out, expected) @@ -1829,7 +1968,11 @@ def test_no_memleak(self): @unittest.skipUnless(support.Py_DEBUG, '-X presite requires a Python debug build') def test_presite(self): - cmd = [sys.executable, "-I", "-X", "presite=test.reperf", "-c", "print('cmd')"] + cmd = [ + sys.executable, + "-I", "-X", "presite=test._test_embed_structseq", + "-c", "print('unique-python-message')", + ] proc = subprocess.run( cmd, stdout=subprocess.PIPE, @@ -1838,9 +1981,8 @@ def test_presite(self): ) self.assertEqual(proc.returncode, 0) out = proc.stdout.strip() - self.assertIn("10 times sub", out) - self.assertIn("CPU seconds", out) - self.assertIn("cmd", out) + self.assertIn("Tests passed", out) + self.assertIn("unique-python-message", out) class StdPrinterTests(EmbeddingTestsMixin, unittest.TestCase): diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index a4b36a90d8815e..6d3c91b0b6d9f9 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -6,7 +6,6 @@ import test.support import unittest import unittest.mock -from importlib.resources.abc import Traversable from pathlib import Path import ensurepip diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 27f8bbaf952afc..8884295b1ab89c 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -14,7 +14,7 @@ from enum import Enum, EnumMeta, IntEnum, StrEnum, EnumType, Flag, IntFlag, unique, auto from enum import STRICT, CONFORM, EJECT, KEEP, _simple_enum, _test_simple_enum from enum import verify, UNIQUE, CONTINUOUS, NAMED_FLAGS, ReprEnum -from enum import member, nonmember, _iter_bits_lsb +from enum import member, nonmember, _iter_bits_lsb, EnumDict from io import StringIO from pickle import dumps, loads, PicklingError, HIGHEST_PROTOCOL from test import support @@ -1495,6 +1495,27 @@ class SpamEnum(Enum): spam = nonmember(SpamEnumIsInner) self.assertTrue(SpamEnum.spam is SpamEnumIsInner) + def test_using_members_as_nonmember(self): + class Example(Flag): + A = 1 + B = 2 + ALL = nonmember(A | B) + + self.assertEqual(Example.A.value, 1) + self.assertEqual(Example.B.value, 2) + self.assertEqual(Example.ALL, 3) + self.assertIs(type(Example.ALL), int) + + class Example(Flag): + A = auto() + B = auto() + ALL = nonmember(A | B) + + self.assertEqual(Example.A.value, 1) + self.assertEqual(Example.B.value, 2) + self.assertEqual(Example.ALL, 3) + self.assertIs(type(Example.ALL), int) + def test_nested_classes_in_enum_with_member(self): """Support locally-defined nested classes.""" class Outer(Enum): @@ -1867,6 +1888,25 @@ def test_wrong_inheritance_order(self): class Wrong(Enum, str): NotHere = 'error before this point' + def test_raise_custom_error_on_creation(self): + class InvalidRgbColorError(ValueError): + def __init__(self, r, g, b): + self.r = r + self.g = g + self.b = b + super().__init__(f'({r}, {g}, {b}) is not a valid RGB color') + + with self.assertRaises(InvalidRgbColorError): + class RgbColor(Enum): + RED = (255, 0, 0) + GREEN = (0, 255, 0) + BLUE = (0, 0, 255) + INVALID = (256, 0, 0) + + def __init__(self, r, g, b): + if not all(0 <= val <= 255 for val in (r, g, b)): + raise InvalidRgbColorError(r, g, b) + def test_intenum_transitivity(self): class number(IntEnum): one = 1 @@ -3409,6 +3449,15 @@ def __new__(cls, int_value, *value_aliases): self.assertIs(Types(2), Types.NetList) self.assertIs(Types('nl'), Types.NetList) + def test_second_tuple_item_is_falsey(self): + class Cardinal(Enum): + RIGHT = (1, 0) + UP = (0, 1) + LEFT = (-1, 0) + DOWN = (0, -1) + self.assertIs(Cardinal(1, 0), Cardinal.RIGHT) + self.assertIs(Cardinal(-1, 0), Cardinal.LEFT) + def test_no_members(self): with self.assertRaisesRegex( TypeError, @@ -3421,6 +3470,23 @@ def test_no_members(self): ): Flag(7) + def test_empty_names(self): + for nothing in '', [], {}: + for e_type in None, int: + empty_enum = Enum('empty_enum', nothing, type=e_type) + self.assertEqual(len(empty_enum), 0) + self.assertRaisesRegex(TypeError, 'has no members', empty_enum, 0) + self.assertRaisesRegex(TypeError, '.int. object is not iterable', Enum, 'bad_enum', names=0) + self.assertRaisesRegex(TypeError, '.int. object is not iterable', Enum, 'bad_enum', 0, type=int) + + def test_nonhashable_matches_hashable(self): # issue 125710 + class Directions(Enum): + DOWN_ONLY = frozenset({"sc"}) + UP_ONLY = frozenset({"cs"}) + UNRESTRICTED = frozenset({"sc", "cs"}) + self.assertIs(Directions({"sc"}), Directions.DOWN_ONLY) + + class TestOrder(unittest.TestCase): "test usage of the `_order_` attribute" @@ -4044,6 +4110,8 @@ def test_global_repr_conform1(self): @reraise_if_not_enum(NoName) def test_global_enum_str(self): + self.assertEqual(repr(NoName.ONE), 'test_enum.ONE') + self.assertEqual(repr(NoName(0)), 'test_enum.NoName(0)') self.assertEqual(str(NoName.ONE & NoName.TWO), 'NoName(0)') self.assertEqual(str(NoName(0)), 'NoName(0)') @@ -5149,7 +5217,57 @@ class Unhashable: self.assertIn('python', Unhashable) self.assertEqual(Unhashable.name.value, 'python') self.assertEqual(Unhashable.name.name, 'name') - _test_simple_enum(Unhashable, Unhashable) + _test_simple_enum(CheckedUnhashable, Unhashable) + ## + class CheckedComplexStatus(IntEnum): + def __new__(cls, value, phrase, description=''): + obj = int.__new__(cls, value) + obj._value_ = value + obj.phrase = phrase + obj.description = description + return obj + CONTINUE = 100, 'Continue', 'Request received, please continue' + PROCESSING = 102, 'Processing' + EARLY_HINTS = 103, 'Early Hints' + SOME_HINTS = 103, 'Some Early Hints' + # + @_simple_enum(IntEnum) + class ComplexStatus: + def __new__(cls, value, phrase, description=''): + obj = int.__new__(cls, value) + obj._value_ = value + obj.phrase = phrase + obj.description = description + return obj + CONTINUE = 100, 'Continue', 'Request received, please continue' + PROCESSING = 102, 'Processing' + EARLY_HINTS = 103, 'Early Hints' + SOME_HINTS = 103, 'Some Early Hints' + _test_simple_enum(CheckedComplexStatus, ComplexStatus) + # + # + class CheckedComplexFlag(IntFlag): + def __new__(cls, value, label): + obj = int.__new__(cls, value) + obj._value_ = value + obj.label = label + return obj + SHIRT = 1, 'upper half' + VEST = 1, 'outer upper half' + PANTS = 2, 'lower half' + self.assertIs(CheckedComplexFlag.SHIRT, CheckedComplexFlag.VEST) + # + @_simple_enum(IntFlag) + class ComplexFlag: + def __new__(cls, value, label): + obj = int.__new__(cls, value) + obj._value_ = value + obj.label = label + return obj + SHIRT = 1, 'upper half' + VEST = 1, 'uppert half' + PANTS = 2, 'lower half' + _test_simple_enum(CheckedComplexFlag, ComplexFlag) class MiscTestCase(unittest.TestCase): @@ -5236,7 +5354,7 @@ def test_convert_value_lookup_priority(self): filter=lambda x: x.startswith('CONVERT_TEST_')) # We don't want the reverse lookup value to vary when there are # multiple possible names for a given value. It should always - # report the first lexigraphical name in that case. + # report the first lexicographical name in that case. self.assertEqual(test_type(5).name, 'CONVERT_TEST_NAME_A') def test_convert_int(self): @@ -5322,6 +5440,37 @@ def test_convert_repr_and_str(self): self.assertEqual(format(test_type.CONVERT_STRING_TEST_NAME_A), '5') +class TestEnumDict(unittest.TestCase): + def test_enum_dict_in_metaclass(self): + """Test that EnumDict is usable as a class namespace""" + class Meta(type): + @classmethod + def __prepare__(metacls, cls, bases, **kwds): + return EnumDict(cls) + + class MyClass(metaclass=Meta): + a = 1 + + with self.assertRaises(TypeError): + a = 2 # duplicate + + with self.assertRaises(ValueError): + _a_sunder_ = 3 + + def test_enum_dict_standalone(self): + """Test that EnumDict is usable on its own""" + enumdict = EnumDict() + enumdict['a'] = 1 + + with self.assertRaises(TypeError): + enumdict['a'] = 'other value' + + # Only MutableMapping interface is overridden for now. + # If this stops passing, update the documentation. + enumdict |= {'a': 'other value'} + self.assertEqual(enumdict['a'], 'other value') + + # helpers def enum_dir(cls): diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py index be4fd73bfdc36b..582e5b6de6e687 100644 --- a/Lib/test/test_eof.py +++ b/Lib/test/test_eof.py @@ -1,7 +1,8 @@ """test script for a few new invalid token catches""" import sys -from test import support +from codecs import BOM_UTF8 +from test.support import force_not_colorized from test.support import os_helper from test.support import script_helper from test.support import warnings_helper @@ -11,67 +12,160 @@ class EOFTestCase(unittest.TestCase): def test_EOF_single_quote(self): expect = "unterminated string literal (detected at line 1) (, line 1)" for quote in ("'", "\""): - try: + with self.assertRaises(SyntaxError) as cm: eval(f"""{quote}this is a test\ """) - except SyntaxError as msg: - self.assertEqual(str(msg), expect) - self.assertEqual(msg.offset, 1) - else: - raise support.TestFailed + self.assertEqual(str(cm.exception), expect) + self.assertEqual(cm.exception.offset, 1) def test_EOFS(self): - expect = ("unterminated triple-quoted string literal (detected at line 1) (, line 1)") - try: - eval("""'''this is a test""") - except SyntaxError as msg: - self.assertEqual(str(msg), expect) - self.assertEqual(msg.offset, 1) - else: - raise support.TestFailed + expect = ("unterminated triple-quoted string literal (detected at line 3) (, line 1)") + with self.assertRaises(SyntaxError) as cm: + eval("""ä = '''thîs is \na \ntest""") + self.assertEqual(str(cm.exception), expect) + self.assertEqual(cm.exception.text, "ä = '''thîs is ") + self.assertEqual(cm.exception.offset, 5) + with self.assertRaises(SyntaxError) as cm: + eval("""ä = '''thîs is \na \ntest""".encode()) + self.assertEqual(str(cm.exception), expect) + self.assertEqual(cm.exception.text, "ä = '''thîs is ") + self.assertEqual(cm.exception.offset, 5) + + with self.assertRaises(SyntaxError) as cm: + eval(BOM_UTF8 + """ä = '''thîs is \na \ntest""".encode()) + self.assertEqual(str(cm.exception), expect) + self.assertEqual(cm.exception.text, "ä = '''thîs is ") + self.assertEqual(cm.exception.offset, 5) + + with self.assertRaises(SyntaxError) as cm: + eval("""# coding: latin1\nä = '''thîs is \na \ntest""".encode('latin1')) + self.assertEqual(str(cm.exception), "unterminated triple-quoted string literal (detected at line 4) (, line 2)") + self.assertEqual(cm.exception.text, "ä = '''thîs is ") + self.assertEqual(cm.exception.offset, 5) + + @force_not_colorized def test_EOFS_with_file(self): expect = ("(, line 1)") with os_helper.temp_dir() as temp_dir: - file_name = script_helper.make_script(temp_dir, 'foo', """'''this is \na \ntest""") - rc, out, err = script_helper.assert_python_failure(file_name) - self.assertIn(b'unterminated triple-quoted string literal (detected at line 3)', err) + file_name = script_helper.make_script(temp_dir, 'foo', + """ä = '''thîs is \na \ntest""") + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-3:], [ + " ä = '''thîs is ", + ' ^', + 'SyntaxError: unterminated triple-quoted string literal (detected at line 3)']) + + file_name = script_helper.make_script(temp_dir, 'foo', + """ä = '''thîs is \na \ntest""".encode()) + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-3:], [ + " ä = '''thîs is ", + ' ^', + 'SyntaxError: unterminated triple-quoted string literal (detected at line 3)']) + + file_name = script_helper.make_script(temp_dir, 'foo', + BOM_UTF8 + """ä = '''thîs is \na \ntest""".encode()) + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-3:], [ + " ä = '''thîs is ", + ' ^', + 'SyntaxError: unterminated triple-quoted string literal (detected at line 3)']) + + file_name = script_helper.make_script(temp_dir, 'foo', + """# coding: latin1\nä = '''thîs is \na \ntest""".encode('latin1')) + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-3:], [ + " ä = '''thîs is ", + ' ^', + 'SyntaxError: unterminated triple-quoted string literal (detected at line 4)']) @warnings_helper.ignore_warnings(category=SyntaxWarning) def test_eof_with_line_continuation(self): expect = "unexpected EOF while parsing (, line 1)" - try: + with self.assertRaises(SyntaxError) as cm: compile('"\\Xhh" \\', '', 'exec') - except SyntaxError as msg: - self.assertEqual(str(msg), expect) - else: - raise support.TestFailed + self.assertEqual(str(cm.exception), expect) def test_line_continuation_EOF(self): """A continuation at the end of input must be an error; bpo2180.""" expect = 'unexpected EOF while parsing (, line 1)' - with self.assertRaises(SyntaxError) as excinfo: - exec('x = 5\\') - self.assertEqual(str(excinfo.exception), expect) - with self.assertRaises(SyntaxError) as excinfo: + with self.assertRaises(SyntaxError) as cm: + exec('ä = 5\\') + self.assertEqual(str(cm.exception), expect) + self.assertEqual(cm.exception.text, 'ä = 5\\\n') + self.assertEqual(cm.exception.offset, 7) + + with self.assertRaises(SyntaxError) as cm: + exec('ä = 5\\'.encode()) + self.assertEqual(str(cm.exception), expect) + self.assertEqual(cm.exception.text, 'ä = 5\\\n') + self.assertEqual(cm.exception.offset, 7) + + with self.assertRaises(SyntaxError) as cm: + exec('# coding:latin1\nä = 5\\'.encode('latin1')) + self.assertEqual(str(cm.exception), + 'unexpected EOF while parsing (, line 2)') + self.assertEqual(cm.exception.text, 'ä = 5\\\n') + self.assertEqual(cm.exception.offset, 7) + + with self.assertRaises(SyntaxError) as cm: + exec(BOM_UTF8 + 'ä = 5\\'.encode()) + self.assertEqual(str(cm.exception), expect) + self.assertEqual(cm.exception.text, 'ä = 5\\\n') + self.assertEqual(cm.exception.offset, 7) + + with self.assertRaises(SyntaxError) as cm: exec('\\') - self.assertEqual(str(excinfo.exception), expect) + self.assertEqual(str(cm.exception), expect) @unittest.skipIf(not sys.executable, "sys.executable required") + @force_not_colorized def test_line_continuation_EOF_from_file_bpo2180(self): """Ensure tok_nextc() does not add too many ending newlines.""" with os_helper.temp_dir() as temp_dir: file_name = script_helper.make_script(temp_dir, 'foo', '\\') - rc, out, err = script_helper.assert_python_failure(file_name) - self.assertIn(b'unexpected EOF while parsing', err) - self.assertIn(b'line 1', err) - self.assertIn(b'\\', err) - - file_name = script_helper.make_script(temp_dir, 'foo', 'y = 6\\') - rc, out, err = script_helper.assert_python_failure(file_name) - self.assertIn(b'unexpected EOF while parsing', err) - self.assertIn(b'line 1', err) - self.assertIn(b'y = 6\\', err) + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-2:], [ + ' \\', + 'SyntaxError: unexpected EOF while parsing']) + self.assertEqual(err[-3][-8:], ', line 1', err) + + file_name = script_helper.make_script(temp_dir, 'foo', 'ä = 6\\') + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-3:], [ + ' ä = 6\\', + ' ^', + 'SyntaxError: unexpected EOF while parsing']) + self.assertEqual(err[-4][-8:], ', line 1', err) + + file_name = script_helper.make_script(temp_dir, 'foo', + '# coding:latin1\n' + 'ä = 7\\'.encode('latin1')) + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-3:], [ + ' ä = 7\\', + ' ^', + 'SyntaxError: unexpected EOF while parsing']) + self.assertEqual(err[-4][-8:], ', line 2', err) + + file_name = script_helper.make_script(temp_dir, 'foo', + BOM_UTF8 + 'ä = 8\\'.encode()) + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-3:], [ + ' ä = 8\\', + ' ^', + 'SyntaxError: unexpected EOF while parsing']) + self.assertEqual(err[-4][-8:], ', line 1', err) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_except_star.py b/Lib/test/test_except_star.py index c49c6008e08e8c..284907f61213f8 100644 --- a/Lib/test/test_except_star.py +++ b/Lib/test/test_except_star.py @@ -952,6 +952,49 @@ def derive(self, excs): self.assertExceptionIsLike(tes, FalsyEG("eg", [TypeError(1)])) self.assertExceptionIsLike(ves, FalsyEG("eg", [ValueError(2)])) + def test_exception_group_subclass_with_bad_split_func(self): + # see gh-128049. + class BadEG1(ExceptionGroup): + def split(self, *args): + return "NOT A 2-TUPLE!" + + class BadEG2(ExceptionGroup): + def split(self, *args): + return ("NOT A 2-TUPLE!",) + + eg_list = [ + (BadEG1("eg", [OSError(123), ValueError(456)]), + r"split must return a tuple, not str"), + (BadEG2("eg", [OSError(123), ValueError(456)]), + r"split must return a 2-tuple, got tuple of size 1") + ] + + for eg_class, msg in eg_list: + with self.assertRaisesRegex(TypeError, msg) as m: + try: + raise eg_class + except* ValueError: + pass + except* OSError: + pass + + self.assertExceptionIsLike(m.exception.__context__, eg_class) + + # we allow tuples of length > 2 for backwards compatibility + class WeirdEG(ExceptionGroup): + def split(self, *args): + return super().split(*args) + ("anything", 123456, None) + + try: + raise WeirdEG("eg", [OSError(123), ValueError(456)]) + except* OSError as e: + oeg = e + except* ValueError as e: + veg = e + + self.assertExceptionIsLike(oeg, WeirdEG("eg", [OSError(123)])) + self.assertExceptionIsLike(veg, WeirdEG("eg", [ValueError(456)])) + class TestExceptStarCleanup(ExceptStarTest): def test_sys_exception_restored(self): diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 20122679223843..53212529c27e28 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -1,7 +1,7 @@ import collections.abc import types import unittest -from test.support import Py_C_RECURSION_LIMIT +from test.support import get_c_recursion_limit, skip_emscripten_stack_overflow class TestExceptionGroupTypeHierarchy(unittest.TestCase): def test_exception_group_types(self): @@ -460,15 +460,17 @@ def test_basics_split_by_predicate__match(self): class DeepRecursionInSplitAndSubgroup(unittest.TestCase): def make_deep_eg(self): e = TypeError(1) - for i in range(Py_C_RECURSION_LIMIT + 1): + for i in range(get_c_recursion_limit() + 1): e = ExceptionGroup('eg', [e]) return e + @skip_emscripten_stack_overflow() def test_deep_split(self): e = self.make_deep_eg() with self.assertRaises(RecursionError): e.split(TypeError) + @skip_emscripten_stack_overflow() def test_deep_subgroup(self): e = self.make_deep_eg() with self.assertRaises(RecursionError): diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index c5eff8ad8ccca1..2d324827451b54 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -7,24 +7,28 @@ import pickle import weakref import errno +from codecs import BOM_UTF8 +from itertools import product from textwrap import dedent from test.support import (captured_stderr, check_impl_detail, cpython_only, gc_collect, no_tracing, script_helper, - SuppressCrashReport) + SuppressCrashReport, + force_not_colorized) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink from test.support.warnings_helper import check_warnings from test import support try: + import _testcapi from _testcapi import INT_MAX except ImportError: + _testcapi = None INT_MAX = 2**31 - 1 - class NaiveException(Exception): def __init__(self, x): self.x = x @@ -40,6 +44,7 @@ def __str__(self): # XXX This is not really enough, each *operation* should be tested! + class ExceptionTests(unittest.TestCase): def raise_catch(self, exc, excname): @@ -314,8 +319,8 @@ def baz(): check('def f():\n global x\n nonlocal x', 2, 3) # Errors thrown by future.c - check('from __future__ import doesnt_exist', 1, 1) - check('from __future__ import braces', 1, 1) + check('from __future__ import doesnt_exist', 1, 24) + check('from __future__ import braces', 1, 24) check('x=1\nfrom __future__ import division', 2, 1) check('foo(1=2)', 1, 5) check('def f():\n x, y: int', 2, 3) @@ -345,8 +350,8 @@ def __init__(self_): class InvalidException: pass + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_capi1(): - import _testcapi try: _testcapi.raise_exception(BadException, 1) except TypeError as err: @@ -356,8 +361,8 @@ def test_capi1(): else: self.fail("Expected exception") + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_capi2(): - import _testcapi try: _testcapi.raise_exception(BadException, 0) except RuntimeError as err: @@ -370,8 +375,8 @@ def test_capi2(): else: self.fail("Expected exception") + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_capi3(): - import _testcapi self.assertRaises(SystemError, _testcapi.raise_exception, InvalidException, 1) @@ -1332,6 +1337,29 @@ def test_unicode_errors_no_object(self): for klass in klasses: self.assertEqual(str(klass.__new__(klass)), "") + def test_unicode_error_str_does_not_crash(self): + # Test that str(UnicodeError(...)) does not crash. + # See https://github.com/python/cpython/issues/123378. + + for start, end, objlen in product( + range(-5, 5), + range(-5, 5), + range(7), + ): + obj = 'a' * objlen + with self.subTest('encode', objlen=objlen, start=start, end=end): + exc = UnicodeEncodeError('utf-8', obj, start, end, '') + self.assertIsInstance(str(exc), str) + + with self.subTest('translate', objlen=objlen, start=start, end=end): + exc = UnicodeTranslateError(obj, start, end, '') + self.assertIsInstance(str(exc), str) + + encoded = obj.encode() + with self.subTest('decode', objlen=objlen, start=start, end=end): + exc = UnicodeDecodeError('utf-8', encoded, start, end, '') + self.assertIsInstance(str(exc), str) + @no_tracing def test_badisinstance(self): # Bug #2542: if issubclass(e, MyException) raises an exception, @@ -1381,6 +1409,7 @@ def foo(): @cpython_only def test_recursion_normalizing_exception(self): + import_module("_testinternalcapi") # Issue #22898. # Test that a RecursionError is raised when tstate->recursion_depth is # equal to recursion_limit in PyErr_NormalizeException() and check @@ -1422,7 +1451,7 @@ def gen(): next(generator) recursionlimit = sys.getrecursionlimit() try: - recurse(support.EXCEEDS_RECURSION_LIMIT) + recurse(support.exceeds_recursion_limit()) finally: sys.setrecursionlimit(recursionlimit) print('Done.') @@ -1435,6 +1464,8 @@ def gen(): self.assertIn(b'Done.', out) @cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") + @force_not_colorized def test_recursion_normalizing_infinite_exception(self): # Issue #30697. Test that a RecursionError is raised when # maximum recursion depth has been exceeded when creating @@ -1448,7 +1479,8 @@ def test_recursion_normalizing_infinite_exception(self): """ rc, out, err = script_helper.assert_python_failure("-c", code) self.assertEqual(rc, 1) - self.assertIn(b'RecursionError: maximum recursion depth exceeded', err) + expected = b'RecursionError: maximum recursion depth exceeded' + self.assertTrue(expected in err, msg=f"{expected!r} not found in {err[:3_000]!r}... (truncated)") self.assertIn(b'Done.', out) @@ -1503,6 +1535,7 @@ def recurse_in_body_and_except(): # Python built with Py_TRACE_REFS fail with a fatal error in # _PyRefchain_Trace() on memory allocation error. @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_recursion_normalizing_with_no_memory(self): # Issue #30697. Test that in the abort that occurs when there is no # memory left and the size of the Python frames stack is greater than @@ -1525,6 +1558,7 @@ def recurse(cnt): self.assertIn(b'MemoryError', err) @cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_MemoryError(self): # PyErr_NoMemory always raises the same exception instance. # Check that the traceback is not doubled. @@ -1544,8 +1578,8 @@ def raiseMemError(): self.assertEqual(tb1, tb2) @cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_exception_with_doc(self): - import _testcapi doc2 = "This is a test docstring." doc4 = "This is another test docstring." @@ -1584,6 +1618,7 @@ class C(object): self.assertEqual(error5.__doc__, "") @cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_memory_error_cleanup(self): # Issue #5437: preallocated MemoryError instances should not keep # traceback objects alive. @@ -1674,6 +1709,7 @@ def test_unhandled(self): # Python built with Py_TRACE_REFS fail with a fatal error in # _PyRefchain_Trace() on memory allocation error. @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_memory_error_in_PyErr_PrintEx(self): code = """if 1: import _testcapi @@ -1792,6 +1828,7 @@ class TestException(MemoryError): gc_collect() + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_memory_error_in_subinterp(self): # gh-109894: subinterpreters shouldn't count on last resort memory error # when MemoryError is raised through PyErr_NoMemory() call, @@ -1806,6 +1843,29 @@ def test_memory_error_in_subinterp(self): rc, _, err = script_helper.assert_python_ok("-c", code) self.assertIn(b'MemoryError', err) + def test_keyerror_context(self): + # Make sure that _PyErr_SetKeyError() chains exceptions + try: + err1 = None + err2 = None + try: + d = {} + try: + raise ValueError("bug") + except Exception as exc: + err1 = exc + d[1] + except Exception as exc: + err2 = exc + + self.assertIsInstance(err1, ValueError) + self.assertIsInstance(err2, KeyError) + self.assertEqual(err2.__context__, err1) + finally: + # Break any potential reference cycle + exc1 = None + exc2 = None + class NameErrorTests(unittest.TestCase): def test_name_error_has_name(self): @@ -1825,6 +1885,8 @@ def f(): except self.failureException: with support.captured_stderr() as err: sys.__excepthook__(*sys.exc_info()) + else: + self.fail("assertRaisesRegex should have failed.") self.assertIn("aab", err.getvalue()) @@ -1975,16 +2037,21 @@ def test_copy_pickle(self): self.assertEqual(exc.path, orig.path) +def run_script(source): + if isinstance(source, str): + with open(TESTFN, 'w', encoding='utf-8') as testfile: + testfile.write(dedent(source)) + else: + with open(TESTFN, 'wb') as testfile: + testfile.write(source) + _rc, _out, err = script_helper.assert_python_failure('-Wd', '-X', 'utf8', TESTFN) + return err.decode('utf-8').splitlines() + class AssertionErrorTests(unittest.TestCase): def tearDown(self): unlink(TESTFN) - def write_source(self, source): - with open(TESTFN, 'w') as testfile: - testfile.write(dedent(source)) - _rc, _out, err = script_helper.assert_python_failure('-Wd', '-X', 'utf8', TESTFN) - return err.decode('utf-8').splitlines() - + @force_not_colorized def test_assertion_error_location(self): cases = [ ('assert None', @@ -2015,11 +2082,32 @@ def test_assertion_error_location(self): 'AssertionError', ], ), - ('assert 1 > 2, "message"', + ('assert 1 > 2, "messäge"', + [ + ' assert 1 > 2, "messäge"', + ' ^^^^^', + 'AssertionError: messäge', + ], + ), + ('assert 1 > 2, "messäge"'.encode(), + [ + ' assert 1 > 2, "messäge"', + ' ^^^^^', + 'AssertionError: messäge', + ], + ), + ('# coding: latin1\nassert 1 > 2, "messäge"'.encode('latin1'), [ - ' assert 1 > 2, "message"', + ' assert 1 > 2, "messäge"', ' ^^^^^', - 'AssertionError: message', + 'AssertionError: messäge', + ], + ), + (BOM_UTF8 + 'assert 1 > 2, "messäge"'.encode(), + [ + ' assert 1 > 2, "messäge"', + ' ^^^^^', + 'AssertionError: messäge', ], ), @@ -2057,10 +2145,11 @@ def test_assertion_error_location(self): ), ] for source, expected in cases: - with self.subTest(source): - result = self.write_source(source) + with self.subTest(source=source): + result = run_script(source) self.assertEqual(result[-3:], expected) + @force_not_colorized def test_multiline_not_highlighted(self): cases = [ (""" @@ -2087,12 +2176,16 @@ def test_multiline_not_highlighted(self): ), ] for source, expected in cases: - with self.subTest(source): - result = self.write_source(source) + with self.subTest(source=source): + result = run_script(source) self.assertEqual(result[-len(expected):], expected) +@support.force_not_colorized_test_class class SyntaxErrorTests(unittest.TestCase): + maxDiff = None + + @force_not_colorized def test_range_of_offsets(self): cases = [ # Basic range from 2->7 @@ -2183,46 +2276,123 @@ def test_range_of_offsets(self): self.assertIn(expected, err.getvalue()) the_exception = exc + @force_not_colorized + def test_subclass(self): + class MySyntaxError(SyntaxError): + pass + + try: + raise MySyntaxError("bad bad", ("bad.py", 1, 2, "abcdefg", 1, 7)) + except SyntaxError as exc: + with support.captured_stderr() as err: + sys.__excepthook__(*sys.exc_info()) + self.assertIn(""" + File "bad.py", line 1 + abcdefg + ^^^^^ +""", err.getvalue()) + def test_encodings(self): + self.addCleanup(unlink, TESTFN) source = ( '# -*- coding: cp437 -*-\n' '"┬ó┬ó┬ó┬ó┬ó┬ó" + f(4, x for x in range(1))\n' ) - try: - with open(TESTFN, 'w', encoding='cp437') as testfile: - testfile.write(source) - rc, out, err = script_helper.assert_python_failure('-Wd', '-X', 'utf8', TESTFN) - err = err.decode('utf-8').splitlines() - - self.assertEqual(err[-3], ' "┬ó┬ó┬ó┬ó┬ó┬ó" + f(4, x for x in range(1))') - self.assertEqual(err[-2], ' ^^^^^^^^^^^^^^^^^^^') - finally: - unlink(TESTFN) + err = run_script(source.encode('cp437')) + self.assertEqual(err[-3], ' "┬ó┬ó┬ó┬ó┬ó┬ó" + f(4, x for x in range(1))') + self.assertEqual(err[-2], ' ^^^^^^^^^^^^^^^^^^^') # Check backwards tokenizer errors source = '# -*- coding: ascii -*-\n\n(\n' - try: - with open(TESTFN, 'w', encoding='ascii') as testfile: - testfile.write(source) - rc, out, err = script_helper.assert_python_failure('-Wd', '-X', 'utf8', TESTFN) - err = err.decode('utf-8').splitlines() - - self.assertEqual(err[-3], ' (') - self.assertEqual(err[-2], ' ^') - finally: - unlink(TESTFN) + err = run_script(source) + self.assertEqual(err[-3], ' (') + self.assertEqual(err[-2], ' ^') def test_non_utf8(self): # Check non utf-8 characters - try: - with open(TESTFN, 'bw') as testfile: - testfile.write(b"\x89") - rc, out, err = script_helper.assert_python_failure('-Wd', '-X', 'utf8', TESTFN) - err = err.decode('utf-8').splitlines() + self.addCleanup(unlink, TESTFN) + err = run_script(b"\x89") + self.assertIn("SyntaxError: Non-UTF-8 code starting with '\\x89' in file", err[-1]) - self.assertIn("SyntaxError: Non-UTF-8 code starting with '\\x89' in file", err[-1]) - finally: - unlink(TESTFN) + def test_string_source(self): + def try_compile(source): + with self.assertRaises(SyntaxError) as cm: + compile(source, '', 'exec') + return cm.exception + + exc = try_compile('return "ä"') + self.assertEqual(str(exc), "'return' outside function (, line 1)") + self.assertIsNone(exc.text) + self.assertEqual(exc.offset, 1) + self.assertEqual(exc.end_offset, 12) + + exc = try_compile('return "ä"'.encode()) + self.assertEqual(str(exc), "'return' outside function (, line 1)") + self.assertIsNone(exc.text) + self.assertEqual(exc.offset, 1) + self.assertEqual(exc.end_offset, 12) + + exc = try_compile(BOM_UTF8 + 'return "ä"'.encode()) + self.assertEqual(str(exc), "'return' outside function (, line 1)") + self.assertIsNone(exc.text) + self.assertEqual(exc.offset, 1) + self.assertEqual(exc.end_offset, 12) + + exc = try_compile('# coding: latin1\nreturn "ä"'.encode('latin1')) + self.assertEqual(str(exc), "'return' outside function (, line 2)") + self.assertIsNone(exc.text) + self.assertEqual(exc.offset, 1) + self.assertEqual(exc.end_offset, 12) + + exc = try_compile('return "ä" #' + 'ä'*1000) + self.assertEqual(str(exc), "'return' outside function (, line 1)") + self.assertIsNone(exc.text) + self.assertEqual(exc.offset, 1) + self.assertEqual(exc.end_offset, 12) + + exc = try_compile('return "ä" # ' + 'ä'*1000) + self.assertEqual(str(exc), "'return' outside function (, line 1)") + self.assertIsNone(exc.text) + self.assertEqual(exc.offset, 1) + self.assertEqual(exc.end_offset, 12) + + def test_file_source(self): + self.addCleanup(unlink, TESTFN) + err = run_script('return "ä"') + self.assertEqual(err[-3:], [ + ' return "ä"', + ' ^^^^^^^^^^', + "SyntaxError: 'return' outside function"]) + + err = run_script('return "ä"'.encode()) + self.assertEqual(err[-3:], [ + ' return "ä"', + ' ^^^^^^^^^^', + "SyntaxError: 'return' outside function"]) + + err = run_script(BOM_UTF8 + 'return "ä"'.encode()) + self.assertEqual(err[-3:], [ + ' return "ä"', + ' ^^^^^^^^^^', + "SyntaxError: 'return' outside function"]) + + err = run_script('# coding: latin1\nreturn "ä"'.encode('latin1')) + self.assertEqual(err[-3:], [ + ' return "ä"', + ' ^^^^^^^^^^', + "SyntaxError: 'return' outside function"]) + + err = run_script('return "ä" #' + 'ä'*1000) + self.assertEqual(err[-2:], [ + ' ^^^^^^^^^^^', + "SyntaxError: 'return' outside function"]) + self.assertEqual(err[-3][:100], ' return "ä" #' + 'ä'*84) + + err = run_script('return "ä" # ' + 'ä'*1000) + self.assertEqual(err[-2:], [ + ' ^^^^^^^^^^^', + "SyntaxError: 'return' outside function"]) + self.assertEqual(err[-3][:100], ' return "ä" # ' + 'ä'*83) def test_attributes_new_constructor(self): args = ("bad.py", 1, 2, "abcdefg", 1, 100) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 86c07de507e39c..2ab48a4778be4d 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -13,8 +13,10 @@ try: from _testexternalinspection import PROCESS_VM_READV_SUPPORTED from _testexternalinspection import get_stack_trace + from _testexternalinspection import get_async_stack_trace except ImportError: - unittest.skip("Test only runs when _testexternalinspection is available") + raise unittest.SkipTest( + "Test only runs when _testexternalinspection is available") def _make_test_script(script_dir, script_basename, source): to_return = make_script(script_dir, script_basename, source) @@ -23,12 +25,14 @@ def _make_test_script(script_dir, script_basename, source): class TestGetStackTrace(unittest.TestCase): - @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") - @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", + "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support") def test_remote_stack_trace(self): # Spawn a process with some realistic Python code script = textwrap.dedent("""\ - import time, sys, os + import time, sys def bar(): for x in range(100): if x == 50: @@ -37,8 +41,8 @@ def baz(): foo() def foo(): - fifo = sys.argv[1] - with open(sys.argv[1], "w") as fifo: + fifo_path = sys.argv[1] + with open(fifo_path, "w") as fifo: fifo.write("ready") time.sleep(1000) @@ -74,8 +78,281 @@ def foo(): ] self.assertEqual(stack_trace, expected_stack_trace) - @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") - @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", + "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support") + def test_async_remote_stack_trace(self): + # Spawn a process with some realistic Python code + script = textwrap.dedent("""\ + import asyncio + import time + import sys + + def c5(): + fifo_path = sys.argv[1] + with open(fifo_path, "w") as fifo: + fifo.write("ready") + time.sleep(10000) + + async def c4(): + await asyncio.sleep(0) + c5() + + async def c3(): + await c4() + + async def c2(): + await c3() + + async def c1(task): + await task + + async def main(): + async with asyncio.TaskGroup() as tg: + task = tg.create_task(c2(), name="c2_root") + tg.create_task(c1(task), name="sub_main_1") + tg.create_task(c1(task), name="sub_main_2") + + def new_eager_loop(): + loop = asyncio.new_event_loop() + eager_task_factory = asyncio.create_eager_task_factory( + asyncio.Task) + loop.set_task_factory(eager_task_factory) + return loop + + asyncio.run(main(), loop_factory={TASK_FACTORY}) + """) + stack_trace = None + for task_factory_variant in "asyncio.new_event_loop", "new_eager_loop": + with ( + self.subTest(task_factory_variant=task_factory_variant), + os_helper.temp_dir() as work_dir, + ): + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + fifo = f"{work_dir}/the_fifo" + os.mkfifo(fifo) + script_name = _make_test_script( + script_dir, 'script', + script.format(TASK_FACTORY=task_factory_variant)) + try: + p = subprocess.Popen( + [sys.executable, script_name, str(fifo)] + ) + with open(fifo, "r") as fifo_file: + response = fifo_file.read() + self.assertEqual(response, "ready") + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace") + finally: + os.remove(fifo) + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + # sets are unordered, so we want to sort "awaited_by"s + stack_trace[2].sort(key=lambda x: x[1]) + + root_task = "Task-1" + expected_stack_trace = [ + ["c5", "c4", "c3", "c2"], + "c2_root", + [ + [["main"], root_task, []], + [["c1"], "sub_main_1", [[["main"], root_task, []]]], + [["c1"], "sub_main_2", [[["main"], root_task, []]]], + ], + ] + self.assertEqual(stack_trace, expected_stack_trace) + + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", + "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support") + def test_asyncgen_remote_stack_trace(self): + # Spawn a process with some realistic Python code + script = textwrap.dedent("""\ + import asyncio + import time + import sys + + async def gen_nested_call(): + fifo_path = sys.argv[1] + with open(fifo_path, "w") as fifo: + fifo.write("ready") + time.sleep(10000) + + async def gen(): + for num in range(2): + yield num + if num == 1: + await gen_nested_call() + + async def main(): + async for el in gen(): + pass + + asyncio.run(main()) + """) + stack_trace = None + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + fifo = f"{work_dir}/the_fifo" + os.mkfifo(fifo) + script_name = _make_test_script(script_dir, 'script', script) + try: + p = subprocess.Popen([sys.executable, script_name, str(fifo)]) + with open(fifo, "r") as fifo_file: + response = fifo_file.read() + self.assertEqual(response, "ready") + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest("Insufficient permissions to read the stack trace") + finally: + os.remove(fifo) + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + # sets are unordered, so we want to sort "awaited_by"s + stack_trace[2].sort(key=lambda x: x[1]) + + expected_stack_trace = [ + ['gen_nested_call', 'gen', 'main'], 'Task-1', [] + ] + self.assertEqual(stack_trace, expected_stack_trace) + + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", + "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support") + def test_async_gather_remote_stack_trace(self): + # Spawn a process with some realistic Python code + script = textwrap.dedent("""\ + import asyncio + import time + import sys + + async def deep(): + await asyncio.sleep(0) + fifo_path = sys.argv[1] + with open(fifo_path, "w") as fifo: + fifo.write("ready") + time.sleep(10000) + + async def c1(): + await asyncio.sleep(0) + await deep() + + async def c2(): + await asyncio.sleep(0) + + async def main(): + await asyncio.gather(c1(), c2()) + + asyncio.run(main()) + """) + stack_trace = None + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + fifo = f"{work_dir}/the_fifo" + os.mkfifo(fifo) + script_name = _make_test_script(script_dir, 'script', script) + try: + p = subprocess.Popen([sys.executable, script_name, str(fifo)]) + with open(fifo, "r") as fifo_file: + response = fifo_file.read() + self.assertEqual(response, "ready") + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace") + finally: + os.remove(fifo) + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + # sets are unordered, so we want to sort "awaited_by"s + stack_trace[2].sort(key=lambda x: x[1]) + + expected_stack_trace = [ + ['deep', 'c1'], 'Task-2', [[['main'], 'Task-1', []]] + ] + self.assertEqual(stack_trace, expected_stack_trace) + + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", + "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support") + def test_async_staggered_race_remote_stack_trace(self): + # Spawn a process with some realistic Python code + script = textwrap.dedent("""\ + import asyncio.staggered + import time + import sys + + async def deep(): + await asyncio.sleep(0) + fifo_path = sys.argv[1] + with open(fifo_path, "w") as fifo: + fifo.write("ready") + time.sleep(10000) + + async def c1(): + await asyncio.sleep(0) + await deep() + + async def c2(): + await asyncio.sleep(10000) + + async def main(): + await asyncio.staggered.staggered_race( + [c1, c2], + delay=None, + ) + + asyncio.run(main()) + """) + stack_trace = None + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + fifo = f"{work_dir}/the_fifo" + os.mkfifo(fifo) + script_name = _make_test_script(script_dir, 'script', script) + try: + p = subprocess.Popen([sys.executable, script_name, str(fifo)]) + with open(fifo, "r") as fifo_file: + response = fifo_file.read() + self.assertEqual(response, "ready") + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace") + finally: + os.remove(fifo) + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + # sets are unordered, so we want to sort "awaited_by"s + stack_trace[2].sort(key=lambda x: x[1]) + + expected_stack_trace = [ + ['deep', 'c1', 'run_one_coro'], 'Task-2', [[['main'], 'Task-1', []]] + ] + self.assertEqual(stack_trace, expected_stack_trace) + + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", + "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support") def test_self_trace(self): stack_trace = get_stack_trace(os.getpid()) self.assertEqual(stack_trace[0], "test_self_trace") diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index d0473500a17735..75d303cd212c82 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -7,7 +7,7 @@ import subprocess import sys from test import support -from test.support import os_helper, script_helper, is_android, MS_WINDOWS +from test.support import os_helper, script_helper, is_android, MS_WINDOWS, threading_helper import tempfile import unittest from textwrap import dedent @@ -100,7 +100,11 @@ def check_error(self, code, lineno, fatal_error, *, Raise an error if the output doesn't match the expected format. """ - if all_threads: + all_threads_disabled = ( + all_threads + and (not sys._is_gil_enabled()) + ) + if all_threads and not all_threads_disabled: if know_current_thread: header = 'Current thread 0x[0-9a-f]+' else: @@ -111,10 +115,15 @@ def check_error(self, code, lineno, fatal_error, *, if py_fatal_error: regex.append("Python runtime state: initialized") regex.append('') + if all_threads_disabled and not py_fatal_error: + regex.append("") regex.append(fr'{header} \(most recent call first\):') - if garbage_collecting: - regex.append(' Garbage-collecting') - regex.append(fr' File "", line {lineno} in {function}') + if support.Py_GIL_DISABLED and py_fatal_error and not know_current_thread: + regex.append(" ") + else: + if garbage_collecting and not all_threads_disabled: + regex.append(' Garbage-collecting') + regex.append(fr' File "", line {lineno} in {function}') regex = '\n'.join(regex) if other_regex: @@ -236,7 +245,7 @@ def test_sigfpe(self): faulthandler._sigfpe() """, 3, - 'Floating point exception') + 'Floating-point exception') @unittest.skipIf(_testcapi is None, 'need _testcapi') @unittest.skipUnless(hasattr(signal, 'SIGBUS'), 'need signal.SIGBUS') @@ -266,6 +275,7 @@ def test_sigill(self): 5, 'Illegal instruction') + @unittest.skipIf(_testcapi is None, 'need _testcapi') def check_fatal_error_func(self, release_gil): # Test that Py_FatalError() dumps a traceback with support.SuppressCrashReport(): @@ -574,10 +584,12 @@ def run(self): lineno = 8 else: lineno = 10 + # When the traceback is dumped, the waiter thread may be in the + # `self.running.set()` call or in `self.stop.wait()`. regex = r""" ^Thread 0x[0-9a-f]+ \(most recent call first\): (?: File ".*threading.py", line [0-9]+ in [_a-z]+ - ){{1,3}} File "", line 23 in run + ){{1,3}} File "", line (?:22|23) in run File ".*threading.py", line [0-9]+ in _bootstrap_inner File ".*threading.py", line [0-9]+ in _bootstrap @@ -893,6 +905,34 @@ def test_cancel_later_without_dump_traceback_later(self): self.assertEqual(output, []) self.assertEqual(exitcode, 0) + @threading_helper.requires_working_threading() + @unittest.skipUnless(support.Py_GIL_DISABLED, "only meaningful if the GIL is disabled") + def test_free_threaded_dump_traceback(self): + # gh-128400: Other threads need to be paused to invoke faulthandler + code = dedent(""" + import faulthandler + from threading import Thread, Event + + class Waiter(Thread): + def __init__(self): + Thread.__init__(self) + self.running = Event() + self.stop = Event() + + def run(self): + self.running.set() + self.stop.wait() + + for _ in range(100): + waiter = Waiter() + waiter.start() + waiter.running.wait() + faulthandler.dump_traceback(all_threads=True) + waiter.stop.set() + waiter.join() + """) + _, exitcode = self.get_output(code) + self.assertEqual(exitcode, 0) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 8b4ed4a9e3a4fe..bb784cbe2ce036 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -117,7 +117,9 @@ def test_fcntl_bad_file(self): @cpython_only def test_fcntl_bad_file_overflow(self): - from _testcapi import INT_MAX, INT_MIN + _testcapi = import_module("_testcapi") + INT_MAX = _testcapi.INT_MAX + INT_MIN = _testcapi.INT_MIN # Issue 15989 with self.assertRaises(OverflowError): fcntl.fcntl(INT_MAX + 1, fcntl.F_SETFL, os.O_NONBLOCK) @@ -129,9 +131,9 @@ def test_fcntl_bad_file_overflow(self): fcntl.fcntl(BadFile(INT_MIN - 1), fcntl.F_SETFL, os.O_NONBLOCK) @unittest.skipIf( - any(platform.machine().startswith(name) for name in {"arm", "aarch"}) - and platform.system() in {"Linux", "Android"}, - "ARM Linux returns EINVAL for F_NOTIFY DN_MULTISHOT") + (platform.machine().startswith("arm") and platform.system() == "Linux") + or platform.system() == "Android", + "this platform returns EINVAL for F_NOTIFY DN_MULTISHOT") def test_fcntl_64_bit(self): # Issue #1309352: fcntl shouldn't fail when the third arg fits in a # C 'long' but not in a C 'int'. @@ -189,7 +191,7 @@ def test_lockf_share(self): @cpython_only def test_flock_overflow(self): - import _testcapi + _testcapi = import_module("_testcapi") self.assertRaises(OverflowError, fcntl.flock, _testcapi.INT_MAX+1, fcntl.LOCK_SH) diff --git a/Lib/test/test_file.py b/Lib/test/test_file.py index 9df55278693531..1206032a93566e 100644 --- a/Lib/test/test_file.py +++ b/Lib/test/test_file.py @@ -126,7 +126,7 @@ def testMethods(self): # it must also return None if an exception was given try: 1/0 - except: + except ZeroDivisionError: self.assertEqual(self.f.__exit__(*sys.exc_info()), None) def testReadWhenWriting(self): diff --git a/Lib/test/test_file_eintr.py b/Lib/test/test_file_eintr.py index f9236f45ca4be8..466f94e389f986 100644 --- a/Lib/test/test_file_eintr.py +++ b/Lib/test/test_file_eintr.py @@ -21,8 +21,8 @@ raise unittest.SkipTest("test module requires subprocess") # Test import all of the things we're about to try testing up front. -import _io -import _pyio +import _io # noqa: F401 +import _pyio # noqa: F401 @unittest.skipUnless(os.name == 'posix', 'tests requires a posix system.') class TestFileIOSignalInterrupt: diff --git a/Lib/test/test_filecmp.py b/Lib/test/test_filecmp.py index b5df71678264a8..2c83667b22feb4 100644 --- a/Lib/test/test_filecmp.py +++ b/Lib/test/test_filecmp.py @@ -1,5 +1,6 @@ import filecmp import os +import re import shutil import tempfile import unittest @@ -155,6 +156,39 @@ def test_cmpfiles(self): (['file'], ['file2'], []), "Comparing mismatched directories fails") + def test_cmpfiles_invalid_names(self): + # See https://github.com/python/cpython/issues/122400. + for file, desc in [ + ('\x00', 'NUL bytes filename'), + (__file__ + '\x00', 'filename with embedded NUL bytes'), + ("\uD834\uDD1E.py", 'surrogate codes (MUSICAL SYMBOL G CLEF)'), + ('a' * 1_000_000, 'very long filename'), + ]: + for other_dir in [self.dir, self.dir_same, self.dir_diff]: + with self.subTest(f'cmpfiles: {desc}', other_dir=other_dir): + res = filecmp.cmpfiles(self.dir, other_dir, [file]) + self.assertTupleEqual(res, ([], [], [file])) + + def test_dircmp_invalid_names(self): + for bad_dir, desc in [ + ('\x00', 'NUL bytes dirname'), + (f'Top{os.sep}Mid\x00', 'dirname with embedded NUL bytes'), + ("\uD834\uDD1E", 'surrogate codes (MUSICAL SYMBOL G CLEF)'), + ('a' * 1_000_000, 'very long dirname'), + ]: + d1 = filecmp.dircmp(self.dir, bad_dir) + d2 = filecmp.dircmp(bad_dir, self.dir) + for target in [ + # attributes where os.listdir() raises OSError or ValueError + 'left_list', 'right_list', + 'left_only', 'right_only', 'common', + ]: + with self.subTest(f'dircmp(ok, bad): {desc}', target=target): + with self.assertRaises((OSError, ValueError)): + getattr(d1, target) + with self.subTest(f'dircmp(bad, ok): {desc}', target=target): + with self.assertRaises((OSError, ValueError)): + getattr(d2, target) def _assert_lists(self, actual, expected): """Assert that two lists are equal, up to ordering.""" @@ -277,6 +311,17 @@ def test_dircmp_shallow_same_file(self): ] self._assert_report(d.report, expected_report) + def test_dircmp_shallow_is_keyword_only(self): + with self.assertRaisesRegex( + TypeError, + re.escape("dircmp.__init__() takes from 3 to 5 positional arguments but 6 were given"), + ): + filecmp.dircmp(self.dir, self.dir_same, None, None, True) + self.assertIsInstance( + filecmp.dircmp(self.dir, self.dir_same, None, None, shallow=True), + filecmp.dircmp, + ) + def test_dircmp_subdirs_type(self): """Check that dircmp.subdirs respects subclassing.""" class MyDirCmp(filecmp.dircmp): diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index b3ad41d2588c4c..b340ef7ed1621c 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -23,10 +23,9 @@ from io import BytesIO, StringIO from fileinput import FileInput, hook_encoded -from pathlib import Path from test.support import verbose -from test.support.os_helper import TESTFN +from test.support.os_helper import TESTFN, FakePath from test.support.os_helper import unlink as safe_unlink from test.support import os_helper from test import support @@ -478,23 +477,23 @@ def test_iteration_buffering(self): self.assertRaises(StopIteration, next, fi) self.assertEqual(src.linesread, []) - def test_pathlib_file(self): - t1 = Path(self.writeTmp("Pathlib file.")) + def test_pathlike_file(self): + t1 = FakePath(self.writeTmp("Path-like file.")) with FileInput(t1, encoding="utf-8") as fi: line = fi.readline() - self.assertEqual(line, 'Pathlib file.') + self.assertEqual(line, 'Path-like file.') self.assertEqual(fi.lineno(), 1) self.assertEqual(fi.filelineno(), 1) self.assertEqual(fi.filename(), os.fspath(t1)) - def test_pathlib_file_inplace(self): - t1 = Path(self.writeTmp('Pathlib file.')) + def test_pathlike_file_inplace(self): + t1 = FakePath(self.writeTmp('Path-like file.')) with FileInput(t1, inplace=True, encoding="utf-8") as fi: line = fi.readline() - self.assertEqual(line, 'Pathlib file.') + self.assertEqual(line, 'Path-like file.') print('Modified %s' % line) with open(t1, encoding="utf-8") as f: - self.assertEqual(f.read(), 'Modified Pathlib file.\n') + self.assertEqual(f.read(), 'Modified Path-like file.\n') class MockFileInput: diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index 06d5a8abf32083..5a0f033ebb82d2 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -10,19 +10,23 @@ from functools import wraps from test.support import ( - cpython_only, swap_attr, gc_collect, is_emscripten, is_wasi, - infinite_recursion, + cpython_only, swap_attr, gc_collect, is_wasi, + infinite_recursion, strace_helper ) from test.support.os_helper import ( TESTFN, TESTFN_ASCII, TESTFN_UNICODE, make_bad_fd, ) from test.support.warnings_helper import check_warnings +from test.support.import_helper import import_module from collections import UserList import _io # C implementation of io import _pyio # Python implementation of io +_strace_flags=["--trace=%file,%desc"] + + class AutoFileTests: # file tests for which a test file is automatically set up @@ -358,6 +362,143 @@ def testErrnoOnClosedReadinto(self, f): a = array('b', b'x'*10) f.readinto(a) + @strace_helper.requires_strace() + def test_syscalls_read(self): + """Check set of system calls during common I/O patterns + + It's expected as bits of the I/O implementation change, this will need + to change. The goal is to catch changes that unintentionally add + additional systemcalls (ex. additional calls have been looked at in + bpo-21679 and gh-120754). + """ + self.f.write(b"Hello, World!") + self.f.close() + + + def check_readall(name, code, prelude="", cleanup="", + extra_checks=None): + with self.subTest(name=name): + syscalls = strace_helper.get_events(code, _strace_flags, + prelude=prelude, + cleanup=cleanup) + + # Some system calls (ex. mmap) can be used for both File I/O and + # memory allocation. Filter out the ones used for memory + # allocation. + syscalls = strace_helper.filter_memory(syscalls) + + # The first call should be an open that returns a + # file descriptor (fd). Afer that calls may vary. Once the file + # is opened, check calls refer to it by fd as the filename + # could be removed from the filesystem, renamed, etc. See: + # Time-of-check time-of-use (TOCTOU) software bug class. + # + # There are a number of related but distinct open system calls + # so not checking precise name here. + self.assertGreater( + len(syscalls), + 1, + f"Should have had at least an open call|calls={syscalls}") + fd_str = syscalls[0].returncode + + # All other calls should contain the fd in their argument set. + for ev in syscalls[1:]: + self.assertIn( + fd_str, + ev.args, + f"Looking for file descriptor in arguments|ev={ev}" + ) + + # There are a number of related syscalls used to implement + # behaviors in a libc (ex. fstat, newfstatat, statx, open, openat). + # Allow any that use the same substring. + def count_similarname(name): + return len([ev for ev in syscalls if name in ev.syscall]) + + checks = [ + # Should open and close the file exactly once + ("open", 1), + ("close", 1), + # There should no longer be an isatty call (All files being + # tested are block devices / not character devices). + ('ioctl', 0), + # Should only have one fstat (bpo-21679, gh-120754) + # note: It's important this uses a fd rather than filename, + # That is validated by the `fd` check above. + # note: fstat, newfstatat, and statx have all been observed + # here in the underlying C library implementations. + ("stat", 1) + ] + + if extra_checks: + checks += extra_checks + + for call, count in checks: + self.assertEqual( + count_similarname(call), + count, + msg=f"call={call}|count={count}|syscalls={syscalls}" + ) + + # "open, read, close" file using different common patterns. + check_readall( + "open builtin with default options", + f""" + f = open('{TESTFN}') + f.read() + f.close() + """ + ) + + check_readall( + "open in binary mode", + f""" + f = open('{TESTFN}', 'rb') + f.read() + f.close() + """ + ) + + check_readall( + "open in text mode", + f""" + f = open('{TESTFN}', 'rt') + f.read() + f.close() + """, + # GH-122111: read_text uses BufferedIO which requires looking up + # position in file. `read_bytes` disables that buffering and avoids + # these calls which is tested the `pathlib read_bytes` case. + extra_checks=[("seek", 1)] + ) + + check_readall( + "pathlib read_bytes", + "p.read_bytes()", + prelude=f"""from pathlib import Path; p = Path("{TESTFN}")""", + # GH-122111: Buffering is disabled so these calls are avoided. + extra_checks=[("seek", 0)] + ) + + check_readall( + "pathlib read_text", + "p.read_text()", + prelude=f"""from pathlib import Path; p = Path("{TESTFN}")""" + ) + + # Focus on just `read()`. + calls = strace_helper.get_syscalls( + prelude=f"f = open('{TESTFN}')", + code="f.read()", + cleanup="f.close()", + strace_flags=_strace_flags + ) + # One to read all the bytes + # One to read the EOF and get a size 0 return. + self.assertEqual(calls.count("read"), 2) + + + class CAutoFileTests(AutoFileTests, unittest.TestCase): FileIO = _io.FileIO modulename = '_io' @@ -390,7 +531,7 @@ def testAbles(self): self.assertEqual(f.isatty(), False) f.close() - if sys.platform != "win32" and not is_emscripten: + if sys.platform != "win32": try: f = self.FileIO("/dev/tty", "a") except OSError: @@ -597,7 +738,7 @@ class COtherFileTests(OtherFileTests, unittest.TestCase): @cpython_only def testInvalidFd_overflow(self): # Issue 15989 - import _testcapi + _testcapi = import_module("_testcapi") self.assertRaises(TypeError, self.FileIO, _testcapi.INT_MAX + 1) self.assertRaises(TypeError, self.FileIO, _testcapi.INT_MIN - 1) diff --git a/Lib/test/test_finalization.py b/Lib/test/test_finalization.py index 1d134430909d84..42871f8a09b16b 100644 --- a/Lib/test/test_finalization.py +++ b/Lib/test/test_finalization.py @@ -13,7 +13,7 @@ def with_tp_del(cls): class C(object): def __new__(cls, *args, **kwargs): - raise TypeError('requires _testcapi.with_tp_del') + raise unittest.SkipTest('requires _testcapi.with_tp_del') return C try: @@ -22,7 +22,7 @@ def __new__(cls, *args, **kwargs): def without_gc(cls): class C: def __new__(cls, *args, **kwargs): - raise TypeError('requires _testcapi.without_gc') + raise unittest.SkipTest('requires _testcapi.without_gc') return C from test import support diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index 5bd640617d6874..f588e16b70123a 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -8,6 +8,7 @@ import unittest from test import support +from test.support.testcase import FloatsAreIdenticalMixin from test.test_grammar import (VALID_UNDERSCORE_LITERALS, INVALID_UNDERSCORE_LITERALS) from math import isinf, isnan, copysign, ldexp @@ -32,6 +33,28 @@ class FloatSubclass(float): class OtherFloatSubclass(float): pass +class MyIndex: + def __init__(self, value): + self.value = value + + def __index__(self): + return self.value + +class MyInt: + def __init__(self, value): + self.value = value + + def __int__(self): + return self.value + +class FloatLike: + def __init__(self, value): + self.value = value + + def __float__(self): + return self.value + + class GeneralFloatCases(unittest.TestCase): def test_float(self): @@ -152,7 +175,7 @@ def check(s): # non-UTF-8 byte string check(b'123\xa0') - @support.run_with_locale('LC_NUMERIC', 'fr_FR', 'de_DE') + @support.run_with_locale('LC_NUMERIC', 'fr_FR', 'de_DE', '') def test_float_with_comma(self): # set locale to something that doesn't use '.' for the decimal point # float must not accept the locale specific decimal point but @@ -181,10 +204,6 @@ def test_float_with_comma(self): def test_floatconversion(self): # Make sure that calls to __float__() work properly - class Foo1(object): - def __float__(self): - return 42. - class Foo2(float): def __float__(self): return 42. @@ -206,45 +225,29 @@ class FooStr(str): def __float__(self): return float(str(self)) + 1 - self.assertEqual(float(Foo1()), 42.) + self.assertEqual(float(FloatLike(42.)), 42.) self.assertEqual(float(Foo2()), 42.) with self.assertWarns(DeprecationWarning): self.assertEqual(float(Foo3(21)), 42.) self.assertRaises(TypeError, float, Foo4(42)) self.assertEqual(float(FooStr('8')), 9.) - class Foo5: - def __float__(self): - return "" - self.assertRaises(TypeError, time.sleep, Foo5()) + self.assertRaises(TypeError, time.sleep, FloatLike("")) # Issue #24731 - class F: - def __float__(self): - return OtherFloatSubclass(42.) + f = FloatLike(OtherFloatSubclass(42.)) with self.assertWarns(DeprecationWarning): - self.assertEqual(float(F()), 42.) + self.assertEqual(float(f), 42.) with self.assertWarns(DeprecationWarning): - self.assertIs(type(float(F())), float) + self.assertIs(type(float(f)), float) with self.assertWarns(DeprecationWarning): - self.assertEqual(FloatSubclass(F()), 42.) + self.assertEqual(FloatSubclass(f), 42.) with self.assertWarns(DeprecationWarning): - self.assertIs(type(FloatSubclass(F())), FloatSubclass) - - class MyIndex: - def __init__(self, value): - self.value = value - def __index__(self): - return self.value + self.assertIs(type(FloatSubclass(f)), FloatSubclass) self.assertEqual(float(MyIndex(42)), 42.0) self.assertRaises(OverflowError, float, MyIndex(2**2000)) - - class MyInt: - def __int__(self): - return 42 - - self.assertRaises(TypeError, float, MyInt()) + self.assertRaises(TypeError, float, MyInt(42)) def test_keyword_args(self): with self.assertRaisesRegex(TypeError, 'keyword argument'): @@ -277,6 +280,37 @@ def __new__(cls, arg, newarg=None): self.assertEqual(float(u), 2.5) self.assertEqual(u.newarg, 3) + def assertEqualAndType(self, actual, expected_value, expected_type): + self.assertEqual(actual, expected_value) + self.assertIs(type(actual), expected_type) + + def test_from_number(self, cls=float): + def eq(actual, expected): + self.assertEqual(actual, expected) + self.assertIs(type(actual), cls) + + eq(cls.from_number(3.14), 3.14) + eq(cls.from_number(314), 314.0) + eq(cls.from_number(OtherFloatSubclass(3.14)), 3.14) + eq(cls.from_number(FloatLike(3.14)), 3.14) + eq(cls.from_number(MyIndex(314)), 314.0) + + x = cls.from_number(NAN) + self.assertTrue(x != x) + self.assertIs(type(x), cls) + if cls is float: + self.assertIs(cls.from_number(NAN), NAN) + + self.assertRaises(TypeError, cls.from_number, '3.14') + self.assertRaises(TypeError, cls.from_number, b'3.14') + self.assertRaises(TypeError, cls.from_number, 3.14j) + self.assertRaises(TypeError, cls.from_number, MyInt(314)) + self.assertRaises(TypeError, cls.from_number, {}) + self.assertRaises(TypeError, cls.from_number) + + def test_from_number_subclass(self): + self.test_from_number(FloatSubclass) + def test_is_integer(self): self.assertFalse((1.1).is_integer()) self.assertTrue((1.).is_integer()) @@ -829,7 +863,7 @@ def test_short_repr(self): self.assertEqual(repr(float(negs)), str(float(negs))) @support.requires_IEEE_754 -class RoundTestCase(unittest.TestCase): +class RoundTestCase(unittest.TestCase, FloatsAreIdenticalMixin): def test_inf_nan(self): self.assertRaises(OverflowError, round, INF) @@ -859,10 +893,10 @@ def test_large_n(self): def test_small_n(self): for n in [-308, -309, -400, 1-2**31, -2**31, -2**31-1, -2**100]: - self.assertEqual(round(123.456, n), 0.0) - self.assertEqual(round(-123.456, n), -0.0) - self.assertEqual(round(1e300, n), 0.0) - self.assertEqual(round(1e-320, n), 0.0) + self.assertFloatsAreIdentical(round(123.456, n), 0.0) + self.assertFloatsAreIdentical(round(-123.456, n), -0.0) + self.assertFloatsAreIdentical(round(1e300, n), 0.0) + self.assertFloatsAreIdentical(round(1e-320, n), 0.0) def test_overflow(self): self.assertRaises(OverflowError, round, 1.6e308, -308) @@ -949,6 +983,13 @@ def test_None_ndigits(self): self.assertEqual(x, 2) self.assertIsInstance(x, int) + @support.cpython_only + def test_round_with_none_arg_direct_call(self): + for val in [(1.0).__round__(None), + round(1.0), + round(1.0, None)]: + self.assertEqual(val, 1) + self.assertIs(type(val), int) # Beginning with Python 2.6 float has cross platform compatible # ways to create and represent inf and nan @@ -1053,21 +1094,14 @@ def test_nan_signs(self): fromHex = float.fromhex toHex = float.hex -class HexFloatTestCase(unittest.TestCase): +class HexFloatTestCase(FloatsAreIdenticalMixin, unittest.TestCase): MAX = fromHex('0x.fffffffffffff8p+1024') # max normal MIN = fromHex('0x1p-1022') # min normal TINY = fromHex('0x0.0000000000001p-1022') # min subnormal EPS = fromHex('0x0.0000000000001p0') # diff between 1.0 and next float up def identical(self, x, y): - # check that floats x and y are identical, or that both - # are NaNs - if isnan(x) or isnan(y): - if isnan(x) == isnan(y): - return - elif x == y and (x != 0.0 or copysign(1.0, x) == copysign(1.0, y)): - return - self.fail('%r not identical to %r' % (x, y)) + self.assertFloatsAreIdentical(x, y) def test_ends(self): self.identical(self.MIN, ldexp(1.0, -1022)) diff --git a/Lib/test/test_fnmatch.py b/Lib/test/test_fnmatch.py index 10ed496d4e2f37..9f360e1dc10f47 100644 --- a/Lib/test/test_fnmatch.py +++ b/Lib/test/test_fnmatch.py @@ -250,6 +250,75 @@ def test_translate(self): self.assertTrue(re.match(fatre, 'cbabcaxc')) self.assertFalse(re.match(fatre, 'dabccbad')) + def test_translate_wildcards(self): + for pattern, expect in [ + ('ab*', r'(?s:ab.*)\Z'), + ('ab*cd', r'(?s:ab.*cd)\Z'), + ('ab*cd*', r'(?s:ab(?>.*?cd).*)\Z'), + ('ab*cd*12', r'(?s:ab(?>.*?cd).*12)\Z'), + ('ab*cd*12*', r'(?s:ab(?>.*?cd)(?>.*?12).*)\Z'), + ('ab*cd*12*34', r'(?s:ab(?>.*?cd)(?>.*?12).*34)\Z'), + ('ab*cd*12*34*', r'(?s:ab(?>.*?cd)(?>.*?12)(?>.*?34).*)\Z'), + ]: + with self.subTest(pattern): + translated = translate(pattern) + self.assertEqual(translated, expect, pattern) + + for pattern, expect in [ + ('*ab', r'(?s:.*ab)\Z'), + ('*ab*', r'(?s:(?>.*?ab).*)\Z'), + ('*ab*cd', r'(?s:(?>.*?ab).*cd)\Z'), + ('*ab*cd*', r'(?s:(?>.*?ab)(?>.*?cd).*)\Z'), + ('*ab*cd*12', r'(?s:(?>.*?ab)(?>.*?cd).*12)\Z'), + ('*ab*cd*12*', r'(?s:(?>.*?ab)(?>.*?cd)(?>.*?12).*)\Z'), + ('*ab*cd*12*34', r'(?s:(?>.*?ab)(?>.*?cd)(?>.*?12).*34)\Z'), + ('*ab*cd*12*34*', r'(?s:(?>.*?ab)(?>.*?cd)(?>.*?12)(?>.*?34).*)\Z'), + ]: + with self.subTest(pattern): + translated = translate(pattern) + self.assertEqual(translated, expect, pattern) + + def test_translate_expressions(self): + for pattern, expect in [ + ('[', r'(?s:\[)\Z'), + ('[!', r'(?s:\[!)\Z'), + ('[]', r'(?s:\[\])\Z'), + ('[abc', r'(?s:\[abc)\Z'), + ('[!abc', r'(?s:\[!abc)\Z'), + ('[abc]', r'(?s:[abc])\Z'), + ('[!abc]', r'(?s:[^abc])\Z'), + ('[!abc][!def]', r'(?s:[^abc][^def])\Z'), + # with [[ + ('[[', r'(?s:\[\[)\Z'), + ('[[a', r'(?s:\[\[a)\Z'), + ('[[]', r'(?s:[\[])\Z'), + ('[[]a', r'(?s:[\[]a)\Z'), + ('[[]]', r'(?s:[\[]\])\Z'), + ('[[]a]', r'(?s:[\[]a\])\Z'), + ('[[a]', r'(?s:[\[a])\Z'), + ('[[a]]', r'(?s:[\[a]\])\Z'), + ('[[a]b', r'(?s:[\[a]b)\Z'), + # backslashes + ('[\\', r'(?s:\[\\)\Z'), + (r'[\]', r'(?s:[\\])\Z'), + (r'[\\]', r'(?s:[\\\\])\Z'), + ]: + with self.subTest(pattern): + translated = translate(pattern) + self.assertEqual(translated, expect, pattern) + + def test_star_indices_locations(self): + from fnmatch import _translate + + blocks = ['a^b', '***', '?', '?', '[a-z]', '[1-9]', '*', '++', '[[a'] + parts, star_indices = _translate(''.join(blocks), '*', '.') + expect_parts = ['a', r'\^', 'b', '*', + '.', '.', '[a-z]', '[1-9]', '*', + r'\+', r'\+', r'\[', r'\[', 'a'] + self.assertListEqual(parts, expect_parts) + self.assertListEqual(star_indices, [3, 8]) + + class FilterTestCase(unittest.TestCase): def test_filter(self): diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index 6fa49dbc0b730c..9dde63e40d06db 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -4,6 +4,7 @@ import re import test.support as support import unittest +from test.support.import_helper import import_module maxsize = support.MAX_Py_ssize_t @@ -35,7 +36,7 @@ def testformat(formatstr, args, output=None, limit=None, overflowok=False): # when 'limit' is specified, it determines how many characters # must match exactly; lengths must always match. # ex: limit=5, '12345678' matches '12345___' - # (mainly for floating point format tests for which an exact match + # (mainly for floating-point format tests for which an exact match # can't be guaranteed due to rounding and representation errors) elif output and limit is not None and ( len(result)!=len(output) or result[:limit]!=output[:limit]): @@ -303,9 +304,9 @@ def test_str_format(self): test_exc('%c', sys.maxunicode+1, OverflowError, "%c arg not in range(0x110000)") #test_exc('%c', 2**128, OverflowError, "%c arg not in range(0x110000)") - test_exc('%c', 3.14, TypeError, "%c requires int or char") - test_exc('%c', 'ab', TypeError, "%c requires int or char") - test_exc('%c', b'x', TypeError, "%c requires int or char") + test_exc('%c', 3.14, TypeError, "%c requires an int or a unicode character, not float") + test_exc('%c', 'ab', TypeError, "%c requires an int or a unicode character, not a string of length 2") + test_exc('%c', b'x', TypeError, "%c requires an int or a unicode character, not bytes") if maxsize == 2**31-1: # crashes 2.2.1 and earlier: @@ -369,11 +370,11 @@ def __bytes__(self): test_exc(b"%c", 2**128, OverflowError, "%c arg not in range(256)") test_exc(b"%c", b"Za", TypeError, - "%c requires an integer in range(256) or a single byte") + "%c requires an integer in range(256) or a single byte, not a bytes object of length 2") test_exc(b"%c", "Y", TypeError, - "%c requires an integer in range(256) or a single byte") + "%c requires an integer in range(256) or a single byte, not str") test_exc(b"%c", 3.14, TypeError, - "%c requires an integer in range(256) or a single byte") + "%c requires an integer in range(256) or a single byte, not float") test_exc(b"%b", "Xc", TypeError, "%b requires a bytes-like object, " "or an object that implements __bytes__, not 'str'") @@ -478,7 +479,8 @@ def test_precision(self): @support.cpython_only def test_precision_c_limits(self): - from _testcapi import INT_MAX + _testcapi = import_module("_testcapi") + INT_MAX = _testcapi.INT_MAX f = 1.2 with self.assertRaises(ValueError) as cm: diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index b45bd098a36684..98dccbec9566ac 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -91,6 +91,204 @@ class DummyFraction(fractions.Fraction): def _components(r): return (r.numerator, r.denominator) +def typed_approx_eq(a, b): + return type(a) == type(b) and (a == b or math.isclose(a, b)) + +class Symbolic: + """Simple non-numeric class for testing mixed arithmetic. + It is not Integral, Rational, Real or Complex, and cannot be converted + to int, float or complex. but it supports some arithmetic operations. + """ + def __init__(self, value): + self.value = value + def __mul__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(f'{self} * {other}') + def __rmul__(self, other): + return self.__class__(f'{other} * {self}') + def __truediv__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(f'{self} / {other}') + def __rtruediv__(self, other): + return self.__class__(f'{other} / {self}') + def __mod__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(f'{self} % {other}') + def __rmod__(self, other): + return self.__class__(f'{other} % {self}') + def __pow__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(f'{self} ** {other}') + def __rpow__(self, other): + return self.__class__(f'{other} ** {self}') + def __eq__(self, other): + if other.__class__ != self.__class__: + return NotImplemented + return self.value == other.value + def __str__(self): + return f'{self.value}' + def __repr__(self): + return f'{self.__class__.__name__}({self.value!r})' + +class SymbolicReal(Symbolic): + pass +numbers.Real.register(SymbolicReal) + +class SymbolicComplex(Symbolic): + pass +numbers.Complex.register(SymbolicComplex) + +class Rat: + """Simple Rational class for testing mixed arithmetic.""" + def __init__(self, n, d): + self.numerator = n + self.denominator = d + def __mul__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.numerator * other.numerator, + self.denominator * other.denominator) + def __rmul__(self, other): + return self.__class__(other.numerator * self.numerator, + other.denominator * self.denominator) + def __truediv__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.numerator * other.denominator, + self.denominator * other.numerator) + def __rtruediv__(self, other): + return self.__class__(other.numerator * self.denominator, + other.denominator * self.numerator) + def __mod__(self, other): + if isinstance(other, F): + return NotImplemented + d = self.denominator * other.numerator + return self.__class__(self.numerator * other.denominator % d, d) + def __rmod__(self, other): + d = other.denominator * self.numerator + return self.__class__(other.numerator * self.denominator % d, d) + + return self.__class__(other.numerator / self.numerator, + other.denominator / self.denominator) + def __pow__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.numerator ** other, + self.denominator ** other) + def __float__(self): + return self.numerator / self.denominator + def __eq__(self, other): + if self.__class__ != other.__class__: + return NotImplemented + return (typed_approx_eq(self.numerator, other.numerator) and + typed_approx_eq(self.denominator, other.denominator)) + def __repr__(self): + return f'{self.__class__.__name__}({self.numerator!r}, {self.denominator!r})' +numbers.Rational.register(Rat) + +class Root: + """Simple Real class for testing mixed arithmetic.""" + def __init__(self, v, n=F(2)): + self.base = v + self.degree = n + def __mul__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.base * other**self.degree, self.degree) + def __rmul__(self, other): + return self.__class__(other**self.degree * self.base, self.degree) + def __truediv__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.base / other**self.degree, self.degree) + def __rtruediv__(self, other): + return self.__class__(other**self.degree / self.base, self.degree) + def __pow__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.base, self.degree / other) + def __float__(self): + return float(self.base) ** (1 / float(self.degree)) + def __eq__(self, other): + if self.__class__ != other.__class__: + return NotImplemented + return typed_approx_eq(self.base, other.base) and typed_approx_eq(self.degree, other.degree) + def __repr__(self): + return f'{self.__class__.__name__}({self.base!r}, {self.degree!r})' +numbers.Real.register(Root) + +class Polar: + """Simple Complex class for testing mixed arithmetic.""" + def __init__(self, r, phi): + self.r = r + self.phi = phi + def __mul__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.r * other, self.phi) + def __rmul__(self, other): + return self.__class__(other * self.r, self.phi) + def __truediv__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.r / other, self.phi) + def __rtruediv__(self, other): + return self.__class__(other / self.r, -self.phi) + def __pow__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.r ** other, self.phi * other) + def __eq__(self, other): + if self.__class__ != other.__class__: + return NotImplemented + return typed_approx_eq(self.r, other.r) and typed_approx_eq(self.phi, other.phi) + def __repr__(self): + return f'{self.__class__.__name__}({self.r!r}, {self.phi!r})' +numbers.Complex.register(Polar) + +class Rect: + """Other simple Complex class for testing mixed arithmetic.""" + def __init__(self, x, y): + self.x = x + self.y = y + def __mul__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.x * other, self.y * other) + def __rmul__(self, other): + return self.__class__(other * self.x, other * self.y) + def __truediv__(self, other): + if isinstance(other, F): + return NotImplemented + return self.__class__(self.x / other, self.y / other) + def __rtruediv__(self, other): + r = self.x * self.x + self.y * self.y + return self.__class__(other * (self.x / r), other * (self.y / r)) + def __rpow__(self, other): + return Polar(other ** self.x, math.log(other) * self.y) + def __complex__(self): + return complex(self.x, self.y) + def __eq__(self, other): + if self.__class__ != other.__class__: + return NotImplemented + return typed_approx_eq(self.x, other.x) and typed_approx_eq(self.y, other.y) + def __repr__(self): + return f'{self.__class__.__name__}({self.x!r}, {self.y!r})' +numbers.Complex.register(Rect) + +class RectComplex(Rect, complex): + pass + +class Ratio: + def __init__(self, ratio): + self._ratio = ratio + def as_integer_ratio(self): + return self._ratio + class FractionTest(unittest.TestCase): @@ -163,6 +361,38 @@ def testInitFromDecimal(self): self.assertRaises(OverflowError, F, Decimal('inf')) self.assertRaises(OverflowError, F, Decimal('-inf')) + def testInitFromIntegerRatio(self): + self.assertEqual((7, 3), _components(F(Ratio((7, 3))))) + errmsg = (r"argument should be a string or a Rational instance or " + r"have the as_integer_ratio\(\) method") + # the type also has an "as_integer_ratio" attribute. + self.assertRaisesRegex(TypeError, errmsg, F, Ratio) + # bad ratio + self.assertRaises(TypeError, F, Ratio(7)) + self.assertRaises(ValueError, F, Ratio((7,))) + self.assertRaises(ValueError, F, Ratio((7, 3, 1))) + # only single-argument form + self.assertRaises(TypeError, F, Ratio((3, 7)), 11) + self.assertRaises(TypeError, F, 2, Ratio((-10, 9))) + + # as_integer_ratio not defined in a class + class A: + pass + a = A() + a.as_integer_ratio = lambda: (9, 5) + self.assertEqual((9, 5), _components(F(a))) + + # as_integer_ratio defined in a metaclass + class M(type): + def as_integer_ratio(self): + return (11, 9) + class B(metaclass=M): + pass + self.assertRaisesRegex(TypeError, errmsg, F, B) + self.assertRaisesRegex(TypeError, errmsg, F, B()) + self.assertRaises(TypeError, F.from_number, B) + self.assertRaises(TypeError, F.from_number, B()) + def testFromString(self): self.assertEqual((5, 1), _components(F("5"))) self.assertEqual((3, 2), _components(F("3/2"))) @@ -368,6 +598,37 @@ def testFromDecimal(self): ValueError, "cannot convert NaN to integer ratio", F.from_decimal, Decimal("snan")) + def testFromNumber(self, cls=F): + def check(arg, numerator, denominator): + f = cls.from_number(arg) + self.assertIs(type(f), cls) + self.assertEqual(f.numerator, numerator) + self.assertEqual(f.denominator, denominator) + + check(10, 10, 1) + check(2.5, 5, 2) + check(Decimal('2.5'), 5, 2) + check(F(22, 7), 22, 7) + check(DummyFraction(22, 7), 22, 7) + check(Rat(22, 7), 22, 7) + check(Ratio((22, 7)), 22, 7) + self.assertRaises(TypeError, cls.from_number, 3+4j) + self.assertRaises(TypeError, cls.from_number, '5/2') + self.assertRaises(TypeError, cls.from_number, []) + self.assertRaises(OverflowError, cls.from_number, float('inf')) + self.assertRaises(OverflowError, cls.from_number, Decimal('inf')) + + # as_integer_ratio not defined in a class + class A: + pass + a = A() + a.as_integer_ratio = lambda: (9, 5) + check(a, 9, 5) + + def testFromNumber_subclass(self): + self.testFromNumber(DummyFraction) + + def test_is_integer(self): self.assertTrue(F(1, 1).is_integer()) self.assertTrue(F(-1, 1).is_integer()) @@ -593,6 +854,7 @@ def testMixedArithmetic(self): self.assertTypedEquals(0.9, 1.0 - F(1, 10)) self.assertTypedEquals(0.9 + 0j, (1.0 + 0j) - F(1, 10)) + def testMixedMultiplication(self): self.assertTypedEquals(F(1, 10), F(1, 10) * 1) self.assertTypedEquals(0.1, F(1, 10) * 1.0) self.assertTypedEquals(0.1 + 0j, F(1, 10) * (1.0 + 0j)) @@ -600,6 +862,29 @@ def testMixedArithmetic(self): self.assertTypedEquals(0.1, 1.0 * F(1, 10)) self.assertTypedEquals(0.1 + 0j, (1.0 + 0j) * F(1, 10)) + self.assertTypedEquals(F(3, 2) * DummyFraction(5, 3), F(5, 2)) + self.assertTypedEquals(DummyFraction(5, 3) * F(3, 2), F(5, 2)) + self.assertTypedEquals(F(3, 2) * Rat(5, 3), Rat(15, 6)) + self.assertTypedEquals(Rat(5, 3) * F(3, 2), F(5, 2)) + + self.assertTypedEquals(F(3, 2) * Root(4), Root(F(9, 1))) + self.assertTypedEquals(Root(4) * F(3, 2), 3.0) + self.assertEqual(F(3, 2) * SymbolicReal('X'), SymbolicReal('3/2 * X')) + self.assertRaises(TypeError, operator.mul, SymbolicReal('X'), F(3, 2)) + + self.assertTypedEquals(F(3, 2) * Polar(4, 2), Polar(F(6, 1), 2)) + self.assertTypedEquals(F(3, 2) * Polar(4.0, 2), Polar(6.0, 2)) + self.assertTypedEquals(F(3, 2) * Rect(4, 3), Rect(F(6, 1), F(9, 2))) + self.assertTypedEquals(F(3, 2) * RectComplex(4, 3), RectComplex(6.0, 4.5)) + self.assertRaises(TypeError, operator.mul, Polar(4, 2), F(3, 2)) + self.assertTypedEquals(Rect(4, 3) * F(3, 2), 6.0 + 4.5j) + self.assertEqual(F(3, 2) * SymbolicComplex('X'), SymbolicComplex('3/2 * X')) + self.assertRaises(TypeError, operator.mul, SymbolicComplex('X'), F(3, 2)) + + self.assertEqual(F(3, 2) * Symbolic('X'), Symbolic('3/2 * X')) + self.assertRaises(TypeError, operator.mul, Symbolic('X'), F(3, 2)) + + def testMixedDivision(self): self.assertTypedEquals(F(1, 10), F(1, 10) / 1) self.assertTypedEquals(0.1, F(1, 10) / 1.0) self.assertTypedEquals(0.1 + 0j, F(1, 10) / (1.0 + 0j)) @@ -607,6 +892,28 @@ def testMixedArithmetic(self): self.assertTypedEquals(10.0, 1.0 / F(1, 10)) self.assertTypedEquals(10.0 + 0j, (1.0 + 0j) / F(1, 10)) + self.assertTypedEquals(F(3, 2) / DummyFraction(3, 5), F(5, 2)) + self.assertTypedEquals(DummyFraction(5, 3) / F(2, 3), F(5, 2)) + self.assertTypedEquals(F(3, 2) / Rat(3, 5), Rat(15, 6)) + self.assertTypedEquals(Rat(5, 3) / F(2, 3), F(5, 2)) + + self.assertTypedEquals(F(2, 3) / Root(4), Root(F(1, 9))) + self.assertTypedEquals(Root(4) / F(2, 3), 3.0) + self.assertEqual(F(3, 2) / SymbolicReal('X'), SymbolicReal('3/2 / X')) + self.assertRaises(TypeError, operator.truediv, SymbolicReal('X'), F(3, 2)) + + self.assertTypedEquals(F(3, 2) / Polar(4, 2), Polar(F(3, 8), -2)) + self.assertTypedEquals(F(3, 2) / Polar(4.0, 2), Polar(0.375, -2)) + self.assertTypedEquals(F(3, 2) / Rect(4, 3), Rect(0.24, 0.18)) + self.assertRaises(TypeError, operator.truediv, Polar(4, 2), F(2, 3)) + self.assertTypedEquals(Rect(4, 3) / F(2, 3), 6.0 + 4.5j) + self.assertEqual(F(3, 2) / SymbolicComplex('X'), SymbolicComplex('3/2 / X')) + self.assertRaises(TypeError, operator.truediv, SymbolicComplex('X'), F(3, 2)) + + self.assertEqual(F(3, 2) / Symbolic('X'), Symbolic('3/2 / X')) + self.assertRaises(TypeError, operator.truediv, Symbolic('X'), F(2, 3)) + + def testMixedIntegerDivision(self): self.assertTypedEquals(0, F(1, 10) // 1) self.assertTypedEquals(0.0, F(1, 10) // 1.0) self.assertTypedEquals(10, 1 // F(1, 10)) @@ -631,6 +938,26 @@ def testMixedArithmetic(self): self.assertTypedTupleEquals(divmod(-0.1, float('inf')), divmod(F(-1, 10), float('inf'))) self.assertTypedTupleEquals(divmod(-0.1, float('-inf')), divmod(F(-1, 10), float('-inf'))) + self.assertTypedEquals(F(3, 2) % DummyFraction(3, 5), F(3, 10)) + self.assertTypedEquals(DummyFraction(5, 3) % F(2, 3), F(1, 3)) + self.assertTypedEquals(F(3, 2) % Rat(3, 5), Rat(3, 6)) + self.assertTypedEquals(Rat(5, 3) % F(2, 3), F(1, 3)) + + self.assertRaises(TypeError, operator.mod, F(2, 3), Root(4)) + self.assertTypedEquals(Root(4) % F(3, 2), 0.5) + self.assertEqual(F(3, 2) % SymbolicReal('X'), SymbolicReal('3/2 % X')) + self.assertRaises(TypeError, operator.mod, SymbolicReal('X'), F(3, 2)) + + self.assertRaises(TypeError, operator.mod, F(3, 2), Polar(4, 2)) + self.assertRaises(TypeError, operator.mod, F(3, 2), RectComplex(4, 3)) + self.assertRaises(TypeError, operator.mod, Rect(4, 3), F(2, 3)) + self.assertEqual(F(3, 2) % SymbolicComplex('X'), SymbolicComplex('3/2 % X')) + self.assertRaises(TypeError, operator.mod, SymbolicComplex('X'), F(3, 2)) + + self.assertEqual(F(3, 2) % Symbolic('X'), Symbolic('3/2 % X')) + self.assertRaises(TypeError, operator.mod, Symbolic('X'), F(2, 3)) + + def testMixedPower(self): # ** has more interesting conversion rules. self.assertTypedEquals(F(100, 1), F(1, 10) ** -2) self.assertTypedEquals(F(100, 1), F(10, 1) ** 2) @@ -647,6 +974,40 @@ def testMixedArithmetic(self): self.assertRaises(ZeroDivisionError, operator.pow, F(0, 1), -2) + self.assertTypedEquals(F(3, 2) ** Rat(3, 1), F(27, 8)) + self.assertTypedEquals(F(3, 2) ** Rat(-3, 1), F(8, 27)) + self.assertTypedEquals(F(-3, 2) ** Rat(-3, 1), F(-8, 27)) + self.assertTypedEquals(F(9, 4) ** Rat(3, 2), 3.375) + self.assertIsInstance(F(4, 9) ** Rat(-3, 2), float) + self.assertAlmostEqual(F(4, 9) ** Rat(-3, 2), 3.375) + self.assertAlmostEqual(F(-4, 9) ** Rat(-3, 2), 3.375j) + self.assertTypedEquals(Rat(9, 4) ** F(3, 2), 3.375) + self.assertTypedEquals(Rat(3, 2) ** F(3, 1), Rat(27, 8)) + self.assertTypedEquals(Rat(3, 2) ** F(-3, 1), F(8, 27)) + self.assertIsInstance(Rat(4, 9) ** F(-3, 2), float) + self.assertAlmostEqual(Rat(4, 9) ** F(-3, 2), 3.375) + + self.assertTypedEquals(Root(4) ** F(2, 3), Root(4, 3.0)) + self.assertTypedEquals(Root(4) ** F(2, 1), Root(4, F(1))) + self.assertTypedEquals(Root(4) ** F(-2, 1), Root(4, -F(1))) + self.assertTypedEquals(Root(4) ** F(-2, 3), Root(4, -3.0)) + self.assertEqual(F(3, 2) ** SymbolicReal('X'), SymbolicReal('3/2 ** X')) + self.assertEqual(SymbolicReal('X') ** F(3, 2), SymbolicReal('X ** 1.5')) + + self.assertTypedEquals(F(3, 2) ** Rect(2, 0), Polar(F(9,4), 0.0)) + self.assertTypedEquals(F(1, 1) ** Rect(2, 3), Polar(F(1), 0.0)) + self.assertTypedEquals(F(3, 2) ** RectComplex(2, 0), Polar(2.25, 0.0)) + self.assertTypedEquals(F(1, 1) ** RectComplex(2, 3), Polar(1.0, 0.0)) + self.assertTypedEquals(Polar(4, 2) ** F(3, 2), Polar(8.0, 3.0)) + self.assertTypedEquals(Polar(4, 2) ** F(3, 1), Polar(64, 6)) + self.assertTypedEquals(Polar(4, 2) ** F(-3, 1), Polar(0.015625, -6)) + self.assertTypedEquals(Polar(4, 2) ** F(-3, 2), Polar(0.125, -3.0)) + self.assertEqual(F(3, 2) ** SymbolicComplex('X'), SymbolicComplex('3/2 ** X')) + self.assertEqual(SymbolicComplex('X') ** F(3, 2), SymbolicComplex('X ** 1.5')) + + self.assertEqual(F(3, 2) ** Symbolic('X'), Symbolic('3/2 ** X')) + self.assertEqual(Symbolic('X') ** F(3, 2), Symbolic('X ** 1.5')) + def testMixingWithDecimal(self): # Decimal refuses mixed arithmetic (but not mixed comparisons) self.assertRaises(TypeError, operator.add, @@ -1341,6 +1702,12 @@ def test_complex_handling(self): message % ("divmod()", "complex", "Fraction"), divmod, b, a) + def test_three_argument_pow(self): + message = "unsupported operand type(s) for ** or pow(): '%s', '%s', '%s'" + self.assertRaisesMessage(TypeError, + message % ("Fraction", "int", "int"), + pow, F(3), 4, 5) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index f8812c281c2deb..7bd13eada8fedf 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -1,10 +1,9 @@ -import gc +import copy import operator import re import sys import textwrap import threading -import types import unittest import weakref try: @@ -12,9 +11,11 @@ except ImportError: _testcapi = None +from collections.abc import Mapping from test import support -from test.support import threading_helper, Py_GIL_DISABLED +from test.support import import_helper, threading_helper from test.support.script_helper import assert_python_ok +from test import mapping_tests class ClearTest(unittest.TestCase): @@ -198,14 +199,6 @@ def inner(): tb = tb.tb_next return frames - def test_locals(self): - f, outer, inner = self.make_frames() - outer_locals = outer.f_locals - self.assertIsInstance(outer_locals.pop('inner'), types.FunctionType) - self.assertEqual(outer_locals, {'x': 5, 'y': 6}) - inner_locals = inner.f_locals - self.assertEqual(inner_locals, {'x': 5, 'z': 7}) - def test_clear_locals(self): # Test f_locals after clear() (issue #21897) f, outer, inner = self.make_frames() @@ -217,8 +210,8 @@ def test_clear_locals(self): def test_locals_clear_locals(self): # Test f_locals before and after clear() (to exercise caching) f, outer, inner = self.make_frames() - outer.f_locals - inner.f_locals + self.assertNotEqual(outer.f_locals, {}) + self.assertNotEqual(inner.f_locals, {}) outer.clear() inner.clear() self.assertEqual(outer.f_locals, {}) @@ -229,6 +222,56 @@ def test_f_lineno_del_segfault(self): with self.assertRaises(AttributeError): del f.f_lineno + def test_f_generator(self): + # Test f_generator in different contexts. + + def t0(): + def nested(): + frame = sys._getframe() + return frame.f_generator + + def gen(): + yield nested() + + g = gen() + try: + return next(g) + finally: + g.close() + + def t1(): + frame = sys._getframe() + return frame.f_generator + + def t2(): + frame = sys._getframe() + yield frame.f_generator + + async def t3(): + frame = sys._getframe() + return frame.f_generator + + # For regular functions f_generator is None + self.assertIsNone(t0()) + self.assertIsNone(t1()) + + # For generators f_generator is equal to self + g = t2() + try: + frame_g = next(g) + self.assertIs(g, frame_g) + finally: + g.close() + + # Ditto for coroutines + c = t3() + try: + c.send(None) + except StopIteration as ex: + self.assertIs(ex.value, c) + else: + raise AssertionError('coroutine did not exit') + class ReprTest(unittest.TestCase): """ @@ -269,6 +312,382 @@ def inner(): r"^$" % (file_repr, offset + 5)) +class TestFrameLocals(unittest.TestCase): + def test_scope(self): + class A: + x = 1 + sys._getframe().f_locals['x'] = 2 + sys._getframe().f_locals['y'] = 2 + + self.assertEqual(A.x, 2) + self.assertEqual(A.y, 2) + + def f(): + x = 1 + sys._getframe().f_locals['x'] = 2 + sys._getframe().f_locals['y'] = 2 + self.assertEqual(x, 2) + self.assertEqual(locals()['y'], 2) + f() + + def test_closure(self): + x = 1 + y = 2 + + def f(): + z = x + y + d = sys._getframe().f_locals + self.assertEqual(d['x'], 1) + self.assertEqual(d['y'], 2) + d['x'] = 2 + d['y'] = 3 + + f() + self.assertEqual(x, 2) + self.assertEqual(y, 3) + + def test_as_dict(self): + x = 1 + y = 2 + d = sys._getframe().f_locals + # self, x, y, d + self.assertEqual(len(d), 4) + self.assertIs(d['d'], d) + self.assertEqual(set(d.keys()), set(['x', 'y', 'd', 'self'])) + self.assertEqual(len(d.values()), 4) + self.assertIn(1, d.values()) + self.assertEqual(len(d.items()), 4) + self.assertIn(('x', 1), d.items()) + self.assertEqual(d.__getitem__('x'), 1) + d.__setitem__('x', 2) + self.assertEqual(d['x'], 2) + self.assertEqual(d.get('x'), 2) + self.assertIs(d.get('non_exist', None), None) + self.assertEqual(d.__len__(), 4) + self.assertEqual(set([key for key in d]), set(['x', 'y', 'd', 'self'])) + self.assertIn('x', d) + self.assertTrue(d.__contains__('x')) + + self.assertEqual(reversed(d), list(reversed(d.keys()))) + + d.update({'x': 3, 'z': 4}) + self.assertEqual(d['x'], 3) + self.assertEqual(d['z'], 4) + + with self.assertRaises(TypeError): + d.update([1, 2]) + + self.assertEqual(d.setdefault('x', 5), 3) + self.assertEqual(d.setdefault('new', 5), 5) + self.assertEqual(d['new'], 5) + + with self.assertRaises(KeyError): + d['non_exist'] + + def test_as_number(self): + x = 1 + y = 2 + d = sys._getframe().f_locals + self.assertIn('z', d | {'z': 3}) + d |= {'z': 3} + self.assertEqual(d['z'], 3) + d |= {'y': 3} + self.assertEqual(d['y'], 3) + with self.assertRaises(TypeError): + d |= 3 + with self.assertRaises(TypeError): + _ = d | [3] + + def test_non_string_key(self): + d = sys._getframe().f_locals + d[1] = 2 + self.assertEqual(d[1], 2) + + def test_write_with_hidden(self): + def f(): + f_locals = [sys._getframe().f_locals for b in [0]][0] + f_locals['b'] = 2 + f_locals['c'] = 3 + self.assertEqual(b, 2) + self.assertEqual(c, 3) + b = 0 + c = 0 + f() + + def test_local_objects(self): + o = object() + k = '.'.join(['a', 'b', 'c']) + f_locals = sys._getframe().f_locals + f_locals['o'] = f_locals['k'] + self.assertEqual(o, 'a.b.c') + + def test_copy(self): + x = 0 + d = sys._getframe().f_locals + d_copy = d.copy() + self.assertIsInstance(d_copy, dict) + self.assertEqual(d_copy['x'], 0) + d_copy['x'] = 1 + self.assertEqual(x, 0) + + def test_update_with_self(self): + def f(): + f_locals = sys._getframe().f_locals + f_locals.update(f_locals) + f_locals.update(f_locals) + f_locals.update(f_locals) + f() + + def test_repr(self): + x = 1 + # Introduce a reference cycle + frame = sys._getframe() + self.assertEqual(repr(frame.f_locals), repr(dict(frame.f_locals))) + + def test_delete(self): + x = 1 + d = sys._getframe().f_locals + + # This needs to be tested before f_extra_locals is created + with self.assertRaisesRegex(KeyError, 'non_exist'): + del d['non_exist'] + + with self.assertRaises(KeyError): + d.pop('non_exist') + + with self.assertRaisesRegex(ValueError, 'local variables'): + del d['x'] + + with self.assertRaises(AttributeError): + d.clear() + + with self.assertRaises(ValueError): + d.pop('x') + + with self.assertRaises(ValueError): + d.pop('x', None) + + # 'm', 'n' is stored in f_extra_locals + d['m'] = 1 + d['n'] = 1 + + with self.assertRaises(KeyError): + d.pop('non_exist') + + del d['m'] + self.assertEqual(d.pop('n'), 1) + + self.assertNotIn('m', d) + self.assertNotIn('n', d) + + self.assertEqual(d.pop('n', 2), 2) + + @support.cpython_only + def test_sizeof(self): + proxy = sys._getframe().f_locals + support.check_sizeof(self, proxy, support.calcobjsize("P")) + + def test_unsupport(self): + x = 1 + d = sys._getframe().f_locals + with self.assertRaises(TypeError): + copy.copy(d) + + with self.assertRaises(TypeError): + copy.deepcopy(d) + + def test_is_mapping(self): + x = 1 + d = sys._getframe().f_locals + self.assertIsInstance(d, Mapping) + match d: + case {"x": value}: + self.assertEqual(value, 1) + kind = "mapping" + case _: + kind = "other" + self.assertEqual(kind, "mapping") + + def _x_stringlikes(self): + class StringSubclass(str): + pass + + class ImpostorX: + def __hash__(self): + return hash('x') + + def __eq__(self, other): + return other == 'x' + + return StringSubclass('x'), ImpostorX(), 'x' + + def test_proxy_key_stringlikes_overwrite(self): + def f(obj): + x = 1 + proxy = sys._getframe().f_locals + proxy[obj] = 2 + return ( + list(proxy.keys()), + dict(proxy), + proxy + ) + + for obj in self._x_stringlikes(): + with self.subTest(cls=type(obj).__name__): + + keys_snapshot, proxy_snapshot, proxy = f(obj) + expected_keys = ['obj', 'x', 'proxy'] + expected_dict = {'obj': 'x', 'x': 2, 'proxy': proxy} + self.assertEqual(proxy.keys(), expected_keys) + self.assertEqual(proxy, expected_dict) + self.assertEqual(keys_snapshot, expected_keys) + self.assertEqual(proxy_snapshot, expected_dict) + + def test_proxy_key_stringlikes_ftrst_write(self): + def f(obj): + proxy = sys._getframe().f_locals + proxy[obj] = 2 + self.assertEqual(x, 2) + x = 1 + + for obj in self._x_stringlikes(): + with self.subTest(cls=type(obj).__name__): + f(obj) + + def test_proxy_key_unhashables(self): + class StringSubclass(str): + __hash__ = None + + class ObjectSubclass: + __hash__ = None + + proxy = sys._getframe().f_locals + + for obj in StringSubclass('x'), ObjectSubclass(): + with self.subTest(cls=type(obj).__name__): + with self.assertRaises(TypeError): + proxy[obj] + with self.assertRaises(TypeError): + proxy[obj] = 0 + + def test_constructor(self): + FrameLocalsProxy = type([sys._getframe().f_locals + for x in range(1)][0]) + self.assertEqual(FrameLocalsProxy.__name__, 'FrameLocalsProxy') + + def make_frame(): + x = 1 + y = 2 + return sys._getframe() + + proxy = FrameLocalsProxy(make_frame()) + self.assertEqual(proxy, {'x': 1, 'y': 2}) + + # constructor expects 1 frame argument + with self.assertRaises(TypeError): + FrameLocalsProxy() # no arguments + with self.assertRaises(TypeError): + FrameLocalsProxy(123) # wrong type + with self.assertRaises(TypeError): + FrameLocalsProxy(frame=sys._getframe()) # no keyword arguments + + +class FrameLocalsProxyMappingTests(mapping_tests.TestHashMappingProtocol): + """Test that FrameLocalsProxy behaves like a Mapping (with exceptions)""" + + def _f(*args, **kwargs): + def _f(): + return sys._getframe().f_locals + return _f() + type2test = _f + + @unittest.skipIf(True, 'Locals proxies for different frames never compare as equal') + def test_constructor(self): + pass + + @unittest.skipIf(True, 'Unlike a mapping: del proxy[key] fails') + def test_write(self): + pass + + @unittest.skipIf(True, 'Unlike a mapping: no proxy.popitem') + def test_popitem(self): + pass + + @unittest.skipIf(True, 'Unlike a mapping: no proxy.pop') + def test_pop(self): + pass + + @unittest.skipIf(True, 'Unlike a mapping: no proxy.clear') + def test_clear(self): + pass + + @unittest.skipIf(True, 'Unlike a mapping: no proxy.fromkeys') + def test_fromkeys(self): + pass + + # no del + def test_getitem(self): + mapping_tests.BasicTestMappingProtocol.test_getitem(self) + d = self._full_mapping({'a': 1, 'b': 2}) + self.assertEqual(d['a'], 1) + self.assertEqual(d['b'], 2) + d['c'] = 3 + d['a'] = 4 + self.assertEqual(d['c'], 3) + self.assertEqual(d['a'], 4) + + @unittest.skipIf(True, 'Unlike a mapping: no proxy.update') + def test_update(self): + pass + + # proxy.copy returns a regular dict + def test_copy(self): + d = self._full_mapping({1:1, 2:2, 3:3}) + self.assertEqual(d.copy(), {1:1, 2:2, 3:3}) + d = self._empty_mapping() + self.assertEqual(d.copy(), d) + self.assertRaises(TypeError, d.copy, None) + + self.assertIsInstance(d.copy(), dict) + + @unittest.skipIf(True, 'Locals proxies for different frames never compare as equal') + def test_eq(self): + pass + + +class TestFrameCApi(unittest.TestCase): + def test_basic(self): + x = 1 + ctypes = import_helper.import_module('ctypes') + PyEval_GetFrameLocals = ctypes.pythonapi.PyEval_GetFrameLocals + PyEval_GetFrameLocals.restype = ctypes.py_object + frame_locals = PyEval_GetFrameLocals() + self.assertTrue(type(frame_locals), dict) + self.assertEqual(frame_locals['x'], 1) + frame_locals['x'] = 2 + self.assertEqual(x, 1) + + PyEval_GetFrameGlobals = ctypes.pythonapi.PyEval_GetFrameGlobals + PyEval_GetFrameGlobals.restype = ctypes.py_object + frame_globals = PyEval_GetFrameGlobals() + self.assertTrue(type(frame_globals), dict) + self.assertIs(frame_globals, globals()) + + PyEval_GetFrameBuiltins = ctypes.pythonapi.PyEval_GetFrameBuiltins + PyEval_GetFrameBuiltins.restype = ctypes.py_object + frame_builtins = PyEval_GetFrameBuiltins() + self.assertEqual(frame_builtins, __builtins__) + + PyFrame_GetLocals = ctypes.pythonapi.PyFrame_GetLocals + PyFrame_GetLocals.argtypes = [ctypes.py_object] + PyFrame_GetLocals.restype = ctypes.py_object + frame = sys._getframe() + f_locals = PyFrame_GetLocals(frame) + self.assertTrue(f_locals['x'], 1) + f_locals['x'] = 2 + self.assertEqual(x, 2) + + class TestIncompleteFrameAreInvisible(unittest.TestCase): def test_issue95818(self): @@ -293,71 +712,6 @@ def gen(): """) assert_python_ok("-c", code) - @support.cpython_only - @unittest.skipIf(Py_GIL_DISABLED, "test requires precise GC scheduling") - def test_sneaky_frame_object(self): - - def trace(frame, event, arg): - """ - Don't actually do anything, just force a frame object to be created. - """ - - def callback(phase, info): - """ - Yo dawg, I heard you like frames, so I'm allocating a frame while - you're allocating a frame, so you can have a frame while you have a - frame! - """ - nonlocal sneaky_frame_object - sneaky_frame_object = sys._getframe().f_back.f_back - # We're done here: - gc.callbacks.remove(callback) - - def f(): - while True: - yield - - old_threshold = gc.get_threshold() - old_callbacks = gc.callbacks[:] - old_enabled = gc.isenabled() - old_trace = sys.gettrace() - try: - # Stop the GC for a second while we set things up: - gc.disable() - # Create a paused generator: - g = f() - next(g) - # Move all objects to the oldest generation, and tell the GC to run - # on the *very next* allocation: - gc.collect() - gc.set_threshold(1, 0, 0) - sys._clear_internal_caches() - # Okay, so here's the nightmare scenario: - # - We're tracing the resumption of a generator, which creates a new - # frame object. - # - The allocation of this frame object triggers a collection - # *before* the frame object is actually created. - # - During the collection, we request the exact same frame object. - # This test does it with a GC callback, but in real code it would - # likely be a trace function, weakref callback, or finalizer. - # - The collection finishes, and the original frame object is - # created. We now have two frame objects fighting over ownership - # of the same interpreter frame! - sys.settrace(trace) - gc.callbacks.append(callback) - sneaky_frame_object = None - gc.enable() - next(g) - # g.gi_frame should be the frame object from the callback (the - # one that was *requested* second, but *created* first): - self.assertIs(g.gi_frame, sneaky_frame_object) - finally: - gc.set_threshold(*old_threshold) - gc.callbacks[:] = old_callbacks - sys.settrace(old_trace) - if old_enabled: - gc.enable() - @support.cpython_only @threading_helper.requires_working_threading() def test_sneaky_frame_object_teardown(self): diff --git a/Lib/test/test_free_threading/__init__.py b/Lib/test/test_free_threading/__init__.py new file mode 100644 index 00000000000000..34e1ad30e11097 --- /dev/null +++ b/Lib/test/test_free_threading/__init__.py @@ -0,0 +1,11 @@ +import os +import unittest + +from test import support + + +if not support.Py_GIL_DISABLED: + raise unittest.SkipTest("GIL enabled") + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_free_threading/test_code.py b/Lib/test/test_free_threading/test_code.py new file mode 100644 index 00000000000000..a5136a3ba4edc7 --- /dev/null +++ b/Lib/test/test_free_threading/test_code.py @@ -0,0 +1,30 @@ +import unittest + +from threading import Thread +from unittest import TestCase + +from test.support import threading_helper + +@threading_helper.requires_working_threading() +class TestCode(TestCase): + def test_code_attrs(self): + """Test concurrent accesses to lazily initialized code attributes""" + code_objects = [] + for _ in range(1000): + code_objects.append(compile("a + b", "", "eval")) + + def run_in_thread(): + for code in code_objects: + self.assertIsInstance(code.co_code, bytes) + self.assertIsInstance(code.co_freevars, tuple) + self.assertIsInstance(code.co_varnames, tuple) + + threads = [Thread(target=run_in_thread) for _ in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_dict.py b/Lib/test/test_free_threading/test_dict.py new file mode 100644 index 00000000000000..4f605e0c51f0d5 --- /dev/null +++ b/Lib/test/test_free_threading/test_dict.py @@ -0,0 +1,232 @@ +import gc +import time +import unittest +import weakref + +from ast import Or +from functools import partial +from threading import Barrier, Thread +from unittest import TestCase + +try: + import _testcapi +except ImportError: + _testcapi = None + +from test.support import threading_helper + + +@threading_helper.requires_working_threading() +class TestDict(TestCase): + def test_racing_creation_shared_keys(self): + """Verify that creating dictionaries is thread safe when we + have a type with shared keys""" + class C(int): + pass + + self.racing_creation(C) + + def test_racing_creation_no_shared_keys(self): + """Verify that creating dictionaries is thread safe when we + have a type with an ordinary dict""" + self.racing_creation(Or) + + def test_racing_creation_inline_values_invalid(self): + """Verify that re-creating a dict after we have invalid inline values + is thread safe""" + class C: + pass + + def make_obj(): + a = C() + # Make object, make inline values invalid, and then delete dict + a.__dict__ = {} + del a.__dict__ + return a + + self.racing_creation(make_obj) + + def test_racing_creation_nonmanaged_dict(self): + """Verify that explicit creation of an unmanaged dict is thread safe + outside of the normal attribute setting code path""" + def make_obj(): + def f(): pass + return f + + def set(func, name, val): + # Force creation of the dict via PyObject_GenericGetDict + func.__dict__[name] = val + + self.racing_creation(make_obj, set) + + def racing_creation(self, cls, set=setattr): + objects = [] + processed = [] + + OBJECT_COUNT = 100 + THREAD_COUNT = 10 + CUR = 0 + + for i in range(OBJECT_COUNT): + objects.append(cls()) + + def writer_func(name): + last = -1 + while True: + if CUR == last: + continue + elif CUR == OBJECT_COUNT: + break + + obj = objects[CUR] + set(obj, name, name) + last = CUR + processed.append(name) + + writers = [] + for x in range(THREAD_COUNT): + writer = Thread(target=partial(writer_func, f"a{x:02}")) + writers.append(writer) + writer.start() + + for i in range(OBJECT_COUNT): + CUR = i + while len(processed) != THREAD_COUNT: + time.sleep(0.001) + processed.clear() + + CUR = OBJECT_COUNT + + for writer in writers: + writer.join() + + for obj_idx, obj in enumerate(objects): + assert ( + len(obj.__dict__) == THREAD_COUNT + ), f"{len(obj.__dict__)} {obj.__dict__!r} {obj_idx}" + for i in range(THREAD_COUNT): + assert f"a{i:02}" in obj.__dict__, f"a{i:02} missing at {obj_idx}" + + def test_racing_set_dict(self): + """Races assigning to __dict__ should be thread safe""" + + def f(): pass + l = [] + THREAD_COUNT = 10 + class MyDict(dict): pass + + def writer_func(l): + for i in range(1000): + d = MyDict() + l.append(weakref.ref(d)) + f.__dict__ = d + + lists = [] + writers = [] + for x in range(THREAD_COUNT): + thread_list = [] + lists.append(thread_list) + writer = Thread(target=partial(writer_func, thread_list)) + writers.append(writer) + + for writer in writers: + writer.start() + + for writer in writers: + writer.join() + + f.__dict__ = {} + gc.collect() + + for thread_list in lists: + for ref in thread_list: + self.assertIsNone(ref()) + + def test_racing_get_set_dict(self): + """Races getting and setting a dict should be thread safe""" + THREAD_COUNT = 10 + barrier = Barrier(THREAD_COUNT) + def work(d): + barrier.wait() + for _ in range(1000): + d[10] = 0 + d.get(10, None) + _ = d[10] + + d = {} + worker_threads = [] + for ii in range(THREAD_COUNT): + worker_threads.append(Thread(target=work, args=[d])) + for t in worker_threads: + t.start() + for t in worker_threads: + t.join() + + + def test_racing_set_object_dict(self): + """Races assigning to __dict__ should be thread safe""" + class C: pass + class MyDict(dict): pass + for cyclic in (False, True): + f = C() + f.__dict__ = {"foo": 42} + THREAD_COUNT = 10 + + def writer_func(l): + for i in range(1000): + if cyclic: + other_d = {} + d = MyDict({"foo": 100}) + if cyclic: + d["x"] = other_d + other_d["bar"] = d + l.append(weakref.ref(d)) + f.__dict__ = d + + def reader_func(): + for i in range(1000): + f.foo + + lists = [] + readers = [] + writers = [] + for x in range(THREAD_COUNT): + thread_list = [] + lists.append(thread_list) + writer = Thread(target=partial(writer_func, thread_list)) + writers.append(writer) + + for x in range(THREAD_COUNT): + reader = Thread(target=partial(reader_func)) + readers.append(reader) + + for writer in writers: + writer.start() + for reader in readers: + reader.start() + + for writer in writers: + writer.join() + + for reader in readers: + reader.join() + + f.__dict__ = {} + gc.collect() + gc.collect() + + count = 0 + ids = set() + for thread_list in lists: + for i, ref in enumerate(thread_list): + if ref() is None: + continue + count += 1 + ids.add(id(ref())) + count += 1 + + self.assertEqual(count, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_gc.py b/Lib/test/test_free_threading/test_gc.py new file mode 100644 index 00000000000000..401067fe9c612c --- /dev/null +++ b/Lib/test/test_free_threading/test_gc.py @@ -0,0 +1,61 @@ +import unittest + +import threading +from threading import Thread +from unittest import TestCase +import gc + +from test.support import threading_helper + + +class MyObj: + pass + + +@threading_helper.requires_working_threading() +class TestGC(TestCase): + def test_get_objects(self): + event = threading.Event() + + def gc_thread(): + for i in range(100): + o = gc.get_objects() + event.set() + + def mutator_thread(): + while not event.is_set(): + o1 = MyObj() + o2 = MyObj() + o3 = MyObj() + o4 = MyObj() + + gcs = [Thread(target=gc_thread)] + mutators = [Thread(target=mutator_thread) for _ in range(4)] + with threading_helper.start_threads(gcs + mutators): + pass + + def test_get_referrers(self): + event = threading.Event() + + obj = MyObj() + + def gc_thread(): + for i in range(100): + o = gc.get_referrers(obj) + event.set() + + def mutator_thread(): + while not event.is_set(): + d1 = { "key": obj } + d2 = { "key": obj } + d3 = { "key": obj } + d4 = { "key": obj } + + gcs = [Thread(target=gc_thread) for _ in range(2)] + mutators = [Thread(target=mutator_thread) for _ in range(4)] + with threading_helper.start_threads(gcs + mutators): + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_list.py b/Lib/test/test_free_threading/test_list.py new file mode 100644 index 00000000000000..a705161369e8dd --- /dev/null +++ b/Lib/test/test_free_threading/test_list.py @@ -0,0 +1,76 @@ +import unittest + +from threading import Thread +from unittest import TestCase + +from test.support import threading_helper + + +NTHREAD = 10 +OBJECT_COUNT = 5_000 + + +class C: + def __init__(self, v): + self.v = v + + +@threading_helper.requires_working_threading() +class TestList(TestCase): + def test_racing_iter_append(self): + l = [] + + def writer_func(): + for i in range(OBJECT_COUNT): + l.append(C(i + OBJECT_COUNT)) + + def reader_func(): + while True: + count = len(l) + for i, x in enumerate(l): + self.assertEqual(x.v, i + OBJECT_COUNT) + if count == OBJECT_COUNT: + break + + writer = Thread(target=writer_func) + readers = [] + for x in range(NTHREAD): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + + def test_racing_iter_extend(self): + l = [] + + def writer_func(): + for i in range(OBJECT_COUNT): + l.extend([C(i + OBJECT_COUNT)]) + + def reader_func(): + while True: + count = len(l) + for i, x in enumerate(l): + self.assertEqual(x.v, i + OBJECT_COUNT) + if count == OBJECT_COUNT: + break + + writer = Thread(target=writer_func) + readers = [] + for x in range(NTHREAD): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_methodcaller.py b/Lib/test/test_free_threading/test_methodcaller.py new file mode 100644 index 00000000000000..8846b0010012f2 --- /dev/null +++ b/Lib/test/test_free_threading/test_methodcaller.py @@ -0,0 +1,33 @@ +import unittest +from threading import Thread +from test.support import threading_helper +from operator import methodcaller + + +class TestMethodcaller(unittest.TestCase): + def test_methodcaller_threading(self): + number_of_threads = 10 + size = 4_000 + + mc = methodcaller("append", 2) + + def work(mc, l, ii): + for _ in range(ii): + mc(l) + + worker_threads = [] + lists = [] + for ii in range(number_of_threads): + l = [] + lists.append(l) + worker_threads.append(Thread(target=work, args=[mc, l, size])) + for t in worker_threads: + t.start() + for t in worker_threads: + t.join() + for l in lists: + assert len(l) == size + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_monitoring.py b/Lib/test/test_free_threading/test_monitoring.py new file mode 100644 index 00000000000000..8fec01715531cb --- /dev/null +++ b/Lib/test/test_free_threading/test_monitoring.py @@ -0,0 +1,248 @@ +"""Tests monitoring, sys.settrace, and sys.setprofile in a multi-threaded +environment to verify things are thread-safe in a free-threaded build""" + +import sys +import time +import unittest +import weakref + +from sys import monitoring +from test.support import threading_helper +from threading import Thread, _PyRLock +from unittest import TestCase + + +class InstrumentationMultiThreadedMixin: + thread_count = 10 + func_count = 50 + fib = 12 + + def after_threads(self): + """Runs once after all the threads have started""" + pass + + def during_threads(self): + """Runs repeatedly while the threads are still running""" + pass + + def work(self, n, funcs): + """Fibonacci function which also calls a bunch of random functions""" + for func in funcs: + func() + if n < 2: + return n + return self.work(n - 1, funcs) + self.work(n - 2, funcs) + + def start_work(self, n, funcs): + # With the GIL builds we need to make sure that the hooks have + # a chance to run as it's possible to run w/o releasing the GIL. + time.sleep(0.1) + self.work(n, funcs) + + def after_test(self): + """Runs once after the test is done""" + pass + + def test_instrumentation(self): + # Setup a bunch of functions which will need instrumentation... + funcs = [] + for i in range(self.func_count): + x = {} + exec("def f(): pass", x) + funcs.append(x["f"]) + + threads = [] + for i in range(self.thread_count): + # Each thread gets a copy of the func list to avoid contention + t = Thread(target=self.start_work, args=(self.fib, list(funcs))) + t.start() + threads.append(t) + + self.after_threads() + + while True: + any_alive = False + for t in threads: + if t.is_alive(): + any_alive = True + break + + if not any_alive: + break + + self.during_threads() + + self.after_test() + + +class MonitoringTestMixin: + def setUp(self): + for i in range(6): + if monitoring.get_tool(i) is None: + self.tool_id = i + monitoring.use_tool_id(i, self.__class__.__name__) + break + + def tearDown(self): + monitoring.free_tool_id(self.tool_id) + + +@threading_helper.requires_working_threading() +class SetPreTraceMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): + """Sets tracing one time after the threads have started""" + + def setUp(self): + super().setUp() + self.called = False + + def after_test(self): + self.assertTrue(self.called) + + def trace_func(self, frame, event, arg): + self.called = True + return self.trace_func + + def after_threads(self): + sys.settrace(self.trace_func) + + +@threading_helper.requires_working_threading() +class MonitoringMultiThreaded( + MonitoringTestMixin, InstrumentationMultiThreadedMixin, TestCase +): + """Uses sys.monitoring and repeatedly toggles instrumentation on and off""" + + def setUp(self): + super().setUp() + self.set = False + self.called = False + monitoring.register_callback( + self.tool_id, monitoring.events.LINE, self.callback + ) + + def tearDown(self): + monitoring.set_events(self.tool_id, 0) + super().tearDown() + + def callback(self, *args): + self.called = True + + def after_test(self): + self.assertTrue(self.called) + + def during_threads(self): + if self.set: + monitoring.set_events( + self.tool_id, monitoring.events.CALL | monitoring.events.LINE + ) + else: + monitoring.set_events(self.tool_id, 0) + self.set = not self.set + + +@threading_helper.requires_working_threading() +class SetTraceMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): + """Uses sys.settrace and repeatedly toggles instrumentation on and off""" + + def setUp(self): + self.set = False + self.called = False + + def after_test(self): + self.assertTrue(self.called) + + def tearDown(self): + sys.settrace(None) + + def trace_func(self, frame, event, arg): + self.called = True + return self.trace_func + + def during_threads(self): + if self.set: + sys.settrace(self.trace_func) + else: + sys.settrace(None) + self.set = not self.set + + +@threading_helper.requires_working_threading() +class SetProfileMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): + """Uses sys.setprofile and repeatedly toggles instrumentation on and off""" + + def setUp(self): + self.set = False + self.called = False + + def after_test(self): + self.assertTrue(self.called) + + def tearDown(self): + sys.setprofile(None) + + def trace_func(self, frame, event, arg): + self.called = True + return self.trace_func + + def during_threads(self): + if self.set: + sys.setprofile(self.trace_func) + else: + sys.setprofile(None) + self.set = not self.set + + +@threading_helper.requires_working_threading() +class MonitoringMisc(MonitoringTestMixin, TestCase): + def register_callback(self): + def callback(*args): + pass + + for i in range(200): + monitoring.register_callback(self.tool_id, monitoring.events.LINE, callback) + + self.refs.append(weakref.ref(callback)) + + def test_register_callback(self): + self.refs = [] + threads = [] + for i in range(50): + t = Thread(target=self.register_callback) + t.start() + threads.append(t) + + for thread in threads: + thread.join() + + monitoring.register_callback(self.tool_id, monitoring.events.LINE, None) + for ref in self.refs: + self.assertEqual(ref(), None) + + def test_set_local_trace_opcodes(self): + def trace(frame, event, arg): + frame.f_trace_opcodes = True + return trace + + loops = 1_000 + + sys.settrace(trace) + try: + l = _PyRLock() + + def f(): + for i in range(loops): + with l: + pass + + t = Thread(target=f) + t.start() + for i in range(loops): + with l: + pass + t.join() + finally: + sys.settrace(None) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_races.py b/Lib/test/test_free_threading/test_races.py new file mode 100644 index 00000000000000..85aa69c8cd494f --- /dev/null +++ b/Lib/test/test_free_threading/test_races.py @@ -0,0 +1,290 @@ +# It's most useful to run these tests with ThreadSanitizer enabled. +import sys +import functools +import threading +import time +import unittest +import _testinternalcapi + +from test.support import threading_helper + + +class TestBase(unittest.TestCase): + pass + + +def do_race(func1, func2): + """Run func1() and func2() repeatedly in separate threads.""" + n = 1000 + + barrier = threading.Barrier(2) + + def repeat(func): + barrier.wait() + for _i in range(n): + func() + + threads = [ + threading.Thread(target=functools.partial(repeat, func1)), + threading.Thread(target=functools.partial(repeat, func2)), + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + +@threading_helper.requires_working_threading() +class TestRaces(TestBase): + def test_racing_cell_set(self): + """Test cell object gettr/settr properties.""" + + def nested_func(): + x = 0 + + def inner(): + nonlocal x + x += 1 + + # This doesn't race because LOAD_DEREF and STORE_DEREF on the + # cell object use critical sections. + do_race(nested_func, nested_func) + + def nested_func2(): + x = 0 + + def inner(): + y = x + frame = sys._getframe(1) + frame.f_locals["x"] = 2 + + return inner + + def mutate_func2(): + inner = nested_func2() + cell = inner.__closure__[0] + old_value = cell.cell_contents + cell.cell_contents = 1000 + time.sleep(0) + cell.cell_contents = old_value + time.sleep(0) + + # This revealed a race with cell_set_contents() since it was missing + # the critical section. + do_race(nested_func2, mutate_func2) + + def test_racing_cell_cmp_repr(self): + """Test cell object compare and repr methods.""" + + def nested_func(): + x = 0 + y = 0 + + def inner(): + return x + y + + return inner.__closure__ + + cell_a, cell_b = nested_func() + + def mutate(): + cell_a.cell_contents += 1 + + def access(): + cell_a == cell_b + s = repr(cell_a) + + # cell_richcompare() and cell_repr used to have data races + do_race(mutate, access) + + def test_racing_load_super_attr(self): + """Test (un)specialization of LOAD_SUPER_ATTR opcode.""" + + class C: + def __init__(self): + try: + super().__init__ + super().__init__() + except RuntimeError: + pass # happens if __class__ is replaced with non-type + + def access(): + C() + + def mutate(): + # Swap out the super() global with a different one + real_super = super + globals()["super"] = lambda s=1: s + time.sleep(0) + globals()["super"] = real_super + time.sleep(0) + # Swap out the __class__ closure value with a non-type + cell = C.__init__.__closure__[0] + real_class = cell.cell_contents + cell.cell_contents = 99 + time.sleep(0) + cell.cell_contents = real_class + + # The initial PR adding specialized opcodes for LOAD_SUPER_ATTR + # had some races (one with the super() global changing and one + # with the cell binding being changed). + do_race(access, mutate) + + def test_racing_to_bool(self): + + seq = [1] + + class C: + def __bool__(self): + return False + + def access(): + if seq: + return 1 + else: + return 2 + + def mutate(): + nonlocal seq + seq = [1] + time.sleep(0) + seq = C() + time.sleep(0) + + do_race(access, mutate) + + def test_racing_store_attr_slot(self): + class C: + __slots__ = ['x', '__dict__'] + + c = C() + + def set_slot(): + for i in range(10): + c.x = i + time.sleep(0) + + def change_type(): + def set_x(self, x): + pass + + def get_x(self): + pass + + C.x = property(get_x, set_x) + time.sleep(0) + del C.x + time.sleep(0) + + do_race(set_slot, change_type) + + def set_getattribute(): + C.__getattribute__ = lambda self, x: x + time.sleep(0) + del C.__getattribute__ + time.sleep(0) + + do_race(set_slot, set_getattribute) + + def test_racing_store_attr_instance_value(self): + class C: + pass + + c = C() + + def set_value(): + for i in range(100): + c.x = i + + set_value() + + def read(): + x = c.x + + def mutate(): + # Adding a property for 'x' should unspecialize it. + C.x = property(lambda self: None, lambda self, x: None) + time.sleep(0) + del C.x + time.sleep(0) + + do_race(read, mutate) + + def test_racing_store_attr_with_hint(self): + class C: + pass + + c = C() + for i in range(29): + setattr(c, f"_{i}", None) + + def set_value(): + for i in range(100): + c.x = i + + set_value() + + def read(): + x = c.x + + def mutate(): + # Adding a property for 'x' should unspecialize it. + C.x = property(lambda self: None, lambda self, x: None) + time.sleep(0) + del C.x + time.sleep(0) + + do_race(read, mutate) + + def make_shared_key_dict(self): + class C: + pass + + a = C() + a.x = 1 + return a.__dict__ + + def test_racing_store_attr_dict(self): + """Test STORE_ATTR with various dictionary types.""" + class C: + pass + + c = C() + + def set_value(): + for i in range(20): + c.x = i + + def mutate(): + nonlocal c + c.x = 1 + self.assertTrue(_testinternalcapi.has_inline_values(c)) + for i in range(30): + setattr(c, f"_{i}", None) + self.assertFalse(_testinternalcapi.has_inline_values(c.__dict__)) + c.__dict__ = self.make_shared_key_dict() + self.assertTrue(_testinternalcapi.has_split_table(c.__dict__)) + c.__dict__[1] = None + self.assertFalse(_testinternalcapi.has_split_table(c.__dict__)) + c = C() + + do_race(set_value, mutate) + + def test_racing_recursion_limit(self): + def something_recursive(): + def count(n): + if n > 0: + return count(n - 1) + 1 + return 0 + + count(50) + + def set_recursion_limit(): + for limit in range(100, 200): + sys.setrecursionlimit(limit) + + do_race(something_recursive, set_recursion_limit) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_slots.py b/Lib/test/test_free_threading/test_slots.py new file mode 100644 index 00000000000000..a3b9f4b0175ae7 --- /dev/null +++ b/Lib/test/test_free_threading/test_slots.py @@ -0,0 +1,287 @@ +import _testcapi +import threading +from test.support import threading_helper +from unittest import TestCase + + +def run_in_threads(targets): + """Run `targets` in separate threads""" + threads = [ + threading.Thread(target=target) + for target in targets + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + +@threading_helper.requires_working_threading() +class TestSlots(TestCase): + + def test_object(self): + class Spam: + __slots__ = [ + "eggs", + ] + + def __init__(self, initial_value): + self.eggs = initial_value + + spam = Spam(0) + iters = 20_000 + + def writer(): + for _ in range(iters): + spam.eggs += 1 + + def reader(): + for _ in range(iters): + eggs = spam.eggs + assert type(eggs) is int + assert 0 <= eggs <= iters + + run_in_threads([writer, reader, reader, reader]) + + def test_T_BOOL(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + # different code paths for True and False + spam_old.T_BOOL = True + spam_new.T_BOOL = True + spam_old.T_BOOL = False + spam_new.T_BOOL = False + + def reader(): + for _ in range(1_000): + spam_old.T_BOOL + spam_new.T_BOOL + + run_in_threads([writer, reader]) + + def test_T_BYTE(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_BYTE = 0 + spam_new.T_BYTE = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_BYTE + spam_new.T_BYTE + + run_in_threads([writer, reader]) + + def test_T_UBYTE(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_UBYTE = 0 + spam_new.T_UBYTE = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_UBYTE + spam_new.T_UBYTE + + run_in_threads([writer, reader]) + + def test_T_SHORT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_SHORT = 0 + spam_new.T_SHORT = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_SHORT + spam_new.T_SHORT + + run_in_threads([writer, reader]) + + def test_T_USHORT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_USHORT = 0 + spam_new.T_USHORT = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_USHORT + spam_new.T_USHORT + + run_in_threads([writer, reader]) + + def test_T_INT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_INT = 0 + spam_new.T_INT = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_INT + spam_new.T_INT + + run_in_threads([writer, reader]) + + def test_T_UINT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_UINT = 0 + spam_new.T_UINT = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_UINT + spam_new.T_UINT + + run_in_threads([writer, reader]) + + def test_T_LONG(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_LONG = 0 + spam_new.T_LONG = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_LONG + spam_new.T_LONG + + run_in_threads([writer, reader]) + + def test_T_ULONG(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_ULONG = 0 + spam_new.T_ULONG = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_ULONG + spam_new.T_ULONG + + run_in_threads([writer, reader]) + + def test_T_PYSSIZET(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_PYSSIZET = 0 + spam_new.T_PYSSIZET = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_PYSSIZET + spam_new.T_PYSSIZET + + run_in_threads([writer, reader]) + + def test_T_FLOAT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_FLOAT = 0.0 + spam_new.T_FLOAT = 0.0 + + def reader(): + for _ in range(1_000): + spam_old.T_FLOAT + spam_new.T_FLOAT + + run_in_threads([writer, reader]) + + def test_T_DOUBLE(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_DOUBLE = 0.0 + spam_new.T_DOUBLE = 0.0 + + def reader(): + for _ in range(1_000): + spam_old.T_DOUBLE + spam_new.T_DOUBLE + + run_in_threads([writer, reader]) + + def test_T_LONGLONG(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_LONGLONG = 0 + spam_new.T_LONGLONG = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_LONGLONG + spam_new.T_LONGLONG + + run_in_threads([writer, reader]) + + def test_T_ULONGLONG(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_ULONGLONG = 0 + spam_new.T_ULONGLONG = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_ULONGLONG + spam_new.T_ULONGLONG + + run_in_threads([writer, reader]) + + def test_T_CHAR(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_CHAR = "c" + spam_new.T_CHAR = "c" + + def reader(): + for _ in range(1_000): + spam_old.T_CHAR + spam_new.T_CHAR + + run_in_threads([writer, reader]) diff --git a/Lib/test/test_free_threading/test_str.py b/Lib/test/test_free_threading/test_str.py new file mode 100644 index 00000000000000..72044e979b0f48 --- /dev/null +++ b/Lib/test/test_free_threading/test_str.py @@ -0,0 +1,74 @@ +import unittest + +from itertools import cycle +from threading import Event, Thread +from unittest import TestCase + +from test.support import threading_helper + +@threading_helper.requires_working_threading() +class TestStr(TestCase): + def test_racing_join_extend(self): + '''Test joining a string being extended by another thread''' + l = [] + ITERS = 100 + READERS = 10 + done_event = Event() + def writer_func(): + for i in range(ITERS): + l.extend(map(str, range(i))) + l.clear() + done_event.set() + def reader_func(): + while not done_event.is_set(): + ''.join(l) + writer = Thread(target=writer_func) + readers = [] + for x in range(READERS): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + + def test_racing_join_replace(self): + ''' + Test joining a string of characters being replaced with ephemeral + strings by another thread. + ''' + l = [*'abcdefg'] + MAX_ORDINAL = 1_000 + READERS = 10 + done_event = Event() + + def writer_func(): + for i, c in zip(cycle(range(len(l))), + map(chr, range(128, MAX_ORDINAL))): + l[i] = c + done_event.set() + + def reader_func(): + while not done_event.is_set(): + ''.join(l) + ''.join(l) + ''.join(l) + ''.join(l) + + writer = Thread(target=writer_func) + readers = [] + for x in range(READERS): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_tokenize.py b/Lib/test/test_free_threading/test_tokenize.py new file mode 100644 index 00000000000000..860cfec4d710f4 --- /dev/null +++ b/Lib/test/test_free_threading/test_tokenize.py @@ -0,0 +1,57 @@ +import io +import time +import unittest +import tokenize +from functools import partial +from threading import Thread + +from test.support import threading_helper + + +@threading_helper.requires_working_threading() +class TestTokenize(unittest.TestCase): + def test_tokenizer_iter(self): + source = io.StringIO("for _ in a:\n pass") + it = tokenize._tokenize.TokenizerIter(source.readline, extra_tokens=False) + + tokens = [] + def next_token(it): + while True: + try: + r = next(it) + tokens.append(tokenize.TokenInfo._make(r)) + time.sleep(0.03) + except StopIteration: + return + + threads = [] + for _ in range(5): + threads.append(Thread(target=partial(next_token, it))) + + for thread in threads: + thread.start() + + for thread in threads: + thread.join() + + expected_tokens = [ + tokenize.TokenInfo(type=1, string='for', start=(1, 0), end=(1, 3), line='for _ in a:\n'), + tokenize.TokenInfo(type=1, string='_', start=(1, 4), end=(1, 5), line='for _ in a:\n'), + tokenize.TokenInfo(type=1, string='in', start=(1, 6), end=(1, 8), line='for _ in a:\n'), + tokenize.TokenInfo(type=1, string='a', start=(1, 9), end=(1, 10), line='for _ in a:\n'), + tokenize.TokenInfo(type=11, string=':', start=(1, 10), end=(1, 11), line='for _ in a:\n'), + tokenize.TokenInfo(type=4, string='', start=(1, 11), end=(1, 11), line='for _ in a:\n'), + tokenize.TokenInfo(type=5, string='', start=(2, -1), end=(2, -1), line=' pass'), + tokenize.TokenInfo(type=1, string='pass', start=(2, 2), end=(2, 6), line=' pass'), + tokenize.TokenInfo(type=4, string='', start=(2, 6), end=(2, 6), line=' pass'), + tokenize.TokenInfo(type=6, string='', start=(2, -1), end=(2, -1), line=' pass'), + tokenize.TokenInfo(type=0, string='', start=(2, -1), end=(2, -1), line=' pass'), + ] + + tokens.sort() + expected_tokens.sort() + self.assertListEqual(tokens, expected_tokens) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py new file mode 100644 index 00000000000000..53f6d778bbecbc --- /dev/null +++ b/Lib/test/test_free_threading/test_type.py @@ -0,0 +1,156 @@ +import threading +import unittest + +from concurrent.futures import ThreadPoolExecutor +from threading import Thread +from unittest import TestCase + +from test.support import threading_helper + + + +NTHREADS = 6 +BOTTOM = 0 +TOP = 1000 +ITERS = 100 + +class A: + attr = 1 + +@threading_helper.requires_working_threading() +class TestType(TestCase): + def test_attr_cache(self): + def read(id0): + for _ in range(ITERS): + for _ in range(BOTTOM, TOP): + A.attr + + def write(id0): + for _ in range(ITERS): + for _ in range(BOTTOM, TOP): + # Make _PyType_Lookup cache hot first + A.attr + A.attr + x = A.attr + x += 1 + A.attr = x + + + with ThreadPoolExecutor(NTHREADS) as pool: + pool.submit(read, (1,)) + pool.submit(write, (1,)) + pool.shutdown(wait=True) + + def test_attr_cache_consistency(self): + class C: + x = 0 + + DONE = False + def writer_func(): + for i in range(3000): + C.x + C.x + C.x += 1 + nonlocal DONE + DONE = True + + def reader_func(): + while True: + # We should always see a greater value read from the type than the + # dictionary + a = C.__dict__['x'] + b = C.x + self.assertGreaterEqual(b, a) + + if DONE: + break + + self.run_one(writer_func, reader_func) + + def test_attr_cache_consistency_subclass(self): + class C: + x = 0 + + class D(C): + pass + + DONE = False + def writer_func(): + for i in range(3000): + D.x + D.x + C.x += 1 + nonlocal DONE + DONE = True + + def reader_func(): + while True: + # We should always see a greater value read from the type than the + # dictionary + a = C.__dict__['x'] + b = D.x + self.assertGreaterEqual(b, a) + + if DONE: + break + + self.run_one(writer_func, reader_func) + + def test___class___modification(self): + loops = 200 + + class Foo: + pass + + class Bar: + pass + + thing = Foo() + def work(): + foo = thing + for _ in range(loops): + foo.__class__ = Bar + type(foo) + foo.__class__ = Foo + type(foo) + + + threads = [] + for i in range(NTHREADS): + thread = threading.Thread(target=work) + thread.start() + threads.append(thread) + + for thread in threads: + thread.join() + + def test_object_class_change(self): + class Base: + def __init__(self): + self.attr = 123 + class ClassA(Base): + pass + class ClassB(Base): + pass + + obj = ClassA() + # keep reference to __dict__ + d = obj.__dict__ + obj.__class__ = ClassB + + + def run_one(self, writer_func, reader_func): + writer = Thread(target=writer_func) + readers = [] + for x in range(30): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_zip.py b/Lib/test/test_free_threading/test_zip.py new file mode 100644 index 00000000000000..b4c5837b1d8936 --- /dev/null +++ b/Lib/test/test_free_threading/test_zip.py @@ -0,0 +1,41 @@ +import unittest +from threading import Thread + +from test.support import threading_helper + + +class ZipThreading(unittest.TestCase): + @staticmethod + def work(enum): + while True: + try: + next(enum) + except StopIteration: + break + + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_threading(self): + number_of_threads = 8 + number_of_iterations = 8 + n = 40_000 + enum = zip(range(n), range(n)) + for _ in range(number_of_iterations): + worker_threads = [] + for ii in range(number_of_threads): + worker_threads.append( + Thread( + target=self.work, + args=[ + enum, + ], + ) + ) + for t in worker_threads: + t.start() + for t in worker_threads: + t.join() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 27c7f70cef32e3..1d96b7a2c2459b 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -8,13 +8,13 @@ # Unicode identifiers in tests is allowed by PEP 3131. import ast +import datetime import dis import os import re import types import decimal import unittest -import warnings from test import support from test.support.os_helper import temp_cwd from test.support.script_helper import assert_python_failure, assert_python_ok @@ -744,13 +744,13 @@ def test_comments(self): }''', 'A complex trick: 2') self.assertEqual(f''' { -40 # fourty +40 # forty + # plus 2 # two }''', '\n42') self.assertEqual(f''' { -40 # fourty +40 # forty + # plus 2 # two }''', '\n42') @@ -896,6 +896,7 @@ def test_missing_expression(self): "f'{:2}'", "f'''{\t\f\r\n:a}'''", "f'{:'", + "F'{[F'{:'}[F'{:'}]]]", ]) self.assertAllRaise(SyntaxError, @@ -1602,6 +1603,12 @@ def f(a): self.assertEqual(f'{f(a=4)}', '3=') self.assertEqual(x, 4) + # Check debug expressions in format spec + y = 20 + self.assertEqual(f"{2:{y=}}", "yyyyyyyyyyyyyyyyyyy2") + self.assertEqual(f"{datetime.datetime.now():h1{y=}h2{y=}h3{y=}}", + 'h1y=20h2y=20h3y=20') + # Make sure __format__ is being called. class C: def __format__(self, s): @@ -1615,9 +1622,11 @@ def __repr__(self): self.assertEqual(f'{C()=: }', 'C()=FORMAT- ') self.assertEqual(f'{C()=:x}', 'C()=FORMAT-x') self.assertEqual(f'{C()=!r:*^20}', 'C()=********REPR********') + self.assertEqual(f"{C():{20=}}", 'FORMAT-20=20') self.assertRaises(SyntaxError, eval, "f'{C=]'") + # Make sure leading and following text works. x = 'foo' self.assertEqual(f'X{x=}Y', 'Xx='+repr(x)+'Y') @@ -1640,6 +1649,14 @@ def __repr__(self): #self.assertEqual(f'X{x =}Y', 'Xx\t='+repr(x)+'Y') #self.assertEqual(f'X{x = }Y', 'Xx\t=\t'+repr(x)+'Y') + def test_debug_expressions_are_raw_strings(self): + + self.assertEqual(f'{b"\N{OX}"=}', 'b"\\N{OX}"=b\'\\\\N{OX}\'') + self.assertEqual(f'{r"\xff"=}', 'r"\\xff"=\'\\\\xff\'') + self.assertEqual(f'{r"\n"=}', 'r"\\n"=\'\\\\n\'') + self.assertEqual(f"{'\''=}", "'\\''=\"'\"") + self.assertEqual(f'{'\xc5'=}', r"'\xc5'='Å'") + def test_walrus(self): x = 20 # This isn't an assignment expression, it's 'x', with a format @@ -1748,5 +1765,23 @@ def get_code(s): for s in ["", "some string"]: self.assertEqual(get_code(f"'{s}'"), get_code(f"f'{s}'")) + def test_gh129093(self): + self.assertEqual(f'{1==2=}', '1==2=False') + self.assertEqual(f'{1 == 2=}', '1 == 2=False') + self.assertEqual(f'{1!=2=}', '1!=2=True') + self.assertEqual(f'{1 != 2=}', '1 != 2=True') + + self.assertEqual(f'{(1) != 2=}', '(1) != 2=True') + self.assertEqual(f'{(1*2) != (3)=}', '(1*2) != (3)=True') + + self.assertEqual(f'{1 != 2 == 3 != 4=}', '1 != 2 == 3 != 4=False') + self.assertEqual(f'{1 == 2 != 3 == 4=}', '1 == 2 != 3 == 4=False') + + self.assertEqual(f'{f'{1==2=}'=}', "f'{1==2=}'='1==2=False'") + self.assertEqual(f'{f'{1 == 2=}'=}', "f'{1 == 2=}'='1 == 2=False'") + self.assertEqual(f'{f'{1!=2=}'=}', "f'{1!=2=}'='1!=2=True'") + self.assertEqual(f'{f'{1 != 2=}'=}', "f'{1 != 2=}'='1 != 2=True'") + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_funcattrs.py b/Lib/test/test_funcattrs.py index b3fc5ad42e7fde..d919d62613ea7c 100644 --- a/Lib/test/test_funcattrs.py +++ b/Lib/test/test_funcattrs.py @@ -98,7 +98,12 @@ def test___globals__(self): (AttributeError, TypeError)) def test___builtins__(self): - self.assertIs(self.b.__builtins__, __builtins__) + if __name__ == "__main__": + builtins_dict = __builtins__.__dict__ + else: + builtins_dict = __builtins__ + + self.assertIs(self.b.__builtins__, builtins_dict) self.cannot_set_attr(self.b, '__builtins__', 2, (AttributeError, TypeError)) @@ -108,7 +113,7 @@ def func(s): return len(s) ns = {} func2 = type(func)(func.__code__, ns) self.assertIs(func2.__globals__, ns) - self.assertIs(func2.__builtins__, __builtins__) + self.assertIs(func2.__builtins__, builtins_dict) # Make sure that the function actually works. self.assertEqual(func2("abc"), 3) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 4eb322644fc541..4beb4380c3ad6b 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -6,6 +6,7 @@ from itertools import permutations import pickle from random import choice +import re import sys from test import support import threading @@ -185,6 +186,19 @@ def test_nested_optimization(self): flat = partial(signature, 'asdf', bar=True) self.assertEqual(signature(nested), signature(flat)) + def test_nested_optimization_bug(self): + partial = self.partial + class Builder: + def __call__(self, tag, *children, **attrib): + return (tag, children, attrib) + + def __getattr__(self, tag): + return partial(self, tag) + + B = Builder() + m = B.m + assert m(1, 2, a=2) == ('m', (1, 2), dict(a=2)) + def test_nested_partial_with_attribute(self): # see issue 25137 partial = self.partial @@ -197,6 +211,51 @@ def foo(bar): p2.new_attr = 'spam' self.assertEqual(p2.new_attr, 'spam') + def test_placeholders_trailing_raise(self): + PH = self.module.Placeholder + for args in [(PH,), (0, PH), (0, PH, 1, PH, PH, PH)]: + with self.assertRaises(TypeError): + self.partial(capture, *args) + + def test_placeholders(self): + PH = self.module.Placeholder + # 1 Placeholder + args = (PH, 0) + p = self.partial(capture, *args) + actual_args, actual_kwds = p('x') + self.assertEqual(actual_args, ('x', 0)) + self.assertEqual(actual_kwds, {}) + # 2 Placeholders + args = (PH, 0, PH, 1) + p = self.partial(capture, *args) + with self.assertRaises(TypeError): + p('x') + actual_args, actual_kwds = p('x', 'y') + self.assertEqual(actual_args, ('x', 0, 'y', 1)) + self.assertEqual(actual_kwds, {}) + + def test_placeholders_optimization(self): + PH = self.module.Placeholder + p = self.partial(capture, PH, 0) + p2 = self.partial(p, PH, 1, 2, 3) + self.assertEqual(p2.args, (PH, 0, 1, 2, 3)) + p3 = self.partial(p2, -1, 4) + actual_args, actual_kwds = p3(5) + self.assertEqual(actual_args, (-1, 0, 1, 2, 3, 4, 5)) + self.assertEqual(actual_kwds, {}) + # inner partial has placeholders and outer partial has no args case + p = self.partial(capture, PH, 0) + p2 = self.partial(p) + self.assertEqual(p2.args, (PH, 0)) + self.assertEqual(p2(1), ((1, 0), {})) + + def test_construct_placeholder_singleton(self): + PH = self.module.Placeholder + tp = type(PH) + self.assertIs(tp(), PH) + self.assertRaises(TypeError, tp, 1, 2) + self.assertRaises(TypeError, tp, a=1, b=2) + def test_repr(self): args = (object(), object()) args_repr = ', '.join(repr(a) for a in args) @@ -298,6 +357,23 @@ def test_setstate(self): self.assertEqual(f(2), ((2,), {})) self.assertEqual(f(), ((), {})) + # Set State with placeholders + PH = self.module.Placeholder + f = self.partial(signature) + f.__setstate__((capture, (PH, 1), dict(a=10), dict(attr=[]))) + self.assertEqual(signature(f), (capture, (PH, 1), dict(a=10), dict(attr=[]))) + msg_regex = re.escape("missing positional arguments in 'partial' call; " + "expected at least 1, got 0") + with self.assertRaisesRegex(TypeError, f'^{msg_regex}$') as cm: + f() + self.assertEqual(f(2), ((2, 1), dict(a=10))) + + # Trailing Placeholder error + f = self.partial(signature) + msg_regex = re.escape("trailing Placeholders are not allowed") + with self.assertRaisesRegex(TypeError, f'^{msg_regex}$') as cm: + f.__setstate__((capture, (1, PH), dict(a=10), dict(attr=[]))) + def test_setstate_errors(self): f = self.partial(signature) self.assertRaises(TypeError, f.__setstate__, (capture, (), {})) @@ -328,14 +404,17 @@ def test_setstate_subclasses(self): self.assertEqual(r, ((1, 2), {})) self.assertIs(type(r[0]), tuple) + @support.skip_emscripten_stack_overflow() def test_recursive_pickle(self): with replaced_module('functools', self.module): f = self.partial(capture) f.__setstate__((f, (), {}, {})) try: for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises(RecursionError): - pickle.dumps(f, proto) + # gh-117008: Small limit since pickle uses C stack memory + with support.infinite_recursion(100): + with self.assertRaises(RecursionError): + pickle.dumps(f, proto) finally: f.__setstate__((capture, (), {}, {})) @@ -380,6 +459,27 @@ def __getitem__(self, key): f = self.partial(object) self.assertRaises(TypeError, f.__setstate__, BadSequence()) + def test_partial_as_method(self): + class A: + meth = self.partial(capture, 1, a=2) + cmeth = classmethod(self.partial(capture, 1, a=2)) + smeth = staticmethod(self.partial(capture, 1, a=2)) + + a = A() + self.assertEqual(A.meth(3, b=4), ((1, 3), {'a': 2, 'b': 4})) + self.assertEqual(A.cmeth(3, b=4), ((1, A, 3), {'a': 2, 'b': 4})) + self.assertEqual(A.smeth(3, b=4), ((1, 3), {'a': 2, 'b': 4})) + self.assertEqual(a.meth(3, b=4), ((1, a, 3), {'a': 2, 'b': 4})) + self.assertEqual(a.cmeth(3, b=4), ((1, A, 3), {'a': 2, 'b': 4})) + self.assertEqual(a.smeth(3, b=4), ((1, 3), {'a': 2, 'b': 4})) + + def test_partial_genericalias(self): + alias = self.partial[int] + self.assertIs(alias.__origin__, self.partial) + self.assertEqual(alias.__args__, (int,)) + self.assertEqual(alias.__parameters__, ()) + + @unittest.skipUnless(c_functools, 'requires the C _functools module') class TestPartialC(TestPartial, unittest.TestCase): if c_functools: @@ -426,6 +526,19 @@ def __str__(self): self.assertIn('astr', r) self.assertIn("['sth']", r) + def test_placeholders_refcount_smoke(self): + PH = self.module.Placeholder + # sum supports vector call + lst1, start = [], [] + sum_lists = self.partial(sum, PH, start) + for i in range(10): + sum_lists([lst1, lst1]) + # collections.ChainMap initializer does not support vectorcall + map1, map2 = {}, {} + partial_cm = self.partial(collections.ChainMap, PH, map1) + for i in range(10): + partial_cm(map2, map2) + class TestPartialPy(TestPartial, unittest.TestCase): module = py_functools @@ -450,6 +563,19 @@ class TestPartialCSubclass(TestPartialC): class TestPartialPySubclass(TestPartialPy): partial = PyPartialSubclass + def test_subclass_optimization(self): + # `partial` input to `partial` subclass + p = py_functools.partial(min, 2) + p2 = self.partial(p, 1) + self.assertIs(p2.func, min) + self.assertEqual(p2(0), 0) + # `partial` subclass input to `partial` subclass + p = self.partial(min, 2) + p2 = self.partial(p, 1) + self.assertIs(p2.func, min) + self.assertEqual(p2(0), 0) + + class TestPartialMethod(unittest.TestCase): class A(object): @@ -519,11 +645,11 @@ def test_bound_method_introspection(self): def test_unbound_method_retrieval(self): obj = self.A - self.assertFalse(hasattr(obj.both, "__self__")) - self.assertFalse(hasattr(obj.nested, "__self__")) - self.assertFalse(hasattr(obj.over_partial, "__self__")) - self.assertFalse(hasattr(obj.static, "__self__")) - self.assertFalse(hasattr(self.a.static, "__self__")) + self.assertNotHasAttr(obj.both, "__self__") + self.assertNotHasAttr(obj.nested, "__self__") + self.assertNotHasAttr(obj.over_partial, "__self__") + self.assertNotHasAttr(obj.static, "__self__") + self.assertNotHasAttr(self.a.static, "__self__") def test_descriptors(self): for obj in [self.A, self.a]: @@ -554,6 +680,14 @@ class B: method = functools.partialmethod(func=capture, a=1) def test_repr(self): + self.assertEqual(repr(vars(self.A)['nothing']), + 'functools.partialmethod({})'.format(capture)) + self.assertEqual(repr(vars(self.A)['positional']), + 'functools.partialmethod({}, 1)'.format(capture)) + self.assertEqual(repr(vars(self.A)['keywords']), + 'functools.partialmethod({}, a=2)'.format(capture)) + self.assertEqual(repr(vars(self.A)['spec_keywords']), + 'functools.partialmethod({}, self=1, func=2)'.format(capture)) self.assertEqual(repr(vars(self.A)['both']), 'functools.partialmethod({}, 3, b=4)'.format(capture)) @@ -579,6 +713,20 @@ def f(a, b, /): p = functools.partial(f, 1) self.assertEqual(p(2), f(1, 2)) + def test_subclass_optimization(self): + class PartialMethodSubclass(functools.partialmethod): + pass + # `partialmethod` input to `partialmethod` subclass + p = functools.partialmethod(min, 2) + p2 = PartialMethodSubclass(p, 1) + self.assertIs(p2.func, min) + self.assertEqual(p2.__get__(0)(), 0) + # `partialmethod` subclass input to `partialmethod` subclass + p = PartialMethodSubclass(min, 2) + p2 = PartialMethodSubclass(p, 1) + self.assertIs(p2.func, min) + self.assertEqual(p2.__get__(0)(), 0) + class TestUpdateWrapper(unittest.TestCase): @@ -643,7 +791,7 @@ def wrapper(): self.assertNotEqual(wrapper.__qualname__, f.__qualname__) self.assertEqual(wrapper.__doc__, None) self.assertEqual(wrapper.__annotations__, {}) - self.assertFalse(hasattr(wrapper, 'attr')) + self.assertNotHasAttr(wrapper, 'attr') def test_selective_update(self): def f(): @@ -692,8 +840,36 @@ def wrapper(): pass functools.update_wrapper(wrapper, max) self.assertEqual(wrapper.__name__, 'max') - self.assertTrue(wrapper.__doc__.startswith('max(')) + self.assertStartsWith(wrapper.__doc__, 'max(') + self.assertEqual(wrapper.__annotations__, {}) + + def test_update_type_wrapper(self): + def wrapper(*args): pass + + functools.update_wrapper(wrapper, type) + self.assertEqual(wrapper.__name__, 'type') self.assertEqual(wrapper.__annotations__, {}) + self.assertEqual(wrapper.__type_params__, ()) + + def test_update_wrapper_annotations(self): + def inner(x: int): pass + def wrapper(*args): pass + + functools.update_wrapper(wrapper, inner) + self.assertEqual(wrapper.__annotations__, {'x': int}) + self.assertIs(wrapper.__annotate__, inner.__annotate__) + + def with_forward_ref(x: undefined): pass + def wrapper(*args): pass + + functools.update_wrapper(wrapper, with_forward_ref) + + self.assertIs(wrapper.__annotate__, with_forward_ref.__annotate__) + with self.assertRaises(NameError): + wrapper.__annotations__ + + undefined = str + self.assertEqual(wrapper.__annotations__, {'x': undefined}) class TestWraps(TestUpdateWrapper): @@ -734,7 +910,7 @@ def wrapper(): self.assertEqual(wrapper.__name__, 'wrapper') self.assertNotEqual(wrapper.__qualname__, f.__qualname__) self.assertEqual(wrapper.__doc__, None) - self.assertFalse(hasattr(wrapper, 'attr')) + self.assertNotHasAttr(wrapper, 'attr') def test_selective_update(self): def f(): @@ -836,6 +1012,29 @@ def __getitem__(self, i): d = {"one": 1, "two": 2, "three": 3} self.assertEqual(self.reduce(add, d), "".join(d.keys())) + # test correctness of keyword usage of `initial` in `reduce` + def test_initial_keyword(self): + def add(x, y): + return x + y + self.assertEqual( + self.reduce(add, ['a', 'b', 'c'], ''), + self.reduce(add, ['a', 'b', 'c'], initial=''), + ) + self.assertEqual( + self.reduce(add, [['a', 'c'], [], ['d', 'w']], []), + self.reduce(add, [['a', 'c'], [], ['d', 'w']], initial=[]), + ) + self.assertEqual( + self.reduce(lambda x, y: x*y, range(2,8), 1), + self.reduce(lambda x, y: x*y, range(2,8), initial=1), + ) + self.assertEqual( + self.reduce(lambda x, y: x*y, range(2,21), 1), + self.reduce(lambda x, y: x*y, range(2,21), initial=1), + ) + self.assertRaises(TypeError, self.reduce, add, [0, 1], initial="") + self.assertEqual(self.reduce(42, "", initial="1"), "1") # func is never called with one item + @unittest.skipUnless(c_functools, 'requires the C _functools module') class TestReduceC(TestReduce, unittest.TestCase): @@ -846,6 +1045,12 @@ class TestReduceC(TestReduce, unittest.TestCase): class TestReducePy(TestReduce, unittest.TestCase): reduce = staticmethod(py_functools.reduce) + def test_reduce_with_kwargs(self): + with self.assertWarns(DeprecationWarning): + self.reduce(function=lambda x, y: x + y, sequence=[1, 2, 3, 4, 5], initial=1) + with self.assertWarns(DeprecationWarning): + self.reduce(lambda x, y: x + y, sequence=[1, 2, 3, 4, 5], initial=1) + class TestCmpToKey: @@ -932,8 +1137,13 @@ def mycmp(x, y): @unittest.skipIf(support.MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_cmp_to_signature(self): - self.assertEqual(str(Signature.from_callable(self.cmp_to_key)), - '(mycmp)') + sig = Signature.from_callable(self.cmp_to_key) + self.assertEqual(str(sig), '(mycmp)') + def mycmp(x, y): + return y - x + sig = Signature.from_callable(self.cmp_to_key(mycmp)) + self.assertEqual(str(sig), '(obj)') + @unittest.skipUnless(c_functools, 'requires the C _functools module') @@ -1848,14 +2058,16 @@ def test_staticmethod(x): self.assertIsNone(ref()) def test_common_signatures(self): - def orig(): ... + def orig(a, /, b, c=True): ... lru = self.module.lru_cache(1)(orig) + self.assertEqual(str(Signature.from_callable(lru)), '(a, /, b, c=True)') self.assertEqual(str(Signature.from_callable(lru.cache_info)), '()') self.assertEqual(str(Signature.from_callable(lru.cache_clear)), '()') @support.skip_on_s390x @unittest.skipIf(support.is_wasi, "WASI has limited C stack") + @support.skip_emscripten_stack_overflow() def test_lru_recursion(self): @self.module.lru_cache @@ -1865,7 +2077,7 @@ def fib(n): return fib(n-1) + fib(n-2) if not support.Py_DEBUG: - depth = support.Py_C_RECURSION_LIMIT*2//7 + depth = support.get_c_recursion_limit()*2//7 with support.infinite_recursion(): fib(depth) if self.module == c_functools: @@ -2454,15 +2666,15 @@ def _(self, arg): a.t(0) self.assertEqual(a.arg, "int") aa = A() - self.assertFalse(hasattr(aa, 'arg')) + self.assertNotHasAttr(aa, 'arg') a.t('') self.assertEqual(a.arg, "str") aa = A() - self.assertFalse(hasattr(aa, 'arg')) + self.assertNotHasAttr(aa, 'arg') a.t(0.0) self.assertEqual(a.arg, "base") aa = A() - self.assertFalse(hasattr(aa, 'arg')) + self.assertNotHasAttr(aa, 'arg') def test_staticmethod_register(self): class A: @@ -2696,7 +2908,10 @@ def static_func(arg: int) -> str: A().static_func ): with self.subTest(meth=meth): - self.assertEqual(meth.__doc__, 'My function docstring') + self.assertEqual(meth.__doc__, + ('My function docstring' + if support.HAVE_DOCSTRINGS + else None)) self.assertEqual(meth.__annotations__['arg'], int) self.assertEqual(A.func.__name__, 'func') @@ -2785,7 +3000,10 @@ def decorated_classmethod(cls, arg: int) -> str: WithSingleDispatch().decorated_classmethod ): with self.subTest(meth=meth): - self.assertEqual(meth.__doc__, 'My function docstring') + self.assertEqual(meth.__doc__, + ('My function docstring' + if support.HAVE_DOCSTRINGS + else None)) self.assertEqual(meth.__annotations__['arg'], int) self.assertEqual( @@ -2818,16 +3036,16 @@ def i(arg): @i.register(42) def _(arg): return "I annotated with a non-type" - self.assertTrue(str(exc.exception).startswith(msg_prefix + "42")) - self.assertTrue(str(exc.exception).endswith(msg_suffix)) + self.assertStartsWith(str(exc.exception), msg_prefix + "42") + self.assertEndsWith(str(exc.exception), msg_suffix) with self.assertRaises(TypeError) as exc: @i.register def _(arg): return "I forgot to annotate" - self.assertTrue(str(exc.exception).startswith(msg_prefix + + self.assertStartsWith(str(exc.exception), msg_prefix + "._" - )) - self.assertTrue(str(exc.exception).endswith(msg_suffix)) + ) + self.assertEndsWith(str(exc.exception), msg_suffix) with self.assertRaises(TypeError) as exc: @i.register @@ -2837,23 +3055,23 @@ def _(arg: typing.Iterable[str]): # types from `typing`. Instead, annotate with regular types # or ABCs. return "I annotated with a generic collection" - self.assertTrue(str(exc.exception).startswith( + self.assertStartsWith(str(exc.exception), "Invalid annotation for 'arg'." - )) - self.assertTrue(str(exc.exception).endswith( + ) + self.assertEndsWith(str(exc.exception), 'typing.Iterable[str] is not a class.' - )) + ) with self.assertRaises(TypeError) as exc: @i.register def _(arg: typing.Union[int, typing.Iterable[str]]): return "Invalid Union" - self.assertTrue(str(exc.exception).startswith( + self.assertStartsWith(str(exc.exception), "Invalid annotation for 'arg'." - )) - self.assertTrue(str(exc.exception).endswith( + ) + self.assertEndsWith(str(exc.exception), 'typing.Union[int, typing.Iterable[str]] not all arguments are classes.' - )) + ) def test_invalid_positional_argument(self): @functools.singledispatch @@ -3000,6 +3218,27 @@ def _(arg: typing.List[float] | bytes): self.assertEqual(f(""), "default") self.assertEqual(f(b""), "default") + def test_forward_reference(self): + @functools.singledispatch + def f(arg, arg2=None): + return "default" + + @f.register + def _(arg: str, arg2: undefined = None): + return "forward reference" + + self.assertEqual(f(1), "default") + self.assertEqual(f(""), "forward reference") + + def test_unresolved_forward_reference(self): + @functools.singledispatch + def f(arg): + return "default" + + with self.assertRaisesRegex(TypeError, "is an unresolved forward reference"): + @f.register + def _(arg: undefined): + return "forward reference" class CachedCostItem: _cost = 1 @@ -3128,7 +3367,10 @@ def test_access_from_class(self): self.assertIsInstance(CachedCostItem.cost, py_functools.cached_property) def test_doc(self): - self.assertEqual(CachedCostItem.cost.__doc__, "The cost of the item.") + self.assertEqual(CachedCostItem.cost.__doc__, + ("The cost of the item." + if support.HAVE_DOCSTRINGS + else None)) def test_module(self): self.assertEqual(CachedCostItem.cost.__module__, CachedCostItem.__module__) diff --git a/Lib/test/test_future_stmt/nested_scope.py b/Lib/test/test_future_stmt/nested_scope.py index 3d7fc860a37655..a8433a42cbb6b0 100644 --- a/Lib/test/test_future_stmt/nested_scope.py +++ b/Lib/test/test_future_stmt/nested_scope.py @@ -1,6 +1,6 @@ """This is a test""" -from __future__ import nested_scopes; import site +from __future__ import nested_scopes; import site # noqa: F401 def f(x): def g(y): diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py index 2c8ceb664cb362..42c6cb3fefac33 100644 --- a/Lib/test/test_future_stmt/test_future.py +++ b/Lib/test/test_future_stmt/test_future.py @@ -67,19 +67,19 @@ def test_future_single_import(self): with import_helper.CleanImport( 'test.test_future_stmt.test_future_single_import', ): - from test.test_future_stmt import test_future_single_import + from test.test_future_stmt import test_future_single_import # noqa: F401 def test_future_multiple_imports(self): with import_helper.CleanImport( 'test.test_future_stmt.test_future_multiple_imports', ): - from test.test_future_stmt import test_future_multiple_imports + from test.test_future_stmt import test_future_multiple_imports # noqa: F401 def test_future_multiple_features(self): with import_helper.CleanImport( "test.test_future_stmt.test_future_multiple_features", ): - from test.test_future_stmt import test_future_multiple_features + from test.test_future_stmt import test_future_multiple_features # noqa: F401 def test_unknown_future_flag(self): code = """ @@ -88,7 +88,7 @@ def test_unknown_future_flag(self): """ self.assertSyntaxError( code, lineno=2, - message='future feature rested_snopes is not defined', + message='future feature rested_snopes is not defined', offset=24, ) def test_future_import_not_on_top(self): @@ -137,23 +137,23 @@ def test_future_import_star(self): code = """ from __future__ import * """ - self.assertSyntaxError(code, message='future feature * is not defined') + self.assertSyntaxError(code, message='future feature * is not defined', offset=24) def test_future_import_braces(self): code = """ from __future__ import braces """ # Congrats, you found an easter egg! - self.assertSyntaxError(code, message='not a chance') + self.assertSyntaxError(code, message='not a chance', offset=24) code = """ from __future__ import nested_scopes, braces """ - self.assertSyntaxError(code, message='not a chance') + self.assertSyntaxError(code, message='not a chance', offset=39) def test_module_with_future_import_not_on_top(self): with self.assertRaises(SyntaxError) as cm: - from test.test_future_stmt import badsyntax_future + from test.test_future_stmt import badsyntax_future # noqa: F401 self.check_syntax_error(cm.exception, "badsyntax_future", lineno=3) def test_ensure_flags_dont_clash(self): @@ -171,26 +171,6 @@ def test_ensure_flags_dont_clash(self): } self.assertCountEqual(set(flags.values()), flags.values()) - def test_parserhack(self): - # test that the parser.c::future_hack function works as expected - # Note: although this test must pass, it's not testing the original - # bug as of 2.6 since the with statement is not optional and - # the parser hack disabled. If a new keyword is introduced in - # 2.6, change this to refer to the new future import. - try: - exec("from __future__ import print_function; print 0") - except SyntaxError: - pass - else: - self.fail("syntax error didn't occur") - - try: - exec("from __future__ import (print_function); print 0") - except SyntaxError: - pass - else: - self.fail("syntax error didn't occur") - def test_unicode_literals_exec(self): scope = {} exec("from __future__ import unicode_literals; x = ''", {}, scope) @@ -203,6 +183,25 @@ def test_syntactical_future_repl(self): out = kill_python(p) self.assertNotIn(b'SyntaxError: invalid syntax', out) + def test_future_dotted_import(self): + with self.assertRaises(ImportError): + exec("from .__future__ import spam") + + code = dedent( + """ + from __future__ import print_function + from ...__future__ import ham + """ + ) + with self.assertRaises(ImportError): + exec(code) + + code = """ + from .__future__ import nested_scopes + from __future__ import barry_as_FLUFL + """ + self.assertSyntaxError(code, lineno=2) + class AnnotationsFutureTestCase(unittest.TestCase): template = dedent( """ diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index dd09643788d62f..b5140057a69d36 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1,11 +1,14 @@ import unittest import unittest.mock +from test import support from test.support import (verbose, refcount_test, - cpython_only, requires_subprocess, Py_GIL_DISABLED) + cpython_only, requires_subprocess, + requires_gil_enabled, + Py_GIL_DISABLED) from test.support.import_helper import import_module from test.support.os_helper import temp_dir, TESTFN, unlink from test.support.script_helper import assert_python_ok, make_script -from test.support import threading_helper +from test.support import threading_helper, gc_threshold import gc import sys @@ -16,18 +19,22 @@ import weakref try: + import _testcapi from _testcapi import with_tp_del + from _testcapi import ContainerNoGC except ImportError: + _testcapi = None def with_tp_del(cls): class C(object): def __new__(cls, *args, **kwargs): - raise TypeError('requires _testcapi.with_tp_del') + raise unittest.SkipTest('requires _testcapi.with_tp_del') return C + ContainerNoGC = None try: - from _testcapi import ContainerNoGC + import _testinternalcapi except ImportError: - ContainerNoGC = None + _testinternalcapi = None ### Support code ############################################################################### @@ -225,7 +232,9 @@ def test_function(self): exec("def f(): pass\n", d) gc.collect() del d - self.assertEqual(gc.collect(), 2) + # In the free-threaded build, the count returned by `gc.collect()` + # is 3 because it includes f's code object. + self.assertIn(gc.collect(), (2, 3)) def test_function_tp_clear_leaves_consistent_state(self): # https://github.com/python/cpython/issues/91636 @@ -363,7 +372,7 @@ def __del__(self): # To minimize variations, though, we first store the get_count() results # and check them at the end. @refcount_test - @unittest.skipIf(Py_GIL_DISABLED, 'needs precise allocation counts') + @requires_gil_enabled('needs precise allocation counts') def test_get_count(self): gc.collect() a, b, c = gc.get_count() @@ -384,19 +393,11 @@ def test_collect_generations(self): # each call to collect(N) x = [] gc.collect(0) - # x is now in gen 1 + # x is now in the old gen a, b, c = gc.get_count() - gc.collect(1) - # x is now in gen 2 - d, e, f = gc.get_count() - gc.collect(2) - # x is now in gen 3 - g, h, i = gc.get_count() - # We don't check a, d, g since their exact values depends on + # We don't check a since its exact values depends on # internal implementation details of the interpreter. self.assertEqual((b, c), (1, 0)) - self.assertEqual((e, f), (0, 1)) - self.assertEqual((h, i), (0, 0)) def test_trashcan(self): class Ouch: @@ -478,7 +479,7 @@ def run_thread(): make_nested() old_switchinterval = sys.getswitchinterval() - sys.setswitchinterval(1e-5) + support.setswitchinterval(1e-5) try: exit = [] threads = [] @@ -689,6 +690,7 @@ def do_work(): @cpython_only @requires_subprocess() + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_garbage_at_shutdown(self): import subprocess code = """if 1: @@ -823,7 +825,7 @@ def test_get_objects(self): any(l is element for element in gc.get_objects()) ) - @unittest.skipIf(Py_GIL_DISABLED, 'need generational GC') + @requires_gil_enabled('need generational GC') def test_get_objects_generations(self): gc.collect() l = [] @@ -831,42 +833,10 @@ def test_get_objects_generations(self): self.assertTrue( any(l is element for element in gc.get_objects(generation=0)) ) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=1)) - ) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=2)) - ) - gc.collect(generation=0) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=0)) - ) - self.assertTrue( - any(l is element for element in gc.get_objects(generation=1)) - ) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=2)) - ) - gc.collect(generation=1) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=0)) - ) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=1)) - ) - self.assertTrue( - any(l is element for element in gc.get_objects(generation=2)) - ) - gc.collect(generation=2) + gc.collect() self.assertFalse( any(l is element for element in gc.get_objects(generation=0)) ) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=1)) - ) - self.assertTrue( - any(l is element for element in gc.get_objects(generation=2)) - ) del l gc.collect() @@ -1076,6 +1046,146 @@ class Z: callback.assert_not_called() gc.enable() + @cpython_only + def test_get_referents_on_capsule(self): + # gh-124538: Calling gc.get_referents() on an untracked capsule must not crash. + import _datetime + import _socket + untracked_capsule = _datetime.datetime_CAPI + tracked_capsule = _socket.CAPI + + # For whoever sees this in the future: if this is failing + # after making datetime's capsule tracked, that's fine -- this isn't something + # users are relying on. Just find a different capsule that is untracked. + self.assertFalse(gc.is_tracked(untracked_capsule)) + self.assertTrue(gc.is_tracked(tracked_capsule)) + + self.assertEqual(len(gc.get_referents(untracked_capsule)), 0) + gc.get_referents(tracked_capsule) + + @cpython_only + def test_get_objects_during_gc(self): + # gh-125859: Calling gc.get_objects() or gc.get_referrers() during a + # collection should not crash. + test = self + collected = False + + class GetObjectsOnDel: + def __del__(self): + nonlocal collected + collected = True + objs = gc.get_objects() + # NB: can't use "in" here because some objects override __eq__ + for obj in objs: + test.assertTrue(obj is not self) + test.assertEqual(gc.get_referrers(self), []) + + obj = GetObjectsOnDel() + obj.cycle = obj + del obj + + gc.collect() + self.assertTrue(collected) + + def test_traverse_frozen_objects(self): + # See GH-126312: Objects that were not frozen could traverse over + # a frozen object on the free-threaded build, which would cause + # a negative reference count. + x = [1, 2, 3] + gc.freeze() + y = [x] + y.append(y) + del y + gc.collect() + gc.unfreeze() + + def test_deferred_refcount_frozen(self): + # Also from GH-126312: objects that use deferred reference counting + # weren't ignored if they were frozen. Unfortunately, it's pretty + # difficult to come up with a case that triggers this. + # + # Calling gc.collect() while the garbage collector is frozen doesn't + # trigger this normally, but it *does* if it's inside unittest for whatever + # reason. We can't call unittest from inside a test, so it has to be + # in a subprocess. + source = textwrap.dedent(""" + import gc + import unittest + + + class Test(unittest.TestCase): + def test_something(self): + gc.freeze() + gc.collect() + gc.unfreeze() + + + if __name__ == "__main__": + unittest.main() + """) + assert_python_ok("-c", source) + + +class IncrementalGCTests(unittest.TestCase): + + def setUp(self): + # Reenable GC as it is disabled module-wide + gc.enable() + + def tearDown(self): + gc.disable() + + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + @requires_gil_enabled("Free threading does not support incremental GC") + # Use small increments to emulate longer running process in a shorter time + @gc_threshold(200, 10) + def test_incremental_gc_handles_fast_cycle_creation(self): + + class LinkedList: + + #Use slots to reduce number of implicit objects + __slots__ = "next", "prev", "surprise" + + def __init__(self, next=None, prev=None): + self.next = next + if next is not None: + next.prev = self + self.prev = prev + if prev is not None: + prev.next = self + + def make_ll(depth): + head = LinkedList() + for i in range(depth): + head = LinkedList(head, head.prev) + return head + + head = make_ll(1000) + count = 1000 + + # There will be some objects we aren't counting, + # e.g. the gc stats dicts. This test checks + # that the counts don't grow, so we try to + # correct for the uncounted objects + # This is just an estimate. + CORRECTION = 20 + + enabled = gc.isenabled() + gc.enable() + olds = [] + initial_heap_size = _testinternalcapi.get_tracked_heap_size() + for i in range(20_000): + newhead = make_ll(20) + count += 20 + newhead.surprise = head + olds.append(newhead) + if len(olds) == 20: + new_objects = _testinternalcapi.get_tracked_heap_size() - initial_heap_size + self.assertLess(new_objects, 27_000, f"Heap growing. Reached limit after {i} iterations") + del olds[:] + if not enabled: + gc.disable() + class GCCallbackTests(unittest.TestCase): def setUp(self): @@ -1198,6 +1308,7 @@ def test_collect_garbage(self): self.assertEqual(len(gc.garbage), 0) + @requires_subprocess() @unittest.skipIf(BUILD_WITH_NDEBUG, 'built with -NDEBUG') def test_refcount_errors(self): @@ -1212,7 +1323,8 @@ def test_refcount_errors(self): from test.support import gc_collect, SuppressCrashReport a = [1, 2, 3] - b = [a] + b = [a, a] + a.append(b) # Avoid coredump when Py_FatalError() calls abort() SuppressCrashReport().__enter__() @@ -1222,6 +1334,8 @@ def test_refcount_errors(self): # (to avoid deallocating it): import ctypes ctypes.pythonapi.Py_DecRef(ctypes.py_object(a)) + del a + del b # The garbage collector should now have a fatal error # when it reaches the broken object @@ -1250,7 +1364,7 @@ def test_refcount_errors(self): self.assertRegex(stderr, br'object type name: list') self.assertRegex(stderr, - br'object repr : \[1, 2, 3\]') + br'object repr : \[1, 2, 3, \[\[...\], \[...\]\]\]') class GCTogglingTests(unittest.TestCase): @@ -1317,6 +1431,10 @@ def callback(ignored): junk = [] i = 0 detector = GC_Detector() + if Py_GIL_DISABLED: + # The free-threaded build doesn't have multiple generations, so + # just trigger a GC manually. + gc.collect() while not detector.gc_happened: i += 1 if i > 10000: @@ -1330,6 +1448,7 @@ def callback(ignored): # with an empty __dict__. self.assertEqual(x, None) + @gc_threshold(1000, 0, 0) def test_bug1055820d(self): # Corresponds to temp2d.py in the bug report. This is very much like # test_bug1055820c, but uses a __del__ method instead of a weakref @@ -1384,6 +1503,10 @@ def __del__(self): detector = GC_Detector() junk = [] i = 0 + if Py_GIL_DISABLED: + # The free-threaded build doesn't have multiple generations, so + # just trigger a GC manually. + gc.collect() while not detector.gc_happened: i += 1 if i > 10000: @@ -1397,6 +1520,32 @@ def __del__(self): # empty __dict__. self.assertEqual(x, None) + @gc_threshold(1000, 0, 0) + def test_indirect_calls_with_gc_disabled(self): + junk = [] + i = 0 + detector = GC_Detector() + while not detector.gc_happened: + i += 1 + if i > 10000: + self.fail("gc didn't happen after 10000 iterations") + junk.append([]) # this will eventually trigger gc + + try: + gc.disable() + junk = [] + i = 0 + detector = GC_Detector() + while not detector.gc_happened: + i += 1 + if i > 10000: + break + junk.append([]) # this may eventually trigger gc (if it is enabled) + + self.assertEqual(i, 10001) + finally: + gc.enable() + class PythonFinalizationTests(unittest.TestCase): def test_ast_fini(self): diff --git a/Lib/test/test_gdb/__init__.py b/Lib/test/test_gdb/__init__.py index 99557739af6748..0dd721780233e6 100644 --- a/Lib/test/test_gdb/__init__.py +++ b/Lib/test/test_gdb/__init__.py @@ -24,6 +24,9 @@ if support.check_cflags_pgo(): raise unittest.SkipTest("test_gdb is not reliable on PGO builds") +if support.check_bolt_optimized(): + raise unittest.SkipTest("test_gdb is not reliable on BOLT optimized builds") + def load_tests(*args): return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_gdb/test_backtrace.py b/Lib/test/test_gdb/test_backtrace.py index c41e7cb7c210de..714853c7b4732d 100644 --- a/Lib/test/test_gdb/test_backtrace.py +++ b/Lib/test/test_gdb/test_backtrace.py @@ -49,6 +49,7 @@ def test_bt_full(self): @unittest.skipIf(python_is_optimized(), "Python was compiled with optimizations") + @support.requires_gil_enabled() @support.requires_resource('cpu') def test_threads(self): 'Verify that "py-bt" indicates threads that are waiting for the GIL' diff --git a/Lib/test/test_gdb/util.py b/Lib/test/test_gdb/util.py index 8fe9cfc543395e..8097fd52ababe6 100644 --- a/Lib/test/test_gdb/util.py +++ b/Lib/test/test_gdb/util.py @@ -280,11 +280,6 @@ def get_stack_trace(self, source=None, script=None, return out - def assertEndsWith(self, actual, exp_end): - '''Ensure that the given "actual" string ends with "exp_end"''' - self.assertTrue(actual.endswith(exp_end), - msg='%r did not end with %r' % (actual, exp_end)) - def assertMultilineMatches(self, actual, pattern): m = re.match(pattern, actual, re.DOTALL) if not m: diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 32c2c2fca05c4e..bb78dd9af83b62 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1,9 +1,11 @@ import contextlib import os +import re import sys import tempfile import unittest +from io import StringIO from test import support from test import test_tools @@ -29,10 +31,12 @@ def skip_if_different_mount_drives(): test_tools.skip_if_missing("cases_generator") with test_tools.imports_under_tool("cases_generator"): - from analyzer import StackItem + from analyzer import analyze_forest, StackItem + from cwriter import CWriter import parser - from stack import Stack + from stack import Local, Stack import tier1_generator + import opcode_metadata_generator import optimizer_generator @@ -43,6 +47,14 @@ def handle_stderr(): return support.captured_stderr() +def parse_src(src): + p = parser.Parser(src, "test.c") + nodes = [] + while node := p.definition(): + nodes.append(node) + return nodes + + class TestEffects(unittest.TestCase): def test_effect_sizes(self): stack = Stack() @@ -60,9 +72,174 @@ def test_effect_sizes(self): stack.pop(y) stack.pop(x) for out in outputs: - stack.push(out) - self.assertEqual(stack.base_offset.to_c(), "-1 - oparg*2 - oparg") - self.assertEqual(stack.top_offset.to_c(), "1 - oparg*2 - oparg + oparg*4") + stack.push(Local.undefined(out)) + self.assertEqual(stack.base_offset.to_c(), "-1 - oparg - oparg*2") + self.assertEqual(stack.top_offset.to_c(), "1 - oparg - oparg*2 + oparg*4") + + +class TestGenerateMaxStackEffect(unittest.TestCase): + def check(self, input, output): + analysis = analyze_forest(parse_src(input)) + buf = StringIO() + writer = CWriter(buf, 0, False) + opcode_metadata_generator.generate_max_stack_effect_function(analysis, writer) + buf.seek(0) + generated = buf.read() + matches = re.search(r"(case OP: {[^}]+})", generated) + if matches is None: + self.fail(f"Couldn't find case statement for OP in:\n {generated}") + self.assertEqual(output.strip(), matches.group(1)) + + def test_push_one(self): + input = """ + inst(OP, (a -- b, c)) { + SPAM(); + } + """ + output = """ + case OP: { + *effect = 1; + return 0; + } + """ + self.check(input, output) + + def test_cond_push(self): + input = """ + inst(OP, (a -- b, c if (oparg))) { + SPAM(); + } + """ + output = """ + case OP: { + *effect = ((oparg) ? 1 : 0); + return 0; + } + """ + self.check(input, output) + + def test_ops_pass_two(self): + input = """ + op(A, (-- val1)) { + val1 = SPAM(); + } + op(B, (-- val2)) { + val2 = SPAM(); + } + op(C, (val1, val2 --)) { + } + macro(OP) = A + B + C; + """ + output = """ + case OP: { + *effect = 2; + return 0; + } + """ + self.check(input, output) + + def test_ops_pass_two_cond_push(self): + input = """ + op(A, (-- val1, val2)) { + val1 = 0; + val2 = 1; + } + op(B, (val1, val2 -- val1, val2, val3 if (oparg))) { + val3 = SPAM(); + } + macro(OP) = A + B; + """ + output = """ + case OP: { + *effect = Py_MAX(2, 2 + ((oparg) ? 1 : 0)); + return 0; + } + """ + self.check(input, output) + + def test_pop_push_array(self): + input = """ + inst(OP, (values[oparg] -- values[oparg], above)) { + SPAM(values, oparg); + above = 0; + } + """ + output = """ + case OP: { + *effect = 1; + return 0; + } + """ + self.check(input, output) + + def test_family(self): + input = """ + op(A, (-- val1, val2)) { + val1 = 0; + val2 = 1; + } + op(B, (val1, val2 -- val3)) { + val3 = 2; + } + macro(OP1) = A + B; + + inst(OP, (-- val)) { + val = 0; + } + + family(OP, 0) = { OP1 }; + """ + output = """ + case OP: { + *effect = 2; + return 0; + } + """ + self.check(input, output) + + def test_family_intermediate_array(self): + input = """ + op(A, (-- values[oparg])) { + val1 = 0; + val2 = 1; + } + op(B, (values[oparg] -- val3)) { + val3 = 2; + } + macro(OP1) = A + B; + + inst(OP, (-- val)) { + val = 0; + } + + family(OP, 0) = { OP1 }; + """ + output = """ + case OP: { + *effect = Py_MAX(1, oparg); + return 0; + } + """ + self.check(input, output) + + def test_negative_effect(self): + input = """ + op(A, (val1 -- )) { + } + op(B, (val2 --)) { + } + op(C, (val3 --)) { + } + + macro(OP) = A + B + C; + """ + output = """ + case OP: { + *effect = -1; + return 0; + } + """ + self.check(input, output) class TestGeneratedCases(unittest.TestCase): @@ -104,12 +281,12 @@ def run_cases_test(self, input: str, expected: str): ) with open(self.temp_output_filename) as temp_output: - lines = temp_output.readlines() - while lines and lines[0].startswith(("// ", "#", " #", "\n")): - lines.pop(0) - while lines and lines[-1].startswith(("#", "\n")): - lines.pop(-1) - actual = "".join(lines) + lines = temp_output.read() + _, rest = lines.split(tier1_generator.INSTRUCTION_START_MARKER) + instructions, labels_with_prelude_and_postlude = rest.split(tier1_generator.INSTRUCTION_END_MARKER) + _, labels_with_postlude = labels_with_prelude_and_postlude.split(tier1_generator.LABEL_START_MARKER) + labels, _ = labels_with_postlude.split(tier1_generator.LABEL_END_MARKER) + actual = instructions + labels # if actual.strip() != expected.strip(): # print("Actual:") # print(actual) @@ -122,7 +299,7 @@ def run_cases_test(self, input: str, expected: str): def test_inst_no_args(self): input = """ inst(OP, (--)) { - spam(); + SPAM(); } """ output = """ @@ -130,7 +307,7 @@ def test_inst_no_args(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - spam(); + SPAM(); DISPATCH(); } """ @@ -139,7 +316,8 @@ def test_inst_no_args(self): def test_inst_one_pop(self): input = """ inst(OP, (value --)) { - spam(); + SPAM(value); + DEAD(value); } """ output = """ @@ -147,10 +325,11 @@ def test_inst_one_pop(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *value; + _PyStackRef value; value = stack_pointer[-1]; - spam(); + SPAM(value); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -159,7 +338,7 @@ def test_inst_one_pop(self): def test_inst_one_push(self): input = """ inst(OP, (-- res)) { - spam(); + res = SPAM(); } """ output = """ @@ -167,10 +346,11 @@ def test_inst_one_push(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *res; - spam(); + _PyStackRef res; + res = SPAM(); stack_pointer[0] = res; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -179,7 +359,8 @@ def test_inst_one_push(self): def test_inst_one_push_one_pop(self): input = """ inst(OP, (value -- res)) { - spam(); + res = SPAM(value); + DEAD(value); } """ output = """ @@ -187,10 +368,10 @@ def test_inst_one_push_one_pop(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - spam(); + res = SPAM(value); stack_pointer[-1] = res; DISPATCH(); } @@ -200,7 +381,9 @@ def test_inst_one_push_one_pop(self): def test_binary_op(self): input = """ inst(OP, (left, right -- res)) { - spam(); + res = SPAM(left, right); + INPUTS_DEAD(); + } """ output = """ @@ -208,14 +391,15 @@ def test_binary_op(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - spam(); + res = SPAM(left, right); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -224,7 +408,8 @@ def test_binary_op(self): def test_overlap(self): input = """ inst(OP, (left, right -- left, result)) { - spam(); + result = SPAM(left, right); + INPUTS_DEAD(); } """ output = """ @@ -232,25 +417,26 @@ def test_overlap(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *right; - PyObject *left; - PyObject *result; + _PyStackRef left; + _PyStackRef right; + _PyStackRef result; right = stack_pointer[-1]; left = stack_pointer[-2]; - spam(); + result = SPAM(left, right); stack_pointer[-1] = result; DISPATCH(); } """ self.run_cases_test(input, output) - def test_predictions_and_eval_breaker(self): + def test_predictions(self): input = """ - inst(OP1, (arg -- rest)) { + inst(OP1, (arg -- res)) { + res = Py_None; } inst(OP3, (arg -- res)) { DEOPT_IF(xxx); - CHECK_EVAL_BREAKER(); + res = Py_None; } family(OP1, INLINE_CACHE_ENTRIES_OP1) = { OP3 }; """ @@ -259,11 +445,10 @@ def test_predictions_and_eval_breaker(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP1); - PREDICTED(OP1); - PyObject *arg; - PyObject *rest; - arg = stack_pointer[-1]; - stack_pointer[-1] = rest; + PREDICTED_OP1:; + _PyStackRef res; + res = Py_None; + stack_pointer[-1] = res; DISPATCH(); } @@ -272,17 +457,76 @@ def test_predictions_and_eval_breaker(self): next_instr += 1; INSTRUCTION_STATS(OP3); static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); - PyObject *arg; - PyObject *res; - arg = stack_pointer[-1]; + _PyStackRef res; DEOPT_IF(xxx, OP1); + res = Py_None; + stack_pointer[-1] = res; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_sync_sp(self): + input = """ + inst(A, (arg -- res)) { + SYNC_SP(); + escaping_call(); + res = Py_None; + } + inst(B, (arg -- res)) { + res = Py_None; + SYNC_SP(); + escaping_call(); + } + """ + output = """ + TARGET(A) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(A); + _PyStackRef res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = Py_None; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(B) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(B); + _PyStackRef res; + res = Py_None; stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); + _PyFrame_SetStackPointer(frame, stack_pointer); + escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } """ self.run_cases_test(input, output) + + def test_pep7_condition(self): + input = """ + inst(OP, (arg1 -- out)) { + if (arg1) + out = 0; + else { + out = 1; + } + } + """ + output = "" + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + def test_error_if_plain(self): input = """ inst(OP, (--)) { @@ -321,6 +565,38 @@ def test_error_if_plain_with_comment(self): def test_error_if_pop(self): input = """ inst(OP, (left, right -- res)) { + SPAM(left, right); + INPUTS_DEAD(); + ERROR_IF(cond, label); + res = 0; + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + SPAM(left, right); + if (cond) goto pop_2_label; + res = 0; + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_error_if_pop_with_result(self): + input = """ + inst(OP, (left, right -- res)) { + res = SPAM(left, right); + INPUTS_DEAD(); ERROR_IF(cond, label); } """ @@ -329,14 +605,16 @@ def test_error_if_pop(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + res = SPAM(left, right); if (cond) goto pop_2_label; stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -349,17 +627,16 @@ def test_cache_effect(self): """ output = """ TARGET(OP) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; INSTRUCTION_STATS(OP); - PyObject *value; - value = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; uint32_t extra = read_u32(&this_instr[2].cache); (void)extra; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -388,10 +665,12 @@ def test_macro_instruction(self): } op(OP2, (extra/2, arg2, left, right -- res)) { res = op2(arg2, left, right); + INPUTS_DEAD(); } macro(OP) = OP1 + cache/2 + OP2; inst(OP3, (unused/5, arg2, left, right -- res)) { res = op3(arg2, left, right); + INPUTS_DEAD(); } family(OP, INLINE_CACHE_ENTRIES_OP) = { OP3 }; """ @@ -400,46 +679,53 @@ def test_macro_instruction(self): frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(OP); - PREDICTED(OP); - _Py_CODEUNIT *this_instr = next_instr - 6; + PREDICTED_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 6; (void)this_instr; - PyObject *right; - PyObject *left; - PyObject *arg2; - PyObject *res; + _PyStackRef left; + _PyStackRef right; + _PyStackRef arg2; + _PyStackRef res; // _OP1 - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; + _PyFrame_SetStackPointer(frame, stack_pointer); op1(left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); } /* Skip 2 cache entries */ // OP2 - arg2 = stack_pointer[-3]; { + arg2 = stack_pointer[-3]; uint32_t extra = read_u32(&this_instr[4].cache); (void)extra; + _PyFrame_SetStackPointer(frame, stack_pointer); res = op2(arg2, left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer[-3] = res; stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(OP1) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(OP1); - PyObject *right; - PyObject *left; + _PyStackRef left; + _PyStackRef right; right = stack_pointer[-1]; left = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; + _PyFrame_SetStackPointer(frame, stack_pointer); op1(left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -448,17 +734,20 @@ def test_macro_instruction(self): next_instr += 6; INSTRUCTION_STATS(OP3); static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *arg2; - PyObject *res; + _PyStackRef arg2; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; /* Skip 5 cache entries */ right = stack_pointer[-1]; left = stack_pointer[-2]; arg2 = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); res = op3(arg2, left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer[-3] = res; stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -467,7 +756,7 @@ def test_macro_instruction(self): def test_unused_caches(self): input = """ inst(OP, (unused/1, unused/2 --)) { - body(); + body; } """ output = """ @@ -477,7 +766,7 @@ def test_unused_caches(self): INSTRUCTION_STATS(OP); /* Skip 1 cache entry */ /* Skip 2 cache entries */ - body(); + body; DISPATCH(); } """ @@ -485,7 +774,7 @@ def test_unused_caches(self): def test_pseudo_instruction_no_flags(self): input = """ - pseudo(OP) = { + pseudo(OP, (in -- out1, out2)) = { OP1, }; @@ -504,7 +793,7 @@ def test_pseudo_instruction_no_flags(self): def test_pseudo_instruction_with_flags(self): input = """ - pseudo(OP, (HAS_ARG, HAS_JUMP)) = { + pseudo(OP, (in1, in2 --), (HAS_ARG, HAS_JUMP)) = { OP1, }; @@ -521,10 +810,40 @@ def test_pseudo_instruction_with_flags(self): """ self.run_cases_test(input, output) + def test_pseudo_instruction_as_sequence(self): + input = """ + pseudo(OP, (in -- out1, out2)) = [ + OP1, OP2 + ]; + + inst(OP1, (--)) { + } + + inst(OP2, (--)) { + } + """ + output = """ + TARGET(OP1) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + DISPATCH(); + } + + TARGET(OP2) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP2); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_array_input(self): input = """ inst(OP, (below, values[oparg*2], above --)) { - spam(); + SPAM(values, oparg); } """ output = """ @@ -532,14 +851,11 @@ def test_array_input(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *above; - PyObject **values; - PyObject *below; - above = stack_pointer[-1]; + _PyStackRef *values; values = &stack_pointer[-1 - oparg*2]; - below = stack_pointer[-2 - oparg*2]; - spam(); + SPAM(values, oparg); stack_pointer += -2 - oparg*2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -548,7 +864,9 @@ def test_array_input(self): def test_array_output(self): input = """ inst(OP, (unused, unused -- below, values[oparg*3], above)) { - spam(values, oparg); + SPAM(values, oparg); + below = 0; + above = 0; } """ output = """ @@ -556,14 +874,17 @@ def test_array_output(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *below; - PyObject **values; - PyObject *above; + _PyStackRef below; + _PyStackRef *values; + _PyStackRef above; values = &stack_pointer[-1]; - spam(values, oparg); + SPAM(values, oparg); + below = 0; + above = 0; stack_pointer[-2] = below; stack_pointer[-1 + oparg*3] = above; stack_pointer += oparg*3; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -572,7 +893,8 @@ def test_array_output(self): def test_array_input_output(self): input = """ inst(OP, (values[oparg] -- values[oparg], above)) { - spam(values, oparg); + SPAM(values, oparg); + above = 0; } """ output = """ @@ -580,12 +902,14 @@ def test_array_input_output(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject **values; - PyObject *above; + _PyStackRef *values; + _PyStackRef above; values = &stack_pointer[-oparg]; - spam(values, oparg); + SPAM(values, oparg); + above = 0; stack_pointer[0] = above; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -602,12 +926,13 @@ def test_array_error_if(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject **values; - PyObject *extra; - values = &stack_pointer[-oparg]; - extra = stack_pointer[-1 - oparg]; - if (oparg == 0) { stack_pointer += -1 - oparg; goto somewhere; } + if (oparg == 0) { + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto somewhere; + } stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -616,7 +941,10 @@ def test_array_error_if(self): def test_cond_effect(self): input = """ inst(OP, (aa, input if ((oparg & 1) == 1), cc -- xx, output if (oparg & 2), zz)) { - output = spam(oparg, input); + output = SPAM(oparg, aa, cc, input); + INPUTS_DEAD(); + xx = 0; + zz = 0; } """ output = """ @@ -624,20 +952,23 @@ def test_cond_effect(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *cc; - PyObject *input = NULL; - PyObject *aa; - PyObject *xx; - PyObject *output = NULL; - PyObject *zz; + _PyStackRef aa; + _PyStackRef input = PyStackRef_NULL; + _PyStackRef cc; + _PyStackRef xx; + _PyStackRef output = PyStackRef_NULL; + _PyStackRef zz; cc = stack_pointer[-1]; if ((oparg & 1) == 1) { input = stack_pointer[-1 - (((oparg & 1) == 1) ? 1 : 0)]; } aa = stack_pointer[-2 - (((oparg & 1) == 1) ? 1 : 0)]; - output = spam(oparg, input); + output = SPAM(oparg, aa, cc, input); + xx = 0; + zz = 0; stack_pointer[-2 - (((oparg & 1) == 1) ? 1 : 0)] = xx; if (oparg & 2) stack_pointer[-1 - (((oparg & 1) == 1) ? 1 : 0)] = output; stack_pointer[-1 - (((oparg & 1) == 1) ? 1 : 0) + ((oparg & 2) ? 1 : 0)] = zz; stack_pointer += -(((oparg & 1) == 1) ? 1 : 0) + ((oparg & 2) ? 1 : 0); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -646,10 +977,14 @@ def test_cond_effect(self): def test_macro_cond_effect(self): input = """ op(A, (left, middle, right --)) { - # Body of A + USE(left, middle, right); + INPUTS_DEAD(); } op(B, (-- deep, extra if (oparg), res)) { - # Body of B + deep = -1; + res = 0; + extra = 1; + INPUTS_DEAD(); } macro(M) = A + B; """ @@ -658,27 +993,30 @@ def test_macro_cond_effect(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); - PyObject *right; - PyObject *middle; - PyObject *left; - PyObject *deep; - PyObject *extra = NULL; - PyObject *res; + _PyStackRef left; + _PyStackRef middle; + _PyStackRef right; + _PyStackRef deep; + _PyStackRef extra = PyStackRef_NULL; + _PyStackRef res; // A - right = stack_pointer[-1]; - middle = stack_pointer[-2]; - left = stack_pointer[-3]; { - # Body of A + right = stack_pointer[-1]; + middle = stack_pointer[-2]; + left = stack_pointer[-3]; + USE(left, middle, right); } // B { - # Body of B + deep = -1; + res = 0; + extra = 1; } stack_pointer[-3] = deep; if (oparg) stack_pointer[-2] = extra; stack_pointer[-2 + ((oparg) ? 1 : 0)] = res; stack_pointer += -1 + ((oparg) ? 1 : 0); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -687,10 +1025,10 @@ def test_macro_cond_effect(self): def test_macro_push_push(self): input = """ op(A, (-- val1)) { - val1 = spam(); + val1 = SPAM(); } op(B, (-- val2)) { - val2 = spam(); + val2 = SPAM(); } macro(M) = A + B; """ @@ -699,19 +1037,20 @@ def test_macro_push_push(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); - PyObject *val1; - PyObject *val2; + _PyStackRef val1; + _PyStackRef val2; // A { - val1 = spam(); + val1 = SPAM(); } // B { - val2 = spam(); + val2 = SPAM(); } stack_pointer[0] = val1; stack_pointer[1] = val2; stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ @@ -720,10 +1059,10 @@ def test_macro_push_push(self): def test_override_inst(self): input = """ inst(OP, (--)) { - spam(); + spam; } override inst(OP, (--)) { - ham(); + ham; } """ output = """ @@ -731,7 +1070,7 @@ def test_override_inst(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - ham(); + ham; DISPATCH(); } """ @@ -740,11 +1079,11 @@ def test_override_inst(self): def test_override_op(self): input = """ op(OP, (--)) { - spam(); + spam; } macro(M) = OP; override op(OP, (--)) { - ham(); + ham; } """ output = """ @@ -752,7 +1091,7 @@ def test_override_op(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); - ham(); + ham; DISPATCH(); } """ @@ -761,7 +1100,7 @@ def test_override_op(self): def test_annotated_inst(self): input = """ pure inst(OP, (--)) { - ham(); + ham; } """ output = """ @@ -769,7 +1108,7 @@ def test_annotated_inst(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - ham(); + ham; DISPATCH(); } """ @@ -778,7 +1117,7 @@ def test_annotated_inst(self): def test_annotated_op(self): input = """ pure op(OP, (--)) { - spam(); + SPAM(); } macro(M) = OP; """ @@ -787,7 +1126,7 @@ def test_annotated_op(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); - spam(); + SPAM(); DISPATCH(); } """ @@ -795,13 +1134,12 @@ def test_annotated_op(self): input = """ pure register specializing op(OP, (--)) { - spam(); + SPAM(); } macro(M) = OP; """ self.run_cases_test(input, output) - def test_deopt_and_exit(self): input = """ pure op(OP, (arg1 -- out)) { @@ -810,106 +1148,766 @@ def test_deopt_and_exit(self): } """ output = "" - with self.assertRaises(Exception): + with self.assertRaises(SyntaxError): self.run_cases_test(input, output) -class TestGeneratedAbstractCases(unittest.TestCase): - def setUp(self) -> None: - super().setUp() - self.maxDiff = None - - self.temp_dir = tempfile.gettempdir() - self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") - self.temp_input2_filename = os.path.join(self.temp_dir, "input2.txt") - self.temp_output_filename = os.path.join(self.temp_dir, "output.txt") - - def tearDown(self) -> None: - for filename in [ - self.temp_input_filename, - self.temp_input2_filename, - self.temp_output_filename, - ]: - try: - os.remove(filename) - except: - pass - super().tearDown() - - def run_cases_test(self, input: str, input2: str, expected: str): - with open(self.temp_input_filename, "w+") as temp_input: - temp_input.write(parser.BEGIN_MARKER) - temp_input.write(input) - temp_input.write(parser.END_MARKER) - temp_input.flush() - - with open(self.temp_input2_filename, "w+") as temp_input: - temp_input.write(parser.BEGIN_MARKER) - temp_input.write(input2) - temp_input.write(parser.END_MARKER) - temp_input.flush() - - with handle_stderr(): - optimizer_generator.generate_tier2_abstract_from_files( - [self.temp_input_filename, self.temp_input2_filename], - self.temp_output_filename - ) - - with open(self.temp_output_filename) as temp_output: - lines = temp_output.readlines() - while lines and lines[0].startswith(("// ", "#", " #", "\n")): - lines.pop(0) - while lines and lines[-1].startswith(("#", "\n")): - lines.pop(-1) - actual = "".join(lines) - self.assertEqual(actual.strip(), expected.strip()) - - def test_overridden_abstract(self): + def test_array_of_one(self): input = """ - pure op(OP, (--)) { - spam(); - } - """ - input2 = """ - pure op(OP, (--)) { - eggs(); + inst(OP, (arg[1] -- out[1])) { + out[0] = arg[0]; } """ output = """ - case OP: { - eggs(); - break; + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef *arg; + _PyStackRef *out; + arg = &stack_pointer[-1]; + out = &stack_pointer[-1]; + out[0] = arg[0]; + DISPATCH(); } """ - self.run_cases_test(input, input2, output) + self.run_cases_test(input, output) - def test_overridden_abstract_args(self): + def test_pointer_to_stackref(self): input = """ - pure op(OP, (arg1 -- out)) { - spam(); - } - op(OP2, (arg1 -- out)) { - eggs(); - } - """ - input2 = """ - op(OP, (arg1 -- out)) { - eggs(); + inst(OP, (arg: _PyStackRef * -- out)) { + out = *arg; + DEAD(arg); } """ output = """ - case OP: { - _Py_UopsSymbol *arg1; - _Py_UopsSymbol *out; - arg1 = stack_pointer[-1]; - eggs(); + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef *arg; + _PyStackRef out; + arg = (_PyStackRef *)stack_pointer[-1].bits; + out = *arg; stack_pointer[-1] = out; - break; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_unused_cached_value(self): + input = """ + op(FIRST, (arg1 -- out)) { + out = arg1; + } + + op(SECOND, (unused -- unused)) { + } + + macro(BOTH) = FIRST + SECOND; + """ + output = """ + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_unused_named_values(self): + input = """ + op(OP, (named -- named)) { + } + + macro(INST) = OP; + """ + output = """ + TARGET(INST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INST); + DISPATCH(); + } + + """ + self.run_cases_test(input, output) + + def test_used_unused_used(self): + input = """ + op(FIRST, (w -- w)) { + USE(w); + } + + op(SECOND, (x -- x)) { + } + + op(THIRD, (y -- y)) { + USE(y); + } + + macro(TEST) = FIRST + SECOND + THIRD; + """ + output = """ + TARGET(TEST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef w; + _PyStackRef y; + // FIRST + { + w = stack_pointer[-1]; + USE(w); + } + // SECOND + { + } + // THIRD + { + y = w; + USE(y); + } + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_unused_used_used(self): + input = """ + op(FIRST, (w -- w)) { + } + + op(SECOND, (x -- x)) { + USE(x); + } + + op(THIRD, (y -- y)) { + USE(y); + } + + macro(TEST) = FIRST + SECOND + THIRD; + """ + output = """ + TARGET(TEST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef x; + _PyStackRef y; + // FIRST + { + } + // SECOND + { + x = stack_pointer[-1]; + USE(x); + } + // THIRD + { + y = x; + USE(y); + } + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_flush(self): + input = """ + op(FIRST, ( -- a, b)) { + a = 0; + b = 1; + } + + op(SECOND, (a, b -- )) { + USE(a, b); + INPUTS_DEAD(); + } + + macro(TEST) = FIRST + flush + SECOND; + """ + output = """ + TARGET(TEST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef a; + _PyStackRef b; + // FIRST + { + a = 0; + b = 1; + } + // flush + stack_pointer[0] = a; + stack_pointer[1] = b; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + // SECOND + { + USE(a, b); + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pop_on_error_peeks(self): + + input = """ + op(FIRST, (x, y -- a, b)) { + a = x; + DEAD(x); + b = y; + DEAD(y); + } + + op(SECOND, (a, b -- a, b)) { + } + + op(THIRD, (j, k --)) { + INPUTS_DEAD(); // Mark j and k as used + ERROR_IF(cond, error); + } + + macro(TEST) = FIRST + SECOND + THIRD; + """ + output = """ + TARGET(TEST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef x; + _PyStackRef y; + _PyStackRef a; + _PyStackRef b; + // FIRST + { + y = stack_pointer[-1]; + x = stack_pointer[-2]; + a = x; + b = y; + } + // SECOND + { + } + // THIRD + { + // Mark j and k as used + if (cond) goto pop_2_error; + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_push_then_error(self): + + input = """ + op(FIRST, ( -- a)) { + a = 1; + } + + op(SECOND, (a -- a, b)) { + b = 1; + ERROR_IF(cond, error); + } + + macro(TEST) = FIRST + SECOND; + """ + + output = """ + TARGET(TEST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef a; + _PyStackRef b; + // FIRST + { + a = 1; + } + // SECOND + { + b = 1; + if (cond) { + stack_pointer[0] = a; + stack_pointer[1] = b; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + stack_pointer[0] = a; + stack_pointer[1] = b; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_error_if_true(self): + + input = """ + inst(OP1, ( --)) { + ERROR_IF(true, here); + } + inst(OP2, ( --)) { + ERROR_IF(1, there); + } + """ + output = """ + TARGET(OP1) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + goto here; + } + + TARGET(OP2) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP2); + goto there; + } + """ + self.run_cases_test(input, output) + + def test_scalar_array_inconsistency(self): + + input = """ + op(FIRST, ( -- a)) { + a = 1; + } + + op(SECOND, (a[1] -- b)) { + b = 1; + } + + macro(TEST) = FIRST + SECOND; + """ + + output = """ + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_array_size_inconsistency(self): + + input = """ + op(FIRST, ( -- a[2])) { + a[0] = 1; + } + + op(SECOND, (a[1] -- b)) { + b = 1; + } + + macro(TEST) = FIRST + SECOND; + """ + + output = """ + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_stack_save_reload(self): + + input = """ + inst(BALANCED, ( -- )) { + SAVE_STACK(); + RELOAD_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + _PyFrame_SetStackPointer(frame, stack_pointer); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_stack_reload_only(self): + + input = """ + inst(BALANCED, ( -- )) { + RELOAD_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + _PyFrame_SetStackPointer(frame, stack_pointer); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_stack_save_only(self): + + input = """ + inst(BALANCED, ( -- )) { + SAVE_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + _PyFrame_SetStackPointer(frame, stack_pointer); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_instruction_size_macro(self): + input = """ + inst(OP, (--)) { + frame->return_offset = INSTRUCTION_SIZE; + } + """ + + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + frame->return_offset = 1 ; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + # Two instructions of different sizes referencing the same + # uop containing the `INSTRUCTION_SIZE` macro is not allowed. + input = """ + inst(OP, (--)) { + frame->return_offset = INSTRUCTION_SIZE; + } + macro(OP2) = unused/1 + OP; + """ + + output = "" # No output needed as this should raise an error. + with self.assertRaisesRegex(SyntaxError, "All instructions containing a uop"): + self.run_cases_test(input, output) + + def test_escaping_call_next_to_cmacro(self): + input = """ + inst(OP, (--)) { + #ifdef Py_GIL_DISABLED + escaping_call(); + #else + another_escaping_call(); + #endif + yet_another_escaping_call(); + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + #else + _PyFrame_SetStackPointer(frame, stack_pointer); + another_escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + #endif + _PyFrame_SetStackPointer(frame, stack_pointer); + yet_another_escaping_call(); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pystackref_frompyobject_new_next_to_cmacro(self): + input = """ + inst(OP, (-- out1, out2)) { + PyObject *obj = SPAM(); + #ifdef Py_GIL_DISABLED + out1 = PyStackRef_FromPyObjectNew(obj); + #else + out1 = PyStackRef_FromPyObjectNew(obj); + #endif + out2 = PyStackRef_FromPyObjectNew(obj); + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef out1; + _PyStackRef out2; + PyObject *obj = SPAM(); + #ifdef Py_GIL_DISABLED + out1 = PyStackRef_FromPyObjectNew(obj); + #else + out1 = PyStackRef_FromPyObjectNew(obj); + #endif + out2 = PyStackRef_FromPyObjectNew(obj); + stack_pointer[0] = out1; + stack_pointer[1] = out2; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pop_input(self): + input = """ + inst(OP, (a, b --)) { + POP_INPUT(b); + HAM(a); + INPUTS_DEAD(); + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef a; + _PyStackRef b; + b = stack_pointer[-1]; + a = stack_pointer[-2]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + HAM(a); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pop_input_with_empty_stack(self): + input = """ + inst(OP, (--)) { + POP_INPUT(foo); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + def test_pop_input_with_non_tos(self): + input = """ + inst(OP, (a, b --)) { + POP_INPUT(a); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + def test_no_escaping_calls_in_branching_macros(self): + + input = """ + inst(OP, ( -- )) { + DEOPT_IF(escaping_call()); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + input = """ + inst(OP, ( -- )) { + EXIT_IF(escaping_call()); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + input = """ + inst(OP, ( -- )) { + ERROR_IF(escaping_call(), error); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + def test_kill_in_wrong_order(self): + input = """ + inst(OP, (a, b -- c)) { + c = b; + PyStackRef_CLOSE(a); + PyStackRef_CLOSE(b); + } + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + def test_complex_label(self): + input = """ + label(my_label) { + // Comment + do_thing() + if (complex) { + goto other_label; + } + goto other_label2; + } + """ + + output = """ + my_label: + { + // Comment + do_thing() + if (complex) { + goto other_label; + } + goto other_label2; + } + """ + self.run_cases_test(input, output) + + def test_multiple_labels(self): + input = """ + label(my_label_1) { + // Comment + do_thing1(); + goto my_label_2; + } + + label(my_label_2) { + // Comment + do_thing2(); + goto my_label_3; + } + """ + + output = """ + my_label_1: + { + // Comment + do_thing1(); + goto my_label_2; + } + + my_label_2: + { + // Comment + do_thing2(); + goto my_label_3; + } + """ + +class TestGeneratedAbstractCases(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.maxDiff = None + + self.temp_dir = tempfile.gettempdir() + self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") + self.temp_input2_filename = os.path.join(self.temp_dir, "input2.txt") + self.temp_output_filename = os.path.join(self.temp_dir, "output.txt") + + def tearDown(self) -> None: + for filename in [ + self.temp_input_filename, + self.temp_input2_filename, + self.temp_output_filename, + ]: + try: + os.remove(filename) + except: + pass + super().tearDown() + + def run_cases_test(self, input: str, input2: str, expected: str): + with open(self.temp_input_filename, "w+") as temp_input: + temp_input.write(parser.BEGIN_MARKER) + temp_input.write(input) + temp_input.write(parser.END_MARKER) + temp_input.flush() + + with open(self.temp_input2_filename, "w+") as temp_input: + temp_input.write(parser.BEGIN_MARKER) + temp_input.write(input2) + temp_input.write(parser.END_MARKER) + temp_input.flush() + + with handle_stderr(): + optimizer_generator.generate_tier2_abstract_from_files( + [self.temp_input_filename, self.temp_input2_filename], + self.temp_output_filename + ) + + with open(self.temp_output_filename) as temp_output: + lines = temp_output.readlines() + while lines and lines[0].startswith(("// ", "#", " #", "\n")): + lines.pop(0) + while lines and lines[-1].startswith(("#", "\n")): + lines.pop(-1) + actual = "".join(lines) + self.assertEqual(actual.strip(), expected.strip()) + + def test_overridden_abstract(self): + input = """ + pure op(OP, (--)) { + SPAM(); + } + """ + input2 = """ + pure op(OP, (--)) { + eggs(); + } + """ + output = """ + case OP: { + eggs(); + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_overridden_abstract_args(self): + input = """ + pure op(OP, (arg1 -- out)) { + out = SPAM(arg1); + } + op(OP2, (arg1 -- out)) { + out = EGGS(arg1); + } + """ + input2 = """ + op(OP, (arg1 -- out)) { + out = EGGS(arg1); + } + """ + output = """ + case OP: { + JitOptSymbol *arg1; + JitOptSymbol *out; + arg1 = stack_pointer[-1]; + out = EGGS(arg1); + stack_pointer[-1] = out; + break; } case OP2: { - _Py_UopsSymbol *out; - out = sym_new_unknown(ctx); - if (out == NULL) goto out_of_space; + JitOptSymbol *out; + out = sym_new_not_null(ctx); stack_pointer[-1] = out; break; } @@ -919,7 +1917,7 @@ def test_overridden_abstract_args(self): def test_no_overridden_case(self): input = """ pure op(OP, (arg1 -- out)) { - spam(); + out = SPAM(arg1); } pure op(OP2, (arg1 -- out)) { @@ -928,21 +1926,20 @@ def test_no_overridden_case(self): """ input2 = """ pure op(OP2, (arg1 -- out)) { + out = NULL; } """ output = """ case OP: { - _Py_UopsSymbol *out; - out = sym_new_unknown(ctx); - if (out == NULL) goto out_of_space; + JitOptSymbol *out; + out = sym_new_not_null(ctx); stack_pointer[-1] = out; break; } case OP2: { - _Py_UopsSymbol *arg1; - _Py_UopsSymbol *out; - arg1 = stack_pointer[-1]; + JitOptSymbol *out; + out = NULL; stack_pointer[-1] = out; break; } @@ -952,7 +1949,7 @@ def test_no_overridden_case(self): def test_missing_override_failure(self): input = """ pure op(OP, (arg1 -- out)) { - spam(); + SPAM(); } """ input2 = """ diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index d48f0d47ba1962..b6985054c33d10 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -6,6 +6,8 @@ import unittest import weakref import inspect +import textwrap +import types from test import support @@ -89,9 +91,12 @@ def gen(): self.assertEqual(gc.garbage, old_garbage) def test_lambda_generator(self): - # Issue #23192: Test that a lambda returning a generator behaves + # bpo-23192, gh-119897: Test that a lambda returning a generator behaves # like the equivalent function f = lambda: (yield 1) + self.assertIsInstance(f(), types.GeneratorType) + self.assertEqual(next(f()), 1) + def g(): return (yield 1) # test 'yield from' @@ -108,6 +113,27 @@ def g3(): return (yield from f()) gen.send(2) self.assertEqual(cm.exception.value, 2) + def test_generator_resurrect(self): + # Test that a resurrected generator still has a valid gi_code + resurrected = [] + + # Resurrect a generator in a finalizer + exec(textwrap.dedent(""" + def gen(): + try: + yield + except: + resurrected.append(g) + + g = gen() + next(g) + """), {"resurrected": resurrected}) + + support.gc_collect() + + self.assertEqual(len(resurrected), 1) + self.assertIsInstance(resurrected[0].gi_code, types.CodeType) + class GeneratorTest(unittest.TestCase): @@ -242,6 +268,79 @@ def loop(): #This should not raise loop() + +class ModifyUnderlyingIterableTest(unittest.TestCase): + iterables = [ + range(0), + range(20), + [1, 2, 3], + (2,), + {13, 48, 211}, + frozenset((15, 8, 6)), + {1: 2, 3: 4}, + ] + + non_iterables = [ + None, + 42, + 3.0, + 2j, + ] + + def genexpr(self): + return (x for x in range(10)) + + def genfunc(self): + def gen(it): + for x in it: + yield x + return gen(range(10)) + + def process_tests(self, get_generator): + for obj in self.iterables: + g_obj = get_generator(obj) + with self.subTest(g_obj=g_obj, obj=obj): + self.assertListEqual(list(g_obj), list(obj)) + + g_iter = get_generator(iter(obj)) + with self.subTest(g_iter=g_iter, obj=obj): + self.assertListEqual(list(g_iter), list(obj)) + + err_regex = "'.*' object is not iterable" + for obj in self.non_iterables: + g_obj = get_generator(obj) + with self.subTest(g_obj=g_obj): + self.assertRaisesRegex(TypeError, err_regex, list, g_obj) + + def test_modify_f_locals(self): + def modify_f_locals(g, local, obj): + g.gi_frame.f_locals[local] = obj + return g + + def get_generator_genexpr(obj): + return modify_f_locals(self.genexpr(), '.0', obj) + + def get_generator_genfunc(obj): + return modify_f_locals(self.genfunc(), 'it', obj) + + self.process_tests(get_generator_genexpr) + self.process_tests(get_generator_genfunc) + + def test_new_gen_from_gi_code(self): + def new_gen_from_gi_code(g, obj): + generator_func = types.FunctionType(g.gi_code, {}) + return generator_func(obj) + + def get_generator_genexpr(obj): + return new_gen_from_gi_code(self.genexpr(), obj) + + def get_generator_genfunc(obj): + return new_gen_from_gi_code(self.genfunc(), obj) + + self.process_tests(get_generator_genexpr) + self.process_tests(get_generator_genfunc) + + class ExceptionTest(unittest.TestCase): # Tests for the issue #23353: check that the currently handled exception # is correctly saved/restored in PyEval_EvalFrameEx(). @@ -532,6 +631,109 @@ def f(): with self.assertRaises(RuntimeError): gen.close() + def test_close_releases_frame_locals(self): + # See gh-118272 + + class Foo: + pass + + f = Foo() + f_wr = weakref.ref(f) + + def genfn(): + a = f + yield + + g = genfn() + next(g) + del f + g.close() + support.gc_collect() + self.assertIsNone(f_wr()) + + +# See https://github.com/python/cpython/issues/125723 +class GeneratorDeallocTest(unittest.TestCase): + def test_frame_outlives_generator(self): + def g1(): + a = 42 + yield sys._getframe() + + def g2(): + a = 42 + yield + + def g3(obj): + a = 42 + obj.frame = sys._getframe() + yield + + class ObjectWithFrame(): + def __init__(self): + self.frame = None + + def get_frame(index): + if index == 1: + return next(g1()) + elif index == 2: + gen = g2() + next(gen) + return gen.gi_frame + elif index == 3: + obj = ObjectWithFrame() + next(g3(obj)) + return obj.frame + else: + return None + + for index in (1, 2, 3): + with self.subTest(index=index): + frame = get_frame(index) + frame_locals = frame.f_locals + self.assertIn('a', frame_locals) + self.assertEqual(frame_locals['a'], 42) + + def test_frame_locals_outlive_generator(self): + frame_locals1 = None + + def g1(): + nonlocal frame_locals1 + frame_locals1 = sys._getframe().f_locals + a = 42 + yield + + def g2(): + a = 42 + yield sys._getframe().f_locals + + def get_frame_locals(index): + if index == 1: + nonlocal frame_locals1 + next(g1()) + return frame_locals1 + if index == 2: + return next(g2()) + else: + return None + + for index in (1, 2): + with self.subTest(index=index): + frame_locals = get_frame_locals(index) + self.assertIn('a', frame_locals) + self.assertEqual(frame_locals['a'], 42) + + def test_frame_locals_outlive_generator_with_exec(self): + def g(): + a = 42 + yield locals(), sys._getframe().f_locals + + locals_ = {'g': g} + for i in range(10): + exec("snapshot, live_locals = next(g())", locals=locals_) + for l in (locals_['snapshot'], locals_['live_locals']): + self.assertIn('a', l) + self.assertEqual(l['a'], 42) + class GeneratorThrowTest(unittest.TestCase): @@ -639,7 +841,8 @@ def check_stack_names(self, frame, expected): while frame: name = frame.f_code.co_name # Stop checking frames when we get to our test helper. - if name.startswith('check_') or name.startswith('call_'): + if (name.startswith('check_') or name.startswith('call_') + or name.startswith('test')): break names.append(name) @@ -680,6 +883,25 @@ def call_throw(gen): self.check_yield_from_example(call_throw) + def test_throw_with_yield_from_custom_generator(self): + + class CustomGen: + def __init__(self, test): + self.test = test + def throw(self, *args): + self.test.check_stack_names(sys._getframe(), ['throw', 'g']) + def __iter__(self): + return self + def __next__(self): + return 42 + + def g(target): + yield from target + + gen = g(CustomGen(self)) + gen.send(None) + gen.throw(RuntimeError) + class YieldFromTests(unittest.TestCase): def test_generator_gi_yieldfrom(self): @@ -887,7 +1109,7 @@ def b(): File "", line 1, in ? File "", line 2, in g File "", line 2, in f - ZeroDivisionError: integer division or modulo by zero + ZeroDivisionError: division by zero >>> next(k) # and the generator cannot be resumed Traceback (most recent call last): File "", line 1, in ? @@ -2223,6 +2445,16 @@ def printsolution(self, x): ... SyntaxError: 'yield' outside function +>>> f=lambda: (yield from (1,2)), (yield from (3,4)) +Traceback (most recent call last): + ... +SyntaxError: 'yield from' outside function + +>>> yield from [1,2] +Traceback (most recent call last): + ... +SyntaxError: 'yield from' outside function + >>> def f(): x = yield = y Traceback (most recent call last): ... diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 04cb810d9babbf..3048d038c782d4 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -49,7 +49,7 @@ ShareableList = None from os import DirEntry from re import Pattern, Match -from types import GenericAlias, MappingProxyType, AsyncGeneratorType +from types import GenericAlias, MappingProxyType, AsyncGeneratorType, CoroutineType, GeneratorType from tempfile import TemporaryDirectory, SpooledTemporaryFile from urllib.parse import SplitResult, ParseResult from unittest.case import _AssertRaisesContext @@ -57,6 +57,10 @@ from weakref import WeakSet, ReferenceType, ref import typing from typing import Unpack +try: + from tkinter import Event +except ImportError: + Event = None from typing import TypeVar T = TypeVar('T') @@ -96,7 +100,7 @@ class BaseTest(unittest.TestCase): """Test basics.""" - generic_types = [type, tuple, list, dict, set, frozenset, enumerate, + generic_types = [type, tuple, list, dict, set, frozenset, enumerate, memoryview, defaultdict, deque, SequenceMatcher, dircmp, @@ -120,6 +124,7 @@ class BaseTest(unittest.TestCase): KeysView, ItemsView, ValuesView, Sequence, MutableSequence, MappingProxyType, AsyncGeneratorType, + GeneratorType, CoroutineType, DirEntry, chain, LoggerAdapter, StreamHandler, @@ -138,6 +143,8 @@ class BaseTest(unittest.TestCase): if ValueProxy is not None: generic_types.extend((ValueProxy, DictProxy, ListProxy, ApplyResult, MPSimpleQueue, MPQueue, MPJoinableQueue)) + if Event is not None: + generic_types.append(Event) def test_subscriptable(self): for t in self.generic_types: diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index aece757fc1933e..e530b463966819 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -1,5 +1,6 @@ import unittest from test import support +from test.support.import_helper import import_module class TestMROEntry(unittest.TestCase): @@ -277,7 +278,9 @@ def __class_getitem__(cls, item): class CAPITest(unittest.TestCase): def test_c_class(self): - from _testcapi import Generic, GenericAlias + _testcapi = import_module("_testcapi") + Generic = _testcapi.Generic + GenericAlias = _testcapi.GenericAlias self.assertIsInstance(Generic.__class_getitem__(int), GenericAlias) IntGeneric = Generic[int] diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index f407ee3caf154c..391158b855624d 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -135,6 +135,9 @@ def test_exists(self): self.assertIs(self.pathmodule.exists(filename), False) self.assertIs(self.pathmodule.exists(bfilename), False) + self.assertIs(self.pathmodule.lexists(filename), False) + self.assertIs(self.pathmodule.lexists(bfilename), False) + create_file(filename) self.assertIs(self.pathmodule.exists(filename), True) @@ -145,17 +148,20 @@ def test_exists(self): self.assertIs(self.pathmodule.exists(filename + '\x00'), False) self.assertIs(self.pathmodule.exists(bfilename + b'\x00'), False) - if self.pathmodule is not genericpath: - self.assertIs(self.pathmodule.lexists(filename), True) - self.assertIs(self.pathmodule.lexists(bfilename), True) + self.assertIs(self.pathmodule.lexists(filename), True) + self.assertIs(self.pathmodule.lexists(bfilename), True) + + self.assertIs(self.pathmodule.lexists(filename + '\udfff'), False) + self.assertIs(self.pathmodule.lexists(bfilename + b'\xff'), False) + self.assertIs(self.pathmodule.lexists(filename + '\x00'), False) + self.assertIs(self.pathmodule.lexists(bfilename + b'\x00'), False) - self.assertIs(self.pathmodule.lexists(filename + '\udfff'), False) - self.assertIs(self.pathmodule.lexists(bfilename + b'\xff'), False) - self.assertIs(self.pathmodule.lexists(filename + '\x00'), False) - self.assertIs(self.pathmodule.lexists(bfilename + b'\x00'), False) + # Keyword arguments are accepted + self.assertIs(self.pathmodule.exists(path=filename), True) + self.assertIs(self.pathmodule.lexists(path=filename), True) @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - @unittest.skipIf(is_emscripten, "Emscripten pipe fds have no stat") + @unittest.skipIf(is_emscripten, "Fixed in next Emscripten release after 4.0.1") def test_exists_fd(self): r, w = os.pipe() try: diff --git a/Lib/test/test_genexps.py b/Lib/test/test_genexps.py index 4f2d3cdcc7943e..7fb58a67368576 100644 --- a/Lib/test/test_genexps.py +++ b/Lib/test/test_genexps.py @@ -223,7 +223,7 @@ next(g) File "", line 1, in g = (10 // i for i in (5, 0, 2)) - ZeroDivisionError: integer division or modulo by zero + ZeroDivisionError: division by zero >>> next(g) Traceback (most recent call last): File "", line 1, in -toplevel- diff --git a/Lib/test/test_getopt.py b/Lib/test/test_getopt.py index c8b3442de4aa77..ed967ad27619ae 100644 --- a/Lib/test/test_getopt.py +++ b/Lib/test/test_getopt.py @@ -1,11 +1,12 @@ # test_getopt.py # David Goodger 2000-08-19 -from test.support.os_helper import EnvironmentVarGuard import doctest -import unittest - import getopt +import sys +import unittest +from test.support.i18n_helper import TestTranslationsBase, update_translation_snapshots +from test.support.os_helper import EnvironmentVarGuard sentinel = object() @@ -19,21 +20,34 @@ def assertError(self, *args, **kwargs): self.assertRaises(getopt.GetoptError, *args, **kwargs) def test_short_has_arg(self): - self.assertTrue(getopt.short_has_arg('a', 'a:')) - self.assertFalse(getopt.short_has_arg('a', 'a')) + self.assertIs(getopt.short_has_arg('a', 'a:'), True) + self.assertIs(getopt.short_has_arg('a', 'a'), False) + self.assertEqual(getopt.short_has_arg('a', 'a::'), '?') self.assertError(getopt.short_has_arg, 'a', 'b') def test_long_has_args(self): has_arg, option = getopt.long_has_args('abc', ['abc=']) - self.assertTrue(has_arg) + self.assertIs(has_arg, True) self.assertEqual(option, 'abc') has_arg, option = getopt.long_has_args('abc', ['abc']) - self.assertFalse(has_arg) + self.assertIs(has_arg, False) self.assertEqual(option, 'abc') + has_arg, option = getopt.long_has_args('abc', ['abc=?']) + self.assertEqual(has_arg, '?') + self.assertEqual(option, 'abc') + + has_arg, option = getopt.long_has_args('abc', ['abcd=']) + self.assertIs(has_arg, True) + self.assertEqual(option, 'abcd') + has_arg, option = getopt.long_has_args('abc', ['abcd']) - self.assertFalse(has_arg) + self.assertIs(has_arg, False) + self.assertEqual(option, 'abcd') + + has_arg, option = getopt.long_has_args('abc', ['abcd=?']) + self.assertEqual(has_arg, '?') self.assertEqual(option, 'abcd') self.assertError(getopt.long_has_args, 'abc', ['def']) @@ -49,9 +63,9 @@ def test_do_shorts(self): self.assertEqual(opts, [('-a', '1')]) self.assertEqual(args, []) - #opts, args = getopt.do_shorts([], 'a=1', 'a:', []) - #self.assertEqual(opts, [('-a', '1')]) - #self.assertEqual(args, []) + opts, args = getopt.do_shorts([], 'a=1', 'a:', []) + self.assertEqual(opts, [('-a', '=1')]) + self.assertEqual(args, []) opts, args = getopt.do_shorts([], 'a', 'a:', ['1']) self.assertEqual(opts, [('-a', '1')]) @@ -61,6 +75,14 @@ def test_do_shorts(self): self.assertEqual(opts, [('-a', '1')]) self.assertEqual(args, ['2']) + opts, args = getopt.do_shorts([], 'a', 'a::', ['1']) + self.assertEqual(opts, [('-a', '')]) + self.assertEqual(args, ['1']) + + opts, args = getopt.do_shorts([], 'a1', 'a::', []) + self.assertEqual(opts, [('-a', '1')]) + self.assertEqual(args, []) + self.assertError(getopt.do_shorts, [], 'a1', 'a', []) self.assertError(getopt.do_shorts, [], 'a', 'a:', []) @@ -77,6 +99,22 @@ def test_do_longs(self): self.assertEqual(opts, [('--abcd', '1')]) self.assertEqual(args, []) + opts, args = getopt.do_longs([], 'abc', ['abc=?'], ['1']) + self.assertEqual(opts, [('--abc', '')]) + self.assertEqual(args, ['1']) + + opts, args = getopt.do_longs([], 'abc', ['abcd=?'], ['1']) + self.assertEqual(opts, [('--abcd', '')]) + self.assertEqual(args, ['1']) + + opts, args = getopt.do_longs([], 'abc=1', ['abc=?'], []) + self.assertEqual(opts, [('--abc', '1')]) + self.assertEqual(args, []) + + opts, args = getopt.do_longs([], 'abc=1', ['abcd=?'], []) + self.assertEqual(opts, [('--abcd', '1')]) + self.assertEqual(args, []) + opts, args = getopt.do_longs([], 'abc', ['ab', 'abc', 'abcd'], []) self.assertEqual(opts, [('--abc', '')]) self.assertEqual(args, []) @@ -95,7 +133,7 @@ def test_getopt(self): # note: the empty string between '-a' and '--beta' is significant: # it simulates an empty string option argument ('-a ""') on the # command line. - cmdline = ['-a', '1', '-b', '--alpha=2', '--beta', '-a', '3', '-a', + cmdline = ['-a1', '-b', '--alpha=2', '--beta', '-a', '3', '-a', '', '--beta', 'arg1', 'arg2'] opts, args = getopt.getopt(cmdline, 'a:b', ['alpha=', 'beta']) @@ -106,33 +144,53 @@ def test_getopt(self): # accounted for in the code that calls getopt(). self.assertEqual(args, ['arg1', 'arg2']) + cmdline = ['-a1', '--alpha=2', '--alpha=', '-a', '--alpha', 'arg1', 'arg2'] + opts, args = getopt.getopt(cmdline, 'a::', ['alpha=?']) + self.assertEqual(opts, [('-a', '1'), ('--alpha', '2'), ('--alpha', ''), + ('-a', ''), ('--alpha', '')]) + self.assertEqual(args, ['arg1', 'arg2']) + self.assertError(getopt.getopt, cmdline, 'a:b', ['alpha', 'beta']) def test_gnu_getopt(self): # Test handling of GNU style scanning mode. - cmdline = ['-a', 'arg1', '-b', '1', '--alpha', '--beta=2'] + cmdline = ['-a', 'arg1', '-b', '1', '--alpha', '--beta=2', '--beta', + '3', 'arg2'] # GNU style opts, args = getopt.gnu_getopt(cmdline, 'ab:', ['alpha', 'beta=']) - self.assertEqual(args, ['arg1']) - self.assertEqual(opts, [('-a', ''), ('-b', '1'), - ('--alpha', ''), ('--beta', '2')]) + self.assertEqual(args, ['arg1', 'arg2']) + self.assertEqual(opts, [('-a', ''), ('-b', '1'), ('--alpha', ''), + ('--beta', '2'), ('--beta', '3')]) + + opts, args = getopt.gnu_getopt(cmdline, 'ab::', ['alpha', 'beta=?']) + self.assertEqual(args, ['arg1', '1', '3', 'arg2']) + self.assertEqual(opts, [('-a', ''), ('-b', ''), ('--alpha', ''), + ('--beta', '2'), ('--beta', '')]) # recognize "-" as an argument opts, args = getopt.gnu_getopt(['-a', '-', '-b', '-'], 'ab:', []) self.assertEqual(args, ['-']) self.assertEqual(opts, [('-a', ''), ('-b', '-')]) + # Return positional arguments intermixed with options. + opts, args = getopt.gnu_getopt(cmdline, '-ab:', ['alpha', 'beta=']) + self.assertEqual(args, ['arg2']) + self.assertEqual(opts, [('-a', ''), (None, ['arg1']), ('-b', '1'), ('--alpha', ''), + ('--beta', '2'), ('--beta', '3')]) + # Posix style via + opts, args = getopt.gnu_getopt(cmdline, '+ab:', ['alpha', 'beta=']) self.assertEqual(opts, [('-a', '')]) - self.assertEqual(args, ['arg1', '-b', '1', '--alpha', '--beta=2']) + self.assertEqual(args, ['arg1', '-b', '1', '--alpha', '--beta=2', + '--beta', '3', 'arg2']) # Posix style via POSIXLY_CORRECT self.env["POSIXLY_CORRECT"] = "1" opts, args = getopt.gnu_getopt(cmdline, 'ab:', ['alpha', 'beta=']) self.assertEqual(opts, [('-a', '')]) - self.assertEqual(args, ['arg1', '-b', '1', '--alpha', '--beta=2']) + self.assertEqual(args, ['arg1', '-b', '1', '--alpha', '--beta=2', + '--beta', '3', 'arg2']) def test_issue4629(self): longopts, shortopts = getopt.getopt(['--help='], '', ['help=']) @@ -173,10 +231,20 @@ def test_libref_examples(): ['a1', 'a2'] """ + +class TestTranslations(TestTranslationsBase): + def test_translations(self): + self.assertMsgidsEqual(getopt) + + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite()) return tests -if __name__ == "__main__": +if __name__ == '__main__': + # To regenerate translation snapshots + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + update_translation_snapshots(getopt) + sys.exit(0) unittest.main() diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 2f7aa69efc184a..ca7cee0c39872a 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -1,7 +1,13 @@ import copy import ntpath +import os import pathlib import posixpath +import shutil +import subprocess +import sys +import sysconfig +import tempfile import unittest from test.support import verbose @@ -92,8 +98,8 @@ def test_venv_win32(self): ]) expected = dict( executable=r"C:\venv\Scripts\python.exe", - prefix=r"C:\Python", - exec_prefix=r"C:\Python", + prefix=r"C:\venv", + exec_prefix=r"C:\venv", base_executable=r"C:\Python\python.exe", base_prefix=r"C:\Python", base_exec_prefix=r"C:\Python", @@ -339,8 +345,8 @@ def test_venv_posix(self): ]) expected = dict( executable="/venv/bin/python", - prefix="/usr", - exec_prefix="/usr", + prefix="/venv", + exec_prefix="/venv", base_executable="/usr/bin/python", base_prefix="/usr", base_exec_prefix="/usr", @@ -371,8 +377,8 @@ def test_venv_changed_name_posix(self): ]) expected = dict( executable="/venv/bin/python", - prefix="/usr", - exec_prefix="/usr", + prefix="/venv", + exec_prefix="/venv", base_executable="/usr/bin/python3", base_prefix="/usr", base_exec_prefix="/usr", @@ -404,8 +410,8 @@ def test_venv_non_installed_zip_path_posix(self): ]) expected = dict( executable="/venv/bin/python", - prefix="/path/to/non-installed", - exec_prefix="/path/to/non-installed", + prefix="/venv", + exec_prefix="/venv", base_executable="/path/to/non-installed/bin/python", base_prefix="/path/to/non-installed", base_exec_prefix="/path/to/non-installed", @@ -435,8 +441,8 @@ def test_venv_changed_name_copy_posix(self): ]) expected = dict( executable="/venv/bin/python", - prefix="/usr", - exec_prefix="/usr", + prefix="/venv", + exec_prefix="/venv", base_executable="/usr/bin/python9", base_prefix="/usr", base_exec_prefix="/usr", @@ -557,7 +563,7 @@ def test_framework_macos(self): ns.add_known_dir("/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8/lib-dynload") ns.add_known_file("/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8/os.py") - # This is definitely not the stdlib (see discusion in bpo-46890) + # This is definitely not the stdlib (see discussion in bpo-46890) #ns.add_known_file("/Library/Frameworks/lib/python98.zip") expected = dict( @@ -605,7 +611,7 @@ def test_alt_framework_macos(self): ns.add_known_dir("/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8/lib-dynload") ns.add_known_xfile("/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8/os.py") - # This is definitely not the stdlib (see discusion in bpo-46890) + # This is definitely not the stdlib (see discussion in bpo-46890) #ns.add_known_xfile("/Library/lib/python98.zip") expected = dict( executable="/Library/Frameworks/DebugPython.framework/Versions/9.8/bin/python9.8", @@ -652,8 +658,8 @@ def test_venv_framework_macos(self): ]) expected = dict( executable=f"{venv_path}/bin/python", - prefix="/Library/Frameworks/Python.framework/Versions/9.8", - exec_prefix="/Library/Frameworks/Python.framework/Versions/9.8", + prefix=venv_path, + exec_prefix=venv_path, base_executable="/Library/Frameworks/Python.framework/Versions/9.8/bin/python9.8", base_prefix="/Library/Frameworks/Python.framework/Versions/9.8", base_exec_prefix="/Library/Frameworks/Python.framework/Versions/9.8", @@ -697,8 +703,8 @@ def test_venv_alt_framework_macos(self): ]) expected = dict( executable=f"{venv_path}/bin/python", - prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8", - exec_prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8", + prefix=venv_path, + exec_prefix=venv_path, base_executable="/Library/Frameworks/DebugPython.framework/Versions/9.8/bin/python9.8", base_prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8", base_exec_prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8", @@ -734,8 +740,8 @@ def test_venv_macos(self): ]) expected = dict( executable="/framework/Python9.8/python", - prefix="/usr", - exec_prefix="/usr", + prefix="/framework/Python9.8", + exec_prefix="/framework/Python9.8", base_executable="/usr/bin/python", base_prefix="/usr", base_exec_prefix="/usr", @@ -832,6 +838,68 @@ def test_explicitly_set_stdlib_dir(self): actual = getpath(ns, expected) self.assertEqual(expected, actual) + def test_PYTHONHOME_in_venv(self): + "Make sure prefix/exec_prefix still point to the venv if PYTHONHOME was used." + ns = MockPosixNamespace( + argv0="/venv/bin/python", + PREFIX="/usr", + ENV_PYTHONHOME="/pythonhome", + ) + # Setup venv + ns.add_known_xfile("/venv/bin/python") + ns.add_known_file("/venv/pyvenv.cfg", [ + r"home = /usr/bin" + ]) + # Seutup PYTHONHOME + ns.add_known_file("/pythonhome/lib/python9.8/os.py") + ns.add_known_dir("/pythonhome/lib/python9.8/lib-dynload") + + expected = dict( + executable="/venv/bin/python", + prefix="/venv", + exec_prefix="/venv", + base_prefix="/pythonhome", + base_exec_prefix="/pythonhome", + module_search_paths_set=1, + module_search_paths=[ + "/pythonhome/lib/python98.zip", + "/pythonhome/lib/python9.8", + "/pythonhome/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + +class RealGetPathTests(unittest.TestCase): + @unittest.skipUnless( + sysconfig.is_python_build(), + 'Test only available when running from the buildir', + ) + @unittest.skipUnless( + any(sys.platform.startswith(p) for p in ('linux', 'freebsd', 'centos')), + 'Test only support on Linux-like OS-es (support LD_LIBRARY_PATH)', + ) + @unittest.skipUnless( + sysconfig.get_config_var('LDLIBRARY') != sysconfig.get_config_var('LIBRARY'), + 'Test only available when using a dynamic libpython', + ) + def test_builddir_wrong_library_warning(self): + library_name = sysconfig.get_config_var('INSTSONAME') + with tempfile.TemporaryDirectory() as tmpdir: + shutil.copy2( + os.path.join(sysconfig.get_config_var('srcdir'), library_name), + os.path.join(tmpdir, library_name) + ) + env = os.environ.copy() + env['LD_LIBRARY_PATH'] = tmpdir + process = subprocess.run( + [sys.executable, '-c', ''], + env=env, check=True, capture_output=True, text=True, + ) + error_msg = 'The runtime library has been loaded from outside the build directory' + self.assertTrue(process.stderr.startswith(error_msg), process.stderr) + # ****************************************************************************** @@ -844,6 +912,7 @@ def test_explicitly_set_stdlib_dir(self): PYDEBUGEXT="", VERSION_MAJOR=9, # fixed version number for ease VERSION_MINOR=8, # of testing + ABI_THREAD="", PYWINVER=None, EXE_SUFFIX=None, diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index 8b2ea8f89f5daf..1a836e34e8712f 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -4,7 +4,10 @@ import shutil import sys import unittest +import warnings +from test import support +from test.support import is_wasi, Py_DEBUG from test.support.os_helper import (TESTFN, skip_unless_symlink, can_symlink, create_empty_file, change_cwd) @@ -41,6 +44,11 @@ def setUp(self): os.symlink(self.norm('broken'), self.norm('sym1')) os.symlink('broken', self.norm('sym2')) os.symlink(os.path.join('a', 'bcd'), self.norm('sym3')) + self.open_dirfd() + + def open_dirfd(self): + if self.dir_fd is not None: + os.close(self.dir_fd) if {os.open, os.stat} <= os.supports_dir_fd and os.scandir in os.supports_fd: self.dir_fd = os.open(self.tempdir, os.O_RDONLY | os.O_DIRECTORY) else: @@ -344,6 +352,24 @@ def test_glob_non_directory(self): eq(self.rglob('nonexistent', '*'), []) eq(self.rglob('nonexistent', '**'), []) + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_glob_named_pipe(self): + path = os.path.join(self.tempdir, 'mypipe') + os.mkfifo(path) + + # gh-117127: Reopen self.dir_fd to pick up directory changes + self.open_dirfd() + + self.assertEqual(self.rglob('mypipe'), [path]) + self.assertEqual(self.rglob('mypipe*'), [path]) + self.assertEqual(self.rglob('mypipe', ''), []) + self.assertEqual(self.rglob('mypipe', 'sub'), []) + self.assertEqual(self.rglob('mypipe', '*'), []) + + + @unittest.skipIf(is_wasi and Py_DEBUG, "requires too much stack") def test_glob_many_open_files(self): depth = 30 base = os.path.join(self.tempdir, 'deep') @@ -361,6 +387,36 @@ def test_glob_many_open_files(self): for it in iters: self.assertEqual(next(it), p) + def test_glob0(self): + with self.assertWarns(DeprecationWarning): + glob.glob0(self.tempdir, 'a') + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + eq = self.assertSequencesEqual_noorder + eq(glob.glob0(self.tempdir, 'a'), ['a']) + eq(glob.glob0(self.tempdir, '.bb'), ['.bb']) + eq(glob.glob0(self.tempdir, '.b*'), []) + eq(glob.glob0(self.tempdir, 'b'), []) + eq(glob.glob0(self.tempdir, '?'), []) + eq(glob.glob0(self.tempdir, '*a'), []) + eq(glob.glob0(self.tempdir, 'a*'), []) + + def test_glob1(self): + with self.assertWarns(DeprecationWarning): + glob.glob1(self.tempdir, 'a') + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + eq = self.assertSequencesEqual_noorder + eq(glob.glob1(self.tempdir, 'a'), ['a']) + eq(glob.glob1(self.tempdir, '.bb'), ['.bb']) + eq(glob.glob1(self.tempdir, '.b*'), ['.bb']) + eq(glob.glob1(self.tempdir, 'b'), []) + eq(glob.glob1(self.tempdir, '?'), ['a']) + eq(glob.glob1(self.tempdir, '*a'), ['a', 'aaa']) + eq(glob.glob1(self.tempdir, 'a*'), ['a', 'aaa', 'aab']) + def test_translate_matching(self): match = re.compile(glob.translate('*')).match self.assertIsNotNone(match('foo')) @@ -440,9 +496,9 @@ def fn(pat): self.assertEqual(fn('?'), r'(?s:[^/])\Z') self.assertEqual(fn('**'), r'(?s:.*)\Z') self.assertEqual(fn('**/**'), r'(?s:.*)\Z') - self.assertRaises(ValueError, fn, '***') - self.assertRaises(ValueError, fn, 'a**') - self.assertRaises(ValueError, fn, '**b') + self.assertEqual(fn('***'), r'(?s:[^/]*)\Z') + self.assertEqual(fn('a**'), r'(?s:a[^/]*)\Z') + self.assertEqual(fn('**b'), r'(?s:[^/]*b)\Z') self.assertEqual(fn('/**/*/*.*/**'), r'(?s:/(?:.+/)?[^/]+/[^/]*\.[^/]*/.*)\Z') def test_translate_seps(self): @@ -455,11 +511,21 @@ def fn(pat): @skip_unless_symlink class SymlinkLoopGlobTests(unittest.TestCase): + # gh-109959: On Linux, glob._isdir() and glob._lexists() can return False + # randomly when checking the "link/" symbolic link. + # https://github.com/python/cpython/issues/109959#issuecomment-2577550700 + @unittest.skip("flaky test") def test_selflink(self): tempdir = TESTFN + "_dir" os.makedirs(tempdir) self.addCleanup(shutil.rmtree, tempdir) with change_cwd(tempdir): + if support.verbose: + cwd = os.getcwd() + print(f"cwd: {cwd} ({len(cwd)} chars)") + cwdb = os.getcwdb() + print(f"cwdb: {cwdb!r} ({len(cwdb)} bytes)") + os.makedirs('dir') create_empty_file(os.path.join('dir', 'file')) os.symlink(os.curdir, os.path.join('dir', 'link')) diff --git a/Lib/test/test_global.py b/Lib/test/test_global.py index f5b38c25ea0728..11d0bd54e8b69b 100644 --- a/Lib/test/test_global.py +++ b/Lib/test/test_global.py @@ -1,7 +1,19 @@ -"""Verify that warnings are issued for global statements following use.""" +"""This module includes tests for syntax errors that occur when a name +declared as `global` is used in ways that violate the language +specification, such as after assignment, usage, or annotation. The tests +verify that syntax errors are correctly raised for improper `global` +statements following variable use or assignment within functions. +Additionally, it tests various name-binding scenarios for global +variables to ensure correct behavior. +See `test_scope.py` for additional related behavioral tests covering +variable scoping and usage in different contexts. +""" + +import contextlib from test.support import check_syntax_error from test.support.warnings_helper import check_warnings +from types import SimpleNamespace import unittest import warnings @@ -12,40 +24,185 @@ def setUp(self): self.enterContext(check_warnings()) warnings.filterwarnings("error", module="") - def test1(self): - prog_text_1 = """\ -def wrong1(): - a = 1 - b = 2 - global a - global b + ###################################################### + ### Syntax error cases as covered in Python/symtable.c + ###################################################### + + def test_name_param(self): + prog_text = """\ +def fn(name_param): + global name_param """ - check_syntax_error(self, prog_text_1, lineno=4, offset=5) + check_syntax_error(self, prog_text, lineno=2, offset=5) - def test2(self): - prog_text_2 = """\ -def wrong2(): - print(x) - global x + def test_name_after_assign(self): + prog_text = """\ +def fn(): + name_assign = 1 + global name_assign """ - check_syntax_error(self, prog_text_2, lineno=3, offset=5) + check_syntax_error(self, prog_text, lineno=3, offset=5) - def test3(self): - prog_text_3 = """\ -def wrong3(): - print(x) - x = 2 - global x + def test_name_after_use(self): + prog_text = """\ +def fn(): + print(name_use) + global name_use """ - check_syntax_error(self, prog_text_3, lineno=4, offset=5) + check_syntax_error(self, prog_text, lineno=3, offset=5) - def test4(self): - prog_text_4 = """\ -global x -x = 2 + def test_name_annot(self): + prog_text_3 = """\ +def fn(): + name_annot: int + global name_annot """ - # this should work - compile(prog_text_4, "", "exec") + check_syntax_error(self, prog_text_3, lineno=3, offset=5) + + ############################################################# + ### Tests for global variables across all name binding cases, + ### as described in executionmodel.rst + ############################################################# + + def test_assignment_statement(self): + global name_assignment_statement + value = object() + name_assignment_statement = value + self.assertIs(globals()["name_assignment_statement"], value) + del name_assignment_statement + + def test_unpacking_assignment(self): + global name_unpacking_assignment + value = object() + _, name_unpacking_assignment = [None, value] + self.assertIs(globals()["name_unpacking_assignment"], value) + del name_unpacking_assignment + + def test_assignment_expression(self): + global name_assignment_expression + value = object() + if name_assignment_expression := value: + pass + self.assertIs(globals()["name_assignment_expression"], value) + del name_assignment_expression + + def test_iteration_variable(self): + global name_iteration_variable + value = object() + for name_iteration_variable in [value]: + pass + self.assertIs(globals()["name_iteration_variable"], value) + del name_iteration_variable + + def test_func_def(self): + global name_func_def + + def name_func_def(): + pass + + value = name_func_def + self.assertIs(globals()["name_func_def"], value) + del name_func_def + + def test_class_def(self): + global name_class_def + + class name_class_def: + pass + + value = name_class_def + self.assertIs(globals()["name_class_def"], value) + del name_class_def + + def test_type_alias(self): + global name_type_alias + type name_type_alias = tuple[int, int] + value = name_type_alias + self.assertIs(globals()["name_type_alias"], value) + del name_type_alias + + def test_caught_exception(self): + global name_caught_exc + + try: + 1 / 0 + except ZeroDivisionError as name_caught_exc: + value = name_caught_exc + # `name_caught_exc` is cleared automatically after the except block + self.assertIs(globals()["name_caught_exc"], value) + + def test_caught_exception_group(self): + global name_caught_exc_group + try: + try: + 1 / 0 + except ZeroDivisionError as exc: + raise ExceptionGroup("eg", [exc]) + except* ZeroDivisionError as name_caught_exc_group: + value = name_caught_exc_group + # `name_caught_exc` is cleared automatically after the except block + self.assertIs(globals()["name_caught_exc_group"], value) + + def test_enter_result(self): + global name_enter_result + value = object() + with contextlib.nullcontext(value) as name_enter_result: + pass + self.assertIs(globals()["name_enter_result"], value) + del name_enter_result + + def test_import_result(self): + global name_import_result + value = contextlib + import contextlib as name_import_result + + self.assertIs(globals()["name_import_result"], value) + del name_import_result + + def test_match(self): + global name_match + value = object() + match value: + case name_match: + pass + self.assertIs(globals()["name_match"], value) + del name_match + + def test_match_as(self): + global name_match_as + value = object() + match value: + case _ as name_match_as: + pass + self.assertIs(globals()["name_match_as"], value) + del name_match_as + + def test_match_seq(self): + global name_match_seq + value = object() + match (None, value): + case (_, name_match_seq): + pass + self.assertIs(globals()["name_match_seq"], value) + del name_match_seq + + def test_match_map(self): + global name_match_map + value = object() + match {"key": value}: + case {"key": name_match_map}: + pass + self.assertIs(globals()["name_match_map"], value) + del name_match_map + + def test_match_attr(self): + global name_match_attr + value = object() + match SimpleNamespace(key=value): + case SimpleNamespace(key=name_match_attr): + pass + self.assertIs(globals()["name_match_attr"], value) + del name_match_attr def setUpModule(): diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 8501006b799262..6a841587f49166 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -3,6 +3,7 @@ from test.support import check_syntax_error from test.support import import_helper +import annotationlib import inspect import unittest import sys @@ -164,7 +165,7 @@ def test_floats(self): x = 3.14 x = 314. x = 0.314 - # XXX x = 000.314 + x = 000.314 x = .314 x = 3e14 x = 3E14 @@ -306,16 +307,6 @@ def test_eof_error(self): var_annot_global: int # a global annotated is necessary for test_var_annot -# custom namespace for testing __annotations__ - -class CNS: - def __init__(self): - self._dct = {} - def __setitem__(self, item, value): - self._dct[item.lower()] = value - def __getitem__(self, item): - return self._dct[item] - class GrammarTests(unittest.TestCase): @@ -446,22 +437,12 @@ class F(C, A): self.assertEqual(E.__annotations__, {}) self.assertEqual(F.__annotations__, {}) - - def test_var_annot_metaclass_semantics(self): - class CMeta(type): - @classmethod - def __prepare__(metacls, name, bases, **kwds): - return {'__annotations__': CNS()} - class CC(metaclass=CMeta): - XX: 'ANNOT' - self.assertEqual(CC.__annotations__['xx'], 'ANNOT') - def test_var_annot_module_semantics(self): self.assertEqual(test.__annotations__, {}) self.assertEqual(ann_module.__annotations__, - {1: 2, 'x': int, 'y': str, 'f': typing.Tuple[int, int], 'u': int | float}) + {'x': int, 'y': str, 'f': typing.Tuple[int, int], 'u': int | float}) self.assertEqual(ann_module.M.__annotations__, - {'123': 123, 'o': type}) + {'o': type}) self.assertEqual(ann_module2.__annotations__, {}) def test_var_annot_in_module(self): @@ -476,51 +457,12 @@ def test_var_annot_in_module(self): ann_module3.D_bad_ann(5) def test_var_annot_simple_exec(self): - gns = {}; lns= {} + gns = {}; lns = {} exec("'docstring'\n" - "__annotations__[1] = 2\n" "x: int = 5\n", gns, lns) - self.assertEqual(lns["__annotations__"], {1: 2, 'x': int}) - with self.assertRaises(KeyError): - gns['__annotations__'] - - def test_var_annot_custom_maps(self): - # tests with custom locals() and __annotations__ - ns = {'__annotations__': CNS()} - exec('X: int; Z: str = "Z"; (w): complex = 1j', ns) - self.assertEqual(ns['__annotations__']['x'], int) - self.assertEqual(ns['__annotations__']['z'], str) + self.assertEqual(lns["__annotate__"](annotationlib.Format.VALUE), {'x': int}) with self.assertRaises(KeyError): - ns['__annotations__']['w'] - nonloc_ns = {} - class CNS2: - def __init__(self): - self._dct = {} - def __setitem__(self, item, value): - nonlocal nonloc_ns - self._dct[item] = value - nonloc_ns[item] = value - def __getitem__(self, item): - return self._dct[item] - exec('x: int = 1', {}, CNS2()) - self.assertEqual(nonloc_ns['__annotations__']['x'], int) - - def test_var_annot_refleak(self): - # complex case: custom locals plus custom __annotations__ - # this was causing refleak - cns = CNS() - nonloc_ns = {'__annotations__': cns} - class CNS2: - def __init__(self): - self._dct = {'__annotations__': cns} - def __setitem__(self, item, value): - nonlocal nonloc_ns - self._dct[item] = value - nonloc_ns[item] = value - def __getitem__(self, item): - return self._dct[item] - exec('X: str', {}, CNS2()) - self.assertEqual(nonloc_ns['__annotations__']['x'], str) + gns['__annotate__'] def test_var_annot_rhs(self): ns = {} diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index d220c7d06e50c9..bf6e1703db8451 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -587,6 +587,8 @@ def test_fileobj_from_fdopen(self): self.assertRaises(AttributeError, f.fileno) def test_fileobj_mode(self): + self.assertEqual(gzip.READ, 'rb') + self.assertEqual(gzip.WRITE, 'wb') gzip.GzipFile(self.filename, "wb").close() with open(self.filename, "r+b") as f: with gzip.GzipFile(fileobj=f, mode='r') as g: @@ -711,8 +713,18 @@ def test_compress_mtime(self): f.read(1) # to set mtime attribute self.assertEqual(f.mtime, mtime) + def test_compress_mtime_default(self): + # test for gh-125260 + datac = gzip.compress(data1, mtime=0) + datac2 = gzip.compress(data1) + self.assertEqual(datac, datac2) + datac3 = gzip.compress(data1, mtime=None) + self.assertNotEqual(datac, datac3) + with gzip.GzipFile(fileobj=io.BytesIO(datac3), mode="rb") as f: + f.read(1) # to set mtime attribute + self.assertGreater(f.mtime, 1) + def test_compress_correct_level(self): - # gzip.compress calls with mtime == 0 take a different code path. for mtime in (0, 42): with self.subTest(mtime=mtime): nocompress = gzip.compress(data1, compresslevel=0, mtime=mtime) @@ -720,6 +732,17 @@ def test_compress_correct_level(self): self.assertIn(data1, nocompress) self.assertNotIn(data1, yescompress) + def test_issue112346(self): + # The OS byte should be 255, this should not change between Python versions. + for mtime in (0, 42): + with self.subTest(mtime=mtime): + compress = gzip.compress(data1, compresslevel=1, mtime=mtime) + self.assertEqual( + struct.unpack("' (unknown location)" + ) + popen = script_helper.spawn_python("-c", script) + stdout, stderr = popen.communicate() + self.assertIn(expected_error, stdout) + + def test_script_shadowing_stdlib(self): + script_errors = [ + ( + "import fractions\nfractions.Fraction", + rb"AttributeError: module 'fractions' has no attribute 'Fraction'" + ), + ( + "from fractions import Fraction", + rb"ImportError: cannot import name 'Fraction' from 'fractions'" + ) + ] + for script, error in script_errors: + with self.subTest(script=script), os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "fractions.py"), "w", encoding='utf-8') as f: + f.write(script) + + expected_error = error + ( + rb" \(consider renaming '.*fractions.py' since it has the " + rb"same name as the standard library module named 'fractions' " + rb"and prevents importing that standard library module\)" + ) + + popen = script_helper.spawn_python(os.path.join(tmp, "fractions.py"), cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-m', 'fractions', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-c', 'import fractions', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + # and there's no error at all when using -P + popen = script_helper.spawn_python('-P', 'fractions.py', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertEqual(stdout, b'') + + tmp_child = os.path.join(tmp, "child") + os.mkdir(tmp_child) + + # test the logic with different cwd + popen = script_helper.spawn_python(os.path.join(tmp, "fractions.py"), cwd=tmp_child) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-m', 'fractions', cwd=tmp_child) + stdout, stderr = popen.communicate() + self.assertEqual(stdout, b'') # no error + + popen = script_helper.spawn_python('-c', 'import fractions', cwd=tmp_child) + stdout, stderr = popen.communicate() + self.assertEqual(stdout, b'') # no error + + def test_package_shadowing_stdlib_module(self): + script_errors = [ + ( + "fractions.Fraction", + rb"AttributeError: module 'fractions' has no attribute 'Fraction'" + ), + ( + "from fractions import Fraction", + rb"ImportError: cannot import name 'Fraction' from 'fractions'" + ) + ] + for script, error in script_errors: + with self.subTest(script=script), os_helper.temp_dir() as tmp: + os.mkdir(os.path.join(tmp, "fractions")) + with open( + os.path.join(tmp, "fractions", "__init__.py"), "w", encoding='utf-8' + ) as f: + f.write("shadowing_module = True") + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write("import fractions; fractions.shadowing_module\n") + f.write(script) + + expected_error = error + ( + rb" \(consider renaming '.*[\\/]fractions[\\/]+__init__.py' since it has the " + rb"same name as the standard library module named 'fractions' " + rb"and prevents importing that standard library module\)" + ) + + popen = script_helper.spawn_python(os.path.join(tmp, "main.py"), cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-m', 'main', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + # and there's no shadowing at all when using -P + popen = script_helper.spawn_python('-P', 'main.py', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, b"module 'fractions' has no attribute 'shadowing_module'") + + def test_script_shadowing_third_party(self): + script_errors = [ + ( + "import numpy\nnumpy.array", + rb"AttributeError: module 'numpy' has no attribute 'array'" + ), + ( + "from numpy import array", + rb"ImportError: cannot import name 'array' from 'numpy'" + ) + ] + for script, error in script_errors: + with self.subTest(script=script), os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "numpy.py"), "w", encoding='utf-8') as f: + f.write(script) + + expected_error = error + ( + rb" \(consider renaming '.*numpy.py' if it has the " + rb"same name as a library you intended to import\)\s+\Z" + ) + + popen = script_helper.spawn_python(os.path.join(tmp, "numpy.py")) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-m', 'numpy', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-c', 'import numpy', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + def test_script_maybe_not_shadowing_third_party(self): + with os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "numpy.py"), "w", encoding='utf-8') as f: + f.write("this_script_does_not_attempt_to_import_numpy = True") + + expected_error = ( + rb"AttributeError: module 'numpy' has no attribute 'attr'\s+\Z" + ) + popen = script_helper.spawn_python('-c', 'import numpy; numpy.attr', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + expected_error = ( + rb"ImportError: cannot import name 'attr' from 'numpy' \(.*\)\s+\Z" + ) + popen = script_helper.spawn_python('-c', 'from numpy import attr', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + def test_script_shadowing_stdlib_edge_cases(self): + with os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "fractions.py"), "w", encoding='utf-8') as f: + f.write("shadowing_module = True") + + # Unhashable str subclass + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module +class substr(str): + __hash__ = None +fractions.__name__ = substr('fractions') +try: + fractions.Fraction +except TypeError as e: + print(str(e)) +""") + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + self.assertEqual(stdout.rstrip(), b"unhashable type: 'substr'") + + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module +class substr(str): + __hash__ = None +fractions.__name__ = substr('fractions') +try: + from fractions import Fraction +except TypeError as e: + print(str(e)) +""") + + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + self.assertEqual(stdout.rstrip(), b"unhashable type: 'substr'") + + # Various issues with sys module + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module + +import sys +sys.stdlib_module_names = None +try: + fractions.Fraction +except AttributeError as e: + print(str(e)) + +del sys.stdlib_module_names +try: + fractions.Fraction +except AttributeError as e: + print(str(e)) + +sys.path = [0] +try: + fractions.Fraction +except AttributeError as e: + print(str(e)) +""") + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + lines = stdout.splitlines() + self.assertEqual(len(lines), 3) + for line in lines: + self.assertEqual(line, b"module 'fractions' has no attribute 'Fraction'") + + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module + +import sys +sys.stdlib_module_names = None +try: + from fractions import Fraction +except ImportError as e: + print(str(e)) + +del sys.stdlib_module_names +try: + from fractions import Fraction +except ImportError as e: + print(str(e)) + +sys.path = [0] +try: + from fractions import Fraction +except ImportError as e: + print(str(e)) +""") + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + lines = stdout.splitlines() + self.assertEqual(len(lines), 3) + for line in lines: + self.assertRegex(line, rb"cannot import name 'Fraction' from 'fractions' \(.*\)") + + # Various issues with origin + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module +del fractions.__spec__.origin +try: + fractions.Fraction +except AttributeError as e: + print(str(e)) + +fractions.__spec__.origin = [] +try: + fractions.Fraction +except AttributeError as e: + print(str(e)) +""") + + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + lines = stdout.splitlines() + self.assertEqual(len(lines), 2) + for line in lines: + self.assertEqual(line, b"module 'fractions' has no attribute 'Fraction'") + + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module +del fractions.__spec__.origin +try: + from fractions import Fraction +except ImportError as e: + print(str(e)) + +fractions.__spec__.origin = [] +try: + from fractions import Fraction +except ImportError as e: + print(str(e)) +""") + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + lines = stdout.splitlines() + self.assertEqual(len(lines), 2) + for line in lines: + self.assertRegex(line, rb"cannot import name 'Fraction' from 'fractions' \(.*\)") + + def test_script_shadowing_stdlib_sys_path_modification(self): + script_errors = [ + ( + "import fractions\nfractions.Fraction", + rb"AttributeError: module 'fractions' has no attribute 'Fraction'" + ), + ( + "from fractions import Fraction", + rb"ImportError: cannot import name 'Fraction' from 'fractions'" + ) + ] + for script, error in script_errors: + with self.subTest(script=script), os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "fractions.py"), "w", encoding='utf-8') as f: + f.write("shadowing_module = True") + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write('import sys; sys.path.insert(0, "this_folder_does_not_exist")\n') + f.write(script) + expected_error = error + ( + rb" \(consider renaming '.*fractions.py' since it has the " + rb"same name as the standard library module named 'fractions' " + rb"and prevents importing that standard library module\)" + ) + + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + def test_create_dynamic_null(self): + with self.assertRaisesRegex(ValueError, 'embedded null character'): + class Spec: + name = "a\x00b" + origin = "abc" + _imp.create_dynamic(Spec()) + + with self.assertRaisesRegex(ValueError, 'embedded null character'): + class Spec2: + name = "abc" + origin = "a\x00b" + _imp.create_dynamic(Spec2()) + @skip_if_dont_write_bytecode class FilePermissionTests(unittest.TestCase): @@ -891,7 +1327,7 @@ class PycRewritingTests(unittest.TestCase): import sys code_filename = sys._getframe().f_code.co_filename module_filename = __file__ -constant = 1 +constant = 1000 def func(): pass func_filename = func.__code__.co_filename @@ -960,7 +1396,7 @@ def test_foreign_code(self): code = marshal.load(f) constants = list(code.co_consts) foreign_code = importlib.import_module.__code__ - pos = constants.index(1) + pos = constants.index(1000) constants[pos] = foreign_code code = code.replace(co_consts=tuple(constants)) with open(self.compiled_name, "wb") as f: @@ -1020,7 +1456,7 @@ def test_UNC_path(self): self.fail("could not import 'test_unc_path' from %r: %r" % (unc, e)) self.assertEqual(mod.testdata, 'test_unc_path') - self.assertTrue(mod.__file__.startswith(unc), mod.__file__) + self.assertStartsWith(mod.__file__, unc) unload("test_unc_path") @@ -1033,7 +1469,7 @@ def tearDown(self): def test_relimport_star(self): # This will import * from .test_import. from .. import relimport - self.assertTrue(hasattr(relimport, "RelativeImportTests")) + self.assertHasAttr(relimport, "RelativeImportTests") def test_issue3221(self): # Note for mergers: the 'absolute' tests from the 2.x branch @@ -1092,6 +1528,34 @@ def test_import_from_unloaded_package(self): import package2.submodule1 package2.submodule1.submodule2 + def test_rebinding(self): + # The same data is also used for testing pkgutil.resolve_name() + # in test_pkgutil and mock.patch in test_unittest. + path = os.path.join(os.path.dirname(__file__), 'data') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + from package3 import submodule + self.assertEqual(submodule.attr, 'rebound') + import package3.submodule as submodule + self.assertEqual(submodule.attr, 'rebound') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + import package3.submodule as submodule + self.assertEqual(submodule.attr, 'rebound') + from package3 import submodule + self.assertEqual(submodule.attr, 'rebound') + + def test_rebinding2(self): + path = os.path.join(os.path.dirname(__file__), 'data') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + import package4.submodule as submodule + self.assertEqual(submodule.attr, 'submodule') + from package4 import submodule + self.assertEqual(submodule.attr, 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + from package4 import submodule + self.assertEqual(submodule.attr, 'origin') + import package4.submodule as submodule + self.assertEqual(submodule.attr, 'submodule') + class OverridingImportBuiltinTests(unittest.TestCase): def test_override_builtin(self): @@ -1335,7 +1799,7 @@ def test_frozen_importlib_is_bootstrap(self): self.assertIs(mod, _bootstrap) self.assertEqual(mod.__name__, 'importlib._bootstrap') self.assertEqual(mod.__package__, 'importlib') - self.assertTrue(mod.__file__.endswith('_bootstrap.py'), mod.__file__) + self.assertEndsWith(mod.__file__, '_bootstrap.py') def test_frozen_importlib_external_is_bootstrap_external(self): from importlib import _bootstrap_external @@ -1343,7 +1807,7 @@ def test_frozen_importlib_external_is_bootstrap_external(self): self.assertIs(mod, _bootstrap_external) self.assertEqual(mod.__name__, 'importlib._bootstrap_external') self.assertEqual(mod.__package__, 'importlib') - self.assertTrue(mod.__file__.endswith('_bootstrap_external.py'), mod.__file__) + self.assertEndsWith(mod.__file__, '_bootstrap_external.py') def test_there_can_be_only_one(self): # Issue #15386 revealed a tricky loophole in the bootstrapping @@ -1650,6 +2114,37 @@ def test_absolute_circular_submodule(self): str(cm.exception), ) + @requires_singlephase_init + @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") + def test_singlephase_circular(self): + """Regression test for gh-123950 + + Import a single-phase-init module that imports itself + from the PyInit_* function (before it's added to sys.modules). + Manages its own cache (which is `static`, and so incompatible + with multiple interpreters or interpreter reset). + """ + name = '_testsinglephase_circular' + helper_name = 'test.test_import.data.circular_imports.singlephase' + with uncache(name, helper_name): + filename = _testsinglephase.__file__ + # We don't put the module in sys.modules: that the *inner* + # import should do that. + mod = import_extension_from_file(name, filename, + put_in_sys_modules=False) + + self.assertEqual(mod.helper_mod_name, helper_name) + self.assertIn(name, sys.modules) + self.assertIn(helper_name, sys.modules) + + self.assertIn(name, sys.modules) + self.assertIn(helper_name, sys.modules) + self.assertNotIn(name, sys.modules) + self.assertNotIn(helper_name, sys.modules) + self.assertIs(mod.clear_static_var(), mod) + _testinternalcapi.clear_extension('_testsinglephase_circular', + mod.__spec__.origin) + def test_unwritable_module(self): self.addCleanup(unload, "test.test_import.data.unwritable") self.addCleanup(unload, "test.test_import.data.unwritable.x") @@ -1697,12 +2192,19 @@ def import_script(self, name, fd, filename=None, check_override=None): _imp._override_multi_interp_extensions_check({check_override}) ''' if filename: + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + loader = "AppleFrameworkLoader" + else: + loader = "ExtensionFileLoader" + return textwrap.dedent(f''' from importlib.util import spec_from_loader, module_from_spec - from importlib.machinery import ExtensionFileLoader + from importlib.machinery import {loader} import os, sys {override_text} - loader = ExtensionFileLoader({name!r}, {filename!r}) + loader = {loader}({name!r}, {filename!r}) spec = spec_from_loader({name!r}, loader) try: module = module_from_spec(spec) @@ -1797,15 +2299,19 @@ def check_compatible_fresh(self, name, *, strict=False, isolated=False): **(self.ISOLATED if isolated else self.NOT_ISOLATED), check_multi_interp_extensions=strict, ) + gil = kwargs['gil'] + kwargs['gil'] = 'default' if gil == 0 else ( + 'shared' if gil == 1 else 'own' if gil == 2 else gil) _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' import _testinternalcapi, sys assert ( {name!r} in sys.builtin_module_names or {name!r} not in sys.modules ), repr({name!r}) + config = type(sys.implementation)(**{kwargs}) ret = _testinternalcapi.run_in_subinterp_with_config( {self.import_script(name, "sys.stdout.fileno()")!r}, - **{kwargs}, + config, ) assert ret == 0, ret ''')) @@ -1821,12 +2327,16 @@ def check_incompatible_fresh(self, name, *, isolated=False): **(self.ISOLATED if isolated else self.NOT_ISOLATED), check_multi_interp_extensions=True, ) + gil = kwargs['gil'] + kwargs['gil'] = 'default' if gil == 0 else ( + 'shared' if gil == 1 else 'own' if gil == 2 else gil) _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' import _testinternalcapi, sys assert {name!r} not in sys.modules, {name!r} + config = type(sys.implementation)(**{kwargs}) ret = _testinternalcapi.run_in_subinterp_with_config( {self.import_script(name, "sys.stdout.fileno()")!r}, - **{kwargs}, + config, ) assert ret == 0, ret ''')) @@ -1836,24 +2346,28 @@ def check_incompatible_fresh(self, name, *, isolated=False): f'ImportError: module {name} does not support loading in subinterpreters', ) + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") def test_builtin_compat(self): # For now we avoid using sys or builtins # since they still don't implement multi-phase init. module = '_imp' require_builtin(module) - with self.subTest(f'{module}: not strict'): - self.check_compatible_here(module, strict=False) + if not Py_GIL_DISABLED: + with self.subTest(f'{module}: not strict'): + self.check_compatible_here(module, strict=False) with self.subTest(f'{module}: strict, not fresh'): self.check_compatible_here(module, strict=True) @cpython_only + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") def test_frozen_compat(self): module = '_frozen_importlib' require_frozen(module, skip=True) if __import__(module).__spec__.origin != 'frozen': raise unittest.SkipTest(f'{module} is unexpectedly not frozen') - with self.subTest(f'{module}: not strict'): - self.check_compatible_here(module, strict=False) + if not Py_GIL_DISABLED: + with self.subTest(f'{module}: not strict'): + self.check_compatible_here(module, strict=False) with self.subTest(f'{module}: strict, not fresh'): self.check_compatible_here(module, strict=True) @@ -1867,13 +2381,16 @@ def test_single_init_extension_compat(self): self.check_incompatible_here(module) with self.subTest(f'{module}: strict, fresh'): self.check_incompatible_fresh(module) + with self.subTest(f'{module}: isolated, fresh'): + self.check_incompatible_fresh(module, isolated=True) @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") def test_multi_init_extension_compat(self): module = '_testmultiphase' require_extension(module) - with self.subTest(f'{module}: not strict'): - self.check_compatible_here(module, strict=False) + if not Py_GIL_DISABLED: + with self.subTest(f'{module}: not strict'): + self.check_compatible_here(module, strict=False) with self.subTest(f'{module}: strict, not fresh'): self.check_compatible_here(module, strict=True) with self.subTest(f'{module}: strict, fresh'): @@ -1883,29 +2400,22 @@ def test_multi_init_extension_compat(self): def test_multi_init_extension_non_isolated_compat(self): modname = '_test_non_isolated' filename = _testmultiphase.__file__ - loader = ExtensionFileLoader(modname, filename) - spec = importlib.util.spec_from_loader(modname, loader) - module = importlib.util.module_from_spec(spec) - loader.exec_module(module) - sys.modules[modname] = module + module = import_extension_from_file(modname, filename) require_extension(module) with self.subTest(f'{modname}: isolated'): self.check_incompatible_here(modname, filename, isolated=True) with self.subTest(f'{modname}: not isolated'): self.check_incompatible_here(modname, filename, isolated=False) - with self.subTest(f'{modname}: not strict'): - self.check_compatible_here(modname, filename, strict=False) + if not Py_GIL_DISABLED: + with self.subTest(f'{modname}: not strict'): + self.check_compatible_here(modname, filename, strict=False) @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") def test_multi_init_extension_per_interpreter_gil_compat(self): modname = '_test_shared_gil_only' filename = _testmultiphase.__file__ - loader = ExtensionFileLoader(modname, filename) - spec = importlib.util.spec_from_loader(modname, loader) - module = importlib.util.module_from_spec(spec) - loader.exec_module(module) - sys.modules[modname] = module + module = import_extension_from_file(modname, filename) require_extension(module) with self.subTest(f'{modname}: isolated, strict'): @@ -1913,15 +2423,18 @@ def test_multi_init_extension_per_interpreter_gil_compat(self): with self.subTest(f'{modname}: not isolated, strict'): self.check_compatible_here(modname, filename, strict=True, isolated=False) - with self.subTest(f'{modname}: not isolated, not strict'): - self.check_compatible_here(modname, filename, - strict=False, isolated=False) + if not Py_GIL_DISABLED: + with self.subTest(f'{modname}: not isolated, not strict'): + self.check_compatible_here(modname, filename, + strict=False, isolated=False) + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") def test_python_compat(self): module = 'threading' require_pure_python(module) - with self.subTest(f'{module}: not strict'): - self.check_compatible_here(module, strict=False) + if not Py_GIL_DISABLED: + with self.subTest(f'{module}: not strict'): + self.check_compatible_here(module, strict=False) with self.subTest(f'{module}: strict, not fresh'): self.check_compatible_here(module, strict=True) with self.subTest(f'{module}: strict, fresh'): @@ -1962,6 +2475,7 @@ def check_incompatible(setting, override): with self.subTest('config: check disabled; override: disabled'): check_compatible(False, -1) + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") def test_isolated_config(self): module = 'threading' require_pure_python(module) @@ -1989,6 +2503,107 @@ def test_disallowed_reimport(self): class TestSinglePhaseSnapshot(ModuleSnapshot): + """A representation of a single-phase init module for testing. + + Fields from ModuleSnapshot: + + * id - id(mod) + * module - mod or a SimpleNamespace with __file__ & __spec__ + * ns - a shallow copy of mod.__dict__ + * ns_id - id(mod.__dict__) + * cached - sys.modules[name] (or None if not there or not snapshotable) + * cached_id - id(sys.modules[name]) (or None if not there) + + Extra fields: + + * summed - the result of calling "mod.sum(1, 2)" + * lookedup - the result of calling "mod.look_up_self()" + * lookedup_id - the object ID of self.lookedup + * state_initialized - the result of calling "mod.state_initialized()" + * init_count - (optional) the result of calling "mod.initialized_count()" + + Overridden methods from ModuleSnapshot: + + * from_module() + * parse() + + Other methods from ModuleSnapshot: + + * build_script() + * from_subinterp() + + ---- + + There are 5 modules in Modules/_testsinglephase.c: + + * _testsinglephase + * has global state + * extra loads skip the init function, copy def.m_base.m_copy + * counts calls to init function + * _testsinglephase_basic_wrapper + * _testsinglephase by another name (and separate init function symbol) + * _testsinglephase_basic_copy + * same as _testsinglephase but with own def (and init func) + * _testsinglephase_with_reinit + * has no global or module state + * mod.state_initialized returns None + * an extra load in the main interpreter calls the cached init func + * an extra load in legacy subinterpreters does a full load + * _testsinglephase_with_state + * has module state + * an extra load in the main interpreter calls the cached init func + * an extra load in legacy subinterpreters does a full load + + (See Modules/_testsinglephase.c for more info.) + + For all those modules, the snapshot after the initial load (not in + the global extensions cache) would look like the following: + + * initial load + * id: ID of nww module object + * ns: exactly what the module init put there + * ns_id: ID of new module's __dict__ + * cached_id: same as self.id + * summed: 3 (never changes) + * lookedup_id: same as self.id + * state_initialized: a timestamp between the time of the load + and the time of the snapshot + * init_count: 1 (None for _testsinglephase_with_reinit) + + For the other scenarios it varies. + + For the _testsinglephase, _testsinglephase_basic_wrapper, and + _testsinglephase_basic_copy modules, the snapshot should look + like the following: + + * reloaded + * id: no change + * ns: matches what the module init function put there, + including the IDs of all contained objects, + plus any extra attributes added before the reload + * ns_id: no change + * cached_id: no change + * lookedup_id: no change + * state_initialized: no change + * init_count: no change + * already loaded + * (same as initial load except for ns and state_initialized) + * ns: matches the initial load, incl. IDs of contained objects + * state_initialized: no change from initial load + + For _testsinglephase_with_reinit: + + * reloaded: same as initial load (old module & ns is discarded) + * already loaded: same as initial load (old module & ns is discarded) + + For _testsinglephase_with_state: + + * reloaded + * (same as initial load (old module & ns is discarded), + except init_count) + * init_count: increase by 1 + * already loaded: same as reloaded + """ @classmethod def from_module(cls, mod): @@ -2034,18 +2649,29 @@ class SinglephaseInitTests(unittest.TestCase): @classmethod def setUpClass(cls): spec = importlib.util.find_spec(cls.NAME) - from importlib.machinery import ExtensionFileLoader - cls.FILE = spec.origin cls.LOADER = type(spec.loader) - assert cls.LOADER is ExtensionFileLoader + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader, and we need to differentiate between the + # spec.origin and the original file location. + if is_apple_mobile: + assert cls.LOADER is AppleFrameworkLoader + + cls.ORIGIN = spec.origin + with open(spec.origin + ".origin", "r") as f: + cls.FILE = os.path.join( + os.path.dirname(sys.executable), + f.read().strip() + ) + else: + assert cls.LOADER is ExtensionFileLoader + + cls.ORIGIN = spec.origin + cls.FILE = spec.origin # Start fresh. cls.clean_up() - @classmethod - def tearDownClass(cls): - restore__testsinglephase() - def tearDown(self): # Clean up the module. self.clean_up() @@ -2053,14 +2679,15 @@ def tearDown(self): @classmethod def clean_up(cls): name = cls.NAME - filename = cls.FILE if name in sys.modules: if hasattr(sys.modules[name], '_clear_globals'): - assert sys.modules[name].__file__ == filename + assert sys.modules[name].__file__ == cls.FILE, \ + f"{sys.modules[name].__file__} != {cls.FILE}" + sys.modules[name]._clear_globals() del sys.modules[name] # Clear all internally cached data for the extension. - _testinternalcapi.clear_extension(name, filename) + _testinternalcapi.clear_extension(name, cls.ORIGIN) ######################### # helpers @@ -2068,7 +2695,7 @@ def clean_up(cls): def add_module_cleanup(self, name): def clean_up(): # Clear all internally cached data for the extension. - _testinternalcapi.clear_extension(name, self.FILE) + _testinternalcapi.clear_extension(name, self.ORIGIN) self.addCleanup(clean_up) def _load_dynamic(self, name, path): @@ -2091,7 +2718,7 @@ def load(self, name): except AttributeError: already_loaded = self.already_loaded = {} assert name not in already_loaded - mod = self._load_dynamic(name, self.FILE) + mod = self._load_dynamic(name, self.ORIGIN) self.assertNotIn(mod, already_loaded.values()) already_loaded[name] = mod return types.SimpleNamespace( @@ -2103,7 +2730,7 @@ def load(self, name): def re_load(self, name, mod): assert sys.modules[name] is mod assert mod.__dict__ == mod.__dict__ - reloaded = self._load_dynamic(name, self.FILE) + reloaded = self._load_dynamic(name, self.ORIGIN) return types.SimpleNamespace( name=name, module=reloaded, @@ -2113,7 +2740,7 @@ def re_load(self, name, mod): # subinterpreters def add_subinterpreter(self): - interpid = _interpreters.create(isolated=False) + interpid = _interpreters.create('legacy') def ensure_destroyed(): try: _interpreters.destroy(interpid) @@ -2129,7 +2756,7 @@ def clean_up(): name = {self.NAME!r} if name in sys.modules: sys.modules.pop(name)._clear_globals() - _testinternalcapi.clear_extension(name, {self.FILE!r}) + _testinternalcapi.clear_extension(name, {self.ORIGIN!r}) ''')) _interpreters.destroy(interpid) self.addCleanup(clean_up) @@ -2146,7 +2773,7 @@ def import_in_subinterp(self, interpid=None, *, postcleanup = f''' {import_} mod._clear_globals() - _testinternalcapi.clear_extension(name, {self.FILE!r}) + _testinternalcapi.clear_extension(name, {self.ORIGIN!r}) ''' try: @@ -2184,9 +2811,9 @@ def check_common(self, loaded): # mod.__name__ might not match, but the spec will. self.assertEqual(mod.__spec__.name, loaded.name) self.assertEqual(mod.__file__, self.FILE) - self.assertEqual(mod.__spec__.origin, self.FILE) + self.assertEqual(mod.__spec__.origin, self.ORIGIN) if not isolated: - self.assertTrue(issubclass(mod.error, Exception)) + self.assertIsSubclass(mod.error, Exception) self.assertEqual(mod.int_const, 1969) self.assertEqual(mod.str_const, 'something different') self.assertIsInstance(mod._module_initialized, float) @@ -2478,6 +3105,16 @@ def test_with_reinit_reloaded(self): self.assertIs(reloaded.snapshot.cached, reloaded.module) + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_check_state_first(self): + for variant in ['', '_with_reinit', '_with_state']: + name = f'{self.NAME}{variant}_check_cache_first' + with self.subTest(name): + mod = self._load_dynamic(name, self.ORIGIN) + self.assertEqual(mod.__name__, name) + sys.modules.pop(name, None) + _testinternalcapi.clear_extension(name, self.ORIGIN) + # Currently, for every single-phrase init module loaded # in multiple interpreters, those interpreters share a # PyModuleDef for that object, which can be a problem. @@ -2524,7 +3161,7 @@ def test_basic_multiple_interpreters_main_no_reset(self): # * alive in 1 interpreter (main) # * module def still in _PyRuntime.imports.extensions # * mod init func ran again - # * m_copy is NULL (claered when the interpreter was destroyed) + # * m_copy is NULL (cleared when the interpreter was destroyed) # (was from main interpreter) # * module's global state was updated, not reset @@ -2578,7 +3215,7 @@ def test_basic_multiple_interpreters_deleted_no_reset(self): # First, load in the main interpreter but then completely clear it. loaded_main = self.load(self.NAME) loaded_main.module._clear_globals() - _testinternalcapi.clear_extension(self.NAME, self.FILE) + _testinternalcapi.clear_extension(self.NAME, self.ORIGIN) # At this point: # * alive in 0 interpreters @@ -2589,31 +3226,32 @@ def test_basic_multiple_interpreters_deleted_no_reset(self): # * module's global state was initialized but cleared # Start with an interpreter that gets destroyed right away. - base = self.import_in_subinterp(postscript=''' - # Attrs set after loading are not in m_copy. - mod.spam = 'spam, spam, mash, spam, eggs, and spam' - ''') + base = self.import_in_subinterp( + postscript=''' + # Attrs set after loading are not in m_copy. + mod.spam = 'spam, spam, mash, spam, eggs, and spam' + ''') self.check_common(base) self.check_fresh(base) # At this point: # * alive in 0 interpreters # * module def in _PyRuntime.imports.extensions - # * mod init func ran again - # * m_copy is NULL (claered when the interpreter was destroyed) + # * mod init func ran for the first time (since reset) + # * m_copy is still set (owned by main interpreter) # * module's global state was initialized, not reset # Use a subinterpreter that sticks around. loaded_interp1 = self.import_in_subinterp(interpid1) self.check_common(loaded_interp1) - self.check_semi_fresh(loaded_interp1, loaded_main, base) + self.check_copied(loaded_interp1, base) # At this point: # * alive in 1 interpreter (interp1) # * module def still in _PyRuntime.imports.extensions - # * mod init func ran again - # * m_copy was copied from interp1 (was NULL) - # * module's global state was updated, not reset + # * mod init func did not run again + # * m_copy was not changed + # * module's global state was not touched # Use a subinterpreter while the previous one is still alive. loaded_interp2 = self.import_in_subinterp(interpid2) @@ -2623,9 +3261,9 @@ def test_basic_multiple_interpreters_deleted_no_reset(self): # At this point: # * alive in 2 interpreters (interp1, interp2) # * module def still in _PyRuntime.imports.extensions - # * mod init func ran again - # * m_copy was copied from interp2 (was from interp1) - # * module's global state was updated, not reset + # * mod init func did not run again + # * m_copy was not changed + # * module's global state was not touched @requires_subinterpreters def test_basic_multiple_interpreters_reset_each(self): @@ -2658,7 +3296,7 @@ def test_basic_multiple_interpreters_reset_each(self): # * alive in 0 interpreters # * module def in _PyRuntime.imports.extensions # * mod init func ran for the first time (since reset, at least) - # * m_copy is NULL (claered when the interpreter was destroyed) + # * m_copy is NULL (cleared when the interpreter was destroyed) # * module's global state was initialized, not reset # Use a subinterpreter that sticks around. @@ -2687,27 +3325,14 @@ def test_basic_multiple_interpreters_reset_each(self): @cpython_only -class CAPITests(unittest.TestCase): - def test_pyimport_addmodule(self): - # gh-105922: Test PyImport_AddModuleRef(), PyImport_AddModule() - # and PyImport_AddModuleObject() - import _testcapi - for name in ( - 'sys', # frozen module - 'test', # package - __name__, # package.module - ): - _testcapi.check_pyimport_addmodule(name) - - def test_pyimport_addmodule_create(self): - # gh-105922: Test PyImport_AddModuleRef(), create a new module - import _testcapi - name = 'dontexist' - self.assertNotIn(name, sys.modules) - self.addCleanup(unload, name) - - mod = _testcapi.check_pyimport_addmodule(name) - self.assertIs(mod, sys.modules[name]) +class TestMagicNumber(unittest.TestCase): + def test_magic_number_endianness(self): + magic_number_bytes = _imp.pyc_magic_number_token.to_bytes(4, 'little') + self.assertEqual(magic_number_bytes[2:], b'\r\n') + # Starting with Python 3.11, Python 3.n starts with magic number 2900+50n. + magic_number = int.from_bytes(magic_number_bytes[:2], 'little') + start = 2900 + sys.version_info.minor * 50 + self.assertIn(magic_number, range(start, start + 50)) if __name__ == '__main__': diff --git a/Lib/test/test_import/data/circular_imports/singlephase.py b/Lib/test/test_import/data/circular_imports/singlephase.py new file mode 100644 index 00000000000000..05618bc72f9327 --- /dev/null +++ b/Lib/test/test_import/data/circular_imports/singlephase.py @@ -0,0 +1,13 @@ +"""Circular import involving a single-phase-init extension. + +This module is imported from the _testsinglephase_circular module from +_testsinglephase, and imports that module again. +""" + +import importlib +import _testsinglephase +from test.test_import import import_extension_from_file + +name = '_testsinglephase_circular' +filename = _testsinglephase.__file__ +mod = import_extension_from_file(name, filename) diff --git a/Lib/test/test_import/data/package3/__init__.py b/Lib/test/test_import/data/package3/__init__.py new file mode 100644 index 00000000000000..7033c22a719ec4 --- /dev/null +++ b/Lib/test/test_import/data/package3/__init__.py @@ -0,0 +1,2 @@ +"""Rebinding the package attribute after importing the module.""" +from .submodule import submodule diff --git a/Lib/test/test_import/data/package3/submodule.py b/Lib/test/test_import/data/package3/submodule.py new file mode 100644 index 00000000000000..cd7b30db15e449 --- /dev/null +++ b/Lib/test/test_import/data/package3/submodule.py @@ -0,0 +1,7 @@ +attr = 'submodule' +class A: + attr = 'submodule' +class submodule: + attr = 'rebound' + class B: + attr = 'rebound' diff --git a/Lib/test/test_import/data/package4/__init__.py b/Lib/test/test_import/data/package4/__init__.py new file mode 100644 index 00000000000000..d8af60ab38a319 --- /dev/null +++ b/Lib/test/test_import/data/package4/__init__.py @@ -0,0 +1,5 @@ +"""Binding the package attribute without importing the module.""" +class submodule: + attr = 'origin' + class B: + attr = 'origin' diff --git a/Lib/test/test_import/data/package4/submodule.py b/Lib/test/test_import/data/package4/submodule.py new file mode 100644 index 00000000000000..c861417aece9c3 --- /dev/null +++ b/Lib/test/test_import/data/package4/submodule.py @@ -0,0 +1,3 @@ +attr = 'submodule' +class A: + attr = 'submodule' diff --git a/Lib/test/test_importlib/extension/_test_nonmodule_cases.py b/Lib/test/test_importlib/extension/_test_nonmodule_cases.py new file mode 100644 index 00000000000000..8ffd18d221df4f --- /dev/null +++ b/Lib/test/test_importlib/extension/_test_nonmodule_cases.py @@ -0,0 +1,44 @@ +import types +import unittest +from test.test_importlib import util + +machinery = util.import_importlib('importlib.machinery') + +from test.test_importlib.extension.test_loader import MultiPhaseExtensionModuleTests + + +class NonModuleExtensionTests: + setUp = MultiPhaseExtensionModuleTests.setUp + load_module_by_name = MultiPhaseExtensionModuleTests.load_module_by_name + + def _test_nonmodule(self): + # Test returning a non-module object from create works. + name = self.name + '_nonmodule' + mod = self.load_module_by_name(name) + self.assertNotEqual(type(mod), type(unittest)) + self.assertEqual(mod.three, 3) + + # issue 27782 + def test_nonmodule_with_methods(self): + # Test creating a non-module object with methods defined. + name = self.name + '_nonmodule_with_methods' + mod = self.load_module_by_name(name) + self.assertNotEqual(type(mod), type(unittest)) + self.assertEqual(mod.three, 3) + self.assertEqual(mod.bar(10, 1), 9) + + def test_null_slots(self): + # Test that NULL slots aren't a problem. + name = self.name + '_null_slots' + module = self.load_module_by_name(name) + self.assertIsInstance(module, types.ModuleType) + self.assertEqual(module.__name__, name) + + +(Frozen_NonModuleExtensionTests, + Source_NonModuleExtensionTests + ) = util.test_both(NonModuleExtensionTests, machinery=machinery) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/extension/test_case_sensitivity.py b/Lib/test/test_importlib/extension/test_case_sensitivity.py index 0bb74fff5fcf96..40311627a144e8 100644 --- a/Lib/test/test_importlib/extension/test_case_sensitivity.py +++ b/Lib/test/test_importlib/extension/test_case_sensitivity.py @@ -8,7 +8,8 @@ machinery = util.import_importlib('importlib.machinery') -@unittest.skipIf(util.EXTENSIONS.filename is None, f'{util.EXTENSIONS.name} not available') +@unittest.skipIf(util.EXTENSIONS is None or util.EXTENSIONS.filename is None, + 'dynamic loading not supported or test module not available') @util.case_insensitive_tests class ExtensionModuleCaseSensitivityTest(util.CASEOKTestBase): diff --git a/Lib/test/test_importlib/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py index 1d5b6e7a5de94b..cdc8884d668a66 100644 --- a/Lib/test/test_importlib/extension/test_finder.py +++ b/Lib/test/test_importlib/extension/test_finder.py @@ -1,3 +1,4 @@ +from test.support import is_apple_mobile from test.test_importlib import abc, util machinery = util.import_importlib('importlib.machinery') @@ -11,7 +12,7 @@ class FinderTests(abc.FinderTests): """Test the finder for extension modules.""" def setUp(self): - if not self.machinery.EXTENSION_SUFFIXES: + if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS: raise unittest.SkipTest("Requires dynamic loading support.") if util.EXTENSIONS.name in sys.builtin_module_names: raise unittest.SkipTest( @@ -19,9 +20,27 @@ def setUp(self): ) def find_spec(self, fullname): - importer = self.machinery.FileFinder(util.EXTENSIONS.path, - (self.machinery.ExtensionFileLoader, - self.machinery.EXTENSION_SUFFIXES)) + if is_apple_mobile: + # Apple mobile platforms require a specialist loader that uses + # .fwork files as placeholders for the true `.so` files. + loaders = [ + ( + self.machinery.AppleFrameworkLoader, + [ + ext.replace(".so", ".fwork") + for ext in self.machinery.EXTENSION_SUFFIXES + ] + ) + ] + else: + loaders = [ + ( + self.machinery.ExtensionFileLoader, + self.machinery.EXTENSION_SUFFIXES + ) + ] + + importer = self.machinery.FileFinder(util.EXTENSIONS.path, *loaders) return importer.find_spec(fullname) diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index 84a0680e4ec653..0dd21e079eba22 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -1,3 +1,4 @@ +from test.support import is_apple_mobile from test.test_importlib import abc, util machinery = util.import_importlib('importlib.machinery') @@ -9,7 +10,8 @@ import warnings import importlib.util import importlib -from test.support import MISSING_C_DOCSTRINGS +from test import support +from test.support import MISSING_C_DOCSTRINGS, script_helper class LoaderTests: @@ -17,14 +19,21 @@ class LoaderTests: """Test ExtensionFileLoader.""" def setUp(self): - if not self.machinery.EXTENSION_SUFFIXES: + if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS: raise unittest.SkipTest("Requires dynamic loading support.") if util.EXTENSIONS.name in sys.builtin_module_names: raise unittest.SkipTest( f"{util.EXTENSIONS.name} is a builtin module" ) - self.loader = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name, - util.EXTENSIONS.file_path) + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + self.LoaderClass = self.machinery.AppleFrameworkLoader + else: + self.LoaderClass = self.machinery.ExtensionFileLoader + + self.loader = self.LoaderClass(util.EXTENSIONS.name, util.EXTENSIONS.file_path) def load_module(self, fullname): with warnings.catch_warnings(): @@ -32,13 +41,11 @@ def load_module(self, fullname): return self.loader.load_module(fullname) def test_equality(self): - other = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name, - util.EXTENSIONS.file_path) + other = self.LoaderClass(util.EXTENSIONS.name, util.EXTENSIONS.file_path) self.assertEqual(self.loader, other) def test_inequality(self): - other = self.machinery.ExtensionFileLoader('_' + util.EXTENSIONS.name, - util.EXTENSIONS.file_path) + other = self.LoaderClass('_' + util.EXTENSIONS.name, util.EXTENSIONS.file_path) self.assertNotEqual(self.loader, other) def test_load_module_API(self): @@ -58,8 +65,7 @@ def test_module(self): ('__package__', '')]: self.assertEqual(getattr(module, attr), value) self.assertIn(util.EXTENSIONS.name, sys.modules) - self.assertIsInstance(module.__loader__, - self.machinery.ExtensionFileLoader) + self.assertIsInstance(module.__loader__, self.LoaderClass) # No extension module as __init__ available for testing. test_package = None @@ -86,7 +92,7 @@ def test_is_package(self): self.assertFalse(self.loader.is_package(util.EXTENSIONS.name)) for suffix in self.machinery.EXTENSION_SUFFIXES: path = os.path.join('some', 'path', 'pkg', '__init__' + suffix) - loader = self.machinery.ExtensionFileLoader('pkg', path) + loader = self.LoaderClass('pkg', path) self.assertTrue(loader.is_package('pkg')) @@ -99,8 +105,16 @@ class SinglePhaseExtensionModuleTests(abc.LoaderTests): # Test loading extension modules without multi-phase initialization. def setUp(self): - if not self.machinery.EXTENSION_SUFFIXES: + if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS: raise unittest.SkipTest("Requires dynamic loading support.") + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + self.LoaderClass = self.machinery.AppleFrameworkLoader + else: + self.LoaderClass = self.machinery.ExtensionFileLoader + self.name = '_testsinglephase' if self.name in sys.builtin_module_names: raise unittest.SkipTest( @@ -109,8 +123,8 @@ def setUp(self): finder = self.machinery.FileFinder(None) self.spec = importlib.util.find_spec(self.name) assert self.spec - self.loader = self.machinery.ExtensionFileLoader( - self.name, self.spec.origin) + + self.loader = self.LoaderClass(self.name, self.spec.origin) def load_module(self): with warnings.catch_warnings(): @@ -120,7 +134,7 @@ def load_module(self): def load_module_by_name(self, fullname): # Load a module from the test extension by name. origin = self.spec.origin - loader = self.machinery.ExtensionFileLoader(fullname, origin) + loader = self.LoaderClass(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) @@ -137,8 +151,7 @@ def test_module(self): with self.assertRaises(AttributeError): module.__path__ self.assertIs(module, sys.modules[self.name]) - self.assertIsInstance(module.__loader__, - self.machinery.ExtensionFileLoader) + self.assertIsInstance(module.__loader__, self.LoaderClass) # No extension module as __init__ available for testing. test_package = None @@ -180,8 +193,16 @@ class MultiPhaseExtensionModuleTests(abc.LoaderTests): # Test loading extension modules with multi-phase initialization (PEP 489). def setUp(self): - if not self.machinery.EXTENSION_SUFFIXES: + if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS: raise unittest.SkipTest("Requires dynamic loading support.") + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + self.LoaderClass = self.machinery.AppleFrameworkLoader + else: + self.LoaderClass = self.machinery.ExtensionFileLoader + self.name = '_testmultiphase' if self.name in sys.builtin_module_names: raise unittest.SkipTest( @@ -190,8 +211,7 @@ def setUp(self): finder = self.machinery.FileFinder(None) self.spec = importlib.util.find_spec(self.name) assert self.spec - self.loader = self.machinery.ExtensionFileLoader( - self.name, self.spec.origin) + self.loader = self.LoaderClass(self.name, self.spec.origin) def load_module(self): # Load the module from the test extension. @@ -202,7 +222,7 @@ def load_module(self): def load_module_by_name(self, fullname): # Load a module from the test extension by name. origin = self.spec.origin - loader = self.machinery.ExtensionFileLoader(fullname, origin) + loader = self.LoaderClass(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) @@ -228,8 +248,7 @@ def test_module(self): with self.assertRaises(AttributeError): module.__path__ self.assertIs(module, sys.modules[self.name]) - self.assertIsInstance(module.__loader__, - self.machinery.ExtensionFileLoader) + self.assertIsInstance(module.__loader__, self.LoaderClass) def test_functionality(self): # Test basic functionality of stuff defined in an extension module. @@ -307,29 +326,6 @@ def test_unloadable_nonascii(self): self.load_module_by_name(name) self.assertEqual(cm.exception.name, name) - def test_nonmodule(self): - # Test returning a non-module object from create works. - name = self.name + '_nonmodule' - mod = self.load_module_by_name(name) - self.assertNotEqual(type(mod), type(unittest)) - self.assertEqual(mod.three, 3) - - # issue 27782 - def test_nonmodule_with_methods(self): - # Test creating a non-module object with methods defined. - name = self.name + '_nonmodule_with_methods' - mod = self.load_module_by_name(name) - self.assertNotEqual(type(mod), type(unittest)) - self.assertEqual(mod.three, 3) - self.assertEqual(mod.bar(10, 1), 9) - - def test_null_slots(self): - # Test that NULL slots aren't a problem. - name = self.name + '_null_slots' - module = self.load_module_by_name(name) - self.assertIsInstance(module, types.ModuleType) - self.assertEqual(module.__name__, name) - def test_bad_modules(self): # Test SystemError is raised for misbehaving extensions. for name_base in [ @@ -383,5 +379,14 @@ def test_nonascii(self): ) = util.test_both(MultiPhaseExtensionModuleTests, machinery=machinery) +class NonModuleExtensionTests(unittest.TestCase): + def test_nonmodule_cases(self): + # The test cases in this file cause the GIL to be enabled permanently + # in free-threaded builds, so they are run in a subprocess to isolate + # this effect. + script = support.findfile("test_importlib/extension/_test_nonmodule_cases.py") + script_helper.run_test_script(script) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/extension/test_path_hook.py b/Lib/test/test_importlib/extension/test_path_hook.py index ec9644dc520534..941dcd5432ce46 100644 --- a/Lib/test/test_importlib/extension/test_path_hook.py +++ b/Lib/test/test_importlib/extension/test_path_hook.py @@ -5,6 +5,8 @@ import unittest +@unittest.skipIf(util.EXTENSIONS is None or util.EXTENSIONS.filename is None, + 'dynamic loading not supported or test module not available') class PathHookTests: """Test the path hook for extension modules.""" @@ -19,7 +21,7 @@ def hook(self, entry): def test_success(self): # Path hook should handle a directory where a known extension module # exists. - self.assertTrue(hasattr(self.hook(util.EXTENSIONS.path), 'find_spec')) + self.assertHasAttr(self.hook(util.EXTENSIONS.path), 'find_spec') (Frozen_PathHooksTests, diff --git a/Lib/test/test_importlib/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py index 1112c0664ad477..c808bb73291a7c 100644 --- a/Lib/test/test_importlib/frozen/test_loader.py +++ b/Lib/test/test_importlib/frozen/test_loader.py @@ -61,7 +61,7 @@ def exec_module(self, name, origname=None): module.main() self.assertTrue(module.initialized) - self.assertTrue(hasattr(module, '__spec__')) + self.assertHasAttr(module, '__spec__') self.assertEqual(module.__spec__.origin, 'frozen') return module, stdout.getvalue() @@ -72,7 +72,7 @@ def test_module(self): for attr, value in check.items(): self.assertEqual(getattr(module, attr), value) self.assertEqual(output, 'Hello world!\n') - self.assertTrue(hasattr(module, '__spec__')) + self.assertHasAttr(module, '__spec__') self.assertEqual(module.__spec__.loader_state.origname, name) def test_package(self): @@ -136,7 +136,7 @@ def test_get_code(self): exec(code, mod.__dict__) with captured_stdout() as stdout: mod.main() - self.assertTrue(hasattr(mod, 'initialized')) + self.assertHasAttr(mod, 'initialized') self.assertEqual(stdout.getvalue(), 'Hello world!\n') def test_get_source(self): diff --git a/Lib/test/test_importlib/import_/test_caching.py b/Lib/test/test_importlib/import_/test_caching.py index aedf0fd4f9db02..718e7d041b0860 100644 --- a/Lib/test/test_importlib/import_/test_caching.py +++ b/Lib/test/test_importlib/import_/test_caching.py @@ -78,7 +78,7 @@ def test_using_cache_for_assigning_to_attribute(self): with self.create_mock('pkg.__init__', 'pkg.module') as importer: with util.import_state(meta_path=[importer]): module = self.__import__('pkg.module') - self.assertTrue(hasattr(module, 'module')) + self.assertHasAttr(module, 'module') self.assertEqual(id(module.module), id(sys.modules['pkg.module'])) @@ -88,7 +88,7 @@ def test_using_cache_for_fromlist(self): with self.create_mock('pkg.__init__', 'pkg.module') as importer: with util.import_state(meta_path=[importer]): module = self.__import__('pkg', fromlist=['module']) - self.assertTrue(hasattr(module, 'module')) + self.assertHasAttr(module, 'module') self.assertEqual(id(module.module), id(sys.modules['pkg.module'])) diff --git a/Lib/test/test_importlib/import_/test_fromlist.py b/Lib/test/test_importlib/import_/test_fromlist.py index 4b4b9bc3f5e04a..feccc7be09a98c 100644 --- a/Lib/test/test_importlib/import_/test_fromlist.py +++ b/Lib/test/test_importlib/import_/test_fromlist.py @@ -63,7 +63,7 @@ def test_nonexistent_object(self): with util.import_state(meta_path=[importer]): module = self.__import__('module', fromlist=['non_existent']) self.assertEqual(module.__name__, 'module') - self.assertFalse(hasattr(module, 'non_existent')) + self.assertNotHasAttr(module, 'non_existent') def test_module_from_package(self): # [module] @@ -71,7 +71,7 @@ def test_module_from_package(self): with util.import_state(meta_path=[importer]): module = self.__import__('pkg', fromlist=['module']) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'module')) + self.assertHasAttr(module, 'module') self.assertEqual(module.module.__name__, 'pkg.module') def test_nonexistent_from_package(self): @@ -79,7 +79,7 @@ def test_nonexistent_from_package(self): with util.import_state(meta_path=[importer]): module = self.__import__('pkg', fromlist=['non_existent']) self.assertEqual(module.__name__, 'pkg') - self.assertFalse(hasattr(module, 'non_existent')) + self.assertNotHasAttr(module, 'non_existent') def test_module_from_package_triggers_ModuleNotFoundError(self): # If a submodule causes an ModuleNotFoundError because it tries @@ -107,7 +107,7 @@ def basic_star_test(self, fromlist=['*']): mock['pkg'].__all__ = ['module'] module = self.__import__('pkg', fromlist=fromlist) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'module')) + self.assertHasAttr(module, 'module') self.assertEqual(module.module.__name__, 'pkg.module') def test_using_star(self): @@ -125,8 +125,8 @@ def test_star_with_others(self): mock['pkg'].__all__ = ['module1'] module = self.__import__('pkg', fromlist=['module2', '*']) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'module1')) - self.assertTrue(hasattr(module, 'module2')) + self.assertHasAttr(module, 'module1') + self.assertHasAttr(module, 'module2') self.assertEqual(module.module1.__name__, 'pkg.module1') self.assertEqual(module.module2.__name__, 'pkg.module2') @@ -136,7 +136,7 @@ def test_nonexistent_in_all(self): importer['pkg'].__all__ = ['non_existent'] module = self.__import__('pkg', fromlist=['*']) self.assertEqual(module.__name__, 'pkg') - self.assertFalse(hasattr(module, 'non_existent')) + self.assertNotHasAttr(module, 'non_existent') def test_star_in_all(self): with util.mock_spec('pkg.__init__') as importer: @@ -144,7 +144,7 @@ def test_star_in_all(self): importer['pkg'].__all__ = ['*'] module = self.__import__('pkg', fromlist=['*']) self.assertEqual(module.__name__, 'pkg') - self.assertFalse(hasattr(module, '*')) + self.assertNotHasAttr(module, '*') def test_invalid_type(self): with util.mock_spec('pkg.__init__') as importer: diff --git a/Lib/test/test_importlib/import_/test_meta_path.py b/Lib/test/test_importlib/import_/test_meta_path.py index 8689017ba43112..4c00f60681acf1 100644 --- a/Lib/test/test_importlib/import_/test_meta_path.py +++ b/Lib/test/test_importlib/import_/test_meta_path.py @@ -43,7 +43,7 @@ def test_empty(self): self.assertIsNone(importlib._bootstrap._find_spec('nothing', None)) self.assertEqual(len(w), 1) - self.assertTrue(issubclass(w[-1].category, ImportWarning)) + self.assertIsSubclass(w[-1].category, ImportWarning) (Frozen_CallingOrder, diff --git a/Lib/test/test_importlib/import_/test_path.py b/Lib/test/test_importlib/import_/test_path.py index 89b52fbd1e1aff..51ff6115e1281e 100644 --- a/Lib/test/test_importlib/import_/test_path.py +++ b/Lib/test/test_importlib/import_/test_path.py @@ -1,3 +1,4 @@ +from test.support import os_helper from test.test_importlib import util importlib = util.import_importlib('importlib') @@ -80,7 +81,7 @@ def test_empty_path_hooks(self): self.assertIsNone(self.find('os')) self.assertIsNone(sys.path_importer_cache[path_entry]) self.assertEqual(len(w), 1) - self.assertTrue(issubclass(w[-1].category, ImportWarning)) + self.assertIsSubclass(w[-1].category, ImportWarning) def test_path_importer_cache_empty_string(self): # The empty string should create a finder using the cwd. @@ -153,6 +154,28 @@ def test_deleted_cwd(self): # Do not want FileNotFoundError raised. self.assertIsNone(self.machinery.PathFinder.find_spec('whatever')) + @os_helper.skip_unless_working_chmod + def test_permission_error_cwd(self): + # gh-115911: Test that an unreadable CWD does not break imports, in + # particular during early stages of interpreter startup. + with ( + os_helper.temp_dir() as new_dir, + os_helper.save_mode(new_dir), + os_helper.change_cwd(new_dir), + util.import_state(path=['']), + ): + # chmod() is done here (inside the 'with' block) because the order + # of teardown operations cannot be the reverse of setup order. See + # https://github.com/python/cpython/pull/116131#discussion_r1739649390 + try: + os.chmod(new_dir, 0o000) + except OSError: + self.skipTest("platform does not allow " + "changing mode of the cwd") + + # Do not want PermissionError raised. + self.assertIsNone(self.machinery.PathFinder.find_spec('whatever')) + def test_invalidate_caches_finders(self): # Finders with an invalidate_caches() method have it called. class FakeFinder: diff --git a/Lib/test/test_importlib/import_/test_relative_imports.py b/Lib/test/test_importlib/import_/test_relative_imports.py index 99c24f1fd9487c..e535d119763148 100644 --- a/Lib/test/test_importlib/import_/test_relative_imports.py +++ b/Lib/test/test_importlib/import_/test_relative_imports.py @@ -81,7 +81,7 @@ def callback(global_): self.__import__('pkg') # For __import__(). module = self.__import__('', global_, fromlist=['mod2'], level=1) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'mod2')) + self.assertHasAttr(module, 'mod2') self.assertEqual(module.mod2.attr, 'pkg.mod2') self.relative_import_test(create, globals_, callback) @@ -107,7 +107,7 @@ def callback(global_): module = self.__import__('', global_, fromlist=['module'], level=1) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'module')) + self.assertHasAttr(module, 'module') self.assertEqual(module.module.attr, 'pkg.module') self.relative_import_test(create, globals_, callback) @@ -131,7 +131,7 @@ def callback(global_): module = self.__import__('', global_, fromlist=['subpkg2'], level=2) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'subpkg2')) + self.assertHasAttr(module, 'subpkg2') self.assertEqual(module.subpkg2.attr, 'pkg.subpkg2.__init__') self.relative_import_test(create, globals_, callback) diff --git a/Lib/test/test_importlib/data/__init__.py b/Lib/test/test_importlib/metadata/__init__.py similarity index 100% rename from Lib/test/test_importlib/data/__init__.py rename to Lib/test/test_importlib/metadata/__init__.py diff --git a/Lib/test/test_importlib/_context.py b/Lib/test/test_importlib/metadata/_context.py similarity index 100% rename from Lib/test/test_importlib/_context.py rename to Lib/test/test_importlib/metadata/_context.py diff --git a/Lib/test/test_importlib/_path.py b/Lib/test/test_importlib/metadata/_path.py similarity index 88% rename from Lib/test/test_importlib/_path.py rename to Lib/test/test_importlib/metadata/_path.py index 25c799fa44cd55..b3cfb9cd549d6c 100644 --- a/Lib/test/test_importlib/_path.py +++ b/Lib/test/test_importlib/metadata/_path.py @@ -17,20 +17,15 @@ class Symlink(str): @runtime_checkable class TreeMaker(Protocol): - def __truediv__(self, *args, **kwargs): - ... # pragma: no cover + def __truediv__(self, *args, **kwargs): ... # pragma: no cover - def mkdir(self, **kwargs): - ... # pragma: no cover + def mkdir(self, **kwargs): ... # pragma: no cover - def write_text(self, content, **kwargs): - ... # pragma: no cover + def write_text(self, content, **kwargs): ... # pragma: no cover - def write_bytes(self, content): - ... # pragma: no cover + def write_bytes(self, content): ... # pragma: no cover - def symlink_to(self, target): - ... # pragma: no cover + def symlink_to(self, target): ... # pragma: no cover def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker: diff --git a/Lib/test/test_importlib/resources/data01/__init__.py b/Lib/test/test_importlib/metadata/data/__init__.py similarity index 100% rename from Lib/test/test_importlib/resources/data01/__init__.py rename to Lib/test/test_importlib/metadata/data/__init__.py diff --git a/Lib/test/test_importlib/data/example-21.12-py3-none-any.whl b/Lib/test/test_importlib/metadata/data/example-21.12-py3-none-any.whl similarity index 100% rename from Lib/test/test_importlib/data/example-21.12-py3-none-any.whl rename to Lib/test/test_importlib/metadata/data/example-21.12-py3-none-any.whl diff --git a/Lib/test/test_importlib/data/example-21.12-py3.6.egg b/Lib/test/test_importlib/metadata/data/example-21.12-py3.6.egg similarity index 100% rename from Lib/test/test_importlib/data/example-21.12-py3.6.egg rename to Lib/test/test_importlib/metadata/data/example-21.12-py3.6.egg diff --git a/Lib/test/test_importlib/data/example2-1.0.0-py3-none-any.whl b/Lib/test/test_importlib/metadata/data/example2-1.0.0-py3-none-any.whl similarity index 100% rename from Lib/test/test_importlib/data/example2-1.0.0-py3-none-any.whl rename to Lib/test/test_importlib/metadata/data/example2-1.0.0-py3-none-any.whl diff --git a/Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py b/Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py new file mode 100644 index 00000000000000..ba73b743394169 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py @@ -0,0 +1,2 @@ +def main(): + return 'example' diff --git a/Lib/test/test_importlib/metadata/data/sources/example/setup.py b/Lib/test/test_importlib/metadata/data/sources/example/setup.py new file mode 100644 index 00000000000000..479488a0348186 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example/setup.py @@ -0,0 +1,11 @@ +from setuptools import setup + +setup( + name='example', + version='21.12', + license='Apache Software License', + packages=['example'], + entry_points={ + 'console_scripts': ['example = example:main', 'Example=example:main'], + }, +) diff --git a/Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py b/Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py new file mode 100644 index 00000000000000..de645c2e8bc75b --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py @@ -0,0 +1,2 @@ +def main(): + return "example" diff --git a/Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml b/Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml new file mode 100644 index 00000000000000..011f4751fb9e32 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml @@ -0,0 +1,10 @@ +[build-system] +build-backend = 'trampolim' +requires = ['trampolim'] + +[project] +name = 'example2' +version = '1.0.0' + +[project.scripts] +example = 'example2:main' diff --git a/Lib/test/test_importlib/fixtures.py b/Lib/test/test_importlib/metadata/fixtures.py similarity index 81% rename from Lib/test/test_importlib/fixtures.py rename to Lib/test/test_importlib/metadata/fixtures.py index 8c973356b5660d..826b1b3259b4cd 100644 --- a/Lib/test/test_importlib/fixtures.py +++ b/Lib/test/test_importlib/metadata/fixtures.py @@ -1,15 +1,14 @@ -import os import sys import copy import json import shutil import pathlib -import tempfile import textwrap import functools import contextlib -from test.support.os_helper import FS_NONASCII +from test.support import import_helper +from test.support import os_helper from test.support import requires_zlib from . import _path @@ -26,29 +25,12 @@ @contextlib.contextmanager -def tempdir(): - tmpdir = tempfile.mkdtemp() - try: - yield pathlib.Path(tmpdir) - finally: - shutil.rmtree(tmpdir) - - -@contextlib.contextmanager -def save_cwd(): - orig = os.getcwd() - try: - yield - finally: - os.chdir(orig) - - -@contextlib.contextmanager -def tempdir_as_cwd(): - with tempdir() as tmp: - with save_cwd(): - os.chdir(str(tmp)) - yield tmp +def tmp_path(): + """ + Like os_helper.temp_dir, but yields a pathlib.Path. + """ + with os_helper.temp_dir() as path: + yield pathlib.Path(path) @contextlib.contextmanager @@ -69,7 +51,7 @@ def setUp(self): class SiteDir(Fixtures): def setUp(self): super().setUp() - self.site_dir = self.fixtures.enter_context(tempdir()) + self.site_dir = self.fixtures.enter_context(tmp_path()) class OnSysPath(Fixtures): @@ -85,6 +67,7 @@ def add_sys_path(dir): def setUp(self): super().setUp() self.fixtures.enter_context(self.add_sys_path(self.site_dir)) + self.fixtures.enter_context(import_helper.isolated_modules()) class SiteBuilder(SiteDir): @@ -141,15 +124,13 @@ class DistInfoPkgEditable(DistInfoPkg): some_hash = '524127ce937f7cb65665130c695abd18ca386f60bb29687efb976faa1596fdcc' files: FilesSpec = { 'distinfo_pkg-1.0.0.dist-info': { - 'direct_url.json': json.dumps( - { - "archive_info": { - "hash": f"sha256={some_hash}", - "hashes": {"sha256": f"{some_hash}"}, - }, - "url": "file:///path/to/distinfo_pkg-1.0.0.editable-py3-none-any.whl", - } - ) + 'direct_url.json': json.dumps({ + "archive_info": { + "hash": f"sha256={some_hash}", + "hashes": {"sha256": f"{some_hash}"}, + }, + "url": "file:///path/to/distinfo_pkg-1.0.0.editable-py3-none-any.whl", + }) }, } @@ -253,6 +234,40 @@ def main(): } +class EggInfoPkgPipInstalledExternalDataFiles(OnSysPath, SiteBuilder): + files: FilesSpec = { + "egg_with_module_pkg.egg-info": { + "PKG-INFO": "Name: egg_with_module-pkg", + # SOURCES.txt is made from the source archive, and contains files + # (setup.py) that are not present after installation. + "SOURCES.txt": """ + egg_with_module.py + setup.py + egg_with_module.json + egg_with_module_pkg.egg-info/PKG-INFO + egg_with_module_pkg.egg-info/SOURCES.txt + egg_with_module_pkg.egg-info/top_level.txt + """, + # installed-files.txt is written by pip, and is a strictly more + # accurate source than SOURCES.txt as to the installed contents of + # the package. + "installed-files.txt": """ + ../../../etc/jupyter/jupyter_notebook_config.d/relative.json + /etc/jupyter/jupyter_notebook_config.d/absolute.json + ../egg_with_module.py + PKG-INFO + SOURCES.txt + top_level.txt + """, + # missing top_level.txt (to trigger fallback to installed-files.txt) + }, + "egg_with_module.py": """ + def main(): + print("hello world") + """, + } + + class EggInfoPkgPipInstalledNoModules(OnSysPath, SiteBuilder): files: FilesSpec = { "egg_with_no_modules_pkg.egg-info": { @@ -338,7 +353,9 @@ def record_names(file_defs): class FileBuilder: def unicode_filename(self): - return FS_NONASCII or self.skip("File system does not support non-ascii.") + return os_helper.FS_NONASCII or self.skip( + "File system does not support non-ascii." + ) def DALS(str): @@ -348,7 +365,7 @@ def DALS(str): @requires_zlib() class ZipFixtures: - root = 'test.test_importlib.data' + root = 'test.test_importlib.metadata.data' def _fixture_on_path(self, filename): pkg_file = resources.files(self.root).joinpath(filename) diff --git a/Lib/test/test_importlib/stubs.py b/Lib/test/test_importlib/metadata/stubs.py similarity index 100% rename from Lib/test/test_importlib/stubs.py rename to Lib/test/test_importlib/metadata/stubs.py diff --git a/Lib/test/test_importlib/test_metadata_api.py b/Lib/test/test_importlib/metadata/test_api.py similarity index 98% rename from Lib/test/test_importlib/test_metadata_api.py rename to Lib/test/test_importlib/metadata/test_api.py index 33c6e85ee94753..2256e0c502e46f 100644 --- a/Lib/test/test_importlib/test_metadata_api.py +++ b/Lib/test/test_importlib/metadata/test_api.py @@ -29,6 +29,7 @@ class APITests( fixtures.EggInfoPkg, fixtures.EggInfoPkgPipInstalledNoToplevel, fixtures.EggInfoPkgPipInstalledNoModules, + fixtures.EggInfoPkgPipInstalledExternalDataFiles, fixtures.EggInfoPkgSourcesFallback, fixtures.DistInfoPkg, fixtures.DistInfoPkgWithDot, @@ -109,7 +110,7 @@ def test_entry_points_unique_packages_normalized(self): Entry points should only be exposed for the first package on sys.path with a given name (even when normalized). """ - alt_site_dir = self.fixtures.enter_context(fixtures.tempdir()) + alt_site_dir = self.fixtures.enter_context(fixtures.tmp_path()) self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) alt_pkg = { "DistInfo_pkg-1.1.0.dist-info": { diff --git a/Lib/test/test_importlib/test_main.py b/Lib/test/test_importlib/metadata/test_main.py similarity index 95% rename from Lib/test/test_importlib/test_main.py rename to Lib/test/test_importlib/metadata/test_main.py index 0a769b89841234..e4218076f8cb0e 100644 --- a/Lib/test/test_importlib/test_main.py +++ b/Lib/test/test_importlib/metadata/test_main.py @@ -2,6 +2,7 @@ import pickle import unittest import warnings +import importlib import importlib.metadata import contextlib from test.support import os_helper @@ -137,7 +138,7 @@ def test_unique_distributions(self): fixtures.build_files(self.make_pkg('abc'), self.site_dir) before = list(_unique(distributions())) - alt_site_dir = self.fixtures.enter_context(fixtures.tempdir()) + alt_site_dir = self.fixtures.enter_context(fixtures.tmp_path()) self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) fixtures.build_files(self.make_pkg('ABC'), alt_site_dir) after = list(_unique(distributions())) @@ -308,12 +309,10 @@ def test_sortable(self): """ EntryPoint objects are sortable, but result is undefined. """ - sorted( - [ - EntryPoint(name='b', value='val', group='group'), - EntryPoint(name='a', value='val', group='group'), - ] - ) + sorted([ + EntryPoint(name='b', value='val', group='group'), + EntryPoint(name='a', value='val', group='group'), + ]) class FileSystem( @@ -380,18 +379,16 @@ def test_packages_distributions_all_module_types(self): 'all_distributions-1.0.0.dist-info': metadata, } for i, suffix in enumerate(suffixes): - files.update( - { - f'importable-name {i}{suffix}': '', - f'in_namespace_{i}': { - f'mod{suffix}': '', - }, - f'in_package_{i}': { - '__init__.py': '', - f'mod{suffix}': '', - }, - } - ) + files.update({ + f'importable-name {i}{suffix}': '', + f'in_namespace_{i}': { + f'mod{suffix}': '', + }, + f'in_package_{i}': { + '__init__.py': '', + f'mod{suffix}': '', + }, + }) metadata.update(RECORD=fixtures.build_record(files)) fixtures.build_files(files, prefix=self.site_dir) diff --git a/Lib/test/test_importlib/test_zip.py b/Lib/test/test_importlib/metadata/test_zip.py similarity index 100% rename from Lib/test/test_importlib/test_zip.py rename to Lib/test/test_importlib/metadata/test_zip.py diff --git a/Lib/test/test_importlib/resources/_path.py b/Lib/test/test_importlib/resources/_path.py index 1f97c96146960d..b144628cb73c77 100644 --- a/Lib/test/test_importlib/resources/_path.py +++ b/Lib/test/test_importlib/resources/_path.py @@ -2,15 +2,44 @@ import functools from typing import Dict, Union +from typing import runtime_checkable +from typing import Protocol #### -# from jaraco.path 3.4.1 +# from jaraco.path 3.7.1 -FilesSpec = Dict[str, Union[str, bytes, 'FilesSpec']] # type: ignore +class Symlink(str): + """ + A string indicating the target of a symlink. + """ + + +FilesSpec = Dict[str, Union[str, bytes, Symlink, 'FilesSpec']] + + +@runtime_checkable +class TreeMaker(Protocol): + def __truediv__(self, *args, **kwargs): ... # pragma: no cover + + def mkdir(self, **kwargs): ... # pragma: no cover + + def write_text(self, content, **kwargs): ... # pragma: no cover + + def write_bytes(self, content): ... # pragma: no cover -def build(spec: FilesSpec, prefix=pathlib.Path()): + def symlink_to(self, target): ... # pragma: no cover + + +def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker: + return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) # type: ignore[return-value] + + +def build( + spec: FilesSpec, + prefix: Union[str, TreeMaker] = pathlib.Path(), # type: ignore[assignment] +): """ Build a set of files/directories, as described by the spec. @@ -25,21 +54,25 @@ def build(spec: FilesSpec, prefix=pathlib.Path()): ... "__init__.py": "", ... }, ... "baz.py": "# Some code", - ... } + ... "bar.py": Symlink("baz.py"), + ... }, + ... "bing": Symlink("foo"), ... } >>> target = getfixture('tmp_path') >>> build(spec, target) >>> target.joinpath('foo/baz.py').read_text(encoding='utf-8') '# Some code' + >>> target.joinpath('bing/bar.py').read_text(encoding='utf-8') + '# Some code' """ for name, contents in spec.items(): - create(contents, pathlib.Path(prefix) / name) + create(contents, _ensure_tree_maker(prefix) / name) @functools.singledispatch def create(content: Union[str, bytes, FilesSpec], path): path.mkdir(exist_ok=True) - build(content, prefix=path) # type: ignore + build(content, prefix=path) # type: ignore[arg-type] @create.register @@ -52,5 +85,10 @@ def _(content: str, path): path.write_text(content, encoding='utf-8') +@create.register +def _(content: Symlink, path): + path.symlink_to(content) + + # end from jaraco.path #### diff --git a/Lib/test/test_importlib/resources/data01/binary.file b/Lib/test/test_importlib/resources/data01/binary.file deleted file mode 100644 index eaf36c1daccfdf..00000000000000 Binary files a/Lib/test/test_importlib/resources/data01/binary.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/data01/subdirectory/binary.file b/Lib/test/test_importlib/resources/data01/subdirectory/binary.file deleted file mode 100644 index eaf36c1daccfdf..00000000000000 Binary files a/Lib/test/test_importlib/resources/data01/subdirectory/binary.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/data01/utf-16.file b/Lib/test/test_importlib/resources/data01/utf-16.file deleted file mode 100644 index 2cb772295ef4b4..00000000000000 Binary files a/Lib/test/test_importlib/resources/data01/utf-16.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/data01/utf-8.file b/Lib/test/test_importlib/resources/data01/utf-8.file deleted file mode 100644 index 1c0132ad90a192..00000000000000 --- a/Lib/test/test_importlib/resources/data01/utf-8.file +++ /dev/null @@ -1 +0,0 @@ -Hello, UTF-8 world! diff --git a/Lib/test/test_importlib/resources/data02/one/__init__.py b/Lib/test/test_importlib/resources/data02/one/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/data02/one/resource1.txt b/Lib/test/test_importlib/resources/data02/one/resource1.txt deleted file mode 100644 index 61a813e40174a6..00000000000000 --- a/Lib/test/test_importlib/resources/data02/one/resource1.txt +++ /dev/null @@ -1 +0,0 @@ -one resource diff --git a/Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt b/Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt deleted file mode 100644 index 48f587a2d0ac53..00000000000000 --- a/Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt +++ /dev/null @@ -1 +0,0 @@ -a resource \ No newline at end of file diff --git a/Lib/test/test_importlib/resources/data02/two/__init__.py b/Lib/test/test_importlib/resources/data02/two/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/data02/two/resource2.txt b/Lib/test/test_importlib/resources/data02/two/resource2.txt deleted file mode 100644 index a80ce46ea362e2..00000000000000 --- a/Lib/test/test_importlib/resources/data02/two/resource2.txt +++ /dev/null @@ -1 +0,0 @@ -two resource diff --git a/Lib/test/test_importlib/resources/data03/__init__.py b/Lib/test/test_importlib/resources/data03/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/data03/namespace/portion1/__init__.py b/Lib/test/test_importlib/resources/data03/namespace/portion1/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/data03/namespace/portion2/__init__.py b/Lib/test/test_importlib/resources/data03/namespace/portion2/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/data03/namespace/resource1.txt b/Lib/test/test_importlib/resources/data03/namespace/resource1.txt deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/namespacedata01/binary.file b/Lib/test/test_importlib/resources/namespacedata01/binary.file deleted file mode 100644 index eaf36c1daccfdf..00000000000000 Binary files a/Lib/test/test_importlib/resources/namespacedata01/binary.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/namespacedata01/utf-16.file b/Lib/test/test_importlib/resources/namespacedata01/utf-16.file deleted file mode 100644 index 2cb772295ef4b4..00000000000000 Binary files a/Lib/test/test_importlib/resources/namespacedata01/utf-16.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/namespacedata01/utf-8.file b/Lib/test/test_importlib/resources/namespacedata01/utf-8.file deleted file mode 100644 index 1c0132ad90a192..00000000000000 --- a/Lib/test/test_importlib/resources/namespacedata01/utf-8.file +++ /dev/null @@ -1 +0,0 @@ -Hello, UTF-8 world! diff --git a/Lib/test/test_importlib/resources/test_contents.py b/Lib/test/test_importlib/resources/test_contents.py index 1a13f043a86f03..4e4e0e9c337f23 100644 --- a/Lib/test/test_importlib/resources/test_contents.py +++ b/Lib/test/test_importlib/resources/test_contents.py @@ -1,7 +1,6 @@ import unittest from importlib import resources -from . import data01 from . import util @@ -19,25 +18,21 @@ def test_contents(self): assert self.expected <= contents -class ContentsDiskTests(ContentsTests, unittest.TestCase): - def setUp(self): - self.data = data01 +class ContentsDiskTests(ContentsTests, util.DiskSetup, unittest.TestCase): + pass class ContentsZipTests(ContentsTests, util.ZipSetup, unittest.TestCase): pass -class ContentsNamespaceTests(ContentsTests, unittest.TestCase): +class ContentsNamespaceTests(ContentsTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' + expected = { # no __init__ because of namespace design - # no subdirectory as incidental difference in fixture 'binary.file', + 'subdirectory', 'utf-16.file', 'utf-8.file', } - - def setUp(self): - from . import namespacedata01 - - self.data = namespacedata01 diff --git a/Lib/test/test_importlib/resources/test_custom.py b/Lib/test/test_importlib/resources/test_custom.py index 73127209a2761b..640f90fc0dd91a 100644 --- a/Lib/test/test_importlib/resources/test_custom.py +++ b/Lib/test/test_importlib/resources/test_custom.py @@ -5,6 +5,7 @@ from test.support import os_helper from importlib import resources +from importlib.resources import abc from importlib.resources.abc import TraversableResources, ResourceReader from . import util @@ -39,8 +40,9 @@ def setUp(self): self.addCleanup(self.fixtures.close) def test_custom_loader(self): - temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + temp_dir = pathlib.Path(self.fixtures.enter_context(os_helper.temp_dir())) loader = SimpleLoader(MagicResources(temp_dir)) pkg = util.create_package_from_loader(loader) files = resources.files(pkg) - assert files is temp_dir + assert isinstance(files, abc.Traversable) + assert list(files.iterdir()) == [] diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index 1450cfb310926a..db8a4e62a32dc6 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -1,4 +1,7 @@ -import typing +import os +import pathlib +import py_compile +import shutil import textwrap import unittest import warnings @@ -7,11 +10,8 @@ from importlib import resources from importlib.resources.abc import Traversable -from . import data01 from . import util -from . import _path -from test.support import os_helper -from test.support import import_helper +from test.support import os_helper, import_helper @contextlib.contextmanager @@ -32,13 +32,14 @@ def test_read_text(self): actual = files.joinpath('utf-8.file').read_text(encoding='utf-8') assert actual == 'Hello, UTF-8 world!\n' - @unittest.skipUnless( - hasattr(typing, 'runtime_checkable'), - "Only suitable when typing supports runtime_checkable", - ) def test_traversable(self): assert isinstance(resources.files(self.data), Traversable) + def test_joinpath_with_multiple_args(self): + files = resources.files(self.data) + binfile = files.joinpath('subdirectory', 'binary.file') + self.assertTrue(binfile.is_file()) + def test_old_parameter(self): """ Files used to take a 'package' parameter. Make sure anyone @@ -48,66 +49,145 @@ def test_old_parameter(self): resources.files(package=self.data) -class OpenDiskTests(FilesTests, unittest.TestCase): - def setUp(self): - self.data = data01 +class OpenDiskTests(FilesTests, util.DiskSetup, unittest.TestCase): + pass class OpenZipTests(FilesTests, util.ZipSetup, unittest.TestCase): pass -class OpenNamespaceTests(FilesTests, unittest.TestCase): - def setUp(self): - from . import namespacedata01 +class OpenNamespaceTests(FilesTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + def test_non_paths_in_dunder_path(self): + """ + Non-path items in a namespace package's ``__path__`` are ignored. + + As reported in python/importlib_resources#311, some tools + like Setuptools, when creating editable packages, will inject + non-paths into a namespace package's ``__path__``, a + sentinel like + ``__editable__.sample_namespace-1.0.finder.__path_hook__`` + to cause the ``PathEntryFinder`` to be called when searching + for packages. In that case, resources should still be loadable. + """ + import namespacedata01 + + namespacedata01.__path__.append( + '__editable__.sample_namespace-1.0.finder.__path_hook__' + ) + + resources.files(namespacedata01) + + +class OpenNamespaceZipTests(FilesTests, util.ZipSetup, unittest.TestCase): + ZIP_MODULE = 'namespacedata01' - self.data = namespacedata01 +class DirectSpec: + """ + Override behavior of ModuleSetup to write a full spec directly. + """ -class SiteDir: - def setUp(self): - self.fixtures = contextlib.ExitStack() - self.addCleanup(self.fixtures.close) - self.site_dir = self.fixtures.enter_context(os_helper.temp_dir()) - self.fixtures.enter_context(import_helper.DirsOnSysPath(self.site_dir)) - self.fixtures.enter_context(import_helper.CleanImport()) + MODULE = 'unused' + def load_fixture(self, name): + self.tree_on_path(self.spec) + + +class ModulesFiles: + spec = { + 'mod.py': '', + 'res.txt': 'resources are the best', + } -class ModulesFilesTests(SiteDir, unittest.TestCase): def test_module_resources(self): """ A module can have resources found adjacent to the module. """ - spec = { - 'mod.py': '', - 'res.txt': 'resources are the best', - } - _path.build(spec, self.site_dir) - import mod + import mod # type: ignore[import-not-found] actual = resources.files(mod).joinpath('res.txt').read_text(encoding='utf-8') - assert actual == spec['res.txt'] + assert actual == self.spec['res.txt'] + + +class ModuleFilesDiskTests(DirectSpec, util.DiskSetup, ModulesFiles, unittest.TestCase): + pass -class ImplicitContextFilesTests(SiteDir, unittest.TestCase): - def test_implicit_files(self): +class ModuleFilesZipTests(DirectSpec, util.ZipSetup, ModulesFiles, unittest.TestCase): + pass + + +class ImplicitContextFiles: + set_val = textwrap.dedent( + f""" + import {resources.__name__} as res + val = res.files().joinpath('res.txt').read_text(encoding='utf-8') + """ + ) + spec = { + 'somepkg': { + '__init__.py': set_val, + 'submod.py': set_val, + 'res.txt': 'resources are the best', + }, + 'frozenpkg': { + '__init__.py': set_val.replace(resources.__name__, 'c_resources'), + 'res.txt': 'resources are the best', + }, + } + + def test_implicit_files_package(self): """ Without any parameter, files() will infer the location as the caller. """ - spec = { - 'somepkg': { - '__init__.py': textwrap.dedent( - """ - import importlib.resources as res - val = res.files().joinpath('res.txt').read_text(encoding='utf-8') - """ - ), - 'res.txt': 'resources are the best', - }, - } - _path.build(spec, self.site_dir) assert importlib.import_module('somepkg').val == 'resources are the best' + def test_implicit_files_submodule(self): + """ + Without any parameter, files() will infer the location as the caller. + """ + assert importlib.import_module('somepkg.submod').val == 'resources are the best' + + def _compile_importlib(self): + """ + Make a compiled-only copy of the importlib resources package. + + Currently only code is copied, as importlib resources doesn't itself + have any resources. + """ + bin_site = self.fixtures.enter_context(os_helper.temp_dir()) + c_resources = pathlib.Path(bin_site, 'c_resources') + sources = pathlib.Path(resources.__file__).parent + + for source_path in sources.glob('**/*.py'): + c_path = c_resources.joinpath(source_path.relative_to(sources)).with_suffix('.pyc') + py_compile.compile(source_path, c_path) + self.fixtures.enter_context(import_helper.DirsOnSysPath(bin_site)) + + def test_implicit_files_with_compiled_importlib(self): + """ + Caller detection works for compiled-only resources module. + + python/cpython#123085 + """ + self._compile_importlib() + assert importlib.import_module('frozenpkg').val == 'resources are the best' + + +class ImplicitContextFilesDiskTests( + DirectSpec, util.DiskSetup, ImplicitContextFiles, unittest.TestCase +): + pass + + +class ImplicitContextFilesZipTests( + DirectSpec, util.ZipSetup, ImplicitContextFiles, unittest.TestCase +): + pass + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/resources/test_functional.py b/Lib/test/test_importlib/resources/test_functional.py new file mode 100644 index 00000000000000..e8d25fa4d9faf0 --- /dev/null +++ b/Lib/test/test_importlib/resources/test_functional.py @@ -0,0 +1,249 @@ +import unittest +import os +import importlib + +from test.support import warnings_helper + +from importlib import resources + +from . import util + +# Since the functional API forwards to Traversable, we only test +# filesystem resources here -- not zip files, namespace packages etc. +# We do test for two kinds of Anchor, though. + + +class StringAnchorMixin: + anchor01 = 'data01' + anchor02 = 'data02' + + +class ModuleAnchorMixin: + @property + def anchor01(self): + return importlib.import_module('data01') + + @property + def anchor02(self): + return importlib.import_module('data02') + + +class FunctionalAPIBase(util.DiskSetup): + def setUp(self): + super().setUp() + self.load_fixture('data02') + + def _gen_resourcetxt_path_parts(self): + """Yield various names of a text file in anchor02, each in a subTest""" + for path_parts in ( + ('subdirectory', 'subsubdir', 'resource.txt'), + ('subdirectory/subsubdir/resource.txt',), + ('subdirectory/subsubdir', 'resource.txt'), + ): + with self.subTest(path_parts=path_parts): + yield path_parts + + def test_read_text(self): + self.assertEqual( + resources.read_text(self.anchor01, 'utf-8.file'), + 'Hello, UTF-8 world!\n', + ) + self.assertEqual( + resources.read_text( + self.anchor02, + 'subdirectory', + 'subsubdir', + 'resource.txt', + encoding='utf-8', + ), + 'a resource', + ) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertEqual( + resources.read_text( + self.anchor02, + *path_parts, + encoding='utf-8', + ), + 'a resource', + ) + # Use generic OSError, since e.g. attempting to read a directory can + # fail with PermissionError rather than IsADirectoryError + with self.assertRaises(OSError): + resources.read_text(self.anchor01) + with self.assertRaises(OSError): + resources.read_text(self.anchor01, 'no-such-file') + with self.assertRaises(UnicodeDecodeError): + resources.read_text(self.anchor01, 'utf-16.file') + self.assertEqual( + resources.read_text( + self.anchor01, + 'binary.file', + encoding='latin1', + ), + '\x00\x01\x02\x03', + ) + self.assertEndsWith( # ignore the BOM + resources.read_text( + self.anchor01, + 'utf-16.file', + errors='backslashreplace', + ), + 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( + errors='backslashreplace', + ), + ) + + def test_read_binary(self): + self.assertEqual( + resources.read_binary(self.anchor01, 'utf-8.file'), + b'Hello, UTF-8 world!\n', + ) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertEqual( + resources.read_binary(self.anchor02, *path_parts), + b'a resource', + ) + + def test_open_text(self): + with resources.open_text(self.anchor01, 'utf-8.file') as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + for path_parts in self._gen_resourcetxt_path_parts(): + with resources.open_text( + self.anchor02, + *path_parts, + encoding='utf-8', + ) as f: + self.assertEqual(f.read(), 'a resource') + # Use generic OSError, since e.g. attempting to read a directory can + # fail with PermissionError rather than IsADirectoryError + with self.assertRaises(OSError): + resources.open_text(self.anchor01) + with self.assertRaises(OSError): + resources.open_text(self.anchor01, 'no-such-file') + with resources.open_text(self.anchor01, 'utf-16.file') as f: + with self.assertRaises(UnicodeDecodeError): + f.read() + with resources.open_text( + self.anchor01, + 'binary.file', + encoding='latin1', + ) as f: + self.assertEqual(f.read(), '\x00\x01\x02\x03') + with resources.open_text( + self.anchor01, + 'utf-16.file', + errors='backslashreplace', + ) as f: + self.assertEndsWith( # ignore the BOM + f.read(), + 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( + errors='backslashreplace', + ), + ) + + def test_open_binary(self): + with resources.open_binary(self.anchor01, 'utf-8.file') as f: + self.assertEqual(f.read(), b'Hello, UTF-8 world!\n') + for path_parts in self._gen_resourcetxt_path_parts(): + with resources.open_binary( + self.anchor02, + *path_parts, + ) as f: + self.assertEqual(f.read(), b'a resource') + + def test_path(self): + with resources.path(self.anchor01, 'utf-8.file') as path: + with open(str(path), encoding='utf-8') as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + with resources.path(self.anchor01) as path: + with open(os.path.join(path, 'utf-8.file'), encoding='utf-8') as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + + def test_is_resource(self): + is_resource = resources.is_resource + self.assertTrue(is_resource(self.anchor01, 'utf-8.file')) + self.assertFalse(is_resource(self.anchor01, 'no_such_file')) + self.assertFalse(is_resource(self.anchor01)) + self.assertFalse(is_resource(self.anchor01, 'subdirectory')) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertTrue(is_resource(self.anchor02, *path_parts)) + + def test_contents(self): + with warnings_helper.check_warnings((".*contents.*", DeprecationWarning)): + c = resources.contents(self.anchor01) + self.assertGreaterEqual( + set(c), + {'utf-8.file', 'utf-16.file', 'binary.file', 'subdirectory'}, + ) + with self.assertRaises(OSError), warnings_helper.check_warnings(( + ".*contents.*", + DeprecationWarning, + )): + list(resources.contents(self.anchor01, 'utf-8.file')) + + for path_parts in self._gen_resourcetxt_path_parts(): + with self.assertRaises(OSError), warnings_helper.check_warnings(( + ".*contents.*", + DeprecationWarning, + )): + list(resources.contents(self.anchor01, *path_parts)) + with warnings_helper.check_warnings((".*contents.*", DeprecationWarning)): + c = resources.contents(self.anchor01, 'subdirectory') + self.assertGreaterEqual( + set(c), + {'binary.file'}, + ) + + @warnings_helper.ignore_warnings(category=DeprecationWarning) + def test_common_errors(self): + for func in ( + resources.read_text, + resources.read_binary, + resources.open_text, + resources.open_binary, + resources.path, + resources.is_resource, + resources.contents, + ): + with self.subTest(func=func): + # Rejecting None anchor + with self.assertRaises(TypeError): + func(None) + # Rejecting invalid anchor type + with self.assertRaises((TypeError, AttributeError)): + func(1234) + # Unknown module + with self.assertRaises(ModuleNotFoundError): + func('$missing module$') + + def test_text_errors(self): + for func in ( + resources.read_text, + resources.open_text, + ): + with self.subTest(func=func): + # Multiple path arguments need explicit encoding argument. + with self.assertRaises(TypeError): + func( + self.anchor02, + 'subdirectory', + 'subsubdir', + 'resource.txt', + ) + + +class FunctionalAPITest_StringAnchor( + StringAnchorMixin, + FunctionalAPIBase, + unittest.TestCase, +): + pass + + +class FunctionalAPITest_ModuleAnchor( + ModuleAnchorMixin, + FunctionalAPIBase, + unittest.TestCase, +): + pass diff --git a/Lib/test/test_importlib/resources/test_open.py b/Lib/test/test_importlib/resources/test_open.py index 86becb4bfaad37..8c00378ad3cc9c 100644 --- a/Lib/test/test_importlib/resources/test_open.py +++ b/Lib/test/test_importlib/resources/test_open.py @@ -1,7 +1,6 @@ import unittest from importlib import resources -from . import data01 from . import util @@ -24,7 +23,7 @@ def test_open_binary(self): target = resources.files(self.data) / 'binary.file' with target.open('rb') as fp: result = fp.read() - self.assertEqual(result, b'\x00\x01\x02\x03') + self.assertEqual(result, bytes(range(4))) def test_open_text_default_encoding(self): target = resources.files(self.data) / 'utf-8.file' @@ -65,21 +64,21 @@ def test_open_text_FileNotFoundError(self): target.open(encoding='utf-8') -class OpenDiskTests(OpenTests, unittest.TestCase): - def setUp(self): - self.data = data01 - +class OpenDiskTests(OpenTests, util.DiskSetup, unittest.TestCase): + pass -class OpenDiskNamespaceTests(OpenTests, unittest.TestCase): - def setUp(self): - from . import namespacedata01 - self.data = namespacedata01 +class OpenDiskNamespaceTests(OpenTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' class OpenZipTests(OpenTests, util.ZipSetup, unittest.TestCase): pass +class OpenNamespaceZipTests(OpenTests, util.ZipSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/resources/test_path.py b/Lib/test/test_importlib/resources/test_path.py index 34a6bdd2d58b91..903911f57b3306 100644 --- a/Lib/test/test_importlib/resources/test_path.py +++ b/Lib/test/test_importlib/resources/test_path.py @@ -1,8 +1,8 @@ import io +import pathlib import unittest from importlib import resources -from . import data01 from . import util @@ -15,23 +15,16 @@ def execute(self, package, path): class PathTests: def test_reading(self): """ - Path should be readable. - - Test also implicitly verifies the returned object is a pathlib.Path - instance. + Path should be readable and a pathlib.Path instance. """ target = resources.files(self.data) / 'utf-8.file' with resources.as_file(target) as path: - self.assertTrue(path.name.endswith("utf-8.file"), repr(path)) - # pathlib.Path.read_text() was introduced in Python 3.5. - with path.open('r', encoding='utf-8') as file: - text = file.read() - self.assertEqual('Hello, UTF-8 world!\n', text) - + self.assertIsInstance(path, pathlib.Path) + self.assertEndsWith(path.name, "utf-8.file") + self.assertEqual('Hello, UTF-8 world!\n', path.read_text(encoding='utf-8')) -class PathDiskTests(PathTests, unittest.TestCase): - data = data01 +class PathDiskTests(PathTests, util.DiskSetup, unittest.TestCase): def test_natural_path(self): # Guarantee the internal implementation detail that # file-system-backed resources do not get the tempdir diff --git a/Lib/test/test_importlib/resources/test_read.py b/Lib/test/test_importlib/resources/test_read.py index 088982681e8b0c..59c237d964121e 100644 --- a/Lib/test/test_importlib/resources/test_read.py +++ b/Lib/test/test_importlib/resources/test_read.py @@ -1,7 +1,7 @@ import unittest from importlib import import_module, resources -from . import data01 + from . import util @@ -18,7 +18,7 @@ def execute(self, package, path): class ReadTests: def test_read_bytes(self): result = resources.files(self.data).joinpath('binary.file').read_bytes() - self.assertEqual(result, b'\0\1\2\3') + self.assertEqual(result, bytes(range(4))) def test_read_text_default_encoding(self): result = ( @@ -51,30 +51,42 @@ def test_read_text_with_errors(self): ) -class ReadDiskTests(ReadTests, unittest.TestCase): - data = data01 +class ReadDiskTests(ReadTests, util.DiskSetup, unittest.TestCase): + pass class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase): def test_read_submodule_resource(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') result = resources.files(submodule).joinpath('binary.file').read_bytes() - self.assertEqual(result, b'\0\1\2\3') + self.assertEqual(result, bytes(range(4, 8))) def test_read_submodule_resource_by_name(self): result = ( - resources.files('ziptestdata.subdirectory') - .joinpath('binary.file') - .read_bytes() + resources.files('data01.subdirectory').joinpath('binary.file').read_bytes() ) - self.assertEqual(result, b'\0\1\2\3') + self.assertEqual(result, bytes(range(4, 8))) + +class ReadNamespaceTests(ReadTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' -class ReadNamespaceTests(ReadTests, unittest.TestCase): - def setUp(self): - from . import namespacedata01 - self.data = namespacedata01 +class ReadNamespaceZipTests(ReadTests, util.ZipSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + def test_read_submodule_resource(self): + submodule = import_module('namespacedata01.subdirectory') + result = resources.files(submodule).joinpath('binary.file').read_bytes() + self.assertEqual(result, bytes(range(12, 16))) + + def test_read_submodule_resource_by_name(self): + result = ( + resources.files('namespacedata01.subdirectory') + .joinpath('binary.file') + .read_bytes() + ) + self.assertEqual(result, bytes(range(12, 16))) if __name__ == '__main__': diff --git a/Lib/test/test_importlib/resources/test_reader.py b/Lib/test/test_importlib/resources/test_reader.py index 8670f72a334585..ed5693ab416798 100644 --- a/Lib/test/test_importlib/resources/test_reader.py +++ b/Lib/test/test_importlib/resources/test_reader.py @@ -1,17 +1,21 @@ import os.path -import sys import pathlib import unittest from importlib import import_module from importlib.readers import MultiplexedPath, NamespaceReader +from . import util -class MultiplexedPathTest(unittest.TestCase): - @classmethod - def setUpClass(cls): - path = pathlib.Path(__file__).parent / 'namespacedata01' - cls.folder = str(path) + +class MultiplexedPathTest(util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + def setUp(self): + super().setUp() + self.folder = pathlib.Path(self.data.__path__[0]) + self.data01 = pathlib.Path(self.load_fixture('data01').__file__).parent + self.data02 = pathlib.Path(self.load_fixture('data02').__file__).parent def test_init_no_paths(self): with self.assertRaises(FileNotFoundError): @@ -19,7 +23,7 @@ def test_init_no_paths(self): def test_init_file(self): with self.assertRaises(NotADirectoryError): - MultiplexedPath(os.path.join(self.folder, 'binary.file')) + MultiplexedPath(self.folder / 'binary.file') def test_iterdir(self): contents = {path.name for path in MultiplexedPath(self.folder).iterdir()} @@ -27,12 +31,13 @@ def test_iterdir(self): contents.remove('__pycache__') except (KeyError, ValueError): pass - self.assertEqual(contents, {'binary.file', 'utf-16.file', 'utf-8.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-16.file', 'utf-8.file'} + ) def test_iterdir_duplicate(self): - data01 = os.path.abspath(os.path.join(__file__, '..', 'data01')) contents = { - path.name for path in MultiplexedPath(self.folder, data01).iterdir() + path.name for path in MultiplexedPath(self.folder, self.data01).iterdir() } for remove in ('__pycache__', '__init__.pyc'): try: @@ -60,17 +65,16 @@ def test_open_file(self): path.open() def test_join_path(self): - prefix = os.path.abspath(os.path.join(__file__, '..')) - data01 = os.path.join(prefix, 'data01') - path = MultiplexedPath(self.folder, data01) + prefix = str(self.folder.parent) + path = MultiplexedPath(self.folder, self.data01) self.assertEqual( str(path.joinpath('binary.file'))[len(prefix) + 1 :], os.path.join('namespacedata01', 'binary.file'), ) - self.assertEqual( - str(path.joinpath('subdirectory'))[len(prefix) + 1 :], - os.path.join('data01', 'subdirectory'), - ) + sub = path.joinpath('subdirectory') + assert isinstance(sub, MultiplexedPath) + assert 'namespacedata01' in str(sub) + assert 'data01' in str(sub) self.assertEqual( str(path.joinpath('imaginary'))[len(prefix) + 1 :], os.path.join('namespacedata01', 'imaginary'), @@ -82,10 +86,8 @@ def test_join_path_compound(self): assert not path.joinpath('imaginary/foo.py').exists() def test_join_path_common_subdir(self): - prefix = os.path.abspath(os.path.join(__file__, '..')) - data01 = os.path.join(prefix, 'data01') - data02 = os.path.join(prefix, 'data02') - path = MultiplexedPath(data01, data02) + prefix = str(self.data02.parent) + path = MultiplexedPath(self.data01, self.data02) self.assertIsInstance(path.joinpath('subdirectory'), MultiplexedPath) self.assertEqual( str(path.joinpath('subdirectory', 'subsubdir'))[len(prefix) + 1 :], @@ -105,16 +107,8 @@ def test_name(self): ) -class NamespaceReaderTest(unittest.TestCase): - site_dir = str(pathlib.Path(__file__).parent) - - @classmethod - def setUpClass(cls): - sys.path.append(cls.site_dir) - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.site_dir) +class NamespaceReaderTest(util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' def test_init_error(self): with self.assertRaises(ValueError): @@ -124,7 +118,7 @@ def test_resource_path(self): namespacedata01 = import_module('namespacedata01') reader = NamespaceReader(namespacedata01.__spec__.submodule_search_locations) - root = os.path.abspath(os.path.join(__file__, '..', 'namespacedata01')) + root = self.data.__path__[0] self.assertEqual( reader.resource_path('binary.file'), os.path.join(root, 'binary.file') ) @@ -133,9 +127,8 @@ def test_resource_path(self): ) def test_files(self): - namespacedata01 = import_module('namespacedata01') - reader = NamespaceReader(namespacedata01.__spec__.submodule_search_locations) - root = os.path.abspath(os.path.join(__file__, '..', 'namespacedata01')) + reader = NamespaceReader(self.data.__spec__.submodule_search_locations) + root = self.data.__path__[0] self.assertIsInstance(reader.files(), MultiplexedPath) self.assertEqual(repr(reader.files()), f"MultiplexedPath('{root}')") diff --git a/Lib/test/test_importlib/resources/test_resource.py b/Lib/test/test_importlib/resources/test_resource.py index 6f75cf57f03d02..fcede14b891a84 100644 --- a/Lib/test/test_importlib/resources/test_resource.py +++ b/Lib/test/test_importlib/resources/test_resource.py @@ -1,15 +1,7 @@ -import contextlib -import sys import unittest -import uuid -import pathlib -from . import data01 -from . import zipdata01, zipdata02 from . import util from importlib import resources, import_module -from test.support import import_helper, os_helper -from test.support.os_helper import unlink class ResourceTests: @@ -29,9 +21,8 @@ def test_is_dir(self): self.assertTrue(target.is_dir()) -class ResourceDiskTests(ResourceTests, unittest.TestCase): - def setUp(self): - self.data = data01 +class ResourceDiskTests(ResourceTests, util.DiskSetup, unittest.TestCase): + pass class ResourceZipTests(ResourceTests, util.ZipSetup, unittest.TestCase): @@ -42,33 +33,39 @@ def names(traversable): return {item.name for item in traversable.iterdir()} -class ResourceLoaderTests(unittest.TestCase): +class ResourceLoaderTests(util.DiskSetup, unittest.TestCase): def test_resource_contents(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C'] + file=self.data, path=self.data.__file__, contents=['A', 'B', 'C'] ) self.assertEqual(names(resources.files(package)), {'A', 'B', 'C'}) def test_is_file(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + file=self.data, + path=self.data.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F'], ) self.assertTrue(resources.files(package).joinpath('B').is_file()) def test_is_dir(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + file=self.data, + path=self.data.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F'], ) self.assertTrue(resources.files(package).joinpath('D').is_dir()) def test_resource_missing(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + file=self.data, + path=self.data.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F'], ) self.assertFalse(resources.files(package).joinpath('Z').is_file()) -class ResourceCornerCaseTests(unittest.TestCase): +class ResourceCornerCaseTests(util.DiskSetup, unittest.TestCase): def test_package_has_no_reader_fallback(self): """ Test odd ball packages which: @@ -77,7 +74,7 @@ def test_package_has_no_reader_fallback(self): # 3. Are not in a zip file """ module = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C'] + file=self.data, path=self.data.__file__, contents=['A', 'B', 'C'] ) # Give the module a dummy loader. module.__loader__ = object() @@ -88,43 +85,39 @@ def test_package_has_no_reader_fallback(self): self.assertFalse(resources.files(module).joinpath('A').is_file()) -class ResourceFromZipsTest01(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata01 # type: ignore - +class ResourceFromZipsTest01(util.ZipSetup, unittest.TestCase): def test_is_submodule_resource(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') self.assertTrue(resources.files(submodule).joinpath('binary.file').is_file()) def test_read_submodule_resource_by_name(self): self.assertTrue( - resources.files('ziptestdata.subdirectory') - .joinpath('binary.file') - .is_file() + resources.files('data01.subdirectory').joinpath('binary.file').is_file() ) def test_submodule_contents(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') self.assertEqual( names(resources.files(submodule)), {'__init__.py', 'binary.file'} ) def test_submodule_contents_by_name(self): self.assertEqual( - names(resources.files('ziptestdata.subdirectory')), + names(resources.files('data01.subdirectory')), {'__init__.py', 'binary.file'}, ) def test_as_file_directory(self): - with resources.as_file(resources.files('ziptestdata')) as data: - assert data.name == 'ziptestdata' + with resources.as_file(resources.files('data01')) as data: + assert data.name == 'data01' assert data.is_dir() assert data.joinpath('subdirectory').is_dir() assert len(list(data.iterdir())) assert not data.parent.exists() -class ResourceFromZipsTest02(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata02 # type: ignore +class ResourceFromZipsTest02(util.ZipSetup, unittest.TestCase): + MODULE = 'data02' def test_unrelated_contents(self): """ @@ -132,93 +125,48 @@ def test_unrelated_contents(self): distinct resources. Ref python/importlib_resources#44. """ self.assertEqual( - names(resources.files('ziptestdata.one')), + names(resources.files('data02.one')), {'__init__.py', 'resource1.txt'}, ) self.assertEqual( - names(resources.files('ziptestdata.two')), + names(resources.files('data02.two')), {'__init__.py', 'resource2.txt'}, ) -@contextlib.contextmanager -def zip_on_path(dir): - data_path = pathlib.Path(zipdata01.__file__) - source_zip_path = data_path.parent.joinpath('ziptestdata.zip') - zip_path = pathlib.Path(dir) / f'{uuid.uuid4()}.zip' - zip_path.write_bytes(source_zip_path.read_bytes()) - sys.path.append(str(zip_path)) - import_module('ziptestdata') - - try: - yield - finally: - with contextlib.suppress(ValueError): - sys.path.remove(str(zip_path)) - - with contextlib.suppress(KeyError): - del sys.path_importer_cache[str(zip_path)] - del sys.modules['ziptestdata'] - - with contextlib.suppress(OSError): - unlink(zip_path) - - -class DeletingZipsTest(unittest.TestCase): +class DeletingZipsTest(util.ZipSetup, unittest.TestCase): """Having accessed resources in a zip file should not keep an open reference to the zip. """ - def setUp(self): - self.fixtures = contextlib.ExitStack() - self.addCleanup(self.fixtures.close) - - modules = import_helper.modules_setup() - self.addCleanup(import_helper.modules_cleanup, *modules) - - temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) - self.fixtures.enter_context(zip_on_path(temp_dir)) - def test_iterdir_does_not_keep_open(self): - [item.name for item in resources.files('ziptestdata').iterdir()] + [item.name for item in resources.files('data01').iterdir()] def test_is_file_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('binary.file').is_file() + resources.files('data01').joinpath('binary.file').is_file() def test_is_file_failure_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('not-present').is_file() + resources.files('data01').joinpath('not-present').is_file() @unittest.skip("Desired but not supported.") def test_as_file_does_not_keep_open(self): # pragma: no cover - resources.as_file(resources.files('ziptestdata') / 'binary.file') + resources.as_file(resources.files('data01') / 'binary.file') def test_entered_path_does_not_keep_open(self): """ Mimic what certifi does on import to make its bundle available for the process duration. """ - resources.as_file(resources.files('ziptestdata') / 'binary.file').__enter__() + resources.as_file(resources.files('data01') / 'binary.file').__enter__() def test_read_binary_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('binary.file').read_bytes() + resources.files('data01').joinpath('binary.file').read_bytes() def test_read_text_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('utf-8.file').read_text( - encoding='utf-8' - ) + resources.files('data01').joinpath('utf-8.file').read_text(encoding='utf-8') -class ResourceFromNamespaceTest01(unittest.TestCase): - site_dir = str(pathlib.Path(__file__).parent) - - @classmethod - def setUpClass(cls): - sys.path.append(cls.site_dir) - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.site_dir) - +class ResourceFromNamespaceTests: def test_is_submodule_resource(self): self.assertTrue( resources.files(import_module('namespacedata01')) @@ -237,7 +185,9 @@ def test_submodule_contents(self): contents.remove('__pycache__') except KeyError: pass - self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} + ) def test_submodule_contents_by_name(self): contents = names(resources.files('namespacedata01')) @@ -245,7 +195,41 @@ def test_submodule_contents_by_name(self): contents.remove('__pycache__') except KeyError: pass - self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} + ) + + def test_submodule_sub_contents(self): + contents = names(resources.files(import_module('namespacedata01.subdirectory'))) + try: + contents.remove('__pycache__') + except KeyError: + pass + self.assertEqual(contents, {'binary.file'}) + + def test_submodule_sub_contents_by_name(self): + contents = names(resources.files('namespacedata01.subdirectory')) + try: + contents.remove('__pycache__') + except KeyError: + pass + self.assertEqual(contents, {'binary.file'}) + + +class ResourceFromNamespaceDiskTests( + util.DiskSetup, + ResourceFromNamespaceTests, + unittest.TestCase, +): + MODULE = 'namespacedata01' + + +class ResourceFromNamespaceZipTests( + util.ZipSetup, + ResourceFromNamespaceTests, + unittest.TestCase, +): + MODULE = 'namespacedata01' if __name__ == '__main__': diff --git a/Lib/test/test_importlib/resources/update-zips.py b/Lib/test/test_importlib/resources/update-zips.py deleted file mode 100755 index 231334aa7e38b4..00000000000000 --- a/Lib/test/test_importlib/resources/update-zips.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -Generate the zip test data files. - -Run to build the tests/zipdataNN/ziptestdata.zip files from -files in tests/dataNN. - -Replaces the file with the working copy, but does commit anything -to the source repo. -""" - -import contextlib -import os -import pathlib -import zipfile - - -def main(): - """ - >>> from unittest import mock - >>> monkeypatch = getfixture('monkeypatch') - >>> monkeypatch.setattr(zipfile, 'ZipFile', mock.MagicMock()) - >>> print(); main() # print workaround for bpo-32509 - - ...data01... -> ziptestdata/... - ... - ...data02... -> ziptestdata/... - ... - """ - suffixes = '01', '02' - tuple(map(generate, suffixes)) - - -def generate(suffix): - root = pathlib.Path(__file__).parent.relative_to(os.getcwd()) - zfpath = root / f'zipdata{suffix}/ziptestdata.zip' - with zipfile.ZipFile(zfpath, 'w') as zf: - for src, rel in walk(root / f'data{suffix}'): - dst = 'ziptestdata' / pathlib.PurePosixPath(rel.as_posix()) - print(src, '->', dst) - zf.write(src, dst) - - -def walk(datapath): - for dirpath, dirnames, filenames in os.walk(datapath): - with contextlib.suppress(ValueError): - dirnames.remove('__pycache__') - for filename in filenames: - res = pathlib.Path(dirpath) / filename - rel = res.relative_to(datapath) - yield res, rel - - -__name__ == '__main__' and main() diff --git a/Lib/test/test_importlib/resources/util.py b/Lib/test/test_importlib/resources/util.py index dbe6ee81476699..e2d995f596317d 100644 --- a/Lib/test/test_importlib/resources/util.py +++ b/Lib/test/test_importlib/resources/util.py @@ -4,11 +4,12 @@ import sys import types import pathlib +import contextlib -from . import data01 -from . import zipdata01 from importlib.resources.abc import ResourceReader -from test.support import import_helper +from test.support import import_helper, os_helper +from . import zip as zip_ +from . import _path from importlib.machinery import ModuleSpec @@ -67,7 +68,7 @@ def create_package(file=None, path=None, is_package=True, contents=()): ) -class CommonTests(metaclass=abc.ABCMeta): +class CommonTestsBase(metaclass=abc.ABCMeta): """ Tests shared by test_open, test_path, and test_read. """ @@ -83,34 +84,34 @@ def test_package_name(self): """ Passing in the package name should succeed. """ - self.execute(data01.__name__, 'utf-8.file') + self.execute(self.data.__name__, 'utf-8.file') def test_package_object(self): """ Passing in the package itself should succeed. """ - self.execute(data01, 'utf-8.file') + self.execute(self.data, 'utf-8.file') def test_string_path(self): """ Passing in a string for the path should succeed. """ path = 'utf-8.file' - self.execute(data01, path) + self.execute(self.data, path) def test_pathlib_path(self): """ Passing in a pathlib.PurePath object for the path should succeed. """ path = pathlib.PurePath('utf-8.file') - self.execute(data01, path) + self.execute(self.data, path) def test_importing_module_as_side_effect(self): """ The anchor package can already be imported. """ - del sys.modules[data01.__name__] - self.execute(data01.__name__, 'utf-8.file') + del sys.modules[self.data.__name__] + self.execute(self.data.__name__, 'utf-8.file') def test_missing_path(self): """ @@ -140,40 +141,66 @@ def test_useless_loader(self): self.execute(package, 'utf-8.file') -class ZipSetupBase: - ZIP_MODULE = None - - @classmethod - def setUpClass(cls): - data_path = pathlib.Path(cls.ZIP_MODULE.__file__) - data_dir = data_path.parent - cls._zip_path = str(data_dir / 'ziptestdata.zip') - sys.path.append(cls._zip_path) - cls.data = importlib.import_module('ziptestdata') - - @classmethod - def tearDownClass(cls): - try: - sys.path.remove(cls._zip_path) - except ValueError: - pass - - try: - del sys.path_importer_cache[cls._zip_path] - del sys.modules[cls.data.__name__] - except KeyError: - pass - - try: - del cls.data - del cls._zip_path - except AttributeError: - pass - +fixtures = dict( + data01={ + '__init__.py': '', + 'binary.file': bytes(range(4)), + 'utf-16.file': '\ufeffHello, UTF-16 world!\n'.encode('utf-16-le'), + 'utf-8.file': 'Hello, UTF-8 world!\n'.encode('utf-8'), + 'subdirectory': { + '__init__.py': '', + 'binary.file': bytes(range(4, 8)), + }, + }, + data02={ + '__init__.py': '', + 'one': {'__init__.py': '', 'resource1.txt': 'one resource'}, + 'two': {'__init__.py': '', 'resource2.txt': 'two resource'}, + 'subdirectory': {'subsubdir': {'resource.txt': 'a resource'}}, + }, + namespacedata01={ + 'binary.file': bytes(range(4)), + 'utf-16.file': '\ufeffHello, UTF-16 world!\n'.encode('utf-16-le'), + 'utf-8.file': 'Hello, UTF-8 world!\n'.encode('utf-8'), + 'subdirectory': { + 'binary.file': bytes(range(12, 16)), + }, + }, +) + + +class ModuleSetup: def setUp(self): - modules = import_helper.modules_setup() - self.addCleanup(import_helper.modules_cleanup, *modules) + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + + self.fixtures.enter_context(import_helper.isolated_modules()) + self.data = self.load_fixture(self.MODULE) + + def load_fixture(self, module): + self.tree_on_path({module: fixtures[module]}) + return importlib.import_module(module) + + +class ZipSetup(ModuleSetup): + MODULE = 'data01' + + def tree_on_path(self, spec): + temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + modules = pathlib.Path(temp_dir) / 'zipped modules.zip' + self.fixtures.enter_context( + import_helper.DirsOnSysPath(str(zip_.make_zip_file(spec, modules))) + ) + + +class DiskSetup(ModuleSetup): + MODULE = 'data01' + + def tree_on_path(self, spec): + temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + _path.build(spec, pathlib.Path(temp_dir)) + self.fixtures.enter_context(import_helper.DirsOnSysPath(temp_dir)) -class ZipSetup(ZipSetupBase): - ZIP_MODULE = zipdata01 # type: ignore +class CommonTests(DiskSetup, CommonTestsBase): + pass diff --git a/Lib/test/test_importlib/resources/zip.py b/Lib/test/test_importlib/resources/zip.py new file mode 100755 index 00000000000000..fc453f02060a31 --- /dev/null +++ b/Lib/test/test_importlib/resources/zip.py @@ -0,0 +1,24 @@ +""" +Generate zip test data files. +""" + +import zipfile + + +def make_zip_file(tree, dst): + """ + Zip the files in tree into a new zipfile at dst. + """ + with zipfile.ZipFile(dst, 'w') as zf: + for name, contents in walk(tree): + zf.writestr(name, contents) + zipfile._path.CompleteDirs.inject(zf) + return dst + + +def walk(tree, prefix=''): + for name, contents in tree.items(): + if isinstance(contents, dict): + yield from walk(contents, prefix=f'{prefix}{name}/') + else: + yield f'{prefix}{name}', contents diff --git a/Lib/test/test_importlib/resources/zipdata01/__init__.py b/Lib/test/test_importlib/resources/zipdata01/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/zipdata01/ziptestdata.zip b/Lib/test/test_importlib/resources/zipdata01/ziptestdata.zip deleted file mode 100644 index 9a3bb0739f87e9..00000000000000 Binary files a/Lib/test/test_importlib/resources/zipdata01/ziptestdata.zip and /dev/null differ diff --git a/Lib/test/test_importlib/resources/zipdata02/__init__.py b/Lib/test/test_importlib/resources/zipdata02/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/zipdata02/ziptestdata.zip b/Lib/test/test_importlib/resources/zipdata02/ziptestdata.zip deleted file mode 100644 index d63ff512d2807e..00000000000000 Binary files a/Lib/test/test_importlib/resources/zipdata02/ziptestdata.zip and /dev/null differ diff --git a/Lib/test/test_importlib/source/test_finder.py b/Lib/test/test_importlib/source/test_finder.py index 8c06c4da1f5cba..4de736a6bf3b2d 100644 --- a/Lib/test/test_importlib/source/test_finder.py +++ b/Lib/test/test_importlib/source/test_finder.py @@ -73,7 +73,7 @@ def run_test(self, test, create=None, *, compile_=None, unlink=None): if error.errno != errno.ENOENT: raise loader = self.import_(mapping['.root'], test) - self.assertTrue(hasattr(loader, 'load_module')) + self.assertHasAttr(loader, 'load_module') return loader def test_module(self): @@ -100,7 +100,7 @@ def test_module_in_package(self): with util.create_modules('pkg.__init__', 'pkg.sub') as mapping: pkg_dir = os.path.dirname(mapping['pkg.__init__']) loader = self.import_(pkg_dir, 'pkg.sub') - self.assertTrue(hasattr(loader, 'load_module')) + self.assertHasAttr(loader, 'load_module') # [sub package] def test_package_in_package(self): @@ -108,7 +108,7 @@ def test_package_in_package(self): with context as mapping: pkg_dir = os.path.dirname(mapping['pkg.__init__']) loader = self.import_(pkg_dir, 'pkg.sub') - self.assertTrue(hasattr(loader, 'load_module')) + self.assertHasAttr(loader, 'load_module') # [package over modules] def test_package_over_module(self): @@ -129,7 +129,7 @@ def test_empty_string_for_dir(self): file.write("# test file for importlib") try: loader = self._find(finder, 'mod', loader_only=True) - self.assertTrue(hasattr(loader, 'load_module')) + self.assertHasAttr(loader, 'load_module') finally: os.unlink('mod.py') diff --git a/Lib/test/test_importlib/source/test_path_hook.py b/Lib/test/test_importlib/source/test_path_hook.py index f274330e0b333b..6e1c23e6a9842b 100644 --- a/Lib/test/test_importlib/source/test_path_hook.py +++ b/Lib/test/test_importlib/source/test_path_hook.py @@ -15,12 +15,12 @@ def path_hook(self): def test_success(self): with util.create_modules('dummy') as mapping: - self.assertTrue(hasattr(self.path_hook()(mapping['.root']), - 'find_spec')) + self.assertHasAttr(self.path_hook()(mapping['.root']), + 'find_spec') def test_empty_string(self): # The empty string represents the cwd. - self.assertTrue(hasattr(self.path_hook()(''), 'find_spec')) + self.assertHasAttr(self.path_hook()(''), 'find_spec') (Frozen_PathHookTest, diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index 603125f6d926f6..b1ab52f966ffdb 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -43,14 +43,12 @@ def setUp(self): def test_subclasses(self): # Test that the expected subclasses inherit. for subclass in self.subclasses: - self.assertTrue(issubclass(subclass, self.__test), - "{0} is not a subclass of {1}".format(subclass, self.__test)) + self.assertIsSubclass(subclass, self.__test) def test_superclasses(self): # Test that the class inherits from the expected superclasses. for superclass in self.superclasses: - self.assertTrue(issubclass(self.__test, superclass), - "{0} is not a superclass of {1}".format(superclass, self.__test)) + self.assertIsSubclass(self.__test, superclass) class MetaPathFinder(InheritanceTests): @@ -226,7 +224,15 @@ class ResourceLoaderDefaultsTests(ABCTestHarness): SPLIT = make_abc_subclasses(ResourceLoader) def test_get_data(self): - with self.assertRaises(IOError): + with ( + self.assertRaises(IOError), + self.assertWarnsRegex( + DeprecationWarning, + r"importlib\.abc\.ResourceLoader is deprecated in favour of " + r"supporting resource loading through importlib\.resources" + r"\.abc\.TraversableResources.", + ), + ): self.ins.get_data('/some/path') @@ -416,14 +422,14 @@ def test_source_to_code_source(self): # Since compile() can handle strings, so should source_to_code(). source = 'attr = 42' module = self.source_to_module(source) - self.assertTrue(hasattr(module, 'attr')) + self.assertHasAttr(module, 'attr') self.assertEqual(module.attr, 42) def test_source_to_code_bytes(self): # Since compile() can handle bytes, so should source_to_code(). source = b'attr = 42' module = self.source_to_module(source) - self.assertTrue(hasattr(module, 'attr')) + self.assertHasAttr(module, 'attr') self.assertEqual(module.attr, 42) def test_source_to_code_path(self): @@ -757,7 +763,7 @@ def test_package_settings(self): warnings.simplefilter('ignore', DeprecationWarning) module = self.loader.load_module(self.name) self.verify_module(module) - self.assertFalse(hasattr(module, '__path__')) + self.assertNotHasAttr(module, '__path__') def test_get_source_encoding(self): # Source is considered encoded in UTF-8 by default unless otherwise @@ -913,5 +919,47 @@ def test_universal_newlines(self): SourceOnlyLoaderMock=SPLIT_SOL) +class SourceLoaderDeprecationWarningsTests(unittest.TestCase): + """Tests SourceLoader deprecation warnings.""" + + def test_deprecated_path_mtime(self): + from importlib.abc import SourceLoader + class DummySourceLoader(SourceLoader): + def get_data(self, path): + return b'' + + def get_filename(self, fullname): + return 'foo.py' + + def path_stats(self, path): + return {'mtime': 1} + with self.assertWarnsRegex( + DeprecationWarning, + r"importlib\.abc\.ResourceLoader is deprecated in favour of " + r"supporting resource loading through importlib\.resources" + r"\.abc\.TraversableResources.", + ): + loader = DummySourceLoader() + + with self.assertWarnsRegex( + DeprecationWarning, + r"SourceLoader\.path_mtime is deprecated in favour of " + r"SourceLoader\.path_stats\(\)\." + ): + loader.path_mtime('foo.py') + + +class ResourceLoaderDeprecationWarningsTests(unittest.TestCase): + """Tests ResourceLoader deprecation warnings.""" + + def test_deprecated_resource_loader(self): + from importlib.abc import ResourceLoader + class DummyLoader(ResourceLoader): + def get_data(self, path): + return b'' + + with self.assertWarns(DeprecationWarning): + DummyLoader() + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py index 2a35f3dcb7210c..1bc531a2fe34e7 100644 --- a/Lib/test/test_importlib/test_api.py +++ b/Lib/test/test_importlib/test_api.py @@ -6,8 +6,10 @@ import os.path import sys +from test import support from test.support import import_helper from test.support import os_helper +import traceback import types import unittest @@ -353,6 +355,20 @@ def test_module_missing_spec(self): with self.assertRaises(ModuleNotFoundError): self.init.reload(module) + def test_reload_traceback_with_non_str(self): + # gh-125519 + with support.captured_stdout() as stdout: + try: + self.init.reload("typing") + except TypeError as exc: + traceback.print_exception(exc, file=stdout) + else: + self.fail("Expected TypeError to be raised") + printed_traceback = stdout.getvalue() + self.assertIn("TypeError", printed_traceback) + self.assertNotIn("AttributeError", printed_traceback) + self.assertNotIn("module.__spec__.name", printed_traceback) + (Frozen_ReloadTests, Source_ReloadTests @@ -414,8 +430,7 @@ def test_everyone_has___loader__(self): for name, module in sys.modules.items(): if isinstance(module, types.ModuleType): with self.subTest(name=name): - self.assertTrue(hasattr(module, '__loader__'), - '{!r} lacks a __loader__ attribute'.format(name)) + self.assertHasAttr(module, '__loader__') if self.machinery.BuiltinImporter.find_spec(name): self.assertIsNot(module.__loader__, None) elif self.machinery.FrozenImporter.find_spec(name): @@ -425,7 +440,7 @@ def test_everyone_has___spec__(self): for name, module in sys.modules.items(): if isinstance(module, types.ModuleType): with self.subTest(name=name): - self.assertTrue(hasattr(module, '__spec__')) + self.assertHasAttr(module, '__spec__') if self.machinery.BuiltinImporter.find_spec(name): self.assertIsNot(module.__spec__, None) elif self.machinery.FrozenImporter.find_spec(name): @@ -437,5 +452,57 @@ def test_everyone_has___spec__(self): ) = test_util.test_both(StartupTests, machinery=machinery) +class TestModuleAll(unittest.TestCase): + def test_machinery(self): + extra = ( + # from importlib._bootstrap and importlib._bootstrap_external + 'AppleFrameworkLoader', + 'BYTECODE_SUFFIXES', + 'BuiltinImporter', + 'DEBUG_BYTECODE_SUFFIXES', + 'EXTENSION_SUFFIXES', + 'ExtensionFileLoader', + 'FileFinder', + 'FrozenImporter', + 'ModuleSpec', + 'NamespaceLoader', + 'OPTIMIZED_BYTECODE_SUFFIXES', + 'PathFinder', + 'SOURCE_SUFFIXES', + 'SourceFileLoader', + 'SourcelessFileLoader', + 'WindowsRegistryFinder', + ) + support.check__all__(self, machinery['Source'], extra=extra) + + def test_util(self): + extra = ( + # from importlib.abc, importlib._bootstrap + # and importlib._bootstrap_external + 'Loader', + 'MAGIC_NUMBER', + 'cache_from_source', + 'decode_source', + 'module_from_spec', + 'source_from_cache', + 'spec_from_file_location', + 'spec_from_loader', + ) + support.check__all__(self, util['Source'], extra=extra) + + +class TestDeprecations(unittest.TestCase): + def test_machinery_deprecated_attributes(self): + from importlib import machinery + attributes = ( + 'DEBUG_BYTECODE_SUFFIXES', + 'OPTIMIZED_BYTECODE_SUFFIXES', + ) + for attr in attributes: + with self.subTest(attr=attr): + with self.assertWarns(DeprecationWarning): + getattr(machinery, attr) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_lazy.py b/Lib/test/test_importlib/test_lazy.py index 38ab21907b58d9..e48fad8898f0ef 100644 --- a/Lib/test/test_importlib/test_lazy.py +++ b/Lib/test/test_importlib/test_lazy.py @@ -125,12 +125,12 @@ def test_delete_eventual_attr(self): # Deleting an attribute should stay deleted. module = self.new_module() del module.attr - self.assertFalse(hasattr(module, 'attr')) + self.assertNotHasAttr(module, 'attr') def test_delete_preexisting_attr(self): module = self.new_module() del module.__name__ - self.assertFalse(hasattr(module, '__name__')) + self.assertNotHasAttr(module, '__name__') def test_module_substitution_error(self): with test_util.uncache(TestingImporter.module_name): @@ -178,6 +178,52 @@ def access_module(): # Or multiple load attempts self.assertEqual(loader.load_count, 1) + def test_lazy_self_referential_modules(self): + # Directory modules with submodules that reference the parent can attempt to access + # the parent module during a load. Verify that this common pattern works with lazy loading. + # json is a good example in the stdlib. + json_modules = [name for name in sys.modules if name.startswith('json')] + with test_util.uncache(*json_modules): + # Standard lazy loading, unwrapped + spec = util.find_spec('json') + loader = util.LazyLoader(spec.loader) + spec.loader = loader + module = util.module_from_spec(spec) + sys.modules['json'] = module + loader.exec_module(module) + + # Trigger load with attribute lookup, ensure expected behavior + test_load = module.loads('{}') + self.assertEqual(test_load, {}) + + def test_lazy_module_type_override(self): + # Verify that lazy loading works with a module that modifies + # its __class__ to be a custom type. + + # Example module from PEP 726 + module = self.new_module(source_code="""\ +import sys +from types import ModuleType + +CONSTANT = 3.14 + +class ImmutableModule(ModuleType): + def __setattr__(self, name, value): + raise AttributeError('Read-only attribute!') + + def __delattr__(self, name): + raise AttributeError('Read-only attribute!') + +sys.modules[__name__].__class__ = ImmutableModule +""") + sys.modules[TestingImporter.module_name] = module + self.assertIsInstance(module, util._LazyModule) + self.assertEqual(module.CONSTANT, 3.14) + with self.assertRaises(AttributeError): + module.CONSTANT = 2.71 + with self.assertRaises(AttributeError): + del module.CONSTANT + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py index 072e198795d394..6ca0978f9bca69 100644 --- a/Lib/test/test_importlib/test_namespace_pkgs.py +++ b/Lib/test/test_importlib/test_namespace_pkgs.py @@ -80,7 +80,7 @@ def test_cant_import_other(self): def test_simple_repr(self): import foo.one - self.assertTrue(repr(foo).startswith(" None: ... + + +def generic_function_2[Eggs, **Spam](x: Eggs, y: Spam): pass + + +class D: + Foo = int + Bar = str + + def generic_method[Foo, **Bar]( + self, x: Foo, y: Bar + ) -> None: ... + + def generic_method_2[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + +# Eggs is `int` in globals, a TypeVar in type_params, and `str` in locals: +class E[Eggs]: + Eggs = str + x: Eggs + + + +def nested(): + from types import SimpleNamespace + from inspect import get_annotations + + Eggs = bytes + Spam = memoryview + + + class F[Eggs, **Spam]: + x: Eggs + y: Spam + + def generic_method[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + + def generic_function[Eggs, **Spam](x: Eggs, y: Spam): pass + + + # Eggs is `int` in globals, `bytes` in the function scope, + # a TypeVar in the type_params, and `str` in locals: + class G[Eggs]: + Eggs = str + x: Eggs + + + return SimpleNamespace( + F=F, + F_annotations=get_annotations(F, eval_str=True), + F_meth_annotations=get_annotations(F.generic_method, eval_str=True), + G_annotations=get_annotations(G, eval_str=True), + generic_func=generic_function, + generic_func_annotations=get_annotations(generic_function, eval_str=True) + ) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 52cf68b93b85fa..8e47df21cfef2e 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -1,9 +1,11 @@ +from annotationlib import Format, ForwardRef import asyncio import builtins import collections import copy import datetime import functools +import gc import importlib import inspect import io @@ -25,6 +27,7 @@ import unittest import unittest.mock import warnings +import weakref try: @@ -32,28 +35,27 @@ except ImportError: ThreadPoolExecutor = None -from test.support import cpython_only +from test.support import cpython_only, import_helper from test.support import MISSING_C_DOCSTRINGS, ALWAYS_EQ +from test.support import run_no_yield_async_fn from test.support.import_helper import DirsOnSysPath, ready_to_import -from test.support.os_helper import TESTFN +from test.support.os_helper import TESTFN, temp_cwd from test.support.script_helper import assert_python_ok, assert_python_failure, kill_python -from test.support import has_subprocess_support, SuppressCrashReport +from test.support import has_subprocess_support from test import support -from . import inspect_fodder as mod -from . import inspect_fodder2 as mod2 -from . import inspect_stock_annotations -from . import inspect_stringized_annotations -from . import inspect_stringized_annotations_2 +from test.test_inspect import inspect_fodder as mod +from test.test_inspect import inspect_fodder2 as mod2 +from test.test_inspect import inspect_stringized_annotations +from test.test_inspect import inspect_deferred_annotations # Functions tested in this suite: # ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode, -# isbuiltin, isroutine, isgenerator, isgeneratorfunction, getmembers, +# isbuiltin, isroutine, isgenerator, ispackage, isgeneratorfunction, getmembers, # getdoc, getfile, getmodule, getsourcefile, getcomments, getsource, -# getclasstree, getargvalues, formatargvalues, -# currentframe, stack, trace, isdatadescriptor, -# ismethodwrapper +# getclasstree, getargvalues, formatargvalues, currentframe, +# stack, trace, ismethoddescriptor, isdatadescriptor, ismethodwrapper # NOTE: There are some additional tests relating to interaction with # zipimport in the test_zipimport_support test module. @@ -72,11 +74,6 @@ def revise(filename, *args): git = mod.StupidGit() -def tearDownModule(): - if support.has_socket_support: - asyncio.set_event_loop_policy(None) - - def signatures_with_lexicographic_keyword_only_parameters(): """ Yields a whole bunch of functions with only keyword-only parameters, @@ -104,7 +101,7 @@ def unsorted_keyword_only_parameters_fn(*, throw, out, the, baby, with_, class IsTestBase(unittest.TestCase): predicates = set([inspect.isbuiltin, inspect.isclass, inspect.iscode, inspect.isframe, inspect.isfunction, inspect.ismethod, - inspect.ismodule, inspect.istraceback, + inspect.ismodule, inspect.istraceback, inspect.ispackage, inspect.isgenerator, inspect.isgeneratorfunction, inspect.iscoroutine, inspect.iscoroutinefunction, inspect.isasyncgen, inspect.isasyncgenfunction, @@ -120,10 +117,13 @@ def istest(self, predicate, exp): predicate == inspect.iscoroutinefunction) and \ other == inspect.isfunction: continue - self.assertFalse(other(obj), 'not %s(%s)' % (other.__name__, exp)) + if predicate == inspect.ispackage and other == inspect.ismodule: + self.assertTrue(predicate(obj), '%s(%s)' % (predicate.__name__, exp)) + else: + self.assertFalse(other(obj), 'not %s(%s)' % (other.__name__, exp)) def test__all__(self): - support.check__all__(self, inspect, not_exported=("modulesbyfile",)) + support.check__all__(self, inspect, not_exported=("modulesbyfile",), extra=("get_annotations",)) def generator_function_example(self): for i in range(2): @@ -175,6 +175,7 @@ def test_excluding_predicates(self): self.istest(inspect.ismethod, 'git.argue') self.istest(inspect.ismethod, 'mod.custom_method') self.istest(inspect.ismodule, 'mod') + self.istest(inspect.ismethoddescriptor, 'int.__add__') self.istest(inspect.isdatadescriptor, 'collections.defaultdict.default_factory') self.istest(inspect.isgenerator, '(x for x in range(2))') self.istest(inspect.isgeneratorfunction, 'generator_function_example') @@ -199,7 +200,17 @@ def test_excluding_predicates(self): self.assertFalse(inspect.ismethodwrapper(int)) self.assertFalse(inspect.ismethodwrapper(type("AnyClass", (), {}))) + def test_ispackage(self): + self.istest(inspect.ispackage, 'unittest') + self.istest(inspect.ispackage, 'importlib') + self.assertFalse(inspect.ispackage(inspect)) + self.assertFalse(inspect.ispackage(mod)) + self.assertFalse(inspect.ispackage(':)')) + + class FakePackage: + __path__ = None + self.assertFalse(inspect.ispackage(FakePackage())) def test_iscoroutine(self): async_gen_coro = async_generator_function_example(1) @@ -233,6 +244,7 @@ class PMClass: gen_coroutine_function_example)))) self.assertFalse(inspect.iscoroutinefunction(gen_coro_pmi)) self.assertFalse(inspect.iscoroutinefunction(gen_coro_pmc)) + self.assertFalse(inspect.iscoroutinefunction(inspect)) self.assertFalse(inspect.iscoroutine(gen_coro)) self.assertTrue( @@ -400,6 +412,8 @@ def test_isroutine(self): self.assertFalse(inspect.isroutine(type)) self.assertFalse(inspect.isroutine(int)) self.assertFalse(inspect.isroutine(type('some_class', (), {}))) + # partial + self.assertTrue(inspect.isroutine(functools.partial(mod.spam))) def test_isclass(self): self.istest(inspect.isclass, 'mod.StupidGit') @@ -668,7 +682,10 @@ def test_cleandoc(self): @cpython_only def test_c_cleandoc(self): - import _testinternalcapi + try: + import _testinternalcapi + except ImportError: + return unittest.skip("requires _testinternalcapi") func = _testinternalcapi.compiler_cleandoc for i, (input, expected) in enumerate(self.cleandoc_testdata): with self.subTest(i=i): @@ -730,6 +747,18 @@ def test_getsourcefile(self): finally: del linecache.cache[co.co_filename] + def test_getsource_empty_file(self): + with temp_cwd() as cwd: + with open('empty_file.py', 'w'): + pass + sys.path.insert(0, cwd) + try: + import empty_file + self.assertEqual(inspect.getsource(empty_file), '\n') + self.assertEqual(inspect.getsourcelines(empty_file), (['\n'], 0)) + finally: + sys.path.remove(cwd) + def test_getfile(self): self.assertEqual(inspect.getfile(mod.StupidGit), mod.__file__) @@ -800,7 +829,64 @@ def monkey(filename, module_globals=None): def test_getsource_on_code_object(self): self.assertSourceEqual(mod.eggs.__code__, 12, 18) + def test_getsource_on_generated_class(self): + A = type('A', (unittest.TestCase,), {}) + self.assertEqual(inspect.getsourcefile(A), __file__) + self.assertEqual(inspect.getfile(A), __file__) + self.assertIs(inspect.getmodule(A), sys.modules[__name__]) + self.assertRaises(OSError, inspect.getsource, A) + self.assertRaises(OSError, inspect.getsourcelines, A) + self.assertIsNone(inspect.getcomments(A)) + + def test_getsource_on_class_without_firstlineno(self): + __firstlineno__ = 1 + class C: + nonlocal __firstlineno__ + self.assertRaises(OSError, inspect.getsource, C) + +class TestGetsourceStdlib(unittest.TestCase): + # Test Python implementations of the stdlib modules + + def test_getsource_stdlib_collections_abc(self): + import collections.abc + lines, lineno = inspect.getsourcelines(collections.abc.Sequence) + self.assertEqual(lines[0], 'class Sequence(Reversible, Collection):\n') + src = inspect.getsource(collections.abc.Sequence) + self.assertEqual(src.splitlines(True), lines) + + def test_getsource_stdlib_tomllib(self): + import tomllib + self.assertRaises(OSError, inspect.getsource, tomllib.TOMLDecodeError) + self.assertRaises(OSError, inspect.getsourcelines, tomllib.TOMLDecodeError) + + def test_getsource_stdlib_abc(self): + # Pure Python implementation + abc = import_helper.import_fresh_module('abc', blocked=['_abc']) + with support.swap_item(sys.modules, 'abc', abc): + self.assertRaises(OSError, inspect.getsource, abc.ABCMeta) + self.assertRaises(OSError, inspect.getsourcelines, abc.ABCMeta) + # With C acceleration + import abc + try: + src = inspect.getsource(abc.ABCMeta) + lines, lineno = inspect.getsourcelines(abc.ABCMeta) + except OSError: + pass + else: + self.assertEqual(lines[0], ' class ABCMeta(type):\n') + self.assertEqual(src.splitlines(True), lines) + + def test_getsource_stdlib_decimal(self): + # Pure Python implementation + decimal = import_helper.import_fresh_module('decimal', blocked=['_decimal']) + with support.swap_item(sys.modules, 'decimal', decimal): + src = inspect.getsource(decimal.Decimal) + lines, lineno = inspect.getsourcelines(decimal.Decimal) + self.assertEqual(lines[0], 'class Decimal(object):\n') + self.assertEqual(src.splitlines(True), lines) + class TestGetsourceInteractive(unittest.TestCase): + @support.force_not_colorized def test_getclasses_interactive(self): # bpo-44648: simulate a REPL session; # there is no `__file__` in the __main__ module @@ -894,6 +980,29 @@ def test_anonymous(self): # as argument to another function. self.assertSourceEqual(mod2.anonymous, 55, 55) + def test_enum(self): + self.assertSourceEqual(mod2.enum322, 322, 323) + self.assertSourceEqual(mod2.enum326, 326, 327) + self.assertSourceEqual(mod2.flag330, 330, 331) + self.assertSourceEqual(mod2.flag334, 334, 335) + self.assertRaises(OSError, inspect.getsource, mod2.simple_enum338) + self.assertRaises(OSError, inspect.getsource, mod2.simple_enum339) + self.assertRaises(OSError, inspect.getsource, mod2.simple_flag340) + self.assertRaises(OSError, inspect.getsource, mod2.simple_flag341) + + def test_namedtuple(self): + self.assertSourceEqual(mod2.nt346, 346, 348) + self.assertRaises(OSError, inspect.getsource, mod2.nt351) + + def test_typeddict(self): + self.assertSourceEqual(mod2.td354, 354, 356) + self.assertRaises(OSError, inspect.getsource, mod2.td359) + + def test_dataclass(self): + self.assertSourceEqual(mod2.dc364, 364, 367) + self.assertRaises(OSError, inspect.getsource, mod2.dc370) + self.assertRaises(OSError, inspect.getsource, mod2.dc371) + class TestBlockComments(GetSourceBase): fodderModule = mod @@ -957,7 +1066,7 @@ def test_findsource_without_filename(self): self.assertRaises(IOError, inspect.findsource, co) self.assertRaises(IOError, inspect.getsource, co) - def test_findsource_with_out_of_bounds_lineno(self): + def test_findsource_on_func_with_out_of_bounds_lineno(self): mod_len = len(inspect.getsource(mod)) src = '\n' * 2* mod_len + "def f(): pass" co = compile(src, mod.__file__, "exec") @@ -965,12 +1074,26 @@ def test_findsource_with_out_of_bounds_lineno(self): eval(co, g, l) func = l['f'] self.assertEqual(func.__code__.co_firstlineno, 1+2*mod_len) - with self.assertRaisesRegex(IOError, "lineno is out of bounds"): + with self.assertRaisesRegex(OSError, "lineno is out of bounds"): inspect.findsource(func) + def test_findsource_on_class_with_out_of_bounds_lineno(self): + mod_len = len(inspect.getsource(mod)) + src = '\n' * 2* mod_len + "class A: pass" + co = compile(src, mod.__file__, "exec") + g, l = {'__name__': mod.__name__}, {} + eval(co, g, l) + cls = l['A'] + self.assertEqual(cls.__firstlineno__, 1+2*mod_len) + with self.assertRaisesRegex(OSError, "lineno is out of bounds"): + inspect.findsource(cls) + def test_getsource_on_method(self): self.assertSourceEqual(mod2.ClassWithMethod.method, 118, 119) + def test_getsource_on_class_code_object(self): + self.assertSourceEqual(mod2.ClassWithCodeObject.code, 315, 317) + def test_nested_func(self): self.assertSourceEqual(mod2.cls135.func136, 136, 139) @@ -991,7 +1114,11 @@ def test_class_decorator(self): self.assertSourceEqual(mod2.cls196, 194, 201) self.assertSourceEqual(mod2.cls196.cls200, 198, 201) + @support.requires_docstrings def test_class_inside_conditional(self): + # We skip this test when docstrings are not present, + # because docstrings are one of the main factors of + # finding the correct class in the source code. self.assertSourceEqual(mod2.cls238.cls239, 239, 240) def test_multiple_children_classes(self): @@ -1036,16 +1163,12 @@ def f(self): # This is necessary when the test is run multiple times. sys.modules.pop("inspect_actual") - @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "socket.accept is broken" - ) def test_nested_class_definition_inside_async_function(self): - import asyncio - self.addCleanup(asyncio.set_event_loop_policy, None) - self.assertSourceEqual(asyncio.run(mod2.func225()), 226, 227) + run = run_no_yield_async_fn + + self.assertSourceEqual(run(mod2.func225), 226, 227) self.assertSourceEqual(mod2.cls226, 231, 235) - self.assertSourceEqual(asyncio.run(mod2.cls226().func232()), 233, 234) + self.assertSourceEqual(run(mod2.cls226().func232), 233, 234) def test_class_definition_same_name_diff_methods(self): self.assertSourceEqual(mod2.cls296, 296, 298) @@ -1201,7 +1324,7 @@ def test_getfullargspec_builtin_methods(self): @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_getfullargspec_builtin_func(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") builtin = _testcapi.docstring_with_signature_with_defaults spec = inspect.getfullargspec(builtin) self.assertEqual(spec.defaults[0], 'avocado') @@ -1210,7 +1333,7 @@ def test_getfullargspec_builtin_func(self): @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_getfullargspec_builtin_func_no_signature(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") builtin = _testcapi.docstring_no_signature with self.assertRaises(TypeError): inspect.getfullargspec(builtin) @@ -1244,7 +1367,7 @@ def test_getfullargspec_builtin_func_no_signature(self): (dict.__class_getitem__, meth_type_o), ] try: - import _stat + import _stat # noqa: F401 except ImportError: # if the _stat extension is not available, stat.S_IMODE() is # implemented in Python, not in C @@ -1508,6 +1631,56 @@ def f(self): self.assertIn(('f', b.f), inspect.getmembers(b)) self.assertIn(('f', b.f), inspect.getmembers(b, inspect.ismethod)) + def test_getmembers_custom_dir(self): + class CorrectDir: + def __init__(self, attr): + self.attr = attr + def method(self): + return self.attr + 1 + def __dir__(self): + return ['attr', 'method'] + + cd = CorrectDir(5) + self.assertEqual(inspect.getmembers(cd), [ + ('attr', 5), + ('method', cd.method), + ]) + self.assertEqual(inspect.getmembers(cd, inspect.ismethod), [ + ('method', cd.method), + ]) + + def test_getmembers_custom_broken_dir(self): + # inspect.getmembers calls `dir()` on the passed object inside. + # if `__dir__` mentions some non-existent attribute, + # we still need to return others correctly. + class BrokenDir: + existing = 1 + def method(self): + return self.existing + 1 + def __dir__(self): + return ['method', 'missing', 'existing'] + + bd = BrokenDir() + self.assertEqual(inspect.getmembers(bd), [ + ('existing', 1), + ('method', bd.method), + ]) + self.assertEqual(inspect.getmembers(bd, inspect.ismethod), [ + ('method', bd.method), + ]) + + def test_getmembers_custom_duplicated_dir(self): + # Duplicates in `__dir__` must not fail and return just one result. + class DuplicatedDir: + attr = 1 + def __dir__(self): + return ['attr', 'attr'] + + dd = DuplicatedDir() + self.assertEqual(inspect.getmembers(dd), [ + ('attr', 1), + ]) + def test_getmembers_VirtualAttribute(self): class M(type): def __getattr__(cls, name): @@ -1552,105 +1725,6 @@ class C(metaclass=M): attrs = [a[0] for a in inspect.getmembers(C)] self.assertNotIn('missing', attrs) - def test_get_annotations_with_stock_annotations(self): - def foo(a:int, b:str): pass - self.assertEqual(inspect.get_annotations(foo), {'a': int, 'b': str}) - - foo.__annotations__ = {'a': 'foo', 'b':'str'} - self.assertEqual(inspect.get_annotations(foo), {'a': 'foo', 'b': 'str'}) - - self.assertEqual(inspect.get_annotations(foo, eval_str=True, locals=locals()), {'a': foo, 'b': str}) - self.assertEqual(inspect.get_annotations(foo, eval_str=True, globals=locals()), {'a': foo, 'b': str}) - - isa = inspect_stock_annotations - self.assertEqual(inspect.get_annotations(isa), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.MyClass), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.function), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function2), {'a': int, 'b': 'str', 'c': isa.MyClass, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function3), {'a': 'int', 'b': 'str', 'c': 'MyClass'}) - self.assertEqual(inspect.get_annotations(inspect), {}) # inspect module has no annotations - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function), {}) - - self.assertEqual(inspect.get_annotations(isa, eval_str=True), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.MyClass, eval_str=True), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.function, eval_str=True), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function2, eval_str=True), {'a': int, 'b': str, 'c': isa.MyClass, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function3, eval_str=True), {'a': int, 'b': str, 'c': isa.MyClass}) - self.assertEqual(inspect.get_annotations(inspect, eval_str=True), {}) - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass, eval_str=True), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function, eval_str=True), {}) - - self.assertEqual(inspect.get_annotations(isa, eval_str=False), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.MyClass, eval_str=False), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.function, eval_str=False), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function2, eval_str=False), {'a': int, 'b': 'str', 'c': isa.MyClass, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function3, eval_str=False), {'a': 'int', 'b': 'str', 'c': 'MyClass'}) - self.assertEqual(inspect.get_annotations(inspect, eval_str=False), {}) - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass, eval_str=False), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function, eval_str=False), {}) - - def times_three(fn): - @functools.wraps(fn) - def wrapper(a, b): - return fn(a*3, b*3) - return wrapper - - wrapped = times_three(isa.function) - self.assertEqual(wrapped(1, 'x'), isa.MyClass(3, 'xxx')) - self.assertIsNot(wrapped.__globals__, isa.function.__globals__) - self.assertEqual(inspect.get_annotations(wrapped), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(wrapped, eval_str=True), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(wrapped, eval_str=False), {'a': int, 'b': str, 'return': isa.MyClass}) - - def test_get_annotations_with_stringized_annotations(self): - isa = inspect_stringized_annotations - self.assertEqual(inspect.get_annotations(isa), {'a': 'int', 'b': 'str'}) - self.assertEqual(inspect.get_annotations(isa.MyClass), {'a': 'int', 'b': 'str'}) - self.assertEqual(inspect.get_annotations(isa.function), {'a': 'int', 'b': 'str', 'return': 'MyClass'}) - self.assertEqual(inspect.get_annotations(isa.function2), {'a': 'int', 'b': "'str'", 'c': 'MyClass', 'return': 'MyClass'}) - self.assertEqual(inspect.get_annotations(isa.function3), {'a': "'int'", 'b': "'str'", 'c': "'MyClass'"}) - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function), {}) - - self.assertEqual(inspect.get_annotations(isa, eval_str=True), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.MyClass, eval_str=True), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.function, eval_str=True), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function2, eval_str=True), {'a': int, 'b': 'str', 'c': isa.MyClass, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function3, eval_str=True), {'a': 'int', 'b': 'str', 'c': 'MyClass'}) - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass, eval_str=True), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function, eval_str=True), {}) - - self.assertEqual(inspect.get_annotations(isa, eval_str=False), {'a': 'int', 'b': 'str'}) - self.assertEqual(inspect.get_annotations(isa.MyClass, eval_str=False), {'a': 'int', 'b': 'str'}) - self.assertEqual(inspect.get_annotations(isa.function, eval_str=False), {'a': 'int', 'b': 'str', 'return': 'MyClass'}) - self.assertEqual(inspect.get_annotations(isa.function2, eval_str=False), {'a': 'int', 'b': "'str'", 'c': 'MyClass', 'return': 'MyClass'}) - self.assertEqual(inspect.get_annotations(isa.function3, eval_str=False), {'a': "'int'", 'b': "'str'", 'c': "'MyClass'"}) - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass, eval_str=False), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function, eval_str=False), {}) - - isa2 = inspect_stringized_annotations_2 - self.assertEqual(inspect.get_annotations(isa2), {}) - self.assertEqual(inspect.get_annotations(isa2, eval_str=True), {}) - self.assertEqual(inspect.get_annotations(isa2, eval_str=False), {}) - - def times_three(fn): - @functools.wraps(fn) - def wrapper(a, b): - return fn(a*3, b*3) - return wrapper - - wrapped = times_three(isa.function) - self.assertEqual(wrapped(1, 'x'), isa.MyClass(3, 'xxx')) - self.assertIsNot(wrapped.__globals__, isa.function.__globals__) - self.assertEqual(inspect.get_annotations(wrapped), {'a': 'int', 'b': 'str', 'return': 'MyClass'}) - self.assertEqual(inspect.get_annotations(wrapped, eval_str=True), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(wrapped, eval_str=False), {'a': 'int', 'b': 'str', 'return': 'MyClass'}) - - # test that local namespace lookups work - self.assertEqual(inspect.get_annotations(isa.MyClassWithLocalAnnotations), {'x': 'mytype'}) - self.assertEqual(inspect.get_annotations(isa.MyClassWithLocalAnnotations, eval_str=True), {'x': int}) - class TestFormatAnnotation(unittest.TestCase): def test_typing_replacement(self): @@ -1659,6 +1733,122 @@ def test_typing_replacement(self): self.assertEqual(inspect.formatannotation(ann1), 'Union[List[testModule.typing.A], int]') +class TestIsMethodDescriptor(unittest.TestCase): + + def test_custom_descriptors(self): + class MethodDescriptor: + def __get__(self, *_): pass + class MethodDescriptorSub(MethodDescriptor): + pass + class DataDescriptorWithNoGet: + def __set__(self, *_): pass + class DataDescriptorWithGetSet: + def __get__(self, *_): pass + def __set__(self, *_): pass + class DataDescriptorWithGetDelete: + def __get__(self, *_): pass + def __delete__(self, *_): pass + class DataDescriptorSub(DataDescriptorWithNoGet, + DataDescriptorWithGetDelete): + pass + + # Custom method descriptors: + self.assertTrue( + inspect.ismethoddescriptor(MethodDescriptor()), + '__get__ and no __set__/__delete__ => method descriptor') + self.assertTrue( + inspect.ismethoddescriptor(MethodDescriptorSub()), + '__get__ (inherited) and no __set__/__delete__' + ' => method descriptor') + + # Custom data descriptors: + self.assertFalse( + inspect.ismethoddescriptor(DataDescriptorWithNoGet()), + '__set__ (and no __get__) => not a method descriptor') + self.assertFalse( + inspect.ismethoddescriptor(DataDescriptorWithGetSet()), + '__get__ and __set__ => not a method descriptor') + self.assertFalse( + inspect.ismethoddescriptor(DataDescriptorWithGetDelete()), + '__get__ and __delete__ => not a method descriptor') + self.assertFalse( + inspect.ismethoddescriptor(DataDescriptorSub()), + '__get__, __set__ and __delete__ => not a method descriptor') + + # Classes of descriptors (are *not* descriptors themselves): + self.assertFalse(inspect.ismethoddescriptor(MethodDescriptor)) + self.assertFalse(inspect.ismethoddescriptor(MethodDescriptorSub)) + self.assertFalse(inspect.ismethoddescriptor(DataDescriptorSub)) + + def test_builtin_descriptors(self): + builtin_slot_wrapper = int.__add__ # This one is mentioned in docs. + class Owner: + def instance_method(self): pass + @classmethod + def class_method(cls): pass + @staticmethod + def static_method(): pass + @property + def a_property(self): pass + class Slotermeyer: + __slots__ = 'a_slot', + def function(): + pass + a_lambda = lambda: None + + # Example builtin method descriptors: + self.assertTrue( + inspect.ismethoddescriptor(builtin_slot_wrapper), + 'a builtin slot wrapper is a method descriptor') + self.assertTrue( + inspect.ismethoddescriptor(Owner.__dict__['class_method']), + 'a classmethod object is a method descriptor') + self.assertTrue( + inspect.ismethoddescriptor(Owner.__dict__['static_method']), + 'a staticmethod object is a method descriptor') + + # Example builtin data descriptors: + self.assertFalse( + inspect.ismethoddescriptor(Owner.__dict__['a_property']), + 'a property is not a method descriptor') + self.assertFalse( + inspect.ismethoddescriptor(Slotermeyer.__dict__['a_slot']), + 'a slot is not a method descriptor') + + # `types.MethodType`/`types.FunctionType` instances (they *are* + # method descriptors, but `ismethoddescriptor()` explicitly + # excludes them): + self.assertFalse(inspect.ismethoddescriptor(Owner().instance_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner().class_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner().static_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner.instance_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner.class_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner.static_method)) + self.assertFalse(inspect.ismethoddescriptor(function)) + self.assertFalse(inspect.ismethoddescriptor(a_lambda)) + self.assertTrue(inspect.ismethoddescriptor(functools.partial(function))) + + def test_descriptor_being_a_class(self): + class MethodDescriptorMeta(type): + def __get__(self, *_): pass + class ClassBeingMethodDescriptor(metaclass=MethodDescriptorMeta): + pass + # `ClassBeingMethodDescriptor` itself *is* a method descriptor, + # but it is *also* a class, and `ismethoddescriptor()` explicitly + # excludes classes. + self.assertFalse( + inspect.ismethoddescriptor(ClassBeingMethodDescriptor), + 'classes (instances of type) are explicitly excluded') + + def test_non_descriptors(self): + class Test: + pass + self.assertFalse(inspect.ismethoddescriptor(Test())) + self.assertFalse(inspect.ismethoddescriptor(Test)) + self.assertFalse(inspect.ismethoddescriptor([42])) + self.assertFalse(inspect.ismethoddescriptor(42)) + + class TestIsDataDescriptor(unittest.TestCase): def test_custom_descriptors(self): @@ -1763,6 +1953,19 @@ def g(local_ref): builtin_vars, unbound_names) self.assertEqual(inspect.getclosurevars(C().f(_arg)), expected) + def test_attribute_same_name_as_global_var(self): + class C: + _global_ref = object() + def f(): + print(C._global_ref, _global_ref) + nonlocal_vars = {"C": C} + global_vars = {"_global_ref": _global_ref} + builtin_vars = {"print": print} + unbound_names = {"_global_ref"} + expected = inspect.ClosureVars(nonlocal_vars, global_vars, + builtin_vars, unbound_names) + self.assertEqual(inspect.getclosurevars(f), expected) + def test_nonlocal_vars(self): # More complex tests of nonlocal resolution def _nonlocal_vars(f): @@ -2280,6 +2483,13 @@ def __dict__(self): self.assertEqual(inspect.getattr_static(foo, 'a'), 3) self.assertFalse(test.called) + class Bar(Foo): pass + + bar = Bar() + bar.a = 5 + self.assertEqual(inspect.getattr_static(bar, 'a'), 3) + self.assertFalse(test.called) + def test_mutated_mro(self): test = self test.called = False @@ -2384,6 +2594,21 @@ def __getattribute__(self, attr): self.assertFalse(test.called) + def test_cache_does_not_cause_classes_to_persist(self): + # regression test for gh-118013: + # check that the internal _shadowed_dict cache does not cause + # dynamically created classes to have extended lifetimes even + # when no other strong references to those classes remain. + # Since these classes can themselves hold strong references to + # other objects, this can cause unexpected memory consumption. + class Foo: pass + Foo.instance = Foo() + weakref_to_class = weakref.ref(Foo) + inspect.getattr_static(Foo.instance, 'whatever', 'irrelevant') + del Foo + gc.collect() + self.assertIsNone(weakref_to_class()) + class TestGetGeneratorState(unittest.TestCase): @@ -2568,6 +2793,10 @@ async def number_asyncgen(): async def asyncTearDown(self): await self.asyncgen.aclose() + @classmethod + def tearDownClass(cls): + asyncio._set_event_loop_policy(None) + def _asyncgenstate(self): return inspect.getasyncgenstate(self.asyncgen) @@ -2760,6 +2989,17 @@ def test2(pod=42, /): with self.assertRaisesRegex(ValueError, 'follows default argument'): S((pkd, pk)) + second_args = args.replace(name="second_args") + with self.assertRaisesRegex(ValueError, 'more than one variadic positional parameter'): + S((args, second_args)) + + with self.assertRaisesRegex(ValueError, 'more than one variadic positional parameter'): + S((args, ko, second_args)) + + second_kwargs = kwargs.replace(name="second_kwargs") + with self.assertRaisesRegex(ValueError, 'more than one variadic keyword parameter'): + S((kwargs, second_kwargs)) + def test_signature_object_pickle(self): def foo(a, b, *, c:1={}, **kw) -> {42:'ham'}: pass foo_partial = functools.partial(foo, a=1) @@ -2871,7 +3111,7 @@ def test_staticmethod(*args): # NOQA @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_signature_on_builtins(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") def test_unbound_method(o): """Use this to test unbound methods (things that should have a self)""" @@ -2952,7 +3192,7 @@ class ThisWorksNow: @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_signature_on_decorated_builtins(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") func = _testcapi.docstring_with_signature_with_defaults def decorator(func): @@ -2973,7 +3213,7 @@ def wrapper_like(*args, **kwargs) -> int: pass @cpython_only def test_signature_on_builtins_no_signature(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") with self.assertRaisesRegex(ValueError, 'no signature found for builtin'): inspect.signature(_testcapi.docstring_no_signature) @@ -3011,7 +3251,7 @@ def test_signature_on_builtins_no_signature(self): (dict.__class_getitem__, meth_o), ] try: - import _stat + import _stat # noqa: F401 except ImportError: # if the _stat extension is not available, stat.S_IMODE() is # implemented in Python, not in C @@ -3023,6 +3263,13 @@ def test_signature_on_builtins_no_signature(self): self.assertEqual(inspect.signature(builtin), inspect.signature(template)) + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_signature_parsing_with_defaults(self): + _testcapi = import_helper.import_module("_testcapi") + meth = _testcapi.DocStringUnrepresentableSignatureTest.with_default + self.assertEqual(str(inspect.signature(meth)), '(self, /, x=1)') + def test_signature_on_non_function(self): with self.assertRaisesRegex(TypeError, 'is not a callable object'): inspect.signature(42) @@ -3184,7 +3431,7 @@ def foo(cls, *, arg): ...)) def test_signature_on_partial(self): - from functools import partial + from functools import partial, Placeholder def test(): pass @@ -3239,6 +3486,25 @@ def test(a, b, *, c, d): ('d', ..., ..., "keyword_only")), ...)) + # With Placeholder + self.assertEqual(self.signature(partial(test, Placeholder, 1)), + ((('a', ..., ..., "positional_only"), + ('c', ..., ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + self.assertEqual(self.signature(partial(test, Placeholder, 1, c=2)), + ((('a', ..., ..., "positional_only"), + ('c', 2, ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + # Ensure unittest.mock.ANY & similar do not get picked up as a Placeholder + self.assertEqual(self.signature(partial(test, unittest.mock.ANY, 1, c=2)), + ((('c', 2, ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + def test(a, *args, b, **kwargs): pass @@ -3286,6 +3552,15 @@ def test(a, *args, b, **kwargs): ('kwargs', ..., ..., "var_keyword")), ...)) + # With Placeholder + p = partial(test, Placeholder, Placeholder, 1, b=0, test=1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('args', ..., ..., "var_positional"), + ('b', 0, ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + def test(a, b, c:int) -> 42: pass @@ -3390,6 +3665,34 @@ def foo(a, b, /, c, d, **kwargs): ('kwargs', ..., ..., 'var_keyword')), ...)) + # Positional only With Placeholder + p = partial(foo, Placeholder, 1, c=0, d=1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('c', 0, ..., "keyword_only"), + ('d', 1, ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + # Optionals Positional With Placeholder + def foo(a=0, b=1, /, c=2, d=3): + pass + + # Positional + p = partial(foo, Placeholder, 1, c=0, d=1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('c', 0, ..., "keyword_only"), + ('d', 1, ..., "keyword_only")), + ...)) + + # Positional or Keyword - transformed to positional + p = partial(foo, Placeholder, 1, Placeholder, 1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('c', ..., ..., "positional_only")), + ...)) + def test_signature_on_partialmethod(self): from functools import partialmethod @@ -3402,18 +3705,32 @@ def test(): inspect.signature(Spam.ham) class Spam: - def test(it, a, *, c) -> 'spam': + def test(it, a, b, *, c) -> 'spam': pass ham = partialmethod(test, c=1) + bar = partialmethod(test, functools.Placeholder, 1, c=1) self.assertEqual(self.signature(Spam.ham, eval_str=False), ((('it', ..., ..., 'positional_or_keyword'), ('a', ..., ..., 'positional_or_keyword'), + ('b', ..., ..., 'positional_or_keyword'), ('c', 1, ..., 'keyword_only')), 'spam')) self.assertEqual(self.signature(Spam().ham, eval_str=False), ((('a', ..., ..., 'positional_or_keyword'), + ('b', ..., ..., 'positional_or_keyword'), + ('c', 1, ..., 'keyword_only')), + 'spam')) + + # With Placeholder + self.assertEqual(self.signature(Spam.bar, eval_str=False), + ((('it', ..., ..., 'positional_only'), + ('a', ..., ..., 'positional_only'), + ('c', 1, ..., 'keyword_only')), + 'spam')) + self.assertEqual(self.signature(Spam().bar, eval_str=False), + ((('a', ..., ..., 'positional_only'), ('c', 1, ..., 'keyword_only')), 'spam')) @@ -3569,14 +3886,14 @@ def __init__(self, b): with self.subTest('partial'): class CM(type): - __call__ = functools.partial(lambda x, a: (x, a), 2) + __call__ = functools.partial(lambda x, a, b: (x, a, b), 2) class C(metaclass=CM): - def __init__(self, b): + def __init__(self, c): pass - self.assertEqual(C(1), (2, 1)) + self.assertEqual(C(1), (2, C, 1)) self.assertEqual(self.signature(C), - ((('a', ..., ..., "positional_or_keyword"),), + ((('b', ..., ..., "positional_or_keyword"),), ...)) with self.subTest('partialmethod'): @@ -3723,11 +4040,11 @@ class C: with self.subTest('partial'): class C: - __init__ = functools.partial(lambda x, a: None, 2) + __init__ = functools.partial(lambda x, a, b: None, 2) C(1) # does not raise self.assertEqual(self.signature(C), - ((('a', ..., ..., "positional_or_keyword"),), + ((('b', ..., ..., "positional_or_keyword"),), ...)) with self.subTest('partialmethod'): @@ -3981,11 +4298,12 @@ class C: with self.subTest('partial'): class C: - __call__ = functools.partial(lambda x, a: (x, a), 2) + __call__ = functools.partial(lambda x, a, b: (x, a, b), 2) - self.assertEqual(C()(1), (2, 1)) + c = C() + self.assertEqual(c(1), (2, c, 1)) self.assertEqual(self.signature(C()), - ((('a', ..., ..., "positional_or_keyword"),), + ((('b', ..., ..., "positional_or_keyword"),), ...)) with self.subTest('partialmethod'): @@ -4044,6 +4362,28 @@ class C: ((('a', ..., ..., "positional_or_keyword"),), ...)) + def test_signature_on_callable_objects_with_text_signature_attr(self): + class C: + __text_signature__ = '(a, /, b, c=True)' + def __call__(self, *args, **kwargs): + pass + + self.assertEqual(self.signature(C), ((), ...)) + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_only"), + ('b', ..., ..., "positional_or_keyword"), + ('c', True, ..., "positional_or_keyword"), + ), + ...)) + + c = C() + c.__text_signature__ = '(x, y)' + self.assertEqual(self.signature(c), + ((('x', ..., ..., "positional_or_keyword"), + ('y', ..., ..., "positional_or_keyword"), + ), + ...)) + def test_signature_on_wrapper(self): class Wrapper: def __call__(self, b): @@ -4315,6 +4655,18 @@ def func( expected_multiline, ) + def test_signature_format_unquote(self): + def func(x: 'int') -> 'str': ... + + self.assertEqual( + inspect.signature(func).format(), + "(x: 'int') -> 'str'" + ) + self.assertEqual( + inspect.signature(func).format(quote_annotation_strings=False), + "(x: int) -> str" + ) + def test_signature_replace_parameters(self): def test(a, b) -> 42: pass @@ -4547,6 +4899,26 @@ def test_signature_eval_str(self): par('b', PORK, annotation=tuple), ))) + def test_signature_annotation_format(self): + ida = inspect_deferred_annotations + sig = inspect.Signature + par = inspect.Parameter + PORK = inspect.Parameter.POSITIONAL_OR_KEYWORD + for signature_func in (inspect.signature, inspect.Signature.from_callable): + with self.subTest(signature_func=signature_func): + self.assertEqual( + signature_func(ida.f, annotation_format=Format.STRING), + sig([par("x", PORK, annotation="undefined")]) + ) + self.assertEqual( + signature_func(ida.f, annotation_format=Format.FORWARDREF), + sig([par("x", PORK, annotation=ForwardRef("undefined"))]) + ) + with self.assertRaisesRegex(NameError, "undefined"): + signature_func(ida.f, annotation_format=Format.VALUE) + with self.assertRaisesRegex(NameError, "undefined"): + signature_func(ida.f) + def test_signature_none_annotation(self): class funclike: # Has to be callable, and have correct @@ -4572,38 +4944,6 @@ def foo(): pass self.assertEqual(signature_func(foo), inspect.Signature()) self.assertEqual(inspect.get_annotations(foo), {}) - def test_signature_as_str(self): - self.maxDiff = None - class S: - __signature__ = '(a, b=2)' - - self.assertEqual(self.signature(S), - ((('a', ..., ..., 'positional_or_keyword'), - ('b', 2, ..., 'positional_or_keyword')), - ...)) - - def test_signature_as_callable(self): - # __signature__ should be either a staticmethod or a bound classmethod - class S: - @classmethod - def __signature__(cls): - return '(a, b=2)' - - self.assertEqual(self.signature(S), - ((('a', ..., ..., 'positional_or_keyword'), - ('b', 2, ..., 'positional_or_keyword')), - ...)) - - class S: - @staticmethod - def __signature__(): - return '(a, b=2)' - - self.assertEqual(self.signature(S), - ((('a', ..., ..., 'positional_or_keyword'), - ('b', 2, ..., 'positional_or_keyword')), - ...)) - def test_signature_on_derived_classes(self): # gh-105080: Make sure that signatures are consistent on derived classes @@ -4622,6 +4962,16 @@ class D2(D1): self.assertEqual(inspect.signature(D2), inspect.signature(D1)) + def test_signature_on_non_comparable(self): + class NoncomparableCallable: + def __call__(self, a): + pass + def __eq__(self, other): + 1/0 + self.assertEqual(self.signature(NoncomparableCallable()), + ((('a', ..., ..., 'positional_or_keyword'),), + ...)) + class TestParameterObject(unittest.TestCase): def test_signature_parameter_kinds(self): @@ -4987,15 +5337,30 @@ def test(a_po, b_po, c_po=3, /, foo=42, *, bar=50, **kwargs): self.assertEqual(self.call(test, 1, 2, foo=4, bar=5), (1, 2, 3, 4, 5, {})) - with self.assertRaisesRegex(TypeError, "but was passed as a keyword"): - self.call(test, 1, 2, foo=4, bar=5, c_po=10) + self.assertEqual(self.call(test, 1, 2, foo=4, bar=5, c_po=10), + (1, 2, 3, 4, 5, {'c_po': 10})) + + self.assertEqual(self.call(test, 1, 2, 30, c_po=31, foo=4, bar=5), + (1, 2, 30, 4, 5, {'c_po': 31})) + + self.assertEqual(self.call(test, 1, 2, 30, foo=4, bar=5, c_po=31), + (1, 2, 30, 4, 5, {'c_po': 31})) - with self.assertRaisesRegex(TypeError, "parameter is positional only"): - self.call(test, 1, 2, c_po=4) + self.assertEqual(self.call(test, 1, 2, c_po=4), + (1, 2, 3, 42, 50, {'c_po': 4})) - with self.assertRaisesRegex(TypeError, "parameter is positional only"): + with self.assertRaisesRegex(TypeError, "missing 2 required positional arguments"): self.call(test, a_po=1, b_po=2) + def without_var_kwargs(c_po=3, d_po=4, /): + return c_po, d_po + + with self.assertRaisesRegex( + TypeError, + "positional-only arguments passed as keyword arguments: 'c_po, d_po'", + ): + self.call(without_var_kwargs, c_po=33, d_po=44) + def test_signature_bind_with_self_arg(self): # Issue #17071: one of the parameters is named "self def test(a, self, b): @@ -5029,7 +5394,7 @@ def test_signature_bind_implicit_arg(self): # Issue #19611: getcallargs should work with comprehensions def make_set(): return set(z * z for z in range(5)) - gencomp_code = make_set.__code__.co_consts[1] + gencomp_code = make_set.__code__.co_consts[0] gencomp_func = types.FunctionType(gencomp_code, {}) iterator = iter(range(5)) @@ -5199,10 +5564,17 @@ class TestSignatureDefinitions(unittest.TestCase): # This test case provides a home for checking that particular APIs # have signatures available for introspection + @staticmethod + def is_public(name): + return not name.startswith('_') or name.startswith('__') and name.endswith('__') + @cpython_only @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") - def test_builtins_have_signatures(self): + def _test_module_has_signatures(self, module, + no_signature=(), unsupported_signature=(), + methods_no_signature={}, methods_unsupported_signature={}, + good_exceptions=()): # This checks all builtin callables in CPython have signatures # A few have signatures Signature can't yet handle, so we skip those # since they will have to wait until PEP 457 adds the required @@ -5211,48 +5583,273 @@ def test_builtins_have_signatures(self): # reasons, so we also skip those for the time being, but design # the test to fail in order to indicate when it needs to be # updated. - no_signature = set() - # These need PEP 457 groups - needs_groups = {"range", "slice", "dir", "getattr", - "next", "iter", "vars"} - no_signature |= needs_groups - # These have unrepresentable parameter default values of NULL - needs_null = {"anext"} - no_signature |= needs_null - # These need *args support in Argument Clinic - needs_varargs = {"min", "max", "__build_class__"} - no_signature |= needs_varargs - # These builtin types are expected to provide introspection info - types_with_signatures = { - 'bool', 'classmethod', 'complex', 'enumerate', 'filter', 'float', - 'frozenset', 'list', 'map', 'memoryview', 'object', 'property', - 'reversed', 'set', 'staticmethod', 'tuple', 'zip' - } + no_signature = no_signature or set() # Check the signatures we expect to be there - ns = vars(builtins) + ns = vars(module) + try: + names = set(module.__all__) + except AttributeError: + names = set(name for name in ns if self.is_public(name)) for name, obj in sorted(ns.items()): + if name not in names: + continue if not callable(obj): continue - # The builtin types haven't been converted to AC yet - if isinstance(obj, type) and (name not in types_with_signatures): - # Note that this also skips all the exception types + if (isinstance(obj, type) and + issubclass(obj, BaseException) and + name not in good_exceptions): no_signature.add(name) - if (name in no_signature): - # Not yet converted - continue - if name in {'classmethod', 'staticmethod'}: - # Bug gh-112006: inspect.unwrap() does not work with types - # with the __wrapped__ data descriptor. - continue - with self.subTest(builtin=name): - self.assertIsNotNone(inspect.signature(obj)) + if name not in no_signature and name not in unsupported_signature: + with self.subTest('supported', builtin=name): + self.assertIsNotNone(inspect.signature(obj)) + if isinstance(obj, type): + with self.subTest(type=name): + self._test_builtin_methods_have_signatures(obj, + methods_no_signature.get(name, ()), + methods_unsupported_signature.get(name, ())) # Check callables that haven't been converted don't claim a signature # This ensures this test will start failing as more signatures are # added, so the affected items can be moved into the scope of the # regression test above - for name in no_signature - needs_null: - with self.subTest(builtin=name): - self.assertIsNone(ns[name].__text_signature__) + for name in no_signature: + with self.subTest('none', builtin=name): + obj = ns[name] + self.assertIsNone(obj.__text_signature__) + self.assertRaises(ValueError, inspect.signature, obj) + for name in unsupported_signature: + with self.subTest('unsupported', builtin=name): + obj = ns[name] + self.assertIsNotNone(obj.__text_signature__) + self.assertRaises(ValueError, inspect.signature, obj) + + def _test_builtin_methods_have_signatures(self, cls, no_signature, unsupported_signature): + ns = vars(cls) + for name in ns: + obj = getattr(cls, name, None) + if not callable(obj) or isinstance(obj, type): + continue + if name not in no_signature and name not in unsupported_signature: + with self.subTest('supported', method=name): + self.assertIsNotNone(inspect.signature(obj)) + for name in no_signature: + with self.subTest('none', method=name): + self.assertIsNone(getattr(cls, name).__text_signature__) + self.assertRaises(ValueError, inspect.signature, getattr(cls, name)) + for name in unsupported_signature: + with self.subTest('unsupported', method=name): + self.assertIsNotNone(getattr(cls, name).__text_signature__) + self.assertRaises(ValueError, inspect.signature, getattr(cls, name)) + + def test_builtins_have_signatures(self): + no_signature = {'type', 'super', 'bytearray', 'bytes', 'dict', 'int', 'str'} + # These need PEP 457 groups + needs_groups = {"range", "slice", "dir", "getattr", + "next", "iter", "vars"} + no_signature |= needs_groups + # These have unrepresentable parameter default values of NULL + unsupported_signature = {"anext"} + # These need *args support in Argument Clinic + needs_varargs = {"min", "max", "__build_class__"} + no_signature |= needs_varargs + + methods_no_signature = { + 'dict': {'update'}, + 'object': {'__class__'}, + } + methods_unsupported_signature = { + 'bytearray': {'count', 'endswith', 'find', 'hex', 'index', 'rfind', 'rindex', 'startswith'}, + 'bytes': {'count', 'endswith', 'find', 'hex', 'index', 'rfind', 'rindex', 'startswith'}, + 'dict': {'pop'}, + 'memoryview': {'cast', 'hex'}, + 'str': {'count', 'endswith', 'find', 'index', 'maketrans', 'rfind', 'rindex', 'startswith'}, + } + self._test_module_has_signatures(builtins, + no_signature, unsupported_signature, + methods_no_signature, methods_unsupported_signature) + + def test_types_module_has_signatures(self): + unsupported_signature = {'CellType'} + methods_no_signature = { + 'AsyncGeneratorType': {'athrow'}, + 'CoroutineType': {'throw'}, + 'GeneratorType': {'throw'}, + } + self._test_module_has_signatures(types, + unsupported_signature=unsupported_signature, + methods_no_signature=methods_no_signature) + + def test_sys_module_has_signatures(self): + no_signature = {'getsizeof', 'set_asyncgen_hooks'} + no_signature |= {name for name in ['getobjects'] + if hasattr(sys, name)} + self._test_module_has_signatures(sys, no_signature) + + def test_abc_module_has_signatures(self): + import abc + self._test_module_has_signatures(abc) + + def test_atexit_module_has_signatures(self): + import atexit + self._test_module_has_signatures(atexit) + + def test_codecs_module_has_signatures(self): + import codecs + methods_no_signature = {'StreamReader': {'charbuffertype'}} + self._test_module_has_signatures(codecs, + methods_no_signature=methods_no_signature) + + def test_collections_module_has_signatures(self): + no_signature = {'OrderedDict', 'defaultdict'} + unsupported_signature = {'deque'} + methods_no_signature = { + 'OrderedDict': {'update'}, + } + methods_unsupported_signature = { + 'deque': {'index'}, + 'OrderedDict': {'pop'}, + 'UserString': {'maketrans'}, + } + self._test_module_has_signatures(collections, + no_signature, unsupported_signature, + methods_no_signature, methods_unsupported_signature) + + def test_collections_abc_module_has_signatures(self): + import collections.abc + self._test_module_has_signatures(collections.abc) + + def test_errno_module_has_signatures(self): + import errno + self._test_module_has_signatures(errno) + + def test_faulthandler_module_has_signatures(self): + import faulthandler + unsupported_signature = {'dump_traceback', 'dump_traceback_later', 'enable'} + unsupported_signature |= {name for name in ['register'] + if hasattr(faulthandler, name)} + self._test_module_has_signatures(faulthandler, unsupported_signature=unsupported_signature) + + def test_functools_module_has_signatures(self): + unsupported_signature = {"reduce"} + self._test_module_has_signatures(functools, unsupported_signature=unsupported_signature) + + def test_gc_module_has_signatures(self): + import gc + no_signature = {'set_threshold'} + self._test_module_has_signatures(gc, no_signature) + + def test_io_module_has_signatures(self): + methods_no_signature = { + 'BufferedRWPair': {'read', 'peek', 'read1', 'readinto', 'readinto1', 'write'}, + } + self._test_module_has_signatures(io, + methods_no_signature=methods_no_signature) + + def test_itertools_module_has_signatures(self): + import itertools + no_signature = {'islice', 'repeat'} + self._test_module_has_signatures(itertools, no_signature) + + def test_locale_module_has_signatures(self): + import locale + self._test_module_has_signatures(locale) + + def test_marshal_module_has_signatures(self): + import marshal + self._test_module_has_signatures(marshal) + + def test_operator_module_has_signatures(self): + import operator + self._test_module_has_signatures(operator) + + def test_os_module_has_signatures(self): + unsupported_signature = {'chmod', 'utime'} + unsupported_signature |= {name for name in + ['get_terminal_size', 'posix_spawn', 'posix_spawnp', + 'register_at_fork', 'startfile'] + if hasattr(os, name)} + self._test_module_has_signatures(os, unsupported_signature=unsupported_signature) + + def test_pwd_module_has_signatures(self): + pwd = import_helper.import_module('pwd') + self._test_module_has_signatures(pwd) + + def test_re_module_has_signatures(self): + import re + methods_no_signature = {'Match': {'group'}} + self._test_module_has_signatures(re, + methods_no_signature=methods_no_signature, + good_exceptions={'error', 'PatternError'}) + + def test_signal_module_has_signatures(self): + import signal + self._test_module_has_signatures(signal) + + def test_stat_module_has_signatures(self): + import stat + self._test_module_has_signatures(stat) + + def test_string_module_has_signatures(self): + import string + self._test_module_has_signatures(string) + + def test_symtable_module_has_signatures(self): + import symtable + self._test_module_has_signatures(symtable) + + def test_sysconfig_module_has_signatures(self): + import sysconfig + self._test_module_has_signatures(sysconfig) + + def test_threading_module_has_signatures(self): + import threading + self._test_module_has_signatures(threading) + + def test_thread_module_has_signatures(self): + import _thread + no_signature = {'RLock'} + self._test_module_has_signatures(_thread, no_signature) + + def test_time_module_has_signatures(self): + no_signature = { + 'asctime', 'ctime', 'get_clock_info', 'gmtime', 'localtime', + 'strftime', 'strptime' + } + no_signature |= {name for name in + ['clock_getres', 'clock_settime', 'clock_settime_ns', + 'pthread_getcpuclockid'] + if hasattr(time, name)} + self._test_module_has_signatures(time, no_signature) + + def test_tokenize_module_has_signatures(self): + import tokenize + self._test_module_has_signatures(tokenize) + + def test_tracemalloc_module_has_signatures(self): + import tracemalloc + self._test_module_has_signatures(tracemalloc) + + def test_typing_module_has_signatures(self): + import typing + no_signature = {'ParamSpec', 'ParamSpecArgs', 'ParamSpecKwargs', + 'Text', 'TypeAliasType', 'TypeVar', 'TypeVarTuple'} + methods_no_signature = { + 'Generic': {'__class_getitem__', '__init_subclass__'}, + } + methods_unsupported_signature = { + 'Text': {'count', 'find', 'index', 'rfind', 'rindex', 'startswith', 'endswith', 'maketrans'}, + } + self._test_module_has_signatures(typing, no_signature, + methods_no_signature=methods_no_signature, + methods_unsupported_signature=methods_unsupported_signature) + + def test_warnings_module_has_signatures(self): + unsupported_signature = {'warn', 'warn_explicit'} + self._test_module_has_signatures(warnings, unsupported_signature=unsupported_signature) + + def test_weakref_module_has_signatures(self): + import weakref + no_signature = {'ReferenceType', 'ref'} + self._test_module_has_signatures(weakref, no_signature) def test_python_function_override_signature(self): def func(*args, **kwargs): @@ -5284,6 +5881,7 @@ def func(*args, **kwargs): with self.assertRaises(ValueError): inspect.signature(func) + @support.requires_docstrings def test_base_class_have_text_signature(self): # see issue 43118 from test.typinganndata.ann_module7 import BufferedReader diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 47fc50a0e20349..8870d7aa5d663d 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -1,5 +1,4 @@ import sys -import time import unittest from unittest import mock @@ -12,6 +11,11 @@ except ImportError: _pylong = None +try: + import _decimal +except ImportError: + _decimal = None + L = [ ('0', 0), ('1', 1), @@ -397,68 +401,8 @@ def __trunc__(self): class JustTrunc(base): def __trunc__(self): return 42 - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(JustTrunc()), 42) - - class ExceptionalTrunc(base): - def __trunc__(self): - 1 / 0 - with self.assertRaises(ZeroDivisionError), \ - self.assertWarns(DeprecationWarning): - int(ExceptionalTrunc()) - - for trunc_result_base in (object, Classic): - class Index(trunc_result_base): - def __index__(self): - return 42 - - class TruncReturnsNonInt(base): - def __trunc__(self): - return Index() - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(TruncReturnsNonInt()), 42) - - class Intable(trunc_result_base): - def __int__(self): - return 42 - - class TruncReturnsNonIndex(base): - def __trunc__(self): - return Intable() - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(TruncReturnsNonInt()), 42) - - class NonIntegral(trunc_result_base): - def __trunc__(self): - # Check that we avoid infinite recursion. - return NonIntegral() - - class TruncReturnsNonIntegral(base): - def __trunc__(self): - return NonIntegral() - try: - with self.assertWarns(DeprecationWarning): - int(TruncReturnsNonIntegral()) - except TypeError as e: - self.assertEqual(str(e), - "__trunc__ returned non-Integral" - " (type NonIntegral)") - else: - self.fail("Failed to raise TypeError with %s" % - ((base, trunc_result_base),)) - - # Regression test for bugs.python.org/issue16060. - class BadInt(trunc_result_base): - def __int__(self): - return 42.0 - - class TruncReturnsBadInt(base): - def __trunc__(self): - return BadInt() - - with self.assertRaises(TypeError), \ - self.assertWarns(DeprecationWarning): - int(TruncReturnsBadInt()) + with self.assertRaises(TypeError): + int(JustTrunc()) def test_int_subclass_with_index(self): class MyIndex(int): @@ -509,18 +453,6 @@ class BadInt2(int): def __int__(self): return True - class TruncReturnsBadIndex: - def __trunc__(self): - return BadIndex() - - class TruncReturnsBadInt: - def __trunc__(self): - return BadInt() - - class TruncReturnsIntSubclass: - def __trunc__(self): - return True - bad_int = BadIndex() with self.assertWarns(DeprecationWarning): n = int(bad_int) @@ -544,26 +476,6 @@ def __trunc__(self): self.assertEqual(n, 1) self.assertIs(type(n), int) - bad_int = TruncReturnsBadIndex() - with self.assertWarns(DeprecationWarning): - n = int(bad_int) - self.assertEqual(n, 1) - self.assertIs(type(n), int) - - bad_int = TruncReturnsBadInt() - with self.assertWarns(DeprecationWarning): - self.assertRaises(TypeError, int, bad_int) - - good_int = TruncReturnsIntSubclass() - with self.assertWarns(DeprecationWarning): - n = int(good_int) - self.assertEqual(n, 1) - self.assertIs(type(n), int) - with self.assertWarns(DeprecationWarning): - n = IntSubclass(good_int) - self.assertEqual(n, 1) - self.assertIs(type(n), IntSubclass) - def test_error_message(self): def check(s, base=None): with self.assertRaises(ValueError, @@ -604,6 +516,13 @@ def test_issue31619(self): self.assertEqual(int('1_2_3_4_5_6_7_8_9', 16), 0x123456789) self.assertEqual(int('1_2_3_4_5_6_7', 32), 1144132807) + @support.cpython_only + def test_round_with_none_arg_direct_call(self): + for val in [(1).__round__(None), + round(1), + round(1, None)]: + self.assertEqual(val, 1) + self.assertIs(type(val), int) class IntStrDigitLimitsTests(unittest.TestCase): @@ -829,17 +748,28 @@ def tearDown(self): sys.set_int_max_str_digits(self._previous_limit) super().tearDown() - def test_pylong_int_to_decimal(self): - n = (1 << 100_000) - 1 - suffix = '9883109375' + def _test_pylong_int_to_decimal(self, n, suffix): s = str(n) - assert s[-10:] == suffix - s = str(-n) - assert s[-10:] == suffix - s = '%d' % n - assert s[-10:] == suffix - s = b'%d' % n - assert s[-10:] == suffix.encode('ascii') + self.assertEqual(s[-10:], suffix) + s2 = str(-n) + self.assertEqual(s2, '-' + s) + s3 = '%d' % n + self.assertEqual(s3, s) + s4 = b'%d' % n + self.assertEqual(s4, s.encode('ascii')) + + def test_pylong_int_to_decimal(self): + self._test_pylong_int_to_decimal((1 << 100_000), '9883109376') + self._test_pylong_int_to_decimal((1 << 100_000) - 1, '9883109375') + self._test_pylong_int_to_decimal(10**30_000, '0000000000') + self._test_pylong_int_to_decimal(10**30_000 - 1, '9999999999') + self._test_pylong_int_to_decimal(3**60_000, '9313200001') + + @support.requires_resource('cpu') + def test_pylong_int_to_decimal_2(self): + self._test_pylong_int_to_decimal(2**1_000_000, '2747109376') + self._test_pylong_int_to_decimal(10**300_000, '0000000000') + self._test_pylong_int_to_decimal(3**600_000, '3132000001') def test_pylong_int_divmod(self): n = (1 << 100_000) @@ -895,6 +825,100 @@ def test_pylong_misbehavior_error_path_from_str( with self.assertRaises(RuntimeError): int(big_value) + def test_pylong_roundtrip(self): + from random import randrange, getrandbits + bits = 5000 + while bits <= 1_000_000: + bits += randrange(-100, 101) # break bitlength patterns + hibit = 1 << (bits - 1) + n = hibit | getrandbits(bits - 1) + assert n.bit_length() == bits + sn = str(n) + self.assertFalse(sn.startswith('0')) + self.assertEqual(n, int(sn)) + bits <<= 1 + + @support.requires_resource('cpu') + @unittest.skipUnless(_decimal, "C _decimal module required") + def test_pylong_roundtrip_huge(self): + # k blocks of 1234567890 + k = 1_000_000 # so 10 million digits in all + tentoten = 10**10 + n = 1234567890 * ((tentoten**k - 1) // (tentoten - 1)) + sn = "1234567890" * k + self.assertEqual(n, int(sn)) + self.assertEqual(sn, str(n)) + + @support.requires_resource('cpu') + @unittest.skipUnless(_pylong, "_pylong module required") + @unittest.skipUnless(_decimal, "C _decimal module required") + def test_whitebox_dec_str_to_int_inner_failsafe(self): + # While I believe the number of GUARD digits in this function is + # always enough so that no more than one correction step is ever + # needed, the code has a "failsafe" path that takes over if I'm + # wrong about that. We have no input that reaches that block. + # Here we test a contrived input that _does_ reach that block, + # provided the number of guard digits is reduced to 1. + sn = "9" * 2000156 + n = 10**len(sn) - 1 + orig_spread = _pylong._spread.copy() + _pylong._spread.clear() + try: + self.assertEqual(n, _pylong._dec_str_to_int_inner(sn, GUARD=1)) + self.assertIn(999, _pylong._spread) + finally: + _pylong._spread.clear() + _pylong._spread.update(orig_spread) + + @unittest.skipUnless(_pylong, "pylong module required") + @unittest.skipUnless(_decimal, "C _decimal module required") + def test_whitebox_dec_str_to_int_inner_monster(self): + # I don't think anyone has enough RAM to build a string long enough + # for this function to complain. So lie about the string length. + + class LyingStr(str): + def __len__(self): + return int((1 << 47) / _pylong._LOG_10_BASE_256) + + liar = LyingStr("42") + # We have to pass the liar directly to the complaining function. If we + # just try `int(liar)`, earlier layers will replace it with plain old + # "43". + # Embedding `len(liar)` into the f-string failed on the WASI testbot + # (don't know what that is): + # OverflowError: cannot fit 'int' into an index-sized integer + # So a random stab at worming around that. + self.assertRaisesRegex(ValueError, + f"^cannot convert string of len {liar.__len__()} to int$", + _pylong._dec_str_to_int_inner, + liar) + + @unittest.skipUnless(_pylong, "_pylong module required") + def test_pylong_compute_powers(self): + # Basic sanity tests. See end of _pylong.py for manual heavy tests. + def consumer(w, base, limit, need_hi): + seen = set() + need = set() + def inner(w): + if w <= limit or w in seen: + return + seen.add(w) + lo = w >> 1 + hi = w - lo + need.add(hi if need_hi else lo) + inner(lo) + inner(hi) + inner(w) + d = _pylong.compute_powers(w, base, limit, need_hi=need_hi) + self.assertEqual(d.keys(), need) + for k, v in d.items(): + self.assertEqual(v, base ** k) + + for base in 2, 5: + for need_hi in False, True: + for limit in 1, 11: + for w in range(250, 550): + consumer(w, base, limit, need_hi) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_interpreters/__init__.py b/Lib/test/test_interpreters/__init__.py index 4b16ecc31156a5..e3d189c4efcd27 100644 --- a/Lib/test/test_interpreters/__init__.py +++ b/Lib/test/test_interpreters/__init__.py @@ -1,5 +1,9 @@ import os -from test.support import load_package_tests +from test.support import load_package_tests, Py_GIL_DISABLED +import unittest + +if Py_GIL_DISABLED: + raise unittest.SkipTest("GIL disabled") def load_tests(*args): return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 363143fa810f35..01856d9bf67657 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -1,15 +1,32 @@ import os -import threading +import pickle from textwrap import dedent +import threading +import types import unittest from test import support from test.support import import_helper # Raise SkipTest if subinterpreters not supported. -import_helper.import_module('_xxsubinterpreters') +_interpreters = import_helper.import_module('_interpreters') +from test.support import Py_GIL_DISABLED from test.support import interpreters -from test.support.interpreters import InterpreterNotFoundError -from .utils import _captured_script, _run_output, _running, TestBase +from test.support import force_not_colorized +from test.support.interpreters import ( + InterpreterError, InterpreterNotFoundError, ExecutionFailed, +) +from .utils import ( + _captured_script, _run_output, _running, TestBase, + requires_test_modules, _testinternalcapi, +) + + +WHENCE_STR_UNKNOWN = 'unknown' +WHENCE_STR_RUNTIME = 'runtime init' +WHENCE_STR_LEGACY_CAPI = 'legacy C-API' +WHENCE_STR_CAPI = 'C-API' +WHENCE_STR_XI = 'cross-interpreter C-API' +WHENCE_STR_STDLIB = '_interpreters module' class ModuleTests(TestBase): @@ -37,6 +54,9 @@ def test_in_main(self): self.assertIsInstance(interp, interpreters.Interpreter) self.assertIn(interp, interpreters.list_all()) + # GH-126221: Passing an invalid Unicode character used to cause a SystemError + self.assertRaises(UnicodeEncodeError, _interpreters.create, '\udc80') + def test_in_thread(self): lock = threading.Lock() interp = None @@ -155,6 +175,18 @@ def test_idempotent(self): id2 = id(interp) self.assertNotEqual(id1, id2) + @requires_test_modules + def test_created_with_capi(self): + expected = _testinternalcapi.next_interpreter_id() + text = self.run_temp_from_capi(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.get_current() + print((interp.id, interp.whence)) + """) + interpid, whence = eval(text) + self.assertEqual(interpid, expected) + self.assertEqual(whence, WHENCE_STR_CAPI) + class ListAllTests(TestBase): @@ -197,6 +229,33 @@ def test_idempotent(self): for interp1, interp2 in zip(actual, expected): self.assertIs(interp1, interp2) + def test_created_with_capi(self): + mainid, *_ = _interpreters.get_main() + interpid1 = _interpreters.create() + interpid2 = _interpreters.create() + interpid3 = _interpreters.create() + interpid4 = interpid3 + 1 + interpid5 = interpid4 + 1 + expected = [ + (mainid, WHENCE_STR_RUNTIME), + (interpid1, WHENCE_STR_STDLIB), + (interpid2, WHENCE_STR_STDLIB), + (interpid3, WHENCE_STR_STDLIB), + (interpid4, WHENCE_STR_CAPI), + (interpid5, WHENCE_STR_STDLIB), + ] + expected2 = expected[:-2] + text = self.run_temp_from_capi(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.create() + print( + [(i.id, i.whence) for i in interpreters.list_all()]) + """) + res = eval(text) + res2 = [(i.id, i.whence) for i in interpreters.list_all()] + self.assertEqual(res, expected) + self.assertEqual(res2, expected2) + class InterpreterObjectTests(TestBase): @@ -249,6 +308,38 @@ def test_id_readonly(self): with self.assertRaises(AttributeError): interp.id = 1_000_000 + def test_whence(self): + main = interpreters.get_main() + interp = interpreters.create() + + with self.subTest('main'): + self.assertEqual(main.whence, WHENCE_STR_RUNTIME) + + with self.subTest('from _interpreters'): + self.assertEqual(interp.whence, WHENCE_STR_STDLIB) + + with self.subTest('from C-API'): + text = self.run_temp_from_capi(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.get_current() + print(repr(interp.whence)) + """) + whence = eval(text) + self.assertEqual(whence, WHENCE_STR_CAPI) + + with self.subTest('readonly'): + for value in [ + None, + WHENCE_STR_UNKNOWN, + WHENCE_STR_RUNTIME, + WHENCE_STR_STDLIB, + WHENCE_STR_CAPI, + ]: + with self.assertRaises(AttributeError): + interp.whence = value + with self.assertRaises(AttributeError): + main.whence = value + def test_hashable(self): interp = interpreters.create() expected = hash(interp.id) @@ -261,6 +352,12 @@ def test_equality(self): self.assertEqual(interp1, interp1) self.assertNotEqual(interp1, interp2) + def test_pickle(self): + interp = interpreters.create() + data = pickle.dumps(interp) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, interp) + class TestInterpreterIsRunning(TestBase): @@ -268,6 +365,7 @@ def test_main(self): main = interpreters.get_main() self.assertTrue(main.is_running()) + # XXX Is this still true? @unittest.skip('Fails on FreeBSD') def test_subinterpreter(self): interp = interpreters.create() @@ -290,7 +388,7 @@ def test_finished(self): def test_from_subinterpreter(self): interp = interpreters.create() out = _run_output(interp, dedent(f""" - import _xxsubinterpreters as _interpreters + import _interpreters if _interpreters.is_running({interp.id}): print(True) else: @@ -329,6 +427,55 @@ def task(): interp.exec('t.join()') self.assertEqual(os.read(r_interp, 1), FINISHED) + def test_created_with_capi(self): + script = dedent(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.get_current() + print(interp.is_running()) + """) + def parse_results(text): + self.assertNotEqual(text, "") + try: + return eval(text) + except Exception: + raise Exception(repr(text)) + + with self.subTest('running __main__ (from self)'): + with self.interpreter_from_capi() as interpid: + text = self.run_from_capi(interpid, script, main=True) + running = parse_results(text) + self.assertTrue(running) + + with self.subTest('running, but not __main__ (from self)'): + text = self.run_temp_from_capi(script) + running = parse_results(text) + self.assertFalse(running) + + with self.subTest('running __main__ (from other)'): + with self.interpreter_obj_from_capi() as (interp, interpid): + before = interp.is_running() + with self.running_from_capi(interpid, main=True): + during = interp.is_running() + after = interp.is_running() + self.assertFalse(before) + self.assertTrue(during) + self.assertFalse(after) + + with self.subTest('running, but not __main__ (from other)'): + with self.interpreter_obj_from_capi() as (interp, interpid): + before = interp.is_running() + with self.running_from_capi(interpid, main=False): + during = interp.is_running() + after = interp.is_running() + self.assertFalse(before) + self.assertFalse(during) + self.assertFalse(after) + + with self.subTest('not running (from other)'): + with self.interpreter_obj_from_capi() as (interp, _): + running = interp.is_running() + self.assertFalse(running) + class TestInterpreterClose(TestBase): @@ -356,11 +503,11 @@ def test_all(self): def test_main(self): main, = interpreters.list_all() - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterError): main.close() def f(): - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterError): main.close() t = threading.Thread(target=f) @@ -381,7 +528,7 @@ def test_from_current(self): interp = interpreters.Interpreter({interp.id}) try: interp.close() - except RuntimeError: + except interpreters.InterpreterError: print('failed') """)) self.assertEqual(out.strip(), 'failed') @@ -411,12 +558,13 @@ def f(): t.start() t.join() + # XXX Is this still true? @unittest.skip('Fails on FreeBSD') def test_still_running(self): main, = interpreters.list_all() interp = interpreters.create() with _running(interp): - with self.assertRaises(RuntimeError): + with self.assertRaises(InterpreterError): interp.close() self.assertTrue(interp.is_running()) @@ -451,6 +599,53 @@ def task(): self.assertEqual(os.read(r_interp, 1), FINISHED) + def test_created_with_capi(self): + script = dedent(f""" + import {interpreters.__name__} as interpreters + interp = interpreters.get_current() + interp.close() + """) + + with self.subTest('running __main__ (from self)'): + with self.interpreter_from_capi() as interpid: + with self.assertRaisesRegex(ExecutionFailed, + 'InterpreterError.*unrecognized'): + self.run_from_capi(interpid, script, main=True) + + with self.subTest('running, but not __main__ (from self)'): + with self.assertRaisesRegex(ExecutionFailed, + 'InterpreterError.*unrecognized'): + self.run_temp_from_capi(script) + + with self.subTest('running __main__ (from other)'): + with self.interpreter_obj_from_capi() as (interp, interpid): + with self.running_from_capi(interpid, main=True): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + interp.close() + # Make sure it wssn't closed. + self.assertTrue( + self.interp_exists(interpid)) + + # The rest would be skipped until we deal with running threads when + # interp.close() is called. However, the "whence" restrictions + # trigger first. + + with self.subTest('running, but not __main__ (from other)'): + with self.interpreter_obj_from_capi() as (interp, interpid): + with self.running_from_capi(interpid, main=False): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + interp.close() + # Make sure it wssn't closed. + self.assertTrue( + self.interp_exists(interpid)) + + with self.subTest('not running (from other)'): + with self.interpreter_obj_from_capi() as (interp, interpid): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + interp.close() + self.assertTrue( + self.interp_exists(interpid)) + class TestInterpreterPrepareMain(TestBase): @@ -503,28 +698,47 @@ def test_not_shareable(self): interp.prepare_main(spam={'spam': 'eggs', 'foo': 'bar'}) # Make sure neither was actually bound. - with self.assertRaises(interpreters.ExecutionFailed): + with self.assertRaises(ExecutionFailed): interp.exec('print(foo)') - with self.assertRaises(interpreters.ExecutionFailed): + with self.assertRaises(ExecutionFailed): interp.exec('print(spam)') + def test_running(self): + interp = interpreters.create() + interp.prepare_main({'spam': True}) + with self.running(interp): + with self.assertRaisesRegex(InterpreterError, 'running'): + interp.prepare_main({'spam': False}) + interp.exec('assert spam is True') + + @requires_test_modules + def test_created_with_capi(self): + with self.interpreter_obj_from_capi() as (interp, interpid): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + interp.prepare_main({'spam': True}) + with self.assertRaisesRegex(ExecutionFailed, 'NameError'): + self.run_from_capi(interpid, 'assert spam is True') + class TestInterpreterExec(TestBase): def test_success(self): interp = interpreters.create() - script, file = _captured_script('print("it worked!", end="")') - with file: + script, results = _captured_script('print("it worked!", end="")') + with results: interp.exec(script) - out = file.read() + results = results.final() + results.raise_if_failed() + out = results.stdout self.assertEqual(out, 'it worked!') def test_failure(self): interp = interpreters.create() - with self.assertRaises(interpreters.ExecutionFailed): + with self.assertRaises(ExecutionFailed): interp.exec('raise Exception') + @force_not_colorized def test_display_preserved_exception(self): tempdir = self.temp_dir() modfile = self.make_module('spam', tempdir, text=""" @@ -575,15 +789,17 @@ def script(): def test_in_thread(self): interp = interpreters.create() - script, file = _captured_script('print("it worked!", end="")') - with file: + script, results = _captured_script('print("it worked!", end="")') + with results: def f(): interp.exec(script) t = threading.Thread(target=f) t.start() t.join() - out = file.read() + results = results.final() + results.raise_if_failed() + out = results.stdout self.assertEqual(out, 'it worked!') @@ -610,6 +826,7 @@ def test_fork(self): content = file.read() self.assertEqual(content, expected) + # XXX Is this still true? @unittest.skip('Fails on FreeBSD') def test_already_running(self): interp = interpreters.create() @@ -658,7 +875,12 @@ def task(): self.assertEqual(os.read(r_interp, 1), RAN) self.assertEqual(os.read(r_interp, 1), FINISHED) - # test_xxsubinterpreters covers the remaining + def test_created_with_capi(self): + with self.interpreter_obj_from_capi() as (interp, _): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + interp.exec('raise Exception("it worked!")') + + # test__interpreters covers the remaining # Interpreter.exec() behavior. @@ -822,7 +1044,7 @@ def test_call(self): raise Exception((args, kwargs)) interp.call(callable) - with self.assertRaises(interpreters.ExecutionFailed): + with self.assertRaises(ExecutionFailed): interp.call(call_func_failure) def test_call_in_thread(self): @@ -925,6 +1147,525 @@ class SubBytes(bytes): interpreters.is_shareable(obj)) +class LowLevelTests(TestBase): + + # The behaviors in the low-level module are important in as much + # as they are exercised by the high-level module. Therefore the + # most important testing happens in the high-level tests. + # These low-level tests cover corner cases that are not + # encountered by the high-level module, thus they + # mostly shouldn't matter as much. + + def test_new_config(self): + # This test overlaps with + # test.test_capi.test_misc.InterpreterConfigTests. + + default = _interpreters.new_config('isolated') + with self.subTest('no arg'): + config = _interpreters.new_config() + self.assert_ns_equal(config, default) + self.assertIsNot(config, default) + + with self.subTest('default'): + config1 = _interpreters.new_config('default') + self.assert_ns_equal(config1, default) + self.assertIsNot(config1, default) + + config2 = _interpreters.new_config('default') + self.assert_ns_equal(config2, config1) + self.assertIsNot(config2, config1) + + for arg in ['', 'default']: + with self.subTest(f'default ({arg!r})'): + config = _interpreters.new_config(arg) + self.assert_ns_equal(config, default) + self.assertIsNot(config, default) + + supported = { + 'isolated': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + check_multi_interp_extensions=True, + gil='own', + ), + 'legacy': types.SimpleNamespace( + use_main_obmalloc=True, + allow_fork=True, + allow_exec=True, + allow_threads=True, + allow_daemon_threads=True, + check_multi_interp_extensions=bool(Py_GIL_DISABLED), + gil='shared', + ), + 'empty': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=False, + allow_daemon_threads=False, + check_multi_interp_extensions=False, + gil='default', + ), + } + gil_supported = ['default', 'shared', 'own'] + + for name, vanilla in supported.items(): + with self.subTest(f'supported ({name})'): + expected = vanilla + config1 = _interpreters.new_config(name) + self.assert_ns_equal(config1, expected) + self.assertIsNot(config1, expected) + + config2 = _interpreters.new_config(name) + self.assert_ns_equal(config2, config1) + self.assertIsNot(config2, config1) + + with self.subTest(f'noop override ({name})'): + expected = vanilla + overrides = vars(vanilla) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest(f'override all ({name})'): + overrides = {k: not v for k, v in vars(vanilla).items()} + for gil in gil_supported: + if vanilla.gil == gil: + continue + overrides['gil'] = gil + expected = types.SimpleNamespace(**overrides) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + # Override individual fields. + for field, old in vars(vanilla).items(): + if field == 'gil': + values = [v for v in gil_supported if v != old] + else: + values = [not old] + for val in values: + with self.subTest(f'{name}.{field} ({old!r} -> {val!r})'): + overrides = {field: val} + expected = types.SimpleNamespace( + **dict(vars(vanilla), **overrides), + ) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest('extra override'): + with self.assertRaises(ValueError): + _interpreters.new_config(spam=True) + + # Bad values for bool fields. + for field, value in vars(supported['empty']).items(): + if field == 'gil': + continue + assert isinstance(value, bool) + for value in [1, '', 'spam', 1.0, None, object()]: + with self.subTest(f'bad override ({field}={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(**{field: value}) + + # Bad values for .gil. + for value in [True, 1, 1.0, None, object()]: + with self.subTest(f'bad override (gil={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(gil=value) + for value in ['', 'spam']: + with self.subTest(f'bad override (gil={value!r})'): + with self.assertRaises(ValueError): + _interpreters.new_config(gil=value) + + def test_get_main(self): + interpid, whence = _interpreters.get_main() + self.assertEqual(interpid, 0) + self.assertEqual(whence, _interpreters.WHENCE_RUNTIME) + self.assertEqual( + _interpreters.whence(interpid), + _interpreters.WHENCE_RUNTIME) + + def test_get_current(self): + with self.subTest('main'): + main, *_ = _interpreters.get_main() + interpid, whence = _interpreters.get_current() + self.assertEqual(interpid, main) + self.assertEqual(whence, _interpreters.WHENCE_RUNTIME) + + script = f""" + import _interpreters + interpid, whence = _interpreters.get_current() + print((interpid, whence)) + """ + def parse_stdout(text): + interpid, whence = eval(text) + return interpid, whence + + with self.subTest('from _interpreters'): + orig = _interpreters.create() + text = self.run_and_capture(orig, script) + interpid, whence = parse_stdout(text) + self.assertEqual(interpid, orig) + self.assertEqual(whence, _interpreters.WHENCE_STDLIB) + + with self.subTest('from C-API'): + last = 0 + for id, *_ in _interpreters.list_all(): + last = max(last, id) + expected = last + 1 + text = self.run_temp_from_capi(script) + interpid, whence = parse_stdout(text) + self.assertEqual(interpid, expected) + self.assertEqual(whence, _interpreters.WHENCE_CAPI) + + def test_list_all(self): + mainid, *_ = _interpreters.get_main() + interpid1 = _interpreters.create() + interpid2 = _interpreters.create() + interpid3 = _interpreters.create() + expected = [ + (mainid, _interpreters.WHENCE_RUNTIME), + (interpid1, _interpreters.WHENCE_STDLIB), + (interpid2, _interpreters.WHENCE_STDLIB), + (interpid3, _interpreters.WHENCE_STDLIB), + ] + + with self.subTest('main'): + res = _interpreters.list_all() + self.assertEqual(res, expected) + + with self.subTest('via interp from _interpreters'): + text = self.run_and_capture(interpid2, f""" + import _interpreters + print( + _interpreters.list_all()) + """) + + res = eval(text) + self.assertEqual(res, expected) + + with self.subTest('via interp from C-API'): + interpid4 = interpid3 + 1 + interpid5 = interpid4 + 1 + expected2 = expected + [ + (interpid4, _interpreters.WHENCE_CAPI), + (interpid5, _interpreters.WHENCE_STDLIB), + ] + expected3 = expected + [ + (interpid5, _interpreters.WHENCE_STDLIB), + ] + text = self.run_temp_from_capi(f""" + import _interpreters + _interpreters.create() + print( + _interpreters.list_all()) + """) + res2 = eval(text) + res3 = _interpreters.list_all() + self.assertEqual(res2, expected2) + self.assertEqual(res3, expected3) + + def test_create(self): + isolated = _interpreters.new_config('isolated') + legacy = _interpreters.new_config('legacy') + default = isolated + + with self.subTest('no args'): + interpid = _interpreters.create() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, default) + + with self.subTest('config: None'): + interpid = _interpreters.create(None) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, default) + + with self.subTest('config: \'empty\''): + with self.assertRaises(InterpreterError): + # The "empty" config isn't viable on its own. + _interpreters.create('empty') + + for arg, expected in { + '': default, + 'default': default, + 'isolated': isolated, + 'legacy': legacy, + }.items(): + with self.subTest(f'str arg: {arg!r}'): + interpid = _interpreters.create(arg) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('custom'): + orig = _interpreters.new_config('empty') + orig.use_main_obmalloc = True + orig.check_multi_interp_extensions = bool(Py_GIL_DISABLED) + orig.gil = 'shared' + interpid = _interpreters.create(orig) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, orig) + + with self.subTest('missing fields'): + orig = _interpreters.new_config() + del orig.gil + with self.assertRaises(ValueError): + _interpreters.create(orig) + + with self.subTest('extra fields'): + orig = _interpreters.new_config() + orig.spam = True + with self.assertRaises(ValueError): + _interpreters.create(orig) + + with self.subTest('whence'): + interpid = _interpreters.create() + self.assertEqual( + _interpreters.whence(interpid), + _interpreters.WHENCE_STDLIB) + + @requires_test_modules + def test_destroy(self): + with self.subTest('from _interpreters'): + interpid = _interpreters.create() + before = [id for id, *_ in _interpreters.list_all()] + _interpreters.destroy(interpid) + after = [id for id, *_ in _interpreters.list_all()] + + self.assertIn(interpid, before) + self.assertNotIn(interpid, after) + self.assertFalse( + self.interp_exists(interpid)) + + with self.subTest('main'): + interpid, *_ = _interpreters.get_main() + with self.assertRaises(InterpreterError): + # It is the current interpreter. + _interpreters.destroy(interpid) + + with self.subTest('from C-API'): + interpid = _testinternalcapi.create_interpreter() + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.destroy(interpid, restrict=True) + self.assertTrue( + self.interp_exists(interpid)) + _interpreters.destroy(interpid) + self.assertFalse( + self.interp_exists(interpid)) + + def test_get_config(self): + # This test overlaps with + # test.test_capi.test_misc.InterpreterConfigTests. + + with self.subTest('main'): + expected = _interpreters.new_config('legacy') + expected.gil = 'own' + if Py_GIL_DISABLED: + expected.check_multi_interp_extensions = False + interpid, *_ = _interpreters.get_main() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('isolated'): + expected = _interpreters.new_config('isolated') + interpid = _interpreters.create('isolated') + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('legacy'): + expected = _interpreters.new_config('legacy') + interpid = _interpreters.create('legacy') + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('from C-API'): + orig = _interpreters.new_config('isolated') + with self.interpreter_from_capi(orig) as interpid: + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.get_config(interpid, restrict=True) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, orig) + + @requires_test_modules + def test_whence(self): + with self.subTest('main'): + interpid, *_ = _interpreters.get_main() + whence = _interpreters.whence(interpid) + self.assertEqual(whence, _interpreters.WHENCE_RUNTIME) + + with self.subTest('stdlib'): + interpid = _interpreters.create() + whence = _interpreters.whence(interpid) + self.assertEqual(whence, _interpreters.WHENCE_STDLIB) + + for orig, name in { + _interpreters.WHENCE_UNKNOWN: 'not ready', + _interpreters.WHENCE_LEGACY_CAPI: 'legacy C-API', + _interpreters.WHENCE_CAPI: 'C-API', + _interpreters.WHENCE_XI: 'cross-interpreter C-API', + }.items(): + with self.subTest(f'from C-API ({orig}: {name})'): + with self.interpreter_from_capi(whence=orig) as interpid: + whence = _interpreters.whence(interpid) + self.assertEqual(whence, orig) + + with self.subTest('from C-API, running'): + text = self.run_temp_from_capi(dedent(f""" + import _interpreters + interpid, *_ = _interpreters.get_current() + print(_interpreters.whence(interpid)) + """), + config=True) + whence = eval(text) + self.assertEqual(whence, _interpreters.WHENCE_CAPI) + + with self.subTest('from legacy C-API, running'): + ... + text = self.run_temp_from_capi(dedent(f""" + import _interpreters + interpid, *_ = _interpreters.get_current() + print(_interpreters.whence(interpid)) + """), + config=False) + whence = eval(text) + self.assertEqual(whence, _interpreters.WHENCE_LEGACY_CAPI) + + def test_is_running(self): + def check(interpid, expected): + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.is_running(interpid, restrict=True) + running = _interpreters.is_running(interpid) + self.assertIs(running, expected) + + with self.subTest('from _interpreters (running)'): + interpid = _interpreters.create() + with self.running(interpid): + running = _interpreters.is_running(interpid) + self.assertTrue(running) + + with self.subTest('from _interpreters (not running)'): + interpid = _interpreters.create() + running = _interpreters.is_running(interpid) + self.assertFalse(running) + + with self.subTest('main'): + interpid, *_ = _interpreters.get_main() + check(interpid, True) + + with self.subTest('from C-API (running __main__)'): + with self.interpreter_from_capi() as interpid: + with self.running_from_capi(interpid, main=True): + check(interpid, True) + + with self.subTest('from C-API (running, but not __main__)'): + with self.interpreter_from_capi() as interpid: + with self.running_from_capi(interpid, main=False): + check(interpid, False) + + with self.subTest('from C-API (not running)'): + with self.interpreter_from_capi() as interpid: + check(interpid, False) + + def test_exec(self): + with self.subTest('run script'): + interpid = _interpreters.create() + script, results = _captured_script('print("it worked!", end="")') + with results: + exc = _interpreters.exec(interpid, script) + results = results.final() + results.raise_if_failed() + out = results.stdout + self.assertEqual(out, 'it worked!') + + with self.subTest('uncaught exception'): + interpid = _interpreters.create() + script, results = _captured_script(""" + raise Exception('uh-oh!') + print("it worked!", end="") + """) + with results: + exc = _interpreters.exec(interpid, script) + out = results.stdout() + self.assertEqual(out, '') + self.assert_ns_equal(exc, types.SimpleNamespace( + type=types.SimpleNamespace( + __name__='Exception', + __qualname__='Exception', + __module__='builtins', + ), + msg='uh-oh!', + # We check these in other tests. + formatted=exc.formatted, + errdisplay=exc.errdisplay, + )) + + with self.subTest('from C-API'): + with self.interpreter_from_capi() as interpid: + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.exec(interpid, 'raise Exception("it worked!")', + restrict=True) + exc = _interpreters.exec(interpid, 'raise Exception("it worked!")') + self.assertIsNot(exc, None) + self.assertEqual(exc.msg, 'it worked!') + + def test_call(self): + with self.subTest('no args'): + interpid = _interpreters.create() + exc = _interpreters.call(interpid, call_func_return_shareable) + self.assertIs(exc, None) + + with self.subTest('uncaught exception'): + interpid = _interpreters.create() + exc = _interpreters.call(interpid, call_func_failure) + self.assertEqual(exc, types.SimpleNamespace( + type=types.SimpleNamespace( + __name__='Exception', + __qualname__='Exception', + __module__='builtins', + ), + msg='spam!', + # We check these in other tests. + formatted=exc.formatted, + errdisplay=exc.errdisplay, + )) + + @requires_test_modules + def test_set___main___attrs(self): + with self.subTest('from _interpreters'): + interpid = _interpreters.create() + before1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'') + before2 = _interpreters.exec(interpid, 'assert ham == 42') + self.assertEqual(before1.type.__name__, 'NameError') + self.assertEqual(before2.type.__name__, 'NameError') + + _interpreters.set___main___attrs(interpid, dict( + spam='eggs', + ham=42, + )) + after1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'') + after2 = _interpreters.exec(interpid, 'assert ham == 42') + after3 = _interpreters.exec(interpid, 'assert spam == 42') + self.assertIs(after1, None) + self.assertIs(after2, None) + self.assertEqual(after3.type.__name__, 'AssertionError') + + with self.assertRaises(ValueError): + # GH-127165: Embedded NULL characters broke the lookup + _interpreters.set___main___attrs(interpid, {"\x00": 1}) + + with self.subTest('from C-API'): + with self.interpreter_from_capi() as interpid: + with self.assertRaisesRegex(InterpreterError, 'unrecognized'): + _interpreters.set___main___attrs(interpid, {'spam': True}, + restrict=True) + _interpreters.set___main___attrs(interpid, {'spam': True}) + rc = _testinternalcapi.exec_interpreter( + interpid, + 'assert spam is True', + ) + self.assertEqual(rc, 0) + + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. unittest.main() diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py index 57204e2776468d..eada18f99d04db 100644 --- a/Lib/test/test_interpreters/test_channels.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -1,4 +1,5 @@ import importlib +import pickle import threading from textwrap import dedent import unittest @@ -6,7 +7,7 @@ from test.support import import_helper # Raise SkipTest if subinterpreters not supported. -_channels = import_helper.import_module('_xxinterpchannels') +_channels = import_helper.import_module('_interpchannels') from test.support import interpreters from test.support.interpreters import channels from .utils import _run_output, TestBase @@ -21,7 +22,7 @@ class LowLevelTests(TestBase): # encountered by the high-level module, thus they # mostly shouldn't matter as much. - # Additional tests are found in Lib/test/test__xxinterpchannels.py. + # Additional tests are found in Lib/test/test__interpchannels.py. # XXX Those should be either moved to LowLevelTests or eliminated # in favor of high-level tests in this file. @@ -47,6 +48,7 @@ def test_list_all(self): self.assertEqual(after, created) def test_shareable(self): + interp = interpreters.create() rch, sch = channels.create() self.assertTrue( @@ -59,8 +61,25 @@ def test_shareable(self): rch2 = rch.recv() sch2 = rch.recv() + interp.prepare_main(rch=rch, sch=sch) + sch.send_nowait(rch) + sch.send_nowait(sch) + interp.exec(dedent(""" + rch2 = rch.recv() + sch2 = rch.recv() + assert rch2 == rch + assert sch2 == sch + + sch.send_nowait(rch2) + sch.send_nowait(sch2) + """)) + rch3 = rch.recv() + sch3 = rch.recv() + self.assertEqual(rch2, rch) self.assertEqual(sch2, sch) + self.assertEqual(rch3, rch) + self.assertEqual(sch3, sch) def test_is_closed(self): rch, sch = channels.create() @@ -100,6 +119,12 @@ def test_equality(self): self.assertEqual(ch1, ch1) self.assertNotEqual(ch1, ch2) + def test_pickle(self): + ch, _ = channels.create() + data = pickle.dumps(ch) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, ch) + class TestSendChannelAttrs(TestBase): @@ -125,6 +150,12 @@ def test_equality(self): self.assertEqual(ch1, ch1) self.assertNotEqual(ch1, ch2) + def test_pickle(self): + _, ch = channels.create() + data = pickle.dumps(ch) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, ch) + class TestSendRecv(TestBase): @@ -341,6 +372,228 @@ def test_send_buffer_nowait(self): obj[4:8] = b'ham.' self.assertEqual(obj, buf) + def test_send_cleared_with_subinterpreter(self): + def common(rch, sch, unbound=None, presize=0): + if not unbound: + extraargs = '' + elif unbound is channels.UNBOUND: + extraargs = ', unbound=channels.UNBOUND' + elif unbound is channels.UNBOUND_ERROR: + extraargs = ', unbound=channels.UNBOUND_ERROR' + elif unbound is channels.UNBOUND_REMOVE: + extraargs = ', unbound=channels.UNBOUND_REMOVE' + else: + raise NotImplementedError(repr(unbound)) + interp = interpreters.create() + + _run_output(interp, dedent(f""" + from test.support.interpreters import channels + sch = channels.SendChannel({sch.id}) + obj1 = b'spam' + obj2 = b'eggs' + sch.send_nowait(obj1{extraargs}) + sch.send_nowait(obj2{extraargs}) + """)) + self.assertEqual( + _channels.get_count(rch.id), + presize + 2, + ) + + if presize == 0: + obj1 = rch.recv() + self.assertEqual(obj1, b'spam') + self.assertEqual( + _channels.get_count(rch.id), + presize + 1, + ) + + return interp + + with self.subTest('default'): # UNBOUND + rch, sch = channels.create() + interp = common(rch, sch) + del interp + self.assertEqual(_channels.get_count(rch.id), 1) + obj1 = rch.recv() + self.assertEqual(_channels.get_count(rch.id), 0) + self.assertIs(obj1, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 0) + with self.assertRaises(channels.ChannelEmptyError): + rch.recv_nowait() + + with self.subTest('UNBOUND'): + rch, sch = channels.create() + interp = common(rch, sch, channels.UNBOUND) + del interp + self.assertEqual(_channels.get_count(rch.id), 1) + obj1 = rch.recv() + self.assertIs(obj1, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 0) + with self.assertRaises(channels.ChannelEmptyError): + rch.recv_nowait() + + with self.subTest('UNBOUND_ERROR'): + rch, sch = channels.create() + interp = common(rch, sch, channels.UNBOUND_ERROR) + + del interp + self.assertEqual(_channels.get_count(rch.id), 1) + with self.assertRaises(channels.ItemInterpreterDestroyed): + rch.recv() + + self.assertEqual(_channels.get_count(rch.id), 0) + with self.assertRaises(channels.ChannelEmptyError): + rch.recv_nowait() + + with self.subTest('UNBOUND_REMOVE'): + rch, sch = channels.create() + + interp = common(rch, sch, channels.UNBOUND_REMOVE) + del interp + self.assertEqual(_channels.get_count(rch.id), 0) + with self.assertRaises(channels.ChannelEmptyError): + rch.recv_nowait() + + sch.send_nowait(b'ham', unbound=channels.UNBOUND_REMOVE) + self.assertEqual(_channels.get_count(rch.id), 1) + interp = common(rch, sch, channels.UNBOUND_REMOVE, 1) + self.assertEqual(_channels.get_count(rch.id), 3) + sch.send_nowait(42, unbound=channels.UNBOUND_REMOVE) + self.assertEqual(_channels.get_count(rch.id), 4) + del interp + self.assertEqual(_channels.get_count(rch.id), 2) + obj1 = rch.recv() + obj2 = rch.recv() + self.assertEqual(obj1, b'ham') + self.assertEqual(obj2, 42) + self.assertEqual(_channels.get_count(rch.id), 0) + with self.assertRaises(channels.ChannelEmptyError): + rch.recv_nowait() + + def test_send_cleared_with_subinterpreter_mixed(self): + rch, sch = channels.create() + interp = interpreters.create() + + # If we don't associate the main interpreter with the channel + # then the channel will be automatically closed when interp + # is destroyed. + sch.send_nowait(None) + rch.recv() + self.assertEqual(_channels.get_count(rch.id), 0) + + _run_output(interp, dedent(f""" + from test.support.interpreters import channels + sch = channels.SendChannel({sch.id}) + sch.send_nowait(1, unbound=channels.UNBOUND) + sch.send_nowait(2, unbound=channels.UNBOUND_ERROR) + sch.send_nowait(3) + sch.send_nowait(4, unbound=channels.UNBOUND_REMOVE) + sch.send_nowait(5, unbound=channels.UNBOUND) + """)) + self.assertEqual(_channels.get_count(rch.id), 5) + + del interp + self.assertEqual(_channels.get_count(rch.id), 4) + + obj1 = rch.recv() + self.assertIs(obj1, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 3) + + with self.assertRaises(channels.ItemInterpreterDestroyed): + rch.recv() + self.assertEqual(_channels.get_count(rch.id), 2) + + obj2 = rch.recv() + self.assertIs(obj2, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 1) + + obj3 = rch.recv() + self.assertIs(obj3, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 0) + + def test_send_cleared_with_subinterpreter_multiple(self): + rch, sch = channels.create() + interp1 = interpreters.create() + interp2 = interpreters.create() + + sch.send_nowait(1) + _run_output(interp1, dedent(f""" + from test.support.interpreters import channels + rch = channels.RecvChannel({rch.id}) + sch = channels.SendChannel({sch.id}) + obj1 = rch.recv() + sch.send_nowait(2, unbound=channels.UNBOUND) + sch.send_nowait(obj1, unbound=channels.UNBOUND_REMOVE) + """)) + _run_output(interp2, dedent(f""" + from test.support.interpreters import channels + rch = channels.RecvChannel({rch.id}) + sch = channels.SendChannel({sch.id}) + obj2 = rch.recv() + obj1 = rch.recv() + """)) + self.assertEqual(_channels.get_count(rch.id), 0) + sch.send_nowait(3) + _run_output(interp1, dedent(""" + sch.send_nowait(4, unbound=channels.UNBOUND) + # interp closed here + sch.send_nowait(5, unbound=channels.UNBOUND_REMOVE) + sch.send_nowait(6, unbound=channels.UNBOUND) + """)) + _run_output(interp2, dedent(""" + sch.send_nowait(7, unbound=channels.UNBOUND_ERROR) + # interp closed here + sch.send_nowait(obj1, unbound=channels.UNBOUND_ERROR) + sch.send_nowait(obj2, unbound=channels.UNBOUND_REMOVE) + sch.send_nowait(8, unbound=channels.UNBOUND) + """)) + _run_output(interp1, dedent(""" + sch.send_nowait(9, unbound=channels.UNBOUND_REMOVE) + sch.send_nowait(10, unbound=channels.UNBOUND) + """)) + self.assertEqual(_channels.get_count(rch.id), 10) + + obj3 = rch.recv() + self.assertEqual(obj3, 3) + self.assertEqual(_channels.get_count(rch.id), 9) + + obj4 = rch.recv() + self.assertEqual(obj4, 4) + self.assertEqual(_channels.get_count(rch.id), 8) + + del interp1 + self.assertEqual(_channels.get_count(rch.id), 6) + + # obj5 was removed + + obj6 = rch.recv() + self.assertIs(obj6, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 5) + + obj7 = rch.recv() + self.assertEqual(obj7, 7) + self.assertEqual(_channels.get_count(rch.id), 4) + + del interp2 + self.assertEqual(_channels.get_count(rch.id), 3) + + # obj1 + with self.assertRaises(channels.ItemInterpreterDestroyed): + rch.recv() + self.assertEqual(_channels.get_count(rch.id), 2) + + # obj2 was removed + + obj8 = rch.recv() + self.assertIs(obj8, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 1) + + # obj9 was removed + + obj10 = rch.recv() + self.assertIs(obj10, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 0) + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. diff --git a/Lib/test/test_interpreters/test_lifecycle.py b/Lib/test/test_interpreters/test_lifecycle.py index becf003e2e5f20..ac24f6568acd95 100644 --- a/Lib/test/test_interpreters/test_lifecycle.py +++ b/Lib/test/test_interpreters/test_lifecycle.py @@ -10,7 +10,7 @@ from test.support import import_helper from test.support import os_helper # Raise SkipTest if subinterpreters not supported. -import_helper.import_module('_xxsubinterpreters') +import_helper.import_module('_interpreters') from .utils import TestBase diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 0a1fdb41f73166..18f83d097eb360 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -1,20 +1,27 @@ import importlib +import pickle import threading from textwrap import dedent import unittest -import time -from test.support import import_helper +from test.support import import_helper, Py_DEBUG # Raise SkipTest if subinterpreters not supported. -_queues = import_helper.import_module('_xxinterpqueues') +_queues = import_helper.import_module('_interpqueues') from test.support import interpreters -from test.support.interpreters import queues -from .utils import _run_output, TestBase +from test.support.interpreters import queues, _crossinterp +from .utils import _run_output, TestBase as _TestBase -class TestBase(TestBase): +REPLACE = _crossinterp._UNBOUND_CONSTANT_TO_FLAG[_crossinterp.UNBOUND] + + +def get_num_queues(): + return len(_queues.list_all()) + + +class TestBase(_TestBase): def tearDown(self): - for qid in _queues.list_all(): + for qid, _, _ in _queues.list_all(): try: _queues.destroy(qid) except Exception: @@ -23,9 +30,9 @@ def tearDown(self): class LowLevelTests(TestBase): - # The behaviors in the low-level module is important in as much - # as it is exercised by the high-level module. Therefore the - # most # important testing happens in the high-level tests. + # The behaviors in the low-level module are important in as much + # as they are exercised by the high-level module. Therefore the + # most important testing happens in the high-level tests. # These low-level tests cover corner cases that are not # encountered by the high-level module, thus they # mostly shouldn't matter as much. @@ -34,6 +41,58 @@ def test_highlevel_reloaded(self): # See gh-115490 (https://github.com/python/cpython/issues/115490). importlib.reload(queues) + def test_create_destroy(self): + qid = _queues.create(2, 0, REPLACE) + _queues.destroy(qid) + self.assertEqual(get_num_queues(), 0) + with self.assertRaises(queues.QueueNotFoundError): + _queues.get(qid) + with self.assertRaises(queues.QueueNotFoundError): + _queues.destroy(qid) + + def test_not_destroyed(self): + # It should have cleaned up any remaining queues. + stdout, stderr = self.assert_python_ok( + '-c', + dedent(f""" + import {_queues.__name__} as _queues + _queues.create(2, 0, {REPLACE}) + """), + ) + self.assertEqual(stdout, '') + if Py_DEBUG: + self.assertNotEqual(stderr, '') + else: + self.assertEqual(stderr, '') + + def test_bind_release(self): + with self.subTest('typical'): + qid = _queues.create(2, 0, REPLACE) + _queues.bind(qid) + _queues.release(qid) + self.assertEqual(get_num_queues(), 0) + + with self.subTest('bind too much'): + qid = _queues.create(2, 0, REPLACE) + _queues.bind(qid) + _queues.bind(qid) + _queues.release(qid) + _queues.destroy(qid) + self.assertEqual(get_num_queues(), 0) + + with self.subTest('nested'): + qid = _queues.create(2, 0, REPLACE) + _queues.bind(qid) + _queues.bind(qid) + _queues.release(qid) + _queues.release(qid) + self.assertEqual(get_num_queues(), 0) + + with self.subTest('release without binding'): + qid = _queues.create(2, 0, REPLACE) + with self.assertRaises(queues.QueueError): + _queues.release(qid) + class QueueTests(TestBase): @@ -127,6 +186,12 @@ def test_equality(self): self.assertEqual(queue1, queue1) self.assertNotEqual(queue1, queue2) + def test_pickle(self): + queue = queues.create() + data = pickle.dumps(queue) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, queue) + class TestQueueOps(TestBase): @@ -364,26 +429,206 @@ def test_put_get_different_interpreters(self): self.assertNotEqual(id(obj2), int(out)) def test_put_cleared_with_subinterpreter(self): - interp = interpreters.create() - queue = queues.create() - - out = _run_output( - interp, - dedent(f""" + def common(queue, unbound=None, presize=0): + if not unbound: + extraargs = '' + elif unbound is queues.UNBOUND: + extraargs = ', unbound=queues.UNBOUND' + elif unbound is queues.UNBOUND_ERROR: + extraargs = ', unbound=queues.UNBOUND_ERROR' + elif unbound is queues.UNBOUND_REMOVE: + extraargs = ', unbound=queues.UNBOUND_REMOVE' + else: + raise NotImplementedError(repr(unbound)) + interp = interpreters.create() + + _run_output(interp, dedent(f""" from test.support.interpreters import queues queue = queues.Queue({queue.id}) obj1 = b'spam' obj2 = b'eggs' - queue.put(obj1, syncobj=True) - queue.put(obj2, syncobj=True) + queue.put(obj1, syncobj=True{extraargs}) + queue.put(obj2, syncobj=True{extraargs}) """)) - self.assertEqual(queue.qsize(), 2) + self.assertEqual(queue.qsize(), presize + 2) + + if presize == 0: + obj1 = queue.get() + self.assertEqual(obj1, b'spam') + self.assertEqual(queue.qsize(), presize + 1) + + return interp + + with self.subTest('default'): # UNBOUND + queue = queues.create() + interp = common(queue) + del interp + obj1 = queue.get() + self.assertIs(obj1, queues.UNBOUND) + self.assertEqual(queue.qsize(), 0) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + with self.subTest('UNBOUND'): + queue = queues.create() + interp = common(queue, queues.UNBOUND) + del interp + obj1 = queue.get() + self.assertIs(obj1, queues.UNBOUND) + self.assertEqual(queue.qsize(), 0) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + with self.subTest('UNBOUND_ERROR'): + queue = queues.create() + interp = common(queue, queues.UNBOUND_ERROR) + + del interp + self.assertEqual(queue.qsize(), 1) + with self.assertRaises(queues.ItemInterpreterDestroyed): + queue.get() + + self.assertEqual(queue.qsize(), 0) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + with self.subTest('UNBOUND_REMOVE'): + queue = queues.create() + + interp = common(queue, queues.UNBOUND_REMOVE) + del interp + self.assertEqual(queue.qsize(), 0) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + queue.put(b'ham', unbound=queues.UNBOUND_REMOVE) + self.assertEqual(queue.qsize(), 1) + interp = common(queue, queues.UNBOUND_REMOVE, 1) + self.assertEqual(queue.qsize(), 3) + queue.put(42, unbound=queues.UNBOUND_REMOVE) + self.assertEqual(queue.qsize(), 4) + del interp + self.assertEqual(queue.qsize(), 2) + obj1 = queue.get() + obj2 = queue.get() + self.assertEqual(obj1, b'ham') + self.assertEqual(obj2, 42) + self.assertEqual(queue.qsize(), 0) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + def test_put_cleared_with_subinterpreter_mixed(self): + queue = queues.create() + interp = interpreters.create() + _run_output(interp, dedent(f""" + from test.support.interpreters import queues + queue = queues.Queue({queue.id}) + queue.put(1, syncobj=True, unbound=queues.UNBOUND) + queue.put(2, syncobj=True, unbound=queues.UNBOUND_ERROR) + queue.put(3, syncobj=True) + queue.put(4, syncobj=True, unbound=queues.UNBOUND_REMOVE) + queue.put(5, syncobj=True, unbound=queues.UNBOUND) + """)) + self.assertEqual(queue.qsize(), 5) + + del interp + self.assertEqual(queue.qsize(), 4) obj1 = queue.get() - self.assertEqual(obj1, b'spam') + self.assertIs(obj1, queues.UNBOUND) + self.assertEqual(queue.qsize(), 3) + + with self.assertRaises(queues.ItemInterpreterDestroyed): + queue.get() + self.assertEqual(queue.qsize(), 2) + + obj2 = queue.get() + self.assertIs(obj2, queues.UNBOUND) self.assertEqual(queue.qsize(), 1) - del interp + obj3 = queue.get() + self.assertIs(obj3, queues.UNBOUND) + self.assertEqual(queue.qsize(), 0) + + def test_put_cleared_with_subinterpreter_multiple(self): + queue = queues.create() + interp1 = interpreters.create() + interp2 = interpreters.create() + + queue.put(1, syncobj=True) + _run_output(interp1, dedent(f""" + from test.support.interpreters import queues + queue = queues.Queue({queue.id}) + obj1 = queue.get() + queue.put(2, syncobj=True, unbound=queues.UNBOUND) + queue.put(obj1, syncobj=True, unbound=queues.UNBOUND_REMOVE) + """)) + _run_output(interp2, dedent(f""" + from test.support.interpreters import queues + queue = queues.Queue({queue.id}) + obj2 = queue.get() + obj1 = queue.get() + """)) + self.assertEqual(queue.qsize(), 0) + queue.put(3) + _run_output(interp1, dedent(""" + queue.put(4, syncobj=True, unbound=queues.UNBOUND) + # interp closed here + queue.put(5, syncobj=True, unbound=queues.UNBOUND_REMOVE) + queue.put(6, syncobj=True, unbound=queues.UNBOUND) + """)) + _run_output(interp2, dedent(""" + queue.put(7, syncobj=True, unbound=queues.UNBOUND_ERROR) + # interp closed here + queue.put(obj1, syncobj=True, unbound=queues.UNBOUND_ERROR) + queue.put(obj2, syncobj=True, unbound=queues.UNBOUND_REMOVE) + queue.put(8, syncobj=True, unbound=queues.UNBOUND) + """)) + _run_output(interp1, dedent(""" + queue.put(9, syncobj=True, unbound=queues.UNBOUND_REMOVE) + queue.put(10, syncobj=True, unbound=queues.UNBOUND) + """)) + self.assertEqual(queue.qsize(), 10) + + obj3 = queue.get() + self.assertEqual(obj3, 3) + self.assertEqual(queue.qsize(), 9) + + obj4 = queue.get() + self.assertEqual(obj4, 4) + self.assertEqual(queue.qsize(), 8) + + del interp1 + self.assertEqual(queue.qsize(), 6) + + # obj5 was removed + + obj6 = queue.get() + self.assertIs(obj6, queues.UNBOUND) + self.assertEqual(queue.qsize(), 5) + + obj7 = queue.get() + self.assertEqual(obj7, 7) + self.assertEqual(queue.qsize(), 4) + + del interp2 + self.assertEqual(queue.qsize(), 3) + + # obj1 + with self.assertRaises(queues.ItemInterpreterDestroyed): + queue.get() + self.assertEqual(queue.qsize(), 2) + + # obj2 was removed + + obj8 = queue.get() + self.assertIs(obj8, queues.UNBOUND) + self.assertEqual(queue.qsize(), 1) + + # obj9 was removed + + obj10 = queue.get() + self.assertIs(obj10, queues.UNBOUND) self.assertEqual(queue.qsize(), 0) def test_put_get_different_threads(self): diff --git a/Lib/test/test_interpreters/test_stress.py b/Lib/test/test_interpreters/test_stress.py index 3cc570b3bf7128..56bfc1721992c8 100644 --- a/Lib/test/test_interpreters/test_stress.py +++ b/Lib/test/test_interpreters/test_stress.py @@ -5,7 +5,7 @@ from test.support import import_helper from test.support import threading_helper # Raise SkipTest if subinterpreters not supported. -import_helper.import_module('_xxsubinterpreters') +import_helper.import_module('_interpreters') from test.support import interpreters from .utils import TestBase @@ -23,6 +23,7 @@ def test_create_many_sequential(self): alive.append(interp) @support.requires_resource('cpu') + @threading_helper.requires_working_threading() def test_create_many_threaded(self): alive = [] def task(): @@ -32,6 +33,35 @@ def task(): with threading_helper.start_threads(threads): pass + @support.requires_resource('cpu') + @threading_helper.requires_working_threading() + def test_many_threads_running_interp_in_other_interp(self): + interp = interpreters.create() + + script = f"""if True: + import _interpreters + _interpreters.run_string({interp.id}, '1') + """ + + def run(): + interp = interpreters.create() + alreadyrunning = (f'{interpreters.InterpreterError}: ' + 'interpreter already running') + success = False + while not success: + try: + interp.exec(script) + except interpreters.ExecutionFailed as exc: + if exc.excinfo.msg != 'interpreter already running': + raise # re-raise + assert exc.excinfo.type.__name__ == 'InterpreterError' + else: + success = True + + threads = (threading.Thread(target=run) for _ in range(200)) + with threading_helper.start_threads(threads): + pass + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index 973d05d4f96dcb..3cab76d0f279e0 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -1,6 +1,9 @@ +from collections import namedtuple import contextlib +import json import os import os.path +#import select import subprocess import sys import tempfile @@ -8,23 +11,334 @@ import threading import types import unittest +import warnings from test import support -from test.support import os_helper +# We would use test.support.import_helper.import_module(), +# but the indirect import of test.support.os_helper causes refleaks. +try: + import _interpreters +except ImportError as exc: + raise unittest.SkipTest(str(exc)) from test.support import interpreters -def _captured_script(script): - r, w = os.pipe() - indented = script.replace('\n', '\n ') - wrapped = dedent(f""" - import contextlib - with open({w}, 'w', encoding='utf-8') as spipe: - with contextlib.redirect_stdout(spipe): +try: + import _testinternalcapi + import _testcapi +except ImportError: + _testinternalcapi = None + _testcapi = None + +def requires_test_modules(func): + return unittest.skipIf(_testinternalcapi is None, "test requires _testinternalcapi module")(func) + + +def _dump_script(text): + lines = text.splitlines() + print() + print('-' * 20) + for i, line in enumerate(lines, 1): + print(f' {i:>{len(str(len(lines)))}} {line}') + print('-' * 20) + + +def _close_file(file): + try: + if hasattr(file, 'close'): + file.close() + else: + os.close(file) + except OSError as exc: + if exc.errno != 9: + raise # re-raise + # It was closed already. + + +def pack_exception(exc=None): + captured = _interpreters.capture_exception(exc) + data = dict(captured.__dict__) + data['type'] = dict(captured.type.__dict__) + return json.dumps(data) + + +def unpack_exception(packed): + try: + data = json.loads(packed) + except json.decoder.JSONDecodeError: + warnings.warn('incomplete exception data', RuntimeWarning) + print(packed if isinstance(packed, str) else packed.decode('utf-8')) + return None + exc = types.SimpleNamespace(**data) + exc.type = types.SimpleNamespace(**exc.type) + return exc; + + +class CapturingResults: + + STDIO = dedent("""\ + with open({w_pipe}, 'wb', buffering=0) as _spipe_{stream}: + _captured_std{stream} = io.StringIO() + with contextlib.redirect_std{stream}(_captured_std{stream}): + ######################### + # begin wrapped script + {indented} - """) - return wrapped, open(r, encoding='utf-8') + + # end wrapped script + ######################### + text = _captured_std{stream}.getvalue() + _spipe_{stream}.write(text.encode('utf-8')) + """)[:-1] + EXC = dedent("""\ + with open({w_pipe}, 'wb', buffering=0) as _spipe_exc: + try: + ######################### + # begin wrapped script + + {indented} + + # end wrapped script + ######################### + except Exception as exc: + text = _interp_utils.pack_exception(exc) + _spipe_exc.write(text.encode('utf-8')) + """)[:-1] + + @classmethod + def wrap_script(cls, script, *, stdout=True, stderr=False, exc=False): + script = dedent(script).strip(os.linesep) + imports = [ + f'import {__name__} as _interp_utils', + ] + wrapped = script + + # Handle exc. + if exc: + exc = os.pipe() + r_exc, w_exc = exc + indented = wrapped.replace('\n', '\n ') + wrapped = cls.EXC.format( + w_pipe=w_exc, + indented=indented, + ) + else: + exc = None + + # Handle stdout. + if stdout: + imports.extend([ + 'import contextlib, io', + ]) + stdout = os.pipe() + r_out, w_out = stdout + indented = wrapped.replace('\n', '\n ') + wrapped = cls.STDIO.format( + w_pipe=w_out, + indented=indented, + stream='out', + ) + else: + stdout = None + + # Handle stderr. + if stderr == 'stdout': + stderr = None + elif stderr: + if not stdout: + imports.extend([ + 'import contextlib, io', + ]) + stderr = os.pipe() + r_err, w_err = stderr + indented = wrapped.replace('\n', '\n ') + wrapped = cls.STDIO.format( + w_pipe=w_err, + indented=indented, + stream='err', + ) + else: + stderr = None + + if wrapped == script: + raise NotImplementedError + else: + for line in imports: + wrapped = f'{line}{os.linesep}{wrapped}' + + results = cls(stdout, stderr, exc) + return wrapped, results + + def __init__(self, out, err, exc): + self._rf_out = None + self._rf_err = None + self._rf_exc = None + self._w_out = None + self._w_err = None + self._w_exc = None + + if out is not None: + r_out, w_out = out + self._rf_out = open(r_out, 'rb', buffering=0) + self._w_out = w_out + + if err is not None: + r_err, w_err = err + self._rf_err = open(r_err, 'rb', buffering=0) + self._w_err = w_err + + if exc is not None: + r_exc, w_exc = exc + self._rf_exc = open(r_exc, 'rb', buffering=0) + self._w_exc = w_exc + + self._buf_out = b'' + self._buf_err = b'' + self._buf_exc = b'' + self._exc = None + + self._closed = False + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + @property + def closed(self): + return self._closed + + def close(self): + if self._closed: + return + self._closed = True + + if self._w_out is not None: + _close_file(self._w_out) + self._w_out = None + if self._w_err is not None: + _close_file(self._w_err) + self._w_err = None + if self._w_exc is not None: + _close_file(self._w_exc) + self._w_exc = None + + self._capture() + + if self._rf_out is not None: + _close_file(self._rf_out) + self._rf_out = None + if self._rf_err is not None: + _close_file(self._rf_err) + self._rf_err = None + if self._rf_exc is not None: + _close_file(self._rf_exc) + self._rf_exc = None + + def _capture(self): + # Ideally this is called only after the script finishes + # (and thus has closed the write end of the pipe. + if self._rf_out is not None: + chunk = self._rf_out.read(100) + while chunk: + self._buf_out += chunk + chunk = self._rf_out.read(100) + if self._rf_err is not None: + chunk = self._rf_err.read(100) + while chunk: + self._buf_err += chunk + chunk = self._rf_err.read(100) + if self._rf_exc is not None: + chunk = self._rf_exc.read(100) + while chunk: + self._buf_exc += chunk + chunk = self._rf_exc.read(100) + + def _unpack_stdout(self): + return self._buf_out.decode('utf-8') + + def _unpack_stderr(self): + return self._buf_err.decode('utf-8') + + def _unpack_exc(self): + if self._exc is not None: + return self._exc + if not self._buf_exc: + return None + self._exc = unpack_exception(self._buf_exc) + return self._exc + + def stdout(self): + if self.closed: + return self.final().stdout + self._capture() + return self._unpack_stdout() + + def stderr(self): + if self.closed: + return self.final().stderr + self._capture() + return self._unpack_stderr() + + def exc(self): + if self.closed: + return self.final().exc + self._capture() + return self._unpack_exc() + + def final(self, *, force=False): + try: + return self._final + except AttributeError: + if not self._closed: + if not force: + raise Exception('no final results available yet') + else: + return CapturedResults.Proxy(self) + self._final = CapturedResults( + self._unpack_stdout(), + self._unpack_stderr(), + self._unpack_exc(), + ) + return self._final + + +class CapturedResults(namedtuple('CapturedResults', 'stdout stderr exc')): + + class Proxy: + def __init__(self, capturing): + self._capturing = capturing + def _finish(self): + if self._capturing is None: + return + self._final = self._capturing.final() + self._capturing = None + def __iter__(self): + self._finish() + yield from self._final + def __len__(self): + self._finish() + return len(self._final) + def __getattr__(self, name): + self._finish() + if name.startswith('_'): + raise AttributeError(name) + return getattr(self._final, name) + + def raise_if_failed(self): + if self.exc is not None: + raise interpreters.ExecutionFailed(self.exc) + + +def _captured_script(script, *, stdout=True, stderr=False, exc=False): + return CapturingResults.wrap_script( + script, + stdout=stdout, + stderr=stderr, + exc=exc, + ) def clean_up_interpreters(): @@ -33,17 +347,17 @@ def clean_up_interpreters(): continue try: interp.close() - except RuntimeError: + except _interpreters.InterpreterError: pass # already destroyed def _run_output(interp, request, init=None): - script, rpipe = _captured_script(request) - with rpipe: + script, results = _captured_script(request) + with results: if init: interp.prepare_main(init) interp.exec(script) - return rpipe.read() + return results.stdout() @contextlib.contextmanager @@ -68,6 +382,9 @@ def run(): class TestBase(unittest.TestCase): + def tearDown(self): + clean_up_interpreters() + def pipe(self): def ensure_closed(fd): try: @@ -82,6 +399,7 @@ def ensure_closed(fd): def temp_dir(self): tempdir = tempfile.mkdtemp() tempdir = os.path.realpath(tempdir) + from test.support import os_helper self.addCleanup(lambda: os_helper.rmtree(tempdir)) return tempdir @@ -156,5 +474,213 @@ def assert_python_failure(self, *argv): self.assertNotEqual(exitcode, 0) return stdout, stderr - def tearDown(self): - clean_up_interpreters() + def assert_ns_equal(self, ns1, ns2, msg=None): + # This is mostly copied from TestCase.assertDictEqual. + self.assertEqual(type(ns1), type(ns2)) + if ns1 == ns2: + return + + import difflib + import pprint + from unittest.util import _common_shorten_repr + standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2) + diff = ('\n' + '\n'.join(difflib.ndiff( + pprint.pformat(vars(ns1)).splitlines(), + pprint.pformat(vars(ns2)).splitlines()))) + diff = f'namespace({diff})' + standardMsg = self._truncateMessage(standardMsg, diff) + self.fail(self._formatMessage(msg, standardMsg)) + + def _run_string(self, interp, script): + wrapped, results = _captured_script(script, exc=False) + #_dump_script(wrapped) + with results: + if isinstance(interp, interpreters.Interpreter): + interp.exec(script) + else: + err = _interpreters.run_string(interp, wrapped) + if err is not None: + return None, err + return results.stdout(), None + + def run_and_capture(self, interp, script): + text, err = self._run_string(interp, script) + if err is not None: + raise interpreters.ExecutionFailed(err) + else: + return text + + def interp_exists(self, interpid): + try: + _interpreters.whence(interpid) + except _interpreters.InterpreterNotFoundError: + return False + else: + return True + + @requires_test_modules + @contextlib.contextmanager + def interpreter_from_capi(self, config=None, whence=None): + if config is False: + if whence is None: + whence = _interpreters.WHENCE_LEGACY_CAPI + else: + assert whence in (_interpreters.WHENCE_LEGACY_CAPI, + _interpreters.WHENCE_UNKNOWN), repr(whence) + config = None + elif config is True: + config = _interpreters.new_config('default') + elif config is None: + if whence not in ( + _interpreters.WHENCE_LEGACY_CAPI, + _interpreters.WHENCE_UNKNOWN, + ): + config = _interpreters.new_config('legacy') + elif isinstance(config, str): + config = _interpreters.new_config(config) + + if whence is None: + whence = _interpreters.WHENCE_XI + + interpid = _testinternalcapi.create_interpreter(config, whence=whence) + try: + yield interpid + finally: + try: + _testinternalcapi.destroy_interpreter(interpid) + except _interpreters.InterpreterNotFoundError: + pass + + @contextlib.contextmanager + def interpreter_obj_from_capi(self, config='legacy'): + with self.interpreter_from_capi(config) as interpid: + interp = interpreters.Interpreter( + interpid, + _whence=_interpreters.WHENCE_CAPI, + _ownsref=False, + ) + yield interp, interpid + + @contextlib.contextmanager + def capturing(self, script): + wrapped, capturing = _captured_script(script, stdout=True, exc=True) + #_dump_script(wrapped) + with capturing: + yield wrapped, capturing.final(force=True) + + @requires_test_modules + def run_from_capi(self, interpid, script, *, main=False): + with self.capturing(script) as (wrapped, results): + rc = _testinternalcapi.exec_interpreter(interpid, wrapped, main=main) + assert rc == 0, rc + results.raise_if_failed() + return results.stdout + + @contextlib.contextmanager + def _running(self, run_interp, exec_interp): + token = b'\0' + r_in, w_in = self.pipe() + r_out, w_out = self.pipe() + + def close(): + _close_file(r_in) + _close_file(w_in) + _close_file(r_out) + _close_file(w_out) + + # Start running (and wait). + script = dedent(f""" + import os + try: + # handshake + token = os.read({r_in}, 1) + os.write({w_out}, token) + # Wait for the "done" message. + os.read({r_in}, 1) + except BrokenPipeError: + pass + except OSError as exc: + if exc.errno != 9: + raise # re-raise + # It was closed already. + """) + failed = None + def run(): + nonlocal failed + try: + run_interp(script) + except Exception as exc: + failed = exc + close() + t = threading.Thread(target=run) + t.start() + + # handshake + try: + os.write(w_in, token) + token2 = os.read(r_out, 1) + assert token2 == token, (token2, token) + except OSError: + t.join() + if failed is not None: + raise failed + + # CM __exit__() + try: + try: + yield + finally: + # Send "done". + os.write(w_in, b'\0') + finally: + close() + t.join() + if failed is not None: + raise failed + + @contextlib.contextmanager + def running(self, interp): + if isinstance(interp, int): + interpid = interp + def exec_interp(script): + exc = _interpreters.exec(interpid, script) + assert exc is None, exc + run_interp = exec_interp + else: + def run_interp(script): + text = self.run_and_capture(interp, script) + assert text == '', repr(text) + def exec_interp(script): + interp.exec(script) + with self._running(run_interp, exec_interp): + yield + + @requires_test_modules + @contextlib.contextmanager + def running_from_capi(self, interpid, *, main=False): + def run_interp(script): + text = self.run_from_capi(interpid, script, main=main) + assert text == '', repr(text) + def exec_interp(script): + rc = _testinternalcapi.exec_interpreter(interpid, script) + assert rc == 0, rc + with self._running(run_interp, exec_interp): + yield + + @requires_test_modules + def run_temp_from_capi(self, script, config='legacy'): + if config is False: + # Force using Py_NewInterpreter(). + run_in_interp = (lambda s, c: _testcapi.run_in_subinterp(s)) + config = None + else: + run_in_interp = _testinternalcapi.run_in_subinterp_with_config + if config is True: + config = 'default' + if isinstance(config, str): + config = _interpreters.new_config(config) + with self.capturing(script) as (wrapped, results): + rc = run_in_interp(wrapped, config) + assert rc == 0, rc + results.raise_if_failed() + return results.stdout diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 5491c0575dbd3f..e59d3977df4134 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -40,7 +40,7 @@ from test.support.script_helper import ( assert_python_ok, assert_python_failure, run_python_until_end) from test.support import ( - import_helper, is_apple, os_helper, skip_if_sanitizer, threading_helper, warnings_helper + import_helper, is_apple, os_helper, threading_helper, warnings_helper, ) from test.support.os_helper import FakePath @@ -639,11 +639,9 @@ def test_large_file_ops(self): def test_with_open(self): for bufsize in (0, 100): - f = None with self.open(os_helper.TESTFN, "wb", bufsize) as f: f.write(b"xxx") self.assertEqual(f.closed, True) - f = None try: with self.open(os_helper.TESTFN, "wb", bufsize) as f: 1/0 @@ -1150,6 +1148,21 @@ def test_disallow_instantiation(self): _io = self._io support.check_disallow_instantiation(self, _io._BytesIOBuffer) + def test_stringio_setstate(self): + # gh-127182: Calling __setstate__() with invalid arguments must not crash + obj = self._io.StringIO() + with self.assertRaisesRegex( + TypeError, + 'initial_value must be str or None, not int', + ): + obj.__setstate__((1, '', 0, {})) + + obj.__setstate__((None, '', 0, {})) # should not crash + self.assertEqual(obj.getvalue(), '') + + obj.__setstate__(('', '', 0, {})) + self.assertEqual(obj.getvalue(), '') + class PyIOTest(IOTest): pass @@ -1160,7 +1173,7 @@ class APIMismatchTest(unittest.TestCase): def test_RawIOBase_io_in_pyio_match(self): """Test that pyio RawIOBase class has all c RawIOBase methods""" mismatch = support.detect_api_mismatch(pyio.RawIOBase, io.RawIOBase, - ignore=('__weakref__',)) + ignore=('__weakref__', '__static_attributes__')) self.assertEqual(mismatch, set(), msg='Python RawIOBase does not have all C RawIOBase methods') def test_RawIOBase_pyio_in_io_match(self): @@ -1697,19 +1710,6 @@ def test_seek_character_device_file(self): class CBufferedReaderTest(BufferedReaderTest, SizeofTest): tp = io.BufferedReader - @skip_if_sanitizer(memory=True, address=True, thread=True, - reason="sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") - def test_constructor(self): - BufferedReaderTest.test_constructor(self) - # The allocation can succeed on 32-bit builds, e.g. with more - # than 2 GiB RAM and a 64-bit kernel. - if sys.maxsize > 0x7FFFFFFF: - rawio = self.MockRawIO() - bufio = self.tp(rawio) - self.assertRaises((OverflowError, MemoryError, ValueError), - bufio.__init__, rawio, sys.maxsize) - def test_initialization(self): rawio = self.MockRawIO([b"abc"]) bufio = self.tp(rawio) @@ -2065,19 +2065,6 @@ def test_slow_close_from_thread(self): class CBufferedWriterTest(BufferedWriterTest, SizeofTest): tp = io.BufferedWriter - @skip_if_sanitizer(memory=True, address=True, thread=True, - reason="sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") - def test_constructor(self): - BufferedWriterTest.test_constructor(self) - # The allocation can succeed on 32-bit builds, e.g. with more - # than 2 GiB RAM and a 64-bit kernel. - if sys.maxsize > 0x7FFFFFFF: - rawio = self.MockRawIO() - bufio = self.tp(rawio) - self.assertRaises((OverflowError, MemoryError, ValueError), - bufio.__init__, rawio, sys.maxsize) - def test_initialization(self): rawio = self.MockRawIO() bufio = self.tp(rawio) @@ -2587,19 +2574,6 @@ def test_interleaved_readline_write(self): class CBufferedRandomTest(BufferedRandomTest, SizeofTest): tp = io.BufferedRandom - @skip_if_sanitizer(memory=True, address=True, thread=True, - reason="sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") - def test_constructor(self): - BufferedRandomTest.test_constructor(self) - # The allocation can succeed on 32-bit builds, e.g. with more - # than 2 GiB RAM and a 64-bit kernel. - if sys.maxsize > 0x7FFFFFFF: - rawio = self.MockRawIO() - bufio = self.tp(rawio) - self.assertRaises((OverflowError, MemoryError, ValueError), - bufio.__init__, rawio, sys.maxsize) - def test_garbage_collection(self): CBufferedReaderTest.test_garbage_collection(self) CBufferedWriterTest.test_garbage_collection(self) @@ -2918,14 +2892,12 @@ def test_reconfigure_line_buffering(self): @unittest.skipIf(sys.flags.utf8_mode, "utf-8 mode is enabled") def test_default_encoding(self): - old_environ = dict(os.environ) - try: + with os_helper.EnvironmentVarGuard() as env: # try to get a user preferred encoding different than the current # locale encoding to check that TextIOWrapper() uses the current # locale encoding and not the user preferred encoding for key in ('LC_ALL', 'LANG', 'LC_CTYPE'): - if key in os.environ: - del os.environ[key] + env.unset(key) current_locale_encoding = locale.getencoding() b = self.BytesIO() @@ -2933,9 +2905,6 @@ def test_default_encoding(self): warnings.simplefilter("ignore", EncodingWarning) t = self.TextIOWrapper(b) self.assertEqual(t.encoding, current_locale_encoding) - finally: - os.environ.clear() - os.environ.update(old_environ) def test_encoding(self): # Check the encoding attribute is always set, and valid @@ -3958,6 +3927,23 @@ def test_issue35928(self): f.write(res) self.assertEqual(res + f.readline(), 'foo\nbar\n') + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + @unittest.skipIf(support.is_emscripten, "Fixed in next Emscripten release after 4.0.1") + def test_read_non_blocking(self): + import os + r, w = os.pipe() + try: + os.set_blocking(r, False) + with self.io.open(r, 'rt') as textfile: + r = None + # Nothing has been written so a non-blocking read raises a BlockingIOError exception. + with self.assertRaises(BlockingIOError): + textfile.read() + finally: + if r is not None: + os.close(r) + os.close(w) + class MemviewBytesIO(io.BytesIO): '''A BytesIO object whose read method returns memoryviews @@ -4055,6 +4041,28 @@ def write(self, data): t.write("x"*chunk_size) self.assertEqual([b"abcdef", b"ghi", b"x"*chunk_size], buf._write_stack) + def test_issue119506(self): + chunk_size = 8192 + + class MockIO(self.MockRawIO): + written = False + def write(self, data): + if not self.written: + self.written = True + t.write("middle") + return super().write(data) + + buf = MockIO() + t = self.TextIOWrapper(buf) + t.write("abc") + t.write("def") + # writing data which size >= chunk_size cause flushing buffer before write. + t.write("g" * chunk_size) + t.flush() + + self.assertEqual([b"abcdef", b"middle", b"g"*chunk_size], + buf._write_stack) + class PyTextIOWrapperTest(TextIOWrapperTest): io = pyio diff --git a/Lib/test/test_ioctl.py b/Lib/test/test_ioctl.py index 7b7067eb7b61d4..04934dfa16a5f0 100644 --- a/Lib/test/test_ioctl.py +++ b/Lib/test/test_ioctl.py @@ -66,23 +66,15 @@ def test_ioctl_mutate_2048(self): # Test with a larger buffer, just for the record. self._check_ioctl_mutate_len(2048) - def test_ioctl_signed_unsigned_code_param(self): - if not pty: - raise unittest.SkipTest('pty module required') + @unittest.skipIf(pty is None, 'pty module required') + def test_ioctl_set_window_size(self): mfd, sfd = pty.openpty() try: - if termios.TIOCSWINSZ < 0: - set_winsz_opcode_maybe_neg = termios.TIOCSWINSZ - set_winsz_opcode_pos = termios.TIOCSWINSZ & 0xffffffff - else: - set_winsz_opcode_pos = termios.TIOCSWINSZ - set_winsz_opcode_maybe_neg, = struct.unpack("i", - struct.pack("I", termios.TIOCSWINSZ)) - - our_winsz = struct.pack("HHHH",80,25,0,0) - # test both with a positive and potentially negative ioctl code - new_winsz = fcntl.ioctl(mfd, set_winsz_opcode_pos, our_winsz) - new_winsz = fcntl.ioctl(mfd, set_winsz_opcode_maybe_neg, our_winsz) + # (rows, columns, xpixel, ypixel) + our_winsz = struct.pack("HHHH", 20, 40, 0, 0) + result = fcntl.ioctl(mfd, termios.TIOCSWINSZ, our_winsz) + new_winsz = struct.unpack("HHHH", result) + self.assertEqual(new_winsz[:2], (20, 40)) finally: os.close(mfd) os.close(sfd) diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index b4952acc2b61b1..b1ac2b94f41b38 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -2189,11 +2189,17 @@ def testIPv6AddressTooLarge(self): ipaddress.ip_address('FFFF::c000:201%scope')) def testIPVersion(self): + self.assertEqual(ipaddress.IPv4Address.version, 4) + self.assertEqual(ipaddress.IPv6Address.version, 6) + self.assertEqual(self.ipv4_address.version, 4) self.assertEqual(self.ipv6_address.version, 6) self.assertEqual(self.ipv6_scoped_address.version, 6) def testMaxPrefixLength(self): + self.assertEqual(ipaddress.IPv4Address.max_prefixlen, 32) + self.assertEqual(ipaddress.IPv6Address.max_prefixlen, 128) + self.assertEqual(self.ipv4_interface.max_prefixlen, 32) self.assertEqual(self.ipv6_interface.max_prefixlen, 128) self.assertEqual(self.ipv6_scoped_interface.max_prefixlen, 128) @@ -2288,6 +2294,10 @@ def testReservedIpv4(self): self.assertEqual(True, ipaddress.ip_address( '172.31.255.255').is_private) self.assertEqual(False, ipaddress.ip_address('172.32.0.0').is_private) + self.assertFalse(ipaddress.ip_address('192.0.0.0').is_global) + self.assertTrue(ipaddress.ip_address('192.0.0.9').is_global) + self.assertTrue(ipaddress.ip_address('192.0.0.10').is_global) + self.assertFalse(ipaddress.ip_address('192.0.0.255').is_global) self.assertEqual(True, ipaddress.ip_address('169.254.100.200').is_link_local) @@ -2313,6 +2323,7 @@ def testPrivateNetworks(self): self.assertEqual(True, ipaddress.ip_network("169.254.0.0/16").is_private) self.assertEqual(True, ipaddress.ip_network("172.16.0.0/12").is_private) self.assertEqual(True, ipaddress.ip_network("192.0.0.0/29").is_private) + self.assertEqual(False, ipaddress.ip_network("192.0.0.9/32").is_private) self.assertEqual(True, ipaddress.ip_network("192.0.0.170/31").is_private) self.assertEqual(True, ipaddress.ip_network("192.0.2.0/24").is_private) self.assertEqual(True, ipaddress.ip_network("192.168.0.0/16").is_private) @@ -2329,8 +2340,8 @@ def testPrivateNetworks(self): self.assertEqual(True, ipaddress.ip_network("::/128").is_private) self.assertEqual(True, ipaddress.ip_network("::ffff:0:0/96").is_private) self.assertEqual(True, ipaddress.ip_network("100::/64").is_private) - self.assertEqual(True, ipaddress.ip_network("2001::/23").is_private) self.assertEqual(True, ipaddress.ip_network("2001:2::/48").is_private) + self.assertEqual(False, ipaddress.ip_network("2001:3::/48").is_private) self.assertEqual(True, ipaddress.ip_network("2001:db8::/32").is_private) self.assertEqual(True, ipaddress.ip_network("2001:10::/28").is_private) self.assertEqual(True, ipaddress.ip_network("fc00::/7").is_private) @@ -2409,6 +2420,22 @@ def testReservedIpv6(self): self.assertEqual(True, ipaddress.ip_address('0::0').is_unspecified) self.assertEqual(False, ipaddress.ip_address('::1').is_unspecified) + self.assertFalse(ipaddress.ip_address('64:ff9b:1::').is_global) + self.assertFalse(ipaddress.ip_address('2001::').is_global) + self.assertTrue(ipaddress.ip_address('2001:1::1').is_global) + self.assertTrue(ipaddress.ip_address('2001:1::2').is_global) + self.assertFalse(ipaddress.ip_address('2001:2::').is_global) + self.assertTrue(ipaddress.ip_address('2001:3::').is_global) + self.assertFalse(ipaddress.ip_address('2001:4::').is_global) + self.assertTrue(ipaddress.ip_address('2001:4:112::').is_global) + self.assertFalse(ipaddress.ip_address('2001:10::').is_global) + self.assertTrue(ipaddress.ip_address('2001:20::').is_global) + self.assertTrue(ipaddress.ip_address('2001:30::').is_global) + self.assertFalse(ipaddress.ip_address('2001:40::').is_global) + self.assertFalse(ipaddress.ip_address('2002::').is_global) + # gh-124217: conform with RFC 9637 + self.assertFalse(ipaddress.ip_address('3fff::').is_global) + # some generic IETF reserved addresses self.assertEqual(True, ipaddress.ip_address('100::').is_reserved) self.assertEqual(True, ipaddress.ip_network('4000::1/128').is_reserved) @@ -2421,12 +2448,52 @@ def testIpv4Mapped(self): self.assertEqual(ipaddress.ip_address('::ffff:c0a8:101').ipv4_mapped, ipaddress.ip_address('192.168.1.1')) + def testIpv4MappedProperties(self): + # Test that an IPv4 mapped IPv6 address has + # the same properties as an IPv4 address. + for addr4 in ( + "178.62.3.251", # global + "169.254.169.254", # link local + "127.0.0.1", # loopback + "224.0.0.1", # multicast + "192.168.0.1", # private + "0.0.0.0", # unspecified + "100.64.0.1", # public and not global + ): + with self.subTest(addr4): + ipv4 = ipaddress.IPv4Address(addr4) + ipv6 = ipaddress.IPv6Address(f"::ffff:{addr4}") + + self.assertEqual(ipv4.is_global, ipv6.is_global) + self.assertEqual(ipv4.is_private, ipv6.is_private) + self.assertEqual(ipv4.is_reserved, ipv6.is_reserved) + self.assertEqual(ipv4.is_multicast, ipv6.is_multicast) + self.assertEqual(ipv4.is_unspecified, ipv6.is_unspecified) + self.assertEqual(ipv4.is_link_local, ipv6.is_link_local) + self.assertEqual(ipv4.is_loopback, ipv6.is_loopback) + def testIpv4MappedPrivateCheck(self): self.assertEqual( True, ipaddress.ip_address('::ffff:192.168.1.1').is_private) self.assertEqual( False, ipaddress.ip_address('::ffff:172.32.0.0').is_private) + def testIpv4MappedLoopbackCheck(self): + # test networks + self.assertEqual(True, ipaddress.ip_network( + '::ffff:127.100.200.254/128').is_loopback) + self.assertEqual(True, ipaddress.ip_network( + '::ffff:127.42.0.0/112').is_loopback) + self.assertEqual(False, ipaddress.ip_network( + '::ffff:128.0.0.0').is_loopback) + # test addresses + self.assertEqual(True, ipaddress.ip_address( + '::ffff:127.100.200.254').is_loopback) + self.assertEqual(True, ipaddress.ip_address( + '::ffff:127.42.0.0').is_loopback) + self.assertEqual(False, ipaddress.ip_address( + '::ffff:128.0.0.0').is_loopback) + def testAddrExclude(self): addr1 = ipaddress.ip_network('10.1.1.0/24') addr2 = ipaddress.ip_network('10.1.1.0/26') @@ -2550,12 +2617,42 @@ def testExplodeShortHandIpStr(self): self.assertEqual('192.168.178.1', addr4.exploded) def testReversePointer(self): - addr1 = ipaddress.IPv4Address('127.0.0.1') - addr2 = ipaddress.IPv6Address('2001:db8::1') - self.assertEqual('1.0.0.127.in-addr.arpa', addr1.reverse_pointer) - self.assertEqual('1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.' + - 'b.d.0.1.0.0.2.ip6.arpa', - addr2.reverse_pointer) + for addr_v4, expected in [ + ('127.0.0.1', '1.0.0.127.in-addr.arpa'), + # test vector: https://www.rfc-editor.org/rfc/rfc1035, §3.5 + ('10.2.0.52', '52.0.2.10.in-addr.arpa'), + ]: + with self.subTest('ipv4_reverse_pointer', addr=addr_v4): + addr = ipaddress.IPv4Address(addr_v4) + self.assertEqual(addr.reverse_pointer, expected) + + for addr_v6, expected in [ + ( + '2001:db8::1', ( + '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.' + '0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.' + 'ip6.arpa' + ) + ), + ( + '::FFFF:192.168.1.35', ( + '3.2.1.0.8.a.0.c.f.f.f.f.0.0.0.0.' + '0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.' + 'ip6.arpa' + ) + ), + # test vector: https://www.rfc-editor.org/rfc/rfc3596, §2.5 + ( + '4321:0:1:2:3:4:567:89ab', ( + 'b.a.9.8.7.6.5.0.4.0.0.0.3.0.0.0.' + '2.0.0.0.1.0.0.0.0.0.0.0.1.2.3.4.' + 'ip6.arpa' + ) + ) + ]: + with self.subTest('ipv6_reverse_pointer', addr=addr_v6): + addr = ipaddress.IPv6Address(addr_v6) + self.assertEqual(addr.reverse_pointer, expected) def testIntRepresentation(self): self.assertEqual(16909060, int(self.ipv4_address)) diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index 7f759fb3317146..abc75c82375d98 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -263,12 +263,14 @@ def test_subclass_tuple(self): self.assertEqual(True, issubclass(int, (int, (float, int)))) self.assertEqual(True, issubclass(str, (str, (Child, str)))) + @support.skip_emscripten_stack_overflow() def test_subclass_recursion_limit(self): # make sure that issubclass raises RecursionError before the C stack is # blown with support.infinite_recursion(): self.assertRaises(RecursionError, blowstack, issubclass, str, str) + @support.skip_emscripten_stack_overflow() def test_isinstance_recursion_limit(self): # make sure that issubclass raises RecursionError before the C stack is # blown @@ -315,6 +317,7 @@ def __bases__(self): self.assertRaises(RecursionError, issubclass, int, X()) self.assertRaises(RecursionError, isinstance, 1, X()) + @support.skip_emscripten_stack_overflow() def test_infinite_recursion_via_bases_tuple(self): """Regression test for bpo-30570.""" class Failure(object): @@ -352,7 +355,7 @@ def blowstack(fxn, arg, compare_to): # Make sure that calling isinstance with a deeply nested tuple for its # argument will raise RecursionError eventually. tuple_arg = (compare_to,) - for cnt in range(support.EXCEEDS_RECURSION_LIMIT): + for cnt in range(support.exceeds_recursion_limit()): tuple_arg = (tuple_arg,) fxn(arg, tuple_arg) diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 9606d5beab71cb..1b9f3cf76240ad 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -5,11 +5,13 @@ from test.support import cpython_only from test.support.os_helper import TESTFN, unlink from test.support import check_free_after_iterating, ALWAYS_EQ, NEVER_EQ +from test.support import BrokenIter import pickle import collections.abc import functools import contextlib import builtins +import traceback # Test result of triple loop (too big to inline) TRIPLETS = [(0, 0, 0), (0, 0, 1), (0, 0, 2), @@ -1143,6 +1145,46 @@ def test_error_iter(self): self.assertRaises(TypeError, iter, typ()) self.assertRaises(ZeroDivisionError, iter, BadIterableClass()) + def test_exception_locations(self): + # The location of an exception raised from __init__ or + # __next__ should should be the iterator expression + + def init_raises(): + try: + for x in BrokenIter(init_raises=True): + pass + except Exception as e: + return e + + def next_raises(): + try: + for x in BrokenIter(next_raises=True): + pass + except Exception as e: + return e + + def iter_raises(): + try: + for x in BrokenIter(iter_raises=True): + pass + except Exception as e: + return e + + for func, expected in [(init_raises, "BrokenIter(init_raises=True)"), + (next_raises, "BrokenIter(next_raises=True)"), + (iter_raises, "BrokenIter(iter_raises=True)"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 9af0730ea98004..f0fd1d28f56f55 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1,7 +1,7 @@ import doctest import unittest from test import support -from test.support import threading_helper +from test.support import threading_helper, script_helper from itertools import * import weakref from decimal import Decimal @@ -15,26 +15,6 @@ import struct import threading import gc -import warnings - -def pickle_deprecated(testfunc): - """ Run the test three times. - First, verify that a Deprecation Warning is raised. - Second, run normally but with DeprecationWarnings temporarily disabled. - Third, run with warnings promoted to errors. - """ - def inner(self): - with self.assertWarns(DeprecationWarning): - testfunc(self) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=DeprecationWarning) - testfunc(self) - with warnings.catch_warnings(): - warnings.simplefilter("error", category=DeprecationWarning) - with self.assertRaises((DeprecationWarning, AssertionError, SystemError)): - testfunc(self) - - return inner maxsize = support.MAX_Py_ssize_t minsize = -maxsize-1 @@ -144,7 +124,6 @@ def expand(it, i=0): c = expand(compare[took:]) self.assertEqual(a, c); - @pickle_deprecated def test_accumulate(self): self.assertEqual(list(accumulate(range(10))), # one positional arg [0, 1, 3, 6, 10, 15, 21, 28, 36, 45]) @@ -171,9 +150,6 @@ def test_accumulate(self): [2, 16, 144, 720, 5040, 0, 0, 0, 0, 0]) with self.assertRaises(TypeError): list(accumulate(s, chr)) # unary-operation - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, accumulate(range(10))) # test pickling - self.pickletest(proto, accumulate(range(10), initial=7)) self.assertEqual(list(accumulate([10, 5, 1], initial=None)), [10, 15, 16]) self.assertEqual(list(accumulate([10, 5, 1], initial=100)), [100, 110, 115, 116]) self.assertEqual(list(accumulate([], initial=100)), [100]) @@ -245,58 +221,12 @@ def test_chain_from_iterable(self): self.assertRaises(TypeError, list, chain.from_iterable([2, 3])) self.assertEqual(list(islice(chain.from_iterable(repeat(range(5))), 2)), [0, 1]) - @pickle_deprecated - def test_chain_reducible(self): - for oper in [copy.deepcopy] + picklecopiers: - it = chain('abc', 'def') - self.assertEqual(list(oper(it)), list('abcdef')) - self.assertEqual(next(it), 'a') - self.assertEqual(list(oper(it)), list('bcdef')) - - self.assertEqual(list(oper(chain(''))), []) - self.assertEqual(take(4, oper(chain('abc', 'def'))), list('abcd')) - self.assertRaises(TypeError, list, oper(chain(2, 3))) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, chain('abc', 'def'), compare=list('abcdef')) - - @pickle_deprecated - def test_chain_setstate(self): - self.assertRaises(TypeError, chain().__setstate__, ()) - self.assertRaises(TypeError, chain().__setstate__, []) - self.assertRaises(TypeError, chain().__setstate__, 0) - self.assertRaises(TypeError, chain().__setstate__, ([],)) - self.assertRaises(TypeError, chain().__setstate__, (iter([]), [])) - it = chain() - it.__setstate__((iter(['abc', 'def']),)) - self.assertEqual(list(it), ['a', 'b', 'c', 'd', 'e', 'f']) - it = chain() - it.__setstate__((iter(['abc', 'def']), iter(['ghi']))) - self.assertEqual(list(it), ['ghi', 'a', 'b', 'c', 'd', 'e', 'f']) - - @pickle_deprecated def test_combinations(self): self.assertRaises(TypeError, combinations, 'abc') # missing r argument self.assertRaises(TypeError, combinations, 'abc', 2, 1) # too many arguments self.assertRaises(TypeError, combinations, None) # pool is not iterable self.assertRaises(ValueError, combinations, 'abc', -2) # r is negative - for op in [lambda a:a] + picklecopiers: - self.assertEqual(list(op(combinations('abc', 32))), []) # r > n - - self.assertEqual(list(op(combinations('ABCD', 2))), - [('A','B'), ('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')]) - testIntermediate = combinations('ABCD', 2) - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), - [('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')]) - - self.assertEqual(list(op(combinations(range(4), 3))), - [(0,1,2), (0,1,3), (0,2,3), (1,2,3)]) - testIntermediate = combinations(range(4), 3) - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), - [(0,1,3), (0,2,3), (1,2,3)]) - def combinations1(iterable, r): 'Pure python version shown in the docs' pool = tuple(iterable) @@ -350,9 +280,6 @@ def combinations3(iterable, r): self.assertEqual(result, list(combinations2(values, r))) # matches second pure python version self.assertEqual(result, list(combinations3(values, r))) # matches second pure python version - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, combinations(values, r)) # test pickling - @support.bigaddrspacetest def test_combinations_overflow(self): with self.assertRaises((OverflowError, MemoryError)): @@ -364,7 +291,6 @@ def test_combinations_tuple_reuse(self): self.assertEqual(len(set(map(id, combinations('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(combinations('abcde', 3))))), 1) - @pickle_deprecated def test_combinations_with_replacement(self): cwr = combinations_with_replacement self.assertRaises(TypeError, cwr, 'abc') # missing r argument @@ -372,15 +298,6 @@ def test_combinations_with_replacement(self): self.assertRaises(TypeError, cwr, None) # pool is not iterable self.assertRaises(ValueError, cwr, 'abc', -2) # r is negative - for op in [lambda a:a] + picklecopiers: - self.assertEqual(list(op(cwr('ABC', 2))), - [('A','A'), ('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')]) - testIntermediate = cwr('ABC', 2) - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), - [('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')]) - - def cwr1(iterable, r): 'Pure python version shown in the docs' # number items returned: (n+r-1)! / r! / (n-1)! when n>0 @@ -438,22 +355,18 @@ def numcombs(n, r): self.assertEqual(result, list(cwr1(values, r))) # matches first pure python version self.assertEqual(result, list(cwr2(values, r))) # matches second pure python version - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, cwr(values,r)) # test pickling - @support.bigaddrspacetest def test_combinations_with_replacement_overflow(self): with self.assertRaises((OverflowError, MemoryError)): combinations_with_replacement("AA", 2**30) - # Test implementation detail: tuple re-use + # Test implementation detail: tuple re-use @support.impl_detail("tuple reuse is specific to CPython") def test_combinations_with_replacement_tuple_reuse(self): cwr = combinations_with_replacement self.assertEqual(len(set(map(id, cwr('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(cwr('abcde', 3))))), 1) - @pickle_deprecated def test_permutations(self): self.assertRaises(TypeError, permutations) # too few arguments self.assertRaises(TypeError, permutations, 'abc', 2, 1) # too many arguments @@ -514,9 +427,6 @@ def permutations2(iterable, r=None): self.assertEqual(result, list(permutations(values, None))) # test r as None self.assertEqual(result, list(permutations(values))) # test default r - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, permutations(values, r)) # test pickling - @support.bigaddrspacetest def test_permutations_overflow(self): with self.assertRaises((OverflowError, MemoryError)): @@ -560,7 +470,6 @@ def test_combinatorics(self): self.assertEqual(comb, list(filter(set(perm).__contains__, cwr))) # comb: cwr that is a perm self.assertEqual(comb, sorted(set(cwr) & set(perm))) # comb: both a cwr and a perm - @pickle_deprecated def test_compress(self): self.assertEqual(list(compress(data='ABCDEF', selectors=[1,0,1,0,1,1])), list('ACEF')) self.assertEqual(list(compress('ABCDEF', [1,0,1,0,1,1])), list('ACEF')) @@ -577,24 +486,6 @@ def test_compress(self): self.assertRaises(TypeError, compress, range(6)) # too few args self.assertRaises(TypeError, compress, range(6), None) # too many args - # check copy, deepcopy, pickle - for op in [lambda a:copy.copy(a), lambda a:copy.deepcopy(a)] + picklecopiers: - for data, selectors, result1, result2 in [ - ('ABCDEF', [1,0,1,0,1,1], 'ACEF', 'CEF'), - ('ABCDEF', [0,0,0,0,0,0], '', ''), - ('ABCDEF', [1,1,1,1,1,1], 'ABCDEF', 'BCDEF'), - ('ABCDEF', [1,0,1], 'AC', 'C'), - ('ABC', [0,1,1,1,1,1], 'BC', 'C'), - ]: - - self.assertEqual(list(op(compress(data=data, selectors=selectors))), list(result1)) - self.assertEqual(list(op(compress(data, selectors))), list(result1)) - testIntermediate = compress(data, selectors) - if result1: - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), list(result2)) - - @pickle_deprecated def test_count(self): self.assertEqual(lzip('abc',count()), [('a', 0), ('b', 1), ('c', 2)]) self.assertEqual(lzip('abc',count(3)), [('a', 3), ('b', 4), ('c', 5)]) @@ -603,6 +494,8 @@ def test_count(self): self.assertEqual(take(2, zip('abc',count(-3))), [('a', -3), ('b', -2)]) self.assertRaises(TypeError, count, 2, 3, 4) self.assertRaises(TypeError, count, 'a') + self.assertEqual(take(3, count(maxsize)), + [maxsize, maxsize + 1, maxsize + 2]) self.assertEqual(take(10, count(maxsize-5)), list(range(maxsize-5, maxsize+5))) self.assertEqual(take(10, count(-maxsize-5)), @@ -625,6 +518,15 @@ def test_count(self): self.assertEqual(next(c), -8) self.assertEqual(repr(count(10.25)), 'count(10.25)') self.assertEqual(repr(count(10.0)), 'count(10.0)') + + self.assertEqual(repr(count(maxsize)), f'count({maxsize})') + c = count(maxsize - 1) + self.assertEqual(repr(c), f'count({maxsize - 1})') + next(c) # c is now at masize + self.assertEqual(repr(c), f'count({maxsize})') + next(c) + self.assertEqual(repr(c), f'count({maxsize + 1})') + self.assertEqual(type(next(count(10.0))), float) for i in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 10, sys.maxsize-5, sys.maxsize+5): # Test repr @@ -632,19 +534,10 @@ def test_count(self): r2 = 'count(%r)'.__mod__(i) self.assertEqual(r1, r2) - # check copy, deepcopy, pickle - for value in -3, 3, maxsize-5, maxsize+5: - c = count(value) - self.assertEqual(next(copy.copy(c)), value) - self.assertEqual(next(copy.deepcopy(c)), value) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, count(value)) - #check proper internal error handling for large "step' sizes count(1, maxsize+5); sys.exc_info() - @pickle_deprecated - def test_count_with_stride(self): + def test_count_with_step(self): self.assertEqual(lzip('abc',count(2,3)), [('a', 2), ('b', 5), ('c', 8)]) self.assertEqual(lzip('abc',count(start=2,step=3)), [('a', 2), ('b', 5), ('c', 8)]) @@ -658,6 +551,12 @@ def test_count_with_stride(self): self.assertEqual(take(20, count(-maxsize-15, 3)), take(20, range(-maxsize-15,-maxsize+100, 3))) self.assertEqual(take(3, count(10, maxsize+5)), list(range(10, 10+3*(maxsize+5), maxsize+5))) + self.assertEqual(take(3, count(maxsize, 2)), + [maxsize, maxsize + 2, maxsize + 4]) + self.assertEqual(take(3, count(maxsize, maxsize)), + [maxsize, 2 * maxsize, 3 * maxsize]) + self.assertEqual(take(3, count(-maxsize, maxsize)), + [-maxsize, 0, maxsize]) self.assertEqual(take(3, count(2, 1.25)), [2, 3.25, 4.5]) self.assertEqual(take(3, count(2, 3.25-4j)), [2, 5.25-4j, 8.5-8j]) self.assertEqual(take(3, count(Decimal('1.1'), Decimal('.1'))), @@ -687,17 +586,42 @@ def test_count_with_stride(self): c = count(10, 1.0) self.assertEqual(type(next(c)), int) self.assertEqual(type(next(c)), float) - for i in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 10, sys.maxsize-5, sys.maxsize+5): - for j in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 1, 10, sys.maxsize-5, sys.maxsize+5): - # Test repr - r1 = repr(count(i, j)) - if j == 1: - r2 = ('count(%r)' % i) - else: - r2 = ('count(%r, %r)' % (i, j)) - self.assertEqual(r1, r2) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, count(i, j)) + + c = count(maxsize -2, 2) + self.assertEqual(repr(c), f'count({maxsize - 2}, 2)') + next(c) # c is now at masize + self.assertEqual(repr(c), f'count({maxsize}, 2)') + next(c) + self.assertEqual(repr(c), f'count({maxsize + 2}, 2)') + + c = count(maxsize + 1, -1) + self.assertEqual(repr(c), f'count({maxsize + 1}, -1)') + next(c) # c is now at masize + self.assertEqual(repr(c), f'count({maxsize}, -1)') + next(c) + self.assertEqual(repr(c), f'count({maxsize - 1}, -1)') + + @threading_helper.requires_working_threading() + def test_count_threading(self, step=1): + # this test verifies multithreading consistency, which is + # mostly for testing builds without GIL, but nice to test anyway + count_to = 10_000 + num_threads = 10 + c = count(step=step) + def counting_thread(): + for i in range(count_to): + next(c) + threads = [] + for i in range(num_threads): + thread = threading.Thread(target=counting_thread) + thread.start() + threads.append(thread) + for thread in threads: + thread.join() + self.assertEqual(next(c), count_to * num_threads * step) + + def test_count_with_step_threading(self): + self.test_count_threading(step=5) def test_cycle(self): self.assertEqual(take(10, cycle('abc')), list('abcabcabca')) @@ -706,113 +630,6 @@ def test_cycle(self): self.assertRaises(TypeError, cycle, 5) self.assertEqual(list(islice(cycle(gen3()),10)), [0,1,2,0,1,2,0,1,2,0]) - @pickle_deprecated - def test_cycle_copy_pickle(self): - # check copy, deepcopy, pickle - c = cycle('abc') - self.assertEqual(next(c), 'a') - #simple copy currently not supported, because __reduce__ returns - #an internal iterator - #self.assertEqual(take(10, copy.copy(c)), list('bcabcabcab')) - self.assertEqual(take(10, copy.deepcopy(c)), list('bcabcabcab')) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.assertEqual(take(10, pickle.loads(pickle.dumps(c, proto))), - list('bcabcabcab')) - next(c) - self.assertEqual(take(10, pickle.loads(pickle.dumps(c, proto))), - list('cabcabcabc')) - next(c) - next(c) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, cycle('abc')) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - # test with partial consumed input iterable - it = iter('abcde') - c = cycle(it) - _ = [next(c) for i in range(2)] # consume 2 of 5 inputs - p = pickle.dumps(c, proto) - d = pickle.loads(p) # rebuild the cycle object - self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) - - # test with completely consumed input iterable - it = iter('abcde') - c = cycle(it) - _ = [next(c) for i in range(7)] # consume 7 of 5 inputs - p = pickle.dumps(c, proto) - d = pickle.loads(p) # rebuild the cycle object - self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) - - @pickle_deprecated - def test_cycle_unpickle_compat(self): - testcases = [ - b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI0\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aK\x00tb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', - b'\x80\x04\x95=\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', - - b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI1\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00K\x01tb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', - b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', - - b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI00\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aI00\ntb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', - b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', - - b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI01\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00I01\ntb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', - b'\x80\x04\x95;\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', - ] - assert len(testcases) == 20 - for t in testcases: - it = pickle.loads(t) - self.assertEqual(take(10, it), [2, 3, 1, 2, 3, 1, 2, 3, 1, 2]) - - @pickle_deprecated - def test_cycle_setstate(self): - # Verify both modes for restoring state - - # Mode 0 is efficient. It uses an incompletely consumed input - # iterator to build a cycle object and then passes in state with - # a list of previously consumed values. There is no data - # overlap between the two. - c = cycle('defg') - c.__setstate__((list('abc'), 0)) - self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) - - # Mode 1 is inefficient. It starts with a cycle object built - # from an iterator over the remaining elements in a partial - # cycle and then passes in state with all of the previously - # seen values (this overlaps values included in the iterator). - c = cycle('defg') - c.__setstate__((list('abcdefg'), 1)) - self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) - - # The first argument to setstate needs to be a tuple - with self.assertRaises(TypeError): - cycle('defg').__setstate__([list('abcdefg'), 0]) - - # The first argument in the setstate tuple must be a list - with self.assertRaises(TypeError): - c = cycle('defg') - c.__setstate__((tuple('defg'), 0)) - take(20, c) - - # The second argument in the setstate tuple must be an int - with self.assertRaises(TypeError): - cycle('defg').__setstate__((list('abcdefg'), 'x')) - - self.assertRaises(TypeError, cycle('').__setstate__, ()) - self.assertRaises(TypeError, cycle('').__setstate__, ([],)) - - @pickle_deprecated def test_groupby(self): # Check whether it accepts arguments correctly self.assertEqual([], list(groupby([]))) @@ -831,15 +648,6 @@ def test_groupby(self): dup.append(elem) self.assertEqual(s, dup) - # Check normal pickled - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - dup = [] - for k, g in pickle.loads(pickle.dumps(groupby(s, testR), proto)): - for elem in g: - self.assertEqual(k, elem[0]) - dup.append(elem) - self.assertEqual(s, dup) - # Check nested case dup = [] for k, g in groupby(s, testR): @@ -850,18 +658,6 @@ def test_groupby(self): dup.append(elem) self.assertEqual(s, dup) - # Check nested and pickled - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - dup = [] - for k, g in pickle.loads(pickle.dumps(groupby(s, testR), proto)): - for ik, ig in pickle.loads(pickle.dumps(groupby(g, testR2), proto)): - for elem in ig: - self.assertEqual(k, elem[0]) - self.assertEqual(ik, elem[2]) - dup.append(elem) - self.assertEqual(s, dup) - - # Check case where inner iterator is not used keys = [k for k, g in groupby(s, testR)] expectedkeys = set([r[0] for r in s]) @@ -881,13 +677,6 @@ def test_groupby(self): list(it) # exhaust the groupby iterator self.assertEqual(list(g3), []) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - it = groupby(s, testR) - _, g = next(it) - next(it) - next(it) - self.assertEqual(list(pickle.loads(pickle.dumps(g, proto))), []) - # Exercise pipes and filters style s = 'abracadabra' # sort s | uniq @@ -970,7 +759,6 @@ def test_filter(self): c = filter(isEven, range(6)) self.pickletest(proto, c) - @pickle_deprecated def test_filterfalse(self): self.assertEqual(list(filterfalse(isEven, range(6))), [1,3,5]) self.assertEqual(list(filterfalse(None, [0,1,0,2,0])), [0,0,0]) @@ -981,8 +769,6 @@ def test_filterfalse(self): self.assertRaises(TypeError, filterfalse, lambda x:x, range(6), 7) self.assertRaises(TypeError, filterfalse, isEven, 3) self.assertRaises(TypeError, next, filterfalse(range(6), range(6))) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, filterfalse(isEven, range(6))) def test_zip(self): # XXX This is rather silly now that builtin zip() calls zip()... @@ -1001,33 +787,12 @@ def test_zip(self): lzip('abc', 'def')) @support.impl_detail("tuple reuse is specific to CPython") - @pickle_deprecated def test_zip_tuple_reuse(self): ids = list(map(id, zip('abc', 'def'))) self.assertEqual(min(ids), max(ids)) ids = list(map(id, list(zip('abc', 'def')))) self.assertEqual(len(dict.fromkeys(ids)), len(ids)) - # check copy, deepcopy, pickle - ans = [(x,y) for x, y in copy.copy(zip('abc',count()))] - self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) - - ans = [(x,y) for x, y in copy.deepcopy(zip('abc',count()))] - self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - ans = [(x,y) for x, y in pickle.loads(pickle.dumps(zip('abc',count()), proto))] - self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - testIntermediate = zip('abc',count()) - next(testIntermediate) - ans = [(x,y) for x, y in pickle.loads(pickle.dumps(testIntermediate, proto))] - self.assertEqual(ans, [('b', 1), ('c', 2)]) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, zip('abc', count())) - def test_ziplongest(self): for args in [ ['abc', range(6)], @@ -1077,14 +842,6 @@ def test_zip_longest_tuple_reuse(self): ids = list(map(id, list(zip_longest('abc', 'def')))) self.assertEqual(len(dict.fromkeys(ids)), len(ids)) - @pickle_deprecated - def test_zip_longest_pickling(self): - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, zip_longest("abc", "def")) - self.pickletest(proto, zip_longest("abc", "defgh")) - self.pickletest(proto, zip_longest("abc", "defgh", fillvalue=1)) - self.pickletest(proto, zip_longest("", "defgh")) - def test_zip_longest_bad_iterable(self): exception = TypeError() @@ -1266,12 +1023,16 @@ def product1(*args, **kwds): else: return - def product2(*args, **kwds): + def product2(*iterables, repeat=1): 'Pure python version used in docs' - pools = list(map(tuple, args)) * kwds.get('repeat', 1) + if repeat < 0: + raise ValueError('repeat argument cannot be negative') + pools = [tuple(pool) for pool in iterables] * repeat + result = [[]] for pool in pools: result = [x+[y] for x in result for y in pool] + for prod in result: yield tuple(prod) @@ -1296,34 +1057,6 @@ def test_product_tuple_reuse(self): self.assertEqual(len(set(map(id, product('abc', 'def')))), 1) self.assertNotEqual(len(set(map(id, list(product('abc', 'def'))))), 1) - @pickle_deprecated - def test_product_pickling(self): - # check copy, deepcopy, pickle - for args, result in [ - ([], [()]), # zero iterables - (['ab'], [('a',), ('b',)]), # one iterable - ([range(2), range(3)], [(0,0), (0,1), (0,2), (1,0), (1,1), (1,2)]), # two iterables - ([range(0), range(2), range(3)], []), # first iterable with zero length - ([range(2), range(0), range(3)], []), # middle iterable with zero length - ([range(2), range(3), range(0)], []), # last iterable with zero length - ]: - self.assertEqual(list(copy.copy(product(*args))), result) - self.assertEqual(list(copy.deepcopy(product(*args))), result) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, product(*args)) - - @pickle_deprecated - def test_product_issue_25021(self): - # test that indices are properly clamped to the length of the tuples - p = product((1, 2),(3,)) - p.__setstate__((0, 0x1000)) # will access tuple element 1 if not clamped - self.assertEqual(next(p), (2, 3)) - # test that empty tuple in the list will result in an immediate StopIteration - p = product((1, 2), (), (3,)) - p.__setstate__((0, 0, 0x1000)) # will access tuple element 1 if not clamped - self.assertRaises(StopIteration, next, p) - - @pickle_deprecated def test_repeat(self): self.assertEqual(list(repeat(object='a', times=3)), ['a', 'a', 'a']) self.assertEqual(lzip(range(3),repeat('a')), @@ -1342,21 +1075,12 @@ def test_repeat(self): list(r) self.assertEqual(repr(r), 'repeat((1+0j), 0)') - # check copy, deepcopy, pickle - c = repeat(object='a', times=10) - self.assertEqual(next(c), 'a') - self.assertEqual(take(2, copy.copy(c)), list('a' * 2)) - self.assertEqual(take(2, copy.deepcopy(c)), list('a' * 2)) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, repeat(object='a', times=10)) - def test_repeat_with_negative_times(self): self.assertEqual(repr(repeat('a', -1)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', -2)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', times=-1)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', times=-2)), "repeat('a', 0)") - @pickle_deprecated def test_map(self): self.assertEqual(list(map(operator.pow, range(3), range(1,7))), [0**1, 1**2, 2**3]) @@ -1374,20 +1098,6 @@ def test_map(self): self.assertRaises(ValueError, next, map(errfunc, [4], [5])) self.assertRaises(TypeError, next, map(onearg, [4], [5])) - # check copy, deepcopy, pickle - ans = [('a',0),('b',1),('c',2)] - - c = map(tupleize, 'abc', count()) - self.assertEqual(list(copy.copy(c)), ans) - - c = map(tupleize, 'abc', count()) - self.assertEqual(list(copy.deepcopy(c)), ans) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - c = map(tupleize, 'abc', count()) - self.pickletest(proto, c) - - @pickle_deprecated def test_starmap(self): self.assertEqual(list(starmap(operator.pow, zip(range(3), range(1,7)))), [0**1, 1**2, 2**3]) @@ -1402,20 +1112,6 @@ def test_starmap(self): self.assertRaises(ValueError, next, starmap(errfunc, [(4,5)])) self.assertRaises(TypeError, next, starmap(onearg, [(4,5)])) - # check copy, deepcopy, pickle - ans = [0**1, 1**2, 2**3] - - c = starmap(operator.pow, zip(range(3), range(1,7))) - self.assertEqual(list(copy.copy(c)), ans) - - c = starmap(operator.pow, zip(range(3), range(1,7))) - self.assertEqual(list(copy.deepcopy(c)), ans) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - c = starmap(operator.pow, zip(range(3), range(1,7))) - self.pickletest(proto, c) - - @pickle_deprecated def test_islice(self): for args in [ # islice(args) should agree with range(args) (10, 20, 3), @@ -1472,21 +1168,6 @@ def test_islice(self): self.assertEqual(list(islice(c, 1, 3, 50)), [1]) self.assertEqual(next(c), 3) - # check copy, deepcopy, pickle - for args in [ # islice(args) should agree with range(args) - (10, 20, 3), - (10, 3, 20), - (10, 20), - (10, 3), - (20,) - ]: - self.assertEqual(list(copy.copy(islice(range(100), *args))), - list(range(*args))) - self.assertEqual(list(copy.deepcopy(islice(range(100), *args))), - list(range(*args))) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, islice(range(100), *args)) - # Issue #21321: check source iterator is not referenced # from islice() after the latter has been exhausted it = (x for x in (1, 2)) @@ -1510,7 +1191,6 @@ def __index__(self): self.assertEqual(list(islice(range(100), IntLike(10), IntLike(50), IntLike(5))), list(range(10,50,5))) - @pickle_deprecated def test_takewhile(self): data = [1, 3, 5, 20, 2, 4, 6, 8] self.assertEqual(list(takewhile(underten, data)), [1, 3, 5]) @@ -1524,14 +1204,6 @@ def test_takewhile(self): self.assertEqual(list(t), [1, 1, 1]) self.assertRaises(StopIteration, next, t) - # check copy, deepcopy, pickle - self.assertEqual(list(copy.copy(takewhile(underten, data))), [1, 3, 5]) - self.assertEqual(list(copy.deepcopy(takewhile(underten, data))), - [1, 3, 5]) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, takewhile(underten, data)) - - @pickle_deprecated def test_dropwhile(self): data = [1, 3, 5, 20, 2, 4, 6, 8] self.assertEqual(list(dropwhile(underten, data)), [20, 2, 4, 6, 8]) @@ -1542,14 +1214,6 @@ def test_dropwhile(self): self.assertRaises(TypeError, next, dropwhile(10, [(4,5)])) self.assertRaises(ValueError, next, dropwhile(errfunc, [(4,5)])) - # check copy, deepcopy, pickle - self.assertEqual(list(copy.copy(dropwhile(underten, data))), [20, 2, 4, 6, 8]) - self.assertEqual(list(copy.deepcopy(dropwhile(underten, data))), - [20, 2, 4, 6, 8]) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, dropwhile(underten, data)) - - @pickle_deprecated def test_tee(self): n = 200 @@ -1616,10 +1280,11 @@ def test_tee(self): self.assertEqual(len(result), n) self.assertEqual([list(x) for x in result], [list('abc')]*n) - # tee pass-through to copyable iterator + # tee objects are independent (see bug gh-123884) a, b = tee('abc') c, d = tee(a) - self.assertTrue(a is c) + e, f = tee(c) + self.assertTrue(len({a, b, c, d, e, f}) == 6) # test tee_new t1, t2 = tee('abc') @@ -1629,7 +1294,7 @@ def test_tee(self): t3 = tnew(t1) self.assertTrue(list(t1) == list(t2) == list(t3) == list('abc')) - # test that tee objects are weak referencable + # test that tee objects are weak referenceable a, b = tee(range(10)) p = weakref.proxy(a) self.assertEqual(getattr(p, '__class__'), type(b)) @@ -1664,40 +1329,13 @@ def test_tee(self): self.assertEqual(list(a), long_ans[100:]) self.assertEqual(list(b), long_ans[60:]) - # check deepcopy - a, b = tee('abc') - self.assertEqual(list(copy.deepcopy(a)), ans) - self.assertEqual(list(copy.deepcopy(b)), ans) - self.assertEqual(list(a), ans) - self.assertEqual(list(b), ans) - a, b = tee(range(10000)) - self.assertEqual(list(copy.deepcopy(a)), long_ans) - self.assertEqual(list(copy.deepcopy(b)), long_ans) - self.assertEqual(list(a), long_ans) - self.assertEqual(list(b), long_ans) - - # check partially consumed deepcopy - a, b = tee('abc') - take(2, a) - take(1, b) - self.assertEqual(list(copy.deepcopy(a)), ans[2:]) - self.assertEqual(list(copy.deepcopy(b)), ans[1:]) - self.assertEqual(list(a), ans[2:]) - self.assertEqual(list(b), ans[1:]) - a, b = tee(range(10000)) - take(100, a) - take(60, b) - self.assertEqual(list(copy.deepcopy(a)), long_ans[100:]) - self.assertEqual(list(copy.deepcopy(b)), long_ans[60:]) - self.assertEqual(list(a), long_ans[100:]) - self.assertEqual(list(b), long_ans[60:]) - - # check pickle - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, iter(tee('abc'))) - a, b = tee('abc') - self.pickletest(proto, a, compare=ans) - self.pickletest(proto, b, compare=ans) + def test_tee_dealloc_segfault(self): + # gh-115874: segfaults when accessing module state in tp_dealloc. + script = ( + "import typing, copyreg, itertools; " + "copyreg.buggy_tee = itertools.tee(())" + ) + script_helper.assert_python_ok("-c", script) # Issue 13454: Crash when deleting backward iterator from tee() def test_tee_del_backward(self): @@ -1813,6 +1451,13 @@ def test_zip_longest_result_gc(self): gc.collect() self.assertTrue(gc.is_tracked(next(it))) + @support.cpython_only + def test_pairwise_result_gc(self): + # Ditto for pairwise. + it = pairwise([None, None]) + gc.collect() + self.assertTrue(gc.is_tracked(next(it))) + @support.cpython_only def test_immutable_types(self): from itertools import _grouper, _tee, _tee_dataobject @@ -1851,33 +1496,6 @@ class TestExamples(unittest.TestCase): def test_accumulate(self): self.assertEqual(list(accumulate([1,2,3,4,5])), [1, 3, 6, 10, 15]) - @pickle_deprecated - def test_accumulate_reducible(self): - # check copy, deepcopy, pickle - data = [1, 2, 3, 4, 5] - accumulated = [1, 3, 6, 10, 15] - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - it = accumulate(data) - self.assertEqual(list(pickle.loads(pickle.dumps(it, proto))), accumulated[:]) - self.assertEqual(next(it), 1) - self.assertEqual(list(pickle.loads(pickle.dumps(it, proto))), accumulated[1:]) - it = accumulate(data) - self.assertEqual(next(it), 1) - self.assertEqual(list(copy.deepcopy(it)), accumulated[1:]) - self.assertEqual(list(copy.copy(it)), accumulated[1:]) - - @pickle_deprecated - def test_accumulate_reducible_none(self): - # Issue #25718: total is None - it = accumulate([None, None, None], operator.is_) - self.assertEqual(next(it), None) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - it_copy = pickle.loads(pickle.dumps(it, proto)) - self.assertEqual(list(it_copy), [True, False]) - self.assertEqual(list(copy.deepcopy(it)), [True, False]) - self.assertEqual(list(copy.copy(it)), [True, False]) - def test_chain(self): self.assertEqual(''.join(chain('ABC', 'DEF')), 'ABCDEF') @@ -1985,27 +1603,169 @@ def batched_recipe(iterable, n): self.assertEqual(r1, r2) self.assertEqual(e1, e2) + + def test_groupby_recipe(self): + + # Begin groupby() recipe ####################################### + + def groupby(iterable, key=None): + # [k for k, g in groupby('AAAABBBCCDAABBB')] → A B C D A B + # [list(g) for k, g in groupby('AAAABBBCCD')] → AAAA BBB CC D + + keyfunc = (lambda x: x) if key is None else key + iterator = iter(iterable) + exhausted = False + + def _grouper(target_key): + nonlocal curr_value, curr_key, exhausted + yield curr_value + for curr_value in iterator: + curr_key = keyfunc(curr_value) + if curr_key != target_key: + return + yield curr_value + exhausted = True + + try: + curr_value = next(iterator) + except StopIteration: + return + curr_key = keyfunc(curr_value) + + while not exhausted: + target_key = curr_key + curr_group = _grouper(target_key) + yield curr_key, curr_group + if curr_key == target_key: + for _ in curr_group: + pass + + # End groupby() recipe ######################################### + + # Check whether it accepts arguments correctly + self.assertEqual([], list(groupby([]))) + self.assertEqual([], list(groupby([], key=id))) + self.assertRaises(TypeError, list, groupby('abc', [])) + if False: + # Test not applicable to the recipe + self.assertRaises(TypeError, list, groupby('abc', None)) + self.assertRaises(TypeError, groupby, 'abc', lambda x:x, 10) + + # Check normal input + s = [(0, 10, 20), (0, 11,21), (0,12,21), (1,13,21), (1,14,22), + (2,15,22), (3,16,23), (3,17,23)] + dup = [] + for k, g in groupby(s, lambda r:r[0]): + for elem in g: + self.assertEqual(k, elem[0]) + dup.append(elem) + self.assertEqual(s, dup) + + # Check nested case + dup = [] + for k, g in groupby(s, testR): + for ik, ig in groupby(g, testR2): + for elem in ig: + self.assertEqual(k, elem[0]) + self.assertEqual(ik, elem[2]) + dup.append(elem) + self.assertEqual(s, dup) + + # Check case where inner iterator is not used + keys = [k for k, g in groupby(s, testR)] + expectedkeys = set([r[0] for r in s]) + self.assertEqual(set(keys), expectedkeys) + self.assertEqual(len(keys), len(expectedkeys)) + + # Check case where inner iterator is used after advancing the groupby + # iterator + s = list(zip('AABBBAAAA', range(9))) + it = groupby(s, testR) + _, g1 = next(it) + _, g2 = next(it) + _, g3 = next(it) + self.assertEqual(list(g1), []) + self.assertEqual(list(g2), []) + self.assertEqual(next(g3), ('A', 5)) + list(it) # exhaust the groupby iterator + self.assertEqual(list(g3), []) + + # Exercise pipes and filters style + s = 'abracadabra' + # sort s | uniq + r = [k for k, g in groupby(sorted(s))] + self.assertEqual(r, ['a', 'b', 'c', 'd', 'r']) + # sort s | uniq -d + r = [k for k, g in groupby(sorted(s)) if list(islice(g,1,2))] + self.assertEqual(r, ['a', 'b', 'r']) + # sort s | uniq -c + r = [(len(list(g)), k) for k, g in groupby(sorted(s))] + self.assertEqual(r, [(5, 'a'), (2, 'b'), (1, 'c'), (1, 'd'), (2, 'r')]) + # sort s | uniq -c | sort -rn | head -3 + r = sorted([(len(list(g)) , k) for k, g in groupby(sorted(s))], reverse=True)[:3] + self.assertEqual(r, [(5, 'a'), (2, 'r'), (2, 'b')]) + + # iter.__next__ failure + class ExpectedError(Exception): + pass + def delayed_raise(n=0): + for i in range(n): + yield 'yo' + raise ExpectedError + def gulp(iterable, keyp=None, func=list): + return [func(g) for k, g in groupby(iterable, keyp)] + + # iter.__next__ failure on outer object + self.assertRaises(ExpectedError, gulp, delayed_raise(0)) + # iter.__next__ failure on inner object + self.assertRaises(ExpectedError, gulp, delayed_raise(1)) + + # __eq__ failure + class DummyCmp: + def __eq__(self, dst): + raise ExpectedError + s = [DummyCmp(), DummyCmp(), None] + + # __eq__ failure on outer object + self.assertRaises(ExpectedError, gulp, s, func=id) + # __eq__ failure on inner object + self.assertRaises(ExpectedError, gulp, s) + + # keyfunc failure + def keyfunc(obj): + if keyfunc.skip > 0: + keyfunc.skip -= 1 + return obj + else: + raise ExpectedError + + # keyfunc failure on outer object + keyfunc.skip = 0 + self.assertRaises(ExpectedError, gulp, [None], keyfunc) + keyfunc.skip = 1 + self.assertRaises(ExpectedError, gulp, [None, None], keyfunc) + + @staticmethod def islice(iterable, *args): + # islice('ABCDEFG', 2) → A B + # islice('ABCDEFG', 2, 4) → C D + # islice('ABCDEFG', 2, None) → C D E F G + # islice('ABCDEFG', 0, None, 2) → A C E G + s = slice(*args) - start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 - it = iter(range(start, stop, step)) - try: - nexti = next(it) - except StopIteration: - # Consume *iterable* up to the *start* position. - for i, element in zip(range(start), iterable): - pass - return - try: - for i, element in enumerate(iterable): - if i == nexti: - yield element - nexti = next(it) - except StopIteration: - # Consume to *stop*. - for i, element in zip(range(i + 1, stop), iterable): - pass + start = 0 if s.start is None else s.start + stop = s.stop + step = 1 if s.step is None else s.step + if start < 0 or (stop is not None and stop < 0) or step <= 0: + raise ValueError + + indices = count() if stop is None else range(max(start, stop)) + next_i = start + for i, element in zip(indices, iterable): + if i == next_i: + yield element + next_i += step def test_islice_recipe(self): self.assertEqual(list(self.islice('ABCDEFG', 2)), list('AB')) @@ -2025,6 +1785,172 @@ def test_islice_recipe(self): self.assertEqual(next(c), 3) + def test_tee_recipe(self): + + # Begin tee() recipe ########################################### + + def tee(iterable, n=2): + if n < 0: + raise ValueError + if n == 0: + return () + iterator = _tee(iterable) + result = [iterator] + for _ in range(n - 1): + result.append(_tee(iterator)) + return tuple(result) + + class _tee: + + def __init__(self, iterable): + it = iter(iterable) + if isinstance(it, _tee): + self.iterator = it.iterator + self.link = it.link + else: + self.iterator = it + self.link = [None, None] + + def __iter__(self): + return self + + def __next__(self): + link = self.link + if link[1] is None: + link[0] = next(self.iterator) + link[1] = [None, None] + value, self.link = link + return value + + # End tee() recipe ############################################# + + n = 200 + + a, b = tee([]) # test empty iterator + self.assertEqual(list(a), []) + self.assertEqual(list(b), []) + + a, b = tee(irange(n)) # test 100% interleaved + self.assertEqual(lzip(a,b), lzip(range(n), range(n))) + + a, b = tee(irange(n)) # test 0% interleaved + self.assertEqual(list(a), list(range(n))) + self.assertEqual(list(b), list(range(n))) + + a, b = tee(irange(n)) # test dealloc of leading iterator + for i in range(100): + self.assertEqual(next(a), i) + del a + self.assertEqual(list(b), list(range(n))) + + a, b = tee(irange(n)) # test dealloc of trailing iterator + for i in range(100): + self.assertEqual(next(a), i) + del b + self.assertEqual(list(a), list(range(100, n))) + + for j in range(5): # test randomly interleaved + order = [0]*n + [1]*n + random.shuffle(order) + lists = ([], []) + its = tee(irange(n)) + for i in order: + value = next(its[i]) + lists[i].append(value) + self.assertEqual(lists[0], list(range(n))) + self.assertEqual(lists[1], list(range(n))) + + # test argument format checking + self.assertRaises(TypeError, tee) + self.assertRaises(TypeError, tee, 3) + self.assertRaises(TypeError, tee, [1,2], 'x') + self.assertRaises(TypeError, tee, [1,2], 3, 'x') + + # tee object should be instantiable + a, b = tee('abc') + c = type(a)('def') + self.assertEqual(list(c), list('def')) + + # test long-lagged and multi-way split + a, b, c = tee(range(2000), 3) + for i in range(100): + self.assertEqual(next(a), i) + self.assertEqual(list(b), list(range(2000))) + self.assertEqual([next(c), next(c)], list(range(2))) + self.assertEqual(list(a), list(range(100,2000))) + self.assertEqual(list(c), list(range(2,2000))) + + # test invalid values of n + self.assertRaises(TypeError, tee, 'abc', 'invalid') + self.assertRaises(ValueError, tee, [], -1) + + for n in range(5): + result = tee('abc', n) + self.assertEqual(type(result), tuple) + self.assertEqual(len(result), n) + self.assertEqual([list(x) for x in result], [list('abc')]*n) + + # tee objects are independent (see bug gh-123884) + a, b = tee('abc') + c, d = tee(a) + e, f = tee(c) + self.assertTrue(len({a, b, c, d, e, f}) == 6) + + # test tee_new + t1, t2 = tee('abc') + tnew = type(t1) + self.assertRaises(TypeError, tnew) + self.assertRaises(TypeError, tnew, 10) + t3 = tnew(t1) + self.assertTrue(list(t1) == list(t2) == list(t3) == list('abc')) + + # test that tee objects are weak referencable + a, b = tee(range(10)) + p = weakref.proxy(a) + self.assertEqual(getattr(p, '__class__'), type(b)) + del a + gc.collect() # For PyPy or other GCs. + self.assertRaises(ReferenceError, getattr, p, '__class__') + + ans = list('abc') + long_ans = list(range(10000)) + + # Tests not applicable to the tee() recipe + if False: + # check copy + a, b = tee('abc') + self.assertEqual(list(copy.copy(a)), ans) + self.assertEqual(list(copy.copy(b)), ans) + a, b = tee(list(range(10000))) + self.assertEqual(list(copy.copy(a)), long_ans) + self.assertEqual(list(copy.copy(b)), long_ans) + + # check partially consumed copy + a, b = tee('abc') + take(2, a) + take(1, b) + self.assertEqual(list(copy.copy(a)), ans[2:]) + self.assertEqual(list(copy.copy(b)), ans[1:]) + self.assertEqual(list(a), ans[2:]) + self.assertEqual(list(b), ans[1:]) + a, b = tee(range(10000)) + take(100, a) + take(60, b) + self.assertEqual(list(copy.copy(a)), long_ans[100:]) + self.assertEqual(list(copy.copy(b)), long_ans[60:]) + self.assertEqual(list(a), long_ans[100:]) + self.assertEqual(list(b), long_ans[60:]) + + # Issue 13454: Crash when deleting backward iterator from tee() + forward, backward = tee(repeat(None, 2000)) # 20000000 + try: + any(forward) # exhaust the iterator + del backward + except: + del forward, backward + raise + + class TestGC(unittest.TestCase): def makecycle(self, iterator, container): @@ -2538,10 +2464,10 @@ class subclass(cls): subclass(*args, newarg=3) for cls, args, result in testcases: - # Constructors of repeat, zip, compress accept keyword arguments. + # Constructors of repeat, zip, map, compress accept keyword arguments. # Their subclasses need overriding __new__ to support new # keyword arguments. - if cls in [repeat, zip, compress]: + if cls in [repeat, zip, map, compress]: continue with self.subTest(cls): class subclass_with_init(cls): diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py index 124045b13184b3..2250af964c022b 100644 --- a/Lib/test/test_json/test_decode.py +++ b/Lib/test/test_json/test_decode.py @@ -8,14 +8,40 @@ class TestDecode: def test_decimal(self): rval = self.loads('1.1', parse_float=decimal.Decimal) - self.assertTrue(isinstance(rval, decimal.Decimal)) + self.assertIsInstance(rval, decimal.Decimal) self.assertEqual(rval, decimal.Decimal('1.1')) def test_float(self): rval = self.loads('1', parse_int=float) - self.assertTrue(isinstance(rval, float)) + self.assertIsInstance(rval, float) self.assertEqual(rval, 1.0) + def test_nonascii_digits_rejected(self): + # JSON specifies only ascii digits, see gh-125687 + for num in ["1\uff10", "0.\uff10", "0e\uff10"]: + with self.assertRaises(self.JSONDecodeError): + self.loads(num) + + def test_bytes(self): + self.assertEqual(self.loads(b"1"), 1) + + def test_parse_constant(self): + for constant, expected in [ + ("Infinity", "INFINITY"), + ("-Infinity", "-INFINITY"), + ("NaN", "NAN"), + ]: + self.assertEqual( + self.loads(constant, parse_constant=str.upper), expected + ) + + def test_constant_invalid_case(self): + for constant in [ + "nan", "NAN", "naN", "infinity", "INFINITY", "inFiniTy" + ]: + with self.assertRaises(self.JSONDecodeError): + self.loads(constant) + def test_empty_objects(self): self.assertEqual(self.loads('{}'), {}) self.assertEqual(self.loads('[]'), []) @@ -88,7 +114,8 @@ def test_string_with_utf8_bom(self): self.json.load(StringIO(bom_json)) self.assertIn('BOM', str(cm.exception)) # make sure that the BOM is not detected in the middle of a string - bom_in_str = '"{}"'.format(''.encode('utf-8-sig').decode('utf-8')) + bom = ''.encode('utf-8-sig').decode('utf-8') + bom_in_str = f'"{bom}"' self.assertEqual(self.loads(bom_in_str), '\ufeff') self.assertEqual(self.json.load(StringIO(bom_in_str)), '\ufeff') diff --git a/Lib/test/test_json/test_default.py b/Lib/test/test_json/test_default.py index 3ce16684a08272..811880a15c8020 100644 --- a/Lib/test/test_json/test_default.py +++ b/Lib/test/test_json/test_default.py @@ -8,6 +8,24 @@ def test_default(self): self.dumps(type, default=repr), self.dumps(repr(type))) + def test_bad_default(self): + def default(obj): + if obj is NotImplemented: + raise ValueError + if obj is ...: + return NotImplemented + if obj is type: + return collections + return [...] + + with self.assertRaises(ValueError) as cm: + self.dumps(type, default=default) + self.assertEqual(cm.exception.__notes__, + ['when serializing ellipsis object', + 'when serializing list item 0', + 'when serializing module object', + 'when serializing type object']) + def test_ordereddict(self): od = collections.OrderedDict(a=1, b=2, c=3, d=4) od.move_to_end('b') diff --git a/Lib/test/test_json/test_encode_basestring_ascii.py b/Lib/test/test_json/test_encode_basestring_ascii.py index 4bbc6c71489a83..6a39b72a09df35 100644 --- a/Lib/test/test_json/test_encode_basestring_ascii.py +++ b/Lib/test/test_json/test_encode_basestring_ascii.py @@ -23,8 +23,7 @@ def test_encode_basestring_ascii(self): for input_string, expect in CASES: result = self.json.encoder.encode_basestring_ascii(input_string) self.assertEqual(result, expect, - '{0!r} != {1!r} for {2}({3!r})'.format( - result, expect, fname, input_string)) + f'{result!r} != {expect!r} for {fname}({input_string!r})') def test_ordered_dict(self): # See issue 6105 diff --git a/Lib/test/test_json/test_fail.py b/Lib/test/test_json/test_fail.py index d6bce605e21463..7c1696cc66d12b 100644 --- a/Lib/test/test_json/test_fail.py +++ b/Lib/test/test_json/test_fail.py @@ -89,7 +89,7 @@ def test_failures(self): except self.JSONDecodeError: pass else: - self.fail("Expected failure for fail{0}.json: {1!r}".format(idx, doc)) + self.fail(f"Expected failure for fail{idx}.json: {doc!r}") def test_non_string_keys_dict(self): data = {'a' : 1, (1, 2) : 2} @@ -100,8 +100,27 @@ def test_non_string_keys_dict(self): def test_not_serializable(self): import sys with self.assertRaisesRegex(TypeError, - 'Object of type module is not JSON serializable'): + 'Object of type module is not JSON serializable') as cm: self.dumps(sys) + self.assertFalse(hasattr(cm.exception, '__notes__')) + + with self.assertRaises(TypeError) as cm: + self.dumps([1, [2, 3, sys]]) + self.assertEqual(cm.exception.__notes__, + ['when serializing list item 2', + 'when serializing list item 1']) + + with self.assertRaises(TypeError) as cm: + self.dumps((1, (2, 3, sys))) + self.assertEqual(cm.exception.__notes__, + ['when serializing tuple item 2', + 'when serializing tuple item 1']) + + with self.assertRaises(TypeError) as cm: + self.dumps({'a': {'b': sys}}) + self.assertEqual(cm.exception.__notes__, + ["when serializing dict item 'b'", + "when serializing dict item 'a'"]) def test_truncated_input(self): test_cases = [ diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index 164ff2013eb552..663c0643579ac8 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -12,8 +12,8 @@ def test_listrecursion(self): x.append(x) try: self.dumps(x) - except ValueError: - pass + except ValueError as exc: + self.assertEqual(exc.__notes__, ["when serializing list item 0"]) else: self.fail("didn't raise ValueError on list recursion") x = [] @@ -21,8 +21,8 @@ def test_listrecursion(self): x.append(y) try: self.dumps(x) - except ValueError: - pass + except ValueError as exc: + self.assertEqual(exc.__notes__, ["when serializing list item 0"]*2) else: self.fail("didn't raise ValueError on alternating list recursion") y = [] @@ -35,8 +35,8 @@ def test_dictrecursion(self): x["test"] = x try: self.dumps(x) - except ValueError: - pass + except ValueError as exc: + self.assertEqual(exc.__notes__, ["when serializing dict item 'test'"]) else: self.fail("didn't raise ValueError on dict recursion") x = {} @@ -60,12 +60,15 @@ def default(self, o): enc.recurse = True try: enc.encode(JSONTestObject) - except ValueError: - pass + except ValueError as exc: + self.assertEqual(exc.__notes__, + ["when serializing list item 0", + "when serializing type object"]) else: self.fail("didn't raise ValueError on default recursion") + @support.skip_emscripten_stack_overflow() def test_highly_nested_objects_decoding(self): # test that loading highly-nested objects doesn't segfault when C # accelerations are used. See #12017 @@ -79,6 +82,7 @@ def test_highly_nested_objects_decoding(self): with support.infinite_recursion(): self.loads('[' * 100000 + '1' + ']' * 100000) + @support.skip_emscripten_stack_overflow() def test_highly_nested_objects_encoding(self): # See #12051 l, d = [], {} @@ -91,6 +95,7 @@ def test_highly_nested_objects_encoding(self): with support.infinite_recursion(5000): self.dumps(d) + @support.skip_emscripten_stack_overflow() def test_endless_recursion(self): # See #12051 class EndlessJSONEncoder(self.json.JSONEncoder): diff --git a/Lib/test/test_json/test_scanstring.py b/Lib/test/test_json/test_scanstring.py index 2d3ee8a8bf0f92..cca556a3b95bab 100644 --- a/Lib/test/test_json/test_scanstring.py +++ b/Lib/test/test_json/test_scanstring.py @@ -116,6 +116,11 @@ def test_bad_escapes(self): '"\\u012z"', '"\\u0x12"', '"\\u0X12"', + '"\\u{0}"'.format("\uff10" * 4), + '"\\u 123"', + '"\\u-123"', + '"\\u+123"', + '"\\u1_23"', '"\\ud834\\"', '"\\ud834\\u"', '"\\ud834\\ud"', @@ -127,6 +132,11 @@ def test_bad_escapes(self): '"\\ud834\\udd2z"', '"\\ud834\\u0x20"', '"\\ud834\\u0X20"', + '"\\ud834\\u{0}"'.format("\uff10" * 4), + '"\\ud834\\u 123"', + '"\\ud834\\u-123"', + '"\\ud834\\u+123"', + '"\\ud834\\u1_23"', ] for s in bad_escapes: with self.assertRaises(self.JSONDecodeError, msg=s): diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 2b63810d53981e..5da7cdcad709fa 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -11,7 +11,7 @@ @support.requires_subprocess() -class TestTool(unittest.TestCase): +class TestMain(unittest.TestCase): data = """ [["blorpie"],[ "whoops" ] , [ @@ -19,6 +19,7 @@ class TestTool(unittest.TestCase): "i-vhbjkhnth", {"nifty":87}, {"morefield" :\tfalse,"field" :"yes"} ] """ + module = 'json' expect_without_sort_keys = textwrap.dedent("""\ [ @@ -87,7 +88,7 @@ class TestTool(unittest.TestCase): """) def test_stdin_stdout(self): - args = sys.executable, '-m', 'json.tool' + args = sys.executable, '-m', self.module process = subprocess.run(args, input=self.data, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, self.expect) self.assertEqual(process.stderr, '') @@ -101,7 +102,7 @@ def _create_infile(self, data=None): def test_infile_stdout(self): infile = self._create_infile() - rc, out, err = assert_python_ok('-m', 'json.tool', infile) + rc, out, err = assert_python_ok('-m', self.module, infile) self.assertEqual(rc, 0) self.assertEqual(out.splitlines(), self.expect.encode().splitlines()) self.assertEqual(err, b'') @@ -115,7 +116,7 @@ def test_non_ascii_infile(self): ''').encode() infile = self._create_infile(data) - rc, out, err = assert_python_ok('-m', 'json.tool', infile) + rc, out, err = assert_python_ok('-m', self.module, infile) self.assertEqual(rc, 0) self.assertEqual(out.splitlines(), expect.splitlines()) @@ -124,7 +125,7 @@ def test_non_ascii_infile(self): def test_infile_outfile(self): infile = self._create_infile() outfile = os_helper.TESTFN + '.out' - rc, out, err = assert_python_ok('-m', 'json.tool', infile, outfile) + rc, out, err = assert_python_ok('-m', self.module, infile, outfile) self.addCleanup(os.remove, outfile) with open(outfile, "r", encoding="utf-8") as fp: self.assertEqual(fp.read(), self.expect) @@ -134,7 +135,7 @@ def test_infile_outfile(self): def test_writing_in_place(self): infile = self._create_infile() - rc, out, err = assert_python_ok('-m', 'json.tool', infile, infile) + rc, out, err = assert_python_ok('-m', self.module, infile, infile) with open(infile, "r", encoding="utf-8") as fp: self.assertEqual(fp.read(), self.expect) self.assertEqual(rc, 0) @@ -142,20 +143,20 @@ def test_writing_in_place(self): self.assertEqual(err, b'') def test_jsonlines(self): - args = sys.executable, '-m', 'json.tool', '--json-lines' + args = sys.executable, '-m', self.module, '--json-lines' process = subprocess.run(args, input=self.jsonlines_raw, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, self.jsonlines_expect) self.assertEqual(process.stderr, '') def test_help_flag(self): - rc, out, err = assert_python_ok('-m', 'json.tool', '-h') + rc, out, err = assert_python_ok('-m', self.module, '-h') self.assertEqual(rc, 0) self.assertTrue(out.startswith(b'usage: ')) self.assertEqual(err, b'') def test_sort_keys_flag(self): infile = self._create_infile() - rc, out, err = assert_python_ok('-m', 'json.tool', '--sort-keys', infile) + rc, out, err = assert_python_ok('-m', self.module, '--sort-keys', infile) self.assertEqual(rc, 0) self.assertEqual(out.splitlines(), self.expect_without_sort_keys.encode().splitlines()) @@ -169,7 +170,7 @@ def test_indent(self): 2 ] ''') - args = sys.executable, '-m', 'json.tool', '--indent', '2' + args = sys.executable, '-m', self.module, '--indent', '2' process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, expect) self.assertEqual(process.stderr, '') @@ -177,7 +178,7 @@ def test_indent(self): def test_no_indent(self): input_ = '[1,\n2]' expect = '[1, 2]\n' - args = sys.executable, '-m', 'json.tool', '--no-indent' + args = sys.executable, '-m', self.module, '--no-indent' process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, expect) self.assertEqual(process.stderr, '') @@ -185,7 +186,7 @@ def test_no_indent(self): def test_tab(self): input_ = '[1, 2]' expect = '[\n\t1,\n\t2\n]\n' - args = sys.executable, '-m', 'json.tool', '--tab' + args = sys.executable, '-m', self.module, '--tab' process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, expect) self.assertEqual(process.stderr, '') @@ -193,7 +194,7 @@ def test_tab(self): def test_compact(self): input_ = '[ 1 ,\n 2]' expect = '[1,2]\n' - args = sys.executable, '-m', 'json.tool', '--compact' + args = sys.executable, '-m', self.module, '--compact' process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, expect) self.assertEqual(process.stderr, '') @@ -202,7 +203,7 @@ def test_no_ensure_ascii_flag(self): infile = self._create_infile('{"key":"💩"}') outfile = os_helper.TESTFN + '.out' self.addCleanup(os.remove, outfile) - assert_python_ok('-m', 'json.tool', '--no-ensure-ascii', infile, outfile) + assert_python_ok('-m', self.module, '--no-ensure-ascii', infile, outfile) with open(outfile, "rb") as f: lines = f.read().splitlines() # asserting utf-8 encoded output file @@ -213,7 +214,7 @@ def test_ensure_ascii_default(self): infile = self._create_infile('{"key":"💩"}') outfile = os_helper.TESTFN + '.out' self.addCleanup(os.remove, outfile) - assert_python_ok('-m', 'json.tool', infile, outfile) + assert_python_ok('-m', self.module, infile, outfile) with open(outfile, "rb") as f: lines = f.read().splitlines() # asserting an ascii encoded output file @@ -222,11 +223,16 @@ def test_ensure_ascii_default(self): @unittest.skipIf(sys.platform =="win32", "The test is failed with ValueError on Windows") def test_broken_pipe_error(self): - cmd = [sys.executable, '-m', 'json.tool'] + cmd = [sys.executable, '-m', self.module] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE) - # bpo-39828: Closing before json.tool attempts to write into stdout. + # bpo-39828: Closing before json attempts to write into stdout. proc.stdout.close() proc.communicate(b'"{}"') self.assertEqual(proc.returncode, errno.EPIPE) + + +@support.requires_subprocess() +class TestTool(TestMain): + module = 'json.tool' diff --git a/Lib/test/test_json/test_unicode.py b/Lib/test/test_json/test_unicode.py index 2e8bba2775256a..68629cceeb9be9 100644 --- a/Lib/test/test_json/test_unicode.py +++ b/Lib/test/test_json/test_unicode.py @@ -20,12 +20,17 @@ def test_encoding4(self): def test_encoding5(self): u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' j = self.dumps(u, ensure_ascii=False) - self.assertEqual(j, '"{0}"'.format(u)) + self.assertEqual(j, f'"{u}"') def test_encoding6(self): u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' j = self.dumps([u], ensure_ascii=False) - self.assertEqual(j, '["{0}"]'.format(u)) + self.assertEqual(j, f'["{u}"]') + + def test_encoding7(self): + u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' + j = self.dumps(u + "\n", ensure_ascii=False) + self.assertEqual(j, f'"{u}\\n"') def test_big_unicode_encode(self): u = '\U0001d120' @@ -34,13 +39,13 @@ def test_big_unicode_encode(self): def test_big_unicode_decode(self): u = 'z\U0001d120x' - self.assertEqual(self.loads('"' + u + '"'), u) + self.assertEqual(self.loads(f'"{u}"'), u) self.assertEqual(self.loads('"z\\ud834\\udd20x"'), u) def test_unicode_decode(self): for i in range(0, 0xd7ff): u = chr(i) - s = '"\\u{0:04x}"'.format(i) + s = f'"\\u{i:04x}"' self.assertEqual(self.loads(s), u) def test_unicode_preservation(self): diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index 3b0930fe69e30e..41f7b70e5cfe81 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -2,7 +2,6 @@ """ import os -import stat import sys import unittest import socket @@ -29,7 +28,7 @@ def setUp(self): mode = 'w+b' with self.open(TESTFN, mode) as f: - current_size = os.fstat(f.fileno())[stat.ST_SIZE] + current_size = os.fstat(f.fileno()).st_size if current_size == size+1: return @@ -40,13 +39,13 @@ def setUp(self): f.seek(size) f.write(b'a') f.flush() - self.assertEqual(os.fstat(f.fileno())[stat.ST_SIZE], size+1) + self.assertEqual(os.fstat(f.fileno()).st_size, size+1) @classmethod def tearDownClass(cls): with cls.open(TESTFN, 'wb'): pass - if not os.stat(TESTFN)[stat.ST_SIZE] == 0: + if not os.stat(TESTFN).st_size == 0: raise cls.failureException('File was not truncated by opening ' 'with mode "wb"') unlink(TESTFN2) @@ -67,7 +66,7 @@ def test_large_read(self, _size): self.assertEqual(f.tell(), size + 1) def test_osstat(self): - self.assertEqual(os.stat(TESTFN)[stat.ST_SIZE], size+1) + self.assertEqual(os.stat(TESTFN).st_size, size+1) def test_seek_read(self): with self.open(TESTFN, 'rb') as f: @@ -142,6 +141,9 @@ def test_truncate(self): f.truncate(1) self.assertEqual(f.tell(), 0) # else pointer moved f.seek(0) + # Verify readall on a truncated file is well behaved. read() + # without a size can be unbounded, this should get just the byte + # that remains. self.assertEqual(len(f.read()), 1) # else wasn't truncated def test_seekable(self): @@ -152,6 +154,22 @@ def test_seekable(self): f.seek(pos) self.assertTrue(f.seekable()) + @bigmemtest(size=size, memuse=2, dry_run=False) + def test_seek_readall(self, _size): + # Seek which doesn't change position should readall successfully. + with self.open(TESTFN, 'rb') as f: + self.assertEqual(f.seek(0, os.SEEK_CUR), 0) + self.assertEqual(len(f.read()), size + 1) + + # Seek which changes (or might change) position should readall + # successfully. + with self.open(TESTFN, 'rb') as f: + self.assertEqual(f.seek(20, os.SEEK_SET), 20) + self.assertEqual(len(f.read()), size - 19) + + with self.open(TESTFN, 'rb') as f: + self.assertEqual(f.seek(-3, os.SEEK_END), size - 2) + self.assertEqual(len(f.read()), 3) def skip_no_disk_space(path, required): def decorator(fun): diff --git a/Lib/test/test_launcher.py b/Lib/test/test_launcher.py index 2528a51240fbf7..58baae25df3df7 100644 --- a/Lib/test/test_launcher.py +++ b/Lib/test/test_launcher.py @@ -764,3 +764,11 @@ def test_shebang_command_in_venv(self): with self.script(f'#! /usr/bin/env {exe.stem} arg1') as script: data = self.run_py([script], env=env) self.assertEqual(data["stdout"].strip(), f"{quote(exe)} arg1 {quote(script)}") + + def test_shebang_executable_extension(self): + with self.script('#! /usr/bin/env python3.99') as script: + data = self.run_py([script], expect_returncode=103) + expect = "# Search PATH for python3.99.exe" + actual = [line.strip() for line in data["stderr"].splitlines() + if line.startswith("# Search PATH")] + self.assertEqual([expect], actual) diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index e42df3d9496bc8..6f5955791407ea 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -83,6 +83,10 @@ def test_getlines(self): class EmptyFile(GetLineTestsGoodData, unittest.TestCase): file_list = [] + def test_getlines(self): + lines = linecache.getlines(self.file_name) + self.assertEqual(lines, ['\n']) + class SingleEmptyLine(GetLineTestsGoodData, unittest.TestCase): file_list = ['\n'] @@ -276,6 +280,37 @@ def test_loader(self): self.assertEqual(linecache.getlines(filename, module_globals), ['source for x.y.z\n']) + def test_invalid_names(self): + for name, desc in [ + ('\x00', 'NUL bytes filename'), + (__file__ + '\x00', 'filename with embedded NUL bytes'), + # A filename with surrogate codes. A UnicodeEncodeError is raised + # by os.stat() upon querying, which is a subclass of ValueError. + ("\uD834\uDD1E.py", 'surrogate codes (MUSICAL SYMBOL G CLEF)'), + # For POSIX platforms, an OSError will be raised but for Windows + # platforms, a ValueError is raised due to the path_t converter. + # See: https://github.com/python/cpython/issues/122170 + ('a' * 1_000_000, 'very long filename'), + ]: + with self.subTest(f'updatecache: {desc}'): + linecache.clearcache() + lines = linecache.updatecache(name) + self.assertListEqual(lines, []) + self.assertNotIn(name, linecache.cache) + + # hack into the cache (it shouldn't be allowed + # but we never know what people do...) + for key, fullname in [(name, 'ok'), ('key', name), (name, name)]: + with self.subTest(f'checkcache: {desc}', + key=key, fullname=fullname): + linecache.clearcache() + linecache.cache[key] = (0, 1234, [], fullname) + linecache.checkcache(key) + self.assertNotIn(key, linecache.cache) + + # just to be sure that we did not mess with cache + linecache.clearcache() + class LineCacheInvalidationTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 2969c6e2f98a23..ad7accf2099f43 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -96,6 +96,11 @@ def imul(a, b): a *= b self.assertRaises((MemoryError, OverflowError), mul, lst, n) self.assertRaises((MemoryError, OverflowError), imul, lst, n) + def test_empty_slice(self): + x = [] + x[:] = x + self.assertEqual(x, []) + def test_list_resize_overflow(self): # gh-97616: test new_allocated * sizeof(PyObject*) overflow # check in list_resize() @@ -229,6 +234,31 @@ def __eq__(self, other): list4 = [1] self.assertFalse(list3 == list4) + def test_lt_operator_modifying_operand(self): + # See gh-120298 + class evil: + def __lt__(self, other): + other.clear() + return NotImplemented + + a = [[evil()]] + with self.assertRaises(TypeError): + a[0] < a + + def test_list_index_modifing_operand(self): + # See gh-120384 + class evil: + def __init__(self, lst): + self.lst = lst + def __iter__(self): + yield from self.lst + self.lst.clear() + + lst = list(range(5)) + operand = evil(lst) + with self.assertRaises(ValueError): + lst[::-1] = operand + @cpython_only def test_preallocation(self): iterable = [0] * 10 @@ -269,6 +299,15 @@ def __eq__(self, other): lst = [X(), X()] X() in lst + def test_tier2_invalidates_iterator(self): + # GH-121012 + for _ in range(100): + a = [1, 2, 3] + it = iter(a) + for _ in it: + pass + a.append(4) + self.assertEqual(list(it), []) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 2868dd01545b95..cffdeeacc5d73b 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -1,8 +1,11 @@ import doctest import textwrap +import traceback import types import unittest +from test.support import BrokenIter + doctests = """ ########### Tests borrowed from or inspired by test_genexps.py ############ @@ -168,6 +171,31 @@ def test_references___class__(self): """ self._check_in_scopes(code, raises=NameError) + def test_references___class___defined(self): + code = """ + __class__ = 2 + res = [__class__ for x in [1]] + """ + self._check_in_scopes( + code, outputs={"res": [2]}, scopes=["module", "function"]) + self._check_in_scopes(code, raises=NameError, scopes=["class"]) + + def test_references___class___enclosing(self): + code = """ + __class__ = 2 + class C: + res = [__class__ for x in [1]] + res = C.res + """ + self._check_in_scopes(code, raises=NameError) + + def test_super_and_class_cell_in_sibling_comps(self): + code = """ + [super for _ in [1]] + [__class__ for _ in [1]] + """ + self._check_in_scopes(code, raises=NameError) + def test_inner_cell_shadows_outer(self): code = """ items = [(lambda: i) for i in range(5)] @@ -581,7 +609,7 @@ def test_comp_in_try_except(self): result = snapshot = None try: result = [{func}(value) for value in value] - except: + except ValueError: snapshot = value raise """ @@ -615,16 +643,21 @@ def test_exception_in_post_comp_call(self): value = [1, None] try: [v for v in value].sort() - except: + except TypeError: pass """ self._check_in_scopes(code, {"value": [1, None]}) def test_frame_locals(self): code = """ - val = [sys._getframe().f_locals for a in [0]][0]["a"] + val = "a" in [sys._getframe().f_locals for a in [0]][0] """ import sys + self._check_in_scopes(code, {"val": False}, ns={"sys": sys}) + + code = """ + val = [sys._getframe().f_locals["a"] for a in [0]][0] + """ self._check_in_scopes(code, {"val": 0}, ns={"sys": sys}) def _recursive_replace(self, maybe_code): @@ -666,6 +699,56 @@ def test_code_replace_extended_arg(self): self._check_in_scopes(code, expected) self._check_in_scopes(code, expected, exec_func=self._replacing_exec) + def test_multiple_comprehension_name_reuse(self): + code = """ + [x for x in [1]] + y = [x for _ in [1]] + """ + self._check_in_scopes(code, {"y": [3]}, ns={"x": 3}) + + code = """ + x = 2 + [x for x in [1]] + y = [x for _ in [1]] + """ + self._check_in_scopes(code, {"x": 2, "y": [3]}, ns={"x": 3}, scopes=["class"]) + self._check_in_scopes(code, {"x": 2, "y": [2]}, ns={"x": 3}, scopes=["function", "module"]) + + def test_exception_locations(self): + # The location of an exception raised from __init__ or + # __next__ should should be the iterator expression + + def init_raises(): + try: + [x for x in BrokenIter(init_raises=True)] + except Exception as e: + return e + + def next_raises(): + try: + [x for x in BrokenIter(next_raises=True)] + except Exception as e: + return e + + def iter_raises(): + try: + [x for x in BrokenIter(iter_raises=True)] + except Exception as e: + return e + + for func, expected in [(init_raises, "BrokenIter(init_raises=True)"), + (next_raises, "BrokenIter(next_raises=True)"), + (iter_raises, "BrokenIter(iter_raises=True)"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) __test__ = {'doctests' : doctests} diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index da4bd79746a476..c025ed4108fb58 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -1,5 +1,5 @@ from decimal import Decimal -from test.support import verbose, is_android, is_emscripten, is_wasi +from test.support import verbose, is_android, is_emscripten, is_wasi, os_helper from test.support.warnings_helper import check_warnings from test.support.import_helper import import_fresh_module from unittest import mock @@ -355,6 +355,8 @@ def setUp(self): is_emscripten or is_wasi, "musl libc issue on Emscripten/WASI, bpo-46390" ) + @unittest.skipIf(sys.platform.startswith("netbsd"), + "gh-124108: NetBSD doesn't support UTF-8 for LC_COLLATE") def test_strcoll_with_diacritic(self): self.assertLess(locale.strcoll('à', 'b'), 0) @@ -364,6 +366,8 @@ def test_strcoll_with_diacritic(self): is_emscripten or is_wasi, "musl libc issue on Emscripten/WASI, bpo-46390" ) + @unittest.skipIf(sys.platform.startswith("netbsd"), + "gh-124108: NetBSD doesn't support UTF-8 for LC_COLLATE") def test_strxfrm_with_diacritic(self): self.assertLess(locale.strxfrm('à'), locale.strxfrm('b')) @@ -495,25 +499,16 @@ def test_defaults_UTF8(self): else: orig_getlocale = None - orig_env = {} try: - for key in ('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE'): - if key in os.environ: - orig_env[key] = os.environ[key] - del os.environ[key] + with os_helper.EnvironmentVarGuard() as env: + for key in ('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE'): + env.unset(key) - os.environ['LC_CTYPE'] = 'UTF-8' - - with check_warnings(('', DeprecationWarning)): - self.assertEqual(locale.getdefaultlocale(), (None, 'UTF-8')) + env.set('LC_CTYPE', 'UTF-8') + with check_warnings(('', DeprecationWarning)): + self.assertEqual(locale.getdefaultlocale(), (None, 'UTF-8')) finally: - for k in orig_env: - os.environ[k] = orig_env[k] - - if 'LC_CTYPE' not in orig_env: - del os.environ['LC_CTYPE'] - if orig_getlocale is not None: _locale._getdefaultlocale = orig_getlocale diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index d71385bf2c78d7..e34fe45fd68e52 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -22,6 +22,7 @@ import logging.handlers import logging.config + import codecs import configparser import copy @@ -60,6 +61,7 @@ import weakref from http.server import HTTPServer, BaseHTTPRequestHandler +from unittest.mock import patch from urllib.parse import urlparse, parse_qs from socketserver import (ThreadingUDPServer, DatagramRequestHandler, ThreadingTCPServer, StreamRequestHandler) @@ -80,6 +82,9 @@ skip_if_asan_fork = unittest.skipIf( support.HAVE_ASAN_FORK_BUG, "libasan has a pthread_create() dead lock related to thread+fork") +skip_if_tsan_fork = unittest.skipIf( + support.check_sanitizer(thread=True), + "TSAN doesn't support threads after fork") class BaseTest(unittest.TestCase): @@ -603,7 +608,7 @@ def test_name(self): def test_builtin_handlers(self): # We can't actually *use* too many handlers in the tests, # but we can try instantiating them with various options - if sys.platform in ('linux', 'darwin'): + if sys.platform in ('linux', 'android', 'darwin'): for existing in (True, False): fn = make_temp_file() if not existing: @@ -653,21 +658,21 @@ def test_builtin_handlers(self): self.assertFalse(h.shouldFlush(r)) h.close() - def test_path_objects(self): + def test_pathlike_objects(self): """ - Test that Path objects are accepted as filename arguments to handlers. + Test that path-like objects are accepted as filename arguments to handlers. See Issue #27493. """ fn = make_temp_file() os.unlink(fn) - pfn = pathlib.Path(fn) + pfn = os_helper.FakePath(fn) cases = ( (logging.FileHandler, (pfn, 'w')), (logging.handlers.RotatingFileHandler, (pfn, 'a')), (logging.handlers.TimedRotatingFileHandler, (pfn, 'h')), ) - if sys.platform in ('linux', 'darwin'): + if sys.platform in ('linux', 'android', 'darwin'): cases += ((logging.handlers.WatchedFileHandler, (pfn, 'w')),) for cls, args in cases: h = cls(*args, encoding="utf-8") @@ -676,9 +681,6 @@ def test_path_objects(self): os.unlink(fn) @unittest.skipIf(os.name == 'nt', 'WatchedFileHandler not appropriate for Windows.') - @unittest.skipIf( - support.is_emscripten, "Emscripten cannot fstat unlinked files." - ) @threading_helper.requires_working_threading() @support.requires_resource('walltime') def test_race(self): @@ -731,6 +733,7 @@ def remove_loop(fname, tries): @support.requires_fork() @threading_helper.requires_working_threading() @skip_if_asan_fork + @skip_if_tsan_fork def test_post_fork_child_no_deadlock(self): """Ensure child logging locks are not held; bpo-6721 & bpo-36533.""" class _OurHandler(logging.Handler): @@ -1033,6 +1036,7 @@ class TestTCPServer(ControlMixin, ThreadingTCPServer): """ allow_reuse_address = True + allow_reuse_port = True def __init__(self, addr, handler, poll_interval=0.5, bind_and_activate=True): @@ -1131,7 +1135,7 @@ def test_basic(self): self.assertEqual(mailfrom, 'me') self.assertEqual(rcpttos, ['you']) self.assertIn('\nSubject: Log\n', data) - self.assertTrue(data.endswith('\n\nHello \u2713')) + self.assertEndsWith(data, '\n\nHello \u2713') h.close() def process_message(self, *args): @@ -2092,6 +2096,18 @@ def test_udp_reconnection(self): self.handled.wait(support.LONG_TIMEOUT) self.assertEqual(self.log_output, b'<11>sp\xc3\xa4m\x00') + @patch('socket.socket') + def test_tcp_timeout(self, mock_socket): + instance_mock_sock = mock_socket.return_value + instance_mock_sock.connect.side_effect = socket.timeout + + with self.assertRaises(socket.timeout): + logging.handlers.SysLogHandler(address=('localhost', 514), + socktype=socket.SOCK_STREAM, + timeout=1) + + instance_mock_sock.close.assert_called() + @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "Unix sockets required") class UnixSysLogHandlerTest(SysLogHandlerTest): @@ -2187,7 +2203,8 @@ def test_output(self): self.handled.clear() msg = "sp\xe4m" logger.error(msg) - self.handled.wait() + handled = self.handled.wait(support.SHORT_TIMEOUT) + self.assertTrue(handled, "HTTP request timed out") self.assertEqual(self.log_data.path, '/frob') self.assertEqual(self.command, method) if method == 'GET': @@ -2361,6 +2378,32 @@ class CustomListener(logging.handlers.QueueListener): class CustomQueue(queue.Queue): pass +class CustomQueueProtocol: + def __init__(self, maxsize=0): + self.queue = queue.Queue(maxsize) + + def __getattr__(self, attribute): + queue = object.__getattribute__(self, 'queue') + return getattr(queue, attribute) + +class CustomQueueFakeProtocol(CustomQueueProtocol): + # An object implementing the minimial Queue API for + # the logging module but with incorrect signatures. + # + # The object will be considered a valid queue class since we + # do not check the signatures (only callability of methods) + # but will NOT be usable in production since a TypeError will + # be raised due to the extra argument in 'put_nowait'. + def put_nowait(self): + pass + +class CustomQueueWrongProtocol(CustomQueueProtocol): + put_nowait = None + +class MinimalQueueProtocol: + def put_nowait(self, x): pass + def get(self): pass + def queueMaker(): return queue.Queue() @@ -3030,6 +3073,30 @@ def format(self, record): }, } + config18 = { + "version": 1, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "DEBUG", + }, + "buffering": { + "class": "logging.handlers.MemoryHandler", + "capacity": 5, + "target": "console", + "level": "DEBUG", + "flushLevel": "ERROR" + } + }, + "loggers": { + "mymodule": { + "level": "DEBUG", + "handlers": ["buffering"], + "propagate": "true" + } + } + } + bad_format = { "version": 1, "formatters": { @@ -3470,7 +3537,7 @@ def test_config14_ok(self): self.assertEqual(h.foo, 'bar') self.assertEqual(h.terminator, '!\n') logging.warning('Exclamation') - self.assertTrue(output.getvalue().endswith('Exclamation!\n')) + self.assertEndsWith(output.getvalue(), 'Exclamation!\n') def test_config15_ok(self): @@ -3516,6 +3583,11 @@ def test_config17_ok(self): h = logging._handlers['hand1'] self.assertEqual(h.formatter.custom_property, 'value') + def test_config18_ok(self): + self.apply_config(self.config18) + handler = logging.getLogger('mymodule').handlers[0] + self.assertEqual(handler.flushLevel, logging.ERROR) + def setup_via_listener(self, text, verify=None): text = text.encode("utf-8") # Ask for a randomly assigned port (by using port 0) @@ -3693,16 +3765,16 @@ def test_baseconfig(self): 'adict': { 'd': 'e', 'f': 3 , 'alpha numeric 1 with spaces' : 5, - 'aplha numeric 1 %( - © ©ß¯' : 9, + 'alpha numeric 1 %( - © ©ß¯' : 9, 'alpha numeric ] 1 with spaces' : 15, - 'aplha ]] numeric 1 %( - © ©ß¯]' : 19, - ' aplha [ numeric 1 %( - © ©ß¯] ' : 11, - ' aplha ' : 32, + 'alpha ]] numeric 1 %( - © ©ß¯]' : 19, + ' alpha [ numeric 1 %( - © ©ß¯] ' : 11, + ' alpha ' : 32, '' : 10, 'nest4' : { 'd': 'e', 'f': 3 , 'alpha numeric 1 with spaces' : 5, - 'aplha numeric 1 %( - © ©ß¯' : 9, + 'alpha numeric 1 %( - © ©ß¯' : 9, '' : 10, 'somelist' : ('g', ('h', 'i'), 'j'), 'somedict' : { @@ -3724,14 +3796,14 @@ def test_baseconfig(self): self.assertEqual(bc.convert('cfg://adict.d'), 'e') self.assertEqual(bc.convert('cfg://adict[f]'), 3) self.assertEqual(bc.convert('cfg://adict[alpha numeric 1 with spaces]'), 5) - self.assertEqual(bc.convert('cfg://adict[aplha numeric 1 %( - © ©ß¯]'), 9) + self.assertEqual(bc.convert('cfg://adict[alpha numeric 1 %( - © ©ß¯]'), 9) self.assertEqual(bc.convert('cfg://adict[]'), 10) self.assertEqual(bc.convert('cfg://adict.nest4.d'), 'e') self.assertEqual(bc.convert('cfg://adict.nest4[d]'), 'e') self.assertEqual(bc.convert('cfg://adict[nest4].d'), 'e') self.assertEqual(bc.convert('cfg://adict[nest4][f]'), 3) self.assertEqual(bc.convert('cfg://adict[nest4][alpha numeric 1 with spaces]'), 5) - self.assertEqual(bc.convert('cfg://adict[nest4][aplha numeric 1 %( - © ©ß¯]'), 9) + self.assertEqual(bc.convert('cfg://adict[nest4][alpha numeric 1 %( - © ©ß¯]'), 9) self.assertEqual(bc.convert('cfg://adict[nest4][]'), 10) self.assertEqual(bc.convert('cfg://adict[nest4][somelist][0]'), 'g') self.assertEqual(bc.convert('cfg://adict[nest4][somelist][1][0]'), 'h') @@ -3751,8 +3823,8 @@ def test_baseconfig(self): self.assertRaises(ValueError, bc.convert, 'cfg://!') self.assertRaises(KeyError, bc.convert, 'cfg://adict[2]') self.assertRaises(KeyError, bc.convert, 'cfg://adict[alpha numeric ] 1 with spaces]') - self.assertRaises(ValueError, bc.convert, 'cfg://adict[ aplha ]] numeric 1 %( - © ©ß¯] ]') - self.assertRaises(ValueError, bc.convert, 'cfg://adict[ aplha [ numeric 1 %( - © ©ß¯] ]') + self.assertRaises(ValueError, bc.convert, 'cfg://adict[ alpha ]] numeric 1 %( - © ©ß¯] ]') + self.assertRaises(ValueError, bc.convert, 'cfg://adict[ alpha [ numeric 1 %( - © ©ß¯] ]') def test_namedtuple(self): # see bpo-39142 @@ -3863,19 +3935,18 @@ def do_queuehandler_configuration(self, qspec, lspec): self.addCleanup(os.remove, fn) @threading_helper.requires_working_threading() + @support.requires_subprocess() def test_config_queue_handler(self): - q = CustomQueue() - dq = { - '()': __name__ + '.CustomQueue', - 'maxsize': 10 - } + qs = [CustomQueue(), CustomQueueProtocol()] + dqs = [{'()': f'{__name__}.{cls}', 'maxsize': 10} + for cls in ['CustomQueue', 'CustomQueueProtocol']] dl = { '()': __name__ + '.listenerMaker', 'arg1': None, 'arg2': None, 'respect_handler_level': True } - qvalues = (None, __name__ + '.queueMaker', __name__ + '.CustomQueue', dq, q) + qvalues = (None, __name__ + '.queueMaker', __name__ + '.CustomQueue', *dqs, *qs) lvalues = (None, __name__ + '.CustomListener', dl, CustomListener) for qspec, lspec in itertools.product(qvalues, lvalues): self.do_queuehandler_configuration(qspec, lspec) @@ -3891,6 +3962,140 @@ def test_config_queue_handler(self): msg = str(ctx.exception) self.assertEqual(msg, "Unable to configure handler 'ah'") + def _apply_simple_queue_listener_configuration(self, qspec): + self.apply_config({ + "version": 1, + "handlers": { + "queue_listener": { + "class": "logging.handlers.QueueHandler", + "queue": qspec, + }, + }, + }) + + @threading_helper.requires_working_threading() + @support.requires_subprocess() + @patch("multiprocessing.Manager") + def test_config_queue_handler_does_not_create_multiprocessing_manager(self, manager): + # gh-120868, gh-121723, gh-124653 + + for qspec in [ + {"()": "queue.Queue", "maxsize": -1}, + queue.Queue(), + # queue.SimpleQueue does not inherit from queue.Queue + queue.SimpleQueue(), + # CustomQueueFakeProtocol passes the checks but will not be usable + # since the signatures are incompatible. Checking the Queue API + # without testing the type of the actual queue is a trade-off + # between usability and the work we need to do in order to safely + # check that the queue object correctly implements the API. + CustomQueueFakeProtocol(), + MinimalQueueProtocol(), + ]: + with self.subTest(qspec=qspec): + self._apply_simple_queue_listener_configuration(qspec) + manager.assert_not_called() + + @patch("multiprocessing.Manager") + def test_config_queue_handler_invalid_config_does_not_create_multiprocessing_manager(self, manager): + # gh-120868, gh-121723 + + for qspec in [object(), CustomQueueWrongProtocol()]: + with self.subTest(qspec=qspec), self.assertRaises(ValueError): + self._apply_simple_queue_listener_configuration(qspec) + manager.assert_not_called() + + @skip_if_tsan_fork + @support.requires_subprocess() + @unittest.skipUnless(support.Py_DEBUG, "requires a debug build for testing" + " assertions in multiprocessing") + def test_config_reject_simple_queue_handler_multiprocessing_context(self): + # multiprocessing.SimpleQueue does not implement 'put_nowait' + # and thus cannot be used as a queue-like object (gh-124653) + + import multiprocessing + + if support.MS_WINDOWS: + start_methods = ['spawn'] + else: + start_methods = ['spawn', 'fork', 'forkserver'] + + for start_method in start_methods: + with self.subTest(start_method=start_method): + ctx = multiprocessing.get_context(start_method) + qspec = ctx.SimpleQueue() + with self.assertRaises(ValueError): + self._apply_simple_queue_listener_configuration(qspec) + + @skip_if_tsan_fork + @support.requires_subprocess() + @unittest.skipUnless(support.Py_DEBUG, "requires a debug build for testing" + " assertions in multiprocessing") + def test_config_queue_handler_multiprocessing_context(self): + # regression test for gh-121723 + if support.MS_WINDOWS: + start_methods = ['spawn'] + else: + start_methods = ['spawn', 'fork', 'forkserver'] + for start_method in start_methods: + with self.subTest(start_method=start_method): + ctx = multiprocessing.get_context(start_method) + with ctx.Manager() as manager: + q = manager.Queue() + records = [] + # use 1 process and 1 task per child to put 1 record + with ctx.Pool(1, initializer=self._mpinit_issue121723, + initargs=(q, "text"), maxtasksperchild=1): + records.append(q.get(timeout=60)) + self.assertTrue(q.empty()) + self.assertEqual(len(records), 1) + + @staticmethod + def _mpinit_issue121723(qspec, message_to_log): + # static method for pickling support + logging.config.dictConfig({ + 'version': 1, + 'disable_existing_loggers': True, + 'handlers': { + 'log_to_parent': { + 'class': 'logging.handlers.QueueHandler', + 'queue': qspec + } + }, + 'root': {'handlers': ['log_to_parent'], 'level': 'DEBUG'} + }) + # log a message (this creates a record put in the queue) + logging.getLogger().info(message_to_log) + + @skip_if_tsan_fork + @support.requires_subprocess() + def test_multiprocessing_queues(self): + # See gh-119819 + + cd = copy.deepcopy(self.config_queue_handler) + from multiprocessing import Queue as MQ, Manager as MM + q1 = MQ() # this can't be pickled + q2 = MM().Queue() # a proxy queue for use when pickling is needed + q3 = MM().JoinableQueue() # a joinable proxy queue + for qspec in (q1, q2, q3): + fn = make_temp_file('.log', 'test_logging-cmpqh-') + cd['handlers']['h1']['filename'] = fn + cd['handlers']['ah']['queue'] = qspec + qh = None + try: + self.apply_config(cd) + qh = logging.getHandlerByName('ah') + self.assertEqual(sorted(logging.getHandlerNames()), ['ah', 'h1']) + self.assertIsNotNone(qh.listener) + self.assertIs(qh.queue, qspec) + self.assertIs(qh.listener.queue, qspec) + finally: + h = logging.getHandlerByName('h1') + if h: + self.addCleanup(closeFileHandler, h, fn) + else: + self.addCleanup(os.remove, fn) + def test_90195(self): # See gh-90195 config = { @@ -3941,6 +4146,35 @@ def test_111615(self): } logging.config.dictConfig(config) + # gh-118868: check if kwargs are passed to logging QueueHandler + def test_kwargs_passing(self): + class CustomQueueHandler(logging.handlers.QueueHandler): + def __init__(self, *args, **kwargs): + super().__init__(queue.Queue()) + self.custom_kwargs = kwargs + + custom_kwargs = {'foo': 'bar'} + + config = { + 'version': 1, + 'handlers': { + 'custom': { + 'class': CustomQueueHandler, + **custom_kwargs + }, + }, + 'root': { + 'level': 'DEBUG', + 'handlers': ['custom'] + } + } + + logging.config.dictConfig(config) + + handler = logging.getHandlerByName('custom') + self.assertEqual(handler.custom_kwargs, custom_kwargs) + + class ManagerTest(BaseTest): def test_manager_loggerclass(self): logged = [] @@ -4060,7 +4294,7 @@ def test_queue_handler(self): msg = self.next_message() self.que_logger.warning(msg) data = self.queue.get_nowait() - self.assertTrue(isinstance(data, logging.LogRecord)) + self.assertIsInstance(data, logging.LogRecord) self.assertEqual(data.name, self.que_logger.name) self.assertEqual((data.msg, data.args), (msg, None)) @@ -4146,6 +4380,7 @@ def test_queue_listener_with_multiple_handlers(self): import multiprocessing from unittest.mock import patch + @skip_if_tsan_fork @threading_helper.requires_working_threading() class QueueListenerTest(BaseTest): """ @@ -4547,6 +4782,77 @@ def test_issue_89047(self): s = f.format(r) self.assertNotIn('.1000', s) + def test_msecs_has_no_floating_point_precision_loss(self): + # See issue gh-102402 + tests = ( + # time_ns is approx. 2023-03-04 04:25:20 UTC + # (time_ns, expected_msecs_value) + (1_677_902_297_100_000_000, 100.0), # exactly 100ms + (1_677_903_920_999_998_503, 999.0), # check truncating doesn't round + (1_677_903_920_000_998_503, 0.0), # check truncating doesn't round + (1_677_903_920_999_999_900, 0.0), # check rounding up + ) + for ns, want in tests: + with patch('time.time_ns') as patched_ns: + patched_ns.return_value = ns + record = logging.makeLogRecord({'msg': 'test'}) + with self.subTest(ns): + self.assertEqual(record.msecs, want) + self.assertEqual(record.created, ns / 1e9) + self.assertAlmostEqual(record.created - int(record.created), + record.msecs / 1e3, + delta=1e-3) + + def test_relativeCreated_has_higher_precision(self): + # See issue gh-102402. + # Run the code in the subprocess, because the time module should + # be patched before the first import of the logging package. + # Temporary unloading and re-importing the logging package has + # side effects (including registering the atexit callback and + # references leak). + start_ns = 1_677_903_920_000_998_503 # approx. 2023-03-04 04:25:20 UTC + offsets_ns = (200, 500, 12_354, 99_999, 1_677_903_456_999_123_456) + code = textwrap.dedent(f""" + start_ns = {start_ns!r} + offsets_ns = {offsets_ns!r} + start_monotonic_ns = start_ns - 1 + + import time + # Only time.time_ns needs to be patched for the current + # implementation, but patch also other functions to make + # the test less implementation depending. + old_time_ns = time.time_ns + old_time = time.time + old_monotonic_ns = time.monotonic_ns + old_monotonic = time.monotonic + time_ns_result = start_ns + time.time_ns = lambda: time_ns_result + time.time = lambda: time.time_ns()/1e9 + time.monotonic_ns = lambda: time_ns_result - start_monotonic_ns + time.monotonic = lambda: time.monotonic_ns()/1e9 + try: + import logging + + for offset_ns in offsets_ns: + # mock for log record creation + time_ns_result = start_ns + offset_ns + record = logging.makeLogRecord({{'msg': 'test'}}) + print(record.created, record.relativeCreated) + finally: + time.time_ns = old_time_ns + time.time = old_time + time.monotonic_ns = old_monotonic_ns + time.monotonic = old_monotonic + """) + rc, out, err = assert_python_ok("-c", code) + out = out.decode() + for offset_ns, line in zip(offsets_ns, out.splitlines(), strict=True): + with self.subTest(offset_ns=offset_ns): + created, relativeCreated = map(float, line.split()) + self.assertAlmostEqual(created, (start_ns + offset_ns) / 1e9, places=6) + # After PR gh-102412, precision (places) increases from 3 to 7 + self.assertAlmostEqual(relativeCreated, offset_ns / 1e6, places=7) + class TestBufferingFormatter(logging.BufferingFormatter): def formatHeader(self, records): @@ -4581,19 +4887,19 @@ def test_formatting(self): r.addHandler(h) try: raise RuntimeError('deliberate mistake') - except: + except RuntimeError: logging.exception('failed', stack_info=True) r.removeHandler(h) h.close() r = h.records[0] - self.assertTrue(r.exc_text.startswith('Traceback (most recent ' - 'call last):\n')) - self.assertTrue(r.exc_text.endswith('\nRuntimeError: ' - 'deliberate mistake')) - self.assertTrue(r.stack_info.startswith('Stack (most recent ' - 'call last):\n')) - self.assertTrue(r.stack_info.endswith('logging.exception(\'failed\', ' - 'stack_info=True)')) + self.assertStartsWith(r.exc_text, + 'Traceback (most recent call last):\n') + self.assertEndsWith(r.exc_text, + '\nRuntimeError: deliberate mistake') + self.assertStartsWith(r.stack_info, + 'Stack (most recent call last):\n') + self.assertEndsWith(r.stack_info, + "logging.exception('failed', stack_info=True)") class LastResortTest(BaseTest): @@ -4936,8 +5242,8 @@ class LogRecordTest(BaseTest): def test_str_rep(self): r = logging.makeLogRecord({}) s = str(r) - self.assertTrue(s.startswith('')) + self.assertStartsWith(s, '') def test_dict_arg(self): h = RecordingHandler() @@ -4975,6 +5281,7 @@ def _extract_logrecord_process_name(key, logMultiprocessing, conn=None): else: return results + @skip_if_tsan_fork def test_multiprocessing(self): support.skip_if_broken_multiprocessing_synchronize() multiprocessing_imported = 'multiprocessing' in sys.modules @@ -5058,7 +5365,7 @@ def test_taskName_with_asyncio_imported(self): logging.logAsyncioTasks = False runner.run(make_record(self.assertIsNone)) finally: - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) @support.requires_working_socket() def test_taskName_without_asyncio_imported(self): @@ -5070,7 +5377,7 @@ def test_taskName_without_asyncio_imported(self): logging.logAsyncioTasks = False runner.run(make_record(self.assertIsNone)) finally: - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class BasicConfigTest(unittest.TestCase): @@ -5374,7 +5681,7 @@ async def log_record(): data = f.read().strip() self.assertRegex(data, r'Task-\d+ - hello world') finally: - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) if handler: handler.close() @@ -5586,14 +5893,14 @@ def test_extra_in_records(self): self.adapter.critical('foo should be here') self.assertEqual(len(self.recording.records), 1) record = self.recording.records[0] - self.assertTrue(hasattr(record, 'foo')) + self.assertHasAttr(record, 'foo') self.assertEqual(record.foo, '1') def test_extra_not_merged_by_default(self): self.adapter.critical('foo should NOT be here', extra={'foo': 'nope'}) self.assertEqual(len(self.recording.records), 1) record = self.recording.records[0] - self.assertFalse(hasattr(record, 'foo')) + self.assertNotHasAttr(record, 'foo') def test_extra_merged(self): self.adapter = logging.LoggerAdapter(logger=self.logger, @@ -5603,8 +5910,8 @@ def test_extra_merged(self): self.adapter.critical('foo and bar should be here', extra={'bar': '2'}) self.assertEqual(len(self.recording.records), 1) record = self.recording.records[0] - self.assertTrue(hasattr(record, 'foo')) - self.assertTrue(hasattr(record, 'bar')) + self.assertHasAttr(record, 'foo') + self.assertHasAttr(record, 'bar') self.assertEqual(record.foo, '1') self.assertEqual(record.bar, '2') @@ -5616,7 +5923,7 @@ def test_extra_merged_log_call_has_precedence(self): self.adapter.critical('foo shall be min', extra={'foo': '2'}) self.assertEqual(len(self.recording.records), 1) record = self.recording.records[0] - self.assertTrue(hasattr(record, 'foo')) + self.assertHasAttr(record, 'foo') self.assertEqual(record.foo, '2') @@ -5919,13 +6226,28 @@ def test_emit_after_closing_in_write_mode(self): self.assertEqual(fp.read().strip(), '1') class RotatingFileHandlerTest(BaseFileTest): - @unittest.skipIf(support.is_wasi, "WASI does not have /dev/null.") def test_should_not_rollover(self): - # If maxbytes is zero rollover never occurs + # If file is empty rollover never occurs + rh = logging.handlers.RotatingFileHandler( + self.fn, encoding="utf-8", maxBytes=1) + self.assertFalse(rh.shouldRollover(None)) + rh.close() + + # If maxBytes is zero rollover never occurs rh = logging.handlers.RotatingFileHandler( self.fn, encoding="utf-8", maxBytes=0) self.assertFalse(rh.shouldRollover(None)) rh.close() + + with open(self.fn, 'wb') as f: + f.write(b'\n') + rh = logging.handlers.RotatingFileHandler( + self.fn, encoding="utf-8", maxBytes=0) + self.assertFalse(rh.shouldRollover(None)) + rh.close() + + @unittest.skipIf(support.is_wasi, "WASI does not have /dev/null.") + def test_should_not_rollover_non_file(self): # bpo-45401 - test with special file # We set maxBytes to 1 so that rollover would normally happen, except # for the check for regular files @@ -5935,18 +6257,47 @@ def test_should_not_rollover(self): rh.close() def test_should_rollover(self): - rh = logging.handlers.RotatingFileHandler(self.fn, encoding="utf-8", maxBytes=1) + with open(self.fn, 'wb') as f: + f.write(b'\n') + rh = logging.handlers.RotatingFileHandler(self.fn, encoding="utf-8", maxBytes=2) self.assertTrue(rh.shouldRollover(self.next_rec())) rh.close() def test_file_created(self): # checks that the file is created and assumes it was created # by us + os.unlink(self.fn) rh = logging.handlers.RotatingFileHandler(self.fn, encoding="utf-8") rh.emit(self.next_rec()) self.assertLogFile(self.fn) rh.close() + def test_max_bytes(self, delay=False): + kwargs = {'delay': delay} if delay else {} + os.unlink(self.fn) + rh = logging.handlers.RotatingFileHandler( + self.fn, encoding="utf-8", backupCount=2, maxBytes=100, **kwargs) + self.assertIs(os.path.exists(self.fn), not delay) + small = logging.makeLogRecord({'msg': 'a'}) + large = logging.makeLogRecord({'msg': 'b'*100}) + self.assertFalse(rh.shouldRollover(small)) + self.assertFalse(rh.shouldRollover(large)) + rh.emit(small) + self.assertLogFile(self.fn) + self.assertFalse(os.path.exists(self.fn + ".1")) + self.assertFalse(rh.shouldRollover(small)) + self.assertTrue(rh.shouldRollover(large)) + rh.emit(large) + self.assertTrue(os.path.exists(self.fn)) + self.assertLogFile(self.fn + ".1") + self.assertFalse(os.path.exists(self.fn + ".2")) + self.assertTrue(rh.shouldRollover(small)) + self.assertTrue(rh.shouldRollover(large)) + rh.close() + + def test_max_bytes_delay(self): + self.test_max_bytes(delay=True) + def test_rollover_filenames(self): def namer(name): return name + ".test" @@ -5955,11 +6306,15 @@ def namer(name): rh.namer = namer rh.emit(self.next_rec()) self.assertLogFile(self.fn) + self.assertFalse(os.path.exists(namer(self.fn + ".1"))) rh.emit(self.next_rec()) self.assertLogFile(namer(self.fn + ".1")) + self.assertFalse(os.path.exists(namer(self.fn + ".2"))) rh.emit(self.next_rec()) self.assertLogFile(namer(self.fn + ".2")) self.assertFalse(os.path.exists(namer(self.fn + ".3"))) + rh.emit(self.next_rec()) + self.assertFalse(os.path.exists(namer(self.fn + ".3"))) rh.close() def test_namer_rotator_inheritance(self): @@ -6080,46 +6435,22 @@ def test_rollover(self): print(tf.read()) self.assertTrue(found, msg=msg) - def test_rollover_at_midnight(self): - atTime = datetime.datetime.now().time() - fmt = logging.Formatter('%(asctime)s %(message)s') - for i in range(3): - fh = logging.handlers.TimedRotatingFileHandler( - self.fn, encoding="utf-8", when='MIDNIGHT', atTime=atTime) - fh.setFormatter(fmt) - r2 = logging.makeLogRecord({'msg': f'testing1 {i}'}) - fh.emit(r2) - fh.close() - self.assertLogFile(self.fn) - with open(self.fn, encoding="utf-8") as f: - for i, line in enumerate(f): - self.assertIn(f'testing1 {i}', line) - - os.utime(self.fn, (time.time() - 1,)*2) - for i in range(2): - fh = logging.handlers.TimedRotatingFileHandler( - self.fn, encoding="utf-8", when='MIDNIGHT', atTime=atTime) - fh.setFormatter(fmt) - r2 = logging.makeLogRecord({'msg': f'testing2 {i}'}) - fh.emit(r2) - fh.close() - rolloverDate = datetime.datetime.now() - datetime.timedelta(days=1) - otherfn = f'{self.fn}.{rolloverDate:%Y-%m-%d}' - self.assertLogFile(otherfn) - with open(self.fn, encoding="utf-8") as f: - for i, line in enumerate(f): - self.assertIn(f'testing2 {i}', line) - with open(otherfn, encoding="utf-8") as f: - for i, line in enumerate(f): - self.assertIn(f'testing1 {i}', line) - - def test_rollover_at_weekday(self): + def test_rollover_at_midnight(self, weekly=False): + os_helper.unlink(self.fn) now = datetime.datetime.now() atTime = now.time() + if not 0.1 < atTime.microsecond/1e6 < 0.9: + # The test requires all records to be emitted within + # the range of the same whole second. + time.sleep((0.1 - atTime.microsecond/1e6) % 1.0) + now = datetime.datetime.now() + atTime = now.time() + atTime = atTime.replace(microsecond=0) fmt = logging.Formatter('%(asctime)s %(message)s') + when = f'W{now.weekday()}' if weekly else 'MIDNIGHT' for i in range(3): fh = logging.handlers.TimedRotatingFileHandler( - self.fn, encoding="utf-8", when=f'W{now.weekday()}', atTime=atTime) + self.fn, encoding="utf-8", when=when, atTime=atTime) fh.setFormatter(fmt) r2 = logging.makeLogRecord({'msg': f'testing1 {i}'}) fh.emit(r2) @@ -6129,15 +6460,15 @@ def test_rollover_at_weekday(self): for i, line in enumerate(f): self.assertIn(f'testing1 {i}', line) - os.utime(self.fn, (time.time() - 1,)*2) + os.utime(self.fn, (now.timestamp() - 1,)*2) for i in range(2): fh = logging.handlers.TimedRotatingFileHandler( - self.fn, encoding="utf-8", when=f'W{now.weekday()}', atTime=atTime) + self.fn, encoding="utf-8", when=when, atTime=atTime) fh.setFormatter(fmt) r2 = logging.makeLogRecord({'msg': f'testing2 {i}'}) fh.emit(r2) fh.close() - rolloverDate = datetime.datetime.now() - datetime.timedelta(days=7) + rolloverDate = now - datetime.timedelta(days=7 if weekly else 1) otherfn = f'{self.fn}.{rolloverDate:%Y-%m-%d}' self.assertLogFile(otherfn) with open(self.fn, encoding="utf-8") as f: @@ -6147,6 +6478,9 @@ def test_rollover_at_weekday(self): for i, line in enumerate(f): self.assertIn(f'testing1 {i}', line) + def test_rollover_at_weekday(self): + self.test_rollover_at_midnight(weekly=True) + def test_invalid(self): assertRaises = self.assertRaises assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler, @@ -6303,18 +6637,19 @@ def namer(filename): p = '%s.log.' % prefix for c in candidates: d, fn = os.path.split(c) - self.assertTrue(fn.startswith(p)) + self.assertStartsWith(fn, p) elif prefix.startswith('d.e'): for c in candidates: d, fn = os.path.split(c) - self.assertTrue(fn.endswith('.log'), fn) - self.assertTrue(fn.startswith(prefix + '.') and - fn[len(prefix) + 2].isdigit()) + self.assertEndsWith(fn, '.log') + self.assertStartsWith(fn, prefix + '.') + self.assertTrue(fn[len(prefix) + 2].isdigit()) elif prefix == 'g': for c in candidates: d, fn = os.path.split(c) - self.assertTrue(fn.endswith('.oldlog')) - self.assertTrue(fn.startswith('g') and fn[1].isdigit()) + self.assertEndsWith(fn, '.oldlog') + self.assertStartsWith(fn, 'g') + self.assertTrue(fn[1].isdigit()) def test_compute_files_to_delete_same_filename_different_extensions(self): # See GH-93205 for background @@ -6352,7 +6687,7 @@ def test_compute_files_to_delete_same_filename_different_extensions(self): matcher = re.compile(r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}\Z") for c in candidates: d, fn = os.path.split(c) - self.assertTrue(fn.startswith(prefix+'.')) + self.assertStartsWith(fn, prefix+'.') suffix = fn[(len(prefix)+1):] self.assertRegex(suffix, matcher) @@ -6594,6 +6929,129 @@ def test(current, expected): fh.close() + # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in + # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0). + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_compute_rollover_MIDNIGHT_local_interval(self): + # DST begins at 2012-3-11T02:00:00 and ends at 2012-11-4T02:00:00. + DT = datetime.datetime + def test(current, expected): + actual = fh.computeRollover(current.timestamp()) + diff = actual - expected.timestamp() + if diff: + self.assertEqual(diff, 0, datetime.timedelta(seconds=diff)) + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, interval=3) + + test(DT(2012, 3, 8, 23, 59, 59), DT(2012, 3, 11, 0, 0)) + test(DT(2012, 3, 9, 0, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 9, 1, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 10, 23, 59, 59), DT(2012, 3, 13, 0, 0)) + test(DT(2012, 3, 11, 0, 0), DT(2012, 3, 14, 0, 0)) + test(DT(2012, 3, 11, 1, 0), DT(2012, 3, 14, 0, 0)) + + test(DT(2012, 11, 1, 23, 59, 59), DT(2012, 11, 4, 0, 0)) + test(DT(2012, 11, 2, 0, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 11, 2, 1, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 11, 3, 23, 59, 59), DT(2012, 11, 6, 0, 0)) + test(DT(2012, 11, 4, 0, 0), DT(2012, 11, 7, 0, 0)) + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 7, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, interval=3, + atTime=datetime.time(12, 0, 0)) + + test(DT(2012, 3, 8, 11, 59, 59), DT(2012, 3, 10, 12, 0)) + test(DT(2012, 3, 8, 12, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 8, 13, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 10, 11, 59, 59), DT(2012, 3, 12, 12, 0)) + test(DT(2012, 3, 10, 12, 0), DT(2012, 3, 13, 12, 0)) + test(DT(2012, 3, 10, 13, 0), DT(2012, 3, 13, 12, 0)) + + test(DT(2012, 11, 1, 11, 59, 59), DT(2012, 11, 3, 12, 0)) + test(DT(2012, 11, 1, 12, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 11, 1, 13, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 11, 3, 11, 59, 59), DT(2012, 11, 5, 12, 0)) + test(DT(2012, 11, 3, 12, 0), DT(2012, 11, 6, 12, 0)) + test(DT(2012, 11, 3, 13, 0), DT(2012, 11, 6, 12, 0)) + + fh.close() + + # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in + # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0). + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_compute_rollover_W6_local_interval(self): + # DST begins at 2012-3-11T02:00:00 and ends at 2012-11-4T02:00:00. + DT = datetime.datetime + def test(current, expected): + actual = fh.computeRollover(current.timestamp()) + diff = actual - expected.timestamp() + if diff: + self.assertEqual(diff, 0, datetime.timedelta(seconds=diff)) + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, interval=3) + + test(DT(2012, 2, 19, 23, 59, 59), DT(2012, 3, 5, 0, 0)) + test(DT(2012, 2, 20, 0, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 2, 20, 1, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 4, 23, 59, 59), DT(2012, 3, 19, 0, 0)) + test(DT(2012, 3, 5, 0, 0), DT(2012, 3, 26, 0, 0)) + test(DT(2012, 3, 5, 1, 0), DT(2012, 3, 26, 0, 0)) + + test(DT(2012, 10, 14, 23, 59, 59), DT(2012, 10, 29, 0, 0)) + test(DT(2012, 10, 15, 0, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 10, 15, 1, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 10, 28, 23, 59, 59), DT(2012, 11, 12, 0, 0)) + test(DT(2012, 10, 29, 0, 0), DT(2012, 11, 19, 0, 0)) + test(DT(2012, 10, 29, 1, 0), DT(2012, 11, 19, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, interval=3, + atTime=datetime.time(0, 0, 0)) + + test(DT(2012, 2, 25, 23, 59, 59), DT(2012, 3, 11, 0, 0)) + test(DT(2012, 2, 26, 0, 0), DT(2012, 3, 18, 0, 0)) + test(DT(2012, 2, 26, 1, 0), DT(2012, 3, 18, 0, 0)) + test(DT(2012, 3, 10, 23, 59, 59), DT(2012, 3, 25, 0, 0)) + test(DT(2012, 3, 11, 0, 0), DT(2012, 4, 1, 0, 0)) + test(DT(2012, 3, 11, 1, 0), DT(2012, 4, 1, 0, 0)) + + test(DT(2012, 10, 20, 23, 59, 59), DT(2012, 11, 4, 0, 0)) + test(DT(2012, 10, 21, 0, 0), DT(2012, 11, 11, 0, 0)) + test(DT(2012, 10, 21, 1, 0), DT(2012, 11, 11, 0, 0)) + test(DT(2012, 11, 3, 23, 59, 59), DT(2012, 11, 18, 0, 0)) + test(DT(2012, 11, 4, 0, 0), DT(2012, 11, 25, 0, 0)) + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 25, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, interval=3, + atTime=datetime.time(12, 0, 0)) + + test(DT(2012, 2, 18, 11, 59, 59), DT(2012, 3, 4, 12, 0)) + test(DT(2012, 2, 19, 12, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 2, 19, 13, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 4, 11, 59, 59), DT(2012, 3, 18, 12, 0)) + test(DT(2012, 3, 4, 12, 0), DT(2012, 3, 25, 12, 0)) + test(DT(2012, 3, 4, 13, 0), DT(2012, 3, 25, 12, 0)) + + test(DT(2012, 10, 14, 11, 59, 59), DT(2012, 10, 28, 12, 0)) + test(DT(2012, 10, 14, 12, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 10, 14, 13, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 10, 28, 11, 59, 59), DT(2012, 11, 11, 12, 0)) + test(DT(2012, 10, 28, 12, 0), DT(2012, 11, 18, 12, 0)) + test(DT(2012, 10, 28, 13, 0), DT(2012, 11, 18, 12, 0)) + + fh.close() + + def secs(**kw): return datetime.timedelta(**kw) // datetime.timedelta(seconds=1) @@ -6605,40 +7063,49 @@ def secs(**kw): # current time (epoch start) is a Thursday, W0 means Monday ('W0', secs(days=4, hours=24)), ): - def test_compute_rollover(self, when=when, exp=exp): - rh = logging.handlers.TimedRotatingFileHandler( - self.fn, encoding="utf-8", when=when, interval=1, backupCount=0, utc=True) - currentTime = 0.0 - actual = rh.computeRollover(currentTime) - if exp != actual: - # Failures occur on some systems for MIDNIGHT and W0. - # Print detailed calculation for MIDNIGHT so we can try to see - # what's going on - if when == 'MIDNIGHT': - try: - if rh.utc: - t = time.gmtime(currentTime) - else: - t = time.localtime(currentTime) - currentHour = t[3] - currentMinute = t[4] - currentSecond = t[5] - # r is the number of seconds left between now and midnight - r = logging.handlers._MIDNIGHT - ((currentHour * 60 + - currentMinute) * 60 + - currentSecond) - result = currentTime + r - print('t: %s (%s)' % (t, rh.utc), file=sys.stderr) - print('currentHour: %s' % currentHour, file=sys.stderr) - print('currentMinute: %s' % currentMinute, file=sys.stderr) - print('currentSecond: %s' % currentSecond, file=sys.stderr) - print('r: %s' % r, file=sys.stderr) - print('result: %s' % result, file=sys.stderr) - except Exception as e: - print('exception in diagnostic code: %s' % e, file=sys.stderr) - self.assertEqual(exp, actual) - rh.close() - setattr(TimedRotatingFileHandlerTest, "test_compute_rollover_%s" % when, test_compute_rollover) + for interval in 1, 3: + def test_compute_rollover(self, when=when, interval=interval, exp=exp): + rh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when=when, interval=interval, backupCount=0, utc=True) + currentTime = 0.0 + actual = rh.computeRollover(currentTime) + if when.startswith('W'): + exp += secs(days=7*(interval-1)) + else: + exp *= interval + if exp != actual: + # Failures occur on some systems for MIDNIGHT and W0. + # Print detailed calculation for MIDNIGHT so we can try to see + # what's going on + if when == 'MIDNIGHT': + try: + if rh.utc: + t = time.gmtime(currentTime) + else: + t = time.localtime(currentTime) + currentHour = t[3] + currentMinute = t[4] + currentSecond = t[5] + # r is the number of seconds left between now and midnight + r = logging.handlers._MIDNIGHT - ((currentHour * 60 + + currentMinute) * 60 + + currentSecond) + result = currentTime + r + print('t: %s (%s)' % (t, rh.utc), file=sys.stderr) + print('currentHour: %s' % currentHour, file=sys.stderr) + print('currentMinute: %s' % currentMinute, file=sys.stderr) + print('currentSecond: %s' % currentSecond, file=sys.stderr) + print('r: %s' % r, file=sys.stderr) + print('result: %s' % result, file=sys.stderr) + except Exception as e: + print('exception in diagnostic code: %s' % e, file=sys.stderr) + self.assertEqual(exp, actual) + rh.close() + name = "test_compute_rollover_%s" % when + if interval > 1: + name += "_interval" + test_compute_rollover.__name__ = name + setattr(TimedRotatingFileHandlerTest, name, test_compute_rollover) @unittest.skipUnless(win32evtlog, 'win32evtlog/win32evtlogutil/pywintypes required for this test.') diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index d299c34cec076d..f336d49fa4f008 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -386,15 +386,6 @@ def __long__(self): return 42 self.assertRaises(TypeError, int, JustLong()) - class LongTrunc: - # __long__ should be ignored in 3.x - def __long__(self): - return 42 - def __trunc__(self): - return 1729 - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(LongTrunc()), 1729) - def check_float_conversion(self, n): # Check that int -> float conversion behaviour matches # that of the pure Python version above. @@ -482,6 +473,12 @@ def test_float_conversion(self): self.check_float_conversion(value) self.check_float_conversion(-value) + @support.requires_IEEE_754 + @support.bigmemtest(2**32, memuse=0.2) + def test_float_conversion_huge_integer(self, size): + v = 1 << size + self.assertRaises(OverflowError, float, v) + def test_float_overflow(self): for x in -2.0, -1.0, 0.0, 1.0, 2.0: self.assertEqual(float(int(x)), x) @@ -623,6 +620,56 @@ def __lt__(self, other): eq(x > y, Rcmp > 0) eq(x >= y, Rcmp >= 0) + @support.requires_IEEE_754 + @support.bigmemtest(2**32, memuse=0.2) + def test_mixed_compares_huge_integer(self, size): + v = 1 << size + f = sys.float_info.max + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, True) + self.assertIs(f <= v, True) + self.assertIs(f > v, False) + self.assertIs(f >= v, False) + f = float('inf') + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, False) + self.assertIs(f <= v, False) + self.assertIs(f > v, True) + self.assertIs(f >= v, True) + f = float('nan') + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, False) + self.assertIs(f <= v, False) + self.assertIs(f > v, False) + self.assertIs(f >= v, False) + + del v + v = (-1) << size + f = -sys.float_info.max + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, False) + self.assertIs(f <= v, False) + self.assertIs(f > v, True) + self.assertIs(f >= v, True) + f = float('-inf') + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, True) + self.assertIs(f <= v, True) + self.assertIs(f > v, False) + self.assertIs(f >= v, False) + f = float('nan') + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, False) + self.assertIs(f <= v, False) + self.assertIs(f > v, False) + self.assertIs(f >= v, False) + def test__format__(self): self.assertEqual(format(123456789, 'd'), '123456789') self.assertEqual(format(123456789, 'd'), '123456789') @@ -942,9 +989,12 @@ def test_huge_lshift_of_zero(self): self.assertEqual(0 << (sys.maxsize + 1), 0) @support.cpython_only - @support.bigmemtest(sys.maxsize + 1000, memuse=2/15 * 2, dry_run=False) + @support.bigmemtest(2**32, memuse=0.2) def test_huge_lshift(self, size): - self.assertEqual(1 << (sys.maxsize + 1000), 1 << 1000 << sys.maxsize) + v = 5 << size + self.assertEqual(v.bit_length(), size + 3) + self.assertEqual(v.bit_count(), 2) + self.assertEqual(v >> size, 5) def test_huge_rshift(self): huge_shift = 1 << 1000 @@ -956,11 +1006,13 @@ def test_huge_rshift(self): self.assertEqual(-2**128 >> huge_shift, -1) @support.cpython_only - @support.bigmemtest(sys.maxsize + 500, memuse=2/15, dry_run=False) + @support.bigmemtest(2**32, memuse=0.2) def test_huge_rshift_of_huge(self, size): - huge = ((1 << 500) + 11) << sys.maxsize - self.assertEqual(huge >> (sys.maxsize + 1), (1 << 499) + 5) - self.assertEqual(huge >> (sys.maxsize + 1000), 0) + huge = ((1 << 500) + 11) << size + self.assertEqual(huge.bit_length(), size + 501) + self.assertEqual(huge.bit_count(), 4) + self.assertEqual(huge >> (size + 1), (1 << 499) + 5) + self.assertEqual(huge >> (size + 1000), 0) def test_small_rshift(self): self.assertEqual(42 >> 1, 21) @@ -1418,7 +1470,6 @@ def equivalent_python(byte_array, byteorder, signed=False): b'\x00': 0, b'\x00\x00': 0, b'\x01': 1, - b'\x00\x01': 256, b'\xff': -1, b'\xff\xff': -1, b'\x81': -127, @@ -1639,6 +1690,8 @@ class MyInt(int): MyInt.__basicsize__ + MyInt.__itemsize__ * ndigits ) + # GH-117195 -- This shouldn't crash + object.__sizeof__(1) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index db290e139327e0..22478c14fb4a65 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -551,19 +551,25 @@ def test_init_with_PathLike_filename(self): with TempFile(filename, COMPRESSED_XZ): with LZMAFile(filename) as f: self.assertEqual(f.read(), INPUT) + self.assertEqual(f.name, TESTFN) with LZMAFile(filename, "a") as f: f.write(INPUT) + self.assertEqual(f.name, TESTFN) with LZMAFile(filename) as f: self.assertEqual(f.read(), INPUT * 2) + self.assertEqual(f.name, TESTFN) def test_init_with_filename(self): with TempFile(TESTFN, COMPRESSED_XZ): with LZMAFile(TESTFN) as f: - pass + self.assertEqual(f.name, TESTFN) + self.assertEqual(f.mode, 'rb') with LZMAFile(TESTFN, "w") as f: - pass + self.assertEqual(f.name, TESTFN) + self.assertEqual(f.mode, 'wb') with LZMAFile(TESTFN, "a") as f: - pass + self.assertEqual(f.name, TESTFN) + self.assertEqual(f.mode, 'wb') def test_init_mode(self): with TempFile(TESTFN): @@ -586,6 +592,7 @@ def test_init_with_x_mode(self): unlink(TESTFN) with LZMAFile(TESTFN, mode) as f: pass + self.assertEqual(f.mode, 'wb') with self.assertRaises(FileExistsError): LZMAFile(TESTFN, mode) @@ -865,13 +872,18 @@ def test_read_from_file(self): with LZMAFile(TESTFN) as f: self.assertEqual(f.read(), INPUT) self.assertEqual(f.read(), b"") + self.assertEqual(f.name, TESTFN) self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'rb') self.assertIs(f.readable(), True) self.assertIs(f.writable(), False) self.assertIs(f.seekable(), True) self.assertIs(f.closed, False) self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) @@ -882,6 +894,7 @@ def test_read_from_file_with_bytes_filename(self): with LZMAFile(bytes_filename) as f: self.assertEqual(f.read(), INPUT) self.assertEqual(f.read(), b"") + self.assertEqual(f.name, bytes_filename) def test_read_from_fileobj(self): with TempFile(TESTFN, COMPRESSED_XZ): @@ -889,13 +902,18 @@ def test_read_from_fileobj(self): with LZMAFile(raw) as f: self.assertEqual(f.read(), INPUT) self.assertEqual(f.read(), b"") + self.assertEqual(f.name, raw.name) self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') self.assertIs(f.readable(), True) self.assertIs(f.writable(), False) self.assertIs(f.seekable(), True) self.assertIs(f.closed, False) self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) @@ -907,13 +925,18 @@ def test_read_from_fileobj_with_int_name(self): with LZMAFile(raw) as f: self.assertEqual(f.read(), INPUT) self.assertEqual(f.read(), b"") + self.assertEqual(f.name, raw.name) self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') self.assertIs(f.readable(), True) self.assertIs(f.writable(), False) self.assertIs(f.seekable(), True) self.assertIs(f.closed, False) self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) @@ -1045,6 +1068,8 @@ def test_write(self): with BytesIO() as dst: with LZMAFile(dst, "w") as f: f.write(INPUT) + with self.assertRaises(AttributeError): + f.name expected = lzma.compress(INPUT) self.assertEqual(dst.getvalue(), expected) with BytesIO() as dst: @@ -1081,23 +1106,31 @@ def test_write_append(self): with BytesIO() as dst: with LZMAFile(dst, "w") as f: f.write(part1) + self.assertEqual(f.mode, 'wb') with LZMAFile(dst, "a") as f: f.write(part2) + self.assertEqual(f.mode, 'wb') with LZMAFile(dst, "a") as f: f.write(part3) + self.assertEqual(f.mode, 'wb') self.assertEqual(dst.getvalue(), expected) def test_write_to_file(self): try: with LZMAFile(TESTFN, "w") as f: f.write(INPUT) + self.assertEqual(f.name, TESTFN) self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'wb') self.assertIs(f.readable(), False) self.assertIs(f.writable(), True) self.assertIs(f.seekable(), False) self.assertIs(f.closed, False) self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) @@ -1113,6 +1146,7 @@ def test_write_to_file_with_bytes_filename(self): try: with LZMAFile(bytes_filename, "w") as f: f.write(INPUT) + self.assertEqual(f.name, bytes_filename) expected = lzma.compress(INPUT) with open(TESTFN, "rb") as f: self.assertEqual(f.read(), expected) @@ -1124,13 +1158,18 @@ def test_write_to_fileobj(self): with open(TESTFN, "wb") as raw: with LZMAFile(raw, "w") as f: f.write(INPUT) + self.assertEqual(f.name, raw.name) self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') self.assertIs(f.readable(), False) self.assertIs(f.writable(), True) self.assertIs(f.seekable(), False) self.assertIs(f.closed, False) self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) @@ -1147,13 +1186,18 @@ def test_write_to_fileobj_with_int_name(self): with open(fd, 'wb') as raw: with LZMAFile(raw, "w") as f: f.write(INPUT) + self.assertEqual(f.name, raw.name) self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') self.assertIs(f.readable(), False) self.assertIs(f.writable(), True) self.assertIs(f.seekable(), False) self.assertIs(f.closed, False) self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') self.assertRaises(ValueError, f.readable) self.assertRaises(ValueError, f.writable) self.assertRaises(ValueError, f.seekable) @@ -1172,10 +1216,13 @@ def test_write_append_to_file(self): try: with LZMAFile(TESTFN, "w") as f: f.write(part1) + self.assertEqual(f.mode, 'wb') with LZMAFile(TESTFN, "a") as f: f.write(part2) + self.assertEqual(f.mode, 'wb') with LZMAFile(TESTFN, "a") as f: f.write(part3) + self.assertEqual(f.mode, 'wb') with open(TESTFN, "rb") as f: self.assertEqual(f.read(), expected) finally: @@ -1373,11 +1420,13 @@ def test_with_pathlike_filename(self): with TempFile(filename): with lzma.open(filename, "wb") as f: f.write(INPUT) + self.assertEqual(f.name, TESTFN) with open(filename, "rb") as f: file_data = lzma.decompress(f.read()) self.assertEqual(file_data, INPUT) with lzma.open(filename, "rb") as f: self.assertEqual(f.read(), INPUT) + self.assertEqual(f.name, TESTFN) def test_bad_params(self): # Test invalid parameter combinations. diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index d4628f91daf7e8..3ecb5eab26d4b9 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -1,7 +1,6 @@ import os import sys import time -import stat import socket import email import email.message @@ -9,6 +8,7 @@ import io import tempfile from test import support +from test.support import import_helper from test.support import os_helper from test.support import refleak_helper from test.support import socket_helper @@ -702,8 +702,7 @@ def _check_basics(self, factory=None): self.assertEqual(self._box._factory, factory) for subdir in '', 'tmp', 'new', 'cur': path = os.path.join(self._path, subdir) - mode = os.stat(path)[stat.ST_MODE] - self.assertTrue(stat.S_ISDIR(mode), "Not a directory: '%s'" % path) + self.assertTrue(os.path.isdir(path), f"Not a directory: {path!r}") def test_list_folders(self): # List folders @@ -1082,6 +1081,47 @@ def test_permissions_after_flush(self): self.assertEqual(os.stat(self._path).st_mode, mode) + @unittest.skipUnless(hasattr(os, 'chown'), 'requires os.chown') + def test_ownership_after_flush(self): + # See issue gh-117467 + + pwd = import_helper.import_module('pwd') + grp = import_helper.import_module('grp') + st = os.stat(self._path) + + for e in pwd.getpwall(): + if e.pw_uid != st.st_uid: + other_uid = e.pw_uid + break + else: + self.skipTest("test needs more than one user") + + for e in grp.getgrall(): + if e.gr_gid != st.st_gid: + other_gid = e.gr_gid + break + else: + self.skipTest("test needs more than one group") + + try: + os.chown(self._path, other_uid, other_gid) + except OSError: + self.skipTest('test needs root privilege') + # Change permissions as in test_permissions_after_flush. + mode = st.st_mode | 0o666 + os.chmod(self._path, mode) + + self._box.add(self._template % 0) + i = self._box.add(self._template % 1) + # Need to remove one message to make flush() create a new file + self._box.remove(i) + self._box.flush() + + st = os.stat(self._path) + self.assertEqual(st.st_uid, other_uid) + self.assertEqual(st.st_gid, other_gid) + self.assertEqual(st.st_mode, mode) + class _TestMboxMMDF(_TestSingleFile): diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 615568e6af2102..4ed9f1fc1b8020 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -1,5 +1,5 @@ from test import support -from test.support import is_apple_mobile, os_helper, requires_debug_ranges +from test.support import is_apple_mobile, os_helper, requires_debug_ranges, is_emscripten from test.support.script_helper import assert_python_ok import array import io @@ -28,6 +28,13 @@ def helper(self, sample, *extra): finally: os_helper.unlink(os_helper.TESTFN) +def omit_last_byte(data): + """return data[:-1]""" + # This file's code is used in CompatibilityTestCase, + # but slices need marshal version 5. + # Avoid the slice literal. + return data[slice(0, -1)] + class IntTestCase(unittest.TestCase, HelperMixin): def test_ints(self): # Test a range of Python ints larger than the machine word size. @@ -118,7 +125,7 @@ def test_code(self): def test_many_codeobjects(self): # Issue2957: bad recursion count on code objects # more than MAX_MARSHAL_STACK_DEPTH - count = support.EXCEEDS_RECURSION_LIMIT + count = support.exceeds_recursion_limit() codes = (ExceptionTestCase.test_exceptions.__code__,) * count marshal.loads(marshal.dumps(codes)) @@ -241,7 +248,8 @@ def test_bug_5888452(self): def test_patch_873224(self): self.assertRaises(Exception, marshal.loads, b'0') self.assertRaises(Exception, marshal.loads, b'f') - self.assertRaises(Exception, marshal.loads, marshal.dumps(2**65)[:-1]) + self.assertRaises(Exception, marshal.loads, + omit_last_byte(marshal.dumps(2**65))) def test_version_argument(self): # Python 2.4.0 crashes for any call to marshal.dumps(x, y) @@ -286,7 +294,7 @@ def test_recursion_limit(self): #if os.name == 'nt' and support.Py_DEBUG: if os.name == 'nt': MAX_MARSHAL_STACK_DEPTH = 1000 - elif sys.platform == 'wasi' or is_apple_mobile: + elif sys.platform == 'wasi' or is_emscripten or is_apple_mobile: MAX_MARSHAL_STACK_DEPTH = 1500 else: MAX_MARSHAL_STACK_DEPTH = 2000 @@ -594,6 +602,19 @@ def testNoIntern(self): s2 = sys.intern(s) self.assertNotEqual(id(s2), id(s)) +class SliceTestCase(unittest.TestCase, HelperMixin): + def test_slice(self): + for obj in ( + slice(None), slice(1), slice(1, 2), slice(1, 2, 3), + slice({'set'}, ('tuple', {'with': 'dict'}, ), self.helper.__code__) + ): + with self.subTest(obj=str(obj)): + self.helper(obj) + + for version in range(4): + with self.assertRaises(ValueError): + marshal.dumps(obj, version) + @support.cpython_only @unittest.skipUnless(_testcapi, 'requires _testcapi') class CAPI_TestCase(unittest.TestCase, HelperMixin): @@ -654,7 +675,7 @@ def test_read_last_object_from_file(self): self.assertEqual(r, obj) with open(os_helper.TESTFN, 'wb') as f: - f.write(data[:1]) + f.write(omit_last_byte(data)) with self.assertRaises(EOFError): _testcapi.pymarshal_read_last_object_from_file(os_helper.TESTFN) os_helper.unlink(os_helper.TESTFN) @@ -671,7 +692,7 @@ def test_read_object_from_file(self): self.assertEqual(p, len(data)) with open(os_helper.TESTFN, 'wb') as f: - f.write(data[:1]) + f.write(omit_last_byte(data)) with self.assertRaises(EOFError): _testcapi.pymarshal_read_object_from_file(os_helper.TESTFN) os_helper.unlink(os_helper.TESTFN) diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index ad382fc2b59891..2c57d288bc03ff 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -187,6 +187,9 @@ def result_check(expected, got, ulp_tol=5, abs_tol=0.0): # Check exactly equal (applies also to strings representing exceptions) if got == expected: + if not got and not expected: + if math.copysign(1, got) != math.copysign(1, expected): + return f"expected {expected}, got {got} (zero has wrong sign)" return None failure = "not equal" @@ -657,7 +660,7 @@ def testFsum(self): def msum(iterable): """Full precision summation. Compute sum(iterable) without any intermediate accumulation of error. Based on the 'lsum' function - at http://code.activestate.com/recipes/393090/ + at https://code.activestate.com/recipes/393090-binary-floating-point-summation-accurate-to-full-p/ """ tmant, texp = 0, 0 @@ -809,11 +812,13 @@ def testHypot(self): # Test allowable types (those with __float__) self.assertEqual(hypot(12.0, 5.0), 13.0) self.assertEqual(hypot(12, 5), 13) - self.assertEqual(hypot(1, -1), math.sqrt(2)) - self.assertEqual(hypot(1, FloatLike(-1.)), math.sqrt(2)) + self.assertEqual(hypot(0.75, -1), 1.25) + self.assertEqual(hypot(-1, 0.75), 1.25) + self.assertEqual(hypot(0.75, FloatLike(-1.)), 1.25) + self.assertEqual(hypot(FloatLike(-1.), 0.75), 1.25) self.assertEqual(hypot(Decimal(12), Decimal(5)), 13) self.assertEqual(hypot(Fraction(12, 32), Fraction(5, 32)), Fraction(13, 32)) - self.assertEqual(hypot(bool(1), bool(0), bool(1), bool(1)), math.sqrt(3)) + self.assertEqual(hypot(True, False, True, True, True), 2.0) # Test corner cases self.assertEqual(hypot(0.0, 0.0), 0.0) # Max input is zero @@ -969,9 +974,9 @@ def testDist(self): self.assertEqual(dist((D(14), D(1)), (D(2), D(-4))), D(13)) self.assertEqual(dist((F(14, 32), F(1, 32)), (F(2, 32), F(-4, 32))), F(13, 32)) - self.assertEqual(dist((True, True, False, True, False), - (True, False, True, True, False)), - sqrt(2.0)) + self.assertEqual(dist((True, True, False, False, True, True), + (True, False, True, False, False, False)), + 2.0) # Test corner cases self.assertEqual(dist((13.25, 12.5, -3.25), @@ -1120,6 +1125,15 @@ def __index__(self): with self.assertRaises(TypeError): math.isqrt(value) + @support.bigmemtest(2**32, memuse=0.85) + def test_isqrt_huge(self, size): + if size & 1: + size += 1 + v = 1 << size + w = math.isqrt(v) + self.assertEqual(w.bit_length(), size // 2 + 1) + self.assertEqual(w.bit_count(), 1) + def test_lcm(self): lcm = math.lcm self.assertEqual(lcm(0, 0), 0) @@ -1261,6 +1275,13 @@ def testLog10(self): self.assertEqual(math.log(INF), INF) self.assertTrue(math.isnan(math.log10(NAN))) + @support.bigmemtest(2**32, memuse=0.2) + def test_log_huge_integer(self, size): + v = 1 << size + self.assertAlmostEqual(math.log2(v), size) + self.assertAlmostEqual(math.log(v), size * 0.6931471805599453) + self.assertAlmostEqual(math.log10(v), size * 0.3010299956639812) + def testSumProd(self): sumprod = math.sumprod Decimal = decimal.Decimal @@ -1397,7 +1418,7 @@ def __repr__(self): return f'Flt({int(self)})' def baseline_sumprod(p, q): - """This defines the target behavior including expections and special values. + """This defines the target behavior including exceptions and special values. However, it is subject to rounding errors, so float inputs should be exactly representable with only a few bits. """ @@ -1881,7 +1902,7 @@ def testTan(self): try: self.assertTrue(math.isnan(math.tan(INF))) self.assertTrue(math.isnan(math.tan(NINF))) - except: + except ValueError: self.assertRaises(ValueError, math.tan, INF) self.assertRaises(ValueError, math.tan, NINF) self.assertTrue(math.isnan(math.tan(NAN))) @@ -2051,6 +2072,13 @@ def test_testfile(self): except OverflowError: result = 'OverflowError' + # C99+ says for math.h's sqrt: If the argument is +∞ or ±0, it is + # returned, unmodified. On another hand, for csqrt: If z is ±0+0i, + # the result is +0+0i. Lets correct zero sign of er to follow + # first convention. + if id in ['sqrt0002', 'sqrt0003', 'sqrt1001', 'sqrt1023']: + er = math.copysign(er, ar) + # Default tolerances ulp_tol, abs_tol = 5, 0.0 @@ -2475,6 +2503,46 @@ def test_input_exceptions(self): self.assertRaises(TypeError, math.atan2, 1.0) self.assertRaises(TypeError, math.atan2, 1.0, 2.0, 3.0) + def test_exception_messages(self): + x = -1.1 + with self.assertRaisesRegex(ValueError, + f"expected a nonnegative input, got {x}"): + math.sqrt(x) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log(x) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log(123, x) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log(x, 123) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log2(x) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log10(x) + x = decimal.Decimal('-1.1') + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log(x) + x = fractions.Fraction(1, 10**400) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {float(x)}"): + math.log(x) + x = -123 + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log(x) + with self.assertRaisesRegex(ValueError, + f"expected a float or nonnegative integer, got {x}"): + math.gamma(x) + x = 1.0 + with self.assertRaisesRegex(ValueError, + f"expected a number between -1 and 1, got {x}"): + math.atanh(x) + # Custom assertions. def assertIsNaN(self, value): @@ -2613,6 +2681,244 @@ def test_fractions(self): self.assertAllNotClose(fraction_examples, rel_tol=1e-9) +class FMATests(unittest.TestCase): + """ Tests for math.fma. """ + + def test_fma_nan_results(self): + # Selected representative values. + values = [ + -math.inf, -1e300, -2.3, -1e-300, -0.0, + 0.0, 1e-300, 2.3, 1e300, math.inf, math.nan + ] + + # If any input is a NaN, the result should be a NaN, too. + for a, b in itertools.product(values, repeat=2): + self.assertIsNaN(math.fma(math.nan, a, b)) + self.assertIsNaN(math.fma(a, math.nan, b)) + self.assertIsNaN(math.fma(a, b, math.nan)) + + def test_fma_infinities(self): + # Cases involving infinite inputs or results. + positives = [1e-300, 2.3, 1e300, math.inf] + finites = [-1e300, -2.3, -1e-300, -0.0, 0.0, 1e-300, 2.3, 1e300] + non_nans = [-math.inf, -2.3, -0.0, 0.0, 2.3, math.inf] + + # ValueError due to inf * 0 computation. + for c in non_nans: + for infinity in [math.inf, -math.inf]: + for zero in [0.0, -0.0]: + with self.assertRaises(ValueError): + math.fma(infinity, zero, c) + with self.assertRaises(ValueError): + math.fma(zero, infinity, c) + + # ValueError when a*b and c both infinite of opposite signs. + for b in positives: + with self.assertRaises(ValueError): + math.fma(math.inf, b, -math.inf) + with self.assertRaises(ValueError): + math.fma(math.inf, -b, math.inf) + with self.assertRaises(ValueError): + math.fma(-math.inf, -b, -math.inf) + with self.assertRaises(ValueError): + math.fma(-math.inf, b, math.inf) + with self.assertRaises(ValueError): + math.fma(b, math.inf, -math.inf) + with self.assertRaises(ValueError): + math.fma(-b, math.inf, math.inf) + with self.assertRaises(ValueError): + math.fma(-b, -math.inf, -math.inf) + with self.assertRaises(ValueError): + math.fma(b, -math.inf, math.inf) + + # Infinite result when a*b and c both infinite of the same sign. + for b in positives: + self.assertEqual(math.fma(math.inf, b, math.inf), math.inf) + self.assertEqual(math.fma(math.inf, -b, -math.inf), -math.inf) + self.assertEqual(math.fma(-math.inf, -b, math.inf), math.inf) + self.assertEqual(math.fma(-math.inf, b, -math.inf), -math.inf) + self.assertEqual(math.fma(b, math.inf, math.inf), math.inf) + self.assertEqual(math.fma(-b, math.inf, -math.inf), -math.inf) + self.assertEqual(math.fma(-b, -math.inf, math.inf), math.inf) + self.assertEqual(math.fma(b, -math.inf, -math.inf), -math.inf) + + # Infinite result when a*b finite, c infinite. + for a, b in itertools.product(finites, finites): + self.assertEqual(math.fma(a, b, math.inf), math.inf) + self.assertEqual(math.fma(a, b, -math.inf), -math.inf) + + # Infinite result when a*b infinite, c finite. + for b, c in itertools.product(positives, finites): + self.assertEqual(math.fma(math.inf, b, c), math.inf) + self.assertEqual(math.fma(-math.inf, b, c), -math.inf) + self.assertEqual(math.fma(-math.inf, -b, c), math.inf) + self.assertEqual(math.fma(math.inf, -b, c), -math.inf) + + self.assertEqual(math.fma(b, math.inf, c), math.inf) + self.assertEqual(math.fma(b, -math.inf, c), -math.inf) + self.assertEqual(math.fma(-b, -math.inf, c), math.inf) + self.assertEqual(math.fma(-b, math.inf, c), -math.inf) + + # gh-73468: On some platforms, libc fma() doesn't implement IEE 754-2008 + # properly: it doesn't use the right sign when the result is zero. + @unittest.skipIf( + sys.platform.startswith(("freebsd", "wasi", "netbsd", "emscripten")) + or (sys.platform == "android" and platform.machine() == "x86_64"), + f"this platform doesn't implement IEE 754-2008 properly") + def test_fma_zero_result(self): + nonnegative_finites = [0.0, 1e-300, 2.3, 1e300] + + # Zero results from exact zero inputs. + for b in nonnegative_finites: + self.assertIsPositiveZero(math.fma(0.0, b, 0.0)) + self.assertIsPositiveZero(math.fma(0.0, b, -0.0)) + self.assertIsNegativeZero(math.fma(0.0, -b, -0.0)) + self.assertIsPositiveZero(math.fma(0.0, -b, 0.0)) + self.assertIsPositiveZero(math.fma(-0.0, -b, 0.0)) + self.assertIsPositiveZero(math.fma(-0.0, -b, -0.0)) + self.assertIsNegativeZero(math.fma(-0.0, b, -0.0)) + self.assertIsPositiveZero(math.fma(-0.0, b, 0.0)) + + self.assertIsPositiveZero(math.fma(b, 0.0, 0.0)) + self.assertIsPositiveZero(math.fma(b, 0.0, -0.0)) + self.assertIsNegativeZero(math.fma(-b, 0.0, -0.0)) + self.assertIsPositiveZero(math.fma(-b, 0.0, 0.0)) + self.assertIsPositiveZero(math.fma(-b, -0.0, 0.0)) + self.assertIsPositiveZero(math.fma(-b, -0.0, -0.0)) + self.assertIsNegativeZero(math.fma(b, -0.0, -0.0)) + self.assertIsPositiveZero(math.fma(b, -0.0, 0.0)) + + # Exact zero result from nonzero inputs. + self.assertIsPositiveZero(math.fma(2.0, 2.0, -4.0)) + self.assertIsPositiveZero(math.fma(2.0, -2.0, 4.0)) + self.assertIsPositiveZero(math.fma(-2.0, -2.0, -4.0)) + self.assertIsPositiveZero(math.fma(-2.0, 2.0, 4.0)) + + # Underflow to zero. + tiny = 1e-300 + self.assertIsPositiveZero(math.fma(tiny, tiny, 0.0)) + self.assertIsNegativeZero(math.fma(tiny, -tiny, 0.0)) + self.assertIsPositiveZero(math.fma(-tiny, -tiny, 0.0)) + self.assertIsNegativeZero(math.fma(-tiny, tiny, 0.0)) + self.assertIsPositiveZero(math.fma(tiny, tiny, -0.0)) + self.assertIsNegativeZero(math.fma(tiny, -tiny, -0.0)) + self.assertIsPositiveZero(math.fma(-tiny, -tiny, -0.0)) + self.assertIsNegativeZero(math.fma(-tiny, tiny, -0.0)) + + # Corner case where rounding the multiplication would + # give the wrong result. + x = float.fromhex('0x1p-500') + y = float.fromhex('0x1p-550') + z = float.fromhex('0x1p-1000') + self.assertIsNegativeZero(math.fma(x-y, x+y, -z)) + self.assertIsPositiveZero(math.fma(y-x, x+y, z)) + self.assertIsNegativeZero(math.fma(y-x, -(x+y), -z)) + self.assertIsPositiveZero(math.fma(x-y, -(x+y), z)) + + def test_fma_overflow(self): + a = b = float.fromhex('0x1p512') + c = float.fromhex('0x1p1023') + # Overflow from multiplication. + with self.assertRaises(OverflowError): + math.fma(a, b, 0.0) + self.assertEqual(math.fma(a, b/2.0, 0.0), c) + # Overflow from the addition. + with self.assertRaises(OverflowError): + math.fma(a, b/2.0, c) + # No overflow, even though a*b overflows a float. + self.assertEqual(math.fma(a, b, -c), c) + + # Extreme case: a * b is exactly at the overflow boundary, so the + # tiniest offset makes a difference between overflow and a finite + # result. + a = float.fromhex('0x1.ffffffc000000p+511') + b = float.fromhex('0x1.0000002000000p+512') + c = float.fromhex('0x0.0000000000001p-1022') + with self.assertRaises(OverflowError): + math.fma(a, b, 0.0) + with self.assertRaises(OverflowError): + math.fma(a, b, c) + self.assertEqual(math.fma(a, b, -c), + float.fromhex('0x1.fffffffffffffp+1023')) + + # Another extreme case: here a*b is about as large as possible subject + # to math.fma(a, b, c) being finite. + a = float.fromhex('0x1.ae565943785f9p+512') + b = float.fromhex('0x1.3094665de9db8p+512') + c = float.fromhex('0x1.fffffffffffffp+1023') + self.assertEqual(math.fma(a, b, -c), c) + + def test_fma_single_round(self): + a = float.fromhex('0x1p-50') + self.assertEqual(math.fma(a - 1.0, a + 1.0, 1.0), a*a) + + def test_random(self): + # A collection of randomly generated inputs for which the naive FMA + # (with two rounds) gives a different result from a singly-rounded FMA. + + # tuples (a, b, c, expected) + test_values = [ + ('0x1.694adde428b44p-1', '0x1.371b0d64caed7p-1', + '0x1.f347e7b8deab8p-4', '0x1.19f10da56c8adp-1'), + ('0x1.605401ccc6ad6p-2', '0x1.ce3a40bf56640p-2', + '0x1.96e3bf7bf2e20p-2', '0x1.1af6d8aa83101p-1'), + ('0x1.e5abd653a67d4p-2', '0x1.a2e400209b3e6p-1', + '0x1.a90051422ce13p-1', '0x1.37d68cc8c0fbbp+0'), + ('0x1.f94e8efd54700p-2', '0x1.123065c812cebp-1', + '0x1.458f86fb6ccd0p-1', '0x1.ccdcee26a3ff3p-1'), + ('0x1.bd926f1eedc96p-1', '0x1.eee9ca68c5740p-1', + '0x1.960c703eb3298p-2', '0x1.3cdcfb4fdb007p+0'), + ('0x1.27348350fbccdp-1', '0x1.3b073914a53f1p-1', + '0x1.e300da5c2b4cbp-1', '0x1.4c51e9a3c4e29p+0'), + ('0x1.2774f00b3497bp-1', '0x1.7038ec336bff0p-2', + '0x1.2f6f2ccc3576bp-1', '0x1.99ad9f9c2688bp-1'), + ('0x1.51d5a99300e5cp-1', '0x1.5cd74abd445a1p-1', + '0x1.8880ab0bbe530p-1', '0x1.3756f96b91129p+0'), + ('0x1.73cb965b821b8p-2', '0x1.218fd3d8d5371p-1', + '0x1.d1ea966a1f758p-2', '0x1.5217b8fd90119p-1'), + ('0x1.4aa98e890b046p-1', '0x1.954d85dff1041p-1', + '0x1.122b59317ebdfp-1', '0x1.0bf644b340cc5p+0'), + ('0x1.e28f29e44750fp-1', '0x1.4bcc4fdcd18fep-1', + '0x1.fd47f81298259p-1', '0x1.9b000afbc9995p+0'), + ('0x1.d2e850717fe78p-3', '0x1.1dd7531c303afp-1', + '0x1.e0869746a2fc2p-2', '0x1.316df6eb26439p-1'), + ('0x1.cf89c75ee6fbap-2', '0x1.b23decdc66825p-1', + '0x1.3d1fe76ac6168p-1', '0x1.00d8ea4c12abbp+0'), + ('0x1.3265ae6f05572p-2', '0x1.16d7ec285f7a2p-1', + '0x1.0b8405b3827fbp-1', '0x1.5ef33c118a001p-1'), + ('0x1.c4d1bf55ec1a5p-1', '0x1.bc59618459e12p-2', + '0x1.ce5b73dc1773dp-1', '0x1.496cf6164f99bp+0'), + ('0x1.d350026ac3946p-1', '0x1.9a234e149a68cp-2', + '0x1.f5467b1911fd6p-2', '0x1.b5cee3225caa5p-1'), + ] + for a_hex, b_hex, c_hex, expected_hex in test_values: + a = float.fromhex(a_hex) + b = float.fromhex(b_hex) + c = float.fromhex(c_hex) + expected = float.fromhex(expected_hex) + self.assertEqual(math.fma(a, b, c), expected) + self.assertEqual(math.fma(b, a, c), expected) + + # Custom assertions. + def assertIsNaN(self, value): + self.assertTrue( + math.isnan(value), + msg="Expected a NaN, got {!r}".format(value) + ) + + def assertIsPositiveZero(self, value): + self.assertTrue( + value == 0 and math.copysign(1, value) > 0, + msg="Expected a positive zero, got {!r}".format(value) + ) + + def assertIsNegativeZero(self, value): + self.assertTrue( + value == 0 and math.copysign(1, value) < 0, + msg="Expected a negative zero, got {!r}".format(value) + ) + + def load_tests(loader, tests, pattern): from doctest import DocFileSuite tests.addTest(DocFileSuite(os.path.join("mathdata", "ieee754.txt"))) diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index 8192502a40791b..95629ed862d6eb 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -801,7 +801,7 @@ def test_sizeof(self): def _test_cow_mutation(self, mutation): # Common code for all BytesIO copy-on-write mutation tests. - imm = b' ' * 1024 + imm = (' ' * 1024).encode("ascii") old_rc = sys.getrefcount(imm) memio = self.ioclass(imm) self.assertEqual(sys.getrefcount(imm), old_rc + 1) diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 0eb2a367603cfc..61b068c630c7ce 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -15,7 +15,13 @@ import pickle import struct -from test.support import import_helper +from itertools import product +from test import support +from test.support import import_helper, threading_helper + + +class MyObject: + pass class AbstractMemoryTests: @@ -54,12 +60,53 @@ def test_getitem(self): for tp in self._types: self.check_getitem_with_type(tp) + def test_index(self): + for tp in self._types: + b = tp(self._source) + m = self._view(b) # may be a sub-view + l = m.tolist() + k = 2 * len(self._source) + + for chi in self._source: + if chi in l: + self.assertEqual(m.index(chi), l.index(chi)) + else: + self.assertRaises(ValueError, m.index, chi) + + for start, stop in product(range(-k, k), range(-k, k)): + index = -1 + try: + index = l.index(chi, start, stop) + except ValueError: + pass + + if index == -1: + self.assertRaises(ValueError, m.index, chi, start, stop) + else: + self.assertEqual(m.index(chi, start, stop), index) + def test_iter(self): for tp in self._types: b = tp(self._source) m = self._view(b) self.assertEqual(list(m), [m[i] for i in range(len(m))]) + def test_count(self): + for tp in self._types: + b = tp(self._source) + m = self._view(b) + l = m.tolist() + for ch in list(m): + self.assertEqual(m.count(ch), l.count(ch)) + + b = tp((b'a' * 5) + (b'c' * 3)) + m = self._view(b) # may be sliced + l = m.tolist() + with self.subTest('count', buffer=b): + self.assertEqual(m.count(ord('a')), l.count(ord('a'))) + self.assertEqual(m.count(ord('b')), l.count(ord('b'))) + self.assertEqual(m.count(ord('c')), l.count(ord('c'))) + def test_setitem_readonly(self): if not self.ro_type: self.skipTest("no read-only type to test") @@ -228,8 +275,6 @@ def __init__(self, base): self.m = memoryview(base) class MySource(tp): pass - class MyObject: - pass # Create a reference cycle through a memoryview object. # This exercises mbuf_clear(). @@ -436,6 +481,18 @@ def _view(self, obj): def _check_contents(self, tp, obj, contents): self.assertEqual(obj, tp(contents)) + def test_count(self): + super().test_count() + for tp in self._types: + b = tp((b'a' * 5) + (b'c' * 3)) + m = self._view(b) # should not be sliced + self.assertEqual(len(b), len(m)) + with self.subTest('count', buffer=b): + self.assertEqual(m.count(ord('a')), 5) + self.assertEqual(m.count(ord('b')), 0) + self.assertEqual(m.count(ord('c')), 3) + + class BaseMemorySliceTests: source_bytes = b"XabcdefY" @@ -656,5 +713,52 @@ def __bool__(self): m[0] = MyBool() self.assertEqual(ba[:8], b'\0'*8) + def test_buffer_reference_loop(self): + m = memoryview(b'abc').__buffer__(0) + o = MyObject() + o.m = m + o.o = o + wr = weakref.ref(o) + del m, o + gc.collect() + self.assertIsNone(wr()) + + def test_picklebuffer_reference_loop(self): + pb = pickle.PickleBuffer(memoryview(b'abc')) + o = MyObject() + o.pb = pb + o.o = o + wr = weakref.ref(o) + del pb, o + gc.collect() + self.assertIsNone(wr()) + + +@threading_helper.requires_working_threading() +@support.requires_resource("cpu") +class RacingTest(unittest.TestCase): + def test_racing_getbuf_and_releasebuf(self): + """Repeatly access the memoryview for racing.""" + try: + from multiprocessing.managers import SharedMemoryManager + except ImportError: + self.skipTest("Test requires multiprocessing") + from threading import Thread + + n = 100 + with SharedMemoryManager() as smm: + obj = smm.ShareableList(range(100)) + threads = [] + for _ in range(n): + # Issue gh-127085, the `ShareableList.count` is just a convenient way to mess the `exports` + # counter of `memoryview`, this issue has no direct relation with `ShareableList`. + threads.append(Thread(target=obj.count, args=(1,))) + + with threading_helper.start_threads(threads): + pass + + del obj + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_metaclass.py b/Lib/test/test_metaclass.py index 36e8ab4cda3dad..07a333f98fa0a9 100644 --- a/Lib/test/test_metaclass.py +++ b/Lib/test/test_metaclass.py @@ -164,9 +164,11 @@ ... d['__module__'] = 'test.test_metaclass' d['__qualname__'] = 'C' + d['__firstlineno__'] = 1 d['foo'] = 4 d['foo'] = 42 d['bar'] = 123 + d['__static_attributes__'] = () >>> Use a metaclass that doesn't derive from type. @@ -182,12 +184,12 @@ ... b = 24 ... meta: C () - ns: [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 42), ('b', 24)] + ns: [('__firstlineno__', 1), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 42), ('b', 24)] kw: [] >>> type(C) is dict True >>> print(sorted(C.items())) - [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 42), ('b', 24)] + [('__firstlineno__', 1), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 42), ('b', 24)] >>> And again, with a __prepare__ attribute. @@ -205,11 +207,13 @@ prepare: C () [('other', 'booh')] d['__module__'] = 'test.test_metaclass' d['__qualname__'] = 'C' + d['__firstlineno__'] = 1 d['a'] = 1 d['a'] = 2 d['b'] = 3 + d['__static_attributes__'] = () meta: C () - ns: [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 2), ('b', 3)] + ns: [('__firstlineno__', 1), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 2), ('b', 3)] kw: [('other', 'booh')] >>> @@ -250,6 +254,33 @@ [...] test.test_metaclass.ObscureException +Test setting attributes with a non-base type in mro() (gh-127773). + + >>> class Base: + ... value = 1 + ... + >>> class Meta(type): + ... def mro(cls): + ... return (cls, Base, object) + ... + >>> class WeirdClass(metaclass=Meta): + ... pass + ... + >>> Base.value + 1 + >>> WeirdClass.value + 1 + >>> Base.value = 2 + >>> Base.value + 2 + >>> WeirdClass.value + 2 + >>> Base.value = 3 + >>> Base.value + 3 + >>> WeirdClass.value + 3 + """ import sys diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 01bba0ac2eed5a..0a5b511e75537c 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -1,6 +1,6 @@ import io import mimetypes -import pathlib +import os import sys import unittest.mock @@ -35,20 +35,28 @@ def setUp(self): def test_case_sensitivity(self): eq = self.assertEqual - eq(self.db.guess_type("foobar.HTML"), self.db.guess_type("foobar.html")) - eq(self.db.guess_type("foobar.TGZ"), self.db.guess_type("foobar.tgz")) - eq(self.db.guess_type("foobar.tar.Z"), ("application/x-tar", "compress")) - eq(self.db.guess_type("foobar.tar.z"), (None, None)) + eq(self.db.guess_file_type("foobar.html"), ("text/html", None)) + eq(self.db.guess_type("scheme:foobar.html"), ("text/html", None)) + eq(self.db.guess_file_type("foobar.HTML"), ("text/html", None)) + eq(self.db.guess_type("scheme:foobar.HTML"), ("text/html", None)) + eq(self.db.guess_file_type("foobar.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_type("scheme:foobar.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foobar.TGZ"), ("application/x-tar", "gzip")) + eq(self.db.guess_type("scheme:foobar.TGZ"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foobar.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_type("scheme:foobar.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_file_type("foobar.tar.z"), (None, None)) + eq(self.db.guess_type("scheme:foobar.tar.z"), (None, None)) def test_default_data(self): eq = self.assertEqual - eq(self.db.guess_type("foo.html"), ("text/html", None)) - eq(self.db.guess_type("foo.HTML"), ("text/html", None)) - eq(self.db.guess_type("foo.tgz"), ("application/x-tar", "gzip")) - eq(self.db.guess_type("foo.tar.gz"), ("application/x-tar", "gzip")) - eq(self.db.guess_type("foo.tar.Z"), ("application/x-tar", "compress")) - eq(self.db.guess_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) - eq(self.db.guess_type("foo.tar.xz"), ("application/x-tar", "xz")) + eq(self.db.guess_file_type("foo.html"), ("text/html", None)) + eq(self.db.guess_file_type("foo.HTML"), ("text/html", None)) + eq(self.db.guess_file_type("foo.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foo.tar.gz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foo.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_file_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) + eq(self.db.guess_file_type("foo.tar.xz"), ("application/x-tar", "xz")) def test_data_urls(self): eq = self.assertEqual @@ -62,7 +70,7 @@ def test_file_parsing(self): eq = self.assertEqual sio = io.StringIO("x-application/x-unittest pyunit\n") self.db.readfp(sio) - eq(self.db.guess_type("foo.pyunit"), + eq(self.db.guess_file_type("foo.pyunit"), ("x-application/x-unittest", None)) eq(self.db.guess_extension("x-application/x-unittest"), ".pyunit") @@ -74,11 +82,19 @@ def test_read_mime_types(self): with os_helper.temp_dir() as directory: data = "x-application/x-unittest pyunit\n" - file = pathlib.Path(directory, "sample.mimetype") - file.write_text(data, encoding="utf-8") + file = os.path.join(directory, "sample.mimetype") + with open(file, 'w', encoding="utf-8") as f: + f.write(data) mime_dict = mimetypes.read_mime_types(file) eq(mime_dict[".pyunit"], "x-application/x-unittest") + data = "x-application/x-unittest2 pyunit2\n" + file = os.path.join(directory, "sample2.mimetype") + with open(file, 'w', encoding="utf-8") as f: + f.write(data) + mime_dict = mimetypes.read_mime_types(os_helper.FakePath(file)) + eq(mime_dict[".pyunit2"], "x-application/x-unittest2") + # bpo-41048: read_mime_types should read the rule file with 'utf-8' encoding. # Not with locale encoding. _bootlocale has been imported because io.open(...) # uses it. @@ -94,12 +110,12 @@ def test_read_mime_types(self): def test_non_standard_types(self): eq = self.assertEqual # First try strict - eq(self.db.guess_type('foo.xul', strict=True), (None, None)) + eq(self.db.guess_file_type('foo.xul', strict=True), (None, None)) eq(self.db.guess_extension('image/jpg', strict=True), None) # And then non-strict - eq(self.db.guess_type('foo.xul', strict=False), ('text/xul', None)) - eq(self.db.guess_type('foo.XUL', strict=False), ('text/xul', None)) - eq(self.db.guess_type('foo.invalid', strict=False), (None, None)) + eq(self.db.guess_file_type('foo.xul', strict=False), ('text/xul', None)) + eq(self.db.guess_file_type('foo.XUL', strict=False), ('text/xul', None)) + eq(self.db.guess_file_type('foo.invalid', strict=False), (None, None)) eq(self.db.guess_extension('image/jpg', strict=False), '.jpg') eq(self.db.guess_extension('image/JPG', strict=False), '.jpg') @@ -109,15 +125,51 @@ def test_filename_with_url_delimiters(self): # compared to when interpreted as filename because of the semicolon. eq = self.assertEqual gzip_expected = ('application/x-tar', 'gzip') - eq(self.db.guess_type(";1.tar.gz"), gzip_expected) - eq(self.db.guess_type("?1.tar.gz"), gzip_expected) - eq(self.db.guess_type("#1.tar.gz"), gzip_expected) - eq(self.db.guess_type("#1#.tar.gz"), gzip_expected) - eq(self.db.guess_type(";1#.tar.gz"), gzip_expected) - eq(self.db.guess_type(";&1=123;?.tar.gz"), gzip_expected) - eq(self.db.guess_type("?k1=v1&k2=v2.tar.gz"), gzip_expected) + for name in ( + ';1.tar.gz', + '?1.tar.gz', + '#1.tar.gz', + '#1#.tar.gz', + ';1#.tar.gz', + ';&1=123;?.tar.gz', + '?k1=v1&k2=v2.tar.gz', + ): + for prefix in ('', '/', '\\', + 'c:', 'c:/', 'c:\\', 'c:/d/', 'c:\\d\\', + '//share/server/', '\\\\share\\server\\'): + path = prefix + name + with self.subTest(path=path): + eq(self.db.guess_file_type(path), gzip_expected) + eq(self.db.guess_type(path), gzip_expected) + expected = (None, None) if os.name == 'nt' else gzip_expected + for prefix in ('//', '\\\\', '//share/', '\\\\share\\'): + path = prefix + name + with self.subTest(path=path): + eq(self.db.guess_file_type(path), expected) + eq(self.db.guess_type(path), expected) + eq(self.db.guess_file_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) eq(self.db.guess_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) + eq(self.db.guess_file_type(r'foo/.tar.gz'), (None, 'gzip')) + eq(self.db.guess_type(r'foo/.tar.gz'), (None, 'gzip')) + expected = (None, 'gzip') if os.name == 'nt' else gzip_expected + eq(self.db.guess_file_type(r'foo\.tar.gz'), expected) + eq(self.db.guess_type(r'foo\.tar.gz'), expected) + eq(self.db.guess_type(r'scheme:foo\.tar.gz'), gzip_expected) + + def test_url(self): + result = self.db.guess_type('http://example.com/host.html') + result = self.db.guess_type('http://host.html') + msg = 'URL only has a host name, not a file' + self.assertSequenceEqual(result, (None, None), msg) + result = self.db.guess_type('http://example.com/host.html') + msg = 'Should be text/html' + self.assertSequenceEqual(result, ('text/html', None), msg) + result = self.db.guess_type('http://example.com/host.html#x.tar') + self.assertSequenceEqual(result, ('text/html', None)) + result = self.db.guess_type('http://example.com/host.html?q=x.tar') + self.assertSequenceEqual(result, ('text/html', None)) + def test_guess_all_types(self): # First try strict. Use a set here for testing the results because if # test_urllib2 is run before test_mimetypes, global state is modified @@ -171,24 +223,46 @@ def test_guess_known_extensions(self): def test_preferred_extension(self): def check_extensions(): - self.assertEqual(mimetypes.guess_extension('application/octet-stream'), '.bin') - self.assertEqual(mimetypes.guess_extension('application/postscript'), '.ps') - self.assertEqual(mimetypes.guess_extension('application/vnd.apple.mpegurl'), '.m3u') - self.assertEqual(mimetypes.guess_extension('application/vnd.ms-excel'), '.xls') - self.assertEqual(mimetypes.guess_extension('application/vnd.ms-powerpoint'), '.ppt') - self.assertEqual(mimetypes.guess_extension('application/x-texinfo'), '.texi') - self.assertEqual(mimetypes.guess_extension('application/x-troff'), '.roff') - self.assertEqual(mimetypes.guess_extension('application/xml'), '.xsl') - self.assertEqual(mimetypes.guess_extension('audio/mpeg'), '.mp3') - self.assertEqual(mimetypes.guess_extension('image/avif'), '.avif') - self.assertEqual(mimetypes.guess_extension('image/webp'), '.webp') - self.assertEqual(mimetypes.guess_extension('image/jpeg'), '.jpg') - self.assertEqual(mimetypes.guess_extension('image/tiff'), '.tiff') - self.assertEqual(mimetypes.guess_extension('message/rfc822'), '.eml') - self.assertEqual(mimetypes.guess_extension('text/html'), '.html') - self.assertEqual(mimetypes.guess_extension('text/plain'), '.txt') - self.assertEqual(mimetypes.guess_extension('video/mpeg'), '.mpeg') - self.assertEqual(mimetypes.guess_extension('video/quicktime'), '.mov') + for mime_type, ext in ( + ("application/octet-stream", ".bin"), + ("application/postscript", ".ps"), + ("application/vnd.apple.mpegurl", ".m3u"), + ("application/vnd.ms-excel", ".xls"), + ("application/vnd.ms-fontobject", ".eot"), + ("application/vnd.ms-powerpoint", ".ppt"), + ("application/x-texinfo", ".texi"), + ("application/x-troff", ".roff"), + ("application/xml", ".xsl"), + ("audio/matroska", ".mka"), + ("audio/mpeg", ".mp3"), + ("font/otf", ".otf"), + ("font/ttf", ".ttf"), + ("font/woff", ".woff"), + ("font/woff2", ".woff2"), + ("image/avif", ".avif"), + ("image/emf", ".emf"), + ("image/fits", ".fits"), + ("image/g3fax", ".g3"), + ("image/jp2", ".jp2"), + ("image/jpm", ".jpm"), + ("image/t38", ".t38"), + ("image/webp", ".webp"), + ("image/wmf", ".wmf"), + ("image/jpeg", ".jpg"), + ("image/tiff", ".tiff"), + ("image/tiff-fx", ".tfx"), + ("message/rfc822", ".eml"), + ("text/html", ".html"), + ("text/plain", ".txt"), + ("text/rtf", ".rtf"), + ("text/x-rst", ".rst"), + ("video/matroska", ".mkv"), + ("video/matroska-3d", ".mk3d"), + ("video/mpeg", ".mpeg"), + ("video/quicktime", ".mov"), + ): + with self.subTest(mime_type=mime_type, ext=ext): + self.assertEqual(mimetypes.guess_extension(mime_type), ext) check_extensions() mimetypes.init() @@ -214,22 +288,38 @@ def test_init_stability(self): def test_path_like_ob(self): filename = "LICENSE.txt" - filepath = pathlib.Path(filename) - filepath_with_abs_dir = pathlib.Path('/dir/'+filename) - filepath_relative = pathlib.Path('../dir/'+filename) - path_dir = pathlib.Path('./') + filepath = os_helper.FakePath(filename) + filepath_with_abs_dir = os_helper.FakePath('/dir/'+filename) + filepath_relative = os_helper.FakePath('../dir/'+filename) + path_dir = os_helper.FakePath('./') - expected = self.db.guess_type(filename) + expected = self.db.guess_file_type(filename) + self.assertEqual(self.db.guess_file_type(filepath), expected) self.assertEqual(self.db.guess_type(filepath), expected) + self.assertEqual(self.db.guess_file_type( + filepath_with_abs_dir), expected) self.assertEqual(self.db.guess_type( filepath_with_abs_dir), expected) + self.assertEqual(self.db.guess_file_type(filepath_relative), expected) self.assertEqual(self.db.guess_type(filepath_relative), expected) + + self.assertEqual(self.db.guess_file_type(path_dir), (None, None)) self.assertEqual(self.db.guess_type(path_dir), (None, None)) + def test_bytes_path(self): + self.assertEqual(self.db.guess_file_type(b'foo.html'), + self.db.guess_file_type('foo.html')) + self.assertEqual(self.db.guess_file_type(b'foo.tar.gz'), + self.db.guess_file_type('foo.tar.gz')) + self.assertEqual(self.db.guess_file_type(b'foo.tgz'), + self.db.guess_file_type('foo.tgz')) + def test_keywords_args_api(self): + self.assertEqual(self.db.guess_file_type( + path="foo.html", strict=True), ("text/html", None)) self.assertEqual(self.db.guess_type( - url="foo.html", strict=True), ("text/html", None)) + url="scheme:foo.html", strict=True), ("text/html", None)) self.assertEqual(self.db.guess_all_extensions( type='image/jpg', strict=True), []) self.assertEqual(self.db.guess_extension( diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index ac759757d24659..b2a299ed172967 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -1,8 +1,10 @@ from test.support import ( requires, _2G, _4G, gc_collect, cpython_only, is_emscripten, is_apple, + in_systemd_nspawn_sync_suppressed, ) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink +from test.support.script_helper import assert_python_ok import unittest import errno import os @@ -12,6 +14,7 @@ import socket import string import sys +import textwrap import weakref # Skip test if we can't import mmap. @@ -837,7 +840,8 @@ def test_flush_return_value(self): mm.write(b'python') result = mm.flush() self.assertIsNone(result) - if sys.platform.startswith('linux'): + if (sys.platform.startswith(('linux', 'android')) + and not in_systemd_nspawn_sync_suppressed()): # 'offset' must be a multiple of mmap.PAGESIZE on Linux. # See bpo-34754 for details. self.assertRaises(OSError, mm.flush, 1, len(b'python')) @@ -1058,6 +1062,81 @@ def __exit__(self, exc_type, exc_value, traceback): with self.assertRaisesRegex(ValueError, "mmap closed or invalid"): m.write_byte(X()) + @unittest.skipUnless(os.name == 'nt', 'requires Windows') + @unittest.skipUnless(hasattr(mmap.mmap, '_protect'), 'test needs debug build') + def test_access_violations(self): + from test.support.os_helper import TESTFN + + code = textwrap.dedent(""" + import faulthandler + import mmap + import os + import sys + from contextlib import suppress + + # Prevent logging access violations to stderr. + faulthandler.disable() + + PAGESIZE = mmap.PAGESIZE + PAGE_NOACCESS = 0x01 + + with open(sys.argv[1], 'bw+') as f: + f.write(b'A'* PAGESIZE) + f.flush() + + m = mmap.mmap(f.fileno(), PAGESIZE) + m._protect(PAGE_NOACCESS, 0, PAGESIZE) + with suppress(OSError): + m.read(PAGESIZE) + assert False, 'mmap.read() did not raise' + with suppress(OSError): + m.read_byte() + assert False, 'mmap.read_byte() did not raise' + with suppress(OSError): + m.readline() + assert False, 'mmap.readline() did not raise' + with suppress(OSError): + m.write(b'A'* PAGESIZE) + assert False, 'mmap.write() did not raise' + with suppress(OSError): + m.write_byte(0) + assert False, 'mmap.write_byte() did not raise' + with suppress(OSError): + m[0] # test mmap_subscript + assert False, 'mmap.__getitem__() did not raise' + with suppress(OSError): + m[0:10] # test mmap_subscript + assert False, 'mmap.__getitem__() did not raise' + with suppress(OSError): + m[0:10:2] # test mmap_subscript + assert False, 'mmap.__getitem__() did not raise' + with suppress(OSError): + m[0] = 1 + assert False, 'mmap.__setitem__() did not raise' + with suppress(OSError): + m[0:10] = b'A'* 10 + assert False, 'mmap.__setitem__() did not raise' + with suppress(OSError): + m[0:10:2] = b'A'* 5 + assert False, 'mmap.__setitem__() did not raise' + with suppress(OSError): + m.move(0, 10, 1) + assert False, 'mmap.move() did not raise' + with suppress(OSError): + list(m) # test mmap_item + assert False, 'mmap.__getitem__() did not raise' + with suppress(OSError): + m.find(b'A') + assert False, 'mmap.find() did not raise' + with suppress(OSError): + m.rfind(b'A') + assert False, 'mmap.rfind() did not raise' + """) + rt, stdout, stderr = assert_python_ok("-c", code, TESTFN) + self.assertEqual(stdout.strip(), b'') + self.assertEqual(stderr.strip(), b'') + + class LargeMmapTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_module/__init__.py b/Lib/test/test_module/__init__.py index 98d1cbe824df12..22132b01c8a056 100644 --- a/Lib/test/test_module/__init__.py +++ b/Lib/test/test_module/__init__.py @@ -357,6 +357,8 @@ def test_annotations_are_created_correctly(self): ann_module4 = import_helper.import_fresh_module( 'test.typinganndata.ann_module4', ) + self.assertFalse("__annotations__" in ann_module4.__dict__) + self.assertEqual(ann_module4.__annotations__, {"a": int, "b": str}) self.assertTrue("__annotations__" in ann_module4.__dict__) del ann_module4.__annotations__ self.assertFalse("__annotations__" in ann_module4.__dict__) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 1e77eb6a2eea4c..3125d190626e38 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -3,14 +3,18 @@ import collections import dis import functools +import math import operator import sys import textwrap import types import unittest -import asyncio -from test import support -from test.support import requires_specialization, script_helper + +import test.support +from test.support import requires_specialization_ft, script_helper + +_testcapi = test.support.import_helper.import_module("_testcapi") +_testinternalcapi = test.support.import_helper.import_module("_testinternalcapi") PAIR = (0,1) @@ -42,10 +46,14 @@ def nth_line(func, offset): class MonitoringBasicTest(unittest.TestCase): + def tearDown(self): + sys.monitoring.free_tool_id(TEST_TOOL) + def test_has_objects(self): m = sys.monitoring m.events m.use_tool_id + m.clear_tool_id m.free_tool_id m.get_tool m.get_events @@ -73,6 +81,43 @@ def test_tool(self): with self.assertRaises(ValueError): sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.CALL) + def test_clear(self): + events = [] + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, lambda *args: events.append(args)) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, lambda *args: events.append(args)) + def f(): + a = 1 + sys.monitoring.set_local_events(TEST_TOOL, f.__code__, E.LINE) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + + f() + sys.monitoring.clear_tool_id(TEST_TOOL) + f() + + # the first f() should trigger a PY_START and a LINE event + # the second f() after clear_tool_id should not trigger any event + # the callback function should be cleared as well + self.assertEqual(len(events), 2) + callback = sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + self.assertIs(callback, None) + + sys.monitoring.free_tool_id(TEST_TOOL) + + events = [] + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") + sys.monitoring.register_callback(TEST_TOOL, E.LINE, lambda *args: events.append(args)) + sys.monitoring.set_local_events(TEST_TOOL, f.__code__, E.LINE) + f() + sys.monitoring.free_tool_id(TEST_TOOL) + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") + f() + # the first f() should trigger a LINE event, and even if we use the + # tool id immediately after freeing it, the second f() should not + # trigger any event + self.assertEqual(len(events), 1) + sys.monitoring.free_tool_id(TEST_TOOL) + class MonitoringTestBase: @@ -651,6 +696,17 @@ def func2(): self.check_lines(func2, [1,2,3,4,5,6]) + def test_generator_with_line(self): + + def f(): + def a(): + yield + def b(): + yield from a() + next(b()) + + self.check_lines(f, [1,3,5,4,2,4]) + class TestDisable(MonitoringTestBase, unittest.TestCase): def gen(self, cond): @@ -729,8 +785,6 @@ def get_events(self, func, tool, recorders): def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): events = self.get_events(func, tool, recorders) - if events != expected: - print(events, file = sys.stderr) self.assertEqual(events, expected) def check_balanced(self, func, recorders): @@ -796,6 +850,7 @@ def __init__(self, events): def __call__(self, code, offset, val): self.events.append(("return", code.co_name, val)) + class ExceptionMonitoringTest(CheckEvents): exception_recorders = ( @@ -817,20 +872,43 @@ def func1(): self.check_events(func1, [("raise", KeyError)]) - # gh-116090: This test doesn't really require specialization, but running - # it without specialization exposes a monitoring bug. - @requires_specialization def test_implicit_stop_iteration(self): + """Generators are documented as raising a StopIteration + when they terminate. + However, we don't do that if we can avoid it, for speed. + sys.monitoring handles that by injecting a STOP_ITERATION + event when we would otherwise have skip the RAISE event. + This test checks that both paths record an equivalent event. + """ def gen(): yield 1 return 2 - def implicit_stop_iteration(): - for _ in gen(): + def implicit_stop_iteration(iterator=None): + if iterator is None: + iterator = gen() + for _ in iterator: pass - self.check_events(implicit_stop_iteration, [("raise", StopIteration)], recorders=(StopiterationRecorder,)) + recorders=(ExceptionRecorder, StopiterationRecorder,) + expected = [("raise", StopIteration)] + + # Make sure that the loop is unspecialized, and that it will not + # re-specialize immediately, so that we can we can test the + # unspecialized version of the loop first. + # Note: this assumes that we don't specialize loops over sets. + implicit_stop_iteration(set(range(_testinternalcapi.SPECIALIZATION_THRESHOLD))) + + # This will record a RAISE event for the StopIteration. + self.check_events(implicit_stop_iteration, expected, recorders=recorders) + + # Now specialize, so that we see a STOP_ITERATION event. + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): + implicit_stop_iteration() + + # This will record a STOP_ITERATION event for the StopIteration. + self.check_events(implicit_stop_iteration, expected, recorders=recorders) initial = [ ("raise", ZeroDivisionError), @@ -968,20 +1046,19 @@ def func(): ) self.assertEqual(events[0], ("throw", IndexError)) - @requires_specialization + @requires_specialization_ft def test_no_unwind_for_shim_frame(self): - - class B: + class ValueErrorRaiser: def __init__(self): raise ValueError() def f(): try: - return B() + return ValueErrorRaiser() except ValueError: pass - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): f() recorders = ( ReturnRecorder, @@ -1128,6 +1205,7 @@ def func1(): ('instruction', 'func1', 10), ('instruction', 'func1', 12), ('instruction', 'func1', 14), + ('instruction', 'func1', 16), ('line', 'get_events', 11)]) def test_c_call(self): @@ -1152,6 +1230,7 @@ def func2(): ('instruction', 'func2', 40), ('instruction', 'func2', 42), ('instruction', 'func2', 44), + ('instruction', 'func2', 46), ('line', 'get_events', 11)]) def test_try_except(self): @@ -1185,6 +1264,7 @@ def func3(): ('instruction', 'func3', 30), ('instruction', 'func3', 32), ('instruction', 'func3', 34), + ('instruction', 'func3', 36), ('line', 'get_events', 11)]) def test_with_restart(self): @@ -1205,6 +1285,7 @@ def func1(): ('instruction', 'func1', 10), ('instruction', 'func1', 12), ('instruction', 'func1', 14), + ('instruction', 'func1', 16), ('line', 'get_events', 11)]) sys.monitoring.restart_events() @@ -1221,6 +1302,7 @@ def func1(): ('instruction', 'func1', 10), ('instruction', 'func1', 12), ('instruction', 'func1', 14), + ('instruction', 'func1', 16), ('line', 'get_events', 11)]) def test_turn_off_only_instruction(self): @@ -1406,12 +1488,45 @@ class BranchRecorder(JumpRecorder): event_type = E.BRANCH name = "branch" +class BranchRightRecorder(JumpRecorder): + + event_type = E.BRANCH_RIGHT + name = "branch right" + +class BranchLeftRecorder(JumpRecorder): + + event_type = E.BRANCH_LEFT + name = "branch left" + +class JumpOffsetRecorder: + + event_type = E.JUMP + name = "jump" + + def __init__(self, events, offsets=False): + self.events = events + + def __call__(self, code, from_, to): + self.events.append((self.name, code.co_name, from_, to)) + +class BranchLeftOffsetRecorder(JumpOffsetRecorder): + + event_type = E.BRANCH_LEFT + name = "branch left" + +class BranchRightOffsetRecorder(JumpOffsetRecorder): + + event_type = E.BRANCH_RIGHT + name = "branch right" JUMP_AND_BRANCH_RECORDERS = JumpRecorder, BranchRecorder JUMP_BRANCH_AND_LINE_RECORDERS = JumpRecorder, BranchRecorder, LineRecorder FLOW_AND_LINE_RECORDERS = JumpRecorder, BranchRecorder, LineRecorder, ExceptionRecorder, ReturnRecorder +BRANCHES_RECORDERS = BranchLeftRecorder, BranchRightRecorder +BRANCH_OFFSET_RECORDERS = BranchLeftOffsetRecorder, BranchRightOffsetRecorder + class TestBranchAndJumpEvents(CheckEvents): maxDiff = None @@ -1426,6 +1541,11 @@ def func(): x = 6 7 + def whilefunc(n=0): + while n < 3: + n += 1 # line 2 + 3 + self.check_events(func, recorders = JUMP_AND_BRANCH_RECORDERS, expected = [ ('branch', 'func', 2, 2), ('branch', 'func', 3, 6), @@ -1455,6 +1575,26 @@ def func(): ('line', 'func', 7), ('line', 'get_events', 11)]) + self.check_events(func, recorders = BRANCHES_RECORDERS, expected = [ + ('branch left', 'func', 2, 2), + ('branch right', 'func', 3, 6), + ('branch left', 'func', 2, 2), + ('branch left', 'func', 3, 4), + ('branch right', 'func', 2, 7)]) + + self.check_events(whilefunc, recorders = BRANCHES_RECORDERS, expected = [ + ('branch left', 'whilefunc', 1, 2), + ('branch left', 'whilefunc', 1, 2), + ('branch left', 'whilefunc', 1, 2), + ('branch right', 'whilefunc', 1, 3)]) + + self.check_events(func, recorders = BRANCH_OFFSET_RECORDERS, expected = [ + ('branch left', 'func', 28, 32), + ('branch right', 'func', 44, 58), + ('branch left', 'func', 28, 32), + ('branch left', 'func', 44, 50), + ('branch right', 'func', 28, 70)]) + def test_except_star(self): class Foo: @@ -1480,8 +1620,8 @@ def func(): ('branch', 'func', 4, 4), ('line', 'func', 5), ('line', 'meth', 1), - ('jump', 'func', 5, '[offset=118]'), - ('branch', 'func', '[offset=122]', '[offset=126]'), + ('jump', 'func', 5, '[offset=120]'), + ('branch', 'func', '[offset=124]', '[offset=130]'), ('line', 'get_events', 11)]) self.check_events(func, recorders = FLOW_AND_LINE_RECORDERS, expected = [ @@ -1495,11 +1635,111 @@ def func(): ('line', 'func', 5), ('line', 'meth', 1), ('return', 'meth', None), - ('jump', 'func', 5, '[offset=118]'), - ('branch', 'func', '[offset=122]', '[offset=126]'), + ('jump', 'func', 5, '[offset=120]'), + ('branch', 'func', '[offset=124]', '[offset=130]'), ('return', 'func', None), ('line', 'get_events', 11)]) + def test_while_offset_consistency(self): + + def foo(n=0): + while n<4: + pass + n += 1 + return None + + in_loop = ('branch left', 'foo', 10, 16) + exit_loop = ('branch right', 'foo', 10, 40) + self.check_events(foo, recorders = BRANCH_OFFSET_RECORDERS, expected = [ + in_loop, + in_loop, + in_loop, + in_loop, + exit_loop]) + + +class TestBranchConsistency(MonitoringTestBase, unittest.TestCase): + + def check_branches(self, func, tool=TEST_TOOL, recorders=BRANCH_OFFSET_RECORDERS): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + event_list = [] + all_events = 0 + for recorder in recorders: + ev = recorder.event_type + sys.monitoring.register_callback(tool, ev, recorder(event_list)) + all_events |= ev + sys.monitoring.set_local_events(tool, func.__code__, all_events) + func() + sys.monitoring.set_local_events(tool, func.__code__, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + lefts = set() + rights = set() + for (src, left, right) in func.__code__.co_branches(): + lefts.add((src, left)) + rights.add((src, right)) + for event in event_list: + way, _, src, dest = event + if "left" in way: + self.assertIn((src, dest), lefts) + else: + self.assertIn("right", way) + self.assertIn((src, dest), rights) + finally: + sys.monitoring.set_local_events(tool, func.__code__, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + + def test_simple(self): + + def func(): + x = 1 + for a in range(2): + if a: + x = 4 + else: + x = 6 + 7 + + self.check_branches(func) + + def whilefunc(n=0): + while n < 3: + n += 1 # line 2 + 3 + + self.check_branches(whilefunc) + + def test_except_star(self): + + class Foo: + def meth(self): + pass + + def func(): + try: + try: + raise KeyError + except* Exception as e: + f = Foo(); f.meth() + except KeyError: + pass + + + self.check_branches(func) + + def test4(self): + + def foo(n=0): + while n<4: + pass + n += 1 + return None + + self.check_branches(foo) + + class TestLoadSuperAttr(CheckEvents): RECORDERS = CallRecorder, LineRecorder, CRaiseRecorder, CReturnRecorder @@ -1560,7 +1800,7 @@ def f(): ('line', 'method', 2), ('line', 'method', 3), ('line', 'method', 2), - ('call', 'method', 1), + ('call', 'method', d["b"]), ('line', 'method', 1), ('line', 'method', 1), ('line', 'get_events', 11), @@ -1731,6 +1971,10 @@ def test_local(self): code = f1.__code__ sys.monitoring.set_local_events(TEST_TOOL, code, E.PY_START) self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), E.PY_START) + sys.monitoring.set_local_events(TEST_TOOL, code, 0) + sys.monitoring.set_local_events(TEST_TOOL, code, E.BRANCH) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), E.BRANCH_LEFT | E.BRANCH_RIGHT) + sys.monitoring.set_local_events(TEST_TOOL, code, 0) sys.monitoring.set_local_events(TEST_TOOL2, code, E.PY_START) self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL2, code), E.PY_START) sys.monitoring.set_local_events(TEST_TOOL, code, 0) @@ -1790,8 +2034,8 @@ def __init__(self, set_event): sys.monitoring.set_events(TEST_TOOL, E.PY_RESUME) def make_foo_optimized_then_set_event(): - for i in range(100): - Foo(i == 99) + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD + 1): + Foo(i == _testinternalcapi.SPECIALIZATION_THRESHOLD) try: make_foo_optimized_then_set_event() @@ -1807,20 +2051,41 @@ def test_gh108976(self): sys.monitoring.set_events(0, E.LINE | E.INSTRUCTION) sys.monitoring.set_events(0, 0) + def test_call_function_ex(self): + def f(a=1, b=2): + return a + b + args = (1, 2) + empty_args = [] -class TestOptimizer(MonitoringTestBase, unittest.TestCase): + call_data = [] + sys.monitoring.use_tool_id(0, "test") + self.addCleanup(sys.monitoring.free_tool_id, 0) + sys.monitoring.set_events(0, 0) + sys.monitoring.register_callback(0, E.CALL, lambda code, offset, callable, arg0: call_data.append((callable, arg0))) + sys.monitoring.set_events(0, E.CALL) + f(*args) + f(*empty_args) + sys.monitoring.set_events(0, 0) + self.assertEqual(call_data[0], (f, 1)) + self.assertEqual(call_data[1], (f, sys.monitoring.MISSING)) + + def test_instruction_explicit_callback(self): + # gh-122247 + # Calling the instruction event callback explicitly should not + # crash CPython + def callback(code, instruction_offset): + pass - def setUp(self): - import _testinternalcapi - self.old_opt = _testinternalcapi.get_optimizer() - opt = _testinternalcapi.new_counter_optimizer() - _testinternalcapi.set_optimizer(opt) - super(TestOptimizer, self).setUp() + sys.monitoring.use_tool_id(0, "test") + self.addCleanup(sys.monitoring.free_tool_id, 0) + sys.monitoring.register_callback(0, sys.monitoring.events.INSTRUCTION, callback) + sys.monitoring.set_events(0, sys.monitoring.events.INSTRUCTION) + callback(None, 0) # call the *same* handler while it is registered + sys.monitoring.restart_events() + sys.monitoring.set_events(0, 0) - def tearDown(self): - import _testinternalcapi - super(TestOptimizer, self).tearDown() - _testinternalcapi.set_optimizer(self.old_opt) + +class TestOptimizer(MonitoringTestBase, unittest.TestCase): def test_for_loop(self): def test_func(x): @@ -1842,9 +2107,9 @@ def test_func(recorder): set_events = sys.monitoring.set_events line = E.LINE i = 0 - for i in range(551): - # Turn on events without branching once i reaches 500. - set_events(TEST_TOOL, line * int(i >= 500)) + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD + 51): + # Turn on events without branching once i reaches _testinternalcapi.SPECIALIZATION_THRESHOLD. + set_events(TEST_TOOL, line * int(i >= _testinternalcapi.SPECIALIZATION_THRESHOLD)) pass pass pass @@ -1866,5 +2131,202 @@ def test_monitoring_live_at_shutdown(self): # gh-115832: An object destructor running during the final GC of # interpreter shutdown triggered an infinite loop in the # instrumentation code. - script = support.findfile("_test_monitoring_shutdown.py") + script = test.support.findfile("_test_monitoring_shutdown.py") script_helper.run_test_script(script) + + +class TestCApiEventGeneration(MonitoringTestBase, unittest.TestCase): + + class Scope: + def __init__(self, *args): + self.args = args + + def __enter__(self): + _testcapi.monitoring_enter_scope(*self.args) + + def __exit__(self, *args): + _testcapi.monitoring_exit_scope() + + def setUp(self): + super(TestCApiEventGeneration, self).setUp() + + capi = _testcapi + + self.codelike = capi.CodeLike(2) + + self.cases = [ + # (Event, function, *args) + ( 1, E.PY_START, capi.fire_event_py_start), + ( 1, E.PY_RESUME, capi.fire_event_py_resume), + ( 1, E.PY_YIELD, capi.fire_event_py_yield, 10), + ( 1, E.PY_RETURN, capi.fire_event_py_return, 20), + ( 2, E.CALL, capi.fire_event_call, callable, 40), + ( 1, E.JUMP, capi.fire_event_jump, 60), + ( 1, E.BRANCH_RIGHT, capi.fire_event_branch_right, 70), + ( 1, E.BRANCH_LEFT, capi.fire_event_branch_left, 80), + ( 1, E.PY_THROW, capi.fire_event_py_throw, ValueError(1)), + ( 1, E.RAISE, capi.fire_event_raise, ValueError(2)), + ( 1, E.EXCEPTION_HANDLED, capi.fire_event_exception_handled, ValueError(5)), + ( 1, E.PY_UNWIND, capi.fire_event_py_unwind, ValueError(6)), + ( 1, E.STOP_ITERATION, capi.fire_event_stop_iteration, 7), + ( 1, E.STOP_ITERATION, capi.fire_event_stop_iteration, StopIteration(8)), + ] + + self.EXPECT_RAISED_EXCEPTION = [E.PY_THROW, E.RAISE, E.EXCEPTION_HANDLED, E.PY_UNWIND] + + + def check_event_count(self, event, func, args, expected, callback_raises=None): + class Counter: + def __init__(self, callback_raises): + self.callback_raises = callback_raises + self.count = 0 + + def __call__(self, *args): + self.count += 1 + if self.callback_raises: + exc = self.callback_raises + self.callback_raises = None + raise exc + + try: + counter = Counter(callback_raises) + sys.monitoring.register_callback(TEST_TOOL, event, counter) + if event == E.C_RETURN or event == E.C_RAISE: + sys.monitoring.set_events(TEST_TOOL, E.CALL) + else: + sys.monitoring.set_events(TEST_TOOL, event) + event_value = int(math.log2(event)) + with self.Scope(self.codelike, event_value): + counter.count = 0 + try: + func(*args) + except ValueError as e: + self.assertIsInstance(expected, ValueError) + self.assertEqual(str(e), str(expected)) + return + else: + self.assertEqual(counter.count, expected) + + prev = sys.monitoring.register_callback(TEST_TOOL, event, None) + with self.Scope(self.codelike, event_value): + counter.count = 0 + func(*args) + self.assertEqual(counter.count, 0) + self.assertEqual(prev, counter) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_fire_event(self): + for expected, event, function, *args in self.cases: + offset = 0 + self.codelike = _testcapi.CodeLike(1) + with self.subTest(function.__name__): + args_ = (self.codelike, offset) + tuple(args) + self.check_event_count(event, function, args_, expected) + + def test_missing_exception(self): + for _, event, function, *args in self.cases: + if event not in self.EXPECT_RAISED_EXCEPTION: + continue + assert args and isinstance(args[-1], BaseException) + offset = 0 + self.codelike = _testcapi.CodeLike(1) + with self.subTest(function.__name__): + args_ = (self.codelike, offset) + tuple(args[:-1]) + (None,) + evt = int(math.log2(event)) + expected = ValueError(f"Firing event {evt} with no exception set") + self.check_event_count(event, function, args_, expected) + + def test_fire_event_failing_callback(self): + for expected, event, function, *args in self.cases: + offset = 0 + self.codelike = _testcapi.CodeLike(1) + with self.subTest(function.__name__): + args_ = (self.codelike, offset) + tuple(args) + exc = OSError(42) + with self.assertRaises(type(exc)): + self.check_event_count(event, function, args_, expected, + callback_raises=exc) + + + CANNOT_DISABLE = { E.PY_THROW, E.RAISE, E.RERAISE, + E.EXCEPTION_HANDLED, E.PY_UNWIND } + + def check_disable(self, event, func, args, expected): + try: + counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, event, counter) + if event == E.C_RETURN or event == E.C_RAISE: + sys.monitoring.set_events(TEST_TOOL, E.CALL) + else: + sys.monitoring.set_events(TEST_TOOL, event) + event_value = int(math.log2(event)) + with self.Scope(self.codelike, event_value): + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected) + counter.disable = True + if event in self.CANNOT_DISABLE: + # use try-except rather then assertRaises to avoid + # events from framework code + try: + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected) + except ValueError: + pass + else: + self.Error("Expected a ValueError") + else: + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected) + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected - 1) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_disable_event(self): + for expected, event, function, *args in self.cases: + offset = 0 + self.codelike = _testcapi.CodeLike(2) + with self.subTest(function.__name__): + args_ = (self.codelike, 0) + tuple(args) + self.check_disable(event, function, args_, expected) + + def test_enter_scope_two_events(self): + try: + yield_counter = CounterWithDisable() + unwind_counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_YIELD, yield_counter) + sys.monitoring.register_callback(TEST_TOOL, E.PY_UNWIND, unwind_counter) + sys.monitoring.set_events(TEST_TOOL, E.PY_YIELD | E.PY_UNWIND) + + yield_value = int(math.log2(E.PY_YIELD)) + unwind_value = int(math.log2(E.PY_UNWIND)) + cl = _testcapi.CodeLike(2) + common_args = (cl, 0) + with self.Scope(cl, yield_value, unwind_value): + yield_counter.count = 0 + unwind_counter.count = 0 + + _testcapi.fire_event_py_unwind(*common_args, ValueError(42)) + assert(yield_counter.count == 0) + assert(unwind_counter.count == 1) + + _testcapi.fire_event_py_yield(*common_args, ValueError(42)) + assert(yield_counter.count == 1) + assert(unwind_counter.count == 1) + + yield_counter.disable = True + _testcapi.fire_event_py_yield(*common_args, ValueError(42)) + assert(yield_counter.count == 2) + assert(unwind_counter.count == 1) + + _testcapi.fire_event_py_yield(*common_args, ValueError(42)) + assert(yield_counter.count == 2) + assert(unwind_counter.count == 1) + + finally: + sys.monitoring.set_events(TEST_TOOL, 0) diff --git a/Lib/test/test_msvcrt.py b/Lib/test/test_msvcrt.py index 600c4446fd5cd4..1c6905bd1ee586 100644 --- a/Lib/test/test_msvcrt.py +++ b/Lib/test/test_msvcrt.py @@ -64,7 +64,7 @@ class TestConsoleIO(unittest.TestCase): # CREATE_NEW_CONSOLE creates a "popup" window. @requires_resource('gui') def run_in_separated_process(self, code): - # Run test in a seprated process to avoid stdin conflicts. + # Run test in a separated process to avoid stdin conflicts. # See: gh-110147 cmd = [sys.executable, '-c', code] subprocess.run(cmd, check=True, capture_output=True, diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index 6451df14696933..1b55f1e70b32f5 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -12,6 +12,7 @@ from test import support from test.support import os_helper from test.support.os_helper import TESTFN +from test.support.import_helper import import_module ALL_CJKENCODINGS = [ # _codecs_cn @@ -212,7 +213,7 @@ def test_issue5640(self): @support.cpython_only def test_subinterp(self): # bpo-42846: Test a CJK codec in a subinterpreter - import _testcapi + _testcapi = import_module("_testcapi") encoding = 'cp932' text = "Python の開発は、1990 年ごろから開始されています。" code = textwrap.dedent(""" @@ -303,7 +304,7 @@ def test_setstate_validates_input(self): self.assertRaises(TypeError, decoder.setstate, 123) self.assertRaises(TypeError, decoder.setstate, ("invalid", 0)) self.assertRaises(TypeError, decoder.setstate, (b"1234", "invalid")) - self.assertRaises(UnicodeError, decoder.setstate, (b"123456789", 0)) + self.assertRaises(UnicodeDecodeError, decoder.setstate, (b"123456789", 0)) class Test_StreamReader(unittest.TestCase): def test_bug1728403(self): diff --git a/Lib/test/test_multiprocessing_fork/__init__.py b/Lib/test/test_multiprocessing_fork/__init__.py index aa1fff50b28f5f..b35e82879d7fe2 100644 --- a/Lib/test/test_multiprocessing_fork/__init__.py +++ b/Lib/test/test_multiprocessing_fork/__init__.py @@ -12,5 +12,8 @@ if sys.platform == 'darwin': raise unittest.SkipTest("test may crash on macOS (bpo-33725)") +if support.check_sanitizer(thread=True): + raise unittest.SkipTest("TSAN doesn't support threads after fork") + def load_tests(*args): return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index f2017bdffcf968..cf44080670dc2e 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -298,6 +298,72 @@ def test_named_expression_invalid_set_comprehension_iterable_expression(self): with self.assertRaisesRegex(SyntaxError, msg): exec(f"lambda: {code}", {}) # Function scope + def test_named_expression_invalid_rebinding_dict_comprehension_iteration_variable(self): + cases = [ + ("Key reuse", 'i', "{(i := 0): 1 for i in range(5)}"), + ("Value reuse", 'i', "{1: (i := 0) for i in range(5)}"), + ("Both reuse", 'i', "{(i := 0): (i := 0) for i in range(5)}"), + ("Nested reuse", 'j', "{{(j := 0): 1 for i in range(5)} for j in range(5)}"), + ("Reuse inner loop target", 'j', "{(j := 0): 1 for i in range(5) for j in range(5)}"), + ("Unpacking key reuse", 'i', "{(i := 0): 1 for i, j in {(0, 1)}}"), + ("Unpacking value reuse", 'i', "{1: (i := 0) for i, j in {(0, 1)}}"), + ("Reuse in loop condition", 'i', "{i+1: 1 for i in range(5) if (i := 0)}"), + ("Unreachable reuse", 'i', "{(False or (i:=0)): 1 for i in range(5)}"), + ("Unreachable nested reuse", 'i', + "{i: j for i in range(5) for j in range(5) if True or (i:=10)}"), + # Regression tests from https://github.com/python/cpython/issues/87447 + ("Complex expression: a", "a", + "{(a := 1): 1 for a, (*b, c[d+e::f(g)], h.i) in j}"), + ("Complex expression: b", "b", + "{(b := 1): 1 for a, (*b, c[d+e::f(g)], h.i) in j}"), + ] + for case, target, code in cases: + msg = f"assignment expression cannot rebind comprehension iteration variable '{target}'" + with self.subTest(case=case): + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}) # Module scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}, {}) # Class scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(f"lambda: {code}", {}) # Function scope + + def test_named_expression_invalid_rebinding_dict_comprehension_inner_loop(self): + cases = [ + ("Inner reuse", 'j', "{i: 1 for i in range(5) if (j := 0) for j in range(5)}"), + ("Inner unpacking reuse", 'j', "{i: 1 for i in range(5) if (j := 0) for j, k in {(0, 1)}}"), + ] + for case, target, code in cases: + msg = f"comprehension inner loop cannot rebind assignment expression target '{target}'" + with self.subTest(case=case): + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}) # Module scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}, {}) # Class scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(f"lambda: {code}", {}) # Function scope + + def test_named_expression_invalid_dict_comprehension_iterable_expression(self): + cases = [ + ("Top level", "{i: 1 for i in (i := range(5))}"), + ("Inside tuple", "{i: 1 for i in (2, 3, i := range(5))}"), + ("Inside list", "{i: 1 for i in [2, 3, i := range(5)]}"), + ("Different name", "{i: 1 for i in (j := range(5))}"), + ("Lambda expression", "{i: 1 for i in (lambda:(j := range(5)))()}"), + ("Inner loop", "{i: 1 for i in range(5) for j in (i := range(5))}"), + ("Nested comprehension", "{i: 1 for i in {j: 2 for j in (k := range(5))}}"), + ("Nested comprehension condition", "{i: 1 for i in {j: 2 for j in range(5) if (j := True)}}"), + ("Nested comprehension body", "{i: 1 for i in {(j := True) for j in range(5)}}"), + ] + msg = "assignment expression cannot be used in a comprehension iterable expression" + for case, code in cases: + with self.subTest(case=case): + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}) # Module scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}, {}) # Class scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(f"lambda: {code}", {}) # Function scope + def test_named_expression_invalid_mangled_class_variables(self): code = """class Foo: def bar(self): @@ -361,7 +427,7 @@ def test_named_expression_assignment_09(self): def test_named_expression_assignment_10(self): if (match := 10) == 10: - pass + self.assertEqual(match, 10) else: self.fail("variable was not assigned using named expression") def test_named_expression_assignment_11(self): @@ -403,7 +469,7 @@ def test_named_expression_assignment_14(self): def test_named_expression_assignment_15(self): while a := False: - pass # This will not run + self.fail("While body executed") # This will not run self.assertEqual(a, False) diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 9cb03e3cd5de8d..da01c65a1c2954 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -347,13 +347,18 @@ def test_normpath(self): tester("ntpath.normpath('..')", r'..') tester("ntpath.normpath('.')", r'.') + tester("ntpath.normpath('c:.')", 'c:') tester("ntpath.normpath('')", r'.') tester("ntpath.normpath('/')", '\\') tester("ntpath.normpath('c:/')", 'c:\\') tester("ntpath.normpath('/../.././..')", '\\') tester("ntpath.normpath('c:/../../..')", 'c:\\') + tester("ntpath.normpath('/./a/b')", r'\a\b') + tester("ntpath.normpath('c:/./a/b')", r'c:\a\b') tester("ntpath.normpath('../.././..')", r'..\..\..') tester("ntpath.normpath('K:../.././..')", r'K:..\..\..') + tester("ntpath.normpath('./a/b')", r'a\b') + tester("ntpath.normpath('c:./a/b')", r'c:a\b') tester("ntpath.normpath('C:////a/b')", r'C:\a\b') tester("ntpath.normpath('//machine/share//a/b')", r'\\machine\share\a\b') @@ -374,6 +379,7 @@ def test_normpath(self): tester("ntpath.normpath('\\\\foo\\')", '\\\\foo\\') tester("ntpath.normpath('\\\\foo')", '\\\\foo') tester("ntpath.normpath('\\\\')", '\\\\') + tester("ntpath.normpath('//?/UNC/server/share/..')", '\\\\?\\UNC\\server\\share\\') def test_realpath_curdir(self): expected = ntpath.normpath(os.getcwd()) @@ -805,6 +811,9 @@ def test_abspath(self): tester('ntpath.abspath("C:\\spam. . .")', "C:\\spam") tester('ntpath.abspath("C:/nul")', "\\\\.\\nul") tester('ntpath.abspath("C:\\nul")', "\\\\.\\nul") + self.assertTrue(ntpath.isabs(ntpath.abspath("C:spam"))) + self.assertEqual(ntpath.abspath("C:\x00"), ntpath.join(ntpath.abspath("C:"), "\x00")) + self.assertEqual(ntpath.abspath("\x00:spam"), "\x00:\\spam") tester('ntpath.abspath("//..")', "\\\\") tester('ntpath.abspath("//../")', "\\\\..\\") tester('ntpath.abspath("//../..")', "\\\\..\\") @@ -866,43 +875,47 @@ def test_commonpath(self): def check(paths, expected): tester(('ntpath.commonpath(%r)' % paths).replace('\\\\', '\\'), expected) - def check_error(exc, paths): - self.assertRaises(exc, ntpath.commonpath, paths) - self.assertRaises(exc, ntpath.commonpath, - [os.fsencode(p) for p in paths]) - + def check_error(paths, expected): + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, paths) + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, paths[::-1]) + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, + [os.fsencode(p) for p in paths]) + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, + [os.fsencode(p) for p in paths[::-1]]) + + self.assertRaises(TypeError, ntpath.commonpath, None) self.assertRaises(ValueError, ntpath.commonpath, []) - check_error(ValueError, ['C:\\Program Files', 'Program Files']) - check_error(ValueError, ['C:\\Program Files', 'C:Program Files']) - check_error(ValueError, ['\\Program Files', 'Program Files']) - check_error(ValueError, ['Program Files', 'C:\\Program Files']) - check(['C:\\Program Files'], 'C:\\Program Files') - check(['C:\\Program Files', 'C:\\Program Files'], 'C:\\Program Files') - check(['C:\\Program Files\\', 'C:\\Program Files'], - 'C:\\Program Files') - check(['C:\\Program Files\\', 'C:\\Program Files\\'], - 'C:\\Program Files') - check(['C:\\\\Program Files', 'C:\\Program Files\\\\'], - 'C:\\Program Files') - check(['C:\\.\\Program Files', 'C:\\Program Files\\.'], - 'C:\\Program Files') - check(['C:\\', 'C:\\bin'], 'C:\\') - check(['C:\\Program Files', 'C:\\bin'], 'C:\\') - check(['C:\\Program Files', 'C:\\Program Files\\Bar'], - 'C:\\Program Files') - check(['C:\\Program Files\\Foo', 'C:\\Program Files\\Bar'], - 'C:\\Program Files') - check(['C:\\Program Files', 'C:\\Projects'], 'C:\\') - check(['C:\\Program Files\\', 'C:\\Projects'], 'C:\\') - - check(['C:\\Program Files\\Foo', 'C:/Program Files/Bar'], - 'C:\\Program Files') - check(['C:\\Program Files\\Foo', 'c:/program files/bar'], - 'C:\\Program Files') - check(['c:/program files/bar', 'C:\\Program Files\\Foo'], - 'c:\\program files') - - check_error(ValueError, ['C:\\Program Files', 'D:\\Program Files']) + self.assertRaises(ValueError, ntpath.commonpath, iter([])) + + # gh-117381: Logical error messages + check_error(['C:\\Foo', 'C:Foo'], "Can't mix absolute and relative paths") + check_error(['C:\\Foo', '\\Foo'], "Paths don't have the same drive") + check_error(['C:\\Foo', 'Foo'], "Paths don't have the same drive") + check_error(['C:Foo', '\\Foo'], "Paths don't have the same drive") + check_error(['C:Foo', 'Foo'], "Paths don't have the same drive") + check_error(['\\Foo', 'Foo'], "Can't mix rooted and not-rooted paths") + + check(['C:\\Foo'], 'C:\\Foo') + check(['C:\\Foo', 'C:\\Foo'], 'C:\\Foo') + check(['C:\\Foo\\', 'C:\\Foo'], 'C:\\Foo') + check(['C:\\Foo\\', 'C:\\Foo\\'], 'C:\\Foo') + check(['C:\\\\Foo', 'C:\\Foo\\\\'], 'C:\\Foo') + check(['C:\\.\\Foo', 'C:\\Foo\\.'], 'C:\\Foo') + check(['C:\\', 'C:\\baz'], 'C:\\') + check(['C:\\Bar', 'C:\\baz'], 'C:\\') + check(['C:\\Foo', 'C:\\Foo\\Baz'], 'C:\\Foo') + check(['C:\\Foo\\Bar', 'C:\\Foo\\Baz'], 'C:\\Foo') + check(['C:\\Bar', 'C:\\Baz'], 'C:\\') + check(['C:\\Bar\\', 'C:\\Baz'], 'C:\\') + + check(['C:\\Foo\\Bar', 'C:/Foo/Baz'], 'C:\\Foo') + check(['C:\\Foo\\Bar', 'c:/foo/baz'], 'C:\\Foo') + check(['c:/foo/bar', 'C:\\Foo\\Baz'], 'c:\\foo') + + # gh-117381: Logical error messages + check_error(['C:\\Foo', 'D:\\Foo'], "Paths don't have the same drive") + check_error(['C:\\Foo', 'D:Foo'], "Paths don't have the same drive") + check_error(['C:Foo', 'D:Foo'], "Paths don't have the same drive") check(['spam'], 'spam') check(['spam', 'spam'], 'spam') @@ -916,22 +929,18 @@ def check_error(exc, paths): check([''], '') check(['', 'spam\\alot'], '') - check_error(ValueError, ['', '\\spam\\alot']) - - self.assertRaises(TypeError, ntpath.commonpath, - [b'C:\\Program Files', 'C:\\Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - [b'C:\\Program Files', 'Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - [b'Program Files', 'C:\\Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - ['C:\\Program Files', b'C:\\Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - ['C:\\Program Files', b'Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - ['Program Files', b'C:\\Program Files\\Foo']) - - @unittest.skipIf(is_emscripten, "Emscripten cannot fstat unnamed files.") + + # gh-117381: Logical error messages + check_error(['', '\\spam\\alot'], "Can't mix rooted and not-rooted paths") + + self.assertRaises(TypeError, ntpath.commonpath, [b'C:\\Foo', 'C:\\Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, [b'C:\\Foo', 'Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, [b'Foo', 'C:\\Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, ['C:\\Foo', b'C:\\Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, ['C:\\Foo', b'Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, ['Foo', b'C:\\Foo\\Baz']) + + @unittest.skipIf(is_emscripten, "Fixed in next Emscripten release after 4.0.1") def test_sameopenfile(self): with TemporaryFile() as tf1, TemporaryFile() as tf2: # Make sure the same file is really the same @@ -1094,6 +1103,27 @@ def test_isfile_driveletter(self): raise unittest.SkipTest('SystemDrive is not defined or malformed') self.assertFalse(os.path.isfile('\\\\.\\' + drive)) + @unittest.skipUnless(hasattr(os, 'pipe'), "need os.pipe()") + def test_isfile_anonymous_pipe(self): + pr, pw = os.pipe() + try: + self.assertFalse(ntpath.isfile(pr)) + finally: + os.close(pr) + os.close(pw) + + @unittest.skipIf(sys.platform != 'win32', "windows only") + def test_isfile_named_pipe(self): + import _winapi + named_pipe = f'//./PIPE/python_isfile_test_{os.getpid()}' + h = _winapi.CreateNamedPipe(named_pipe, + _winapi.PIPE_ACCESS_INBOUND, + 0, 1, 0, 0, 0, 0) + try: + self.assertFalse(ntpath.isfile(named_pipe)) + finally: + _winapi.CloseHandle(h) + @unittest.skipIf(sys.platform != 'win32', "windows only") def test_con_device(self): self.assertFalse(os.path.isfile(r"\\.\CON")) @@ -1107,14 +1137,22 @@ def test_fast_paths_in_use(self): # There are fast paths of these functions implemented in posixmodule.c. # Confirm that they are being used, and not the Python fallbacks in # genericpath.py. + self.assertTrue(os.path.splitroot is nt._path_splitroot_ex) + self.assertFalse(inspect.isfunction(os.path.splitroot)) + self.assertTrue(os.path.normpath is nt._path_normpath) + self.assertFalse(inspect.isfunction(os.path.normpath)) self.assertTrue(os.path.isdir is nt._path_isdir) self.assertFalse(inspect.isfunction(os.path.isdir)) self.assertTrue(os.path.isfile is nt._path_isfile) self.assertFalse(inspect.isfunction(os.path.isfile)) self.assertTrue(os.path.islink is nt._path_islink) self.assertFalse(inspect.isfunction(os.path.islink)) + self.assertTrue(os.path.isjunction is nt._path_isjunction) + self.assertFalse(inspect.isfunction(os.path.isjunction)) self.assertTrue(os.path.exists is nt._path_exists) self.assertFalse(inspect.isfunction(os.path.exists)) + self.assertTrue(os.path.lexists is nt._path_lexists) + self.assertFalse(inspect.isfunction(os.path.lexists)) @unittest.skipIf(os.name != 'nt', "Dev Drives only exist on Win32") def test_isdevdrive(self): diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 5fb4b815c95d07..2defe74892786d 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -4,25 +4,28 @@ import threading import types import unittest -from test.support import threading_helper, check_impl_detail, requires_specialization +from test.support import (threading_helper, check_impl_detail, + requires_specialization, requires_specialization_ft, + cpython_only, requires_jit_disabled) +from test.support.import_helper import import_module # Skip this module on other interpreters, it is cpython specific: if check_impl_detail(cpython=False): raise unittest.SkipTest('implementation detail specific to cpython') -import _testinternalcapi +_testinternalcapi = import_module("_testinternalcapi") -def disabling_optimizer(func): - def wrapper(*args, **kwargs): - old_opt = _testinternalcapi.get_optimizer() - _testinternalcapi.set_optimizer(None) - try: - return func(*args, **kwargs) - finally: - _testinternalcapi.set_optimizer(old_opt) +class TestBase(unittest.TestCase): + def assert_specialized(self, f, opname): + instructions = dis.get_instructions(f, adaptive=True) + opnames = {instruction.opname for instruction in instructions} + self.assertIn(opname, opnames) - return wrapper + def assert_no_opcode(self, f, opname): + instructions = dis.get_instructions(f, adaptive=True) + opnames = {instruction.opname for instruction in instructions} + self.assertNotIn(opname, opnames) class TestLoadSuperAttrCache(unittest.TestCase): @@ -42,7 +45,8 @@ def f(self): d = D() - self.assertEqual(d.f(), 1) # warmup + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD - 1): + self.assertEqual(d.f(), 1) # warmup calls.clear() self.assertEqual(d.f(), 1) # try to specialize self.assertEqual(calls, [(d, D)]) @@ -62,7 +66,7 @@ def f(o): return o.x o = C() - for i in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): assert f(o) == 1 Descriptor.__get__ = lambda self, instance, value: 2 @@ -89,13 +93,13 @@ def __set__(self, instance, value): def f(): return Class.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) Descriptor.__get__ = __get__ Descriptor.__set__ = __set__ - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_metaclass_descriptor_shadows_class_attribute(self): @@ -110,7 +114,7 @@ class Class(metaclass=Metaclass): def f(): return Class.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) def test_metaclass_set_descriptor_after_optimization(self): @@ -127,12 +131,12 @@ def attribute(self): def f(): return Class.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) Metaclass.attribute = attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_metaclass_del_descriptor_after_optimization(self): @@ -147,12 +151,12 @@ class Class(metaclass=Metaclass): def f(): return Class.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) del Metaclass.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_type_descriptor_shadows_attribute_method(self): @@ -162,7 +166,7 @@ class Class: def f(): return Class.mro - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertIsNone(f()) def test_type_descriptor_shadows_attribute_member(self): @@ -172,7 +176,7 @@ class Class: def f(): return Class.__base__ - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertIs(f(), object) def test_type_descriptor_shadows_attribute_getset(self): @@ -182,7 +186,7 @@ class Class: def f(): return Class.__name__ - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertEqual(f(), "Class") def test_metaclass_getattribute(self): @@ -196,7 +200,7 @@ class Class(metaclass=Metaclass): def f(): return Class.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) def test_metaclass_swap(self): @@ -216,12 +220,12 @@ class Class(metaclass=OldMetaclass): def f(): return Class.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) Class.__class__ = NewMetaclass - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_load_shadowing_slot_should_raise_type_error(self): @@ -238,7 +242,7 @@ def f(o): o = Sneaky() o.shadowed = 42 - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): with self.assertRaises(TypeError): f(o) @@ -255,7 +259,7 @@ def f(o): o = Sneaky() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): with self.assertRaises(TypeError): f(o) @@ -271,7 +275,7 @@ def f(o): o = Sneaky() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): with self.assertRaises(TypeError): f(o) @@ -287,7 +291,7 @@ def f(o): o = Sneaky() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): with self.assertRaises(TypeError): f(o) @@ -315,13 +319,13 @@ def attribute(): def f(): return instance.attribute() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) Descriptor.__get__ = __get__ Descriptor.__set__ = __set__ - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_metaclass_descriptor_added_after_optimization(self): @@ -344,13 +348,13 @@ def __set__(self, instance, value): def f(): return Class.attribute() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) Descriptor.__get__ = __get__ Descriptor.__set__ = __set__ - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_metaclass_descriptor_shadows_class_attribute(self): @@ -366,7 +370,7 @@ def attribute(): def f(): return Class.attribute() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) def test_metaclass_set_descriptor_after_optimization(self): @@ -384,12 +388,12 @@ def attribute(self): def f(): return Class.attribute() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) Metaclass.attribute = attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_metaclass_del_descriptor_after_optimization(self): @@ -405,12 +409,12 @@ def attribute(): def f(): return Class.attribute() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) del Metaclass.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_type_descriptor_shadows_attribute_method(self): @@ -421,7 +425,7 @@ def mro(): def f(): return Class.mro() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertEqual(f(), ["Spam", "eggs"]) def test_type_descriptor_shadows_attribute_member(self): @@ -432,7 +436,7 @@ def __base__(): def f(): return Class.__base__() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertNotEqual(f(), "Spam") def test_metaclass_getattribute(self): @@ -447,7 +451,7 @@ def attribute(): def f(): return Class.attribute() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) def test_metaclass_swap(self): @@ -467,22 +471,27 @@ class Class(metaclass=OldMetaclass): def f(): return Class.attribute() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) Class.__class__ = NewMetaclass - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) -class TestCallCache(unittest.TestCase): +class InitTakesArg: + def __init__(self, arg): + self.arg = arg + + +class TestCallCache(TestBase): def test_too_many_defaults_0(self): def f(): pass f.__defaults__ = (None,) - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): f() def test_too_many_defaults_1(self): @@ -490,7 +499,7 @@ def f(x): pass f.__defaults__ = (None, None) - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): f(None) f() @@ -499,28 +508,68 @@ def f(x, y): pass f.__defaults__ = (None, None, None) - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): f(None, None) f(None) f() + @requires_jit_disabled + @requires_specialization_ft + def test_assign_init_code(self): + class MyClass: + def __init__(self): + pass + + def instantiate(): + return MyClass() + + # Trigger specialization + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + instantiate() + self.assert_specialized(instantiate, "CALL_ALLOC_AND_ENTER_INIT") + + def count_args(self, *args): + self.num_args = len(args) + + # Set MyClass.__init__.__code__ to a code object that uses different + # args + MyClass.__init__.__code__ = count_args.__code__ + instantiate() + + @requires_jit_disabled + @requires_specialization_ft + def test_push_init_frame_fails(self): + def instantiate(): + return InitTakesArg() + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + with self.assertRaises(TypeError): + instantiate() + self.assert_specialized(instantiate, "CALL_ALLOC_AND_ENTER_INIT") + + with self.assertRaises(TypeError): + instantiate() + + +def make_deferred_ref_count_obj(): + """Create an object that uses deferred reference counting. + + Only objects that use deferred refence counting may be stored in inline + caches in free-threaded builds. This constructs a new class named Foo, + which uses deferred reference counting. + """ + return type("Foo", (object,), {}) + @threading_helper.requires_working_threading() -@requires_specialization -class TestRacesDoNotCrash(unittest.TestCase): +class TestRacesDoNotCrash(TestBase): # Careful with these. Bigger numbers have a higher chance of catching bugs, # but you can also burn through a *ton* of type/dict/function versions: ITEMS = 1000 LOOPS = 4 - WARMUPS = 2 WRITERS = 2 - def assert_specialized(self, f, opname): - instructions = dis.get_instructions(f, adaptive=True) - opnames = {instruction.opname for instruction in instructions} - self.assertIn(opname, opnames) - - @disabling_optimizer + @requires_jit_disabled def assert_races_do_not_crash( self, opname, get_items, read, write, *, check_items=False ): @@ -534,7 +583,7 @@ def assert_races_do_not_crash( else: read.__code__ = read.__code__.replace() # Specialize: - for _ in range(self.WARMUPS): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): read(items) if check_items: for item in items: @@ -553,6 +602,7 @@ def assert_races_do_not_crash( for writer in writers: writer.join() + @requires_specialization_ft def test_binary_subscr_getitem(self): def get_items(): class C: @@ -582,6 +632,7 @@ def write(items): opname = "BINARY_SUBSCR_GETITEM" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_binary_subscr_list_int(self): def get_items(): items = [] @@ -605,6 +656,7 @@ def write(items): opname = "BINARY_SUBSCR_LIST_INT" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization def test_for_iter_gen(self): def get_items(): def g(): @@ -636,6 +688,7 @@ def write(items): opname = "FOR_ITER_GEN" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization def test_for_iter_list(self): def get_items(): items = [] @@ -657,10 +710,11 @@ def write(items): opname = "FOR_ITER_LIST" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_load_attr_class(self): def get_items(): class C: - a = object() + a = make_deferred_ref_count_obj() items = [] for _ in range(self.ITEMS): @@ -681,11 +735,45 @@ def write(items): del item.a except AttributeError: pass - item.a = object() + item.a = make_deferred_ref_count_obj() opname = "LOAD_ATTR_CLASS" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft + def test_load_attr_class_with_metaclass_check(self): + def get_items(): + class Meta(type): + pass + + class C(metaclass=Meta): + a = make_deferred_ref_count_obj() + + items = [] + for _ in range(self.ITEMS): + item = C + items.append(item) + return items + + def read(items): + for item in items: + try: + item.a + except AttributeError: + pass + + def write(items): + for item in items: + try: + del item.a + except AttributeError: + pass + item.a = make_deferred_ref_count_obj() + + opname = "LOAD_ATTR_CLASS_WITH_METACLASS_CHECK" + self.assert_races_do_not_crash(opname, get_items, read, write) + + @requires_specialization_ft def test_load_attr_getattribute_overridden(self): def get_items(): class C: @@ -715,6 +803,7 @@ def write(items): opname = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_load_attr_instance_value(self): def get_items(): class C: @@ -738,6 +827,7 @@ def write(items): opname = "LOAD_ATTR_INSTANCE_VALUE" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_load_attr_method_lazy_dict(self): def get_items(): class C(Exception): @@ -767,6 +857,7 @@ def write(items): opname = "LOAD_ATTR_METHOD_LAZY_DICT" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_load_attr_method_no_dict(self): def get_items(): class C: @@ -797,6 +888,7 @@ def write(items): opname = "LOAD_ATTR_METHOD_NO_DICT" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_load_attr_method_with_values(self): def get_items(): class C: @@ -826,6 +918,7 @@ def write(items): opname = "LOAD_ATTR_METHOD_WITH_VALUES" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_load_attr_module(self): def get_items(): items = [] @@ -850,6 +943,7 @@ def write(items): opname = "LOAD_ATTR_MODULE" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_load_attr_property(self): def get_items(): class C: @@ -879,6 +973,34 @@ def write(items): opname = "LOAD_ATTR_PROPERTY" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft + def test_load_attr_slot(self): + def get_items(): + class C: + __slots__ = ["a", "b"] + + items = [] + for i in range(self.ITEMS): + item = C() + item.a = i + item.b = i + self.ITEMS + items.append(item) + return items + + def read(items): + for item in items: + item.a + item.b + + def write(items): + for item in items: + item.a = 100 + item.b = 200 + + opname = "LOAD_ATTR_SLOT" + self.assert_races_do_not_crash(opname, get_items, read, write) + + @requires_specialization_ft def test_load_attr_with_hint(self): def get_items(): class C: @@ -889,7 +1011,7 @@ class C: item = C() item.a = None # Resize into a combined unicode dict: - for i in range(29): + for i in range(_testinternalcapi.SHARED_KEYS_MAX_SIZE - 1): setattr(item, f"_{i}", None) items.append(item) return items @@ -905,6 +1027,7 @@ def write(items): opname = "LOAD_ATTR_WITH_HINT" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_load_global_module(self): def get_items(): items = [] @@ -926,6 +1049,7 @@ def write(items): opname, get_items, read, write, check_items=True ) + @requires_specialization def test_store_attr_instance_value(self): def get_items(): class C: @@ -948,6 +1072,7 @@ def write(items): opname = "STORE_ATTR_INSTANCE_VALUE" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization def test_store_attr_with_hint(self): def get_items(): class C: @@ -957,7 +1082,7 @@ class C: for _ in range(self.ITEMS): item = C() # Resize into a combined unicode dict: - for i in range(29): + for i in range(_testinternalcapi.SHARED_KEYS_MAX_SIZE - 1): setattr(item, f"_{i}", None) items.append(item) return items @@ -973,6 +1098,7 @@ def write(items): opname = "STORE_ATTR_WITH_HINT" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_store_subscr_list_int(self): def get_items(): items = [] @@ -996,6 +1122,7 @@ def write(items): opname = "STORE_SUBSCR_LIST_INT" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_unpack_sequence_list(self): def get_items(): items = [] @@ -1044,21 +1171,14 @@ def test_dict_materialization(self): c.a = 1 c.b = 2 c.__dict__ - self.assertIs( - _testinternalcapi.get_object_dict_values(c), - None - ) + self.assertEqual(c.__dict__, {"a":1, "b": 2}) def test_dict_dematerialization(self): c = C() c.a = 1 c.b = 2 c.__dict__ - self.assertIs( - _testinternalcapi.get_object_dict_values(c), - None - ) - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): c.a self.assertEqual( _testinternalcapi.get_object_dict_values(c), @@ -1070,12 +1190,8 @@ def test_dict_dematerialization_multiple_refs(self): c.a = 1 c.b = 2 d = c.__dict__ - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): c.a - self.assertIs( - _testinternalcapi.get_object_dict_values(c), - None - ) self.assertIs(c.__dict__, d) def test_dict_dematerialization_copy(self): @@ -1083,7 +1199,7 @@ def test_dict_dematerialization_copy(self): c.a = 1 c.b = 2 c2 = copy.copy(c) - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): c.a c2.a self.assertEqual( @@ -1095,7 +1211,7 @@ def test_dict_dematerialization_copy(self): (1, 2, '') ) c3 = copy.deepcopy(c) - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): c.a c3.a self.assertEqual( @@ -1109,7 +1225,7 @@ def test_dict_dematerialization_pickle(self): c.a = 1 c.b = 2 c2 = pickle.loads(pickle.dumps(c)) - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): c.a c2.a self.assertEqual( @@ -1127,7 +1243,7 @@ class D(dict): pass c.a = 1 c.b = 2 c.__dict__ = D(c.__dict__) - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): c.a self.assertIs( _testinternalcapi.get_object_dict_values(c), @@ -1138,6 +1254,551 @@ class D(dict): pass {'a':1, 'b':2} ) + def test_125868(self): + + def make_special_dict(): + """Create a dictionary an object with a this table: + index | key | value + ----- | --- | ----- + 0 | 'b' | 'value' + 1 | 'b' | NULL + """ + class A: + pass + a = A() + a.a = 1 + a.b = 2 + d = a.__dict__.copy() + del d['a'] + del d['b'] + d['b'] = "value" + return d + + class NoInlineAorB: + pass + for i in range(ord('c'), ord('z')): + setattr(NoInlineAorB(), chr(i), i) + + c = NoInlineAorB() + c.a = 0 + c.b = 1 + self.assertFalse(_testinternalcapi.has_inline_values(c)) + + def f(o, n): + for i in range(n): + o.b = i + # Prime f to store to dict slot 1 + f(c, _testinternalcapi.SPECIALIZATION_THRESHOLD) + + test_obj = NoInlineAorB() + test_obj.__dict__ = make_special_dict() + self.assertEqual(test_obj.b, "value") + + #This should set x.b = 0 + f(test_obj, 1) + self.assertEqual(test_obj.b, 0) + + +class TestSpecializer(TestBase): + + @cpython_only + @requires_specialization_ft + def test_binary_op(self): + def binary_op_add_int(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = 1, 2 + c = a + b + self.assertEqual(c, 3) + + binary_op_add_int() + self.assert_specialized(binary_op_add_int, "BINARY_OP_ADD_INT") + self.assert_no_opcode(binary_op_add_int, "BINARY_OP") + + def binary_op_add_unicode(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = "foo", "bar" + c = a + b + self.assertEqual(c, "foobar") + + binary_op_add_unicode() + self.assert_specialized(binary_op_add_unicode, "BINARY_OP_ADD_UNICODE") + self.assert_no_opcode(binary_op_add_unicode, "BINARY_OP") + + def binary_op_add_extend(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = 6, 3.0 + c = a + b + self.assertEqual(c, 9.0) + c = b + a + self.assertEqual(c, 9.0) + c = a - b + self.assertEqual(c, 3.0) + c = b - a + self.assertEqual(c, -3.0) + c = a * b + self.assertEqual(c, 18.0) + c = b * a + self.assertEqual(c, 18.0) + c = a / b + self.assertEqual(c, 2.0) + c = b / a + self.assertEqual(c, 0.5) + + binary_op_add_extend() + self.assert_specialized(binary_op_add_extend, "BINARY_OP_EXTEND") + self.assert_no_opcode(binary_op_add_extend, "BINARY_OP") + + def binary_op_zero_division(): + def compactlong_lhs(arg): + 42 / arg + def float_lhs(arg): + 42.0 / arg + + with self.assertRaises(ZeroDivisionError): + compactlong_lhs(0) + with self.assertRaises(ZeroDivisionError): + compactlong_lhs(0.0) + with self.assertRaises(ZeroDivisionError): + float_lhs(0.0) + with self.assertRaises(ZeroDivisionError): + float_lhs(0) + + self.assert_no_opcode(compactlong_lhs, "BINARY_OP_EXTEND") + self.assert_no_opcode(float_lhs, "BINARY_OP_EXTEND") + + binary_op_zero_division() + + def binary_op_nan(): + def compactlong_lhs(arg): + return ( + 42 + arg, + 42 - arg, + 42 * arg, + 42 / arg, + ) + def compactlong_rhs(arg): + return ( + arg + 42, + arg - 42, + arg * 2, + arg / 42, + ) + nan = float('nan') + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(compactlong_lhs(1.0), (43.0, 41.0, 42.0, 42.0)) + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): + self.assertTrue(all(filter(lambda x: x is nan, compactlong_lhs(nan)))) + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(compactlong_rhs(42.0), (84.0, 0.0, 84.0, 1.0)) + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): + self.assertTrue(all(filter(lambda x: x is nan, compactlong_rhs(nan)))) + + self.assert_no_opcode(compactlong_lhs, "BINARY_OP_EXTEND") + self.assert_no_opcode(compactlong_rhs, "BINARY_OP_EXTEND") + + binary_op_nan() + + def binary_op_bitwise_extend(): + for _ in range(100): + a, b = 2, 7 + x = a | b + self.assertEqual(x, 7) + y = a & b + self.assertEqual(y, 2) + z = a ^ b + self.assertEqual(z, 5) + a, b = 3, 9 + a |= b + self.assertEqual(a, 11) + a, b = 11, 9 + a &= b + self.assertEqual(a, 9) + a, b = 3, 9 + a ^= b + self.assertEqual(a, 10) + + binary_op_bitwise_extend() + self.assert_specialized(binary_op_bitwise_extend, "BINARY_OP_EXTEND") + self.assert_no_opcode(binary_op_bitwise_extend, "BINARY_OP") + + @cpython_only + @requires_specialization_ft + def test_load_super_attr(self): + """Ensure that LOAD_SUPER_ATTR is specialized as expected.""" + + class A: + def __init__(self): + meth = super().__init__ + super().__init__() + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + A() + + self.assert_specialized(A.__init__, "LOAD_SUPER_ATTR_ATTR") + self.assert_specialized(A.__init__, "LOAD_SUPER_ATTR_METHOD") + self.assert_no_opcode(A.__init__, "LOAD_SUPER_ATTR") + + # Temporarily replace super() with something else. + real_super = super + + def fake_super(): + def init(self): + pass + + return init + + # Force unspecialize + globals()['super'] = fake_super + try: + # Should be unspecialized after enough calls. + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): + A() + finally: + globals()['super'] = real_super + + # Ensure the specialized instructions are not present + self.assert_no_opcode(A.__init__, "LOAD_SUPER_ATTR_ATTR") + self.assert_no_opcode(A.__init__, "LOAD_SUPER_ATTR_METHOD") + + @cpython_only + @requires_specialization_ft + def test_contain_op(self): + def contains_op_dict(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = 1, {1: 2, 2: 5} + self.assertTrue(a in b) + self.assertFalse(3 in b) + + contains_op_dict() + self.assert_specialized(contains_op_dict, "CONTAINS_OP_DICT") + self.assert_no_opcode(contains_op_dict, "CONTAINS_OP") + + def contains_op_set(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = 1, {1, 2} + self.assertTrue(a in b) + self.assertFalse(3 in b) + + contains_op_set() + self.assert_specialized(contains_op_set, "CONTAINS_OP_SET") + self.assert_no_opcode(contains_op_set, "CONTAINS_OP") + + @cpython_only + @requires_specialization_ft + def test_send_with(self): + def run_async(coro): + while True: + try: + coro.send(None) + except StopIteration: + break + + class CM: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + pass + + async def send_with(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + async with CM(): + x = 1 + + run_async(send_with()) + # Note there are still unspecialized "SEND" opcodes in the + # cleanup paths of the 'with' statement. + self.assert_specialized(send_with, "SEND_GEN") + + @cpython_only + @requires_specialization_ft + def test_send_yield_from(self): + def g(): + yield None + + def send_yield_from(): + yield from g() + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + list(send_yield_from()) + + self.assert_specialized(send_yield_from, "SEND_GEN") + self.assert_no_opcode(send_yield_from, "SEND") + + @cpython_only + @requires_specialization_ft + def test_store_attr_slot(self): + class C: + __slots__ = ['x'] + + def set_slot(n): + c = C() + for i in range(n): + c.x = i + + set_slot(_testinternalcapi.SPECIALIZATION_THRESHOLD) + + self.assert_specialized(set_slot, "STORE_ATTR_SLOT") + self.assert_no_opcode(set_slot, "STORE_ATTR") + + # Adding a property for 'x' should unspecialize it. + C.x = property(lambda self: None, lambda self, x: None) + set_slot(_testinternalcapi.SPECIALIZATION_COOLDOWN) + self.assert_no_opcode(set_slot, "STORE_ATTR_SLOT") + + @cpython_only + @requires_specialization_ft + def test_store_attr_instance_value(self): + class C: + pass + + def set_value(n): + c = C() + for i in range(n): + c.x = i + + set_value(_testinternalcapi.SPECIALIZATION_THRESHOLD) + + self.assert_specialized(set_value, "STORE_ATTR_INSTANCE_VALUE") + self.assert_no_opcode(set_value, "STORE_ATTR") + + # Adding a property for 'x' should unspecialize it. + C.x = property(lambda self: None, lambda self, x: None) + set_value(_testinternalcapi.SPECIALIZATION_COOLDOWN) + self.assert_no_opcode(set_value, "STORE_ATTR_INSTANCE_VALUE") + + @cpython_only + @requires_specialization_ft + def test_store_attr_with_hint(self): + class C: + pass + + c = C() + for i in range(_testinternalcapi.SHARED_KEYS_MAX_SIZE - 1): + setattr(c, f"_{i}", None) + + def set_value(n): + for i in range(n): + c.x = i + + set_value(_testinternalcapi.SPECIALIZATION_THRESHOLD) + + self.assert_specialized(set_value, "STORE_ATTR_WITH_HINT") + self.assert_no_opcode(set_value, "STORE_ATTR") + + # Adding a property for 'x' should unspecialize it. + C.x = property(lambda self: None, lambda self, x: None) + set_value(_testinternalcapi.SPECIALIZATION_COOLDOWN) + self.assert_no_opcode(set_value, "STORE_ATTR_WITH_HINT") + + @cpython_only + @requires_specialization_ft + def test_to_bool(self): + def to_bool_bool(): + true_cnt, false_cnt = 0, 0 + elems = [e % 2 == 0 for e in range(_testinternalcapi.SPECIALIZATION_THRESHOLD)] + for e in elems: + if e: + true_cnt += 1 + else: + false_cnt += 1 + d, m = divmod(_testinternalcapi.SPECIALIZATION_THRESHOLD, 2) + self.assertEqual(true_cnt, d + m) + self.assertEqual(false_cnt, d) + + to_bool_bool() + self.assert_specialized(to_bool_bool, "TO_BOOL_BOOL") + self.assert_no_opcode(to_bool_bool, "TO_BOOL") + + def to_bool_int(): + count = 0 + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + if i: + count += 1 + else: + count -= 1 + self.assertEqual(count, _testinternalcapi.SPECIALIZATION_THRESHOLD - 2) + + to_bool_int() + self.assert_specialized(to_bool_int, "TO_BOOL_INT") + self.assert_no_opcode(to_bool_int, "TO_BOOL") + + def to_bool_list(): + count = 0 + elems = list(range(_testinternalcapi.SPECIALIZATION_THRESHOLD)) + while elems: + count += elems.pop() + self.assertEqual(elems, []) + self.assertEqual(count, sum(range(_testinternalcapi.SPECIALIZATION_THRESHOLD))) + + to_bool_list() + self.assert_specialized(to_bool_list, "TO_BOOL_LIST") + self.assert_no_opcode(to_bool_list, "TO_BOOL") + + def to_bool_none(): + count = 0 + elems = [None] * _testinternalcapi.SPECIALIZATION_THRESHOLD + for e in elems: + if not e: + count += 1 + self.assertEqual(count, _testinternalcapi.SPECIALIZATION_THRESHOLD) + + to_bool_none() + self.assert_specialized(to_bool_none, "TO_BOOL_NONE") + self.assert_no_opcode(to_bool_none, "TO_BOOL") + + def to_bool_str(): + count = 0 + elems = [""] + ["foo"] * (_testinternalcapi.SPECIALIZATION_THRESHOLD - 1) + for e in elems: + if e: + count += 1 + self.assertEqual(count, _testinternalcapi.SPECIALIZATION_THRESHOLD - 1) + + to_bool_str() + self.assert_specialized(to_bool_str, "TO_BOOL_STR") + self.assert_no_opcode(to_bool_str, "TO_BOOL") + + @cpython_only + @requires_specialization_ft + def test_unpack_sequence(self): + def unpack_sequence_two_tuple(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = 1, 2 + self.assertEqual(a, 1) + self.assertEqual(b, 2) + + unpack_sequence_two_tuple() + self.assert_specialized(unpack_sequence_two_tuple, + "UNPACK_SEQUENCE_TWO_TUPLE") + self.assert_no_opcode(unpack_sequence_two_tuple, "UNPACK_SEQUENCE") + + def unpack_sequence_tuple(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, = 1, + self.assertEqual(a, 1) + + unpack_sequence_tuple() + self.assert_specialized(unpack_sequence_tuple, "UNPACK_SEQUENCE_TUPLE") + self.assert_no_opcode(unpack_sequence_tuple, "UNPACK_SEQUENCE") + + def unpack_sequence_list(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = [1, 2] + self.assertEqual(a, 1) + self.assertEqual(b, 2) + + unpack_sequence_list() + self.assert_specialized(unpack_sequence_list, "UNPACK_SEQUENCE_LIST") + self.assert_no_opcode(unpack_sequence_list, "UNPACK_SEQUENCE") + + @cpython_only + @requires_specialization_ft + def test_binary_subscr(self): + def binary_subscr_list_int(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = [1, 2, 3] + for idx, expected in enumerate(a): + self.assertEqual(a[idx], expected) + + binary_subscr_list_int() + self.assert_specialized(binary_subscr_list_int, + "BINARY_SUBSCR_LIST_INT") + self.assert_no_opcode(binary_subscr_list_int, "BINARY_SUBSCR") + + def binary_subscr_tuple_int(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = (1, 2, 3) + for idx, expected in enumerate(a): + self.assertEqual(a[idx], expected) + + binary_subscr_tuple_int() + self.assert_specialized(binary_subscr_tuple_int, + "BINARY_SUBSCR_TUPLE_INT") + self.assert_no_opcode(binary_subscr_tuple_int, "BINARY_SUBSCR") + + def binary_subscr_dict(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = {1: 2, 2: 3} + self.assertEqual(a[1], 2) + self.assertEqual(a[2], 3) + + binary_subscr_dict() + self.assert_specialized(binary_subscr_dict, "BINARY_SUBSCR_DICT") + self.assert_no_opcode(binary_subscr_dict, "BINARY_SUBSCR") + + def binary_subscr_str_int(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = "foobar" + for idx, expected in enumerate(a): + self.assertEqual(a[idx], expected) + + binary_subscr_str_int() + self.assert_specialized(binary_subscr_str_int, "BINARY_SUBSCR_STR_INT") + self.assert_no_opcode(binary_subscr_str_int, "BINARY_SUBSCR") + + def binary_subscr_getitems(): + class C: + def __init__(self, val): + self.val = val + def __getitem__(self, item): + return self.val + + items = [C(i) for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD)] + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(items[i][i], i) + + binary_subscr_getitems() + self.assert_specialized(binary_subscr_getitems, "BINARY_SUBSCR_GETITEM") + self.assert_no_opcode(binary_subscr_getitems, "BINARY_SUBSCR") + + @cpython_only + @requires_specialization_ft + def test_compare_op(self): + def compare_op_int(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = 1, 2 + c = a == b + self.assertFalse(c) + + compare_op_int() + self.assert_specialized(compare_op_int, "COMPARE_OP_INT") + self.assert_no_opcode(compare_op_int, "COMPARE_OP") + + def compare_op_float(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = 1.0, 2.0 + c = a == b + self.assertFalse(c) + + compare_op_float() + self.assert_specialized(compare_op_float, "COMPARE_OP_FLOAT") + self.assert_no_opcode(compare_op_float, "COMPARE_OP") + + def compare_op_str(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = "spam", "ham" + c = a == b + self.assertFalse(c) + + compare_op_str() + self.assert_specialized(compare_op_str, "COMPARE_OP_STR") + self.assert_no_opcode(compare_op_str, "COMPARE_OP") + + @cpython_only + @requires_specialization_ft + def test_load_const(self): + def load_const(): + def unused(): pass + # Currently, the empty tuple is immortal, and the otherwise + # unused nested function's code object is mortal. This test will + # have to use different values if either of that changes. + return () + + load_const() + self.assert_specialized(load_const, "LOAD_CONST_IMMORTAL") + self.assert_specialized(load_const, "LOAD_CONST_MORTAL") + self.assert_no_opcode(load_const, "LOAD_CONST") if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_opcodes.py b/Lib/test/test_opcodes.py index 72488b2bb6b4ff..f7cc8331b8d844 100644 --- a/Lib/test/test_opcodes.py +++ b/Lib/test/test_opcodes.py @@ -39,16 +39,19 @@ class C: pass def test_use_existing_annotations(self): ns = {'__annotations__': {1: 2}} exec('x: int', ns) - self.assertEqual(ns['__annotations__'], {'x': int, 1: 2}) + self.assertEqual(ns['__annotations__'], {1: 2}) def test_do_not_recreate_annotations(self): # Don't rely on the existence of the '__annotations__' global. with support.swap_item(globals(), '__annotations__', {}): - del globals()['__annotations__'] + globals().pop('__annotations__', None) class C: - del __annotations__ - with self.assertRaises(NameError): - x: int + try: + del __annotations__ + except NameError: + pass + x: int + self.assertEqual(C.__annotations__, {"x": int}) def test_raise_class_exceptions(self): diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index 1db738d228b1b9..82578a0ef1e6f2 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -1,6 +1,9 @@ import unittest +import inspect import pickle import sys +from decimal import Decimal +from fractions import Fraction from test import support from test.support import import_helper @@ -344,6 +347,26 @@ def test_is_not(self): self.assertFalse(operator.is_not(a, b)) self.assertTrue(operator.is_not(a,c)) + def test_is_none(self): + operator = self.module + a = 'xyzpdq' + b = '' + c = None + self.assertRaises(TypeError, operator.is_none) + self.assertFalse(operator.is_none(a)) + self.assertFalse(operator.is_none(b)) + self.assertTrue(operator.is_none(c)) + + def test_is_not_none(self): + operator = self.module + a = 'xyzpdq' + b = '' + c = None + self.assertRaises(TypeError, operator.is_not_none) + self.assertTrue(operator.is_not_none(a)) + self.assertTrue(operator.is_not_none(b)) + self.assertFalse(operator.is_not_none(c)) + def test_attrgetter(self): operator = self.module class A: @@ -459,6 +482,8 @@ def bar(self, f=42): return f def baz(*args, **kwds): return kwds['name'], kwds['self'] + def return_arguments(self, *args, **kwds): + return args, kwds a = A() f = operator.methodcaller('foo') self.assertRaises(IndexError, f, a) @@ -475,6 +500,17 @@ def baz(*args, **kwds): f = operator.methodcaller('baz', name='spam', self='eggs') self.assertEqual(f(a), ('spam', 'eggs')) + many_positional_arguments = tuple(range(10)) + many_kw_arguments = dict(zip('abcdefghij', range(10))) + f = operator.methodcaller('return_arguments', *many_positional_arguments) + self.assertEqual(f(a), (many_positional_arguments, {})) + + f = operator.methodcaller('return_arguments', **many_kw_arguments) + self.assertEqual(f(a), ((), many_kw_arguments)) + + f = operator.methodcaller('return_arguments', *many_positional_arguments, **many_kw_arguments) + self.assertEqual(f(a), (many_positional_arguments, many_kw_arguments)) + def test_inplace(self): operator = self.module class C(object): @@ -508,6 +544,44 @@ def __getitem__(self, other): return 5 # so that C is a sequence self.assertEqual(operator.ixor (c, 5), "ixor") self.assertEqual(operator.iconcat (c, c), "iadd") + def test_iconcat_without_getitem(self): + operator = self.module + + msg = "'int' object can't be concatenated" + with self.assertRaisesRegex(TypeError, msg): + operator.iconcat(1, 0.5) + + def test_index(self): + operator = self.module + class X: + def __index__(self): + return 1 + + self.assertEqual(operator.index(X()), 1) + self.assertEqual(operator.index(0), 0) + self.assertEqual(operator.index(1), 1) + self.assertEqual(operator.index(2), 2) + with self.assertRaises((AttributeError, TypeError)): + operator.index(1.5) + with self.assertRaises((AttributeError, TypeError)): + operator.index(Fraction(3, 7)) + with self.assertRaises((AttributeError, TypeError)): + operator.index(Decimal(1)) + with self.assertRaises((AttributeError, TypeError)): + operator.index(None) + + def test_not_(self): + operator = self.module + class C: + def __bool__(self): + raise SyntaxError + self.assertRaises(TypeError, operator.not_) + self.assertRaises(SyntaxError, operator.not_, C()) + self.assertFalse(operator.not_(5)) + self.assertFalse(operator.not_([0])) + self.assertTrue(operator.not_(0)) + self.assertTrue(operator.not_([])) + def test_length_hint(self): operator = self.module class X(object): @@ -533,6 +607,13 @@ def __length_hint__(self): with self.assertRaises(LookupError): operator.length_hint(X(LookupError)) + class Y: pass + + msg = "'str' object cannot be interpreted as an integer" + with self.assertRaisesRegex(TypeError, msg): + operator.length_hint(X(2), "abc") + self.assertEqual(operator.length_hint(Y(), 10), 10) + def test_call(self): operator = self.module @@ -555,6 +636,28 @@ def test_dunder_is_original(self): if dunder: self.assertIs(dunder, orig) + def test_attrgetter_signature(self): + operator = self.module + sig = inspect.signature(operator.attrgetter) + self.assertEqual(str(sig), '(attr, /, *attrs)') + sig = inspect.signature(operator.attrgetter('x', 'z', 'y')) + self.assertEqual(str(sig), '(obj, /)') + + def test_itemgetter_signature(self): + operator = self.module + sig = inspect.signature(operator.itemgetter) + self.assertEqual(str(sig), '(item, /, *items)') + sig = inspect.signature(operator.itemgetter(2, 3, 5)) + self.assertEqual(str(sig), '(obj, /)') + + def test_methodcaller_signature(self): + operator = self.module + sig = inspect.signature(operator.methodcaller) + self.assertEqual(str(sig), '(name, /, *args, **kwargs)') + sig = inspect.signature(operator.methodcaller('foo', 2, y=3)) + self.assertEqual(str(sig), '(obj, /)') + + class PyOperatorTestCase(OperatorTestCase, unittest.TestCase): module = py_operator diff --git a/Lib/test/test_optimizer.py b/Lib/test/test_optimizer.py index 899a4507317334..fac4d1a4ab44c4 100644 --- a/Lib/test/test_optimizer.py +++ b/Lib/test/test_optimizer.py @@ -80,6 +80,8 @@ def func(x=0): class TestOptimizerSymbols(unittest.TestCase): + @unittest.skipUnless(hasattr(_testinternalcapi, "uop_symbols_test"), + "requires _testinternalcapi.uop_symbols_test") def test_optimizer_symbols(self): _testinternalcapi.uop_symbols_test() diff --git a/Lib/test/test_optparse.py b/Lib/test/test_optparse.py index 28b274462388ed..8655a0537a5e56 100644 --- a/Lib/test/test_optparse.py +++ b/Lib/test/test_optparse.py @@ -15,7 +15,7 @@ from io import StringIO from test import support from test.support import os_helper - +from test.support.i18n_helper import TestTranslationsBase, update_translation_snapshots import optparse from optparse import make_option, Option, \ @@ -1656,5 +1656,14 @@ def test__all__(self): support.check__all__(self, optparse, not_exported=not_exported) +class TestTranslations(TestTranslationsBase): + def test_translations(self): + self.assertMsgidsEqual(optparse) + + if __name__ == '__main__': + # To regenerate translation snapshots + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + update_translation_snapshots(optparse) + sys.exit(0) unittest.main() diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index 4571b23dfe7c1a..9f131a9110dccb 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -2,7 +2,9 @@ import contextlib import copy import gc +import operator import pickle +import re from random import randrange, shuffle import struct import sys @@ -739,11 +741,44 @@ def test_ordered_dict_items_result_gc(self): # when it's mutated and returned from __next__: self.assertTrue(gc.is_tracked(next(it))) + +class _TriggerSideEffectOnEqual: + count = 0 # number of calls to __eq__ + trigger = 1 # count value when to trigger side effect + + def __eq__(self, other): + if self.__class__.count == self.__class__.trigger: + self.side_effect() + self.__class__.count += 1 + return True + + def __hash__(self): + # all instances represent the same key + return -1 + + def side_effect(self): + raise NotImplementedError + class PurePythonOrderedDictTests(OrderedDictTests, unittest.TestCase): module = py_coll OrderedDict = py_coll.OrderedDict + def test_issue119004_attribute_error(self): + class Key(_TriggerSideEffectOnEqual): + def side_effect(self): + del dict1[TODEL] + + TODEL = Key() + dict1 = self.OrderedDict(dict.fromkeys((0, TODEL, 4.2))) + dict2 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + # This causes an AttributeError due to the linked list being changed + msg = re.escape("'NoneType' object has no attribute 'key'") + self.assertRaisesRegex(AttributeError, msg, operator.eq, dict1, dict2) + self.assertEqual(Key.count, 2) + self.assertDictEqual(dict1, dict.fromkeys((0, 4.2))) + self.assertDictEqual(dict2, dict.fromkeys((0, Key(), 4.2))) + class CPythonBuiltinDictTests(unittest.TestCase): """Builtin dict preserves insertion order. @@ -764,8 +799,85 @@ class CPythonBuiltinDictTests(unittest.TestCase): del method +class CPythonOrderedDictSideEffects: + + def check_runtime_error_issue119004(self, dict1, dict2): + msg = re.escape("OrderedDict mutated during iteration") + self.assertRaisesRegex(RuntimeError, msg, operator.eq, dict1, dict2) + + def test_issue119004_change_size_by_clear(self): + class Key(_TriggerSideEffectOnEqual): + def side_effect(self): + dict1.clear() + + dict1 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + dict2 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + self.check_runtime_error_issue119004(dict1, dict2) + self.assertEqual(Key.count, 2) + self.assertDictEqual(dict1, {}) + self.assertDictEqual(dict2, dict.fromkeys((0, Key(), 4.2))) + + def test_issue119004_change_size_by_delete_key(self): + class Key(_TriggerSideEffectOnEqual): + def side_effect(self): + del dict1[TODEL] + + TODEL = Key() + dict1 = self.OrderedDict(dict.fromkeys((0, TODEL, 4.2))) + dict2 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + self.check_runtime_error_issue119004(dict1, dict2) + self.assertEqual(Key.count, 2) + self.assertDictEqual(dict1, dict.fromkeys((0, 4.2))) + self.assertDictEqual(dict2, dict.fromkeys((0, Key(), 4.2))) + + def test_issue119004_change_linked_list_by_clear(self): + class Key(_TriggerSideEffectOnEqual): + def side_effect(self): + dict1.clear() + dict1['a'] = dict1['b'] = 'c' + + dict1 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + dict2 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + self.check_runtime_error_issue119004(dict1, dict2) + self.assertEqual(Key.count, 2) + self.assertDictEqual(dict1, dict.fromkeys(('a', 'b'), 'c')) + self.assertDictEqual(dict2, dict.fromkeys((0, Key(), 4.2))) + + def test_issue119004_change_linked_list_by_delete_key(self): + class Key(_TriggerSideEffectOnEqual): + def side_effect(self): + del dict1[TODEL] + dict1['a'] = 'c' + + TODEL = Key() + dict1 = self.OrderedDict(dict.fromkeys((0, TODEL, 4.2))) + dict2 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + self.check_runtime_error_issue119004(dict1, dict2) + self.assertEqual(Key.count, 2) + self.assertDictEqual(dict1, {0: None, 'a': 'c', 4.2: None}) + self.assertDictEqual(dict2, dict.fromkeys((0, Key(), 4.2))) + + def test_issue119004_change_size_by_delete_key_in_dict_eq(self): + class Key(_TriggerSideEffectOnEqual): + trigger = 0 + def side_effect(self): + del dict1[TODEL] + + TODEL = Key() + dict1 = self.OrderedDict(dict.fromkeys((0, TODEL, 4.2))) + dict2 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + self.assertEqual(Key.count, 0) + # the side effect is in dict.__eq__ and modifies the length + self.assertNotEqual(dict1, dict2) + self.assertEqual(Key.count, 2) + self.assertDictEqual(dict1, dict.fromkeys((0, 4.2))) + self.assertDictEqual(dict2, dict.fromkeys((0, Key(), 4.2))) + + @unittest.skipUnless(c_coll, 'requires the C version of the collections module') -class CPythonOrderedDictTests(OrderedDictTests, unittest.TestCase): +class CPythonOrderedDictTests(OrderedDictTests, + CPythonOrderedDictSideEffects, + unittest.TestCase): module = c_coll OrderedDict = c_coll.OrderedDict diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 3b2d5fccc30f3c..6e40cb4f58bfee 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -13,6 +13,7 @@ import locale import os import pickle +import platform import select import selectors import shutil @@ -34,7 +35,7 @@ from test.support import import_helper from test.support import os_helper from test.support import socket_helper -from test.support import set_recursion_limit +from test.support import infinite_recursion from test.support import warnings_helper from platform import win32_is_iot @@ -56,8 +57,10 @@ except (ImportError, AttributeError): all_users = [] try: + import _testcapi from _testcapi import INT_MAX, PY_SSIZE_T_MAX except ImportError: + _testcapi = None INT_MAX = PY_SSIZE_T_MAX = sys.maxsize try: @@ -102,7 +105,7 @@ def create_file(filename, content=b'content'): def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class MiscTests(unittest.TestCase): @@ -185,9 +188,6 @@ def test_access(self): os.close(f) self.assertTrue(os.access(os_helper.TESTFN, os.W_OK)) - @unittest.skipIf( - support.is_emscripten, "Test is unstable under Emscripten." - ) @unittest.skipIf( support.is_wasi, "WASI does not support dup." ) @@ -230,6 +230,94 @@ def test_read(self): self.assertEqual(type(s), bytes) self.assertEqual(s, b"spam") + def test_readinto(self): + with open(os_helper.TESTFN, "w+b") as fobj: + fobj.write(b"spam") + fobj.flush() + fd = fobj.fileno() + os.lseek(fd, 0, 0) + # Oversized so readinto without hitting end. + buffer = bytearray(7) + s = os.readinto(fd, buffer) + self.assertEqual(type(s), int) + self.assertEqual(s, 4) + # Should overwrite the first 4 bytes of the buffer. + self.assertEqual(buffer[:4], b"spam") + + # Readinto at EOF should return 0 and not touch buffer. + buffer[:] = b"notspam" + s = os.readinto(fd, buffer) + self.assertEqual(type(s), int) + self.assertEqual(s, 0) + self.assertEqual(bytes(buffer), b"notspam") + s = os.readinto(fd, buffer) + self.assertEqual(s, 0) + self.assertEqual(bytes(buffer), b"notspam") + + # Readinto a 0 length bytearray when at EOF should return 0 + self.assertEqual(os.readinto(fd, bytearray()), 0) + + # Readinto a 0 length bytearray with data available should return 0. + os.lseek(fd, 0, 0) + self.assertEqual(os.readinto(fd, bytearray()), 0) + + @unittest.skipUnless(hasattr(os, 'get_blocking'), + 'needs os.get_blocking() and os.set_blocking()') + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + @unittest.skipIf(support.is_emscripten, "set_blocking does not work correctly") + def test_readinto_non_blocking(self): + # Verify behavior of a readinto which would block on a non-blocking fd. + r, w = os.pipe() + try: + os.set_blocking(r, False) + with self.assertRaises(BlockingIOError): + os.readinto(r, bytearray(5)) + + # Pass some data through + os.write(w, b"spam") + self.assertEqual(os.readinto(r, bytearray(4)), 4) + + # Still don't block or return 0. + with self.assertRaises(BlockingIOError): + os.readinto(r, bytearray(5)) + + # At EOF should return size 0 + os.close(w) + w = None + self.assertEqual(os.readinto(r, bytearray(5)), 0) + self.assertEqual(os.readinto(r, bytearray(5)), 0) # Still EOF + + finally: + os.close(r) + if w is not None: + os.close(w) + + def test_readinto_badarg(self): + with open(os_helper.TESTFN, "w+b") as fobj: + fobj.write(b"spam") + fobj.flush() + fd = fobj.fileno() + os.lseek(fd, 0, 0) + + for bad_arg in ("test", bytes(), 14): + with self.subTest(f"bad buffer {type(bad_arg)}"): + with self.assertRaises(TypeError): + os.readinto(fd, bad_arg) + + with self.subTest("doesn't work on file objects"): + with self.assertRaises(TypeError): + os.readinto(fobj, bytearray(5)) + + # takes two args + with self.assertRaises(TypeError): + os.readinto(fd) + + # No data should have been read with the bad arguments. + buffer = bytearray(4) + s = os.readinto(fd, buffer) + self.assertEqual(s, 4) + self.assertEqual(buffer, b"spam") + @support.cpython_only # Skip the test on 32-bit platforms: the number of bytes must fit in a # Py_ssize_t type @@ -249,6 +337,29 @@ def test_large_read(self, size): # operating system is free to return less bytes than requested. self.assertEqual(data, b'test') + + @support.cpython_only + # Skip the test on 32-bit platforms: the number of bytes must fit in a + # Py_ssize_t type + @unittest.skipUnless(INT_MAX < PY_SSIZE_T_MAX, + "needs INT_MAX < PY_SSIZE_T_MAX") + @support.bigmemtest(size=INT_MAX + 10, memuse=1, dry_run=False) + def test_large_readinto(self, size): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + create_file(os_helper.TESTFN, b'test') + + # Issue #21932: For readinto the buffer contains the length rather than + # a length being passed explicitly to read, should still get capped to a + # valid size / not raise an OverflowError for sizes larger than INT_MAX. + buffer = bytearray(INT_MAX + 10) + with open(os_helper.TESTFN, "rb") as fp: + length = os.readinto(fp.fileno(), buffer) + + # The test does not try to read more than 2 GiB at once because the + # operating system is free to return less bytes than requested. + self.assertEqual(length, 4) + self.assertEqual(buffer[:4], b'test') + def test_write(self): # os.write() accepts bytes- and buffer-like objects but not strings fd = os.open(os_helper.TESTFN, os.O_CREAT | os.O_WRONLY) @@ -802,14 +913,28 @@ def _test_utime(self, set_time, filename=None): set_time(filename, (atime_ns, mtime_ns)) st = os.stat(filename) - if support_subsecond: - self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-6) - self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-6) + if support.is_emscripten: + # Emscripten timestamps are roundtripped through a 53 bit integer of + # nanoseconds. If we want to represent ~50 years which is an 11 + # digits number of seconds: + # 2*log10(60) + log10(24) + log10(365) + log10(60) + log10(50) + # is about 11. Because 53 * log10(2) is about 16, we only have 5 + # digits worth of sub-second precision. + # Some day it would be good to fix this upstream. + delta=1e-5 + self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-5) + self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-5) + self.assertAlmostEqual(st.st_atime_ns, atime_ns, delta=1e9 * 1e-5) + self.assertAlmostEqual(st.st_mtime_ns, mtime_ns, delta=1e9 * 1e-5) else: - self.assertEqual(st.st_atime, atime_ns * 1e-9) - self.assertEqual(st.st_mtime, mtime_ns * 1e-9) - self.assertEqual(st.st_atime_ns, atime_ns) - self.assertEqual(st.st_mtime_ns, mtime_ns) + if support_subsecond: + self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-6) + self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-6) + else: + self.assertEqual(st.st_atime, atime_ns * 1e-9) + self.assertEqual(st.st_mtime, mtime_ns * 1e-9) + self.assertEqual(st.st_atime_ns, atime_ns) + self.assertEqual(st.st_mtime_ns, mtime_ns) def test_utime(self): def set_time(filename, ns): @@ -825,7 +950,7 @@ def ns_to_sec(ns): return (ns * 1e-9) + 0.5e-9 def test_utime_by_indexed(self): - # pass times as floating point seconds as the second indexed parameter + # pass times as floating-point seconds as the second indexed parameter def set_time(filename, ns): atime_ns, mtime_ns = ns atime = self.ns_to_sec(atime_ns) @@ -1295,9 +1420,56 @@ def test_ror_operator(self): self._test_underlying_process_env('_A_', '') self._test_underlying_process_env(overridden_key, original_value) + def test_reload_environ(self): + # Test os.reload_environ() + has_environb = hasattr(os, 'environb') + + # Test with putenv() which doesn't update os.environ + os.environ['test_env'] = 'python_value' + os.putenv("test_env", "new_value") + self.assertEqual(os.environ['test_env'], 'python_value') + if has_environb: + self.assertEqual(os.environb[b'test_env'], b'python_value') + + os.reload_environ() + self.assertEqual(os.environ['test_env'], 'new_value') + if has_environb: + self.assertEqual(os.environb[b'test_env'], b'new_value') + + # Test with unsetenv() which doesn't update os.environ + os.unsetenv('test_env') + self.assertEqual(os.environ['test_env'], 'new_value') + if has_environb: + self.assertEqual(os.environb[b'test_env'], b'new_value') + + os.reload_environ() + self.assertNotIn('test_env', os.environ) + if has_environb: + self.assertNotIn(b'test_env', os.environb) + + if has_environb: + # test reload_environ() on os.environb with putenv() + os.environb[b'test_env'] = b'python_value2' + os.putenv("test_env", "new_value2") + self.assertEqual(os.environb[b'test_env'], b'python_value2') + self.assertEqual(os.environ['test_env'], 'python_value2') + + os.reload_environ() + self.assertEqual(os.environb[b'test_env'], b'new_value2') + self.assertEqual(os.environ['test_env'], 'new_value2') + + # test reload_environ() on os.environb with unsetenv() + os.unsetenv('test_env') + self.assertEqual(os.environb[b'test_env'], b'new_value2') + self.assertEqual(os.environ['test_env'], 'new_value2') + + os.reload_environ() + self.assertNotIn(b'test_env', os.environb) + self.assertNotIn('test_env', os.environ) class WalkTests(unittest.TestCase): """Tests for os.walk().""" + is_fwalk = False # Wrapper to hide minor differences between os.walk and os.fwalk # to tests both functions with the same code base @@ -1332,14 +1504,14 @@ def setUp(self): self.sub11_path = join(self.sub1_path, "SUB11") sub2_path = join(self.walk_path, "SUB2") sub21_path = join(sub2_path, "SUB21") - tmp1_path = join(self.walk_path, "tmp1") + self.tmp1_path = join(self.walk_path, "tmp1") tmp2_path = join(self.sub1_path, "tmp2") tmp3_path = join(sub2_path, "tmp3") tmp5_path = join(sub21_path, "tmp3") self.link_path = join(sub2_path, "link") t2_path = join(os_helper.TESTFN, "TEST2") tmp4_path = join(os_helper.TESTFN, "TEST2", "tmp4") - broken_link_path = join(sub2_path, "broken_link") + self.broken_link_path = join(sub2_path, "broken_link") broken_link2_path = join(sub2_path, "broken_link2") broken_link3_path = join(sub2_path, "broken_link3") @@ -1349,13 +1521,13 @@ def setUp(self): os.makedirs(sub21_path) os.makedirs(t2_path) - for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path: + for path in self.tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path: with open(path, "x", encoding='utf-8') as f: f.write("I'm " + path + " and proud of it. Blame test_os.\n") if os_helper.can_symlink(): os.symlink(os.path.abspath(t2_path), self.link_path) - os.symlink('broken', broken_link_path, True) + os.symlink('broken', self.broken_link_path, True) os.symlink(join('tmp3', 'broken'), broken_link2_path, True) os.symlink(join('SUB21', 'tmp5'), broken_link3_path, True) self.sub2_tree = (sub2_path, ["SUB21", "link"], @@ -1364,9 +1536,7 @@ def setUp(self): else: self.sub2_tree = (sub2_path, ["SUB21"], ["tmp3"]) - if not support.is_emscripten: - # Emscripten fails with inaccessible directory - os.chmod(sub21_path, 0) + os.chmod(sub21_path, 0) try: os.listdir(sub21_path) except PermissionError: @@ -1451,6 +1621,11 @@ def test_walk_symlink(self): else: self.fail("Didn't follow symlink with followlinks=True") + walk_it = self.walk(self.broken_link_path, follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(FileNotFoundError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + def test_walk_bad_dir(self): # Walk top-down. errors = [] @@ -1472,6 +1647,73 @@ def test_walk_bad_dir(self): finally: os.rename(path1new, path1) + def test_walk_bad_dir2(self): + walk_it = self.walk('nonexisting') + if self.is_fwalk: + self.assertRaises(FileNotFoundError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk('nonexisting', follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(FileNotFoundError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk(self.tmp1_path) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk(self.tmp1_path, follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(NotADirectoryError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_walk_named_pipe(self): + path = os_helper.TESTFN + '-pipe' + os.mkfifo(path) + self.addCleanup(os.unlink, path) + + walk_it = self.walk(path) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk(path, follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(NotADirectoryError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_walk_named_pipe2(self): + path = os_helper.TESTFN + '-dir' + os.mkdir(path) + self.addCleanup(shutil.rmtree, path) + os.mkfifo(os.path.join(path, 'mypipe')) + + errors = [] + walk_it = self.walk(path, onerror=errors.append) + next(walk_it) + self.assertRaises(StopIteration, next, walk_it) + self.assertEqual(errors, []) + + errors = [] + walk_it = self.walk(path, onerror=errors.append) + root, dirs, files = next(walk_it) + self.assertEqual(root, path) + self.assertEqual(dirs, []) + self.assertEqual(files, ['mypipe']) + dirs.extend(files) + files.clear() + if self.is_fwalk: + self.assertRaises(NotADirectoryError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + if self.is_fwalk: + self.assertEqual(errors, []) + else: + self.assertEqual(len(errors), 1, errors) + self.assertIsInstance(errors[0], NotADirectoryError) + def test_walk_many_open_files(self): depth = 30 base = os.path.join(os_helper.TESTFN, 'deep') @@ -1496,7 +1738,7 @@ def test_walk_many_open_files(self): def test_walk_above_recursion_limit(self): depth = 50 os.makedirs(os.path.join(self.walk_path, *(['d'] * depth))) - with set_recursion_limit(depth - 5): + with infinite_recursion(depth - 5): all = list(self.walk(self.walk_path)) sub2_path = self.sub2_tree[0] @@ -1537,6 +1779,7 @@ def test_walk_above_recursion_limit(self): @unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()") class FwalkTests(WalkTests): """Tests for os.fwalk().""" + is_fwalk = True def walk(self, top, **kwargs): for root, dirs, files, root_fd in self.fwalk(top, **kwargs): @@ -1589,9 +1832,6 @@ def test_yields_correct_dir_fd(self): # check that listdir() returns consistent information self.assertEqual(set(os.listdir(rootfd)), set(dirs) | set(files)) - @unittest.skipIf( - support.is_emscripten, "Cannot dup stdout on Emscripten" - ) @unittest.skipIf( support.is_android, "dup return value is unpredictable on Android" ) @@ -1608,10 +1848,26 @@ def test_fd_leak(self): self.addCleanup(os.close, newfd) self.assertEqual(newfd, minfd) + @unittest.skipIf( + support.is_android, "dup return value is unpredictable on Android" + ) + def test_fd_finalization(self): + # Check that close()ing the fwalk() generator closes FDs + def getfd(): + fd = os.dup(1) + os.close(fd) + return fd + for topdown in (False, True): + old_fd = getfd() + it = self.fwalk(os_helper.TESTFN, topdown=topdown) + self.assertEqual(getfd(), old_fd) + next(it) + self.assertGreater(getfd(), old_fd) + it.close() + self.assertEqual(getfd(), old_fd) + # fwalk() keeps file descriptors open test_walk_many_open_files = None - # fwalk() still uses recursion - test_walk_above_recursion_limit = None class BytesWalkTests(WalkTests): @@ -1660,8 +1916,8 @@ def test_makedir(self): os.makedirs(path) @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "Emscripten's/WASI's umask is a stub." + support.is_wasi, + "WASI's umask is a stub." ) def test_mode(self): with os_helper.temp_umask(0o002): @@ -1676,8 +1932,8 @@ def test_mode(self): self.assertEqual(os.stat(parent).st_mode & 0o777, 0o775) @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "Emscripten's/WASI's umask is a stub." + support.is_wasi, + "WASI's umask is a stub." ) def test_exist_ok_existing_directory(self): path = os.path.join(os_helper.TESTFN, 'dir1') @@ -1694,8 +1950,8 @@ def test_exist_ok_existing_directory(self): os.makedirs(os.path.abspath('/'), exist_ok=True) @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "Emscripten's/WASI's umask is a stub." + support.is_wasi, + "WASI's umask is a stub." ) def test_exist_ok_s_isgid_directory(self): path = os.path.join(os_helper.TESTFN, 'dir1') @@ -1734,6 +1990,19 @@ def test_exist_ok_existing_regular_file(self): self.assertRaises(OSError, os.makedirs, path, exist_ok=True) os.remove(path) + @unittest.skipUnless(os.name == 'nt', "requires Windows") + def test_win32_mkdir_700(self): + base = os_helper.TESTFN + path = os.path.abspath(os.path.join(os_helper.TESTFN, 'dir')) + os.mkdir(path, mode=0o700) + out = subprocess.check_output(["cacls.exe", path, "/s"], encoding="oem") + os.rmdir(path) + out = out.strip().rsplit(" ", 1)[1] + self.assertEqual( + out, + '"D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)"', + ) + def tearDown(self): path = os.path.join(os_helper.TESTFN, 'dir1', 'dir2', 'dir3', 'dir4', 'dir5', 'dir6') @@ -2196,9 +2465,13 @@ def test_chmod(self): @unittest.skipIf(support.is_wasi, "Cannot create invalid FD on WASI.") class TestInvalidFD(unittest.TestCase): - singles = ["fchdir", "dup", "fdatasync", "fstat", - "fstatvfs", "fsync", "tcgetpgrp", "ttyname"] - singles_fildes = {"fchdir", "fdatasync", "fsync"} + singles = ["fchdir", "dup", "fstat", "fstatvfs", "tcgetpgrp", "ttyname"] + singles_fildes = {"fchdir"} + # systemd-nspawn --suppress-sync=true does not verify fd passed + # fdatasync() and fsync(), and always returns success + if not support.in_systemd_nspawn_sync_suppressed(): + singles += ["fdatasync", "fsync"] + singles_fildes |= {"fdatasync", "fsync"} #singles.append("close") #We omit close because it doesn't raise an exception on some platforms def get_single(f): @@ -2256,10 +2529,6 @@ def test_dup2(self): self.check(os.dup2, 20) @unittest.skipUnless(hasattr(os, 'dup2'), 'test needs os.dup2()') - @unittest.skipIf( - support.is_emscripten, - "dup2() with negative fds is broken on Emscripten (see gh-102179)" - ) def test_dup2_negative_fd(self): valid_fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, valid_fd) @@ -2284,13 +2553,14 @@ def test_fchown(self): self.check(os.fchown, -1, -1) @unittest.skipUnless(hasattr(os, 'fpathconf'), 'test needs os.fpathconf()') - @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "musl libc issue on Emscripten/WASI, bpo-46390" - ) def test_fpathconf(self): - self.check(os.pathconf, "PC_NAME_MAX") - self.check(os.fpathconf, "PC_NAME_MAX") + self.assertIn("PC_NAME_MAX", os.pathconf_names) + if not (support.is_emscripten or support.is_wasi): + # musl libc pathconf ignores the file descriptor and always returns + # a constant, so the assertion that it should notice a bad file + # descriptor and return EBADF fails. + self.check(os.pathconf, "PC_NAME_MAX") + self.check(os.fpathconf, "PC_NAME_MAX") self.check_bool(os.pathconf, "PC_NAME_MAX") self.check_bool(os.fpathconf, "PC_NAME_MAX") @@ -2308,6 +2578,10 @@ def test_lseek(self): def test_read(self): self.check(os.read, 1) + @unittest.skipUnless(hasattr(os, 'readinto'), 'test needs os.readinto()') + def test_readinto(self): + self.check(os.readinto, bytearray(5)) + @unittest.skipUnless(hasattr(os, 'readv'), 'test needs os.readv()') def test_readv(self): buf = bytearray(10) @@ -3017,7 +3291,8 @@ class Win32NtTests(unittest.TestCase): def test_getfinalpathname_handles(self): nt = import_helper.import_module('nt') ctypes = import_helper.import_module('ctypes') - import ctypes.wintypes + # Ruff false positive -- it thinks we're redefining `ctypes` here + import ctypes.wintypes # noqa: F811 kernel = ctypes.WinDLL('Kernel32.dll', use_last_error=True) kernel.GetCurrentProcess.restype = ctypes.wintypes.HANDLE @@ -3129,9 +3404,8 @@ def test_stat_inaccessible_file(self): self.skipTest("Unable to create inaccessible file") def cleanup(): - # Give delete permission. We are the file owner, so we can do this - # even though we removed all permissions earlier. - subprocess.check_output([ICACLS, filename, "/grant", "Everyone:(D)"], + # Give delete permission to the owner (us) + subprocess.check_output([ICACLS, filename, "/grant", "*WD:(D)"], stderr=subprocess.STDOUT) os.unlink(filename) @@ -3221,9 +3495,6 @@ def test_bad_fd(self): @unittest.skipUnless(os.isatty(0) and not win32_is_iot() and (sys.platform.startswith('win') or (hasattr(locale, 'nl_langinfo') and hasattr(locale, 'CODESET'))), 'test requires a tty and either Windows or nl_langinfo(CODESET)') - @unittest.skipIf( - support.is_emscripten, "Cannot get encoding of stdin on Emscripten" - ) def test_device_encoding(self): encoding = os.device_encoding(0) self.assertIsNotNone(encoding) @@ -3533,9 +3804,8 @@ def test_set_get_priority(self): class TestSendfile(unittest.IsolatedAsyncioTestCase): DATA = b"12345abcde" * 16 * 1024 # 160 KiB - SUPPORT_HEADERS_TRAILERS = not sys.platform.startswith("linux") and \ - not sys.platform.startswith("solaris") and \ - not sys.platform.startswith("sunos") + SUPPORT_HEADERS_TRAILERS = ( + not sys.platform.startswith(("linux", "android", "solaris", "sunos"))) requires_headers_trailers = unittest.skipUnless(SUPPORT_HEADERS_TRAILERS, 'requires headers and trailers support') requires_32b = unittest.skipUnless(sys.maxsize < 2**32, @@ -3808,10 +4078,10 @@ def _check_xattrs_str(self, s, getxattr, setxattr, removexattr, listxattr, **kwa xattr.remove("user.test") self.assertEqual(set(listxattr(fn)), xattr) self.assertEqual(getxattr(fn, s("user.test2"), **kwargs), b"foo") - setxattr(fn, s("user.test"), b"a"*1024, **kwargs) - self.assertEqual(getxattr(fn, s("user.test"), **kwargs), b"a"*1024) + setxattr(fn, s("user.test"), b"a"*256, **kwargs) + self.assertEqual(getxattr(fn, s("user.test"), **kwargs), b"a"*256) removexattr(fn, s("user.test"), **kwargs) - many = sorted("user.test{}".format(i) for i in range(100)) + many = sorted("user.test{}".format(i) for i in range(32)) for thing in many: setxattr(fn, thing, b"x", **kwargs) self.assertEqual(set(listxattr(fn)), set(init_xattr) | set(many)) @@ -3858,7 +4128,12 @@ def test_does_not_crash(self): try: size = os.get_terminal_size() except OSError as e: - if sys.platform == "win32" or e.errno in (errno.EINVAL, errno.ENOTTY): + known_errnos = [errno.EINVAL, errno.ENOTTY] + if sys.platform == "android": + # The Android testbed redirects the native stdout to a pipe, + # which returns a different error code. + known_errnos.append(errno.EACCES) + if sys.platform == "win32" or e.errno in known_errnos: # Under win32 a generic OSError can be thrown if the # handle cannot be retrieved self.skipTest("failed to query terminal size") @@ -4010,11 +4285,18 @@ def test_eventfd_select(self): os.eventfd_read(fd) @unittest.skipUnless(hasattr(os, 'timerfd_create'), 'requires os.timerfd_create') +@unittest.skipIf(sys.platform == "android", "gh-124873: Test is flaky on Android") @support.requires_linux_version(2, 6, 30) class TimerfdTests(unittest.TestCase): - # Tolerate a difference of 1 ms - CLOCK_RES_NS = 1_000_000 - CLOCK_RES = CLOCK_RES_NS * 1e-9 + # 1 ms accuracy is reliably achievable on every platform except Android + # emulators, where we allow 10 ms (gh-108277). + if sys.platform == "android" and platform.android_ver().is_emulator: + CLOCK_RES_PLACES = 2 + else: + CLOCK_RES_PLACES = 3 + + CLOCK_RES = 10 ** -CLOCK_RES_PLACES + CLOCK_RES_NS = 10 ** (9 - CLOCK_RES_PLACES) def timerfd_create(self, *args, **kwargs): fd = os.timerfd_create(*args, **kwargs) @@ -4036,18 +4318,18 @@ def test_timerfd_initval(self): # 1st call next_expiration, interval2 = os.timerfd_settime(fd, initial=initial_expiration, interval=interval) - self.assertAlmostEqual(interval2, 0.0, places=3) - self.assertAlmostEqual(next_expiration, 0.0, places=3) + self.assertAlmostEqual(interval2, 0.0, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, 0.0, places=self.CLOCK_RES_PLACES) # 2nd call next_expiration, interval2 = os.timerfd_settime(fd, initial=initial_expiration, interval=interval) - self.assertAlmostEqual(interval2, interval, places=3) - self.assertAlmostEqual(next_expiration, initial_expiration, places=3) + self.assertAlmostEqual(interval2, interval, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, initial_expiration, places=self.CLOCK_RES_PLACES) # timerfd_gettime next_expiration, interval2 = os.timerfd_gettime(fd) - self.assertAlmostEqual(interval2, interval, places=3) - self.assertAlmostEqual(next_expiration, initial_expiration, places=3) + self.assertAlmostEqual(interval2, interval, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, initial_expiration, places=self.CLOCK_RES_PLACES) def test_timerfd_non_blocking(self): fd = self.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) @@ -4101,8 +4383,8 @@ def test_timerfd_interval(self): # timerfd_gettime next_expiration, interval2 = os.timerfd_gettime(fd) - self.assertAlmostEqual(interval2, interval, places=3) - self.assertAlmostEqual(next_expiration, initial_expiration, places=3) + self.assertAlmostEqual(interval2, interval, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, initial_expiration, places=self.CLOCK_RES_PLACES) count = 3 t = time.perf_counter() @@ -4133,8 +4415,8 @@ def test_timerfd_TFD_TIMER_ABSTIME(self): # timerfd_gettime # Note: timerfd_gettime returns relative values even if TFD_TIMER_ABSTIME is specified. next_expiration, interval2 = os.timerfd_gettime(fd) - self.assertAlmostEqual(interval2, interval, places=3) - self.assertAlmostEqual(next_expiration, offset, places=3) + self.assertAlmostEqual(interval2, interval, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, offset, places=self.CLOCK_RES_PLACES) t = time.perf_counter() count_signaled = self.read_count_signaled(fd) @@ -5256,8 +5538,9 @@ def test_fork(self): else: assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug") - @unittest.skipUnless(sys.platform in ("linux", "darwin"), + @unittest.skipUnless(sys.platform in ("linux", "android", "darwin"), "Only Linux and macOS detect this today.") + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_fork_warns_when_non_python_thread_exists(self): code = """if 1: import os, threading, warnings @@ -5284,20 +5567,21 @@ def test_fork_warns_when_non_python_thread_exists(self): self.assertEqual(err.decode("utf-8"), "") self.assertEqual(out.decode("utf-8"), "") - def test_fork_at_exit(self): + def test_fork_at_finalization(self): code = """if 1: import atexit import os - def exit_handler(): - pid = os.fork() - if pid != 0: - print("shouldn't be printed") - - atexit.register(exit_handler) + class AtFinalization: + def __del__(self): + print("OK") + pid = os.fork() + if pid != 0: + print("shouldn't be printed") + at_finalization = AtFinalization() """ _, out, err = assert_python_ok("-c", code) - self.assertEqual(b"", out) + self.assertEqual(b"OK\n", out) self.assertIn(b"can't fork at interpreter shutdown", err) diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 7e44ae61a5eba7..d64092b710a4d6 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -1,3 +1,5 @@ +import collections +import contextlib import io import os import sys @@ -15,27 +17,65 @@ from test.support import import_helper from test.support import is_emscripten, is_wasi -from test.support import set_recursion_limit +from test.support import infinite_recursion +from test.support import swap_attr from test.support import os_helper from test.support.os_helper import TESTFN, FakePath from test.test_pathlib import test_pathlib_abc -from test.test_pathlib.test_pathlib_abc import needs_posix, needs_windows, needs_symlinks +from test.test_pathlib.test_pathlib_abc import needs_posix, needs_windows +try: + import fcntl +except ImportError: + fcntl = None try: import grp, pwd except ImportError: grp = pwd = None +try: + import posix +except ImportError: + posix = None root_in_posix = False if hasattr(os, 'geteuid'): root_in_posix = (os.geteuid() == 0) + +def patch_replace(old_test): + def new_replace(self, target): + raise OSError(errno.EXDEV, "Cross-device link", self, target) + + def new_test(self): + old_replace = self.cls.replace + self.cls.replace = new_replace + try: + old_test(self) + finally: + self.cls.replace = old_replace + return new_test + + +_tests_needing_symlinks = set() +def needs_symlinks(fn): + """Decorator that marks a test as requiring a path class that supports symlinks.""" + _tests_needing_symlinks.add(fn.__name__) + return fn + + + +class UnsupportedOperationTest(unittest.TestCase): + def test_is_notimplemented(self): + self.assertTrue(issubclass(pathlib.UnsupportedOperation, NotImplementedError)) + self.assertTrue(isinstance(pathlib.UnsupportedOperation(), NotImplementedError)) + + # # Tests for the pure classes. # -class PurePathTest(test_pathlib_abc.DummyPurePathTest): +class PurePathTest(test_pathlib_abc.DummyJoinablePathTest): cls = pathlib.PurePath # Make sure any symbolic links in the base test path are resolved. @@ -65,7 +105,7 @@ def test_concrete_class(self): p = self.cls('a') self.assertIs(type(p), expected) - def test_concrete_pathmod(self): + def test_concrete_parser(self): if self.cls is pathlib.PurePosixPath: expected = posixpath elif self.cls is pathlib.PureWindowsPath: @@ -73,19 +113,19 @@ def test_concrete_pathmod(self): else: expected = os.path p = self.cls('a') - self.assertIs(p.pathmod, expected) + self.assertIs(p.parser, expected) - def test_different_pathmods_unequal(self): + def test_different_parsers_unequal(self): p = self.cls('a') - if p.pathmod is posixpath: + if p.parser is posixpath: q = pathlib.PureWindowsPath('a') else: q = pathlib.PurePosixPath('a') self.assertNotEqual(p, q) - def test_different_pathmods_unordered(self): + def test_different_parsers_unordered(self): p = self.cls('a') - if p.pathmod is posixpath: + if p.parser is posixpath: q = pathlib.PureWindowsPath('a') else: q = pathlib.PurePosixPath('a') @@ -107,17 +147,26 @@ def test_constructor_nested(self): self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) self.assertEqual(P(P('./a:b')), P('./a:b')) + @needs_windows + def test_constructor_nested_foreign_flavour(self): + # See GH-125069. + p1 = pathlib.PurePosixPath('b/c:\\d') + p2 = pathlib.PurePosixPath('b/', 'c:\\d') + self.assertEqual(p1, p2) + self.assertEqual(self.cls(p1), self.cls('b/c:/d')) + self.assertEqual(self.cls(p2), self.cls('b/c:/d')) + def _check_parse_path(self, raw_path, *expected): - sep = self.pathmod.sep + sep = self.parser.sep actual = self.cls._parse_path(raw_path.replace('/', sep)) self.assertEqual(actual, expected) - if altsep := self.pathmod.altsep: + if altsep := self.parser.altsep: actual = self.cls._parse_path(raw_path.replace('/', altsep)) self.assertEqual(actual, expected) def test_parse_path_common(self): check = self._check_parse_path - sep = self.pathmod.sep + sep = self.parser.sep check('', '', '', []) check('a', '', '', ['a']) check('a/', '', '', ['a']) @@ -139,15 +188,6 @@ def test_empty_path(self): # Special case for the empty path. self._check_str('.', ('',)) - def test_parts_interning(self): - P = self.cls - p = P('/usr/bin/foo') - q = P('/usr/local/bin') - # 'usr' - self.assertIs(p.parts[1], q.parts[1]) - # 'bin' - self.assertIs(p.parts[2], q.parts[3]) - def test_join_nested(self): P = self.cls p = P('a/b').joinpath(P('c')) @@ -189,6 +229,31 @@ def test_fspath_common(self): self._check_str(p.__fspath__(), ('a/b',)) self._check_str(os.fspath(p), ('a/b',)) + def test_bytes(self): + P = self.cls + with self.assertRaises(TypeError): + P(b'a') + with self.assertRaises(TypeError): + P(b'a', 'b') + with self.assertRaises(TypeError): + P('a', b'b') + with self.assertRaises(TypeError): + P('a').joinpath(b'b') + with self.assertRaises(TypeError): + P('a') / b'b' + with self.assertRaises(TypeError): + b'a' / P('b') + with self.assertRaises(TypeError): + P('a').match(b'b') + with self.assertRaises(TypeError): + P('a').relative_to(b'b') + with self.assertRaises(TypeError): + P('a').with_name(b'b') + with self.assertRaises(TypeError): + P('a').with_stem(b'b') + with self.assertRaises(TypeError): + P('a').with_suffix(b'b') + def test_bytes_exc_message(self): P = self.cls message = (r"argument should be a str or an os\.PathLike object " @@ -205,6 +270,12 @@ def test_as_bytes_common(self): P = self.cls self.assertEqual(bytes(P('a/b')), b'a' + sep + b'b') + def test_as_posix_common(self): + P = self.cls + for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): + self.assertEqual(P(pathstr).as_posix(), pathstr) + # Other tests for as_posix() are in test_equivalences(). + def test_eq_common(self): P = self.cls self.assertEqual(P('a/b'), P('a/b')) @@ -284,6 +355,51 @@ def test_repr_roundtrips(self): self.assertEqual(q, p) self.assertEqual(repr(q), r) + def test_drive_common(self): + P = self.cls + self.assertEqual(P('a/b').drive, '') + self.assertEqual(P('/a/b').drive, '') + self.assertEqual(P('').drive, '') + + @needs_windows + def test_drive_windows(self): + P = self.cls + self.assertEqual(P('c:').drive, 'c:') + self.assertEqual(P('c:a/b').drive, 'c:') + self.assertEqual(P('c:/').drive, 'c:') + self.assertEqual(P('c:/a/b/').drive, 'c:') + self.assertEqual(P('//a/b').drive, '\\\\a\\b') + self.assertEqual(P('//a/b/').drive, '\\\\a\\b') + self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') + self.assertEqual(P('./c:a').drive, '') + + + def test_root_common(self): + P = self.cls + sep = self.sep + self.assertEqual(P('').root, '') + self.assertEqual(P('a/b').root, '') + self.assertEqual(P('/').root, sep) + self.assertEqual(P('/a/b').root, sep) + + @needs_posix + def test_root_posix(self): + P = self.cls + self.assertEqual(P('/a/b').root, '/') + # POSIX special case for two leading slashes. + self.assertEqual(P('//a/b').root, '//') + + @needs_windows + def test_root_windows(self): + P = self.cls + self.assertEqual(P('c:').root, '') + self.assertEqual(P('c:a/b').root, '') + self.assertEqual(P('c:/').root, '\\') + self.assertEqual(P('c:/a/b/').root, '\\') + self.assertEqual(P('//a/b').root, '\\') + self.assertEqual(P('//a/b/').root, '\\') + self.assertEqual(P('//a/b/c/d').root, '\\') + def test_name_empty(self): P = self.cls self.assertEqual(P('').name, '') @@ -311,19 +427,6 @@ def test_with_stem_empty(self): self.assertRaises(ValueError, P('a/b').with_stem, '') self.assertRaises(ValueError, P('a/b').with_stem, '.') - def test_relative_to_several_args(self): - P = self.cls - p = P('a/b') - with self.assertWarns(DeprecationWarning): - p.relative_to('a', 'b') - p.relative_to('a', 'b', walk_up=True) - - def test_is_relative_to_several_args(self): - P = self.cls - p = P('a/b') - with self.assertWarns(DeprecationWarning): - p.is_relative_to('a', 'b') - def test_is_reserved_deprecated(self): P = self.cls p = P('a/b') @@ -335,6 +438,84 @@ def test_match_empty(self): self.assertRaises(ValueError, P('a').match, '') self.assertRaises(ValueError, P('a').match, '.') + def test_match_common(self): + P = self.cls + # Simple relative pattern. + self.assertTrue(P('b.py').match('b.py')) + self.assertTrue(P('a/b.py').match('b.py')) + self.assertTrue(P('/a/b.py').match('b.py')) + self.assertFalse(P('a.py').match('b.py')) + self.assertFalse(P('b/py').match('b.py')) + self.assertFalse(P('/a.py').match('b.py')) + self.assertFalse(P('b.py/c').match('b.py')) + # Wildcard relative pattern. + self.assertTrue(P('b.py').match('*.py')) + self.assertTrue(P('a/b.py').match('*.py')) + self.assertTrue(P('/a/b.py').match('*.py')) + self.assertFalse(P('b.pyc').match('*.py')) + self.assertFalse(P('b./py').match('*.py')) + self.assertFalse(P('b.py/c').match('*.py')) + # Multi-part relative pattern. + self.assertTrue(P('ab/c.py').match('a*/*.py')) + self.assertTrue(P('/d/ab/c.py').match('a*/*.py')) + self.assertFalse(P('a.py').match('a*/*.py')) + self.assertFalse(P('/dab/c.py').match('a*/*.py')) + self.assertFalse(P('ab/c.py/d').match('a*/*.py')) + # Absolute pattern. + self.assertTrue(P('/b.py').match('/*.py')) + self.assertFalse(P('b.py').match('/*.py')) + self.assertFalse(P('a/b.py').match('/*.py')) + self.assertFalse(P('/a/b.py').match('/*.py')) + # Multi-part absolute pattern. + self.assertTrue(P('/a/b.py').match('/a/*.py')) + self.assertFalse(P('/ab.py').match('/a/*.py')) + self.assertFalse(P('/a/b/c.py').match('/a/*.py')) + # Multi-part glob-style pattern. + self.assertFalse(P('/a/b/c.py').match('/**/*.py')) + self.assertTrue(P('/a/b/c.py').match('/a/**/*.py')) + # Case-sensitive flag + self.assertFalse(P('A.py').match('a.PY', case_sensitive=True)) + self.assertTrue(P('A.py').match('a.PY', case_sensitive=False)) + self.assertFalse(P('c:/a/B.Py').match('C:/A/*.pY', case_sensitive=True)) + self.assertTrue(P('/a/b/c.py').match('/A/*/*.Py', case_sensitive=False)) + # Matching against empty path + self.assertFalse(P('').match('*')) + self.assertFalse(P('').match('**')) + self.assertFalse(P('').match('**/*')) + + @needs_posix + def test_match_posix(self): + P = self.cls + self.assertFalse(P('A.py').match('a.PY')) + + @needs_windows + def test_match_windows(self): + P = self.cls + # Absolute patterns. + self.assertTrue(P('c:/b.py').match('*:/*.py')) + self.assertTrue(P('c:/b.py').match('c:/*.py')) + self.assertFalse(P('d:/b.py').match('c:/*.py')) # wrong drive + self.assertFalse(P('b.py').match('/*.py')) + self.assertFalse(P('b.py').match('c:*.py')) + self.assertFalse(P('b.py').match('c:/*.py')) + self.assertFalse(P('c:b.py').match('/*.py')) + self.assertFalse(P('c:b.py').match('c:/*.py')) + self.assertFalse(P('/b.py').match('c:*.py')) + self.assertFalse(P('/b.py').match('c:/*.py')) + # UNC patterns. + self.assertTrue(P('//some/share/a.py').match('//*/*/*.py')) + self.assertTrue(P('//some/share/a.py').match('//some/share/*.py')) + self.assertFalse(P('//other/share/a.py').match('//some/share/*.py')) + self.assertFalse(P('//some/share/a/b.py').match('//some/share/*.py')) + # Case-insensitivity. + self.assertTrue(P('B.py').match('b.PY')) + self.assertTrue(P('c:/a/B.Py').match('C:/A/*.pY')) + self.assertTrue(P('//Some/Share/B.Py').match('//somE/sharE/*.pY')) + # Path anchor doesn't match pattern anchor + self.assertFalse(P('c:/b.py').match('/*.py')) # 'c:/' vs '/' + self.assertFalse(P('c:/b.py').match('c:*.py')) # 'c:/' vs 'c:' + self.assertFalse(P('//some/share/a.py').match('/*.py')) # '//some/share/' vs '/' + @needs_posix def test_parse_path_posix(self): check = self._check_parse_path @@ -495,6 +676,311 @@ def assertOrderedEqual(a, b): self.assertFalse(p < q) self.assertFalse(p > q) + @needs_posix + def test_is_absolute_posix(self): + P = self.cls + self.assertFalse(P('').is_absolute()) + self.assertFalse(P('a').is_absolute()) + self.assertFalse(P('a/b/').is_absolute()) + self.assertTrue(P('/').is_absolute()) + self.assertTrue(P('/a').is_absolute()) + self.assertTrue(P('/a/b/').is_absolute()) + self.assertTrue(P('//a').is_absolute()) + self.assertTrue(P('//a/b').is_absolute()) + + @needs_windows + def test_is_absolute_windows(self): + P = self.cls + # Under NT, only paths with both a drive and a root are absolute. + self.assertFalse(P().is_absolute()) + self.assertFalse(P('a').is_absolute()) + self.assertFalse(P('a/b/').is_absolute()) + self.assertFalse(P('/').is_absolute()) + self.assertFalse(P('/a').is_absolute()) + self.assertFalse(P('/a/b/').is_absolute()) + self.assertFalse(P('c:').is_absolute()) + self.assertFalse(P('c:a').is_absolute()) + self.assertFalse(P('c:a/b/').is_absolute()) + self.assertTrue(P('c:/').is_absolute()) + self.assertTrue(P('c:/a').is_absolute()) + self.assertTrue(P('c:/a/b/').is_absolute()) + # UNC paths are absolute by definition. + self.assertTrue(P('//').is_absolute()) + self.assertTrue(P('//a').is_absolute()) + self.assertTrue(P('//a/b').is_absolute()) + self.assertTrue(P('//a/b/').is_absolute()) + self.assertTrue(P('//a/b/c').is_absolute()) + self.assertTrue(P('//a/b/c/d').is_absolute()) + self.assertTrue(P('//?/UNC/').is_absolute()) + self.assertTrue(P('//?/UNC/spam').is_absolute()) + + def test_relative_to_common(self): + P = self.cls + p = P('a/b') + self.assertRaises(TypeError, p.relative_to) + self.assertRaises(TypeError, p.relative_to, b'a') + self.assertEqual(p.relative_to(P('')), P('a/b')) + self.assertEqual(p.relative_to(''), P('a/b')) + self.assertEqual(p.relative_to(P('a')), P('b')) + self.assertEqual(p.relative_to('a'), P('b')) + self.assertEqual(p.relative_to('a/'), P('b')) + self.assertEqual(p.relative_to(P('a/b')), P('')) + self.assertEqual(p.relative_to('a/b'), P('')) + self.assertEqual(p.relative_to(P(''), walk_up=True), P('a/b')) + self.assertEqual(p.relative_to('', walk_up=True), P('a/b')) + self.assertEqual(p.relative_to(P('a'), walk_up=True), P('b')) + self.assertEqual(p.relative_to('a', walk_up=True), P('b')) + self.assertEqual(p.relative_to('a/', walk_up=True), P('b')) + self.assertEqual(p.relative_to(P('a/b'), walk_up=True), P('')) + self.assertEqual(p.relative_to('a/b', walk_up=True), P('')) + self.assertEqual(p.relative_to(P('a/c'), walk_up=True), P('../b')) + self.assertEqual(p.relative_to('a/c', walk_up=True), P('../b')) + self.assertEqual(p.relative_to(P('a/b/c'), walk_up=True), P('..')) + self.assertEqual(p.relative_to('a/b/c', walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('c'), walk_up=True), P('../a/b')) + self.assertEqual(p.relative_to('c', walk_up=True), P('../a/b')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('c')) + self.assertRaises(ValueError, p.relative_to, P('a/b/c')) + self.assertRaises(ValueError, p.relative_to, P('a/c')) + self.assertRaises(ValueError, p.relative_to, P('/a')) + self.assertRaises(ValueError, p.relative_to, P("../a")) + self.assertRaises(ValueError, p.relative_to, P("a/..")) + self.assertRaises(ValueError, p.relative_to, P("/a/..")) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/a'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) + p = P('/a/b') + self.assertEqual(p.relative_to(P('/')), P('a/b')) + self.assertEqual(p.relative_to('/'), P('a/b')) + self.assertEqual(p.relative_to(P('/a')), P('b')) + self.assertEqual(p.relative_to('/a'), P('b')) + self.assertEqual(p.relative_to('/a/'), P('b')) + self.assertEqual(p.relative_to(P('/a/b')), P('')) + self.assertEqual(p.relative_to('/a/b'), P('')) + self.assertEqual(p.relative_to(P('/'), walk_up=True), P('a/b')) + self.assertEqual(p.relative_to('/', walk_up=True), P('a/b')) + self.assertEqual(p.relative_to(P('/a'), walk_up=True), P('b')) + self.assertEqual(p.relative_to('/a', walk_up=True), P('b')) + self.assertEqual(p.relative_to('/a/', walk_up=True), P('b')) + self.assertEqual(p.relative_to(P('/a/b'), walk_up=True), P('')) + self.assertEqual(p.relative_to('/a/b', walk_up=True), P('')) + self.assertEqual(p.relative_to(P('/a/c'), walk_up=True), P('../b')) + self.assertEqual(p.relative_to('/a/c', walk_up=True), P('../b')) + self.assertEqual(p.relative_to(P('/a/b/c'), walk_up=True), P('..')) + self.assertEqual(p.relative_to('/a/b/c', walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('/c'), walk_up=True), P('../a/b')) + self.assertEqual(p.relative_to('/c', walk_up=True), P('../a/b')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('/c')) + self.assertRaises(ValueError, p.relative_to, P('/a/b/c')) + self.assertRaises(ValueError, p.relative_to, P('/a/c')) + self.assertRaises(ValueError, p.relative_to, P('')) + self.assertRaises(ValueError, p.relative_to, '') + self.assertRaises(ValueError, p.relative_to, P('a')) + self.assertRaises(ValueError, p.relative_to, P("../a")) + self.assertRaises(ValueError, p.relative_to, P("a/..")) + self.assertRaises(ValueError, p.relative_to, P("/a/..")) + self.assertRaises(ValueError, p.relative_to, P(''), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('a'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) + + @needs_windows + def test_relative_to_windows(self): + P = self.cls + p = P('C:Foo/Bar') + self.assertEqual(p.relative_to(P('c:')), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:'), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:foO')), P('Bar')) + self.assertEqual(p.relative_to('c:foO'), P('Bar')) + self.assertEqual(p.relative_to('c:foO/'), P('Bar')) + self.assertEqual(p.relative_to(P('c:foO/baR')), P()) + self.assertEqual(p.relative_to('c:foO/baR'), P()) + self.assertEqual(p.relative_to(P('c:'), walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:foO'), walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:foO', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:foO/', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to(P('c:foO/baR'), walk_up=True), P()) + self.assertEqual(p.relative_to('c:foO/baR', walk_up=True), P()) + self.assertEqual(p.relative_to(P('C:Foo/Bar/Baz'), walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('C:Foo/Baz'), walk_up=True), P('../Bar')) + self.assertEqual(p.relative_to(P('C:Baz/Bar'), walk_up=True), P('../../Foo/Bar')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P()) + self.assertRaises(ValueError, p.relative_to, '') + self.assertRaises(ValueError, p.relative_to, P('d:')) + self.assertRaises(ValueError, p.relative_to, P('/')) + self.assertRaises(ValueError, p.relative_to, P('Foo')) + self.assertRaises(ValueError, p.relative_to, P('/Foo')) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo')) + self.assertRaises(ValueError, p.relative_to, P('C:Foo/Bar/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:Foo/Baz')) + self.assertRaises(ValueError, p.relative_to, P(), walk_up=True) + self.assertRaises(ValueError, p.relative_to, '', walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo'), walk_up=True) + p = P('C:/Foo/Bar') + self.assertEqual(p.relative_to(P('c:/')), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:/'), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:/foO')), P('Bar')) + self.assertEqual(p.relative_to('c:/foO'), P('Bar')) + self.assertEqual(p.relative_to('c:/foO/'), P('Bar')) + self.assertEqual(p.relative_to(P('c:/foO/baR')), P()) + self.assertEqual(p.relative_to('c:/foO/baR'), P()) + self.assertEqual(p.relative_to(P('c:/'), walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:/', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:/foO'), walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:/foO', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:/foO/', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to(P('c:/foO/baR'), walk_up=True), P()) + self.assertEqual(p.relative_to('c:/foO/baR', walk_up=True), P()) + self.assertEqual(p.relative_to('C:/Baz', walk_up=True), P('../Foo/Bar')) + self.assertEqual(p.relative_to('C:/Foo/Bar/Baz', walk_up=True), P('..')) + self.assertEqual(p.relative_to('C:/Foo/Baz', walk_up=True), P('../Bar')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, 'c:') + self.assertRaises(ValueError, p.relative_to, P('c:')) + self.assertRaises(ValueError, p.relative_to, P('C:/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Bar/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:Foo')) + self.assertRaises(ValueError, p.relative_to, P('d:')) + self.assertRaises(ValueError, p.relative_to, P('d:/')) + self.assertRaises(ValueError, p.relative_to, P('/')) + self.assertRaises(ValueError, p.relative_to, P('/Foo')) + self.assertRaises(ValueError, p.relative_to, P('//C/Foo')) + self.assertRaises(ValueError, p.relative_to, 'c:', walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('c:'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('C:Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('d:/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('//C/Foo'), walk_up=True) + # UNC paths. + p = P('//Server/Share/Foo/Bar') + self.assertEqual(p.relative_to(P('//sErver/sHare')), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare'), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/'), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo')), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo'), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/'), P('Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar')), P()) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar'), P()) + self.assertEqual(p.relative_to(P('//sErver/sHare'), walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo'), walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar'), walk_up=True), P()) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar', walk_up=True), P()) + self.assertEqual(p.relative_to(P('//sErver/sHare/bar'), walk_up=True), P('../Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/bar', walk_up=True), P('../Foo/Bar')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo')) + self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo')) + self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo')) + self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo')) + self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo'), walk_up=True) + + def test_is_relative_to_common(self): + P = self.cls + p = P('a/b') + self.assertRaises(TypeError, p.is_relative_to) + self.assertRaises(TypeError, p.is_relative_to, b'a') + self.assertTrue(p.is_relative_to(P(''))) + self.assertTrue(p.is_relative_to('')) + self.assertTrue(p.is_relative_to(P('a'))) + self.assertTrue(p.is_relative_to('a/')) + self.assertTrue(p.is_relative_to(P('a/b'))) + self.assertTrue(p.is_relative_to('a/b')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('c'))) + self.assertFalse(p.is_relative_to(P('a/b/c'))) + self.assertFalse(p.is_relative_to(P('a/c'))) + self.assertFalse(p.is_relative_to(P('/a'))) + p = P('/a/b') + self.assertTrue(p.is_relative_to(P('/'))) + self.assertTrue(p.is_relative_to('/')) + self.assertTrue(p.is_relative_to(P('/a'))) + self.assertTrue(p.is_relative_to('/a')) + self.assertTrue(p.is_relative_to('/a/')) + self.assertTrue(p.is_relative_to(P('/a/b'))) + self.assertTrue(p.is_relative_to('/a/b')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('/c'))) + self.assertFalse(p.is_relative_to(P('/a/b/c'))) + self.assertFalse(p.is_relative_to(P('/a/c'))) + self.assertFalse(p.is_relative_to(P(''))) + self.assertFalse(p.is_relative_to('')) + self.assertFalse(p.is_relative_to(P('a'))) + + @needs_windows + def test_is_relative_to_windows(self): + P = self.cls + p = P('C:Foo/Bar') + self.assertTrue(p.is_relative_to(P('c:'))) + self.assertTrue(p.is_relative_to('c:')) + self.assertTrue(p.is_relative_to(P('c:foO'))) + self.assertTrue(p.is_relative_to('c:foO')) + self.assertTrue(p.is_relative_to('c:foO/')) + self.assertTrue(p.is_relative_to(P('c:foO/baR'))) + self.assertTrue(p.is_relative_to('c:foO/baR')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P())) + self.assertFalse(p.is_relative_to('')) + self.assertFalse(p.is_relative_to(P('d:'))) + self.assertFalse(p.is_relative_to(P('/'))) + self.assertFalse(p.is_relative_to(P('Foo'))) + self.assertFalse(p.is_relative_to(P('/Foo'))) + self.assertFalse(p.is_relative_to(P('C:/Foo'))) + self.assertFalse(p.is_relative_to(P('C:Foo/Bar/Baz'))) + self.assertFalse(p.is_relative_to(P('C:Foo/Baz'))) + p = P('C:/Foo/Bar') + self.assertTrue(p.is_relative_to(P('c:/'))) + self.assertTrue(p.is_relative_to(P('c:/foO'))) + self.assertTrue(p.is_relative_to('c:/foO/')) + self.assertTrue(p.is_relative_to(P('c:/foO/baR'))) + self.assertTrue(p.is_relative_to('c:/foO/baR')) + # Unrelated paths. + self.assertFalse(p.is_relative_to('c:')) + self.assertFalse(p.is_relative_to(P('C:/Baz'))) + self.assertFalse(p.is_relative_to(P('C:/Foo/Bar/Baz'))) + self.assertFalse(p.is_relative_to(P('C:/Foo/Baz'))) + self.assertFalse(p.is_relative_to(P('C:Foo'))) + self.assertFalse(p.is_relative_to(P('d:'))) + self.assertFalse(p.is_relative_to(P('d:/'))) + self.assertFalse(p.is_relative_to(P('/'))) + self.assertFalse(p.is_relative_to(P('/Foo'))) + self.assertFalse(p.is_relative_to(P('//C/Foo'))) + # UNC paths. + p = P('//Server/Share/Foo/Bar') + self.assertTrue(p.is_relative_to(P('//sErver/sHare'))) + self.assertTrue(p.is_relative_to('//sErver/sHare')) + self.assertTrue(p.is_relative_to('//sErver/sHare/')) + self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo'))) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo')) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/')) + self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo/Bar'))) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/Bar')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('/Server/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('c:/Server/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('//z/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('//Server/z/Foo'))) + class PurePosixPathTest(PurePathTest): cls = pathlib.PurePosixPath @@ -516,17 +1002,54 @@ class cls(pathlib.PurePath): # Tests for the concrete classes. # -class PathTest(test_pathlib_abc.DummyPathTest, PurePathTest): +class PathTest(test_pathlib_abc.DummyRWPathTest, PurePathTest): """Tests for the FS-accessing functionalities of the Path classes.""" cls = pathlib.Path can_symlink = os_helper.can_symlink() def setUp(self): + name = self.id().split('.')[-1] + if name in _tests_needing_symlinks and not self.can_symlink: + self.skipTest('requires symlinks') super().setUp() - os.chmod(self.pathmod.join(self.base, 'dirE'), 0) + + def createTestHierarchy(self): + os.mkdir(self.base) + os.mkdir(os.path.join(self.base, 'dirA')) + os.mkdir(os.path.join(self.base, 'dirB')) + os.mkdir(os.path.join(self.base, 'dirC')) + os.mkdir(os.path.join(self.base, 'dirC', 'dirD')) + os.mkdir(os.path.join(self.base, 'dirE')) + with open(os.path.join(self.base, 'fileA'), 'wb') as f: + f.write(b"this is file A\n") + with open(os.path.join(self.base, 'dirB', 'fileB'), 'wb') as f: + f.write(b"this is file B\n") + with open(os.path.join(self.base, 'dirC', 'fileC'), 'wb') as f: + f.write(b"this is file C\n") + with open(os.path.join(self.base, 'dirC', 'novel.txt'), 'wb') as f: + f.write(b"this is a novel\n") + with open(os.path.join(self.base, 'dirC', 'dirD', 'fileD'), 'wb') as f: + f.write(b"this is file D\n") + os.chmod(os.path.join(self.base, 'dirE'), 0) + if self.can_symlink: + # Relative symlinks. + os.symlink('fileA', os.path.join(self.base, 'linkA')) + os.symlink('non-existing', os.path.join(self.base, 'brokenLink')) + os.symlink('dirB', + os.path.join(self.base, 'linkB'), + target_is_directory=True) + os.symlink(os.path.join('..', 'dirB'), + os.path.join(self.base, 'dirA', 'linkC'), + target_is_directory=True) + # This one goes upwards, creating a loop. + os.symlink(os.path.join('..', 'dirB'), + os.path.join(self.base, 'dirB', 'linkD'), + target_is_directory=True) + # Broken symlink (pointing to itself). + os.symlink('brokenLinkLoop', os.path.join(self.base, 'brokenLinkLoop')) def tearDown(self): - os.chmod(self.pathmod.join(self.base, 'dirE'), 0o777) + os.chmod(os.path.join(self.base, 'dirE'), 0o777) os_helper.rmtree(self.base) def tempdir(self): @@ -535,18 +1058,15 @@ def tempdir(self): self.addCleanup(os_helper.rmtree, d) return d - def test_matches_pathbase_api(self): - our_names = {name for name in dir(self.cls) if name[0] != '_'} - our_names.remove('is_reserved') # only present in PurePath - path_names = {name for name in dir(pathlib._abc.PathBase) if name[0] != '_'} - self.assertEqual(our_names, path_names) - for attr_name in our_names: - if attr_name == 'pathmod': - # On Windows, Path.pathmod is ntpath, but PathBase.pathmod is + def test_matches_writablepath_docstrings(self): + path_names = {name for name in dir(pathlib._abc.WritablePath) if name[0] != '_'} + for attr_name in path_names: + if attr_name == 'parser': + # On Windows, Path.parser is ntpath, but WritablePath.parser is # posixpath, and so their docstrings differ. continue our_attr = getattr(self.cls, attr_name) - path_attr = getattr(pathlib._abc.PathBase, attr_name) + path_attr = getattr(pathlib._abc.WritablePath, attr_name) self.assertEqual(our_attr.__doc__, path_attr.__doc__) def test_concrete_class(self): @@ -557,9 +1077,9 @@ def test_concrete_class(self): p = self.cls('a') self.assertIs(type(p), expected) - def test_unsupported_pathmod(self): - if self.cls.pathmod is os.path: - self.skipTest("path flavour is supported") + def test_unsupported_parser(self): + if self.cls.parser is os.path: + self.skipTest("path parser is supported") else: self.assertRaises(pathlib.UnsupportedOperation, self.cls) @@ -660,12 +1180,646 @@ def with_segments(self, *pathsegments): for dirpath, dirnames, filenames in p.walk(): self.assertEqual(42, dirpath.session_id) + def test_open_common(self): + p = self.cls(self.base) + with (p / 'fileA').open('r') as f: + self.assertIsInstance(f, io.TextIOBase) + self.assertEqual(f.read(), "this is file A\n") + with (p / 'fileA').open('rb') as f: + self.assertIsInstance(f, io.BufferedIOBase) + self.assertEqual(f.read().strip(), b"this is file A") + def test_open_unbuffered(self): p = self.cls(self.base) with (p / 'fileA').open('rb', buffering=0) as f: self.assertIsInstance(f, io.RawIOBase) self.assertEqual(f.read().strip(), b"this is file A") + def test_copy_file_preserve_metadata(self): + base = self.cls(self.base) + source = base / 'fileA' + if hasattr(os, 'chmod'): + os.chmod(source, stat.S_IRWXU | stat.S_IRWXO) + if hasattr(os, 'chflags') and hasattr(stat, 'UF_NODUMP'): + os.chflags(source, stat.UF_NODUMP) + source_st = source.stat() + target = base / 'copyA' + source.copy(target, preserve_metadata=True) + self.assertTrue(target.exists()) + self.assertEqual(source.read_text(), target.read_text()) + target_st = target.stat() + self.assertLessEqual(source_st.st_atime, target_st.st_atime) + self.assertLessEqual(source_st.st_mtime, target_st.st_mtime) + self.assertEqual(source_st.st_mode, target_st.st_mode) + if hasattr(source_st, 'st_flags'): + self.assertEqual(source_st.st_flags, target_st.st_flags) + + @needs_symlinks + def test_copy_file_to_existing_symlink(self): + base = self.cls(self.base) + source = base / 'dirB' / 'fileB' + target = base / 'linkA' + real_target = base / 'fileA' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertTrue(real_target.exists()) + self.assertFalse(real_target.is_symlink()) + self.assertEqual(source.read_text(), real_target.read_text()) + + @needs_symlinks + def test_copy_file_to_existing_symlink_follow_symlinks_false(self): + base = self.cls(self.base) + source = base / 'dirB' / 'fileB' + target = base / 'linkA' + real_target = base / 'fileA' + result = source.copy(target, follow_symlinks=False) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertTrue(real_target.exists()) + self.assertFalse(real_target.is_symlink()) + self.assertEqual(source.read_text(), real_target.read_text()) + + @os_helper.skip_unless_xattr + def test_copy_file_preserve_metadata_xattrs(self): + base = self.cls(self.base) + source = base / 'fileA' + os.setxattr(source, b'user.foo', b'42') + target = base / 'copyA' + source.copy(target, preserve_metadata=True) + self.assertEqual(os.getxattr(target, b'user.foo'), b'42') + + @needs_symlinks + def test_copy_symlink_follow_symlinks_true(self): + base = self.cls(self.base) + source = base / 'linkA' + target = base / 'copyA' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertFalse(target.is_symlink()) + self.assertEqual(source.read_text(), target.read_text()) + + @needs_symlinks + def test_copy_symlink_follow_symlinks_false(self): + base = self.cls(self.base) + source = base / 'linkA' + target = base / 'copyA' + result = source.copy(target, follow_symlinks=False) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source.readlink(), target.readlink()) + + @needs_symlinks + def test_copy_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkA' + self.assertRaises(OSError, source.copy, source) + + @needs_symlinks + def test_copy_symlink_to_existing_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'fileA') + target.symlink_to(base / 'dirC') + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_symlink_to_existing_directory_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'fileA') + target.symlink_to(base / 'dirC') + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_directory_symlink_follow_symlinks_false(self): + base = self.cls(self.base) + source = base / 'linkB' + target = base / 'copyA' + result = source.copy(target, follow_symlinks=False) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source.readlink(), target.readlink()) + + @needs_symlinks + def test_copy_directory_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkB' + self.assertRaises(OSError, source.copy, source) + self.assertRaises(OSError, source.copy, source, follow_symlinks=False) + + @needs_symlinks + def test_copy_directory_symlink_into_itself(self): + base = self.cls(self.base) + source = base / 'linkB' + target = base / 'linkB' / 'copyB' + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + self.assertFalse(target.exists()) + + @needs_symlinks + def test_copy_directory_symlink_to_existing_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'dirC') + target.symlink_to(base / 'fileA') + self.assertRaises(FileExistsError, source.copy, target) + self.assertRaises(FileExistsError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_directory_symlink_to_existing_directory_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'dirC' / 'dirD') + target.symlink_to(base / 'dirC') + self.assertRaises(FileExistsError, source.copy, target) + self.assertRaises(FileExistsError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_dangling_symlink(self): + base = self.cls(self.base) + source = base / 'source' + target = base / 'target' + + source.mkdir() + source.joinpath('link').symlink_to('nonexistent') + + self.assertRaises(FileNotFoundError, source.copy, target) + + target2 = base / 'target2' + result = source.copy(target2, follow_symlinks=False) + self.assertEqual(result, target2) + self.assertTrue(target2.joinpath('link').is_symlink()) + self.assertEqual(target2.joinpath('link').readlink(), self.cls('nonexistent')) + + @needs_symlinks + def test_copy_link_preserve_metadata(self): + base = self.cls(self.base) + source = base / 'linkA' + if hasattr(os, 'lchmod'): + os.lchmod(source, stat.S_IRWXU | stat.S_IRWXO) + if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'): + os.lchflags(source, stat.UF_NODUMP) + source_st = source.lstat() + target = base / 'copyA' + source.copy(target, follow_symlinks=False, preserve_metadata=True) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source.readlink(), target.readlink()) + target_st = target.lstat() + self.assertLessEqual(source_st.st_atime, target_st.st_atime) + self.assertLessEqual(source_st.st_mtime, target_st.st_mtime) + self.assertEqual(source_st.st_mode, target_st.st_mode) + if hasattr(source_st, 'st_flags'): + self.assertEqual(source_st.st_flags, target_st.st_flags) + + def test_copy_error_handling(self): + def make_raiser(err): + def raiser(*args, **kwargs): + raise OSError(err, os.strerror(err)) + return raiser + + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'copyA' + + # Raise non-fatal OSError from all available fast copy functions. + with contextlib.ExitStack() as ctx: + if fcntl and hasattr(fcntl, 'FICLONE'): + ctx.enter_context(mock.patch('fcntl.ioctl', make_raiser(errno.EXDEV))) + if posix and hasattr(posix, '_fcopyfile'): + ctx.enter_context(mock.patch('posix._fcopyfile', make_raiser(errno.ENOTSUP))) + if hasattr(os, 'copy_file_range'): + ctx.enter_context(mock.patch('os.copy_file_range', make_raiser(errno.EXDEV))) + if hasattr(os, 'sendfile'): + ctx.enter_context(mock.patch('os.sendfile', make_raiser(errno.ENOTSOCK))) + + source.copy(target) + self.assertTrue(target.exists()) + self.assertEqual(source.read_text(), target.read_text()) + + # Raise fatal OSError from first available fast copy function. + if fcntl and hasattr(fcntl, 'FICLONE'): + patchpoint = 'fcntl.ioctl' + elif posix and hasattr(posix, '_fcopyfile'): + patchpoint = 'posix._fcopyfile' + elif hasattr(os, 'copy_file_range'): + patchpoint = 'os.copy_file_range' + elif hasattr(os, 'sendfile'): + patchpoint = 'os.sendfile' + else: + return + with mock.patch(patchpoint, make_raiser(errno.ENOENT)): + self.assertRaises(FileNotFoundError, source.copy, target) + + @unittest.skipIf(sys.platform == "win32" or sys.platform == "wasi", "directories are always readable on Windows and WASI") + @unittest.skipIf(root_in_posix, "test fails with root privilege") + def test_copy_dir_no_read_permission(self): + base = self.cls(self.base) + source = base / 'dirE' + target = base / 'copyE' + self.assertRaises(PermissionError, source.copy, target) + self.assertFalse(target.exists()) + + def test_copy_dir_preserve_metadata(self): + base = self.cls(self.base) + source = base / 'dirC' + if hasattr(os, 'chmod'): + os.chmod(source / 'dirD', stat.S_IRWXU | stat.S_IRWXO) + if hasattr(os, 'chflags') and hasattr(stat, 'UF_NODUMP'): + os.chflags(source / 'fileC', stat.UF_NODUMP) + target = base / 'copyA' + source.copy(target, preserve_metadata=True) + + for subpath in ['.', 'fileC', 'dirD', 'dirD/fileD']: + source_st = source.joinpath(subpath).stat() + target_st = target.joinpath(subpath).stat() + self.assertLessEqual(source_st.st_atime, target_st.st_atime) + self.assertLessEqual(source_st.st_mtime, target_st.st_mtime) + self.assertEqual(source_st.st_mode, target_st.st_mode) + if hasattr(source_st, 'st_flags'): + self.assertEqual(source_st.st_flags, target_st.st_flags) + + @os_helper.skip_unless_xattr + def test_copy_dir_preserve_metadata_xattrs(self): + base = self.cls(self.base) + source = base / 'dirC' + source_file = source.joinpath('dirD', 'fileD') + os.setxattr(source_file, b'user.foo', b'42') + target = base / 'copyA' + source.copy(target, preserve_metadata=True) + target_file = target.joinpath('dirD', 'fileD') + self.assertEqual(os.getxattr(target_file, b'user.foo'), b'42') + + @needs_symlinks + def test_move_file_symlink(self): + base = self.cls(self.base) + source = base / 'linkA' + source_readlink = source.readlink() + target = base / 'linkA_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source_readlink, target.readlink()) + + @needs_symlinks + def test_move_file_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkA' + self.assertRaises(OSError, source.move, source) + + @needs_symlinks + def test_move_dir_symlink(self): + base = self.cls(self.base) + source = base / 'linkB' + source_readlink = source.readlink() + target = base / 'linkB_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source_readlink, target.readlink()) + + @needs_symlinks + def test_move_dir_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkB' + self.assertRaises(OSError, source.move, source) + + @needs_symlinks + def test_move_dangling_symlink(self): + base = self.cls(self.base) + source = base / 'brokenLink' + source_readlink = source.readlink() + target = base / 'brokenLink_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source_readlink, target.readlink()) + + def test_move_file(self): + base = self.cls(self.base) + source = base / 'fileA' + source_text = source.read_text() + target = base / 'fileA_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.exists()) + self.assertEqual(source_text, target.read_text()) + + @patch_replace + def test_move_file_other_fs(self): + self.test_move_file() + + def test_move_file_to_file(self): + base = self.cls(self.base) + source = base / 'fileA' + source_text = source.read_text() + target = base / 'dirB' / 'fileB' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.exists()) + self.assertEqual(source_text, target.read_text()) + + @patch_replace + def test_move_file_to_file_other_fs(self): + self.test_move_file_to_file() + + def test_move_file_to_dir(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'dirB' + self.assertRaises(OSError, source.move, target) + + @patch_replace + def test_move_file_to_dir_other_fs(self): + self.test_move_file_to_dir() + + def test_move_file_to_itself(self): + base = self.cls(self.base) + source = base / 'fileA' + self.assertRaises(OSError, source.move, source) + + def test_move_dir(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'dirC_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_dir()) + self.assertTrue(target.joinpath('dirD').is_dir()) + self.assertTrue(target.joinpath('dirD', 'fileD').is_file()) + self.assertEqual(target.joinpath('dirD', 'fileD').read_text(), + "this is file D\n") + self.assertTrue(target.joinpath('fileC').is_file()) + self.assertTrue(target.joinpath('fileC').read_text(), + "this is file C\n") + + @patch_replace + def test_move_dir_other_fs(self): + self.test_move_dir() + + def test_move_dir_to_dir(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'dirB' + self.assertRaises(OSError, source.move, target) + self.assertTrue(source.exists()) + self.assertTrue(target.exists()) + + @patch_replace + def test_move_dir_to_dir_other_fs(self): + self.test_move_dir_to_dir() + + def test_move_dir_to_itself(self): + base = self.cls(self.base) + source = base / 'dirC' + self.assertRaises(OSError, source.move, source) + self.assertTrue(source.exists()) + + def test_move_dir_into_itself(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'dirC' / 'bar' + self.assertRaises(OSError, source.move, target) + self.assertTrue(source.exists()) + self.assertFalse(target.exists()) + + @patch_replace + def test_move_dir_into_itself_other_fs(self): + self.test_move_dir_into_itself() + + @patch_replace + @needs_symlinks + def test_move_file_symlink_other_fs(self): + self.test_move_file_symlink() + + @patch_replace + @needs_symlinks + def test_move_file_symlink_to_itself_other_fs(self): + self.test_move_file_symlink_to_itself() + + @patch_replace + @needs_symlinks + def test_move_dir_symlink_other_fs(self): + self.test_move_dir_symlink() + + @patch_replace + @needs_symlinks + def test_move_dir_symlink_to_itself_other_fs(self): + self.test_move_dir_symlink_to_itself() + + @patch_replace + @needs_symlinks + def test_move_dangling_symlink_other_fs(self): + self.test_move_dangling_symlink() + + def test_move_into(self): + base = self.cls(self.base) + source = base / 'fileA' + source_text = source.read_text() + target_dir = base / 'dirA' + result = source.move_into(target_dir) + self.assertEqual(result, target_dir / 'fileA') + self.assertFalse(source.exists()) + self.assertTrue(result.exists()) + self.assertEqual(source_text, result.read_text()) + + @patch_replace + def test_move_into_other_os(self): + self.test_move_into() + + def test_move_into_empty_name(self): + source = self.cls('') + target_dir = self.base + self.assertRaises(ValueError, source.move_into, target_dir) + + @patch_replace + def test_move_into_empty_name_other_os(self): + self.test_move_into_empty_name() + + @needs_symlinks + def test_complex_symlinks_absolute(self): + self._check_complex_symlinks(self.base) + + @needs_symlinks + def test_complex_symlinks_relative(self): + self._check_complex_symlinks('.') + + @needs_symlinks + def test_complex_symlinks_relative_dot_dot(self): + self._check_complex_symlinks(self.parser.join('dirA', '..')) + + def _check_complex_symlinks(self, link0_target): + # Test solving a non-looping chain of symlinks (issue #19887). + parser = self.parser + P = self.cls(self.base) + P.joinpath('link1').symlink_to(parser.join('link0', 'link0'), target_is_directory=True) + P.joinpath('link2').symlink_to(parser.join('link1', 'link1'), target_is_directory=True) + P.joinpath('link3').symlink_to(parser.join('link2', 'link2'), target_is_directory=True) + P.joinpath('link0').symlink_to(link0_target, target_is_directory=True) + + # Resolve absolute paths. + p = (P / 'link0').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = (P / 'link1').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = (P / 'link2').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = (P / 'link3').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + + # Resolve relative paths. + old_path = os.getcwd() + os.chdir(self.base) + try: + p = self.cls('link0').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = self.cls('link1').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = self.cls('link2').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = self.cls('link3').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + finally: + os.chdir(old_path) + + def _check_resolve(self, p, expected, strict=True): + q = p.resolve(strict) + self.assertEqual(q, expected) + + # This can be used to check both relative and absolute resolutions. + _check_resolve_relative = _check_resolve_absolute = _check_resolve + + @needs_symlinks + def test_resolve_common(self): + P = self.cls + p = P(self.base, 'foo') + with self.assertRaises(OSError) as cm: + p.resolve(strict=True) + self.assertEqual(cm.exception.errno, errno.ENOENT) + # Non-strict + parser = self.parser + self.assertEqualNormCase(str(p.resolve(strict=False)), + parser.join(self.base, 'foo')) + p = P(self.base, 'foo', 'in', 'spam') + self.assertEqualNormCase(str(p.resolve(strict=False)), + parser.join(self.base, 'foo', 'in', 'spam')) + p = P(self.base, '..', 'foo', 'in', 'spam') + self.assertEqualNormCase(str(p.resolve(strict=False)), + parser.join(parser.dirname(self.base), 'foo', 'in', 'spam')) + # These are all relative symlinks. + p = P(self.base, 'dirB', 'fileB') + self._check_resolve_relative(p, p) + p = P(self.base, 'linkA') + self._check_resolve_relative(p, P(self.base, 'fileA')) + p = P(self.base, 'dirA', 'linkC', 'fileB') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) + p = P(self.base, 'dirB', 'linkD', 'fileB') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) + # Non-strict + p = P(self.base, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB', 'foo', 'in', + 'spam'), False) + p = P(self.base, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') + if self.cls.parser is not posixpath: + # In Windows, if linkY points to dirB, 'dirA\linkY\..' + # resolves to 'dirA' without resolving linkY first. + self._check_resolve_relative(p, P(self.base, 'dirA', 'foo', 'in', + 'spam'), False) + else: + # In Posix, if linkY points to dirB, 'dirA/linkY/..' + # resolves to 'dirB/..' first before resolving to parent of dirB. + self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) + # Now create absolute symlinks. + d = self.tempdir() + P(self.base, 'dirA', 'linkX').symlink_to(d) + P(self.base, str(d), 'linkY').symlink_to(self.parser.join(self.base, 'dirB')) + p = P(self.base, 'dirA', 'linkX', 'linkY', 'fileB') + self._check_resolve_absolute(p, P(self.base, 'dirB', 'fileB')) + # Non-strict + p = P(self.base, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') + self._check_resolve_relative(p, P(self.base, 'dirB', 'foo', 'in', 'spam'), + False) + p = P(self.base, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') + if self.cls.parser is not posixpath: + # In Windows, if linkY points to dirB, 'dirA\linkY\..' + # resolves to 'dirA' without resolving linkY first. + self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) + else: + # In Posix, if linkY points to dirB, 'dirA/linkY/..' + # resolves to 'dirB/..' first before resolving to parent of dirB. + self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) + + @needs_symlinks + def test_resolve_dot(self): + # See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/ + parser = self.parser + p = self.cls(self.base) + p.joinpath('0').symlink_to('.', target_is_directory=True) + p.joinpath('1').symlink_to(parser.join('0', '0'), target_is_directory=True) + p.joinpath('2').symlink_to(parser.join('1', '1'), target_is_directory=True) + q = p / '2' + self.assertEqual(q.resolve(strict=True), p) + r = q / '3' / '4' + self.assertRaises(FileNotFoundError, r.resolve, strict=True) + # Non-strict + self.assertEqual(r.resolve(strict=False), p / '3' / '4') + + def _check_symlink_loop(self, *args): + path = self.cls(*args) + with self.assertRaises(OSError) as cm: + path.resolve(strict=True) + self.assertEqual(cm.exception.errno, errno.ELOOP) + + @needs_posix + @needs_symlinks + def test_resolve_loop(self): + # Loops with relative symlinks. + self.cls(self.base, 'linkX').symlink_to('linkX/inside') + self._check_symlink_loop(self.base, 'linkX') + self.cls(self.base, 'linkY').symlink_to('linkY') + self._check_symlink_loop(self.base, 'linkY') + self.cls(self.base, 'linkZ').symlink_to('linkZ/../linkZ') + self._check_symlink_loop(self.base, 'linkZ') + # Non-strict + p = self.cls(self.base, 'linkZ', 'foo') + self.assertEqual(p.resolve(strict=False), p) + # Loops with absolute symlinks. + self.cls(self.base, 'linkU').symlink_to(self.parser.join(self.base, 'linkU/inside')) + self._check_symlink_loop(self.base, 'linkU') + self.cls(self.base, 'linkV').symlink_to(self.parser.join(self.base, 'linkV')) + self._check_symlink_loop(self.base, 'linkV') + self.cls(self.base, 'linkW').symlink_to(self.parser.join(self.base, 'linkW/../linkW')) + self._check_symlink_loop(self.base, 'linkW') + # Non-strict + q = self.cls(self.base, 'linkW', 'foo') + self.assertEqual(q.resolve(strict=False), q) + def test_resolve_nonexist_relative_issue38671(self): p = self.cls('non', 'exist') @@ -676,6 +1830,24 @@ def test_resolve_nonexist_relative_issue38671(self): finally: os.chdir(old_cwd) + @needs_symlinks + def test_readlink(self): + P = self.cls(self.base) + self.assertEqual((P / 'linkA').readlink(), self.cls('fileA')) + self.assertEqual((P / 'brokenLink').readlink(), + self.cls('non-existing')) + self.assertEqual((P / 'linkB').readlink(), self.cls('dirB')) + self.assertEqual((P / 'linkB' / 'linkD').readlink(), self.cls('../dirB')) + with self.assertRaises(OSError): + (P / 'fileA').readlink() + + @unittest.skipIf(hasattr(os, "readlink"), "os.readlink() is present") + def test_readlink_unsupported(self): + P = self.cls(self.base) + p = P / 'fileA' + with self.assertRaises(pathlib.UnsupportedOperation): + q.readlink(p) + @os_helper.skip_unless_working_chmod def test_chmod(self): p = self.cls(self.base) / 'fileA' @@ -796,6 +1968,174 @@ def test_rmdir(self): self.assertFileNotFound(p.stat) self.assertFileNotFound(p.unlink) + def test_delete_file(self): + p = self.cls(self.base) / 'fileA' + p._delete() + self.assertFalse(p.exists()) + self.assertFileNotFound(p._delete) + + def test_delete_dir(self): + base = self.cls(self.base) + base.joinpath('dirA')._delete() + self.assertFalse(base.joinpath('dirA').exists()) + self.assertFalse(base.joinpath('dirA', 'linkC').exists( + follow_symlinks=False)) + base.joinpath('dirB')._delete() + self.assertFalse(base.joinpath('dirB').exists()) + self.assertFalse(base.joinpath('dirB', 'fileB').exists()) + self.assertFalse(base.joinpath('dirB', 'linkD').exists( + follow_symlinks=False)) + base.joinpath('dirC')._delete() + self.assertFalse(base.joinpath('dirC').exists()) + self.assertFalse(base.joinpath('dirC', 'dirD').exists()) + self.assertFalse(base.joinpath('dirC', 'dirD', 'fileD').exists()) + self.assertFalse(base.joinpath('dirC', 'fileC').exists()) + self.assertFalse(base.joinpath('dirC', 'novel.txt').exists()) + + def test_delete_missing(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + # filename is guaranteed not to exist + filename = tmp / 'foo' + self.assertRaises(FileNotFoundError, filename._delete) + + @needs_symlinks + def test_delete_symlink(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + dir_ = tmp / 'dir' + dir_.mkdir() + link = tmp / 'link' + link.symlink_to(dir_) + link._delete() + self.assertTrue(dir_.exists()) + self.assertFalse(link.exists(follow_symlinks=False)) + + @needs_symlinks + def test_delete_inner_symlink(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + dir1 = tmp / 'dir1' + dir2 = dir1 / 'dir2' + dir3 = tmp / 'dir3' + for d in dir1, dir2, dir3: + d.mkdir() + file1 = tmp / 'file1' + file1.write_text('foo') + link1 = dir1 / 'link1' + link1.symlink_to(dir2) + link2 = dir1 / 'link2' + link2.symlink_to(dir3) + link3 = dir1 / 'link3' + link3.symlink_to(file1) + # make sure symlinks are removed but not followed + dir1._delete() + self.assertFalse(dir1.exists()) + self.assertTrue(dir3.exists()) + self.assertTrue(file1.exists()) + + @unittest.skipIf(sys.platform[:6] == 'cygwin', + "This test can't be run on Cygwin (issue #1071513).") + @os_helper.skip_if_dac_override + @os_helper.skip_unless_working_chmod + def test_delete_unwritable(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + child_file_path = tmp / 'a' + child_dir_path = tmp / 'b' + child_file_path.write_text("") + child_dir_path.mkdir() + old_dir_mode = tmp.stat().st_mode + old_child_file_mode = child_file_path.stat().st_mode + old_child_dir_mode = child_dir_path.stat().st_mode + # Make unwritable. + new_mode = stat.S_IREAD | stat.S_IEXEC + try: + child_file_path.chmod(new_mode) + child_dir_path.chmod(new_mode) + tmp.chmod(new_mode) + + self.assertRaises(PermissionError, tmp._delete) + finally: + tmp.chmod(old_dir_mode) + child_file_path.chmod(old_child_file_mode) + child_dir_path.chmod(old_child_dir_mode) + + @needs_windows + def test_delete_inner_junction(self): + import _winapi + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + dir1 = tmp / 'dir1' + dir2 = dir1 / 'dir2' + dir3 = tmp / 'dir3' + for d in dir1, dir2, dir3: + d.mkdir() + file1 = tmp / 'file1' + file1.write_text('foo') + link1 = dir1 / 'link1' + _winapi.CreateJunction(str(dir2), str(link1)) + link2 = dir1 / 'link2' + _winapi.CreateJunction(str(dir3), str(link2)) + link3 = dir1 / 'link3' + _winapi.CreateJunction(str(file1), str(link3)) + # make sure junctions are removed but not followed + dir1._delete() + self.assertFalse(dir1.exists()) + self.assertTrue(dir3.exists()) + self.assertTrue(file1.exists()) + + @needs_windows + def test_delete_outer_junction(self): + import _winapi + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + src = tmp / 'cheese' + dst = tmp / 'shop' + src.mkdir() + spam = src / 'spam' + spam.write_text('') + _winapi.CreateJunction(str(src), str(dst)) + dst._delete() + self.assertFalse(dst.exists()) + self.assertTrue(spam.exists()) + self.assertTrue(src.exists()) + + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_delete_on_named_pipe(self): + p = self.cls(self.base, 'pipe') + os.mkfifo(p) + p._delete() + self.assertFalse(p.exists()) + + p = self.cls(self.base, 'dir') + p.mkdir() + os.mkfifo(p / 'mypipe') + p._delete() + self.assertFalse(p.exists()) + + def test_delete_does_not_choke_on_failing_lstat(self): + try: + orig_lstat = os.lstat + tmp = self.cls(self.base, 'delete') + + def raiser(fn, *args, **kwargs): + if fn != tmp: + raise OSError() + else: + return orig_lstat(fn) + + os.lstat = raiser + + tmp.mkdir() + foo = tmp / 'foo' + foo.write_text('') + tmp._delete() + finally: + os.lstat = orig_lstat + @os_helper.skip_unless_hardlink def test_hardlink_to(self): P = self.cls(self.base) @@ -809,7 +2149,7 @@ def test_hardlink_to(self): self.assertTrue(target.exists()) # Linking to a str of a relative path. link2 = P / 'dirA' / 'fileAAA' - target2 = self.pathmod.join(TESTFN, 'fileA') + target2 = self.parser.join(TESTFN, 'fileA') link2.hardlink_to(target2) self.assertEqual(os.stat(target2).st_size, size) self.assertTrue(link2.exists()) @@ -834,7 +2174,7 @@ def test_rename(self): self.assertEqual(q.stat().st_size, size) self.assertFileNotFound(p.stat) # Renaming to a str of a relative path. - r = self.pathmod.join(TESTFN, 'fileAAA') + r = self.parser.join(TESTFN, 'fileAAA') renamed_q = q.rename(r) self.assertEqual(renamed_q, self.cls(r)) self.assertEqual(os.stat(r).st_size, size) @@ -851,7 +2191,7 @@ def test_replace(self): self.assertEqual(q.stat().st_size, size) self.assertFileNotFound(p.stat) # Replacing another (existing) path. - r = self.pathmod.join(TESTFN, 'dirB', 'fileB') + r = self.parser.join(TESTFN, 'dirB', 'fileB') replaced_q = q.replace(r) self.assertEqual(replaced_q, self.cls(r)) self.assertEqual(os.stat(r).st_size, size) @@ -953,7 +2293,6 @@ def test_mkdir_exist_ok_with_parent(self): self.assertTrue(p.exists()) self.assertEqual(p.stat().st_ctime, st_ctime_first) - @unittest.skipIf(is_emscripten, "FS root cannot be modified on Emscripten.") def test_mkdir_exist_ok_root(self): # Issue #25803: A drive root could raise PermissionError on Windows. self.cls('/').resolve().mkdir(exist_ok=True) @@ -1057,12 +2396,72 @@ def test_symlink_to_unsupported(self): with self.assertRaises(pathlib.UnsupportedOperation): q.symlink_to(p) - def test_is_junction(self): + def test_stat(self): + statA = self.cls(self.base).joinpath('fileA').stat() + statB = self.cls(self.base).joinpath('dirB', 'fileB').stat() + statC = self.cls(self.base).joinpath('dirC').stat() + # st_mode: files are the same, directory differs. + self.assertIsInstance(statA.st_mode, int) + self.assertEqual(statA.st_mode, statB.st_mode) + self.assertNotEqual(statA.st_mode, statC.st_mode) + self.assertNotEqual(statB.st_mode, statC.st_mode) + # st_ino: all different, + self.assertIsInstance(statA.st_ino, int) + self.assertNotEqual(statA.st_ino, statB.st_ino) + self.assertNotEqual(statA.st_ino, statC.st_ino) + self.assertNotEqual(statB.st_ino, statC.st_ino) + # st_dev: all the same. + self.assertIsInstance(statA.st_dev, int) + self.assertEqual(statA.st_dev, statB.st_dev) + self.assertEqual(statA.st_dev, statC.st_dev) + # other attributes not used by pathlib. + + def test_stat_no_follow_symlinks_nosymlink(self): + p = self.cls(self.base) / 'fileA' + st = p.stat() + self.assertEqual(st, p.stat(follow_symlinks=False)) + + @needs_symlinks + def test_stat_no_follow_symlinks(self): + p = self.cls(self.base) / 'linkA' + st = p.stat() + self.assertNotEqual(st, p.stat(follow_symlinks=False)) + + @needs_symlinks + def test_lstat(self): + p = self.cls(self.base)/ 'linkA' + st = p.stat() + self.assertNotEqual(st, p.lstat()) + + def test_lstat_nosymlink(self): + p = self.cls(self.base) / 'fileA' + st = p.stat() + self.assertEqual(st, p.lstat()) + + def test_is_junction_false(self): P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_junction()) + self.assertFalse((P / 'dirA').is_junction()) + self.assertFalse((P / 'non-existing').is_junction()) + self.assertFalse((P / 'fileA' / 'bah').is_junction()) + self.assertFalse((P / 'fileA\udfff').is_junction()) + self.assertFalse((P / 'fileA\x00').is_junction()) + + def test_is_junction_true(self): + P = self.cls(self.base) + + with mock.patch.object(P.parser, 'isjunction'): + self.assertEqual(P.is_junction(), P.parser.isjunction.return_value) + P.parser.isjunction.assert_called_once_with(P) - with mock.patch.object(P.pathmod, 'isjunction'): - self.assertEqual(P.is_junction(), P.pathmod.isjunction.return_value) - P.pathmod.isjunction.assert_called_once_with(P) + def test_is_fifo_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_fifo()) + self.assertFalse((P / 'dirA').is_fifo()) + self.assertFalse((P / 'non-existing').is_fifo()) + self.assertFalse((P / 'fileA' / 'bah').is_fifo()) + self.assertIs((P / 'fileA\udfff').is_fifo(), False) + self.assertIs((P / 'fileA\x00').is_fifo(), False) @unittest.skipUnless(hasattr(os, "mkfifo"), "os.mkfifo() required") @unittest.skipIf(sys.platform == "vxworks", @@ -1079,6 +2478,15 @@ def test_is_fifo_true(self): self.assertIs(self.cls(self.base, 'myfifo\udfff').is_fifo(), False) self.assertIs(self.cls(self.base, 'myfifo\x00').is_fifo(), False) + def test_is_socket_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_socket()) + self.assertFalse((P / 'dirA').is_socket()) + self.assertFalse((P / 'non-existing').is_socket()) + self.assertFalse((P / 'fileA' / 'bah').is_socket()) + self.assertIs((P / 'fileA\udfff').is_socket(), False) + self.assertIs((P / 'fileA\x00').is_socket(), False) + @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "Unix sockets required") @unittest.skipIf( is_emscripten, "Unix sockets are not implemented on Emscripten." @@ -1102,18 +2510,43 @@ def test_is_socket_true(self): self.assertIs(self.cls(self.base, 'mysock\udfff').is_socket(), False) self.assertIs(self.cls(self.base, 'mysock\x00').is_socket(), False) + def test_is_block_device_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_block_device()) + self.assertFalse((P / 'dirA').is_block_device()) + self.assertFalse((P / 'non-existing').is_block_device()) + self.assertFalse((P / 'fileA' / 'bah').is_block_device()) + self.assertIs((P / 'fileA\udfff').is_block_device(), False) + self.assertIs((P / 'fileA\x00').is_block_device(), False) + + def test_is_char_device_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_char_device()) + self.assertFalse((P / 'dirA').is_char_device()) + self.assertFalse((P / 'non-existing').is_char_device()) + self.assertFalse((P / 'fileA' / 'bah').is_char_device()) + self.assertIs((P / 'fileA\udfff').is_char_device(), False) + self.assertIs((P / 'fileA\x00').is_char_device(), False) + def test_is_char_device_true(self): - # Under Unix, /dev/null should generally be a char device. - P = self.cls('/dev/null') + # os.devnull should generally be a char device. + P = self.cls(os.devnull) if not P.exists(): - self.skipTest("/dev/null required") + self.skipTest("null device required") self.assertTrue(P.is_char_device()) self.assertFalse(P.is_block_device()) self.assertFalse(P.is_file()) - self.assertIs(self.cls('/dev/null\udfff').is_char_device(), False) - self.assertIs(self.cls('/dev/null\x00').is_char_device(), False) + self.assertIs(self.cls(f'{os.devnull}\udfff').is_char_device(), False) + self.assertIs(self.cls(f'{os.devnull}\x00').is_char_device(), False) - def test_is_mount_root(self): + def test_is_mount(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_mount()) + self.assertFalse((P / 'dirA').is_mount()) + self.assertFalse((P / 'non-existing').is_mount()) + self.assertFalse((P / 'fileA' / 'bah').is_mount()) + if self.can_symlink: + self.assertFalse((P / 'linkA').is_mount()) if os.name == 'nt': R = self.cls('c:\\') else: @@ -1121,87 +2554,39 @@ def test_is_mount_root(self): self.assertTrue(R.is_mount()) self.assertFalse((R / '\udfff').is_mount()) - def test_passing_kwargs_deprecated(self): - with self.assertWarns(DeprecationWarning): + def test_samefile(self): + parser = self.parser + fileA_path = parser.join(self.base, 'fileA') + fileB_path = parser.join(self.base, 'dirB', 'fileB') + p = self.cls(fileA_path) + pp = self.cls(fileA_path) + q = self.cls(fileB_path) + self.assertTrue(p.samefile(fileA_path)) + self.assertTrue(p.samefile(pp)) + self.assertFalse(p.samefile(fileB_path)) + self.assertFalse(p.samefile(q)) + # Test the non-existent file case + non_existent = parser.join(self.base, 'foo') + r = self.cls(non_existent) + self.assertRaises(FileNotFoundError, p.samefile, r) + self.assertRaises(FileNotFoundError, p.samefile, non_existent) + self.assertRaises(FileNotFoundError, r.samefile, p) + self.assertRaises(FileNotFoundError, r.samefile, non_existent) + self.assertRaises(FileNotFoundError, r.samefile, r) + self.assertRaises(FileNotFoundError, r.samefile, non_existent) + + def test_passing_kwargs_errors(self): + with self.assertRaises(TypeError): self.cls(foo="bar") - def setUpWalk(self): - super().setUpWalk() - sub21_path= self.sub2_path / "SUB21" - tmp5_path = sub21_path / "tmp3" - broken_link3_path = self.sub2_path / "broken_link3" - - os.makedirs(sub21_path) - tmp5_path.write_text("I am tmp5, blame test_pathlib.") - if self.can_symlink: - os.symlink(tmp5_path, broken_link3_path) - self.sub2_tree[2].append('broken_link3') - self.sub2_tree[2].sort() - if not is_emscripten: - # Emscripten fails with inaccessible directories. - os.chmod(sub21_path, 0) - try: - os.listdir(sub21_path) - except PermissionError: - self.sub2_tree[1].append('SUB21') - else: - os.chmod(sub21_path, stat.S_IRWXU) - os.unlink(tmp5_path) - os.rmdir(sub21_path) - - def test_walk_bad_dir(self): - self.setUpWalk() - errors = [] - walk_it = self.walk_path.walk(on_error=errors.append) - root, dirs, files = next(walk_it) - self.assertEqual(errors, []) - dir1 = 'SUB1' - path1 = root / dir1 - path1new = (root / dir1).with_suffix(".new") - path1.rename(path1new) - try: - roots = [r for r, _, _ in walk_it] - self.assertTrue(errors) - self.assertNotIn(path1, roots) - self.assertNotIn(path1new, roots) - for dir2 in dirs: - if dir2 != dir1: - self.assertIn(root / dir2, roots) - finally: - path1new.rename(path1) - - def test_walk_many_open_files(self): - depth = 30 - base = self.cls(self.base, 'deep') - path = self.cls(base, *(['d']*depth)) - path.mkdir(parents=True) - - iters = [base.walk(top_down=False) for _ in range(100)] - for i in range(depth + 1): - expected = (path, ['d'] if i else [], []) - for it in iters: - self.assertEqual(next(it), expected) - path = path.parent - - iters = [base.walk(top_down=True) for _ in range(100)] - path = base - for i in range(depth + 1): - expected = (path, ['d'] if i < depth else [], []) - for it in iters: - self.assertEqual(next(it), expected) - path = path / 'd' - - def test_walk_above_recursion_limit(self): - recursion_limit = 40 - # directory_depth > recursion_limit - directory_depth = recursion_limit + 10 - base = self.cls(self.base, 'deep') - path = base.joinpath(*(['d'] * directory_depth)) - path.mkdir(parents=True) - - with set_recursion_limit(recursion_limit): - list(base.walk()) - list(base.walk(top_down=False)) + @needs_symlinks + def test_iterdir_symlink(self): + # __iter__ on a symlink to a directory. + P = self.cls + p = P(self.base, 'linkB') + paths = set(p.iterdir()) + expected = { P(self.base, 'linkB', q) for q in ['fileB', 'linkD'] } + self.assertEqual(paths, expected) def test_glob_empty_pattern(self): p = self.cls('') @@ -1239,7 +2624,7 @@ def test_glob_above_recursion_limit(self): path = base.joinpath(*(['d'] * directory_depth)) path.mkdir(parents=True) - with set_recursion_limit(recursion_limit): + with infinite_recursion(recursion_limit): list(base.glob('**/')) def test_glob_pathlike(self): @@ -1263,6 +2648,32 @@ def test_glob_dot(self): self.assertEqual( set(P('.').glob('**/*/*')), {P("dirD/fileD")}) + # See https://github.com/WebAssembly/wasi-filesystem/issues/26 + @unittest.skipIf(is_wasi, "WASI resolution of '..' parts doesn't match POSIX") + def test_glob_dotdot(self): + # ".." is not special in globs. + P = self.cls + p = P(self.base) + self.assertEqual(set(p.glob("..")), { P(self.base, "..") }) + self.assertEqual(set(p.glob("../..")), { P(self.base, "..", "..") }) + self.assertEqual(set(p.glob("dirA/..")), { P(self.base, "dirA", "..") }) + self.assertEqual(set(p.glob("dirA/../file*")), { P(self.base, "dirA/../fileA") }) + self.assertEqual(set(p.glob("dirA/../file*/..")), set()) + self.assertEqual(set(p.glob("../xyzzy")), set()) + if self.cls.parser is posixpath: + self.assertEqual(set(p.glob("xyzzy/..")), set()) + else: + # ".." segments are normalized first on Windows, so this path is stat()able. + self.assertEqual(set(p.glob("xyzzy/..")), { P(self.base, "xyzzy", "..") }) + self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(self.base, *[".."] * 50)}) + + def test_glob_inaccessible(self): + P = self.cls + p = P(self.base, "mydir1", "mydir2") + p.mkdir(parents=True) + p.parent.chmod(0) + self.assertEqual(set(p.glob('*')), set()) + def test_rglob_pathlike(self): P = self.cls p = P(self.base, "dirC") @@ -1271,6 +2682,122 @@ def test_rglob_pathlike(self): self.assertEqual(expect, set(p.rglob(P(pattern)))) self.assertEqual(expect, set(p.rglob(FakePath(pattern)))) + @needs_symlinks + def test_glob_recurse_symlinks_common(self): + def _check(path, glob, expected): + actual = {path for path in path.glob(glob, recurse_symlinks=True) + if path.parts.count("linkD") <= 1} # exclude symlink loop. + self.assertEqual(actual, { P(self.base, q) for q in expected }) + P = self.cls + p = P(self.base) + _check(p, "fileB", []) + _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) + _check(p, "*A", ["dirA", "fileA", "linkA"]) + _check(p, "*B/*", ["dirB/fileB", "dirB/linkD", "linkB/fileB", "linkB/linkD"]) + _check(p, "*/fileB", ["dirB/fileB", "linkB/fileB"]) + _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"]) + _check(p, "dir*/*/..", ["dirC/dirD/..", "dirA/linkC/..", "dirB/linkD/.."]) + _check(p, "dir*/**", [ + "dirA/", "dirA/linkC", "dirA/linkC/fileB", "dirA/linkC/linkD", "dirA/linkC/linkD/fileB", + "dirB/", "dirB/fileB", "dirB/linkD", "dirB/linkD/fileB", + "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt", + "dirE/"]) + _check(p, "dir*/**/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirC/dirD/", "dirE/"]) + _check(p, "dir*/**/..", ["dirA/..", "dirA/linkC/..", "dirB/..", + "dirB/linkD/..", "dirA/linkC/linkD/..", + "dirC/..", "dirC/dirD/..", "dirE/.."]) + _check(p, "dir*/*/**", [ + "dirA/linkC/", "dirA/linkC/linkD", "dirA/linkC/fileB", "dirA/linkC/linkD/fileB", + "dirB/linkD/", "dirB/linkD/fileB", + "dirC/dirD/", "dirC/dirD/fileD"]) + _check(p, "dir*/*/**/", ["dirA/linkC/", "dirA/linkC/linkD/", "dirB/linkD/", "dirC/dirD/"]) + _check(p, "dir*/*/**/..", ["dirA/linkC/..", "dirA/linkC/linkD/..", + "dirB/linkD/..", "dirC/dirD/.."]) + _check(p, "dir*/**/fileC", ["dirC/fileC"]) + _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) + _check(p, "*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) + _check(p, "*/dirD/**/", ["dirC/dirD/"]) + + @needs_symlinks + def test_rglob_recurse_symlinks_common(self): + def _check(path, glob, expected): + actual = {path for path in path.rglob(glob, recurse_symlinks=True) + if path.parts.count("linkD") <= 1} # exclude symlink loop. + self.assertEqual(actual, { P(self.base, q) for q in expected }) + P = self.cls + p = P(self.base) + _check(p, "fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB", + "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB"]) + _check(p, "*/fileA", []) + _check(p, "*/fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB", + "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB"]) + _check(p, "file*", ["fileA", "dirA/linkC/fileB", "dirB/fileB", + "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB", + "dirC/fileC", "dirC/dirD/fileD", "linkB/fileB"]) + _check(p, "*/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirC/dirD/", "dirE/", "linkB/", "linkB/linkD/"]) + _check(p, "", ["", "dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirE/", "dirC/dirD/", "linkB/", "linkB/linkD/"]) + + p = P(self.base, "dirC") + _check(p, "*", ["dirC/fileC", "dirC/novel.txt", + "dirC/dirD", "dirC/dirD/fileD"]) + _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "*/*", ["dirC/dirD/fileD"]) + _check(p, "*/", ["dirC/dirD/"]) + _check(p, "", ["dirC/", "dirC/dirD/"]) + # gh-91616, a re module regression + _check(p, "*.txt", ["dirC/novel.txt"]) + _check(p, "*.*", ["dirC/novel.txt"]) + + @needs_symlinks + def test_rglob_symlink_loop(self): + # Don't get fooled by symlink loops (Issue #26012). + P = self.cls + p = P(self.base) + given = set(p.rglob('*', recurse_symlinks=False)) + expect = {'brokenLink', + 'dirA', 'dirA/linkC', + 'dirB', 'dirB/fileB', 'dirB/linkD', + 'dirC', 'dirC/dirD', 'dirC/dirD/fileD', + 'dirC/fileC', 'dirC/novel.txt', + 'dirE', + 'fileA', + 'linkA', + 'linkB', + 'brokenLinkLoop', + } + self.assertEqual(given, {p / x for x in expect}) + + @needs_symlinks + def test_glob_permissions(self): + # See bpo-38894 + P = self.cls + base = P(self.base) / 'permissions' + base.mkdir() + + for i in range(100): + link = base / f"link{i}" + if i % 2: + link.symlink_to(P(self.base, "dirE", "nonexistent")) + else: + link.symlink_to(P(self.base, "dirC"), target_is_directory=True) + + self.assertEqual(len(set(base.glob("*"))), 100) + self.assertEqual(len(set(base.glob("*/"))), 50) + self.assertEqual(len(set(base.glob("*/fileC"))), 50) + self.assertEqual(len(set(base.glob("*/file*"))), 50) + + @needs_symlinks + def test_glob_long_symlink(self): + # See gh-87695 + base = self.cls(self.base) / 'long_symlink' + base.mkdir() + bad_link = base / 'bad_link' + bad_link.symlink_to("bad" * 200) + self.assertEqual(sorted(base.glob('**/*')), [bad_link]) + @needs_posix def test_absolute_posix(self): P = self.cls @@ -1289,18 +2816,20 @@ def test_absolute_posix(self): ) @needs_posix def test_open_mode(self): - old_mask = os.umask(0) + # Unmask all permissions except world-write, which may + # not be supported on some filesystems (see GH-85633.) + old_mask = os.umask(0o002) self.addCleanup(os.umask, old_mask) p = self.cls(self.base) with (p / 'new_file').open('wb'): pass - st = os.stat(self.pathmod.join(self.base, 'new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o666) - os.umask(0o022) + st = os.stat(self.parser.join(self.base, 'new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o664) + os.umask(0o026) with (p / 'other_new_file').open('wb'): pass - st = os.stat(self.pathmod.join(self.base, 'other_new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) + st = os.stat(self.parser.join(self.base, 'other_new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o640) @needs_posix def test_resolve_root(self): @@ -1318,18 +2847,20 @@ def test_resolve_root(self): ) @needs_posix def test_touch_mode(self): - old_mask = os.umask(0) + # Unmask all permissions except world-write, which may + # not be supported on some filesystems (see GH-85633.) + old_mask = os.umask(0o002) self.addCleanup(os.umask, old_mask) p = self.cls(self.base) (p / 'new_file').touch() - st = os.stat(self.pathmod.join(self.base, 'new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o666) - os.umask(0o022) + st = os.stat(self.parser.join(self.base, 'new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o664) + os.umask(0o026) (p / 'other_new_file').touch() - st = os.stat(self.pathmod.join(self.base, 'other_new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) + st = os.stat(self.parser.join(self.base, 'other_new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o640) (p / 'masked_new_file').touch(mode=0o750) - st = os.stat(self.pathmod.join(self.base, 'masked_new_file')) + st = os.stat(self.parser.join(self.base, 'masked_new_file')) self.assertEqual(stat.S_IMODE(st.st_mode), 0o750) @unittest.skipUnless(hasattr(pwd, 'getpwall'), @@ -1575,6 +3106,162 @@ def test_group_windows(self): P('c:/').group() +class PathWalkTest(test_pathlib_abc.DummyReadablePathWalkTest): + cls = pathlib.Path + base = PathTest.base + can_symlink = PathTest.can_symlink + + def setUp(self): + name = self.id().split('.')[-1] + if name in _tests_needing_symlinks and not self.can_symlink: + self.skipTest('requires symlinks') + super().setUp() + + def createTestHierarchy(self): + # Build: + # TESTFN/ + # TEST1/ a file kid and two directory kids + # tmp1 + # SUB1/ a file kid and a directory kid + # tmp2 + # SUB11/ no kids + # SUB2/ a file kid and a dirsymlink kid + # tmp3 + # link/ a symlink to TEST2 + # broken_link + # broken_link2 + # TEST2/ + # tmp4 a lone file + t2_path = self.cls(self.base, "TEST2") + os.makedirs(self.sub11_path) + os.makedirs(self.sub2_path) + os.makedirs(t2_path) + + tmp1_path = self.walk_path / "tmp1" + tmp2_path = self.sub1_path / "tmp2" + tmp3_path = self.sub2_path / "tmp3" + tmp4_path = self.cls(self.base, "TEST2", "tmp4") + for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path: + with open(path, "w", encoding='utf-8') as f: + f.write(f"I'm {path} and proud of it. Blame test_pathlib.\n") + + if self.can_symlink: + broken_link_path = self.sub2_path / "broken_link" + broken_link2_path = self.sub2_path / "broken_link2" + os.symlink(t2_path, self.link_path, target_is_directory=True) + os.symlink('broken', broken_link_path) + os.symlink(os.path.join('tmp3', 'broken'), broken_link2_path) + self.sub2_tree = (self.sub2_path, [], ["broken_link", "broken_link2", "link", "tmp3"]) + sub21_path= self.sub2_path / "SUB21" + tmp5_path = sub21_path / "tmp3" + broken_link3_path = self.sub2_path / "broken_link3" + + os.makedirs(sub21_path) + tmp5_path.write_text("I am tmp5, blame test_pathlib.") + if self.can_symlink: + os.symlink(tmp5_path, broken_link3_path) + self.sub2_tree[2].append('broken_link3') + self.sub2_tree[2].sort() + os.chmod(sub21_path, 0) + try: + os.listdir(sub21_path) + except PermissionError: + self.sub2_tree[1].append('SUB21') + else: + os.chmod(sub21_path, stat.S_IRWXU) + os.unlink(tmp5_path) + os.rmdir(sub21_path) + + def tearDown(self): + if 'SUB21' in self.sub2_tree[1]: + os.chmod(self.sub2_path / "SUB21", stat.S_IRWXU) + os_helper.rmtree(self.base) + + def test_walk_bad_dir(self): + errors = [] + walk_it = self.walk_path.walk(on_error=errors.append) + root, dirs, files = next(walk_it) + self.assertEqual(errors, []) + dir1 = 'SUB1' + path1 = root / dir1 + path1new = (root / dir1).with_suffix(".new") + path1.rename(path1new) + try: + roots = [r for r, _, _ in walk_it] + self.assertTrue(errors) + self.assertNotIn(path1, roots) + self.assertNotIn(path1new, roots) + for dir2 in dirs: + if dir2 != dir1: + self.assertIn(root / dir2, roots) + finally: + path1new.rename(path1) + + def test_walk_many_open_files(self): + depth = 30 + base = self.cls(self.base, 'deep') + path = self.cls(base, *(['d']*depth)) + path.mkdir(parents=True) + + iters = [base.walk(top_down=False) for _ in range(100)] + for i in range(depth + 1): + expected = (path, ['d'] if i else [], []) + for it in iters: + self.assertEqual(next(it), expected) + path = path.parent + + iters = [base.walk(top_down=True) for _ in range(100)] + path = base + for i in range(depth + 1): + expected = (path, ['d'] if i < depth else [], []) + for it in iters: + self.assertEqual(next(it), expected) + path = path / 'd' + + def test_walk_above_recursion_limit(self): + recursion_limit = 40 + # directory_depth > recursion_limit + directory_depth = recursion_limit + 10 + base = self.cls(self.base, 'deep') + path = base.joinpath(*(['d'] * directory_depth)) + path.mkdir(parents=True) + + with infinite_recursion(recursion_limit): + list(base.walk()) + list(base.walk(top_down=False)) + + @needs_symlinks + def test_walk_follow_symlinks(self): + walk_it = self.walk_path.walk(follow_symlinks=True) + for root, dirs, files in walk_it: + if root == self.link_path: + self.assertEqual(dirs, []) + self.assertEqual(files, ["tmp4"]) + break + else: + self.fail("Didn't follow symlink with follow_symlinks=True") + + @needs_symlinks + def test_walk_symlink_location(self): + # Tests whether symlinks end up in filenames or dirnames depending + # on the `follow_symlinks` argument. + walk_it = self.walk_path.walk(follow_symlinks=False) + for root, dirs, files in walk_it: + if root == self.sub2_path: + self.assertIn("link", files) + break + else: + self.fail("symlink not found") + + walk_it = self.walk_path.walk(follow_symlinks=True) + for root, dirs, files in walk_it: + if root == self.sub2_path: + self.assertIn("link", dirs) + break + else: + self.fail("symlink not found") + + @unittest.skipIf(os.name == 'nt', 'test requires a POSIX-compatible system') class PosixPathTest(PathTest, PurePosixPathTest): cls = pathlib.PosixPath diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index fb467a015a80d2..e67bead4297829 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -2,10 +2,10 @@ import io import os import errno -import stat import unittest -from pathlib._abc import UnsupportedOperation, PathModuleBase, PurePathBase, PathBase +from pathlib._abc import JoinablePath, ReadablePath, WritablePath, magic_open +from pathlib._types import Parser import posixpath from test.support.os_helper import TESTFN @@ -13,7 +13,6 @@ _tests_needing_posix = set() _tests_needing_windows = set() -_tests_needing_symlinks = set() def needs_posix(fn): @@ -26,75 +25,14 @@ def needs_windows(fn): _tests_needing_windows.add(fn.__name__) return fn -def needs_symlinks(fn): - """Decorator that marks a test as requiring a path class that supports symlinks.""" - _tests_needing_symlinks.add(fn.__name__) - return fn - - -class UnsupportedOperationTest(unittest.TestCase): - def test_is_notimplemented(self): - self.assertTrue(issubclass(UnsupportedOperation, NotImplementedError)) - self.assertTrue(isinstance(UnsupportedOperation(), NotImplementedError)) - - -class PathModuleBaseTest(unittest.TestCase): - cls = PathModuleBase - - def test_unsupported_operation(self): - m = self.cls() - e = UnsupportedOperation - with self.assertRaises(e): - m.sep - self.assertRaises(e, m.join, 'foo') - self.assertRaises(e, m.split, 'foo') - self.assertRaises(e, m.splitdrive, 'foo') - self.assertRaises(e, m.normcase, 'foo') - self.assertRaises(e, m.isabs, 'foo') # # Tests for the pure classes. # -class PurePathBaseTest(unittest.TestCase): - cls = PurePathBase - - def test_unsupported_operation_pure(self): - p = self.cls('foo') - e = UnsupportedOperation - with self.assertRaises(e): - p.drive - with self.assertRaises(e): - p.root - with self.assertRaises(e): - p.anchor - with self.assertRaises(e): - p.parts - with self.assertRaises(e): - p.parent - with self.assertRaises(e): - p.parents - with self.assertRaises(e): - p.name - with self.assertRaises(e): - p.stem - with self.assertRaises(e): - p.suffix - with self.assertRaises(e): - p.suffixes - with self.assertRaises(e): - p / 'bar' - with self.assertRaises(e): - 'bar' / p - self.assertRaises(e, p.joinpath, 'bar') - self.assertRaises(e, p.with_name, 'bar') - self.assertRaises(e, p.with_stem, 'bar') - self.assertRaises(e, p.with_suffix, '.txt') - self.assertRaises(e, p.relative_to, '') - self.assertRaises(e, p.is_relative_to, '') - self.assertRaises(e, p.is_absolute) - self.assertRaises(e, p.match, '*') +class JoinablePathTest(unittest.TestCase): + cls = JoinablePath def test_magic_methods(self): P = self.cls @@ -109,16 +47,23 @@ def test_magic_methods(self): self.assertIs(P.__gt__, object.__gt__) self.assertIs(P.__ge__, object.__ge__) - def test_pathmod(self): - self.assertIsInstance(self.cls.pathmod, PathModuleBase) + def test_parser(self): + self.assertIs(self.cls.parser, posixpath) -class DummyPurePath(PurePathBase): - __slots__ = () - pathmod = posixpath +class DummyJoinablePath(JoinablePath): + __slots__ = ('_segments',) + + def __init__(self, *segments): + self._segments = segments + + def __str__(self): + if self._segments: + return self.parser.join(*self._segments) + return '' def __eq__(self, other): - if not isinstance(other, DummyPurePath): + if not isinstance(other, DummyJoinablePath): return NotImplemented return str(self) == str(other) @@ -126,60 +71,42 @@ def __hash__(self): return hash(str(self)) def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + return "{}({!r})".format(self.__class__.__name__, str(self)) + def with_segments(self, *pathsegments): + return type(self)(*pathsegments) -class DummyPurePathTest(unittest.TestCase): - cls = DummyPurePath + +class DummyJoinablePathTest(unittest.TestCase): + cls = DummyJoinablePath # Use a base path that's unrelated to any real filesystem path. base = f'/this/path/kills/fascists/{TESTFN}' def setUp(self): name = self.id().split('.')[-1] - if name in _tests_needing_posix and self.cls.pathmod is not posixpath: + if name in _tests_needing_posix and self.cls.parser is not posixpath: self.skipTest('requires POSIX-flavoured path class') - if name in _tests_needing_windows and self.cls.pathmod is posixpath: + if name in _tests_needing_windows and self.cls.parser is posixpath: self.skipTest('requires Windows-flavoured path class') p = self.cls('a') - self.pathmod = p.pathmod - self.sep = self.pathmod.sep - self.altsep = self.pathmod.altsep + self.parser = p.parser + self.sep = self.parser.sep + self.altsep = self.parser.altsep + + def test_parser(self): + self.assertIsInstance(self.cls.parser, Parser) def test_constructor_common(self): P = self.cls p = P('a') self.assertIsInstance(p, P) + P() P('a', 'b', 'c') P('/a', 'b', 'c') P('a/b/c') P('/a/b/c') - def test_bytes(self): - P = self.cls - with self.assertRaises(TypeError): - P(b'a') - with self.assertRaises(TypeError): - P(b'a', 'b') - with self.assertRaises(TypeError): - P('a', b'b') - with self.assertRaises(TypeError): - P('a').joinpath(b'b') - with self.assertRaises(TypeError): - P('a') / b'b' - with self.assertRaises(TypeError): - b'a' / P('b') - with self.assertRaises(TypeError): - P('a').match(b'b') - with self.assertRaises(TypeError): - P('a').relative_to(b'b') - with self.assertRaises(TypeError): - P('a').with_name(b'b') - with self.assertRaises(TypeError): - P('a').with_stem(b'b') - with self.assertRaises(TypeError): - P('a').with_suffix(b'b') - def _check_str_subclass(self, *args): # Issue #21127: it should be possible to construct a PurePath object # from a str subclass instance, and it then gets converted to @@ -228,7 +155,6 @@ def with_segments(self, *pathsegments): self.assertEqual(42, p.with_stem('foo').session_id) self.assertEqual(42, p.with_suffix('.foo').session_id) self.assertEqual(42, p.with_segments('foo').session_id) - self.assertEqual(42, p.relative_to('foo').session_id) self.assertEqual(42, p.parent.session_id) for parent in p.parents: self.assertEqual(42, parent.session_id) @@ -370,94 +296,6 @@ def test_str_windows(self): p = self.cls('//a/b/c/d') self.assertEqual(str(p), '\\\\a\\b\\c\\d') - def test_as_posix_common(self): - P = self.cls - for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): - self.assertEqual(P(pathstr).as_posix(), pathstr) - # Other tests for as_posix() are in test_equivalences(). - - def test_match_empty(self): - P = self.cls - self.assertRaises(ValueError, P('a').match, '') - - def test_match_common(self): - P = self.cls - # Simple relative pattern. - self.assertTrue(P('b.py').match('b.py')) - self.assertTrue(P('a/b.py').match('b.py')) - self.assertTrue(P('/a/b.py').match('b.py')) - self.assertFalse(P('a.py').match('b.py')) - self.assertFalse(P('b/py').match('b.py')) - self.assertFalse(P('/a.py').match('b.py')) - self.assertFalse(P('b.py/c').match('b.py')) - # Wildcard relative pattern. - self.assertTrue(P('b.py').match('*.py')) - self.assertTrue(P('a/b.py').match('*.py')) - self.assertTrue(P('/a/b.py').match('*.py')) - self.assertFalse(P('b.pyc').match('*.py')) - self.assertFalse(P('b./py').match('*.py')) - self.assertFalse(P('b.py/c').match('*.py')) - # Multi-part relative pattern. - self.assertTrue(P('ab/c.py').match('a*/*.py')) - self.assertTrue(P('/d/ab/c.py').match('a*/*.py')) - self.assertFalse(P('a.py').match('a*/*.py')) - self.assertFalse(P('/dab/c.py').match('a*/*.py')) - self.assertFalse(P('ab/c.py/d').match('a*/*.py')) - # Absolute pattern. - self.assertTrue(P('/b.py').match('/*.py')) - self.assertFalse(P('b.py').match('/*.py')) - self.assertFalse(P('a/b.py').match('/*.py')) - self.assertFalse(P('/a/b.py').match('/*.py')) - # Multi-part absolute pattern. - self.assertTrue(P('/a/b.py').match('/a/*.py')) - self.assertFalse(P('/ab.py').match('/a/*.py')) - self.assertFalse(P('/a/b/c.py').match('/a/*.py')) - # Multi-part glob-style pattern. - self.assertFalse(P('/a/b/c.py').match('/**/*.py')) - self.assertTrue(P('/a/b/c.py').match('/a/**/*.py')) - # Case-sensitive flag - self.assertFalse(P('A.py').match('a.PY', case_sensitive=True)) - self.assertTrue(P('A.py').match('a.PY', case_sensitive=False)) - self.assertFalse(P('c:/a/B.Py').match('C:/A/*.pY', case_sensitive=True)) - self.assertTrue(P('/a/b/c.py').match('/A/*/*.Py', case_sensitive=False)) - # Matching against empty path - self.assertFalse(P('').match('*')) - self.assertFalse(P('').match('**')) - self.assertFalse(P('').match('**/*')) - - @needs_posix - def test_match_posix(self): - P = self.cls - self.assertFalse(P('A.py').match('a.PY')) - - @needs_windows - def test_match_windows(self): - P = self.cls - # Absolute patterns. - self.assertTrue(P('c:/b.py').match('*:/*.py')) - self.assertTrue(P('c:/b.py').match('c:/*.py')) - self.assertFalse(P('d:/b.py').match('c:/*.py')) # wrong drive - self.assertFalse(P('b.py').match('/*.py')) - self.assertFalse(P('b.py').match('c:*.py')) - self.assertFalse(P('b.py').match('c:/*.py')) - self.assertFalse(P('c:b.py').match('/*.py')) - self.assertFalse(P('c:b.py').match('c:/*.py')) - self.assertFalse(P('/b.py').match('c:*.py')) - self.assertFalse(P('/b.py').match('c:/*.py')) - # UNC patterns. - self.assertTrue(P('//some/share/a.py').match('//*/*/*.py')) - self.assertTrue(P('//some/share/a.py').match('//some/share/*.py')) - self.assertFalse(P('//other/share/a.py').match('//some/share/*.py')) - self.assertFalse(P('//some/share/a/b.py').match('//some/share/*.py')) - # Case-insensitivity. - self.assertTrue(P('B.py').match('b.PY')) - self.assertTrue(P('c:/a/B.Py').match('C:/A/*.pY')) - self.assertTrue(P('//Some/Share/B.Py').match('//somE/sharE/*.pY')) - # Path anchor doesn't match pattern anchor - self.assertFalse(P('c:/b.py').match('/*.py')) # 'c:/' vs '/' - self.assertFalse(P('c:/b.py').match('c:*.py')) # 'c:/' vs 'c:' - self.assertFalse(P('//some/share/a.py').match('/*.py')) # '//some/share/' vs '/' - def test_full_match_common(self): P = self.cls # Simple relative pattern. @@ -512,8 +350,6 @@ def test_full_match_common(self): self.assertFalse(P('a/b/c.py').full_match('**/a/b/c./**')) self.assertFalse(P('a/b/c.py').full_match('/a/b/c.py/**')) self.assertFalse(P('a/b/c.py').full_match('/**/a/b/c.py')) - self.assertRaises(ValueError, P('a').full_match, '**a/b/c') - self.assertRaises(ValueError, P('a').full_match, 'a/b/c**') # Case-sensitive flag self.assertFalse(P('A.py').full_match('a.PY', case_sensitive=True)) self.assertTrue(P('A.py').full_match('a.PY', case_sensitive=False)) @@ -684,50 +520,6 @@ def test_parents_windows(self): with self.assertRaises(IndexError): par[2] - def test_drive_common(self): - P = self.cls - self.assertEqual(P('a/b').drive, '') - self.assertEqual(P('/a/b').drive, '') - self.assertEqual(P('').drive, '') - - @needs_windows - def test_drive_windows(self): - P = self.cls - self.assertEqual(P('c:').drive, 'c:') - self.assertEqual(P('c:a/b').drive, 'c:') - self.assertEqual(P('c:/').drive, 'c:') - self.assertEqual(P('c:/a/b/').drive, 'c:') - self.assertEqual(P('//a/b').drive, '\\\\a\\b') - self.assertEqual(P('//a/b/').drive, '\\\\a\\b') - self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') - self.assertEqual(P('./c:a').drive, '') - - def test_root_common(self): - P = self.cls - sep = self.sep - self.assertEqual(P('').root, '') - self.assertEqual(P('a/b').root, '') - self.assertEqual(P('/').root, sep) - self.assertEqual(P('/a/b').root, sep) - - @needs_posix - def test_root_posix(self): - P = self.cls - self.assertEqual(P('/a/b').root, '/') - # POSIX special case for two leading slashes. - self.assertEqual(P('//a/b').root, '//') - - @needs_windows - def test_root_windows(self): - P = self.cls - self.assertEqual(P('c:').root, '') - self.assertEqual(P('c:a/b').root, '') - self.assertEqual(P('c:/').root, '\\') - self.assertEqual(P('c:/a/b/').root, '\\') - self.assertEqual(P('//a/b').root, '\\') - self.assertEqual(P('//a/b/').root, '\\') - self.assertEqual(P('//a/b/c/d').root, '\\') - def test_anchor_common(self): P = self.cls sep = self.sep @@ -790,8 +582,12 @@ def test_suffix_common(self): self.assertEqual(P('/a/.hg.rc').suffix, '.rc') self.assertEqual(P('a/b.tar.gz').suffix, '.gz') self.assertEqual(P('/a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('/a/Some name. Ending with a dot.').suffix, '') + self.assertEqual(P('a/trailing.dot.').suffix, '.') + self.assertEqual(P('/a/trailing.dot.').suffix, '.') + self.assertEqual(P('a/..d.o.t..').suffix, '.') + self.assertEqual(P('a/inn.er..dots').suffix, '.dots') + self.assertEqual(P('photo').suffix, '') + self.assertEqual(P('photo.jpg').suffix, '.jpg') @needs_windows def test_suffix_windows(self): @@ -808,8 +604,8 @@ def test_suffix_windows(self): self.assertEqual(P('c:/a/.hg.rc').suffix, '.rc') self.assertEqual(P('c:a/b.tar.gz').suffix, '.gz') self.assertEqual(P('c:/a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('c:a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffix, '') + self.assertEqual(P('c:a/trailing.dot.').suffix, '.') + self.assertEqual(P('c:/a/trailing.dot.').suffix, '.') self.assertEqual(P('//My.py/Share.php').suffix, '') self.assertEqual(P('//My.py/Share.php/a/b').suffix, '') @@ -829,8 +625,12 @@ def test_suffixes_common(self): self.assertEqual(P('/a/.hg.rc').suffixes, ['.rc']) self.assertEqual(P('a/b.tar.gz').suffixes, ['.tar', '.gz']) self.assertEqual(P('/a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('a/Some name. Ending with a dot.').suffixes, []) - self.assertEqual(P('/a/Some name. Ending with a dot.').suffixes, []) + self.assertEqual(P('a/trailing.dot.').suffixes, ['.dot', '.']) + self.assertEqual(P('/a/trailing.dot.').suffixes, ['.dot', '.']) + self.assertEqual(P('a/..d.o.t..').suffixes, ['.o', '.t', '.', '.']) + self.assertEqual(P('a/inn.er..dots').suffixes, ['.er', '.', '.dots']) + self.assertEqual(P('photo').suffixes, []) + self.assertEqual(P('photo.jpg').suffixes, ['.jpg']) @needs_windows def test_suffixes_windows(self): @@ -849,8 +649,8 @@ def test_suffixes_windows(self): self.assertEqual(P('c:/a/b.tar.gz').suffixes, ['.tar', '.gz']) self.assertEqual(P('//My.py/Share.php').suffixes, []) self.assertEqual(P('//My.py/Share.php/a/b').suffixes, []) - self.assertEqual(P('c:a/Some name. Ending with a dot.').suffixes, []) - self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffixes, []) + self.assertEqual(P('c:a/trailing.dot.').suffixes, ['.dot', '.']) + self.assertEqual(P('c:/a/trailing.dot.').suffixes, ['.dot', '.']) def test_stem_empty(self): P = self.cls @@ -866,8 +666,11 @@ def test_stem_common(self): self.assertEqual(P('a/.hgrc').stem, '.hgrc') self.assertEqual(P('a/.hg.rc').stem, '.hg') self.assertEqual(P('a/b.tar.gz').stem, 'b.tar') - self.assertEqual(P('a/Some name. Ending with a dot.').stem, - 'Some name. Ending with a dot.') + self.assertEqual(P('a/trailing.dot.').stem, 'trailing.dot') + self.assertEqual(P('a/..d.o.t..').stem, '..d.o.t.') + self.assertEqual(P('a/inn.er..dots').stem, 'inn.er.') + self.assertEqual(P('photo').stem, 'photo') + self.assertEqual(P('photo.jpg').stem, 'photo') @needs_windows def test_stem_windows(self): @@ -881,8 +684,8 @@ def test_stem_windows(self): self.assertEqual(P('c:a/.hgrc').stem, '.hgrc') self.assertEqual(P('c:a/.hg.rc').stem, '.hg') self.assertEqual(P('c:a/b.tar.gz').stem, 'b.tar') - self.assertEqual(P('c:a/Some name. Ending with a dot.').stem, - 'Some name. Ending with a dot.') + self.assertEqual(P('c:a/trailing.dot.').stem, 'trailing.dot') + def test_with_name_common(self): P = self.cls self.assertEqual(P('a/b').with_name('d.xml'), P('a/d.xml')) @@ -930,16 +733,16 @@ def test_with_stem_common(self): self.assertEqual(P('a/b.py').with_stem('d'), P('a/d.py')) self.assertEqual(P('/a/b.py').with_stem('d'), P('/a/d.py')) self.assertEqual(P('/a/b.tar.gz').with_stem('d'), P('/a/d.gz')) - self.assertEqual(P('a/Dot ending.').with_stem('d'), P('a/d')) - self.assertEqual(P('/a/Dot ending.').with_stem('d'), P('/a/d')) + self.assertEqual(P('a/Dot ending.').with_stem('d'), P('a/d.')) + self.assertEqual(P('/a/Dot ending.').with_stem('d'), P('/a/d.')) @needs_windows def test_with_stem_windows(self): P = self.cls self.assertEqual(P('c:a/b').with_stem('d'), P('c:a/d')) self.assertEqual(P('c:/a/b').with_stem('d'), P('c:/a/d')) - self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d')) - self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d')) + self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d.')) + self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d.')) self.assertRaises(ValueError, P('c:').with_stem, 'd') self.assertRaises(ValueError, P('c:/').with_stem, 'd') self.assertRaises(ValueError, P('//My/Share').with_stem, 'd') @@ -975,6 +778,11 @@ def test_with_suffix_common(self): # Stripping suffix. self.assertEqual(P('a/b.py').with_suffix(''), P('a/b')) self.assertEqual(P('/a/b').with_suffix(''), P('/a/b')) + # Single dot + self.assertEqual(P('a/b').with_suffix('.'), P('a/b.')) + self.assertEqual(P('/a/b').with_suffix('.'), P('/a/b.')) + self.assertEqual(P('a/b.py').with_suffix('.'), P('a/b.')) + self.assertEqual(P('/a/b.py').with_suffix('.'), P('/a/b.')) @needs_windows def test_with_suffix_windows(self): @@ -1000,6 +808,7 @@ def test_with_suffix_windows(self): self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c\\d') self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c/d') self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c\\d') + self.assertRaises(TypeError, P('c:a/b').with_suffix, None) def test_with_suffix_empty(self): P = self.cls @@ -1007,389 +816,27 @@ def test_with_suffix_empty(self): self.assertRaises(ValueError, P('').with_suffix, '.gz') self.assertRaises(ValueError, P('/').with_suffix, '.gz') - def test_with_suffix_seps(self): + def test_with_suffix_invalid(self): P = self.cls # Invalid suffix. self.assertRaises(ValueError, P('a/b').with_suffix, 'gz') self.assertRaises(ValueError, P('a/b').with_suffix, '/') - self.assertRaises(ValueError, P('a/b').with_suffix, '.') self.assertRaises(ValueError, P('a/b').with_suffix, '/.gz') self.assertRaises(ValueError, P('a/b').with_suffix, 'c/d') self.assertRaises(ValueError, P('a/b').with_suffix, '.c/.d') self.assertRaises(ValueError, P('a/b').with_suffix, './.d') self.assertRaises(ValueError, P('a/b').with_suffix, '.d/.') - - def test_relative_to_common(self): - P = self.cls - p = P('a/b') - self.assertRaises(TypeError, p.relative_to) - self.assertRaises(TypeError, p.relative_to, b'a') - self.assertEqual(p.relative_to(P('')), P('a/b')) - self.assertEqual(p.relative_to(''), P('a/b')) - self.assertEqual(p.relative_to(P('a')), P('b')) - self.assertEqual(p.relative_to('a'), P('b')) - self.assertEqual(p.relative_to('a/'), P('b')) - self.assertEqual(p.relative_to(P('a/b')), P('')) - self.assertEqual(p.relative_to('a/b'), P('')) - self.assertEqual(p.relative_to(P(''), walk_up=True), P('a/b')) - self.assertEqual(p.relative_to('', walk_up=True), P('a/b')) - self.assertEqual(p.relative_to(P('a'), walk_up=True), P('b')) - self.assertEqual(p.relative_to('a', walk_up=True), P('b')) - self.assertEqual(p.relative_to('a/', walk_up=True), P('b')) - self.assertEqual(p.relative_to(P('a/b'), walk_up=True), P('')) - self.assertEqual(p.relative_to('a/b', walk_up=True), P('')) - self.assertEqual(p.relative_to(P('a/c'), walk_up=True), P('../b')) - self.assertEqual(p.relative_to('a/c', walk_up=True), P('../b')) - self.assertEqual(p.relative_to(P('a/b/c'), walk_up=True), P('..')) - self.assertEqual(p.relative_to('a/b/c', walk_up=True), P('..')) - self.assertEqual(p.relative_to(P('c'), walk_up=True), P('../a/b')) - self.assertEqual(p.relative_to('c', walk_up=True), P('../a/b')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('c')) - self.assertRaises(ValueError, p.relative_to, P('a/b/c')) - self.assertRaises(ValueError, p.relative_to, P('a/c')) - self.assertRaises(ValueError, p.relative_to, P('/a')) - self.assertRaises(ValueError, p.relative_to, P("../a")) - self.assertRaises(ValueError, p.relative_to, P("a/..")) - self.assertRaises(ValueError, p.relative_to, P("/a/..")) - self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/a'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) - p = P('/a/b') - self.assertEqual(p.relative_to(P('/')), P('a/b')) - self.assertEqual(p.relative_to('/'), P('a/b')) - self.assertEqual(p.relative_to(P('/a')), P('b')) - self.assertEqual(p.relative_to('/a'), P('b')) - self.assertEqual(p.relative_to('/a/'), P('b')) - self.assertEqual(p.relative_to(P('/a/b')), P('')) - self.assertEqual(p.relative_to('/a/b'), P('')) - self.assertEqual(p.relative_to(P('/'), walk_up=True), P('a/b')) - self.assertEqual(p.relative_to('/', walk_up=True), P('a/b')) - self.assertEqual(p.relative_to(P('/a'), walk_up=True), P('b')) - self.assertEqual(p.relative_to('/a', walk_up=True), P('b')) - self.assertEqual(p.relative_to('/a/', walk_up=True), P('b')) - self.assertEqual(p.relative_to(P('/a/b'), walk_up=True), P('')) - self.assertEqual(p.relative_to('/a/b', walk_up=True), P('')) - self.assertEqual(p.relative_to(P('/a/c'), walk_up=True), P('../b')) - self.assertEqual(p.relative_to('/a/c', walk_up=True), P('../b')) - self.assertEqual(p.relative_to(P('/a/b/c'), walk_up=True), P('..')) - self.assertEqual(p.relative_to('/a/b/c', walk_up=True), P('..')) - self.assertEqual(p.relative_to(P('/c'), walk_up=True), P('../a/b')) - self.assertEqual(p.relative_to('/c', walk_up=True), P('../a/b')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('/c')) - self.assertRaises(ValueError, p.relative_to, P('/a/b/c')) - self.assertRaises(ValueError, p.relative_to, P('/a/c')) - self.assertRaises(ValueError, p.relative_to, P('')) - self.assertRaises(ValueError, p.relative_to, '') - self.assertRaises(ValueError, p.relative_to, P('a')) - self.assertRaises(ValueError, p.relative_to, P("../a")) - self.assertRaises(ValueError, p.relative_to, P("a/..")) - self.assertRaises(ValueError, p.relative_to, P("/a/..")) - self.assertRaises(ValueError, p.relative_to, P(''), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('a'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) - - @needs_windows - def test_relative_to_windows(self): - P = self.cls - p = P('C:Foo/Bar') - self.assertEqual(p.relative_to(P('c:')), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:'), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:foO')), P('Bar')) - self.assertEqual(p.relative_to('c:foO'), P('Bar')) - self.assertEqual(p.relative_to('c:foO/'), P('Bar')) - self.assertEqual(p.relative_to(P('c:foO/baR')), P()) - self.assertEqual(p.relative_to('c:foO/baR'), P()) - self.assertEqual(p.relative_to(P('c:'), walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:foO'), walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:foO', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:foO/', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to(P('c:foO/baR'), walk_up=True), P()) - self.assertEqual(p.relative_to('c:foO/baR', walk_up=True), P()) - self.assertEqual(p.relative_to(P('C:Foo/Bar/Baz'), walk_up=True), P('..')) - self.assertEqual(p.relative_to(P('C:Foo/Baz'), walk_up=True), P('../Bar')) - self.assertEqual(p.relative_to(P('C:Baz/Bar'), walk_up=True), P('../../Foo/Bar')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P()) - self.assertRaises(ValueError, p.relative_to, '') - self.assertRaises(ValueError, p.relative_to, P('d:')) - self.assertRaises(ValueError, p.relative_to, P('/')) - self.assertRaises(ValueError, p.relative_to, P('Foo')) - self.assertRaises(ValueError, p.relative_to, P('/Foo')) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo')) - self.assertRaises(ValueError, p.relative_to, P('C:Foo/Bar/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:Foo/Baz')) - self.assertRaises(ValueError, p.relative_to, P(), walk_up=True) - self.assertRaises(ValueError, p.relative_to, '', walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo'), walk_up=True) - p = P('C:/Foo/Bar') - self.assertEqual(p.relative_to(P('c:/')), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:/'), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:/foO')), P('Bar')) - self.assertEqual(p.relative_to('c:/foO'), P('Bar')) - self.assertEqual(p.relative_to('c:/foO/'), P('Bar')) - self.assertEqual(p.relative_to(P('c:/foO/baR')), P()) - self.assertEqual(p.relative_to('c:/foO/baR'), P()) - self.assertEqual(p.relative_to(P('c:/'), walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:/', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:/foO'), walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:/foO', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('c:/foO/', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to(P('c:/foO/baR'), walk_up=True), P()) - self.assertEqual(p.relative_to('c:/foO/baR', walk_up=True), P()) - self.assertEqual(p.relative_to('C:/Baz', walk_up=True), P('../Foo/Bar')) - self.assertEqual(p.relative_to('C:/Foo/Bar/Baz', walk_up=True), P('..')) - self.assertEqual(p.relative_to('C:/Foo/Baz', walk_up=True), P('../Bar')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, 'c:') - self.assertRaises(ValueError, p.relative_to, P('c:')) - self.assertRaises(ValueError, p.relative_to, P('C:/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Bar/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:Foo')) - self.assertRaises(ValueError, p.relative_to, P('d:')) - self.assertRaises(ValueError, p.relative_to, P('d:/')) - self.assertRaises(ValueError, p.relative_to, P('/')) - self.assertRaises(ValueError, p.relative_to, P('/Foo')) - self.assertRaises(ValueError, p.relative_to, P('//C/Foo')) - self.assertRaises(ValueError, p.relative_to, 'c:', walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('c:'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('C:Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('d:/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('//C/Foo'), walk_up=True) - # UNC paths. - p = P('//Server/Share/Foo/Bar') - self.assertEqual(p.relative_to(P('//sErver/sHare')), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare'), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/'), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo')), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo'), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/'), P('Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar')), P()) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar'), P()) - self.assertEqual(p.relative_to(P('//sErver/sHare'), walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/', walk_up=True), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo'), walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/', walk_up=True), P('Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar'), walk_up=True), P()) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar', walk_up=True), P()) - self.assertEqual(p.relative_to(P('//sErver/sHare/bar'), walk_up=True), P('../Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/bar', walk_up=True), P('../Foo/Bar')) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo')) - self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo')) - self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo')) - self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo')) - self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo'), walk_up=True) - self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo'), walk_up=True) - - def test_is_relative_to_common(self): - P = self.cls - p = P('a/b') - self.assertRaises(TypeError, p.is_relative_to) - self.assertRaises(TypeError, p.is_relative_to, b'a') - self.assertTrue(p.is_relative_to(P(''))) - self.assertTrue(p.is_relative_to('')) - self.assertTrue(p.is_relative_to(P('a'))) - self.assertTrue(p.is_relative_to('a/')) - self.assertTrue(p.is_relative_to(P('a/b'))) - self.assertTrue(p.is_relative_to('a/b')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('c'))) - self.assertFalse(p.is_relative_to(P('a/b/c'))) - self.assertFalse(p.is_relative_to(P('a/c'))) - self.assertFalse(p.is_relative_to(P('/a'))) - p = P('/a/b') - self.assertTrue(p.is_relative_to(P('/'))) - self.assertTrue(p.is_relative_to('/')) - self.assertTrue(p.is_relative_to(P('/a'))) - self.assertTrue(p.is_relative_to('/a')) - self.assertTrue(p.is_relative_to('/a/')) - self.assertTrue(p.is_relative_to(P('/a/b'))) - self.assertTrue(p.is_relative_to('/a/b')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('/c'))) - self.assertFalse(p.is_relative_to(P('/a/b/c'))) - self.assertFalse(p.is_relative_to(P('/a/c'))) - self.assertFalse(p.is_relative_to(P(''))) - self.assertFalse(p.is_relative_to('')) - self.assertFalse(p.is_relative_to(P('a'))) - - @needs_windows - def test_is_relative_to_windows(self): - P = self.cls - p = P('C:Foo/Bar') - self.assertTrue(p.is_relative_to(P('c:'))) - self.assertTrue(p.is_relative_to('c:')) - self.assertTrue(p.is_relative_to(P('c:foO'))) - self.assertTrue(p.is_relative_to('c:foO')) - self.assertTrue(p.is_relative_to('c:foO/')) - self.assertTrue(p.is_relative_to(P('c:foO/baR'))) - self.assertTrue(p.is_relative_to('c:foO/baR')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P())) - self.assertFalse(p.is_relative_to('')) - self.assertFalse(p.is_relative_to(P('d:'))) - self.assertFalse(p.is_relative_to(P('/'))) - self.assertFalse(p.is_relative_to(P('Foo'))) - self.assertFalse(p.is_relative_to(P('/Foo'))) - self.assertFalse(p.is_relative_to(P('C:/Foo'))) - self.assertFalse(p.is_relative_to(P('C:Foo/Bar/Baz'))) - self.assertFalse(p.is_relative_to(P('C:Foo/Baz'))) - p = P('C:/Foo/Bar') - self.assertTrue(p.is_relative_to(P('c:/'))) - self.assertTrue(p.is_relative_to(P('c:/foO'))) - self.assertTrue(p.is_relative_to('c:/foO/')) - self.assertTrue(p.is_relative_to(P('c:/foO/baR'))) - self.assertTrue(p.is_relative_to('c:/foO/baR')) - # Unrelated paths. - self.assertFalse(p.is_relative_to('c:')) - self.assertFalse(p.is_relative_to(P('C:/Baz'))) - self.assertFalse(p.is_relative_to(P('C:/Foo/Bar/Baz'))) - self.assertFalse(p.is_relative_to(P('C:/Foo/Baz'))) - self.assertFalse(p.is_relative_to(P('C:Foo'))) - self.assertFalse(p.is_relative_to(P('d:'))) - self.assertFalse(p.is_relative_to(P('d:/'))) - self.assertFalse(p.is_relative_to(P('/'))) - self.assertFalse(p.is_relative_to(P('/Foo'))) - self.assertFalse(p.is_relative_to(P('//C/Foo'))) - # UNC paths. - p = P('//Server/Share/Foo/Bar') - self.assertTrue(p.is_relative_to(P('//sErver/sHare'))) - self.assertTrue(p.is_relative_to('//sErver/sHare')) - self.assertTrue(p.is_relative_to('//sErver/sHare/')) - self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo'))) - self.assertTrue(p.is_relative_to('//sErver/sHare/Foo')) - self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/')) - self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo/Bar'))) - self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/Bar')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('/Server/Share/Foo'))) - self.assertFalse(p.is_relative_to(P('c:/Server/Share/Foo'))) - self.assertFalse(p.is_relative_to(P('//z/Share/Foo'))) - self.assertFalse(p.is_relative_to(P('//Server/z/Foo'))) - - @needs_posix - def test_is_absolute_posix(self): - P = self.cls - self.assertFalse(P('').is_absolute()) - self.assertFalse(P('a').is_absolute()) - self.assertFalse(P('a/b/').is_absolute()) - self.assertTrue(P('/').is_absolute()) - self.assertTrue(P('/a').is_absolute()) - self.assertTrue(P('/a/b/').is_absolute()) - self.assertTrue(P('//a').is_absolute()) - self.assertTrue(P('//a/b').is_absolute()) - - @needs_windows - def test_is_absolute_windows(self): - P = self.cls - # Under NT, only paths with both a drive and a root are absolute. - self.assertFalse(P().is_absolute()) - self.assertFalse(P('a').is_absolute()) - self.assertFalse(P('a/b/').is_absolute()) - self.assertFalse(P('/').is_absolute()) - self.assertFalse(P('/a').is_absolute()) - self.assertFalse(P('/a/b/').is_absolute()) - self.assertFalse(P('c:').is_absolute()) - self.assertFalse(P('c:a').is_absolute()) - self.assertFalse(P('c:a/b/').is_absolute()) - self.assertTrue(P('c:/').is_absolute()) - self.assertTrue(P('c:/a').is_absolute()) - self.assertTrue(P('c:/a/b/').is_absolute()) - # UNC paths are absolute by definition. - self.assertTrue(P('//').is_absolute()) - self.assertTrue(P('//a').is_absolute()) - self.assertTrue(P('//a/b').is_absolute()) - self.assertTrue(P('//a/b/').is_absolute()) - self.assertTrue(P('//a/b/c').is_absolute()) - self.assertTrue(P('//a/b/c/d').is_absolute()) - self.assertTrue(P('//?/UNC/').is_absolute()) - self.assertTrue(P('//?/UNC/spam').is_absolute()) + self.assertRaises(TypeError, P('a/b').with_suffix, None) # # Tests for the virtual classes. # -class PathBaseTest(PurePathBaseTest): - cls = PathBase - def test_unsupported_operation(self): - P = self.cls - p = self.cls('') - e = UnsupportedOperation - self.assertRaises(e, p.stat) - self.assertRaises(e, p.lstat) - self.assertRaises(e, p.exists) - self.assertRaises(e, p.samefile, 'foo') - self.assertRaises(e, p.is_dir) - self.assertRaises(e, p.is_file) - self.assertRaises(e, p.is_mount) - self.assertRaises(e, p.is_symlink) - self.assertRaises(e, p.is_block_device) - self.assertRaises(e, p.is_char_device) - self.assertRaises(e, p.is_fifo) - self.assertRaises(e, p.is_socket) - self.assertRaises(e, p.open) - self.assertRaises(e, p.read_bytes) - self.assertRaises(e, p.read_text) - self.assertRaises(e, p.write_bytes, b'foo') - self.assertRaises(e, p.write_text, 'foo') - self.assertRaises(e, p.iterdir) - self.assertRaises(e, p.glob, '*') - self.assertRaises(e, p.rglob, '*') - self.assertRaises(e, lambda: list(p.walk())) - self.assertRaises(e, p.absolute) - self.assertRaises(e, P.cwd) - self.assertRaises(e, p.expanduser) - self.assertRaises(e, p.home) - self.assertRaises(e, p.readlink) - self.assertRaises(e, p.symlink_to, 'foo') - self.assertRaises(e, p.hardlink_to, 'foo') - self.assertRaises(e, p.mkdir) - self.assertRaises(e, p.touch) - self.assertRaises(e, p.rename, 'foo') - self.assertRaises(e, p.replace, 'foo') - self.assertRaises(e, p.chmod, 0o755) - self.assertRaises(e, p.lchmod, 0o755) - self.assertRaises(e, p.unlink) - self.assertRaises(e, p.rmdir) - self.assertRaises(e, p.owner) - self.assertRaises(e, p.group) - self.assertRaises(e, p.as_uri) - - def test_as_uri_common(self): - e = UnsupportedOperation - self.assertRaises(e, self.cls('').as_uri) - - def test_fspath_common(self): - self.assertRaises(TypeError, os.fspath, self.cls('')) - - def test_as_bytes_common(self): - self.assertRaises(TypeError, bytes, self.cls('')) - - -class DummyPathIO(io.BytesIO): +class DummyWritablePathIO(io.BytesIO): """ - Used by DummyPath to implement `open('w')` + Used by DummyWritablePath to implement `__open_wb__()` """ def __init__(self, files, path): @@ -1402,89 +849,63 @@ def close(self): super().close() -DummyPathStatResult = collections.namedtuple( - 'DummyPathStatResult', - 'st_mode st_ino st_dev st_nlink st_uid st_gid st_size st_atime st_mtime st_ctime') - - -class DummyPath(PathBase): +class DummyReadablePath(ReadablePath, DummyJoinablePath): """ - Simple implementation of PathBase that keeps files and directories in - memory. + Simple implementation of DummyReadablePath that keeps files and + directories in memory. """ __slots__ = () - pathmod = posixpath _files = {} _directories = {} - _symlinks = {} - def __eq__(self, other): - if not isinstance(other, DummyPath): - return NotImplemented - return str(self) == str(other) + def exists(self, *, follow_symlinks=True): + return self.is_dir() or self.is_file() - def __hash__(self): - return hash(str(self)) + def is_dir(self, *, follow_symlinks=True): + return str(self).rstrip('/') in self._directories - def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + def is_file(self, *, follow_symlinks=True): + return str(self) in self._files - def stat(self, *, follow_symlinks=True): - if follow_symlinks: - path = str(self.resolve()) - else: - path = str(self.parent.resolve() / self.name) - if path in self._files: - st_mode = stat.S_IFREG - elif path in self._directories: - st_mode = stat.S_IFDIR - elif path in self._symlinks: - st_mode = stat.S_IFLNK - else: - raise FileNotFoundError(errno.ENOENT, "Not found", str(self)) - return DummyPathStatResult(st_mode, hash(str(self)), 0, 0, 0, 0, 0, 0, 0, 0) - - def open(self, mode='r', buffering=-1, encoding=None, - errors=None, newline=None): - if buffering != -1: - raise NotImplementedError - path_obj = self.resolve() - path = str(path_obj) - name = path_obj.name - parent = str(path_obj.parent) + def is_symlink(self): + return False + + def __open_rb__(self, buffering=-1): + path = str(self) if path in self._directories: raise IsADirectoryError(errno.EISDIR, "Is a directory", path) - - text = 'b' not in mode - mode = ''.join(c for c in mode if c not in 'btU') - if mode == 'r': - if path not in self._files: - raise FileNotFoundError(errno.ENOENT, "File not found", path) - stream = io.BytesIO(self._files[path]) - elif mode == 'w': - if parent not in self._directories: - raise FileNotFoundError(errno.ENOENT, "File not found", parent) - stream = DummyPathIO(self._files, path) - self._files[path] = b'' - self._directories[parent].add(name) - else: - raise NotImplementedError - if text: - stream = io.TextIOWrapper(stream, encoding=encoding, errors=errors, newline=newline) - return stream + elif path not in self._files: + raise FileNotFoundError(errno.ENOENT, "File not found", path) + return io.BytesIO(self._files[path]) def iterdir(self): - path = str(self.resolve()) + path = str(self).rstrip('/') if path in self._files: raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path) elif path in self._directories: - return (self / name for name in self._directories[path]) + return iter([self / name for name in self._directories[path]]) else: raise FileNotFoundError(errno.ENOENT, "File not found", path) + +class DummyWritablePath(WritablePath, DummyJoinablePath): + __slots__ = () + + def __open_wb__(self, buffering=-1): + path = str(self) + if path in self._directories: + raise IsADirectoryError(errno.EISDIR, "Is a directory", path) + parent, name = posixpath.split(path) + if parent not in self._directories: + raise FileNotFoundError(errno.ENOENT, "File not found", parent) + self._files[path] = b'' + self._directories[parent].add(name) + return DummyWritablePathIO(self._files, path) + def mkdir(self, mode=0o777, parents=False, exist_ok=False): - path = str(self.resolve()) + path = str(self) + parent = str(self.parent) if path in self._directories: if exist_ok: return @@ -1492,19 +913,19 @@ def mkdir(self, mode=0o777, parents=False, exist_ok=False): raise FileExistsError(errno.EEXIST, "File exists", path) try: if self.name: - self._directories[str(self.parent)].add(self.name) + self._directories[parent].add(self.name) self._directories[path] = set() except KeyError: if not parents: - raise FileNotFoundError(errno.ENOENT, "File not found", str(self.parent)) from None + raise FileNotFoundError(errno.ENOENT, "File not found", parent) from None self.parent.mkdir(parents=True, exist_ok=True) self.mkdir(mode, parents=False, exist_ok=exist_ok) -class DummyPathTest(DummyPurePathTest): - """Tests for PathBase methods that use stat(), open() and iterdir().""" +class DummyReadablePathTest(DummyJoinablePathTest): + """Tests for ReadablePathTest methods that use stat(), open() and iterdir().""" - cls = DummyPath + cls = DummyReadablePath can_symlink = False # (self.base) @@ -1529,40 +950,30 @@ class DummyPathTest(DummyPurePathTest): def setUp(self): super().setUp() - name = self.id().split('.')[-1] - if name in _tests_needing_symlinks and not self.can_symlink: - self.skipTest('requires symlinks') - pathmod = self.cls.pathmod - p = self.cls(self.base) - p.mkdir(parents=True) - p.joinpath('dirA').mkdir() - p.joinpath('dirB').mkdir() - p.joinpath('dirC').mkdir() - p.joinpath('dirC', 'dirD').mkdir() - p.joinpath('dirE').mkdir() - with p.joinpath('fileA').open('wb') as f: - f.write(b"this is file A\n") - with p.joinpath('dirB', 'fileB').open('wb') as f: - f.write(b"this is file B\n") - with p.joinpath('dirC', 'fileC').open('wb') as f: - f.write(b"this is file C\n") - with p.joinpath('dirC', 'novel.txt').open('wb') as f: - f.write(b"this is a novel\n") - with p.joinpath('dirC', 'dirD', 'fileD').open('wb') as f: - f.write(b"this is file D\n") - if self.can_symlink: - p.joinpath('linkA').symlink_to('fileA') - p.joinpath('brokenLink').symlink_to('non-existing') - p.joinpath('linkB').symlink_to('dirB') - p.joinpath('dirA', 'linkC').symlink_to(pathmod.join('..', 'dirB')) - p.joinpath('dirB', 'linkD').symlink_to(pathmod.join('..', 'dirB')) - p.joinpath('brokenLinkLoop').symlink_to('brokenLinkLoop') + self.createTestHierarchy() + + def createTestHierarchy(self): + cls = self.cls + cls._files = { + f'{self.base}/fileA': b'this is file A\n', + f'{self.base}/dirB/fileB': b'this is file B\n', + f'{self.base}/dirC/fileC': b'this is file C\n', + f'{self.base}/dirC/dirD/fileD': b'this is file D\n', + f'{self.base}/dirC/novel.txt': b'this is a novel\n', + } + cls._directories = { + f'{self.base}': {'fileA', 'dirA', 'dirB', 'dirC', 'dirE'}, + f'{self.base}/dirA': set(), + f'{self.base}/dirB': {'fileB'}, + f'{self.base}/dirC': {'fileC', 'dirD', 'novel.txt'}, + f'{self.base}/dirC/dirD': {'fileD'}, + f'{self.base}/dirE': set(), + } def tearDown(self): cls = self.cls cls._files.clear() cls._directories.clear() - cls._symlinks.clear() def tempdir(self): path = self.cls(self.base).with_name('tmp-dirD') @@ -1575,30 +986,9 @@ def assertFileNotFound(self, func, *args, **kwargs): self.assertEqual(cm.exception.errno, errno.ENOENT) def assertEqualNormCase(self, path_a, path_b): - normcase = self.pathmod.normcase + normcase = self.parser.normcase self.assertEqual(normcase(path_a), normcase(path_b)) - def test_samefile(self): - pathmod = self.pathmod - fileA_path = pathmod.join(self.base, 'fileA') - fileB_path = pathmod.join(self.base, 'dirB', 'fileB') - p = self.cls(fileA_path) - pp = self.cls(fileA_path) - q = self.cls(fileB_path) - self.assertTrue(p.samefile(fileA_path)) - self.assertTrue(p.samefile(pp)) - self.assertFalse(p.samefile(fileB_path)) - self.assertFalse(p.samefile(q)) - # Test the non-existent file case - non_existent = pathmod.join(self.base, 'foo') - r = self.cls(non_existent) - self.assertRaises(FileNotFoundError, p.samefile, r) - self.assertRaises(FileNotFoundError, p.samefile, non_existent) - self.assertRaises(FileNotFoundError, r.samefile, p) - self.assertRaises(FileNotFoundError, r.samefile, non_existent) - self.assertRaises(FileNotFoundError, r.samefile, r) - self.assertRaises(FileNotFoundError, r.samefile, non_existent) - def test_exists(self): P = self.cls p = P(self.base) @@ -1618,67 +1008,15 @@ def test_exists(self): self.assertIs(False, P(self.base + '\udfff').exists()) self.assertIs(False, P(self.base + '\x00').exists()) - def test_open_common(self): + def test_magic_open(self): p = self.cls(self.base) - with (p / 'fileA').open('r') as f: + with magic_open(p / 'fileA', 'r') as f: self.assertIsInstance(f, io.TextIOBase) self.assertEqual(f.read(), "this is file A\n") - with (p / 'fileA').open('rb') as f: + with magic_open(p / 'fileA', 'rb') as f: self.assertIsInstance(f, io.BufferedIOBase) self.assertEqual(f.read().strip(), b"this is file A") - def test_read_write_bytes(self): - p = self.cls(self.base) - (p / 'fileA').write_bytes(b'abcdefg') - self.assertEqual((p / 'fileA').read_bytes(), b'abcdefg') - # Check that trying to write str does not truncate the file. - self.assertRaises(TypeError, (p / 'fileA').write_bytes, 'somestr') - self.assertEqual((p / 'fileA').read_bytes(), b'abcdefg') - - def test_read_write_text(self): - p = self.cls(self.base) - (p / 'fileA').write_text('äbcdefg', encoding='latin-1') - self.assertEqual((p / 'fileA').read_text( - encoding='utf-8', errors='ignore'), 'bcdefg') - # Check that trying to write bytes does not truncate the file. - self.assertRaises(TypeError, (p / 'fileA').write_text, b'somebytes') - self.assertEqual((p / 'fileA').read_text(encoding='latin-1'), 'äbcdefg') - - def test_read_text_with_newlines(self): - p = self.cls(self.base) - # Check that `\n` character change nothing - (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') - self.assertEqual((p / 'fileA').read_text(newline='\n'), - 'abcde\r\nfghlk\n\rmnopq') - # Check that `\r` character replaces `\n` - (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') - self.assertEqual((p / 'fileA').read_text(newline='\r'), - 'abcde\r\nfghlk\n\rmnopq') - # Check that `\r\n` character replaces `\n` - (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') - self.assertEqual((p / 'fileA').read_text(newline='\r\n'), - 'abcde\r\nfghlk\n\rmnopq') - - def test_write_text_with_newlines(self): - p = self.cls(self.base) - # Check that `\n` character change nothing - (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\n') - self.assertEqual((p / 'fileA').read_bytes(), - b'abcde\r\nfghlk\n\rmnopq') - # Check that `\r` character replaces `\n` - (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\r') - self.assertEqual((p / 'fileA').read_bytes(), - b'abcde\r\rfghlk\r\rmnopq') - # Check that `\r\n` character replaces `\n` - (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\r\n') - self.assertEqual((p / 'fileA').read_bytes(), - b'abcde\r\r\nfghlk\r\n\rmnopq') - # Check that no argument passed will change `\n` to `os.linesep` - os_linesep_byte = bytes(os.linesep, encoding='ascii') - (p / 'fileA').write_text('abcde\nfghlk\n\rmnopq') - self.assertEqual((p / 'fileA').read_bytes(), - b'abcde' + os_linesep_byte + b'fghlk' + os_linesep_byte + b'\rmnopq') - def test_iterdir(self): P = self.cls p = P(self.base) @@ -1689,15 +1027,6 @@ def test_iterdir(self): expected += ['linkA', 'linkB', 'brokenLink', 'brokenLinkLoop'] self.assertEqual(paths, { P(self.base, q) for q in expected }) - @needs_symlinks - def test_iterdir_symlink(self): - # __iter__ on a symlink to a directory. - P = self.cls - p = P(self.base, 'linkB') - paths = set(p.iterdir()) - expected = { P(self.base, 'linkB', q) for q in ['fileB', 'linkD'] } - self.assertEqual(paths, expected) - def test_iterdir_nodir(self): # __iter__ on something that is not a directory. p = self.cls(self.base, 'fileA') @@ -1708,6 +1037,22 @@ def test_iterdir_nodir(self): self.assertIn(cm.exception.errno, (errno.ENOTDIR, errno.ENOENT, errno.EINVAL)) + def test_scandir(self): + p = self.cls(self.base) + with p._scandir() as entries: + self.assertTrue(list(entries)) + with p._scandir() as entries: + for entry in entries: + child = p / entry.name + self.assertIsNotNone(entry) + self.assertEqual(entry.name, child.name) + self.assertEqual(entry.is_symlink(), + child.is_symlink()) + self.assertEqual(entry.is_dir(follow_symlinks=False), + child.is_dir(follow_symlinks=False)) + if entry.name != 'brokenLinkLoop': + self.assertEqual(entry.is_dir(), child.is_dir()) + def test_glob_common(self): def _check(glob, expected): self.assertEqual(set(glob), { P(self.base, q) for q in expected }) @@ -1743,8 +1088,9 @@ def _check(glob, expected): def test_glob_posix(self): P = self.cls p = P(self.base) + q = p / "FILEa" given = set(p.glob("FILEa")) - expect = set() + expect = {q} if q.exists() else set() self.assertEqual(given, expect) self.assertEqual(set(p.glob("FILEa*")), set()) @@ -1755,15 +1101,11 @@ def test_glob_windows(self): self.assertEqual(set(p.glob("FILEa")), { P(self.base, "fileA") }) self.assertEqual(set(p.glob("*a\\")), { P(self.base, "dirA/") }) self.assertEqual(set(p.glob("F*a")), { P(self.base, "fileA") }) - self.assertEqual(set(map(str, p.glob("FILEa"))), {f"{p}\\fileA"}) - self.assertEqual(set(map(str, p.glob("F*a"))), {f"{p}\\fileA"}) def test_glob_empty_pattern(self): P = self.cls p = P(self.base) self.assertEqual(list(p.glob("")), [p]) - self.assertEqual(list(p.glob(".")), [p / "."]) - self.assertEqual(list(p.glob("./")), [p / "./"]) def test_glob_case_sensitive(self): P = self.cls @@ -1777,76 +1119,9 @@ def _check(path, pattern, case_sensitive, expected): _check(path, "dirb/file*", True, []) _check(path, "dirb/file*", False, ["dirB/fileB"]) - @needs_symlinks - def test_glob_follow_symlinks_common(self): + def test_rglob_recurse_symlinks_false(self): def _check(path, glob, expected): - actual = {path for path in path.glob(glob, follow_symlinks=True) - if path.parts.count("linkD") <= 1} # exclude symlink loop. - self.assertEqual(actual, { P(self.base, q) for q in expected }) - P = self.cls - p = P(self.base) - _check(p, "fileB", []) - _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) - _check(p, "*A", ["dirA", "fileA", "linkA"]) - _check(p, "*B/*", ["dirB/fileB", "dirB/linkD", "linkB/fileB", "linkB/linkD"]) - _check(p, "*/fileB", ["dirB/fileB", "linkB/fileB"]) - _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"]) - _check(p, "dir*/*/..", ["dirC/dirD/..", "dirA/linkC/..", "dirB/linkD/.."]) - _check(p, "dir*/**", [ - "dirA/", "dirA/linkC", "dirA/linkC/fileB", "dirA/linkC/linkD", "dirA/linkC/linkD/fileB", - "dirB/", "dirB/fileB", "dirB/linkD", "dirB/linkD/fileB", - "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt", - "dirE/"]) - _check(p, "dir*/**/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", - "dirC/", "dirC/dirD/", "dirE/"]) - _check(p, "dir*/**/..", ["dirA/..", "dirA/linkC/..", "dirB/..", - "dirB/linkD/..", "dirA/linkC/linkD/..", - "dirC/..", "dirC/dirD/..", "dirE/.."]) - _check(p, "dir*/*/**", [ - "dirA/linkC/", "dirA/linkC/linkD", "dirA/linkC/fileB", "dirA/linkC/linkD/fileB", - "dirB/linkD/", "dirB/linkD/fileB", - "dirC/dirD/", "dirC/dirD/fileD"]) - _check(p, "dir*/*/**/", ["dirA/linkC/", "dirA/linkC/linkD/", "dirB/linkD/", "dirC/dirD/"]) - _check(p, "dir*/*/**/..", ["dirA/linkC/..", "dirA/linkC/linkD/..", - "dirB/linkD/..", "dirC/dirD/.."]) - _check(p, "dir*/**/fileC", ["dirC/fileC"]) - _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) - _check(p, "*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) - _check(p, "*/dirD/**/", ["dirC/dirD/"]) - - @needs_symlinks - def test_glob_no_follow_symlinks_common(self): - def _check(path, glob, expected): - actual = {path for path in path.glob(glob, follow_symlinks=False)} - self.assertEqual(actual, { P(self.base, q) for q in expected }) - P = self.cls - p = P(self.base) - _check(p, "fileB", []) - _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) - _check(p, "*A", ["dirA", "fileA", "linkA"]) - _check(p, "*B/*", ["dirB/fileB", "dirB/linkD"]) - _check(p, "*/fileB", ["dirB/fileB"]) - _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/"]) - _check(p, "dir*/*/..", ["dirC/dirD/.."]) - _check(p, "dir*/**", [ - "dirA/", "dirA/linkC", - "dirB/", "dirB/fileB", "dirB/linkD", - "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt", - "dirE/"]) - _check(p, "dir*/**/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) - _check(p, "dir*/**/..", ["dirA/..", "dirB/..", "dirC/..", "dirC/dirD/..", "dirE/.."]) - _check(p, "dir*/*/**", ["dirC/dirD/", "dirC/dirD/fileD"]) - _check(p, "dir*/*/**/", ["dirC/dirD/"]) - _check(p, "dir*/*/**/..", ["dirC/dirD/.."]) - _check(p, "dir*/**/fileC", ["dirC/fileC"]) - _check(p, "dir*/*/../dirD/**", ["dirC/dirD/../dirD/", "dirC/dirD/../dirD/fileD"]) - _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) - _check(p, "*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) - _check(p, "*/dirD/**/", ["dirC/dirD/"]) - - def test_rglob_follow_symlinks_none(self): - def _check(path, glob, expected): - actual = set(path.rglob(glob, follow_symlinks=None)) + actual = set(path.rglob(glob, recurse_symlinks=False)) self.assertEqual(actual, { P(self.base, q) for q in expected }) P = self.cls p = P(self.base) @@ -1889,8 +1164,9 @@ def _check(path, glob, expected): def test_rglob_posix(self): P = self.cls p = P(self.base, "dirC") + q = p / "dirD" / "FILEd" given = set(p.rglob("FILEd")) - expect = set() + expect = {q} if q.exists() else set() self.assertEqual(given, expect) self.assertEqual(set(p.rglob("FILEd*")), set()) @@ -1900,296 +1176,6 @@ def test_rglob_windows(self): p = P(self.base, "dirC") self.assertEqual(set(p.rglob("FILEd")), { P(self.base, "dirC/dirD/fileD") }) self.assertEqual(set(p.rglob("*\\")), { P(self.base, "dirC/dirD/") }) - self.assertEqual(set(map(str, p.rglob("FILEd"))), {f"{p}\\dirD\\fileD"}) - - @needs_symlinks - def test_rglob_follow_symlinks_common(self): - def _check(path, glob, expected): - actual = {path for path in path.rglob(glob, follow_symlinks=True) - if path.parts.count("linkD") <= 1} # exclude symlink loop. - self.assertEqual(actual, { P(self.base, q) for q in expected }) - P = self.cls - p = P(self.base) - _check(p, "fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB", - "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB"]) - _check(p, "*/fileA", []) - _check(p, "*/fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB", - "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB"]) - _check(p, "file*", ["fileA", "dirA/linkC/fileB", "dirB/fileB", - "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB", - "dirC/fileC", "dirC/dirD/fileD", "linkB/fileB"]) - _check(p, "*/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", - "dirC/", "dirC/dirD/", "dirE/", "linkB/", "linkB/linkD/"]) - _check(p, "", ["", "dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", - "dirC/", "dirE/", "dirC/dirD/", "linkB/", "linkB/linkD/"]) - - p = P(self.base, "dirC") - _check(p, "*", ["dirC/fileC", "dirC/novel.txt", - "dirC/dirD", "dirC/dirD/fileD"]) - _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p, "*/*", ["dirC/dirD/fileD"]) - _check(p, "*/", ["dirC/dirD/"]) - _check(p, "", ["dirC/", "dirC/dirD/"]) - # gh-91616, a re module regression - _check(p, "*.txt", ["dirC/novel.txt"]) - _check(p, "*.*", ["dirC/novel.txt"]) - - @needs_symlinks - def test_rglob_no_follow_symlinks_common(self): - def _check(path, glob, expected): - actual = {path for path in path.rglob(glob, follow_symlinks=False)} - self.assertEqual(actual, { P(self.base, q) for q in expected }) - P = self.cls - p = P(self.base) - _check(p, "fileB", ["dirB/fileB"]) - _check(p, "*/fileA", []) - _check(p, "*/fileB", ["dirB/fileB"]) - _check(p, "file*", ["fileA", "dirB/fileB", "dirC/fileC", "dirC/dirD/fileD", ]) - _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) - _check(p, "", ["", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) - - p = P(self.base, "dirC") - _check(p, "*", ["dirC/fileC", "dirC/novel.txt", - "dirC/dirD", "dirC/dirD/fileD"]) - _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p, "*/*", ["dirC/dirD/fileD"]) - _check(p, "*/", ["dirC/dirD/"]) - _check(p, "", ["dirC/", "dirC/dirD/"]) - # gh-91616, a re module regression - _check(p, "*.txt", ["dirC/novel.txt"]) - _check(p, "*.*", ["dirC/novel.txt"]) - - @needs_symlinks - def test_rglob_symlink_loop(self): - # Don't get fooled by symlink loops (Issue #26012). - P = self.cls - p = P(self.base) - given = set(p.rglob('*', follow_symlinks=None)) - expect = {'brokenLink', - 'dirA', 'dirA/linkC', - 'dirB', 'dirB/fileB', 'dirB/linkD', - 'dirC', 'dirC/dirD', 'dirC/dirD/fileD', - 'dirC/fileC', 'dirC/novel.txt', - 'dirE', - 'fileA', - 'linkA', - 'linkB', - 'brokenLinkLoop', - } - self.assertEqual(given, {p / x for x in expect}) - - def test_glob_dotdot(self): - # ".." is not special in globs. - P = self.cls - p = P(self.base) - self.assertEqual(set(p.glob("..")), { P(self.base, "..") }) - self.assertEqual(set(p.glob("../..")), { P(self.base, "..", "..") }) - self.assertEqual(set(p.glob("dirA/..")), { P(self.base, "dirA", "..") }) - self.assertEqual(set(p.glob("dirA/../file*")), { P(self.base, "dirA/../fileA") }) - self.assertEqual(set(p.glob("dirA/../file*/..")), set()) - self.assertEqual(set(p.glob("../xyzzy")), set()) - self.assertEqual(set(p.glob("xyzzy/..")), set()) - self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(self.base, *[".."] * 50)}) - - @needs_symlinks - def test_glob_permissions(self): - # See bpo-38894 - P = self.cls - base = P(self.base) / 'permissions' - base.mkdir() - - for i in range(100): - link = base / f"link{i}" - if i % 2: - link.symlink_to(P(self.base, "dirE", "nonexistent")) - else: - link.symlink_to(P(self.base, "dirC")) - - self.assertEqual(len(set(base.glob("*"))), 100) - self.assertEqual(len(set(base.glob("*/"))), 50) - self.assertEqual(len(set(base.glob("*/fileC"))), 50) - self.assertEqual(len(set(base.glob("*/file*"))), 50) - - @needs_symlinks - def test_glob_long_symlink(self): - # See gh-87695 - base = self.cls(self.base) / 'long_symlink' - base.mkdir() - bad_link = base / 'bad_link' - bad_link.symlink_to("bad" * 200) - self.assertEqual(sorted(base.glob('**/*')), [bad_link]) - - @needs_symlinks - def test_readlink(self): - P = self.cls(self.base) - self.assertEqual((P / 'linkA').readlink(), self.cls('fileA')) - self.assertEqual((P / 'brokenLink').readlink(), - self.cls('non-existing')) - self.assertEqual((P / 'linkB').readlink(), self.cls('dirB')) - self.assertEqual((P / 'linkB' / 'linkD').readlink(), self.cls('../dirB')) - with self.assertRaises(OSError): - (P / 'fileA').readlink() - - @unittest.skipIf(hasattr(os, "readlink"), "os.readlink() is present") - def test_readlink_unsupported(self): - P = self.cls(self.base) - p = P / 'fileA' - with self.assertRaises(UnsupportedOperation): - q.readlink(p) - - def _check_resolve(self, p, expected, strict=True): - q = p.resolve(strict) - self.assertEqual(q, expected) - - # This can be used to check both relative and absolute resolutions. - _check_resolve_relative = _check_resolve_absolute = _check_resolve - - @needs_symlinks - def test_resolve_common(self): - P = self.cls - p = P(self.base, 'foo') - with self.assertRaises(OSError) as cm: - p.resolve(strict=True) - self.assertEqual(cm.exception.errno, errno.ENOENT) - # Non-strict - pathmod = self.pathmod - self.assertEqualNormCase(str(p.resolve(strict=False)), - pathmod.join(self.base, 'foo')) - p = P(self.base, 'foo', 'in', 'spam') - self.assertEqualNormCase(str(p.resolve(strict=False)), - pathmod.join(self.base, 'foo', 'in', 'spam')) - p = P(self.base, '..', 'foo', 'in', 'spam') - self.assertEqualNormCase(str(p.resolve(strict=False)), - pathmod.join(pathmod.dirname(self.base), 'foo', 'in', 'spam')) - # These are all relative symlinks. - p = P(self.base, 'dirB', 'fileB') - self._check_resolve_relative(p, p) - p = P(self.base, 'linkA') - self._check_resolve_relative(p, P(self.base, 'fileA')) - p = P(self.base, 'dirA', 'linkC', 'fileB') - self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) - p = P(self.base, 'dirB', 'linkD', 'fileB') - self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) - # Non-strict - p = P(self.base, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB', 'foo', 'in', - 'spam'), False) - p = P(self.base, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') - if self.cls.pathmod is not posixpath: - # In Windows, if linkY points to dirB, 'dirA\linkY\..' - # resolves to 'dirA' without resolving linkY first. - self._check_resolve_relative(p, P(self.base, 'dirA', 'foo', 'in', - 'spam'), False) - else: - # In Posix, if linkY points to dirB, 'dirA/linkY/..' - # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) - # Now create absolute symlinks. - d = self.tempdir() - P(self.base, 'dirA', 'linkX').symlink_to(d) - P(self.base, str(d), 'linkY').symlink_to(self.pathmod.join(self.base, 'dirB')) - p = P(self.base, 'dirA', 'linkX', 'linkY', 'fileB') - self._check_resolve_absolute(p, P(self.base, 'dirB', 'fileB')) - # Non-strict - p = P(self.base, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(self.base, 'dirB', 'foo', 'in', 'spam'), - False) - p = P(self.base, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') - if self.cls.pathmod is not posixpath: - # In Windows, if linkY points to dirB, 'dirA\linkY\..' - # resolves to 'dirA' without resolving linkY first. - self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) - else: - # In Posix, if linkY points to dirB, 'dirA/linkY/..' - # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) - - @needs_symlinks - def test_resolve_dot(self): - # See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/ - pathmod = self.pathmod - p = self.cls(self.base) - p.joinpath('0').symlink_to('.', target_is_directory=True) - p.joinpath('1').symlink_to(pathmod.join('0', '0'), target_is_directory=True) - p.joinpath('2').symlink_to(pathmod.join('1', '1'), target_is_directory=True) - q = p / '2' - self.assertEqual(q.resolve(strict=True), p) - r = q / '3' / '4' - self.assertRaises(FileNotFoundError, r.resolve, strict=True) - # Non-strict - self.assertEqual(r.resolve(strict=False), p / '3' / '4') - - def _check_symlink_loop(self, *args): - path = self.cls(*args) - with self.assertRaises(OSError) as cm: - path.resolve(strict=True) - self.assertEqual(cm.exception.errno, errno.ELOOP) - - @needs_posix - @needs_symlinks - def test_resolve_loop(self): - # Loops with relative symlinks. - self.cls(self.base, 'linkX').symlink_to('linkX/inside') - self._check_symlink_loop(self.base, 'linkX') - self.cls(self.base, 'linkY').symlink_to('linkY') - self._check_symlink_loop(self.base, 'linkY') - self.cls(self.base, 'linkZ').symlink_to('linkZ/../linkZ') - self._check_symlink_loop(self.base, 'linkZ') - # Non-strict - p = self.cls(self.base, 'linkZ', 'foo') - self.assertEqual(p.resolve(strict=False), p) - # Loops with absolute symlinks. - self.cls(self.base, 'linkU').symlink_to(self.pathmod.join(self.base, 'linkU/inside')) - self._check_symlink_loop(self.base, 'linkU') - self.cls(self.base, 'linkV').symlink_to(self.pathmod.join(self.base, 'linkV')) - self._check_symlink_loop(self.base, 'linkV') - self.cls(self.base, 'linkW').symlink_to(self.pathmod.join(self.base, 'linkW/../linkW')) - self._check_symlink_loop(self.base, 'linkW') - # Non-strict - q = self.cls(self.base, 'linkW', 'foo') - self.assertEqual(q.resolve(strict=False), q) - - def test_stat(self): - statA = self.cls(self.base).joinpath('fileA').stat() - statB = self.cls(self.base).joinpath('dirB', 'fileB').stat() - statC = self.cls(self.base).joinpath('dirC').stat() - # st_mode: files are the same, directory differs. - self.assertIsInstance(statA.st_mode, int) - self.assertEqual(statA.st_mode, statB.st_mode) - self.assertNotEqual(statA.st_mode, statC.st_mode) - self.assertNotEqual(statB.st_mode, statC.st_mode) - # st_ino: all different, - self.assertIsInstance(statA.st_ino, int) - self.assertNotEqual(statA.st_ino, statB.st_ino) - self.assertNotEqual(statA.st_ino, statC.st_ino) - self.assertNotEqual(statB.st_ino, statC.st_ino) - # st_dev: all the same. - self.assertIsInstance(statA.st_dev, int) - self.assertEqual(statA.st_dev, statB.st_dev) - self.assertEqual(statA.st_dev, statC.st_dev) - # other attributes not used by pathlib. - - @needs_symlinks - def test_stat_no_follow_symlinks(self): - p = self.cls(self.base) / 'linkA' - st = p.stat() - self.assertNotEqual(st, p.stat(follow_symlinks=False)) - - def test_stat_no_follow_symlinks_nosymlink(self): - p = self.cls(self.base) / 'fileA' - st = p.stat() - self.assertEqual(st, p.stat(follow_symlinks=False)) - - @needs_symlinks - def test_lstat(self): - p = self.cls(self.base)/ 'linkA' - st = p.stat() - self.assertNotEqual(st, p.lstat()) - - def test_lstat_nosymlink(self): - p = self.cls(self.base) / 'fileA' - st = p.stat() - self.assertEqual(st, p.lstat()) def test_is_dir(self): P = self.cls(self.base) @@ -2243,15 +1229,6 @@ def test_is_file_no_follow_symlinks(self): self.assertFalse((P / 'fileA\udfff').is_file(follow_symlinks=False)) self.assertFalse((P / 'fileA\x00').is_file(follow_symlinks=False)) - def test_is_mount(self): - P = self.cls(self.base) - self.assertFalse((P / 'fileA').is_mount()) - self.assertFalse((P / 'dirA').is_mount()) - self.assertFalse((P / 'non-existing').is_mount()) - self.assertFalse((P / 'fileA' / 'bah').is_mount()) - if self.can_symlink: - self.assertFalse((P / 'linkA').is_mount()) - def test_is_symlink(self): P = self.cls(self.base) self.assertFalse((P / 'fileA').is_symlink()) @@ -2268,155 +1245,264 @@ def test_is_symlink(self): self.assertIs((P / 'linkA\udfff').is_file(), False) self.assertIs((P / 'linkA\x00').is_file(), False) - def test_is_junction_false(self): - P = self.cls(self.base) - self.assertFalse((P / 'fileA').is_junction()) - self.assertFalse((P / 'dirA').is_junction()) - self.assertFalse((P / 'non-existing').is_junction()) - self.assertFalse((P / 'fileA' / 'bah').is_junction()) - self.assertFalse((P / 'fileA\udfff').is_junction()) - self.assertFalse((P / 'fileA\x00').is_junction()) - - def test_is_fifo_false(self): - P = self.cls(self.base) - self.assertFalse((P / 'fileA').is_fifo()) - self.assertFalse((P / 'dirA').is_fifo()) - self.assertFalse((P / 'non-existing').is_fifo()) - self.assertFalse((P / 'fileA' / 'bah').is_fifo()) - self.assertIs((P / 'fileA\udfff').is_fifo(), False) - self.assertIs((P / 'fileA\x00').is_fifo(), False) - - def test_is_socket_false(self): - P = self.cls(self.base) - self.assertFalse((P / 'fileA').is_socket()) - self.assertFalse((P / 'dirA').is_socket()) - self.assertFalse((P / 'non-existing').is_socket()) - self.assertFalse((P / 'fileA' / 'bah').is_socket()) - self.assertIs((P / 'fileA\udfff').is_socket(), False) - self.assertIs((P / 'fileA\x00').is_socket(), False) - - def test_is_block_device_false(self): - P = self.cls(self.base) - self.assertFalse((P / 'fileA').is_block_device()) - self.assertFalse((P / 'dirA').is_block_device()) - self.assertFalse((P / 'non-existing').is_block_device()) - self.assertFalse((P / 'fileA' / 'bah').is_block_device()) - self.assertIs((P / 'fileA\udfff').is_block_device(), False) - self.assertIs((P / 'fileA\x00').is_block_device(), False) - - def test_is_char_device_false(self): - P = self.cls(self.base) - self.assertFalse((P / 'fileA').is_char_device()) - self.assertFalse((P / 'dirA').is_char_device()) - self.assertFalse((P / 'non-existing').is_char_device()) - self.assertFalse((P / 'fileA' / 'bah').is_char_device()) - self.assertIs((P / 'fileA\udfff').is_char_device(), False) - self.assertIs((P / 'fileA\x00').is_char_device(), False) - - def _check_complex_symlinks(self, link0_target): - # Test solving a non-looping chain of symlinks (issue #19887). - pathmod = self.pathmod - P = self.cls(self.base) - P.joinpath('link1').symlink_to(pathmod.join('link0', 'link0'), target_is_directory=True) - P.joinpath('link2').symlink_to(pathmod.join('link1', 'link1'), target_is_directory=True) - P.joinpath('link3').symlink_to(pathmod.join('link2', 'link2'), target_is_directory=True) - P.joinpath('link0').symlink_to(link0_target, target_is_directory=True) - - # Resolve absolute paths. - p = (P / 'link0').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), self.base) - p = (P / 'link1').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), self.base) - p = (P / 'link2').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), self.base) - p = (P / 'link3').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), self.base) - - # Resolve relative paths. - try: - self.cls('').absolute() - except UnsupportedOperation: - return - old_path = os.getcwd() - os.chdir(self.base) - try: - p = self.cls('link0').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), self.base) - p = self.cls('link1').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), self.base) - p = self.cls('link2').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), self.base) - p = self.cls('link3').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), self.base) - finally: - os.chdir(old_path) - - @needs_symlinks - def test_complex_symlinks_absolute(self): - self._check_complex_symlinks(self.base) - - @needs_symlinks - def test_complex_symlinks_relative(self): - self._check_complex_symlinks('.') - - @needs_symlinks - def test_complex_symlinks_relative_dot_dot(self): - self._check_complex_symlinks(self.pathmod.join('dirA', '..')) - - def setUpWalk(self): - # Build: - # TESTFN/ - # TEST1/ a file kid and two directory kids - # tmp1 - # SUB1/ a file kid and a directory kid - # tmp2 - # SUB11/ no kids - # SUB2/ a file kid and a dirsymlink kid - # tmp3 - # link/ a symlink to TEST2 - # broken_link - # broken_link2 - # TEST2/ - # tmp4 a lone file + +class DummyWritablePathTest(DummyJoinablePathTest): + cls = DummyWritablePath + + +class DummyRWPath(DummyWritablePath, DummyReadablePath): + __slots__ = () + + +class DummyRWPathTest(DummyWritablePathTest, DummyReadablePathTest): + cls = DummyRWPath + can_symlink = False + + def test_read_write_bytes(self): + p = self.cls(self.base) + (p / 'fileA').write_bytes(b'abcdefg') + self.assertEqual((p / 'fileA').read_bytes(), b'abcdefg') + # Check that trying to write str does not truncate the file. + self.assertRaises(TypeError, (p / 'fileA').write_bytes, 'somestr') + self.assertEqual((p / 'fileA').read_bytes(), b'abcdefg') + + def test_read_write_text(self): + p = self.cls(self.base) + (p / 'fileA').write_text('äbcdefg', encoding='latin-1') + self.assertEqual((p / 'fileA').read_text( + encoding='utf-8', errors='ignore'), 'bcdefg') + # Check that trying to write bytes does not truncate the file. + self.assertRaises(TypeError, (p / 'fileA').write_text, b'somebytes') + self.assertEqual((p / 'fileA').read_text(encoding='latin-1'), 'äbcdefg') + + def test_read_text_with_newlines(self): + p = self.cls(self.base) + # Check that `\n` character change nothing + (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') + self.assertEqual((p / 'fileA').read_text(newline='\n'), + 'abcde\r\nfghlk\n\rmnopq') + # Check that `\r` character replaces `\n` + (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') + self.assertEqual((p / 'fileA').read_text(newline='\r'), + 'abcde\r\nfghlk\n\rmnopq') + # Check that `\r\n` character replaces `\n` + (p / 'fileA').write_bytes(b'abcde\r\nfghlk\n\rmnopq') + self.assertEqual((p / 'fileA').read_text(newline='\r\n'), + 'abcde\r\nfghlk\n\rmnopq') + + def test_write_text_with_newlines(self): + p = self.cls(self.base) + # Check that `\n` character change nothing + (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\n') + self.assertEqual((p / 'fileA').read_bytes(), + b'abcde\r\nfghlk\n\rmnopq') + # Check that `\r` character replaces `\n` + (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\r') + self.assertEqual((p / 'fileA').read_bytes(), + b'abcde\r\rfghlk\r\rmnopq') + # Check that `\r\n` character replaces `\n` + (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\r\n') + self.assertEqual((p / 'fileA').read_bytes(), + b'abcde\r\r\nfghlk\r\n\rmnopq') + # Check that no argument passed will change `\n` to `os.linesep` + os_linesep_byte = bytes(os.linesep, encoding='ascii') + (p / 'fileA').write_text('abcde\nfghlk\n\rmnopq') + self.assertEqual((p / 'fileA').read_bytes(), + b'abcde' + os_linesep_byte + b'fghlk' + os_linesep_byte + b'\rmnopq') + + def test_copy_file(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'copyA' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertEqual(source.read_text(), target.read_text()) + + def test_copy_file_to_existing_file(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'dirB' / 'fileB' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertEqual(source.read_text(), target.read_text()) + + def test_copy_file_to_existing_directory(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'dirA' + self.assertRaises(OSError, source.copy, target) + + def test_copy_file_empty(self): + base = self.cls(self.base) + source = base / 'empty' + target = base / 'copyA' + source.write_bytes(b'') + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertEqual(target.read_bytes(), b'') + + def test_copy_file_to_itself(self): + base = self.cls(self.base) + source = base / 'empty' + source.write_bytes(b'') + self.assertRaises(OSError, source.copy, source) + self.assertRaises(OSError, source.copy, source, follow_symlinks=False) + + def test_copy_dir_simple(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'copyC' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(target.is_dir()) + self.assertTrue(target.joinpath('dirD').is_dir()) + self.assertTrue(target.joinpath('dirD', 'fileD').is_file()) + self.assertEqual(target.joinpath('dirD', 'fileD').read_text(), + "this is file D\n") + self.assertTrue(target.joinpath('fileC').is_file()) + self.assertTrue(target.joinpath('fileC').read_text(), + "this is file C\n") + + def test_copy_dir_complex(self, follow_symlinks=True): + def ordered_walk(path): + for dirpath, dirnames, filenames in path.walk(follow_symlinks=follow_symlinks): + dirnames.sort() + filenames.sort() + yield dirpath, dirnames, filenames + base = self.cls(self.base) + source = base / 'dirC' + + if self.can_symlink: + # Add some symlinks + source.joinpath('linkC').symlink_to('fileC') + source.joinpath('linkD').symlink_to('dirD', target_is_directory=True) + + # Perform the copy + target = base / 'copyC' + result = source.copy(target, follow_symlinks=follow_symlinks) + self.assertEqual(result, target) + + # Compare the source and target trees + source_walk = ordered_walk(source) + target_walk = ordered_walk(target) + for source_item, target_item in zip(source_walk, target_walk, strict=True): + self.assertEqual(source_item[0].parts[len(source.parts):], + target_item[0].parts[len(target.parts):]) # dirpath + self.assertEqual(source_item[1], target_item[1]) # dirnames + self.assertEqual(source_item[2], target_item[2]) # filenames + # Compare files and symlinks + for filename in source_item[2]: + source_file = source_item[0].joinpath(filename) + target_file = target_item[0].joinpath(filename) + if follow_symlinks or not source_file.is_symlink(): + # Regular file. + self.assertEqual(source_file.read_bytes(), target_file.read_bytes()) + elif source_file.is_dir(): + # Symlink to directory. + self.assertTrue(target_file.is_dir()) + self.assertEqual(source_file.readlink(), target_file.readlink()) + else: + # Symlink to file. + self.assertEqual(source_file.read_bytes(), target_file.read_bytes()) + self.assertEqual(source_file.readlink(), target_file.readlink()) + + def test_copy_dir_complex_follow_symlinks_false(self): + self.test_copy_dir_complex(follow_symlinks=False) + + def test_copy_dir_to_existing_directory(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'copyC' + target.mkdir() + target.joinpath('dirD').mkdir() + self.assertRaises(FileExistsError, source.copy, target) + + def test_copy_dir_to_existing_directory_dirs_exist_ok(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'copyC' + target.mkdir() + target.joinpath('dirD').mkdir() + result = source.copy(target, dirs_exist_ok=True) + self.assertEqual(result, target) + self.assertTrue(target.is_dir()) + self.assertTrue(target.joinpath('dirD').is_dir()) + self.assertTrue(target.joinpath('dirD', 'fileD').is_file()) + self.assertEqual(target.joinpath('dirD', 'fileD').read_text(), + "this is file D\n") + self.assertTrue(target.joinpath('fileC').is_file()) + self.assertTrue(target.joinpath('fileC').read_text(), + "this is file C\n") + + def test_copy_dir_to_itself(self): + base = self.cls(self.base) + source = base / 'dirC' + self.assertRaises(OSError, source.copy, source) + self.assertRaises(OSError, source.copy, source, follow_symlinks=False) + + def test_copy_dir_into_itself(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'dirC' / 'dirD' / 'copyC' + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + self.assertFalse(target.exists()) + + def test_copy_into(self): + base = self.cls(self.base) + source = base / 'fileA' + target_dir = base / 'dirA' + result = source.copy_into(target_dir) + self.assertEqual(result, target_dir / 'fileA') + self.assertTrue(result.exists()) + self.assertEqual(source.read_text(), result.read_text()) + + def test_copy_into_empty_name(self): + source = self.cls('') + target_dir = self.base + self.assertRaises(ValueError, source.copy_into, target_dir) + + +class DummyReadablePathWalkTest(unittest.TestCase): + cls = DummyReadablePath + base = DummyReadablePathTest.base + can_symlink = False + + def setUp(self): self.walk_path = self.cls(self.base, "TEST1") self.sub1_path = self.walk_path / "SUB1" self.sub11_path = self.sub1_path / "SUB11" self.sub2_path = self.walk_path / "SUB2" - tmp1_path = self.walk_path / "tmp1" - tmp2_path = self.sub1_path / "tmp2" - tmp3_path = self.sub2_path / "tmp3" self.link_path = self.sub2_path / "link" - t2_path = self.cls(self.base, "TEST2") - tmp4_path = self.cls(self.base, "TEST2", "tmp4") - broken_link_path = self.sub2_path / "broken_link" - broken_link2_path = self.sub2_path / "broken_link2" + self.sub2_tree = (self.sub2_path, [], ["tmp3"]) + self.createTestHierarchy() - self.sub11_path.mkdir(parents=True) - self.sub2_path.mkdir(parents=True) - t2_path.mkdir(parents=True) - - for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path: - with path.open("w", encoding='utf-8') as f: - f.write(f"I'm {path} and proud of it. Blame test_pathlib.\n") + def createTestHierarchy(self): + cls = self.cls + cls._files = { + f'{self.base}/TEST1/tmp1': b'this is tmp1\n', + f'{self.base}/TEST1/SUB1/tmp2': b'this is tmp2\n', + f'{self.base}/TEST1/SUB2/tmp3': b'this is tmp3\n', + f'{self.base}/TEST2/tmp4': b'this is tmp4\n', + } + cls._directories = { + f'{self.base}': {'TEST1', 'TEST2'}, + f'{self.base}/TEST1': {'SUB1', 'SUB2', 'tmp1'}, + f'{self.base}/TEST1/SUB1': {'SUB11', 'tmp2'}, + f'{self.base}/TEST1/SUB1/SUB11': set(), + f'{self.base}/TEST1/SUB2': {'tmp3'}, + f'{self.base}/TEST2': {'tmp4'}, + } - if self.can_symlink: - self.link_path.symlink_to(t2_path) - broken_link_path.symlink_to('broken') - broken_link2_path.symlink_to(self.cls('tmp3', 'broken')) - self.sub2_tree = (self.sub2_path, [], ["broken_link", "broken_link2", "link", "tmp3"]) - else: - self.sub2_tree = (self.sub2_path, [], ["tmp3"]) + def tearDown(self): + cls = self.cls + cls._files.clear() + cls._directories.clear() def test_walk_topdown(self): - self.setUpWalk() walker = self.walk_path.walk() entry = next(walker) entry[1].sort() # Ensure we visit SUB1 before SUB2 @@ -2433,7 +1519,6 @@ def test_walk_topdown(self): next(walker) def test_walk_prune(self): - self.setUpWalk() # Prune the search. all = [] for root, dirs, files in self.walk_path.walk(): @@ -2450,7 +1535,6 @@ def test_walk_prune(self): self.assertEqual(all[1], self.sub2_tree) def test_walk_bottom_up(self): - self.setUpWalk() seen_testfn = seen_sub1 = seen_sub11 = seen_sub2 = False for path, dirnames, filenames in self.walk_path.walk(top_down=False): if path == self.walk_path: @@ -2483,63 +1567,6 @@ def test_walk_bottom_up(self): raise AssertionError(f"Unexpected path: {path}") self.assertTrue(seen_testfn) - @needs_symlinks - def test_walk_follow_symlinks(self): - self.setUpWalk() - walk_it = self.walk_path.walk(follow_symlinks=True) - for root, dirs, files in walk_it: - if root == self.link_path: - self.assertEqual(dirs, []) - self.assertEqual(files, ["tmp4"]) - break - else: - self.fail("Didn't follow symlink with follow_symlinks=True") - - @needs_symlinks - def test_walk_symlink_location(self): - self.setUpWalk() - # Tests whether symlinks end up in filenames or dirnames depending - # on the `follow_symlinks` argument. - walk_it = self.walk_path.walk(follow_symlinks=False) - for root, dirs, files in walk_it: - if root == self.sub2_path: - self.assertIn("link", files) - break - else: - self.fail("symlink not found") - - walk_it = self.walk_path.walk(follow_symlinks=True) - for root, dirs, files in walk_it: - if root == self.sub2_path: - self.assertIn("link", dirs) - break - else: - self.fail("symlink not found") - - -class DummyPathWithSymlinks(DummyPath): - __slots__ = () - - # Reduce symlink traversal limit to make tests run faster. - _max_symlinks = 20 - - def readlink(self): - path = str(self.parent.resolve() / self.name) - if path in self._symlinks: - return self.with_segments(self._symlinks[path]) - elif path in self._files or path in self._directories: - raise OSError(errno.EINVAL, "Not a symlink", path) - else: - raise FileNotFoundError(errno.ENOENT, "File not found", path) - - def symlink_to(self, target, target_is_directory=False): - self._directories[str(self.parent)].add(self.name) - self._symlinks[str(self)] = str(target) - - -class DummyPathWithSymlinksTest(DummyPathTest): - cls = DummyPathWithSymlinks - can_symlink = True if __name__ == "__main__": diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py index 298e78ccee3875..dae6d898964cff 100644 --- a/Lib/test/test_patma.py +++ b/Lib/test/test_patma.py @@ -1,6 +1,7 @@ import array import collections import dataclasses +import dis import enum import inspect import sys @@ -2957,6 +2958,14 @@ def test_invalid_syntax_3(self): pass """) + def test_len1_tuple_sequence_pattern_comma(self): + # correct syntax would be `case(*x,):` + self.assert_syntax_error(""" + match ...: + case (*x): + pass + """) + def test_mapping_pattern_keys_may_only_match_literals_and_attribute_lookups(self): self.assert_syntax_error(""" match ...: @@ -3006,6 +3015,13 @@ def test_multiple_assignments_to_name_in_pattern_5(self): pass """) + def test_multiple_assignments_to_name_in_pattern_6(self): + self.assert_syntax_error(""" + match ...: + case a as a + 1: # NAME and expression with no () + pass + """) + def test_multiple_starred_names_in_sequence_pattern_0(self): self.assert_syntax_error(""" match ...: @@ -3369,6 +3385,24 @@ class Keys: self.assertIs(y, None) self.assertIs(z, None) +class TestSourceLocations(unittest.TestCase): + def test_jump_threading(self): + # See gh-123048 + def f(): + x = 0 + v = 1 + match v: + case 1: + if x < 0: + x = 1 + case 2: + if x < 0: + x = 1 + x += 1 + + for inst in dis.get_instructions(f): + if inst.opcode in dis.hasjump: + self.assertIsNotNone(inst.positions.lineno, "jump without location") class TestTracing(unittest.TestCase): diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 3dd275caf43200..4d371a6e754b96 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -10,17 +10,17 @@ import subprocess import textwrap import linecache +import zipapp from contextlib import ExitStack, redirect_stdout from io import StringIO from test import support -from test.support import os_helper +from test.support import force_not_colorized, os_helper from test.support.import_helper import import_module from test.support.pty_helper import run_pty, FakeInput from unittest.mock import patch -# gh-114275: WASI fails to run asyncio tests, similar skip than test_asyncio. -SKIP_ASYNCIO_TESTS = (not support.has_socket_support) +SKIP_CORO_TESTS = False class PdbTestInput(object): @@ -45,7 +45,6 @@ def test_pdb_displayhook(): >>> def test_function(foo, bar): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ ... 'foo', @@ -54,8 +53,8 @@ def test_pdb_displayhook(): ... 'continue', ... ]): ... test_function(1, None) - > (3)test_function() - -> pass + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) foo 1 (Pdb) bar @@ -97,6 +96,7 @@ def test_pdb_basic_commands(): ... print(ret) >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'step', # go to line ret = test_function_2('baz') ... 'step', # entering the function call ... 'args', # display function args ... 'list', # list function source @@ -121,6 +121,9 @@ def test_pdb_basic_commands(): ... 'continue', ... ]): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> ret = test_function_2('baz') (Pdb) step @@ -144,7 +147,7 @@ def test_pdb_basic_commands(): [EOF] (Pdb) bt ... - (25)() + (26)() -> test_function() (3)test_function() -> ret = test_function_2('baz') @@ -254,6 +257,8 @@ def test_pdb_breakpoint_commands(): ... 'clear 3', ... 'break', ... 'condition 1', + ... 'commands 1', + ... 'EOF', # Simulate Ctrl-D/Ctrl-Z from user, should end input ... 'enable 1', ... 'clear 1', ... 'commands 2', @@ -274,8 +279,8 @@ def test_pdb_breakpoint_commands(): ... 'continue', ... ]): ... test_function() - > (3)test_function() - -> print(1) + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break 3 Breakpoint 1 at :3 (Pdb) break 4, + @@ -309,6 +314,9 @@ def test_pdb_breakpoint_commands(): 2 breakpoint keep yes at :4 (Pdb) condition 1 Breakpoint 1 is now unconditional. + (Pdb) commands 1 + (com) EOF + (Pdb) enable 1 Enabled breakpoint 1 at :3 (Pdb) clear 1 @@ -354,6 +362,130 @@ def test_pdb_breakpoint_commands(): 4 """ +def test_pdb_breakpoint_on_annotated_function_def(): + """Test breakpoints on function definitions with annotation. + + >>> def foo[T](): + ... return 0 + + >>> def bar() -> int: + ... return 0 + + >>> def foobar[T]() -> int: + ... return 0 + + >>> reset_Breakpoint() + + >>> def test_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... pass + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'break foo', + ... 'break bar', + ... 'break foobar', + ... 'continue', + ... ]): + ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) break foo + Breakpoint 1 at :2 + (Pdb) break bar + Breakpoint 2 at :2 + (Pdb) break foobar + Breakpoint 3 at :2 + (Pdb) continue + """ + +def test_pdb_commands(): + """Test the commands command of pdb. + + >>> def test_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... print(1) + ... print(2) + ... print(3) + + >>> reset_Breakpoint() + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'b 3', + ... 'commands', + ... 'silent', # suppress the frame status output + ... 'p "hello"', + ... 'end', + ... 'b 4', + ... 'commands', + ... 'until 5', # no output, should stop at line 5 + ... 'continue', # hit breakpoint at line 3 + ... '', # repeat continue, hit breakpoint at line 4 then `until` to line 5 + ... '', + ... ]): + ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) b 3 + Breakpoint 1 at :3 + (Pdb) commands + (com) silent + (com) p "hello" + (com) end + (Pdb) b 4 + Breakpoint 2 at :4 + (Pdb) commands + (com) until 5 + (Pdb) continue + 'hello' + (Pdb) + 1 + 2 + > (5)test_function() + -> print(3) + (Pdb) + 3 + """ + +def test_pdb_breakpoint_with_filename(): + """Breakpoints with filename:lineno + + >>> def test_function(): + ... # inspect_fodder2 is a great module as the line number is stable + ... from test.test_inspect import inspect_fodder2 as mod2 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... mod2.func88() + ... mod2.func114() + ... # Be a good citizen and clean up the mess + ... reset_Breakpoint() + + First, need to clear bdb state that might be left over from previous tests. + Otherwise, the new breakpoints might get assigned different numbers. + + >>> reset_Breakpoint() + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS + ... 'break test.test_inspect.inspect_fodder2:90', + ... 'continue', # will stop at func88 + ... 'break test/test_inspect/inspect_fodder2.py:115', + ... 'continue', # will stop at func114 + ... 'continue', + ... ]): + ... test_function() + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) break test.test_inspect.inspect_fodder2:90 + Breakpoint 1 at ...inspect_fodder2.py:90 + (Pdb) continue + > ...inspect_fodder2.py(90)func88() + -> return 90 + (Pdb) break test/test_inspect/inspect_fodder2.py:115 + Breakpoint 2 at ...inspect_fodder2.py:115 + (Pdb) continue + > ...inspect_fodder2.py(115)func114() + -> return 115 + (Pdb) continue + """ + def test_pdb_breakpoints_preserved_across_interactive_sessions(): """Breakpoints are remembered between interactive sessions @@ -421,6 +553,43 @@ def test_pdb_breakpoints_preserved_across_interactive_sessions(): (Pdb) continue """ +def test_pdb_break_anywhere(): + """Test break_anywhere() method of Pdb. + + >>> def outer(): + ... def inner(): + ... import pdb + ... import sys + ... p = pdb.Pdb(nosigint=True, readrc=False) + ... p.set_trace() + ... frame = sys._getframe() + ... print(p.break_anywhere(frame)) # inner + ... print(p.break_anywhere(frame.f_back)) # outer + ... print(p.break_anywhere(frame.f_back.f_back)) # caller + ... inner() + + >>> def caller(): + ... outer() + + >>> def test_function(): + ... caller() + + >>> reset_Breakpoint() + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'b 3', + ... 'c', + ... ]): + ... test_function() + > (6)inner() + -> p.set_trace() + (Pdb) b 3 + Breakpoint 1 at :3 + (Pdb) c + True + False + False + """ + def test_pdb_pp_repr_exc(): """Test that do_p/do_pp do not swallow exceptions. @@ -438,8 +607,7 @@ def test_pdb_pp_repr_exc(): ... 'continue', ... ]): ... test_function() - --Return-- - > (2)test_function()->None + > (2)test_function() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) p obj *** Exception: repr_exc @@ -448,6 +616,37 @@ def test_pdb_pp_repr_exc(): (Pdb) continue """ +def test_pdb_empty_line(): + """Test that empty line repeats the last command. + + >>> def test_function(): + ... x = 1 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... y = 2 + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'p x', + ... '', # Should repeat p x + ... 'n ;; p 0 ;; p x', # Fill cmdqueue with multiple commands + ... '', # Should still repeat p x + ... 'continue', + ... ]): + ... test_function() + > (3)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) p x + 1 + (Pdb) + 1 + (Pdb) n ;; p 0 ;; p x + 0 + 1 + > (4)test_function() + -> y = 2 + (Pdb) + 1 + (Pdb) continue + """ def do_nothing(): pass @@ -479,6 +678,7 @@ def test_list_commands(): ... ret = test_function_2('baz') >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'step', # go to the test function line ... 'list', # list first function ... 'step', # step into second function ... 'list', # list second function @@ -494,6 +694,9 @@ def test_list_commands(): ... 'continue', ... ]): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> ret = test_function_2('baz') (Pdb) list @@ -574,8 +777,7 @@ def test_pdb_whatis_command(): ... 'continue', ... ]): ... test_function() - --Return-- - > (2)test_function()->None + > (2)test_function() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) whatis myvar @@ -604,6 +806,7 @@ def test_pdb_display_command(): ... a = 4 >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 's', ... 'display +', ... 'display', ... 'display a', @@ -619,6 +822,9 @@ def test_pdb_display_command(): ... 'continue', ... ]): ... test_function() + > (3)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) s > (4)test_function() -> a = 1 (Pdb) display + @@ -667,6 +873,7 @@ def test_pdb_alias_command(): ... o.method() >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 's', ... 'alias pi', ... 'alias pi for k in %1.__dict__.keys(): print(f"%1.{k} = {%1.__dict__[k]}")', ... 'alias ps pi self', @@ -685,6 +892,9 @@ def test_pdb_alias_command(): ... 'continue', ... ]): ... test_function() + > (3)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) s > (4)test_function() -> o.method() (Pdb) alias pi @@ -727,7 +937,7 @@ def test_pdb_where_command(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() >>> def f(): - ... g(); + ... g() >>> def test_function(): ... f() @@ -735,50 +945,114 @@ def test_pdb_where_command(): >>> with PdbTestInput([ # doctest: +ELLIPSIS ... 'w', ... 'where', + ... 'w 1', + ... 'w invalid', ... 'u', ... 'w', + ... 'w 0', + ... 'w 100', + ... 'w -100', ... 'continue', ... ]): ... test_function() - --Return-- - > (2)g()->None + > (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) w ... - (8)() + (13)() -> test_function() (2)test_function() -> f() (2)f() - -> g(); - > (2)g()->None + -> g() + > (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) where ... - (8)() + (13)() -> test_function() (2)test_function() -> f() (2)f() - -> g(); - > (2)g()->None + -> g() + > (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) w 1 + > (2)g() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) w invalid + *** Invalid count (invalid) (Pdb) u > (2)f() - -> g(); + -> g() (Pdb) w ... - (8)() + (13)() + -> test_function() + (2)test_function() + -> f() + > (2)f() + -> g() + (2)g() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) w 0 + > (2)f() + -> g() + (Pdb) w 100 + ... + (13)() + -> test_function() + (2)test_function() + -> f() + > (2)f() + -> g() + (2)g() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) w -100 + ... + (13)() -> test_function() (2)test_function() -> f() > (2)f() - -> g(); - (2)g()->None + -> g() + (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) continue """ +def test_pdb_restart_command(): + """Test restart command + + >>> def test_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False, mode='inline').set_trace() + ... x = 1 + + >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 'restart', + ... 'continue', + ... ]): + ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False, mode='inline').set_trace() + (Pdb) restart + *** run/restart command is disabled when pdb is running in inline mode. + Use the command line interface to enable restarting your program + e.g. "python -m pdb myscript.py" + (Pdb) continue + """ + +def test_pdb_commands_with_set_trace(): + """Test that commands can be passed to Pdb.set_trace() + + >>> def test_function(): + ... x = 1 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace(commands=['p x', 'c']) + + >>> test_function() + 1 + """ + # skip this test if sys.flags.no_site = True; # exit() isn't defined unless there's a site module. @@ -810,8 +1084,7 @@ def test_pdb_interact_command(): ... 'continue', ... ]): ... test_function() - --Return-- - > (4)test_function()->None + > (4)test_function() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) interact *pdb interact start* @@ -843,7 +1116,7 @@ def test_convenience_variables(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... try: ... raise Exception('test') - ... except: + ... except Exception: ... pass ... return 1 @@ -851,6 +1124,7 @@ def test_convenience_variables(): ... util_function() >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'step', # Step to try statement ... '$_frame.f_lineno', # Check frame convenience variable ... '$ _frame', # This should be a syntax error ... '$a = 10', # Set a convenience variable @@ -873,6 +1147,9 @@ def test_convenience_variables(): ... 'continue', ... ]): ... test_function() + > (2)util_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)util_function() -> try: (Pdb) $_frame.f_lineno @@ -911,7 +1188,7 @@ def test_convenience_variables(): Exception('test') (Pdb) next > (5)util_function() - -> except: + -> except Exception: (Pdb) $_exception *** KeyError: '_exception' (Pdb) return @@ -1138,7 +1415,7 @@ def test_post_mortem_context_of_the_cause(): def test_post_mortem_from_none(): """Test post mortem traceback debugging of chained exception - In particular that cause from None (which sets __supress_context__ to True) + In particular that cause from None (which sets __suppress_context__ to True) does not show context. @@ -1346,6 +1623,7 @@ def test_post_mortem(): ... print('Not reached.') >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'step', # step to test_function_2() line ... 'next', # step over exception-raising call ... 'bt', # get a backtrace ... 'list', # list code of test_function() @@ -1357,6 +1635,9 @@ def test_post_mortem(): ... test_function() ... except ZeroDivisionError: ... print('Correctly reraised.') + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> test_function_2() (Pdb) next @@ -1366,7 +1647,7 @@ def test_post_mortem(): -> test_function_2() (Pdb) bt ... - (10)() + (11)() -> test_function() > (3)test_function() -> test_function_2() @@ -1393,6 +1674,58 @@ def test_post_mortem(): """ +def test_pdb_return_to_different_file(): + """When pdb returns to a different file, it should not skip if f_trace is + not already set + + >>> import pprint + + >>> class A: + ... def __repr__(self): + ... return 'A' + + >>> def test_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... pprint.pprint(A()) + + >>> reset_Breakpoint() + >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'b A.__repr__', + ... 'continue', + ... 'return', + ... 'next', + ... 'return', + ... 'return', + ... 'continue', + ... ]): + ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) b A.__repr__ + Breakpoint 1 at :3 + (Pdb) continue + > (3)__repr__() + -> return 'A' + (Pdb) return + --Return-- + > (3)__repr__()->'A' + -> return 'A' + (Pdb) next + > ...pprint.py..._safe_repr() + -> return rep,... + (Pdb) return + --Return-- + > ...pprint.py..._safe_repr()->('A'...) + -> return rep,... + (Pdb) return + --Return-- + > ...pprint.py...format()->('A'...) + -> return... + (Pdb) continue + A + """ + + def test_pdb_skip_modules(): """This illustrates the simple case of module skipping. @@ -1403,9 +1736,13 @@ def test_pdb_skip_modules(): >>> with PdbTestInput([ ... 'step', + ... 'step', ... 'continue', ... ]): ... skip_module() + > (3)skip_module() + -> import pdb; pdb.Pdb(skip=['stri*'], nosigint=True, readrc=False).set_trace() + (Pdb) step > (4)skip_module() -> string.capwords('FOO') (Pdb) step @@ -1420,7 +1757,6 @@ def test_pdb_invalid_arg(): >>> def test_function(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ ... 'a = 3', @@ -1429,8 +1765,8 @@ def test_pdb_invalid_arg(): ... 'continue' ... ]): ... test_function() - > (3)test_function() - -> pass + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) a = 3 *** Invalid argument: = 3 Usage: a(rgs) @@ -1464,10 +1800,14 @@ def test_pdb_skip_modules_with_callback(): ... 'step', ... 'step', ... 'step', + ... 'step', ... 'continue', ... ]): ... skip_module() ... pass # provides something to "step" to + > (4)skip_module() + -> import pdb; pdb.Pdb(skip=['module_to_skip*'], nosigint=True, readrc=False).set_trace() + (Pdb) step > (5)skip_module() -> mod.foo_pony(callback) (Pdb) step @@ -1486,7 +1826,7 @@ def test_pdb_skip_modules_with_callback(): > (5)skip_module()->None -> mod.foo_pony(callback) (Pdb) step - > (10)() + > (11)() -> pass # provides something to "step" to (Pdb) continue """ @@ -1505,6 +1845,7 @@ def test_pdb_continue_in_bottomframe(): ... print(4) >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 'step', ... 'next', ... 'break 7', ... 'continue', @@ -1513,6 +1854,9 @@ def test_pdb_continue_in_bottomframe(): ... 'continue', ... ]): ... test_function() + > (3)test_function() + -> inst.set_trace() + (Pdb) step > (4)test_function() -> inst.botframe = sys._getframe() # hackery to get the right botframe (Pdb) next @@ -1604,8 +1948,8 @@ def test_next_until_return_at_return_event(): ... 'return', ... 'continue']): ... test_function() - > (3)test_function() - -> test_function_2() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break test_function_2 Breakpoint 1 at :2 (Pdb) continue @@ -1642,7 +1986,7 @@ def test_next_until_return_at_return_event(): """ def test_pdb_next_command_for_generator(): - """Testing skip unwindng stack on yield for generators for "next" command + """Testing skip unwinding stack on yield for generators for "next" command >>> def test_gen(): ... yield 0 @@ -1664,12 +2008,16 @@ def test_pdb_next_command_for_generator(): >>> with PdbTestInput(['step', ... 'step', ... 'step', + ... 'step', ... 'next', ... 'next', ... 'step', ... 'step', ... 'continue']): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> it = test_gen() (Pdb) step @@ -1700,36 +2048,37 @@ def test_pdb_next_command_for_generator(): finished """ -if not SKIP_ASYNCIO_TESTS: +if not SKIP_CORO_TESTS: def test_pdb_next_command_for_coroutine(): - """Testing skip unwindng stack on yield for coroutines for "next" command + """Testing skip unwinding stack on yield for coroutines for "next" command - >>> import asyncio + >>> from test.support import run_yielding_async_fn, async_yield >>> async def test_coro(): - ... await asyncio.sleep(0) - ... await asyncio.sleep(0) - ... await asyncio.sleep(0) + ... await async_yield(0) + ... await async_yield(0) + ... await async_yield(0) >>> async def test_main(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... await test_coro() >>> def test_function(): - ... loop = asyncio.new_event_loop() - ... loop.run_until_complete(test_main()) - ... loop.close() - ... asyncio.set_event_loop_policy(None) + ... run_yielding_async_fn(test_main) ... print("finished") >>> with PdbTestInput(['step', ... 'step', + ... 'step', ... 'next', ... 'next', ... 'next', ... 'step', ... 'continue']): ... test_function() + > (2)test_main() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_main() -> await test_coro() (Pdb) step @@ -1738,13 +2087,13 @@ def test_pdb_next_command_for_coroutine(): -> async def test_coro(): (Pdb) step > (2)test_coro() - -> await asyncio.sleep(0) + -> await async_yield(0) (Pdb) next > (3)test_coro() - -> await asyncio.sleep(0) + -> await async_yield(0) (Pdb) next > (4)test_coro() - -> await asyncio.sleep(0) + -> await async_yield(0) (Pdb) next Internal StopIteration > (3)test_main() @@ -1758,13 +2107,13 @@ def test_pdb_next_command_for_coroutine(): """ def test_pdb_next_command_for_asyncgen(): - """Testing skip unwindng stack on yield for coroutines for "next" command + """Testing skip unwinding stack on yield for coroutines for "next" command - >>> import asyncio + >>> from test.support import run_yielding_async_fn, async_yield >>> async def agen(): ... yield 1 - ... await asyncio.sleep(0) + ... await async_yield(0) ... yield 2 >>> async def test_coro(): @@ -1776,20 +2125,21 @@ def test_pdb_next_command_for_asyncgen(): ... await test_coro() >>> def test_function(): - ... loop = asyncio.new_event_loop() - ... loop.run_until_complete(test_main()) - ... loop.close() - ... asyncio.set_event_loop_policy(None) + ... run_yielding_async_fn(test_main) ... print("finished") >>> with PdbTestInput(['step', ... 'step', + ... 'step', ... 'next', ... 'next', ... 'step', ... 'next', ... 'continue']): ... test_function() + > (2)test_main() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_main() -> await test_coro() (Pdb) step @@ -1812,14 +2162,14 @@ def test_pdb_next_command_for_asyncgen(): -> yield 1 (Pdb) next > (3)agen() - -> await asyncio.sleep(0) + -> await async_yield(0) (Pdb) continue 2 finished """ def test_pdb_return_command_for_generator(): - """Testing no unwindng stack on yield for generators + """Testing no unwinding stack on yield for generators for "return" command >>> def test_gen(): @@ -1842,11 +2192,15 @@ def test_pdb_return_command_for_generator(): >>> with PdbTestInput(['step', ... 'step', ... 'step', + ... 'step', ... 'return', ... 'step', ... 'step', ... 'continue']): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> it = test_gen() (Pdb) step @@ -1873,33 +2227,34 @@ def test_pdb_return_command_for_generator(): finished """ -if not SKIP_ASYNCIO_TESTS: +if not SKIP_CORO_TESTS: def test_pdb_return_command_for_coroutine(): - """Testing no unwindng stack on yield for coroutines for "return" command + """Testing no unwinding stack on yield for coroutines for "return" command - >>> import asyncio + >>> from test.support import run_yielding_async_fn, async_yield >>> async def test_coro(): - ... await asyncio.sleep(0) - ... await asyncio.sleep(0) - ... await asyncio.sleep(0) + ... await async_yield(0) + ... await async_yield(0) + ... await async_yield(0) >>> async def test_main(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... await test_coro() >>> def test_function(): - ... loop = asyncio.new_event_loop() - ... loop.run_until_complete(test_main()) - ... loop.close() - ... asyncio.set_event_loop_policy(None) + ... run_yielding_async_fn(test_main) ... print("finished") >>> with PdbTestInput(['step', ... 'step', + ... 'step', ... 'next', ... 'continue']): ... test_function() + > (2)test_main() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_main() -> await test_coro() (Pdb) step @@ -1908,16 +2263,16 @@ def test_pdb_return_command_for_coroutine(): -> async def test_coro(): (Pdb) step > (2)test_coro() - -> await asyncio.sleep(0) + -> await async_yield(0) (Pdb) next > (3)test_coro() - -> await asyncio.sleep(0) + -> await async_yield(0) (Pdb) continue finished """ def test_pdb_until_command_for_generator(): - """Testing no unwindng stack on yield for generators + """Testing no unwinding stack on yield for generators for "until" command if target breakpoint is not reached >>> def test_gen(): @@ -1932,11 +2287,15 @@ def test_pdb_until_command_for_generator(): ... print("finished") >>> with PdbTestInput(['step', + ... 'step', ... 'until 4', ... 'step', ... 'step', ... 'continue']): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> for i in test_gen(): (Pdb) step @@ -1960,20 +2319,20 @@ def test_pdb_until_command_for_generator(): finished """ -if not SKIP_ASYNCIO_TESTS: +if not SKIP_CORO_TESTS: def test_pdb_until_command_for_coroutine(): - """Testing no unwindng stack for coroutines + """Testing no unwinding stack for coroutines for "until" command if target breakpoint is not reached - >>> import asyncio + >>> from test.support import run_yielding_async_fn, async_yield >>> async def test_coro(): ... print(0) - ... await asyncio.sleep(0) + ... await async_yield(0) ... print(1) - ... await asyncio.sleep(0) + ... await async_yield(0) ... print(2) - ... await asyncio.sleep(0) + ... await async_yield(0) ... print(3) >>> async def test_main(): @@ -1981,16 +2340,17 @@ def test_pdb_until_command_for_coroutine(): ... await test_coro() >>> def test_function(): - ... loop = asyncio.new_event_loop() - ... loop.run_until_complete(test_main()) - ... loop.close() - ... asyncio.set_event_loop_policy(None) + ... run_yielding_async_fn(test_main) ... print("finished") >>> with PdbTestInput(['step', + ... 'step', ... 'until 8', ... 'continue']): ... test_function() + > (2)test_main() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_main() -> await test_coro() (Pdb) step @@ -2029,8 +2389,8 @@ def test_pdb_next_command_in_generator_for_loop(): ... 'next', ... 'continue']): ... test_function() - > (3)test_function() - -> for i in test_gen(): + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break test_gen Breakpoint 1 at :2 (Pdb) continue @@ -2069,11 +2429,15 @@ def test_pdb_next_command_subiterator(): >>> with PdbTestInput(['step', ... 'step', + ... 'step', ... 'next', ... 'next', ... 'next', ... 'continue']): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> for i in test_gen(): (Pdb) step @@ -2102,34 +2466,105 @@ def test_pdb_multiline_statement(): >>> def test_function(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'def f(x):', ... ' return x * 2', ... '', - ... 'f(2)', + ... 'val = 2', + ... 'if val > 0:', + ... ' val = f(val)', + ... '', + ... '', # empty line should repeat the multi-line statement + ... 'val', ... 'c' ... ]): ... test_function() - > (3)test_function() - -> pass + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) def f(x): ... return x * 2 ... - (Pdb) f(2) - 4 + (Pdb) val = 2 + (Pdb) if val > 0: + ... val = f(val) + ... + (Pdb) + (Pdb) val + 8 + (Pdb) c + """ + +def test_pdb_closure(): + """Test for all expressions/statements that involve closure + + >>> k = 0 + >>> g = 1 + >>> def test_function(): + ... x = 2 + ... g = 3 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'k', + ... 'g', + ... 'y = y', + ... 'global g; g', + ... 'global g; (lambda: g)()', + ... '(lambda: x)()', + ... '(lambda: g)()', + ... 'lst = [n for n in range(10) if (n % x) == 0]', + ... 'lst', + ... 'sum(n for n in lst if n > x)', + ... 'x = 1; raise Exception()', + ... 'x', + ... 'def f():', + ... ' return x', + ... '', + ... 'f()', + ... 'c' + ... ]): + ... test_function() + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) k + 0 + (Pdb) g + 3 + (Pdb) y = y + *** NameError: name 'y' is not defined + (Pdb) global g; g + 1 + (Pdb) global g; (lambda: g)() + 1 + (Pdb) (lambda: x)() + 2 + (Pdb) (lambda: g)() + 3 + (Pdb) lst = [n for n in range(10) if (n % x) == 0] + (Pdb) lst + [0, 2, 4, 6, 8] + (Pdb) sum(n for n in lst if n > x) + 18 + (Pdb) x = 1; raise Exception() + *** Exception + (Pdb) x + 1 + (Pdb) def f(): + ... return x + ... + (Pdb) f() + 1 (Pdb) c """ def test_pdb_show_attribute_and_item(): - """Test for multiline statement + """Test for expressions with command prefix >>> def test_function(): ... n = lambda x: x ... c = {"a": 1} ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'c["a"]', @@ -2143,8 +2578,8 @@ def test_pdb_show_attribute_and_item(): ... 'c' ... ]): ... test_function() - > (5)test_function() - -> pass + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) c["a"] 1 (Pdb) c.get("a") @@ -2163,13 +2598,56 @@ def test_pdb_show_attribute_and_item(): (Pdb) c """ -def test_pdb_issue_20766(): - """Test for reference leaks when the SIGINT handler is set. +# doctest will modify pdb.set_trace during the test, so we need to backup +# the original function to use it in the test +original_pdb_settrace = pdb.set_trace + +def test_pdb_with_inline_breakpoint(): + """Hard-coded breakpoint() calls should invoke the same debugger instance >>> def test_function(): - ... i = 1 - ... while i <= 2: - ... sess = pdb.Pdb() + ... x = 1 + ... import pdb; pdb.Pdb().set_trace() + ... original_pdb_settrace() + ... x = 2 + + >>> with PdbTestInput(['display x', + ... 'n', + ... 'n', + ... 'n', + ... 'n', + ... 'undisplay', + ... 'c']): + ... test_function() + > (3)test_function() + -> import pdb; pdb.Pdb().set_trace() + (Pdb) display x + display x: 1 + (Pdb) n + > (4)test_function() + -> original_pdb_settrace() + (Pdb) n + > (4)test_function() + -> original_pdb_settrace() + (Pdb) n + > (5)test_function() + -> x = 2 + (Pdb) n + --Return-- + > (5)test_function()->None + -> x = 2 + display x: 2 [old: 1] + (Pdb) undisplay + (Pdb) c + """ + +def test_pdb_issue_20766(): + """Test for reference leaks when the SIGINT handler is set. + + >>> def test_function(): + ... i = 1 + ... while i <= 2: + ... sess = pdb.Pdb() ... sess.set_trace(sys._getframe()) ... print('pdb %d: %s' % (i, sess._previous_sigint_handler)) ... i += 1 @@ -2178,12 +2656,12 @@ def test_pdb_issue_20766(): >>> with PdbTestInput(['continue', ... 'continue']): ... test_function() - > (6)test_function() - -> print('pdb %d: %s' % (i, sess._previous_sigint_handler)) + > (5)test_function() + -> sess.set_trace(sys._getframe()) (Pdb) continue pdb 1: - > (6)test_function() - -> print('pdb %d: %s' % (i, sess._previous_sigint_handler)) + > (5)test_function() + -> sess.set_trace(sys._getframe()) (Pdb) continue pdb 2: """ @@ -2204,8 +2682,8 @@ def test_pdb_issue_43318(): ... 'continue' ... ]): ... test_function() - > (3)test_function() - -> print(1) + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break 3 Breakpoint 1 at :3 (Pdb) clear :3 @@ -2237,12 +2715,16 @@ def test_pdb_issue_gh_91742(): >>> reset_Breakpoint() >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'step', + ... 'step', ... 'next', ... 'next', ... 'jump 5', ... 'continue' ... ]): ... test_function() + > (11)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (12)test_function() -> about() (Pdb) step @@ -2280,6 +2762,7 @@ def test_pdb_issue_gh_94215(): >>> reset_Breakpoint() >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'step', + ... 'step', ... 'next', ... 'next', ... 'jump 3', @@ -2292,6 +2775,9 @@ def test_pdb_issue_gh_94215(): ... 'continue' ... ]): ... test_function() + > (8)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (9)test_function() -> func() (Pdb) step @@ -2331,7 +2817,7 @@ def test_pdb_issue_gh_94215(): def test_pdb_issue_gh_101673(): """See GH-101673 - Make sure ll won't revert local variable assignment + Make sure ll and switching frames won't revert local variable assignment >>> def test_function(): ... a = 1 @@ -2341,11 +2827,14 @@ def test_pdb_issue_gh_101673(): ... '!a = 2', ... 'll', ... 'p a', + ... 'u', + ... 'p a', + ... 'd', + ... 'p a', ... 'continue' ... ]): ... test_function() - --Return-- - > (3)test_function()->None + > (3)test_function() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) !a = 2 (Pdb) ll @@ -2354,6 +2843,16 @@ def test_pdb_issue_gh_101673(): 3 -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) p a 2 + (Pdb) u + > (11)() + -> test_function() + (Pdb) p a + *** NameError: name 'a' is not defined + (Pdb) d + > (3)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) p a + 2 (Pdb) continue """ @@ -2369,16 +2868,16 @@ def test_pdb_issue_gh_103225(): ... a = 1 ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... b = 2 - > (7)() - -> b = 2 + > (6)() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) longlist 1 with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE 2 'longlist', 3 'continue' 4 ]): 5 a = 1 - 6 import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - 7 -> b = 2 + 6 -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + 7 b = 2 (Pdb) continue """ @@ -2397,9 +2896,8 @@ def test_pdb_issue_gh_101517(): ... 'continue' ... ]): ... test_function() - --Return-- - > (None)test_function()->None - -> Warning: lineno is None + > (5)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) continue """ @@ -2415,8 +2913,23 @@ def test_pdb_issue_gh_108976(): ... 'continue' ... ]): ... test_function() - bdb.Bdb.dispatch: unknown debugging event: 'opcode' - > (5)test_function() + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) continue + """ + +def test_pdb_issue_gh_127321(): + """See GH-127321 + breakpoint() should stop at a opcode that has a line number + >>> def test_function(): + ... import pdb; pdb_instance = pdb.Pdb(nosigint=True, readrc=False) + ... [1, 2] and pdb_instance.set_trace() + ... a = 1 + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'continue' + ... ]): + ... test_function() + > (4)test_function() -> a = 1 (Pdb) continue """ @@ -2436,9 +2949,8 @@ def test_pdb_issue_gh_80731(): ... raise ValueError('Correct') ... except ValueError: ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass - > (10)() - -> pass + > (9)() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) import sys (Pdb) sys.exc_info() (, ValueError('Correct'), ) @@ -2452,6 +2964,7 @@ def test_pdb_ambiguous_statements(): Make sure that ambiguous statements prefixed by '!' are properly disambiguated >>> with PdbTestInput([ + ... 's', # step to the print line ... '! n = 42', # disambiguated statement: reassign the name n ... 'n', # advance the debugger into the print() ... 'continue' @@ -2460,6 +2973,9 @@ def test_pdb_ambiguous_statements(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... print(f"The value of n is {n}") > (8)() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) s + > (9)() -> print(f"The value of n is {n}") (Pdb) ! n = 42 (Pdb) n @@ -2488,9 +3004,60 @@ def test_pdb_f_trace_lines(): ... 'continue' ... ]): ... test_function() - > (6)test_function() - -> if frame.f_trace_lines != False: + > (5)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) continue + """ + +def test_pdb_frame_refleak(): + """ + pdb should not leak reference to frames + + >>> def frame_leaker(container): + ... import sys + ... container.append(sys._getframe()) + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... pass + + >>> def test_function(): + ... import gc + ... container = [] + ... frame_leaker(container) # c + ... print(len(gc.get_referrers(container[0]))) + ... container = [] + ... frame_leaker(container) # n c + ... print(len(gc.get_referrers(container[0]))) + ... container = [] + ... frame_leaker(container) # r c + ... print(len(gc.get_referrers(container[0]))) + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'continue', + ... 'next', + ... 'continue', + ... 'return', + ... 'continue', + ... ]): + ... test_function() + > (4)frame_leaker() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) continue + 1 + > (4)frame_leaker() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) next + > (5)frame_leaker() + -> pass + (Pdb) continue + 1 + > (4)frame_leaker() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) return + --Return-- + > (5)frame_leaker()->None + -> pass + (Pdb) continue + 1 """ def test_pdb_function_break(): @@ -2512,7 +3079,6 @@ def test_pdb_function_break(): >>> def test_function(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ... 'break foo', @@ -2522,8 +3088,8 @@ def test_pdb_function_break(): ... 'continue' ... ]): ... test_function() - > (3)test_function() - -> pass + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break foo Breakpoint ... at :1 (Pdb) break bar @@ -2553,6 +3119,7 @@ def test_pdb_issue_gh_65052(): ... A() >>> with PdbTestInput([ # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ... 's', + ... 's', ... 'retval', ... 'continue', ... 'args', @@ -2561,6 +3128,9 @@ def test_pdb_issue_gh_65052(): ... 'continue', ... ]): ... test_function() + > (3)__new__() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) s > (4)__new__() -> return object.__new__(cls) (Pdb) s @@ -2570,8 +3140,8 @@ def test_pdb_issue_gh_65052(): (Pdb) retval *** repr(retval) failed: AttributeError: 'A' object has no attribute 'a' *** (Pdb) continue - > (7)__init__() - -> self.a = 1 + > (6)__init__() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) args self = *** repr(self) failed: AttributeError: 'A' object has no attribute 'a' *** (Pdb) display self @@ -2603,12 +3173,12 @@ def _run_pdb(self, pdb_args, commands, cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.PIPE, env = {**env, 'PYTHONIOENCODING': 'utf-8'} ) as proc: stdout, stderr = proc.communicate(str.encode(commands)) - stdout = stdout and bytes.decode(stdout) - stderr = stderr and bytes.decode(stderr) + stdout = bytes.decode(stdout) if isinstance(stdout, bytes) else stdout + stderr = bytes.decode(stderr) if isinstance(stderr, bytes) else stderr self.assertEqual( proc.returncode, expected_returncode, @@ -2618,13 +3188,28 @@ def _run_pdb(self, pdb_args, commands, def run_pdb_script(self, script, commands, expected_returncode=0, - extra_env=None): + extra_env=None, + script_args=None, + pdbrc=None, + remove_home=False): """Run 'script' lines with pdb and the pdb 'commands'.""" filename = 'main.py' with open(filename, 'w') as f: f.write(textwrap.dedent(script)) + + if pdbrc is not None: + with open('.pdbrc', 'w') as f: + f.write(textwrap.dedent(pdbrc)) + self.addCleanup(os_helper.unlink, '.pdbrc') self.addCleanup(os_helper.unlink, filename) - return self._run_pdb([filename], commands, expected_returncode, extra_env) + + with os_helper.EnvironmentVarGuard() as env: + if remove_home: + env.unset('HOME') + if script_args is None: + script_args = [] + stdout, stderr = self._run_pdb([filename] + script_args, commands, expected_returncode, extra_env) + return stdout, stderr def run_pdb_module(self, script, commands): """Runs the script code as part of a module""" @@ -2740,7 +3325,7 @@ def test_issue7964(self): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.PIPE, ) self.addCleanup(proc.stdout.close) stdout, stderr = proc.communicate(b'quit\n') @@ -2824,7 +3409,7 @@ def start_pdb(): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.PIPE, env={**os.environ, 'PYTHONIOENCODING': 'utf-8'} ) self.addCleanup(proc.stdout.close) @@ -2854,7 +3439,7 @@ def start_pdb(): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.PIPE, env = {**os.environ, 'PYTHONIOENCODING': 'utf-8'} ) self.addCleanup(proc.stdout.close) @@ -2862,6 +3447,7 @@ def start_pdb(): self.assertNotIn(b'Error', stdout, "Got an error running test script under PDB") + @force_not_colorized def test_issue16180(self): # A syntax error in the debuggee. script = "def f: pass\n" @@ -2870,11 +3456,12 @@ def test_issue16180(self): stdout, stderr = self.run_pdb_script( script, commands ) - self.assertIn(expected, stdout, + self.assertIn(expected, stderr, '\n\nExpected:\n{}\nGot:\n{}\n' 'Fail to handle a syntax error in the debuggee.' - .format(expected, stdout)) + .format(expected, stderr)) + @force_not_colorized def test_issue84583(self): # A syntax error from ast.literal_eval should not make pdb exit. script = "import ast; ast.literal_eval('')\n" @@ -2884,11 +3471,12 @@ def test_issue84583(self): quit """ stdout, stderr = self.run_pdb_script(script, commands) - # The code should appear 3 times in the stdout: - # 1. when pdb starts - # 2. when the exception is raised, in trackback - # 3. in where command - self.assertEqual(stdout.count("ast.literal_eval('')"), 3) + # The code should appear 3 times in the stdout/stderr: + # 1. when pdb starts (stdout) + # 2. when the exception is raised, in trackback (stderr) + # 3. in where command (stdout) + self.assertEqual(stdout.count("ast.literal_eval('')"), 2) + self.assertEqual(stderr.count("ast.literal_eval('')"), 1) def test_issue26053(self): # run command of pdb prompt echoes the correct args @@ -2904,91 +3492,166 @@ def test_issue26053(self): self.assertRegex(res, "Restarting .* with arguments:\na b c") self.assertRegex(res, "Restarting .* with arguments:\nd e f") - def test_readrc_kwarg(self): + def test_issue58956(self): + # Set a breakpoint in a function that already exists on the call stack + # should enable the trace function for the frame. + script = """ + import bar + def foo(): + ret = bar.bar() + pass + foo() + """ + commands = """ + b bar.bar + c + b main.py:5 + c + p ret + quit + """ + bar = """ + def bar(): + return 42 + """ + with open('bar.py', 'w') as f: + f.write(textwrap.dedent(bar)) + self.addCleanup(os_helper.unlink, 'bar.py') + stdout, stderr = self.run_pdb_script(script, commands) + lines = stdout.splitlines() + self.assertIn('-> pass', lines) + self.assertIn('(Pdb) 42', lines) + + def test_step_into_botframe(self): + # gh-125422 + # pdb should not be able to step into the botframe (bdb.py) + script = "x = 1" + commands = """ + step + step + step + quit + """ + stdout, _ = self.run_pdb_script(script, commands) + self.assertIn("The program finished", stdout) + self.assertNotIn("bdb.py", stdout) + + def test_pdbrc_basic(self): script = textwrap.dedent(""" - import pdb; pdb.Pdb(readrc=False).set_trace() + a = 1 + b = 2 + """) - print('hello') + pdbrc = textwrap.dedent(""" + # Comments should be fine + n + p f"{a+8=}" """) - save_home = os.environ.pop('HOME', None) - try: - with os_helper.temp_cwd(): - with open('.pdbrc', 'w') as f: - f.write("invalid\n") - - with open('main.py', 'w') as f: - f.write(script) - - cmd = [sys.executable, 'main.py'] - proc = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - with proc: - stdout, stderr = proc.communicate(b'q\n') - self.assertNotIn(b"NameError: name 'invalid' is not defined", - stdout) + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertNotIn("SyntaxError", stdout) + self.assertIn("a+8=9", stdout) + self.assertIn("-> b = 2", stdout) - finally: - if save_home is not None: - os.environ['HOME'] = save_home + def test_pdbrc_empty_line(self): + """Test that empty lines in .pdbrc are ignored.""" - def test_readrc_homedir(self): - save_home = os.environ.pop("HOME", None) - with os_helper.temp_dir() as temp_dir, patch("os.path.expanduser"): - rc_path = os.path.join(temp_dir, ".pdbrc") - os.path.expanduser.return_value = rc_path - try: - with open(rc_path, "w") as f: - f.write("invalid") - self.assertEqual(pdb.Pdb().rcLines[0], "invalid") - finally: - if save_home is not None: - os.environ["HOME"] = save_home + script = textwrap.dedent(""" + a = 1 + b = 2 + c = 3 + """) + + pdbrc = textwrap.dedent(""" + n + + """) - def test_read_pdbrc_with_ascii_encoding(self): + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertIn("b = 2", stdout) + self.assertNotIn("c = 3", stdout) + + def test_pdbrc_alias(self): + script = textwrap.dedent(""" + class A: + def __init__(self): + self.attr = 1 + a = A() + b = 2 + """) + + pdbrc = textwrap.dedent(""" + alias pi for k in %1.__dict__.keys(): print(f"%1.{k} = {%1.__dict__[k]}") + until 6 + pi a + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertIn("a.attr = 1", stdout) + + def test_pdbrc_semicolon(self): + script = textwrap.dedent(""" + class A: + def __init__(self): + self.attr = 1 + a = A() + b = 2 + """) + + pdbrc = textwrap.dedent(""" + b 5;;c;;n + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertIn("-> b = 2", stdout) + + def test_pdbrc_commands(self): + script = textwrap.dedent(""" + class A: + def __init__(self): + self.attr = 1 + a = A() + b = 2 + """) + + pdbrc = textwrap.dedent(""" + b 6 + commands 1 ;; p a;; end + c + """) + + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) + self.assertIn("<__main__.A object at", stdout) + + def test_readrc_kwarg(self): script = textwrap.dedent(""" - import pdb; pdb.Pdb().set_trace() print('hello') """) - save_home = os.environ.pop('HOME', None) - try: - with os_helper.temp_cwd(): - with open('.pdbrc', 'w', encoding='utf-8') as f: - f.write("Fran\u00E7ais") - - with open('main.py', 'w', encoding='utf-8') as f: - f.write(script) - - cmd = [sys.executable, 'main.py'] - env = {'PYTHONIOENCODING': 'ascii'} - if sys.platform == 'win32': - env['PYTHONLEGACYWINDOWSSTDIO'] = 'non-empty-string' - proc = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - env={**os.environ, **env} - ) - with proc: - stdout, stderr = proc.communicate(b'c\n') - self.assertIn(b"UnicodeEncodeError: \'ascii\' codec can\'t encode character " - b"\'\\xe7\' in position 21: ordinal not in range(128)", stderr) - finally: - if save_home is not None: - os.environ['HOME'] = save_home + stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc='invalid', remove_home=True) + self.assertIn("NameError: name 'invalid' is not defined", stdout) + + def test_readrc_homedir(self): + with os_helper.EnvironmentVarGuard() as env: + env.unset("HOME") + with os_helper.temp_dir() as temp_dir, patch("os.path.expanduser"): + rc_path = os.path.join(temp_dir, ".pdbrc") + os.path.expanduser.return_value = rc_path + with open(rc_path, "w") as f: + f.write("invalid") + self.assertEqual(pdb.Pdb().rcLines[0], "invalid") def test_header(self): stdout = StringIO() header = 'Nobody expects... blah, blah, blah' with ExitStack() as resources: resources.enter_context(patch('sys.stdout', stdout)) + # patch pdb.Pdb.set_trace() to avoid entering the debugger resources.enter_context(patch.object(pdb.Pdb, 'set_trace')) + # We need to manually clear pdb.Pdb._last_pdb_instance so a + # new instance with stdout redirected could be created when + # pdb.set_trace() is called. + pdb.Pdb._last_pdb_instance = None pdb.set_trace(header=header) self.assertEqual(stdout.getvalue(), header + '\n') @@ -3013,6 +3676,31 @@ def test_module_is_run_as_main(self): stdout, stderr = self.run_pdb_module(script, commands) self.assertTrue(any("SUCCESS" in l for l in stdout.splitlines()), stdout) + def test_run_module_with_args(self): + commands = """ + continue + """ + self._run_pdb(["calendar", "-m"], commands, expected_returncode=2) + + stdout, _ = self._run_pdb(["-m", "calendar", "1"], commands) + self.assertIn("December", stdout) + + stdout, _ = self._run_pdb(["-m", "calendar", "--type", "text"], commands) + self.assertIn("December", stdout) + + def test_run_script_with_args(self): + script = """ + import sys + print(sys.argv[1:]) + """ + commands = """ + continue + quit + """ + + stdout, stderr = self.run_pdb_script(script, commands, script_args=["--bar", "foo"]) + self.assertIn("['--bar', 'foo']", stdout) + def test_breakpoint(self): script = """ if __name__ == '__main__': @@ -3080,9 +3768,9 @@ def test_dir_as_script(self): def test_invalid_cmd_line_options(self): stdout, stderr = self._run_pdb(["-c"], "", expected_returncode=2) - self.assertIn(f"pdb: error: argument -c/--command: expected one argument", stdout.split('\n')[1]) + self.assertIn(f"pdb: error: argument -c/--command: expected one argument", stderr.split('\n')[1]) stdout, stderr = self._run_pdb(["--spam", "-m", "pdb"], "", expected_returncode=2) - self.assertIn(f"pdb: error: unrecognized arguments: --spam", stdout.split('\n')[1]) + self.assertIn(f"pdb: error: unrecognized arguments: --spam", stderr.split('\n')[1]) def test_blocks_at_first_code_line(self): script = """ @@ -3102,10 +3790,12 @@ def test_file_modified_after_execution(self): print("hello") """ + # the time.sleep is needed for low-resolution filesystems like HFS+ commands = """ filename = $_frame.f_code.co_filename f = open(filename, "w") f.write("print('goodbye')") + import time; time.sleep(1) f.close() ll """ @@ -3114,11 +3804,32 @@ def test_file_modified_after_execution(self): self.assertIn("WARNING:", stdout) self.assertIn("was edited", stdout) + def test_file_modified_and_immediately_restarted(self): + script = """ + print("hello") + """ + + # the time.sleep is needed for low-resolution filesystems like HFS+ + commands = """ + filename = $_frame.f_code.co_filename + f = open(filename, "w") + f.write("print('goodbye')") + import time; time.sleep(1) + f.close() + restart + """ + + stdout, stderr = self.run_pdb_script(script, commands) + self.assertNotIn("WARNING:", stdout) + self.assertNotIn("was edited", stdout) + def test_file_modified_after_execution_with_multiple_instances(self): + # the time.sleep is needed for low-resolution filesystems like HFS+ script = """ import pdb; pdb.Pdb().set_trace() with open(__file__, "w") as f: f.write("print('goodbye')\\n" * 5) + import time; time.sleep(1) import pdb; pdb.Pdb().set_trace() """ @@ -3137,7 +3848,7 @@ def test_file_modified_after_execution_with_multiple_instances(self): cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.PIPE, env = {**os.environ, 'PYTHONIOENCODING': 'utf-8'}, ) as proc: stdout, _ = proc.communicate(str.encode(commands)) @@ -3178,6 +3889,23 @@ def change_file(content, filename): # the file as up to date self.assertNotIn("WARNING:", stdout) + def test_post_mortem_restart(self): + script = """ + def foo(): + raise ValueError("foo") + foo() + """ + + commands = """ + continue + restart + continue + quit + """ + + stdout, stderr = self.run_pdb_script(script, commands) + self.assertIn("Restarting", stdout) + def test_relative_imports(self): self.module_name = 't_main' os_helper.rmtree(self.module_name) @@ -3432,12 +4160,46 @@ def _create_fake_frozen_module(): # verify that pdb found the source of the "frozen" function self.assertIn('x = "Sentinel string for gh-93696"', stdout, "Sentinel statement not found") + def test_empty_file(self): + script = '' + commands = 'q\n' + # We check that pdb stopped at line 0, but anything reasonable + # is acceptable here, as long as it does not halt + stdout, _ = self.run_pdb_script(script, commands) + self.assertIn('main.py(0)', stdout) + stdout, _ = self.run_pdb_module(script, commands) + self.assertIn('__main__.py(0)', stdout) + def test_non_utf8_encoding(self): script_dir = os.path.join(os.path.dirname(__file__), 'encoded_modules') for filename in os.listdir(script_dir): if filename.endswith(".py"): self._run_pdb([os.path.join(script_dir, filename)], 'q') + def test_zipapp(self): + with os_helper.temp_dir() as temp_dir: + os.mkdir(os.path.join(temp_dir, 'source')) + script = textwrap.dedent( + """ + def f(x): + return x + 1 + f(21 + 21) + """ + ) + with open(os.path.join(temp_dir, 'source', '__main__.py'), 'w') as f: + f.write(script) + zipapp.create_archive(os.path.join(temp_dir, 'source'), + os.path.join(temp_dir, 'zipapp.pyz')) + stdout, _ = self._run_pdb([os.path.join(temp_dir, 'zipapp.pyz')], '\n'.join([ + 'b f', + 'c', + 'p x', + 'q' + ])) + self.assertIn('42', stdout) + self.assertIn('return x + 1', stdout) + + class ChecklineTests(unittest.TestCase): def setUp(self): linecache.clearcache() # Pdb.checkline() uses linecache.getline() @@ -3475,6 +4237,62 @@ def test_checkline_is_not_executable(self): self.assertFalse(db.checkline(os_helper.TESTFN, lineno)) +@support.requires_subprocess() +class PdbTestInline(unittest.TestCase): + @unittest.skipIf(sys.flags.safe_path, + 'PYTHONSAFEPATH changes default sys.path') + def _run_script(self, script, commands, + expected_returncode=0, + extra_env=None): + self.addCleanup(os_helper.rmtree, '__pycache__') + filename = 'main.py' + with open(filename, 'w') as f: + f.write(textwrap.dedent(script)) + self.addCleanup(os_helper.unlink, filename) + + commands = textwrap.dedent(commands) + + cmd = [sys.executable, 'main.py'] + if extra_env is not None: + env = os.environ | extra_env + else: + env = os.environ + with subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + stderr=subprocess.PIPE, + env = {**env, 'PYTHONIOENCODING': 'utf-8'} + ) as proc: + stdout, stderr = proc.communicate(str.encode(commands)) + stdout = bytes.decode(stdout) if isinstance(stdout, bytes) else stdout + stderr = bytes.decode(stderr) if isinstance(stderr, bytes) else stderr + self.assertEqual( + proc.returncode, + expected_returncode, + f"Unexpected return code\nstdout: {stdout}\nstderr: {stderr}" + ) + return stdout, stderr + + def test_quit(self): + script = """ + x = 1 + breakpoint() + """ + + commands = """ + quit + n + p x + 1 + quit + y + """ + + stdout, stderr = self._run_script(script, commands) + self.assertIn("2", stdout) + self.assertIn("Quit anyway", stdout) + + @support.requires_subprocess() class PdbTestReadline(unittest.TestCase): def setUpClass(): @@ -3523,6 +4341,57 @@ def test_expression_completion(self): self.assertIn(b'species', output) self.assertIn(b'$_frame', output) + def test_builtin_completion(self): + script = textwrap.dedent(""" + value = "speci" + import pdb; pdb.Pdb().set_trace() + """) + + # Complete: print(value + 'al') + input = b"pri\tval\t + 'al')\n" + + # Continue + input += b"c\n" + + output = run_pty(script, input) + + self.assertIn(b'special', output) + + def test_local_namespace(self): + script = textwrap.dedent(""" + def f(): + original = "I live Pythin" + import pdb; pdb.Pdb().set_trace() + f() + """) + + # Complete: original.replace('i', 'o') + input = b"orig\t.repl\t('i', 'o')\n" + + # Continue + input += b"c\n" + + output = run_pty(script, input) + + self.assertIn(b'I love Python', output) + + def test_multiline_completion(self): + script = textwrap.dedent(""" + import pdb; pdb.Pdb().set_trace() + """) + + input = b"def func():\n" + # Complete: \treturn 40 + 2 + input += b"\tret\t 40 + 2\n" + input += b"\n" + # Complete: func() + input += b"fun\t()\n" + input += b"c\n" + + output = run_pty(script, input) + + self.assertIn(b'42', output) + def load_tests(loader, tests, pattern): from test import test_pdb diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index dffedd0b1fc476..b5b2b350e77a3b 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -114,7 +114,7 @@ def f(): return None self.assertNotInBytecode(f, 'LOAD_GLOBAL') - self.assertInBytecode(f, 'RETURN_CONST', None) + self.assertInBytecode(f, 'LOAD_CONST', None) self.check_lnotab(f) def test_while_one(self): @@ -131,7 +131,7 @@ def f(): def test_pack_unpack(self): for line, elem in ( - ('a, = a,', 'RETURN_CONST',), + ('a, = a,', 'LOAD_CONST',), ('a, b = a, b', 'SWAP',), ('a, b, c = a, b, c', 'SWAP',), ): @@ -162,7 +162,7 @@ def test_folding_of_tuples_of_constants(self): # One LOAD_CONST for the tuple, one for the None return value load_consts = [instr for instr in dis.get_instructions(code) if instr.opname == 'LOAD_CONST'] - self.assertEqual(len(load_consts), 1) + self.assertEqual(len(load_consts), 2) self.check_lnotab(code) # Bug 1053819: Tuple of constants misidentified when presented with: @@ -248,14 +248,17 @@ def test_folding_of_binops_on_constants(self): ): with self.subTest(line=line): code = compile(line, '', 'single') - self.assertInBytecode(code, 'LOAD_CONST', elem) + if isinstance(elem, int): + self.assertInBytecode(code, 'LOAD_SMALL_INT', elem) + else: + self.assertInBytecode(code, 'LOAD_CONST', elem) for instr in dis.get_instructions(code): self.assertFalse(instr.opname.startswith('BINARY_')) self.check_lnotab(code) # Verify that unfoldables are skipped code = compile('a=2+"b"', '', 'single') - self.assertInBytecode(code, 'LOAD_CONST', 2) + self.assertInBytecode(code, 'LOAD_SMALL_INT', 2) self.assertInBytecode(code, 'LOAD_CONST', 'b') self.check_lnotab(code) @@ -307,7 +310,10 @@ def test_folding_of_unaryops_on_constants(self): ): with self.subTest(line=line): code = compile(line, '', 'single') - self.assertInBytecode(code, 'LOAD_CONST', elem) + if isinstance(elem, int): + self.assertInBytecode(code, 'LOAD_SMALL_INT', elem) + else: + self.assertInBytecode(code, 'LOAD_CONST', elem) for instr in dis.get_instructions(code): self.assertFalse(instr.opname.startswith('UNARY_')) self.check_lnotab(code) @@ -766,7 +772,7 @@ def test_load_fast_unknown_after_error_2(self): def f(): try: 1 / 0 - except: + except ZeroDivisionError: print(a, b, c, d, e, f, g) a = b = c = d = e = f = g = 1 self.assertInBytecode(f, 'LOAD_FAST_CHECK') @@ -932,23 +938,6 @@ def f(): self.assertNotInBytecode(f, "LOAD_FAST_CHECK") return f - def test_deleting_local_warns_and_assigns_none(self): - f = self.make_function_with_no_checks() - co_code = f.__code__.co_code - def trace(frame, event, arg): - if event == 'line' and frame.f_lineno == 4: - del frame.f_locals["x"] - sys.settrace(None) - return None - return trace - e = r"assigning None to unbound local 'x'" - with self.assertWarnsRegex(RuntimeWarning, e): - sys.settrace(trace) - f() - self.assertInBytecode(f, "LOAD_FAST") - self.assertNotInBytecode(f, "LOAD_FAST_CHECK") - self.assertEqual(f.__code__.co_code, co_code) - def test_modifying_local_does_not_add_check(self): f = self.make_function_with_no_checks() def trace(frame, event, arg): @@ -981,10 +970,15 @@ class DirectCfgOptimizerTests(CfgOptimizationTestCase): def cfg_optimization_test(self, insts, expected_insts, consts=None, expected_consts=None, nlocals=0): + + self.check_instructions(insts) + self.check_instructions(expected_insts) + if expected_consts is None: expected_consts = consts - opt_insts, opt_consts = self.get_optimized(insts, consts, nlocals) - expected_insts = self.normalize_insts(expected_insts) + seq = self.seq_from_insts(insts) + opt_insts, opt_consts = self.get_optimized(seq, consts, nlocals) + expected_insts = self.seq_from_insts(expected_insts).get_instructions() self.assertInstructionsMatch(opt_insts, expected_insts) self.assertEqual(opt_consts, expected_consts) @@ -993,17 +987,19 @@ def test_conditional_jump_forward_non_const_condition(self): ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl := self.Label(), 12), ('LOAD_CONST', 2, 13), - ('RETURN_VALUE', 13), + ('RETURN_VALUE', None, 13), lbl, ('LOAD_CONST', 3, 14), - ('RETURN_VALUE', 14), + ('RETURN_VALUE', None, 14), ] expected_insts = [ ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl := self.Label(), 12), - ('RETURN_CONST', 1, 13), + ('LOAD_CONST', 1, 13), + ('RETURN_VALUE', None, 13), lbl, - ('RETURN_CONST', 2, 14), + ('LOAD_CONST', 2, 14), + ('RETURN_VALUE', None, 14), ] self.cfg_optimization_test(insts, expected_insts, @@ -1020,12 +1016,13 @@ def test_conditional_jump_forward_const_condition(self): ('LOAD_CONST', 2, 13), lbl, ('LOAD_CONST', 3, 14), - ('RETURN_VALUE', 14), + ('RETURN_VALUE', None, 14), ] expected_insts = [ - ('NOP', 11), - ('NOP', 12), - ('RETURN_CONST', 1, 14), + ('NOP', None, 11), + ('NOP', None, 12), + ('LOAD_CONST', 1, 14), + ('RETURN_VALUE', None, 14), ] self.cfg_optimization_test(insts, expected_insts, @@ -1038,14 +1035,14 @@ def test_conditional_jump_backward_non_const_condition(self): ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl1, 12), ('LOAD_NAME', 2, 13), - ('RETURN_VALUE', 13), + ('RETURN_VALUE', None, 13), ] expected = [ lbl := self.Label(), ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl, 12), ('LOAD_NAME', 2, 13), - ('RETURN_VALUE', 13), + ('RETURN_VALUE', None, 13), ] self.cfg_optimization_test(insts, expected, consts=list(range(5))) @@ -1056,11 +1053,11 @@ def test_conditional_jump_backward_const_condition(self): ('LOAD_CONST', 3, 11), ('POP_JUMP_IF_TRUE', lbl1, 12), ('LOAD_CONST', 2, 13), - ('RETURN_VALUE', 13), + ('RETURN_VALUE', None, 13), ] expected_insts = [ lbl := self.Label(), - ('NOP', 11), + ('NOP', None, 11), ('JUMP', lbl, 12), ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) @@ -1068,16 +1065,20 @@ def test_conditional_jump_backward_const_condition(self): def test_except_handler_label(self): insts = [ ('SETUP_FINALLY', handler := self.Label(), 10), - ('POP_BLOCK', 0, -1), - ('RETURN_CONST', 1, 11), + ('POP_BLOCK', None, -1), + ('LOAD_CONST', 1, 11), + ('RETURN_VALUE', None, 11), handler, - ('RETURN_CONST', 2, 12), + ('LOAD_CONST', 2, 12), + ('RETURN_VALUE', None, 12), ] expected_insts = [ ('SETUP_FINALLY', handler := self.Label(), 10), - ('RETURN_CONST', 1, 11), + ('LOAD_CONST', 1, 11), + ('RETURN_VALUE', None, 11), handler, - ('RETURN_CONST', 2, 12), + ('LOAD_CONST', 2, 12), + ('RETURN_VALUE', None, 12), ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) @@ -1090,17 +1091,18 @@ def test_no_unsafe_static_swap(self): ('SWAP', 3, 4), ('STORE_FAST', 1, 4), ('STORE_FAST', 1, 4), - ('POP_TOP', 0, 4), + ('POP_TOP', None, 4), ('LOAD_CONST', 0, 5), - ('RETURN_VALUE', 5) + ('RETURN_VALUE', None, 5) ] expected_insts = [ ('LOAD_CONST', 0, 1), ('LOAD_CONST', 1, 2), - ('NOP', 0, 3), + ('NOP', None, 3), ('STORE_FAST', 1, 4), - ('POP_TOP', 0, 4), - ('RETURN_CONST', 0) + ('POP_TOP', None, 4), + ('LOAD_CONST', 0, 5), + ('RETURN_VALUE', None, 5) ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1) @@ -1113,15 +1115,16 @@ def test_dead_store_elimination_in_same_lineno(self): ('STORE_FAST', 1, 4), ('STORE_FAST', 1, 4), ('LOAD_CONST', 0, 5), - ('RETURN_VALUE', 5) + ('RETURN_VALUE', None, 5) ] expected_insts = [ ('LOAD_CONST', 0, 1), ('LOAD_CONST', 1, 2), - ('NOP', 0, 3), - ('POP_TOP', 0, 4), + ('NOP', None, 3), + ('POP_TOP', None, 4), ('STORE_FAST', 1, 4), - ('RETURN_CONST', 0, 5) + ('LOAD_CONST', 0, 5), + ('RETURN_VALUE', None, 5) ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1) @@ -1134,7 +1137,7 @@ def test_no_dead_store_elimination_in_different_lineno(self): ('STORE_FAST', 1, 5), ('STORE_FAST', 1, 6), ('LOAD_CONST', 0, 5), - ('RETURN_VALUE', 5) + ('RETURN_VALUE', None, 5) ] expected_insts = [ ('LOAD_CONST', 0, 1), @@ -1143,7 +1146,8 @@ def test_no_dead_store_elimination_in_different_lineno(self): ('STORE_FAST', 1, 4), ('STORE_FAST', 1, 5), ('STORE_FAST', 1, 6), - ('RETURN_CONST', 0, 5) + ('LOAD_CONST', 0, 5), + ('RETURN_VALUE', None, 5) ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1) @@ -1169,7 +1173,7 @@ def get_insts(lno1, lno2, op1, op2): op = 'JUMP' if 'JUMP' in (op1, op2) else 'JUMP_NO_INTERRUPT' expected_insts = [ ('LOAD_NAME', 0, 10), - ('NOP', 0, 4), + ('NOP', None, 4), (op, 0, 5), ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) @@ -1189,5 +1193,56 @@ def get_insts(lno1, lno2, op1, op2): ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) + def test_list_to_tuple_get_iter(self): + # for _ in (*foo, *bar) -> for _ in [*foo, *bar] + INTRINSIC_LIST_TO_TUPLE = 6 + insts = [ + ("BUILD_LIST", 0, 1), + ("LOAD_FAST", 0, 2), + ("LIST_EXTEND", 1, 3), + ("LOAD_FAST", 1, 4), + ("LIST_EXTEND", 1, 5), + ("CALL_INTRINSIC_1", INTRINSIC_LIST_TO_TUPLE, 6), + ("GET_ITER", None, 7), + top := self.Label(), + ("FOR_ITER", end := self.Label(), 8), + ("STORE_FAST", 2, 9), + ("JUMP", top, 10), + end, + ("END_FOR", None, 11), + ("POP_TOP", None, 12), + ("LOAD_CONST", 0, 13), + ("RETURN_VALUE", None, 14), + ] + expected_insts = [ + ("BUILD_LIST", 0, 1), + ("LOAD_FAST", 0, 2), + ("LIST_EXTEND", 1, 3), + ("LOAD_FAST", 1, 4), + ("LIST_EXTEND", 1, 5), + ("NOP", None, 6), # ("CALL_INTRINSIC_1", INTRINSIC_LIST_TO_TUPLE, 6), + ("GET_ITER", None, 7), + top := self.Label(), + ("FOR_ITER", end := self.Label(), 8), + ("STORE_FAST", 2, 9), + ("JUMP", top, 10), + end, + ("END_FOR", None, 11), + ("POP_TOP", None, 12), + ("LOAD_CONST", 0, 13), + ("RETURN_VALUE", None, 14), + ] + self.cfg_optimization_test(insts, expected_insts, consts=[None]) + + def test_list_to_tuple_get_iter_is_safe(self): + a, b = [], [] + for item in (*(items := [0, 1, 2, 3]),): + a.append(item) + b.append(items.pop()) + self.assertEqual(a, [0, 1, 2, 3]) + self.assertEqual(b, [3, 2, 1, 0]) + self.assertEqual(items, []) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index 9e273e99e387a4..1411e55dd0f293 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -13,9 +13,7 @@ from test.support import os_helper, import_helper from test.support.script_helper import assert_python_ok -_py_cflags_nodist = sysconfig.get_config_var("PY_CFLAGS_NODIST") -_pgo_flag = sysconfig.get_config_var("PGO_PROF_USE_FLAG") -if _pgo_flag and _py_cflags_nodist and _pgo_flag in _py_cflags_nodist: +if support.check_cflags_pgo(): raise unittest.SkipTest("peg_generator test disabled under PGO build") test_tools.skip_if_missing("peg_generator") diff --git a/Lib/test/test_peg_generator/test_grammar_validator.py b/Lib/test/test_peg_generator/test_grammar_validator.py index 72c3d2054fa8f9..c7f20e1de802ce 100644 --- a/Lib/test/test_peg_generator/test_grammar_validator.py +++ b/Lib/test/test_peg_generator/test_grammar_validator.py @@ -4,7 +4,7 @@ test_tools.skip_if_missing("peg_generator") with test_tools.imports_under_tool("peg_generator"): from pegen.grammar_parser import GeneratedParser as GrammarParser - from pegen.validator import SubRuleValidator, ValidationError + from pegen.validator import SubRuleValidator, ValidationError, RaiseRuleValidator from pegen.testutil import parse_string from pegen.grammar import Grammar @@ -49,3 +49,13 @@ def test_rule_with_collision_after_some_other_rules(self) -> None: with self.assertRaises(ValidationError): for rule_name, rule in grammar.rules.items(): validator.validate_rule(rule_name, rule) + + def test_raising_valid_rule(self) -> None: + grammar_source = """ + start: NAME { RAISE_SYNTAX_ERROR("this is not allowed") } + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + validator = RaiseRuleValidator(grammar) + with self.assertRaises(ValidationError): + for rule_name, rule in grammar.rules.items(): + validator.validate_rule(rule_name, rule) diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py index 86db767b99a228..d8606521345390 100644 --- a/Lib/test/test_peg_generator/test_pegen.py +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -484,7 +484,7 @@ def test_left_recursive(self) -> None: def test_python_expr(self) -> None: grammar = """ - start: expr NEWLINE? $ { ast.Expression(expr, lineno=1, col_offset=0) } + start: expr NEWLINE? $ { ast.Expression(expr) } expr: ( expr '+' term { ast.BinOp(expr, ast.Add(), term, lineno=expr.lineno, col_offset=expr.col_offset, end_lineno=term.end_lineno, end_col_offset=term.end_col_offset) } | expr '-' term { ast.BinOp(expr, ast.Sub(), term, lineno=expr.lineno, col_offset=expr.col_offset, end_lineno=term.end_lineno, end_col_offset=term.end_col_offset) } | term { term } @@ -506,6 +506,14 @@ def test_python_expr(self) -> None: val = eval(code) self.assertEqual(val, 3.0) + def test_f_string_in_action(self) -> None: + grammar = """ + start: n=NAME NEWLINE? $ { f"name -> {n.string}" } + """ + parser_class = make_parser(grammar) + node = parse_string("a", parser_class) + self.assertEqual(node.strip(), "name -> a") + def test_nullable(self) -> None: grammar_source = """ start: sign NUMBER @@ -893,7 +901,7 @@ def test_unreachable_implicit3(self) -> None: def test_locations_in_alt_action_and_group(self) -> None: grammar = """ - start: t=term NEWLINE? $ { ast.Expression(t, LOCATIONS) } + start: t=term NEWLINE? $ { ast.Expression(t) } term: | l=term '*' r=factor { ast.BinOp(l, ast.Mult(), r, LOCATIONS) } | l=term '/' r=factor { ast.BinOp(l, ast.Div(), r, LOCATIONS) } diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 040be63da11447..6f1fd8d38e4ea0 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -47,6 +47,7 @@ def tearDown(self) -> None: for file in files_to_delete: file.unlink() + @unittest.skipIf(support.check_bolt_optimized, "fails on BOLT instrumented binaries") def test_trampoline_works(self): code = """if 1: def foo(): @@ -62,11 +63,13 @@ def baz(): """ with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) + env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, "-Xperf", script], text=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, + env=env, ) as process: stdout, stderr = process.communicate() @@ -76,15 +79,29 @@ def baz(): perf_file = pathlib.Path(f"/tmp/perf-{process.pid}.map") self.assertTrue(perf_file.exists()) perf_file_contents = perf_file.read_text() - perf_lines = perf_file_contents.splitlines(); - expected_symbols = [f"py::foo:{script}", f"py::bar:{script}", f"py::baz:{script}"] + perf_lines = perf_file_contents.splitlines() + expected_symbols = [ + f"py::foo:{script}", + f"py::bar:{script}", + f"py::baz:{script}", + ] for expected_symbol in expected_symbols: - perf_line = next((line for line in perf_lines if expected_symbol in line), None) - self.assertIsNotNone(perf_line, f"Could not find {expected_symbol} in perf file") + perf_line = next( + (line for line in perf_lines if expected_symbol in line), None + ) + self.assertIsNotNone( + perf_line, f"Could not find {expected_symbol} in perf file" + ) perf_addr = perf_line.split(" ")[0] - self.assertFalse(perf_addr.startswith("0x"), "Address should not be prefixed with 0x") - self.assertTrue(set(perf_addr).issubset(string.hexdigits), "Address should contain only hex characters") + self.assertFalse( + perf_addr.startswith("0x"), "Address should not be prefixed with 0x" + ) + self.assertTrue( + set(perf_addr).issubset(string.hexdigits), + "Address should contain only hex characters", + ) + @unittest.skipIf(support.check_bolt_optimized, "fails on BOLT instrumented binaries") def test_trampoline_works_with_forks(self): code = """if 1: import os, sys @@ -117,11 +134,13 @@ def baz(): """ with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) + env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, "-Xperf", script], text=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, + env=env, ) as process: stdout, stderr = process.communicate() @@ -143,6 +162,7 @@ def baz(): self.assertIn(f"py::bar_fork:{script}", child_perf_file_contents) self.assertIn(f"py::baz_fork:{script}", child_perf_file_contents) + @unittest.skipIf(support.check_bolt_optimized, "fails on BOLT instrumented binaries") def test_sys_api(self): code = """if 1: import sys @@ -166,11 +186,13 @@ def baz(): """ with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) + env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, script], text=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, + env=env, ) as process: stdout, stderr = process.communicate() @@ -191,14 +213,14 @@ def test_sys_api_with_existing_trampoline(self): sys.activate_stack_trampoline("perf") sys.activate_stack_trampoline("perf") """ - assert_python_ok("-c", code) + assert_python_ok("-c", code, PYTHON_JIT="0") def test_sys_api_with_invalid_trampoline(self): code = """if 1: import sys sys.activate_stack_trampoline("invalid") """ - rc, out, err = assert_python_failure("-c", code) + rc, out, err = assert_python_failure("-c", code, PYTHON_JIT="0") self.assertIn("invalid backend: invalid", err.decode()) def test_sys_api_get_status(self): @@ -209,10 +231,10 @@ def test_sys_api_get_status(self): sys.deactivate_stack_trampoline() assert sys.is_stack_trampoline_active() is False """ - assert_python_ok("-c", code) + assert_python_ok("-c", code, PYTHON_JIT="0") -def is_unwinding_reliable(): +def is_unwinding_reliable_with_frame_pointers(): cflags = sysconfig.get_config_var("PY_CORE_CFLAGS") if not cflags: return False @@ -247,8 +269,9 @@ def perf_command_works(): "-c", 'print("hello")', ) + env = {**os.environ, "PYTHON_JIT": "0"} stdout = subprocess.check_output( - cmd, cwd=script_dir, text=True, stderr=subprocess.STDOUT + cmd, cwd=script_dir, text=True, stderr=subprocess.STDOUT, env=env ) except (subprocess.SubprocessError, OSError): return False @@ -259,14 +282,26 @@ def perf_command_works(): return True -def run_perf(cwd, *args, **env_vars): +def run_perf(cwd, *args, use_jit=False, **env_vars): + env = os.environ.copy() if env_vars: - env = os.environ.copy() env.update(env_vars) - else: - env = None + env["PYTHON_JIT"] = "0" output_file = cwd + "/perf_output.perf" - base_cmd = ("perf", "record", "-g", "--call-graph=fp", "-o", output_file, "--") + if not use_jit: + base_cmd = ("perf", "record", "-g", "--call-graph=fp", "-o", output_file, "--") + else: + base_cmd = ( + "perf", + "record", + "-g", + "--call-graph=dwarf,65528", + "-F99", + "-k1", + "-o", + output_file, + "--", + ) proc = subprocess.run( base_cmd + args, stdout=subprocess.PIPE, @@ -274,9 +309,21 @@ def run_perf(cwd, *args, **env_vars): env=env, ) if proc.returncode: - print(proc.stderr) + print(proc.stderr, file=sys.stderr) raise ValueError(f"Perf failed with return code {proc.returncode}") + if use_jit: + jit_output_file = cwd + "/jit_output.dump" + command = ("perf", "inject", "-j", "-i", output_file, "-o", jit_output_file) + proc = subprocess.run( + command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env + ) + if proc.returncode: + print(proc.stderr) + raise ValueError(f"Perf failed with return code {proc.returncode}") + # Copy the jit_output_file to the output_file + os.rename(jit_output_file, output_file) + base_cmd = ("perf", "script") proc = subprocess.run( ("perf", "script", "-i", output_file), @@ -290,20 +337,9 @@ def run_perf(cwd, *args, **env_vars): ) -@unittest.skipUnless(perf_command_works(), "perf command doesn't work") -@unittest.skipUnless(is_unwinding_reliable(), "Unwinding is unreliable") -class TestPerfProfiler(unittest.TestCase): - def setUp(self): - super().setUp() - self.perf_files = set(pathlib.Path("/tmp/").glob("perf-*.map")) - - def tearDown(self) -> None: - super().tearDown() - files_to_delete = ( - set(pathlib.Path("/tmp/").glob("perf-*.map")) - self.perf_files - ) - for file in files_to_delete: - file.unlink() +class TestPerfProfilerMixin: + def run_perf(self, script_dir, perf_mode, script): + raise NotImplementedError() def test_python_calls_appear_in_the_stack_if_perf_activated(self): with temp_dir() as script_dir: @@ -322,14 +358,14 @@ def baz(n): baz(10000000) """ script = make_script(script_dir, "perftest", code) - stdout, stderr = run_perf(script_dir, sys.executable, "-Xperf", script) + stdout, stderr = self.run_perf(script_dir, script) self.assertEqual(stderr, "") self.assertIn(f"py::foo:{script}", stdout) self.assertIn(f"py::bar:{script}", stdout) self.assertIn(f"py::baz:{script}", stdout) - def test_python_calls_do_not_appear_in_the_stack_if_perf_activated(self): + def test_python_calls_do_not_appear_in_the_stack_if_perf_deactivated(self): with temp_dir() as script_dir: code = """if 1: def foo(n): @@ -346,13 +382,39 @@ def baz(n): baz(10000000) """ script = make_script(script_dir, "perftest", code) - stdout, stderr = run_perf(script_dir, sys.executable, script) + stdout, stderr = self.run_perf( + script_dir, script, activate_trampoline=False + ) self.assertEqual(stderr, "") self.assertNotIn(f"py::foo:{script}", stdout) self.assertNotIn(f"py::bar:{script}", stdout) self.assertNotIn(f"py::baz:{script}", stdout) + +@unittest.skipUnless(perf_command_works(), "perf command doesn't work") +@unittest.skipUnless( + is_unwinding_reliable_with_frame_pointers(), + "Unwinding is unreliable with frame pointers", +) +class TestPerfProfiler(unittest.TestCase, TestPerfProfilerMixin): + def run_perf(self, script_dir, script, activate_trampoline=True): + if activate_trampoline: + return run_perf(script_dir, sys.executable, "-Xperf", script) + return run_perf(script_dir, sys.executable, script) + + def setUp(self): + super().setUp() + self.perf_files = set(pathlib.Path("/tmp/").glob("perf-*.map")) + + def tearDown(self) -> None: + super().tearDown() + files_to_delete = ( + set(pathlib.Path("/tmp/").glob("perf-*.map")) - self.perf_files + ) + for file in files_to_delete: + file.unlink() + def test_pre_fork_compile(self): code = """if 1: import sys @@ -370,7 +432,7 @@ def bar_fork(): foo_fork() def foo(): - pass + import time; time.sleep(1) def bar(): foo() @@ -393,11 +455,13 @@ def compile_trampolines_for_all_functions(): with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) + env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, "-Xperf", script], universal_newlines=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, + env=env, ) as process: stdout, stderr = process.communicate() @@ -423,12 +487,49 @@ def compile_trampolines_for_all_functions(): # identical in both the parent and child perf-map files. perf_file_lines = perf_file_contents.split("\n") for line in perf_file_lines: - if ( - f"py::foo_fork:{script}" in line - or f"py::bar_fork:{script}" in line - ): + if f"py::foo_fork:{script}" in line or f"py::bar_fork:{script}" in line: self.assertIn(line, child_perf_file_contents) +def _is_perf_version_at_least(major, minor): + # The output of perf --version looks like "perf version 6.7-3" but + # it can also be perf version "perf version 5.15.143" + try: + output = subprocess.check_output(["perf", "--version"], text=True) + except (subprocess.CalledProcessError, FileNotFoundError): + return False + version = output.split()[2] + version = version.split("-")[0] + version = version.split(".") + version = tuple(map(int, version)) + return version >= (major, minor) + + +@unittest.skipUnless(perf_command_works(), "perf command doesn't work") +@unittest.skipUnless( + _is_perf_version_at_least(6, 6), "perf command may not work due to a perf bug" +) +class TestPerfProfilerWithDwarf(unittest.TestCase, TestPerfProfilerMixin): + def run_perf(self, script_dir, script, activate_trampoline=True): + if activate_trampoline: + return run_perf( + script_dir, sys.executable, "-Xperf_jit", script, use_jit=True + ) + return run_perf(script_dir, sys.executable, script, use_jit=True) + + def setUp(self): + super().setUp() + self.perf_files = set(pathlib.Path("/tmp/").glob("jit*.dump")) + self.perf_files |= set(pathlib.Path("/tmp/").glob("jitted-*.so")) + + def tearDown(self) -> None: + super().tearDown() + files_to_delete = set(pathlib.Path("/tmp/").glob("jit*.dump")) + files_to_delete |= set(pathlib.Path("/tmp/").glob("jitted-*.so")) + files_to_delete = files_to_delete - self.perf_files + for file in files_to_delete: + file.unlink() + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_perfmaps.py b/Lib/test/test_perfmaps.py index a17adb89f55360..d4c6fe0124af18 100644 --- a/Lib/test/test_perfmaps.py +++ b/Lib/test/test_perfmaps.py @@ -2,7 +2,11 @@ import sys import unittest -from _testinternalcapi import perf_map_state_teardown, write_perf_map_entry +try: + from _testinternalcapi import perf_map_state_teardown, write_perf_map_entry +except ImportError: + raise unittest.SkipTest("requires _testinternalcapi") + if sys.platform != 'linux': raise unittest.SkipTest('Linux only') diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 19f977971570b7..4ec966d8351490 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -16,6 +16,7 @@ from test.pickletester import AbstractHookTests from test.pickletester import AbstractUnpickleTests +from test.pickletester import AbstractPicklingErrorTests from test.pickletester import AbstractPickleTests from test.pickletester import AbstractPickleModuleTests from test.pickletester import AbstractPersistentPicklerTests @@ -55,6 +56,18 @@ def loads(self, buf, **kwds): return u.load() +class PyPicklingErrorTests(AbstractPicklingErrorTests, unittest.TestCase): + + pickler = pickle._Pickler + + def dumps(self, arg, proto=None, **kwargs): + f = io.BytesIO() + p = self.pickler(f, proto, **kwargs) + p.dump(arg) + f.seek(0) + return bytes(f.read()) + + class PyPicklerTests(AbstractPickleTests, unittest.TestCase): pickler = pickle._Pickler @@ -88,6 +101,8 @@ def loads(self, buf, **kwds): return pickle.loads(buf, **kwds) test_framed_write_sizes_with_delayed_writer = None + test_find_class = None + test_custom_find_class = None class PersistentPicklerUnpicklerMixin(object): @@ -209,25 +224,109 @@ def persistent_load(pid): def test_pickler_super(self): class PersPickler(self.pickler): def persistent_id(subself, obj): + called.append(obj) self.assertIsNone(super().persistent_id(obj)) return obj for proto in range(pickle.HIGHEST_PROTOCOL + 1): f = io.BytesIO() pickler = PersPickler(f, proto) + called = [] pickler.dump('abc') + self.assertEqual(called, ['abc']) self.assertEqual(self.loads(f.getvalue()), 'abc') def test_unpickler_super(self): class PersUnpickler(self.unpickler): def persistent_load(subself, pid): + called.append(pid) with self.assertRaises(self.persistent_load_error): super().persistent_load(pid) return pid for proto in range(pickle.HIGHEST_PROTOCOL + 1): unpickler = PersUnpickler(io.BytesIO(self.dumps('abc', proto))) + called = [] self.assertEqual(unpickler.load(), 'abc') + self.assertEqual(called, ['abc']) + + def test_pickler_instance_attribute(self): + def persistent_id(obj): + called.append(obj) + return obj + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f = io.BytesIO() + pickler = self.pickler(f, proto) + called = [] + old_persistent_id = pickler.persistent_id + pickler.persistent_id = persistent_id + self.assertEqual(pickler.persistent_id, persistent_id) + pickler.dump('abc') + self.assertEqual(called, ['abc']) + self.assertEqual(self.loads(f.getvalue()), 'abc') + del pickler.persistent_id + self.assertEqual(pickler.persistent_id, old_persistent_id) + + def test_unpickler_instance_attribute(self): + def persistent_load(pid): + called.append(pid) + return pid + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + unpickler = self.unpickler(io.BytesIO(self.dumps('abc', proto))) + called = [] + old_persistent_load = unpickler.persistent_load + unpickler.persistent_load = persistent_load + self.assertEqual(unpickler.persistent_load, persistent_load) + self.assertEqual(unpickler.load(), 'abc') + self.assertEqual(called, ['abc']) + del unpickler.persistent_load + self.assertEqual(unpickler.persistent_load, old_persistent_load) + + def test_pickler_super_instance_attribute(self): + class PersPickler(self.pickler): + def persistent_id(subself, obj): + raise AssertionError('should never be called') + def _persistent_id(subself, obj): + called.append(obj) + self.assertIsNone(super().persistent_id(obj)) + return obj + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f = io.BytesIO() + pickler = PersPickler(f, proto) + called = [] + old_persistent_id = pickler.persistent_id + pickler.persistent_id = pickler._persistent_id + self.assertEqual(pickler.persistent_id, pickler._persistent_id) + pickler.dump('abc') + self.assertEqual(called, ['abc']) + self.assertEqual(self.loads(f.getvalue()), 'abc') + del pickler.persistent_id + self.assertEqual(pickler.persistent_id, old_persistent_id) + + def test_unpickler_super_instance_attribute(self): + class PersUnpickler(self.unpickler): + def persistent_load(subself, pid): + raise AssertionError('should never be called') + def _persistent_load(subself, pid): + called.append(pid) + with self.assertRaises(self.persistent_load_error): + super().persistent_load(pid) + return pid + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + unpickler = PersUnpickler(io.BytesIO(self.dumps('abc', proto))) + called = [] + old_persistent_load = unpickler.persistent_load + unpickler.persistent_load = unpickler._persistent_load + self.assertEqual(unpickler.persistent_load, unpickler._persistent_load) + self.assertEqual(unpickler.load(), 'abc') + self.assertEqual(called, ['abc']) + del unpickler.persistent_load + self.assertEqual(unpickler.persistent_load, old_persistent_load) + class PyPicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests, unittest.TestCase): @@ -267,6 +366,9 @@ class CUnpicklerTests(PyUnpicklerTests): bad_stack_errors = (pickle.UnpicklingError,) truncated_errors = (pickle.UnpicklingError,) + class CPicklingErrorTests(PyPicklingErrorTests): + pickler = _pickle.Pickler + class CPicklerTests(PyPicklerTests): pickler = _pickle.Pickler unpickler = _pickle.Unpickler @@ -349,7 +451,7 @@ class SizeofTests(unittest.TestCase): check_sizeof = support.check_sizeof def test_pickler(self): - basesize = support.calcobjsize('6P2n3i2n3i2P') + basesize = support.calcobjsize('7P2n3i2n3i2P') p = _pickle.Pickler(io.BytesIO()) self.assertEqual(object.__sizeof__(p), basesize) MT_size = struct.calcsize('3nP0n') @@ -366,7 +468,7 @@ def test_pickler(self): 0) # Write buffer is cleared after every dump(). def test_unpickler(self): - basesize = support.calcobjsize('2P2nP 2P2n2i5P 2P3n8P2n2i') + basesize = support.calcobjsize('2P2n2P 2P2n2i5P 2P3n8P2n2i') unpickler = _pickle.Unpickler P = struct.calcsize('P') # Size of memo table entry. n = struct.calcsize('n') # Size of mark table entry. @@ -569,7 +671,7 @@ def test_exceptions(self): EncodingWarning, BaseExceptionGroup, ExceptionGroup, - IncompleteInputError): + _IncompleteInputError): continue if exc is not OSError and issubclass(exc, OSError): self.assertEqual(reverse_mapping('builtins', name), diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index d37af79e878a2e..a178d3353eecdf 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -1,3 +1,4 @@ +import io import pickle import pickletools from test import support @@ -62,6 +63,424 @@ def test_optimize_binput_and_memoize(self): self.assertNotIn(pickle.BINPUT, pickled2) +class SimpleReader: + def __init__(self, data): + self.data = data + self.pos = 0 + + def read(self, n): + data = self.data[self.pos: self.pos + n] + self.pos += n + return data + + def readline(self): + nl = self.data.find(b'\n', self.pos) + 1 + if not nl: + nl = len(self.data) + data = self.data[self.pos: nl] + self.pos = nl + return data + + +class GenopsTests(unittest.TestCase): + def test_genops(self): + it = pickletools.genops(b'(I123\nK\x12J\x12\x34\x56\x78t.') + self.assertEqual([(item[0].name,) + item[1:] for item in it], [ + ('MARK', None, 0), + ('INT', 123, 1), + ('BININT1', 0x12, 6), + ('BININT', 0x78563412, 8), + ('TUPLE', None, 13), + ('STOP', None, 14), + ]) + + def test_from_file(self): + f = io.BytesIO(b'prefix(I123\nK\x12J\x12\x34\x56\x78t.suffix') + self.assertEqual(f.read(6), b'prefix') + it = pickletools.genops(f) + self.assertEqual([(item[0].name,) + item[1:] for item in it], [ + ('MARK', None, 6), + ('INT', 123, 7), + ('BININT1', 0x12, 12), + ('BININT', 0x78563412, 14), + ('TUPLE', None, 19), + ('STOP', None, 20), + ]) + self.assertEqual(f.read(), b'suffix') + + def test_without_pos(self): + f = SimpleReader(b'(I123\nK\x12J\x12\x34\x56\x78t.') + it = pickletools.genops(f) + self.assertEqual([(item[0].name,) + item[1:] for item in it], [ + ('MARK', None, None), + ('INT', 123, None), + ('BININT1', 0x12, None), + ('BININT', 0x78563412, None), + ('TUPLE', None, None), + ('STOP', None, None), + ]) + + def test_no_stop(self): + it = pickletools.genops(b'N') + item = next(it) + self.assertEqual(item[0].name, 'NONE') + with self.assertRaisesRegex(ValueError, + 'pickle exhausted before seeing STOP'): + next(it) + + def test_truncated_data(self): + it = pickletools.genops(b'I123') + with self.assertRaisesRegex(ValueError, + 'no newline found when trying to read stringnl'): + next(it) + it = pickletools.genops(b'J\x12\x34') + with self.assertRaisesRegex(ValueError, + 'not enough data in stream to read int4'): + next(it) + + def test_unknown_opcode(self): + it = pickletools.genops(b'N\xff') + item = next(it) + self.assertEqual(item[0].name, 'NONE') + with self.assertRaisesRegex(ValueError, + r"at position 1, opcode b'\\xff' unknown"): + next(it) + + def test_unknown_opcode_without_pos(self): + f = SimpleReader(b'N\xff') + it = pickletools.genops(f) + item = next(it) + self.assertEqual(item[0].name, 'NONE') + with self.assertRaisesRegex(ValueError, + r"at position , opcode b'\\xff' unknown"): + next(it) + + +class DisTests(unittest.TestCase): + maxDiff = None + + def check_dis(self, data, expected, **kwargs): + out = io.StringIO() + pickletools.dis(data, out=out, **kwargs) + self.assertEqual(out.getvalue(), expected) + + def check_dis_error(self, data, expected, expected_error, **kwargs): + out = io.StringIO() + with self.assertRaisesRegex(ValueError, expected_error): + pickletools.dis(data, out=out, **kwargs) + self.assertEqual(out.getvalue(), expected) + + def test_mark(self): + self.check_dis(b'(N(tl.', '''\ + 0: ( MARK + 1: N NONE + 2: ( MARK + 3: t TUPLE (MARK at 2) + 4: l LIST (MARK at 0) + 5: . STOP +highest protocol among opcodes = 0 +''') + + def test_indentlevel(self): + self.check_dis(b'(N(tl.', '''\ + 0: ( MARK + 1: N NONE + 2: ( MARK + 3: t TUPLE (MARK at 2) + 4: l LIST (MARK at 0) + 5: . STOP +highest protocol among opcodes = 0 +''', indentlevel=2) + + def test_mark_without_pos(self): + self.check_dis(SimpleReader(b'(N(tl.'), '''\ +( MARK +N NONE +( MARK +t TUPLE (MARK at unknown opcode offset) +l LIST (MARK at unknown opcode offset) +. STOP +highest protocol among opcodes = 0 +''') + + def test_no_mark(self): + self.check_dis_error(b'Nt.', '''\ + 0: N NONE + 1: t TUPLE +''', 'no MARK exists on stack') + + def test_put(self): + self.check_dis(b'Np0\nq\x01r\x02\x00\x00\x00\x94.', '''\ + 0: N NONE + 1: p PUT 0 + 4: q BINPUT 1 + 6: r LONG_BINPUT 2 + 11: \\x94 MEMOIZE (as 3) + 12: . STOP +highest protocol among opcodes = 4 +''') + + def test_put_redefined(self): + self.check_dis(b'Np1\np1\nq\x01r\x01\x00\x00\x00\x94.', '''\ + 0: N NONE + 1: p PUT 1 + 4: p PUT 1 + 7: q BINPUT 1 + 9: r LONG_BINPUT 1 + 14: \\x94 MEMOIZE (as 1) + 15: . STOP +highest protocol among opcodes = 4 +''') + + def test_put_empty_stack(self): + self.check_dis_error(b'p0\n', '''\ + 0: p PUT 0 +''', "stack is empty -- can't store into memo") + + def test_put_markobject(self): + self.check_dis_error(b'(p0\n', '''\ + 0: ( MARK + 1: p PUT 0 +''', "can't store markobject in the memo") + + def test_get(self): + self.check_dis(b'(Np1\ng1\nh\x01j\x01\x00\x00\x00t.', '''\ + 0: ( MARK + 1: N NONE + 2: p PUT 1 + 5: g GET 1 + 8: h BINGET 1 + 10: j LONG_BINGET 1 + 15: t TUPLE (MARK at 0) + 16: . STOP +highest protocol among opcodes = 1 +''') + + def test_get_without_put(self): + self.check_dis_error(b'g1\n.', '''\ + 0: g GET 1 +''', 'memo key 1 has never been stored into') + self.check_dis_error(b'h\x01.', '''\ + 0: h BINGET 1 +''', 'memo key 1 has never been stored into') + self.check_dis_error(b'j\x01\x00\x00\x00.', '''\ + 0: j LONG_BINGET 1 +''', 'memo key 1 has never been stored into') + + def test_memo(self): + memo = {} + self.check_dis(b'Np1\n.', '''\ + 0: N NONE + 1: p PUT 1 + 4: . STOP +highest protocol among opcodes = 0 +''', memo=memo) + self.check_dis(b'g1\n.', '''\ + 0: g GET 1 + 3: . STOP +highest protocol among opcodes = 0 +''', memo=memo) + + def test_mark_pop(self): + self.check_dis(b'(N00N.', '''\ + 0: ( MARK + 1: N NONE + 2: 0 POP + 3: 0 POP (MARK at 0) + 4: N NONE + 5: . STOP +highest protocol among opcodes = 0 +''') + + def test_too_small_stack(self): + self.check_dis_error(b'a', '''\ + 0: a APPEND +''', 'tries to pop 2 items from stack with only 0 items') + self.check_dis_error(b']a', '''\ + 0: ] EMPTY_LIST + 1: a APPEND +''', 'tries to pop 2 items from stack with only 1 items') + + def test_no_stop(self): + self.check_dis_error(b'N', '''\ + 0: N NONE +''', 'pickle exhausted before seeing STOP') + + def test_truncated_data(self): + self.check_dis_error(b'NI123', '''\ + 0: N NONE +''', 'no newline found when trying to read stringnl') + self.check_dis_error(b'NJ\x12\x34', '''\ + 0: N NONE +''', 'not enough data in stream to read int4') + + def test_unknown_opcode(self): + self.check_dis_error(b'N\xff', '''\ + 0: N NONE +''', r"at position 1, opcode b'\\xff' unknown") + + def test_stop_not_empty_stack(self): + self.check_dis_error(b']N.', '''\ + 0: ] EMPTY_LIST + 1: N NONE + 2: . STOP +highest protocol among opcodes = 1 +''', r'stack not empty after STOP: \[list\]') + + def test_annotate(self): + self.check_dis(b'(Nt.', '''\ + 0: ( MARK Push markobject onto the stack. + 1: N NONE Push None on the stack. + 2: t TUPLE (MARK at 0) Build a tuple out of the topmost stack slice, after markobject. + 3: . STOP Stop the unpickling machine. +highest protocol among opcodes = 0 +''', annotate=1) + self.check_dis(b'(Nt.', '''\ + 0: ( MARK Push markobject onto the stack. + 1: N NONE Push None on the stack. + 2: t TUPLE (MARK at 0) Build a tuple out of the topmost stack slice, after markobject. + 3: . STOP Stop the unpickling machine. +highest protocol among opcodes = 0 +''', annotate=20) + self.check_dis(b'(((((((ttttttt.', '''\ + 0: ( MARK Push markobject onto the stack. + 1: ( MARK Push markobject onto the stack. + 2: ( MARK Push markobject onto the stack. + 3: ( MARK Push markobject onto the stack. + 4: ( MARK Push markobject onto the stack. + 5: ( MARK Push markobject onto the stack. + 6: ( MARK Push markobject onto the stack. + 7: t TUPLE (MARK at 6) Build a tuple out of the topmost stack slice, after markobject. + 8: t TUPLE (MARK at 5) Build a tuple out of the topmost stack slice, after markobject. + 9: t TUPLE (MARK at 4) Build a tuple out of the topmost stack slice, after markobject. + 10: t TUPLE (MARK at 3) Build a tuple out of the topmost stack slice, after markobject. + 11: t TUPLE (MARK at 2) Build a tuple out of the topmost stack slice, after markobject. + 12: t TUPLE (MARK at 1) Build a tuple out of the topmost stack slice, after markobject. + 13: t TUPLE (MARK at 0) Build a tuple out of the topmost stack slice, after markobject. + 14: . STOP Stop the unpickling machine. +highest protocol among opcodes = 0 +''', annotate=20) + + def test_string(self): + self.check_dis(b"S'abc'\n.", '''\ + 0: S STRING 'abc' + 7: . STOP +highest protocol among opcodes = 0 +''') + self.check_dis(b'S"abc"\n.', '''\ + 0: S STRING 'abc' + 7: . STOP +highest protocol among opcodes = 0 +''') + self.check_dis(b"S'\xc3\xb5'\n.", '''\ + 0: S STRING '\\xc3\\xb5' + 6: . STOP +highest protocol among opcodes = 0 +''') + + def test_string_without_quotes(self): + self.check_dis_error(b"Sabc'\n.", '', + 'no string quotes around b"abc\'"') + self.check_dis_error(b'Sabc"\n.', '', + "no string quotes around b'abc\"'") + self.check_dis_error(b"S'abc\n.", '', + '''strinq quote b"'" not found at both ends of b"'abc"''') + self.check_dis_error(b'S"abc\n.', '', + r"""strinq quote b'"' not found at both ends of b'"abc'""") + self.check_dis_error(b"S'abc\"\n.", '', + r"""strinq quote b"'" not found at both ends of b'\\'abc"'""") + self.check_dis_error(b"S\"abc'\n.", '', + r"""strinq quote b'"' not found at both ends of b'"abc\\''""") + + def test_binstring(self): + self.check_dis(b"T\x03\x00\x00\x00abc.", '''\ + 0: T BINSTRING 'abc' + 8: . STOP +highest protocol among opcodes = 1 +''') + self.check_dis(b"T\x02\x00\x00\x00\xc3\xb5.", '''\ + 0: T BINSTRING '\\xc3\\xb5' + 7: . STOP +highest protocol among opcodes = 1 +''') + + def test_short_binstring(self): + self.check_dis(b"U\x03abc.", '''\ + 0: U SHORT_BINSTRING 'abc' + 5: . STOP +highest protocol among opcodes = 1 +''') + self.check_dis(b"U\x02\xc3\xb5.", '''\ + 0: U SHORT_BINSTRING '\\xc3\\xb5' + 4: . STOP +highest protocol among opcodes = 1 +''') + + def test_global(self): + self.check_dis(b"cmodule\nname\n.", '''\ + 0: c GLOBAL 'module name' + 13: . STOP +highest protocol among opcodes = 0 +''') + self.check_dis(b"cm\xc3\xb6dule\nn\xc3\xa4me\n.", '''\ + 0: c GLOBAL 'm\xf6dule n\xe4me' + 15: . STOP +highest protocol among opcodes = 0 +''') + + def test_inst(self): + self.check_dis(b"(imodule\nname\n.", '''\ + 0: ( MARK + 1: i INST 'module name' (MARK at 0) + 14: . STOP +highest protocol among opcodes = 0 +''') + + def test_persid(self): + self.check_dis(b"Pabc\n.", '''\ + 0: P PERSID 'abc' + 5: . STOP +highest protocol among opcodes = 0 +''') + + def test_constants(self): + self.check_dis(b"(NI00\nI01\n\x89\x88t.", '''\ + 0: ( MARK + 1: N NONE + 2: I INT False + 6: I INT True + 10: \\x89 NEWFALSE + 11: \\x88 NEWTRUE + 12: t TUPLE (MARK at 0) + 13: . STOP +highest protocol among opcodes = 2 +''') + + def test_integers(self): + self.check_dis(b"(I0\nI1\nI10\nI011\nL12\nL13L\nL014\nL015L\nt.", '''\ + 0: ( MARK + 1: I INT 0 + 4: I INT 1 + 7: I INT 10 + 11: I INT 11 + 16: L LONG 12 + 20: L LONG 13 + 25: L LONG 14 + 30: L LONG 15 + 36: t TUPLE (MARK at 0) + 37: . STOP +highest protocol among opcodes = 0 +''') + + def test_nondecimal_integers(self): + self.check_dis_error(b'I0b10\n.', '', 'invalid literal for int') + self.check_dis_error(b'I0o10\n.', '', 'invalid literal for int') + self.check_dis_error(b'I0x10\n.', '', 'invalid literal for int') + self.check_dis_error(b'L0b10L\n.', '', 'invalid literal for int') + self.check_dis_error(b'L0o10L\n.', '', 'invalid literal for int') + self.check_dis_error(b'L0x10L\n.', '', 'invalid literal for int') + + class MiscTestCase(unittest.TestCase): def test__all__(self): not_exported = { diff --git a/Lib/test/test_pkg.py b/Lib/test/test_pkg.py index eed0fd1c6b73fa..a7a1c2affbe1fb 100644 --- a/Lib/test/test_pkg.py +++ b/Lib/test/test_pkg.py @@ -94,7 +94,7 @@ def mkhier(self, descr): def test_1(self): hier = [("t1", None), ("t1 __init__.py", "")] self.mkhier(hier) - import t1 + import t1 # noqa: F401 def test_2(self): hier = [ @@ -124,7 +124,7 @@ def test_2(self): from t2 import sub from t2.sub import subsub - from t2.sub.subsub import spam + from t2.sub.subsub import spam # noqa: F401 self.assertEqual(sub.__name__, "t2.sub") self.assertEqual(subsub.__name__, "t2.sub.subsub") self.assertEqual(sub.subsub.__name__, "t2.sub.subsub") diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index 6fcd726345eeac..736b83711def03 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -12,6 +12,10 @@ import shutil import zipfile +from test.support.import_helper import DirsOnSysPath +from test.support.os_helper import FakePath +from test.test_importlib.util import uncache + # Note: pkgutil.walk_packages is currently tested in test_runpy. This is # a hack to get a major issue resolved for 3.3b2. Longer term, it should # be moved back here, perhaps by factoring out the helper code for @@ -118,7 +122,7 @@ def test_issue44061_iter_modules(self): # make sure iter_modules accepts Path objects names = [] - for moduleinfo in pkgutil.iter_modules([Path(zip_file)]): + for moduleinfo in pkgutil.iter_modules([FakePath(zip_file)]): self.assertIsInstance(moduleinfo, pkgutil.ModuleInfo) names.append(moduleinfo.name) self.assertEqual(names, [pkg]) @@ -318,6 +322,38 @@ def test_name_resolution(self): with self.assertRaises(exc): pkgutil.resolve_name(s) + def test_name_resolution_import_rebinding(self): + # The same data is also used for testing import in test_import and + # mock.patch in test_unittest. + path = os.path.join(os.path.dirname(__file__), 'test_import', 'data') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3.submodule.attr'), 'submodule') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3.submodule:attr'), 'submodule') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + self.assertEqual(pkgutil.resolve_name('package3.submodule.attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + self.assertEqual(pkgutil.resolve_name('package3.submodule:attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + + def test_name_resolution_import_rebinding2(self): + path = os.path.join(os.path.dirname(__file__), 'test_import', 'data') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4.submodule.attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4.submodule:attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'origin') + self.assertEqual(pkgutil.resolve_name('package4.submodule.attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'origin') + self.assertEqual(pkgutil.resolve_name('package4.submodule:attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'submodule') + class PkgutilPEP302Tests(unittest.TestCase): @@ -486,7 +522,43 @@ def test_mixed_namespace(self): del sys.modules['foo.bar'] del sys.modules['foo.baz'] - # XXX: test .pkg files + + def test_extend_path_argument_types(self): + pkgname = 'foo' + dirname_0 = self.create_init(pkgname) + + # If the input path is not a list it is returned unchanged + self.assertEqual('notalist', pkgutil.extend_path('notalist', 'foo')) + self.assertEqual(('not', 'a', 'list'), pkgutil.extend_path(('not', 'a', 'list'), 'foo')) + self.assertEqual(123, pkgutil.extend_path(123, 'foo')) + self.assertEqual(None, pkgutil.extend_path(None, 'foo')) + + # Cleanup + shutil.rmtree(dirname_0) + del sys.path[0] + + + def test_extend_path_pkg_files(self): + pkgname = 'foo' + dirname_0 = self.create_init(pkgname) + + with open(os.path.join(dirname_0, 'bar.pkg'), 'w') as pkg_file: + pkg_file.write('\n'.join([ + 'baz', + '/foo/bar/baz', + '', + '#comment' + ])) + + extended_paths = pkgutil.extend_path(sys.path, 'bar') + + self.assertEqual(extended_paths[:-2], sys.path) + self.assertEqual(extended_paths[-2], 'baz') + self.assertEqual(extended_paths[-1], '/foo/bar/baz') + + # Cleanup + shutil.rmtree(dirname_0) + del sys.path[0] class NestedNamespacePackageTest(unittest.TestCase): @@ -535,67 +607,6 @@ class ImportlibMigrationTests(unittest.TestCase): # PEP 302 emulation in this module is in the process of being # deprecated in favour of importlib proper - @unittest.skipIf(__name__ == '__main__', 'not compatible with __main__') - @ignore_warnings(category=DeprecationWarning) - def test_get_loader_handles_missing_loader_attribute(self): - global __loader__ - this_loader = __loader__ - del __loader__ - try: - self.assertIsNotNone(pkgutil.get_loader(__name__)) - finally: - __loader__ = this_loader - - @ignore_warnings(category=DeprecationWarning) - def test_get_loader_handles_missing_spec_attribute(self): - name = 'spam' - mod = type(sys)(name) - del mod.__spec__ - with CleanImport(name): - sys.modules[name] = mod - loader = pkgutil.get_loader(name) - self.assertIsNone(loader) - - @ignore_warnings(category=DeprecationWarning) - def test_get_loader_handles_spec_attribute_none(self): - name = 'spam' - mod = type(sys)(name) - mod.__spec__ = None - with CleanImport(name): - sys.modules[name] = mod - loader = pkgutil.get_loader(name) - self.assertIsNone(loader) - - @ignore_warnings(category=DeprecationWarning) - def test_get_loader_None_in_sys_modules(self): - name = 'totally bogus' - sys.modules[name] = None - try: - loader = pkgutil.get_loader(name) - finally: - del sys.modules[name] - self.assertIsNone(loader) - - def test_get_loader_is_deprecated(self): - with check_warnings( - (r".*\bpkgutil.get_loader\b.*", DeprecationWarning), - ): - res = pkgutil.get_loader("sys") - self.assertIsNotNone(res) - - def test_find_loader_is_deprecated(self): - with check_warnings( - (r".*\bpkgutil.find_loader\b.*", DeprecationWarning), - ): - res = pkgutil.find_loader("sys") - self.assertIsNotNone(res) - - @ignore_warnings(category=DeprecationWarning) - def test_find_loader_missing_module(self): - name = 'totally bogus' - loader = pkgutil.find_loader(name) - self.assertIsNone(loader) - def test_get_importer_avoids_emulation(self): # We use an illegal path so *none* of the path hooks should fire with check_warnings() as w: diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 648e18d0150ef0..e04ad142061ad3 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -10,6 +10,14 @@ from test import support from test.support import os_helper +try: + # Some of the iOS tests need ctypes to operate. + # Confirm that the ctypes module is available + # is available. + import _ctypes +except ImportError: + _ctypes = None + FEDORA_OS_RELEASE = """\ NAME=Fedora VERSION="32 (Thirty Two)" @@ -75,6 +83,38 @@ def clear_caches(self): platform._uname_cache = None platform._os_release_cache = None + def test_invalidate_caches(self): + self.clear_caches() + + self.assertDictEqual(platform._platform_cache, {}) + self.assertDictEqual(platform._sys_version_cache, {}) + self.assertIsNone(platform._uname_cache) + self.assertIsNone(platform._os_release_cache) + + # fill the cached entries (some have side effects on others) + platform.platform() # for platform._platform_cache + platform.python_implementation() # for platform._sys_version_cache + platform.uname() # for platform._uname_cache + + # check that the cache are filled + self.assertNotEqual(platform._platform_cache, {}) + self.assertNotEqual(platform._sys_version_cache, {}) + self.assertIsNotNone(platform._uname_cache) + + try: + platform.freedesktop_os_release() + except OSError: + self.assertIsNone(platform._os_release_cache) + else: + self.assertIsNotNone(platform._os_release_cache) + + with self.subTest('clear platform caches'): + platform.invalidate_caches() + self.assertDictEqual(platform._platform_cache, {}) + self.assertDictEqual(platform._sys_version_cache, {}) + self.assertIsNone(platform._uname_cache) + self.assertIsNone(platform._os_release_cache) + def test_architecture(self): res = platform.architecture() @@ -219,6 +259,30 @@ def test_uname(self): self.assertEqual(res[-1], res.processor) self.assertEqual(len(res), 6) + if os.name == "posix": + uname = os.uname() + self.assertEqual(res.node, uname.nodename) + self.assertEqual(res.version, uname.version) + self.assertEqual(res.machine, uname.machine) + + if sys.platform == "android": + self.assertEqual(res.system, "Android") + self.assertEqual(res.release, platform.android_ver().release) + elif sys.platform == "ios": + # Platform module needs ctypes for full operation. If ctypes + # isn't available, there's no ObjC module, and dummy values are + # returned. + if _ctypes: + self.assertIn(res.system, {"iOS", "iPadOS"}) + self.assertEqual(res.release, platform.ios_ver().release) + else: + self.assertEqual(res.system, "") + self.assertEqual(res.release, "") + else: + self.assertEqual(res.system, uname.sysname) + self.assertEqual(res.release, uname.release) + + @unittest.skipUnless(sys.platform.startswith('win'), "windows only test") def test_uname_win32_without_wmi(self): def raises_oserror(*a): @@ -318,12 +382,44 @@ def raises_oserror(*a): platform._uname_cache = None def test_java_ver(self): - res = platform.java_ver() - if sys.platform == 'java': # Is never actually checked in CI - self.assertTrue(all(res)) + import re + msg = re.escape( + "'java_ver' is deprecated and slated for removal in Python 3.15" + ) + with self.assertWarnsRegex(DeprecationWarning, msg): + res = platform.java_ver() + self.assertEqual(len(res), 4) + @unittest.skipUnless(support.MS_WINDOWS, 'This test only makes sense on Windows') def test_win32_ver(self): - res = platform.win32_ver() + release1, version1, csd1, ptype1 = 'a', 'b', 'c', 'd' + res = platform.win32_ver(release1, version1, csd1, ptype1) + self.assertEqual(len(res), 4) + release, version, csd, ptype = res + if release: + # Currently, release names always come from internal dicts, + # but this could change over time. For now, we just check that + # release is something different from what we have passed. + self.assertNotEqual(release, release1) + if version: + # It is rather hard to test explicit version without + # going deep into the details. + self.assertIn('.', version) + for v in version.split('.'): + int(v) # should not fail + if csd: + self.assertTrue(csd.startswith('SP'), msg=csd) + if ptype: + if os.cpu_count() > 1: + self.assertIn('Multiprocessor', ptype) + else: + self.assertIn('Uniprocessor', ptype) + + @unittest.skipIf(support.MS_WINDOWS, 'This test only makes sense on non Windows') + def test_win32_ver_on_non_windows(self): + release, version, csd, ptype = 'a', '1.0', 'c', 'd' + res = platform.win32_ver(release, version, csd, ptype) + self.assertSequenceEqual(res, (release, version, csd, ptype), seq_type=tuple) def test_mac_ver(self): res = platform.mac_ver() @@ -377,6 +473,56 @@ def test_mac_ver_with_fork(self): # parent support.wait_process(pid, exitcode=0) + def test_ios_ver(self): + result = platform.ios_ver() + + # ios_ver is only fully available on iOS where ctypes is available. + if sys.platform == "ios" and _ctypes: + system, release, model, is_simulator = result + # Result is a namedtuple + self.assertEqual(result.system, system) + self.assertEqual(result.release, release) + self.assertEqual(result.model, model) + self.assertEqual(result.is_simulator, is_simulator) + + # We can't assert specific values without reproducing the logic of + # ios_ver(), so we check that the values are broadly what we expect. + + # System is either iOS or iPadOS, depending on the test device + self.assertIn(system, {"iOS", "iPadOS"}) + + # Release is a numeric version specifier with at least 2 parts + parts = release.split(".") + self.assertGreaterEqual(len(parts), 2) + self.assertTrue(all(part.isdigit() for part in parts)) + + # If this is a simulator, we get a high level device descriptor + # with no identifying model number. If this is a physical device, + # we get a model descriptor like "iPhone13,1" + if is_simulator: + self.assertIn(model, {"iPhone", "iPad"}) + else: + self.assertTrue( + (model.startswith("iPhone") or model.startswith("iPad")) + and "," in model + ) + + self.assertEqual(type(is_simulator), bool) + else: + # On non-iOS platforms, calling ios_ver doesn't fail; you get + # default values + self.assertEqual(result.system, "") + self.assertEqual(result.release, "") + self.assertEqual(result.model, "") + self.assertFalse(result.is_simulator) + + # Check the fallback values can be overridden by arguments + override = platform.ios_ver("Foo", "Bar", "Whiz", True) + self.assertEqual(override.system, "Foo") + self.assertEqual(override.release, "Bar") + self.assertEqual(override.model, "Whiz") + self.assertTrue(override.is_simulator) + @unittest.skipIf(support.is_emscripten, "Does not apply to Emscripten") def test_libc_ver(self): # check that libc_ver(executable) doesn't raise an exception @@ -426,6 +572,43 @@ def test_libc_ver(self): self.assertEqual(platform.libc_ver(filename, chunksize=chunksize), ('glibc', '1.23.4')) + def test_android_ver(self): + res = platform.android_ver() + self.assertIsInstance(res, tuple) + self.assertEqual(res, (res.release, res.api_level, res.manufacturer, + res.model, res.device, res.is_emulator)) + + if sys.platform == "android": + for name in ["release", "manufacturer", "model", "device"]: + with self.subTest(name): + value = getattr(res, name) + self.assertIsInstance(value, str) + self.assertNotEqual(value, "") + + self.assertIsInstance(res.api_level, int) + self.assertGreaterEqual(res.api_level, sys.getandroidapilevel()) + + self.assertIsInstance(res.is_emulator, bool) + + # When not running on Android, it should return the default values. + else: + self.assertEqual(res.release, "") + self.assertEqual(res.api_level, 0) + self.assertEqual(res.manufacturer, "") + self.assertEqual(res.model, "") + self.assertEqual(res.device, "") + self.assertEqual(res.is_emulator, False) + + # Default values may also be overridden using parameters. + res = platform.android_ver( + "alpha", 1, "bravo", "charlie", "delta", True) + self.assertEqual(res.release, "alpha") + self.assertEqual(res.api_level, 1) + self.assertEqual(res.manufacturer, "bravo") + self.assertEqual(res.model, "charlie") + self.assertEqual(res.device, "delta") + self.assertEqual(res.is_emulator, True) + @support.cpython_only def test__comparable_version(self): from platform import _comparable_version as V diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index 001f86f2893f2f..a0c76e5dec5ebe 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -1,4 +1,4 @@ -# Copyright (C) 2003-2013 Python Software Foundation +# Copyright (C) 2003 Python Software Foundation import copy import operator import pickle @@ -13,7 +13,6 @@ import subprocess import binascii import collections -import time import zoneinfo from test import support from test.support import os_helper diff --git a/Lib/test/test_poll.py b/Lib/test/test_poll.py index 1847ae95db9292..5675db8d1cab6e 100644 --- a/Lib/test/test_poll.py +++ b/Lib/test/test_poll.py @@ -172,7 +172,10 @@ def test_poll3(self): @cpython_only def test_poll_c_limits(self): - from _testcapi import USHRT_MAX, INT_MAX, UINT_MAX + try: + from _testcapi import USHRT_MAX, INT_MAX, UINT_MAX + except ImportError: + raise unittest.SkipTest("requires _testcapi") pollster = select.poll() pollster.register(1) diff --git a/Lib/test/test_positional_only_arg.py b/Lib/test/test_positional_only_arg.py index 1a193814d7535d..eea0625012da6d 100644 --- a/Lib/test/test_positional_only_arg.py +++ b/Lib/test/test_positional_only_arg.py @@ -2,6 +2,7 @@ import dis import pickle +import types import unittest from test.support import check_syntax_error @@ -440,7 +441,9 @@ def f(x: not (int is int), /): ... # without constant folding we end up with # COMPARE_OP(is), IS_OP (0) # with constant folding we should expect a IS_OP (1) - codes = [(i.opname, i.argval) for i in dis.get_instructions(g)] + code_obj = next(const for const in g.__code__.co_consts + if isinstance(const, types.CodeType) and const.co_name == "__annotate__") + codes = [(i.opname, i.argval) for i in dis.get_instructions(code_obj)] self.assertNotIn(('UNARY_NOT', None), codes) self.assertIn(('IS_OP', 1), codes) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 2706d5eb6d9830..c9cbe1541e733e 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -6,12 +6,14 @@ from test.support import warnings_helper from test.support.script_helper import assert_python_ok +import copy import errno import sys import signal import time import os import platform +import pickle import stat import tempfile import unittest @@ -411,8 +413,10 @@ def test_posix_fallocate(self): # issue33655: Also ignore EINVAL on *BSD since ZFS is also # often used there. if inst.errno == errno.EINVAL and sys.platform.startswith( - ('sunos', 'freebsd', 'netbsd', 'openbsd', 'gnukfreebsd')): + ('sunos', 'freebsd', 'openbsd', 'gnukfreebsd')): raise unittest.SkipTest("test may fail on ZFS filesystems") + elif inst.errno == errno.EOPNOTSUPP and sys.platform.startswith("netbsd"): + raise unittest.SkipTest("test may fail on FFS filesystems") else: raise finally: @@ -565,8 +569,37 @@ def test_dup(self): @unittest.skipUnless(hasattr(posix, 'confstr'), 'test needs posix.confstr()') def test_confstr(self): - self.assertRaises(ValueError, posix.confstr, "CS_garbage") - self.assertEqual(len(posix.confstr("CS_PATH")) > 0, True) + with self.assertRaisesRegex( + ValueError, "unrecognized configuration name" + ): + posix.confstr("CS_garbage") + + with self.assertRaisesRegex( + TypeError, "configuration names must be strings or integers" + ): + posix.confstr(1.23) + + path = posix.confstr("CS_PATH") + self.assertGreater(len(path), 0) + self.assertEqual(posix.confstr(posix.confstr_names["CS_PATH"]), path) + + @unittest.skipUnless(hasattr(posix, 'sysconf'), + 'test needs posix.sysconf()') + def test_sysconf(self): + with self.assertRaisesRegex( + ValueError, "unrecognized configuration name" + ): + posix.sysconf("SC_garbage") + + with self.assertRaisesRegex( + TypeError, "configuration names must be strings or integers" + ): + posix.sysconf(1.23) + + arg_max = posix.sysconf("SC_ARG_MAX") + self.assertGreater(arg_max, 0) + self.assertEqual( + posix.sysconf(posix.sysconf_names["SC_ARG_MAX"]), arg_max) @unittest.skipUnless(hasattr(posix, 'dup2'), 'test needs posix.dup2()') @@ -703,7 +736,8 @@ def test_makedev(self): self.assertEqual(posix.major(dev), major) self.assertRaises(TypeError, posix.major, float(dev)) self.assertRaises(TypeError, posix.major) - self.assertRaises((ValueError, OverflowError), posix.major, -1) + for x in -2, 2**64, -2**63-1: + self.assertRaises((ValueError, OverflowError), posix.major, x) minor = posix.minor(dev) self.assertIsInstance(minor, int) @@ -711,13 +745,23 @@ def test_makedev(self): self.assertEqual(posix.minor(dev), minor) self.assertRaises(TypeError, posix.minor, float(dev)) self.assertRaises(TypeError, posix.minor) - self.assertRaises((ValueError, OverflowError), posix.minor, -1) + for x in -2, 2**64, -2**63-1: + self.assertRaises((ValueError, OverflowError), posix.minor, x) self.assertEqual(posix.makedev(major, minor), dev) self.assertRaises(TypeError, posix.makedev, float(major), minor) self.assertRaises(TypeError, posix.makedev, major, float(minor)) self.assertRaises(TypeError, posix.makedev, major) self.assertRaises(TypeError, posix.makedev) + for x in -2, 2**32, 2**64, -2**63-1: + self.assertRaises((ValueError, OverflowError), posix.makedev, x, minor) + self.assertRaises((ValueError, OverflowError), posix.makedev, major, x) + + if sys.platform == 'linux': + NODEV = -1 + self.assertEqual(posix.major(NODEV), NODEV) + self.assertEqual(posix.minor(NODEV), NODEV) + self.assertEqual(posix.makedev(NODEV, NODEV), NODEV) def _test_all_chown_common(self, chown_func, first_param, stat_func): """Common code for chown, fchown and lchown tests.""" @@ -1257,7 +1301,7 @@ def test_sched_priority(self): self.assertIsInstance(lo, int) self.assertIsInstance(hi, int) self.assertGreaterEqual(hi, lo) - # Apple plaforms return 15 without checking the argument. + # Apple platforms return 15 without checking the argument. if not is_apple: self.assertRaises(OSError, posix.sched_get_priority_min, -23) self.assertRaises(OSError, posix.sched_get_priority_max, -23) @@ -1303,6 +1347,25 @@ def test_get_and_set_scheduler_and_param(self): param = posix.sched_param(sched_priority=-large) self.assertRaises(OverflowError, posix.sched_setparam, 0, param) + @requires_sched + def test_sched_param(self): + param = posix.sched_param(1) + for proto in range(pickle.HIGHEST_PROTOCOL+1): + newparam = pickle.loads(pickle.dumps(param, proto)) + self.assertEqual(newparam, param) + newparam = copy.copy(param) + self.assertIsNot(newparam, param) + self.assertEqual(newparam, param) + newparam = copy.deepcopy(param) + self.assertIsNot(newparam, param) + self.assertEqual(newparam, param) + newparam = copy.replace(param) + self.assertIsNot(newparam, param) + self.assertEqual(newparam, param) + newparam = copy.replace(param, sched_priority=0) + self.assertNotEqual(newparam, param) + self.assertEqual(newparam.sched_priority, 0) + @unittest.skipUnless(hasattr(posix, "sched_rr_get_interval"), "no function") def test_sched_rr_get_interval(self): try: @@ -1335,12 +1398,21 @@ def test_sched_getaffinity(self): def test_sched_setaffinity(self): mask = posix.sched_getaffinity(0) self.addCleanup(posix.sched_setaffinity, 0, list(mask)) + if len(mask) > 1: # Empty masks are forbidden mask.pop() posix.sched_setaffinity(0, mask) self.assertEqual(posix.sched_getaffinity(0), mask) - self.assertRaises(OSError, posix.sched_setaffinity, 0, []) + + try: + posix.sched_setaffinity(0, []) + # gh-117061: On RHEL9, sched_setaffinity(0, []) does not fail + except OSError: + # sched_setaffinity() manual page documents EINVAL error + # when the mask is empty. + pass + self.assertRaises(ValueError, posix.sched_setaffinity, 0, [-10]) self.assertRaises(ValueError, posix.sched_setaffinity, 0, map(int, "0X")) self.assertRaises(OverflowError, posix.sched_setaffinity, 0, [1<<128]) @@ -2119,6 +2191,13 @@ def test_stat(self): with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): os.stat("file", dir_fd=0) + def test_ptsname_r(self): + self._verify_available("HAVE_PTSNAME_R") + if self.mac_ver >= (10, 13, 4): + self.assertIn("HAVE_PTSNAME_R", posix._have_functions) + else: + self.assertNotIn("HAVE_PTSNAME_R", posix._have_functions) + def test_access(self): self._verify_available("HAVE_FACCESSAT") if self.mac_ver >= (10, 10): diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index cbb7c4c52d9697..43e4fbc610e5f7 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -1,11 +1,12 @@ +import inspect import os import posixpath import sys import unittest from posixpath import realpath, abspath, dirname, basename from test import test_genericpath -from test.support import import_helper -from test.support import os_helper +from test.support import get_attribute, import_helper +from test.support import cpython_only, os_helper from test.support.os_helper import FakePath from unittest import mock @@ -56,6 +57,8 @@ def test_join(self): self.assertEqual(fn(b"/foo", b"bar", b"baz"), b"/foo/bar/baz") self.assertEqual(fn(b"/foo/", b"bar/", b"baz/"), b"/foo/bar/baz/") + self.assertEqual(fn("a", ""), "a/") + self.assertEqual(fn("a", "", ""), "a/") self.assertEqual(fn("a", "b"), "a/b") self.assertEqual(fn("a", "b/"), "a/b/") self.assertEqual(fn("a/", "b"), "a/b") @@ -281,6 +284,16 @@ def fake_lstat(path): def test_isjunction(self): self.assertFalse(posixpath.isjunction(ABSTFN)) + @unittest.skipIf(sys.platform == 'win32', "Fast paths are not for win32") + @cpython_only + def test_fast_paths_in_use(self): + # There are fast paths of these functions implemented in posixmodule.c. + # Confirm that they are being used, and not the Python fallbacks + self.assertTrue(os.path.splitroot is posix._path_splitroot_ex) + self.assertFalse(inspect.isfunction(os.path.splitroot)) + self.assertTrue(os.path.normpath is posix._path_normpath) + self.assertFalse(inspect.isfunction(os.path.normpath)) + def test_expanduser(self): self.assertEqual(posixpath.expanduser("foo"), "foo") self.assertEqual(posixpath.expanduser(b"foo"), b"foo") @@ -342,12 +355,31 @@ def test_expanduser_pwd(self): for path in ('~', '~/.local', '~vstinner/'): self.assertEqual(posixpath.expanduser(path), path) + @unittest.skipIf(sys.platform == "vxworks", + "no home directory on VxWorks") + def test_expanduser_pwd2(self): + pwd = import_helper.import_module('pwd') + for all_entry in get_attribute(pwd, 'getpwall')(): + name = all_entry.pw_name + + # gh-121200: pw_dir can be different between getpwall() and + # getpwnam(), so use getpwnam() pw_dir as expanduser() does. + entry = pwd.getpwnam(name) + home = entry.pw_dir + home = home.rstrip('/') or '/' + + with self.subTest(all_entry=all_entry, entry=entry): + self.assertEqual(posixpath.expanduser('~' + name), home) + self.assertEqual(posixpath.expanduser(os.fsencode('~' + name)), + os.fsencode(home)) + NORMPATH_CASES = [ ("", "."), ("/", "/"), ("/.", "/"), ("/./", "/"), ("/.//.", "/"), + ("/./foo/bar", "/foo/bar"), ("/foo", "/foo"), ("/foo/bar", "/foo/bar"), ("//", "//"), @@ -357,6 +389,7 @@ def test_expanduser_pwd(self): ("///..//./foo/.//bar", "/foo/bar"), (".", "."), (".//.", "."), + ("./foo/bar", "foo/bar"), ("..", ".."), ("../", ".."), ("../foo", "../foo"), @@ -456,6 +489,15 @@ def test_realpath_relative(self): finally: os_helper.unlink(ABSTFN) + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + def test_realpath_missing_pardir(self): + try: + os.symlink(os_helper.TESTFN + "1", os_helper.TESTFN) + self.assertEqual(realpath("nonexistent/../" + os_helper.TESTFN), ABSTFN + "1") + finally: + os_helper.unlink(os_helper.TESTFN) + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash def test_realpath_symlink_loops(self): @@ -475,7 +517,7 @@ def test_realpath_symlink_loops(self): self.assertEqual(realpath(ABSTFN+"1/../x"), dirname(ABSTFN) + "/x") os.symlink(ABSTFN+"x", ABSTFN+"y") self.assertEqual(realpath(ABSTFN+"1/../" + basename(ABSTFN) + "y"), - ABSTFN + "y") + ABSTFN + "x") self.assertEqual(realpath(ABSTFN+"1/../" + basename(ABSTFN) + "1"), ABSTFN + "1") @@ -637,10 +679,88 @@ def test_realpath_resolve_first(self): safe_rmdir(ABSTFN + "/k") safe_rmdir(ABSTFN) + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + @unittest.skipIf(os.chmod not in os.supports_follow_symlinks, "Can't set symlink permissions") + @unittest.skipIf(sys.platform != "darwin", "only macOS requires read permission to readlink()") + def test_realpath_unreadable_symlink(self): + try: + os.symlink(ABSTFN+"1", ABSTFN) + os.chmod(ABSTFN, 0o000, follow_symlinks=False) + self.assertEqual(realpath(ABSTFN), ABSTFN) + self.assertEqual(realpath(ABSTFN + '/foo'), ABSTFN + '/foo') + self.assertEqual(realpath(ABSTFN + '/../foo'), dirname(ABSTFN) + '/foo') + self.assertEqual(realpath(ABSTFN + '/foo/..'), ABSTFN) + with self.assertRaises(PermissionError): + realpath(ABSTFN, strict=True) + finally: + os.chmod(ABSTFN, 0o755, follow_symlinks=False) + os.unlink(ABSTFN) + + @skip_if_ABSTFN_contains_backslash + def test_realpath_nonterminal_file(self): + try: + with open(ABSTFN, 'w') as f: + f.write('test_posixpath wuz ere') + self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN) + self.assertEqual(realpath(ABSTFN + "/", strict=False), ABSTFN) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=True) + self.assertEqual(realpath(ABSTFN + "/.", strict=False), ABSTFN) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=True) + self.assertEqual(realpath(ABSTFN + "/..", strict=False), dirname(ABSTFN)) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=True) + self.assertEqual(realpath(ABSTFN + "/subdir", strict=False), ABSTFN + "/subdir") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) + finally: + os_helper.unlink(ABSTFN) + + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + def test_realpath_nonterminal_symlink_to_file(self): + try: + with open(ABSTFN + "1", 'w') as f: + f.write('test_posixpath wuz ere') + os.symlink(ABSTFN + "1", ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN + "1") + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN + "1") + self.assertEqual(realpath(ABSTFN + "/", strict=False), ABSTFN + "1") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=True) + self.assertEqual(realpath(ABSTFN + "/.", strict=False), ABSTFN + "1") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=True) + self.assertEqual(realpath(ABSTFN + "/..", strict=False), dirname(ABSTFN)) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=True) + self.assertEqual(realpath(ABSTFN + "/subdir", strict=False), ABSTFN + "1/subdir") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) + finally: + os_helper.unlink(ABSTFN) + + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + def test_realpath_nonterminal_symlink_to_symlinks_to_file(self): + try: + with open(ABSTFN + "2", 'w') as f: + f.write('test_posixpath wuz ere') + os.symlink(ABSTFN + "2", ABSTFN + "1") + os.symlink(ABSTFN + "1", ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN + "2") + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN + "2") + self.assertEqual(realpath(ABSTFN + "/", strict=False), ABSTFN + "2") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=True) + self.assertEqual(realpath(ABSTFN + "/.", strict=False), ABSTFN + "2") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=True) + self.assertEqual(realpath(ABSTFN + "/..", strict=False), dirname(ABSTFN)) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=True) + self.assertEqual(realpath(ABSTFN + "/subdir", strict=False), ABSTFN + "2/subdir") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) + finally: + os_helper.unlink(ABSTFN) + def test_relpath(self): (real_getcwd, os.getcwd) = (os.getcwd, lambda: r"/home/user/bar") try: curdir = os.path.split(os.getcwd())[-1] + self.assertRaises(TypeError, posixpath.relpath, None) self.assertRaises(ValueError, posixpath.relpath, "") self.assertEqual(posixpath.relpath("a"), "a") self.assertEqual(posixpath.relpath(posixpath.abspath("a")), "a") diff --git a/Lib/test/test_pprint.py b/Lib/test/test_pprint.py index 4e6fed1ab969ac..dfbc2a06e7346f 100644 --- a/Lib/test/test_pprint.py +++ b/Lib/test/test_pprint.py @@ -8,7 +8,6 @@ import pprint import random import re -import test.support import types import unittest diff --git a/Lib/test/test_print.py b/Lib/test/test_print.py index 5f1bfd9e30db98..f4805a1d6c6602 100644 --- a/Lib/test/test_print.py +++ b/Lib/test/test_print.py @@ -188,38 +188,6 @@ def test_string_in_loop_on_same_line(self): self.assertIn("Missing parentheses in call to 'print'. Did you mean print(...)", str(context.exception)) - def test_stream_redirection_hint_for_py2_migration(self): - # Test correct hint produced for Py2 redirection syntax - with self.assertRaises(TypeError) as context: - print >> sys.stderr, "message" - self.assertIn('Did you mean "print(, ' - 'file=)"?', str(context.exception)) - - # Test correct hint is produced in the case where RHS implements - # __rrshift__ but returns NotImplemented - with self.assertRaises(TypeError) as context: - print >> 42 - self.assertIn('Did you mean "print(, ' - 'file=)"?', str(context.exception)) - - # Test stream redirection hint is specific to print - with self.assertRaises(TypeError) as context: - max >> sys.stderr - self.assertNotIn('Did you mean ', str(context.exception)) - - # Test stream redirection hint is specific to rshift - with self.assertRaises(TypeError) as context: - print << sys.stderr - self.assertNotIn('Did you mean', str(context.exception)) - - # Ensure right operand implementing rrshift still works - class OverrideRRShift: - def __rrshift__(self, lhs): - return 42 # Force result independent of LHS - - self.assertEqual(print >> OverrideRRShift(), 42) - - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_property.py b/Lib/test/test_property.py index 408e64f53142db..cea241b0f200d0 100644 --- a/Lib/test/test_property.py +++ b/Lib/test/test_property.py @@ -438,7 +438,7 @@ def getter3(self): self.assertEqual(p2.__doc__, "doc-A") # Case-3: with no user-provided doc new getter doc - # takes precendence + # takes precedence p = property(getter2, None, None, None) p2 = p.getter(getter3) @@ -463,6 +463,40 @@ def getter3(self): self.assertEqual(p.__doc__, "user") self.assertEqual(p2.__doc__, "user") + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_prefer_explicit_doc(self): + # Issue 25757: subclasses of property lose docstring + self.assertEqual(property(doc="explicit doc").__doc__, "explicit doc") + self.assertEqual(PropertySub(doc="explicit doc").__doc__, "explicit doc") + + class Foo: + spam = PropertySub(doc="spam explicit doc") + + @spam.getter + def spam(self): + """ignored as doc already set""" + return 1 + + def _stuff_getter(self): + """ignored as doc set directly""" + stuff = PropertySub(doc="stuff doc argument", fget=_stuff_getter) + + #self.assertEqual(Foo.spam.__doc__, "spam explicit doc") + self.assertEqual(Foo.stuff.__doc__, "stuff doc argument") + + def test_property_no_doc_on_getter(self): + # If a property's getter has no __doc__ then the property's doc should + # be None; test that this is consistent with subclasses as well; see + # GH-2487 + class NoDoc: + @property + def __doc__(self): + raise AttributeError + + self.assertEqual(property(NoDoc()).__doc__, None) + self.assertEqual(PropertySub(NoDoc()).__doc__, None) + @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def test_property_setter_copies_getter_docstring(self): diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index c7c5419ffe3e37..25b313f6c25a4e 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -31,14 +31,6 @@ def assertListEq(self, l1, l2, ignore): print("l1=%r\nl2=%r\nignore=%r" % (l1, l2, ignore), file=sys.stderr) self.fail("%r missing" % missing.pop()) - def assertHasattr(self, obj, attr, ignore): - ''' succeed iff hasattr(obj,attr) or attr in ignore. ''' - if attr in ignore: return - if not hasattr(obj, attr): print("???", attr) - self.assertTrue(hasattr(obj, attr), - 'expected hasattr(%r, %r)' % (obj, attr)) - - def assertHaskey(self, obj, key, ignore): ''' succeed iff key in obj or key in ignore. ''' if key in ignore: return @@ -78,14 +70,15 @@ def ismethod(oclass, obj, name): objname = obj.__name__ if objname.startswith("__") and not objname.endswith("__"): - objname = "_%s%s" % (oclass.__name__, objname) + if stripped_typename := oclass.__name__.lstrip('_'): + objname = f"_{stripped_typename}{objname}" return objname == name # Make sure the toplevel functions and classes are the same. for name, value in dict.items(): if name in ignore: continue - self.assertHasattr(module, name, ignore) + self.assertHasAttr(module, name, ignore) py_item = getattr(module, name) if isinstance(value, pyclbr.Function): self.assertIsInstance(py_item, (FunctionType, BuiltinFunctionType)) @@ -109,14 +102,20 @@ def ismethod(oclass, obj, name): actualMethods = [] for m in py_item.__dict__.keys(): + if m == "__annotate__": + continue if ismethod(py_item, getattr(py_item, m), m): actualMethods.append(m) - foundMethods = [] - for m in value.methods.keys(): - if m[:2] == '__' and m[-2:] != '__': - foundMethods.append('_'+name+m) - else: - foundMethods.append(m) + + if stripped_typename := name.lstrip('_'): + foundMethods = [] + for m in value.methods.keys(): + if m.startswith('__') and not m.endswith('__'): + foundMethods.append(f"_{stripped_typename}{m}") + else: + foundMethods.append(m) + else: + foundMethods = list(value.methods.keys()) try: self.assertListEq(foundMethods, actualMethods, ignore) @@ -150,8 +149,9 @@ def test_easy(self): "DocTestCase", '_DocTestSuite')) self.checkModule('difflib', ignore=("Match",)) - def test_decorators(self): - self.checkModule('test.pyclbr_input', ignore=['om']) + def test_cases(self): + # see test.pyclbr_input for the rationale behind the ignored symbols + self.checkModule('test.pyclbr_input', ignore=['om', 'f']) def test_nested(self): mb = pyclbr @@ -226,7 +226,7 @@ def test_others(self): cm( 'pdb', # pyclbr does not handle elegantly `typing` or properties - ignore=('Union', '_ModuleTarget', '_ScriptTarget'), + ignore=('Union', '_ModuleTarget', '_ScriptTarget', '_ZipTarget'), ) cm('pydoc', ignore=('input', 'output',)) # properties diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 9d40234ed01697..b02ba3aafd4d20 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -4,6 +4,7 @@ import contextlib import importlib.util import inspect +import io import pydoc import py_compile import keyword @@ -17,6 +18,7 @@ import types import typing import unittest +import unittest.mock import urllib.parse import xml.etree import xml.etree.ElementTree @@ -30,7 +32,7 @@ from test.support.script_helper import (assert_python_ok, assert_python_failure, spawn_python) from test.support import threading_helper -from test.support import (reap_children, captured_output, captured_stdout, +from test.support import (reap_children, captured_stdout, captured_stderr, is_emscripten, is_wasi, requires_docstrings, MISSING_C_DOCSTRINGS) from test.support.os_helper import (TESTFN, rmtree, unlink) @@ -76,6 +78,11 @@ class A(builtins.object) | __weakref__%s class B(builtins.object) + | Methods defined here: + | + | __annotate__(format, /) + | + | ---------------------------------------------------------------------- | Data descriptors defined here: | | __dict__%s @@ -86,8 +93,6 @@ class B(builtins.object) | Data and other attributes defined here: | | NO_MEANING = 'eggs' - | - | __annotations__ = {'NO_MEANING': } class C(builtins.object) | Methods defined here: @@ -175,6 +180,9 @@ class A(builtins.object) list of weak references to the object class B(builtins.object) + Methods defined here: + __annotate__(format, /) + ---------------------------------------------------------------------- Data descriptors defined here: __dict__ dictionary for instance variables @@ -183,7 +191,6 @@ class B(builtins.object) ---------------------------------------------------------------------- Data and other attributes defined here: NO_MEANING = 'eggs' - __annotations__ = {'NO_MEANING': } class C(builtins.object) @@ -379,6 +386,11 @@ def html2text(html): class PydocBaseTest(unittest.TestCase): + def tearDown(self): + # Self-testing. Mocking only works if sys.modules['pydoc'] and pydoc + # are the same. But some pydoc functions reload the module and change + # sys.modules, so check that it was restored. + self.assertIs(sys.modules['pydoc'], pydoc) def _restricted_walk_packages(self, walk_packages, path=None): """ @@ -410,6 +422,8 @@ def call_url_handler(self, url, expected_title): class PydocDocTest(unittest.TestCase): maxDiff = None + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __locals__ unexpectedly') @@ -450,6 +464,14 @@ class BinaryInteger(enum.IntEnum): doc = pydoc.render_doc(BinaryInteger) self.assertIn('BinaryInteger.zero', doc) + def test_slotted_dataclass_with_field_docs(self): + import dataclasses + @dataclasses.dataclass(slots=True) + class My: + x: int = dataclasses.field(doc='Docstring for x') + doc = pydoc.render_doc(My) + self.assertIn('Docstring for x', doc) + def test_mixed_case_module_names_are_lower_cased(self): # issue16484 doc_link = get_pydoc_link(xml.etree.ElementTree) @@ -534,6 +556,14 @@ class object | ... and 82 other subclasses """ doc = pydoc.TextDoc() + try: + # Make sure HeapType, which has no __module__ attribute, is one + # of the known subclasses of object. (doc.docclass() used to + # fail if HeapType was imported before running this test, like + # when running tests sequentially.) + from _testcapi import HeapType + except ImportError: + pass text = doc.docclass(object) snip = (" | Built-in subclasses:\n" " | async_generator\n" @@ -658,16 +688,13 @@ def test_fail_help_output_redirect(self): @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __locals__ unexpectedly') + @unittest.mock.patch('pydoc.pager') @requires_docstrings - def test_help_output_redirect(self): + def test_help_output_redirect(self, pager_mock): # issue 940286, if output is set in Helper, then all output from # Helper.help should be redirected - getpager_old = pydoc.getpager - getpager_new = lambda: (lambda x: x) self.maxDiff = None - buf = StringIO() - helper = pydoc.Helper(output=buf) unused, doc_loc = get_pydoc_text(pydoc_mod) module = "test.test_pydoc.pydoc_mod" help_header = """ @@ -677,26 +704,160 @@ def test_help_output_redirect(self): help_header = textwrap.dedent(help_header) expected_help_pattern = help_header + expected_text_pattern - pydoc.getpager = getpager_new - try: - with captured_output('stdout') as output, \ - captured_output('stderr') as err: - helper.help(module) + with captured_stdout() as output, captured_stderr() as err: + buf = StringIO() + helper = pydoc.Helper(output=buf) + helper.help(module) + result = buf.getvalue().strip() + expected_text = expected_help_pattern % ( + (doc_loc,) + + expected_text_data_docstrings + + (inspect.getabsfile(pydoc_mod),)) + self.assertEqual('', output.getvalue()) + self.assertEqual('', err.getvalue()) + self.assertEqual(expected_text, result) + + pager_mock.assert_not_called() + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + @unittest.mock.patch('pydoc.pager') + def test_help_output_redirect_various_requests(self, pager_mock): + # issue 940286, if output is set in Helper, then all output from + # Helper.help should be redirected + + def run_pydoc_for_request(request, expected_text_part): + """Helper function to run pydoc with its output redirected""" + with captured_stdout() as output, captured_stderr() as err: + buf = StringIO() + helper = pydoc.Helper(output=buf) + helper.help(request) result = buf.getvalue().strip() - expected_text = expected_help_pattern % ( - (doc_loc,) + - expected_text_data_docstrings + - (inspect.getabsfile(pydoc_mod),)) - self.assertEqual('', output.getvalue()) + self.assertEqual('', output.getvalue(), msg=f'failed on request "{request}"') + self.assertEqual('', err.getvalue(), msg=f'failed on request "{request}"') + self.assertIn(expected_text_part, result, msg=f'failed on request "{request}"') + pager_mock.assert_not_called() + + self.maxDiff = None + + # test for "keywords" + run_pydoc_for_request('keywords', 'Here is a list of the Python keywords.') + # test for "symbols" + run_pydoc_for_request('symbols', 'Here is a list of the punctuation symbols') + # test for "topics" + run_pydoc_for_request('topics', 'Here is a list of available topics.') + # test for "modules" skipped, see test_modules() + # test for symbol "%" + run_pydoc_for_request('%', 'The power operator') + # test for special True, False, None keywords + run_pydoc_for_request('True', 'class bool(int)') + run_pydoc_for_request('False', 'class bool(int)') + run_pydoc_for_request('None', 'class NoneType(object)') + # test for keyword "assert" + run_pydoc_for_request('assert', 'The "assert" statement') + # test for topic "TYPES" + run_pydoc_for_request('TYPES', 'The standard type hierarchy') + # test for "pydoc.Helper.help" + run_pydoc_for_request('pydoc.Helper.help', 'Help on function help in pydoc.Helper:') + # test for pydoc.Helper.help + run_pydoc_for_request(pydoc.Helper.help, 'Help on function help in module pydoc:') + # test for pydoc.Helper() instance skipped because it is always meant to be interactive + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + def test_help_output_pager(self): + def run_pydoc_pager(request, what, expected_first_line): + with (captured_stdout() as output, + captured_stderr() as err, + unittest.mock.patch('pydoc.pager') as pager_mock, + self.subTest(repr(request))): + helper = pydoc.Helper() + helper.help(request) self.assertEqual('', err.getvalue()) - self.assertEqual(expected_text, result) - finally: - pydoc.getpager = getpager_old + self.assertEqual('\n', output.getvalue()) + pager_mock.assert_called_once() + result = clean_text(pager_mock.call_args.args[0]) + self.assertEqual(result.splitlines()[0], expected_first_line) + self.assertEqual(pager_mock.call_args.args[1], f'Help on {what}') + + run_pydoc_pager('%', 'EXPRESSIONS', 'Operator precedence') + run_pydoc_pager('True', 'bool object', 'Help on bool object:') + run_pydoc_pager(True, 'bool object', 'Help on bool object:') + run_pydoc_pager('assert', 'assert', 'The "assert" statement') + run_pydoc_pager('TYPES', 'TYPES', 'The standard type hierarchy') + run_pydoc_pager('pydoc.Helper.help', 'pydoc.Helper.help', + 'Help on function help in pydoc.Helper:') + run_pydoc_pager(pydoc.Helper.help, 'Helper.help', + 'Help on function help in module pydoc:') + run_pydoc_pager('str', 'str', 'Help on class str in module builtins:') + run_pydoc_pager(str, 'str', 'Help on class str in module builtins:') + run_pydoc_pager('str.upper', 'str.upper', + 'Help on method descriptor upper in str:') + run_pydoc_pager(str.upper, 'str.upper', + 'Help on method descriptor upper:') + run_pydoc_pager(''.upper, 'str.upper', + 'Help on built-in function upper:') + run_pydoc_pager(str.__add__, + 'str.__add__', 'Help on method descriptor __add__:') + run_pydoc_pager(''.__add__, + 'str.__add__', 'Help on method wrapper __add__:') + run_pydoc_pager(int.numerator, 'int.numerator', + 'Help on getset descriptor builtins.int.numerator:') + run_pydoc_pager(list[int], 'list', + 'Help on GenericAlias in module builtins:') + run_pydoc_pager('sys', 'sys', 'Help on built-in module sys:') + run_pydoc_pager(sys, 'sys', 'Help on built-in module sys:') + + def test_showtopic(self): + with captured_stdout() as showtopic_io: + helper = pydoc.Helper() + helper.showtopic('with') + helptext = showtopic_io.getvalue() + self.assertIn('The "with" statement', helptext) + + def test_fail_showtopic(self): + with captured_stdout() as showtopic_io: + helper = pydoc.Helper() + helper.showtopic('abd') + expected = "no documentation found for 'abd'" + self.assertEqual(expected, showtopic_io.getvalue().strip()) + + @unittest.mock.patch('pydoc.pager') + def test_fail_showtopic_output_redirect(self, pager_mock): + with StringIO() as buf: + helper = pydoc.Helper(output=buf) + helper.showtopic("abd") + expected = "no documentation found for 'abd'" + self.assertEqual(expected, buf.getvalue().strip()) + + pager_mock.assert_not_called() + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + @unittest.mock.patch('pydoc.pager') + def test_showtopic_output_redirect(self, pager_mock): + # issue 940286, if output is set in Helper, then all output from + # Helper.showtopic should be redirected + self.maxDiff = None + + with captured_stdout() as output, captured_stderr() as err: + buf = StringIO() + helper = pydoc.Helper(output=buf) + helper.showtopic('with') + result = buf.getvalue().strip() + self.assertEqual('', output.getvalue()) + self.assertEqual('', err.getvalue()) + self.assertIn('The "with" statement', result) + + pager_mock.assert_not_called() def test_lambda_with_return_annotation(self): func = lambda a, b, c: 1 func.__annotations__ = {"return": int} - with captured_output('stdout') as help_io: + with captured_stdout() as help_io: pydoc.help(func) helptext = help_io.getvalue() self.assertIn("lambda (a, b, c) -> int", helptext) @@ -704,7 +865,7 @@ def test_lambda_with_return_annotation(self): def test_lambda_without_return_annotation(self): func = lambda a, b, c: 1 func.__annotations__ = {"a": int, "b": int, "c": int} - with captured_output('stdout') as help_io: + with captured_stdout() as help_io: pydoc.help(func) helptext = help_io.getvalue() self.assertIn("lambda (a: int, b: int, c: int)", helptext) @@ -712,7 +873,7 @@ def test_lambda_without_return_annotation(self): def test_lambda_with_return_and_params_annotation(self): func = lambda a, b, c: 1 func.__annotations__ = {"a": int, "b": int, "c": int, "return": int} - with captured_output('stdout') as help_io: + with captured_stdout() as help_io: pydoc.help(func) helptext = help_io.getvalue() self.assertIn("lambda (a: int, b: int, c: int) -> int", helptext) @@ -747,6 +908,82 @@ def test_synopsis(self): synopsis = pydoc.synopsis(TESTFN, {}) self.assertEqual(synopsis, 'line 1: h\xe9') + def test_source_synopsis(self): + def check(source, expected, encoding=None): + if isinstance(source, str): + source_file = StringIO(source) + else: + source_file = io.TextIOWrapper(io.BytesIO(source), encoding=encoding) + with source_file: + result = pydoc.source_synopsis(source_file) + self.assertEqual(result, expected) + + check('"""Single line docstring."""', + 'Single line docstring.') + check('"""First line of docstring.\nSecond line.\nThird line."""', + 'First line of docstring.') + check('"""First line of docstring.\\nSecond line.\\nThird line."""', + 'First line of docstring.') + check('""" Whitespace around docstring. """', + 'Whitespace around docstring.') + check('import sys\n"""No docstring"""', + None) + check(' \n"""Docstring after empty line."""', + 'Docstring after empty line.') + check('# Comment\n"""Docstring after comment."""', + 'Docstring after comment.') + check(' # Indented comment\n"""Docstring after comment."""', + 'Docstring after comment.') + check('""""""', # Empty docstring + '') + check('', # Empty file + None) + check('"""Embedded\0null byte"""', + None) + check('"""Embedded null byte"""\0', + None) + check('"""Café and résumé."""', + 'Café and résumé.') + check("'''Triple single quotes'''", + 'Triple single quotes') + check('"Single double quotes"', + 'Single double quotes') + check("'Single single quotes'", + 'Single single quotes') + check('"""split\\\nline"""', + 'splitline') + check('"""Unrecognized escape \\sequence"""', + 'Unrecognized escape \\sequence') + check('"""Invalid escape seq\\uence"""', + None) + check('r"""Raw \\stri\\ng"""', + 'Raw \\stri\\ng') + check('b"""Bytes literal"""', + None) + check('f"""f-string"""', + None) + check('"""Concatenated""" \\\n"string" \'literals\'', + 'Concatenatedstringliterals') + check('"""String""" + """expression"""', + None) + check('("""In parentheses""")', + 'In parentheses') + check('("""Multiple lines """\n"""in parentheses""")', + 'Multiple lines in parentheses') + check('()', # tuple + None) + check(b'# coding: iso-8859-15\n"""\xa4uro sign"""', + '€uro sign', encoding='iso-8859-15') + check(b'"""\xa4"""', # Decoding error + None, encoding='utf-8') + + with tempfile.NamedTemporaryFile(mode='w+', encoding='utf-8') as temp_file: + temp_file.write('"""Real file test."""\n') + temp_file.flush() + temp_file.seek(0) + result = pydoc.source_synopsis(temp_file) + self.assertEqual(result, "Real file test.") + @requires_docstrings def test_synopsis_sourceless(self): os = import_helper.import_fresh_module('os') @@ -921,7 +1158,7 @@ def __init__(self, class A(builtins.object) | A( - | arg1: collections.abc.Callable[[int, int, int], str], + | arg1: Callable[[int, int, int], str], | arg2: Literal['some value', 'other value'], | arg3: Annotated[int, 'some docs about this type'] | ) -> None @@ -930,7 +1167,7 @@ class A(builtins.object) | | __init__( | self, - | arg1: collections.abc.Callable[[int, int, int], str], + | arg1: Callable[[int, int, int], str], | arg2: Literal['some value', 'other value'], | arg3: Annotated[int, 'some docs about this type'] | ) -> None @@ -957,7 +1194,7 @@ def func( self.assertEqual(doc, '''Python Library Documentation: function func in module %s func( - arg1: collections.abc.Callable[[typing.Annotated[int, 'Some doc']], str], + arg1: Callable[[Annotated[int, 'Some doc']], str], arg2: Literal[1, 2, 3, 4, 5, 6, 7, 8] ) -> Annotated[int, 'Some other'] ''' % __name__) @@ -1072,7 +1309,6 @@ def test_apropos_with_unreadable_dir(self): self.assertEqual(err.getvalue(), '') @os_helper.skip_unless_working_chmod - @unittest.skipIf(is_emscripten, "cannot remove x bit") def test_apropos_empty_doc(self): pkgdir = os.path.join(TESTFN, 'walkpkg') os.mkdir(pkgdir) @@ -1102,7 +1338,8 @@ def test_url_search_package_error(self): sys.path.insert(0, TESTFN) try: with self.assertRaisesRegex(ValueError, "ouch"): - import test_error_package # Sanity check + # Sanity check + import test_error_package # noqa: F401 text = self.call_url_handler("search?key=test_error_package", "Pydoc: Search Results") @@ -1154,12 +1391,15 @@ def test_modules_search_builtin(self): self.assertTrue(result.startswith(expected)) def test_importfile(self): - loaded_pydoc = pydoc.importfile(pydoc.__file__) + try: + loaded_pydoc = pydoc.importfile(pydoc.__file__) - self.assertIsNot(loaded_pydoc, pydoc) - self.assertEqual(loaded_pydoc.__name__, 'pydoc') - self.assertEqual(loaded_pydoc.__file__, pydoc.__file__) - self.assertEqual(loaded_pydoc.__spec__, pydoc.__spec__) + self.assertIsNot(loaded_pydoc, pydoc) + self.assertEqual(loaded_pydoc.__name__, 'pydoc') + self.assertEqual(loaded_pydoc.__file__, pydoc.__file__) + self.assertEqual(loaded_pydoc.__spec__, pydoc.__spec__) + finally: + sys.modules['pydoc'] = pydoc class Rect: @@ -1174,6 +1414,8 @@ class Square(Rect): class TestDescriptions(unittest.TestCase): + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) def test_module(self): # Check that pydocfodder module can be described @@ -1236,8 +1478,8 @@ def foo(data: typing.List[typing.Any], T = typing.TypeVar('T') class C(typing.Generic[T], typing.Mapping[int, str]): ... self.assertEqual(pydoc.render_doc(foo).splitlines()[-1], - 'f\x08fo\x08oo\x08o(data: List[Any], x: int)' - ' -> Iterator[Tuple[int, Any]]') + 'f\x08fo\x08oo\x08o(data: typing.List[typing.Any], x: int)' + ' -> typing.Iterator[typing.Tuple[int, typing.Any]]') self.assertEqual(pydoc.render_doc(C).splitlines()[2], 'class C\x08C(collections.abc.Mapping, typing.Generic)') @@ -1372,7 +1614,7 @@ def test_bound_builtin_classmethod_o(self): @support.cpython_only @requires_docstrings def test_module_level_callable_unrepresentable_default(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") builtin = _testcapi.func_with_unrepresentable_signature self.assertEqual(self._get_summary_line(builtin), "func_with_unrepresentable_signature(a, b=)") @@ -1382,7 +1624,7 @@ def test_module_level_callable_unrepresentable_default(self): def test_builtin_staticmethod_unrepresentable_default(self): self.assertEqual(self._get_summary_line(str.maketrans), "maketrans(x, y=, z=, /)") - import _testcapi + _testcapi = import_helper.import_module("_testcapi") cls = _testcapi.DocStringUnrepresentableSignatureTest self.assertEqual(self._get_summary_line(cls.staticmeth), "staticmeth(a, b=)") @@ -1393,7 +1635,7 @@ def test_unbound_builtin_method_unrepresentable_default(self): self.assertEqual(self._get_summary_line(dict.pop), "pop(self, key, default=, /) " "unbound builtins.dict method") - import _testcapi + _testcapi = import_helper.import_module("_testcapi") cls = _testcapi.DocStringUnrepresentableSignatureTest self.assertEqual(self._get_summary_line(cls.meth), "meth(self, /, a, b=) unbound " @@ -1405,7 +1647,7 @@ def test_bound_builtin_method_unrepresentable_default(self): self.assertEqual(self._get_summary_line({}.pop), "pop(key, default=, /) " "method of builtins.dict instance") - import _testcapi + _testcapi = import_helper.import_module("_testcapi") obj = _testcapi.DocStringUnrepresentableSignatureTest() self.assertEqual(self._get_summary_line(obj.meth), "meth(a, b=) " @@ -1414,7 +1656,7 @@ def test_bound_builtin_method_unrepresentable_default(self): @support.cpython_only @requires_docstrings def test_unbound_builtin_classmethod_unrepresentable_default(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") cls = _testcapi.DocStringUnrepresentableSignatureTest descr = cls.__dict__['classmeth'] self.assertEqual(self._get_summary_line(descr), @@ -1424,7 +1666,7 @@ def test_unbound_builtin_classmethod_unrepresentable_default(self): @support.cpython_only @requires_docstrings def test_bound_builtin_classmethod_unrepresentable_default(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") cls = _testcapi.DocStringUnrepresentableSignatureTest self.assertEqual(self._get_summary_line(cls.classmeth), "classmeth(a, b=) class method of " @@ -1663,6 +1905,8 @@ def a_fn_with_https_link(): class PydocFodderTest(unittest.TestCase): + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) def getsection(self, text, beginline, endline): lines = text.splitlines() @@ -1802,6 +2046,8 @@ def test_html_doc_routines_in_module(self): ) class PydocServerTest(unittest.TestCase): """Tests for pydoc._start_server""" + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) def test_server(self): # Minimal test that starts the server, checks that it works, then stops @@ -1864,9 +2110,14 @@ def test_url_requests(self): ("foobar", "Pydoc: Error - foobar"), ] - with self.restrict_walk_packages(): - for url, title in requests: - self.call_url_handler(url, title) + self.assertIs(sys.modules['pydoc'], pydoc) + try: + with self.restrict_walk_packages(): + for url, title in requests: + self.call_url_handler(url, title) + finally: + # Some requests reload the module and change sys.modules. + sys.modules['pydoc'] = pydoc class TestHelper(unittest.TestCase): @@ -1876,6 +2127,9 @@ def test_keywords(self): class PydocWithMetaClasses(unittest.TestCase): + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __locals__ unexpectedly') @requires_docstrings diff --git a/Lib/test/test_pyrepl/__init__.py b/Lib/test/test_pyrepl/__init__.py new file mode 100644 index 00000000000000..8359d9844623c2 --- /dev/null +++ b/Lib/test/test_pyrepl/__init__.py @@ -0,0 +1,15 @@ +import os +import sys +from test.support import requires, load_package_tests +from test.support.import_helper import import_module + +if sys.platform != "win32": + # On non-Windows platforms, testing pyrepl currently requires that the + # 'curses' resource be given on the regrtest command line using the -u + # option. Additionally, we need to attempt to import curses and readline. + requires("curses") + curses = import_module("curses") + + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_pyrepl/__main__.py b/Lib/test/test_pyrepl/__main__.py new file mode 100644 index 00000000000000..cbe9e01d0df820 --- /dev/null +++ b/Lib/test/test_pyrepl/__main__.py @@ -0,0 +1,4 @@ +import unittest +from test.test_pyrepl import load_tests + +unittest.main() diff --git a/Lib/test/test_pyrepl/support.py b/Lib/test/test_pyrepl/support.py new file mode 100644 index 00000000000000..45e3bf758f17de --- /dev/null +++ b/Lib/test/test_pyrepl/support.py @@ -0,0 +1,158 @@ +import os +from code import InteractiveConsole +from functools import partial +from typing import Iterable +from unittest.mock import MagicMock + +from _pyrepl.console import Console, Event +from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig +from _pyrepl.simple_interact import _strip_final_indent + + +def multiline_input(reader: ReadlineAlikeReader, namespace: dict | None = None): + saved = reader.more_lines + try: + reader.more_lines = partial(more_lines, namespace=namespace) + reader.ps1 = reader.ps2 = ">>>" + reader.ps3 = reader.ps4 = "..." + return reader.readline() + finally: + reader.more_lines = saved + reader.paste_mode = False + + +def more_lines(text: str, namespace: dict | None = None): + if namespace is None: + namespace = {} + src = _strip_final_indent(text) + console = InteractiveConsole(namespace, filename="") + try: + code = console.compile(src, "", "single") + except (OverflowError, SyntaxError, ValueError): + return False + else: + return code is None + + +def code_to_events(code: str): + for c in code: + yield Event(evt="key", data=c, raw=bytearray(c.encode("utf-8"))) + + +def clean_screen(screen: Iterable[str]): + """Cleans color and console characters out of a screen output. + + This is useful for screen testing, it increases the test readability since + it strips out all the unreadable side of the screen. + """ + output = [] + for line in screen: + if line.startswith(">>>") or line.startswith("..."): + line = line[3:] + output.append(line) + return "\n".join(output).strip() + + +def prepare_reader(console: Console, **kwargs): + config = ReadlineConfig(readline_completer=kwargs.pop("readline_completer", None)) + reader = ReadlineAlikeReader(console=console, config=config) + reader.more_lines = partial(more_lines, namespace=None) + reader.paste_mode = True # Avoid extra indents + + def get_prompt(lineno, cursor_on_line) -> str: + return "" + + reader.get_prompt = get_prompt # Remove prompt for easier calculations of (x, y) + + for key, val in kwargs.items(): + setattr(reader, key, val) + + return reader + + +def prepare_console(events: Iterable[Event], **kwargs) -> MagicMock | Console: + console = MagicMock() + console.get_event.side_effect = events + console.height = 100 + console.width = 80 + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + +def handle_all_events( + events, prepare_console=prepare_console, prepare_reader=prepare_reader +): + console = prepare_console(events) + reader = prepare_reader(console) + try: + while True: + reader.handle1() + except StopIteration: + pass + except KeyboardInterrupt: + pass + return reader, console + + +handle_events_narrow_console = partial( + handle_all_events, + prepare_console=partial(prepare_console, width=10), +) + + +class FakeConsole(Console): + def __init__(self, events, encoding="utf-8") -> None: + self.events = iter(events) + self.encoding = encoding + self.screen = [] + self.height = 100 + self.width = 80 + + def get_event(self, block: bool = True) -> Event | None: + return next(self.events) + + def getpending(self) -> Event: + return self.get_event(block=False) + + def getheightwidth(self) -> tuple[int, int]: + return self.height, self.width + + def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: + pass + + def prepare(self) -> None: + pass + + def restore(self) -> None: + pass + + def move_cursor(self, x: int, y: int) -> None: + pass + + def set_cursor_vis(self, visible: bool) -> None: + pass + + def push_char(self, char: int | bytes) -> None: + pass + + def beep(self) -> None: + pass + + def clear(self) -> None: + pass + + def finish(self) -> None: + pass + + def flushoutput(self) -> None: + pass + + def forgetinput(self) -> None: + pass + + def wait(self, timeout: float | None = None) -> bool: + return True + + def repaint(self) -> None: + pass diff --git a/Lib/test/test_pyrepl/test_input.py b/Lib/test/test_pyrepl/test_input.py new file mode 100644 index 00000000000000..c78c876c2c4c2a --- /dev/null +++ b/Lib/test/test_pyrepl/test_input.py @@ -0,0 +1,102 @@ +import unittest + +from _pyrepl.console import Event +from _pyrepl.input import KeymapTranslator + + +class KeymapTranslatorTests(unittest.TestCase): + def test_push_single_key(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "a") + translator.push(evt) + result = translator.get() + self.assertEqual(result, ("command_a", ["a"])) + + def test_push_multiple_keys(self): + keymap = [("ab", "command_ab")] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "b") + translator.push(evt1) + translator.push(evt2) + result = translator.get() + self.assertEqual(result, ("command_ab", ["a", "b"])) + + def test_push_invalid_key(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "b") + translator.push(evt) + result = translator.get() + self.assertEqual(result, (None, ["b"])) + + def test_push_invalid_key_with_stack(self): + keymap = [("ab", "command_ab")] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "c") + translator.push(evt1) + translator.push(evt2) + result = translator.get() + self.assertEqual(result, (None, ["a", "c"])) + + def test_push_character_key(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "a") + translator.push(evt) + result = translator.get() + self.assertEqual(result, ("command_a", ["a"])) + + def test_push_character_key_with_stack(self): + keymap = [("ab", "command_ab")] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "b") + evt3 = Event("key", "c") + translator.push(evt1) + translator.push(evt2) + translator.push(evt3) + result = translator.get() + self.assertEqual(result, ("command_ab", ["a", "b"])) + + def test_push_transition_key(self): + keymap = [("a", {"b": "command_ab"})] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "b") + translator.push(evt1) + translator.push(evt2) + result = translator.get() + self.assertEqual(result, ("command_ab", ["a", "b"])) + + def test_push_transition_key_interrupted(self): + keymap = [("a", {"b": "command_ab"})] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "c") + evt3 = Event("key", "b") + translator.push(evt1) + translator.push(evt2) + translator.push(evt3) + result = translator.get() + self.assertEqual(result, (None, ["a", "c"])) + + def test_push_invalid_key_with_unicode_category(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "\u0003") # Control character + translator.push(evt) + result = translator.get() + self.assertEqual(result, (None, ["\u0003"])) + + def test_empty(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + self.assertTrue(translator.empty()) + evt = Event("key", "a") + translator.push(evt) + self.assertFalse(translator.empty()) + translator.get() + self.assertTrue(translator.empty()) diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py new file mode 100644 index 00000000000000..e0ee310e2c4dbc --- /dev/null +++ b/Lib/test/test_pyrepl/test_interact.py @@ -0,0 +1,262 @@ +import contextlib +import io +import unittest +from unittest.mock import patch +from textwrap import dedent + +from test.support import force_not_colorized + +from _pyrepl.console import InteractiveColoredConsole +from _pyrepl.simple_interact import _more_lines + +class TestSimpleInteract(unittest.TestCase): + def test_multiple_statements(self): + namespace = {} + code = dedent("""\ + class A: + def foo(self): + + + pass + + class B: + def bar(self): + pass + + a = 1 + a + """) + console = InteractiveColoredConsole(namespace, filename="") + f = io.StringIO() + with ( + patch.object(InteractiveColoredConsole, "showsyntaxerror") as showsyntaxerror, + patch.object(InteractiveColoredConsole, "runsource", wraps=console.runsource) as runsource, + contextlib.redirect_stdout(f), + ): + more = console.push(code, filename="", _symbol="single") # type: ignore[call-arg] + self.assertFalse(more) + showsyntaxerror.assert_not_called() + + + def test_multiple_statements_output(self): + namespace = {} + code = dedent("""\ + b = 1 + b + a = 1 + a + """) + console = InteractiveColoredConsole(namespace, filename="") + f = io.StringIO() + with contextlib.redirect_stdout(f): + more = console.push(code, filename="", _symbol="single") # type: ignore[call-arg] + self.assertFalse(more) + self.assertEqual(f.getvalue(), "1\n") + + def test_empty(self): + namespace = {} + code = "" + console = InteractiveColoredConsole(namespace, filename="") + f = io.StringIO() + with contextlib.redirect_stdout(f): + more = console.push(code, filename="", _symbol="single") # type: ignore[call-arg] + self.assertFalse(more) + self.assertEqual(f.getvalue(), "") + + def test_runsource_compiles_and_runs_code(self): + console = InteractiveColoredConsole() + source = "print('Hello, world!')" + with patch.object(console, "runcode") as mock_runcode: + console.runsource(source) + mock_runcode.assert_called_once() + + def test_runsource_returns_false_for_successful_compilation(self): + console = InteractiveColoredConsole() + source = "print('Hello, world!')" + f = io.StringIO() + with contextlib.redirect_stdout(f): + result = console.runsource(source) + self.assertFalse(result) + + @force_not_colorized + def test_runsource_returns_false_for_failed_compilation(self): + console = InteractiveColoredConsole() + source = "print('Hello, world!'" + f = io.StringIO() + with contextlib.redirect_stderr(f): + result = console.runsource(source) + self.assertFalse(result) + self.assertIn('SyntaxError', f.getvalue()) + + @force_not_colorized + def test_runsource_show_syntax_error_location(self): + console = InteractiveColoredConsole() + source = "def f(x, x): ..." + f = io.StringIO() + with contextlib.redirect_stderr(f): + result = console.runsource(source) + self.assertFalse(result) + r = """ + def f(x, x): ... + ^ +SyntaxError: duplicate argument 'x' in function definition""" + self.assertIn(r, f.getvalue()) + + def test_runsource_shows_syntax_error_for_failed_compilation(self): + console = InteractiveColoredConsole() + source = "print('Hello, world!'" + with patch.object(console, "showsyntaxerror") as mock_showsyntaxerror: + console.runsource(source) + mock_showsyntaxerror.assert_called_once() + source = dedent("""\ + match 1: + case {0: _, 0j: _}: + pass + """) + with patch.object(console, "showsyntaxerror") as mock_showsyntaxerror: + console.runsource(source) + mock_showsyntaxerror.assert_called_once() + + def test_runsource_survives_null_bytes(self): + console = InteractiveColoredConsole() + source = "\x00\n" + f = io.StringIO() + with contextlib.redirect_stdout(f), contextlib.redirect_stderr(f): + result = console.runsource(source) + self.assertFalse(result) + self.assertIn("source code string cannot contain null bytes", f.getvalue()) + + def test_no_active_future(self): + console = InteractiveColoredConsole() + source = dedent("""\ + x: int = 1 + print(__annotate__(1)) + """) + f = io.StringIO() + with contextlib.redirect_stdout(f): + result = console.runsource(source) + self.assertFalse(result) + self.assertEqual(f.getvalue(), "{'x': }\n") + + def test_future_annotations(self): + console = InteractiveColoredConsole() + source = dedent("""\ + from __future__ import annotations + def g(x: int): ... + print(g.__annotations__) + """) + f = io.StringIO() + with contextlib.redirect_stdout(f): + result = console.runsource(source) + self.assertFalse(result) + self.assertEqual(f.getvalue(), "{'x': 'int'}\n") + + def test_future_barry_as_flufl(self): + console = InteractiveColoredConsole() + f = io.StringIO() + with contextlib.redirect_stdout(f): + result = console.runsource("from __future__ import barry_as_FLUFL\n") + result = console.runsource("""print("black" <> 'blue')\n""") + self.assertFalse(result) + self.assertEqual(f.getvalue(), "True\n") + + +class TestMoreLines(unittest.TestCase): + def test_invalid_syntax_single_line(self): + namespace = {} + code = "if foo" + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_empty_line(self): + namespace = {} + code = "" + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_valid_single_statement(self): + namespace = {} + code = "foo = 1" + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_multiline_single_assignment(self): + namespace = {} + code = dedent("""\ + foo = [ + 1, + 2, + 3, + ]""") + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_multiline_single_block(self): + namespace = {} + code = dedent("""\ + def foo(): + '''docs''' + + return 1""") + console = InteractiveColoredConsole(namespace, filename="") + self.assertTrue(_more_lines(console, code)) + + def test_multiple_statements_single_line(self): + namespace = {} + code = "foo = 1;bar = 2" + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_multiple_statements(self): + namespace = {} + code = dedent("""\ + import time + + foo = 1""") + console = InteractiveColoredConsole(namespace, filename="") + self.assertTrue(_more_lines(console, code)) + + def test_multiple_blocks(self): + namespace = {} + code = dedent("""\ + from dataclasses import dataclass + + @dataclass + class Point: + x: float + y: float""") + console = InteractiveColoredConsole(namespace, filename="") + self.assertTrue(_more_lines(console, code)) + + def test_multiple_blocks_empty_newline(self): + namespace = {} + code = dedent("""\ + from dataclasses import dataclass + + @dataclass + class Point: + x: float + y: float + """) + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_multiple_blocks_indented_newline(self): + namespace = {} + code = ( + "from dataclasses import dataclass\n" + "\n" + "@dataclass\n" + "class Point:\n" + " x: float\n" + " y: float\n" + " " + ) + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_incomplete_statement(self): + namespace = {} + code = "if foo:" + console = InteractiveColoredConsole(namespace, filename="") + self.assertTrue(_more_lines(console, code)) diff --git a/Lib/test/test_pyrepl/test_keymap.py b/Lib/test/test_pyrepl/test_keymap.py new file mode 100644 index 00000000000000..2c97066b2c7043 --- /dev/null +++ b/Lib/test/test_pyrepl/test_keymap.py @@ -0,0 +1,120 @@ +import string +import unittest + +from _pyrepl.keymap import _keynames, _escapes, parse_keys, compile_keymap, KeySpecError + + +class TestParseKeys(unittest.TestCase): + def test_single_character(self): + """Ensure that single ascii characters or single digits are parsed as single characters.""" + test_cases = [(key, [key]) for key in string.ascii_letters + string.digits] + for test_key, expected_keys in test_cases: + with self.subTest(f"{test_key} should be parsed as {expected_keys}"): + self.assertEqual(parse_keys(test_key), expected_keys) + + def test_keynames(self): + """Ensure that keynames are parsed to their corresponding mapping. + + A keyname is expected to be of the following form: \\ such as \\ + which would get parsed as "left". + """ + test_cases = [(f"\\<{keyname}>", [parsed_keyname]) for keyname, parsed_keyname in _keynames.items()] + for test_key, expected_keys in test_cases: + with self.subTest(f"{test_key} should be parsed as {expected_keys}"): + self.assertEqual(parse_keys(test_key), expected_keys) + + def test_escape_sequences(self): + """Ensure that escaping sequences are parsed to their corresponding mapping.""" + test_cases = [(f"\\{escape}", [parsed_escape]) for escape, parsed_escape in _escapes.items()] + for test_key, expected_keys in test_cases: + with self.subTest(f"{test_key} should be parsed as {expected_keys}"): + self.assertEqual(parse_keys(test_key), expected_keys) + + def test_control_sequences(self): + """Ensure that supported control sequences are parsed successfully.""" + keys = ["@", "[", "]", "\\", "^", "_", "\\", "\\"] + keys.extend(string.ascii_letters) + test_cases = [(f"\\C-{key}", chr(ord(key) & 0x1F)) for key in []] + for test_key, expected_keys in test_cases: + with self.subTest(f"{test_key} should be parsed as {expected_keys}"): + self.assertEqual(parse_keys(test_key), expected_keys) + + def test_meta_sequences(self): + self.assertEqual(parse_keys("\\M-a"), ["\033", "a"]) + self.assertEqual(parse_keys("\\M-b"), ["\033", "b"]) + self.assertEqual(parse_keys("\\M-c"), ["\033", "c"]) + + def test_combinations(self): + self.assertEqual(parse_keys("\\C-a\\n\\"), ["\x01", "\n", "up"]) + self.assertEqual(parse_keys("\\M-a\\t\\"), ["\033", "a", "\t", "down"]) + + def test_keyspec_errors(self): + cases = [ + ("\\Ca", "\\C must be followed by `-'"), + ("\\ca", "\\C must be followed by `-'"), + ("\\C-\\C-", "doubled \\C-"), + ("\\Ma", "\\M must be followed by `-'"), + ("\\ma", "\\M must be followed by `-'"), + ("\\M-\\M-", "doubled \\M-"), + ("\\", "unrecognised keyname"), + ("\\大", "unknown backslash escape"), + ("\\C-\\", "\\C- followed by invalid key") + ] + for test_keys, expected_err in cases: + with self.subTest(f"{test_keys} should give error {expected_err}"): + with self.assertRaises(KeySpecError) as e: + parse_keys(test_keys) + self.assertIn(expected_err, str(e.exception)) + + def test_index_errors(self): + test_cases = ["\\", "\\C", "\\C-\\C"] + for test_keys in test_cases: + with self.assertRaises(IndexError): + parse_keys(test_keys) + + +class TestCompileKeymap(unittest.TestCase): + def test_empty_keymap(self): + keymap = {} + result = compile_keymap(keymap) + self.assertEqual(result, {}) + + def test_single_keymap(self): + keymap = {b"a": "action"} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": "action"}) + + def test_nested_keymap(self): + keymap = {b"a": {b"b": "action"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"b": "action"}}) + + def test_empty_value(self): + keymap = {b"a": {b"": "action"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"": "action"}}) + + def test_multiple_empty_values(self): + keymap = {b"a": {b"": "action1", b"b": "action2"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"": "action1", b"b": "action2"}}) + + def test_multiple_keymaps(self): + keymap = {b"a": {b"b": "action1", b"c": "action2"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"b": "action1", b"c": "action2"}}) + + def test_nested_multiple_keymaps(self): + keymap = {b"a": {b"b": {b"c": "action"}}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"b": {b"c": "action"}}}) + + def test_clashing_definitions(self): + km = {b'a': 'c', b'a' + b'b': 'd'} + with self.assertRaises(KeySpecError): + compile_keymap(km) + + def test_non_bytes_key(self): + with self.assertRaises(TypeError): + compile_keymap({123: 'a'}) diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py new file mode 100644 index 00000000000000..3540d2a5a41662 --- /dev/null +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -0,0 +1,1358 @@ +import io +import itertools +import os +import pathlib +import re +import rlcompleter +import select +import subprocess +import sys +import tempfile +from unittest import TestCase, skipUnless, skipIf +from unittest.mock import patch +from test.support import force_not_colorized, make_clean_env +from test.support import SHORT_TIMEOUT +from test.support.import_helper import import_module +from test.support.os_helper import unlink + +from .support import ( + FakeConsole, + handle_all_events, + handle_events_narrow_console, + more_lines, + multiline_input, + code_to_events, + clean_screen, +) +from _pyrepl.console import Event +from _pyrepl.readline import (ReadlineAlikeReader, ReadlineConfig, + _ReadlineWrapper) +from _pyrepl.readline import multiline_input as readline_multiline_input + +try: + import pty +except ImportError: + pty = None + + +class ReplTestCase(TestCase): + def run_repl( + self, + repl_input: str | list[str], + env: dict | None = None, + *, + cmdline_args: list[str] | None = None, + cwd: str | None = None, + ) -> tuple[str, int]: + temp_dir = None + if cwd is None: + temp_dir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) + cwd = temp_dir.name + try: + return self._run_repl( + repl_input, env=env, cmdline_args=cmdline_args, cwd=cwd + ) + finally: + if temp_dir is not None: + temp_dir.cleanup() + + def _run_repl( + self, + repl_input: str | list[str], + *, + env: dict | None, + cmdline_args: list[str] | None, + cwd: str, + ) -> tuple[str, int]: + assert pty + master_fd, slave_fd = pty.openpty() + cmd = [sys.executable, "-i", "-u"] + if env is None: + cmd.append("-I") + elif "PYTHON_HISTORY" not in env: + env["PYTHON_HISTORY"] = os.path.join(cwd, ".regrtest_history") + if cmdline_args is not None: + cmd.extend(cmdline_args) + + try: + import termios + except ModuleNotFoundError: + pass + else: + term_attr = termios.tcgetattr(slave_fd) + term_attr[6][termios.VREPRINT] = 0 # pass through CTRL-R + term_attr[6][termios.VINTR] = 0 # pass through CTRL-C + termios.tcsetattr(slave_fd, termios.TCSANOW, term_attr) + + process = subprocess.Popen( + cmd, + stdin=slave_fd, + stdout=slave_fd, + stderr=slave_fd, + cwd=cwd, + text=True, + close_fds=True, + env=env if env else os.environ, + ) + os.close(slave_fd) + if isinstance(repl_input, list): + repl_input = "\n".join(repl_input) + "\n" + os.write(master_fd, repl_input.encode("utf-8")) + + output = [] + while select.select([master_fd], [], [], SHORT_TIMEOUT)[0]: + try: + data = os.read(master_fd, 1024).decode("utf-8") + if not data: + break + except OSError: + break + output.append(data) + else: + os.close(master_fd) + process.kill() + self.fail(f"Timeout while waiting for output, got: {''.join(output)}") + + os.close(master_fd) + try: + exit_code = process.wait(timeout=SHORT_TIMEOUT) + except subprocess.TimeoutExpired: + process.kill() + exit_code = process.wait() + return "".join(output), exit_code + + +class TestCursorPosition(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_up_arrow_simple(self): + # fmt: off + code = ( + "def f():\n" + " ...\n" + ) + # fmt: on + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (0, 1)) + console.move_cursor.assert_called_once_with(0, 1) + + def test_down_arrow_end_of_input(self): + # fmt: off + code = ( + "def f():\n" + " ...\n" + ) + # fmt: on + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (0, 2)) + console.move_cursor.assert_called_once_with(0, 2) + + def test_left_arrow_simple(self): + events = itertools.chain( + code_to_events("11+11"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (4, 0)) + console.move_cursor.assert_called_once_with(4, 0) + + def test_right_arrow_end_of_line(self): + events = itertools.chain( + code_to_events("11+11"), + [ + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (5, 0)) + console.move_cursor.assert_called_once_with(5, 0) + + def test_cursor_position_simple_character(self): + events = itertools.chain(code_to_events("k")) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 1 for simple character + self.assertEqual(reader.cxy, (1, 0)) + + def test_cursor_position_double_width_character(self): + events = itertools.chain(code_to_events("樂")) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 2 for wide character + self.assertEqual(reader.cxy, (2, 0)) + + def test_cursor_position_double_width_character_move_left(self): + events = itertools.chain( + code_to_events("樂"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 0) + self.assertEqual(reader.cxy, (0, 0)) + + def test_cursor_position_double_width_character_move_left_right(self): + events = itertools.chain( + code_to_events("樂"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 2 for wide character + self.assertEqual(reader.cxy, (2, 0)) + + def test_cursor_position_double_width_characters_move_up(self): + for_loop = "for _ in _:" + + # fmt: off + code = ( + f"{for_loop}\n" + " ' 可口可乐; 可口可樂'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + + reader, _ = handle_all_events(events) + + # cursor at end of first line + self.assertEqual(reader.pos, len(for_loop)) + self.assertEqual(reader.cxy, (len(for_loop), 0)) + + def test_cursor_position_double_width_characters_move_up_down(self): + for_loop = "for _ in _:" + + # fmt: off + code = ( + f"{for_loop}\n" + " ' 可口可乐; 可口可樂'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader, _ = handle_all_events(events) + + # cursor here (showing 2nd line only): + # < ' 可口可乐; 可口可樂'> + # ^ + self.assertEqual(reader.pos, 19) + self.assertEqual(reader.cxy, (10, 1)) + + def test_cursor_position_multiple_double_width_characters_move_left(self): + events = itertools.chain( + code_to_events("' 可口可乐; 可口可樂'"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 10) + + # 1 for quote, 1 for space, 2 per wide character, + # 1 for semicolon, 1 for space, 2 per wide character + self.assertEqual(reader.cxy, (16, 0)) + + def test_cursor_position_move_up_to_eol(self): + first_line = "for _ in _:" + second_line = " hello" + + # fmt: off + code = ( + f"{first_line}\n" + f"{second_line}\n" + " h\n" + " hel" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + + reader, _ = handle_all_events(events) + + # Cursor should be at end of line 1, even though line 2 is shorter + # for _ in _: + # hello + # h + # hel + self.assertEqual( + reader.pos, len(first_line) + len(second_line) + 1 + ) # +1 for newline + self.assertEqual(reader.cxy, (len(second_line), 1)) + + def test_cursor_position_move_down_to_eol(self): + last_line = " hel" + + # fmt: off + code = ( + "for _ in _:\n" + " hello\n" + " h\n" + f"{last_line}" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader, _ = handle_all_events(events) + + # Cursor should be at end of line 3, even though line 2 is shorter + # for _ in _: + # hello + # h + # hel + self.assertEqual(reader.pos, len(code)) + self.assertEqual(reader.cxy, (len(last_line), 3)) + + def test_cursor_position_multiple_mixed_lines_move_up(self): + # fmt: off + code = ( + "def foo():\n" + " x = '可口可乐; 可口可樂'\n" + " y = 'abckdfjskldfjslkdjf'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + 13 * [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], + ) + + reader, _ = handle_all_events(events) + + # By moving left, we're before the s: + # y = 'abckdfjskldfjslkdjf' + # ^ + # And we should move before the semi-colon despite the different offset + # x = '可口可乐; 可口可樂' + # ^ + self.assertEqual(reader.pos, 22) + self.assertEqual(reader.cxy, (15, 1)) + + def test_cursor_position_after_wrap_and_move_up(self): + # fmt: off + code = ( + "def foo():\n" + " hello" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + reader, _ = handle_events_narrow_console(events) + + # The code looks like this: + # def foo()\ + # : + # hello + # After moving up we should be after the colon in line 2 + self.assertEqual(reader.pos, 10) + self.assertEqual(reader.cxy, (1, 1)) + + +class TestPyReplAutoindent(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_auto_indent_default(self): + # fmt: off + input_code = ( + "def f():\n" + "pass\n\n" + ) + + output_code = ( + "def f():\n" + " pass\n" + " " + ) + # fmt: on + + def test_auto_indent_continuation(self): + # auto indenting according to previous user indentation + # fmt: off + events = itertools.chain( + code_to_events("def f():\n"), + # add backspace to delete default auto-indent + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + code_to_events( + " pass\n" + "pass\n\n" + ), + ) + + output_code = ( + "def f():\n" + " pass\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_auto_indent_prev_block(self): + # auto indenting according to indentation in different block + # fmt: off + events = itertools.chain( + code_to_events("def f():\n"), + # add backspace to delete default auto-indent + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + code_to_events( + " pass\n" + "pass\n\n" + ), + code_to_events( + "def g():\n" + "pass\n\n" + ), + ) + + output_code = ( + "def g():\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output1 = multiline_input(reader) + output2 = multiline_input(reader) + self.assertEqual(output2, output_code) + + def test_auto_indent_multiline(self): + # fmt: off + events = itertools.chain( + code_to_events( + "def f():\n" + "pass" + ), + [ + # go to the end of the first line + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\x05", raw=bytearray(b"\x1bO5")), + # new line should be autoindented + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + code_to_events( + "pass" + ), + [ + # go to end of last line + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="\x05", raw=bytearray(b"\x1bO5")), + # double newline to terminate the block + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + output_code = ( + "def f():\n" + " pass\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_auto_indent_with_comment(self): + # fmt: off + events = code_to_events( + "def f(): # foo\n" + "pass\n\n" + ) + + output_code = ( + "def f(): # foo\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_auto_indent_with_multicomment(self): + # fmt: off + events = code_to_events( + "def f(): ## foo\n" + "pass\n\n" + ) + + output_code = ( + "def f(): ## foo\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_auto_indent_ignore_comments(self): + # fmt: off + events = code_to_events( + "pass #:\n" + ) + + output_code = ( + "pass #:" + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + +class TestPyReplOutput(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + reader.can_colorize = False + return reader + + def test_stdin_is_tty(self): + # Used during test log analysis to figure out if a TTY was available. + try: + if os.isatty(sys.stdin.fileno()): + return + except OSError as ose: + self.skipTest(f"stdin tty check failed: {ose}") + else: + self.skipTest("stdin is not a tty") + + def test_stdout_is_tty(self): + # Used during test log analysis to figure out if a TTY was available. + try: + if os.isatty(sys.stdout.fileno()): + return + except OSError as ose: + self.skipTest(f"stdout tty check failed: {ose}") + else: + self.skipTest("stdout is not a tty") + + def test_basic(self): + reader = self.prepare_reader(code_to_events("1+1\n")) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") + + def test_get_line_buffer_returns_str(self): + reader = self.prepare_reader(code_to_events("\n")) + wrapper = _ReadlineWrapper(f_in=None, f_out=None, reader=reader) + self.assertIs(type(wrapper.get_line_buffer()), str) + + def test_multiline_edit(self): + events = itertools.chain( + code_to_events("def f():\n...\n\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="backspace", raw=bytearray(b"\x08")), + Event(evt="key", data="g", raw=bytearray(b"g")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="backspace", raw=bytearray(b"\x08")), + Event(evt="key", data="delete", raw=bytearray(b"\x7F")), + Event(evt="key", data="right", raw=bytearray(b"g")), + Event(evt="key", data="backspace", raw=bytearray(b"\x08")), + Event(evt="key", data="p", raw=bytearray(b"p")), + Event(evt="key", data="a", raw=bytearray(b"a")), + Event(evt="key", data="s", raw=bytearray(b"s")), + Event(evt="key", data="s", raw=bytearray(b"s")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "def f():\n ...\n ") + self.assertEqual(clean_screen(reader.screen), "def f():\n ...") + output = multiline_input(reader) + self.assertEqual(output, "def g():\n pass\n ") + self.assertEqual(clean_screen(reader.screen), "def g():\n pass") + + def test_history_navigation_with_up_arrow(self): + events = itertools.chain( + code_to_events("1+1\n2+2\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") + output = multiline_input(reader) + self.assertEqual(output, "2+2") + self.assertEqual(clean_screen(reader.screen), "2+2") + output = multiline_input(reader) + self.assertEqual(output, "2+2") + self.assertEqual(clean_screen(reader.screen), "2+2") + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") + + def test_history_with_multiline_entries(self): + code = "def foo():\nx = 1\ny = 2\nz = 3\n\ndef bar():\nreturn 42\n\n" + events = list(itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ] + )) + + reader = self.prepare_reader(events) + output = multiline_input(reader) + output = multiline_input(reader) + output = multiline_input(reader) + self.assertEqual( + clean_screen(reader.screen), + 'def foo():\n x = 1\n y = 2\n z = 3' + ) + self.assertEqual(output, "def foo():\n x = 1\n y = 2\n z = 3\n ") + + + def test_history_navigation_with_down_arrow(self): + events = itertools.chain( + code_to_events("1+1\n2+2\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") + + def test_history_search(self): + events = itertools.chain( + code_to_events("1+1\n2+2\n3+3\n"), + [ + Event(evt="key", data="\x12", raw=bytearray(b"\x12")), + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") + output = multiline_input(reader) + self.assertEqual(output, "2+2") + self.assertEqual(clean_screen(reader.screen), "2+2") + output = multiline_input(reader) + self.assertEqual(output, "3+3") + self.assertEqual(clean_screen(reader.screen), "3+3") + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") + + def test_control_character(self): + events = code_to_events("c\x1d\n") + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, "c\x1d") + self.assertEqual(clean_screen(reader.screen), "c") + + def test_history_search_backward(self): + # Test history search backward with "imp" input + events = itertools.chain( + code_to_events("import os\n"), + code_to_events("imp"), + [ + Event(evt='key', data='page up', raw=bytearray(b'\x1b[5~')), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + # fill the history + reader = self.prepare_reader(events) + multiline_input(reader) + + # search for "imp" in history + output = multiline_input(reader) + self.assertEqual(output, "import os") + self.assertEqual(clean_screen(reader.screen), "import os") + + def test_history_search_backward_empty(self): + # Test history search backward with an empty input + events = itertools.chain( + code_to_events("import os\n"), + [ + Event(evt='key', data='page up', raw=bytearray(b'\x1b[5~')), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + # fill the history + reader = self.prepare_reader(events) + multiline_input(reader) + + # search backward in history + output = multiline_input(reader) + self.assertEqual(output, "import os") + self.assertEqual(clean_screen(reader.screen), "import os") + + +class TestPyReplCompleter(TestCase): + def prepare_reader(self, events, namespace): + console = FakeConsole(events) + config = ReadlineConfig() + config.readline_completer = rlcompleter.Completer(namespace).complete + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + @patch("rlcompleter._readline_available", False) + def test_simple_completion(self): + events = code_to_events("os.getpid\t\n") + + namespace = {"os": os} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.getpid()") + + def test_completion_with_many_options(self): + # Test with something that initially displays many options + # and then complete from one of them. The first time tab is + # pressed, the options are displayed (which corresponds to + # when the repl shows [ not unique ]) and the second completes + # from one of them. + events = code_to_events("os.\t\tO_AP\t\n") + + namespace = {"os": os} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.O_APPEND") + + def test_empty_namespace_completion(self): + events = code_to_events("os.geten\t\n") + namespace = {} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.geten") + + def test_global_namespace_completion(self): + events = code_to_events("py\t\n") + namespace = {"python": None} + reader = self.prepare_reader(events, namespace) + output = multiline_input(reader, namespace) + self.assertEqual(output, "python") + + def test_up_down_arrow_with_completion_menu(self): + """Up arrow in the middle of unfinished tab completion when the menu is displayed + should work and trigger going back in history. Down arrow should subsequently + get us back to the incomplete command.""" + code = "import os\nos.\t\t" + namespace = {"os": os} + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + code_to_events("\n"), + ) + reader = self.prepare_reader(events, namespace=namespace) + output = multiline_input(reader, namespace) + # This is the first line, nothing to see here + self.assertEqual(output, "import os") + # This is the second line. We pressed up and down arrows + # so we should end up where we were when we initiated tab completion. + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.") + + @patch("_pyrepl.readline._ReadlineWrapper.get_reader") + @patch("sys.stderr", new_callable=io.StringIO) + def test_completion_with_warnings(self, mock_stderr, mock_get_reader): + class Dummy: + @property + def test_func(self): + import warnings + + warnings.warn("warnings\n") + return None + + dummy = Dummy() + events = code_to_events("dummy.test_func.\t\n\n") + namespace = {"dummy": dummy} + reader = self.prepare_reader(events, namespace) + mock_get_reader.return_value = reader + output = readline_multiline_input(more_lines, ">>>", "...") + self.assertEqual(output, "dummy.test_func.__") + self.assertEqual(mock_stderr.getvalue(), "") + + +class TestPasteEvent(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_paste(self): + # fmt: off + code = ( + "def a():\n" + " for x in range(10):\n" + " if x%2:\n" + " print(x)\n" + " else:\n" + " pass\n" + ) + # fmt: on + + events = itertools.chain( + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events(code), + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, code) + + def test_paste_mid_newlines(self): + # fmt: off + code = ( + "def f():\n" + " x = y\n" + " \n" + " y = z\n" + ) + # fmt: on + + events = itertools.chain( + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events(code), + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, code) + + def test_paste_mid_newlines_not_in_paste_mode(self): + # fmt: off + code = ( + "def f():\n" + "x = y\n" + "\n" + "y = z\n\n" + ) + + expected = ( + "def f():\n" + " x = y\n" + " " + ) + # fmt: on + + events = code_to_events(code) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, expected) + + def test_paste_not_in_paste_mode(self): + # fmt: off + input_code = ( + "def a():\n" + "for x in range(10):\n" + "if x%2:\n" + "print(x)\n" + "else:\n" + "pass\n\n" + ) + + output_code = ( + "def a():\n" + " for x in range(10):\n" + " if x%2:\n" + " print(x)\n" + " else:" + ) + # fmt: on + + events = code_to_events(input_code) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_bracketed_paste(self): + """Test that bracketed paste using \x1b[200~ and \x1b[201~ works.""" + # fmt: off + input_code = ( + "def a():\n" + " for x in range(10):\n" + "\n" + " if x%2:\n" + " print(x)\n" + "\n" + " else:\n" + " pass\n" + ) + + output_code = ( + "def a():\n" + " for x in range(10):\n" + "\n" + " if x%2:\n" + " print(x)\n" + "\n" + " else:\n" + " pass\n" + ) + # fmt: on + + paste_start = "\x1b[200~" + paste_end = "\x1b[201~" + + events = itertools.chain( + code_to_events(paste_start), + code_to_events(input_code), + code_to_events(paste_end), + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_bracketed_paste_single_line(self): + input_code = "oneline" + + paste_start = "\x1b[200~" + paste_end = "\x1b[201~" + + events = itertools.chain( + code_to_events(paste_start), + code_to_events(input_code), + code_to_events(paste_end), + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, input_code) + + +@skipUnless(pty, "requires pty") +class TestDumbTerminal(ReplTestCase): + def test_dumb_terminal_exits_cleanly(self): + env = os.environ.copy() + env.update({"TERM": "dumb"}) + output, exit_code = self.run_repl("exit()\n", env=env) + self.assertEqual(exit_code, 0) + self.assertIn("warning: can't use pyrepl", output) + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + + +@skipUnless(pty, "requires pty") +@skipIf((os.environ.get("TERM") or "dumb") == "dumb", "can't use pyrepl in dumb terminal") +class TestMain(ReplTestCase): + def setUp(self): + # Cleanup from PYTHON* variables to isolate from local + # user settings, see #121359. Such variables should be + # added later in test methods to patched os.environ. + patcher = patch('os.environ', new=make_clean_env()) + self.addCleanup(patcher.stop) + patcher.start() + + @force_not_colorized + def test_exposed_globals_in_repl(self): + pre = "['__builtins__'" + post = "'__loader__', '__name__', '__package__', '__spec__']" + output, exit_code = self.run_repl(["sorted(dir())", "exit()"]) + if "can't use pyrepl" in output: + self.skipTest("pyrepl not available") + self.assertEqual(exit_code, 0) + + # if `__main__` is not a file (impossible with pyrepl) + case1 = f"{pre}, '__doc__', {post}" in output + + # if `__main__` is an uncached .py file (no .pyc) + case2 = f"{pre}, '__doc__', '__file__', {post}" in output + + # if `__main__` is a cached .pyc file and the .py source exists + case3 = f"{pre}, '__cached__', '__doc__', '__file__', {post}" in output + + # if `__main__` is a cached .pyc file but there's no .py source file + case4 = f"{pre}, '__cached__', '__doc__', {post}" in output + + self.assertTrue(case1 or case2 or case3 or case4, output) + + def _assertMatchOK( + self, var: str, expected: str | re.Pattern, actual: str + ) -> None: + if isinstance(expected, re.Pattern): + self.assertTrue( + expected.match(actual), + f"{var}={actual} does not match {expected.pattern}", + ) + else: + self.assertEqual( + actual, + expected, + f"expected {var}={expected}, got {var}={actual}", + ) + + @force_not_colorized + def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False): + clean_env = make_clean_env() + clean_env["NO_COLOR"] = "1" # force_not_colorized doesn't touch subprocesses + + with tempfile.TemporaryDirectory() as td: + blue = pathlib.Path(td) / "blue" + blue.mkdir() + mod = blue / "calx.py" + mod.write_text("FOO = 42", encoding="utf-8") + commands = [ + "print(f'^{" + var + "=}')" for var in expectations + ] + ["exit()"] + if as_file and as_module: + self.fail("as_file and as_module are mutually exclusive") + elif as_file: + output, exit_code = self.run_repl( + commands, + cmdline_args=[str(mod)], + env=clean_env, + ) + elif as_module: + output, exit_code = self.run_repl( + commands, + cmdline_args=["-m", "blue.calx"], + env=clean_env, + cwd=td, + ) + else: + self.fail("Choose one of as_file or as_module") + + if "can't use pyrepl" in output: + self.skipTest("pyrepl not available") + + self.assertEqual(exit_code, 0) + for var, expected in expectations.items(): + with self.subTest(var=var, expected=expected): + if m := re.search(rf"\^{var}=(.+?)[\r\n]", output): + self._assertMatchOK(var, expected, actual=m.group(1)) + else: + self.fail(f"{var}= not found in output: {output!r}\n\n{output}") + + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + + def test_inspect_keeps_globals_from_inspected_file(self): + expectations = { + "FOO": "42", + "__name__": "'__main__'", + "__package__": "None", + # "__file__" is missing in -i, like in the basic REPL + } + self._run_repl_globals_test(expectations, as_file=True) + + def test_inspect_keeps_globals_from_inspected_module(self): + expectations = { + "FOO": "42", + "__name__": "'__main__'", + "__package__": "'blue'", + "__file__": re.compile(r"^'.*calx.py'$"), + } + self._run_repl_globals_test(expectations, as_module=True) + + @force_not_colorized + def test_python_basic_repl(self): + env = os.environ.copy() + commands = ("from test.support import initialized_with_pyrepl\n" + "initialized_with_pyrepl()\n" + "exit()\n") + + env.pop("PYTHON_BASIC_REPL", None) + output, exit_code = self.run_repl(commands, env=env) + if "can\'t use pyrepl" in output: + self.skipTest("pyrepl not available") + self.assertEqual(exit_code, 0) + self.assertIn("True", output) + self.assertNotIn("False", output) + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + + env["PYTHON_BASIC_REPL"] = "1" + output, exit_code = self.run_repl(commands, env=env) + self.assertEqual(exit_code, 0) + self.assertIn("False", output) + self.assertNotIn("True", output) + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + + # The site module must not load _pyrepl if PYTHON_BASIC_REPL is set + commands = ("import sys\n" + "print('_pyrepl' in sys.modules)\n" + "exit()\n") + env["PYTHON_BASIC_REPL"] = "1" + output, exit_code = self.run_repl(commands, env=env) + self.assertEqual(exit_code, 0) + self.assertIn("False", output) + self.assertNotIn("True", output) + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + + @force_not_colorized + def test_bad_sys_excepthook_doesnt_crash_pyrepl(self): + env = os.environ.copy() + commands = ("import sys\n" + "sys.excepthook = 1\n" + "1/0\n" + "exit()\n") + + def check(output, exitcode): + self.assertIn("Error in sys.excepthook:", output) + self.assertEqual(output.count("'int' object is not callable"), 1) + self.assertIn("Original exception was:", output) + self.assertIn("division by zero", output) + self.assertEqual(exitcode, 0) + env.pop("PYTHON_BASIC_REPL", None) + output, exit_code = self.run_repl(commands, env=env) + if "can\'t use pyrepl" in output: + self.skipTest("pyrepl not available") + check(output, exit_code) + + env["PYTHON_BASIC_REPL"] = "1" + output, exit_code = self.run_repl(commands, env=env) + check(output, exit_code) + + def test_not_wiping_history_file(self): + # skip, if readline module is not available + import_module('readline') + + hfile = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(unlink, hfile.name) + env = os.environ.copy() + env["PYTHON_HISTORY"] = hfile.name + commands = "123\nspam\nexit()\n" + + env.pop("PYTHON_BASIC_REPL", None) + output, exit_code = self.run_repl(commands, env=env) + self.assertEqual(exit_code, 0) + self.assertIn("123", output) + self.assertIn("spam", output) + self.assertNotEqual(pathlib.Path(hfile.name).stat().st_size, 0) + + hfile.file.truncate() + hfile.close() + + env["PYTHON_BASIC_REPL"] = "1" + output, exit_code = self.run_repl(commands, env=env) + self.assertEqual(exit_code, 0) + self.assertIn("123", output) + self.assertIn("spam", output) + self.assertNotEqual(pathlib.Path(hfile.name).stat().st_size, 0) + + @force_not_colorized + def test_correct_filename_in_syntaxerrors(self): + env = os.environ.copy() + commands = "a b c\nexit()\n" + output, exit_code = self.run_repl(commands, env=env) + if "can't use pyrepl" in output: + self.skipTest("pyrepl not available") + self.assertIn("SyntaxError: invalid syntax", output) + self.assertIn("", output) + commands = " b\nexit()\n" + output, exit_code = self.run_repl(commands, env=env) + self.assertIn("IndentationError: unexpected indent", output) + self.assertIn("", output) + + @force_not_colorized + def test_proper_tracebacklimit(self): + env = os.environ.copy() + for set_tracebacklimit in [True, False]: + commands = ("import sys\n" + + ("sys.tracebacklimit = 1\n" if set_tracebacklimit else "") + + "def x1(): 1/0\n\n" + "def x2(): x1()\n\n" + "def x3(): x2()\n\n" + "x3()\n" + "exit()\n") + + for basic_repl in [True, False]: + if basic_repl: + env["PYTHON_BASIC_REPL"] = "1" + else: + env.pop("PYTHON_BASIC_REPL", None) + with self.subTest(set_tracebacklimit=set_tracebacklimit, + basic_repl=basic_repl): + output, exit_code = self.run_repl(commands, env=env) + if "can't use pyrepl" in output: + self.skipTest("pyrepl not available") + self.assertIn("in x1", output) + if set_tracebacklimit: + self.assertNotIn("in x2", output) + self.assertNotIn("in x3", output) + self.assertNotIn("in ", output) + else: + self.assertIn("in x2", output) + self.assertIn("in x3", output) + self.assertIn("in ", output) + + def test_null_byte(self): + output, exit_code = self.run_repl("\x00\nexit()\n") + self.assertEqual(exit_code, 0) + self.assertNotIn("TypeError", output) + + def test_readline_history_file(self): + # skip, if readline module is not available + readline = import_module('readline') + if readline.backend != "editline": + self.skipTest("GNU readline is not affected by this issue") + + with tempfile.NamedTemporaryFile() as hfile: + env = os.environ.copy() + env["PYTHON_HISTORY"] = hfile.name + + env["PYTHON_BASIC_REPL"] = "1" + output, exit_code = self.run_repl("spam \nexit()\n", env=env) + self.assertEqual(exit_code, 0) + self.assertIn("spam ", output) + self.assertNotEqual(pathlib.Path(hfile.name).stat().st_size, 0) + self.assertIn("spam\\040", pathlib.Path(hfile.name).read_text()) + + env.pop("PYTHON_BASIC_REPL", None) + output, exit_code = self.run_repl("exit\n", env=env) + self.assertEqual(exit_code, 0) + self.assertNotIn("\\040", pathlib.Path(hfile.name).read_text()) + + def test_keyboard_interrupt_after_isearch(self): + output, exit_code = self.run_repl(["\x12", "\x03", "exit"]) + self.assertEqual(exit_code, 0) + + def test_prompt_after_help(self): + output, exit_code = self.run_repl(["help", "q", "exit"]) + + # Regex pattern to remove ANSI escape sequences + ansi_escape = re.compile(r"(\x1B(=|>|(\[)[0-?]*[ -\/]*[@-~]))") + cleaned_output = ansi_escape.sub("", output) + self.assertEqual(exit_code, 0) + + # Ensure that we don't see multiple prompts after exiting `help` + # Extra stuff (newline and `exit` rewrites) are necessary + # because of how run_repl works. + self.assertNotIn(">>> \n>>> >>>", cleaned_output) diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py new file mode 100644 index 00000000000000..863ecc61ddd432 --- /dev/null +++ b/Lib/test/test_pyrepl/test_reader.py @@ -0,0 +1,314 @@ +import itertools +import functools +import rlcompleter +from unittest import TestCase +from unittest.mock import MagicMock + +from .support import handle_all_events, handle_events_narrow_console, code_to_events, prepare_reader +from _pyrepl.console import Event +from _pyrepl.reader import Reader + + +class TestReader(TestCase): + def assert_screen_equals(self, reader, expected): + actual = reader.screen + expected = expected.split("\n") + self.assertListEqual(actual, expected) + + def test_calc_screen_wrap_simple(self): + events = code_to_events(10 * "a") + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{9*"a"}\\\na") + + def test_calc_screen_wrap_wide_characters(self): + events = code_to_events(8 * "a" + "樂") + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{8*"a"}\\\n樂") + + def test_calc_screen_wrap_three_lines(self): + events = code_to_events(20 * "a") + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{9*"a"}\\\n{9*"a"}\\\naa") + + def test_calc_screen_prompt_handling(self): + def prepare_reader_keep_prompts(*args, **kwargs): + reader = prepare_reader(*args, **kwargs) + del reader.get_prompt + reader.ps1 = ">>> " + reader.ps2 = ">>> " + reader.ps3 = "... " + reader.ps4 = "" + reader.can_colorize = False + reader.paste_mode = False + return reader + + events = code_to_events("if some_condition:\nsome_function()") + reader, _ = handle_events_narrow_console( + events, + prepare_reader=prepare_reader_keep_prompts, + ) + # fmt: off + self.assert_screen_equals( + reader, + ( + ">>> if so\\\n" + "me_condit\\\n" + "ion:\n" + "... s\\\n" + "ome_funct\\\n" + "ion()" + ) + ) + # fmt: on + + def test_calc_screen_wrap_three_lines_mixed_character(self): + # fmt: off + code = ( + "def f():\n" + f" {8*"a"}\n" + f" {5*"樂"}" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + + # fmt: off + self.assert_screen_equals(reader, ( + "def f():\n" + f" {7*"a"}\\\n" + "a\n" + f" {3*"樂"}\\\n" + "樂樂" + )) + # fmt: on + + def test_calc_screen_backspace(self): + events = itertools.chain( + code_to_events("aaa"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) + reader, _ = handle_all_events(events) + self.assert_screen_equals(reader, "aa") + + def test_calc_screen_wrap_removes_after_backspace(self): + events = itertools.chain( + code_to_events(10 * "a"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, 9 * "a") + + def test_calc_screen_backspace_in_second_line_after_wrap(self): + events = itertools.chain( + code_to_events(11 * "a"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{9*"a"}\\\na") + + def test_setpos_for_xy_simple(self): + events = code_to_events("11+11") + reader, _ = handle_all_events(events) + reader.setpos_from_xy(0, 0) + self.assertEqual(reader.pos, 0) + + def test_control_characters(self): + code = 'flag = "🏳️‍🌈"' + events = code_to_events(code) + reader, _ = handle_all_events(events) + self.assert_screen_equals(reader, 'flag = "🏳️\\u200d🌈"') + + def test_setpos_from_xy_multiple_lines(self): + # fmt: off + code = ( + "def foo():\n" + " return 1" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_all_events(events) + reader.setpos_from_xy(2, 1) + self.assertEqual(reader.pos, 13) + + def test_setpos_from_xy_after_wrap(self): + # fmt: off + code = ( + "def foo():\n" + " hello" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + reader.setpos_from_xy(2, 2) + self.assertEqual(reader.pos, 13) + + def test_setpos_fromxy_in_wrapped_line(self): + # fmt: off + code = ( + "def foo():\n" + " hello" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + reader.setpos_from_xy(0, 1) + self.assertEqual(reader.pos, 9) + + def test_up_arrow_after_ctrl_r(self): + events = iter( + [ + Event(evt="key", data="\x12", raw=bytearray(b"\x12")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ] + ) + + reader, _ = handle_all_events(events) + self.assert_screen_equals(reader, "") + + def test_newline_within_block_trailing_whitespace(self): + # fmt: off + code = ( + "def foo():\n" + "a = 1\n" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + # go to the end of the first line + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\x05", raw=bytearray(b"\x1bO5")), + # new lines in-block shouldn't terminate the block + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + # end of line 2 + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="\x05", raw=bytearray(b"\x1bO5")), + # a double new line in-block should terminate the block + # even if its followed by whitespace + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + no_paste_reader = functools.partial(prepare_reader, paste_mode=False) + reader, _ = handle_all_events(events, prepare_reader=no_paste_reader) + + expected = ( + "def foo():\n" + " \n" + " \n" + " a = 1\n" + " \n" + " " # HistoricalReader will trim trailing whitespace + ) + self.assert_screen_equals(reader, expected) + self.assertTrue(reader.finished) + + def test_input_hook_is_called_if_set(self): + input_hook = MagicMock() + def _prepare_console(events): + console = MagicMock() + console.get_event.side_effect = events + console.height = 100 + console.width = 80 + console.input_hook = input_hook + return console + + events = code_to_events("a") + reader, _ = handle_all_events(events, prepare_console=_prepare_console) + + self.assertEqual(len(input_hook.mock_calls), 4) + + def test_keyboard_interrupt_clears_screen(self): + namespace = {"itertools": itertools} + code = "import itertools\nitertools." + events = itertools.chain(code_to_events(code), [ + Event(evt='key', data='\t', raw=bytearray(b'\t')), # Two tabs for completion + Event(evt='key', data='\t', raw=bytearray(b'\t')), + Event(evt='key', data='\x03', raw=bytearray(b'\x03')), # Ctrl-C + ]) + + completing_reader = functools.partial( + prepare_reader, + readline_completer=rlcompleter.Completer(namespace).complete + ) + reader, _ = handle_all_events(events, prepare_reader=completing_reader) + self.assertEqual(reader.calc_screen(), code.split("\n")) + + def test_prompt_length(self): + # Handles simple ASCII prompt + ps1 = ">>> " + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, ps1) + self.assertEqual(l, 4) + + # Handles ANSI escape sequences + ps1 = "\033[0;32m>>> \033[0m" + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, "\033[0;32m>>> \033[0m") + self.assertEqual(l, 4) + + # Handles ANSI escape sequences bracketed in \001 .. \002 + ps1 = "\001\033[0;32m\002>>> \001\033[0m\002" + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, "\033[0;32m>>> \033[0m") + self.assertEqual(l, 4) + + # Handles wide characters in prompt + ps1 = "樂>> " + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, ps1) + self.assertEqual(l, 5) + + # Handles wide characters AND ANSI sequences together + ps1 = "\001\033[0;32m\002樂>\001\033[0m\002> " + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, "\033[0;32m樂>\033[0m> ") + self.assertEqual(l, 5) + + def test_completions_updated_on_key_press(self): + namespace = {"itertools": itertools} + code = "itertools." + events = itertools.chain(code_to_events(code), [ + Event(evt='key', data='\t', raw=bytearray(b'\t')), # Two tabs for completion + Event(evt='key', data='\t', raw=bytearray(b'\t')), + ], code_to_events("a")) + + completing_reader = functools.partial( + prepare_reader, + readline_completer=rlcompleter.Completer(namespace).complete + ) + reader, _ = handle_all_events(events, prepare_reader=completing_reader) + + actual = reader.screen + self.assertEqual(len(actual), 2) + self.assertEqual(actual[0], f"{code}a") + self.assertEqual(actual[1].rstrip(), "itertools.accumulate(") + + def test_key_press_on_tab_press_once(self): + namespace = {"itertools": itertools} + code = "itertools." + events = itertools.chain(code_to_events(code), [ + Event(evt='key', data='\t', raw=bytearray(b'\t')), + ], code_to_events("a")) + + completing_reader = functools.partial( + prepare_reader, + readline_completer=rlcompleter.Completer(namespace).complete + ) + reader, _ = handle_all_events(events, prepare_reader=completing_reader) + + self.assert_screen_equals(reader, f"{code}a") diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py new file mode 100644 index 00000000000000..15dbf48bcf0f1c --- /dev/null +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -0,0 +1,327 @@ +import itertools +import os +import sys +import unittest +from functools import partial +from test.support import os_helper +from unittest import TestCase +from unittest.mock import MagicMock, call, patch, ANY + +from .support import handle_all_events, code_to_events + +try: + from _pyrepl.console import Event + from _pyrepl.unix_console import UnixConsole +except ImportError: + pass + + +def unix_console(events, **kwargs): + console = UnixConsole() + console.get_event = MagicMock(side_effect=events) + + height = kwargs.get("height", 25) + width = kwargs.get("width", 80) + console.getheightwidth = MagicMock(side_effect=lambda: (height, width)) + + console.prepare() + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + +handle_events_unix_console = partial( + handle_all_events, + prepare_console=partial(unix_console), +) +handle_events_narrow_unix_console = partial( + handle_all_events, + prepare_console=partial(unix_console, width=5), +) +handle_events_short_unix_console = partial( + handle_all_events, + prepare_console=partial(unix_console, height=1), +) +handle_events_unix_console_height_3 = partial( + handle_all_events, prepare_console=partial(unix_console, height=3) +) + + +TERM_CAPABILITIES = { + "bel": b"\x07", + "civis": b"\x1b[?25l", + "clear": b"\x1b[H\x1b[2J", + "cnorm": b"\x1b[?12l\x1b[?25h", + "cub": b"\x1b[%p1%dD", + "cub1": b"\x08", + "cud": b"\x1b[%p1%dB", + "cud1": b"\n", + "cuf": b"\x1b[%p1%dC", + "cuf1": b"\x1b[C", + "cup": b"\x1b[%i%p1%d;%p2%dH", + "cuu": b"\x1b[%p1%dA", + "cuu1": b"\x1b[A", + "dch1": b"\x1b[P", + "dch": b"\x1b[%p1%dP", + "el": b"\x1b[K", + "hpa": b"\x1b[%i%p1%dG", + "ich": b"\x1b[%p1%d@", + "ich1": None, + "ind": b"\n", + "pad": None, + "ri": b"\x1bM", + "rmkx": b"\x1b[?1l\x1b>", + "smkx": b"\x1b[?1h\x1b=", +} + + +@unittest.skipIf(sys.platform == "win32", "No Unix event queue on Windows") +@patch("_pyrepl.curses.tigetstr", lambda s: TERM_CAPABILITIES.get(s)) +@patch( + "_pyrepl.curses.tparm", + lambda s, *args: s + b":" + b",".join(str(i).encode() for i in args), +) +@patch("_pyrepl.curses.setupterm", lambda a, b: None) +@patch( + "termios.tcgetattr", + lambda _: [ + 27394, + 3, + 19200, + 536872399, + 38400, + 38400, + [ + b"\x04", + b"\xff", + b"\xff", + b"\x7f", + b"\x17", + b"\x15", + b"\x12", + b"\x00", + b"\x03", + b"\x1c", + b"\x1a", + b"\x19", + b"\x11", + b"\x13", + b"\x16", + b"\x0f", + b"\x01", + b"\x00", + b"\x14", + b"\x00", + ], + ], +) +@patch("termios.tcsetattr", lambda a, b, c: None) +@patch("os.write") +class TestConsole(TestCase): + def test_simple_addition(self, _os_write): + code = "12+34" + events = code_to_events(code) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, b"1") + _os_write.assert_any_call(ANY, b"2") + _os_write.assert_any_call(ANY, b"+") + _os_write.assert_any_call(ANY, b"3") + _os_write.assert_any_call(ANY, b"4") + con.restore() + + def test_wrap(self, _os_write): + code = "12+34" + events = code_to_events(code) + _, con = handle_events_narrow_unix_console(events) + _os_write.assert_any_call(ANY, b"1") + _os_write.assert_any_call(ANY, b"2") + _os_write.assert_any_call(ANY, b"+") + _os_write.assert_any_call(ANY, b"3") + _os_write.assert_any_call(ANY, b"\\") + _os_write.assert_any_call(ANY, b"\n") + _os_write.assert_any_call(ANY, b"4") + con.restore() + + def test_cursor_left(self, _os_write): + code = "1" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + ) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") + con.restore() + + def test_cursor_left_right(self, _os_write): + code = "1" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuf"] + b":1") + con.restore() + + def test_cursor_up(self, _os_write): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], + ) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuu"] + b":1") + con.restore() + + def test_cursor_up_down(self, _os_write): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuu"] + b":1") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cud"] + b":1") + con.restore() + + def test_cursor_back_write(self, _os_write): + events = itertools.chain( + code_to_events("1"), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + code_to_events("2"), + ) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, b"1") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") + _os_write.assert_any_call(ANY, b"2") + con.restore() + + def test_multiline_function_move_up_short_terminal(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + ], + ) + _, con = handle_events_short_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ri"] + b":") + con.restore() + + def test_multiline_function_move_up_down_short_terminal(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="scroll", data=None), + ], + ) + _, con = handle_events_short_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ri"] + b":") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ind"] + b":") + con.restore() + + def test_resize_bigger_on_multiline_function(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = handle_events_short_unix_console(events) + + console.height = 2 + console.getheightwidth = MagicMock(lambda _: (2, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + _os_write.assert_has_calls( + [ + call(ANY, TERM_CAPABILITIES["ri"] + b":"), + call(ANY, TERM_CAPABILITIES["cup"] + b":0,0"), + call(ANY, b"def f():"), + ] + ) + console.restore() + con.restore() + + def test_resize_smaller_on_multiline_function(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = handle_events_unix_console_height_3(events) + + console.height = 1 + console.getheightwidth = MagicMock(lambda _: (1, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + _os_write.assert_has_calls( + [ + call(ANY, TERM_CAPABILITIES["ind"] + b":"), + call(ANY, TERM_CAPABILITIES["cup"] + b":0,0"), + call(ANY, b" foo"), + ] + ) + console.restore() + con.restore() + + def test_getheightwidth_with_invalid_environ(self, _os_write): + # gh-128636 + console = UnixConsole() + with os_helper.EnvironmentVarGuard() as env: + env["LINES"] = "" + self.assertIsInstance(console.getheightwidth(), tuple) + env["COLUMNS"] = "" + self.assertIsInstance(console.getheightwidth(), tuple) + os.environ = [] + self.assertIsInstance(console.getheightwidth(), tuple) diff --git a/Lib/test/test_pyrepl/test_unix_eventqueue.py b/Lib/test/test_pyrepl/test_unix_eventqueue.py new file mode 100644 index 00000000000000..301f79927a741f --- /dev/null +++ b/Lib/test/test_pyrepl/test_unix_eventqueue.py @@ -0,0 +1,117 @@ +import tempfile +import unittest +import sys +from unittest.mock import patch + +try: + from _pyrepl.console import Event + from _pyrepl.unix_eventqueue import EventQueue +except ImportError: + pass + +@unittest.skipIf(sys.platform == "win32", "No Unix event queue on Windows") +@patch("_pyrepl.curses.tigetstr", lambda x: b"") +class TestUnixEventQueue(unittest.TestCase): + def setUp(self): + self.file = tempfile.TemporaryFile() + + def tearDown(self) -> None: + self.file.close() + + def test_get(self): + eq = EventQueue(self.file.fileno(), "utf-8") + event = Event("key", "a", b"a") + eq.insert(event) + self.assertEqual(eq.get(), event) + + def test_empty(self): + eq = EventQueue(self.file.fileno(), "utf-8") + self.assertTrue(eq.empty()) + eq.insert(Event("key", "a", b"a")) + self.assertFalse(eq.empty()) + + def test_flush_buf(self): + eq = EventQueue(self.file.fileno(), "utf-8") + eq.buf.extend(b"test") + self.assertEqual(eq.flush_buf(), b"test") + self.assertEqual(eq.buf, bytearray()) + + def test_insert(self): + eq = EventQueue(self.file.fileno(), "utf-8") + event = Event("key", "a", b"a") + eq.insert(event) + self.assertEqual(eq.events[0], event) + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_with_key_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {b"a": "b"} + eq.push("a") + mock_keymap.compile_keymap.assert_called() + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "b") + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_without_key_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {b"c": "d"} + eq.push("a") + mock_keymap.compile_keymap.assert_called() + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "a") + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_with_keymap_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {b"a": {b"b": "c"}} + eq.push("a") + mock_keymap.compile_keymap.assert_called() + self.assertTrue(eq.empty()) + eq.push("b") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "c") + eq.push("d") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "d") + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_with_keymap_in_keymap_and_escape(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {b"a": {b"b": "c"}} + eq.push("a") + mock_keymap.compile_keymap.assert_called() + self.assertTrue(eq.empty()) + eq.flush_buf() + eq.push("\033") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\033") + eq.push("b") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "b") + + def test_push_special_key(self): + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {} + eq.push("\x1b") + eq.push("[") + eq.push("A") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\x1b") + + def test_push_unrecognized_escape_sequence(self): + eq = EventQueue(self.file.fileno(), "utf-8") + eq.keymap = {} + eq.push("\x1b") + eq.push("[") + eq.push("Z") + self.assertEqual(len(eq.events), 3) + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\x1b") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "[") + self.assertEqual(eq.events[2].evt, "key") + self.assertEqual(eq.events[2].data, "Z") diff --git a/Lib/test/test_pyrepl/test_windows_console.py b/Lib/test/test_pyrepl/test_windows_console.py new file mode 100644 index 00000000000000..07eaccd1124cd6 --- /dev/null +++ b/Lib/test/test_pyrepl/test_windows_console.py @@ -0,0 +1,348 @@ +import sys +import unittest + +if sys.platform != "win32": + raise unittest.SkipTest("test only relevant on win32") + + +import itertools +from functools import partial +from typing import Iterable +from unittest import TestCase +from unittest.mock import MagicMock, call + +from .support import handle_all_events, code_to_events + +try: + from _pyrepl.console import Event, Console + from _pyrepl.windows_console import ( + WindowsConsole, + MOVE_LEFT, + MOVE_RIGHT, + MOVE_UP, + MOVE_DOWN, + ERASE_IN_LINE, + ) +except ImportError: + pass + + +class WindowsConsoleTests(TestCase): + def console(self, events, **kwargs) -> Console: + console = WindowsConsole() + console.get_event = MagicMock(side_effect=events) + console._scroll = MagicMock() + console._hide_cursor = MagicMock() + console._show_cursor = MagicMock() + console._getscrollbacksize = MagicMock(42) + console.out = MagicMock() + + height = kwargs.get("height", 25) + width = kwargs.get("width", 80) + console.getheightwidth = MagicMock(side_effect=lambda: (height, width)) + + console.prepare() + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + def handle_events(self, events: Iterable[Event], **kwargs): + return handle_all_events(events, partial(self.console, **kwargs)) + + def handle_events_narrow(self, events): + return self.handle_events(events, width=5) + + def handle_events_short(self, events): + return self.handle_events(events, height=1) + + def handle_events_height_3(self, events): + return self.handle_events(events, height=3) + + def test_simple_addition(self): + code = "12+34" + events = code_to_events(code) + _, con = self.handle_events(events) + con.out.write.assert_any_call(b"1") + con.out.write.assert_any_call(b"2") + con.out.write.assert_any_call(b"+") + con.out.write.assert_any_call(b"3") + con.out.write.assert_any_call(b"4") + con.restore() + + def test_wrap(self): + code = "12+34" + events = code_to_events(code) + _, con = self.handle_events_narrow(events) + con.out.write.assert_any_call(b"1") + con.out.write.assert_any_call(b"2") + con.out.write.assert_any_call(b"+") + con.out.write.assert_any_call(b"3") + con.out.write.assert_any_call(b"\\") + con.out.write.assert_any_call(b"\n") + con.out.write.assert_any_call(b"4") + con.restore() + + def test_resize_wider(self): + code = "1234567890" + events = code_to_events(code) + reader, console = self.handle_events_narrow(events) + + console.height = 20 + console.width = 80 + console.getheightwidth = MagicMock(lambda _: (20, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + + con.out.write.assert_any_call(self.move_right(2)) + con.out.write.assert_any_call(self.move_up(2)) + con.out.write.assert_any_call(b"567890") + + con.restore() + + def test_resize_narrower(self): + code = "1234567890" + events = code_to_events(code) + reader, console = self.handle_events(events) + + console.height = 20 + console.width = 4 + console.getheightwidth = MagicMock(lambda _: (20, 4)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + + con.out.write.assert_any_call(b"456\\") + con.out.write.assert_any_call(b"789\\") + + con.restore() + + def test_cursor_left(self): + code = "1" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(self.move_left()) + con.restore() + + def test_cursor_left_right(self): + code = "1" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(self.move_left()) + con.out.write.assert_any_call(self.move_right()) + con.restore() + + def test_cursor_up(self): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(self.move_up()) + con.restore() + + def test_cursor_up_down(self): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(self.move_up()) + con.out.write.assert_any_call(self.move_down()) + con.restore() + + def test_cursor_back_write(self): + events = itertools.chain( + code_to_events("1"), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + code_to_events("2"), + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(b"1") + con.out.write.assert_any_call(self.move_left()) + con.out.write.assert_any_call(b"21") + con.restore() + + def test_multiline_function_move_up_short_terminal(self): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + ], + ) + _, con = self.handle_events_short(events) + con.out.write.assert_any_call(self.move_left(5)) + con.out.write.assert_any_call(self.move_up()) + con.restore() + + def test_multiline_function_move_up_down_short_terminal(self): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="scroll", data=None), + ], + ) + _, con = self.handle_events_short(events) + con.out.write.assert_any_call(self.move_left(8)) + con.out.write.assert_any_call(self.erase_in_line()) + con.restore() + + def test_resize_bigger_on_multiline_function(self): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = self.handle_events_short(events) + + console.height = 2 + console.getheightwidth = MagicMock(lambda _: (2, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + con.out.write.assert_has_calls( + [ + call(self.move_left(5)), + call(self.move_up()), + call(b"def f():"), + call(self.move_left(3)), + call(self.move_down()), + ] + ) + console.restore() + con.restore() + + def test_resize_smaller_on_multiline_function(self): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = self.handle_events_height_3(events) + + console.height = 1 + console.getheightwidth = MagicMock(lambda _: (1, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + con.out.write.assert_has_calls( + [ + call(self.move_left(5)), + call(self.move_up()), + call(self.erase_in_line()), + call(b" foo"), + ] + ) + console.restore() + con.restore() + + def move_up(self, lines=1): + return MOVE_UP.format(lines).encode("utf8") + + def move_down(self, lines=1): + return MOVE_DOWN.format(lines).encode("utf8") + + def move_left(self, cols=1): + return MOVE_LEFT.format(cols).encode("utf8") + + def move_right(self, cols=1): + return MOVE_RIGHT.format(cols).encode("utf8") + + def erase_in_line(self): + return ERASE_IN_LINE.encode("utf8") + + def test_multiline_ctrl_z(self): + # see gh-126332 + code = "abcdefghi" + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data='\x1a', raw=bytearray(b'\x1a')), + Event(evt="key", data='\x1a', raw=bytearray(b'\x1a')), + ], + ) + reader, _ = self.handle_events_narrow(events) + self.assertEqual(reader.cxy, (2, 3)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 92d670ca6f8f5b..7f4fe357034b71 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -2,7 +2,6 @@ # to ensure the Queue locks remain stable. import itertools import random -import sys import threading import time import unittest @@ -317,97 +316,107 @@ def test_shutdown_all_methods_in_one_thread(self): def test_shutdown_immediate_all_methods_in_one_thread(self): return self._shutdown_all_methods_in_one_thread(True) - def _write_msg_thread(self, q, n, results, delay, - i_when_exec_shutdown, - event_start, event_end): - event_start.wait() - for i in range(1, n+1): + def _write_msg_thread(self, q, n, results, + i_when_exec_shutdown, event_shutdown, + barrier_start): + # All `write_msg_threads` + # put several items into the queue. + for i in range(0, i_when_exec_shutdown//2): + q.put((i, 'LOYD')) + # Wait for the barrier to be complete. + barrier_start.wait() + + for i in range(i_when_exec_shutdown//2, n): try: q.put((i, "YDLO")) - results.append(True) except self.queue.ShutDown: results.append(False) - # triggers shutdown of queue - if i == i_when_exec_shutdown: - event_end.set() - time.sleep(delay) - # end of all puts - q.join() + break - def _read_msg_thread(self, q, nb, results, delay, event_start): - event_start.wait() - block = True - while nb: - time.sleep(delay) + # Trigger queue shutdown. + if i == i_when_exec_shutdown: + # Only one thread should call shutdown(). + if not event_shutdown.is_set(): + event_shutdown.set() + results.append(True) + + def _read_msg_thread(self, q, results, barrier_start): + # Get at least one item. + q.get(True) + q.task_done() + # Wait for the barrier to be complete. + barrier_start.wait() + while True: try: - # Get at least one message - q.get(block) - block = False + q.get(False) q.task_done() - results.append(True) - nb -= 1 except self.queue.ShutDown: - results.append(False) - nb -= 1 + results.append(True) + break except self.queue.Empty: pass - q.join() - def _shutdown_thread(self, q, event_end, immediate): + def _shutdown_thread(self, q, results, event_end, immediate): event_end.wait() q.shutdown(immediate) - q.join() + results.append(q.qsize() == 0) - def _join_thread(self, q, delay, event_start): - event_start.wait() - time.sleep(delay) + def _join_thread(self, q, barrier_start): + # Wait for the barrier to be complete. + barrier_start.wait() q.join() def _shutdown_all_methods_in_many_threads(self, immediate): + # Run a 'multi-producers/consumers queue' use case, + # with enough items into the queue. + # When shutdown, all running threads will be joined. q = self.type2test() ps = [] - ev_start = threading.Event() - ev_exec_shutdown = threading.Event() res_puts = [] res_gets = [] - delay = 1e-4 - read_process = 4 - nb_msgs = read_process * 16 - nb_msgs_r = nb_msgs // read_process - when_exec_shutdown = nb_msgs // 2 - lprocs = ( - (self._write_msg_thread, 1, (q, nb_msgs, res_puts, delay, - when_exec_shutdown, - ev_start, ev_exec_shutdown)), - (self._read_msg_thread, read_process, (q, nb_msgs_r, - res_gets, delay*2, - ev_start)), - (self._join_thread, 2, (q, delay*2, ev_start)), - (self._shutdown_thread, 1, (q, ev_exec_shutdown, immediate)), - ) - # start all threds + res_shutdown = [] + write_threads = 4 + read_threads = 6 + join_threads = 2 + nb_msgs = 1024*64 + nb_msgs_w = nb_msgs // write_threads + when_exec_shutdown = nb_msgs_w // 2 + # Use of a Barrier to ensure that + # - all write threads put all their items into the queue, + # - all read thread get at least one item from the queue, + # and keep on running until shutdown. + # The join thread is started only when shutdown is immediate. + nparties = write_threads + read_threads + if immediate: + nparties += join_threads + barrier_start = threading.Barrier(nparties) + ev_exec_shutdown = threading.Event() + lprocs = [ + (self._write_msg_thread, write_threads, (q, nb_msgs_w, res_puts, + when_exec_shutdown, ev_exec_shutdown, + barrier_start)), + (self._read_msg_thread, read_threads, (q, res_gets, barrier_start)), + (self._shutdown_thread, 1, (q, res_shutdown, ev_exec_shutdown, immediate)), + ] + if immediate: + lprocs.append((self._join_thread, join_threads, (q, barrier_start))) + # start all threads. for func, n, args in lprocs: for i in range(n): ps.append(threading.Thread(target=func, args=args)) ps[-1].start() - # set event in order to run q.shutdown() - ev_start.set() - - if not immediate: - assert(len(res_gets) == len(res_puts)) - assert(res_gets.count(True) == res_puts.count(True)) - else: - assert(len(res_gets) <= len(res_puts)) - assert(res_gets.count(True) <= res_puts.count(True)) - - for thread in ps[1:]: + for thread in ps: thread.join() - @unittest.skip("test times out (gh-115258)") + self.assertTrue(True in res_puts) + self.assertEqual(res_gets.count(True), read_threads) + if immediate: + self.assertListEqual(res_shutdown, [True]) + self.assertTrue(q.empty()) + def test_shutdown_all_methods_in_many_threads(self): return self._shutdown_all_methods_in_many_threads(False) - @unittest.skip("test times out (gh-115258)") def test_shutdown_immediate_all_methods_in_many_threads(self): return self._shutdown_all_methods_in_many_threads(True) @@ -476,7 +485,7 @@ def _shutdown_get(self, immediate): else: thrds = ( # on shutdown(immediate=False) - # one of these threads shoud raise Shutdown + # one of these threads should raise Shutdown (self._get, (q, go, results)), (self._get, (q, go, results)), (self._get, (q, go, results)), @@ -567,7 +576,6 @@ def _shutdown_put_join(self, immediate): results = [] go = threading.Event() q.put("Y") - nb = q.qsize() # queue not fulled thrds = ( @@ -578,13 +586,19 @@ def _shutdown_put_join(self, immediate): for func, params in thrds: threads.append(threading.Thread(target=func, args=params)) threads[-1].start() - self.assertEqual(q.unfinished_tasks, nb) - for i in range(nb): - t = threading.Thread(target=q.task_done) - t.start() - threads.append(t) + self.assertEqual(q.unfinished_tasks, 1) + q.shutdown(immediate) go.set() + + if immediate: + with self.assertRaises(self.queue.ShutDown): + q.get_nowait() + else: + result = q.get() + self.assertEqual(result, "Y") + q.task_done() + for t in threads: t.join() @@ -621,6 +635,23 @@ def test_shutdown_get_task_done_join(self): self.assertEqual(results, [True]*len(thrds)) + def test_shutdown_pending_get(self): + def get(): + try: + results.append(q.get()) + except Exception as e: + results.append(e) + + q = self.type2test() + results = [] + get_thread = threading.Thread(target=get) + get_thread.start() + q.shutdown(immediate=False) + get_thread.join(timeout=10.0) + self.assertFalse(get_thread.is_alive()) + self.assertEqual(len(results), 1) + self.assertIsInstance(results[0], self.queue.ShutDown) + class QueueTest(BaseQueueTestMixin): diff --git a/Lib/test/test_raise.py b/Lib/test/test_raise.py index 6d26a61bee4292..dcf0753bc828f3 100644 --- a/Lib/test/test_raise.py +++ b/Lib/test/test_raise.py @@ -48,7 +48,7 @@ def test_except_reraise(self): def reraise(): try: raise TypeError("foo") - except: + except TypeError: try: raise KeyError("caught") except KeyError: @@ -60,7 +60,7 @@ def test_finally_reraise(self): def reraise(): try: raise TypeError("foo") - except: + except TypeError: try: raise KeyError("caught") finally: @@ -73,7 +73,7 @@ def nested_reraise(): def reraise(): try: raise TypeError("foo") - except: + except TypeError: nested_reraise() self.assertRaises(TypeError, reraise) @@ -81,7 +81,7 @@ def test_raise_from_None(self): try: try: raise TypeError("foo") - except: + except TypeError: raise ValueError() from None except ValueError as e: self.assertIsInstance(e.__context__, TypeError) @@ -91,7 +91,7 @@ def test_with_reraise1(self): def reraise(): try: raise TypeError("foo") - except: + except TypeError: with Context(): pass raise @@ -101,7 +101,7 @@ def test_with_reraise2(self): def reraise(): try: raise TypeError("foo") - except: + except TypeError: with Context(): raise KeyError("caught") raise @@ -111,7 +111,7 @@ def test_yield_reraise(self): def reraise(): try: raise TypeError("foo") - except: + except TypeError: yield 1 raise g = reraise() @@ -314,7 +314,7 @@ def test_instance_context_instance_raise(self): try: try: raise context - except: + except IndexError: raise OSError() except OSError as e: self.assertIs(e.__context__, context) @@ -326,7 +326,7 @@ def test_class_context_instance_raise(self): try: try: raise context - except: + except IndexError: raise OSError() except OSError as e: self.assertIsNot(e.__context__, context) @@ -339,7 +339,7 @@ def test_class_context_class_raise(self): try: try: raise context - except: + except IndexError: raise OSError except OSError as e: self.assertIsNot(e.__context__, context) @@ -351,7 +351,7 @@ def test_c_exception_context(self): try: try: 1/0 - except: + except ZeroDivisionError: raise OSError except OSError as e: self.assertIsInstance(e.__context__, ZeroDivisionError) @@ -362,7 +362,7 @@ def test_c_exception_raise(self): try: try: 1/0 - except: + except ZeroDivisionError: xyzzy except NameError as e: self.assertIsInstance(e.__context__, ZeroDivisionError) @@ -459,7 +459,7 @@ def f(): try: try: raise ValueError - except: + except ValueError: del g raise KeyError except Exception as e: @@ -475,7 +475,7 @@ class C: def __del__(self): try: 1/0 - except: + except ZeroDivisionError: raise def f(): diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index b1e4ef4197d130..51f9193b269eee 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -4,6 +4,7 @@ import os import time import pickle +import shlex import warnings import test.support @@ -1397,5 +1398,47 @@ def test_after_fork(self): support.wait_process(pid, exitcode=0) +class CommandLineTest(unittest.TestCase): + def test_parse_args(self): + args, help_text = random._parse_args(shlex.split("--choice a b c")) + self.assertEqual(args.choice, ["a", "b", "c"]) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("--integer 5")) + self.assertEqual(args.integer, 5) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("--float 2.5")) + self.assertEqual(args.float, 2.5) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("a b c")) + self.assertEqual(args.input, ["a", "b", "c"]) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("5")) + self.assertEqual(args.input, ["5"]) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("2.5")) + self.assertEqual(args.input, ["2.5"]) + self.assertTrue(help_text.startswith("usage: ")) + + def test_main(self): + for command, expected in [ + ("--choice a b c", "b"), + ('"a b c"', "b"), + ("a b c", "b"), + ("--choice 'a a' 'b b' 'c c'", "b b"), + ("'a a' 'b b' 'c c'", "b b"), + ("--integer 5", 4), + ("5", 4), + ("--float 2.5", 2.1110546288126204), + ("2.5", 2.1110546288126204), + ]: + random.seed(0) + self.assertEqual(random.main(shlex.split(command)), expected) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index b1ac22c28cf7c1..5538de60b2a03a 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1,12 +1,11 @@ from test.support import (gc_collect, bigmemtest, _2G, cpython_only, captured_stdout, check_disallow_instantiation, is_emscripten, is_wasi, - warnings_helper, SHORT_TIMEOUT, CPUStopwatch) + warnings_helper, SHORT_TIMEOUT, CPUStopwatch, requires_resource) import locale import re import string import sys -import time import unittest import warnings from re import Scanner @@ -14,7 +13,7 @@ # some platforms lack working multiprocessing try: - import _multiprocessing + import _multiprocessing # noqa: F401 except ImportError: multiprocessing = None else: @@ -884,31 +883,134 @@ def test_named_unicode_escapes(self): self.checkPatternError(br'\N{LESS-THAN SIGN}', r'bad escape \N', 0) self.checkPatternError(br'[\N{LESS-THAN SIGN}]', r'bad escape \N', 1) - def test_string_boundaries(self): + def test_word_boundaries(self): # See http://bugs.python.org/issue10713 - self.assertEqual(re.search(r"\b(abc)\b", "abc").group(1), - "abc") + self.assertEqual(re.search(r"\b(abc)\b", "abc").group(1), "abc") + self.assertEqual(re.search(r"\b(abc)\b", "abc", re.ASCII).group(1), "abc") + self.assertEqual(re.search(br"\b(abc)\b", b"abc").group(1), b"abc") + self.assertEqual(re.search(br"\b(abc)\b", b"abc", re.LOCALE).group(1), b"abc") + self.assertEqual(re.search(r"\b(ьюя)\b", "ьюя").group(1), "ьюя") + self.assertIsNone(re.search(r"\b(ьюя)\b", "ьюя", re.ASCII)) + # There's a word boundary between a word and a non-word. + self.assertTrue(re.match(r".\b", "a=")) + self.assertTrue(re.match(r".\b", "a=", re.ASCII)) + self.assertTrue(re.match(br".\b", b"a=")) + self.assertTrue(re.match(br".\b", b"a=", re.LOCALE)) + self.assertTrue(re.match(r".\b", "я=")) + self.assertIsNone(re.match(r".\b", "я=", re.ASCII)) + # There's a word boundary between a non-word and a word. + self.assertTrue(re.match(r".\b", "=a")) + self.assertTrue(re.match(r".\b", "=a", re.ASCII)) + self.assertTrue(re.match(br".\b", b"=a")) + self.assertTrue(re.match(br".\b", b"=a", re.LOCALE)) + self.assertTrue(re.match(r".\b", "=я")) + self.assertIsNone(re.match(r".\b", "=я", re.ASCII)) + # There is no word boundary inside a word. + self.assertIsNone(re.match(r".\b", "ab")) + self.assertIsNone(re.match(r".\b", "ab", re.ASCII)) + self.assertIsNone(re.match(br".\b", b"ab")) + self.assertIsNone(re.match(br".\b", b"ab", re.LOCALE)) + self.assertIsNone(re.match(r".\b", "юя")) + self.assertIsNone(re.match(r".\b", "юя", re.ASCII)) + # There is no word boundary between a non-word characters. + self.assertIsNone(re.match(r".\b", "=-")) + self.assertIsNone(re.match(r".\b", "=-", re.ASCII)) + self.assertIsNone(re.match(br".\b", b"=-")) + self.assertIsNone(re.match(br".\b", b"=-", re.LOCALE)) + # There is no non-boundary match between a word and a non-word. + self.assertIsNone(re.match(r".\B", "a=")) + self.assertIsNone(re.match(r".\B", "a=", re.ASCII)) + self.assertIsNone(re.match(br".\B", b"a=")) + self.assertIsNone(re.match(br".\B", b"a=", re.LOCALE)) + self.assertIsNone(re.match(r".\B", "я=")) + self.assertTrue(re.match(r".\B", "я=", re.ASCII)) + # There is no non-boundary match between a non-word and a word. + self.assertIsNone(re.match(r".\B", "=a")) + self.assertIsNone(re.match(r".\B", "=a", re.ASCII)) + self.assertIsNone(re.match(br".\B", b"=a")) + self.assertIsNone(re.match(br".\B", b"=a", re.LOCALE)) + self.assertIsNone(re.match(r".\B", "=я")) + self.assertTrue(re.match(r".\B", "=я", re.ASCII)) + # There's a non-boundary match inside a word. + self.assertTrue(re.match(r".\B", "ab")) + self.assertTrue(re.match(r".\B", "ab", re.ASCII)) + self.assertTrue(re.match(br".\B", b"ab")) + self.assertTrue(re.match(br".\B", b"ab", re.LOCALE)) + self.assertTrue(re.match(r".\B", "юя")) + self.assertTrue(re.match(r".\B", "юя", re.ASCII)) + # There's a non-boundary match between a non-word characters. + self.assertTrue(re.match(r".\B", "=-")) + self.assertTrue(re.match(r".\B", "=-", re.ASCII)) + self.assertTrue(re.match(br".\B", b"=-")) + self.assertTrue(re.match(br".\B", b"=-", re.LOCALE)) # There's a word boundary at the start of a string. self.assertTrue(re.match(r"\b", "abc")) + self.assertTrue(re.match(r"\b", "abc", re.ASCII)) + self.assertTrue(re.match(br"\b", b"abc")) + self.assertTrue(re.match(br"\b", b"abc", re.LOCALE)) + self.assertTrue(re.match(r"\b", "ьюя")) + self.assertIsNone(re.match(r"\b", "ьюя", re.ASCII)) + # There's a word boundary at the end of a string. + self.assertTrue(re.fullmatch(r".+\b", "abc")) + self.assertTrue(re.fullmatch(r".+\b", "abc", re.ASCII)) + self.assertTrue(re.fullmatch(br".+\b", b"abc")) + self.assertTrue(re.fullmatch(br".+\b", b"abc", re.LOCALE)) + self.assertTrue(re.fullmatch(r".+\b", "ьюя")) + self.assertIsNone(re.search(r"\b", "ьюя", re.ASCII)) # A non-empty string includes a non-boundary zero-length match. - self.assertTrue(re.search(r"\B", "abc")) + self.assertEqual(re.search(r"\B", "abc").span(), (1, 1)) + self.assertEqual(re.search(r"\B", "abc", re.ASCII).span(), (1, 1)) + self.assertEqual(re.search(br"\B", b"abc").span(), (1, 1)) + self.assertEqual(re.search(br"\B", b"abc", re.LOCALE).span(), (1, 1)) + self.assertEqual(re.search(r"\B", "ьюя").span(), (1, 1)) + self.assertEqual(re.search(r"\B", "ьюя", re.ASCII).span(), (0, 0)) # There is no non-boundary match at the start of a string. - self.assertFalse(re.match(r"\B", "abc")) - # However, an empty string contains no word boundaries, and also no - # non-boundaries. - self.assertIsNone(re.search(r"\B", "")) - # This one is questionable and different from the perlre behaviour, - # but describes current behavior. + self.assertIsNone(re.match(r"\B", "abc")) + self.assertIsNone(re.match(r"\B", "abc", re.ASCII)) + self.assertIsNone(re.match(br"\B", b"abc")) + self.assertIsNone(re.match(br"\B", b"abc", re.LOCALE)) + self.assertIsNone(re.match(r"\B", "ьюя")) + self.assertTrue(re.match(r"\B", "ьюя", re.ASCII)) + # There is no non-boundary match at the end of a string. + self.assertIsNone(re.fullmatch(r".+\B", "abc")) + self.assertIsNone(re.fullmatch(r".+\B", "abc", re.ASCII)) + self.assertIsNone(re.fullmatch(br".+\B", b"abc")) + self.assertIsNone(re.fullmatch(br".+\B", b"abc", re.LOCALE)) + self.assertIsNone(re.fullmatch(r".+\B", "ьюя")) + self.assertTrue(re.fullmatch(r".+\B", "ьюя", re.ASCII)) + # However, an empty string contains no word boundaries. self.assertIsNone(re.search(r"\b", "")) + self.assertIsNone(re.search(r"\b", "", re.ASCII)) + self.assertIsNone(re.search(br"\b", b"")) + self.assertIsNone(re.search(br"\b", b"", re.LOCALE)) + self.assertTrue(re.search(r"\B", "")) + self.assertTrue(re.search(r"\B", "", re.ASCII)) + self.assertTrue(re.search(br"\B", b"")) + self.assertTrue(re.search(br"\B", b"", re.LOCALE)) # A single word-character string has two boundaries, but no # non-boundary gaps. self.assertEqual(len(re.findall(r"\b", "a")), 2) + self.assertEqual(len(re.findall(r"\b", "a", re.ASCII)), 2) + self.assertEqual(len(re.findall(br"\b", b"a")), 2) + self.assertEqual(len(re.findall(br"\b", b"a", re.LOCALE)), 2) self.assertEqual(len(re.findall(r"\B", "a")), 0) + self.assertEqual(len(re.findall(r"\B", "a", re.ASCII)), 0) + self.assertEqual(len(re.findall(br"\B", b"a")), 0) + self.assertEqual(len(re.findall(br"\B", b"a", re.LOCALE)), 0) # If there are no words, there are no boundaries self.assertEqual(len(re.findall(r"\b", " ")), 0) + self.assertEqual(len(re.findall(r"\b", " ", re.ASCII)), 0) + self.assertEqual(len(re.findall(br"\b", b" ")), 0) + self.assertEqual(len(re.findall(br"\b", b" ", re.LOCALE)), 0) self.assertEqual(len(re.findall(r"\b", " ")), 0) + self.assertEqual(len(re.findall(r"\b", " ", re.ASCII)), 0) + self.assertEqual(len(re.findall(br"\b", b" ")), 0) + self.assertEqual(len(re.findall(br"\b", b" ", re.LOCALE)), 0) # Can match around the whitespace. self.assertEqual(len(re.findall(r"\B", " ")), 2) + self.assertEqual(len(re.findall(r"\B", " ", re.ASCII)), 2) + self.assertEqual(len(re.findall(br"\B", b" ")), 2) + self.assertEqual(len(re.findall(br"\B", b" ", re.LOCALE)), 2) def test_bigcharset(self): self.assertEqual(re.match("([\u2222\u2223])", @@ -1031,6 +1133,39 @@ def test_ignore_case_set(self): self.assertTrue(re.match(br'[19a]', b'a', re.I)) self.assertTrue(re.match(br'[19a]', b'A', re.I)) self.assertTrue(re.match(br'[19A]', b'a', re.I)) + self.assertTrue(re.match(r'[19\xc7]', '\xc7', re.I)) + self.assertTrue(re.match(r'[19\xc7]', '\xe7', re.I)) + self.assertTrue(re.match(r'[19\xe7]', '\xc7', re.I)) + self.assertTrue(re.match(r'[19\xe7]', '\xe7', re.I)) + self.assertTrue(re.match(r'[19\u0400]', '\u0400', re.I)) + self.assertTrue(re.match(r'[19\u0400]', '\u0450', re.I)) + self.assertTrue(re.match(r'[19\u0450]', '\u0400', re.I)) + self.assertTrue(re.match(r'[19\u0450]', '\u0450', re.I)) + self.assertTrue(re.match(r'[19\U00010400]', '\U00010400', re.I)) + self.assertTrue(re.match(r'[19\U00010400]', '\U00010428', re.I)) + self.assertTrue(re.match(r'[19\U00010428]', '\U00010400', re.I)) + self.assertTrue(re.match(r'[19\U00010428]', '\U00010428', re.I)) + + self.assertTrue(re.match(br'[19A]', b'A', re.I)) + self.assertTrue(re.match(br'[19a]', b'a', re.I)) + self.assertTrue(re.match(br'[19a]', b'A', re.I)) + self.assertTrue(re.match(br'[19A]', b'a', re.I)) + self.assertTrue(re.match(r'[19A]', 'A', re.I|re.A)) + self.assertTrue(re.match(r'[19a]', 'a', re.I|re.A)) + self.assertTrue(re.match(r'[19a]', 'A', re.I|re.A)) + self.assertTrue(re.match(r'[19A]', 'a', re.I|re.A)) + self.assertTrue(re.match(r'[19\xc7]', '\xc7', re.I|re.A)) + self.assertIsNone(re.match(r'[19\xc7]', '\xe7', re.I|re.A)) + self.assertIsNone(re.match(r'[19\xe7]', '\xc7', re.I|re.A)) + self.assertTrue(re.match(r'[19\xe7]', '\xe7', re.I|re.A)) + self.assertTrue(re.match(r'[19\u0400]', '\u0400', re.I|re.A)) + self.assertIsNone(re.match(r'[19\u0400]', '\u0450', re.I|re.A)) + self.assertIsNone(re.match(r'[19\u0450]', '\u0400', re.I|re.A)) + self.assertTrue(re.match(r'[19\u0450]', '\u0450', re.I|re.A)) + self.assertTrue(re.match(r'[19\U00010400]', '\U00010400', re.I|re.A)) + self.assertIsNone(re.match(r'[19\U00010400]', '\U00010428', re.I|re.A)) + self.assertIsNone(re.match(r'[19\U00010428]', '\U00010400', re.I|re.A)) + self.assertTrue(re.match(r'[19\U00010428]', '\U00010428', re.I|re.A)) # Two different characters have the same lowercase. assert 'K'.lower() == '\u212a'.lower() == 'k' # 'K' @@ -1067,8 +1202,10 @@ def test_ignore_case_range(self): self.assertTrue(re.match(br'[9-a]', b'_', re.I)) self.assertIsNone(re.match(br'[9-A]', b'_', re.I)) self.assertTrue(re.match(r'[\xc0-\xde]', '\xd7', re.I)) + self.assertTrue(re.match(r'[\xc0-\xde]', '\xe7', re.I)) self.assertIsNone(re.match(r'[\xc0-\xde]', '\xf7', re.I)) self.assertTrue(re.match(r'[\xe0-\xfe]', '\xf7', re.I)) + self.assertTrue(re.match(r'[\xe0-\xfe]', '\xc7', re.I)) self.assertIsNone(re.match(r'[\xe0-\xfe]', '\xd7', re.I)) self.assertTrue(re.match(r'[\u0430-\u045f]', '\u0450', re.I)) self.assertTrue(re.match(r'[\u0430-\u045f]', '\u0400', re.I)) @@ -1079,6 +1216,26 @@ def test_ignore_case_range(self): self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010428', re.I)) self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010400', re.I)) + self.assertTrue(re.match(r'[\xc0-\xde]', '\xd7', re.I|re.A)) + self.assertIsNone(re.match(r'[\xc0-\xde]', '\xe7', re.I|re.A)) + self.assertTrue(re.match(r'[\xe0-\xfe]', '\xf7', re.I|re.A)) + self.assertIsNone(re.match(r'[\xe0-\xfe]', '\xc7', re.I|re.A)) + self.assertTrue(re.match(r'[\u0430-\u045f]', '\u0450', re.I|re.A)) + self.assertIsNone(re.match(r'[\u0430-\u045f]', '\u0400', re.I|re.A)) + self.assertIsNone(re.match(r'[\u0400-\u042f]', '\u0450', re.I|re.A)) + self.assertTrue(re.match(r'[\u0400-\u042f]', '\u0400', re.I|re.A)) + self.assertTrue(re.match(r'[\U00010428-\U0001044f]', '\U00010428', re.I|re.A)) + self.assertIsNone(re.match(r'[\U00010428-\U0001044f]', '\U00010400', re.I|re.A)) + self.assertIsNone(re.match(r'[\U00010400-\U00010427]', '\U00010428', re.I|re.A)) + self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010400', re.I|re.A)) + + self.assertTrue(re.match(r'[N-\x7f]', 'A', re.I|re.A)) + self.assertTrue(re.match(r'[n-\x7f]', 'Z', re.I|re.A)) + self.assertTrue(re.match(r'[N-\uffff]', 'A', re.I|re.A)) + self.assertTrue(re.match(r'[n-\uffff]', 'Z', re.I|re.A)) + self.assertTrue(re.match(r'[N-\U00010000]', 'A', re.I|re.A)) + self.assertTrue(re.match(r'[n-\U00010000]', 'Z', re.I|re.A)) + # Two different characters have the same lowercase. assert 'K'.lower() == '\u212a'.lower() == 'k' # 'K' self.assertTrue(re.match(r'[J-M]', '\u212a', re.I)) @@ -1116,47 +1273,76 @@ def test_not_literal(self): def test_possible_set_operations(self): s = bytes(range(128)).decode() - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set difference') as w: p = re.compile(r'[0-9--1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('-./0123456789')) + with self.assertWarnsRegex(FutureWarning, 'Possible set difference') as w: + self.assertEqual(re.findall(r'[0-9--2]', s), list('-./0123456789')) + self.assertEqual(w.filename, __file__) + self.assertEqual(re.findall(r'[--1]', s), list('-./01')) - with self.assertWarns(FutureWarning): + + with self.assertWarnsRegex(FutureWarning, 'Possible set difference') as w: p = re.compile(r'[%--1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list("%&'()*+,-1")) - with self.assertWarns(FutureWarning): + + with self.assertWarnsRegex(FutureWarning, 'Possible set difference ') as w: p = re.compile(r'[%--]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list("%&'()*+,-")) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set intersection ') as w: p = re.compile(r'[0-9&&1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('&0123456789')) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set intersection ') as w: + self.assertEqual(re.findall(r'[0-8&&1]', s), list('&012345678')) + self.assertEqual(w.filename, __file__) + + with self.assertWarnsRegex(FutureWarning, 'Possible set intersection ') as w: p = re.compile(r'[\d&&1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('&0123456789')) + self.assertEqual(re.findall(r'[&&1]', s), list('&1')) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set union ') as w: p = re.compile(r'[0-9||a]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789a|')) - with self.assertWarns(FutureWarning): + + with self.assertWarnsRegex(FutureWarning, 'Possible set union ') as w: p = re.compile(r'[\d||a]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789a|')) + self.assertEqual(re.findall(r'[||1]', s), list('1|')) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set symmetric difference ') as w: p = re.compile(r'[0-9~~1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789~')) - with self.assertWarns(FutureWarning): + + with self.assertWarnsRegex(FutureWarning, 'Possible set symmetric difference ') as w: p = re.compile(r'[\d~~1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789~')) + self.assertEqual(re.findall(r'[~~1]', s), list('1~')) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible nested set ') as w: p = re.compile(r'[[0-9]|]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789[]')) + with self.assertWarnsRegex(FutureWarning, 'Possible nested set ') as w: + self.assertEqual(re.findall(r'[[0-8]|]', s), list('012345678[]')) + self.assertEqual(w.filename, __file__) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible nested set ') as w: p = re.compile(r'[[:digit:]|]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list(':[]dgit')) def test_search_coverage(self): @@ -1229,7 +1415,7 @@ def test_pickling(self): newpat = pickle.loads(pickled) self.assertEqual(newpat, oldpat) # current pickle expects the _compile() reconstructor in re module - from re import _compile + from re import _compile # noqa: F401 def test_copying(self): import copy @@ -2282,6 +2468,9 @@ def test_bug_40736(self): with self.assertRaisesRegex(TypeError, "got 'type'"): re.search("x*", type) + # gh-117594: The test is not slow by itself, but it relies on + # the absolute computation time and can fail on very slow computers. + @requires_resource('cpu') def test_search_anchor_at_beginning(self): s = 'x'*10**7 with CPUStopwatch() as stopwatch: @@ -2448,6 +2637,12 @@ def test_bug_gh100061(self): self.assertEqual(re.match("(?>(?:ab?c){1,3})", "aca").span(), (0, 2)) self.assertEqual(re.match("(?:ab?c){1,3}+", "aca").span(), (0, 2)) + def test_bug_gh101955(self): + # Possessive quantifier with nested alternative with capture groups + self.assertEqual(re.match('((x)|y|z)*+', 'xyz').groups(), ('z', 'x')) + self.assertEqual(re.match('((x)|y|z){3}+', 'xyz').groups(), ('z', 'x')) + self.assertEqual(re.match('((x)|y|z){3,}+', 'xyz').groups(), ('z', 'x')) + @unittest.skipIf(multiprocessing is None, 'test requires multiprocessing') def test_regression_gh94675(self): pattern = re.compile(r'(?<=[({}])(((//[^\n]*)?[\n])([\000-\040])*)*' @@ -2471,6 +2666,47 @@ def test_regression_gh94675(self): def test_fail(self): self.assertEqual(re.search(r'12(?!)|3', '123')[0], '3') + def test_character_set_any(self): + # The union of complementary character sets matches any character + # and is equivalent to "(?s:.)". + s = '1x\n' + for p in r'[\s\S]', r'[\d\D]', r'[\w\W]', r'[\S\s]', r'\s|\S': + with self.subTest(pattern=p): + self.assertEqual(re.findall(p, s), list(s)) + self.assertEqual(re.fullmatch('(?:' + p + ')+', s).group(), s) + + def test_character_set_none(self): + # Negation of the union of complementary character sets does not match + # any character. + s = '1x\n' + for p in r'[^\s\S]', r'[^\d\D]', r'[^\w\W]', r'[^\S\s]': + with self.subTest(pattern=p): + self.assertIsNone(re.search(p, s)) + self.assertIsNone(re.search('(?s:.)' + p, s)) + + def check_interrupt(self, pattern, string, maxcount): + class Interrupt(Exception): + pass + p = re.compile(pattern) + for n in range(maxcount): + try: + p._fail_after(n, Interrupt) + p.match(string) + return n + except Interrupt: + pass + finally: + p._fail_after(-1, None) + + @unittest.skipUnless(hasattr(re.Pattern, '_fail_after'), 'requires debug build') + def test_memory_leaks(self): + self.check_interrupt(r'(.)*:', 'abc:', 100) + self.check_interrupt(r'([^:])*?:', 'abc:', 100) + self.check_interrupt(r'([^:])*+:', 'abc:', 100) + self.check_interrupt(r'(.){2,4}:', 'abc:', 100) + self.check_interrupt(r'([^:]){2,4}?:', 'abc:', 100) + self.check_interrupt(r'([^:]){2,4}+:', 'abc:', 100) + def get_debug_out(pat): with captured_stdout() as out: diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index 5e0e6f8dfac651..8b8772c66ee654 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -12,6 +12,7 @@ from test.support.os_helper import unlink, temp_dir, TESTFN from test.support.pty_helper import run_pty from test.support.script_helper import assert_python_ok +from test.support.threading_helper import requires_working_threading # Skip tests if there is no readline module readline = import_module('readline') @@ -113,6 +114,14 @@ def test_write_read_append(self): # write_history_file can create the target readline.write_history_file(hfilename) + # Negative values should be disallowed + with self.assertRaises(ValueError): + readline.append_history_file(-42, hfilename) + + # See gh-122431, using the minimum signed integer value caused a segfault + with self.assertRaises(ValueError): + readline.append_history_file(-2147483648, hfilename) + def test_nonascii_history(self): readline.clear_history() try: @@ -132,6 +141,32 @@ def test_nonascii_history(self): self.assertEqual(readline.get_history_item(1), "entrée 1") self.assertEqual(readline.get_history_item(2), "entrée 22") + def test_write_read_limited_history(self): + previous_length = readline.get_history_length() + self.addCleanup(readline.set_history_length, previous_length) + + readline.clear_history() + readline.add_history("first line") + readline.add_history("second line") + readline.add_history("third line") + + readline.set_history_length(2) + self.assertEqual(readline.get_history_length(), 2) + readline.write_history_file(TESTFN) + self.addCleanup(os.remove, TESTFN) + + readline.clear_history() + self.assertEqual(readline.get_current_history_length(), 0) + self.assertEqual(readline.get_history_length(), 2) + + readline.read_history_file(TESTFN) + self.assertEqual(readline.get_history_item(1), "second line") + self.assertEqual(readline.get_history_item(2), "third line") + self.assertEqual(readline.get_history_item(3), None) + + # Readline seems to report an additional history element. + self.assertIn(readline.get_current_history_length(), (2, 3)) + class TestReadline(unittest.TestCase): @@ -323,6 +358,50 @@ def test_history_size(self): self.assertEqual(len(lines), history_size) self.assertEqual(lines[-1].strip(), b"last input") + @requires_working_threading() + def test_gh123321_threadsafe(self): + """gh-123321: readline should be thread-safe and not crash""" + script = textwrap.dedent(r""" + import threading + from test.support.threading_helper import join_thread + + def func(): + input() + + thread1 = threading.Thread(target=func) + thread2 = threading.Thread(target=func) + thread1.start() + thread2.start() + join_thread(thread1) + join_thread(thread2) + print("done") + """) + + output = run_pty(script, input=b"input1\rinput2\r") + + self.assertIn(b"done", output) + + + def test_write_read_limited_history(self): + previous_length = readline.get_history_length() + self.addCleanup(readline.set_history_length, previous_length) + + readline.add_history("first line") + readline.add_history("second line") + readline.add_history("third line") + + readline.set_history_length(2) + self.assertEqual(readline.get_history_length(), 2) + readline.write_history_file(TESTFN) + self.addCleanup(os.remove, TESTFN) + + readline.read_history_file(TESTFN) + # Without clear_history() there's no good way to test if + # the correct entries are present (we're combining history limiting and + # possible deduplication with arbitrary previous content). + # So, we've only tested that the read did not fail. + # See TestHistoryManipulation for the full test. + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 7e1eaa7d6a515e..969f483814d08d 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -4,6 +4,7 @@ Note: test_regrtest cannot be run twice in parallel. """ +import _colorize import contextlib import dataclasses import glob @@ -21,13 +22,17 @@ import tempfile import textwrap import unittest +import unittest.mock +from xml.etree import ElementTree + from test import support -from test.support import os_helper, without_optimizer +from test.support import import_helper +from test.support import os_helper from test.libregrtest import cmdline from test.libregrtest import main from test.libregrtest import setup from test.libregrtest import utils -from test.libregrtest.filter import set_match_tests, match_test +from test.libregrtest.filter import get_match_tests, set_match_tests, match_test from test.libregrtest.result import TestStats from test.libregrtest.utils import normalize_test_name @@ -464,6 +469,28 @@ def test_bisect(self): regrtest = self.create_regrtest(args) self.assertTrue(regrtest.want_bisect) + def test_verbose3_huntrleaks(self): + args = ['-R', '3:10', '--verbose3'] + with support.captured_stderr(): + regrtest = self.create_regrtest(args) + self.assertIsNotNone(regrtest.hunt_refleak) + self.assertEqual(regrtest.hunt_refleak.warmups, 3) + self.assertEqual(regrtest.hunt_refleak.runs, 10) + self.assertFalse(regrtest.output_on_failure) + + def test_single_process(self): + args = ['-j2', '--single-process'] + with support.captured_stderr(): + regrtest = self.create_regrtest(args) + self.assertEqual(regrtest.num_workers, 0) + self.assertTrue(regrtest.single_process) + + args = ['--fast-ci', '--single-process'] + with support.captured_stderr(): + regrtest = self.create_regrtest(args) + self.assertEqual(regrtest.num_workers, 0) + self.assertTrue(regrtest.single_process) + @dataclasses.dataclass(slots=True) class Rerun: @@ -765,6 +792,7 @@ def test_finds_expected_number_of_tests(self): f'{", ".join(output.splitlines())}') +@support.force_not_colorized_test_class class ProgramsTestCase(BaseTestCase): """ Test various ways to run the Python test suite. Use options close @@ -878,6 +906,7 @@ def test_pcbuild_rt(self): self.run_batch(script, *rt_args, *self.regrtest_args, *self.tests) +@support.force_not_colorized_test_class class ArgsTestCase(BaseTestCase): """ Test arguments of the Python test suite. @@ -1113,7 +1142,7 @@ def test_coverage(self): output = self.run_tests("--coverage", test) self.check_executed_tests(output, [test], stats=1) regex = (r'lines +cov% +module +\(path\)\n' - r'(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+') + r'(?: *[0-9]+ *[0-9]{1,2}\.[0-9]% *[^ ]+ +\([^)]+\)+)+') self.check_line(output, regex) def test_wait(self): @@ -1156,7 +1185,7 @@ def test_run(self): stats=TestStats(4, 1), forever=True) - @without_optimizer + @support.requires_jit_disabled def check_leak(self, code, what, *, run_workers=False): test = self.create_test('huntrleaks', code=code) @@ -1724,6 +1753,9 @@ def test_other_bug(self): @support.cpython_only def test_uncollectable(self): + # Skip test if _testcapi is missing + import_helper.import_module('_testcapi') + code = textwrap.dedent(r""" import _testcapi import gc @@ -2106,30 +2138,34 @@ def test_unload_tests(self): def check_add_python_opts(self, option): # --fast-ci and --slow-ci add "-u -W default -bb -E" options to Python + + # Skip test if _testinternalcapi is missing + import_helper.import_module('_testinternalcapi') + code = textwrap.dedent(r""" import sys import unittest from test import support try: - from _testinternalcapi import get_config + from _testcapi import config_get except ImportError: - get_config = None + config_get = None # WASI/WASM buildbots don't use -E option use_environment = (support.is_emscripten or support.is_wasi) class WorkerTests(unittest.TestCase): - @unittest.skipUnless(get_config is None, 'need get_config()') + @unittest.skipUnless(config_get is None, 'need config_get()') def test_config(self): - config = get_config()['config'] + config = config_get() # -u option - self.assertEqual(config['buffered_stdio'], 0) + self.assertEqual(config_get('buffered_stdio'), 0) # -W default option - self.assertTrue(config['warnoptions'], ['default']) + self.assertTrue(config_get('warnoptions'), ['default']) # -bb option - self.assertTrue(config['bytes_warning'], 2) + self.assertTrue(config_get('bytes_warning'), 2) # -E option - self.assertTrue(config['use_environment'], use_environment) + self.assertTrue(config_get('use_environment'), use_environment) def test_python_opts(self): # -u option @@ -2168,10 +2204,8 @@ def test_add_python_opts(self): @unittest.skipIf(support.is_android, 'raising SIGSEGV on Android is unreliable') def test_worker_output_on_failure(self): - try: - from faulthandler import _sigsegv - except ImportError: - self.skipTest("need faulthandler._sigsegv") + # Skip test if faulthandler is missing + import_helper.import_module('faulthandler') code = textwrap.dedent(r""" import faulthandler @@ -2226,6 +2260,44 @@ def test_pass(self): self.check_executed_tests(output, testname, stats=1, parallel=True) self.assertNotIn('SPAM SPAM SPAM', output) + def test_xml(self): + code = textwrap.dedent(r""" + import unittest + from test import support + + class VerboseTests(unittest.TestCase): + def test_failed(self): + print("abc \x1b def") + self.fail() + """) + testname = self.create_test(code=code) + + # Run sequentially + filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, filename) + + output = self.run_tests(testname, "--junit-xml", filename, + exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=testname, + stats=TestStats(1, 1, 0)) + + # Test generated XML + with open(filename, encoding="utf8") as fp: + content = fp.read() + + testsuite = ElementTree.fromstring(content) + self.assertEqual(int(testsuite.get('tests')), 1) + self.assertEqual(int(testsuite.get('errors')), 0) + self.assertEqual(int(testsuite.get('failures')), 1) + + testcase = testsuite[0][0] + self.assertEqual(testcase.get('status'), 'run') + self.assertEqual(testcase.get('result'), 'completed') + self.assertGreater(float(testcase.get('time')), 0) + for out in testcase.iter('system-out'): + self.assertEqual(out.text, r"abc \x1b def") + class TestUtils(unittest.TestCase): def test_format_duration(self): @@ -2261,15 +2333,6 @@ def test_normalize_test_name(self): self.assertIsNone(normalize('setUpModule (test.test_x)', is_error=True)) self.assertIsNone(normalize('tearDownModule (test.test_module)', is_error=True)) - def test_get_signal_name(self): - for exitcode, expected in ( - (-int(signal.SIGINT), 'SIGINT'), - (-int(signal.SIGSEGV), 'SIGSEGV'), - (3221225477, "STATUS_ACCESS_VIOLATION"), - (0xC00000FD, "STATUS_STACK_OVERFLOW"), - ): - self.assertEqual(utils.get_signal_name(exitcode), expected, exitcode) - def test_format_resources(self): format_resources = utils.format_resources ALL_RESOURCES = utils.ALL_RESOURCES @@ -2298,6 +2361,10 @@ def __init__(self, test_id): def id(self): return self.test_id + # Restore patterns once the test completes + patterns = get_match_tests() + self.addCleanup(set_match_tests, patterns) + test_access = Test('test.test_os.FileTests.test_access') test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir') test_copy = Test('test.test_shutil.TestCopy.test_copy') @@ -2404,6 +2471,69 @@ def id(self): self.assertTrue(match_test(test_chdir)) self.assertFalse(match_test(test_copy)) + def test_sanitize_xml(self): + sanitize_xml = utils.sanitize_xml + + # escape invalid XML characters + self.assertEqual(sanitize_xml('abc \x1b\x1f def'), + r'abc \x1b\x1f def') + self.assertEqual(sanitize_xml('nul:\x00, bell:\x07'), + r'nul:\x00, bell:\x07') + self.assertEqual(sanitize_xml('surrogate:\uDC80'), + r'surrogate:\udc80') + self.assertEqual(sanitize_xml('illegal \uFFFE and \uFFFF'), + r'illegal \ufffe and \uffff') + + # no escape for valid XML characters + self.assertEqual(sanitize_xml('a\n\tb'), + 'a\n\tb') + self.assertEqual(sanitize_xml('valid t\xe9xt \u20ac'), + 'valid t\xe9xt \u20ac') + + +from test.libregrtest.results import TestResults + + +class TestColorized(unittest.TestCase): + def test_test_result_get_state(self): + # Arrange + green = _colorize.ANSIColors.GREEN + red = _colorize.ANSIColors.BOLD_RED + reset = _colorize.ANSIColors.RESET + yellow = _colorize.ANSIColors.YELLOW + + good_results = TestResults() + good_results.good = ["good1", "good2"] + bad_results = TestResults() + bad_results.bad = ["bad1", "bad2"] + no_results = TestResults() + no_results.bad = [] + interrupted_results = TestResults() + interrupted_results.interrupted = True + interrupted_worker_bug = TestResults() + interrupted_worker_bug.interrupted = True + interrupted_worker_bug.worker_bug = True + + for results, expected in ( + (good_results, f"{green}SUCCESS{reset}"), + (bad_results, f"{red}FAILURE{reset}"), + (no_results, f"{yellow}NO TESTS RAN{reset}"), + (interrupted_results, f"{yellow}INTERRUPTED{reset}"), + ( + interrupted_worker_bug, + f"{yellow}INTERRUPTED{reset}, {red}WORKER BUG{reset}", + ), + ): + with self.subTest(results=results, expected=expected): + # Act + with unittest.mock.patch( + "_colorize.can_colorize", return_value=True + ): + result = results.get_state(fail_env_changed=False) + + # Assert + self.assertEqual(result, expected) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index a28d1595f44533..356ff5b198d637 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -1,13 +1,26 @@ """Test the interactive interpreter.""" -import sys import os -import unittest +import select import subprocess +import sys +import unittest from textwrap import dedent from test import support -from test.support import cpython_only, has_subprocess_support, SuppressCrashReport +from test.support import ( + cpython_only, + has_subprocess_support, + os_helper, + SuppressCrashReport, + SHORT_TIMEOUT, +) from test.support.script_helper import kill_python +from test.support.import_helper import import_module + +try: + import pty +except ImportError: + pty = None if not has_subprocess_support: @@ -28,7 +41,7 @@ def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw): # path may be used by Py_GetPath() to build the default module search # path. stdin_fname = os.path.join(os.path.dirname(sys.executable), "") - cmd_line = [stdin_fname, '-E', '-i'] + cmd_line = [stdin_fname, '-I', '-i'] cmd_line.extend(args) # Set TERM=vt100, for the rationale see the comments in spawn_python() of @@ -57,6 +70,7 @@ def run_on_interactive_mode(source): return output +@support.force_not_colorized_test_class class TestInteractiveInterpreter(unittest.TestCase): @cpython_only @@ -64,6 +78,7 @@ class TestInteractiveInterpreter(unittest.TestCase): # _PyRefchain_Trace() on memory allocation error. @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') def test_no_memory(self): + import_module("_testcapi") # Issue #30696: Fix the interactive interpreter looping endlessly when # no memory. Check also that the fix does not break the interactive # loop when an exception is raised. @@ -173,6 +188,19 @@ def foo(x): ] self.assertEqual(traceback_lines, expected_lines) + def test_runsource_show_syntax_error_location(self): + user_input = dedent("""def f(x, x): ... + """) + p = spawn_repl() + p.stdin.write(user_input) + output = kill_python(p) + expected_lines = [ + ' def f(x, x): ...', + ' ^', + "SyntaxError: duplicate argument 'x' in function definition" + ] + self.assertEqual(output.splitlines()[4:-1], expected_lines) + def test_interactive_source_is_in_linecache(self): user_input = dedent(""" def foo(x): @@ -193,8 +221,61 @@ def bar(x): expected = "(30, None, [\'def foo(x):\\n\', \' return x + 1\\n\', \'\\n\'], \'\')" self.assertIn(expected, output, expected) + def test_asyncio_repl_reaches_python_startup_script(self): + with os_helper.temp_dir() as tmpdir: + script = os.path.join(tmpdir, "pythonstartup.py") + with open(script, "w") as f: + f.write("print('pythonstartup done!')" + os.linesep) + f.write("exit(0)" + os.linesep) + + env = os.environ.copy() + env["PYTHON_HISTORY"] = os.path.join(tmpdir, ".asyncio_history") + env["PYTHONSTARTUP"] = script + subprocess.check_call( + [sys.executable, "-m", "asyncio"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + timeout=SHORT_TIMEOUT, + ) + @unittest.skipUnless(pty, "requires pty") + def test_asyncio_repl_is_ok(self): + m, s = pty.openpty() + cmd = [sys.executable, "-I", "-m", "asyncio"] + env = os.environ.copy() + proc = subprocess.Popen( + cmd, + stdin=s, + stdout=s, + stderr=s, + text=True, + close_fds=True, + env=env, + ) + os.close(s) + os.write(m, b"await asyncio.sleep(0)\n") + os.write(m, b"exit()\n") + output = [] + while select.select([m], [], [], SHORT_TIMEOUT)[0]: + try: + data = os.read(m, 1024).decode("utf-8") + if not data: + break + except OSError: + break + output.append(data) + os.close(m) + try: + exit_code = proc.wait(timeout=SHORT_TIMEOUT) + except subprocess.TimeoutExpired: + proc.kill() + exit_code = proc.wait() + self.assertEqual(exit_code, 0, "".join(output)) + + +@support.force_not_colorized_test_class class TestInteractiveModeSyntaxErrors(unittest.TestCase): def test_interactive_syntax_error_correct_line(self): @@ -213,5 +294,42 @@ def f(): self.assertEqual(traceback_lines, expected_lines) +class TestAsyncioREPLContextVars(unittest.TestCase): + def test_toplevel_contextvars_sync(self): + user_input = dedent("""\ + from contextvars import ContextVar + var = ContextVar("var", default="failed") + var.set("ok") + """) + p = spawn_repl("-m", "asyncio") + p.stdin.write(user_input) + user_input2 = dedent(""" + print(f"toplevel contextvar test: {var.get()}") + """) + p.stdin.write(user_input2) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + expected = "toplevel contextvar test: ok" + self.assertIn(expected, output, expected) + + def test_toplevel_contextvars_async(self): + user_input = dedent("""\ + from contextvars import ContextVar + var = ContextVar('var', default='failed') + """) + p = spawn_repl("-m", "asyncio") + p.stdin.write(user_input) + user_input2 = "async def set_var(): var.set('ok')\n" + p.stdin.write(user_input2) + user_input3 = "await set_var()\n" + p.stdin.write(user_input3) + user_input4 = "print(f'toplevel contextvar test: {var.get()}')\n" + p.stdin.write(user_input4) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + expected = "toplevel contextvar test: ok" + self.assertIn(expected, output, expected) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_reprlib.py b/Lib/test/test_reprlib.py index 3e93b561c143d8..ffeb1fba7b80c6 100644 --- a/Lib/test/test_reprlib.py +++ b/Lib/test/test_reprlib.py @@ -580,6 +580,50 @@ def test_invalid_indent(self): with self.assertRaisesRegex(expected_error, expected_msg): r.repr(test_object) + def test_shadowed_stdlib_array(self): + # Issue #113570: repr() should not be fooled by an array + class array: + def __repr__(self): + return "not array.array" + + self.assertEqual(r(array()), "not array.array") + + def test_shadowed_builtin(self): + # Issue #113570: repr() should not be fooled + # by a shadowed builtin function + class list: + def __repr__(self): + return "not builtins.list" + + self.assertEqual(r(list()), "not builtins.list") + + def test_custom_repr(self): + class MyRepr(Repr): + + def repr_TextIOWrapper(self, obj, level): + if obj.name in {'', '', ''}: + return obj.name + return repr(obj) + + aRepr = MyRepr() + self.assertEqual(aRepr.repr(sys.stdin), "") + + def test_custom_repr_class_with_spaces(self): + class TypeWithSpaces: + pass + + t = TypeWithSpaces() + type(t).__name__ = "type with spaces" + self.assertEqual(type(t).__name__, "type with spaces") + + class MyRepr(Repr): + def repr_type_with_spaces(self, obj, level): + return "Type With Spaces" + + + aRepr = MyRepr() + self.assertEqual(aRepr.repr(t), "Type With Spaces") + def write_file(path, text): with open(path, 'w', encoding='ASCII') as fp: fp.write(text) diff --git a/Lib/test/test_resource.py b/Lib/test/test_resource.py index 317e7ca8f8c853..d23d3623235f38 100644 --- a/Lib/test/test_resource.py +++ b/Lib/test/test_resource.py @@ -138,7 +138,7 @@ def test_pagesize(self): self.assertIsInstance(pagesize, int) self.assertGreaterEqual(pagesize, 0) - @unittest.skipUnless(sys.platform == 'linux', 'test requires Linux') + @unittest.skipUnless(sys.platform in ('linux', 'android'), 'Linux only') def test_linux_constants(self): for attr in ['MSGQUEUE', 'NICE', 'RTPRIO', 'RTTIME', 'SIGPENDING']: with contextlib.suppress(AttributeError): diff --git a/Lib/test/test_richcmp.py b/Lib/test/test_richcmp.py index 5f449cdc05c6ba..b967c7623c57b0 100644 --- a/Lib/test/test_richcmp.py +++ b/Lib/test/test_richcmp.py @@ -28,9 +28,6 @@ def __gt__(self, other): def __ge__(self, other): return self.x >= other - def __cmp__(self, other): - raise support.TestFailed("Number.__cmp__() should not be called") - def __repr__(self): return "Number(%r)" % (self.x, ) @@ -53,9 +50,6 @@ def __setitem__(self, i, v): def __bool__(self): raise TypeError("Vectors cannot be used in Boolean contexts") - def __cmp__(self, other): - raise support.TestFailed("Vector.__cmp__() should not be called") - def __repr__(self): return "Vector(%r)" % (self.data, ) diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index 273ce2cf5c7dd2..1cff6a218f8d75 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -55,7 +55,7 @@ def test_attr_matches(self): if x.startswith('s')]) self.assertEqual(self.stdcompleter.attr_matches('tuple.foospamegg'), []) expected = sorted({'None.%s%s' % (x, - '()' if x == '__init_subclass__' + '()' if x in ('__init_subclass__', '__class__') else '' if x == '__doc__' else '(') for x in dir(None)}) diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index 9d76764c75be3e..ada78ec8e6b0c7 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -12,10 +12,16 @@ import textwrap import unittest import warnings -from test.support import (infinite_recursion, no_tracing, verbose, - requires_subprocess, requires_resource) +from test.support import ( + force_not_colorized_test_class, + infinite_recursion, + no_tracing, + requires_resource, + requires_subprocess, + verbose, +) from test.support.import_helper import forget, make_legacy_pyc, unload -from test.support.os_helper import create_empty_file, temp_dir +from test.support.os_helper import create_empty_file, temp_dir, FakePath from test.support.script_helper import make_script, make_zip_script @@ -657,14 +663,13 @@ def test_basic_script(self): self._check_script(script_name, "", script_name, script_name, expect_spec=False) - def test_basic_script_with_path_object(self): + def test_basic_script_with_pathlike_object(self): with temp_dir() as script_dir: mod_name = 'script' - script_name = pathlib.Path(self._make_test_script(script_dir, - mod_name)) - self._check_script(script_name, "", - os.fsdecode(script_name), - os.fsdecode(script_name), + script_name = self._make_test_script(script_dir, mod_name) + self._check_script(FakePath(script_name), "", + script_name, + script_name, expect_spec=False) def test_basic_script_no_suffix(self): @@ -759,6 +764,7 @@ def test_encoding(self): self.assertEqual(result['s'], "non-ASCII: h\xe9") +@force_not_colorized_test_class class TestExit(unittest.TestCase): STATUS_CONTROL_C_EXIT = 0xC000013A EXPECTED_CODE = ( diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py index 97e96668f85c8a..0d0f86c145b499 100644 --- a/Lib/test/test_sax.py +++ b/Lib/test/test_sax.py @@ -16,6 +16,7 @@ from xml.sax.handler import (feature_namespaces, feature_external_ges, LexicalHandler) from xml.sax.xmlreader import InputSource, AttributesImpl, AttributesNSImpl +from xml import sax from io import BytesIO, StringIO import codecs import os.path @@ -25,7 +26,7 @@ from urllib.error import URLError import urllib.request from test.support import os_helper -from test.support import findfile +from test.support import findfile, check__all__ from test.support.os_helper import FakePath, TESTFN @@ -1215,10 +1216,10 @@ def test_expat_incremental_reset(self): self.assertEqual(result.getvalue(), start + b"text") + @unittest.skipIf(pyexpat.version_info < (2, 6, 0), + f'Expat {pyexpat.version_info} does not ' + 'support reparse deferral') def test_flush_reparse_deferral_enabled(self): - if pyexpat.version_info < (2, 6, 0): - self.skipTest(f'Expat {pyexpat.version_info} does not support reparse deferral') - result = BytesIO() xmlgen = XMLGenerator(result) parser = create_parser() @@ -1251,8 +1252,8 @@ def test_flush_reparse_deferral_disabled(self): if pyexpat.version_info >= (2, 6, 0): parser._parser.SetReparseDeferralEnabled(False) + self.assertEqual(result.getvalue(), start) # i.e. no elements started - self.assertEqual(result.getvalue(), start) # i.e. no elements started self.assertFalse(parser._parser.GetReparseDeferralEnabled()) parser.flush() @@ -1557,5 +1558,20 @@ def characters(self, content): self.assertEqual(self.char_index, 2) +class TestModuleAll(unittest.TestCase): + def test_all(self): + extra = ( + 'ContentHandler', + 'ErrorHandler', + 'InputSource', + 'SAXException', + 'SAXNotRecognizedException', + 'SAXNotSupportedException', + 'SAXParseException', + 'SAXReaderNotAvailable', + ) + check__all__(self, sax, extra=extra) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py index 6e46dfa96a664f..24a366efc6ca05 100644 --- a/Lib/test/test_scope.py +++ b/Lib/test/test_scope.py @@ -810,6 +810,30 @@ def dig(self): gc_collect() # For PyPy or other GCs. self.assertIsNone(ref()) + def test_multiple_nesting(self): + # Regression test for https://github.com/python/cpython/issues/121863 + class MultiplyNested: + def f1(self): + __arg = 1 + class D: + def g(self, __arg): + return __arg + return D().g(_MultiplyNested__arg=2) + + def f2(self): + __arg = 1 + class D: + def g(self, __arg): + return __arg + return D().g + + inst = MultiplyNested() + with self.assertRaises(TypeError): + inst.f1() + + closure = inst.f2() + with self.assertRaises(TypeError): + closure(_MultiplyNested__arg=2) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index d9102eb98a54a6..a8531d466e56e7 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -635,6 +635,16 @@ def __le__(self, some_set): myset >= myobj self.assertTrue(myobj.le_called) + def test_set_membership(self): + myfrozenset = frozenset(range(3)) + myset = {myfrozenset, "abc", 1} + self.assertIn(set(range(3)), myset) + self.assertNotIn(set(range(1)), myset) + myset.discard(set(range(3))) + self.assertEqual(myset, {"abc", 1}) + self.assertRaises(KeyError, myset.remove, set(range(1))) + self.assertRaises(KeyError, myset.remove, set(range(3))) + class SetSubclass(set): pass diff --git a/Lib/test/test_setcomps.py b/Lib/test/test_setcomps.py index 976fa885bd8ef9..0bb02ef11f6b4b 100644 --- a/Lib/test/test_setcomps.py +++ b/Lib/test/test_setcomps.py @@ -1,6 +1,9 @@ import doctest +import traceback import unittest +from test.support import BrokenIter + doctests = """ ########### Tests mostly copied from test_listcomps.py ############ @@ -148,6 +151,42 @@ """ +class SetComprehensionTest(unittest.TestCase): + def test_exception_locations(self): + # The location of an exception raised from __init__ or + # __next__ should should be the iterator expression + + def init_raises(): + try: + {x for x in BrokenIter(init_raises=True)} + except Exception as e: + return e + + def next_raises(): + try: + {x for x in BrokenIter(next_raises=True)} + except Exception as e: + return e + + def iter_raises(): + try: + {x for x in BrokenIter(iter_raises=True)} + except Exception as e: + return e + + for func, expected in [(init_raises, "BrokenIter(init_raises=True)"), + (next_raises, "BrokenIter(next_raises=True)"), + (iter_raises, "BrokenIter(iter_raises=True)"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) __test__ = {'doctests' : doctests} diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index d96dad4eb9475d..1f18b1f09b5858 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -70,18 +70,17 @@ def wrap(*args, **kwargs): os.rename = builtin_rename return wrap -def write_file(path, content, binary=False): +def create_file(path, content=b''): """Write *content* to a file located at *path*. If *path* is a tuple instead of a string, os.path.join will be used to - make a path. If *binary* is true, the file will be opened in binary - mode. + make a path. """ if isinstance(path, tuple): path = os.path.join(*path) - mode = 'wb' if binary else 'w' - encoding = None if binary else "utf-8" - with open(path, mode, encoding=encoding) as fp: + if isinstance(content, str): + content = content.encode() + with open(path, 'xb') as fp: fp.write(content) def write_test_file(path, size): @@ -190,7 +189,7 @@ def test_rmtree_works_on_bytes(self): tmp = self.mkdtemp() victim = os.path.join(tmp, 'killme') os.mkdir(victim) - write_file(os.path.join(victim, 'somefile'), 'foo') + create_file(os.path.join(victim, 'somefile'), 'foo') victim = os.fsencode(victim) self.assertIsInstance(victim, bytes) shutil.rmtree(victim) @@ -242,7 +241,7 @@ def test_rmtree_works_on_symlinks(self): for d in dir1, dir2, dir3: os.mkdir(d) file1 = os.path.join(tmp, 'file1') - write_file(file1, 'foo') + create_file(file1, 'foo') link1 = os.path.join(dir1, 'link1') os.symlink(dir2, link1) link2 = os.path.join(dir1, 'link2') @@ -304,7 +303,7 @@ def test_rmtree_works_on_junctions(self): for d in dir1, dir2, dir3: os.mkdir(d) file1 = os.path.join(tmp, 'file1') - write_file(file1, 'foo') + create_file(file1, 'foo') link1 = os.path.join(dir1, 'link1') _winapi.CreateJunction(dir2, link1) link2 = os.path.join(dir1, 'link2') @@ -327,7 +326,7 @@ def test_rmtree_errors(self): # existing file tmpdir = self.mkdtemp() filename = os.path.join(tmpdir, "tstfile") - write_file(filename, "") + create_file(filename) with self.assertRaises(NotADirectoryError) as cm: shutil.rmtree(filename) self.assertEqual(cm.exception.filename, filename) @@ -347,7 +346,7 @@ def test_rmtree_errors(self): def test_rmtree_errors_onerror(self): tmpdir = self.mkdtemp() filename = os.path.join(tmpdir, "tstfile") - write_file(filename, "") + create_file(filename) errors = [] def onerror(*args): errors.append(args) @@ -365,7 +364,7 @@ def onerror(*args): def test_rmtree_errors_onexc(self): tmpdir = self.mkdtemp() filename = os.path.join(tmpdir, "tstfile") - write_file(filename, "") + create_file(filename) errors = [] def onexc(*args): errors.append(args) @@ -547,7 +546,7 @@ def raiser(fn, *args, **kwargs): os.lstat = raiser os.mkdir(TESTFN) - write_file((TESTFN, 'foo'), 'foo') + create_file((TESTFN, 'foo'), 'foo') shutil.rmtree(TESTFN) finally: os.lstat = orig_lstat @@ -558,25 +557,23 @@ def test_rmtree_uses_safe_fd_version_if_available(self): os.listdir in os.supports_fd and os.stat in os.supports_follow_symlinks) if _use_fd_functions: - self.assertTrue(shutil._use_fd_functions) self.assertTrue(shutil.rmtree.avoids_symlink_attacks) tmp_dir = self.mkdtemp() d = os.path.join(tmp_dir, 'a') os.mkdir(d) try: - real_rmtree = shutil._rmtree_safe_fd + real_open = os.open class Called(Exception): pass def _raiser(*args, **kwargs): raise Called - shutil._rmtree_safe_fd = _raiser + os.open = _raiser self.assertRaises(Called, shutil.rmtree, d) finally: - shutil._rmtree_safe_fd = real_rmtree + os.open = real_open else: - self.assertFalse(shutil._use_fd_functions) self.assertFalse(shutil.rmtree.avoids_symlink_attacks) - @unittest.skipUnless(shutil._use_fd_functions, "requires safe rmtree") + @unittest.skipUnless(shutil.rmtree.avoids_symlink_attacks, "requires safe rmtree") def test_rmtree_fails_on_close(self): # Test that the error handler is called for failed os.close() and that # os.close() is only called once for a file descriptor. @@ -611,7 +608,7 @@ def onexc(*args): self.assertEqual(errors[1][1], dir1) self.assertEqual(close_count, 2) - @unittest.skipUnless(shutil._use_fd_functions, "dir_fd is not supported") + @unittest.skipUnless(shutil.rmtree.avoids_symlink_attacks, "dir_fd is not supported") def test_rmtree_with_dir_fd(self): tmp_dir = self.mkdtemp() victim = 'killme' @@ -620,12 +617,12 @@ def test_rmtree_with_dir_fd(self): self.addCleanup(os.close, dir_fd) os.mkdir(fullname) os.mkdir(os.path.join(fullname, 'subdir')) - write_file(os.path.join(fullname, 'subdir', 'somefile'), 'foo') + create_file(os.path.join(fullname, 'subdir', 'somefile'), 'foo') self.assertTrue(os.path.exists(fullname)) shutil.rmtree(victim, dir_fd=dir_fd) self.assertFalse(os.path.exists(fullname)) - @unittest.skipIf(shutil._use_fd_functions, "dir_fd is supported") + @unittest.skipIf(shutil.rmtree.avoids_symlink_attacks, "dir_fd is supported") def test_rmtree_with_dir_fd_unsupported(self): tmp_dir = self.mkdtemp() with self.assertRaises(NotImplementedError): @@ -660,13 +657,30 @@ def test_rmtree_on_junction(self): src = os.path.join(TESTFN, 'cheese') dst = os.path.join(TESTFN, 'shop') os.mkdir(src) - open(os.path.join(src, 'spam'), 'wb').close() + create_file(os.path.join(src, 'spam')) _winapi.CreateJunction(src, dst) self.assertRaises(OSError, shutil.rmtree, dst) shutil.rmtree(dst, ignore_errors=True) finally: shutil.rmtree(TESTFN, ignore_errors=True) + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_rmtree_on_named_pipe(self): + os.mkfifo(TESTFN) + try: + with self.assertRaises(NotADirectoryError): + shutil.rmtree(TESTFN) + self.assertTrue(os.path.exists(TESTFN)) + finally: + os.unlink(TESTFN) + + os.mkdir(TESTFN) + os.mkfifo(os.path.join(TESTFN, 'mypipe')) + shutil.rmtree(TESTFN) + self.assertFalse(os.path.exists(TESTFN)) + @unittest.skipIf(sys.platform[:6] == 'cygwin', "This test can't be run on Cygwin (issue #1071513).") @os_helper.skip_if_dac_override @@ -703,7 +717,7 @@ def _onexc(fn, path, exc): for path in dirs: os.mkdir(path) for path in files: - write_file(path, '') + create_file(path) old_modes = [os.stat(path).st_mode for path in paths] @@ -724,6 +738,16 @@ def _onexc(fn, path, exc): shutil.rmtree(TESTFN) raise + def test_rmtree_above_recursion_limit(self): + recursion_limit = 40 + # directory_depth > recursion_limit + directory_depth = recursion_limit + 10 + base = os.path.join(TESTFN, *(['d'] * directory_depth)) + os.makedirs(base) + + with support.infinite_recursion(recursion_limit): + shutil.rmtree(TESTFN) + class TestCopyTree(BaseTest, unittest.TestCase): @@ -732,9 +756,9 @@ def test_copytree_simple(self): dst_dir = os.path.join(self.mkdtemp(), 'destination') self.addCleanup(shutil.rmtree, src_dir) self.addCleanup(shutil.rmtree, os.path.dirname(dst_dir)) - write_file((src_dir, 'test.txt'), '123') + create_file((src_dir, 'test.txt'), '123') os.mkdir(os.path.join(src_dir, 'test_dir')) - write_file((src_dir, 'test_dir', 'test.txt'), '456') + create_file((src_dir, 'test_dir', 'test.txt'), '456') shutil.copytree(src_dir, dst_dir) self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test.txt'))) @@ -752,11 +776,11 @@ def test_copytree_dirs_exist_ok(self): self.addCleanup(shutil.rmtree, src_dir) self.addCleanup(shutil.rmtree, dst_dir) - write_file((src_dir, 'nonexisting.txt'), '123') + create_file((src_dir, 'nonexisting.txt'), '123') os.mkdir(os.path.join(src_dir, 'existing_dir')) os.mkdir(os.path.join(dst_dir, 'existing_dir')) - write_file((dst_dir, 'existing_dir', 'existing.txt'), 'will be replaced') - write_file((src_dir, 'existing_dir', 'existing.txt'), 'has been replaced') + create_file((dst_dir, 'existing_dir', 'existing.txt'), 'will be replaced') + create_file((src_dir, 'existing_dir', 'existing.txt'), 'has been replaced') shutil.copytree(src_dir, dst_dir, dirs_exist_ok=True) self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'nonexisting.txt'))) @@ -779,7 +803,7 @@ def test_copytree_symlinks(self): sub_dir = os.path.join(src_dir, 'sub') os.mkdir(src_dir) os.mkdir(sub_dir) - write_file((src_dir, 'file.txt'), 'foo') + create_file((src_dir, 'file.txt'), 'foo') src_link = os.path.join(sub_dir, 'link') dst_link = os.path.join(dst_dir, 'sub/link') os.symlink(os.path.join(src_dir, 'file.txt'), @@ -810,16 +834,16 @@ def test_copytree_with_exclude(self): src_dir = self.mkdtemp() try: dst_dir = join(self.mkdtemp(), 'destination') - write_file((src_dir, 'test.txt'), '123') - write_file((src_dir, 'test.tmp'), '123') + create_file((src_dir, 'test.txt'), '123') + create_file((src_dir, 'test.tmp'), '123') os.mkdir(join(src_dir, 'test_dir')) - write_file((src_dir, 'test_dir', 'test.txt'), '456') + create_file((src_dir, 'test_dir', 'test.txt'), '456') os.mkdir(join(src_dir, 'test_dir2')) - write_file((src_dir, 'test_dir2', 'test.txt'), '456') + create_file((src_dir, 'test_dir2', 'test.txt'), '456') os.mkdir(join(src_dir, 'test_dir2', 'subdir')) os.mkdir(join(src_dir, 'test_dir2', 'subdir2')) - write_file((src_dir, 'test_dir2', 'subdir', 'test.txt'), '456') - write_file((src_dir, 'test_dir2', 'subdir2', 'test.py'), '456') + create_file((src_dir, 'test_dir2', 'subdir', 'test.txt'), '456') + create_file((src_dir, 'test_dir2', 'subdir2', 'test.py'), '456') # testing glob-like patterns try: @@ -878,12 +902,12 @@ def test_copytree_arg_types_of_ignore(self): os.mkdir(join(src_dir)) os.mkdir(join(src_dir, 'test_dir')) os.mkdir(os.path.join(src_dir, 'test_dir', 'subdir')) - write_file((src_dir, 'test_dir', 'subdir', 'test.txt'), '456') + create_file((src_dir, 'test_dir', 'subdir', 'test.txt'), '456') - invokations = [] + invocations = [] def _ignore(src, names): - invokations.append(src) + invocations.append(src) self.assertIsInstance(src, str) self.assertIsInstance(names, list) self.assertEqual(len(names), len(set(names))) @@ -897,7 +921,7 @@ def _ignore(src, names): 'test.txt'))) dst_dir = join(self.mkdtemp(), 'destination') - shutil.copytree(pathlib.Path(src_dir), dst_dir, ignore=_ignore) + shutil.copytree(FakePath(src_dir), dst_dir, ignore=_ignore) self.assertTrue(exists(join(dst_dir, 'test_dir', 'subdir', 'test.txt'))) @@ -908,7 +932,7 @@ def _ignore(src, names): self.assertTrue(exists(join(dst_dir, 'test_dir', 'subdir', 'test.txt'))) - self.assertEqual(len(invokations), 9) + self.assertEqual(len(invocations), 9) def test_copytree_retains_permissions(self): tmp_dir = self.mkdtemp() @@ -918,9 +942,9 @@ def test_copytree_retains_permissions(self): self.addCleanup(shutil.rmtree, tmp_dir) os.chmod(src_dir, 0o777) - write_file((src_dir, 'permissive.txt'), '123') + create_file((src_dir, 'permissive.txt'), '123') os.chmod(os.path.join(src_dir, 'permissive.txt'), 0o777) - write_file((src_dir, 'restrictive.txt'), '456') + create_file((src_dir, 'restrictive.txt'), '456') os.chmod(os.path.join(src_dir, 'restrictive.txt'), 0o600) restrictive_subdir = tempfile.mkdtemp(dir=src_dir) self.addCleanup(os_helper.rmtree, restrictive_subdir) @@ -963,8 +987,7 @@ def custom_cpfun(a, b): flag = [] src = self.mkdtemp() dst = tempfile.mktemp(dir=self.mkdtemp()) - with open(os.path.join(src, 'foo'), 'w', encoding='utf-8') as f: - f.close() + create_file(os.path.join(src, 'foo')) shutil.copytree(src, dst, copy_function=custom_cpfun) self.assertEqual(len(flag), 1) @@ -999,9 +1022,9 @@ def test_copytree_named_pipe(self): def test_copytree_special_func(self): src_dir = self.mkdtemp() dst_dir = os.path.join(self.mkdtemp(), 'destination') - write_file((src_dir, 'test.txt'), '123') + create_file((src_dir, 'test.txt'), '123') os.mkdir(os.path.join(src_dir, 'test_dir')) - write_file((src_dir, 'test_dir', 'test.txt'), '456') + create_file((src_dir, 'test_dir', 'test.txt'), '456') copied = [] def _copy(src, dst): @@ -1014,7 +1037,7 @@ def _copy(src, dst): def test_copytree_dangling_symlinks(self): src_dir = self.mkdtemp() valid_file = os.path.join(src_dir, 'test.txt') - write_file(valid_file, 'abc') + create_file(valid_file, 'abc') dir_a = os.path.join(src_dir, 'dir_a') os.mkdir(dir_a) for d in src_dir, dir_a: @@ -1042,8 +1065,7 @@ def test_copytree_symlink_dir(self): src_dir = self.mkdtemp() dst_dir = os.path.join(self.mkdtemp(), 'destination') os.mkdir(os.path.join(src_dir, 'real_dir')) - with open(os.path.join(src_dir, 'real_dir', 'test.txt'), 'wb'): - pass + create_file(os.path.join(src_dir, 'real_dir', 'test.txt')) os.symlink(os.path.join(src_dir, 'real_dir'), os.path.join(src_dir, 'link_to_dir'), target_is_directory=True) @@ -1063,7 +1085,7 @@ def test_copytree_return_value(self): dst_dir = src_dir + "dest" self.addCleanup(shutil.rmtree, dst_dir, True) src = os.path.join(src_dir, 'foo') - write_file(src, 'foo') + create_file(src, 'foo') rv = shutil.copytree(src_dir, dst_dir) self.assertEqual(['foo'], os.listdir(rv)) @@ -1075,7 +1097,7 @@ def test_copytree_subdirectory(self): dst_dir = os.path.join(src_dir, "somevendor", "1.0") os.makedirs(src_dir) src = os.path.join(src_dir, 'pol') - write_file(src, 'pol') + create_file(src, 'pol') rv = shutil.copytree(src_dir, dst_dir) self.assertEqual(['pol'], os.listdir(rv)) @@ -1090,8 +1112,8 @@ def test_copymode_follow_symlinks(self): dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') dst_link = os.path.join(tmp_dir, 'quux') - write_file(src, 'foo') - write_file(dst, 'foo') + create_file(src, 'foo') + create_file(dst, 'foo') os.symlink(src, src_link) os.symlink(dst, dst_link) os.chmod(src, stat.S_IRWXU|stat.S_IRWXG) @@ -1122,8 +1144,8 @@ def test_copymode_symlink_to_symlink(self): dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') dst_link = os.path.join(tmp_dir, 'quux') - write_file(src, 'foo') - write_file(dst, 'foo') + create_file(src, 'foo') + create_file(dst, 'foo') os.symlink(src, src_link) os.symlink(dst, dst_link) os.chmod(src, stat.S_IRWXU|stat.S_IRWXG) @@ -1153,8 +1175,8 @@ def test_copymode_symlink_to_symlink_wo_lchmod(self): dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') dst_link = os.path.join(tmp_dir, 'quux') - write_file(src, 'foo') - write_file(dst, 'foo') + create_file(src, 'foo') + create_file(dst, 'foo') os.symlink(src, src_link) os.symlink(dst, dst_link) shutil.copymode(src_link, dst_link, follow_symlinks=False) # silent fail @@ -1168,11 +1190,11 @@ def test_copystat_symlinks(self): dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') dst_link = os.path.join(tmp_dir, 'qux') - write_file(src, 'foo') + create_file(src, 'foo') src_stat = os.stat(src) os.utime(src, (src_stat.st_atime, src_stat.st_mtime - 42.0)) # ensure different mtimes - write_file(dst, 'bar') + create_file(dst, 'bar') self.assertNotEqual(os.stat(src).st_mtime, os.stat(dst).st_mtime) os.symlink(src, src_link) os.symlink(dst, dst_link) @@ -1210,8 +1232,8 @@ def test_copystat_handles_harmless_chflags_errors(self): tmpdir = self.mkdtemp() file1 = os.path.join(tmpdir, 'file1') file2 = os.path.join(tmpdir, 'file2') - write_file(file1, 'xxx') - write_file(file2, 'xxx') + create_file(file1, 'xxx') + create_file(file2, 'xxx') def make_chflags_raiser(err): ex = OSError() @@ -1237,9 +1259,9 @@ def _chflags_raiser(path, flags, *, follow_symlinks=True): def test_copyxattr(self): tmp_dir = self.mkdtemp() src = os.path.join(tmp_dir, 'foo') - write_file(src, 'foo') + create_file(src, 'foo') dst = os.path.join(tmp_dir, 'bar') - write_file(dst, 'bar') + create_file(dst, 'bar') # no xattr == no problem shutil._copyxattr(src, dst) @@ -1253,7 +1275,7 @@ def test_copyxattr(self): os.getxattr(dst, 'user.foo')) # check errors don't affect other attrs os.remove(dst) - write_file(dst, 'bar') + create_file(dst, 'bar') os_error = OSError(errno.EPERM, 'EPERM') def _raise_on_user_foo(fname, attr, val, **kwargs): @@ -1283,15 +1305,15 @@ def _raise_on_src(fname, *, follow_symlinks=True): # test that shutil.copystat copies xattrs src = os.path.join(tmp_dir, 'the_original') srcro = os.path.join(tmp_dir, 'the_original_ro') - write_file(src, src) - write_file(srcro, srcro) + create_file(src, src) + create_file(srcro, srcro) os.setxattr(src, 'user.the_value', b'fiddly') os.setxattr(srcro, 'user.the_value', b'fiddly') os.chmod(srcro, 0o444) dst = os.path.join(tmp_dir, 'the_copy') dstro = os.path.join(tmp_dir, 'the_copy_ro') - write_file(dst, dst) - write_file(dstro, dstro) + create_file(dst, dst) + create_file(dstro, dstro) shutil.copystat(src, dst) shutil.copystat(srcro, dstro) self.assertEqual(os.getxattr(dst, 'user.the_value'), b'fiddly') @@ -1307,13 +1329,13 @@ def test_copyxattr_symlinks(self): tmp_dir = self.mkdtemp() src = os.path.join(tmp_dir, 'foo') src_link = os.path.join(tmp_dir, 'baz') - write_file(src, 'foo') + create_file(src, 'foo') os.symlink(src, src_link) os.setxattr(src, 'trusted.foo', b'42') os.setxattr(src_link, 'trusted.foo', b'43', follow_symlinks=False) dst = os.path.join(tmp_dir, 'bar') dst_link = os.path.join(tmp_dir, 'qux') - write_file(dst, 'bar') + create_file(dst, 'bar') os.symlink(dst, dst_link) shutil._copyxattr(src_link, dst_link, follow_symlinks=False) self.assertEqual(os.getxattr(dst_link, 'trusted.foo', follow_symlinks=False), b'43') @@ -1326,7 +1348,7 @@ def test_copyxattr_symlinks(self): def _copy_file(self, method): fname = 'test.txt' tmpdir = self.mkdtemp() - write_file((tmpdir, fname), 'xxx') + create_file((tmpdir, fname), 'xxx') file1 = os.path.join(tmpdir, fname) tmpdir2 = self.mkdtemp() method(file1, tmpdir2) @@ -1345,7 +1367,7 @@ def test_copy_symlinks(self): src = os.path.join(tmp_dir, 'foo') dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') - write_file(src, 'foo') + create_file(src, 'foo') os.symlink(src, src_link) if hasattr(os, 'lchmod'): os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO) @@ -1387,7 +1409,7 @@ def test_copy2_symlinks(self): src = os.path.join(tmp_dir, 'foo') dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') - write_file(src, 'foo') + create_file(src, 'foo') os.symlink(src, src_link) if hasattr(os, 'lchmod'): os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO) @@ -1421,7 +1443,7 @@ def test_copy2_xattr(self): tmp_dir = self.mkdtemp() src = os.path.join(tmp_dir, 'foo') dst = os.path.join(tmp_dir, 'bar') - write_file(src, 'foo') + create_file(src, 'foo') os.setxattr(src, 'user.foo', b'42') shutil.copy2(src, dst) self.assertEqual( @@ -1435,7 +1457,7 @@ def test_copy_return_value(self): src_dir = self.mkdtemp() dst_dir = self.mkdtemp() src = os.path.join(src_dir, 'foo') - write_file(src, 'foo') + create_file(src, 'foo') rv = fn(src, dst_dir) self.assertEqual(rv, os.path.join(dst_dir, 'foo')) rv = fn(src, os.path.join(dst_dir, 'bar')) @@ -1452,7 +1474,7 @@ def _test_copy_dir(self, copy_func): src_file = os.path.join(src_dir, 'foo') dir2 = self.mkdtemp() dst = os.path.join(src_dir, 'does_not_exist/') - write_file(src_file, 'foo') + create_file(src_file, 'foo') if sys.platform == "win32": err = PermissionError else: @@ -1472,7 +1494,7 @@ def test_copyfile_symlinks(self): dst = os.path.join(tmp_dir, 'dst') dst_link = os.path.join(tmp_dir, 'dst_link') link = os.path.join(tmp_dir, 'link') - write_file(src, 'foo') + create_file(src, 'foo') os.symlink(src, link) # don't follow shutil.copyfile(link, dst_link, follow_symlinks=False) @@ -1489,8 +1511,7 @@ def test_dont_copy_file_onto_link_to_itself(self): src = os.path.join(TESTFN, 'cheese') dst = os.path.join(TESTFN, 'shop') try: - with open(src, 'w', encoding='utf-8') as f: - f.write('cheddar') + create_file(src, 'cheddar') try: os.link(src, dst) except PermissionError as e: @@ -1509,8 +1530,7 @@ def test_dont_copy_file_onto_symlink_to_itself(self): src = os.path.join(TESTFN, 'cheese') dst = os.path.join(TESTFN, 'shop') try: - with open(src, 'w', encoding='utf-8') as f: - f.write('cheddar') + create_file(src, 'cheddar') # Using `src` here would mean we end up with a symlink pointing # to TESTFN/TESTFN/cheese, while it should point at # TESTFN/cheese. @@ -1545,7 +1565,7 @@ def test_copyfile_return_value(self): dst_dir = self.mkdtemp() dst_file = os.path.join(dst_dir, 'bar') src_file = os.path.join(src_dir, 'foo') - write_file(src_file, 'foo') + create_file(src_file, 'foo') rv = shutil.copyfile(src_file, dst_file) self.assertTrue(os.path.exists(rv)) self.assertEqual(read_file(src_file), read_file(dst_file)) @@ -1555,7 +1575,7 @@ def test_copyfile_same_file(self): # are the same. src_dir = self.mkdtemp() src_file = os.path.join(src_dir, 'foo') - write_file(src_file, 'foo') + create_file(src_file, 'foo') self.assertRaises(SameFileError, shutil.copyfile, src_file, src_file) # But Error should work too, to stay backward compatible. self.assertRaises(Error, shutil.copyfile, src_file, src_file) @@ -1572,7 +1592,7 @@ def test_copyfile_nonexistent_dir(self): src_dir = self.mkdtemp() src_file = os.path.join(src_dir, 'foo') dst = os.path.join(src_dir, 'does_not_exist/') - write_file(src_file, 'foo') + create_file(src_file, 'foo') self.assertRaises(FileNotFoundError, shutil.copyfile, src_file, dst) def test_copyfile_copy_dir(self): @@ -1583,7 +1603,7 @@ def test_copyfile_copy_dir(self): src_file = os.path.join(src_dir, 'foo') dir2 = self.mkdtemp() dst = os.path.join(src_dir, 'does_not_exist/') - write_file(src_file, 'foo') + create_file(src_file, 'foo') if sys.platform == "win32": err = PermissionError else: @@ -1598,42 +1618,6 @@ class TestArchives(BaseTest, unittest.TestCase): ### shutil.make_archive - @support.requires_zlib() - def test_make_tarball(self): - # creating something to tar - root_dir, base_dir = self._create_files('') - - tmpdir2 = self.mkdtemp() - # force shutil to create the directory - os.rmdir(tmpdir2) - # working with relative paths - work_dir = os.path.dirname(tmpdir2) - rel_base_name = os.path.join(os.path.basename(tmpdir2), 'archive') - - with os_helper.change_cwd(work_dir), no_chdir: - base_name = os.path.abspath(rel_base_name) - tarball = make_archive(rel_base_name, 'gztar', root_dir, '.') - - # check if the compressed tarball was created - self.assertEqual(tarball, base_name + '.tar.gz') - self.assertTrue(os.path.isfile(tarball)) - self.assertTrue(tarfile.is_tarfile(tarball)) - with tarfile.open(tarball, 'r:gz') as tf: - self.assertCountEqual(tf.getnames(), - ['.', './sub', './sub2', - './file1', './file2', './sub/file3']) - - # trying an uncompressed one - with os_helper.change_cwd(work_dir), no_chdir: - tarball = make_archive(rel_base_name, 'tar', root_dir, '.') - self.assertEqual(tarball, base_name + '.tar') - self.assertTrue(os.path.isfile(tarball)) - self.assertTrue(tarfile.is_tarfile(tarball)) - with tarfile.open(tarball, 'r') as tf: - self.assertCountEqual(tf.getnames(), - ['.', './sub', './sub2', - './file1', './file2', './sub/file3']) - def _tarinfo(self, path): with tarfile.open(path) as tar: names = tar.getnames() @@ -1645,15 +1629,101 @@ def _create_files(self, base_dir='dist'): root_dir = self.mkdtemp() dist = os.path.join(root_dir, base_dir) os.makedirs(dist, exist_ok=True) - write_file((dist, 'file1'), 'xxx') - write_file((dist, 'file2'), 'xxx') + create_file((dist, 'file1'), 'xxx') + create_file((dist, 'file2'), 'xxx') os.mkdir(os.path.join(dist, 'sub')) - write_file((dist, 'sub', 'file3'), 'xxx') + create_file((dist, 'sub', 'file3'), 'xxx') os.mkdir(os.path.join(dist, 'sub2')) if base_dir: - write_file((root_dir, 'outer'), 'xxx') + create_file((root_dir, 'outer'), 'xxx') return root_dir, base_dir + @support.requires_zlib() + def test_make_tarfile(self): + root_dir, base_dir = self._create_files() + # Test without base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'tar', root_dir) + # check if the compressed tarball was created + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['.', './dist', './dist/sub', './dist/sub2', + './dist/file1', './dist/file2', './dist/sub/file3', + './outer']) + + # Test with base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst2', 'archive') + archive = make_archive(base_name, 'tar', root_dir, base_dir) + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + # check if the uncompressed tarball was created + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['dist', 'dist/sub', 'dist/sub2', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + # Test with multi-component base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst3', 'archive') + archive = make_archive(base_name, 'tar', root_dir, + os.path.join(base_dir, 'sub')) + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['dist/sub', 'dist/sub/file3']) + + @support.requires_zlib() + def test_make_tarfile_without_rootdir(self): + root_dir, base_dir = self._create_files() + # Test without base_dir. + base_name = os.path.join(self.mkdtemp(), 'dst', 'archive') + base_name = os.path.relpath(base_name, root_dir) + with os_helper.change_cwd(root_dir), no_chdir: + archive = make_archive(base_name, 'gztar') + self.assertEqual(archive, base_name + '.tar.gz') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r:gz') as tf: + self.assertCountEqual(tf.getnames(), + ['.', './dist', './dist/sub', './dist/sub2', + './dist/file1', './dist/file2', './dist/sub/file3', + './outer']) + + # Test with base_dir. + with os_helper.change_cwd(root_dir), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'tar', base_dir=base_dir) + self.assertEqual(archive, base_name + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['dist', 'dist/sub', 'dist/sub2', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + def test_make_tarfile_with_explicit_curdir(self): + # Test with base_dir=os.curdir. + root_dir, base_dir = self._create_files() + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'tar', root_dir, os.curdir) + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['.', './dist', './dist/sub', './dist/sub2', + './dist/file1', './dist/file2', './dist/sub/file3', + './outer']) + @support.requires_zlib() @unittest.skipUnless(shutil.which('tar'), 'Need the tar command to run') @@ -1703,40 +1773,89 @@ def test_tarfile_vs_tar(self): @support.requires_zlib() def test_make_zipfile(self): - # creating something to zip root_dir, base_dir = self._create_files() + # Test without base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'zip', root_dir) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3', + 'outer']) + + # Test with base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst2', 'archive') + archive = make_archive(base_name, 'zip', root_dir, base_dir) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + # Test with multi-component base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst3', 'archive') + archive = make_archive(base_name, 'zip', root_dir, + os.path.join(base_dir, 'sub')) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/sub/', 'dist/sub/file3']) - tmpdir2 = self.mkdtemp() - # force shutil to create the directory - os.rmdir(tmpdir2) - # working with relative paths - work_dir = os.path.dirname(tmpdir2) - rel_base_name = os.path.join(os.path.basename(tmpdir2), 'archive') - - with os_helper.change_cwd(work_dir), no_chdir: - base_name = os.path.abspath(rel_base_name) - res = make_archive(rel_base_name, 'zip', root_dir) + @support.requires_zlib() + def test_make_zipfile_without_rootdir(self): + root_dir, base_dir = self._create_files() + # Test without base_dir. + base_name = os.path.join(self.mkdtemp(), 'dst', 'archive') + base_name = os.path.relpath(base_name, root_dir) + with os_helper.change_cwd(root_dir), no_chdir: + archive = make_archive(base_name, 'zip') + self.assertEqual(archive, base_name + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3', + 'outer']) + + # Test with base_dir. + root_dir, base_dir = self._create_files() + with os_helper.change_cwd(root_dir), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'zip', base_dir=base_dir) + self.assertEqual(archive, base_name + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) - self.assertEqual(res, base_name + '.zip') - self.assertTrue(os.path.isfile(res)) - self.assertTrue(zipfile.is_zipfile(res)) - with zipfile.ZipFile(res) as zf: - self.assertCountEqual(zf.namelist(), - ['dist/', 'dist/sub/', 'dist/sub2/', - 'dist/file1', 'dist/file2', 'dist/sub/file3', - 'outer']) - - with os_helper.change_cwd(work_dir), no_chdir: - base_name = os.path.abspath(rel_base_name) - res = make_archive(rel_base_name, 'zip', root_dir, base_dir) - - self.assertEqual(res, base_name + '.zip') - self.assertTrue(os.path.isfile(res)) - self.assertTrue(zipfile.is_zipfile(res)) - with zipfile.ZipFile(res) as zf: - self.assertCountEqual(zf.namelist(), - ['dist/', 'dist/sub/', 'dist/sub2/', - 'dist/file1', 'dist/file2', 'dist/sub/file3']) + @support.requires_zlib() + def test_make_zipfile_with_explicit_curdir(self): + # Test with base_dir=os.curdir. + root_dir, base_dir = self._create_files() + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'zip', root_dir, os.curdir) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3', + 'outer']) @support.requires_zlib() @unittest.skipUnless(shutil.which('zip'), @@ -1785,7 +1904,10 @@ def test_unzip_zipfile(self): subprocess.check_output(zip_cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exc: details = exc.output.decode(errors="replace") - if 'unrecognized option: t' in details: + if any(message in details for message in [ + 'unrecognized option: t', # BusyBox + 'invalid option -- t', # Android + ]): self.skipTest("unzip doesn't support -t") msg = "{}\n\n**Unzip Output**\n{}" self.fail(msg.format(exc, details)) @@ -1906,17 +2028,19 @@ def archiver(base_name, base_dir, **kw): unregister_archive_format('xxx') def test_make_tarfile_in_curdir(self): - # Issue #21280 + # Issue #21280: Test with the archive in the current directory. root_dir = self.mkdtemp() with os_helper.change_cwd(root_dir), no_chdir: + # root_dir must be None, so the archive path is relative. self.assertEqual(make_archive('test', 'tar'), 'test.tar') self.assertTrue(os.path.isfile('test.tar')) @support.requires_zlib() def test_make_zipfile_in_curdir(self): - # Issue #21280 + # Issue #21280: Test with the archive in the current directory. root_dir = self.mkdtemp() with os_helper.change_cwd(root_dir), no_chdir: + # root_dir must be None, so the archive path is relative. self.assertEqual(make_archive('test', 'zip'), 'test.zip') self.assertTrue(os.path.isfile('test.zip')) @@ -1937,10 +2061,11 @@ def test_register_archive_format(self): self.assertNotIn('xxx', formats) def test_make_tarfile_rootdir_nodir(self): - # GH-99203 + # GH-99203: Test with root_dir is not a real directory. self.addCleanup(os_helper.unlink, f'{TESTFN}.tar') for dry_run in (False, True): with self.subTest(dry_run=dry_run): + # root_dir does not exist. tmp_dir = self.mkdtemp() nonexisting_file = os.path.join(tmp_dir, 'nonexisting') with self.assertRaises(FileNotFoundError) as cm: @@ -1949,6 +2074,7 @@ def test_make_tarfile_rootdir_nodir(self): self.assertEqual(cm.exception.filename, nonexisting_file) self.assertFalse(os.path.exists(f'{TESTFN}.tar')) + # root_dir is a file. tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) os.close(tmp_fd) with self.assertRaises(NotADirectoryError) as cm: @@ -1959,10 +2085,11 @@ def test_make_tarfile_rootdir_nodir(self): @support.requires_zlib() def test_make_zipfile_rootdir_nodir(self): - # GH-99203 + # GH-99203: Test with root_dir is not a real directory. self.addCleanup(os_helper.unlink, f'{TESTFN}.zip') for dry_run in (False, True): with self.subTest(dry_run=dry_run): + # root_dir does not exist. tmp_dir = self.mkdtemp() nonexisting_file = os.path.join(tmp_dir, 'nonexisting') with self.assertRaises(FileNotFoundError) as cm: @@ -1971,6 +2098,7 @@ def test_make_zipfile_rootdir_nodir(self): self.assertEqual(cm.exception.filename, nonexisting_file) self.assertFalse(os.path.exists(f'{TESTFN}.zip')) + # root_dir is a file. tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) os.close(tmp_fd) with self.assertRaises(NotADirectoryError) as cm: @@ -1985,7 +2113,7 @@ def check_unpack_archive(self, format, **kwargs): self.check_unpack_archive_with_converter( format, lambda path: path, **kwargs) self.check_unpack_archive_with_converter( - format, pathlib.Path, **kwargs) + format, FakePath, **kwargs) self.check_unpack_archive_with_converter(format, FakePath, **kwargs) def check_unpack_archive_with_converter(self, format, converter, **kwargs): @@ -2015,9 +2143,6 @@ def check_unpack_archive_with_converter(self, format, converter, **kwargs): def check_unpack_tarball(self, format): self.check_unpack_archive(format, filter='fully_trusted') self.check_unpack_archive(format, filter='data') - with warnings_helper.check_warnings( - ('Python 3.14', DeprecationWarning)): - self.check_unpack_archive(format) def test_unpack_archive_tar(self): self.check_unpack_tarball('tar') @@ -2090,7 +2215,9 @@ def test_disk_usage(self): def test_chown(self): dirname = self.mkdtemp() filename = tempfile.mktemp(dir=dirname) - write_file(filename, 'testing chown function') + linkname = os.path.join(dirname, "chown_link") + create_file(filename, 'testing chown function') + os.symlink(filename, linkname) with self.assertRaises(ValueError): shutil.chown(filename) @@ -2111,7 +2238,7 @@ def test_chown(self): gid = os.getgid() def check_chown(path, uid=None, gid=None): - s = os.stat(filename) + s = os.stat(path) if uid is not None: self.assertEqual(uid, s.st_uid) if gid is not None: @@ -2147,42 +2274,76 @@ def check_chown(path, uid=None, gid=None): shutil.chown(dirname, user, group) check_chown(dirname, uid, gid) + dirfd = os.open(dirname, os.O_RDONLY) + self.addCleanup(os.close, dirfd) + basename = os.path.basename(filename) + baselinkname = os.path.basename(linkname) + shutil.chown(basename, uid, gid, dir_fd=dirfd) + check_chown(filename, uid, gid) + shutil.chown(basename, uid, dir_fd=dirfd) + check_chown(filename, uid) + shutil.chown(basename, group=gid, dir_fd=dirfd) + check_chown(filename, gid=gid) + shutil.chown(basename, uid, gid, dir_fd=dirfd, follow_symlinks=True) + check_chown(filename, uid, gid) + shutil.chown(basename, uid, gid, dir_fd=dirfd, follow_symlinks=False) + check_chown(filename, uid, gid) + shutil.chown(linkname, uid, follow_symlinks=True) + check_chown(filename, uid) + shutil.chown(baselinkname, group=gid, dir_fd=dirfd, follow_symlinks=False) + check_chown(filename, gid=gid) + shutil.chown(baselinkname, uid, gid, dir_fd=dirfd, follow_symlinks=True) + check_chown(filename, uid, gid) + + with self.assertRaises(TypeError): + shutil.chown(filename, uid, dir_fd=dirname) + + with self.assertRaises(FileNotFoundError): + shutil.chown('missingfile', uid, gid, dir_fd=dirfd) + + with self.assertRaises(ValueError): + shutil.chown(filename, dir_fd=dirfd) + @support.requires_subprocess() class TestWhich(BaseTest, unittest.TestCase): def setUp(self): - self.temp_dir = self.mkdtemp(prefix="Tmp") + temp_dir = self.mkdtemp(prefix="Tmp") + base_dir = os.path.join(temp_dir, TESTFN + '-basedir') + os.mkdir(base_dir) + self.dir = os.path.join(base_dir, TESTFN + '-dir') + os.mkdir(self.dir) + self.other_dir = os.path.join(base_dir, TESTFN + '-dir2') + os.mkdir(self.other_dir) # Give the temp_file an ".exe" suffix for all. # It's needed on Windows and not harmful on other platforms. - self.temp_file = tempfile.NamedTemporaryFile(dir=self.temp_dir, - prefix="Tmp", - suffix=".Exe") - os.chmod(self.temp_file.name, stat.S_IXUSR) - self.addCleanup(self.temp_file.close) - self.dir, self.file = os.path.split(self.temp_file.name) + self.file = TESTFN + '.Exe' + self.filepath = os.path.join(self.dir, self.file) + self.create_file(self.filepath) self.env_path = self.dir self.curdir = os.curdir self.ext = ".EXE" - def to_text_type(self, s): - ''' - In this class we're testing with str, so convert s to a str - ''' - if isinstance(s, bytes): - return s.decode() - return s + to_text_type = staticmethod(os.fsdecode) + + def create_file(self, path): + create_file(path) + os.chmod(path, 0o755) + + def assertNormEqual(self, actual, expected): + self.assertEqual(os.path.normcase(actual), os.path.normcase(expected)) def test_basic(self): # Given an EXE in a directory, it should be returned. rv = shutil.which(self.file, path=self.dir) - self.assertEqual(rv, self.temp_file.name) + self.assertEqual(rv, self.filepath) def test_absolute_cmd(self): # When given the fully qualified path to an executable that exists, # it should be returned. - rv = shutil.which(self.temp_file.name, path=self.temp_dir) - self.assertEqual(rv, self.temp_file.name) + rv = shutil.which(self.filepath, path=self.other_dir) + self.assertEqual(rv, self.filepath) def test_relative_cmd(self): # When given the relative path with a directory part to an executable @@ -2190,7 +2351,7 @@ def test_relative_cmd(self): base_dir, tail_dir = os.path.split(self.dir) relpath = os.path.join(tail_dir, self.file) with os_helper.change_cwd(path=base_dir): - rv = shutil.which(relpath, path=self.temp_dir) + rv = shutil.which(relpath, path=self.other_dir) self.assertEqual(rv, relpath) # But it shouldn't be searched in PATH directories (issue #16957). with os_helper.change_cwd(path=self.dir): @@ -2201,9 +2362,8 @@ def test_relative_cmd(self): "test is for non win32") def test_cwd_non_win32(self): # Issue #16957 - base_dir = os.path.dirname(self.dir) with os_helper.change_cwd(path=self.dir): - rv = shutil.which(self.file, path=base_dir) + rv = shutil.which(self.file, path=self.other_dir) # non-win32: shouldn't match in the current directory. self.assertIsNone(rv) @@ -2213,57 +2373,32 @@ def test_cwd_win32(self): base_dir = os.path.dirname(self.dir) with os_helper.change_cwd(path=self.dir): with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=True): - rv = shutil.which(self.file, path=base_dir) + rv = shutil.which(self.file, path=self.other_dir) # Current directory implicitly on PATH self.assertEqual(rv, os.path.join(self.curdir, self.file)) with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=False): - rv = shutil.which(self.file, path=base_dir) + rv = shutil.which(self.file, path=self.other_dir) # Current directory not on PATH self.assertIsNone(rv) @unittest.skipUnless(sys.platform == "win32", "test is for win32") def test_cwd_win32_added_before_all_other_path(self): - base_dir = pathlib.Path(os.fsdecode(self.dir)) - - elsewhere_in_path_dir = base_dir / 'dir1' - elsewhere_in_path_dir.mkdir() - match_elsewhere_in_path = elsewhere_in_path_dir / 'hello.exe' - match_elsewhere_in_path.touch() - - exe_in_cwd = base_dir / 'hello.exe' - exe_in_cwd.touch() - - with os_helper.change_cwd(path=base_dir): - with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=True): - rv = shutil.which('hello.exe', path=elsewhere_in_path_dir) - - self.assertEqual(os.path.abspath(rv), os.path.abspath(exe_in_cwd)) - - @unittest.skipUnless(sys.platform == "win32", - "test is for win32") - def test_pathext_match_before_path_full_match(self): - base_dir = pathlib.Path(os.fsdecode(self.dir)) - dir1 = base_dir / 'dir1' - dir2 = base_dir / 'dir2' - dir1.mkdir() - dir2.mkdir() - - pathext_match = dir1 / 'hello.com.exe' - path_match = dir2 / 'hello.com' - pathext_match.touch() - path_match.touch() - - test_path = os.pathsep.join([str(dir1), str(dir2)]) - assert os.path.basename(shutil.which( - 'hello.com', path=test_path, mode = os.F_OK - )).lower() == 'hello.com.exe' + other_file_path = os.path.join(self.other_dir, self.file) + self.create_file(other_file_path) + with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=True): + with os_helper.change_cwd(path=self.dir): + rv = shutil.which(self.file, path=self.other_dir) + self.assertEqual(rv, os.path.join(self.curdir, self.file)) + with os_helper.change_cwd(path=self.other_dir): + rv = shutil.which(self.file, path=self.dir) + self.assertEqual(rv, os.path.join(self.curdir, self.file)) @os_helper.skip_if_dac_override def test_non_matching_mode(self): # Set the file read-only and ask for writeable files. - os.chmod(self.temp_file.name, stat.S_IREAD) - if os.access(self.temp_file.name, os.W_OK): + os.chmod(self.filepath, stat.S_IREAD) + if os.access(self.filepath, os.W_OK): self.skipTest("can't set the file read-only") rv = shutil.which(self.file, path=self.dir, mode=os.W_OK) self.assertIsNone(rv) @@ -2285,13 +2420,13 @@ def test_pathext_checking(self): # Ask for the file without the ".exe" extension, then ensure that # it gets found properly with the extension. rv = shutil.which(self.file[:-4], path=self.dir) - self.assertEqual(rv, self.temp_file.name[:-4] + self.ext) + self.assertEqual(rv, self.filepath[:-4] + self.ext) def test_environ_path(self): with os_helper.EnvironmentVarGuard() as env: env['PATH'] = self.env_path rv = shutil.which(self.file) - self.assertEqual(rv, self.temp_file.name) + self.assertEqual(rv, self.filepath) def test_environ_path_empty(self): # PATH='': no match @@ -2305,12 +2440,9 @@ def test_environ_path_empty(self): self.assertIsNone(rv) def test_environ_path_cwd(self): - expected_cwd = os.path.basename(self.temp_file.name) + expected_cwd = self.file if sys.platform == "win32": - curdir = os.curdir - if isinstance(expected_cwd, bytes): - curdir = os.fsencode(curdir) - expected_cwd = os.path.join(curdir, expected_cwd) + expected_cwd = os.path.join(self.curdir, expected_cwd) # PATH=':': explicitly looks in the current directory with os_helper.EnvironmentVarGuard() as env: @@ -2335,14 +2467,14 @@ def test_environ_path_missing(self): create=True), \ support.swap_attr(os, 'defpath', self.dir): rv = shutil.which(self.file) - self.assertEqual(rv, self.temp_file.name) + self.assertEqual(rv, self.filepath) # with confstr with unittest.mock.patch('os.confstr', return_value=self.dir, \ create=True), \ support.swap_attr(os, 'defpath', ''): rv = shutil.which(self.file) - self.assertEqual(rv, self.temp_file.name) + self.assertEqual(rv, self.filepath) def test_empty_path(self): base_dir = os.path.dirname(self.dir) @@ -2360,50 +2492,88 @@ def test_empty_path_no_PATH(self): @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') def test_pathext(self): - ext = self.to_text_type(".xyz") - temp_filexyz = tempfile.NamedTemporaryFile(dir=self.temp_dir, - prefix=self.to_text_type("Tmp2"), suffix=ext) - os.chmod(temp_filexyz.name, stat.S_IXUSR) - self.addCleanup(temp_filexyz.close) - - # strip path and extension - program = os.path.basename(temp_filexyz.name) - program = os.path.splitext(program)[0] - + ext = '.xyz' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) with os_helper.EnvironmentVarGuard() as env: - env['PATHEXT'] = ext if isinstance(ext, str) else ext.decode() - rv = shutil.which(program, path=self.temp_dir) - self.assertEqual(rv, temp_filexyz.name) + env['PATHEXT'] = ext + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmdext, path=self.dir), filepath) # Issue 40592: See https://bugs.python.org/issue40592 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') def test_pathext_with_empty_str(self): - ext = self.to_text_type(".xyz") - temp_filexyz = tempfile.NamedTemporaryFile(dir=self.temp_dir, - prefix=self.to_text_type("Tmp2"), suffix=ext) - self.addCleanup(temp_filexyz.close) + ext = '.xyz' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) + with os_helper.EnvironmentVarGuard() as env: + env['PATHEXT'] = ext + ';' # note the ; + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmdext, path=self.dir), filepath) - # strip path and extension - program = os.path.basename(temp_filexyz.name) - program = os.path.splitext(program)[0] + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_with_multidot_extension(self): + ext = '.foo.bar' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) + with os_helper.EnvironmentVarGuard() as env: + env['PATHEXT'] = ext + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmdext, path=self.dir), filepath) + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_with_null_extension(self): + cmd = self.to_text_type(TESTFN2) + cmddot = cmd + self.to_text_type('.') + filepath = os.path.join(self.dir, cmd) + self.create_file(filepath) + with os_helper.EnvironmentVarGuard() as env: + env['PATHEXT'] = '.xyz' + self.assertIsNone(shutil.which(cmd, path=self.dir)) + self.assertIsNone(shutil.which(cmddot, path=self.dir)) + env['PATHEXT'] = '.xyz;.' # note the . + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmddot, path=self.dir), + filepath + self.to_text_type('.')) + env['PATHEXT'] = '.xyz;..' # multiple dots + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmddot, path=self.dir), + filepath + self.to_text_type('.')) + + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_extension_ends_with_dot(self): + ext = '.xyz' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + dot = self.to_text_type('.') + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) with os_helper.EnvironmentVarGuard() as env: - env['PATHEXT'] = f"{ext if isinstance(ext, str) else ext.decode()};" # note the ; - rv = shutil.which(program, path=self.temp_dir) - self.assertEqual(rv, temp_filexyz.name) + env['PATHEXT'] = ext + '.' + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) # cmd.exe hangs here + self.assertEqual(shutil.which(cmdext, path=self.dir), filepath) + self.assertIsNone(shutil.which(cmd + dot, path=self.dir)) + self.assertIsNone(shutil.which(cmdext + dot, path=self.dir)) # See GH-75586 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') def test_pathext_applied_on_files_in_path(self): + ext = '.xyz' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) with os_helper.EnvironmentVarGuard() as env: - env["PATH"] = self.temp_dir if isinstance(self.temp_dir, str) else self.temp_dir.decode() - env["PATHEXT"] = ".test" - - test_path = os.path.join(self.temp_dir, self.to_text_type("test_program.test")) - open(test_path, 'w').close() - os.chmod(test_path, 0o755) - - self.assertEqual(shutil.which(self.to_text_type("test_program")), test_path) + env["PATH"] = os.fsdecode(self.dir) + env["PATHEXT"] = ext + self.assertEqual(shutil.which(cmd), filepath) + self.assertEqual(shutil.which(cmdext), filepath) # See GH-75586 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') @@ -2419,49 +2589,107 @@ def test_win_path_needs_curdir(self): self.assertFalse(shutil._win_path_needs_curdir('dontcare', os.X_OK)) need_curdir_mock.assert_called_once_with('dontcare') - # See GH-109590 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') - def test_pathext_preferred_for_execute(self): - with os_helper.EnvironmentVarGuard() as env: - env["PATH"] = self.temp_dir if isinstance(self.temp_dir, str) else self.temp_dir.decode() - env["PATHEXT"] = ".test" - - exe = os.path.join(self.temp_dir, self.to_text_type("test.exe")) - open(exe, 'w').close() - os.chmod(exe, 0o755) - - # default behavior allows a direct match if nothing in PATHEXT matches - self.assertEqual(shutil.which(self.to_text_type("test.exe")), exe) - - dot_test = os.path.join(self.temp_dir, self.to_text_type("test.exe.test")) - open(dot_test, 'w').close() - os.chmod(dot_test, 0o755) - - # now we have a PATHEXT match, so it take precedence - self.assertEqual(shutil.which(self.to_text_type("test.exe")), dot_test) - - # but if we don't use os.X_OK we don't change the order based off PATHEXT - # and therefore get the direct match. - self.assertEqual(shutil.which(self.to_text_type("test.exe"), mode=os.F_OK), exe) + def test_same_dir_with_pathext_extension(self): + cmd = self.file # with .exe extension + # full match + self.assertNormEqual(shutil.which(cmd, path=self.dir), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=self.dir, mode=os.F_OK), + self.filepath) + + cmd2 = cmd + self.to_text_type('.com') # with .exe.com extension + other_file_path = os.path.join(self.dir, cmd2) + self.create_file(other_file_path) + + # full match + self.assertNormEqual(shutil.which(cmd, path=self.dir), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=self.dir, mode=os.F_OK), + self.filepath) + self.assertNormEqual(shutil.which(cmd2, path=self.dir), other_file_path) + self.assertNormEqual(shutil.which(cmd2, path=self.dir, mode=os.F_OK), + other_file_path) - # See GH-109590 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') - def test_pathext_given_extension_preferred(self): - with os_helper.EnvironmentVarGuard() as env: - env["PATH"] = self.temp_dir if isinstance(self.temp_dir, str) else self.temp_dir.decode() - env["PATHEXT"] = ".exe2;.exe" - - exe = os.path.join(self.temp_dir, self.to_text_type("test.exe")) - open(exe, 'w').close() - os.chmod(exe, 0o755) + def test_same_dir_without_pathext_extension(self): + cmd = self.file[:-4] # without .exe extension + # pathext match + self.assertNormEqual(shutil.which(cmd, path=self.dir), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=self.dir, mode=os.F_OK), + self.filepath) + + # without extension + other_file_path = os.path.join(self.dir, cmd) + self.create_file(other_file_path) + + # pathext match if mode contains X_OK + self.assertNormEqual(shutil.which(cmd, path=self.dir), self.filepath) + # full match + self.assertNormEqual(shutil.which(cmd, path=self.dir, mode=os.F_OK), + other_file_path) + self.assertNormEqual(shutil.which(self.file, path=self.dir), self.filepath) + self.assertNormEqual(shutil.which(self.file, path=self.dir, mode=os.F_OK), + self.filepath) - exe2 = os.path.join(self.temp_dir, self.to_text_type("test.exe2")) - open(exe2, 'w').close() - os.chmod(exe2, 0o755) + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_dir_order_with_pathext_extension(self): + cmd = self.file # with .exe extension + search_path = os.pathsep.join([os.fsdecode(self.other_dir), + os.fsdecode(self.dir)]) + # full match in the second directory + self.assertNormEqual(shutil.which(cmd, path=search_path), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + self.filepath) + + cmd2 = cmd + self.to_text_type('.com') # with .exe.com extension + other_file_path = os.path.join(self.other_dir, cmd2) + self.create_file(other_file_path) + + # pathext match in the first directory + self.assertNormEqual(shutil.which(cmd, path=search_path), other_file_path) + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + other_file_path) + # full match in the first directory + self.assertNormEqual(shutil.which(cmd2, path=search_path), other_file_path) + self.assertNormEqual(shutil.which(cmd2, path=search_path, mode=os.F_OK), + other_file_path) + + # full match in the first directory + search_path = os.pathsep.join([os.fsdecode(self.dir), + os.fsdecode(self.other_dir)]) + self.assertEqual(shutil.which(cmd, path=search_path), self.filepath) + self.assertEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + self.filepath) - # even though .exe2 is preferred in PATHEXT, we matched directly to test.exe - self.assertEqual(shutil.which(self.to_text_type("test.exe")), exe) - self.assertEqual(shutil.which(self.to_text_type("test")), exe2) + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_dir_order_without_pathext_extension(self): + cmd = self.file[:-4] # without .exe extension + search_path = os.pathsep.join([os.fsdecode(self.other_dir), + os.fsdecode(self.dir)]) + # pathext match in the second directory + self.assertNormEqual(shutil.which(cmd, path=search_path), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + self.filepath) + + # without extension + other_file_path = os.path.join(self.other_dir, cmd) + self.create_file(other_file_path) + + # pathext match in the second directory + self.assertNormEqual(shutil.which(cmd, path=search_path), self.filepath) + # full match in the first directory + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + other_file_path) + # full match in the second directory + self.assertNormEqual(shutil.which(self.file, path=search_path), self.filepath) + self.assertNormEqual(shutil.which(self.file, path=search_path, mode=os.F_OK), + self.filepath) + + # pathext match in the first directory + search_path = os.pathsep.join([os.fsdecode(self.dir), + os.fsdecode(self.other_dir)]) + self.assertNormEqual(shutil.which(cmd, path=search_path), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + self.filepath) class TestWhichBytes(TestWhich): @@ -2469,18 +2697,12 @@ def setUp(self): TestWhich.setUp(self) self.dir = os.fsencode(self.dir) self.file = os.fsencode(self.file) - self.temp_file.name = os.fsencode(self.temp_file.name) - self.temp_dir = os.fsencode(self.temp_dir) + self.filepath = os.fsencode(self.filepath) + self.other_dir = os.fsencode(self.other_dir) self.curdir = os.fsencode(self.curdir) self.ext = os.fsencode(self.ext) - def to_text_type(self, s): - ''' - In this class we're testing with bytes, so convert s to a bytes - ''' - if isinstance(s, str): - return s.encode() - return s + to_text_type = staticmethod(os.fsencode) class TestMove(BaseTest, unittest.TestCase): @@ -2491,8 +2713,7 @@ def setUp(self): self.dst_dir = self.mkdtemp() self.src_file = os.path.join(self.src_dir, filename) self.dst_file = os.path.join(self.dst_dir, filename) - with open(self.src_file, "wb") as f: - f.write(b"spam") + create_file(self.src_file, b"spam") def _check_move_file(self, src, dst, real_dst): with open(src, "rb") as f: @@ -2518,12 +2739,12 @@ def test_move_file_to_dir(self): def test_move_file_to_dir_pathlike_src(self): # Move a pathlike file to another location on the same filesystem. - src = pathlib.Path(self.src_file) + src = FakePath(self.src_file) self._check_move_file(src, self.dst_dir, self.dst_file) def test_move_file_to_dir_pathlike_dst(self): # Move a file to another pathlike location on the same filesystem. - dst = pathlib.Path(self.dst_dir) + dst = FakePath(self.dst_dir) self._check_move_file(self.src_file, dst, self.dst_file) @mock_rename @@ -2570,8 +2791,7 @@ def test_move_dir_altsep_to_dir(self): def test_existing_file_inside_dest_dir(self): # A file with the same name inside the destination dir already exists. - with open(self.dst_file, "wb"): - pass + create_file(self.dst_file) self.assertRaises(shutil.Error, shutil.move, self.src_file, self.dst_dir) def test_dont_move_dir_in_itself(self): @@ -2986,8 +3206,7 @@ def test_empty_file(self): dstname = TESTFN + 'dst' self.addCleanup(lambda: os_helper.unlink(srcname)) self.addCleanup(lambda: os_helper.unlink(dstname)) - with open(srcname, "wb"): - pass + create_file(srcname) with open(srcname, "rb") as src: with open(dstname, "wb") as dst: @@ -3110,7 +3329,7 @@ def test_blocksize_arg(self): self.assertEqual(blocksize, os.path.getsize(TESTFN)) # ...unless we're dealing with a small file. os_helper.unlink(TESTFN2) - write_file(TESTFN2, b"hello", binary=True) + create_file(TESTFN2, b"hello") self.addCleanup(os_helper.unlink, TESTFN2 + '3') self.assertRaises(ZeroDivisionError, shutil.copyfile, TESTFN2, TESTFN2 + '3') @@ -3234,7 +3453,7 @@ def test_module_all_attribute(self): self.assertTrue(hasattr(shutil, '__all__')) target_api = ['copyfileobj', 'copyfile', 'copymode', 'copystat', 'copy', 'copy2', 'copytree', 'move', 'rmtree', 'Error', - 'SpecialFileError', 'ExecError', 'make_archive', + 'SpecialFileError', 'make_archive', 'get_archive_formats', 'register_archive_format', 'unregister_archive_format', 'get_unpack_formats', 'register_unpack_format', 'unregister_unpack_format', @@ -3243,6 +3462,8 @@ def test_module_all_attribute(self): if hasattr(os, 'statvfs') or os.name == 'nt': target_api.append('disk_usage') self.assertEqual(set(shutil.__all__), set(target_api)) + with self.assertWarns(DeprecationWarning): + from shutil import ExecError if __name__ == '__main__': diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 61fb047caf6dab..96ae79b0eb18b3 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -123,6 +123,8 @@ def __repr__(self): self.assertEqual(signal.getsignal(signal.SIGHUP), hup) self.assertEqual(0, argument.repr_count) + @unittest.skipIf(sys.platform.startswith("netbsd"), + "gh-124083: strsignal is not supported on NetBSD") def test_strsignal(self): self.assertIn("Interrupt", signal.strsignal(signal.SIGINT)) self.assertIn("Terminated", signal.strsignal(signal.SIGTERM)) @@ -251,9 +253,7 @@ def test_invalid_socket(self): self.assertRaises((ValueError, OSError), signal.set_wakeup_fd, fd) - # Emscripten does not support fstat on pipes yet. - # https://github.com/emscripten-core/emscripten/issues/16414 - @unittest.skipIf(support.is_emscripten, "Emscripten cannot fstat pipes.") + @unittest.skipIf(support.is_emscripten, "Fixed in next Emscripten release after 4.0.1") @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_set_wakeup_fd_result(self): r1, w1 = os.pipe() @@ -272,7 +272,7 @@ def test_set_wakeup_fd_result(self): self.assertEqual(signal.set_wakeup_fd(-1), w2) self.assertEqual(signal.set_wakeup_fd(-1), -1) - @unittest.skipIf(support.is_emscripten, "Emscripten cannot fstat pipes.") + @unittest.skipIf(support.is_emscripten, "Fixed in next Emscripten release after 4.0.1") @unittest.skipUnless(support.has_socket_support, "needs working sockets.") def test_set_wakeup_fd_socket_result(self): sock1 = socket.socket() @@ -293,7 +293,7 @@ def test_set_wakeup_fd_socket_result(self): # On Windows, files are always blocking and Windows does not provide a # function to test if a socket is in non-blocking mode. @unittest.skipIf(sys.platform == "win32", "tests specific to POSIX") - @unittest.skipIf(support.is_emscripten, "Emscripten cannot fstat pipes.") + @unittest.skipIf(support.is_emscripten, "Fixed in next Emscripten release after 4.0.1") @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_set_wakeup_fd_blocking(self): rfd, wfd = os.pipe() @@ -698,7 +698,7 @@ def handler(signum, frame): @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") class SiginterruptTest(unittest.TestCase): - def readpipe_interrupted(self, interrupt): + def readpipe_interrupted(self, interrupt, timeout=support.SHORT_TIMEOUT): """Perform a read during which a signal will arrive. Return True if the read is interrupted by the signal and raises an exception. Return False if it returns normally. @@ -746,7 +746,7 @@ def handler(signum, frame): # wait until the child process is loaded and has started first_line = process.stdout.readline() - stdout, stderr = process.communicate(timeout=support.SHORT_TIMEOUT) + stdout, stderr = process.communicate(timeout=timeout) except subprocess.TimeoutExpired: process.kill() return False @@ -777,7 +777,7 @@ def test_siginterrupt_off(self): # If a signal handler is installed and siginterrupt is called with # a false value for the second argument, when that signal arrives, it # does not interrupt a syscall that's in progress. - interrupted = self.readpipe_interrupted(False) + interrupted = self.readpipe_interrupted(False, timeout=2) self.assertFalse(interrupted) @@ -1325,15 +1325,18 @@ def test_stress_delivery_simultaneous(self): def handler(signum, frame): sigs.append(signum) - self.setsig(signal.SIGUSR1, handler) + # On Android, SIGUSR1 is unreliable when used in close proximity to + # another signal – see Android/testbed/app/src/main/python/main.py. + # So we use a different signal. + self.setsig(signal.SIGUSR2, handler) self.setsig(signal.SIGALRM, handler) # for ITIMER_REAL expected_sigs = 0 while expected_sigs < N: # Hopefully the SIGALRM will be received somewhere during - # initial processing of SIGUSR1. + # initial processing of SIGUSR2. signal.setitimer(signal.ITIMER_REAL, 1e-6 + random.random() * 1e-5) - os.kill(os.getpid(), signal.SIGUSR1) + os.kill(os.getpid(), signal.SIGUSR2) expected_sigs += 2 # Wait for handlers to run to avoid signal coalescing @@ -1345,6 +1348,7 @@ def handler(signum, frame): # Python handler self.assertEqual(len(sigs), N, "Some signals were lost") + @support.requires_gil_enabled("gh-121065: test is flaky on free-threaded build") @unittest.skipIf(is_apple, "crashes due to system bug (FB13453490)") @unittest.skipUnless(hasattr(signal, "SIGUSR1"), "test needs SIGUSR1") diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 0502181854f52b..035913cdd05f34 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -328,13 +328,13 @@ def test_getsitepackages(self): if sys.platlibdir != "lib": self.assertEqual(len(dirs), 2) wanted = os.path.join('xoxo', sys.platlibdir, - 'python%d.%d' % sys.version_info[:2], + f'python{sysconfig._get_python_version_abi()}', 'site-packages') self.assertEqual(dirs[0], wanted) else: self.assertEqual(len(dirs), 1) wanted = os.path.join('xoxo', 'lib', - 'python%d.%d' % sys.version_info[:2], + f'python{sysconfig._get_python_version_abi()}', 'site-packages') self.assertEqual(dirs[-1], wanted) else: @@ -513,7 +513,7 @@ def test_sitecustomize_executed(self): # If sitecustomize is available, it should have been imported. if "sitecustomize" not in sys.modules: try: - import sitecustomize + import sitecustomize # noqa: F401 except ImportError: pass else: diff --git a/Lib/test/test_smtpnet.py b/Lib/test/test_smtpnet.py index 2e0dc1aa276f35..d765746987bc4b 100644 --- a/Lib/test/test_smtpnet.py +++ b/Lib/test/test_smtpnet.py @@ -2,6 +2,7 @@ from test import support from test.support import import_helper from test.support import socket_helper +import os import smtplib import socket @@ -9,6 +10,8 @@ support.requires("network") +SMTP_TEST_SERVER = os.getenv('CPYTHON_TEST_SMTP_SERVER', 'smtp.gmail.com') + def check_ssl_verifiy(host, port): context = ssl.create_default_context() with socket.create_connection((host, port)) as sock: @@ -22,7 +25,7 @@ def check_ssl_verifiy(host, port): class SmtpTest(unittest.TestCase): - testServer = 'smtp.gmail.com' + testServer = SMTP_TEST_SERVER remotePort = 587 def test_connect_starttls(self): @@ -44,7 +47,7 @@ def test_connect_starttls(self): class SmtpSSLTest(unittest.TestCase): - testServer = 'smtp.gmail.com' + testServer = SMTP_TEST_SERVER remotePort = 465 def test_connect(self): diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index b936e9ae91daca..faf326d9164e1b 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -3,7 +3,6 @@ from test.support import ( is_apple, os_helper, refleak_helper, socket_helper, threading_helper ) - import _thread as thread import array import contextlib @@ -37,6 +36,10 @@ import fcntl except ImportError: fcntl = None +try: + import _testcapi +except ImportError: + _testcapi = None support.requires_working_socket(module=True) @@ -157,8 +160,8 @@ def _have_socket_qipcrtr(): def _have_socket_vsock(): """Check whether AF_VSOCK sockets are supported on this host.""" - ret = get_cid() is not None - return ret + cid = get_cid() + return (cid is not None) def _have_socket_bluetooth(): @@ -209,7 +212,10 @@ def socket_setdefaulttimeout(timeout): HAVE_SOCKET_VSOCK = _have_socket_vsock() -HAVE_SOCKET_UDPLITE = hasattr(socket, "IPPROTO_UDPLITE") +# Older Android versions block UDPLITE with SELinux. +HAVE_SOCKET_UDPLITE = ( + hasattr(socket, "IPPROTO_UDPLITE") + and not (support.is_android and platform.android_ver().api_level < 29)) HAVE_SOCKET_BLUETOOTH = _have_socket_bluetooth() @@ -514,8 +520,6 @@ def clientTearDown(self): @unittest.skipIf(WSL, 'VSOCK does not work on Microsoft WSL') @unittest.skipUnless(HAVE_SOCKET_VSOCK, 'VSOCK sockets required for this test.') -@unittest.skipUnless(get_cid() != 2, - "This test can only be run on a virtual guest.") class ThreadedVSOCKSocketStreamTest(unittest.TestCase, ThreadableTest): def __init__(self, methodName='runTest'): @@ -537,10 +541,16 @@ def clientSetUp(self): self.cli = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) self.addCleanup(self.cli.close) cid = get_cid() + if cid in (socket.VMADDR_CID_HOST, socket.VMADDR_CID_ANY): + # gh-119461: Use the local communication address (loopback) + cid = socket.VMADDR_CID_LOCAL self.cli.connect((cid, VSOCKPORT)) def testStream(self): - msg = self.conn.recv(1024) + try: + msg = self.conn.recv(1024) + except PermissionError as exc: + self.skipTest(repr(exc)) self.assertEqual(msg, MSG) def _testStream(self): @@ -585,19 +595,27 @@ class SocketPairTest(unittest.TestCase, ThreadableTest): def __init__(self, methodName='runTest'): unittest.TestCase.__init__(self, methodName=methodName) ThreadableTest.__init__(self) + self.cli = None + self.serv = None + + def socketpair(self): + # To be overridden by some child classes. + return socket.socketpair() def setUp(self): - self.serv, self.cli = socket.socketpair() + self.serv, self.cli = self.socketpair() def tearDown(self): - self.serv.close() + if self.serv: + self.serv.close() self.serv = None def clientSetUp(self): pass def clientTearDown(self): - self.cli.close() + if self.cli: + self.cli.close() self.cli = None ThreadableTest.clientTearDown(self) @@ -1096,6 +1114,7 @@ def test_sethostname(self): @unittest.skipUnless(hasattr(socket, 'if_nameindex'), 'socket.if_nameindex() not available.') + @support.skip_android_selinux('if_nameindex') def testInterfaceNameIndex(self): interfaces = socket.if_nameindex() for index, name in interfaces: @@ -1112,6 +1131,7 @@ def testInterfaceNameIndex(self): @unittest.skipUnless(hasattr(socket, 'if_indextoname'), 'socket.if_indextoname() not available.') + @support.skip_android_selinux('if_indextoname') def testInvalidInterfaceIndexToName(self): self.assertRaises(OSError, socket.if_indextoname, 0) self.assertRaises(OverflowError, socket.if_indextoname, -1) @@ -1131,6 +1151,7 @@ def testInvalidInterfaceIndexToName(self): @unittest.skipUnless(hasattr(socket, 'if_nametoindex'), 'socket.if_nametoindex() not available.') + @support.skip_android_selinux('if_nametoindex') def testInvalidInterfaceNameToIndex(self): self.assertRaises(TypeError, socket.if_nametoindex, 0) self.assertRaises(OSError, socket.if_nametoindex, '_DEADBEEF') @@ -1170,6 +1191,7 @@ def testNtoH(self): self.assertRaises(OverflowError, func, 1<<34) @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def testNtoHErrors(self): import _testcapi s_good_values = [0, 1, 2, 0xffff] @@ -1199,8 +1221,8 @@ def testGetServBy(self): # I've ordered this by protocols that have both a tcp and udp # protocol, at least for modern Linuxes. if ( - sys.platform.startswith(('freebsd', 'netbsd', 'gnukfreebsd')) - or sys.platform == 'linux' + sys.platform.startswith( + ('linux', 'android', 'freebsd', 'netbsd', 'gnukfreebsd')) or is_apple ): # avoid the 'echo' service on this platform, as there is an @@ -1217,9 +1239,8 @@ def testGetServBy(self): else: raise OSError # Try same call with optional protocol omitted - # Issue #26936: Android getservbyname() was broken before API 23. - if (not hasattr(sys, 'getandroidapilevel') or - sys.getandroidapilevel() >= 23): + # Issue gh-71123: this fails on Android before API level 23. + if not (support.is_android and platform.android_ver().api_level < 23): port2 = socket.getservbyname(service) eq(port, port2) # Try udp, but don't barf if it doesn't exist @@ -1230,8 +1251,9 @@ def testGetServBy(self): else: eq(udpport, port) # Now make sure the lookup by port returns the same service name - # Issue #26936: Android getservbyport() is broken. - if not support.is_android: + # Issue #26936: when the protocol is omitted, this fails on Android + # before API level 28. + if not (support.is_android and platform.android_ver().api_level < 28): eq(socket.getservbyport(port2), service) eq(socket.getservbyport(port, 'tcp'), service) if udpport is not None: @@ -1576,9 +1598,8 @@ def testGetaddrinfo(self): socket.getaddrinfo('::1', 80) # port can be a string service name such as "http", a numeric # port number or None - # Issue #26936: Android getaddrinfo() was broken before API level 23. - if (not hasattr(sys, 'getandroidapilevel') or - sys.getandroidapilevel() >= 23): + # Issue #26936: this fails on Android before API level 23. + if not (support.is_android and platform.android_ver().api_level < 23): socket.getaddrinfo(HOST, "http") socket.getaddrinfo(HOST, 80) socket.getaddrinfo(HOST, None) @@ -1636,6 +1657,7 @@ def testGetaddrinfo(self): except socket.gaierror: pass + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_getaddrinfo_int_port_overflow(self): # gh-74895: Test that getaddrinfo does not raise OverflowError on port. # @@ -1653,7 +1675,7 @@ def test_getaddrinfo_int_port_overflow(self): try: socket.getaddrinfo(None, ULONG_MAX + 1, type=socket.SOCK_STREAM) except OverflowError: - # Platforms differ as to what values consitute a getaddrinfo() error + # Platforms differ as to what values constitute a getaddrinfo() error # return. Some fail for LONG_MAX+1, others ULONG_MAX+1, and Windows # silently accepts such huge "port" aka "service" numeric values. self.fail("Either no error or socket.gaierror expected.") @@ -1829,6 +1851,7 @@ def test_listen_backlog(self): srv.listen() @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_listen_backlog_overflow(self): # Issue 15989 import _testcapi @@ -1861,6 +1884,7 @@ def test_getfqdn_filter_localhost(self): @unittest.skipIf(sys.platform == 'win32', 'does not work on Windows') @unittest.skipIf(AIX, 'Symbolic scope id does not work') @unittest.skipUnless(hasattr(socket, 'if_nameindex'), "test needs socket.if_nameindex()") + @support.skip_android_selinux('if_nameindex') def test_getaddrinfo_ipv6_scopeid_symbolic(self): # Just pick up any network interface (Linux, Mac OS X) (ifindex, test_interface) = socket.if_nameindex()[0] @@ -1894,6 +1918,7 @@ def test_getaddrinfo_ipv6_scopeid_numeric(self): @unittest.skipIf(sys.platform == 'win32', 'does not work on Windows') @unittest.skipIf(AIX, 'Symbolic scope id does not work') @unittest.skipUnless(hasattr(socket, 'if_nameindex'), "test needs socket.if_nameindex()") + @support.skip_android_selinux('if_nameindex') def test_getnameinfo_ipv6_scopeid_symbolic(self): # Just pick up any network interface. (ifindex, test_interface) = socket.if_nameindex()[0] @@ -2507,6 +2532,7 @@ def testVSOCKConstants(self): socket.SO_VM_SOCKETS_BUFFER_MAX_SIZE socket.VMADDR_CID_ANY socket.VMADDR_PORT_ANY + socket.VMADDR_CID_LOCAL socket.VMADDR_CID_HOST socket.VM_SOCKETS_INVALID_VERSION socket.IOCTL_VM_SOCKETS_GET_LOCAL_CID @@ -2710,22 +2736,29 @@ def testDup(self): def _testDup(self): self.serv_conn.send(MSG) - def testShutdown(self): - # Testing shutdown() + def check_shutdown(self): + # Test shutdown() helper msg = self.cli_conn.recv(1024) self.assertEqual(msg, MSG) - # wait for _testShutdown to finish: on OS X, when the server + # wait for _testShutdown[_overflow] to finish: on OS X, when the server # closes the connection the client also becomes disconnected, # and the client's shutdown call will fail. (Issue #4397.) self.done.wait() + def testShutdown(self): + self.check_shutdown() + def _testShutdown(self): self.serv_conn.send(MSG) self.serv_conn.shutdown(2) - testShutdown_overflow = support.cpython_only(testShutdown) + @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def testShutdown_overflow(self): + self.check_shutdown() @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def _testShutdown_overflow(self): import _testcapi self.serv_conn.send(MSG) @@ -3196,7 +3229,7 @@ def _testSendmsgTimeout(self): # Linux supports MSG_DONTWAIT when sending, but in general, it # only works when receiving. Could add other platforms if they # support it too. - @skipWithClientIf(sys.platform not in {"linux"}, + @skipWithClientIf(sys.platform not in {"linux", "android"}, "MSG_DONTWAIT not known to work on this platform when " "sending") def testSendmsgDontWait(self): @@ -4781,15 +4814,13 @@ def testInterruptedSendmsgTimeout(self): class TCPCloserTest(ThreadedTCPSocketTest): - def testClose(self): - conn, addr = self.serv.accept() - conn.close() + conn, _ = self.serv.accept() - sd = self.cli - read, write, err = select.select([sd], [], [], 1.0) - self.assertEqual(read, [sd]) - self.assertEqual(sd.recv(1), b'') + read, _, _ = select.select([conn], [], [], support.SHORT_TIMEOUT) + self.assertEqual(read, [conn]) + self.assertEqual(conn.recv(1), b'x') + conn.close() # Calling close() many times should be safe. conn.close() @@ -4797,7 +4828,10 @@ def testClose(self): def _testClose(self): self.cli.connect((HOST, self.port)) - time.sleep(1.0) + self.cli.send(b'x') + read, _, _ = select.select([self.cli], [], [], support.SHORT_TIMEOUT) + self.assertEqual(read, [self.cli]) + self.assertEqual(self.cli.recv(1), b'') class BasicSocketPairTest(SocketPairTest): @@ -4835,6 +4869,112 @@ def _testSend(self): self.assertEqual(msg, MSG) +class PurePythonSocketPairTest(SocketPairTest): + # Explicitly use socketpair AF_INET or AF_INET6 to ensure that is the + # code path we're using regardless platform is the pure python one where + # `_socket.socketpair` does not exist. (AF_INET does not work with + # _socket.socketpair on many platforms). + def socketpair(self): + # called by super().setUp(). + try: + return socket.socketpair(socket.AF_INET6) + except OSError: + return socket.socketpair(socket.AF_INET) + + # Local imports in this class make for easy security fix backporting. + + def setUp(self): + if hasattr(_socket, "socketpair"): + self._orig_sp = socket.socketpair + # This forces the version using the non-OS provided socketpair + # emulation via an AF_INET socket in Lib/socket.py. + socket.socketpair = socket._fallback_socketpair + else: + # This platform already uses the non-OS provided version. + self._orig_sp = None + super().setUp() + + def tearDown(self): + super().tearDown() + if self._orig_sp is not None: + # Restore the default socket.socketpair definition. + socket.socketpair = self._orig_sp + + def test_recv(self): + msg = self.serv.recv(1024) + self.assertEqual(msg, MSG) + + def _test_recv(self): + self.cli.send(MSG) + + def test_send(self): + self.serv.send(MSG) + + def _test_send(self): + msg = self.cli.recv(1024) + self.assertEqual(msg, MSG) + + def test_ipv4(self): + cli, srv = socket.socketpair(socket.AF_INET) + cli.close() + srv.close() + + def _test_ipv4(self): + pass + + @unittest.skipIf(not hasattr(_socket, 'IPPROTO_IPV6') or + not hasattr(_socket, 'IPV6_V6ONLY'), + "IPV6_V6ONLY option not supported") + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 required for this test') + def test_ipv6(self): + cli, srv = socket.socketpair(socket.AF_INET6) + cli.close() + srv.close() + + def _test_ipv6(self): + pass + + def test_injected_authentication_failure(self): + orig_getsockname = socket.socket.getsockname + inject_sock = None + + def inject_getsocketname(self): + nonlocal inject_sock + sockname = orig_getsockname(self) + # Connect to the listening socket ahead of the + # client socket. + if inject_sock is None: + inject_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + inject_sock.setblocking(False) + try: + inject_sock.connect(sockname[:2]) + except (BlockingIOError, InterruptedError): + pass + inject_sock.setblocking(True) + return sockname + + sock1 = sock2 = None + try: + socket.socket.getsockname = inject_getsocketname + with self.assertRaises(OSError): + sock1, sock2 = socket.socketpair() + finally: + socket.socket.getsockname = orig_getsockname + if inject_sock: + inject_sock.close() + if sock1: # This cleanup isn't needed on a successful test. + sock1.close() + if sock2: + sock2.close() + + def _test_injected_authentication_failure(self): + # No-op. Exists for base class threading infrastructure to call. + # We could refactor this test into its own lesser class along with the + # setUp and tearDown code to construct an ideal; it is simpler to keep + # it here and live with extra overhead one this _one_ failure test. + pass + + class NonBlockingTCPTests(ThreadedTCPSocketTest): def __init__(self, methodName='runTest'): @@ -4882,6 +5022,7 @@ def _testSetBlocking(self): pass @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def testSetBlocking_overflow(self): # Issue 15989 import _testcapi @@ -4994,6 +5135,39 @@ def _testRecv(self): # send data: recv() will no longer block self.cli.sendall(MSG) + def testLargeTimeout(self): + # gh-126876: Check that a timeout larger than INT_MAX is replaced with + # INT_MAX in the poll() code path. The following assertion must not + # fail: assert(INT_MIN <= ms && ms <= INT_MAX). + if _testcapi is not None: + large_timeout = _testcapi.INT_MAX + 1 + else: + large_timeout = 2147483648 + + # test recv() with large timeout + conn, addr = self.serv.accept() + self.addCleanup(conn.close) + try: + conn.settimeout(large_timeout) + except OverflowError: + # On Windows, settimeout() fails with OverflowError, whereas + # we want to test recv(). Just give up silently. + return + msg = conn.recv(len(MSG)) + + def _testLargeTimeout(self): + # test sendall() with large timeout + if _testcapi is not None: + large_timeout = _testcapi.INT_MAX + 1 + else: + large_timeout = 2147483648 + self.cli.connect((HOST, self.port)) + try: + self.cli.settimeout(large_timeout) + except OverflowError: + return + self.cli.sendall(MSG) + class FileObjectClassTestCase(SocketConnectedTest): """Unit tests for the object returned by socket.makefile() @@ -5196,6 +5370,8 @@ def _testMakefileClose(self): self.write_file.write(self.write_msg) self.write_file.flush() + @unittest.skipUnless(hasattr(sys, 'getrefcount'), + 'test needs sys.getrefcount()') def testMakefileCloseSocketDestroy(self): refcount_before = sys.getrefcount(self.cli_conn) self.read_file.close() @@ -5634,7 +5810,7 @@ def test_setblocking_invalidfd(self): sock.setblocking(False) -@unittest.skipUnless(sys.platform == 'linux', 'Linux specific test') +@unittest.skipUnless(sys.platform in ('linux', 'android'), 'Linux specific test') class TestLinuxAbstractNamespace(unittest.TestCase): UNIX_PATH_MAX = 108 @@ -5759,7 +5935,8 @@ def testUnencodableAddr(self): self.addCleanup(os_helper.unlink, path) self.assertEqual(self.sock.getsockname(), path) - @unittest.skipIf(sys.platform == 'linux', 'Linux specific test') + @unittest.skipIf(sys.platform in ('linux', 'android'), + 'Linux behavior is tested by TestLinuxAbstractNamespace') def testEmptyAddress(self): # Test that binding empty address fails. self.assertRaises(OSError, self.sock.bind, "") @@ -6693,6 +6870,28 @@ class TestMacOSTCPFlags(unittest.TestCase): def test_tcp_keepalive(self): self.assertTrue(socket.TCP_KEEPALIVE) +@unittest.skipUnless(hasattr(socket, 'TCP_QUICKACK'), 'need socket.TCP_QUICKACK') +class TestQuickackFlag(unittest.TestCase): + def check_set_quickack(self, sock): + # quickack already true by default on some OS distributions + opt = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK) + if opt: + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK, 0) + + opt = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK) + self.assertFalse(opt) + + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK, 1) + + opt = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK) + self.assertTrue(opt) + + def test_set_quickack(self): + sock = socket.socket(family=socket.AF_INET, type=socket.SOCK_STREAM, + proto=socket.IPPROTO_TCP) + with sock: + self.check_set_quickack(sock) + @unittest.skipUnless(sys.platform.startswith("win"), "requires Windows") class TestMSWindowsTCPFlags(unittest.TestCase): @@ -6706,7 +6905,9 @@ class TestMSWindowsTCPFlags(unittest.TestCase): 'TCP_KEEPCNT', # available starting with Windows 10 1709 'TCP_KEEPIDLE', - 'TCP_KEEPINTVL' + 'TCP_KEEPINTVL', + # available starting with Windows 7 / Server 2008 R2 + 'TCP_QUICKACK', } def test_new_tcp_flags(self): @@ -6874,6 +7075,26 @@ def close_fds(fds): self.assertEqual(data, str(index).encode()) +class FreeThreadingTests(unittest.TestCase): + + def test_close_detach_race(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + def close(): + for _ in range(1000): + s.close() + + def detach(): + for _ in range(1000): + s.detach() + + t1 = threading.Thread(target=close) + t2 = threading.Thread(target=detach) + + with threading_helper.start_threads([t1, t2]): + pass + + def setUpModule(): thread_info = threading_helper.threading_setup() unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info) diff --git a/Lib/test/test_sort.py b/Lib/test/test_sort.py index 3b6ad4d17b0416..2a7cfb7affaa21 100644 --- a/Lib/test/test_sort.py +++ b/Lib/test/test_sort.py @@ -128,6 +128,27 @@ def bad_key(x): x = [e for e, i in augmented] # a stable sort of s check("stability", x, s) + def test_small_stability(self): + from itertools import product + from operator import itemgetter + + # Exhaustively test stability across all lists of small lengths + # and only a few distinct elements. + # This can provoke edge cases that randomization is unlikely to find. + # But it can grow very expensive quickly, so don't overdo it. + NELTS = 3 + MAXSIZE = 9 + + pick0 = itemgetter(0) + for length in range(MAXSIZE + 1): + # There are NELTS ** length distinct lists. + for t in product(range(NELTS), repeat=length): + xs = list(zip(t, range(length))) + # Stability forced by index in each element. + forced = sorted(xs) + # Use key= to hide the index from compares. + native = sorted(xs, key=pick0) + self.assertEqual(forced, native) #============================================================================== class TestBugs(unittest.TestCase): diff --git a/Lib/test/test_sqlite3/test_cli.py b/Lib/test/test_sqlite3/test_cli.py index 303f9e03b5383f..dcd90d11d46819 100644 --- a/Lib/test/test_sqlite3/test_cli.py +++ b/Lib/test/test_sqlite3/test_cli.py @@ -34,7 +34,9 @@ def expect_failure(self, *args): def test_cli_help(self): out = self.expect_success("-h") - self.assertIn("usage: python -m sqlite3", out) + self.assertIn("usage: ", out) + self.assertIn(" [-h] [-v] [filename] [sql]", out) + self.assertIn("Python sqlite3 CLI", out) def test_cli_version(self): out = self.expect_success("-v") @@ -88,14 +90,14 @@ def test_interact(self): out, err = self.run_cli() self.assertIn(self.MEMORY_DB_MSG, err) self.assertIn(self.MEMORY_DB_MSG, err) - self.assertTrue(out.endswith(self.PS1)) + self.assertEndsWith(out, self.PS1) self.assertEqual(out.count(self.PS1), 1) self.assertEqual(out.count(self.PS2), 0) def test_interact_quit(self): out, err = self.run_cli(commands=(".quit",)) self.assertIn(self.MEMORY_DB_MSG, err) - self.assertTrue(out.endswith(self.PS1)) + self.assertEndsWith(out, self.PS1) self.assertEqual(out.count(self.PS1), 1) self.assertEqual(out.count(self.PS2), 0) @@ -103,7 +105,7 @@ def test_interact_version(self): out, err = self.run_cli(commands=(".version",)) self.assertIn(self.MEMORY_DB_MSG, err) self.assertIn(sqlite3.sqlite_version + "\n", out) - self.assertTrue(out.endswith(self.PS1)) + self.assertEndsWith(out, self.PS1) self.assertEqual(out.count(self.PS1), 2) self.assertEqual(out.count(self.PS2), 0) self.assertIn(sqlite3.sqlite_version, out) @@ -112,14 +114,14 @@ def test_interact_valid_sql(self): out, err = self.run_cli(commands=("SELECT 1;",)) self.assertIn(self.MEMORY_DB_MSG, err) self.assertIn("(1,)\n", out) - self.assertTrue(out.endswith(self.PS1)) + self.assertEndsWith(out, self.PS1) self.assertEqual(out.count(self.PS1), 2) self.assertEqual(out.count(self.PS2), 0) def test_interact_incomplete_multiline_sql(self): out, err = self.run_cli(commands=("SELECT 1",)) self.assertIn(self.MEMORY_DB_MSG, err) - self.assertTrue(out.endswith(self.PS2)) + self.assertEndsWith(out, self.PS2) self.assertEqual(out.count(self.PS1), 1) self.assertEqual(out.count(self.PS2), 1) @@ -128,7 +130,7 @@ def test_interact_valid_multiline_sql(self): self.assertIn(self.MEMORY_DB_MSG, err) self.assertIn(self.PS2, out) self.assertIn("(1,)\n", out) - self.assertTrue(out.endswith(self.PS1)) + self.assertEndsWith(out, self.PS1) self.assertEqual(out.count(self.PS1), 2) self.assertEqual(out.count(self.PS2), 1) @@ -136,7 +138,7 @@ def test_interact_invalid_sql(self): out, err = self.run_cli(commands=("sel;",)) self.assertIn(self.MEMORY_DB_MSG, err) self.assertIn("OperationalError (SQLITE_ERROR)", err) - self.assertTrue(out.endswith(self.PS1)) + self.assertEndsWith(out, self.PS1) self.assertEqual(out.count(self.PS1), 2) self.assertEqual(out.count(self.PS2), 0) @@ -145,7 +147,7 @@ def test_interact_on_disk_file(self): out, err = self.run_cli(TESTFN, commands=("CREATE TABLE t(t);",)) self.assertIn(TESTFN, err) - self.assertTrue(out.endswith(self.PS1)) + self.assertEndsWith(out, self.PS1) out, _ = self.run_cli(TESTFN, commands=("SELECT count(t) FROM t;",)) self.assertIn("(0,)\n", out) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 588272448bbfda..f5ffe2427430e2 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -28,14 +28,14 @@ import threading import unittest import urllib.parse +import warnings from test.support import ( SHORT_TIMEOUT, check_disallow_instantiation, requires_subprocess, is_apple, is_emscripten, is_wasi ) from test.support import gc_collect -from test.support import threading_helper -from _testcapi import INT_MAX, ULLONG_MAX +from test.support import threading_helper, import_helper from os import SEEK_SET, SEEK_CUR, SEEK_END from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE, unlink, temp_dir, FakePath @@ -48,17 +48,6 @@ def test_api_level(self): self.assertEqual(sqlite.apilevel, "2.0", "apilevel is %s, should be 2.0" % sqlite.apilevel) - def test_deprecated_version(self): - msg = "deprecated and will be removed in Python 3.14" - for attr in "version", "version_info": - with self.subTest(attr=attr): - with self.assertWarnsRegex(DeprecationWarning, msg) as cm: - getattr(sqlite, attr) - self.assertEqual(cm.filename, __file__) - with self.assertWarnsRegex(DeprecationWarning, msg) as cm: - getattr(sqlite.dbapi2, attr) - self.assertEqual(cm.filename, __file__) - def test_thread_safety(self): self.assertIn(sqlite.threadsafety, {0, 1, 3}, "threadsafety is %d, should be 0, 1 or 3" % @@ -70,45 +59,34 @@ def test_param_style(self): sqlite.paramstyle) def test_warning(self): - self.assertTrue(issubclass(sqlite.Warning, Exception), - "Warning is not a subclass of Exception") + self.assertIsSubclass(sqlite.Warning, Exception) def test_error(self): - self.assertTrue(issubclass(sqlite.Error, Exception), - "Error is not a subclass of Exception") + self.assertIsSubclass(sqlite.Error, Exception) def test_interface_error(self): - self.assertTrue(issubclass(sqlite.InterfaceError, sqlite.Error), - "InterfaceError is not a subclass of Error") + self.assertIsSubclass(sqlite.InterfaceError, sqlite.Error) def test_database_error(self): - self.assertTrue(issubclass(sqlite.DatabaseError, sqlite.Error), - "DatabaseError is not a subclass of Error") + self.assertIsSubclass(sqlite.DatabaseError, sqlite.Error) def test_data_error(self): - self.assertTrue(issubclass(sqlite.DataError, sqlite.DatabaseError), - "DataError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.DataError, sqlite.DatabaseError) def test_operational_error(self): - self.assertTrue(issubclass(sqlite.OperationalError, sqlite.DatabaseError), - "OperationalError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.OperationalError, sqlite.DatabaseError) def test_integrity_error(self): - self.assertTrue(issubclass(sqlite.IntegrityError, sqlite.DatabaseError), - "IntegrityError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.IntegrityError, sqlite.DatabaseError) def test_internal_error(self): - self.assertTrue(issubclass(sqlite.InternalError, sqlite.DatabaseError), - "InternalError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.InternalError, sqlite.DatabaseError) def test_programming_error(self): - self.assertTrue(issubclass(sqlite.ProgrammingError, sqlite.DatabaseError), - "ProgrammingError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.ProgrammingError, sqlite.DatabaseError) def test_not_supported_error(self): - self.assertTrue(issubclass(sqlite.NotSupportedError, - sqlite.DatabaseError), - "NotSupportedError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.NotSupportedError, sqlite.DatabaseError) def test_module_constants(self): consts = [ @@ -285,7 +263,7 @@ def test_module_constants(self): consts.append("SQLITE_IOERR_CORRUPTFS") for const in consts: with self.subTest(const=const): - self.assertTrue(hasattr(sqlite, const)) + self.assertHasAttr(sqlite, const) def test_error_code_on_exception(self): err_msg = "unable to open database file" @@ -299,7 +277,7 @@ def test_error_code_on_exception(self): sqlite.connect(db) e = cm.exception self.assertEqual(e.sqlite_errorcode, err_code) - self.assertTrue(e.sqlite_errorname.startswith("SQLITE_CANTOPEN")) + self.assertStartsWith(e.sqlite_errorname, "SQLITE_CANTOPEN") def test_extended_error_code_on_exception(self): with memory_database() as con: @@ -436,7 +414,7 @@ def test_connection_exceptions(self): ] for exc in exceptions: with self.subTest(exc=exc): - self.assertTrue(hasattr(self.cx, exc)) + self.assertHasAttr(self.cx, exc) self.assertIs(getattr(sqlite, exc), getattr(self.cx, exc)) def test_interrupt_on_closed_db(self): @@ -590,6 +568,11 @@ def test_connection_resource_warning(self): del cx gc_collect() + def test_connection_signature(self): + from inspect import signature + sig = signature(self.cx) + self.assertEqual(str(sig), "(sql, /)") + class UninitialisedConnectionTests(unittest.TestCase): def setUp(self): @@ -884,9 +867,21 @@ def test_execute_named_param_and_sequence(self): msg = "Binding.*is a named parameter" for query, params in dataset: with self.subTest(query=query, params=params): - with self.assertWarnsRegex(DeprecationWarning, msg) as cm: + with self.assertRaisesRegex(sqlite.ProgrammingError, msg) as cm: self.cu.execute(query, params) - self.assertEqual(cm.filename, __file__) + + def test_execute_indexed_nameless_params(self): + # See gh-117995: "'?1' is considered a named placeholder" + for query, params, expected in ( + ("select ?1, ?2", (1, 2), (1, 2)), + ("select ?2, ?1", (1, 2), (2, 1)), + ): + with self.subTest(query=query, params=params): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + cu = self.cu.execute(query, params) + actual, = cu.fetchall() + self.assertEqual(actual, expected) def test_execute_too_many_params(self): category = sqlite.SQLITE_LIMIT_VARIABLE_NUMBER @@ -1202,7 +1197,6 @@ def test_blob_seek_and_tell(self): def test_blob_seek_error(self): msg_oor = "offset out of blob range" msg_orig = "'origin' should be os.SEEK_SET, os.SEEK_CUR, or os.SEEK_END" - msg_of = "seek offset results in overflow" dataset = ( (ValueError, msg_oor, lambda: self.blob.seek(1000)), @@ -1214,12 +1208,15 @@ def test_blob_seek_error(self): with self.subTest(exc=exc, msg=msg, fn=fn): self.assertRaisesRegex(exc, msg, fn) + def test_blob_seek_overflow_error(self): # Force overflow errors + msg_of = "seek offset results in overflow" + _testcapi = import_helper.import_module("_testcapi") self.blob.seek(1, SEEK_SET) with self.assertRaisesRegex(OverflowError, msg_of): - self.blob.seek(INT_MAX, SEEK_CUR) + self.blob.seek(_testcapi.INT_MAX, SEEK_CUR) with self.assertRaisesRegex(OverflowError, msg_of): - self.blob.seek(INT_MAX, SEEK_END) + self.blob.seek(_testcapi.INT_MAX, SEEK_END) def test_blob_read(self): buf = self.blob.read() @@ -1379,14 +1376,17 @@ def test_blob_get_item_error(self): with self.subTest(idx=idx): with self.assertRaisesRegex(IndexError, "index out of range"): self.blob[idx] - with self.assertRaisesRegex(IndexError, "cannot fit 'int'"): - self.blob[ULLONG_MAX] # Provoke read error self.cx.execute("update test set b='aaaa' where rowid=1") with self.assertRaises(sqlite.OperationalError): self.blob[0] + def test_blob_get_item_error_bigint(self): + _testcapi = import_helper.import_module("_testcapi") + with self.assertRaisesRegex(IndexError, "cannot fit 'int'"): + self.blob[_testcapi.ULLONG_MAX] + def test_blob_set_item_error(self): with self.assertRaisesRegex(TypeError, "cannot be interpreted"): self.blob[0] = b"multiple" @@ -1423,7 +1423,7 @@ def test_blob_sequence_not_supported(self): self.blob + self.blob with self.assertRaisesRegex(TypeError, "unsupported operand"): self.blob * 5 - with self.assertRaisesRegex(TypeError, "is not iterable"): + with self.assertRaisesRegex(TypeError, "is not.+iterable"): b"a" in self.blob def test_blob_context_manager(self): diff --git a/Lib/test/test_sqlite3/test_dump.py b/Lib/test/test_sqlite3/test_dump.py index 7261b7f0dc93d0..550cea41976441 100644 --- a/Lib/test/test_sqlite3/test_dump.py +++ b/Lib/test/test_sqlite3/test_dump.py @@ -10,6 +10,7 @@ class DumpTests(MemoryDatabaseMixin, unittest.TestCase): def test_table_dump(self): expected_sqls = [ + "PRAGMA foreign_keys=OFF;", """CREATE TABLE "index"("index" blob);""" , """INSERT INTO "index" VALUES(X'01');""" @@ -48,7 +49,7 @@ def test_table_dump(self): expected_sqls = [ "PRAGMA foreign_keys=OFF;", "BEGIN TRANSACTION;", - *expected_sqls, + *expected_sqls[1:], "COMMIT;", ] [self.assertEqual(expected_sqls[i], actual_sqls[i]) @@ -190,6 +191,21 @@ def __getitem__(self, index): got = list(self.cx.iterdump()) self.assertEqual(expected, got) + def test_dump_custom_row_factory(self): + # gh-118221: iterdump should be able to cope with custom row factories. + def dict_factory(cu, row): + fields = [col[0] for col in cu.description] + return dict(zip(fields, row)) + + self.cx.row_factory = dict_factory + CREATE_TABLE = "CREATE TABLE test(t);" + expected = ["BEGIN TRANSACTION;", CREATE_TABLE, "COMMIT;"] + + self.cu.execute(CREATE_TABLE) + actual = list(self.cx.iterdump()) + self.assertEqual(expected, actual) + self.assertEqual(self.cx.row_factory, dict_factory) + def test_dump_virtual_tables(self): # gh-64662 expected = [ diff --git a/Lib/test/test_sqlite3/test_factory.py b/Lib/test/test_sqlite3/test_factory.py index 48d35b54a2e239..cc9f1ec5c4bec5 100644 --- a/Lib/test/test_sqlite3/test_factory.py +++ b/Lib/test/test_sqlite3/test_factory.py @@ -280,7 +280,7 @@ def test_custom(self): austria = "Österreich" row = self.con.execute("select ?", (austria,)).fetchone() self.assertEqual(type(row[0]), str, "type of row[0] must be unicode") - self.assertTrue(row[0].endswith("reich"), "column must contain original data") + self.assertEndsWith(row[0], "reich", "column must contain original data") class TextFactoryTestsWithEmbeddedZeroBytes(unittest.TestCase): diff --git a/Lib/test/test_sqlite3/test_regression.py b/Lib/test/test_sqlite3/test_regression.py index db4e13222da9da..50cced3891d13a 100644 --- a/Lib/test/test_sqlite3/test_regression.py +++ b/Lib/test/test_sqlite3/test_regression.py @@ -433,6 +433,7 @@ def test_table_lock_cursor_dealloc(self): con.commit() cur = con.execute("select t from t") del cur + support.gc_collect() con.execute("drop table t") con.commit() @@ -448,6 +449,7 @@ def dup(v): con.create_function("dup", 1, dup) cur = con.execute("select dup(t) from t") del cur + support.gc_collect() con.execute("drop table t") con.commit() diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 1b18230d83577d..9863f3ffe97656 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -3,6 +3,8 @@ import sys import unittest import unittest.mock +from ast import literal_eval +from threading import Thread from test import support from test.support import import_helper from test.support import os_helper @@ -38,7 +40,7 @@ ssl = import_helper.import_module("ssl") import _ssl -from ssl import TLSVersion, _TLSContentType, _TLSMessageType, _TLSAlertType +from ssl import Purpose, TLSVersion, _TLSContentType, _TLSMessageType, _TLSAlertType Py_DEBUG_WIN32 = support.Py_DEBUG and sys.platform == 'win32' @@ -82,52 +84,32 @@ def data_file(*name): CAFILE_NEURONIO = data_file("capath", "4e1295a3.0") CAFILE_CACERT = data_file("capath", "5ed36f99.0") -CERTFILE_INFO = { - 'issuer': ((('countryName', 'XY'),), - (('localityName', 'Castle Anthrax'),), - (('organizationName', 'Python Software Foundation'),), - (('commonName', 'localhost'),)), - 'notAfter': 'Aug 26 14:23:15 2028 GMT', - 'notBefore': 'Aug 29 14:23:15 2018 GMT', - 'serialNumber': '98A7CF88C74A32ED', - 'subject': ((('countryName', 'XY'),), - (('localityName', 'Castle Anthrax'),), - (('organizationName', 'Python Software Foundation'),), - (('commonName', 'localhost'),)), - 'subjectAltName': (('DNS', 'localhost'),), - 'version': 3 -} +with open(data_file('keycert.pem.reference')) as file: + CERTFILE_INFO = literal_eval(file.read()) # empty CRL CRLFILE = data_file("revocation.crl") # Two keys and certs signed by the same CA (for SNI tests) SIGNED_CERTFILE = data_file("keycert3.pem") +SINGED_CERTFILE_ONLY = data_file("cert3.pem") SIGNED_CERTFILE_HOSTNAME = 'localhost' -SIGNED_CERTFILE_INFO = { - 'OCSP': ('http://testca.pythontest.net/testca/ocsp/',), - 'caIssuers': ('http://testca.pythontest.net/testca/pycacert.cer',), - 'crlDistributionPoints': ('http://testca.pythontest.net/testca/revocation.crl',), - 'issuer': ((('countryName', 'XY'),), - (('organizationName', 'Python Software Foundation CA'),), - (('commonName', 'our-ca-server'),)), - 'notAfter': 'Oct 28 14:23:16 2037 GMT', - 'notBefore': 'Aug 29 14:23:16 2018 GMT', - 'serialNumber': 'CB2D80995A69525C', - 'subject': ((('countryName', 'XY'),), - (('localityName', 'Castle Anthrax'),), - (('organizationName', 'Python Software Foundation'),), - (('commonName', 'localhost'),)), - 'subjectAltName': (('DNS', 'localhost'),), - 'version': 3 -} +with open(data_file('keycert3.pem.reference')) as file: + SIGNED_CERTFILE_INFO = literal_eval(file.read()) SIGNED_CERTFILE2 = data_file("keycert4.pem") SIGNED_CERTFILE2_HOSTNAME = 'fakehostname' SIGNED_CERTFILE_ECC = data_file("keycertecc.pem") SIGNED_CERTFILE_ECC_HOSTNAME = 'localhost-ecc' +# A custom testcase, extracted from `rfc5280::aki::leaf-missing-aki` in x509-limbo: +# The leaf (server) certificate has no AKI, which is forbidden under RFC 5280. +# See: https://x509-limbo.com/testcases/rfc5280/#rfc5280akileaf-missing-aki +LEAF_MISSING_AKI_CERTFILE = data_file("leaf-missing-aki.keycert.pem") +LEAF_MISSING_AKI_CERTFILE_HOSTNAME = "example.com" +LEAF_MISSING_AKI_CA = data_file("leaf-missing-aki.ca.pem") + # Same certificate as pycacert.pem, but without extra text in file SIGNING_CA = data_file("capath", "ceff1710.0") # cert with all kinds of subject alt names @@ -169,7 +151,7 @@ def is_ubuntu(): if is_ubuntu(): def seclevel_workaround(*ctxs): - """"Lower security level to '1' and allow all ciphers for TLS 1.0/1""" + """Lower security level to '1' and allow all ciphers for TLS 1.0/1""" for ctx in ctxs: if ( hasattr(ctx, "minimum_version") and @@ -296,11 +278,19 @@ def test_wrap_socket(sock, *, return context.wrap_socket(sock, **kwargs) +USE_SAME_TEST_CONTEXT = False +_TEST_CONTEXT = None + def testing_context(server_cert=SIGNED_CERTFILE, *, server_chain=True): """Create context client_context, server_context, hostname = testing_context() """ + global _TEST_CONTEXT + if USE_SAME_TEST_CONTEXT: + if _TEST_CONTEXT is not None: + return _TEST_CONTEXT + if server_cert == SIGNED_CERTFILE: hostname = SIGNED_CERTFILE_HOSTNAME elif server_cert == SIGNED_CERTFILE2: @@ -318,6 +308,10 @@ def testing_context(server_cert=SIGNED_CERTFILE, *, server_chain=True): if server_chain: server_context.load_verify_locations(SIGNING_CA) + if USE_SAME_TEST_CONTEXT: + if _TEST_CONTEXT is not None: + _TEST_CONTEXT = client_context, server_context, hostname + return client_context, server_context, hostname @@ -402,6 +396,7 @@ def test_random(self): ssl.RAND_add(bytearray(b"this is a random bytearray object"), 75.0) def test_parse_cert(self): + self.maxDiff = None # note that this uses an 'unofficial' function in _ssl.c, # provided solely for this test, to exercise the certificate # parsing code @@ -544,7 +539,7 @@ def test_openssl_version(self): else: openssl_ver = f"OpenSSL {major:d}.{minor:d}.{fix:d}" self.assertTrue( - s.startswith((openssl_ver, libressl_ver)), + s.startswith((openssl_ver, libressl_ver, "AWS-LC")), (s, t, hex(n)) ) @@ -1162,24 +1157,30 @@ def test_load_cert_chain(self): with self.assertRaises(OSError) as cm: ctx.load_cert_chain(NONEXISTINGCERT) self.assertEqual(cm.exception.errno, errno.ENOENT) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_cert_chain(BADCERT) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_cert_chain(EMPTYCERT) # Separate key and cert ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) ctx.load_cert_chain(ONLYCERT, ONLYKEY) ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) ctx.load_cert_chain(certfile=BYTES_ONLYCERT, keyfile=BYTES_ONLYKEY) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_cert_chain(ONLYCERT) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_cert_chain(ONLYKEY) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_cert_chain(certfile=ONLYKEY, keyfile=ONLYCERT) # Mismatching key and cert ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - with self.assertRaisesRegex(ssl.SSLError, "key values mismatch"): + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + key values mismatch # OpenSSL + | + KEY_VALUES_MISMATCH # AWS-LC + )""", re.X) + with self.assertRaisesRegex(ssl.SSLError, regex): ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY) # Password protected key and cert ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD) @@ -1247,7 +1248,7 @@ def test_load_verify_locations(self): with self.assertRaises(OSError) as cm: ctx.load_verify_locations(NONEXISTINGCERT) self.assertEqual(cm.exception.errno, errno.ENOENT) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_verify_locations(BADCERT) ctx.load_verify_locations(CERTFILE, CAPATH) ctx.load_verify_locations(CERTFILE, capath=BYTES_CAPATH) @@ -1324,8 +1325,7 @@ def test_load_verify_cadata(self): def test_load_dh_params(self): ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) ctx.load_dh_params(DHFILE) - if os.name != 'nt': - ctx.load_dh_params(BYTES_DHFILE) + ctx.load_dh_params(BYTES_DHFILE) self.assertRaises(TypeError, ctx.load_dh_params) self.assertRaises(TypeError, ctx.load_dh_params, None) with self.assertRaises(FileNotFoundError) as cm: @@ -1497,6 +1497,10 @@ def test_create_default_context(self): self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS_CLIENT) self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertEqual(ctx.verify_flags & ssl.VERIFY_X509_PARTIAL_CHAIN, + ssl.VERIFY_X509_PARTIAL_CHAIN) + self.assertEqual(ctx.verify_flags & ssl.VERIFY_X509_STRICT, + ssl.VERIFY_X509_STRICT) self.assertTrue(ctx.check_hostname) self._assert_context_options(ctx) @@ -1651,9 +1655,10 @@ def test_lib_reason(self): with self.assertRaises(ssl.SSLError) as cm: ctx.load_dh_params(CERTFILE) self.assertEqual(cm.exception.library, 'PEM') - self.assertEqual(cm.exception.reason, 'NO_START_LINE') + regex = "(NO_START_LINE|UNSUPPORTED_PUBLIC_KEY_TYPE)" + self.assertRegex(cm.exception.reason, regex) s = str(cm.exception) - self.assertTrue(s.startswith("[PEM: NO_START_LINE] no start line"), s) + self.assertTrue("NO_START_LINE" in s, s) def test_subclass(self): # Check that the appropriate SSLError subclass is raised @@ -1833,7 +1838,13 @@ def test_connect_fail(self): s = test_wrap_socket(socket.socket(socket.AF_INET), cert_reqs=ssl.CERT_REQUIRED) self.addCleanup(s.close) - self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + self.assertRaisesRegex(ssl.SSLError, regex, s.connect, self.server_addr) def test_connect_ex(self): @@ -1901,7 +1912,13 @@ def test_connect_with_context_fail(self): server_hostname=SIGNED_CERTFILE_HOSTNAME ) self.addCleanup(s.close) - self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + self.assertRaisesRegex(ssl.SSLError, regex, s.connect, self.server_addr) def test_connect_capath(self): @@ -2118,14 +2135,16 @@ def test_bio_handshake(self): self.assertIsNone(sslobj.version()) self.assertIsNone(sslobj.shared_ciphers()) self.assertRaises(ValueError, sslobj.getpeercert) - if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + # tls-unique is not defined for TLSv1.3 + # https://datatracker.ietf.org/doc/html/rfc8446#appendix-C.5 + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES and sslobj.version() != "TLSv1.3": self.assertIsNone(sslobj.get_channel_binding('tls-unique')) self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) self.assertTrue(sslobj.cipher()) self.assertIsNone(sslobj.shared_ciphers()) self.assertIsNotNone(sslobj.version()) self.assertTrue(sslobj.getpeercert()) - if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES and sslobj.version() != "TLSv1.3": self.assertTrue(sslobj.get_channel_binding('tls-unique')) try: self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) @@ -2279,7 +2298,6 @@ def wrap_conn(self): # See also http://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/ if e.errno != errno.EPROTOTYPE and sys.platform != "darwin": self.running = False - self.server.stop() self.close() return False else: @@ -2397,16 +2415,18 @@ def run(self): self.write(msg.lower()) except OSError as e: # handles SSLError and socket errors + if isinstance(e, ConnectionError): + # OpenSSL 1.1.1 sometimes raises + # ConnectionResetError when connection is not + # shut down gracefully. + if self.server.chatty and support.verbose: + print(f" Connection reset by peer: {self.addr}") + + self.close() + self.running = False + return if self.server.chatty and support.verbose: - if isinstance(e, ConnectionError): - # OpenSSL 1.1.1 sometimes raises - # ConnectionResetError when connection is not - # shut down gracefully. - print( - f" Connection reset by peer: {self.addr}" - ) - else: - handle_error("Test server failure:\n") + handle_error("Test server failure:\n") try: self.write(b"ERROR\n") except OSError: @@ -2414,10 +2434,6 @@ def run(self): self.close() self.running = False - # normally, we'd just stop here, but for the test - # harness, we want to stop the server - self.server.stop() - def __init__(self, certificate=None, ssl_version=None, certreqs=None, cacerts=None, chatty=True, connectionchatty=False, starttls_server=False, @@ -2451,21 +2467,33 @@ def __init__(self, certificate=None, ssl_version=None, self.conn_errors = [] threading.Thread.__init__(self) self.daemon = True + self._in_context = False def __enter__(self): + if self._in_context: + raise ValueError('Re-entering ThreadedEchoServer context') + self._in_context = True self.start(threading.Event()) self.flag.wait() return self def __exit__(self, *args): + assert self._in_context + self._in_context = False self.stop() self.join() def start(self, flag=None): + if not self._in_context: + raise ValueError( + 'ThreadedEchoServer must be used as a context manager') self.flag = flag threading.Thread.start(self) def run(self): + if not self._in_context: + raise ValueError( + 'ThreadedEchoServer must be used as a context manager') self.sock.settimeout(1.0) self.sock.listen(5) self.active = True @@ -2792,6 +2820,44 @@ def test_echo(self): 'Cannot create a client socket with a PROTOCOL_TLS_SERVER context', str(e.exception)) + @unittest.skipUnless(support.Py_GIL_DISABLED, "test is only useful if the GIL is disabled") + def test_ssl_in_multiple_threads(self): + # See GH-124984: OpenSSL is not thread safe. + threads = [] + + global USE_SAME_TEST_CONTEXT + USE_SAME_TEST_CONTEXT = True + try: + for func in ( + self.test_echo, + self.test_alpn_protocols, + self.test_getpeercert, + self.test_crl_check, + self.test_check_hostname_idn, + self.test_wrong_cert_tls12, + self.test_wrong_cert_tls13, + ): + # Be careful with the number of threads here. + # Too many can result in failing tests. + for num in range(5): + with self.subTest(func=func, num=num): + threads.append(Thread(target=func)) + + with threading_helper.catch_threading_exception() as cm: + for thread in threads: + with self.subTest(thread=thread): + thread.start() + + for thread in threads: + with self.subTest(thread=thread): + thread.join() + if cm.exc_value is not None: + # Some threads can skip their test + if not isinstance(cm.exc_value, unittest.SkipTest): + raise cm.exc_value + finally: + USE_SAME_TEST_CONTEXT = False + def test_getpeercert(self): if support.verbose: sys.stdout.write("\n") @@ -2850,11 +2916,16 @@ def test_crl_check(self): client_context.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF server = ThreadedEchoServer(context=server_context, chatty=True) + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) with server: with client_context.wrap_socket(socket.socket(), server_hostname=hostname) as s: - with self.assertRaisesRegex(ssl.SSLError, - "certificate verify failed"): + with self.assertRaisesRegex(ssl.SSLError, regex): s.connect((HOST, server.port)) # now load a CRL file. The CRL file is signed by the CA. @@ -2885,12 +2956,16 @@ def test_check_hostname(self): # incorrect hostname should raise an exception server = ThreadedEchoServer(context=server_context, chatty=True) + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) with server: with client_context.wrap_socket(socket.socket(), server_hostname="invalid") as s: - with self.assertRaisesRegex( - ssl.CertificateError, - "Hostname mismatch, certificate is not valid for 'invalid'."): + with self.assertRaisesRegex(ssl.CertificateError, regex): s.connect((HOST, server.port)) # missing server_hostname arg should cause an exception, too @@ -2946,6 +3021,38 @@ def test_ecc_cert(self): cipher = s.cipher()[0].split('-') self.assertTrue(cipher[:2], ('ECDHE', 'ECDSA')) + @unittest.skipUnless(IS_OPENSSL_3_0_0, + "test requires RFC 5280 check added in OpenSSL 3.0+") + def test_verify_strict(self): + # verification fails by default, since the server cert is non-conforming + client_context = ssl.create_default_context() + client_context.load_verify_locations(LEAF_MISSING_AKI_CA) + hostname = LEAF_MISSING_AKI_CERTFILE_HOSTNAME + + server_context = ssl.create_default_context(purpose=Purpose.CLIENT_AUTH) + server_context.load_cert_chain(LEAF_MISSING_AKI_CERTFILE) + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + with self.assertRaises(ssl.SSLError): + s.connect((HOST, server.port)) + + # explicitly disabling VERIFY_X509_STRICT allows it to succeed + client_context = ssl.create_default_context() + client_context.load_verify_locations(LEAF_MISSING_AKI_CA) + client_context.verify_flags &= ~ssl.VERIFY_X509_STRICT + + server_context = ssl.create_default_context(purpose=Purpose.CLIENT_AUTH) + server_context.load_cert_chain(LEAF_MISSING_AKI_CERTFILE) + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + def test_dual_rsa_ecc(self): client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) client_context.load_verify_locations(SIGNING_CA) @@ -3093,8 +3200,10 @@ def test_wrong_cert_tls13(self): suppress_ragged_eofs=False) as s: s.connect((HOST, server.port)) with self.assertRaisesRegex( - ssl.SSLError, - 'alert unknown ca|EOF occurred' + OSError, + 'alert unknown ca|EOF occurred|TLSV1_ALERT_UNKNOWN_CA|' + 'closed by the remote host|Connection reset by peer|' + 'Broken pipe' ): # TLS 1.3 perform client cert exchange after handshake s.write(b'data') @@ -3158,13 +3267,21 @@ def test_ssl_cert_verify_error(self): server_hostname=SIGNED_CERTFILE_HOSTNAME) as s: try: s.connect((HOST, server.port)) + self.fail("Expected connection failure") except ssl.SSLError as e: msg = 'unable to get local issuer certificate' self.assertIsInstance(e, ssl.SSLCertVerificationError) self.assertEqual(e.verify_code, 20) self.assertEqual(e.verify_message, msg) - self.assertIn(msg, repr(e)) - self.assertIn('certificate verify failed', repr(e)) + # Allow for flexible libssl error messages. + regex = f"({msg}|CERTIFICATE_VERIFY_FAILED)" + self.assertRegex(repr(e), regex) + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + self.assertRegex(repr(e), regex) def test_PROTOCOL_TLS(self): """Connecting to an SSLv23 server with various client options""" @@ -3696,7 +3813,7 @@ def test_no_shared_ciphers(self): server_hostname=hostname) as s: with self.assertRaises(OSError): s.connect((HOST, server.port)) - self.assertIn("no shared cipher", server.conn_errors[0]) + self.assertIn("NO_SHARED_CIPHER", server.conn_errors[0]) def test_version_basic(self): """ @@ -3784,7 +3901,7 @@ def test_min_max_version_mismatch(self): server_hostname=hostname) as s: with self.assertRaises(ssl.SSLError) as e: s.connect((HOST, server.port)) - self.assertIn("alert", str(e.exception)) + self.assertRegex(str(e.exception), "(alert|ALERT)") @requires_tls_version('SSLv3') def test_min_max_version_sslv3(self): @@ -3826,6 +3943,10 @@ def test_tls_unique_channel_binding(self): client_context, server_context, hostname = testing_context() + # tls-unique is not defined for TLSv1.3 + # https://datatracker.ietf.org/doc/html/rfc8446#appendix-C.5 + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + server = ThreadedEchoServer(context=server_context, chatty=True, connectionchatty=False) @@ -3926,7 +4047,7 @@ def test_dh_params(self): cipher = stats["cipher"][0] parts = cipher.split("-") if "ADH" not in parts and "EDH" not in parts and "DHE" not in parts: - self.fail("Non-DH cipher: " + cipher[0]) + self.fail("Non-DH key exchange: " + cipher[0]) def test_ecdh_curve(self): # server secp384r1, client auto @@ -4093,8 +4214,9 @@ def cb_raising(ssl_sock, server_name, initial_context): chatty=False, sni_name='supermessage') - self.assertEqual(cm.exception.reason, - 'SSLV3_ALERT_HANDSHAKE_FAILURE') + # Allow for flexible libssl error messages. + regex = "(SSLV3_ALERT_HANDSHAKE_FAILURE|NO_PRIVATE_VALUE)" + self.assertRegex(cm.exception.reason, regex) self.assertEqual(catch.unraisable.exc_type, ZeroDivisionError) def test_sni_callback_wrong_return_type(self): @@ -4371,7 +4493,8 @@ def server_callback(identity): s.connect((HOST, server.port)) -@unittest.skipUnless(has_tls_version('TLSv1_3'), "Test needs TLS 1.3") +@unittest.skipUnless(has_tls_version('TLSv1_3') and ssl.HAS_PHA, + "Test needs TLS 1.3 PHA") class TestPostHandshakeAuth(unittest.TestCase): def test_pha_setter(self): protocols = [ @@ -4446,8 +4569,12 @@ def msg_cb(conn, direction, version, content_type, msg_type, data): # test sometimes fails with EOF error. Test passes as long as # server aborts connection with an error. with self.assertRaisesRegex( - ssl.SSLError, - '(certificate required|EOF occurred)' + OSError, + ('certificate required' + '|EOF occurred' + '|closed by the remote host' + '|Connection reset by peer' + '|Broken pipe') ): # receive CertificateRequest data = s.recv(1024) @@ -4626,6 +4753,40 @@ def test_internal_chain_client(self): ssl.PEM_cert_to_DER_cert(pem), der ) + def test_certificate_chain(self): + client_context, server_context, hostname = testing_context( + server_chain=False + ) + server = ThreadedEchoServer(context=server_context, chatty=False) + + with open(SIGNING_CA) as f: + expected_ca_cert = ssl.PEM_cert_to_DER_cert(f.read()) + + with open(SINGED_CERTFILE_ONLY) as f: + expected_ee_cert = ssl.PEM_cert_to_DER_cert(f.read()) + + with server: + with client_context.wrap_socket( + socket.socket(), + server_hostname=hostname + ) as s: + s.connect((HOST, server.port)) + vc = s.get_verified_chain() + self.assertEqual(len(vc), 2) + + ee, ca = vc + self.assertIsInstance(ee, bytes) + self.assertIsInstance(ca, bytes) + self.assertEqual(expected_ca_cert, ca) + self.assertEqual(expected_ee_cert, ee) + + uvc = s.get_unverified_chain() + self.assertEqual(len(uvc), 1) + self.assertIsInstance(uvc[0], bytes) + + self.assertEqual(ee, uvc[0]) + self.assertNotEqual(ee, ca) + def test_internal_chain_server(self): client_context, server_context, hostname = testing_context() client_context.load_cert_chain(SIGNED_CERTFILE) @@ -4871,11 +5032,11 @@ def run(self): pass # closed, protocol error, etc. def non_linux_skip_if_other_okay_error(self, err): - if sys.platform == "linux": + if sys.platform in ("linux", "android"): return # Expect the full test setup to always work on Linux. if (isinstance(err, ConnectionResetError) or (isinstance(err, OSError) and err.errno == errno.EINVAL) or - re.search('wrong.version.number', getattr(err, "reason", ""), re.I)): + re.search('wrong.version.number', str(getattr(err, "reason", "")), re.I)): # On Windows the TCP RST leads to a ConnectionResetError # (ECONNRESET) which Linux doesn't appear to surface to userspace. # If wrap_socket() winds up on the "if connected:" path and doing diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 8bd373976426ef..f3724ce6d4d15a 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -6,7 +6,10 @@ import sys import unittest from test.support.import_helper import import_module -from _testcapi import get_feature_macros +try: + from _testcapi import get_feature_macros +except ImportError: + raise unittest.SkipTest("requires _testcapi") feature_macros = get_feature_macros() @@ -224,6 +227,9 @@ def test_windows_feature_macros(self): "PyEval_EvalFrameEx", "PyEval_GetBuiltins", "PyEval_GetFrame", + "PyEval_GetFrameBuiltins", + "PyEval_GetFrameGlobals", + "PyEval_GetFrameLocals", "PyEval_GetFuncDesc", "PyEval_GetFuncName", "PyEval_GetGlobals", @@ -261,7 +267,6 @@ def test_windows_feature_macros(self): "PyExc_IOError", "PyExc_ImportError", "PyExc_ImportWarning", - "PyExc_IncompleteInputError", "PyExc_IndentationError", "PyExc_IndexError", "PyExc_InterruptedError", @@ -366,6 +371,7 @@ def test_windows_feature_macros(self): "PyInterpreterState_New", "PyIter_Check", "PyIter_Next", + "PyIter_NextItem", "PyIter_Send", "PyListIter_Type", "PyListRevIter_Type", @@ -385,23 +391,31 @@ def test_windows_feature_macros(self): "PyLongRangeIter_Type", "PyLong_AsDouble", "PyLong_AsInt", + "PyLong_AsInt32", + "PyLong_AsInt64", "PyLong_AsLong", "PyLong_AsLongAndOverflow", "PyLong_AsLongLong", "PyLong_AsLongLongAndOverflow", "PyLong_AsSize_t", "PyLong_AsSsize_t", + "PyLong_AsUInt32", + "PyLong_AsUInt64", "PyLong_AsUnsignedLong", "PyLong_AsUnsignedLongLong", "PyLong_AsUnsignedLongLongMask", "PyLong_AsUnsignedLongMask", "PyLong_AsVoidPtr", "PyLong_FromDouble", + "PyLong_FromInt32", + "PyLong_FromInt64", "PyLong_FromLong", "PyLong_FromLongLong", "PyLong_FromSize_t", "PyLong_FromSsize_t", "PyLong_FromString", + "PyLong_FromUInt32", + "PyLong_FromUInt64", "PyLong_FromUnsignedLong", "PyLong_FromUnsignedLongLong", "PyLong_FromVoidPtr", @@ -699,14 +713,19 @@ def test_windows_feature_macros(self): "PyTuple_Size", "PyTuple_Type", "PyType_ClearCache", + "PyType_Freeze", "PyType_FromMetaclass", "PyType_FromModuleAndSpec", "PyType_FromSpec", "PyType_FromSpecWithBases", "PyType_GenericAlloc", "PyType_GenericNew", + "PyType_GetBaseByToken", "PyType_GetFlags", + "PyType_GetFullyQualifiedName", "PyType_GetModule", + "PyType_GetModuleByDef", + "PyType_GetModuleName", "PyType_GetModuleState", "PyType_GetName", "PyType_GetQualName", @@ -787,6 +806,7 @@ def test_windows_feature_macros(self): "PyUnicode_DecodeUnicodeEscape", "PyUnicode_EncodeFSDefault", "PyUnicode_EncodeLocale", + "PyUnicode_Equal", "PyUnicode_EqualToUTF8", "PyUnicode_EqualToUTF8AndSize", "PyUnicode_FSConverter", @@ -854,6 +874,8 @@ def test_windows_feature_macros(self): "Py_GetArgcArgv", "Py_GetBuildInfo", "Py_GetCompiler", + "Py_GetConstant", + "Py_GetConstantBorrowed", "Py_GetCopyright", "Py_GetExecPrefix", "Py_GetPath", @@ -879,12 +901,16 @@ def test_windows_feature_macros(self): "Py_MakePendingCalls", "Py_NewInterpreter", "Py_NewRef", + "Py_PACK_FULL_VERSION", + "Py_PACK_VERSION", + "Py_REFCNT", "Py_ReprEnter", "Py_ReprLeave", "Py_SetPath", "Py_SetProgramName", "Py_SetPythonHome", "Py_SetRecursionLimit", + "Py_TYPE", "Py_UTF8Mode", "Py_VaBuildValue", "Py_Version", diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 1cf41638a7f01a..c69baa4bf4d1b1 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -1072,7 +1072,7 @@ def test_no_inplace_modifications(self): def test_order_doesnt_matter(self): # Test that the order of data points doesn't change the result. - # CAUTION: due to floating point rounding errors, the result actually + # CAUTION: due to floating-point rounding errors, the result actually # may depend on the order. Consider this test representing an ideal. # To avoid this test failing, only test with exact values such as ints # or Fractions. @@ -2379,6 +2379,18 @@ def integrate(func, low, high, steps=10_000): area = integrate(f_hat, -20, 20) self.assertAlmostEqual(area, 1.0, places=4) + # Check CDF against an integral of the PDF + + data = [3, 5, 10, 12] + h = 2.3 + x = 10.5 + for kernel in kernels: + with self.subTest(kernel=kernel): + cdf = kde(data, h, kernel, cumulative=True) + f_hat = kde(data, h, kernel) + area = integrate(f_hat, -20, x, 100_000) + self.assertAlmostEqual(cdf(x), area, places=4) + # Check error cases with self.assertRaises(StatisticsError): @@ -2390,11 +2402,13 @@ def integrate(func, low, high, steps=10_000): with self.assertRaises(StatisticsError): kde(sample, h=0.0) # Zero bandwidth with self.assertRaises(StatisticsError): - kde(sample, h=0.0) # Negative bandwidth + kde(sample, h=-1.0) # Negative bandwidth with self.assertRaises(TypeError): kde(sample, h='str') # Wrong bandwidth type with self.assertRaises(StatisticsError): kde(sample, h=1.0, kernel='bogus') # Invalid kernel + with self.assertRaises(TypeError): + kde(sample, 1.0, 'gauss', True) # Positional cumulative argument # Test name and docstring of the generated function @@ -2403,7 +2417,7 @@ def integrate(func, low, high, steps=10_000): f_hat = kde(sample, h, kernel) self.assertEqual(f_hat.__name__, 'pdf') self.assertIn(kernel, f_hat.__doc__) - self.assertIn(str(h), f_hat.__doc__) + self.assertIn(repr(h), f_hat.__doc__) # Test closed interval for the support boundaries. # In particular, 'uniform' should non-zero at the boundaries. @@ -2412,6 +2426,109 @@ def integrate(func, low, high, steps=10_000): self.assertEqual(f_hat(-1.0), 1/2) self.assertEqual(f_hat(1.0), 1/2) + # Test online updates to data + + data = [1, 2] + f_hat = kde(data, 5.0, 'triangular') + self.assertEqual(f_hat(100), 0.0) + data.append(100) + self.assertGreater(f_hat(100), 0.0) + + def test_kde_kernel_specs(self): + # White-box test for the kernel formulas in isolation from + # their downstream use in kde() and kde_random() + kernel_specs = statistics._kernel_specs + + # Verify that cdf / invcdf will round trip + xarr = [i/100 for i in range(-100, 101)] + parr = [i/1000 + 5/10000 for i in range(1000)] + for kernel, spec in kernel_specs.items(): + cdf = spec['cdf'] + invcdf = spec['invcdf'] + with self.subTest(kernel=kernel): + for x in xarr: + self.assertAlmostEqual(invcdf(cdf(x)), x, places=6) + for p in parr: + self.assertAlmostEqual(cdf(invcdf(p)), p, places=11) + + @support.requires_resource('cpu') + def test_kde_random(self): + kde_random = statistics.kde_random + StatisticsError = statistics.StatisticsError + kernels = ['normal', 'gauss', 'logistic', 'sigmoid', 'rectangular', + 'uniform', 'triangular', 'parabolic', 'epanechnikov', + 'quartic', 'biweight', 'triweight', 'cosine'] + sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + + # Smoke test + + for kernel in kernels: + with self.subTest(kernel=kernel): + rand = kde_random(sample, h=1.5, kernel=kernel) + selections = [rand() for i in range(10)] + + # Check error cases + + with self.assertRaises(StatisticsError): + kde_random([], h=1.0) # Empty dataset + with self.assertRaises(TypeError): + kde_random(['abc', 'def'], 1.5) # Non-numeric data + with self.assertRaises(TypeError): + kde_random(iter(sample), 1.5) # Data is not a sequence + with self.assertRaises(StatisticsError): + kde_random(sample, h=-1.0) # Zero bandwidth + with self.assertRaises(StatisticsError): + kde_random(sample, h=0.0) # Negative bandwidth + with self.assertRaises(TypeError): + kde_random(sample, h='str') # Wrong bandwidth type + with self.assertRaises(StatisticsError): + kde_random(sample, h=1.0, kernel='bogus') # Invalid kernel + + # Test name and docstring of the generated function + + h = 1.5 + kernel = 'cosine' + rand = kde_random(sample, h, kernel) + self.assertEqual(rand.__name__, 'rand') + self.assertIn(kernel, rand.__doc__) + self.assertIn(repr(h), rand.__doc__) + + # Approximate distribution test: Compare a random sample to the expected distribution + + data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2, 7.8, 14.3, 15.1, 15.3, 15.8, 17.0] + xarr = [x / 10 for x in range(-100, 250)] + n = 1_000_000 + h = 1.75 + dx = 0.1 + + def p_observed(x): + # P(x <= X < x+dx) + i = bisect.bisect_left(big_sample, x) + j = bisect.bisect_left(big_sample, x + dx) + return (j - i) / len(big_sample) + + def p_expected(x): + # P(x <= X < x+dx) + return F_hat(x + dx) - F_hat(x) + + for kernel in kernels: + with self.subTest(kernel=kernel): + + rand = kde_random(data, h, kernel, seed=8675309**2) + big_sample = sorted([rand() for i in range(n)]) + F_hat = statistics.kde(data, h, kernel, cumulative=True) + + for x in xarr: + self.assertTrue(math.isclose(p_observed(x), p_expected(x), abs_tol=0.0005)) + + # Test online updates to data + + data = [1, 2] + rand = kde_random(data, 5, 'triangular') + self.assertLess(max([rand() for i in range(5000)]), 10) + data.append(100) + self.assertGreater(max(rand() for i in range(5000)), 10) + class TestQuantiles(unittest.TestCase): @@ -2682,7 +2799,7 @@ def test_sqrtprod_helper_function_fundamentals(self): @requires_IEEE_754 @unittest.skipIf(HAVE_DOUBLE_ROUNDING, "accuracy not guaranteed on machines with double rounding") - @support.cpython_only # Allow for a weaker sumprod() implmentation + @support.cpython_only # Allow for a weaker sumprod() implementation def test_sqrtprod_helper_function_improved_accuracy(self): # Test a known example where accuracy is improved x, y, target = 0.8035720646477457, 0.7957468097636939, 0.7996498651651661 diff --git a/Lib/test/test_str.py b/Lib/test/test_str.py index b4927113db44e3..d1c9542c7d1317 100644 --- a/Lib/test/test_str.py +++ b/Lib/test/test_str.py @@ -7,6 +7,7 @@ """ import _string import codecs +import datetime import itertools import operator import pickle @@ -1578,7 +1579,7 @@ def __int__(self): self.assertRaisesRegex(TypeError, '%u format: a real number is required, not complex', operator.mod, '%u', 3j) self.assertRaisesRegex(TypeError, '%i format: a real number is required, not complex', operator.mod, '%i', 2j) self.assertRaisesRegex(TypeError, '%d format: a real number is required, not complex', operator.mod, '%d', 1j) - self.assertRaisesRegex(TypeError, '%c requires int or char', operator.mod, '%c', pi) + self.assertRaisesRegex(TypeError, r'%c requires an int or a unicode character, not .*\.PseudoFloat', operator.mod, '%c', pi) class RaisingNumber: def __int__(self): @@ -1665,7 +1666,7 @@ def test_startswith_endswith_errors(self): self.assertIn('str', exc) self.assertIn('tuple', exc) - @support.run_with_locale('LC_ALL', 'de_DE', 'fr_FR') + @support.run_with_locale('LC_ALL', 'de_DE', 'fr_FR', '') def test_format_float(self): # should not format with a comma, but always with C locale self.assertEqual('1.0', '%.1f' % 1.0) @@ -1736,8 +1737,6 @@ def __str__(self): 'character buffers are decoded to unicode' ) - self.assertRaises(TypeError, str, 42, 42, 42) - def test_constructor_keyword_args(self): """Pass various keyword argument combinations to the constructor.""" # The object argument can be passed as a keyword. @@ -1910,6 +1909,12 @@ def test_utf8_decode_invalid_sequences(self): self.assertRaises(UnicodeDecodeError, (b'\xF4'+cb+b'\xBF\xBF').decode, 'utf-8') + def test_issue127903(self): + # gh-127903: ``_copy_characters`` crashes on DEBUG builds when + # there is nothing to copy. + d = datetime.datetime(2013, 11, 10, 14, 20, 59) + self.assertEqual(d.strftime('%z'), '') + def test_issue8271(self): # Issue #8271: during the decoding of an invalid UTF-8 byte sequence, # only the start byte and the continuation byte(s) are now considered @@ -2432,8 +2437,10 @@ def __repr__(self): self.assertEqual(repr(s1()), '\\n') def test_printable_repr(self): - self.assertEqual(repr('\U00010000'), "'%c'" % (0x10000,)) # printable - self.assertEqual(repr('\U00014000'), "'\\U00014000'") # nonprintable + # printable + self.assertEqual(repr('\U00010000'), "'%c'" % (0x10000,)) + # nonprintable (private use area) + self.assertEqual(repr('\U00100001'), "'\\U00100001'") # This test only affects 32-bit platforms because expandtabs can only take # an int as the max value, not a 64-bit C long. If expandtabs is changed @@ -2651,6 +2658,47 @@ def test_check_encoding_errors(self): proc = assert_python_failure('-X', 'dev', '-c', code) self.assertEqual(proc.rc, 10, proc) + def test_str_invalid_call(self): + # too many args + with self.assertRaisesRegex(TypeError, r"str expected at most 3 arguments, got 4"): + str("too", "many", "argu", "ments") + with self.assertRaisesRegex(TypeError, r"str expected at most 3 arguments, got 4"): + str(1, "", "", 1) + + # no such kw arg + with self.assertRaisesRegex(TypeError, r"str\(\) got an unexpected keyword argument 'test'"): + str(test=1) + + # 'encoding' must be str + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'encoding' must be str, not int"): + str(1, 1) + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'encoding' must be str, not int"): + str(1, encoding=1) + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'encoding' must be str, not bytes"): + str(b"x", b"ascii") + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'encoding' must be str, not bytes"): + str(b"x", encoding=b"ascii") + + # 'errors' must be str + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'encoding' must be str, not int"): + str(1, 1, 1) + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'errors' must be str, not int"): + str(1, errors=1) + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'errors' must be str, not int"): + str(1, "", errors=1) + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'errors' must be str, not bytes"): + str(b"x", "ascii", b"strict") + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'errors' must be str, not bytes"): + str(b"x", "ascii", errors=b"strict") + + # both positional and kwarg + with self.assertRaisesRegex(TypeError, r"argument for str\(\) given by name \('encoding'\) and position \(2\)"): + str(b"x", "utf-8", encoding="ascii") + with self.assertRaisesRegex(TypeError, r"str\(\) takes at most 3 arguments \(4 given\)"): + str(b"x", "utf-8", "ignore", encoding="ascii") + with self.assertRaisesRegex(TypeError, r"str\(\) takes at most 3 arguments \(4 given\)"): + str(b"x", "utf-8", "strict", errors="ignore") + class StringModuleTest(unittest.TestCase): def test_formatter_parser(self): diff --git a/Lib/test/test_strftime.py b/Lib/test/test_strftime.py index cebfc8927862a7..752e31359cf206 100644 --- a/Lib/test/test_strftime.py +++ b/Lib/test/test_strftime.py @@ -183,8 +183,7 @@ class Y1900Tests(unittest.TestCase): def test_y_before_1900(self): # Issue #13674, #19634 t = (1899, 1, 1, 0, 0, 0, 0, 0, 0) - if (sys.platform == "win32" - or sys.platform.startswith(("aix", "sunos", "solaris"))): + if sys.platform.startswith(("aix", "sunos", "solaris")): with self.assertRaises(ValueError): time.strftime("%y", t) else: diff --git a/Lib/test/test_string_literals.py b/Lib/test/test_string_literals.py index 371e8193b3544d..f56195ca27672c 100644 --- a/Lib/test/test_string_literals.py +++ b/Lib/test/test_string_literals.py @@ -116,7 +116,9 @@ def test_eval_str_invalid_escape(self): warnings.simplefilter('always', category=SyntaxWarning) eval("'''\n\\z'''") self.assertEqual(len(w), 1) - self.assertEqual(str(w[0].message), r"invalid escape sequence '\z'") + self.assertEqual(str(w[0].message), r'"\z" is an invalid escape sequence. ' + r'Such sequences will not work in the future. ' + r'Did you mean "\\z"? A raw string is also an option.') self.assertEqual(w[0].filename, '') self.assertEqual(w[0].lineno, 1) @@ -126,12 +128,13 @@ def test_eval_str_invalid_escape(self): eval("'''\n\\z'''") exc = cm.exception self.assertEqual(w, []) - self.assertEqual(exc.msg, r"invalid escape sequence '\z'") + self.assertEqual(exc.msg, r'"\z" is an invalid escape sequence. ' + r'Did you mean "\\z"? A raw string is also an option.') self.assertEqual(exc.filename, '') self.assertEqual(exc.lineno, 1) self.assertEqual(exc.offset, 1) - # Check that the warning is raised ony once if there are syntax errors + # Check that the warning is raised only once if there are syntax errors with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always', category=SyntaxWarning) @@ -153,7 +156,9 @@ def test_eval_str_invalid_octal_escape(self): eval("'''\n\\407'''") self.assertEqual(len(w), 1) self.assertEqual(str(w[0].message), - r"invalid octal escape sequence '\407'") + r'"\407" is an invalid octal escape sequence. ' + r'Such sequences will not work in the future. ' + r'Did you mean "\\407"? A raw string is also an option.') self.assertEqual(w[0].filename, '') self.assertEqual(w[0].lineno, 1) @@ -163,7 +168,8 @@ def test_eval_str_invalid_octal_escape(self): eval("'''\n\\407'''") exc = cm.exception self.assertEqual(w, []) - self.assertEqual(exc.msg, r"invalid octal escape sequence '\407'") + self.assertEqual(exc.msg, r'"\407" is an invalid octal escape sequence. ' + r'Did you mean "\\407"? A raw string is also an option.') self.assertEqual(exc.filename, '') self.assertEqual(exc.lineno, 1) self.assertEqual(exc.offset, 1) @@ -205,7 +211,9 @@ def test_eval_bytes_invalid_escape(self): warnings.simplefilter('always', category=SyntaxWarning) eval("b'''\n\\z'''") self.assertEqual(len(w), 1) - self.assertEqual(str(w[0].message), r"invalid escape sequence '\z'") + self.assertEqual(str(w[0].message), r'"\z" is an invalid escape sequence. ' + r'Such sequences will not work in the future. ' + r'Did you mean "\\z"? A raw string is also an option.') self.assertEqual(w[0].filename, '') self.assertEqual(w[0].lineno, 1) @@ -215,7 +223,8 @@ def test_eval_bytes_invalid_escape(self): eval("b'''\n\\z'''") exc = cm.exception self.assertEqual(w, []) - self.assertEqual(exc.msg, r"invalid escape sequence '\z'") + self.assertEqual(exc.msg, r'"\z" is an invalid escape sequence. ' + r'Did you mean "\\z"? A raw string is also an option.') self.assertEqual(exc.filename, '') self.assertEqual(exc.lineno, 1) @@ -228,8 +237,9 @@ def test_eval_bytes_invalid_octal_escape(self): warnings.simplefilter('always', category=SyntaxWarning) eval("b'''\n\\407'''") self.assertEqual(len(w), 1) - self.assertEqual(str(w[0].message), - r"invalid octal escape sequence '\407'") + self.assertEqual(str(w[0].message), r'"\407" is an invalid octal escape sequence. ' + r'Such sequences will not work in the future. ' + r'Did you mean "\\407"? A raw string is also an option.') self.assertEqual(w[0].filename, '') self.assertEqual(w[0].lineno, 1) @@ -239,7 +249,8 @@ def test_eval_bytes_invalid_octal_escape(self): eval("b'''\n\\407'''") exc = cm.exception self.assertEqual(w, []) - self.assertEqual(exc.msg, r"invalid octal escape sequence '\407'") + self.assertEqual(exc.msg, r'"\407" is an invalid octal escape sequence. ' + r'Did you mean "\\407"? A raw string is also an option.') self.assertEqual(exc.filename, '') self.assertEqual(exc.lineno, 1) diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py index 05c8afc907ad3c..0d30a63ab0c140 100644 --- a/Lib/test/test_strptime.py +++ b/Lib/test/test_strptime.py @@ -5,13 +5,22 @@ import locale import re import os +import platform import sys from test import support -from test.support import skip_if_buggy_ucrt_strfptime +from test.support import warnings_helper +from test.support import skip_if_buggy_ucrt_strfptime, run_with_locales from datetime import date as datetime_date import _strptime +libc_ver = platform.libc_ver() +if libc_ver[0] == 'glibc': + glibc_ver = tuple(map(int, libc_ver[1].split('.'))) +else: + glibc_ver = None + + class getlang_Tests(unittest.TestCase): """Test _getlang""" def test_basic(self): @@ -70,9 +79,6 @@ def test_am_pm(self): self.assertEqual(self.LT_ins.am_pm[position], strftime_output, "AM/PM representation in the wrong position within the tuple") - @unittest.skipIf( - support.is_emscripten, "musl libc issue on Emscripten, bpo-46390" - ) def test_timezone(self): # Make sure timezone is correct timezone = time.strftime("%Z", self.time_tuple).lower() @@ -120,7 +126,7 @@ def setUp(self): def test_pattern(self): # Test TimeRE.pattern - pattern_string = self.time_re.pattern(r"%a %A %d") + pattern_string = self.time_re.pattern(r"%a %A %d %Y") self.assertTrue(pattern_string.find(self.locale_time.a_weekday[2]) != -1, "did not find abbreviated weekday in pattern string '%s'" % pattern_string) @@ -160,10 +166,11 @@ def test_compile(self): found.group('b'))) for directive in ('a','A','b','B','c','d','G','H','I','j','m','M','p', 'S','u','U','V','w','W','x','X','y','Y','Z','%'): - compiled = self.time_re.compile("%" + directive) - found = compiled.match(time.strftime("%" + directive)) + fmt = "%d %Y" if directive == 'd' else "%" + directive + compiled = self.time_re.compile(fmt) + found = compiled.match(time.strftime(fmt)) self.assertTrue(found, "Matching failed on '%s' using '%s' regex" % - (time.strftime("%" + directive), + (time.strftime(fmt), compiled.pattern)) def test_blankpattern(self): @@ -206,8 +213,8 @@ class StrptimeTests(unittest.TestCase): """Tests for _strptime.strptime.""" def setUp(self): - """Create testing time tuple.""" - self.time_tuple = time.gmtime() + """Create testing time tuples.""" + self.time_tuple = time.localtime() def test_ValueError(self): # Make sure ValueError is raised when match fails or format is bad @@ -282,59 +289,73 @@ def test_strptime_exception_context(self): # additional check for IndexError branch (issue #19545) with self.assertRaises(ValueError) as e: _strptime._strptime_time('19', '%Y %') - self.assertIs(e.exception.__suppress_context__, True) + self.assertIsNone(e.exception.__context__) def test_unconverteddata(self): # Check ValueError is raised when there is unconverted data self.assertRaises(ValueError, _strptime._strptime_time, "10 12", "%m") - def helper(self, directive, position): + def roundtrip(self, fmt, position, time_tuple=None): """Helper fxn in testing.""" - strf_output = time.strftime("%" + directive, self.time_tuple) - strp_output = _strptime._strptime_time(strf_output, "%" + directive) - self.assertTrue(strp_output[position] == self.time_tuple[position], - "testing of '%s' directive failed; '%s' -> %s != %s" % - (directive, strf_output, strp_output[position], - self.time_tuple[position])) + if time_tuple is None: + time_tuple = self.time_tuple + strf_output = time.strftime(fmt, time_tuple) + strp_output = _strptime._strptime_time(strf_output, fmt) + self.assertEqual(strp_output[position], time_tuple[position], + "testing of %r format failed; %r -> %r != %r" % + (fmt, strf_output, strp_output[position], + time_tuple[position])) + if support.verbose >= 3: + print("testing of %r format: %r -> %r" % + (fmt, strf_output, strp_output[position])) def test_year(self): # Test that the year is handled properly - for directive in ('y', 'Y'): - self.helper(directive, 0) + self.roundtrip('%Y', 0) + self.roundtrip('%y', 0) + self.roundtrip('%Y', 0, (1900, 1, 1, 0, 0, 0, 0, 1, 0)) + # Must also make sure %y values are correct for bounds set by Open Group - for century, bounds in ((1900, ('69', '99')), (2000, ('00', '68'))): - for bound in bounds: - strp_output = _strptime._strptime_time(bound, '%y') - expected_result = century + int(bound) - self.assertTrue(strp_output[0] == expected_result, - "'y' test failed; passed in '%s' " - "and returned '%s'" % (bound, strp_output[0])) + strptime = _strptime._strptime_time + self.assertEqual(strptime('00', '%y')[0], 2000) + self.assertEqual(strptime('68', '%y')[0], 2068) + self.assertEqual(strptime('69', '%y')[0], 1969) + self.assertEqual(strptime('99', '%y')[0], 1999) def test_month(self): # Test for month directives - for directive in ('B', 'b', 'm'): - self.helper(directive, 1) + self.roundtrip('%m', 1) + + @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', 'he_IL', '') + def test_month_locale(self): + # Test for month directives + self.roundtrip('%B', 1) + self.roundtrip('%b', 1) + for m in range(1, 13): + self.roundtrip('%B', 1, (1900, m, 1, 0, 0, 0, 0, 1, 0)) + self.roundtrip('%b', 1, (1900, m, 1, 0, 0, 0, 0, 1, 0)) def test_day(self): # Test for day directives - self.helper('d', 2) + self.roundtrip('%d %Y', 2) def test_hour(self): # Test hour directives - self.helper('H', 3) - strf_output = time.strftime("%I %p", self.time_tuple) - strp_output = _strptime._strptime_time(strf_output, "%I %p") - self.assertTrue(strp_output[3] == self.time_tuple[3], - "testing of '%%I %%p' directive failed; '%s' -> %s != %s" % - (strf_output, strp_output[3], self.time_tuple[3])) + self.roundtrip('%H', 3) + + # NB: Only works on locales with AM/PM + @run_with_locales('LC_TIME', 'C', 'en_US', 'ja_JP') + def test_hour_locale(self): + # Test hour directives + self.roundtrip('%I %p', 3) def test_minute(self): # Test minute directives - self.helper('M', 4) + self.roundtrip('%M', 4) def test_second(self): # Test second directives - self.helper('S', 5) + self.roundtrip('%S', 5) def test_fraction(self): # Test microseconds @@ -345,12 +366,18 @@ def test_fraction(self): def test_weekday(self): # Test weekday directives - for directive in ('A', 'a', 'w', 'u'): - self.helper(directive,6) + self.roundtrip('%w', 6) + self.roundtrip('%u', 6) + + @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', '') + def test_weekday_locale(self): + # Test weekday directives + self.roundtrip('%A', 6) + self.roundtrip('%a', 6) def test_julian(self): # Test julian directives - self.helper('j', 7) + self.roundtrip('%j', 7) def test_offset(self): one_hour = 60 * 60 @@ -401,9 +428,6 @@ def test_bad_offset(self): self.assertEqual("Inconsistent use of : in -01:3030", str(err.exception)) @skip_if_buggy_ucrt_strfptime - @unittest.skipIf( - support.is_emscripten, "musl libc issue on Emscripten, bpo-46390" - ) def test_timezone(self): # Test timezone directives. # When gmtime() is used with %Z, entire result of strftime() is empty. @@ -447,20 +471,121 @@ def test_bad_timezone(self): "time.daylight set to %s and passing in %s" % (time.tzname, tz_value, time.daylight, tz_name)) - def test_date_time(self): + # NB: Does not roundtrip in some locales due to the ambiguity of + # the date and time representation (bugs in locales?): + # * Seconds are not included: bem_ZM, bokmal, ff_SN, nb_NO, nn_NO, + # no_NO, norwegian, nynorsk. + # * Hours are in 12-hour notation without AM/PM indication: hy_AM, + # id_ID, ms_MY. + # * Year is not included: ha_NG. + # * Use non-Gregorian calendar: lo_LA, thai, th_TH. + # On Windows: ar_IN, ar_SA, fa_IR, ps_AF. + # + # BUG: Generates regexp that does not match the current date and time + # for lzh_TW. + @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', + 'he_IL', 'eu_ES', 'ar_AE', 'mfe_MU', 'yo_NG', + 'csb_PL', 'br_FR', 'gez_ET', 'brx_IN', + 'my_MM', 'or_IN', 'shn_MM', 'az_IR') + def test_date_time_locale(self): # Test %c directive - for position in range(6): - self.helper('c', position) - - def test_date(self): + loc = locale.getlocale(locale.LC_TIME)[0] + if glibc_ver and glibc_ver < (2, 31) and loc == 'br_FR': + self.skipTest('%c in locale br_FR does not include time') + now = time.time() + self.roundtrip('%c', slice(0, 6), time.localtime(now)) + # 1 hour 20 minutes 30 seconds ago + self.roundtrip('%c', slice(0, 6), time.localtime(now - 4830)) + # 12 hours ago + self.roundtrip('%c', slice(0, 6), time.localtime(now - 12*3600)) + # different days of the week + for i in range(1, 7): + self.roundtrip('%c', slice(0, 6), time.localtime(now - i*24*3600)) + # different months + for i in range(1, 12): + self.roundtrip('%c', slice(0, 6), time.localtime(now - i*30*24*3600)) + # different year + self.roundtrip('%c', slice(0, 6), time.localtime(now - 366*24*3600)) + + # NB: Dates before 1969 do not roundtrip on some locales: + # az_IR, bo_CN, bo_IN, dz_BT, eu_ES, eu_FR, fa_IR, or_IN. + @support.run_with_tz('STD-1DST,M4.1.0,M10.1.0') + @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', + 'he_IL', 'ar_AE', 'mfe_MU', 'yo_NG', + 'csb_PL', 'br_FR', 'gez_ET', 'brx_IN', + 'my_MM', 'shn_MM') + def test_date_time_locale2(self): + # Test %c directive + loc = locale.getlocale(locale.LC_TIME)[0] + if sys.platform.startswith('sunos'): + if loc in ('ar_AE',): + self.skipTest(f'locale {loc!r} may not work on this platform') + self.roundtrip('%c', slice(0, 6), (1900, 1, 1, 0, 0, 0, 0, 1, 0)) + self.roundtrip('%c', slice(0, 6), (1800, 1, 1, 0, 0, 0, 0, 1, 0)) + + # NB: Does not roundtrip because use non-Gregorian calendar: + # lo_LA, thai, th_TH. On Windows: ar_IN, ar_SA, fa_IR, ps_AF. + # BUG: Generates regexp that does not match the current date + # for lzh_TW. + @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', + 'he_IL', 'eu_ES', 'ar_AE', + 'az_IR', 'my_MM', 'or_IN', 'shn_MM') + def test_date_locale(self): # Test %x directive - for position in range(0,3): - self.helper('x', position) - - def test_time(self): + now = time.time() + self.roundtrip('%x', slice(0, 3), time.localtime(now)) + # different days of the week + for i in range(1, 7): + self.roundtrip('%x', slice(0, 3), time.localtime(now - i*24*3600)) + # different months + for i in range(1, 12): + self.roundtrip('%x', slice(0, 3), time.localtime(now - i*30*24*3600)) + # different year + self.roundtrip('%x', slice(0, 3), time.localtime(now - 366*24*3600)) + + # NB: Dates before 1969 do not roundtrip on many locales, including C. + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "musl libc issue on Emscripten, bpo-46390" + ) + @run_with_locales('LC_TIME', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', + 'eu_ES', 'ar_AE', 'my_MM', 'shn_MM') + def test_date_locale2(self): + # Test %x directive + loc = locale.getlocale(locale.LC_TIME)[0] + if sys.platform.startswith('sunos'): + if loc in ('en_US', 'de_DE', 'ar_AE'): + self.skipTest(f'locale {loc!r} may not work on this platform') + self.roundtrip('%x', slice(0, 3), (1900, 1, 1, 0, 0, 0, 0, 1, 0)) + self.roundtrip('%x', slice(0, 3), (1800, 1, 1, 0, 0, 0, 0, 1, 0)) + + # NB: Does not roundtrip in some locales due to the ambiguity of + # the time representation (bugs in locales?): + # * Seconds are not included: bokmal, ff_SN, nb_NO, nn_NO, no_NO, + # norwegian, nynorsk. + # * Hours are in 12-hour notation without AM/PM indication: hy_AM, + # ms_MY, sm_WS. + # BUG: Generates regexp that does not match the current time for lzh_TW. + @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', + 'aa_ET', 'am_ET', 'az_IR', 'byn_ER', 'fa_IR', 'gez_ET', + 'my_MM', 'om_ET', 'or_IN', 'shn_MM', 'sid_ET', 'so_SO', + 'ti_ET', 'tig_ER', 'wal_ET') + def test_time_locale(self): # Test %X directive - for position in range(3,6): - self.helper('X', position) + loc = locale.getlocale(locale.LC_TIME)[0] + pos = slice(3, 6) + if glibc_ver and glibc_ver < (2, 29) and loc in { + 'aa_ET', 'am_ET', 'byn_ER', 'gez_ET', 'om_ET', + 'sid_ET', 'so_SO', 'ti_ET', 'tig_ER', 'wal_ET'}: + # Hours are in 12-hour notation without AM/PM indication. + # Ignore hours. + pos = slice(4, 6) + now = time.time() + self.roundtrip('%X', pos, time.localtime(now)) + # 1 hour 20 minutes 30 seconds ago + self.roundtrip('%X', pos, time.localtime(now - 4830)) + # 12 hours ago + self.roundtrip('%X', pos, time.localtime(now - 12*3600)) def test_percent(self): # Make sure % signs are handled properly @@ -497,9 +622,11 @@ def test_escaping(self): need_escaping = r".^$*+?{}\[]|)(" self.assertTrue(_strptime._strptime_time(need_escaping, need_escaping)) + @warnings_helper.ignore_warnings(category=DeprecationWarning) # gh-70647 def test_feb29_on_leap_year_without_year(self): time.strptime("Feb 29", "%b %d") + @warnings_helper.ignore_warnings(category=DeprecationWarning) # gh-70647 def test_mar1_comes_after_feb29_even_when_omitting_the_year(self): self.assertLess( time.strptime("Feb 29", "%b %d"), @@ -679,25 +806,25 @@ class CacheTests(unittest.TestCase): def test_time_re_recreation(self): # Make sure cache is recreated when current locale does not match what # cached object was created with. - _strptime._strptime_time("10", "%d") + _strptime._strptime_time("10 2004", "%d %Y") _strptime._strptime_time("2005", "%Y") _strptime._TimeRE_cache.locale_time.lang = "Ni" original_time_re = _strptime._TimeRE_cache - _strptime._strptime_time("10", "%d") + _strptime._strptime_time("10 2004", "%d %Y") self.assertIsNot(original_time_re, _strptime._TimeRE_cache) self.assertEqual(len(_strptime._regex_cache), 1) def test_regex_cleanup(self): # Make sure cached regexes are discarded when cache becomes "full". try: - del _strptime._regex_cache['%d'] + del _strptime._regex_cache['%d %Y'] except KeyError: pass bogus_key = 0 while len(_strptime._regex_cache) <= _strptime._CACHE_MAX_SIZE: _strptime._regex_cache[bogus_key] = None bogus_key += 1 - _strptime._strptime_time("10", "%d") + _strptime._strptime_time("10 2004", "%d %Y") self.assertEqual(len(_strptime._regex_cache), 1) def test_new_localetime(self): @@ -705,24 +832,19 @@ def test_new_localetime(self): # is created. locale_time_id = _strptime._TimeRE_cache.locale_time _strptime._TimeRE_cache.locale_time.lang = "Ni" - _strptime._strptime_time("10", "%d") + _strptime._strptime_time("10 2004", "%d %Y") self.assertIsNot(locale_time_id, _strptime._TimeRE_cache.locale_time) def test_TimeRE_recreation_locale(self): # The TimeRE instance should be recreated upon changing the locale. - locale_info = locale.getlocale(locale.LC_TIME) - try: - locale.setlocale(locale.LC_TIME, ('en_US', 'UTF8')) - except locale.Error: - self.skipTest('test needs en_US.UTF8 locale') - try: - _strptime._strptime_time('10', '%d') + with support.run_with_locale('LC_TIME', 'en_US.UTF8'): + _strptime._strptime_time('10 2004', '%d %Y') # Get id of current cache object. first_time_re = _strptime._TimeRE_cache try: # Change the locale and force a recreation of the cache. locale.setlocale(locale.LC_TIME, ('de_DE', 'UTF8')) - _strptime._strptime_time('10', '%d') + _strptime._strptime_time('10 2004', '%d %Y') # Get the new cache object's id. second_time_re = _strptime._TimeRE_cache # They should not be equal. @@ -732,10 +854,6 @@ def test_TimeRE_recreation_locale(self): # to the resetting to the original locale. except locale.Error: self.skipTest('test needs de_DE.UTF8 locale') - # Make sure we don't trample on the locale setting once we leave the - # test. - finally: - locale.setlocale(locale.LC_TIME, locale_info) @support.run_with_tz('STD-1DST,M4.1.0,M10.1.0') def test_TimeRE_recreation_timezone(self): diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 15f6ee06ffe19b..5fee9fbb92acf4 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -1,4 +1,5 @@ from collections import abc +from itertools import combinations import array import gc import math @@ -11,12 +12,22 @@ from test import support from test.support import import_helper from test.support.script_helper import assert_python_ok +from test.support.testcase import ComplexesAreIdenticalMixin ISBIGENDIAN = sys.byteorder == "big" integer_codes = 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'n', 'N' byteorders = '', '@', '=', '<', '>', '!' +INF = float('inf') +NAN = float('nan') + +try: + struct.pack('C', 1j) + have_c_complex = True +except struct.error: + have_c_complex = False + def iter_integer_formats(byteorders=byteorders): for code in integer_codes: for byteorder in byteorders: @@ -33,7 +44,7 @@ def bigendian_to_native(value): else: return string_reverse(value) -class StructTest(unittest.TestCase): +class StructTest(ComplexesAreIdenticalMixin, unittest.TestCase): def test_isbigendian(self): self.assertEqual((struct.pack('=i', 1)[0] == 0), ISBIGENDIAN) @@ -96,6 +107,13 @@ def test_new_features(self): ('10s', b'helloworld', b'helloworld', b'helloworld', 0), ('11s', b'helloworld', b'helloworld\0', b'helloworld\0', 1), ('20s', b'helloworld', b'helloworld'+10*b'\0', b'helloworld'+10*b'\0', 1), + ('0p', b'helloworld', b'', b'', 1), + ('1p', b'helloworld', b'\x00', b'\x00', 1), + ('2p', b'helloworld', b'\x01h', b'\x01h', 1), + ('10p', b'helloworld', b'\x09helloworl', b'\x09helloworl', 1), + ('11p', b'helloworld', b'\x0Ahelloworld', b'\x0Ahelloworld', 0), + ('12p', b'helloworld', b'\x0Ahelloworld\0', b'\x0Ahelloworld\0', 1), + ('20p', b'helloworld', b'\x0Ahelloworld'+9*b'\0', b'\x0Ahelloworld'+9*b'\0', 1), ('b', 7, b'\7', b'\7', 0), ('b', -7, b'\371', b'\371', 0), ('B', 7, b'\7', b'\7', 0), @@ -339,6 +357,7 @@ def assertStructError(func, *args, **kwargs): def test_p_code(self): # Test p ("Pascal string") code. for code, input, expected, expectedback in [ + ('0p', b'abc', b'', b''), ('p', b'abc', b'\x00', b''), ('1p', b'abc', b'\x00', b''), ('2p', b'abc', b'\x01a', b'a'), @@ -521,6 +540,9 @@ def __bool__(self): for c in [b'\x01', b'\x7f', b'\xff', b'\x0f', b'\xf0']: self.assertTrue(struct.unpack('>?', c)[0]) + self.assertTrue(struct.unpack('E', '>C', '"), ('a' * 100, True, 0, @@ -1532,7 +1529,8 @@ def test_repr(self): (["ls"], False, None, ""), (["ls", '--my-opts', 'a' * 100], False, None, ""), - (path_cmd, False, 7, f"") + (os_helper.FakePath("my-tool.py"), False, 7, + ">") ] with unittest.mock.patch.object(subprocess.Popen, '_execute_child'): for cmd, shell, code, sx in cases: @@ -1607,6 +1605,22 @@ def test_class_getitems(self): self.assertIsInstance(subprocess.Popen[bytes], types.GenericAlias) self.assertIsInstance(subprocess.CompletedProcess[str], types.GenericAlias) + @unittest.skipUnless(hasattr(subprocess, '_winapi'), + 'need subprocess._winapi') + def test_wait_negative_timeout(self): + proc = subprocess.Popen(ZERO_RETURN_CMD) + with proc: + patch = mock.patch.object( + subprocess._winapi, + 'WaitForSingleObject', + return_value=subprocess._winapi.WAIT_OBJECT_0) + with patch as mock_wait: + proc.wait(-1) # negative timeout + mock_wait.assert_called_once_with(proc._handle, 0) + proc.returncode = None + + self.assertEqual(proc.wait(), 0) + class RunFuncTestCase(BaseTestCase): def run_python(self, code, **kwargs): @@ -1747,6 +1761,13 @@ def test_capture_output(self): self.assertIn(b'BDFL', cp.stdout) self.assertIn(b'FLUFL', cp.stderr) + def test_stdout_stdout(self): + # run() refuses to accept stdout=STDOUT + with self.assertRaises(ValueError, + msg=("STDOUT can only be used for stderr")): + self.run_python("print('will not be run')", + stdout=subprocess.STDOUT) + def test_stdout_with_capture_output_arg(self): # run() refuses to accept 'stdout' with 'capture_output' tf = tempfile.TemporaryFile() @@ -3258,7 +3279,7 @@ def __int__(self): 1, 2, 3, 4, True, True, 0, None, None, None, -1, - None, True) + None) self.assertIn('fds_to_keep', str(c.exception)) finally: if not gc_enabled: @@ -3382,38 +3403,20 @@ def test_preexec_at_exit(self): def dummy(): pass - def exit_handler(): - subprocess.Popen({ZERO_RETURN_CMD}, preexec_fn=dummy) - print("shouldn't be printed") - - atexit.register(exit_handler) + class AtFinalization: + def __del__(self): + print("OK") + subprocess.Popen({ZERO_RETURN_CMD}, preexec_fn=dummy) + print("shouldn't be printed") + at_finalization = AtFinalization() """ _, out, err = assert_python_ok("-c", code) - self.assertEqual(out, b'') + self.assertEqual(out.strip(), b"OK") self.assertIn(b"preexec_fn not supported at interpreter shutdown", err) @unittest.skipIf(not sysconfig.get_config_var("HAVE_VFORK"), "vfork() not enabled by configure.") - @mock.patch("subprocess._fork_exec") - @mock.patch("subprocess._USE_POSIX_SPAWN", new=False) - def test__use_vfork(self, mock_fork_exec): - self.assertTrue(subprocess._USE_VFORK) # The default value regardless. - mock_fork_exec.side_effect = RuntimeError("just testing args") - with self.assertRaises(RuntimeError): - subprocess.run([sys.executable, "-c", "pass"]) - mock_fork_exec.assert_called_once() - # NOTE: These assertions are *ugly* as they require the last arg - # to remain the have_vfork boolean. We really need to refactor away - # from the giant "wall of args" internal C extension API. - self.assertTrue(mock_fork_exec.call_args.args[-1]) - with mock.patch.object(subprocess, '_USE_VFORK', False): - with self.assertRaises(RuntimeError): - subprocess.run([sys.executable, "-c", "pass"]) - self.assertFalse(mock_fork_exec.call_args_list[-1].args[-1]) - - @unittest.skipIf(not sysconfig.get_config_var("HAVE_VFORK"), - "vfork() not enabled by configure.") - @unittest.skipIf(sys.platform != "linux", "Linux only, requires strace.") + @strace_helper.requires_strace() @mock.patch("subprocess._USE_POSIX_SPAWN", new=False) def test_vfork_used_when_expected(self): # This is a performance regression test to ensure we default to using @@ -3421,52 +3424,39 @@ def test_vfork_used_when_expected(self): # Technically this test could pass when posix_spawn is used as well # because libc tends to implement that internally using vfork. But # that'd just be testing a libc+kernel implementation detail. - strace_binary = "/usr/bin/strace" - # The only system calls we are interested in. - strace_filter = "--trace=clone,clone2,clone3,fork,vfork,exit,exit_group" - true_binary = "/bin/true" - strace_command = [strace_binary, strace_filter] - try: - does_strace_work_process = subprocess.run( - strace_command + [true_binary], - stderr=subprocess.PIPE, - stdout=subprocess.DEVNULL, - ) - rc = does_strace_work_process.returncode - stderr = does_strace_work_process.stderr - except OSError: - rc = -1 - stderr = "" - if rc or (b"+++ exited with 0 +++" not in stderr): - self.skipTest("strace not found or not working as expected.") + # Are intersted in the system calls: + # clone,clone2,clone3,fork,vfork,exit,exit_group + # Unfortunately using `--trace` with that list to strace fails because + # not all are supported on all platforms (ex. clone2 is ia64 only...) + # So instead use `%process` which is recommended by strace, and contains + # the above. + true_binary = "/bin/true" + strace_args = ["--trace=%process"] with self.subTest(name="default_is_vfork"): - vfork_result = assert_python_ok( - "-c", - textwrap.dedent(f"""\ - import subprocess - subprocess.check_call([{true_binary!r}])"""), - __run_using_command=strace_command, + vfork_result = strace_helper.strace_python( + f"""\ + import subprocess + subprocess.check_call([{true_binary!r}])""", + strace_args ) # Match both vfork() and clone(..., flags=...|CLONE_VFORK|...) - self.assertRegex(vfork_result.err, br"(?i)vfork") + self.assertRegex(vfork_result.event_bytes, br"(?i)vfork") # Do NOT check that fork() or other clones did not happen. # If the OS denys the vfork it'll fallback to plain fork(). # Test that each individual thing that would disable the use of vfork # actually disables it. for sub_name, preamble, sp_kwarg, expect_permission_error in ( - ("!use_vfork", "subprocess._USE_VFORK = False", "", False), ("preexec", "", "preexec_fn=lambda: None", False), ("setgid", "", f"group={os.getgid()}", True), ("setuid", "", f"user={os.getuid()}", True), ("setgroups", "", "extra_groups=[]", True), ): with self.subTest(name=sub_name): - non_vfork_result = assert_python_ok( - "-c", - textwrap.dedent(f"""\ + non_vfork_result = strace_helper.strace_python( + f"""\ import subprocess {preamble} try: @@ -3474,11 +3464,11 @@ def test_vfork_used_when_expected(self): [{true_binary!r}], **dict({sp_kwarg})) except PermissionError: if not {expect_permission_error}: - raise"""), - __run_using_command=strace_command, + raise""", + strace_args ) # Ensure neither vfork() or clone(..., flags=...|CLONE_VFORK|...). - self.assertNotRegex(non_vfork_result.err, br"(?i)vfork") + self.assertNotRegex(non_vfork_result.event_bytes, br"(?i)vfork") @unittest.skipUnless(mswindows, "Windows specific tests") diff --git a/Lib/test/test_sundry.py b/Lib/test/test_sundry.py index f4a8d434ed1b8c..d6d08ee53f821c 100644 --- a/Lib/test/test_sundry.py +++ b/Lib/test/test_sundry.py @@ -18,10 +18,11 @@ def test_untested_modules_can_be_imported(self): self.fail('{} has tests even though test_sundry claims ' 'otherwise'.format(name)) - import html.entities + import html.entities # noqa: F401 try: - import tty # Not available on Windows + # Not available on Windows + import tty # noqa: F401 except ImportError: if support.verbose: print("skipping tty") diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index 256b416caaa584..5cef612a340be9 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -1,12 +1,12 @@ """Unit tests for zero-argument super() & related machinery.""" +import copy +import pickle import textwrap +import threading import unittest from unittest.mock import patch -from test.support import import_helper - - -ADAPTIVE_WARMUP_DELAY = 2 +from test.support import import_helper, threading_helper class A: @@ -463,7 +463,8 @@ def test(name): super(MyType, type(mytype)).__setattr__(mytype, "bar", 1) self.assertEqual(mytype.bar, 1) - for _ in range(ADAPTIVE_WARMUP_DELAY): + _testinternalcapi = import_helper.import_module("_testinternalcapi") + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): test("foo1") def test_reassigned_new(self): @@ -482,7 +483,8 @@ class C(B): def __new__(cls): return super().__new__(cls) - for _ in range(ADAPTIVE_WARMUP_DELAY): + _testinternalcapi = import_helper.import_module("_testinternalcapi") + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): C() def test_mixed_staticmethod_hierarchy(self): @@ -502,9 +504,111 @@ class C(B): def some(cls): return super().some(cls) - for _ in range(ADAPTIVE_WARMUP_DELAY): + _testinternalcapi = import_helper.import_module("_testinternalcapi") + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): C.some(C) + @threading_helper.requires_working_threading() + def test___class___modification_multithreaded(self): + """ Note: this test isn't actually testing anything on its own. + It requires a sys audithook to be set to crash on older Python. + This should be the case anyways as our test suite sets + an audit hook. + """ + + class Foo: + pass + + class Bar: + pass + + thing = Foo() + def work(): + foo = thing + for _ in range(200): + foo.__class__ = Bar + type(foo) + foo.__class__ = Foo + type(foo) + + + threads = [] + for _ in range(6): + thread = threading.Thread(target=work) + thread.start() + threads.append(thread) + + for thread in threads: + thread.join() + + def test_special_methods(self): + for e in E(), E: + s = super(C, e) + self.assertEqual(s.__reduce__, e.__reduce__) + self.assertEqual(s.__reduce_ex__, e.__reduce_ex__) + self.assertEqual(s.__getstate__, e.__getstate__) + self.assertFalse(hasattr(s, '__getnewargs__')) + self.assertFalse(hasattr(s, '__getnewargs_ex__')) + self.assertFalse(hasattr(s, '__setstate__')) + self.assertFalse(hasattr(s, '__copy__')) + self.assertFalse(hasattr(s, '__deepcopy__')) + + def test_pickling(self): + e = E() + e.x = 1 + s = super(C, e) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + u = pickle.loads(pickle.dumps(s, proto)) + self.assertEqual(u.f(), s.f()) + self.assertIs(type(u), type(s)) + self.assertIs(type(u.__self__), E) + self.assertEqual(u.__self__.x, 1) + self.assertIs(u.__thisclass__, C) + self.assertIs(u.__self_class__, E) + + s = super(C, E) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + u = pickle.loads(pickle.dumps(s, proto)) + self.assertEqual(u.cm(), s.cm()) + self.assertEqual(u.f, s.f) + self.assertIs(type(u), type(s)) + self.assertIs(u.__self__, E) + self.assertIs(u.__thisclass__, C) + self.assertIs(u.__self_class__, E) + + def test_shallow_copying(self): + s = super(C, E()) + self.assertIs(copy.copy(s), s) + s = super(C, E) + self.assertIs(copy.copy(s), s) + + def test_deep_copying(self): + e = E() + e.x = [1] + s = super(C, e) + u = copy.deepcopy(s) + self.assertEqual(u.f(), s.f()) + self.assertIs(type(u), type(s)) + self.assertIsNot(u, s) + self.assertIs(type(u.__self__), E) + self.assertIsNot(u.__self__, e) + self.assertIsNot(u.__self__.x, e.x) + self.assertEqual(u.__self__.x, [1]) + self.assertIs(u.__thisclass__, C) + self.assertIs(u.__self_class__, E) + + s = super(C, E) + u = copy.deepcopy(s) + self.assertEqual(u.cm(), s.cm()) + self.assertEqual(u.f, s.f) + self.assertIsNot(u, s) + self.assertIs(type(u), type(s)) + self.assertIs(u.__self__, E) + self.assertIs(u.__thisclass__, C) + self.assertIs(u.__self_class__, E) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index d160cbf0645b47..d900db546ada8d 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -3,6 +3,7 @@ import io import os import shutil +import signal import socket import stat import subprocess @@ -70,7 +71,7 @@ def test_get_original_stdout(self): self.assertEqual(support.get_original_stdout(), sys.stdout) def test_unload(self): - import sched + import sched # noqa: F401 self.assertIn("sched", sys.modules) import_helper.unload("sched") self.assertNotIn("sched", sys.modules) @@ -547,13 +548,13 @@ def test_optim_args_from_interpreter_flags(self): with self.subTest(opts=opts): self.check_options(opts, 'optim_args_from_interpreter_flags') - @unittest.skipIf(support.is_emscripten, "Unstable in Emscripten") + @unittest.skipIf(support.is_apple_mobile, "Unstable on Apple Mobile") @unittest.skipIf(support.is_wasi, "Unavailable on WASI") def test_fd_count(self): - # We cannot test the absolute value of fd_count(): on old Linux - # kernel or glibc versions, os.urandom() keeps a FD open on - # /dev/urandom device and Python has 4 FD opens instead of 3. - # Test is unstable on Emscripten. The platform starts and stops + # We cannot test the absolute value of fd_count(): on old Linux kernel + # or glibc versions, os.urandom() keeps a FD open on /dev/urandom + # device and Python has 4 FD opens instead of 3. Test is unstable on + # Emscripten and Apple Mobile platforms; these platforms start and stop # background threads that use pipes and epoll fds. start = os_helper.fd_count() fd = os.open(__file__, os.O_RDONLY) @@ -576,7 +577,7 @@ def test_print_warning(self): 'Warning -- a\nWarning -- b\n') def test_has_strftime_extensions(self): - if support.is_emscripten or sys.platform == "win32": + if sys.platform == "win32": self.assertFalse(support.has_strftime_extensions) else: self.assertTrue(support.has_strftime_extensions) @@ -732,6 +733,17 @@ def test_copy_python_src_ignore(self): self.assertEqual(support.copy_python_src_ignore(path, os.listdir(path)), ignored) + def test_get_signal_name(self): + for exitcode, expected in ( + (-int(signal.SIGINT), 'SIGINT'), + (-int(signal.SIGSEGV), 'SIGSEGV'), + (128 + int(signal.SIGABRT), 'SIGABRT'), + (3221225477, "STATUS_ACCESS_VIOLATION"), + (0xC00000FD, "STATUS_STACK_OVERFLOW"), + ): + self.assertEqual(support.get_signal_name(exitcode), expected, + exitcode) + # XXX -follows a list of untested API # make_legacy_pyc # is_resource_enabled diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 92b78a8086a83d..24d89b09d946ad 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -1,6 +1,9 @@ """ Test the API of the symtable module. """ + +import re +import textwrap import symtable import unittest @@ -13,7 +16,7 @@ glob = 42 some_var = 12 -some_non_assigned_global_var = 11 +some_non_assigned_global_var: int some_assigned_global_var = 11 class Mine: @@ -49,10 +52,124 @@ def namespace_test(): pass def generic_spam[T](a): pass -class GenericMine[T: int]: +class GenericMine[T: int, U: (int, str) = int]: pass """ +TEST_COMPLEX_CLASS_CODE = """ +# The following symbols are defined in ComplexClass +# without being introduced by a 'global' statement. +glob_unassigned_meth: Any +glob_unassigned_meth_pep_695: Any + +glob_unassigned_async_meth: Any +glob_unassigned_async_meth_pep_695: Any + +def glob_assigned_meth(): pass +def glob_assigned_meth_pep_695[T](): pass + +async def glob_assigned_async_meth(): pass +async def glob_assigned_async_meth_pep_695[T](): pass + +# The following symbols are defined in ComplexClass after +# being introduced by a 'global' statement (and therefore +# are not considered as local symbols of ComplexClass). +glob_unassigned_meth_ignore: Any +glob_unassigned_meth_pep_695_ignore: Any + +glob_unassigned_async_meth_ignore: Any +glob_unassigned_async_meth_pep_695_ignore: Any + +def glob_assigned_meth_ignore(): pass +def glob_assigned_meth_pep_695_ignore[T](): pass + +async def glob_assigned_async_meth_ignore(): pass +async def glob_assigned_async_meth_pep_695_ignore[T](): pass + +class ComplexClass: + a_var = 1234 + a_genexpr = (x for x in []) + a_lambda = lambda x: x + + type a_type_alias = int + type a_type_alias_pep_695[T] = list[T] + + class a_class: pass + class a_class_pep_695[T]: pass + + def a_method(self): pass + def a_method_pep_695[T](self): pass + + async def an_async_method(self): pass + async def an_async_method_pep_695[T](self): pass + + @classmethod + def a_classmethod(cls): pass + @classmethod + def a_classmethod_pep_695[T](self): pass + + @classmethod + async def an_async_classmethod(cls): pass + @classmethod + async def an_async_classmethod_pep_695[T](self): pass + + @staticmethod + def a_staticmethod(): pass + @staticmethod + def a_staticmethod_pep_695[T](self): pass + + @staticmethod + async def an_async_staticmethod(): pass + @staticmethod + async def an_async_staticmethod_pep_695[T](self): pass + + # These ones will be considered as methods because of the 'def' although + # they are *not* valid methods at runtime since they are not decorated + # with @staticmethod. + def a_fakemethod(): pass + def a_fakemethod_pep_695[T](): pass + + async def an_async_fakemethod(): pass + async def an_async_fakemethod_pep_695[T](): pass + + # Check that those are still considered as methods + # since they are not using the 'global' keyword. + def glob_unassigned_meth(): pass + def glob_unassigned_meth_pep_695[T](): pass + + async def glob_unassigned_async_meth(): pass + async def glob_unassigned_async_meth_pep_695[T](): pass + + def glob_assigned_meth(): pass + def glob_assigned_meth_pep_695[T](): pass + + async def glob_assigned_async_meth(): pass + async def glob_assigned_async_meth_pep_695[T](): pass + + # The following are not picked as local symbols because they are not + # visible by the class at runtime (this is equivalent to having the + # definitions outside of the class). + global glob_unassigned_meth_ignore + def glob_unassigned_meth_ignore(): pass + global glob_unassigned_meth_pep_695_ignore + def glob_unassigned_meth_pep_695_ignore[T](): pass + + global glob_unassigned_async_meth_ignore + async def glob_unassigned_async_meth_ignore(): pass + global glob_unassigned_async_meth_pep_695_ignore + async def glob_unassigned_async_meth_pep_695_ignore[T](): pass + + global glob_assigned_meth_ignore + def glob_assigned_meth_ignore(): pass + global glob_assigned_meth_pep_695_ignore + def glob_assigned_meth_pep_695_ignore[T](): pass + + global glob_assigned_async_meth_ignore + async def glob_assigned_async_meth_ignore(): pass + global glob_assigned_async_meth_pep_695_ignore + async def glob_assigned_async_meth_pep_695_ignore[T](): pass +""" + def find_block(block, name): for ch in block.get_children(): @@ -65,6 +182,7 @@ class SymtableTest(unittest.TestCase): top = symtable.symtable(TEST_CODE, "?", "exec") # These correspond to scopes in TEST_CODE Mine = find_block(top, "Mine") + a_method = find_block(Mine, "a_method") spam = find_block(top, "spam") internal = find_block(spam, "internal") @@ -78,6 +196,7 @@ class SymtableTest(unittest.TestCase): GenericMine = find_block(top, "GenericMine") GenericMine_inner = find_block(GenericMine, "GenericMine") T = find_block(GenericMine, "T") + U = find_block(GenericMine, "U") def test_type(self): self.assertEqual(self.top.get_type(), "module") @@ -87,13 +206,14 @@ def test_type(self): self.assertEqual(self.internal.get_type(), "function") self.assertEqual(self.foo.get_type(), "function") self.assertEqual(self.Alias.get_type(), "type alias") - self.assertEqual(self.GenericAlias.get_type(), "type parameter") + self.assertEqual(self.GenericAlias.get_type(), "type parameters") self.assertEqual(self.GenericAlias_inner.get_type(), "type alias") - self.assertEqual(self.generic_spam.get_type(), "type parameter") + self.assertEqual(self.generic_spam.get_type(), "type parameters") self.assertEqual(self.generic_spam_inner.get_type(), "function") - self.assertEqual(self.GenericMine.get_type(), "type parameter") + self.assertEqual(self.GenericMine.get_type(), "type parameters") self.assertEqual(self.GenericMine_inner.get_type(), "class") - self.assertEqual(self.T.get_type(), "TypeVar bound") + self.assertEqual(self.T.get_type(), "type variable") + self.assertEqual(self.U.get_type(), "type variable") def test_id(self): self.assertGreater(self.top.get_id(), 0) @@ -205,12 +325,14 @@ def test_assigned(self): def test_annotated(self): st1 = symtable.symtable('def f():\n x: int\n', 'test', 'exec') - st2 = st1.get_children()[0] + st2 = st1.get_children()[1] + self.assertEqual(st2.get_type(), "function") self.assertTrue(st2.lookup('x').is_local()) self.assertTrue(st2.lookup('x').is_annotated()) self.assertFalse(st2.lookup('x').is_global()) st3 = symtable.symtable('def f():\n x = 1\n', 'test', 'exec') - st4 = st3.get_children()[0] + st4 = st3.get_children()[1] + self.assertEqual(st4.get_type(), "function") self.assertTrue(st4.lookup('x').is_local()) self.assertFalse(st4.lookup('x').is_annotated()) @@ -237,8 +359,86 @@ def test_name(self): self.assertEqual(self.spam.lookup("x").get_name(), "x") self.assertEqual(self.Mine.get_name(), "Mine") - def test_class_info(self): - self.assertEqual(self.Mine.get_methods(), ('a_method',)) + def test_class_get_methods(self): + deprecation_mess = ( + re.escape('symtable.Class.get_methods() is deprecated ' + 'and will be removed in Python 3.16.') + ) + + with self.assertWarnsRegex(DeprecationWarning, deprecation_mess): + self.assertEqual(self.Mine.get_methods(), ('a_method',)) + + top = symtable.symtable(TEST_COMPLEX_CLASS_CODE, "?", "exec") + this = find_block(top, "ComplexClass") + + with self.assertWarnsRegex(DeprecationWarning, deprecation_mess): + self.assertEqual(this.get_methods(), ( + 'a_method', 'a_method_pep_695', + 'an_async_method', 'an_async_method_pep_695', + 'a_classmethod', 'a_classmethod_pep_695', + 'an_async_classmethod', 'an_async_classmethod_pep_695', + 'a_staticmethod', 'a_staticmethod_pep_695', + 'an_async_staticmethod', 'an_async_staticmethod_pep_695', + 'a_fakemethod', 'a_fakemethod_pep_695', + 'an_async_fakemethod', 'an_async_fakemethod_pep_695', + 'glob_unassigned_meth', 'glob_unassigned_meth_pep_695', + 'glob_unassigned_async_meth', 'glob_unassigned_async_meth_pep_695', + 'glob_assigned_meth', 'glob_assigned_meth_pep_695', + 'glob_assigned_async_meth', 'glob_assigned_async_meth_pep_695', + )) + + # Test generator expressions that are of type TYPE_FUNCTION + # but will not be reported by get_methods() since they are + # not functions per se. + # + # Other kind of comprehensions such as list, set or dict + # expressions do not have the TYPE_FUNCTION type. + + def check_body(body, expected_methods): + indented = textwrap.indent(body, ' ' * 4) + top = symtable.symtable(f"class A:\n{indented}", "?", "exec") + this = find_block(top, "A") + with self.assertWarnsRegex(DeprecationWarning, deprecation_mess): + self.assertEqual(this.get_methods(), expected_methods) + + # statements with 'genexpr' inside it + GENEXPRS = ( + 'x = (x for x in [])', + 'x = (x async for x in [])', + 'type x[genexpr = (x for x in [])] = (x for x in [])', + 'type x[genexpr = (x async for x in [])] = (x async for x in [])', + 'genexpr = (x for x in [])', + 'genexpr = (x async for x in [])', + 'type genexpr[genexpr = (x for x in [])] = (x for x in [])', + 'type genexpr[genexpr = (x async for x in [])] = (x async for x in [])', + ) + + for gen in GENEXPRS: + # test generator expression + with self.subTest(gen=gen): + check_body(gen, ()) + + # test generator expression + variable named 'genexpr' + with self.subTest(gen=gen, isvar=True): + check_body('\n'.join((gen, 'genexpr = 1')), ()) + check_body('\n'.join(('genexpr = 1', gen)), ()) + + for paramlist in ('()', '(x)', '(x, y)', '(z: T)'): + for func in ( + f'def genexpr{paramlist}:pass', + f'async def genexpr{paramlist}:pass', + f'def genexpr[T]{paramlist}:pass', + f'async def genexpr[T]{paramlist}:pass', + ): + with self.subTest(func=func): + # test function named 'genexpr' + check_body(func, ('genexpr',)) + + for gen in GENEXPRS: + with self.subTest(gen=gen, func=func): + # test generator expression + function named 'genexpr' + check_body('\n'.join((gen, func)), ('genexpr',)) + check_body('\n'.join((func, gen)), ('genexpr',)) def test_filename_correct(self): ### Bug tickler: SyntaxError file name correct whether error raised @@ -299,12 +499,56 @@ def test_symbol_repr(self): "") self.assertEqual(repr(self.other_internal.lookup("some_var")), "") + self.assertEqual(repr(self.GenericMine.lookup("T")), + "") + + st1 = symtable.symtable("[x for x in [1]]", "?", "exec") + self.assertEqual(repr(st1.lookup("x")), + "") + + st2 = symtable.symtable("[(lambda: x) for x in [1]]", "?", "exec") + self.assertEqual(repr(st2.lookup("x")), + "") + + st3 = symtable.symtable("def f():\n" + " x = 1\n" + " class A:\n" + " x = 2\n" + " def method():\n" + " return x\n", + "?", "exec") + # child 0 is for __annotate__ + func_f = st3.get_children()[1] + class_A = func_f.get_children()[0] + self.assertEqual(repr(class_A.lookup('x')), + "") def test_symtable_entry_repr(self): expected = f"" self.assertEqual(repr(self.top._table), expected) +class ComprehensionTests(unittest.TestCase): + def get_identifiers_recursive(self, st, res): + res.extend(st.get_identifiers()) + for ch in st.get_children(): + self.get_identifiers_recursive(ch, res) + + def test_loopvar_in_only_one_scope(self): + # ensure that the loop variable appears only once in the symtable + comps = [ + "[x for x in [1]]", + "{x for x in [1]}", + "{x:x*x for x in [1]}", + ] + for comp in comps: + with self.subTest(comp=comp): + st = symtable.symtable(comp, "?", "exec") + ids = [] + self.get_identifiers_recursive(st, ids) + self.assertEqual(len([x for x in ids if x == 'x']), 1) + + class CommandLineTest(unittest.TestCase): maxDiff = None diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index ee9b873d9023f0..132e2b839627bc 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -59,6 +59,18 @@ Traceback (most recent call last): SyntaxError: cannot assign to __debug__ +>>> def __debug__(): pass +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + +>>> async def __debug__(): pass +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + +>>> class __debug__: pass +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + >>> del __debug__ Traceback (most recent call last): SyntaxError: cannot delete __debug__ @@ -786,6 +798,9 @@ >>> __debug__: int Traceback (most recent call last): SyntaxError: cannot assign to __debug__ +>>> x.__debug__: int +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ >>> f(a=) Traceback (most recent call last): SyntaxError: expected argument value expression @@ -1182,6 +1197,24 @@ Traceback (most recent call last): SyntaxError: expected ':' + >>> match x: + ... case a, __debug__, b: + ... pass + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + + >>> match x: + ... case a, b, *__debug__: + ... pass + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + + >>> match x: + ... case Foo(a, __debug__=1, b=2): + ... pass + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + >>> if x = 3: ... pass Traceback (most recent call last): @@ -1213,6 +1246,22 @@ Traceback (most recent call last): SyntaxError: expected '(' + >>> def f -> int: + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> async def f -> int: # type: int + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> async def f[T]: + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> def f[T] -> str: + Traceback (most recent call last): + SyntaxError: expected '(' + Parenthesized arguments in function definitions >>> def f(x, (y, z), w): @@ -1259,6 +1308,15 @@ Traceback (most recent call last): SyntaxError: expected 'except' or 'finally' block +Custom error message for __debug__ as exception variable + + >>> try: + ... pass + ... except TypeError as __debug__: + ... pass + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + Custom error message for try block mixing except and except* >>> try: @@ -1301,6 +1359,36 @@ Traceback (most recent call last): SyntaxError: cannot have both 'except' and 'except*' on the same 'try' +Better error message for using `except as` with not a name: + + >>> try: + ... pass + ... except TypeError as obj.attr: + ... pass + Traceback (most recent call last): + SyntaxError: cannot use except statement with attribute + + >>> try: + ... pass + ... except TypeError as obj[1]: + ... pass + Traceback (most recent call last): + SyntaxError: cannot use except statement with subscript + + >>> try: + ... pass + ... except* TypeError as (obj, name): + ... pass + Traceback (most recent call last): + SyntaxError: cannot use except* statement with tuple + + >>> try: + ... pass + ... except* TypeError as 1: + ... pass + Traceback (most recent call last): + SyntaxError: cannot use except* statement with literal + Ensure that early = are not matched by the parser as invalid comparisons >>> f(2, 4, x=34); 1 $ 2 Traceback (most recent call last): @@ -1506,6 +1594,19 @@ Traceback (most recent call last): IndentationError: expected an indented block after class definition on line 1 + >>> class C(__debug__=42): ... + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + + >>> class Meta(type): + ... def __new__(*args, **kwargs): + ... pass + + >>> class C(metaclass=Meta, __debug__=42): + ... pass + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + >>> match something: ... pass Traceback (most recent call last): @@ -1692,6 +1793,26 @@ Traceback (most recent call last): SyntaxError: Did you mean to use 'from ... import ...' instead? +>>> import __debug__ +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + +>>> import a as __debug__ +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + +>>> import a.b.c as __debug__ +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + +>>> from a import __debug__ +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + +>>> from a import b as __debug__ +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + # Check that we dont raise the "trailing comma" error if there is more # input to the left of the valid part that we parsed. @@ -1699,6 +1820,18 @@ Traceback (most recent call last): SyntaxError: invalid syntax +>>> from i import +Traceback (most recent call last): +SyntaxError: Expected one or more names after 'import' + +>>> from .. import +Traceback (most recent call last): +SyntaxError: Expected one or more names after 'import' + +>>> import +Traceback (most recent call last): +SyntaxError: Expected one or more names after 'import' + >>> (): int Traceback (most recent call last): SyntaxError: only single target (not tuple) can be annotated @@ -1712,6 +1845,49 @@ Traceback (most recent call last): SyntaxError: only single target (not list) can be annotated +# 'not' after operators: + +>>> 3 + not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> 3 * not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> + not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> - not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> ~ not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> 3 + - not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> 3 + not -1 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +# Check that we don't introduce misleading errors +>>> not 1 */ 2 +Traceback (most recent call last): +SyntaxError: invalid syntax + +>>> not 1 + +Traceback (most recent call last): +SyntaxError: invalid syntax + +>>> not + 1 + +Traceback (most recent call last): +SyntaxError: invalid syntax + Corner-cases that used to fail to raise the correct error: >>> def f(*, x=lambda __debug__:0): pass @@ -1756,53 +1932,55 @@ ... case 42 as 1+2+4: ... ... Traceback (most recent call last): - SyntaxError: invalid pattern target + SyntaxError: cannot use expression as pattern target >>> match ...: - ... case Foo(z=1, y=2, x): + ... case 42 as a.b: ... ... Traceback (most recent call last): - SyntaxError: positional patterns follow keyword patterns + SyntaxError: cannot use attribute as pattern target >>> match ...: - ... case Foo(a, z=1, y=2, x): + ... case 42 as (a, b): ... ... Traceback (most recent call last): - SyntaxError: positional patterns follow keyword patterns + SyntaxError: cannot use tuple as pattern target >>> match ...: - ... case Foo(z=1, x, y=2): + ... case 42 as (a + 1): ... ... Traceback (most recent call last): - SyntaxError: positional patterns follow keyword patterns + SyntaxError: cannot use expression as pattern target >>> match ...: - ... case C(a=b, c, d=e, f, g=h, i, j=k, ...): + ... case (32 as x) | (42 as a()): ... ... Traceback (most recent call last): - SyntaxError: positional patterns follow keyword patterns + SyntaxError: cannot use function call as pattern target -Non-matching 'elif'/'else' statements: - - >>> if a == b: + >>> match ...: + ... case Foo(z=1, y=2, x): ... ... - ... elif a == c: Traceback (most recent call last): - SyntaxError: 'elif' must match an if-statement here + SyntaxError: positional patterns follow keyword patterns - >>> if x == y: + >>> match ...: + ... case Foo(a, z=1, y=2, x): ... ... - ... else: Traceback (most recent call last): - SyntaxError: 'else' must match a valid statement here + SyntaxError: positional patterns follow keyword patterns - >>> elif m == n: + >>> match ...: + ... case Foo(z=1, x, y=2): + ... ... Traceback (most recent call last): - SyntaxError: 'elif' must match an if-statement here + SyntaxError: positional patterns follow keyword patterns - >>> else: + >>> match ...: + ... case C(a=b, c, d=e, f, g=h, i, j=k, ...): + ... ... Traceback (most recent call last): - SyntaxError: 'else' must match a valid statement here + SyntaxError: positional patterns follow keyword patterns Uses of the star operator which should fail: @@ -1868,22 +2046,22 @@ >>> A[*(1:2)] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> A[*(1:2)] = 1 Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> del A[*(1:2)] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression A[*:] and A[:*] >>> A[*:] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> A[:*] Traceback (most recent call last): ... @@ -1894,7 +2072,7 @@ >>> A[*] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression A[**] @@ -1972,16 +2150,116 @@ def f(x: *b) Invalid expressions in type scopes: + >>> type A[] = int + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> class A[]: ... + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def some[](): ... + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def some[]() + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> async def some[]: # type: int + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def f[T: (x:=3)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar bound + + >>> def f[T: ((x:= 3), int)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar constraint + + >>> def f[T = ((x:=3))](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar default + + >>> async def f[T: (x:=3)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar bound + + >>> async def f[T: ((x:= 3), int)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar constraint + + >>> async def f[T = ((x:=3))](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar default + >>> type A[T: (x:=3)] = int Traceback (most recent call last): ... SyntaxError: named expression cannot be used within a TypeVar bound + >>> type A[T: ((x:= 3), int)] = int + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar constraint + + >>> type A[T = ((x:=3))] = int + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar default + + >>> def f[T: (yield)](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> def f[T: (int, (yield))](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar constraint + + >>> def f[T = (yield)](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar default + + >>> def f[*Ts = (yield)](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVarTuple default + + >>> def f[**P = [(yield), int]](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a ParamSpec default + >>> type A[T: (yield 3)] = int Traceback (most recent call last): ... SyntaxError: yield expression cannot be used within a TypeVar bound + >>> type A[T: (int, (yield 3))] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar constraint + + >>> type A[T = (yield 3)] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar default + >>> type A[T: (await 3)] = int Traceback (most recent call last): ... @@ -1992,6 +2270,31 @@ def f(x: *b) ... SyntaxError: yield expression cannot be used within a TypeVar bound + >>> class A[T: (yield 3)]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> class A[T: (int, (yield 3))]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar constraint + + >>> class A[T = (yield)]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar default + + >>> class A[*Ts = (yield)]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVarTuple default + + >>> class A[**P = [(yield), int]]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a ParamSpec default + >>> type A = (x := 3) Traceback (most recent call last): ... @@ -2012,6 +2315,14 @@ def f(x: *b) ... SyntaxError: yield expression cannot be used within a type alias + >>> type __debug__ = int + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + + >>> class A[__debug__]: pass + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + >>> class A[T]((x := 3)): ... Traceback (most recent call last): ... @@ -2038,11 +2349,23 @@ def f(x: *b) >>> f(**x, *) Traceback (most recent call last): - SyntaxError: iterable argument unpacking follows keyword argument unpacking + SyntaxError: Invalid star expression >>> f(x, *:) Traceback (most recent call last): - SyntaxError: invalid syntax + SyntaxError: Invalid star expression + + >>> f(x, *) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x = 5, *) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x = 5, *:) + Traceback (most recent call last): + SyntaxError: Invalid star expression """ import re @@ -2059,8 +2382,8 @@ def _check_error(self, code, errtext, lineno=None, offset=None, end_lineno=None, end_offset=None): """Check that compiling code raises SyntaxError with errtext. - errtext is a regular expression that must be present in the - test of the exception raised. If subclass is specified, it + errtest is a regular expression that must be present in the + test of the exception raised. If subclass is specified it is the expected subclass of SyntaxError (e.g. IndentationError). """ try: @@ -2084,22 +2407,6 @@ def _check_error(self, code, errtext, else: self.fail("compile() did not raise SyntaxError") - def _check_noerror(self, code, - errtext="compile() raised unexpected SyntaxError", - filename="", mode="exec", subclass=None): - """Check that compiling code does not raise a SyntaxError. - - errtext is the message passed to self.fail if there is - a SyntaxError. If the subclass parameter is specified, - it is the subclass of SyntaxError (e.g. IndentationError) - that the raised error is checked against. - """ - try: - compile(code, filename, mode) - except SyntaxError as err: - if (not subclass) or isinstance(err, subclass): - self.fail(errtext) - def test_expression_with_assignment(self): self._check_error( "print(end1 + end2 = ' ')", @@ -2325,13 +2632,40 @@ def bug(): code += "): yield a" return code - CO_MAXBLOCKS = 20 # static nesting limit of the compiler + CO_MAXBLOCKS = 21 # static nesting limit of the compiler + MAX_MANAGERS = CO_MAXBLOCKS - 1 # One for the StopIteration block + + for n in range(MAX_MANAGERS): + with self.subTest(f"within range: {n=}"): + compile(get_code(n), "", "exec") + + for n in range(MAX_MANAGERS, MAX_MANAGERS + 5): + with self.subTest(f"out of range: {n=}"): + self._check_error(get_code(n), "too many statically nested blocks") + + @support.cpython_only + def test_async_with_statement_many_context_managers(self): + # See gh-116767 + + def get_code(n): + code = [ textwrap.dedent(""" + async def bug(): + async with ( + a + """) ] + for i in range(n): + code.append(f" as a{i}, a\n") + code.append("): yield a") + return "".join(code) + + CO_MAXBLOCKS = 21 # static nesting limit of the compiler + MAX_MANAGERS = CO_MAXBLOCKS - 1 # One for the StopIteration block - for n in range(CO_MAXBLOCKS): + for n in range(MAX_MANAGERS): with self.subTest(f"within range: {n=}"): compile(get_code(n), "", "exec") - for n in range(CO_MAXBLOCKS, CO_MAXBLOCKS + 5): + for n in range(MAX_MANAGERS, MAX_MANAGERS + 5): with self.subTest(f"out of range: {n=}"): self._check_error(get_code(n), "too many statically nested blocks") @@ -2469,29 +2803,11 @@ def test_syntax_error_on_deeply_nested_blocks(self): while 20: while 21: while 22: - break + while 23: + break """ self._check_error(source, "too many statically nested blocks") - def test_syntax_error_non_matching_elif_else_statements(self): - # Check bpo-45759: 'elif' statements that doesn't match an - # if-statement or 'else' statements that doesn't match any - # valid else-able statement (e.g. 'while') - self._check_error( - "elif m == n:\n ...", - "'elif' must match an if-statement here") - self._check_error( - "else:\n ...", - "'else' must match a valid statement here") - self._check_noerror("if a == b:\n ...\nelif a == c:\n ...") - self._check_noerror("if x == y:\n ...\nelse:\n ...") - self._check_error( - "else = 123", - "invalid syntax") - self._check_error( - "elif 55 = 123", - "cannot assign to literal here") - @support.cpython_only def test_error_on_parser_stack_overflow(self): source = "-" * 100000 + "4" @@ -2508,6 +2824,39 @@ def test_deep_invalid_rule(self): with self.assertRaises(SyntaxError): compile(source, "", "exec") + def test_except_stmt_invalid_as_expr(self): + self._check_error( + textwrap.dedent( + """ + try: + pass + except ValueError as obj.attr: + pass + """ + ), + errtext="cannot use except statement with attribute", + lineno=4, + end_lineno=4, + offset=22, + end_offset=22 + len("obj.attr"), + ) + + def test_match_stmt_invalid_as_expr(self): + self._check_error( + textwrap.dedent( + """ + match 1: + case x as obj.attr: + ... + """ + ), + errtext="cannot use attribute as pattern target", + lineno=3, + end_lineno=3, + offset=15, + end_offset=15 + len("obj.attr"), + ) + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite()) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 38dcabd84d8170..39857445a02255 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -16,6 +16,7 @@ from test.support.script_helper import assert_python_ok, assert_python_failure from test.support import threading_helper from test.support import import_helper +from test.support import force_not_colorized try: from test.support import interpreters except ImportError: @@ -145,6 +146,7 @@ def f(): class ExceptHookTest(unittest.TestCase): + @force_not_colorized def test_original_excepthook(self): try: raise ValueError(42) @@ -156,6 +158,7 @@ def test_original_excepthook(self): self.assertRaises(TypeError, sys.__excepthook__) + @force_not_colorized def test_excepthook_bytes_filename(self): # bpo-37467: sys.excepthook() must not crash if a filename # is a bytes string @@ -203,6 +206,20 @@ def test_exit(self): self.assertEqual(out, b'') self.assertEqual(err, b'') + # gh-125842: Windows uses 32-bit unsigned integers for exit codes + # so a -1 exit code is sometimes interpreted as 0xffff_ffff. + rc, out, err = assert_python_failure('-c', 'import sys; sys.exit(0xffff_ffff)') + self.assertIn(rc, (-1, 0xff, 0xffff_ffff)) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + + # Overflow results in a -1 exit code, which may be converted to 0xff + # or 0xffff_ffff. + rc, out, err = assert_python_failure('-c', 'import sys; sys.exit(2**128)') + self.assertIn(rc, (-1, 0xff, 0xffff_ffff)) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + # call with integer argument with self.assertRaises(SystemExit) as cm: sys.exit(42) @@ -391,10 +408,15 @@ def test_dlopenflags(self): @test.support.refcount_test def test_refcount(self): - # n here must be a global in order for this test to pass while - # tracing with a python function. Tracing calls PyFrame_FastToLocals - # which will add a copy of any locals to the frame object, causing - # the reference count to increase by 2 instead of 1. + # n here originally had to be a global in order for this test to pass + # while tracing with a python function. Tracing used to call + # PyFrame_FastToLocals, which would add a copy of any locals to the + # frame object, causing the ref count to increase by 2 instead of 1. + # While that no longer happens (due to PEP 667), this test case retains + # its original global-based implementation + # PEP 683's immortal objects also made this point moot, since the + # refcount for None doesn't change anyway. Maybe this test should be + # using a different constant value? (e.g. an integer) global n self.assertRaises(TypeError, sys.getrefcount) c = sys.getrefcount(None) @@ -562,7 +584,8 @@ def g456(): # And the next record must be for g456(). filename, lineno, funcname, sourceline = stack[i+1] self.assertEqual(funcname, "g456") - self.assertTrue(sourceline.startswith("if leave_g.wait(")) + self.assertTrue((sourceline.startswith("if leave_g.wait(") or + sourceline.startswith("g_raised.set()"))) finally: # Reap the spawned thread. leave_g.set() @@ -668,7 +691,7 @@ def test_thread_info(self): self.assertEqual(len(info), 3) self.assertIn(info.name, ('nt', 'pthread', 'pthread-stubs', 'solaris', None)) self.assertIn(info.lock, ('semaphore', 'mutex+cond', None)) - if sys.platform.startswith(("linux", "freebsd")): + if sys.platform.startswith(("linux", "android", "freebsd")): self.assertEqual(info.name, "pthread") elif sys.platform == "win32": self.assertEqual(info.name, "nt") @@ -722,8 +745,11 @@ def __hash__(self): if has_is_interned: self.assertIs(sys._is_interned(S("abc")), False) + @support.cpython_only @requires_subinterpreters def test_subinterp_intern_dynamically_allocated(self): + # Implementation detail: Dynamically allocated strings + # are distinct between interpreters s = "never interned before" + str(random.randrange(0, 10**9)) t = sys.intern(s) self.assertIs(t, s) @@ -731,24 +757,58 @@ def test_subinterp_intern_dynamically_allocated(self): interp = interpreters.create() interp.exec(textwrap.dedent(f''' import sys - t = sys.intern({s!r}) + + # set `s`, avoid parser interning & constant folding + s = str({s.encode()!r}, 'utf-8') + + t = sys.intern(s) + assert id(t) != {id(s)}, (id(t), {id(s)}) assert id(t) != {id(t)}, (id(t), {id(t)}) ''')) + @support.cpython_only @requires_subinterpreters def test_subinterp_intern_statically_allocated(self): + # Implementation detail: Statically allocated strings are shared + # between interpreters. # See Tools/build/generate_global_objects.py for the list # of strings that are always statically allocated. - s = '__init__' - t = sys.intern(s) + for s in ('__init__', 'CANCELLED', '', 'utf-8', + '{{', '', '\n', '_', 'x', '\0', '\N{CEDILLA}', '\xff', + ): + with self.subTest(s=s): + t = sys.intern(s) - interp = interpreters.create() - interp.exec(textwrap.dedent(f''' - import sys - t = sys.intern({s!r}) - assert id(t) == {id(t)}, (id(t), {id(t)}) - ''')) + interp = interpreters.create() + interp.exec(textwrap.dedent(f''' + import sys + + # set `s`, avoid parser interning & constant folding + s = str({s.encode()!r}, 'utf-8') + + t = sys.intern(s) + assert id(t) == {id(t)}, (id(t), {id(t)}) + ''')) + + @support.cpython_only + @requires_subinterpreters + def test_subinterp_intern_singleton(self): + # Implementation detail: singletons are used for 0- and 1-character + # latin1 strings. + for s in '', '\n', '_', 'x', '\0', '\N{CEDILLA}', '\xff': + with self.subTest(s=s): + interp = interpreters.create() + interp.exec(textwrap.dedent(f''' + import sys + + # set `s`, avoid parser interning & constant folding + s = str({s.encode()!r}, 'utf-8') + + assert id(s) == {id(s)} + t = sys.intern(s) + ''')) + self.assertTrue(sys._is_interned(s)) def test_sys_flags(self): self.assertTrue(sys.flags) @@ -792,6 +852,7 @@ def test_sys_getwindowsversion_no_instantiation(self): def test_clear_type_cache(self): sys._clear_type_cache() + @force_not_colorized @support.requires_subprocess() def test_ioencoding(self): env = dict(os.environ) @@ -995,14 +1056,10 @@ def test_debugmallocstats(self): # Output of sys._debugmallocstats() depends on configure flags. # The sysconfig vars are not available on Windows. if sys.platform != "win32": - with_freelists = sysconfig.get_config_var("WITH_FREELISTS") with_pymalloc = sysconfig.get_config_var("WITH_PYMALLOC") - if with_freelists: - self.assertIn(b"free PyDictObjects", err) + self.assertIn(b"free PyDictObjects", err) if with_pymalloc: self.assertIn(b'Small block threshold', err) - if not with_freelists and not with_pymalloc: - self.assertFalse(err) # The function has no parameter self.assertRaises(TypeError, sys._debugmallocstats, True) @@ -1037,7 +1094,14 @@ def test_getallocatedblocks(self): # While we could imagine a Python session where the number of # multiple buffer objects would exceed the sharing of references, # it is unlikely to happen in a normal test run. - self.assertLess(a, sys.gettotalrefcount()) + # + # In free-threaded builds each code object owns an array of + # pointers to copies of the bytecode. When the number of + # code objects is a large fraction of the total number of + # references, this can cause the total number of allocated + # blocks to exceed the total number of references. + if not support.Py_GIL_DISABLED: + self.assertLess(a, sys.gettotalrefcount()) except AttributeError: # gettotalrefcount() not available pass @@ -1048,6 +1112,12 @@ def test_getallocatedblocks(self): c = sys.getallocatedblocks() self.assertIn(c, range(b - 50, b + 50)) + def test_is_gil_enabled(self): + if support.Py_GIL_DISABLED: + self.assertIs(type(sys._is_gil_enabled()), bool) + else: + self.assertTrue(sys._is_gil_enabled()) + def test_is_finalizing(self): self.assertIs(sys.is_finalizing(), False) # Don't use the atexit module because _Py_Finalizing is only set @@ -1101,13 +1171,13 @@ def __del__(self): self.assertEqual(stdout.rstrip(), b"") self.assertEqual(stderr.rstrip(), b"") - @unittest.skipUnless(hasattr(sys, 'getandroidapilevel'), - 'need sys.getandroidapilevel()') + @unittest.skipUnless(sys.platform == "android", "Android only") def test_getandroidapilevel(self): level = sys.getandroidapilevel() self.assertIsInstance(level, int) self.assertGreater(level, 0) + @force_not_colorized @support.requires_subprocess() def test_sys_tracebacklimit(self): code = """if 1: @@ -1390,7 +1460,7 @@ class SizeofTest(unittest.TestCase): def setUp(self): self.P = struct.calcsize('P') self.longdigit = sys.int_info.sizeof_digit - import _testinternalcapi + _testinternalcapi = import_helper.import_module("_testinternalcapi") self.gc_headsize = _testinternalcapi.SIZEOF_PYGC_HEAD self.managed_pre_header_size = _testinternalcapi.SIZEOF_MANAGED_PRE_HEADER @@ -1550,10 +1620,14 @@ class C(object): pass def func(): return sys._getframe() x = func() - check(x, size('3Pi3c7P2ic??2P')) + if support.Py_GIL_DISABLED: + INTERPRETER_FRAME = '9PihcP' + else: + INTERPRETER_FRAME = '9PhcP' + check(x, size('3PiccPP' + INTERPRETER_FRAME + 'P')) # function def func(): pass - check(func, size('15Pi')) + check(func, size('16Pi')) class c(): @staticmethod def foo(): @@ -1567,7 +1641,7 @@ def bar(cls): check(bar, size('PP')) # generator def get_gen(): yield 1 - check(get_gen(), size('PP4P4c7P2ic??2P')) + check(get_gen(), size('6P4c' + INTERPRETER_FRAME + 'P')) # iterator check(iter('abc'), size('lP')) # callable-iterator @@ -1595,7 +1669,10 @@ def get_gen(): yield 1 check(int(PyLong_BASE**2-1), vsize('') + 2*self.longdigit) check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit) # module - check(unittest, size('PnPPP')) + if support.Py_GIL_DISABLED: + check(unittest, size('PPPPPP')) + else: + check(unittest, size('PPPPP')) # None check(None, size('')) # NotImplementedType @@ -1653,6 +1730,7 @@ def delx(self): del self.__x fmt = 'P2nPI13Pl4Pn9Pn12PIPc' s = vsize(fmt) check(int, s) + typeid = 'n' if support.Py_GIL_DISABLED else '' # class s = vsize(fmt + # PyTypeObject '4P' # PyAsyncMethods @@ -1660,8 +1738,9 @@ def delx(self): del self.__x '3P' # PyMappingMethods '10P' # PySequenceMethods '2P' # PyBufferProcs - '6P' - '1PIP' # Specializer cache + '7P' + '1PIP' # Specializer cache + + typeid # heap type id (free-threaded only) ) class newstyleclass(object): pass # Separate block for PyDictKeysObject with 8 keys and 5 entries @@ -1708,11 +1787,15 @@ class newstyleclass(object): pass # TODO: add check that forces layout of unicodefields # weakref import weakref - check(weakref.ref(int), size('2Pn3P')) + if support.Py_GIL_DISABLED: + expected = size('2Pn4P') + else: + expected = size('2Pn3P') + check(weakref.ref(int), expected) # weakproxy # XXX # weakcallableproxy - check(weakref.proxy(int), size('2Pn3P')) + check(weakref.proxy(int), expected) def check_slots(self, obj, base, extra): expected = sys.getsizeof(base) + struct.calcsize(extra) @@ -1762,7 +1845,8 @@ def test_pythontypes(self): # symtable entry # XXX # sys.flags - check(sys.flags, vsize('') + self.P * len(sys.flags)) + # FIXME: The +1 will not be necessary once gh-122575 is fixed + check(sys.flags, vsize('') + self.P * (1 + len(sys.flags))) def test_asyncgen_hooks(self): old = sys.get_asyncgen_hooks() @@ -1770,6 +1854,21 @@ def test_asyncgen_hooks(self): self.assertIsNone(old.finalizer) firstiter = lambda *a: None + finalizer = lambda *a: None + + with self.assertRaises(TypeError): + sys.set_asyncgen_hooks(firstiter=firstiter, finalizer="invalid") + cur = sys.get_asyncgen_hooks() + self.assertIsNone(cur.firstiter) + self.assertIsNone(cur.finalizer) + + # gh-118473 + with self.assertRaises(TypeError): + sys.set_asyncgen_hooks(firstiter="invalid", finalizer=finalizer) + cur = sys.get_asyncgen_hooks() + self.assertIsNone(cur.firstiter) + self.assertIsNone(cur.finalizer) + sys.set_asyncgen_hooks(firstiter=firstiter) hooks = sys.get_asyncgen_hooks() self.assertIs(hooks.firstiter, firstiter) @@ -1777,7 +1876,6 @@ def test_asyncgen_hooks(self): self.assertIs(hooks.finalizer, None) self.assertIs(hooks[1], None) - finalizer = lambda *a: None sys.set_asyncgen_hooks(finalizer=finalizer) hooks = sys.get_asyncgen_hooks() self.assertIs(hooks.firstiter, firstiter) diff --git a/Lib/test/test_sys_setprofile.py b/Lib/test/test_sys_setprofile.py index 32e03d7cd25dbe..311a4d2cafe88d 100644 --- a/Lib/test/test_sys_setprofile.py +++ b/Lib/test/test_sys_setprofile.py @@ -124,7 +124,7 @@ def f(p): def test_caught_exception(self): def f(p): try: 1/0 - except: pass + except ZeroDivisionError: pass f_ident = ident(f) self.check_events(f, [(1, 'call', f_ident), (1, 'return', f_ident), @@ -133,7 +133,7 @@ def f(p): def test_caught_nested_exception(self): def f(p): try: 1/0 - except: pass + except ZeroDivisionError: pass f_ident = ident(f) self.check_events(f, [(1, 'call', f_ident), (1, 'return', f_ident), @@ -156,9 +156,9 @@ def f(p): def g(p): try: f(p) - except: + except ZeroDivisionError: try: f(p) - except: pass + except ZeroDivisionError: pass f_ident = ident(f) g_ident = ident(g) self.check_events(g, [(1, 'call', g_ident), @@ -187,7 +187,7 @@ def g(p): def test_raise_twice(self): def f(p): try: 1/0 - except: 1/0 + except ZeroDivisionError: 1/0 f_ident = ident(f) self.check_events(f, [(1, 'call', f_ident), (1, 'return', f_ident), @@ -196,7 +196,7 @@ def f(p): def test_raise_reraise(self): def f(p): try: 1/0 - except: raise + except ZeroDivisionError: raise f_ident = ident(f) self.check_events(f, [(1, 'call', f_ident), (1, 'return', f_ident), @@ -320,7 +320,7 @@ def f(p): def test_caught_exception(self): def f(p): try: 1/0 - except: pass + except ZeroDivisionError: pass f_ident = ident(f) self.check_events(f, [(1, 'call', f_ident), (1, 'return', f_ident), @@ -479,6 +479,20 @@ def f(): sys.setprofile(lambda *args: None) f() + def test_method_with_c_function(self): + # gh-122029 + # When we have a PyMethodObject whose im_func is a C function, we + # should record both the call and the return. f = classmethod(repr) + # is just a way to create a PyMethodObject with a C function. + class A: + f = classmethod(repr) + events = [] + sys.setprofile(lambda frame, event, args: events.append(event)) + A().f() + sys.setprofile(None) + # The last c_call is the call to sys.setprofile + self.assertEqual(events, ['c_call', 'c_return', 'c_call']) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 125f40227118f6..28c2c681babe18 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -6,8 +6,7 @@ import difflib import gc from functools import wraps -import asyncio -from test.support import import_helper, requires_subprocess +from test.support import import_helper, requires_subprocess, run_no_yield_async_fn import contextlib import os import tempfile @@ -19,8 +18,6 @@ except ImportError: _testinternalcapi = None -support.requires_working_socket(module=True) - class tracecontext: """Context manager that traces its enter and exit.""" def __init__(self, output, value): @@ -1650,15 +1647,15 @@ def func(): EXPECTED_EVENTS = [ (0, 'call'), (2, 'line'), - (1, 'line'), (-3, 'call'), (-2, 'line'), (-2, 'return'), - (4, 'line'), (1, 'line'), + (4, 'line'), + (2, 'line'), (-2, 'call'), (-2, 'return'), - (1, 'return'), + (2, 'return'), ] # C level events should be the same as expected and the same as Python level. @@ -2067,10 +2064,9 @@ def run_async_test(self, func, jumpFrom, jumpTo, expected, error=None, stack.enter_context(self.assertRaisesRegex(*error)) if warning is not None: stack.enter_context(self.assertWarnsRegex(*warning)) - asyncio.run(func(output)) + run_no_yield_async_fn(func, output) sys.settrace(None) - asyncio.set_event_loop_policy(None) self.compare_jump_output(expected, output) def jump_test(jumpFrom, jumpTo, expected, error=None, event='line', warning=None): @@ -2857,7 +2853,7 @@ def test_no_jump_from_exception_event(output): output.append(1) 1 / 0 - @jump_test(3, 2, [2, 5], event='return') + @jump_test(3, 2, [2, 2, 5], event='return') def test_jump_from_yield(output): def gen(): output.append(2) @@ -3039,7 +3035,7 @@ def test_trace_unpack_long_sequence(self): def test_trace_lots_of_globals(self): - count = min(1000, int(support.Py_C_RECURSION_LIMIT * 0.8)) + count = min(1000, int(support.get_c_recursion_limit() * 0.8)) code = """if 1: def f(): diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index bb87bf00dc2d1a..3738914cf17de8 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -1,23 +1,34 @@ +import platform +import re import unittest import sys import os import subprocess import shutil +import json +import textwrap from copy import copy from test.support import ( - captured_stdout, PythonSymlink, requires_subprocess, is_wasi + captured_stdout, + is_android, + is_apple_mobile, + is_wasi, + PythonSymlink, + requires_subprocess, ) from test.support.import_helper import import_module from test.support.os_helper import (TESTFN, unlink, skip_unless_symlink, change_cwd) +from test.support.venv import VirtualEnvironmentMixin import sysconfig from sysconfig import (get_paths, get_platform, get_config_vars, get_path, get_path_names, _INSTALL_SCHEMES, get_default_scheme, get_scheme_names, get_config_var, - _expand_vars, _get_preferred_schemes) -from sysconfig.__main__ import _main, _parse_makefile + _expand_vars, _get_preferred_schemes, + is_python_build, _PROJECT_BASE) +from sysconfig.__main__ import _main, _parse_makefile, _get_pybuilddir, _get_json_data_name import _imp import _osx_support import _sysconfig @@ -26,10 +37,11 @@ HAS_USER_BASE = sysconfig._HAS_USER_BASE -class TestSysConfig(unittest.TestCase): +class TestSysConfig(unittest.TestCase, VirtualEnvironmentMixin): def setUp(self): super(TestSysConfig, self).setUp() + self.maxDiff = None self.sys_path = sys.path[:] # patching os.uname if hasattr(os, 'uname'): @@ -41,6 +53,8 @@ def setUp(self): os.uname = self._get_uname # saving the environment self.name = os.name + self.prefix = sys.prefix + self.exec_prefix = sys.exec_prefix self.platform = sys.platform self.version = sys.version self._framework = sys._framework @@ -65,6 +79,8 @@ def tearDown(self): else: del os.uname os.name = self.name + sys.prefix = self.prefix + sys.exec_prefix = self.exec_prefix sys.platform = self.platform sys.version = self.version sys._framework = self._framework @@ -151,7 +167,7 @@ def test_posix_venv_scheme(self): binpath = 'bin' incpath = 'include' libpath = os.path.join('lib', - 'python%d.%d' % sys.version_info[:2], + f'python{sysconfig._get_python_version_abi()}', 'site-packages') # Resolve the paths in an imaginary venv/ directory @@ -226,6 +242,11 @@ def test_get_config_vars(self): self.assertTrue(cvars) def test_get_platform(self): + # Check the actual platform returns something reasonable. + actual_platform = get_platform() + self.assertIsInstance(actual_platform, str) + self.assertTrue(actual_platform) + # windows XP, 32bits os.name = 'nt' sys.version = ('2.4.4 (#71, Oct 18 2006, 08:34:43) ' @@ -341,9 +362,26 @@ def test_get_platform(self): self.assertEqual(get_platform(), 'linux-i686') + # Android + os.name = 'posix' + sys.platform = 'android' + get_config_vars()['ANDROID_API_LEVEL'] = 9 + for machine, abi in { + 'x86_64': 'x86_64', + 'i686': 'x86', + 'aarch64': 'arm64_v8a', + 'armv7l': 'armeabi_v7a', + }.items(): + with self.subTest(machine): + self._set_uname(('Linux', 'localhost', '3.18.91+', + '#1 Tue Jan 9 20:35:43 UTC 2018', machine)) + self.assertEqual(get_platform(), f'android-9-{abi}') + # XXX more platforms to tests here @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't distribute header files in the runtime environment") def test_get_config_h_filename(self): config_h = sysconfig.get_config_h_filename() self.assertTrue(os.path.isfile(config_h), config_h) @@ -389,8 +427,8 @@ def test_user_similar(self): if name == 'platlib': # Replace "/lib64/python3.11/site-packages" suffix # with "/lib/python3.11/site-packages". - py_version_short = sysconfig.get_python_version() - suffix = f'python{py_version_short}/site-packages' + py_version_abi = sysconfig._get_python_version_abi() + suffix = f'python{py_version_abi}/site-packages' expected = expected.replace(f'/{sys.platlibdir}/{suffix}', f'/lib/{suffix}') self.assertEqual(user_path, expected) @@ -421,6 +459,9 @@ def test_library(self): self.assertTrue(library.startswith(f'python{major}{minor}')) self.assertTrue(library.endswith('.dll')) self.assertEqual(library, ldlibrary) + elif is_apple_mobile: + framework = sysconfig.get_config_var('PYTHONFRAMEWORK') + self.assertEqual(ldlibrary, f"{framework}.framework/{framework}") else: self.assertTrue(library.startswith(f'libpython{major}.{minor}')) self.assertTrue(library.endswith('.a')) @@ -474,6 +515,8 @@ def test_platform_in_subprocess(self): self.assertEqual(my_platform, test_platform) @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't include config folder at runtime") def test_srcdir(self): # See Issues #15322, #15364. srcdir = sysconfig.get_config_var('srcdir') @@ -516,12 +559,9 @@ def test_EXT_SUFFIX_in_vars(self): vars = sysconfig.get_config_vars() self.assertEqual(vars['EXT_SUFFIX'], _imp.extension_suffixes()[0]) - @unittest.skipUnless(sys.platform == 'linux' and - hasattr(sys.implementation, '_multiarch'), - 'multiarch-specific test') - def test_triplet_in_ext_suffix(self): + @unittest.skipUnless(sys.platform == 'linux', 'Linux-specific test') + def test_linux_ext_suffix(self): ctypes = import_module('ctypes') - import platform, re machine = platform.machine() suffix = sysconfig.get_config_var('EXT_SUFFIX') if re.match('(aarch64|arm|mips|ppc|powerpc|s390|sparc)', machine): @@ -534,16 +574,124 @@ def test_triplet_in_ext_suffix(self): self.assertTrue(suffix.endswith(expected_suffixes), f'unexpected suffix {suffix!r}') + @unittest.skipUnless(sys.platform == 'android', 'Android-specific test') + def test_android_ext_suffix(self): + machine = platform.machine() + suffix = sysconfig.get_config_var('EXT_SUFFIX') + expected_triplet = { + "x86_64": "x86_64-linux-android", + "i686": "i686-linux-android", + "aarch64": "aarch64-linux-android", + "armv7l": "arm-linux-androideabi", + }[machine] + self.assertTrue(suffix.endswith(f"-{expected_triplet}.so"), + f"{machine=}, {suffix=}") + @unittest.skipUnless(sys.platform == 'darwin', 'OS X-specific test') def test_osx_ext_suffix(self): suffix = sysconfig.get_config_var('EXT_SUFFIX') self.assertTrue(suffix.endswith('-darwin.so'), suffix) + @requires_subprocess() + def test_makefile_overwrites_config_vars(self): + script = textwrap.dedent(""" + import sys, sysconfig + + data = { + 'prefix': sys.prefix, + 'exec_prefix': sys.exec_prefix, + 'base_prefix': sys.base_prefix, + 'base_exec_prefix': sys.base_exec_prefix, + 'config_vars': sysconfig.get_config_vars(), + } + + import json + print(json.dumps(data, indent=2)) + """) + + # We need to run the test inside a virtual environment so that + # sys.prefix/sys.exec_prefix have a different value from the + # prefix/exec_prefix Makefile variables. + with self.venv() as venv: + data = json.loads(venv.run('-c', script).stdout) + + # We expect sysconfig.get_config_vars to correctly reflect sys.prefix/sys.exec_prefix + self.assertEqual(data['prefix'], data['config_vars']['prefix']) + self.assertEqual(data['exec_prefix'], data['config_vars']['exec_prefix']) + # As a sanity check, just make sure sys.prefix/sys.exec_prefix really + # are different from the Makefile values. + # sys.base_prefix/sys.base_exec_prefix should reflect the value of the + # prefix/exec_prefix Makefile variables, so we use them in the comparison. + self.assertNotEqual(data['prefix'], data['base_prefix']) + self.assertNotEqual(data['exec_prefix'], data['base_exec_prefix']) + + @unittest.skipIf(os.name != 'posix', '_sysconfig-vars JSON file is only available on POSIX') + @unittest.skipIf(is_wasi, "_sysconfig-vars JSON file currently isn't available on WASI") + @unittest.skipIf(is_android or is_apple_mobile, 'Android and iOS change the prefix') + def test_sysconfigdata_json(self): + if '_PYTHON_SYSCONFIGDATA_PATH' in os.environ: + data_dir = os.environ['_PYTHON_SYSCONFIGDATA_PATH'] + elif is_python_build(): + data_dir = os.path.join(_PROJECT_BASE, _get_pybuilddir()) + else: + data_dir = sys._stdlib_dir + + json_data_path = os.path.join(data_dir, _get_json_data_name()) + + with open(json_data_path) as f: + json_config_vars = json.load(f) + + system_config_vars = get_config_vars() + + ignore_keys = set() + # Keys dependent on Python being run outside the build directrory + if sysconfig.is_python_build(): + ignore_keys |= {'srcdir'} + # Keys dependent on the executable location + if os.path.dirname(sys.executable) != system_config_vars['BINDIR']: + ignore_keys |= {'projectbase'} + # Keys dependent on the environment (different inside virtual environments) + if sys.prefix != sys.base_prefix: + ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase'} + # Keys dependent on Python being run from the prefix targetted when building (different on relocatable installs) + if sysconfig._installation_is_relocated(): + ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase', 'installed_base', 'installed_platbase'} + + for key in ignore_keys: + json_config_vars.pop(key) + system_config_vars.pop(key) + + self.assertEqual(system_config_vars, json_config_vars) + + def test_sysconfig_config_vars_no_prefix_cache(self): + sys.prefix = 'prefix-AAA' + sys.exec_prefix = 'exec-prefix-AAA' + + config_vars = sysconfig.get_config_vars() + + self.assertEqual(config_vars['prefix'], sys.prefix) + self.assertEqual(config_vars['base'], sys.prefix) + self.assertEqual(config_vars['exec_prefix'], sys.exec_prefix) + self.assertEqual(config_vars['platbase'], sys.exec_prefix) + + sys.prefix = 'prefix-BBB' + sys.exec_prefix = 'exec-prefix-BBB' + + config_vars = sysconfig.get_config_vars() + + self.assertEqual(config_vars['prefix'], sys.prefix) + self.assertEqual(config_vars['base'], sys.prefix) + self.assertEqual(config_vars['exec_prefix'], sys.exec_prefix) + self.assertEqual(config_vars['platbase'], sys.exec_prefix) + + class MakefileTests(unittest.TestCase): @unittest.skipIf(sys.platform.startswith('win'), 'Test is not Windows compatible') @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't include config folder at runtime") def test_get_makefile_filename(self): makefile = sysconfig.get_makefile_filename() self.assertTrue(os.path.isfile(makefile), makefile) @@ -569,5 +717,38 @@ def test_parse_makefile(self): }) +class DeprecationTests(unittest.TestCase): + def deprecated(self, removal_version, deprecation_msg=None, error=Exception, error_msg=None): + if sys.version_info >= removal_version: + return self.assertRaises(error, msg=error_msg) + else: + return self.assertWarns(DeprecationWarning, msg=deprecation_msg) + + def test_expand_makefile_vars(self): + with self.deprecated( + removal_version=(3, 16), + deprecation_msg=( + 'sysconfig.expand_makefile_vars is deprecated and will be removed in ' + 'Python 3.16. Use sysconfig.get_paths(vars=...) instead.', + ), + error=AttributeError, + error_msg="module 'sysconfig' has no attribute 'expand_makefile_vars'", + ): + sysconfig.expand_makefile_vars('', {}) + + def test_is_python_build_check_home(self): + with self.deprecated( + removal_version=(3, 15), + deprecation_msg=( + 'The check_home argument of sysconfig.is_python_build is ' + 'deprecated and its value is ignored. ' + 'It will be removed in Python 3.15.' + ), + error=TypeError, + error_msg="is_python_build() takes 0 positional arguments but 1 were given", + ): + sysconfig.is_python_build('foo') + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py index cc122cafc7985c..30dcb3e3c4f4f9 100644 --- a/Lib/test/test_tabnanny.py +++ b/Lib/test/test_tabnanny.py @@ -315,7 +315,7 @@ def validate_cmd(self, *args, stdout="", stderr="", partial=False, expect_failur def test_with_errored_file(self): """Should displays error when errored python file is given.""" with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path: - stderr = f"{file_path!r}: Token Error: " + stderr = f"{file_path!r}: Indentation Error: " stderr += ('unindent does not match any outer indentation level' ' (, line 3)') self.validate_cmd(file_path, stderr=stderr, expect_failure=True) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index a047780fdd7e17..54d329a15d4d25 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -386,7 +386,7 @@ def test_is_tarfile_erroneous(self): self.assertFalse(tarfile.is_tarfile(tmpname)) # is_tarfile works on path-like objects - self.assertFalse(tarfile.is_tarfile(pathlib.Path(tmpname))) + self.assertFalse(tarfile.is_tarfile(os_helper.FakePath(tmpname))) # is_tarfile works on file objects with open(tmpname, "rb") as fobj: @@ -400,7 +400,7 @@ def test_is_tarfile_valid(self): self.assertTrue(tarfile.is_tarfile(self.tarname)) # is_tarfile works on path-like objects - self.assertTrue(tarfile.is_tarfile(pathlib.Path(self.tarname))) + self.assertTrue(tarfile.is_tarfile(os_helper.FakePath(self.tarname))) # is_tarfile works on file objects with open(self.tarname, "rb") as fobj: @@ -513,6 +513,7 @@ def test_extractfile_attrs(self): with self.tar.extractfile(file) as fobj: self.assertEqual(fobj.name, 'ustar/regtype') self.assertRaises(AttributeError, fobj.fileno) + self.assertEqual(fobj.mode, 'rb') self.assertIs(fobj.readable(), True) self.assertIs(fobj.writable(), False) if self.is_stream: @@ -523,6 +524,7 @@ def test_extractfile_attrs(self): self.assertIs(fobj.closed, True) self.assertEqual(fobj.name, 'ustar/regtype') self.assertRaises(AttributeError, fobj.fileno) + self.assertEqual(fobj.mode, 'rb') self.assertIs(fobj.readable(), True) self.assertIs(fobj.writable(), False) if self.is_stream: @@ -533,11 +535,8 @@ def test_extractfile_attrs(self): class MiscReadTestBase(CommonReadTest): is_stream = False - def requires_name_attribute(self): - pass def test_no_name_argument(self): - self.requires_name_attribute() with open(self.tarname, "rb") as fobj: self.assertIsInstance(fobj.name, str) with tarfile.open(fileobj=fobj, mode=self.mode) as tar: @@ -570,7 +569,6 @@ def test_int_name_attribute(self): self.assertIsNone(tar.name) def test_bytes_name_attribute(self): - self.requires_name_attribute() tarname = os.fsencode(self.tarname) with open(tarname, 'rb') as fobj: self.assertIsInstance(fobj.name, bytes) @@ -578,21 +576,23 @@ def test_bytes_name_attribute(self): self.assertIsInstance(tar.name, bytes) self.assertEqual(tar.name, os.path.abspath(fobj.name)) - def test_pathlike_name(self): - tarname = pathlib.Path(self.tarname) + def test_pathlike_name(self, tarname=None): + if tarname is None: + tarname = self.tarname + expected = os.path.abspath(tarname) + tarname = os_helper.FakePath(tarname) with tarfile.open(tarname, mode=self.mode) as tar: - self.assertIsInstance(tar.name, str) - self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + self.assertEqual(tar.name, expected) with self.taropen(tarname) as tar: - self.assertIsInstance(tar.name, str) - self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + self.assertEqual(tar.name, expected) with tarfile.TarFile.open(tarname, mode=self.mode) as tar: - self.assertIsInstance(tar.name, str) - self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + self.assertEqual(tar.name, expected) if self.suffix == '': with tarfile.TarFile(tarname, mode='r') as tar: - self.assertIsInstance(tar.name, str) - self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + self.assertEqual(tar.name, expected) + + def test_pathlike_bytes_name(self): + self.test_pathlike_name(os.fsencode(self.tarname)) def test_illegal_mode_arg(self): with open(tmpname, 'wb'): @@ -722,6 +722,24 @@ def format_mtime(mtime): tar.close() os_helper.rmtree(DIR) + @staticmethod + def test_extractall_default_filter(): + # Test that the default filter is now "data", and the other filter types are not used. + DIR = pathlib.Path(TEMPDIR) / "extractall_default_filter" + with ( + os_helper.temp_dir(DIR), + tarfile.open(tarname, encoding="iso8859-1") as tar, + unittest.mock.patch("tarfile.data_filter", wraps=tarfile.data_filter) as mock_data_filter, + unittest.mock.patch("tarfile.tar_filter", wraps=tarfile.tar_filter) as mock_tar_filter, + unittest.mock.patch("tarfile.fully_trusted_filter", wraps=tarfile.fully_trusted_filter) as mock_ft_filter + ): + directories = [t for t in tar if t.isdir()] + tar.extractall(DIR, directories) + + mock_data_filter.assert_called() + mock_ft_filter.assert_not_called() + mock_tar_filter.assert_not_called() + @os_helper.skip_unless_working_chmod def test_extract_directory(self): dirtype = "ustar/dirtype" @@ -738,24 +756,24 @@ def test_extract_directory(self): finally: os_helper.rmtree(DIR) - def test_extractall_pathlike_name(self): - DIR = pathlib.Path(TEMPDIR) / "extractall" + def test_extractall_pathlike_dir(self): + DIR = os.path.join(TEMPDIR, "extractall") with os_helper.temp_dir(DIR), \ tarfile.open(tarname, encoding="iso8859-1") as tar: directories = [t for t in tar if t.isdir()] - tar.extractall(DIR, directories, filter='fully_trusted') + tar.extractall(os_helper.FakePath(DIR), directories, filter='fully_trusted') for tarinfo in directories: - path = DIR / tarinfo.name + path = os.path.join(DIR, tarinfo.name) self.assertEqual(os.path.getmtime(path), tarinfo.mtime) - def test_extract_pathlike_name(self): + def test_extract_pathlike_dir(self): dirtype = "ustar/dirtype" - DIR = pathlib.Path(TEMPDIR) / "extractall" + DIR = os.path.join(TEMPDIR, "extractall") with os_helper.temp_dir(DIR), \ tarfile.open(tarname, encoding="iso8859-1") as tar: tarinfo = tar.getmember(dirtype) - tar.extract(tarinfo, path=DIR, filter='fully_trusted') - extracted = DIR / dirtype + tar.extract(tarinfo, path=os_helper.FakePath(DIR), filter='fully_trusted') + extracted = os.path.join(DIR, dirtype) self.assertEqual(os.path.getmtime(extracted), tarinfo.mtime) def test_init_close_fobj(self): @@ -814,12 +832,10 @@ class GzipMiscReadTest(GzipTest, MiscReadTestBase, unittest.TestCase): pass class Bz2MiscReadTest(Bz2Test, MiscReadTestBase, unittest.TestCase): - def requires_name_attribute(self): - self.skipTest("BZ2File have no name attribute") + pass class LzmaMiscReadTest(LzmaTest, MiscReadTestBase, unittest.TestCase): - def requires_name_attribute(self): - self.skipTest("LZMAFile have no name attribute") + pass class StreamReadTest(CommonReadTest, unittest.TestCase): @@ -1186,7 +1202,7 @@ def _fs_supports_holes(): # # The function returns False if page size is larger than 4 KiB. # For example, ppc64 uses pages of 64 KiB. - if sys.platform.startswith("linux"): + if sys.platform.startswith(("linux", "android")): # Linux evidentially has 512 byte st_blocks units. name = os.path.join(TEMPDIR, "sparse-test") with open(name, "wb") as fobj: @@ -1245,6 +1261,48 @@ def test_pax_number_fields(self): finally: tar.close() + def test_pax_header_bad_formats(self): + # The fields from the pax header have priority over the + # TarInfo. + pax_header_replacements = ( + b" foo=bar\n", + b"0 \n", + b"1 \n", + b"2 \n", + b"3 =\n", + b"4 =a\n", + b"1000000 foo=bar\n", + b"0 foo=bar\n", + b"-12 foo=bar\n", + b"000000000000000000000000036 foo=bar\n", + ) + pax_headers = {"foo": "bar"} + + for replacement in pax_header_replacements: + with self.subTest(header=replacement): + tar = tarfile.open(tmpname, "w", format=tarfile.PAX_FORMAT, + encoding="iso8859-1") + try: + t = tarfile.TarInfo() + t.name = "pax" # non-ASCII + t.uid = 1 + t.pax_headers = pax_headers + tar.addfile(t) + finally: + tar.close() + + with open(tmpname, "rb") as f: + data = f.read() + self.assertIn(b"11 foo=bar\n", data) + data = data.replace(b"11 foo=bar\n", replacement) + + with open(tmpname, "wb") as f: + f.truncate() + f.write(data) + + with self.assertRaisesRegex(tarfile.ReadError, r"method tar: ReadError\('invalid header'\)"): + tarfile.open(tmpname, encoding="iso8859-1") + class WriteTestBase(TarTest): # Put all write tests in here that are supposed to be tested @@ -1369,11 +1427,11 @@ def test_ordered_recursion(self): def test_gettarinfo_pathlike_name(self): with tarfile.open(tmpname, self.mode) as tar: - path = pathlib.Path(TEMPDIR) / "file" + path = os.path.join(TEMPDIR, "file") with open(path, "wb") as fobj: fobj.write(b"aaa") - tarinfo = tar.gettarinfo(path) - tarinfo2 = tar.gettarinfo(os.fspath(path)) + tarinfo = tar.gettarinfo(os_helper.FakePath(path)) + tarinfo2 = tar.gettarinfo(path) self.assertIsInstance(tarinfo.name, str) self.assertEqual(tarinfo.name, tarinfo2.name) self.assertEqual(tarinfo.size, 3) @@ -1587,6 +1645,12 @@ def write(self, data): pax_headers={'non': 'empty'}) self.assertFalse(f.closed) + def test_missing_fileobj(self): + with tarfile.open(tmpname, self.mode) as tar: + tarinfo = tar.gettarinfo(tarname) + with self.assertRaises(ValueError): + tar.addfile(tarinfo) + class GzipWriteTest(GzipTest, WriteTest): pass @@ -1920,10 +1984,10 @@ def test_create_existing_taropen(self): self.assertIn("spameggs42", names[0]) def test_create_pathlike_name(self): - with tarfile.open(pathlib.Path(tmpname), self.mode) as tobj: + with tarfile.open(os_helper.FakePath(tmpname), self.mode) as tobj: self.assertIsInstance(tobj.name, str) self.assertEqual(tobj.name, os.path.abspath(tmpname)) - tobj.add(pathlib.Path(self.file_path)) + tobj.add(os_helper.FakePath(self.file_path)) names = tobj.getnames() self.assertEqual(len(names), 1) self.assertIn('spameggs42', names[0]) @@ -1934,10 +1998,10 @@ def test_create_pathlike_name(self): self.assertIn('spameggs42', names[0]) def test_create_taropen_pathlike_name(self): - with self.taropen(pathlib.Path(tmpname), "x") as tobj: + with self.taropen(os_helper.FakePath(tmpname), "x") as tobj: self.assertIsInstance(tobj.name, str) self.assertEqual(tobj.name, os.path.abspath(tmpname)) - tobj.add(pathlib.Path(self.file_path)) + tobj.add(os_helper.FakePath(self.file_path)) names = tobj.getnames() self.assertEqual(len(names), 1) self.assertIn('spameggs42', names[0]) @@ -3258,7 +3322,8 @@ def test_add(self): tar = tarfile.open(fileobj=bio, mode='w', format=tarformat) tarinfo = tar.gettarinfo(tarname) try: - tar.addfile(tarinfo) + with open(tarname, 'rb') as f: + tar.addfile(tarinfo, f) except Exception: if tarformat == tarfile.USTAR_FORMAT: # In the old, limited format, adding might fail for @@ -3273,7 +3338,8 @@ def test_add(self): replaced = tarinfo.replace(**{attr_name: None}) with self.assertRaisesRegex(ValueError, f"{attr_name}"): - tar.addfile(replaced) + with open(tarname, 'rb') as f: + tar.addfile(replaced, f) def test_list(self): # Change some metadata to None, then compare list() output @@ -3980,15 +4046,6 @@ def test_data_filter(self): self.assertIs(filtered.name, tarinfo.name) self.assertIs(filtered.type, tarinfo.type) - def test_default_filter_warns(self): - """Ensure the default filter warns""" - with ArchiveMaker() as arc: - arc.add('foo') - with warnings_helper.check_warnings( - ('Python 3.14', DeprecationWarning)): - with self.check_context(arc.open(), None): - self.expect_file('foo') - def test_change_default_filter_on_instance(self): tar = tarfile.TarFile(tarname, 'r') def strict_filter(tarinfo, path): diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index ebdb58f91d3d8a..d479f7d7515d9b 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -51,7 +51,7 @@ def test_eval_null_in_result(self): def test_eval_surrogates_in_result(self): tcl = self.interp - self.assertIn(tcl.eval(r'set a "<\ud83d\udcbb>"'), '<\U0001f4bb>') + self.assertEqual(tcl.eval(r'set a "<\ud83d\udcbb>"'), '<\U0001f4bb>') def testEvalException(self): tcl = self.interp @@ -61,11 +61,30 @@ def testEvalException2(self): tcl = self.interp self.assertRaises(TclError,tcl.eval,'this is wrong') + def test_eval_returns_tcl_obj(self): + tcl = self.interp.tk + tcl.eval(r'set a "\u20ac \ud83d\udcbb \0 \udcab"; regexp -about $a') + a = tcl.eval('set a') + expected = '\u20ac \U0001f4bb \0 \udced\udcb2\udcab' + self.assertEqual(a, expected) + def testCall(self): tcl = self.interp tcl.call('set','a','1') self.assertEqual(tcl.call('set','a'),'1') + def test_call_passing_null(self): + tcl = self.interp + tcl.call('set', 'a', 'a\0b') # ASCII-only + self.assertEqual(tcl.getvar('a'), 'a\x00b') + self.assertEqual(tcl.call('set', 'a'), 'a\x00b') + self.assertEqual(tcl.eval('set a'), 'a\x00b') + + tcl.call('set', 'a', '\u20ac\0') # non-ASCII + self.assertEqual(tcl.getvar('a'), '\u20ac\x00') + self.assertEqual(tcl.call('set', 'a'), '\u20ac\x00') + self.assertEqual(tcl.eval('set a'), '\u20ac\x00') + def testCallException(self): tcl = self.interp self.assertRaises(TclError,tcl.call,'set','a') @@ -74,11 +93,35 @@ def testCallException2(self): tcl = self.interp self.assertRaises(TclError,tcl.call,'this','is','wrong') + def test_call_returns_tcl_obj(self): + tcl = self.interp.tk + tcl.eval(r'set a "\u20ac \ud83d\udcbb \0 \udcab"; regexp -about $a') + a = tcl.call('set', 'a') + expected = '\u20ac \U0001f4bb \0 \udced\udcb2\udcab' + if self.wantobjects: + self.assertEqual(str(a), expected) + self.assertEqual(a.string, expected) + self.assertEqual(a.typename, 'regexp') + else: + self.assertEqual(a, expected) + def testSetVar(self): tcl = self.interp tcl.setvar('a','1') self.assertEqual(tcl.eval('set a'),'1') + def test_setvar_passing_null(self): + tcl = self.interp + tcl.setvar('a', 'a\0b') # ASCII-only + self.assertEqual(tcl.getvar('a'), 'a\x00b') + self.assertEqual(tcl.call('set', 'a'), 'a\x00b') + self.assertEqual(tcl.eval('set a'), 'a\x00b') + + tcl.setvar('a', '\u20ac\0') # non-ASCII + self.assertEqual(tcl.getvar('a'), '\u20ac\x00') + self.assertEqual(tcl.call('set', 'a'), '\u20ac\x00') + self.assertEqual(tcl.eval('set a'), '\u20ac\x00') + def testSetVarArray(self): tcl = self.interp tcl.setvar('a(1)','1') @@ -102,6 +145,18 @@ def testGetVarArrayException(self): tcl = self.interp self.assertRaises(TclError,tcl.getvar,'a(1)') + def test_getvar_returns_tcl_obj(self): + tcl = self.interp.tk + tcl.eval(r'set a "\u20ac \ud83d\udcbb \0 \udcab"; regexp -about $a') + a = tcl.getvar('a') + expected = '\u20ac \U0001f4bb \0 \udced\udcb2\udcab' + if self.wantobjects: + self.assertEqual(str(a), expected) + self.assertEqual(a.string, expected) + self.assertEqual(a.typename, 'regexp') + else: + self.assertEqual(a, expected) + def testUnsetVar(self): tcl = self.interp tcl.setvar('a',1) @@ -219,10 +274,18 @@ def test_evalfile_surrogates_in_result(self): with open(filename, 'wb') as f: f.write(b""" set a "<\xed\xa0\xbd\xed\xb2\xbb>" + """) + if tcl_version >= (9, 0): + self.assertRaises(TclError, tcl.evalfile, filename) + else: + tcl.evalfile(filename) + self.assertEqual(tcl.eval('set a'), '<\U0001f4bb>') + + with open(filename, 'wb') as f: + f.write(b""" set b "<\\ud83d\\udcbb>" """) tcl.evalfile(filename) - self.assertEqual(tcl.eval('set a'), '<\U0001f4bb>') self.assertEqual(tcl.eval('set b'), '<\U0001f4bb>') def testEvalFileException(self): @@ -482,29 +545,36 @@ def testfunc(arg): return arg self.interp.createcommand('testfunc', testfunc) self.addCleanup(self.interp.tk.deletecommand, 'testfunc') - def check(value, expected=None, *, eq=self.assertEqual): - if expected is None: - expected = value + def check(value, expected1=None, expected2=None, *, eq=self.assertEqual): + expected = value + if self.wantobjects >= 2: + if expected2 is not None: + expected = expected2 + expected_type = type(expected) + else: + if expected1 is not None: + expected = expected1 + expected_type = str nonlocal result result = None r = self.interp.call('testfunc', value) - self.assertIsInstance(result, str) + self.assertIsInstance(result, expected_type) eq(result, expected) - self.assertIsInstance(r, str) + self.assertIsInstance(r, expected_type) eq(r, expected) def float_eq(actual, expected): self.assertAlmostEqual(float(actual), expected, delta=abs(expected) * 1e-10) - check(True, '1') - check(False, '0') + check(True, '1', 1) + check(False, '0', 0) check('string') check('string\xbd') check('string\u20ac') check('string\U0001f4bb') if sys.platform != 'win32': - check('<\udce2\udc82\udcac>', '<\u20ac>') - check('<\udced\udca0\udcbd\udced\udcb2\udcbb>', '<\U0001f4bb>') + check('<\udce2\udc82\udcac>', '<\u20ac>', '<\u20ac>') + check('<\udced\udca0\udcbd\udced\udcb2\udcbb>', '<\U0001f4bb>', '<\U0001f4bb>') check('') check(b'string', 'string') check(b'string\xe2\x82\xac', 'string\xe2\x82\xac') @@ -526,9 +596,31 @@ def float_eq(actual, expected): check(float('inf'), eq=float_eq) check(-float('inf'), eq=float_eq) # XXX NaN representation can be not parsable by float() - check((), '') - check((1, (2,), (3, 4), '5 6', ()), '1 2 {3 4} {5 6} {}') - check([1, [2,], [3, 4], '5 6', []], '1 2 {3 4} {5 6} {}') + check((), '', '') + check((1, (2,), (3, 4), '5 6', ()), + '1 2 {3 4} {5 6} {}', + (1, (2,), (3, 4), '5 6', '')) + check([1, [2,], [3, 4], '5 6', []], + '1 2 {3 4} {5 6} {}', + (1, (2,), (3, 4), '5 6', '')) + + def test_passing_tcl_obj(self): + tcl = self.interp.tk + a = None + def testfunc(arg): + nonlocal a + a = arg + self.interp.createcommand('testfunc', testfunc) + self.addCleanup(self.interp.tk.deletecommand, 'testfunc') + tcl.eval(r'set a "\u20ac \ud83d\udcbb \0 \udcab"; regexp -about $a') + tcl.eval(r'testfunc $a') + expected = '\u20ac \U0001f4bb \0 \udced\udcb2\udcab' + if self.wantobjects >= 2: + self.assertEqual(str(a), expected) + self.assertEqual(a.string, expected) + self.assertEqual(a.typename, 'regexp') + else: + self.assertEqual(a, expected) def test_splitlist(self): splitlist = self.interp.tk.splitlist @@ -654,6 +746,7 @@ def test_new_tcl_obj(self): support.check_disallow_instantiation(self, _tkinter.TkttType) support.check_disallow_instantiation(self, _tkinter.TkappType) + class BigmemTclTest(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index b64b6a4f2baeb5..7adc021d298254 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -13,6 +13,7 @@ import weakref import gc import shutil +import subprocess from unittest import mock import unittest @@ -62,16 +63,10 @@ def test_infer_return_type_multiples_and_none(self): tempfile._infer_return_type(b'', None, '') def test_infer_return_type_pathlib(self): - self.assertIs(str, tempfile._infer_return_type(pathlib.Path('/'))) + self.assertIs(str, tempfile._infer_return_type(os_helper.FakePath('/'))) def test_infer_return_type_pathlike(self): - class Path: - def __init__(self, path): - self.path = path - - def __fspath__(self): - return self.path - + Path = os_helper.FakePath self.assertIs(str, tempfile._infer_return_type(Path('/'))) self.assertIs(bytes, tempfile._infer_return_type(Path(b'/'))) self.assertIs(str, tempfile._infer_return_type('', Path(''))) @@ -333,10 +328,6 @@ def _mock_candidate_names(*names): class TestBadTempdir: - - @unittest.skipIf( - support.is_emscripten, "Emscripten cannot remove write bits." - ) def test_read_only_directory(self): with _inside_empty_temp_dir(): oldmode = mode = os.stat(tempfile.tempdir).st_mode @@ -442,7 +433,7 @@ def test_choose_directory(self): dir = tempfile.mkdtemp() try: self.do_create(dir=dir).write(b"blat") - self.do_create(dir=pathlib.Path(dir)).write(b"blat") + self.do_create(dir=os_helper.FakePath(dir)).write(b"blat") finally: support.gc_collect() # For PyPy or other GCs. os.rmdir(dir) @@ -680,7 +671,7 @@ def test_choose_directory(self): dir = tempfile.mkdtemp() try: self.do_create(dir=dir) - self.do_create(dir=pathlib.Path(dir)) + self.do_create(dir=os_helper.FakePath(dir)) finally: os.rmdir(dir) @@ -781,7 +772,7 @@ def test_choose_directory(self): dir = tempfile.mkdtemp() try: os.rmdir(self.do_create(dir=dir)) - os.rmdir(self.do_create(dir=pathlib.Path(dir))) + os.rmdir(self.do_create(dir=os_helper.FakePath(dir))) finally: os.rmdir(dir) @@ -803,6 +794,33 @@ def test_mode(self): finally: os.rmdir(dir) + @unittest.skipUnless(os.name == "nt", "Only on Windows.") + def test_mode_win32(self): + # Use icacls.exe to extract the users with some level of access + # Main thing we are testing is that the BUILTIN\Users group has + # no access. The exact ACL is going to vary based on which user + # is running the test. + dir = self.do_create() + try: + out = subprocess.check_output(["icacls.exe", dir], encoding="oem").casefold() + finally: + os.rmdir(dir) + + dir = dir.casefold() + users = set() + found_user = False + for line in out.strip().splitlines(): + acl = None + # First line of result includes our directory + if line.startswith(dir): + acl = line.removeprefix(dir).strip() + elif line and line[:1].isspace(): + acl = line.strip() + if acl: + users.add(acl.partition(":")[0]) + + self.assertNotIn(r"BUILTIN\Users".casefold(), users) + def test_collision_with_existing_file(self): # mkdtemp tries another name when a file with # the chosen name already exists @@ -1094,11 +1112,14 @@ def my_func(dir): # Testing extreme case, where the file is not explicitly closed # f.close() return tmp_name - # Make sure that the garbage collector has finalized the file object. - gc.collect() dir = tempfile.mkdtemp() try: - tmp_name = my_func(dir) + with self.assertWarnsRegex( + expected_warning=ResourceWarning, + expected_regex=r"Implicitly cleaning up <_TemporaryFileWrapper file=.*>", + ): + tmp_name = my_func(dir) + support.gc_collect() self.assertFalse(os.path.exists(tmp_name), f"NamedTemporaryFile {tmp_name!r} " f"exists after finalizer ") @@ -1218,9 +1239,6 @@ def test_del_unrolled_file(self): with self.assertWarns(ResourceWarning): f.__del__() - @unittest.skipIf( - support.is_emscripten, "Emscripten cannot fstat renamed files." - ) def test_del_rolled_file(self): # The rolled file should be deleted when the SpooledTemporaryFile # object is deleted. This should raise a ResourceWarning since the file @@ -1446,9 +1464,6 @@ def use_closed(): pass self.assertRaises(ValueError, use_closed) - @unittest.skipIf( - support.is_emscripten, "Emscripten cannot fstat renamed files." - ) def test_truncate_with_size_parameter(self): # A SpooledTemporaryFile can be truncated to zero size f = tempfile.SpooledTemporaryFile(max_size=10) diff --git a/Lib/test/test_termios.py b/Lib/test/test_termios.py index 58698ffac2d981..d6eb00175b4c85 100644 --- a/Lib/test/test_termios.py +++ b/Lib/test/test_termios.py @@ -3,6 +3,7 @@ import sys import tempfile import unittest +from test import support from test.support.import_helper import import_module termios = import_module('termios') @@ -18,10 +19,16 @@ def setUp(self): tmp = self.enterContext(tempfile.TemporaryFile(mode='wb', buffering=0)) self.bad_fd = tmp.fileno() - def assertRaisesTermiosError(self, errno, callable, *args): + def assertRaisesTermiosError(self, err, callable, *args): + # Some versions of Android return EACCES when calling termios functions + # on a regular file. + errs = [err] + if sys.platform == 'android' and err == errno.ENOTTY: + errs.append(errno.EACCES) + with self.assertRaises(termios.error) as cm: callable(*args) - self.assertEqual(cm.exception.args[0], errno) + self.assertIn(cm.exception.args[0], errs) def test_tcgetattr(self): attrs = termios.tcgetattr(self.fd) @@ -90,16 +97,18 @@ def test_tcsetattr_errors(self): self.assertRaises(TypeError, termios.tcsetattr, object(), termios.TCSANOW, attrs) self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW) + @support.skip_android_selinux('tcsendbreak') def test_tcsendbreak(self): try: termios.tcsendbreak(self.fd, 1) except termios.error as exc: - if exc.args[0] == errno.ENOTTY and sys.platform.startswith('freebsd'): + if exc.args[0] == errno.ENOTTY and sys.platform.startswith(('freebsd', "netbsd")): self.skipTest('termios.tcsendbreak() is not supported ' 'with pseudo-terminals (?) on this platform') raise termios.tcsendbreak(self.stream, 1) + @support.skip_android_selinux('tcsendbreak') def test_tcsendbreak_errors(self): self.assertRaises(OverflowError, termios.tcsendbreak, self.fd, 2**1000) self.assertRaises(TypeError, termios.tcsendbreak, self.fd, 0.0) @@ -110,10 +119,12 @@ def test_tcsendbreak_errors(self): self.assertRaises(TypeError, termios.tcsendbreak, object(), 0) self.assertRaises(TypeError, termios.tcsendbreak, self.fd) + @support.skip_android_selinux('tcdrain') def test_tcdrain(self): termios.tcdrain(self.fd) termios.tcdrain(self.stream) + @support.skip_android_selinux('tcdrain') def test_tcdrain_errors(self): self.assertRaisesTermiosError(errno.ENOTTY, termios.tcdrain, self.bad_fd) self.assertRaises(ValueError, termios.tcdrain, -1) @@ -136,12 +147,14 @@ def test_tcflush_errors(self): self.assertRaises(TypeError, termios.tcflush, object(), termios.TCIFLUSH) self.assertRaises(TypeError, termios.tcflush, self.fd) + @support.skip_android_selinux('tcflow') def test_tcflow(self): termios.tcflow(self.fd, termios.TCOOFF) termios.tcflow(self.fd, termios.TCOON) termios.tcflow(self.fd, termios.TCIOFF) termios.tcflow(self.fd, termios.TCION) + @support.skip_android_selinux('tcflow') def test_tcflow_errors(self): self.assertRaisesTermiosError(errno.EINVAL, termios.tcflow, self.fd, -1) self.assertRaises(OverflowError, termios.tcflow, self.fd, 2**1000) @@ -211,6 +224,15 @@ def test_constants(self): self.assertLess(termios.VTIME, termios.NCCS) self.assertLess(termios.VMIN, termios.NCCS) + def test_ioctl_constants(self): + # gh-119770: ioctl() constants must be positive + for name in dir(termios): + if not name.startswith('TIO'): + continue + value = getattr(termios, name) + with self.subTest(name=name): + self.assertGreaterEqual(value, 0) + def test_exception(self): self.assertTrue(issubclass(termios.error, Exception)) self.assertFalse(issubclass(termios.error, OSError)) diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index 83235230d5c112..d94e04250c9307 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -289,6 +289,54 @@ def joiner(): with self.assertRaisesRegex(RuntimeError, "Cannot join current thread"): raise error + def test_join_with_timeout(self): + lock = thread.allocate_lock() + lock.acquire() + + def thr(): + lock.acquire() + + with threading_helper.wait_threads_exit(): + handle = thread.start_joinable_thread(thr) + handle.join(0.1) + self.assertFalse(handle.is_done()) + lock.release() + handle.join() + self.assertTrue(handle.is_done()) + + def test_join_unstarted(self): + handle = thread._ThreadHandle() + with self.assertRaisesRegex(RuntimeError, "thread not started"): + handle.join() + + def test_set_done_unstarted(self): + handle = thread._ThreadHandle() + with self.assertRaisesRegex(RuntimeError, "thread not started"): + handle._set_done() + + def test_start_duplicate_handle(self): + lock = thread.allocate_lock() + lock.acquire() + + def func(): + lock.acquire() + + handle = thread._ThreadHandle() + with threading_helper.wait_threads_exit(): + thread.start_joinable_thread(func, handle=handle) + with self.assertRaisesRegex(RuntimeError, "thread already started"): + thread.start_joinable_thread(func, handle=handle) + lock.release() + handle.join() + + def test_start_with_none_handle(self): + def func(): + pass + + with threading_helper.wait_threads_exit(): + handle = thread.start_joinable_thread(func, handle=None) + handle.join() + class Barrier: def __init__(self, num_threads): diff --git a/Lib/test/test_thread_local_bytecode.py b/Lib/test/test_thread_local_bytecode.py new file mode 100644 index 00000000000000..7a8809c5ae7697 --- /dev/null +++ b/Lib/test/test_thread_local_bytecode.py @@ -0,0 +1,198 @@ +"""Tests for thread-local bytecode.""" +import dis +import textwrap +import unittest + +from test import support +from test.support import cpython_only, import_helper, requires_specialization_ft +from test.support.script_helper import assert_python_ok +from test.support.threading_helper import requires_working_threading + +# Skip this test if the _testinternalcapi module isn't available +_testinternalcapi = import_helper.import_module("_testinternalcapi") + + +@cpython_only +@requires_working_threading() +@unittest.skipUnless(support.Py_GIL_DISABLED, "only in free-threaded builds") +class TLBCTests(unittest.TestCase): + @requires_specialization_ft + def test_new_threads_start_with_unspecialized_code(self): + code = textwrap.dedent(""" + import dis + import queue + import threading + + from _testinternalcapi import get_tlbc + + def all_opnames(bc): + return {i.opname for i in dis._get_instructions_bytes(bc)} + + def f(a, b, q=None): + if q is not None: + q.put(get_tlbc(f)) + return a + b + + for _ in range(100): + # specialize + f(1, 2) + + q = queue.Queue() + t = threading.Thread(target=f, args=('a', 'b', q)) + t.start() + t.join() + + assert "BINARY_OP_ADD_INT" in all_opnames(get_tlbc(f)) + assert "BINARY_OP_ADD_INT" not in all_opnames(q.get()) + """) + assert_python_ok("-X", "tlbc=1", "-c", code) + + @requires_specialization_ft + def test_threads_specialize_independently(self): + code = textwrap.dedent(""" + import dis + import queue + import threading + + from _testinternalcapi import get_tlbc + + def all_opnames(bc): + return {i.opname for i in dis._get_instructions_bytes(bc)} + + def f(a, b): + return a + b + + def g(a, b, q=None): + for _ in range(100): + f(a, b) + if q is not None: + q.put(get_tlbc(f)) + + # specialize in main thread + g(1, 2) + + # specialize in other thread + q = queue.Queue() + t = threading.Thread(target=g, args=('a', 'b', q)) + t.start() + t.join() + + assert "BINARY_OP_ADD_INT" in all_opnames(get_tlbc(f)) + t_opnames = all_opnames(q.get()) + assert "BINARY_OP_ADD_INT" not in t_opnames + assert "BINARY_OP_ADD_UNICODE" in t_opnames + """) + assert_python_ok("-X", "tlbc=1", "-c", code) + + def test_reuse_tlbc_across_threads_different_lifetimes(self): + code = textwrap.dedent(""" + import queue + import threading + + from _testinternalcapi import get_tlbc_id + + def f(a, b, q=None): + if q is not None: + q.put(get_tlbc_id(f)) + return a + b + + q = queue.Queue() + tlbc_ids = [] + for _ in range(3): + t = threading.Thread(target=f, args=('a', 'b', q)) + t.start() + t.join() + tlbc_ids.append(q.get()) + + assert tlbc_ids[0] == tlbc_ids[1] + assert tlbc_ids[1] == tlbc_ids[2] + """) + assert_python_ok("-X", "tlbc=1", "-c", code) + + def test_no_copies_if_tlbc_disabled(self): + code = textwrap.dedent(""" + import queue + import threading + + from _testinternalcapi import get_tlbc_id + + def f(a, b, q=None): + if q is not None: + q.put(get_tlbc_id(f)) + return a + b + + q = queue.Queue() + threads = [] + for _ in range(3): + t = threading.Thread(target=f, args=('a', 'b', q)) + t.start() + threads.append(t) + + tlbc_ids = [] + for t in threads: + t.join() + tlbc_ids.append(q.get()) + + main_tlbc_id = get_tlbc_id(f) + assert main_tlbc_id is not None + assert tlbc_ids[0] == main_tlbc_id + assert tlbc_ids[1] == main_tlbc_id + assert tlbc_ids[2] == main_tlbc_id + """) + assert_python_ok("-X", "tlbc=0", "-c", code) + + def test_no_specialization_if_tlbc_disabled(self): + code = textwrap.dedent(""" + import dis + import queue + import threading + + from _testinternalcapi import get_tlbc + + def all_opnames(f): + bc = get_tlbc(f) + return {i.opname for i in dis._get_instructions_bytes(bc)} + + def f(a, b): + return a + b + + for _ in range(100): + f(1, 2) + + assert "BINARY_OP_ADD_INT" not in all_opnames(f) + """) + assert_python_ok("-X", "tlbc=0", "-c", code) + + def test_generator_throw(self): + code = textwrap.dedent(""" + import queue + import threading + + from _testinternalcapi import get_tlbc_id + + def g(): + try: + yield + except: + yield get_tlbc_id(g) + + def f(q): + gen = g() + next(gen) + q.put(gen.throw(ValueError)) + + q = queue.Queue() + t = threading.Thread(target=f, args=(q,)) + t.start() + t.join() + + gen = g() + next(gen) + main_id = gen.throw(ValueError) + assert main_id != q.get() + """) + assert_python_ok("-X", "tlbc=1", "-c", code) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 3b5c37c948c8c3..214e1ba0b53dd2 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -3,10 +3,11 @@ """ import test.support -from test.support import threading_helper, requires_subprocess +from test.support import threading_helper, requires_subprocess, requires_gil_enabled from test.support import verbose, cpython_only, os_helper from test.support.import_helper import import_module from test.support.script_helper import assert_python_ok, assert_python_failure +from test.support import force_not_colorized import random import sys @@ -47,6 +48,8 @@ def skip_unless_reliable_fork(test): return unittest.skip("due to known OS bug related to thread+fork")(test) if support.HAVE_ASAN_FORK_BUG: return unittest.skip("libasan has a pthread_create() dead lock related to thread+fork")(test) + if support.check_sanitizer(thread=True): + return unittest.skip("TSAN doesn't support threads after fork")(test) return test @@ -406,7 +409,7 @@ def run(self): def test_limbo_cleanup(self): # Issue 7481: Failure to start thread should cleanup the limbo map. - def fail_new_thread(*args): + def fail_new_thread(*args, **kwargs): raise threading.ThreadError() _start_joinable_thread = threading._start_joinable_thread threading._start_joinable_thread = fail_new_thread @@ -423,6 +426,10 @@ def test_finalize_running_thread(self): # Issue 1402: the PyGILState_Ensure / _Release functions may be called # very late on python exit: on deallocation of a running thread for # example. + if support.check_sanitizer(thread=True): + # the thread running `time.sleep(100)` below will still be alive + # at process exit + self.skipTest("TSAN would report thread leak") import_module("ctypes") rc, out, err = assert_python_failure("-c", """if 1: @@ -455,6 +462,11 @@ def waitingThread(): def test_finalize_with_trace(self): # Issue1733757 # Avoid a deadlock when sys.settrace steps into threading._shutdown + if support.check_sanitizer(thread=True): + # the thread running `time.sleep(2)` below will still be alive + # at process exit + self.skipTest("TSAN would report thread leak") + assert_python_ok("-c", """if 1: import sys, threading @@ -504,7 +516,7 @@ def test_enumerate_after_join(self): old_interval = sys.getswitchinterval() try: for i in range(1, 100): - sys.setswitchinterval(i * 0.0002) + support.setswitchinterval(i * 0.0002) t = threading.Thread(target=lambda: None) t.start() t.join() @@ -769,8 +781,7 @@ def func(): "current is main True\n" ) - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - @support.requires_fork() + @skip_unless_reliable_fork @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") def test_main_thread_after_fork_from_foreign_thread(self, create_dummy=False): code = """if 1: @@ -901,41 +912,6 @@ def f(): rc, out, err = assert_python_ok("-c", code) self.assertEqual(err, b"") - def test_tstate_lock(self): - # Test an implementation detail of Thread objects. - started = _thread.allocate_lock() - finish = _thread.allocate_lock() - started.acquire() - finish.acquire() - def f(): - started.release() - finish.acquire() - time.sleep(0.01) - # The tstate lock is None until the thread is started - t = threading.Thread(target=f) - self.assertIs(t._tstate_lock, None) - t.start() - started.acquire() - self.assertTrue(t.is_alive()) - # The tstate lock can't be acquired when the thread is running - # (or suspended). - tstate_lock = t._tstate_lock - self.assertFalse(tstate_lock.acquire(timeout=0), False) - finish.release() - # When the thread ends, the state_lock can be successfully - # acquired. - self.assertTrue(tstate_lock.acquire(timeout=support.SHORT_TIMEOUT), False) - # But is_alive() is still True: we hold _tstate_lock now, which - # prevents is_alive() from knowing the thread's end-of-life C code - # is done. - self.assertTrue(t.is_alive()) - # Let is_alive() find out the C code is done. - tstate_lock.release() - self.assertFalse(t.is_alive()) - # And verify the thread disposed of _tstate_lock. - self.assertIsNone(t._tstate_lock) - t.join() - def test_repr_stopped(self): # Verify that "stopped" shows up in repr(Thread) appropriately. started = _thread.allocate_lock() @@ -983,6 +959,7 @@ def test_BoundedSemaphore_limit(self): @cpython_only def test_frame_tstate_tracing(self): + _testcapi = import_module("_testcapi") # Issue #14432: Crash when a generator is created in a C thread that is # destroyed while the generator is still used. The issue was that a # generator contains a frame, and the frame kept a reference to the @@ -1010,7 +987,6 @@ def callback(): threading.settrace(noop_trace) # Create a generator in a C thread which exits after the call - import _testcapi _testcapi.call_in_temporary_c_thread(callback) # Call the generator in a different Python thread, check that the @@ -1101,30 +1077,6 @@ def checker(): self.assertEqual(threading.getprofile(), old_profile) self.assertEqual(sys.getprofile(), old_profile) - @cpython_only - def test_shutdown_locks(self): - for daemon in (False, True): - with self.subTest(daemon=daemon): - event = threading.Event() - thread = threading.Thread(target=event.wait, daemon=daemon) - - # Thread.start() must add lock to _shutdown_locks, - # but only for non-daemon thread - thread.start() - tstate_lock = thread._tstate_lock - if not daemon: - self.assertIn(tstate_lock, threading._shutdown_locks) - else: - self.assertNotIn(tstate_lock, threading._shutdown_locks) - - # unblock the thread and join it - event.set() - thread.join() - - # Thread._stop() must remove tstate_lock from _shutdown_locks. - # Daemon threads must never add it to _shutdown_locks. - self.assertNotIn(tstate_lock, threading._shutdown_locks) - def test_locals_at_exit(self): # bpo-19466: thread locals must not be deleted before destructors # are called @@ -1202,23 +1154,128 @@ def import_threading(): self.assertEqual(out, b'') self.assertEqual(err, b'') - def test_start_new_thread_at_exit(self): + def test_start_new_thread_at_finalization(self): code = """if 1: - import atexit import _thread def f(): print("shouldn't be printed") - def exit_handler(): - _thread.start_new_thread(f, ()) - - atexit.register(exit_handler) + class AtFinalization: + def __del__(self): + print("OK") + _thread.start_new_thread(f, ()) + at_finalization = AtFinalization() """ _, out, err = assert_python_ok("-c", code) - self.assertEqual(out, b'') + self.assertEqual(out.strip(), b"OK") self.assertIn(b"can't create new thread at interpreter shutdown", err) + def test_start_new_thread_failed(self): + # gh-109746: if Python fails to start newly created thread + # due to failure of underlying PyThread_start_new_thread() call, + # its state should be removed from interpreter' thread states list + # to avoid its double cleanup + try: + from resource import setrlimit, RLIMIT_NPROC + except ImportError as err: + self.skipTest(err) # RLIMIT_NPROC is specific to Linux and BSD + code = """if 1: + import resource + import _thread + + def f(): + print("shouldn't be printed") + + limits = resource.getrlimit(resource.RLIMIT_NPROC) + [_, hard] = limits + resource.setrlimit(resource.RLIMIT_NPROC, (0, hard)) + + try: + handle = _thread.start_joinable_thread(f) + except RuntimeError: + print('ok') + else: + print('!skip!') + handle.join() + """ + _, out, err = assert_python_ok("-u", "-c", code) + out = out.strip() + if b'!skip!' in out: + self.skipTest('RLIMIT_NPROC had no effect; probably superuser') + self.assertEqual(out, b'ok') + self.assertEqual(err, b'') + + @cpython_only + def test_finalize_daemon_thread_hang(self): + if support.check_sanitizer(thread=True, memory=True): + # the thread running `time.sleep(100)` below will still be alive + # at process exit + self.skipTest( + "https://github.com/python/cpython/issues/124878 - Known" + " race condition that TSAN identifies.") + # gh-87135: tests that daemon threads hang during finalization + script = textwrap.dedent(''' + import os + import sys + import threading + import time + import _testcapi + + lock = threading.Lock() + lock.acquire() + thread_started_event = threading.Event() + def thread_func(): + try: + thread_started_event.set() + _testcapi.finalize_thread_hang(lock.acquire) + finally: + # Control must not reach here. + os._exit(2) + + t = threading.Thread(target=thread_func) + t.daemon = True + t.start() + thread_started_event.wait() + # Sleep to ensure daemon thread is blocked on `lock.acquire` + # + # Note: This test is designed so that in the unlikely case that + # `0.1` seconds is not sufficient time for the thread to become + # blocked on `lock.acquire`, the test will still pass, it just + # won't be properly testing the thread behavior during + # finalization. + time.sleep(0.1) + + def run_during_finalization(): + # Wake up daemon thread + lock.release() + # Sleep to give the daemon thread time to crash if it is going + # to. + # + # Note: If due to an exceptionally slow execution this delay is + # insufficient, the test will still pass but will simply be + # ineffective as a test. + time.sleep(0.1) + # If control reaches here, the test succeeded. + os._exit(0) + + # Replace sys.stderr.flush as a way to run code during finalization + orig_flush = sys.stderr.flush + def do_flush(*args, **kwargs): + orig_flush(*args, **kwargs) + if not sys.is_finalizing: + return + sys.stderr.flush = orig_flush + run_during_finalization() + + sys.stderr.flush = do_flush + + # If the follow exit code is retained, `run_during_finalization` + # did not run. + sys.exit(1) + ''') + assert_python_ok("-c", script) + class ThreadJoinOnShutdown(BaseTestCase): def _run_and_join(self, script): @@ -1303,6 +1360,11 @@ def test_4_daemon_threads(self): # Check that a daemon thread cannot crash the interpreter on shutdown # by manipulating internal structures that are being disposed of in # the main thread. + if support.check_sanitizer(thread=True): + # some of the threads running `random_io` below will still be alive + # at process exit + self.skipTest("TSAN would report thread leak") + script = """if True: import os import random @@ -1340,6 +1402,30 @@ def main(): rc, out, err = assert_python_ok('-c', script) self.assertFalse(err) + def test_thread_from_thread(self): + script = """if True: + import threading + import time + + def thread2(): + time.sleep(0.05) + print("OK") + + def thread1(): + time.sleep(0.05) + t2 = threading.Thread(target=thread2) + t2.start() + + t = threading.Thread(target=thread1) + t.start() + # do not join() -- the interpreter waits for non-daemon threads to + # finish. + """ + rc, out, err = assert_python_ok('-c', script) + self.assertEqual(err, b"") + self.assertEqual(out.strip(), b"OK") + self.assertEqual(rc, 0) + @skip_unless_reliable_fork def test_reinit_tls_after_fork(self): # Issue #13817: fork() would deadlock in a multithreaded program with @@ -1509,6 +1595,7 @@ def task(): @cpython_only def test_daemon_threads_fatal_error(self): + import_module("_testcapi") subinterp_code = f"""if 1: import os import threading @@ -1535,6 +1622,7 @@ def _check_allowed(self, before_start='', *, daemon_allowed=True, daemon=False, ): + import_module("_testinternalcapi") subinterp_code = textwrap.dedent(f""" import test.support import threading @@ -1544,6 +1632,7 @@ def func(): {before_start} t.start() """) + check_multi_interp_extensions = bool(support.Py_GIL_DISABLED) script = textwrap.dedent(f""" import test.support test.support.run_in_subinterp_with_config( @@ -1553,7 +1642,7 @@ def func(): allow_exec=True, allow_threads={allowed}, allow_daemon_threads={daemon_allowed}, - check_multi_interp_extensions=False, + check_multi_interp_extensions={check_multi_interp_extensions}, own_gil=False, ) """) @@ -1809,6 +1898,7 @@ def setUp(self): restore_default_excepthook(self) super().setUp() + @force_not_colorized def test_excepthook(self): with support.captured_output("stderr") as stderr: thread = ThreadRunFail(name="excepthook thread") @@ -1822,6 +1912,7 @@ def test_excepthook(self): self.assertIn('ValueError: run failed', stderr) @support.cpython_only + @force_not_colorized def test_excepthook_thread_None(self): # threading.excepthook called with thread=None: log the thread # identifier in this case. @@ -2013,6 +2104,110 @@ def test__all__(self): support.check__all__(self, threading, ('threading', '_thread'), extra=extra, not_exported=not_exported) + @unittest.skipUnless(hasattr(_thread, 'set_name'), "missing _thread.set_name") + @unittest.skipUnless(hasattr(_thread, '_get_name'), "missing _thread._get_name") + def test_set_name(self): + # set_name() limit in bytes + truncate = getattr(_thread, "_NAME_MAXLEN", None) + limit = truncate or 100 + + tests = [ + # test short ASCII name + "CustomName", + + # test short non-ASCII name + "namé€", + + # embedded null character: name is truncated + # at the first null character + "embed\0null", + + # Test long ASCII names (not truncated) + "x" * limit, + + # Test long ASCII names (truncated) + "x" * (limit + 10), + + # Test long non-ASCII name (truncated) + "x" * (limit - 1) + "é€", + + # Test long non-BMP names (truncated) creating surrogate pairs + # on Windows + "x" * (limit - 1) + "\U0010FFFF", + "x" * (limit - 2) + "\U0010FFFF" * 2, + "x" + "\U0001f40d" * limit, + "xx" + "\U0001f40d" * limit, + "xxx" + "\U0001f40d" * limit, + "xxxx" + "\U0001f40d" * limit, + ] + if os_helper.FS_NONASCII: + tests.append(f"nonascii:{os_helper.FS_NONASCII}") + if os_helper.TESTFN_UNENCODABLE: + tests.append(os_helper.TESTFN_UNENCODABLE) + + if sys.platform.startswith("solaris"): + encoding = "utf-8" + else: + encoding = sys.getfilesystemencoding() + + def work(): + nonlocal work_name + work_name = _thread._get_name() + + for name in tests: + if not support.MS_WINDOWS: + encoded = name.encode(encoding, "replace") + if b'\0' in encoded: + encoded = encoded.split(b'\0', 1)[0] + if truncate is not None: + encoded = encoded[:truncate] + if sys.platform.startswith("solaris"): + expected = encoded.decode("utf-8", "surrogateescape") + else: + expected = os.fsdecode(encoded) + else: + size = 0 + chars = [] + for ch in name: + if ord(ch) > 0xFFFF: + size += 2 + else: + size += 1 + if size > truncate: + break + chars.append(ch) + expected = ''.join(chars) + + if '\0' in expected: + expected = expected.split('\0', 1)[0] + + with self.subTest(name=name, expected=expected): + work_name = None + thread = threading.Thread(target=work, name=name) + thread.start() + thread.join() + self.assertEqual(work_name, expected, + f"{len(work_name)=} and {len(expected)=}") + + @unittest.skipUnless(hasattr(_thread, 'set_name'), "missing _thread.set_name") + @unittest.skipUnless(hasattr(_thread, '_get_name'), "missing _thread._get_name") + def test_change_name(self): + # Change the name of a thread while the thread is running + + name1 = None + name2 = None + def work(): + nonlocal name1, name2 + name1 = _thread._get_name() + threading.current_thread().name = "new name" + name2 = _thread._get_name() + + thread = threading.Thread(target=work, name="name") + thread.start() + thread.join() + self.assertEqual(name1, "name") + self.assertEqual(name2, "new name") + class InterruptMainTests(unittest.TestCase): def check_interrupt_main_with_signal_handler(self, signum): @@ -2038,6 +2233,7 @@ def check_interrupt_main_noerror(self, signum): # Restore original handler signal.signal(signum, handler) + @requires_gil_enabled("gh-118433: Flaky due to a longstanding bug") def test_interrupt_main_subthread(self): # Calling start_new_thread with a function that executes interrupt_main # should raise KeyboardInterrupt upon completion. diff --git a/Lib/test/test_threading_local.py b/Lib/test/test_threading_local.py index f0b829a978feb5..3a58afd8194a32 100644 --- a/Lib/test/test_threading_local.py +++ b/Lib/test/test_threading_local.py @@ -208,6 +208,21 @@ def test_threading_local_clear_race(self): _testcapi.join_temporary_c_thread() + @support.cpython_only + def test_error(self): + class Loop(self._local): + attr = 1 + + # Trick the "if name == '__dict__':" test of __setattr__() + # to always be true + class NameCompareTrue: + def __eq__(self, other): + return True + + loop = Loop() + with self.assertRaisesRegex(AttributeError, 'Loop.*read-only'): + loop.__setattr__(NameCompareTrue(), 2) + class ThreadLocalTest(unittest.TestCase, BaseLocalTest): _local = _thread._local diff --git a/Lib/test/test_threadsignals.py b/Lib/test/test_threadsignals.py index 6a53d655015cdb..bf241ada90ea35 100644 --- a/Lib/test/test_threadsignals.py +++ b/Lib/test/test_threadsignals.py @@ -32,39 +32,28 @@ def handle_signals(sig,frame): # a function that will be spawned as a separate thread. def send_signals(): - os.kill(process_pid, signal.SIGUSR1) - os.kill(process_pid, signal.SIGUSR2) + # We use `raise_signal` rather than `kill` because: + # * It verifies that a signal delivered to a background thread still has + # its Python-level handler called on the main thread. + # * It ensures the signal is handled before the thread exits. + signal.raise_signal(signal.SIGUSR1) + signal.raise_signal(signal.SIGUSR2) signalled_all.release() @threading_helper.requires_working_threading() -@unittest.skipUnless(hasattr(signal, "alarm"), "test requires signal.alarm") class ThreadSignals(unittest.TestCase): def test_signals(self): with threading_helper.wait_threads_exit(): # Test signal handling semantics of threads. - # We spawn a thread, have the thread send two signals, and + # We spawn a thread, have the thread send itself two signals, and # wait for it to finish. Check that we got both signals # and that they were run by the main thread. signalled_all.acquire() self.spawnSignallingThread() signalled_all.acquire() - # the signals that we asked the kernel to send - # will come back, but we don't know when. - # (it might even be after the thread exits - # and might be out of order.) If we haven't seen - # the signals yet, send yet another signal and - # wait for it return. - if signal_blackboard[signal.SIGUSR1]['tripped'] == 0 \ - or signal_blackboard[signal.SIGUSR2]['tripped'] == 0: - try: - signal.alarm(1) - signal.pause() - finally: - signal.alarm(0) - self.assertEqual( signal_blackboard[signal.SIGUSR1]['tripped'], 1) self.assertEqual( signal_blackboard[signal.SIGUSR1]['tripped_by'], thread.get_ident()) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index a0aeea515afbd6..1147997d8d86bf 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -2,7 +2,6 @@ from test.support import warnings_helper import decimal import enum -import locale import math import platform import sys @@ -19,7 +18,7 @@ except ImportError: _testinternalcapi = None -from test.support import skip_if_buggy_ucrt_strfptime +from test.support import skip_if_buggy_ucrt_strfptime, SuppressCrashReport # Max year is only limited by the size of C int. SIZEOF_INT = sysconfig.get_config_var('SIZEOF_INT') or 4 @@ -117,6 +116,7 @@ def test_clock_monotonic(self): 'need time.pthread_getcpuclockid()') @unittest.skipUnless(hasattr(time, 'clock_gettime'), 'need time.clock_gettime()') + @unittest.skipIf(support.is_emscripten, "Fails to find clock") def test_pthread_getcpuclockid(self): clk_id = time.pthread_getcpuclockid(threading.get_ident()) self.assertTrue(type(clk_id) is int) @@ -158,10 +158,19 @@ def test_conversions(self): self.assertEqual(int(time.mktime(time.localtime(self.t))), int(self.t)) - def test_sleep(self): + def test_sleep_exceptions(self): + self.assertRaises(TypeError, time.sleep, []) + self.assertRaises(TypeError, time.sleep, "a") + self.assertRaises(TypeError, time.sleep, complex(0, 0)) + self.assertRaises(ValueError, time.sleep, -2) self.assertRaises(ValueError, time.sleep, -1) - time.sleep(1.2) + self.assertRaises(ValueError, time.sleep, -0.1) + + def test_sleep(self): + for value in [-0.0, 0, 0.0, 1e-100, 1e-9, 1e-6, 1, 1.2]: + with self.subTest(value=value): + time.sleep(value) def test_epoch(self): # bpo-43869: Make sure that Python use the same Epoch on all platforms: @@ -182,8 +191,44 @@ def test_strftime(self): self.fail('conversion specifier: %r failed.' % format) self.assertRaises(TypeError, time.strftime, b'%S', tt) - # embedded null character - self.assertRaises(ValueError, time.strftime, '%S\0', tt) + + def test_strftime_invalid_format(self): + tt = time.gmtime(self.t) + with SuppressCrashReport(): + for i in range(1, 128): + format = ' %' + chr(i) + with self.subTest(format=format): + try: + time.strftime(format, tt) + except ValueError as exc: + self.assertEqual(str(exc), 'Invalid format string') + + def test_strftime_special(self): + tt = time.gmtime(self.t) + s1 = time.strftime('%c', tt) + s2 = time.strftime('%B', tt) + # gh-52551, gh-78662: Unicode strings should pass through strftime, + # independently from locale. + self.assertEqual(time.strftime('\U0001f40d', tt), '\U0001f40d') + self.assertEqual(time.strftime('\U0001f4bb%c\U0001f40d%B', tt), f'\U0001f4bb{s1}\U0001f40d{s2}') + self.assertEqual(time.strftime('%c\U0001f4bb%B\U0001f40d', tt), f'{s1}\U0001f4bb{s2}\U0001f40d') + # Lone surrogates should pass through. + self.assertEqual(time.strftime('\ud83d', tt), '\ud83d') + self.assertEqual(time.strftime('\udc0d', tt), '\udc0d') + self.assertEqual(time.strftime('\ud83d%c\udc0d%B', tt), f'\ud83d{s1}\udc0d{s2}') + self.assertEqual(time.strftime('%c\ud83d%B\udc0d', tt), f'{s1}\ud83d{s2}\udc0d') + self.assertEqual(time.strftime('%c\udc0d%B\ud83d', tt), f'{s1}\udc0d{s2}\ud83d') + # Surrogate pairs should not recombine. + self.assertEqual(time.strftime('\ud83d\udc0d', tt), '\ud83d\udc0d') + self.assertEqual(time.strftime('%c\ud83d\udc0d%B', tt), f'{s1}\ud83d\udc0d{s2}') + # Surrogate-escaped bytes should not recombine. + self.assertEqual(time.strftime('\udcf0\udc9f\udc90\udc8d', tt), '\udcf0\udc9f\udc90\udc8d') + self.assertEqual(time.strftime('%c\udcf0\udc9f\udc90\udc8d%B', tt), f'{s1}\udcf0\udc9f\udc90\udc8d{s2}') + # gh-124531: The null character should not terminate the format string. + self.assertEqual(time.strftime('\0', tt), '\0') + self.assertEqual(time.strftime('\0'*1000, tt), '\0'*1000) + self.assertEqual(time.strftime('\0%c\0%B', tt), f'\0{s1}\0{s2}') + self.assertEqual(time.strftime('%c\0%B\0', tt), f'{s1}\0{s2}\0') def _bounds_checking(self, func): # Make sure that strftime() checks the bounds of the various parts @@ -277,6 +322,8 @@ def test_strptime(self): 'j', 'm', 'M', 'p', 'S', 'U', 'w', 'W', 'x', 'X', 'y', 'Y', 'Z', '%'): format = '%' + directive + if directive == 'd': + format += ',%Y' # Avoid GH-70647. strf_output = time.strftime(format, tt) try: time.strptime(strf_output, format) @@ -297,7 +344,13 @@ def test_strptime_exception_context(self): # additional check for IndexError branch (issue #19545) with self.assertRaises(ValueError) as e: time.strptime('19', '%Y %') - self.assertIs(e.exception.__suppress_context__, True) + self.assertIsNone(e.exception.__context__) + + def test_strptime_leap_year(self): + # GH-70647: warns if parsing a format with a day and no year. + with self.assertWarnsRegex(DeprecationWarning, + r'.*day of month without a year.*'): + time.strptime('02-07 18:28', '%m-%d %H:%M') def test_asctime(self): time.asctime(time.gmtime(self.t)) @@ -317,9 +370,6 @@ def test_asctime(self): def test_asctime_bounding_check(self): self._bounds_checking(time.asctime) - @unittest.skipIf( - support.is_emscripten, "musl libc issue on Emscripten, bpo-46390" - ) def test_ctime(self): t = time.mktime((1973, 9, 16, 1, 3, 52, 0, 0, -1)) self.assertEqual(time.ctime(t), 'Sun Sep 16 01:03:52 1973') @@ -496,6 +546,9 @@ def test_perf_counter(self): @unittest.skipIf( support.is_wasi, "process_time not available on WASI" ) + @unittest.skipIf( + support.is_emscripten, "process_time present but doesn't exclude sleep" + ) def test_process_time(self): # process_time() should not include time spend during a sleep start = time.process_time() @@ -511,7 +564,7 @@ def test_process_time(self): def test_thread_time(self): if not hasattr(time, 'thread_time'): - if sys.platform.startswith(('linux', 'win')): + if sys.platform.startswith(('linux', 'android', 'win')): self.fail("time.thread_time() should be available on %r" % (sys.platform,)) else: @@ -592,17 +645,8 @@ def test_get_clock_info(self): class TestLocale(unittest.TestCase): - def setUp(self): - self.oldloc = locale.setlocale(locale.LC_ALL) - - def tearDown(self): - locale.setlocale(locale.LC_ALL, self.oldloc) - + @support.run_with_locale('LC_ALL', 'fr_FR', '') def test_bug_3061(self): - try: - tmp = locale.setlocale(locale.LC_ALL, "fr_FR") - except locale.Error: - self.skipTest('could not set locale.LC_ALL to fr_FR') # This should not cause an exception time.strftime("%B", (2009,2,1,0,0,0,0,0,0)) @@ -646,8 +690,7 @@ def year4d(y): self.test_year('%04d', func=year4d) def skip_if_not_supported(y): - msg = "strftime() is limited to [1; 9999] with Visual Studio" - # Check that it doesn't crash for year > 9999 + msg = f"strftime() does not support year {y} on this platform" try: time.strftime('%Y', (y,) + (0,) * 8) except ValueError: @@ -709,9 +752,6 @@ class TestStrftime4dyear(_TestStrftimeYear, _Test4dYear, unittest.TestCase): class TestPytime(unittest.TestCase): @skip_if_buggy_ucrt_strfptime @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") - @unittest.skipIf( - support.is_emscripten, "musl libc issue on Emscripten, bpo-46390" - ) def test_localtime_timezone(self): # Get the localtime and examine it for the offset and zone. diff --git a/Lib/test/test_tkinter/test_geometry_managers.py b/Lib/test/test_tkinter/test_geometry_managers.py index 59fe592b492adc..d71a634a767310 100644 --- a/Lib/test/test_tkinter/test_geometry_managers.py +++ b/Lib/test/test_tkinter/test_geometry_managers.py @@ -10,6 +10,11 @@ requires('gui') +EXPECTED_FLOAT_ERRMSG = 'expected floating-point number but got "{}"' +EXPECTED_FLOAT_OR_EMPTY_ERRMSG = 'expected floating-point number (or "" )?but got "{}"' +EXPECTED_SCREEN_DISTANCE_ERRMSG = '(bad|expected) screen distance (but got )?"{}"' +EXPECTED_SCREEN_DISTANCE_OR_EMPTY_ERRMSG = '(bad|expected) screen distance (or "" but got )?"{}"' + class PackTest(AbstractWidgetTest, unittest.TestCase): test_keys = None @@ -317,7 +322,8 @@ def test_place_configure_x(self): self.assertEqual(f2.place_info()['x'], '-10') self.root.update() self.assertEqual(f2.winfo_x(), 190) - with self.assertRaisesRegex(TclError, 'bad screen distance "spam"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('spam')): f2.place_configure(in_=f, x='spam') def test_place_configure_y(self): @@ -334,7 +340,8 @@ def test_place_configure_y(self): self.assertEqual(f2.place_info()['y'], '-10') self.root.update() self.assertEqual(f2.winfo_y(), 110) - with self.assertRaisesRegex(TclError, 'bad screen distance "spam"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('spam')): f2.place_configure(in_=f, y='spam') def test_place_configure_relx(self): @@ -351,8 +358,7 @@ def test_place_configure_relx(self): self.assertEqual(f2.place_info()['relx'], '1') self.root.update() self.assertEqual(f2.winfo_x(), 200) - with self.assertRaisesRegex(TclError, 'expected floating-point number ' - 'but got "spam"'): + with self.assertRaisesRegex(TclError, EXPECTED_FLOAT_ERRMSG.format('spam')): f2.place_configure(in_=f, relx='spam') def test_place_configure_rely(self): @@ -369,8 +375,7 @@ def test_place_configure_rely(self): self.assertEqual(f2.place_info()['rely'], '1') self.root.update() self.assertEqual(f2.winfo_y(), 120) - with self.assertRaisesRegex(TclError, 'expected floating-point number ' - 'but got "spam"'): + with self.assertRaisesRegex(TclError, EXPECTED_FLOAT_ERRMSG.format('spam')): f2.place_configure(in_=f, rely='spam') def test_place_configure_anchor(self): @@ -391,7 +396,8 @@ def test_place_configure_width(self): f2.place_configure(width='') self.root.update() self.assertEqual(f2.winfo_width(), 30) - with self.assertRaisesRegex(TclError, 'bad screen distance "abcd"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_OR_EMPTY_ERRMSG.format('abcd')): f2.place_configure(width='abcd') def test_place_configure_height(self): @@ -402,7 +408,8 @@ def test_place_configure_height(self): f2.place_configure(height='') self.root.update() self.assertEqual(f2.winfo_height(), 60) - with self.assertRaisesRegex(TclError, 'bad screen distance "abcd"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_OR_EMPTY_ERRMSG.format('abcd')): f2.place_configure(height='abcd') def test_place_configure_relwidth(self): @@ -413,8 +420,7 @@ def test_place_configure_relwidth(self): f2.place_configure(relwidth='') self.root.update() self.assertEqual(f2.winfo_width(), 30) - with self.assertRaisesRegex(TclError, 'expected floating-point number ' - 'but got "abcd"'): + with self.assertRaisesRegex(TclError, EXPECTED_FLOAT_OR_EMPTY_ERRMSG.format('abcd')): f2.place_configure(relwidth='abcd') def test_place_configure_relheight(self): @@ -425,8 +431,7 @@ def test_place_configure_relheight(self): f2.place_configure(relheight='') self.root.update() self.assertEqual(f2.winfo_height(), 60) - with self.assertRaisesRegex(TclError, 'expected floating-point number ' - 'but got "abcd"'): + with self.assertRaisesRegex(TclError, EXPECTED_FLOAT_OR_EMPTY_ERRMSG.format('abcd')): f2.place_configure(relheight='abcd') def test_place_configure_bordermode(self): @@ -629,7 +634,8 @@ def test_grid_columnconfigure(self): self.assertEqual(self.root.grid_columnconfigure(0, 'weight'), 4) def test_grid_columnconfigure_minsize(self): - with self.assertRaisesRegex(TclError, 'bad screen distance "foo"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('foo')): self.root.grid_columnconfigure(0, minsize='foo') self.root.grid_columnconfigure(0, minsize=10) self.assertEqual(self.root.grid_columnconfigure(0, 'minsize'), 10) @@ -646,7 +652,8 @@ def test_grid_columnconfigure_weight(self): self.assertEqual(self.root.grid_columnconfigure(0)['weight'], 3) def test_grid_columnconfigure_pad(self): - with self.assertRaisesRegex(TclError, 'bad screen distance "foo"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('foo')): self.root.grid_columnconfigure(0, pad='foo') with self.assertRaisesRegex(TclError, 'invalid arg "-pad": ' 'should be non-negative'): @@ -683,7 +690,8 @@ def test_grid_rowconfigure(self): self.assertEqual(self.root.grid_rowconfigure(0, 'weight'), 4) def test_grid_rowconfigure_minsize(self): - with self.assertRaisesRegex(TclError, 'bad screen distance "foo"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('foo')): self.root.grid_rowconfigure(0, minsize='foo') self.root.grid_rowconfigure(0, minsize=10) self.assertEqual(self.root.grid_rowconfigure(0, 'minsize'), 10) @@ -700,7 +708,8 @@ def test_grid_rowconfigure_weight(self): self.assertEqual(self.root.grid_rowconfigure(0)['weight'], 3) def test_grid_rowconfigure_pad(self): - with self.assertRaisesRegex(TclError, 'bad screen distance "foo"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('foo')): self.root.grid_rowconfigure(0, pad='foo') with self.assertRaisesRegex(TclError, 'invalid arg "-pad": ' 'should be non-negative'): @@ -818,9 +827,11 @@ def test_grid_location(self): self.root.grid_location(0) with self.assertRaises(TypeError): self.root.grid_location(0, 0, 0) - with self.assertRaisesRegex(TclError, 'bad screen distance "x"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('x')): self.root.grid_location('x', 'y') - with self.assertRaisesRegex(TclError, 'bad screen distance "y"'): + with self.assertRaisesRegex(TclError, + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('y')): self.root.grid_location('1c', 'y') t = self.root # de-maximize @@ -893,9 +904,5 @@ def test_grid_slaves(self): self.assertEqual(self.root.grid_slaves(row=1, column=1), [d, c]) -tests_gui = ( - PackTest, PlaceTest, GridTest, -) - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_tkinter/test_images.py b/Lib/test/test_tkinter/test_images.py index ef1c99f57c6f47..38371fe00d6eb5 100644 --- a/Lib/test/test_tkinter/test_images.py +++ b/Lib/test/test_tkinter/test_images.py @@ -302,7 +302,37 @@ def test_copy(self): image2 = image.copy() self.assertEqual(image2.width(), 16) self.assertEqual(image2.height(), 16) - self.assertEqual(image.get(4, 6), image.get(4, 6)) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) + + image2 = image.copy(from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = image.copy(from_coords=(2, 3, 14, 11), zoom=2) + self.assertEqual(image2.width(), 24) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(23, 15), image.get(13, 10)) + self.assertEqual(image2.get(2*2, 4*2), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2*2+1, 4*2+1), image.get(6+2, 2+3)) + + image2 = image.copy(from_coords=(2, 3, 14, 11), subsample=2) + self.assertEqual(image2.width(), 6) + self.assertEqual(image2.height(), 4) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(5, 3), image.get(12, 9)) + self.assertEqual(image2.get(3, 2), image.get(3*2+2, 2*2+3)) + + image2 = image.copy(from_coords=(2, 3, 14, 11), subsample=2, zoom=3) + self.assertEqual(image2.width(), 18) + self.assertEqual(image2.height(), 12) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(17, 11), image.get(12, 9)) + self.assertEqual(image2.get(1*3, 2*3), image.get(1*2+2, 2*2+3)) + self.assertEqual(image2.get(1*3+2, 2*3+2), image.get(1*2+2, 2*2+3)) def test_subsample(self): image = self.create() @@ -316,6 +346,13 @@ def test_subsample(self): self.assertEqual(image2.height(), 8) self.assertEqual(image2.get(2, 3), image.get(4, 6)) + image2 = image.subsample(2, from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 6) + self.assertEqual(image2.height(), 4) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(5, 3), image.get(12, 9)) + self.assertEqual(image2.get(1, 2), image.get(1*2+2, 2*2+3)) + def test_zoom(self): image = self.create() image2 = image.zoom(2, 3) @@ -330,6 +367,118 @@ def test_zoom(self): self.assertEqual(image2.get(8, 12), image.get(4, 6)) self.assertEqual(image2.get(9, 13), image.get(4, 6)) + image2 = image.zoom(2, from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 24) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(23, 15), image.get(13, 10)) + self.assertEqual(image2.get(2*2, 4*2), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2*2+1, 4*2+1), image.get(6+2, 2+3)) + + def test_copy_replace(self): + image = self.create() + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image) + self.assertEqual(image2.width(), 16) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), shrink=True) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), to=(3, 6)) + self.assertEqual(image2.width(), 15) + self.assertEqual(image2.height(), 14) + self.assertEqual(image2.get(0+3, 0+6), image.get(2, 3)) + self.assertEqual(image2.get(11+3, 7+6), image.get(13, 10)) + self.assertEqual(image2.get(2+3, 4+6), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), to=(0, 0, 100, 50)) + self.assertEqual(image2.width(), 100) + self.assertEqual(image2.height(), 50) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2+12, 4+8), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2+12*2, 4), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2, 4+8*3), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), zoom=2) + self.assertEqual(image2.width(), 24) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(23, 15), image.get(13, 10)) + self.assertEqual(image2.get(2*2, 4*2), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2*2+1, 4*2+1), image.get(6+2, 2+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), subsample=2) + self.assertEqual(image2.width(), 6) + self.assertEqual(image2.height(), 4) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(5, 3), image.get(12, 9)) + self.assertEqual(image2.get(1, 2), image.get(1*2+2, 2*2+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), subsample=2, zoom=3) + self.assertEqual(image2.width(), 18) + self.assertEqual(image2.height(), 12) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(17, 11), image.get(12, 9)) + self.assertEqual(image2.get(3*3, 2*3), image.get(3*2+2, 2*2+3)) + self.assertEqual(image2.get(3*3+2, 2*3+2), image.get(3*2+2, 2*2+3)) + self.assertEqual(image2.get(1*3, 2*3), image.get(1*2+2, 2*2+3)) + self.assertEqual(image2.get(1*3+2, 2*3+2), image.get(1*2+2, 2*2+3)) + + def checkImgTrans(self, image, expected): + actual = {(x, y) + for x in range(image.width()) + for y in range(image.height()) + if image.transparency_get(x, y)} + self.assertEqual(actual, expected) + + def test_copy_replace_compositingrule(self): + image1 = tkinter.PhotoImage(master=self.root, width=2, height=2) + image1.blank() + image1.put('black', to=(0, 0, 2, 2)) + image1.transparency_set(0, 0, True) + + # default compositingrule + image2 = tkinter.PhotoImage(master=self.root, width=3, height=3) + image2.blank() + image2.put('white', to=(0, 0, 2, 2)) + image2.copy_replace(image1, to=(1, 1)) + self.checkImgTrans(image2, {(0, 2), (2, 0)}) + + image3 = tkinter.PhotoImage(master=self.root, width=3, height=3) + image3.blank() + image3.put('white', to=(0, 0, 2, 2)) + image3.copy_replace(image1, to=(1, 1), compositingrule='overlay') + self.checkImgTrans(image3, {(0, 2), (2, 0)}) + + image4 = tkinter.PhotoImage(master=self.root, width=3, height=3) + image4.blank() + image4.put('white', to=(0, 0, 2, 2)) + image4.copy_replace(image1, to=(1, 1), compositingrule='set') + self.checkImgTrans(image4, {(0, 2), (1, 1), (2, 0)}) + def test_put(self): image = self.create() image.put('{red green} {blue yellow}', to=(4, 6)) @@ -356,6 +505,50 @@ def test_get(self): self.assertRaises(tkinter.TclError, image.get, 16, 15) self.assertRaises(tkinter.TclError, image.get, 15, 16) + def test_read(self): + # Due to the Tk bug https://core.tcl-lang.org/tk/tktview/1576528 + # the -from option does not work correctly for GIF and PNG files. + # Use the PPM file for this test. + testfile = support.findfile('python.ppm', subdir='tkinterdata') + image = tkinter.PhotoImage(master=self.root, file=testfile) + + image2 = tkinter.PhotoImage(master=self.root) + image2.read(testfile) + self.assertEqual(image2.type(), 'photo') + self.assertEqual(image2.width(), 16) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(0, 0)) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) + + self.assertRaises(tkinter.TclError, image2.read, self.testfile, 'ppm') + + image2 = tkinter.PhotoImage(master=self.root) + image2.read(testfile, from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root, file=testfile) + self.assertEqual(image2.width(), 16) + self.assertEqual(image2.height(), 16) + image2.read(testfile, from_coords=(2, 3, 14, 11), shrink=True) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.read(testfile, from_coords=(2, 3, 14, 11), to=(3, 6)) + self.assertEqual(image2.type(), 'photo') + self.assertEqual(image2.width(), 15) + self.assertEqual(image2.height(), 14) + self.assertEqual(image2.get(0+3, 0+6), image.get(2, 3)) + self.assertEqual(image2.get(11+3, 7+6), image.get(13, 10)) + self.assertEqual(image2.get(2+3, 4+6), image.get(2+2, 4+3)) + def test_write(self): filename = os_helper.TESTFN import locale @@ -367,19 +560,17 @@ def test_write(self): image.write(filename) image2 = tkinter.PhotoImage('::img::test2', master=self.root, - format='ppm', - file=filename) + format='ppm', file=filename) self.assertEqual(str(image2), '::img::test2') self.assertEqual(image2.type(), 'photo') self.assertEqual(image2.width(), 16) self.assertEqual(image2.height(), 16) self.assertEqual(image2.get(0, 0), image.get(0, 0)) - self.assertEqual(image2.get(15, 8), image.get(15, 8)) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) image.write(filename, format='gif', from_coords=(4, 6, 6, 9)) image3 = tkinter.PhotoImage('::img::test3', master=self.root, - format='gif', - file=filename) + format='gif', file=filename) self.assertEqual(str(image3), '::img::test3') self.assertEqual(image3.type(), 'photo') self.assertEqual(image3.width(), 2) @@ -387,6 +578,67 @@ def test_write(self): self.assertEqual(image3.get(0, 0), image.get(4, 6)) self.assertEqual(image3.get(1, 2), image.get(5, 8)) + image.write(filename, background='#ff0000') + image4 = tkinter.PhotoImage('::img::test4', master=self.root, + format='ppm', file=filename) + self.assertEqual(image4.get(0, 0), (255, 0, 0) if self.wantobjects else '255 0 0') + self.assertEqual(image4.get(4, 6), image.get(4, 6)) + + image.write(filename, grayscale=True) + image5 = tkinter.PhotoImage('::img::test5', master=self.root, + format='ppm', file=filename) + c = image5.get(4, 6) + if not self.wantobjects: + c = c.split() + self.assertTrue(c[0] == c[1] == c[2], c) + + def test_data(self): + image = self.create() + + data = image.data() + self.assertIsInstance(data, tuple) + for row in data: + self.assertIsInstance(row, str) + c = image.get(4, 6) + if not self.wantobjects: + c = tuple(map(int, c.split())) + self.assertEqual(data[6].split()[4], '#%02x%02x%02x' % c) + + data = image.data('ppm') + image2 = tkinter.PhotoImage('::img::test2', master=self.root, + format='ppm', data=data) + self.assertEqual(str(image2), '::img::test2') + self.assertEqual(image2.type(), 'photo') + self.assertEqual(image2.width(), 16) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(0, 0)) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) + + data = image.data(format='gif', from_coords=(4, 6, 6, 9)) + image3 = tkinter.PhotoImage('::img::test3', master=self.root, + format='gif', data=data) + self.assertEqual(str(image3), '::img::test3') + self.assertEqual(image3.type(), 'photo') + self.assertEqual(image3.width(), 2) + self.assertEqual(image3.height(), 3) + self.assertEqual(image3.get(0, 0), image.get(4, 6)) + self.assertEqual(image3.get(1, 2), image.get(5, 8)) + + data = image.data('ppm', background='#ff0000') + image4 = tkinter.PhotoImage('::img::test4', master=self.root, + format='ppm', data=data) + self.assertEqual(image4.get(0, 0), (255, 0, 0) if self.wantobjects else '255 0 0') + self.assertEqual(image4.get(4, 6), image.get(4, 6)) + + data = image.data('ppm', grayscale=True) + image5 = tkinter.PhotoImage('::img::test5', master=self.root, + format='ppm', data=data) + c = image5.get(4, 6) + if not self.wantobjects: + c = c.split() + self.assertTrue(c[0] == c[1] == c[2], c) + + def test_transparency(self): image = self.create() self.assertEqual(image.transparency_get(0, 0), True) diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index 81a20b698a72eb..96ea3f0117ca03 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -4,7 +4,8 @@ from tkinter import TclError import enum from test import support -from test.test_tkinter.support import AbstractTkTest, AbstractDefaultRootTest, requires_tk +from test.test_tkinter.support import (AbstractTkTest, AbstractDefaultRootTest, + requires_tk, get_tk_patchlevel) support.requires('gui') @@ -30,12 +31,20 @@ def test_repr(self): self.assertEqual(repr(f), '') def test_generated_names(self): + class Button2(tkinter.Button): + pass + t = tkinter.Toplevel(self.root) f = tkinter.Frame(t) f2 = tkinter.Frame(t) + self.assertNotEqual(str(f), str(f2)) b = tkinter.Button(f2) - for name in str(b).split('.'): + b2 = Button2(f2) + for name in str(b).split('.') + str(b2).split('.'): self.assertFalse(name.isidentifier(), msg=repr(name)) + b3 = tkinter.Button(f2) + b4 = Button2(f2) + self.assertEqual(len({str(b), str(b2), str(b3), str(b4)}), 4) @requires_tk(8, 6, 6) def test_tk_busy(self): @@ -66,9 +75,10 @@ def test_tk_busy(self): f.tk_busy_forget() self.assertFalse(f.tk_busy_status()) self.assertFalse(f.tk_busy_current()) - with self.assertRaisesRegex(TclError, "can't find busy window"): + errmsg = r"can(no|')t find busy window.*" + with self.assertRaisesRegex(TclError, errmsg): f.tk_busy_configure() - with self.assertRaisesRegex(TclError, "can't find busy window"): + with self.assertRaisesRegex(TclError, errmsg): f.tk_busy_forget() @requires_tk(8, 6, 6) @@ -87,7 +97,8 @@ def test_tk_busy_with_cursor(self): self.assertEqual(f.tk_busy_configure('cursor')[4], 'heart') f.tk_busy_forget() - with self.assertRaisesRegex(TclError, "can't find busy window"): + errmsg = r"can(no|')t find busy window.*" + with self.assertRaisesRegex(TclError, errmsg): f.tk_busy_cget('cursor') def test_tk_setPalette(self): @@ -121,9 +132,9 @@ def test_tk_setPalette(self): def test_after(self): root = self.root - def callback(start=0, step=1): + def callback(start=0, step=1, *, end=0): nonlocal count - count = start + step + count = start + step + end # Without function, sleeps for ms. self.assertIsNone(root.after(1)) @@ -159,12 +170,18 @@ def callback(start=0, step=1): root.update() # Process all pending events. self.assertEqual(count, 53) + # Set up with callback with keyword args. + count = 0 + timer1 = root.after(0, callback, 42, step=11, end=1) + root.update() # Process all pending events. + self.assertEqual(count, 54) + def test_after_idle(self): root = self.root - def callback(start=0, step=1): + def callback(start=0, step=1, *, end=0): nonlocal count - count = start + step + count = start + step + end # Set up with callback with no args. count = 0 @@ -191,6 +208,12 @@ def callback(start=0, step=1): with self.assertRaises(tkinter.TclError): root.tk.call(script) + # Set up with callback with keyword args. + count = 0 + idle1 = root.after_idle(callback, 42, step=11, end=1) + root.update() # Process all pending events. + self.assertEqual(count, 54) + def test_after_cancel(self): root = self.root @@ -232,6 +255,46 @@ def callback(): with self.assertRaises(tkinter.TclError): root.tk.call('after', 'info', idle1) + def test_after_info(self): + root = self.root + + # No events. + self.assertEqual(root.after_info(), ()) + + # Add timer. + timer = root.after(1, lambda: 'break') + + # With no parameter, it returns a tuple of the event handler ids. + self.assertEqual(root.after_info(), (timer, )) + root.after_cancel(timer) + + timer1 = root.after(5000, lambda: 'break') + timer2 = root.after(5000, lambda: 'break') + idle1 = root.after_idle(lambda: 'break') + # Only contains new events and not 'timer'. + self.assertEqual(root.after_info(), (idle1, timer2, timer1)) + + # With a parameter returns a tuple of (script, type). + timer1_info = root.after_info(timer1) + self.assertEqual(len(timer1_info), 2) + self.assertEqual(timer1_info[1], 'timer') + idle1_info = root.after_info(idle1) + self.assertEqual(len(idle1_info), 2) + self.assertEqual(idle1_info[1], 'idle') + + root.after_cancel(timer1) + with self.assertRaises(tkinter.TclError): + root.after_info(timer1) + root.after_cancel(timer2) + with self.assertRaises(tkinter.TclError): + root.after_info(timer2) + root.after_cancel(idle1) + with self.assertRaises(tkinter.TclError): + root.after_info(idle1) + + # No events. + self.assertEqual(root.after_info(), ()) + def test_clipboard(self): root = self.root root.clipboard_clear() @@ -436,6 +499,15 @@ def test_info_patchlevel(self): self.assertEqual(vi.micro, 0) self.assertTrue(str(vi).startswith(f'{vi.major}.{vi.minor}')) + def test_embedded_null(self): + widget = tkinter.Entry(self.root) + widget.insert(0, 'abc\0def') # ASCII-only + widget.selection_range(0, 'end') + self.assertEqual(widget.selection_get(), 'abc\x00def') + widget.insert(0, '\u20ac\0') # non-ASCII + widget.selection_range(0, 'end') + self.assertEqual(widget.selection_get(), '\u20ac\0abc\x00def') + class WmTest(AbstractTkTest, unittest.TestCase): @@ -491,6 +563,309 @@ def test_wm_attribute(self): self.assertEqual(w.wm_attributes('alpha'), 1.0 if self.wantobjects else '1.0') + def test_wm_iconbitmap(self): + t = tkinter.Toplevel(self.root) + self.assertEqual(t.wm_iconbitmap(), '') + t.wm_iconbitmap('hourglass') + bug = False + if t._windowingsystem == 'aqua': + # Tk bug 13ac26b35dc55f7c37f70b39d59d7ef3e63017c8. + patchlevel = get_tk_patchlevel(t) + if patchlevel < (8, 6, 17) or (9, 0) <= patchlevel < (9, 0, 2): + bug = True + if not bug: + self.assertEqual(t.wm_iconbitmap(), 'hourglass') + self.assertEqual(self.root.wm_iconbitmap(), '') + t.wm_iconbitmap('') + self.assertEqual(t.wm_iconbitmap(), '') + + if t._windowingsystem == 'win32': + t.wm_iconbitmap(default='hourglass') + self.assertEqual(t.wm_iconbitmap(), 'hourglass') + self.assertEqual(self.root.wm_iconbitmap(), '') + t.wm_iconbitmap(default='') + self.assertEqual(t.wm_iconbitmap(), '') + + t.destroy() + + +class EventTest(AbstractTkTest, unittest.TestCase): + + def test_focus(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + + f.focus_force() + self.root.update() + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.FocusIn) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, '??') + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, '??') + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, '??') + self.assertEqual(e.y, '??') + self.assertEqual(e.x_root, '??') + self.assertEqual(e.y_root, '??') + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), '') + + def test_configure(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + + f.configure(height=120, borderwidth=10) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.Configure) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, '??') + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, '??') + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, 150) + self.assertEqual(e.height, 100) + self.assertEqual(e.x, 0) + self.assertEqual(e.y, 0) + self.assertEqual(e.x_root, '??') + self.assertEqual(e.y_root, '??') + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), '') + + def test_event_generate_key_press(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + f.focus_force() + + f.event_generate('') + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.KeyPress) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, 0) + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, '??') + self.assertIsInstance(e.state, int) + self.assertNotEqual(e.state, 0) + self.assertEqual(e.char, 'z') + self.assertIsInstance(e.keycode, int) + self.assertNotEqual(e.keycode, 0) + self.assertEqual(e.keysym, 'z') + self.assertEqual(e.keysym_num, ord('z')) + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, -1 - f.winfo_rootx()) + self.assertEqual(e.y, -1 - f.winfo_rooty()) + self.assertEqual(e.x_root, -1) + self.assertEqual(e.y_root, -1) + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), + f"") + + def test_event_generate_enter(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + + f.event_generate('', x=100, y=50) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.Enter) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, 0) + self.assertIs(e.send_event, False) + self.assertIs(e.focus, False) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, 0) + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, 100) + self.assertEqual(e.y, 50) + self.assertEqual(e.x_root, 100 + f.winfo_rootx()) + self.assertEqual(e.y_root, 50 + f.winfo_rooty()) + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), '') + + def test_event_generate_button_press(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + f.focus_force() + + f.event_generate('', x=100, y=50) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.ButtonPress) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, 0) + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, 1) + self.assertEqual(e.state, 0) + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, 100) + self.assertEqual(e.y, 50) + self.assertEqual(e.x_root, f.winfo_rootx() + 100) + self.assertEqual(e.y_root, f.winfo_rooty() + 50) + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), '') + + def test_event_generate_motion(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + f.focus_force() + + f.event_generate('', x=100, y=50) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.Motion) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, 0) + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, 0x100) + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, 100) + self.assertEqual(e.y, 50) + self.assertEqual(e.x_root, f.winfo_rootx() + 100) + self.assertEqual(e.y_root, f.winfo_rooty() + 50) + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), '') + + def test_event_generate_mouse_wheel(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('', events.append) + f.focus_force() + + f.event_generate('', x=100, y=50, delta=-5) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.MouseWheel) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.time, 0) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, 0) + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, 100) + self.assertEqual(e.y, 50) + self.assertEqual(e.x_root, f.winfo_rootx() + 100) + self.assertEqual(e.y_root, f.winfo_rooty() + 50) + self.assertEqual(e.delta, -5) + self.assertEqual(repr(e), '') + + def test_generate_event_virtual_event(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + + events = [] + f.bind('<>', events.append) + f.focus_force() + + f.event_generate('<>', x=50) + self.assertEqual(len(events), 1, events) + e = events[0] + self.assertIs(e.type, tkinter.EventType.VirtualEvent) + self.assertIs(e.widget, f) + self.assertIsInstance(e.serial, int) + self.assertEqual(e.time, 0) + self.assertIs(e.send_event, False) + self.assertFalse(hasattr(e, 'focus')) + self.assertEqual(e.num, '??') + self.assertEqual(e.state, 0) + self.assertEqual(e.char, '??') + self.assertEqual(e.keycode, '??') + self.assertEqual(e.keysym, '??') + self.assertEqual(e.keysym_num, '??') + self.assertEqual(e.width, '??') + self.assertEqual(e.height, '??') + self.assertEqual(e.x, 50) + self.assertEqual(e.y, 0) + self.assertEqual(e.x_root, f.winfo_rootx() + 50) + self.assertEqual(e.y_root, -1) + self.assertEqual(e.delta, 0) + self.assertEqual(repr(e), + f"") + class BindTest(AbstractTkTest, unittest.TestCase): diff --git a/Lib/test/test_tkinter/test_variables.py b/Lib/test/test_tkinter/test_variables.py index c1d232e2febc7a..def7aec077e800 100644 --- a/Lib/test/test_tkinter/test_variables.py +++ b/Lib/test/test_tkinter/test_variables.py @@ -6,7 +6,7 @@ from tkinter import (Variable, StringVar, IntVar, DoubleVar, BooleanVar, Tcl, TclError) from test.support import ALWAYS_EQ -from test.test_tkinter.support import AbstractDefaultRootTest +from test.test_tkinter.support import AbstractDefaultRootTest, tcl_version class Var(Variable): @@ -112,6 +112,8 @@ def test_initialize(self): self.assertTrue(v.side_effect) def test_trace_old(self): + if tcl_version >= (9, 0): + self.skipTest('requires Tcl version < 9.0') # Old interface v = Variable(self.root) vname = str(v) diff --git a/Lib/test/test_tkinter/test_widgets.py b/Lib/test/test_tkinter/test_widgets.py index d3f942db7baf9a..f6e77973061956 100644 --- a/Lib/test/test_tkinter/test_widgets.py +++ b/Lib/test/test_tkinter/test_widgets.py @@ -4,22 +4,37 @@ import os from test.support import requires -from test.test_tkinter.support import (requires_tk, +from test.test_tkinter.support import (requires_tk, tk_version, get_tk_patchlevel, widget_eq, AbstractDefaultRootTest) + from test.test_tkinter.widget_tests import ( - add_standard_options, - AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests) + add_configure_tests, + AbstractWidgetTest, + StandardOptionsTests, + IntegerSizeTests, + PixelSizeTests) requires('gui') +EXPECTED_SCREEN_DISTANCE_ERRMSG = '(bad|expected) screen distance (but got )?"{}"' +EXPECTED_SCREEN_DISTANCE_OR_EMPTY_ERRMSG = '(bad|expected) screen distance (or "" but got )?"{}"' + def float_round(x): return float(round(x)) - class AbstractToplevelTest(AbstractWidgetTest, PixelSizeTests): - _conv_pad_pixels = False + if tk_version < (9, 0): + _no_round = {'padx', 'pady'} + else: + _no_round = {'borderwidth', 'height', 'highlightthickness', 'padx', + 'pady', 'width'} + if tk_version < (9, 0): + _clipped = {'highlightthickness'} + else: + _clipped = {'borderwidth', 'height', 'highlightthickness', 'padx', + 'pady', 'width'} def test_configure_class(self): widget = self.create() @@ -55,14 +70,14 @@ def test_configure_visual(self): self.assertEqual(widget2['visual'], 'default') -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class ToplevelTest(AbstractToplevelTest, unittest.TestCase): OPTIONS = ( - 'background', 'borderwidth', + 'background', 'backgroundimage', 'borderwidth', 'class', 'colormap', 'container', 'cursor', 'height', 'highlightbackground', 'highlightcolor', 'highlightthickness', 'menu', 'padx', 'pady', 'relief', 'screen', - 'takefocus', 'use', 'visual', 'width', + 'takefocus', 'tile', 'use', 'visual', 'width', ) def create(self, **kwargs): @@ -98,20 +113,25 @@ def test_configure_use(self): self.assertEqual(widget2['use'], wid) -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class FrameTest(AbstractToplevelTest, unittest.TestCase): OPTIONS = ( - 'background', 'borderwidth', + 'background', 'backgroundimage', 'borderwidth', 'class', 'colormap', 'container', 'cursor', 'height', 'highlightbackground', 'highlightcolor', 'highlightthickness', - 'padx', 'pady', 'relief', 'takefocus', 'visual', 'width', + 'padx', 'pady', 'relief', 'takefocus', 'tile', 'visual', 'width', ) + if tk_version < (9, 0): + _no_round = {'padx', 'pady'} + else: + _no_round = {'borderwidth', 'height', 'highlightthickness', 'padx', + 'pady', 'width'} def create(self, **kwargs): return tkinter.Frame(self.root, **kwargs) -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class LabelFrameTest(AbstractToplevelTest, unittest.TestCase): OPTIONS = ( 'background', 'borderwidth', @@ -121,6 +141,11 @@ class LabelFrameTest(AbstractToplevelTest, unittest.TestCase): 'labelanchor', 'labelwidget', 'padx', 'pady', 'relief', 'takefocus', 'text', 'visual', 'width', ) + if tk_version < (9, 0): + _no_round = {'padx', 'pady'} + else: + _no_round = {'borderwidth', 'height', 'highlightthickness', 'padx', + 'pady', 'width'} def create(self, **kwargs): return tkinter.LabelFrame(self.root, **kwargs) @@ -138,17 +163,16 @@ def test_configure_labelwidget(self): self.checkParam(widget, 'labelwidget', label, expected='.foo') label.destroy() - +# Label, Button, Checkbutton, Radiobutton, MenuButton class AbstractLabelTest(AbstractWidgetTest, IntegerSizeTests): - _conv_pixels = False - - def test_configure_highlightthickness(self): - widget = self.create() - self.checkPixelsParam(widget, 'highlightthickness', - 0, 1.3, 2.6, 6, -2, '10p') - - -@add_standard_options(StandardOptionsTests) + _rounds_pixels = False + if tk_version < (9, 0): + _clipped = {} + else: + _clipped = {'borderwidth', 'insertborderwidth', 'highlightthickness', + 'padx', 'pady'} + +@add_configure_tests(StandardOptionsTests) class LabelTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'activebackground', 'activeforeground', 'anchor', @@ -164,7 +188,7 @@ def create(self, **kwargs): return tkinter.Label(self.root, **kwargs) -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class ButtonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'activebackground', 'activeforeground', 'anchor', @@ -185,7 +209,7 @@ def test_configure_default(self): self.checkEnumParam(widget, 'default', 'active', 'disabled', 'normal') -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class CheckbuttonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'activebackground', 'activeforeground', 'anchor', @@ -239,8 +263,7 @@ def test_same_name(self): b2.deselect() self.assertEqual(v.get(), 0) - -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class RadiobuttonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'activebackground', 'activeforeground', 'anchor', @@ -263,7 +286,7 @@ def test_configure_value(self): self.checkParams(widget, 'value', 1, 2.3, '', 'any string') -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class MenubuttonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'activebackground', 'activeforeground', 'anchor', @@ -276,7 +299,11 @@ class MenubuttonTest(AbstractLabelTest, unittest.TestCase): 'takefocus', 'text', 'textvariable', 'underline', 'width', 'wraplength', ) - _conv_pixels = round + _rounds_pixels = (tk_version < (9, 0)) + if tk_version < (9, 0): + _clipped = {'highlightthickness', 'padx', 'pady'} + else: + _clipped ={ 'insertborderwidth', 'highlightthickness', 'padx', 'pady'} def create(self, **kwargs): return tkinter.Menubutton(self.root, **kwargs) @@ -290,14 +317,14 @@ def test_configure_height(self): widget = self.create() self.checkIntegerParam(widget, 'height', 100, -100, 0, conv=str) - test_configure_highlightthickness = \ - StandardOptionsTests.test_configure_highlightthickness - def test_configure_image(self): widget = self.create() image = tkinter.PhotoImage(master=self.root, name='image1') self.checkParam(widget, 'image', image, conv=str) - errmsg = 'image "spam" doesn\'t exist' + if tk_version < (9, 0): + errmsg = 'image "spam" doesn\'t exist' + else: + errmsg = 'image "spam" does not exist' with self.assertRaises(tkinter.TclError) as cm: widget['image'] = 'spam' if errmsg is not None: @@ -313,16 +340,6 @@ def test_configure_menu(self): self.checkParam(widget, 'menu', menu, eq=widget_eq) menu.destroy() - def test_configure_padx(self): - widget = self.create() - self.checkPixelsParam(widget, 'padx', 3, 4.4, 5.6, '12m') - self.checkParam(widget, 'padx', -2, expected=0) - - def test_configure_pady(self): - widget = self.create() - self.checkPixelsParam(widget, 'pady', 3, 4.4, 5.6, '12m') - self.checkParam(widget, 'pady', -2, expected=0) - def test_configure_width(self): widget = self.create() self.checkIntegerParam(widget, 'width', 402, -402, 0, conv=str) @@ -337,9 +354,15 @@ def test_bad_kwarg(self): with self.assertRaisesRegex(TclError, r"^unknown option -image$"): tkinter.OptionMenu(self.root, None, 'b', image='') - -@add_standard_options(IntegerSizeTests, StandardOptionsTests) +@add_configure_tests(IntegerSizeTests, StandardOptionsTests) class EntryTest(AbstractWidgetTest, unittest.TestCase): + _rounds_pixels = (tk_version < (9, 0)) + if tk_version < (9, 0): + _clipped = {'highlightthickness'} + else: + _clipped = {'highlightthickness', 'borderwidth', 'insertborderwidth', + 'selectborderwidth'} + OPTIONS = ( 'background', 'borderwidth', 'cursor', 'disabledbackground', 'disabledforeground', @@ -347,7 +370,8 @@ class EntryTest(AbstractWidgetTest, unittest.TestCase): 'highlightbackground', 'highlightcolor', 'highlightthickness', 'insertbackground', 'insertborderwidth', 'insertofftime', 'insertontime', 'insertwidth', - 'invalidcommand', 'justify', 'readonlybackground', 'relief', + 'invalidcommand', 'justify', 'placeholder', 'placeholderforeground', + 'readonlybackground', 'relief', 'selectbackground', 'selectborderwidth', 'selectforeground', 'show', 'state', 'takefocus', 'textvariable', 'validate', 'validatecommand', 'width', 'xscrollcommand', @@ -363,16 +387,23 @@ def test_configure_disabledbackground(self): def test_configure_insertborderwidth(self): widget = self.create(insertwidth=100) self.checkPixelsParam(widget, 'insertborderwidth', - 0, 1.3, 2.6, 6, -2, '10p') + 0, 1.3, 2.6, 6, '10p') + self.checkParam(widget, 'insertborderwidth', -2) # insertborderwidth is bounded above by a half of insertwidth. - self.checkParam(widget, 'insertborderwidth', 60, expected=100//2) + expected = 100 // 2 if tk_version < (9, 0) else 60 + self.checkParam(widget, 'insertborderwidth', 60, expected=expected) def test_configure_insertwidth(self): widget = self.create() self.checkPixelsParam(widget, 'insertwidth', 1.3, 3.6, '10p') - self.checkParam(widget, 'insertwidth', 0.1, expected=2) - self.checkParam(widget, 'insertwidth', -2, expected=2) - self.checkParam(widget, 'insertwidth', 0.9, expected=1) + if tk_version < (9, 0): + self.checkParam(widget, 'insertwidth', 0.1, expected=2) + self.checkParam(widget, 'insertwidth', -2, expected=2) + self.checkParam(widget, 'insertwidth', 0.9, expected=1) + else: + self.checkParam(widget, 'insertwidth', 0.1) + self.checkParam(widget, 'insertwidth', -2, expected=0) + self.checkParam(widget, 'insertwidth', 0.9) def test_configure_invalidcommand(self): widget = self.create() @@ -430,7 +461,7 @@ def test_selection_methods(self): widget.selection_adjust(0) -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class SpinboxTest(EntryTest, unittest.TestCase): OPTIONS = ( 'activebackground', 'background', 'borderwidth', @@ -441,8 +472,8 @@ class SpinboxTest(EntryTest, unittest.TestCase): 'increment', 'insertbackground', 'insertborderwidth', 'insertofftime', 'insertontime', 'insertwidth', - 'invalidcommand', 'justify', 'relief', 'readonlybackground', - 'repeatdelay', 'repeatinterval', + 'invalidcommand', 'justify', 'placeholder', 'placeholderforeground', + 'relief', 'readonlybackground', 'repeatdelay', 'repeatinterval', 'selectbackground', 'selectborderwidth', 'selectforeground', 'state', 'takefocus', 'textvariable', 'to', 'validate', 'validatecommand', 'values', @@ -489,8 +520,12 @@ def test_configure_from(self): widget = self.create() self.checkParam(widget, 'to', 100.0) self.checkFloatParam(widget, 'from', -10, 10.2, 11.7) - self.checkInvalidParam(widget, 'from', 200, - errmsg='-to value must be greater than -from value') + if tk_version >= (8, 7): + self.checkFloatParam(widget, 'from', 200, expected=100) + else: + self.checkInvalidParam( + widget, 'from', 200, + errmsg='-to value must be greater than -from value') def test_configure_increment(self): widget = self.create() @@ -500,8 +535,12 @@ def test_configure_to(self): widget = self.create() self.checkParam(widget, 'from', -100.0) self.checkFloatParam(widget, 'to', -10, 10.2, 11.7) - self.checkInvalidParam(widget, 'to', -200, - errmsg='-to value must be greater than -from value') + if tk_version >= (8, 7): + self.checkFloatParam(widget, 'to', -200, expected=-100) + else: + self.checkInvalidParam( + widget, 'to', -200, + errmsg='-to value must be greater than -from value') def test_configure_values(self): # XXX @@ -559,7 +598,7 @@ def test_selection_element(self): self.assertEqual(widget.selection_element(), "buttondown") -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class TextTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'autoseparators', 'background', 'blockcursor', 'borderwidth', @@ -574,6 +613,9 @@ class TextTest(AbstractWidgetTest, unittest.TestCase): 'tabs', 'tabstyle', 'takefocus', 'undo', 'width', 'wrap', 'xscrollcommand', 'yscrollcommand', ) + _rounds_pixels = (tk_version < (9, 0)) + _no_round = {'selectborderwidth'} + _clipped = {'highlightthickness'} def create(self, **kwargs): return tkinter.Text(self.root, **kwargs) @@ -602,8 +644,10 @@ def test_configure_endline(self): def test_configure_height(self): widget = self.create() self.checkPixelsParam(widget, 'height', 100, 101.2, 102.6, '3c') - self.checkParam(widget, 'height', -100, expected=1) - self.checkParam(widget, 'height', 0, expected=1) + self.checkParam(widget, 'height', -100, + expected=1 if tk_version < (9, 0) else -100) + self.checkParam(widget, 'height', 0, + expected=1 if tk_version < (9, 0) else 0 ) def test_configure_maxundo(self): widget = self.create() @@ -660,11 +704,13 @@ def test_configure_tabs(self): widget = self.create() self.checkParam(widget, 'tabs', (10.2, 20.7, '1i', '2i')) self.checkParam(widget, 'tabs', '10.2 20.7 1i 2i', - expected=('10.2', '20.7', '1i', '2i')) + expected=(10.2, 20.7, '1i', '2i') + if get_tk_patchlevel(self.root) >= (8, 6, 14) + else ('10.2', '20.7', '1i', '2i')) self.checkParam(widget, 'tabs', '2c left 4c 6c center', expected=('2c', 'left', '4c', '6c', 'center')) self.checkInvalidParam(widget, 'tabs', 'spam', - errmsg='bad screen distance "spam"') + errmsg=EXPECTED_SCREEN_DISTANCE_ERRMSG.format('spam')) def test_configure_tabstyle(self): widget = self.create() @@ -694,7 +740,7 @@ def test_bbox(self): self.assertRaises(TypeError, widget.bbox, '1.1', 'end') -@add_standard_options(PixelSizeTests, StandardOptionsTests) +@add_configure_tests(PixelSizeTests, StandardOptionsTests) class CanvasTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'background', 'borderwidth', @@ -708,8 +754,15 @@ class CanvasTest(AbstractWidgetTest, unittest.TestCase): 'xscrollcommand', 'xscrollincrement', 'yscrollcommand', 'yscrollincrement', 'width', ) - - _conv_pixels = round + _rounds_pixels = True + if tk_version < (9, 0): + _noround = {} + _clipped = {'highlightthickness'} + else: + _no_round = {'borderwidth', 'height', 'highlightthickness', 'width', + 'xscrollincrement', 'yscrollincrement'} + _clipped = {'borderwidth', 'height', 'highlightthickness', 'width', + 'xscrollincrement', 'yscrollincrement'} _stringify = True def create(self, **kwargs): @@ -858,24 +911,27 @@ def test_create_line(self): def test_create_polygon(self): c = self.create() - i1 = c.create_polygon(20, 30, 40, 50, 60, 10) + tk87 = tk_version >= (8, 7) + # In Tk < 8.7 polygons are filled, but has no outline by default. + # This affects its size, so always explicitly specify outline. + i1 = c.create_polygon(20, 30, 40, 50, 60, 10, outline='red') self.assertEqual(c.coords(i1), [20.0, 30.0, 40.0, 50.0, 60.0, 10.0]) - self.assertEqual(c.bbox(i1), (19, 9, 61, 51)) + self.assertEqual(c.bbox(i1), (18, 8, 62, 52)) self.assertEqual(c.itemcget(i1, 'joinstyle'), 'round') self.assertEqual(c.itemcget(i1, 'smooth'), '0') self.assertEqual(c.itemcget(i1, 'splinestep'), '12') - i2 = c.create_polygon([21, 31, 41, 51, 61, 11]) + i2 = c.create_polygon([21, 31, 41, 51, 61, 11], outline='red') self.assertEqual(c.coords(i2), [21.0, 31.0, 41.0, 51.0, 61.0, 11.0]) - self.assertEqual(c.bbox(i2), (20, 10, 62, 52)) + self.assertEqual(c.bbox(i2), (19, 9, 63, 53)) - i3 = c.create_polygon((22, 32), (42, 52), (62, 12)) + i3 = c.create_polygon((22, 32), (42, 52), (62, 12), outline='red') self.assertEqual(c.coords(i3), [22.0, 32.0, 42.0, 52.0, 62.0, 12.0]) - self.assertEqual(c.bbox(i3), (21, 11, 63, 53)) + self.assertEqual(c.bbox(i3), (20, 10, 64, 54)) - i4 = c.create_polygon([(23, 33), (43, 53), (63, 13)]) + i4 = c.create_polygon([(23, 33), (43, 53), (63, 13)], outline='red') self.assertEqual(c.coords(i4), [23.0, 33.0, 43.0, 53.0, 63.0, 13.0]) - self.assertEqual(c.bbox(i4), (22, 12, 64, 54)) + self.assertEqual(c.bbox(i4), (21, 11, 65, 55)) self.assertRaises(TclError, c.create_polygon, 20, 30, 60) self.assertRaises(TclError, c.create_polygon, [20, 30, 60]) @@ -948,7 +1004,7 @@ def test_moveto(self): self.assertEqual(y2_2 - y1_2, y2_3 - y1_3) -@add_standard_options(IntegerSizeTests, StandardOptionsTests) +@add_configure_tests(IntegerSizeTests, StandardOptionsTests) class ListboxTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'activestyle', 'background', 'borderwidth', 'cursor', @@ -960,6 +1016,11 @@ class ListboxTest(AbstractWidgetTest, unittest.TestCase): 'selectmode', 'setgrid', 'state', 'takefocus', 'width', 'xscrollcommand', 'yscrollcommand', ) + _rounds_pixels = (tk_version < (9, 0)) + if tk_version < (9, 0): + _clipped = {'highlightthickness'} + else: + _clipped = { 'borderwidth', 'highlightthickness', 'selectborderwidth'} def create(self, **kwargs): return tkinter.Listbox(self.root, **kwargs) @@ -999,12 +1060,16 @@ def test_itemconfigure(self): widget.itemconfigure() with self.assertRaisesRegex(TclError, 'bad listbox index "red"'): widget.itemconfigure('red') + if get_tk_patchlevel(self.root) >= (8, 6, 14): + prefix = ('background', '', '', '') + else: + prefix = ('background', 'background', 'Background', '') self.assertEqual(widget.itemconfigure(0, 'background'), - ('background', 'background', 'Background', '', 'red')) + (*prefix, 'red')) self.assertEqual(widget.itemconfigure('end', 'background'), - ('background', 'background', 'Background', '', 'violet')) + (*prefix, 'violet')) self.assertEqual(widget.itemconfigure('@0,0', 'background'), - ('background', 'background', 'Background', '', 'red')) + (*prefix, 'red')) d = widget.itemconfigure(0) self.assertIsInstance(d, dict) @@ -1082,7 +1147,7 @@ def test_get(self): self.assertRaises(TclError, lb.get, 2.4) -@add_standard_options(PixelSizeTests, StandardOptionsTests) +@add_configure_tests(PixelSizeTests, StandardOptionsTests) class ScaleTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'activebackground', 'background', 'bigincrement', 'borderwidth', @@ -1093,6 +1158,8 @@ class ScaleTest(AbstractWidgetTest, unittest.TestCase): 'resolution', 'showvalue', 'sliderlength', 'sliderrelief', 'state', 'takefocus', 'tickinterval', 'to', 'troughcolor', 'variable', 'width', ) + _rounds_pixels = (tk_version < (9, 0)) + _clipped = {'highlightthickness'} default_orient = 'vertical' def create(self, **kwargs): @@ -1150,7 +1217,7 @@ def test_configure_to(self): conv=float_round) -@add_standard_options(PixelSizeTests, StandardOptionsTests) +@add_configure_tests(PixelSizeTests, StandardOptionsTests) class ScrollbarTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'activebackground', 'activerelief', @@ -1161,25 +1228,30 @@ class ScrollbarTest(AbstractWidgetTest, unittest.TestCase): 'repeatdelay', 'repeatinterval', 'takefocus', 'troughcolor', 'width', ) - _conv_pixels = round + _rounds_pixels = True + if tk_version >= (9, 0): + _no_round = {'borderwidth', 'elementborderwidth', 'highlightthickness', + 'width'} + if tk_version < (9, 0): + _clipped = {'highlightthickness'} + else: + _clipped = {'borderwidth', 'highlightthickness', 'width'} _stringify = True default_orient = 'vertical' def create(self, **kwargs): return tkinter.Scrollbar(self.root, **kwargs) - def test_configure_activerelief(self): - widget = self.create() - self.checkReliefParam(widget, 'activerelief') - def test_configure_elementborderwidth(self): widget = self.create() - self.checkPixelsParam(widget, 'elementborderwidth', 4.3, 5.6, -2, '1m') + self.checkPixelsParam(widget, 'elementborderwidth', 4.3, 5.6, '1m') + expected = self._default_pixels if tk_version >= (8, 7) else -2 + self.checkParam(widget, 'elementborderwidth', -2, expected=expected) def test_configure_orient(self): widget = self.create() self.checkEnumParam(widget, 'orient', 'vertical', 'horizontal', - errmsg='bad orientation "{}": must be vertical or horizontal') + fullname='orientation', allow_empty=True) def test_activate(self): sb = self.create() @@ -1201,7 +1273,7 @@ def test_set(self): self.assertRaises(TypeError, sb.set, 0.6, 0.7, 0.8) -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class PanedWindowTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'background', 'borderwidth', 'cursor', @@ -1212,6 +1284,15 @@ class PanedWindowTest(AbstractWidgetTest, unittest.TestCase): 'sashcursor', 'sashpad', 'sashrelief', 'sashwidth', 'showhandle', 'width', ) + _rounds_pixels = True + if tk_version < (9, 0): + _no_round = {'handlesize', 'height', 'proxyborderwidth', 'sashwidth', + 'selectborderwidth', 'width'} + else: + _no_round = {'borderwidth', 'handlepad', 'handlesize', 'height', + 'proxyborderwidth', 'sashpad', 'sashwidth', + 'selectborderwidth', 'width'} + _clipped = {} default_orient = 'horizontal' def create(self, **kwargs): @@ -1250,7 +1331,8 @@ def test_configure_proxyborderwidth(self): @requires_tk(8, 6, 5) def test_configure_proxyrelief(self): widget = self.create() - self.checkReliefParam(widget, 'proxyrelief') + self.checkReliefParam(widget, 'proxyrelief', + allow_empty=(tk_version >= (8, 7))) def test_configure_sashcursor(self): widget = self.create() @@ -1323,7 +1405,7 @@ def test_paneconfigure_height(self): p, b, c = self.create2() self.check_paneconfigure(p, b, 'height', 10, 10) self.check_paneconfigure_bad(p, b, 'height', - 'bad screen distance "badValue"') + EXPECTED_SCREEN_DISTANCE_OR_EMPTY_ERRMSG.format('badValue')) def test_paneconfigure_hide(self): p, b, c = self.create2() @@ -1335,19 +1417,19 @@ def test_paneconfigure_minsize(self): p, b, c = self.create2() self.check_paneconfigure(p, b, 'minsize', 10, 10) self.check_paneconfigure_bad(p, b, 'minsize', - 'bad screen distance "badValue"') + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('badValue')) def test_paneconfigure_padx(self): p, b, c = self.create2() - self.check_paneconfigure(p, b, 'padx', 1.3, 1) + self.check_paneconfigure(p, b, 'padx', 1.3, 1 if tk_version < (9, 0) else 1.3) self.check_paneconfigure_bad(p, b, 'padx', - 'bad screen distance "badValue"') + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('badValue')) def test_paneconfigure_pady(self): p, b, c = self.create2() - self.check_paneconfigure(p, b, 'pady', 1.3, 1) + self.check_paneconfigure(p, b, 'pady', 1.3, 1 if tk_version < (9, 0) else 1.3) self.check_paneconfigure_bad(p, b, 'pady', - 'bad screen distance "badValue"') + EXPECTED_SCREEN_DISTANCE_ERRMSG.format('badValue')) def test_paneconfigure_sticky(self): p, b, c = self.create2() @@ -1368,19 +1450,20 @@ def test_paneconfigure_width(self): p, b, c = self.create2() self.check_paneconfigure(p, b, 'width', 10, 10) self.check_paneconfigure_bad(p, b, 'width', - 'bad screen distance "badValue"') + EXPECTED_SCREEN_DISTANCE_OR_EMPTY_ERRMSG.format('badValue')) -@add_standard_options(StandardOptionsTests) +@add_configure_tests(StandardOptionsTests) class MenuTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'activebackground', 'activeborderwidth', 'activeforeground', - 'background', 'borderwidth', 'cursor', + 'activerelief', 'background', 'borderwidth', 'cursor', 'disabledforeground', 'font', 'foreground', 'postcommand', 'relief', 'selectcolor', 'takefocus', 'tearoff', 'tearoffcommand', 'title', 'type', ) - _conv_pixels = False + _rounds_pixels = False + _clipped = {} def create(self, **kwargs): return tkinter.Menu(self.root, **kwargs) @@ -1390,6 +1473,8 @@ def test_indexcommand_none(self): i = widget.index('none') self.assertIsNone(i) + test_configure_activerelief = requires_tk(8, 7)(StandardOptionsTests.test_configure_activerelief) + def test_configure_postcommand(self): widget = self.create() self.checkCommandParam(widget, 'postcommand') @@ -1408,14 +1493,10 @@ def test_configure_title(self): def test_configure_type(self): widget = self.create() - opts = ('normal, tearoff, or menubar' - if widget.info_patchlevel() < (8, 7) else - 'menubar, normal, or tearoff') - self.checkEnumParam( - widget, 'type', - 'normal', 'tearoff', 'menubar', - errmsg='bad type "{}": must be ' + opts, - ) + values = ('normal', 'tearoff', 'menubar') + self.checkEnumParam(widget, 'type', *values, + allow_empty=tk_version < (8, 7), + sort=tk_version >= (8, 7)) def test_entryconfigure(self): m1 = self.create() @@ -1451,7 +1532,7 @@ def test_entryconfigure_variable(self): self.assertEqual(str(m1.entrycget(1, 'variable')), str(v2)) -@add_standard_options(PixelSizeTests, StandardOptionsTests) +@add_configure_tests(PixelSizeTests, StandardOptionsTests) class MessageTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'anchor', 'aspect', 'background', 'borderwidth', @@ -1460,7 +1541,12 @@ class MessageTest(AbstractWidgetTest, unittest.TestCase): 'justify', 'padx', 'pady', 'relief', 'takefocus', 'text', 'textvariable', 'width', ) - _conv_pad_pixels = False + _rounds_pixels = (tk_version < (9, 0)) + _no_round = {'padx', 'pady'} + if tk_version < (9, 0): + _clipped = {'highlightthickness'} + else: + _clipped = {'borderwidth', 'highlightthickness', 'padx', 'pady'} def create(self, **kwargs): return tkinter.Message(self.root, **kwargs) @@ -1469,6 +1555,24 @@ def test_configure_aspect(self): widget = self.create() self.checkIntegerParam(widget, 'aspect', 250, 0, -300) + def test_configure_padx(self): + widget = self.create() + self.checkPixelsParam(widget, 'padx', 3, 4.4, 5.6, '12m') + expected = -2 if tk_version < (9, 0) else self._default_pixels + self.checkParam(widget, 'padx', -2, expected=expected) + + def test_configure_pady(self): + widget = self.create() + self.checkPixelsParam(widget, 'pady', 3, 4.4, 5.6, '12m') + expected = -2 if tk_version < (9, 0) else self._default_pixels + self.checkParam(widget, 'pady', -2, expected=expected) + + def test_configure_width(self): + widget = self.create() + self.checkPixelsParam(widget, 'width', 402, 403.4, 404.6, 0, '5i') + expected = 0 if tk_version >= (8, 7) else -402 + self.checkParam(widget, 'width', -402, expected=expected) + class DefaultRootTest(AbstractDefaultRootTest, unittest.TestCase): @@ -1479,13 +1583,5 @@ def test_label(self): self._test_widget(tkinter.Label) -tests_gui = ( - ButtonTest, CanvasTest, CheckbuttonTest, EntryTest, - FrameTest, LabelFrameTest,LabelTest, ListboxTest, - MenubuttonTest, MenuTest, MessageTest, OptionMenuTest, - PanedWindowTest, RadiobuttonTest, ScaleTest, ScrollbarTest, - SpinboxTest, TextTest, ToplevelTest, DefaultRootTest, -) - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_tkinter/widget_tests.py b/Lib/test/test_tkinter/widget_tests.py index 31f82f459beefd..ac7fb5977e04fc 100644 --- a/Lib/test/test_tkinter/widget_tests.py +++ b/Lib/test/test_tkinter/widget_tests.py @@ -1,17 +1,22 @@ # Common tests for test_tkinter/test_widgets.py and test_ttk/test_widgets.py +import re import tkinter -from test.test_tkinter.support import (AbstractTkTest, tk_version, +from test.test_tkinter.support import (AbstractTkTest, requires_tk, tk_version, pixels_conv, tcl_obj_eq) import test.support - _sentinel = object() +# Options which accept all values allowed by Tk_GetPixels +# borderwidth = bd + class AbstractWidgetTest(AbstractTkTest): - _conv_pixels = round - _conv_pad_pixels = None - _stringify = False + _default_pixels = '' # Value for unset pixel options. + _rounds_pixels = True # True if some pixel options are rounded. + _no_round = {} # Pixel options which are not rounded nonetheless + _stringify = False # Whether to convert tuples to strings + _allow_empty_justify = False @property def scaling(self): @@ -38,6 +43,9 @@ def checkParam(self, widget, name, value, *, expected=_sentinel, widget[name] = value if expected is _sentinel: expected = value + if name in self._clipped: + if not isinstance(expected, str): + expected = max(expected, 0) if conv: expected = conv(expected) if self._stringify or not self.wantobjects: @@ -56,16 +64,13 @@ def checkParam(self, widget, name, value, *, expected=_sentinel, def checkInvalidParam(self, widget, name, value, errmsg=None): orig = widget[name] if errmsg is not None: - errmsg = errmsg.format(value) - with self.assertRaises(tkinter.TclError) as cm: + errmsg = errmsg.format(re.escape(str(value))) + errmsg = fr'\A{errmsg}\Z' + with self.assertRaisesRegex(tkinter.TclError, errmsg or ''): widget[name] = value - if errmsg is not None: - self.assertEqual(str(cm.exception), errmsg) self.assertEqual(widget[name], orig) - with self.assertRaises(tkinter.TclError) as cm: + with self.assertRaisesRegex(tkinter.TclError, errmsg or ''): widget.configure({name: value}) - if errmsg is not None: - self.assertEqual(str(cm.exception), errmsg) self.assertEqual(widget[name], orig) def checkParams(self, widget, name, *values, **kwargs): @@ -74,30 +79,26 @@ def checkParams(self, widget, name, *values, **kwargs): def checkIntegerParam(self, widget, name, *values, **kwargs): self.checkParams(widget, name, *values, **kwargs) - self.checkInvalidParam(widget, name, '', - errmsg='expected integer but got ""') - self.checkInvalidParam(widget, name, '10p', - errmsg='expected integer but got "10p"') - self.checkInvalidParam(widget, name, 3.2, - errmsg='expected integer but got "3.2"') + errmsg = 'expected integer but got "{}"' + self.checkInvalidParam(widget, name, '', errmsg=errmsg) + self.checkInvalidParam(widget, name, '10p', errmsg=errmsg) + self.checkInvalidParam(widget, name, 3.2, errmsg=errmsg) def checkFloatParam(self, widget, name, *values, conv=float, **kwargs): for value in values: self.checkParam(widget, name, value, conv=conv, **kwargs) - self.checkInvalidParam(widget, name, '', - errmsg='expected floating-point number but got ""') - self.checkInvalidParam(widget, name, 'spam', - errmsg='expected floating-point number but got "spam"') + errmsg = 'expected floating-point number but got "{}"' + self.checkInvalidParam(widget, name, '', errmsg=errmsg) + self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) def checkBooleanParam(self, widget, name): for value in (False, 0, 'false', 'no', 'off'): self.checkParam(widget, name, value, expected=0) for value in (True, 1, 'true', 'yes', 'on'): self.checkParam(widget, name, value, expected=1) - self.checkInvalidParam(widget, name, '', - errmsg='expected boolean value but got ""') - self.checkInvalidParam(widget, name, 'spam', - errmsg='expected boolean value but got "spam"') + errmsg = 'expected boolean value but got "{}"' + self.checkInvalidParam(widget, name, '', errmsg=errmsg) + self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) def checkColorParam(self, widget, name, *, allow_empty=None, **kwargs): self.checkParams(widget, name, @@ -120,52 +121,68 @@ def command(*args): self.assertTrue(widget[name]) self.checkParams(widget, name, '') - def checkEnumParam(self, widget, name, *values, errmsg=None, **kwargs): + def checkEnumParam(self, widget, name, *values, + errmsg=None, allow_empty=False, fullname=None, + sort=False, **kwargs): self.checkParams(widget, name, *values, **kwargs) if errmsg is None: + if sort: + if values[-1]: + values = tuple(sorted(values)) + else: + values = tuple(sorted(values[:-1])) + ('',) errmsg2 = ' %s "{}": must be %s%s or %s' % ( - name, + fullname or name, ', '.join(values[:-1]), ',' if len(values) > 2 else '', - values[-1]) - self.checkInvalidParam(widget, name, '', - errmsg='ambiguous' + errmsg2) + values[-1] or '""') + if '' not in values and not allow_empty: + self.checkInvalidParam(widget, name, '', + errmsg='ambiguous' + errmsg2) errmsg = 'bad' + errmsg2 self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) - def checkPixelsParam(self, widget, name, *values, - conv=None, **kwargs): - if conv is None: - conv = self._conv_pixels + def checkPixelsParam(self, widget, name, *values, conv=None, **kwargs): + if not self._rounds_pixels or name in self._no_round: + conv = False + elif conv != str: + conv = round for value in values: expected = _sentinel conv1 = conv if isinstance(value, str): + if not getattr(self, '_converts_pixels', True): + conv1 = str if conv1 and conv1 is not str: expected = pixels_conv(value) * self.scaling conv1 = round self.checkParam(widget, name, value, expected=expected, conv=conv1, **kwargs) - self.checkInvalidParam(widget, name, '6x', - errmsg='bad screen distance "6x"') - self.checkInvalidParam(widget, name, 'spam', - errmsg='bad screen distance "spam"') + errmsg = '(bad|expected) screen distance ((or "" )?but got )?"{}"' + self.checkInvalidParam(widget, name, '6x', errmsg=errmsg) + self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) - def checkReliefParam(self, widget, name): - self.checkParams(widget, name, - 'flat', 'groove', 'raised', 'ridge', 'solid', 'sunken') - errmsg='bad relief "spam": must be '\ - 'flat, groove, raised, ridge, solid, or sunken' + def checkReliefParam(self, widget, name, *, allow_empty=False): + values = ('flat', 'groove', 'raised', 'ridge', 'solid', 'sunken') + if allow_empty: + values += ('',) + self.checkParams(widget, name, *values) + errmsg = 'bad relief "{}": must be %s, or %s' % ( + ', '.join(values[:-1]), + values[-1] or '""') if tk_version < (8, 6): errmsg = None - self.checkInvalidParam(widget, name, 'spam', - errmsg=errmsg) + self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) def checkImageParam(self, widget, name): image = tkinter.PhotoImage(master=self.root, name='image1') self.checkParam(widget, name, image, conv=str) + if tk_version < (9, 0): + errmsg = 'image "spam" doesn\'t exist' + else: + errmsg = 'image "spam" does not exist' self.checkInvalidParam(widget, name, 'spam', - errmsg='image "spam" doesn\'t exist') + errmsg=errmsg) widget[name] = '' def checkVariableParam(self, widget, name, var): @@ -193,6 +210,7 @@ def test_keys(self): aliases = { 'bd': 'borderwidth', 'bg': 'background', + 'bgimg': 'backgroundimage', 'fg': 'foreground', 'invcmd': 'invalidcommand', 'vcmd': 'validatecommand', @@ -206,35 +224,88 @@ def test_keys(self): print('%s.OPTIONS doesn\'t contain "%s"' % (self.__class__.__name__, k)) +class PixelOptionsTests: + """Standard options that accept all formats acceptable to Tk_GetPixels. -class StandardOptionsTests: - STANDARD_OPTIONS = ( - 'activebackground', 'activeborderwidth', 'activeforeground', 'anchor', - 'background', 'bitmap', 'borderwidth', 'compound', 'cursor', - 'disabledforeground', 'exportselection', 'font', 'foreground', - 'highlightbackground', 'highlightcolor', 'highlightthickness', - 'image', 'insertbackground', 'insertborderwidth', - 'insertofftime', 'insertontime', 'insertwidth', - 'jump', 'justify', 'orient', 'padx', 'pady', 'relief', - 'repeatdelay', 'repeatinterval', - 'selectbackground', 'selectborderwidth', 'selectforeground', - 'setgrid', 'takefocus', 'text', 'textvariable', 'troughcolor', - 'underline', 'wraplength', 'xscrollcommand', 'yscrollcommand', - ) - - def test_configure_activebackground(self): - widget = self.create() - self.checkColorParam(widget, 'activebackground') + In addition to numbers, these options can be set with distances + specified as a string consisting of a number followed by a single + character giving the unit of distance. The allowed units are: + millimeters ('m'), centimeters ('c'), inches ('i') or points ('p'). + In Tk 9 a cget call for one of these options returns a Tcl_Obj of + type "pixels", whose string representation is the distance string + passed to configure. + """ + PIXEL_OPTIONS = ('activeborderwidth', 'borderwidth', 'highlightthickness', + 'insertborderwidth', 'insertwidth', 'padx', 'pady', 'selectborderwidth') def test_configure_activeborderwidth(self): widget = self.create() self.checkPixelsParam(widget, 'activeborderwidth', 0, 1.3, 2.9, 6, -2, '10p') + def test_configure_borderwidth(self): + widget = self.create() + self.checkPixelsParam(widget, 'borderwidth', + 0, 1.3, 2.6, 6, '10p') + self.checkParam(widget, 'borderwidth', -2) + if 'bd' in self.OPTIONS: + self.checkPixelsParam(widget, 'bd', 0, 1.3, 2.6, 6, '10p') + self.checkParam(widget, 'bd', -2, expected=expected) + + def test_configure_highlightthickness(self): + widget = self.create() + self.checkPixelsParam(widget, 'highlightthickness', + 0, 1.3, 2.6, 6, '10p') + self.checkParam(widget, 'highlightthickness', -2) + + def test_configure_insertborderwidth(self): + widget = self.create() + self.checkPixelsParam(widget, 'insertborderwidth', + 0, 1.3, 2.6, 6, '10p') + self.checkParam(widget, 'insertborderwidth', -2) + + def test_configure_insertwidth(self): + widget = self.create() + self.checkPixelsParam(widget, 'insertwidth', 1.3, 2.6, -2, '10p') + + def test_configure_padx(self): + widget = self.create() + self.checkPixelsParam(widget, 'padx', 3, 4.4, 5.6, '12m') + self.checkParam(widget, 'padx', -2) + + def test_configure_pady(self): + widget = self.create() + self.checkPixelsParam(widget, 'pady', 3, 4.4, 5.6, '12m') + self.checkParam(widget, 'pady', -2) + + def test_configure_selectborderwidth(self): + widget = self.create() + self.checkPixelsParam(widget, 'selectborderwidth', 1.3, 2.6, -2, '10p') + +class StandardOptionsTests(PixelOptionsTests): + + STANDARD_OPTIONS = ( 'activebackground', 'activeforeground', + 'anchor', 'background', 'bitmap', 'compound', 'cursor', + 'disabledforeground', 'exportselection', 'font', 'foreground', + 'highlightbackground', 'highlightcolor', 'image', + 'insertbackground', 'insertofftime', 'insertontime', 'jump', + 'justify', 'orient', 'relief', 'repeatdelay', 'repeatinterval', + 'selectbackground', 'selectforeground', 'setgrid', 'takefocus', + 'text', 'textvariable', 'troughcolor', 'underline', 'wraplength', + 'xscrollcommand', 'yscrollcommand', ) + PixelOptionsTests.PIXEL_OPTIONS + + def test_configure_activebackground(self): + widget = self.create() + self.checkColorParam(widget, 'activebackground') + def test_configure_activeforeground(self): widget = self.create() self.checkColorParam(widget, 'activeforeground') + def test_configure_activerelief(self): + widget = self.create() + self.checkReliefParam(widget, 'activerelief') + def test_configure_anchor(self): widget = self.create() self.checkEnumParam(widget, 'anchor', @@ -246,6 +317,11 @@ def test_configure_background(self): if 'bg' in self.OPTIONS: self.checkColorParam(widget, 'bg') + @requires_tk(8, 7) + def test_configure_backgroundimage(self): + widget = self.create() + self.checkImageParam(widget, 'backgroundimage') + def test_configure_bitmap(self): widget = self.create() self.checkParam(widget, 'bitmap', 'questhead') @@ -259,13 +335,6 @@ def test_configure_bitmap(self): self.checkInvalidParam(widget, 'bitmap', 'spam', errmsg='bitmap "spam" not defined') - def test_configure_borderwidth(self): - widget = self.create() - self.checkPixelsParam(widget, 'borderwidth', - 0, 1.3, 2.6, 6, -2, '10p') - if 'bd' in self.OPTIONS: - self.checkPixelsParam(widget, 'bd', 0, 1.3, 2.6, 6, -2, '10p') - def test_configure_compound(self): widget = self.create() self.checkEnumParam(widget, 'compound', @@ -287,8 +356,10 @@ def test_configure_font(self): widget = self.create() self.checkParam(widget, 'font', '-Adobe-Helvetica-Medium-R-Normal--*-120-*-*-*-*-*-*') - self.checkInvalidParam(widget, 'font', '', - errmsg='font "" doesn\'t exist') + is_ttk = widget.__class__.__module__ == 'tkinter.ttk' + if not is_ttk: + errmsg = 'font "" does ?n[o\']t exist' + self.checkInvalidParam(widget, 'font', '', errmsg=errmsg) def test_configure_foreground(self): widget = self.create() @@ -304,13 +375,6 @@ def test_configure_highlightcolor(self): widget = self.create() self.checkColorParam(widget, 'highlightcolor') - def test_configure_highlightthickness(self): - widget = self.create() - self.checkPixelsParam(widget, 'highlightthickness', - 0, 1.3, 2.6, 6, '10p') - self.checkParam(widget, 'highlightthickness', -2, expected=0, - conv=self._conv_pixels) - def test_configure_image(self): widget = self.create() self.checkImageParam(widget, 'image') @@ -319,11 +383,6 @@ def test_configure_insertbackground(self): widget = self.create() self.checkColorParam(widget, 'insertbackground') - def test_configure_insertborderwidth(self): - widget = self.create() - self.checkPixelsParam(widget, 'insertborderwidth', - 0, 1.3, 2.6, 6, -2, '10p') - def test_configure_insertofftime(self): widget = self.create() self.checkIntegerParam(widget, 'insertofftime', 100) @@ -332,37 +391,32 @@ def test_configure_insertontime(self): widget = self.create() self.checkIntegerParam(widget, 'insertontime', 100) - def test_configure_insertwidth(self): - widget = self.create() - self.checkPixelsParam(widget, 'insertwidth', 1.3, 2.6, -2, '10p') - def test_configure_jump(self): widget = self.create() self.checkBooleanParam(widget, 'jump') def test_configure_justify(self): widget = self.create() - self.checkEnumParam(widget, 'justify', 'left', 'right', 'center', - errmsg='bad justification "{}": must be ' - 'left, right, or center') - self.checkInvalidParam(widget, 'justify', '', - errmsg='ambiguous justification "": must be ' - 'left, right, or center') + values = ('left', 'right', 'center') + if self._allow_empty_justify: + values += ('',) + self.checkEnumParam(widget, 'justify', *values, + fullname='justification') def test_configure_orient(self): widget = self.create() self.assertEqual(str(widget['orient']), self.default_orient) self.checkEnumParam(widget, 'orient', 'horizontal', 'vertical') - def test_configure_padx(self): + @requires_tk(8, 7) + def test_configure_placeholder(self): widget = self.create() - self.checkPixelsParam(widget, 'padx', 3, 4.4, 5.6, -2, '12m', - conv=self._conv_pad_pixels) + self.checkParam(widget, 'placeholder', 'xxx') - def test_configure_pady(self): + @requires_tk(8, 7) + def test_configure_placeholderforeground(self): widget = self.create() - self.checkPixelsParam(widget, 'pady', 3, 4.4, 5.6, -2, '12m', - conv=self._conv_pad_pixels) + self.checkColorParam(widget, 'placeholderforeground') def test_configure_relief(self): widget = self.create() @@ -380,10 +434,6 @@ def test_configure_selectbackground(self): widget = self.create() self.checkColorParam(widget, 'selectbackground') - def test_configure_selectborderwidth(self): - widget = self.create() - self.checkPixelsParam(widget, 'selectborderwidth', 1.3, 2.6, -2, '10p') - def test_configure_selectforeground(self): widget = self.create() self.checkColorParam(widget, 'selectforeground') @@ -409,13 +459,35 @@ def test_configure_textvariable(self): var = tkinter.StringVar(self.root) self.checkVariableParam(widget, 'textvariable', var) + @requires_tk(8, 7) + def test_configure_tile(self): + widget = self.create() + self.checkBooleanParam(widget, 'tile') + def test_configure_troughcolor(self): widget = self.create() self.checkColorParam(widget, 'troughcolor') def test_configure_underline(self): widget = self.create() - self.checkIntegerParam(widget, 'underline', 0, 1, 10) + self.checkParams(widget, 'underline', 0, 1, 10) + if tk_version >= (8, 7): + is_ttk = widget.__class__.__module__ == 'tkinter.ttk' + self.checkParam(widget, 'underline', '', + expected='' if is_ttk else self._default_pixels) + self.checkParam(widget, 'underline', '5+2', + expected='5+2' if is_ttk else 7) + self.checkParam(widget, 'underline', '5-2', + expected='5-2' if is_ttk else 3) + self.checkParam(widget, 'underline', 'end', expected='end') + self.checkParam(widget, 'underline', 'end-2', expected='end-2') + errmsg = (r'bad index "{}": must be integer\?\[\+-\]integer\?, ' + r'end\?\[\+-\]integer\?, or ""') + else: + errmsg = 'expected integer but got "{}"' + self.checkInvalidParam(widget, 'underline', '', errmsg=errmsg) + self.checkInvalidParam(widget, 'underline', '10p', errmsg=errmsg) + self.checkInvalidParam(widget, 'underline', 3.2, errmsg=errmsg) def test_configure_wraplength(self): widget = self.create() @@ -445,7 +517,8 @@ def test_configure_offrelief(self): def test_configure_overrelief(self): widget = self.create() - self.checkReliefParam(widget, 'overrelief') + self.checkReliefParam(widget, 'overrelief', + allow_empty=(tk_version >= (8, 7))) def test_configure_selectcolor(self): widget = self.create() @@ -470,6 +543,7 @@ def test_configure_variable(self): class IntegerSizeTests: + """ Tests widgets which only accept integral width and height.""" def test_configure_height(self): widget = self.create() self.checkIntegerParam(widget, 'height', 100, -100, 0) @@ -480,6 +554,7 @@ def test_configure_width(self): class PixelSizeTests: + """ Tests widgets which accept screen distances for width and height.""" def test_configure_height(self): widget = self.create() self.checkPixelsParam(widget, 'height', 100, 101.2, 102.6, -100, 0, '3c') @@ -489,7 +564,7 @@ def test_configure_width(self): self.checkPixelsParam(widget, 'width', 402, 403.4, 404.6, -402, 0, '5i') -def add_standard_options(*source_classes): +def add_configure_tests(*source_classes): # This decorator adds test_configure_xxx methods from source classes for # every xxx option in the OPTIONS class attribute if they are not defined # explicitly. diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 4428e8cea1964c..480bff743a9f8a 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1,4 +1,5 @@ import os +import re import token import tokenize import unittest @@ -228,7 +229,7 @@ def test_long(self): """) def test_float(self): - # Floating point numbers + # Floating-point numbers self.check_tokenize("x = 3.14159", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) @@ -1199,6 +1200,31 @@ def test_closing_parenthesis_from_different_line(self): NAME 'x' (1, 3) (1, 4) """) + def test_multiline_non_ascii_fstring(self): + self.check_tokenize("""\ +a = f''' + Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli'''""", """\ + NAME 'a' (1, 0) (1, 1) + OP '=' (1, 2) (1, 3) + FSTRING_START "f\'\'\'" (1, 4) (1, 8) + FSTRING_MIDDLE '\\n Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli' (1, 8) (2, 68) + FSTRING_END "\'\'\'" (2, 68) (2, 71) + """) + + def test_multiline_non_ascii_fstring_with_expr(self): + self.check_tokenize("""\ +f''' + 🔗 This is a test {test_arg1}🔗 +🔗'''""", """\ + FSTRING_START "f\'\'\'" (1, 0) (1, 4) + FSTRING_MIDDLE '\\n 🔗 This is a test ' (1, 4) (2, 21) + OP '{' (2, 21) (2, 22) + NAME 'test_arg1' (2, 22) (2, 31) + OP '}' (2, 31) (2, 32) + FSTRING_MIDDLE '🔗\\n🔗' (2, 32) (3, 1) + FSTRING_END "\'\'\'" (3, 1) (3, 4) + """) + class GenerateTokensTest(TokenizeTest): def check_tokenize(self, s, expected): # Format the tokens in s in a table format. @@ -1794,6 +1820,22 @@ def test_iter_compat(self): self.assertEqual(tokenize.untokenize(iter(tokens)), b'Hello ') +def contains_ambiguous_backslash(source): + """Return `True` if the source contains a backslash on a + line by itself. For example: + + a = (1 + \\ + ) + + Code like this cannot be untokenized exactly. This is because + the tokenizer does not produce any tokens for the line containing + the backslash and so there is no way to know its indent. + """ + pattern = re.compile(br'\n\s*\\\r?\n') + return pattern.search(source) is not None + + class TestRoundtrip(TestCase): def check_roundtrip(self, f): @@ -1804,6 +1846,9 @@ def check_roundtrip(self, f): tokenize.untokenize(), and the latter tokenized again to 2-tuples. The test fails if the 3 pair tokenizations do not match. + If the source code can be untokenized unambiguously, the + untokenized code must match the original code exactly. + When untokenize bugs are fixed, untokenize with 5-tuples should reproduce code that does not contain a backslash continuation following spaces. A proper test should test this. @@ -1827,6 +1872,13 @@ def check_roundtrip(self, f): tokens2_from5 = [tok[:2] for tok in tokenize.tokenize(readline5)] self.assertEqual(tokens2_from5, tokens2) + if not contains_ambiguous_backslash(code): + # The BOM does not produce a token so there is no way to preserve it. + code_without_bom = code.removeprefix(b'\xef\xbb\xbf') + readline = iter(code_without_bom.splitlines(keepends=True)).__next__ + untokenized_code = tokenize.untokenize(tokenize.tokenize(readline)) + self.assertEqual(code_without_bom, untokenized_code) + def check_line_extraction(self, f): if isinstance(f, str): code = f.encode('utf-8') @@ -1894,6 +1946,26 @@ def test_roundtrip(self): self.check_roundtrip(r"f'\\\\N{{'") self.check_roundtrip(r"f'\\\\\\N{{'") self.check_roundtrip(r"f'\\\\\\\\N{{'") + + self.check_roundtrip(r"f'\n{{foo}}'") + self.check_roundtrip(r"f'\\n{{foo}}'") + self.check_roundtrip(r"f'\\\n{{foo}}'") + self.check_roundtrip(r"f'\\\\n{{foo}}'") + + self.check_roundtrip(r"f'\t{{foo}}'") + self.check_roundtrip(r"f'\\t{{foo}}'") + self.check_roundtrip(r"f'\\\t{{foo}}'") + self.check_roundtrip(r"f'\\\\t{{foo}}'") + + self.check_roundtrip(r"rf'\t{{foo}}'") + self.check_roundtrip(r"rf'\\t{{foo}}'") + self.check_roundtrip(r"rf'\\\t{{foo}}'") + self.check_roundtrip(r"rf'\\\\t{{foo}}'") + + self.check_roundtrip(r"rf'\{{foo}}'") + self.check_roundtrip(r"f'\\{{foo}}'") + self.check_roundtrip(r"rf'\\\{{foo}}'") + self.check_roundtrip(r"f'\\\\{{foo}}'") cases = [ """ if 1: diff --git a/Lib/test/test_tomllib/test_error.py b/Lib/test/test_tomllib/test_error.py index 72446267f04759..3a8587492859ca 100644 --- a/Lib/test/test_tomllib/test_error.py +++ b/Lib/test/test_tomllib/test_error.py @@ -39,8 +39,19 @@ def test_invalid_char_quotes(self): tomllib.loads("v = '\n'") self.assertTrue(" '\\n' " in str(exc_info.exception)) + def test_type_error(self): + with self.assertRaises(TypeError) as exc_info: + tomllib.loads(b"v = 1") # type: ignore[arg-type] + self.assertEqual(str(exc_info.exception), "Expected str object, not 'bytes'") + + with self.assertRaises(TypeError) as exc_info: + tomllib.loads(False) # type: ignore[arg-type] + self.assertEqual(str(exc_info.exception), "Expected str object, not 'bool'") + def test_module_name(self): - self.assertEqual(tomllib.TOMLDecodeError().__module__, tomllib.__name__) + self.assertEqual( + tomllib.TOMLDecodeError("", "", 0).__module__, tomllib.__name__ + ) def test_invalid_parse_float(self): def dict_returner(s: str) -> dict: @@ -55,3 +66,33 @@ def list_returner(s: str) -> list: self.assertEqual( str(exc_info.exception), "parse_float must not return dicts or lists" ) + + def test_deprecated_tomldecodeerror(self): + for args in [ + (), + ("err msg",), + (None,), + (None, "doc"), + ("err msg", None), + (None, "doc", None), + ("err msg", "doc", None), + ("one", "two", "three", "four"), + ("one", "two", 3, "four", "five"), + ]: + with self.assertWarns(DeprecationWarning): + e = tomllib.TOMLDecodeError(*args) # type: ignore[arg-type] + self.assertEqual(e.args, args) + + def test_tomldecodeerror(self): + msg = "error parsing" + doc = "v=1\n[table]\nv='val'" + pos = 13 + formatted_msg = "error parsing (at line 3, column 2)" + e = tomllib.TOMLDecodeError(msg, doc, pos) + self.assertEqual(e.args, (formatted_msg,)) + self.assertEqual(str(e), formatted_msg) + self.assertEqual(e.msg, msg) + self.assertEqual(e.doc, doc) + self.assertEqual(e.pos, pos) + self.assertEqual(e.lineno, 3) + self.assertEqual(e.colno, 2) diff --git a/Lib/test/test_tomllib/test_misc.py b/Lib/test/test_tomllib/test_misc.py index 9e677a337a2835..59116afa1f36ad 100644 --- a/Lib/test/test_tomllib/test_misc.py +++ b/Lib/test/test_tomllib/test_misc.py @@ -5,6 +5,7 @@ import copy import datetime from decimal import Decimal as D +import importlib from pathlib import Path import sys import tempfile @@ -113,3 +114,11 @@ def test_inline_table_recursion_limit(self): nest_count=nest_count): recursive_table_toml = nest_count * "key = {" + nest_count * "}" tomllib.loads(recursive_table_toml) + + def test_types_import(self): + """Test that `_types` module runs. + + The module is for type annotations only, so it is otherwise + never imported by tests. + """ + importlib.import_module(f"{tomllib.__name__}._types") diff --git a/Lib/test/test_tools/i18n_data/docstrings.pot b/Lib/test/test_tools/i18n_data/docstrings.pot new file mode 100644 index 00000000000000..5af1d41422ff62 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/docstrings.pot @@ -0,0 +1,40 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: docstrings.py:7 +#, docstring +msgid "" +msgstr "" + +#: docstrings.py:18 +#, docstring +msgid "" +"multiline\n" +" docstring\n" +" " +msgstr "" + +#: docstrings.py:25 +#, docstring +msgid "docstring1" +msgstr "" + +#: docstrings.py:30 +#, docstring +msgid "Hello, {}!" +msgstr "" + diff --git a/Lib/test/test_tools/i18n_data/docstrings.py b/Lib/test/test_tools/i18n_data/docstrings.py new file mode 100644 index 00000000000000..85d7f159d37775 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/docstrings.py @@ -0,0 +1,41 @@ +# Test docstring extraction +from gettext import gettext as _ + + +# Empty docstring +def test(x): + """""" + + +# Leading empty line +def test2(x): + + """docstring""" # XXX This should be extracted but isn't. + + +# XXX Multiline docstrings should be cleaned with `inspect.cleandoc`. +def test3(x): + """multiline + docstring + """ + + +# Multiple docstrings - only the first should be extracted +def test4(x): + """docstring1""" + """docstring2""" + + +def test5(x): + """Hello, {}!""".format("world!") # XXX This should not be extracted. + + +# Nested docstrings +def test6(x): + def inner(y): + """nested docstring""" # XXX This should be extracted but isn't. + + +class Outer: + class Inner: + "nested class docstring" # XXX This should be extracted but isn't. diff --git a/Lib/test/test_tools/i18n_data/fileloc.pot b/Lib/test/test_tools/i18n_data/fileloc.pot new file mode 100644 index 00000000000000..dbd28687a73556 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/fileloc.pot @@ -0,0 +1,35 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: fileloc.py:5 fileloc.py:6 +msgid "foo" +msgstr "" + +#: fileloc.py:9 +msgid "bar" +msgstr "" + +#: fileloc.py:14 fileloc.py:18 +#, docstring +msgid "docstring" +msgstr "" + +#: fileloc.py:22 fileloc.py:26 +#, docstring +msgid "baz" +msgstr "" + diff --git a/Lib/test/test_tools/i18n_data/fileloc.py b/Lib/test/test_tools/i18n_data/fileloc.py new file mode 100644 index 00000000000000..c5d4d0595fea52 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/fileloc.py @@ -0,0 +1,26 @@ +# Test file locations +from gettext import gettext as _ + +# Duplicate strings +_('foo') +_('foo') + +# Duplicate strings on the same line should only add one location to the output +_('bar'), _('bar') + + +# Duplicate docstrings +class A: + """docstring""" + + +def f(): + """docstring""" + + +# Duplicate message and docstring +_('baz') + + +def g(): + """baz""" diff --git a/Lib/test/test_tools/i18n_data/messages.pot b/Lib/test/test_tools/i18n_data/messages.pot new file mode 100644 index 00000000000000..8d66fbc4f3a937 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/messages.pot @@ -0,0 +1,89 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: messages.py:16 +msgid "" +msgstr "" + +#: messages.py:19 messages.py:20 +msgid "parentheses" +msgstr "" + +#: messages.py:23 +msgid "Hello, world!" +msgstr "" + +#: messages.py:26 +msgid "" +"Hello,\n" +" multiline!\n" +msgstr "" + +#: messages.py:46 messages.py:89 messages.py:90 messages.py:93 messages.py:94 +#: messages.py:99 +msgid "foo" +msgid_plural "foos" +msgstr[0] "" +msgstr[1] "" + +#: messages.py:47 +msgid "something" +msgstr "" + +#: messages.py:50 +msgid "Hello, {}!" +msgstr "" + +#: messages.py:54 +msgid "1" +msgstr "" + +#: messages.py:54 +msgid "2" +msgstr "" + +#: messages.py:55 messages.py:56 +msgid "A" +msgstr "" + +#: messages.py:55 messages.py:56 +msgid "B" +msgstr "" + +#: messages.py:57 +msgid "set" +msgstr "" + +#: messages.py:63 +msgid "nested string" +msgstr "" + +#: messages.py:68 +msgid "baz" +msgstr "" + +#: messages.py:91 messages.py:92 messages.py:95 messages.py:96 +msgctxt "context" +msgid "foo" +msgid_plural "foos" +msgstr[0] "" +msgstr[1] "" + +#: messages.py:100 +msgid "domain foo" +msgstr "" + diff --git a/Lib/test/test_tools/i18n_data/messages.py b/Lib/test/test_tools/i18n_data/messages.py new file mode 100644 index 00000000000000..1e03f4e556830d --- /dev/null +++ b/Lib/test/test_tools/i18n_data/messages.py @@ -0,0 +1,110 @@ +# Test message extraction +from gettext import ( + gettext, + ngettext, + pgettext, + npgettext, + dgettext, + dngettext, + dpgettext, + dnpgettext +) + +_ = gettext + +# Empty string +_("") + +# Extra parentheses +(_("parentheses")) +((_("parentheses"))) + +# Multiline strings +_("Hello, " + "world!") + +_("""Hello, + multiline! +""") + +# Invalid arguments +_() +_(None) +_(1) +_(False) +_(("invalid")) +_(["invalid"]) +_({"invalid"}) +_("string"[3]) +_("string"[:3]) +_({"string": "foo"}) + +# pygettext does not allow keyword arguments, but both xgettext and pybabel do +_(x="kwargs work!") + +# Unusual, but valid arguments +_("foo", "bar") +_("something", x="something else") + +# .format() +_("Hello, {}!").format("world") # valid +_("Hello, {}!".format("world")) # invalid, but xgettext and pybabel extract the first string + +# Nested structures +_("1"), _("2") +arr = [_("A"), _("B")] +obj = {'a': _("A"), 'b': _("B")} +{{{_('set')}}} + + +# Nested functions and classes +def test(): + _("nested string") # XXX This should be extracted but isn't. + [_("nested string")] + + +class Foo: + def bar(self): + return _("baz") + + +def bar(x=_('default value')): # XXX This should be extracted but isn't. + pass + + +def baz(x=[_('default value')]): # XXX This should be extracted but isn't. + pass + + +# Shadowing _() +def _(x): + pass + + +def _(x="don't extract me"): + pass + + +# Other gettext functions +gettext("foo") +ngettext("foo", "foos", 1) +pgettext("context", "foo") +npgettext("context", "foo", "foos", 1) +dgettext("domain", "foo") +dngettext("domain", "foo", "foos", 1) +dpgettext("domain", "context", "foo") +dnpgettext("domain", "context", "foo", "foos", 1) + +# Complex arguments +ngettext("foo", "foos", 42 + (10 - 20)) +dgettext(["some", {"complex"}, ("argument",)], "domain foo") + +# Invalid calls which are not extracted +gettext() +ngettext('foo') +pgettext('context') +npgettext('context', 'foo') +dgettext('domain') +dngettext('domain', 'foo') +dpgettext('domain', 'context') +dnpgettext('domain', 'context', 'foo') diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index c083a04475e726..29c3423e234d20 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -1,9 +1,11 @@ """Tests to cover the Tools/i18n package""" import os +import re import sys import unittest from textwrap import dedent +from pathlib import Path from test.support.script_helper import assert_python_ok from test.test_tools import skip_if_missing, toolsdir @@ -12,20 +14,47 @@ skip_if_missing() +DATA_DIR = Path(__file__).resolve().parent / 'i18n_data' + + +def normalize_POT_file(pot): + """Normalize the POT creation timestamp, charset and + file locations to make the POT file easier to compare. + + """ + # Normalize the creation date. + date_pattern = re.compile(r'"POT-Creation-Date: .+?\\n"') + header = r'"POT-Creation-Date: 2000-01-01 00:00+0000\\n"' + pot = re.sub(date_pattern, header, pot) + + # Normalize charset to UTF-8 (currently there's no way to specify the output charset). + charset_pattern = re.compile(r'"Content-Type: text/plain; charset=.+?\\n"') + charset = r'"Content-Type: text/plain; charset=UTF-8\\n"' + pot = re.sub(charset_pattern, charset, pot) + + # Normalize file location path separators in case this test is + # running on Windows (which uses '\'). + fileloc_pattern = re.compile(r'#:.+') + + def replace(match): + return match[0].replace(os.sep, "/") + pot = re.sub(fileloc_pattern, replace, pot) + return pot + class Test_pygettext(unittest.TestCase): """Tests for the pygettext.py tool""" - script = os.path.join(toolsdir,'i18n', 'pygettext.py') + script = Path(toolsdir, 'i18n', 'pygettext.py') def get_header(self, data): """ utility: return the header of a .po file as a dictionary """ headers = {} for line in data.split('\n'): - if not line or line.startswith(('#', 'msgid','msgstr')): + if not line or line.startswith(('#', 'msgid', 'msgstr')): continue line = line.strip('"') - key, val = line.split(':',1) + key, val = line.split(':', 1) headers[key] = val.strip() return headers @@ -53,23 +82,34 @@ def get_msgids(self, data): return msgids - def extract_docstrings_from_str(self, module_content): - """ utility: return all msgids extracted from module_content """ - filename = 'test_docstrings.py' - with temp_cwd(None) as cwd: + def assert_POT_equal(self, expected, actual): + """Check if two POT files are equal""" + self.maxDiff = None + self.assertEqual(normalize_POT_file(expected), normalize_POT_file(actual)) + + def extract_from_str(self, module_content, *, args=(), strict=True): + """Return all msgids extracted from module_content.""" + filename = 'test.py' + with temp_cwd(None): with open(filename, 'w', encoding='utf-8') as fp: fp.write(module_content) - assert_python_ok(self.script, '-D', filename) + res = assert_python_ok('-Xutf8', self.script, *args, filename) + if strict: + self.assertEqual(res.err, b'') with open('messages.pot', encoding='utf-8') as fp: data = fp.read() return self.get_msgids(data) + def extract_docstrings_from_str(self, module_content): + """Return all docstrings extracted from module_content.""" + return self.extract_from_str(module_content, args=('--docstrings',), strict=False) + def test_header(self): """Make sure the required fields are in the header, according to: http://www.gnu.org/software/gettext/manual/gettext.html#Header-Entry """ with temp_cwd(None) as cwd: - assert_python_ok(self.script) + assert_python_ok('-Xutf8', self.script) with open('messages.pot', encoding='utf-8') as fp: data = fp.read() header = self.get_header(data) @@ -96,7 +136,7 @@ def test_POT_Creation_Date(self): """ Match the date format from xgettext for POT-Creation-Date """ from datetime import datetime with temp_cwd(None) as cwd: - assert_python_ok(self.script) + assert_python_ok('-Xutf8', self.script) with open('messages.pot', encoding='utf-8') as fp: data = fp.read() header = self.get_header(data) @@ -292,14 +332,14 @@ def test_calls_in_fstring_with_multiple_args(self): msgids = self.extract_docstrings_from_str(dedent('''\ f"{_('foo', 'bar')}" ''')) - self.assertNotIn('foo', msgids) + self.assertIn('foo', msgids) self.assertNotIn('bar', msgids) def test_calls_in_fstring_with_keyword_args(self): msgids = self.extract_docstrings_from_str(dedent('''\ f"{_('foo', bar='baz')}" ''')) - self.assertNotIn('foo', msgids) + self.assertIn('foo', msgids) self.assertNotIn('bar', msgids) self.assertNotIn('baz', msgids) @@ -310,6 +350,37 @@ def test_calls_in_fstring_with_partially_wrong_expression(self): self.assertNotIn('foo', msgids) self.assertIn('bar', msgids) + def test_function_and_class_names(self): + """Test that function and class names are not mistakenly extracted.""" + msgids = self.extract_from_str(dedent('''\ + def _(x): + pass + + def _(x="foo"): + pass + + async def _(x): + pass + + class _(object): + pass + ''')) + self.assertEqual(msgids, ['']) + + def test_pygettext_output(self): + """Test that the pygettext output exactly matches snapshots.""" + for input_file in DATA_DIR.glob('*.py'): + output_file = input_file.with_suffix('.pot') + with self.subTest(input_file=f'i18n_data/{input_file}'): + contents = input_file.read_text(encoding='utf-8') + with temp_cwd(None): + Path(input_file.name).write_text(contents) + assert_python_ok('-Xutf8', self.script, '--docstrings', input_file.name) + output = Path('messages.pot').read_text(encoding='utf-8') + + expected = output_file.read_text(encoding='utf-8') + self.assert_POT_equal(expected, output) + def test_files_list(self): """Make sure the directories are inspected for source files bpo-31920 @@ -318,21 +389,41 @@ def test_files_list(self): text2 = 'Text to translate2' text3 = 'Text to ignore' with temp_cwd(None), temp_dir(None) as sdir: - os.mkdir(os.path.join(sdir, 'pypkg')) - with open(os.path.join(sdir, 'pypkg', 'pymod.py'), 'w', - encoding='utf-8') as sfile: - sfile.write(f'_({text1!r})') - os.mkdir(os.path.join(sdir, 'pkg.py')) - with open(os.path.join(sdir, 'pkg.py', 'pymod2.py'), 'w', - encoding='utf-8') as sfile: - sfile.write(f'_({text2!r})') - os.mkdir(os.path.join(sdir, 'CVS')) - with open(os.path.join(sdir, 'CVS', 'pymod3.py'), 'w', - encoding='utf-8') as sfile: - sfile.write(f'_({text3!r})') - assert_python_ok(self.script, sdir) - with open('messages.pot', encoding='utf-8') as fp: - data = fp.read() + pymod = Path(sdir, 'pypkg', 'pymod.py') + pymod.parent.mkdir() + pymod.write_text(f'_({text1!r})', encoding='utf-8') + + pymod2 = Path(sdir, 'pkg.py', 'pymod2.py') + pymod2.parent.mkdir() + pymod2.write_text(f'_({text2!r})', encoding='utf-8') + + pymod3 = Path(sdir, 'CVS', 'pymod3.py') + pymod3.parent.mkdir() + pymod3.write_text(f'_({text3!r})', encoding='utf-8') + + assert_python_ok('-Xutf8', self.script, sdir) + data = Path('messages.pot').read_text(encoding='utf-8') self.assertIn(f'msgid "{text1}"', data) self.assertIn(f'msgid "{text2}"', data) self.assertNotIn(text3, data) + + +def update_POT_snapshots(): + for input_file in DATA_DIR.glob('*.py'): + output_file = input_file.with_suffix('.pot') + contents = input_file.read_bytes() + with temp_cwd(None): + Path(input_file.name).write_bytes(contents) + assert_python_ok('-Xutf8', Test_pygettext.script, '--docstrings', input_file.name) + output = Path('messages.pot').read_text(encoding='utf-8') + + output = normalize_POT_file(output) + output_file.write_text(output, encoding='utf-8') + + +if __name__ == '__main__': + # To regenerate POT files + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + update_POT_snapshots() + sys.exit(0) + unittest.main() diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py new file mode 100644 index 00000000000000..4c7588d4d93fc3 --- /dev/null +++ b/Lib/test/test_tools/test_makefile.py @@ -0,0 +1,81 @@ +""" +Tests for `Makefile`. +""" + +import os +import unittest +from test import support +import sysconfig + +MAKEFILE = sysconfig.get_makefile_filename() + +if not support.check_impl_detail(cpython=True): + raise unittest.SkipTest('cpython only') +if not os.path.exists(MAKEFILE) or not os.path.isfile(MAKEFILE): + raise unittest.SkipTest('Makefile could not be found') + + +class TestMakefile(unittest.TestCase): + def list_test_dirs(self): + result = [] + found_testsubdirs = False + with open(MAKEFILE, 'r', encoding='utf-8') as f: + for line in f: + if line.startswith('TESTSUBDIRS='): + found_testsubdirs = True + result.append( + line.removeprefix('TESTSUBDIRS=').replace( + '\\', '', + ).strip(), + ) + continue + if found_testsubdirs: + if '\t' not in line: + break + result.append(line.replace('\\', '').strip()) + return result + + @unittest.skipUnless(support.TEST_MODULES_ENABLED, "requires test modules") + def test_makefile_test_folders(self): + test_dirs = self.list_test_dirs() + idle_test = 'idlelib/idle_test' + self.assertIn(idle_test, test_dirs) + + used = set([idle_test]) + for dirpath, dirs, files in os.walk(support.TEST_HOME_DIR): + dirname = os.path.basename(dirpath) + # Skip temporary dirs: + if dirname == '__pycache__' or dirname.startswith('.'): + dirs.clear() # do not process subfolders + continue + # Skip empty dirs: + if not dirs and not files: + continue + # Skip dirs with hidden-only files: + if files and all( + filename.startswith('.') or filename == '__pycache__' + for filename in files + ): + continue + + relpath = os.path.relpath(dirpath, support.STDLIB_DIR) + with self.subTest(relpath=relpath): + self.assertIn( + relpath, + test_dirs, + msg=( + f"{relpath!r} is not included in the Makefile's list " + "of test directories to install" + ) + ) + used.add(relpath) + + # Don't check the wheel dir when Python is built --with-wheel-pkg-dir + if sysconfig.get_config_var('WHEEL_PKG_DIR'): + test_dirs.remove('test/wheeldata') + used.discard('test/wheeldata') + + # Check that there are no extra entries: + unique_test_dirs = set(test_dirs) + self.assertSetEqual(unique_test_dirs, used) + self.assertEqual(len(test_dirs), len(unique_test_dirs)) diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index c1e289bcaff9e5..e7e42531916d0d 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -6,6 +6,7 @@ from test.support.script_helper import assert_python_ok, assert_python_failure import textwrap import unittest +from types import FunctionType import trace from trace import Trace @@ -411,7 +412,7 @@ def test_issue9936(self): coverage = {} for line in stdout: lines, cov, module = line.split()[:3] - coverage[module] = (int(lines), int(cov[:-1])) + coverage[module] = (float(lines), float(cov[:-1])) # XXX This is needed to run regrtest.py as a script modname = trace._fullmodname(sys.modules[modname].__file__) self.assertIn(modname, coverage) @@ -552,12 +553,36 @@ def f(): stdout = stdout.decode() self.assertEqual(status, 0) self.assertIn('lines cov% module (path)', stdout) - self.assertIn(f'6 100% {modulename} ({filename})', stdout) + self.assertIn(f'6 100.0% {modulename} ({filename})', stdout) def test_run_as_module(self): assert_python_ok('-m', 'trace', '-l', '--module', 'timeit', '-n', '1') assert_python_failure('-m', 'trace', '-l', '--module', 'not_a_module_zzz') +class TestTrace(unittest.TestCase): + def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) + self.tracer = Trace(count=0, trace=1) + self.filemod = my_file_and_modname() + + def test_no_source_file(self): + filename = "" + co = traced_func_linear.__code__ + co = co.replace(co_filename=filename) + f = FunctionType(co, globals()) + + with captured_stdout() as out: + self.tracer.runfunc(f, 2, 3) + + out = out.getvalue().splitlines() + firstlineno = get_firstlineno(f) + self.assertIn(f" --- modulename: {self.filemod[1]}, funcname: {f.__code__.co_name}", out[0]) + self.assertIn(f"{filename}({firstlineno + 1})", out[1]) + self.assertIn(f"{filename}({firstlineno + 2})", out[2]) + self.assertIn(f"{filename}({firstlineno + 3})", out[3]) + self.assertIn(f"{filename}({firstlineno + 4})", out[4]) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index dd9b1850adf086..89980ae6f8573a 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -21,13 +21,14 @@ from test.support.os_helper import TESTFN, unlink from test.support.script_helper import assert_python_ok, assert_python_failure from test.support.import_helper import forget +from test.support import force_not_colorized, force_not_colorized_test_class import json import textwrap import traceback -import contextlib from functools import partial from pathlib import Path +import _colorize MODULE_PREFIX = f'{__name__}.' if __name__ == '__main__' else '' @@ -45,12 +46,12 @@ class TracebackCases(unittest.TestCase): # formatting of SyntaxErrors works based on changes for 2.1. def setUp(self): super().setUp() - self.colorize = traceback._COLORIZE - traceback._COLORIZE = False + self.colorize = _colorize.COLORIZE + _colorize.COLORIZE = False def tearDown(self): super().tearDown() - traceback._COLORIZE = self.colorize + _colorize.COLORIZE = self.colorize def get_exception_format(self, func, exc): try: @@ -85,7 +86,7 @@ def test_caret(self): err = self.get_exception_format(self.syntax_error_with_caret, SyntaxError) self.assertEqual(len(err), 4) - self.assertTrue(err[1].strip() == "return x!") + self.assertEqual(err[1].strip(), "return x!") self.assertIn("^", err[2]) # third line has caret self.assertEqual(err[1].find("!"), err[2].find("^")) # in the right place self.assertEqual(err[2].count("^"), 1) @@ -124,6 +125,7 @@ def test_nocaret(self): self.assertEqual(len(err), 3) self.assertEqual(err[1].strip(), "bad syntax") + @force_not_colorized def test_no_caret_with_no_debug_ranges_flag(self): # Make sure that if `-X no_debug_ranges` is used, there are no carets # in the traceback. @@ -148,7 +150,7 @@ def test_no_caret_with_no_debug_ranges_flag_python_traceback(self): import traceback try: x = 1 / 0 - except: + except ZeroDivisionError: traceback.print_exc() """) try: @@ -374,7 +376,32 @@ def f(): ' ValueError: 0\n', ]) + def test_format_exception_group_syntax_error_with_custom_values(self): + # See https://github.com/python/cpython/issues/128894 + for exc in [ + SyntaxError('error', 'abcd'), + SyntaxError('error', [None] * 4), + SyntaxError('error', (1, 2, 3, 4)), + SyntaxError('error', (1, 2, 3, 4)), + SyntaxError('error', (1, 'a', 'b', 2)), + # with end_lineno and end_offset: + SyntaxError('error', 'abcdef'), + SyntaxError('error', [None] * 6), + SyntaxError('error', (1, 2, 3, 4, 5, 6)), + SyntaxError('error', (1, 'a', 'b', 2, 'c', 'd')), + ]: + with self.subTest(exc=exc): + err = traceback.format_exception_only(exc, show_group=True) + # Should not raise an exception: + if exc.lineno is not None: + self.assertEqual(len(err), 2) + self.assertTrue(err[0].startswith(' File')) + else: + self.assertEqual(len(err), 1) + self.assertEqual(err[-1], 'SyntaxError: error\n') + @requires_subprocess() + @force_not_colorized def test_encoded_file(self): # Test that tracebacks are correctly printed for encoded source files: # - correct line number (Issue2384) @@ -416,16 +443,10 @@ def do_test(firstlines, message, charset, lineno): err_line = "raise RuntimeError('{0}')".format(message_ascii) err_msg = "RuntimeError: {0}".format(message_ascii) - self.assertIn(("line %s" % lineno), stdout[1], - "Invalid line number: {0!r} instead of {1}".format( - stdout[1], lineno)) - self.assertTrue(stdout[2].endswith(err_line), - "Invalid traceback line: {0!r} instead of {1!r}".format( - stdout[2], err_line)) + self.assertIn("line %s" % lineno, stdout[1]) + self.assertEndsWith(stdout[2], err_line) actual_err_msg = stdout[3] - self.assertTrue(actual_err_msg == err_msg, - "Invalid error message: {0!r} instead of {1!r}".format( - actual_err_msg, err_msg)) + self.assertEqual(actual_err_msg, err_msg) do_test("", "foo", "ascii", 3) for charset in ("ascii", "iso-8859-1", "utf-8", "GBK"): @@ -497,7 +518,7 @@ def test_format_exception_exc(self): traceback.format_exception(e.__class__, e) with self.assertRaisesRegex(ValueError, 'Both or neither'): traceback.format_exception(e.__class__, tb=e.__traceback__) - with self.assertRaisesRegex(TypeError, 'positional-only'): + with self.assertRaisesRegex(TypeError, 'required positional argument'): traceback.format_exception(exc=e) def test_format_exception_only_exc(self): @@ -536,20 +557,21 @@ def test_signatures(self): self.assertEqual( str(inspect.signature(traceback.format_exception)), ('(exc, /, value=, tb=, limit=None, ' - 'chain=True)')) + 'chain=True, **kwargs)')) self.assertEqual( str(inspect.signature(traceback.format_exception_only)), - '(exc, /, value=, *, show_group=False)') + '(exc, /, value=, *, show_group=False, **kwargs)') class PurePythonExceptionFormattingMixin: def get_exception(self, callable, slice_start=0, slice_end=-1): try: callable() - self.fail("No exception thrown.") - except: + except BaseException: return traceback.format_exc().splitlines()[slice_start:slice_end] + else: + self.fail("No exception thrown.") callable_line = get_exception.__code__.co_firstlineno + 2 @@ -619,6 +641,7 @@ def test_caret_in_type_annotation(self): def f_with_type(): def foo(a: THIS_DOES_NOT_EXIST ) -> int: return 0 + foo.__annotations__ lineno_f = f_with_type.__code__.co_firstlineno expected_f = ( @@ -626,7 +649,9 @@ def foo(a: THIS_DOES_NOT_EXIST ) -> int: f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' ' ~~~~~~~~^^\n' - f' File "{__file__}", line {lineno_f+1}, in f_with_type\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_type\n' + ' foo.__annotations__\n' + f' File "{__file__}", line {lineno_f+1}, in __annotate__\n' ' def foo(a: THIS_DOES_NOT_EXIST ) -> int:\n' ' ^^^^^^^^^^^^^^^^^^^\n' ) @@ -685,7 +710,6 @@ def f_with_multiline(): ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+2}, in f_with_multiline\n' ' return compile(code, "?", "exec")\n' - ' ~~~~~~~^^^^^^^^^^^^^^^^^^^\n' ' File "?", line 7\n' ' foo(a, z\n' ' ^' @@ -694,6 +718,35 @@ def f_with_multiline(): result_lines = self.get_exception(f_with_multiline) self.assertEqual(result_lines, expected_f.splitlines()) + # Check custom error messages covering multiple lines + code = textwrap.dedent(""" + dummy_call( + "dummy value" + foo="bar", + ) + """) + + def f_with_multiline(): + # Need to defer the compilation until in self.get_exception(..) + return compile(code, "?", "exec") + + lineno_f = f_with_multiline.__code__.co_firstlineno + + expected_f = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_multiline\n' + ' return compile(code, "?", "exec")\n' + ' File "?", line 3\n' + ' "dummy value"\n' + ' ^^^^^^^^^^^^^' + ) + + result_lines = self.get_exception(f_with_multiline) + self.assertEqual(result_lines, expected_f.splitlines()) + def test_caret_multiline_expression_bin_op(self): # Make sure no carets are printed for expressions spanning multiple # lines. @@ -775,8 +828,8 @@ def f_with_binary_operator(): def test_caret_for_binary_operators_with_spaces_and_parenthesis(self): def f_with_binary_operator(): a = 1 - b = "" - return ( a ) +b + b = c = "" + return ( a ) +b + c lineno_f = f_with_binary_operator.__code__.co_firstlineno expected_error = ( @@ -785,7 +838,7 @@ def f_with_binary_operator(): ' callable()\n' ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' - ' return ( a ) +b\n' + ' return ( a ) +b + c\n' ' ~~~~~~~~~~^~\n' ) result_lines = self.get_exception(f_with_binary_operator) @@ -973,7 +1026,7 @@ def f1(a): def f2(b): raise RuntimeError("fail") return f2 - return f1("x")("y") + return f1("x")("y")("z") lineno_f = f_with_call.__code__.co_firstlineno expected_error = ( @@ -982,7 +1035,7 @@ def f2(b): ' callable()\n' ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+5}, in f_with_call\n' - ' return f1("x")("y")\n' + ' return f1("x")("y")("z")\n' ' ~~~~~~~^^^^^\n' f' File "{__file__}", line {lineno_f+3}, in f2\n' ' raise RuntimeError("fail")\n' @@ -1497,8 +1550,187 @@ def f(): ' raise MemoryError()'] self.assertEqual(actual, expected) + def test_anchors_for_simple_return_statements_are_elided(self): + def g(): + 1/0 + + def f(): + return g() + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g()", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(): + 1/0 + + def f(): + return g() + 1 + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g() + 1", + " ~^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + return g(1, + 2, 4, + 5) + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g(1,", + " 2, 4,", + " 5)", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + return g(1, + 2, 4, + 5) + 1 + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g(1,", + " ~^^^", + " 2, 4,", + " ^^^^^", + " 5) + 1", + " ^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def test_anchors_for_simple_assign_statements_are_elided(self): + def g(): + 1/0 + + def f(): + x = g() + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = g()", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + x = g(1, + 2, 3, + 4) + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = g(1,", + " 2, 3,", + " 4)", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(): + 1/0 + + def f(): + x = y = g() + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = y = g()", + " ~^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + x = y = g(1, + 2, 3, + 4) + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = y = g(1,", + " ~^^^", + " 2, 3,", + " ^^^^^", + " 4)", + " ^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + @requires_debug_ranges() +@force_not_colorized_test_class class PurePythonTracebackErrorCaretTests( PurePythonExceptionFormattingMixin, TracebackErrorLocationCaretTestBase, @@ -1512,6 +1744,7 @@ class PurePythonTracebackErrorCaretTests( @cpython_only @requires_debug_ranges() +@force_not_colorized_test_class class CPythonTracebackErrorCaretTests( CAPIExceptionFormattingMixin, TracebackErrorLocationCaretTestBase, @@ -1523,6 +1756,7 @@ class CPythonTracebackErrorCaretTests( @cpython_only @requires_debug_ranges() +@force_not_colorized_test_class class CPythonTracebackLegacyErrorCaretTests( CAPIExceptionFormattingLegacyMixin, TracebackErrorLocationCaretTestBase, @@ -1593,9 +1827,9 @@ def check_traceback_format(self, cleanup_func=None): banner = tb_lines[0] self.assertEqual(len(tb_lines), 5) location, source_line = tb_lines[-2], tb_lines[-1] - self.assertTrue(banner.startswith('Traceback')) - self.assertTrue(location.startswith(' File')) - self.assertTrue(source_line.startswith(' raise')) + self.assertStartsWith(banner, 'Traceback') + self.assertStartsWith(location, ' File') + self.assertStartsWith(source_line, ' raise') def test_traceback_format(self): self.check_traceback_format() @@ -1691,7 +1925,7 @@ def f(): # Check a known (limited) number of recursive invocations def g(count=10): if count: - return g(count-1) + return g(count-1) + 1 raise ValueError with captured_output("stderr") as stderr_g: @@ -1705,13 +1939,13 @@ def g(count=10): lineno_g = g.__code__.co_firstlineno result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' ' [Previous line repeated 7 more times]\n' f' File "{__file__}", line {lineno_g+3}, in g\n' @@ -1750,13 +1984,10 @@ def h(count=10): ' ~^^\n' f' File "{__file__}", line {lineno_h+2}, in h\n' ' return h(count-1)\n' - ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_h+2}, in h\n' ' return h(count-1)\n' - ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_h+2}, in h\n' ' return h(count-1)\n' - ' ~^^^^^^^^^\n' ' [Previous line repeated 7 more times]\n' f' File "{__file__}", line {lineno_h+3}, in h\n' ' g()\n' @@ -1776,13 +2007,13 @@ def h(count=10): self.fail("no error raised") result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+3}, in g\n' ' raise ValueError\n' @@ -1790,7 +2021,7 @@ def h(count=10): ) tb_line = ( 'Traceback (most recent call last):\n' - f' File "{__file__}", line {lineno_g+80}, in _check_recursive_traceback_display\n' + f' File "{__file__}", line {lineno_g+77}, in _check_recursive_traceback_display\n' ' g(traceback._RECURSIVE_CUTOFF)\n' ' ~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' ) @@ -1808,13 +2039,13 @@ def h(count=10): self.fail("no error raised") result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' ' [Previous line repeated 1 more time]\n' f' File "{__file__}", line {lineno_g+3}, in g\n' @@ -1823,7 +2054,7 @@ def h(count=10): ) tb_line = ( 'Traceback (most recent call last):\n' - f' File "{__file__}", line {lineno_g+112}, in _check_recursive_traceback_display\n' + f' File "{__file__}", line {lineno_g+109}, in _check_recursive_traceback_display\n' ' g(traceback._RECURSIVE_CUTOFF + 1)\n' ' ~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' ) @@ -1887,6 +2118,7 @@ def deep_eg(self): return e @cpython_only + @support.skip_emscripten_stack_overflow() def test_exception_group_deep_recursion_capi(self): from _testcapi import exception_print LIMIT = 75 @@ -1898,6 +2130,7 @@ def test_exception_group_deep_recursion_capi(self): self.assertIn('ExceptionGroup', output) self.assertLessEqual(output.count('ExceptionGroup'), LIMIT) + @support.skip_emscripten_stack_overflow() def test_exception_group_deep_recursion_traceback(self): LIMIT = 75 eg = self.deep_eg() @@ -1937,10 +2170,12 @@ def test_print_exception_bad_type_python(self): boundaries = re.compile( '(%s|%s)' % (re.escape(cause_message), re.escape(context_message))) +@force_not_colorized_test_class class TestTracebackFormat(unittest.TestCase, TracebackFormatMixin): pass @cpython_only +@force_not_colorized_test_class class TestFallbackTracebackFormat(unittest.TestCase, TracebackFormatMixin): DEBUG_RANGES = False def setUp(self) -> None: @@ -1973,12 +2208,12 @@ def zero_div(self): def check_zero_div(self, msg): lines = msg.splitlines() if has_no_debug_ranges(): - self.assertTrue(lines[-3].startswith(' File')) + self.assertStartsWith(lines[-3], ' File') self.assertIn('1/0 # In zero_div', lines[-2]) else: - self.assertTrue(lines[-4].startswith(' File')) + self.assertStartsWith(lines[-4], ' File') self.assertIn('1/0 # In zero_div', lines[-3]) - self.assertTrue(lines[-1].startswith('ZeroDivisionError'), lines[-1]) + self.assertStartsWith(lines[-1], 'ZeroDivisionError') def test_simple(self): try: @@ -1988,12 +2223,12 @@ def test_simple(self): lines = self.get_report(e).splitlines() if has_no_debug_ranges(): self.assertEqual(len(lines), 4) - self.assertTrue(lines[3].startswith('ZeroDivisionError')) + self.assertStartsWith(lines[3], 'ZeroDivisionError') else: self.assertEqual(len(lines), 5) - self.assertTrue(lines[4].startswith('ZeroDivisionError')) - self.assertTrue(lines[0].startswith('Traceback')) - self.assertTrue(lines[1].startswith(' File')) + self.assertStartsWith(lines[4], 'ZeroDivisionError') + self.assertStartsWith(lines[0], 'Traceback') + self.assertStartsWith(lines[1], ' File') self.assertIn('1/0 # Marker', lines[2]) def test_cause(self): @@ -2028,15 +2263,15 @@ def test_context_suppression(self): try: try: raise Exception - except: + except Exception: raise ZeroDivisionError from None except ZeroDivisionError as _: e = _ lines = self.get_report(e).splitlines() self.assertEqual(len(lines), 4) - self.assertTrue(lines[3].startswith('ZeroDivisionError')) - self.assertTrue(lines[0].startswith('Traceback')) - self.assertTrue(lines[1].startswith(' File')) + self.assertStartsWith(lines[3], 'ZeroDivisionError') + self.assertStartsWith(lines[0], 'Traceback') + self.assertStartsWith(lines[1], ' File') self.assertIn('ZeroDivisionError from None', lines[2]) def test_cause_and_context(self): @@ -2132,19 +2367,22 @@ def test_message_none(self): def test_syntax_error_various_offsets(self): for offset in range(-5, 10): for add in [0, 2]: - text = " "*add + "text%d" % offset + text = " " * add + "text%d" % offset expected = [' File "file.py", line 1'] if offset < 1: expected.append(" %s" % text.lstrip()) elif offset <= 6: expected.append(" %s" % text.lstrip()) - expected.append(" %s^" % (" "*(offset-1))) + # Set the caret length to match the length of the text minus the offset. + caret_length = max(1, len(text.lstrip()) - offset + 1) + expected.append(" %s%s" % (" " * (offset - 1), "^" * caret_length)) else: + caret_length = max(1, len(text.lstrip()) - 4) expected.append(" %s" % text.lstrip()) - expected.append(" %s^" % (" "*5)) + expected.append(" %s%s" % (" " * 5, "^" * caret_length)) expected.append("SyntaxError: msg") expected.append("") - err = self.get_report(SyntaxError("msg", ("file.py", 1, offset+add, text))) + err = self.get_report(SyntaxError("msg", ("file.py", 1, offset + add, text))) exp = "\n".join(expected) self.assertEqual(exp, err) @@ -2377,9 +2615,9 @@ def exc(): try: try: raise EG("eg1", [ValueError(1), TypeError(2)]) - except: + except EG: raise EG("eg2", [ValueError(3), TypeError(4)]) - except: + except EG: raise ImportError(5) expected = ( @@ -2429,7 +2667,7 @@ def exc(): except Exception as e: exc = e raise EG("eg", [VE(1), exc, VE(4)]) - except: + except EG: raise EG("top", [VE(5)]) expected = (f' + Exception Group Traceback (most recent call last):\n' @@ -2699,6 +2937,33 @@ def exc(): report = self.get_report(exc) self.assertEqual(report, expected) + def test_exception_group_wrapped_naked(self): + # See gh-128799 + + def exc(): + try: + raise Exception(42) + except* Exception as e: + raise + + expected = (f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {self.callable_line}, in get_exception\n' + f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 3}, in exc\n' + f' | except* Exception as e:\n' + f' | raise\n' + f' | ExceptionGroup: (1 sub-exception)\n' + f' +-+---------------- 1 ----------------\n' + f' | Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 2}, in exc\n' + f' | raise Exception(42)\n' + f' | Exception: 42\n' + f' +------------------------------------\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) + def test_KeyboardInterrupt_at_first_line_of_frame(self): # see GH-93249 def f(): @@ -2725,6 +2990,7 @@ def f(): self.assertEqual(report, expected) +@force_not_colorized_test_class class PyExcReportingTests(BaseExceptionReportingTests, unittest.TestCase): # # This checks reporting through the 'traceback' module, with both @@ -2741,6 +3007,7 @@ def get_report(self, e): return s +@force_not_colorized_test_class class CExcReportingTests(BaseExceptionReportingTests, unittest.TestCase): # # This checks built-in reporting by the interpreter. @@ -3095,6 +3362,41 @@ def format_frame_summary(self, frame_summary, colorize=False): f' File "{__file__}", line {lno}, in f\n 1/0\n' ) + def test_summary_should_show_carets(self): + # See: https://github.com/python/cpython/issues/122353 + + # statement to execute and to get a ZeroDivisionError for a traceback + statement = "abcdef = 1 / 0 and 2.0" + colno = statement.index('1 / 0') + end_colno = colno + len('1 / 0') + + # Actual line to use when rendering the traceback + # and whose AST will be extracted (it will be empty). + cached_line = '# this line will be used during rendering' + self.addCleanup(unlink, TESTFN) + with open(TESTFN, "w") as file: + file.write(cached_line) + linecache.updatecache(TESTFN, {}) + + try: + exec(compile(statement, TESTFN, "exec")) + except ZeroDivisionError as exc: + # This is the simplest way to create a StackSummary + # whose FrameSummary items have their column offsets. + s = traceback.TracebackException.from_exception(exc).stack + self.assertIsInstance(s, traceback.StackSummary) + with unittest.mock.patch.object(s, '_should_show_carets', + wraps=s._should_show_carets) as ff: + self.assertEqual(len(s), 2) + self.assertListEqual( + s.format_frame_summary(s[1]).splitlines(), + [ + f' File "{TESTFN}", line 1, in ', + f' {cached_line}' + ] + ) + ff.assert_called_with(colno, end_colno, [cached_line], None) + class Unrepresentable: def __repr__(self) -> str: raise Exception("Unrepresentable") @@ -3207,7 +3509,7 @@ def test_long_context_chain(self): def f(): try: 1/0 - except: + except ZeroDivisionError: f() try: @@ -3311,7 +3613,7 @@ def test_comparison_params_variations(self): def raise_exc(): try: raise ValueError('bad value') - except: + except ValueError: raise def raise_with_locals(): @@ -3705,6 +4007,27 @@ class CaseChangeOverSubstitution: actual = self.get_suggestion(cls(), 'bluch') self.assertIn(suggestion, actual) + def test_getattr_suggestions_underscored(self): + class A: + bluch = None + + self.assertIn("'bluch'", self.get_suggestion(A(), 'blach')) + self.assertIn("'bluch'", self.get_suggestion(A(), '_luch')) + self.assertIn("'bluch'", self.get_suggestion(A(), '_bluch')) + + class B: + _bluch = None + def method(self, name): + getattr(self, name) + + self.assertIn("'_bluch'", self.get_suggestion(B(), '_blach')) + self.assertIn("'_bluch'", self.get_suggestion(B(), '_luch')) + self.assertNotIn("'_bluch'", self.get_suggestion(B(), 'bluch')) + + self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, '_blach'))) + self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, '_luch'))) + self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, 'bluch'))) + def test_getattr_suggestions_do_not_trigger_for_long_attributes(self): class A: blech = None @@ -3897,6 +4220,17 @@ def test_import_from_suggestions(self): actual = self.get_import_from_suggestion(code, 'bluch') self.assertIn(suggestion, actual) + def test_import_from_suggestions_underscored(self): + code = "bluch = None" + self.assertIn("'bluch'", self.get_import_from_suggestion(code, 'blach')) + self.assertIn("'bluch'", self.get_import_from_suggestion(code, '_luch')) + self.assertIn("'bluch'", self.get_import_from_suggestion(code, '_bluch')) + + code = "_bluch = None" + self.assertIn("'_bluch'", self.get_import_from_suggestion(code, '_blach')) + self.assertIn("'_bluch'", self.get_import_from_suggestion(code, '_luch')) + self.assertNotIn("'_bluch'", self.get_import_from_suggestion(code, 'bluch')) + def test_import_from_suggestions_do_not_trigger_for_long_attributes(self): code = "blech = None" @@ -4208,9 +4542,8 @@ class MiscTest(unittest.TestCase): def test_all(self): expected = set() - denylist = {'print_list'} for name in dir(traceback): - if name.startswith('_') or name in denylist: + if name.startswith('_'): continue module_object = getattr(traceback, name) if getattr(module_object, '__module__', None) == 'traceback': @@ -4277,11 +4610,14 @@ def foo(*args): x = {'a':{'b': None}} y = x['a']['b']['c'] - def baz(*args): - return foo(1,2,3,4) + def baz2(*args): + return (lambda *args: foo(*args))(1,2,3,4) + + def baz1(*args): + return baz2(1,2,3,4) def bar(): - return baz(1, + return baz1(1, 2,3 ,4) try: @@ -4291,14 +4627,14 @@ def bar(): e, capture_locals=True ) lines = "".join(exc.format(colorize=True)) - red = traceback._ANSIColors.RED - boldr = traceback._ANSIColors.BOLD_RED - reset = traceback._ANSIColors.RESET + red = _colorize.ANSIColors.RED + boldr = _colorize.ANSIColors.BOLD_RED + reset = _colorize.ANSIColors.RESET self.assertIn("y = " + red + "x['a']['b']" + reset + boldr + "['c']" + reset, lines) - self.assertIn("return " + red + "foo" + reset + boldr + "(1,2,3,4)" + reset, lines) - self.assertIn("return " + red + "baz" + reset + boldr + "(1," + reset, lines) - self.assertIn(boldr + "2,3" + reset, lines) - self.assertIn(boldr + ",4)" + reset, lines) + self.assertIn("return " + red + "(lambda *args: foo(*args))" + reset + boldr + "(1,2,3,4)" + reset, lines) + self.assertIn("return (lambda *args: " + red + "foo" + reset + boldr + "(*args)" + reset + ")(1,2,3,4)", lines) + self.assertIn("return baz2(1,2,3,4)", lines) + self.assertIn("return baz1(1,\n 2,3\n ,4)", lines) self.assertIn(red + "bar" + reset + boldr + "()" + reset, lines) def test_colorized_syntax_error(self): @@ -4309,11 +4645,11 @@ def test_colorized_syntax_error(self): e, capture_locals=True ) actual = "".join(exc.format(colorize=True)) - red = traceback._ANSIColors.RED - magenta = traceback._ANSIColors.MAGENTA - boldm = traceback._ANSIColors.BOLD_MAGENTA - boldr = traceback._ANSIColors.BOLD_RED - reset = traceback._ANSIColors.RESET + red = _colorize.ANSIColors.RED + magenta = _colorize.ANSIColors.MAGENTA + boldm = _colorize.ANSIColors.BOLD_MAGENTA + boldr = _colorize.ANSIColors.BOLD_RED + reset = _colorize.ANSIColors.RESET expected = "".join([ f' File {magenta}""{reset}, line {magenta}1{reset}\n', f' a {boldr}${reset} b\n', @@ -4332,15 +4668,15 @@ def foo(): self.fail("No exception thrown.") except Exception as e: with captured_output("stderr") as tbstderr: - with unittest.mock.patch('traceback._can_colorize', return_value=True): + with unittest.mock.patch('_colorize.can_colorize', return_value=True): exception_print(e) actual = tbstderr.getvalue().splitlines() - red = traceback._ANSIColors.RED - boldr = traceback._ANSIColors.BOLD_RED - magenta = traceback._ANSIColors.MAGENTA - boldm = traceback._ANSIColors.BOLD_MAGENTA - reset = traceback._ANSIColors.RESET + red = _colorize.ANSIColors.RED + boldr = _colorize.ANSIColors.BOLD_RED + magenta = _colorize.ANSIColors.MAGENTA + boldm = _colorize.ANSIColors.BOLD_MAGENTA + reset = _colorize.ANSIColors.RESET lno_foo = foo.__code__.co_firstlineno expected = ['Traceback (most recent call last):', f' File {magenta}"{__file__}"{reset}, ' @@ -4354,32 +4690,49 @@ def foo(): f'{boldm}ZeroDivisionError{reset}: {magenta}division by zero{reset}'] self.assertEqual(actual, expected) - def test_colorized_detection_checks_for_environment_variables(self): - if sys.platform == "win32": - virtual_patching = unittest.mock.patch("nt._supports_virtual_terminal", return_value=True) - else: - virtual_patching = contextlib.nullcontext() - with virtual_patching: - with unittest.mock.patch("os.isatty") as isatty_mock: - isatty_mock.return_value = True - with unittest.mock.patch("os.environ", {'TERM': 'dumb'}): - self.assertEqual(traceback._can_colorize(), False) - with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '1'}): - self.assertEqual(traceback._can_colorize(), True) - with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '0'}): - self.assertEqual(traceback._can_colorize(), False) - with unittest.mock.patch("os.environ", {'NO_COLOR': '1'}): - self.assertEqual(traceback._can_colorize(), False) - with unittest.mock.patch("os.environ", {'NO_COLOR': '1', "PYTHON_COLORS": '1'}): - self.assertEqual(traceback._can_colorize(), True) - with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1'}): - self.assertEqual(traceback._can_colorize(), True) - with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1', 'NO_COLOR': '1'}): - self.assertEqual(traceback._can_colorize(), False) - with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1', "PYTHON_COLORS": '0'}): - self.assertEqual(traceback._can_colorize(), False) - isatty_mock.return_value = False - self.assertEqual(traceback._can_colorize(), False) + def test_colorized_traceback_from_exception_group(self): + def foo(): + exceptions = [] + try: + 1 / 0 + except ZeroDivisionError as inner_exc: + exceptions.append(inner_exc) + raise ExceptionGroup("test", exceptions) + + try: + foo() + except Exception as e: + exc = traceback.TracebackException.from_exception( + e, capture_locals=True + ) + + red = _colorize.ANSIColors.RED + boldr = _colorize.ANSIColors.BOLD_RED + magenta = _colorize.ANSIColors.MAGENTA + boldm = _colorize.ANSIColors.BOLD_MAGENTA + reset = _colorize.ANSIColors.RESET + lno_foo = foo.__code__.co_firstlineno + actual = "".join(exc.format(colorize=True)).splitlines() + expected = [f" + Exception Group Traceback (most recent call last):", + f' | File {magenta}"{__file__}"{reset}, line {magenta}{lno_foo+9}{reset}, in {magenta}test_colorized_traceback_from_exception_group{reset}', + f' | {red}foo{reset}{boldr}(){reset}', + f' | {red}~~~{reset}{boldr}^^{reset}', + f" | e = ExceptionGroup('test', [ZeroDivisionError('division by zero')])", + f" | foo = {foo}", + f' | self = <{__name__}.TestColorizedTraceback testMethod=test_colorized_traceback_from_exception_group>', + f' | File {magenta}"{__file__}"{reset}, line {magenta}{lno_foo+6}{reset}, in {magenta}foo{reset}', + f' | raise ExceptionGroup("test", exceptions)', + f" | exceptions = [ZeroDivisionError('division by zero')]", + f' | {boldm}ExceptionGroup{reset}: {magenta}test (1 sub-exception){reset}', + f' +-+---------------- 1 ----------------', + f' | Traceback (most recent call last):', + f' | File {magenta}"{__file__}"{reset}, line {magenta}{lno_foo+3}{reset}, in {magenta}foo{reset}', + f' | {red}1 {reset}{boldr}/{reset}{red} 0{reset}', + f' | {red}~~{reset}{boldr}^{reset}{red}~~{reset}', + f" | exceptions = [ZeroDivisionError('division by zero')]", + f' | {boldm}ZeroDivisionError{reset}: {magenta}division by zero{reset}', + f' +------------------------------------'] + self.assertEqual(actual, expected) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index bea124521032d1..0220a83d24b428 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -1,13 +1,16 @@ import contextlib import os import sys +import textwrap import tracemalloc import unittest from unittest.mock import patch from test.support.script_helper import (assert_python_ok, assert_python_failure, interpreter_requires_environment) from test import support +from test.support import force_not_colorized from test.support import os_helper +from test.support import threading_helper try: import _testcapi @@ -17,6 +20,7 @@ _testinternalcapi = None +DEFAULT_DOMAIN = 0 EMPTY_STRING_SIZE = sys.getsizeof(b'') INVALID_NFRAME = (-1, 2**30) @@ -938,6 +942,7 @@ def test_env_limit(self): stdout = stdout.rstrip() self.assertEqual(stdout, b'10') + @force_not_colorized def check_env_var_invalid(self, nframe): with support.SuppressCrashReport(): ok, stdout, stderr = assert_python_failure( @@ -950,7 +955,6 @@ def check_env_var_invalid(self, nframe): return self.fail(f"unexpected output: {stderr!a}") - def test_env_var_invalid(self): for nframe in INVALID_NFRAME: with self.subTest(nframe=nframe): @@ -979,6 +983,7 @@ def check_sys_xoptions_invalid(self, nframe): return self.fail(f"unexpected output: {stderr!a}") + @force_not_colorized def test_sys_xoptions_invalid(self): for nframe in INVALID_NFRAME: with self.subTest(nframe=nframe): @@ -1024,8 +1029,8 @@ def track(self, release_gil=False, nframe=1): release_gil) return frames - def untrack(self): - _testcapi.tracemalloc_untrack(self.domain, self.ptr) + def untrack(self, release_gil=False): + _testcapi.tracemalloc_untrack(self.domain, self.ptr, release_gil) def get_traced_memory(self): # Get the traced size in the domain @@ -1067,7 +1072,7 @@ def test_track_already_tracked(self): self.assertEqual(self.get_traceback(), tracemalloc.Traceback(frames)) - def test_untrack(self): + def check_untrack(self, release_gil): tracemalloc.start() self.track() @@ -1075,13 +1080,19 @@ def test_untrack(self): self.assertEqual(self.get_traced_memory(), self.size) # untrack must remove the trace - self.untrack() + self.untrack(release_gil) self.assertIsNone(self.get_traceback()) self.assertEqual(self.get_traced_memory(), 0) # calling _PyTraceMalloc_Untrack() multiple times must not crash - self.untrack() - self.untrack() + self.untrack(release_gil) + self.untrack(release_gil) + + def test_untrack(self): + self.check_untrack(False) + + def test_untrack_without_gil(self): + self.check_untrack(True) def test_stop_track(self): tracemalloc.start() @@ -1099,6 +1110,37 @@ def test_stop_untrack(self): with self.assertRaises(RuntimeError): self.untrack() + @unittest.skipIf(_testcapi is None, 'need _testcapi') + @threading_helper.requires_working_threading() + # gh-128679: Test crash on a debug build (especially on FreeBSD). + @unittest.skipIf(support.Py_DEBUG, 'need release build') + def test_tracemalloc_track_race(self): + # gh-128679: Test fix for tracemalloc.stop() race condition + _testcapi.tracemalloc_track_race() + + def test_late_untrack(self): + code = textwrap.dedent(f""" + from test import support + import tracemalloc + import _testcapi + + class Tracked: + def __init__(self, domain, size): + self.domain = domain + self.ptr = id(self) + self.size = size + _testcapi.tracemalloc_track(self.domain, self.ptr, self.size) + + def __del__(self, untrack=_testcapi.tracemalloc_untrack): + untrack(self.domain, self.ptr, 1) + + domain = {DEFAULT_DOMAIN} + tracemalloc.start() + obj = Tracked(domain, 1024 * 1024) + support.late_deletion(obj) + """) + assert_python_ok("-c", code) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ttk/test_style.py b/Lib/test/test_ttk/test_style.py index 9a04a95dc40d65..19918772514ad4 100644 --- a/Lib/test/test_ttk/test_style.py +++ b/Lib/test/test_ttk/test_style.py @@ -205,7 +205,8 @@ def test_element_create_from_errors(self): style = self.style with self.assertRaises(IndexError): style.element_create('plain.newelem', 'from') - with self.assertRaisesRegex(TclError, 'theme "spam" doesn\'t exist'): + with self.assertRaisesRegex(TclError, + 'theme "spam" (does not|doesn\'t) exist'): style.element_create('plain.newelem', 'from', 'spam') def test_element_create_image(self): @@ -227,13 +228,13 @@ def test_element_create_image(self): foreground='blue', background='yellow') img3 = tkinter.BitmapImage(master=self.root, file=imgfile, foreground='white', background='black') - style.element_create('Button.button', 'image', + style.element_create('TestButton.button', 'image', img1, ('pressed', img2), ('active', img3), border=(2, 4), sticky='we') - self.assertIn('Button.button', style.element_names()) + self.assertIn('TestButton.button', style.element_names()) - style.layout('Button', [('Button.button', {'sticky': 'news'})]) - b = ttk.Button(self.root, style='Button') + style.layout('TestButton', [('TestButton.button', {'sticky': 'news'})]) + b = ttk.Button(self.root, style='TestButton') b.pack(expand=True, fill='both') self.assertEqual(b.winfo_reqwidth(), 16) self.assertEqual(b.winfo_reqheight(), 16) diff --git a/Lib/test/test_ttk/test_widgets.py b/Lib/test/test_ttk/test_widgets.py index fd1a748a498ac5..d5620becfa7187 100644 --- a/Lib/test/test_ttk/test_widgets.py +++ b/Lib/test/test_ttk/test_widgets.py @@ -5,9 +5,10 @@ import sys from test.test_ttk_textonly import MockTclObj -from test.test_tkinter.support import (AbstractTkTest, tk_version, get_tk_patchlevel, - simulate_mouse_click, AbstractDefaultRootTest) -from test.test_tkinter.widget_tests import (add_standard_options, +from test.test_tkinter.support import ( + AbstractTkTest, requires_tk, tk_version, get_tk_patchlevel, + simulate_mouse_click, AbstractDefaultRootTest) +from test.test_tkinter.widget_tests import (add_configure_tests, AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests) requires('gui') @@ -27,16 +28,27 @@ def test_configure_class(self): def test_configure_padding(self): widget = self.create() - self.checkParam(widget, 'padding', 0, expected=('0',)) - self.checkParam(widget, 'padding', 5, expected=('5',)) - self.checkParam(widget, 'padding', (5, 6), expected=('5', '6')) + if get_tk_patchlevel(self.root) < (8, 6, 14): + def padding_conv(value): + self.assertIsInstance(value, tuple) + return tuple(map(str, value)) + else: + padding_conv = None + self.checkParam(widget, 'padding', 0, expected=(0,), conv=padding_conv) + self.checkParam(widget, 'padding', 5, expected=(5,), conv=padding_conv) + self.checkParam(widget, 'padding', (5, 6), + expected=(5, 6), conv=padding_conv) self.checkParam(widget, 'padding', (5, 6, 7), - expected=('5', '6', '7')) + expected=(5, 6, 7), conv=padding_conv) self.checkParam(widget, 'padding', (5, 6, 7, 8), - expected=('5', '6', '7', '8')) + expected=(5, 6, 7, 8), conv=padding_conv) self.checkParam(widget, 'padding', ('5p', '6p', '7p', '8p')) self.checkParam(widget, 'padding', (), expected='') + def test_configure_state(self): + widget = self.create() + self.checkParams(widget, 'state', 'active', 'disabled', 'readonly') + def test_configure_style(self): widget = self.create() self.assertEqual(widget['style'], '') @@ -50,6 +62,11 @@ def test_configure_style(self): self.assertEqual(widget2['class'], 'Foo') # XXX + def test_configure_relief(self): + widget = self.create() + self.checkReliefParam(widget, 'relief', + allow_empty=(tk_version >= (8, 7))) + class WidgetTest(AbstractTkTest, unittest.TestCase): """Tests methods available in every ttk widget.""" @@ -108,10 +125,11 @@ def test_cb(arg1, **kw): class AbstractToplevelTest(AbstractWidgetTest, PixelSizeTests): - _conv_pixels = False + _rounds_pixels = False + _clipped = {} -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class FrameTest(AbstractToplevelTest, unittest.TestCase): OPTIONS = ( 'borderwidth', 'class', 'cursor', 'height', @@ -123,7 +141,7 @@ def create(self, **kwargs): return ttk.Frame(self.root, **kwargs) -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class LabelFrameTest(AbstractToplevelTest, unittest.TestCase): OPTIONS = ( 'borderwidth', 'class', 'cursor', 'height', @@ -150,6 +168,9 @@ def test_configure_labelwidget(self): class AbstractLabelTest(AbstractWidgetTest): + _allow_empty_justify = True + _rounds_pixels = False + _clipped = {} def checkImageParam(self, widget, name): image = tkinter.PhotoImage(master=self.root, name='image1') @@ -161,28 +182,27 @@ def checkImageParam(self, widget, name): expected=('image1', 'active', 'image2')) self.checkParam(widget, name, 'image1 active image2', expected=('image1', 'active', 'image2')) - self.checkInvalidParam(widget, name, 'spam', - errmsg='image "spam" doesn\'t exist') + if tk_version < (9, 0): + errmsg = 'image "spam" doesn\'t exist' + else: + errmsg = 'image "spam" does not exist' + self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) def test_configure_compound(self): - options = 'none text image center top bottom left right'.split() - errmsg = ( - 'bad compound "{}": must be' - f' {", ".join(options[:-1])}, or {options[-1]}' - ) + values = ('none', 'text', 'image', 'center', 'top', 'bottom', 'left', 'right') + if tk_version >= (8, 7): + values += ('',) widget = self.create() - self.checkEnumParam(widget, 'compound', *options, errmsg=errmsg) + self.checkEnumParam(widget, 'compound', *values, allow_empty=True) - def test_configure_state(self): - widget = self.create() - self.checkParams(widget, 'state', 'active', 'disabled', 'normal') + test_configure_justify = requires_tk(8, 7)(StandardOptionsTests.test_configure_justify) def test_configure_width(self): widget = self.create() self.checkParams(widget, 'width', 402, -402, 0) -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class LabelTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'anchor', 'background', 'borderwidth', @@ -192,21 +212,19 @@ class LabelTest(AbstractLabelTest, unittest.TestCase): 'underline', 'width', 'wraplength', ) _conv_pixels = False + _allow_empty_justify = tk_version >= (8, 7) def create(self, **kwargs): return ttk.Label(self.root, **kwargs) - def test_configure_font(self): - widget = self.create() - self.checkParam(widget, 'font', - '-Adobe-Helvetica-Medium-R-Normal--*-120-*-*-*-*-*-*') + test_configure_justify = StandardOptionsTests.test_configure_justify -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class ButtonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'class', 'command', 'compound', 'cursor', 'default', - 'image', 'padding', 'state', 'style', + 'image', 'justify', 'padding', 'state', 'style', 'takefocus', 'text', 'textvariable', 'underline', 'width', ) @@ -216,7 +234,9 @@ def create(self, **kwargs): def test_configure_default(self): widget = self.create() - self.checkEnumParam(widget, 'default', 'normal', 'active', 'disabled') + values = ('normal', 'active', 'disabled') + self.checkEnumParam(widget, 'default', *values, + sort=tk_version >= (8, 7)) def test_invoke(self): success = [] @@ -225,11 +245,11 @@ def test_invoke(self): self.assertTrue(success) -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class CheckbuttonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'class', 'command', 'compound', 'cursor', - 'image', + 'image', 'justify', 'offvalue', 'onvalue', 'padding', 'state', 'style', 'takefocus', 'text', 'textvariable', @@ -268,7 +288,10 @@ def cb_test(): cbtn['command'] = '' res = cbtn.invoke() - self.assertFalse(str(res)) + if tk_version >= (8, 7) and self.wantobjects: + self.assertEqual(res, ()) + else: + self.assertEqual(str(res), '') self.assertLessEqual(len(success), 1) self.assertEqual(cbtn['offvalue'], cbtn.tk.globalgetvar(cbtn['variable'])) @@ -285,20 +308,44 @@ def test_unique_variables(self): b.pack() buttons.append(b) variables = [str(b['variable']) for b in buttons] - print(variables) self.assertEqual(len(set(variables)), 4, variables) + def test_unique_variables2(self): + buttons = [] + f = ttk.Frame(self.root) + f.pack() + f = ttk.Frame(self.root) + f.pack() + for j in 'AB': + b = tkinter.Checkbutton(f, text=j) + b.pack() + buttons.append(b) + # Should be larger than the number of all previously created + # tkinter.Checkbutton widgets: + for j in range(100): + b = ttk.Checkbutton(f, text=str(j)) + b.pack() + buttons.append(b) + names = [str(b) for b in buttons] + self.assertEqual(len(set(names)), len(buttons), names) + variables = [str(b['variable']) for b in buttons] + self.assertEqual(len(set(variables)), len(buttons), variables) + -@add_standard_options(IntegerSizeTests, StandardTtkOptionsTests) +@add_configure_tests(IntegerSizeTests, StandardTtkOptionsTests) class EntryTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'background', 'class', 'cursor', 'exportselection', 'font', 'foreground', 'invalidcommand', 'justify', + 'placeholder', 'placeholderforeground', 'show', 'state', 'style', 'takefocus', 'textvariable', 'validate', 'validatecommand', 'width', 'xscrollcommand', ) - IDENTIFY_AS = 'Entry.field' if sys.platform == 'darwin' else 'textarea' + _rounds_pixels = False + _clipped = {} + # bpo-27313: macOS Tk/Tcl may or may not report 'Entry.field'. + IDENTIFY_AS = {'Entry.field', 'textarea'} def setUp(self): super().setUp() @@ -317,11 +364,6 @@ def test_configure_show(self): self.checkParam(widget, 'show', '') self.checkParam(widget, 'show', ' ') - def test_configure_state(self): - widget = self.create() - self.checkParams(widget, 'state', - 'disabled', 'normal', 'readonly') - def test_configure_validate(self): widget = self.create() self.checkEnumParam(widget, 'validate', @@ -337,11 +379,14 @@ def test_bbox(self): self.assertRaises(tkinter.TclError, self.entry.bbox, None) def test_identify(self): + if (tk_version >= (9, 0) and sys.platform == 'darwin' + and isinstance(self.entry, ttk.Combobox)): + self.skipTest('Test does not work on macOS Tk 9.') + # https://core.tcl-lang.org/tk/tktview/8b49e9cfa6 self.entry.pack() - self.entry.update() + self.root.update() - # bpo-27313: macOS Cocoa widget differs from X, allow either - self.assertEqual(self.entry.identify(5, 5), self.IDENTIFY_AS) + self.assertIn(self.entry.identify(5, 5), self.IDENTIFY_AS) self.assertEqual(self.entry.identify(-1, -1), "") self.assertRaises(tkinter.TclError, self.entry.identify, None, 5) @@ -417,17 +462,18 @@ def validate(content): self.assertEqual(self.entry.state(), ()) -@add_standard_options(IntegerSizeTests, StandardTtkOptionsTests) +@add_configure_tests(IntegerSizeTests, StandardTtkOptionsTests) class ComboboxTest(EntryTest, unittest.TestCase): OPTIONS = ( 'background', 'class', 'cursor', 'exportselection', 'font', 'foreground', 'height', 'invalidcommand', - 'justify', 'postcommand', 'show', 'state', 'style', + 'justify', 'placeholder', 'placeholderforeground', 'postcommand', + 'show', 'state', 'style', 'takefocus', 'textvariable', 'validate', 'validatecommand', 'values', 'width', 'xscrollcommand', ) - IDENTIFY_AS = 'Combobox.button' if sys.platform == 'darwin' else 'textarea' + IDENTIFY_AS = {'Combobox.button', 'textarea'} def setUp(self): super().setUp() @@ -445,11 +491,14 @@ def _show_drop_down_listbox(self): x, y = width - 5, 5 if sys.platform != 'darwin': # there's no down arrow on macOS self.assertRegex(self.combo.identify(x, y), r'.*downarrow\Z') - self.combo.event_generate('', x=x, y=y) + self.combo.event_generate('', x=x, y=y) self.combo.event_generate('', x=x, y=y) - self.combo.update_idletasks() def test_virtual_event(self): + if (tk_version >= (9, 0) and sys.platform == 'darwin' + and isinstance(self.entry, ttk.Combobox)): + self.skipTest('Test does not work on macOS Tk 9.') + # https://core.tcl-lang.org/tk/tktview/8b49e9cfa6 success = [] self.combo['values'] = [1] @@ -467,6 +516,10 @@ def test_virtual_event(self): self.assertTrue(success) def test_configure_postcommand(self): + if (tk_version >= (9, 0) and sys.platform == 'darwin' + and isinstance(self.entry, ttk.Combobox)): + self.skipTest('Test does not work on macOS Tk 9.') + # https://core.tcl-lang.org/tk/tktview/8b49e9cfa6 success = [] self.combo['postcommand'] = lambda: success.append(True) @@ -486,7 +539,7 @@ def check_get_current(getval, currval): self.assertEqual(self.combo.get(), getval) self.assertEqual(self.combo.current(), currval) - self.assertEqual(self.combo['values'], '') + self.assertIn(self.combo['values'], ((), '')) check_get_current('', -1) self.checkParam(self.combo, 'values', 'mon tue wed thur', @@ -542,12 +595,14 @@ def check_get_current(getval, currval): combo2.destroy() -@add_standard_options(IntegerSizeTests, StandardTtkOptionsTests) +@add_configure_tests(IntegerSizeTests, StandardTtkOptionsTests) class PanedWindowTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'class', 'cursor', 'height', 'orient', 'style', 'takefocus', 'width', ) + _rounds_pixels = False + _clipped = {} def setUp(self): super().setUp() @@ -611,8 +666,14 @@ def test_insert(self): child2 = ttk.Label(self.root) child3 = ttk.Label(self.root) - self.assertRaises(tkinter.TclError, self.paned.insert, 0, child) + if tk_version >= (8, 7): + self.paned.insert(0, child) + self.assertEqual(self.paned.panes(), (str(child),)) + self.paned.forget(0) + else: + self.assertRaises(tkinter.TclError, self.paned.insert, 0, child) + self.assertEqual(self.paned.panes(), ()) self.paned.insert('end', child2) self.paned.insert(0, child) self.assertEqual(self.paned.panes(), (str(child), str(child2))) @@ -672,11 +733,11 @@ def test_sashpos(self): self.assertIsInstance(self.paned.sashpos(0), int) -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class RadiobuttonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'class', 'command', 'compound', 'cursor', - 'image', + 'image', 'justify', 'padding', 'state', 'style', 'takefocus', 'text', 'textvariable', 'underline', 'value', 'variable', 'width', @@ -715,7 +776,10 @@ def cb_test(): cbtn2['command'] = '' res = cbtn2.invoke() - self.assertEqual(str(res), '') + if tk_version >= (8, 7) and self.wantobjects: + self.assertEqual(res, ()) + else: + self.assertEqual(str(res), '') self.assertLessEqual(len(success), 1) self.assertEqual(conv(cbtn2['value']), myvar.get()) self.assertEqual(myvar.get(), @@ -727,7 +791,7 @@ def cb_test(): class MenubuttonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'class', 'compound', 'cursor', 'direction', - 'image', 'menu', 'padding', 'state', 'style', + 'image', 'justify', 'menu', 'padding', 'state', 'style', 'takefocus', 'text', 'textvariable', 'underline', 'width', ) @@ -735,10 +799,11 @@ class MenubuttonTest(AbstractLabelTest, unittest.TestCase): def create(self, **kwargs): return ttk.Menubutton(self.root, **kwargs) - def test_direction(self): + def test_configure_direction(self): widget = self.create() - self.checkEnumParam(widget, 'direction', - 'above', 'below', 'left', 'right', 'flush') + values = ('above', 'below', 'left', 'right', 'flush') + self.checkEnumParam(widget, 'direction', *values, + sort=tk_version >= (8, 7)) def test_configure_menu(self): widget = self.create() @@ -747,13 +812,14 @@ def test_configure_menu(self): menu.destroy() -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class ScaleTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'class', 'command', 'cursor', 'from', 'length', - 'orient', 'style', 'takefocus', 'to', 'value', 'variable', + 'orient', 'state', 'style', 'takefocus', 'to', 'value', 'variable', ) - _conv_pixels = False + _rounds_pixels = False + _clipped = {} default_orient = 'horizontal' def setUp(self): @@ -773,6 +839,8 @@ def test_configure_length(self): widget = self.create() self.checkPixelsParam(widget, 'length', 130, 131.2, 135.6, '5i') + test_configure_state = requires_tk(8, 6, 9)(StandardTtkOptionsTests.test_configure_state) + def test_configure_to(self): widget = self.create() self.checkFloatParam(widget, 'to', 300, 14.9, 15.1, -10, conv=False) @@ -853,19 +921,32 @@ def test_set(self): self.assertRaises(tkinter.TclError, self.scale.set, None) -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class ProgressbarTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( - 'class', 'cursor', 'orient', 'length', - 'mode', 'maximum', 'phase', + 'anchor', 'class', 'cursor', 'font', 'foreground', 'justify', + 'orient', 'length', + 'mode', 'maximum', 'phase', 'text', 'wraplength', 'style', 'takefocus', 'value', 'variable', ) - _conv_pixels = False + _rounds_pixels = False + _clipped = {} + _allow_empty_justify = True default_orient = 'horizontal' def create(self, **kwargs): return ttk.Progressbar(self.root, **kwargs) + @requires_tk(8, 7) + def test_configure_anchor(self): + widget = self.create() + self.checkEnumParam(widget, 'anchor', + 'n', 'ne', 'e', 'se', 's', 'sw', 'w', 'nw', 'center', '') + + test_configure_font = requires_tk(8, 7)(StandardOptionsTests.test_configure_font) + test_configure_foreground = requires_tk(8, 7)(StandardOptionsTests.test_configure_foreground) + test_configure_justify = requires_tk(8, 7)(StandardTtkOptionsTests.test_configure_justify) + def test_configure_length(self): widget = self.create() self.checkPixelsParam(widget, 'length', 100.1, 56.7, '2i') @@ -882,30 +963,39 @@ def test_configure_phase(self): # XXX pass + test_configure_text = requires_tk(8, 7)(StandardOptionsTests.test_configure_text) + def test_configure_value(self): widget = self.create() self.checkFloatParam(widget, 'value', 150.2, 77.7, 0, -10, conv=False) + test_configure_wraplength = requires_tk(8, 7)(StandardOptionsTests.test_configure_wraplength) + @unittest.skipIf(sys.platform == 'darwin', 'ttk.Scrollbar is special on MacOSX') -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class ScrollbarTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'class', 'command', 'cursor', 'orient', 'style', 'takefocus', ) + _rounds_pixels = False + _clipped = {} default_orient = 'vertical' def create(self, **kwargs): return ttk.Scrollbar(self.root, **kwargs) -@add_standard_options(IntegerSizeTests, StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class NotebookTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'class', 'cursor', 'height', 'padding', 'style', 'takefocus', 'width', ) + _rounds_pixels = (tk_version < (9,0)) + _converts_pixels = False + _clipped = {} def setUp(self): super().setUp() @@ -918,6 +1008,20 @@ def setUp(self): def create(self, **kwargs): return ttk.Notebook(self.root, **kwargs) + def test_configure_height(self): + widget = self.create() + if get_tk_patchlevel(self.root) < (8, 6, 15): + self.checkIntegerParam(widget, 'height', 402, -402, 0) + else: + self.checkPixelsParam(widget, 'height', '10c', 402, -402, 0) + + def test_configure_width(self): + widget = self.create() + if get_tk_patchlevel(self.root) < (8, 6, 15): + self.checkIntegerParam(widget, 'width', 402, -402, 0) + else: + self.checkPixelsParam(widget, 'width', '10c', 402, -402, 0) + def test_tab_identifiers(self): self.nb.forget(0) self.nb.hide(self.child2) @@ -1024,7 +1128,11 @@ def test_insert(self): self.nb.insert(self.child1, child3) self.assertEqual(self.nb.tabs(), (str(child3), ) + tabs) self.nb.forget(child3) - self.assertRaises(tkinter.TclError, self.nb.insert, 2, child3) + if tk_version >= (8, 7): + self.nb.insert(2, child3) + self.assertEqual(self.nb.tabs(), (*tabs, str(child3))) + else: + self.assertRaises(tkinter.TclError, self.nb.insert, 2, child3) self.assertRaises(tkinter.TclError, self.nb.insert, -1, child3) # bad inserts @@ -1078,7 +1186,12 @@ def test_traversal(self): self.nb.select(0) - focus_identify_as = 'focus' if sys.platform != 'darwin' else '' + if sys.platform == 'darwin': + focus_identify_as = '' + elif sys.platform == 'win32': + focus_identify_as = 'focus' + else: + focus_identify_as = 'focus' if tk_version < (9,0) else 'padding' self.assertEqual(self.nb.identify(5, 5), focus_identify_as) simulate_mouse_click(self.nb, 5, 5) self.nb.focus_force() @@ -1111,16 +1224,18 @@ def test_traversal(self): self.assertEqual(self.nb.select(), str(self.child2)) -@add_standard_options(IntegerSizeTests, StandardTtkOptionsTests) +@add_configure_tests(IntegerSizeTests, StandardTtkOptionsTests) class SpinboxTest(EntryTest, unittest.TestCase): OPTIONS = ( 'background', 'class', 'command', 'cursor', 'exportselection', 'font', 'foreground', 'format', 'from', 'increment', - 'invalidcommand', 'justify', 'show', 'state', 'style', + 'invalidcommand', 'justify', + 'placeholder', 'placeholderforeground', + 'show', 'state', 'style', 'takefocus', 'textvariable', 'to', 'validate', 'validatecommand', 'values', 'width', 'wrap', 'xscrollcommand', ) - IDENTIFY_AS = 'Spinbox.field' if sys.platform == 'darwin' else 'textarea' + IDENTIFY_AS = {'Spinbox.field', 'textarea'} def setUp(self): super().setUp() @@ -1286,13 +1401,16 @@ def test_configure_values(self): spin2.destroy() -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class TreeviewTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'class', 'columns', 'cursor', 'displaycolumns', - 'height', 'padding', 'selectmode', 'show', - 'style', 'takefocus', 'xscrollcommand', 'yscrollcommand', + 'height', 'padding', 'selectmode', 'selecttype', 'show', 'striped', + 'style', 'takefocus', 'titlecolumns', 'titleitems', + 'xscrollcommand', 'yscrollcommand', ) + _rounds_pixels = False + _clipped = {} def setUp(self): super().setUp() @@ -1306,7 +1424,8 @@ def test_configure_columns(self): self.checkParam(widget, 'columns', 'a b c', expected=('a', 'b', 'c')) self.checkParam(widget, 'columns', ('a', 'b', 'c')) - self.checkParam(widget, 'columns', '') + self.checkParam(widget, 'columns', '', + expected=() if tk_version >= (8, 7) else '') def test_configure_displaycolumns(self): widget = self.create() @@ -1318,22 +1437,30 @@ def test_configure_displaycolumns(self): expected=('#all',)) self.checkParam(widget, 'displaycolumns', (2, 1, 0)) self.checkInvalidParam(widget, 'displaycolumns', ('a', 'b', 'd'), - errmsg='Invalid column index d') + errmsg='Invalid column index "?d"?') + errmsg = 'Column index "?{}"? out of bounds' self.checkInvalidParam(widget, 'displaycolumns', (1, 2, 3), - errmsg='Column index 3 out of bounds') + errmsg=errmsg.format(3)) self.checkInvalidParam(widget, 'displaycolumns', (1, -2), - errmsg='Column index -2 out of bounds') + errmsg=errmsg.format(-2)) def test_configure_height(self): widget = self.create() - self.checkPixelsParam(widget, 'height', 100, -100, 0, '3c', conv=False) - self.checkPixelsParam(widget, 'height', 101.2, 102.6, conv=False) + self.checkPixelsParam(widget, 'height', 100, -100, 0, '3c', + conv=False) + self.checkPixelsParam(widget, 'height', 101.2, 102.6, '3c', + conv=False) def test_configure_selectmode(self): widget = self.create() self.checkEnumParam(widget, 'selectmode', 'none', 'browse', 'extended') + @requires_tk(8, 7) + def test_configure_selecttype(self): + widget = self.create() + self.checkEnumParam(widget, 'selecttype', 'item', 'cell') + def test_configure_show(self): widget = self.create() self.checkParam(widget, 'show', 'tree headings', @@ -1343,6 +1470,23 @@ def test_configure_show(self): self.checkParam(widget, 'show', 'tree', expected=('tree',)) self.checkParam(widget, 'show', 'headings', expected=('headings',)) + @requires_tk(8, 7) + def test_configure_striped(self): + widget = self.create() + self.checkBooleanParam(widget, 'striped') + + @requires_tk(8, 7) + def test_configure_titlecolumns(self): + widget = self.create() + self.checkIntegerParam(widget, 'titlecolumns', 0, 1, 5) + self.checkInvalidParam(widget, 'titlecolumns', -2) + + @requires_tk(8, 7) + def test_configure_titleitems(self): + widget = self.create() + self.checkIntegerParam(widget, 'titleitems', 0, 1, 5) + self.checkInvalidParam(widget, 'titleitems', -2) + def test_bbox(self): self.tv.pack() self.assertEqual(self.tv.bbox(''), '') @@ -1827,24 +1971,28 @@ def test_tag_has(self): self.assertEqual(self.tv.tag_has('tag3'), ()) -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class SeparatorTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'class', 'cursor', 'orient', 'style', 'takefocus', # 'state'? ) + _rounds_pixels = False + _clipped = {} default_orient = 'horizontal' def create(self, **kwargs): return ttk.Separator(self.root, **kwargs) -@add_standard_options(StandardTtkOptionsTests) +@add_configure_tests(StandardTtkOptionsTests) class SizegripTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'class', 'cursor', 'style', 'takefocus', # 'state'? ) + _rounds_pixels = False + _clipped = {} def create(self, **kwargs): return ttk.Sizegrip(self.root, **kwargs) @@ -1859,13 +2007,5 @@ def test_label(self): self._test_widget(ttk.Label) -tests_gui = ( - ButtonTest, CheckbuttonTest, ComboboxTest, EntryTest, - FrameTest, LabelFrameTest, LabelTest, MenubuttonTest, - NotebookTest, PanedWindowTest, ProgressbarTest, - RadiobuttonTest, ScaleTest, ScrollbarTest, SeparatorTest, - SizegripTest, SpinboxTest, TreeviewTest, WidgetTest, DefaultRootTest, - ) - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_turtle.py b/Lib/test/test_turtle.py index 14121a590a5026..d02cac284a909a 100644 --- a/Lib/test/test_turtle.py +++ b/Lib/test/test_turtle.py @@ -1,5 +1,9 @@ +import os import pickle +import re +import tempfile import unittest +import unittest.mock from test import support from test.support import import_helper from test.support import os_helper @@ -50,6 +54,21 @@ """ +def patch_screen(): + """Patch turtle._Screen for testing without a display. + + We must patch the _Screen class itself instead of the _Screen + instance because instantiating it requires a display. + """ + return unittest.mock.patch( + "turtle._Screen.__new__", + **{ + "return_value.__class__": turtle._Screen, + "return_value.mode.return_value": "standard", + }, + ) + + class TurtleConfigTest(unittest.TestCase): def get_cfg_file(self, cfg_str): @@ -130,6 +149,7 @@ def assertVectorsAlmostEqual(self, vec1, vec2): self.assertAlmostEqual( i, j, msg='values at index {} do not match'.format(idx)) + class Multiplier: def __mul__(self, other): @@ -461,6 +481,161 @@ def test_teleport(self): self.assertTrue(tpen.isdown()) +class TestTurtleScreen(unittest.TestCase): + def test_save_raises_if_wrong_extension(self) -> None: + screen = unittest.mock.Mock() + + msg = "Unknown file extension: '.png', must be one of {'.ps', '.eps'}" + with ( + tempfile.TemporaryDirectory() as tmpdir, + self.assertRaisesRegex(ValueError, re.escape(msg)) + ): + turtle.TurtleScreen.save(screen, os.path.join(tmpdir, "file.png")) + + def test_save_raises_if_parent_not_found(self) -> None: + screen = unittest.mock.Mock() + + with tempfile.TemporaryDirectory() as tmpdir: + parent = os.path.join(tmpdir, "unknown_parent") + msg = f"The directory '{parent}' does not exist. Cannot save to it" + + with self.assertRaisesRegex(FileNotFoundError, re.escape(msg)): + turtle.TurtleScreen.save(screen, os.path.join(parent, "a.ps")) + + def test_save_raises_if_file_found(self) -> None: + screen = unittest.mock.Mock() + + with tempfile.TemporaryDirectory() as tmpdir: + file_path = os.path.join(tmpdir, "some_file.ps") + with open(file_path, "w") as f: + f.write("some text") + + msg = ( + f"The file '{file_path}' already exists. To overwrite it use" + " the 'overwrite=True' argument of the save function." + ) + with self.assertRaisesRegex(FileExistsError, re.escape(msg)): + turtle.TurtleScreen.save(screen, file_path) + + def test_save_overwrites_if_specified(self) -> None: + screen = unittest.mock.Mock() + screen.cv.postscript.return_value = "postscript" + + with tempfile.TemporaryDirectory() as tmpdir: + file_path = os.path.join(tmpdir, "some_file.ps") + with open(file_path, "w") as f: + f.write("some text") + + turtle.TurtleScreen.save(screen, file_path, overwrite=True) + with open(file_path) as f: + self.assertEqual(f.read(), "postscript") + + def test_save(self) -> None: + screen = unittest.mock.Mock() + screen.cv.postscript.return_value = "postscript" + + with tempfile.TemporaryDirectory() as tmpdir: + file_path = os.path.join(tmpdir, "some_file.ps") + + turtle.TurtleScreen.save(screen, file_path) + with open(file_path) as f: + self.assertEqual(f.read(), "postscript") + + def test_no_animation_sets_tracer_0(self): + s = turtle.TurtleScreen(cv=unittest.mock.MagicMock()) + + with s.no_animation(): + self.assertEqual(s.tracer(), 0) + + def test_no_animation_resets_tracer_to_old_value(self): + s = turtle.TurtleScreen(cv=unittest.mock.MagicMock()) + + for tracer in [0, 1, 5]: + s.tracer(tracer) + with s.no_animation(): + pass + self.assertEqual(s.tracer(), tracer) + + def test_no_animation_calls_update_at_exit(self): + s = turtle.TurtleScreen(cv=unittest.mock.MagicMock()) + s.update = unittest.mock.MagicMock() + + with s.no_animation(): + s.update.assert_not_called() + s.update.assert_called_once() + + +class TestTurtle(unittest.TestCase): + def setUp(self): + with patch_screen(): + self.turtle = turtle.Turtle() + + # Reset the Screen singleton to avoid reference leaks + self.addCleanup(setattr, turtle.Turtle, '_screen', None) + + def test_begin_end_fill(self): + self.assertFalse(self.turtle.filling()) + self.turtle.begin_fill() + self.assertTrue(self.turtle.filling()) + self.turtle.end_fill() + self.assertFalse(self.turtle.filling()) + + def test_fill(self): + # The context manager behaves like begin_fill and end_fill. + self.assertFalse(self.turtle.filling()) + with self.turtle.fill(): + self.assertTrue(self.turtle.filling()) + self.assertFalse(self.turtle.filling()) + + def test_fill_resets_after_exception(self): + # The context manager cleans up correctly after exceptions. + try: + with self.turtle.fill(): + self.assertTrue(self.turtle.filling()) + raise ValueError + except ValueError: + self.assertFalse(self.turtle.filling()) + + def test_fill_context_when_filling(self): + # The context manager works even when the turtle is already filling. + self.turtle.begin_fill() + self.assertTrue(self.turtle.filling()) + with self.turtle.fill(): + self.assertTrue(self.turtle.filling()) + self.assertFalse(self.turtle.filling()) + + def test_begin_end_poly(self): + self.assertFalse(self.turtle._creatingPoly) + self.turtle.begin_poly() + self.assertTrue(self.turtle._creatingPoly) + self.turtle.end_poly() + self.assertFalse(self.turtle._creatingPoly) + + def test_poly(self): + # The context manager behaves like begin_poly and end_poly. + self.assertFalse(self.turtle._creatingPoly) + with self.turtle.poly(): + self.assertTrue(self.turtle._creatingPoly) + self.assertFalse(self.turtle._creatingPoly) + + def test_poly_resets_after_exception(self): + # The context manager cleans up correctly after exceptions. + try: + with self.turtle.poly(): + self.assertTrue(self.turtle._creatingPoly) + raise ValueError + except ValueError: + self.assertFalse(self.turtle._creatingPoly) + + def test_poly_context_when_creating_poly(self): + # The context manager works when the turtle is already creating poly. + self.turtle.begin_poly() + self.assertTrue(self.turtle._creatingPoly) + with self.turtle.poly(): + self.assertTrue(self.turtle._creatingPoly) + self.assertFalse(self.turtle._creatingPoly) + + class TestModuleLevel(unittest.TestCase): def test_all_signatures(self): import inspect diff --git a/Lib/test/test_type_aliases.py b/Lib/test/test_type_aliases.py index 9c325bc595f585..230bbe646baf28 100644 --- a/Lib/test/test_type_aliases.py +++ b/Lib/test/test_type_aliases.py @@ -4,7 +4,9 @@ from test.support import check_syntax_error, run_code from test.typinganndata import mod_generics_cache -from typing import Callable, TypeAliasType, TypeVar, get_args +from typing import ( + Callable, TypeAliasType, TypeVar, TypeVarTuple, ParamSpec, get_args, +) class TypeParamsInvalidTest(unittest.TestCase): @@ -211,6 +213,59 @@ def test_generic(self): self.assertEqual(TA.__value__, list[T]) self.assertEqual(TA.__type_params__, (T,)) self.assertEqual(TA.__module__, __name__) + self.assertIs(type(TA[int]), types.GenericAlias) + + def test_not_generic(self): + TA = TypeAliasType("TA", list[int], type_params=()) + self.assertEqual(TA.__name__, "TA") + self.assertEqual(TA.__value__, list[int]) + self.assertEqual(TA.__type_params__, ()) + self.assertEqual(TA.__module__, __name__) + with self.assertRaisesRegex( + TypeError, + "Only generic type aliases are subscriptable", + ): + TA[int] + + def test_type_params_order_with_defaults(self): + HasNoDefaultT = TypeVar("HasNoDefaultT") + WithDefaultT = TypeVar("WithDefaultT", default=int) + + HasNoDefaultP = ParamSpec("HasNoDefaultP") + WithDefaultP = ParamSpec("WithDefaultP", default=HasNoDefaultP) + + HasNoDefaultTT = TypeVarTuple("HasNoDefaultTT") + WithDefaultTT = TypeVarTuple("WithDefaultTT", default=HasNoDefaultTT) + + for type_params in [ + (HasNoDefaultT, WithDefaultT), + (HasNoDefaultP, WithDefaultP), + (HasNoDefaultTT, WithDefaultTT), + ]: + with self.subTest(type_params=type_params): + TypeAliasType("A", int, type_params=type_params) # ok + + msg = "follows default type parameter" + for type_params in [ + (WithDefaultT, HasNoDefaultT), + (WithDefaultP, HasNoDefaultP), + (WithDefaultTT, HasNoDefaultTT), + (WithDefaultT, HasNoDefaultP), # different types + ]: + with self.subTest(type_params=type_params): + with self.assertRaisesRegex(TypeError, msg): + TypeAliasType("A", int, type_params=type_params) + + def test_expects_type_like(self): + T = TypeVar("T") + + msg = "Expected a type param" + with self.assertRaisesRegex(TypeError, msg): + TypeAliasType("A", int, type_params=(1,)) + with self.assertRaisesRegex(TypeError, msg): + TypeAliasType("A", int, type_params=(1, 2)) + with self.assertRaisesRegex(TypeError, msg): + TypeAliasType("A", int, type_params=(T, 2)) def test_keywords(self): TA = TypeAliasType(name="TA", value=int) @@ -328,3 +383,22 @@ def test_pickling_local(self): with self.subTest(thing=thing, proto=proto): with self.assertRaises(pickle.PickleError): pickle.dumps(thing, protocol=proto) + + +class TypeParamsExoticGlobalsTest(unittest.TestCase): + def test_exec_with_unusual_globals(self): + class customdict(dict): + def __missing__(self, key): + return key + + code = compile("type Alias = undefined", "test", "exec") + ns = customdict() + exec(code, ns) + Alias = ns["Alias"] + self.assertEqual(Alias.__value__, "undefined") + + code = compile("class A: type Alias = undefined", "test", "exec") + ns = customdict() + exec(code, ns) + Alias = ns["A"].Alias + self.assertEqual(Alias.__value__, "undefined") diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 3dbb35afcb620f..0afcd76af153e7 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -1,6 +1,10 @@ +import annotationlib +import inspect import textwrap +import types import unittest -from test.support import run_code +from test.support import run_code, check_syntax_error + class TypeAnnotationTests(unittest.TestCase): @@ -48,6 +52,7 @@ def test_annotations_are_created_correctly(self): class C: a:int=3 b:str=4 + self.assertEqual(C.__annotations__, {"a": int, "b": str}) self.assertTrue("__annotations__" in C.__dict__) del C.__annotations__ self.assertFalse("__annotations__" in C.__dict__) @@ -105,6 +110,13 @@ class D(metaclass=C): self.assertEqual(D.__annotations__, {}) +def build_module(code: str, name: str = "top") -> types.ModuleType: + ns = run_code(code) + mod = types.ModuleType(name) + mod.__dict__.update(ns) + return mod + + class TestSetupAnnotations(unittest.TestCase): def check(self, code: str): code = textwrap.dedent(code) @@ -112,11 +124,10 @@ def check(self, code: str): with self.subTest(scope=scope): if scope == "class": code = f"class C:\n{textwrap.indent(code, ' ')}" - ns = run_code(code) - if scope == "class": + ns = run_code(code) annotations = ns["C"].__annotations__ else: - annotations = ns["__annotations__"] + annotations = build_module(code).__annotations__ self.assertEqual(annotations, {"x": int}) def test_top_level(self): @@ -212,3 +223,237 @@ def test_match(self): case 0: x: int = 1 """) + + +class AnnotateTests(unittest.TestCase): + """See PEP 649.""" + def test_manual_annotate(self): + def f(): + pass + mod = types.ModuleType("mod") + class X: + pass + + for obj in (f, mod, X): + with self.subTest(obj=obj): + self.check_annotations(obj) + + def check_annotations(self, f): + self.assertEqual(f.__annotations__, {}) + self.assertIs(f.__annotate__, None) + + with self.assertRaisesRegex(TypeError, "__annotate__ must be callable or None"): + f.__annotate__ = 42 + f.__annotate__ = lambda: 42 + with self.assertRaisesRegex(TypeError, r"takes 0 positional arguments but 1 was given"): + print(f.__annotations__) + + f.__annotate__ = lambda x: 42 + with self.assertRaisesRegex(TypeError, r"__annotate__ returned non-dict of type 'int'"): + print(f.__annotations__) + + f.__annotate__ = lambda x: {"x": x} + self.assertEqual(f.__annotations__, {"x": 1}) + + # Setting annotate to None does not invalidate the cached __annotations__ + f.__annotate__ = None + self.assertEqual(f.__annotations__, {"x": 1}) + + # But setting it to a new callable does + f.__annotate__ = lambda x: {"y": x} + self.assertEqual(f.__annotations__, {"y": 1}) + + # Setting f.__annotations__ also clears __annotate__ + f.__annotations__ = {"z": 43} + self.assertIs(f.__annotate__, None) + + +class DeferredEvaluationTests(unittest.TestCase): + def test_function(self): + def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined: + pass + + with self.assertRaises(NameError): + func.__annotations__ + + undefined = 1 + self.assertEqual(func.__annotations__, { + "x": 1, + "y": 1, + "args": 1, + "z": 1, + "kwargs": 1, + "return": 1, + }) + + def test_async_function(self): + async def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined: + pass + + with self.assertRaises(NameError): + func.__annotations__ + + undefined = 1 + self.assertEqual(func.__annotations__, { + "x": 1, + "y": 1, + "args": 1, + "z": 1, + "kwargs": 1, + "return": 1, + }) + + def test_class(self): + class X: + a: undefined + + with self.assertRaises(NameError): + X.__annotations__ + + undefined = 1 + self.assertEqual(X.__annotations__, {"a": 1}) + + def test_module(self): + ns = run_code("x: undefined = 1") + anno = ns["__annotate__"] + with self.assertRaises(NotImplementedError): + anno(3) + + with self.assertRaises(NameError): + anno(1) + + ns["undefined"] = 1 + self.assertEqual(anno(1), {"x": 1}) + + def test_class_scoping(self): + class Outer: + def meth(self, x: Nested): ... + x: Nested + class Nested: ... + + self.assertEqual(Outer.meth.__annotations__, {"x": Outer.Nested}) + self.assertEqual(Outer.__annotations__, {"x": Outer.Nested}) + + def test_no_exotic_expressions(self): + check_syntax_error(self, "def func(x: (yield)): ...", "yield expression cannot be used within an annotation") + check_syntax_error(self, "def func(x: (yield from x)): ...", "yield expression cannot be used within an annotation") + check_syntax_error(self, "def func(x: (y := 3)): ...", "named expression cannot be used within an annotation") + check_syntax_error(self, "def func(x: (await 42)): ...", "await expression cannot be used within an annotation") + + def test_no_exotic_expressions_in_unevaluated_annotations(self): + preludes = [ + "", + "class X: ", + "def f(): ", + "async def f(): ", + ] + for prelude in preludes: + with self.subTest(prelude=prelude): + check_syntax_error(self, prelude + "(x): (yield)", "yield expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (yield from x)", "yield expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (y := 3)", "named expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (await 42)", "await expression cannot be used within an annotation") + + def test_ignore_non_simple_annotations(self): + ns = run_code("class X: (y): int") + self.assertEqual(ns["X"].__annotations__, {}) + ns = run_code("class X: int.b: int") + self.assertEqual(ns["X"].__annotations__, {}) + ns = run_code("class X: int[str]: int") + self.assertEqual(ns["X"].__annotations__, {}) + + def test_generated_annotate(self): + def func(x: int): + pass + class X: + x: int + mod = build_module("x: int") + for obj in (func, X, mod): + with self.subTest(obj=obj): + annotate = obj.__annotate__ + self.assertIsInstance(annotate, types.FunctionType) + self.assertEqual(annotate.__name__, "__annotate__") + with self.assertRaises(NotImplementedError): + annotate(annotationlib.Format.FORWARDREF) + with self.assertRaises(NotImplementedError): + annotate(annotationlib.Format.STRING) + with self.assertRaises(TypeError): + annotate(None) + self.assertEqual(annotate(annotationlib.Format.VALUE), {"x": int}) + + sig = inspect.signature(annotate) + self.assertEqual(sig, inspect.Signature([ + inspect.Parameter("format", inspect.Parameter.POSITIONAL_ONLY) + ])) + + def test_comprehension_in_annotation(self): + # This crashed in an earlier version of the code + ns = run_code("x: [y for y in range(10)]") + self.assertEqual(ns["__annotate__"](1), {"x": list(range(10))}) + + def test_future_annotations(self): + code = """ + from __future__ import annotations + + def f(x: int) -> int: pass + """ + ns = run_code(code) + f = ns["f"] + self.assertIsInstance(f.__annotate__, types.FunctionType) + annos = {"x": "int", "return": "int"} + self.assertEqual(f.__annotate__(annotationlib.Format.VALUE), annos) + self.assertEqual(f.__annotations__, annos) + + def test_name_clash_with_format(self): + # this test would fail if __annotate__'s parameter was called "format" + # during symbol table construction + code = """ + class format: pass + + def f(x: format): pass + """ + ns = run_code(code) + f = ns["f"] + self.assertEqual(f.__annotations__, {"x": ns["format"]}) + + code = """ + class Outer: + class format: pass + + def meth(self, x: format): ... + """ + ns = run_code(code) + self.assertEqual(ns["Outer"].meth.__annotations__, {"x": ns["Outer"].format}) + + code = """ + def f(format): + def inner(x: format): pass + return inner + res = f("closure var") + """ + ns = run_code(code) + self.assertEqual(ns["res"].__annotations__, {"x": "closure var"}) + + code = """ + def f(x: format): + pass + """ + ns = run_code(code) + # picks up the format() builtin + self.assertEqual(ns["f"].__annotations__, {"x": format}) + + code = """ + def outer(): + def f(x: format): + pass + if False: + class format: pass + return f + f = outer() + """ + ns = run_code(code) + with self.assertRaisesRegex( + NameError, + "cannot access free variable 'format' where it is not associated with a value in enclosing scope", + ): + ns["f"].__annotations__ diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index e90e315c808361..ee64f89358ed55 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -2,7 +2,7 @@ import unittest import dis from test import support -from test.support import import_helper, requires_specialization +from test.support import import_helper, requires_specialization, requires_specialization_ft try: from sys import _clear_type_cache except ImportError: @@ -10,8 +10,9 @@ # Skip this test if the _testcapi module isn't available. _testcapi = import_helper.import_module("_testcapi") +_testinternalcapi = import_helper.import_module("_testinternalcapi") type_get_version = _testcapi.type_get_version -type_assign_specific_version_unsafe = _testcapi.type_assign_specific_version_unsafe +type_assign_specific_version_unsafe = _testinternalcapi.type_assign_specific_version_unsafe type_assign_version = _testcapi.type_assign_version type_modified = _testcapi.type_modified @@ -92,9 +93,23 @@ class C: new_version = type_get_version(C) self.assertEqual(new_version, 0) + def test_119462(self): + + class Holder: + value = None + + @classmethod + def set_value(cls): + cls.value = object() + + class HolderSub(Holder): + pass + + for _ in range(1050): + Holder.set_value() + HolderSub.value @support.cpython_only -@requires_specialization class TypeCacheWithSpecializationTests(unittest.TestCase): def tearDown(self): _clear_type_cache() @@ -105,8 +120,10 @@ def _assign_valid_version_or_skip(self, type_): if type_get_version(type_) == 0: self.skipTest("Could not assign valid type version") - def _assign_and_check_version_0(self, user_type): + def _no_more_versions(self, user_type): type_modified(user_type) + for _ in range(1001): + type_assign_specific_version_unsafe(user_type, 1000_000_000) type_assign_specific_version_unsafe(user_type, 0) self.assertEqual(type_get_version(user_type), 0) @@ -114,7 +131,7 @@ def _all_opnames(self, func): return set(instr.opname for instr in dis.Bytecode(func, adaptive=True)) def _check_specialization(self, func, arg, opname, *, should_specialize): - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): func(arg) if should_specialize: @@ -122,6 +139,7 @@ def _check_specialization(self, func, arg, opname, *, should_specialize): else: self.assertIn(opname, self._all_opnames(func)) + @requires_specialization def test_class_load_attr_specialization_user_type(self): class A: def foo(self): @@ -135,16 +153,17 @@ def load_foo_1(type_): self._check_specialization(load_foo_1, A, "LOAD_ATTR", should_specialize=True) del load_foo_1 - self._assign_and_check_version_0(A) + self._no_more_versions(A) def load_foo_2(type_): return type_.foo self._check_specialization(load_foo_2, A, "LOAD_ATTR", should_specialize=False) + @requires_specialization def test_class_load_attr_specialization_static_type(self): - self._assign_valid_version_or_skip(str) - self._assign_valid_version_or_skip(bytes) + self.assertNotEqual(type_get_version(str), 0) + self.assertNotEqual(type_get_version(bytes), 0) def get_capitalize_1(type_): return type_.capitalize @@ -152,26 +171,8 @@ def get_capitalize_1(type_): self._check_specialization(get_capitalize_1, str, "LOAD_ATTR", should_specialize=True) self.assertEqual(get_capitalize_1(str)('hello'), 'Hello') self.assertEqual(get_capitalize_1(bytes)(b'hello'), b'Hello') - del get_capitalize_1 - - # Permanently overflow the static type version counter, and force str and bytes - # to have tp_version_tag == 0 - for _ in range(2**16): - type_modified(str) - type_assign_version(str) - type_modified(bytes) - type_assign_version(bytes) - - self.assertEqual(type_get_version(str), 0) - self.assertEqual(type_get_version(bytes), 0) - - def get_capitalize_2(type_): - return type_.capitalize - - self._check_specialization(get_capitalize_2, str, "LOAD_ATTR", should_specialize=False) - self.assertEqual(get_capitalize_2(str)('hello'), 'Hello') - self.assertEqual(get_capitalize_2(bytes)(b'hello'), b'Hello') + @requires_specialization def test_property_load_attr_specialization_user_type(self): class G: @property @@ -186,13 +187,14 @@ def load_x_1(instance): self._check_specialization(load_x_1, G(), "LOAD_ATTR", should_specialize=True) del load_x_1 - self._assign_and_check_version_0(G) + self._no_more_versions(G) def load_x_2(instance): instance.x self._check_specialization(load_x_2, G(), "LOAD_ATTR", should_specialize=False) + @requires_specialization def test_store_attr_specialization_user_type(self): class B: __slots__ = ("bar",) @@ -205,13 +207,14 @@ def store_bar_1(type_): self._check_specialization(store_bar_1, B(), "STORE_ATTR", should_specialize=True) del store_bar_1 - self._assign_and_check_version_0(B) + self._no_more_versions(B) def store_bar_2(type_): type_.bar = 10 self._check_specialization(store_bar_2, B(), "STORE_ATTR", should_specialize=False) + @requires_specialization_ft def test_class_call_specialization_user_type(self): class F: def __init__(self): @@ -225,13 +228,14 @@ def call_class_1(type_): self._check_specialization(call_class_1, F, "CALL", should_specialize=True) del call_class_1 - self._assign_and_check_version_0(F) + self._no_more_versions(F) def call_class_2(type_): type_() self._check_specialization(call_class_2, F, "CALL", should_specialize=False) + @requires_specialization def test_to_bool_specialization_user_type(self): class H: pass @@ -244,7 +248,7 @@ def to_bool_1(instance): self._check_specialization(to_bool_1, H(), "TO_BOOL", should_specialize=True) del to_bool_1 - self._assign_and_check_version_0(H) + self._no_more_versions(H) def to_bool_2(instance): not instance diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 25ee188731f31f..0f393def827271 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -1,12 +1,12 @@ -import asyncio +import annotationlib import textwrap import types import unittest import pickle import weakref -from test.support import requires_working_socket, check_syntax_error, run_code +from test.support import check_syntax_error, run_code, run_no_yield_async_fn -from typing import Generic, Sequence, TypeVar, TypeVarTuple, ParamSpec, get_args +from typing import Generic, NoDefault, Sequence, TypeAliasType, TypeVar, TypeVarTuple, ParamSpec, get_args class TypeParamsInvalidTest(unittest.TestCase): @@ -412,6 +412,14 @@ def test_comprehension_02(self): func, = T.__bound__ self.assertEqual(func(), 1) + def test_comprehension_03(self): + def F[T: [lambda: T for T in (T, [1])[1]]](): return [lambda: T for T in T.__name__] + func, = F() + self.assertEqual(func(), "T") + T, = F.__type_params__ + func, = T.__bound__ + self.assertEqual(func(), 1) + def test_gen_exp_in_nested_class(self): code = """ from test.test_type_params import make_base @@ -436,9 +444,11 @@ class C[T]: class Inner[U](make_base(T for _ in (1,)), make_base(T)): pass """ - with self.assertRaisesRegex(SyntaxError, - "Cannot use comprehension in annotation scope within class scope"): - run_code(code) + ns = run_code(code) + inner = ns["C"].Inner + base1, base2, _ = inner.__bases__ + self.assertEqual(list(base1.__arg__), [ns["C"].__type_params__[0]]) + self.assertEqual(base2.__arg__, "class") def test_listcomp_in_nested_class(self): code = """ @@ -464,9 +474,11 @@ class C[T]: class Inner[U](make_base([T for _ in (1,)]), make_base(T)): pass """ - with self.assertRaisesRegex(SyntaxError, - "Cannot use comprehension in annotation scope within class scope"): - run_code(code) + ns = run_code(code) + inner = ns["C"].Inner + base1, base2, _ = inner.__bases__ + self.assertEqual(base1.__arg__, [ns["C"].__type_params__[0]]) + self.assertEqual(base2.__arg__, "class") def test_gen_exp_in_generic_method(self): code = """ @@ -475,29 +487,86 @@ class C[T]: def meth[U](x: (T for _ in (1,)), y: T): pass """ - with self.assertRaisesRegex(SyntaxError, - "Cannot use comprehension in annotation scope within class scope"): - run_code(code) + ns = run_code(code) + meth = ns["C"].meth + self.assertEqual(list(meth.__annotations__["x"]), [ns["C"].__type_params__[0]]) + self.assertEqual(meth.__annotations__["y"], "class") def test_nested_scope_in_generic_alias(self): code = """ - class C[T]: + T = "global" + class C: T = "class" {} """ - error_cases = [ - "type Alias1[T] = lambda: T", - "type Alias2 = lambda: T", - "type Alias3[T] = (T for _ in (1,))", - "type Alias4 = (T for _ in (1,))", - "type Alias5[T] = [T for _ in (1,)]", - "type Alias6 = [T for _ in (1,)]", + cases = [ + "type Alias[T] = (T for _ in (1,))", + "type Alias = (T for _ in (1,))", + "type Alias[T] = [T for _ in (1,)]", + "type Alias = [T for _ in (1,)]", ] - for case in error_cases: + for case in cases: with self.subTest(case=case): - with self.assertRaisesRegex(SyntaxError, - r"Cannot use [a-z]+ in annotation scope within class scope"): - run_code(code.format(case)) + ns = run_code(code.format(case)) + alias = ns["C"].Alias + value = list(alias.__value__)[0] + if alias.__type_params__: + self.assertIs(value, alias.__type_params__[0]) + else: + self.assertEqual(value, "global") + + def test_lambda_in_alias_in_class(self): + code = """ + T = "global" + class C: + T = "class" + type Alias = lambda: T + """ + C = run_code(code)["C"] + self.assertEqual(C.Alias.__value__(), "global") + + def test_lambda_in_alias_in_generic_class(self): + code = """ + class C[T]: + T = "class" + type Alias = lambda: T + """ + C = run_code(code)["C"] + self.assertIs(C.Alias.__value__(), C.__type_params__[0]) + + def test_lambda_in_generic_alias_in_class(self): + # A lambda nested in the alias cannot see the class scope, but can see + # a surrounding annotation scope. + code = """ + T = U = "global" + class C: + T = "class" + U = "class" + type Alias[T] = lambda: (T, U) + """ + C = run_code(code)["C"] + T, U = C.Alias.__value__() + self.assertIs(T, C.Alias.__type_params__[0]) + self.assertEqual(U, "global") + + def test_lambda_in_generic_alias_in_generic_class(self): + # A lambda nested in the alias cannot see the class scope, but can see + # a surrounding annotation scope. + code = """ + class C[T, U]: + T = "class" + U = "class" + type Alias[T] = lambda: (T, U) + """ + C = run_code(code)["C"] + T, U = C.Alias.__value__() + self.assertIs(T, C.Alias.__type_params__[0]) + self.assertIs(U, C.__type_params__[1]) + + def test_type_special_case(self): + # https://github.com/python/cpython/issues/119011 + self.assertEqual(type.__type_params__, ()) + self.assertEqual(object.__type_params__, ()) def make_base(arg): @@ -535,10 +604,12 @@ class Foo[T: Foo, U: (Foo, Foo)]: self.assertEqual(type_params[0].__name__, "T") self.assertIs(type_params[0].__bound__, Foo) self.assertEqual(type_params[0].__constraints__, ()) + self.assertIs(type_params[0].__default__, NoDefault) self.assertEqual(type_params[1].__name__, "U") self.assertIs(type_params[1].__bound__, None) self.assertEqual(type_params[1].__constraints__, (Foo, Foo)) + self.assertIs(type_params[1].__default__, NoDefault) def test_evaluation_error(self): class Foo[T: Undefined, U: (Undefined,)]: @@ -549,6 +620,8 @@ class Foo[T: Undefined, U: (Undefined,)]: type_params[0].__bound__ self.assertEqual(type_params[0].__constraints__, ()) self.assertIs(type_params[1].__bound__, None) + self.assertIs(type_params[0].__default__, NoDefault) + self.assertIs(type_params[1].__default__, NoDefault) with self.assertRaises(NameError): type_params[1].__constraints__ @@ -708,6 +781,31 @@ class D[U](T): self.assertIn(int, C.D.__bases__) self.assertIs(C.D.x, str) + +class DynamicClassTest(unittest.TestCase): + def _set_type_params(self, ns, params): + ns['__type_params__'] = params + + def test_types_new_class_with_callback(self): + T = TypeVar('T', infer_variance=True) + Klass = types.new_class('Klass', (Generic[T],), {}, + lambda ns: self._set_type_params(ns, (T,))) + + self.assertEqual(Klass.__bases__, (Generic,)) + self.assertEqual(Klass.__orig_bases__, (Generic[T],)) + self.assertEqual(Klass.__type_params__, (T,)) + self.assertEqual(Klass.__parameters__, (T,)) + + def test_types_new_class_no_callback(self): + T = TypeVar('T', infer_variance=True) + Klass = types.new_class('Klass', (Generic[T],), {}) + + self.assertEqual(Klass.__bases__, (Generic,)) + self.assertEqual(Klass.__orig_bases__, (Generic[T],)) + self.assertEqual(Klass.__type_params__, ()) # must be explicitly set + self.assertEqual(Klass.__parameters__, (T,)) + + class TypeParamsManglingTest(unittest.TestCase): def test_mangling(self): class Foo[__T]: @@ -730,6 +828,100 @@ def meth[__U](self, arg: __T, arg2: __U): self.assertEqual(Foo.Alias.__value__, (T, V)) + def test_no_leaky_mangling_in_module(self): + ns = run_code(""" + __before = "before" + class X[T]: pass + __after = "after" + """) + self.assertEqual(ns["__before"], "before") + self.assertEqual(ns["__after"], "after") + + def test_no_leaky_mangling_in_function(self): + ns = run_code(""" + def f(): + class X[T]: pass + _X_foo = 2 + __foo = 1 + assert locals()['__foo'] == 1 + return __foo + """) + self.assertEqual(ns["f"](), 1) + + def test_no_leaky_mangling_in_class(self): + ns = run_code(""" + class Outer: + __before = "before" + class Inner[T]: + __x = "inner" + __after = "after" + """) + Outer = ns["Outer"] + self.assertEqual(Outer._Outer__before, "before") + self.assertEqual(Outer.Inner._Inner__x, "inner") + self.assertEqual(Outer._Outer__after, "after") + + def test_no_mangling_in_bases(self): + ns = run_code(""" + class __Base: + def __init_subclass__(self, **kwargs): + self.kwargs = kwargs + + class Derived[T](__Base, __kwarg=1): + pass + """) + Derived = ns["Derived"] + self.assertEqual(Derived.__bases__, (ns["__Base"], Generic)) + self.assertEqual(Derived.kwargs, {"__kwarg": 1}) + + def test_no_mangling_in_nested_scopes(self): + ns = run_code(""" + from test.test_type_params import make_base + + class __X: + pass + + class Y[T: __X]( + make_base(lambda: __X), + # doubly nested scope + make_base(lambda: (lambda: __X)), + # list comprehension + make_base([__X for _ in (1,)]), + # genexp + make_base(__X for _ in (1,)), + ): + pass + """) + Y = ns["Y"] + T, = Y.__type_params__ + self.assertIs(T.__bound__, ns["__X"]) + base0 = Y.__bases__[0] + self.assertIs(base0.__arg__(), ns["__X"]) + base1 = Y.__bases__[1] + self.assertIs(base1.__arg__()(), ns["__X"]) + base2 = Y.__bases__[2] + self.assertEqual(base2.__arg__, [ns["__X"]]) + base3 = Y.__bases__[3] + self.assertEqual(list(base3.__arg__), [ns["__X"]]) + + def test_type_params_are_mangled(self): + ns = run_code(""" + from test.test_type_params import make_base + + class Foo[__T, __U: __T](make_base(__T), make_base(lambda: __T)): + param = __T + """) + Foo = ns["Foo"] + T, U = Foo.__type_params__ + self.assertEqual(T.__name__, "__T") + self.assertEqual(U.__name__, "__U") + self.assertIs(U.__bound__, T) + self.assertIs(Foo.param, T) + + base1, base2, *_ = Foo.__bases__ + self.assertIs(base1.__arg__, T) + self.assertIs(base2.__arg__(), T) + class TypeParamsComplexCallsTest(unittest.TestCase): def test_defaults(self): @@ -758,6 +950,7 @@ class C[T](Base, a=1, b=2, **kwargs): T, = C.__type_params__ self.assertEqual(T.__name__, "T") self.assertEqual(C.kwargs, {"a": 1, "b": 2, "c": 3}) + self.assertEqual(C.__bases__, (Base, Generic)) bases = (Base,) class C2[T](*bases, **kwargs): @@ -766,6 +959,22 @@ class C2[T](*bases, **kwargs): T, = C2.__type_params__ self.assertEqual(T.__name__, "T") self.assertEqual(C2.kwargs, {"c": 3}) + self.assertEqual(C2.__bases__, (Base, Generic)) + + def test_starargs_base(self): + class C1[T](*()): pass + + T, = C1.__type_params__ + self.assertEqual(T.__name__, "T") + self.assertEqual(C1.__bases__, (Generic,)) + + class Base: pass + bases = [Base] + class C2[T](*bases): pass + + T, = C2.__type_params__ + self.assertEqual(T.__name__, "T") + self.assertEqual(C2.__bases__, (Base, Generic)) class TypeParamsTraditionalTypeVarsTest(unittest.TestCase): @@ -841,7 +1050,6 @@ def generator2[B](): self.assertIsInstance(c, TypeVar) self.assertEqual(c.__name__, "C") - @requires_working_socket() def test_typevar_coroutine(self): def get_coroutine[A](): async def coroutine[B](): @@ -850,8 +1058,7 @@ async def coroutine[B](): co = get_coroutine() - self.addCleanup(asyncio.set_event_loop_policy, None) - a, b = asyncio.run(co()) + a, b = run_no_yield_async_fn(co) self.assertIsInstance(a, TypeVar) self.assertEqual(a.__name__, "A") @@ -1102,3 +1309,154 @@ class Inner[U](T): """ with self.assertRaises(RuntimeError): run_code(code) + + +class DefaultsTest(unittest.TestCase): + def test_defaults_on_func(self): + ns = run_code(""" + def func[T=int, **U=float, *V=None](): + pass + """) + + T, U, V = ns["func"].__type_params__ + self.assertIs(T.__default__, int) + self.assertIs(U.__default__, float) + self.assertIs(V.__default__, None) + + def test_defaults_on_class(self): + ns = run_code(""" + class C[T=int, **U=float, *V=None]: + pass + """) + + T, U, V = ns["C"].__type_params__ + self.assertIs(T.__default__, int) + self.assertIs(U.__default__, float) + self.assertIs(V.__default__, None) + + def test_defaults_on_type_alias(self): + ns = run_code(""" + type Alias[T = int, **U = float, *V = None] = int + """) + + T, U, V = ns["Alias"].__type_params__ + self.assertIs(T.__default__, int) + self.assertIs(U.__default__, float) + self.assertIs(V.__default__, None) + + def test_starred_invalid(self): + check_syntax_error(self, "type Alias[T = *int] = int") + check_syntax_error(self, "type Alias[**P = *int] = int") + + def test_starred_typevartuple(self): + ns = run_code(""" + default = tuple[int, str] + type Alias[*Ts = *default] = Ts + """) + + Ts, = ns["Alias"].__type_params__ + self.assertEqual(Ts.__default__, next(iter(ns["default"]))) + + def test_nondefault_after_default(self): + check_syntax_error(self, "def func[T=int, U](): pass", "non-default type parameter 'U' follows default type parameter") + check_syntax_error(self, "class C[T=int, U]: pass", "non-default type parameter 'U' follows default type parameter") + check_syntax_error(self, "type A[T=int, U] = int", "non-default type parameter 'U' follows default type parameter") + + def test_lazy_evaluation(self): + ns = run_code(""" + type Alias[T = Undefined, *U = Undefined, **V = Undefined] = int + """) + + T, U, V = ns["Alias"].__type_params__ + + with self.assertRaises(NameError): + T.__default__ + with self.assertRaises(NameError): + U.__default__ + with self.assertRaises(NameError): + V.__default__ + + ns["Undefined"] = "defined" + self.assertEqual(T.__default__, "defined") + self.assertEqual(U.__default__, "defined") + self.assertEqual(V.__default__, "defined") + + # Now it is cached + ns["Undefined"] = "redefined" + self.assertEqual(T.__default__, "defined") + self.assertEqual(U.__default__, "defined") + self.assertEqual(V.__default__, "defined") + + def test_symtable_key_regression_default(self): + # Test against the bugs that would happen if we used .default_ + # as the key in the symtable. + ns = run_code(""" + type X[T = [T for T in [T]]] = T + """) + + T, = ns["X"].__type_params__ + self.assertEqual(T.__default__, [T]) + + def test_symtable_key_regression_name(self): + # Test against the bugs that would happen if we used .name + # as the key in the symtable. + ns = run_code(""" + type X1[T = A] = T + type X2[T = B] = T + A = "A" + B = "B" + """) + + self.assertEqual(ns["X1"].__type_params__[0].__default__, "A") + self.assertEqual(ns["X2"].__type_params__[0].__default__, "B") + + +class TestEvaluateFunctions(unittest.TestCase): + def test_general(self): + type Alias = int + Alias2 = TypeAliasType("Alias2", int) + def f[T: int = int, **P = int, *Ts = int](): pass + T, P, Ts = f.__type_params__ + T2 = TypeVar("T2", bound=int, default=int) + P2 = ParamSpec("P2", default=int) + Ts2 = TypeVarTuple("Ts2", default=int) + cases = [ + Alias.evaluate_value, + Alias2.evaluate_value, + T.evaluate_bound, + T.evaluate_default, + P.evaluate_default, + Ts.evaluate_default, + T2.evaluate_bound, + T2.evaluate_default, + P2.evaluate_default, + Ts2.evaluate_default, + ] + for case in cases: + with self.subTest(case=case): + self.assertIs(case(1), int) + self.assertIs(annotationlib.call_evaluate_function(case, annotationlib.Format.VALUE), int) + self.assertIs(annotationlib.call_evaluate_function(case, annotationlib.Format.FORWARDREF), int) + self.assertEqual(annotationlib.call_evaluate_function(case, annotationlib.Format.STRING), 'int') + + def test_constraints(self): + def f[T: (int, str)](): pass + T, = f.__type_params__ + T2 = TypeVar("T2", int, str) + for case in [T, T2]: + with self.subTest(case=case): + self.assertEqual(case.evaluate_constraints(1), (int, str)) + self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.VALUE), (int, str)) + self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.FORWARDREF), (int, str)) + self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.STRING), '(int, str)') + + def test_const_evaluator(self): + T = TypeVar("T", bound=int) + self.assertEqual(repr(T.evaluate_bound), ">") + + ConstEvaluator = type(T.evaluate_bound) + + with self.assertRaisesRegex(TypeError, r"cannot create '_typing\._ConstEvaluator' instances"): + ConstEvaluator() # This used to segfault. + with self.assertRaisesRegex(TypeError, r"cannot set 'attribute' attribute of immutable type '_typing\._ConstEvaluator'"): + ConstEvaluator.attribute = 1 diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 16985122bc0219..d1161719d98040 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -1,8 +1,12 @@ # Python test set -- part 6, built-in types -from test.support import run_with_locale, cpython_only, MISSING_C_DOCSTRINGS +from test.support import ( + run_with_locale, is_apple_mobile, cpython_only, no_rerun, + iter_builtin_types, iter_slot_wrappers, + MISSING_C_DOCSTRINGS, +) import collections.abc -from collections import namedtuple +from collections import namedtuple, UserDict import copy import _datetime import gc @@ -10,6 +14,7 @@ import pickle import locale import sys +import textwrap import types import unittest.mock import weakref @@ -393,7 +398,7 @@ def test(i, format_spec, result): test(123456, "1=20", '11111111111111123456') test(123456, "*=20", '**************123456') - @run_with_locale('LC_NUMERIC', 'en_US.UTF8') + @run_with_locale('LC_NUMERIC', 'en_US.UTF8', '') def test_float__format__locale(self): # test locale support for __format__ code 'n' @@ -402,7 +407,7 @@ def test_float__format__locale(self): self.assertEqual(locale.format_string('%g', x, grouping=True), format(x, 'n')) self.assertEqual(locale.format_string('%.10g', x, grouping=True), format(x, '.10n')) - @run_with_locale('LC_NUMERIC', 'en_US.UTF8') + @run_with_locale('LC_NUMERIC', 'en_US.UTF8', '') def test_int__format__locale(self): # test locale support for __format__ code 'n' for integers @@ -1755,21 +1760,50 @@ class Model(metaclass=ModelBase): class SimpleNamespaceTests(unittest.TestCase): def test_constructor(self): - ns1 = types.SimpleNamespace() - ns2 = types.SimpleNamespace(x=1, y=2) - ns3 = types.SimpleNamespace(**dict(x=1, y=2)) + def check(ns, expected): + self.assertEqual(len(ns.__dict__), len(expected)) + self.assertEqual(vars(ns), expected) + # check order + self.assertEqual(list(vars(ns).items()), list(expected.items())) + for name in expected: + self.assertEqual(getattr(ns, name), expected[name]) + + check(types.SimpleNamespace(), {}) + check(types.SimpleNamespace(x=1, y=2), {'x': 1, 'y': 2}) + check(types.SimpleNamespace(**dict(x=1, y=2)), {'x': 1, 'y': 2}) + check(types.SimpleNamespace({'x': 1, 'y': 2}, x=4, z=3), + {'x': 4, 'y': 2, 'z': 3}) + check(types.SimpleNamespace([['x', 1], ['y', 2]], x=4, z=3), + {'x': 4, 'y': 2, 'z': 3}) + check(types.SimpleNamespace(UserDict({'x': 1, 'y': 2}), x=4, z=3), + {'x': 4, 'y': 2, 'z': 3}) + check(types.SimpleNamespace({'x': 1, 'y': 2}), {'x': 1, 'y': 2}) + check(types.SimpleNamespace([['x', 1], ['y', 2]]), {'x': 1, 'y': 2}) + check(types.SimpleNamespace([], x=4, z=3), {'x': 4, 'z': 3}) + check(types.SimpleNamespace({}, x=4, z=3), {'x': 4, 'z': 3}) + check(types.SimpleNamespace([]), {}) + check(types.SimpleNamespace({}), {}) with self.assertRaises(TypeError): - types.SimpleNamespace(1, 2, 3) + types.SimpleNamespace([], []) # too many positional arguments with self.assertRaises(TypeError): - types.SimpleNamespace(**{1: 2}) - - self.assertEqual(len(ns1.__dict__), 0) - self.assertEqual(vars(ns1), {}) - self.assertEqual(len(ns2.__dict__), 2) - self.assertEqual(vars(ns2), {'y': 2, 'x': 1}) - self.assertEqual(len(ns3.__dict__), 2) - self.assertEqual(vars(ns3), {'y': 2, 'x': 1}) + types.SimpleNamespace(1) # not a mapping or iterable + with self.assertRaises(TypeError): + types.SimpleNamespace([1]) # non-iterable + with self.assertRaises(ValueError): + types.SimpleNamespace([['x']]) # not a pair + with self.assertRaises(ValueError): + types.SimpleNamespace([['x', 'y', 'z']]) + with self.assertRaises(TypeError): + types.SimpleNamespace(**{1: 2}) # non-string key + with self.assertRaises(TypeError): + types.SimpleNamespace({1: 2}) + with self.assertRaises(TypeError): + types.SimpleNamespace([[1, 2]]) + with self.assertRaises(TypeError): + types.SimpleNamespace(UserDict({1: 2})) + with self.assertRaises(TypeError): + types.SimpleNamespace([[[], 2]]) # non-hashable key def test_unbound(self): ns1 = vars(types.SimpleNamespace()) @@ -2316,5 +2350,90 @@ def ex(a, /, b, *, c): ) +class SubinterpreterTests(unittest.TestCase): + + NUMERIC_METHODS = { + '__abs__', + '__add__', + '__bool__', + '__divmod__', + '__float__', + '__floordiv__', + '__index__', + '__int__', + '__lshift__', + '__mod__', + '__mul__', + '__neg__', + '__pos__', + '__pow__', + '__radd__', + '__rdivmod__', + '__rfloordiv__', + '__rlshift__', + '__rmod__', + '__rmul__', + '__rpow__', + '__rrshift__', + '__rshift__', + '__rsub__', + '__rtruediv__', + '__sub__', + '__truediv__', + } + + @classmethod + def setUpClass(cls): + global interpreters + try: + from test.support import interpreters + except ModuleNotFoundError: + raise unittest.SkipTest('subinterpreters required') + import test.support.interpreters.channels + + @cpython_only + @no_rerun('channels (and queues) might have a refleak; see gh-122199') + def test_static_types_inherited_slots(self): + rch, sch = interpreters.channels.create() + + script = textwrap.dedent(""" + import test.support + results = [] + for cls in test.support.iter_builtin_types(): + for attr, _ in test.support.iter_slot_wrappers(cls): + wrapper = getattr(cls, attr) + res = (cls, attr, wrapper) + results.append(res) + results = tuple((repr(c), a, repr(w)) for c, a, w in results) + sch.send_nowait(results) + """) + def collate_results(raw): + results = {} + for cls, attr, wrapper in raw: + key = cls, attr + assert key not in results, (results, key, wrapper) + results[key] = wrapper + return results + + exec(script) + raw = rch.recv_nowait() + main_results = collate_results(raw) + + interp = interpreters.create() + interp.exec('from test.support import interpreters') + interp.prepare_main(sch=sch) + interp.exec(script) + raw = rch.recv_nowait() + interp_results = collate_results(raw) + + for key, expected in main_results.items(): + cls, attr = key + with self.subTest(cls=cls, slotattr=attr): + actual = interp_results.pop(key) + self.assertEqual(actual, expected) + self.maxDiff = None + self.assertEqual(interp_results, {}) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index dbbbe63e95ad46..f002d28df60e9c 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1,3 +1,4 @@ +import annotationlib import contextlib import collections import collections.abc @@ -7,6 +8,7 @@ import inspect import itertools import operator +import os import pickle import re import sys @@ -31,22 +33,23 @@ from typing import dataclass_transform from typing import no_type_check, no_type_check_decorator from typing import Type -from typing import NamedTuple, NotRequired, Required, TypedDict +from typing import NamedTuple, NotRequired, Required, ReadOnly, TypedDict from typing import IO, TextIO, BinaryIO from typing import Pattern, Match from typing import Annotated, ForwardRef from typing import Self, LiteralString from typing import TypeAlias from typing import ParamSpec, Concatenate, ParamSpecArgs, ParamSpecKwargs -from typing import TypeGuard +from typing import TypeGuard, TypeIs, NoDefault import abc import textwrap import typing import weakref +import warnings import types -from test.support import captured_stderr, cpython_only, infinite_recursion -from test.typinganndata import mod_generics_cache, _typed_dict_helper +from test.support import captured_stderr, cpython_only, infinite_recursion, requires_docstrings, import_helper, run_code +from test.typinganndata import ann_module695, mod_generics_cache, _typed_dict_helper CANNOT_SUBCLASS_TYPE = 'Cannot subclass special typing classes' @@ -56,20 +59,6 @@ class BaseTestCase(TestCase): - def assertIsSubclass(self, cls, class_or_tuple, msg=None): - if not issubclass(cls, class_or_tuple): - message = '%r is not a subclass of %r' % (cls, class_or_tuple) - if msg is not None: - message += ' : %s' % msg - raise self.failureException(message) - - def assertNotIsSubclass(self, cls, class_or_tuple, msg=None): - if issubclass(cls, class_or_tuple): - message = '%r is a subclass of %r' % (cls, class_or_tuple) - if msg is not None: - message += ' : %s' % msg - raise self.failureException(message) - def clear_caches(self): for f in typing._cleanups: f() @@ -120,26 +109,49 @@ class Sub(Any): pass def test_errors(self): with self.assertRaises(TypeError): - issubclass(42, Any) + isinstance(42, Any) with self.assertRaises(TypeError): Any[int] # Any is not a generic type. def test_can_subclass(self): class Mock(Any): pass - self.assertTrue(issubclass(Mock, Any)) + self.assertIsSubclass(Mock, Any) self.assertIsInstance(Mock(), Mock) class Something: pass - self.assertFalse(issubclass(Something, Any)) + self.assertNotIsSubclass(Something, Any) self.assertNotIsInstance(Something(), Mock) class MockSomething(Something, Mock): pass - self.assertTrue(issubclass(MockSomething, Any)) + self.assertIsSubclass(MockSomething, Any) + self.assertIsSubclass(MockSomething, MockSomething) + self.assertIsSubclass(MockSomething, Something) + self.assertIsSubclass(MockSomething, Mock) ms = MockSomething() self.assertIsInstance(ms, MockSomething) self.assertIsInstance(ms, Something) self.assertIsInstance(ms, Mock) + def test_subclassing_with_custom_constructor(self): + class Sub(Any): + def __init__(self, *args, **kwargs): pass + # The instantiation must not fail. + Sub(0, s="") + + def test_multiple_inheritance_with_custom_constructors(self): + class Foo: + def __init__(self, x): + self.x = x + + class Bar(Any, Foo): + def __init__(self, x, y): + self.y = y + super().__init__(x) + + b = Bar(1, 2) + self.assertEqual(b.x, 1) + self.assertEqual(b.y, 2) + def test_cannot_instantiate(self): with self.assertRaises(TypeError): Any() @@ -558,6 +570,240 @@ def test_constructor(self): self.assertEqual(T.__name__, "T") self.assertEqual(T.__constraints__, ()) self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", bound=type) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, type) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", default=()) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, ()) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", covariant=True) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, True) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", contravariant=True) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, True) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", infer_variance=True) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, True) + + +class TypeParameterDefaultsTests(BaseTestCase): + def test_typevar(self): + T = TypeVar('T', default=int) + self.assertEqual(T.__default__, int) + self.assertTrue(T.has_default()) + self.assertIsInstance(T, TypeVar) + + class A(Generic[T]): ... + Alias = Optional[T] + + def test_typevar_none(self): + U = TypeVar('U') + U_None = TypeVar('U_None', default=None) + self.assertIs(U.__default__, NoDefault) + self.assertFalse(U.has_default()) + self.assertIs(U_None.__default__, None) + self.assertTrue(U_None.has_default()) + + class X[T]: ... + T, = X.__type_params__ + self.assertIs(T.__default__, NoDefault) + self.assertFalse(T.has_default()) + + def test_paramspec(self): + P = ParamSpec('P', default=(str, int)) + self.assertEqual(P.__default__, (str, int)) + self.assertTrue(P.has_default()) + self.assertIsInstance(P, ParamSpec) + + class A(Generic[P]): ... + Alias = typing.Callable[P, None] + + P_default = ParamSpec('P_default', default=...) + self.assertIs(P_default.__default__, ...) + + def test_paramspec_none(self): + U = ParamSpec('U') + U_None = ParamSpec('U_None', default=None) + self.assertIs(U.__default__, NoDefault) + self.assertFalse(U.has_default()) + self.assertIs(U_None.__default__, None) + self.assertTrue(U_None.has_default()) + + class X[**P]: ... + P, = X.__type_params__ + self.assertIs(P.__default__, NoDefault) + self.assertFalse(P.has_default()) + + def test_typevartuple(self): + Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) + self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) + self.assertTrue(Ts.has_default()) + self.assertIsInstance(Ts, TypeVarTuple) + + class A(Generic[Unpack[Ts]]): ... + Alias = Optional[Unpack[Ts]] + + def test_typevartuple_specialization(self): + T = TypeVar("T") + Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) + self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) + class A(Generic[T, Unpack[Ts]]): ... + self.assertEqual(A[float].__args__, (float, str, int)) + self.assertEqual(A[float, range].__args__, (float, range)) + self.assertEqual(A[float, *tuple[int, ...]].__args__, (float, *tuple[int, ...])) + + def test_typevar_and_typevartuple_specialization(self): + T = TypeVar("T") + U = TypeVar("U", default=float) + Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) + self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) + class A(Generic[T, U, Unpack[Ts]]): ... + self.assertEqual(A[int].__args__, (int, float, str, int)) + self.assertEqual(A[int, str].__args__, (int, str, str, int)) + self.assertEqual(A[int, str, range].__args__, (int, str, range)) + self.assertEqual(A[int, str, *tuple[int, ...]].__args__, (int, str, *tuple[int, ...])) + + def test_no_default_after_typevar_tuple(self): + T = TypeVar("T", default=int) + Ts = TypeVarTuple("Ts") + Ts_default = TypeVarTuple("Ts_default", default=Unpack[Tuple[str, int]]) + + with self.assertRaises(TypeError): + class X(Generic[*Ts, T]): ... + + with self.assertRaises(TypeError): + class Y(Generic[*Ts_default, T]): ... + + def test_allow_default_after_non_default_in_alias(self): + T_default = TypeVar('T_default', default=int) + T = TypeVar('T') + Ts = TypeVarTuple('Ts') + + a1 = Callable[[T_default], T] + self.assertEqual(a1.__args__, (T_default, T)) + + a2 = dict[T_default, T] + self.assertEqual(a2.__args__, (T_default, T)) + + a3 = typing.Dict[T_default, T] + self.assertEqual(a3.__args__, (T_default, T)) + + a4 = Callable[*Ts, T] + self.assertEqual(a4.__args__, (*Ts, T)) + + def test_paramspec_specialization(self): + T = TypeVar("T") + P = ParamSpec('P', default=[str, int]) + self.assertEqual(P.__default__, [str, int]) + class A(Generic[T, P]): ... + self.assertEqual(A[float].__args__, (float, (str, int))) + self.assertEqual(A[float, [range]].__args__, (float, (range,))) + + def test_typevar_and_paramspec_specialization(self): + T = TypeVar("T") + U = TypeVar("U", default=float) + P = ParamSpec('P', default=[str, int]) + self.assertEqual(P.__default__, [str, int]) + class A(Generic[T, U, P]): ... + self.assertEqual(A[float].__args__, (float, float, (str, int))) + self.assertEqual(A[float, int].__args__, (float, int, (str, int))) + self.assertEqual(A[float, int, [range]].__args__, (float, int, (range,))) + + def test_paramspec_and_typevar_specialization(self): + T = TypeVar("T") + P = ParamSpec('P', default=[str, int]) + U = TypeVar("U", default=float) + self.assertEqual(P.__default__, [str, int]) + class A(Generic[T, P, U]): ... + self.assertEqual(A[float].__args__, (float, (str, int), float)) + self.assertEqual(A[float, [range]].__args__, (float, (range,), float)) + self.assertEqual(A[float, [range], int].__args__, (float, (range,), int)) + + def test_typevartuple_none(self): + U = TypeVarTuple('U') + U_None = TypeVarTuple('U_None', default=None) + self.assertIs(U.__default__, NoDefault) + self.assertFalse(U.has_default()) + self.assertIs(U_None.__default__, None) + self.assertTrue(U_None.has_default()) + + class X[**Ts]: ... + Ts, = X.__type_params__ + self.assertIs(Ts.__default__, NoDefault) + self.assertFalse(Ts.has_default()) + + def test_no_default_after_non_default(self): + DefaultStrT = TypeVar('DefaultStrT', default=str) + T = TypeVar('T') + + with self.assertRaisesRegex( + TypeError, r"Type parameter ~T without a default follows type parameter with a default" + ): + Test = Generic[DefaultStrT, T] + + def test_need_more_params(self): + DefaultStrT = TypeVar('DefaultStrT', default=str) + T = TypeVar('T') + U = TypeVar('U') + + class A(Generic[T, U, DefaultStrT]): ... + A[int, bool] + A[int, bool, str] + + with self.assertRaisesRegex( + TypeError, r"Too few arguments for .+; actual 1, expected at least 2" + ): + Test = A[int] + + def test_pickle(self): + global U, U_co, U_contra, U_default # pickle wants to reference the class by name + U = TypeVar('U') + U_co = TypeVar('U_co', covariant=True) + U_contra = TypeVar('U_contra', contravariant=True) + U_default = TypeVar('U_default', default=int) + for proto in range(pickle.HIGHEST_PROTOCOL): + for typevar in (U, U_co, U_contra, U_default): + z = pickle.loads(pickle.dumps(typevar, proto)) + self.assertEqual(z.__name__, typevar.__name__) + self.assertEqual(z.__covariant__, typevar.__covariant__) + self.assertEqual(z.__contravariant__, typevar.__contravariant__) + self.assertEqual(z.__bound__, typevar.__bound__) + self.assertEqual(z.__default__, typevar.__default__) def template_replace(templates: list[str], replacements: dict[str, list[str]]) -> list[tuple[str]]: @@ -958,11 +1204,39 @@ def foo(**kwargs: Unpack[Movie]): ... self.assertEqual(repr(foo.__annotations__['kwargs']), f"typing.Unpack[{__name__}.Movie]") -class TypeVarTupleTests(BaseTestCase): + def test_builtin_tuple(self): + Ts = TypeVarTuple("Ts") + + class Old(Generic[*Ts]): ... + class New[*Ts]: ... + + PartOld = Old[int, *Ts] + self.assertEqual(PartOld[str].__args__, (int, str)) + self.assertEqual(PartOld[*tuple[str]].__args__, (int, str)) + self.assertEqual(PartOld[*Tuple[str]].__args__, (int, str)) + self.assertEqual(PartOld[Unpack[tuple[str]]].__args__, (int, str)) + self.assertEqual(PartOld[Unpack[Tuple[str]]].__args__, (int, str)) + + PartNew = New[int, *Ts] + self.assertEqual(PartNew[str].__args__, (int, str)) + self.assertEqual(PartNew[*tuple[str]].__args__, (int, str)) + self.assertEqual(PartNew[*Tuple[str]].__args__, (int, str)) + self.assertEqual(PartNew[Unpack[tuple[str]]].__args__, (int, str)) + self.assertEqual(PartNew[Unpack[Tuple[str]]].__args__, (int, str)) + + def test_unpack_wrong_type(self): + Ts = TypeVarTuple("Ts") + class Gen[*Ts]: ... + PartGen = Gen[int, *Ts] + + bad_unpack_param = re.escape("Unpack[...] must be used with a tuple type") + with self.assertRaisesRegex(TypeError, bad_unpack_param): + PartGen[Unpack[list[int]]] + with self.assertRaisesRegex(TypeError, bad_unpack_param): + PartGen[Unpack[List[int]]] - def assertEndsWith(self, string, tail): - if not string.endswith(tail): - self.fail(f"String {string!r} does not end with {tail!r}") + +class TypeVarTupleTests(BaseTestCase): def test_name(self): Ts = TypeVarTuple('Ts') @@ -1722,13 +1996,81 @@ def test_basics(self): u = Union[int, float] self.assertNotEqual(u, Union) - def test_subclass_error(self): + def test_union_isinstance(self): + self.assertIsInstance(42, Union[int, str]) + self.assertIsInstance('abc', Union[int, str]) + self.assertNotIsInstance(3.14, Union[int, str]) + self.assertIsInstance(42, Union[int, list[int]]) + self.assertIsInstance(42, Union[int, Any]) + + def test_union_isinstance_type_error(self): + with self.assertRaises(TypeError): + isinstance(42, Union[str, list[int]]) + with self.assertRaises(TypeError): + isinstance(42, Union[list[int], int]) + with self.assertRaises(TypeError): + isinstance(42, Union[list[int], str]) + with self.assertRaises(TypeError): + isinstance(42, Union[str, Any]) + with self.assertRaises(TypeError): + isinstance(42, Union[Any, int]) + with self.assertRaises(TypeError): + isinstance(42, Union[Any, str]) + + def test_optional_isinstance(self): + self.assertIsInstance(42, Optional[int]) + self.assertIsInstance(None, Optional[int]) + self.assertNotIsInstance('abc', Optional[int]) + + def test_optional_isinstance_type_error(self): + with self.assertRaises(TypeError): + isinstance(42, Optional[list[int]]) + with self.assertRaises(TypeError): + isinstance(None, Optional[list[int]]) + with self.assertRaises(TypeError): + isinstance(42, Optional[Any]) + with self.assertRaises(TypeError): + isinstance(None, Optional[Any]) + + def test_union_issubclass(self): + self.assertIsSubclass(int, Union[int, str]) + self.assertIsSubclass(str, Union[int, str]) + self.assertNotIsSubclass(float, Union[int, str]) + self.assertIsSubclass(int, Union[int, list[int]]) + self.assertIsSubclass(int, Union[int, Any]) + self.assertNotIsSubclass(int, Union[str, Any]) + self.assertIsSubclass(int, Union[Any, int]) + self.assertNotIsSubclass(int, Union[Any, str]) + + def test_union_issubclass_type_error(self): with self.assertRaises(TypeError): issubclass(int, Union) with self.assertRaises(TypeError): issubclass(Union, int) with self.assertRaises(TypeError): issubclass(Union[int, str], int) + with self.assertRaises(TypeError): + issubclass(int, Union[str, list[int]]) + with self.assertRaises(TypeError): + issubclass(int, Union[list[int], int]) + with self.assertRaises(TypeError): + issubclass(int, Union[list[int], str]) + + def test_optional_issubclass(self): + self.assertIsSubclass(int, Optional[int]) + self.assertIsSubclass(type(None), Optional[int]) + self.assertNotIsSubclass(str, Optional[int]) + self.assertIsSubclass(Any, Optional[Any]) + self.assertIsSubclass(type(None), Optional[Any]) + self.assertNotIsSubclass(int, Optional[Any]) + + def test_optional_issubclass_type_error(self): + with self.assertRaises(TypeError): + issubclass(list[int], Optional[list[int]]) + with self.assertRaises(TypeError): + issubclass(type(None), Optional[list[int]]) + with self.assertRaises(TypeError): + issubclass(int, Optional[list[int]]) def test_union_any(self): u = Union[Any] @@ -2213,7 +2555,7 @@ def test_concatenate(self): def test_nested_paramspec(self): # Since Callable has some special treatment, we want to be sure - # that substituion works correctly, see gh-103054 + # that substitution works correctly, see gh-103054 Callable = self.Callable P = ParamSpec('P') P2 = ParamSpec('P2') @@ -3486,7 +3828,7 @@ def meth(self): pass acceptable_extra_attrs = { '_is_protocol', '_is_runtime_protocol', '__parameters__', - '__init__', '__annotations__', '__subclasshook__', + '__init__', '__annotations__', '__subclasshook__', '__annotate__', } self.assertLessEqual(vars(NonP).keys(), vars(C).keys() | acceptable_extra_attrs) self.assertLessEqual( @@ -3708,8 +4050,8 @@ def test_generic_protocols_repr(self): class P(Protocol[T, S]): pass - self.assertTrue(repr(P[T, S]).endswith('P[~T, ~S]')) - self.assertTrue(repr(P[int, str]).endswith('P[int, str]')) + self.assertEndsWith(repr(P[T, S]), 'P[~T, ~S]') + self.assertEndsWith(repr(P[int, str]), 'P[int, str]') def test_generic_protocols_eq(self): T = TypeVar('T') @@ -3965,9 +4307,15 @@ def test_builtin_protocol_allowlist(self): class CustomProtocol(TestCase, Protocol): pass + class CustomPathLikeProtocol(os.PathLike, Protocol): + pass + class CustomContextManager(typing.ContextManager, Protocol): pass + class CustomAsyncIterator(typing.AsyncIterator, Protocol): + pass + def test_non_runtime_protocol_isinstance_check(self): class P(Protocol): x: int @@ -4293,8 +4641,7 @@ class C(Generic[T]): self.assertNotEqual(Z, Y[int]) self.assertNotEqual(Z, Y[T]) - self.assertTrue(str(Z).endswith( - '.C[typing.Tuple[str, int]]')) + self.assertEndsWith(str(Z), '.C[typing.Tuple[str, int]]') def test_new_repr(self): T = TypeVar('T') @@ -4522,12 +4869,12 @@ class A(Generic[T]): self.assertNotEqual(typing.FrozenSet[A[str]], typing.FrozenSet[mod_generics_cache.B.A[str]]) - self.assertTrue(repr(Tuple[A[str]]).endswith('.A[str]]')) - self.assertTrue(repr(Tuple[B.A[str]]).endswith('.B.A[str]]')) - self.assertTrue(repr(Tuple[mod_generics_cache.A[str]]) - .endswith('mod_generics_cache.A[str]]')) - self.assertTrue(repr(Tuple[mod_generics_cache.B.A[str]]) - .endswith('mod_generics_cache.B.A[str]]')) + self.assertEndsWith(repr(Tuple[A[str]]), '.A[str]]') + self.assertEndsWith(repr(Tuple[B.A[str]]), '.B.A[str]]') + self.assertEndsWith(repr(Tuple[mod_generics_cache.A[str]]), + 'mod_generics_cache.A[str]]') + self.assertEndsWith(repr(Tuple[mod_generics_cache.B.A[str]]), + 'mod_generics_cache.B.A[str]]') def test_extended_generic_rules_eq(self): T = TypeVar('T') @@ -4621,6 +4968,88 @@ def f(x: X): ... {'x': list[list[ForwardRef('X')]]} ) + def test_pep695_generic_class_with_future_annotations(self): + original_globals = dict(ann_module695.__dict__) + + hints_for_A = get_type_hints(ann_module695.A) + A_type_params = ann_module695.A.__type_params__ + self.assertIs(hints_for_A["x"], A_type_params[0]) + self.assertEqual(hints_for_A["y"].__args__[0], Unpack[A_type_params[1]]) + self.assertIs(hints_for_A["z"].__args__[0], A_type_params[2]) + + # should not have changed as a result of the get_type_hints() calls! + self.assertEqual(ann_module695.__dict__, original_globals) + + def test_pep695_generic_class_with_future_annotations_and_local_shadowing(self): + hints_for_B = get_type_hints(ann_module695.B) + self.assertEqual(hints_for_B, {"x": int, "y": str, "z": bytes}) + + def test_pep695_generic_class_with_future_annotations_name_clash_with_global_vars(self): + hints_for_C = get_type_hints(ann_module695.C) + self.assertEqual( + set(hints_for_C.values()), + set(ann_module695.C.__type_params__) + ) + + def test_pep_695_generic_function_with_future_annotations(self): + hints_for_generic_function = get_type_hints(ann_module695.generic_function) + func_t_params = ann_module695.generic_function.__type_params__ + self.assertEqual( + hints_for_generic_function.keys(), {"x", "y", "z", "zz", "return"} + ) + self.assertIs(hints_for_generic_function["x"], func_t_params[0]) + self.assertEqual(hints_for_generic_function["y"], Unpack[func_t_params[1]]) + self.assertIs(hints_for_generic_function["z"].__origin__, func_t_params[2]) + self.assertIs(hints_for_generic_function["zz"].__origin__, func_t_params[2]) + + def test_pep_695_generic_function_with_future_annotations_name_clash_with_global_vars(self): + self.assertEqual( + set(get_type_hints(ann_module695.generic_function_2).values()), + set(ann_module695.generic_function_2.__type_params__) + ) + + def test_pep_695_generic_method_with_future_annotations(self): + hints_for_generic_method = get_type_hints(ann_module695.D.generic_method) + params = { + param.__name__: param + for param in ann_module695.D.generic_method.__type_params__ + } + self.assertEqual( + hints_for_generic_method, + {"x": params["Foo"], "y": params["Bar"], "return": types.NoneType} + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_vars(self): + self.assertEqual( + set(get_type_hints(ann_module695.D.generic_method_2).values()), + set(ann_module695.D.generic_method_2.__type_params__) + ) + + def test_pep_695_generics_with_future_annotations_nested_in_function(self): + results = ann_module695.nested() + + self.assertEqual( + set(results.hints_for_E.values()), + set(results.E.__type_params__) + ) + self.assertEqual( + set(results.hints_for_E_meth.values()), + set(results.E.generic_method.__type_params__) + ) + self.assertNotEqual( + set(results.hints_for_E_meth.values()), + set(results.E.__type_params__) + ) + self.assertEqual( + set(results.hints_for_E_meth.values()).intersection(results.E.__type_params__), + set() + ) + + self.assertEqual( + set(results.hints_for_generic_func.values()), + set(results.generic_func.__type_params__) + ) + def test_extended_generic_rules_subclassing(self): class T1(Tuple[T, KT]): ... class T2(Tuple[T, ...]): ... @@ -4655,8 +5084,6 @@ def test_fail_with_bare_union(self): List[Union] with self.assertRaises(TypeError): Tuple[Optional] - with self.assertRaises(TypeError): - ClassVar[ClassVar[int]] with self.assertRaises(TypeError): List[ClassVar[int]] @@ -4737,6 +5164,18 @@ class C(B[int]): x = pickle.loads(z) self.assertEqual(s, x) + # Test ParamSpec args and kwargs + global PP + PP = ParamSpec('PP') + for thing in [PP.args, PP.kwargs]: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(thing=thing, proto=proto): + self.assertEqual( + pickle.loads(pickle.dumps(thing, proto)), + thing, + ) + del PP + def test_copy_and_deepcopy(self): T = TypeVar('T') class Node(Generic[T]): ... @@ -5189,6 +5628,7 @@ def test_subclass_special_form(self): Literal[1, 2], Concatenate[int, ParamSpec("P")], TypeGuard[int], + TypeIs[range], ): with self.subTest(msg=obj): with self.assertRaisesRegex( @@ -5376,7 +5816,7 @@ def __call__(self, *args, **kwargs): @Wrapper def wrapped(): ... self.assertIsInstance(wrapped, Wrapper) - self.assertIs(False, hasattr(wrapped, "__final__")) + self.assertNotHasAttr(wrapped, "__final__") class Meta(type): @property @@ -5388,7 +5828,7 @@ class WithMeta(metaclass=Meta): ... # Builtin classes throw TypeError if you try to set an # attribute. final(int) - self.assertIs(False, hasattr(int, "__final__")) + self.assertNotHasAttr(int, "__final__") # Make sure it works with common builtin decorators class Methods: @@ -5469,19 +5909,19 @@ def static_method_bad_order(): self.assertEqual(Derived.class_method_good_order(), 42) self.assertIs(True, Derived.class_method_good_order.__override__) self.assertEqual(Derived.class_method_bad_order(), 42) - self.assertIs(False, hasattr(Derived.class_method_bad_order, "__override__")) + self.assertNotHasAttr(Derived.class_method_bad_order, "__override__") self.assertEqual(Derived.static_method_good_order(), 42) self.assertIs(True, Derived.static_method_good_order.__override__) self.assertEqual(Derived.static_method_bad_order(), 42) - self.assertIs(False, hasattr(Derived.static_method_bad_order, "__override__")) + self.assertNotHasAttr(Derived.static_method_bad_order, "__override__") # Base object is not changed: - self.assertIs(False, hasattr(Base.normal_method, "__override__")) - self.assertIs(False, hasattr(Base.class_method_good_order, "__override__")) - self.assertIs(False, hasattr(Base.class_method_bad_order, "__override__")) - self.assertIs(False, hasattr(Base.static_method_good_order, "__override__")) - self.assertIs(False, hasattr(Base.static_method_bad_order, "__override__")) + self.assertNotHasAttr(Base.normal_method, "__override__") + self.assertNotHasAttr(Base.class_method_good_order, "__override__") + self.assertNotHasAttr(Base.class_method_bad_order, "__override__") + self.assertNotHasAttr(Base.static_method_good_order, "__override__") + self.assertNotHasAttr(Base.static_method_bad_order, "__override__") def test_property(self): class Base: @@ -5506,8 +5946,8 @@ def wrong(self) -> int: self.assertEqual(instance.correct, 2) self.assertTrue(Child.correct.fget.__override__) self.assertEqual(instance.wrong, 2) - self.assertFalse(hasattr(Child.wrong, "__override__")) - self.assertFalse(hasattr(Child.wrong.fset, "__override__")) + self.assertNotHasAttr(Child.wrong, "__override__") + self.assertNotHasAttr(Child.wrong.fset, "__override__") def test_silent_failure(self): class CustomProp: @@ -5524,7 +5964,7 @@ def some(self): return 1 self.assertEqual(WithOverride.some, 1) - self.assertFalse(hasattr(WithOverride.some, "__override__")) + self.assertNotHasAttr(WithOverride.some, "__override__") def test_multiple_decorators(self): def with_wraps(f): # similar to `lru_cache` definition @@ -5867,6 +6307,12 @@ def foo(a: 'Node[T'): with self.assertRaises(SyntaxError): get_type_hints(foo) + def test_syntax_error_empty_string(self): + for form in [typing.List, typing.Set, typing.Type, typing.Deque]: + with self.subTest(form=form): + with self.assertRaises(SyntaxError): + form[''] + def test_name_error(self): def foo(a: 'Noode[T]'): @@ -6008,16 +6454,6 @@ class F: for clazz in [C, D, E, F]: self.assertEqual(get_type_hints(clazz), expected_result) - def test_nested_classvar_fails_forward_ref_check(self): - class E: - foo: 'typing.ClassVar[typing.ClassVar[int]]' = 7 - class F: - foo: ClassVar['ClassVar[int]'] = 7 - - for clazz in [E, F]: - with self.assertRaises(TypeError): - get_type_hints(clazz) - def test_meta_no_type_check(self): depr_msg = ( "'typing.no_type_check_decorator' is deprecated " @@ -6070,6 +6506,42 @@ def test_or(self): self.assertEqual("x" | X, Union["x", X]) +class InternalsTests(BaseTestCase): + def test_deprecation_for_no_type_params_passed_to__evaluate(self): + with self.assertWarnsRegex( + DeprecationWarning, + ( + "Failing to pass a value to the 'type_params' parameter " + "of 'typing._eval_type' is deprecated" + ) + ) as cm: + self.assertEqual(typing._eval_type(list["int"], globals(), {}), list[int]) + + self.assertEqual(cm.filename, __file__) + + f = ForwardRef("int") + + with self.assertWarnsRegex( + DeprecationWarning, + ( + "Failing to pass a value to the 'type_params' parameter " + "of 'typing.ForwardRef._evaluate' is deprecated" + ) + ) as cm: + self.assertIs(f._evaluate(globals(), {}, recursive_guard=frozenset()), int) + + self.assertEqual(cm.filename, __file__) + + def test_collect_parameters(self): + typing = import_helper.import_fresh_module("typing") + with self.assertWarnsRegex( + DeprecationWarning, + "The private _collect_parameters function is deprecated" + ) as cm: + typing._collect_parameters + self.assertEqual(cm.filename, __file__) + + @lru_cache() def cached_func(x, y): return 3 * x + y @@ -6342,7 +6814,7 @@ def test_get_type_hints_from_various_objects(self): gth(None) def test_get_type_hints_modules(self): - ann_module_type_hints = {1: 2, 'f': Tuple[int, int], 'x': int, 'y': str, 'u': int | float} + ann_module_type_hints = {'f': Tuple[int, int], 'x': int, 'y': str, 'u': int | float} self.assertEqual(gth(ann_module), ann_module_type_hints) self.assertEqual(gth(ann_module2), {}) self.assertEqual(gth(ann_module3), {}) @@ -6360,7 +6832,7 @@ def test_get_type_hints_classes(self): self.assertEqual(gth(ann_module.C), # gth will find the right globalns {'y': Optional[ann_module.C]}) self.assertIsInstance(gth(ann_module.j_class), dict) - self.assertEqual(gth(ann_module.M), {'123': 123, 'o': type}) + self.assertEqual(gth(ann_module.M), {'o': type}) self.assertEqual(gth(ann_module.D), {'j': str, 'k': str, 'y': Optional[ann_module.C]}) self.assertEqual(gth(ann_module.Y), {'z': int}) @@ -6643,6 +7115,44 @@ def h(x: collections.abc.Callable[P, int]): ... self.assertEqual(get_type_hints(g), {'x': collections.abc.Callable[..., int]}) self.assertEqual(get_type_hints(h), {'x': collections.abc.Callable[P, int]}) + def test_get_type_hints_format(self): + class C: + x: undefined + + with self.assertRaises(NameError): + get_type_hints(C) + + with self.assertRaises(NameError): + get_type_hints(C, format=annotationlib.Format.VALUE) + + annos = get_type_hints(C, format=annotationlib.Format.FORWARDREF) + self.assertIsInstance(annos, dict) + self.assertEqual(list(annos), ['x']) + self.assertIsInstance(annos['x'], annotationlib.ForwardRef) + self.assertEqual(annos['x'].__arg__, 'undefined') + + self.assertEqual(get_type_hints(C, format=annotationlib.Format.STRING), + {'x': 'undefined'}) + + def test_get_type_hints_format_function(self): + def func(x: undefined) -> undefined: ... + + # VALUE + with self.assertRaises(NameError): + get_type_hints(func) + with self.assertRaises(NameError): + get_type_hints(func, format=annotationlib.Format.VALUE) + + # FORWARDREF + self.assertEqual( + get_type_hints(func, format=annotationlib.Format.FORWARDREF), + {'x': ForwardRef('undefined'), 'return': ForwardRef('undefined')}, + ) + + # STRING + self.assertEqual(get_type_hints(func, format=annotationlib.Format.STRING), + {'x': 'undefined', 'return': 'undefined'}) + class GetUtilitiesTestCase(TestCase): def test_get_origin(self): @@ -6734,6 +7244,7 @@ class C(Generic[T]): pass self.assertEqual(get_args(NotRequired[int]), (int,)) self.assertEqual(get_args(TypeAlias), ()) self.assertEqual(get_args(TypeGuard[int]), (int,)) + self.assertEqual(get_args(TypeIs[range]), (range,)) Ts = TypeVarTuple('Ts') self.assertEqual(get_args(Ts), ()) self.assertEqual(get_args((*Ts,)[0]), (Ts,)) @@ -6744,6 +7255,51 @@ class C(Generic[T]): pass self.assertEqual(get_args(Unpack[tuple[Unpack[Ts]]]), (tuple[Unpack[Ts]],)) +class EvaluateForwardRefTests(BaseTestCase): + def test_evaluate_forward_ref(self): + int_ref = ForwardRef('int') + missing = ForwardRef('missing') + self.assertIs( + typing.evaluate_forward_ref(int_ref, type_params=()), + int, + ) + self.assertIs( + typing.evaluate_forward_ref( + int_ref, type_params=(), format=annotationlib.Format.FORWARDREF, + ), + int, + ) + self.assertIs( + typing.evaluate_forward_ref( + missing, type_params=(), format=annotationlib.Format.FORWARDREF, + ), + missing, + ) + self.assertEqual( + typing.evaluate_forward_ref( + int_ref, type_params=(), format=annotationlib.Format.STRING, + ), + 'int', + ) + + def test_evaluate_forward_ref_no_type_params(self): + ref = ForwardRef('int') + with self.assertWarnsRegex( + DeprecationWarning, + ( + "Failing to pass a value to the 'type_params' parameter " + "of 'typing.evaluate_forward_ref' is deprecated, " + "as it leads to incorrect behaviour" + ), + ): + typing.evaluate_forward_ref(ref) + + # No warnings when `type_params` is passed: + with warnings.catch_warnings(record=True) as w: + typing.evaluate_forward_ref(ref, type_params=()) + self.assertEqual(w, []) + + class CollectionsAbcTests(BaseTestCase): def test_hashable(self): @@ -6767,24 +7323,16 @@ def test_iterator(self): self.assertNotIsInstance(42, typing.Iterator) def test_awaitable(self): - ns = {} - exec( - "async def foo() -> typing.Awaitable[int]:\n" - " return await AwaitableWrapper(42)\n", - globals(), ns) - foo = ns['foo'] + async def foo() -> typing.Awaitable[int]: + return await AwaitableWrapper(42) g = foo() self.assertIsInstance(g, typing.Awaitable) self.assertNotIsInstance(foo, typing.Awaitable) g.send(None) # Run foo() till completion, to avoid warning. def test_coroutine(self): - ns = {} - exec( - "async def foo():\n" - " return\n", - globals(), ns) - foo = ns['foo'] + async def foo(): + return g = foo() self.assertIsInstance(g, typing.Coroutine) with self.assertRaises(TypeError): @@ -6846,16 +7394,6 @@ def test_mutablesequence(self): self.assertIsInstance([], typing.MutableSequence) self.assertNotIsInstance((), typing.MutableSequence) - def test_bytestring(self): - with self.assertWarns(DeprecationWarning): - self.assertIsInstance(b'', typing.ByteString) - with self.assertWarns(DeprecationWarning): - self.assertIsInstance(bytearray(b''), typing.ByteString) - with self.assertWarns(DeprecationWarning): - class Foo(typing.ByteString): ... - with self.assertWarns(DeprecationWarning): - class Bar(typing.ByteString, typing.Awaitable): ... - def test_list(self): self.assertIsSubclass(list, typing.List) @@ -7049,6 +7587,17 @@ def foo(): g = foo() self.assertIsSubclass(type(g), typing.Generator) + def test_generator_default(self): + g1 = typing.Generator[int] + g2 = typing.Generator[int, None, None] + self.assertEqual(get_args(g1), (int, type(None), type(None))) + self.assertEqual(get_args(g1), get_args(g2)) + + g3 = typing.Generator[int, float] + g4 = typing.Generator[int, float, None] + self.assertEqual(get_args(g3), (int, float, type(None))) + self.assertEqual(get_args(g3), get_args(g4)) + def test_no_generator_instantiation(self): with self.assertRaises(TypeError): typing.Generator() @@ -7058,10 +7607,9 @@ def test_no_generator_instantiation(self): typing.Generator[int, int, int]() def test_async_generator(self): - ns = {} - exec("async def f():\n" - " yield 42\n", globals(), ns) - g = ns['f']() + async def f(): + yield 42 + g = f() self.assertIsSubclass(type(g), typing.AsyncGenerator) def test_no_async_generator_instantiation(self): @@ -7148,9 +7696,8 @@ def asend(self, value): def athrow(self, typ, val=None, tb=None): pass - ns = {} - exec('async def g(): yield 0', globals(), ns) - g = ns['g'] + async def g(): yield 0 + self.assertIsSubclass(G, typing.AsyncGenerator) self.assertIsSubclass(G, typing.AsyncIterable) self.assertIsSubclass(G, collections.abc.AsyncGenerator) @@ -7235,6 +7782,15 @@ def manager(): self.assertIsInstance(cm, typing.ContextManager) self.assertNotIsInstance(42, typing.ContextManager) + def test_contextmanager_type_params(self): + cm1 = typing.ContextManager[int] + self.assertEqual(get_args(cm1), (int, bool | None)) + cm2 = typing.ContextManager[int, None] + self.assertEqual(get_args(cm2), (int, types.NoneType)) + + type gen_cm[T1, T2] = typing.ContextManager[T1, T2] + self.assertEqual(get_args(gen_cm.__value__[int, None]), (int, types.NoneType)) + def test_async_contextmanager(self): class NotACM: pass @@ -7246,11 +7802,17 @@ def manager(): cm = manager() self.assertNotIsInstance(cm, typing.AsyncContextManager) - self.assertEqual(typing.AsyncContextManager[int].__args__, (int,)) + self.assertEqual(typing.AsyncContextManager[int].__args__, (int, bool | None)) with self.assertRaises(TypeError): isinstance(42, typing.AsyncContextManager[int]) with self.assertRaises(TypeError): - typing.AsyncContextManager[int, str] + typing.AsyncContextManager[int, str, float] + + def test_asynccontextmanager_type_params(self): + cm1 = typing.AsyncContextManager[int] + self.assertEqual(get_args(cm1), (int, bool | None)) + cm2 = typing.AsyncContextManager[int, None] + self.assertEqual(get_args(cm2), (int, types.NoneType)) class TypeTests(BaseTestCase): @@ -7455,6 +8017,48 @@ class XMethBad2(NamedTuple): def _source(self): return 'no chance for this as well' + def test_annotation_type_check(self): + # These are rejected by _type_check + with self.assertRaises(TypeError): + class X(NamedTuple): + a: Final + with self.assertRaises(TypeError): + class Y(NamedTuple): + a: (1, 2) + + # Conversion by _type_convert + class Z(NamedTuple): + a: None + b: "str" + annos = {'a': type(None), 'b': ForwardRef("str")} + self.assertEqual(Z.__annotations__, annos) + self.assertEqual(Z.__annotate__(annotationlib.Format.VALUE), annos) + self.assertEqual(Z.__annotate__(annotationlib.Format.FORWARDREF), annos) + self.assertEqual(Z.__annotate__(annotationlib.Format.STRING), {"a": "None", "b": "str"}) + + def test_future_annotations(self): + code = """ + from __future__ import annotations + from typing import NamedTuple + class X(NamedTuple): + a: int + b: None + """ + ns = run_code(textwrap.dedent(code)) + X = ns['X'] + self.assertEqual(X.__annotations__, {'a': ForwardRef("int"), 'b': ForwardRef("None")}) + + def test_deferred_annotations(self): + class X(NamedTuple): + y: undefined + + self.assertEqual(X._fields, ('y',)) + with self.assertRaises(NameError): + X.__annotations__ + + undefined = int + self.assertEqual(X.__annotations__, {'y': int}) + def test_multiple_inheritance(self): class A: pass @@ -7769,7 +8373,11 @@ def test_basics_functional_syntax(self): self.assertEqual(Emp.__name__, 'Emp') self.assertEqual(Emp.__module__, __name__) self.assertEqual(Emp.__bases__, (dict,)) - self.assertEqual(Emp.__annotations__, {'name': str, 'id': int}) + annos = {'name': str, 'id': int} + self.assertEqual(Emp.__annotations__, annos) + self.assertEqual(Emp.__annotate__(annotationlib.Format.VALUE), annos) + self.assertEqual(Emp.__annotate__(annotationlib.Format.FORWARDREF), annos) + self.assertEqual(Emp.__annotate__(annotationlib.Format.STRING), {'name': 'str', 'id': 'int'}) self.assertEqual(Emp.__total__, True) self.assertEqual(Emp.__required_keys__, {'name', 'id'}) self.assertIsInstance(Emp.__required_keys__, frozenset) @@ -8130,6 +8738,8 @@ class A[T](TypedDict): self.assertEqual(A.__bases__, (Generic, dict)) self.assertEqual(A.__orig_bases__, (TypedDict, Generic[T])) self.assertEqual(A.__mro__, (A, Generic, dict, object)) + self.assertEqual(A.__annotations__, {'a': T}) + self.assertEqual(A.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) self.assertEqual(A.__parameters__, (T,)) self.assertEqual(A[str].__parameters__, ()) self.assertEqual(A[str].__args__, (str,)) @@ -8141,6 +8751,8 @@ class A(TypedDict, Generic[T]): self.assertEqual(A.__bases__, (Generic, dict)) self.assertEqual(A.__orig_bases__, (TypedDict, Generic[T])) self.assertEqual(A.__mro__, (A, Generic, dict, object)) + self.assertEqual(A.__annotations__, {'a': T}) + self.assertEqual(A.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) self.assertEqual(A.__parameters__, (T,)) self.assertEqual(A[str].__parameters__, ()) self.assertEqual(A[str].__args__, (str,)) @@ -8151,6 +8763,8 @@ class A2(Generic[T], TypedDict): self.assertEqual(A2.__bases__, (Generic, dict)) self.assertEqual(A2.__orig_bases__, (Generic[T], TypedDict)) self.assertEqual(A2.__mro__, (A2, Generic, dict, object)) + self.assertEqual(A2.__annotations__, {'a': T}) + self.assertEqual(A2.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) self.assertEqual(A2.__parameters__, (T,)) self.assertEqual(A2[str].__parameters__, ()) self.assertEqual(A2[str].__args__, (str,)) @@ -8161,6 +8775,8 @@ class B(A[KT], total=False): self.assertEqual(B.__bases__, (Generic, dict)) self.assertEqual(B.__orig_bases__, (A[KT],)) self.assertEqual(B.__mro__, (B, Generic, dict, object)) + self.assertEqual(B.__annotations__, {'a': T, 'b': KT}) + self.assertEqual(B.__annotate__(annotationlib.Format.STRING), {'a': 'T', 'b': 'KT'}) self.assertEqual(B.__parameters__, (KT,)) self.assertEqual(B.__total__, False) self.assertEqual(B.__optional_keys__, frozenset(['b'])) @@ -8185,6 +8801,11 @@ class C(B[int]): 'b': KT, 'c': int, }) + self.assertEqual(C.__annotate__(annotationlib.Format.STRING), { + 'a': 'T', + 'b': 'KT', + 'c': 'int', + }) with self.assertRaises(TypeError): C[str] @@ -8204,6 +8825,11 @@ class Point3D(Point2DGeneric[T], Generic[T, KT]): 'b': T, 'c': KT, }) + self.assertEqual(Point3D.__annotate__(annotationlib.Format.STRING), { + 'a': 'T', + 'b': 'T', + 'c': 'KT', + }) self.assertEqual(Point3D[int, str].__origin__, Point3D) with self.assertRaises(TypeError): @@ -8235,6 +8861,11 @@ class WithImplicitAny(B): 'b': KT, 'c': int, }) + self.assertEqual(WithImplicitAny.__annotate__(annotationlib.Format.STRING), { + 'a': 'T', + 'b': 'KT', + 'c': 'int', + }) with self.assertRaises(TypeError): WithImplicitAny[str] @@ -8328,6 +8959,117 @@ class T4(TypedDict, Generic[S]): pass self.assertEqual(klass.__optional_keys__, set()) self.assertIsInstance(klass(), dict) + def test_readonly_inheritance(self): + class Base1(TypedDict): + a: ReadOnly[int] + + class Child1(Base1): + b: str + + self.assertEqual(Child1.__readonly_keys__, frozenset({'a'})) + self.assertEqual(Child1.__mutable_keys__, frozenset({'b'})) + + class Base2(TypedDict): + a: int + + class Child2(Base2): + b: ReadOnly[str] + + self.assertEqual(Child2.__readonly_keys__, frozenset({'b'})) + self.assertEqual(Child2.__mutable_keys__, frozenset({'a'})) + + def test_cannot_make_mutable_key_readonly(self): + class Base(TypedDict): + a: int + + with self.assertRaises(TypeError): + class Child(Base): + a: ReadOnly[int] + + def test_can_make_readonly_key_mutable(self): + class Base(TypedDict): + a: ReadOnly[int] + + class Child(Base): + a: int + + self.assertEqual(Child.__readonly_keys__, frozenset()) + self.assertEqual(Child.__mutable_keys__, frozenset({'a'})) + + def test_combine_qualifiers(self): + class AllTheThings(TypedDict): + a: Annotated[Required[ReadOnly[int]], "why not"] + b: Required[Annotated[ReadOnly[int], "why not"]] + c: ReadOnly[NotRequired[Annotated[int, "why not"]]] + d: NotRequired[Annotated[int, "why not"]] + + self.assertEqual(AllTheThings.__required_keys__, frozenset({'a', 'b'})) + self.assertEqual(AllTheThings.__optional_keys__, frozenset({'c', 'd'})) + self.assertEqual(AllTheThings.__readonly_keys__, frozenset({'a', 'b', 'c'})) + self.assertEqual(AllTheThings.__mutable_keys__, frozenset({'d'})) + + self.assertEqual( + get_type_hints(AllTheThings, include_extras=False), + {'a': int, 'b': int, 'c': int, 'd': int}, + ) + self.assertEqual( + get_type_hints(AllTheThings, include_extras=True), + { + 'a': Annotated[Required[ReadOnly[int]], 'why not'], + 'b': Required[Annotated[ReadOnly[int], 'why not']], + 'c': ReadOnly[NotRequired[Annotated[int, 'why not']]], + 'd': NotRequired[Annotated[int, 'why not']], + }, + ) + + def test_annotations(self): + # _type_check is applied + with self.assertRaisesRegex(TypeError, "Plain typing.Final is not valid as type argument"): + class X(TypedDict): + a: Final + + # _type_convert is applied + class Y(TypedDict): + a: None + b: "int" + fwdref = ForwardRef('int', module=__name__) + self.assertEqual(Y.__annotations__, {'a': type(None), 'b': fwdref}) + self.assertEqual(Y.__annotate__(annotationlib.Format.FORWARDREF), {'a': type(None), 'b': fwdref}) + + # _type_check is also applied later + class Z(TypedDict): + a: undefined + + with self.assertRaises(NameError): + Z.__annotations__ + + undefined = Final + with self.assertRaisesRegex(TypeError, "Plain typing.Final is not valid as type argument"): + Z.__annotations__ + + undefined = None + self.assertEqual(Z.__annotations__, {'a': type(None)}) + + def test_deferred_evaluation(self): + class A(TypedDict): + x: NotRequired[undefined] + y: ReadOnly[undefined] + z: Required[undefined] + + self.assertEqual(A.__required_keys__, frozenset({'y', 'z'})) + self.assertEqual(A.__optional_keys__, frozenset({'x'})) + self.assertEqual(A.__readonly_keys__, frozenset({'y'})) + self.assertEqual(A.__mutable_keys__, frozenset({'x', 'z'})) + + with self.assertRaises(NameError): + A.__annotations__ + + self.assertEqual( + A.__annotate__(annotationlib.Format.STRING), + {'x': 'NotRequired[undefined]', 'y': 'ReadOnly[undefined]', + 'z': 'Required[undefined]'}, + ) + class RequiredTests(BaseTestCase): @@ -8710,6 +9452,34 @@ class C: self.assertEqual(get_type_hints(C, globals())['classvar'], ClassVar[int]) self.assertEqual(get_type_hints(C, globals())['const'], Final[int]) + def test_special_forms_nesting(self): + # These are uncommon types and are to ensure runtime + # is lax on validation. See gh-89547 for more context. + class CF: + x: ClassVar[Final[int]] + + class FC: + x: Final[ClassVar[int]] + + class ACF: + x: Annotated[ClassVar[Final[int]], "a decoration"] + + class CAF: + x: ClassVar[Annotated[Final[int], "a decoration"]] + + class AFC: + x: Annotated[Final[ClassVar[int]], "a decoration"] + + class FAC: + x: Final[Annotated[ClassVar[int], "a decoration"]] + + self.assertEqual(get_type_hints(CF, globals())['x'], ClassVar[Final[int]]) + self.assertEqual(get_type_hints(FC, globals())['x'], Final[ClassVar[int]]) + self.assertEqual(get_type_hints(ACF, globals())['x'], ClassVar[Final[int]]) + self.assertEqual(get_type_hints(CAF, globals())['x'], ClassVar[Final[int]]) + self.assertEqual(get_type_hints(AFC, globals())['x'], Final[ClassVar[int]]) + self.assertEqual(get_type_hints(FAC, globals())['x'], Final[ClassVar[int]]) + def test_cannot_subclass(self): with self.assertRaisesRegex(TypeError, "Cannot subclass .*Annotated"): class C(Annotated): @@ -9417,6 +10187,18 @@ def test_valid_uses(self): self.assertEqual(C4.__args__, (Concatenate[int, T, P], T)) self.assertEqual(C4.__parameters__, (T, P)) + def test_invalid_uses(self): + with self.assertRaisesRegex(TypeError, 'Concatenate of no types'): + Concatenate[()] + with self.assertRaisesRegex( + TypeError, + ( + 'The last parameter to Concatenate should be a ' + 'ParamSpec variable or ellipsis' + ), + ): + Concatenate[int] + def test_var_substitution(self): T = TypeVar('T') P = ParamSpec('P') @@ -9487,6 +10269,56 @@ def test_no_isinstance(self): issubclass(int, TypeGuard) +class TypeIsTests(BaseTestCase): + def test_basics(self): + TypeIs[int] # OK + + def foo(arg) -> TypeIs[int]: ... + self.assertEqual(gth(foo), {'return': TypeIs[int]}) + + with self.assertRaises(TypeError): + TypeIs[int, str] + + def test_repr(self): + self.assertEqual(repr(TypeIs), 'typing.TypeIs') + cv = TypeIs[int] + self.assertEqual(repr(cv), 'typing.TypeIs[int]') + cv = TypeIs[Employee] + self.assertEqual(repr(cv), 'typing.TypeIs[%s.Employee]' % __name__) + cv = TypeIs[tuple[int]] + self.assertEqual(repr(cv), 'typing.TypeIs[tuple[int]]') + + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class C(type(TypeIs)): + pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class D(type(TypeIs[int])): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.TypeIs'): + class E(TypeIs): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.TypeIs\[int\]'): + class F(TypeIs[int]): + pass + + def test_cannot_init(self): + with self.assertRaises(TypeError): + TypeIs() + with self.assertRaises(TypeError): + type(TypeIs)() + with self.assertRaises(TypeError): + type(TypeIs[Optional[int]])() + + def test_no_isinstance(self): + with self.assertRaises(TypeError): + isinstance(1, TypeIs[int]) + with self.assertRaises(TypeError): + issubclass(int, TypeIs) + + SpecialAttrsP = typing.ParamSpec('SpecialAttrsP') SpecialAttrsT = typing.TypeVar('SpecialAttrsT', int, float, complex) @@ -9502,7 +10334,6 @@ def test_special_attrs(self): typing.AsyncIterable: 'AsyncIterable', typing.AsyncIterator: 'AsyncIterator', typing.Awaitable: 'Awaitable', - typing.ByteString: 'ByteString', typing.Callable: 'Callable', typing.ChainMap: 'ChainMap', typing.Collection: 'Collection', @@ -9536,7 +10367,7 @@ def test_special_attrs(self): typing.ValuesView: 'ValuesView', # Subscribed ABC classes typing.AbstractSet[Any]: 'AbstractSet', - typing.AsyncContextManager[Any]: 'AsyncContextManager', + typing.AsyncContextManager[Any, Any]: 'AsyncContextManager', typing.AsyncGenerator[Any, Any]: 'AsyncGenerator', typing.AsyncIterable[Any]: 'AsyncIterable', typing.AsyncIterator[Any]: 'AsyncIterator', @@ -9546,7 +10377,7 @@ def test_special_attrs(self): typing.ChainMap[Any, Any]: 'ChainMap', typing.Collection[Any]: 'Collection', typing.Container[Any]: 'Container', - typing.ContextManager[Any]: 'ContextManager', + typing.ContextManager[Any, Any]: 'ContextManager', typing.Coroutine[Any, Any, Any]: 'Coroutine', typing.Counter[Any]: 'Counter', typing.DefaultDict[Any, Any]: 'DefaultDict', @@ -9578,7 +10409,6 @@ def test_special_attrs(self): typing.ClassVar: 'ClassVar', typing.Concatenate: 'Concatenate', typing.Final: 'Final', - typing.ForwardRef: 'ForwardRef', typing.Literal: 'Literal', typing.NewType: 'NewType', typing.NoReturn: 'NoReturn', @@ -9586,10 +10416,11 @@ def test_special_attrs(self): typing.Optional: 'Optional', typing.TypeAlias: 'TypeAlias', typing.TypeGuard: 'TypeGuard', + typing.TypeIs: 'TypeIs', typing.TypeVar: 'TypeVar', typing.Union: 'Union', typing.Self: 'Self', - # Subscribed special forms + # Subscripted special forms typing.Annotated[Any, "Annotation"]: 'Annotated', typing.Annotated[int, 'Annotation']: 'Annotated', typing.ClassVar[Any]: 'ClassVar', @@ -9600,10 +10431,10 @@ def test_special_attrs(self): typing.Literal[True, 2]: 'Literal', typing.Optional[Any]: 'Optional', typing.TypeGuard[Any]: 'TypeGuard', + typing.TypeIs[Any]: 'TypeIs', typing.Union[Any]: 'Any', typing.Union[int, float]: 'Union', # Incompatible special forms (tested in test_special_attrs2) - # - typing.ForwardRef('set[Any]') # - typing.NewType('TypeName', Any) # - typing.ParamSpec('SpecialAttrsP') # - typing.TypeVar('T') @@ -9622,18 +10453,6 @@ def test_special_attrs(self): TypeName = typing.NewType('SpecialAttrsTests.TypeName', Any) def test_special_attrs2(self): - # Forward refs provide a different introspection API. __name__ and - # __qualname__ make little sense for forward refs as they can store - # complex typing expressions. - fr = typing.ForwardRef('set[Any]') - self.assertFalse(hasattr(fr, '__name__')) - self.assertFalse(hasattr(fr, '__qualname__')) - self.assertEqual(fr.__module__, 'typing') - # Forward refs are currently unpicklable. - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises(TypeError): - pickle.dumps(fr, proto) - self.assertEqual(SpecialAttrsTests.TypeName.__name__, 'TypeName') self.assertEqual( SpecialAttrsTests.TypeName.__qualname__, @@ -9654,7 +10473,7 @@ def test_special_attrs2(self): # to the variable name to which it is assigned". Thus, providing # __qualname__ is unnecessary. self.assertEqual(SpecialAttrsT.__name__, 'SpecialAttrsT') - self.assertFalse(hasattr(SpecialAttrsT, '__qualname__')) + self.assertNotHasAttr(SpecialAttrsT, '__qualname__') self.assertEqual(SpecialAttrsT.__module__, __name__) # Module-level type variables are picklable. for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -9663,7 +10482,7 @@ def test_special_attrs2(self): self.assertIs(SpecialAttrsT, loaded) self.assertEqual(SpecialAttrsP.__name__, 'SpecialAttrsP') - self.assertFalse(hasattr(SpecialAttrsP, '__qualname__')) + self.assertNotHasAttr(SpecialAttrsP, '__qualname__') self.assertEqual(SpecialAttrsP.__module__, __name__) # Module-level ParamSpecs are picklable. for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -9786,6 +10605,45 @@ class CustomerModel(ModelBase, init=False): self.assertIsInstance(CustomerModel, Decorated) +class NoDefaultTests(BaseTestCase): + def test_pickling(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + s = pickle.dumps(NoDefault, proto) + loaded = pickle.loads(s) + self.assertIs(NoDefault, loaded) + + def test_constructor(self): + self.assertIs(NoDefault, type(NoDefault)()) + with self.assertRaises(TypeError): + type(NoDefault)(1) + + def test_repr(self): + self.assertEqual(repr(NoDefault), 'typing.NoDefault') + + @requires_docstrings + def test_doc(self): + self.assertIsInstance(NoDefault.__doc__, str) + + def test_class(self): + self.assertIs(NoDefault.__class__, type(NoDefault)) + + def test_no_call(self): + with self.assertRaises(TypeError): + NoDefault() + + def test_no_attributes(self): + with self.assertRaises(AttributeError): + NoDefault.foo = 3 + with self.assertRaises(AttributeError): + NoDefault.foo + + # TypeError is consistent with the behavior of NoneType + with self.assertRaises(TypeError): + type(NoDefault).foo = 3 + with self.assertRaises(AttributeError): + type(NoDefault).foo + + class AllTests(BaseTestCase): """Tests for __all__.""" diff --git a/Lib/test/test_unicode_file_functions.py b/Lib/test/test_unicode_file_functions.py index 25c16e3a0b7e43..4a067d714e12e3 100644 --- a/Lib/test/test_unicode_file_functions.py +++ b/Lib/test/test_unicode_file_functions.py @@ -125,8 +125,8 @@ def test_open(self): # open(), os.stat(), etc. don't raise any exception. @unittest.skipIf(is_apple, 'irrelevant test on Apple platforms') @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "test fails on Emscripten/WASI when host platform is macOS." + support.is_wasi, + "test fails on WASI when host platform is macOS." ) def test_normalize(self): files = set(self.files) diff --git a/Lib/test/test_unicode_identifiers.py b/Lib/test/test_unicode_identifiers.py index 63c6c055824b20..3680072d643792 100644 --- a/Lib/test/test_unicode_identifiers.py +++ b/Lib/test/test_unicode_identifiers.py @@ -19,7 +19,7 @@ def test_non_bmp_normalized(self): def test_invalid(self): try: - from test.tokenizedata import badsyntax_3131 + from test.tokenizedata import badsyntax_3131 # noqa: F401 except SyntaxError as err: self.assertEqual(str(err), "invalid character '€' (U+20AC) (badsyntax_3131.py, line 2)") diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index d3bf4ea7c7d437..0285f0d51f2365 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -11,14 +11,20 @@ import sys import unicodedata import unittest -from test.support import (open_urlresource, requires_resource, script_helper, - cpython_only, check_disallow_instantiation) +from test.support import ( + open_urlresource, + requires_resource, + script_helper, + cpython_only, + check_disallow_instantiation, + force_not_colorized, +) class UnicodeMethodsTest(unittest.TestCase): # update this, if the database changes - expectedchecksum = '63aa77dcb36b0e1df082ee2a6071caeda7f0955e' + expectedchecksum = '9e43ee3929471739680c0e705482b4ae1c4122e4' @requires_resource('cpu') def test_method_checksum(self): @@ -71,7 +77,7 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest): # Update this if the database changes. Make sure to do a full rebuild # (e.g. 'make distclean && make') to get the correct checksum. - expectedchecksum = '232affd2a50ec4bd69d2482aa0291385cbdefaba' + expectedchecksum = '23ab09ed4abdf93db23b97359108ed630dd8311d' @requires_resource('cpu') def test_function_checksum(self): @@ -114,7 +120,7 @@ def test_no_names_in_pua(self): def test_lookup_nonexistant(self): # just make sure that lookup can fail - for nonexistant in [ + for nonexistent in [ "LATIN SMLL LETR A", "OPEN HANDS SIGHS", "DREGS", @@ -122,7 +128,7 @@ def test_lookup_nonexistant(self): "MODIFIER LETTER CYRILLIC SMALL QUESTION MARK", "???", ]: - self.assertRaises(KeyError, self.db.lookup, nonexistant) + self.assertRaises(KeyError, self.db.lookup, nonexistent) def test_digit(self): self.assertEqual(self.db.digit('A', None), None) @@ -277,6 +283,7 @@ def test_disallow_instantiation(self): # Ensure that the type disallows instantiation (bpo-43916) check_disallow_instantiation(self, unicodedata.UCD) + @force_not_colorized def test_failed_import_during_compiling(self): # Issue 4367 # Decoding \N escapes requires the unicodedata module. If it can't be diff --git a/Lib/test/test_importlib/resources/data01/subdirectory/__init__.py b/Lib/test/test_unittest/namespace_test_pkg/bar/__init__.py similarity index 100% rename from Lib/test/test_importlib/resources/data01/subdirectory/__init__.py rename to Lib/test/test_unittest/namespace_test_pkg/bar/__init__.py diff --git a/Lib/test/test_unittest/namespace_test_pkg/bar/test_bar.py b/Lib/test/test_unittest/namespace_test_pkg/bar/test_bar.py new file mode 100644 index 00000000000000..05b184d9eba685 --- /dev/null +++ b/Lib/test/test_unittest/namespace_test_pkg/bar/test_bar.py @@ -0,0 +1,5 @@ +import unittest + +class PassingTest(unittest.TestCase): + def test_true(self): + self.assertTrue(True) diff --git a/Lib/test/test_importlib/resources/data02/__init__.py b/Lib/test/test_unittest/namespace_test_pkg/noop/no2/__init__.py similarity index 100% rename from Lib/test/test_importlib/resources/data02/__init__.py rename to Lib/test/test_unittest/namespace_test_pkg/noop/no2/__init__.py diff --git a/Lib/test/test_unittest/namespace_test_pkg/noop/no2/test_no2.py b/Lib/test/test_unittest/namespace_test_pkg/noop/no2/test_no2.py new file mode 100644 index 00000000000000..05b184d9eba685 --- /dev/null +++ b/Lib/test/test_unittest/namespace_test_pkg/noop/no2/test_no2.py @@ -0,0 +1,5 @@ +import unittest + +class PassingTest(unittest.TestCase): + def test_true(self): + self.assertTrue(True) diff --git a/Lib/test/test_unittest/namespace_test_pkg/noop/test_noop.py b/Lib/test/test_unittest/namespace_test_pkg/noop/test_noop.py new file mode 100644 index 00000000000000..05b184d9eba685 --- /dev/null +++ b/Lib/test/test_unittest/namespace_test_pkg/noop/test_noop.py @@ -0,0 +1,5 @@ +import unittest + +class PassingTest(unittest.TestCase): + def test_true(self): + self.assertTrue(True) diff --git a/Lib/test/test_unittest/namespace_test_pkg/test_foo.py b/Lib/test/test_unittest/namespace_test_pkg/test_foo.py new file mode 100644 index 00000000000000..05b184d9eba685 --- /dev/null +++ b/Lib/test/test_unittest/namespace_test_pkg/test_foo.py @@ -0,0 +1,5 @@ +import unittest + +class PassingTest(unittest.TestCase): + def test_true(self): + self.assertTrue(True) diff --git a/Lib/test/test_unittest/test_assertions.py b/Lib/test/test_unittest/test_assertions.py index 5c1a28ecda5b49..1dec947ea76d23 100644 --- a/Lib/test/test_unittest/test_assertions.py +++ b/Lib/test/test_unittest/test_assertions.py @@ -386,6 +386,16 @@ def testAssertWarns(self): '^UserWarning not triggered$', '^UserWarning not triggered : oops$']) + def test_assertNotWarns(self): + def warn_future(): + warnings.warn('xyz', FutureWarning, stacklevel=2) + self.assertMessagesCM('_assertNotWarns', (FutureWarning,), + warn_future, + ['^FutureWarning triggered$', + '^oops$', + '^FutureWarning triggered$', + '^FutureWarning triggered : oops$']) + def testAssertWarnsRegex(self): # test error not raised self.assertMessagesCM('assertWarnsRegex', (UserWarning, 'unused regex'), diff --git a/Lib/test/test_unittest/test_async_case.py b/Lib/test/test_unittest/test_async_case.py index ba1ab838cd4a22..fc996b42149dcb 100644 --- a/Lib/test/test_unittest/test_async_case.py +++ b/Lib/test/test_unittest/test_async_case.py @@ -2,6 +2,7 @@ import contextvars import unittest from test import support +from test.support import force_not_colorized support.requires_working_socket(module=True) @@ -11,7 +12,7 @@ class MyException(Exception): def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class TestCM: @@ -252,6 +253,7 @@ async def on_cleanup(self): test.doCleanups() self.assertEqual(events, ['asyncSetUp', 'test', 'asyncTearDown', 'cleanup']) + @force_not_colorized def test_exception_in_tear_clean_up(self): class Test(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self): @@ -312,18 +314,21 @@ async def test3(self): self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test1', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'int'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Test('test2').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test2', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'async_generator'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Test('test3').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test3', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn(f'returned {Nothing.__name__!r}', str(w.warning)) def test_cleanups_interleave_order(self): events = [] @@ -471,11 +476,11 @@ async def cleanup(self, fut): def test_setup_get_event_loop(self): # See https://github.com/python/cpython/issues/95736 # Make sure the default event loop is not used - asyncio.set_event_loop(None) + asyncio._set_event_loop(None) class TestCase1(unittest.IsolatedAsyncioTestCase): def setUp(self): - asyncio.get_event_loop_policy().get_event_loop() + asyncio._get_event_loop_policy().get_event_loop() async def test_demo1(self): pass @@ -485,7 +490,7 @@ async def test_demo1(self): self.assertTrue(result.wasSuccessful()) def test_loop_factory(self): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class TestCase1(unittest.IsolatedAsyncioTestCase): loop_factory = asyncio.EventLoop diff --git a/Lib/test/test_unittest/test_case.py b/Lib/test/test_unittest/test_case.py index ed5eb5609a5dd1..a04af55f3fc0ae 100644 --- a/Lib/test/test_unittest/test_case.py +++ b/Lib/test/test_unittest/test_case.py @@ -10,6 +10,7 @@ import inspect import types +from collections import UserString from copy import deepcopy from test import support @@ -54,6 +55,10 @@ def tearDown(self): self.events.append('tearDown') +class List(list): + pass + + class Test_TestCase(unittest.TestCase, TestEquality, TestHashing): ### Set up attributes used by inherited tests @@ -85,7 +90,7 @@ class Test(unittest.TestCase): def runTest(self): raise MyException() def test(self): pass - self.assertEqual(Test().id()[-13:], '.Test.runTest') + self.assertEndsWith(Test().id(), '.Test.runTest') # test that TestCase can be instantiated with no args # primarily for use at the interactive interpreter @@ -106,7 +111,7 @@ class Test(unittest.TestCase): def runTest(self): raise MyException() def test(self): pass - self.assertEqual(Test('test').id()[-10:], '.Test.test') + self.assertEndsWith(Test('test').id(), '.Test.test') # "class TestCase([methodName])" # ... @@ -325,18 +330,40 @@ def test3(self): self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test1', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'int'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Foo('test2').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test2', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'generator'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Foo('test3').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test3', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn(f'returned {Nothing.__name__!r}', str(w.warning)) + + def test_deprecation_of_return_val_from_test_async_method(self): + class Foo(unittest.TestCase): + async def test1(self): + return 1 + + with self.assertWarns(DeprecationWarning) as w: + warnings.filterwarnings('ignore', + 'coroutine .* was never awaited', RuntimeWarning) + Foo('test1').run() + support.gc_collect() + self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) + self.assertIn('test1', str(w.warning)) + self.assertEqual(w.filename, __file__) + self.assertIn("returned 'coroutine'", str(w.warning)) + self.assertIn( + 'Maybe you forgot to use IsolatedAsyncioTestCase as the base class?', + str(w.warning), + ) def _check_call_order__subtests(self, result, events, expected_events): class Foo(Test.LoggingTestCase): @@ -678,16 +705,136 @@ def testAssertIsNot(self): self.assertRaises(self.failureException, self.assertIsNot, thing, thing) def testAssertIsInstance(self): - thing = [] + thing = List() self.assertIsInstance(thing, list) - self.assertRaises(self.failureException, self.assertIsInstance, - thing, dict) + self.assertIsInstance(thing, (int, list)) + with self.assertRaises(self.failureException) as cm: + self.assertIsInstance(thing, int) + self.assertEqual(str(cm.exception), + "[] is not an instance of ") + with self.assertRaises(self.failureException) as cm: + self.assertIsInstance(thing, (int, float)) + self.assertEqual(str(cm.exception), + "[] is not an instance of any of (, )") + + with self.assertRaises(self.failureException) as cm: + self.assertIsInstance(thing, int, 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertIsInstance(thing, int, msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) def testAssertNotIsInstance(self): - thing = [] - self.assertNotIsInstance(thing, dict) - self.assertRaises(self.failureException, self.assertNotIsInstance, - thing, list) + thing = List() + self.assertNotIsInstance(thing, int) + self.assertNotIsInstance(thing, (int, float)) + with self.assertRaises(self.failureException) as cm: + self.assertNotIsInstance(thing, list) + self.assertEqual(str(cm.exception), + "[] is an instance of ") + with self.assertRaises(self.failureException) as cm: + self.assertNotIsInstance(thing, (int, list)) + self.assertEqual(str(cm.exception), + "[] is an instance of ") + + with self.assertRaises(self.failureException) as cm: + self.assertNotIsInstance(thing, list, 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertNotIsInstance(thing, list, msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertIsSubclass(self): + self.assertIsSubclass(List, list) + self.assertIsSubclass(List, (int, list)) + with self.assertRaises(self.failureException) as cm: + self.assertIsSubclass(List, int) + self.assertEqual(str(cm.exception), + f"{List!r} is not a subclass of ") + with self.assertRaises(self.failureException) as cm: + self.assertIsSubclass(List, (int, float)) + self.assertEqual(str(cm.exception), + f"{List!r} is not a subclass of any of (, )") + with self.assertRaises(self.failureException) as cm: + self.assertIsSubclass(1, int) + self.assertEqual(str(cm.exception), "1 is not a class") + + with self.assertRaises(self.failureException) as cm: + self.assertIsSubclass(List, int, 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertIsSubclass(List, int, msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertNotIsSubclass(self): + self.assertNotIsSubclass(List, int) + self.assertNotIsSubclass(List, (int, float)) + with self.assertRaises(self.failureException) as cm: + self.assertNotIsSubclass(List, list) + self.assertEqual(str(cm.exception), + f"{List!r} is a subclass of ") + with self.assertRaises(self.failureException) as cm: + self.assertNotIsSubclass(List, (int, list)) + self.assertEqual(str(cm.exception), + f"{List!r} is a subclass of ") + with self.assertRaises(self.failureException) as cm: + self.assertNotIsSubclass(1, int) + self.assertEqual(str(cm.exception), "1 is not a class") + + with self.assertRaises(self.failureException) as cm: + self.assertNotIsSubclass(List, list, 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertNotIsSubclass(List, list, msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertHasAttr(self): + a = List() + a.x = 1 + self.assertHasAttr(a, 'x') + with self.assertRaises(self.failureException) as cm: + self.assertHasAttr(a, 'y') + self.assertEqual(str(cm.exception), + "'List' object has no attribute 'y'") + with self.assertRaises(self.failureException) as cm: + self.assertHasAttr(List, 'spam') + self.assertEqual(str(cm.exception), + "type object 'List' has no attribute 'spam'") + with self.assertRaises(self.failureException) as cm: + self.assertHasAttr(sys, 'nonexistent') + self.assertEqual(str(cm.exception), + "module 'sys' has no attribute 'nonexistent'") + + with self.assertRaises(self.failureException) as cm: + self.assertHasAttr(a, 'y', 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertHasAttr(a, 'y', msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertNotHasAttr(self): + a = List() + a.x = 1 + self.assertNotHasAttr(a, 'y') + with self.assertRaises(self.failureException) as cm: + self.assertNotHasAttr(a, 'x') + self.assertEqual(str(cm.exception), + "'List' object has unexpected attribute 'x'") + with self.assertRaises(self.failureException) as cm: + self.assertNotHasAttr(List, 'append') + self.assertEqual(str(cm.exception), + "type object 'List' has unexpected attribute 'append'") + with self.assertRaises(self.failureException) as cm: + self.assertNotHasAttr(sys, 'modules') + self.assertEqual(str(cm.exception), + "module 'sys' has unexpected attribute 'modules'") + + with self.assertRaises(self.failureException) as cm: + self.assertNotHasAttr(a, 'x', 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertNotHasAttr(a, 'x', msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) def testAssertIn(self): animals = {'monkey': 'banana', 'cow': 'grass', 'seal': 'fish'} @@ -1132,6 +1279,8 @@ def testAssertMultiLineEqual(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualSingleLine(self): sample_text = "laden swallows fly slowly" @@ -1148,6 +1297,8 @@ def testAssertEqualSingleLine(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualwithEmptyString(self): '''Verify when there is an empty string involved, the diff output @@ -1165,6 +1316,8 @@ def testAssertEqualwithEmptyString(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualMultipleLinesMissingNewlineTerminator(self): '''Verifying format of diff output from assertEqual involving strings @@ -1185,6 +1338,8 @@ def testAssertEqualMultipleLinesMissingNewlineTerminator(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualMultipleLinesMismatchedNewlinesTerminators(self): '''Verifying format of diff output from assertEqual involving strings @@ -1208,6 +1363,8 @@ def testAssertEqualMultipleLinesMismatchedNewlinesTerminators(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testEqualityBytesWarning(self): if sys.flags.bytes_warning: @@ -1832,6 +1989,186 @@ def testAssertNoLogsYieldsNone(self): pass self.assertIsNone(value) + def testAssertStartswith(self): + self.assertStartsWith('ababahalamaha', 'ababa') + self.assertStartsWith('ababahalamaha', ('x', 'ababa', 'y')) + self.assertStartsWith(UserString('ababahalamaha'), 'ababa') + self.assertStartsWith(UserString('ababahalamaha'), ('x', 'ababa', 'y')) + self.assertStartsWith(bytearray(b'ababahalamaha'), b'ababa') + self.assertStartsWith(bytearray(b'ababahalamaha'), (b'x', b'ababa', b'y')) + self.assertStartsWith(b'ababahalamaha', bytearray(b'ababa')) + self.assertStartsWith(b'ababahalamaha', + (bytearray(b'x'), bytearray(b'ababa'), bytearray(b'y'))) + + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', 'amaha') + self.assertEqual(str(cm.exception), + "'ababahalamaha' doesn't start with 'amaha'") + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', ('x', 'y')) + self.assertEqual(str(cm.exception), + "'ababahalamaha' doesn't start with any of ('x', 'y')") + + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith(b'ababahalamaha', 'ababa') + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith(b'ababahalamaha', ('amaha', 'ababa')) + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith([], 'ababa') + self.assertEqual(str(cm.exception), 'Expected str, not list') + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', b'ababa') + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', (b'amaha', b'ababa')) + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(TypeError): + self.assertStartsWith('ababahalamaha', ord('a')) + + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', 'amaha', 'abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', 'amaha', msg='abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertNotStartswith(self): + self.assertNotStartsWith('ababahalamaha', 'amaha') + self.assertNotStartsWith('ababahalamaha', ('x', 'amaha', 'y')) + self.assertNotStartsWith(UserString('ababahalamaha'), 'amaha') + self.assertNotStartsWith(UserString('ababahalamaha'), ('x', 'amaha', 'y')) + self.assertNotStartsWith(bytearray(b'ababahalamaha'), b'amaha') + self.assertNotStartsWith(bytearray(b'ababahalamaha'), (b'x', b'amaha', b'y')) + self.assertNotStartsWith(b'ababahalamaha', bytearray(b'amaha')) + self.assertNotStartsWith(b'ababahalamaha', + (bytearray(b'x'), bytearray(b'amaha'), bytearray(b'y'))) + + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', 'ababa') + self.assertEqual(str(cm.exception), + "'ababahalamaha' starts with 'ababa'") + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', ('x', 'ababa', 'y')) + self.assertEqual(str(cm.exception), + "'ababahalamaha' starts with 'ababa'") + + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith(b'ababahalamaha', 'ababa') + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith(b'ababahalamaha', ('amaha', 'ababa')) + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith([], 'ababa') + self.assertEqual(str(cm.exception), 'Expected str, not list') + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', b'ababa') + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', (b'amaha', b'ababa')) + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(TypeError): + self.assertNotStartsWith('ababahalamaha', ord('a')) + + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', 'ababa', 'abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', 'ababa', msg='abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertEndswith(self): + self.assertEndsWith('ababahalamaha', 'amaha') + self.assertEndsWith('ababahalamaha', ('x', 'amaha', 'y')) + self.assertEndsWith(UserString('ababahalamaha'), 'amaha') + self.assertEndsWith(UserString('ababahalamaha'), ('x', 'amaha', 'y')) + self.assertEndsWith(bytearray(b'ababahalamaha'), b'amaha') + self.assertEndsWith(bytearray(b'ababahalamaha'), (b'x', b'amaha', b'y')) + self.assertEndsWith(b'ababahalamaha', bytearray(b'amaha')) + self.assertEndsWith(b'ababahalamaha', + (bytearray(b'x'), bytearray(b'amaha'), bytearray(b'y'))) + + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', 'ababa') + self.assertEqual(str(cm.exception), + "'ababahalamaha' doesn't end with 'ababa'") + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', ('x', 'y')) + self.assertEqual(str(cm.exception), + "'ababahalamaha' doesn't end with any of ('x', 'y')") + + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith(b'ababahalamaha', 'amaha') + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith(b'ababahalamaha', ('ababa', 'amaha')) + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith([], 'amaha') + self.assertEqual(str(cm.exception), 'Expected str, not list') + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', b'amaha') + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', (b'ababa', b'amaha')) + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(TypeError): + self.assertEndsWith('ababahalamaha', ord('a')) + + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', 'ababa', 'abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', 'ababa', msg='abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertNotEndswith(self): + self.assertNotEndsWith('ababahalamaha', 'ababa') + self.assertNotEndsWith('ababahalamaha', ('x', 'ababa', 'y')) + self.assertNotEndsWith(UserString('ababahalamaha'), 'ababa') + self.assertNotEndsWith(UserString('ababahalamaha'), ('x', 'ababa', 'y')) + self.assertNotEndsWith(bytearray(b'ababahalamaha'), b'ababa') + self.assertNotEndsWith(bytearray(b'ababahalamaha'), (b'x', b'ababa', b'y')) + self.assertNotEndsWith(b'ababahalamaha', bytearray(b'ababa')) + self.assertNotEndsWith(b'ababahalamaha', + (bytearray(b'x'), bytearray(b'ababa'), bytearray(b'y'))) + + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', 'amaha') + self.assertEqual(str(cm.exception), + "'ababahalamaha' ends with 'amaha'") + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', ('x', 'amaha', 'y')) + self.assertEqual(str(cm.exception), + "'ababahalamaha' ends with 'amaha'") + + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith(b'ababahalamaha', 'amaha') + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith(b'ababahalamaha', ('ababa', 'amaha')) + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith([], 'amaha') + self.assertEqual(str(cm.exception), 'Expected str, not list') + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', b'amaha') + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', (b'ababa', b'amaha')) + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(TypeError): + self.assertNotEndsWith('ababahalamaha', ord('a')) + + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', 'amaha', 'abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', 'amaha', msg='abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + def testDeprecatedFailMethods(self): """Test that the deprecated fail* methods get removed in 3.12""" deprecated_names = [ diff --git a/Lib/test/test_unittest/test_discovery.py b/Lib/test/test_unittest/test_discovery.py index 004898ed431834..38c9779daaf87d 100644 --- a/Lib/test/test_unittest/test_discovery.py +++ b/Lib/test/test_unittest/test_discovery.py @@ -4,12 +4,14 @@ import sys import types import pickle +from importlib._bootstrap_external import NamespaceLoader from test import support from test.support import import_helper import unittest import unittest.mock import test.test_unittest +from test.test_importlib import util as test_util class TestableTestProgram(unittest.TestProgram): @@ -395,7 +397,7 @@ def restore_isdir(): self.addCleanup(restore_isdir) _find_tests_args = [] - def _find_tests(start_dir, pattern): + def _find_tests(start_dir, pattern, namespace=None): _find_tests_args.append((start_dir, pattern)) return ['tests'] loader._find_tests = _find_tests @@ -406,10 +408,34 @@ def _find_tests(start_dir, pattern): top_level_dir = os.path.abspath('/foo/bar') start_dir = os.path.abspath('/foo/bar/baz') self.assertEqual(suite, "['tests']") - self.assertEqual(loader._top_level_dir, top_level_dir) + self.assertEqual(loader._top_level_dir, os.path.abspath('/foo')) self.assertEqual(_find_tests_args, [(start_dir, 'pattern')]) self.assertIn(top_level_dir, sys.path) + def test_discover_should_not_persist_top_level_dir_between_calls(self): + original_isfile = os.path.isfile + original_isdir = os.path.isdir + original_sys_path = sys.path[:] + def restore(): + os.path.isfile = original_isfile + os.path.isdir = original_isdir + sys.path[:] = original_sys_path + self.addCleanup(restore) + + os.path.isfile = lambda path: True + os.path.isdir = lambda path: True + loader = unittest.TestLoader() + loader.suiteClass = str + dir = '/foo/bar' + top_level_dir = '/foo' + + loader.discover(dir, top_level_dir=top_level_dir) + self.assertEqual(loader._top_level_dir, None) + + loader._top_level_dir = dir2 = '/previous/dir' + loader.discover(dir, top_level_dir=top_level_dir) + self.assertEqual(loader._top_level_dir, dir2) + def test_discover_start_dir_is_package_calls_package_load_tests(self): # This test verifies that the package load_tests in a package is indeed # invoked when the start_dir is a package (and not the top level). @@ -791,7 +817,7 @@ def test_discovery_from_dotted_path(self): expectedPath = os.path.abspath(os.path.dirname(test.test_unittest.__file__)) self.wasRun = False - def _find_tests(start_dir, pattern): + def _find_tests(start_dir, pattern, namespace=None): self.wasRun = True self.assertEqual(start_dir, expectedPath) return tests @@ -824,6 +850,54 @@ def restore(): 'Can not use builtin modules ' 'as dotted module names') + def test_discovery_from_dotted_namespace_packages(self): + loader = unittest.TestLoader() + + package = types.ModuleType('package') + package.__name__ = "tests" + package.__path__ = ['/a', '/b'] + package.__file__ = None + package.__spec__ = types.SimpleNamespace( + name=package.__name__, + loader=NamespaceLoader(package.__name__, package.__path__, None), + submodule_search_locations=['/a', '/b'] + ) + + def _import(packagename, *args, **kwargs): + sys.modules[packagename] = package + return package + + _find_tests_args = [] + def _find_tests(start_dir, pattern, namespace=None): + _find_tests_args.append((start_dir, pattern)) + return ['%s/tests' % start_dir] + + loader._find_tests = _find_tests + loader.suiteClass = list + + with unittest.mock.patch('builtins.__import__', _import): + # Since loader.discover() can modify sys.path, restore it when done. + with import_helper.DirsOnSysPath(): + # Make sure to remove 'package' from sys.modules when done. + with test_util.uncache('package'): + suite = loader.discover('package') + + self.assertEqual(suite, ['/a/tests', '/b/tests']) + + def test_discovery_start_dir_is_namespace(self): + """Subdirectory discovery not affected if start_dir is a namespace pkg.""" + loader = unittest.TestLoader() + with ( + import_helper.DirsOnSysPath(os.path.join(os.path.dirname(__file__))), + test_util.uncache('namespace_test_pkg') + ): + suite = loader.discover('namespace_test_pkg') + self.assertEqual( + {list(suite)[0]._tests[0].__module__ for suite in suite._tests if list(suite)}, + # files under namespace_test_pkg.noop not discovered. + {'namespace_test_pkg.test_foo', 'namespace_test_pkg.bar.test_bar'}, + ) + def test_discovery_failed_discovery(self): from test.test_importlib import util diff --git a/Lib/test/test_unittest/test_loader.py b/Lib/test/test_unittest/test_loader.py index 83dd25ca54623f..cdff6d1a20c8df 100644 --- a/Lib/test/test_unittest/test_loader.py +++ b/Lib/test/test_unittest/test_loader.py @@ -76,7 +76,7 @@ def runTest(self): loader = unittest.TestLoader() # This has to be false for the test to succeed - self.assertFalse('runTest'.startswith(loader.testMethodPrefix)) + self.assertNotStartsWith('runTest', loader.testMethodPrefix) suite = loader.loadTestsFromTestCase(Foo) self.assertIsInstance(suite, loader.suiteClass) diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py index 7241cf59f73d4f..6092ed292d8f60 100644 --- a/Lib/test/test_unittest/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -7,6 +7,7 @@ from test.test_unittest.test_result import BufferedWriter +@support.force_not_colorized_test_class class Test_TestProgram(unittest.TestCase): def test_discovery_from_dotted_path(self): @@ -126,14 +127,14 @@ def test_NonExit(self): argv=["foobar"], testRunner=unittest.TextTestRunner(stream=stream), testLoader=self.TestLoader(self.FooBar)) - self.assertTrue(hasattr(program, 'result')) + self.assertHasAttr(program, 'result') out = stream.getvalue() self.assertIn('\nFAIL: testFail ', out) self.assertIn('\nERROR: testError ', out) self.assertIn('\nUNEXPECTED SUCCESS: testUnexpectedSuccess ', out) expected = ('\n\nFAILED (failures=1, errors=1, skipped=1, ' 'expected failures=1, unexpected successes=1)\n') - self.assertTrue(out.endswith(expected)) + self.assertEndsWith(out, expected) def test_Exit(self): stream = BufferedWriter() @@ -150,7 +151,7 @@ def test_Exit(self): self.assertIn('\nUNEXPECTED SUCCESS: testUnexpectedSuccess ', out) expected = ('\n\nFAILED (failures=1, errors=1, skipped=1, ' 'expected failures=1, unexpected successes=1)\n') - self.assertTrue(out.endswith(expected)) + self.assertEndsWith(out, expected) def test_ExitAsDefault(self): stream = BufferedWriter() @@ -165,7 +166,7 @@ def test_ExitAsDefault(self): self.assertIn('\nUNEXPECTED SUCCESS: testUnexpectedSuccess ', out) expected = ('\n\nFAILED (failures=1, errors=1, skipped=1, ' 'expected failures=1, unexpected successes=1)\n') - self.assertTrue(out.endswith(expected)) + self.assertEndsWith(out, expected) def test_ExitSkippedSuite(self): stream = BufferedWriter() @@ -177,7 +178,7 @@ def test_ExitSkippedSuite(self): self.assertEqual(cm.exception.code, 0) out = stream.getvalue() expected = '\n\nOK (skipped=1)\n' - self.assertTrue(out.endswith(expected)) + self.assertEndsWith(out, expected) def test_ExitEmptySuite(self): stream = BufferedWriter() diff --git a/Lib/test/test_unittest/test_result.py b/Lib/test/test_unittest/test_result.py index 15e3f62ef66a4b..9ac4c52449c2ff 100644 --- a/Lib/test/test_unittest/test_result.py +++ b/Lib/test/test_unittest/test_result.py @@ -1,12 +1,11 @@ import io import sys import textwrap - -from test.support import warnings_helper, captured_stdout - import traceback import unittest from unittest.util import strclass +from test.support import warnings_helper +from test.support import captured_stdout, force_not_colorized_test_class from test.test_unittest.support import BufferedWriter @@ -14,7 +13,7 @@ class MockTraceback(object): class TracebackException: def __init__(self, *args, **kwargs): self.capture_locals = kwargs.get('capture_locals', False) - def format(self): + def format(self, **kwargs): result = ['A traceback'] if self.capture_locals: result.append('locals') @@ -34,6 +33,7 @@ def bad_cleanup2(): raise ValueError('bad cleanup2') +@force_not_colorized_test_class class Test_TestResult(unittest.TestCase): # Note: there are not separate tests for TestResult.wasSuccessful(), # TestResult.errors, TestResult.failures, TestResult.testsRun or @@ -186,7 +186,7 @@ def test_1(self): test = Foo('test_1') try: test.fail("foo") - except: + except AssertionError: exc_info_tuple = sys.exc_info() result = unittest.TestResult() @@ -214,7 +214,7 @@ def test_1(self): def get_exc_info(): try: test.fail("foo") - except: + except AssertionError: return sys.exc_info() exc_info_tuple = get_exc_info() @@ -241,9 +241,9 @@ def get_exc_info(): try: try: test.fail("foo") - except: + except AssertionError: raise ValueError(42) - except: + except ValueError: return sys.exc_info() exc_info_tuple = get_exc_info() @@ -271,7 +271,7 @@ def get_exc_info(): loop.__cause__ = loop loop.__context__ = loop raise loop - except: + except Exception: return sys.exc_info() exc_info_tuple = get_exc_info() @@ -300,7 +300,7 @@ def get_exc_info(): ex1.__cause__ = ex2 ex2.__context__ = ex1 raise C - except: + except Exception: return sys.exc_info() exc_info_tuple = get_exc_info() @@ -345,7 +345,7 @@ def test_1(self): test = Foo('test_1') try: raise TypeError() - except: + except TypeError: exc_info_tuple = sys.exc_info() result = unittest.TestResult() @@ -454,9 +454,10 @@ def test(result): self.assertTrue(result.failfast) result = runner.run(test) stream.flush() - self.assertTrue(stream.getvalue().endswith('\n\nOK\n')) + self.assertEndsWith(stream.getvalue(), '\n\nOK\n') +@force_not_colorized_test_class class Test_TextTestResult(unittest.TestCase): maxDiff = None @@ -758,6 +759,7 @@ def testFoo(self): runner.run(Test('testFoo')) +@force_not_colorized_test_class class TestOutputBuffering(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_unittest/test_runner.py b/Lib/test/test_unittest/test_runner.py index 1b9cef43e3f9c5..4d3cfd60b8d9c3 100644 --- a/Lib/test/test_unittest/test_runner.py +++ b/Lib/test/test_unittest/test_runner.py @@ -106,6 +106,7 @@ def cleanup2(*args, **kwargs): self.assertTrue(test.doCleanups()) self.assertEqual(cleanups, [(2, (), {}), (1, (1, 2, 3), dict(four='hello', five='goodbye'))]) + @support.force_not_colorized def testCleanUpWithErrors(self): class TestableTest(unittest.TestCase): def testNothing(self): @@ -249,6 +250,7 @@ def testNothing(self): self.assertEqual(test._cleanups, []) +@support.force_not_colorized_test_class class TestClassCleanup(unittest.TestCase): def test_addClassCleanUp(self): class TestableTest(unittest.TestCase): @@ -601,6 +603,7 @@ class EmptyTest(unittest.TestCase): self.assertIn("\nNO TESTS RAN\n", runner.stream.getvalue()) +@support.force_not_colorized_test_class class TestModuleCleanUp(unittest.TestCase): def test_add_and_do_ModuleCleanup(self): module_cleanups = [] @@ -1318,6 +1321,7 @@ def MockResultClass(*args): expectedresult = (runner.stream, DESCRIPTIONS, VERBOSITY) self.assertEqual(runner._makeResult(), expectedresult) + @support.force_not_colorized @support.requires_subprocess() def test_warnings(self): """ diff --git a/Lib/test/test_unittest/test_skipping.py b/Lib/test/test_unittest/test_skipping.py index f146dcac18ecc0..f5cb860c60b156 100644 --- a/Lib/test/test_unittest/test_skipping.py +++ b/Lib/test/test_unittest/test_skipping.py @@ -1,5 +1,6 @@ import unittest +from test.support import force_not_colorized from test.test_unittest.support import LoggingResult @@ -293,6 +294,7 @@ def test_die(self): self.assertFalse(result.unexpectedSuccesses) self.assertTrue(result.wasSuccessful()) + @force_not_colorized def test_expected_failure_and_fail_in_cleanup(self): class Foo(unittest.TestCase): @unittest.expectedFailure @@ -372,6 +374,7 @@ def test_die(self): self.assertEqual(result.unexpectedSuccesses, [test]) self.assertFalse(result.wasSuccessful()) + @force_not_colorized def test_unexpected_success_and_fail_in_cleanup(self): class Foo(unittest.TestCase): @unittest.expectedFailure diff --git a/Lib/test/test_unittest/test_util.py b/Lib/test/test_unittest/test_util.py new file mode 100644 index 00000000000000..d590a333930278 --- /dev/null +++ b/Lib/test/test_unittest/test_util.py @@ -0,0 +1,33 @@ +import unittest +from unittest.util import safe_repr, sorted_list_difference, unorderable_list_difference + + +class TestUtil(unittest.TestCase): + def test_safe_repr(self): + class RaisingRepr: + def __repr__(self): + raise ValueError("Invalid repr()") + + class LongRepr: + def __repr__(self): + return 'x' * 100 + + safe_repr(RaisingRepr()) + self.assertEqual(safe_repr('foo'), "'foo'") + self.assertEqual(safe_repr(LongRepr(), short=True), 'x'*80 + ' [truncated]...') + + def test_sorted_list_difference(self): + self.assertEqual(sorted_list_difference([], []), ([], [])) + self.assertEqual(sorted_list_difference([1, 2], [2, 3]), ([1], [3])) + self.assertEqual(sorted_list_difference([1, 2], [1, 3]), ([2], [3])) + self.assertEqual(sorted_list_difference([1, 1, 1], [1, 2, 3]), ([], [2, 3])) + self.assertEqual(sorted_list_difference([4], [1, 2, 3, 4]), ([], [1, 2, 3])) + self.assertEqual(sorted_list_difference([1, 1], [2]), ([1], [2])) + self.assertEqual(sorted_list_difference([2], [1, 1]), ([2], [1])) + self.assertEqual(sorted_list_difference([1, 2], [1, 1]), ([2], [])) + + def test_unorderable_list_difference(self): + self.assertEqual(unorderable_list_difference([], []), ([], [])) + self.assertEqual(unorderable_list_difference([1, 2], []), ([2, 1], [])) + self.assertEqual(unorderable_list_difference([], [1, 2]), ([], [1, 2])) + self.assertEqual(unorderable_list_difference([1, 2], [1, 3]), ([2], [3])) diff --git a/Lib/test/test_unittest/testmock/support.py b/Lib/test/test_unittest/testmock/support.py index 49986d65dc47af..6c535b7944f261 100644 --- a/Lib/test/test_unittest/testmock/support.py +++ b/Lib/test/test_unittest/testmock/support.py @@ -14,3 +14,14 @@ def wibble(self): pass class X(object): pass + +# A standin for weurkzeug.local.LocalProxy - issue 119600 +def _inaccessible(*args, **kwargs): + raise AttributeError + + +class OpaqueProxy: + __getattribute__ = _inaccessible + + +g = OpaqueProxy() diff --git a/Lib/test/test_unittest/testmock/testasync.py b/Lib/test/test_unittest/testmock/testasync.py index f57b83f457f279..0791675b5401ca 100644 --- a/Lib/test/test_unittest/testmock/testasync.py +++ b/Lib/test/test_unittest/testmock/testasync.py @@ -8,14 +8,14 @@ support.requires_working_socket(module=True) -from asyncio import run, iscoroutinefunction +from asyncio import run from unittest import IsolatedAsyncioTestCase from unittest.mock import (ANY, call, AsyncMock, patch, MagicMock, Mock, create_autospec, sentinel, _CallList, seal) def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) class AsyncClass: @@ -60,7 +60,7 @@ class AsyncPatchDecoratorTest(unittest.TestCase): def test_is_coroutine_function_patch(self): @patch.object(AsyncClass, 'async_method') def test_async(mock_method): - self.assertTrue(iscoroutinefunction(mock_method)) + self.assertTrue(inspect.iscoroutinefunction(mock_method)) test_async() def test_is_async_patch(self): @@ -121,7 +121,7 @@ class AsyncPatchCMTest(unittest.TestCase): def test_is_async_function_cm(self): def test_async(): with patch.object(AsyncClass, 'async_method') as mock_method: - self.assertTrue(iscoroutinefunction(mock_method)) + self.assertTrue(inspect.iscoroutinefunction(mock_method)) test_async() @@ -155,7 +155,7 @@ def test_patch_dict_async_def(self): async def test_async(): self.assertEqual(foo['a'], 'b') - self.assertTrue(iscoroutinefunction(test_async)) + self.assertTrue(inspect.iscoroutinefunction(test_async)) run(test_async()) def test_patch_dict_async_def_context(self): @@ -170,12 +170,11 @@ async def test_async(): class AsyncMockTest(unittest.TestCase): def test_iscoroutinefunction_default(self): mock = AsyncMock() - self.assertTrue(iscoroutinefunction(mock)) + self.assertTrue(inspect.iscoroutinefunction(mock)) def test_iscoroutinefunction_function(self): async def foo(): pass mock = AsyncMock(foo) - self.assertTrue(iscoroutinefunction(mock)) self.assertTrue(inspect.iscoroutinefunction(mock)) def test_isawaitable(self): @@ -188,7 +187,6 @@ def test_isawaitable(self): def test_iscoroutinefunction_normal_function(self): def foo(): pass mock = AsyncMock(foo) - self.assertTrue(iscoroutinefunction(mock)) self.assertTrue(inspect.iscoroutinefunction(mock)) def test_future_isfuture(self): @@ -231,7 +229,6 @@ async def main(): run(main()) - self.assertTrue(iscoroutinefunction(spec)) self.assertTrue(inspect.iscoroutinefunction(spec)) self.assertTrue(asyncio.iscoroutine(awaitable)) self.assertTrue(inspect.iscoroutine(awaitable)) @@ -273,7 +270,6 @@ async def test_async(): awaitable = mock_method(1, 2, c=3) self.assertIsInstance(mock_method.mock, AsyncMock) - self.assertTrue(iscoroutinefunction(mock_method)) self.assertTrue(inspect.iscoroutinefunction(mock_method)) self.assertTrue(asyncio.iscoroutine(awaitable)) self.assertTrue(inspect.iscoroutine(awaitable)) @@ -430,13 +426,13 @@ def test_async(async_method): def test_is_async_AsyncMock(self): mock = AsyncMock(spec_set=AsyncClass.async_method) - self.assertTrue(iscoroutinefunction(mock)) + self.assertTrue(inspect.iscoroutinefunction(mock)) self.assertIsInstance(mock, AsyncMock) def test_is_child_AsyncMock(self): mock = MagicMock(spec_set=AsyncClass) - self.assertTrue(iscoroutinefunction(mock.async_method)) - self.assertFalse(iscoroutinefunction(mock.normal_method)) + self.assertTrue(inspect.iscoroutinefunction(mock.async_method)) + self.assertFalse(inspect.iscoroutinefunction(mock.normal_method)) self.assertIsInstance(mock.async_method, AsyncMock) self.assertIsInstance(mock.normal_method, MagicMock) self.assertIsInstance(mock, MagicMock) @@ -590,24 +586,24 @@ def test_sync_magic_methods_return_magic_mocks(self): def test_magicmock_has_async_magic_methods(self): m_mock = MagicMock() - self.assertTrue(hasattr(m_mock, "__aenter__")) - self.assertTrue(hasattr(m_mock, "__aexit__")) - self.assertTrue(hasattr(m_mock, "__anext__")) + self.assertHasAttr(m_mock, "__aenter__") + self.assertHasAttr(m_mock, "__aexit__") + self.assertHasAttr(m_mock, "__anext__") def test_asyncmock_has_sync_magic_methods(self): a_mock = AsyncMock() - self.assertTrue(hasattr(a_mock, "__enter__")) - self.assertTrue(hasattr(a_mock, "__exit__")) - self.assertTrue(hasattr(a_mock, "__next__")) - self.assertTrue(hasattr(a_mock, "__len__")) + self.assertHasAttr(a_mock, "__enter__") + self.assertHasAttr(a_mock, "__exit__") + self.assertHasAttr(a_mock, "__next__") + self.assertHasAttr(a_mock, "__len__") def test_magic_methods_are_async_functions(self): m_mock = MagicMock() self.assertIsInstance(m_mock.__aenter__, AsyncMock) self.assertIsInstance(m_mock.__aexit__, AsyncMock) # AsyncMocks are also coroutine functions - self.assertTrue(iscoroutinefunction(m_mock.__aenter__)) - self.assertTrue(iscoroutinefunction(m_mock.__aexit__)) + self.assertTrue(inspect.iscoroutinefunction(m_mock.__aenter__)) + self.assertTrue(inspect.iscoroutinefunction(m_mock.__aexit__)) class AsyncContextManagerTest(unittest.TestCase): @@ -746,11 +742,11 @@ def inner_test(mock_type): mock_instance = mock_type(instance) # Check that the mock and the real thing bahave the same # __aiter__ is not actually async, so not a coroutinefunction - self.assertFalse(iscoroutinefunction(instance.__aiter__)) - self.assertFalse(iscoroutinefunction(mock_instance.__aiter__)) + self.assertFalse(inspect.iscoroutinefunction(instance.__aiter__)) + self.assertFalse(inspect.iscoroutinefunction(mock_instance.__aiter__)) # __anext__ is async - self.assertTrue(iscoroutinefunction(instance.__anext__)) - self.assertTrue(iscoroutinefunction(mock_instance.__anext__)) + self.assertTrue(inspect.iscoroutinefunction(instance.__anext__)) + self.assertTrue(inspect.iscoroutinefunction(mock_instance.__anext__)) for mock_type in [AsyncMock, MagicMock]: with self.subTest(f"test aiter and anext corourtine with {mock_type}"): @@ -806,7 +802,7 @@ def test_assert_called_but_not_awaited(self): mock = AsyncMock(AsyncClass) with assertNeverAwaited(self): mock.async_method() - self.assertTrue(iscoroutinefunction(mock.async_method)) + self.assertTrue(inspect.iscoroutinefunction(mock.async_method)) mock.async_method.assert_called() mock.async_method.assert_called_once() mock.async_method.assert_called_once_with() diff --git a/Lib/test/test_unittest/testmock/testcallable.py b/Lib/test/test_unittest/testmock/testcallable.py index ca88511f63959d..03cb983e447c70 100644 --- a/Lib/test/test_unittest/testmock/testcallable.py +++ b/Lib/test/test_unittest/testmock/testcallable.py @@ -23,21 +23,21 @@ def assertNotCallable(self, mock): def test_non_callable(self): for mock in NonCallableMagicMock(), NonCallableMock(): self.assertRaises(TypeError, mock) - self.assertFalse(hasattr(mock, '__call__')) + self.assertNotHasAttr(mock, '__call__') self.assertIn(mock.__class__.__name__, repr(mock)) def test_hierarchy(self): - self.assertTrue(issubclass(MagicMock, Mock)) - self.assertTrue(issubclass(NonCallableMagicMock, NonCallableMock)) + self.assertIsSubclass(MagicMock, Mock) + self.assertIsSubclass(NonCallableMagicMock, NonCallableMock) def test_attributes(self): one = NonCallableMock() - self.assertTrue(issubclass(type(one.one), Mock)) + self.assertIsSubclass(type(one.one), Mock) two = NonCallableMagicMock() - self.assertTrue(issubclass(type(two.two), MagicMock)) + self.assertIsSubclass(type(two.two), MagicMock) def test_subclasses(self): @@ -45,13 +45,13 @@ class MockSub(Mock): pass one = MockSub() - self.assertTrue(issubclass(type(one.one), MockSub)) + self.assertIsSubclass(type(one.one), MockSub) class MagicSub(MagicMock): pass two = MagicSub() - self.assertTrue(issubclass(type(two.two), MagicSub)) + self.assertIsSubclass(type(two.two), MagicSub) def test_patch_spec(self): diff --git a/Lib/test/test_unittest/testmock/testhelpers.py b/Lib/test/test_unittest/testmock/testhelpers.py index 74785a83757a92..8d0f3ebc5cba88 100644 --- a/Lib/test/test_unittest/testmock/testhelpers.py +++ b/Lib/test/test_unittest/testmock/testhelpers.py @@ -8,8 +8,10 @@ Mock, ANY, _CallList, patch, PropertyMock, _callable ) +from dataclasses import dataclass, field, InitVar from datetime import datetime from functools import partial +from typing import ClassVar class SomeClass(object): def one(self, a, b): pass @@ -949,7 +951,7 @@ def __getattr__(self, attribute): proxy = Foo() autospec = create_autospec(proxy) - self.assertFalse(hasattr(autospec, '__name__')) + self.assertNotHasAttr(autospec, '__name__') def test_autospec_signature_staticmethod(self): @@ -1034,6 +1036,76 @@ def f(a): pass self.assertEqual(mock.mock_calls, []) self.assertEqual(rv.mock_calls, []) + def test_dataclass_post_init(self): + @dataclass + class WithPostInit: + a: int = field(init=False) + b: int = field(init=False) + def __post_init__(self): + self.a = 1 + self.b = 2 + + for mock in [ + create_autospec(WithPostInit, instance=True), + create_autospec(WithPostInit()), + ]: + with self.subTest(mock=mock): + self.assertIsInstance(mock.a, int) + self.assertIsInstance(mock.b, int) + + # Classes do not have these fields: + mock = create_autospec(WithPostInit) + msg = "Mock object has no attribute" + with self.assertRaisesRegex(AttributeError, msg): + mock.a + with self.assertRaisesRegex(AttributeError, msg): + mock.b + + def test_dataclass_default(self): + @dataclass + class WithDefault: + a: int + b: int = 0 + + for mock in [ + create_autospec(WithDefault, instance=True), + create_autospec(WithDefault(1)), + ]: + with self.subTest(mock=mock): + self.assertIsInstance(mock.a, int) + self.assertIsInstance(mock.b, int) + + def test_dataclass_with_method(self): + @dataclass + class WithMethod: + a: int + def b(self) -> int: + return 1 + + for mock in [ + create_autospec(WithMethod, instance=True), + create_autospec(WithMethod(1)), + ]: + with self.subTest(mock=mock): + self.assertIsInstance(mock.a, int) + mock.b.assert_not_called() + + def test_dataclass_with_non_fields(self): + @dataclass + class WithNonFields: + a: ClassVar[int] + b: InitVar[int] + + msg = "Mock object has no attribute" + for mock in [ + create_autospec(WithNonFields, instance=True), + create_autospec(WithNonFields(1)), + ]: + with self.subTest(mock=mock): + with self.assertRaisesRegex(AttributeError, msg): + mock.a + with self.assertRaisesRegex(AttributeError, msg): + mock.b class TestCallList(unittest.TestCase): @@ -1127,6 +1199,14 @@ def test_propertymock_side_effect(self): p.assert_called_once_with() + def test_propertymock_attach(self): + m = Mock() + p = PropertyMock() + type(m).foo = p + m.attach_mock(p, 'foo') + self.assertEqual(m.mock_calls, []) + + class TestCallablePredicate(unittest.TestCase): def test_type(self): diff --git a/Lib/test/test_unittest/testmock/testmagicmethods.py b/Lib/test/test_unittest/testmock/testmagicmethods.py index a4feae7e9d3b73..acdbd699d18134 100644 --- a/Lib/test/test_unittest/testmock/testmagicmethods.py +++ b/Lib/test/test_unittest/testmock/testmagicmethods.py @@ -1,7 +1,7 @@ import math import unittest import os -from asyncio import iscoroutinefunction +from inspect import iscoroutinefunction from unittest.mock import AsyncMock, Mock, MagicMock, _magics @@ -10,13 +10,13 @@ class TestMockingMagicMethods(unittest.TestCase): def test_deleting_magic_methods(self): mock = Mock() - self.assertFalse(hasattr(mock, '__getitem__')) + self.assertNotHasAttr(mock, '__getitem__') mock.__getitem__ = Mock() - self.assertTrue(hasattr(mock, '__getitem__')) + self.assertHasAttr(mock, '__getitem__') del mock.__getitem__ - self.assertFalse(hasattr(mock, '__getitem__')) + self.assertNotHasAttr(mock, '__getitem__') def test_magicmock_del(self): @@ -252,12 +252,12 @@ def test_magicmock(self): self.assertEqual(list(mock), [1, 2, 3]) getattr(mock, '__bool__').return_value = False - self.assertFalse(hasattr(mock, '__nonzero__')) + self.assertNotHasAttr(mock, '__nonzero__') self.assertFalse(bool(mock)) for entry in _magics: - self.assertTrue(hasattr(mock, entry)) - self.assertFalse(hasattr(mock, '__imaginary__')) + self.assertHasAttr(mock, entry) + self.assertNotHasAttr(mock, '__imaginary__') def test_magic_mock_equality(self): @@ -331,6 +331,45 @@ def test_magic_methods_fspath(self): self.assertEqual(os.fspath(mock), expected_path) mock.__fspath__.assert_called_once() + def test_magic_mock_does_not_reset_magic_returns(self): + # https://github.com/python/cpython/issues/123934 + for reset in (True, False): + with self.subTest(reset=reset): + mm = MagicMock() + self.assertIs(type(mm.__str__()), str) + mm.__str__.assert_called_once() + + self.assertIs(type(mm.__hash__()), int) + mm.__hash__.assert_called_once() + + for _ in range(3): + # Repeat reset several times to be sure: + mm.reset_mock(return_value=reset) + + self.assertIs(type(mm.__str__()), str) + mm.__str__.assert_called_once() + + self.assertIs(type(mm.__hash__()), int) + mm.__hash__.assert_called_once() + + def test_magic_mock_resets_manual_mocks(self): + mm = MagicMock() + mm.__iter__ = MagicMock(return_value=iter([1])) + mm.custom = MagicMock(return_value=2) + self.assertEqual(list(iter(mm)), [1]) + self.assertEqual(mm.custom(), 2) + + mm.reset_mock(return_value=True) + self.assertEqual(list(iter(mm)), []) + self.assertIsInstance(mm.custom(), MagicMock) + + def test_magic_mock_resets_manual_mocks_empty_iter(self): + mm = MagicMock() + mm.__iter__.return_value = [] + self.assertEqual(list(iter(mm)), []) + + mm.reset_mock(return_value=True) + self.assertEqual(list(iter(mm)), []) def test_magic_methods_and_spec(self): class Iterable(object): diff --git a/Lib/test/test_unittest/testmock/testmock.py b/Lib/test/test_unittest/testmock/testmock.py index 1725406bcfb9e4..5d1bf4258afacd 100644 --- a/Lib/test/test_unittest/testmock/testmock.py +++ b/Lib/test/test_unittest/testmock/testmock.py @@ -115,6 +115,24 @@ def f(): pass with self.assertRaises(TypeError): mock() + def test_create_autospec_should_be_configurable_by_kwargs(self): + """If kwargs are given to configure mock, the function must configure + the parent mock during initialization.""" + mocked_result = 'mocked value' + class_mock = create_autospec(spec=Something, **{ + 'return_value.meth.side_effect': [ValueError, DEFAULT], + 'return_value.meth.return_value': mocked_result}) + with self.assertRaises(ValueError): + class_mock().meth(a=None, b=None, c=None) + self.assertEqual(class_mock().meth(a=None, b=None, c=None), mocked_result) + # Only the parent mock should be configurable because the user will + # pass kwargs with respect to the parent mock. + self.assertEqual(class_mock().return_value.meth.side_effect, None) + + def test_create_autospec_correctly_handles_name(self): + class X: ... + mock = create_autospec(X, spec_set=True, name="Y") + self.assertEqual(mock._mock_name, "Y") def test_repr(self): mock = Mock(name='foo') @@ -245,6 +263,65 @@ class B(object): with mock.patch('builtins.open', mock.mock_open()): mock.mock_open() # should still be valid with open() mocked + def test_create_autospec_wraps_class(self): + """Autospec a class with wraps & test if the call is passed to the + wrapped object.""" + result = "real result" + + class Result: + def get_result(self): + return result + class_mock = create_autospec(spec=Result, wraps=Result) + # Have to reassign the return_value to DEFAULT to return the real + # result (actual instance of "Result") when the mock is called. + class_mock.return_value = mock.DEFAULT + self.assertEqual(class_mock().get_result(), result) + # Autospec should also wrap child attributes of parent. + self.assertEqual(class_mock.get_result._mock_wraps, Result.get_result) + + def test_create_autospec_instance_wraps_class(self): + """Autospec a class instance with wraps & test if the call is passed + to the wrapped object.""" + result = "real result" + + class Result: + @staticmethod + def get_result(): + """This is a static method because when the mocked instance of + 'Result' will call this method, it won't be able to consume + 'self' argument.""" + return result + instance_mock = create_autospec(spec=Result, instance=True, wraps=Result) + # Have to reassign the return_value to DEFAULT to return the real + # result from "Result.get_result" when the mocked instance of "Result" + # calls "get_result". + instance_mock.get_result.return_value = mock.DEFAULT + self.assertEqual(instance_mock.get_result(), result) + # Autospec should also wrap child attributes of the instance. + self.assertEqual(instance_mock.get_result._mock_wraps, Result.get_result) + + def test_create_autospec_wraps_function_type(self): + """Autospec a function or a method with wraps & test if the call is + passed to the wrapped object.""" + result = "real result" + + class Result: + def get_result(self): + return result + func_mock = create_autospec(spec=Result.get_result, wraps=Result.get_result) + self.assertEqual(func_mock(Result()), result) + + def test_explicit_return_value_even_if_mock_wraps_object(self): + """If the mock has an explicit return_value set then calls are not + passed to the wrapped object and the return_value is returned instead. + """ + def my_func(): + return None + func_mock = create_autospec(spec=my_func, wraps=my_func) + return_value = "explicit return value" + func_mock.return_value = return_value + self.assertEqual(func_mock(), return_value) + def test_explicit_parent(self): parent = Mock() mock1 = Mock(parent=parent, return_value=None) @@ -622,6 +699,14 @@ def test_wraps_calls(self): real = Mock() mock = Mock(wraps=real) + # If "Mock" wraps an object, just accessing its + # "return_value" ("NonCallableMock.__get_return_value") should not + # trigger its descriptor ("NonCallableMock.__set_return_value") so + # the default "return_value" should always be "sentinel.DEFAULT". + self.assertEqual(mock.return_value, DEFAULT) + # It will not be "sentinel.DEFAULT" if the mock is not wrapping any + # object. + self.assertNotEqual(real.return_value, DEFAULT) self.assertEqual(mock(), real()) real.reset_mock() @@ -2130,13 +2215,13 @@ def test_attach_mock_patch_autospec_signature(self): def test_attribute_deletion(self): for mock in (Mock(), MagicMock(), NonCallableMagicMock(), NonCallableMock()): - self.assertTrue(hasattr(mock, 'm')) + self.assertHasAttr(mock, 'm') del mock.m - self.assertFalse(hasattr(mock, 'm')) + self.assertNotHasAttr(mock, 'm') del mock.f - self.assertFalse(hasattr(mock, 'f')) + self.assertNotHasAttr(mock, 'f') self.assertRaises(AttributeError, getattr, mock, 'f') @@ -2145,18 +2230,18 @@ def test_mock_does_not_raise_on_repeated_attribute_deletion(self): for mock in (Mock(), MagicMock(), NonCallableMagicMock(), NonCallableMock()): mock.foo = 3 - self.assertTrue(hasattr(mock, 'foo')) + self.assertHasAttr(mock, 'foo') self.assertEqual(mock.foo, 3) del mock.foo - self.assertFalse(hasattr(mock, 'foo')) + self.assertNotHasAttr(mock, 'foo') mock.foo = 4 - self.assertTrue(hasattr(mock, 'foo')) + self.assertHasAttr(mock, 'foo') self.assertEqual(mock.foo, 4) del mock.foo - self.assertFalse(hasattr(mock, 'foo')) + self.assertNotHasAttr(mock, 'foo') def test_mock_raises_when_deleting_nonexistent_attribute(self): @@ -2174,7 +2259,7 @@ def test_reset_mock_does_not_raise_on_attr_deletion(self): mock.child = True del mock.child mock.reset_mock() - self.assertFalse(hasattr(mock, 'child')) + self.assertNotHasAttr(mock, 'child') def test_class_assignable(self): diff --git a/Lib/test/test_unittest/testmock/testpatch.py b/Lib/test/test_unittest/testmock/testpatch.py index d0046d702a53f4..7c5fc3deed2ca2 100644 --- a/Lib/test/test_unittest/testmock/testpatch.py +++ b/Lib/test/test_unittest/testmock/testpatch.py @@ -7,9 +7,11 @@ from collections import OrderedDict import unittest +import test from test.test_unittest.testmock import support from test.test_unittest.testmock.support import SomeClass, is_instance +from test.support.import_helper import DirsOnSysPath from test.test_importlib.util import uncache from unittest.mock import ( NonCallableMock, CallableMixin, sentinel, @@ -364,7 +366,7 @@ def test(): self.assertEqual(SomeClass.frooble, sentinel.Frooble) test() - self.assertFalse(hasattr(SomeClass, 'frooble')) + self.assertNotHasAttr(SomeClass, 'frooble') def test_patch_wont_create_by_default(self): @@ -381,7 +383,7 @@ def test_patchobject_wont_create_by_default(self): @patch.object(SomeClass, 'ord', sentinel.Frooble) def test(): pass test() - self.assertFalse(hasattr(SomeClass, 'ord')) + self.assertNotHasAttr(SomeClass, 'ord') def test_patch_builtins_without_create(self): @@ -743,6 +745,54 @@ def test_stop_idempotent(self): self.assertIsNone(patcher.stop()) + def test_exit_idempotent(self): + patcher = patch(foo_name, 'bar', 3) + with patcher: + patcher.stop() + + + def test_second_start_failure(self): + patcher = patch(foo_name, 'bar', 3) + patcher.start() + try: + self.assertRaises(RuntimeError, patcher.start) + finally: + patcher.stop() + + + def test_second_enter_failure(self): + patcher = patch(foo_name, 'bar', 3) + with patcher: + self.assertRaises(RuntimeError, patcher.start) + + + def test_second_start_after_stop(self): + patcher = patch(foo_name, 'bar', 3) + patcher.start() + patcher.stop() + patcher.start() + patcher.stop() + + + def test_property_setters(self): + mock_object = Mock() + mock_bar = mock_object.bar + patcher = patch.object(mock_object, 'bar', 'x') + with patcher: + self.assertEqual(patcher.is_local, False) + self.assertIs(patcher.target, mock_object) + self.assertEqual(patcher.temp_original, mock_bar) + patcher.is_local = True + patcher.target = mock_bar + patcher.temp_original = mock_object + self.assertEqual(patcher.is_local, True) + self.assertIs(patcher.target, mock_bar) + self.assertEqual(patcher.temp_original, mock_object) + # if changes are left intact, they may lead to disruption as shown below (it might be what someone needs though) + self.assertEqual(mock_bar.bar, mock_object) + self.assertEqual(mock_object.bar, 'x') + + def test_patchobject_start_stop(self): original = something patcher = patch.object(PTModule, 'something', 'foo') @@ -1096,7 +1146,7 @@ def test_new_callable_patch(self): self.assertIsNot(m1, m2) for mock in m1, m2: - self.assertNotCallable(m1) + self.assertNotCallable(mock) def test_new_callable_patch_object(self): @@ -1109,7 +1159,7 @@ def test_new_callable_patch_object(self): self.assertIsNot(m1, m2) for mock in m1, m2: - self.assertNotCallable(m1) + self.assertNotCallable(mock) def test_new_callable_keyword_arguments(self): @@ -1427,7 +1477,7 @@ def test_patch_multiple_create(self): finally: patcher.stop() - self.assertFalse(hasattr(Foo, 'blam')) + self.assertNotHasAttr(Foo, 'blam') def test_patch_multiple_spec_set(self): @@ -1728,6 +1778,71 @@ def test(mock): 'exception traceback not propagated') + def test_name_resolution_import_rebinding(self): + # Currently mock.patch uses pkgutil.resolve_name(), but repeat + # similar tests just for the case. + # The same data is also used for testing import in test_import and + # pkgutil.resolve_name() in test_pkgutil. + path = os.path.join(os.path.dirname(test.__file__), 'test_import', 'data') + def check(name): + p = patch(name) + p.start() + p.stop() + def check_error(name): + p = patch(name) + self.assertRaises(AttributeError, p.start) + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3.submodule.A.attr') + check_error('package3.submodule.B.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3.submodule:A.attr') + check_error('package3.submodule:B.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + check('package3.submodule.A.attr') + check_error('package3.submodule.B.attr') + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + check('package3.submodule:A.attr') + check_error('package3.submodule:B.attr') + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + + def test_name_resolution_import_rebinding2(self): + path = os.path.join(os.path.dirname(test.__file__), 'test_import', 'data') + def check(name): + p = patch(name) + p.start() + p.stop() + def check_error(name): + p = patch(name) + self.assertRaises(AttributeError, p.start) + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4.submodule.A.attr') + check_error('package4.submodule.B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4.submodule:A.attr') + check_error('package4.submodule:B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4:submodule.B.attr') + check_error('package4:submodule.A.attr') + check('package4.submodule.A.attr') + check_error('package4.submodule.B.attr') + check('package4:submodule.A.attr') + check_error('package4:submodule.B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4:submodule.B.attr') + check_error('package4:submodule.A.attr') + check('package4.submodule:A.attr') + check_error('package4.submodule:B.attr') + check('package4:submodule.A.attr') + check_error('package4:submodule.B.attr') + + def test_create_and_specs(self): for kwarg in ('spec', 'spec_set', 'autospec'): p = patch('%s.doesnotexist' % __name__, create=True, @@ -1978,6 +2093,13 @@ def test(): pass with self.assertRaises(TypeError): test() + def test_patch_proxy_object(self): + @patch("test.test_unittest.testmock.support.g", new_callable=MagicMock()) + def test(_): + pass + + test() + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_unpack.py b/Lib/test/test_unpack.py index 515ec128a08a9c..adb30c0febbb6f 100644 --- a/Lib/test/test_unpack.py +++ b/Lib/test/test_unpack.py @@ -18,6 +18,13 @@ >>> a == 4 and b == 5 and c == 6 True +Unpack dict + + >>> d = {4: 'four', 5: 'five', 6: 'six'} + >>> a, b, c = d + >>> a == 4 and b == 5 and c == 6 + True + Unpack implied tuple >>> a, b, c = 7, 8, 9 @@ -66,14 +73,14 @@ >>> a, b = t Traceback (most recent call last): ... - ValueError: too many values to unpack (expected 2) + ValueError: too many values to unpack (expected 2, got 3) Unpacking tuple of wrong size >>> a, b = l Traceback (most recent call last): ... - ValueError: too many values to unpack (expected 2) + ValueError: too many values to unpack (expected 2, got 3) Unpacking sequence too short @@ -140,8 +147,52 @@ >>> () = [42] Traceback (most recent call last): ... - ValueError: too many values to unpack (expected 0) + ValueError: too many values to unpack (expected 0, got 1) + +Unpacking a larger iterable should raise ValuleError, but it +should not entirely consume the iterable + >>> it = iter(range(100)) + >>> x, y, z = it + Traceback (most recent call last): + ... + ValueError: too many values to unpack (expected 3) + >>> next(it) + 4 + +Unpacking unbalanced dict + + >>> d = {4: 'four', 5: 'five', 6: 'six', 7: 'seven'} + >>> a, b, c = d + Traceback (most recent call last): + ... + ValueError: too many values to unpack (expected 3, got 4) + +Ensure that custom `__len__()` is NOT called when showing the error message + + >>> class LengthTooLong: + ... def __len__(self): + ... return 5 + ... def __getitem__(self, i): + ... return i*2 + ... + >>> x, y, z = LengthTooLong() + Traceback (most recent call last): + ... + ValueError: too many values to unpack (expected 3) + +For evil cases like these as well, no actual count to be shown + + >>> class BadLength: + ... def __len__(self): + ... return 1 + ... def __getitem__(self, i): + ... return i*2 + ... + >>> x, y, z = BadLength() + Traceback (most recent call last): + ... + ValueError: too many values to unpack (expected 3) """ __test__ = {'doctests' : doctests} diff --git a/Lib/test/test_unpack_ex.py b/Lib/test/test_unpack_ex.py index c201d08f61b8cd..9e2d54bd3a8c4e 100644 --- a/Lib/test/test_unpack_ex.py +++ b/Lib/test/test_unpack_ex.py @@ -26,6 +26,12 @@ >>> a == [7, 8, 9] True +Unpack nested implied tuple + + >>> [*[*a]] = [[7,8,9]] + >>> a == [[7,8,9]] + True + Unpack string... fun! >>> a, *b = 'one' diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index bb15f64c59dbd1..332919540da4d6 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -651,7 +651,9 @@ def test_multiquote_joined_string(self): def test_backslash_in_format_spec(self): import re - msg = re.escape("invalid escape sequence '\\ '") + msg = re.escape('"\\ " is an invalid escape sequence. ' + 'Such sequences will not work in the future. ' + 'Did you mean "\\\\ "? A raw string is also an option.') with self.assertWarnsRegex(SyntaxWarning, msg): self.check_ast_roundtrip("""f"{x:\\ }" """) self.check_ast_roundtrip("""f"{x:\\n}" """) @@ -673,6 +675,20 @@ def test_quote_in_format_spec(self): self.check_ast_roundtrip("""f'\\'{x:\\"}' """) self.check_ast_roundtrip("""f'\\'{x:\\\\"}' """) + def test_type_params(self): + self.check_ast_roundtrip("type A = int") + self.check_ast_roundtrip("type A[T] = int") + self.check_ast_roundtrip("type A[T: int] = int") + self.check_ast_roundtrip("type A[T = int] = int") + self.check_ast_roundtrip("type A[T: int = int] = int") + self.check_ast_roundtrip("type A[**P] = int") + self.check_ast_roundtrip("type A[**P = int] = int") + self.check_ast_roundtrip("type A[*Ts] = int") + self.check_ast_roundtrip("type A[*Ts = int] = int") + self.check_ast_roundtrip("type A[*Ts = *int] = int") + self.check_ast_roundtrip("def f[T: int = int, **P = int, *Ts = *int]():\n pass") + self.check_ast_roundtrip("class C[T: int = int, **P = int, *Ts = *int]():\n pass") + class ManualASTCreationTestCase(unittest.TestCase): """Test that AST nodes created without a type_params field unparse correctly.""" @@ -723,6 +739,20 @@ def test_function_with_type_params_and_bound(self): ast.fix_missing_locations(node) self.assertEqual(ast.unparse(node), "def f[T: int]():\n pass") + def test_function_with_type_params_and_default(self): + node = ast.FunctionDef( + name="f", + args=ast.arguments(), + body=[ast.Pass()], + type_params=[ + ast.TypeVar("T", default_value=ast.Constant(value=1)), + ast.TypeVarTuple("Ts", default_value=ast.Starred(value=ast.Constant(value=1), ctx=ast.Load())), + ast.ParamSpec("P", default_value=ast.Constant(value=1)), + ], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "def f[T = 1, *Ts = *1, **P = 1]():\n pass") + def test_async_function(self): node = ast.AsyncFunctionDef( name="f", @@ -746,6 +776,20 @@ def test_async_function_with_type_params(self): ast.fix_missing_locations(node) self.assertEqual(ast.unparse(node), "async def f[T]():\n pass") + def test_async_function_with_type_params_and_default(self): + node = ast.AsyncFunctionDef( + name="f", + args=ast.arguments(), + body=[ast.Pass()], + type_params=[ + ast.TypeVar("T", default_value=ast.Constant(value=1)), + ast.TypeVarTuple("Ts", default_value=ast.Starred(value=ast.Constant(value=1), ctx=ast.Load())), + ast.ParamSpec("P", default_value=ast.Constant(value=1)), + ], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "async def f[T = 1, *Ts = *1, **P = 1]():\n pass") + class DirectoryTestCase(ASTTestCase): """Test roundtrip behaviour on all files in Lib and Lib/test.""" diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 6bb0fb303fa202..4842428d6fd103 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -7,11 +7,9 @@ import email.message import io import unittest -from unittest.mock import patch from test import support from test.support import os_helper from test.support import socket_helper -from test.support import warnings_helper import os try: import ssl @@ -19,9 +17,7 @@ ssl = None import sys import tempfile -from nturl2path import url2pathname, pathname2url -from base64 import b64encode import collections @@ -36,32 +32,6 @@ def hexescape(char): hex_repr = "0%s" % hex_repr return "%" + hex_repr -# Shortcut for testing FancyURLopener -_urlopener = None - - -def urlopen(url, data=None, proxies=None): - """urlopen(url [, data]) -> open file-like object""" - global _urlopener - if proxies is not None: - opener = urllib.request.FancyURLopener(proxies=proxies) - elif not _urlopener: - opener = FancyURLopener() - _urlopener = opener - else: - opener = _urlopener - if data is None: - return opener.open(url) - else: - return opener.open(url, data) - - -def FancyURLopener(): - with warnings_helper.check_warnings( - ('FancyURLopener style of invoking requests is deprecated.', - DeprecationWarning)): - return urllib.request.FancyURLopener() - def fakehttp(fakedata, mock_close=False): class FakeSocket(io.BytesIO): @@ -120,26 +90,6 @@ def unfakehttp(self): http.client.HTTPConnection = self._connection_class -class FakeFTPMixin(object): - def fakeftp(self): - class FakeFtpWrapper(object): - def __init__(self, user, passwd, host, port, dirs, timeout=None, - persistent=True): - pass - - def retrfile(self, file, type): - return io.BytesIO(), 0 - - def close(self): - pass - - self._ftpwrapper_class = urllib.request.ftpwrapper - urllib.request.ftpwrapper = FakeFtpWrapper - - def unfakeftp(self): - urllib.request.ftpwrapper = self._ftpwrapper_class - - class urlopen_FileTests(unittest.TestCase): """Test urlopen() opening a temporary file. @@ -159,7 +109,7 @@ def setUp(self): f.close() self.pathname = os_helper.TESTFN self.quoted_pathname = urllib.parse.quote(self.pathname) - self.returned_obj = urlopen("file:%s" % self.quoted_pathname) + self.returned_obj = urllib.request.urlopen("file:%s" % self.quoted_pathname) def tearDown(self): """Shut down the open object""" @@ -206,7 +156,7 @@ def test_headers(self): self.assertIsInstance(self.returned_obj.headers, email.message.Message) def test_url(self): - self.assertEqual(self.returned_obj.url, self.quoted_pathname) + self.assertEqual(self.returned_obj.url, "file:" + self.quoted_pathname) def test_status(self): self.assertIsNone(self.returned_obj.status) @@ -215,7 +165,7 @@ def test_info(self): self.assertIsInstance(self.returned_obj.info(), email.message.Message) def test_geturl(self): - self.assertEqual(self.returned_obj.geturl(), self.quoted_pathname) + self.assertEqual(self.returned_obj.geturl(), "file:" + self.quoted_pathname) def test_getcode(self): self.assertIsNone(self.returned_obj.getcode()) @@ -340,13 +290,13 @@ def test_getproxies_environment_prefer_lowercase(self): self.assertEqual('http://somewhere:3128', proxies['http']) -class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin, FakeFTPMixin): +class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin): """Test urlopen() opening a fake http connection.""" def check_read(self, ver): self.fakehttp(b"HTTP/" + ver + b" 200 OK\r\n\r\nHello!") try: - fp = urlopen("http://python.org/") + fp = urllib.request.urlopen("http://python.org/") self.assertEqual(fp.readline(), b"Hello!") self.assertEqual(fp.readline(), b"") self.assertEqual(fp.geturl(), 'http://python.org/') @@ -367,8 +317,8 @@ def test_url_fragment(self): def test_willclose(self): self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello!") try: - resp = urlopen("http://www.python.org") - self.assertTrue(resp.fp.will_close) + resp = urllib.request.urlopen("http://www.python.org") + self.assertTrue(resp.will_close) finally: self.unfakehttp() @@ -393,9 +343,6 @@ def test_url_path_with_control_char_rejected(self): with self.assertRaisesRegex( InvalidURL, f"contain control.*{escaped_char_repr}"): urllib.request.urlopen(f"https:{schemeless_url}") - # This code path quotes the URL so there is no injection. - resp = urlopen(f"http:{schemeless_url}") - self.assertNotIn(char, resp.geturl()) finally: self.unfakehttp() @@ -417,11 +364,6 @@ def test_url_path_with_newline_header_injection_rejected(self): urllib.request.urlopen(f"http:{schemeless_url}") with self.assertRaisesRegex(InvalidURL, r"contain control.*\\n"): urllib.request.urlopen(f"https:{schemeless_url}") - # This code path quotes the URL so there is no injection. - resp = urlopen(f"http:{schemeless_url}") - self.assertNotIn(' ', resp.geturl()) - self.assertNotIn('\r', resp.geturl()) - self.assertNotIn('\n', resp.geturl()) finally: self.unfakehttp() @@ -436,9 +378,9 @@ def test_url_host_with_control_char_rejected(self): InvalidURL = http.client.InvalidURL with self.assertRaisesRegex( InvalidURL, f"contain control.*{escaped_char_repr}"): - urlopen(f"http:{schemeless_url}") + urllib.request.urlopen(f"http:{schemeless_url}") with self.assertRaisesRegex(InvalidURL, f"contain control.*{escaped_char_repr}"): - urlopen(f"https:{schemeless_url}") + urllib.request.urlopen(f"https:{schemeless_url}") finally: self.unfakehttp() @@ -451,9 +393,9 @@ def test_url_host_with_newline_header_injection_rejected(self): InvalidURL = http.client.InvalidURL with self.assertRaisesRegex( InvalidURL, r"contain control.*\\r"): - urlopen(f"http:{schemeless_url}") + urllib.request.urlopen(f"http:{schemeless_url}") with self.assertRaisesRegex(InvalidURL, r"contain control.*\\n"): - urlopen(f"https:{schemeless_url}") + urllib.request.urlopen(f"https:{schemeless_url}") finally: self.unfakehttp() @@ -477,7 +419,9 @@ def test_read_bogus(self): Content-Type: text/html; charset=iso-8859-1 ''', mock_close=True) try: - self.assertRaises(OSError, urlopen, "http://python.org/") + with self.assertRaises(urllib.error.HTTPError) as cm: + urllib.request.urlopen("http://python.org/") + cm.exception.close() finally: self.unfakehttp() @@ -492,22 +436,24 @@ def test_invalid_redirect(self): ''', mock_close=True) try: msg = "Redirection to url 'file:" - with self.assertRaisesRegex(urllib.error.HTTPError, msg): - urlopen("http://python.org/") + with self.assertRaisesRegex(urllib.error.HTTPError, msg) as cm: + urllib.request.urlopen("http://python.org/") + cm.exception.close() finally: self.unfakehttp() def test_redirect_limit_independent(self): # Ticket #12923: make sure independent requests each use their # own retry limit. - for i in range(FancyURLopener().maxtries): + for i in range(urllib.request.HTTPRedirectHandler.max_redirections): self.fakehttp(b'''HTTP/1.1 302 Found Location: file://guidocomputer.athome.com:/python/license Connection: close ''', mock_close=True) try: - self.assertRaises(urllib.error.HTTPError, urlopen, - "http://something") + with self.assertRaises(urllib.error.HTTPError) as cm: + urllib.request.urlopen("http://something") + cm.exception.close() finally: self.unfakehttp() @@ -516,87 +462,47 @@ def test_empty_socket(self): # data. (#1680230) self.fakehttp(b'') try: - self.assertRaises(OSError, urlopen, "http://something") + self.assertRaises(OSError, urllib.request.urlopen, "http://something") finally: self.unfakehttp() def test_missing_localfile(self): # Test for #10836 with self.assertRaises(urllib.error.URLError) as e: - urlopen('file://localhost/a/file/which/doesnot/exists.py') + urllib.request.urlopen('file://localhost/a/file/which/doesnot/exists.py') self.assertTrue(e.exception.filename) self.assertTrue(e.exception.reason) def test_file_notexists(self): fd, tmp_file = tempfile.mkstemp() - tmp_fileurl = 'file://localhost/' + tmp_file.replace(os.path.sep, '/') + tmp_file_canon_url = 'file:' + urllib.request.pathname2url(tmp_file) + parsed = urllib.parse.urlsplit(tmp_file_canon_url) + tmp_fileurl = parsed._replace(netloc='localhost').geturl() try: self.assertTrue(os.path.exists(tmp_file)) - with urlopen(tmp_fileurl) as fobj: + with urllib.request.urlopen(tmp_fileurl) as fobj: self.assertTrue(fobj) + self.assertEqual(fobj.url, tmp_file_canon_url) finally: os.close(fd) os.unlink(tmp_file) self.assertFalse(os.path.exists(tmp_file)) with self.assertRaises(urllib.error.URLError): - urlopen(tmp_fileurl) + urllib.request.urlopen(tmp_fileurl) def test_ftp_nohost(self): test_ftp_url = 'ftp:///path' with self.assertRaises(urllib.error.URLError) as e: - urlopen(test_ftp_url) + urllib.request.urlopen(test_ftp_url) self.assertFalse(e.exception.filename) self.assertTrue(e.exception.reason) def test_ftp_nonexisting(self): with self.assertRaises(urllib.error.URLError) as e: - urlopen('ftp://localhost/a/file/which/doesnot/exists.py') + urllib.request.urlopen('ftp://localhost/a/file/which/doesnot/exists.py') self.assertFalse(e.exception.filename) self.assertTrue(e.exception.reason) - @patch.object(urllib.request, 'MAXFTPCACHE', 0) - def test_ftp_cache_pruning(self): - self.fakeftp() - try: - urllib.request.ftpcache['test'] = urllib.request.ftpwrapper('user', 'pass', 'localhost', 21, []) - urlopen('ftp://localhost') - finally: - self.unfakeftp() - - def test_userpass_inurl(self): - self.fakehttp(b"HTTP/1.0 200 OK\r\n\r\nHello!") - try: - fp = urlopen("http://user:pass@python.org/") - self.assertEqual(fp.readline(), b"Hello!") - self.assertEqual(fp.readline(), b"") - self.assertEqual(fp.geturl(), 'http://user:pass@python.org/') - self.assertEqual(fp.getcode(), 200) - finally: - self.unfakehttp() - - def test_userpass_inurl_w_spaces(self): - self.fakehttp(b"HTTP/1.0 200 OK\r\n\r\nHello!") - try: - userpass = "a b:c d" - url = "http://{}@python.org/".format(userpass) - fakehttp_wrapper = http.client.HTTPConnection - authorization = ("Authorization: Basic %s\r\n" % - b64encode(userpass.encode("ASCII")).decode("ASCII")) - fp = urlopen(url) - # The authorization header must be in place - self.assertIn(authorization, fakehttp_wrapper.buf.decode("UTF-8")) - self.assertEqual(fp.readline(), b"Hello!") - self.assertEqual(fp.readline(), b"") - # the spaces are quoted in URL so no match - self.assertNotEqual(fp.geturl(), url) - self.assertEqual(fp.getcode(), 200) - finally: - self.unfakehttp() - - def test_URLopener_deprecation(self): - with warnings_helper.check_warnings(('',DeprecationWarning)): - urllib.request.URLopener() - class urlopen_DataTests(unittest.TestCase): """Test urlopen() opening a data URL.""" @@ -627,10 +533,11 @@ def setUp(self): "QOjdAAAAAXNSR0IArs4c6QAAAA9JREFUCNdj%0AYGBg%2BP//PwAGAQL%2BCm8 " "vHgAAAABJRU5ErkJggg%3D%3D%0A%20") - self.text_url_resp = urllib.request.urlopen(self.text_url) - self.text_url_base64_resp = urllib.request.urlopen( - self.text_url_base64) - self.image_url_resp = urllib.request.urlopen(self.image_url) + self.text_url_resp = self.enterContext( + urllib.request.urlopen(self.text_url)) + self.text_url_base64_resp = self.enterContext( + urllib.request.urlopen(self.text_url_base64)) + self.image_url_resp = self.enterContext(urllib.request.urlopen(self.image_url)) def test_interface(self): # Make sure object returned by urlopen() has the specified methods @@ -646,8 +553,10 @@ def test_info(self): [('text/plain', ''), ('charset', 'ISO-8859-1')]) self.assertEqual(self.image_url_resp.info()['content-length'], str(len(self.image))) - self.assertEqual(urllib.request.urlopen("data:,").info().get_params(), + r = urllib.request.urlopen("data:,") + self.assertEqual(r.info().get_params(), [('text/plain', ''), ('charset', 'US-ASCII')]) + r.close() def test_geturl(self): self.assertEqual(self.text_url_resp.geturl(), self.text_url) @@ -710,11 +619,7 @@ def tearDown(self): def constructLocalFileUrl(self, filePath): filePath = os.path.abspath(filePath) - try: - filePath.encode("utf-8") - except UnicodeEncodeError: - raise unittest.SkipTest("filePath is not encodable to utf8") - return "file://%s" % urllib.request.pathname2url(filePath) + return "file:" + urllib.request.pathname2url(filePath) def createNewTempFile(self, data=b""): """Creates a new temporary file containing the specified data, @@ -1520,39 +1425,122 @@ def test_quoting(self): (expect, result)) @unittest.skipUnless(sys.platform == 'win32', - 'test specific to the nturl2path functions.') - def test_prefixes(self): + 'test specific to Windows pathnames.') + def test_pathname2url_win(self): # Test special prefixes are correctly handled in pathname2url() - given = '\\\\?\\C:\\dir' - expect = '///C:/dir' - result = urllib.request.pathname2url(given) - self.assertEqual(expect, result, - "pathname2url() failed; %s != %s" % - (expect, result)) - given = '\\\\?\\unc\\server\\share\\dir' - expect = '/server/share/dir' - result = urllib.request.pathname2url(given) - self.assertEqual(expect, result, - "pathname2url() failed; %s != %s" % - (expect, result)) - + fn = urllib.request.pathname2url + self.assertEqual(fn('\\\\?\\C:\\dir'), '///C:/dir') + self.assertEqual(fn('\\\\?\\unc\\server\\share\\dir'), '//server/share/dir') + self.assertEqual(fn("C:"), '///C:') + self.assertEqual(fn("C:\\"), '///C:/') + self.assertEqual(fn('c:\\a\\b.c'), '///c:/a/b.c') + self.assertEqual(fn('C:\\a\\b.c'), '///C:/a/b.c') + self.assertEqual(fn('C:\\a\\b.c\\'), '///C:/a/b.c/') + self.assertEqual(fn('C:\\a\\\\b.c'), '///C:/a//b.c') + self.assertEqual(fn('C:\\a\\b%#c'), '///C:/a/b%25%23c') + self.assertEqual(fn('C:\\a\\b\xe9'), '///C:/a/b%C3%A9') + self.assertEqual(fn('C:\\foo\\bar\\spam.foo'), "///C:/foo/bar/spam.foo") + # NTFS alternate data streams + self.assertEqual(fn('C:\\foo:bar'), '///C:/foo%3Abar') + self.assertEqual(fn('foo:bar'), 'foo%3Abar') + # No drive letter + self.assertEqual(fn("\\folder\\test\\"), '///folder/test/') + self.assertEqual(fn("\\\\folder\\test\\"), '//folder/test/') + self.assertEqual(fn("\\\\\\folder\\test\\"), '///folder/test/') + self.assertEqual(fn('\\\\some\\share\\'), '//some/share/') + self.assertEqual(fn('\\\\some\\share\\a\\b.c'), '//some/share/a/b.c') + self.assertEqual(fn('\\\\some\\share\\a\\b%#c\xe9'), '//some/share/a/b%25%23c%C3%A9') + # Alternate path separator + self.assertEqual(fn('C:/a/b.c'), '///C:/a/b.c') + self.assertEqual(fn('//some/share/a/b.c'), '//some/share/a/b.c') + self.assertEqual(fn('//?/C:/dir'), '///C:/dir') + self.assertEqual(fn('//?/unc/server/share/dir'), '//server/share/dir') + # Round-tripping + urls = ['///C:', + '///folder/test/', + '///C:/foo/bar/spam.foo'] + for url in urls: + self.assertEqual(fn(urllib.request.url2pathname(url)), url) + + @unittest.skipIf(sys.platform == 'win32', + 'test specific to POSIX pathnames') + def test_pathname2url_posix(self): + fn = urllib.request.pathname2url + self.assertEqual(fn('/'), '///') + self.assertEqual(fn('/a/b.c'), '///a/b.c') + self.assertEqual(fn('//a/b.c'), '////a/b.c') + self.assertEqual(fn('///a/b.c'), '/////a/b.c') + self.assertEqual(fn('////a/b.c'), '//////a/b.c') + self.assertEqual(fn('/a/b%#c'), '///a/b%25%23c') + + @unittest.skipUnless(os_helper.FS_NONASCII, 'need os_helper.FS_NONASCII') + def test_pathname2url_nonascii(self): + encoding = sys.getfilesystemencoding() + errors = sys.getfilesystemencodeerrors() + url = urllib.parse.quote(os_helper.FS_NONASCII, encoding=encoding, errors=errors) + self.assertEqual(urllib.request.pathname2url(os_helper.FS_NONASCII), url) @unittest.skipUnless(sys.platform == 'win32', - 'test specific to the urllib.url2path function.') - def test_ntpath(self): - given = ('/C:/', '///C:/', '/C|//') - expect = 'C:\\' - for url in given: - result = urllib.request.url2pathname(url) - self.assertEqual(expect, result, - 'urllib.request..url2pathname() failed; %s != %s' % - (expect, result)) - given = '///C|/path' - expect = 'C:\\path' - result = urllib.request.url2pathname(given) - self.assertEqual(expect, result, - 'urllib.request.url2pathname() failed; %s != %s' % - (expect, result)) + 'test specific to Windows pathnames.') + def test_url2pathname_win(self): + fn = urllib.request.url2pathname + self.assertEqual(fn('/C:/'), 'C:\\') + self.assertEqual(fn("///C|"), 'C:') + self.assertEqual(fn("///C:"), 'C:') + self.assertEqual(fn('///C:/'), 'C:\\') + self.assertEqual(fn('/C|//'), 'C:\\\\') + self.assertEqual(fn('///C|/path'), 'C:\\path') + # No DOS drive + self.assertEqual(fn("///C/test/"), '\\C\\test\\') + self.assertEqual(fn("////C/test/"), '\\\\C\\test\\') + # DOS drive paths + self.assertEqual(fn('c:/path/to/file'), 'c:\\path\\to\\file') + self.assertEqual(fn('C:/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('C:/path/to/file/'), 'C:\\path\\to\\file\\') + self.assertEqual(fn('C:/path/to//file'), 'C:\\path\\to\\\\file') + self.assertEqual(fn('C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('/C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('///C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn("///C|/foo/bar/spam.foo"), 'C:\\foo\\bar\\spam.foo') + # Non-ASCII drive letter + self.assertRaises(IOError, fn, "///\u00e8|/") + # UNC paths + self.assertEqual(fn('//server/path/to/file'), '\\\\server\\path\\to\\file') + self.assertEqual(fn('////server/path/to/file'), '\\\\server\\path\\to\\file') + self.assertEqual(fn('/////server/path/to/file'), '\\\\server\\path\\to\\file') + # Localhost paths + self.assertEqual(fn('//localhost/C:/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('//localhost/C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('//localhost/path/to/file'), '\\path\\to\\file') + self.assertEqual(fn('//localhost//server/path/to/file'), '\\\\server\\path\\to\\file') + # Percent-encoded forward slashes are preserved for backwards compatibility + self.assertEqual(fn('C:/foo%2fbar'), 'C:\\foo/bar') + self.assertEqual(fn('//server/share/foo%2fbar'), '\\\\server\\share\\foo/bar') + # Round-tripping + paths = ['C:', + r'\C\test\\', + r'C:\foo\bar\spam.foo'] + for path in paths: + self.assertEqual(fn(urllib.request.pathname2url(path)), path) + + @unittest.skipIf(sys.platform == 'win32', + 'test specific to POSIX pathnames') + def test_url2pathname_posix(self): + fn = urllib.request.url2pathname + self.assertEqual(fn('/foo/bar'), '/foo/bar') + self.assertEqual(fn('//foo/bar'), '//foo/bar') + self.assertEqual(fn('///foo/bar'), '/foo/bar') + self.assertEqual(fn('////foo/bar'), '//foo/bar') + self.assertEqual(fn('//localhost/foo/bar'), '/foo/bar') + + @unittest.skipUnless(os_helper.FS_NONASCII, 'need os_helper.FS_NONASCII') + def test_url2pathname_nonascii(self): + encoding = sys.getfilesystemencoding() + errors = sys.getfilesystemencodeerrors() + url = os_helper.FS_NONASCII + self.assertEqual(urllib.request.url2pathname(url), os_helper.FS_NONASCII) + url = urllib.parse.quote(url, encoding=encoding, errors=errors) + self.assertEqual(urllib.request.url2pathname(url), os_helper.FS_NONASCII) class Utility_Tests(unittest.TestCase): """Testcase to test the various utility functions in the urllib.""" @@ -1562,56 +1550,6 @@ def test_thishost(self): self.assertIsInstance(urllib.request.thishost(), tuple) -class URLopener_Tests(FakeHTTPMixin, unittest.TestCase): - """Testcase to test the open method of URLopener class.""" - - def test_quoted_open(self): - class DummyURLopener(urllib.request.URLopener): - def open_spam(self, url): - return url - with warnings_helper.check_warnings( - ('DummyURLopener style of invoking requests is deprecated.', - DeprecationWarning)): - self.assertEqual(DummyURLopener().open( - 'spam://example/ /'),'//example/%20/') - - # test the safe characters are not quoted by urlopen - self.assertEqual(DummyURLopener().open( - "spam://c:|windows%/:=&?~#+!$,;'@()*[]|/path/"), - "//c:|windows%/:=&?~#+!$,;'@()*[]|/path/") - - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_urlopener_retrieve_file(self): - with os_helper.temp_dir() as tmpdir: - fd, tmpfile = tempfile.mkstemp(dir=tmpdir) - os.close(fd) - fileurl = "file:" + urllib.request.pathname2url(tmpfile) - filename, _ = urllib.request.URLopener().retrieve(fileurl) - # Some buildbots have TEMP folder that uses a lowercase drive letter. - self.assertEqual(os.path.normcase(filename), os.path.normcase(tmpfile)) - - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_urlopener_retrieve_remote(self): - url = "http://www.python.org/file.txt" - self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello!") - self.addCleanup(self.unfakehttp) - filename, _ = urllib.request.URLopener().retrieve(url) - self.assertEqual(os.path.splitext(filename)[1], ".txt") - - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_local_file_open(self): - # bpo-35907, CVE-2019-9948: urllib must reject local_file:// scheme - class DummyURLopener(urllib.request.URLopener): - def open_local_file(self, url): - return url - for url in ('local_file://example', 'local-file://example'): - self.assertRaises(OSError, urllib.request.urlopen, url) - self.assertRaises(OSError, urllib.request.URLopener().open, url) - self.assertRaises(OSError, urllib.request.URLopener().retrieve, url) - self.assertRaises(OSError, DummyURLopener().open, url) - self.assertRaises(OSError, DummyURLopener().retrieve, url) - - class RequestTests(unittest.TestCase): """Unit tests for urllib.request.Request.""" @@ -1636,60 +1574,5 @@ def test_with_method_arg(self): self.assertEqual(request.get_method(), 'HEAD') -class URL2PathNameTests(unittest.TestCase): - - def test_converting_drive_letter(self): - self.assertEqual(url2pathname("///C|"), 'C:') - self.assertEqual(url2pathname("///C:"), 'C:') - self.assertEqual(url2pathname("///C|/"), 'C:\\') - - def test_converting_when_no_drive_letter(self): - # cannot end a raw string in \ - self.assertEqual(url2pathname("///C/test/"), r'\\\C\test' '\\') - self.assertEqual(url2pathname("////C/test/"), r'\\C\test' '\\') - - def test_simple_compare(self): - self.assertEqual(url2pathname("///C|/foo/bar/spam.foo"), - r'C:\foo\bar\spam.foo') - - def test_non_ascii_drive_letter(self): - self.assertRaises(IOError, url2pathname, "///\u00e8|/") - - def test_roundtrip_url2pathname(self): - list_of_paths = ['C:', - r'\\\C\test\\', - r'C:\foo\bar\spam.foo' - ] - for path in list_of_paths: - self.assertEqual(url2pathname(pathname2url(path)), path) - -class PathName2URLTests(unittest.TestCase): - - def test_converting_drive_letter(self): - self.assertEqual(pathname2url("C:"), '///C:') - self.assertEqual(pathname2url("C:\\"), '///C:') - - def test_converting_when_no_drive_letter(self): - self.assertEqual(pathname2url(r"\\\folder\test" "\\"), - '/////folder/test/') - self.assertEqual(pathname2url(r"\\folder\test" "\\"), - '////folder/test/') - self.assertEqual(pathname2url(r"\folder\test" "\\"), - '/folder/test/') - - def test_simple_compare(self): - self.assertEqual(pathname2url(r'C:\foo\bar\spam.foo'), - "///C:/foo/bar/spam.foo" ) - - def test_long_drive_letter(self): - self.assertRaises(IOError, pathname2url, "XX:\\") - - def test_roundtrip_pathname2url(self): - list_of_paths = ['///C:', - '/////folder/test/', - '///C:/foo/bar/spam.foo'] - for path in list_of_paths: - self.assertEqual(pathname2url(url2pathname(path)), path) - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 739c15df13de21..44e6af8c6b6868 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -8,6 +8,7 @@ import os import io +import ftplib import socket import array import sys @@ -22,10 +23,11 @@ _proxy_bypass_winreg_override, _proxy_bypass_macosx_sysconf, AbstractDigestAuthHandler) -from urllib.parse import urlparse +from urllib.parse import urlsplit import urllib.error import http.client + support.requires_working_socket(module=True) # XXX @@ -716,18 +718,6 @@ def test_processors(self): self.assertIsInstance(args[1], MockResponse) -def sanepathname2url(path): - try: - path.encode("utf-8") - except UnicodeEncodeError: - raise unittest.SkipTest("path is not encodable to utf8") - urlpath = urllib.request.pathname2url(path) - if os.name == "nt" and urlpath.startswith("///"): - urlpath = urlpath[2:] - # XXX don't ask me about the mac... - return urlpath - - class HandlerTests(unittest.TestCase): def test_ftp(self): @@ -754,7 +744,6 @@ def connect_ftp(self, user, passwd, host, port, dirs, self.ftpwrapper = MockFTPWrapper(self.data) return self.ftpwrapper - import ftplib data = "rheum rhaponicum" h = NullFTPHandler(data) h.parent = MockOpener() @@ -777,7 +766,7 @@ def connect_ftp(self, user, passwd, host, port, dirs, ["foo", "bar"], "", None), ("ftp://localhost/baz.gif;type=a", "localhost", ftplib.FTP_PORT, "", "", "A", - [], "baz.gif", None), # XXX really this should guess image/gif + [], "baz.gif", "image/gif"), ]: req = Request(url) req.timeout = None @@ -793,6 +782,29 @@ def connect_ftp(self, user, passwd, host, port, dirs, headers = r.info() self.assertEqual(headers.get("Content-type"), mimetype) self.assertEqual(int(headers["Content-length"]), len(data)) + r.close() + + @support.requires_resource("network") + def test_ftp_error(self): + class ErrorFTPHandler(urllib.request.FTPHandler): + def __init__(self, exception): + self._exception = exception + + def connect_ftp(self, user, passwd, host, port, dirs, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + raise self._exception + + exception = ftplib.error_perm( + "500 OOPS: cannot change directory:/nonexistent") + h = ErrorFTPHandler(exception) + urlopen = urllib.request.build_opener(h).open + try: + urlopen("ftp://www.pythontest.net/") + except urllib.error.URLError as raised: + self.assertEqual(raised.reason, + f"ftp error: {exception.args[0]}") + else: + self.fail("Did not raise ftplib exception") def test_file(self): import email.utils @@ -800,19 +812,22 @@ def test_file(self): o = h.parent = MockOpener() TESTFN = os_helper.TESTFN - urlpath = sanepathname2url(os.path.abspath(TESTFN)) towrite = b"hello, world\n" + canonurl = 'file:' + urllib.request.pathname2url(os.path.abspath(TESTFN)) + parsed = urlsplit(canonurl) + if parsed.netloc: + raise unittest.SkipTest("non-local working directory") urls = [ - "file://localhost%s" % urlpath, - "file://%s" % urlpath, - "file://%s%s" % (socket.gethostbyname('localhost'), urlpath), + canonurl, + parsed._replace(netloc='localhost').geturl(), + parsed._replace(netloc=socket.gethostbyname('localhost')).geturl(), ] try: localaddr = socket.gethostbyname(socket.gethostname()) except socket.gaierror: localaddr = '' if localaddr: - urls.append("file://%s%s" % (localaddr, urlpath)) + urls.append(parsed._replace(netloc=localaddr).geturl()) for url in urls: f = open(TESTFN, "wb") @@ -837,10 +852,10 @@ def test_file(self): self.assertEqual(headers["Content-type"], "text/plain") self.assertEqual(headers["Content-length"], "13") self.assertEqual(headers["Last-modified"], modified) - self.assertEqual(respurl, url) + self.assertEqual(respurl, canonurl) for url in [ - "file://localhost:80%s" % urlpath, + parsed._replace(netloc='localhost:80').geturl(), "file:///file_does_not_exist.txt", "file://not-a-local-host.com//dir/file.txt", "file://%s:80%s/%s" % (socket.gethostbyname('localhost'), @@ -1138,13 +1153,13 @@ def test_full_url_setter(self): r = Request('http://example.com') for url in urls: r.full_url = url - parsed = urlparse(url) + parsed = urlsplit(url) self.assertEqual(r.get_full_url(), url) # full_url setter uses splittag to split into components. # splittag sets the fragment as None while urlparse sets it to '' self.assertEqual(r.fragment or '', parsed.fragment) - self.assertEqual(urlparse(r.get_full_url()).query, parsed.query) + self.assertEqual(urlsplit(r.get_full_url()).query, parsed.query) def test_full_url_deleter(self): r = Request('http://www.example.com') @@ -1233,10 +1248,11 @@ def test_redirect(self): try: method(req, MockFile(), code, "Blah", MockHeaders({"location": to_url})) - except urllib.error.HTTPError: + except urllib.error.HTTPError as err: # 307 and 308 in response to POST require user OK self.assertIn(code, (307, 308)) self.assertIsNotNone(data) + err.close() self.assertEqual(o.req.get_full_url(), to_url) try: self.assertEqual(o.req.get_method(), "GET") @@ -1272,9 +1288,10 @@ def redirect(h, req, url=to_url): while 1: redirect(h, req, "http://example.com/") count = count + 1 - except urllib.error.HTTPError: + except urllib.error.HTTPError as err: # don't stop until max_repeats, because cookies may introduce state self.assertEqual(count, urllib.request.HTTPRedirectHandler.max_repeats) + err.close() # detect endless non-repeating chain of redirects req = Request(from_url, origin_req_host="example.com") @@ -1284,9 +1301,10 @@ def redirect(h, req, url=to_url): while 1: redirect(h, req, "http://example.com/%d" % count) count = count + 1 - except urllib.error.HTTPError: + except urllib.error.HTTPError as err: self.assertEqual(count, urllib.request.HTTPRedirectHandler.max_redirections) + err.close() def test_invalid_redirect(self): from_url = "http://example.com/a.html" @@ -1300,9 +1318,11 @@ def test_invalid_redirect(self): for scheme in invalid_schemes: invalid_url = scheme + '://' + schemeless_url - self.assertRaises(urllib.error.HTTPError, h.http_error_302, + with self.assertRaises(urllib.error.HTTPError) as cm: + h.http_error_302( req, MockFile(), 302, "Security Loophole", MockHeaders({"location": invalid_url})) + cm.exception.close() for scheme in valid_schemes: valid_url = scheme + '://' + schemeless_url @@ -1402,6 +1422,15 @@ def http_open(self, req): request = handler.last_buf self.assertTrue(request.startswith(expected), repr(request)) + def test_redirect_head_request(self): + from_url = "http://example.com/a.html" + to_url = "http://example.com/b.html" + h = urllib.request.HTTPRedirectHandler() + req = Request(from_url, method="HEAD") + fp = MockFile() + new_req = h.redirect_request(req, fp, 302, "Found", {}, to_url) + self.assertEqual(new_req.get_method(), "HEAD") + def test_proxy(self): u = "proxy.example.com:3128" for d in dict(http=u), dict(HTTP=u): @@ -1889,11 +1918,13 @@ def test_HTTPError_interface(self): self.assertEqual(str(err), expected_errmsg) expected_errmsg = '' % (err.code, err.msg) self.assertEqual(repr(err), expected_errmsg) + err.close() def test_gh_98778(self): x = urllib.error.HTTPError("url", 405, "METHOD NOT ALLOWED", None, None) self.assertEqual(getattr(x, "__notes__", ()), ()) self.assertIsInstance(x.fp.read(), bytes) + x.close() def test_parse_proxy(self): parse_proxy_test_cases = [ @@ -1940,10 +1971,38 @@ def test_parse_proxy(self): self.assertRaises(ValueError, _parse_proxy, 'file:/ftp.example.com'), - def test_unsupported_algorithm(self): - handler = AbstractDigestAuthHandler() + +skip_libssl_fips_mode = unittest.skipIf( + support.is_libssl_fips_mode(), + "conservative skip due to OpenSSL FIPS mode possible algorithm nerfing", +) + + +class TestDigestAuthAlgorithms(unittest.TestCase): + def setUp(self): + self.handler = AbstractDigestAuthHandler() + + @skip_libssl_fips_mode + def test_md5_algorithm(self): + H, KD = self.handler.get_algorithm_impls('MD5') + self.assertEqual(H("foo"), "acbd18db4cc2f85cedef654fccc4a4d8") + self.assertEqual(KD("foo", "bar"), "4e99e8c12de7e01535248d2bac85e732") + + @skip_libssl_fips_mode + def test_sha_algorithm(self): + H, KD = self.handler.get_algorithm_impls('SHA') + self.assertEqual(H("foo"), "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") + self.assertEqual(KD("foo", "bar"), "54dcbe67d21d5eb39493d46d89ae1f412d3bd6de") + + @skip_libssl_fips_mode + def test_sha256_algorithm(self): + H, KD = self.handler.get_algorithm_impls('SHA-256') + self.assertEqual(H("foo"), "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae") + self.assertEqual(KD("foo", "bar"), "a765a8beaa9d561d4c5cbed29d8f4e30870297fdfa9cb7d6e9848a95fec9f937") + + def test_invalid_algorithm(self): with self.assertRaises(ValueError) as exc: - handler.get_algorithm_impls('invalid') + self.handler.get_algorithm_impls('invalid') self.assertEqual( str(exc.exception), "Unsupported digest authentication algorithm 'invalid'" diff --git a/Lib/test/test_urllib2_localnet.py b/Lib/test/test_urllib2_localnet.py index 50c491a3cfd3d0..9cb15d61c2ad4d 100644 --- a/Lib/test/test_urllib2_localnet.py +++ b/Lib/test/test_urllib2_localnet.py @@ -316,7 +316,9 @@ def test_basic_auth_httperror(self): ah = urllib.request.HTTPBasicAuthHandler() ah.add_password(self.REALM, self.server_url, self.USER, self.INCORRECT_PASSWD) urllib.request.install_opener(urllib.request.build_opener(ah)) - self.assertRaises(urllib.error.HTTPError, urllib.request.urlopen, self.server_url) + with self.assertRaises(urllib.error.HTTPError) as cm: + urllib.request.urlopen(self.server_url) + cm.exception.close() @hashlib_helper.requires_hashdigest("md5", openssl=True) @@ -362,15 +364,15 @@ def test_proxy_with_bad_password_raises_httperror(self): self.proxy_digest_handler.add_password(self.REALM, self.URL, self.USER, self.PASSWD+"bad") self.digest_auth_handler.set_qop("auth") - self.assertRaises(urllib.error.HTTPError, - self.opener.open, - self.URL) + with self.assertRaises(urllib.error.HTTPError) as cm: + self.opener.open(self.URL) + cm.exception.close() def test_proxy_with_no_password_raises_httperror(self): self.digest_auth_handler.set_qop("auth") - self.assertRaises(urllib.error.HTTPError, - self.opener.open, - self.URL) + with self.assertRaises(urllib.error.HTTPError) as cm: + self.opener.open(self.URL) + cm.exception.close() def test_proxy_qop_auth_works(self): self.proxy_digest_handler.add_password(self.REALM, self.URL, diff --git a/Lib/test/test_urllib2net.py b/Lib/test/test_urllib2net.py index f0874d8d3ce463..b84290a7368c29 100644 --- a/Lib/test/test_urllib2net.py +++ b/Lib/test/test_urllib2net.py @@ -4,7 +4,6 @@ from test.support import os_helper from test.support import socket_helper from test.support import ResourceDenied -from test.test_urllib2 import sanepathname2url import os import socket @@ -151,7 +150,7 @@ def test_file(self): f.write('hi there\n') f.close() urls = [ - 'file:' + sanepathname2url(os.path.abspath(TESTFN)), + 'file:' + urllib.request.pathname2url(os.path.abspath(TESTFN)), ('file:///nonsensename/etc/passwd', None, urllib.error.URLError), ] diff --git a/Lib/test/test_urllib_response.py b/Lib/test/test_urllib_response.py index b76763f4ed824f..d949fa38bfc42f 100644 --- a/Lib/test/test_urllib_response.py +++ b/Lib/test/test_urllib_response.py @@ -48,6 +48,7 @@ def test_addinfo(self): info = urllib.response.addinfo(self.fp, self.test_headers) self.assertEqual(info.info(), self.test_headers) self.assertEqual(info.headers, self.test_headers) + info.close() def test_addinfourl(self): url = "http://www.python.org" @@ -60,6 +61,7 @@ def test_addinfourl(self): self.assertEqual(infourl.headers, self.test_headers) self.assertEqual(infourl.url, url) self.assertEqual(infourl.status, code) + infourl.close() def tearDown(self): self.sock.close() diff --git a/Lib/test/test_urllibnet.py b/Lib/test/test_urllibnet.py index 49a3b5afdebb2f..ce4e60e3a8011d 100644 --- a/Lib/test/test_urllibnet.py +++ b/Lib/test/test_urllibnet.py @@ -5,6 +5,7 @@ import contextlib import socket +import urllib.error import urllib.parse import urllib.request import os @@ -101,13 +102,11 @@ def test_getcode(self): # test getcode() with the fancy opener to get 404 error codes URL = self.url + "XXXinvalidXXX" with socket_helper.transient_internet(URL): - with self.assertWarns(DeprecationWarning): - open_url = urllib.request.FancyURLopener().open(URL) - try: - code = open_url.getcode() - finally: - open_url.close() - self.assertEqual(code, 404) + with self.assertRaises(urllib.error.URLError) as e: + with urllib.request.urlopen(URL): + pass + self.assertEqual(e.exception.code, 404) + e.exception.close() @support.requires_resource('walltime') def test_bad_address(self): diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 625c6dc88796b6..4516bdea6adb19 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -19,6 +19,10 @@ ("=a", [('', 'a')]), ("a", [('a', '')]), ("a=", [('a', '')]), + ("a=b=c", [('a', 'b=c')]), + ("a%3Db=c", [('a=b', 'c')]), + ("a=b&c=d", [('a', 'b'), ('c', 'd')]), + ("a=b%26c=d", [('a', 'b&c=d')]), ("&a=b", [('a', 'b')]), ("a=a+b&b=b+c", [('a', 'a b'), ('b', 'b c')]), ("a=1&a=2", [('a', '1'), ('a', '2')]), @@ -29,6 +33,10 @@ (b"=a", [(b'', b'a')]), (b"a", [(b'a', b'')]), (b"a=", [(b'a', b'')]), + (b"a=b=c", [(b'a', b'b=c')]), + (b"a%3Db=c", [(b'a=b', b'c')]), + (b"a=b&c=d", [(b'a', b'b'), (b'c', b'd')]), + (b"a=b%26c=d", [(b'a', b'b&c=d')]), (b"&a=b", [(b'a', b'b')]), (b"a=a+b&b=b+c", [(b'a', b'a b'), (b'b', b'b c')]), (b"a=1&a=2", [(b'a', b'1'), (b'a', b'2')]), @@ -36,6 +44,14 @@ ("a=a+b;b=b+c", [('a', 'a b;b=b c')]), (b";a=b", [(b';a', b'b')]), (b"a=a+b;b=b+c", [(b'a', b'a b;b=b c')]), + + ("\u0141=\xE9", [('\u0141', '\xE9')]), + ("%C5%81=%C3%A9", [('\u0141', '\xE9')]), + ("%81=%A9", [('\ufffd', '\ufffd')]), + (b"\xc5\x81=\xc3\xa9", [(b'\xc5\x81', b'\xc3\xa9')]), + (b"%C5%81=%C3%A9", [(b'\xc5\x81', b'\xc3\xa9')]), + (b"\x81=\xA9", [(b'\x81', b'\xa9')]), + (b"%81=%A9", [(b'\x81', b'\xa9')]), ] # Each parse_qs testcase is a two-tuple that contains @@ -49,6 +65,10 @@ ("=a", {'': ['a']}), ("a", {'a': ['']}), ("a=", {'a': ['']}), + ("a=b=c", {'a': ['b=c']}), + ("a%3Db=c", {'a=b': ['c']}), + ("a=b&c=d", {'a': ['b'], 'c': ['d']}), + ("a=b%26c=d", {'a': ['b&c=d']}), ("&a=b", {'a': ['b']}), ("a=a+b&b=b+c", {'a': ['a b'], 'b': ['b c']}), ("a=1&a=2", {'a': ['1', '2']}), @@ -59,6 +79,10 @@ (b"=a", {b'': [b'a']}), (b"a", {b'a': [b'']}), (b"a=", {b'a': [b'']}), + (b"a=b=c", {b'a': [b'b=c']}), + (b"a%3Db=c", {b'a=b': [b'c']}), + (b"a=b&c=d", {b'a': [b'b'], b'c': [b'd']}), + (b"a=b%26c=d", {b'a': [b'b&c=d']}), (b"&a=b", {b'a': [b'b']}), (b"a=a+b&b=b+c", {b'a': [b'a b'], b'b': [b'b c']}), (b"a=1&a=2", {b'a': [b'1', b'2']}), @@ -66,11 +90,22 @@ ("a=a+b;b=b+c", {'a': ['a b;b=b c']}), (b";a=b", {b';a': [b'b']}), (b"a=a+b;b=b+c", {b'a':[ b'a b;b=b c']}), + (b"a=a%E2%80%99b", {b'a': [b'a\xe2\x80\x99b']}), + + ("\u0141=\xE9", {'\u0141': ['\xE9']}), + ("%C5%81=%C3%A9", {'\u0141': ['\xE9']}), + ("%81=%A9", {'\ufffd': ['\ufffd']}), + (b"\xc5\x81=\xc3\xa9", {b'\xc5\x81': [b'\xc3\xa9']}), + (b"%C5%81=%C3%A9", {b'\xc5\x81': [b'\xc3\xa9']}), + (b"\x81=\xA9", {b'\x81': [b'\xa9']}), + (b"%81=%A9", {b'\x81': [b'\xa9']}), ] class UrlParseTestCase(unittest.TestCase): - def checkRoundtrips(self, url, parsed, split): + def checkRoundtrips(self, url, parsed, split, url2=None): + if url2 is None: + url2 = url result = urllib.parse.urlparse(url) self.assertSequenceEqual(result, parsed) t = (result.scheme, result.netloc, result.path, @@ -78,7 +113,7 @@ def checkRoundtrips(self, url, parsed, split): self.assertSequenceEqual(t, parsed) # put it back together and it should be the same result2 = urllib.parse.urlunparse(result) - self.assertSequenceEqual(result2, url) + self.assertSequenceEqual(result2, url2) self.assertSequenceEqual(result2, result.geturl()) # the result of geturl() is a fixpoint; we can always parse it @@ -104,7 +139,7 @@ def checkRoundtrips(self, url, parsed, split): result.query, result.fragment) self.assertSequenceEqual(t, split) result2 = urllib.parse.urlunsplit(result) - self.assertSequenceEqual(result2, url) + self.assertSequenceEqual(result2, url2) self.assertSequenceEqual(result2, result.geturl()) # check the fixpoint property of re-parsing the result of geturl() @@ -142,9 +177,60 @@ def test_qs(self): def test_roundtrips(self): str_cases = [ + ('path/to/file', + ('', '', 'path/to/file', '', '', ''), + ('', '', 'path/to/file', '', '')), + ('/path/to/file', + ('', '', '/path/to/file', '', '', ''), + ('', '', '/path/to/file', '', '')), + ('//path/to/file', + ('', 'path', '/to/file', '', '', ''), + ('', 'path', '/to/file', '', '')), + ('////path/to/file', + ('', '', '//path/to/file', '', '', ''), + ('', '', '//path/to/file', '', '')), + ('/////path/to/file', + ('', '', '///path/to/file', '', '', ''), + ('', '', '///path/to/file', '', '')), + ('scheme:path/to/file', + ('scheme', '', 'path/to/file', '', '', ''), + ('scheme', '', 'path/to/file', '', '')), + ('scheme:/path/to/file', + ('scheme', '', '/path/to/file', '', '', ''), + ('scheme', '', '/path/to/file', '', '')), + ('scheme://path/to/file', + ('scheme', 'path', '/to/file', '', '', ''), + ('scheme', 'path', '/to/file', '', '')), + ('scheme:////path/to/file', + ('scheme', '', '//path/to/file', '', '', ''), + ('scheme', '', '//path/to/file', '', '')), + ('scheme://///path/to/file', + ('scheme', '', '///path/to/file', '', '', ''), + ('scheme', '', '///path/to/file', '', '')), + ('file:tmp/junk.txt', + ('file', '', 'tmp/junk.txt', '', '', ''), + ('file', '', 'tmp/junk.txt', '', '')), ('file:///tmp/junk.txt', ('file', '', '/tmp/junk.txt', '', '', ''), ('file', '', '/tmp/junk.txt', '', '')), + ('file:////tmp/junk.txt', + ('file', '', '//tmp/junk.txt', '', '', ''), + ('file', '', '//tmp/junk.txt', '', '')), + ('file://///tmp/junk.txt', + ('file', '', '///tmp/junk.txt', '', '', ''), + ('file', '', '///tmp/junk.txt', '', '')), + ('http:tmp/junk.txt', + ('http', '', 'tmp/junk.txt', '', '', ''), + ('http', '', 'tmp/junk.txt', '', '')), + ('http://example.com/tmp/junk.txt', + ('http', 'example.com', '/tmp/junk.txt', '', '', ''), + ('http', 'example.com', '/tmp/junk.txt', '', '')), + ('http:///example.com/tmp/junk.txt', + ('http', '', '/example.com/tmp/junk.txt', '', '', ''), + ('http', '', '/example.com/tmp/junk.txt', '', '')), + ('http:////example.com/tmp/junk.txt', + ('http', '', '//example.com/tmp/junk.txt', '', '', ''), + ('http', '', '//example.com/tmp/junk.txt', '', '')), ('imap://mail.python.org/mbox1', ('imap', 'mail.python.org', '/mbox1', '', '', ''), ('imap', 'mail.python.org', '/mbox1', '', '')), @@ -171,14 +257,59 @@ def test_roundtrips(self): 'action=download-manifest&url=https://example.com/app', ''), ('itms-services', '', '', 'action=download-manifest&url=https://example.com/app', '')), + ('+scheme:path/to/file', + ('', '', '+scheme:path/to/file', '', '', ''), + ('', '', '+scheme:path/to/file', '', '')), + ('sch_me:path/to/file', + ('', '', 'sch_me:path/to/file', '', '', ''), + ('', '', 'sch_me:path/to/file', '', '')), ] def _encode(t): return (t[0].encode('ascii'), tuple(x.encode('ascii') for x in t[1]), tuple(x.encode('ascii') for x in t[2])) bytes_cases = [_encode(x) for x in str_cases] + str_cases += [ + ('schème:path/to/file', + ('', '', 'schème:path/to/file', '', '', ''), + ('', '', 'schème:path/to/file', '', '')), + ] for url, parsed, split in str_cases + bytes_cases: - self.checkRoundtrips(url, parsed, split) + with self.subTest(url): + self.checkRoundtrips(url, parsed, split) + + def test_roundtrips_normalization(self): + str_cases = [ + ('///path/to/file', + '/path/to/file', + ('', '', '/path/to/file', '', '', ''), + ('', '', '/path/to/file', '', '')), + ('scheme:///path/to/file', + 'scheme:/path/to/file', + ('scheme', '', '/path/to/file', '', '', ''), + ('scheme', '', '/path/to/file', '', '')), + ('file:/tmp/junk.txt', + 'file:///tmp/junk.txt', + ('file', '', '/tmp/junk.txt', '', '', ''), + ('file', '', '/tmp/junk.txt', '', '')), + ('http:/tmp/junk.txt', + 'http:///tmp/junk.txt', + ('http', '', '/tmp/junk.txt', '', '', ''), + ('http', '', '/tmp/junk.txt', '', '')), + ('https:/tmp/junk.txt', + 'https:///tmp/junk.txt', + ('https', '', '/tmp/junk.txt', '', '', ''), + ('https', '', '/tmp/junk.txt', '', '')), + ] + def _encode(t): + return (t[0].encode('ascii'), + t[1].encode('ascii'), + tuple(x.encode('ascii') for x in t[2]), + tuple(x.encode('ascii') for x in t[3])) + bytes_cases = [_encode(x) for x in str_cases] + for url, url2, parsed, split in str_cases + bytes_cases: + with self.subTest(url): + self.checkRoundtrips(url, parsed, split, url2) def test_http_roundtrips(self): # urllib.parse.urlsplit treats 'http:' as an optimized special case, @@ -218,12 +349,19 @@ def _encode(t): split = (scheme,) + split self.checkRoundtrips(url, parsed, split) - def checkJoin(self, base, relurl, expected): - str_components = (base, relurl, expected) - self.assertEqual(urllib.parse.urljoin(base, relurl), expected) - bytes_components = baseb, relurlb, expectedb = [ - x.encode('ascii') for x in str_components] - self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) + def checkJoin(self, base, relurl, expected, *, relroundtrip=True): + with self.subTest(base=base, relurl=relurl): + self.assertEqual(urllib.parse.urljoin(base, relurl), expected) + baseb = base.encode('ascii') + relurlb = relurl.encode('ascii') + expectedb = expected.encode('ascii') + self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) + + if relroundtrip: + relurl = urllib.parse.urlunsplit(urllib.parse.urlsplit(relurl)) + self.assertEqual(urllib.parse.urljoin(base, relurl), expected) + relurlb = urllib.parse.urlunsplit(urllib.parse.urlsplit(relurlb)) + self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) def test_unparse_parse(self): str_cases = ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',] @@ -389,8 +527,6 @@ def test_RFC3986(self): def test_urljoins(self): self.checkJoin(SIMPLE_BASE, 'g:h','g:h') - self.checkJoin(SIMPLE_BASE, 'http:g','http://a/b/c/g') - self.checkJoin(SIMPLE_BASE, 'http:','http://a/b/c/d') self.checkJoin(SIMPLE_BASE, 'g','http://a/b/c/g') self.checkJoin(SIMPLE_BASE, './g','http://a/b/c/g') self.checkJoin(SIMPLE_BASE, 'g/','http://a/b/c/g/') @@ -411,8 +547,6 @@ def test_urljoins(self): self.checkJoin(SIMPLE_BASE, 'g/./h','http://a/b/c/g/h') self.checkJoin(SIMPLE_BASE, 'g/../h','http://a/b/c/h') self.checkJoin(SIMPLE_BASE, 'http:g','http://a/b/c/g') - self.checkJoin(SIMPLE_BASE, 'http:','http://a/b/c/d') - self.checkJoin(SIMPLE_BASE, 'http:?y','http://a/b/c/d?y') self.checkJoin(SIMPLE_BASE, 'http:g?y','http://a/b/c/g?y') self.checkJoin(SIMPLE_BASE, 'http:g?y/./x','http://a/b/c/g?y/./x') self.checkJoin('http:///', '..','http:///') @@ -442,6 +576,125 @@ def test_urljoins(self): # issue 23703: don't duplicate filename self.checkJoin('a', 'b', 'b') + # Test with empty (but defined) components. + self.checkJoin(RFC1808_BASE, '', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, '#', 'http://a/b/c/d;p?q#', relroundtrip=False) + self.checkJoin(RFC1808_BASE, '#z', 'http://a/b/c/d;p?q#z') + self.checkJoin(RFC1808_BASE, '?', 'http://a/b/c/d;p?', relroundtrip=False) + self.checkJoin(RFC1808_BASE, '?#z', 'http://a/b/c/d;p?#z', relroundtrip=False) + self.checkJoin(RFC1808_BASE, '?y', 'http://a/b/c/d;p?y') + self.checkJoin(RFC1808_BASE, ';', 'http://a/b/c/;') + self.checkJoin(RFC1808_BASE, ';?y', 'http://a/b/c/;?y') + self.checkJoin(RFC1808_BASE, ';#z', 'http://a/b/c/;#z') + self.checkJoin(RFC1808_BASE, ';x', 'http://a/b/c/;x') + self.checkJoin(RFC1808_BASE, '/w', 'http://a/w') + self.checkJoin(RFC1808_BASE, '//', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, '//#z', 'http://a/b/c/d;p?q#z') + self.checkJoin(RFC1808_BASE, '//?y', 'http://a/b/c/d;p?y') + self.checkJoin(RFC1808_BASE, '//;x', 'http://;x') + self.checkJoin(RFC1808_BASE, '///w', 'http://a/w') + self.checkJoin(RFC1808_BASE, '//v', 'http://v') + # For backward compatibility with RFC1630, the scheme name is allowed + # to be present in a relative reference if it is the same as the base + # URI scheme. + self.checkJoin(RFC1808_BASE, 'http:', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, 'http:#', 'http://a/b/c/d;p?q#', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'http:#z', 'http://a/b/c/d;p?q#z') + self.checkJoin(RFC1808_BASE, 'http:?', 'http://a/b/c/d;p?', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'http:?#z', 'http://a/b/c/d;p?#z', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'http:?y', 'http://a/b/c/d;p?y') + self.checkJoin(RFC1808_BASE, 'http:;', 'http://a/b/c/;') + self.checkJoin(RFC1808_BASE, 'http:;?y', 'http://a/b/c/;?y') + self.checkJoin(RFC1808_BASE, 'http:;#z', 'http://a/b/c/;#z') + self.checkJoin(RFC1808_BASE, 'http:;x', 'http://a/b/c/;x') + self.checkJoin(RFC1808_BASE, 'http:/w', 'http://a/w') + self.checkJoin(RFC1808_BASE, 'http://', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, 'http://#z', 'http://a/b/c/d;p?q#z') + self.checkJoin(RFC1808_BASE, 'http://?y', 'http://a/b/c/d;p?y') + self.checkJoin(RFC1808_BASE, 'http://;x', 'http://;x') + self.checkJoin(RFC1808_BASE, 'http:///w', 'http://a/w') + self.checkJoin(RFC1808_BASE, 'http://v', 'http://v') + # Different scheme is not ignored. + self.checkJoin(RFC1808_BASE, 'https:', 'https:', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:#', 'https:#', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:#z', 'https:#z', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:?', 'https:?', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:?y', 'https:?y', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:;', 'https:;') + self.checkJoin(RFC1808_BASE, 'https:;x', 'https:;x') + + def test_urljoins_relative_base(self): + # According to RFC 3986, Section 5.1, a base URI must conform to + # the absolute-URI syntax rule (Section 4.3). But urljoin() lacks + # a context to establish missed components of the relative base URI. + # It still has to return a sensible result for backwards compatibility. + # The following tests are figments of the imagination and artifacts + # of the current implementation that are not based on any standard. + self.checkJoin('', '', '') + self.checkJoin('', '//', '//', relroundtrip=False) + self.checkJoin('', '//v', '//v') + self.checkJoin('', '//v/w', '//v/w') + self.checkJoin('', '/w', '/w') + self.checkJoin('', '///w', '///w', relroundtrip=False) + self.checkJoin('', 'w', 'w') + + self.checkJoin('//', '', '//') + self.checkJoin('//', '//', '//') + self.checkJoin('//', '//v', '//v') + self.checkJoin('//', '//v/w', '//v/w') + self.checkJoin('//', '/w', '///w') + self.checkJoin('//', '///w', '///w') + self.checkJoin('//', 'w', '///w') + + self.checkJoin('//a', '', '//a') + self.checkJoin('//a', '//', '//a') + self.checkJoin('//a', '//v', '//v') + self.checkJoin('//a', '//v/w', '//v/w') + self.checkJoin('//a', '/w', '//a/w') + self.checkJoin('//a', '///w', '//a/w') + self.checkJoin('//a', 'w', '//a/w') + + for scheme in '', 'http:': + self.checkJoin('http:', scheme + '', 'http:') + self.checkJoin('http:', scheme + '//', 'http:') + self.checkJoin('http:', scheme + '//v', 'http://v') + self.checkJoin('http:', scheme + '//v/w', 'http://v/w') + self.checkJoin('http:', scheme + '/w', 'http:/w') + self.checkJoin('http:', scheme + '///w', 'http:/w') + self.checkJoin('http:', scheme + 'w', 'http:/w') + + self.checkJoin('http://', scheme + '', 'http://') + self.checkJoin('http://', scheme + '//', 'http://') + self.checkJoin('http://', scheme + '//v', 'http://v') + self.checkJoin('http://', scheme + '//v/w', 'http://v/w') + self.checkJoin('http://', scheme + '/w', 'http:///w') + self.checkJoin('http://', scheme + '///w', 'http:///w') + self.checkJoin('http://', scheme + 'w', 'http:///w') + + self.checkJoin('http://a', scheme + '', 'http://a') + self.checkJoin('http://a', scheme + '//', 'http://a') + self.checkJoin('http://a', scheme + '//v', 'http://v') + self.checkJoin('http://a', scheme + '//v/w', 'http://v/w') + self.checkJoin('http://a', scheme + '/w', 'http://a/w') + self.checkJoin('http://a', scheme + '///w', 'http://a/w') + self.checkJoin('http://a', scheme + 'w', 'http://a/w') + + self.checkJoin('/b/c', '', '/b/c') + self.checkJoin('/b/c', '//', '/b/c') + self.checkJoin('/b/c', '//v', '//v') + self.checkJoin('/b/c', '//v/w', '//v/w') + self.checkJoin('/b/c', '/w', '/w') + self.checkJoin('/b/c', '///w', '/w') + self.checkJoin('/b/c', 'w', '/b/w') + + self.checkJoin('///b/c', '', '///b/c') + self.checkJoin('///b/c', '//', '///b/c') + self.checkJoin('///b/c', '//v', '//v') + self.checkJoin('///b/c', '//v/w', '//v/w') + self.checkJoin('///b/c', '/w', '///w') + self.checkJoin('///b/c', '///w', '///w') + self.checkJoin('///b/c', 'w', '///b/w') + def test_RFC2732(self): str_cases = [ ('http://Test.python.org:5432/foo/', 'test.python.org', 5432), @@ -504,16 +757,31 @@ def test_urldefrag(self): ('http://python.org/p?q', 'http://python.org/p?q', ''), (RFC1808_BASE, 'http://a/b/c/d;p?q', 'f'), (RFC2396_BASE, 'http://a/b/c/d;p?q', ''), + ('http://a/b/c;p?q#f', 'http://a/b/c;p?q', 'f'), + ('http://a/b/c;p?q#', 'http://a/b/c;p?q', ''), + ('http://a/b/c;p?q', 'http://a/b/c;p?q', ''), + ('http://a/b/c;p?#f', 'http://a/b/c;p?', 'f'), + ('http://a/b/c;p#f', 'http://a/b/c;p', 'f'), + ('http://a/b/c;?q#f', 'http://a/b/c;?q', 'f'), + ('http://a/b/c?q#f', 'http://a/b/c?q', 'f'), + ('http:///b/c;p?q#f', 'http:///b/c;p?q', 'f'), + ('http:b/c;p?q#f', 'http:b/c;p?q', 'f'), + ('http:;?q#f', 'http:;?q', 'f'), + ('http:?q#f', 'http:?q', 'f'), + ('//a/b/c;p?q#f', '//a/b/c;p?q', 'f'), + ('://a/b/c;p?q#f', '://a/b/c;p?q', 'f'), ] def _encode(t): return type(t)(x.encode('ascii') for x in t) bytes_cases = [_encode(x) for x in str_cases] for url, defrag, frag in str_cases + bytes_cases: - result = urllib.parse.urldefrag(url) - self.assertEqual(result.geturl(), url) - self.assertEqual(result, (defrag, frag)) - self.assertEqual(result.url, defrag) - self.assertEqual(result.fragment, frag) + with self.subTest(url): + result = urllib.parse.urldefrag(url) + hash = '#' if isinstance(url, str) else b'#' + self.assertEqual(result.geturl(), url.rstrip(hash)) + self.assertEqual(result, (defrag, frag)) + self.assertEqual(result.url, defrag) + self.assertEqual(result.fragment, frag) def test_urlsplit_scoped_IPv6(self): p = urllib.parse.urlsplit('http://[FE80::822a:a8ff:fe49:470c%tESt]:1234') @@ -995,8 +1263,8 @@ def test_parse_qsl_encoding(self): def test_parse_qsl_max_num_fields(self): with self.assertRaises(ValueError): - urllib.parse.parse_qs('&'.join(['a=a']*11), max_num_fields=10) - urllib.parse.parse_qs('&'.join(['a=a']*10), max_num_fields=10) + urllib.parse.parse_qsl('&'.join(['a=a']*11), max_num_fields=10) + urllib.parse.parse_qsl('&'.join(['a=a']*10), max_num_fields=10) def test_parse_qs_separator(self): parse_qs_semicolon_cases = [ @@ -1039,6 +1307,38 @@ def test_parse_qsl_separator(self): result_bytes = urllib.parse.parse_qsl(orig, separator=b';') self.assertEqual(result_bytes, expect, "Error parsing %r" % orig) + def test_parse_qsl_bytes(self): + self.assertEqual(urllib.parse.parse_qsl(b'a=b'), [(b'a', b'b')]) + self.assertEqual(urllib.parse.parse_qsl(bytearray(b'a=b')), [(b'a', b'b')]) + self.assertEqual(urllib.parse.parse_qsl(memoryview(b'a=b')), [(b'a', b'b')]) + + def test_parse_qsl_false_value(self): + kwargs = dict(keep_blank_values=True, strict_parsing=True) + for x in '', b'', None, memoryview(b''): + self.assertEqual(urllib.parse.parse_qsl(x, **kwargs), []) + self.assertRaises(ValueError, urllib.parse.parse_qsl, x, separator=1) + for x in 0, 0.0, [], {}: + with self.assertWarns(DeprecationWarning) as cm: + self.assertEqual(urllib.parse.parse_qsl(x, **kwargs), []) + self.assertEqual(cm.filename, __file__) + with self.assertWarns(DeprecationWarning) as cm: + self.assertEqual(urllib.parse.parse_qs(x, **kwargs), {}) + self.assertEqual(cm.filename, __file__) + self.assertRaises(ValueError, urllib.parse.parse_qsl, x, separator=1) + + def test_parse_qsl_errors(self): + self.assertRaises(TypeError, urllib.parse.parse_qsl, list(b'a=b')) + self.assertRaises(TypeError, urllib.parse.parse_qsl, iter(b'a=b')) + self.assertRaises(TypeError, urllib.parse.parse_qsl, 1) + self.assertRaises(TypeError, urllib.parse.parse_qsl, object()) + + for separator in '', b'', None, 0, 1, 0.0, 1.5: + with self.assertRaises(ValueError): + urllib.parse.parse_qsl('a=b', separator=separator) + with self.assertRaises(UnicodeEncodeError): + urllib.parse.parse_qsl(b'a=b', separator='\xa6') + with self.assertRaises(UnicodeDecodeError): + urllib.parse.parse_qsl('a=b', separator=b'\xa6') def test_urlencode_sequences(self): # Other tests incidentally urlencode things; test non-covered cases: @@ -1369,13 +1669,6 @@ def test_unwrap(self): class DeprecationTest(unittest.TestCase): - - def test_Quoter_deprecation(self): - with self.assertWarns(DeprecationWarning) as cm: - old_class = urllib.parse.Quoter - self.assertIs(old_class, urllib.parse._Quoter) - self.assertIn('Quoter will be removed', str(cm.warning)) - def test_splittype_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splittype('') diff --git a/Lib/test/test_utf8source.py b/Lib/test/test_utf8source.py index c42b6aaaab579d..7336cf00a71183 100644 --- a/Lib/test/test_utf8source.py +++ b/Lib/test/test_utf8source.py @@ -14,7 +14,7 @@ def test_pep3120(self): def test_badsyntax(self): try: - import test.tokenizedata.badsyntax_pep3120 + import test.tokenizedata.badsyntax_pep3120 # noqa: F401 except SyntaxError as msg: msg = str(msg).lower() self.assertTrue('utf-8' in msg) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 9cec1e87fd3c2d..8216c4dd00e35a 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -8,8 +8,10 @@ import io import os import pickle +import random import sys import weakref +from itertools import product from unittest import mock py_uuid = import_helper.import_fresh_module('uuid', blocked=['_uuid']) @@ -19,7 +21,7 @@ def importable(name): try: __import__(name) return True - except: + except ModuleNotFoundError: return False @@ -32,6 +34,47 @@ def get_command_stdout(command, args): class BaseTestUUID: uuid = None + def test_nil_uuid(self): + nil_uuid = self.uuid.NIL + + s = '00000000-0000-0000-0000-000000000000' + i = 0 + self.assertEqual(nil_uuid, self.uuid.UUID(s)) + self.assertEqual(nil_uuid, self.uuid.UUID(int=i)) + self.assertEqual(nil_uuid.int, i) + self.assertEqual(str(nil_uuid), s) + # The Nil UUID falls within the range of the Apollo NCS variant as per + # RFC 9562. + # See https://www.rfc-editor.org/rfc/rfc9562.html#section-5.9-4 + self.assertEqual(nil_uuid.variant, self.uuid.RESERVED_NCS) + # A version field of all zeros is "Unused" in RFC 9562, but the version + # field also only applies to the 10xx variant, i.e. the variant + # specified in RFC 9562. As such, because the Nil UUID falls under a + # different variant, its version is considered undefined. + # See https://www.rfc-editor.org/rfc/rfc9562.html#table2 + self.assertIsNone(nil_uuid.version) + + def test_max_uuid(self): + max_uuid = self.uuid.MAX + + s = 'ffffffff-ffff-ffff-ffff-ffffffffffff' + i = (1 << 128) - 1 + self.assertEqual(max_uuid, self.uuid.UUID(s)) + self.assertEqual(max_uuid, self.uuid.UUID(int=i)) + self.assertEqual(max_uuid.int, i) + self.assertEqual(str(max_uuid), s) + # The Max UUID falls within the range of the "yet-to-be defined" future + # UUID variant as per RFC 9562. + # See https://www.rfc-editor.org/rfc/rfc9562.html#section-5.10-4 + self.assertEqual(max_uuid.variant, self.uuid.RESERVED_FUTURE) + # A version field of all ones is "Reserved for future definition" in + # RFC 9562, but the version field also only applies to the 10xx + # variant, i.e. the variant specified in RFC 9562. As such, because the + # Max UUID falls under a different variant, its version is considered + # undefined. + # See https://www.rfc-editor.org/rfc/rfc9562.html#table2 + self.assertIsNone(max_uuid.version) + def test_safe_uuid_enum(self): class CheckedSafeUUID(enum.Enum): safe = 0 @@ -267,7 +310,7 @@ def test_exceptions(self): # Version number out of range. badvalue(lambda: self.uuid.UUID('00'*16, version=0)) - badvalue(lambda: self.uuid.UUID('00'*16, version=6)) + badvalue(lambda: self.uuid.UUID('00'*16, version=42)) # Integer value out of range. badvalue(lambda: self.uuid.UUID(int=-1)) @@ -530,7 +573,14 @@ def test_uuid1(self): @support.requires_mac_ver(10, 5) @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') def test_uuid1_safe(self): - if not self.uuid._has_uuid_generate_time_safe: + try: + import _uuid + except ImportError: + has_uuid_generate_time_safe = False + else: + has_uuid_generate_time_safe = _uuid.has_uuid_generate_time_safe + + if not has_uuid_generate_time_safe or not self.uuid._generate_time_safe: self.skipTest('requires uuid_generate_time_safe(3)') u = self.uuid.uuid1() @@ -546,7 +596,6 @@ def mock_generate_time_safe(self, safe_value): """ if os.name != 'posix': self.skipTest('POSIX-only test') - self.uuid._load_system_functions() f = self.uuid._generate_time_safe if f is None: self.skipTest('need uuid._generate_time_safe') @@ -581,8 +630,7 @@ def test_uuid1_bogus_return_value(self): self.assertEqual(u.is_safe, self.uuid.SafeUUID.unknown) def test_uuid1_time(self): - with mock.patch.object(self.uuid, '_has_uuid_generate_time_safe', False), \ - mock.patch.object(self.uuid, '_generate_time_safe', None), \ + with mock.patch.object(self.uuid, '_generate_time_safe', None), \ mock.patch.object(self.uuid, '_last_timestamp', None), \ mock.patch.object(self.uuid, 'getnode', return_value=93328246233727), \ mock.patch('time.time_ns', return_value=1545052026752910643), \ @@ -590,8 +638,7 @@ def test_uuid1_time(self): u = self.uuid.uuid1() self.assertEqual(u, self.uuid.UUID('a7a55b92-01fc-11e9-94c5-54e1acf6da7f')) - with mock.patch.object(self.uuid, '_has_uuid_generate_time_safe', False), \ - mock.patch.object(self.uuid, '_generate_time_safe', None), \ + with mock.patch.object(self.uuid, '_generate_time_safe', None), \ mock.patch.object(self.uuid, '_last_timestamp', None), \ mock.patch('time.time_ns', return_value=1545052026752910643): u = self.uuid.uuid1(node=93328246233727, clock_seq=5317) @@ -677,6 +724,41 @@ def test_uuid5(self): equal(u, self.uuid.UUID(v)) equal(str(u), v) + def test_uuid8(self): + equal = self.assertEqual + u = self.uuid.uuid8() + + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 8) + + for (_, hi, mid, lo) in product( + range(10), # repeat 10 times + [None, 0, random.getrandbits(48)], + [None, 0, random.getrandbits(12)], + [None, 0, random.getrandbits(62)], + ): + u = self.uuid.uuid8(hi, mid, lo) + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 8) + if hi is not None: + equal((u.int >> 80) & 0xffffffffffff, hi) + if mid is not None: + equal((u.int >> 64) & 0xfff, mid) + if lo is not None: + equal(u.int & 0x3fffffffffffffff, lo) + + def test_uuid8_uniqueness(self): + # Test that UUIDv8-generated values are unique (up to a negligible + # probability of failure). There are 122 bits of entropy and assuming + # that the underlying mt-19937-based random generator is sufficiently + # good, it is unlikely to have a collision of two UUIDs. + N = 1000 + uuids = {self.uuid.uuid8() for _ in range(N)} + self.assertEqual(len(uuids), N) + + versions = {u.version for u in uuids} + self.assertSetEqual(versions, {8}) + @support.requires_fork() def testIssue8621(self): # On at least some versions of OSX self.uuid.uuid4 generates diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index f60c662d322e38..6e23097deaf221 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -17,13 +17,15 @@ import sys import sysconfig import tempfile +import shlex from test.support import (captured_stdout, captured_stderr, skip_if_broken_multiprocessing_synchronize, verbose, requires_subprocess, is_android, is_apple_mobile, is_emscripten, is_wasi, requires_venv_with_pip, TEST_HOME_DIR, requires_resource, copy_python_src_ignore) -from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) +from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree, + TESTFN, FakePath) import unittest import venv from unittest.mock import patch, Mock @@ -74,7 +76,7 @@ def setUp(self): self.include = 'Include' else: self.bindir = 'bin' - self.lib = ('lib', 'python%d.%d' % sys.version_info[:2]) + self.lib = ('lib', f'python{sysconfig._get_python_version_abi()}') self.include = 'include' executable = sys._base_executable self.exe = os.path.split(executable)[-1] @@ -124,12 +126,12 @@ def test_defaults_with_str_path(self): self.run_with_capture(venv.create, self.env_dir) self._check_output_of_default_create() - def test_defaults_with_pathlib_path(self): + def test_defaults_with_pathlike(self): """ - Test the create function with default arguments and a pathlib.Path path. + Test the create function with default arguments and a path-like path. """ rmtree(self.env_dir) - self.run_with_capture(venv.create, pathlib.Path(self.env_dir)) + self.run_with_capture(venv.create, FakePath(self.env_dir)) self._check_output_of_default_create() def _check_output_of_default_create(self): @@ -273,7 +275,8 @@ def test_prefixes(self): ('base_exec_prefix', sys.base_exec_prefix)): cmd[2] = 'import sys; print(sys.%s)' % prefix out, err = check_output(cmd) - self.assertEqual(out.strip(), expected.encode(), prefix) + self.assertEqual(pathlib.Path(out.strip().decode()), + pathlib.Path(expected), prefix) @requireVenvCreate def test_sysconfig(self): @@ -486,6 +489,82 @@ def test_executable_symlinks(self): 'import sys; print(sys.executable)']) self.assertEqual(out.strip(), envpy.encode()) + # gh-124651: test quoted strings + @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows') + def test_special_chars_bash(self): + """ + Test that the template strings are quoted properly (bash) + """ + rmtree(self.env_dir) + bash = shutil.which('bash') + if bash is None: + self.skipTest('bash required for this test') + env_name = '"\';&&$e|\'"' + env_dir = os.path.join(os.path.realpath(self.env_dir), env_name) + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate') + test_script = os.path.join(self.env_dir, 'test_special_chars.sh') + with open(test_script, "w") as f: + f.write(f'source {shlex.quote(activate)}\n' + 'python -c \'import sys; print(sys.executable)\'\n' + 'python -c \'import os; print(os.environ["VIRTUAL_ENV"])\'\n' + 'deactivate\n') + out, err = check_output([bash, test_script]) + lines = out.splitlines() + self.assertTrue(env_name.encode() in lines[0]) + self.assertEndsWith(lines[1], env_name.encode()) + + # gh-124651: test quoted strings + @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows') + def test_special_chars_csh(self): + """ + Test that the template strings are quoted properly (csh) + """ + rmtree(self.env_dir) + csh = shutil.which('tcsh') or shutil.which('csh') + if csh is None: + self.skipTest('csh required for this test') + env_name = '"\';&&$e|\'"' + env_dir = os.path.join(os.path.realpath(self.env_dir), env_name) + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate.csh') + test_script = os.path.join(self.env_dir, 'test_special_chars.csh') + with open(test_script, "w") as f: + f.write(f'source {shlex.quote(activate)}\n' + 'python -c \'import sys; print(sys.executable)\'\n' + 'python -c \'import os; print(os.environ["VIRTUAL_ENV"])\'\n' + 'deactivate\n') + out, err = check_output([csh, test_script]) + lines = out.splitlines() + self.assertTrue(env_name.encode() in lines[0]) + self.assertEndsWith(lines[1], env_name.encode()) + + # gh-124651: test quoted strings on Windows + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') + def test_special_chars_windows(self): + """ + Test that the template strings are quoted properly on Windows + """ + rmtree(self.env_dir) + env_name = "'&&^$e" + env_dir = os.path.join(os.path.realpath(self.env_dir), env_name) + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate.bat') + test_batch = os.path.join(self.env_dir, 'test_special_chars.bat') + with open(test_batch, "w") as f: + f.write('@echo off\n' + f'"{activate}" & ' + f'{self.exe} -c "import sys; print(sys.executable)" & ' + f'{self.exe} -c "import os; print(os.environ[\'VIRTUAL_ENV\'])" & ' + 'deactivate') + out, err = check_output([test_batch]) + lines = out.splitlines() + self.assertTrue(env_name.encode() in lines[0]) + self.assertEndsWith(lines[1], env_name.encode()) + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') def test_unicode_in_batch_file(self): """ @@ -502,6 +581,21 @@ def test_unicode_in_batch_file(self): ) self.assertEqual(out.strip(), '0') + @unittest.skipUnless(os.name == 'nt' and can_symlink(), + 'symlinks on Windows') + def test_failed_symlink(self): + """ + Test handling of failed symlinks on Windows. + """ + rmtree(self.env_dir) + env_dir = os.path.join(os.path.realpath(self.env_dir), 'venv') + with patch('os.symlink') as mock_symlink: + mock_symlink.side_effect = OSError() + builder = venv.EnvBuilder(clear=True, symlinks=True) + _, err = self.run_with_capture(builder.create, env_dir) + filepath_regex = r"'[A-Z]:\\\\(?:[^\\\\]+\\\\)*[^\\\\]+'" + self.assertRegex(err, rf"Unable to symlink {filepath_regex} to {filepath_regex}") + @requireVenvCreate def test_multiprocessing(self): """ @@ -531,7 +625,7 @@ def test_multiprocessing_recursion(self): rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) script = os.path.join(TEST_HOME_DIR, '_test_venv_multiprocessing.py') - subprocess.check_call([self.envpy(real_env_dir=True), script]) + subprocess.check_call([self.envpy(real_env_dir=True), "-I", script]) @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') def test_deactivate_with_strict_bash_opts(self): @@ -570,7 +664,7 @@ def test_pathsep_error(self): rmtree(self.env_dir) bad_itempath = self.env_dir + os.pathsep self.assertRaises(ValueError, venv.create, bad_itempath) - self.assertRaises(ValueError, venv.create, pathlib.Path(bad_itempath)) + self.assertRaises(ValueError, venv.create, FakePath(bad_itempath)) @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') @requireVenvCreate @@ -591,7 +685,8 @@ def test_zippath_from_non_installed_posix(self): libdir = os.path.join(non_installed_dir, platlibdir, self.lib[1]) os.makedirs(libdir) landmark = os.path.join(libdir, "os.py") - stdlib_zip = "python%d%d.zip" % sys.version_info[:2] + abi_thread = "t" if sysconfig.get_config_var("Py_GIL_DISABLED") else "" + stdlib_zip = f"python{sys.version_info.major}{sys.version_info.minor}{abi_thread}" zip_landmark = os.path.join(non_installed_dir, platlibdir, stdlib_zip) @@ -743,6 +838,57 @@ def test_cli_without_scm_ignore_files(self): with self.assertRaises(FileNotFoundError): self.get_text_file_contents('.gitignore') + def test_venv_same_path(self): + same_path = venv.EnvBuilder._same_path + if sys.platform == 'win32': + # Case-insensitive, and handles short/long names + tests = [ + (True, TESTFN, TESTFN), + (True, TESTFN.lower(), TESTFN.upper()), + ] + import _winapi + # ProgramFiles is the most reliable path that will have short/long + progfiles = os.getenv('ProgramFiles') + if progfiles: + tests = [ + *tests, + (True, progfiles, progfiles), + (True, _winapi.GetShortPathName(progfiles), _winapi.GetLongPathName(progfiles)), + ] + else: + # Just a simple case-sensitive comparison + tests = [ + (True, TESTFN, TESTFN), + (False, TESTFN.lower(), TESTFN.upper()), + ] + for r, path1, path2 in tests: + with self.subTest(f"{path1}-{path2}"): + if r: + self.assertTrue(same_path(path1, path2)) + else: + self.assertFalse(same_path(path1, path2)) + + # gh-126084: venvwlauncher should run pythonw, not python + @requireVenvCreate + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') + def test_venvwlauncher(self): + """ + Test that the GUI launcher runs the GUI python. + """ + rmtree(self.env_dir) + venv.create(self.env_dir) + exename = self.exe + # Retain the debug suffix if present + if "python" in exename and not "pythonw" in exename: + exename = exename.replace("python", "pythonw") + envpyw = os.path.join(self.env_dir, self.bindir, exename) + try: + subprocess.check_call([envpyw, "-c", "import sys; " + "assert sys._base_executable.endswith('%s')" % exename]) + except subprocess.CalledProcessError: + self.fail("venvwlauncher.exe did not run %s" % exename) + + @requireVenvCreate class EnsurePipTest(BaseTest): """Test venv module installation of pip.""" @@ -853,6 +999,14 @@ def do_test_with_pip(self, system_site_packages): err = re.sub("^(WARNING: )?The directory .* or its parent directory " "is not owned or is not writable by the current user.*$", "", err, flags=re.MULTILINE) + # Ignore warning about missing optional module: + try: + import ssl + except ImportError: + err = re.sub( + "^WARNING: Disabling truststore since ssl support is missing$", + "", + err, flags=re.MULTILINE) self.assertEqual(err.rstrip(), "") # Being fairly specific regarding the expected behaviour for the # initial bundling phase in Python 3.4. If the output changes in diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 50b0f3fff04c57..4bd164b8a9a82b 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -1,6 +1,8 @@ from contextlib import contextmanager import linecache import os +import importlib +import inspect from io import StringIO import re import sys @@ -12,6 +14,7 @@ from test.support import import_helper from test.support import os_helper from test.support import warnings_helper +from test.support import force_not_colorized from test.support.script_helper import assert_python_ok, assert_python_failure from test.test_warnings.data import package_helper @@ -154,40 +157,42 @@ def f(): f() self.assertEqual(len(w), 1) - def test_always(self): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: - self.module.resetwarnings() - self.module.filterwarnings("always", category=UserWarning) - message = "FilterTests.test_always" - def f(): - self.module.warn(message, UserWarning) - f() - self.assertEqual(len(w), 1) - self.assertEqual(w[-1].message.args[0], message) - f() - self.assertEqual(len(w), 2) - self.assertEqual(w[-1].message.args[0], message) + def test_always_and_all(self): + for mode in {"always", "all"}: + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.resetwarnings() + self.module.filterwarnings(mode, category=UserWarning) + message = "FilterTests.test_always_and_all" + def f(): + self.module.warn(message, UserWarning) + f() + self.assertEqual(len(w), 1) + self.assertEqual(w[-1].message.args[0], message) + f() + self.assertEqual(len(w), 2) + self.assertEqual(w[-1].message.args[0], message) - def test_always_after_default(self): - with original_warnings.catch_warnings(record=True, - module=self.module) as w: - self.module.resetwarnings() - message = "FilterTests.test_always_after_ignore" - def f(): - self.module.warn(message, UserWarning) - f() - self.assertEqual(len(w), 1) - self.assertEqual(w[-1].message.args[0], message) - f() - self.assertEqual(len(w), 1) - self.module.filterwarnings("always", category=UserWarning) - f() - self.assertEqual(len(w), 2) - self.assertEqual(w[-1].message.args[0], message) - f() - self.assertEqual(len(w), 3) - self.assertEqual(w[-1].message.args[0], message) + def test_always_and_all_after_default(self): + for mode in {"always", "all"}: + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.resetwarnings() + message = "FilterTests.test_always_and_all_after_ignore" + def f(): + self.module.warn(message, UserWarning) + f() + self.assertEqual(len(w), 1) + self.assertEqual(w[-1].message.args[0], message) + f() + self.assertEqual(len(w), 1) + self.module.filterwarnings(mode, category=UserWarning) + f() + self.assertEqual(len(w), 2) + self.assertEqual(w[-1].message.args[0], message) + f() + self.assertEqual(len(w), 3) + self.assertEqual(w[-1].message.args[0], message) def test_default(self): with original_warnings.catch_warnings(record=True, @@ -489,7 +494,7 @@ def test_stacklevel(self): warning_tests.inner("spam7", stacklevel=9999) self.assertEqual(os.path.basename(w[-1].filename), - "sys") + "") def test_stacklevel_import(self): # Issue #24305: With stacklevel=2, module-level warnings should work. @@ -498,7 +503,7 @@ def test_stacklevel_import(self): with original_warnings.catch_warnings(record=True, module=self.module) as w: self.module.simplefilter('always') - import test.test_warnings.data.import_warning + import test.test_warnings.data.import_warning # noqa: F401 self.assertEqual(len(w), 1) self.assertEqual(w[0].filename, __file__) @@ -528,6 +533,18 @@ def test_skip_file_prefixes(self): warning_tests.package("prefix02", stacklevel=3) self.assertIn("unittest", w[-1].filename) + def test_skip_file_prefixes_file_path(self): + # see: gh-126209 + with warnings_state(self.module): + skipped = warning_tests.__file__ + with original_warnings.catch_warnings( + record=True, module=self.module, + ) as w: + warning_tests.outer("msg", skip_file_prefixes=(skipped,)) + + self.assertEqual(len(w), 1) + self.assertNotEqual(w[-1].filename, skipped) + def test_skip_file_prefixes_type_errors(self): with warnings_state(self.module): warn = warning_tests.warnings.warn @@ -635,6 +652,97 @@ class NonWarningSubclass: self.module.warn('good warning category', MyWarningClass) self.assertIsInstance(cm.warning, Warning) + def check_module_globals(self, module_globals): + with original_warnings.catch_warnings(module=self.module, record=True) as w: + self.module.filterwarnings('default') + self.module.warn_explicit( + 'eggs', UserWarning, 'bar', 1, + module_globals=module_globals) + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, UserWarning) + self.assertEqual(str(w[0].message), 'eggs') + + def check_module_globals_error(self, module_globals, errmsg, errtype=ValueError): + if self.module is py_warnings: + self.check_module_globals(module_globals) + return + with original_warnings.catch_warnings(module=self.module, record=True) as w: + self.module.filterwarnings('always') + with self.assertRaisesRegex(errtype, re.escape(errmsg)): + self.module.warn_explicit( + 'eggs', UserWarning, 'bar', 1, + module_globals=module_globals) + self.assertEqual(len(w), 0) + + def check_module_globals_deprecated(self, module_globals, msg): + if self.module is py_warnings: + self.check_module_globals(module_globals) + return + with original_warnings.catch_warnings(module=self.module, record=True) as w: + self.module.filterwarnings('always') + self.module.warn_explicit( + 'eggs', UserWarning, 'bar', 1, + module_globals=module_globals) + self.assertEqual(len(w), 2) + self.assertEqual(w[0].category, DeprecationWarning) + self.assertEqual(str(w[0].message), msg) + self.assertEqual(w[1].category, UserWarning) + self.assertEqual(str(w[1].message), 'eggs') + + def test_gh86298_no_loader_and_no_spec(self): + self.check_module_globals({'__name__': 'bar'}) + + def test_gh86298_loader_is_none_and_no_spec(self): + self.check_module_globals({'__name__': 'bar', '__loader__': None}) + + def test_gh86298_no_loader_and_spec_is_none(self): + self.check_module_globals_error( + {'__name__': 'bar', '__spec__': None}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_loader_is_none_and_spec_is_none(self): + self.check_module_globals_error( + {'__name__': 'bar', '__loader__': None, '__spec__': None}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_loader_is_none_and_spec_loader_is_none(self): + self.check_module_globals_error( + {'__name__': 'bar', '__loader__': None, + '__spec__': types.SimpleNamespace(loader=None)}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_no_spec(self): + self.check_module_globals_deprecated( + {'__name__': 'bar', '__loader__': object()}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_spec_is_none(self): + self.check_module_globals_deprecated( + {'__name__': 'bar', '__loader__': object(), '__spec__': None}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_no_spec_loader(self): + self.check_module_globals_deprecated( + {'__name__': 'bar', '__loader__': object(), + '__spec__': types.SimpleNamespace()}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_loader_and_spec_loader_disagree(self): + self.check_module_globals_deprecated( + {'__name__': 'bar', '__loader__': object(), + '__spec__': types.SimpleNamespace(loader=object())}, + 'Module globals; __loader__ != __spec__.loader') + + def test_gh86298_no_loader_and_no_spec_loader(self): + self.check_module_globals_error( + {'__name__': 'bar', '__spec__': types.SimpleNamespace()}, + 'Module globals is missing a __spec__.loader', AttributeError) + + def test_gh86298_no_loader_with_spec_loader_okay(self): + self.check_module_globals( + {'__name__': 'bar', + '__spec__': types.SimpleNamespace(loader=object())}) + class CWarnTests(WarnTests, unittest.TestCase): module = c_warnings @@ -883,37 +991,46 @@ def test_issue31285(self): # warn_explicit() should neither raise a SystemError nor cause an # assertion failure, in case the return value of get_source() has a # bad splitlines() method. - def get_bad_loader(splitlines_ret_val): + get_source_called = [] + def get_module_globals(*, splitlines_ret_val): + class BadSource(str): + def splitlines(self): + return splitlines_ret_val + class BadLoader: def get_source(self, fullname): - class BadSource(str): - def splitlines(self): - return splitlines_ret_val + get_source_called.append(splitlines_ret_val) return BadSource('spam') - return BadLoader() + + loader = BadLoader() + spec = importlib.machinery.ModuleSpec('foobar', loader) + return {'__loader__': loader, + '__spec__': spec, + '__name__': 'foobar'} + wmod = self.module with original_warnings.catch_warnings(module=wmod): wmod.filterwarnings('default', category=UserWarning) + linecache.clearcache() with support.captured_stderr() as stderr: wmod.warn_explicit( 'foo', UserWarning, 'bar', 1, - module_globals={'__loader__': get_bad_loader(42), - '__name__': 'foobar'}) + module_globals=get_module_globals(splitlines_ret_val=42)) self.assertIn('UserWarning: foo', stderr.getvalue()) + self.assertEqual(get_source_called, [42]) - show = wmod._showwarnmsg - try: + linecache.clearcache() + with support.swap_attr(wmod, '_showwarnmsg', None): del wmod._showwarnmsg with support.captured_stderr() as stderr: wmod.warn_explicit( 'eggs', UserWarning, 'bar', 1, - module_globals={'__loader__': get_bad_loader([42]), - '__name__': 'foobar'}) + module_globals=get_module_globals(splitlines_ret_val=[42])) self.assertIn('UserWarning: eggs', stderr.getvalue()) - finally: - wmod._showwarnmsg = show + self.assertEqual(get_source_called, [42, [42]]) + linecache.clearcache() @support.cpython_only def test_issue31411(self): @@ -1239,6 +1356,7 @@ def test_comma_separated_warnings(self): self.assertEqual(stdout, b"['ignore::DeprecationWarning', 'ignore::UnicodeWarning']") + @force_not_colorized def test_envvar_and_command_line(self): rc, stdout, stderr = assert_python_ok("-Wignore::UnicodeWarning", "-c", "import sys; sys.stdout.write(str(sys.warnoptions))", @@ -1247,6 +1365,7 @@ def test_envvar_and_command_line(self): self.assertEqual(stdout, b"['ignore::DeprecationWarning', 'ignore::UnicodeWarning']") + @force_not_colorized def test_conflicting_envvar_and_command_line(self): rc, stdout, stderr = assert_python_failure("-Werror::DeprecationWarning", "-c", "import sys, warnings; sys.stdout.write(str(sys.warnoptions)); " @@ -1388,7 +1507,7 @@ def test_late_resource_warning(self): # Issue #21925: Emitting a ResourceWarning late during the Python # shutdown must be logged. - expected = b"sys:1: ResourceWarning: unclosed file " + expected = b":0: ResourceWarning: unclosed file " # don't import the warnings module # (_warnings will try to import it) @@ -1402,7 +1521,7 @@ def test_late_resource_warning(self): self.assertTrue(err.startswith(expected), ascii(err)) -class DeprecatedTests(unittest.TestCase): +class DeprecatedTests(PyPublicAPITests): def test_dunder_deprecated(self): @deprecated("A will go away soon") class A: @@ -1679,6 +1798,29 @@ def d(): pass isinstance(cell.cell_contents, deprecated) for cell in d.__closure__ )) + def test_inspect(self): + @deprecated("depr") + def sync(): + pass + + @deprecated("depr") + async def coro(): + pass + + class Cls: + @deprecated("depr") + def sync(self): + pass + + @deprecated("depr") + async def coro(self): + pass + + self.assertFalse(inspect.iscoroutinefunction(sync)) + self.assertTrue(inspect.iscoroutinefunction(coro)) + self.assertFalse(inspect.iscoroutinefunction(Cls.sync)) + self.assertTrue(inspect.iscoroutinefunction(Cls.coro)) + def setUpModule(): py_warnings.onceregistry.clear() c_warnings.onceregistry.clear() diff --git a/Lib/test/test_warnings/data/stacklevel.py b/Lib/test/test_warnings/data/stacklevel.py index c6dd24733b3b74..fe36242d3d20c2 100644 --- a/Lib/test/test_warnings/data/stacklevel.py +++ b/Lib/test/test_warnings/data/stacklevel.py @@ -4,11 +4,13 @@ import warnings from test.test_warnings.data import package_helper -def outer(message, stacklevel=1): - inner(message, stacklevel) -def inner(message, stacklevel=1): - warnings.warn(message, stacklevel=stacklevel) +def outer(message, stacklevel=1, skip_file_prefixes=()): + inner(message, stacklevel, skip_file_prefixes) + +def inner(message, stacklevel=1, skip_file_prefixes=()): + warnings.warn(message, stacklevel=stacklevel, + skip_file_prefixes=skip_file_prefixes) def package(message, *, stacklevel): package_helper.inner_api(message, stacklevel=stacklevel, diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 4cdd66d3769e0c..4faad6629fe23c 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -10,11 +10,14 @@ import threading import time import random +import textwrap from test import support from test.support import script_helper, ALWAYS_EQ from test.support import gc_collect +from test.support import import_helper from test.support import threading_helper +from test.support import is_wasi, Py_DEBUG # Used in ReferencesTestCase.test_ref_created_during_del() . ref_from_del = None @@ -79,7 +82,7 @@ def callback(self, ref): @contextlib.contextmanager -def collect_in_thread(period=0.0001): +def collect_in_thread(period=0.005): """ Ensure GC collections happen in a different thread, at a high frequency. """ @@ -116,6 +119,38 @@ def test_basic_ref(self): del o repr(wr) + @support.cpython_only + def test_ref_repr(self): + obj = C() + ref = weakref.ref(obj) + regex = ( + rf"" + ) + self.assertRegex(repr(ref), regex) + + obj = None + gc_collect() + self.assertRegex(repr(ref), + rf'') + + # test type with __name__ + class WithName: + @property + def __name__(self): + return "custom_name" + + obj2 = WithName() + ref2 = weakref.ref(obj2) + regex = ( + rf"" + ) + self.assertRegex(repr(ref2), regex) + def test_repr_failure_gh99184(self): class MyConfig(dict): def __getattr__(self, x): @@ -134,7 +169,7 @@ def test_basic_callback(self): @support.cpython_only def test_cfunction(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") create_cfunction = _testcapi.create_cfunction f = create_cfunction() wr = weakref.ref(f) @@ -195,6 +230,22 @@ def check(proxy): self.assertRaises(ReferenceError, bool, ref3) self.assertEqual(self.cbcalled, 2) + @support.cpython_only + def test_proxy_repr(self): + obj = C() + ref = weakref.proxy(obj, self.callback) + regex = ( + rf"" + ) + self.assertRegex(repr(ref), regex) + + obj = None + gc_collect() + self.assertRegex(repr(ref), + rf'') + def check_basic_ref(self, factory): o = factory() ref = weakref.ref(o) @@ -914,6 +965,7 @@ def test_hashing(self): self.assertEqual(hash(a), hash(42)) self.assertRaises(TypeError, hash, b) + @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") def test_trashcan_16602(self): # Issue #16602: when a weakref's target was part of a long # deallocation chain, the trashcan mechanism could delay clearing @@ -967,6 +1019,31 @@ def __del__(self): pass del x support.gc_collect() + @support.cpython_only + def test_no_memory_when_clearing(self): + # gh-118331: Make sure we do not raise an exception from the destructor + # when clearing weakrefs if allocating the intermediate tuple fails. + code = textwrap.dedent(""" + import _testcapi + import weakref + + class TestObj: + pass + + def callback(obj): + pass + + obj = TestObj() + # The choice of 50 is arbitrary, but must be large enough to ensure + # the allocation won't be serviced by the free list. + wrs = [weakref.ref(obj, callback) for _ in range(50)] + _testcapi.set_nomemory(0) + del obj + """).strip() + res, _ = script_helper.run_python_until_end("-c", code) + stderr = res.err.decode("ascii", "backslashreplace") + self.assertNotRegex(stderr, "_Py_Dealloc: Deallocator of type 'TestObj'") + class SubclassableWeakrefTestCase(TestBase): @@ -1213,6 +1290,12 @@ class MappingTestCase(TestBase): COUNT = 10 + if support.check_sanitizer(thread=True) and support.Py_GIL_DISABLED: + # Reduce iteration count to get acceptable latency + NUM_THREADED_ITERATIONS = 1000 + else: + NUM_THREADED_ITERATIONS = 100000 + def check_len_cycles(self, dict_type, cons): N = 20 items = [RefCycle() for i in range(N)] @@ -1838,7 +1921,7 @@ def test_make_weak_keyed_dict_repr(self): def test_threaded_weak_valued_setdefault(self): d = weakref.WeakValueDictionary() with collect_in_thread(): - for i in range(100000): + for i in range(self.NUM_THREADED_ITERATIONS): x = d.setdefault(10, RefCycle()) self.assertIsNot(x, None) # we never put None in there! del x @@ -1847,7 +1930,7 @@ def test_threaded_weak_valued_setdefault(self): def test_threaded_weak_valued_pop(self): d = weakref.WeakValueDictionary() with collect_in_thread(): - for i in range(100000): + for i in range(self.NUM_THREADED_ITERATIONS): d[10] = RefCycle() x = d.pop(10, 10) self.assertIsNot(x, None) # we never put None in there! @@ -1858,13 +1941,32 @@ def test_threaded_weak_valued_consistency(self): # WeakValueDictionary when collecting from another thread. d = weakref.WeakValueDictionary() with collect_in_thread(): - for i in range(200000): + for i in range(2 * self.NUM_THREADED_ITERATIONS): o = RefCycle() d[10] = o # o is still alive, so the dict can't be empty self.assertEqual(len(d), 1) o = None # lose ref + @support.cpython_only + def test_weak_valued_consistency(self): + # A single-threaded, deterministic repro for issue #28427: old keys + # should not remove new values from WeakValueDictionary. This relies on + # an implementation detail of CPython's WeakValueDictionary (its + # underlying dictionary of KeyedRefs) to reproduce the issue. + d = weakref.WeakValueDictionary() + with support.disable_gc(): + d[10] = RefCycle() + # Keep the KeyedRef alive after it's replaced so that GC will invoke + # the callback. + wr = d.data[10] + # Replace the value with something that isn't cyclic garbage + o = RefCycle() + d[10] = o + # Trigger GC, which will invoke the callback for `wr` + gc.collect() + self.assertEqual(len(d), 1) + def check_threaded_weak_dict_copy(self, type_, deepcopy): # `type_` should be either WeakKeyDictionary or WeakValueDictionary. # `deepcopy` should be either True or False. @@ -1927,6 +2029,7 @@ def pop_and_collect(lst): raise exc[0] @threading_helper.requires_working_threading() + @support.requires_resource('cpu') def test_threaded_weak_key_dict_copy(self): # Issue #35615: Weakref keys or values getting GC'ed during dict # copying should not result in a crash. @@ -1940,6 +2043,7 @@ def test_threaded_weak_key_dict_deepcopy(self): self.check_threaded_weak_dict_copy(weakref.WeakKeyDictionary, True) @threading_helper.requires_working_threading() + @support.requires_resource('cpu') def test_threaded_weak_value_dict_copy(self): # Issue #35615: Weakref keys or values getting GC'ed during dict # copying should not result in a crash. diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index 8c074cb28a87e3..4fcbc5c2e59ea3 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -1,15 +1,20 @@ -import webbrowser -import unittest import os -import sys +import re +import shlex import subprocess -from unittest import mock +import sys +import unittest +import webbrowser from test import support from test.support import import_helper +from test.support import is_apple_mobile from test.support import os_helper +from test.support import requires_subprocess +from test.support import threading_helper +from unittest import mock -if not support.has_subprocess_support: - raise unittest.SkipTest("test webserver requires subprocess") +# The webbrowser module uses threading locks +threading_helper.requires_working_threading(module=True) URL = 'https://www.example.com' CMD_NAME = 'test' @@ -24,6 +29,7 @@ def wait(self, seconds=None): return 0 +@requires_subprocess() class CommandTestMixin: def _test(self, meth, *, args=[URL], kw={}, options, arguments): @@ -94,6 +100,15 @@ def test_open_new_tab(self): options=[], arguments=[URL]) + def test_open_bad_new_parameter(self): + with self.assertRaisesRegex(webbrowser.Error, + re.escape("Bad 'new' parameter to open(); " + "expected 0, 1, or 2, got 999")): + self._test('open', + options=[], + arguments=[URL], + kw=dict(new=999)) + class EdgeCommandTest(CommandTestMixin, unittest.TestCase): @@ -201,22 +216,89 @@ class ELinksCommandTest(CommandTestMixin, unittest.TestCase): def test_open(self): self._test('open', options=['-remote'], - arguments=['openURL({})'.format(URL)]) + arguments=[f'openURL({URL})']) def test_open_with_autoraise_false(self): self._test('open', options=['-remote'], - arguments=['openURL({})'.format(URL)]) + arguments=[f'openURL({URL})']) def test_open_new(self): self._test('open_new', options=['-remote'], - arguments=['openURL({},new-window)'.format(URL)]) + arguments=[f'openURL({URL},new-window)']) def test_open_new_tab(self): self._test('open_new_tab', options=['-remote'], - arguments=['openURL({},new-tab)'.format(URL)]) + arguments=[f'openURL({URL},new-tab)']) + + +@unittest.skipUnless(sys.platform == "ios", "Test only applicable to iOS") +class IOSBrowserTest(unittest.TestCase): + def _obj_ref(self, *args): + # Construct a string representation of the arguments that can be used + # as a proxy for object instance references + return "|".join(str(a) for a in args) + + @unittest.skipIf(getattr(webbrowser, "objc", None) is None, + "iOS Webbrowser tests require ctypes") + def setUp(self): + # Intercept the objc library. Wrap the calls to get the + # references to classes and selectors to return strings, and + # wrap msgSend to return stringified object references + self.orig_objc = webbrowser.objc + + webbrowser.objc = mock.Mock() + webbrowser.objc.objc_getClass = lambda cls: f"C#{cls.decode()}" + webbrowser.objc.sel_registerName = lambda sel: f"S#{sel.decode()}" + webbrowser.objc.objc_msgSend.side_effect = self._obj_ref + + def tearDown(self): + webbrowser.objc = self.orig_objc + + def _test(self, meth, **kwargs): + # The browser always gets focus, there's no concept of separate browser + # windows, and there's no API-level control over creating a new tab. + # Therefore, all calls to webbrowser are effectively the same. + getattr(webbrowser, meth)(URL, **kwargs) + + # The ObjC String version of the URL is created with UTF-8 encoding + url_string_args = [ + "C#NSString", + "S#stringWithCString:encoding:", + b'https://www.example.com', + 4, + ] + # The NSURL version of the URL is created from that string + url_obj_args = [ + "C#NSURL", + "S#URLWithString:", + self._obj_ref(*url_string_args), + ] + # The openURL call is invoked on the shared application + shared_app_args = ["C#UIApplication", "S#sharedApplication"] + + # Verify that the last call is the one that opens the URL. + webbrowser.objc.objc_msgSend.assert_called_with( + self._obj_ref(*shared_app_args), + "S#openURL:options:completionHandler:", + self._obj_ref(*url_obj_args), + None, + None + ) + + def test_open(self): + self._test('open') + + def test_open_with_autoraise_false(self): + self._test('open', autoraise=False) + + def test_open_new(self): + self._test('open_new') + + def test_open_new_tab(self): + self._test('open_new_tab') class BrowserRegistrationTest(unittest.TestCase): @@ -271,7 +353,6 @@ def test_register_default(self): def test_register_preferred(self): self._check_registration(preferred=True) - @unittest.skipUnless(sys.platform == "darwin", "macOS specific test") def test_no_xdg_settings_on_macOS(self): # On macOS webbrowser should not use xdg-settings to @@ -314,6 +395,10 @@ def test_synthesize(self): webbrowser.register(name, None, webbrowser.GenericBrowser(name)) webbrowser.get(sys.executable) + @unittest.skipIf( + is_apple_mobile, + "Apple mobile doesn't allow modifying browser with environment" + ) def test_environment(self): webbrowser = import_helper.import_fresh_module('webbrowser') try: @@ -325,6 +410,10 @@ def test_environment(self): webbrowser = import_helper.import_fresh_module('webbrowser') webbrowser.get() + @unittest.skipIf( + is_apple_mobile, + "Apple mobile doesn't allow modifying browser with environment" + ) def test_environment_preferred(self): webbrowser = import_helper.import_fresh_module('webbrowser') try: @@ -344,5 +433,74 @@ def test_environment_preferred(self): self.assertEqual(webbrowser.get().name, sys.executable) -if __name__=='__main__': +class CliTest(unittest.TestCase): + def test_parse_args(self): + for command, url, new_win in [ + # No optional arguments + ("https://example.com", "https://example.com", 0), + # Each optional argument + ("https://example.com -n", "https://example.com", 1), + ("-n https://example.com", "https://example.com", 1), + ("https://example.com -t", "https://example.com", 2), + ("-t https://example.com", "https://example.com", 2), + # Long form + ("https://example.com --new-window", "https://example.com", 1), + ("--new-window https://example.com", "https://example.com", 1), + ("https://example.com --new-tab", "https://example.com", 2), + ("--new-tab https://example.com", "https://example.com", 2), + ]: + args = webbrowser.parse_args(shlex.split(command)) + + self.assertEqual(args.url, url) + self.assertEqual(args.new_win, new_win) + + def test_parse_args_error(self): + for command in [ + # Arguments must not both be given + "https://example.com -n -t", + "https://example.com --new-window --new-tab", + "https://example.com -n --new-tab", + "https://example.com --new-window -t", + ]: + with support.captured_stderr() as stderr: + with self.assertRaises(SystemExit): + webbrowser.parse_args(shlex.split(command)) + self.assertIn( + 'error: argument -t/--new-tab: not allowed with argument -n/--new-window', + stderr.getvalue(), + ) + + # Ensure ambiguous shortening fails + with support.captured_stderr() as stderr: + with self.assertRaises(SystemExit): + webbrowser.parse_args(shlex.split("https://example.com --new")) + self.assertIn( + 'error: ambiguous option: --new could match --new-window, --new-tab', + stderr.getvalue() + ) + + def test_main(self): + for command, expected_url, expected_new_win in [ + # No optional arguments + ("https://example.com", "https://example.com", 0), + # Each optional argument + ("https://example.com -n", "https://example.com", 1), + ("-n https://example.com", "https://example.com", 1), + ("https://example.com -t", "https://example.com", 2), + ("-t https://example.com", "https://example.com", 2), + # Long form + ("https://example.com --new-window", "https://example.com", 1), + ("--new-window https://example.com", "https://example.com", 1), + ("https://example.com --new-tab", "https://example.com", 2), + ("--new-tab https://example.com", "https://example.com", 2), + ]: + with ( + mock.patch("webbrowser.open", return_value=None) as mock_open, + mock.patch("builtins.print", return_value=None), + ): + webbrowser.main(shlex.split(command)) + mock_open.assert_called_once_with(expected_url, expected_new_win) + + +if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_winapi.py b/Lib/test/test_winapi.py index 014aeea7239e2b..e64208330ad2f9 100644 --- a/Lib/test/test_winapi.py +++ b/Lib/test/test_winapi.py @@ -1,10 +1,10 @@ # Test the Windows-only _winapi module -import random -import threading -import time +import os +import pathlib +import re import unittest -from test.support import import_helper +from test.support import import_helper, os_helper _winapi = import_helper.import_module('_winapi', required_on=['win']) @@ -92,3 +92,67 @@ def test_many_events_waitany(self): def test_max_events_waitany(self): self._events_waitany_test(MAXIMUM_BATCHED_WAIT_OBJECTS) + + +class WinAPITests(unittest.TestCase): + def test_getlongpathname(self): + testfn = pathlib.Path(os.getenv("ProgramFiles")).parents[-1] / "PROGRA~1" + if not os.path.isdir(testfn): + raise unittest.SkipTest("require x:\\PROGRA~1 to test") + + # pathlib.Path will be rejected - only str is accepted + with self.assertRaises(TypeError): + _winapi.GetLongPathName(testfn) + + actual = _winapi.GetLongPathName(os.fsdecode(testfn)) + + # Can't assume that PROGRA~1 expands to any particular variation, so + # ensure it matches any one of them. + candidates = set(testfn.parent.glob("Progra*")) + self.assertIn(pathlib.Path(actual), candidates) + + def test_getshortpathname(self): + testfn = pathlib.Path(os.getenv("ProgramFiles")) + if not os.path.isdir(testfn): + raise unittest.SkipTest("require '%ProgramFiles%' to test") + + # pathlib.Path will be rejected - only str is accepted + with self.assertRaises(TypeError): + _winapi.GetShortPathName(testfn) + + actual = _winapi.GetShortPathName(os.fsdecode(testfn)) + + # Should contain "PROGRA~" but we can't predict the number + self.assertIsNotNone(re.match(r".\:\\PROGRA~\d", actual.upper()), actual) + + def test_namedpipe(self): + pipe_name = rf"\\.\pipe\LOCAL\{os_helper.TESTFN}" + + # Pipe does not exist, so this raises + with self.assertRaises(FileNotFoundError): + _winapi.WaitNamedPipe(pipe_name, 0) + + pipe = _winapi.CreateNamedPipe( + pipe_name, + _winapi.PIPE_ACCESS_DUPLEX, + 8, # 8=PIPE_REJECT_REMOTE_CLIENTS + 2, # two instances available + 32, 32, 0, 0) + self.addCleanup(_winapi.CloseHandle, pipe) + + # Pipe instance is available, so this passes + _winapi.WaitNamedPipe(pipe_name, 0) + + with open(pipe_name, 'w+b') as pipe2: + # No instances available, so this times out + # (WinError 121 does not get mapped to TimeoutError) + with self.assertRaises(OSError): + _winapi.WaitNamedPipe(pipe_name, 0) + + _winapi.WriteFile(pipe, b'testdata') + self.assertEqual(b'testdata', pipe2.read(8)) + + self.assertEqual((b'', 0), _winapi.PeekNamedPipe(pipe, 8)[:2]) + pipe2.write(b'testdata') + pipe2.flush() + self.assertEqual((b'testdata', 8), _winapi.PeekNamedPipe(pipe, 8)[:2]) diff --git a/Lib/test/test_winconsoleio.py b/Lib/test/test_winconsoleio.py index 209e4464e1a5c0..d9076e77c158a2 100644 --- a/Lib/test/test_winconsoleio.py +++ b/Lib/test/test_winconsoleio.py @@ -43,6 +43,9 @@ def test_open_fd(self): self.assertEqual(0, f.fileno()) f.close() # multiple close should not crash f.close() + with self.assertWarns(RuntimeWarning): + with ConIO(False): + pass try: f = ConIO(1, 'w') @@ -55,6 +58,9 @@ def test_open_fd(self): self.assertEqual(1, f.fileno()) f.close() f.close() + with self.assertWarns(RuntimeWarning): + with ConIO(False): + pass try: f = ConIO(2, 'w') @@ -136,6 +142,29 @@ def test_write_empty_data(self): with ConIO('CONOUT$', 'w') as f: self.assertEqual(f.write(b''), 0) + @requires_resource('console') + def test_write(self): + testcases = [] + with ConIO('CONOUT$', 'w') as f: + for a in [ + b'', + b'abc', + b'\xc2\xa7\xe2\x98\x83\xf0\x9f\x90\x8d', + b'\xff'*10, + ]: + for b in b'\xc2\xa7', b'\xe2\x98\x83', b'\xf0\x9f\x90\x8d': + testcases.append(a + b) + for i in range(1, len(b)): + data = a + b[:i] + testcases.append(data + b'z') + testcases.append(data + b'\xff') + # incomplete multibyte sequence + with self.subTest(data=data): + self.assertEqual(f.write(data), len(a)) + for data in testcases: + with self.subTest(data=data): + self.assertEqual(f.write(data), len(data)) + def assertStdinRoundTrip(self, text): stdin = open('CONIN$', 'r') old_stdin = sys.stdin diff --git a/Lib/test/test_winsound.py b/Lib/test/test_winsound.py index a59d0d24f5db48..870ab7bd41d8ce 100644 --- a/Lib/test/test_winsound.py +++ b/Lib/test/test_winsound.py @@ -1,12 +1,13 @@ # Ridiculously simple test of the winsound module for Windows. import functools -import pathlib +import os import time import unittest from test import support from test.support import import_helper +from test.support import os_helper support.requires('audio') @@ -85,13 +86,6 @@ def test_keyword_args(self): safe_MessageBeep(type=winsound.MB_OK) -# A class for testing winsound when the given path resolves -# to bytes rather than str. -class BytesPath(pathlib.WindowsPath): - def __fspath__(self): - return bytes(super().__fspath__(), 'UTF-8') - - class PlaySoundTest(unittest.TestCase): def test_errors(self): @@ -126,7 +120,7 @@ def test_snd_filename(self): def test_snd_filepath(self): fn = support.findfile('pluck-pcm8.wav', subdir='audiodata') - path = pathlib.Path(fn) + path = os_helper.FakePath(fn) safe_PlaySound(path, winsound.SND_FILENAME | winsound.SND_NODEFAULT) def test_snd_filepath_as_bytes(self): @@ -134,7 +128,7 @@ def test_snd_filepath_as_bytes(self): self.assertRaises( TypeError, winsound.PlaySound, - BytesPath(fn), + os_helper.FakePath(os.fsencode(fn)), winsound.SND_FILENAME | winsound.SND_NODEFAULT ) diff --git a/Lib/test/test_with.py b/Lib/test/test_with.py index d81902327a7e0a..e3e2de09496728 100644 --- a/Lib/test/test_with.py +++ b/Lib/test/test_with.py @@ -5,6 +5,7 @@ __email__ = "mbland at acm dot org" import sys +import traceback import unittest from collections import deque from contextlib import _GeneratorContextManager, contextmanager, nullcontext @@ -170,7 +171,10 @@ def __exit__(self, *args): def shouldThrow(): ct = EnterThrows() self.foo = None - with ct as self.foo: + # Ruff complains that we're redefining `self.foo` here, + # but the whole point of the test is to check that `self.foo` + # is *not* redefined (because `__enter__` raises) + with ct as self.foo: # ruff: noqa: F811 pass self.assertRaises(RuntimeError, shouldThrow) self.assertEqual(self.foo, None) @@ -251,7 +255,6 @@ def testInlineGeneratorBoundSyntax(self): self.assertAfterWithGeneratorInvariantsNoError(foo) def testInlineGeneratorBoundToExistingVariable(self): - foo = None with mock_contextmanager_generator() as foo: self.assertInWithGeneratorInvariants(foo) self.assertAfterWithGeneratorInvariantsNoError(foo) @@ -716,7 +719,7 @@ def testExceptionInExprList(self): try: with self.Dummy() as a, self.InitRaises(): pass - except: + except RuntimeError: pass self.assertTrue(a.enter_called) self.assertTrue(a.exit_called) @@ -749,5 +752,48 @@ def testEnterReturnsTuple(self): self.assertEqual(10, b1) self.assertEqual(20, b2) + def testExceptionLocation(self): + # The location of an exception raised from + # __init__, __enter__ or __exit__ of a context + # manager should be just the context manager expression, + # pinpointing the precise context manager in case there + # is more than one. + + def init_raises(): + try: + with self.Dummy(), self.InitRaises() as cm, self.Dummy() as d: + pass + except Exception as e: + return e + + def enter_raises(): + try: + with self.EnterRaises(), self.Dummy() as d: + pass + except Exception as e: + return e + + def exit_raises(): + try: + with self.ExitRaises(), self.Dummy() as d: + pass + except Exception as e: + return e + + for func, expected in [(init_raises, "self.InitRaises()"), + (enter_raises, "self.EnterRaises()"), + (exit_raises, "self.ExitRaises()"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_wmi.py b/Lib/test/test_wmi.py index bf8c52e646dc18..f667926d1f8ddf 100644 --- a/Lib/test/test_wmi.py +++ b/Lib/test/test_wmi.py @@ -14,11 +14,13 @@ def wmi_exec_query(query): # gh-112278: WMI maybe slow response when first call. try: return _wmi.exec_query(query) + except BrokenPipeError: + pass except WindowsError as e: if e.winerror != 258: raise - time.sleep(LOOPBACK_TIMEOUT) - return _wmi.exec_query(query) + time.sleep(LOOPBACK_TIMEOUT) + return _wmi.exec_query(query) class WmiTests(unittest.TestCase): diff --git a/Lib/test/test_wsgiref.py b/Lib/test/test_wsgiref.py index 9316d0ecbcf1ae..b047f7b06f85d3 100644 --- a/Lib/test/test_wsgiref.py +++ b/Lib/test/test_wsgiref.py @@ -137,7 +137,7 @@ def test_environ(self): def test_request_length(self): out, err = run_amock(data=b"GET " + (b"x" * 65537) + b" HTTP/1.0\n\n") self.assertEqual(out.splitlines()[0], - b"HTTP/1.0 414 Request-URI Too Long") + b"HTTP/1.0 414 URI Too Long") def test_validated_hello(self): out, err = run_amock(validator(hello_app)) diff --git a/Lib/test/test_xml_dom_xmlbuilder.py b/Lib/test/test_xml_dom_xmlbuilder.py new file mode 100644 index 00000000000000..5f5f2eb328df9f --- /dev/null +++ b/Lib/test/test_xml_dom_xmlbuilder.py @@ -0,0 +1,88 @@ +import io +import unittest +from http import client +from test.test_httplib import FakeSocket +from unittest import mock +from xml.dom import getDOMImplementation, minidom, xmlbuilder + +SMALL_SAMPLE = b""" + + +Introduction to XSL +
    +

    A. Namespace

    +""" + + +class XMLBuilderTest(unittest.TestCase): + def test_entity_resolver(self): + body = ( + b"HTTP/1.1 200 OK\r\nContent-Type: text/xml; charset=utf-8\r\n\r\n" + + SMALL_SAMPLE + ) + + sock = FakeSocket(body) + response = client.HTTPResponse(sock) + response.begin() + attrs = {"open.return_value": response} + opener = mock.Mock(**attrs) + + resolver = xmlbuilder.DOMEntityResolver() + + with mock.patch("urllib.request.build_opener") as mock_build: + mock_build.return_value = opener + source = resolver.resolveEntity(None, "http://example.com/2000/svg") + + self.assertIsInstance(source, xmlbuilder.DOMInputSource) + self.assertIsNone(source.publicId) + self.assertEqual(source.systemId, "http://example.com/2000/svg") + self.assertEqual(source.baseURI, "http://example.com/2000/") + self.assertEqual(source.encoding, "utf-8") + self.assertIs(source.byteStream, response) + + self.assertIsNone(source.characterStream) + self.assertIsNone(source.stringData) + + def test_builder(self): + imp = getDOMImplementation() + self.assertIsInstance(imp, xmlbuilder.DOMImplementationLS) + + builder = imp.createDOMBuilder(imp.MODE_SYNCHRONOUS, None) + self.assertIsInstance(builder, xmlbuilder.DOMBuilder) + + def test_parse_uri(self): + body = ( + b"HTTP/1.1 200 OK\r\nContent-Type: text/xml; charset=utf-8\r\n\r\n" + + SMALL_SAMPLE + ) + + sock = FakeSocket(body) + response = client.HTTPResponse(sock) + response.begin() + attrs = {"open.return_value": response} + opener = mock.Mock(**attrs) + + with mock.patch("urllib.request.build_opener") as mock_build: + mock_build.return_value = opener + + imp = getDOMImplementation() + builder = imp.createDOMBuilder(imp.MODE_SYNCHRONOUS, None) + document = builder.parseURI("http://example.com/2000/svg") + + self.assertIsInstance(document, minidom.Document) + self.assertEqual(len(document.childNodes), 1) + + def test_parse_with_systemId(self): + response = io.BytesIO(SMALL_SAMPLE) + + with mock.patch("urllib.request.urlopen") as mock_open: + mock_open.return_value = response + + imp = getDOMImplementation() + source = imp.createDOMInputSource() + builder = imp.createDOMBuilder(imp.MODE_SYNCHRONOUS, None) + source.systemId = "http://example.com/2000/svg" + document = builder.parse(source) + + self.assertIsInstance(document, minidom.Document) + self.assertEqual(len(document.childNodes), 1) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 14df482ba6c207..ae06a9cc11855f 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -138,9 +138,9 @@ class ModuleTest(unittest.TestCase): def test_sanity(self): # Import sanity. - from xml.etree import ElementTree - from xml.etree import ElementInclude - from xml.etree import ElementPath + from xml.etree import ElementTree # noqa: F401 + from xml.etree import ElementInclude # noqa: F401 + from xml.etree import ElementPath # noqa: F401 def test_all(self): names = ("xml.etree.ElementTree", "_elementtree") @@ -329,7 +329,7 @@ def test_simpleops(self): self.serialize_check(element, '') # 5 with self.assertRaises(ValueError) as cm: element.remove(subelement) - self.assertIn('not in list', str(cm.exception)) + self.assertEqual(str(cm.exception), 'list.remove(x): x not in list') self.serialize_check(element, '') # 6 element[0:0] = [subelement, subelement, subelement] self.serialize_check(element[1], '') @@ -1707,11 +1707,10 @@ def test_unknown_event(self): with self.assertRaises(ValueError): ET.XMLPullParser(events=('start', 'end', 'bogus')) + @unittest.skipIf(pyexpat.version_info < (2, 6, 0), + f'Expat {pyexpat.version_info} does not ' + 'support reparse deferral') def test_flush_reparse_deferral_enabled(self): - if pyexpat.version_info < (2, 6, 0): - self.skipTest(f'Expat {pyexpat.version_info} does not ' - 'support reparse deferral') - parser = ET.XMLPullParser(events=('start', 'end')) for chunk in (""): @@ -1743,8 +1742,8 @@ def test_flush_reparse_deferral_disabled(self): self.skipTest(f'XMLParser.(Get|Set)ReparseDeferralEnabled ' 'methods not available in C') parser._parser._parser.SetReparseDeferralEnabled(False) + self.assert_event_tags(parser, []) # i.e. no elements started - self.assert_event_tags(parser, []) # i.e. no elements started if ET is pyET: self.assertFalse(parser._parser._parser.GetReparseDeferralEnabled()) @@ -2424,6 +2423,22 @@ def test_39495_treebuilder_start(self): self.assertRaises(TypeError, ET.TreeBuilder().start, "tag") self.assertRaises(TypeError, ET.TreeBuilder().start, "tag", None) + def test_issue123213_correct_extend_exception(self): + # Does not hide the internal exception when extending the element + self.assertRaises(ZeroDivisionError, ET.Element('tag').extend, + (1/0 for i in range(2))) + + # Still raises the TypeError when extending with a non-iterable + self.assertRaises(TypeError, ET.Element('tag').extend, None) + + # Preserves the TypeError message when extending with a generator + def f(): + raise TypeError("mymessage") + + self.assertRaisesRegex( + TypeError, 'mymessage', + ET.Element('tag').extend, (f() for i in range(2))) + # -------------------------------------------------------------------- @@ -3749,6 +3764,22 @@ def test_setslice_negative_steps(self): e[1::-sys.maxsize<<64] = [ET.Element('d')] self.assertEqual(self._subelem_tags(e), ['a0', 'd', 'a2', 'a3']) + def test_issue123213_setslice_exception(self): + e = ET.Element('tag') + # Does not hide the internal exception when assigning to the element + with self.assertRaises(ZeroDivisionError): + e[:1] = (1/0 for i in range(2)) + + # Still raises the TypeError when assigning with a non-iterable + with self.assertRaises(TypeError): + e[:1] = None + + # Preserve the original TypeError message when assigning. + def f(): + raise TypeError("mymessage") + + with self.assertRaisesRegex(TypeError, 'mymessage'): + e[:1] = (f() for i in range(2)) class IOTest(unittest.TestCase): def test_encoding(self): @@ -4089,7 +4120,7 @@ class BoolTest(unittest.TestCase): def test_warning(self): e = ET.fromstring('') msg = ( - r"Testing an element's truth value will raise an exception in " + r"Testing an element's truth value will always return True in " r"future versions. " r"Use specific 'len\(elem\)' or 'elem is not None' test instead.") with self.assertWarnsRegex(DeprecationWarning, msg): diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py index 3a0fc572f457ff..db19af419bdeab 100644 --- a/Lib/test/test_xml_etree_c.py +++ b/Lib/test/test_xml_etree_c.py @@ -57,6 +57,7 @@ def test_del_attribute(self): del element.attrib self.assertEqual(element.attrib, {'A': 'B', 'C': 'D'}) + @unittest.skipIf(support.is_emscripten, "segfaults") def test_trashcan(self): # If this test fails, it will most likely die via segfault. e = root = cET.Element('root') diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index 6c4b8384a3202e..2803c6d45c27bf 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -308,7 +308,7 @@ def test_get_host_info(self): def test_ssl_presence(self): try: - import ssl + import ssl # noqa: F401 except ImportError: has_ssl = False else: diff --git a/Lib/test/test_yield_from.py b/Lib/test/test_yield_from.py index 1a60357a1bcd62..b90e15e20027dc 100644 --- a/Lib/test/test_yield_from.py +++ b/Lib/test/test_yield_from.py @@ -1576,6 +1576,19 @@ def outer(): self.assertIsNone(caught.exception.__context__) self.assert_stop_iteration(g) + def test_throws_in_iter(self): + # See GH-126366: NULL pointer dereference if __iter__ + # threw an exception. + class Silly: + def __iter__(self): + raise RuntimeError("nobody expects the spanish inquisition") + + def my_generator(): + yield from Silly() + + with self.assertRaisesRegex(RuntimeError, "nobody expects the spanish inquisition"): + next(iter(my_generator())) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_zipapp.py b/Lib/test/test_zipapp.py index f1c6b2d97621ee..00a5ed6626ddc5 100644 --- a/Lib/test/test_zipapp.py +++ b/Lib/test/test_zipapp.py @@ -265,14 +265,15 @@ def test_write_shebang_to_fileobj(self): zipapp.create_archive(str(target), new_target, interpreter='python2.7') self.assertTrue(new_target.getvalue().startswith(b'#!python2.7\n')) - def test_read_from_pathobj(self): - # Test that we can copy an archive using a pathlib.Path object + def test_read_from_pathlike_obj(self): + # Test that we can copy an archive using a path-like object # for the source. source = self.tmpdir / 'source' source.mkdir() (source / '__main__.py').touch() - target1 = self.tmpdir / 'target1.pyz' - target2 = self.tmpdir / 'target2.pyz' + source = os_helper.FakePath(str(source)) + target1 = os_helper.FakePath(str(self.tmpdir / 'target1.pyz')) + target2 = os_helper.FakePath(str(self.tmpdir / 'target2.pyz')) zipapp.create_archive(source, target1, interpreter='python') zipapp.create_archive(target1, target2, interpreter='python2.7') self.assertEqual(zipapp.get_interpreter(target2), 'python2.7') diff --git a/Lib/test/test_zipfile/_path/test_complexity.py b/Lib/test/test_zipfile/_path/test_complexity.py index 7050937738af18..b505dd7c376462 100644 --- a/Lib/test/test_zipfile/_path/test_complexity.py +++ b/Lib/test/test_zipfile/_path/test_complexity.py @@ -20,7 +20,7 @@ class TestComplexity(unittest.TestCase): @pytest.mark.flaky def test_implied_dirs_performance(self): best, others = big_o.big_o( - compose(consume, zipfile.CompleteDirs._implied_dirs), + compose(consume, zipfile._path.CompleteDirs._implied_dirs), lambda size: [ '/'.join(string.ascii_lowercase + str(n)) for n in range(size) ], @@ -43,13 +43,17 @@ def make_zip_path(self, depth=1, width=1) -> zipfile.Path: @classmethod def make_names(cls, width, letters=string.ascii_lowercase): """ + >>> list(TestComplexity.make_names(1)) + ['a'] >>> list(TestComplexity.make_names(2)) ['a', 'b'] >>> list(TestComplexity.make_names(30)) ['aa', 'ab', ..., 'bd'] + >>> list(TestComplexity.make_names(17124)) + ['aaa', 'aab', ..., 'zip'] """ # determine how many products are needed to produce width - n_products = math.ceil(math.log(width, len(letters))) + n_products = max(1, math.ceil(math.log(width, len(letters)))) inputs = (letters,) * n_products combinations = itertools.product(*inputs) names = map(''.join, combinations) @@ -80,7 +84,7 @@ def test_glob_depth(self): max_n=100, min_n=1, ) - assert best <= big_o.complexities.Quadratic + assert best <= big_o.complexities.Linear @pytest.mark.flaky def test_glob_width(self): diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py index 171ab6fdb5fc28..1ee45f5fc57104 100644 --- a/Lib/test/test_zipfile/_path/test_path.py +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -3,31 +3,28 @@ import contextlib import pathlib import pickle +import stat import sys +import time import unittest import zipfile +import zipfile._path + +from test.support.os_helper import temp_dir, FakePath from ._functools import compose from ._itertools import Counter from ._test_params import parameterize, Invoked -from test.support.os_helper import temp_dir - class jaraco: class itertools: Counter = Counter -def add_dirs(zf): - """ - Given a writable zip file zf, inject directory entries for - any directories implied by the presence of children. - """ - for name in zipfile.CompleteDirs._implied_dirs(zf.namelist()): - zf.writestr(name, b"") - return zf +def _make_link(info: zipfile.ZipInfo): # type: ignore[name-defined] + info.external_attr |= stat.S_IFLNK << 16 def build_alpharep_fixture(): @@ -36,6 +33,7 @@ def build_alpharep_fixture(): . ├── a.txt + ├── n.txt (-> a.txt) ├── b │ ├── c.txt │ ├── d @@ -56,6 +54,7 @@ def build_alpharep_fixture(): - multiple files in a directory (b/c, b/f) - a directory containing only a directory (g/h) - a directory with files of different extensions (j/klm) + - a symlink (n) pointing to (a) "alpha" because it uses alphabet "rep" because it's a representative example @@ -70,13 +69,16 @@ def build_alpharep_fixture(): zf.writestr("j/k.bin", b"content of k") zf.writestr("j/l.baz", b"content of l") zf.writestr("j/m.bar", b"content of m") + zf.writestr("n.txt", b"a.txt") + _make_link(zf.infolist()[-1]) + zf.filename = "alpharep.zip" return zf alpharep_generators = [ Invoked.wrap(build_alpharep_fixture), - Invoked.wrap(compose(add_dirs, build_alpharep_fixture)), + Invoked.wrap(compose(zipfile._path.CompleteDirs.inject, build_alpharep_fixture)), ] pass_alpharep = parameterize(['alpharep'], alpharep_generators) @@ -100,7 +102,7 @@ def zipfile_ondisk(self, alpharep): def test_iterdir_and_types(self, alpharep): root = zipfile.Path(alpharep) assert root.is_dir() - a, b, g, j = root.iterdir() + a, n, b, g, j = root.iterdir() assert a.is_file() assert b.is_dir() assert g.is_dir() @@ -120,7 +122,7 @@ def test_is_file_missing(self, alpharep): @pass_alpharep def test_iterdir_on_file(self, alpharep): root = zipfile.Path(alpharep) - a, b, g, j = root.iterdir() + a, n, b, g, j = root.iterdir() with self.assertRaises(ValueError): a.iterdir() @@ -135,7 +137,7 @@ def test_subdir_is_dir(self, alpharep): @pass_alpharep def test_open(self, alpharep): root = zipfile.Path(alpharep) - a, b, g, j = root.iterdir() + a, n, b, g, j = root.iterdir() with a.open(encoding="utf-8") as strm: data = strm.read() self.assertEqual(data, "content of a") @@ -210,11 +212,12 @@ def test_open_write(self): with zf.joinpath('file.txt').open('w', encoding="utf-8") as strm: strm.write('text file') - def test_open_extant_directory(self): + @pass_alpharep + def test_open_extant_directory(self, alpharep): """ Attempting to open a directory raises IsADirectoryError. """ - zf = zipfile.Path(add_dirs(build_alpharep_fixture())) + zf = zipfile.Path(alpharep) with self.assertRaises(IsADirectoryError): zf.joinpath('b').open() @@ -226,18 +229,19 @@ def test_open_binary_invalid_args(self, alpharep): with self.assertRaises(ValueError): root.joinpath('a.txt').open('rb', 'utf-8') - def test_open_missing_directory(self): + @pass_alpharep + def test_open_missing_directory(self, alpharep): """ Attempting to open a missing directory raises FileNotFoundError. """ - zf = zipfile.Path(add_dirs(build_alpharep_fixture())) + zf = zipfile.Path(alpharep) with self.assertRaises(FileNotFoundError): zf.joinpath('z').open() @pass_alpharep def test_read(self, alpharep): root = zipfile.Path(alpharep) - a, b, g, j = root.iterdir() + a, n, b, g, j = root.iterdir() assert a.read_text(encoding="utf-8") == "content of a" # Also check positional encoding arg (gh-101144). assert a.read_text("utf-8") == "content of a" @@ -271,13 +275,13 @@ def test_pathlike_construction(self, alpharep): zipfile.Path should be constructable from a path-like object """ zipfile_ondisk = self.zipfile_ondisk(alpharep) - pathlike = pathlib.Path(str(zipfile_ondisk)) + pathlike = FakePath(str(zipfile_ondisk)) zipfile.Path(pathlike) @pass_alpharep def test_traverse_pathlike(self, alpharep): root = zipfile.Path(alpharep) - root / pathlib.Path("a") + root / FakePath("a") @pass_alpharep def test_parent(self, alpharep): @@ -303,7 +307,7 @@ def test_mutability(self, alpharep): reflect that change. """ root = zipfile.Path(alpharep) - a, b, g, j = root.iterdir() + a, n, b, g, j = root.iterdir() alpharep.writestr('foo.txt', 'foo') alpharep.writestr('bar/baz.txt', 'baz') assert any(child.name == 'foo.txt' for child in root.iterdir()) @@ -472,6 +476,18 @@ def test_glob_recursive(self, alpharep): assert list(root.glob("**/*.txt")) == list(root.rglob("*.txt")) + @pass_alpharep + def test_glob_dirs(self, alpharep): + root = zipfile.Path(alpharep) + assert list(root.glob('b')) == [zipfile.Path(alpharep, "b/")] + assert list(root.glob('b*')) == [zipfile.Path(alpharep, "b/")] + + @pass_alpharep + def test_glob_subdir(self, alpharep): + root = zipfile.Path(alpharep) + assert list(root.glob('g/h')) == [zipfile.Path(alpharep, "g/h/")] + assert list(root.glob('g*/h*')) == [zipfile.Path(alpharep, "g/h/")] + @pass_alpharep def test_glob_subdirs(self, alpharep): root = zipfile.Path(alpharep) @@ -520,12 +536,9 @@ def test_eq_hash(self, alpharep): @pass_alpharep def test_is_symlink(self, alpharep): - """ - See python/cpython#82102 for symlink support beyond this object. - """ - root = zipfile.Path(alpharep) - assert not root.is_symlink() + assert not root.joinpath('a.txt').is_symlink() + assert root.joinpath('n.txt').is_symlink() @pass_alpharep def test_relative_to(self, alpharep): @@ -546,12 +559,12 @@ def test_inheritance(self, alpharep): ['alpharep', 'path_type', 'subpath'], itertools.product( alpharep_generators, - [str, pathlib.Path], + [str, FakePath], ['', 'b/'], ), ) def test_pickle(self, alpharep, path_type, subpath): - zipfile_ondisk = path_type(self.zipfile_ondisk(alpharep)) + zipfile_ondisk = path_type(str(self.zipfile_ondisk(alpharep))) saved_1 = pickle.dumps(zipfile.Path(zipfile_ondisk, at=subpath)) restored_1 = pickle.loads(saved_1) @@ -578,14 +591,69 @@ def test_getinfo_missing(self, alpharep): with self.assertRaises(KeyError): alpharep.getinfo('does-not-exist') - def test_root_folder_in_zipfile(self): + def test_malformed_paths(self): """ - gh-112795: Some tools or self constructed codes will add '/' folder to - the zip file, this is a strange behavior, but we should support it. + Path should handle malformed paths gracefully. + + Paths with leading slashes are not visible. + + Paths with dots are treated like regular files. """ - in_memory_file = io.BytesIO() - zf = zipfile.ZipFile(in_memory_file, "w") - zf.mkdir('/') - zf.writestr('./a.txt', 'aaa') - tmpdir = pathlib.Path(self.fixtures.enter_context(temp_dir())) - zf.extractall(tmpdir) + data = io.BytesIO() + zf = zipfile.ZipFile(data, "w") + zf.writestr("/one-slash.txt", b"content") + zf.writestr("//two-slash.txt", b"content") + zf.writestr("../parent.txt", b"content") + zf.filename = '' + root = zipfile.Path(zf) + assert list(map(str, root.iterdir())) == ['../'] + assert root.joinpath('..').joinpath('parent.txt').read_bytes() == b'content' + + def test_unsupported_names(self): + """ + Path segments with special characters are readable. + + On some platforms or file systems, characters like + ``:`` and ``?`` are not allowed, but they are valid + in the zip file. + """ + data = io.BytesIO() + zf = zipfile.ZipFile(data, "w") + zf.writestr("path?", b"content") + zf.writestr("V: NMS.flac", b"fLaC...") + zf.filename = '' + root = zipfile.Path(zf) + contents = root.iterdir() + assert next(contents).name == 'path?' + assert next(contents).name == 'V: NMS.flac' + assert root.joinpath('V: NMS.flac').read_bytes() == b"fLaC..." + + def test_backslash_not_separator(self): + """ + In a zip file, backslashes are not separators. + """ + data = io.BytesIO() + zf = zipfile.ZipFile(data, "w") + zf.writestr(DirtyZipInfo("foo\\bar")._for_archive(zf), b"content") + zf.filename = '' + root = zipfile.Path(zf) + (first,) = root.iterdir() + assert not first.is_dir() + assert first.name == 'foo\\bar' + + @pass_alpharep + def test_interface(self, alpharep): + from importlib.resources.abc import Traversable + + zf = zipfile.Path(alpharep) + assert isinstance(zf, Traversable) + + +class DirtyZipInfo(zipfile.ZipInfo): + """ + Bypass name sanitization. + """ + + def __init__(self, filename, *args, **kwargs): + super().__init__(filename, *args, **kwargs) + self.filename = filename diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 7d89f753448dd7..6b1fe56074d561 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -1,3 +1,4 @@ +import _pyio import array import contextlib import importlib.util @@ -5,6 +6,7 @@ import itertools import os import posixpath +import stat import struct import subprocess import sys @@ -18,10 +20,11 @@ from random import randint, random, randbytes from test import archiver_tests -from test.support import script_helper +from test.support import script_helper, os_helper from test.support import ( findfile, requires_zlib, requires_bz2, requires_lzma, - captured_stdout, captured_stderr, requires_subprocess + captured_stdout, captured_stderr, requires_subprocess, + is_emscripten ) from test.support.os_helper import ( TESTFN, unlink, rmtree, temp_dir, temp_cwd, fd_count, FakePath @@ -389,7 +392,6 @@ def test_repr(self): with zipfp.open(fname) as zipopen: r = repr(zipopen) self.assertIn('name=%r' % fname, r) - self.assertIn("mode='r'", r) if self.compression != zipfile.ZIP_STORED: self.assertIn('compress_type=', r) self.assertIn('[closed]', repr(zipopen)) @@ -455,14 +457,14 @@ def test_zipextfile_attrs(self): with zipfp.open(fname) as fid: self.assertEqual(fid.name, fname) self.assertRaises(io.UnsupportedOperation, fid.fileno) - self.assertEqual(fid.mode, 'r') + self.assertEqual(fid.mode, 'rb') self.assertIs(fid.readable(), True) self.assertIs(fid.writable(), False) self.assertIs(fid.seekable(), True) self.assertIs(fid.closed, False) self.assertIs(fid.closed, True) self.assertEqual(fid.name, fname) - self.assertEqual(fid.mode, 'r') + self.assertEqual(fid.mode, 'rb') self.assertRaises(io.UnsupportedOperation, fid.fileno) self.assertRaises(ValueError, fid.readable) self.assertIs(fid.writable(), False) @@ -1117,7 +1119,7 @@ def test_force_zip64(self): # Because this is hard to verify by parsing the data as a zip, the raw # bytes are checked to ensure that they line up with the zip spec. # The spec for this can be found at: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT - # The relevent sections for this test are: + # The relevant sections for this test are: # - 4.3.7 for local file header # - 4.5.3 for zip64 extra field @@ -1188,7 +1190,7 @@ def test_unseekable_zip_known_filesize(self): # in as a zip, this test looks at the raw bytes created to ensure that # the correct data has been generated. # The spec for this can be found at: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT - # The relevent sections for this test are: + # The relevant sections for this test are: # - 4.3.7 for local file header # - 4.3.9 for the data descriptor # - 4.5.3 for zip64 extra field @@ -1308,12 +1310,16 @@ def test_zipwritefile_attrs(self): fname = "somefile.txt" with zipfile.ZipFile(TESTFN2, mode="w", compression=self.compression) as zipfp: with zipfp.open(fname, 'w') as fid: + self.assertEqual(fid.name, fname) self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertEqual(fid.mode, 'wb') self.assertIs(fid.readable(), False) self.assertIs(fid.writable(), True) self.assertIs(fid.seekable(), False) self.assertIs(fid.closed, False) self.assertIs(fid.closed, True) + self.assertEqual(fid.name, fname) + self.assertEqual(fid.mode, 'wb') self.assertRaises(io.UnsupportedOperation, fid.fileno) self.assertIs(fid.readable(), False) self.assertIs(fid.writable(), True) @@ -1777,6 +1783,35 @@ def test_writestr_extended_local_header_issue1202(self): zinfo.flag_bits |= zipfile._MASK_USE_DATA_DESCRIPTOR # Include an extended local header. orig_zip.writestr(zinfo, data) + def test_write_with_source_date_epoch(self): + with os_helper.EnvironmentVarGuard() as env: + # Set the SOURCE_DATE_EPOCH environment variable to a specific timestamp + env['SOURCE_DATE_EPOCH'] = "1735715999" + + with zipfile.ZipFile(TESTFN, "w") as zf: + zf.writestr("test_source_date_epoch.txt", "Testing SOURCE_DATE_EPOCH") + + with zipfile.ZipFile(TESTFN, "r") as zf: + zip_info = zf.getinfo("test_source_date_epoch.txt") + get_time = time.localtime(int(os.environ['SOURCE_DATE_EPOCH']))[:6] + # Compare each element of the date_time tuple + # Allow for a 1-second difference + for z_time, g_time in zip(zip_info.date_time, get_time): + self.assertAlmostEqual(z_time, g_time, delta=1) + + def test_write_without_source_date_epoch(self): + with os_helper.EnvironmentVarGuard() as env: + del env['SOURCE_DATE_EPOCH'] + + with zipfile.ZipFile(TESTFN, "w") as zf: + zf.writestr("test_no_source_date_epoch.txt", "Testing without SOURCE_DATE_EPOCH") + + with zipfile.ZipFile(TESTFN, "r") as zf: + zip_info = zf.getinfo("test_no_source_date_epoch.txt") + current_time = time.localtime()[:6] + for z_time, c_time in zip(zip_info.date_time, current_time): + self.assertAlmostEqual(z_time, c_time, delta=1) + def test_close(self): """Check that the zipfile is closed after the 'with' block.""" with zipfile.ZipFile(TESTFN2, "w") as zipfp: @@ -1966,10 +2001,16 @@ def test_is_zip_valid_file(self): zip_contents = fp.read() # - passing a file-like object fp = io.BytesIO() - fp.write(zip_contents) + end = fp.write(zip_contents) + self.assertEqual(fp.tell(), end) + mid = end // 2 + fp.seek(mid, 0) self.assertTrue(zipfile.is_zipfile(fp)) - fp.seek(0, 0) + # check that the position is left unchanged after the call + # see: https://github.com/python/cpython/issues/122356 + self.assertEqual(fp.tell(), mid) self.assertTrue(zipfile.is_zipfile(fp)) + self.assertEqual(fp.tell(), mid) def test_non_existent_file_raises_OSError(self): # make sure we don't raise an AttributeError when a partially-constructed @@ -2202,6 +2243,34 @@ def test_create_empty_zipinfo_repr(self): zi = zipfile.ZipInfo(filename="empty") self.assertEqual(repr(zi), "") + def test_for_archive(self): + base_filename = TESTFN2.rstrip('/') + + with zipfile.ZipFile(TESTFN, mode="w", compresslevel=1, + compression=zipfile.ZIP_STORED) as zf: + # no trailing forward slash + zi = zipfile.ZipInfo(base_filename)._for_archive(zf) + self.assertEqual(zi.compress_level, 1) + self.assertEqual(zi.compress_type, zipfile.ZIP_STORED) + # ?rw- --- --- + filemode = stat.S_IRUSR | stat.S_IWUSR + # filemode is stored as the highest 16 bits of external_attr + self.assertEqual(zi.external_attr >> 16, filemode) + self.assertEqual(zi.external_attr & 0xFF, 0) # no MS-DOS flag + + with zipfile.ZipFile(TESTFN, mode="w", compresslevel=1, + compression=zipfile.ZIP_STORED) as zf: + # with a trailing slash + zi = zipfile.ZipInfo(f'{base_filename}/')._for_archive(zf) + self.assertEqual(zi.compress_level, 1) + self.assertEqual(zi.compress_type, zipfile.ZIP_STORED) + # d rwx rwx r-x + filemode = stat.S_IFDIR + filemode |= stat.S_IRWXU | stat.S_IRWXG + filemode |= stat.S_IROTH | stat.S_IXOTH + self.assertEqual(zi.external_attr >> 16, filemode) + self.assertEqual(zi.external_attr & 0xFF, 0x10) # MS-DOS flag + def test_create_empty_zipinfo_default_attributes(self): """Ensure all required attributes are set.""" zi = zipfile.ZipInfo() @@ -2324,6 +2393,18 @@ def test_read_after_seek(self): fp.seek(1, os.SEEK_CUR) self.assertEqual(fp.read(-1), b'men!') + def test_uncompressed_interleaved_seek_read(self): + # gh-127847: Make sure the position in the archive is correct + # in the special case of seeking in a ZIP_STORED entry. + with zipfile.ZipFile(TESTFN, "w") as zipf: + zipf.writestr("a.txt", "123") + zipf.writestr("b.txt", "456") + with zipfile.ZipFile(TESTFN, "r") as zipf: + with zipf.open("a.txt", "r") as a, zipf.open("b.txt", "r") as b: + self.assertEqual(a.read(1), b"1") + self.assertEqual(b.seek(1), 1) + self.assertEqual(b.read(1), b"5") + @requires_bz2() def test_decompress_without_3rd_party_library(self): data = b'PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' @@ -2937,6 +3018,22 @@ def test_bug_6050(self): os.mkdir(os.path.join(TESTFN2, "a")) self.test_extract_dir() + def test_extract_dir_backslash(self): + zfname = findfile("zipdir_backslash.zip", subdir="archivetestdata") + with zipfile.ZipFile(zfname) as zipf: + zipf.extractall(TESTFN2) + if os.name == 'nt': + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "a"))) + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "a", "b"))) + self.assertTrue(os.path.isfile(os.path.join(TESTFN2, "a", "b", "c"))) + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "d"))) + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "d", "e"))) + else: + self.assertTrue(os.path.isfile(os.path.join(TESTFN2, "a\\b\\c"))) + self.assertTrue(os.path.isfile(os.path.join(TESTFN2, "d\\e\\"))) + self.assertFalse(os.path.exists(os.path.join(TESTFN2, "a"))) + self.assertFalse(os.path.exists(os.path.join(TESTFN2, "d"))) + def test_write_dir(self): dirpath = os.path.join(TESTFN2, "x") os.mkdir(dirpath) @@ -3033,6 +3130,17 @@ def test_create_directory_with_write(self): self.assertEqual(set(os.listdir(target)), {"directory", "directory2"}) + def test_root_folder_in_zipfile(self): + """ + gh-112795: Some tools or self constructed codes will add '/' folder to + the zip file, this is a strange behavior, but we should support it. + """ + in_memory_file = io.BytesIO() + zf = zipfile.ZipFile(in_memory_file, "w") + zf.mkdir('/') + zf.writestr('./a.txt', 'aaa') + zf.extractall(TESTFN2) + def tearDown(self): rmtree(TESTFN2) if os.path.exists(TESTFN): @@ -3412,5 +3520,87 @@ def test_too_short(self): b"zzz", zipfile._Extra.strip(b"zzz", (self.ZIP64_EXTRA,))) +class StatIO(_pyio.BytesIO): + """Buffer which remembers the number of bytes that were read.""" + + def __init__(self): + super().__init__() + self.bytes_read = 0 + + def read(self, size=-1): + bs = super().read(size) + self.bytes_read += len(bs) + return bs + + +class StoredZipExtFileRandomReadTest(unittest.TestCase): + """Tests whether an uncompressed, unencrypted zip entry can be randomly + seek and read without reading redundant bytes.""" + def test_stored_seek_and_read(self): + + sio = StatIO() + # 20000 bytes + txt = b'0123456789' * 2000 + + # The seek length must be greater than ZipExtFile.MIN_READ_SIZE + # as `ZipExtFile._read2()` reads in blocks of this size and we + # need to seek out of the buffered data + read_buffer_size = zipfile.ZipExtFile.MIN_READ_SIZE + self.assertGreaterEqual(10002, read_buffer_size) # for forward seek test + self.assertGreaterEqual(5003, read_buffer_size) # for backward seek test + # The read length must be less than MIN_READ_SIZE, since we assume that + # only 1 block is read in the test. + read_length = 100 + self.assertGreaterEqual(read_buffer_size, read_length) # for read() calls + + with zipfile.ZipFile(sio, "w", compression=zipfile.ZIP_STORED) as zipf: + zipf.writestr("foo.txt", txt) + + # check random seek and read on a file + with zipfile.ZipFile(sio, "r") as zipf: + with zipf.open("foo.txt", "r") as fp: + # Test this optimized read hasn't rewound and read from the + # start of the file (as in the case of the unoptimized path) + + # forward seek + old_count = sio.bytes_read + forward_seek_len = 10002 + current_pos = 0 + fp.seek(forward_seek_len, os.SEEK_CUR) + current_pos += forward_seek_len + self.assertEqual(fp.tell(), current_pos) + self.assertEqual(fp._left, fp._compress_left) + arr = fp.read(read_length) + current_pos += read_length + self.assertEqual(fp.tell(), current_pos) + self.assertEqual(arr, txt[current_pos - read_length:current_pos]) + self.assertEqual(fp._left, fp._compress_left) + read_count = sio.bytes_read - old_count + self.assertLessEqual(read_count, read_buffer_size) + + # backward seek + old_count = sio.bytes_read + backward_seek_len = 5003 + fp.seek(-backward_seek_len, os.SEEK_CUR) + current_pos -= backward_seek_len + self.assertEqual(fp.tell(), current_pos) + self.assertEqual(fp._left, fp._compress_left) + arr = fp.read(read_length) + current_pos += read_length + self.assertEqual(fp.tell(), current_pos) + self.assertEqual(arr, txt[current_pos - read_length:current_pos]) + self.assertEqual(fp._left, fp._compress_left) + read_count = sio.bytes_read - old_count + self.assertLessEqual(read_count, read_buffer_size) + + # eof flags test + fp.seek(0, os.SEEK_END) + fp.seek(12345, os.SEEK_SET) + current_pos = 12345 + arr = fp.read(read_length) + current_pos += read_length + self.assertEqual(arr, txt[current_pos - read_length:current_pos]) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index c12798d221e9b7..1f288c8b45d589 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -1,8 +1,10 @@ import sys import os import marshal +import glob import importlib import importlib.util +import re import struct import time import unittest @@ -50,10 +52,14 @@ def module_path_to_dotted_name(path): TESTMOD = "ziptestmodule" +TESTMOD2 = "ziptestmodule2" +TESTMOD3 = "ziptestmodule3" TESTPACK = "ziptestpackage" TESTPACK2 = "ziptestpackage2" +TESTPACK3 = "ziptestpackage3" TEMP_DIR = os.path.abspath("junk95142") TEMP_ZIP = os.path.abspath("junk95142.zip") +TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "zipimport_data") pyc_file = importlib.util.cache_from_source(TESTMOD + '.py') pyc_ext = '.pyc' @@ -92,8 +98,10 @@ def makeTree(self, files, dirName=TEMP_DIR): # defined by files under the directory dirName. self.addCleanup(os_helper.rmtree, dirName) - for name, (mtime, data) in files.items(): - path = os.path.join(dirName, name) + for name, data in files.items(): + if isinstance(data, tuple): + mtime, data = data + path = os.path.join(dirName, *name.split('/')) if path[-1] == os.sep: if not os.path.isdir(path): os.makedirs(path) @@ -104,22 +112,18 @@ def makeTree(self, files, dirName=TEMP_DIR): with open(path, 'wb') as fp: fp.write(data) - def makeZip(self, files, zipName=TEMP_ZIP, **kw): + def makeZip(self, files, zipName=TEMP_ZIP, *, + comment=None, file_comment=None, stuff=None, prefix='', **kw): # Create a zip archive based set of modules/packages - # defined by files in the zip file zipName. If the - # key 'stuff' exists in kw it is prepended to the archive. + # defined by files in the zip file zipName. + # If stuff is not None, it is prepended to the archive. self.addCleanup(os_helper.unlink, zipName) - with ZipFile(zipName, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - z.writestr(zinfo, data) - comment = kw.get("comment", None) + with ZipFile(zipName, "w", compression=self.compression) as z: + self.writeZip(z, files, file_comment=file_comment, prefix=prefix) if comment is not None: z.comment = comment - stuff = kw.get("stuff", None) if stuff is not None: # Prepend 'stuff' to the start of the zipfile with open(zipName, "rb") as f: @@ -128,20 +132,47 @@ def makeZip(self, files, zipName=TEMP_ZIP, **kw): f.write(stuff) f.write(data) + def writeZip(self, z, files, *, file_comment=None, prefix=''): + for name, data in files.items(): + if isinstance(data, tuple): + mtime, data = data + else: + mtime = NOW + name = name.replace(os.sep, '/') + zinfo = ZipInfo(prefix + name, time.localtime(mtime)) + zinfo.compress_type = self.compression + if file_comment is not None: + zinfo.comment = file_comment + if data is None: + zinfo.CRC = 0 + z.mkdir(zinfo) + else: + assert name[-1] != '/' + z.writestr(zinfo, data) + + def getZip64Files(self): + # This is the simplest way to make zipfile generate the zip64 EOCD block + return {f"f{n}.py": test_src for n in range(65537)} + def doTest(self, expected_ext, files, *modules, **kw): + if 'prefix' not in kw: + kw['prefix'] = 'pre/fix/' self.makeZip(files, **kw) + self.doTestWithPreBuiltZip(expected_ext, *modules, **kw) - sys.path.insert(0, TEMP_ZIP) + def doTestWithPreBuiltZip(self, expected_ext, *modules, + call=None, prefix='', **kw): + zip_path = os.path.join(TEMP_ZIP, *prefix.split('/')[:-1]) + sys.path.insert(0, zip_path) mod = importlib.import_module(".".join(modules)) - call = kw.get('call') if call is not None: call(mod) if expected_ext: file = mod.get_file() - self.assertEqual(file, os.path.join(TEMP_ZIP, + self.assertEqual(file, os.path.join(zip_path, *modules) + expected_ext) def testAFakeZlib(self): @@ -167,7 +198,7 @@ def testAFakeZlib(self): self.skipTest('zlib is a builtin module') if "zlib" in sys.modules: del sys.modules["zlib"] - files = {"zlib.py": (NOW, test_src)} + files = {"zlib.py": test_src} try: self.doTest(".py", files, "zlib") except ImportError: @@ -178,16 +209,16 @@ def testAFakeZlib(self): self.fail("expected test to raise ImportError") def testPy(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD) def testPyc(self): - files = {TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTMOD) def testBoth(self): - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTMOD) def testUncheckedHashBasedPyc(self): @@ -220,22 +251,22 @@ def check(mod): self.doTest(None, files, TESTMOD, call=check) def testEmptyPy(self): - files = {TESTMOD + ".py": (NOW, "")} + files = {TESTMOD + ".py": ""} self.doTest(None, files, TESTMOD) def testBadMagic(self): # make pyc magic word invalid, forcing loading from .py badmagic_pyc = bytearray(test_pyc) badmagic_pyc[0] ^= 0x04 # flip an arbitrary bit - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, badmagic_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: badmagic_pyc} self.doTest(".py", files, TESTMOD) def testBadMagic2(self): # make pyc magic word invalid, causing an ImportError badmagic_pyc = bytearray(test_pyc) badmagic_pyc[0] ^= 0x04 # flip an arbitrary bit - files = {TESTMOD + pyc_ext: (NOW, badmagic_pyc)} + files = {TESTMOD + pyc_ext: badmagic_pyc} try: self.doTest(".py", files, TESTMOD) self.fail("This should not be reached") @@ -248,22 +279,22 @@ def testBadMTime(self): # flip the second bit -- not the first as that one isn't stored in the # .py's mtime in the zip archive. badtime_pyc[11] ^= 0x02 - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, badtime_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: badtime_pyc} self.doTest(".py", files, TESTMOD) def test2038MTime(self): # Make sure we can handle mtimes larger than what a 32-bit signed number # can hold. twenty_thirty_eight_pyc = make_pyc(test_co, 2**32 - 1, len(test_src)) - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, twenty_thirty_eight_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: twenty_thirty_eight_pyc} self.doTest(".py", files, TESTMOD) def testPackage(self): packdir = TESTPACK + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir + TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTPACK, TESTMOD) def testSubPackage(self): @@ -271,9 +302,9 @@ def testSubPackage(self): # archives. packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTPACK, TESTPACK2, TESTMOD) def testSubNamespacePackage(self): @@ -282,29 +313,104 @@ def testSubNamespacePackage(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep # The first two files are just directory entries (so have no data). - files = {packdir: (NOW, ""), - packdir2: (NOW, ""), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {packdir: None, + packdir2: None, + packdir2 + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTPACK, TESTPACK2, TESTMOD) + def testPackageExplicitDirectories(self): + # Test explicit namespace packages with explicit directory entries. + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.mkdir('a') + z.writestr('a/__init__.py', test_src) + z.mkdir('a/b') + z.writestr('a/b/__init__.py', test_src) + z.mkdir('a/b/c') + z.writestr('a/b/c/__init__.py', test_src) + z.writestr('a/b/c/d.py', test_src) + self._testPackage(initfile='__init__.py') + + def testPackageImplicitDirectories(self): + # Test explicit namespace packages without explicit directory entries. + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.writestr('a/__init__.py', test_src) + z.writestr('a/b/__init__.py', test_src) + z.writestr('a/b/c/__init__.py', test_src) + z.writestr('a/b/c/d.py', test_src) + self._testPackage(initfile='__init__.py') + + def testNamespacePackageExplicitDirectories(self): + # Test implicit namespace packages with explicit directory entries. + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.mkdir('a') + z.mkdir('a/b') + z.mkdir('a/b/c') + z.writestr('a/b/c/d.py', test_src) + self._testPackage(initfile=None) + + def testNamespacePackageImplicitDirectories(self): + # Test implicit namespace packages without explicit directory entries. + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.writestr('a/b/c/d.py', test_src) + self._testPackage(initfile=None) + + def _testPackage(self, initfile): + zi = zipimport.zipimporter(os.path.join(TEMP_ZIP, 'a')) + if initfile is None: + # XXX Should it work? + self.assertRaises(zipimport.ZipImportError, zi.is_package, 'b') + self.assertRaises(zipimport.ZipImportError, zi.get_source, 'b') + self.assertRaises(zipimport.ZipImportError, zi.get_code, 'b') + else: + self.assertTrue(zi.is_package('b')) + self.assertEqual(zi.get_source('b'), test_src) + self.assertEqual(zi.get_code('b').co_filename, + os.path.join(TEMP_ZIP, 'a', 'b', initfile)) + + sys.path.insert(0, TEMP_ZIP) + self.assertNotIn('a', sys.modules) + + mod = importlib.import_module(f'a.b') + self.assertIn('a', sys.modules) + self.assertIs(sys.modules['a.b'], mod) + if initfile is None: + self.assertIsNone(mod.__file__) + else: + self.assertEqual(mod.__file__, + os.path.join(TEMP_ZIP, 'a', 'b', initfile)) + self.assertEqual(len(mod.__path__), 1, mod.__path__) + self.assertEqual(mod.__path__[0], os.path.join(TEMP_ZIP, 'a', 'b')) + + mod2 = importlib.import_module(f'a.b.c.d') + self.assertIn('a.b.c', sys.modules) + self.assertIn('a.b.c.d', sys.modules) + self.assertIs(sys.modules['a.b.c.d'], mod2) + self.assertIs(mod.c.d, mod2) + self.assertEqual(mod2.__file__, + os.path.join(TEMP_ZIP, 'a', 'b', 'c', 'd.py')) + def testMixedNamespacePackage(self): # Test implicit namespace packages spread between a # real filesystem and a zip archive. packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - packdir3 = packdir2 + TESTPACK + '3' + os.sep - files1 = {packdir: (NOW, ""), - packdir + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir3: (NOW, ""), - packdir3 + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + '3' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} - files2 = {packdir: (NOW, ""), - packdir + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir2 + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + packdir3 = packdir2 + TESTPACK3 + os.sep + files1 = {packdir: None, + packdir + TESTMOD + pyc_ext: test_pyc, + packdir2: None, + packdir3: None, + packdir3 + TESTMOD + pyc_ext: test_pyc, + packdir2 + TESTMOD3 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} + files2 = {packdir: None, + packdir + TESTMOD2 + pyc_ext: test_pyc, + packdir2: None, + packdir2 + TESTMOD2 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} zip1 = os.path.abspath("path1.zip") self.makeZip(files1, zip1) @@ -337,8 +443,8 @@ def testMixedNamespacePackage(self): mod = importlib.import_module('.'.join((TESTPACK, TESTMOD))) self.assertEqual("path1.zip", mod.__file__.split(os.sep)[-3]) - # And TESTPACK/(TESTMOD + '2') only exists in path2. - mod = importlib.import_module('.'.join((TESTPACK, TESTMOD + '2'))) + # And TESTPACK/(TESTMOD2) only exists in path2. + mod = importlib.import_module('.'.join((TESTPACK, TESTMOD2))) self.assertEqual(os.path.basename(TEMP_DIR), mod.__file__.split(os.sep)[-3]) @@ -355,13 +461,13 @@ def testMixedNamespacePackage(self): self.assertEqual(os.path.basename(TEMP_DIR), mod.__file__.split(os.sep)[-4]) - # subpkg.TESTMOD + '2' only exists in zip2. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '2'))) + # subpkg.TESTMOD2 only exists in zip2. + mod = importlib.import_module('.'.join((subpkg, TESTMOD2))) self.assertEqual(os.path.basename(TEMP_DIR), mod.__file__.split(os.sep)[-4]) - # Finally subpkg.TESTMOD + '3' only exists in zip1. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '3'))) + # Finally subpkg.TESTMOD3 only exists in zip1. + mod = importlib.import_module('.'.join((subpkg, TESTMOD3))) self.assertEqual('path1.zip', mod.__file__.split(os.sep)[-4]) def testNamespacePackage(self): @@ -369,22 +475,22 @@ def testNamespacePackage(self): # archives. packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - packdir3 = packdir2 + TESTPACK + '3' + os.sep - files1 = {packdir: (NOW, ""), - packdir + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir3: (NOW, ""), - packdir3 + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + '3' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + packdir3 = packdir2 + TESTPACK3 + os.sep + files1 = {packdir: None, + packdir + TESTMOD + pyc_ext: test_pyc, + packdir2: None, + packdir3: None, + packdir3 + TESTMOD + pyc_ext: test_pyc, + packdir2 + TESTMOD3 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} zip1 = os.path.abspath("path1.zip") self.makeZip(files1, zip1) - files2 = {packdir: (NOW, ""), - packdir + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir2 + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + files2 = {packdir: None, + packdir + TESTMOD2 + pyc_ext: test_pyc, + packdir2: None, + packdir2 + TESTMOD2 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} zip2 = os.path.abspath("path2.zip") self.makeZip(files2, zip2) @@ -413,8 +519,8 @@ def testNamespacePackage(self): mod = importlib.import_module('.'.join((TESTPACK, TESTMOD))) self.assertEqual("path1.zip", mod.__file__.split(os.sep)[-3]) - # And TESTPACK/(TESTMOD + '2') only exists in path2. - mod = importlib.import_module('.'.join((TESTPACK, TESTMOD + '2'))) + # And TESTPACK/(TESTMOD2) only exists in path2. + mod = importlib.import_module('.'.join((TESTPACK, TESTMOD2))) self.assertEqual("path2.zip", mod.__file__.split(os.sep)[-3]) # One level deeper... @@ -429,29 +535,22 @@ def testNamespacePackage(self): mod = importlib.import_module('.'.join((subpkg, TESTMOD))) self.assertEqual('path2.zip', mod.__file__.split(os.sep)[-4]) - # subpkg.TESTMOD + '2' only exists in zip2. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '2'))) + # subpkg.TESTMOD2 only exists in zip2. + mod = importlib.import_module('.'.join((subpkg, TESTMOD2))) self.assertEqual('path2.zip', mod.__file__.split(os.sep)[-4]) - # Finally subpkg.TESTMOD + '3' only exists in zip1. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '3'))) + # Finally subpkg.TESTMOD3 only exists in zip1. + mod = importlib.import_module('.'.join((subpkg, TESTMOD3))) self.assertEqual('path1.zip', mod.__file__.split(os.sep)[-4]) def testZipImporterMethods(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc), - "spam" + pyc_ext: (NOW, test_pyc)} - - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc, + "spam" + pyc_ext: test_pyc} + self.makeZip(files, file_comment=b"spam") zi = zipimport.zipimporter(TEMP_ZIP) self.assertEqual(zi.archive, TEMP_ZIP) @@ -507,35 +606,26 @@ def testZipImporterMethods(self): def testInvalidateCaches(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc), - "spam" + pyc_ext: (NOW, test_pyc)} - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc, + "spam" + pyc_ext: test_pyc} + extra_files = [packdir, packdir2] + self.makeZip(files, file_comment=b"spam") zi = zipimport.zipimporter(TEMP_ZIP) - self.assertEqual(zi._get_files().keys(), files.keys()) + self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) # Check that the file information remains accurate after reloading zi.invalidate_caches() - self.assertEqual(zi._get_files().keys(), files.keys()) + self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) # Add a new file to the ZIP archive - newfile = {"spam2" + pyc_ext: (NOW, test_pyc)} + newfile = {"spam2" + pyc_ext: test_pyc} files.update(newfile) - with ZipFile(TEMP_ZIP, "a") as z: - for name, (mtime, data) in newfile.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + with ZipFile(TEMP_ZIP, "a", compression=self.compression) as z: + self.writeZip(z, newfile, file_comment=b"spam") # Check that we can detect the new file after invalidating the cache zi.invalidate_caches() - self.assertEqual(zi._get_files().keys(), files.keys()) + self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) spec = zi.find_spec('spam2') self.assertIsNotNone(spec) self.assertIsInstance(spec.loader, zipimport.zipimporter) @@ -549,36 +639,27 @@ def testInvalidateCaches(self): def testInvalidateCachesWithMultipleZipimports(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc), - "spam" + pyc_ext: (NOW, test_pyc)} - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc, + "spam" + pyc_ext: test_pyc} + extra_files = [packdir, packdir2] + self.makeZip(files, file_comment=b"spam") zi = zipimport.zipimporter(TEMP_ZIP) - self.assertEqual(zi._get_files().keys(), files.keys()) + self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) # Zipimporter for the same path. zi2 = zipimport.zipimporter(TEMP_ZIP) - self.assertEqual(zi2._get_files().keys(), files.keys()) + self.assertEqual(sorted(zi2._get_files()), sorted([*files, *extra_files])) # Add a new file to the ZIP archive to make the cache wrong. - newfile = {"spam2" + pyc_ext: (NOW, test_pyc)} + newfile = {"spam2" + pyc_ext: test_pyc} files.update(newfile) - with ZipFile(TEMP_ZIP, "a") as z: - for name, (mtime, data) in newfile.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + with ZipFile(TEMP_ZIP, "a", compression=self.compression) as z: + self.writeZip(z, newfile, file_comment=b"spam") # Invalidate the cache of the first zipimporter. zi.invalidate_caches() # Check that the second zipimporter detects the new file and isn't using a stale cache. - self.assertEqual(zi2._get_files().keys(), files.keys()) + self.assertEqual(sorted(zi2._get_files()), sorted([*files, *extra_files])) spec = zi2.find_spec('spam2') self.assertIsNotNone(spec) self.assertIsInstance(spec.loader, zipimport.zipimporter) @@ -586,16 +667,9 @@ def testInvalidateCachesWithMultipleZipimports(self): def testZipImporterMethodsInSubDirectory(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} - - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"eggs" - z.writestr(zinfo, data) + files = {packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} + self.makeZip(files, file_comment=b"eggs") zi = zipimport.zipimporter(TEMP_ZIP + os.sep + packdir) self.assertEqual(zi.archive, TEMP_ZIP) @@ -641,17 +715,33 @@ def testZipImporterMethodsInSubDirectory(self): self.assertIsNone(loader.get_source(mod_name)) self.assertEqual(loader.get_filename(mod_name), mod.__file__) - def testGetData(self): + def testGetDataExplicitDirectories(self): self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - z.compression = self.compression - name = "testdata.dat" - data = bytes(x for x in range(256)) - z.writestr(name, data) - - zi = zipimport.zipimporter(TEMP_ZIP) - self.assertEqual(data, zi.get_data(name)) - self.assertIn('zipimporter object', repr(zi)) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.mkdir('a') + z.mkdir('a/b') + z.mkdir('a/b/c') + data = bytes(range(256)) + z.writestr('a/b/c/testdata.dat', data) + self._testGetData() + + def testGetDataImplicitDirectories(self): + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + data = bytes(range(256)) + z.writestr('a/b/c/testdata.dat', data) + self._testGetData() + + def _testGetData(self): + zi = zipimport.zipimporter(os.path.join(TEMP_ZIP, 'ignored')) + pathname = os.path.join('a', 'b', 'c', 'testdata.dat') + data = bytes(range(256)) + self.assertEqual(zi.get_data(pathname), data) + self.assertEqual(zi.get_data(os.path.join(TEMP_ZIP, pathname)), data) + self.assertEqual(zi.get_data(os.path.join('a', 'b', '')), b'') + self.assertEqual(zi.get_data(os.path.join(TEMP_ZIP, 'a', 'b', '')), b'') + self.assertRaises(OSError, zi.get_data, os.path.join('a', 'b')) + self.assertRaises(OSError, zi.get_data, os.path.join(TEMP_ZIP, 'a', 'b')) def testImporterAttr(self): src = """if 1: # indent hack @@ -660,9 +750,9 @@ def get_file(): if __loader__.get_data("some.data") != b"some data": raise AssertionError("bad data")\n""" pyc = make_pyc(compile(src, "", "exec"), NOW, len(src)) - files = {TESTMOD + pyc_ext: (NOW, pyc), - "some.data": (NOW, "some data")} - self.doTest(pyc_ext, files, TESTMOD) + files = {TESTMOD + pyc_ext: pyc, + "some.data": "some data"} + self.doTest(pyc_ext, files, TESTMOD, prefix='') def testDefaultOptimizationLevel(self): # zipimport should use the default optimization level (#28131) @@ -670,7 +760,7 @@ def testDefaultOptimizationLevel(self): def test(val): assert(val) return val\n""" - files = {TESTMOD + '.py': (NOW, src)} + files = {TESTMOD + '.py': src} self.makeZip(files) sys.path.insert(0, TEMP_ZIP) mod = importlib.import_module(TESTMOD) @@ -683,7 +773,7 @@ def test(val): def testImport_WithStuff(self): # try importing from a zipfile which contains additional # stuff at the beginning of the file - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, stuff=b"Some Stuff"*31) @@ -691,18 +781,18 @@ def assertModuleSource(self, module): self.assertEqual(inspect.getsource(module), test_src) def testGetSource(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, call=self.assertModuleSource) def testGetCompiledSource(self): pyc = make_pyc(compile(test_src, "", "exec"), NOW, len(test_src)) - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: pyc} self.doTest(pyc_ext, files, TESTMOD, call=self.assertModuleSource) def runDoctest(self, callback): - files = {TESTMOD + ".py": (NOW, test_src), - "xyz.txt": (NOW, ">>> log.append(True)\n")} + files = {TESTMOD + ".py": test_src, + "xyz.txt": ">>> log.append(True)\n"} self.doTest(".py", files, TESTMOD, call=callback) def doDoctestFile(self, module): @@ -754,29 +844,21 @@ def doTraceback(self, module): raise AssertionError("This ought to be impossible") def testTraceback(self): - files = {TESTMOD + ".py": (NOW, raise_src)} + files = {TESTMOD + ".py": raise_src} self.doTest(None, files, TESTMOD, call=self.doTraceback) @unittest.skipIf(os_helper.TESTFN_UNENCODABLE is None, "need an unencodable filename") def testUnencodable(self): filename = os_helper.TESTFN_UNENCODABLE + ".zip" - self.addCleanup(os_helper.unlink, filename) - with ZipFile(filename, "w") as z: - zinfo = ZipInfo(TESTMOD + ".py", time.localtime(NOW)) - zinfo.compress_type = self.compression - z.writestr(zinfo, test_src) + self.makeZip({TESTMOD + ".py": test_src}, filename) spec = zipimport.zipimporter(filename).find_spec(TESTMOD) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) def testBytesPath(self): filename = os_helper.TESTFN + ".zip" - self.addCleanup(os_helper.unlink, filename) - with ZipFile(filename, "w") as z: - zinfo = ZipInfo(TESTMOD + ".py", time.localtime(NOW)) - zinfo.compress_type = self.compression - z.writestr(zinfo, test_src) + self.makeZip({TESTMOD + ".py": test_src}, filename) zipimport.zipimporter(filename) with self.assertRaises(TypeError): @@ -787,17 +869,143 @@ def testBytesPath(self): zipimport.zipimporter(memoryview(os.fsencode(filename))) def testComment(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, comment=b"comment") def testBeginningCruftAndComment(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, stuff=b"cruft" * 64, comment=b"hi") def testLargestPossibleComment(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, comment=b"c" * ((1 << 16) - 1)) + @support.requires_resource('cpu') + def testZip64(self): + files = self.getZip64Files() + self.doTest(".py", files, "f6") + + @support.requires_resource('cpu') + def testZip64CruftAndComment(self): + files = self.getZip64Files() + self.doTest(".py", files, "f65536", comment=b"c" * ((1 << 16) - 1)) + + def testZip64LargeFile(self): + support.requires( + "largefile", + f"test generates files >{0xFFFFFFFF} bytes and takes a long time " + "to run" + ) + + # N.B.: We do a lot of gymnastics below in the ZIP_STORED case to save + # and reconstruct a sparse zip on systems that support sparse files. + # Instead of creating a ~8GB zip file mainly consisting of null bytes + # for every run of the test, we create the zip once and save off the + # non-null portions of the resulting file as data blobs with offsets + # that allow re-creating the zip file sparsely. This drops disk space + # usage to ~9KB for the ZIP_STORED case and drops that test time by ~2 + # orders of magnitude. For the ZIP_DEFLATED case, however, we bite the + # bullet. The resulting zip file is ~8MB of non-null data; so the sparse + # trick doesn't work and would result in that full ~8MB zip data file + # being checked in to source control. + parts_glob = f"sparse-zip64-c{self.compression:d}-0x*.part" + full_parts_glob = os.path.join(TEST_DATA_DIR, parts_glob) + pre_built_zip_parts = glob.glob(full_parts_glob) + + self.addCleanup(os_helper.unlink, TEMP_ZIP) + if not pre_built_zip_parts: + if self.compression != ZIP_STORED: + support.requires( + "cpu", + "test requires a lot of CPU for compression." + ) + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with open(os_helper.TESTFN, "wb") as f: + f.write(b"data") + f.write(os.linesep.encode()) + f.seek(0xffff_ffff, os.SEEK_CUR) + f.write(os.linesep.encode()) + os.utime(os_helper.TESTFN, (0.0, 0.0)) + with ZipFile( + TEMP_ZIP, + "w", + compression=self.compression, + strict_timestamps=False + ) as z: + z.write(os_helper.TESTFN, "data1") + z.writestr( + ZipInfo("module.py", (1980, 1, 1, 0, 0, 0)), test_src + ) + z.write(os_helper.TESTFN, "data2") + + # This "works" but relies on the zip format having a non-empty + # final page due to the trailing central directory to wind up with + # the correct length file. + def make_sparse_zip_parts(name): + empty_page = b"\0" * 4096 + with open(name, "rb") as f: + part = None + try: + while True: + offset = f.tell() + data = f.read(len(empty_page)) + if not data: + break + if data != empty_page: + if not part: + part_fullname = os.path.join( + TEST_DATA_DIR, + f"sparse-zip64-c{self.compression:d}-" + f"{offset:#011x}.part", + ) + os.makedirs( + os.path.dirname(part_fullname), + exist_ok=True + ) + part = open(part_fullname, "wb") + print("Created", part_fullname) + part.write(data) + else: + if part: + part.close() + part = None + finally: + if part: + part.close() + + if self.compression == ZIP_STORED: + print(f"Creating sparse parts to check in into {TEST_DATA_DIR}:") + make_sparse_zip_parts(TEMP_ZIP) + + else: + def extract_offset(name): + if m := re.search(r"-(0x[0-9a-f]{9})\.part$", name): + return int(m.group(1), base=16) + raise ValueError(f"{name=} does not fit expected pattern.") + offset_parts = [(extract_offset(n), n) for n in pre_built_zip_parts] + with open(TEMP_ZIP, "wb") as f: + for offset, part_fn in sorted(offset_parts): + with open(part_fn, "rb") as part: + f.seek(offset, os.SEEK_SET) + f.write(part.read()) + # Confirm that the reconstructed zip file works and looks right. + with ZipFile(TEMP_ZIP, "r") as z: + self.assertEqual( + z.getinfo("module.py").date_time, (1980, 1, 1, 0, 0, 0) + ) + self.assertEqual( + z.read("module.py"), test_src.encode(), + msg=f"Recreate {full_parts_glob}, unexpected contents." + ) + def assertDataEntry(name): + zinfo = z.getinfo(name) + self.assertEqual(zinfo.date_time, (1980, 1, 1, 0, 0, 0)) + self.assertGreater(zinfo.file_size, 0xffff_ffff) + assertDataEntry("data1") + assertDataEntry("data2") + + self.doTestWithPreBuiltZip(".py", "module") + @support.requires_zlib() class CompressedZipImportTestCase(UncompressedZipImportTestCase): diff --git a/Lib/test/test_zipimport_support.py b/Lib/test/test_zipimport_support.py index 71039d2a8e7ab9..ae8a8c99762313 100644 --- a/Lib/test/test_zipimport_support.py +++ b/Lib/test/test_zipimport_support.py @@ -31,7 +31,7 @@ # Retrieve some helpers from other test cases from test.test_doctest import (test_doctest, sample_doctest, sample_doctest_no_doctests, - sample_doctest_no_docstrings) + sample_doctest_no_docstrings, sample_doctest_skip) def _run_object_doctest(obj, module): @@ -110,7 +110,7 @@ def test_doctest_issue4197(self): # The sample doctest files rewritten to include in the zipped version. sample_sources = {} for mod in [sample_doctest, sample_doctest_no_doctests, - sample_doctest_no_docstrings]: + sample_doctest_no_docstrings, sample_doctest_skip]: src = inspect.getsource(mod) src = src.replace("test.test_doctest.test_doctest", "test_zipped_doctest") # Rewrite the module name so that, for example, diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index ef02c64f886f8a..4d97fe56f3a094 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -6,7 +6,7 @@ import pickle import random import sys -from test.support import bigmemtest, _1G, _4G, skip_on_s390x +from test.support import bigmemtest, _1G, _4G, is_s390x zlib = import_helper.import_module('zlib') @@ -33,8 +33,9 @@ def _zlib_runtime_version_tuple(zlib_version=zlib.ZLIB_RUNTIME_VERSION): ZLIB_RUNTIME_VERSION_TUPLE = _zlib_runtime_version_tuple() -# bpo-46623: On s390x, when a hardware accelerator is used, using different -# ways to compress data with zlib can produce different compressed data. +# bpo-46623: When a hardware accelerator is used (currently only on s390x), +# using different ways to compress data with zlib can produce different +# compressed data. # Simplified test_pair() code: # # def func1(data): @@ -57,8 +58,10 @@ def _zlib_runtime_version_tuple(zlib_version=zlib.ZLIB_RUNTIME_VERSION): # # zlib.decompress(func1(data)) == zlib.decompress(func2(data)) == data # -# Make the assumption that s390x always has an accelerator to simplify the skip -# condition. +# To simplify the skip condition, make the assumption that s390x always has an +# accelerator, and nothing else has it. +HW_ACCELERATED = is_s390x + class VersionTestCase(unittest.TestCase): @@ -223,12 +226,14 @@ def test_keywords(self): bufsize=zlib.DEF_BUF_SIZE), HAMLET_SCENE) - @skip_on_s390x def test_speech128(self): # compress more data data = HAMLET_SCENE * 128 x = zlib.compress(data) - self.assertEqual(zlib.compress(bytearray(data)), x) + # With hardware acceleration, the compressed bytes + # might not be identical. + if not HW_ACCELERATED: + self.assertEqual(zlib.compress(bytearray(data)), x) for ob in x, bytearray(x): self.assertEqual(zlib.decompress(ob), data) @@ -275,7 +280,6 @@ def test_64bit_compress(self, size): class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): # Test compression object - @skip_on_s390x def test_pair(self): # straightforward compress/decompress objects datasrc = HAMLET_SCENE * 128 @@ -286,7 +290,10 @@ def test_pair(self): x1 = co.compress(data) x2 = co.flush() self.assertRaises(zlib.error, co.flush) # second flush should not work - self.assertEqual(x1 + x2, datazip) + # With hardware acceleration, the compressed bytes might not + # be identical. + if not HW_ACCELERATED: + self.assertEqual(x1 + x2, datazip) for v1, v2 in ((x1, x2), (bytearray(x1), bytearray(x2))): dco = zlib.decompressobj() y1 = dco.decompress(v1 + v2) @@ -498,20 +505,16 @@ def test_flushes(self): for sync in sync_opt: for level in range(10): - try: + with self.subTest(sync=sync, level=level): obj = zlib.compressobj( level ) a = obj.compress( data[:3000] ) b = obj.flush( sync ) c = obj.compress( data[3000:] ) d = obj.flush() - except: - print("Error for flush mode={}, level={}" - .format(sync, level)) - raise - self.assertEqual(zlib.decompress(b''.join([a,b,c,d])), - data, ("Decompress failed: flush " - "mode=%i, level=%i") % (sync, level)) - del obj + self.assertEqual(zlib.decompress(b''.join([a,b,c,d])), + data, ("Decompress failed: flush " + "mode=%i, level=%i") % (sync, level)) + del obj @unittest.skipUnless(hasattr(zlib, 'Z_SYNC_FLUSH'), 'requires zlib.Z_SYNC_FLUSH') diff --git a/Lib/test/translationdata/argparse/msgids.txt b/Lib/test/translationdata/argparse/msgids.txt new file mode 100644 index 00000000000000..ae89ac74726ecf --- /dev/null +++ b/Lib/test/translationdata/argparse/msgids.txt @@ -0,0 +1,32 @@ + (default: %(default)s) +%(heading)s: +%(prog)s: error: %(message)s\n +%(prog)s: warning: %(message)s\n +ambiguous option: %(option)s could match %(matches)s +argument "-" with mode %r +argument %(argument_name)s: %(message)s +argument '%(argument_name)s' is deprecated +can't open '%(filename)s': %(error)s +command '%(parser_name)s' is deprecated +conflicting option string: %s +expected %s argument +expected at least one argument +expected at most one argument +expected one argument +ignored explicit argument %r +invalid %(type)s value: %(value)r +invalid choice: %(value)r (choose from %(choices)s) +invalid choice: %(value)r, maybe you meant %(closest)r? (choose from %(choices)s) +not allowed with argument %s +one of the arguments %s is required +option '%(option)s' is deprecated +options +positional arguments +show program's version number and exit +show this help message and exit +subcommands +the following arguments are required: %s +unexpected option string: %s +unknown parser %(parser_name)r (choices: %(choices)s) +unrecognized arguments: %s +usage: \ No newline at end of file diff --git a/Lib/test/translationdata/getopt/msgids.txt b/Lib/test/translationdata/getopt/msgids.txt new file mode 100644 index 00000000000000..5c0c02d452d156 --- /dev/null +++ b/Lib/test/translationdata/getopt/msgids.txt @@ -0,0 +1,6 @@ +option -%s not recognized +option -%s requires argument +option --%s must not have an argument +option --%s not a unique prefix; possible options: %s +option --%s not recognized +option --%s requires argument \ No newline at end of file diff --git a/Lib/test/translationdata/optparse/msgids.txt b/Lib/test/translationdata/optparse/msgids.txt new file mode 100644 index 00000000000000..8f405a2bf26dbe --- /dev/null +++ b/Lib/test/translationdata/optparse/msgids.txt @@ -0,0 +1,15 @@ +%(option)s option requires %(number)d argument +%prog [options] +%s option does not take a value +Options +Usage +Usage: %s\n +ambiguous option: %s (%s?) +complex +floating-point +integer +no such option: %s +option %s: invalid %s value: %r +option %s: invalid choice: %r (choose from %s) +show program's version number and exit +show this help message and exit \ No newline at end of file diff --git a/Lib/test/typinganndata/ann_module.py b/Lib/test/typinganndata/ann_module.py index 5081e6b58345a9..e1a1792cb4a867 100644 --- a/Lib/test/typinganndata/ann_module.py +++ b/Lib/test/typinganndata/ann_module.py @@ -8,8 +8,6 @@ from typing import Optional from functools import wraps -__annotations__[1] = 2 - class C: x = 5; y: Optional['C'] = None @@ -18,8 +16,6 @@ class C: x: int = 5; y: str = x; f: Tuple[int, int] class M(type): - - __annotations__['123'] = 123 o: type = object (pars): bool = True diff --git a/Lib/test/typinganndata/ann_module695.py b/Lib/test/typinganndata/ann_module695.py new file mode 100644 index 00000000000000..b6f3b06bd5065f --- /dev/null +++ b/Lib/test/typinganndata/ann_module695.py @@ -0,0 +1,72 @@ +from __future__ import annotations +from typing import Callable + + +class A[T, *Ts, **P]: + x: T + y: tuple[*Ts] + z: Callable[P, str] + + +class B[T, *Ts, **P]: + T = int + Ts = str + P = bytes + x: T + y: Ts + z: P + + +Eggs = int +Spam = str + + +class C[Eggs, **Spam]: + x: Eggs + y: Spam + + +def generic_function[T, *Ts, **P]( + x: T, *y: *Ts, z: P.args, zz: P.kwargs +) -> None: ... + + +def generic_function_2[Eggs, **Spam](x: Eggs, y: Spam): pass + + +class D: + Foo = int + Bar = str + + def generic_method[Foo, **Bar]( + self, x: Foo, y: Bar + ) -> None: ... + + def generic_method_2[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + +def nested(): + from types import SimpleNamespace + from typing import get_type_hints + + Eggs = bytes + Spam = memoryview + + + class E[Eggs, **Spam]: + x: Eggs + y: Spam + + def generic_method[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + + def generic_function[Eggs, **Spam](x: Eggs, y: Spam): pass + + + return SimpleNamespace( + E=E, + hints_for_E=get_type_hints(E), + hints_for_E_meth=get_type_hints(E.generic_method), + generic_func=generic_function, + hints_for_generic_func=get_type_hints(generic_function) + ) diff --git a/Lib/test/wheeldata/wheel-0.40.0-py3-none-any.whl b/Lib/test/wheeldata/wheel-0.40.0-py3-none-any.whl deleted file mode 100644 index 410132385bba4d..00000000000000 Binary files a/Lib/test/wheeldata/wheel-0.40.0-py3-none-any.whl and /dev/null differ diff --git a/Lib/test/wheeldata/wheel-0.43.0-py3-none-any.whl b/Lib/test/wheeldata/wheel-0.43.0-py3-none-any.whl new file mode 100644 index 00000000000000..67e2308717d675 Binary files /dev/null and b/Lib/test/wheeldata/wheel-0.43.0-py3-none-any.whl differ diff --git a/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part b/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part new file mode 100644 index 00000000000000..c6beae8e2552d6 Binary files /dev/null and b/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part differ diff --git a/Lib/test/zipimport_data/sparse-zip64-c0-0x100000000.part b/Lib/test/zipimport_data/sparse-zip64-c0-0x100000000.part new file mode 100644 index 00000000000000..74ab03b4648948 Binary files /dev/null and b/Lib/test/zipimport_data/sparse-zip64-c0-0x100000000.part differ diff --git a/Lib/test/zipimport_data/sparse-zip64-c0-0x200000000.part b/Lib/test/zipimport_data/sparse-zip64-c0-0x200000000.part new file mode 100644 index 00000000000000..9769a404f675d4 Binary files /dev/null and b/Lib/test/zipimport_data/sparse-zip64-c0-0x200000000.part differ diff --git a/Lib/textwrap.py b/Lib/textwrap.py index 7ca393d1c371aa..1bf07aa46cad99 100644 --- a/Lib/textwrap.py +++ b/Lib/textwrap.py @@ -2,7 +2,7 @@ """ # Copyright (C) 1999-2001 Gregory P. Ward. -# Copyright (C) 2002, 2003 Python Software Foundation. +# Copyright (C) 2002 Python Software Foundation. # Written by Greg Ward import re diff --git a/Lib/threading.py b/Lib/threading.py index ec89550d6b022e..da9cdf0b09d83c 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -3,7 +3,6 @@ import os as _os import sys as _sys import _thread -import warnings from time import monotonic as _time from _weakrefset import WeakSet @@ -36,8 +35,11 @@ _daemon_threads_allowed = _thread.daemon_threads_allowed _allocate_lock = _thread.allocate_lock _LockType = _thread.LockType -_set_sentinel = _thread._set_sentinel +_thread_shutdown = _thread._shutdown +_make_thread_handle = _thread._make_thread_handle +_ThreadHandle = _thread._ThreadHandle get_ident = _thread.get_ident +_get_main_thread_ident = _thread._get_main_thread_ident _is_main_interpreter = _thread._is_main_interpreter try: get_native_id = _thread.get_native_id @@ -45,6 +47,10 @@ __all__.append('get_native_id') except AttributeError: _HAVE_THREAD_NATIVE_ID = False +try: + _set_name = _thread.set_name +except AttributeError: + _set_name = None ThreadError = _thread.error try: _CRLock = _thread.RLock @@ -126,6 +132,7 @@ def RLock(*args, **kwargs): """ if args or kwargs: + import warnings warnings.warn( 'Passing arguments to RLock is deprecated and will be removed in 3.15', DeprecationWarning, @@ -333,7 +340,7 @@ def wait(self, timeout=None): awakened or timed out, it re-acquires the lock and returns. When the timeout argument is present and not None, it should be a - floating point number specifying a timeout for the operation in seconds + floating-point number specifying a timeout for the operation in seconds (or fractions thereof). When the underlying lock is an RLock, it is not released using its @@ -643,7 +650,7 @@ def wait(self, timeout=None): the optional timeout occurs. When the timeout argument is present and not None, it should be a - floating point number specifying a timeout for the operation in seconds + floating-point number specifying a timeout for the operation in seconds (or fractions thereof). This method returns the internal flag on exit, so it will always return @@ -686,6 +693,8 @@ def __init__(self, parties, action=None, timeout=None): default for all subsequent 'wait()' calls. """ + if parties < 1: + raise ValueError("parties must be >= 1") self._cond = Condition(Lock()) self._action = action self._timeout = timeout @@ -847,25 +856,6 @@ def _newname(name_template): _limbo = {} _dangling = WeakSet() -# Set of Thread._tstate_lock locks of non-daemon threads used by _shutdown() -# to wait until all Python thread states get deleted: -# see Thread._set_tstate_lock(). -_shutdown_locks_lock = _allocate_lock() -_shutdown_locks = set() - -def _maintain_shutdown_locks(): - """ - Drop any shutdown locks that don't correspond to running threads anymore. - - Calling this from time to time avoids an ever-growing _shutdown_locks - set when Thread objects are not joined explicitly. See bpo-37788. - - This must be called with _shutdown_locks_lock acquired. - """ - # If a lock was released, the corresponding thread has exited - to_remove = [lock for lock in _shutdown_locks if not lock.locked()] - _shutdown_locks.difference_update(to_remove) - # Main class for threads @@ -930,10 +920,8 @@ class is implemented. self._ident = None if _HAVE_THREAD_NATIVE_ID: self._native_id = None - self._tstate_lock = None - self._handle = None + self._handle = _ThreadHandle() self._started = Event() - self._is_stopped = False self._initialized = True # Copy of sys.stderr used by self._invoke_excepthook() self._stderr = _sys.stderr @@ -947,28 +935,18 @@ def _after_fork(self, new_ident=None): if new_ident is not None: # This thread is alive. self._ident = new_ident - if self._handle is not None: - assert self._handle.ident == new_ident - # bpo-42350: If the fork happens when the thread is already stopped - # (ex: after threading._shutdown() has been called), _tstate_lock - # is None. Do nothing in this case. - if self._tstate_lock is not None: - self._tstate_lock._at_fork_reinit() - self._tstate_lock.acquire() + assert self._handle.ident == new_ident else: - # This thread isn't alive after fork: it doesn't have a tstate - # anymore. - self._is_stopped = True - self._tstate_lock = None - self._handle = None + # Otherwise, the thread is dead, Jim. _PyThread_AfterFork() + # already marked our handle done. + pass def __repr__(self): assert self._initialized, "Thread.__init__() was not called" status = "initial" if self._started.is_set(): status = "started" - self.is_alive() # easy way to get ._is_stopped set when appropriate - if self._is_stopped: + if self._handle.is_done(): status = "stopped" if self._daemonic: status += " daemon" @@ -996,7 +974,8 @@ def start(self): _limbo[self] = self try: # Start joinable thread - self._handle = _start_joinable_thread(self._bootstrap) + _start_joinable_thread(self._bootstrap, handle=self._handle, + daemon=self.daemon) except Exception: with _active_limbo_lock: del _limbo[self] @@ -1047,25 +1026,20 @@ def _set_ident(self): def _set_native_id(self): self._native_id = get_native_id() - def _set_tstate_lock(self): - """ - Set a lock object which will be released by the interpreter when - the underlying thread state (see pystate.h) gets deleted. - """ - self._tstate_lock = _set_sentinel() - self._tstate_lock.acquire() - - if not self.daemon: - with _shutdown_locks_lock: - _maintain_shutdown_locks() - _shutdown_locks.add(self._tstate_lock) + def _set_os_name(self): + if _set_name is None or not self._name: + return + try: + _set_name(self._name) + except OSError: + pass def _bootstrap_inner(self): try: self._set_ident() - self._set_tstate_lock() if _HAVE_THREAD_NATIVE_ID: self._set_native_id() + self._set_os_name() self._started.set() with _active_limbo_lock: _active[self._ident] = self @@ -1083,33 +1057,6 @@ def _bootstrap_inner(self): finally: self._delete() - def _stop(self): - # After calling ._stop(), .is_alive() returns False and .join() returns - # immediately. ._tstate_lock must be released before calling ._stop(). - # - # Normal case: C code at the end of the thread's life - # (release_sentinel in _threadmodule.c) releases ._tstate_lock, and - # that's detected by our ._wait_for_tstate_lock(), called by .join() - # and .is_alive(). Any number of threads _may_ call ._stop() - # simultaneously (for example, if multiple threads are blocked in - # .join() calls), and they're not serialized. That's harmless - - # they'll just make redundant rebindings of ._is_stopped and - # ._tstate_lock. Obscure: we rebind ._tstate_lock last so that the - # "assert self._is_stopped" in ._wait_for_tstate_lock() always works - # (the assert is executed only if ._tstate_lock is None). - # - # Special case: _main_thread releases ._tstate_lock via this - # module's _shutdown() function. - lock = self._tstate_lock - if lock is not None: - assert not lock.locked() - self._is_stopped = True - self._tstate_lock = None - if not self.daemon: - with _shutdown_locks_lock: - # Remove our lock and other released locks from _shutdown_locks - _maintain_shutdown_locks() - def _delete(self): "Remove current thread from the dict of currently running threads." with _active_limbo_lock: @@ -1127,7 +1074,7 @@ def join(self, timeout=None): or until the optional timeout occurs. When the timeout argument is present and not None, it should be a - floating point number specifying a timeout for the operation in seconds + floating-point number specifying a timeout for the operation in seconds (or fractions thereof). As join() always returns None, you must call is_alive() after join() to decide whether a timeout happened -- if the thread is still alive, the join() call timed out. @@ -1150,47 +1097,12 @@ def join(self, timeout=None): if self is current_thread(): raise RuntimeError("cannot join current thread") - if timeout is None: - self._wait_for_tstate_lock() - else: - # the behavior of a negative timeout isn't documented, but - # historically .join(timeout=x) for x<0 has acted as if timeout=0 - self._wait_for_tstate_lock(timeout=max(timeout, 0)) - - if self._is_stopped: - self._join_os_thread() - - def _join_os_thread(self): - # self._handle may be cleared post-fork - if self._handle is not None: - self._handle.join() - - def _wait_for_tstate_lock(self, block=True, timeout=-1): - # Issue #18808: wait for the thread state to be gone. - # At the end of the thread's life, after all knowledge of the thread - # is removed from C data structures, C code releases our _tstate_lock. - # This method passes its arguments to _tstate_lock.acquire(). - # If the lock is acquired, the C code is done, and self._stop() is - # called. That sets ._is_stopped to True, and ._tstate_lock to None. - lock = self._tstate_lock - if lock is None: - # already determined that the C code is done - assert self._is_stopped - return + # the behavior of a negative timeout isn't documented, but + # historically .join(timeout=x) for x<0 has acted as if timeout=0 + if timeout is not None: + timeout = max(timeout, 0) - try: - if lock.acquire(block, timeout): - lock.release() - self._stop() - except: - if lock.locked(): - # bpo-45274: lock.acquire() acquired the lock, but the function - # was interrupted with an exception before reaching the - # lock.release(). It can happen if a signal handler raises an - # exception, like CTRL+C which raises KeyboardInterrupt. - lock.release() - self._stop() - raise + self._handle.join(timeout) @property def name(self): @@ -1207,6 +1119,8 @@ def name(self): def name(self, name): assert self._initialized, "Thread.__init__() not called" self._name = str(name) + if get_ident() == self._ident: + self._set_os_name() @property def ident(self): @@ -1241,13 +1155,7 @@ def is_alive(self): """ assert self._initialized, "Thread.__init__() not called" - if self._is_stopped or not self._started.is_set(): - return False - self._wait_for_tstate_lock(False) - if not self._is_stopped: - return True - self._join_os_thread() - return False + return self._started.is_set() and not self._handle.is_done() @property def daemon(self): @@ -1456,18 +1364,14 @@ class _MainThread(Thread): def __init__(self): Thread.__init__(self, name="MainThread", daemon=False) - self._set_tstate_lock() self._started.set() - self._set_ident() + self._ident = _get_main_thread_ident() + self._handle = _make_thread_handle(self._ident) if _HAVE_THREAD_NATIVE_ID: self._set_native_id() with _active_limbo_lock: _active[self._ident] = self - def _join_os_thread(self): - # No ThreadHandle for main thread - pass - # Helper thread-local instance to detect when a _DummyThread # is collected. Not a part of the public API. @@ -1510,17 +1414,15 @@ def __init__(self): daemon=_daemon_threads_allowed()) self._started.set() self._set_ident() + self._handle = _make_thread_handle(self._ident) if _HAVE_THREAD_NATIVE_ID: self._set_native_id() with _active_limbo_lock: _active[self._ident] = self _DeleteDummyThreadOnDel(self) - def _stop(self): - pass - def is_alive(self): - if not self._is_stopped and self._started.is_set(): + if not self._handle.is_done() and self._started.is_set(): return True raise RuntimeError("thread is not alive") @@ -1532,7 +1434,6 @@ def _after_fork(self, new_ident=None): self.__class__ = _MainThread self._name = 'MainThread' self._daemonic = False - self._set_tstate_lock() Thread._after_fork(self, new_ident=new_ident) @@ -1631,12 +1532,11 @@ def _shutdown(): """ Wait until the Python thread state of all non-daemon threads get deleted. """ - # Obscure: other threads may be waiting to join _main_thread. That's - # dubious, but some code does it. We can't wait for C code to release - # the main thread's tstate_lock - that won't happen until the interpreter - # is nearly dead. So we release it here. Note that just calling _stop() - # isn't enough: other threads may already be waiting on _tstate_lock. - if _main_thread._is_stopped and _is_main_interpreter(): + # Obscure: other threads may be waiting to join _main_thread. That's + # dubious, but some code does it. We can't wait for it to be marked as done + # normally - that won't happen until the interpreter is nearly dead. So + # mark it done here. + if _main_thread._handle.is_done() and _is_main_interpreter(): # _shutdown() was already called return @@ -1648,42 +1548,11 @@ def _shutdown(): for atexit_call in reversed(_threading_atexits): atexit_call() - # Main thread - if _main_thread.ident == get_ident(): - tlock = _main_thread._tstate_lock - # The main thread isn't finished yet, so its thread state lock can't - # have been released. - assert tlock is not None - if tlock.locked(): - # It should have been released already by - # _PyInterpreterState_SetNotRunningMain(), but there may be - # embedders that aren't calling that yet. - tlock.release() - _main_thread._stop() - else: - # bpo-1596321: _shutdown() must be called in the main thread. - # If the threading module was not imported by the main thread, - # _main_thread is the thread which imported the threading module. - # In this case, ignore _main_thread, similar behavior than for threads - # spawned by C libraries or using _thread.start_new_thread(). - pass - - # Join all non-deamon threads - while True: - with _shutdown_locks_lock: - locks = list(_shutdown_locks) - _shutdown_locks.clear() - - if not locks: - break - - for lock in locks: - # mimic Thread.join() - lock.acquire() - lock.release() - - # new threads can be spawned while we were waiting for the other - # threads to complete + if _is_main_interpreter(): + _main_thread._handle._set_done() + + # Wait for all non-daemon threads to exit. + _thread_shutdown() def main_thread(): @@ -1703,7 +1572,6 @@ def _after_fork(): # Reset _active_limbo_lock, in case we forked while the lock was held # by another (non-forked) thread. http://bugs.python.org/issue874900 global _active_limbo_lock, _main_thread - global _shutdown_locks_lock, _shutdown_locks _active_limbo_lock = RLock() # fork() only copied the current thread; clear references to others. @@ -1719,10 +1587,6 @@ def _after_fork(): _main_thread = current - # reset _shutdown() locks: threads re-register their _tstate_lock below - _shutdown_locks_lock = _allocate_lock() - _shutdown_locks = set() - with _active_limbo_lock: # Dangling thread instances must still have their locks reset, # because someone may join() them. @@ -1739,7 +1603,6 @@ def _after_fork(): else: # All the others are already stopped. thread._after_fork() - thread._stop() _limbo.clear() _active.clear() diff --git a/Lib/timeit.py b/Lib/timeit.py old mode 100755 new mode 100644 index 02cfafaf36e5d1..c106e0f67356da --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """Tool for measuring execution time of small code snippets. This module avoids a number of common traps for measuring execution diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 175bfbd7d912d2..0baed8b569e40f 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -41,6 +41,7 @@ import re wantobjects = 1 +_debug = False # set to True to print executed Tcl/Tk commands TkVersion = float(_tkinter.TK_VERSION) TclVersion = float(_tkinter.TCL_VERSION) @@ -69,7 +70,10 @@ def _stringify(value): else: value = '{%s}' % _join(value) else: - value = str(value) + if isinstance(value, bytes): + value = str(value, 'latin1') + else: + value = str(value) if not value: value = '{}' elif _magic_re.search(value): @@ -291,6 +295,8 @@ def __repr__(self): ''.join(' %s=%s' % (k, attrs[k]) for k in keys if k in attrs) ) + __class_getitem__ = classmethod(types.GenericAlias) + _support_default_root = True _default_root = None @@ -411,7 +417,6 @@ def __del__(self): self._tk.globalunsetvar(self._name) if self._tclCommands is not None: for name in self._tclCommands: - #print '- Tkinter: deleted command', name self._tk.deletecommand(name) self._tclCommands = None @@ -683,7 +688,6 @@ def destroy(self): this widget in the Tcl interpreter.""" if self._tclCommands is not None: for name in self._tclCommands: - #print '- Tkinter: deleted command', name self.tk.deletecommand(name) self._tclCommands = None @@ -691,7 +695,6 @@ def deletecommand(self, name): """Internal function. Delete the Tcl command provided in NAME.""" - #print '- Tkinter: deleted command', name self.tk.deletecommand(name) try: self._tclCommands.remove(name) @@ -844,7 +847,7 @@ def tk_focusPrev(self): if not name: return None return self._nametowidget(name) - def after(self, ms, func=None, *args): + def after(self, ms, func=None, *args, **kw): """Call function once after given time. MS specifies the time in milliseconds. FUNC gives the @@ -858,7 +861,7 @@ def after(self, ms, func=None, *args): else: def callit(): try: - func(*args) + func(*args, **kw) finally: try: self.deletecommand(name) @@ -872,13 +875,13 @@ def callit(): name = self._register(callit) return self.tk.call('after', ms, name) - def after_idle(self, func, *args): + def after_idle(self, func, *args, **kw): """Call FUNC once if the Tcl main loop has no event to process. Return an identifier to cancel the scheduling with after_cancel.""" - return self.after('idle', func, *args) + return self.after('idle', func, *args, **kw) def after_cancel(self, id): """Cancel scheduling of function identified with ID. @@ -897,6 +900,21 @@ def after_cancel(self, id): pass self.tk.call('after', 'cancel', id) + def after_info(self, id=None): + """Return information about existing event handlers. + + With no argument, return a tuple of the identifiers for all existing + event handlers created by the after and after_idle commands for this + interpreter. If id is supplied, it specifies an existing handler; id + must have been the return value from some previous call to after or + after_idle and it must not have triggered yet or been canceled. If the + id doesn't exist, a TclError is raised. Otherwise, the return value is + a tuple containing (script, type) where script is a reference to the + function to be called by the event handler and type is either 'idle' + or 'timer' to indicate what kind of event handler it is. + """ + return self.tk.splitlist(self.tk.call('after', 'info', id)) + def bell(self, displayof=0): """Ring a display's bell.""" self.tk.call(('bell',) + self._displayof(displayof)) @@ -1711,6 +1729,9 @@ def getint_event(s): except (ValueError, TclError): return s + if any(isinstance(s, tuple) for s in args): + args = [s[0] if isinstance(s, tuple) and len(s) == 1 else s + for s in args] nsign, b, f, h, k, s, t, w, x, y, A, E, K, N, W, T, X, Y, D = args # Missing: (a, c, d, m, o, v, B, R) e = Event() @@ -1746,7 +1767,10 @@ def getint_event(s): try: e.type = EventType(T) except ValueError: - e.type = T + try: + e.type = EventType(str(T)) # can be int + except ValueError: + e.type = T try: e.widget = self._nametowidget(W) except KeyError: @@ -2241,7 +2265,7 @@ def wm_iconbitmap(self, bitmap=None, default=None): explicitly. DEFAULT can be the relative path to a .ico file (example: root.iconbitmap(default='myicon.ico') ). See Tk documentation for more information.""" - if default: + if default is not None: return self.tk.call('wm', 'iconbitmap', self._w, '-default', default) else: return self.tk.call('wm', 'iconbitmap', self._w, bitmap) @@ -2435,6 +2459,8 @@ def __init__(self, screenName=None, baseName=None, className='Tk', baseName = baseName + ext interactive = False self.tk = _tkinter.create(screenName, baseName, className, interactive, wantobjects, useTk, sync, use) + if _debug: + self.tk.settrace(_print_command) if useTk: self._loadtk() if not sys.flags.ignore_environment: @@ -2521,6 +2547,14 @@ def __getattr__(self, attr): "Delegate attribute access to the interpreter object" return getattr(self.tk, attr) + +def _print_command(cmd, *, file=sys.stderr): + # Print executed Tcl/Tk commands. + assert isinstance(cmd, tuple) + cmd = _join(cmd) + print(cmd, file=file) + + # Ideally, the classes Pack, Place and Grid disappear, the # pack/place/grid methods are defined on the Widget class, and # everybody uses w.pack_whatever(...) instead of Pack.whatever(w, @@ -2707,6 +2741,8 @@ def _setup(self, master, cnf): del cnf['name'] if not name: name = self.__class__.__name__.lower() + if name[-1].isdigit(): + name += "!" # Avoid duplication when calculating names below if master._last_child_ids is None: master._last_child_ids = {} count = master._last_child_ids.get(name, 0) + 1 @@ -3166,11 +3202,16 @@ def __init__(self, master=None, cnf={}, **kw): Widget.__init__(self, master, 'checkbutton', cnf, kw) def _setup(self, master, cnf): + # Because Checkbutton defaults to a variable with the same name as + # the widget, Checkbutton default names must be globally unique, + # not just unique within the parent widget. if not cnf.get('name'): global _checkbutton_count name = self.__class__.__name__.lower() _checkbutton_count += 1 - cnf['name'] = f'!{name}{_checkbutton_count}' + # To avoid collisions with ttk.Checkbutton, use the different + # name template. + cnf['name'] = f'!{name}-{_checkbutton_count}' super()._setup(master, cnf) def deselect(self): @@ -4258,33 +4299,112 @@ def cget(self, option): def __getitem__(self, key): return self.tk.call(self.name, 'cget', '-' + key) - # XXX copy -from, -to, ...? - def copy(self): - """Return a new PhotoImage with the same image as this widget.""" + def copy(self, *, from_coords=None, zoom=None, subsample=None): + """Return a new PhotoImage with the same image as this widget. + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied. It must be a tuple or a list of 1 to 4 + integers (x1, y1, x2, y2). (x1, y1) and (x2, y2) specify diagonally + opposite corners of the rectangle. If x2 and y2 are not specified, + the default value is the bottom-right corner of the source image. + The pixels copied will include the left and top edges of the + specified rectangle but not the bottom or right edges. If the + FROM_COORDS option is not given, the default is the whole source + image. + + If SUBSAMPLE or ZOOM are specified, the image is transformed as in + the subsample() or zoom() methods. The value must be a single + integer or a pair of integers. + """ destImage = PhotoImage(master=self.tk) - self.tk.call(destImage, 'copy', self.name) + destImage.copy_replace(self, from_coords=from_coords, + zoom=zoom, subsample=subsample) return destImage - def zoom(self, x, y=''): + def zoom(self, x, y='', *, from_coords=None): """Return a new PhotoImage with the same image as this widget - but zoom it with a factor of x in the X direction and y in the Y - direction. If y is not given, the default value is the same as x. + but zoom it with a factor of X in the X direction and Y in the Y + direction. If Y is not given, the default value is the same as X. + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied, as in the copy() method. """ - destImage = PhotoImage(master=self.tk) if y=='': y=x - self.tk.call(destImage, 'copy', self.name, '-zoom',x,y) - return destImage + return self.copy(zoom=(x, y), from_coords=from_coords) - def subsample(self, x, y=''): + def subsample(self, x, y='', *, from_coords=None): """Return a new PhotoImage based on the same image as this widget - but use only every Xth or Yth pixel. If y is not given, the - default value is the same as x. + but use only every Xth or Yth pixel. If Y is not given, the + default value is the same as X. + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied, as in the copy() method. """ - destImage = PhotoImage(master=self.tk) if y=='': y=x - self.tk.call(destImage, 'copy', self.name, '-subsample',x,y) - return destImage + return self.copy(subsample=(x, y), from_coords=from_coords) + + def copy_replace(self, sourceImage, *, from_coords=None, to=None, shrink=False, + zoom=None, subsample=None, compositingrule=None): + """Copy a region from the source image (which must be a PhotoImage) to + this image, possibly with pixel zooming and/or subsampling. If no + options are specified, this command copies the whole of the source + image into this image, starting at coordinates (0, 0). + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied. It must be a tuple or a list of 1 to 4 + integers (x1, y1, x2, y2). (x1, y1) and (x2, y2) specify diagonally + opposite corners of the rectangle. If x2 and y2 are not specified, + the default value is the bottom-right corner of the source image. + The pixels copied will include the left and top edges of the + specified rectangle but not the bottom or right edges. If the + FROM_COORDS option is not given, the default is the whole source + image. + + The TO option specifies a rectangular sub-region of the destination + image to be affected. It must be a tuple or a list of 1 to 4 + integers (x1, y1, x2, y2). (x1, y1) and (x2, y2) specify diagonally + opposite corners of the rectangle. If x2 and y2 are not specified, + the default value is (x1,y1) plus the size of the source region + (after subsampling and zooming, if specified). If x2 and y2 are + specified, the source region will be replicated if necessary to fill + the destination region in a tiled fashion. + + If SHRINK is true, the size of the destination image should be + reduced, if necessary, so that the region being copied into is at + the bottom-right corner of the image. + + If SUBSAMPLE or ZOOM are specified, the image is transformed as in + the subsample() or zoom() methods. The value must be a single + integer or a pair of integers. + + The COMPOSITINGRULE option specifies how transparent pixels in the + source image are combined with the destination image. When a + compositing rule of 'overlay' is set, the old contents of the + destination image are visible, as if the source image were printed + on a piece of transparent film and placed over the top of the + destination. When a compositing rule of 'set' is set, the old + contents of the destination image are discarded and the source image + is used as-is. The default compositing rule is 'overlay'. + """ + options = [] + if from_coords is not None: + options.extend(('-from', *from_coords)) + if to is not None: + options.extend(('-to', *to)) + if shrink: + options.append('-shrink') + if zoom is not None: + if not isinstance(zoom, (tuple, list)): + zoom = (zoom,) + options.extend(('-zoom', *zoom)) + if subsample is not None: + if not isinstance(subsample, (tuple, list)): + subsample = (subsample,) + options.extend(('-subsample', *subsample)) + if compositingrule: + options.extend(('-compositingrule', compositingrule)) + self.tk.call(self.name, 'copy', sourceImage, *options) def get(self, x, y): """Return the color (red, green, blue) of the pixel at X,Y.""" @@ -4299,17 +4419,117 @@ def put(self, data, to=None): to = to[1:] args = args + ('-to',) + tuple(to) self.tk.call(args) - # XXX read - - def write(self, filename, format=None, from_coords=None): - """Write image to file FILENAME in FORMAT starting from - position FROM_COORDS.""" - args = (self.name, 'write', filename) - if format: - args = args + ('-format', format) - if from_coords: - args = args + ('-from',) + tuple(from_coords) - self.tk.call(args) + + def read(self, filename, format=None, *, from_coords=None, to=None, shrink=False): + """Reads image data from the file named FILENAME into the image. + + The FORMAT option specifies the format of the image data in the + file. + + The FROM_COORDS option specifies a rectangular sub-region of the image + file data to be copied to the destination image. It must be a tuple + or a list of 1 to 4 integers (x1, y1, x2, y2). (x1, y1) and + (x2, y2) specify diagonally opposite corners of the rectangle. If + x2 and y2 are not specified, the default value is the bottom-right + corner of the source image. The default, if this option is not + specified, is the whole of the image in the image file. + + The TO option specifies the coordinates of the top-left corner of + the region of the image into which data from filename are to be + read. The default is (0, 0). + + If SHRINK is true, the size of the destination image will be + reduced, if necessary, so that the region into which the image file + data are read is at the bottom-right corner of the image. + """ + options = () + if format is not None: + options += ('-format', format) + if from_coords is not None: + options += ('-from', *from_coords) + if shrink: + options += ('-shrink',) + if to is not None: + options += ('-to', *to) + self.tk.call(self.name, 'read', filename, *options) + + def write(self, filename, format=None, from_coords=None, *, + background=None, grayscale=False): + """Writes image data from the image to a file named FILENAME. + + The FORMAT option specifies the name of the image file format + handler to be used to write the data to the file. If this option + is not given, the format is guessed from the file extension. + + The FROM_COORDS option specifies a rectangular region of the image + to be written to the image file. It must be a tuple or a list of 1 + to 4 integers (x1, y1, x2, y2). If only x1 and y1 are specified, + the region extends from (x1,y1) to the bottom-right corner of the + image. If all four coordinates are given, they specify diagonally + opposite corners of the rectangular region. The default, if this + option is not given, is the whole image. + + If BACKGROUND is specified, the data will not contain any + transparency information. In all transparent pixels the color will + be replaced by the specified color. + + If GRAYSCALE is true, the data will not contain color information. + All pixel data will be transformed into grayscale. + """ + options = () + if format is not None: + options += ('-format', format) + if from_coords is not None: + options += ('-from', *from_coords) + if grayscale: + options += ('-grayscale',) + if background is not None: + options += ('-background', background) + self.tk.call(self.name, 'write', filename, *options) + + def data(self, format=None, *, from_coords=None, + background=None, grayscale=False): + """Returns image data. + + The FORMAT option specifies the name of the image file format + handler to be used. If this option is not given, this method uses + a format that consists of a tuple (one element per row) of strings + containing space-separated (one element per pixel/column) colors + in “#RRGGBB” format (where RR is a pair of hexadecimal digits for + the red channel, GG for green, and BB for blue). + + The FROM_COORDS option specifies a rectangular region of the image + to be returned. It must be a tuple or a list of 1 to 4 integers + (x1, y1, x2, y2). If only x1 and y1 are specified, the region + extends from (x1,y1) to the bottom-right corner of the image. If + all four coordinates are given, they specify diagonally opposite + corners of the rectangular region, including (x1, y1) and excluding + (x2, y2). The default, if this option is not given, is the whole + image. + + If BACKGROUND is specified, the data will not contain any + transparency information. In all transparent pixels the color will + be replaced by the specified color. + + If GRAYSCALE is true, the data will not contain color information. + All pixel data will be transformed into grayscale. + """ + options = () + if format is not None: + options += ('-format', format) + if from_coords is not None: + options += ('-from', *from_coords) + if grayscale: + options += ('-grayscale',) + if background is not None: + options += ('-background', background) + data = self.tk.call(self.name, 'data', *options) + if isinstance(data, str): # For wantobjects = 0. + if format is None: + data = self.tk.splitlist(data) + else: + data = bytes(data, 'latin1') + return data def transparency_get(self, x, y): """Return True if the pixel at x,y is transparent.""" diff --git a/Lib/tkinter/simpledialog.py b/Lib/tkinter/simpledialog.py index 0f0dc66460f798..6e5b025a9f9d7d 100644 --- a/Lib/tkinter/simpledialog.py +++ b/Lib/tkinter/simpledialog.py @@ -357,7 +357,7 @@ def askinteger(title, prompt, **kw): class _QueryFloat(_QueryDialog): - errormessage = "Not a floating point value." + errormessage = "Not a floating-point value." def getresult(self): return self.getdouble(self.entry.get()) diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py index 5ca938a670831a..8ddb7f97e3b233 100644 --- a/Lib/tkinter/ttk.py +++ b/Lib/tkinter/ttk.py @@ -321,6 +321,8 @@ def _tclobj_to_py(val): elif hasattr(val, 'typename'): # some other (single) Tcl object val = _convert_stringval(val) + if isinstance(val, tuple) and len(val) == 0: + return '' return val def tclobjs_to_py(adict): @@ -690,7 +692,10 @@ def current(self, newindex=None): returns the index of the current value in the list of values or -1 if the current value does not appear in the list.""" if newindex is None: - return self.tk.getint(self.tk.call(self._w, "current")) + res = self.tk.call(self._w, "current") + if res == '': + return -1 + return self.tk.getint(res) return self.tk.call(self._w, "current", newindex) @@ -1522,7 +1527,7 @@ def __init__(self, master=None, variable=None, from_=0, to=10, **kw): self.label.place(anchor='n' if label_side == 'top' else 's') # update the label as scale or variable changes - self.__tracecb = self._variable.trace_variable('w', self._adjust) + self.__tracecb = self._variable.trace_add('write', self._adjust) self.bind('', self._adjust) self.bind('', self._adjust) @@ -1530,7 +1535,7 @@ def __init__(self, master=None, variable=None, from_=0, to=10, **kw): def destroy(self): """Destroy this widget and possibly its associated variable.""" try: - self._variable.trace_vdelete('w', self.__tracecb) + self._variable.trace_remove('write', self.__tracecb) except AttributeError: pass else: diff --git a/Lib/token.py b/Lib/token.py index b620317106e173..54d7cdccadc79a 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -1,7 +1,8 @@ """Token constants.""" # Auto-generated by Tools/build/generate_token.py -__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] +__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF', + 'EXACT_TOKEN_TYPES'] ENDMARKER = 0 NAME = 1 diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 7f418bb7a1b37f..9ce95a62d961ba 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -169,6 +169,7 @@ def __init__(self): self.prev_row = 1 self.prev_col = 0 self.prev_type = None + self.prev_line = "" self.encoding = None def add_whitespace(self, start): @@ -176,14 +177,28 @@ def add_whitespace(self, start): if row < self.prev_row or row == self.prev_row and col < self.prev_col: raise ValueError("start ({},{}) precedes previous end ({},{})" .format(row, col, self.prev_row, self.prev_col)) - row_offset = row - self.prev_row - if row_offset: - self.tokens.append("\\\n" * row_offset) - self.prev_col = 0 + self.add_backslash_continuation(start) col_offset = col - self.prev_col if col_offset: self.tokens.append(" " * col_offset) + def add_backslash_continuation(self, start): + """Add backslash continuation characters if the row has increased + without encountering a newline token. + + This also inserts the correct amount of whitespace before the backslash. + """ + row = start[0] + row_offset = row - self.prev_row + if row_offset == 0: + return + + newline = '\r\n' if self.prev_line.endswith('\r\n') else '\n' + line = self.prev_line.rstrip('\\\r\n') + ws = ''.join(_itertools.takewhile(str.isspace, reversed(line))) + self.tokens.append(ws + f"\\{newline}" * row_offset) + self.prev_col = 0 + def escape_brackets(self, token): characters = [] consume_until_next_bracket = False @@ -200,7 +215,7 @@ def escape_brackets(self, token): characters[-2::-1] ) ) - if n_backslashes % 2 == 0: + if n_backslashes % 2 == 0 or characters[-1] != "N": characters.append(character) else: consume_until_next_bracket = True @@ -243,8 +258,6 @@ def untokenize(self, iterable): end_line, end_col = end extra_chars = last_line.count("{{") + last_line.count("}}") end = (end_line, end_col + extra_chars) - elif tok_type in (STRING, FSTRING_START) and self.prev_type in (STRING, FSTRING_END): - self.tokens.append(" ") self.add_whitespace(start) self.tokens.append(token) @@ -253,6 +266,7 @@ def untokenize(self, iterable): self.prev_row += 1 self.prev_col = 0 self.prev_type = tok_type + self.prev_line = line return "".join(self.tokens) def compat(self, token, iterable): @@ -318,16 +332,10 @@ def untokenize(iterable): with at least two elements, a token number and token value. If only two tokens are passed, the resulting output is poor. - Round-trip invariant for full input: - Untokenized source will match input source exactly - - Round-trip invariant for limited input: - # Output bytes will tokenize back to the input - t1 = [tok[:2] for tok in tokenize(f.readline)] - newcode = untokenize(t1) - readline = BytesIO(newcode).readline - t2 = [tok[:2] for tok in tokenize(readline)] - assert t1 == t2 + The result is guaranteed to tokenize back to match the input so + that the conversion is lossless and round-trips are assured. + The guarantee applies only to the token type and token string as + the spacing between tokens (column positions) may change. """ ut = Untokenizer() out = ut.untokenize(iterable) @@ -510,7 +518,7 @@ def error(message, filename=None, location=None): sys.exit(1) # Parse the arguments and options - parser = argparse.ArgumentParser(prog='python -m tokenize') + parser = argparse.ArgumentParser() parser.add_argument(dest='filename', nargs='?', metavar='filename.py', help='the file to tokenize; defaults to stdin') diff --git a/Lib/tomllib/_parser.py b/Lib/tomllib/_parser.py index 45ca7a89630f0e..0e522c3a69e6fe 100644 --- a/Lib/tomllib/_parser.py +++ b/Lib/tomllib/_parser.py @@ -4,10 +4,7 @@ from __future__ import annotations -from collections.abc import Iterable -import string from types import MappingProxyType -from typing import Any, BinaryIO, NamedTuple from ._re import ( RE_DATETIME, @@ -17,7 +14,13 @@ match_to_localtime, match_to_number, ) -from ._types import Key, ParseFloat, Pos + +TYPE_CHECKING = False +if TYPE_CHECKING: + from collections.abc import Iterable + from typing import IO, Any + + from ._types import Key, ParseFloat, Pos ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127)) @@ -33,9 +36,11 @@ TOML_WS = frozenset(" \t") TOML_WS_AND_NEWLINE = TOML_WS | frozenset("\n") -BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + "-_") +BARE_KEY_CHARS = frozenset( + "abcdefghijklmnopqrstuvwxyz" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "0123456789" "-_" +) KEY_INITIAL_CHARS = BARE_KEY_CHARS | frozenset("\"'") -HEXDIGIT_CHARS = frozenset(string.hexdigits) +HEXDIGIT_CHARS = frozenset("abcdef" "ABCDEF" "0123456789") BASIC_STR_ESCAPE_REPLACEMENTS = MappingProxyType( { @@ -50,11 +55,73 @@ ) +class DEPRECATED_DEFAULT: + """Sentinel to be used as default arg during deprecation + period of TOMLDecodeError's free-form arguments.""" + + class TOMLDecodeError(ValueError): - """An error raised if a document is not valid TOML.""" + """An error raised if a document is not valid TOML. + + Adds the following attributes to ValueError: + msg: The unformatted error message + doc: The TOML document being parsed + pos: The index of doc where parsing failed + lineno: The line corresponding to pos + colno: The column corresponding to pos + """ + + def __init__( + self, + msg: str = DEPRECATED_DEFAULT, # type: ignore[assignment] + doc: str = DEPRECATED_DEFAULT, # type: ignore[assignment] + pos: Pos = DEPRECATED_DEFAULT, # type: ignore[assignment] + *args: Any, + ): + if ( + args + or not isinstance(msg, str) + or not isinstance(doc, str) + or not isinstance(pos, int) + ): + import warnings + + warnings.warn( + "Free-form arguments for TOMLDecodeError are deprecated. " + "Please set 'msg' (str), 'doc' (str) and 'pos' (int) arguments only.", + DeprecationWarning, + stacklevel=2, + ) + if pos is not DEPRECATED_DEFAULT: # type: ignore[comparison-overlap] + args = pos, *args + if doc is not DEPRECATED_DEFAULT: # type: ignore[comparison-overlap] + args = doc, *args + if msg is not DEPRECATED_DEFAULT: # type: ignore[comparison-overlap] + args = msg, *args + ValueError.__init__(self, *args) + return + + lineno = doc.count("\n", 0, pos) + 1 + if lineno == 1: + colno = pos + 1 + else: + colno = pos - doc.rindex("\n", 0, pos) + + if pos >= len(doc): + coord_repr = "end of document" + else: + coord_repr = f"line {lineno}, column {colno}" + errmsg = f"{msg} (at {coord_repr})" + ValueError.__init__(self, errmsg) + + self.msg = msg + self.doc = doc + self.pos = pos + self.lineno = lineno + self.colno = colno -def load(fp: BinaryIO, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: +def load(fp: IO[bytes], /, *, parse_float: ParseFloat = float) -> dict[str, Any]: """Parse TOML from a binary file object.""" b = fp.read() try: @@ -71,9 +138,14 @@ def loads(s: str, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: # n # The spec allows converting "\r\n" to "\n", even in string # literals. Let's do so to simplify parsing. - src = s.replace("\r\n", "\n") + try: + src = s.replace("\r\n", "\n") + except (AttributeError, TypeError): + raise TypeError( + f"Expected str object, not '{type(s).__qualname__}'" + ) from None pos = 0 - out = Output(NestedDict(), Flags()) + out = Output() header: Key = () parse_float = make_safe_parse_float(parse_float) @@ -113,7 +185,7 @@ def loads(s: str, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: # n pos, header = create_dict_rule(src, pos, out) pos = skip_chars(src, pos, TOML_WS) elif char != "#": - raise suffixed_err(src, pos, "Invalid statement") + raise TOMLDecodeError("Invalid statement", src, pos) # 3. Skip comment pos = skip_comment(src, pos) @@ -124,8 +196,8 @@ def loads(s: str, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: # n except IndexError: break if char != "\n": - raise suffixed_err( - src, pos, "Expected newline or end of document after a statement" + raise TOMLDecodeError( + "Expected newline or end of document after a statement", src, pos ) pos += 1 @@ -224,9 +296,10 @@ def append_nest_to_list(self, key: Key) -> None: cont[last_key] = [{}] -class Output(NamedTuple): - data: NestedDict - flags: Flags +class Output: + def __init__(self) -> None: + self.data = NestedDict() + self.flags = Flags() def skip_chars(src: str, pos: Pos, chars: Iterable[str]) -> Pos: @@ -251,12 +324,12 @@ def skip_until( except ValueError: new_pos = len(src) if error_on_eof: - raise suffixed_err(src, new_pos, f"Expected {expect!r}") from None + raise TOMLDecodeError(f"Expected {expect!r}", src, new_pos) from None if not error_on.isdisjoint(src[pos:new_pos]): while src[pos] not in error_on: pos += 1 - raise suffixed_err(src, pos, f"Found invalid character {src[pos]!r}") + raise TOMLDecodeError(f"Found invalid character {src[pos]!r}", src, pos) return new_pos @@ -287,15 +360,17 @@ def create_dict_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: pos, key = parse_key(src, pos) if out.flags.is_(key, Flags.EXPLICIT_NEST) or out.flags.is_(key, Flags.FROZEN): - raise suffixed_err(src, pos, f"Cannot declare {key} twice") + raise TOMLDecodeError(f"Cannot declare {key} twice", src, pos) out.flags.set(key, Flags.EXPLICIT_NEST, recursive=False) try: out.data.get_or_create_nest(key) except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None + raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None if not src.startswith("]", pos): - raise suffixed_err(src, pos, "Expected ']' at the end of a table declaration") + raise TOMLDecodeError( + "Expected ']' at the end of a table declaration", src, pos + ) return pos + 1, key @@ -305,7 +380,7 @@ def create_list_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: pos, key = parse_key(src, pos) if out.flags.is_(key, Flags.FROZEN): - raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}") + raise TOMLDecodeError(f"Cannot mutate immutable namespace {key}", src, pos) # Free the namespace now that it points to another empty list item... out.flags.unset_all(key) # ...but this key precisely is still prohibited from table declaration @@ -313,10 +388,12 @@ def create_list_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: try: out.data.append_nest_to_list(key) except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None + raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None if not src.startswith("]]", pos): - raise suffixed_err(src, pos, "Expected ']]' at the end of an array declaration") + raise TOMLDecodeError( + "Expected ']]' at the end of an array declaration", src, pos + ) return pos + 2, key @@ -331,22 +408,22 @@ def key_value_rule( for cont_key in relative_path_cont_keys: # Check that dotted key syntax does not redefine an existing table if out.flags.is_(cont_key, Flags.EXPLICIT_NEST): - raise suffixed_err(src, pos, f"Cannot redefine namespace {cont_key}") + raise TOMLDecodeError(f"Cannot redefine namespace {cont_key}", src, pos) # Containers in the relative path can't be opened with the table syntax or # dotted key/value syntax in following table sections. out.flags.add_pending(cont_key, Flags.EXPLICIT_NEST) if out.flags.is_(abs_key_parent, Flags.FROZEN): - raise suffixed_err( - src, pos, f"Cannot mutate immutable namespace {abs_key_parent}" + raise TOMLDecodeError( + f"Cannot mutate immutable namespace {abs_key_parent}", src, pos ) try: nest = out.data.get_or_create_nest(abs_key_parent) except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None + raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None if key_stem in nest: - raise suffixed_err(src, pos, "Cannot overwrite a value") + raise TOMLDecodeError("Cannot overwrite a value", src, pos) # Mark inline table and array namespaces recursively immutable if isinstance(value, (dict, list)): out.flags.set(header + key, Flags.FROZEN, recursive=True) @@ -363,7 +440,7 @@ def parse_key_value_pair( except IndexError: char = None if char != "=": - raise suffixed_err(src, pos, "Expected '=' after a key in a key/value pair") + raise TOMLDecodeError("Expected '=' after a key in a key/value pair", src, pos) pos += 1 pos = skip_chars(src, pos, TOML_WS) pos, value = parse_value(src, pos, parse_float) @@ -401,7 +478,7 @@ def parse_key_part(src: str, pos: Pos) -> tuple[Pos, str]: return parse_literal_str(src, pos) if char == '"': return parse_one_line_basic_str(src, pos) - raise suffixed_err(src, pos, "Invalid initial character for a key part") + raise TOMLDecodeError("Invalid initial character for a key part", src, pos) def parse_one_line_basic_str(src: str, pos: Pos) -> tuple[Pos, str]: @@ -425,7 +502,7 @@ def parse_array(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, list] if c == "]": return pos + 1, array if c != ",": - raise suffixed_err(src, pos, "Unclosed array") + raise TOMLDecodeError("Unclosed array", src, pos) pos += 1 pos = skip_comments_and_array_ws(src, pos) @@ -445,20 +522,20 @@ def parse_inline_table(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos pos, key, value = parse_key_value_pair(src, pos, parse_float) key_parent, key_stem = key[:-1], key[-1] if flags.is_(key, Flags.FROZEN): - raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}") + raise TOMLDecodeError(f"Cannot mutate immutable namespace {key}", src, pos) try: nest = nested_dict.get_or_create_nest(key_parent, access_lists=False) except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None + raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None if key_stem in nest: - raise suffixed_err(src, pos, f"Duplicate inline table key {key_stem!r}") + raise TOMLDecodeError(f"Duplicate inline table key {key_stem!r}", src, pos) nest[key_stem] = value pos = skip_chars(src, pos, TOML_WS) c = src[pos : pos + 1] if c == "}": return pos + 1, nested_dict.dict if c != ",": - raise suffixed_err(src, pos, "Unclosed inline table") + raise TOMLDecodeError("Unclosed inline table", src, pos) if isinstance(value, (dict, list)): flags.set(key, Flags.FROZEN, recursive=True) pos += 1 @@ -480,7 +557,7 @@ def parse_basic_str_escape( except IndexError: return pos, "" if char != "\n": - raise suffixed_err(src, pos, "Unescaped '\\' in a string") + raise TOMLDecodeError("Unescaped '\\' in a string", src, pos) pos += 1 pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE) return pos, "" @@ -491,7 +568,7 @@ def parse_basic_str_escape( try: return pos, BASIC_STR_ESCAPE_REPLACEMENTS[escape_id] except KeyError: - raise suffixed_err(src, pos, "Unescaped '\\' in a string") from None + raise TOMLDecodeError("Unescaped '\\' in a string", src, pos) from None def parse_basic_str_escape_multiline(src: str, pos: Pos) -> tuple[Pos, str]: @@ -501,11 +578,13 @@ def parse_basic_str_escape_multiline(src: str, pos: Pos) -> tuple[Pos, str]: def parse_hex_char(src: str, pos: Pos, hex_len: int) -> tuple[Pos, str]: hex_str = src[pos : pos + hex_len] if len(hex_str) != hex_len or not HEXDIGIT_CHARS.issuperset(hex_str): - raise suffixed_err(src, pos, "Invalid hex value") + raise TOMLDecodeError("Invalid hex value", src, pos) pos += hex_len hex_int = int(hex_str, 16) if not is_unicode_scalar_value(hex_int): - raise suffixed_err(src, pos, "Escaped character is not a Unicode scalar value") + raise TOMLDecodeError( + "Escaped character is not a Unicode scalar value", src, pos + ) return pos, chr(hex_int) @@ -562,7 +641,7 @@ def parse_basic_str(src: str, pos: Pos, *, multiline: bool) -> tuple[Pos, str]: try: char = src[pos] except IndexError: - raise suffixed_err(src, pos, "Unterminated string") from None + raise TOMLDecodeError("Unterminated string", src, pos) from None if char == '"': if not multiline: return pos + 1, result + src[start_pos:pos] @@ -577,7 +656,7 @@ def parse_basic_str(src: str, pos: Pos, *, multiline: bool) -> tuple[Pos, str]: start_pos = pos continue if char in error_on: - raise suffixed_err(src, pos, f"Illegal character {char!r}") + raise TOMLDecodeError(f"Illegal character {char!r}", src, pos) pos += 1 @@ -625,7 +704,7 @@ def parse_value( # noqa: C901 try: datetime_obj = match_to_datetime(datetime_match) except ValueError as e: - raise suffixed_err(src, pos, "Invalid date or datetime") from e + raise TOMLDecodeError("Invalid date or datetime", src, pos) from e return datetime_match.end(), datetime_obj localtime_match = RE_LOCALTIME.match(src, pos) if localtime_match: @@ -646,24 +725,7 @@ def parse_value( # noqa: C901 if first_four in {"-inf", "+inf", "-nan", "+nan"}: return pos + 4, parse_float(first_four) - raise suffixed_err(src, pos, "Invalid value") - - -def suffixed_err(src: str, pos: Pos, msg: str) -> TOMLDecodeError: - """Return a `TOMLDecodeError` where error message is suffixed with - coordinates in source.""" - - def coord_repr(src: str, pos: Pos) -> str: - if pos >= len(src): - return "end of document" - line = src.count("\n", 0, pos) + 1 - if line == 1: - column = pos + 1 - else: - column = pos - src.rindex("\n", 0, pos) - return f"line {line}, column {column}" - - return TOMLDecodeError(f"{msg} (at {coord_repr(src, pos)})") + raise TOMLDecodeError("Invalid value", src, pos) def is_unicode_scalar_value(codepoint: int) -> bool: diff --git a/Lib/tomllib/_re.py b/Lib/tomllib/_re.py index 994bb7493fd928..1ca6bef77a0b03 100644 --- a/Lib/tomllib/_re.py +++ b/Lib/tomllib/_re.py @@ -7,9 +7,12 @@ from datetime import date, datetime, time, timedelta, timezone, tzinfo from functools import lru_cache import re -from typing import Any -from ._types import ParseFloat +TYPE_CHECKING = False +if TYPE_CHECKING: + from typing import Any + + from ._types import ParseFloat # E.g. # - 00:32:00.999999 @@ -84,6 +87,9 @@ def match_to_datetime(match: re.Match) -> datetime | date: return datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz) +# No need to limit cache size. This is only ever called on input +# that matched RE_DATETIME, so there is an implicit bound of +# 24 (hours) * 60 (minutes) * 2 (offset direction) = 2880. @lru_cache(maxsize=None) def cached_tz(hour_str: str, minute_str: str, sign_str: str) -> timezone: sign = 1 if sign_str == "+" else -1 diff --git a/Lib/trace.py b/Lib/trace.py old mode 100755 new mode 100644 index 7886959fa64f68..a87bc6d61a884f --- a/Lib/trace.py +++ b/Lib/trace.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - # portions copyright 2001, Autonomous Zones Industries, Inc., all rights... # err... reserved and offered to the public under the terms of the # Python 2.2 license. @@ -281,14 +279,13 @@ def write_results(self, show_missing=True, summary=False, coverdir=None, *, n_hits, n_lines = self.write_results_file(coverpath, source, lnotab, count, encoding) if summary and n_lines: - percent = int(100 * n_hits / n_lines) - sums[modulename] = n_lines, percent, modulename, filename + sums[modulename] = n_lines, n_hits, modulename, filename if summary and sums: print("lines cov% module (path)") for m in sorted(sums): - n_lines, percent, modulename, filename = sums[m] - print("%5d %3d%% %s (%s)" % sums[m]) + n_lines, n_hits, modulename, filename = sums[m] + print(f"{n_lines:5d} {n_hits/n_lines:.1%} {modulename} ({filename})") if self.outfile: # try and store counts and module info into self.outfile @@ -402,7 +399,7 @@ def __init__(self, count=1, trace=1, countfuncs=0, countcallers=0, @param countfuncs true iff it should just output a list of (filename, modulename, funcname,) for functions that were called at least once; This overrides - `count' and `trace' + 'count' and 'trace' @param ignoremods a list of the names of modules to ignore @param ignoredirs a list of the names of directories to ignore all of the (recursive) contents of @@ -534,7 +531,7 @@ def globaltrace_countfuncs(self, frame, why, arg): def globaltrace_lt(self, frame, why, arg): """Handler for call events. - If the code block being entered is to be ignored, returns `None', + If the code block being entered is to be ignored, returns 'None', else returns self.localtrace. """ if why == 'call': @@ -565,8 +562,12 @@ def localtrace_trace_and_count(self, frame, why, arg): if self.start_time: print('%.2f' % (_time() - self.start_time), end=' ') bname = os.path.basename(filename) - print("%s(%d): %s" % (bname, lineno, - linecache.getline(filename, lineno)), end='') + line = linecache.getline(filename, lineno) + print("%s(%d)" % (bname, lineno), end='') + if line: + print(": ", line, end='') + else: + print() return self.localtrace def localtrace_trace(self, frame, why, arg): @@ -578,8 +579,12 @@ def localtrace_trace(self, frame, why, arg): if self.start_time: print('%.2f' % (_time() - self.start_time), end=' ') bname = os.path.basename(filename) - print("%s(%d): %s" % (bname, lineno, - linecache.getline(filename, lineno)), end='') + line = linecache.getline(filename, lineno) + print("%s(%d)" % (bname, lineno), end='') + if line: + print(": ", line, end='') + else: + print() return self.localtrace def localtrace_count(self, frame, why, arg): diff --git a/Lib/traceback.py b/Lib/traceback.py index d27c7a726d2bb6..31c73efcef5a52 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -1,7 +1,5 @@ """Extract, format and print information about Python stack traces.""" -import os -import io import collections.abc import itertools import linecache @@ -9,19 +7,20 @@ import textwrap import warnings from contextlib import suppress +import _colorize +from _colorize import ANSIColors __all__ = ['extract_stack', 'extract_tb', 'format_exception', 'format_exception_only', 'format_list', 'format_stack', 'format_tb', 'print_exc', 'format_exc', 'print_exception', 'print_last', 'print_stack', 'print_tb', 'clear_frames', 'FrameSummary', 'StackSummary', 'TracebackException', - 'walk_stack', 'walk_tb'] + 'walk_stack', 'walk_tb', 'print_list'] # # Formatting and printing lists of traceback lines. # -_COLORIZE = True def print_list(extracted_list, file=None): """Print the list of tuples as returned by extract_tb() or @@ -133,40 +132,15 @@ def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ BUILTIN_EXCEPTION_LIMIT = object() -def _can_colorize(): - if sys.platform == "win32": - try: - import nt - if not nt._supports_virtual_terminal(): - return False - except (ImportError, AttributeError): - return False - - if os.environ.get("PYTHON_COLORS") == "0": - return False - if os.environ.get("PYTHON_COLORS") == "1": - return True - if "NO_COLOR" in os.environ: - return False - if not _COLORIZE: - return False - if "FORCE_COLOR" in os.environ: - return True - if os.environ.get("TERM") == "dumb": - return False - try: - return os.isatty(sys.stderr.fileno()) - except io.UnsupportedOperation: - return sys.stderr.isatty() def _print_exception_bltin(exc, /): file = sys.stderr if sys.stderr is not None else sys.__stderr__ - colorize = _can_colorize() + colorize = _colorize.can_colorize(file=file) return print_exception(exc, limit=BUILTIN_EXCEPTION_LIMIT, file=file, colorize=colorize) def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ - chain=True): + chain=True, **kwargs): """Format a stack trace and the exception information. The arguments have the same meaning as the corresponding arguments @@ -175,12 +149,13 @@ def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ these lines are concatenated and printed, exactly the same text is printed as does print_exception(). """ + colorize = kwargs.get("colorize", False) value, tb = _parse_value_tb(exc, value, tb) te = TracebackException(type(value), value, tb, limit=limit, compact=True) - return list(te.format(chain=chain)) + return list(te.format(chain=chain, colorize=colorize)) -def format_exception_only(exc, /, value=_sentinel, *, show_group=False): +def format_exception_only(exc, /, value=_sentinel, *, show_group=False, **kwargs): """Format the exception part of a traceback. The return value is a list of strings, each ending in a newline. @@ -195,10 +170,11 @@ def format_exception_only(exc, /, value=_sentinel, *, show_group=False): :exc:`BaseExceptionGroup`, the nested exceptions are included as well, recursively, with indentation relative to their nesting depth. """ + colorize = kwargs.get("colorize", False) if value is _sentinel: value = exc te = TracebackException(type(value), value, None, compact=True) - return list(te.format_exception_only(show_group=show_group)) + return list(te.format_exception_only(show_group=show_group, colorize=colorize)) # -- not official API but folk probably use these two functions. @@ -208,9 +184,9 @@ def _format_final_exc_line(etype, value, *, insert_final_newline=True, colorize= end_char = "\n" if insert_final_newline else "" if colorize: if value is None or not valuestr: - line = f"{_ANSIColors.BOLD_MAGENTA}{etype}{_ANSIColors.RESET}{end_char}" + line = f"{ANSIColors.BOLD_MAGENTA}{etype}{ANSIColors.RESET}{end_char}" else: - line = f"{_ANSIColors.BOLD_MAGENTA}{etype}{_ANSIColors.RESET}: {_ANSIColors.MAGENTA}{valuestr}{_ANSIColors.RESET}{end_char}" + line = f"{ANSIColors.BOLD_MAGENTA}{etype}{ANSIColors.RESET}: {ANSIColors.MAGENTA}{valuestr}{ANSIColors.RESET}{end_char}" else: if value is None or not valuestr: line = f"{etype}{end_char}" @@ -218,6 +194,7 @@ def _format_final_exc_line(etype, value, *, insert_final_newline=True, colorize= line = f"{etype}: {valuestr}{end_char}" return line + def _safe_string(value, what, func=str): try: return func(value) @@ -443,13 +420,6 @@ def _get_code_position(code, instruction_index): _RECURSIVE_CUTOFF = 3 # Also hardcoded in traceback.c. -class _ANSIColors: - RED = '\x1b[31m' - BOLD_RED = '\x1b[1;31m' - MAGENTA = '\x1b[35m' - BOLD_MAGENTA = '\x1b[1;35m' - GREY = '\x1b[90m' - RESET = '\x1b[0m' class StackSummary(list): """A list of FrameSummary objects, representing a stack of frames.""" @@ -554,15 +524,15 @@ def format_frame_summary(self, frame_summary, **kwargs): filename = "" if colorize: row.append(' File {}"{}"{}, line {}{}{}, in {}{}{}\n'.format( - _ANSIColors.MAGENTA, + ANSIColors.MAGENTA, filename, - _ANSIColors.RESET, - _ANSIColors.MAGENTA, + ANSIColors.RESET, + ANSIColors.MAGENTA, frame_summary.lineno, - _ANSIColors.RESET, - _ANSIColors.MAGENTA, + ANSIColors.RESET, + ANSIColors.MAGENTA, frame_summary.name, - _ANSIColors.RESET, + ANSIColors.RESET, ) ) else: @@ -607,13 +577,10 @@ def format_frame_summary(self, frame_summary, **kwargs): # attempt to parse for anchors anchors = None + show_carets = False with suppress(Exception): anchors = _extract_caret_anchors_from_line_segment(segment) - - # only use carets if there are anchors or the carets do not span all lines - show_carets = False - if anchors or all_lines[0][:start_offset].lstrip() or all_lines[-1][end_offset:].rstrip(): - show_carets = True + show_carets = self._should_show_carets(start_offset, end_offset, all_lines, anchors) result = [] @@ -689,11 +656,11 @@ def output_line(lineno): for color, group in itertools.groupby(itertools.zip_longest(line, carets, fillvalue=""), key=lambda x: x[1]): caret_group = list(group) if color == "^": - colorized_line_parts.append(_ANSIColors.BOLD_RED + "".join(char for char, _ in caret_group) + _ANSIColors.RESET) - colorized_carets_parts.append(_ANSIColors.BOLD_RED + "".join(caret for _, caret in caret_group) + _ANSIColors.RESET) + colorized_line_parts.append(ANSIColors.BOLD_RED + "".join(char for char, _ in caret_group) + ANSIColors.RESET) + colorized_carets_parts.append(ANSIColors.BOLD_RED + "".join(caret for _, caret in caret_group) + ANSIColors.RESET) elif color == "~": - colorized_line_parts.append(_ANSIColors.RED + "".join(char for char, _ in caret_group) + _ANSIColors.RESET) - colorized_carets_parts.append(_ANSIColors.RED + "".join(caret for _, caret in caret_group) + _ANSIColors.RESET) + colorized_line_parts.append(ANSIColors.RED + "".join(char for char, _ in caret_group) + ANSIColors.RESET) + colorized_carets_parts.append(ANSIColors.RED + "".join(caret for _, caret in caret_group) + ANSIColors.RESET) else: colorized_line_parts.append("".join(char for char, _ in caret_group)) colorized_carets_parts.append("".join(caret for _, caret in caret_group)) @@ -727,6 +694,39 @@ def output_line(lineno): return ''.join(row) + def _should_show_carets(self, start_offset, end_offset, all_lines, anchors): + with suppress(SyntaxError, ImportError): + import ast + tree = ast.parse('\n'.join(all_lines)) + if not tree.body: + return False + statement = tree.body[0] + value = None + def _spawns_full_line(value): + return ( + value.lineno == 1 + and value.end_lineno == len(all_lines) + and value.col_offset == start_offset + and value.end_col_offset == end_offset + ) + match statement: + case ast.Return(value=ast.Call()): + if isinstance(statement.value.func, ast.Name): + value = statement.value + case ast.Assign(value=ast.Call()): + if ( + len(statement.targets) == 1 and + isinstance(statement.targets[0], ast.Name) + ): + value = statement.value + if value is not None and _spawns_full_line(value): + return False + if anchors: + return True + if all_lines[0][:start_offset].lstrip() or all_lines[-1][end_offset:].rstrip(): + return True + return False + def format(self, **kwargs): """Format the stack ready for printing. @@ -1269,12 +1269,12 @@ def _format_syntax_error(self, stype, **kwargs): if self.lineno is not None: if colorize: yield ' File {}"{}"{}, line {}{}{}\n'.format( - _ANSIColors.MAGENTA, + ANSIColors.MAGENTA, self.filename or "", - _ANSIColors.RESET, - _ANSIColors.MAGENTA, + ANSIColors.RESET, + ANSIColors.MAGENTA, self.lineno, - _ANSIColors.RESET, + ANSIColors.RESET, ) else: yield ' File "{}", line {}\n'.format( @@ -1283,7 +1283,7 @@ def _format_syntax_error(self, stype, **kwargs): filename_suffix = ' ({})'.format(self.filename) text = self.text - if text is not None: + if isinstance(text, str): # text = " foo\n" # rtext = " foo" # ltext = "foo" @@ -1292,13 +1292,24 @@ def _format_syntax_error(self, stype, **kwargs): spaces = len(rtext) - len(ltext) if self.offset is None: yield ' {}\n'.format(ltext) - else: + elif isinstance(self.offset, int): offset = self.offset - end_offset = self.end_offset if self.end_offset not in {None, 0} else offset + if self.lineno == self.end_lineno: + end_offset = ( + self.end_offset + if ( + isinstance(self.end_offset, int) + and self.end_offset != 0 + ) + else offset + ) + else: + end_offset = len(rtext) + 1 + if self.text and offset > len(self.text): - offset = len(self.text) + 1 + offset = len(rtext) + 1 if self.text and end_offset > len(self.text): - end_offset = len(self.text) + 1 + end_offset = len(rtext) + 1 if offset >= end_offset or end_offset < 0: end_offset = offset + 1 @@ -1314,11 +1325,11 @@ def _format_syntax_error(self, stype, **kwargs): # colorize from colno to end_colno ltext = ( ltext[:colno] + - _ANSIColors.BOLD_RED + ltext[colno:end_colno] + _ANSIColors.RESET + + ANSIColors.BOLD_RED + ltext[colno:end_colno] + ANSIColors.RESET + ltext[end_colno:] ) - start_color = _ANSIColors.BOLD_RED - end_color = _ANSIColors.RESET + start_color = ANSIColors.BOLD_RED + end_color = ANSIColors.RESET yield ' {}\n'.format(ltext) yield ' {}{}{}{}\n'.format( "".join(caretspace), @@ -1331,12 +1342,12 @@ def _format_syntax_error(self, stype, **kwargs): msg = self.msg or "" if colorize: yield "{}{}{}: {}{}{}{}\n".format( - _ANSIColors.BOLD_MAGENTA, + ANSIColors.BOLD_MAGENTA, stype, - _ANSIColors.RESET, - _ANSIColors.MAGENTA, + ANSIColors.RESET, + ANSIColors.MAGENTA, msg, - _ANSIColors.RESET, + ANSIColors.RESET, filename_suffix) else: yield "{}: {}{}\n".format(stype, msg, filename_suffix) @@ -1424,7 +1435,7 @@ def format(self, *, chain=True, _ctx=None, **kwargs): f'+---------------- {title} ----------------\n') _ctx.exception_group_depth += 1 if not truncated: - yield from exc.exceptions[i].format(chain=chain, _ctx=_ctx) + yield from exc.exceptions[i].format(chain=chain, _ctx=_ctx, colorize=colorize) else: remaining = num_excs - self.max_group_width plural = 's' if remaining > 1 else '' @@ -1472,12 +1483,23 @@ def _compute_suggestion_error(exc_value, tb, wrong_name): obj = exc_value.obj try: d = dir(obj) + hide_underscored = (wrong_name[:1] != '_') + if hide_underscored and tb is not None: + while tb.tb_next is not None: + tb = tb.tb_next + frame = tb.tb_frame + if 'self' in frame.f_locals and frame.f_locals['self'] is obj: + hide_underscored = False + if hide_underscored: + d = [x for x in d if x[:1] != '_'] except Exception: return None elif isinstance(exc_value, ImportError): try: mod = __import__(exc_value.name) d = dir(mod) + if wrong_name[:1] != '_': + d = [x for x in d if x[:1] != '_'] except Exception: return None else: diff --git a/Lib/turtle.py b/Lib/turtle.py index 7bfe81351b0b34..e88981d298ad52 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -51,7 +51,7 @@ turtle. So the turtles can more easily be used as a visual feedback instrument by the (beginning) programmer. -- Different turtle shapes, gif-images as turtle shapes, user defined +- Different turtle shapes, image files as turtle shapes, user defined and user controllable turtle shapes, among them compound (multicolored) shapes. Turtle shapes can be stretched and tilted, which makes turtles very versatile geometrical objects. @@ -106,6 +106,8 @@ import sys from os.path import isfile, split, join +from pathlib import Path +from contextlib import contextmanager from copy import deepcopy from tkinter import simpledialog @@ -113,23 +115,24 @@ 'RawTurtle', 'Turtle', 'RawPen', 'Pen', 'Shape', 'Vec2D'] _tg_screen_functions = ['addshape', 'bgcolor', 'bgpic', 'bye', 'clearscreen', 'colormode', 'delay', 'exitonclick', 'getcanvas', - 'getshapes', 'listen', 'mainloop', 'mode', 'numinput', + 'getshapes', 'listen', 'mainloop', 'mode', 'no_animation', 'numinput', 'onkey', 'onkeypress', 'onkeyrelease', 'onscreenclick', 'ontimer', - 'register_shape', 'resetscreen', 'screensize', 'setup', - 'setworldcoordinates', 'textinput', 'title', 'tracer', 'turtles', 'update', - 'window_height', 'window_width'] + 'register_shape', 'resetscreen', 'screensize', 'save', 'setup', + 'setworldcoordinates', 'textinput', 'title', 'tracer', 'turtles', + 'update', 'window_height', 'window_width'] _tg_turtle_functions = ['back', 'backward', 'begin_fill', 'begin_poly', 'bk', 'circle', 'clear', 'clearstamp', 'clearstamps', 'clone', 'color', 'degrees', 'distance', 'dot', 'down', 'end_fill', 'end_poly', 'fd', - 'fillcolor', 'filling', 'forward', 'get_poly', 'getpen', 'getscreen', 'get_shapepoly', - 'getturtle', 'goto', 'heading', 'hideturtle', 'home', 'ht', 'isdown', - 'isvisible', 'left', 'lt', 'onclick', 'ondrag', 'onrelease', 'pd', - 'pen', 'pencolor', 'pendown', 'pensize', 'penup', 'pos', 'position', - 'pu', 'radians', 'right', 'reset', 'resizemode', 'rt', - 'seth', 'setheading', 'setpos', 'setposition', - 'setundobuffer', 'setx', 'sety', 'shape', 'shapesize', 'shapetransform', 'shearfactor', 'showturtle', - 'speed', 'st', 'stamp', 'teleport', 'tilt', 'tiltangle', 'towards', - 'turtlesize', 'undo', 'undobufferentries', 'up', 'width', + 'fillcolor', 'fill', 'filling', 'forward', 'get_poly', 'getpen', + 'getscreen', 'get_shapepoly', 'getturtle', 'goto', 'heading', + 'hideturtle', 'home', 'ht', 'isdown', 'isvisible', 'left', 'lt', + 'onclick', 'ondrag', 'onrelease', 'pd', 'pen', 'pencolor', 'pendown', + 'pensize', 'penup', 'poly', 'pos', 'position', 'pu', 'radians', 'right', + 'reset', 'resizemode', 'rt', 'seth', 'setheading', 'setpos', + 'setposition', 'setundobuffer', 'setx', 'sety', 'shape', 'shapesize', + 'shapetransform', 'shearfactor', 'showturtle', 'speed', 'st', 'stamp', + 'teleport', 'tilt', 'tiltangle', 'towards', 'turtlesize', 'undo', + 'undobufferentries', 'up', 'width', 'write', 'xcor', 'ycor'] _tg_utilities = ['write_docstringdict', 'done'] @@ -467,7 +470,7 @@ def _blankimage(self): def _image(self, filename): """return an image object containing the - imagedata from a gif-file named filename. + imagedata from an image file named filename. """ return TK.PhotoImage(file=filename, master=self.cv) @@ -871,10 +874,7 @@ def __init__(self, type_, data=None): if isinstance(data, list): data = tuple(data) elif type_ == "image": - if isinstance(data, str): - if data.lower().endswith(".gif") and isfile(data): - data = TurtleScreen._image(data) - # else data assumed to be PhotoImage + assert(isinstance(data, TK.PhotoImage)) elif type_ == "compound": data = [] else: @@ -1099,14 +1099,18 @@ def register_shape(self, name, shape=None): """Adds a turtle shape to TurtleScreen's shapelist. Arguments: - (1) name is the name of a gif-file and shape is None. + (1) name is the name of an image file (PNG, GIF, PGM, and PPM) and shape is None. Installs the corresponding image shape. !! Image-shapes DO NOT rotate when turning the turtle, !! so they do not display the heading of the turtle! - (2) name is an arbitrary string and shape is a tuple + (2) name is an arbitrary string and shape is the name of an image file (PNG, GIF, PGM, and PPM). + Installs the corresponding image shape. + !! Image-shapes DO NOT rotate when turning the turtle, + !! so they do not display the heading of the turtle! + (3) name is an arbitrary string and shape is a tuple of pairs of coordinates. Installs the corresponding polygon shape - (3) name is an arbitrary string and shape is a + (4) name is an arbitrary string and shape is a (compound) Shape object. Installs the corresponding compound shape. To use a shape, you have to issue the command shape(shapename). @@ -1119,12 +1123,9 @@ def register_shape(self, name, shape=None): """ if shape is None: - # image - if name.lower().endswith(".gif"): - shape = Shape("image", self._image(name)) - else: - raise TurtleGraphicsError("Bad arguments for register_shape.\n" - + "Use help(register_shape)" ) + shape = Shape("image", self._image(name)) + elif isinstance(shape, str): + shape = Shape("image", self._image(shape)) elif isinstance(shape, tuple): shape = Shape("polygon", shape) ## else shape assumed to be Shape-instance @@ -1276,6 +1277,26 @@ def delay(self, delay=None): return self._delayvalue self._delayvalue = int(delay) + @contextmanager + def no_animation(self): + """Temporarily turn off auto-updating the screen. + + This is useful for drawing complex shapes where even the fastest setting + is too slow. Once this context manager is exited, the drawing will + be displayed. + + Example (for a TurtleScreen instance named screen + and a Turtle instance named turtle): + >>> with screen.no_animation(): + ... turtle.circle(50) + """ + tracer = self.tracer() + try: + self.tracer(0) + yield + finally: + self.tracer(tracer) + def _incrementudc(self): """Increment update counter.""" if not TurtleScreen._RUNNING: @@ -1453,7 +1474,7 @@ def bgpic(self, picname=None): """Set background image or return name of current backgroundimage. Optional argument: - picname -- a string, name of a gif-file or "nopic". + picname -- a string, name of an image file (PNG, GIF, PGM, and PPM) or "nopic". If picname is a filename, set the corresponding image as background. If picname is "nopic", delete backgroundimage, if present. @@ -1492,6 +1513,39 @@ def screensize(self, canvwidth=None, canvheight=None, bg=None): """ return self._resize(canvwidth, canvheight, bg) + def save(self, filename, *, overwrite=False): + """Save the drawing as a PostScript file + + Arguments: + filename -- a string, the path of the created file. + Must end with '.ps' or '.eps'. + + Optional arguments: + overwrite -- boolean, if true, then existing files will be overwritten + + Example (for a TurtleScreen instance named screen): + >>> screen.save('my_drawing.eps') + """ + filename = Path(filename) + if not filename.parent.exists(): + raise FileNotFoundError( + f"The directory '{filename.parent}' does not exist." + " Cannot save to it." + ) + if not overwrite and filename.exists(): + raise FileExistsError( + f"The file '{filename}' already exists. To overwrite it use" + " the 'overwrite=True' argument of the save function." + ) + if (ext := filename.suffix) not in {".ps", ".eps"}: + raise ValueError( + f"Unknown file extension: '{ext}'," + " must be one of {'.ps', '.eps'}" + ) + + postscript = self.cv.postscript() + filename.write_text(postscript) + onscreenclick = onclick resetscreen = reset clearscreen = clear @@ -1718,7 +1772,7 @@ def xcor(self): >>> reset() >>> turtle.left(60) >>> turtle.forward(100) - >>> print turtle.xcor() + >>> print(turtle.xcor()) 50.0 """ return self._position[0] @@ -1732,7 +1786,7 @@ def ycor(self): >>> reset() >>> turtle.left(60) >>> turtle.forward(100) - >>> print turtle.ycor() + >>> print(turtle.ycor()) 86.6025403784 """ return self._position[1] @@ -2335,7 +2389,7 @@ def isvisible(self): Example (for a Turtle instance named turtle): >>> turtle.hideturtle() - >>> print turtle.isvisible(): + >>> print(turtle.isvisible()) False """ return self._shown @@ -3348,6 +3402,24 @@ def filling(self): """ return isinstance(self._fillpath, list) + @contextmanager + def fill(self): + """A context manager for filling a shape. + + Implicitly ensures the code block is wrapped with + begin_fill() and end_fill(). + + Example (for a Turtle instance named turtle): + >>> turtle.color("black", "red") + >>> with turtle.fill(): + ... turtle.circle(60) + """ + self.begin_fill() + try: + yield + finally: + self.end_fill() + def begin_fill(self): """Called just before drawing a shape to be filled. @@ -3368,7 +3440,6 @@ def begin_fill(self): self.undobuffer.push(("beginfill", self._fillitem)) self._update() - def end_fill(self): """Fill the shape drawn after the call begin_fill(). @@ -3472,6 +3543,27 @@ def write(self, arg, move=False, align="left", font=("Arial", 8, "normal")): if self.undobuffer: self.undobuffer.cumulate = False + @contextmanager + def poly(self): + """A context manager for recording the vertices of a polygon. + + Implicitly ensures that the code block is wrapped with + begin_poly() and end_poly() + + Example (for a Turtle instance named turtle) where we create a + triangle as the polygon and move the turtle 100 steps forward: + >>> with turtle.poly(): + ... for side in range(3) + ... turtle.forward(50) + ... turtle.right(60) + >>> turtle.forward(100) + """ + self.begin_poly() + try: + yield + finally: + self.end_poly() + def begin_poly(self): """Start recording the vertices of a polygon. diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py old mode 100755 new mode 100644 index 2ab6c15e2c079e..b49c0beab3ccf7 --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - """ ---------------------------------------------- turtleDemo - Help @@ -92,20 +90,21 @@ from idlelib.colorizer import ColorDelegator, color_config from idlelib.percolator import Percolator from idlelib.textview import view_text +import turtle from turtledemo import __doc__ as about_turtledemo -import turtle +if sys.platform == 'win32': + from idlelib.util import fix_win_hidpi + fix_win_hidpi() demo_dir = os.path.dirname(os.path.abspath(__file__)) darwin = sys.platform == 'darwin' - STARTUP = 1 READY = 2 RUNNING = 3 DONE = 4 EVENTDRIVEN = 5 -menufont = ("Arial", 12, NORMAL) btnfont = ("Arial", 12, 'bold') txtfont = ['Lucida Console', 10, 'normal'] @@ -216,7 +215,7 @@ def makeTextFrame(self, root): self.vbar = vbar = Scrollbar(text_frame, name='vbar') vbar['command'] = text.yview - vbar.pack(side=LEFT, fill=Y) + vbar.pack(side=RIGHT, fill=Y) self.hbar = hbar = Scrollbar(text_frame, name='hbar', orient=HORIZONTAL) hbar['command'] = text.xview hbar.pack(side=BOTTOM, fill=X) @@ -292,37 +291,35 @@ def configGUI(self, start, stop, clear, txt="", color="blue"): self.output_lbl.config(text=txt, fg=color) def makeLoadDemoMenu(self, master): - menu = Menu(master) + menu = Menu(master, tearoff=1) # TJR: leave this one. for entry in getExampleEntries(): def load(entry=entry): self.loadfile(entry) - menu.add_command(label=entry, underline=0, - font=menufont, command=load) + menu.add_command(label=entry, underline=0, command=load) return menu def makeFontMenu(self, master): - menu = Menu(master) - menu.add_command(label="Decrease (C-'-')", command=self.decrease_size, - font=menufont) - menu.add_command(label="Increase (C-'+')", command=self.increase_size, - font=menufont) + menu = Menu(master, tearoff=0) + menu.add_command(label="Decrease", command=self.decrease_size, + accelerator=f"{'Command' if darwin else 'Ctrl'}+-") + menu.add_command(label="Increase", command=self.increase_size, + accelerator=f"{'Command' if darwin else 'Ctrl'}+=") menu.add_separator() for size in font_sizes: def resize(size=size): self.set_txtsize(size) - menu.add_command(label=str(size), underline=0, - font=menufont, command=resize) + menu.add_command(label=str(size), underline=0, command=resize) return menu def makeHelpMenu(self, master): - menu = Menu(master) + menu = Menu(master, tearoff=0) for help_label, help_file in help_entries: def show(help_label=help_label, help_file=help_file): view_text(self.root, help_label, help_file) - menu.add_command(label=help_label, font=menufont, command=show) + menu.add_command(label=help_label, command=show) return menu def refreshCanvas(self): diff --git a/Lib/turtledemo/bytedesign.py b/Lib/turtledemo/bytedesign.py old mode 100755 new mode 100644 index 1b7452b512c6eb..a5d76a6b6ff295 --- a/Lib/turtledemo/bytedesign.py +++ b/Lib/turtledemo/bytedesign.py @@ -1,7 +1,4 @@ -#!/usr/bin/env python3 -""" turtle-example-suite: - - tdemo_bytedesign.py +"""turtledemo/bytedesign.py An example adapted from the example-suite of PythonCard's turtle graphics. diff --git a/Lib/turtledemo/chaos.py b/Lib/turtledemo/chaos.py index 6a45d0d807ef0b..b25f0fa42c901d 100644 --- a/Lib/turtledemo/chaos.py +++ b/Lib/turtledemo/chaos.py @@ -1,9 +1,7 @@ -# File: tdemo_chaos.py -# Author: Gregor Lingl -# Date: 2009-06-24 - -# A demonstration of chaos +"""turtledemo/chaos.py +A demonstration of chaos. +""" from turtle import * N = 80 diff --git a/Lib/turtledemo/clock.py b/Lib/turtledemo/clock.py old mode 100755 new mode 100644 index 9f8585bd11e053..8b639066c4f440 --- a/Lib/turtledemo/clock.py +++ b/Lib/turtledemo/clock.py @@ -1,18 +1,14 @@ -#!/usr/bin/env python3 -# -*- coding: cp1252 -*- -""" turtle-example-suite: - - tdemo_clock.py +"""turtledemo/clock.py Enhanced clock-program, showing date -and time - ------------------------------------ - Press STOP to exit the program! - ------------------------------------ +and time. """ from turtle import * from datetime import datetime +dtfont = "TkFixedFont", 14, "bold" +current_day = None + def jump(distanz, winkel=0): penup() right(winkel) @@ -53,11 +49,23 @@ def clockface(radius): jump(-radius) rt(6) +def display_date_time(): + global current_day + writer.clear() + now = datetime.now() + current_day = now.day + writer.home() + writer.forward(distance=65) + writer.write(wochentag(now), align="center", font=dtfont) + writer.back(distance=150) + writer.write(datum(now), align="center", font=dtfont) + writer.forward(distance=85) + def setup(): global second_hand, minute_hand, hour_hand, writer mode("logo") make_hand_shape("second_hand", 125, 25) - make_hand_shape("minute_hand", 130, 25) + make_hand_shape("minute_hand", 115, 25) make_hand_shape("hour_hand", 90, 25) clockface(160) second_hand = Turtle() @@ -75,10 +83,10 @@ def setup(): hand.speed(0) ht() writer = Turtle() - #writer.mode("logo") writer.ht() writer.pu() writer.bk(85) + display_date_time() def wochentag(t): wochentag = ["Monday", "Tuesday", "Wednesday", @@ -100,18 +108,11 @@ def tick(): stunde = t.hour + minute/60.0 try: tracer(False) # Terminator can occur here - writer.clear() - writer.home() - writer.forward(65) - writer.write(wochentag(t), - align="center", font=("Courier", 14, "bold")) - writer.back(150) - writer.write(datum(t), - align="center", font=("Courier", 14, "bold")) - writer.forward(85) second_hand.setheading(6*sekunde) # or here minute_hand.setheading(6*minute) hour_hand.setheading(30*stunde) + if t.day != current_day: + display_date_time() tracer(True) ontimer(tick, 100) except Terminator: diff --git a/Lib/turtledemo/colormixer.py b/Lib/turtledemo/colormixer.py index 448db83361a649..f66012c8154317 100644 --- a/Lib/turtledemo/colormixer.py +++ b/Lib/turtledemo/colormixer.py @@ -1,5 +1,4 @@ -# colormixer - +"""turtledemo/colormixer.py""" from turtle import Screen, Turtle, mainloop class ColorTurtle(Turtle): diff --git a/Lib/turtledemo/forest.py b/Lib/turtledemo/forest.py old mode 100755 new mode 100644 index 55b7da947d2476..e1fa85a577ffce --- a/Lib/turtledemo/forest.py +++ b/Lib/turtledemo/forest.py @@ -1,15 +1,11 @@ -#!/usr/bin/env python3 -""" turtlegraphics-example-suite: +"""turtledemo/forest.py - tdemo_forest.py +Displays a 'forest' of 3 breadth-first trees, +similar to the one in tree.py. +For further details, see tree.py. -Displays a 'forest' of 3 breadth-first-trees -similar to the one in tree. -For further remarks see tree.py - -This example is a 'breadth-first'-rewrite of -a Logo program written by Erich Neuwirth. See -http://homepage.univie.ac.at/erich.neuwirth/ +This example is a breadth-first rewrite of +a Logo program by Erich Neuwirth. """ from turtle import Turtle, colormode, tracer, mainloop from random import randrange diff --git a/Lib/turtledemo/fractalcurves.py b/Lib/turtledemo/fractalcurves.py old mode 100755 new mode 100644 index 54ade96a0ad05e..2d0a506a4f5b9f --- a/Lib/turtledemo/fractalcurves.py +++ b/Lib/turtledemo/fractalcurves.py @@ -1,7 +1,4 @@ -#!/usr/bin/env python3 -""" turtle-example-suite: - - tdemo_fractalCurves.py +"""turtledemo/fractalcurves.py This program draws two fractal-curve-designs: (1) A hilbert curve (in a box) diff --git a/Lib/turtledemo/lindenmayer.py b/Lib/turtledemo/lindenmayer.py old mode 100755 new mode 100644 index 3925f25da61870..eb309afb9381b1 --- a/Lib/turtledemo/lindenmayer.py +++ b/Lib/turtledemo/lindenmayer.py @@ -1,7 +1,4 @@ -#!/usr/bin/env python3 -""" turtle-example-suite: - - xtx_lindenmayer_indian.py +"""turtledemo/lindenmayer.py Each morning women in Tamil Nadu, in southern India, place designs, created by using rice diff --git a/Lib/turtledemo/minimal_hanoi.py b/Lib/turtledemo/minimal_hanoi.py old mode 100755 new mode 100644 index 4a432f2b2908d5..e44330eaaf7f18 --- a/Lib/turtledemo/minimal_hanoi.py +++ b/Lib/turtledemo/minimal_hanoi.py @@ -1,7 +1,4 @@ -#!/usr/bin/env python3 -""" turtle-example-suite: - - tdemo_minimal_hanoi.py +"""turtledemo/minimal_hanoi.py A minimal 'Towers of Hanoi' animation: A tower of 6 discs is transferred from the @@ -13,9 +10,6 @@ Discs are turtles with shape "square", but stretched to rectangles by shapesize() - --------------------------------------- - To exit press STOP button - --------------------------------------- """ from turtle import * diff --git a/Lib/turtledemo/nim.py b/Lib/turtledemo/nim.py index 9ae6cc5c01b903..f87c479714d662 100644 --- a/Lib/turtledemo/nim.py +++ b/Lib/turtledemo/nim.py @@ -1,6 +1,4 @@ -""" turtle-example-suite: - - tdemo_nim.py +"""turtledemo/nim.py Play nim against the computer. The player who takes the last stick is the winner. diff --git a/Lib/turtledemo/paint.py b/Lib/turtledemo/paint.py old mode 100755 new mode 100644 index fc6852a20082f5..780300fb2da9d1 --- a/Lib/turtledemo/paint.py +++ b/Lib/turtledemo/paint.py @@ -1,13 +1,9 @@ -#!/usr/bin/env python3 -""" turtle-example-suite: +"""turtledemo/paint.py - tdemo_paint.py - -A simple event-driven paint program - -- left mouse button moves turtle -- middle mouse button changes color -- right mouse button toggles between pen up +A simple event-driven paint program. +- Left mouse button moves turtle. +- Middle mouse button changes color. +- Right mouse button toggles between pen up (no line drawn when the turtle moves) and pen down (line is drawn). If pen up follows at least two pen-down moves, the polygon that @@ -15,8 +11,6 @@ ------------------------------------------- Play around by clicking into the canvas using all three mouse buttons. - ------------------------------------------- - To exit press STOP button ------------------------------------------- """ from turtle import * diff --git a/Lib/turtledemo/peace.py b/Lib/turtledemo/peace.py old mode 100755 new mode 100644 index e2ba9288d9e42e..d86c94a48a2472 --- a/Lib/turtledemo/peace.py +++ b/Lib/turtledemo/peace.py @@ -1,7 +1,4 @@ -#!/usr/bin/env python3 -""" turtle-example-suite: - - tdemo_peace.py +"""turtledemo/peace.py A simple drawing suitable as a beginner's programming example. Aside from the diff --git a/Lib/turtledemo/penrose.py b/Lib/turtledemo/penrose.py old mode 100755 new mode 100644 index 045722a2286061..ceaefedac24a67 --- a/Lib/turtledemo/penrose.py +++ b/Lib/turtledemo/penrose.py @@ -1,7 +1,4 @@ -#!/usr/bin/env python3 -""" xturtle-example-suite: - - xtx_kites_and_darts.py +"""turtledemo/penrose.py Constructs two aperiodic penrose-tilings, consisting of kites and darts, by the method @@ -12,7 +9,7 @@ consisting of five darts. For more information see: - http://en.wikipedia.org/wiki/Penrose_tiling + https://en.wikipedia.org/wiki/Penrose_tiling ------------------------------------------- """ from turtle import * diff --git a/Lib/turtledemo/planet_and_moon.py b/Lib/turtledemo/planet_and_moon.py old mode 100755 new mode 100644 index 021ff99383aa65..571afcf922103f --- a/Lib/turtledemo/planet_and_moon.py +++ b/Lib/turtledemo/planet_and_moon.py @@ -1,7 +1,4 @@ -#!/usr/bin/env python3 -""" turtle-example-suite: - - tdemo_planets_and_moon.py +"""turtledemo/planets_and_moon.py Gravitational system simulation using the approximation method from Feynman-lectures, diff --git a/Lib/turtledemo/rosette.py b/Lib/turtledemo/rosette.py index 47d0f00e9da9d1..48897a620f9d8b 100644 --- a/Lib/turtledemo/rosette.py +++ b/Lib/turtledemo/rosette.py @@ -1,6 +1,4 @@ -""" turtle-example-suite: - - tdemo_wikipedia3.py +"""turtledemo/rosette.py This example is inspired by the Wikipedia article on turtle diff --git a/Lib/turtledemo/round_dance.py b/Lib/turtledemo/round_dance.py index 10383614c6e974..9da6389b213207 100644 --- a/Lib/turtledemo/round_dance.py +++ b/Lib/turtledemo/round_dance.py @@ -1,9 +1,4 @@ -""" turtle-example-suite: - - tdemo_round_dance.py - -(Needs version 1.1 of the turtle module that -comes with Python 3.1) +"""turtledemo/round_dance.py Dancing turtles have a compound shape consisting of a series of triangles of diff --git a/Lib/turtledemo/sorting_animate.py b/Lib/turtledemo/sorting_animate.py old mode 100755 new mode 100644 index d25a0ab6cebdc0..e0a2877cd5d621 --- a/Lib/turtledemo/sorting_animate.py +++ b/Lib/turtledemo/sorting_animate.py @@ -1,19 +1,13 @@ -#!/usr/bin/env python3 -""" - - sorting_animation.py +"""turtledemo/sorting_animation.py A minimal sorting algorithm animation: Sorts a shelf of 10 blocks using insertion sort, selection sort and quicksort. -Shelfs are implemented using builtin lists. +Shelves are implemented using builtin lists. Blocks are turtles with shape "square", but stretched to rectangles by shapesize() - --------------------------------------- - To exit press space button - --------------------------------------- """ from turtle import * import random diff --git a/Lib/turtledemo/tree.py b/Lib/turtledemo/tree.py old mode 100755 new mode 100644 index 98a20da7f15c11..6ad8fcc854a155 --- a/Lib/turtledemo/tree.py +++ b/Lib/turtledemo/tree.py @@ -1,7 +1,4 @@ -#!/usr/bin/env python3 -""" turtle-example-suite: - - tdemo_tree.py +"""turtledemo/tree.py Displays a 'breadth-first-tree' - in contrast to the classical Logo tree drawing programs, diff --git a/Lib/turtledemo/two_canvases.py b/Lib/turtledemo/two_canvases.py index f3602585ab0592..2c8020378edf1b 100644 --- a/Lib/turtledemo/two_canvases.py +++ b/Lib/turtledemo/two_canvases.py @@ -1,4 +1,4 @@ -"""turtledemo.two_canvases +"""turtledemo/two_canvases.py Use TurtleScreen and RawTurtle to draw on two distinct canvases in a separate window. The diff --git a/Lib/turtledemo/yinyang.py b/Lib/turtledemo/yinyang.py old mode 100755 new mode 100644 index 11d1f47cae2549..6e92d4bf739194 --- a/Lib/turtledemo/yinyang.py +++ b/Lib/turtledemo/yinyang.py @@ -1,7 +1,4 @@ -#!/usr/bin/env python3 -""" turtle-example-suite: - - tdemo_yinyang.py +"""turtledemo/yinyang.py Another drawing suitable as a beginner's programming example. diff --git a/Lib/typing.py b/Lib/typing.py index 2cb4a714ea3a85..66570db7a5bd74 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -19,6 +19,8 @@ """ from abc import abstractmethod, ABCMeta +import annotationlib +from annotationlib import ForwardRef import collections from collections import defaultdict import collections.abc @@ -27,7 +29,7 @@ import operator import sys import types -from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType, GenericAlias +from types import GenericAlias from _typing import ( _idfunc, @@ -38,6 +40,7 @@ ParamSpecKwargs, TypeAliasType, Generic, + NoDefault, ) # Please keep __all__ alphabetized within each category. @@ -63,7 +66,6 @@ # ABCs (from collections.abc). 'AbstractSet', # collections.abc.Set. - 'ByteString', 'Container', 'ContextManager', 'Hashable', @@ -125,6 +127,7 @@ 'cast', 'clear_overloads', 'dataclass_transform', + 'evaluate_forward_ref', 'final', 'get_args', 'get_origin', @@ -138,12 +141,14 @@ 'NewType', 'no_type_check', 'no_type_check_decorator', + 'NoDefault', 'NoReturn', 'NotRequired', 'overload', 'override', 'ParamSpecArgs', 'ParamSpecKwargs', + 'ReadOnly', 'Required', 'reveal_type', 'runtime_checkable', @@ -152,6 +157,7 @@ 'TYPE_CHECKING', 'TypeAlias', 'TypeGuard', + 'TypeIs', 'TypeAliasType', 'Unpack', ] @@ -162,7 +168,7 @@ def _type_convert(arg, module=None, *, allow_special_forms=False): if arg is None: return type(None) if isinstance(arg, str): - return ForwardRef(arg, module=module, is_class=allow_special_forms) + return _make_forward_ref(arg, module=module, is_class=allow_special_forms) return arg @@ -236,34 +242,27 @@ def _type_repr(obj): typically enough to uniquely identify a type. For everything else, we fall back on repr(obj). """ - # When changing this function, don't forget about - # `_collections_abc._type_repr`, which does the same thing - # and must be consistent with this one. - if isinstance(obj, type): - if obj.__module__ == 'builtins': - return obj.__qualname__ - return f'{obj.__module__}.{obj.__qualname__}' - if obj is ...: - return '...' - if isinstance(obj, types.FunctionType): - return obj.__name__ if isinstance(obj, tuple): # Special case for `repr` of types with `ParamSpec`: return '[' + ', '.join(_type_repr(t) for t in obj) + ']' - return repr(obj) + return annotationlib.value_to_string(obj) -def _collect_parameters(args): - """Collect all type variables and parameter specifications in args +def _collect_type_parameters(args, *, enforce_default_ordering: bool = True): + """Collect all type parameters in args in order of first appearance (lexicographic order). For example:: >>> P = ParamSpec('P') >>> T = TypeVar('T') - >>> _collect_parameters((T, Callable[P, T])) + >>> _collect_type_parameters((T, Callable[P, T])) (~T, ~P) """ + # required type parameter cannot appear after parameter with default + default_encountered = False + # or after TypeVarTuple + type_var_tuple_encountered = False parameters = [] for t in args: if isinstance(t, type): @@ -273,32 +272,64 @@ def _collect_parameters(args): # `t` might be a tuple, when `ParamSpec` is substituted with # `[T, int]`, or `[int, *Ts]`, etc. for x in t: - for collected in _collect_parameters([x]): + for collected in _collect_type_parameters([x]): if collected not in parameters: parameters.append(collected) elif hasattr(t, '__typing_subst__'): if t not in parameters: + if enforce_default_ordering: + if type_var_tuple_encountered and t.has_default(): + raise TypeError('Type parameter with a default' + ' follows TypeVarTuple') + + if t.has_default(): + default_encountered = True + elif default_encountered: + raise TypeError(f'Type parameter {t!r} without a default' + ' follows type parameter with a default') + parameters.append(t) else: + if _is_unpacked_typevartuple(t): + type_var_tuple_encountered = True for x in getattr(t, '__parameters__', ()): if x not in parameters: parameters.append(x) return tuple(parameters) -def _check_generic(cls, parameters, elen): +def _check_generic_specialization(cls, arguments): """Check correct count for parameters of a generic cls (internal helper). This gives a nice error message in case of count mismatch. """ - if not elen: + expected_len = len(cls.__parameters__) + if not expected_len: raise TypeError(f"{cls} is not a generic class") - alen = len(parameters) - if alen != elen: - raise TypeError(f"Too {'many' if alen > elen else 'few'} arguments for {cls};" - f" actual {alen}, expected {elen}") + actual_len = len(arguments) + if actual_len != expected_len: + # deal with defaults + if actual_len < expected_len: + # If the parameter at index `actual_len` in the parameters list + # has a default, then all parameters after it must also have + # one, because we validated as much in _collect_type_parameters(). + # That means that no error needs to be raised here, despite + # the number of arguments being passed not matching the number + # of parameters: all parameters that aren't explicitly + # specialized in this call are parameters with default values. + if cls.__parameters__[actual_len].has_default(): + return + + expected_len -= sum(p.has_default() for p in cls.__parameters__) + expect_val = f"at least {expected_len}" + else: + expect_val = expected_len + + raise TypeError(f"Too {'many' if actual_len > expected_len else 'few'} arguments" + f" for {cls}; actual {actual_len}, expected {expect_val}") -def _unpack_args(args): + +def _unpack_args(*args): newargs = [] for arg in args: subargs = getattr(arg, '__typing_unpacked_tuple_args__', None) @@ -397,19 +428,52 @@ def inner(*args, **kwds): return decorator -def _eval_type(t, globalns, localns, recursive_guard=frozenset()): + +def _deprecation_warning_for_no_type_params_passed(funcname: str) -> None: + import warnings + + depr_message = ( + f"Failing to pass a value to the 'type_params' parameter " + f"of {funcname!r} is deprecated, as it leads to incorrect behaviour " + f"when calling {funcname} on a stringified annotation " + f"that references a PEP 695 type parameter. " + f"It will be disallowed in Python 3.15." + ) + warnings.warn(depr_message, category=DeprecationWarning, stacklevel=3) + + +class _Sentinel: + __slots__ = () + def __repr__(self): + return '' + + +_sentinel = _Sentinel() + + +def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=frozenset(), + format=annotationlib.Format.VALUE, owner=None): """Evaluate all forward references in the given type t. For use of globalns and localns see the docstring for get_type_hints(). recursive_guard is used to prevent infinite recursion with a recursive ForwardRef. """ + if type_params is _sentinel: + _deprecation_warning_for_no_type_params_passed("typing._eval_type") + type_params = () if isinstance(t, ForwardRef): - return t._evaluate(globalns, localns, recursive_guard) + # If the forward_ref has __forward_module__ set, evaluate() infers the globals + # from the module, and it will probably pick better than the globals we have here. + if t.__forward_module__ is not None: + globalns = None + return evaluate_forward_ref(t, globals=globalns, locals=localns, + type_params=type_params, owner=owner, + _recursive_guard=recursive_guard, format=format) if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)): if isinstance(t, GenericAlias): args = tuple( - ForwardRef(arg) if isinstance(arg, str) else arg + _make_forward_ref(arg) if isinstance(arg, str) else arg for arg in t.__args__ ) is_unpacked = t.__unpacked__ @@ -419,7 +483,14 @@ def _eval_type(t, globalns, localns, recursive_guard=frozenset()): t = t.__origin__[args] if is_unpacked: t = Unpack[t] - ev_args = tuple(_eval_type(a, globalns, localns, recursive_guard) for a in t.__args__) + + ev_args = tuple( + _eval_type( + a, globalns, localns, type_params, recursive_guard=recursive_guard, + format=format, owner=owner, + ) + for a in t.__args__ + ) if ev_args == t.__args__: return t if isinstance(t, GenericAlias): @@ -538,7 +609,7 @@ class Any(metaclass=_AnyMeta): def __new__(cls, *args, **kwargs): if cls is Any: raise TypeError("Any cannot be instantiated") - return super().__new__(cls, *args, **kwargs) + return super().__new__(cls) @_SpecialForm @@ -653,7 +724,7 @@ class Starship: Note that ClassVar is not a class itself, and should not be used with isinstance() or issubclass(). """ - item = _type_check(parameters, f'{self} accepts only single type.') + item = _type_check(parameters, f'{self} accepts only single type.', allow_special_forms=True) return _GenericAlias(self, (item,)) @_SpecialForm @@ -675,7 +746,7 @@ class FastConnector(Connection): There is no runtime checking of these properties. """ - item = _type_check(parameters, f'{self} accepts only single type.') + item = _type_check(parameters, f'{self} accepts only single type.', allow_special_forms=True) return _GenericAlias(self, (item,)) @_SpecialForm @@ -817,46 +888,52 @@ def Concatenate(self, parameters): @_SpecialForm def TypeGuard(self, parameters): - """Special typing construct for marking user-defined type guard functions. + """Special typing construct for marking user-defined type predicate functions. ``TypeGuard`` can be used to annotate the return type of a user-defined - type guard function. ``TypeGuard`` only accepts a single type argument. + type predicate function. ``TypeGuard`` only accepts a single type argument. At runtime, functions marked this way should return a boolean. ``TypeGuard`` aims to benefit *type narrowing* -- a technique used by static type checkers to determine a more precise type of an expression within a program's code flow. Usually type narrowing is done by analyzing conditional code flow and applying the narrowing to a block of code. The - conditional expression here is sometimes referred to as a "type guard". + conditional expression here is sometimes referred to as a "type predicate". Sometimes it would be convenient to use a user-defined boolean function - as a type guard. Such a function should use ``TypeGuard[...]`` as its - return type to alert static type checkers to this intention. + as a type predicate. Such a function should use ``TypeGuard[...]`` or + ``TypeIs[...]`` as its return type to alert static type checkers to + this intention. ``TypeGuard`` should be used over ``TypeIs`` when narrowing + from an incompatible type (e.g., ``list[object]`` to ``list[int]``) or when + the function does not return ``True`` for all instances of the narrowed type. - Using ``-> TypeGuard`` tells the static type checker that for a given - function: + Using ``-> TypeGuard[NarrowedType]`` tells the static type checker that + for a given function: 1. The return value is a boolean. 2. If the return value is ``True``, the type of its argument - is the type inside ``TypeGuard``. + is ``NarrowedType``. + + For example:: - For example:: + def is_str_list(val: list[object]) -> TypeGuard[list[str]]: + '''Determines whether all objects in the list are strings''' + return all(isinstance(x, str) for x in val) - def is_str(val: Union[str, float]): - # "isinstance" type guard - if isinstance(val, str): - # Type of ``val`` is narrowed to ``str`` - ... - else: - # Else, type of ``val`` is narrowed to ``float``. - ... + def func1(val: list[object]): + if is_str_list(val): + # Type of ``val`` is narrowed to ``list[str]``. + print(" ".join(val)) + else: + # Type of ``val`` remains as ``list[object]``. + print("Not a list of strings!") Strict type narrowing is not enforced -- ``TypeB`` need not be a narrower form of ``TypeA`` (it can even be a wider form) and this may lead to type-unsafe results. The main reason is to allow for things like - narrowing ``List[object]`` to ``List[str]`` even though the latter is not - a subtype of the former, since ``List`` is invariant. The responsibility of - writing type-safe type guards is left to the user. + narrowing ``list[object]`` to ``list[str]`` even though the latter is not + a subtype of the former, since ``list`` is invariant. The responsibility of + writing type-safe type predicates is left to the user. ``TypeGuard`` also works with type variables. For more information, see PEP 647 (User-Defined Type Guards). @@ -865,88 +942,146 @@ def is_str(val: Union[str, float]): return _GenericAlias(self, (item,)) -class ForwardRef(_Final, _root=True): - """Internal wrapper to hold a forward reference.""" +@_SpecialForm +def TypeIs(self, parameters): + """Special typing construct for marking user-defined type predicate functions. - __slots__ = ('__forward_arg__', '__forward_code__', - '__forward_evaluated__', '__forward_value__', - '__forward_is_argument__', '__forward_is_class__', - '__forward_module__') + ``TypeIs`` can be used to annotate the return type of a user-defined + type predicate function. ``TypeIs`` only accepts a single type argument. + At runtime, functions marked this way should return a boolean and accept + at least one argument. - def __init__(self, arg, is_argument=True, module=None, *, is_class=False): - if not isinstance(arg, str): - raise TypeError(f"Forward reference must be a string -- got {arg!r}") + ``TypeIs`` aims to benefit *type narrowing* -- a technique used by static + type checkers to determine a more precise type of an expression within a + program's code flow. Usually type narrowing is done by analyzing + conditional code flow and applying the narrowing to a block of code. The + conditional expression here is sometimes referred to as a "type predicate". - # If we do `def f(*args: *Ts)`, then we'll have `arg = '*Ts'`. - # Unfortunately, this isn't a valid expression on its own, so we - # do the unpacking manually. - if arg[0] == '*': - arg_to_compile = f'({arg},)[0]' # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] - else: - arg_to_compile = arg - try: - code = compile(arg_to_compile, '', 'eval') - except SyntaxError: - raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}") - - self.__forward_arg__ = arg - self.__forward_code__ = code - self.__forward_evaluated__ = False - self.__forward_value__ = None - self.__forward_is_argument__ = is_argument - self.__forward_is_class__ = is_class - self.__forward_module__ = module - - def _evaluate(self, globalns, localns, recursive_guard): - if self.__forward_arg__ in recursive_guard: - return self - if not self.__forward_evaluated__ or localns is not globalns: - if globalns is None and localns is None: - globalns = localns = {} - elif globalns is None: - globalns = localns - elif localns is None: - localns = globalns - if self.__forward_module__ is not None: - globalns = getattr( - sys.modules.get(self.__forward_module__, None), '__dict__', globalns - ) - type_ = _type_check( - eval(self.__forward_code__, globalns, localns), - "Forward references must evaluate to types.", - is_argument=self.__forward_is_argument__, - allow_special_forms=self.__forward_is_class__, - ) - self.__forward_value__ = _eval_type( - type_, globalns, localns, recursive_guard | {self.__forward_arg__} - ) - self.__forward_evaluated__ = True - return self.__forward_value__ + Sometimes it would be convenient to use a user-defined boolean function + as a type predicate. Such a function should use ``TypeIs[...]`` or + ``TypeGuard[...]`` as its return type to alert static type checkers to + this intention. ``TypeIs`` usually has more intuitive behavior than + ``TypeGuard``, but it cannot be used when the input and output types + are incompatible (e.g., ``list[object]`` to ``list[int]``) or when the + function does not return ``True`` for all instances of the narrowed type. - def __eq__(self, other): - if not isinstance(other, ForwardRef): - return NotImplemented - if self.__forward_evaluated__ and other.__forward_evaluated__: - return (self.__forward_arg__ == other.__forward_arg__ and - self.__forward_value__ == other.__forward_value__) - return (self.__forward_arg__ == other.__forward_arg__ and - self.__forward_module__ == other.__forward_module__) + Using ``-> TypeIs[NarrowedType]`` tells the static type checker that for + a given function: - def __hash__(self): - return hash((self.__forward_arg__, self.__forward_module__)) + 1. The return value is a boolean. + 2. If the return value is ``True``, the type of its argument + is the intersection of the argument's original type and + ``NarrowedType``. + 3. If the return value is ``False``, the type of its argument + is narrowed to exclude ``NarrowedType``. - def __or__(self, other): - return Union[self, other] + For example:: - def __ror__(self, other): - return Union[other, self] + from typing import assert_type, final, TypeIs - def __repr__(self): - if self.__forward_module__ is None: - module_repr = '' + class Parent: pass + class Child(Parent): pass + @final + class Unrelated: pass + + def is_parent(val: object) -> TypeIs[Parent]: + return isinstance(val, Parent) + + def run(arg: Child | Unrelated): + if is_parent(arg): + # Type of ``arg`` is narrowed to the intersection + # of ``Parent`` and ``Child``, which is equivalent to + # ``Child``. + assert_type(arg, Child) + else: + # Type of ``arg`` is narrowed to exclude ``Parent``, + # so only ``Unrelated`` is left. + assert_type(arg, Unrelated) + + The type inside ``TypeIs`` must be consistent with the type of the + function's argument; if it is not, static type checkers will raise + an error. An incorrectly written ``TypeIs`` function can lead to + unsound behavior in the type system; it is the user's responsibility + to write such functions in a type-safe manner. + + ``TypeIs`` also works with type variables. For more information, see + PEP 742 (Narrowing types with ``TypeIs``). + """ + item = _type_check(parameters, f'{self} accepts only single type.') + return _GenericAlias(self, (item,)) + + +def _make_forward_ref(code, **kwargs): + forward_ref = ForwardRef(code, **kwargs) + # For compatibility, eagerly compile the forwardref's code. + forward_ref.__forward_code__ + return forward_ref + + +def evaluate_forward_ref( + forward_ref, + *, + owner=None, + globals=None, + locals=None, + type_params=_sentinel, + format=annotationlib.Format.VALUE, + _recursive_guard=frozenset(), +): + """Evaluate a forward reference as a type hint. + + This is similar to calling the ForwardRef.evaluate() method, + but unlike that method, evaluate_forward_ref() also: + + * Recursively evaluates forward references nested within the type hint. + * Rejects certain objects that are not valid type hints. + * Replaces type hints that evaluate to None with types.NoneType. + * Supports the *FORWARDREF* and *STRING* formats. + + *forward_ref* must be an instance of ForwardRef. *owner*, if given, + should be the object that holds the annotations that the forward reference + derived from, such as a module, class object, or function. It is used to + infer the namespaces to use for looking up names. *globals* and *locals* + can also be explicitly given to provide the global and local namespaces. + *type_params* is a tuple of type parameters that are in scope when + evaluating the forward reference. This parameter must be provided (though + it may be an empty tuple) if *owner* is not given and the forward reference + does not already have an owner set. *format* specifies the format of the + annotation and is a member of the annotationlib.Format enum. + + """ + if type_params is _sentinel: + _deprecation_warning_for_no_type_params_passed("typing.evaluate_forward_ref") + type_params = () + if format == annotationlib.Format.STRING: + return forward_ref.__forward_arg__ + if forward_ref.__forward_arg__ in _recursive_guard: + return forward_ref + + try: + value = forward_ref.evaluate(globals=globals, locals=locals, + type_params=type_params, owner=owner) + except NameError: + if format == annotationlib.Format.FORWARDREF: + return forward_ref else: - module_repr = f', module={self.__forward_module__!r}' - return f'ForwardRef({self.__forward_arg__!r}{module_repr})' + raise + + type_ = _type_check( + value, + "Forward references must evaluate to types.", + is_argument=forward_ref.__forward_is_argument__, + allow_special_forms=forward_ref.__forward_is_class__, + ) + return _eval_type( + type_, + globals, + locals, + type_params, + recursive_guard=_recursive_guard | {forward_ref.__forward_arg__}, + format=format, + owner=owner, + ) def _is_unpacked_typevartuple(x: Any) -> bool: @@ -994,11 +1129,15 @@ def _typevartuple_prepare_subst(self, alias, args): elif left + right > alen: raise TypeError(f"Too few arguments for {alias};" f" actual {alen}, expected at least {plen-1}") + if left == alen - right and self.has_default(): + replacement = _unpack_args(self.__default__) + else: + replacement = args[left: alen - right] return ( *args[:left], *([fillarg]*(typevartuple_index - left)), - tuple(args[left: alen - right]), + replacement, *([fillarg]*(plen - right - left - typevartuple_index - 1)), *args[alen - right:], ) @@ -1016,6 +1155,8 @@ def _paramspec_subst(self, arg): def _paramspec_prepare_subst(self, alias, args): params = alias.__parameters__ i = params.index(self) + if i == len(args) and self.has_default(): + args = [*args, self.__default__] if i >= len(args): raise TypeError(f"Too few arguments for {alias}") # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612. @@ -1029,33 +1170,33 @@ def _paramspec_prepare_subst(self, alias, args): @_tp_cache -def _generic_class_getitem(cls, params): +def _generic_class_getitem(cls, args): """Parameterizes a generic class. At least, parameterizing a generic class is the *main* thing this method does. For example, for some generic class `Foo`, this is called when we - do `Foo[int]` - there, with `cls=Foo` and `params=int`. + do `Foo[int]` - there, with `cls=Foo` and `args=int`. However, note that this method is also called when defining generic classes in the first place with `class Foo(Generic[T]): ...`. """ - if not isinstance(params, tuple): - params = (params,) + if not isinstance(args, tuple): + args = (args,) - params = tuple(_type_convert(p) for p in params) + args = tuple(_type_convert(p) for p in args) is_generic_or_protocol = cls in (Generic, Protocol) if is_generic_or_protocol: # Generic and Protocol can only be subscripted with unique type variables. - if not params: + if not args: raise TypeError( f"Parameter list to {cls.__qualname__}[...] cannot be empty" ) - if not all(_is_typevar_like(p) for p in params): + if not all(_is_typevar_like(p) for p in args): raise TypeError( f"Parameters to {cls.__name__}[...] must all be type variables " f"or parameter specification variables.") - if len(set(params)) != len(params): + if len(set(args)) != len(args): raise TypeError( f"Parameters to {cls.__name__}[...] must all be unique") else: @@ -1063,18 +1204,18 @@ def _generic_class_getitem(cls, params): for param in cls.__parameters__: prepare = getattr(param, '__typing_prepare_subst__', None) if prepare is not None: - params = prepare(cls, params) - _check_generic(cls, params, len(cls.__parameters__)) + args = prepare(cls, args) + _check_generic_specialization(cls, args) new_args = [] - for param, new_arg in zip(cls.__parameters__, params): + for param, new_arg in zip(cls.__parameters__, args): if isinstance(param, TypeVarTuple): new_args.extend(new_arg) else: new_args.append(new_arg) - params = tuple(new_args) + args = tuple(new_args) - return _GenericAlias(cls, params) + return _GenericAlias(cls, args) def _generic_init_subclass(cls, *args, **kwargs): @@ -1089,7 +1230,7 @@ def _generic_init_subclass(cls, *args, **kwargs): if error: raise TypeError("Cannot inherit from plain Generic") if '__orig_bases__' in cls.__dict__: - tvars = _collect_parameters(cls.__orig_bases__) + tvars = _collect_type_parameters(cls.__orig_bases__) # Look for Generic[T1, ..., Tn]. # If found, tvars must be a subset of it. # If not found, tvars is it. @@ -1190,7 +1331,7 @@ def __getattr__(self, attr): raise AttributeError(attr) def __setattr__(self, attr, val): - if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams'}: + if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams', '_defaults'}: super().__setattr__(attr, val) else: setattr(self.__origin__, attr, val) @@ -1237,11 +1378,12 @@ class _GenericAlias(_BaseGenericAlias, _root=True): # A = Callable[[], None] # _CallableGenericAlias # B = Callable[[T], None] # _CallableGenericAlias # C = B[int] # _CallableGenericAlias - # * Parameterized `Final`, `ClassVar` and `TypeGuard`: + # * Parameterized `Final`, `ClassVar`, `TypeGuard`, and `TypeIs`: # # All _GenericAlias # Final[int] # ClassVar[float] - # TypeVar[bool] + # TypeGuard[bool] + # TypeIs[range] def __init__(self, origin, args, *, inst=True, name=None): super().__init__(origin, inst=inst, name=name) @@ -1249,7 +1391,11 @@ def __init__(self, origin, args, *, inst=True, name=None): args = (args,) self.__args__ = tuple(... if a is _TypingEllipsis else a for a in args) - self.__parameters__ = _collect_parameters(args) + enforce_default_ordering = origin in (Generic, Protocol) + self.__parameters__ = _collect_type_parameters( + args, + enforce_default_ordering=enforce_default_ordering, + ) if not name: self.__module__ = origin.__module__ @@ -1294,8 +1440,7 @@ def __getitem__(self, args): # Preprocess `args`. if not isinstance(args, tuple): args = (args,) - args = tuple(_type_convert(p) for p in args) - args = _unpack_args(args) + args = _unpack_args(*(_type_convert(p) for p in args)) new_args = self._determine_new_args(args) r = self.copy_with(new_args) return r @@ -1440,11 +1585,12 @@ def __iter__(self): # parameters are accepted (needs custom __getitem__). class _SpecialGenericAlias(_NotIterable, _BaseGenericAlias, _root=True): - def __init__(self, origin, nparams, *, inst=True, name=None): + def __init__(self, origin, nparams, *, inst=True, name=None, defaults=()): if name is None: name = origin.__name__ super().__init__(origin, inst=inst, name=name) self._nparams = nparams + self._defaults = defaults if origin.__module__ == 'builtins': self.__doc__ = f'A generic version of {origin.__qualname__}.' else: @@ -1456,7 +1602,22 @@ def __getitem__(self, params): params = (params,) msg = "Parameters to generic types must be types." params = tuple(_type_check(p, msg) for p in params) - _check_generic(self, params, self._nparams) + if (self._defaults + and len(params) < self._nparams + and len(params) + len(self._defaults) >= self._nparams + ): + params = (*params, *self._defaults[len(params) - self._nparams:]) + actual_len = len(params) + + if actual_len != self._nparams: + if self._defaults: + expected = f"at least {self._nparams - len(self._defaults)}" + else: + expected = str(self._nparams) + if not self._nparams: + raise TypeError(f"{self} is not a generic class") + raise TypeError(f"Too {'many' if actual_len > self._nparams else 'few'} arguments for {self};" + f" actual {actual_len}, expected {expected}") return self.copy_with(params) def copy_with(self, params): @@ -1483,21 +1644,6 @@ def __ror__(self, left): return Union[left, self] -class _DeprecatedGenericAlias(_SpecialGenericAlias, _root=True): - def __init__( - self, origin, nparams, *, removal_version, inst=True, name=None - ): - super().__init__(origin, nparams, inst=inst, name=name) - self._removal_version = removal_version - - def __instancecheck__(self, inst): - import warnings - warnings._deprecated( - f"{self.__module__}.{self._name}", remove=self._removal_version - ) - return super().__instancecheck__(inst) - - class _CallableGenericAlias(_NotIterable, _GenericAlias, _root=True): def __repr__(self): assert self._name == 'Callable' @@ -1587,12 +1733,16 @@ def __repr__(self): return super().__repr__() def __instancecheck__(self, obj): - return self.__subclasscheck__(type(obj)) + for arg in self.__args__: + if isinstance(obj, arg): + return True + return False def __subclasscheck__(self, cls): for arg in self.__args__: if issubclass(cls, arg): return True + return False def __reduce__(self): func, (origin, args) = super().__reduce__() @@ -1690,8 +1840,9 @@ def __typing_unpacked_tuple_args__(self): assert self.__origin__ is Unpack assert len(self.__args__) == 1 arg, = self.__args__ - if isinstance(arg, _GenericAlias): - assert arg.__origin__ is tuple + if isinstance(arg, (_GenericAlias, types.GenericAlias)): + if arg.__origin__ is not tuple: + raise TypeError("Unpack[...] must be used with a tuple type") return arg.__args__ return None @@ -1716,7 +1867,8 @@ class _TypingEllipsis: '__abstractmethods__', '__annotations__', '__dict__', '__doc__', '__init__', '__module__', '__new__', '__slots__', '__subclasshook__', '__weakref__', '__class_getitem__', - '__match_args__', + '__match_args__', '__static_attributes__', '__firstlineno__', + '__annotate__', }) # These special attributes will be not collected as protocol members. @@ -1792,9 +1944,11 @@ def _allow_reckless_class_checks(depth=2): _PROTO_ALLOWLIST = { 'collections.abc': [ 'Callable', 'Awaitable', 'Iterable', 'Iterator', 'AsyncIterable', - 'Hashable', 'Sized', 'Container', 'Collection', 'Reversible', 'Buffer', + 'AsyncIterator', 'Hashable', 'Sized', 'Container', 'Collection', + 'Reversible', 'Buffer', ], 'contextlib': ['AbstractContextManager', 'AbstractAsyncContextManager'], + 'os': ['PathLike'], } @@ -2014,7 +2168,7 @@ class _AnnotatedAlias(_NotIterable, _GenericAlias, _root=True): """Runtime representation of an annotated type. At its core 'Annotated[t, dec1, dec2, ...]' is an alias for the type 't' - with extra annotations. The alias behaves like a normal typing alias. + with extra metadata. The alias behaves like a normal typing alias. Instantiating is the same as instantiating the underlying type; binding it to types is also the same. @@ -2193,12 +2347,8 @@ def greet(name: str) -> None: return val -_allowed_types = (types.FunctionType, types.BuiltinFunctionType, - types.MethodType, types.ModuleType, - WrapperDescriptorType, MethodWrapperType, MethodDescriptorType) - - -def get_type_hints(obj, globalns=None, localns=None, include_extras=False): +def get_type_hints(obj, globalns=None, localns=None, include_extras=False, + *, format=annotationlib.Format.VALUE): """Return type hints for an object. This is often the same as obj.__annotations__, but it handles @@ -2235,13 +2385,14 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): if isinstance(obj, type): hints = {} for base in reversed(obj.__mro__): + ann = annotationlib.get_annotations(base, format=format) + if format is annotationlib.Format.STRING: + hints.update(ann) + continue if globalns is None: base_globals = getattr(sys.modules.get(base.__module__, None), '__dict__', {}) else: base_globals = globalns - ann = base.__dict__.get('__annotations__', {}) - if isinstance(ann, types.GetSetDescriptorType): - ann = {} base_locals = dict(vars(base)) if localns is None else localns if localns is None and globalns is None: # This is surprising, but required. Before Python 3.10, @@ -2255,10 +2406,26 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): if value is None: value = type(None) if isinstance(value, str): - value = ForwardRef(value, is_argument=False, is_class=True) - value = _eval_type(value, base_globals, base_locals) + value = _make_forward_ref(value, is_argument=False, is_class=True) + value = _eval_type(value, base_globals, base_locals, base.__type_params__, + format=format, owner=obj) hints[name] = value - return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} + if include_extras or format is annotationlib.Format.STRING: + return hints + else: + return {k: _strip_annotations(t) for k, t in hints.items()} + + hints = annotationlib.get_annotations(obj, format=format) + if ( + not hints + and not isinstance(obj, types.ModuleType) + and not callable(obj) + and not hasattr(obj, '__annotations__') + and not hasattr(obj, '__annotate__') + ): + raise TypeError(f"{obj!r} is not a module, class, or callable.") + if format is annotationlib.Format.STRING: + return hints if globalns is None: if isinstance(obj, types.ModuleType): @@ -2273,27 +2440,19 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): localns = globalns elif localns is None: localns = globalns - hints = getattr(obj, '__annotations__', None) - if hints is None: - # Return empty annotations for something that _could_ have them. - if isinstance(obj, _allowed_types): - return {} - else: - raise TypeError('{!r} is not a module, class, method, ' - 'or function.'.format(obj)) - hints = dict(hints) + type_params = getattr(obj, "__type_params__", ()) for name, value in hints.items(): if value is None: value = type(None) if isinstance(value, str): # class-level forward refs were handled above, this must be either # a module-level annotation or a function argument annotation - value = ForwardRef( + value = _make_forward_ref( value, is_argument=not isinstance(obj, types.ModuleType), is_class=False, ) - hints[name] = _eval_type(value, globalns, localns) + hints[name] = _eval_type(value, globalns, localns, type_params, format=format, owner=obj) return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} @@ -2301,7 +2460,7 @@ def _strip_annotations(t): """Strip the annotations from a given type.""" if isinstance(t, _AnnotatedAlias): return _strip_annotations(t.__origin__) - if hasattr(t, "__origin__") and t.__origin__ in (Required, NotRequired): + if hasattr(t, "__origin__") and t.__origin__ in (Required, NotRequired, ReadOnly): return _strip_annotations(t.__args__[0]) if isinstance(t, _GenericAlias): stripped_args = tuple(_strip_annotations(a) for a in t.__args__) @@ -2639,9 +2798,6 @@ class Other(Leaf): # Error reported by type checker MutableMapping = _alias(collections.abc.MutableMapping, 2) Sequence = _alias(collections.abc.Sequence, 1) MutableSequence = _alias(collections.abc.MutableSequence, 1) -ByteString = _DeprecatedGenericAlias( - collections.abc.ByteString, 0, removal_version=(3, 14) # Not generic. -) # Tuple accepts variable number of parameters. Tuple = _TupleType(tuple, -1, inst=False, name='Tuple') Tuple.__doc__ = \ @@ -2668,8 +2824,8 @@ class Other(Leaf): # Error reported by type checker OrderedDict = _alias(collections.OrderedDict, 2) Counter = _alias(collections.Counter, 1) ChainMap = _alias(collections.ChainMap, 2) -Generator = _alias(collections.abc.Generator, 3) -AsyncGenerator = _alias(collections.abc.AsyncGenerator, 2) +Generator = _alias(collections.abc.Generator, 3, defaults=(types.NoneType, types.NoneType)) +AsyncGenerator = _alias(collections.abc.AsyncGenerator, 2, defaults=(types.NoneType,)) Type = _alias(type, 1, inst=False, name='Type') Type.__doc__ = \ """Deprecated alias to builtins.type. @@ -2773,22 +2929,33 @@ def __round__(self, ndigits: int = 0) -> T: pass -def _make_nmtuple(name, types, module, defaults = ()): - fields = [n for n, t in types] - types = {n: _type_check(t, f"field {n} annotation must be a type") - for n, t in types} +def _make_nmtuple(name, fields, annotate_func, module, defaults = ()): nm_tpl = collections.namedtuple(name, fields, defaults=defaults, module=module) - nm_tpl.__annotations__ = nm_tpl.__new__.__annotations__ = types + nm_tpl.__annotate__ = nm_tpl.__new__.__annotate__ = annotate_func return nm_tpl +def _make_eager_annotate(types): + checked_types = {key: _type_check(val, f"field {key} annotation must be a type") + for key, val in types.items()} + def annotate(format): + match format: + case annotationlib.Format.VALUE | annotationlib.Format.FORWARDREF: + return checked_types + case annotationlib.Format.STRING: + return annotationlib.annotations_to_string(types) + case _: + raise NotImplementedError(format) + return annotate + + # attributes prohibited to set in NamedTuple class syntax _prohibited = frozenset({'__new__', '__init__', '__slots__', '__getnewargs__', '_fields', '_field_defaults', '_make', '_replace', '_asdict', '_source'}) -_special = frozenset({'__module__', '__name__', '__annotations__'}) +_special = frozenset({'__module__', '__name__', '__annotations__', '__annotate__'}) class NamedTupleMeta(type): @@ -2799,9 +2966,31 @@ def __new__(cls, typename, bases, ns): raise TypeError( 'can only inherit from a NamedTuple type and Generic') bases = tuple(tuple if base is _NamedTuple else base for base in bases) - types = ns.get('__annotations__', {}) + if "__annotations__" in ns: + types = ns["__annotations__"] + field_names = list(types) + annotate = _make_eager_annotate(types) + elif "__annotate__" in ns: + original_annotate = ns["__annotate__"] + types = annotationlib.call_annotate_function(original_annotate, annotationlib.Format.FORWARDREF) + field_names = list(types) + + # For backward compatibility, type-check all the types at creation time + for typ in types.values(): + _type_check(typ, "field annotation must be a type") + + def annotate(format): + annos = annotationlib.call_annotate_function(original_annotate, format) + if format != annotationlib.Format.STRING: + return {key: _type_check(val, f"field {key} annotation must be a type") + for key, val in annos.items()} + return annos + else: + # Empty NamedTuple + field_names = [] + annotate = lambda format: {} default_names = [] - for field_name in types: + for field_name in field_names: if field_name in ns: default_names.append(field_name) elif default_names: @@ -2809,7 +2998,7 @@ def __new__(cls, typename, bases, ns): f"cannot follow default field" f"{'s' if len(default_names) > 1 else ''} " f"{', '.join(default_names)}") - nm_tpl = _make_nmtuple(typename, types.items(), + nm_tpl = _make_nmtuple(typename, field_names, annotate, defaults=[ns[n] for n in default_names], module=ns['__module__']) nm_tpl.__bases__ = bases @@ -2842,15 +3031,6 @@ def __new__(cls, typename, bases, ns): return nm_tpl -class _Sentinel: - __slots__ = () - def __repr__(self): - return '' - - -_sentinel = _Sentinel() - - def NamedTuple(typename, fields=_sentinel, /, **kwargs): """Typed version of namedtuple. @@ -2909,7 +3089,11 @@ class Employee(NamedTuple): import warnings warnings._deprecated(deprecated_thing, message=deprecation_msg, remove=(3, 15)) fields = kwargs.items() - nt = _make_nmtuple(typename, fields, module=_caller()) + types = {n: _type_check(t, f"field {n} annotation must be a type") + for n, t in fields} + field_names = [n for n, _ in fields] + + nt = _make_nmtuple(typename, field_names, _make_eager_annotate(types), module=_caller()) nt.__orig_bases__ = (NamedTuple,) return nt @@ -2922,6 +3106,28 @@ def _namedtuple_mro_entries(bases): NamedTuple.__mro_entries__ = _namedtuple_mro_entries +def _get_typeddict_qualifiers(annotation_type): + while True: + annotation_origin = get_origin(annotation_type) + if annotation_origin is Annotated: + annotation_args = get_args(annotation_type) + if annotation_args: + annotation_type = annotation_args[0] + else: + break + elif annotation_origin is Required: + yield Required + (annotation_type,) = get_args(annotation_type) + elif annotation_origin is NotRequired: + yield NotRequired + (annotation_type,) = get_args(annotation_type) + elif annotation_origin is ReadOnly: + yield ReadOnly + (annotation_type,) = get_args(annotation_type) + else: + break + + class _TypedDictMeta(type): def __new__(cls, name, bases, ns, total=True): """Create a new typed dict class object. @@ -2946,19 +3152,28 @@ def __new__(cls, name, bases, ns, total=True): if not hasattr(tp_dict, '__orig_bases__'): tp_dict.__orig_bases__ = bases - annotations = {} - own_annotations = ns.get('__annotations__', {}) + if "__annotations__" in ns: + own_annotate = None + own_annotations = ns["__annotations__"] + elif "__annotate__" in ns: + own_annotate = ns["__annotate__"] + own_annotations = annotationlib.call_annotate_function( + own_annotate, annotationlib.Format.FORWARDREF, owner=tp_dict + ) + else: + own_annotate = None + own_annotations = {} msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type" - own_annotations = { + own_checked_annotations = { n: _type_check(tp, msg, module=tp_dict.__module__) for n, tp in own_annotations.items() } required_keys = set() optional_keys = set() + readonly_keys = set() + mutable_keys = set() for base in bases: - annotations.update(base.__dict__.get('__annotations__', {})) - base_required = base.__dict__.get('__required_keys__', set()) required_keys |= base_required optional_keys -= base_required @@ -2967,18 +3182,14 @@ def __new__(cls, name, bases, ns, total=True): required_keys -= base_optional optional_keys |= base_optional - annotations.update(own_annotations) - for annotation_key, annotation_type in own_annotations.items(): - annotation_origin = get_origin(annotation_type) - if annotation_origin is Annotated: - annotation_args = get_args(annotation_type) - if annotation_args: - annotation_type = annotation_args[0] - annotation_origin = get_origin(annotation_type) + readonly_keys.update(base.__dict__.get('__readonly_keys__', ())) + mutable_keys.update(base.__dict__.get('__mutable_keys__', ())) - if annotation_origin is Required: + for annotation_key, annotation_type in own_checked_annotations.items(): + qualifiers = set(_get_typeddict_qualifiers(annotation_type)) + if Required in qualifiers: is_required = True - elif annotation_origin is NotRequired: + elif NotRequired in qualifiers: is_required = False else: is_required = total @@ -2990,13 +3201,53 @@ def __new__(cls, name, bases, ns, total=True): optional_keys.add(annotation_key) required_keys.discard(annotation_key) + if ReadOnly in qualifiers: + if annotation_key in mutable_keys: + raise TypeError( + f"Cannot override mutable key {annotation_key!r}" + " with read-only key" + ) + readonly_keys.add(annotation_key) + else: + mutable_keys.add(annotation_key) + readonly_keys.discard(annotation_key) + assert required_keys.isdisjoint(optional_keys), ( f"Required keys overlap with optional keys in {name}:" f" {required_keys=}, {optional_keys=}" ) - tp_dict.__annotations__ = annotations + + def __annotate__(format): + annos = {} + for base in bases: + if base is Generic: + continue + base_annotate = base.__annotate__ + if base_annotate is None: + continue + base_annos = annotationlib.call_annotate_function(base.__annotate__, format, owner=base) + annos.update(base_annos) + if own_annotate is not None: + own = annotationlib.call_annotate_function(own_annotate, format, owner=tp_dict) + if format != annotationlib.Format.STRING: + own = { + n: _type_check(tp, msg, module=tp_dict.__module__) + for n, tp in own.items() + } + elif format == annotationlib.Format.STRING: + own = annotationlib.annotations_to_string(own_annotations) + elif format in (annotationlib.Format.FORWARDREF, annotationlib.Format.VALUE): + own = own_checked_annotations + else: + raise NotImplementedError(format) + annos.update(own) + return annos + + tp_dict.__annotate__ = __annotate__ tp_dict.__required_keys__ = frozenset(required_keys) tp_dict.__optional_keys__ = frozenset(optional_keys) + tp_dict.__readonly_keys__ = frozenset(readonly_keys) + tp_dict.__mutable_keys__ = frozenset(mutable_keys) tp_dict.__total__ = total return tp_dict @@ -3055,6 +3306,14 @@ class Point2D(TypedDict): y: NotRequired[int] # the "y" key can be omitted See PEP 655 for more details on Required and NotRequired. + + The ReadOnly special form can be used + to mark individual keys as immutable for type checkers:: + + class DatabaseUser(TypedDict): + id: ReadOnly[int] # the "id" key must not be modified + username: str # the "username" key can be changed + """ if fields is _sentinel or fields is None: import warnings @@ -3131,6 +3390,26 @@ class Movie(TypedDict): return _GenericAlias(self, (item,)) +@_SpecialForm +def ReadOnly(self, parameters): + """A special typing construct to mark an item of a TypedDict as read-only. + + For example:: + + class Movie(TypedDict): + title: ReadOnly[str] + year: int + + def mutate_movie(m: Movie) -> None: + m["year"] = 1992 # allowed + m["title"] = "The Matrix" # typechecker error + + There is no runtime checking for this property. + """ + item = _type_check(parameters, f'{self._name} accepts only a single type.') + return _GenericAlias(self, (item,)) + + class NewType: """NewType creates simple unique types with almost zero runtime overhead. @@ -3546,7 +3825,17 @@ def __getattr__(attr): obj = _alias(getattr(re, attr), 1) elif attr in {"ContextManager", "AsyncContextManager"}: import contextlib - obj = _alias(getattr(contextlib, f"Abstract{attr}"), 1, name=attr) + obj = _alias(getattr(contextlib, f"Abstract{attr}"), 2, name=attr, defaults=(bool | None,)) + elif attr == "_collect_parameters": + import warnings + + depr_message = ( + "The private _collect_parameters function is deprecated and will be" + " removed in a future version of Python. Any use of private functions" + " is discouraged and may break in the future." + ) + warnings.warn(depr_message, category=DeprecationWarning, stacklevel=2) + obj = _collect_type_parameters else: raise AttributeError(f"module {__name__!r} has no attribute {attr!r}") globals()[attr] = obj diff --git a/Lib/unittest/__init__.py b/Lib/unittest/__init__.py index f1f6c911ef17d9..78ff6bb4fdcce5 100644 --- a/Lib/unittest/__init__.py +++ b/Lib/unittest/__init__.py @@ -27,7 +27,7 @@ def testMultiply(self): http://docs.python.org/library/unittest.html Copyright (c) 1999-2003 Steve Purcell -Copyright (c) 2003-2010 Python Software Foundation +Copyright (c) 2003 Python Software Foundation This module is free software, and you may redistribute it and/or modify it under the same terms as Python itself, so long as this copyright message and disclaimer are retained in their original form. @@ -57,9 +57,9 @@ def testMultiply(self): from .case import (addModuleCleanup, TestCase, FunctionTestCase, SkipTest, skip, skipIf, skipUnless, expectedFailure, doModuleCleanups, enterModuleContext) -from .suite import BaseTestSuite, TestSuite +from .suite import BaseTestSuite, TestSuite # noqa: F401 from .loader import TestLoader, defaultTestLoader -from .main import TestProgram, main +from .main import TestProgram, main # noqa: F401 from .runner import TextTestRunner, TextTestResult from .signals import installHandler, registerResult, removeResult, removeHandler # IsolatedAsyncioTestCase will be imported lazily. diff --git a/Lib/unittest/async_case.py b/Lib/unittest/async_case.py index 63ff6a5d1f8b61..6000af1cef0a78 100644 --- a/Lib/unittest/async_case.py +++ b/Lib/unittest/async_case.py @@ -5,6 +5,7 @@ from .case import TestCase +__unittest = True class IsolatedAsyncioTestCase(TestCase): # Names intentionally have a long prefix @@ -90,9 +91,13 @@ def _callSetUp(self): self._callAsync(self.asyncSetUp) def _callTestMethod(self, method): - if self._callMaybeAsync(method) is not None: - warnings.warn(f'It is deprecated to return a value that is not None from a ' - f'test case ({method})', DeprecationWarning, stacklevel=4) + result = self._callMaybeAsync(method) + if result is not None: + msg = ( + f'It is deprecated to return a value that is not None ' + f'from a test case ({method} returned {type(result).__name__!r})', + ) + warnings.warn(msg, DeprecationWarning, stacklevel=4) def _callTearDown(self): self._callAsync(self.asyncTearDown) diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index 001b640dc43ad6..10c3b7e122371e 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -332,6 +332,23 @@ def __exit__(self, exc_type, exc_value, tb): self._raiseFailure("{} not triggered".format(exc_name)) +class _AssertNotWarnsContext(_AssertWarnsContext): + + def __exit__(self, exc_type, exc_value, tb): + self.warnings_manager.__exit__(exc_type, exc_value, tb) + if exc_type is not None: + # let unexpected exceptions pass through + return + try: + exc_name = self.expected.__name__ + except AttributeError: + exc_name = str(self.expected) + for m in self.warnings: + w = m.message + if isinstance(w, self.expected): + self._raiseFailure(f"{exc_name} triggered") + + class _OrderedChainMap(collections.ChainMap): def __iter__(self): seen = set() @@ -586,9 +603,18 @@ def _callSetUp(self): self.setUp() def _callTestMethod(self, method): - if method() is not None: - warnings.warn(f'It is deprecated to return a value that is not None from a ' - f'test case ({method})', DeprecationWarning, stacklevel=3) + result = method() + if result is not None: + import inspect + msg = ( + f'It is deprecated to return a value that is not None ' + f'from a test case ({method} returned {type(result).__name__!r})' + ) + if inspect.iscoroutine(result): + msg += ( + '. Maybe you forgot to use IsolatedAsyncioTestCase as the base class?' + ) + warnings.warn(msg, DeprecationWarning, stacklevel=3) def _callTearDown(self): self.tearDown() @@ -811,6 +837,11 @@ def assertWarns(self, expected_warning, *args, **kwargs): context = _AssertWarnsContext(expected_warning, self) return context.handle('assertWarns', args, kwargs) + def _assertNotWarns(self, expected_warning, *args, **kwargs): + """The opposite of assertWarns. Private due to low demand.""" + context = _AssertNotWarnsContext(expected_warning, self) + return context.handle('_assertNotWarns', args, kwargs) + def assertLogs(self, logger=None, level=None): """Fail unless a log message of level *level* or higher is emitted on *logger_name* or its children. If omitted, *level* defaults to @@ -1290,13 +1321,71 @@ def assertIsInstance(self, obj, cls, msg=None): """Same as self.assertTrue(isinstance(obj, cls)), with a nicer default message.""" if not isinstance(obj, cls): - standardMsg = '%s is not an instance of %r' % (safe_repr(obj), cls) + if isinstance(cls, tuple): + standardMsg = f'{safe_repr(obj)} is not an instance of any of {cls!r}' + else: + standardMsg = f'{safe_repr(obj)} is not an instance of {cls!r}' self.fail(self._formatMessage(msg, standardMsg)) def assertNotIsInstance(self, obj, cls, msg=None): """Included for symmetry with assertIsInstance.""" if isinstance(obj, cls): - standardMsg = '%s is an instance of %r' % (safe_repr(obj), cls) + if isinstance(cls, tuple): + for x in cls: + if isinstance(obj, x): + cls = x + break + standardMsg = f'{safe_repr(obj)} is an instance of {cls!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertIsSubclass(self, cls, superclass, msg=None): + try: + if issubclass(cls, superclass): + return + except TypeError: + if not isinstance(cls, type): + self.fail(self._formatMessage(msg, f'{cls!r} is not a class')) + raise + if isinstance(superclass, tuple): + standardMsg = f'{cls!r} is not a subclass of any of {superclass!r}' + else: + standardMsg = f'{cls!r} is not a subclass of {superclass!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotIsSubclass(self, cls, superclass, msg=None): + try: + if not issubclass(cls, superclass): + return + except TypeError: + if not isinstance(cls, type): + self.fail(self._formatMessage(msg, f'{cls!r} is not a class')) + raise + if isinstance(superclass, tuple): + for x in superclass: + if issubclass(cls, x): + superclass = x + break + standardMsg = f'{cls!r} is a subclass of {superclass!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertHasAttr(self, obj, name, msg=None): + if not hasattr(obj, name): + if isinstance(obj, types.ModuleType): + standardMsg = f'module {obj.__name__!r} has no attribute {name!r}' + elif isinstance(obj, type): + standardMsg = f'type object {obj.__name__!r} has no attribute {name!r}' + else: + standardMsg = f'{type(obj).__name__!r} object has no attribute {name!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotHasAttr(self, obj, name, msg=None): + if hasattr(obj, name): + if isinstance(obj, types.ModuleType): + standardMsg = f'module {obj.__name__!r} has unexpected attribute {name!r}' + elif isinstance(obj, type): + standardMsg = f'type object {obj.__name__!r} has unexpected attribute {name!r}' + else: + standardMsg = f'{type(obj).__name__!r} object has unexpected attribute {name!r}' self.fail(self._formatMessage(msg, standardMsg)) def assertRaisesRegex(self, expected_exception, expected_regex, @@ -1360,6 +1449,80 @@ def assertNotRegex(self, text, unexpected_regex, msg=None): msg = self._formatMessage(msg, standardMsg) raise self.failureException(msg) + def _tail_type_check(self, s, tails, msg): + if not isinstance(tails, tuple): + tails = (tails,) + for tail in tails: + if isinstance(tail, str): + if not isinstance(s, str): + self.fail(self._formatMessage(msg, + f'Expected str, not {type(s).__name__}')) + elif isinstance(tail, (bytes, bytearray)): + if not isinstance(s, (bytes, bytearray)): + self.fail(self._formatMessage(msg, + f'Expected bytes, not {type(s).__name__}')) + + def assertStartsWith(self, s, prefix, msg=None): + try: + if s.startswith(prefix): + return + except (AttributeError, TypeError): + self._tail_type_check(s, prefix, msg) + raise + a = safe_repr(s, short=True) + b = safe_repr(prefix) + if isinstance(prefix, tuple): + standardMsg = f"{a} doesn't start with any of {b}" + else: + standardMsg = f"{a} doesn't start with {b}" + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotStartsWith(self, s, prefix, msg=None): + try: + if not s.startswith(prefix): + return + except (AttributeError, TypeError): + self._tail_type_check(s, prefix, msg) + raise + if isinstance(prefix, tuple): + for x in prefix: + if s.startswith(x): + prefix = x + break + a = safe_repr(s, short=True) + b = safe_repr(prefix) + self.fail(self._formatMessage(msg, f"{a} starts with {b}")) + + def assertEndsWith(self, s, suffix, msg=None): + try: + if s.endswith(suffix): + return + except (AttributeError, TypeError): + self._tail_type_check(s, suffix, msg) + raise + a = safe_repr(s, short=True) + b = safe_repr(suffix) + if isinstance(suffix, tuple): + standardMsg = f"{a} doesn't end with any of {b}" + else: + standardMsg = f"{a} doesn't end with {b}" + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotEndsWith(self, s, suffix, msg=None): + try: + if not s.endswith(suffix): + return + except (AttributeError, TypeError): + self._tail_type_check(s, suffix, msg) + raise + if isinstance(suffix, tuple): + for x in suffix: + if s.endswith(x): + suffix = x + break + a = safe_repr(s, short=True) + b = safe_repr(suffix) + self.fail(self._formatMessage(msg, f"{a} ends with {b}")) class FunctionTestCase(TestCase): diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py index 9a3e5cc4bf30e5..a52950dad224ee 100644 --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -254,6 +254,7 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None): Paths are sorted before being imported to ensure reproducible execution order even on filesystems with non-alphabetical ordering like ext3/4. """ + original_top_level_dir = self._top_level_dir set_implicit_top = False if top_level_dir is None and self._top_level_dir is not None: # make top_level_dir optional if called from load_tests in a package @@ -273,6 +274,8 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None): self._top_level_dir = top_level_dir is_not_importable = False + is_namespace = False + tests = [] if os.path.isdir(os.path.abspath(start_dir)): start_dir = os.path.abspath(start_dir) if start_dir != top_level_dir: @@ -285,12 +288,25 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None): is_not_importable = True else: the_module = sys.modules[start_dir] - top_part = start_dir.split('.')[0] - try: - start_dir = os.path.abspath( - os.path.dirname((the_module.__file__))) - except AttributeError: - if the_module.__name__ in sys.builtin_module_names: + if not hasattr(the_module, "__file__") or the_module.__file__ is None: + # look for namespace packages + try: + spec = the_module.__spec__ + except AttributeError: + spec = None + + if spec and spec.submodule_search_locations is not None: + is_namespace = True + + for path in the_module.__path__: + if (not set_implicit_top and + not path.startswith(top_level_dir)): + continue + self._top_level_dir = \ + (path.split(the_module.__name__ + .replace(".", os.path.sep))[0]) + tests.extend(self._find_tests(path, pattern, namespace=True)) + elif the_module.__name__ in sys.builtin_module_names: # builtin module raise TypeError('Can not use builtin modules ' 'as dotted module names') from None @@ -299,14 +315,28 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None): f"don't know how to discover from {the_module!r}" ) from None + else: + top_part = start_dir.split('.')[0] + start_dir = os.path.abspath(os.path.dirname((the_module.__file__))) + if set_implicit_top: - self._top_level_dir = self._get_directory_containing_module(top_part) + if not is_namespace: + if sys.modules[top_part].__file__ is None: + self._top_level_dir = os.path.dirname(the_module.__file__) + if self._top_level_dir not in sys.path: + sys.path.insert(0, self._top_level_dir) + else: + self._top_level_dir = \ + self._get_directory_containing_module(top_part) sys.path.remove(top_level_dir) if is_not_importable: raise ImportError('Start directory is not importable: %r' % start_dir) - tests = list(self._find_tests(start_dir, pattern)) + if not is_namespace: + tests = list(self._find_tests(start_dir, pattern)) + + self._top_level_dir = original_top_level_dir return self.suiteClass(tests) def _get_directory_containing_module(self, module_name): @@ -341,7 +371,7 @@ def _match_path(self, path, full_path, pattern): # override this method to use alternative matching strategy return fnmatch(path, pattern) - def _find_tests(self, start_dir, pattern): + def _find_tests(self, start_dir, pattern, namespace=False): """Used by discovery. Yields test suites it loads.""" # Handle the __init__ in this package name = self._get_name_from_path(start_dir) @@ -350,7 +380,8 @@ def _find_tests(self, start_dir, pattern): if name != '.' and name not in self._loading_packages: # name is in self._loading_packages while we have called into # loadTestsFromModule with name. - tests, should_recurse = self._find_test_path(start_dir, pattern) + tests, should_recurse = self._find_test_path( + start_dir, pattern, namespace) if tests is not None: yield tests if not should_recurse: @@ -361,7 +392,8 @@ def _find_tests(self, start_dir, pattern): paths = sorted(os.listdir(start_dir)) for path in paths: full_path = os.path.join(start_dir, path) - tests, should_recurse = self._find_test_path(full_path, pattern) + tests, should_recurse = self._find_test_path( + full_path, pattern, False) if tests is not None: yield tests if should_recurse: @@ -369,11 +401,11 @@ def _find_tests(self, start_dir, pattern): name = self._get_name_from_path(full_path) self._loading_packages.add(name) try: - yield from self._find_tests(full_path, pattern) + yield from self._find_tests(full_path, pattern, False) finally: self._loading_packages.discard(name) - def _find_test_path(self, full_path, pattern): + def _find_test_path(self, full_path, pattern, namespace=False): """Used by discovery. Loads tests from a single file, or a directories' __init__.py when @@ -417,7 +449,8 @@ def _find_test_path(self, full_path, pattern): msg % (mod_name, module_dir, expected_dir)) return self.loadTestsFromModule(module, pattern=pattern), False elif os.path.isdir(full_path): - if not os.path.isfile(os.path.join(full_path, '__init__.py')): + if (not namespace and + not os.path.isfile(os.path.join(full_path, '__init__.py'))): return None, False load_tests = None diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 93f4d9743ed2fa..55cb4b1f6aff90 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -32,8 +32,9 @@ import sys import builtins import pkgutil -from asyncio import iscoroutinefunction +from inspect import iscoroutinefunction import threading +from dataclasses import fields, is_dataclass from types import CodeType, ModuleType, MethodType from unittest.util import safe_repr from functools import wraps, partial @@ -573,7 +574,7 @@ def __get_return_value(self): if self._mock_delegate is not None: ret = self._mock_delegate.return_value - if ret is DEFAULT: + if ret is DEFAULT and self._mock_wraps is None: ret = self._get_child_mock( _new_parent=self, _new_name='()' ) @@ -628,7 +629,9 @@ def __set_side_effect(self, value): side_effect = property(__get_side_effect, __set_side_effect) - def reset_mock(self, visited=None,*, return_value=False, side_effect=False): + def reset_mock(self, visited=None, *, + return_value: bool = False, + side_effect: bool = False): "Restore the mock object to its initial state." if visited is None: visited = [] @@ -830,6 +833,9 @@ def __setattr__(self, name, value): mock_name = f'{self._extract_mock_name()}.{name}' raise AttributeError(f'Cannot set {mock_name}') + if isinstance(value, PropertyMock): + self.__dict__[name] = value + return return object.__setattr__(self, name, value) @@ -1234,6 +1240,9 @@ def _execute_mock_call(self, /, *args, **kwargs): if self._mock_return_value is not DEFAULT: return self.return_value + if self._mock_delegate and self._mock_delegate.return_value is not DEFAULT: + return self.return_value + if self._mock_wraps is not None: return self._mock_wraps(*args, **kwargs) @@ -1351,6 +1360,7 @@ def __init__( self.autospec = autospec self.kwargs = kwargs self.additional_patchers = [] + self.is_started = False def copy(self): @@ -1463,6 +1473,9 @@ def get_original(self): def __enter__(self): """Perform the patch.""" + if self.is_started: + raise RuntimeError("Patch is already started") + new, spec, spec_set = self.new, self.spec, self.spec_set autospec, kwargs = self.autospec, self.kwargs new_callable = self.new_callable @@ -1505,13 +1518,12 @@ def __enter__(self): if isinstance(original, type): # If we're patching out a class and there is a spec inherit = True - if spec is None and _is_async_obj(original): - Klass = AsyncMock - else: - Klass = MagicMock - _kwargs = {} + + # Determine the Klass to use if new_callable is not None: Klass = new_callable + elif spec is None and _is_async_obj(original): + Klass = AsyncMock elif spec is not None or spec_set is not None: this_spec = spec if spec_set is not None: @@ -1524,7 +1536,12 @@ def __enter__(self): Klass = AsyncMock elif not_callable: Klass = NonCallableMagicMock + else: + Klass = MagicMock + else: + Klass = MagicMock + _kwargs = {} if spec is not None: _kwargs['spec'] = spec if spec_set is not None: @@ -1590,6 +1607,7 @@ def __enter__(self): self.temp_original = original self.is_local = local self._exit_stack = contextlib.ExitStack() + self.is_started = True try: setattr(self.target, self.attribute, new_attr) if self.attribute_name is not None: @@ -1609,6 +1627,9 @@ def __enter__(self): def __exit__(self, *exc_info): """Undo the patch.""" + if not self.is_started: + return + if self.is_local and self.temp_original is not DEFAULT: setattr(self.target, self.attribute, self.temp_original) else: @@ -1625,6 +1646,7 @@ def __exit__(self, *exc_info): del self.target exit_stack = self._exit_stack del self._exit_stack + self.is_started = False return exit_stack.__exit__(*exc_info) @@ -1745,7 +1767,7 @@ def patch( the patch is undone. If `new` is omitted, then the target is replaced with an - `AsyncMock if the patched object is an async function or a + `AsyncMock` if the patched object is an async function or a `MagicMock` otherwise. If `patch` is used as a decorator and `new` is omitted, the created mock is passed in as an extra argument to the decorated function. If `patch` is used as a context manager the created @@ -1819,7 +1841,8 @@ def patch( class _patch_dict(object): """ Patch a dictionary, or dictionary like object, and restore the dictionary - to its original state after the test. + to its original state after the test, where the restored dictionary is + a copy of the dictionary as it was before the test. `in_dict` can be a dictionary or a mapping like container. If it is a mapping then it must at least support getting, setting and deleting items @@ -2155,8 +2178,6 @@ def _mock_set_magics(self): if getattr(self, "_mock_methods", None) is not None: these_magics = orig_magics.intersection(self._mock_methods) - - remove_magics = set() remove_magics = orig_magics - these_magics for entry in remove_magics: @@ -2209,6 +2230,17 @@ def mock_add_spec(self, spec, spec_set=False): self._mock_add_spec(spec, spec_set) self._mock_set_magics() + def reset_mock(self, /, *args, return_value: bool = False, **kwargs): + if ( + return_value + and self._mock_name + and _is_magic(self._mock_name) + ): + # Don't reset return values for magic methods, + # otherwise `m.__str__` will start + # to return `MagicMock` instances, instead of `str` instances. + return_value = False + super().reset_mock(*args, return_value=return_value, **kwargs) class MagicProxy(Base): @@ -2445,7 +2477,7 @@ class AsyncMock(AsyncMockMixin, AsyncMagicMixin, Mock): recognized as an async function, and the result of a call is an awaitable: >>> mock = AsyncMock() - >>> iscoroutinefunction(mock) + >>> inspect.iscoroutinefunction(mock) True >>> inspect.isawaitable(mock()) True @@ -2734,7 +2766,15 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, raise InvalidSpecError(f'Cannot autospec a Mock object. ' f'[object={spec!r}]') is_async_func = _is_async_func(spec) - _kwargs = {'spec': spec} + + entries = [(entry, _missing) for entry in dir(spec)] + if is_type and instance and is_dataclass(spec): + dataclass_fields = fields(spec) + entries.extend((f.name, f.type) for f in dataclass_fields) + _kwargs = {'spec': [f.name for f in dataclass_fields]} + else: + _kwargs = {'spec': spec} + if spec_set: _kwargs = {'spec_set': spec} elif spec is None: @@ -2745,6 +2785,12 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, if not unsafe: _check_spec_arg_typos(kwargs) + _name = kwargs.pop('name', _name) + _new_name = _name + if _parent is None: + # for a top level object no _new_name should be set + _new_name = '' + _kwargs.update(kwargs) Klass = MagicMock @@ -2762,13 +2808,6 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, elif is_type and instance and not _instance_callable(spec): Klass = NonCallableMagicMock - _name = _kwargs.pop('name', _name) - - _new_name = _name - if _parent is None: - # for a top level object no _new_name should be set - _new_name = '' - mock = Klass(parent=_parent, _new_parent=_parent, _new_name=_new_name, name=_name, **_kwargs) @@ -2785,11 +2824,14 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, if _parent is not None and not instance: _parent._mock_children[_name] = mock + # Pop wraps from kwargs because it must not be passed to configure_mock. + wrapped = kwargs.pop('wraps', None) if is_type and not instance and 'return_value' not in kwargs: mock.return_value = create_autospec(spec, spec_set, instance=True, - _name='()', _parent=mock) + _name='()', _parent=mock, + wraps=wrapped) - for entry in dir(spec): + for entry, original in entries: if _is_magic(entry): # MagicMock already does the useful magic methods for us continue @@ -2803,14 +2845,18 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, # AttributeError on being fetched? # we could be resilient against it, or catch and propagate the # exception when the attribute is fetched from the mock - try: - original = getattr(spec, entry) - except AttributeError: - continue + if original is _missing: + try: + original = getattr(spec, entry) + except AttributeError: + continue - kwargs = {'spec': original} + child_kwargs = {'spec': original} + # Wrap child attributes also. + if wrapped and hasattr(wrapped, entry): + child_kwargs.update(wraps=original) if spec_set: - kwargs = {'spec_set': original} + child_kwargs = {'spec_set': original} if not isinstance(original, FunctionTypes): new = _SpecState(original, spec_set, mock, entry, instance) @@ -2821,14 +2867,13 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, parent = mock.mock skipfirst = _must_skip(spec, entry, is_type) - kwargs['_eat_self'] = skipfirst + child_kwargs['_eat_self'] = skipfirst if iscoroutinefunction(original): child_klass = AsyncMock else: child_klass = MagicMock new = child_klass(parent=parent, name=entry, _new_name=entry, - _new_parent=parent, - **kwargs) + _new_parent=parent, **child_kwargs) mock._mock_children[entry] = new new.return_value = child_klass() _check_signature(original, new, skipfirst=skipfirst) @@ -2839,6 +2884,11 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, # setting as an instance attribute? if isinstance(new, FunctionTypes): setattr(mock, entry, new) + # kwargs are passed with respect to the parent mock so, they are not used + # for creating return_value of the parent mock. So, this condition + # should be true only for the parent mock if kwargs are given. + if _is_instance_mock(mock) and kwargs: + mock.configure_mock(**kwargs) return mock diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py index 3ace0a5b7bf2ef..b8ea396db6772e 100644 --- a/Lib/unittest/result.py +++ b/Lib/unittest/result.py @@ -189,7 +189,10 @@ def _exc_info_to_string(self, err, test): tb_e = traceback.TracebackException( exctype, value, tb, capture_locals=self.tb_locals, compact=True) - msgLines = list(tb_e.format()) + from _colorize import can_colorize + + colorize = hasattr(self, "stream") and can_colorize(file=self.stream) + msgLines = list(tb_e.format(colorize=colorize)) if self.buffer: output = sys.stdout.getvalue() diff --git a/Lib/unittest/runner.py b/Lib/unittest/runner.py index 2bcadf0c998bd9..eb0234a2617680 100644 --- a/Lib/unittest/runner.py +++ b/Lib/unittest/runner.py @@ -4,6 +4,8 @@ import time import warnings +from _colorize import get_colors + from . import result from .case import _SubTest from .signals import registerResult @@ -13,18 +15,18 @@ class _WritelnDecorator(object): """Used to decorate file-like objects with a handy 'writeln' method""" - def __init__(self,stream): + def __init__(self, stream): self.stream = stream def __getattr__(self, attr): if attr in ('stream', '__getstate__'): raise AttributeError(attr) - return getattr(self.stream,attr) + return getattr(self.stream, attr) def writeln(self, arg=None): if arg: self.write(arg) - self.write('\n') # text-mode streams translate to \r\n if needed + self.write('\n') # text-mode streams translate to \r\n if needed class TextTestResult(result.TestResult): @@ -43,6 +45,7 @@ def __init__(self, stream, descriptions, verbosity, *, durations=None): self.showAll = verbosity > 1 self.dots = verbosity == 1 self.descriptions = descriptions + self._ansi = get_colors(file=stream) self._newline = True self.durations = durations @@ -76,86 +79,102 @@ def _write_status(self, test, status): def addSubTest(self, test, subtest, err): if err is not None: + red, reset = self._ansi.RED, self._ansi.RESET if self.showAll: if issubclass(err[0], subtest.failureException): - self._write_status(subtest, "FAIL") + self._write_status(subtest, f"{red}FAIL{reset}") else: - self._write_status(subtest, "ERROR") + self._write_status(subtest, f"{red}ERROR{reset}") elif self.dots: if issubclass(err[0], subtest.failureException): - self.stream.write('F') + self.stream.write(f"{red}F{reset}") else: - self.stream.write('E') + self.stream.write(f"{red}E{reset}") self.stream.flush() super(TextTestResult, self).addSubTest(test, subtest, err) def addSuccess(self, test): super(TextTestResult, self).addSuccess(test) + green, reset = self._ansi.GREEN, self._ansi.RESET if self.showAll: - self._write_status(test, "ok") + self._write_status(test, f"{green}ok{reset}") elif self.dots: - self.stream.write('.') + self.stream.write(f"{green}.{reset}") self.stream.flush() def addError(self, test, err): super(TextTestResult, self).addError(test, err) + red, reset = self._ansi.RED, self._ansi.RESET if self.showAll: - self._write_status(test, "ERROR") + self._write_status(test, f"{red}ERROR{reset}") elif self.dots: - self.stream.write('E') + self.stream.write(f"{red}E{reset}") self.stream.flush() def addFailure(self, test, err): super(TextTestResult, self).addFailure(test, err) + red, reset = self._ansi.RED, self._ansi.RESET if self.showAll: - self._write_status(test, "FAIL") + self._write_status(test, f"{red}FAIL{reset}") elif self.dots: - self.stream.write('F') + self.stream.write(f"{red}F{reset}") self.stream.flush() def addSkip(self, test, reason): super(TextTestResult, self).addSkip(test, reason) + yellow, reset = self._ansi.YELLOW, self._ansi.RESET if self.showAll: - self._write_status(test, "skipped {0!r}".format(reason)) + self._write_status(test, f"{yellow}skipped{reset} {reason!r}") elif self.dots: - self.stream.write("s") + self.stream.write(f"{yellow}s{reset}") self.stream.flush() def addExpectedFailure(self, test, err): super(TextTestResult, self).addExpectedFailure(test, err) + yellow, reset = self._ansi.YELLOW, self._ansi.RESET if self.showAll: - self.stream.writeln("expected failure") + self.stream.writeln(f"{yellow}expected failure{reset}") self.stream.flush() elif self.dots: - self.stream.write("x") + self.stream.write(f"{yellow}x{reset}") self.stream.flush() def addUnexpectedSuccess(self, test): super(TextTestResult, self).addUnexpectedSuccess(test) + red, reset = self._ansi.RED, self._ansi.RESET if self.showAll: - self.stream.writeln("unexpected success") + self.stream.writeln(f"{red}unexpected success{reset}") self.stream.flush() elif self.dots: - self.stream.write("u") + self.stream.write(f"{red}u{reset}") self.stream.flush() def printErrors(self): + bold_red = self._ansi.BOLD_RED + red = self._ansi.RED + reset = self._ansi.RESET if self.dots or self.showAll: self.stream.writeln() self.stream.flush() - self.printErrorList('ERROR', self.errors) - self.printErrorList('FAIL', self.failures) - unexpectedSuccesses = getattr(self, 'unexpectedSuccesses', ()) + self.printErrorList(f"{red}ERROR{reset}", self.errors) + self.printErrorList(f"{red}FAIL{reset}", self.failures) + unexpectedSuccesses = getattr(self, "unexpectedSuccesses", ()) if unexpectedSuccesses: self.stream.writeln(self.separator1) for test in unexpectedSuccesses: - self.stream.writeln(f"UNEXPECTED SUCCESS: {self.getDescription(test)}") + self.stream.writeln( + f"{red}UNEXPECTED SUCCESS{bold_red}: " + f"{self.getDescription(test)}{reset}" + ) self.stream.flush() def printErrorList(self, flavour, errors): + bold_red, reset = self._ansi.BOLD_RED, self._ansi.RESET for test, err in errors: self.stream.writeln(self.separator1) - self.stream.writeln("%s: %s" % (flavour,self.getDescription(test))) + self.stream.writeln( + f"{flavour}{bold_red}: {self.getDescription(test)}{reset}" + ) self.stream.writeln(self.separator2) self.stream.writeln("%s" % err) self.stream.flush() @@ -232,7 +251,7 @@ def run(self, test): if self.warnings: # if self.warnings is set, use it to filter all the warnings warnings.simplefilter(self.warnings) - startTime = time.perf_counter() + start_time = time.perf_counter() startTestRun = getattr(result, 'startTestRun', None) if startTestRun is not None: startTestRun() @@ -242,8 +261,8 @@ def run(self, test): stopTestRun = getattr(result, 'stopTestRun', None) if stopTestRun is not None: stopTestRun() - stopTime = time.perf_counter() - timeTaken = stopTime - startTime + stop_time = time.perf_counter() + time_taken = stop_time - start_time result.printErrors() if self.durations is not None: self._printDurations(result) @@ -253,10 +272,10 @@ def run(self, test): run = result.testsRun self.stream.writeln("Ran %d test%s in %.3fs" % - (run, run != 1 and "s" or "", timeTaken)) + (run, run != 1 and "s" or "", time_taken)) self.stream.writeln() - expectedFails = unexpectedSuccesses = skipped = 0 + expected_fails = unexpected_successes = skipped = 0 try: results = map(len, (result.expectedFailures, result.unexpectedSuccesses, @@ -264,26 +283,35 @@ def run(self, test): except AttributeError: pass else: - expectedFails, unexpectedSuccesses, skipped = results + expected_fails, unexpected_successes, skipped = results infos = [] + ansi = get_colors(file=self.stream) + bold_red = ansi.BOLD_RED + green = ansi.GREEN + red = ansi.RED + reset = ansi.RESET + yellow = ansi.YELLOW + if not result.wasSuccessful(): - self.stream.write("FAILED") + self.stream.write(f"{bold_red}FAILED{reset}") failed, errored = len(result.failures), len(result.errors) if failed: - infos.append("failures=%d" % failed) + infos.append(f"{bold_red}failures={failed}{reset}") if errored: - infos.append("errors=%d" % errored) + infos.append(f"{bold_red}errors={errored}{reset}") elif run == 0 and not skipped: - self.stream.write("NO TESTS RAN") + self.stream.write(f"{yellow}NO TESTS RAN{reset}") else: - self.stream.write("OK") + self.stream.write(f"{green}OK{reset}") if skipped: - infos.append("skipped=%d" % skipped) - if expectedFails: - infos.append("expected failures=%d" % expectedFails) - if unexpectedSuccesses: - infos.append("unexpected successes=%d" % unexpectedSuccesses) + infos.append(f"{yellow}skipped={skipped}{reset}") + if expected_fails: + infos.append(f"{yellow}expected failures={expected_fails}{reset}") + if unexpected_successes: + infos.append( + f"{red}unexpected successes={unexpected_successes}{reset}" + ) if infos: self.stream.writeln(" (%s)" % (", ".join(infos),)) else: diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index c129b0d7971d71..c412c729852272 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -247,11 +247,11 @@ def _hostinfo(self): return hostname, port -_DefragResultBase = namedtuple('DefragResult', 'url fragment') +_DefragResultBase = namedtuple('_DefragResultBase', 'url fragment') _SplitResultBase = namedtuple( - 'SplitResult', 'scheme netloc path query fragment') + '_SplitResultBase', 'scheme netloc path query fragment') _ParseResultBase = namedtuple( - 'ParseResult', 'scheme netloc path params query fragment') + '_ParseResultBase', 'scheme netloc path params query fragment') _DefragResultBase.__doc__ = """ DefragResult(url, fragment) @@ -392,20 +392,23 @@ def urlparse(url, scheme='', allow_fragments=True): Note that % escapes are not expanded. """ url, scheme, _coerce_result = _coerce_args(url, scheme) - splitresult = urlsplit(url, scheme, allow_fragments) - scheme, netloc, url, query, fragment = splitresult - if scheme in uses_params and ';' in url: - url, params = _splitparams(url) - else: - params = '' - result = ParseResult(scheme, netloc, url, params, query, fragment) + scheme, netloc, url, params, query, fragment = _urlparse(url, scheme, allow_fragments) + result = ParseResult(scheme or '', netloc or '', url, params or '', query or '', fragment or '') return _coerce_result(result) -def _splitparams(url): +def _urlparse(url, scheme=None, allow_fragments=True): + scheme, netloc, url, query, fragment = _urlsplit(url, scheme, allow_fragments) + if (scheme or '') in uses_params and ';' in url: + url, params = _splitparams(url, allow_none=True) + else: + params = None + return (scheme, netloc, url, params, query, fragment) + +def _splitparams(url, allow_none=False): if '/' in url: i = url.find(';', url.rfind('/')) if i < 0: - return url, '' + return url, None if allow_none else '' else: i = url.find(';') return url[:i], url[i+1:] @@ -472,17 +475,23 @@ def urlsplit(url, scheme='', allow_fragments=True): """ url, scheme, _coerce_result = _coerce_args(url, scheme) + scheme, netloc, url, query, fragment = _urlsplit(url, scheme, allow_fragments) + v = SplitResult(scheme or '', netloc or '', url, query or '', fragment or '') + return _coerce_result(v) + +def _urlsplit(url, scheme=None, allow_fragments=True): # Only lstrip url as some applications rely on preserving trailing space. # (https://url.spec.whatwg.org/#concept-basic-url-parser would strip both) url = url.lstrip(_WHATWG_C0_CONTROL_OR_SPACE) - scheme = scheme.strip(_WHATWG_C0_CONTROL_OR_SPACE) - for b in _UNSAFE_URL_BYTES_TO_REMOVE: url = url.replace(b, "") - scheme = scheme.replace(b, "") + if scheme is not None: + scheme = scheme.strip(_WHATWG_C0_CONTROL_OR_SPACE) + for b in _UNSAFE_URL_BYTES_TO_REMOVE: + scheme = scheme.replace(b, "") allow_fragments = bool(allow_fragments) - netloc = query = fragment = '' + netloc = query = fragment = None i = url.find(':') if i > 0 and url[0].isascii() and url[0].isalpha(): for c in url[:i]: @@ -503,8 +512,7 @@ def urlsplit(url, scheme='', allow_fragments=True): if '?' in url: url, query = url.split('?', 1) _checknetloc(netloc) - v = SplitResult(scheme, netloc, url, query, fragment) - return _coerce_result(v) + return (scheme, netloc, url, query, fragment) def urlunparse(components): """Put a parsed URL back together again. This may result in a @@ -513,9 +521,15 @@ def urlunparse(components): (the draft states that these are equivalent).""" scheme, netloc, url, params, query, fragment, _coerce_result = ( _coerce_args(*components)) + if not netloc: + if scheme and scheme in uses_netloc and (not url or url[:1] == '/'): + netloc = '' + else: + netloc = None if params: url = "%s;%s" % (url, params) - return _coerce_result(urlunsplit((scheme, netloc, url, query, fragment))) + return _coerce_result(_urlunsplit(scheme or None, netloc, url, + query or None, fragment or None)) def urlunsplit(components): """Combine the elements of a tuple as returned by urlsplit() into a @@ -525,16 +539,27 @@ def urlunsplit(components): empty query; the RFC states that these are equivalent).""" scheme, netloc, url, query, fragment, _coerce_result = ( _coerce_args(*components)) - if netloc or (scheme and scheme in uses_netloc and url[:2] != '//'): + if not netloc: + if scheme and scheme in uses_netloc and (not url or url[:1] == '/'): + netloc = '' + else: + netloc = None + return _coerce_result(_urlunsplit(scheme or None, netloc, url, + query or None, fragment or None)) + +def _urlunsplit(scheme, netloc, url, query, fragment): + if netloc is not None: if url and url[:1] != '/': url = '/' + url - url = '//' + (netloc or '') + url + url = '//' + netloc + url + elif url[:2] == '//': + url = '//' + url if scheme: url = scheme + ':' + url - if query: + if query is not None: url = url + '?' + query - if fragment: + if fragment is not None: url = url + '#' + fragment - return _coerce_result(url) + return url def urljoin(base, url, allow_fragments=True): """Join a base URL and a possibly relative URL to form an absolute @@ -545,26 +570,29 @@ def urljoin(base, url, allow_fragments=True): return base base, url, _coerce_result = _coerce_args(base, url) - bscheme, bnetloc, bpath, bparams, bquery, bfragment = \ - urlparse(base, '', allow_fragments) - scheme, netloc, path, params, query, fragment = \ - urlparse(url, bscheme, allow_fragments) - - if scheme != bscheme or scheme not in uses_relative: + bscheme, bnetloc, bpath, bquery, bfragment = \ + _urlsplit(base, None, allow_fragments) + scheme, netloc, path, query, fragment = \ + _urlsplit(url, None, allow_fragments) + + if scheme is None: + scheme = bscheme + if scheme != bscheme or (scheme and scheme not in uses_relative): return _coerce_result(url) - if scheme in uses_netloc: + if not scheme or scheme in uses_netloc: if netloc: - return _coerce_result(urlunparse((scheme, netloc, path, - params, query, fragment))) + return _coerce_result(_urlunsplit(scheme, netloc, path, + query, fragment)) netloc = bnetloc - if not path and not params: + if not path: path = bpath - params = bparams - if not query: + if query is None: query = bquery - return _coerce_result(urlunparse((scheme, netloc, path, - params, query, fragment))) + if fragment is None: + fragment = bfragment + return _coerce_result(_urlunsplit(scheme, netloc, path, + query, fragment)) base_parts = bpath.split('/') if base_parts[-1] != '': @@ -601,8 +629,8 @@ def urljoin(base, url, allow_fragments=True): # then we need to append the trailing '/' resolved_path.append('') - return _coerce_result(urlunparse((scheme, netloc, '/'.join( - resolved_path) or '/', params, query, fragment))) + return _coerce_result(_urlunsplit(scheme, netloc, '/'.join( + resolved_path) or '/', query, fragment)) def urldefrag(url): @@ -614,12 +642,12 @@ def urldefrag(url): """ url, _coerce_result = _coerce_args(url) if '#' in url: - s, n, p, a, q, frag = urlparse(url) - defrag = urlunparse((s, n, p, a, q, '')) + s, n, p, q, frag = _urlsplit(url) + defrag = _urlunsplit(s, n, p, q, None) else: frag = '' defrag = url - return _coerce_result(DefragResult(defrag, frag)) + return _coerce_result(DefragResult(defrag, frag or '')) _hexdig = '0123456789ABCDEFabcdef' _hextobyte = None @@ -725,7 +753,8 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, parsed_result = {} pairs = parse_qsl(qs, keep_blank_values, strict_parsing, encoding=encoding, errors=errors, - max_num_fields=max_num_fields, separator=separator) + max_num_fields=max_num_fields, separator=separator, + _stacklevel=2) for name, value in pairs: if name in parsed_result: parsed_result[name].append(value) @@ -735,7 +764,7 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, - encoding='utf-8', errors='replace', max_num_fields=None, separator='&'): + encoding='utf-8', errors='replace', max_num_fields=None, separator='&', *, _stacklevel=1): """Parse a query given as a string argument. Arguments: @@ -763,42 +792,56 @@ def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, Returns a list, as G-d intended. """ - qs, _coerce_result = _coerce_args(qs) - separator, _ = _coerce_args(separator) - - if not separator or (not isinstance(separator, (str, bytes))): + if not separator or not isinstance(separator, (str, bytes)): raise ValueError("Separator must be of type string or bytes.") + if isinstance(qs, str): + if not isinstance(separator, str): + separator = str(separator, 'ascii') + eq = '=' + def _unquote(s): + return unquote_plus(s, encoding=encoding, errors=errors) + elif qs is None: + return [] + else: + try: + # Use memoryview() to reject integers and iterables, + # acceptable by the bytes constructor. + qs = bytes(memoryview(qs)) + except TypeError: + if not qs: + warnings.warn(f"Accepting {type(qs).__name__} objects with " + f"false value in urllib.parse.parse_qsl() is " + f"deprecated as of 3.14", + DeprecationWarning, stacklevel=_stacklevel + 1) + return [] + raise + if isinstance(separator, str): + separator = bytes(separator, 'ascii') + eq = b'=' + def _unquote(s): + return unquote_to_bytes(s.replace(b'+', b' ')) + + if not qs: + return [] # If max_num_fields is defined then check that the number of fields # is less than max_num_fields. This prevents a memory exhaustion DOS # attack via post bodies with many fields. if max_num_fields is not None: - num_fields = 1 + qs.count(separator) if qs else 0 + num_fields = 1 + qs.count(separator) if max_num_fields < num_fields: raise ValueError('Max number of fields exceeded') r = [] - query_args = qs.split(separator) if qs else [] - for name_value in query_args: - if not name_value and not strict_parsing: - continue - nv = name_value.split('=', 1) - if len(nv) != 2: - if strict_parsing: + for name_value in qs.split(separator): + if name_value or strict_parsing: + name, has_eq, value = name_value.partition(eq) + if not has_eq and strict_parsing: raise ValueError("bad query field: %r" % (name_value,)) - # Handle case of a control-name with no equal sign - if keep_blank_values: - nv.append('') - else: - continue - if len(nv[1]) or keep_blank_values: - name = nv[0].replace('+', ' ') - name = unquote(name, encoding=encoding, errors=errors) - name = _coerce_result(name) - value = nv[1].replace('+', ' ') - value = unquote(value, encoding=encoding, errors=errors) - value = _coerce_result(value) - r.append((name, value)) + if value or keep_blank_values: + name = _unquote(name) + value = _unquote(value) + r.append((name, value)) return r def unquote_plus(string, encoding='utf-8', errors='replace'): @@ -816,14 +859,6 @@ def unquote_plus(string, encoding='utf-8', errors='replace'): b'_.-~') _ALWAYS_SAFE_BYTES = bytes(_ALWAYS_SAFE) -def __getattr__(name): - if name == 'Quoter': - warnings.warn('Deprecated in 3.11. ' - 'urllib.parse.Quoter will be removed in Python 3.14. ' - 'It was not intended to be a public API.', - DeprecationWarning, stacklevel=2) - return _Quoter - raise AttributeError(f'module {__name__!r} has no attribute {name!r}') class _Quoter(dict): """A mapping from bytes numbers (in range(0,256)) to strings. diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index d22af6618d80f1..0d1b594b8cf20b 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -83,6 +83,7 @@ import base64 import bisect +import contextlib import email import hashlib import http.client @@ -94,21 +95,19 @@ import sys import time import tempfile -import contextlib -import warnings from urllib.error import URLError, HTTPError, ContentTooShortError from urllib.parse import ( urlparse, urlsplit, urljoin, unwrap, quote, unquote, _splittype, _splithost, _splitport, _splituser, _splitpasswd, - _splitattr, _splitquery, _splitvalue, _splittag, _to_bytes, + _splitattr, _splitvalue, _splittag, unquote_to_bytes, urlunparse) from urllib.response import addinfourl, addclosehook # check for SSL try: - import ssl + import ssl # noqa: F401 except ImportError: _have_ssl = False else: @@ -128,7 +127,7 @@ 'urlopen', 'install_opener', 'build_opener', 'pathname2url', 'url2pathname', 'getproxies', # Legacy interface - 'urlretrieve', 'urlcleanup', 'URLopener', 'FancyURLopener', + 'urlretrieve', 'urlcleanup', ] # used in User-Agent header sent @@ -165,8 +164,7 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, the reason phrase returned by the server --- instead of the response headers as it is specified in the documentation for HTTPResponse. - For FTP, file, and data URLs and requests explicitly handled by legacy - URLopener and FancyURLopener classes, this function returns a + For FTP, file, and data URLs, this function returns a urllib.response.addinfourl object. Note that None may be returned if no handler handles the request (though @@ -650,6 +648,7 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): newheaders = {k: v for k, v in req.headers.items() if k.lower() not in CONTENT_HEADERS} return Request(newurl, + method="HEAD" if m == "HEAD" else "GET", headers=newheaders, origin_req_host=req.origin_req_host, unverifiable=True) @@ -877,9 +876,9 @@ def find_user_password(self, realm, authuri): class HTTPPasswordMgrWithPriorAuth(HTTPPasswordMgrWithDefaultRealm): - def __init__(self, *args, **kwargs): + def __init__(self): self.authenticated = {} - super().__init__(*args, **kwargs) + super().__init__() def add_password(self, realm, uri, user, passwd, is_authenticated=False): self.update_authenticated(uri, is_authenticated) @@ -939,6 +938,7 @@ def _parse_realm(self, header): for mo in AbstractBasicAuthHandler.rx.finditer(header): scheme, quote, realm = mo.groups() if quote not in ['"', "'"]: + import warnings warnings.warn("Basic Auth Realm was unquoted", UserWarning, 3) @@ -1048,7 +1048,7 @@ def http_error_407(self, req, fp, code, msg, headers): class AbstractDigestAuthHandler: - # Digest authentication is specified in RFC 2617. + # Digest authentication is specified in RFC 2617/7616. # XXX The client does not inspect the Authentication-Info header # in a successful response. @@ -1176,11 +1176,14 @@ def get_authorization(self, req, chal): return base def get_algorithm_impls(self, algorithm): + # algorithm names taken from RFC 7616 Section 6.1 # lambdas assume digest modules are imported at the top level if algorithm == 'MD5': H = lambda x: hashlib.md5(x.encode("ascii")).hexdigest() - elif algorithm == 'SHA': + elif algorithm == 'SHA': # non-standard, retained for compatibility. H = lambda x: hashlib.sha1(x.encode("ascii")).hexdigest() + elif algorithm == 'SHA-256': + H = lambda x: hashlib.sha256(x.encode("ascii")).hexdigest() # XXX MD5-sess else: raise ValueError("Unsupported digest authentication " @@ -1488,13 +1491,10 @@ def open_local_file(self, req): host, port = _splitport(host) if not host or \ (not port and _safe_gethostbyname(host) in self.get_names()): - if host: - origurl = 'file://' + host + filename - else: - origurl = 'file://' + filename + origurl = 'file:' + pathname2url(localfile) return addinfourl(open(localfile, 'rb'), headers, origurl) except OSError as exp: - raise URLError(exp) + raise URLError(exp, exp.filename) raise URLError('file not on local host') def _safe_gethostbyname(host): @@ -1554,7 +1554,7 @@ def ftp_open(self, req): headers = email.message_from_string(headers) return addinfourl(fp, headers, req.full_url) except ftplib.all_errors as exp: - raise URLError(exp) from exp + raise URLError(f"ftp error: {exp}") from exp def connect_ftp(self, user, passwd, host, port, dirs, timeout): return ftpwrapper(user, passwd, host, port, dirs, timeout, @@ -1646,8 +1646,6 @@ def data_open(self, req): # Code move from the old urllib module -MAXFTPCACHE = 10 # Trim the ftp cache beyond this size - # Helper for non-unix systems if os.name == 'nt': from nturl2path import url2pathname, pathname2url @@ -1655,684 +1653,29 @@ def data_open(self, req): def url2pathname(pathname): """OS-specific conversion from a relative URL of the 'file' scheme to a file system path; not recommended for general use.""" - return unquote(pathname) + if pathname[:3] == '///': + # URL has an empty authority section, so the path begins on the + # third character. + pathname = pathname[2:] + elif pathname[:12] == '//localhost/': + # Skip past 'localhost' authority. + pathname = pathname[11:] + encoding = sys.getfilesystemencoding() + errors = sys.getfilesystemencodeerrors() + return unquote(pathname, encoding=encoding, errors=errors) def pathname2url(pathname): """OS-specific conversion from a file system path to a relative URL of the 'file' scheme; not recommended for general use.""" - return quote(pathname) - - -ftpcache = {} - - -class URLopener: - """Class to open URLs. - This is a class rather than just a subroutine because we may need - more than one set of global protocol-specific options. - Note -- this is a base class for those who don't want the - automatic handling of errors type 302 (relocated) and 401 - (authorization needed).""" - - __tempfiles = None - - version = "Python-urllib/%s" % __version__ - - # Constructor - def __init__(self, proxies=None, **x509): - msg = "%(class)s style of invoking requests is deprecated. " \ - "Use newer urlopen functions/methods" % {'class': self.__class__.__name__} - warnings.warn(msg, DeprecationWarning, stacklevel=3) - if proxies is None: - proxies = getproxies() - assert hasattr(proxies, 'keys'), "proxies must be a mapping" - self.proxies = proxies - self.key_file = x509.get('key_file') - self.cert_file = x509.get('cert_file') - self.addheaders = [('User-Agent', self.version), ('Accept', '*/*')] - self.__tempfiles = [] - self.__unlink = os.unlink # See cleanup() - self.tempcache = None - # Undocumented feature: if you assign {} to tempcache, - # it is used to cache files retrieved with - # self.retrieve(). This is not enabled by default - # since it does not work for changing documents (and I - # haven't got the logic to check expiration headers - # yet). - self.ftpcache = ftpcache - # Undocumented feature: you can use a different - # ftp cache by assigning to the .ftpcache member; - # in case you want logically independent URL openers - # XXX This is not threadsafe. Bah. - - def __del__(self): - self.close() - - def close(self): - self.cleanup() - - def cleanup(self): - # This code sometimes runs when the rest of this module - # has already been deleted, so it can't use any globals - # or import anything. - if self.__tempfiles: - for file in self.__tempfiles: - try: - self.__unlink(file) - except OSError: - pass - del self.__tempfiles[:] - if self.tempcache: - self.tempcache.clear() - - def addheader(self, *args): - """Add a header to be used by the HTTP interface only - e.g. u.addheader('Accept', 'sound/basic')""" - self.addheaders.append(args) - - # External interface - def open(self, fullurl, data=None): - """Use URLopener().open(file) instead of open(file, 'r').""" - fullurl = unwrap(_to_bytes(fullurl)) - fullurl = quote(fullurl, safe="%/:=&?~#+!$,;'@()*[]|") - if self.tempcache and fullurl in self.tempcache: - filename, headers = self.tempcache[fullurl] - fp = open(filename, 'rb') - return addinfourl(fp, headers, fullurl) - urltype, url = _splittype(fullurl) - if not urltype: - urltype = 'file' - if urltype in self.proxies: - proxy = self.proxies[urltype] - urltype, proxyhost = _splittype(proxy) - host, selector = _splithost(proxyhost) - url = (host, fullurl) # Signal special case to open_*() - else: - proxy = None - name = 'open_' + urltype - self.type = urltype - name = name.replace('-', '_') - if not hasattr(self, name) or name == 'open_local_file': - if proxy: - return self.open_unknown_proxy(proxy, fullurl, data) - else: - return self.open_unknown(fullurl, data) - try: - if data is None: - return getattr(self, name)(url) - else: - return getattr(self, name)(url, data) - except (HTTPError, URLError): - raise - except OSError as msg: - raise OSError('socket error', msg) from msg - - def open_unknown(self, fullurl, data=None): - """Overridable interface to open unknown URL type.""" - type, url = _splittype(fullurl) - raise OSError('url error', 'unknown url type', type) - - def open_unknown_proxy(self, proxy, fullurl, data=None): - """Overridable interface to open unknown URL type.""" - type, url = _splittype(fullurl) - raise OSError('url error', 'invalid proxy for %s' % type, proxy) - - # External interface - def retrieve(self, url, filename=None, reporthook=None, data=None): - """retrieve(url) returns (filename, headers) for a local object - or (tempfilename, headers) for a remote object.""" - url = unwrap(_to_bytes(url)) - if self.tempcache and url in self.tempcache: - return self.tempcache[url] - type, url1 = _splittype(url) - if filename is None and (not type or type == 'file'): - try: - fp = self.open_local_file(url1) - hdrs = fp.info() - fp.close() - return url2pathname(_splithost(url1)[1]), hdrs - except OSError: - pass - fp = self.open(url, data) - try: - headers = fp.info() - if filename: - tfp = open(filename, 'wb') - else: - garbage, path = _splittype(url) - garbage, path = _splithost(path or "") - path, garbage = _splitquery(path or "") - path, garbage = _splitattr(path or "") - suffix = os.path.splitext(path)[1] - (fd, filename) = tempfile.mkstemp(suffix) - self.__tempfiles.append(filename) - tfp = os.fdopen(fd, 'wb') - try: - result = filename, headers - if self.tempcache is not None: - self.tempcache[url] = result - bs = 1024*8 - size = -1 - read = 0 - blocknum = 0 - if "content-length" in headers: - size = int(headers["Content-Length"]) - if reporthook: - reporthook(blocknum, bs, size) - while block := fp.read(bs): - read += len(block) - tfp.write(block) - blocknum += 1 - if reporthook: - reporthook(blocknum, bs, size) - finally: - tfp.close() - finally: - fp.close() - - # raise exception if actual size does not match content-length header - if size >= 0 and read < size: - raise ContentTooShortError( - "retrieval incomplete: got only %i out of %i bytes" - % (read, size), result) - - return result - - # Each method named open_ knows how to open that type of URL - - def _open_generic_http(self, connection_factory, url, data): - """Make an HTTP connection using connection_class. - - This is an internal method that should be called from - open_http() or open_https(). - - Arguments: - - connection_factory should take a host name and return an - HTTPConnection instance. - - url is the url to retrieval or a host, relative-path pair. - - data is payload for a POST request or None. - """ - - user_passwd = None - proxy_passwd= None - if isinstance(url, str): - host, selector = _splithost(url) - if host: - user_passwd, host = _splituser(host) - host = unquote(host) - realhost = host - else: - host, selector = url - # check whether the proxy contains authorization information - proxy_passwd, host = _splituser(host) - # now we proceed with the url we want to obtain - urltype, rest = _splittype(selector) - url = rest - user_passwd = None - if urltype.lower() != 'http': - realhost = None - else: - realhost, rest = _splithost(rest) - if realhost: - user_passwd, realhost = _splituser(realhost) - if user_passwd: - selector = "%s://%s%s" % (urltype, realhost, rest) - if proxy_bypass(realhost): - host = realhost - - if not host: raise OSError('http error', 'no host given') - - if proxy_passwd: - proxy_passwd = unquote(proxy_passwd) - proxy_auth = base64.b64encode(proxy_passwd.encode()).decode('ascii') - else: - proxy_auth = None - - if user_passwd: - user_passwd = unquote(user_passwd) - auth = base64.b64encode(user_passwd.encode()).decode('ascii') - else: - auth = None - http_conn = connection_factory(host) - headers = {} - if proxy_auth: - headers["Proxy-Authorization"] = "Basic %s" % proxy_auth - if auth: - headers["Authorization"] = "Basic %s" % auth - if realhost: - headers["Host"] = realhost - - # Add Connection:close as we don't support persistent connections yet. - # This helps in closing the socket and avoiding ResourceWarning - - headers["Connection"] = "close" - - for header, value in self.addheaders: - headers[header] = value - - if data is not None: - headers["Content-Type"] = "application/x-www-form-urlencoded" - http_conn.request("POST", selector, data, headers) - else: - http_conn.request("GET", selector, headers=headers) - - try: - response = http_conn.getresponse() - except http.client.BadStatusLine: - # something went wrong with the HTTP status line - raise URLError("http protocol error: bad status line") - - # According to RFC 2616, "2xx" code indicates that the client's - # request was successfully received, understood, and accepted. - if 200 <= response.status < 300: - return addinfourl(response, response.msg, "http:" + url, - response.status) - else: - return self.http_error( - url, response.fp, - response.status, response.reason, response.msg, data) - - def open_http(self, url, data=None): - """Use HTTP protocol.""" - return self._open_generic_http(http.client.HTTPConnection, url, data) - - def http_error(self, url, fp, errcode, errmsg, headers, data=None): - """Handle http errors. - - Derived class can override this, or provide specific handlers - named http_error_DDD where DDD is the 3-digit error code.""" - # First check if there's a specific handler for this error - name = 'http_error_%d' % errcode - if hasattr(self, name): - method = getattr(self, name) - if data is None: - result = method(url, fp, errcode, errmsg, headers) - else: - result = method(url, fp, errcode, errmsg, headers, data) - if result: return result - return self.http_error_default(url, fp, errcode, errmsg, headers) - - def http_error_default(self, url, fp, errcode, errmsg, headers): - """Default error handler: close the connection and raise OSError.""" - fp.close() - raise HTTPError(url, errcode, errmsg, headers, None) - - if _have_ssl: - def _https_connection(self, host): - if self.key_file or self.cert_file: - http_version = http.client.HTTPSConnection._http_vsn - context = http.client._create_https_context(http_version) - context.load_cert_chain(self.cert_file, self.key_file) - # cert and key file means the user wants to authenticate. - # enable TLS 1.3 PHA implicitly even for custom contexts. - if context.post_handshake_auth is not None: - context.post_handshake_auth = True - else: - context = None - return http.client.HTTPSConnection(host, context=context) - - def open_https(self, url, data=None): - """Use HTTPS protocol.""" - return self._open_generic_http(self._https_connection, url, data) - - def open_file(self, url): - """Use local file or FTP depending on form of URL.""" - if not isinstance(url, str): - raise URLError('file error: proxy support for file protocol currently not implemented') - if url[:2] == '//' and url[2:3] != '/' and url[2:12].lower() != 'localhost/': - raise ValueError("file:// scheme is supported only on localhost") - else: - return self.open_local_file(url) - - def open_local_file(self, url): - """Use local file.""" - import email.utils - import mimetypes - host, file = _splithost(url) - localname = url2pathname(file) - try: - stats = os.stat(localname) - except OSError as e: - raise URLError(e.strerror, e.filename) - size = stats.st_size - modified = email.utils.formatdate(stats.st_mtime, usegmt=True) - mtype = mimetypes.guess_type(url)[0] - headers = email.message_from_string( - 'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' % - (mtype or 'text/plain', size, modified)) - if not host: - urlfile = file - if file[:1] == '/': - urlfile = 'file://' + file - return addinfourl(open(localname, 'rb'), headers, urlfile) - host, port = _splitport(host) - if (not port - and socket.gethostbyname(host) in ((localhost(),) + thishost())): - urlfile = file - if file[:1] == '/': - urlfile = 'file://' + file - elif file[:2] == './': - raise ValueError("local file url may start with / or file:. Unknown url of type: %s" % url) - return addinfourl(open(localname, 'rb'), headers, urlfile) - raise URLError('local file error: not on local host') - - def open_ftp(self, url): - """Use FTP protocol.""" - if not isinstance(url, str): - raise URLError('ftp error: proxy support for ftp protocol currently not implemented') - import mimetypes - host, path = _splithost(url) - if not host: raise URLError('ftp error: no host given') - host, port = _splitport(host) - user, host = _splituser(host) - if user: user, passwd = _splitpasswd(user) - else: passwd = None - host = unquote(host) - user = unquote(user or '') - passwd = unquote(passwd or '') - host = socket.gethostbyname(host) - if not port: - import ftplib - port = ftplib.FTP_PORT - else: - port = int(port) - path, attrs = _splitattr(path) - path = unquote(path) - dirs = path.split('/') - dirs, file = dirs[:-1], dirs[-1] - if dirs and not dirs[0]: dirs = dirs[1:] - if dirs and not dirs[0]: dirs[0] = '/' - key = user, host, port, '/'.join(dirs) - # XXX thread unsafe! - if len(self.ftpcache) > MAXFTPCACHE: - # Prune the cache, rather arbitrarily - for k in list(self.ftpcache): - if k != key: - v = self.ftpcache[k] - del self.ftpcache[k] - v.close() - try: - if key not in self.ftpcache: - self.ftpcache[key] = \ - ftpwrapper(user, passwd, host, port, dirs) - if not file: type = 'D' - else: type = 'I' - for attr in attrs: - attr, value = _splitvalue(attr) - if attr.lower() == 'type' and \ - value in ('a', 'A', 'i', 'I', 'd', 'D'): - type = value.upper() - (fp, retrlen) = self.ftpcache[key].retrfile(file, type) - mtype = mimetypes.guess_type("ftp:" + url)[0] - headers = "" - if mtype: - headers += "Content-Type: %s\n" % mtype - if retrlen is not None and retrlen >= 0: - headers += "Content-Length: %d\n" % retrlen - headers = email.message_from_string(headers) - return addinfourl(fp, headers, "ftp:" + url) - except ftperrors() as exp: - raise URLError(f'ftp error: {exp}') from exp - - def open_data(self, url, data=None): - """Use "data" URL.""" - if not isinstance(url, str): - raise URLError('data error: proxy support for data protocol currently not implemented') - # ignore POSTed data - # - # syntax of data URLs: - # dataurl := "data:" [ mediatype ] [ ";base64" ] "," data - # mediatype := [ type "/" subtype ] *( ";" parameter ) - # data := *urlchar - # parameter := attribute "=" value - try: - [type, data] = url.split(',', 1) - except ValueError: - raise OSError('data error', 'bad data URL') - if not type: - type = 'text/plain;charset=US-ASCII' - semi = type.rfind(';') - if semi >= 0 and '=' not in type[semi:]: - encoding = type[semi+1:] - type = type[:semi] - else: - encoding = '' - msg = [] - msg.append('Date: %s'%time.strftime('%a, %d %b %Y %H:%M:%S GMT', - time.gmtime(time.time()))) - msg.append('Content-type: %s' % type) - if encoding == 'base64': - # XXX is this encoding/decoding ok? - data = base64.decodebytes(data.encode('ascii')).decode('latin-1') - else: - data = unquote(data) - msg.append('Content-Length: %d' % len(data)) - msg.append('') - msg.append(data) - msg = '\n'.join(msg) - headers = email.message_from_string(msg) - f = io.StringIO(msg) - #f.fileno = None # needed for addinfourl - return addinfourl(f, headers, url) - - -class FancyURLopener(URLopener): - """Derived class with handlers for errors we can handle (perhaps).""" - - def __init__(self, *args, **kwargs): - URLopener.__init__(self, *args, **kwargs) - self.auth_cache = {} - self.tries = 0 - self.maxtries = 10 - - def http_error_default(self, url, fp, errcode, errmsg, headers): - """Default error handling -- don't raise an exception.""" - return addinfourl(fp, headers, "http:" + url, errcode) - - def http_error_302(self, url, fp, errcode, errmsg, headers, data=None): - """Error 302 -- relocated (temporarily).""" - self.tries += 1 - try: - if self.maxtries and self.tries >= self.maxtries: - if hasattr(self, "http_error_500"): - meth = self.http_error_500 - else: - meth = self.http_error_default - return meth(url, fp, 500, - "Internal Server Error: Redirect Recursion", - headers) - result = self.redirect_internal(url, fp, errcode, errmsg, - headers, data) - return result - finally: - self.tries = 0 - - def redirect_internal(self, url, fp, errcode, errmsg, headers, data): - if 'location' in headers: - newurl = headers['location'] - elif 'uri' in headers: - newurl = headers['uri'] - else: - return - fp.close() - - # In case the server sent a relative URL, join with original: - newurl = urljoin(self.type + ":" + url, newurl) - - urlparts = urlparse(newurl) - - # For security reasons, we don't allow redirection to anything other - # than http, https and ftp. - - # We are using newer HTTPError with older redirect_internal method - # This older method will get deprecated in 3.3 - - if urlparts.scheme not in ('http', 'https', 'ftp', ''): - raise HTTPError(newurl, errcode, - errmsg + - " Redirection to url '%s' is not allowed." % newurl, - headers, fp) - - return self.open(newurl) - - def http_error_301(self, url, fp, errcode, errmsg, headers, data=None): - """Error 301 -- also relocated (permanently).""" - return self.http_error_302(url, fp, errcode, errmsg, headers, data) - - def http_error_303(self, url, fp, errcode, errmsg, headers, data=None): - """Error 303 -- also relocated (essentially identical to 302).""" - return self.http_error_302(url, fp, errcode, errmsg, headers, data) - - def http_error_307(self, url, fp, errcode, errmsg, headers, data=None): - """Error 307 -- relocated, but turn POST into error.""" - if data is None: - return self.http_error_302(url, fp, errcode, errmsg, headers, data) - else: - return self.http_error_default(url, fp, errcode, errmsg, headers) - - def http_error_308(self, url, fp, errcode, errmsg, headers, data=None): - """Error 308 -- relocated, but turn POST into error.""" - if data is None: - return self.http_error_301(url, fp, errcode, errmsg, headers, data) - else: - return self.http_error_default(url, fp, errcode, errmsg, headers) - - def http_error_401(self, url, fp, errcode, errmsg, headers, data=None, - retry=False): - """Error 401 -- authentication required. - This function supports Basic authentication only.""" - if 'www-authenticate' not in headers: - URLopener.http_error_default(self, url, fp, - errcode, errmsg, headers) - stuff = headers['www-authenticate'] - match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff) - if not match: - URLopener.http_error_default(self, url, fp, - errcode, errmsg, headers) - scheme, realm = match.groups() - if scheme.lower() != 'basic': - URLopener.http_error_default(self, url, fp, - errcode, errmsg, headers) - if not retry: - URLopener.http_error_default(self, url, fp, errcode, errmsg, - headers) - name = 'retry_' + self.type + '_basic_auth' - if data is None: - return getattr(self,name)(url, realm) - else: - return getattr(self,name)(url, realm, data) - - def http_error_407(self, url, fp, errcode, errmsg, headers, data=None, - retry=False): - """Error 407 -- proxy authentication required. - This function supports Basic authentication only.""" - if 'proxy-authenticate' not in headers: - URLopener.http_error_default(self, url, fp, - errcode, errmsg, headers) - stuff = headers['proxy-authenticate'] - match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff) - if not match: - URLopener.http_error_default(self, url, fp, - errcode, errmsg, headers) - scheme, realm = match.groups() - if scheme.lower() != 'basic': - URLopener.http_error_default(self, url, fp, - errcode, errmsg, headers) - if not retry: - URLopener.http_error_default(self, url, fp, errcode, errmsg, - headers) - name = 'retry_proxy_' + self.type + '_basic_auth' - if data is None: - return getattr(self,name)(url, realm) - else: - return getattr(self,name)(url, realm, data) - - def retry_proxy_http_basic_auth(self, url, realm, data=None): - host, selector = _splithost(url) - newurl = 'http://' + host + selector - proxy = self.proxies['http'] - urltype, proxyhost = _splittype(proxy) - proxyhost, proxyselector = _splithost(proxyhost) - i = proxyhost.find('@') + 1 - proxyhost = proxyhost[i:] - user, passwd = self.get_user_passwd(proxyhost, realm, i) - if not (user or passwd): return None - proxyhost = "%s:%s@%s" % (quote(user, safe=''), - quote(passwd, safe=''), proxyhost) - self.proxies['http'] = 'http://' + proxyhost + proxyselector - if data is None: - return self.open(newurl) - else: - return self.open(newurl, data) - - def retry_proxy_https_basic_auth(self, url, realm, data=None): - host, selector = _splithost(url) - newurl = 'https://' + host + selector - proxy = self.proxies['https'] - urltype, proxyhost = _splittype(proxy) - proxyhost, proxyselector = _splithost(proxyhost) - i = proxyhost.find('@') + 1 - proxyhost = proxyhost[i:] - user, passwd = self.get_user_passwd(proxyhost, realm, i) - if not (user or passwd): return None - proxyhost = "%s:%s@%s" % (quote(user, safe=''), - quote(passwd, safe=''), proxyhost) - self.proxies['https'] = 'https://' + proxyhost + proxyselector - if data is None: - return self.open(newurl) - else: - return self.open(newurl, data) - - def retry_http_basic_auth(self, url, realm, data=None): - host, selector = _splithost(url) - i = host.find('@') + 1 - host = host[i:] - user, passwd = self.get_user_passwd(host, realm, i) - if not (user or passwd): return None - host = "%s:%s@%s" % (quote(user, safe=''), - quote(passwd, safe=''), host) - newurl = 'http://' + host + selector - if data is None: - return self.open(newurl) - else: - return self.open(newurl, data) - - def retry_https_basic_auth(self, url, realm, data=None): - host, selector = _splithost(url) - i = host.find('@') + 1 - host = host[i:] - user, passwd = self.get_user_passwd(host, realm, i) - if not (user or passwd): return None - host = "%s:%s@%s" % (quote(user, safe=''), - quote(passwd, safe=''), host) - newurl = 'https://' + host + selector - if data is None: - return self.open(newurl) - else: - return self.open(newurl, data) - - def get_user_passwd(self, host, realm, clear_cache=0): - key = realm + '@' + host.lower() - if key in self.auth_cache: - if clear_cache: - del self.auth_cache[key] - else: - return self.auth_cache[key] - user, passwd = self.prompt_user_passwd(host, realm) - if user or passwd: self.auth_cache[key] = (user, passwd) - return user, passwd - - def prompt_user_passwd(self, host, realm): - """Override this in a GUI environment!""" - import getpass - try: - user = input("Enter username for %s at %s: " % (realm, host)) - passwd = getpass.getpass("Enter password for %s in %s at %s: " % - (user, realm, host)) - return user, passwd - except KeyboardInterrupt: - print() - return None, None + if pathname[:1] == '/': + # Add explicitly empty authority to absolute path. If the path + # starts with exactly one slash then this change is mostly + # cosmetic, but if it begins with two or more slashes then this + # avoids interpreting the path as a URL authority. + pathname = '//' + pathname + encoding = sys.getfilesystemencoding() + errors = sys.getfilesystemencodeerrors() + return quote(pathname, encoding=encoding, errors=errors) # Utility functions @@ -2480,9 +1823,7 @@ def getproxies_environment(): """Return a dictionary of scheme -> proxy server URL mappings. Scan the environment for variables named _proxy; - this seems to be the standard convention. If you need a - different way, you can pass a proxies dictionary to the - [Fancy]URLopener constructor. + this seems to be the standard convention. """ # in order to prefer lowercase variables, process environment in # two passes: first matches any, second pass matches lowercase only diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index c58565e3945146..409f2b2e48de6e 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -11,6 +11,7 @@ """ import collections +import urllib.error import urllib.parse import urllib.request @@ -65,6 +66,7 @@ def read(self): self.disallow_all = True elif err.code >= 400 and err.code < 500: self.allow_all = True + err.close() else: raw = f.read() self.parse(raw.decode("utf-8").splitlines()) diff --git a/Lib/uuid.py b/Lib/uuid.py index 470bc0d68597ab..36809b85cb8ceb 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -1,8 +1,8 @@ -r"""UUID objects (universally unique identifiers) according to RFC 4122. +r"""UUID objects (universally unique identifiers) according to RFC 4122/9562. This module provides immutable UUID objects (class UUID) and the functions -uuid1(), uuid3(), uuid4(), uuid5() for generating version 1, 3, 4, and 5 -UUIDs as specified in RFC 4122. +uuid1(), uuid3(), uuid4(), uuid5(), and uuid8() for generating version 1, 3, +4, 5, and 8 UUIDs as specified in RFC 4122/9562. If all you want is a unique ID, you should probably call uuid1() or uuid4(). Note that uuid1() may compromise privacy since it creates a UUID containing @@ -42,6 +42,14 @@ # make a UUID from a 16-byte string >>> uuid.UUID(bytes=x.bytes) UUID('00010203-0405-0607-0809-0a0b0c0d0e0f') + + # get the Nil UUID + >>> uuid.NIL + UUID('00000000-0000-0000-0000-000000000000') + + # get the Max UUID + >>> uuid.MAX + UUID('ffffffff-ffff-ffff-ffff-ffffffffffff') """ import os @@ -53,13 +61,16 @@ __author__ = 'Ka-Ping Yee ' # The recognized platforms - known behaviors -if sys.platform in ('win32', 'darwin', 'emscripten', 'wasi'): +if sys.platform in {'win32', 'darwin', 'emscripten', 'wasi'}: _AIX = _LINUX = False +elif sys.platform == 'linux': + _LINUX = True + _AIX = False else: import platform _platform_system = platform.system() _AIX = _platform_system == 'AIX' - _LINUX = _platform_system == 'Linux' + _LINUX = _platform_system in ('Linux', 'Android') _MAC_DELIM = b':' _MAC_OMITS_LEADING_ZEROES = False @@ -82,6 +93,17 @@ class SafeUUID: unknown = None +_UINT_128_MAX = (1 << 128) - 1 +# 128-bit mask to clear the variant and version bits of a UUID integral value +_RFC_4122_CLEARFLAGS_MASK = ~((0xf000 << 64) | (0xc000 << 48)) +# RFC 4122 variant bits and version bits to activate on a UUID integral value. +_RFC_4122_VERSION_1_FLAGS = ((1 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_3_FLAGS = ((3 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_4_FLAGS = ((4 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_5_FLAGS = ((5 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_8_FLAGS = ((8 << 76) | (0x8000 << 48)) + + class UUID: """Instances of the UUID class represent UUIDs as specified in RFC 4122. UUID objects are immutable, hashable, and usable as dictionary keys. @@ -121,12 +143,12 @@ class UUID: int the UUID as a 128-bit integer - urn the UUID as a URN as specified in RFC 4122 + urn the UUID as a URN as specified in RFC 4122/9562 variant the UUID variant (one of the constants RESERVED_NCS, RFC_4122, RESERVED_MICROSOFT, or RESERVED_FUTURE) - version the UUID version number (1 through 5, meaningful only + version the UUID version number (1 through 8, meaningful only when the variant is RFC_4122) is_safe An enum indicating whether the UUID has been generated in @@ -171,57 +193,69 @@ def __init__(self, hex=None, bytes=None, bytes_le=None, fields=None, if [hex, bytes, bytes_le, fields, int].count(None) != 4: raise TypeError('one of the hex, bytes, bytes_le, fields, ' 'or int arguments must be given') - if hex is not None: + if int is not None: + pass + elif hex is not None: hex = hex.replace('urn:', '').replace('uuid:', '') hex = hex.strip('{}').replace('-', '') if len(hex) != 32: raise ValueError('badly formed hexadecimal UUID string') int = int_(hex, 16) - if bytes_le is not None: + elif bytes_le is not None: if len(bytes_le) != 16: raise ValueError('bytes_le is not a 16-char string') + assert isinstance(bytes_le, bytes_), repr(bytes_le) bytes = (bytes_le[4-1::-1] + bytes_le[6-1:4-1:-1] + bytes_le[8-1:6-1:-1] + bytes_le[8:]) - if bytes is not None: + int = int_.from_bytes(bytes) # big endian + elif bytes is not None: if len(bytes) != 16: raise ValueError('bytes is not a 16-char string') assert isinstance(bytes, bytes_), repr(bytes) int = int_.from_bytes(bytes) # big endian - if fields is not None: + elif fields is not None: if len(fields) != 6: raise ValueError('fields is not a 6-tuple') (time_low, time_mid, time_hi_version, clock_seq_hi_variant, clock_seq_low, node) = fields - if not 0 <= time_low < 1<<32: + if not 0 <= time_low < (1 << 32): raise ValueError('field 1 out of range (need a 32-bit value)') - if not 0 <= time_mid < 1<<16: + if not 0 <= time_mid < (1 << 16): raise ValueError('field 2 out of range (need a 16-bit value)') - if not 0 <= time_hi_version < 1<<16: + if not 0 <= time_hi_version < (1 << 16): raise ValueError('field 3 out of range (need a 16-bit value)') - if not 0 <= clock_seq_hi_variant < 1<<8: + if not 0 <= clock_seq_hi_variant < (1 << 8): raise ValueError('field 4 out of range (need an 8-bit value)') - if not 0 <= clock_seq_low < 1<<8: + if not 0 <= clock_seq_low < (1 << 8): raise ValueError('field 5 out of range (need an 8-bit value)') - if not 0 <= node < 1<<48: + if not 0 <= node < (1 << 48): raise ValueError('field 6 out of range (need a 48-bit value)') clock_seq = (clock_seq_hi_variant << 8) | clock_seq_low int = ((time_low << 96) | (time_mid << 80) | (time_hi_version << 64) | (clock_seq << 48) | node) - if int is not None: - if not 0 <= int < 1<<128: - raise ValueError('int is out of range (need a 128-bit value)') + if not 0 <= int <= _UINT_128_MAX: + raise ValueError('int is out of range (need a 128-bit value)') if version is not None: - if not 1 <= version <= 5: + if not 1 <= version <= 8: raise ValueError('illegal version number') - # Set the variant to RFC 4122. - int &= ~(0xc000 << 48) - int |= 0x8000 << 48 + # clear the variant and the version number bits + int &= _RFC_4122_CLEARFLAGS_MASK + # Set the variant to RFC 4122/9562. + int |= 0x8000_0000_0000_0000 # (0x8000 << 48) # Set the version number. - int &= ~(0xf000 << 64) int |= version << 76 object.__setattr__(self, 'int', int) object.__setattr__(self, 'is_safe', is_safe) + @classmethod + def _from_int(cls, value): + """Create a UUID from an integer *value*. Internal use only.""" + assert 0 <= value <= _UINT_128_MAX, repr(value) + self = object.__new__(cls) + object.__setattr__(self, 'int', value) + object.__setattr__(self, 'is_safe', SafeUUID.unknown) + return self + def __getstate__(self): d = {'int': self.int} if self.is_safe != SafeUUID.unknown: @@ -352,7 +386,7 @@ def variant(self): @property def version(self): - # The version bits are only meaningful for RFC 4122 UUIDs. + # The version bits are only meaningful for RFC 4122/9562 UUIDs. if self.variant == RFC_4122: return int((self.int >> 76) & 0xf) @@ -371,7 +405,7 @@ def _get_command_stdout(command, *args): # for are actually localized, but in theory some system could do so.) env = dict(os.environ) env['LC_ALL'] = 'C' - # Empty strings will be quoted by popen so we should just ommit it + # Empty strings will be quoted by popen so we should just omit it if args != ('',): command = (executable, *args) else: @@ -564,32 +598,16 @@ def _netstat_getnode(): # This works on AIX and might work on Tru64 UNIX. return _find_mac_under_heading('netstat', '-ian', b'Address') -def _ipconfig_getnode(): - """[DEPRECATED] Get the hardware address on Windows.""" - # bpo-40501: UuidCreateSequential() is now the only supported approach - return _windll_getnode() - -def _netbios_getnode(): - """[DEPRECATED] Get the hardware address on Windows.""" - # bpo-40501: UuidCreateSequential() is now the only supported approach - return _windll_getnode() - # Import optional C extension at toplevel, to help disabling it when testing try: import _uuid _generate_time_safe = getattr(_uuid, "generate_time_safe", None) _UuidCreate = getattr(_uuid, "UuidCreate", None) - _has_uuid_generate_time_safe = _uuid.has_uuid_generate_time_safe except ImportError: _uuid = None _generate_time_safe = None _UuidCreate = None - _has_uuid_generate_time_safe = None - - -def _load_system_functions(): - """[DEPRECATED] Platform-specific functions loaded at import time""" def _unix_getnode(): @@ -713,25 +731,55 @@ def uuid3(namespace, name): """Generate a UUID from the MD5 hash of a namespace UUID and a name.""" if isinstance(name, str): name = bytes(name, "utf-8") - from hashlib import md5 - digest = md5( - namespace.bytes + name, - usedforsecurity=False - ).digest() - return UUID(bytes=digest[:16], version=3) + import hashlib + h = hashlib.md5(namespace.bytes + name, usedforsecurity=False) + int_uuid_3 = int.from_bytes(h.digest()) + int_uuid_3 &= _RFC_4122_CLEARFLAGS_MASK + int_uuid_3 |= _RFC_4122_VERSION_3_FLAGS + return UUID._from_int(int_uuid_3) def uuid4(): """Generate a random UUID.""" - return UUID(bytes=os.urandom(16), version=4) + int_uuid_4 = int.from_bytes(os.urandom(16)) + int_uuid_4 &= _RFC_4122_CLEARFLAGS_MASK + int_uuid_4 |= _RFC_4122_VERSION_4_FLAGS + return UUID._from_int(int_uuid_4) def uuid5(namespace, name): """Generate a UUID from the SHA-1 hash of a namespace UUID and a name.""" if isinstance(name, str): name = bytes(name, "utf-8") - from hashlib import sha1 - hash = sha1(namespace.bytes + name).digest() - return UUID(bytes=hash[:16], version=5) + import hashlib + h = hashlib.sha1(namespace.bytes + name, usedforsecurity=False) + int_uuid_5 = int.from_bytes(h.digest()[:16]) + int_uuid_5 &= _RFC_4122_CLEARFLAGS_MASK + int_uuid_5 |= _RFC_4122_VERSION_5_FLAGS + return UUID._from_int(int_uuid_5) + +def uuid8(a=None, b=None, c=None): + """Generate a UUID from three custom blocks. + * 'a' is the first 48-bit chunk of the UUID (octets 0-5); + * 'b' is the mid 12-bit chunk (octets 6-7); + * 'c' is the last 62-bit chunk (octets 8-15). + + When a value is not specified, a pseudo-random value is generated. + """ + if a is None: + import random + a = random.getrandbits(48) + if b is None: + import random + b = random.getrandbits(12) + if c is None: + import random + c = random.getrandbits(62) + int_uuid_8 = (a & 0xffff_ffff_ffff) << 80 + int_uuid_8 |= (b & 0xfff) << 64 + int_uuid_8 |= c & 0x3fff_ffff_ffff_ffff + # by construction, the variant and version bits are already cleared + int_uuid_8 |= _RFC_4122_VERSION_8_FLAGS + return UUID._from_int(int_uuid_8) def main(): """Run the uuid command line interface.""" @@ -739,7 +787,8 @@ def main(): "uuid1": uuid1, "uuid3": uuid3, "uuid4": uuid4, - "uuid5": uuid5 + "uuid5": uuid5, + "uuid8": uuid8, } uuid_namespace_funcs = ("uuid3", "uuid5") namespaces = { @@ -789,5 +838,10 @@ def main(): NAMESPACE_OID = UUID('6ba7b812-9dad-11d1-80b4-00c04fd430c8') NAMESPACE_X500 = UUID('6ba7b814-9dad-11d1-80b4-00c04fd430c8') +# RFC 9562 Sections 5.9 and 5.10 define the special Nil and Max UUID formats. + +NIL = UUID('00000000-0000-0000-0000-000000000000') +MAX = UUID('ffffffff-ffff-ffff-ffff-ffffffffffff') + if __name__ == "__main__": main() diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 4856594755ae57..dc4c9ef3531991 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -11,6 +11,7 @@ import sys import sysconfig import types +import shlex CORE_VENV_DEPS = ('pip',) @@ -102,11 +103,36 @@ def _venv_path(self, env_dir, name): vars = { 'base': env_dir, 'platbase': env_dir, - 'installed_base': env_dir, - 'installed_platbase': env_dir, } return sysconfig.get_path(name, scheme='venv', vars=vars) + @classmethod + def _same_path(cls, path1, path2): + """Check whether two paths appear the same. + + Whether they refer to the same file is irrelevant; we're testing for + whether a human reader would look at the path string and easily tell + that they're the same file. + """ + if sys.platform == 'win32': + if os.path.normcase(path1) == os.path.normcase(path2): + return True + # gh-90329: Don't display a warning for short/long names + import _winapi + try: + path1 = _winapi.GetLongPathName(os.fsdecode(path1)) + except OSError: + pass + try: + path2 = _winapi.GetLongPathName(os.fsdecode(path2)) + except OSError: + pass + if os.path.normcase(path1) == os.path.normcase(path2): + return True + return False + else: + return path1 == path2 + def ensure_directories(self, env_dir): """ Create the directories for the environment. @@ -147,9 +173,20 @@ def create_if_needed(d): context.python_dir = dirname context.python_exe = exename binpath = self._venv_path(env_dir, 'scripts') - incpath = self._venv_path(env_dir, 'include') libpath = self._venv_path(env_dir, 'purelib') + # PEP 405 says venvs should create a local include directory. + # See https://peps.python.org/pep-0405/#include-files + # XXX: This directory is not exposed in sysconfig or anywhere else, and + # doesn't seem to be utilized by modern packaging tools. We keep it + # for backwards-compatibility, and to follow the PEP, but I would + # recommend against using it, as most tooling does not pass it to + # compilers. Instead, until we standardize a site-specific include + # directory, I would recommend installing headers as package data, + # and providing some sort of API to get the include directories. + # Example: https://numpy.org/doc/2.1/reference/generated/numpy.get_include.html + incpath = os.path.join(env_dir, 'Include' if os.name == 'nt' else 'include') + context.inc_path = incpath create_if_needed(incpath) context.lib_path = libpath @@ -171,7 +208,7 @@ def create_if_needed(d): # bpo-45337: Fix up env_exec_cmd to account for file system redirections. # Some redirects only apply to CreateFile and not CreateProcess real_env_exe = os.path.realpath(context.env_exe) - if os.path.normcase(real_env_exe) != os.path.normcase(context.env_exe): + if not self._same_path(real_env_exe, context.env_exe): logger.warning('Actual environment location may have moved due to ' 'redirects, links or junctions.\n' ' Requested location: "%s"\n' @@ -276,8 +313,11 @@ def setup_python(self, context): copier(context.executable, path) if not os.path.islink(path): os.chmod(path, 0o755) - for suffix in ('python', 'python3', - f'python3.{sys.version_info[1]}'): + + suffixes = ['python', 'python3', f'python3.{sys.version_info[1]}'] + if sys.version_info[:2] == (3, 14): + suffixes.append('𝜋thon') + for suffix in suffixes: path = os.path.join(binpath, suffix) if not os.path.exists(path): # Issue 18807: make copies if @@ -366,7 +406,7 @@ def setup_python(self, context): os.symlink(src, dest) to_unlink.append(dest) except OSError: - logger.warning('Unable to symlink %r to %r', src, dst) + logger.warning('Unable to symlink %r to %r', src, dest) do_copies = True for f in to_unlink: try: @@ -454,11 +494,41 @@ def replace_variables(self, text, context): :param context: The information for the environment creation request being processed. """ - text = text.replace('__VENV_DIR__', context.env_dir) - text = text.replace('__VENV_NAME__', context.env_name) - text = text.replace('__VENV_PROMPT__', context.prompt) - text = text.replace('__VENV_BIN_NAME__', context.bin_name) - text = text.replace('__VENV_PYTHON__', context.env_exe) + replacements = { + '__VENV_DIR__': context.env_dir, + '__VENV_NAME__': context.env_name, + '__VENV_PROMPT__': context.prompt, + '__VENV_BIN_NAME__': context.bin_name, + '__VENV_PYTHON__': context.env_exe, + } + + def quote_ps1(s): + """ + This should satisfy PowerShell quoting rules [1], unless the quoted + string is passed directly to Windows native commands [2]. + [1]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_quoting_rules + [2]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_parsing#passing-arguments-that-contain-quote-characters + """ + s = s.replace("'", "''") + return f"'{s}'" + + def quote_bat(s): + return s + + # gh-124651: need to quote the template strings properly + quote = shlex.quote + script_path = context.script_path + if script_path.endswith('.ps1'): + quote = quote_ps1 + elif script_path.endswith('.bat'): + quote = quote_bat + else: + # fallbacks to POSIX shell compliant quote + quote = shlex.quote + + replacements = {key: quote(s) for key, s in replacements.items()} + for key, quoted in replacements.items(): + text = text.replace(key, quoted) return text def install_scripts(self, context, path): @@ -508,6 +578,7 @@ def skip_file(f): with open(srcfile, 'rb') as f: data = f.read() try: + context.script_path = srcfile new_data = ( self.replace_variables(data.decode('utf-8'), context) .encode('utf-8') @@ -545,8 +616,7 @@ def create(env_dir, system_site_packages=False, clear=False, def main(args=None): import argparse - parser = argparse.ArgumentParser(prog=__name__, - description='Creates virtual Python ' + parser = argparse.ArgumentParser(description='Creates virtual Python ' 'environments in one or ' 'more target ' 'directories.', diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index cbd4873f012246..70673a265d41f8 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -14,8 +14,9 @@ deactivate () { unset _OLD_VIRTUAL_PYTHONHOME fi - # Call hash to forget past commands. Without forgetting - # past commands the $PATH changes we made may not be respected + # Call hash to forget past locations. Without forgetting + # past locations the $PATH changes we made may not be respected. + # See "man bash" for more details. hash is usually a builtin of your shell hash -r 2> /dev/null if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then @@ -37,23 +38,23 @@ deactivate nondestructive # on Windows, a path can contain colons and backslashes and has to be converted: case "$(uname)" in - CYGWIN*|MSYS*) - # transform D:\path\to\venv to /d/path/to/venv on MSYS + CYGWIN*|MSYS*|MINGW*) + # transform D:\path\to\venv to /d/path/to/venv on MSYS and MINGW # and to /cygdrive/d/path/to/venv on Cygwin - VIRTUAL_ENV=$(cygpath "__VENV_DIR__") + VIRTUAL_ENV=$(cygpath __VENV_DIR__) export VIRTUAL_ENV ;; *) # use the path as-is - export VIRTUAL_ENV="__VENV_DIR__" + export VIRTUAL_ENV=__VENV_DIR__ ;; esac _OLD_VIRTUAL_PATH="$PATH" -PATH="$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" +PATH="$VIRTUAL_ENV/"__VENV_BIN_NAME__":$PATH" export PATH -VIRTUAL_ENV_PROMPT="__VENV_PROMPT__" +VIRTUAL_ENV_PROMPT=__VENV_PROMPT__ export VIRTUAL_ENV_PROMPT # unset PYTHONHOME if set @@ -66,7 +67,7 @@ fi if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then _OLD_VIRTUAL_PS1="${PS1:-}" - PS1="(__VENV_PROMPT__) ${PS1:-}" + PS1="("__VENV_PROMPT__") ${PS1:-}" export PS1 fi diff --git a/Lib/venv/scripts/posix/activate.fish b/Lib/venv/scripts/common/activate.fish similarity index 90% rename from Lib/venv/scripts/posix/activate.fish rename to Lib/venv/scripts/common/activate.fish index 25c42756789bbc..284a7469c99b57 100644 --- a/Lib/venv/scripts/posix/activate.fish +++ b/Lib/venv/scripts/common/activate.fish @@ -33,11 +33,11 @@ end # Unset irrelevant variables. deactivate nondestructive -set -gx VIRTUAL_ENV "__VENV_DIR__" +set -gx VIRTUAL_ENV __VENV_DIR__ set -gx _OLD_VIRTUAL_PATH $PATH -set -gx PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__" $PATH -set -gx VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" +set -gx PATH "$VIRTUAL_ENV/"__VENV_BIN_NAME__ $PATH +set -gx VIRTUAL_ENV_PROMPT __VENV_PROMPT__ # Unset PYTHONHOME if set. if set -q PYTHONHOME @@ -57,7 +57,7 @@ if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" set -l old_status $status # Output the venv prompt; color taken from the blue of the Python logo. - printf "%s(%s)%s " (set_color 4B8BBE) "__VENV_PROMPT__" (set_color normal) + printf "%s(%s)%s " (set_color 4B8BBE) __VENV_PROMPT__ (set_color normal) # Restore the return status of the previous command. echo "exit $old_status" | . diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat index dd5ea8eb67b90a..35533e4b551155 100644 --- a/Lib/venv/scripts/nt/activate.bat +++ b/Lib/venv/scripts/nt/activate.bat @@ -8,7 +8,7 @@ if defined _OLD_CODEPAGE ( "%SystemRoot%\System32\chcp.com" 65001 > nul ) -set VIRTUAL_ENV=__VENV_DIR__ +set "VIRTUAL_ENV=__VENV_DIR__" if not defined PROMPT set PROMPT=$P$G @@ -24,8 +24,8 @@ set PYTHONHOME= if defined _OLD_VIRTUAL_PATH set PATH=%_OLD_VIRTUAL_PATH% if not defined _OLD_VIRTUAL_PATH set _OLD_VIRTUAL_PATH=%PATH% -set PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH% -set VIRTUAL_ENV_PROMPT=__VENV_PROMPT__ +set "PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%" +set "VIRTUAL_ENV_PROMPT=__VENV_PROMPT__" :END if defined _OLD_CODEPAGE ( diff --git a/Lib/venv/scripts/posix/activate.csh b/Lib/venv/scripts/posix/activate.csh index c707f1988b0acc..2a3fa835476ab9 100644 --- a/Lib/venv/scripts/posix/activate.csh +++ b/Lib/venv/scripts/posix/activate.csh @@ -9,17 +9,17 @@ alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PA # Unset irrelevant variables. deactivate nondestructive -setenv VIRTUAL_ENV "__VENV_DIR__" +setenv VIRTUAL_ENV __VENV_DIR__ set _OLD_VIRTUAL_PATH="$PATH" -setenv PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" -setenv VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" +setenv PATH "$VIRTUAL_ENV/"__VENV_BIN_NAME__":$PATH" +setenv VIRTUAL_ENV_PROMPT __VENV_PROMPT__ set _OLD_VIRTUAL_PROMPT="$prompt" if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then - set prompt = "(__VENV_PROMPT__) $prompt" + set prompt = "("__VENV_PROMPT__") $prompt:q" endif alias pydoc python -m pydoc diff --git a/Lib/warnings.py b/Lib/warnings.py index 4ad6ad027192e8..f20b01372dd7a4 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -132,7 +132,7 @@ def filterwarnings(action, message="", category=Warning, module="", lineno=0, append=False): """Insert an entry into the list of warnings filters (at the front). - 'action' -- one of "error", "ignore", "always", "default", "module", + 'action' -- one of "error", "ignore", "always", "all", "default", "module", or "once" 'message' -- a regex that the warning message must match 'category' -- a class that the warning must be a subclass of @@ -140,7 +140,7 @@ def filterwarnings(action, message="", category=Warning, module="", lineno=0, 'lineno' -- an integer line number, 0 matches all warnings 'append' -- if true, append to the list of filters """ - if action not in {"error", "ignore", "always", "default", "module", "once"}: + if action not in {"error", "ignore", "always", "all", "default", "module", "once"}: raise ValueError(f"invalid action: {action!r}") if not isinstance(message, str): raise TypeError("message must be a string") @@ -171,13 +171,13 @@ def simplefilter(action, category=Warning, lineno=0, append=False): """Insert a simple entry into the list of warnings filters (at the front). A simple filter matches all modules and messages. - 'action' -- one of "error", "ignore", "always", "default", "module", + 'action' -- one of "error", "ignore", "always", "all", "default", "module", or "once" 'category' -- a class that the warning must be a subclass of 'lineno' -- an integer line number, 0 matches all warnings 'append' -- if true, append to the list of filters """ - if action not in {"error", "ignore", "always", "default", "module", "once"}: + if action not in {"error", "ignore", "always", "all", "default", "module", "once"}: raise ValueError(f"invalid action: {action!r}") if not isinstance(lineno, int): raise TypeError("lineno must be an int") @@ -185,24 +185,32 @@ def simplefilter(action, category=Warning, lineno=0, append=False): raise ValueError("lineno must be an int >= 0") _add_filter(action, None, category, None, lineno, append=append) +def _filters_mutated(): + # Even though this function is not part of the public API, it's used by + # a fair amount of user code. + with _lock: + _filters_mutated_lock_held() + def _add_filter(*item, append): - # Remove possible duplicate filters, so new one will be placed - # in correct place. If append=True and duplicate exists, do nothing. - if not append: - try: - filters.remove(item) - except ValueError: - pass - filters.insert(0, item) - else: - if item not in filters: - filters.append(item) - _filters_mutated() + with _lock: + if not append: + # Remove possible duplicate filters, so new one will be placed + # in correct place. If append=True and duplicate exists, do nothing. + try: + filters.remove(item) + except ValueError: + pass + filters.insert(0, item) + else: + if item not in filters: + filters.append(item) + _filters_mutated_lock_held() def resetwarnings(): """Clear the list of warning filters, so that no filters are active.""" - filters[:] = [] - _filters_mutated() + with _lock: + filters[:] = [] + _filters_mutated_lock_held() class _OptionError(Exception): """Exception used by option processing helpers.""" @@ -248,8 +256,7 @@ def _setoption(arg): def _getaction(action): if not action: return "default" - if action == "all": return "always" # Alias - for a in ('default', 'always', 'ignore', 'module', 'once', 'error'): + for a in ('default', 'always', 'all', 'ignore', 'module', 'once', 'error'): if a.startswith(action): return a raise _OptionError("invalid action: %r" % (action,)) @@ -332,8 +339,8 @@ def warn(message, category=None, stacklevel=1, source=None, raise ValueError except ValueError: globals = sys.__dict__ - filename = "sys" - lineno = 1 + filename = "" + lineno = 0 else: globals = frame.f_globals filename = frame.f_code.co_filename @@ -354,11 +361,6 @@ def warn_explicit(message, category, filename, lineno, module = filename or "" if module[-3:].lower() == ".py": module = module[:-3] # XXX What about leading pathname? - if registry is None: - registry = {} - if registry.get('version', 0) != _filters_version: - registry.clear() - registry['version'] = _filters_version if isinstance(message, Warning): text = str(message) category = message.__class__ @@ -366,52 +368,59 @@ def warn_explicit(message, category, filename, lineno, text = message message = category(message) key = (text, category, lineno) - # Quick test for common case - if registry.get(key): - return - # Search the filters - for item in filters: - action, msg, cat, mod, ln = item - if ((msg is None or msg.match(text)) and - issubclass(category, cat) and - (mod is None or mod.match(module)) and - (ln == 0 or lineno == ln)): - break - else: - action = defaultaction - # Early exit actions - if action == "ignore": - return + with _lock: + if registry is None: + registry = {} + if registry.get('version', 0) != _filters_version: + registry.clear() + registry['version'] = _filters_version + # Quick test for common case + if registry.get(key): + return + # Search the filters + for item in filters: + action, msg, cat, mod, ln = item + if ((msg is None or msg.match(text)) and + issubclass(category, cat) and + (mod is None or mod.match(module)) and + (ln == 0 or lineno == ln)): + break + else: + action = defaultaction + # Early exit actions + if action == "ignore": + return + + if action == "error": + raise message + # Other actions + if action == "once": + registry[key] = 1 + oncekey = (text, category) + if onceregistry.get(oncekey): + return + onceregistry[oncekey] = 1 + elif action in {"always", "all"}: + pass + elif action == "module": + registry[key] = 1 + altkey = (text, category, 0) + if registry.get(altkey): + return + registry[altkey] = 1 + elif action == "default": + registry[key] = 1 + else: + # Unrecognized actions are errors + raise RuntimeError( + "Unrecognized action (%r) in warnings.filters:\n %s" % + (action, item)) # Prime the linecache for formatting, in case the # "file" is actually in a zipfile or something. import linecache linecache.getlines(filename, module_globals) - if action == "error": - raise message - # Other actions - if action == "once": - registry[key] = 1 - oncekey = (text, category) - if onceregistry.get(oncekey): - return - onceregistry[oncekey] = 1 - elif action == "always": - pass - elif action == "module": - registry[key] = 1 - altkey = (text, category, 0) - if registry.get(altkey): - return - registry[altkey] = 1 - elif action == "default": - registry[key] = 1 - else: - # Unrecognized actions are errors - raise RuntimeError( - "Unrecognized action (%r) in warnings.filters:\n %s" % - (action, item)) # Print message and context msg = WarningMessage(message, category, filename, lineno, source) _showwarnmsg(msg) @@ -489,30 +498,32 @@ def __enter__(self): if self._entered: raise RuntimeError("Cannot enter %r twice" % self) self._entered = True - self._filters = self._module.filters - self._module.filters = self._filters[:] - self._module._filters_mutated() - self._showwarning = self._module.showwarning - self._showwarnmsg_impl = self._module._showwarnmsg_impl + with _lock: + self._filters = self._module.filters + self._module.filters = self._filters[:] + self._module._filters_mutated_lock_held() + self._showwarning = self._module.showwarning + self._showwarnmsg_impl = self._module._showwarnmsg_impl + if self._record: + log = [] + self._module._showwarnmsg_impl = log.append + # Reset showwarning() to the default implementation to make sure + # that _showwarnmsg() calls _showwarnmsg_impl() + self._module.showwarning = self._module._showwarning_orig + else: + log = None if self._filter is not None: simplefilter(*self._filter) - if self._record: - log = [] - self._module._showwarnmsg_impl = log.append - # Reset showwarning() to the default implementation to make sure - # that _showwarnmsg() calls _showwarnmsg_impl() - self._module.showwarning = self._module._showwarning_orig - return log - else: - return None + return log def __exit__(self, *exc_info): if not self._entered: raise RuntimeError("Cannot exit %r without entering first" % self) - self._module.filters = self._filters - self._module._filters_mutated() - self._module.showwarning = self._showwarning - self._module._showwarnmsg_impl = self._showwarnmsg_impl + with _lock: + self._module.filters = self._filters + self._module._filters_mutated_lock_held() + self._module.showwarning = self._showwarning + self._module._showwarnmsg_impl = self._showwarnmsg_impl class deprecated: @@ -629,12 +640,16 @@ def __init_subclass__(*args, **kwargs): return arg elif callable(arg): import functools + import inspect @functools.wraps(arg) def wrapper(*args, **kwargs): warn(msg, category=category, stacklevel=stacklevel + 1) return arg(*args, **kwargs) + if inspect.iscoroutinefunction(arg): + wrapper = inspect.markcoroutinefunction(wrapper) + arg.__deprecated__ = wrapper.__deprecated__ = msg return wrapper else: @@ -690,7 +705,7 @@ def extract(): # filters contains a sequence of filter 5-tuples # The components of the 5-tuple are: -# - an action: error, ignore, always, default, module, or once +# - an action: error, ignore, always, all, default, module, or once # - a compiled regex that must match the warning message # - a class representing the warning category # - a compiled regex that must match the module that is being warned @@ -698,18 +713,36 @@ def extract(): # If either if the compiled regexs are None, match anything. try: from _warnings import (filters, _defaultaction, _onceregistry, - warn, warn_explicit, _filters_mutated) + warn, warn_explicit, + _filters_mutated_lock_held, + _acquire_lock, _release_lock, + ) defaultaction = _defaultaction onceregistry = _onceregistry _warnings_defaults = True + + class _Lock: + def __enter__(self): + _acquire_lock() + return self + + def __exit__(self, *args): + _release_lock() + + _lock = _Lock() + except ImportError: filters = [] defaultaction = "default" onceregistry = {} + import _thread + + _lock = _thread.RLock() + _filters_version = 1 - def _filters_mutated(): + def _filters_mutated_lock_held(): global _filters_version _filters_version += 1 diff --git a/Lib/weakref.py b/Lib/weakref.py index 25b70927e29c31..94e4278143c987 100644 --- a/Lib/weakref.py +++ b/Lib/weakref.py @@ -19,7 +19,7 @@ ReferenceType, _remove_dead_weakref) -from _weakrefset import WeakSet, _IterationGuard +from _weakrefset import WeakSet import _collections_abc # Import after _weakref to avoid circular import. import sys @@ -105,34 +105,14 @@ def __init__(self, other=(), /, **kw): def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref): self = selfref() if self is not None: - if self._iterating: - self._pending_removals.append(wr.key) - else: - # Atomic removal is necessary since this function - # can be called asynchronously by the GC - _atomic_removal(self.data, wr.key) + # Atomic removal is necessary since this function + # can be called asynchronously by the GC + _atomic_removal(self.data, wr.key) self._remove = remove - # A list of keys to be removed - self._pending_removals = [] - self._iterating = set() self.data = {} self.update(other, **kw) - def _commit_removals(self, _atomic_removal=_remove_dead_weakref): - pop = self._pending_removals.pop - d = self.data - # We shouldn't encounter any KeyError, because this method should - # always be called *before* mutating the dict. - while True: - try: - key = pop() - except IndexError: - return - _atomic_removal(d, key) - def __getitem__(self, key): - if self._pending_removals: - self._commit_removals() o = self.data[key]() if o is None: raise KeyError(key) @@ -140,18 +120,12 @@ def __getitem__(self, key): return o def __delitem__(self, key): - if self._pending_removals: - self._commit_removals() del self.data[key] def __len__(self): - if self._pending_removals: - self._commit_removals() return len(self.data) def __contains__(self, key): - if self._pending_removals: - self._commit_removals() try: o = self.data[key]() except KeyError: @@ -162,38 +136,28 @@ def __repr__(self): return "<%s at %#x>" % (self.__class__.__name__, id(self)) def __setitem__(self, key, value): - if self._pending_removals: - self._commit_removals() self.data[key] = KeyedRef(value, self._remove, key) def copy(self): - if self._pending_removals: - self._commit_removals() new = WeakValueDictionary() - with _IterationGuard(self): - for key, wr in self.data.items(): - o = wr() - if o is not None: - new[key] = o + for key, wr in self.data.copy().items(): + o = wr() + if o is not None: + new[key] = o return new __copy__ = copy def __deepcopy__(self, memo): from copy import deepcopy - if self._pending_removals: - self._commit_removals() new = self.__class__() - with _IterationGuard(self): - for key, wr in self.data.items(): - o = wr() - if o is not None: - new[deepcopy(key, memo)] = o + for key, wr in self.data.copy().items(): + o = wr() + if o is not None: + new[deepcopy(key, memo)] = o return new def get(self, key, default=None): - if self._pending_removals: - self._commit_removals() try: wr = self.data[key] except KeyError: @@ -207,21 +171,15 @@ def get(self, key, default=None): return o def items(self): - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - for k, wr in self.data.items(): - v = wr() - if v is not None: - yield k, v + for k, wr in self.data.copy().items(): + v = wr() + if v is not None: + yield k, v def keys(self): - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - for k, wr in self.data.items(): - if wr() is not None: - yield k + for k, wr in self.data.copy().items(): + if wr() is not None: + yield k __iter__ = keys @@ -235,23 +193,15 @@ def itervaluerefs(self): keep the values around longer than needed. """ - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - yield from self.data.values() + yield from self.data.copy().values() def values(self): - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - for wr in self.data.values(): - obj = wr() - if obj is not None: - yield obj + for wr in self.data.copy().values(): + obj = wr() + if obj is not None: + yield obj def popitem(self): - if self._pending_removals: - self._commit_removals() while True: key, wr = self.data.popitem() o = wr() @@ -259,8 +209,6 @@ def popitem(self): return key, o def pop(self, key, *args): - if self._pending_removals: - self._commit_removals() try: o = self.data.pop(key)() except KeyError: @@ -279,16 +227,12 @@ def setdefault(self, key, default=None): except KeyError: o = None if o is None: - if self._pending_removals: - self._commit_removals() self.data[key] = KeyedRef(default, self._remove, key) return default else: return o def update(self, other=None, /, **kwargs): - if self._pending_removals: - self._commit_removals() d = self.data if other is not None: if not hasattr(other, "items"): @@ -308,9 +252,7 @@ def valuerefs(self): keep the values around longer than needed. """ - if self._pending_removals: - self._commit_removals() - return list(self.data.values()) + return list(self.data.copy().values()) def __ior__(self, other): self.update(other) @@ -369,57 +311,22 @@ def __init__(self, dict=None): def remove(k, selfref=ref(self)): self = selfref() if self is not None: - if self._iterating: - self._pending_removals.append(k) - else: - try: - del self.data[k] - except KeyError: - pass + try: + del self.data[k] + except KeyError: + pass self._remove = remove - # A list of dead weakrefs (keys to be removed) - self._pending_removals = [] - self._iterating = set() - self._dirty_len = False if dict is not None: self.update(dict) - def _commit_removals(self): - # NOTE: We don't need to call this method before mutating the dict, - # because a dead weakref never compares equal to a live weakref, - # even if they happened to refer to equal objects. - # However, it means keys may already have been removed. - pop = self._pending_removals.pop - d = self.data - while True: - try: - key = pop() - except IndexError: - return - - try: - del d[key] - except KeyError: - pass - - def _scrub_removals(self): - d = self.data - self._pending_removals = [k for k in self._pending_removals if k in d] - self._dirty_len = False - def __delitem__(self, key): - self._dirty_len = True del self.data[ref(key)] def __getitem__(self, key): return self.data[ref(key)] def __len__(self): - if self._dirty_len and self._pending_removals: - # self._pending_removals may still contain keys which were - # explicitly removed, we have to scrub them (see issue #21173). - self._scrub_removals() - return len(self.data) - len(self._pending_removals) + return len(self.data) def __repr__(self): return "<%s at %#x>" % (self.__class__.__name__, id(self)) @@ -429,11 +336,10 @@ def __setitem__(self, key, value): def copy(self): new = WeakKeyDictionary() - with _IterationGuard(self): - for key, value in self.data.items(): - o = key() - if o is not None: - new[o] = value + for key, value in self.data.copy().items(): + o = key() + if o is not None: + new[o] = value return new __copy__ = copy @@ -441,11 +347,10 @@ def copy(self): def __deepcopy__(self, memo): from copy import deepcopy new = self.__class__() - with _IterationGuard(self): - for key, value in self.data.items(): - o = key() - if o is not None: - new[o] = deepcopy(value, memo) + for key, value in self.data.copy().items(): + o = key() + if o is not None: + new[o] = deepcopy(value, memo) return new def get(self, key, default=None): @@ -459,26 +364,23 @@ def __contains__(self, key): return wr in self.data def items(self): - with _IterationGuard(self): - for wr, value in self.data.items(): - key = wr() - if key is not None: - yield key, value + for wr, value in self.data.copy().items(): + key = wr() + if key is not None: + yield key, value def keys(self): - with _IterationGuard(self): - for wr in self.data: - obj = wr() - if obj is not None: - yield obj + for wr in self.data.copy(): + obj = wr() + if obj is not None: + yield obj __iter__ = keys def values(self): - with _IterationGuard(self): - for wr, value in self.data.items(): - if wr() is not None: - yield value + for wr, value in self.data.copy().items(): + if wr() is not None: + yield value def keyrefs(self): """Return a list of weak references to the keys. @@ -493,7 +395,6 @@ def keyrefs(self): return list(self.data) def popitem(self): - self._dirty_len = True while True: key, value = self.data.popitem() o = key() @@ -501,7 +402,6 @@ def popitem(self): return o, value def pop(self, key, *args): - self._dirty_len = True return self.data.pop(ref(key), *args) def setdefault(self, key, default=None): diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py old mode 100755 new mode 100644 index 0424c53b7ccaf9..d2efc72113a917 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -1,4 +1,3 @@ -#! /usr/bin/env python3 """Interfaces for launching and remotely controlling web browsers.""" # Maintained by Georg Brandl. @@ -11,14 +10,17 @@ __all__ = ["Error", "open", "open_new", "open_new_tab", "get", "register"] + class Error(Exception): pass + _lock = threading.RLock() _browsers = {} # Dictionary of available browser controllers _tryorder = None # Preference order of available browsers _os_preferred_browser = None # The preferred browser + def register(name, klass, instance=None, *, preferred=False): """Register a browser connector.""" with _lock: @@ -29,11 +31,12 @@ def register(name, klass, instance=None, *, preferred=False): # Preferred browsers go to the front of the list. # Need to match to the default browser returned by xdg-settings, which # may be of the form e.g. "firefox.desktop". - if preferred or (_os_preferred_browser and name in _os_preferred_browser): + if preferred or (_os_preferred_browser and f'{name}.desktop' == _os_preferred_browser): _tryorder.insert(0, name) else: _tryorder.append(name) + def get(using=None): """Return a browser launcher instance appropriate for the environment.""" if _tryorder is None: @@ -64,6 +67,7 @@ def get(using=None): return command[0]() raise Error("could not locate runnable browser") + # Please note: the following definition hides a builtin function. # It is recommended one does "import webbrowser" and uses webbrowser.open(url) # instead of "from webbrowser import *". @@ -76,6 +80,9 @@ def open(url, new=0, autoraise=True): - 1: a new browser window. - 2: a new browser page ("tab"). If possible, autoraise raises the window (the default) or not. + + If opening the browser succeeds, return True. + If there is a problem, return False. """ if _tryorder is None: with _lock: @@ -87,6 +94,7 @@ def open(url, new=0, autoraise=True): return True return False + def open_new(url): """Open url in a new window of the default browser. @@ -94,6 +102,7 @@ def open_new(url): """ return open(url, 1) + def open_new_tab(url): """Open url in a new page ("tab") of the default browser. @@ -136,7 +145,7 @@ def _synthesize(browser, *, preferred=False): # General parent classes -class BaseBrowser(object): +class BaseBrowser: """Parent class for all browsers. Do not use directly.""" args = ['%s'] @@ -197,7 +206,7 @@ def open(self, url, new=0, autoraise=True): else: p = subprocess.Popen(cmdline, close_fds=True, start_new_session=True) - return (p.poll() is None) + return p.poll() is None except OSError: return False @@ -225,7 +234,8 @@ def _invoke(self, args, remote, autoraise, url=None): # use autoraise argument only for remote invocation autoraise = int(autoraise) opt = self.raise_opts[autoraise] - if opt: raise_opt = [opt] + if opt: + raise_opt = [opt] cmdline = [self.name] + raise_opt + args @@ -266,8 +276,8 @@ def open(self, url, new=0, autoraise=True): else: action = self.remote_action_newtab else: - raise Error("Bad 'new' parameter to open(); " + - "expected 0, 1, or 2, got %s" % new) + raise Error("Bad 'new' parameter to open(); " + f"expected 0, 1, or 2, got {new}") args = [arg.replace("%s", url).replace("%action", action) for arg in self.remote_args] @@ -302,7 +312,7 @@ class Epiphany(UnixBrowser): class Chrome(UnixBrowser): - "Launcher class for Google Chrome browser." + """Launcher class for Google Chrome browser.""" remote_args = ['%action', '%s'] remote_action = "" @@ -310,11 +320,12 @@ class Chrome(UnixBrowser): remote_action_newtab = "" background = True + Chromium = Chrome class Opera(UnixBrowser): - "Launcher class for Opera browser." + """Launcher class for Opera browser.""" remote_args = ['%action', '%s'] remote_action = "" @@ -324,7 +335,7 @@ class Opera(UnixBrowser): class Elinks(UnixBrowser): - "Launcher class for Elinks browsers." + """Launcher class for Elinks browsers.""" remote_args = ['-remote', 'openURL(%s%action)'] remote_action = "" @@ -387,11 +398,11 @@ def open(self, url, new=0, autoraise=True): except OSError: return False else: - return (p.poll() is None) + return p.poll() is None class Edge(UnixBrowser): - "Launcher class for Microsoft Edge browser." + """Launcher class for Microsoft Edge browser.""" remote_args = ['%action', '%s'] remote_action = "" @@ -461,7 +472,6 @@ def register_X_browsers(): if shutil.which("opera"): register("opera", None, Opera("opera")) - if shutil.which("microsoft-edge"): register("microsoft-edge", None, Edge("microsoft-edge")) @@ -478,6 +488,9 @@ def register_standard_browsers(): # OS X can use below Unix support (but we prefer using the OS X # specific stuff) + if sys.platform == "ios": + register("iosbrowser", None, IOSBrowser(), preferred=True) + if sys.platform == "serenityos": # SerenityOS webbrowser, simply called "Browser". register("Browser", None, BackgroundBrowser("Browser")) @@ -511,7 +524,8 @@ def register_standard_browsers(): cmd = "xdg-settings get default-web-browser".split() raw_result = subprocess.check_output(cmd, stderr=subprocess.DEVNULL) result = raw_result.decode().strip() - except (FileNotFoundError, subprocess.CalledProcessError, PermissionError, NotADirectoryError) : + except (FileNotFoundError, subprocess.CalledProcessError, + PermissionError, NotADirectoryError): pass else: global _os_preferred_browser @@ -581,15 +595,16 @@ def __init__(self, name='default'): def open(self, url, new=0, autoraise=True): sys.audit("webbrowser.open", url) + url = url.replace('"', '%22') if self.name == 'default': - script = 'open location "%s"' % url.replace('"', '%22') # opens in default browser + script = f'open location "{url}"' # opens in default browser else: script = f''' - tell application "%s" + tell application "{self.name}" activate - open location "%s" + open location "{url}" end - '''%(self.name, url.replace('"', '%22')) + ''' osapipe = os.popen("osascript", "w") if osapipe is None: @@ -599,34 +614,96 @@ def open(self, url, new=0, autoraise=True): rc = osapipe.close() return not rc +# +# Platform support for iOS +# +if sys.platform == "ios": + from _ios_support import objc + if objc: + # If objc exists, we know ctypes is also importable. + from ctypes import c_void_p, c_char_p, c_ulong -def main(): - import getopt - usage = """Usage: %s [-n | -t | -h] url - -n: open new window - -t: open new tab - -h, --help: show help""" % sys.argv[0] - try: - opts, args = getopt.getopt(sys.argv[1:], 'ntdh',['help']) - except getopt.error as msg: - print(msg, file=sys.stderr) - print(usage, file=sys.stderr) - sys.exit(1) - new_win = 0 - for o, a in opts: - if o == '-n': new_win = 1 - elif o == '-t': new_win = 2 - elif o == '-h' or o == '--help': - print(usage, file=sys.stderr) - sys.exit() - if len(args) != 1: - print(usage, file=sys.stderr) - sys.exit(1) - - url = args[0] - open(url, new_win) + class IOSBrowser(BaseBrowser): + def open(self, url, new=0, autoraise=True): + sys.audit("webbrowser.open", url) + # If ctypes isn't available, we can't open a browser + if objc is None: + return False + + # All the messages in this call return object references. + objc.objc_msgSend.restype = c_void_p + + # This is the equivalent of: + # NSString url_string = + # [NSString stringWithCString:url.encode("utf-8") + # encoding:NSUTF8StringEncoding]; + NSString = objc.objc_getClass(b"NSString") + constructor = objc.sel_registerName(b"stringWithCString:encoding:") + objc.objc_msgSend.argtypes = [c_void_p, c_void_p, c_char_p, c_ulong] + url_string = objc.objc_msgSend( + NSString, + constructor, + url.encode("utf-8"), + 4, # NSUTF8StringEncoding = 4 + ) + + # Create an NSURL object representing the URL + # This is the equivalent of: + # NSURL *nsurl = [NSURL URLWithString:url]; + NSURL = objc.objc_getClass(b"NSURL") + urlWithString_ = objc.sel_registerName(b"URLWithString:") + objc.objc_msgSend.argtypes = [c_void_p, c_void_p, c_void_p] + ns_url = objc.objc_msgSend(NSURL, urlWithString_, url_string) + + # Get the shared UIApplication instance + # This code is the equivalent of: + # UIApplication shared_app = [UIApplication sharedApplication] + UIApplication = objc.objc_getClass(b"UIApplication") + sharedApplication = objc.sel_registerName(b"sharedApplication") + objc.objc_msgSend.argtypes = [c_void_p, c_void_p] + shared_app = objc.objc_msgSend(UIApplication, sharedApplication) + + # Open the URL on the shared application + # This code is the equivalent of: + # [shared_app openURL:ns_url + # options:NIL + # completionHandler:NIL]; + openURL_ = objc.sel_registerName(b"openURL:options:completionHandler:") + objc.objc_msgSend.argtypes = [ + c_void_p, c_void_p, c_void_p, c_void_p, c_void_p + ] + # Method returns void + objc.objc_msgSend.restype = None + objc.objc_msgSend(shared_app, openURL_, ns_url, None, None) + + return True + + +def parse_args(arg_list: list[str] | None): + import argparse + parser = argparse.ArgumentParser(description="Open URL in a web browser.") + parser.add_argument("url", help="URL to open") + + group = parser.add_mutually_exclusive_group() + group.add_argument("-n", "--new-window", action="store_const", + const=1, default=0, dest="new_win", + help="open new window") + group.add_argument("-t", "--new-tab", action="store_const", + const=2, default=0, dest="new_win", + help="open new tab") + + args = parser.parse_args(arg_list) + + return args + + +def main(arg_list: list[str] | None = None): + args = parse_args(arg_list) + + open(args.url, args.new_win) print("\a") + if __name__ == "__main__": main() diff --git a/Lib/wsgiref/headers.py b/Lib/wsgiref/headers.py index fab851c5a44430..c78879f80c7df2 100644 --- a/Lib/wsgiref/headers.py +++ b/Lib/wsgiref/headers.py @@ -1,11 +1,11 @@ """Manage HTTP Response Headers Much of this module is red-handedly pilfered from email.message in the stdlib, -so portions are Copyright (C) 2001,2002 Python Software Foundation, and were +so portions are Copyright (C) 2001 Python Software Foundation, and were written by Barry Warsaw. """ -# Regular expression that matches `special' characters in parameters, the +# Regular expression that matches 'special' characters in parameters, the # existence of which force quoting of the parameter value. import re tspecials = re.compile(r'[ \(\)<>@,;:\\"/\[\]\?=]') diff --git a/Lib/xml/dom/__init__.py b/Lib/xml/dom/__init__.py index 97cf9a6429993d..dd7fb996afd616 100644 --- a/Lib/xml/dom/__init__.py +++ b/Lib/xml/dom/__init__.py @@ -137,4 +137,4 @@ class UserDataHandler: EMPTY_NAMESPACE = None EMPTY_PREFIX = None -from .domreg import getDOMImplementation, registerDOMImplementation +from .domreg import getDOMImplementation, registerDOMImplementation # noqa: F401 diff --git a/Lib/xml/dom/xmlbuilder.py b/Lib/xml/dom/xmlbuilder.py index 8a200263497b89..a8852625a2f9a2 100644 --- a/Lib/xml/dom/xmlbuilder.py +++ b/Lib/xml/dom/xmlbuilder.py @@ -189,7 +189,7 @@ def parse(self, input): options.filter = self.filter options.errorHandler = self.errorHandler fp = input.byteStream - if fp is None and options.systemId: + if fp is None and input.systemId: import urllib.request fp = urllib.request.urlopen(input.systemId) return self._parse_bytestream(fp, options) @@ -247,10 +247,12 @@ def _create_opener(self): def _guess_media_encoding(self, source): info = source.byteStream.info() - if "Content-Type" in info: - for param in info.getplist(): - if param.startswith("charset="): - return param.split("=", 1)[1].lower() + # import email.message + # assert isinstance(info, email.message.Message) + charset = info.get_param('charset') + if charset is not None: + return charset.lower() + return None class DOMInputSource(object): diff --git a/Lib/xml/etree/ElementInclude.py b/Lib/xml/etree/ElementInclude.py index 40a9b22292479f..986e6c3bbe90f6 100644 --- a/Lib/xml/etree/ElementInclude.py +++ b/Lib/xml/etree/ElementInclude.py @@ -79,8 +79,8 @@ class LimitedRecursiveIncludeError(FatalIncludeError): # @param parse Parse mode. Either "xml" or "text". # @param encoding Optional text encoding (UTF-8 by default for "text"). # @return The expanded resource. If the parse mode is "xml", this -# is an ElementTree instance. If the parse mode is "text", this -# is a Unicode string. If the loader fails, it can return None +# is an Element instance. If the parse mode is "text", this +# is a string. If the loader fails, it can return None # or raise an OSError exception. # @throws OSError If the loader fails to load the resource. @@ -98,7 +98,7 @@ def default_loader(href, parse, encoding=None): ## # Expand XInclude directives. # -# @param elem Root element. +# @param elem Root Element or any ElementTree of a tree to be expanded # @param loader Optional resource loader. If omitted, it defaults # to {@link default_loader}. If given, it should be a callable # that implements the same interface as default_loader. @@ -106,12 +106,13 @@ def default_loader(href, parse, encoding=None): # relative include file references. # @param max_depth The maximum number of recursive inclusions. # Limited to reduce the risk of malicious content explosion. -# Pass a negative value to disable the limitation. +# Pass None to disable the limitation. # @throws LimitedRecursiveIncludeError If the {@link max_depth} was exceeded. # @throws FatalIncludeError If the function fails to include a given # resource, or if the tree contains malformed XInclude elements. -# @throws IOError If the function fails to load a given resource. -# @returns the node or its replacement if it was an XInclude node +# @throws OSError If the function fails to load a given resource. +# @throws ValueError If negative {@link max_depth} is passed. +# @returns None. Modifies tree pointed by {@link elem} def include(elem, loader=None, base_url=None, max_depth=DEFAULT_MAX_INCLUSION_DEPTH): diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 9e15d34d22aa6c..ce67d7d7d54748 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -201,7 +201,7 @@ def __len__(self): def __bool__(self): warnings.warn( - "Testing an element's truth value will raise an exception in " + "Testing an element's truth value will always return True in " "future versions. " "Use specific 'len(elem)' or 'elem is not None' test instead.", DeprecationWarning, stacklevel=2 diff --git a/Lib/xml/sax/__init__.py b/Lib/xml/sax/__init__.py index b657310207cfe5..fe4582c6f8b758 100644 --- a/Lib/xml/sax/__init__.py +++ b/Lib/xml/sax/__init__.py @@ -21,9 +21,9 @@ from .xmlreader import InputSource from .handler import ContentHandler, ErrorHandler -from ._exceptions import SAXException, SAXNotRecognizedException, \ - SAXParseException, SAXNotSupportedException, \ - SAXReaderNotAvailable +from ._exceptions import (SAXException, SAXNotRecognizedException, + SAXParseException, SAXNotSupportedException, + SAXReaderNotAvailable) def parse(source, handler, errorHandler=ErrorHandler()): @@ -55,7 +55,7 @@ def parseString(string, handler, errorHandler=ErrorHandler()): # tell modulefinder that importing sax potentially imports expatreader _false = 0 if _false: - import xml.sax.expatreader + import xml.sax.expatreader # noqa: F401 import os, sys if not sys.flags.ignore_environment and "PY_SAX_PARSER" in os.environ: @@ -92,3 +92,9 @@ def make_parser(parser_list=()): def _create_parser(parser_name): drv_module = __import__(parser_name,{},{},['create_parser']) return drv_module.create_parser() + + +__all__ = ['ContentHandler', 'ErrorHandler', 'InputSource', 'SAXException', + 'SAXNotRecognizedException', 'SAXNotSupportedException', + 'SAXParseException', 'SAXReaderNotAvailable', + 'default_parser_list', 'make_parser', 'parse', 'parseString'] diff --git a/Lib/xml/sax/handler.py b/Lib/xml/sax/handler.py index e8d417e5194232..3183c3fe96d74f 100644 --- a/Lib/xml/sax/handler.py +++ b/Lib/xml/sax/handler.py @@ -371,7 +371,7 @@ def startDTD(self, name, public_id, system_id): name is the name of the document element type, public_id the public identifier of the DTD (or None if none were supplied) - and system_id the system identfier of the external subset (or + and system_id the system identifier of the external subset (or None if none were supplied).""" def endDTD(self): diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 4dddb1d10e08bd..90a356fbb8eae4 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -578,6 +578,7 @@ class SimpleXMLRPCServer(socketserver.TCPServer, """ allow_reuse_address = True + allow_reuse_port = True # Warning: this is for debugging purposes only! Never set this to True in # production code, as will be sending out sensitive information (exception diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index cc08f602fe44e0..b8b496ad9471f4 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -241,7 +241,9 @@ def is_zipfile(filename): result = False try: if hasattr(filename, "read"): + pos = filename.tell() result = _check_zipfile(fp=filename) + filename.seek(pos) else: with open(filename, "rb") as fp: result = _check_zipfile(fp) @@ -309,7 +311,7 @@ def _EndRecData(fpin): fpin.seek(-sizeEndCentDir, 2) except OSError: return None - data = fpin.read() + data = fpin.read(sizeEndCentDir) if (len(data) == sizeEndCentDir and data[0:4] == stringEndArchive and data[-2:] == b"\000\000"): @@ -329,9 +331,9 @@ def _EndRecData(fpin): # record signature. The comment is the last item in the ZIP file and may be # up to 64K long. It is assumed that the "end of central directory" magic # number does not appear in the comment. - maxCommentStart = max(filesize - (1 << 16) - sizeEndCentDir, 0) + maxCommentStart = max(filesize - ZIP_MAX_COMMENT - sizeEndCentDir, 0) fpin.seek(maxCommentStart, 0) - data = fpin.read() + data = fpin.read(ZIP_MAX_COMMENT + sizeEndCentDir) start = data.rfind(stringEndArchive) if start >= 0: # found the magic number; attempt to unpack and interpret @@ -603,9 +605,39 @@ def from_file(cls, filename, arcname=None, *, strict_timestamps=True): return zinfo + def _for_archive(self, archive): + """Resolve suitable defaults from the archive. + + Resolve the date_time, compression attributes, and external attributes + to suitable defaults as used by :method:`ZipFile.writestr`. + + Return self. + """ + # gh-91279: Set the SOURCE_DATE_EPOCH to a specific timestamp + epoch = os.environ.get('SOURCE_DATE_EPOCH') + get_time = int(epoch) if epoch else time.time() + self.date_time = time.localtime(get_time)[:6] + + self.compress_type = archive.compression + self.compress_level = archive.compresslevel + if self.filename.endswith('/'): # pragma: no cover + self.external_attr = 0o40775 << 16 # drwxrwxr-x + self.external_attr |= 0x10 # MS-DOS directory flag + else: + self.external_attr = 0o600 << 16 # ?rw------- + return self + def is_dir(self): """Return True if this archive member is a directory.""" - return self.filename.endswith('/') + if self.filename.endswith('/'): + return True + # The ZIP format specification requires to use forward slashes + # as the directory separator, but in practice some ZIP files + # created on Windows can use backward slashes. For compatibility + # with the extraction code which already handles this: + if os.path.altsep: + return self.filename.endswith((os.path.sep, os.path.altsep)) + return False # ZIP encryption uses the CRC32 one-byte primitive for scrambling some @@ -809,7 +841,10 @@ def seek(self, offset, whence=0): raise ValueError("Can't reposition in the ZIP file while " "there is an open writing handle on it. " "Close the writing handle before trying to read.") - self._file.seek(offset, whence) + if whence == os.SEEK_CUR: + self._file.seek(self._pos + offset) + else: + self._file.seek(offset, whence) self._pos = self._file.tell() return self._pos @@ -932,7 +967,7 @@ def __repr__(self): result = ['<%s.%s' % (self.__class__.__module__, self.__class__.__qualname__)] if not self.closed: - result.append(' name=%r mode=%r' % (self.name, self.mode)) + result.append(' name=%r' % (self.name,)) if self._compress_type != ZIP_STORED: result.append(' compress_type=%s' % compressor_names.get(self._compress_type, @@ -1152,13 +1187,15 @@ def seek(self, offset, whence=os.SEEK_SET): self._offset = buff_offset read_offset = 0 # Fast seek uncompressed unencrypted file - elif self._compress_type == ZIP_STORED and self._decrypter is None and read_offset > 0: + elif self._compress_type == ZIP_STORED and self._decrypter is None and read_offset != 0: # disable CRC checking after first seeking - it would be invalid self._expected_crc = None # seek actual file taking already buffered data into account read_offset -= len(self._readbuffer) - self._offset self._fileobj.seek(read_offset, os.SEEK_CUR) self._left -= read_offset + self._compress_left -= read_offset + self._eof = self._left <= 0 read_offset = 0 # flush read buffer self._readbuffer = b'' @@ -1209,6 +1246,14 @@ def __init__(self, zf, zinfo, zip64): def _fileobj(self): return self._zipfile.fp + @property + def name(self): + return self._zinfo.filename + + @property + def mode(self): + return 'wb' + def writable(self): return True @@ -1577,7 +1622,8 @@ def comment(self, comment): self._didModify = True def read(self, name, pwd=None): - """Return file bytes for name.""" + """Return file bytes for name. 'pwd' is the password to decrypt + encrypted files.""" with self.open(name, "r", pwd) as fp: return fp.read() @@ -1678,7 +1724,7 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): else: pwd = None - return ZipExtFile(zef_file, mode, zinfo, pwd, True) + return ZipExtFile(zef_file, mode + 'b', zinfo, pwd, True) except: zef_file.close() raise @@ -1728,8 +1774,9 @@ def _open_to_write(self, zinfo, force_zip64=False): def extract(self, member, path=None, pwd=None): """Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately - as possible. `member' may be a filename or a ZipInfo object. You can - specify a different directory using `path'. + as possible. 'member' may be a filename or a ZipInfo object. You can + specify a different directory using 'path'. You can specify the + password to decrypt the file using 'pwd'. """ if path is None: path = os.getcwd() @@ -1740,9 +1787,10 @@ def extract(self, member, path=None, pwd=None): def extractall(self, path=None, members=None, pwd=None): """Extract all members from the archive to the current working - directory. `path' specifies a different directory to extract to. - `members' is optional and must be a subset of the list returned - by namelist(). + directory. 'path' specifies a different directory to extract to. + 'members' is optional and must be a subset of the list returned + by namelist(). You can specify the password to decrypt all files + using 'pwd'. """ if members is None: members = self.namelist() @@ -1884,18 +1932,10 @@ def writestr(self, zinfo_or_arcname, data, the name of the file in the archive.""" if isinstance(data, str): data = data.encode("utf-8") - if not isinstance(zinfo_or_arcname, ZipInfo): - zinfo = ZipInfo(filename=zinfo_or_arcname, - date_time=time.localtime(time.time())[:6]) - zinfo.compress_type = self.compression - zinfo.compress_level = self.compresslevel - if zinfo.filename.endswith('/'): - zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x - zinfo.external_attr |= 0x10 # MS-DOS directory flag - else: - zinfo.external_attr = 0o600 << 16 # ?rw------- - else: + if isinstance(zinfo_or_arcname, ZipInfo): zinfo = zinfo_or_arcname + else: + zinfo = ZipInfo(zinfo_or_arcname)._for_archive(self) if not self.fp: raise ValueError( diff --git a/Lib/zipfile/_path/__init__.py b/Lib/zipfile/_path/__init__.py index 78c413563bb2b1..5ae16ec970dda4 100644 --- a/Lib/zipfile/_path/__init__.py +++ b/Lib/zipfile/_path/__init__.py @@ -1,3 +1,12 @@ +""" +A Path-like interface for zipfiles. + +This codebase is shared between zipfile.Path in the stdlib +and zipp in PyPI. See +https://github.com/python/importlib_metadata/wiki/Development-Methodology +for more detail. +""" + import io import posixpath import zipfile @@ -5,8 +14,10 @@ import contextlib import pathlib import re +import stat +import sys -from .glob import translate +from .glob import Translator __all__ = ['Path'] @@ -34,7 +45,7 @@ def _parents(path): def _ancestry(path): """ Given a path with elements separated by - posixpath.sep, generate all elements of that path + posixpath.sep, generate all elements of that path. >>> list(_ancestry('b/d')) ['b/d', 'b'] @@ -46,9 +57,14 @@ def _ancestry(path): ['b'] >>> list(_ancestry('')) [] + + Multiple separators are treated like a single. + + >>> list(_ancestry('//b//d///f//')) + ['//b//d///f', '//b//d', '//b'] """ path = path.rstrip(posixpath.sep) - while path and path != posixpath.sep: + while path.rstrip(posixpath.sep): yield path path, tail = posixpath.split(path) @@ -147,6 +163,16 @@ def make(cls, source): source.__class__ = cls return source + @classmethod + def inject(cls, zf: zipfile.ZipFile) -> zipfile.ZipFile: + """ + Given a writable zip file zf, inject directory entries for + any directories implied by the presence of children. + """ + for name in cls._implied_dirs(zf.namelist()): + zf.writestr(name, b"") + return zf + class FastLookup(CompleteDirs): """ @@ -168,13 +194,18 @@ def _name_set(self): def _extract_text_encoding(encoding=None, *args, **kwargs): - # stacklevel=3 so that the caller of the caller see any warning. - return io.text_encoding(encoding, 3), args, kwargs + # compute stack level so that the caller of the caller sees any warning. + is_pypy = sys.implementation.name == 'pypy' + stack_level = 3 + is_pypy + return io.text_encoding(encoding, stack_level), args, kwargs class Path: """ - A pathlib-compatible interface for zip files. + A :class:`importlib.resources.abc.Traversable` interface for zip files. + + Implements many of the features users enjoy from + :class:`pathlib.Path`. Consider a zip file with this structure:: @@ -194,13 +225,13 @@ class Path: Path accepts the zipfile object itself or a filename - >>> root = Path(zf) + >>> path = Path(zf) From there, several path operations are available. Directory iteration (including the zip file itself): - >>> a, b = root.iterdir() + >>> a, b = path.iterdir() >>> a Path('mem/abcde.zip', 'a.txt') >>> b @@ -238,16 +269,38 @@ class Path: 'mem/abcde.zip/b/c.txt' At the root, ``name``, ``filename``, and ``parent`` - resolve to the zipfile. Note these attributes are not - valid and will raise a ``ValueError`` if the zipfile - has no filename. + resolve to the zipfile. - >>> root.name + >>> str(path) + 'mem/abcde.zip/' + >>> path.name 'abcde.zip' - >>> str(root.filename).replace(os.sep, posixpath.sep) - 'mem/abcde.zip' - >>> str(root.parent) + >>> path.filename == pathlib.Path('mem/abcde.zip') + True + >>> str(path.parent) 'mem' + + If the zipfile has no filename, such attributes are not + valid and accessing them will raise an Exception. + + >>> zf.filename = None + >>> path.name + Traceback (most recent call last): + ... + TypeError: ... + + >>> path.filename + Traceback (most recent call last): + ... + TypeError: ... + + >>> path.parent + Traceback (most recent call last): + ... + TypeError: ... + + # workaround python/cpython#106763 + >>> pass """ __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})" @@ -286,7 +339,7 @@ def open(self, mode='r', *args, pwd=None, **kwargs): if self.is_dir(): raise IsADirectoryError(self) zip_mode = mode[0] - if not self.exists() and zip_mode == 'r': + if zip_mode == 'r' and not self.exists(): raise FileNotFoundError(self) stream = self.root.open(self.at, zip_mode, pwd=pwd) if 'b' in mode: @@ -355,16 +408,19 @@ def match(self, path_pattern): def is_symlink(self): """ - Return whether this path is a symlink. Always false (python/cpython#82102). + Return whether this path is a symlink. """ - return False + info = self.root.getinfo(self.at) + mode = info.external_attr >> 16 + return stat.S_ISLNK(mode) def glob(self, pattern): if not pattern: raise ValueError(f"Unacceptable pattern: {pattern!r}") prefix = re.escape(self.at) - matches = re.compile(prefix + translate(pattern)).fullmatch + tr = Translator(seps='/') + matches = re.compile(prefix + tr.translate(pattern)).fullmatch return map(self._next, filter(matches, self.root.namelist())) def rglob(self, pattern): diff --git a/Lib/zipfile/_path/glob.py b/Lib/zipfile/_path/glob.py index 4a2e665e27078a..4320f1c0badcf9 100644 --- a/Lib/zipfile/_path/glob.py +++ b/Lib/zipfile/_path/glob.py @@ -1,18 +1,105 @@ +import os import re -def translate(pattern): - r""" - Given a glob pattern, produce a regex that matches it. +_default_seps = os.sep + str(os.altsep) * bool(os.altsep) - >>> translate('*.txt') - '[^/]*\\.txt' - >>> translate('a?txt') - 'a.txt' - >>> translate('**/*') - '.*/[^/]*' + +class Translator: + """ + >>> Translator('xyz') + Traceback (most recent call last): + ... + AssertionError: Invalid separators + + >>> Translator('') + Traceback (most recent call last): + ... + AssertionError: Invalid separators """ - return ''.join(map(replace, separate(pattern))) + + seps: str + + def __init__(self, seps: str = _default_seps): + assert seps and set(seps) <= set(_default_seps), "Invalid separators" + self.seps = seps + + def translate(self, pattern): + """ + Given a glob pattern, produce a regex that matches it. + """ + return self.extend(self.match_dirs(self.translate_core(pattern))) + + def extend(self, pattern): + r""" + Extend regex for pattern-wide concerns. + + Apply '(?s:)' to create a non-matching group that + matches newlines (valid on Unix). + + Append '\Z' to imply fullmatch even when match is used. + """ + return rf'(?s:{pattern})\Z' + + def match_dirs(self, pattern): + """ + Ensure that zipfile.Path directory names are matched. + + zipfile.Path directory names always end in a slash. + """ + return rf'{pattern}[/]?' + + def translate_core(self, pattern): + r""" + Given a glob pattern, produce a regex that matches it. + + >>> t = Translator() + >>> t.translate_core('*.txt').replace('\\\\', '') + '[^/]*\\.txt' + >>> t.translate_core('a?txt') + 'a[^/]txt' + >>> t.translate_core('**/*').replace('\\\\', '') + '.*/[^/][^/]*' + """ + self.restrict_rglob(pattern) + return ''.join(map(self.replace, separate(self.star_not_empty(pattern)))) + + def replace(self, match): + """ + Perform the replacements for a match from :func:`separate`. + """ + return match.group('set') or ( + re.escape(match.group(0)) + .replace('\\*\\*', r'.*') + .replace('\\*', rf'[^{re.escape(self.seps)}]*') + .replace('\\?', r'[^/]') + ) + + def restrict_rglob(self, pattern): + """ + Raise ValueError if ** appears in anything but a full path segment. + + >>> Translator().translate('**foo') + Traceback (most recent call last): + ... + ValueError: ** must appear alone in a path segment + """ + seps_pattern = rf'[{re.escape(self.seps)}]+' + segments = re.split(seps_pattern, pattern) + if any('**' in segment and segment != '**' for segment in segments): + raise ValueError("** must appear alone in a path segment") + + def star_not_empty(self, pattern): + """ + Ensure that * will not match an empty segment. + """ + + def handle_segment(match): + segment = match.group(0) + return '?*' if segment == '*' else segment + + not_seps_pattern = rf'[^{re.escape(self.seps)}]+' + return re.sub(not_seps_pattern, handle_segment, pattern) def separate(pattern): @@ -25,16 +112,3 @@ def separate(pattern): ['a', '[?]', 'txt'] """ return re.finditer(r'([^\[]+)|(?P[\[].*?[\]])|([\[][^\]]*$)', pattern) - - -def replace(match): - """ - Perform the replacements for a match from :func:`separate`. - """ - - return match.group('set') or ( - re.escape(match.group(0)) - .replace('\\*\\*', r'.*') - .replace('\\*', r'[^/]*') - .replace('\\?', r'.') - ) diff --git a/Lib/zipimport.py b/Lib/zipimport.py index 823a82ee830465..e5192c4d074c4b 100644 --- a/Lib/zipimport.py +++ b/Lib/zipimport.py @@ -1,11 +1,9 @@ """zipimport provides support for importing Python modules from Zip archives. -This module exports three objects: +This module exports two objects: - zipimporter: a class; its constructor takes a path to a Zip archive. - ZipImportError: exception raised by zipimporter objects. It's a subclass of ImportError, so it can be caught as ImportError, too. -- _zip_directory_cache: a dict, mapping archive paths to zip directory - info dicts, as used in zipimporter._files. It is usually not needed to use the zipimport module explicitly; it is used by the builtin import mechanism for sys.path items that are paths @@ -15,7 +13,7 @@ #from importlib import _bootstrap_external #from importlib import _bootstrap # for _verbose_message import _frozen_importlib_external as _bootstrap_external -from _frozen_importlib_external import _unpack_uint16, _unpack_uint32 +from _frozen_importlib_external import _unpack_uint16, _unpack_uint32, _unpack_uint64 import _frozen_importlib as _bootstrap # for _verbose_message import _imp # for check_hash_based_pycs import _io # for open @@ -40,8 +38,14 @@ class ZipImportError(ImportError): _module_type = type(sys) END_CENTRAL_DIR_SIZE = 22 -STRING_END_ARCHIVE = b'PK\x05\x06' +END_CENTRAL_DIR_SIZE_64 = 56 +END_CENTRAL_DIR_LOCATOR_SIZE_64 = 20 +STRING_END_ARCHIVE = b'PK\x05\x06' # standard EOCD signature +STRING_END_LOCATOR_64 = b'PK\x06\x07' # Zip64 EOCD Locator signature +STRING_END_ZIP_64 = b'PK\x06\x06' # Zip64 EOCD signature MAX_COMMENT_LEN = (1 << 16) - 1 +MAX_UINT32 = 0xffffffff +ZIP64_EXTRA_TAG = 0x1 class zipimporter(_bootstrap_external._LoaderBasics): """zipimporter(archivepath) -> zipimporter object @@ -151,6 +155,8 @@ def get_data(self, pathname): toc_entry = self._get_files()[key] except KeyError: raise OSError(0, '', key) + if toc_entry is None: + return b'' return _get_data(self.archive, toc_entry) @@ -250,17 +256,9 @@ def load_module(self, fullname): def get_resource_reader(self, fullname): - """Return the ResourceReader for a package in a zip file. - - If 'fullname' is a package within the zip file, return the - 'ResourceReader' object for the package. Otherwise return None. - """ - try: - if not self.is_package(fullname): - return None - except ZipImportError: - return None + """Return the ResourceReader for a module in a zip file.""" from importlib.readers import ZipReader + return ZipReader(self, fullname) @@ -356,49 +354,72 @@ def _read_directory(archive): # to not cause problems when some runs 'python3 /dev/fd/9 9= 0 and pos64+END_CENTRAL_DIR_SIZE_64+END_CENTRAL_DIR_LOCATOR_SIZE_64==pos): + # Zip64 at "correct" offset from standard EOCD + buffer = data[pos64:pos64 + END_CENTRAL_DIR_SIZE_64] + if len(buffer) != END_CENTRAL_DIR_SIZE_64: + raise ZipImportError( + f"corrupt Zip64 file: Expected {END_CENTRAL_DIR_SIZE_64} byte " + f"zip64 central directory, but read {len(buffer)} bytes.", + path=archive) + header_position = file_size - len(data) + pos64 + + central_directory_size = _unpack_uint64(buffer[40:48]) + central_directory_position = _unpack_uint64(buffer[48:56]) + num_entries = _unpack_uint64(buffer[24:32]) + elif pos >= 0: buffer = data[pos:pos+END_CENTRAL_DIR_SIZE] if len(buffer) != END_CENTRAL_DIR_SIZE: raise ZipImportError(f"corrupt Zip file: {archive!r}", path=archive) + header_position = file_size - len(data) + pos - header_size = _unpack_uint32(buffer[12:16]) - header_offset = _unpack_uint32(buffer[16:20]) - if header_position < header_size: + # Buffer now contains a valid EOCD, and header_position gives the + # starting position of it. + central_directory_size = _unpack_uint32(buffer[12:16]) + central_directory_position = _unpack_uint32(buffer[16:20]) + num_entries = _unpack_uint16(buffer[8:10]) + + # N.b. if someday you want to prefer the standard (non-zip64) EOCD, + # you need to adjust position by 76 for arc to be 0. + else: + raise ZipImportError(f'not a Zip file: {archive!r}', + path=archive) + + # Buffer now contains a valid EOCD, and header_position gives the + # starting position of it. + # XXX: These are cursory checks but are not as exact or strict as they + # could be. Checking the arc-adjusted value is probably good too. + if header_position < central_directory_size: raise ZipImportError(f'bad central directory size: {archive!r}', path=archive) - if header_position < header_offset: + if header_position < central_directory_position: raise ZipImportError(f'bad central directory offset: {archive!r}', path=archive) - header_position -= header_size - arc_offset = header_position - header_offset + header_position -= central_directory_size + # On just-a-zipfile these values are the same and arc_offset is zero; if + # the file has some bytes prepended, `arc_offset` is the number of such + # bytes. This is used for pex as well as self-extracting .exe. + arc_offset = header_position - central_directory_position if arc_offset < 0: raise ZipImportError(f'bad central directory size or offset: {archive!r}', path=archive) @@ -415,6 +436,11 @@ def _read_directory(archive): raise EOFError('EOF read where not expected') # Start of file header if buffer[:4] != b'PK\x01\x02': + if count != num_entries: + raise ZipImportError( + f"mismatched num_entries: {count} should be {num_entries} in {archive!r}", + path=archive, + ) break # Bad: Central Dir File Header if len(buffer) != 46: raise EOFError('EOF read where not expected') @@ -430,9 +456,6 @@ def _read_directory(archive): comment_size = _unpack_uint16(buffer[32:34]) file_offset = _unpack_uint32(buffer[42:46]) header_size = name_size + extra_size + comment_size - if file_offset > header_offset: - raise ZipImportError(f'bad local header offset: {archive!r}', path=archive) - file_offset += arc_offset try: name = fp.read(name_size) @@ -444,7 +467,10 @@ def _read_directory(archive): # slower than reading the data because fseek flushes stdio's # internal buffers. See issue #8745. try: - if len(fp.read(header_size - name_size)) != header_size - name_size: + extra_data_len = header_size - name_size + extra_data = memoryview(fp.read(extra_data_len)) + + if len(extra_data) != extra_data_len: raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) except OSError: raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) @@ -461,12 +487,83 @@ def _read_directory(archive): name = name.replace('/', path_sep) path = _bootstrap_external._path_join(archive, name) + + # Ordering matches unpacking below. + if ( + file_size == MAX_UINT32 or + data_size == MAX_UINT32 or + file_offset == MAX_UINT32 + ): + # need to decode extra_data looking for a zip64 extra (which might not + # be present) + while extra_data: + if len(extra_data) < 4: + raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) + tag = _unpack_uint16(extra_data[:2]) + size = _unpack_uint16(extra_data[2:4]) + if len(extra_data) < 4 + size: + raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) + if tag == ZIP64_EXTRA_TAG: + if (len(extra_data) - 4) % 8 != 0: + raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) + num_extra_values = (len(extra_data) - 4) // 8 + if num_extra_values > 3: + raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) + import struct + values = list(struct.unpack_from(f"<{min(num_extra_values, 3)}Q", + extra_data, offset=4)) + + # N.b. Here be dragons: the ordering of these is different than + # the header fields, and it's really easy to get it wrong since + # naturally-occurring zips that use all 3 are >4GB + if file_size == MAX_UINT32: + file_size = values.pop(0) + if data_size == MAX_UINT32: + data_size = values.pop(0) + if file_offset == MAX_UINT32: + file_offset = values.pop(0) + + break + + # For a typical zip, this bytes-slicing only happens 2-3 times, on + # small data like timestamps and filesizes. + extra_data = extra_data[4+size:] + else: + _bootstrap._verbose_message( + "zipimport: suspected zip64 but no zip64 extra for {!r}", + path, + ) + # XXX These two statements seem swapped because `central_directory_position` + # is a position within the actual file, but `file_offset` (when compared) is + # as encoded in the entry, not adjusted for this file. + # N.b. this must be after we've potentially read the zip64 extra which can + # change `file_offset`. + if file_offset > central_directory_position: + raise ZipImportError(f'bad local header offset: {archive!r}', path=archive) + file_offset += arc_offset + t = (path, compress, data_size, file_size, file_offset, time, date, crc) files[name] = t count += 1 finally: fp.seek(start_offset) _bootstrap._verbose_message('zipimport: found {} names in {!r}', count, archive) + + # Add implicit directories. + count = 0 + for name in list(files): + while True: + i = name.rstrip(path_sep).rfind(path_sep) + if i < 0: + break + name = name[:i + 1] + if name in files: + break + files[name] = None + count += 1 + if count: + _bootstrap._verbose_message('zipimport: added {} implicit directories in {!r}', + count, archive) return files # During bootstrap, we may need to load the encodings @@ -600,7 +697,7 @@ def _unmarshal_code(self, pathname, fullpath, fullname, data): source_bytes = _get_pyc_source(self, fullpath) if source_bytes is not None: source_hash = _imp.source_hash( - _bootstrap_external._RAW_MAGIC_NUMBER, + _imp.pyc_magic_number_token, source_bytes, ) diff --git a/Mac/BuildScript/README.rst b/Mac/BuildScript/README.rst index 4f74e7dc00520a..e44e74f3a49234 100644 --- a/Mac/BuildScript/README.rst +++ b/Mac/BuildScript/README.rst @@ -17,7 +17,7 @@ Gatekeeper download quarantine, the final package must be signed with a valid Apple Developer ID certificate using productsign. Starting with macOS 10.15 Catalina, Gatekeeper now also requires that installer packages are submitted to and pass Apple's automated -notarization service using the altool command. To pass notarization, +notarization service using the ``notarytool`` command. To pass notarization, the binaries included in the package must be built with at least the macOS 10.9 SDK, must now be signed with the codesign utility, and executables must opt in to the hardened run time option with @@ -27,7 +27,7 @@ available in the on-line Apple Developer Documentation and man pages. A goal of PSF-provided (python.org) Python binaries for macOS is to support a wide-range of operating system releases with one set of binaries. Currently, the oldest release supported by python.org -binaries is macOS 10.9; it is still possible to build Python and +binaries is macOS 10.9; it should still be possible to build Python and Python installers on older versions of macOS but we not regularly test on those systems nor provide binaries for them. @@ -49,20 +49,17 @@ Starting with 3.9.1, Python fully supports macOS "weaklinking", meaning it is now possible to build a Python on a current macOS version with a deployment target of an earlier macOS system. For 3.9.1 and later systems, we provide a "macOS 64-bit universal2 installer" -variant, currently build on macOS 11 Big Sur with fat binaries +variant, currently built on macOS 11 Big Sur with fat binaries natively supporting both Apple Silicon (arm64) and Intel-64 (x86_64) Macs running macOS 10.9 or later. -The legacy "macOS 64-bit Intel installer" variant is expected to -be retired prior to the end of 3.9.x support. - build-installer.py requires Apple Developer tools, either from the Command Line Tools package or from a full Xcode installation. You should use the most recent version of either for the operating system version in use. (One notable exception: on macOS 10.6, Snow Leopard, use Xcode 3, not Xcode 4 which was released later in the 10.6 support cycle.) build-installer.py also must be run -with recent versions of Python 3.x or 2.7. On older systems, +with recent versions of Python 3.x. On older systems, due to changes in TLS practices, it may be easier to manually download and cache third-party source distributions used by build-installer.py rather than have it attempt to automatically @@ -76,12 +73,12 @@ download them. - builds the following third-party libraries - * OpenSSL 1.1.1 - * Tcl/Tk 8.6 + * OpenSSL 3.0.x + * Tcl/Tk 8.6.x * NCurses * SQLite * XZ - * libffi + * mpdecimal - uses system-supplied versions of third-party libraries @@ -97,35 +94,6 @@ download them. * ``MACOSX_DEPLOYMENT_TARGET=10.9`` * Apple ``clang`` -2. legacy Intel 64-bit, x86_64, for OS X 10.9 (and later):: - - /path/to/bootstrap/python3 build-installer.py \ - --universal-archs=intel-64 \ - --dep-target=10.9 - - - builds the following third-party libraries - - * OpenSSL 1.1.1 - * Tcl/Tk 8.6 - * NCurses - * SQLite - * XZ - * libffi - - - uses system-supplied versions of third-party libraries - - * readline module links with Apple BSD editline (libedit) - * zlib - * bz2 - - - recommended build environment: - - * Mac OS X 10.9.5 - * Xcode Command Line Tools 6.2 - * ``MacOSX10.9`` SDK - * ``MACOSX_DEPLOYMENT_TARGET=10.9`` - * Apple ``clang`` - General Prerequisites --------------------- diff --git a/Mac/BuildScript/backport_gh92603_fix.patch b/Mac/BuildScript/backport_gh92603_fix.patch deleted file mode 100644 index 9a37b029650340..00000000000000 --- a/Mac/BuildScript/backport_gh92603_fix.patch +++ /dev/null @@ -1,82 +0,0 @@ -Accepted upstream for release in Tk 8.6.14: -https://core.tcl-lang.org/tk/info/cf3830280b - ---- tk8.6.13/macosx/tkMacOSXWindowEvent.c.orig -+++ tk8.6.13-patched/macosx/tkMacOSXWindowEvent.c -@@ -239,8 +239,8 @@ extern NSString *NSWindowDidOrderOffScreenNotification; - if (winPtr) { - TKContentView *view = [window contentView]; - --#if MAC_OS_X_VERSION_MAX_ALLOWED >= 101500 -- if (@available(macOS 10.15, *)) { -+#if MAC_OS_X_VERSION_MAX_ALLOWED >= 101400 -+ if (@available(macOS 10.14, *)) { - [view viewDidChangeEffectiveAppearance]; - } - #endif -@@ -1237,29 +1237,8 @@ static const char *const accentNames[] = { - } else if (effectiveAppearanceName == NSAppearanceNameDarkAqua) { - TkSendVirtualEvent(tkwin, "DarkAqua", NULL); - } -- if ([NSApp macOSVersion] < 101500) { -- -- /* -- * Mojave cannot handle the KVO shenanigans that we need for the -- * highlight and accent color notifications. -- */ -- -- return; -- } - if (!defaultColor) { - defaultColor = [NSApp macOSVersion] < 110000 ? "Blue" : "Multicolor"; -- preferences = [[NSUserDefaults standardUserDefaults] retain]; -- -- /* -- * AppKit calls this method when the user changes the Accent Color -- * but not when the user changes the Highlight Color. So we register -- * to receive KVO notifications for Highlight Color as well. -- */ -- -- [preferences addObserver:self -- forKeyPath:@"AppleHighlightColor" -- options:NSKeyValueObservingOptionNew -- context:NULL]; - } - NSString *accent = [preferences stringForKey:@"AppleAccentColor"]; - NSArray *words = [[preferences stringForKey:@"AppleHighlightColor"] ---- tk8.6.13/macosx/tkMacOSXWm.c.orig -+++ tk8.6.13-patched/macosx/tkMacOSXWm.c -@@ -1289,6 +1289,11 @@ TkWmDeadWindow( - [NSApp _setMainWindow:nil]; - } - [deadNSWindow close]; -+#if MAC_OS_X_VERSION_MAX_ALLOWED >= 101400 -+ NSUserDefaults *preferences = [NSUserDefaults standardUserDefaults]; -+ [preferences removeObserver:deadNSWindow.contentView -+ forKeyPath:@"AppleHighlightColor"]; -+#endif - [deadNSWindow release]; - - #if DEBUG_ZOMBIES > 1 -@@ -6763,6 +6768,21 @@ TkMacOSXMakeRealWindowExist( - } - TKContentView *contentView = [[TKContentView alloc] - initWithFrame:NSZeroRect]; -+#if MAC_OS_X_VERSION_MAX_ALLOWED >= 101400 -+ NSUserDefaults *preferences = [NSUserDefaults standardUserDefaults]; -+ -+ /* -+ * AppKit calls the viewDidChangeEffectiveAppearance method when the -+ * user changes the Accent Color but not when the user changes the -+ * Highlight Color. So we register to receive KVO notifications for -+ * Highlight Color as well. -+ */ -+ -+ [preferences addObserver:contentView -+ forKeyPath:@"AppleHighlightColor" -+ options:NSKeyValueObservingOptionNew -+ context:NULL]; -+#endif - [window setContentView:contentView]; - [contentView release]; - [window setDelegate:NSApp]; diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 0af90563cbbb2b..f5f0ed44884142 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 3.0.13", - url="https://www.openssl.org/source/openssl-3.0.13.tar.gz", - checksum='88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313', + name="OpenSSL 3.0.15", + url="https://github.com/openssl/openssl/releases/download/openssl-3.0.15/openssl-3.0.15.tar.gz", + checksum='23c666d0edf20f14249b3d8f0368acaee9ab585b09e1de82107c66e1f3ec9533', buildrecipe=build_universal_openssl, configure=None, install=None, @@ -264,11 +264,11 @@ def library_recipes(): tk_patches = ['backport_gh71383_fix.patch', 'tk868_on_10_8_10_9.patch', 'backport_gh110950_fix.patch'] else: - tcl_tk_ver='8.6.13' - tcl_checksum='43a1fae7412f61ff11de2cfd05d28cfc3a73762f354a417c62370a54e2caf066' + tcl_tk_ver='8.6.15' + tcl_checksum='861e159753f2e2fbd6ec1484103715b0be56be3357522b858d3cbb5f893ffef1' - tk_checksum='2e65fa069a23365440a3c56c556b8673b5e32a283800d8d9b257e3f584ce0675' - tk_patches = ['backport_gh92603_fix.patch', 'backport_gh71383_fix.patch', 'backport_gh110950_fix.patch'] + tk_checksum='550969f35379f952b3020f3ab7b9dd5bfd11c1ef7c9b7c6a75f5c49aca793fec' + tk_patches = [] base_url = "https://prdownloads.sourceforge.net/tcl/{what}{version}-src.tar.gz" @@ -359,9 +359,9 @@ def library_recipes(): ), ), dict( - name="SQLite 3.45.1", - url="https://sqlite.org/2024/sqlite-autoconf-3450100.tar.gz", - checksum="cd9c27841b7a5932c9897651e20b86c701dd740556989b01ca596fcfa3d49a0a", + name="SQLite 3.45.3", + url="https://sqlite.org/2024/sqlite-autoconf-3450300.tar.gz", + checksum="b2809ca53124c19c60f42bf627736eae011afdcc205bb48270a5ee9a38191531", extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' @@ -378,6 +378,15 @@ def library_recipes(): '--disable-dependency-tracking', ] ), + dict( + name="libmpdec 4.0.0", + url="https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-4.0.0.tar.gz", + checksum="942445c3245b22730fd41a67a7c5c231d11cb1b9936b9c0f76334fb7d0b4468c", + configure_pre=[ + "--disable-cxx", + "MACHINE=universal", + ] + ), ]) if not PYTHON_3: @@ -1150,6 +1159,7 @@ def buildPython(): print(" NOTE: --with-mimalloc=no pending resolution of weak linking issues") runCommand("%s -C --enable-framework --enable-universalsdk=/ " "--with-mimalloc=no " + "--with-system-libmpdec " "--with-universal-archs=%s " "%s " "%s " diff --git a/Mac/BuildScript/resources/License.rtf b/Mac/BuildScript/resources/License.rtf index 1255d1ce48ed6c..b5cb8ec41c86e2 100644 --- a/Mac/BuildScript/resources/License.rtf +++ b/Mac/BuildScript/resources/License.rtf @@ -64,7 +64,7 @@ Some software incorporated into Python is under different licenses. The licenses \f1\b0 \ 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated documentation.\ \ -2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, i.e., "Copyright \'a9 2001-2020 Python Software Foundation; All Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.\ +2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, i.e., "Copyright \'a9 2001 Python Software Foundation; All Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.\ \ 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in any such work a brief summary of the changes made to Python.\ \ diff --git a/Mac/BuildScript/resources/ReadMe.rtf b/Mac/BuildScript/resources/ReadMe.rtf index efd76b9b1ae64b..ee5ba4707dfea4 100644 --- a/Mac/BuildScript/resources/ReadMe.rtf +++ b/Mac/BuildScript/resources/ReadMe.rtf @@ -1,4 +1,4 @@ -{\rtf1\ansi\ansicpg1252\cocoartf2709 +{\rtf1\ansi\ansicpg1252\cocoartf2761 \cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;\f2\fswiss\fcharset0 Helvetica-Oblique; \f3\fmodern\fcharset0 CourierNewPSMT;\f4\fmodern\fcharset0 Courier;} {\colortbl;\red255\green255\blue255;} @@ -11,7 +11,7 @@ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 \f1\b \cf0 NOTE: -\f0\b0 This is an alpha preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ +\f0\b0 This is a beta preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 \cf0 \ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 @@ -69,7 +69,7 @@ Due to new security checks on macOS 10.15 Catalina, when launching IDLE macOS ma \f1\b \ul Apple Silicon Mac support\ \f0\b0 \ulnone \ -On Apple Silicon Macs, it is possible to run Python either with native ARM64 code or under Intel 64 emulation using Rosetta2. This option might be useful for testing or if binary wheels are not yet available with native ARM64 binaries. To easily force Python to run in emulation mode, invoke it from a command line shell with the +On Apple Silicon Macs, it is possible to run Python either with native ARM64 code or under Intel 64 emulation using Rosetta2. This option might be useful for testing or if binary wheels are not yet available with native ARM64 binaries. To easily force Python to run in emulation mode, invoke it from a command line shell with the \f4 python3-intel64 \f0 command instead of just \f4 python3 diff --git a/Mac/BuildScript/resources/Welcome.rtf b/Mac/BuildScript/resources/Welcome.rtf index 79851e1f4a69cc..49d6e22286be26 100644 --- a/Mac/BuildScript/resources/Welcome.rtf +++ b/Mac/BuildScript/resources/Welcome.rtf @@ -1,4 +1,4 @@ -{\rtf1\ansi\ansicpg1252\cocoartf2709 +{\rtf1\ansi\ansicpg1252\cocoartf2761 \cocoascreenfonts1\cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;\f2\fmodern\fcharset0 CourierNewPSMT; } {\colortbl;\red255\green255\blue255;} @@ -26,5 +26,5 @@ At the end of this install, click on \ \f1\b NOTE: -\f0\b0 This is an alpha test preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ +\f0\b0 This is a beta test preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ } \ No newline at end of file diff --git a/Mac/BuildScript/scripts/postflight.patch-profile b/Mac/BuildScript/scripts/postflight.patch-profile index 68b8e4bb044e10..9caf62211ddd16 100755 --- a/Mac/BuildScript/scripts/postflight.patch-profile +++ b/Mac/BuildScript/scripts/postflight.patch-profile @@ -77,16 +77,17 @@ bash) fi ;; fish) - CONFIG_DIR="${HOME}/.config/fish" - RC="${CONFIG_DIR}/config.fish" + CONFIG_DIR="${HOME}/.config/fish/conf.d/" + RC="${CONFIG_DIR}/python-${PYVER}.fish" mkdir -p "$CONFIG_DIR" if [ -f "${RC}" ]; then cp -fp "${RC}" "${RC}.pysave" fi - echo "" >> "${RC}" - echo "# Setting PATH for Python ${PYVER}" >> "${RC}" - echo "# The original version is saved in ${RC}.pysave" >> "${RC}" - echo "set -x PATH \"${PYTHON_ROOT}/bin\" \"\$PATH\"" >> "${RC}" + echo "# Setting PATH for Python ${PYVER}" > "${RC}" + if [ -f "${RC}.pysave" ]; then + echo "# The original version is saved in ${RC}.pysave" >> "${RC}" + fi + echo "fish_add_path -g \"${PYTHON_ROOT}/bin\"" >> "${RC}" if [ `id -ur` = 0 ]; then chown "${USER}" "${RC}" fi diff --git a/Mac/PythonLauncher/Info.plist.in b/Mac/PythonLauncher/Info.plist.in index 233694788ac2b7..ce8f27cd7d4de7 100644 --- a/Mac/PythonLauncher/Info.plist.in +++ b/Mac/PythonLauncher/Info.plist.in @@ -40,9 +40,9 @@ CFBundleExecutable Python Launcher NSHumanReadableCopyright - Copyright © 2001-2024 Python Software Foundation + Copyright © 2001 Python Software Foundation CFBundleGetInfoString - %VERSION%, © 2001-2024 Python Software Foundation + %VERSION%, © 2001 Python Software Foundation CFBundleIconFile PythonLauncher.icns CFBundleIdentifier diff --git a/Mac/Resources/app-store-compliance.patch b/Mac/Resources/app-store-compliance.patch new file mode 100644 index 00000000000000..f4b7decc01cf1f --- /dev/null +++ b/Mac/Resources/app-store-compliance.patch @@ -0,0 +1,29 @@ +diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py +index d6c83a75c1c..19ed4e01091 100644 +--- a/Lib/test/test_urlparse.py ++++ b/Lib/test/test_urlparse.py +@@ -237,11 +237,6 @@ def test_roundtrips(self): + '','',''), + ('git+ssh', 'git@github.com','/user/project.git', + '', '')), +- ('itms-services://?action=download-manifest&url=https://example.com/app', +- ('itms-services', '', '', '', +- 'action=download-manifest&url=https://example.com/app', ''), +- ('itms-services', '', '', +- 'action=download-manifest&url=https://example.com/app', '')), + ('+scheme:path/to/file', + ('', '', '+scheme:path/to/file', '', '', ''), + ('', '', '+scheme:path/to/file', '', '')), +diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py +index 8f724f907d4..148caf742c9 100644 +--- a/Lib/urllib/parse.py ++++ b/Lib/urllib/parse.py +@@ -59,7 +59,7 @@ + 'imap', 'wais', 'file', 'mms', 'https', 'shttp', + 'snews', 'prospero', 'rtsp', 'rtsps', 'rtspu', 'rsync', + 'svn', 'svn+ssh', 'sftp', 'nfs', 'git', 'git+ssh', +- 'ws', 'wss', 'itms-services'] ++ 'ws', 'wss'] + + uses_params = ['', 'ftp', 'hdl', 'prospero', 'http', 'imap', + 'https', 'shttp', 'rtsp', 'rtsps', 'rtspu', 'sip', diff --git a/Makefile.pre.in b/Makefile.pre.in index ee65ecd918ce2a..67acf0fc520087 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -40,8 +40,10 @@ LINKCC= @LINKCC@ AR= @AR@ READELF= @READELF@ SOABI= @SOABI@ +ABIFLAGS= @ABIFLAGS@ +ABI_THREAD= @ABI_THREAD@ LDVERSION= @LDVERSION@ -MODULE_LDFLAGS=@MODULE_LDFLAGS@ +LIBPYTHON=@LIBPYTHON@ GITVERSION= @GITVERSION@ GITTAG= @GITTAG@ GITBRANCH= @GITBRANCH@ @@ -139,6 +141,13 @@ prefix= @prefix@ # Install prefix for architecture-dependent files exec_prefix= @exec_prefix@ +# For cross compilation, we distinguish between "prefix" (where we install the +# files) and "host_prefix" (where getpath.c expects to find the files at +# runtime) +host_prefix= @host_prefix@ +host_exec_prefix= @host_exec_prefix@ + + # Install prefix for data files datarootdir= @datarootdir@ @@ -150,7 +159,6 @@ INCLUDEDIR= @includedir@ CONFINCLUDEDIR= $(exec_prefix)/include PLATLIBDIR= @PLATLIBDIR@ SCRIPTDIR= $(prefix)/$(PLATLIBDIR) -ABIFLAGS= @ABIFLAGS@ # executable name for shebangs EXENAME= $(BINDIR)/python$(LDVERSION)$(EXE) # Variable used by ensurepip @@ -158,7 +166,7 @@ WHEEL_PKG_DIR= @WHEEL_PKG_DIR@ # Detailed destination directories BINLIBDEST= @BINLIBDEST@ -LIBDEST= $(SCRIPTDIR)/python$(VERSION) +LIBDEST= $(SCRIPTDIR)/python$(VERSION)$(ABI_THREAD) INCLUDEPY= $(INCLUDEDIR)/python$(LDVERSION) CONFINCLUDEPY= $(CONFINCLUDEDIR)/python$(LDVERSION) @@ -167,7 +175,7 @@ SHLIB_SUFFIX= @SHLIB_SUFFIX@ EXT_SUFFIX= @EXT_SUFFIX@ LDSHARED= @LDSHARED@ $(PY_LDFLAGS) BLDSHARED= @BLDSHARED@ $(PY_CORE_LDFLAGS) -LDCXXSHARED= @LDCXXSHARED@ +LDCXXSHARED= @LDCXXSHARED@ $(PY_LDFLAGS) DESTSHARED= $(BINLIBDEST)/lib-dynload # List of exported symbols for AIX @@ -178,6 +186,9 @@ EXPORTSFROM= @EXPORTSFROM@ EXE= @EXEEXT@ BUILDEXE= @BUILDEXEEXT@ +# Name of the patch file to apply for app store compliance +APP_STORE_COMPLIANCE_PATCH=@APP_STORE_COMPLIANCE_PATCH@ + # Short name and location for Mac OS X Python framework UNIVERSALSDK=@UNIVERSALSDK@ PYTHONFRAMEWORK= @PYTHONFRAMEWORK@ @@ -186,12 +197,18 @@ PYTHONFRAMEWORKPREFIX= @PYTHONFRAMEWORKPREFIX@ PYTHONFRAMEWORKINSTALLDIR= @PYTHONFRAMEWORKINSTALLDIR@ PYTHONFRAMEWORKINSTALLNAMEPREFIX= @PYTHONFRAMEWORKINSTALLNAMEPREFIX@ RESSRCDIR= @RESSRCDIR@ -# Deployment target selected during configure, to be checked +# macOS deployment target selected during configure, to be checked # by distutils. The export statement is needed to ensure that the # deployment target is active during build. MACOSX_DEPLOYMENT_TARGET=@CONFIGURE_MACOSX_DEPLOYMENT_TARGET@ @EXPORT_MACOSX_DEPLOYMENT_TARGET@export MACOSX_DEPLOYMENT_TARGET +# iOS Deployment target selected during configure. Unlike macOS, the iOS +# deployment target is controlled using `-mios-version-min` arguments added to +# CFLAGS and LDFLAGS by the configure script. This variable is not used during +# the build, and is only listed here so it will be included in sysconfigdata. +IPHONEOS_DEPLOYMENT_TARGET=@IPHONEOS_DEPLOYMENT_TARGET@ + # Option to install to strip binaries STRIPFLAG=-s @@ -211,6 +228,12 @@ ENSUREPIP= @ENSUREPIP@ LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a LIBEXPAT_A= Modules/expat/libexpat.a LIBHACL_SHA2_A= Modules/_hacl/libHacl_Hash_SHA2.a +LIBHACL_BLAKE2_A= Modules/_hacl/libHacl_Hash_Blake2.a +LIBHACL_CFLAGS=@LIBHACL_CFLAGS@ +LIBHACL_SIMD128_FLAGS=@LIBHACL_SIMD128_FLAGS@ +LIBHACL_SIMD256_FLAGS=@LIBHACL_SIMD256_FLAGS@ +LIBHACL_SIMD128_OBJS=@LIBHACL_SIMD128_OBJS@ +LIBHACL_SIMD256_OBJS=@LIBHACL_SIMD256_OBJS@ # Module state, compiler flags and linker flags # Empty CFLAGS and LDFLAGS are omitted. @@ -227,6 +250,9 @@ LIBHACL_SHA2_A= Modules/_hacl/libHacl_Hash_SHA2.a # Default zoneinfo.TZPATH. Added here to expose it in sysconfig.get_config_var TZPATH=@TZPATH@ +# If to install mimalloc headers +INSTALL_MIMALLOC=@INSTALL_MIMALLOC@ + # Modes for directories, executables and data files created by the # install process. Default to user-only-writable for all file types. DIRMODE= 755 @@ -243,10 +269,6 @@ SRCDIRS= @SRCDIRS@ # Other subdirectories SUBDIRSTOO= Include Lib Misc -# assets for Emscripten browser builds -WASM_ASSETS_DIR=.$(prefix) -WASM_STDLIB=$(WASM_ASSETS_DIR)/lib/python$(VERSION)/os.py - # Files and directories to be distributed CONFIGFILES= configure configure.ac acconfig.h pyconfig.h.in Makefile.pre.in DISTFILES= README.rst ChangeLog $(CONFIGFILES) @@ -410,6 +432,7 @@ PYTHON_OBJS= \ Python/brc.o \ Python/ceval.o \ Python/codecs.o \ + Python/codegen.o \ Python/compile.o \ Python/context.o \ Python/critical_section.o \ @@ -433,8 +456,11 @@ PYTHON_OBJS= \ Python/hashtable.o \ Python/import.o \ Python/importdl.o \ + Python/index_pool.o \ Python/initconfig.o \ + Python/interpconfig.o \ Python/instrumentation.o \ + Python/instruction_sequence.o \ Python/intrinsics.o \ Python/jit.o \ Python/legacy_tracing.o \ @@ -462,12 +488,14 @@ PYTHON_OBJS= \ Python/qsbr.o \ Python/bootstrap_hash.o \ Python/specialize.o \ + Python/stackrefs.o \ Python/structmember.o \ Python/symtable.o \ Python/sysmodule.o \ Python/thread.o \ Python/traceback.o \ Python/tracemalloc.o \ + Python/uniqueid.o \ Python/getopt.o \ Python/pystrcmp.o \ Python/pystrtod.o \ @@ -477,6 +505,7 @@ PYTHON_OBJS= \ Python/fileutils.o \ Python/suggestions.o \ Python/perf_trampoline.o \ + Python/perf_jit_trampoline.o \ Python/$(DYNLOADFILE) \ $(LIBOBJS) \ $(MACHDEP_OBJS) \ @@ -507,7 +536,6 @@ OBJECT_OBJS= \ Objects/floatobject.o \ Objects/frameobject.o \ Objects/funcobject.o \ - Objects/interpreteridobject.o \ Objects/iterobject.o \ Objects/listobject.o \ Objects/longobject.o \ @@ -620,7 +648,9 @@ LIBEXPAT_HEADERS= \ Modules/expat/utf8tab.h \ Modules/expat/xmlrole.h \ Modules/expat/xmltok.h \ - Modules/expat/xmltok_impl.h + Modules/expat/xmltok_impl.h \ + Modules/expat/xmltok_impl.c \ + Modules/expat/xmltok_ns.c ########################################################################## # hashlib's HACL* library @@ -628,6 +658,13 @@ LIBEXPAT_HEADERS= \ LIBHACL_SHA2_OBJS= \ Modules/_hacl/Hacl_Hash_SHA2.o +LIBHACL_BLAKE2_OBJS= \ + Modules/_hacl/Hacl_Hash_Blake2s.o \ + Modules/_hacl/Hacl_Hash_Blake2b.o \ + Modules/_hacl/Lib_Memzero0.o \ + $(LIBHACL_SIMD128_OBJS) \ + $(LIBHACL_SIMD256_OBJS) + LIBHACL_HEADERS= \ Modules/_hacl/include/krml/FStar_UInt128_Verified.h \ Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h \ @@ -643,6 +680,18 @@ LIBHACL_SHA2_HEADERS= \ Modules/_hacl/internal/Hacl_Hash_SHA2.h \ $(LIBHACL_HEADERS) +LIBHACL_BLAKE2_HEADERS= \ + Modules/_hacl/Hacl_Hash_Blake2b.h \ + Modules/_hacl/Hacl_Hash_Blake2s.h \ + Modules/_hacl/Hacl_Hash_Blake2s_Simd128.h \ + Modules/_hacl/Hacl_Hash_Blake2b_Simd256.h \ + Modules/_hacl/internal/Hacl_Hash_Blake2b.h \ + Modules/_hacl/internal/Hacl_Hash_Blake2s.h \ + Modules/_hacl/internal/Hacl_Impl_Blake2_Constants.h \ + Modules/_hacl/internal/Hacl_Hash_Blake2s_Simd128.h \ + Modules/_hacl/internal/Hacl_Hash_Blake2b_Simd256.h \ + $(LIBHACL_HEADERS) + ######################################################################### # Rules @@ -653,14 +702,41 @@ all: @DEF_MAKE_ALL_RULE@ # all. .PHONY: all +# Provide quick help for common Makefile targets. +.PHONY: help +help: + @echo "Run 'make' to build the Python executable and extension modules" + @echo "" + @echo "or 'make ' where is one of:" + @echo " test run the test suite" + @echo " install install built files" + @echo " regen-all regenerate a number of generated source files" + @echo " clinic run Argument Clinic over source files" + @echo "" + @echo " clean to remove build files" + @echo " distclean 'clean' + remove other generated files (patch, exe, etc)" + @echo "" + @echo " recheck rerun configure with last cmdline options" + @echo " reindent reindent .py files in Lib directory" + @echo " tags build a tags file (useful for Emacs and other editors)" + @echo " list-targets list all targets in the Makefile" + +# Display a full list of Makefile targets +.PHONY: list-targets +list-targets: + @grep -E '^[A-Za-z][-A-Za-z0-9]+:' Makefile | awk -F : '{print $$1}' + .PHONY: build_all -build_all: check-clean-src $(BUILDPYTHON) platform sharedmods \ +build_all: check-clean-src check-app-store-compliance $(BUILDPYTHON) platform sharedmods \ gdbhooks Programs/_testembed scripts checksharedmods rundsymutil .PHONY: build_wasm build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ python-config checksharedmods +.PHONY: build_emscripten +build_emscripten: build_wasm web_example + # Check that the source is clean when building out of source. .PHONY: check-clean-src check-clean-src: @@ -677,6 +753,16 @@ check-clean-src: exit 1; \ fi +# Check that the app store compliance patch can be applied (if configured). +# This is checked as a dry-run against the original library sources; +# the patch will be actually applied during the install phase. +.PHONY: check-app-store-compliance +check-app-store-compliance: + @if [ "$(APP_STORE_COMPLIANCE_PATCH)" != "" ]; then \ + patch --dry-run --quiet --force --strip 1 --directory "$(abs_srcdir)" --input "$(abs_srcdir)/$(APP_STORE_COMPLIANCE_PATCH)"; \ + echo "App store compliance patch can be applied."; \ + fi + # Profile generation build must start from a clean tree. profile-clean-stamp: $(MAKE) clean @@ -788,7 +874,7 @@ coverage-lcov: @ # remove 3rd party modules, system headers and internal files with @ # debug, test or dummy functions. @lcov $(COVERAGE_LCOV_OPTIONS) --remove $(COVERAGE_INFO) \ - '*/Modules/_blake2/impl/*' \ + '*/Modules/_hacl/*' \ '*/Modules/_ctypes/libffi*/*' \ '*/Modules/_decimal/libmpdec/*' \ '*/Modules/expat/*' \ @@ -818,8 +904,8 @@ coverage-report: regen-token regen-frozen # Run "Argument Clinic" over all source files .PHONY: clinic -clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --make --exclude Lib/test/clinic.test.c --srcdir $(srcdir) +clinic: check-clean-src + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --force --make --exclude Lib/test/clinic.test.c --srcdir $(srcdir) .PHONY: clinic-tests clinic-tests: check-clean-src $(srcdir)/Lib/test/clinic.test.c @@ -848,11 +934,6 @@ pybuilddir.txt: $(PYTHON_FOR_BUILD_DEPS) exit 1 ; \ fi -# blake2s is auto-generated from blake2b -$(srcdir)/Modules/_blake2/blake2s_impl.c: $(srcdir)/Modules/_blake2/blake2b_impl.c $(srcdir)/Modules/_blake2/blake2b2s.py - $(PYTHON_FOR_REGEN) $(srcdir)/Modules/_blake2/blake2b2s.py - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py -f $@ - # Build static library $(LIBRARY): $(LIBRARY_OBJS) -rm -f $@ @@ -911,7 +992,7 @@ $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK): \ $(LN) -fsn Versions/Current/$(PYTHONFRAMEWORK) $(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK) $(LN) -fsn Versions/Current/Resources $(PYTHONFRAMEWORKDIR)/Resources -# This rule is for iOS, which requires an annoyingly just slighly different +# This rule is for iOS, which requires an annoyingly just slightly different # format for frameworks to macOS. It *doesn't* use a versioned framework, and # the Info.plist must be in the root of the framework. $(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK): \ @@ -935,23 +1016,38 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) else true; \ fi -# wasm32-emscripten browser build -# wasm assets directory is relative to current build dir, e.g. "./usr/local". -# --preload-file turns a relative asset path into an absolute path. +# wasm32-emscripten browser web example + +WEBEX_DIR=$(srcdir)/Tools/wasm/emscripten/web_example/ +web_example/python.html: $(WEBEX_DIR)/python.html + @mkdir -p web_example + @cp $< $@ + +web_example/python.worker.mjs: $(WEBEX_DIR)/python.worker.mjs + @mkdir -p web_example + @cp $< $@ -.PHONY: wasm_stdlib -wasm_stdlib: $(WASM_STDLIB) -$(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ - $(srcdir)/Tools/wasm/wasm_assets.py \ +web_example/server.py: $(WEBEX_DIR)/server.py + @mkdir -p web_example + @cp $< $@ + +WEB_STDLIB=web_example/python$(VERSION)$(ABI_THREAD).zip +$(WEB_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ + $(WEBEX_DIR)/wasm_assets.py \ Makefile pybuilddir.txt Modules/Setup.local - $(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \ - --buildroot . --prefix $(prefix) + $(PYTHON_FOR_BUILD) $(WEBEX_DIR)/wasm_assets.py \ + --buildroot . --prefix $(prefix) -o $@ -python.html: $(srcdir)/Tools/wasm/python.html python.worker.js - @cp $(srcdir)/Tools/wasm/python.html $@ +web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON) + @if test $(HOST_GNU_TYPE) != 'wasm32-unknown-emscripten' ; then \ + echo "Can only build web_example when target is Emscripten" ;\ + exit 1 ;\ + fi + cp python.mjs web_example/python.mjs + cp python.wasm web_example/python.wasm -python.worker.js: $(srcdir)/Tools/wasm/python.worker.js - @cp $(srcdir)/Tools/wasm/python.worker.js $@ +.PHONY: web_example +web_example: web_example/python.mjs web_example/python.worker.mjs web_example/python.html web_example/server.py $(WEB_STDLIB) ############################################################################ # Header files @@ -959,6 +1055,7 @@ python.worker.js: $(srcdir)/Tools/wasm/python.worker.js PYTHON_HEADERS= \ $(srcdir)/Include/Python.h \ $(srcdir)/Include/abstract.h \ + $(srcdir)/Include/audit.h \ $(srcdir)/Include/bltinmodule.h \ $(srcdir)/Include/boolobject.h \ $(srcdir)/Include/bytearrayobject.h \ @@ -967,6 +1064,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/codecs.h \ $(srcdir)/Include/compile.h \ $(srcdir)/Include/complexobject.h \ + $(srcdir)/Include/critical_section.h \ $(srcdir)/Include/descrobject.h \ $(srcdir)/Include/dictobject.h \ $(srcdir)/Include/dynamic_annotations.h \ @@ -979,16 +1077,17 @@ PYTHON_HEADERS= \ $(srcdir)/Include/frameobject.h \ $(srcdir)/Include/genericaliasobject.h \ $(srcdir)/Include/import.h \ - $(srcdir)/Include/interpreteridobject.h \ $(srcdir)/Include/intrcheck.h \ $(srcdir)/Include/iterobject.h \ $(srcdir)/Include/listobject.h \ + $(srcdir)/Include/lock.h \ $(srcdir)/Include/longobject.h \ $(srcdir)/Include/marshal.h \ $(srcdir)/Include/memoryobject.h \ $(srcdir)/Include/methodobject.h \ $(srcdir)/Include/modsupport.h \ $(srcdir)/Include/moduleobject.h \ + $(srcdir)/Include/monitoring.h \ $(srcdir)/Include/object.h \ $(srcdir)/Include/objimpl.h \ $(srcdir)/Include/opcode.h \ @@ -1018,6 +1117,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/pythread.h \ $(srcdir)/Include/pytypedefs.h \ $(srcdir)/Include/rangeobject.h \ + $(srcdir)/Include/refcount.h \ $(srcdir)/Include/setobject.h \ $(srcdir)/Include/sliceobject.h \ $(srcdir)/Include/structmember.h \ @@ -1034,6 +1134,7 @@ PYTHON_HEADERS= \ $(PARSER_HEADERS) \ \ $(srcdir)/Include/cpython/abstract.h \ + $(srcdir)/Include/cpython/audit.h \ $(srcdir)/Include/cpython/bytearrayobject.h \ $(srcdir)/Include/cpython/bytesobject.h \ $(srcdir)/Include/cpython/cellobject.h \ @@ -1043,6 +1144,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/compile.h \ $(srcdir)/Include/cpython/complexobject.h \ $(srcdir)/Include/cpython/context.h \ + $(srcdir)/Include/cpython/critical_section.h \ $(srcdir)/Include/cpython/descrobject.h \ $(srcdir)/Include/cpython/dictobject.h \ $(srcdir)/Include/cpython/fileobject.h \ @@ -1053,16 +1155,17 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/genobject.h \ $(srcdir)/Include/cpython/import.h \ $(srcdir)/Include/cpython/initconfig.h \ - $(srcdir)/Include/cpython/interpreteridobject.h \ $(srcdir)/Include/cpython/listobject.h \ + $(srcdir)/Include/cpython/lock.h \ $(srcdir)/Include/cpython/longintrepr.h \ $(srcdir)/Include/cpython/longobject.h \ $(srcdir)/Include/cpython/memoryobject.h \ $(srcdir)/Include/cpython/methodobject.h \ + $(srcdir)/Include/cpython/modsupport.h \ + $(srcdir)/Include/cpython/monitoring.h \ $(srcdir)/Include/cpython/object.h \ $(srcdir)/Include/cpython/objimpl.h \ $(srcdir)/Include/cpython/odictobject.h \ - $(srcdir)/Include/cpython/optimizer.h \ $(srcdir)/Include/cpython/picklebufobject.h \ $(srcdir)/Include/cpython/pthread_stubs.h \ $(srcdir)/Include/cpython/pyatomic.h \ @@ -1081,7 +1184,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/pythonrun.h \ $(srcdir)/Include/cpython/pythread.h \ $(srcdir)/Include/cpython/setobject.h \ - $(srcdir)/Include/cpython/sysmodule.h \ $(srcdir)/Include/cpython/traceback.h \ $(srcdir)/Include/cpython/tracemalloc.h \ $(srcdir)/Include/cpython/tupleobject.h \ @@ -1096,6 +1198,8 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_ast.h \ $(srcdir)/Include/internal/pycore_ast_state.h \ $(srcdir)/Include/internal/pycore_atexit.h \ + $(srcdir)/Include/internal/pycore_audit.h \ + $(srcdir)/Include/internal/pycore_backoff.h \ $(srcdir)/Include/internal/pycore_bitutils.h \ $(srcdir)/Include/internal/pycore_blocks_output_buffer.h \ $(srcdir)/Include/internal/pycore_brc.h \ @@ -1103,6 +1207,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_bytesobject.h \ $(srcdir)/Include/internal/pycore_call.h \ $(srcdir)/Include/internal/pycore_capsule.h \ + $(srcdir)/Include/internal/pycore_cell.h \ $(srcdir)/Include/internal/pycore_ceval.h \ $(srcdir)/Include/internal/pycore_ceval_state.h \ $(srcdir)/Include/internal/pycore_code.h \ @@ -1113,6 +1218,8 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_context.h \ $(srcdir)/Include/internal/pycore_critical_section.h \ $(srcdir)/Include/internal/pycore_crossinterp.h \ + $(srcdir)/Include/internal/pycore_crossinterp_data_registry.h \ + $(srcdir)/Include/internal/pycore_debug_offsets.h \ $(srcdir)/Include/internal/pycore_descrobject.h \ $(srcdir)/Include/internal/pycore_dict.h \ $(srcdir)/Include/internal/pycore_dict_state.h \ @@ -1125,6 +1232,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_format.h \ $(srcdir)/Include/internal/pycore_frame.h \ $(srcdir)/Include/internal/pycore_freelist.h \ + $(srcdir)/Include/internal/pycore_freelist_state.h \ $(srcdir)/Include/internal/pycore_function.h \ $(srcdir)/Include/internal/pycore_gc.h \ $(srcdir)/Include/internal/pycore_genobject.h \ @@ -1135,11 +1243,12 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_global_strings.h \ $(srcdir)/Include/internal/pycore_hamt.h \ $(srcdir)/Include/internal/pycore_hashtable.h \ - $(srcdir)/Include/internal/pycore_identifier.h \ $(srcdir)/Include/internal/pycore_import.h \ $(srcdir)/Include/internal/pycore_importdl.h \ + $(srcdir)/Include/internal/pycore_index_pool.h \ $(srcdir)/Include/internal/pycore_initconfig.h \ $(srcdir)/Include/internal/pycore_instruments.h \ + $(srcdir)/Include/internal/pycore_instruction_sequence.h \ $(srcdir)/Include/internal/pycore_interp.h \ $(srcdir)/Include/internal/pycore_intrinsics.h \ $(srcdir)/Include/internal/pycore_jit.h \ @@ -1154,6 +1263,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_namespace.h \ $(srcdir)/Include/internal/pycore_object.h \ $(srcdir)/Include/internal/pycore_object_alloc.h \ + $(srcdir)/Include/internal/pycore_object_deferred.h \ $(srcdir)/Include/internal/pycore_object_stack.h \ $(srcdir)/Include/internal/pycore_object_state.h \ $(srcdir)/Include/internal/pycore_obmalloc.h \ @@ -1190,12 +1300,14 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_structseq.h \ $(srcdir)/Include/internal/pycore_symtable.h \ $(srcdir)/Include/internal/pycore_sysmodule.h \ + $(srcdir)/Include/internal/pycore_stackref.h \ $(srcdir)/Include/internal/pycore_time.h \ $(srcdir)/Include/internal/pycore_token.h \ $(srcdir)/Include/internal/pycore_traceback.h \ $(srcdir)/Include/internal/pycore_tracemalloc.h \ $(srcdir)/Include/internal/pycore_tstate.h \ $(srcdir)/Include/internal/pycore_tuple.h \ + $(srcdir)/Include/internal/pycore_uniqueid.h \ $(srcdir)/Include/internal/pycore_typeobject.h \ $(srcdir)/Include/internal/pycore_typevarobject.h \ $(srcdir)/Include/internal/pycore_ucnhash.h \ @@ -1283,8 +1395,9 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS) $(AR) $(ARFLAGS) $@ $(LIBEXPAT_OBJS) ########################################################################## -# Build HACL* static libraries for hashlib: libHacl_Hash_SHA2.a -LIBHACL_CFLAGS=-I$(srcdir)/Modules/_hacl/include -D_BSD_SOURCE -D_DEFAULT_SOURCE $(PY_STDMODULE_CFLAGS) $(CCSHARED) +# Build HACL* static libraries for hashlib: libHacl_Hash_SHA2.a, and +# libHacl_Blake2.a -- the contents of the latter vary depending on whether we +# have the ability to compile vectorized versions Modules/_hacl/Hacl_Hash_SHA2.o: $(srcdir)/Modules/_hacl/Hacl_Hash_SHA2.c $(LIBHACL_SHA2_HEADERS) $(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_SHA2.c @@ -1293,6 +1406,31 @@ $(LIBHACL_SHA2_A): $(LIBHACL_SHA2_OBJS) -rm -f $@ $(AR) $(ARFLAGS) $@ $(LIBHACL_SHA2_OBJS) +Modules/_hacl/Hacl_Hash_Blake2s.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s.c $(LIBHACL_BLAKE2_HEADERS) + $(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s.c + +Modules/_hacl/Hacl_Hash_Blake2b.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b.c $(LIBHACL_BLAKE2_HEADERS) + $(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b.c + +Modules/_hacl/Hacl_Hash_Blake2s_Simd128.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c $(LIBHACL_BLAKE2_HEADERS) + $(CC) -c $(LIBHACL_CFLAGS) $(LIBHACL_SIMD128_FLAGS) -DHACL_CAN_COMPILE_VEC128 -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c + +Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.c $(LIBHACL_BLAKE2_HEADERS) + $(CC) -c $(LIBHACL_CFLAGS) $(LIBHACL_SIMD128_FLAGS) -DHACL_CAN_COMPILE_VEC128 -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.c + +Modules/_hacl/Hacl_Hash_Blake2b_Simd256.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c $(LIBHACL_BLAKE2_HEADERS) + $(CC) -c $(LIBHACL_CFLAGS) $(LIBHACL_SIMD256_FLAGS) -DHACL_CAN_COMPILE_VEC256 -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c + +Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.c $(LIBHACL_BLAKE2_HEADERS) + $(CC) -c $(LIBHACL_CFLAGS) $(LIBHACL_SIMD256_FLAGS) -DHACL_CAN_COMPILE_VEC256 -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.c + +Modules/_hacl/Lib_Memzero0.o: $(srcdir)/Modules/_hacl/Lib_Memzero0.c $(LIBHACL_BLAKE2_HEADERS) + $(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Lib_Memzero0.c + +$(LIBHACL_BLAKE2_A): $(LIBHACL_BLAKE2_OBJS) + -rm -f $@ + $(AR) $(ARFLAGS) $@ $(LIBHACL_BLAKE2_OBJS) + # create relative links from build/lib.platform/egg.so to Modules/egg.so # pybuilddir.txt is created too late. We cannot use it in Makefile # targets. ln --relative is not portable. @@ -1376,7 +1514,7 @@ Programs/_testembed: Programs/_testembed.o $(LINK_PYTHON_DEPS) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) ############################################################################ -# "Bootstrap Python" used to run deepfreeze.py +# "Bootstrap Python" used to run Programs/_freeze_module.py BOOTSTRAP_HEADERS = \ Python/frozen_modules/importlib._bootstrap.h \ @@ -1395,7 +1533,7 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu # # Freezing is a multi step process. It works differently for standard builds # and cross builds. Standard builds use Programs/_freeze_module and -# _bootstrap_python for freezing and deepfreezing, so users can build Python +# _bootstrap_python for freezing, so users can build Python # without an existing Python installation. Cross builds cannot execute # compiled binaries and therefore rely on an external build Python # interpreter. The build interpreter must have same version and same bytecode @@ -1409,12 +1547,10 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu # 5) create remaining frozen module headers with # ``./_bootstrap_python Programs/_freeze_module.py``. The pure Python # script is used to test the cross compile code path. -# 6) deepfreeze modules with _bootstrap_python # # Cross compile process: # 1) create all frozen module headers with external build Python and # Programs/_freeze_module.py script. -# 2) deepfreeze modules with external build Python. # # FROZEN_FILES_* are auto-generated by Tools/build/freeze_modules.py. @@ -1560,41 +1696,6 @@ regen-frozen: Tools/build/freeze_modules.py $(FROZEN_FILES_IN) $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/freeze_modules.py --frozen-modules @echo "The Makefile was updated, you may need to re-run make." -############################################################################ -# Deepfreeze targets - -DEEPFREEZE_C = Python/deepfreeze/deepfreeze.c -DEEPFREEZE_DEPS=$(srcdir)/Tools/build/deepfreeze.py Include/internal/pycore_global_strings.h $(FREEZE_MODULE_DEPS) $(FROZEN_FILES_OUT) - -# BEGIN: deepfreeze modules -$(DEEPFREEZE_C): $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/build/deepfreeze.py \ - Python/frozen_modules/importlib._bootstrap.h:importlib._bootstrap \ - Python/frozen_modules/importlib._bootstrap_external.h:importlib._bootstrap_external \ - Python/frozen_modules/zipimport.h:zipimport \ - Python/frozen_modules/abc.h:abc \ - Python/frozen_modules/codecs.h:codecs \ - Python/frozen_modules/io.h:io \ - Python/frozen_modules/_collections_abc.h:_collections_abc \ - Python/frozen_modules/_sitebuiltins.h:_sitebuiltins \ - Python/frozen_modules/genericpath.h:genericpath \ - Python/frozen_modules/ntpath.h:ntpath \ - Python/frozen_modules/posixpath.h:posixpath \ - Python/frozen_modules/os.h:os \ - Python/frozen_modules/site.h:site \ - Python/frozen_modules/stat.h:stat \ - Python/frozen_modules/importlib.util.h:importlib.util \ - Python/frozen_modules/importlib.machinery.h:importlib.machinery \ - Python/frozen_modules/runpy.h:runpy \ - Python/frozen_modules/__hello__.h:__hello__ \ - Python/frozen_modules/__phello__.h:__phello__ \ - Python/frozen_modules/__phello__.ham.h:__phello__.ham \ - Python/frozen_modules/__phello__.ham.eggs.h:__phello__.ham.eggs \ - Python/frozen_modules/__phello__.spam.h:__phello__.spam \ - Python/frozen_modules/frozen_only.h:frozen_only \ - -o Python/deepfreeze/deepfreeze.c -# END: deepfreeze modules - # We keep this renamed target around for folks with muscle memory. .PHONY: regen-importlib regen-importlib: regen-frozen @@ -1623,7 +1724,7 @@ check-abidump: all .PHONY: regen-limited-abi regen-limited-abi: all - $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/stable_abi.py --generate-all $(srcdir)/Misc/stable_abi.toml + $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/stable_abi.py --generate-all ############################################################################ # Regenerate Unicode Data @@ -1641,7 +1742,7 @@ regen-unicodedata: regen-all: regen-cases regen-typeslots \ regen-token regen-ast regen-keyword regen-sre regen-frozen \ regen-pegen-metaparser regen-pegen regen-test-frozenmain \ - regen-test-levenshtein regen-global-objects regen-jit + regen-test-levenshtein regen-global-objects @echo @echo "Note: make regen-stdlib-module-names, make regen-limited-abi, " @echo "make regen-configure, make regen-sbom, and make regen-unicodedata should be run manually" @@ -1664,8 +1765,8 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \ Modules/getpath.o: $(srcdir)/Modules/getpath.c Python/frozen_modules/getpath.h Makefile $(PYTHON_HEADERS) $(CC) -c $(PY_CORE_CFLAGS) -DPYTHONPATH='"$(PYTHONPATH)"' \ - -DPREFIX='"$(prefix)"' \ - -DEXEC_PREFIX='"$(exec_prefix)"' \ + -DPREFIX='"$(host_prefix)"' \ + -DEXEC_PREFIX='"$(host_exec_prefix)"' \ -DVERSION='"$(VERSION)"' \ -DVPATH='"$(VPATH)"' \ -DPLATLIBDIR='"$(PLATLIBDIR)"' \ @@ -1688,14 +1789,18 @@ Modules/pwdmodule.o: $(srcdir)/Modules/pwdmodule.c $(srcdir)/Modules/posixmodule Modules/signalmodule.o: $(srcdir)/Modules/signalmodule.c $(srcdir)/Modules/posixmodule.h -Modules/_xxsubinterpretersmodule.o: $(srcdir)/Modules/_xxsubinterpretersmodule.c $(srcdir)/Modules/_interpreters_common.h +Modules/_interpretersmodule.o: $(srcdir)/Modules/_interpretersmodule.c $(srcdir)/Modules/_interpreters_common.h -Modules/_xxinterpqueuesmodule.o: $(srcdir)/Modules/_xxinterpqueuesmodule.c $(srcdir)/Modules/_interpreters_common.h +Modules/_interpqueuesmodule.o: $(srcdir)/Modules/_interpqueuesmodule.c $(srcdir)/Modules/_interpreters_common.h -Modules/_xxinterpchannelsmodule.o: $(srcdir)/Modules/_xxinterpchannelsmodule.c $(srcdir)/Modules/_interpreters_common.h +Modules/_interpchannelsmodule.o: $(srcdir)/Modules/_interpchannelsmodule.c $(srcdir)/Modules/_interpreters_common.h Python/crossinterp.o: $(srcdir)/Python/crossinterp.c $(srcdir)/Python/crossinterp_data_lookup.h $(srcdir)/Python/crossinterp_exceptions.h +Python/initconfig.o: $(srcdir)/Python/initconfig.c $(srcdir)/Python/config_common.h + +Python/interpconfig.o: $(srcdir)/Python/interpconfig.c $(srcdir)/Python/config_common.h + Python/dynload_shlib.o: $(srcdir)/Python/dynload_shlib.c Makefile $(CC) -c $(PY_CORE_CFLAGS) \ -DSOABI='"$(SOABI)"' \ @@ -1803,7 +1908,7 @@ regen-sre: $(srcdir)/Modules/_sre/sre_constants.h \ $(srcdir)/Modules/_sre/sre_targets.h -Python/compile.o Python/symtable.o Python/ast_unparse.o Python/ast.o Python/future.o: $(srcdir)/Include/internal/pycore_ast.h +Python/compile.o Python/codegen.o Python/symtable.o Python/ast_unparse.o Python/ast.o Python/future.o: $(srcdir)/Include/internal/pycore_ast.h $(srcdir)/Include/internal/pycore_ast.h Python/getplatform.o: $(srcdir)/Python/getplatform.c $(CC) -c $(PY_CORE_CFLAGS) -DPLATFORM='"$(MACHDEP)"' -o $@ $(srcdir)/Python/getplatform.c @@ -1835,6 +1940,7 @@ UNICODE_DEPS = \ $(srcdir)/Objects/stringlib/localeutil.h \ $(srcdir)/Objects/stringlib/partition.h \ $(srcdir)/Objects/stringlib/replace.h \ + $(srcdir)/Objects/stringlib/repr.h \ $(srcdir)/Objects/stringlib/split.h \ $(srcdir)/Objects/stringlib/ucs1lib.h \ $(srcdir)/Objects/stringlib/ucs2lib.h \ @@ -1872,41 +1978,78 @@ Objects/obmalloc.o: $(srcdir)/Objects/mimalloc/alloc.c \ Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c + +# Regenerate various files from Python/bytecodes.c +# Pass CASESFLAG=-l to insert #line directives in the output + .PHONY: regen-cases -regen-cases: - # Regenerate various files from Python/bytecodes.c - # Pass CASESFLAG=-l to insert #line directives in the output +regen-cases: \ + regen-opcode-ids regen-opcode-targets regen-uop-ids regen-opcode-metadata-py \ + regen-generated-cases regen-executor-cases regen-optimizer-cases \ + regen-opcode-metadata regen-uop-metadata + +.PHONY: regen-opcode-ids +regen-opcode-ids: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/opcode_id_generator.py \ -o $(srcdir)/Include/opcode_ids.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Include/opcode_ids.h $(srcdir)/Include/opcode_ids.h.new + +.PHONY: regen-opcode-targets +regen-opcode-targets: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/target_generator.py \ -o $(srcdir)/Python/opcode_targets.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new + +.PHONY: regen-uop-ids +regen-uop-ids: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/uop_id_generator.py \ -o $(srcdir)/Include/internal/pycore_uop_ids.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_uop_ids.h $(srcdir)/Include/internal/pycore_uop_ids.h.new + +.PHONY: regen-opcode-metadata-py +regen-opcode-metadata-py: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/py_metadata_generator.py \ -o $(srcdir)/Lib/_opcode_metadata.py.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Lib/_opcode_metadata.py $(srcdir)/Lib/_opcode_metadata.py.new + +.PHONY: regen-generated-cases +regen-generated-cases: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier1_generator.py \ -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new + +.PHONY: regen-executor-cases +regen-executor-cases: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier2_generator.py \ -o $(srcdir)/Python/executor_cases.c.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Python/executor_cases.c.h $(srcdir)/Python/executor_cases.c.h.new + +.PHONY: regen-optimizer-cases +regen-optimizer-cases: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/optimizer_generator.py \ -o $(srcdir)/Python/optimizer_cases.c.h.new \ $(srcdir)/Python/optimizer_bytecodes.c \ $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Python/optimizer_cases.c.h $(srcdir)/Python/optimizer_cases.c.h.new + +.PHONY: regen-opcode-metadata +regen-opcode-metadata: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/opcode_metadata_generator.py \ -o $(srcdir)/Include/internal/pycore_opcode_metadata.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode_metadata.h $(srcdir)/Include/internal/pycore_opcode_metadata.h.new + +.PHONY: regen-uop-metadata +regen-uop-metadata: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/uop_metadata_generator.py -o \ $(srcdir)/Include/internal/pycore_uop_metadata.h.new $(srcdir)/Python/bytecodes.c - $(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new - $(UPDATE_FILE) $(srcdir)/Include/opcode_ids.h $(srcdir)/Include/opcode_ids.h.new - $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_uop_ids.h $(srcdir)/Include/internal/pycore_uop_ids.h.new - $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new - $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode_metadata.h $(srcdir)/Include/internal/pycore_opcode_metadata.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_uop_metadata.h $(srcdir)/Include/internal/pycore_uop_metadata.h.new - $(UPDATE_FILE) $(srcdir)/Python/executor_cases.c.h $(srcdir)/Python/executor_cases.c.h.new - $(UPDATE_FILE) $(srcdir)/Python/optimizer_cases.c.h $(srcdir)/Python/optimizer_cases.c.h.new - $(UPDATE_FILE) $(srcdir)/Lib/_opcode_metadata.py $(srcdir)/Lib/_opcode_metadata.py.new -Python/compile.o: $(srcdir)/Include/internal/pycore_opcode_metadata.h +Python/compile.o Python/codegen.o Python/assemble.o Python/flowgraph.o Python/instruction_sequence.o: \ + $(srcdir)/Include/internal/pycore_compile.h \ + $(srcdir)/Include/internal/pycore_flowgraph.h \ + $(srcdir)/Include/internal/pycore_instruction_sequence.h \ + $(srcdir)/Include/internal/pycore_opcode_metadata.h \ + $(srcdir)/Include/internal/pycore_opcode_utils.h Python/ceval.o: \ $(srcdir)/Python/ceval_macros.h \ @@ -1979,6 +2122,12 @@ cleantest: all # Similar to buildbottest, but use --fast-ci option, instead of --slow-ci. .PHONY: test test: all + $(TESTRUNNER) --fast-ci -u-gui --timeout=$(TESTTIMEOUT) $(TESTOPTS) + +# Run a basic set of regression tests inside the CI. +# This excludes some tests that are particularly resource-intensive. +# Similar to test, but also runs GUI tests. +ci: all $(TESTRUNNER) --fast-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS) # Run the test suite for both architectures in a Universal build on OSX. @@ -1993,6 +2142,36 @@ testuniversal: all $(RUNSHARED) /usr/libexec/oah/translate \ ./$(BUILDPYTHON) -E -m test -j 0 -u all $(TESTOPTS) +# Run the test suite on the iOS simulator. Must be run on a macOS machine with +# a full Xcode install that has an iPhone SE (3rd edition) simulator available. +# This must be run *after* a `make install` has completed the build. The +# `--with-framework-name` argument *cannot* be used when configuring the build. +XCFOLDER:=iOSTestbed.$(MULTIARCH).$(shell date +%s) +.PHONY: testios +testios: + @if test "$(MACHDEP)" != "ios"; then \ + echo "Cannot run the iOS testbed for a non-iOS build."; \ + exit 1;\ + fi + @if test "$(findstring -iphonesimulator,$(MULTIARCH))" != "-iphonesimulator"; then \ + echo "Cannot run the iOS testbed for non-simulator builds."; \ + exit 1;\ + fi + @if test $(PYTHONFRAMEWORK) != "Python"; then \ + echo "Cannot run the iOS testbed with a non-default framework name."; \ + exit 1;\ + fi + @if ! test -d $(PYTHONFRAMEWORKPREFIX); then \ + echo "Cannot find a finalized iOS Python.framework. Have you run 'make install' to finalize the framework build?"; \ + exit 1;\ + fi + + # Clone the testbed project into the XCFOLDER + $(PYTHON_FOR_BUILD) $(srcdir)/iOS/testbed clone --framework $(PYTHONFRAMEWORKPREFIX) "$(XCFOLDER)" + + # Run the testbed project + $(PYTHON_FOR_BUILD) "$(XCFOLDER)" run --verbose -- test -uall --single-process --rerun -W + # Like test, but using --slow-ci which enables all test resources and use # longer timeout. Run an optional pybuildbot.identify script to include # information about the build environment. @@ -2168,10 +2347,10 @@ bininstall: commoninstall altbininstall -if test "$(VERSION)" != "$(LDVERSION)"; then \ rm -f $(DESTDIR)$(BINDIR)/python$(VERSION)-config; \ (cd $(DESTDIR)$(BINDIR); $(LN) -s python$(LDVERSION)-config python$(VERSION)-config); \ - rm -f $(DESTDIR)$(LIBPC)/python-$(LDVERSION).pc; \ - (cd $(DESTDIR)$(LIBPC); $(LN) -s python-$(VERSION).pc python-$(LDVERSION).pc); \ - rm -f $(DESTDIR)$(LIBPC)/python-$(LDVERSION)-embed.pc; \ - (cd $(DESTDIR)$(LIBPC); $(LN) -s python-$(VERSION)-embed.pc python-$(LDVERSION)-embed.pc); \ + rm -f $(DESTDIR)$(LIBPC)/python-$(VERSION).pc; \ + (cd $(DESTDIR)$(LIBPC); $(LN) -s python-$(LDVERSION).pc python-$(VERSION).pc); \ + rm -f $(DESTDIR)$(LIBPC)/python-$(VERSION)-embed.pc; \ + (cd $(DESTDIR)$(LIBPC); $(LN) -s python-$(LDVERSION)-embed.pc python-$(VERSION)-embed.pc); \ fi -rm -f $(DESTDIR)$(BINDIR)/python3-config (cd $(DESTDIR)$(BINDIR); $(LN) -s python$(VERSION)-config python3-config) @@ -2248,9 +2427,12 @@ LIBSUBDIRS= asyncio \ xmlrpc \ zipfile zipfile/_path \ zoneinfo \ + _pyrepl \ __phello__ TESTSUBDIRS= idlelib/idle_test \ test \ + test/test_ast \ + test/test_ast/data \ test/archivetestdata \ test/audiodata \ test/certdata \ @@ -2274,6 +2456,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/support/interpreters \ test/test_asyncio \ test/test_capi \ + test/test_cext \ test/test_concurrent_futures \ test/test_cppext \ test/test_ctypes \ @@ -2281,6 +2464,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_doctest \ test/test_email \ test/test_email/data \ + test/test_free_threading \ test/test_future_stmt \ test/test_gdb \ test/test_import \ @@ -2291,13 +2475,21 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_import/data/circular_imports/subpkg2/parent \ test/test_import/data/package \ test/test_import/data/package2 \ + test/test_import/data/package3 \ + test/test_import/data/package4 \ test/test_import/data/unwritable \ test/test_importlib \ test/test_importlib/builtin \ - test/test_importlib/data \ test/test_importlib/extension \ test/test_importlib/frozen \ test/test_importlib/import_ \ + test/test_importlib/metadata \ + test/test_importlib/metadata/data \ + test/test_importlib/metadata/data/sources \ + test/test_importlib/metadata/data/sources/example \ + test/test_importlib/metadata/data/sources/example/example \ + test/test_importlib/metadata/data/sources/example2 \ + test/test_importlib/metadata/data/sources/example2/example2 \ test/test_importlib/namespace_pkgs \ test/test_importlib/namespace_pkgs/both_portions \ test/test_importlib/namespace_pkgs/both_portions/foo \ @@ -2320,20 +2512,6 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_importlib/namespace_pkgs/project3/parent/child \ test/test_importlib/partial \ test/test_importlib/resources \ - test/test_importlib/resources/data01 \ - test/test_importlib/resources/data01/subdirectory \ - test/test_importlib/resources/data02 \ - test/test_importlib/resources/data02/one \ - test/test_importlib/resources/data02/subdirectory \ - test/test_importlib/resources/data02/subdirectory/subsubdir \ - test/test_importlib/resources/data02/two \ - test/test_importlib/resources/data03 \ - test/test_importlib/resources/data03/namespace \ - test/test_importlib/resources/data03/namespace/portion1 \ - test/test_importlib/resources/data03/namespace/portion2 \ - test/test_importlib/resources/namespacedata01 \ - test/test_importlib/resources/zipdata01 \ - test/test_importlib/resources/zipdata02 \ test/test_importlib/source \ test/test_inspect \ test/test_interpreters \ @@ -2345,6 +2523,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_pathlib \ test/test_peg_generator \ test/test_pydoc \ + test/test_pyrepl \ test/test_sqlite3 \ test/test_tkinter \ test/test_tomllib \ @@ -2366,8 +2545,13 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_tomllib/data/valid/dates-and-times \ test/test_tomllib/data/valid/multiline-basic-str \ test/test_tools \ + test/test_tools/i18n_data \ test/test_ttk \ test/test_unittest \ + test/test_unittest/namespace_test_pkg \ + test/test_unittest/namespace_test_pkg/bar \ + test/test_unittest/namespace_test_pkg/noop \ + test/test_unittest/namespace_test_pkg/noop/no2 \ test/test_unittest/testmock \ test/test_warnings \ test/test_warnings/data \ @@ -2378,10 +2562,15 @@ TESTSUBDIRS= idlelib/idle_test \ test/tkinterdata \ test/tokenizedata \ test/tracedmodules \ + test/translationdata \ + test/translationdata/argparse \ + test/translationdata/getopt \ + test/translationdata/optparse \ test/typinganndata \ test/wheeldata \ test/xmltestdata \ - test/xmltestdata/c14n-20 + test/xmltestdata/c14n-20 \ + test/zipimport_data COMPILEALL_OPTS=-j0 @@ -2453,9 +2642,17 @@ libinstall: all $(srcdir)/Modules/xxmodule.c esac; \ done; \ done - $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py \ - $(DESTDIR)$(LIBDEST); \ + $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py $(DESTDIR)$(LIBDEST); \ + $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfig_vars_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt + @ # If app store compliance has been configured, apply the patch to the + @ # installed library code. The patch has been previously validated against + @ # the original source tree, so we can ignore any errors that are raised + @ # due to files that are missing because of --disable-test-modules etc. + @if [ "$(APP_STORE_COMPLIANCE_PATCH)" != "" ]; then \ + echo "Applying app store compliance patch"; \ + patch --force --reject-file "$(abs_builddir)/app-store-compliance.rej" --strip 2 --directory "$(DESTDIR)$(LIBDEST)" --input "$(abs_srcdir)/$(APP_STORE_COMPLIANCE_PATCH)" || true ; \ + fi @ # Build PYC files for the 3 optimization levels (0, 1, 2) -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ $(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \ @@ -2524,6 +2721,12 @@ inclinstall: $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(INCLUDEPY)/internal; \ else true; \ fi + @if test "$(INSTALL_MIMALLOC)" = "yes"; then \ + if test ! -d $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc; then \ + echo "Creating directory $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc"; \ + $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc; \ + fi; \ + fi @for i in $(srcdir)/Include/*.h; \ do \ echo $(INSTALL_DATA) $$i $(INCLUDEPY); \ @@ -2539,6 +2742,16 @@ inclinstall: echo $(INSTALL_DATA) $$i $(INCLUDEPY)/internal; \ $(INSTALL_DATA) $$i $(DESTDIR)$(INCLUDEPY)/internal; \ done + @if test "$(INSTALL_MIMALLOC)" = "yes"; then \ + echo $(INSTALL_DATA) $(srcdir)/Include/internal/mimalloc/mimalloc.h $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc.h; \ + $(INSTALL_DATA) $(srcdir)/Include/internal/mimalloc/mimalloc.h $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc.h; \ + for i in $(srcdir)/Include/internal/mimalloc/mimalloc/*.h; \ + do \ + echo $(INSTALL_DATA) $$i $(INCLUDEPY)/internal/mimalloc/mimalloc; \ + $(INSTALL_DATA) $$i $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc; \ + done; \ + fi + echo $(INSTALL_DATA) pyconfig.h $(DESTDIR)$(CONFINCLUDEPY)/pyconfig.h $(INSTALL_DATA) pyconfig.h $(DESTDIR)$(CONFINCLUDEPY)/pyconfig.h # Install the library and miscellaneous stuff needed for extending/embedding @@ -2579,8 +2792,8 @@ libainstall: all scripts $(INSTALL_DATA) Modules/Setup.bootstrap $(DESTDIR)$(LIBPL)/Setup.bootstrap $(INSTALL_DATA) Modules/Setup.stdlib $(DESTDIR)$(LIBPL)/Setup.stdlib $(INSTALL_DATA) Modules/Setup.local $(DESTDIR)$(LIBPL)/Setup.local - $(INSTALL_DATA) Misc/python.pc $(DESTDIR)$(LIBPC)/python-$(VERSION).pc - $(INSTALL_DATA) Misc/python-embed.pc $(DESTDIR)$(LIBPC)/python-$(VERSION)-embed.pc + $(INSTALL_DATA) Misc/python.pc $(DESTDIR)$(LIBPC)/python-$(LDVERSION).pc + $(INSTALL_DATA) Misc/python-embed.pc $(DESTDIR)$(LIBPC)/python-$(LDVERSION)-embed.pc $(INSTALL_SCRIPT) $(srcdir)/Modules/makesetup $(DESTDIR)$(LIBPL)/makesetup $(INSTALL_SCRIPT) $(srcdir)/install-sh $(DESTDIR)$(LIBPL)/install-sh $(INSTALL_SCRIPT) python-config.py $(DESTDIR)$(LIBPL)/python-config.py @@ -2712,13 +2925,13 @@ frameworkinstallextras: # subdirectory. The install has put these folders in the same folder as # Python.framework; Move the headers to their final framework-compatible home. .PHONY: frameworkinstallmobileheaders -frameworkinstallmobileheaders: +frameworkinstallmobileheaders: frameworkinstallunversionedstructure inclinstall if test -d $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers; then \ echo "Removing old framework headers"; \ rm -rf $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers; \ fi - mv "$(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include/python$(VERSION)" "$(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers" - $(LN) -fs "../$(PYTHONFRAMEWORKDIR)/Headers" "$(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include/python$(VERSION)" + mv "$(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include/python$(LDVERSION)" "$(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers" + $(LN) -fs "../$(PYTHONFRAMEWORKDIR)/Headers" "$(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include/python$(LDVERSION)" # Build the toplevel Makefile Makefile.pre: $(srcdir)/Makefile.pre.in config.status @@ -2837,16 +3050,19 @@ clean-retain-profile: pycremoval find build -name '*.py[co]' -exec rm -f {} ';' || true -rm -f pybuilddir.txt -rm -f _bootstrap_python - -rm -f python.html python*.js python.data python*.symbols python*.map - -rm -f $(WASM_STDLIB) + -rm -rf web_example python.mjs python.wasm python*.symbols python*.map -rm -f Programs/_testembed Programs/_freeze_module - -rm -f Python/deepfreeze/*.[co] + -rm -rf Python/deepfreeze -rm -f Python/frozen_modules/*.h -rm -f Python/frozen_modules/MANIFEST -rm -f jit_stencils.h -find build -type f -a ! -name '*.gc??' -exec rm -f {} ';' -rm -f Include/pydtrace_probes.h -rm -f profile-gen-stamp + -rm -rf iOS/testbed/Python.xcframework/ios-*/bin + -rm -rf iOS/testbed/Python.xcframework/ios-*/lib + -rm -rf iOS/testbed/Python.xcframework/ios-*/include + -rm -rf iOS/testbed/Python.xcframework/ios-*/Python.framework .PHONY: profile-removal profile-removal: @@ -2872,6 +3088,8 @@ clobber: clean config.cache config.log pyconfig.h Modules/config.c -rm -rf build platform -rm -rf $(PYTHONFRAMEWORKDIR) + -rm -rf iOS/Frameworks + -rm -rf iOSTestbed.* -rm -f python-config.py python-config -rm -rf cross-build @@ -2949,7 +3167,7 @@ patchcheck: all .PHONY: check-limited-abi check-limited-abi: all - $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/stable_abi.py --all $(srcdir)/Misc/stable_abi.toml + $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/stable_abi.py --all .PHONY: update-config update-config: @@ -2975,20 +3193,22 @@ MODULE_CMATH_DEPS=$(srcdir)/Modules/_math.h MODULE_MATH_DEPS=$(srcdir)/Modules/_math.h MODULE_PYEXPAT_DEPS=@LIBEXPAT_INTERNAL@ MODULE_UNICODEDATA_DEPS=$(srcdir)/Modules/unicodedata_db.h $(srcdir)/Modules/unicodename_db.h -MODULE__BLAKE2_DEPS=$(srcdir)/Modules/_blake2/impl/blake2-config.h $(srcdir)/Modules/_blake2/impl/blake2-impl.h $(srcdir)/Modules/_blake2/impl/blake2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2b-ref.c $(srcdir)/Modules/_blake2/impl/blake2b-round.h $(srcdir)/Modules/_blake2/impl/blake2b.c $(srcdir)/Modules/_blake2/impl/blake2s-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2s-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2s-load-xop.h $(srcdir)/Modules/_blake2/impl/blake2s-ref.c $(srcdir)/Modules/_blake2/impl/blake2s-round.h $(srcdir)/Modules/_blake2/impl/blake2s.c $(srcdir)/Modules/_blake2/blake2module.h $(srcdir)/Modules/hashlib.h -MODULE__CTYPES_DEPS=$(srcdir)/Modules/_ctypes/ctypes.h +MODULE__CTYPES_DEPS=$(srcdir)/Modules/_ctypes/ctypes.h $(srcdir)/Modules/_complex.h +MODULE__CTYPES_TEST_DEPS=$(srcdir)/Modules/_ctypes/_ctypes_test_generated.c.h MODULE__CTYPES_MALLOC_CLOSURE=@MODULE__CTYPES_MALLOC_CLOSURE@ MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h -MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_MD5.h Modules/_hacl/Hacl_Hash_MD5.c -MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA1.h Modules/_hacl/Hacl_Hash_SHA1.c +MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_MD5.h Modules/_hacl/internal/Hacl_Hash_MD5.h Modules/_hacl/Hacl_Hash_MD5.c +MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA1.h Modules/_hacl/internal/Hacl_Hash_SHA1.h Modules/_hacl/Hacl_Hash_SHA1.c MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_A) -MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA3.h Modules/_hacl/Hacl_Hash_SHA3.c +MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA3.h Modules/_hacl/internal/Hacl_Hash_SHA3.h Modules/_hacl/Hacl_Hash_SHA3.c +MODULE__BLAKE2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_A) MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h -MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/testcapi_long.h $(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h +MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h +MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h diff --git a/Misc/ACKS b/Misc/ACKS index f01c7a70a65dc5..4901609a178bc3 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -24,6 +24,7 @@ Eitan Adler Anton Afanasyev Ali Afshar Nitika Agarwal +Maxim Ageev Anjani Agrawal Pablo S. Blum de Aguiar Jim Ahlstrom @@ -191,6 +192,7 @@ Finn Bock Paul Boddie Matthew Boedicker Robin Boerdijk +Wannes Boeykens Andra Bogildea Matt Bogosian Nikolay Bogoychev @@ -256,6 +258,7 @@ Colm Buckley Erik de Bueger Jan-Hein Bührman Marc Bürg +Calvin Bui Lars Buitinck Artem Bulgakov Dick Bulterman @@ -314,6 +317,7 @@ Greg Chapman Mitch Chapman Matt Chaput William Chargin +Ben Chatterton Yogesh Chaudhari Gautam Chaudhuri David Chaum @@ -470,6 +474,7 @@ Allen Downey Cesar Douady Dean Draayer Fred L. Drake, Jr. +Mehdi Drissi Derk Drukker John DuBois Paul Dubois @@ -482,6 +487,7 @@ Luke Dunstan Virgil Dupras Bruno Dupuis Andy Dustman +Sayandip Dutta Gary Duzan Eugene Dvurechenski Karmen Dykstra @@ -494,6 +500,7 @@ David Edelsohn John Edmonds Benjamin Edwards Grant Edwards +Vlad Efanov Zvi Effron John Ehresman Tal Einat @@ -517,6 +524,7 @@ Michael Ernst Ben Escoto Andy Eskilsson André Espaze +Lucas Esposito Stefan Esser Nicolas Estibals Jonathan Eunice @@ -606,6 +614,7 @@ Nitin Ganatra Soumendra Ganguly (गङ्गोपाध्याय) Fred Gansevles Paul Ganssle +Tian Gao Lars Marius Garshol Jake Garver Dan Gass @@ -639,6 +648,7 @@ Neil Girdhar Matt Giuca Andrea Giudiceandrea Franz Glasner +Jeff Glass Wim Glenn Michael Goderbauer Karan Goel @@ -725,6 +735,7 @@ Larry Hastings Tim Hatch Zac Hatfield-Dodds Shane Hathaway +Akinori Hattori Michael Haubenwallner Janko Hauser Flavian Hautbois @@ -747,6 +758,7 @@ Kasun Herath Chris Herborth Ivan Herman Jürgen Hermann +Joshua Jay Herman Gary Herron Ernie Hershey Thomas Herve @@ -812,6 +824,7 @@ Tomáš Hrnčiar Miro Hrončok Chiu-Hsiang Hsu Chih-Hao Huang +Xuanteng Huang Christian Hudon Benoît Hudson Lawrence Hudson @@ -926,6 +939,7 @@ Hiroaki Kawai Dmitry Kazakov Brian Kearns Sebastien Keim +Russell Keith-Magee Ryan Kelly Hugo van Kemenade Dan Kenigsberg @@ -943,6 +957,7 @@ Sanyam Khurana Tyler Kieft Mads Kiilerich Jason Killen +Beomsoo Bombs Kim Derek D. Kim Gihwan Kim Jan Kim @@ -1023,6 +1038,7 @@ Erno Kuusela Kabir Kwatra Ross Lagerwall Cameron Laird +Filipe Laíns Loïc Lajeanne Alexander Lakeev David Lam @@ -1091,6 +1107,7 @@ Ivan Levkivskyi Ben Lewis William Lewis Akira Li +Jiahao Li Robert Li Xuanji Li Zekun Li @@ -1113,6 +1130,7 @@ Gregor Lingl Everett Lipman Mirko Liss Alexander Liu +Hui Liu Yuan Liu Nick Lockwood Stephanie Lockwood @@ -1140,6 +1158,7 @@ Mark Lutz Taras Lyapun Jim Lynch Mikael Lyngvig +Ilya Lyubavski Jeff MacDonald John Machin Andrew I MacIntyre @@ -1154,6 +1173,7 @@ Grzegorz Makarewicz David Malcolm Greg Malcolm William Mallard +Cody Maloney Ken Manheimer Vladimir Marangozov Colin Marc @@ -1262,6 +1282,7 @@ Emily Morehouse Derek Morr James A Morrison Martin Morrison +Stephen Morton Derek McTavish Mounce Alessandro Moura Pablo Mouzo @@ -1309,6 +1330,7 @@ Hrvoje Nikšić Gregory Nofi Jesse Noller Bill Noon +Janek Nouvertné Stefan Norberg Tim Northover Joe Norton @@ -1453,6 +1475,7 @@ Michael Pomraning Martin Pool Iustin Pop Claudiu Popa +Nick Pope John Popplewell Matheus Vieira Portela Davin Potts @@ -1541,7 +1564,9 @@ Lisa Roach Carl Robben Ben Roberts Mark Roberts +Tony Roberts Andy Robinson +Izan "TizzySaurus" Robinson Jim Robinson Yolanda Robla Daniel Rocco @@ -1642,6 +1667,7 @@ Scott Schram Robin Schreiber Chad J. Schroeder Simon-Martin Schroeder +Brian Schubert Christian Schubert Sam Schulenburg Andreas Schwab @@ -1660,6 +1686,7 @@ Fred Sells Jiwon Seo Iñigo Serna Joakim Sernbrant +Rodrigo Girão Serrão Roger D. Serwy Jerry Seutter Pete Sevander @@ -1714,6 +1741,7 @@ Ng Pheng Siong Yann Sionneau George Sipe J. Sipprell +Giovanni Siragusa Ngalim Siregar Kragen Sitaker Kaartic Sivaraam @@ -1732,6 +1760,7 @@ Christopher Smith Eric V. Smith Ethan H. Smith Gregory P. Smith +Malcolm Smith Mark Smith Nathaniel J. Smith Roy Smith @@ -1798,6 +1827,7 @@ Reuben Sumner Eryk Sun Sanjay Sundaresan Marek Šuppa +Danica J. Sutherland Hisao Suzuki Kalle Svensson Andrew Svetlov @@ -1893,6 +1923,7 @@ Bill Tutt Fraser Tweedale Doobee R. Tzeck Eren Türkay +Stan Ulbrych Lionel Ulmer Adnan Umer Utkarsh Upadhyay @@ -2049,11 +2080,13 @@ Doug Wyatt Xiang Zhang Robert Xiao Florent Xicluna +Yanbo, Xie Xinhang Xu Arnon Yaari Alakshendra Yadav Hirokazu Yamamoto Masayuki Yamamoto +Zhikang Yan Jingchen Ye Ka-Ping Yee Chi Hsuan Yen diff --git a/Misc/HISTORY b/Misc/HISTORY index b66413277259dc..d68aaa066771fb 100644 --- a/Misc/HISTORY +++ b/Misc/HISTORY @@ -607,7 +607,7 @@ Library MemoryError. - Issue #18473: Fixed 2to3 and 3to2 compatible pickle mappings. Fixed - ambigious reverse mappings. Added many new mappings. Import mapping is no + ambiguous reverse mappings. Added many new mappings. Import mapping is no longer applied to modules already mapped with full name mapping. - Issue #23745: The new email header parser now handles duplicate MIME @@ -2030,7 +2030,7 @@ Library initialization of the unquote_to_bytes() table of the urllib.parse module, to not waste memory if these modules are not used. -- Issue #19157: Include the broadcast address in the usuable hosts for IPv6 +- Issue #19157: Include the broadcast address in the usable hosts for IPv6 in ipaddress. - Issue #11599: When an external command (e.g. compiler) fails, distutils now @@ -2620,7 +2620,7 @@ Library - asyncio: Various improvements and small changes not all covered by issues listed below. E.g. wait_for() now cancels the inner task if - the timeout occcurs; tweaked the set of exported symbols; renamed + the timeout occurs; tweaked the set of exported symbols; renamed Empty/Full to QueueEmpty/QueueFull; "with (yield from lock)" now uses a separate context manager; readexactly() raises if not enough data was read; PTY support tweaks. @@ -3944,7 +3944,7 @@ Library - Issue #18996: TestCase.assertEqual() now more cleverly shorten differing strings in error report. -- Issue #19034: repr() for tkinter.Tcl_Obj now exposes string reperesentation. +- Issue #19034: repr() for tkinter.Tcl_Obj now exposes string representation. - Issue #18978: ``urllib.request.Request`` now allows the method to be indicated on the class and no longer sets it to None in ``__init__``. @@ -3952,7 +3952,7 @@ Library - Issue #18626: the inspect module now offers a basic command line introspection interface (Initial patch by Claudiu Popa) -- Issue #3015: Fixed tkinter with wantobject=False. Any Tcl command call +- Issue #3015: Fixed tkinter with ``wantobjects=False``. Any Tcl command call returned empty string. - Issue #19037: The mailbox module now makes all changes to maildir files @@ -4191,7 +4191,7 @@ Library - Issue #18532: Change the builtin hash algorithms' names to lower case names as promised by hashlib's documentation. -- Issue #8713: add new spwan and forkserver start methods, and new functions +- Issue #8713: add new spawn and forkserver start methods, and new functions get_all_start_methods, get_start_method, and set_start_method, to multiprocessing. @@ -4524,7 +4524,7 @@ Core and Builtins - Issue #16613: Add *m* argument to ``collections.Chainmap.new_child`` to allow the new child map to be specified explicitly. -- Issue #16730: importlib.machinery.FileFinder now no longers raises an +- Issue #16730: importlib.machinery.FileFinder now no longer raises an exception when trying to populate its cache and it finds out the directory is unreadable or has turned into a file. Reported and diagnosed by David Pritchard. @@ -4832,7 +4832,7 @@ Library on Windows and adds no value over and above python -m pydoc ... - Issue #18155: The csv module now correctly handles csv files that use - a delimter character that has a special meaning in regexes, instead of + a delimiter character that has a special meaning in regexes, instead of throwing an exception. - Issue #14360: encode_quopri can now be successfully used as an encoder @@ -5590,7 +5590,7 @@ Library - Issue #16248: Disable code execution from the user's home directory by tkinter when the -E flag is passed to Python. Patch by Zachary Ware. -- Issue #13390: New function :func:`sys.getallocatedblocks()` returns the +- Issue #13390: New function :func:`sys.getallocatedblocks` returns the number of memory blocks currently allocated. - Issue #16628: Fix a memory leak in ctypes.resize(). @@ -6157,7 +6157,7 @@ Tests starting with a ".". Patch by Sebastian Kreft. - Issue #13390: The ``-R`` option to regrtest now also checks for memory - allocation leaks, using :func:`sys.getallocatedblocks()`. + allocation leaks, using :func:`sys.getallocatedblocks`. - Issue #16559: Add more tests for the json module, including some from the official test suite at json.org. Patch by Serhiy Storchaka. @@ -6329,7 +6329,7 @@ Documentation - Issue #15940: Specify effect of locale on time functions. -- Issue #17538: Document XML vulnerabilties +- Issue #17538: Document XML vulnerabilities - Issue #16642: sched.scheduler timefunc initial default is time.monotonic. Patch by Ramchandra Apte @@ -6676,7 +6676,7 @@ Library - Issue #14669: Fix pickling of connections and sockets on Mac OS X by sending/receiving an acknowledgment after file descriptor transfer. - TestPicklingConnection has been reenabled for Mac OS X. + TestPicklingConnection has been re-enabled for Mac OS X. - Issue #11062: Fix adding a message from file to Babyl mailbox. @@ -7114,7 +7114,7 @@ Build - Issue #14330: For cross builds, don't use host python, use host search paths for host compiler. -- Issue #15235: Allow Berkley DB versions up to 5.3 to build the dbm module. +- Issue #15235: Allow Berkeley DB versions up to 5.3 to build the dbm module. - Issue #15268: Search curses.h in /usr/include/ncursesw. @@ -7264,7 +7264,7 @@ Library called with no arguments. - Issue #14653: email.utils.mktime_tz() no longer relies on system - mktime() when timezone offest is supplied. + mktime() when timezone offset is supplied. - Issue #14684: zlib.compressobj() and zlib.decompressobj() now support the use of predefined compression dictionaries. Original patch by Sam Rushing. @@ -7606,7 +7606,7 @@ Library - Issue #14773: Fix os.fwalk() failing on dangling symlinks. - Issue #12541: Be lenient with quotes around Realm field of HTTP Basic - Authentation in urllib2. + Authentication in urllib2. - Issue #14807: move undocumented tarfile.filemode() to stat.filemode() and add doc entry. Add tarfile.filemode alias with deprecation warning. @@ -7673,7 +7673,7 @@ Library IDLE ---- -- Issue #14958: Change IDLE systax highlighting to recognize all string and +- Issue #14958: Change IDLE syntax highlighting to recognize all string and byte literals supported in Python 3.3. - Issue #10997: Prevent a duplicate entry in IDLE's "Recent Files" menu. @@ -10176,7 +10176,7 @@ IDLE - Issue #13296: Fix IDLE to clear compile __future__ flags on shell restart. (Patch by Roger Serwy) -- Issue #9871: Prevent IDLE 3 crash when given byte stings +- Issue #9871: Prevent IDLE 3 crash when given byte strings with invalid hex escape sequences, like b'\x0'. (Original patch by Claudiu Popa.) @@ -12098,7 +12098,7 @@ Library - Issue #9632: Remove sys.setfilesystemencoding() function: use PYTHONFSENCODING environment variable to set the filesystem encoding at Python startup. sys.setfilesystemencoding() creates inconsistencies because it is unable to - reencode all filenames in all objects. + re-encode all filenames in all objects. - Issue #9410: Various optimizations to the pickle module, leading to speedups up to 4x (depending on the benchmark). Mostly ported from Unladen Swallow; @@ -12509,7 +12509,7 @@ Library - Issue #9605: posix.getlogin() decodes the username with file filesystem encoding and surrogateescape error handler. Patch written by David Watson. -- Issue #9604: posix.initgroups() encodes the username using the fileystem +- Issue #9604: posix.initgroups() encodes the username using the filesystem encoding and surrogateescape error handler. Patch written by David Watson. - Issue #9603: posix.ttyname() and posix.ctermid() decode the terminal name @@ -12667,7 +12667,7 @@ What's New in Python 3.2 Alpha 1? Core and Builtins ----------------- -- Issue #8991: convertbuffer() rejects discontigious buffers. +- Issue #8991: convertbuffer() rejects discontiguous buffers. - Issue #7616: Fix copying of overlapping memoryview slices with the Intel compiler. @@ -13211,7 +13211,7 @@ Library - Issue #7989: Added pure python implementation of the `datetime` module. The C module is renamed to `_datetime` and if available, overrides all classes - defined in datetime with fast C impementation. Python implementation is based + defined in datetime with fast C implementation. Python implementation is based on the original python prototype for the datetime module by Tim Peters with minor modifications by the PyPy project. The test suite now tests `datetime` module with and without `_datetime` acceleration using the same test cases. @@ -15049,7 +15049,7 @@ Extension Modules an error. The _PY_STRUCT_FLOAT_COERCE constant has been removed. The version number has been bumped to 0.3. -- Issue #5359: Readd the Berkeley DB detection code to allow _dbm be built +- Issue #5359: Re-add the Berkeley DB detection code to allow _dbm be built using Berkeley DB. Tests @@ -17028,7 +17028,7 @@ Extension Modules and renamed to filter(), map(), and zip(). Also, renamed izip_longest() to zip_longest() and ifilterfalse() to filterfalse(). -- Issue #1762972: Readded the reload() function as imp.reload(). +- Issue #1762972: Re-added the reload() function as imp.reload(). - Bug #2111: mmap segfaults when trying to write a block opened with PROT_READ. @@ -18448,7 +18448,7 @@ Core and builtins - Fixed bug #1459029 - unicode reprs were double-escaped. -- Patch #1396919: The system scope threads are reenabled on FreeBSD +- Patch #1396919: The system scope threads are re-enabled on FreeBSD 5.4 and later versions. - Bug #1115379: Compiling a Unicode string with an encoding declaration @@ -21803,7 +21803,7 @@ Library - New csv package makes it easy to read/write CSV files. - Module shlex has been extended to allow posix-like shell parsings, - including a split() function for easy spliting of quoted strings and + including a split() function for easy splitting of quoted strings and commands. An iterator interface was also implemented. Tools/Demos @@ -27751,7 +27751,7 @@ Fri Mar 12 22:15:43 1999 Guido van Rossum The filename to URL conversion didn't properly quote special characters. - The URL to filename didn't properly unquote special chatacters. + The URL to filename didn't properly unquote special characters. * Objects/floatobject.c: OK, try again. Vladimir gave me a fix for the alignment bus error, @@ -27807,7 +27807,7 @@ Wed Mar 10 22:55:47 1999 Guido van Rossum classes in selected module methods of selected class - Sinlge clicking in a directory, module or class item updates the next + Single clicking in a directory, module or class item updates the next column with info about the selected item. Double clicking in a module, class or method item opens the file (and selects the clicked item if it is a class or method). @@ -28130,7 +28130,7 @@ webchecker and other ftp retrieves. - ConfigParser's get() method now accepts an optional keyword argument (vars) that is substituted on top of the defaults that were setup in -__init__. You can now also have recusive references in your +__init__. You can now also have recursive references in your configuration file. - Some improvements to the Queue module, including a put_nowait() @@ -28209,7 +28209,7 @@ core. not. - The curses module implements an optional nlines argument to -w.scroll(). (It then calls wscrl(win, nlines) instead of scoll(win).) +w.scroll(). (It then calls wscrl(win, nlines) instead of scroll(win).) Changes to tools ---------------- @@ -28504,7 +28504,7 @@ PyEval_GetGlobals. - glmodule.c: check in the changed version after running the stubber again -- this solves the conflict with curses over the 'clear' entry point much nicer. (Jack Jansen had checked in the changes to cstubs -eons ago, but I never regenrated glmodule.c :-( ) +eons ago, but I never regenerated glmodule.c :-( ) - frameobject.c: fix reference count bug in PyFrame_New. Vladimir Marangozov. @@ -28581,7 +28581,7 @@ idiom L1[len(L1):] = L2. - Better error messages when a sequence is indexed with a non-integer. -- Bettter error message when calling a non-callable object (include +- Better error message when calling a non-callable object (include the type in the message). Python services @@ -28656,7 +28656,7 @@ Internet Protocols and Support - imaplib.py: new version from Piers Lauder. - smtplib.py: change sendmail() method to accept a single string or a -list or strings as the destination (commom newbie mistake). +list or strings as the destination (common newbie mistake). - poplib.py: LIST with a msg argument fixed. @@ -31109,7 +31109,7 @@ encoding/decoding CGI form arguments. Catch all errors from the ftp module. HTTP requests now add the Host: header line. The proxy variable names are now mapped to lower case, for Windows. The spliturl() function no longer erroneously throws away all data past -the first newline. The basejoin() function now intereprets "../" +the first newline. The basejoin() function now interprets "../" correctly. I *believe* that the problems with "exception raised in __del__" under certain circumstances have been fixed (mostly by changes elsewher in the interpreter). @@ -31397,7 +31397,7 @@ changes and fixes. - Added a bunch of new winfo options to Tkinter.py; we should now be up to date with Tk 4.2. The new winfo options supported are: -mananger, pointerx, pointerxy, pointery, server, viewable, visualid, +manager, pointerx, pointerxy, pointery, server, viewable, visualid, visualsavailable. - The broken bind() method on Canvas objects defined in the Canvas.py @@ -32552,7 +32552,7 @@ The same applies to posixfile.open() and the socket method makefile(). is being maintained and distributed separately. - Improved support for the Apple Macintosh, in part by Jack Jansen, -e.g. interfaces to (a few) resource mananger functions, get/set file +e.g. interfaces to (a few) resource manager functions, get/set file type and creator, gestalt, sound manager, speech manager, MacTCP, comm toolbox, and the think C console library. This is being maintained and distributed separately. @@ -33229,7 +33229,7 @@ sys.argv[0]; it can simply do "if __name__ == '__main__': main()". * When an object is printed by the print statement, its implementation of str() is used. This means that classes can define __str__(self) to direct how their instances are printed. This is different from -__repr__(self), which should define an unambigous string +__repr__(self), which should define an unambiguous string representation of the instance. (If __str__() is not defined, it defaults to __repr__().) @@ -34366,7 +34366,7 @@ eval_code) and ceval.h (which doesn't need compile.hand declares the rest) ceval.h defines macros BGN_SAVE / END_SAVE for use with threads (to -improve the parallellism of multi-threaded programs by letting other +improve the parallelism of multi-threaded programs by letting other Python code run when a blocking system call or something similar is made) @@ -34514,7 +34514,7 @@ names listed in a 'global' statement must not be used in the function before the statement is reached. Remember that you don't need to use 'global' if you only want to *use* -a global variable in a function; nor do you need ot for assignments to +a global variable in a function; nor do you need to for assignments to parts of global variables (e.g., list or dictionary items or attributes of class instances). This has not changed; in fact assignment to part of a global variable was the standard workaround. diff --git a/Misc/NEWS.d/3.10.0a1.rst b/Misc/NEWS.d/3.10.0a1.rst index 4842a026aa49f7..f09842f1e77dea 100644 --- a/Misc/NEWS.d/3.10.0a1.rst +++ b/Misc/NEWS.d/3.10.0a1.rst @@ -5,7 +5,7 @@ .. section: Security Fixes ``python3x._pth`` being ignored on Windows, caused by the fix for -:issue:`29778` (CVE-2020-15801). +:issue:`29778` (:cve:`2020-15801`). .. @@ -25,7 +25,7 @@ events. .. section: Security Ensure :file:`python3.dll` is loaded from correct locations when Python is -embedded (CVE-2020-15523). +embedded (:cve:`2020-15523`). .. @@ -97,7 +97,7 @@ convention. Patch by Donghee Na. .. nonce: aJS9B3 .. section: Core and Builtins -Port the :mod:`_bisect` module to the multi-phase initialization API +Port the :mod:`!_bisect` module to the multi-phase initialization API (:pep:`489`). .. @@ -128,7 +128,7 @@ Taskaya. .. nonce: lh335O .. section: Core and Builtins -Port the :mod:`_lsprof` extension module to multi-phase initialization +Port the :mod:`!_lsprof` extension module to multi-phase initialization (:pep:`489`). .. @@ -148,7 +148,7 @@ Port the :mod:`cmath` extension module to multi-phase initialization .. nonce: jiXmyT .. section: Core and Builtins -Port the :mod:`_scproxy` extension module to multi-phase initialization +Port the :mod:`!_scproxy` extension module to multi-phase initialization (:pep:`489`). .. @@ -168,7 +168,7 @@ Port the :mod:`termios` extension module to multi-phase initialization .. nonce: QuDIut .. section: Core and Builtins -Convert the :mod:`_sha256` extension module types to heap types. +Convert the :mod:`!_sha256` extension module types to heap types. .. @@ -187,7 +187,7 @@ classes with a huge amount of arguments. Patch by Pablo Galindo. .. nonce: CnRME3 .. section: Core and Builtins -Port the :mod:`_overlapped` extension module to multi-phase initialization +Port the :mod:`!_overlapped` extension module to multi-phase initialization (:pep:`489`). .. @@ -197,7 +197,7 @@ Port the :mod:`_overlapped` extension module to multi-phase initialization .. nonce: X9CZgo .. section: Core and Builtins -Port the :mod:`_curses_panel` extension module to multi-phase initialization +Port the :mod:`!_curses_panel` extension module to multi-phase initialization (:pep:`489`). .. @@ -207,7 +207,7 @@ Port the :mod:`_curses_panel` extension module to multi-phase initialization .. nonce: 5jZymK .. section: Core and Builtins -Port the :mod:`_opcode` extension module to multi-phase initialization +Port the :mod:`!_opcode` extension module to multi-phase initialization (:pep:`489`). .. @@ -282,7 +282,7 @@ initialized ``_ast`` module. .. nonce: vcxSUa .. section: Core and Builtins -Convert :mod:`_operator` to use :c:func:`PyType_FromSpec`. +Convert :mod:`!_operator` to use :c:func:`PyType_FromSpec`. .. @@ -291,7 +291,7 @@ Convert :mod:`_operator` to use :c:func:`PyType_FromSpec`. .. nonce: fubBkb .. section: Core and Builtins -Port :mod:`_sha3` to multi-phase init. Convert static types to heap types. +Port :mod:`!_sha3` to multi-phase init. Convert static types to heap types. .. @@ -300,7 +300,7 @@ Port :mod:`_sha3` to multi-phase init. Convert static types to heap types. .. nonce: FC13e7 .. section: Core and Builtins -Port the :mod:`_blake2` extension module to the multi-phase initialization +Port the :mod:`!_blake2` extension module to the multi-phase initialization API (:pep:`489`). .. @@ -339,7 +339,7 @@ The output of ``python --help`` contains now only ASCII characters. .. nonce: O0d3ym .. section: Core and Builtins -Port the :mod:`_sha1`, :mod:`_sha512`, and :mod:`_md5` extension modules to +Port the :mod:`!_sha1`, :mod:`!_sha512`, and :mod:`!_md5` extension modules to multi-phase initialization API (:pep:`489`). .. @@ -636,7 +636,7 @@ Remove the remaining files from the old parser and the :mod:`symbol` module. .. nonce: _yI-ax .. section: Core and Builtins -Convert :mod:`_bz2` to use :c:func:`PyType_FromSpec`. +Convert :mod:`!_bz2` to use :c:func:`PyType_FromSpec`. .. @@ -666,7 +666,7 @@ by Brandt Bucher. .. nonce: 61iyYh .. section: Core and Builtins -Port :mod:`_gdbm` to multiphase initialization. +Port :mod:`!_gdbm` to multiphase initialization. .. @@ -696,7 +696,7 @@ for emitting syntax errors. Patch by Pablo Galindo. .. nonce: mmlp3Q .. section: Core and Builtins -Port :mod:`_dbm` to multiphase initialization. +Port :mod:`!_dbm` to multiphase initialization. .. @@ -1010,7 +1010,7 @@ Port :mod:`mmap` to multiphase initialization. .. nonce: Kfe9fT .. section: Core and Builtins -Port :mod:`_lzma` to multiphase initialization. +Port :mod:`!_lzma` to multiphase initialization. .. @@ -1032,7 +1032,7 @@ the :meth:`~object.__int__` method but do not have the .. nonce: AkRzjb .. section: Core and Builtins -Add :meth:`int.bit_count()`, counting the number of ones in the binary +Add :meth:`int.bit_count`, counting the number of ones in the binary representation of an integer. Patch by Niklas Fiekas. .. @@ -1499,7 +1499,7 @@ used to cause ZeroDivisionError now cause an OverflowError instead. .. nonce: rju34k .. section: Library -Add :func:`os.cpu_count()` support for VxWorks RTOS. +Add :func:`os.cpu_count` support for VxWorks RTOS. .. @@ -1596,7 +1596,7 @@ UnpicklingError instead of crashing. .. section: Library Avoid infinite loop when reading specially crafted TAR files using the -tarfile module (CVE-2019-20907). +tarfile module (:cve:`2019-20907`). .. @@ -1861,8 +1861,8 @@ bundled versions of ``pip`` and ``setuptools``. Patch by Krzysztof Konopko. .. nonce: _dx3OO .. section: Library -Removed :meth:`asyncio.Task.current_task` and -:meth:`asyncio.Task.all_tasks`. Patch contributed by Rémi Lapeyre. +Removed :meth:`!asyncio.Task.current_task` and +:meth:`!asyncio.Task.all_tasks`. Patch contributed by Rémi Lapeyre. .. @@ -2452,7 +2452,7 @@ Added the *root_dir* and *dir_fd* parameters in :func:`glob.glob`. .. nonce: X-TJZO .. section: Library -Fix :meth:`IMAP4.noop()` when debug mode is enabled (ex: ``imaplib.Debug = +Fix :meth:`IMAP4.noop` when debug mode is enabled (ex: ``imaplib.Debug = 3``). .. diff --git a/Misc/NEWS.d/3.10.0a2.rst b/Misc/NEWS.d/3.10.0a2.rst index 79f570439b52b8..3e82de9ef266d6 100644 --- a/Misc/NEWS.d/3.10.0a2.rst +++ b/Misc/NEWS.d/3.10.0a2.rst @@ -226,8 +226,8 @@ thread at the time the function is called. .. section: Core and Builtins Enable ``from __future__ import annotations`` (:pep:`563`) by default. The -values found in :attr:`__annotations__` dicts are now strings, e.g. ``{"x": -"int"}`` instead of ``{"x": int}``. +values found in :attr:`~object.__annotations__` dicts are now strings, for +example ``{"x": "int"}`` instead of ``{"x": int}``. .. @@ -362,7 +362,7 @@ plistlib: fix parsing XML plists with hexadecimal integer values .. nonce: 85BsRA .. section: Library -Fix an incorrectly formatted error from :meth:`_codecs.charmap_decode` when +Fix an incorrectly formatted error from :meth:`!_codecs.charmap_decode` when called with a mapped value outside the range of valid Unicode code points. PR by Max Bernstein. @@ -383,7 +383,7 @@ Inwood. .. nonce: jd_gkA .. section: Library -:meth:`sched.scheduler.cancel()` will now cancel the correct event, if two +:meth:`sched.scheduler.cancel` will now cancel the correct event, if two events with same priority are scheduled for the same time. Patch by Bar Harel. diff --git a/Misc/NEWS.d/3.10.0a3.rst b/Misc/NEWS.d/3.10.0a3.rst index 179cf3e9cfb08c..33c3e14b7a4bcf 100644 --- a/Misc/NEWS.d/3.10.0a3.rst +++ b/Misc/NEWS.d/3.10.0a3.rst @@ -477,7 +477,7 @@ object belongs to, potentially breaking the unpickling of those objects. Simplify the :mod:`importlib` external bootstrap code: ``importlib._bootstrap_external`` now uses regular imports to import builtin -modules. When it is imported, the builtin :func:`__import__()` function is +modules. When it is imported, the builtin :func:`__import__` function is already fully working and so can be used to import builtin modules like :mod:`sys`. Patch by Victor Stinner. @@ -517,8 +517,8 @@ Port the ``_signal`` extension module to the multi-phase initialization API .. nonce: Wh5svI .. section: Library -:func:`time.time()`, :func:`time.perf_counter()` and -:func:`time.monotonic()` functions can no longer fail with a Python fatal +:func:`time.time`, :func:`time.perf_counter` and +:func:`time.monotonic` functions can no longer fail with a Python fatal error, instead raise a regular Python exception on failure. .. @@ -550,10 +550,10 @@ deduplicate, use type to cache key). Patch provided by Yurii Karabas. .. nonce: iDbHrw .. section: Library -:func:`time.perf_counter()` on Windows and :func:`time.monotonic()` on macOS +:func:`time.perf_counter` on Windows and :func:`time.monotonic` on macOS are now system-wide. Previously, they used an offset computed at startup to reduce the precision loss caused by the float type. Use -:func:`time.perf_counter_ns()` and :func:`time.monotonic_ns()` added in +:func:`time.perf_counter_ns` and :func:`time.monotonic_ns` added in Python 3.7 to avoid this precision loss. .. @@ -1386,7 +1386,7 @@ Python already implicitly installs signal handlers: see The ``Py_TRASHCAN_BEGIN`` macro no longer accesses PyTypeObject attributes, but now can get the condition by calling the new private -:c:func:`_PyTrash_cond()` function which hides implementation details. +:c:func:`!_PyTrash_cond()` function which hides implementation details. .. diff --git a/Misc/NEWS.d/3.10.0a4.rst b/Misc/NEWS.d/3.10.0a4.rst index 398f7e5d3422cb..19f0db9a6be5e9 100644 --- a/Misc/NEWS.d/3.10.0a4.rst +++ b/Misc/NEWS.d/3.10.0a4.rst @@ -193,7 +193,7 @@ subinterpreters. Patch by Victor Stinner. .. nonce: j7nl6A .. section: Core and Builtins -Make :c:func:`_PyUnicode_FromId` function compatible with subinterpreters. +Make :c:func:`!_PyUnicode_FromId` function compatible with subinterpreters. Each interpreter now has an array of identifier objects (interned strings decoded from UTF-8). Patch by Victor Stinner. @@ -367,7 +367,7 @@ uses "options" instead. .. nonce: Quy3zn .. section: Library -Port the :mod:`_thread` extension module to the multiphase initialization +Port the :mod:`!_thread` extension module to the multiphase initialization API (:pep:`489`) and convert its static types to heap types. .. @@ -412,7 +412,7 @@ be created automatically. ``logging.disable`` will now validate the types and value of its parameter. It also now accepts strings representing the levels (as does -``loging.setLevel``) instead of only the numerical values. +``logging.setLevel``) instead of only the numerical values. .. @@ -709,7 +709,7 @@ directories. .. nonce: ek38d_ .. section: Library -Add :func:`os.set_blocking()` support for VxWorks RTOS. +Add :func:`os.set_blocking` support for VxWorks RTOS. .. @@ -960,8 +960,8 @@ explicitly and so not exported. .. nonce: Je08Ny .. section: C API -Remove the private :c:func:`_Py_fopen` function which is no longer needed. -Use :c:func:`_Py_wfopen` or :c:func:`_Py_fopen_obj` instead. Patch by Victor +Remove the private :c:func:`!_Py_fopen` function which is no longer needed. +Use :c:func:`!_Py_wfopen` or :c:func:`!_Py_fopen_obj` instead. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.10.0a5.rst b/Misc/NEWS.d/3.10.0a5.rst index dc95e8ce072fd9..a85ea1ff1c2817 100644 --- a/Misc/NEWS.d/3.10.0a5.rst +++ b/Misc/NEWS.d/3.10.0a5.rst @@ -108,7 +108,7 @@ a slice at the start of the ``bytearray`` to a shorter byte string). .. nonce: WfTdfg .. section: Core and Builtins -Fix the :c:func:`_PyUnicode_FromId` function (_Py_IDENTIFIER(var) API) when +Fix the :c:func:`!_PyUnicode_FromId` function (_Py_IDENTIFIER(var) API) when :c:func:`Py_Initialize` / :c:func:`Py_Finalize` is called multiple times: preserve ``_PyRuntime.unicode_ids.next_index`` value. diff --git a/Misc/NEWS.d/3.10.0a6.rst b/Misc/NEWS.d/3.10.0a6.rst index bad3528084897b..31b7df2c61158e 100644 --- a/Misc/NEWS.d/3.10.0a6.rst +++ b/Misc/NEWS.d/3.10.0a6.rst @@ -315,7 +315,7 @@ Adds :const:`resource.RLIMIT_KQUEUES` constant from FreeBSD to the .. section: Library Make the pure Python implementation of :mod:`xml.etree.ElementTree` behave -the same as the C implementation (:mod:`_elementree`) regarding default +the same as the C implementation (:mod:`!_elementree`) regarding default attribute values (by not setting ``specified_attributes=1``). .. diff --git a/Misc/NEWS.d/3.10.0a7.rst b/Misc/NEWS.d/3.10.0a7.rst index 74120a3b40c012..d866e805fd3a7e 100644 --- a/Misc/NEWS.d/3.10.0a7.rst +++ b/Misc/NEWS.d/3.10.0a7.rst @@ -4,7 +4,7 @@ .. release date: 2021-04-05 .. section: Security -CVE-2021-3426: Remove the ``getfile`` feature of the :mod:`pydoc` module +:cve:`2021-3426`: Remove the ``getfile`` feature of the :mod:`pydoc` module which could be abused to read arbitrary files on the disk (directory traversal vulnerability). Moreover, even source code of Python modules can contain sensitive data like passwords. Vulnerability reported by David @@ -83,7 +83,7 @@ instruction dispatch a bit. .. nonce: PhaT-B .. section: Core and Builtins -Fix reference leak in the :mod:`_hashopenssl` extension. Patch by Pablo +Fix reference leak in the :mod:`!_hashopenssl` extension. Patch by Pablo Galindo. .. @@ -654,7 +654,7 @@ support importlib.invalidate_caches(). Patch by Desmond Cheong. .. nonce: 3r0HFY .. section: Library -Fail fast in :func:`shutil.move()` to avoid creating destination directories +Fail fast in :func:`shutil.move` to avoid creating destination directories on failure. .. @@ -701,8 +701,8 @@ sessions in :mod:`pdb`'s interactive mode. When the :data:`tempfile.tempdir` global variable is set to a value of type bytes, it is now handled consistently. Previously exceptions could be raised from some tempfile APIs when the directory did not already exist in -this situation. Also ensures that the :func:`tempfile.gettempdir()` and -:func:`tempfile.gettempdirb()` functions *always* return ``str`` and +this situation. Also ensures that the :func:`tempfile.gettempdir` and +:func:`tempfile.gettempdirb` functions *always* return ``str`` and ``bytes`` respectively. .. @@ -715,7 +715,7 @@ this situation. Also ensures that the :func:`tempfile.gettempdir()` and Expose ``X509_V_FLAG_ALLOW_PROXY_CERTS`` as :const:`~ssl.VERIFY_ALLOW_PROXY_CERTS` to allow proxy certificate validation as explained in -https://www.openssl.org/docs/man1.1.1/man7/proxy-certificates.html. +https://docs.openssl.org/1.1.1/man7/proxy-certificates/. .. diff --git a/Misc/NEWS.d/3.10.0b1.rst b/Misc/NEWS.d/3.10.0b1.rst index 640f3ee58adbae..406a5d7853edc0 100644 --- a/Misc/NEWS.d/3.10.0b1.rst +++ b/Misc/NEWS.d/3.10.0b1.rst @@ -182,7 +182,7 @@ normally be possible, but might occur in some unusual circumstances. .. nonce: u5Y6bS .. section: Core and Builtins -Importing the :mod:`_signal` module in a subinterpreter has no longer side +Importing the :mod:`!_signal` module in a subinterpreter has no longer side effects. .. @@ -277,7 +277,7 @@ cause any runtime effects with ``from __future__ import annotations``. .. section: Core and Builtins :exc:`SyntaxError` exceptions raised by the interpreter will highlight the -full error range of the expression that consistutes the syntax error itself, +full error range of the expression that constitutes the syntax error itself, instead of just where the problem is detected. Patch by Pablo Galindo. .. @@ -776,11 +776,11 @@ builtins.open() is now io.open(). .. nonce: o1zEk_ .. section: Library -The Python :func:`_pyio.open` function becomes a static method to behave as +The Python :func:`!_pyio.open` function becomes a static method to behave as :func:`io.open` built-in function: don't become a bound method when stored as a class variable. It becomes possible since static methods are now -callable in Python 3.10. Moreover, :func:`_pyio.OpenWrapper` becomes a -simple alias to :func:`_pyio.open`. Patch by Victor Stinner. +callable in Python 3.10. Moreover, :func:`!_pyio.OpenWrapper` becomes a +simple alias to :func:`!_pyio.open`. Patch by Victor Stinner. .. @@ -941,7 +941,7 @@ result from ``entry_points()`` as deprecated. .. -.. gh: 47383 +.. gh-issue: 47383 .. date: 2021-04-08-19-32-26 .. nonce: YI1hdL .. section: Library @@ -958,7 +958,7 @@ Patch by Jelle Zijlstra. .. nonce: nnVd3h .. section: Library -Add an ``encoding`` parameter :func:`logging.fileConfig()`. +Add an ``encoding`` parameter :func:`logging.fileConfig`. .. @@ -1270,7 +1270,7 @@ Fix thread locks in zlib module may go wrong in rare case. Patch by Ma Lin. .. nonce: oi6Kdb .. section: Library -Fix dataclasses with ``InitVar``\s and :func:`~dataclasses.replace()`. Patch +Fix dataclasses with ``InitVar``\s and :func:`~dataclasses.replace`. Patch by Claudiu Popa. .. @@ -1310,11 +1310,11 @@ functions in the :mod:`os` module. .. nonce: 9adF3E .. section: Library -:func:`os.path.expanduser()` now refuses to guess Windows home directories +:func:`os.path.expanduser` now refuses to guess Windows home directories if the basename of current user's home directory does not match their username. -:meth:`pathlib.Path.expanduser()` and :meth:`~pathlib.Path.home()` now +:meth:`pathlib.Path.expanduser` and :meth:`~pathlib.Path.home` now consistently raise :exc:`RuntimeError` exception when a home directory cannot be resolved. Previously a :exc:`KeyError` exception could be raised on Windows when the ``"USERNAME"`` environment variable was unset. @@ -1516,7 +1516,7 @@ Adds additional arguments to :func:`os.startfile` function. .. nonce: F0Cg6X .. section: Windows -Avoid raising errors from :meth:`pathlib.Path.exists()` when passed an +Avoid raising errors from :meth:`pathlib.Path.exists` when passed an invalid filename. .. diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index 754e782dfe661b..0b49c2a78771d2 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -38,7 +38,7 @@ significant performance overhead when loading from ``.pyc`` files. .. section: Security Update the vendored copy of libexpat to 2.4.1 (from 2.2.8) to get the fix -for the CVE-2013-0340 "Billion Laughs" vulnerability. This copy is most used +for the :cve:`2013-0340` "Billion Laughs" vulnerability. This copy is most used on Windows and macOS. .. @@ -613,7 +613,7 @@ Rename ``types.Union`` to ``types.UnionType``. .. section: Core and Builtins Expose specialization stats in python via -:func:`_opcode.get_specialization_stats`. +:func:`!_opcode.get_specialization_stats`. .. @@ -972,7 +972,7 @@ manager` protocols correspondingly. .. section: Core and Builtins Make sure that the line number is set when entering a comprehension scope. -Ensures that backtraces inclusing generator expressions show the correct +This ensures that backtraces including generator expressions show the correct line number. .. @@ -1701,7 +1701,7 @@ Remove many old deprecated :mod:`unittest` features: .. nonce: y1kEfP .. section: Library -Remove the deprecated ``split()`` method of :class:`_tkinter.TkappType`. +Remove the deprecated ``split()`` method of :class:`!_tkinter.TkappType`. Patch by Erlend E. Aasland. .. @@ -2000,7 +2000,7 @@ during file extraction. .. nonce: roUl0G .. section: Library -:mod:`subprocess` on Solaris now also uses :func:`os.posix_spawn()` for +:mod:`subprocess` on Solaris now also uses :func:`os.posix_spawn` for better performance. .. @@ -2298,9 +2298,9 @@ Adopt *binacii.a2b_base64*'s strict mode in *base64.b64decode*. .. nonce: ThuDMI .. section: Library -Fixed a bug in the :mod:`_ssl` module that was throwing :exc:`OverflowError` -when using :meth:`_ssl._SSLSocket.write` and :meth:`_ssl._SSLSocket.read` -for a big value of the ``len`` parameter. Patch by Pablo Galindo +Fixed a bug in the :mod:`!_ssl` module that was throwing :exc:`OverflowError` +when using :meth:`!_ssl._SSLSocket.write` and :meth:`!_ssl._SSLSocket.read` +for a big value of the ``len`` parameter. Patch by Pablo Galindo. .. @@ -2398,7 +2398,7 @@ class in the interactive session. Instead of :exc:`TypeError`, it should be .. nonce: R3IcM1 .. section: Library -Fix memory leak in :func:`_tkinter._flatten` if it is called with a sequence +Fix memory leak in :func:`!_tkinter._flatten` if it is called with a sequence or set, but not list or tuple. .. @@ -2722,7 +2722,7 @@ Importing typing.io or typing.re now prints a ``DeprecationWarning``. .. section: Library argparse actions store_const and append_const each receive a default value -of None when the ``const`` kwarg is not provided. Previously, this raised a +of ``None`` when the ``const`` kwarg is not provided. Previously, this raised a :exc:`TypeError`. .. @@ -3995,7 +3995,7 @@ operator expressions. .. section: Documentation Document that :class:`collections.defaultdict` parameter ``default_factory`` -defaults to None and is positional-only. +defaults to ``None`` and is positional-only. .. @@ -4187,7 +4187,7 @@ Add calls of :func:`gc.collect` in tests to support PyPy. .. nonce: mQZdXU .. section: Tests -Made tests relying on the :mod:`_asyncio` C extension module optional to +Made tests relying on the :mod:`!_asyncio` C extension module optional to allow running on alternative Python implementations. Patch by Serhiy Storchaka. @@ -4238,7 +4238,7 @@ harmless "malloc can't allocate region" messages spewed by test_decimal. .. nonce: KKsNOV .. section: Tests -Fixed floating point precision issue in turtle tests. +Fixed floating-point precision issue in turtle tests. .. diff --git a/Misc/NEWS.d/3.11.0a2.rst b/Misc/NEWS.d/3.11.0a2.rst index a6b5fe54b391c5..48cf2c1e428d87 100644 --- a/Misc/NEWS.d/3.11.0a2.rst +++ b/Misc/NEWS.d/3.11.0a2.rst @@ -15,7 +15,7 @@ Improve the :exc:`SyntaxError` message when using ``True``, ``None`` or .. section: Core and Builtins :data:`sys.stdlib_module_names` now contains the macOS-specific module -:mod:`_scproxy`. +:mod:`!_scproxy`. .. @@ -34,7 +34,7 @@ module but frozen modules are disabled. .. nonce: veL4lJ .. section: Core and Builtins -Specialize simple calls to Python functions (no starargs, keyowrd dict, or +Specialize simple calls to Python functions (no starargs, keyword dict, or closure) .. @@ -331,8 +331,8 @@ underlying SQLite API signals memory error. Patch by Erlend E. Aasland. .. nonce: 4MQt4r .. section: Library -pprint.pprint() now handles underscore_numbers correctly. Previously it was -always setting it to False. +:func:`pprint.pprint` now handles *underscore_numbers* correctly. +Previously it was always setting it to ``False``. .. @@ -1023,7 +1023,7 @@ compile shared modules. .. nonce: 61gM2A .. section: Build -:mod:`pyexpat` and :mod:`_elementtree` no longer define obsolete macros +:mod:`pyexpat` and :mod:`!_elementtree` no longer define obsolete macros ``HAVE_EXPAT_CONFIG_H`` and ``USE_PYEXPAT_CAPI``. ``XML_POOR_ENTROPY`` is now defined in ``expat_config.h``. diff --git a/Misc/NEWS.d/3.11.0a3.rst b/Misc/NEWS.d/3.11.0a3.rst index 2842aad0e163d6..6a0ae20d1fb5ed 100644 --- a/Misc/NEWS.d/3.11.0a3.rst +++ b/Misc/NEWS.d/3.11.0a3.rst @@ -27,7 +27,7 @@ invalid targets. Patch by Pablo Galindo .. nonce: 3TmTSw .. section: Core and Builtins -:c:func:`_PyErr_ChainStackItem` no longer normalizes ``exc_info`` (including +:c:func:`!_PyErr_ChainStackItem` no longer normalizes ``exc_info`` (including setting the traceback on the exception instance) because ``exc_info`` is always normalized. diff --git a/Misc/NEWS.d/3.11.0a4.rst b/Misc/NEWS.d/3.11.0a4.rst index 78b682f7a22cc6..64e2f39ad9db18 100644 --- a/Misc/NEWS.d/3.11.0a4.rst +++ b/Misc/NEWS.d/3.11.0a4.rst @@ -7,7 +7,7 @@ :c:func:`Py_EndInterpreter` now explicitly untracks all objects currently tracked by the GC. Previously, if an object was used later by another interpreter, calling :c:func:`PyObject_GC_UnTrack` on the object crashed if -the previous or the next object of the :c:type:`PyGC_Head` structure became +the previous or the next object of the :c:type:`!PyGC_Head` structure became a dangling pointer. Patch by Victor Stinner. .. @@ -258,7 +258,7 @@ instruction which performs the same operation, but without the loop. .. nonce: ADVaPT .. section: Core and Builtins -The code called from :c:func:`_PyErr_Display` was refactored to improve +The code called from :c:func:`!_PyErr_Display` was refactored to improve error handling. It now exits immediately upon an unrecoverable error. .. diff --git a/Misc/NEWS.d/3.11.0a5.rst b/Misc/NEWS.d/3.11.0a5.rst index 30a462e9bfdcbf..5418d5d59dd583 100644 --- a/Misc/NEWS.d/3.11.0a5.rst +++ b/Misc/NEWS.d/3.11.0a5.rst @@ -486,8 +486,8 @@ Use ``dis.Positions`` in ``dis.Instruction`` instead of a regular ``tuple``. .. nonce: geS-aP .. section: Library -:mod:`pdb` now gracefully handles ``help`` when :attr:`__doc__` is missing, -for example when run with pregenerated optimized ``.pyc`` files. +:mod:`pdb` now gracefully handles ``help`` when :attr:`~module.__doc__` is +missing, for example when run with pregenerated optimized ``.pyc`` files. .. @@ -748,7 +748,7 @@ tests to use ``support.infinite_recursion()``. Patch by Victor Stinner. Skip test_builtin PTY tests on non-ASCII characters if the readline module is loaded. The readline module changes input() behavior, but test_builtin is -not intented to test the readline module. Patch by Victor Stinner. +not intended to test the readline module. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.11.0a6.rst b/Misc/NEWS.d/3.11.0a6.rst index 2fdceef7746d4e..e88142e641f040 100644 --- a/Misc/NEWS.d/3.11.0a6.rst +++ b/Misc/NEWS.d/3.11.0a6.rst @@ -1054,7 +1054,7 @@ Patch by Victor Stinner. .. nonce: ajJjkh .. section: Build -Building Python now requires support for floating point Not-a-Number (NaN): +Building Python now requires support for floating-point Not-a-Number (NaN): remove the ``Py_NO_NAN`` macro. Patch by Victor Stinner. .. @@ -1088,7 +1088,7 @@ Patch by Kumar Aditya. Fix wasm32-emscripten test failures and platform issues. - Disable syscalls that are not supported or don't work, e.g. wait, getrusage, prlimit, -mkfifo, mknod, setres[gu]id, setgroups. - Use fd_count to cound open fds. - +mkfifo, mknod, setres[gu]id, setgroups. - Use fd_count to count open fds. - Add more checks for subprocess and fork. - Add workarounds for missing _multiprocessing and failing socket.accept(). - Enable bzip2. - Disable large file support. - Disable signal.alarm. @@ -1162,7 +1162,7 @@ Terry Jan Reedy. .. section: C API Python's public headers no longer import ````, leaving code that -embedd/extends Python free to define ``bool``, ``true`` and ``false``. +embeds/extends Python free to define ``bool``, ``true`` and ``false``. .. @@ -1182,7 +1182,7 @@ internal C API ``pycore_frame.h`` header file. Patch by Victor Stinner. .. section: C API Rename ``Include/buffer.h`` header file to ``Include/pybuffer.h`` to avoid -conflits with projects having an existing ``buffer.h`` header file. Patch by +conflicts with projects having an existing ``buffer.h`` header file. Patch by Victor Stinner. .. @@ -1202,5 +1202,5 @@ API). Patch by Victor Stinner. .. nonce: __ZdpH .. section: C API -Added function :c:func:`PyType_GetModuleByDef`, which allows accesss to +Added function :c:func:`PyType_GetModuleByDef`, which allows access to module state when a method's defining class is not available. diff --git a/Misc/NEWS.d/3.11.0a7.rst b/Misc/NEWS.d/3.11.0a7.rst index ec99bd0294ceca..eff2ea2dac13f8 100644 --- a/Misc/NEWS.d/3.11.0a7.rst +++ b/Misc/NEWS.d/3.11.0a7.rst @@ -224,7 +224,7 @@ Kumar Aditya. .. nonce: ZI05b5 .. section: Core and Builtins -Improved the performance of :meth:`list.append()` and list comprehensions by +Improved the performance of :meth:`list.append` and list comprehensions by optimizing for the common case, where no resize is needed. Patch by Dennis Sweeney. @@ -1127,7 +1127,7 @@ Raise more accurate and :pep:`249` compatible exceptions in :mod:`sqlite3`. * Don't overwrite :exc:`BufferError` with :exc:`ValueError` when conversion to BLOB fails. * Raise :exc:`~sqlite3.ProgrammingError` instead of :exc:`~sqlite3.Warning` if - user tries to :meth:`~sqlite3.Cursor.execute()` more than one SQL statement. + user tries to :meth:`~sqlite3.Cursor.execute` more than one SQL statement. * Raise :exc:`~sqlite3.ProgrammingError` instead of :exc:`ValueError` if an SQL query contains null characters. @@ -1173,7 +1173,7 @@ implemented. .. section: Library Add an Barrier object in synchronization primitives of *asyncio* Lib in -order to be consistant with Barrier from *threading* and *multiprocessing* +order to be consistent with Barrier from *threading* and *multiprocessing* libs* .. @@ -1211,7 +1211,7 @@ Update PEP URLs to :pep:`676`'s new canonical form. .. nonce: 4Dn48U .. section: Documentation -Clarified the old Python versions compatiblity note of +Clarified the old Python versions compatibility note of :func:`binascii.crc32` / :func:`zlib.adler32` / :func:`zlib.crc32` functions. @@ -1401,7 +1401,7 @@ Christian's container image ``quay.io/tiran/cpython_autoconf:269``. .. nonce: fry4aK .. section: Build -Building Python now requires support of IEEE 754 floating point numbers. +Building Python now requires support of IEEE 754 floating-point numbers. Patch by Victor Stinner. .. @@ -1421,7 +1421,7 @@ Patch by Victor Stinner. .. nonce: IB0XL4 .. section: Windows -Update ``zlib`` to v1.2.12 to resolve CVE-2018-25032. +Update ``zlib`` to v1.2.12 to resolve :cve:`2018-25032`. .. @@ -1472,8 +1472,8 @@ Update Windows installer to use SQLite 3.38.1. .. nonce: SPrGS9 .. section: Windows -Update bzip2 to 1.0.8 in Windows builds to mitigate CVE-2016-3189 and -CVE-2019-12900 +Update bzip2 to 1.0.8 in Windows builds to mitigate :cve:`2016-3189` and +:cve:`2019-12900`. .. @@ -1482,7 +1482,7 @@ CVE-2019-12900 .. nonce: Ufd4tG .. section: Windows -Prevent CVE-2022-26488 by ensuring the Add to PATH option in the Windows +Prevent :cve:`2022-26488` by ensuring the Add to PATH option in the Windows installer uses the correct path when being repaired. .. diff --git a/Misc/NEWS.d/3.11.0b1.rst b/Misc/NEWS.d/3.11.0b1.rst index f9296679655573..87442dbbbd17f5 100644 --- a/Misc/NEWS.d/3.11.0b1.rst +++ b/Misc/NEWS.d/3.11.0b1.rst @@ -58,10 +58,10 @@ may have prevented Python-to-Python calls respecting PEP 523. .. nonce: -igcjS .. section: Core and Builtins -Add a closure keyword-only parameter to exec(). It can only be specified +Add a closure keyword-only parameter to :func:`exec`. It can only be specified when exec-ing a code object that uses free variables. When specified, it must be a tuple, with exactly the number of cell variables referenced by the -code object. closure has a default value of None, and it must be None if the +code object. closure has a default value of ``None``, and it must be ``None`` if the code object doesn't refer to any free variables. .. @@ -285,7 +285,7 @@ macros. .. nonce: 11YXHQ .. section: Core and Builtins -Add a new :c:func:`_PyFrame_IsEntryFrame` API function, to check if a +Add a new :c:func:`!_PyFrame_IsEntryFrame` API function, to check if a :c:type:`PyFrameObject` is an entry frame. Patch by Pablo Galindo. .. @@ -570,7 +570,7 @@ planned). Patch by Alex Waygood. .. -.. gh: 78157 +.. gh-issue: 78157 .. date: 2022-05-05-20-40-45 .. nonce: IA_9na .. section: Library @@ -664,8 +664,9 @@ for :func:`os.fcopyfile` available in macOs. .. nonce: l1p7CJ .. section: Library -For @dataclass, add weakref_slot. Default is False. If True, and if -slots=True, add a slot named "__weakref__", which will allow instances to be +For :func:`@dataclass `, add *weakref_slot*. +The new parameter defaults to ``False``. If true, and if +``slots=True``, add a slot named ``"__weakref__"``, which will allow instances to be weakref'd. Contributed by Eric V. Smith .. @@ -1288,7 +1289,7 @@ Deprecate the chunk module. .. -.. gh: 91498 +.. gh-issue: 91498 .. date: 2022-04-10-08-39-44 .. nonce: 8oII92 .. section: Library @@ -1800,8 +1801,8 @@ The documentation now lists which members of C structs are part of the .. section: Documentation All docstrings in code snippets are now wrapped into :c:macro:`PyDoc_STR` to -follow the guideline of `PEP 7's Documentation Strings paragraph -`_. Patch +follow the guideline of :pep:`PEP 7's Documentation Strings paragraph +<0007#documentation-strings>`. Patch by Oleg Iarygin. .. diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst index 4739e0fb89a4a8..f2668e99a6299b 100644 --- a/Misc/NEWS.d/3.12.0a1.rst +++ b/Misc/NEWS.d/3.12.0a1.rst @@ -29,8 +29,7 @@ process. This was a potential privilege escalation. Filesystem based socket permissions restrict this to the *forkserver* process user as was the default in Python 3.8 and earlier. -This prevents Linux `CVE-2022-42919 -`_. +This prevents Linux :cve:`2022-42919`. .. @@ -103,7 +102,7 @@ well as generator expressions. .. section: Core and Builtins Added unicode check for ``name`` attribute of ``spec`` argument passed in -:func:`_imp.create_builtin` function. +:func:`!_imp.create_builtin` function. .. @@ -484,7 +483,7 @@ Fix case of undefined behavior in ceval.c .. nonce: AfCi36 .. section: Core and Builtins -Convert :mod:`_functools` to argument clinic. +Convert :mod:`!_functools` to argument clinic. .. @@ -493,7 +492,7 @@ Convert :mod:`_functools` to argument clinic. .. nonce: wky0Fc .. section: Core and Builtins -Do not expose ``KeyWrapper`` in :mod:`_functools`. +Do not expose ``KeyWrapper`` in :mod:`!_functools`. .. @@ -561,7 +560,7 @@ versions prior to 3.11 .. nonce: 9lmTCC .. section: Core and Builtins -Remove two cases of undefined behavoir, by adding NULL checks. +Remove two cases of undefined behavior, by adding NULL checks. .. @@ -1732,7 +1731,7 @@ tracing functions implemented in C. .. nonce: lenv9h .. section: Core and Builtins -:meth:`_warnings.warn_explicit` is ported to Argument Clinic. +:meth:`!_warnings.warn_explicit` is ported to Argument Clinic. .. @@ -2592,7 +2591,7 @@ Update bundled pip to 22.2.2. Fix :class:`asyncio.TaskGroup` to propagate exception when :exc:`asyncio.CancelledError` was replaced with another exception by a -context manger. Patch by Kumar Aditya and Guido van Rossum. +context manager. Patch by Kumar Aditya and Guido van Rossum. .. @@ -3143,8 +3142,8 @@ test.test_codecs.EncodedFileTest`` instead. .. nonce: VhS1eS .. section: Library -Made :class:`_struct.Struct` GC-tracked in order to fix a reference leak in -the :mod:`_struct` module. +Made :class:`!_struct.Struct` GC-tracked in order to fix a reference leak in +the :mod:`!_struct` module. .. @@ -3200,9 +3199,8 @@ Remove the :func:`ssl.wrap_socket` function, deprecated in Python 3.7: instead, create a :class:`ssl.SSLContext` object and call its :class:`ssl.SSLContext.wrap_socket` method. Any package that still uses :func:`ssl.wrap_socket` is broken and insecure. The function neither sends a -SNI TLS extension nor validates server hostname. Code is subject to `CWE-295 -`_: Improper Certificate -Validation. Patch by Victor Stinner. +SNI TLS extension nor validates server hostname. Code is subject to :cwe:`295` +Improper Certificate Validation. Patch by Victor Stinner. .. @@ -3223,9 +3221,9 @@ Stinner. .. section: Library :mod:`hashlib`: Remove the pure Python implementation of -:func:`hashlib.pbkdf2_hmac()`, deprecated in Python 3.10. Python 3.10 and +:func:`hashlib.pbkdf2_hmac`, deprecated in Python 3.10. Python 3.10 and newer requires OpenSSL 1.1.1 (:pep:`644`): this OpenSSL version provides a C -implementation of :func:`~hashlib.pbkdf2_hmac()` which is faster. Patch by +implementation of :func:`~hashlib.pbkdf2_hmac` which is faster. Patch by Victor Stinner. .. @@ -3260,7 +3258,7 @@ on the main thread Remove ``io.OpenWrapper`` and ``_pyio.OpenWrapper``, deprecated in Python 3.10: just use :func:`open` instead. The :func:`open` (:func:`io.open`) -function is a built-in function. Since Python 3.10, :func:`_pyio.open` is +function is a built-in function. Since Python 3.10, :func:`!_pyio.open` is also a static method. Patch by Victor Stinner. .. @@ -3500,7 +3498,7 @@ Illia Volochii. .. nonce: tjfu9L .. section: Library -Deprecate :data:`sqlite3.version` and :data:`sqlite3.version_info`. +Deprecate :data:`!version` and :data:`!version_info`. .. @@ -3563,8 +3561,8 @@ with :func:`os.pidfd_open` in non-blocking mode. Patch by Kumar Aditya. .. nonce: mkYl5q .. section: Library -Implement Enum __contains__ that returns True or False to replace the -deprecated behaviour that would sometimes raise a TypeError. +Implement ``Enum.__contains__`` that returns ``True`` or ``False`` to replace the +deprecated behaviour that would sometimes raise a :exc:`TypeError`. .. @@ -3731,7 +3729,7 @@ In a very special case, the email package tried to append the nonexistent .. nonce: e6uKxj .. section: Library -Fix :func:`ast.unparse` when ``ImportFrom.level`` is None +Fix :func:`ast.unparse` when ``ImportFrom.level`` is ``None`` .. @@ -3793,7 +3791,7 @@ the :c:type:`time_t` type in C. .. section: Library Fixed crash resulting from calling bisect.insort() or bisect.insort_left() -with the key argument not equal to None. +with the key argument not equal to ``None``. .. @@ -4082,7 +4080,7 @@ replacement strings containing group references by 2--3 times. .. section: Library Fix findtext in the xml module to only give an empty string when the text -attribute is set to None. +attribute is set to ``None``. .. @@ -4239,8 +4237,8 @@ by :mod:`asyncio` to AIX platform only. .. nonce: 4dzB80 .. section: Library -Set :attr:`doctest.DocTest.lineno` to ``None`` when object does not have -:attr:`__doc__`. +Set :attr:`doctest.DocTest.lineno` to ``None`` when an object does not have +:attr:`~definition.__doc__`. .. @@ -4404,8 +4402,7 @@ Remove extra row .. section: Documentation Deprecated tools ``make suspicious`` and ``rstlint.py`` are now removed. -They have been replaced by `spinx-lint -`_. +They have been replaced by :pypi:`sphinx-lint`. .. @@ -4479,7 +4476,7 @@ they are deprecated. Contributed by C.A.M. Gerlach. .. nonce: we7AFm .. section: Documentation -Replaced incorrectly written true/false values in documentiation. Patch by +Replaced incorrectly written true/false values in documentation. Patch by Robert O'Shea .. @@ -5613,7 +5610,7 @@ Accept os.PathLike for the argument to winsound.PlaySound Support native Windows case-insensitive path comparisons by using ``LCMapStringEx`` instead of :func:`str.lower` in :func:`ntpath.normcase`. -Add ``LCMapStringEx`` to the :mod:`_winapi` module. +Add ``LCMapStringEx`` to the :mod:`!_winapi` module. .. diff --git a/Misc/NEWS.d/3.12.0a2.rst b/Misc/NEWS.d/3.12.0a2.rst index a9c5038fa489bb..bc028f30636bf7 100644 --- a/Misc/NEWS.d/3.12.0a2.rst +++ b/Misc/NEWS.d/3.12.0a2.rst @@ -527,7 +527,7 @@ Stinner. .. nonce: Ai2KDh .. section: Library -Now :mod:`_pyio` is consistent with :mod:`_io` in raising ``ValueError`` +Now :mod:`!_pyio` is consistent with :mod:`!_io` in raising ``ValueError`` when executing methods over closed buffers. .. @@ -537,7 +537,7 @@ when executing methods over closed buffers. .. nonce: 0v8iyw .. section: Library -Clean up refleak on failed module initialisation in :mod:`_zoneinfo` +Clean up refleak on failed module initialisation in :mod:`!_zoneinfo` .. @@ -546,7 +546,7 @@ Clean up refleak on failed module initialisation in :mod:`_zoneinfo` .. nonce: qc_KHr .. section: Library -Clean up refleaks on failed module initialisation in :mod:`_pickle` +Clean up refleaks on failed module initialisation in :mod:`!_pickle` .. @@ -555,7 +555,7 @@ Clean up refleaks on failed module initialisation in :mod:`_pickle` .. nonce: LBl79O .. section: Library -Clean up refleak on failed module initialisation in :mod:`_io`. +Clean up refleak on failed module initialisation in :mod:`!_io`. .. @@ -706,7 +706,7 @@ Remove modules :mod:`!asyncore` and :mod:`!asynchat`, which were deprecated by .. section: Library Fix handling of ``bytes`` :term:`path-like objects ` in -:func:`os.ismount()`. +:func:`os.ismount`. .. @@ -822,7 +822,7 @@ and to indicate when it became late-bound. .. nonce: 7KinCV .. section: Tests -The Python test suite now fails wit exit code 4 if no tests ran. It should +The Python test suite now fails with exit code 4 if no tests ran. It should help detecting typos in test names and test methods. .. @@ -968,7 +968,7 @@ if :option:`--with-system-expat` is passed to :program:`configure`. .. nonce: 0f6e_N .. section: Windows -Update Windows builds to zlib v1.2.13. v1.2.12 has CVE-2022-37434, but the +Update Windows builds to zlib v1.2.13. v1.2.12 has :cve:`2022-37434`, but the vulnerable ``inflateGetHeader`` API is not used by Python. .. diff --git a/Misc/NEWS.d/3.12.0a3.rst b/Misc/NEWS.d/3.12.0a3.rst index ce128fd5f80c77..04a2bf9fb916b7 100644 --- a/Misc/NEWS.d/3.12.0a3.rst +++ b/Misc/NEWS.d/3.12.0a3.rst @@ -70,7 +70,7 @@ Fix bug where compiler crashes on an if expression with an empty body block. .. nonce: DcKoBJ .. section: Core and Builtins -Fix a reference bug in :func:`_imp.create_builtin()` after the creation of +Fix a reference bug in :func:`!_imp.create_builtin` after the creation of the first sub-interpreter for modules ``builtins`` and ``sys``. Patch by Victor Stinner. @@ -82,7 +82,7 @@ Victor Stinner. .. section: Core and Builtins Fixed a bug that was causing a buffer overflow if the tokenizer copies a -line missing the newline caracter from a file that is as long as the +line missing the newline character from a file that is as long as the available tokenizer buffer. Patch by Pablo galindo .. @@ -221,7 +221,7 @@ returns an invalid file descriptor. Also \ escape \s in the http.server BaseHTTPRequestHandler.log_message so that it is technically possible to parse the line and reconstruct what the -original data was. Without this a \xHH is ambiguious as to if it is a hex +original data was. Without this a \xHH is ambiguous as to if it is a hex replacement we put in or the characters r"\x" came through in the original request line. @@ -399,7 +399,7 @@ Fix refcount error when arguments are packed to tuple in Argument Clinic. .. nonce: 7uCiIB .. section: Library -:meth:`pathlib.PurePath.relative_to()` now treats naked Windows drive paths +:meth:`pathlib.PurePath.relative_to` now treats naked Windows drive paths as relative. This brings its behaviour in line with other parts of pathlib. .. @@ -496,7 +496,7 @@ Created packages from zipfile and test_zipfile modules, separating Fix :attr:`~ipaddress.IPv4Address.is_private` properties in the :mod:`ipaddress` module. Previously non-private networks (0.0.0.0/0) would -return True from this method; now they correctly return False. +return ``True`` from this method; now they correctly return ``False``. .. diff --git a/Misc/NEWS.d/3.12.0a4.rst b/Misc/NEWS.d/3.12.0a4.rst index 82faa5ad0b2031..57fb2052764b6f 100644 --- a/Misc/NEWS.d/3.12.0a4.rst +++ b/Misc/NEWS.d/3.12.0a4.rst @@ -65,8 +65,8 @@ redundant. .. nonce: M2n6Kg .. section: Core and Builtins -Fix :func:`int.__sizeof__` calculation to include the 1 element ob_digit -array for 0 and False. +Fix :func:`int.__sizeof__` calculation to include the 1-element ``ob_digit`` +array for ``0`` and ``False``. .. @@ -241,7 +241,7 @@ are now always dumped, even if switched off. Improve ``BUILD_LIST`` opcode so that it works similarly to the ``BUILD_TUPLE`` opcode, by stealing references from the stack rather than repeatedly using stack operations to set list elements. Implementation -details are in a new private API :c:func:`_PyList_FromArraySteal`. +details are in a new private API :c:func:`!_PyList_FromArraySteal`. .. @@ -611,8 +611,8 @@ random.expovariate(). .. nonce: bgtzMV .. section: Library -A :exc:`DeprecationWarning` may be raised when :func:`os.fork()` or -:func:`os.forkpty()` is called from multi-threaded processes. Forking with +A :exc:`DeprecationWarning` may be raised when :func:`os.fork` or +:func:`os.forkpty` is called from multi-threaded processes. Forking with threads is unsafe and can cause deadlocks, crashes and subtle problems. Lack of a warning does not indicate that the fork call was actually safe, as Python may not be aware of all threads. @@ -830,7 +830,7 @@ Reduced the memory usage of :func:`urllib.parse.unquote` and .. section: Library ``inspect.signature`` was raising ``TypeError`` on call with mock objects. -Now it correctly returns ``(*args, **kwargs)`` as infered signature. +Now it correctly returns ``(*args, **kwargs)`` as inferred signature. .. diff --git a/Misc/NEWS.d/3.12.0a5.rst b/Misc/NEWS.d/3.12.0a5.rst index 8cf90b0e9cde46..5dc443bb55b617 100644 --- a/Misc/NEWS.d/3.12.0a5.rst +++ b/Misc/NEWS.d/3.12.0a5.rst @@ -287,7 +287,7 @@ a positional argument would lead to a :exc:`TypeError`. .. section: Library Group-related variables of ``_posixsubprocess`` module are renamed to stress -that supplimentary group affinity is added to a fork, not replace the +that supplementary group affinity is added to a fork, not replace the inherited ones. Patch by Oleg Iarygin. .. @@ -307,7 +307,7 @@ It must not drop the ``Unpack`` part. .. nonce: wz4Xgc .. section: Library -Add :func:`os.path.splitroot()`, which splits a path into a 3-item tuple +Add :func:`os.path.splitroot`, which splits a path into a 3-item tuple ``(drive, root, tail)``. This new function is used by :mod:`pathlib` to improve the performance of path construction by up to a third. @@ -506,7 +506,7 @@ inheritance. .. nonce: 7sQz5l .. section: Build -Update BOLT configration not to use depreacted usage of ``--split +Update BOLT configuration not to use deprecated usage of ``--split functions``. Patch by Donghee Na. .. diff --git a/Misc/NEWS.d/3.12.0a6.rst b/Misc/NEWS.d/3.12.0a6.rst index cf28bdb9258820..bc708d163ce0e9 100644 --- a/Misc/NEWS.d/3.12.0a6.rst +++ b/Misc/NEWS.d/3.12.0a6.rst @@ -15,9 +15,9 @@ from the HACL* project. .. section: Security Updated the OpenSSL version used in Windows and macOS binary release builds -to 1.1.1t to address CVE-2023-0286, CVE-2022-4303, and CVE-2022-4303 per +to 1.1.1t to address :cve:`2023-0286`, :cve:`2022-4303`, and :cve:`2022-4303` per `the OpenSSL 2023-02-07 security advisory -`_. +`_. .. @@ -453,7 +453,7 @@ E. Aasland. .. section: Library Change repr of :class:`collections.OrderedDict` to use regular dictionary -formating instead of pairs of keys and values. +formatting instead of pairs of keys and values. .. diff --git a/Misc/NEWS.d/3.12.0a7.rst b/Misc/NEWS.d/3.12.0a7.rst index a859be8a047456..f48b9ce0550440 100644 --- a/Misc/NEWS.d/3.12.0a7.rst +++ b/Misc/NEWS.d/3.12.0a7.rst @@ -219,7 +219,7 @@ Aasland. .. nonce: DqNehf .. section: Library -Pure python :func:`locale.getencoding()` will not warn deprecation. +Pure python :func:`locale.getencoding` will not warn deprecation. .. diff --git a/Misc/NEWS.d/3.12.0b1.rst b/Misc/NEWS.d/3.12.0b1.rst index d9804be764c9a9..3a3870ac9fe621 100644 --- a/Misc/NEWS.d/3.12.0b1.rst +++ b/Misc/NEWS.d/3.12.0b1.rst @@ -37,7 +37,7 @@ or lacks SHA3. :func:`urllib.parse.urlsplit` now strips leading C0 control and space characters following the specification for URLs defined by WHATWG in -response to CVE-2023-24329. Patch by Illia Volochii. +response to :cve:`2023-24329`. Patch by Illia Volochii. .. @@ -395,7 +395,7 @@ Fix bug in line numbers of instructions emitted for :keyword:`except* .. section: Core and Builtins Clarify :exc:`SyntaxWarning` with literal ``is`` comparison by specifying -which literal is problematic, since comparisons using ``is`` with e.g. None +which literal is problematic, since comparisons using ``is`` with e.g. ``None`` and bool literals are idiomatic. .. @@ -1446,7 +1446,7 @@ Adapt the :mod:`winsound` extension module to :pep:`687`. .. nonce: jurMzv .. section: Library -Remove deprecation of enum ``memmber.member`` access. +Remove deprecation of enum ``member.member`` access. .. @@ -1750,7 +1750,7 @@ Remove the long-deprecated ``imp`` module. .. nonce: N46coo .. section: Library -Deprecate :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` in +Deprecate :func:`!pkgutil.find_loader` and :func:`!pkgutil.get_loader` in favor of :func:`importlib.util.find_spec`. .. @@ -1828,7 +1828,7 @@ is relative. .. nonce: 511Tbh .. section: Library -Convert private :meth:`_posixsubprocess.fork_exec` to use Argument Clinic. +Convert private :meth:`!_posixsubprocess.fork_exec` to use Argument Clinic. .. diff --git a/Misc/NEWS.d/3.13.0a1.rst b/Misc/NEWS.d/3.13.0a1.rst index 16715bee5a8e49..c32c9a537d7b53 100644 --- a/Misc/NEWS.d/3.13.0a1.rst +++ b/Misc/NEWS.d/3.13.0a1.rst @@ -8,9 +8,7 @@ Fixed an issue where instances of :class:`ssl.SSLSocket` were vulnerable to a bypass of the TLS handshake and included protections (like certificate verification) and treating sent unencrypted data as if it were post-handshake TLS encrypted data. Security issue reported as -`CVE-2023-40217 -`_ by Aapo -Oksman. Patch by Gregory P. Smith. +:cve:`2023-40217` by Aapo Oksman. Patch by Gregory P. Smith. .. @@ -565,7 +563,7 @@ on deallocation. Fix :meth:`multiprocessing.synchronize.SemLock.__setstate__` to properly initialize :attr:`multiprocessing.synchronize.SemLock._is_fork_ctx`. This -fixes a regression when passing a SemLock accross nested processes. +fixes a regression when passing a SemLock across nested processes. Rename :attr:`multiprocessing.synchronize.SemLock.is_fork_ctx` to :attr:`multiprocessing.synchronize.SemLock._is_fork_ctx` to avoid exposing @@ -710,7 +708,7 @@ Fixes crash when tracing in recursive calls to Python classes. .. section: Core and Builtins Remove the ``_PyCFrame`` struct, moving the pointer to the current -intepreter frame back to the threadstate, as it was for 3.10 and earlier. +interpreter frame back to the threadstate, as it was for 3.10 and earlier. The ``_PyCFrame`` existed as a performance optimization for tracing. Since PEP 669 has been implemented, this optimization no longer applies. @@ -928,7 +926,7 @@ Isolate :mod:`!_decimal` (apply :pep:`687`). Patch by Charlie Zhao. Add the exception as the third argument to ``PY_UNIND`` callbacks in ``sys.monitoring``. This makes the ``PY_UNWIND`` callback consistent with -the other exception hanlding callbacks. +the other exception handling callbacks. .. @@ -937,7 +935,7 @@ the other exception hanlding callbacks. .. nonce: DdEwV8 .. section: Core and Builtins -Raise a ``ValueError`` when a monitoring callback funtion returns +Raise a ``ValueError`` when a monitoring callback function returns ``DISABLE`` for events that cannot be disabled locally. .. @@ -1008,7 +1006,7 @@ Add :meth:`dbm.gnu.gdbm.clear` to :mod:`dbm.gnu`. Patch By Donghee Na. .. section: Core and Builtins The ASYNC and AWAIT tokens are removed from the Grammar, which removes the -posibility of making ``async`` and ``await`` soft keywords when using +possibility of making ``async`` and ``await`` soft keywords when using ``feature_version<7`` in :func:`ast.parse`. .. @@ -1030,7 +1028,7 @@ the call is not a classmethod. .. nonce: DdqHFg .. section: Core and Builtins -Python no longer crashes due an infrequent race when initialzing +Python no longer crashes due an infrequent race when initializing per-interpreter interned strings. The crash would manifest when the interpreter was finalized. @@ -1924,7 +1922,7 @@ objects .. nonce: RDGe8- .. section: Library -Deprecation warning about non-integer number in :mod:`gettext` now alwais +Deprecation warning about non-integer number in :mod:`gettext` now always refers to the line in the user code where gettext function or method is used. Previously it could refer to a line in ``gettext`` code. @@ -1966,7 +1964,7 @@ debugging. .. nonce: LCxiau .. section: Library -Fix :func:`termios.tcsetattr` bug that was overwritting existing errors +Fix :func:`termios.tcsetattr` bug that was overwriting existing errors during parsing integers from ``term`` list. .. @@ -2049,7 +2047,7 @@ point. .. nonce: fECxTj .. section: Library -On Windows, multiprocessing ``Popen.terminate()`` now catchs +On Windows, multiprocessing ``Popen.terminate()`` now catches :exc:`PermissionError` and get the process exit code. If the process is still running, raise again the :exc:`PermissionError`. Otherwise, the process terminated as expected: store its exit code. Patch by Victor @@ -2859,7 +2857,7 @@ Seems that in some conditions, OpenSSL will return ``SSL_ERROR_SYSCALL`` instead of ``SSL_ERROR_SSL`` when a certification verification has failed, but the error parameters will still contain ``ERR_LIB_SSL`` and ``SSL_R_CERTIFICATE_VERIFY_FAILED``. We are now detecting this situation and -raising the appropiate ``ssl.SSLCertVerificationError``. Patch by Pablo +raising the appropriate ``ssl.SSLCertVerificationError``. Patch by Pablo Galindo .. @@ -2890,9 +2888,9 @@ documented and were not intended to be used externally. .. nonce: vMbmj_ .. section: Library -:data:`opcode.ENABLE_SPECIALIZATION` (which was added in 3.12 but never +:data:`!opcode.ENABLE_SPECIALIZATION` (which was added in 3.12 but never documented or intended for external usage) is moved to -:data:`_opcode.ENABLE_SPECIALIZATION` where tests can access it. +:data:`!_opcode.ENABLE_SPECIALIZATION` where tests can access it. .. @@ -2981,7 +2979,7 @@ method. Patch by James Cave. .. section: Library Fix overflow on 32-bit systems with :mod:`asyncio` :func:`os.sendfile` -implemention. +implementation. .. @@ -3055,7 +3053,7 @@ Donghee Na. .. nonce: U9nD_B .. section: Library -Optimize :meth:`_PollLikeSelector.select` for many iteration case. +Optimize :meth:`!_PollLikeSelector.select` for many iteration case. .. @@ -3175,7 +3173,7 @@ Disable tab completion in multiline mode of :mod:`pdb` .. nonce: pYSwMj .. section: Library -Expose opcode metadata through :mod:`_opcode`. +Expose opcode metadata through :mod:`!_opcode`. .. @@ -3253,7 +3251,7 @@ Eliseev. .. nonce: NN35-U .. section: Library -Optimize ``(?!)`` (pattern which alwais fails) in regular expressions. +Optimize ``(?!)`` (pattern which always fails) in regular expressions. .. @@ -3438,7 +3436,8 @@ added support for this decorator. Patch by Alex Waygood. .. nonce: C1ahtk .. section: Library -Make pydoc.doc catch bad module ImportError when output stream is not None. +Make :func:`pydoc.doc` catch bad module :exc:`ImportError` +when output stream is not ``None``. .. @@ -3496,7 +3495,7 @@ star imports. .. nonce: TJEUkd .. section: Library -Zipapp will now skip over apending an archive to itself. +Zipapp will now skip over appending an archive to itself. .. @@ -3736,7 +3735,7 @@ overwritten. .. nonce: _sZilh .. section: Library -Fix bugs in :mod:`_ctypes` where exceptions could end up being overwritten. +Fix bugs in :mod:`!_ctypes` where exceptions could end up being overwritten. .. @@ -4184,8 +4183,7 @@ Hugo van Kemenade. .. section: Library :pep:`594`: Remove the :mod:`!spwd` module, deprecated in Python 3.11: the -`python-pam project `_ can be used -instead. Patch by Victor Stinner. +:pypi:`python-pam` project can be used instead. Patch by Victor Stinner. .. @@ -4566,7 +4564,7 @@ Deprecate passing any arguments to :func:`threading.RLock`. .. nonce: o5Zb0t .. section: Library -Refactored ``zipfile._strip_extra`` to use higher level abstactions for +Refactored ``zipfile._strip_extra`` to use higher level abstractions for extras instead of a heavy-state loop. .. @@ -5016,7 +5014,7 @@ by Victor Stinner. Fix test_timeout() of test_concurrent_futures.test_wait. Remove the future which may or may not complete depending if it takes longer than the timeout -ot not. Keep the second future which does not complete before wait() +or not. Keep the second future which does not complete before wait() timeout. Patch by Victor Stinner. .. @@ -5106,7 +5104,7 @@ Victor Stinner. regrtest: Add ``--fast-ci`` and ``--slow-ci`` options. ``--fast-ci`` uses a default timeout of 10 minutes and ``-u all,-cpu`` (skip slowest tests). -``--slow-ci`` uses a default timeout of 20 minues and ``-u all`` (run all +``--slow-ci`` uses a default timeout of 20 minutes and ``-u all`` (run all tests). Patch by Victor Stinner. .. @@ -5150,7 +5148,7 @@ frame did not save the PC`` is found. Patch by Victor Stinner. .. nonce: qxI4OG .. section: Tests -libregrtest now calls :func:`random.seed()` before running each test file +libregrtest now calls :func:`random.seed` before running each test file when ``-r/--randomize`` command line option is used. Moreover, it's also called in worker processes. It should help to make tests more deterministic. Previously, it was only called once in the main process @@ -5234,7 +5232,7 @@ and ``sysctl net.inet.udp.blackhole=1``). Patch by Victor Stinner. Skip ``test_gdb`` if gdb is unable to retrieve Python frame objects: if a frame is ````. When Python is built with "clang -Og", gdb can -fail to retrive the *frame* parameter of ``_PyEval_EvalFrameDefault()``. In +fail to retrieve the *frame* parameter of ``_PyEval_EvalFrameDefault()``. In this case, tests like ``py_bt()`` are likely to fail. Without getting access to Python frames, ``python-gdb.py`` is mostly clueless on retrieving the Python traceback. Moreover, ``test_gdb`` is no longer skipped on macOS if diff --git a/Misc/NEWS.d/3.13.0a2.rst b/Misc/NEWS.d/3.13.0a2.rst index e5841e14c02efb..a1a2d8a42ec03b 100644 --- a/Misc/NEWS.d/3.13.0a2.rst +++ b/Misc/NEWS.d/3.13.0a2.rst @@ -228,7 +228,7 @@ cross-interpreter API. Patch by Anthony Shaw. .. nonce: ageUWQ .. section: Core and Builtins -Add support for sharing of True and False between interpreters using the +Add support for sharing of ``True`` and ``False`` between interpreters using the cross-interpreter API. Patch by Anthony Shaw. .. @@ -632,8 +632,8 @@ Add extra argument validation for ``alias`` command in :mod:`pdb` .. nonce: FWqZIU .. section: Library -:mod:`time`: Make :func:`time.clock_gettime()` and -:func:`time.clock_gettime_ns()` functions up to 2x faster by faster calling +:mod:`time`: Make :func:`time.clock_gettime` and +:func:`time.clock_gettime_ns` functions up to 2x faster by faster calling convention. Patch by Victor Stinner. .. @@ -674,7 +674,7 @@ Unix socket when the server is closed. .. nonce: Bc8LvA .. section: Library -Added :func:`io.text_encoding()`, :data:`io.DEFAULT_BUFFER_SIZE`, and +Added :func:`io.text_encoding`, :data:`io.DEFAULT_BUFFER_SIZE`, and :class:`io.IncrementalNewlineDecoder` to ``io.__all__``. .. @@ -736,7 +736,7 @@ which allows to format :exc:`ExceptionGroup` instances. .. nonce: 5ePgFl .. section: Library -Another attempt at fixing :func:`asyncio.Server.wait_closed()`. It now +Another attempt at fixing :func:`asyncio.Server.wait_closed`. It now blocks until both conditions are true: the server is closed, *and* there are no more active connections. (This means that in some cases where in 3.12.0 this function would *incorrectly* have returned immediately, it will now @@ -777,7 +777,7 @@ Add error checking during :mod:`!_socket` module init. .. nonce: urFYtn .. section: Library -Fix :mod:`_blake2` not checking for errors when initializing. +Fix :mod:`!_blake2` not checking for errors when initializing. .. @@ -880,7 +880,7 @@ Make :mod:`pdb` enter post-mortem mode even for :exc:`SyntaxError` .. nonce: _M-cQC .. section: Library -Set ``f_trace_lines = True`` on all frames upon :func:`pdb.set_trace()` +Set ``f_trace_lines = True`` on all frames upon :func:`pdb.set_trace` .. @@ -974,7 +974,7 @@ pattern. .. nonce: 6ah-aw .. section: Library -Add the :attr:`ipaddress.IPv4Address.ipv6_mapped` property, which retuns the +Add the :attr:`ipaddress.IPv4Address.ipv6_mapped` property, which returns the IPv4-mapped IPv6 address. .. @@ -1354,8 +1354,8 @@ crash encountered after the first :meth:`tkinter.Tk` instance is destroyed. .. section: IDLE Add docstrings to the IDLE debugger module. Fix two bugs: initialize -Idb.botframe (should be in Bdb); in Idb.in_rpc_code, check whether -prev_frame is None before trying to use it. Greatly expand test_debugger. +``Idb.botframe`` (should be in Bdb); in ``Idb.in_rpc_code``, check whether +``prev_frame`` is ``None`` before trying to use it. Greatly expand test_debugger. .. diff --git a/Misc/NEWS.d/3.13.0a3.rst b/Misc/NEWS.d/3.13.0a3.rst index 95aa66603de7cb..0f8dee261c6589 100644 --- a/Misc/NEWS.d/3.13.0a3.rst +++ b/Misc/NEWS.d/3.13.0a3.rst @@ -269,7 +269,7 @@ Correctly compute end column offsets for multiline tokens in the .. nonce: 4ADN7i .. section: Core and Builtins -Fix None.__ne__(None) returning NotImplemented instead of False +Fix ``None.__ne__(None)`` returning ``NotImplemented`` instead of ``False``. .. @@ -449,8 +449,8 @@ well-formed for surrogateescape encoding. Patch by Sidney Markowitz. .. nonce: N8E1zw .. section: Core and Builtins -Use the object's actual class name in :meth:`_io.FileIO.__repr__`, -:meth:`_io._WindowsConsoleIO` and :meth:`_io.TextIOWrapper.__repr__`, to +Use the object's actual class name in :meth:`!_io.FileIO.__repr__`, +:meth:`!_io._WindowsConsoleIO` and :meth:`!_io.TextIOWrapper.__repr__`, to make these methods subclass friendly. .. @@ -555,7 +555,7 @@ Added :data:`mmap.MAP_NORESERVE`, :data:`mmap.MAP_NOEXTEND`, .. nonce: kXoCy0 .. section: Library -:func:`asyncio.TaskGroup()` and :func:`asyncio.timeout()` context managers +:func:`asyncio.TaskGroup` and :func:`asyncio.timeout` context managers now handle :exc:`~asyncio.CancelledError` subclasses as well as exact :exc:`!CancelledError`. @@ -607,9 +607,9 @@ with the documentation) .. nonce: xN2LuL .. section: Library -:func:`asyncio.Condition.wait()` now re-raises the same +:func:`asyncio.Condition.wait` now re-raises the same :exc:`CancelledError` instance that may have caused it to be interrupted. -Fixed race condition in :func:`asyncio.Semaphore.aquire` when interrupted +Fixed race condition in :func:`asyncio.Semaphore.acquire` when interrupted with a :exc:`CancelledError`. .. @@ -863,7 +863,7 @@ Fixed tarfile list() method to show file type. .. nonce: jLWGlr .. section: Library -:meth:`asyncio.futures.Future.set_exception()` now transforms +:meth:`asyncio.futures.Future.set_exception` now transforms :exc:`StopIteration` into :exc:`RuntimeError` instead of hanging or other misbehavior. Patch contributed by Jamie Phan. @@ -928,7 +928,7 @@ on Windows. .. section: Library Fix :func:`shutil.copymode` and :func:`shutil.copystat` on Windows. -Previously they worked differenly if *dst* is a symbolic link: they modified +Previously they worked differently if *dst* is a symbolic link: they modified the permission bits of *dst* itself rather than the file it points to if *follow_symlinks* is true or *src* is not a symbolic link, and did not modify the permission bits if *follow_symlinks* is false and *src* is a @@ -1550,8 +1550,8 @@ addresses are encountered instead of potentially inaccurate values. Add optional *strict* parameter to these two functions: use ``strict=False`` to get the old behavior, accept malformed inputs. ``getattr(email.utils, 'supports_strict_parsing', False)`` can be use to check if the *strict* -paramater is available. Patch by Thomas Dwyer and Victor Stinner to improve -the CVE-2023-27043 fix. +parameter is available. Patch by Thomas Dwyer and Victor Stinner to improve +the :cve:`2023-27043` fix. .. @@ -1615,7 +1615,7 @@ method of :class:`itertools.pairwise`. .. section: Library Small (10 - 20%) and trivial performance improvement of -:func:`urrlib.request.getproxies_environment`, typically useful when there +:func:`urllib.request.getproxies_environment`, typically useful when there are many environment variables to go over. .. @@ -1805,7 +1805,7 @@ size. .. nonce: xPOBBY .. section: Library -:func:`warnings.filterwarnings()` and :func:`warnings.simplefilter()` now +:func:`warnings.filterwarnings` and :func:`warnings.simplefilter` now raise appropriate exceptions instead of ``AssertionError``. Patch contributed by Rémi Lapeyre. diff --git a/Misc/NEWS.d/3.13.0a4.rst b/Misc/NEWS.d/3.13.0a4.rst index 39af0534cf8fb5..1b971113173e0a 100644 --- a/Misc/NEWS.d/3.13.0a4.rst +++ b/Misc/NEWS.d/3.13.0a4.rst @@ -336,7 +336,7 @@ for decorated functions. .. nonce: RzxNYT .. section: Library -Fix several :func:`format()` bugs when using the C implementation of +Fix several :func:`format` bugs when using the C implementation of :class:`~decimal.Decimal`: * memory leak in some rare cases when using the ``z`` format option (coerce negative 0) * incorrect output when applying the ``z`` format option to type ``F`` (fixed-point with capital ``NAN`` / @@ -1181,7 +1181,7 @@ configure.ac. .. nonce: XcEXEZ .. section: Build -configure and Makefile were refactored to accomodate framework builds on +configure and Makefile were refactored to accommodate framework builds on Apple platforms other than macOS. .. @@ -1283,7 +1283,7 @@ Update macOS installer to use OpenSSL 3.0.13. .. nonce: FrQOQ0 .. section: macOS -Add Mach-O linkage support for :func:`platform.architecture()`. +Add Mach-O linkage support for :func:`platform.architecture`. .. diff --git a/Misc/NEWS.d/3.13.0a5.rst b/Misc/NEWS.d/3.13.0a5.rst new file mode 100644 index 00000000000000..d56b1542b01823 --- /dev/null +++ b/Misc/NEWS.d/3.13.0a5.rst @@ -0,0 +1,1163 @@ +.. date: 2024-02-18-03-14-40 +.. gh-issue: 115398 +.. nonce: tzvxH8 +.. release date: 2024-03-12 +.. section: Security + +Allow controlling Expat >=2.6.0 reparse deferral (:cve:`2023-52425`) by adding +five new methods: + +* :meth:`xml.etree.ElementTree.XMLParser.flush` +* :meth:`xml.etree.ElementTree.XMLPullParser.flush` +* :meth:`xml.parsers.expat.xmlparser.GetReparseDeferralEnabled` +* :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` +* :meth:`xml.sax.expatreader.ExpatParser.flush` + +.. + +.. date: 2024-01-26-22-14-09 +.. gh-issue: 114572 +.. nonce: t1QMQD +.. section: Security + +:meth:`ssl.SSLContext.cert_store_stats` and +:meth:`ssl.SSLContext.get_ca_certs` now correctly lock access to the +certificate store, when the :class:`ssl.SSLContext` is shared across +multiple threads. + +.. + +.. date: 2024-03-11-22-24-59 +.. gh-issue: 116604 +.. nonce: LCEzAT +.. section: Core and Builtins + +Respect the status of the garbage collector when indirect calls are made via +:c:func:`PyErr_CheckSignals` and the evaluation breaker. Patch by Pablo +Galindo + +.. + +.. date: 2024-03-09-11-10-53 +.. gh-issue: 112087 +.. nonce: nbI0Pw +.. section: Core and Builtins + +:class:`list` is now compatible with the implementation of :pep:`703`. + +.. + +.. date: 2024-03-05-22-00-58 +.. gh-issue: 116381 +.. nonce: 0Nq9iO +.. section: Core and Builtins + +Add specialization for ``CONTAINS_OP``. + +.. + +.. date: 2024-03-04-10-19-51 +.. gh-issue: 116296 +.. nonce: gvtxyU +.. section: Core and Builtins + +Fix possible refleak in :meth:`!object.__reduce__` internal error handling. + +.. + +.. date: 2024-02-22-16-17-53 +.. gh-issue: 115823 +.. nonce: c1TreJ +.. section: Core and Builtins + +Properly calculate error ranges in the parser when raising +:exc:`SyntaxError` exceptions caused by invalid byte sequences. Patch by +Pablo Galindo + +.. + +.. date: 2024-02-22-11-33-20 +.. gh-issue: 115778 +.. nonce: jksd1D +.. section: Core and Builtins + +Add ``tierN`` annotation for instruction definition in interpreter DSL. + +.. + +.. date: 2024-02-20-18-49-02 +.. gh-issue: 115733 +.. nonce: 51Zb85 +.. section: Core and Builtins + +Fix crash when calling ``next()`` on exhausted list iterators. + +.. + +.. date: 2024-02-20-12-46-20 +.. gh-issue: 115700 +.. nonce: KLJ5r4 +.. section: Core and Builtins + +The regen-cases build stage now works on Windows. + +.. + +.. date: 2024-02-14-23-50-43 +.. gh-issue: 115347 +.. nonce: VkHvQC +.. section: Core and Builtins + +Fix bug where docstring was replaced by a redundant NOP when Python is run +with ``-OO``. + +.. + +.. date: 2024-02-12-23-29-17 +.. gh-issue: 115323 +.. nonce: 3t6687 +.. section: Core and Builtins + +Make error message more meaningful for when :meth:`bytearray.extend` is +called with a :class:`str` object. + +.. + +.. date: 2024-02-09-18-59-22 +.. gh-issue: 112175 +.. nonce: qglugr +.. section: Core and Builtins + +Every ``PyThreadState`` now has its own ``eval_breaker``, allowing specific +threads to be interrupted. + +.. + +.. date: 2024-02-08-16-01-18 +.. gh-issue: 115154 +.. nonce: ji96FV +.. section: Core and Builtins + +Fix a bug that was causing the :func:`tokenize.untokenize` function to +handle unicode named literals incorrectly. Patch by Pablo Galindo + +.. + +.. date: 2024-01-28-02-46-12 +.. gh-issue: 112433 +.. nonce: FUX-nT +.. section: Core and Builtins + +Add ability to force alignment of :mod:`ctypes.Structure` by way of the new +``_align_`` attribute on the class. + +.. + +.. date: 2023-07-16-15-02-47 +.. gh-issue: 104090 +.. nonce: oMjNa9 +.. section: Core and Builtins + +The multiprocessing resource tracker now exits with non-zero status code if +a resource leak was detected. It still exits with status code 0 otherwise. + +.. + +.. date: 2023-06-16-21-29-06 +.. gh-issue: 105858 +.. nonce: Q7h0EV +.. section: Core and Builtins + +Improve the constructors for :mod:`ast` nodes. Arguments of list types now +default to an empty list if omitted, and optional fields default to +``None``. AST nodes now have an ``__annotations__`` attribute with the +expected types of their attributes. Passing unrecognized extra arguments to +AST nodes is deprecated and will become an error in Python 3.15. Omitting a +required argument to an AST node is deprecated and will become an error in +Python 3.15. Patch by Jelle Zijlstra. + +.. + +.. date: 2023-02-13-11-36-50 +.. gh-issue: 101860 +.. nonce: CKCMbC +.. section: Core and Builtins + +Expose ``__name__`` attribute on property. + +.. + +.. date: 2022-09-04-16-51-56 +.. gh-issue: 96497 +.. nonce: HTBuIL +.. section: Core and Builtins + +Fix incorrect resolution of mangled class variables used in assignment +expressions in comprehensions. + +.. + +.. date: 2024-03-11-12-11-10 +.. gh-issue: 116600 +.. nonce: FcNBy_ +.. section: Library + +Fix :func:`repr` for global :class:`~enum.Flag` members. + +.. + +.. date: 2024-03-07-21-57-50 +.. gh-issue: 116349 +.. nonce: fD2pbP +.. section: Library + +:func:`platform.java_ver` is deprecated and will be removed in 3.15. It was +largely untested, had a confusing API, and was only useful for Jython +support. + +.. + +.. date: 2024-03-05-20-53-34 +.. gh-issue: 116143 +.. nonce: sww6Zl +.. section: Library + +Fix a race in pydoc ``_start_server``, eliminating a window in which +``_start_server`` can return a thread that is "serving" but without a +``docserver`` set. + +.. + +.. date: 2024-03-05-14-34-22 +.. gh-issue: 116127 +.. nonce: 5uktu3 +.. section: Library + +:mod:`typing`: implement :pep:`705` which adds :data:`typing.ReadOnly` +support to :class:`typing.TypedDict`. + +.. + +.. date: 2024-03-05-02-09-18 +.. gh-issue: 116325 +.. nonce: FmlBYv +.. section: Library + +:mod:`typing`: raise :exc:`SyntaxError` instead of :exc:`AttributeError` on +forward references as empty strings. + +.. + +.. date: 2024-03-02-11-31-49 +.. gh-issue: 115957 +.. nonce: C-3Z_U +.. section: Library + +When ``asyncio.TaskGroup.create_task`` is called on an inactive +``asyncio.TaskGroup``, the given coroutine will be closed (which prevents a +``RuntimeWarning``). + +.. + +.. date: 2024-03-01-14-22-08 +.. gh-issue: 115978 +.. nonce: r2ePTo +.. section: Library + +Disable preadv(), readv(), pwritev(), and writev() on WASI. + +Under wasmtime for WASI 0.2, these functions don't pass test_posix +(https://github.com/bytecodealliance/wasmtime/issues/7830). + +.. + +.. date: 2024-03-01-11-57-32 +.. gh-issue: 88352 +.. nonce: bZ68rw +.. section: Library + +Fix the computation of the next rollover time in the +:class:`logging.TimedRotatingFileHandler` handler. :meth:`!computeRollover` +now always returns a timestamp larger than the specified time and works +correctly during the DST change. :meth:`!doRollover` no longer overwrite the +already rolled over file, saving from data loss when run at midnight or +during repeated time at the DST change. + +.. + +.. date: 2024-02-29-20-06-06 +.. gh-issue: 87115 +.. nonce: FVMiOR +.. section: Library + +Set ``__main__.__spec__`` to ``None`` when running a script with :mod:`pdb` + +.. + +.. date: 2024-02-29-17-06-54 +.. gh-issue: 76511 +.. nonce: WqjRLP +.. section: Library + +Fix UnicodeEncodeError in :meth:`email.Message.as_string` that results when +a message that claims to be in the ascii character set actually has +non-ascii characters. Non-ascii characters are now replaced with the U+FFFD +replacement character, like in the ``replace`` error handler. + +.. + +.. date: 2024-02-28-17-50-42 +.. gh-issue: 89547 +.. nonce: GetF38 +.. section: Library + +Add support for nested typing special forms like Final[ClassVar[int]]. + +.. + +.. date: 2024-02-28-17-04-28 +.. gh-issue: 65824 +.. nonce: gG8KR1 +.. section: Library + +Improve the ``less`` prompt in :mod:`pydoc`. + +.. + +.. date: 2024-02-28-13-10-17 +.. gh-issue: 116040 +.. nonce: wDidHd +.. section: Library + +[Enum] fix by-value calls when second value is falsey; e.g. Cardinal(1, 0) + +.. + +.. date: 2024-02-28-12-14-31 +.. gh-issue: 115821 +.. nonce: YO2vKA +.. section: Library + +[Enum] Improve error message when calling super().__new__() in custom +__new__. + +.. + +.. date: 2024-02-27-20-11-29 +.. gh-issue: 85644 +.. nonce: 3rgcBm +.. section: Library + +Use the ``XDG_CURRENT_DESKTOP`` environment variable in :mod:`webbrowser` to +check desktop. Prefer it to the deprecated ``GNOME_DESKTOP_SESSION_ID`` for +GNOME detection. + +.. + +.. date: 2024-02-27-13-05-51 +.. gh-issue: 75988 +.. nonce: In6LlB +.. section: Library + +Fixed :func:`unittest.mock.create_autospec` to pass the call through to the +wrapped object to return the real result. + +.. + +.. date: 2024-02-25-19-20-05 +.. gh-issue: 115881 +.. nonce: ro_Kuw +.. section: Library + +Fix issue where :func:`ast.parse` would incorrectly flag conditional context +managers (such as ``with (x() if y else z()): ...``) as invalid syntax if +``feature_version=(3, 8)`` was passed. This reverts changes to the grammar +made as part of gh-94949. + +.. + +.. date: 2024-02-24-18-48-14 +.. gh-issue: 115886 +.. nonce: rgM6AF +.. section: Library + +Fix silent truncation of the name with an embedded null character in +:class:`multiprocessing.shared_memory.SharedMemory`. + +.. + +.. date: 2024-02-23-11-08-31 +.. gh-issue: 115532 +.. nonce: zVd3gK +.. section: Library + +Add kernel density estimation to the statistics module. + +.. + +.. date: 2024-02-22-12-10-18 +.. gh-issue: 115714 +.. nonce: P2JsU1 +.. section: Library + +On WASI, the :mod:`time` module no longer get process time using ``times()`` +or ``CLOCK_PROCESS_CPUTIME_ID``, system API is that is unreliable and is +likely to be removed from WASI. The affected clock functions fall back to +calling ``clock()``. + +.. + +.. date: 2024-02-22-11-29-27 +.. gh-issue: 115809 +.. nonce: 9H1DhB +.. section: Library + +Improve algorithm for computing which rolled-over log files to delete in +:class:`logging.TimedRotatingFileHandler`. It is now reliable for handlers +without ``namer`` and with arbitrary deterministic ``namer`` that leaves the +datetime part in the file name unmodified. + +.. + +.. date: 2024-02-21-17-54-59 +.. gh-issue: 74668 +.. nonce: JT-Q8W +.. section: Library + +:mod:`urllib.parse` functions :func:`~urllib.parse.parse_qs` and +:func:`~urllib.parse.parse_qsl` now support bytes arguments containing raw +and percent-encoded non-ASCII data. + +.. + +.. date: 2024-02-20-22-02-34 +.. gh-issue: 67044 +.. nonce: QF9_Ru +.. section: Library + +:func:`csv.writer` now always quotes or escapes ``'\r'`` and ``'\n'``, +regardless of *lineterminator* value. + +.. + +.. date: 2024-02-20-16-42-54 +.. gh-issue: 115712 +.. nonce: EXVMXw +.. section: Library + +Restore support of space delimiter with ``skipinitialspace=True`` in +:mod:`csv`. :func:`csv.writer` now quotes empty fields if delimiter is a +space and skipinitialspace is true and raises exception if quoting is not +possible. + +.. + +.. date: 2024-02-20-07-38-15 +.. gh-issue: 112364 +.. nonce: EX7uGI +.. section: Library + +Fixed :func:`ast.unparse` to handle format_spec with ``"``, ``'`` or ``\\``. +Patched by Frank Hoffmann. + +.. + +.. date: 2024-02-19-16-53-48 +.. gh-issue: 112997 +.. nonce: sYBXRZ +.. section: Library + +Stop logging potentially sensitive callback arguments in :mod:`asyncio` +unless debug mode is active. + +.. + +.. date: 2024-02-19-15-52-30 +.. gh-issue: 114914 +.. nonce: M5-1d8 +.. section: Library + +Fix an issue where an abandoned :class:`StreamWriter` would not be garbage +collected. + +.. + +.. date: 2024-02-18-12-18-12 +.. gh-issue: 111358 +.. nonce: 9yJUMD +.. section: Library + +Fix a bug in :meth:`asyncio.BaseEventLoop.shutdown_default_executor` to +ensure the timeout passed to the coroutine behaves as expected. + +.. + +.. date: 2024-02-17-18-47-12 +.. gh-issue: 115618 +.. nonce: napiNp +.. section: Library + +Fix improper decreasing the reference count for ``None`` argument in +:class:`property` methods :meth:`~property.getter`, :meth:`~property.setter` +and :meth:`~property.deleter`. + +.. + +.. date: 2024-02-16-16-40-10 +.. gh-issue: 112720 +.. nonce: io6_Ac +.. section: Library + +Refactor :class:`dis.ArgResolver` to make it possible to subclass and change +the way jump args are interpreted. + +.. + +.. date: 2024-02-15-23-42-54 +.. gh-issue: 112006 +.. nonce: 4wxcK- +.. section: Library + +Fix :func:`inspect.unwrap` for types with the ``__wrapper__`` data +descriptor. Fix :meth:`inspect.Signature.from_callable` for builtins +:func:`classmethod` and :func:`staticmethod`. + +.. + +.. date: 2024-02-15-19-11-49 +.. gh-issue: 101293 +.. nonce: 898b8l +.. section: Library + +Support callables with the ``__call__()`` method and types with +``__new__()`` and ``__init__()`` methods set to class methods, static +methods, bound methods, partial functions, and other types of methods and +descriptors in :meth:`inspect.Signature.from_callable`. + +.. + +.. date: 2024-02-12-11-42-48 +.. gh-issue: 103092 +.. nonce: sGMKr0 +.. section: Library + +Isolate :mod:`!_lsprof` (apply :pep:`687`). + +.. + +.. date: 2024-02-11-20-12-39 +.. gh-issue: 113942 +.. nonce: i72sMJ +.. section: Library + +:mod:`pydoc` no longer skips global functions implemented as builtin +methods, such as :class:`~type.MethodDescriptorType` and +:class:`~type.WrapperDescriptorType`. + +.. + +.. date: 2024-02-10-17-18-49 +.. gh-issue: 115256 +.. nonce: 41Fy9P +.. section: Library + +Added DeprecationWarning when accessing the tarfile attribute of TarInfo +objects. The attribute is never used internally and is only attached to +TarInfos when the tarfile is opened in write-mode, not read-mode. The +attribute creates an unnecessary reference cycle which may cause corruption +when not closing the handle after writing a tarfile. + +.. + +.. date: 2024-02-09-19-41-48 +.. gh-issue: 115197 +.. nonce: 20wkWH +.. section: Library + +``urllib.request`` no longer resolves the hostname before checking it +against the system's proxy bypass list on macOS and Windows. + +.. + +.. date: 2024-02-09-12-22-47 +.. gh-issue: 113812 +.. nonce: wOraaG +.. section: Library + +:meth:`DatagramTransport.sendto` will now send zero-length datagrams if +called with an empty bytes object. The transport flow control also now +accounts for the datagram header when calculating the buffer size. + +.. + +.. date: 2024-01-30-23-28-29 +.. gh-issue: 114763 +.. nonce: BRjKkg +.. section: Library + +Protect modules loaded with :class:`importlib.util.LazyLoader` from race +conditions when multiple threads try to access attributes before the loading +is complete. + +.. + +.. date: 2024-01-29-13-46-41 +.. gh-issue: 114709 +.. nonce: SQ998l +.. section: Library + +:func:`posixpath.commonpath` now raises a :exc:`ValueError` exception when +passed an empty iterable. Previously, :exc:`IndexError` was raised. + +:func:`posixpath.commonpath` now raises a :exc:`TypeError` exception when +passed ``None``. Previously, :exc:`ValueError` was raised. + +.. + +.. date: 2024-01-26-16-42-31 +.. gh-issue: 114610 +.. nonce: S18Vuz +.. section: Library + +Fix bug where :meth:`pathlib.PurePath.with_stem` converted a non-empty path +suffix to a stem when given an empty *stem* argument. It now raises +:exc:`ValueError`, just like :meth:`pathlib.PurePath.with_suffix` does when +called on a path with an empty stem, given a non-empty *suffix* argument. + +.. + +.. date: 2023-11-24-23-40-00 +.. gh-issue: 107361 +.. nonce: v54gh46 +.. section: Library + +Add :data:`ssl.VERIFY_X509_PARTIAL_CHAIN` and :data:`VERIFY_X509_STRICT` to +the default SSL context created with :func:`ssl.create_default_context`. + +.. + +.. date: 2023-11-20-16-15-44 +.. gh-issue: 112281 +.. nonce: gH4EVk +.. section: Library + +Allow creating :ref:`union of types` for +:class:`typing.Annotated` with unhashable metadata. + +.. + +.. date: 2023-11-07-10-22-06 +.. gh-issue: 111775 +.. nonce: IoVxfX +.. section: Library + +Fix :meth:`importlib.resources.simple.ResourceHandle.open` for text mode, +added missed ``stream`` argument. + +.. + +.. date: 2023-10-07-06-15-13 +.. gh-issue: 90095 +.. nonce: gWn1ka +.. section: Library + +Make .pdbrc and -c work with any valid pdb commands. + +.. + +.. date: 2023-08-05-08-41-58 +.. gh-issue: 107625 +.. nonce: cVSHCT +.. section: Library + +Raise :exc:`configparser.ParsingError` from +:meth:`~configparser.ConfigParser.read` and +:meth:`~configparser.ConfigParser.read_file` methods of +:class:`configparser.ConfigParser` if a key without a corresponding value is +continued (that is, followed by an indented line). + +.. + +.. date: 2023-08-02-01-17-32 +.. gh-issue: 107155 +.. nonce: Mj1K9L +.. section: Library + +Fix incorrect output of ``help(x)`` where ``x`` is a :keyword:`lambda` +function, which has an ``__annotations__`` dictionary attribute with a +``"return"`` key. + +.. + +.. date: 2023-07-12-14-52-04 +.. gh-issue: 57141 +.. nonce: L2k8Xb +.. section: Library + +Add option for *non-shallow* comparisons to :class:`filecmp.dircmp` like +:func:`filecmp.cmp`. Original patch by Steven Ward. Enhanced by Tobias +Rautenkranz + +.. + +.. date: 2023-05-17-21-33-21 +.. gh-issue: 69990 +.. nonce: Blvz9G +.. section: Library + +:meth:`Profile.print_stats` has been improved to accept multiple sort +arguments. Patched by Chiu-Hsiang Hsu and Furkan Onder. + +.. + +.. date: 2023-05-01-22-28-57 +.. gh-issue: 104061 +.. nonce: vxfBXf +.. section: Library + +Add :data:`socket.SO_BINDTOIFINDEX` constant. + +.. + +.. date: 2023-04-02-21-20-35 +.. gh-issue: 60346 +.. nonce: 7mjgua +.. section: Library + +Fix ArgumentParser inconsistent with parse_known_args. + +.. + +.. date: 2023-03-03-09-05-42 +.. gh-issue: 102389 +.. nonce: ucmo0_ +.. section: Library + +Add ``windows_31j`` to aliases for ``cp932`` codec + +.. + +.. date: 2023-02-14-17-19-59 +.. gh-issue: 72249 +.. nonce: fv35wU +.. section: Library + +:func:`functools.partial`s of :func:`repr` has been improved to include the +:term:`module` name. Patched by Furkan Onder and Anilyka Barry. + +.. + +.. date: 2023-01-12-14-16-01 +.. gh-issue: 100985 +.. nonce: GT5Fvd +.. section: Library + +Update HTTPSConnection to consistently wrap IPv6 Addresses when using a +proxy. + +.. + +.. date: 2023-01-09-14-08-02 +.. gh-issue: 100884 +.. nonce: DcmdLl +.. section: Library + +email: fix misfolding of comma in address-lists over multiple lines in +combination with unicode encoding. + +.. + +.. date: 2022-11-22-23-17-43 +.. gh-issue: 95782 +.. nonce: an_and +.. section: Library + +Fix :func:`io.BufferedReader.tell`, :func:`io.BufferedReader.seek`, +:func:`!_pyio.BufferedReader.tell`, :func:`io.BufferedRandom.tell`, +:func:`io.BufferedRandom.seek` and :func:`!_pyio.BufferedRandom.tell` being +able to return negative offsets. + +.. + +.. date: 2022-08-26-15-50-53 +.. gh-issue: 96310 +.. nonce: 0NssDh +.. section: Library + +Fix a traceback in :mod:`argparse` when all options in a mutually exclusive +group are suppressed. + +.. + +.. date: 2022-05-25-17-49-04 +.. gh-issue: 93205 +.. nonce: DjhFVR +.. section: Library + +Fixed a bug in :class:`logging.handlers.TimedRotatingFileHandler` where +multiple rotating handler instances pointing to files with the same name but +different extensions would conflict and not delete the correct files. + +.. + +.. bpo: 31116 +.. date: 2022-01-14-10-50-17 +.. nonce: 0bduV9 +.. section: Library + +Add Z85 encoding to ``base64``. + +.. + +.. bpo: 44865 +.. date: 2021-08-24-20-47-37 +.. nonce: c3BhZS +.. section: Library + +Add missing call to localization function in :mod:`argparse`. + +.. + +.. bpo: 43952 +.. date: 2021-05-03-11-04-12 +.. nonce: Me7fJe +.. section: Library + +Fix :meth:`multiprocessing.connection.Listener.accept` to accept empty +bytes as authkey. Not accepting empty bytes as key causes it to hang +indefinitely. + +.. + +.. bpo: 42125 +.. date: 2020-12-15-22-30-49 +.. nonce: UGyseY +.. section: Library + +linecache: get module name from ``__spec__`` if available. This allows +getting source code for the ``__main__`` module when a custom loader is +used. + +.. + +.. bpo: 41122 +.. date: 2020-07-13-23-59-42 +.. nonce: 8P_Brh +.. section: Library + +Failing to pass arguments properly to :func:`functools.singledispatchmethod` +now throws a TypeError instead of hitting an index out of bounds internally. + +.. + +.. bpo: 40818 +.. date: 2020-05-29-18-08-54 +.. nonce: Ij8ffq +.. section: Library + +The asyncio REPL now runs :data:`sys.__interactivehook__` on startup. The +default implementation of :data:`sys.__interactivehook__` provides +auto-completion to the asyncio REPL. Patch contributed by Rémi Lapeyre. + +.. + +.. bpo: 33775 +.. date: 2019-04-06-23-50-59 +.. nonce: 0yhMDc +.. section: Library + +Add 'default' and 'version' help text for localization in argparse. + +.. + +.. date: 2024-02-14-20-17-04 +.. gh-issue: 115399 +.. nonce: fb9a0R +.. section: Documentation + +Document :cve:`2023-52425` of Expat <2.6.0 under "XML vulnerabilities". + +.. + +.. date: 2024-02-08-08-51-37 +.. gh-issue: 109653 +.. nonce: QHLW4w +.. section: Documentation + +Improve import time of :mod:`uuid` on Linux. + +.. + +.. date: 2024-02-25-16-28-26 +.. gh-issue: 71052 +.. nonce: lSb9EC +.. section: Tests + +Add test exclusions to support running the test suite on Android. + +.. + +.. date: 2024-02-25-15-58-28 +.. gh-issue: 71052 +.. nonce: lxBjqY +.. section: Tests + +Enable ``test_concurrent_futures`` on platforms that support threading but +not multiprocessing. + +.. + +.. date: 2024-02-22-00-17-06 +.. gh-issue: 115796 +.. nonce: d4hpKy +.. section: Tests + +Make '_testinternalcapi.assemble_code_object' construct the exception table +for the code object. + +.. + +.. date: 2024-02-20-15-47-41 +.. gh-issue: 115720 +.. nonce: w8i8UG +.. section: Tests + +Leak tests (``-R``, ``--huntrleaks``) now show a summary of the number of +leaks found in each iteration. + +.. + +.. date: 2024-02-18-14-20-52 +.. gh-issue: 115122 +.. nonce: 3rGNo9 +.. section: Tests + +Add ``--bisect`` option to regrtest test runner: run failed tests with +``test.bisect_cmd`` to identify failing tests. Patch by Victor Stinner. + +.. + +.. date: 2024-02-17-08-25-01 +.. gh-issue: 115596 +.. nonce: RGPCrR +.. section: Tests + +Fix ``ProgramPriorityTests`` in ``test_os`` permanently changing the process +priority. + +.. + +.. date: 2024-02-16-13-04-28 +.. gh-issue: 115556 +.. nonce: rjaQ9w +.. section: Tests + +On Windows, commas passed in arguments to ``Tools\buildbot\test.bat`` and +``PCbuild\\rt.bat`` are now properly handled. + +.. + +.. date: 2024-02-13-18-24-04 +.. gh-issue: 115420 +.. nonce: -dlzfI +.. section: Tests + +Fix translation of exception handler targets by +``_testinternalcapi.optimize_cfg``. + +.. + +.. date: 2024-02-12-22-35-01 +.. gh-issue: 115376 +.. nonce: n9vubZ +.. section: Tests + +Fix segfault in ``_testinternalcapi.compiler_codegen`` on bad input. + +.. + +.. date: 2024-03-04-12-43-42 +.. gh-issue: 116313 +.. nonce: cLLb8S +.. section: Build + +Get WASI builds to work under wasmtime 18 w/ WASI 0.2/preview2 primitives. + +.. + +.. date: 2024-03-01-16-44-19 +.. gh-issue: 71052 +.. nonce: Hs-9EP +.. section: Build + +Change Android's :data:`sys.platform` from ``"linux"`` to ``"android"``. + +.. + +.. date: 2024-02-29-15-12-31 +.. gh-issue: 116117 +.. nonce: eENkQK +.. section: Build + +Backport ``libb2``'s PR #42 to fix compiling CPython on 32-bit Windows with +``clang-cl``. + +.. + +.. date: 2024-02-26-14-54-58 +.. gh-issue: 71052 +.. nonce: XvFay1 +.. section: Build + +Fix several Android build issues + +.. + +.. date: 2024-02-26-13-13-53 +.. gh-issue: 114099 +.. nonce: 8lpX-7 +.. section: Build + +A testbed project was added to run the test suite on iOS. + +.. + +.. date: 2024-02-24-12-50-43 +.. gh-issue: 115350 +.. nonce: naQA6y +.. section: Build + +Fix building ctypes module with -DWIN32_LEAN_AND_MEAN defined + +.. + +.. date: 2024-02-21-18-22-49 +.. gh-issue: 111225 +.. nonce: Z8C3av +.. section: Build + +Link extension modules against libpython on Android. + +.. + +.. date: 2024-02-21-11-58-30 +.. gh-issue: 115737 +.. nonce: dpNl2T +.. section: Build + +The install name for libPython is now correctly set for non-framework macOS +builds. + +.. + +.. date: 2024-02-13-14-52-59 +.. gh-issue: 114099 +.. nonce: zjXsQr +.. section: Build + +Makefile targets were added to support compiling an iOS-compatible framework +build. + +.. + +.. date: 2024-02-27-23-21-55 +.. gh-issue: 116012 +.. nonce: B9_IwM +.. section: Windows + +Ensure the value of ``GetLastError()`` is preserved across GIL operations. + +.. + +.. date: 2024-02-23-11-43-43 +.. gh-issue: 115582 +.. nonce: sk1XPi +.. section: Windows + +Building extensions intended for free-threaded builds of CPython now require +compiling with ``/DPy_GIL_DISABLED`` manually when using a regular install. +This is expected to change in future releases. + +.. + +.. date: 2024-02-21-23-48-59 +.. gh-issue: 115554 +.. nonce: 02mpQC +.. section: Windows + +The installer now has more strict rules about updating the :ref:`launcher`. +In general, most users only have a single launcher installed and will see no +difference. When multiple launchers have been installed, the option to +install the launcher is disabled until all but one have been removed. +Downgrading the launcher (which was never allowed) is now more obviously +blocked. + +.. + +.. date: 2024-02-15-23-16-31 +.. gh-issue: 115543 +.. nonce: otrWnw +.. section: Windows + +:ref:`launcher` can now detect Python 3.13 when installed from the Microsoft +Store, and will install Python 3.12 by default when +:envvar:`PYLAUNCHER_ALLOW_INSTALL` is set. + +.. + +.. date: 2024-02-29-20-52-23 +.. gh-issue: 116145 +.. nonce: ygafim +.. section: macOS + +Update macOS installer to Tcl/Tk 8.6.14. + +.. + +.. date: 2023-12-09-11-04-26 +.. gh-issue: 88516 +.. nonce: SIIvfs +.. section: IDLE + +On macOS show a proxy icon in the title bar of editor windows to match +platform behaviour. + +.. + +.. date: 2023-02-12-19-28-08 +.. gh-issue: 100176 +.. nonce: Kzs4Zw +.. section: Tools/Demos + +Remove outdated Tools/{io,cc,string}bench + +.. + +.. bpo: 45101 +.. date: 2021-09-05-02-47-48 +.. nonce: 60Zqmt +.. section: Tools/Demos + +Add consistency in usage message IO between 2 versions of python-config. + +.. + +.. date: 2024-02-16-15-56-53 +.. gh-issue: 114626 +.. nonce: ie2esA +.. section: C API + +Add again ``_PyCFunctionFastWithKeywords`` name, removed in Python 3.13 +alpha 4 by mistake. Keep the old private ``_PyCFunctionFastWithKeywords`` +name (Python 3.7) as an alias to the new public name +``PyCFunctionFastWithKeywords`` (Python 3.13a4). Patch by Victor Stinner. + +.. + +.. date: 2023-11-15-09-24-51 +.. gh-issue: 111418 +.. nonce: FYYetY +.. section: C API + +Add :c:macro:`PyHASH_MODULUS`, :c:macro:`PyHASH_BITS`, :c:macro:`PyHASH_INF` +and :c:macro:`PyHASH_IMAG` C macros. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/3.13.0a6.rst b/Misc/NEWS.d/3.13.0a6.rst new file mode 100644 index 00000000000000..2740b4f0d967ba --- /dev/null +++ b/Misc/NEWS.d/3.13.0a6.rst @@ -0,0 +1,1270 @@ +.. date: 2024-04-04-13-42-59 +.. gh-issue: 117494 +.. nonce: GPQH64 +.. release date: 2024-04-09 +.. section: Core and Builtins + +Refactored the instruction sequence data structure out of compile.c into +instruction_sequence.c. + +.. + +.. date: 2024-04-03-13-44-04 +.. gh-issue: 116968 +.. nonce: zgcdG2 +.. section: Core and Builtins + +Introduce a unified 16-bit backoff counter type (``_Py_BackoffCounter``), +shared between the Tier 1 adaptive specializer and the Tier 2 optimizer. The +API used for adaptive specialization counters is changed but the behavior is +(supposed to be) identical. + +The behavior of the Tier 2 counters is changed: + +* There are no longer dynamic thresholds (we never varied these). +* All counters now use the same exponential backoff. +* The counter for ``JUMP_BACKWARD`` starts counting down from 16. +* The ``temperature`` in side exits starts counting down from 64. + +.. + +.. date: 2024-04-03-09-49-15 +.. gh-issue: 117431 +.. nonce: WAqRgc +.. section: Core and Builtins + +Improve the performance of the following :class:`bytes` and +:class:`bytearray` methods by adapting them to the :c:macro:`METH_FASTCALL` +calling convention: + +* :meth:`!endswith` +* :meth:`!startswith` + +.. + +.. date: 2024-04-02-17-37-35 +.. gh-issue: 117431 +.. nonce: vDKAOn +.. section: Core and Builtins + +Improve the performance of the following :class:`str` methods by adapting +them to the :c:macro:`METH_FASTCALL` calling convention: + +* :meth:`~str.count` +* :meth:`~str.endswith` +* :meth:`~str.find` +* :meth:`~str.index` +* :meth:`~str.rfind` +* :meth:`~str.rindex` +* :meth:`~str.startswith` + +.. + +.. date: 2024-04-02-10-04-57 +.. gh-issue: 117411 +.. nonce: YdyVmG +.. section: Core and Builtins + +Move ``PyFutureFeatures`` to an internal header and make it private. + +.. + +.. date: 2024-04-02-06-16-49 +.. gh-issue: 109120 +.. nonce: X485oN +.. section: Core and Builtins + +Added handle of incorrect star expressions, e.g ``f(3, *)``. Patch by +Grigoryev Semyon + +.. + +.. date: 2024-03-26-17-22-38 +.. gh-issue: 117266 +.. nonce: Kwh79O +.. section: Core and Builtins + +Fix crashes for certain user-created subclasses of :class:`ast.AST`. Such +classes are now expected to set the ``_field_types`` attribute. + +.. + +.. date: 2024-03-25-17-04-54 +.. gh-issue: 99108 +.. nonce: 8bjdO6 +.. section: Core and Builtins + +Updated the :mod:`hashlib` built-in `HACL\* project`_ C code from upstream +that we use for many implementations when they are not present via OpenSSL +in a given build. This also avoids the rare potential for a C symbol name +one definition rule linking issue. + +.. _HACL\* project: https://github.com/hacl-star/hacl-star + +.. + +.. date: 2024-03-25-12-51-12 +.. gh-issue: 117108 +.. nonce: tNqDEo +.. section: Core and Builtins + +Change the old space bit of objects in the young generation from 0 to +gcstate->visited, so that any objects created during GC will have the old +bit set correctly if they get moved into the old generation. + +.. + +.. date: 2024-03-21-12-10-11 +.. gh-issue: 117108 +.. nonce: _6jIrB +.. section: Core and Builtins + +The cycle GC now chooses the size of increments based on the total heap +size, instead of the rate of object creation. This ensures that it can keep +up with growing heaps. + +.. + +.. date: 2024-03-13-16-55-25 +.. gh-issue: 116735 +.. nonce: o3w6y8 +.. section: Core and Builtins + +For ``INSTRUMENTED_CALL_FUNCTION_EX``, set ``arg0`` to +``sys.monitoring.MISSING`` instead of ``None`` for :monitoring-event:`CALL` +event. + +.. + +.. date: 2024-03-12-20-31-57 +.. gh-issue: 113964 +.. nonce: bJppzg +.. section: Core and Builtins + +Starting new threads and process creation through :func:`os.fork` are now +only prevented once all non-daemon threads exit. + +.. + +.. date: 2024-03-11-22-05-56 +.. gh-issue: 116626 +.. nonce: GsyczB +.. section: Core and Builtins + +Ensure ``INSTRUMENTED_CALL_FUNCTION_EX`` always emits +:monitoring-event:`CALL` + +.. + +.. date: 2024-03-11-00-45-39 +.. gh-issue: 116554 +.. nonce: gYumG5 +.. section: Core and Builtins + +``list.sort()`` now exploits more cases of partial ordering, particularly +those with long descending runs with sub-runs of equal values. Those are +recognized as single runs now (previously, each block of repeated values +caused a new run to be created). + +.. + +.. date: 2024-03-07-16-12-39 +.. gh-issue: 114099 +.. nonce: ujdjn2 +.. section: Core and Builtins + +Added a Loader that can discover extension modules in an iOS-style +Frameworks folder. + +.. + +.. date: 2024-02-25-14-17-25 +.. gh-issue: 115775 +.. nonce: CNbGbJ +.. section: Core and Builtins + +Compiler populates the new ``__static_attributes__`` field on a class with +the names of attributes of this class which are accessed through self.X from +any function in its body. + +.. + +.. date: 2024-02-24-03-39-09 +.. gh-issue: 115776 +.. nonce: THJXqg +.. section: Core and Builtins + +The array of values, the ``PyDictValues`` struct is now embedded in the +object during allocation. This provides better performance in the common +case, and does not degrade as much when the object's ``__dict__`` is +materialized. + +.. + +.. date: 2024-01-07-04-22-51 +.. gh-issue: 108362 +.. nonce: oB9Gcf +.. section: Core and Builtins + +Implement an incremental cyclic garbage collector. By collecting the old +generation in increments, there is no need for a full heap scan. This can +hugely reduce maximum pause time for programs with large heaps. + +Reduce the number of generations from three to two. The old generation is +split into two spaces, "visited" and "pending". + +Collection happens in two steps:: * An increment is formed from the young +generation and a small part of the pending space. * This increment is +scanned and the survivors moved to the end of the visited space. + +When the collecting space becomes empty, the two spaces are swapped. + +.. + +.. date: 2023-10-14-00-05-17 +.. gh-issue: 109870 +.. nonce: oKpJ3P +.. section: Core and Builtins + +Dataclasses now calls :func:`exec` once per dataclass, instead of once per +method being added. This can speed up dataclass creation by up to 20%. + +.. + +.. date: 2022-10-05-09-33-48 +.. gh-issue: 97901 +.. nonce: BOLluU +.. section: Core and Builtins + +Mime type ``text/rtf`` is now supported by :mod:`mimetypes`. + +.. + +.. bpo: 24612 +.. date: 2021-09-04-22-33-01 +.. nonce: SsTuUX +.. section: Core and Builtins + +Improve the :exc:`SyntaxError` that happens when 'not' appears after an +operator. Patch by Pablo Galindo + +.. + +.. date: 2024-04-08-20-26-15 +.. gh-issue: 117648 +.. nonce: NzVEa7 +.. section: Library + +Improve performance of :func:`os.path.join` and :func:`os.path.expanduser`. + +.. + +.. date: 2024-04-06-16-42-34 +.. gh-issue: 117584 +.. nonce: hqk9Hn +.. section: Library + +Raise :exc:`TypeError` for non-paths in :func:`posixpath.relpath`. + +.. + +.. date: 2024-04-03-18-36-53 +.. gh-issue: 117467 +.. nonce: l6rWlj +.. section: Library + +Preserve mailbox ownership when rewriting in :func:`mailbox.mbox.flush`. +Patch by Tony Mountifield. + +.. + +.. date: 2024-04-02-20-30-12 +.. gh-issue: 114848 +.. nonce: YX4pEc +.. section: Library + +Raise :exc:`FileNotFoundError` when ``getcwd()`` returns '(unreachable)', +which can happen on Linux >= 2.6.36 with glibc < 2.27. + +.. + +.. date: 2024-04-02-13-13-46 +.. gh-issue: 117459 +.. nonce: jiIZmH +.. section: Library + +:meth:`asyncio.asyncio.run_coroutine_threadsafe` now keeps the traceback of +:class:`CancelledError`, :class:`TimeoutError` and +:class:`InvalidStateError` which are raised in the coroutine. + +.. + +.. date: 2024-03-29-21-43-19 +.. gh-issue: 117381 +.. nonce: fT0JFM +.. section: Library + +Fix error message for :func:`ntpath.commonpath`. + +.. + +.. date: 2024-03-29-15-58-01 +.. gh-issue: 117337 +.. nonce: 7w3Qwp +.. section: Library + +Deprecate undocumented :func:`!glob.glob0` and :func:`!glob.glob1` +functions. Use :func:`glob.glob` and pass a directory to its *root_dir* +argument instead. + +.. + +.. date: 2024-03-29-15-04-13 +.. gh-issue: 117349 +.. nonce: OB9kQQ +.. section: Library + +Optimise several functions in :mod:`os.path`. + +.. + +.. date: 2024-03-29-12-07-26 +.. gh-issue: 117348 +.. nonce: WjCYvK +.. section: Library + +Refactored :meth:`configparser.RawConfigParser._read` to reduce cyclometric +complexity and improve comprehensibility. + +.. + +.. date: 2024-03-28-19-13-20 +.. gh-issue: 117335 +.. nonce: d6uKJu +.. section: Library + +Raise TypeError for non-sequences for :func:`ntpath.commonpath`. + +.. + +.. date: 2024-03-28-17-55-22 +.. gh-issue: 66449 +.. nonce: 4jhuEV +.. section: Library + +:class:`configparser.ConfigParser` now accepts unnamed sections before named +ones, if configured to do so. + +.. + +.. date: 2024-03-28-13-54-20 +.. gh-issue: 88014 +.. nonce: zJz31I +.. section: Library + +In documentation of :class:`gzip.GzipFile` in module gzip, explain data type +of optional constructor argument *mtime*, and recommend ``mtime = 0`` for +generating deterministic streams. + +.. + +.. date: 2024-03-27-21-05-52 +.. gh-issue: 117310 +.. nonce: Bt2wox +.. section: Library + +Fixed an unlikely early & extra ``Py_DECREF`` triggered crash in :mod:`ssl` +when creating a new ``_ssl._SSLContext`` if CPython was built implausibly +such that the default cipher list is empty **or** the SSL library it was +linked against reports a failure from its C ``SSL_CTX_set_cipher_list()`` +API. + +.. + +.. date: 2024-03-27-16-43-42 +.. gh-issue: 117294 +.. nonce: wbXNFv +.. section: Library + +A ``DocTestCase`` now reports as skipped if all examples in the doctest are +skipped. + +.. + +.. date: 2024-03-26-11-48-39 +.. gh-issue: 98966 +.. nonce: SayV9y +.. section: Library + +In :mod:`subprocess`, raise a more informative message when +``stdout=STDOUT``. + +.. + +.. date: 2024-03-25-21-15-56 +.. gh-issue: 117225 +.. nonce: oOaZXb +.. section: Library + +doctest: only print "and X failed" when non-zero, don't pluralise "1 items". +Patch by Hugo van Kemenade. + +.. + +.. date: 2024-03-25-00-20-16 +.. gh-issue: 117205 +.. nonce: yV7xGb +.. section: Library + +Speed up :func:`compileall.compile_dir` by 20% when using multiprocessing by +increasing ``chunksize``. + +.. + +.. date: 2024-03-23-14-26-18 +.. gh-issue: 117178 +.. nonce: vTisTG +.. section: Library + +Fix regression in lazy loading of self-referential modules, introduced in +gh-114781. + +.. + +.. date: 2024-03-23-13-40-13 +.. gh-issue: 112383 +.. nonce: XuHf3G +.. section: Library + +Fix :mod:`dis` module's handling of ``ENTER_EXECUTOR`` instructions. + +.. + +.. date: 2024-03-23-12-28-05 +.. gh-issue: 117182 +.. nonce: a0KANW +.. section: Library + +Lazy-loading of modules that modify their own ``__class__`` no longer +reverts the ``__class__`` to :class:`types.ModuleType`. + +.. + +.. date: 2024-03-21-17-07-38 +.. gh-issue: 117084 +.. nonce: w1mTpT +.. section: Library + +Fix :mod:`zipfile` extraction for directory entries with the name containing +backslashes on Windows. + +.. + +.. date: 2024-03-21-09-57-57 +.. gh-issue: 117114 +.. nonce: Qu-p55 +.. section: Library + +Make :func:`os.path.isdevdrive` available on all platforms. For those that +do not offer Dev Drives, it will always return ``False``. + +.. + +.. date: 2024-03-21-07-27-36 +.. gh-issue: 117110 +.. nonce: 9K1InX +.. section: Library + +Fix a bug that prevents subclasses of :class:`typing.Any` to be instantiated +with arguments. Patch by Chris Fu. + +.. + +.. date: 2024-03-20-23-07-58 +.. gh-issue: 109653 +.. nonce: uu3lrX +.. section: Library + +Deferred select imports in importlib.metadata and importlib.resources for a +14% speedup. + +.. + +.. date: 2024-03-20-16-10-29 +.. gh-issue: 70647 +.. nonce: FpD6Ar +.. section: Library + +Start the deprecation period for the current behavior of +:func:`datetime.datetime.strptime` and :func:`time.strptime` which always +fails to parse a date string with a :exc:`ValueError` involving a day of +month such as ``strptime("02-29", "%m-%d")`` when a year is **not** +specified and the date happen to be February 29th. This should help avoid +users finding new bugs every four years due to a natural mistaken assumption +about the API when parsing partial date values. + +.. + +.. date: 2024-03-19-19-42-25 +.. gh-issue: 116987 +.. nonce: ZVKUH1 +.. section: Library + +Fixed :func:`inspect.findsource` for class code objects. + +.. + +.. date: 2024-03-19-14-35-57 +.. gh-issue: 114099 +.. nonce: siNSpK +.. section: Library + +Modify standard library to allow for iOS platform differences. + +.. + +.. date: 2024-03-19-11-08-26 +.. gh-issue: 90872 +.. nonce: ghys95 +.. section: Library + +On Windows, :meth:`subprocess.Popen.wait` no longer calls +``WaitForSingleObject()`` with a negative timeout: pass ``0`` ms if the +timeout is negative. Patch by Victor Stinner. + +.. + +.. date: 2024-03-18-14-36-50 +.. gh-issue: 116957 +.. nonce: dTCs4f +.. section: Library + +configparser: Don't leave ConfigParser values in an invalid state (stored as +a list instead of a str) after an earlier read raised DuplicateSectionError +or DuplicateOptionError. + +.. + +.. date: 2024-03-17-18-12-39 +.. gh-issue: 115538 +.. nonce: PBiRQB +.. section: Library + +:class:`!_io.WindowsConsoleIO` now emit a warning if a boolean value is +passed as a filedescriptor argument. + +.. + +.. date: 2024-03-14-20-59-28 +.. gh-issue: 90095 +.. nonce: 7UaJ1U +.. section: Library + +Ignore empty lines and comments in ``.pdbrc`` + +.. + +.. date: 2024-03-14-17-24-59 +.. gh-issue: 106531 +.. nonce: 9ehywi +.. section: Library + +Refreshed zipfile._path from `zipp 3.18 +`_, providing +better compatibility for PyPy, better glob performance for deeply nested +zipfiles, and providing internal access to ``CompleteDirs.inject`` for use +in other tests (like importlib.resources). + +.. + +.. date: 2024-03-14-17-21-25 +.. gh-issue: 63207 +.. nonce: LV16SL +.. section: Library + +On Windows, :func:`time.time` now uses the +``GetSystemTimePreciseAsFileTime()`` clock to have a resolution better than +1 us, instead of the ``GetSystemTimeAsFileTime()`` clock which has a +resolution of 15.6 ms. Patch by Victor Stinner. + +.. + +.. date: 2024-03-14-14-01-46 +.. gh-issue: 116764 +.. nonce: moB3Lc +.. section: Library + +Restore support of ``None`` and other false values in :mod:`urllib.parse` +functions :func:`~urllib.parse.parse_qs` and +:func:`~urllib.parse.parse_qsl`. Also, they now raise a TypeError for +non-zero integers and non-empty sequences. + +.. + +.. date: 2024-03-14-10-01-23 +.. gh-issue: 116811 +.. nonce: _h5iKP +.. section: Library + +In ``PathFinder.invalidate_caches``, delegate to +``MetadataPathFinder.invalidate_caches``. + +.. + +.. date: 2024-03-14-09-38-51 +.. gh-issue: 116647 +.. nonce: h0d_zj +.. section: Library + +Fix recursive child in dataclasses + +.. + +.. date: 2024-03-14-01-38-44 +.. gh-issue: 113171 +.. nonce: VFnObz +.. section: Library + +Fixed various false positives and false negatives in + +* :attr:`ipaddress.IPv4Address.is_private` (see these docs for details) +* :attr:`ipaddress.IPv4Address.is_global` +* :attr:`ipaddress.IPv6Address.is_private` +* :attr:`ipaddress.IPv6Address.is_global` + +Also in the corresponding :class:`ipaddress.IPv4Network` and +:class:`ipaddress.IPv6Network` attributes. + +.. + +.. date: 2024-03-13-15-45-54 +.. gh-issue: 63283 +.. nonce: OToJnG +.. section: Library + +In :mod:`encodings.idna`, any capitalization of the ACE prefix +(``xn--``) is now acceptable. Patch by Pepijn de Vos and Zackery Spytz. + +.. + +.. date: 2024-03-12-19-32-17 +.. gh-issue: 71042 +.. nonce: oI0Ron +.. section: Library + +Add :func:`platform.android_ver`, which provides device and OS information +on Android. + +.. + +.. date: 2024-03-12-17-53-14 +.. gh-issue: 73468 +.. nonce: z4ZzvJ +.. section: Library + +Added new :func:`math.fma` function, wrapping C99's ``fma()`` operation: +fused multiply-add function. Patch by Mark Dickinson and Victor Stinner. + +.. + +.. date: 2024-03-11-17-04-55 +.. gh-issue: 116608 +.. nonce: 30f58- +.. section: Library + +The :mod:`importlib.resources` functions +:func:`~importlib.resources.is_resource`, +:func:`~importlib.resources.open_binary`, +:func:`~importlib.resources.open_text`, +:func:`~importlib.resources.path`, +:func:`~importlib.resources.read_binary`, and +:func:`~importlib.resources.read_text` are un-deprecated, and support +subdirectories via multiple positional arguments. The +:func:`~importlib.resources.contents` function also allows subdirectories, +but remains deprecated. + +.. + +.. date: 2024-03-08-11-31-49 +.. gh-issue: 116484 +.. nonce: VMAsU7 +.. section: Library + +Change automatically generated :class:`tkinter.Checkbutton` widget names to +avoid collisions with automatically generated +:class:`tkinter.ttk.Checkbutton` widget names within the same parent widget. + +.. + +.. date: 2024-03-07-11-10-27 +.. gh-issue: 114314 +.. nonce: iEhAMH +.. section: Library + +In :mod:`ctypes`, ctype data is now stored in type objects directly rather +than in a dict subclass. This is an internal change that should not affect +usage. + +.. + +.. date: 2024-03-06-18-30-37 +.. gh-issue: 116401 +.. nonce: 3Wcda2 +.. section: Library + +Fix blocking :func:`os.fwalk` and :func:`shutil.rmtree` on opening named +pipe. + +.. + +.. date: 2024-03-05-19-56-29 +.. gh-issue: 71052 +.. nonce: PMDK-- +.. section: Library + +Implement :func:`ctypes.util.find_library` on Android. + +.. + +.. date: 2024-03-01-20-23-57 +.. gh-issue: 90535 +.. nonce: wXm-jC +.. section: Library + +Fix support of *interval* values > 1 in +:class:`logging.TimedRotatingFileHandler` for ``when='MIDNIGHT'`` and +``when='Wx'``. + +.. + +.. date: 2024-02-26-10-06-50 +.. gh-issue: 113308 +.. nonce: MbvOFt +.. section: Library + +Remove some internal protected parts from :mod:`uuid`: +``_has_uuid_generate_time_safe``, ``_netbios_getnode``, +``_ipconfig_getnode``, and ``_load_system_functions``. They were unused. + +.. + +.. date: 2024-02-18-09-50-31 +.. gh-issue: 115627 +.. nonce: HGchj0 +.. section: Library + +Fix the :mod:`ssl` module error handling of connection terminate by peer. It +now throws an OSError with the appropriate error code instead of an +EOFError. + +.. + +.. date: 2024-02-01-08-09-20 +.. gh-issue: 114847 +.. nonce: -JrWrR +.. section: Library + +Speed up :func:`os.path.realpath` on non-Windows platforms. + +.. + +.. date: 2024-02-01-03-09-38 +.. gh-issue: 114271 +.. nonce: raCkt5 +.. section: Library + +Fix a race in ``threading.Thread.join()``. + +``threading._MainThread`` now always represents the main thread of the main +interpreter. + +``PyThreadState.on_delete`` and ``PyThreadState.on_delete_data`` have been +removed. + +.. + +.. date: 2024-01-22-15-50-58 +.. gh-issue: 113538 +.. nonce: v2wrwg +.. section: Library + +Add :meth:`asyncio.Server.close_clients` and +:meth:`asyncio.Server.abort_clients` methods which allow to more forcefully +close an asyncio server. + +.. + +.. date: 2024-01-02-22-47-12 +.. gh-issue: 85287 +.. nonce: ZC5DLj +.. section: Library + +Changes Unicode codecs to return UnicodeEncodeError or UnicodeDecodeError, +rather than just UnicodeError. + +.. + +.. date: 2023-12-28-22-52-45 +.. gh-issue: 113548 +.. nonce: j6TJ7O +.. section: Library + +:mod:`pdb` now allows CLI arguments to ``pdb -m``. + +.. + +.. date: 2023-12-11-00-51-51 +.. gh-issue: 112948 +.. nonce: k-OKp5 +.. section: Library + +Make completion of :mod:`pdb` similar to Python REPL + +.. + +.. date: 2023-06-16-19-17-06 +.. gh-issue: 105866 +.. nonce: 0NBveV +.. section: Library + +Fixed ``_get_slots`` bug which caused error when defining dataclasses with +slots and a weakref_slot. + +.. + +.. date: 2023-05-06-05-00-42 +.. gh-issue: 96471 +.. nonce: S3X5I- +.. section: Library + +Add :py:class:`asyncio.Queue` termination with +:py:meth:`~asyncio.Queue.shutdown` method. + +.. + +.. date: 2022-06-22-14-45-32 +.. gh-issue: 89739 +.. nonce: CqZcRL +.. section: Library + +The :mod:`zipimport` module can now read ZIP64 files. + +.. + +.. bpo: 33533 +.. date: 2020-10-02-17-35-19 +.. nonce: GLIhM5 +.. section: Library + +:func:`asyncio.as_completed` now returns an object that is both an +asynchronous iterator and plain iterator. The new asynchronous iteration +pattern allows for easier correlation between prior tasks and their +completed results. This is a closer match to +:func:`concurrent.futures.as_completed`'s iteration pattern. Patch by Justin +Arthur. + +.. + +.. bpo: 27578 +.. date: 2020-06-11-16-20-33 +.. nonce: CIA-fu +.. section: Library + +:func:`inspect.getsource` (and related functions) work with empty module +files, returning ``'\n'`` (or reasonable equivalent) instead of raising +``OSError``. Patch by Kernc. + +.. + +.. bpo: 37141 +.. date: 2019-09-26-17-52-52 +.. nonce: onYY2- +.. section: Library + +Accept an iterable of separators in :meth:`asyncio.StreamReader.readuntil`, +stopping when one of them is encountered. + +.. + +.. date: 2019-08-27-01-03-26 +.. gh-issue: 66543 +.. nonce: _TRpYr +.. section: Library + +Make :func:`mimetypes.guess_type` properly parsing of URLs with only a host +name, URLs containing fragment or query, and filenames with only a UNC +sharepoint on Windows. Based on patch by Dong-hee Na. + +.. + +.. bpo: 15010 +.. date: 2019-08-12-19-08-06 +.. nonce: 3bY2CF +.. section: Library + +:meth:`unittest.TestLoader.discover` now saves the original value of +``unittest.TestLoader._top_level_dir`` and restores it at the end of the +call. + +.. + +.. date: 2024-03-20-15-12-37 +.. gh-issue: 115977 +.. nonce: IMLi6K +.. section: Documentation + +Remove compatibility references to Emscripten. + +.. + +.. date: 2024-03-20-12-41-47 +.. gh-issue: 114099 +.. nonce: ad_Ck9 +.. section: Documentation + +Add an iOS platform guide, and flag modules not available on iOS. + +.. + +.. date: 2022-04-15-13-15-23 +.. gh-issue: 91565 +.. nonce: OznXwC +.. section: Documentation + +Changes to documentation files and config outputs to reflect the new +location for reporting bugs - i.e. GitHub rather than bugs.python.org. + +.. + +.. date: 2024-03-25-21-31-49 +.. gh-issue: 83434 +.. nonce: U7Z8cY +.. section: Tests + +Disable JUnit XML output (``--junit-xml=FILE`` command line option) in +regrtest when hunting for reference leaks (``-R`` option). Patch by Victor +Stinner. + +.. + +.. date: 2024-03-24-23-49-25 +.. gh-issue: 117187 +.. nonce: eMLT5n +.. section: Tests + +Fix XML tests for vanilla Expat <2.6.0. + +.. + +.. date: 2024-03-21-11-32-29 +.. gh-issue: 116333 +.. nonce: F-9Ram +.. section: Tests + +Tests of TLS related things (error codes, etc) were updated to be more +lenient about specific error message strings and behaviors as seen in the +BoringSSL and AWS-LC forks of OpenSSL. + +.. + +.. date: 2024-03-20-14-19-32 +.. gh-issue: 117089 +.. nonce: WwR1Z1 +.. section: Tests + +Consolidated tests for importlib.metadata in their own ``metadata`` package. + +.. + +.. date: 2024-03-13-12-06-49 +.. gh-issue: 115979 +.. nonce: zsNpQD +.. section: Tests + +Update test_importlib so that it passes under WASI SDK 21. + +.. + +.. date: 2024-03-11-23-20-28 +.. gh-issue: 112536 +.. nonce: Qv1RrX +.. section: Tests + +Add --tsan to test.regrtest for running TSAN tests in reasonable execution +times. Patch by Donghee Na. + +.. + +.. date: 2024-03-06-11-00-36 +.. gh-issue: 116307 +.. nonce: Uij0t_ +.. section: Tests + +Added import helper ``isolated_modules`` as ``CleanImport`` does not remove +modules imported during the context. Use it in importlib.resources tests to +avoid leaving ``mod`` around to impede importlib.metadata tests. + +.. + +.. date: 2024-03-13-16-16-43 +.. gh-issue: 114736 +.. nonce: ZhmauG +.. section: Build + +Have WASI builds use WASI SDK 21. + +.. + +.. date: 2024-03-08-17-05-15 +.. gh-issue: 115983 +.. nonce: ZQqk0Q +.. section: Build + +Skip building test modules that must be built as shared under WASI. + +.. + +.. date: 2024-03-06-17-26-55 +.. gh-issue: 71052 +.. nonce: vLbu9u +.. section: Build + +Add Android build script and instructions. + +.. + +.. date: 2024-03-28-22-12-00 +.. gh-issue: 117267 +.. nonce: K_tki1 +.. section: Windows + +Ensure ``DirEntry.stat().st_ctime`` behaves consistently with +:func:`os.stat` during the deprecation period of ``st_ctime`` by containing +the same value as ``st_birthtime``. After the deprecation period, +``st_ctime`` will be the metadata change time (or unavailable through +``DirEntry``), and only ``st_birthtime`` will contain the creation time. + +.. + +.. date: 2024-03-14-20-46-23 +.. gh-issue: 116195 +.. nonce: Cu_rYs +.. section: Windows + +Improves performance of :func:`os.getppid` by using an alternate system API +when available. Contributed by vxiiduu. + +.. + +.. date: 2024-03-14-09-14-21 +.. gh-issue: 88494 +.. nonce: Bwfmp7 +.. section: Windows + +On Windows, :func:`time.monotonic` now uses the +``QueryPerformanceCounter()`` clock to have a resolution better than 1 us, +instead of the ``GetTickCount64()`` clock which has a resolution of 15.6 ms. +Patch by Victor Stinner. + +.. + +.. date: 2024-03-14-01-58-22 +.. gh-issue: 116773 +.. nonce: H2UldY +.. section: Windows + +Fix instances of ``<_overlapped.Overlapped object at 0xXXX> still has +pending operation at deallocation, the process may crash``. + +.. + +.. date: 2024-02-24-23-03-43 +.. gh-issue: 91227 +.. nonce: sL4zWC +.. section: Windows + +Fix the asyncio ProactorEventLoop implementation so that sending a datagram +to an address that is not listening does not prevent receiving any more +datagrams. + +.. + +.. date: 2024-02-08-14-48-15 +.. gh-issue: 115119 +.. nonce: qMt32O +.. section: Windows + +Switched from vendored ``libmpdecimal`` code to a separately-hosted external +package in the ``cpython-source-deps`` repository when building the +``_decimal`` module. + +.. + +.. date: 2024-04-08-18-53-33 +.. gh-issue: 117642 +.. nonce: _-tYH_ +.. section: C API + +Fix :pep:`737` implementation for ``%#T`` and ``%#N``. + +.. + +.. date: 2024-03-22-19-29-24 +.. gh-issue: 87193 +.. nonce: u7O-jY +.. section: C API + +:c:func:`_PyBytes_Resize` can now be called for bytes objects with reference +count > 1, including 1-byte bytes objects. It creates a new bytes object and +destroys the old one if it has reference count > 1. + +.. + +.. date: 2024-03-20-13-13-22 +.. gh-issue: 117021 +.. nonce: 0Q5jBx +.. section: C API + +Fix integer overflow in :c:func:`PyLong_AsPid` on non-Windows 64-bit +platforms. + +.. + +.. date: 2024-03-19-09-49-04 +.. gh-issue: 115756 +.. nonce: 4Ls_Tl +.. section: C API + +:c:func:`!PyCode_GetFirstFree` is an ustable API now and has been renamed to +:c:func:`PyUnstable_Code_GetFirstFree`. (Contributed by Bogdan Romanyuk in +:gh:`115781`) + +.. + +.. date: 2024-03-18-10-58-47 +.. gh-issue: 116869 +.. nonce: lN0GBl +.. section: C API + +Add ``test_cext`` test: build a C extension to check if the Python C API +emits C compiler warnings. Patch by Victor Stinner. + +.. + +.. date: 2024-03-18-09-58-46 +.. gh-issue: 116869 +.. nonce: LFDVKM +.. section: C API + +Make the C API compatible with ``-Werror=declaration-after-statement`` +compiler flag again. Patch by Victor Stinner. + +.. + +.. date: 2024-03-17-22-42-21 +.. gh-issue: 116936 +.. nonce: tNrzfm +.. section: C API + +Add :c:func:`PyType_GetModuleByDef` to the limited C API. Patch by Victor +Stinner. + +.. + +.. date: 2024-03-16-12-21-00 +.. gh-issue: 116809 +.. nonce: JL786L +.. section: C API + +Restore removed private ``_PyErr_ChainExceptions1()`` function. Patch by +Victor Stinner. + +.. + +.. date: 2024-03-15-23-57-33 +.. gh-issue: 115754 +.. nonce: zLdv82 +.. section: C API + +In the limited C API version 3.13, getting ``Py_None``, ``Py_False``, +``Py_True``, ``Py_Ellipsis`` and ``Py_NotImplemented`` singletons is now +implemented as function calls at the stable ABI level to hide implementation +details. Getting these constants still return borrowed references. Patch by +Victor Stinner. + +.. + +.. date: 2024-03-15-23-55-24 +.. gh-issue: 115754 +.. nonce: xnzc__ +.. section: C API + +Add :c:func:`Py_GetConstant` and :c:func:`Py_GetConstantBorrowed` functions +to get constants. For example, ``Py_GetConstant(Py_CONSTANT_ZERO)`` returns +a :term:`strong reference` to the constant zero. Patch by Victor Stinner. + +.. + +.. date: 2024-03-14-22-30-07 +.. gh-issue: 111696 +.. nonce: 76UMKi +.. section: C API + +Add support for ``%T``, ``%T#``, ``%N`` and ``%N#`` formats to +:c:func:`PyUnicode_FromFormat`: format the fully qualified name of an object +type and of a type: call :c:func:`PyType_GetModuleName`. See :pep:`737` for +more information. Patch by Victor Stinner. + +.. + +.. date: 2024-03-14-18-00-32 +.. gh-issue: 111696 +.. nonce: L6oIPq +.. section: C API + +Add :c:func:`PyType_GetModuleName` function to get the type's module name. +Equivalent to getting the ``type.__module__`` attribute. Patch by Eric Snow +and Victor Stinner. + +.. + +.. date: 2024-03-14-15-17-11 +.. gh-issue: 111696 +.. nonce: YmnvAi +.. section: C API + +Add :c:func:`PyType_GetFullyQualifiedName` function to get the type's fully +qualified name. Equivalent to ``f"{type.__module__}.{type.__qualname__}"``, +or ``type.__qualname__`` if ``type.__module__`` is not a string or is equal +to ``"builtins"``. Patch by Victor Stinner. + +.. + +.. date: 2024-03-14-10-33-58 +.. gh-issue: 85283 +.. nonce: LOgmdU +.. section: C API + +The ``fcntl``, ``grp``, ``pwd``, ``termios``, ``_statistics`` and +``_testconsole`` C extensions are now built with the :ref:`limited C API +`. Patch by Victor Stinner. + +.. + +.. date: 2024-02-28-15-50-01 +.. gh-issue: 111140 +.. nonce: mpwcUg +.. section: C API + +Add additional flags to :c:func:`PyLong_AsNativeBytes` and +:c:func:`PyLong_FromNativeBytes` to allow the caller to determine how to +handle edge cases around values that fill the entire buffer. + +.. + +.. date: 2023-12-12-19-48-31 +.. gh-issue: 113024 +.. nonce: rXcQs7 +.. section: C API + +Add :c:func:`PyObject_GenericHash` function. diff --git a/Misc/NEWS.d/3.13.0b1.rst b/Misc/NEWS.d/3.13.0b1.rst new file mode 100644 index 00000000000000..97731276679ba6 --- /dev/null +++ b/Misc/NEWS.d/3.13.0b1.rst @@ -0,0 +1,1671 @@ +.. date: 2024-03-27-13-50-02 +.. gh-issue: 116741 +.. nonce: ZoGryG +.. release date: 2024-05-08 +.. section: Security + +Update bundled libexpat to 2.6.2 + +.. + +.. date: 2024-03-25-21-25-28 +.. gh-issue: 117233 +.. nonce: E4CyI_ +.. section: Security + +Detect BLAKE2, SHA3, Shake, & truncated SHA512 support in the OpenSSL-ish +libcrypto library at build time. This allows :mod:`hashlib` to be used with +libraries that do not to support every algorithm that upstream OpenSSL does. + +.. + +.. date: 2024-05-07-01-39-24 +.. gh-issue: 118414 +.. nonce: G5GG7l +.. section: Core and Builtins + +Add instrumented opcodes to YIELD_VALUE assertion for tracing cases. + +.. + +.. date: 2024-05-06-10-57-54 +.. gh-issue: 117953 +.. nonce: DqCzIs +.. section: Core and Builtins + +When a builtin or extension module is imported for the first time, while a +subinterpreter is active, the module's init function is now run by the main +interpreter first before import continues in the subinterpreter. +Consequently, single-phase init modules now fail in an isolated +subinterpreter without the init function running under that interpreter, +whereas before it would run under the subinterpreter *before* failing, +potentially leaving behind global state and callbacks and otherwise leaving +the module in an inconsistent state. + +.. + +.. date: 2024-05-05-12-04-02 +.. gh-issue: 117549 +.. nonce: kITawD +.. section: Core and Builtins + +Don't use designated initializer syntax in inline functions in internal +headers. They cause problems for C++ or MSVC users who aren't yet using the +latest C++ standard (C++20). While internal, pycore_backoff.h, is included +(indirectly, via pycore_code.h) by some key 3rd party software that does so +for speed. + +.. + +.. date: 2024-05-03-18-01-26 +.. gh-issue: 95382 +.. nonce: 73FSEv +.. section: Core and Builtins + +Improve performance of :func:`json.dumps` and :func:`json.dump` when using +the argument *indent*. Depending on the data the encoding using +:func:`json.dumps` with *indent* can be up to 2 to 3 times faster. + +.. + +.. date: 2024-05-03-17-49-37 +.. gh-issue: 116322 +.. nonce: Gy6M4j +.. section: Core and Builtins + +In ``--disable-gil`` builds, the GIL will be enabled while loading C +extension modules. If the module indicates that it supports running without +the GIL, the GIL will be disabled once loading is complete. Otherwise, the +GIL will remain enabled for the remainder of the interpreter's lifetime. +This behavior does not apply if the GIL has been explicitly enabled or +disabled with ``PYTHON_GIL`` or ``-Xgil``. + +.. + +.. date: 2024-05-02-21-19-35 +.. gh-issue: 118513 +.. nonce: qHODjb +.. section: Core and Builtins + +Fix incorrect :exc:`UnboundLocalError` when two comprehensions in the same +function both reference the same name, and in one comprehension the name is +bound while in the other it's an implicit global. + +.. + +.. date: 2024-05-02-20-32-42 +.. gh-issue: 118518 +.. nonce: m-JbTi +.. section: Core and Builtins + +Allow the Linux perf support to work without frame pointers using perf's +advanced JIT support. The feature is activated when using the +``PYTHON_PERF_JIT_SUPPORT`` environment variable or when running Python with +``-Xperf_jit``. Patch by Pablo Galindo. + +.. + +.. date: 2024-05-02-16-04-51 +.. gh-issue: 117514 +.. nonce: CJiuC0 +.. section: Core and Builtins + +Add ``sys._is_gil_enabled()`` function that returns whether the GIL is +currently enabled. In the default build it always returns ``True`` because +the GIL is always enabled. In the free-threaded build, it may return +``True`` or ``False``. + +.. + +.. date: 2024-05-02-15-57-07 +.. gh-issue: 118164 +.. nonce: AF6kwI +.. section: Core and Builtins + +Break a loop between the Python implementation of the :mod:`decimal` module +and the Python code for integer to string conversion. Also optimize integer +to string conversion for values in the range from 9_000 to 135_000 decimal +digits. + +.. + +.. date: 2024-05-01-22-43-54 +.. gh-issue: 118473 +.. nonce: QIvq9R +.. section: Core and Builtins + +Fix :func:`sys.set_asyncgen_hooks` not to be partially set when raising +:exc:`TypeError`. + +.. + +.. date: 2024-05-01-17-12-36 +.. gh-issue: 118465 +.. nonce: g3Q8iE +.. section: Core and Builtins + +Compiler populates the new ``__firstlineno__`` field on a class with the +line number of the first line of the class definition. + +.. + +.. date: 2024-05-01-14-20-28 +.. gh-issue: 118492 +.. nonce: VUsSfn +.. section: Core and Builtins + +Fix an issue where the type cache can expose a previously accessed attribute +when a finalizer is run. + +.. + +.. date: 2024-05-01-07-06-48 +.. gh-issue: 117714 +.. nonce: Ip_dm5 +.. section: Core and Builtins + +update ``async_generator.athrow().close()`` and +``async_generator.asend().close()`` to close their section of the underlying +async generator + +.. + +.. date: 2024-04-28-00-41-17 +.. gh-issue: 111201 +.. nonce: cQsh5U +.. section: Core and Builtins + +The :term:`interactive` interpreter is now implemented in Python, which +allows for a number of new features like colors, multiline input, history +viewing, and paste mode. Contributed by Pablo Galindo, Łukasz Langa and +Lysandros Nikolaou based on code from the PyPy project. + +.. + +.. date: 2024-04-27-21-44-40 +.. gh-issue: 74929 +.. nonce: C2nESp +.. section: Core and Builtins + +Implement PEP 667: converted :attr:`FrameType.f_locals ` and +:c:func:`PyFrame_GetLocals` to return a write-through proxy object when the +frame refers to a function or comprehension. + +.. + +.. date: 2024-04-27-16-23-29 +.. gh-issue: 116767 +.. nonce: z9UFpr +.. section: Core and Builtins + +Fix crash in compiler on 'async with' that has many context managers. + +.. + +.. date: 2024-04-26-14-06-18 +.. gh-issue: 118335 +.. nonce: SRFsxO +.. section: Core and Builtins + +Change how to use the tier 2 interpreter. Instead of running Python with +``-X uops`` or setting the environment variable ``PYTHON_UOPS=1``, this +choice is now made at build time by configuring with +``--enable-experimental-jit=interpreter``. + +**Beware!** This changes the environment variable to enable or disable +micro-ops to ``PYTHON_JIT``. The old ``PYTHON_UOPS`` is no longer used. + +.. + +.. date: 2024-04-26-05-38-18 +.. gh-issue: 118306 +.. nonce: vRUEOU +.. section: Core and Builtins + +Update JIT compilation to use LLVM 18 + +.. + +.. date: 2024-04-25-21-18-19 +.. gh-issue: 118160 +.. nonce: GH5SMc +.. section: Core and Builtins + +:ref:`Annotation scopes ` within classes can now contain +comprehensions. However, such comprehensions are not inlined into their +parent scope at runtime. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-04-25-12-55-47 +.. gh-issue: 118272 +.. nonce: 5ptjk_ +.. section: Core and Builtins + +Fix bug where ``generator.close`` does not free the generator frame's +locals. + +.. + +.. date: 2024-04-25-11-48-28 +.. gh-issue: 118216 +.. nonce: SVg700 +.. section: Core and Builtins + +Don't consider :mod:`__future__` imports with dots before the module name. + +.. + +.. date: 2024-04-22-08-34-28 +.. gh-issue: 118074 +.. nonce: 5_JnIa +.. section: Core and Builtins + +Make sure that the Executor objects in the COLD_EXITS array aren't assumed +to be GC-able (which would access bytes outside the object). + +.. + +.. date: 2024-04-20-20-30-15 +.. gh-issue: 107674 +.. nonce: GZPOP7 +.. section: Core and Builtins + +Lazy load frame line number to improve performance of tracing + +.. + +.. date: 2024-04-19-11-59-57 +.. gh-issue: 118082 +.. nonce: _FLuOT +.. section: Core and Builtins + +Improve :exc:`SyntaxError` message for imports without names, like in ``from +x import`` and ``import`` cases. It now points out to users that +:keyword:`import` expects at least one name after it. + +.. + +.. date: 2024-04-19-11-57-46 +.. gh-issue: 118090 +.. nonce: eGAQ0B +.. section: Core and Builtins + +Improve :exc:`SyntaxError` message for empty type param brackets. + +.. + +.. date: 2024-04-18-03-49-41 +.. gh-issue: 117958 +.. nonce: -EsfUs +.. section: Core and Builtins + +Added a ``get_jit_code()`` method to access JIT compiled machine code from +the UOp Executor when the experimental JIT is enabled. Patch by Anthony +Shaw. + +.. + +.. date: 2024-04-17-22-53-52 +.. gh-issue: 117901 +.. nonce: SsEcVJ +.. section: Core and Builtins + +Add option for compiler's codegen to save nested instruction sequences for +introspection. + +.. + +.. date: 2024-04-17-22-49-15 +.. gh-issue: 116622 +.. nonce: tthNUF +.. section: Core and Builtins + +Redirect stdout and stderr to system log when embedded in an Android app. + +.. + +.. date: 2024-04-17-17-52-32 +.. gh-issue: 109118 +.. nonce: q9iPEI +.. section: Core and Builtins + +:ref:`annotation scope ` within class scopes can now +contain lambdas. + +.. + +.. date: 2024-04-15-13-53-59 +.. gh-issue: 117894 +.. nonce: 8LpZ6m +.. section: Core and Builtins + +Prevent ``agen.aclose()`` objects being re-used after ``.throw()``. + +.. + +.. date: 2024-04-15-07-37-09 +.. gh-issue: 117881 +.. nonce: 07H0wI +.. section: Core and Builtins + +prevent concurrent access to an async generator via athrow().throw() or +asend().throw() + +.. + +.. date: 2024-04-13-16-55-53 +.. gh-issue: 117536 +.. nonce: xkVbfv +.. section: Core and Builtins + +Fix a :exc:`RuntimeWarning` when calling ``agen.aclose().throw(Exception)``. + +.. + +.. date: 2024-04-12-12-28-49 +.. gh-issue: 117755 +.. nonce: 6ct8kU +.. section: Core and Builtins + +Fix mimalloc allocator for huge memory allocation (around 8,589,934,592 GiB) +on s390x. Patch by Victor Stinner. + +.. + +.. date: 2024-04-12-11-19-18 +.. gh-issue: 117750 +.. nonce: YttK6h +.. section: Core and Builtins + +Fix issue where an object's dict would get out of sync with the object's +internal values when being cleared. ``obj.__dict__.clear()`` now clears the +internal values, but leaves the dict attached to the object. + +.. + +.. date: 2024-04-12-09-09-11 +.. gh-issue: 117431 +.. nonce: lxFEeJ +.. section: Core and Builtins + +Improve the performance of the following :class:`bytes` and +:class:`bytearray` methods by adapting them to the :c:macro:`METH_FASTCALL` +calling convention: + +* :meth:`!count` +* :meth:`!find` +* :meth:`!index` +* :meth:`!rfind` +* :meth:`!rindex` + +.. + +.. date: 2024-04-10-22-16-18 +.. gh-issue: 117709 +.. nonce: -_1YL0 +.. section: Core and Builtins + +Speed up calls to :func:`str` with positional-only argument, by using the +:pep:`590` ``vectorcall`` calling convention. Patch by Erlend Aasland. + +.. + +.. date: 2024-04-09-16-07-00 +.. gh-issue: 117680 +.. nonce: MRZ78K +.. section: Core and Builtins + +Give ``_PyInstructionSequence`` a Python interface and use it in tests. + +.. + +.. date: 2024-04-09-11-31-25 +.. gh-issue: 115776 +.. nonce: 5Nthd0 +.. section: Core and Builtins + +Statically allocated objects are, by definition, immortal so must be marked +as such regardless of whether they are in extension modules or not. + +.. + +.. date: 2024-03-30-00-37-53 +.. gh-issue: 117385 +.. nonce: h0OJti +.. section: Core and Builtins + +Remove unhandled ``PY_MONITORING_EVENT_BRANCH`` and +``PY_MONITORING_EVENT_EXCEPTION_HANDLED`` events from :func:`sys.settrace`. + +.. + +.. date: 2024-03-12-13-51-09 +.. gh-issue: 116322 +.. nonce: q8TcDQ +.. section: Core and Builtins + +Extension modules may indicate to the runtime that they can run without the +GIL. Multi-phase init modules do so by calling providing +``Py_MOD_GIL_NOT_USED`` for the ``Py_mod_gil`` slot, while single-phase init +modules call ``PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED)`` from +their init function. + +.. + +.. date: 2024-02-29-18-55-45 +.. gh-issue: 116129 +.. nonce: wsFnIq +.. section: Core and Builtins + +Implement :pep:`696`, adding support for defaults on type parameters. Patch +by Jelle Zijlstra. + +.. + +.. date: 2024-02-26-13-14-52 +.. gh-issue: 93502 +.. nonce: JMWRvA +.. section: Core and Builtins + +Add two new functions to the C-API, :c:func:`PyRefTracer_SetTracer` and +:c:func:`PyRefTracer_GetTracer`, that allows to track object creation and +destruction the same way the :mod:`tracemalloc` module does. Patch by Pablo +Galindo + +.. + +.. date: 2024-02-04-07-45-29 +.. gh-issue: 107674 +.. nonce: q8mCmi +.. section: Core and Builtins + +Improved the performance of :func:`sys.settrace` significantly + +.. + +.. date: 2024-01-07-03-38-34 +.. gh-issue: 95754 +.. nonce: aPjEBG +.. section: Core and Builtins + +Improve the error message when a script shadowing a module from the standard +library causes :exc:`AttributeError` to be raised. Similarly, improve the +error message when a script shadowing a third party module attempts to +access an attribute from that third party module while still initialising. + +.. + +.. date: 2023-12-03-18-21-59 +.. gh-issue: 99180 +.. nonce: 5m0V0q +.. section: Core and Builtins + +Elide uninformative traceback indicators in ``return`` and simple +``assignment`` statements. Patch by Pablo Galindo. + +.. + +.. date: 2023-06-18-00-27-57 +.. gh-issue: 105879 +.. nonce: dPw78k +.. section: Core and Builtins + +Allow the *globals* and *locals* arguments to :func:`exec` and :func:`eval` +to be passed as keywords. + +.. + +.. date: 2024-05-07-11-23-11 +.. gh-issue: 118418 +.. nonce: QPMdJm +.. section: Library + +A :exc:`DeprecationWarning` is now emitted if you fail to pass a value to +the new *type_params* parameter of ``typing._eval_type()`` or +``typing.ForwardRef._evaluate()``. (Using either of these private and +undocumented functions is discouraged to begin with, but failing to pass a +value to the ``type_params`` parameter may lead to incorrect behaviour on +Python 3.12 or newer.) + +.. + +.. date: 2024-05-06-18-13-02 +.. gh-issue: 118660 +.. nonce: n01Vb7 +.. section: Library + +Add an optional second type parameter to :class:`typing.ContextManager` and +:class:`typing.AsyncContextManager`, representing the return types of +:meth:`~object.__exit__` and :meth:`~object.__aexit__` respectively. This +parameter defaults to ``bool | None``. + +.. + +.. date: 2024-05-06-16-52-40 +.. gh-issue: 118650 +.. nonce: qKz5lp +.. section: Library + +The ``enum`` module allows method named ``_repr_*`` to be defined on +``Enum`` types. + +.. + +.. date: 2024-05-06-08-23-01 +.. gh-issue: 118648 +.. nonce: OVA3jJ +.. section: Library + +Add type parameter defaults to :class:`typing.Generator` and +:class:`typing.AsyncGenerator`. + +.. + +.. date: 2024-05-05-16-08-03 +.. gh-issue: 101137 +.. nonce: 71ECXu +.. section: Library + +Mime type ``text/x-rst`` is now supported by :mod:`mimetypes`. + +.. + +.. date: 2024-05-04-20-22-59 +.. gh-issue: 118164 +.. nonce: 9D02MQ +.. section: Library + +The Python implementation of the ``decimal`` module could appear to hang in +relatively small power cases (like ``2**117``) if context precision was set +to a very high value. A different method to check for exactly representable +results is used now that doesn't rely on computing ``10**precision`` (which +could be effectively too large to compute). + +.. + +.. date: 2024-05-04-18-40-43 +.. gh-issue: 111744 +.. nonce: nuCtwN +.. section: Library + +``breakpoint()`` and ``pdb.set_trace()`` now enter the debugger immediately +after the call rather than before the next line is executed. + +.. + +.. date: 2024-05-02-04-27-12 +.. gh-issue: 118500 +.. nonce: pBGGtQ +.. section: Library + +Add :mod:`pdb` support for zipapps + +.. + +.. date: 2024-04-30-15-18-19 +.. gh-issue: 118406 +.. nonce: y-GnMo +.. section: Library + +Add signature for :class:`sqlite3.Connection` objects. + +.. + +.. date: 2024-04-30-12-59-04 +.. gh-issue: 101732 +.. nonce: 29zUDu +.. section: Library + +Use a Y2038 compatible openssl time function when available. + +.. + +.. date: 2024-04-29-22-11-54 +.. gh-issue: 118404 +.. nonce: GYfMaD +.. section: Library + +Fix :func:`inspect.signature` for non-comparable callables. + +.. + +.. date: 2024-04-29-21-51-28 +.. gh-issue: 118402 +.. nonce: Z_06Th +.. section: Library + +Fix :func:`inspect.signature` for the result of the +:func:`functools.cmp_to_key` call. + +.. + +.. date: 2024-04-27-20-34-56 +.. gh-issue: 116622 +.. nonce: YlQgXv +.. section: Library + +On Android, :any:`sysconfig.get_platform` now returns the format specified +by :pep:`738`. + +.. + +.. date: 2024-04-26-14-53-28 +.. gh-issue: 118285 +.. nonce: A0_pte +.. section: Library + +Allow to specify the signature of custom callable instances of extension +type by the ``__text_signature__`` attribute. Specify signatures of +:class:`operator.attrgetter`, :class:`operator.itemgetter`, and +:class:`operator.methodcaller` instances. + +.. + +.. date: 2024-04-26-12-42-29 +.. gh-issue: 118314 +.. nonce: Z7reGc +.. section: Library + +Fix an edge case in :func:`binascii.a2b_base64` strict mode, where excessive +padding is not detected when no padding is necessary. + +.. + +.. date: 2024-04-25-11-49-11 +.. gh-issue: 118271 +.. nonce: 5N2Xcy +.. section: Library + +Add the :class:`!PhotoImage` methods :meth:`!read` to +read an image from a file and :meth:`!data` to get the +image data. Add *background* and *grayscale* parameters to +:class:`!PhotoImage` method :meth:`!write`. + +.. + +.. date: 2024-04-24-16-07-26 +.. gh-issue: 118225 +.. nonce: KdrcgL +.. section: Library + +Add the :class:`!PhotoImage` method :meth:`!copy_replace` to copy a region +from one image to other image, possibly with pixel zooming and/or +subsampling. Add *from_coords* parameter to :class:`!PhotoImage` methods +:meth:`!copy`, :meth:`!zoom` and :meth:`!subsample`. Add *zoom* and +*subsample* parameters to :class:`!PhotoImage` method :meth:`!copy`. + +.. + +.. date: 2024-04-24-12-29-33 +.. gh-issue: 118221 +.. nonce: 2k_bac +.. section: Library + +Fix a bug where :meth:`sqlite3.Connection.iterdump` could fail if a custom +:attr:`row factory ` was used. Patch by +Erlend Aasland. + +.. + +.. date: 2024-04-24-12-20-48 +.. gh-issue: 118013 +.. nonce: TKn_kZ +.. section: Library + +Fix regression introduced in gh-103193 that meant that calling +:func:`inspect.getattr_static` on an instance would cause a strong reference +to that instance's class to persist in an internal cache in the +:mod:`inspect` module. This caused unexpected memory consumption if the +class was dynamically created, the class held strong references to other +objects which took up a significant amount of memory, and the cache +contained the sole strong reference to the class. The fix for the regression +leads to a slowdown in :func:`!getattr_static`, but the function should +still be significantly faster than it was in Python 3.11. Patch by Alex +Waygood. + +.. + +.. date: 2024-04-24-07-45-08 +.. gh-issue: 118218 +.. nonce: m1OHbN +.. section: Library + +Speed up :func:`itertools.pairwise` in the common case by up to 1.8x. + +.. + +.. date: 2024-04-23-21-17-00 +.. gh-issue: 117486 +.. nonce: ea3KYD +.. section: Library + +Improve the behavior of user-defined subclasses of :class:`ast.AST`. Such +classes will now require no changes in the usual case to conform with the +behavior changes of the :mod:`ast` module in Python 3.13. Patch by Jelle +Zijlstra. + +.. + +.. date: 2024-04-22-21-54-12 +.. gh-issue: 90848 +.. nonce: 5jHEEc +.. section: Library + +Fixed :func:`unittest.mock.create_autospec` to configure parent mock with +keyword arguments. + +.. + +.. date: 2024-04-22-20-42-29 +.. gh-issue: 118168 +.. nonce: Igni7h +.. section: Library + +Fix incorrect argument substitution when :data:`typing.Unpack` is used with +the builtin :class:`tuple`. :data:`!typing.Unpack` now raises +:exc:`TypeError` when used with certain invalid types. Patch by Jelle +Zijlstra. + +.. + +.. date: 2024-04-21-18-55-42 +.. gh-issue: 118131 +.. nonce: eAT0is +.. section: Library + +Add command-line interface for the :mod:`random` module. Patch by Hugo van +Kemenade. + +.. + +.. date: 2024-04-19-09-28-43 +.. gh-issue: 118107 +.. nonce: Mdsr1J +.. section: Library + +Fix :mod:`zipimport` reading of ZIP64 files with file entries that are too +big or offset too far. + +.. + +.. date: 2024-04-19-08-50-48 +.. gh-issue: 102511 +.. nonce: qDEB66 +.. section: Library + +Fix :func:`os.path.normpath` for UNC paths on Windows. +Speed up :func:`os.path.splitroot` with a native implementation. + +.. + +.. date: 2024-04-18-00-35-11 +.. gh-issue: 117535 +.. nonce: 0m6SIM +.. section: Library + +Change the unknown filename of :mod:`warnings` from ``sys`` to ```` to +clarify that it's not a real filename. + +.. + +.. date: 2024-04-17-22-00-15 +.. gh-issue: 114053 +.. nonce: _JBV4D +.. section: Library + +Fix erroneous :exc:`NameError` when calling :func:`typing.get_type_hints` on +a class that made use of :pep:`695` type parameters in a module that had +``from __future__ import annotations`` at the top of the file. Patch by Alex +Waygood. + +.. + +.. date: 2024-04-17-21-28-24 +.. gh-issue: 116931 +.. nonce: _AS09h +.. section: Library + +Add parameter *fileobj* check for :func:`tarfile.TarFile.addfile` + +.. + +.. date: 2024-04-17-19-41-59 +.. gh-issue: 117995 +.. nonce: Vt76Rv +.. section: Library + +Don't raise :exc:`DeprecationWarning` when a :term:`sequence` of parameters +is used to bind indexed, nameless placeholders. See also :gh:`100668`. + +.. + +.. date: 2024-04-17-18-00-30 +.. gh-issue: 80361 +.. nonce: RstWg- +.. section: Library + +Fix TypeError in :func:`email.message.Message.get_payload` when the charset is +:rfc:`2231` encoded. + +.. + +.. date: 2024-04-16-18-34-11 +.. gh-issue: 86650 +.. nonce: Zeydyg +.. section: Library + +Fix IndexError when parse some emails with invalid Message-ID (including +one-off addresses generated by Microsoft Outlook). + +.. + +.. date: 2024-04-14-15-59-28 +.. gh-issue: 117691 +.. nonce: 1mtREE +.. section: Library + +Improve the error messages emitted by :mod:`tarfile` deprecation warnings +relating to PEP 706. If a ``filter`` argument is not provided to +``extract()`` or ``extractall``, the deprecation warning now points to the +line in the user's code where the relevant function was called. Patch by +Alex Waygood. + +.. + +.. date: 2024-04-13-18-59-25 +.. gh-issue: 115874 +.. nonce: c3xG-E +.. section: Library + +Fixed a possible segfault during garbage collection of +``_asyncio.FutureIter`` objects. Patch by Savannah Ostrowski. + +.. + +.. date: 2024-04-13-01-45-15 +.. gh-issue: 115060 +.. nonce: IxoM03 +.. section: Library + +Speed up :meth:`pathlib.Path.glob` by omitting an initial +:meth:`~pathlib.Path.is_dir` call. As a result of this change, +:meth:`~pathlib.Path.glob` can no longer raise :exc:`OSError`. + +.. + +.. date: 2024-04-12-17-37-11 +.. gh-issue: 77102 +.. nonce: Mk6X_E +.. section: Library + +:mod:`site` module now parses ``.pth`` file with UTF-8 first, and +:term:`locale encoding` if ``UnicodeDecodeError`` happened. It supported +only locale encoding before. + +.. + +.. date: 2024-04-11-18-11-37 +.. gh-issue: 76785 +.. nonce: BWNkhC +.. section: Library + +We've exposed the low-level :mod:`!_interpreters` module for the sake of the +PyPI implementation of :pep:`734`. It was sometimes available as the +:mod:`!_xxsubinterpreters` module and was formerly used only for testing. +For the most part, it should be considered an internal module, like +:mod:`!_thread` and :mod:`!_imp`. See +https://discuss.python.org/t/pep-734-multiple-interpreters-in-the-stdlib/41147/26. + +.. + +.. date: 2024-04-10-22-35-24 +.. gh-issue: 115060 +.. nonce: XEVuOb +.. section: Library + +Speed up :meth:`pathlib.Path.glob` by not scanning directories for +non-wildcard pattern segments. + +.. + +.. date: 2024-04-10-21-30-37 +.. gh-issue: 117727 +.. nonce: uAYNVS +.. section: Library + +Speed up :meth:`pathlib.Path.iterdir` by using :func:`os.scandir` +internally. + +.. + +.. date: 2024-04-10-21-08-32 +.. gh-issue: 117586 +.. nonce: UCL__1 +.. section: Library + +Speed up :meth:`pathlib.Path.walk` by working with strings internally. + +.. + +.. date: 2024-04-10-20-59-10 +.. gh-issue: 117722 +.. nonce: oxIUEI +.. section: Library + +Change the new multi-separator support in :meth:`asyncio.StreamReader.readuntil` +to only accept tuples of separators rather than arbitrary iterables. + +.. + +.. date: 2024-04-09-23-22-21 +.. gh-issue: 117692 +.. nonce: EciInD +.. section: Library + +Fixes a bug when :class:`doctest.DocTestFinder` was failing on wrapped +``builtin_function_or_method``. + +.. + +.. date: 2024-04-09-20-14-44 +.. gh-issue: 117348 +.. nonce: A2NAAz +.. section: Library + +Largely restored import time performance of configparser by avoiding +dataclasses. + +.. + +.. date: 2024-04-08-19-30-38 +.. gh-issue: 117641 +.. nonce: oaBGSJ +.. section: Library + +Speedup :func:`os.path.commonpath` on Unix. + +.. + +.. date: 2024-04-08-19-12-26 +.. gh-issue: 117663 +.. nonce: CPfc_p +.. section: Library + +Fix ``_simple_enum`` to detect aliases when multiple arguments are present +but only one is the member value. + +.. + +.. date: 2024-04-08-14-33-38 +.. gh-issue: 117636 +.. nonce: exnRKd +.. section: Library + +Speedup :func:`os.path.join`. + +.. + +.. date: 2024-04-08-03-23-22 +.. gh-issue: 117618 +.. nonce: -4DCUw +.. section: Library + +Support ``package.module`` as ``filename`` for ``break`` command of +:mod:`pdb` + +.. + +.. date: 2024-04-07-19-39-20 +.. gh-issue: 102247 +.. nonce: h8rqiX +.. section: Library + +the status codes enum with constants in http.HTTPStatus are updated to +include the names from RFC9110. This RFC includes some HTTP statuses +previously only used for WEBDAV and assigns more generic names to them. + +The old constants are preserved for backwards compatibility. + +.. + +.. date: 2024-04-07-18-42-09 +.. gh-issue: 117607 +.. nonce: C978BD +.. section: Library + +Speedup :func:`os.path.relpath`. + +.. + +.. date: 2024-04-06-20-31-09 +.. gh-issue: 117586 +.. nonce: UgWdRK +.. section: Library + +Speed up :meth:`pathlib.Path.glob` by working with strings internally. + +.. + +.. date: 2024-04-06-18-41-36 +.. gh-issue: 117225 +.. nonce: tJh1Hw +.. section: Library + +Add colour to doctest output. Patch by Hugo van Kemenade. + +.. + +.. date: 2024-04-05-15-51-01 +.. gh-issue: 117566 +.. nonce: 54nABf +.. section: Library + +:meth:`ipaddress.IPv6Address.is_loopback` will now return ``True`` for +IPv4-mapped loopback addresses, i.e. addresses in the +``::ffff:127.0.0.0/104`` address space. + +.. + +.. date: 2024-04-05-13-38-53 +.. gh-issue: 117546 +.. nonce: lWjhHE +.. section: Library + +Fix issue where :func:`os.path.realpath` stopped resolving symlinks after +encountering a symlink loop on POSIX. + +.. + +.. date: 2024-04-04-15-28-12 +.. gh-issue: 116720 +.. nonce: aGhXns +.. section: Library + +Improved behavior of :class:`asyncio.TaskGroup` when an external +cancellation collides with an internal cancellation. For example, when two +task groups are nested and both experience an exception in a child task +simultaneously, it was possible that the outer task group would misbehave, +because its internal cancellation was swallowed by the inner task group. + +In the case where a task group is cancelled externally and also must raise +an :exc:`ExceptionGroup`, it will now call the parent task's +:meth:`~asyncio.Task.cancel` method. This ensures that a +:exc:`asyncio.CancelledError` will be raised at the next :keyword:`await`, +so the cancellation is not lost. + +An added benefit of these changes is that task groups now preserve the +cancellation count (:meth:`asyncio.Task.cancelling`). + +In order to handle some corner cases, :meth:`asyncio.Task.uncancel` may now +reset the undocumented ``_must_cancel`` flag when the cancellation count +reaches zero. + +.. + +.. date: 2024-04-03-16-01-31 +.. gh-issue: 117516 +.. nonce: 7DlHje +.. section: Library + +Add :data:`typing.TypeIs`, implementing :pep:`742`. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-04-03-15-04-23 +.. gh-issue: 117503 +.. nonce: NMfwup +.. section: Library + +Fix support of non-ASCII user names in bytes paths in +:func:`os.path.expanduser` on Posix. + +.. + +.. date: 2024-04-02-11-17-44 +.. gh-issue: 117394 +.. nonce: 2aoSlb +.. section: Library + +:func:`os.path.ismount` is now 2-3 times faster if the user has permissions. + +.. + +.. date: 2024-03-29-15-14-51 +.. gh-issue: 117313 +.. nonce: ks_ONu +.. section: Library + +Only treat ``'\n'``, ``'\r'`` and ``'\r\n'`` as line separators in +re-folding the :mod:`email` messages. Preserve control characters ``'\v'``, +``'\f'``, ``'\x1c'``, ``'\x1d'`` and ``'\x1e'`` and Unicode line separators +``'\x85'``, ``'\u2028'`` and ``'\u2029'`` as is. + +.. + +.. date: 2024-03-29-12-21-40 +.. gh-issue: 117142 +.. nonce: U0agfh +.. section: Library + +Convert :mod:`!_ctypes` to multi-phase initialisation (:pep:`489`). + +.. + +.. date: 2024-03-26-15-29-39 +.. gh-issue: 66543 +.. nonce: OZBhU5 +.. section: Library + +Add the :func:`mimetypes.guess_file_type` function which works with file +path. Passing file path instead of URL in :func:`~mimetypes.guess_type` is +:term:`soft deprecated`. + +.. + +.. date: 2024-03-20-00-11-39 +.. gh-issue: 68583 +.. nonce: mIlxxb +.. section: Library + +webbrowser CLI: replace getopt with argparse, add long options. Patch by +Hugo van Kemenade. + +.. + +.. date: 2024-03-17-18-24-23 +.. gh-issue: 116871 +.. nonce: 9uSl8M +.. section: Library + +Name suggestions for :exc:`AttributeError` and :exc:`ImportError` now only +include underscored names if the original name was underscored. + +.. + +.. date: 2024-02-28-11-51-51 +.. gh-issue: 116023 +.. nonce: CGYhFh +.. section: Library + +Don't show empty fields (value ``None`` or ``[]``) in :func:`ast.dump` by +default. Add ``show_empty=False`` parameter to optionally show them. + +.. + +.. date: 2024-02-28-10-41-24 +.. gh-issue: 115961 +.. nonce: P-_DU0 +.. section: Library + +Added :attr:`!name` and :attr:`!mode` attributes for compressed and archived +file-like objects in modules :mod:`bz2`, :mod:`lzma`, :mod:`tarfile` and +:mod:`zipfile`. The value of the :attr:`!mode` attribute of +:class:`gzip.GzipFile` was changed from integer (``1`` or ``2``) to string +(``'rb'`` or ``'wb'``). The value of the :attr:`!mode` attribute of the +readable file-like object returned by :meth:`zipfile.ZipFile.open` was +changed from ``'r'`` to ``'rb'``. + +.. + +.. date: 2024-02-11-07-31-43 +.. gh-issue: 82062 +.. nonce: eeS6w7 +.. section: Library + +Fix :func:`inspect.signature` to correctly handle parameter defaults on +methods in extension modules that use names defined in the module namespace. + +.. + +.. date: 2024-01-19-05-40-46 +.. gh-issue: 83856 +.. nonce: jN5M80 +.. section: Library + +Honor :mod:`atexit` for all :mod:`multiprocessing` start methods + +.. + +.. date: 2023-12-14-02-51-38 +.. gh-issue: 113081 +.. nonce: S-9Qyn +.. section: Library + +Print colorized exception just like built-in traceback in :mod:`pdb` + +.. + +.. date: 2023-12-07-20-05-54 +.. gh-issue: 112855 +.. nonce: ph4ehh +.. section: Library + +Speed up pickling of :class:`pathlib.PurePath` objects. Patch by Barney +Gale. + +.. + +.. date: 2023-11-07-22-41-42 +.. gh-issue: 111744 +.. nonce: TbLxF0 +.. section: Library + +Support opcode events in :mod:`bdb` + +.. + +.. date: 2023-10-24-12-39-04 +.. gh-issue: 109617 +.. nonce: YoI8TV +.. section: Library + +:mod:`!ncurses`: fixed a crash that could occur on macOS 13 or earlier when +Python was built with Apple Xcode 15's SDK. + +.. + +.. date: 2023-10-20-03-50-17 +.. gh-issue: 83151 +.. nonce: bcsD40 +.. section: Library + +Enabled arbitrary statements and evaluations in :mod:`pdb` shell to access +the local variables of the current frame, which made it possible for +multi-scope code like generators or nested function to work. + +.. + +.. date: 2023-10-02-10-35-58 +.. gh-issue: 110209 +.. nonce: b5zfIz +.. section: Library + +Add :meth:`~object.__class_getitem__` to :class:`types.GeneratorType` and +:class:`types.CoroutineType` for type hinting purposes. Patch by James +Hilton-Balfe. + +.. + +.. date: 2023-08-21-10-34-43 +.. gh-issue: 108191 +.. nonce: GZM3mv +.. section: Library + +The :class:`types.SimpleNamespace` now accepts an optional positional +argument which specifies initial values of attributes as a dict or an +iterable of key-value pairs. + +.. + +.. date: 2023-05-28-11-25-18 +.. gh-issue: 62090 +.. nonce: opAhDn +.. section: Library + +Fix assertion errors caused by whitespace in metavars or ``SUPPRESS``-ed +groups in :mod:`argparse` by simplifying usage formatting. Patch by Ali +Hamdan. + +.. + +.. date: 2023-03-03-21-13-08 +.. gh-issue: 102402 +.. nonce: fpkRO1 +.. section: Library + +Adjust ``logging.LogRecord`` to use ``time.time_ns()`` and fix minor bug +related to floating-point math. + +.. + +.. date: 2022-12-14-15-53-38 +.. gh-issue: 100242 +.. nonce: Ny7VUO +.. section: Library + +Bring pure Python implementation ``functools.partial.__new__`` more in line +with the C-implementation by not just always checking for the presence of +the attribute ``'func'`` on the first argument of ``partial``. Instead, both +the Python version and the C version perform an ``isinstance(func, +partial)`` check on the first argument of ``partial``. + +.. + +.. date: 2022-11-23-17-16-31 +.. gh-issue: 99730 +.. nonce: bDQdaX +.. section: Library + +HEAD requests are no longer upgraded to GET request during redirects in +urllib. + +.. + +.. date: 2022-10-24-12-05-19 +.. gh-issue: 66410 +.. nonce: du4UKW +.. section: Library + +Setting the :mod:`!tkinter` module global :data:`!wantobjects` to ``2`` +before creating the :class:`~tkinter.Tk` object or call the +:meth:`!wantobjects` method of the :class:`!Tk` object with argument +``2`` makes now arguments to callbacks registered in the :mod:`tkinter` module +to be passed as various Python objects (``int``, ``float``, ``bytes``, ``tuple``), +depending on their internal representation in Tcl, instead of always ``str``. +:data:`!tkinter.wantobjects` is now set to ``2`` by default. + +.. + +.. bpo: 40943 +.. date: 2020-06-10-19-24-17 +.. nonce: vjiiN_ +.. section: Library + +Fix several IndexError when parse emails with truncated Message-ID, address, +routes, etc, e.g. ``example@``. + +.. + +.. bpo: 39324 +.. date: 2020-01-14-09-46-51 +.. nonce: qUcDrM +.. section: Library + +Add mime type mapping for .md <-> text/markdown + +.. + +.. bpo: 18108 +.. date: 2019-09-09-18-18-34 +.. nonce: ajPLAO +.. section: Library + +:func:`shutil.chown` now supports *dir_fd* and *follow_symlinks* keyword +arguments. + +.. + +.. bpo: 30988 +.. date: 2019-08-29-20-26-08 +.. nonce: b-_h5O +.. section: Library + +Fix parsing of emails with invalid address headers having a leading or +trailing dot. Patch by tsufeki. + +.. + +.. bpo: 32839 +.. date: 2018-02-13-10-02-54 +.. nonce: McbVz3 +.. section: Library + +Add the :meth:`!after_info` method for Tkinter widgets. + +.. + +.. date: 2024-04-25-22-12-20 +.. gh-issue: 117928 +.. nonce: LKdTno +.. section: Documentation + +The minimum Sphinx version required for the documentation is now 6.2.1. + +.. + +.. date: 2024-05-07-21-15-47 +.. gh-issue: 118734 +.. nonce: --GHiS +.. section: Build + +Fixes Windows build when invoked directly (not through the :file:`build.bat` +script) without specifying a value for ``UseTIER2``. + +.. + +.. date: 2024-05-06-00-39-06 +.. gh-issue: 115119 +.. nonce: LT27pF +.. section: Build + +The :file:`configure` option :option:`--with-system-libmpdec` now defaults +to ``yes``. The bundled copy of ``libmpdecimal`` will be removed in Python +3.15. + +.. + +.. date: 2024-04-15-08-35-06 +.. gh-issue: 117845 +.. nonce: IowzyW +.. section: Build + +Fix building against recent libedit versions by detecting readline hook +signatures in :program:`configure`. + +.. + +.. date: 2024-04-14-19-35-35 +.. gh-issue: 116622 +.. nonce: 8lpX-7 +.. section: Build + +A testbed project was added to run the test suite on Android. + +.. + +.. date: 2024-04-09-12-59-06 +.. gh-issue: 117645 +.. nonce: 0oEVAa +.. section: Build + +Increase WASI stack size from 512 KiB to 8 MiB and the initial memory from +10 MiB to 20 MiB. Patch by Victor Stinner. + +.. + +.. date: 2024-02-13-15-31-28 +.. gh-issue: 115119 +.. nonce: FnQzAW +.. section: Build + +:program:`configure` now uses :program:`pkg-config` to detect :mod:`decimal` +dependencies if the :option:`--with-system-libmpdec` option is given. + +.. + +.. date: 2024-05-02-09-28-04 +.. gh-issue: 115119 +.. nonce: cUKMXo +.. section: Windows + +Update Windows installer to use libmpdecimal 4.0.0. + +.. + +.. date: 2024-05-01-20-57-09 +.. gh-issue: 118486 +.. nonce: K44KJG +.. section: Windows + +:func:`os.mkdir` now accepts *mode* of ``0o700`` to restrict the new +directory to the current user. + +.. + +.. date: 2024-04-29-13-53-25 +.. gh-issue: 118347 +.. nonce: U5ZRm_ +.. section: Windows + +Fixes launcher updates not being installed. + +.. + +.. date: 2024-04-26-14-23-07 +.. gh-issue: 118293 +.. nonce: ohhPtW +.. section: Windows + +The ``multiprocessing`` module now passes the ``STARTF_FORCEOFFFEEDBACK`` +flag when spawning processes to tell Windows not to change the mouse cursor. + +.. + +.. date: 2024-04-15-21-23-34 +.. gh-issue: 115009 +.. nonce: uhisHP +.. section: Windows + +Update Windows installer to use SQLite 3.45.3. + +.. + +.. date: 2024-04-12-14-02-58 +.. gh-issue: 90329 +.. nonce: YpEeaO +.. section: Windows + +Suppress the warning displayed on virtual environment creation when the +requested and created paths differ only by a short (8.3 style) name. +Warnings will continue to be shown if a junction or symlink in the path +caused the venv to be created in a different location than originally +requested. + +.. + +.. date: 2024-04-12-13-18-42 +.. gh-issue: 117786 +.. nonce: LpI01s +.. section: Windows + +Fixes virtual environments not correctly launching when created from a Store +install. + +.. + +.. date: 2024-05-03-12-13-27 +.. gh-issue: 115119 +.. nonce: ltDtoR +.. section: macOS + +Update macOS installer to use libmpdecimal 4.0.0. + +.. + +.. date: 2024-04-19-08-40-00 +.. gh-issue: 114099 +.. nonce: _iDfrQ +.. section: macOS + +iOS preprocessor symbol usage was made compatible with older macOS SDKs. + +.. + +.. date: 2024-04-15-21-19-39 +.. gh-issue: 115009 +.. nonce: IdxH9N +.. section: macOS + +Update macOS installer to use SQLite 3.45.3. + +.. + +.. date: 2022-04-17-01-07-42 +.. gh-issue: 91629 +.. nonce: YBGAAt +.. section: macOS + +Use :file:`~/.config/fish/conf.d` configs and :program:`fish_add_path` to +set :envvar:`PATH` when installing for the Fish shell. + +.. + +.. bpo: 34774 +.. date: 2018-09-23-01-36-39 +.. nonce: VeM-X- +.. section: IDLE + +Use user-selected color theme for Help => IDLE Doc. + +.. + +.. date: 2024-04-29-17-44-15 +.. gh-issue: 118124 +.. nonce: czQQ9G +.. section: C API + +Fix :c:macro:`Py_BUILD_ASSERT` and :c:macro:`Py_BUILD_ASSERT_EXPR` for +non-constant expressions: use ``static_assert()`` on C11 and newer. Patch by +Victor Stinner. + +.. + +.. date: 2024-04-29-17-19-07 +.. gh-issue: 110850 +.. nonce: vcpLn1 +.. section: C API + +Add "Raw" variant of PyTime functions + +* :c:func:`PyTime_MonotonicRaw` +* :c:func:`PyTime_PerfCounterRaw` +* :c:func:`PyTime_TimeRaw` + +Patch by Victor Stinner. + +.. + +.. date: 2024-04-17-16-48-17 +.. gh-issue: 117987 +.. nonce: zsvNL1 +.. section: C API + +Restore functions removed in Python 3.13 alpha 1: + +* :c:func:`Py_SetPythonHome` +* :c:func:`Py_SetProgramName` +* :c:func:`PySys_SetArgvEx` +* :c:func:`PySys_SetArgv` + +Patch by Victor Stinner. + +.. + +.. date: 2024-04-16-13-34-01 +.. gh-issue: 117929 +.. nonce: HSr419 +.. section: C API + +Restore removed :c:func:`PyEval_InitThreads` function. Patch by Victor +Stinner. + +.. + +.. date: 2024-04-08-09-44-29 +.. gh-issue: 117534 +.. nonce: 54ZE_n +.. section: C API + +Improve validation logic in the C implementation of +:meth:`datetime.datetime.fromisoformat` to better handle invalid years. +Patch by Vlad Efanov. + +.. + +.. date: 2024-03-18-17-29-52 +.. gh-issue: 68114 +.. nonce: W7R_lI +.. section: C API + +Fixed skipitem()'s handling of the old 'w' and 'w#' formatters. These are +no longer supported and now raise an exception if used. + +.. + +.. date: 2024-03-13-17-48-24 +.. gh-issue: 111997 +.. nonce: 8ZbHlA +.. section: C API + +Add a C-API for firing monitoring events. diff --git a/Misc/NEWS.d/3.14.0a1.rst b/Misc/NEWS.d/3.14.0a1.rst new file mode 100644 index 00000000000000..dfc33812611185 --- /dev/null +++ b/Misc/NEWS.d/3.14.0a1.rst @@ -0,0 +1,6604 @@ +.. date: 2024-09-24-10-48-46 +.. gh-issue: 124448 +.. nonce: bFMrS6 +.. release date: 2024-10-15 +.. section: macOS + +Update bundled Tcl/Tk in macOS installer to 8.6.15. + +.. + +.. date: 2024-09-07-12-14-54 +.. gh-issue: 123797 +.. nonce: yFDeug +.. section: macOS + +Check for runtime availability of ``ptsname_r`` function on macos. + +.. + +.. date: 2024-09-04-11-55-29 +.. gh-issue: 123418 +.. nonce: 8P4bmN +.. section: macOS + +Updated macOS installer build to use OpenSSL 3.0.15. + +.. + +.. date: 2024-09-27-15-07-30 +.. gh-issue: 124487 +.. nonce: 7LrwHC +.. section: Windows + +Increases Windows required OS and API level to Windows 10. + +.. + +.. date: 2024-09-27-13-40-25 +.. gh-issue: 124609 +.. nonce: WaKk8G +.. section: Windows + +Fix ``_Py_ThreadId`` for Windows builds using MinGW. Patch by Tony Roberts. + +.. + +.. date: 2024-09-24-19-04-56 +.. gh-issue: 124448 +.. nonce: srVT3d +.. section: Windows + +Updated bundled Tcl/Tk to 8.6.15. + +.. + +.. date: 2024-09-20-11-18-50 +.. gh-issue: 124254 +.. nonce: iPin-L +.. section: Windows + +Ensures experimental free-threaded binaries remain installed when updating. + +.. + +.. date: 2024-09-10-19-23-00 +.. gh-issue: 123915 +.. nonce: yZMEDO +.. section: Windows + +Ensure that ``Tools\msi\buildrelease.bat`` uses different directories for +AMD64 and ARM64 builds. + +.. + +.. date: 2024-09-04-09-59-18 +.. gh-issue: 123418 +.. nonce: QaMC12 +.. section: Windows + +Updated Windows build to use OpenSSL 3.0.15. + +.. + +.. date: 2024-08-29-16-13-45 +.. gh-issue: 123476 +.. nonce: m2DFS4 +.. section: Windows + +Add support for ``socket.TCP_QUICKACK`` on Windows platforms. + +.. + +.. date: 2024-08-01-10-55-15 +.. gh-issue: 122573 +.. nonce: 4-UCFY +.. section: Windows + +The Windows build of CPython now requires 3.10 or newer. + +.. + +.. date: 2024-07-19-21-50-54 +.. gh-issue: 100256 +.. nonce: GDrKba +.. section: Windows + +:mod:`mimetypes` no longer fails when it encounters an inaccessible registry +key. + +.. + +.. date: 2024-05-30-17-39-25 +.. gh-issue: 119679 +.. nonce: mZC87w +.. section: Windows + +Ensures correct import libraries are included in Windows installs. + +.. + +.. date: 2024-05-29-11-06-12 +.. gh-issue: 119690 +.. nonce: 8q6e1p +.. section: Windows + +Adds Unicode support and fixes audit events for ``_winapi.CreateNamedPipe``. + +.. + +.. date: 2024-05-25-18-43-10 +.. gh-issue: 111201 +.. nonce: SLPJIx +.. section: Windows + +Add support for new pyrepl on Windows + +.. + +.. date: 2024-05-22-19-43-29 +.. gh-issue: 119070 +.. nonce: _enton +.. section: Windows + +Fixes ``py.exe`` handling of shebangs like ``/usr/bin/env python3.12``, +which were previously interpreted as ``python3.exe`` instead of +``python3.12.exe``. + +.. + +.. date: 2024-04-24-22-50-33 +.. gh-issue: 117505 +.. nonce: gcTb_p +.. section: Windows + +Fixes an issue with the Windows installer not running ensurepip in a fully +isolated environment. This could cause unexpected interactions with the user +site-packages. + +.. + +.. date: 2024-04-24-05-16-32 +.. gh-issue: 118209 +.. nonce: Ryyzlz +.. section: Windows + +Avoid crashing in :mod:`mmap` on Windows when the mapped memory is +inaccessible due to file system errors or access violations. + +.. + +.. date: 2022-04-20-18-32-30 +.. gh-issue: 79846 +.. nonce: Vggv3f +.. section: Windows + +Makes :code:`ssl.create_default_context()` ignore invalid certificates in +the Windows certificate store + +.. + +.. date: 2024-09-04-10-07-51 +.. gh-issue: 123418 +.. nonce: 1eIFZb +.. section: Tools/Demos + +Update GitHub CI workflows to use OpenSSL 3.0.15 and multissltests to use +3.0.15, 3.1.7, and 3.2.3. + +.. + +.. date: 2024-10-07-14-13-38 +.. gh-issue: 125041 +.. nonce: PKLWDf +.. section: Tests + +Re-enable skipped tests for :mod:`zlib` on the s390x architecture: only skip +checks of the compressed bytes, which can be different between zlib's +software implementation and the hardware-accelerated implementation. + +.. + +.. date: 2024-09-25-12-39-34 +.. gh-issue: 124378 +.. nonce: Ywwgia +.. section: Tests + +Updated ``test_ttk`` to pass with Tcl/Tk 8.6.15. + +.. + +.. date: 2024-09-18-18-39-21 +.. gh-issue: 124213 +.. nonce: AQq_xg +.. section: Tests + +Detect whether the test suite is running inside a systemd-nspawn container +with ``--suppress-sync=true`` option, and skip the ``test_os`` and +``test_mmap`` tests that are failing in this scenario. + +.. + +.. date: 2024-09-17-22-21-58 +.. gh-issue: 124190 +.. nonce: 3fWhiX +.. section: Tests + +Add capability to ignore entire files or directories in check warning CI +tool + +.. + +.. date: 2024-07-17-08-25-06 +.. gh-issue: 121921 +.. nonce: HW8CIS +.. section: Tests + +Update ``Lib/test/crashers/bogus_code_obj.py`` so that it crashes properly +again. + +.. + +.. date: 2024-07-13-21-55-58 +.. gh-issue: 112301 +.. nonce: YJS1dl +.. section: Tests + +Add tooling to check for changes in compiler warnings. Patch by Nate Ohlson. + +.. + +.. date: 2024-07-13-11-48-20 +.. gh-issue: 59022 +.. nonce: fYNbQ8 +.. section: Tests + +Add tests for :func:`pkgutil.extend_path`. Patch by Andreas Stocker. + +.. + +.. date: 2024-07-13-11-04-44 +.. gh-issue: 99242 +.. nonce: aGxnwz +.. section: Tests + +:func:`os.getloadavg` may throw :exc:`OSError` when running regression tests +under certain conditions (e.g. chroot). This error is now caught and +ignored, since reporting load average is optional. + +.. + +.. date: 2024-07-04-15-10-29 +.. gh-issue: 121084 +.. nonce: qxcd5d +.. section: Tests + +Fix test_typing random leaks. Clear typing ABC caches when running tests for +refleaks (``-R`` option): call ``_abc_caches_clear()`` on typing abstract +classes and their subclasses. Patch by Victor Stinner. + +.. + +.. date: 2024-07-03-14-41-00 +.. gh-issue: 121160 +.. nonce: LEtiTd +.. section: Tests + +Add a test for :func:`readline.set_history_length`. Note that this test may +fail on readline libraries. + +.. + +.. date: 2024-07-01-16-15-06 +.. gh-issue: 121200 +.. nonce: 4Pc-gc +.. section: Tests + +Fix ``test_expanduser_pwd2()`` of ``test_posixpath``. Call ``getpwnam()`` +to get ``pw_dir``, since it can be different than ``getpwall()`` ``pw_dir``. +Patch by Victor Stinner. + +.. + +.. date: 2024-07-01-09-04-32 +.. gh-issue: 121188 +.. nonce: XbuTVa +.. section: Tests + +When creating the JUnit XML file, regrtest now escapes characters which are +invalid in XML, such as the chr(27) control character used in ANSI escape +sequences. Patch by Victor Stinner. + +.. + +.. date: 2024-06-20-12-51-26 +.. gh-issue: 120801 +.. nonce: lMVXC9 +.. section: Tests + +Cleaned up fixtures for importlib.metadata tests and consolidated behavior +with 'test.support.os_helper'. + +.. + +.. date: 2024-05-29-15-28-08 +.. gh-issue: 119727 +.. nonce: dVkaZM +.. section: Tests + +Add ``--single-process`` command line option to Python test runner +(regrtest). Patch by Victor Stinner. + +.. + +.. date: 2024-05-20-18-06-31 +.. gh-issue: 119273 +.. nonce: hf-yhX +.. section: Tests + +Python test runner no longer runs tests using TTY (ex: test_ioctl) in a +process group (using ``setsid()``). Previously, tests using TTY were +skipped. Patch by Victor Stinner. + +.. + +.. date: 2024-05-18-10-59-27 +.. gh-issue: 119050 +.. nonce: g4qiH7 +.. section: Tests + +regrtest test runner: Add XML support to the refleak checker (-R option). +Patch by Victor Stinner. + +.. + +.. date: 2024-05-04-22-56-41 +.. gh-issue: 101525 +.. nonce: LHK166 +.. section: Tests + +Skip ``test_gdb`` if the binary is relocated by BOLT. Patch by Donghee Na. + +.. + +.. date: 2023-08-03-17-26-55 +.. gh-issue: 107562 +.. nonce: ZnbscS +.. section: Tests + +Test certificates have been updated to expire far in the future. This allows +testing Y2038 with system time set to after that, so that actual Y2038 +issues can be exposed, and not masked by expired certificate errors. + +.. + +.. date: 2024-10-09-20-08-13 +.. gh-issue: 125140 +.. nonce: YgNWRB +.. section: Security + +Remove the current directory from ``sys.path`` when using PyREPL. + +.. + +.. date: 2024-09-04-12-41-35 +.. gh-issue: 123678 +.. nonce: N41y9n +.. section: Security + +Upgrade libexpat to 2.6.3 + +.. + +.. date: 2024-08-14-19-43-57 +.. gh-issue: 112301 +.. nonce: IQUcOy +.. section: Security + +Enable compiler options that warn of potential security vulnerabilities. + +.. + +.. date: 2024-08-07-10-42-13 +.. gh-issue: 122792 +.. nonce: oiTMo9 +.. section: Security + +Changed IPv4-mapped ``ipaddress.IPv6Address`` to consistently use the mapped +IPv4 address value for deciding properties. Properties which have their +behavior fixed are ``is_multicast``, ``is_reserved``, ``is_link_local``, +``is_global``, and ``is_unspecified``. + +.. + +.. date: 2024-08-06-00-06-23 +.. gh-issue: 112301 +.. nonce: 4k4lw6 +.. section: Security + +Add ability to ignore warnings per file with warning count in warning +checking tooling. Patch by Nate Ohlson. + +.. + +.. date: 2024-07-24-05-18-25 +.. gh-issue: 112301 +.. nonce: lfINgZ +.. section: Security + +Add macOS warning tracking to warning check tooling. Patch by Nate Ohlson. + +.. + +.. date: 2024-07-22-13-11-28 +.. gh-issue: 122133 +.. nonce: 0mPeta +.. section: Security + +Authenticate the socket connection for the ``socket.socketpair()`` fallback +on platforms where ``AF_UNIX`` is not available like Windows. + +Patch by Gregory P. Smith and Seth Larson +. Reported by Ellie + +.. + +.. date: 2024-07-18-13-17-47 +.. gh-issue: 121957 +.. nonce: QemKLU +.. section: Security + +Fixed missing audit events around interactive use of Python, now also +properly firing for ``python -i``, as well as for ``python -m asyncio``. The +events in question are ``cpython.run_stdin`` and ``cpython.run_startup``. + +.. + +.. date: 2024-07-08-23-39-04 +.. gh-issue: 112301 +.. nonce: TD8G01 +.. section: Security + +Enable runtime protections for glibc to abort execution when unsafe behavior +is encountered, for all platforms except Windows. + +.. + +.. date: 2024-07-02-13-39-20 +.. gh-issue: 121285 +.. nonce: hrl-yI +.. section: Security + +Remove backtracking from tarfile header parsing for ``hdrcharset``, PAX, and +GNU sparse headers. + +.. + +.. date: 2024-06-25-04-42-43 +.. gh-issue: 112301 +.. nonce: god4IC +.. section: Security + +Add default compiler options to improve security. Enable +-Wimplicit-fallthrough, -fstack-protector-strong, -Wtrampolines. + +.. + +.. date: 2024-05-08-21-59-38 +.. gh-issue: 118773 +.. nonce: 7dFRJY +.. section: Security + +Fixes creation of ACLs in :func:`os.mkdir` on Windows to work correctly on +non-English machines. + +.. + +.. date: 2024-05-01-20-57-09 +.. gh-issue: 118486 +.. nonce: K44KJG +.. section: Security + +:func:`os.mkdir` on Windows now accepts *mode* of ``0o700`` to restrict the +new directory to the current user. This fixes :cve:`2024-4030` affecting +:func:`tempfile.mkdtemp` in scenarios where the base temporary directory is +more permissive than the default. + +.. + +.. date: 2024-10-14-04-44-12 +.. gh-issue: 125422 +.. nonce: MlVuC6 +.. section: Library + +Fixed the bug where :mod:`pdb` and :mod:`bdb` can step into the bottom +caller frame. + +.. + +.. date: 2024-10-14-02-27-03 +.. gh-issue: 100141 +.. nonce: NuAcwa +.. section: Library + +Fixed the bug where :mod:`pdb` will be stuck in an infinite loop when +debugging an empty file. + +.. + +.. date: 2024-10-13-20-21-35 +.. gh-issue: 53203 +.. nonce: Rz1c8A +.. section: Library + +Fix :func:`time.strptime` for ``%c``, ``%x`` and ``%X`` formats in many +locales that use non-ASCII digits, like Persian, Burmese, Odia and Shan. + +.. + +.. date: 2024-10-11-18-03-05 +.. gh-issue: 125206 +.. nonce: pWRRK6 +.. section: Library + +Workaround for old libffi versions is added. Module ctypes supports +:c:expr:`double complex` only with libffi >= 3.3.0. Patch by Mikhail Efimov. + +.. + +.. date: 2024-10-11-16-19-46 +.. gh-issue: 89967 +.. nonce: vhWUOR +.. section: Library + +Make :class:`~weakref.WeakKeyDictionary` and +:class:`~weakref.WeakValueDictionary` safe against concurrent mutations from +other threads. Patch by Kumar Aditya. + +.. + +.. date: 2024-10-11-04-04-38 +.. gh-issue: 125260 +.. nonce: PeZ0Mb +.. section: Library + +The :func:`gzip.compress` *mtime* parameter now defaults to 0 for +reproducible output. Patch by Bernhard M. Wiedemann and Adam Turner. + +.. + +.. date: 2024-10-10-20-39-57 +.. gh-issue: 125243 +.. nonce: eUbbtu +.. section: Library + +Fix data race when creating :class:`zoneinfo.ZoneInfo` objects in the free +threading build. + +.. + +.. date: 2024-10-10-19-57-35 +.. gh-issue: 125254 +.. nonce: RtZxXS +.. section: Library + +Fix a bug where ArgumentError includes the incorrect ambiguous option in +:mod:`argparse`. + +.. + +.. date: 2024-10-10-18-33-31 +.. gh-issue: 125235 +.. nonce: 0kOB5I +.. section: Library + +Keep :mod:`tkinter` TCL paths in venv pointing to base installation on +Windows. + +.. + +.. date: 2024-10-09-21-42-43 +.. gh-issue: 61011 +.. nonce: pQXZb1 +.. section: Library + +Fix inheritance of nested mutually exclusive groups from parent parser in +:class:`argparse.ArgumentParser`. Previously, all nested mutually exclusive +groups lost their connection to the group containing them and were displayed +as belonging directly to the parser. + +.. + +.. date: 2024-10-09-07-09-00 +.. gh-issue: 125118 +.. nonce: J9rQ1S +.. section: Library + +Don't copy arbitrary values to :c:expr:`_Bool` in the :mod:`struct` module. + +.. + +.. date: 2024-10-08-21-17-16 +.. gh-issue: 125069 +.. nonce: 0RP0Mx +.. section: Library + +Fix an issue where providing a :class:`pathlib.PurePath` object as an +initializer argument to a second :class:`~pathlib.PurePath` object with a +different :attr:`~pathlib.PurePath.parser` resulted in arguments to the +former object's initializer being joined by the latter object's parser. + +.. + +.. date: 2024-10-08-13-28-22 +.. gh-issue: 125096 +.. nonce: Vz0W5g +.. section: Library + +If the :envvar:`PYTHON_BASIC_REPL` environment variable is set, the +:mod:`site` module no longer imports the :mod:`!_pyrepl` module. Moreover, +the :mod:`site` module now respects :option:`-E` and :option:`-I` command +line options: ignore :envvar:`PYTHON_BASIC_REPL` in this case. Patch by +Victor Stinner. + +.. + +.. date: 2024-10-08-12-09-09 +.. gh-issue: 124969 +.. nonce: _VBQLq +.. section: Library + +Fix ``locale.nl_langinfo(locale.ALT_DIGITS)``. Now it returns a tuple of up +to 100 strings (an empty tuple on most locales). Previously it returned the +first item of that tuple or an empty string. + +.. + +.. date: 2024-10-05-15-49-53 +.. gh-issue: 124960 +.. nonce: Bol9hT +.. section: Library + +Fix support for the ``barry_as_FLUFL`` future flag in the new REPL. + +.. + +.. date: 2024-10-04-12-43-03 +.. gh-issue: 69998 +.. nonce: DVqOXX +.. section: Library + +Fix :func:`locale.nl_langinfo` in case when different categories have +different locales. The function now sets temporarily the ``LC_CTYPE`` locale +in some cases. This temporary change affects other threads. + +.. + +.. date: 2024-10-04-08-46-00 +.. gh-issue: 124958 +.. nonce: rea9-x +.. section: Library + +Fix refcycles in exceptions raised from :class:`asyncio.TaskGroup` and the +python implementation of :class:`asyncio.Future` + +.. + +.. date: 2024-10-03-20-45-57 +.. gh-issue: 53203 +.. nonce: 3Sk4Ia +.. section: Library + +Fix :func:`time.strptime` for ``%c`` and ``%x`` formats in many locales: +Arabic, Bislama, Breton, Bodo, Kashubian, Chuvash, Estonian, French, Irish, +Ge'ez, Gurajati, Manx Gaelic, Hebrew, Hindi, Chhattisgarhi, Haitian Kreyol, +Japanese, Kannada, Korean, Marathi, Malay, Norwegian, Nynorsk, Punjabi, +Rajasthani, Tok Pisin, Yoruba, Yue Chinese, Yau/Nungon and Chinese. + +.. + +.. date: 2024-10-03-19-16-38 +.. gh-issue: 123961 +.. nonce: ik1Dgs +.. section: Library + +Convert :mod:`curses` to multi-phase initialization (:pep:`489`), thereby +fixing reference leaks at interpreter shutdown. Patch by Bénédikt Tran. + +.. + +.. date: 2024-10-03-17-13-22 +.. gh-issue: 124944 +.. nonce: YyLAzf +.. section: Library + +Add ``SO_ORIGINAL_DST`` to the :mod:`socket` module. + +.. + +.. date: 2024-10-03-05-00-25 +.. gh-issue: 117151 +.. nonce: Prdw_W +.. section: Library + +The default buffer size used by :func:`shutil.copyfileobj` has been +increased from 64k to 256k on non-Windows platforms. It was already larger +on Windows. + +.. + +.. date: 2024-10-02-22-53-48 +.. gh-issue: 90102 +.. nonce: 4qX52R +.. section: Library + +Skip the ``isatty`` system call during open() when the file is known to not +be a character device. This provides a slight performance improvement when +reading whole files. + +.. + +.. date: 2024-10-02-21-11-18 +.. gh-issue: 124917 +.. nonce: Lnwh5b +.. section: Library + +Allow calling :func:`os.path.exists` and :func:`os.path.lexists` with +keyword arguments on Windows. Fixes a regression in 3.13.0. + +.. + +.. date: 2024-10-02-16-35-07 +.. gh-issue: 65865 +.. nonce: S2D4wq +.. section: Library + +:mod:`argparse` now raises early error for invalid ``help`` arguments to +:meth:`~argparse.ArgumentParser.add_argument`, +:meth:`~argparse.ArgumentParser.add_subparsers` and :meth:`!add_parser`. + +.. + +.. date: 2024-10-02-15-05-45 +.. gh-issue: 124653 +.. nonce: tqsTu9 +.. section: Library + +Fix detection of the minimal Queue API needed by the :mod:`logging` module. +Patch by Bénédikt Tran. + +.. + +.. date: 2024-10-01-23-29-09 +.. gh-issue: 91818 +.. nonce: Kz8cPI +.. section: Library + +The CLI of many modules (:mod:`ast`, :mod:`ensurepip`, :mod:`json`, +:mod:`pdb`, :mod:`sqlite3`, :mod:`tokenize`, :mod:`venv`) now uses the +actual executable name instead of simply "python" to display in the usage +message. + +.. + +.. date: 2024-10-01-17-12-20 +.. gh-issue: 124858 +.. nonce: Zy0tvT +.. section: Library + +Fix reference cycles left in tracebacks in :func:`asyncio.open_connection` +when used with ``happy_eyeballs_delay`` + +.. + +.. date: 2024-10-01-13-46-58 +.. gh-issue: 124390 +.. nonce: dK1Zcm +.. section: Library + +Fixed :exc:`AssertionError` when using +:func:`!asyncio.staggered.staggered_race` with +:attr:`asyncio.eager_task_factory`. + +.. + +.. date: 2024-10-01-13-11-53 +.. gh-issue: 85935 +.. nonce: CTwJUy +.. section: Library + +:meth:`argparse.ArgumentParser.add_argument` now raises an exception if an +:ref:`action` that does not consume arguments (like 'store_const' or +'store_true') or explicit ``nargs=0`` are specified for positional +arguments. + +.. + +.. date: 2024-10-01-12-43-42 +.. gh-issue: 124835 +.. nonce: SVyp3K +.. section: Library + +Make :func:`tomllib.loads` raise :exc:`TypeError` not :exc:`AttributeError` +on bad input types that do not have the ``replace`` attribute. Improve error +message when :class:`bytes` is received. + +.. + +.. date: 2024-10-01-02-31-13 +.. gh-issue: 124693 +.. nonce: qzbXKB +.. section: Library + +Fix a bug where :mod:`argparse` doesn't recognize negative complex numbers +or negative numbers using scientific notation. + +.. + +.. date: 2024-09-30-20-46-32 +.. gh-issue: 124787 +.. nonce: 3FnJnP +.. section: Library + +Fix :class:`typing.TypeAliasType` with incorrect ``type_params`` argument. +Now it raises a :exc:`TypeError` when a type parameter without a default +follows one with a default, and when an entry in the ``type_params`` tuple +is not a type parameter object. + +.. + +.. date: 2024-09-30-19-59-28 +.. gh-issue: 66436 +.. nonce: 4gYN_n +.. section: Library + +Improved :ref:`prog` default value for :class:`argparse.ArgumentParser`. It +will now include the name of the Python executable along with the module or +package name, or the path to a directory, ZIP file, or directory within a +ZIP file if the code was run that way. + +.. + +.. date: 2024-09-27-15-16-04 +.. gh-issue: 116850 +.. nonce: dBkR0- +.. section: Library + +Fix :mod:`argparse` for namespaces with not directly writable dict (e.g. +classes). + +.. + +.. date: 2024-09-27-06-39-32 +.. gh-issue: 101552 +.. nonce: xYkzag +.. section: Library + +Add an *annoation_format* parameter to :func:`inspect.signature`. Add an +*quote_annotation_strings* parameter to :meth:`inspect.Signature.format`. +Use the new functionality to improve the display of annotations in +signatures in :mod:`pydoc`. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-09-26-22-14-12 +.. gh-issue: 58573 +.. nonce: hozbm9 +.. section: Library + +Fix conflicts between abbreviated long options in the parent parser and +subparsers in :mod:`argparse`. + +.. + +.. date: 2024-09-26-13-43-39 +.. gh-issue: 124594 +.. nonce: peYhsP +.. section: Library + +All :mod:`asyncio` REPL prompts run in the same :class:`context +`. Contributed by Bartosz Sławecki. + +.. + +.. date: 2024-09-26-09-18-09 +.. gh-issue: 61181 +.. nonce: dwjmch +.. section: Library + +Fix support of :ref:`choices` with string value in :mod:`argparse`. +Substrings of the specified string no longer considered valid values. + +.. + +.. date: 2024-09-26-00-35-24 +.. gh-issue: 116750 +.. nonce: X1aMHI +.. section: Library + +Provide :func:`sys.monitoring.clear_tool_id` to unregister all events and +callbacks set by the tool. + +.. + +.. date: 2024-09-25-22-06-52 +.. gh-issue: 124552 +.. nonce: 1nQKNM +.. section: Library + +Improve the accuracy of :mod:`bdb`'s check for the possibility of breakpoint +in a frame. This makes it possible to disable unnecessary events in +functions. + +.. + +.. date: 2024-09-25-18-34-48 +.. gh-issue: 124538 +.. nonce: nXZk4R +.. section: Library + +Fixed crash when using :func:`gc.get_referents` on a capsule object. + +.. + +.. date: 2024-09-25-18-08-29 +.. gh-issue: 80259 +.. nonce: kO5Tw7 +.. section: Library + +Fix :mod:`argparse` support of positional arguments with ``nargs='?'``, +``default=argparse.SUPPRESS`` and specified ``type``. + +.. + +.. date: 2024-09-25-18-07-51 +.. gh-issue: 120378 +.. nonce: NlBSz_ +.. section: Library + +Fix a crash related to an integer overflow in :func:`curses.resizeterm` and +:func:`curses.resize_term`. + +.. + +.. date: 2024-09-25-12-14-58 +.. gh-issue: 124498 +.. nonce: Ozxs55 +.. section: Library + +Fix :class:`typing.TypeAliasType` not to be generic, when ``type_params`` is +an empty tuple. + +.. + +.. date: 2024-09-25-10-25-57 +.. gh-issue: 53834 +.. nonce: uyIckw +.. section: Library + +Fix support of arguments with :ref:`choices` in :mod:`argparse`. Positional +arguments with :ref:`nargs` equal to ``'?'`` or ``'*'`` no longer check +:ref:`default` against ``choices``. Optional arguments with ``nargs`` equal +to ``'?'`` no longer check :ref:`const` against ``choices``. + +.. + +.. date: 2024-09-24-22-38-51 +.. gh-issue: 123884 +.. nonce: iEPTK4 +.. section: Library + +Fixed bug in itertools.tee() handling of other tee inputs (a tee in a tee). +The output now has the promised *n* independent new iterators. Formerly, +the first iterator was identical (not independent) to the input iterator. +This would sometimes give surprising results. + +.. + +.. date: 2024-09-24-21-15-27 +.. gh-issue: 123017 +.. nonce: dSAr2f +.. section: Library + +Due to unreliable results on some devices, :func:`time.strftime` no longer +accepts negative years on Android. + +.. + +.. date: 2024-09-24-19-32-14 +.. gh-issue: 123014 +.. nonce: zVcfkZ +.. section: Library + +:func:`os.pidfd_open` and :func:`signal.pidfd_send_signal` are now +unavailable when building against Android API levels older than 31, since +the underlying system calls may cause a crash. + +.. + +.. date: 2024-09-24-13-32-16 +.. gh-issue: 124176 +.. nonce: 6hmOPz +.. section: Library + +Add support for :func:`dataclasses.dataclass` in +:func:`unittest.mock.create_autospec`. Now ``create_autospec`` will check +for potential dataclasses and use :func:`dataclasses.fields` function to +retrieve the spec information. + +.. + +.. date: 2024-09-24-12-34-48 +.. gh-issue: 124345 +.. nonce: s3vKql +.. section: Library + +:mod:`argparse` vim supports abbreviated single-dash long options separated +by ``=`` from its value. + +.. + +.. date: 2024-09-24-00-01-24 +.. gh-issue: 124400 +.. nonce: 0XCgfe +.. section: Library + +Fixed a :mod:`pdb` bug where ``until`` has no effect when it appears in a +``commands`` sequence. Also avoid printing the frame information at a +breakpoint that has a command list containing a command that resumes +execution. + +.. + +.. date: 2024-09-23-18-26-17 +.. gh-issue: 90562 +.. nonce: Yj566G +.. section: Library + +Modify dataclasses to support zero-argument super() when ``slots=True`` is +specified. This works by modifying all references to ``__class__`` to point +to the newly created class. + +.. + +.. date: 2024-09-23-17-33-47 +.. gh-issue: 104860 +.. nonce: O86OSc +.. section: Library + +Fix disallowing abbreviation of single-dash long options in :mod:`argparse` +with ``allow_abbrev=False``. + +.. + +.. date: 2024-09-21-23-56-41 +.. gh-issue: 63143 +.. nonce: YKu-LQ +.. section: Library + +Fix parsing mutually exclusive arguments in :mod:`argparse`. Arguments with +the value identical to the default value (e.g. booleans, small integers, +empty or 1-character strings) are no longer considered "not present". + +.. + +.. date: 2024-09-21-22-32-21 +.. gh-issue: 72795 +.. nonce: naLmkX +.. section: Library + +Positional arguments with :ref:`nargs` equal to ``'*'`` or +:data:`!argparse.REMAINDER` are no longer required. This allows to use +positional argument with ``nargs='*'`` and without ``default`` in mutually +exclusive group and improves error message about required arguments. + +.. + +.. date: 2024-09-21-19-02-37 +.. gh-issue: 59317 +.. nonce: OAhNZZ +.. section: Library + +Fix parsing positional argument with :ref:`nargs` equal to ``'?'`` or +``'*'`` if it is preceded by an option and another positional argument. + +.. + +.. date: 2024-09-20-18-23-19 +.. gh-issue: 100980 +.. nonce: 8nVAB6 +.. section: Library + +The :attr:`~ctypes.Structure._fields_` attribute of +:class:`ctypes.Structure` and :class:`~ctypes.Union` is no longer set if the +setattr operation raises an error. + +.. + +.. date: 2024-09-20-12-23-11 +.. gh-issue: 53780 +.. nonce: mrV1zi +.. section: Library + +:mod:`argparse` now ignores the first ``"--"`` (double dash) between an +option and command. + +.. + +.. date: 2024-09-19-20-15-00 +.. gh-issue: 124217 +.. nonce: j0KlQB +.. section: Library + +Add RFC 9637 reserved IPv6 block ``3fff::/20`` in :mod:`ipaddress` module. + +.. + +.. date: 2024-09-19-16-00-22 +.. gh-issue: 111513 +.. nonce: 6jHm02 +.. section: Library + +Improve the error message that may be raised by +:meth:`datetime.date.fromtimestamp`. + +.. + +.. date: 2024-09-19-11-47-39 +.. gh-issue: 124248 +.. nonce: g7rufd +.. section: Library + +Fixed potential crash when using :mod:`struct` to process zero-width 'Pascal +string' fields (``0p``). + +.. + +.. date: 2024-09-19-10-36-18 +.. gh-issue: 81691 +.. nonce: Hyhp_U +.. section: Library + +Fix handling of multiple ``"--"`` (double dashes) in :mod:`argparse`. Only +the first one has now been removed, all subsequent ones are now taken +literally. + +.. + +.. date: 2024-09-19-03-46-59 +.. gh-issue: 87041 +.. nonce: 9Ox7Bv +.. section: Library + +Fix a bug in :mod:`argparse` where lengthy subparser argument help is +incorrectly indented. + +.. + +.. date: 2024-09-19-00-09-48 +.. gh-issue: 84559 +.. nonce: IrxvQe +.. section: Library + +The default :mod:`multiprocessing` start method on Linux and other POSIX +systems has been changed away from often unsafe ``"fork"`` to +``"forkserver"`` (when the platform supports sending file handles over pipes +as most do) or ``"spawn"``. Mac and Windows are unchanged as they already +default to ``"spawn"``. + +.. + +.. date: 2024-09-18-17-45-52 +.. gh-issue: 124212 +.. nonce: n6kIby +.. section: Library + +Fix invalid variable in :mod:`venv` handling of failed symlink on Windows + +.. + +.. date: 2024-09-17-18-06-42 +.. gh-issue: 124171 +.. nonce: PHCvRJ +.. section: Library + +Add workaround for broken :c:func:`!fmod()` implementations on Windows, that +loose zero sign (e.g. ``fmod(-10, 1)`` returns ``0.0``). Patch by Sergey B +Kirpichev. + +.. + +.. date: 2024-09-16-12-31-48 +.. gh-issue: 123978 +.. nonce: z3smEu +.. section: Library + +Remove broken :func:`time.thread_time` and :func:`time.thread_time_ns` on +NetBSD. + +.. + +.. date: 2024-09-13-10-34-19 +.. gh-issue: 123934 +.. nonce: yMe7mL +.. section: Library + +Fix :class:`unittest.mock.MagicMock` resetting magic methods return values +after ``.reset_mock(return_value=True)`` was called. + +.. + +.. date: 2024-09-12-10-55-19 +.. gh-issue: 124016 +.. nonce: ncs0hd +.. section: Library + +Update :mod:`unicodedata` database to Unicode 16.0.0. + +.. + +.. date: 2024-09-11-19-12-23 +.. gh-issue: 123968 +.. nonce: OwHON_ +.. section: Library + +Fix the command-line interface for the :mod:`random` module to select floats +between 0 and N, not 1 and N. + +.. + +.. date: 2024-09-11-19-05-32 +.. gh-issue: 123945 +.. nonce: jLwybB +.. section: Library + +Fix a bug where :mod:`argparse` doesn't recognize negative numbers with +underscores + +.. + +.. date: 2024-09-11-13-33-19 +.. gh-issue: 123935 +.. nonce: fRZ_56 +.. section: Library + +Fix parent slots detection for dataclasses that inherit from classes with +``__dictoffset__``. + +.. + +.. date: 2024-09-10-11-26-14 +.. gh-issue: 123892 +.. nonce: 2gzIrz +.. section: Library + +Add ``"_wmi"`` to :data:`sys.stdlib_module_names`. Patch by Victor Stinner. + +.. + +.. date: 2024-09-06-10-17-54 +.. gh-issue: 84808 +.. nonce: ION67Z +.. section: Library + +Fix error handling in :py:class:`~socket.socket` method +:py:func:`~socket.socket.connect_ex` on platforms where :c:data:`errno` can +be negative. + +.. + +.. date: 2024-09-06-01-35-11 +.. gh-issue: 123756 +.. nonce: Ozbhke +.. section: Library + +Added a new argument ``mode`` to :class:`pdb.Pdb`. Only allow :mod:`pdb` +from command line to use ``restart`` command. + +.. + +.. date: 2024-09-06-00-00-43 +.. gh-issue: 122765 +.. nonce: tx4hsr +.. section: Library + +Fix unbalanced quote errors occurring when activate.csh in :mod:`venv` was +sourced with a custom prompt containing unpaired quotes or newlines. + +.. + +.. date: 2024-09-04-18-23-43 +.. gh-issue: 123657 +.. nonce: Oks4So +.. section: Library + +Fix crash and memory leak in :func:`decimal.getcontext`. It crashed when +using a thread-local context by ``--with-decimal-contextvar=no``. + +.. + +.. date: 2024-09-02-20-34-04 +.. gh-issue: 123339 +.. nonce: czgcSu +.. section: Library + +Fix :func:`inspect.getsource` for classes in :mod:`collections.abc` and +:mod:`decimal` (for pure Python implementation) modules. +:func:`inspect.getcomments` now raises OSError instead of IndexError if the +``__firstlineno__`` value for a class is out of bound. + +.. + +.. date: 2024-08-31-12-34-44 +.. gh-issue: 123374 +.. nonce: 3kE7rb +.. section: Library + +Remove check for redefined memo entry in :func:`pickletools.dis`. + +.. + +.. date: 2024-08-30-09-01-35 +.. gh-issue: 123504 +.. nonce: lJ9_BB +.. section: Library + +Fixed reference leak in the finalization of :mod:`tkinter`. + +.. + +.. date: 2024-08-29-14-51-36 +.. gh-issue: 123430 +.. nonce: M7wXl9 +.. section: Library + +Pages generated by the :mod:`http.server` module allow the browser to apply +its default dark mode. + +.. + +.. date: 2024-08-29-09-27-12 +.. gh-issue: 123446 +.. nonce: _I_mMr +.. section: Library + +Fix empty function name in :exc:`TypeError` when :func:`csv.reader`, +:func:`csv.writer`, or :func:`csv.register_dialect` are used without the +required args. + +.. + +.. date: 2024-08-28-20-08-19 +.. gh-issue: 123448 +.. nonce: tItJlp +.. section: Library + +Fixed memory leak of :class:`typing.NoDefault` by moving it to the static +types array. + +.. + +.. date: 2024-08-28-13-03-36 +.. gh-issue: 123409 +.. nonce: lW0YF- +.. section: Library + +Fix :attr:`ipaddress.IPv6Address.reverse_pointer` output according to +:rfc:`RFC 3596, §2.5 <3596#section-2.5>`. Patch by Bénédikt Tran. + +.. + +.. date: 2024-08-27-12-38-42 +.. gh-issue: 123089 +.. nonce: vA7iFR +.. section: Library + +Make :class:`weakref.WeakSet` safe against concurrent mutations while it is +being iterated. Patch by Kumar Aditya. + +.. + +.. date: 2024-08-27-12-11-00 +.. gh-issue: 123363 +.. nonce: gKuJp6 +.. section: Library + +Show string value of :opcode:`CONTAINS_OP` oparg in :mod:`dis` output. Patch +by Alexandr153. + +.. + +.. date: 2024-08-27-10-30-37 +.. gh-issue: 123341 +.. nonce: 5e-fjt +.. section: Library + +Add :meth:`~object.__class_getitem__` to :class:`!tkinter.Event` for type +subscript support at runtime. Patch by Adonis Rakateli. + +.. + +.. date: 2024-08-26-19-36-00 +.. gh-issue: 123340 +.. nonce: mQKI1H +.. section: Library + +Show string value of :opcode:`IS_OP` oparg in :mod:`dis` output. + +.. + +.. date: 2024-08-26-18-48-13 +.. gh-issue: 119518 +.. nonce: QFYH9q +.. section: Library + +Speed up normalization of :class:`pathlib.PurePath` and +:class:`~pathlib.Path` objects by not interning string parts. + +.. + +.. date: 2024-08-26-13-45-20 +.. gh-issue: 123270 +.. nonce: gXHvNJ +.. section: Library + +Applied a more surgical fix for malformed payloads in :class:`zipfile.Path` +causing infinite loops (gh-122905) without breaking contents using +legitimate characters. + +.. + +.. date: 2024-08-25-16-59-20 +.. gh-issue: 73991 +.. nonce: 1w8u3K +.. section: Library + +Add :meth:`pathlib.Path.copy_into` and :meth:`~pathlib.Path.move_into`, +which copy and move files and directories into *existing* directories. + +.. + +.. date: 2024-08-24-06-05-41 +.. gh-issue: 123228 +.. nonce: jR_5O5 +.. section: Library + +Fix return type for +:func:`!_pyrepl.readline._ReadlineWrapper.get_line_buffer` to be +:func:`str`. Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-08-24-00-03-01 +.. gh-issue: 123240 +.. nonce: uFPG3l +.. section: Library + +Raise audit events for the :func:`input` in the new REPL. + +.. + +.. date: 2024-08-23-22-01-30 +.. gh-issue: 76960 +.. nonce: vsANPu +.. section: Library + +Fix :func:`urllib.parse.urljoin` and :func:`urllib.parse.urldefrag` for URIs +containing empty components. For example, :func:`!urljoin` with relative +reference "?" now sets empty query and removes fragment. Preserve empty +components (authority, params, query, fragment) in :func:`!urljoin`. +Preserve empty components (authority, params, query) in :func:`!urldefrag`. + +.. + +.. date: 2024-08-23-15-49-10 +.. gh-issue: 116810 +.. nonce: QLBUU8 +.. section: Library + +Resolve a memory leak introduced in CPython 3.10's :mod:`ssl` when the +:attr:`ssl.SSLSocket.session` property was accessed. Speeds up read and +write access to said property by no longer unnecessarily cloning session +objects via serialization. + +.. + +.. date: 2024-08-22-20-10-13 +.. gh-issue: 123243 +.. nonce: Kifj1L +.. section: Library + +Fix memory leak in :mod:`!_decimal`. + +.. + +.. date: 2024-08-22-11-25-19 +.. gh-issue: 122546 +.. nonce: BSmeE7 +.. section: Library + +Consistently use same file name for different exceptions in the new repl. +Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-08-22-09-37-48 +.. gh-issue: 123213 +.. nonce: owmXnP +.. section: Library + +:meth:`xml.etree.ElementTree.Element.extend` and +:class:`~xml.etree.ElementTree.Element` assignment no longer hide the +internal exception if an erroneous generator is passed. Patch by Bar Harel. + +.. + +.. date: 2024-08-20-18-02-27 +.. gh-issue: 85110 +.. nonce: 8_iDQy +.. section: Library + +Preserve relative path in URL without netloc in +:func:`urllib.parse.urlunsplit` and :func:`urllib.parse.urlunparse`. + +.. + +.. date: 2024-08-20-14-22-49 +.. gh-issue: 123165 +.. nonce: vOZZOA +.. section: Library + +Add support for rendering :class:`~dis.Positions` in :mod:`dis`. + +.. + +.. date: 2024-08-19-17-37-18 +.. gh-issue: 122909 +.. nonce: kP12SK +.. section: Library + +In urllib.request when URLError is raised opening an ftp URL, the exception +argument is now consistently a string. Earlier versions passed either a +string or an ftplib exception instance as the argument to URLError. + +.. + +.. date: 2024-08-18-08-25-32 +.. gh-issue: 123084 +.. nonce: rf8izX +.. section: Library + +Deprecate :class:`!shutil.ExecError`, which hasn't been raised by any +:mod:`shutil` function since Python 3.4. It's now an alias for +:exc:`RuntimeError`. + +.. + +.. date: 2024-08-17-08-17-20 +.. gh-issue: 123085 +.. nonce: 7Io2yH +.. section: Library + +In a bare call to :func:`importlib.resources.files`, ensure the caller's +frame is properly detected when ``importlib.resources`` is itself available +as a compiled module only (no source). + +.. + +.. date: 2024-08-16-19-13-21 +.. gh-issue: 123067 +.. nonce: Nx9O4R +.. section: Library + +Fix quadratic complexity in parsing ``"``-quoted cookie values with +backslashes by :mod:`http.cookies`. + +.. + +.. date: 2024-08-16-16-53-52 +.. gh-issue: 123049 +.. nonce: izx_fH +.. section: Library + +Add support for :const:`~configparser.UNNAMED_SECTION` in +:meth:`configparser.ConfigParser.add_section`. + +.. + +.. date: 2024-08-15-09-45-34 +.. gh-issue: 121735 +.. nonce: _1q0qf +.. section: Library + +When working with zip archives, importlib.resources now properly honors +module-adjacent references (e.g. ``files(pkg.mod)`` and not just +``files(pkg)``). + +.. + +.. date: 2024-08-14-10-41-11 +.. gh-issue: 122981 +.. nonce: BHV0Z9 +.. section: Library + +Fix :func:`inspect.getsource` for generated classes with Python base classes +(e.g. enums). + +.. + +.. date: 2024-08-11-14-23-07 +.. gh-issue: 122903 +.. nonce: xktZta +.. section: Library + +``zipfile.Path.glob`` now correctly matches directories instead of silently +omitting them. + +.. + +.. date: 2024-08-11-14-08-04 +.. gh-issue: 122905 +.. nonce: 7tDsxA +.. section: Library + +:class:`zipfile.Path` objects now sanitize names from the zipfile. + +.. + +.. date: 2024-08-10-14-16-59 +.. gh-issue: 122873 +.. nonce: XlHaUn +.. section: Library + +Enable :mod:`json` module to work as a script using the :option:`-m` switch: +``python -m json``. See the :ref:`JSON command-line interface +` documentation. Patch by Trey Hunner. + +.. + +.. date: 2024-08-10-10-21-44 +.. gh-issue: 122858 +.. nonce: ZC1rJD +.. section: Library + +Deprecate :func:`!asyncio.iscoroutinefunction` in favor of +:func:`inspect.iscoroutinefunction`. + +.. + +.. date: 2024-08-07-17-41-16 +.. gh-issue: 116263 +.. nonce: EcXir0 +.. section: Library + +:class:`logging.handlers.RotatingFileHandler` no longer rolls over empty log +files. + +.. + +.. date: 2024-08-07-14-12-19 +.. gh-issue: 105376 +.. nonce: QbGPdE +.. section: Library + +Restore the deprecated :mod:`logging` ``warn()`` method. It was removed in +Python 3.13 alpha 1. Keep the deprecated ``warn()`` method in Python 3.13. +Patch by Victor Stinner. + +.. + +.. date: 2024-08-07-11-57-41 +.. gh-issue: 122311 +.. nonce: LDExnJ +.. section: Library + +Improve errors in the :mod:`pickle` module. :exc:`~pickle.PicklingError` is +now raised more often instead of :exc:`UnicodeEncodeError`, +:exc:`ValueError` and :exc:`AttributeError`, and the original exception is +chained to it. Improve and unify error messages in Python and C +implementations. + +.. + +.. date: 2024-08-06-18-07-19 +.. gh-issue: 122744 +.. nonce: kCzNDI +.. section: Library + +Bump the version of pip bundled in ensurepip to version 24.2. + +.. + +.. date: 2024-08-06-10-36-55 +.. gh-issue: 118761 +.. nonce: q_x_1A +.. section: Library + +Improve import time of :mod:`pprint` by around seven times. Patch by Hugo +van Kemenade. + +.. + +.. date: 2024-08-06-07-24-00 +.. gh-issue: 118974 +.. nonce: qamsCQ +.. section: Library + +Add ``decorator`` parameter to :func:`dataclasses.make_dataclass` to +customize the functional creation of dataclasses. + +.. + +.. date: 2024-08-04-14-07-18 +.. gh-issue: 118814 +.. nonce: uiyks1 +.. section: Library + +Fix the :class:`typing.TypeVar` constructor when name is passed by keyword. + +.. + +.. date: 2024-08-03-06-51-08 +.. gh-issue: 122637 +.. nonce: gpas8J +.. section: Library + +Adjust ``cmath.tanh(nanj)`` and ``cmath.tanh(infj)`` for recent C standards. + +.. + +.. date: 2024-07-31-20-43-21 +.. gh-issue: 122478 +.. nonce: sCU2Le +.. section: Library + +Remove internal frames from tracebacks shown in +:class:`code.InteractiveInterpreter` with non-default +:func:`sys.excepthook`. Save correct tracebacks in +:attr:`sys.last_traceback` and update ``__traceback__`` attribute of +:attr:`sys.last_value` and :attr:`sys.last_exc`. + +.. + +.. date: 2024-07-31-15-08-42 +.. gh-issue: 116622 +.. nonce: aKxIQA +.. section: Library + +On Android, the ``FICLONE`` and ``FICLONERANGE`` constants are no longer +exposed by :mod:`fcntl`, as these ioctls are blocked by SELinux. + +.. + +.. date: 2024-07-31-14-55-41 +.. gh-issue: 82378 +.. nonce: eZvYmR +.. section: Library + +Make sure that the new :term:`REPL` interprets :data:`sys.tracebacklimit` in +the same way that the classic REPL did. + +.. + +.. date: 2024-07-30-21-29-30 +.. gh-issue: 122334 +.. nonce: LeoE1x +.. section: Library + +Fix crash when importing :mod:`ssl` after the main interpreter restarts. + +.. + +.. date: 2024-07-30-15-57-07 +.. gh-issue: 122459 +.. nonce: AYIoeN +.. section: Library + +Optimize :mod:`pickling ` by name objects without the ``__module__`` +attribute. + +.. + +.. date: 2024-07-30-14-46-16 +.. gh-issue: 87320 +.. nonce: -Yk1wb +.. section: Library + +In :class:`code.InteractiveInterpreter`, handle exceptions caused by calling +a non-default :func:`sys.excepthook`. Before, the exception bubbled up to +the caller, ending the :term:`REPL`. + +.. + +.. date: 2024-07-30-04-27-55 +.. gh-issue: 122272 +.. nonce: 6Wwa1V +.. section: Library + +On some platforms such as Linux, year with century was not 0-padded when +formatted by :meth:`~.datetime.strftime` with C99-specific specifiers +``'%C'`` or ``'%F'``. The 0-padding behavior is now guaranteed when the +format specifiers ``'%C'`` and ``'%F'`` are supported by the C library. +Patch by Ben Hsing + +.. + +.. date: 2024-07-29-16-47-08 +.. gh-issue: 122400 +.. nonce: fM0YSv +.. section: Library + +Handle :exc:`ValueError`\s raised by :func:`os.stat` in +:class:`filecmp.dircmp` and :func:`filecmp.cmpfiles`. Patch by Bénédikt +Tran. + +.. + +.. date: 2024-07-29-10-24-48 +.. gh-issue: 122311 +.. nonce: xChV1b +.. section: Library + +Fix some error messages in :mod:`pickle`. + +.. + +.. date: 2024-07-27-16-10-41 +.. gh-issue: 121650 +.. nonce: nf6oc9 +.. section: Library + +:mod:`email` headers with embedded newlines are now quoted on output. The +:mod:`~email.generator` will now refuse to serialize (write) headers that +are unsafely folded or delimited; see +:attr:`~email.policy.Policy.verify_generated_headers`. (Contributed by Bas +Bloemsaat and Petr Viktorin in :gh:`121650`.) + +.. + +.. date: 2024-07-26-21-21-13 +.. gh-issue: 122332 +.. nonce: fvw88r +.. section: Library + +Fixed segfault with :meth:`asyncio.Task.get_coro` when using an eager task +factory. + +.. + +.. date: 2024-07-25-15-41-14 +.. gh-issue: 105733 +.. nonce: o3koJA +.. section: Library + +:func:`ctypes.ARRAY` is now :term:`soft deprecated`: it no longer emits +deprecation warnings and is not scheduled for removal. + +.. + +.. date: 2024-07-24-08-48-22 +.. gh-issue: 122213 +.. nonce: o3pdgA +.. section: Library + +Add notes for pickle serialization errors that allow to identify the source +of the error. + +.. + +.. date: 2024-07-23-22-26-00 +.. gh-issue: 119180 +.. nonce: B2IVT8 +.. section: Library + +As part of :pep:`749`, add the following attributes for customizing +evaluation of annotation scopes: + +* ``evaluate_value`` on :class:`typing.TypeAliasType` +* ``evaluate_bound``, ``evaluate_constraints``, and ``evaluate_default`` on :class:`typing.TypeVar` +* ``evaluate_default`` on :class:`typing.ParamSpec` +* ``evaluate_default`` on :class:`typing.TypeVarTuple` + +.. + +.. date: 2024-07-23-17-13-10 +.. gh-issue: 119180 +.. nonce: 5PZELo +.. section: Library + +Fix handling of classes with custom metaclasses in +``annotationlib.get_annotations``. + +.. + +.. date: 2024-07-23-15-30-23 +.. gh-issue: 122170 +.. nonce: Z9gi3Y +.. section: Library + +Handle :exc:`ValueError`\s raised by :func:`os.stat` in :mod:`linecache`. +Patch by Bénédikt Tran. + +.. + +.. date: 2024-07-23-15-11-13 +.. gh-issue: 122163 +.. nonce: 4wRUuM +.. section: Library + +Add notes for JSON serialization errors that allow to identify the source of +the error. + +.. + +.. date: 2024-07-23-13-07-12 +.. gh-issue: 122129 +.. nonce: PwbC8q +.. section: Library + +Improve support of method descriptors and wrappers in the help title. + +.. + +.. date: 2024-07-23-12-38-14 +.. gh-issue: 122145 +.. nonce: sTO8nX +.. section: Library + +Fix an issue when reporting tracebacks corresponding to Python code emitting +an empty AST body. Patch by Nikita Sobolev and Bénédikt Tran. + +.. + +.. date: 2024-07-23-10-59-38 +.. gh-issue: 121723 +.. nonce: iJEf7e +.. section: Library + +Make :func:`logging.config.dictConfig` accept any object implementing the +Queue public API. See the :ref:`queue configuration ` +section for details. Patch by Bénédikt Tran. + +.. + +.. date: 2024-07-23-09-14-44 +.. gh-issue: 82951 +.. nonce: -F5p5A +.. section: Library + +Serializing objects with complex ``__qualname__`` (such as unbound methods +and nested classes) by name no longer involves serializing parent objects by +value in pickle protocols < 4. + +.. + +.. date: 2024-07-22-08-57-28 +.. gh-issue: 120754 +.. nonce: Eo5puP +.. section: Library + +``Pathlib.read_bytes`` no longer opens the file in Python's buffered I/O +mode. This reduces overheads as the code reads a file in whole leading to a +modest speedup. + +.. + +.. date: 2024-07-22-08-14-04 +.. gh-issue: 113785 +.. nonce: 6B_KNB +.. section: Library + +:mod:`csv` now correctly parses numeric fields (when used with +:const:`csv.QUOTE_NONNUMERIC` or :const:`csv.QUOTE_STRINGS`) which start +with an escape character. + +.. + +.. date: 2024-07-21-18-03-30 +.. gh-issue: 122088 +.. nonce: vi2bP- +.. section: Library + +:func:`@warnings.deprecated ` now copies the coroutine +status of functions and methods so that :func:`inspect.iscoroutinefunction` +returns the correct result. + +.. + +.. date: 2024-07-21-10-45-24 +.. gh-issue: 122081 +.. nonce: dNrYMq +.. section: Library + +Fix a crash in the :func:`!decimal.IEEEContext` optional function available +via the ``EXTRA_FUNCTIONALITY`` configuration flag. + +.. + +.. date: 2024-07-21-02-00-46 +.. gh-issue: 73991 +.. nonce: pLxdtJ +.. section: Library + +Add :meth:`pathlib.Path.move`, which moves a file or directory tree. + +.. + +.. date: 2024-07-17-12-55-22 +.. gh-issue: 121268 +.. nonce: 41RmjR +.. section: Library + +Remove workarounds for non-IEEE 754 systems in :mod:`cmath`. + +.. + +.. date: 2024-07-17-09-44-35 +.. gh-issue: 119698 +.. nonce: WlygzR +.. section: Library + +Due to the lack of interest for :meth:`symtable.Class.get_methods`, the +method is marked as deprecated and will be removed in Python 3.16. Patch by +Bénédikt Tran. + +.. + +.. date: 2024-07-17-09-23-03 +.. gh-issue: 121889 +.. nonce: 6se9jS +.. section: Library + +Adjusts ``cmath.acosh(complex('0+nanj'))`` for recent C standards. + +.. + +.. date: 2024-07-16-20-49-07 +.. gh-issue: 121804 +.. nonce: gYN-In +.. section: Library + +Correctly show error locations, when :exc:`SyntaxError` raised in new repl. +Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-07-15-19-34-56 +.. gh-issue: 121797 +.. nonce: qDqj59 +.. section: Library + +Add alternative :class:`~fractions.Fraction` constructor +:meth:`Fraction.from_number() `. + +.. + +.. date: 2024-07-15-19-25-25 +.. gh-issue: 121798 +.. nonce: GmuBDu +.. section: Library + +Add alternative :class:`~decimal.Decimal` constructor +:meth:`Decimal.from_number() `. + +.. + +.. date: 2024-07-14-11-18-28 +.. gh-issue: 120930 +.. nonce: Kuo4L0 +.. section: Library + +Fixed a bug introduced by gh-92081 that added an incorrect extra blank to +encoded words occurring in wrapped headers. + +.. + +.. date: 2024-07-14-06-24-02 +.. gh-issue: 57141 +.. nonce: C3jhDh +.. section: Library + +The *shallow* argument to :class:`filecmp.dircmp` (new in Python 3.13) is +now keyword-only. + +.. + +.. date: 2024-07-13-06-23-24 +.. gh-issue: 121245 +.. nonce: RfOgf4 +.. section: Library + +Simplify handling of the history file in ``site.register_readline()`` +helper. The ``CAN_USE_PYREPL`` variable now will be initialized, when +imported. Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-07-10-08-13-34 +.. gh-issue: 121249 +.. nonce: W9Gd09 +.. section: Library + +Support the :c:expr:`float complex` and :c:expr:`double complex` C types in +the :mod:`struct` module if the compiler has C11 complex arithmetic. Patch +by Sergey B Kirpichev. + +.. + +.. date: 2024-07-09-12-23-32 +.. gh-issue: 121486 +.. nonce: Iultjh +.. section: Library + +:mod:`math` functions :func:`~math.isqrt`, :func:`~math.log`, +:func:`~math.log2` and :func:`~math.log10` now support integers larger than +``2**2**32`` on 32-bit platforms. + +.. + +.. date: 2024-07-08-03-45-34 +.. gh-issue: 121474 +.. nonce: NsvrUN +.. section: Library + +Fix missing sanity check for ``parties`` arg in :class:`threading.Barrier` +constructor. Patch by Clinton Christian (pygeek). + +.. + +.. date: 2024-07-06-23-39-38 +.. gh-issue: 121450 +.. nonce: vGqb3c +.. section: Library + +Hard-coded breakpoints (:func:`breakpoint` and :func:`pdb.set_trace`) now +reuse the most recent ``Pdb`` instance that calls ``Pdb.set_trace()``, +instead of creating a new one each time. As a result, all the instance +specific data like ``display`` and ``commands`` are preserved across +Hard-coded breakpoints. + +.. + +.. date: 2024-07-06-16-08-39 +.. gh-issue: 119169 +.. nonce: o0YymL +.. section: Library + +Slightly speed up :func:`os.walk` by simplifying exception handling. + +.. + +.. date: 2024-07-06-12-37-10 +.. gh-issue: 121423 +.. nonce: vnxrl4 +.. section: Library + +Improve import time of :mod:`socket` by lazy importing modules and writing +:data:`!socket.errorTab` as a constant. + +.. + +.. date: 2024-07-04-17-36-03 +.. gh-issue: 59110 +.. nonce: IlI9Fz +.. section: Library + +:mod:`zipimport` supports now namespace packages when no directory entry +exists. + +.. + +.. date: 2024-07-03-14-23-04 +.. gh-issue: 119004 +.. nonce: L5MoUu +.. section: Library + +Fix a crash in :ref:`OrderedDict.__eq__ ` +when operands are mutated during the check. Patch by Bénédikt Tran. + +.. + +.. date: 2024-07-03-10-11-53 +.. gh-issue: 121313 +.. nonce: D7gARW +.. section: Library + +Limit the reading size in the :class:`multiprocessing.connection.Connection` +class to 64 KiB to prevent memory overallocation and unnecessary memory +management system calls. + +.. + +.. date: 2024-07-03-07-25-21 +.. gh-issue: 121332 +.. nonce: Iz6FEq +.. section: Library + +Fix constructor of :mod:`ast` nodes with custom ``_attributes``. Previously, +passing custom attributes would raise a :py:exc:`DeprecationWarning`. +Passing arguments to the constructor that are not in ``_fields`` or +``_attributes`` remains deprecated. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-07-02-11-34-06 +.. gh-issue: 121245 +.. nonce: sSkDAr +.. section: Library + +Fix a bug in the handling of the command history of the new :term:`REPL` +that caused the history file to be wiped at REPL exit. + +.. + +.. date: 2024-07-01-11-23-18 +.. gh-issue: 121210 +.. nonce: cD0zfn +.. section: Library + +Handle AST nodes with missing runtime fields or attributes in +:func:`ast.compare`. Patch by Bénédikt Tran. + +.. + +.. date: 2024-06-29-19-30-15 +.. gh-issue: 121163 +.. nonce: SJKDFq +.. section: Library + +Add support for ``all`` as an valid ``action`` for +:func:`warnings.simplefilter` and :func:`warnings.filterwarnings`. + +.. + +.. date: 2024-06-29-15-23-26 +.. gh-issue: 121151 +.. nonce: HeLEvq +.. section: Library + +Fix wrapping of long usage text of arguments inside a mutually exclusive +group in :mod:`argparse`. + +.. + +.. date: 2024-06-29-15-21-12 +.. gh-issue: 121141 +.. nonce: 4evD6q +.. section: Library + +Add support for :func:`copy.replace` to AST nodes. Patch by Bénédikt Tran. + +.. + +.. date: 2024-06-29-05-08-59 +.. gh-issue: 87744 +.. nonce: rpF6Jw +.. section: Library + +Fix waitpid race while calling +:meth:`~asyncio.subprocess.Process.send_signal` in asyncio. Patch by Kumar +Aditya. + +.. + +.. date: 2024-06-27-13-47-14 +.. gh-issue: 121027 +.. nonce: jh55EC +.. section: Library + +Add a future warning in :meth:`!functools.partial.__get__`. In future Python +versions :class:`functools.partial` will be a method descriptor. + +.. + +.. date: 2024-06-27-12-27-52 +.. gh-issue: 121027 +.. nonce: D4K1OX +.. section: Library + +Make the :class:`functools.partial` object a method descriptor. + +.. + +.. date: 2024-06-26-17-00-39 +.. gh-issue: 117784 +.. nonce: inCtAV +.. section: Library + +CPython now detects whether its linked TLS library supports TLSv1.3 +post-handshake authentication and disables that feature if support is +lacking. + +.. + +.. date: 2024-06-26-10-13-40 +.. gh-issue: 121025 +.. nonce: M-XXlV +.. section: Library + +Improve the :meth:`~object.__repr__` of :class:`functools.partialmethod`. +Patch by Bénédikt Tran. + +.. + +.. date: 2024-06-26-03-04-24 +.. gh-issue: 121018 +.. nonce: clVSc4 +.. section: Library + +Fixed issues where :meth:`!argparse.ArgumentParser.parse_args` did not honor +``exit_on_error=False``. Based on patch by Ben Hsing. + +.. + +.. date: 2024-06-23-17-50-40 +.. gh-issue: 119614 +.. nonce: vwPGLB +.. section: Library + +Fix truncation of strings with embedded null characters in some internal +operations in :mod:`tkinter`. + +.. + +.. date: 2024-06-23-11-21-27 +.. gh-issue: 120910 +.. nonce: t0QXdB +.. section: Library + +When reading installed files from an egg, use ``relative_to(walk_up=True)`` +to honor files installed outside of the installation root. + +.. + +.. date: 2024-06-23-07-23-08 +.. gh-issue: 61103 +.. nonce: ca_U_l +.. section: Library + +Support :c:expr:`float complex`, :c:expr:`double complex` and :c:expr:`long +double complex` C types in :mod:`ctypes` as +:class:`~ctypes.c_float_complex`, :class:`~ctypes.c_double_complex` and +:class:`~ctypes.c_longdouble_complex` if the compiler has C11 complex +arithmetic. Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-06-22-22-52-24 +.. gh-issue: 120888 +.. nonce: sd8I3N +.. section: Library + +Upgrade pip wheel bundled with ensurepip (pip 24.1.1) + +.. + +.. date: 2024-06-22-22-23-56 +.. gh-issue: 101830 +.. nonce: 1BAoxH +.. section: Library + +Accessing the :mod:`tkinter` object's string representation no longer +converts the underlying Tcl object to a string on Windows. + +.. + +.. date: 2024-06-22-17-01-56 +.. gh-issue: 120678 +.. nonce: Ik8dCg +.. section: Library + +Fix regression in the new REPL that meant that globals from files passed +using the ``-i`` argument would not be included in the REPL's global +namespace. Patch by Alex Waygood. + +.. + +.. date: 2024-06-21-14-32-56 +.. gh-issue: 120811 +.. nonce: eBmVTV +.. section: Library + +Fix possible memory leak in :meth:`contextvars.Context.run`. + +.. + +.. date: 2024-06-21-12-00-16 +.. gh-issue: 120782 +.. nonce: LOE8tj +.. section: Library + +Fix wrong references of the :mod:`datetime` types after reloading the +module. + +.. + +.. date: 2024-06-21-06-37-46 +.. gh-issue: 120713 +.. nonce: WBbQx4 +.. section: Library + +:meth:`datetime.datetime.strftime` now 0-pads years with less than four +digits for the format specifiers ``%Y`` and ``%G`` on Linux. Patch by Ben +Hsing + +.. + +.. date: 2024-06-20-01-31-24 +.. gh-issue: 120769 +.. nonce: PfiMrc +.. section: Library + +Make empty line in :mod:`pdb` repeats the last command even when the command +is from ``cmdqueue``. + +.. + +.. date: 2024-06-19-23-08-25 +.. gh-issue: 120780 +.. nonce: 0Omopb +.. section: Library + +Show string value of LOAD_SPECIAL oparg in :mod:`dis` output. + +.. + +.. date: 2024-06-19-19-53-42 +.. gh-issue: 41431 +.. nonce: gnkUc5 +.. section: Library + +Add :meth:`datetime.time.strptime` and :meth:`datetime.date.strptime`. +Contributed by Wannes Boeykens. + +.. + +.. date: 2024-06-19-15-43-04 +.. gh-issue: 120743 +.. nonce: CMMl2P +.. section: Library + +:term:`Soft deprecate ` :func:`os.popen` and +:func:`os.spawn* ` functions. They should no longer be used to +write new code. The :mod:`subprocess` module is recommended instead. Patch +by Victor Stinner. + +.. + +.. date: 2024-06-19-15-06-58 +.. gh-issue: 120732 +.. nonce: OvYV9b +.. section: Library + +Fix ``name`` passing to :class:`unittest.mock.Mock` object when using +:func:`unittest.mock.create_autospec`. + +.. + +.. date: 2024-06-19-13-20-01 +.. gh-issue: 111259 +.. nonce: Wki5PV +.. section: Library + +:mod:`re` now handles patterns like ``"[\s\S]"`` or ``"\s|\S"`` which match +any character as effectively as a dot with the ``DOTALL`` modifier +(``"(?s:.)"``). + +.. + +.. date: 2024-06-18-19-18-10 +.. gh-issue: 120683 +.. nonce: xmRez7 +.. section: Library + +Fix an error in :class:`logging.LogRecord`, when the integer part of the +timestamp is rounded up, while the millisecond calculation truncates, +causing the log timestamp to be wrong by up to 999 ms (affected roughly 1 in +8 million timestamps). + +.. + +.. date: 2024-06-18-14-45-38 +.. gh-issue: 118710 +.. nonce: 5GZZPX +.. section: Library + +:class:`ipaddress.IPv4Address` and :class:`ipaddress.IPv6Address` attributes +``version`` and ``max_prefixlen`` are now available on the class. + +.. + +.. date: 2024-06-17-20-04-13 +.. gh-issue: 120633 +.. nonce: kZC5wt +.. section: Library + +Move scrollbar and remove tear-off menus in turtledemo. + +.. + +.. date: 2024-06-16-21-33-56 +.. gh-issue: 120606 +.. nonce: kugbwR +.. section: Library + +Allow users to use EOF to exit ``commands`` definition in :mod:`pdb` + +.. + +.. date: 2024-06-15-23-38-36 +.. gh-issue: 120284 +.. nonce: HwsAtY +.. section: Library + +Allow :meth:`asyncio.Runner.run` to accept :term:`awaitable` objects instead +of simply :term:`coroutine`\s. + +.. + +.. date: 2024-06-15-12-04-46 +.. gh-issue: 120541 +.. nonce: d3cc5y +.. section: Library + +Improve the prompt in the "less" pager when :func:`help` is called with +non-string argument. + +.. + +.. date: 2024-06-14-20-05-25 +.. gh-issue: 120495 +.. nonce: OxgZKB +.. section: Library + +Fix incorrect exception handling in Tab Nanny. Patch by Wulian233. + +.. + +.. date: 2024-06-12-15-07-58 +.. gh-issue: 120388 +.. nonce: VuTQMT +.. section: Library + +Improve a warning message when a test method in :mod:`unittest` returns +something other than ``None``. Now we show the returned object type and +optional asyncio-related tip. + +.. + +.. date: 2024-06-12-11-54-05 +.. gh-issue: 120381 +.. nonce: O-BNLs +.. section: Library + +Correct :func:`inspect.ismethoddescriptor` to check also for the lack of +:meth:`~object.__delete__`. Patch by Jan Kaliszewski. + +.. + +.. date: 2024-06-12-10-00-31 +.. gh-issue: 90425 +.. nonce: 5CfkKG +.. section: Library + +The OS byte in gzip headers is now always set to 255 when using +:func:`gzip.compress`. + +.. + +.. date: 2024-06-11-16-34-41 +.. gh-issue: 120343 +.. nonce: hdiXeU +.. section: Library + +Fix column offset reporting for tokens that come after multiline f-strings +in the :mod:`tokenize` module. + +.. + +.. date: 2024-06-11-07-17-25 +.. gh-issue: 119180 +.. nonce: iH-2zy +.. section: Library + +As part of implementing :pep:`649` and :pep:`749`, add a new module +``annotationlib``. Add support for unresolved forward references in +annotations to :mod:`dataclasses`, :class:`typing.TypedDict`, and +:class:`typing.NamedTuple`. + +.. + +.. date: 2024-06-10-14-00-40 +.. gh-issue: 119600 +.. nonce: jJMf4C +.. section: Library + +Fix :func:`unittest.mock.patch` to not read attributes of the target when +``new_callable`` is set. Patch by Robert Collins. + +.. + +.. date: 2024-06-09-19-53-11 +.. gh-issue: 120289 +.. nonce: s4HXR0 +.. section: Library + +Fixed the use-after-free issue in :mod:`cProfile` by disallowing +``disable()`` and ``clear()`` in external timers. + +.. + +.. date: 2024-06-08-17-41-11 +.. gh-issue: 82017 +.. nonce: WpSTGi +.. section: Library + +Added support for converting any objects that have the +:meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`. + +.. + +.. date: 2024-06-08-15-46-35 +.. gh-issue: 114053 +.. nonce: Ub2XgJ +.. section: Library + +Fix edge-case bug where :func:`typing.get_type_hints` would produce +incorrect results if type parameters in a class scope were overridden by +assignments in a class scope and ``from __future__ import annotations`` +semantics were enabled. Patch by Alex Waygood. + +.. + +.. date: 2024-06-08-15-15-29 +.. gh-issue: 114053 +.. nonce: WQLAFG +.. section: Library + +Fix erroneous :exc:`NameError` when calling :func:`inspect.get_annotations` +with ``eval_str=True``` on a class that made use of :pep:`695` type +parameters in a module that had ``from __future__ import annotations`` at +the top of the file. Patch by Alex Waygood. + +.. + +.. date: 2024-06-08-14-36-40 +.. gh-issue: 120268 +.. nonce: MNpd1q +.. section: Library + +Prohibit passing ``None`` to pure-Python :meth:`datetime.date.fromtimestamp` +to achieve consistency with C-extension implementation. + +.. + +.. date: 2024-06-08-09-45-31 +.. gh-issue: 120244 +.. nonce: 8o9Dzr +.. section: Library + +Fix memory leak in :func:`re.sub` when the replacement string contains +backreferences. + +.. + +.. date: 2024-06-08-03-29-01 +.. gh-issue: 120254 +.. nonce: h682ke +.. section: Library + +Added ``commands`` argument to :func:`pdb.set_trace` which allows users to +send debugger commands from the source file. + +.. + +.. date: 2024-06-07-13-21-11 +.. gh-issue: 120211 +.. nonce: Rws_gf +.. section: Library + +Fix :mod:`tkinter.ttk` with Tcl/Tk 9.0. + +.. + +.. date: 2024-06-07-11-23-31 +.. gh-issue: 71587 +.. nonce: IjFajE +.. section: Library + +Fix crash in C version of :meth:`datetime.datetime.strptime` when called +again on the restarted interpreter. + +.. + +.. date: 2024-06-07-10-10-32 +.. gh-issue: 117983 +.. nonce: NeMR9n +.. section: Library + +Defer the ``threading`` import in ``importlib.util`` until lazy loading is +used. + +.. + +.. date: 2024-06-07-02-00-31 +.. gh-issue: 120157 +.. nonce: HnWcF9 +.. section: Library + +Remove unused constant ``concurrent.futures._base._FUTURE_STATES`` in +:mod:`concurrent.futures`. Patch by Clinton Christian (pygeek). + +.. + +.. date: 2024-06-06-17-24-43 +.. gh-issue: 120161 +.. nonce: DahNXV +.. section: Library + +:mod:`datetime` no longer crashes in certain complex reference cycle +situations. + +.. + +.. date: 2024-06-06-12-07-57 +.. gh-issue: 119698 +.. nonce: rRrprk +.. section: Library + +Fix :meth:`symtable.Class.get_methods` and document its behaviour. Patch by +Bénédikt Tran. + +.. + +.. date: 2024-06-05-16-30-28 +.. gh-issue: 120121 +.. nonce: 9dz8i7 +.. section: Library + +Add :exc:`concurrent.futures.InvalidStateError` to module's ``__all__``. + +.. + +.. date: 2024-06-05-11-39-21 +.. gh-issue: 119933 +.. nonce: ooJXQV +.. section: Library + +Add the :class:`symtable.SymbolTableType` enumeration to represent the +possible outputs of the :class:`symtable.SymbolTable.get_type` method. Patch +by Bénédikt Tran. + +.. + +.. date: 2024-06-05-11-03-10 +.. gh-issue: 120029 +.. nonce: QBsw47 +.. section: Library + +Expose :class:`symtable.Symbol` methods +:meth:`~symtable.Symbol.is_free_class`, +:meth:`~symtable.Symbol.is_comp_iter` and +:meth:`~symtable.Symbol.is_comp_cell`. Patch by Bénédikt Tran. + +.. + +.. date: 2024-06-05-08-02-46 +.. gh-issue: 120108 +.. nonce: 4U9BL8 +.. section: Library + +Fix calling :func:`copy.deepcopy` on :mod:`ast` trees that have been +modified to have references to parent nodes. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-06-04-19-49-16 +.. gh-issue: 120056 +.. nonce: 5aqozw +.. section: Library + +Add :data:`!socket.IP_RECVERR` and :data:`!socket.IP_RECVTTL` constants +(both available since Linux 2.2). And :data:`!socket.IP_RECVORIGDSTADDR` +constant (available since Linux 2.6.29). + +.. + +.. date: 2024-06-04-19-03-25 +.. gh-issue: 112672 +.. nonce: K2XfZH +.. section: Library + +Support building :mod:`tkinter` with Tcl 9.0. + +.. + +.. date: 2024-06-04-18-53-10 +.. gh-issue: 120057 +.. nonce: RSD9_Z +.. section: Library + +Added the :data:`os.environ.refresh() ` method to update +:data:`os.environ` with changes to the environment made by +:func:`os.putenv`, by :func:`os.unsetenv`, or made outside Python in the +same process. Patch by Victor Stinner. + +.. + +.. date: 2024-06-04-14-54-46 +.. gh-issue: 120029 +.. nonce: _1YdTf +.. section: Library + +Expose :meth:`symtable.Symbol.is_type_parameter` in the :mod:`symtable` +module. Patch by Bénédikt Tran. + +.. + +.. date: 2024-06-04-12-23-01 +.. gh-issue: 119819 +.. nonce: WKKrYh +.. section: Library + +Fix regression to allow logging configuration with multiprocessing queue +types. + +.. + +.. date: 2024-06-04-08-57-02 +.. gh-issue: 65454 +.. nonce: o9j4wF +.. section: Library + +:func:`unittest.mock.Mock.attach_mock` no longer triggers a call to a +``PropertyMock`` being attached. + +.. + +.. date: 2024-06-03-11-18-16 +.. gh-issue: 117142 +.. nonce: kWTXQo +.. section: Library + +The :mod:`ctypes` module may now be imported in all subinterpreters, +including those that have their own GIL. + +.. + +.. date: 2024-06-02-15-09-17 +.. gh-issue: 118835 +.. nonce: KUAuz6 +.. section: Library + +Fix _pyrepl crash when using custom prompt with ANSI escape codes. + +.. + +.. date: 2024-06-02-13-35-11 +.. gh-issue: 81936 +.. nonce: ETeW9x +.. section: Library + +:meth:`!help` and :meth:`!showtopic` methods now respect a configured +*output* argument to :class:`!pydoc.Helper` and not use the pager in such +cases. Patch by Enrico Tröger. + +.. + +.. date: 2024-06-01-16-58-43 +.. gh-issue: 117398 +.. nonce: kR0RW7 +.. section: Library + +The ``_datetime`` module (C implementation for :mod:`datetime`) now supports +being imported in multiple interpreters. + +.. + +.. date: 2024-05-31-21-17-43 +.. gh-issue: 119824 +.. nonce: CQlxWV +.. section: Library + +Print stack entry in :mod:`pdb` when and only when user input is needed. + +.. + +.. date: 2024-05-31-13-56-21 +.. gh-issue: 119838 +.. nonce: H6XHlE +.. section: Library + +In mixed arithmetic operations with :class:`~fractions.Fraction` and +complex, the fraction is now converted to :class:`float` instead of +:class:`complex`. + +.. + +.. date: 2024-05-31-12-57-31 +.. gh-issue: 119770 +.. nonce: NCtels +.. section: Library + +Make :mod:`termios` ``ioctl()`` constants positive. Patch by Victor Stinner. + +.. + +.. date: 2024-05-30-21-37-05 +.. gh-issue: 89727 +.. nonce: D6S9ig +.. section: Library + +Fix issue with :func:`shutil.rmtree` where a :exc:`RecursionError` is raised +on deep directory trees. + +.. + +.. date: 2024-05-29-21-50-05 +.. gh-issue: 119577 +.. nonce: S3BlKJ +.. section: Library + +The :exc:`DeprecationWarning` emitted when testing the truth value of an +:class:`xml.etree.ElementTree.Element` now describes unconditionally +returning ``True`` in a future version rather than raising an exception in +Python 3.14. + +.. + +.. date: 2024-05-29-20-42-17 +.. gh-issue: 89727 +.. nonce: 5lPTTW +.. section: Library + +Partially fix issue with :func:`shutil.rmtree` where a :exc:`RecursionError` +is raised on deep directory trees. A recursion error is no longer raised +when :data:`!rmtree.avoids_symlink_attacks` is false. + +.. + +.. date: 2024-05-29-12-42-40 +.. gh-issue: 93963 +.. nonce: cb1oJS +.. section: Library + +Remove deprecated names from ``importlib.abc`` as found in +``importlib.resources.abc``. + +.. + +.. date: 2024-05-28-12-15-03 +.. gh-issue: 119118 +.. nonce: FMKz1F +.. section: Library + +Fix performance regression in the :mod:`tokenize` module by caching the +``line`` token attribute and calculating the column offset more efficiently. + +.. + +.. date: 2024-05-28-00-56-59 +.. gh-issue: 89727 +.. nonce: _bxoL3 +.. section: Library + +Fix issue with :func:`os.fwalk` where a :exc:`RecursionError` was raised on +deep directory trees by adjusting the implementation to be iterative instead +of recursive. + +.. + +.. date: 2024-05-26-22-22-51 +.. gh-issue: 119594 +.. nonce: fnQNM8 +.. section: Library + +If one calls pow(fractions.Fraction, x, module) with modulo not None, the +error message now says that the types are incompatible rather than saying +pow only takes 2 arguments. Patch by Wim Jeantine-Glenn and Mark Dickinson. + +.. + +.. date: 2024-05-26-21-28-11 +.. gh-issue: 119588 +.. nonce: wlLBK5 +.. section: Library + +``zipfile.Path.is_symlink`` now assesses if the given path is a symlink. + +.. + +.. date: 2024-05-25-20-20-42 +.. gh-issue: 119562 +.. nonce: DyplWc +.. section: Library + +Remove :class:`!ast.Num`, :class:`!ast.Str`, :class:`!ast.Bytes`, +:class:`!ast.NameConstant` and :class:`!ast.Ellipsis`. They had all emitted +deprecation warnings since Python 3.12. Patch by Alex Waygood. + +.. + +.. date: 2024-05-25-20-15-26 +.. gh-issue: 119555 +.. nonce: mvHbEL +.. section: Library + +Catch :exc:`SyntaxError` from :func:`compile` in the runsource() method of +the InteractiveColoredConsole. Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-05-25-10-40-38 +.. gh-issue: 118908 +.. nonce: XcZiq4 +.. section: Library + +Limit exposed globals from internal imports and definitions on new REPL +startup. Patch by Eugene Triguba and Pablo Galindo. + +.. + +.. date: 2024-05-25-07-25-07 +.. gh-issue: 117865 +.. nonce: 1A0Xpi +.. section: Library + +Improve the import time of the :mod:`ast` module by deferring the import of +:mod:`re`. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-05-25-00-54-26 +.. gh-issue: 119127 +.. nonce: LpPvag +.. section: Library + +Positional arguments of :func:`functools.partial` objects now support +placeholders via :data:`functools.Placeholder`. + +.. + +.. date: 2024-05-24-21-54-55 +.. gh-issue: 113892 +.. nonce: JKDFqq +.. section: Library + +Now, the method ``sock_connect`` of :class:`asyncio.ProactorEventLoop` +raises a :exc:`ValueError` if given socket is not in non-blocking mode, as +well as in other loop implementations. + +.. + +.. date: 2024-05-24-14-32-24 +.. gh-issue: 119506 +.. nonce: -nMNqq +.. section: Library + +Fix :meth:`!io.TextIOWrapper.write` method breaks internal buffer when the +method is called again during flushing internal buffer. + +.. + +.. date: 2024-05-24-11-47-08 +.. gh-issue: 69214 +.. nonce: Grl6zF +.. section: Library + +Fix ``fcntl.ioctl()`` *request* parameter: use an ``unsigned long`` instead +of an ``unsigned int`` for the *request* parameter of :func:`fcntl.ioctl` to +support requests larger than ``UINT_MAX``. Patch by Victor Stinner. + +.. + +.. date: 2024-05-24-04-05-37 +.. gh-issue: 119105 +.. nonce: aDSRFn +.. section: Library + +``difflib``'s ``DIffer.compare()`` (and so also ``ndiff``) can no longer be +provoked into cubic-time behavior, or into unbounded recursion, and should +generally be faster in ordinary cases too. Results may change in some cases, +although that should be rare. Correctness of diffs is not affected. Some +similar lines far apart may be reported as deleting one and adding the +other, where before they were displayed on adjacent output lines with markup +showing the intraline differences. + +.. + +.. date: 2024-05-23-22-29-59 +.. gh-issue: 119443 +.. nonce: KAGz6S +.. section: Library + +The interactive REPL no longer runs with ``from __future__ import +annotations`` enabled. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-05-23-15-48-17 +.. gh-issue: 119461 +.. nonce: 82KqUW +.. section: Library + +Add ``socket.VMADDR_CID_LOCAL`` constant. Patch by Victor Stinner. + +.. + +.. date: 2024-05-23-11-52-36 +.. gh-issue: 117398 +.. nonce: 2FG1Mk +.. section: Library + +Objects in the datetime C-API are now all statically allocated, which means +better memory safety, especially when the module is reloaded. This should be +transparent to users. + +.. + +.. date: 2024-05-22-21-20-43 +.. gh-issue: 118894 +.. nonce: xHdxR_ +.. section: Library + +:mod:`asyncio` REPL now has the same capabilities as PyREPL. + +.. + +.. date: 2024-05-21-23-39-22 +.. gh-issue: 118830 +.. nonce: YTqvEo +.. section: Library + +Bump :mod:`pickle` default protocol to ``5``. + +.. + +.. date: 2024-05-21-20-13-23 +.. gh-issue: 118911 +.. nonce: iG8nMq +.. section: Library + +In PyREPL, updated ``maybe-accept``'s logic so that if the user hits +:kbd:`Enter` twice, they are able to terminate the block even if there's +trailing whitespace. Also, now when the user hits arrow up, the cursor is on +the last functional line. This matches IPython's behavior. Patch by Aya +Elsayed. + +.. + +.. date: 2024-05-21-19-10-30 +.. gh-issue: 115225 +.. nonce: eRmfJH +.. section: Library + +Raise error on certain technically valid but pathological ISO 8601 strings +passed to :meth:`datetime.time.fromisoformat` that were previously parsed +incorrectly. + +.. + +.. date: 2024-05-20-20-30-57 +.. gh-issue: 111201 +.. nonce: DAA5lC +.. section: Library + +Remove dependency to :mod:`readline` from the new Python REPL. + +.. + +.. date: 2024-05-20-13-48-37 +.. gh-issue: 119189 +.. nonce: dhJVs5 +.. section: Library + +When using the ``**`` operator or :func:`pow` with +:class:`~fractions.Fraction` as the base and an exponent that is not +rational, a float, or a complex, the fraction is no longer converted to a +float. + +.. + +.. date: 2024-05-19-18-49-04 +.. gh-issue: 119174 +.. nonce: 5GTv7d +.. section: Library + +Fix high DPI causes turtledemo(turtle-graphics examples) windows blurry +Patch by Wulian233 and Terry Jan Reedy + +.. + +.. date: 2024-05-19-13-05-59 +.. gh-issue: 119121 +.. nonce: P1gnh1 +.. section: Library + +Fix a NameError happening in ``asyncio.staggered.staggered_race``. This +function is now tested. + +.. + +.. date: 2024-05-19-12-25-36 +.. gh-issue: 119105 +.. nonce: VcR4ig +.. section: Library + +``difflib.Differ`` is much faster for some cases of diffs where many pairs +of lines are equally similar. + +.. + +.. date: 2024-05-17-17-32-12 +.. gh-issue: 119113 +.. nonce: kEv1Ll +.. section: Library + +Fix issue where :meth:`pathlib.PurePath.with_suffix` didn't raise +:exc:`TypeError` when given ``None`` as a suffix. + +.. + +.. date: 2024-05-16-17-31-46 +.. gh-issue: 118643 +.. nonce: hAWH4C +.. section: Library + +Fix an AttributeError in the :mod:`email` module when re-fold a long address +list. Also fix more cases of incorrect encoding of the address separator in +the address list. + +.. + +.. date: 2024-05-15-01-36-08 +.. gh-issue: 73991 +.. nonce: CGknDf +.. section: Library + +Add :meth:`pathlib.Path.copy`, which copies a file or directory to another. + +.. + +.. date: 2024-05-12-21-38-42 +.. gh-issue: 58933 +.. nonce: 0kgU2l +.. section: Library + +Make :mod:`pdb` return to caller frame correctly when ``f_trace`` of the +caller frame is not set + +.. + +.. date: 2024-05-11-20-23-45 +.. gh-issue: 82805 +.. nonce: F9bz4J +.. section: Library + +Support single-dot file extensions in :attr:`pathlib.PurePath.suffix` and +related attributes and methods. For example, the +:attr:`~pathlib.PurePath.suffixes` of ``PurePath('foo.bar.')`` are now +``['.bar', '.']`` rather than ``[]``. This brings file extension splitting +in line with :func:`os.path.splitext`. + +.. + +.. date: 2024-05-10-22-59-01 +.. gh-issue: 118924 +.. nonce: 9nyvSH +.. section: Library + +Remove :data:`!version` and :data:`!version_info` from :mod:`sqlite3`. Patch +by Hugo van Kemenade. + +.. + +.. date: 2024-05-10-22-36-01 +.. gh-issue: 118928 +.. nonce: IW7Ukv +.. section: Library + +Disallow using a sequence of parameters with named placeholders in +:mod:`sqlite3` queries. Patch by Erlend E. Aasland. + +.. + +.. date: 2024-05-10-05-24-32 +.. gh-issue: 118895 +.. nonce: wUm5r2 +.. section: Library + +Setting attributes on :data:`typing.NoDefault` now raises +:exc:`AttributeError` instead of :exc:`TypeError`. + +.. + +.. date: 2024-05-09-21-36-11 +.. gh-issue: 118868 +.. nonce: uckxxP +.. section: Library + +Fixed issue where kwargs were no longer passed to the logging handler +QueueHandler + +.. + +.. date: 2024-05-09-12-33-25 +.. gh-issue: 118827 +.. nonce: JrzHz1 +.. section: Library + +Remove deprecated :class:`!Quoter` class from :mod:`urllib.parse`. It had +previously raised a :exc:`DeprecationWarning` since Python 3.11. Patch by +Nikita Sobolev. + +.. + +.. date: 2024-05-09-11-50-26 +.. gh-issue: 118824 +.. nonce: -jBJQC +.. section: Library + +Remove deprecated :func:`!pty.master_open` and :func:`!pty.slave_open`. Use +:func:`pty.openpty` instead. Patch by Nikita Sobolev. + +.. + +.. date: 2024-05-09-08-46-12 +.. gh-issue: 118851 +.. nonce: aPAoJw +.. section: Library + +``ctx`` arguments to the constructors of :mod:`ast` node classes now default +to :class:`ast.Load() `. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-05-09-02-43-37 +.. gh-issue: 101588 +.. nonce: 30bNAr +.. section: Library + +Remove copy, deepcopy, and pickle from itertools. These had previously +raised a DeprecationWarning since Python 3.12. + +.. + +.. date: 2024-05-09-01-05-52 +.. gh-issue: 118805 +.. nonce: N7dm07 +.. section: Library + +Remove *type*, *choices*, and *metavar* parameters of +:class:`!argparse.BooleanOptionalAction`. They were deprecated since Python +3.12. + +.. + +.. date: 2024-05-09-00-52-30 +.. gh-issue: 118803 +.. nonce: Wv3AvU +.. section: Library + +:class:`!typing.ByteString` and :class:`!collections.abc.ByteString` are +removed. They had previously raised a :exc:`DeprecationWarning` since Python +3.12. + +.. + +.. date: 2024-05-08-23-16-50 +.. gh-issue: 118798 +.. nonce: Q_ybqP +.. section: Library + +The *isdst* parameter has been removed from :func:`email.utils.localtime`. +Patch by Hugo van Kemenade. + +.. + +.. date: 2024-05-08-21-30-33 +.. gh-issue: 118760 +.. nonce: XvyMHn +.. section: Library + +Restore the default value of ``tkiter.wantobjects`` to ``1``. + +.. + +.. date: 2024-05-08-21-13-56 +.. gh-issue: 118760 +.. nonce: mdmH3T +.. section: Library + +Fix errors in calling Tkinter bindings on Windows. + +.. + +.. date: 2024-05-08-20-41-48 +.. gh-issue: 74033 +.. nonce: YebHZj +.. section: Library + +Drop support for passing keyword arguments to :class:`pathlib.Path`. + +.. + +.. date: 2024-05-08-19-47-34 +.. gh-issue: 101357 +.. nonce: e4R_9x +.. section: Library + +Suppress all :exc:`OSError` exceptions from :meth:`pathlib.Path.exists` and +``is_*()`` methods, rather than a selection of more common errors. The new +behaviour is consistent with :func:`os.path.exists`, :func:`os.path.isdir`, +etc. Use :meth:`pathlib.Path.stat` to retrieve the file status without +suppressing exceptions. + +.. + +.. date: 2024-05-08-18-59-19 +.. gh-issue: 78707 +.. nonce: _Lz1sw +.. section: Library + +Drop support for passing additional positional arguments to +:meth:`pathlib.PurePath.relative_to` and +:meth:`~pathlib.PurePath.is_relative_to`. + +.. + +.. date: 2024-05-08-18-33-07 +.. gh-issue: 118507 +.. nonce: OCQsAY +.. section: Library + +Fix :func:`os.path.isfile` on Windows for pipes. Speedup +:func:`os.path.isjunction` and :func:`os.path.lexists` on Windows with a +native implementation. + +.. + +.. date: 2024-05-08-09-21-49 +.. gh-issue: 118772 +.. nonce: c16E8X +.. section: Library + +Allow :class:`typing.TypeVar` instances without a default to follow +instances without a default in some cases. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-05-07-17-38-53 +.. gh-issue: 118714 +.. nonce: XXKpVZ +.. section: Library + +Allow ``restart`` in post-mortem debugging of :mod:`pdb`. Removed restart +message when the user quits pdb from post-mortem mode. + +.. + +.. date: 2024-05-06-17-39-52 +.. gh-issue: 118673 +.. nonce: sTXBit +.. section: Library + +Removed executable bits and shebang from stdlib modules. + +.. + +.. date: 2024-05-01-22-24-05 +.. gh-issue: 110863 +.. nonce: GjYBbq +.. section: Library + +:func:`os.path.realpath` now suppresses any :exc:`OSError` from +:func:`os.readlink` when *strict* mode is disabled (the default). + +.. + +.. date: 2024-04-28-19-51-00 +.. gh-issue: 118263 +.. nonce: Gaap3S +.. section: Library + +Speed up :func:`os.path.splitroot` & :func:`os.path.normpath` with a direct +C call. + +.. + +.. date: 2024-04-24-16-23-04 +.. gh-issue: 110190 +.. nonce: TGd5qx +.. section: Library + +Fix ctypes structs with array on SPARC by setting ``MAX_STRUCT_SIZE`` to 32 +in stgdict. Patch by Jakub Kulik + +.. + +.. date: 2024-04-19-14-59-53 +.. gh-issue: 118033 +.. nonce: amS4Gw +.. section: Library + +Fix :func:`dataclasses.dataclass` not creating a ``__weakref__`` slot when +subclassing :class:`typing.Generic`. + +.. + +.. date: 2024-04-19-05-58-50 +.. gh-issue: 117766 +.. nonce: J3xepp +.. section: Library + +Always use :func:`str` to print ``choices`` in :mod:`argparse`. + +.. + +.. date: 2024-03-19-21-41-31 +.. gh-issue: 106531 +.. nonce: Mgd--6 +.. section: Library + +In :mod:`importlib.resources`, sync with `importlib_resources 6.3.2 +`_, +including: ``MultiplexedPath`` now expects ``Traversable`` paths, +deprecating string arguments to ``MultiplexedPath``; Enabled support for +resources in namespace packages in zip files; Fixed ``NotADirectoryError`` +when calling files on a subdirectory of a namespace package. + +.. + +.. date: 2024-02-27-10-22-15 +.. gh-issue: 115937 +.. nonce: 0cVNur +.. section: Library + +Removed extra preprocessing for the ``__signature__`` attribute: the code +just check if it's a :class:`inspect.Signature` instance. Patch by Sergey B +Kirpichev. + +.. + +.. date: 2024-02-22-10-12-59 +.. gh-issue: 115808 +.. nonce: F2g2Ku +.. section: Library + +Add :func:`operator.is_none` and :func:`operator.is_not_none` functions. + +.. + +.. date: 2024-01-18-21-44-23 +.. gh-issue: 114264 +.. nonce: DBKn29 +.. section: Library + +Improve performance of :func:`copy.deepcopy` by adding a fast path for +atomic types. + +.. + +.. date: 2024-01-14-11-43-31 +.. gh-issue: 113878 +.. nonce: dmEIN3 +.. section: Library + +Add *doc* parameter to :func:`dataclasses.field`, so it can be stored and +shown as a documentation / metadata. If ``@dataclass(slots=True)`` is used, +then the supplied string is availabl in the :attr:`~object.__slots__` dict. +Otherwise, the supplied string is only available in the corresponding +:class:`dataclasses.Field` object. + +In order to support this feature we are changing the ``__slots__`` format in +dataclasses from :class:`tuple` to :class:`dict` when documentation / +metadata is present. + +.. + +.. date: 2024-01-12-08-51-03 +.. gh-issue: 113978 +.. nonce: MqTgB0 +.. section: Library + +Ignore warnings on text completion inside REPL. + +.. + +.. date: 2023-12-14-13-43-27 +.. gh-issue: 113008 +.. nonce: jWYn8T +.. section: Library + +Correct argparse usage output for required, mutually exclusive groups +containing a positional argument + +.. + +.. date: 2023-12-12-15-19-58 +.. gh-issue: 108172 +.. nonce: KyDPuG +.. section: Library + +``webbrowser`` honors OS preferred browser on Linux when its desktop entry +name contains the text of a known browser name. + +.. + +.. date: 2023-09-19-17-56-24 +.. gh-issue: 109109 +.. nonce: WJvvX2 +.. section: Library + +You can now get the raw TLS certificate chains from TLS connections via +:meth:`ssl.SSLSocket.get_verified_chain` and +:meth:`ssl.SSLSocket.get_unverified_chain` methods. + +Contributed by Mateusz Nowak. + +.. + +.. date: 2023-06-17-09-07-06 +.. gh-issue: 105623 +.. nonce: 5G06od +.. section: Library + +Fix performance degradation in +:class:`logging.handlers.RotatingFileHandler`. Patch by Craig Robson. + +.. + +.. date: 2023-06-16-14-52-00 +.. gh-issue: 102450 +.. nonce: MfeR6A +.. section: Library + +Add missing ISO-8601 24:00 alternative to midnight of next day to +:meth:`datetime.datetime.fromisoformat` and +:meth:`datetime.time.fromisoformat`. Patch by Izan "TizzySaurus" Robinson +(tizzysaurus@gmail.com) + +.. + +.. date: 2023-04-28-09-54-15 +.. gh-issue: 103956 +.. nonce: EyLDPS +.. section: Library + +Fix lack of newline characters in :mod:`trace` module output when line +tracing is enabled but source code line for current frame is not available. + +.. + +.. date: 2023-04-26-22-24-17 +.. gh-issue: 92081 +.. nonce: V8xMot +.. section: Library + +Fix missing spaces in email headers when the spaces are mixed with encoded +8-bit characters. + +.. + +.. date: 2023-04-24-05-34-23 +.. gh-issue: 103194 +.. nonce: GwBwWL +.. section: Library + +Prepare Tkinter for C API changes in Tcl 8.7/9.0 to avoid +:class:`!_tkinter.Tcl_Obj` being unexpectedly returned instead of +:class:`bool`, :class:`str`, :class:`bytearray`, or :class:`int`. + +.. + +.. date: 2023-04-10-00-04-37 +.. gh-issue: 87106 +.. nonce: UyBnPQ +.. section: Library + +Fixed handling in :meth:`inspect.Signature.bind` of keyword arguments having +the same name as positional-only arguments when a variadic keyword argument +(e.g. ``**kwargs``) is present. + +.. + +.. date: 2023-03-30-18-19-53 +.. gh-issue: 103134 +.. nonce: bHrn91 +.. section: Library + +Add additional methods to :ref:`proxy objects +` in the :mod:`!multiprocessing` module: + +* :meth:`!clear` and :meth:`!copy` for proxies of :class:`list` +* :meth:`~dict.fromkeys`, ``reversed(d)``, ``d | {}``, ``{} | d``, + ``d |= {'b': 2}`` for proxies of :class:`dict` + +.. + +.. bpo: 45767 +.. date: 2022-03-10-16-47-57 +.. nonce: ywmyo1 +.. section: Library + +Fix integer conversion in :func:`os.major`, :func:`os.minor`, and +:func:`os.makedev`. Support device numbers larger than ``2**63-1``. Support +non-existent device number (``NODEV``). + +.. + +.. bpo: 44864 +.. date: 2021-08-24-19-37-46 +.. nonce: KzxaDh +.. section: Library + +Do not translate user-provided strings in :class:`argparse.ArgumentParser`. + +.. + +.. bpo: 41843 +.. date: 2020-12-22-18-08-12 +.. nonce: q9Nh2r +.. section: Library + +Solaris now uses :func:`os.sendfile` fast-copy syscall for more efficient +:mod:`shutil` file copy related functions. + +.. + +.. bpo: 15987 +.. date: 2020-03-28-21-00-54 +.. nonce: aBL8XS +.. section: Library + +Implemented :func:`ast.compare` for comparing two ASTs. Patch by Batuhan +Taskaya with some help from Jeremy Hylton. + +.. + +.. date: 2019-08-27-01-16-50 +.. gh-issue: 67693 +.. nonce: 4NIAiy +.. section: Library + +Fix :func:`urllib.parse.urlunparse` and :func:`urllib.parse.urlunsplit` for +URIs with path starting with multiple slashes and no authority. Based on +patch by Ashwin Ramaswami. + +.. + +.. bpo: 14074 +.. date: 2018-12-04-07-36-27 +.. nonce: fMLKCu +.. section: Library + +Fix :mod:`argparse` metavar processing to allow positional arguments to have +a tuple metavar. + +.. + +.. date: 2024-10-04-15-34-34 +.. gh-issue: 122392 +.. nonce: V8K3w2 +.. section: IDLE + +Increase currently inadequate vertical spacing for the IDLE browsers (path, +module, and stack) on high-resolution monitors. + +.. + +.. date: 2024-09-21-23-12-18 +.. gh-issue: 112938 +.. nonce: OeiDru +.. section: IDLE + +Fix uninteruptable hang when Shell gets rapid continuous output. + +.. + +.. date: 2024-07-30-18-02-55 +.. gh-issue: 122482 +.. nonce: TerE0g +.. section: IDLE + +Change About IDLE to direct users to discuss.python.org instead of the now +unused idle-dev email and mailing list. + +.. + +.. date: 2024-07-16-16-57-03 +.. gh-issue: 78889 +.. nonce: U7ghFD +.. section: IDLE + +Stop Shell freezes by blocking user access to non-method sys.stdout.shell +attributes, which are all private. + +.. + +.. date: 2024-06-16-21-42-45 +.. gh-issue: 120083 +.. nonce: nczuyv +.. section: IDLE + +Add explicit black IDLE Hovertip foreground color needed for recent macOS. +Fixes Sonoma showing unreadable white on pale yellow. Patch by John +Riggles. + +.. + +.. date: 2024-06-05-14-54-24 +.. gh-issue: 120104 +.. nonce: j_thj4 +.. section: IDLE + +Fix padding in config and search dialog windows in IDLE. + +.. + +.. date: 2024-10-10-02-56-24 +.. gh-issue: 124872 +.. nonce: 0mDDOq +.. section: Documentation + +Added definitions for :term:`context`, :term:`current context`, and +:term:`context management protocol`, updated related definitions to be +consistent, and expanded the documentation for :class:`contextvars.Context`. + +.. + +.. date: 2024-10-07-00-31-17 +.. gh-issue: 125018 +.. nonce: yKnymn +.. section: Documentation + +The :mod:`importlib.metadata` documentation now includes semantic +cross-reference targets for the significant documented APIs. This means +intersphinx references like :func:`importlib.metadata.version` will now work +as expected. + +.. + +.. date: 2024-09-27-16-47-48 +.. gh-issue: 124720 +.. nonce: nVSTVb +.. section: Documentation + +Update "Using Python on a Mac" section of the "Python Setup and Usage" +document and include information on installing free-threading support. + +.. + +.. date: 2024-09-24-11-52-36 +.. gh-issue: 124457 +.. nonce: yrCjSV +.. section: Documentation + +Remove coverity scan from the CPython repo. It has not been used since 2020 +and is currently unmaintained. + +.. + +.. date: 2024-09-19-19-33-25 +.. gh-issue: 116622 +.. nonce: M65UZ6 +.. section: Documentation + +Add an Android platform guide, and flag modules not available on Android. + +.. + +.. date: 2024-09-11-16-52-08 +.. gh-issue: 123976 +.. nonce: jhOfNR +.. section: Documentation + +Refresh docs around custom providers. + +.. + +.. date: 2024-08-01-17-18-21 +.. gh-issue: 70870 +.. nonce: fZnBM9 +.. section: Documentation + +Clarified the dual usage of the term "free variable" (both the formal +meaning of any reference to names defined outside the local scope, and the +narrower pragmatic meaning of nonlocal variables named in ``co_freevars``). + +.. + +.. date: 2024-07-19-12-22-48 +.. gh-issue: 121277 +.. nonce: wF_zKd +.. section: Documentation + +Writers of CPython's documentation can now use ``next`` as the version for +the ``versionchanged``, ``versionadded``, ``deprecated`` directives. + +.. + +.. date: 2024-07-14-12-25-53 +.. gh-issue: 117765 +.. nonce: YFMOUv +.. section: Documentation + +Improved documentation for :func:`unittest.mock.patch.dict` + +.. + +.. date: 2024-07-14-11-48-10 +.. gh-issue: 121749 +.. nonce: nxHoTk +.. section: Documentation + +Fix documentation for :c:func:`PyModule_AddObjectRef`. + +.. + +.. date: 2024-06-05-12-36-18 +.. gh-issue: 120012 +.. nonce: f14DbQ +.. section: Documentation + +Clarify the behaviours of :meth:`multiprocessing.Queue.empty` and +:meth:`multiprocessing.SimpleQueue.empty` on closed queues. Patch by +Bénédikt Tran. + +.. + +.. date: 2024-06-03-22-06-26 +.. gh-issue: 119574 +.. nonce: Ik9kOO +.. section: Documentation + +Added some missing environment variables to the output of +:option:`--help-env`. + +.. + +.. bpo: 34008 +.. date: 2018-07-04-20-35-25 +.. nonce: bqecIb +.. section: Documentation + +The :c:func:`Py_Main` documentation moved from the "Very High Level API" +section to the "Initialization and Finalization" section. + +Also make it explicit that we expect ``Py_Main`` to typically be called +instead of ``Py_Initialize`` rather than after it (since ``Py_Main`` makes +its own call to ``Py_Initialize``). Document that calling both is supported +but is version dependent on which settings will be applied correctly. + +.. + +.. date: 2024-10-15-15-29-41 +.. gh-issue: 124375 +.. nonce: wNrWVa +.. section: Core and Builtins + +Fix a crash in the free threading build when the GC runs concurrently with a +new thread starting. + +.. + +.. date: 2024-10-10-14-47-13 +.. gh-issue: 125221 +.. nonce: nfSQzT +.. section: Core and Builtins + +Fix possible race condition when calling :meth:`~object.__reduce_ex__` for +the first time in the free threading build. + +.. + +.. date: 2024-10-10-12-04-56 +.. gh-issue: 125174 +.. nonce: _8h6T7 +.. section: Core and Builtins + +Make the handling of reference counts of immortal objects more robust. +Immortal objects with reference counts that deviate from their original +reference count by up to a billion (half a billion on 32 bit builds) are +still counted as immortal. + +.. + +.. date: 2024-10-07-23-33-18 +.. gh-issue: 125039 +.. nonce: MKTyNI +.. section: Core and Builtins + +Make ``this_instr`` and ``prev_instr`` const in cases generator. + +.. + +.. date: 2024-10-05-23-53-06 +.. gh-issue: 125008 +.. nonce: ETANpd +.. section: Core and Builtins + +Fix :func:`tokenize.untokenize` producing invalid syntax for double braces +preceded by certain escape characters. + +.. + +.. date: 2024-10-03-22-26-39 +.. gh-issue: 124871 +.. nonce: tAMF47 +.. section: Core and Builtins + +Fix compiler bug (in some versions of 3.13) where an assertion fails during +reachability analysis. + +.. + +.. date: 2024-10-03-14-39-41 +.. gh-issue: 123378 +.. nonce: dCxANf +.. section: Core and Builtins + +Fix a crash in the :meth:`~object.__str__` method of :exc:`UnicodeError` +objects when the :attr:`UnicodeError.start` and :attr:`UnicodeError.end` +values are invalid or out-of-range. Patch by Bénédikt Tran. + +.. + +.. date: 2024-09-30-16-39-37 +.. gh-issue: 118093 +.. nonce: J2A3gz +.. section: Core and Builtins + +Improve the experimental JIT compiler's ability to stay "on trace" when +encountering highly-biased branches. + +.. + +.. date: 2024-09-27-17-18-53 +.. gh-issue: 124642 +.. nonce: OCjhBJ +.. section: Core and Builtins + +Fixed scalability issue in free-threaded builds for lock-free reads from +dictionaries in multi-threaded scenarios + +.. + +.. date: 2024-09-26-18-21-06 +.. gh-issue: 116510 +.. nonce: FacUWO +.. section: Core and Builtins + +Fix a crash caused by immortal interned strings being shared between +sub-interpreters that use basic single-phase init. In that case, the string +can be used by an interpreter that outlives the interpreter that created and +interned it. For interpreters that share obmalloc state, also share the +interned dict with the main interpreter. + +.. + +.. date: 2024-09-26-17-55-34 +.. gh-issue: 116510 +.. nonce: dhn8w8 +.. section: Core and Builtins + +Fix a bug that can cause a crash when sub-interpreters use "basic" +single-phase extension modules. Shared objects could refer to PyGC_Head +nodes that had been freed as part of interpreter cleanup. + +.. + +.. date: 2024-09-26-13-25-01 +.. gh-issue: 119180 +.. nonce: k_JCX0 +.. section: Core and Builtins + +The ``__main__`` module no longer always contains an ``__annotations__`` +dictionary in its global namespace. + +.. + +.. date: 2024-09-26-12-19-13 +.. gh-issue: 124547 +.. nonce: P_SHfU +.. section: Core and Builtins + +When deallocating an object with inline values whose ``__dict__`` is still +live: if memory allocation for the inline values fails, clear the +dictionary. Prevents an interpreter crash. + +.. + +.. date: 2024-09-25-14-45-56 +.. gh-issue: 124513 +.. nonce: ywiXtr +.. section: Core and Builtins + +Fix a crash in FrameLocalsProxy constructor: check the number of arguments. +Patch by Victor Stinner. + +.. + +.. date: 2024-09-25-11-53-22 +.. gh-issue: 124442 +.. nonce: EXC1Ve +.. section: Core and Builtins + +Fix nondeterminism in compilation by sorting the value of +:attr:`~type.__static_attributes__`. Patch by kp2pml30. + +.. + +.. date: 2024-09-23-23-06-19 +.. gh-issue: 124285 +.. nonce: mahGTg +.. section: Core and Builtins + +Fix bug where ``bool(a)`` can be invoked more than once during the +evaluation of a compound boolean expression. + +.. + +.. date: 2024-09-23-15-23-14 +.. gh-issue: 123856 +.. nonce: yrgJ9m +.. section: Core and Builtins + +Fix PyREPL failure when a keyboard interrupt is triggered after using a +history search + +.. + +.. date: 2024-09-23-13-25-27 +.. gh-issue: 65961 +.. nonce: LDqXV2 +.. section: Core and Builtins + +Deprecate the setting and using ``__package__`` and ``__cached__``. + +.. + +.. date: 2024-09-19-16-57-34 +.. gh-issue: 119726 +.. nonce: DseseK +.. section: Core and Builtins + +The JIT now generates more efficient code for calls to C functions resulting +in up to 0.8% memory savings and 1.5% speed improvement on AArch64. Patch by +Diego Russo. + +.. + +.. date: 2024-09-19-13-17-31 +.. gh-issue: 122878 +.. nonce: 4iFpsB +.. section: Core and Builtins + +Use the ``pager`` binary, if available (e.g. on Debian and derivatives), to +display REPL ``help()``. + +.. + +.. date: 2024-09-17-22-06-01 +.. gh-issue: 124188 +.. nonce: aFqNAB +.. section: Core and Builtins + +Fix reading and decoding a line from the source file witn non-UTF-8 encoding +for syntax errors raised in the compiler. + +.. + +.. date: 2024-09-13-02-25-06 +.. gh-issue: 124027 +.. nonce: to_9DY +.. section: Core and Builtins + +Support ````, ````, and ```` keys in the Python +REPL when ``$TERM`` is set to ``vt100``. + +.. + +.. date: 2024-09-12-21-53-26 +.. gh-issue: 124022 +.. nonce: fQzUiW +.. section: Core and Builtins + +Fix bug where docstring is removed from classes in interactive mode. + +.. + +.. date: 2024-09-11-15-48-36 +.. gh-issue: 123958 +.. nonce: 5VW2r0 +.. section: Core and Builtins + +docstrings are now removed from the optimized AST in optimization level 2. + +.. + +.. date: 2024-09-10-20-25-00 +.. gh-issue: 123923 +.. nonce: A7uxqa +.. section: Core and Builtins + +The ``f_executable`` field in the internal :c:struct:`_PyInterpreterFrame` +struct now uses a tagged pointer. Profilers and debuggers that uses this +field should clear the least significant bit to recover the +:c:expr:`PyObject*` pointer. + +.. + +.. date: 2024-09-10-13-27-16 +.. gh-issue: 77894 +.. nonce: ZC-Olu +.. section: Core and Builtins + +Fix possible crash in the garbage collector when it tries to break a +reference loop containing a :class:`memoryview` object. Now a +:class:`!memoryview` object can only be cleared if there are no buffers that +refer it. + +.. + +.. date: 2024-09-06-19-23-44 +.. gh-issue: 120221 +.. nonce: giJEDT +.. section: Core and Builtins + +asyncio REPL is now again properly recognizing KeyboardInterrupts. Display +of exceptions raised in secondary threads is fixed. + +.. + +.. date: 2024-09-06-14-13-01 +.. gh-issue: 119310 +.. nonce: WQxyDF +.. section: Core and Builtins + +Allow the new interactive shell to read history files written with the +editline library that use unicode-escaped entries. Patch by aorcajo and +Łukasz Langa. + +.. + +.. date: 2024-09-03-13-34-35 +.. gh-issue: 123572 +.. nonce: uuqoYV +.. section: Core and Builtins + +Fix key mappings for various F-keys in Windows for the new REPL. Patch by +devdanzin + +.. + +.. date: 2024-09-02-20-39-10 +.. gh-issue: 123614 +.. nonce: 26TMHp +.. section: Core and Builtins + +Add :func:`turtle.save` to easily save Turtle drawings as PostScript files. +Patch by Marie Roald and Yngve Mardal Moe. + +.. + +.. date: 2024-09-02-20-36-45 +.. gh-issue: 123339 +.. nonce: QcmpSs +.. section: Core and Builtins + +Setting the :attr:`!__module__` attribute for a class now removes the +``__firstlineno__`` item from the type's dict, so they will no longer be +inconsistent. + +.. + +.. date: 2024-09-02-17-32-15 +.. gh-issue: 119034 +.. nonce: HYh5Vj +.. section: Core and Builtins + +Change ```` and ```` keys of the Python REPL to history +search forward/backward. Patch by Victor Stinner. + +.. + +.. date: 2024-09-01-12-08-39 +.. gh-issue: 123562 +.. nonce: aJPKVu +.. section: Core and Builtins + +Improve :exc:`SyntaxError` message for using ``case ... as ...`` with not a +name. + +.. + +.. date: 2024-09-01-00-02-05 +.. gh-issue: 123545 +.. nonce: 8nQNbL +.. section: Core and Builtins + +Fix a double decref in rare cases on experimental JIT builds. + +.. + +.. date: 2024-08-29-19-46-07 +.. gh-issue: 123484 +.. nonce: rjUn_F +.. section: Core and Builtins + +Fix ``_Py_DebugOffsets`` for long objects to be relative to the start of the +object rather than the start of a subobject. + +.. + +.. date: 2024-08-29-13-18-18 +.. gh-issue: 123446 +.. nonce: KWDrgq +.. section: Core and Builtins + +Fix empty function name in :exc:`TypeError` when builtin magic methods are +used without the required args. + +.. + +.. date: 2024-08-28-22-42-51 +.. gh-issue: 123440 +.. nonce: yOFB0N +.. section: Core and Builtins + +Improve :exc:`SyntaxError` message for using ``except as`` with not a name. + +.. + +.. date: 2024-08-27-21-44-14 +.. gh-issue: 116017 +.. nonce: ZY3yBY +.. section: Core and Builtins + +Improved JIT memory consumption by periodically freeing memory used by +infrequently-executed code. This change is especially likely to improve the +memory footprint of long-running programs. + +.. + +.. date: 2024-08-27-13-16-40 +.. gh-issue: 123344 +.. nonce: 56Or78 +.. section: Core and Builtins + +Add AST optimizations for type parameter defaults. + +.. + +.. date: 2024-08-26-00-58-26 +.. gh-issue: 123321 +.. nonce: ApxcnE +.. section: Core and Builtins + +Prevent Parser/myreadline race condition from segfaulting on multi-threaded +use. Patch by Bar Harel and Amit Wienner. + +.. + +.. date: 2024-08-25-18-27-49 +.. gh-issue: 123177 +.. nonce: yLuyqE +.. section: Core and Builtins + +Fix a bug causing stray prompts to appear in the middle of wrapped lines in +the new REPL. + +.. + +.. date: 2024-08-25-10-54-22 +.. gh-issue: 122982 +.. nonce: KLD91q +.. section: Core and Builtins + +Extend the deprecation period for bool inversion (``~``) by two years. + +.. + +.. date: 2024-08-23-21-20-34 +.. gh-issue: 123271 +.. nonce: xeVViR +.. section: Core and Builtins + +Make concurrent iterations over the same :func:`zip` iterator safe under +free-threading. + +.. + +.. date: 2024-08-23-18-31-10 +.. gh-issue: 123275 +.. nonce: DprIrj +.. section: Core and Builtins + +Support :option:`-X gil=1 <-X>` and :envvar:`PYTHON_GIL=1 ` on +non-free-threaded builds. + +.. + +.. date: 2024-08-23-15-59-54 +.. gh-issue: 123177 +.. nonce: OLcaC5 +.. section: Core and Builtins + +Deactivate line wrap in the Apple Terminal via a ANSI escape code. Patch by +Pablo Galindo + +.. + +.. date: 2024-08-23-13-08-27 +.. gh-issue: 123229 +.. nonce: aHm-dw +.. section: Core and Builtins + +Fix valgrind warning by initializing the f-string buffers to 0 in the +tokenizer. Patch by Pablo Galindo + +.. + +.. date: 2024-08-23-11-26-54 +.. gh-issue: 122298 +.. nonce: ZMyln4 +.. section: Core and Builtins + +Restore printout of GC stats when ``gc.set_debug(gc.DEBUG_STATS)`` is +called. This feature was accidentally removed when implementing incremental +GC. + +.. + +.. date: 2024-08-21-15-22-53 +.. gh-issue: 121804 +.. nonce: r5K3PS +.. section: Core and Builtins + +Correctly show error locations when a :exc:`SyntaxError` is raised in the +basic REPL. Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-08-21-08-53-00 +.. gh-issue: 115776 +.. nonce: 9A7Dv_ +.. section: Core and Builtins + +Enables inline values (Python's equivalent of hidden classes) on any class +who's instances are of a fixed size. + +.. + +.. date: 2024-08-20-12-29-52 +.. gh-issue: 123142 +.. nonce: 3PXiNb +.. section: Core and Builtins + +Fix too-wide source location in exception tracebacks coming from broken +iterables in comprehensions. + +.. + +.. date: 2024-08-20-11-09-16 +.. gh-issue: 123048 +.. nonce: 2TISpv +.. section: Core and Builtins + +Fix a bug where pattern matching code could emit a :opcode:`JUMP_FORWARD` +with no source location. + +.. + +.. date: 2024-08-19-15-13-13 +.. gh-issue: 118093 +.. nonce: dLZ8qS +.. section: Core and Builtins + +Break up ``CALL_ALLOC_AND_ENTER_INIT`` into micro-ops and relax requirement +for exact args, in order to increase the amount of code supported by tier 2. + +.. + +.. date: 2024-08-18-18-25-54 +.. gh-issue: 123123 +.. nonce: 0ZcaEB +.. section: Core and Builtins + +Fix displaying :exc:`SyntaxError` exceptions covering multiple lines. Patch +by Pablo Galindo + +.. + +.. date: 2024-08-17-17-26-25 +.. gh-issue: 123083 +.. nonce: 9xWLJ- +.. section: Core and Builtins + +Fix a potential use-after-free in ``STORE_ATTR_WITH_HINT``. + +.. + +.. date: 2024-08-15-19-28-43 +.. gh-issue: 123022 +.. nonce: m3EF9E +.. section: Core and Builtins + +Fix crash in free-threaded build when calling :c:func:`Py_Initialize` from a +non-main thread. + +.. + +.. date: 2024-08-14-11-38-56 +.. gh-issue: 118093 +.. nonce: 3BywDP +.. section: Core and Builtins + +Add three specializations for :opcode:`CALL_KW`: + +* :opcode:`!CALL_KW_PY` for calls to Python functions +* :opcode:`!CALL_KW_BOUND_METHOD` for calls to bound methods +* :opcode:`!CALL_KW_NON_PY` for all other calls + +.. + +.. date: 2024-08-12-11-45-47 +.. gh-issue: 122821 +.. nonce: WnAzTK +.. section: Core and Builtins + +Make sure that branches in :keyword:`while` statements have consistent +offsets for :mod:`sys.monitoring`. :keyword:`!while` statements are now +compiled with a simple jump at the end of the body, instead of duplicating +the test. + +.. + +.. date: 2024-08-12-11-19-37 +.. gh-issue: 122907 +.. nonce: q68096 +.. section: Core and Builtins + +Building with ``HAVE_DYNAMIC_LOADING`` now works as well as it did in 3.12. +Existing deficiences will be addressed separately. (See +https://github.com/python/cpython/issues/122950.) + +.. + +.. date: 2024-08-10-12-44-03 +.. gh-issue: 122888 +.. nonce: TUyu9r +.. section: Core and Builtins + +Fix crash on certain calls to ``str()`` with positional arguments of the +wrong type. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-08-08-16-02-28 +.. gh-issue: 118093 +.. nonce: m6Mrvy +.. section: Core and Builtins + +Improve the experimental JIT's handling of polymorphic code. + +.. + +.. date: 2024-08-05-19-28-12 +.. gh-issue: 122697 +.. nonce: 17MvYl +.. section: Core and Builtins + +Fixed memory leaks at interpreter shutdown in the free-threaded build, and +also reporting of leaked memory blocks via :option:`-X showrefcount <-X>`. + +.. + +.. date: 2024-08-05-19-04-06 +.. gh-issue: 116622 +.. nonce: 3LWUzE +.. section: Core and Builtins + +Fix Android stdout and stderr messages being truncated or lost. + +.. + +.. date: 2024-08-01-19-13-58 +.. gh-issue: 122527 +.. nonce: eztso6 +.. section: Core and Builtins + +Fix a crash that occurred when a ``PyStructSequence`` was deallocated after +its type's dictionary was cleared by the GC. The type's +:c:member:`~PyTypeObject.tp_basicsize` now accounts for non-sequence fields +that aren't included in the :c:macro:`Py_SIZE` of the sequence. + +.. + +.. date: 2024-07-30-11-41-35 +.. gh-issue: 122445 +.. nonce: Rq0bjS +.. section: Core and Builtins + +Add only fields which are modified via self.* to +:attr:`~type.__static_attributes__`. + +.. + +.. date: 2024-07-29-19-20-25 +.. gh-issue: 122417 +.. nonce: NVgs0a +.. section: Core and Builtins + +In the free-threaded build, the reference counts for heap type objects are +now partially stored in a distributed manner in per-thread arrays. This +reduces contention on the heap type's reference count fields when creating +or destroying instances of the same type from multiple threads concurrently. + +.. + +.. date: 2024-07-29-10-55-46 +.. gh-issue: 116090 +.. nonce: p1MhU0 +.. section: Core and Builtins + +Fix an issue in JIT builds that prevented some :keyword:`for` loops from +correctly firing :monitoring-event:`RAISE` monitoring events. + +.. + +.. date: 2024-07-26-14-05-51 +.. gh-issue: 122300 +.. nonce: SVIF-l +.. section: Core and Builtins + +Preserve AST nodes for f-string with single-element format specifiers. Patch +by Pablo Galindo + +.. + +.. date: 2024-07-26-13-56-32 +.. gh-issue: 120906 +.. nonce: qBh2I9 +.. section: Core and Builtins + +:attr:`frame.f_locals` now supports arbitrary hashable objects as keys. + +.. + +.. date: 2024-07-25-01-45-21 +.. gh-issue: 122239 +.. nonce: 7zh-sW +.. section: Core and Builtins + +When a :class:`list`, :class:`tuple` or :class:`dict` with too many elements +is unpacked, show the actual length in the error message. + +.. + +.. date: 2024-07-24-22-39-07 +.. gh-issue: 122245 +.. nonce: LVa9v8 +.. section: Core and Builtins + +Detection of writes to ``__debug__`` is moved from the compiler's codegen +stage to the symtable. This means that these errors are now detected even in +code that is optimized away before codegen (such as assertions with the +:option:`-O` command line option). + +.. + +.. date: 2024-07-24-17-11-51 +.. gh-issue: 122234 +.. nonce: VxsP_F +.. section: Core and Builtins + +Specializations for sums with float and complex inputs in :func:`sum` now +always use compensated summation. Also, for integer items in above +specializations: :c:func:`PyLong_AsDouble` is used, instead of +:c:func:`PyLong_AsLongAndOverflow`. Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-07-23-23-59-04 +.. gh-issue: 122208 +.. nonce: z8KHsY +.. section: Core and Builtins + +Dictionary watchers now only deliver the PyDict_EVENT_ADDED event when the +insertion is in a known good state to succeed. + +.. + +.. date: 2024-07-23-11-57-36 +.. gh-issue: 122160 +.. nonce: HSnrAP +.. section: Core and Builtins + +Remove the ``BUILD_CONST_KEY_MAP`` opcode. Use :opcode:`BUILD_MAP` instead. + +.. + +.. date: 2024-07-21-01-23-54 +.. gh-issue: 122029 +.. nonce: gKv-e2 +.. section: Core and Builtins + +Emit ``c_call`` events in :func:`sys.setprofile` when a ``PyMethodObject`` +pointing to a ``PyCFunction`` is called. + +.. + +.. date: 2024-07-19-15-28-05 +.. gh-issue: 122026 +.. nonce: sta2Ca +.. section: Core and Builtins + +Fix a bug that caused the tokenizer to not correctly identify mismatched +parentheses inside f-strings in some situations. Patch by Pablo Galindo + +.. + +.. date: 2024-07-19-09-38-01 +.. gh-issue: 99108 +.. nonce: qzM6gl +.. section: Core and Builtins + +Python's hashlib now unconditionally uses the vendored HACL* library for +Blake2. Python no longer accepts libb2 as an optional dependency for Blake2. + +We refreshed HACL* to the latest version, and now vendor HACL*'s 128-bit and +256-bit wide vector implementations for Blake2, which are used on x86/x64 +toolchains when the required CPU features are available at runtime. + +HACL*'s 128-bit wide vector implementation of Blake2 can also run on ARM +NEON and Power8, but lacking evidence of a performance gain, these are not +enabled (yet). + +.. + +.. date: 2024-07-18-21-19-04 +.. gh-issue: 121999 +.. nonce: 8IBbTK +.. section: Core and Builtins + +The default extraction filter for the :mod:`tarfile` module is now set to +:func:`'data' `. + +.. + +.. date: 2024-07-16-18-23-22 +.. gh-issue: 121860 +.. nonce: -FTauD +.. section: Core and Builtins + +Fix crash when rematerializing a managed dictionary after it was deleted. + +.. + +.. date: 2024-07-16-15-11-51 +.. gh-issue: 121795 +.. nonce: xkIHrI +.. section: Core and Builtins + +Improve performance of set membership testing, ``set.remove()`` and +``set.discard()`` when the argument is a set. + +.. + +.. date: 2024-07-15-20-41-06 +.. gh-issue: 121814 +.. nonce: oR2ixR +.. section: Core and Builtins + +Fixed the SegFault when :c:func:`PyEval_SetTrace` is used with no Python +frame on stack. + +.. + +.. date: 2024-07-15-20-03-29 +.. gh-issue: 121295 +.. nonce: w53ucI +.. section: Core and Builtins + +Fix PyREPL console getting into a blocked state after interrupting a long +paste + +.. + +.. date: 2024-07-15-16-26-32 +.. gh-issue: 121794 +.. nonce: fhBtiQ +.. section: Core and Builtins + +Fix bug in free-threaded Python where a resurrected object could lead to a +negative ref count assertion failure. + +.. + +.. date: 2024-07-13-12-27-31 +.. gh-issue: 121657 +.. nonce: wgOYLw +.. section: Core and Builtins + +Improve the :exc:`SyntaxError` message if the user tries to use +:keyword:`yield from ` outside a function. + +.. + +.. date: 2024-07-13-09-51-44 +.. gh-issue: 121609 +.. nonce: jWsE5t +.. section: Core and Builtins + +Fix pasting of characters containing unicode character joiners in the new +REPL. Patch by Marta Gomez Macias + +.. + +.. date: 2024-07-12-18-18-44 +.. gh-issue: 121297 +.. nonce: 67VE7b +.. section: Core and Builtins + +Previously, incorrect usage of :keyword:`await` or asynchronous +comprehensions in code removed by the :option:`-O` option was not flagged by +the Python compiler. Now, such code raises :exc:`SyntaxError`. Patch by +Jelle Zijlstra. + +.. + +.. date: 2024-07-10-15-43-54 +.. gh-issue: 117482 +.. nonce: 5WYaXR +.. section: Core and Builtins + +Unexpected slot wrappers are no longer created for builtin static types in +subinterpreters. + +.. + +.. date: 2024-07-10-02-02-32 +.. gh-issue: 121562 +.. nonce: 8beIMi +.. section: Core and Builtins + +Optimized performance of hex_from_char by replacing switch-case with a +lookup table + +.. + +.. date: 2024-07-09-13-53-18 +.. gh-issue: 121499 +.. nonce: rpp7il +.. section: Core and Builtins + +Fix a bug affecting how multi-line history was being rendered in the new +REPL after interacting with the new screen cache. Patch by Pablo Galindo + +.. + +.. date: 2024-07-08-17-15-14 +.. gh-issue: 121497 +.. nonce: I8hMDC +.. section: Core and Builtins + +Fix a bug that was preventing the REPL to correctly respect the history when +an input hook was set. Patch by Pablo Galindo + +.. + +.. date: 2024-07-08-10-31-08 +.. gh-issue: 121012 +.. nonce: M5hHk- +.. section: Core and Builtins + +Tier 2 execution now ensures that list iterators remain exhausted, once they +become exhausted. + +.. + +.. date: 2024-07-08-02-24-55 +.. gh-issue: 121439 +.. nonce: jDHod3 +.. section: Core and Builtins + +Allow tuples of length 20 in the freelist to be reused. + +.. + +.. date: 2024-07-05-11-29-27 +.. gh-issue: 121288 +.. nonce: lYKYYP +.. section: Core and Builtins + +:exc:`ValueError` messages for :meth:`!list.index`, :meth:`!range.index`, +:meth:`!deque.index`, :meth:`!deque.remove` and :meth:`!ShareableList.index` +no longer contain the repr of the searched value (which can be arbitrary +large) and are consistent with error messages for other :meth:`!index` and +:meth:`!remove` methods. + +.. + +.. date: 2024-07-04-23-38-30 +.. gh-issue: 121368 +.. nonce: m3EF9E +.. section: Core and Builtins + +Fix race condition in ``_PyType_Lookup`` in the free-threaded build due to a +missing memory fence. This could lead to ``_PyType_Lookup`` returning +incorrect results on arm64. + +.. + +.. date: 2024-06-30-03-48-10 +.. gh-issue: 121149 +.. nonce: lLBMKe +.. section: Core and Builtins + +Added specialization for summation of complexes, this also improves accuracy +of builtin :func:`sum` for such inputs. Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-06-29-10-46-14 +.. gh-issue: 121130 +.. nonce: Rj66Xs +.. section: Core and Builtins + +Fix f-strings with debug expressions in format specifiers. Patch by Pablo +Galindo + +.. + +.. date: 2024-06-28-23-17-22 +.. gh-issue: 121381 +.. nonce: i2xL7P +.. section: Core and Builtins + +Remove ``subprocess._USE_VFORK`` escape hatch code and documentation. It was +added just in case, and doesn't have any known cases that require it. + +.. + +.. date: 2024-06-28-18-34-49 +.. gh-issue: 119726 +.. nonce: Fjv_Ab +.. section: Core and Builtins + +Optimize code layout for calls to C functions from the JIT on AArch64. Patch +by Diego Russo. + +.. + +.. date: 2024-06-28-10-02-58 +.. gh-issue: 121115 +.. nonce: EeSLfc +.. section: Core and Builtins + +:c:func:`PyLong_AsNativeBytes` no longer uses :meth:`~object.__index__` +methods by default. The ``Py_ASNATIVEBYTES_ALLOW_INDEX`` flag has been added +to allow it. + +.. + +.. date: 2024-06-26-14-09-31 +.. gh-issue: 120838 +.. nonce: nFeTL9 +.. section: Core and Builtins + +:c:func:`Py_Finalize()` and :c:func:`Py_FinalizeEx()` now always run with +the main interpreter active. + +.. + +.. date: 2024-06-26-13-42-36 +.. gh-issue: 113433 +.. nonce: xKAtLB +.. section: Core and Builtins + +Subinterpreters now get cleaned up automatically during runtime +finalization. + +.. + +.. date: 2024-06-25-16-26-44 +.. gh-issue: 119726 +.. nonce: WqvHxB +.. section: Core and Builtins + +Improve the speed and memory use of C function calls from JIT code on +AArch64. Patch by Diego Russo + +.. + +.. date: 2024-06-24-08-39-23 +.. gh-issue: 116017 +.. nonce: -Bw2UY +.. section: Core and Builtins + +Simplify the warmup mechanism used for "side exits" in JIT code, resulting +in slightly better performance and slightly lower memory usage for most +platforms. + +.. + +.. date: 2024-06-19-21-34-21 +.. gh-issue: 98442 +.. nonce: cqhjkN +.. section: Core and Builtins + +Fix too wide source locations of the cleanup instructions of a with +statement. + +.. + +.. date: 2024-06-19-19-54-35 +.. gh-issue: 120754 +.. nonce: uF29sj +.. section: Core and Builtins + +Reduce the number of system calls invoked when reading a whole file (ex. +``open('a.txt').read()``). For a sample program that reads the contents of +the 400+ ``.rst`` files in the cpython repository ``Doc`` folder, there is +an over 10% reduction in system call count. + +.. + +.. date: 2024-06-19-11-10-50 +.. gh-issue: 119462 +.. nonce: DpcqSe +.. section: Core and Builtins + +Make sure that invariants of type versioning are maintained: * Superclasses +always have their version number assigned before subclasses * The version +tag is always zero if the tag is not valid. * The version tag is always +non-if the tag is valid. + +.. + +.. date: 2024-06-19-01-58-54 +.. gh-issue: 120437 +.. nonce: nCkIoI +.. section: Core and Builtins + +Fix ``_CHECK_STACK_SPACE`` optimization problems introduced in :gh:`118322`. + +.. + +.. date: 2024-06-18-22-41-05 +.. gh-issue: 120722 +.. nonce: rS7tkE +.. section: Core and Builtins + +Correctly set the bytecode position on return instructions within lambdas. +Patch by Jelle Zijlstra. + +.. + +.. date: 2024-06-18-21-34-30 +.. gh-issue: 120367 +.. nonce: zDwffP +.. section: Core and Builtins + +Fix bug where compiler creates a redundant jump during pseudo-op +replacement. Can only happen with a synthetic AST that has a try on the same +line as the instruction following the exception handler. + +.. + +.. date: 2024-06-17-12-20-20 +.. gh-issue: 120507 +.. nonce: 94lz2J +.. section: Core and Builtins + +Remove the ``BEFORE_WITH`` and ``BEFORE_ASYNC_WITH`` instructions. Add the +new :opcode:`LOAD_SPECIAL` instruction. Generate code for ``with`` and +``async with`` statements using the new instruction. + +.. + +.. date: 2024-06-14-22-02-25 +.. gh-issue: 113993 +.. nonce: MiA0vX +.. section: Core and Builtins + +Strings interned with :func:`sys.intern` are again garbage-collected when no +longer used, as per the documentation. Strings interned with the C function +:c:func:`PyUnicode_InternInPlace` are still immortal. Internals of the +string interning mechanism have been changed. This may affect performance +and identities of :class:`str` objects. + +.. + +.. date: 2024-06-14-07-52-00 +.. gh-issue: 120485 +.. nonce: yy4K4b +.. section: Core and Builtins + +Add an override of ``allow_reuse_port`` on classes subclassing +``socketserver.TCPServer`` where ``allow_reuse_address`` is also overridden. + +.. + +.. date: 2024-06-13-12-17-52 +.. gh-issue: 120384 +.. nonce: w1UBGl +.. section: Core and Builtins + +Fix an array out of bounds crash in ``list_ass_subscript``, which could be +invoked via some specifically tailored input: including concurrent +modification of a list object, where one thread assigns a slice and another +clears it. + +.. + +.. date: 2024-06-12-18-50-29 +.. gh-issue: 120367 +.. nonce: LmXx2y +.. section: Core and Builtins + +Fix crash in compiler on code with redundant NOPs and JUMPs which show up +after exception handlers are moved to the end of the code. + +.. + +.. date: 2024-06-12-18-23-15 +.. gh-issue: 120380 +.. nonce: edtqjq +.. section: Core and Builtins + +Fix Python implementation of :class:`pickle.Pickler` for :class:`bytes` and +:class:`bytearray` objects when using protocol version 5. Patch by Bénédikt +Tran. + +.. + +.. date: 2024-06-12-13-47-25 +.. gh-issue: 120397 +.. nonce: n-I_cc +.. section: Core and Builtins + +Improve the throughput by up to two times for the :meth:`str.count`, +:meth:`bytes.count` and :meth:`bytearray.count` methods for counting single +characters. + +.. + +.. date: 2024-06-11-17-56-12 +.. gh-issue: 120221 +.. nonce: si9hM9 +.. section: Core and Builtins + +Deliver real signals on Ctrl-C and Ctrl-Z in the new REPL. Patch by Pablo +Galindo + +.. + +.. date: 2024-06-11-12-47-54 +.. gh-issue: 120346 +.. nonce: hhn_6X +.. section: Core and Builtins + +Respect :envvar:`PYTHON_BASIC_REPL` when running in interactive inspect mode +(``python -i``). Patch by Pablo Galindo + +.. + +.. date: 2024-06-10-22-30-26 +.. gh-issue: 93691 +.. nonce: 68WOTS +.. section: Core and Builtins + +Fix source locations of instructions generated for the iterator of a for +statement. + +.. + +.. date: 2024-06-10-15-07-16 +.. gh-issue: 120198 +.. nonce: WW_pjO +.. section: Core and Builtins + +Fix a crash when multiple threads read and write to the same ``__class__`` +of an object concurrently. + +.. + +.. date: 2024-06-10-10-42-48 +.. gh-issue: 120298 +.. nonce: napREA +.. section: Core and Builtins + +Fix use-after free in ``list_richcompare_impl`` which can be invoked via +some specifically tailored evil input. + +.. + +.. date: 2024-06-09-19-13-38 +.. gh-issue: 119666 +.. nonce: S0G4rZ +.. section: Core and Builtins + +Fix a compiler crash in the case where two comprehensions in class scope +both reference ``__class__``. + +.. + +.. date: 2024-06-07-22-54-15 +.. gh-issue: 119726 +.. nonce: D9EE-o +.. section: Core and Builtins + +JIT: Re-use trampolines on AArch64 when creating stencils. Patch by Diego +Russo + +.. + +.. date: 2024-06-07-16-09-04 +.. gh-issue: 120225 +.. nonce: kuYf9t +.. section: Core and Builtins + +Fix crash in compiler on empty block at end of exception handler. + +.. + +.. date: 2024-06-05-18-29-18 +.. gh-issue: 93691 +.. nonce: 6OautB +.. section: Core and Builtins + +Fix source locations of instructions generated for with statements. + +.. + +.. date: 2024-06-05-10-32-44 +.. gh-issue: 120097 +.. nonce: 9S2klk +.. section: Core and Builtins + +``FrameLocalsProxy`` now subclasses ``collections.abc.Mapping`` and can be +matched as a mapping in ``match`` statements + +.. + +.. date: 2024-06-05-08-39-40 +.. gh-issue: 120080 +.. nonce: DJFK11 +.. section: Core and Builtins + +Direct call to the :meth:`!int.__round__` now accepts ``None`` as a valid +argument. + +.. + +.. date: 2024-06-03-13-48-44 +.. gh-issue: 119933 +.. nonce: Kc0HG5 +.. section: Core and Builtins + +Improve :exc:`SyntaxError` messages for invalid expressions in a type +parameters bound, a type parameter constraint tuple or a default type +parameter. Patch by Bénédikt Tran. + +.. + +.. date: 2024-06-03-13-25-04 +.. gh-issue: 119724 +.. nonce: EH1dkA +.. section: Core and Builtins + +Reverted improvements to error messages for ``elif``/``else`` statements not +matching any valid statements, which made in hard to locate the syntax +errors inside those ``elif``/``else`` blocks. + +.. + +.. date: 2024-06-02-06-12-35 +.. gh-issue: 119879 +.. nonce: Jit951 +.. section: Core and Builtins + +String search is now slightly faster for certain cases. It now utilizes last +character gap (good suffix rule) for two-way periodic needles. + +.. + +.. date: 2024-05-31-12-06-11 +.. gh-issue: 119842 +.. nonce: tCGVsv +.. section: Core and Builtins + +Honor :c:func:`PyOS_InputHook` in the new REPL. Patch by Pablo Galindo + +.. + +.. date: 2024-05-31-08-23-41 +.. gh-issue: 119180 +.. nonce: KL4VxZ +.. section: Core and Builtins + +:func:`classmethod` and :func:`staticmethod` now wrap the +:attr:`~object.__annotations__` and :attr:`~object.__annotate__` attributes +of their underlying callable lazily. See :pep:`649`. Patch by Jelle +Zijlstra. + +.. + +.. date: 2024-05-30-23-01-00 +.. gh-issue: 119821 +.. nonce: jPGfvt +.. section: Core and Builtins + +Fix execution of :ref:`annotation scopes ` within classes +when ``globals`` is set to a non-dict. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-05-30-04-11-36 +.. gh-issue: 118934 +.. nonce: fbDqve +.. section: Core and Builtins + +Make ``PyEval_GetLocals`` return borrowed reference + +.. + +.. date: 2024-05-29-18-53-43 +.. gh-issue: 119740 +.. nonce: zP2JNM +.. section: Core and Builtins + +Remove the previously-deprecated delegation of :func:`int` to +:meth:`~object.__trunc__`. + +.. + +.. date: 2024-05-28-22-49-56 +.. gh-issue: 119689 +.. nonce: VwLFD5 +.. section: Core and Builtins + +Generate stack effect metadata for pseudo instructions from bytecodes.c. + +.. + +.. date: 2024-05-27-19-13-49 +.. gh-issue: 109218 +.. nonce: -sdDg0 +.. section: Core and Builtins + +:func:`complex` accepts now a string only as a positional argument. Passing +a complex number as the "real" or "imag" argument is deprecated; it should +only be passed as a single positional argument. + +.. + +.. date: 2024-05-25-16-45-27 +.. gh-issue: 119548 +.. nonce: pqF9Y6 +.. section: Core and Builtins + +Add a ``clear`` command to the REPL. Patch by Pablo Galindo + +.. + +.. date: 2024-05-25-13-51-48 +.. gh-issue: 111999 +.. nonce: L0q1gh +.. section: Core and Builtins + +Fix the signature of :meth:`str.format_map`. + +.. + +.. date: 2024-05-25-12-52-25 +.. gh-issue: 119560 +.. nonce: wSlm8q +.. section: Core and Builtins + +An invalid assert in beta 1 has been removed. The assert would fail if +``PyState_FindModule()`` was used in an extension module's init function +before the module def had been initialized. + +.. + +.. date: 2024-05-24-21-16-52 +.. gh-issue: 119369 +.. nonce: qBThho +.. section: Core and Builtins + +Fix deadlock during thread deletion in free-threaded build, which could +occur when the GIL was enabled at runtime. + +.. + +.. date: 2024-05-24-21-04-00 +.. gh-issue: 119525 +.. nonce: zLFLf1 +.. section: Core and Builtins + +Fix deadlock involving ``_PyType_Lookup()`` cache in the free-threaded build +when the GIL is dynamically enabled at runtime. + +.. + +.. date: 2024-05-23-20-17-37 +.. gh-issue: 119258 +.. nonce: wZFIpt +.. section: Core and Builtins + +Eliminate type version guards in the tier two interpreter. + +Note that setting the ``tp_version_tag`` manually (which has never been +supported) may result in crashes. + +.. + +.. date: 2024-05-23-06-34-45 +.. gh-issue: 119311 +.. nonce: 2DBwKR +.. section: Core and Builtins + +Fix bug where names are unexpectedly mangled in the bases of generic +classes. + +.. + +.. date: 2024-05-23-06-34-14 +.. gh-issue: 119395 +.. nonce: z-Hsqb +.. section: Core and Builtins + +Fix bug where names appearing after a generic class are mangled as if they +are in the generic class. + +.. + +.. date: 2024-05-22-12-49-03 +.. gh-issue: 119372 +.. nonce: PXig1R +.. section: Core and Builtins + +Correct invalid corner cases in complex division (resulted in ``(nan+nanj)`` +output), e.g. ``1/complex('(inf+infj)')``. Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-05-22-06-22-47 +.. gh-issue: 119180 +.. nonce: vZMiXm +.. section: Core and Builtins + +Evaluation of annotations is now deferred. See :pep:`649` for details. + +.. + +.. date: 2024-05-21-12-17-02 +.. gh-issue: 119180 +.. nonce: UNDUb9 +.. section: Core and Builtins + +Replace :opcode:`!LOAD_ASSERTION_ERROR` opcode with +:opcode:`LOAD_COMMON_CONSTANT` and add support for +:exc:`NotImplementedError`. + +.. + +.. date: 2024-05-21-11-27-14 +.. gh-issue: 119213 +.. nonce: nxjxrt +.. section: Core and Builtins + +Non-builtin modules built with argument clinic were crashing if used in a +subinterpreter before the main interpreter. The objects that were causing +the problem by leaking between interpreters carelessly have been fixed. + +.. + +.. date: 2024-05-21-09-46-51 +.. gh-issue: 119011 +.. nonce: WOe3bu +.. section: Core and Builtins + +Fixes ``type.__type_params__`` to return an empty tuple instead of a +descriptor. + +.. + +.. date: 2024-05-20-14-57-39 +.. gh-issue: 118692 +.. nonce: Qadm7F +.. section: Core and Builtins + +Avoid creating unnecessary :exc:`StopIteration` instances for monitoring. + +.. + +.. date: 2024-05-20-10-10-51 +.. gh-issue: 119180 +.. nonce: 35xqpu +.. section: Core and Builtins + +Add an ``__annotate__`` attribute to functions, classes, and modules as part +of :pep:`649`. Patch by Jelle Zijlstra. + +.. + +.. date: 2024-05-16-23-02-03 +.. gh-issue: 119049 +.. nonce: qpd_S- +.. section: Core and Builtins + +Fix displaying the source line for warnings created by the C API if the +:mod:`warnings` module had not yet been imported. + +.. + +.. date: 2024-05-15-12-15-58 +.. gh-issue: 119057 +.. nonce: P3G9G2 +.. section: Core and Builtins + +Improve :exc:`ZeroDivisionError` error message. Now, all error messages are +harmonized: all ``/``, ``//``, and ``%`` operations just use "division by +zero" message. And ``0 ** -1`` operation uses "zero to a negative power". + +.. + +.. date: 2024-05-11-21-44-17 +.. gh-issue: 118844 +.. nonce: q2H_km +.. section: Core and Builtins + +Fix build failures when configuring with both ``--disable-gil`` and +``--enable-experimental-jit``. + +.. + +.. date: 2024-05-10-19-54-18 +.. gh-issue: 118921 +.. nonce: O4ztZG +.. section: Core and Builtins + +Add ``copy()`` method for ``FrameLocalsProxy`` which returns a snapshot +``dict`` for local variables. + +.. + +.. date: 2024-05-09-19-47-12 +.. gh-issue: 117657 +.. nonce: Vn0Yey +.. section: Core and Builtins + +Fix data races on the field that stores a pointer to the interpreter's main +thread that occur in free-threaded builds. + +.. + +.. date: 2024-05-09-02-37-25 +.. gh-issue: 118750 +.. nonce: 7aLfT- +.. section: Core and Builtins + +If the C version of the ``decimal`` module is available, ``int(str)`` now +uses it to supply an asymptotically much faster conversion. However, this +only applies if the string contains over about 2 million digits. + +.. + +.. date: 2024-05-08-09-44-15 +.. gh-issue: 118767 +.. nonce: iFF5F5 +.. section: Core and Builtins + +Using :data:`NotImplemented` in a boolean context now raises +:exc:`TypeError`. Contributed by Jelle Zijlstra. + +.. + +.. date: 2024-05-07-16-57-56 +.. gh-issue: 118561 +.. nonce: wNMKVd +.. section: Core and Builtins + +Fix race condition in free-threaded build where :meth:`!list.extend` could +expose uninitialised memory to concurrent readers. + +.. + +.. date: 2024-05-01-05-09-16 +.. gh-issue: 117139 +.. nonce: t41w_D +.. section: Core and Builtins + +Convert the Python evaluation stack to use internal stack references. The +purpose is to support tagged pointers. In :pep:`703`, this will allow for +its form of deferred reference counting. For both the default and +free-threaded builds, this sets up the infrastructure for unboxed integers +in the future. + +.. + +.. date: 2024-04-27-18-36-46 +.. gh-issue: 115801 +.. nonce: SVeHSy +.. section: Core and Builtins + +Raise ``TypeError`` when passing a string to :func:`difflib.unified_diff` +and :func:`difflib.context_diff`. + +.. + +.. date: 2024-04-08-13-49-02 +.. gh-issue: 117558 +.. nonce: 9lSEpR +.. section: Core and Builtins + +Improve error messages when a string, bytes or bytearray object of length 1 +is expected. + +.. + +.. date: 2024-03-25-15-07-01 +.. gh-issue: 117195 +.. nonce: OWakgD +.. section: Core and Builtins + +Avoid assertion failure for debug builds when calling +``object.__sizeof__(1)`` + +.. + +.. date: 2024-03-19-22-21-22 +.. gh-issue: 116022 +.. nonce: iyHENN +.. section: Core and Builtins + +Improve the :meth:`~object.__repr__` output of :class:`~ast.AST` nodes. + +.. + +.. date: 2024-01-15-21-51-26 +.. gh-issue: 114091 +.. nonce: VOtSJl +.. section: Core and Builtins + +Changed the error message for awaiting something that can't be awaited from +"object can't be used in an await expression" to "'' object +can't be awaited". + +.. + +.. date: 2024-01-15-18-11-48 +.. gh-issue: 113190 +.. nonce: OwQX64 +.. section: Core and Builtins + +:c:func:`Py_Finalize` now deletes all interned strings. + +.. + +.. date: 2023-10-14-23-05-40 +.. gh-issue: 84978 +.. nonce: Z0t6dg +.. section: Core and Builtins + +Add class methods :meth:`float.from_number` and :meth:`complex.from_number`. + +.. + +.. date: 2022-07-22-15-56-35 +.. gh-issue: 95144 +.. nonce: FZYWX- +.. section: Core and Builtins + +Improve the error message from ``a in b`` when ``b`` is not a container to +mention the term "container". + +.. + +.. bpo: 24766 +.. date: 2018-10-09-15-14-53 +.. nonce: c_C1Wc +.. section: Core and Builtins + +Fix handling of ``doc`` argument to subclasses of ``property``. + +.. + +.. date: 2024-10-03-14-06-08 +.. gh-issue: 123961 +.. nonce: uwJQTY +.. section: C API + +Convert the :ref:`curses.window ` static type exposed +by the :c:macro:`!PyCursesWindow_Type` macro in ``Include/py_curses.h`` to a +:ref:`heap type `. Patch by Bénédikt Tran. + +.. + +.. date: 2024-09-25-11-44-02 +.. gh-issue: 124502 +.. nonce: qWuDjT +.. section: C API + +Add :c:func:`PyUnicode_Equal` function to the limited C API: test if two +strings are equal. Patch by Victor Stinner. + +.. + +.. date: 2024-09-24-20-34-21 +.. gh-issue: 124296 +.. nonce: S4QoS1 +.. section: C API + +:c:type:`PyDictObject` no longer maintains a private version tag field +``ma_version_tag`` per :pep:`699`. This field was originally added in +Python 3.6 (:pep:`509`) and deprecated in Python 3.12. + +.. + +.. date: 2024-09-18-18-40-30 +.. gh-issue: 124160 +.. nonce: Zy-VKi +.. section: C API + +Fix crash when importing modules containing state and single-phase +initialization in a subinterpreter. + +.. + +.. date: 2024-09-17-05-23-35 +.. gh-issue: 124153 +.. nonce: L8TWmx +.. section: C API + +Add :c:func:`PyType_GetBaseByToken` and :c:data:`Py_tp_token` slot for +easier type checking, related to :pep:`489` and :pep:`630`. + +.. + +.. date: 2024-09-16-16-21-39 +.. gh-issue: 124127 +.. nonce: LB8DBU +.. section: C API + +In the limited C API 3.14 and newer, :c:func:`Py_REFCNT` is now implemented +as an opaque function call to hide implementation details. Patch by Victor +Stinner. + +.. + +.. date: 2024-09-12-16-16-24 +.. gh-issue: 123880 +.. nonce: 2-8vcj +.. section: C API + +Fixed a bug that prevented circular imports of extension modules that use +single-phase initialization. + +.. + +.. date: 2024-09-10-16-54-27 +.. gh-issue: 123909 +.. nonce: CTGxDR +.. section: C API + +:c:func:`PyType_FromSpec`, :c:func:`PyType_FromSpecWithBases` and +:c:func:`PyType_FromModuleAndSpec` will now fail if the metaclass of the new +type has custom :c:member:`~PyTypeObject.tp_new`. + +.. + +.. date: 2024-08-30-14-02-17 +.. gh-issue: 107954 +.. nonce: TPvj4u +.. section: C API + +Add functions to configure the Python initialization (:pep:`741`): + +* :c:func:`PyInitConfig_Create` +* :c:func:`PyInitConfig_Free` +* :c:func:`PyInitConfig_GetError` +* :c:func:`PyInitConfig_GetExitCode` +* :c:func:`PyInitConfig_HasOption` +* :c:func:`PyInitConfig_GetInt` +* :c:func:`PyInitConfig_GetStr` +* :c:func:`PyInitConfig_GetStrList` +* :c:func:`PyInitConfig_FreeStrList` +* :c:func:`PyInitConfig_SetInt` +* :c:func:`PyInitConfig_SetStr` +* :c:func:`PyInitConfig_SetStrList` +* :c:func:`PyInitConfig_AddModule` +* :c:func:`Py_InitializeFromInitConfig` + +Patch by Victor Stinner. + +.. + +.. date: 2024-08-29-15-55-55 +.. gh-issue: 107954 +.. nonce: pr2O50 +.. section: C API + +Add functions to get and set the current runtime Python configuration: + +* :c:func:`PyConfig_Get` +* :c:func:`PyConfig_GetInt` +* :c:func:`PyConfig_Set` +* :c:func:`PyConfig_Names` + +Patch by Victor Stinner. + +.. + +.. date: 2024-08-29-15-05-19 +.. gh-issue: 123465 +.. nonce: eqwNWq +.. section: C API + +:c:macro:`Py_RELATIVE_OFFSET` is now allowed in :c:type:`PyMemberDef` for +the special offset member ``"__vectorcalloffset__"``, as well as the +discouraged special offset members ``"__dictoffset__"`` and +``"__weaklistoffset__"`` + +.. + +.. date: 2024-08-26-13-01-20 +.. gh-issue: 100554 +.. nonce: 0ku85o +.. section: C API + +Added a slot ``Py_tp_vectorcall`` to set +:c:member:`~PyTypeObject.tp_vectorcall` via the :c:func:`PyType_FromSpec` +function family. Limited API extensions can use this feature to provide more +efficient vector call-based implementation of ``__new__`` and ``__init__``. + +.. + +.. date: 2024-08-09-13-12-20 +.. gh-issue: 122854 +.. nonce: -1OgvU +.. section: C API + +Add :c:func:`Py_HashBuffer` to compute and return the hash value of a +buffer. Patch by Antoine Pitrou and Victor Stinner. + +.. + +.. date: 2024-08-06-14-23-11 +.. gh-issue: 122728 +.. nonce: l-fQ-v +.. section: C API + +Fix :c:func:`PyEval_GetLocals` to avoid :exc:`SystemError` ("bad argument to +internal function"). Patch by Victor Stinner. + +.. + +.. date: 2024-07-30-23-48-26 +.. gh-issue: 116622 +.. nonce: yTTtil +.. section: C API + +Make :any:`PyObject_Print` work around a bug in Android and OpenBSD which +prevented it from throwing an exception when trying to write to a read-only +stream. + +.. + +.. date: 2024-07-27-00-28-35 +.. gh-issue: 105201 +.. nonce: 0-xUWq +.. section: C API + +Add :c:func:`PyIter_NextItem` to replace :c:func:`PyIter_Next`, which has an +ambiguous return value. Patch by Irit Katriel and Erlend Aasland. + +.. + +.. date: 2024-07-21-17-40-07 +.. gh-issue: 121489 +.. nonce: SUMFCr +.. section: C API + +Export private :c:func:`!_PyBytes_Join` again. + +.. + +.. date: 2024-07-12-13-40-59 +.. gh-issue: 121645 +.. nonce: 96QvD3 +.. section: C API + +Add :c:func:`PyBytes_Join(sep, iterable) ` function, similar +to ``sep.join(iterable)`` in Python. Patch by Victor Stinner. + +.. + +.. date: 2024-07-09-15-55-20 +.. gh-issue: 89364 +.. nonce: yYYroI +.. section: C API + +Export the :c:func:`PySignal_SetWakeupFd` function. Previously, the function +was documented but it couldn't be used in 3rd party code. Patch by Victor +Stinner. + +.. + +.. date: 2024-07-04-15-41-10 +.. gh-issue: 113993 +.. nonce: cLSiWV +.. section: C API + +:c:func:`PyUnicode_InternInPlace` no longer prevents its argument from being +garbage collected. + +Several functions that take ``char *`` are now documented as possibly +preventing string objects from being garbage collected; refer to their +documentation for details: :c:func:`PyUnicode_InternFromString`, +:c:func:`PyDict_SetItemString`, :c:func:`PyObject_SetAttrString`, +:c:func:`PyObject_DelAttrString`, :c:func:`PyUnicode_InternFromString`, and +``PyModule_Add*`` convenience functions. + +.. + +.. date: 2024-07-04-13-23-27 +.. gh-issue: 113601 +.. nonce: K3RLqp +.. section: C API + +Removed debug build assertions related to interning strings, which were +falsely triggered by stable ABI extensions. + +.. + +.. date: 2024-07-02-11-03-40 +.. gh-issue: 112136 +.. nonce: f3fiY8 +.. section: C API + +Restore the private ``_PyArg_Parser`` structure and the private +``_PyArg_ParseTupleAndKeywordsFast()`` function, previously removed in +Python 3.13 alpha 1. Patch by Victor Stinner. + +.. + +.. date: 2024-06-26-11-29-01 +.. gh-issue: 120642 +.. nonce: H7P9qK +.. section: C API + +Remove the private ``_Py_CODEUNIT`` type from the public C API. The internal +``pycore_code.h`` header should now be used to get this internal type. Patch +by Victor Stinner. + +.. + +.. date: 2024-06-21-16-41-21 +.. gh-issue: 120858 +.. nonce: Z5_-Mn +.. section: C API + +:c:func:`PyDict_Next` no longer locks the dictionary in the free-threaded +build. The locking needs to be done by the caller around the entire +iteration loop. + +.. + +.. date: 2024-06-19-21-27-42 +.. gh-issue: 120642 +.. nonce: UlKClN +.. section: C API + +Remove the following unstable functions: + +* ``PyUnstable_Replace_Executor()`` +* ``PyUnstable_SetOptimizer()`` +* ``PyUnstable_GetOptimizer()`` +* ``PyUnstable_GetExecutor()`` +* ``PyUnstable_Optimizer_NewCounter()`` +* ``PyUnstable_Optimizer_NewUOpOptimizer()`` + +Patch by Victor Stinner. + +.. + +.. date: 2024-06-19-17-27-22 +.. gh-issue: 120389 +.. nonce: GSZeHF +.. section: C API + +Add new functions to convert C ```` numbers from/to Python +:class:`int`: + +* :c:func:`PyLong_FromInt32` +* :c:func:`PyLong_FromUInt32` +* :c:func:`PyLong_FromInt64` +* :c:func:`PyLong_FromUInt64` +* :c:func:`PyLong_AsInt32` +* :c:func:`PyLong_AsUInt32` +* :c:func:`PyLong_AsInt64` +* :c:func:`PyLong_AsUInt64` + +Patch by Victor Stinner. + +.. + +.. date: 2024-06-16-22-58-47 +.. gh-issue: 120600 +.. nonce: TJdf0w +.. section: C API + +In the limited C API 3.14 and newer, :c:func:`Py_TYPE` is now implemented as +an opaque function call to hide implementation details. Patch by Victor +Stinner. + +.. + +.. date: 2024-06-11-21-38-32 +.. gh-issue: 70278 +.. nonce: WDE4zM +.. section: C API + +:c:func:`PyUnicode_FromFormat` no longer produces the ending ``\ufffd`` +character for truncated C string when use precision with ``%s`` and ``%V``. +It now truncates the string before the start of truncated multibyte +sequences. + +.. + +.. date: 2024-06-08-08-33-40 +.. gh-issue: 119771 +.. nonce: Oip2dL +.. section: C API + +Set :data:`errno` in :c:func:`_Py_c_pow` on overflows. Patch by Sergey B +Kirpichev. + +.. + +.. date: 2024-06-07-22-12-30 +.. gh-issue: 119182 +.. nonce: yt8Ar7 +.. section: C API + +Add a new :c:type:`PyUnicodeWriter` API to create a Python :class:`str` +object: + +* :c:func:`PyUnicodeWriter_Create`. +* :c:func:`PyUnicodeWriter_Discard`. +* :c:func:`PyUnicodeWriter_Finish`. +* :c:func:`PyUnicodeWriter_WriteChar`. +* :c:func:`PyUnicodeWriter_WriteUTF8`. +* :c:func:`PyUnicodeWriter_WriteUCS4`. +* :c:func:`PyUnicodeWriter_WriteWideChar`. +* :c:func:`PyUnicodeWriter_WriteStr`. +* :c:func:`PyUnicodeWriter_WriteRepr`. +* :c:func:`PyUnicodeWriter_WriteSubstring`. +* :c:func:`PyUnicodeWriter_Format`. +* :c:func:`PyUnicodeWriter_DecodeUTF8Stateful`. + +Patch by Victor Stinner. + +.. + +.. date: 2024-06-04-10-58-20 +.. gh-issue: 119613 +.. nonce: qOr9GF +.. section: C API + +Soft deprecate the :c:macro:`!Py_MEMCPY` macro: use directly ``memcpy()`` +instead. Patch by Victor Stinner. + +.. + +.. date: 2024-05-30-12-51-21 +.. gh-issue: 119775 +.. nonce: CBq9IG +.. section: C API + +Creating :c:data:`immutable types ` with mutable +bases was deprecated since 3.12 and now raises a :exc:`TypeError`. + +.. + +.. date: 2024-05-29-21-05-59 +.. gh-issue: 119585 +.. nonce: Sn7JL3 +.. section: C API + +Fix crash when a thread state that was created by +:c:func:`PyGILState_Ensure` calls a destructor that during +:c:func:`PyThreadState_Clear` that calls back into +:c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release`. This might +occur when in the free-threaded build or when using thread-local variables +whose destructors call :c:func:`PyGILState_Ensure`. + +.. + +.. date: 2024-05-29-09-21-37 +.. gh-issue: 119613 +.. nonce: J2xfrC +.. section: C API + +Macros ``Py_IS_NAN``, ``Py_IS_INFINITY`` and ``Py_IS_FINITE`` are +:term:`soft deprecated`. + +.. + +.. date: 2024-05-22-17-50-48 +.. gh-issue: 119336 +.. nonce: ff3qnS +.. section: C API + +Restore the removed ``_PyLong_NumBits()`` function. It is used by the +pywin32 project. Patch by Ethan Smith + +.. + +.. date: 2024-05-21-19-41-41 +.. gh-issue: 119344 +.. nonce: QKvzQb +.. section: C API + +The critical section API is now public as part of the non-limited C API. + +.. + +.. date: 2024-05-21-18-28-44 +.. gh-issue: 119333 +.. nonce: OTsYVX +.. section: C API + +Add :c:func:`PyContext_AddWatcher` and :c:func:`PyContext_ClearWatcher` APIs +to register callbacks to receive notification on enter and exit of context +objects. + +.. + +.. date: 2024-05-21-11-35-11 +.. gh-issue: 119247 +.. nonce: U6n6mh +.. section: C API + +Added ``Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST`` and +``Py_END_CRITICAL_SECTION_SEQUENCE_FAST`` macros to make it possible to use +PySequence_Fast APIs safely when free-threaded, and update str.join to work +without the GIL using them. + +.. + +.. date: 2024-05-20-10-35-22 +.. gh-issue: 111389 +.. nonce: a6axBk +.. section: C API + +Add :c:macro:`PyHASH_MULTIPLIER` constant: prime multiplier used in string +and various other hashes. Patch by Victor Stinner. + +.. + +.. date: 2024-05-10-15-43-14 +.. gh-issue: 118771 +.. nonce: 5KVglT +.. section: C API + +Several C declarations with names that didn't start with the ``Py`` or +``_Py`` prefixes, which were added by mistake in 3.13 alpha and beta +releases, were moved to internal headers. + +.. + +.. date: 2024-05-08-23-14-06 +.. gh-issue: 116984 +.. nonce: 5sgcDo +.. section: C API + +Make mimalloc includes relative to the current file to avoid embedders or +extensions needing to include ``Internal/mimalloc`` if they are already +including internal CPython headers. + +.. + +.. date: 2024-05-08-21-57-50 +.. gh-issue: 118789 +.. nonce: Ni4UQx +.. section: C API + +Add :c:func:`PyUnstable_Object_ClearWeakRefsNoCallbacks`, which clears +weakrefs without calling their callbacks. + +.. + +.. date: 2024-05-08-20-13-00 +.. gh-issue: 118789 +.. nonce: m88uUa +.. section: C API + +Restore ``_PyWeakref_ClearRef`` that was previously removed in Python 3.13 +alpha 1. + +.. + +.. date: 2024-04-10-16-48-04 +.. gh-issue: 117511 +.. nonce: RZtBRK +.. section: C API + +Make the :c:type:`PyMutex` public in the non-limited C API. + +.. + +.. date: 2024-03-10-14-55-51 +.. gh-issue: 116560 +.. nonce: x2mZaO +.. section: C API + +Add :c:func:`PyLong_GetSign` function. Patch by Sergey B Kirpichev. + +.. + +.. date: 2022-10-01-09-56-27 +.. gh-issue: 97588 +.. nonce: Gvg54o +.. section: C API + +Fix creating bitfields in :mod:`ctypes` structures and unions. Fields no +longer overlap. + +.. + +.. date: 2022-08-05-19-41-20 +.. gh-issue: 87135 +.. nonce: SCNBYj +.. section: C API + +Attempting to acquire the GIL after runtime finalization has begun in a +different thread now causes the thread to hang rather than terminate, which +avoids potential crashes or memory corruption caused by attempting to +terminate a thread that is running code not specifically designed to support +termination. In most cases this hanging is harmless since the process will +soon exit anyway. + +The ``PyThread_exit_thread`` function is now deprecated. Its behavior is +inconsistent across platforms, and it can only be used safely in the +unlikely case that every function in the entire call stack has been designed +to support the platform-dependent termination mechanism. It is recommended +that users of this function change their design to not require thread +termination. In the unlikely case that thread termination is needed and can +be done safely, users may migrate to calling platform-specific APIs such as +``pthread_exit`` (POSIX) or ``_endthreadex`` (Windows) directly. + +.. + +.. bpo: 34008 +.. date: 2018-06-30-21-48-16 +.. nonce: 2Wjtm0 +.. section: C API + +Added ``Py_IsInitialized`` to the list of APIs that are safe to call before +the interpreter is initialized, and updated the embedding tests to cover it. + +.. + +.. date: 2024-10-13-21-11-30 +.. gh-issue: 125269 +.. nonce: BC-fdo +.. section: Build + +Fix detection of whether ``-latomic`` is needed when cross-compiling CPython +using the configure script. + +.. + +.. date: 2024-09-23-11-27-25 +.. gh-issue: 123990 +.. nonce: d6HrYC +.. section: Build + +Remove ``WITH_FREELISTS`` macro and ``--without-freelists`` build +configuration + +.. + +.. date: 2024-09-16-09-42-05 +.. gh-issue: 124102 +.. nonce: Ow254j +.. section: Build + +Update internal documentation under PCbuild, so it now correctly states that +Windows requires VS2017 or later and Python 3.10 or later + +.. + +.. date: 2024-09-13-17-48-37 +.. gh-issue: 124043 +.. nonce: Bruxpq +.. section: Build + +Building using :option:`--with-trace-refs` is (temporarily) disallowed when +the GIL is disabled. + +.. + +.. date: 2024-09-04-12-01-43 +.. gh-issue: 123418 +.. nonce: ynzspB +.. section: Build + +Updated Android build to use OpenSSL 3.0.15. + +.. + +.. date: 2024-08-24-19-09-31 +.. gh-issue: 123297 +.. nonce: fdtXoe +.. section: Build + +Propagate the value of ``LDFLAGS`` to ``LDCXXSHARED`` in :mod:`sysconfig`. +Patch by Pablo Galindo + +.. + +.. date: 2024-08-14-19-17-34 +.. gh-issue: 121634 +.. nonce: eOMfHG +.. section: Build + +Allow for specifying the target compile triple for WASI. + +.. + +.. date: 2024-08-12-15-48-49 +.. gh-issue: 122578 +.. nonce: YJ3xEa +.. section: Build + +Use WASI SDK 24 for testing. + +.. + +.. date: 2024-08-07-00-20-18 +.. gh-issue: 116622 +.. nonce: U9cxHM +.. section: Build + +Rename build variable ``MODULE_LDFLAGS`` back to ``LIBPYTHON``, as it's used +by package build systems (e.g. Meson). + +.. + +.. date: 2024-08-02-12-59-15 +.. gh-issue: 118943 +.. nonce: vZQtET +.. section: Build + +Fix an issue where the experimental JIT could be built several times by the +``make regen-all`` target, leading to possible race conditions on heavily +parallelized builds. + +.. + +.. date: 2024-07-19-10-14-31 +.. gh-issue: 121996 +.. nonce: IEb2sz +.. section: Build + +Introduce ./configure --disable-safety and --enable-slower-safety options. +Patch by Donghee Na. + +.. + +.. date: 2024-07-18-07-53-07 +.. gh-issue: 120522 +.. nonce: dg3o5A +.. section: Build + +Added a :option:`--with-app-store-compliance` option to patch out known +issues with macOS/iOS App Store review processes. + +.. + +.. date: 2024-07-16-12-29-54 +.. gh-issue: 120371 +.. nonce: E7x858 +.. section: Build + +Support WASI SDK 22 by explicitly skipping functions that are just stubs in +wasi-libc. + +.. + +.. date: 2024-07-14-01-29-47 +.. gh-issue: 121731 +.. nonce: RMPGP3 +.. section: Build + +Fix mimalloc compile error on GNU/Hurd + +.. + +.. date: 2024-07-08-14-01-17 +.. gh-issue: 121487 +.. nonce: ekHmpR +.. section: Build + +Fix deprecation warning for ATOMIC_VAR_INIT in mimalloc. + +.. + +.. date: 2024-07-08-01-11-54 +.. gh-issue: 121467 +.. nonce: 3qWRQj +.. section: Build + +Fix a Makefile bug that prevented mimalloc header files from being +installed. + +.. + +.. date: 2024-07-02-20-16-09 +.. gh-issue: 121103 +.. nonce: TMef9j +.. section: Build + +On POSIX systems, excluding macOS framework installs, the lib directory for +the free-threaded build now includes a "t" suffix to avoid conflicts with a +co-located default build installation. + +.. + +.. date: 2024-07-02-12-42-25 +.. gh-issue: 120831 +.. nonce: i3eIjg +.. section: Build + +The default minimum iOS version was increased to 13.0. + +.. + +.. date: 2024-06-27-18-03-20 +.. gh-issue: 121082 +.. nonce: w3AfRx +.. section: Build + +Fix build failure when the developer use ``--enable-pystats`` arguments in +configuration command after #118450. + +.. + +.. date: 2024-06-21-09-24-03 +.. gh-issue: 120671 +.. nonce: Z8sBQB +.. section: Build + +Fix failing configure tests due to a missing space when appending to CFLAGS. + +.. + +.. date: 2024-06-19-21-05-15 +.. gh-issue: 120602 +.. nonce: UyDARz +.. section: Build + +Correctly handle LLVM installs with ``LLVM_VERSION_SUFFIX`` when building +with ``--enable-experimental-jit``. + +.. + +.. date: 2024-06-18-15-32-36 +.. gh-issue: 120688 +.. nonce: tjIPLD +.. section: Build + +On WASI in debug mode, Python is now built with compiler flag ``-O3`` +instead of ``-Og``, to support more recursive calls. Patch by Victor +Stinner. + +.. + +.. date: 2024-06-18-15-28-25 +.. gh-issue: 118943 +.. nonce: aie7nn +.. section: Build + +Fix a possible race condition affecting parallel builds configured with +``--enable-experimental-jit``, in which :exc:`FileNotFoundError` could be +caused by another process already moving ``jit_stencils.h.new`` to +``jit_stencils.h``. + +.. + +.. date: 2024-06-11-00-38-05 +.. gh-issue: 120326 +.. nonce: JHSDF1 +.. section: Build + +On Windows, fix build error when ``--disable-gil`` and +``--experimental-jit`` options are combined. + +.. + +.. date: 2024-06-09-15-54-22 +.. gh-issue: 120291 +.. nonce: IpfHzE +.. section: Build + +Make the ``python-config`` shell script compatible with non-bash shells. + +.. + +.. date: 2024-06-02-13-23-26 +.. gh-issue: 113565 +.. nonce: 8xBlId +.. section: Build + +Improve :mod:`curses` and :mod:`curses.panel` dependency checks in +:program:`configure`. + +.. + +.. date: 2024-05-29-17-40-50 +.. gh-issue: 119729 +.. nonce: k0xJ5U +.. section: Build + +On POSIX systems, the pkg-config (``.pc``) filenames now include the ABI +flags, which may include debug ("d") and free-threaded ("t"). For example: +* ``python-3.14.pc`` (default, non-debug build) * ``python-3.14d.pc`` +(default, debug build) * ``python-3.14t.pc`` (free-threaded build) + +.. + +.. date: 2024-05-22-13-18-02 +.. gh-issue: 119400 +.. nonce: WEt83v +.. section: Build + +``make_ssl_certs``, the script that prepares certificate data for the test +suite, now allows specifying expiration dates. + +.. + +.. date: 2024-05-19-22-54-55 +.. gh-issue: 115119 +.. nonce: DwMwev +.. section: Build + +Fall back to the bundled libmpdec if a system version cannot be found. + +.. + +.. date: 2024-05-17-19-53-27 +.. gh-issue: 119132 +.. nonce: wepPgM +.. section: Build + +Update :data:`sys.version` to identify whether the build is default build or +free-threading build. Patch By Donghee Na. + +.. + +.. date: 2024-05-13-15-57-58 +.. gh-issue: 118836 +.. nonce: 7yN1iB +.. section: Build + +Fix an ``AssertionError`` when building with ``--enable-experimental-jit`` +and the compiler emits a ``SHT_NOTE`` section. + +.. + +.. date: 2024-05-11-15-11-30 +.. gh-issue: 118943 +.. nonce: VI_MnY +.. section: Build + +Fix a possible race condition affecting parallel builds configured with +``--enable-experimental-jit``, in which compilation errors could be caused +by an incompletely-generated header file. diff --git a/Misc/NEWS.d/3.14.0a2.rst b/Misc/NEWS.d/3.14.0a2.rst new file mode 100644 index 00000000000000..d82ec98b7a3c87 --- /dev/null +++ b/Misc/NEWS.d/3.14.0a2.rst @@ -0,0 +1,1762 @@ +.. date: 2024-11-16-22-08-41 +.. gh-issue: 126911 +.. nonce: HchCZZ +.. release date: 2024-11-19 +.. section: Windows + +Update credits command output. + +.. + +.. date: 2024-11-12-22-31-13 +.. gh-issue: 118973 +.. nonce: _lfxW6 +.. section: Windows + +Ensures the experimental free-threaded install includes the ``_tkinter`` +module. The optional Tcl/Tk component must also be installed in order for +the module to work. + +.. + +.. date: 2024-11-07-20-42-31 +.. gh-issue: 126497 +.. nonce: EARpd- +.. section: Windows + +Fixes venv failure due to missing redirector executables in experimental +free-threaded installs. + +.. + +.. date: 2024-10-29-20-09-52 +.. gh-issue: 126074 +.. nonce: 83ZzZs +.. section: Windows + +Removed unnecessary DLLs from Windows embeddable package + +.. + +.. date: 2024-10-29-19-48-03 +.. gh-issue: 125315 +.. nonce: jdB9qN +.. section: Windows + +Avoid crashing in :mod:`platform` due to slow WMI calls on some Windows +machines. + +.. + +.. date: 2024-10-29-09-39-06 +.. gh-issue: 126084 +.. nonce: 3wAL8o +.. section: Windows + +Fix venvwlauncher to launch pythonw instead of python so no extra console +window is created. + +.. + +.. date: 2024-10-23-17-24-23 +.. gh-issue: 125842 +.. nonce: m3EF9E +.. section: Windows + +Fix a :exc:`SystemError` when :func:`sys.exit` is called with ``0xffffffff`` +on Windows. + +.. + +.. date: 2024-10-15-21-28-43 +.. gh-issue: 125550 +.. nonce: hmGWCP +.. section: Windows + +Enable the :ref:`launcher` to detect Python 3.14 installs from the Windows +Store. + +.. + +.. date: 2024-09-07-15-16-24 +.. gh-issue: 123803 +.. nonce: J9VNQU +.. section: Windows + +All Windows code pages are now supported as "cpXXX" codecs on Windows. + +.. + +.. date: 2024-11-13-22-23-36 +.. gh-issue: 126807 +.. nonce: vpaWuN +.. section: Tools/Demos + +Fix extraction warnings in :program:`pygettext.py` caused by mistaking +function definitions for function calls. + +.. + +.. date: 2024-10-30-13-59-07 +.. gh-issue: 126167 +.. nonce: j5cCWE +.. section: Tools/Demos + +The iOS testbed was modified so that it can be used by third-party projects +for testing purposes. + +.. + +.. date: 2024-11-17-16-56-48 +.. gh-issue: 126909 +.. nonce: 60VTxW +.. section: Tests + +Fix test_os extended attribute tests to work on filesystems with 1 KiB xattr +size limit. + +.. + +.. date: 2024-10-21-14-10-56 +.. gh-issue: 125730 +.. nonce: kcWbvI +.. section: Tests + +Change ``make test`` to not run GUI tests by default. Use ``make ci`` to run +tests with GUI tests instead. + +.. + +.. date: 2024-09-30-22-52-44 +.. gh-issue: 124295 +.. nonce: VZy5kx +.. section: Tests + +Add translation tests to the :mod:`argparse` module. + +.. + +.. date: 2024-11-13-11-09-12 +.. gh-issue: 126623 +.. nonce: TO7NnR +.. section: Security + +Upgrade libexpat to 2.6.4 + +.. + +.. date: 2024-11-18-15-33-25 +.. gh-issue: 85957 +.. nonce: 8gT3B- +.. section: Library + +Add missing MIME types for images with RFCs: emf, fits, g3fax, jp2, jpm, +jpx, t38, tiff-fx and wmf. Patch by Hugo van Kemenade. + +.. + +.. date: 2024-11-17-01-14-59 +.. gh-issue: 126920 +.. nonce: s8-f_L +.. section: Library + +Fix the ``prefix`` and ``exec_prefix`` keys from +:py:func:`sysconfig.get_config_vars` incorrectly having the same value as +:py:const:`sys.base_prefix` and :py:const:`sys.base_exec_prefix`, +respectively, inside virtual environments. They now accurately reflect +:py:const:`sys.prefix` and :py:const:`sys.exec_prefix`. + +.. + +.. date: 2024-11-14-22-25-49 +.. gh-issue: 67877 +.. nonce: G9hw0w +.. section: Library + +Fix memory leaks when :mod:`regular expression ` matching terminates +abruptly, either because of a signal or because memory allocation fails. + +.. + +.. date: 2024-11-14-13-16-20 +.. gh-issue: 125063 +.. nonce: kJ-WnH +.. section: Library + +:mod:`marshal` now supports :class:`slice` objects. The marshal format +version was increased to 5. + +.. + +.. date: 2024-11-13-22-25-57 +.. gh-issue: 126789 +.. nonce: lKzlc7 +.. section: Library + +Fixed the values of :py:func:`sysconfig.get_config_vars`, +:py:func:`sysconfig.get_paths`, and their siblings when the :py:mod:`site` +initialization happens after :py:mod:`sysconfig` has built a cache for +:py:func:`sysconfig.get_config_vars`. + +.. + +.. date: 2024-11-13-20-03-18 +.. gh-issue: 126188 +.. nonce: RJLKk- +.. section: Library + +Update bundled pip to 24.3.1 + +.. + +.. date: 2024-11-12-21-43-12 +.. gh-issue: 126766 +.. nonce: oi2KJ7 +.. section: Library + +Fix issue where :func:`urllib.request.url2pathname` failed to discard two +leading slashes introducing an empty authority section. + +.. + +.. date: 2024-11-11-14-52-21 +.. gh-issue: 126705 +.. nonce: 0W7jFW +.. section: Library + +Allow :class:`os.PathLike` to be a base for Protocols. + +.. + +.. date: 2024-11-11-13-24-22 +.. gh-issue: 126699 +.. nonce: ONGbMd +.. section: Library + +Allow :class:`collections.abc.AsyncIterator` to be a base for Protocols. + +.. + +.. date: 2024-11-11-13-00-21 +.. gh-issue: 126654 +.. nonce: 4gfP2y +.. section: Library + +Fix crash when non-dict was passed to several functions in ``_interpreters`` +module. + +.. + +.. date: 2024-11-10-18-14-51 +.. gh-issue: 104745 +.. nonce: zAa5Ke +.. section: Library + +Limit starting a patcher (from :func:`unittest.mock.patch` or +:func:`unittest.mock.patch.object`) more than once without stopping it + +.. + +.. date: 2024-11-09-10-31-10 +.. gh-issue: 126595 +.. nonce: A-7MyC +.. section: Library + +Fix a crash when instantiating :class:`itertools.count` with an initial +count of :data:`sys.maxsize` on debug builds. Patch by Bénédikt Tran. + +.. + +.. date: 2024-11-08-17-05-10 +.. gh-issue: 120423 +.. nonce: 7rdLVV +.. section: Library + +Fix issue where :func:`urllib.request.pathname2url` mishandled Windows paths +with embedded forward slashes. + +.. + +.. date: 2024-11-08-11-06-14 +.. gh-issue: 126565 +.. nonce: dFFO22 +.. section: Library + +Improve performances of :meth:`zipfile.Path.open` for non-reading modes. + +.. + +.. date: 2024-11-07-22-41-47 +.. gh-issue: 126505 +.. nonce: iztYE1 +.. section: Library + +Fix bugs in compiling case-insensitive :mod:`regular expressions ` with +character classes containing non-BMP characters: upper-case non-BMP +character did was ignored and the ASCII flag was ignored when matching a +character range whose upper bound is beyond the BMP region. + +.. + +.. date: 2024-11-07-01-40-11 +.. gh-issue: 117378 +.. nonce: o9O5uM +.. section: Library + +Fixed the :mod:`multiprocessing` ``"forkserver"`` start method forkserver +process to correctly inherit the parent's :data:`sys.path` during the +importing of :func:`multiprocessing.set_forkserver_preload` modules in the +same manner as :data:`sys.path` is configured in workers before executing +work items. + +This bug caused some forkserver module preloading to silently fail to +preload. This manifested as a performance degration in child processes when +the ``sys.path`` was required due to additional repeated work in every +worker. + +It could also have a side effect of ``""`` remaining in :data:`sys.path` +during forkserver preload imports instead of the absolute path from +:func:`os.getcwd` at multiprocessing import time used in the worker +``sys.path``. + +The ``sys.path`` differences between phases in the child process could +potentially have caused preload to import incorrect things from the wrong +location. We are unaware of that actually having happened in practice. + +.. + +.. date: 2024-11-06-23-40-28 +.. gh-issue: 125679 +.. nonce: Qq9xF5 +.. section: Library + +The :class:`multiprocessing.Lock` and :class:`multiprocessing.RLock` +``repr`` values no longer say "unknown" on macOS. + +.. + +.. date: 2024-11-06-18-30-50 +.. gh-issue: 126476 +.. nonce: F1wh3c +.. section: Library + +Raise :class:`calendar.IllegalMonthError` (now a subclass of +:class:`IndexError`) for :func:`calendar.month` when the input month is not +correct. + +.. + +.. date: 2024-11-06-13-41-38 +.. gh-issue: 126489 +.. nonce: toaf-0 +.. section: Library + +The Python implementation of :mod:`pickle` no longer calls +:meth:`pickle.Pickler.persistent_id` for the result of +:meth:`!persistent_id`. + +.. + +.. date: 2024-11-05-11-28-45 +.. gh-issue: 126451 +.. nonce: XJMtqz +.. section: Library + +Register the :class:`contextvars.Context` type to +:class:`collections.abc.Mapping`. + +.. + +.. date: 2024-11-05-09-54-49 +.. gh-issue: 126175 +.. nonce: spnjJr +.. section: Library + +Add ``msg``, ``doc``, ``pos``, ``lineno`` and ``colno`` attributes to +:exc:`tomllib.TOMLDecodeError`. Deprecate instantiating with free-form +arguments. + +.. + +.. date: 2024-11-04-22-53-09 +.. gh-issue: 89416 +.. nonce: YVQaas +.. section: Library + +Add :rfc:`9559` MIME types for Matroska audiovisual container formats. Patch +by Hugo van Kemenade. + +.. + +.. date: 2024-11-04-16-40-02 +.. gh-issue: 126417 +.. nonce: OWPqn0 +.. section: Library + +Register the :class:`!multiprocessing.managers.DictProxy` and +:class:`!multiprocessing.managers.ListProxy` types in +:mod:`multiprocessing.managers` to :class:`collections.abc.MutableMapping` +and :class:`collections.abc.MutableSequence`, respectively. + +.. + +.. date: 2024-11-04-13-16-18 +.. gh-issue: 126390 +.. nonce: Cxvqa5 +.. section: Library + +Add support for returning intermixed options and non-option arguments in +order in :func:`getopt.gnu_getopt`. + +.. + +.. date: 2024-11-03-23-25-07 +.. gh-issue: 126374 +.. nonce: Xu_THP +.. section: Library + +Add support for options with optional arguments in the :mod:`getopt` module. + +.. + +.. date: 2024-11-03-14-43-51 +.. gh-issue: 126363 +.. nonce: Xus7vU +.. section: Library + +Speed up pattern parsing in :meth:`pathlib.Path.glob` by skipping creation +of a :class:`pathlib.Path` object for the pattern. + +.. + +.. date: 2024-11-03-10-48-07 +.. gh-issue: 126353 +.. nonce: ChDzot +.. section: Library + +:func:`asyncio.get_event_loop` now does not implicitly creates an event +loop. It now raises a :exc:`RuntimeError` if there is no set event loop. +Patch by Kumar Aditya. + +.. + +.. date: 2024-11-03-09-42-42 +.. gh-issue: 126313 +.. nonce: EFP6Dl +.. section: Library + +Fix an issue in :func:`curses.napms` when :func:`curses.initscr` has not yet +been called. Patch by Bénédikt Tran. + +.. + +.. date: 2024-11-02-19-20-44 +.. gh-issue: 126303 +.. nonce: yVvyWB +.. section: Library + +Fix pickling and copying of :class:`os.sched_param` objects. + +.. + +.. date: 2024-11-01-14-31-41 +.. gh-issue: 126138 +.. nonce: yTniOG +.. section: Library + +Fix a use-after-free crash on :class:`asyncio.Task` objects whose underlying +coroutine yields an object that implements an evil +:meth:`~object.__getattribute__`. Patch by Nico Posada. + +.. + +.. date: 2024-11-01-10-35-49 +.. gh-issue: 120057 +.. nonce: YWy81Q +.. section: Library + +Replace the ``os.environ.refresh()`` method with a new +:func:`os.reload_environ` function. Patch by Victor Stinner. + +.. + +.. date: 2024-10-31-14-06-28 +.. gh-issue: 126220 +.. nonce: uJAJCU +.. section: Library + +Fix crash in :class:`!cProfile.Profile` and :class:`!_lsprof.Profiler` when +their callbacks were directly called with 0 arguments. + +.. + +.. date: 2024-10-30-23-59-36 +.. gh-issue: 126212 +.. nonce: _9uYjT +.. section: Library + +Fix issue where :func:`urllib.request.pathname2url` and +:func:`~urllib.request.url2pathname` removed slashes from Windows DOS drive +paths and URLs. + +.. + +.. date: 2024-10-30-23-42-44 +.. gh-issue: 126223 +.. nonce: k2qooc +.. section: Library + +Raise a :exc:`UnicodeEncodeError` instead of a :exc:`SystemError` upon +calling :func:`!_interpreters.create` with an invalid Unicode character. + +.. + +.. date: 2024-10-30-20-45-17 +.. gh-issue: 126205 +.. nonce: CHEmtx +.. section: Library + +Fix issue where :func:`urllib.request.pathname2url` generated URLs beginning +with four slashes (rather than two) when given a Windows UNC path. + +.. + +.. date: 2024-10-30-00-12-22 +.. gh-issue: 126156 +.. nonce: BOSqv0 +.. section: Library + +Improved performances of creating :py:class:`~http.cookies.Morsel` objects +by a factor of 3.8x. + +.. + +.. date: 2024-10-29-11-45-44 +.. gh-issue: 126105 +.. nonce: cOL-R6 +.. section: Library + +Fix a crash in :mod:`ast` when the :attr:`ast.AST._fields` attribute is +deleted. + +.. + +.. date: 2024-10-29-10-58-52 +.. gh-issue: 126106 +.. nonce: rlF798 +.. section: Library + +Fixes a possible ``NULL`` pointer dereference in :mod:`ssl`. + +.. + +.. date: 2024-10-29-10-38-28 +.. gh-issue: 126080 +.. nonce: qKRBuo +.. section: Library + +Fix a use-after-free crash on :class:`asyncio.Task` objects for which the +underlying event loop implements an evil :meth:`~object.__getattribute__`. +Reported by Nico-Posada. Patch by Bénédikt Tran. + +.. + +.. date: 2024-10-29-07-24-52 +.. gh-issue: 125322 +.. nonce: sstOM- +.. section: Library + +Correct detection of complex numbers support in libffi. + +.. + +.. date: 2024-10-28-22-35-22 +.. gh-issue: 126083 +.. nonce: TuI--n +.. section: Library + +Fixed a reference leak in :class:`asyncio.Task` objects when reinitializing +the same object with a non-``None`` context. Patch by Nico Posada. + +.. + +.. date: 2024-10-28-11-33-59 +.. gh-issue: 126068 +.. nonce: Pdznm_ +.. section: Library + +Fix exceptions in the :mod:`argparse` module so that only error messages for +ArgumentError and ArgumentTypeError are now translated. ArgumentError is now +only used for command line errors, not for logical errors in the program. +TypeError is now raised instead of ValueError for some logical errors. + +.. + +.. date: 2024-10-28-01-24-52 +.. gh-issue: 125413 +.. nonce: Jat5kq +.. section: Library + +Add :meth:`!pathlib.Path.scandir` method to efficiently fetch directory +children and their file attributes. This is a trivial wrapper of +:func:`os.scandir`. + +.. + +.. date: 2024-10-26-12-50-48 +.. gh-issue: 125984 +.. nonce: d4vp5_ +.. section: Library + +Fix use-after-free crashes on :class:`asyncio.Future` objects for which the +underlying event loop implements an evil :meth:`~object.__getattribute__`. +Reported by Nico-Posada. Patch by Bénédikt Tran. + +.. + +.. date: 2024-10-25-20-52-15 +.. gh-issue: 125926 +.. nonce: pp8rtZ +.. section: Library + +Fix :func:`urllib.parse.urljoin` for base URI with undefined authority. +Although :rfc:`3986` only specify reference resolution for absolute base +URI, :func:`!urljoin` should continue to return sensible result for relative +base URI. + +.. + +.. date: 2024-10-25-11-13-24 +.. gh-issue: 125969 +.. nonce: YvbrTr +.. section: Library + +Fix an out-of-bounds crash when an evil :meth:`asyncio.loop.call_soon` +mutates the length of the internal callbacks list. Patch by Bénédikt Tran. + +.. + +.. date: 2024-10-25-10-53-56 +.. gh-issue: 125966 +.. nonce: eOCYU_ +.. section: Library + +Fix a use-after-free crash in :meth:`asyncio.Future.remove_done_callback`. +Patch by Bénédikt Tran. + +.. + +.. date: 2024-10-24-14-08-10 +.. gh-issue: 125789 +.. nonce: eaiAMw +.. section: Library + +Fix possible crash when mutating list of callbacks returned by +:attr:`!asyncio.Future._callbacks`. It now always returns a new copy in C +implementation :mod:`!_asyncio`. Patch by Kumar Aditya. + +.. + +.. date: 2024-10-24-13-40-20 +.. gh-issue: 126916 +.. nonce: MAgz6D +.. section: Library + +Allow the *initial* parameter of :func:`functools.reduce` to be passed as a +keyword argument. Patch by Sayandip Dutta. + +.. + +.. date: 2024-10-24-10-49-47 +.. gh-issue: 124452 +.. nonce: eqTRgx +.. section: Library + +Fix an issue in :meth:`email.policy.EmailPolicy.header_source_parse` and +:meth:`email.policy.Compat32.header_source_parse` that introduced spurious +leading whitespaces into header values when the header includes a newline +character after the header name delimiter (``:``) and before the value. + +.. + +.. date: 2024-10-23-20-44-30 +.. gh-issue: 117941 +.. nonce: Y9jdlW +.. section: Library + +:class:`!argparse.BooleanOptionalAction` now rejects option names starting +with ``--no-``. + +.. + +.. date: 2024-10-23-17-45-40 +.. gh-issue: 125884 +.. nonce: 41E_PD +.. section: Library + +Fixed the bug for :mod:`pdb` where it can't set breakpoints on functions +with certain annotations. + +.. + +.. date: 2024-10-22-13-28-00 +.. gh-issue: 125355 +.. nonce: zssHm_ +.. section: Library + +Fix several bugs in :meth:`argparse.ArgumentParser.parse_intermixed_args`. + +* The parser no longer changes temporarily during parsing. +* Default values are not processed twice. +* Required mutually exclusive groups containing positional arguments are now supported. +* The missing arguments report now includes the names of all required optional and positional arguments. +* Unknown options can be intermixed with positional arguments in parse_known_intermixed_args(). + +.. + +.. date: 2024-10-21-13-52-37 +.. gh-issue: 125767 +.. nonce: 0kK4lX +.. section: Library + +:class:`super` objects are now :mod:`pickleable ` and :mod:`copyable +`. + +.. + +.. date: 2024-10-21-12-06-55 +.. gh-issue: 124969 +.. nonce: xiY8UP +.. section: Library + +``locale.nl_langinfo(locale.ALT_DIGITS)`` now returns a string again. The +returned value consists of up to 100 semicolon-separated symbols. + +.. + +.. date: 2024-10-20-00-56-44 +.. gh-issue: 84850 +.. nonce: p5TeUB +.. section: Library + +Remove :class:`!URLopener` and :class:`!FancyURLopener` classes from +:mod:`urllib.request`. They had previously raised :exc:`DeprecationWarning` +since Python 3.3. + +.. + +.. date: 2024-10-19-16-06-52 +.. gh-issue: 125666 +.. nonce: jGfdCP +.. section: Library + +Avoid the exiting the interpreter if a null byte is given as input in the +new REPL. + +.. + +.. date: 2024-10-19-13-37-37 +.. gh-issue: 125710 +.. nonce: FyFAAr +.. section: Library + +[Enum] fix hashable<->nonhashable comparisons for member values + +.. + +.. date: 2024-10-19-11-06-06 +.. gh-issue: 125631 +.. nonce: BlhVvR +.. section: Library + +Restore ability to set :attr:`~pickle.Pickler.persistent_id` and +:attr:`~pickle.Unpickler.persistent_load` attributes of instances of the +:class:`!Pickler` and :class:`!Unpickler` classes in the :mod:`pickle` +module. + +.. + +.. date: 2024-10-19-01-30-40 +.. gh-issue: 125378 +.. nonce: WTosxX +.. section: Library + +Fixed the bug in :mod:`pdb` where after a multi-line command, an empty line +repeats the first line of the multi-line command, instead of the full +command. + +.. + +.. date: 2024-10-18-09-51-29 +.. gh-issue: 125682 +.. nonce: vsj4cU +.. section: Library + +Reject non-ASCII digits in the Python implementation of :func:`json.loads` +conforming to the JSON specification. + +.. + +.. date: 2024-10-18-08-58-10 +.. gh-issue: 125660 +.. nonce: sDdDqO +.. section: Library + +Reject invalid unicode escapes for Python implementation of +:func:`json.loads`. + +.. + +.. date: 2024-10-17-20-36-06 +.. gh-issue: 52551 +.. nonce: EIVNYY +.. section: Library + +Use :c:func:`!wcsftime` to implement :func:`time.strftime` on Windows. + +.. + +.. date: 2024-10-17-16-10-29 +.. gh-issue: 125259 +.. nonce: oMew0c +.. section: Library + +Fix the notes removal logic for errors thrown in enum initialization. + +.. + +.. date: 2024-10-17-04-52-00 +.. gh-issue: 125633 +.. nonce: lMck06 +.. section: Library + +Add function :func:`inspect.ispackage` to determine whether an object is a +:term:`package` or not. + +.. + +.. date: 2024-10-16-22-45-50 +.. gh-issue: 125614 +.. nonce: 3OEo_Q +.. section: Library + +In the :data:`~annotationlib.Format.FORWARDREF` format of +:mod:`annotationlib`, fix bug where nested expressions were not returned as +:class:`annotationlib.ForwardRef` format. + +.. + +.. date: 2024-10-16-20-32-40 +.. gh-issue: 125590 +.. nonce: stHzOP +.. section: Library + +Allow ``FrameLocalsProxy`` to delete and pop if the key is not a fast +variable. + +.. + +.. date: 2024-10-16-15-55-50 +.. gh-issue: 125600 +.. nonce: yMsJx0 +.. section: Library + +Only show stale code warning in :mod:`pdb` when we display source code. + +.. + +.. date: 2024-10-16-04-50-53 +.. gh-issue: 125542 +.. nonce: vZJ-Ns +.. section: Library + +Deprecate passing keyword-only *prefix_chars* argument to +:meth:`argparse.ArgumentParser.add_argument_group`. + +.. + +.. date: 2024-10-15-16-50-03 +.. gh-issue: 125541 +.. nonce: FfhmWo +.. section: Library + +Pressing :kbd:`Ctrl-C` while blocked in :meth:`threading.Lock.acquire`, +:meth:`threading.RLock.acquire`, and :meth:`threading.Thread.join` now +interrupts the function call and raises a :exc:`KeyboardInterrupt` exception +on Windows, similar to how those functions behave on macOS and Linux. + +.. + +.. date: 2024-10-15-14-01-03 +.. gh-issue: 125519 +.. nonce: TqGh6a +.. section: Library + +Improve traceback if :func:`importlib.reload` is called with an object that +is not a module. Patch by Alex Waygood. + +.. + +.. date: 2024-10-14-17-29-34 +.. gh-issue: 125451 +.. nonce: fmP3T9 +.. section: Library + +Fix deadlock when :class:`concurrent.futures.ProcessPoolExecutor` shuts down +concurrently with an error when feeding a job to a worker process. + +.. + +.. date: 2024-10-14-02-07-44 +.. gh-issue: 125115 +.. nonce: IOf3ON +.. section: Library + +Fixed a bug in :mod:`pdb` where arguments starting with ``-`` can't be +passed to the debugged script. + +.. + +.. date: 2024-10-13-15-04-58 +.. gh-issue: 125398 +.. nonce: UW7Ndv +.. section: Library + +Fix the conversion of the :envvar:`!VIRTUAL_ENV` path in the activate script +in :mod:`venv` when running in Git Bash for Windows. + +.. + +.. date: 2024-10-11-00-40-13 +.. gh-issue: 125245 +.. nonce: 8vReM- +.. section: Library + +Fix race condition when importing :mod:`collections.abc`, which could +incorrectly return an empty module. + +.. + +.. date: 2024-10-09-17-07-33 +.. gh-issue: 52551 +.. nonce: PBakSY +.. section: Library + +Fix encoding issues in :func:`time.strftime`, the +:meth:`~datetime.datetime.strftime` method of the :mod:`datetime` classes +:class:`~datetime.datetime`, :class:`~datetime.date` and +:class:`~datetime.time` and formatting of these classes. Characters not +encodable in the current locale are now acceptable in the format string. +Surrogate pairs and sequence of surrogatescape-encoded bytes are no longer +recombinated. Embedded null character no longer terminates the format +string. + +.. + +.. date: 2024-10-04-22-43-48 +.. gh-issue: 124984 +.. nonce: xjMv9b +.. section: Library + +Fixed thread safety in :mod:`ssl` in the free-threaded build. OpenSSL +operations are now protected by a per-object lock. + +.. + +.. date: 2024-09-28-02-03-04 +.. gh-issue: 124651 +.. nonce: bLBGtH +.. section: Library + +Properly quote template strings in :mod:`venv` activation scripts. + +.. + +.. date: 2024-09-27-15-42-55 +.. gh-issue: 124694 +.. nonce: uUy32y +.. section: Library + +We've added :class:`concurrent.futures.InterpreterPoolExecutor`, which +allows you to run code in multiple isolated interpreters. This allows you +to circumvent the limitations of CPU-bound threads (due to the GIL). Patch +by Eric Snow. + +This addition is unrelated to :pep:`734`. + +.. + +.. date: 2024-09-27-13-10-17 +.. gh-issue: 58032 +.. nonce: 0aNAQ0 +.. section: Library + +Deprecate the :class:`argparse.FileType` type converter. + +.. + +.. date: 2024-09-24-18-49-16 +.. gh-issue: 99749 +.. nonce: gBDJX7 +.. section: Library + +Adds a feature to optionally enable suggestions for argument choices and +subparser names if mistyped by the user. + +.. + +.. date: 2024-09-24-18-16-59 +.. gh-issue: 58956 +.. nonce: 0wFrBR +.. section: Library + +Fixed a bug in :mod:`pdb` where sometimes the breakpoint won't trigger if it +was set on a function which is already in the call stack. + +.. + +.. date: 2024-09-17-10-38-26 +.. gh-issue: 124111 +.. nonce: Hd53VN +.. section: Library + +The tkinter module can now be built to use either the new version 9.0.0 of +Tcl/Tk or the latest release 8.6.15 of Tcl/Tk 8. Tcl/Tk 9 includes many +improvements, both to the Tcl language and to the appearance and utility of +the graphical user interface provided by Tk. + +.. + +.. date: 2024-09-07-13-57-49 +.. gh-issue: 80958 +.. nonce: fVYnqV +.. section: Library + +unittest discovery supports PEP 420 namespace packages as start directory +again. + +.. + +.. date: 2024-08-28-19-27-35 +.. gh-issue: 123370 +.. nonce: SPZ9Ux +.. section: Library + +Fix the canvas not clearing after running turtledemo clock. + +.. + +.. date: 2024-08-22-12-12-35 +.. gh-issue: 89083 +.. nonce: b6zFh0 +.. section: Library + +Add :func:`uuid.uuid8` for generating UUIDv8 objects as specified in +:rfc:`9562`. Patch by Bénédikt Tran + +.. + +.. date: 2024-08-01-11-15-55 +.. gh-issue: 122549 +.. nonce: ztV4Kz +.. section: Library + +Add :func:`platform.invalidate_caches` to invalidate cached results. + +.. + +.. date: 2024-07-23-02-24-50 +.. gh-issue: 120754 +.. nonce: nHb5mG +.. section: Library + +Update unbounded ``read`` calls in :mod:`zipfile` to specify an explicit +``size`` putting a limit on how much data they may read. This also updates +handling around ZIP max comment size to match the standard instead of +reading comments that are one byte too long. + +.. + +.. date: 2024-07-02-15-56-42 +.. gh-issue: 121267 +.. nonce: yFBWkh +.. section: Library + +Improve the performance of :mod:`tarfile` when writing files, by caching +user names and group names. + +.. + +.. date: 2024-06-06-04-06-05 +.. gh-issue: 70764 +.. nonce: 6511hw +.. section: Library + +Fixed an issue where :func:`inspect.getclosurevars` would incorrectly +classify an attribute name as a global variable when the name exists both as +an attribute name and a global variable. + +.. + +.. date: 2024-06-05-19-09-36 +.. gh-issue: 118289 +.. nonce: moL9_d +.. section: Library + +:func:`!posixpath.realpath` now raises :exc:`NotADirectoryError` when +*strict* mode is enabled and a non-directory path with a trailing slash is +supplied. + +.. + +.. date: 2024-06-02-11-48-19 +.. gh-issue: 119826 +.. nonce: N1obGa +.. section: Library + +Always return an absolute path for :func:`os.path.abspath` on Windows. + +.. + +.. date: 2024-05-28-14-35-23 +.. gh-issue: 97850 +.. nonce: dCtjel +.. section: Library + +Remove deprecated :func:`!pkgutil.get_loader` and +:func:`!pkgutil.find_loader`. + +.. + +.. date: 2024-05-13-10-09-41 +.. gh-issue: 118986 +.. nonce: -r4W9h +.. section: Library + +Add :data:`!socket.IPV6_RECVERR` constant (available since Linux 2.2). + +.. + +.. date: 2024-03-16-13-38-27 +.. gh-issue: 116897 +.. nonce: UDQTjp +.. section: Library + +Accepting objects with false values (like ``0`` and ``[]``) except empty +strings, byte-like objects and ``None`` in :mod:`urllib.parse` functions +:func:`~urllib.parse.parse_qsl` and :func:`~urllib.parse.parse_qs` is now +deprecated. + +.. + +.. date: 2023-10-26-16-36-22 +.. gh-issue: 101955 +.. nonce: Ixu3IF +.. section: Library + +Fix SystemError when match regular expression pattern containing some +combination of possessive quantifier, alternative and capture group. + +.. + +.. date: 2022-10-15-10-18-20 +.. gh-issue: 71936 +.. nonce: MzJjc_ +.. section: Library + +Fix a race condition in :class:`multiprocessing.pool.Pool`. + +.. + +.. bpo: 46128 +.. date: 2021-12-19-10-47-24 +.. nonce: Qv3EK1 +.. section: Library + +Strip :class:`unittest.IsolatedAsyncioTestCase` stack frames from reported +stacktraces. + +.. + +.. date: 2020-05-19-01-12-47 +.. gh-issue: 84852 +.. nonce: FEjHJW +.. section: Library + +Add MIME types for MS Embedded OpenType, OpenType Layout, TrueType, WOFF 1.0 +and 2.0 fonts. Patch by Sahil Prajapati and Hugo van Kemenade. + +.. + +.. date: 2024-11-09-19-43-10 +.. gh-issue: 126622 +.. nonce: YacfDc +.. section: Documentation + +Added stub pages for removed modules explaining their removal, where to find +replacements, and linking to the last Python version that supported them. +Contributed by Ned Batchelder. + +.. + +.. date: 2024-10-10-23-46-54 +.. gh-issue: 125277 +.. nonce: QAby09 +.. section: Documentation + +Require Sphinx 7.2.6 or later to build the Python documentation. Patch by +Adam Turner. + +.. + +.. date: 2023-03-28-22-24-45 +.. gh-issue: 60712 +.. nonce: So5uad +.. section: Documentation + +Include the :class:`object` type in the lists of documented types. Change by +Furkan Onder and Martin Panter. + +.. + +.. date: 2024-11-13-17-18-13 +.. gh-issue: 126795 +.. nonce: _JBX9e +.. section: Core and Builtins + +Increase the threshold for JIT code warmup. Depending on platform and +workload, this can result in performance gains of 1-9% and memory savings of +3-5%. + +.. + +.. date: 2024-11-12-19-24-00 +.. gh-issue: 126341 +.. nonce: 5SdAe1 +.. section: Core and Builtins + +Now :exc:`ValueError` is raised instead of :exc:`SystemError` when trying to +iterate over a released :class:`memoryview` object. + +.. + +.. date: 2024-11-11-17-02-48 +.. gh-issue: 126688 +.. nonce: QiOXUi +.. section: Core and Builtins + +Fix a crash when calling :func:`os.fork` on some operating systems, +including SerenityOS. + +.. + +.. date: 2024-11-09-16-10-22 +.. gh-issue: 126066 +.. nonce: 9zs4m4 +.. section: Core and Builtins + +Fix :mod:`importlib` to not write an incomplete .pyc files when a ulimit or +some other operating system mechanism is preventing the write to go through +fully. + +.. + +.. date: 2024-11-06-16-34-11 +.. gh-issue: 126222 +.. nonce: 9NBfTn +.. section: Core and Builtins + +Do not include count of "peek" items in ``_PyUop_num_popped``. This ensures +that the correct number of items are popped from the stack when a micro-op +exits with an error. + +.. + +.. date: 2024-11-03-15-15-36 +.. gh-issue: 126366 +.. nonce: 8BBdGU +.. section: Core and Builtins + +Fix crash when using ``yield from`` on an object that raises an exception in +its ``__iter__``. + +.. + +.. date: 2024-11-02-18-01-31 +.. gh-issue: 126209 +.. nonce: 2ZIhrS +.. section: Core and Builtins + +Fix an issue with ``skip_file_prefixes`` parameter which resulted in an +inconsistent behaviour between the C and Python implementations of +:func:`warnings.warn`. Patch by Daehee Kim. + +.. + +.. date: 2024-11-02-14-43-46 +.. gh-issue: 126312 +.. nonce: LMHzLT +.. section: Core and Builtins + +Fix crash during garbage collection on an object frozen by :func:`gc.freeze` +on the free-threaded build. + +.. + +.. date: 2024-11-01-09-58-06 +.. gh-issue: 103951 +.. nonce: 6qduwj +.. section: Core and Builtins + +Relax optimization requirements to allow fast attribute access to module +subclasses. + +.. + +.. date: 2024-10-31-21-49-00 +.. gh-issue: 126072 +.. nonce: o9k8Ns +.. section: Core and Builtins + +Following :gh:`126101`, for :ref:`codeobjects` like lambda, annotation and +type alias, we no longer add ``None`` to its :attr:`~codeobject.co_consts`. + +.. + +.. date: 2024-10-30-18-16-10 +.. gh-issue: 126195 +.. nonce: 6ezBpr +.. section: Core and Builtins + +Improve JIT performance by 1.4% on macOS Apple Silicon by using +platform-specific memory protection APIs. Patch by Diego Russo. + +.. + +.. date: 2024-10-29-15-17-31 +.. gh-issue: 126139 +.. nonce: B4OQ8a +.. section: Core and Builtins + +Provide better error location when attempting to use a :term:`future +statement <__future__>` with an unknown future feature. + +.. + +.. date: 2024-10-29-10-37-39 +.. gh-issue: 126072 +.. nonce: XLKlxv +.. section: Core and Builtins + +Add a new attribute in :attr:`~codeobject.co_flags` to indicate whether the +first item in :attr:`~codeobject.co_consts` is the docstring. If a code +object has no docstring, ``None`` will **NOT** be inserted. + +.. + +.. date: 2024-10-28-13-18-16 +.. gh-issue: 126076 +.. nonce: MebZuS +.. section: Core and Builtins + +Relocated objects such as ``tuple``, ``bytes`` and ``str`` objects are +properly tracked by :mod:`tracemalloc` and its associated hooks. Patch by +Pablo Galindo. + +.. + +.. date: 2024-10-27-20-31-43 +.. gh-issue: 90370 +.. nonce: IP_W3a +.. section: Core and Builtins + +Avoid temporary tuple creation for vararg in argument passing with Argument +Clinic generated code (if arguments either vararg or positional-only). + +.. + +.. date: 2024-10-26-23-50-03 +.. gh-issue: 126018 +.. nonce: Hq-qcM +.. section: Core and Builtins + +Fix a crash in :func:`sys.audit` when passing a non-string as first argument +and Python was compiled in debug mode. + +.. + +.. date: 2024-10-26-13-32-48 +.. gh-issue: 126012 +.. nonce: 2KalhG +.. section: Core and Builtins + +The :class:`memoryview` type now supports subscription, making it a +:term:`generic type`. + +.. + +.. date: 2024-10-25-15-56-14 +.. gh-issue: 125837 +.. nonce: KlCdgD +.. section: Core and Builtins + +Adds :opcode:`LOAD_SMALL_INT` and :opcode:`LOAD_CONST_IMMORTAL` +instructions. ``LOAD_SMALL_INT`` pushes a small integer equal to the +``oparg`` to the stack. ``LOAD_CONST_IMMORTAL`` does the same as +``LOAD_CONST`` but is more efficient for immortal objects. Removes +``RETURN_CONST`` instruction. + +.. + +.. date: 2024-10-24-22-43-03 +.. gh-issue: 125942 +.. nonce: 3UQht1 +.. section: Core and Builtins + +On Android, the ``errors`` setting of :any:`sys.stdout` was changed from +``surrogateescape`` to ``backslashreplace``. + +.. + +.. date: 2024-10-23-14-42-27 +.. gh-issue: 125859 +.. nonce: m3EF9E +.. section: Core and Builtins + +Fix a crash in the free threading build when :func:`gc.get_objects` or +:func:`gc.get_referrers` is called during an in-progress garbage collection. + +.. + +.. date: 2024-10-23-14-05-47 +.. gh-issue: 125868 +.. nonce: uLfXYB +.. section: Core and Builtins + +It was possible in 3.14.0a1 only for attribute lookup to give the wrong +value. This was due to an incorrect specialization in very specific +circumstances. This is fixed in 3.14.0a2. + +.. + +.. date: 2024-10-22-04-18-53 +.. gh-issue: 125498 +.. nonce: cFjPIn +.. section: Core and Builtins + +The JIT has been updated to leverage Clang 19’s new ``preserve_none`` +attribute, which supports more platforms and is more useful than LLVM's +existing ``ghccc`` calling convention. This also removes the need to +manually patch the calling convention in LLVM IR, simplifying the JIT +compilation process. + +.. + +.. date: 2024-10-18-16-00-10 +.. gh-issue: 125703 +.. nonce: QRoqMo +.. section: Core and Builtins + +Correctly honour :mod:`tracemalloc` hooks in specialized ``Py_DECREF`` +paths. Patch by Pablo Galindo + +.. + +.. date: 2024-10-18-10-11-43 +.. gh-issue: 125593 +.. nonce: Q97m3A +.. section: Core and Builtins + +Use color to highlight error locations in traceback from exception group + +.. + +.. date: 2024-10-16-23-06-06 +.. gh-issue: 125017 +.. nonce: fcltj0 +.. section: Core and Builtins + +Fix crash on certain accesses to the ``__annotations__`` of +:class:`staticmethod` and :class:`classmethod` objects. + +.. + +.. date: 2024-10-16-13-52-48 +.. gh-issue: 125588 +.. nonce: kCahyO +.. section: Core and Builtins + +The Python PEG generator can now use f-strings in the grammar actions. Patch +by Pablo Galindo + +.. + +.. date: 2024-10-16-12-12-39 +.. gh-issue: 125444 +.. nonce: 9tG2X6 +.. section: Core and Builtins + +Fix illegal instruction for older Arm architectures. Patch by Diego Russo, +testing by Ross Burton. + +.. + +.. date: 2024-10-14-17-13-12 +.. gh-issue: 118423 +.. nonce: SkBoda +.. section: Core and Builtins + +Add a new ``INSTRUCTION_SIZE`` macro to the cases generator which returns +the current instruction size. + +.. + +.. date: 2024-10-09-13-53-50 +.. gh-issue: 125038 +.. nonce: ffSLCz +.. section: Core and Builtins + +Fix crash when iterating over a generator expression after direct changes on +``gi_frame.f_locals``. Patch by Mikhail Efimov. + +.. + +.. date: 2024-10-01-17-31-32 +.. gh-issue: 124855 +.. nonce: sdsv_H +.. section: Core and Builtins + +Don't allow the JIT and perf support to be active at the same time. Patch by +Pablo Galindo + +.. + +.. date: 2024-09-14-20-09-39 +.. gh-issue: 123714 +.. nonce: o1mbe4 +.. section: Core and Builtins + +Update JIT compilation to use LLVM 19 + +.. + +.. date: 2024-09-11-01-32-07 +.. gh-issue: 123930 +.. nonce: BkPfB6 +.. section: Core and Builtins + +Improve the error message when a script shadowing a module from the standard +library causes :exc:`ImportError` to be raised during a "from" import. +Similarly, improve the error message when a script shadowing a third party +module attempts to "from" import an attribute from that third party module +while still initialising. + +.. + +.. date: 2024-06-13-19-12-49 +.. gh-issue: 119793 +.. nonce: FDVCDk +.. section: Core and Builtins + +The :func:`map` built-in now has an optional keyword-only *strict* flag like +:func:`zip` to check that all the iterables are of equal length. Patch by +Wannes Boeykens. + +.. + +.. date: 2024-05-12-03-10-36 +.. gh-issue: 118950 +.. nonce: 5Wc4vp +.. section: Core and Builtins + +Fix bug where SSLProtocol.connection_lost wasn't getting called when OSError +was thrown on writing to socket. + +.. + +.. date: 2023-12-30-00-21-45 +.. gh-issue: 113570 +.. nonce: _XQgsW +.. section: Core and Builtins + +Fixed a bug in ``reprlib.repr`` where it incorrectly called the repr method +on shadowed Python built-in types. + +.. + +.. date: 2024-11-07-20-24-58 +.. gh-issue: 126554 +.. nonce: ri12eb +.. section: C API + +Fix error handling in :class:`ctypes.CDLL` objects which could result in a +crash in rare situations. + +.. + +.. date: 2024-10-28-15-56-03 +.. gh-issue: 126061 +.. nonce: Py51_1 +.. section: C API + +Add :c:func:`PyLong_IsPositive`, :c:func:`PyLong_IsNegative` and +:c:func:`PyLong_IsZero` for checking if a :c:type:`PyLongObject` is +positive, negative, or zero, respectively. + +.. + +.. date: 2024-10-16-19-28-23 +.. gh-issue: 125608 +.. nonce: gTsU2g +.. section: C API + +Fix a bug where dictionary watchers (e.g., :c:func:`PyDict_Watch`) on an +object's attribute dictionary (:attr:`~object.__dict__`) were not triggered +when the object's attributes were modified. + +.. + +.. date: 2024-09-03-13-33-33 +.. gh-issue: 123619 +.. nonce: HhgUUI +.. section: C API + +Added the :c:func:`PyUnstable_Object_EnableDeferredRefcount` function for +enabling :pep:`703` deferred reference counting. + +.. + +.. date: 2024-07-30-14-40-08 +.. gh-issue: 121654 +.. nonce: tgGeAl +.. section: C API + +Add :c:func:`PyType_Freeze` function to make a type immutable. Patch by +Victor Stinner. + +.. + +.. date: 2024-06-04-13-38-44 +.. gh-issue: 120026 +.. nonce: uhEvJ9 +.. section: C API + +The :c:macro:`!Py_HUGE_VAL` macro is :term:`soft deprecated`. + +.. + +.. date: 2024-11-13-15-47-09 +.. gh-issue: 126691 +.. nonce: ni4K-b +.. section: Build + +Removed the ``--with-emscripten-target`` configure flag. We unified the +``node`` and ``browser`` options and the same build can now be used, +independent of target runtime. + +.. + +.. date: 2024-11-07-11-09-31 +.. gh-issue: 123877 +.. nonce: CVdd0b +.. section: Build + +Use ``wasm32-wasip1`` as the target triple for WASI instead of +``wasm32-wasi``. The latter will eventually be reclaimed for WASI 1.0 while +CPython currently only supports WASI preview1. + +.. + +.. date: 2024-11-06-11-12-04 +.. gh-issue: 126458 +.. nonce: 7vzHtx +.. section: Build + +Disable SIMD support for HACL under WASI. + +.. + +.. date: 2024-11-04-09-42-04 +.. gh-issue: 89640 +.. nonce: QBv05o +.. section: Build + +Hard-code float word ordering as little endian on WASM. + +.. + +.. date: 2024-10-31-15-37-05 +.. gh-issue: 126206 +.. nonce: oC6z2i +.. section: Build + +``make clinic`` now runs Argument Clinic using the ``--force`` option, thus +forcefully regenerating generated code. + +.. + +.. date: 2024-10-30-17-47-15 +.. gh-issue: 126187 +.. nonce: 0jFCZB +.. section: Build + +Introduced ``Tools/wasm/emscripten.py`` to simplify doing Emscripten builds. + +.. + +.. date: 2024-10-25-17-20-50 +.. gh-issue: 124932 +.. nonce: F-aNuS +.. section: Build + +For cross builds, there is now support for having a different install +``prefix`` than the ``host_prefix`` used by ``getpath.py``. This is set to +``/`` by default for Emscripten, on other platforms the default behavior is +the same as before. + +.. + +.. date: 2024-10-25-00-29-15 +.. gh-issue: 125946 +.. nonce: KPA3g0 +.. section: Build + +The minimum supported Android version is now 7.0 (API level 24). + +.. + +.. date: 2024-10-24-22-14-35 +.. gh-issue: 125940 +.. nonce: 2wMtTA +.. section: Build + +The Android build now supports `16 KB page sizes +`__. + +.. + +.. date: 2024-10-16-09-37-51 +.. gh-issue: 89640 +.. nonce: UDsW-j +.. section: Build + +Improve detection of float word ordering on Linux when link-time +optimizations are enabled. + +.. + +.. date: 2024-10-04-17-29-23 +.. gh-issue: 124928 +.. nonce: FsGffe +.. section: Build + +Emscripten builds now require node >= 18. + +.. + +.. date: 2024-03-03-20-28-23 +.. gh-issue: 115382 +.. nonce: 97hJFE +.. section: Build + +Fix cross compile failures when the host and target SOABIs match. diff --git a/Misc/NEWS.d/3.14.0a3.rst b/Misc/NEWS.d/3.14.0a3.rst new file mode 100644 index 00000000000000..8393be8909ff8f --- /dev/null +++ b/Misc/NEWS.d/3.14.0a3.rst @@ -0,0 +1,1152 @@ +.. date: 2024-11-28-15-55-48 +.. gh-issue: 127353 +.. nonce: i-XOXg +.. release date: 2024-12-17 +.. section: Windows + +Allow to force color output on Windows using environment variables. Patch by +Andrey Efremov. + +.. + +.. date: 2024-10-31-09-46-53 +.. gh-issue: 125729 +.. nonce: KdKVLa +.. section: Windows + +Makes the presence of the :mod:`turtle` module dependent on the Tcl/Tk +installer option. Previously, the module was always installed but would be +unusable without Tcl/Tk. + +.. + +.. date: 2024-11-16-20-47-20 +.. gh-issue: 126700 +.. nonce: ayrHv4 +.. section: Tools/Demos + +Add support for multi-argument :mod:`gettext` functions in +:program:`pygettext.py`. + +.. + +.. date: 2024-12-13-13-41-34 +.. gh-issue: 127906 +.. nonce: NuRHlB +.. section: Tests + +Test the limited C API in test_cppext. Patch by Victor Stinner. + +.. + +.. date: 2024-12-09-12-35-44 +.. gh-issue: 127637 +.. nonce: KLx-9I +.. section: Tests + +Add tests for the :mod:`dis` command-line interface. Patch by Bénédikt Tran. + +.. + +.. date: 2024-12-04-15-03-24 +.. gh-issue: 126925 +.. nonce: uxAMK- +.. section: Tests + +iOS test results are now streamed during test execution, and the deprecated +xcresulttool is no longer used. + +.. + +.. date: 2024-11-21-02-03-48 +.. gh-issue: 127076 +.. nonce: a3avV1 +.. section: Tests + +Disable strace based system call tests when LD_PRELOAD is set. + +.. + +.. date: 2024-11-20-18-49-01 +.. gh-issue: 127076 +.. nonce: DHnXxo +.. section: Tests + +Filter out memory-related ``mmap``, ``munmap``, and ``mprotect`` calls from +file-related ones when testing :mod:`io` behavior using strace. + +.. + +.. date: 2024-12-05-21-35-19 +.. gh-issue: 127655 +.. nonce: xpPoOf +.. section: Security + +Fixed the :class:`!asyncio.selector_events._SelectorSocketTransport` +transport not pausing writes for the protocol when the buffer reaches the +high water mark when using :meth:`asyncio.WriteTransport.writelines`. + +.. + +.. date: 2024-12-13-22-20-54 +.. gh-issue: 126907 +.. nonce: fWRL_R +.. section: Library + +Fix crash when using :mod:`atexit` concurrently on the :term:`free-threaded +` build. + +.. + +.. date: 2024-12-12-16-59-42 +.. gh-issue: 127870 +.. nonce: _NFG-3 +.. section: Library + +Detect recursive calls in ctypes ``_as_parameter_`` handling. Patch by +Victor Stinner. + +.. + +.. date: 2024-12-08-08-36-18 +.. gh-issue: 127732 +.. nonce: UEKxoa +.. section: Library + +The :mod:`platform` module now correctly detects Windows Server 2025. + +.. + +.. date: 2024-12-07-23-06-44 +.. gh-issue: 126789 +.. nonce: 4dqfV1 +.. section: Library + +Fixed :func:`sysconfig.get_config_vars`, :func:`sysconfig.get_paths`, and +siblings, returning outdated cached data if the value of :data:`sys.prefix` +or :data:`sys.exec_prefix` changes. Overwriting :data:`sys.prefix` or +:data:`sys.exec_prefix` still is discouraged, as that might break other +parts of the code. + +.. + +.. date: 2024-12-07-15-28-31 +.. gh-issue: 127718 +.. nonce: 9dpLfi +.. section: Library + +Add colour to :mod:`test.regrtest` output. Patch by Hugo van Kemenade. + +.. + +.. date: 2024-12-06-17-28-55 +.. gh-issue: 127610 +.. nonce: ctv_NP +.. section: Library + +Added validation for more than one var-positional or var-keyword parameters +in :class:`inspect.Signature`. Patch by Maxim Ageev. + +.. + +.. date: 2024-12-05-10-14-52 +.. gh-issue: 127627 +.. nonce: fgCHOZ +.. section: Library + +Added ``posix._emscripten_debugger()`` to help with debugging the test suite +on the Emscripten target. + +.. + +.. date: 2024-12-04-15-04-12 +.. gh-issue: 126821 +.. nonce: lKCLVV +.. section: Library + +macOS and iOS apps can now choose to redirect stdout and stderr to the +system log during interpreter configuration. + +.. + +.. date: 2024-12-04-11-01-16 +.. gh-issue: 93312 +.. nonce: 9sB-Qw +.. section: Library + +Include ```` to get ``os.PIDFD_NONBLOCK`` constant. Patch by +Victor Stinner. + +.. + +.. date: 2024-12-01-23-18-43 +.. gh-issue: 127481 +.. nonce: K36AoP +.. section: Library + +Add the ``EPOLLWAKEUP`` constant to the :mod:`select` module. + +.. + +.. date: 2024-12-01-22-28-41 +.. gh-issue: 127065 +.. nonce: tFpRer +.. section: Library + +Make :func:`operator.methodcaller` thread-safe and re-entrant safe. + +.. + +.. date: 2024-11-30-21-46-15 +.. gh-issue: 127321 +.. nonce: M78fBv +.. section: Library + +:func:`pdb.set_trace` will not stop at an opcode that does not have an +associated line number anymore. + +.. + +.. date: 2024-11-29-23-02-43 +.. gh-issue: 127429 +.. nonce: dQf2w4 +.. section: Library + +Fixed bug where, on cross-builds, the :mod:`sysconfig` POSIX data was being +generated with the host Python's ``Makefile``. The data is now generated +from current build's ``Makefile``. + +.. + +.. date: 2024-11-29-14-45-26 +.. gh-issue: 127413 +.. nonce: z11AUc +.. section: Library + +Add the :option:`dis --specialized` command-line option to show specialized +bytecode. Patch by Bénédikt Tran. + +.. + +.. date: 2024-11-29-00-15-59 +.. gh-issue: 125413 +.. nonce: WCN0vv +.. section: Library + +Revert addition of :meth:`!pathlib.Path.scandir`. This method was added in +3.14.0a2. The optimizations remain for file system paths, but other +subclasses should only have to implement :meth:`pathlib.Path.iterdir`. + +.. + +.. date: 2024-11-28-14-14-46 +.. gh-issue: 127257 +.. nonce: n6-jU9 +.. section: Library + +In :mod:`ssl`, system call failures that OpenSSL reports using +``ERR_LIB_SYS`` are now raised as :exc:`OSError`. + +.. + +.. date: 2024-11-27-17-04-38 +.. gh-issue: 59705 +.. nonce: sAGyvs +.. section: Library + +On Linux, :class:`threading.Thread` now sets the thread name to the +operating system. Patch by Victor Stinner. + +.. + +.. date: 2024-11-27-16-06-10 +.. gh-issue: 127303 +.. nonce: asqkgh +.. section: Library + +Publicly expose :data:`~token.EXACT_TOKEN_TYPES` in :attr:`!token.__all__`. + +.. + +.. date: 2024-11-27-14-23-02 +.. gh-issue: 127331 +.. nonce: 9sNEC9 +.. section: Library + +:mod:`ssl` can show descriptions for errors added in OpenSSL 3.4. + +.. + +.. date: 2024-11-27-14-06-35 +.. gh-issue: 123967 +.. nonce: wxUmnW +.. section: Library + +Fix faulthandler for trampoline frames. If the top-most frame is a +trampoline frame, skip it. Patch by Victor Stinner. + +.. + +.. date: 2024-11-26-17-42-00 +.. gh-issue: 127178 +.. nonce: U8hxjc +.. section: Library + +A ``_sysconfig_vars_(...).json`` file is now shipped in the standard library +directory. It contains the output of :func:`sysconfig.get_config_vars` on +the default environment encoded as JSON data. This is an implementation +detail, and may change at any time. + +.. + +.. date: 2024-11-25-19-04-10 +.. gh-issue: 127072 +.. nonce: -c284K +.. section: Library + +Remove outdated ``socket.NETLINK_*`` constants not present in Linux kernels +beyond 2.6.17. + +.. + +.. date: 2024-11-25-15-02-44 +.. gh-issue: 127255 +.. nonce: UXeljc +.. section: Library + +The :func:`~ctypes.CopyComPointer` function is now public. Previously, this +was private and only available in ``_ctypes``. + +.. + +.. date: 2024-11-24-14-20-17 +.. gh-issue: 127182 +.. nonce: WmfY2g +.. section: Library + +Fix :meth:`!io.StringIO.__setstate__` crash, when :const:`None` was passed +as the first value. + +.. + +.. date: 2024-11-24-12-41-31 +.. gh-issue: 127217 +.. nonce: UAXGFr +.. section: Library + +Fix :func:`urllib.request.pathname2url` for paths starting with multiple +slashes on Posix. + +.. + +.. date: 2024-11-23-12-25-06 +.. gh-issue: 125866 +.. nonce: wEOP66 +.. section: Library + +:func:`urllib.request.pathname2url` now adds an empty authority when +generating a URL for a path that begins with exactly one slash. For example, +the path ``/etc/hosts`` is converted to the scheme-less URL +``///etc/hosts``. As a result of this change, URLs without authorities are +only generated for relative paths. + +.. + +.. date: 2024-11-23-00-17-29 +.. gh-issue: 127221 +.. nonce: OSXdFE +.. section: Library + +Add colour to :mod:`unittest` output. Patch by Hugo van Kemenade. + +.. + +.. date: 2024-11-22-10-42-34 +.. gh-issue: 127035 +.. nonce: UnbDlr +.. section: Library + +Fix :mod:`shutil.which` on Windows. Now it looks at direct match if and only +if the command ends with a PATHEXT extension or X_OK is not in mode. Support +extensionless files if "." is in PATHEXT. Support PATHEXT extensions that +end with a dot. + +.. + +.. date: 2024-11-22-09-23-41 +.. gh-issue: 122273 +.. nonce: H8M6fd +.. section: Library + +Support PyREPL history on Windows. Patch by devdanzin and Victor Stinner. + +.. + +.. date: 2024-11-22-04-49-31 +.. gh-issue: 125866 +.. nonce: TUtvPK +.. section: Library + +:func:`urllib.request.pathname2url` and :func:`~urllib.request.url2pathname` +no longer convert Windows drive letters to uppercase. + +.. + +.. date: 2024-11-22-03-40-02 +.. gh-issue: 127078 +.. nonce: gI_PaP +.. section: Library + +Fix issue where :func:`urllib.request.url2pathname` failed to discard an +extra slash before a UNC drive in the URL path on Windows. + +.. + +.. date: 2024-11-22-02-31-55 +.. gh-issue: 126766 +.. nonce: jfkhBH +.. section: Library + +Fix issue where :func:`urllib.request.url2pathname` failed to discard any +'localhost' authority present in the URL. + +.. + +.. date: 2024-11-21-16-23-16 +.. gh-issue: 127065 +.. nonce: cfL1zd +.. section: Library + +Fix crash when calling a :func:`operator.methodcaller` instance from +multiple threads in the free threading build. + +.. + +.. date: 2024-11-21-06-03-46 +.. gh-issue: 127090 +.. nonce: yUYwdh +.. section: Library + +Fix value of :attr:`urllib.response.addinfourl.url` for ``file:`` URLs that +express relative paths and absolute Windows paths. The canonical URL +generated by :func:`urllib.request.pathname2url` is now used. + +.. + +.. date: 2024-11-20-21-20-56 +.. gh-issue: 126992 +.. nonce: RbU0FZ +.. section: Library + +Fix LONG and INT opcodes to only use base 10 for string to integer +conversion in :mod:`pickle`. + +.. + +.. date: 2024-11-20-16-58-59 +.. gh-issue: 126997 +.. nonce: 0PI41Y +.. section: Library + +Fix support of STRING and GLOBAL opcodes with non-ASCII arguments in +:mod:`pickletools`. :func:`pickletools.dis` now outputs non-ASCII bytes in +STRING, BINSTRING and SHORT_BINSTRING arguments as escaped (``\xXX``). + +.. + +.. date: 2024-11-20-11-37-08 +.. gh-issue: 126316 +.. nonce: ElkZmE +.. section: Library + +:mod:`grp`: Make :func:`grp.getgrall` thread-safe by adding a mutex. Patch +by Victor Stinner. + +.. + +.. date: 2024-11-20-08-54-11 +.. gh-issue: 126618 +.. nonce: ef_53g +.. section: Library + +Fix the representation of :class:`itertools.count` objects when the count +value is :data:`sys.maxsize`. + +.. + +.. date: 2024-11-19-14-34-05 +.. gh-issue: 126615 +.. nonce: LOskwi +.. section: Library + +The :exc:`~ctypes.COMError` exception is now public. Previously, this was +private and only available in ``_ctypes``. + +.. + +.. date: 2024-11-18-23-42-06 +.. gh-issue: 126985 +.. nonce: 7XplY9 +.. section: Library + +When running under a virtual environment with the :mod:`site` disabled (see +:option:`-S`), :data:`sys.prefix` and :data:`sys.base_prefix` will now point +to the virtual environment, instead of the base installation. + +.. + +.. date: 2024-11-18-23-18-27 +.. gh-issue: 112192 +.. nonce: DRdRgP +.. section: Library + +In the :mod:`trace` module, increase the coverage precision (``cov%``) to +one decimal. + +.. + +.. date: 2024-11-18-22-02-47 +.. gh-issue: 118761 +.. nonce: GQKD_J +.. section: Library + +Improve import time of :mod:`mimetypes` by around 11-16 times. Patch by Hugo +van Kemenade. + +.. + +.. date: 2024-11-18-19-03-46 +.. gh-issue: 126947 +.. nonce: NiDYUe +.. section: Library + +Raise :exc:`TypeError` in :meth:`!_pydatetime.timedelta.__new__` if the +passed arguments are not :class:`int` or :class:`float`, so that the Python +implementation is in line with the C implementation. + +.. + +.. date: 2024-11-18-16-43-11 +.. gh-issue: 126946 +.. nonce: 52Ou-B +.. section: Library + +Improve the :exc:`~getopt.GetoptError` error message when a long option +prefix matches multiple accepted options in :func:`getopt.getopt` and +:func:`getopt.gnu_getopt`. + +.. + +.. date: 2024-11-16-10-52-48 +.. gh-issue: 126899 +.. nonce: GFnfBt +.. section: Library + +Make tkinter widget methods :meth:`!after` and :meth:`!after_idle` accept +arguments passed by keyword. + +.. + +.. date: 2024-11-15-01-50-36 +.. gh-issue: 85168 +.. nonce: bP8VIN +.. section: Library + +Fix issue where :func:`urllib.request.url2pathname` and +:func:`~urllib.request.pathname2url` always used UTF-8 when quoting and +unquoting file URIs. They now use the :term:`filesystem encoding and error +handler`. + +.. + +.. date: 2024-11-13-19-15-18 +.. gh-issue: 126780 +.. nonce: ZZqJvI +.. section: Library + +Fix :func:`os.path.normpath` for drive-relative paths on Windows. + +.. + +.. date: 2024-11-13-10-44-25 +.. gh-issue: 126775 +.. nonce: a3ubjh +.. section: Library + +Make :func:`linecache.checkcache` thread safe and GC re-entrancy safe. + +.. + +.. date: 2024-11-12-20-05-09 +.. gh-issue: 126601 +.. nonce: Nj7bA9 +.. section: Library + +Fix issue where :func:`urllib.request.pathname2url` raised :exc:`OSError` +when given a Windows path containing a colon character not following a drive +letter, such as before an NTFS alternate data stream. + +.. + +.. date: 2024-11-12-13-14-47 +.. gh-issue: 126727 +.. nonce: 5Eqfqd +.. section: Library + +``locale.nl_langinfo(locale.ERA)`` now returns multiple era description +segments separated by semicolons. Previously it only returned the first +segment on platforms with Glibc. + +.. + +.. date: 2024-11-04-22-02-30 +.. gh-issue: 85046 +.. nonce: Y5d_ZN +.. section: Library + +Add :data:`~errno.EHWPOISON` error code to :mod:`errno`. + +.. + +.. date: 2024-10-28-19-49-18 +.. gh-issue: 118201 +.. nonce: v41XXh +.. section: Library + +Fixed intermittent failures of :any:`os.confstr`, :any:`os.pathconf` and +:any:`os.sysconf` on iOS and Android. + +.. + +.. date: 2024-10-23-20-05-54 +.. gh-issue: 86463 +.. nonce: jvFTI_ +.. section: Library + +The ``usage`` parameter of :class:`argparse.ArgumentParser` no longer +affects the default value of the ``prog`` parameter in subparsers. + +.. + +.. date: 2024-09-13-18-24-27 +.. gh-issue: 124008 +.. nonce: XaiPQx +.. section: Library + +Fix possible crash (in debug build), incorrect output or returning incorrect +value from raw binary ``write()`` when writing to console on Windows. + +.. + +.. date: 2024-08-27-18-58-01 +.. gh-issue: 123401 +.. nonce: t4-FpI +.. section: Library + +The :mod:`http.cookies` module now supports parsing obsolete :rfc:`850` date +formats, in accordance with :rfc:`9110` requirements. Patch by Nano Zheng. + +.. + +.. date: 2024-07-30-11-37-40 +.. gh-issue: 122431 +.. nonce: lAzVtu +.. section: Library + +:func:`readline.append_history_file` now raises a :exc:`ValueError` when +given a negative value. + +.. + +.. date: 2024-07-29-15-20-30 +.. gh-issue: 122356 +.. nonce: wKCmFx +.. section: Library + +Guarantee that the position of a file-like object passed to +:func:`zipfile.is_zipfile` is left untouched after the call. Patch by +Bénédikt Tran. + +.. + +.. date: 2024-07-25-18-06-51 +.. gh-issue: 122288 +.. nonce: -_xxOR +.. section: Library + +Improve the performances of :func:`fnmatch.translate` by a factor 1.7. Patch +by Bénédikt Tran. + +.. + +.. date: 2023-02-15-23-54-42 +.. gh-issue: 88110 +.. nonce: KU6erv +.. section: Library + +Fixed :class:`multiprocessing.Process` reporting a ``.exitcode`` of 1 even +on success when using the ``"fork"`` start method while using a +:class:`concurrent.futures.ThreadPoolExecutor`. + +.. + +.. date: 2022-11-10-17-16-45 +.. gh-issue: 97514 +.. nonce: kzA0zl +.. section: Library + +Authentication was added to the :mod:`multiprocessing` forkserver start +method control socket so that only processes with the authentication key +generated by the process that spawned the forkserver can control it. This +is an enhancement over the other :gh:`97514` fixes so that access is no +longer limited only by filesystem permissions. + +The file descriptor exchange of control pipes with the forked worker process +now requires an explicit acknowledgement byte to be sent over the socket +after the exchange on all forkserver supporting platforms. That makes +testing the above much easier. + +.. + +.. date: 2024-11-27-22-56-48 +.. gh-issue: 127347 +.. nonce: xyddWS +.. section: Documentation + +Publicly expose :func:`traceback.print_list` in :attr:`!traceback.__all__`. + +.. + +.. date: 2024-12-10-21-08-05 +.. gh-issue: 127740 +.. nonce: 0tWC9h +.. section: Core and Builtins + +Fix error message in :func:`bytes.fromhex` when given an odd number of +digits to properly indicate that an even number of hexadecimal digits is +required. + +.. + +.. date: 2024-12-09-11-29-10 +.. gh-issue: 127058 +.. nonce: pqtBcZ +.. section: Core and Builtins + +``PySequence_Tuple`` now creates the resulting tuple atomically, preventing +partially created tuples being visible to the garbage collector or through +``gc.get_referrers()`` + +.. + +.. date: 2024-12-07-13-06-09 +.. gh-issue: 127599 +.. nonce: tXCZb_ +.. section: Core and Builtins + +Fix statistics for increments of object reference counts (in particular, +when a reference count was increased by more than 1 in a single operation). + +.. + +.. date: 2024-12-06-01-09-40 +.. gh-issue: 127651 +.. nonce: 80cm6j +.. section: Core and Builtins + +When raising :exc:`ImportError` for missing symbols in ``from`` imports, use +``__file__`` in the error message if ``__spec__.origin`` is not a location + +.. + +.. date: 2024-12-05-19-25-00 +.. gh-issue: 127582 +.. nonce: ogUY2a +.. section: Core and Builtins + +Fix non-thread-safe object resurrection when calling finalizers and watcher +callbacks in the free threading build. + +.. + +.. date: 2024-12-04-09-52-08 +.. gh-issue: 127434 +.. nonce: RjkGT_ +.. section: Core and Builtins + +The iOS compiler shims can now accept arguments with spaces. + +.. + +.. date: 2024-12-03-21-07-06 +.. gh-issue: 127536 +.. nonce: 3jMMrT +.. section: Core and Builtins + +Add missing locks around some list assignment operations in the free +threading build. + +.. + +.. date: 2024-11-30-23-35-45 +.. gh-issue: 127085 +.. nonce: KLKylb +.. section: Core and Builtins + +Fix race when exporting a buffer from a :class:`memoryview` object on the +:term:`free-threaded ` build. + +.. + +.. date: 2024-11-25-05-15-21 +.. gh-issue: 127238 +.. nonce: O8wkH- +.. section: Core and Builtins + +Correct error message for :func:`sys.set_int_max_str_digits`. + +.. + +.. date: 2024-11-24-07-01-28 +.. gh-issue: 113841 +.. nonce: WFg-Bu +.. section: Core and Builtins + +Fix possible undefined behavior division by zero in :class:`complex`'s +:c:func:`_Py_c_pow`. + +.. + +.. date: 2024-11-23-04-54-42 +.. gh-issue: 127133 +.. nonce: WMoJjF +.. section: Core and Builtins + +Calling :meth:`argparse.ArgumentParser.add_argument_group` on an argument +group, and calling :meth:`argparse.ArgumentParser.add_argument_group` or +:meth:`argparse.ArgumentParser.add_mutually_exclusive_group` on a mutually +exclusive group now raise exceptions. This nesting was never supported, +often failed to work correctly, and was unintentionally exposed through +inheritance. This functionality has been deprecated since Python 3.11. + +.. + +.. date: 2024-11-21-16-13-52 +.. gh-issue: 126491 +.. nonce: 0YvL94 +.. section: Core and Builtins + +Add a marking phase to the GC. All objects that can be transitively reached +from builtin modules or the stacks are marked as reachable before cycle +detection. This reduces the amount of work done by the GC by approximately +half. + +.. + +.. date: 2024-11-19-21-49-58 +.. gh-issue: 127020 +.. nonce: 5vvI17 +.. section: Core and Builtins + +Fix a crash in the free threading build when :c:func:`PyCode_GetCode`, +:c:func:`PyCode_GetVarnames`, :c:func:`PyCode_GetCellvars`, or +:c:func:`PyCode_GetFreevars` were called from multiple threads at the same +time. + +.. + +.. date: 2024-11-19-17-17-32 +.. gh-issue: 127010 +.. nonce: 9Cl4bb +.. section: Core and Builtins + +Simplify GC tracking of dictionaries. All dictionaries are tracked when +created, rather than being lazily tracked when a trackable object was added +to them. This simplifies the code considerably and results in a slight +speedup. + +.. + +.. date: 2024-11-18-23-18-17 +.. gh-issue: 126980 +.. nonce: r8QHdi +.. section: Core and Builtins + +Fix :meth:`~object.__buffer__` of :class:`bytearray` crashing when +:attr:`~inspect.BufferFlags.READ` or :attr:`~inspect.BufferFlags.WRITE` are +passed as flags. + +.. + +.. date: 2024-11-17-21-35-55 +.. gh-issue: 126937 +.. nonce: qluVM0 +.. section: Core and Builtins + +Fix :exc:`TypeError` when a :class:`ctypes.Structure` has a field size that +doesn't fit into an unsigned 16-bit integer. Instead, the maximum number of +*bits* is :data:`sys.maxsize`. + +.. + +.. date: 2024-11-16-22-37-46 +.. gh-issue: 126868 +.. nonce: yOoHSY +.. section: Core and Builtins + +Increase performance of :class:`int` by adding a freelist for compact ints. + +.. + +.. date: 2024-11-16-11-11-35 +.. gh-issue: 126881 +.. nonce: ijofLZ +.. section: Core and Builtins + +Fix crash in finalization of dtoa state. Patch by Kumar Aditya. + +.. + +.. date: 2024-11-15-16-39-37 +.. gh-issue: 126892 +.. nonce: QR6Yo3 +.. section: Core and Builtins + +Require cold or invalidated code to "warm up" before being JIT compiled +again. + +.. + +.. date: 2024-11-07-21-48-23 +.. gh-issue: 126091 +.. nonce: ETaRGE +.. section: Core and Builtins + +Ensure stack traces are complete when throwing into a generator chain that +ends in a custom generator. + +.. + +.. date: 2024-10-27-04-47-28 +.. gh-issue: 126024 +.. nonce: XCQSqT +.. section: Core and Builtins + +Optimize decoding of short UTF-8 sequences containing non-ASCII characters +by approximately 15%. + +.. + +.. date: 2024-10-14-13-28-16 +.. gh-issue: 125420 +.. nonce: hNKixM +.. section: Core and Builtins + +Add :meth:`memoryview.index` to :class:`memoryview` objects. Patch by +Bénédikt Tran. + +.. + +.. date: 2024-10-14-12-34-51 +.. gh-issue: 125420 +.. nonce: jABXoZ +.. section: Core and Builtins + +Add :meth:`memoryview.count` to :class:`memoryview` objects. Patch by +Bénédikt Tran. + +.. + +.. date: 2024-09-25-21-50-23 +.. gh-issue: 124470 +.. nonce: pFr3_d +.. section: Core and Builtins + +Fix crash in free-threaded builds when replacing object dictionary while +reading attribute on another thread + +.. + +.. date: 2024-08-03-14-02-27 +.. gh-issue: 69639 +.. nonce: mW3iKq +.. section: Core and Builtins + +Implement mixed-mode arithmetic rules combining real and complex numbers as +specified by C standards since C99. Patch by Sergey B Kirpichev. + +.. + +.. date: 2024-06-04-08-26-25 +.. gh-issue: 120010 +.. nonce: _z-AWz +.. section: Core and Builtins + +Correct invalid corner cases which resulted in ``(nan+nanj)`` output in +complex multiplication, e.g., ``(1e300+1j)*(nan+infj)``. Patch by Sergey B +Kirpichev. + +.. + +.. date: 2023-09-22-21-01-56 +.. gh-issue: 109746 +.. nonce: 32MHt9 +.. section: Core and Builtins + +If :func:`!_thread.start_new_thread` fails to start a new thread, it deletes +its state from interpreter and thus avoids its repeated cleanup on +finalization. + +.. + +.. date: 2024-12-16-07-12-15 +.. gh-issue: 127896 +.. nonce: HmI9pk +.. section: C API + +The previously undocumented function :c:func:`PySequence_In` is :term:`soft +deprecated`. Use :c:func:`PySequence_Contains` instead. + +.. + +.. date: 2024-12-10-14-25-22 +.. gh-issue: 127791 +.. nonce: YRw4GU +.. section: C API + +Fix loss of callbacks after more than one call to +:c:func:`PyUnstable_AtExit`. + +.. + +.. date: 2024-12-06-16-53-34 +.. gh-issue: 127691 +.. nonce: k_Jitp +.. section: C API + +The :ref:`Unicode Exception Objects ` C API now raises a +:exc:`TypeError` if its exception argument is not a :exc:`UnicodeError` +object. Patch by Bénédikt Tran. + +.. + +.. date: 2024-12-02-16-10-36 +.. gh-issue: 123378 +.. nonce: Q6YRwe +.. section: C API + +Ensure that the value of :attr:`UnicodeEncodeError.end ` +retrieved by :c:func:`PyUnicodeEncodeError_GetEnd` lies in ``[min(1, +objlen), max(min(1, objlen), objlen)]`` where *objlen* is the length of +:attr:`UnicodeEncodeError.object `. Similar arguments +apply to :exc:`UnicodeDecodeError` and :exc:`UnicodeTranslateError` and +their corresponding C interface. Patch by Bénédikt Tran. + +.. + +.. date: 2024-11-26-22-06-10 +.. gh-issue: 127314 +.. nonce: SsRrIu +.. section: C API + +Improve error message when calling the C API without an active thread state +on the :term:`free-threaded ` build. + +.. + +.. date: 2024-08-27-09-07-56 +.. gh-issue: 123378 +.. nonce: JJ6n_u +.. section: C API + +Ensure that the value of :attr:`UnicodeEncodeError.start +` retrieved by :c:func:`PyUnicodeEncodeError_GetStart` +lies in ``[0, max(0, objlen - 1)]`` where *objlen* is the length of +:attr:`UnicodeEncodeError.object `. Similar arguments +apply to :exc:`UnicodeDecodeError` and :exc:`UnicodeTranslateError` and +their corresponding C interface. Patch by Bénédikt Tran. + +.. + +.. date: 2024-08-12-10-15-19 +.. gh-issue: 109523 +.. nonce: S2c3fi +.. section: C API + +Reading text from a non-blocking stream with ``read`` may now raise a +:exc:`BlockingIOError` if the operation cannot immediately return bytes. + +.. + +.. date: 2024-07-03-17-26-53 +.. gh-issue: 102471 +.. nonce: XpmKYk +.. section: C API + +Add a new import and export API for Python :class:`int` objects +(:pep:`757`): + +* :c:func:`PyLong_GetNativeLayout`; +* :c:func:`PyLong_Export`; +* :c:func:`PyLong_FreeExport`; +* :c:func:`PyLongWriter_Create`; +* :c:func:`PyLongWriter_Finish`; +* :c:func:`PyLongWriter_Discard`. + +Patch by Victor Stinner. + +.. + +.. date: 2024-07-03-13-39-13 +.. gh-issue: 121058 +.. nonce: MKi1MV +.. section: C API + +``PyThreadState_Clear()`` now warns (and calls ``sys.excepthook``) if the +thread state still has an active exception. + +.. + +.. date: 2024-12-12-17-21-45 +.. gh-issue: 127865 +.. nonce: 30GDzs +.. section: Build + +Fix build failure on systems without thread-locals support. + +.. + +.. date: 2024-12-06-12-47-52 +.. gh-issue: 127629 +.. nonce: tD-ERQ +.. section: Build + +Emscripten builds now include ctypes support. + +.. + +.. date: 2024-11-30-16-36-09 +.. gh-issue: 127111 +.. nonce: QI9mMZ +.. section: Build + +Updated the Emscripten web example to use ES6 modules and be built into a +distinct ``web_example`` subfolder. + +.. + +.. date: 2024-11-22-08-46-46 +.. gh-issue: 115869 +.. nonce: UVLSKd +.. section: Build + +Make ``jit_stencils.h`` (which is produced during JIT builds) reproducible. + +.. + +.. date: 2024-11-20-17-12-40 +.. gh-issue: 126898 +.. nonce: I2zILt +.. section: Build + +The Emscripten build of Python is now based on ES6 modules. diff --git a/Misc/NEWS.d/3.14.0a4.rst b/Misc/NEWS.d/3.14.0a4.rst new file mode 100644 index 00000000000000..1e08b36020386c --- /dev/null +++ b/Misc/NEWS.d/3.14.0a4.rst @@ -0,0 +1,830 @@ +.. date: 2024-12-22-08-54-30 +.. gh-issue: 127592 +.. nonce: iyuFCC +.. release date: 2025-01-14 +.. section: macOS + +Usage of the unified Apple System Log APIs was disabled when the minimum +macOS version is earlier than 10.12. + +.. + +.. date: 2025-01-03-23-51-07 +.. gh-issue: 128152 +.. nonce: IhzElS +.. section: Tools/Demos + +Fix a bug where Argument Clinic's C pre-processor parser tried to parse +pre-processor directives inside C comments. Patch by Erlend Aasland. + +.. + +.. date: 2025-01-13-01-29-08 +.. gh-issue: 128690 +.. nonce: cPFVDb +.. section: Tests + +Temporarily do not use test_embed in PGO profile builds until the problem +with test_init_pyvenv_cfg failing in some configurations is resolved. + +.. + +.. date: 2025-01-11-13-40-12 +.. gh-issue: 128731 +.. nonce: qpKlai +.. section: Library + +Fix :exc:`ResourceWarning` in +:meth:`urllib.robotparser.RobotFileParser.read`. + +.. + +.. date: 2025-01-10-15-06-45 +.. gh-issue: 71339 +.. nonce: EKnpzw +.. section: Library + +Add new assertion methods for :mod:`unittest`: +:meth:`~unittest.TestCase.assertHasAttr`, +:meth:`~unittest.TestCase.assertNotHasAttr`, +:meth:`~unittest.TestCase.assertIsSubclass`, +:meth:`~unittest.TestCase.assertNotIsSubclass` +:meth:`~unittest.TestCase.assertStartsWith`, +:meth:`~unittest.TestCase.assertNotStartsWith`, +:meth:`~unittest.TestCase.assertEndsWith` and +:meth:`~unittest.TestCase.assertNotEndsWith`. + +.. + +.. date: 2025-01-10-13-34-33 +.. gh-issue: 118761 +.. nonce: qRB8nS +.. section: Library + +Improve import time of :mod:`pickle` by 25% by removing an unnecessary +regular expression. As such, :mod:`re` is no more implicitly available as +``pickle.re``. Patch by Bénédikt Tran. + +.. + +.. date: 2025-01-09-12-06-52 +.. gh-issue: 128661 +.. nonce: ixx_0z +.. section: Library + +Fixes :func:`typing.evaluate_forward_ref` not showing deprecation when +``type_params`` arg is not passed. + +.. + +.. date: 2025-01-08-03-09-29 +.. gh-issue: 128562 +.. nonce: Mlv-yO +.. section: Library + +Fix possible conflicts in generated :mod:`tkinter` widget names if the +widget class name ends with a digit. + +.. + +.. date: 2025-01-06-21-35-00 +.. gh-issue: 128559 +.. nonce: 6fxcDM +.. section: Library + +Improved import time of :mod:`asyncio`. + +.. + +.. date: 2025-01-06-18-41-08 +.. gh-issue: 128552 +.. nonce: fV-f8j +.. section: Library + +Fix cyclic garbage introduced by :meth:`asyncio.loop.create_task` and +:meth:`asyncio.TaskGroup.create_task` holding a reference to the created +task if it is eager. + +.. + +.. date: 2025-01-05-11-46-14 +.. gh-issue: 128340 +.. nonce: gKI0uU +.. section: Library + +Add internal thread safe handle to be used in +:meth:`asyncio.loop.call_soon_threadsafe` for thread safe cancellation. + +.. + +.. date: 2025-01-04-11-32-46 +.. gh-issue: 128182 +.. nonce: SJ2Zsa +.. section: Library + +Fix crash when using :mod:`ctypes` pointers concurrently on the :term:`free +threaded ` build. + +.. + +.. date: 2025-01-02-15-20-17 +.. gh-issue: 128400 +.. nonce: UMiG4f +.. section: Library + +Only show the current thread in :mod:`faulthandler` on the :term:`free +threaded ` build to prevent races. + +.. + +.. date: 2025-01-02-13-05-16 +.. gh-issue: 128400 +.. nonce: 5N43fF +.. section: Library + +Fix crash when using :func:`faulthandler.dump_traceback` while other threads +are active on the :term:`free threaded ` build. + +.. + +.. date: 2025-01-01-19-24-43 +.. gh-issue: 128388 +.. nonce: 8UdMz_ +.. section: Library + +Fix ``PyREPL`` on Windows to support more keybindings, like the +:kbd:`Control-←` and :kbd:`Control-→` word-skipping keybindings and those +with meta (i.e. :kbd:`Alt`), e.g. :kbd:`Alt-d` to ``kill-word`` or +:kbd:`Alt-Backspace` ``backward-kill-word``. + +.. + +.. date: 2024-12-30-20-48-28 +.. gh-issue: 88834 +.. nonce: RIvgwc +.. section: Library + +Unify the instance check for :class:`typing.Union` and +:class:`types.UnionType`: :class:`!Union` now uses the instance checks +against its parameters instead of the subclass checks. + +.. + +.. date: 2024-12-29-13-49-46 +.. gh-issue: 128302 +.. nonce: psRpPN +.. section: Library + +Fix :meth:`!xml.dom.xmlbuilder.DOMEntityResolver.resolveEntity`, which was +broken by the Python 3.0 transition. + +.. + +.. date: 2024-12-29-00-33-34 +.. gh-issue: 128317 +.. nonce: WgFina +.. section: Library + +Highlight today in colour in :mod:`calendar`'s CLI output. Patch by Hugo van +Kemenade. + +.. + +.. date: 2024-12-27-16-28-57 +.. gh-issue: 128302 +.. nonce: 2GMvyl +.. section: Library + +Allow :meth:`!xml.dom.xmlbuilder.DOMParser.parse` to correctly handle +:class:`!xml.dom.xmlbuilder.DOMInputSource` instances that only have a +:attr:`!systemId` attribute set. + +.. + +.. date: 2024-12-21-11-12-50 +.. gh-issue: 128151 +.. nonce: aq7vpG +.. section: Library + +Improve generation of :class:`~uuid.UUID` objects version 3, 4, 5, and 8 via +their dedicated functions by 30%. Patch by Bénédikt Tran. + +.. + +.. date: 2024-12-20-10-57-10 +.. gh-issue: 128118 +.. nonce: mYak8i +.. section: Library + +Improve performance of :func:`copy.copy` by 30% via a fast path for atomic +types and container types. + +.. + +.. date: 2024-12-19-20-46-01 +.. gh-issue: 127946 +.. nonce: 4lM3Op +.. section: Library + +Fix crash when modifying :class:`ctypes._CFuncPtr` objects concurrently on +the :term:`free threaded ` build. + +.. + +.. date: 2024-12-18-10-18-55 +.. gh-issue: 128062 +.. nonce: E9oU7- +.. section: Library + +Revert the font of :mod:`turtledemo`'s menu bar to its default value and +display the shortcut keys in the correct position. + +.. + +.. date: 2024-12-18-00-07-50 +.. gh-issue: 128014 +.. nonce: F3aUbz +.. section: Library + +Fix resetting the default window icon by passing ``default=''`` to the +:mod:`tkinter` method :meth:`!wm_iconbitmap`. + +.. + +.. date: 2024-12-17-15-23-40 +.. gh-issue: 41872 +.. nonce: 31LjKY +.. section: Library + +Fix quick extraction of module docstrings from a file in :mod:`pydoc`. It +now supports docstrings with single quotes, escape sequences, raw string +literals, and other Python syntax. + +.. + +.. date: 2024-12-17-13-21-52 +.. gh-issue: 127060 +.. nonce: mv2bX6 +.. section: Library + +Set TERM environment variable to "dumb" to disable traceback colors in IDLE, +since IDLE doesn't understand ANSI escape sequences. Patch by Victor +Stinner. + +.. + +.. date: 2024-12-17-12-41-07 +.. gh-issue: 126742 +.. nonce: l07qvT +.. section: Library + +Fix support of localized error messages reported by :manpage:`dlerror(3)` +and :manpage:`gdbm_strerror ` in :mod:`ctypes` and :mod:`dbm.gnu` +functions respectively. Patch by Bénédikt Tran. + +.. + +.. date: 2024-12-13-14-21-04 +.. gh-issue: 122548 +.. nonce: hq3Vud +.. section: Library + +Adds two new local events to sys.monitoring, ``BRANCH_LEFT`` and +``BRANCH_RIGHT``. This allows the two arms of the branch to be disabled +independently, which should hugely improve performance of branch-level +coverage tools. The old branch event, ``BRANCH`` is now deprecated. + +.. + +.. date: 2024-12-12-07-27-51 +.. gh-issue: 127847 +.. nonce: ksfNKM +.. section: Library + +Fix the position when doing interleaved seeks and reads in uncompressed, +unencrypted zip files returned by :meth:`zipfile.ZipFile.open`. + +.. + +.. date: 2024-12-06-21-03-11 +.. gh-issue: 127688 +.. nonce: NJqtc- +.. section: Library + +Add the :data:`~os.SCHED_DEADLINE` and :data:`~os.SCHED_NORMAL` constants to +the :mod:`os` module. + +.. + +.. date: 2024-12-04-10-39-29 +.. gh-issue: 83662 +.. nonce: CG1s3m +.. section: Library + +Add missing ``__class_getitem__`` method to the Python implementation of +:func:`functools.partial`, to make it compatible with the C version. This is +mainly relevant for alternative Python implementations like PyPy and +GraalPy, because CPython will usually use the C-implementation of that +function. + +.. + +.. date: 2024-12-03-20-28-08 +.. gh-issue: 127586 +.. nonce: zgotYF +.. section: Library + +:class:`multiprocessing.pool.Pool` now properly restores blocked signal +handlers of the parent thread when creating processes via either *spawn* or +*forkserver*. + +.. + +.. date: 2024-12-03-14-45-16 +.. gh-issue: 98188 +.. nonce: GX9i2b +.. section: Library + +Fix an issue in :meth:`email.message.Message.get_payload` where data cannot +be decoded if the Content Transfer Encoding mechanism contains trailing +whitespaces or additional junk text. Patch by Hui Liu. + +.. + +.. date: 2024-12-02-19-13-19 +.. gh-issue: 127529 +.. nonce: Pj1Xtf +.. section: Library + +Correct behavior of +:func:`!asyncio.selector_events.BaseSelectorEventLoop._accept_connection` in +handling :exc:`ConnectionAbortedError` in a loop. This improves performance +on OpenBSD. + +.. + +.. date: 2024-11-28-14-24-12 +.. gh-issue: 127360 +.. nonce: HVKt-c +.. section: Library + +When a descriptive error message cannot be provided for an +:exc:`ssl.SSLError`, the "unknown error" message now shows the internal +error code (as retrieved by ``ERR_get_error`` and similar OpenSSL +functions). + +.. + +.. date: 2024-11-24-14-53-35 +.. gh-issue: 127196 +.. nonce: 8CBkUa +.. section: Library + +Fix crash when dict with keys in invalid encoding were passed to several +functions in ``_interpreters`` module. + +.. + +.. date: 2024-11-19-10-46-57 +.. gh-issue: 124130 +.. nonce: OZ_vR5 +.. section: Library + +Fix a bug in matching regular expression ``\B`` in empty input string. Now +it is always the opposite of ``\b``. To get an old behavior, use +``(?!\A\Z)\B``. To get a new behavior in old Python versions, use +``(?!\b)``. + +.. + +.. date: 2024-11-11-07-56-03 +.. gh-issue: 126639 +.. nonce: AmVSt- +.. section: Library + +:class:`tempfile.NamedTemporaryFile` will now issue a :exc:`ResourceWarning` +when it is finalized by the garbage collector without being explicitly +closed. + +.. + +.. date: 2024-11-09-15-59-51 +.. gh-issue: 126624 +.. nonce: bN53Va +.. section: Library + +Expose error code :data:`~xml.parsers.expat.errors.XML_ERROR_NOT_STARTED` of +Expat >=2.6.4 in :mod:`xml.parsers.expat.errors`. + +.. + +.. date: 2024-10-31-14-31-36 +.. gh-issue: 126225 +.. nonce: vTxGXm +.. section: Library + +:mod:`getopt` and :mod:`optparse` are no longer marked as deprecated. There +are legitimate reasons to use one of these modules in preference to +:mod:`argparse`, and none of these modules are at risk of being removed from +the standard library. Of the three, ``argparse`` remains the recommended +default choice, *unless* one of the concerns noted at the top of the +``optparse`` module documentation applies. + +.. + +.. date: 2024-10-04-09-56-45 +.. gh-issue: 124761 +.. nonce: N4pSD6 +.. section: Library + +Add :data:`~socket.SO_REUSEPORT_LB` constant to :mod:`socket` for FreeBSD. + +.. + +.. date: 2024-09-04-14-13-14 +.. gh-issue: 121720 +.. nonce: z9hhXQ +.. section: Library + +:class:`enum.EnumDict` can now be used without resorting to private API. + +.. + +.. date: 2024-08-28-16-10-37 +.. gh-issue: 123424 +.. nonce: u96_i6 +.. section: Library + +Add :meth:`zipfile.ZipInfo._for_archive` setting default properties on +:class:`~zipfile.ZipInfo` objects. Patch by Bénédikt Tran and Jason R. +Coombs. + +.. + +.. date: 2024-07-13-13-25-31 +.. gh-issue: 121676 +.. nonce: KDLS11 +.. section: Library + +Deprecate calling the Python implementation of :meth:`functools.reduce` with +a ``function`` or ``sequence`` as a :term:`keyword argument`. This will be +forbidden in Python 3.16 in order to match the C implementation. + +.. + +.. date: 2023-11-12-21-53-40 +.. gh-issue: 112015 +.. nonce: 2WPRxE +.. section: Library + +:func:`ctypes.memoryview_at` now exists to create a :class:`memoryview` +object that refers to the supplied pointer and length. This works like +:func:`ctypes.string_at` except it avoids a buffer copy, and is typically +useful when implementing pure Python callback functions that are passed +dynamically-sized buffers. + +.. + +.. date: 2022-07-28-12-32-59 +.. gh-issue: 95371 +.. nonce: F24IFC +.. section: Library + +Added support for other image formats (PNG, PGM, and PPM) to the turtle +module. Patch by Shin-myoung-serp. + +.. + +.. date: 2025-01-13-12-48-30 +.. gh-issue: 128078 +.. nonce: qOsl9B +.. section: Core and Builtins + +Fix a :exc:`SystemError` when using :func:`anext` with a default tuple +value. Patch by Bénédikt Tran. + +.. + +.. date: 2025-01-11-12-39-17 +.. gh-issue: 128717 +.. nonce: i65d06 +.. section: Core and Builtins + +Fix a crash when setting the recursion limit while other threads are active +on the :term:`free threaded ` build. + +.. + +.. date: 2025-01-09-11-46-57 +.. gh-issue: 124483 +.. nonce: KRtBeQ +.. section: Core and Builtins + +Treat ``Py_DECREF`` and variants as escaping when generating opcode and uop +metadata. This prevents the possibility of a ``__del__`` method causing the +JIT to behave incorrectly. + +.. + +.. date: 2025-01-07-19-48-56 +.. gh-issue: 126703 +.. nonce: 0ISs-7 +.. section: Core and Builtins + +Improve performance of class methods by using a freelist. + +.. + +.. date: 2024-12-24-01-40-12 +.. gh-issue: 128137 +.. nonce: gsTwr_ +.. section: Core and Builtins + +Update :c:type:`PyASCIIObject` layout to handle interned field with the +atomic operation. Patch by Donghee Na. + +.. + +.. date: 2024-12-23-11-14-07 +.. gh-issue: 128192 +.. nonce: 02mEhD +.. section: Core and Builtins + +Upgrade HTTP digest authentication algorithm for :mod:`urllib.request` by +supporting SHA-256 digest authentication as specified in :rfc:`7616`. + +.. + +.. date: 2024-12-22-15-47-44 +.. gh-issue: 126868 +.. nonce: RpjKez +.. section: Core and Builtins + +Increase usage of freelist for :class:`int` allocation. + +.. + +.. date: 2024-12-20-23-07-33 +.. gh-issue: 114203 +.. nonce: 84NgoW +.. section: Core and Builtins + +Optimize ``Py_BEGIN_CRITICAL_SECTION`` for simple recursive calls. + +.. + +.. date: 2024-12-20-12-25-16 +.. gh-issue: 127705 +.. nonce: WmCz1z +.. section: Core and Builtins + +Adds stackref debugging when ``Py_STACKREF_DEBUG`` is set. Finds all +double-closes and leaks, logging the origin and last borrow. + +Inspired by HPy's debug mode. +https://docs.hpyproject.org/en/latest/debug-mode.html + +.. + +.. date: 2024-12-18-14-22-48 +.. gh-issue: 128079 +.. nonce: SUD5le +.. section: Core and Builtins + +Fix a bug where :keyword:`except* ` does not properly check the +return value of an :exc:`ExceptionGroup`'s :meth:`~BaseExceptionGroup.split` +function, leading to a crash in some cases. Now when +:meth:`~BaseExceptionGroup.split` returns an invalid object, +:keyword:`except* ` raises a :exc:`TypeError` with the original +raised :exc:`ExceptionGroup` object chained to it. + +.. + +.. date: 2024-12-17-22-28-15 +.. gh-issue: 128030 +.. nonce: H1ptOD +.. section: Core and Builtins + +Avoid error from calling ``PyModule_GetFilenameObject`` on a non-module +object when importing a non-existent symbol from a non-module object. + +.. + +.. date: 2024-12-17-18-20-37 +.. gh-issue: 128035 +.. nonce: JwqHdB +.. section: Core and Builtins + +Indicate through :data:`ssl.HAS_PHA` whether the :mod:`ssl` module supports +TLSv1.3 post-handshake client authentication (PHA). Patch by Will +Childs-Klein. + +.. + +.. date: 2024-12-17-13-45-33 +.. gh-issue: 127274 +.. nonce: deNxNC +.. section: Core and Builtins + +Add a new flag, ``CO_METHOD``, to :attr:`~codeobject.co_flags` that +indicates whether the code object belongs to a function defined in class +scope. + +.. + +.. date: 2024-12-15-21-11-26 +.. gh-issue: 66409 +.. nonce: wv109z +.. section: Core and Builtins + +During the :ref:`path initialization `, we now check if +``base_exec_prefix`` is the same as ``base_prefix`` before falling back to +searching the Python interpreter directory. + +.. + +.. date: 2024-12-15-19-51-54 +.. gh-issue: 127970 +.. nonce: vdUp-y +.. section: Core and Builtins + +We now use the location of the ``libpython`` runtime library used in the +current proccess to determine :data:`sys.base_prefix` on all platforms +implementing the `dladdr +`_ +function defined by the UNIX standard — this includes Linux, Android, macOS, +iOS, FreeBSD, etc. This was already the case on Windows and macOS Framework +builds. + +.. + +.. date: 2024-12-13-15-21-45 +.. gh-issue: 127773 +.. nonce: E-DZR4 +.. section: Core and Builtins + +Do not use the type attribute cache for types with incompatible :term:`MRO`. + +.. + +.. date: 2024-12-13-14-17-24 +.. gh-issue: 127903 +.. nonce: vemHSl +.. section: Core and Builtins + +``Objects/unicodeobject.c``: fix a crash on DEBUG builds in +``_copy_characters`` when there is nothing to copy. + +.. + +.. date: 2024-12-11-14-32-22 +.. gh-issue: 127809 +.. nonce: 0W8khe +.. section: Core and Builtins + +Fix an issue where the experimental JIT may infer an incorrect result type +for exponentiation (``**`` and ``**=``), leading to bugs or crashes. + +.. + +.. date: 2024-12-02-18-15-37 +.. gh-issue: 126862 +.. nonce: fdIK7T +.. section: Core and Builtins + +Fix a possible overflow when a class inherits from an absurd number of +super-classes. Reported by Valery Fedorenko. Patch by Bénédikt Tran. + +.. + +.. date: 2025-01-12-12-19-51 +.. gh-issue: 128400 +.. nonce: OwoIDw +.. section: C API + +:c:func:`Py_FatalError` no longer shows all threads on the :term:`free +threaded ` build to prevent crashes. + +.. + +.. date: 2025-01-08-13-13-18 +.. gh-issue: 128629 +.. nonce: gSmzyl +.. section: C API + +Add macros :c:func:`Py_PACK_VERSION` and :c:func:`Py_PACK_FULL_VERSION` for +bit-packing Python version numbers. + +.. + +.. date: 2024-12-16-21-59-06 +.. gh-issue: 128008 +.. nonce: fa9Jt0 +.. section: C API + +Add :c:func:`PyWeakref_IsDead` function, which tests if a weak reference is +dead. + +.. + +.. date: 2024-12-11-13-01-26 +.. gh-issue: 127350 +.. nonce: uEBZZ4 +.. section: C API + +Add :c:func:`Py_fopen` function to open a file. Similar to the +:c:func:`!fopen` function, but the *path* parameter is a Python object and +an exception is set on error. Add also :c:func:`Py_fclose` function to close +a file, function needed for Windows support. Patch by Victor Stinner. + +.. + +.. date: 2025-01-09-19-44-00 +.. gh-issue: 128627 +.. nonce: mHzsEd +.. section: Build + +For Emscripten builds the function pointer cast call trampoline now uses the +wasm-gc ref.test instruction if it's available instead of Wasm JS type +reflection. + +.. + +.. date: 2025-01-04-22-39-10 +.. gh-issue: 128472 +.. nonce: Wt5E6M +.. section: Build + +Skip BOLT optimization of functions using computed gotos, fixing errors on +build with LLVM 19. + +.. + +.. date: 2025-01-02-12-50-46 +.. gh-issue: 115765 +.. nonce: jko7Fg +.. section: Build + +GNU Autoconf 2.72 is now required to generate :file:`!configure`. Patch by +Erlend Aasland. + +.. + +.. date: 2025-01-02-11-02-45 +.. gh-issue: 123925 +.. nonce: TLlyUi +.. section: Build + +Fix building the :mod:`curses` module on platforms with libncurses but +without libncursesw. + +.. + +.. date: 2024-12-31-17-09-37 +.. gh-issue: 90905 +.. nonce: PjLNai +.. section: Build + +Add support for cross-compiling to x86_64 on aarch64/arm64 macOS. + +.. + +.. date: 2024-12-28-21-05-19 +.. gh-issue: 128321 +.. nonce: 0UvbXw +.. section: Build + +Set ``LIBS`` instead of ``LDFLAGS`` when checking if :mod:`sqlite3` library +functions are available. This fixes the ordering of linked libraries during +checks, which was incorrect when using a statically linked ``libsqlite3``. + +.. + +.. date: 2024-12-21-09-56-37 +.. gh-issue: 100384 +.. nonce: Ib-XrN +.. section: Build + +Error on ``unguarded-availability`` in macOS builds, preventing invalid use +of symbols that are not available in older versions of the OS. + +.. + +.. date: 2024-12-20-09-03-22 +.. gh-issue: 128104 +.. nonce: m_SoVx +.. section: Build + +Remove ``Py_STRFTIME_C99_SUPPORT`` conditions in favor of requiring C99 +:manpage:`strftime(3)` specifier support at build time. When +cross-compiling, there is no build time check and support is assumed. + +.. + +.. date: 2024-12-16-16-16-35 +.. gh-issue: 127951 +.. nonce: lpE13- +.. section: Build + +Add option ``--pystats`` to the Windows build to enable performance +statistics collection. diff --git a/Misc/NEWS.d/3.5.0a1.rst b/Misc/NEWS.d/3.5.0a1.rst index 26b3d8253dbdee..8f9d8ce57caa05 100644 --- a/Misc/NEWS.d/3.5.0a1.rst +++ b/Misc/NEWS.d/3.5.0a1.rst @@ -1284,7 +1284,7 @@ Add function :func:`sys.is_finalizing` to know about interpreter shutdown. .. section: Library Add a default limit for the amount of data xmlrpclib.gzip_decode will -return. This resolves CVE-2013-1753. +return. This resolves :cve:`2013-1753`. .. @@ -1345,7 +1345,7 @@ newer worked. .. section: Library The "ip" command is now used on Linux to determine MAC address in -uuid.getnode(). Pach by Bruno Cauet. +uuid.getnode(). Patch by Bruno Cauet. .. @@ -3447,7 +3447,8 @@ tkinter.ttk now works when default root window is not set. .. nonce: FE_PII .. section: Library -_tkinter.create() now creates tkapp object with wantobject=1 by default. +``_tkinter.create()`` now creates ``tkapp`` object with ``wantobjects=1`` by +default. .. @@ -3930,7 +3931,7 @@ has been called. .. nonce: 5CDoox .. section: Library -New keyword argument ``unsafe`` to Mock. It raises ``AttributeError`` incase of +New keyword argument ``unsafe`` to Mock. It raises ``AttributeError`` in case of an attribute startswith assert or assret. .. @@ -4030,7 +4031,7 @@ unittest.mock.MagicMock now supports division. Patch by Johannes Baiter. .. section: Library Fix arbitrary memory access in JSONDecoder.raw_decode with a negative second -parameter. Bug reported by Guido Vranken. (See also: CVE-2014-4616) +parameter. Bug reported by Guido Vranken. (See also: :cve:`2014-4616`) .. @@ -4339,7 +4340,7 @@ these modules are not used. .. nonce: V1-XhC .. section: Library -Include the broadcast address in the usuable hosts for IPv6 in ipaddress. +Include the broadcast address in the usable hosts for IPv6 in ipaddress. .. @@ -5467,7 +5468,7 @@ All resources are now allowed when tests are not run by regrtest.py. .. section: Tests Fix pystone micro-benchmark: use floor division instead of true division to -benchmark integers instead of floating point numbers. Set pystone version to +benchmark integers instead of floating-point numbers. Set pystone version to 1.2. Patch written by Lennart Regebro. .. diff --git a/Misc/NEWS.d/3.5.2rc1.rst b/Misc/NEWS.d/3.5.2rc1.rst index a7e5c1b130f9e9..f9409b62e352ac 100644 --- a/Misc/NEWS.d/3.5.2rc1.rst +++ b/Misc/NEWS.d/3.5.2rc1.rst @@ -5,7 +5,7 @@ .. original section: Library .. section: Security -Update expat to 2.1.1, fixes CVE-2015-1283. +Update expat to 2.1.1, fixes :cve:`2015-1283`. .. @@ -15,8 +15,8 @@ Update expat to 2.1.1, fixes CVE-2015-1283. .. original section: Library .. section: Security -Fix TLS stripping vulnerability in smtplib, CVE-2016-0772. Reported by Team -Oststrom +Fix TLS stripping vulnerability in smtplib, :cve:`2016-0772`. Reported by Team +Oststrom. .. diff --git a/Misc/NEWS.d/3.5.3rc1.rst b/Misc/NEWS.d/3.5.3rc1.rst index bf4ef9302c9d1d..2424604249a65c 100644 --- a/Misc/NEWS.d/3.5.3rc1.rst +++ b/Misc/NEWS.d/3.5.3rc1.rst @@ -1048,7 +1048,7 @@ certs. .. section: Library Remove 3DES from ssl module's default cipher list to counter measure sweet32 -attack (CVE-2016-2183). +attack (:cve:`2016-2183`). .. @@ -1251,7 +1251,7 @@ Fix possible integer overflow in the _csv module for large record lengths. .. nonce: OnuO9s .. section: Library -Prevent HTTPoxy attack (CVE-2016-1000110). Ignore the HTTP_PROXY variable +Prevent HTTPoxy attack (:cve:`2016-1000110`). Ignore the HTTP_PROXY variable when REQUEST_METHOD environment is set, which indicates that the script is in CGI mode. diff --git a/Misc/NEWS.d/3.5.4rc1.rst b/Misc/NEWS.d/3.5.4rc1.rst index d65d5d14ee78bb..d5a85b3a2d8666 100644 --- a/Misc/NEWS.d/3.5.4rc1.rst +++ b/Misc/NEWS.d/3.5.4rc1.rst @@ -17,10 +17,10 @@ passing other environment variables and command arguments. .. section: Security Upgrade expat copy from 2.2.0 to 2.2.1 to get fixes of multiple security -vulnerabilities including: CVE-2017-9233 (External entity infinite loop -DoS), CVE-2016-9063 (Integer overflow, re-fix), CVE-2016-0718 (Fix -regression bugs from 2.2.0's fix to CVE-2016-0718) and CVE-2012-0876 -(Counter hash flooding with SipHash). Note: the CVE-2016-5300 (Use +vulnerabilities including: :cve:`2017-9233` (External entity infinite loop +DoS), :cve:`2016-9063` (Integer overflow, re-fix), :cve:`2016-0718` (Fix +regression bugs from 2.2.0's fix to :cve:`2016-0718`) and :cve:`2012-0876` +(Counter hash flooding with SipHash). Note: the :cve:`2016-5300` (Use os-specific entropy sources like getrandom) doesn't impact Python, since Python already gets entropy from the OS to set the expat secret using ``XML_SetHashSalt()``. @@ -46,8 +46,8 @@ authentication (``login@host``). .. original section: Library .. section: Security -Update expat copy from 2.1.1 to 2.2.0 to get fixes of CVE-2016-0718 and -CVE-2016-4472. See https://sourceforge.net/p/expat/bugs/537/ for more +Update expat copy from 2.1.1 to 2.2.0 to get fixes of :cve:`2016-0718` and +:cve:`2016-4472`. See https://sourceforge.net/p/expat/bugs/537/ for more information. .. diff --git a/Misc/NEWS.d/3.5.5rc1.rst b/Misc/NEWS.d/3.5.5rc1.rst index 9ccbf7b8060cd4..4a44840039e388 100644 --- a/Misc/NEWS.d/3.5.5rc1.rst +++ b/Misc/NEWS.d/3.5.5rc1.rst @@ -24,7 +24,7 @@ also be affected) .. nonce: Fd8kId .. section: Security -Fixed possible integer overflow in PyBytes_DecodeEscape, CVE-2017-1000158. +Fixed possible integer overflow in PyBytes_DecodeEscape, :cve:`2017-1000158`. Original patch by Jay Bosamiya; rebased to Python 3 by Miro Hrončok. .. diff --git a/Misc/NEWS.d/3.6.0a1.rst b/Misc/NEWS.d/3.6.0a1.rst index 144d217f6098a1..803c9fc5925fa6 100644 --- a/Misc/NEWS.d/3.6.0a1.rst +++ b/Misc/NEWS.d/3.6.0a1.rst @@ -1401,7 +1401,7 @@ array is extended. .. section: Library doctest.DocFileTest and doctest.testfile() now support packages (module -splitted into multiple directories) for the package parameter. +split into multiple directories) for the package parameter. .. @@ -1484,9 +1484,9 @@ on UNIX signals (SIGSEGV, SIGFPE, SIGABRT). .. nonce: RWN1jR .. section: Library -Add C functions :c:func:`_PyTraceMalloc_Track` and -:c:func:`_PyTraceMalloc_Untrack` to track memory blocks using the -:mod:`tracemalloc` module. Add :c:func:`_PyTraceMalloc_GetTraceback` to get +Add C functions :c:func:`!_PyTraceMalloc_Track` and +:c:func:`!_PyTraceMalloc_Untrack` to track memory blocks using the +:mod:`tracemalloc` module. Add :c:func:`!_PyTraceMalloc_GetTraceback` to get the traceback of an object. .. diff --git a/Misc/NEWS.d/3.6.0a2.rst b/Misc/NEWS.d/3.6.0a2.rst index 05b3d9f0463c1c..89d68ab3f8078f 100644 --- a/Misc/NEWS.d/3.6.0a2.rst +++ b/Misc/NEWS.d/3.6.0a2.rst @@ -5,7 +5,7 @@ .. original section: Library .. section: Security -Update expat to 2.1.1, fixes CVE-2015-1283. +Update expat to 2.1.1, fixes :cve:`2015-1283`. .. @@ -15,7 +15,7 @@ Update expat to 2.1.1, fixes CVE-2015-1283. .. original section: Library .. section: Security -Fix TLS stripping vulnerability in smtplib, CVE-2016-0772. Reported by Team +Fix TLS stripping vulnerability in smtplib, :cve:`2016-0772`. Reported by Team Oststrom. .. diff --git a/Misc/NEWS.d/3.6.0a4.rst b/Misc/NEWS.d/3.6.0a4.rst index d613fd5d928b65..3abbdecb57038b 100644 --- a/Misc/NEWS.d/3.6.0a4.rst +++ b/Misc/NEWS.d/3.6.0a4.rst @@ -359,7 +359,7 @@ Fix possible integer overflow in the _csv module for large record lengths. .. nonce: OnuO9s .. section: Library -Prevent HTTPoxy attack (CVE-2016-1000110). Ignore the HTTP_PROXY variable +Prevent HTTPoxy attack (:cve:`2016-1000110`). Ignore the HTTP_PROXY variable when REQUEST_METHOD environment is set, which indicates that the script is in CGI mode. diff --git a/Misc/NEWS.d/3.6.0b1.rst b/Misc/NEWS.d/3.6.0b1.rst index 4fb6bdd6f89c9b..bd54cf601d053b 100644 --- a/Misc/NEWS.d/3.6.0b1.rst +++ b/Misc/NEWS.d/3.6.0b1.rst @@ -949,7 +949,7 @@ Add scrypt (password-based key derivation function) to hashlib module .. section: Library Remove 3DES from ssl module's default cipher list to counter measure sweet32 -attack (CVE-2016-2183). +attack (:cve:`2016-2183`). .. diff --git a/Misc/NEWS.d/3.6.0b2.rst b/Misc/NEWS.d/3.6.0b2.rst index 9413c6e01917d5..23dd69efb23b88 100644 --- a/Misc/NEWS.d/3.6.0b2.rst +++ b/Misc/NEWS.d/3.6.0b2.rst @@ -215,7 +215,7 @@ memcpy(). .. nonce: e5xc1i .. section: Core and Builtins -Fix dict.pop() for splitted dictionary when trying to remove a "pending key" +Fix dict.pop() for split dictionary when trying to remove a "pending key" (Not yet inserted in split-table). Patch by Xiang Zhang. .. diff --git a/Misc/NEWS.d/3.6.2rc1.rst b/Misc/NEWS.d/3.6.2rc1.rst index 28eb88f79130c5..8e28bc9691921b 100644 --- a/Misc/NEWS.d/3.6.2rc1.rst +++ b/Misc/NEWS.d/3.6.2rc1.rst @@ -5,8 +5,8 @@ .. original section: Library .. section: Security -Update expat copy from 2.1.1 to 2.2.0 to get fixes of CVE-2016-0718 and -CVE-2016-4472. See https://sourceforge.net/p/expat/bugs/537/ for more +Update expat copy from 2.1.1 to 2.2.0 to get fixes of :cve:`2016-0718` and +:cve:`2016-4472`. See https://sourceforge.net/p/expat/bugs/537/ for more information. .. diff --git a/Misc/NEWS.d/3.6.2rc2.rst b/Misc/NEWS.d/3.6.2rc2.rst index 8c6545f6dbbeec..5ae7425828b692 100644 --- a/Misc/NEWS.d/3.6.2rc2.rst +++ b/Misc/NEWS.d/3.6.2rc2.rst @@ -17,10 +17,10 @@ passing other environment variables and command arguments. .. section: Security Upgrade expat copy from 2.2.0 to 2.2.1 to get fixes of multiple security -vulnerabilities including: CVE-2017-9233 (External entity infinite loop -DoS), CVE-2016-9063 (Integer overflow, re-fix), CVE-2016-0718 (Fix -regression bugs from 2.2.0's fix to CVE-2016-0718) and CVE-2012-0876 -(Counter hash flooding with SipHash). Note: the CVE-2016-5300 (Use +vulnerabilities including: :cve:`2017-9233` (External entity infinite loop +DoS), :cve:`2016-9063` (Integer overflow, re-fix), :cve:`2016-0718` (Fix +regression bugs from 2.2.0's fix to :cve:`2016-0718`) and :cve:`2012-0876` +(Counter hash flooding with SipHash). Note: the :cve:`2016-5300` (Use os-specific entropy sources like getrandom) doesn't impact Python, since Python already gets entropy from the OS to set the expat secret using ``XML_SetHashSalt()``. diff --git a/Misc/NEWS.d/3.6.3rc1.rst b/Misc/NEWS.d/3.6.3rc1.rst index ebda7665e2b6ea..6a20e07f05956c 100644 --- a/Misc/NEWS.d/3.6.3rc1.rst +++ b/Misc/NEWS.d/3.6.3rc1.rst @@ -85,7 +85,7 @@ wrong line (typically the first line of the file). .. nonce: Kl_fS5 .. section: Core and Builtins -Include sys/sysmacros.h for major(), minor(), and makedev(). GNU C libray +Include sys/sysmacros.h for major(), minor(), and makedev(). GNU C library plans to remove the functions from sys/types.h. .. diff --git a/Misc/NEWS.d/3.6.5rc1.rst b/Misc/NEWS.d/3.6.5rc1.rst index 056bacb5267c41..a45e97fb29045b 100644 --- a/Misc/NEWS.d/3.6.5rc1.rst +++ b/Misc/NEWS.d/3.6.5rc1.rst @@ -15,7 +15,7 @@ Minimal fix to prevent buffer overrun in os.symlink on Windows Regexes in difflib and poplib were vulnerable to catastrophic backtracking. These regexes formed potential DOS vectors (REDOS). They have been -refactored. This resolves CVE-2018-1060 and CVE-2018-1061. Patch by Jamie +refactored. This resolves :cve:`2018-1060` and :cve:`2018-1061`. Patch by Jamie Davis. .. @@ -474,7 +474,7 @@ platforms with OpenSSL 1.0.2+ or inet_pton. .. nonce: ideco .. section: Library -:func:`urllib.parse.urlsplit()` does not convert zone-id (scope) to lower +:func:`urllib.parse.urlsplit` does not convert zone-id (scope) to lower case for scoped IPv6 addresses in hostnames now. .. diff --git a/Misc/NEWS.d/3.7.0a1.rst b/Misc/NEWS.d/3.7.0a1.rst index aca79c4cc8c1b8..fd6ba07b53a617 100644 --- a/Misc/NEWS.d/3.7.0a1.rst +++ b/Misc/NEWS.d/3.7.0a1.rst @@ -46,10 +46,10 @@ passing other environment variables and command arguments. .. section: Security Upgrade expat copy from 2.2.0 to 2.2.1 to get fixes of multiple security -vulnerabilities including: CVE-2017-9233 (External entity infinite loop -DoS), CVE-2016-9063 (Integer overflow, re-fix), CVE-2016-0718 (Fix -regression bugs from 2.2.0's fix to CVE-2016-0718) and CVE-2012-0876 -(Counter hash flooding with SipHash). Note: the CVE-2016-5300 (Use +vulnerabilities including: :cve:`2017-9233` (External entity infinite loop +DoS), :cve:`2016-9063` (Integer overflow, re-fix), :cve:`2016-0718` (Fix +regression bugs from 2.2.0's fix to :cve:`2016-0718`) and :cve:`2012-0876` +(Counter hash flooding with SipHash). Note: the :cve:`2016-5300` (Use os-specific entropy sources like getrandom) doesn't impact Python, since Python already gets entropy from the OS to set the expat secret using ``XML_SetHashSalt()``. @@ -75,8 +75,8 @@ authentication (``login@host``). .. original section: Library .. section: Security -Update expat copy from 2.1.1 to 2.2.0 to get fixes of CVE-2016-0718 and -CVE-2016-4472. See https://sourceforge.net/p/expat/bugs/537/ for more +Update expat copy from 2.1.1 to 2.2.0 to get fixes of :cve:`2016-0718` and +:cve:`2016-4472`. See https://sourceforge.net/p/expat/bugs/537/ for more information. .. @@ -214,7 +214,7 @@ Fix possible undefined behavior in _PyObject_FastCall_Prepend. .. nonce: Kl_fS5 .. section: Core and Builtins -Include sys/sysmacros.h for major(), minor(), and makedev(). GNU C libray +Include sys/sysmacros.h for major(), minor(), and makedev(). GNU C library plans to remove the functions from sys/types.h. .. @@ -1479,7 +1479,7 @@ memcpy(). .. nonce: e5xc1i .. section: Core and Builtins -Fix dict.pop() for splitted dictionary when trying to remove a "pending key" +Fix dict.pop() for split dictionary when trying to remove a "pending key" (Not yet inserted in split-table). Patch by Xiang Zhang. .. @@ -2516,7 +2516,7 @@ stdin.write() if the child process is still running but closed the pipe. .. nonce: CdOuSl .. section: Library -Addded empty __slots__ to abc.ABC. This allows subclassers to deny __dict__ +Added empty __slots__ to abc.ABC. This allows subclassers to deny __dict__ and __weakref__ creation. Patch by Aaron Hall. .. diff --git a/Misc/NEWS.d/3.7.0a4.rst b/Misc/NEWS.d/3.7.0a4.rst index f2c6559037d84f..2ceb9e78e0421b 100644 --- a/Misc/NEWS.d/3.7.0a4.rst +++ b/Misc/NEWS.d/3.7.0a4.rst @@ -434,7 +434,7 @@ loop.getaddrinfo, loop.getnameinfo. .. nonce: ideco .. section: Library -:func:`urllib.parse.urlsplit()` does not convert zone-id (scope) to lower +:func:`urllib.parse.urlsplit` does not convert zone-id (scope) to lower case for scoped IPv6 addresses in hostnames now. .. @@ -463,7 +463,7 @@ Fix ``stop_serving`` in asyncio proactor loop kill all listening servers .. nonce: CUbsb2 .. section: Library -:func:`re.sub()` now replaces empty matches adjacent to a previous non-empty +:func:`re.sub` now replaces empty matches adjacent to a previous non-empty match. .. @@ -595,7 +595,7 @@ Add asyncio.get_running_loop() function. .. section: Library All class and static methods of builtin types now are correctly classified -by inspect.classify_class_attrs() and grouped in pydoc ouput. Added +by inspect.classify_class_attrs() and grouped in pydoc output. Added types.ClassMethodDescriptorType for unbound class methods of builtin types. .. diff --git a/Misc/NEWS.d/3.7.0b1.rst b/Misc/NEWS.d/3.7.0b1.rst index d1beec9cdcc33a..c9786e55c20739 100644 --- a/Misc/NEWS.d/3.7.0b1.rst +++ b/Misc/NEWS.d/3.7.0b1.rst @@ -414,7 +414,7 @@ callback. .. section: Library Fix memory leak in asyncio.Queue, when the queue has limited size and it is -full, the cancelation of queue.put() can cause a memory leak. Patch by: José +full, the cancellation of queue.put() can cause a memory leak. Patch by: José Melero. .. @@ -601,7 +601,7 @@ Add socket.getblocking() method. Add :mod:`importlib.resources` and :class:`importlib.abc.ResourceReader` as the unified API for reading resources contained within packages. Loaders wishing to support resource reading must implement the -:meth:`get_resource_reader()` method. File-based and zipimport-based +:meth:`get_resource_reader` method. File-based and zipimport-based loaders both implement these APIs. :class:`importlib.abc.ResourceLoader` is deprecated in favor of these new APIs. @@ -875,4 +875,4 @@ by Stéphane Wirtel .. section: C API Add C API access to the ``datetime.timezone`` constructor and -``datetime.timzone.UTC`` singleton. +``datetime.timezone.UTC`` singleton. diff --git a/Misc/NEWS.d/3.7.0b2.rst b/Misc/NEWS.d/3.7.0b2.rst index 702dbc960c018d..10cd57ea7edfce 100644 --- a/Misc/NEWS.d/3.7.0b2.rst +++ b/Misc/NEWS.d/3.7.0b2.rst @@ -274,7 +274,7 @@ collections.ChainMap() preserves the order of the underlying mappings. .. nonce: -T77_c .. section: Library -:func:`fnmatch.translate()` no longer produces patterns which contain set +:func:`fnmatch.translate` no longer produces patterns which contain set operations. Sets starting with '[' or containing '--', '&&', '~~' or '||' will be interpreted differently in regular expressions in future versions. Currently they emit warnings. fnmatch.translate() now avoids producing diff --git a/Misc/NEWS.d/3.7.0b3.rst b/Misc/NEWS.d/3.7.0b3.rst index c86963b7e42daf..a0c4cb15dc2b40 100644 --- a/Misc/NEWS.d/3.7.0b3.rst +++ b/Misc/NEWS.d/3.7.0b3.rst @@ -4,7 +4,7 @@ .. release date: 2018-03-29 .. section: Security -Harden ssl module against LibreSSL CVE-2018-8970. +Harden ssl module against LibreSSL :cve:`2018-8970`. X509_VERIFY_PARAM_set1_host() is called with an explicit namelen. A new test ensures that NULL bytes are not allowed. @@ -26,7 +26,7 @@ Minimal fix to prevent buffer overrun in os.symlink on Windows Regexes in difflib and poplib were vulnerable to catastrophic backtracking. These regexes formed potential DOS vectors (REDOS). They have been -refactored. This resolves CVE-2018-1060 and CVE-2018-1061. Patch by Jamie +refactored. This resolves :cve:`2018-1060` and :cve:`2018-1061`. Patch by Jamie Davis. .. diff --git a/Misc/NEWS.d/3.7.0b4.rst b/Misc/NEWS.d/3.7.0b4.rst index b17c7e08d1d408..93627f54900ddd 100644 --- a/Misc/NEWS.d/3.7.0b4.rst +++ b/Misc/NEWS.d/3.7.0b4.rst @@ -46,8 +46,8 @@ Fix potential memory leak in ``normalizestring()``. Change dict growth function from ``round_up_to_power_2(used*2+hashtable_size/2)`` to -``round_up_to_power_2(used*3)``. Previously, dict is shrinked only when -``used == 0``. Now dict has more chance to be shrinked. +``round_up_to_power_2(used*3)``. Previously, dict is shrunk only when +``used == 0``. Now dict has more chance to be shrunk. .. @@ -235,7 +235,7 @@ End framing at the end of C implementation of :func:`pickle.Pickler.dump`. .. section: Library Improved error handling and fixed a reference leak in -:func:`os.posix_spawn()`. +:func:`os.posix_spawn`. .. diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index bd9061601fe190..93995bc8feaad7 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -4,7 +4,7 @@ .. release date: 2019-02-03 .. section: Security -[CVE-2019-5010] Fix a NULL pointer deref in ssl module. The cert parser did +:cve:`2019-5010`: Fix a NULL pointer deref in ssl module. The cert parser did not handle CRL distribution points with empty DP or URI correctly. A malicious or buggy certificate can result into segfault. Vulnerability (TALOS-2018-0758) reported by Colin Read and Nicolas Edet of Cisco. @@ -50,7 +50,7 @@ files or create network connections. .. nonce: Ua9jMv .. section: Security -CVE-2018-14647: The C accelerated _elementtree module now initializes hash +:cve:`2018-14647`: The C accelerated _elementtree module now initializes hash randomization salt from _Py_HashSecret instead of libexpat's default CSPRNG. .. @@ -89,7 +89,7 @@ Fixed thread-safety of error handling in _ssl. .. nonce: TzSN4x .. section: Security -Harden ssl module against LibreSSL CVE-2018-8970. +Harden ssl module against LibreSSL :cve:`2018-8970`. X509_VERIFY_PARAM_set1_host() is called with an explicit namelen. A new test ensures that NULL bytes are not allowed. @@ -111,7 +111,7 @@ Minimal fix to prevent buffer overrun in os.symlink on Windows Regexes in difflib and poplib were vulnerable to catastrophic backtracking. These regexes formed potential DOS vectors (REDOS). They have been -refactored. This resolves CVE-2018-1060 and CVE-2018-1061. Patch by Jamie +refactored. This resolves :cve:`2018-1060` and :cve:`2018-1061`. Patch by Jamie Davis. .. @@ -224,7 +224,7 @@ positives from posix, socket, time, test_io, and test_faulthandler. .. nonce: 9vMWSP .. section: Core and Builtins -Fix an assertion error in :func:`format` in debug build for floating point +Fix an assertion error in :func:`format` in debug build for floating-point formatting with "n" format, zero padding and small width. Release build is not impacted. Patch by Karthikeyan Singaravelan. @@ -625,7 +625,7 @@ Spytz. The C function ``property_descr_get()`` uses a "cached" tuple to optimize function calls. But this tuple can be discovered in debug mode with -:func:`sys.getobjects()`. Remove the optimization, it's not really worth it +:func:`sys.getobjects`. Remove the optimization, it's not really worth it and it causes 3 different crashes last years. .. @@ -1406,8 +1406,8 @@ Fix potential memory leak in ``normalizestring()``. Change dict growth function from ``round_up_to_power_2(used*2+hashtable_size/2)`` to -``round_up_to_power_2(used*3)``. Previously, dict is shrinked only when -``used == 0``. Now dict has more chance to be shrinked. +``round_up_to_power_2(used*3)``. Previously, dict is shrunk only when +``used == 0``. Now dict has more chance to be shrunk. .. @@ -2519,7 +2519,7 @@ non-Windows systems. .. nonce: dQS1ng .. section: Library -Fix incorrect parsing of :class:`_io.IncrementalNewlineDecoder`'s +Fix incorrect parsing of :class:`io.IncrementalNewlineDecoder`'s *translate* argument. .. @@ -2850,8 +2850,8 @@ allow for tweaking of protocols and also to add support by default for .. nonce: 37IdsA .. section: Library -Fixed integer overflow in the :meth:`~hashlib.shake.digest()` and -:meth:`~hashlib.shake.hexdigest()` methods for the SHAKE algorithm in the +Fixed integer overflow in the :meth:`~hashlib.shake.digest` and +:meth:`~hashlib.shake.hexdigest` methods for the SHAKE algorithm in the :mod:`hashlib` module. .. @@ -3211,10 +3211,10 @@ bytes objects. (microoptimization) .. nonce: i-F_E5 .. section: Library -Add :func:`~unittest.addModuleCleanup()` and -:meth:`~unittest.TestCase.addClassCleanup()` to unittest to support cleanups -for :func:`~unittest.setUpModule()` and -:meth:`~unittest.TestCase.setUpClass()`. Patch by Lisa Roach. +Add :func:`~unittest.addModuleCleanup` and +:meth:`~unittest.TestCase.addClassCleanup` to unittest to support cleanups +for :func:`~unittest.setUpModule` and +:meth:`~unittest.TestCase.setUpClass`. Patch by Lisa Roach. .. @@ -3458,7 +3458,7 @@ Running the :mod:`trace` module no longer creates the ``trace.cover`` file. .. section: Library Fix crash when an ``ABC``-derived class with invalid ``__subclasses__`` is -passed as the second argument to :func:`issubclass()`. Patch by Alexey +passed as the second argument to :func:`issubclass`. Patch by Alexey Izbyshev. .. @@ -3664,7 +3664,7 @@ Add pure Python fallback for functools.reduce. Patch by Robert Wright. .. section: Library The default asyncio task class now always has a name which can be get or set -using two new methods (:meth:`~asyncio.Task.get_name()` and +using two new methods (:meth:`~asyncio.Task.get_name` and :meth:`~asyncio.Task.set_name`) and is visible in the :func:`repr` output. An initial name can also be set using the new ``name`` keyword argument to :func:`asyncio.create_task` or the @@ -4152,12 +4152,12 @@ Convert content length to string before putting to header. :func:`~os.path.exists`, :func:`~os.path.lexists`, :func:`~os.path.isdir`, :func:`~os.path.isfile`, :func:`~os.path.islink`, and :func:`~os.path.ismount`, and :mod:`pathlib.Path` methods that return a -boolean result like :meth:`~pathlib.Path.exists()`, -:meth:`~pathlib.Path.is_dir()`, :meth:`~pathlib.Path.is_file()`, -:meth:`~pathlib.Path.is_mount()`, :meth:`~pathlib.Path.is_symlink()`, -:meth:`~pathlib.Path.is_block_device()`, -:meth:`~pathlib.Path.is_char_device()`, :meth:`~pathlib.Path.is_fifo()`, -:meth:`~pathlib.Path.is_socket()` now return ``False`` instead of raising +boolean result like :meth:`~pathlib.Path.exists`, +:meth:`~pathlib.Path.is_dir`, :meth:`~pathlib.Path.is_file`, +:meth:`~pathlib.Path.is_mount`, :meth:`~pathlib.Path.is_symlink`, +:meth:`~pathlib.Path.is_block_device`, +:meth:`~pathlib.Path.is_char_device`, :meth:`~pathlib.Path.is_fifo`, +:meth:`~pathlib.Path.is_socket` now return ``False`` instead of raising :exc:`ValueError` or its subclasses :exc:`UnicodeEncodeError` and :exc:`UnicodeDecodeError` for paths that contain characters or bytes unrepresentable at the OS level. @@ -5269,7 +5269,7 @@ performance and smaller size compared to protocol 3 introduced in Python .. section: Library Improved error handling and fixed a reference leak in -:func:`os.posix_spawn()`. +:func:`os.posix_spawn`. .. @@ -5857,7 +5857,7 @@ collections.ChainMap() preserves the order of the underlying mappings. .. nonce: -T77_c .. section: Library -:func:`fnmatch.translate()` no longer produces patterns which contain set +:func:`fnmatch.translate` no longer produces patterns which contain set operations. Sets starting with '[' or containing '--', '&&', '~~' or '||' will be interpreted differently in regular expressions in future versions. Currently they emit warnings. fnmatch.translate() now avoids producing @@ -6282,7 +6282,7 @@ Add documentation about the new command line interface of the gzip module. .. nonce: YO9CYm .. section: Documentation -chm document displays non-ASCII charaters properly on some MBCS Windows +chm document displays non-ASCII characters properly on some MBCS Windows systems. .. @@ -8051,7 +8051,7 @@ Update macOS 10.9+ installer to Tcl/Tk 8.6.8. .. nonce: K6jCVG .. section: macOS -In :mod:`_scproxy`, drop the GIL when calling into ``SystemConfiguration`` +In :mod:`!_scproxy`, drop the GIL when calling into ``SystemConfiguration`` to avoid deadlocks. .. diff --git a/Misc/NEWS.d/3.8.0a2.rst b/Misc/NEWS.d/3.8.0a2.rst index c8620aeea7f133..0dbfa2758fe601 100644 --- a/Misc/NEWS.d/3.8.0a2.rst +++ b/Misc/NEWS.d/3.8.0a2.rst @@ -202,7 +202,7 @@ the mean and standard deviation of measurement data as single entity. .. nonce: V88MCD .. section: Library -Added statistics.fmean() as a faster, floating point variant of the existing +Added statistics.fmean() as a faster, floating-point variant of the existing mean() function. .. diff --git a/Misc/NEWS.d/3.8.0a4.rst b/Misc/NEWS.d/3.8.0a4.rst index fa5eb697d9202d..41eea7303bd899 100644 --- a/Misc/NEWS.d/3.8.0a4.rst +++ b/Misc/NEWS.d/3.8.0a4.rst @@ -13,7 +13,7 @@ Fixes mishandling of pre-normalization characters in urlsplit(). .. nonce: 51E-DA .. section: Security -Address CVE-2019-9740 by disallowing URL paths with embedded whitespace or +Address :cve:`2019-9740` by disallowing URL paths with embedded whitespace or control characters through into the underlying http client request. Such potentially malicious header injection URLs now cause an http.client.InvalidURL exception to be raised. @@ -600,7 +600,7 @@ exceptions. .. nonce: 9sjd38 .. section: Library -Add time module support and fix test_time faiures for VxWorks. +Add time module support and fix test_time failures for VxWorks. .. @@ -663,8 +663,8 @@ followed imports. Patch by Brandt Bucher. .. nonce: QmfNmY .. section: Library -Added :meth:`~socket.create_server()` and -:meth:`~socket.has_dualstack_ipv6()` convenience functions to automate the +Added :meth:`~socket.create_server` and +:meth:`~socket.has_dualstack_ipv6` convenience functions to automate the necessary tasks usually involved when creating a server socket, including accepting both IPv4 and IPv6 connections on the same socket. (Contributed by Giampaolo Rodola in :issue:`17561`.) @@ -843,7 +843,7 @@ Using the code of the ``Tools/scripts/serve.py`` script as an example in the .. nonce: nF1pP1 .. section: Documentation -Added Documention for PyInterpreterState_Main(). +Added documentation for PyInterpreterState_Main(). .. @@ -945,7 +945,7 @@ P. Hemsley. .. nonce: __FTq9 .. section: Tests -Add a new :mod:`_testinternalcapi` module to test the internal C API. +Add a new :mod:`!_testinternalcapi` module to test the internal C API. .. @@ -1383,7 +1383,7 @@ Since Python 3.7.0, calling :c:func:`Py_DecodeLocale` before coerced and/or if the UTF-8 Mode is enabled by the user configuration. The LC_CTYPE coercion and UTF-8 Mode are now disabled by default to fix the mojibake issue. They must now be enabled explicitly (opt-in) using the new -:c:func:`_Py_PreInitialize` API with ``_PyPreConfig``. +:c:func:`!_Py_PreInitialize` API with ``_PyPreConfig``. .. diff --git a/Misc/NEWS.d/3.8.0b1.rst b/Misc/NEWS.d/3.8.0b1.rst index 4eb0c0451e97b5..5010473269d92f 100644 --- a/Misc/NEWS.d/3.8.0b1.rst +++ b/Misc/NEWS.d/3.8.0b1.rst @@ -4,7 +4,7 @@ .. release date: 2019-06-04 .. section: Security -CVE-2019-9948: Avoid file reading by disallowing ``local-file://`` and +:cve:`2019-9948`: Avoid file reading by disallowing ``local-file://`` and ``local_file://`` URL schemes in ``URLopener().open()`` and ``URLopener().retrieve()`` of :mod:`urllib.request`. @@ -176,8 +176,8 @@ Added new ``replace()`` method to the code type (:class:`types.CodeType`). .. nonce: d1SOtF .. section: Core and Builtins -Implement :func:`socket.if_nameindex()`, :func:`socket.if_nametoindex()`, -and :func:`socket.if_indextoname()` on Windows. +Implement :func:`socket.if_nameindex`, :func:`socket.if_nametoindex`, +and :func:`socket.if_indextoname` on Windows. .. @@ -538,7 +538,7 @@ module. .. nonce: TQFOR4 .. section: Library -:meth:`!msilib.Directory.start_component()` no longer fails if *keyfile* is +:meth:`!msilib.Directory.start_component` no longer fails if *keyfile* is not ``None``. .. @@ -600,7 +600,7 @@ default. .. nonce: sLULGQ .. section: Library -Fix destructor :class:`_pyio.BytesIO` and :class:`_pyio.TextIOWrapper`: +Fix destructor :class:`!_pyio.BytesIO` and :class:`!_pyio.TextIOWrapper`: initialize their ``_buffer`` attribute as soon as possible (in the class body), because it's used by ``__del__()`` which calls ``close()``. @@ -1371,7 +1371,7 @@ Asyncio: Remove inner callback on outer cancellation in shield .. nonce: d8djAJ .. section: Library -Fix :meth:`asyncio.SelectorEventLoop.subprocess_exec()` leaks file +Fix :meth:`asyncio.SelectorEventLoop.subprocess_exec` leaks file descriptors if ``Popen`` fails and called with ``stdin=subprocess.PIPE``. Patch by Niklas Fiekas. diff --git a/Misc/NEWS.d/3.9.0a1.rst b/Misc/NEWS.d/3.9.0a1.rst index 66d7fc1f32e705..d75132b0aac040 100644 --- a/Misc/NEWS.d/3.9.0a1.rst +++ b/Misc/NEWS.d/3.9.0a1.rst @@ -44,7 +44,7 @@ rendering the document page as HTML. (Contributed by Donghee Na in .. section: Security Update vendorized expat library version to 2.2.8, which resolves -CVE-2019-15903. +:cve:`2019-15903`. .. @@ -149,7 +149,7 @@ exception in :meth:`float.__getformat__`. .. nonce: 9-vKtO .. section: Core and Builtins -Optimized :func:`math.floor()`, :func:`math.ceil()` and :func:`math.trunc()` +Optimized :func:`math.floor`, :func:`math.ceil` and :func:`math.trunc` for floats. .. @@ -299,7 +299,7 @@ Check the error from the system's underlying ``crypt`` or ``crypt_r``. .. section: Core and Builtins On FreeBSD, Python no longer calls ``fedisableexcept()`` at startup to -control the floating point control mode. The call became useless since +control the floating-point control mode. The call became useless since FreeBSD 6: it became the default mode. .. @@ -1384,7 +1384,7 @@ Nested subclasses of :class:`typing.NamedTuple` are now pickleable. .. nonce: hwrPN7 .. section: Library -Prevent :exc:`KeyError` thrown by :func:`_encoded_words.decode` when given +Prevent :exc:`KeyError` thrown by :func:`!_encoded_words.decode` when given an encoded-word with invalid content-type encoding from propagating all the way to :func:`email.message.get`. @@ -1395,8 +1395,8 @@ way to :func:`email.message.get`. .. nonce: S6Klvm .. section: Library -Deprecated the ``split()`` method in :class:`_tkinter.TkappType` in favour -of the ``splitlist()`` method which has more consistent and predicable +Deprecated the ``split()`` method in :class:`!_tkinter.TkappType` in favour +of the ``splitlist()`` method which has more consistent and predictable behavior. .. @@ -2990,7 +2990,7 @@ mode. .. nonce: FRGH4I .. section: Library -:func:`ctypes.create_unicode_buffer()` now also supports non-BMP characters +:func:`ctypes.create_unicode_buffer` now also supports non-BMP characters on platforms with 16-bit :c:type:`wchar_t` (for example, Windows and AIX). .. @@ -3013,7 +3013,7 @@ thread was still running. .. section: Library Allow pure Python implementation of :mod:`pickle` to work even when the C -:mod:`_pickle` module is unavailable. +:mod:`!_pickle` module is unavailable. .. @@ -3054,7 +3054,7 @@ Change the format of feature_version to be a (major, minor) tuple. .. nonce: 5_mJkQ .. section: Library -Eliminate :exc:`RuntimeError` raised by :func:`asyncio.all_tasks()` if +Eliminate :exc:`RuntimeError` raised by :func:`asyncio.all_tasks` if internal tasks weak set is changed by another thread during iteration. .. @@ -3064,8 +3064,8 @@ internal tasks weak set is changed by another thread during iteration. .. nonce: ADqCkq .. section: Library -:class:`_pyio.IOBase` destructor now does nothing if getting the ``closed`` -attribute fails to better mimic :class:`_io.IOBase` finalizer. +:class:`!_pyio.IOBase` destructor now does nothing if getting the ``closed`` +attribute fails to better mimic :class:`!_io.IOBase` finalizer. .. @@ -3536,7 +3536,7 @@ Add :meth:`~pathlib.Path.readlink`. Patch by Girts Folkmanis. .. nonce: La3TZz .. section: Library -Made :func:`urllib.parse.unquote()` accept bytes in addition to strings. +Made :func:`urllib.parse.unquote` accept bytes in addition to strings. Patch by Stein Karlsen. .. @@ -3839,7 +3839,7 @@ Added possible exceptions to the description of os.chdir(). .. nonce: r_wGRc .. section: Documentation -Documented that in :meth:`datetime.datetime.strptime()`, the leading zero in +Documented that in :meth:`datetime.datetime.strptime`, the leading zero in some two-digit formats is optional. Patch by Mike Gleen. .. @@ -4993,7 +4993,7 @@ Make :const:`winreg.REG_MULTI_SZ` support zero-length strings. .. section: Windows Replace use of :c:func:`strcasecmp` for the system function -:c:func:`_stricmp`. Patch by Minmin Gong. +:c:func:`!_stricmp`. Patch by Minmin Gong. .. @@ -5616,7 +5616,7 @@ heap type .. nonce: 4DcUaI .. section: C API -Add :c:func:`_PyObject_FunctionStr` to get a user-friendly string +Add :c:func:`!_PyObject_FunctionStr` to get a user-friendly string representation of a function-like object. Patch by Jeroen Demeyer. .. @@ -5696,8 +5696,8 @@ Add :c:func:`PyConfig_SetWideStringList` function. .. section: C API Add fast functions for calling methods: -:c:func:`_PyObject_VectorcallMethod`, :c:func:`_PyObject_CallMethodNoArgs` -and :c:func:`_PyObject_CallMethodOneArg`. +:c:func:`!_PyObject_VectorcallMethod`, :c:func:`!_PyObject_CallMethodNoArgs` +and :c:func:`!_PyObject_CallMethodOneArg`. .. diff --git a/Misc/NEWS.d/3.9.0a2.rst b/Misc/NEWS.d/3.9.0a2.rst index 7d878cfe227552..39b1c308312aa4 100644 --- a/Misc/NEWS.d/3.9.0a2.rst +++ b/Misc/NEWS.d/3.9.0a2.rst @@ -844,7 +844,7 @@ test.regrtest now can receive a list of test patterns to ignore (using the .. nonce: cNsA7S .. section: Build -:mod:`asyncio` now raises :exc:`TyperError` when calling incompatible +:mod:`asyncio` now raises :exc:`TypeError` when calling incompatible methods with an :class:`ssl.SSLSocket` socket. Patch by Ido Michael. .. diff --git a/Misc/NEWS.d/3.9.0a4.rst b/Misc/NEWS.d/3.9.0a4.rst index ca0eb2abf1d654..cce0c4c9acdf1b 100644 --- a/Misc/NEWS.d/3.9.0a4.rst +++ b/Misc/NEWS.d/3.9.0a4.rst @@ -755,7 +755,7 @@ dependencies. .. nonce: X7FRaN .. section: Windows -:meth:`~pathlib.Path.home()` and :meth:`~pathlib.Path.expanduser()` on +:meth:`~pathlib.Path.home` and :meth:`~pathlib.Path.expanduser` on Windows now prefer :envvar:`USERPROFILE` and no longer use :envvar:`HOME`, which is not normally set for regular user accounts. This makes them again behave like :func:`os.path.expanduser`, which was changed to ignore diff --git a/Misc/NEWS.d/3.9.0a5.rst b/Misc/NEWS.d/3.9.0a5.rst index f0015ac54df307..9402e5077c2e77 100644 --- a/Misc/NEWS.d/3.9.0a5.rst +++ b/Misc/NEWS.d/3.9.0a5.rst @@ -5,7 +5,7 @@ .. section: Security Disallow control characters in hostnames in http.client, addressing -CVE-2019-18348. Such potentially malicious header injection URLs now cause a +:cve:`2019-18348`. Such potentially malicious header injection URLs now cause a InvalidURL to be raised. .. @@ -548,7 +548,7 @@ large for an AF_UNIX socket address. Patch by Pablo Galindo. .. nonce: mxr5m8 .. section: Library -:func:`ast.dump()` no longer outputs optional fields and attributes with +:func:`ast.dump` no longer outputs optional fields and attributes with default values. The default values for optional fields and attributes of AST nodes are now set as class attributes (e.g. ``Constant.kind`` is set to ``None``). diff --git a/Misc/NEWS.d/3.9.0a6.rst b/Misc/NEWS.d/3.9.0a6.rst index 366a260172efb8..11309fccc0a20c 100644 --- a/Misc/NEWS.d/3.9.0a6.rst +++ b/Misc/NEWS.d/3.9.0a6.rst @@ -23,7 +23,7 @@ header injection attacks. .. nonce: B299Yq .. section: Security -CVE-2020-8492: The :class:`~urllib.request.AbstractBasicAuthHandler` class +:cve:`2020-8492`: The :class:`~urllib.request.AbstractBasicAuthHandler` class of the :mod:`urllib.request` module uses an inefficient regular expression which can be exploited by an attacker to cause a denial of service. Fix the regex to prevent the catastrophic backtracking. Vulnerability reported by @@ -111,7 +111,7 @@ str.decode(). .. nonce: m15TTX .. section: Core and Builtins -Fix possible refleaks in :mod:`_json`, memo of PyScannerObject should be +Fix possible refleaks in :mod:`!_json`, memo of PyScannerObject should be traversed. .. @@ -403,7 +403,7 @@ after encoding it to utf-8, not before. .. nonce: pDZR6V .. section: Library -Added :meth:`pathlib.Path.with_stem()` to create a new Path with the stem +Added :meth:`pathlib.Path.with_stem` to create a new Path with the stem replaced. .. @@ -564,7 +564,7 @@ Implement traverse and clear slots in _abc._abc_data type. .. nonce: 3rO_q7 .. section: Library -Remove deprecated :meth:`symtable.SymbolTable.has_exec`. +Remove deprecated :meth:`!symtable.SymbolTable.has_exec`. .. @@ -635,7 +635,7 @@ script is killed by signal 11, it now logs: "CGI script exit code -11." .. section: Library Improve the error message when triying to import a module using :mod:`runpy` -and incorrently use the ".py" extension at the end of the module name. Patch +and incorrectly using the ".py" extension at the end of the module name. Patch by Pablo Galindo. .. @@ -666,8 +666,8 @@ for _main_thread, instead of a _DummyThread instance. .. nonce: VTq_8s .. section: Library -Add a private ``_at_fork_reinit()`` method to :class:`_thread.Lock`, -:class:`_thread.RLock`, :class:`threading.RLock` and +Add a private ``_at_fork_reinit()`` method to :class:`!_thread.Lock`, +:class:`!_thread.RLock`, :class:`threading.RLock` and :class:`threading.Condition` classes: reinitialize the lock at fork in the child process, reset the lock to the unlocked state. Rename also the private ``_reset_internal_locks()`` method of :class:`threading.Event` to @@ -866,7 +866,7 @@ of source of the class. Patch by Karthikeyan Singaravelan. .. nonce: vHC7YQ .. section: Library -Deprecate passing None as an argument for :func:`shlex.split()`'s ``s`` +Deprecate passing None as an argument for :func:`shlex.split`'s ``s`` parameter. Patch by Zackery Spytz. .. @@ -1118,7 +1118,7 @@ into an exit code. .. nonce: _FOf7E .. section: C API -Move the :c:type:`PyGC_Head` structure to the internal C API. +Move the :c:type:`!PyGC_Head` structure to the internal C API. .. @@ -1149,8 +1149,8 @@ the garbage collector respectively. Patch by Pablo Galindo. .. nonce: Seuh3D .. section: C API -The :c:func:`PyObject_NEW` macro becomes an alias to the -:c:func:`PyObject_New` macro, and the :c:func:`PyObject_NEW_VAR` macro +The :c:func:`!PyObject_NEW` macro becomes an alias to the +:c:func:`PyObject_New` macro, and the :c:func:`!PyObject_NEW_VAR` macro becomes an alias to the :c:func:`PyObject_NewVar` macro, to hide implementation details. They no longer access directly the :c:member:`PyTypeObject.tp_basicsize` member. @@ -1174,7 +1174,7 @@ used. .. nonce: 6nFYbY .. section: C API -Convert the :c:func:`PyObject_GET_WEAKREFS_LISTPTR` macro to a function to +Convert the :c:func:`!PyObject_GET_WEAKREFS_LISTPTR` macro to a function to hide implementation details: the macro accessed directly to the :c:member:`PyTypeObject.tp_weaklistoffset` member. diff --git a/Misc/NEWS.d/3.9.0b1.rst b/Misc/NEWS.d/3.9.0b1.rst index 40fb8474bf9364..9a3630ddf216a8 100644 --- a/Misc/NEWS.d/3.9.0b1.rst +++ b/Misc/NEWS.d/3.9.0b1.rst @@ -532,7 +532,7 @@ Remove ``_random.Random.randbytes()``: the C implementation of .. section: Library Added default arguments to -:meth:`difflib.SequenceMatcher.find_longest_match()`. +:meth:`difflib.SequenceMatcher.find_longest_match`. .. diff --git a/Misc/NEWS.d/next/Build/2024-02-13-14-52-59.gh-issue-114099.zjXsQr.rst b/Misc/NEWS.d/next/Build/2024-02-13-14-52-59.gh-issue-114099.zjXsQr.rst deleted file mode 100644 index e2858bd71d28cb..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-02-13-14-52-59.gh-issue-114099.zjXsQr.rst +++ /dev/null @@ -1,2 +0,0 @@ -Makefile targets were added to support compiling an iOS-compatible framework -build. diff --git a/Misc/NEWS.d/next/Build/2024-02-21-11-58-30.gh-issue-115737.dpNl2T.rst b/Misc/NEWS.d/next/Build/2024-02-21-11-58-30.gh-issue-115737.dpNl2T.rst deleted file mode 100644 index 112f65258dd84b..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-02-21-11-58-30.gh-issue-115737.dpNl2T.rst +++ /dev/null @@ -1,2 +0,0 @@ -The install name for libPython is now correctly set for non-framework macOS -builds. diff --git a/Misc/NEWS.d/next/Build/2024-02-21-18-22-49.gh-issue-111225.Z8C3av.rst b/Misc/NEWS.d/next/Build/2024-02-21-18-22-49.gh-issue-111225.Z8C3av.rst deleted file mode 100644 index 8cdeba46ba2313..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-02-21-18-22-49.gh-issue-111225.Z8C3av.rst +++ /dev/null @@ -1 +0,0 @@ -Link extension modules against libpython on Android. diff --git a/Misc/NEWS.d/next/Build/2024-02-24-12-50-43.gh-issue-115350.naQA6y.rst b/Misc/NEWS.d/next/Build/2024-02-24-12-50-43.gh-issue-115350.naQA6y.rst deleted file mode 100644 index 5492a804255838..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-02-24-12-50-43.gh-issue-115350.naQA6y.rst +++ /dev/null @@ -1 +0,0 @@ -Fix building ctypes module with -DWIN32_LEAN_AND_MEAN defined diff --git a/Misc/NEWS.d/next/Build/2024-02-26-14-54-58.gh-issue-71052.XvFay1.rst b/Misc/NEWS.d/next/Build/2024-02-26-14-54-58.gh-issue-71052.XvFay1.rst deleted file mode 100644 index bda91335814936..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-02-26-14-54-58.gh-issue-71052.XvFay1.rst +++ /dev/null @@ -1 +0,0 @@ -Fix several Android build issues diff --git a/Misc/NEWS.d/next/Build/2024-02-29-15-12-31.gh-issue-116117.eENkQK.rst b/Misc/NEWS.d/next/Build/2024-02-29-15-12-31.gh-issue-116117.eENkQK.rst deleted file mode 100644 index 22477b343c06f0..00000000000000 --- a/Misc/NEWS.d/next/Build/2024-02-29-15-12-31.gh-issue-116117.eENkQK.rst +++ /dev/null @@ -1,2 +0,0 @@ -Backport ``libb2``'s PR #42 to fix compiling CPython on 32-bit Windows -with ``clang-cl``. diff --git a/Misc/NEWS.d/next/Build/2025-01-16-03-35-37.gh-issue-128902.Dt7xtV.rst b/Misc/NEWS.d/next/Build/2025-01-16-03-35-37.gh-issue-128902.Dt7xtV.rst new file mode 100644 index 00000000000000..42ac492498e029 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-01-16-03-35-37.gh-issue-128902.Dt7xtV.rst @@ -0,0 +1 @@ +Fix compile errors with Clang 9 and older due to lack of ``__attribute__((fallthrough))`` support. diff --git a/Misc/NEWS.d/next/C API/2024-02-16-15-56-53.gh-issue-114626.ie2esA.rst b/Misc/NEWS.d/next/C API/2024-02-16-15-56-53.gh-issue-114626.ie2esA.rst deleted file mode 100644 index 763f4cee6d3f0b..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-02-16-15-56-53.gh-issue-114626.ie2esA.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add again ``_PyCFunctionFastWithKeywords`` name, removed in Python 3.13 -alpha 4 by mistake. Keep the old private ``_PyCFunctionFastWithKeywords`` -name (Python 3.7) as an alias to the new public name -``PyCFunctionFastWithKeywords`` (Python 3.13a4). Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2024-12-14-03-40-15.gh-issue-127925.FF7aov.rst b/Misc/NEWS.d/next/C_API/2024-12-14-03-40-15.gh-issue-127925.FF7aov.rst new file mode 100644 index 00000000000000..6cf5fd2872cd43 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-12-14-03-40-15.gh-issue-127925.FF7aov.rst @@ -0,0 +1,3 @@ +Convert the :mod:`decimal` module to use :pep:`757` C API (export-import +integers), offering some speed-up if the integer part of the +:class:`~decimal.Decimal` instance is small. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/C_API/2025-01-01-03-25-38.gh-issue-126599.MRCYlH.rst b/Misc/NEWS.d/next/C_API/2025-01-01-03-25-38.gh-issue-126599.MRCYlH.rst new file mode 100644 index 00000000000000..8362ee3a2b1760 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-01-01-03-25-38.gh-issue-126599.MRCYlH.rst @@ -0,0 +1 @@ +Remove some internal test APIs for the experimental JIT compiler. diff --git a/Misc/NEWS.d/next/C_API/2025-01-15-11-42-07.gh-issue-128863.C9MkB_.rst b/Misc/NEWS.d/next/C_API/2025-01-15-11-42-07.gh-issue-128863.C9MkB_.rst new file mode 100644 index 00000000000000..b2e5664138fba2 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-01-15-11-42-07.gh-issue-128863.C9MkB_.rst @@ -0,0 +1,19 @@ +The following private functions are deprecated and planned for removal in +Python 3.18: + +* :c:func:`!_PyBytes_Join`: use :c:func:`PyBytes_Join`. +* :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`. +* :c:func:`!_PyDict_Pop()`: use :c:func:`PyDict_Pop`. +* :c:func:`!_PyLong_Sign()`: use :c:func:`PyLong_GetSign`. +* :c:func:`!_PyLong_FromDigits` and :c:func:`!_PyLong_New`: + use :c:func:`PyLongWriter_Create`. +* :c:func:`!_PyThreadState_UncheckedGet`: use :c:func:`PyThreadState_GetUnchecked`. +* :c:func:`!_PyUnicode_AsString`: use :c:func:`PyUnicode_AsUTF8`. +* :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`. +* :c:func:`!_Py_fopen_obj`: use :c:func:`Py_fopen`. + +The `pythoncapi-compat project +`__ can be used to get these new +public functions on Python 3.13 and older. + +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2025-01-16-21-56-49.gh-issue-128844.ZPiJuo.rst b/Misc/NEWS.d/next/C_API/2025-01-16-21-56-49.gh-issue-128844.ZPiJuo.rst new file mode 100644 index 00000000000000..d9e1962631026a --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-01-16-21-56-49.gh-issue-128844.ZPiJuo.rst @@ -0,0 +1,3 @@ +Add :c:func:`PyUnstable_TryIncRef` and :c:func:`PyUnstable_EnableTryIncRef` +unstable APIs. These are helpers for dealing with unowned references in +a thread-safe way, particularly in the free threading build. diff --git a/Misc/NEWS.d/next/C_API/2025-01-19-23-17-58.gh-issue-129033.cpRivP.rst b/Misc/NEWS.d/next/C_API/2025-01-19-23-17-58.gh-issue-129033.cpRivP.rst new file mode 100644 index 00000000000000..3cd19cc48e3416 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-01-19-23-17-58.gh-issue-129033.cpRivP.rst @@ -0,0 +1,3 @@ +Remove the private ``_Py_InitializeMain()`` function. It was a +:term:`provisional API` added to Python 3.8 by :pep:`587`. Patch by Victor +Stinner. diff --git a/Misc/NEWS.d/next/C_API/2025-01-20-10-40-11.gh-issue-129033.d1jltB.rst b/Misc/NEWS.d/next/C_API/2025-01-20-10-40-11.gh-issue-129033.d1jltB.rst new file mode 100644 index 00000000000000..c0c109d5ce1ca2 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-01-20-10-40-11.gh-issue-129033.d1jltB.rst @@ -0,0 +1,4 @@ +Remove ``_PyInterpreterState_GetConfigCopy()`` and +``_PyInterpreterState_SetConfig()`` private functions. Use instead +:c:func:`PyConfig_Get` and :c:func:`PyConfig_Set`, public C API added by +:pep:`741` "Python Configuration C API". Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2025-01-22-09-28-04.gh-issue-128509.gqQ36L.rst b/Misc/NEWS.d/next/C_API/2025-01-22-09-28-04.gh-issue-128509.gqQ36L.rst new file mode 100644 index 00000000000000..c4a048fe3195d1 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-01-22-09-28-04.gh-issue-128509.gqQ36L.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyUnstable_IsImmortal` for determining whether an object is +:term:`immortal`. diff --git a/Misc/NEWS.d/next/C API/README.rst b/Misc/NEWS.d/next/C_API/README.rst similarity index 100% rename from Misc/NEWS.d/next/C API/README.rst rename to Misc/NEWS.d/next/C_API/README.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-04-16-51-56.gh-issue-96497.HTBuIL.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-04-16-51-56.gh-issue-96497.HTBuIL.rst deleted file mode 100644 index 6881dde2e6cf44..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-09-04-16-51-56.gh-issue-96497.HTBuIL.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix incorrect resolution of mangled class variables used in assignment -expressions in comprehensions. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-11-36-50.gh-issue-101860.CKCMbC.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-13-11-36-50.gh-issue-101860.CKCMbC.rst deleted file mode 100644 index 5a274353466973..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-11-36-50.gh-issue-101860.CKCMbC.rst +++ /dev/null @@ -1 +0,0 @@ -Expose ``__name__`` attribute on property. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-16-21-29-06.gh-issue-105858.Q7h0EV.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-16-21-29-06.gh-issue-105858.Q7h0EV.rst deleted file mode 100644 index 9338f662374c9a..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-06-16-21-29-06.gh-issue-105858.Q7h0EV.rst +++ /dev/null @@ -1,8 +0,0 @@ -Improve the constructors for :mod:`ast` nodes. Arguments of list types now -default to an empty list if omitted, and optional fields default to ``None``. -AST nodes now have an -``__annotations__`` attribute with the expected types of their attributes. -Passing unrecognized extra arguments to AST nodes is deprecated and will -become an error in Python 3.15. Omitting a required argument to an AST node -is deprecated and will become an error in Python 3.15. Patch by Jelle -Zijlstra. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-16-15-02-47.gh-issue-104090.oMjNa9.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-16-15-02-47.gh-issue-104090.oMjNa9.rst deleted file mode 100644 index e581d291d047ae..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-16-15-02-47.gh-issue-104090.oMjNa9.rst +++ /dev/null @@ -1,2 +0,0 @@ -The multiprocessing resource tracker now exits with non-zero status code if a resource -leak was detected. It still exits with status code 0 otherwise. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-28-02-46-12.gh-issue-112433.FUX-nT.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-28-02-46-12.gh-issue-112433.FUX-nT.rst deleted file mode 100644 index fdd11bdf4241b9..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-28-02-46-12.gh-issue-112433.FUX-nT.rst +++ /dev/null @@ -1 +0,0 @@ -Add ability to force alignment of :mod:`ctypes.Structure` by way of the new ``_align_`` attribute on the class. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-08-16-01-18.gh-issue-115154.ji96FV.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-08-16-01-18.gh-issue-115154.ji96FV.rst deleted file mode 100644 index 045596bfcdca43..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-08-16-01-18.gh-issue-115154.ji96FV.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug that was causing the :func:`tokenize.untokenize` function to -handle unicode named literals incorrectly. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-09-18-59-22.gh-issue-112175.qglugr.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-09-18-59-22.gh-issue-112175.qglugr.rst deleted file mode 100644 index 6d919134bf4d9c..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-09-18-59-22.gh-issue-112175.qglugr.rst +++ /dev/null @@ -1 +0,0 @@ -Every ``PyThreadState`` now has its own ``eval_breaker``, allowing specific threads to be interrupted. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-12-23-29-17.gh-issue-115323.3t6687.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-12-23-29-17.gh-issue-115323.3t6687.rst deleted file mode 100644 index 171855608fbc6a..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-12-23-29-17.gh-issue-115323.3t6687.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make error message more meaningful for when :meth:`bytearray.extend` is -called with a :class:`str` object. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-14-23-50-43.gh-issue-115347.VkHvQC.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-14-23-50-43.gh-issue-115347.VkHvQC.rst deleted file mode 100644 index 16f357a944ed7a..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-14-23-50-43.gh-issue-115347.VkHvQC.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix bug where docstring was replaced by a redundant NOP when Python is run -with ``-OO``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-20-12-46-20.gh-issue-115700.KLJ5r4.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-20-12-46-20.gh-issue-115700.KLJ5r4.rst deleted file mode 100644 index 5b7b8e410b5063..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-20-12-46-20.gh-issue-115700.KLJ5r4.rst +++ /dev/null @@ -1 +0,0 @@ -The regen-cases build stage now works on Windows. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-20-18-49-02.gh-issue-115733.51Zb85.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-20-18-49-02.gh-issue-115733.51Zb85.rst deleted file mode 100644 index 5cbb292065b5da..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-20-18-49-02.gh-issue-115733.51Zb85.rst +++ /dev/null @@ -1 +0,0 @@ -Fix crash when calling ``next()`` on exhausted list iterators. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-22-11-33-20.gh-issue-115778.jksd1D.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-22-11-33-20.gh-issue-115778.jksd1D.rst deleted file mode 100644 index 023f6aa3026733..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-22-11-33-20.gh-issue-115778.jksd1D.rst +++ /dev/null @@ -1 +0,0 @@ -Add ``tierN`` annotation for instruction definition in interpreter DSL. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-22-16-17-53.gh-issue-115823.c1TreJ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-22-16-17-53.gh-issue-115823.c1TreJ.rst deleted file mode 100644 index 8cda4c9343d4d7..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-02-22-16-17-53.gh-issue-115823.c1TreJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Properly calculate error ranges in the parser when raising -:exc:`SyntaxError` exceptions caused by invalid byte sequences. Patch by -Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-04-10-19-51.gh-issue-116296.gvtxyU.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-04-10-19-51.gh-issue-116296.gvtxyU.rst deleted file mode 100644 index 0781e9282205d1..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2024-03-04-10-19-51.gh-issue-116296.gvtxyU.rst +++ /dev/null @@ -1 +0,0 @@ -Fix possible refleak in :meth:`!object.__reduce__` internal error handling. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2023-12-04-15-53-25.gh-issue-112713.Zrhv77.rst b/Misc/NEWS.d/next/Core_and_Builtins/2023-12-04-15-53-25.gh-issue-112713.Zrhv77.rst new file mode 100644 index 00000000000000..ee1f33f95647bd --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2023-12-04-15-53-25.gh-issue-112713.Zrhv77.rst @@ -0,0 +1 @@ +Added support for the ``Partitioned`` cookie flag in :mod:`http.cookies`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-02-29-16-55-52.gh-issue-115911.Vnkue_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-02-29-16-55-52.gh-issue-115911.Vnkue_.rst new file mode 100644 index 00000000000000..717804be95b18b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-02-29-16-55-52.gh-issue-115911.Vnkue_.rst @@ -0,0 +1,3 @@ +If the current working directory cannot be determined due to permissions, +then import will no longer raise :exc:`PermissionError`. Patch by Alex +Willmer. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-03-06-05-16.gh-issue-126349.7YwWsI.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-03-06-05-16.gh-issue-126349.7YwWsI.rst new file mode 100644 index 00000000000000..aecc8c9abf3ce9 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-03-06-05-16.gh-issue-126349.7YwWsI.rst @@ -0,0 +1,2 @@ +Add :func:`turtle.fill`, :func:`turtle.poly` and :func:`turtle.no_animation` context managers. +Patch by Marie Roald and Yngve Mardal Moe. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-18-12-17-45.gh-issue-125723.tW_hFG.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-18-12-17-45.gh-issue-125723.tW_hFG.rst new file mode 100644 index 00000000000000..62ca6f62f521a8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-18-12-17-45.gh-issue-125723.tW_hFG.rst @@ -0,0 +1,2 @@ +Fix crash with ``gi_frame.f_locals`` when generator frames outlive their +generator. Patch by Mikhail Efimov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-04-22-14-40.gh-issue-127119._hpyFE.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-04-22-14-40.gh-issue-127119._hpyFE.rst new file mode 100644 index 00000000000000..f021bd490f488c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-04-22-14-40.gh-issue-127119._hpyFE.rst @@ -0,0 +1 @@ +Slightly optimize the :class:`int` deallocator. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-17-46.gh-issue-126004.-p8MAS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-17-46.gh-issue-126004.-p8MAS.rst new file mode 100644 index 00000000000000..60b1c5d8b80793 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-17-46.gh-issue-126004.-p8MAS.rst @@ -0,0 +1,3 @@ +Fix handling of :attr:`UnicodeError.start` and :attr:`UnicodeError.end` +values in the :func:`codecs.xmlcharrefreplace_errors` error handler. +Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-30-58.gh-issue-126004.-p8MAS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-30-58.gh-issue-126004.-p8MAS.rst new file mode 100644 index 00000000000000..619d73042a9bb8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-30-58.gh-issue-126004.-p8MAS.rst @@ -0,0 +1,3 @@ +Fix handling of :attr:`UnicodeError.start` and :attr:`UnicodeError.end` +values in the :func:`codecs.backslashreplace_errors` error handler. Patch by +Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-32-58.gh-issue-126004.CYAwTB.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-32-58.gh-issue-126004.CYAwTB.rst new file mode 100644 index 00000000000000..de70c59ee48eec --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-32-58.gh-issue-126004.CYAwTB.rst @@ -0,0 +1,3 @@ +Fix handling of :attr:`UnicodeError.start` and :attr:`UnicodeError.end` +values in the :func:`codecs.replace_errors` error handler. Patch by Bénédikt +Tran. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-09-28-17.gh-issue-128016.DPqhah.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-09-28-17.gh-issue-128016.DPqhah.rst new file mode 100644 index 00000000000000..0832d777bc3251 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-17-09-28-17.gh-issue-128016.DPqhah.rst @@ -0,0 +1 @@ +Improved the ``SyntaxWarning`` message for invalid escape sequences to clarify that such sequences will raise a ``SyntaxError`` in future Python releases. The new message also suggests a potential fix, i.e., ``Did you mean "\\e"?``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-29-15-09-21.gh-issue-128330.IaYL7G.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-29-15-09-21.gh-issue-128330.IaYL7G.rst new file mode 100644 index 00000000000000..8fe628d18f4bd4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-29-15-09-21.gh-issue-128330.IaYL7G.rst @@ -0,0 +1 @@ +Restore terminal control characters on REPL exit. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-30-15-49-31.gh-issue-127953.B4_6L9.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-30-15-49-31.gh-issue-127953.B4_6L9.rst new file mode 100644 index 00000000000000..f19afcd90b16ea --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-30-15-49-31.gh-issue-127953.B4_6L9.rst @@ -0,0 +1,2 @@ +The time to handle a ``LINE`` event in sys.monitoring (and sys.settrace) is +now independent of the number of lines in the code object. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-07-19-26-40.gh-issue-126703.9i-S5t.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-07-19-26-40.gh-issue-126703.9i-S5t.rst new file mode 100644 index 00000000000000..dcd5f449c98ef3 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-07-19-26-40.gh-issue-126703.9i-S5t.rst @@ -0,0 +1 @@ +Improve performance of iterating over lists and tuples by using a freelist for the iterator objects. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-23-54-16.gh-issue-100239.ijOOUs.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-23-54-16.gh-issue-100239.ijOOUs.rst new file mode 100644 index 00000000000000..f58c1fc767515e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-23-54-16.gh-issue-100239.ijOOUs.rst @@ -0,0 +1,2 @@ +Add opcode ``BINARY_OP_EXTEND`` which executes a pair of functions (guard and +specialization functions) accessed from the inline cache. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-13-17-03-49.gh-issue-128807.BGGBxD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-13-17-03-49.gh-issue-128807.BGGBxD.rst new file mode 100644 index 00000000000000..34952e9abb66e5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-13-17-03-49.gh-issue-128807.BGGBxD.rst @@ -0,0 +1,6 @@ +Add a marking phase to the free-threaded GC. This is similar to what was +done in GH-126491. Since the free-threaded GC does not have generations and +is not incremental, the marking phase looks for all objects reachable from +known roots. The roots are objects known to not be garbage, like the module +dictionary for :mod:`sys`. For most programs, this marking phase should +make the GC a bit faster since typically less work is done per object. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-18-16-18.gh-issue-128910.9pqfab.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-18-16-18.gh-issue-128910.9pqfab.rst new file mode 100644 index 00000000000000..e095ba9ebf6be4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-18-16-18.gh-issue-128910.9pqfab.rst @@ -0,0 +1,2 @@ +Undocumented and unused private C-API functions ``_PyTrash_begin`` and +``_PyTrash_end`` are removed. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-22-54-12.gh-issue-100239.7_HpBU.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-22-54-12.gh-issue-100239.7_HpBU.rst new file mode 100644 index 00000000000000..6f086b7ecc0036 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-16-22-54-12.gh-issue-100239.7_HpBU.rst @@ -0,0 +1 @@ +Specialize ``BINARY_OP`` for bitwise logical operations on compact ints. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-18-01-06-58.gh-issue-128799.vSNagk.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-18-01-06-58.gh-issue-128799.vSNagk.rst new file mode 100644 index 00000000000000..eb2361bb5d4525 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-18-01-06-58.gh-issue-128799.vSNagk.rst @@ -0,0 +1 @@ +Add frame of ``except*`` to traceback when it wraps a naked exception. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-21-19-48-30.gh-issue-124363.vOFhHW.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-21-19-48-30.gh-issue-124363.vOFhHW.rst new file mode 100644 index 00000000000000..553aa5a4dd573e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-21-19-48-30.gh-issue-124363.vOFhHW.rst @@ -0,0 +1 @@ +Treat debug expressions in f-string as raw strings. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-21-23-35-41.gh-issue-129093.0rvETC.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-21-23-35-41.gh-issue-129093.0rvETC.rst new file mode 100644 index 00000000000000..067d52eff2da1e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-21-23-35-41.gh-issue-129093.0rvETC.rst @@ -0,0 +1,2 @@ +Fix f-strings such as ``f'{expr=}'`` sometimes not displaying the full +expression when the expression contains ``!=``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-22-14-24-44.gh-issue-129149.wAYu43.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-22-14-24-44.gh-issue-129149.wAYu43.rst new file mode 100644 index 00000000000000..d946f6a2fea185 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-22-14-24-44.gh-issue-129149.wAYu43.rst @@ -0,0 +1,2 @@ +Add fast path for medium-size integers in :c:func:`PyLong_FromUnsignedLong`, +:c:func:`PyLong_FromUnsignedLongLong` and :c:func:`PyLong_FromSize_t`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-28-06-23-59.gh-issue-129345.uOjkML.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-28-06-23-59.gh-issue-129345.uOjkML.rst new file mode 100644 index 00000000000000..68e1103db45652 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-28-06-23-59.gh-issue-129345.uOjkML.rst @@ -0,0 +1 @@ +Fix null pointer dereference in :func:`syslog.openlog` when an audit hook raises an exception. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-28-11-13-41.gh-issue-128563.xElppE.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-28-11-13-41.gh-issue-128563.xElppE.rst new file mode 100644 index 00000000000000..dfd932e836bda6 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-28-11-13-41.gh-issue-128563.xElppE.rst @@ -0,0 +1,2 @@ +Fix an issue where the "lltrace" debug feature could have been incorrectly +enabled for some frames. diff --git a/Misc/NEWS.d/next/Core and Builtins/README.rst b/Misc/NEWS.d/next/Core_and_Builtins/README.rst similarity index 100% rename from Misc/NEWS.d/next/Core and Builtins/README.rst rename to Misc/NEWS.d/next/Core_and_Builtins/README.rst diff --git a/Misc/NEWS.d/next/Documentation/2024-02-14-20-17-04.gh-issue-115399.fb9a0R.rst b/Misc/NEWS.d/next/Documentation/2024-02-14-20-17-04.gh-issue-115399.fb9a0R.rst deleted file mode 100644 index 587aea802168bd..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2024-02-14-20-17-04.gh-issue-115399.fb9a0R.rst +++ /dev/null @@ -1 +0,0 @@ -Document CVE-2023-52425 of Expat <2.6.0 under "XML vulnerabilities". diff --git a/Misc/NEWS.d/next/Documentation/2025-01-14-11-06-41.gh-issue-67206.LYKmi5.rst b/Misc/NEWS.d/next/Documentation/2025-01-14-11-06-41.gh-issue-67206.LYKmi5.rst new file mode 100644 index 00000000000000..11fb617e98edc3 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2025-01-14-11-06-41.gh-issue-67206.LYKmi5.rst @@ -0,0 +1,3 @@ +Document that :const:`string.printable` is not printable in the POSIX sense. +In particular, :meth:`string.printable.isprintable() ` returns +:const:`False`. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Documentation/2025-01-16-18-59-11.gh-issue-125722.eHHRga.rst b/Misc/NEWS.d/next/Documentation/2025-01-16-18-59-11.gh-issue-125722.eHHRga.rst new file mode 100644 index 00000000000000..bf6253eed2eb90 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2025-01-16-18-59-11.gh-issue-125722.eHHRga.rst @@ -0,0 +1,2 @@ +Require Sphinx 8.1.3 or later to build the Python documentation. Patch by +Adam Turner. diff --git a/Misc/NEWS.d/next/IDLE/2023-12-09-11-04-26.gh-issue-88516.SIIvfs.rst b/Misc/NEWS.d/next/IDLE/2023-12-09-11-04-26.gh-issue-88516.SIIvfs.rst deleted file mode 100644 index b6dea5029bf353..00000000000000 --- a/Misc/NEWS.d/next/IDLE/2023-12-09-11-04-26.gh-issue-88516.SIIvfs.rst +++ /dev/null @@ -1,2 +0,0 @@ -On macOS show a proxy icon in the title bar of editor windows to match -platform behaviour. diff --git a/Misc/NEWS.d/next/Library/2019-04-06-23-50-59.bpo-33775.0yhMDc.rst b/Misc/NEWS.d/next/Library/2019-04-06-23-50-59.bpo-33775.0yhMDc.rst deleted file mode 100644 index 2a663ac7940dcb..00000000000000 --- a/Misc/NEWS.d/next/Library/2019-04-06-23-50-59.bpo-33775.0yhMDc.rst +++ /dev/null @@ -1 +0,0 @@ -Add 'default' and 'version' help text for localization in argparse. diff --git a/Misc/NEWS.d/next/Library/2020-05-29-18-08-54.bpo-40818.Ij8ffq.rst b/Misc/NEWS.d/next/Library/2020-05-29-18-08-54.bpo-40818.Ij8ffq.rst deleted file mode 100644 index 27f6a6daa52f53..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-05-29-18-08-54.bpo-40818.Ij8ffq.rst +++ /dev/null @@ -1,3 +0,0 @@ -The asyncio REPL now runs :data:`sys.__interactivehook__` on startup. The -default implementation of :data:`sys.__interactivehook__` provides -auto-completion to the asyncio REPL. Patch contributed by Rémi Lapeyre. diff --git a/Misc/NEWS.d/next/Library/2020-07-13-23-59-42.bpo-41122.8P_Brh.rst b/Misc/NEWS.d/next/Library/2020-07-13-23-59-42.bpo-41122.8P_Brh.rst deleted file mode 100644 index 76568d407449f5..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-07-13-23-59-42.bpo-41122.8P_Brh.rst +++ /dev/null @@ -1,3 +0,0 @@ -Failing to pass arguments properly to :func:`functools.singledispatchmethod` -now throws a TypeError instead of hitting an index out of bounds -internally. diff --git a/Misc/NEWS.d/next/Library/2020-12-15-22-30-49.bpo-42125.UGyseY.rst b/Misc/NEWS.d/next/Library/2020-12-15-22-30-49.bpo-42125.UGyseY.rst deleted file mode 100644 index 49d4462e257702..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-12-15-22-30-49.bpo-42125.UGyseY.rst +++ /dev/null @@ -1,2 +0,0 @@ -linecache: get module name from ``__spec__`` if available. This allows getting -source code for the ``__main__`` module when a custom loader is used. diff --git a/Misc/NEWS.d/next/Library/2021-05-03-11-04-12.bpo-43952.Me7fJe.rst b/Misc/NEWS.d/next/Library/2021-05-03-11-04-12.bpo-43952.Me7fJe.rst deleted file mode 100644 index e164619e44a301..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-03-11-04-12.bpo-43952.Me7fJe.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`multiprocessing.connection.Listener.accept()` to accept empty bytes -as authkey. Not accepting empty bytes as key causes it to hang indefinitely. diff --git a/Misc/NEWS.d/next/Library/2021-08-24-20-47-37.bpo-44865.c3BhZS.rst b/Misc/NEWS.d/next/Library/2021-08-24-20-47-37.bpo-44865.c3BhZS.rst deleted file mode 100644 index ecdb26cdd6edd6..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-24-20-47-37.bpo-44865.c3BhZS.rst +++ /dev/null @@ -1 +0,0 @@ -Add missing call to localization function in :mod:`argparse`. diff --git a/Misc/NEWS.d/next/Library/2022-01-14-10-50-17.bpo-31116.0bduV9.rst b/Misc/NEWS.d/next/Library/2022-01-14-10-50-17.bpo-31116.0bduV9.rst deleted file mode 100644 index d77a96b442bcbb..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-01-14-10-50-17.bpo-31116.0bduV9.rst +++ /dev/null @@ -1 +0,0 @@ -Add Z85 encoding to ``base64``. diff --git a/Misc/NEWS.d/next/Library/2022-05-25-17-49-04.gh-issue-93205.DjhFVR.rst b/Misc/NEWS.d/next/Library/2022-05-25-17-49-04.gh-issue-93205.DjhFVR.rst deleted file mode 100644 index 4a280b93d93347..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-05-25-17-49-04.gh-issue-93205.DjhFVR.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a bug in :class:`logging.handlers.TimedRotatingFileHandler` where multiple rotating handler instances pointing to files with the same name but different extensions would conflict and not delete the correct files. diff --git a/Misc/NEWS.d/next/Library/2022-08-26-15-50-53.gh-issue-96310.0NssDh.rst b/Misc/NEWS.d/next/Library/2022-08-26-15-50-53.gh-issue-96310.0NssDh.rst deleted file mode 100644 index f8efb0002e104a..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-08-26-15-50-53.gh-issue-96310.0NssDh.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a traceback in :mod:`argparse` when all options in a mutually exclusive -group are suppressed. diff --git a/Misc/NEWS.d/next/Library/2022-11-22-23-17-43.gh-issue-95782.an_and.rst b/Misc/NEWS.d/next/Library/2022-11-22-23-17-43.gh-issue-95782.an_and.rst deleted file mode 100644 index 123c3944aa3a3a..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-22-23-17-43.gh-issue-95782.an_and.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix :func:`io.BufferedReader.tell`, :func:`io.BufferedReader.seek`, -:func:`_pyio.BufferedReader.tell`, :func:`io.BufferedRandom.tell`, -:func:`io.BufferedRandom.seek` and :func:`_pyio.BufferedRandom.tell` -being able to return negative offsets. diff --git a/Misc/NEWS.d/next/Library/2023-01-09-14-08-02.gh-issue-100884.DcmdLl.rst b/Misc/NEWS.d/next/Library/2023-01-09-14-08-02.gh-issue-100884.DcmdLl.rst deleted file mode 100644 index 2a388178810835..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-09-14-08-02.gh-issue-100884.DcmdLl.rst +++ /dev/null @@ -1,2 +0,0 @@ -email: fix misfolding of comma in address-lists over multiple lines in -combination with unicode encoding. diff --git a/Misc/NEWS.d/next/Library/2023-01-12-14-16-01.gh-issue-100985.GT5Fvd.rst b/Misc/NEWS.d/next/Library/2023-01-12-14-16-01.gh-issue-100985.GT5Fvd.rst deleted file mode 100644 index 8d8693a5edb3d4..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-12-14-16-01.gh-issue-100985.GT5Fvd.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update HTTPSConnection to consistently wrap IPv6 Addresses when using a -proxy. diff --git a/Misc/NEWS.d/next/Library/2023-02-01-16-41-31.gh-issue-101410.Dt2aQE.rst b/Misc/NEWS.d/next/Library/2023-02-01-16-41-31.gh-issue-101410.Dt2aQE.rst new file mode 100644 index 00000000000000..31493686daec97 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-01-16-41-31.gh-issue-101410.Dt2aQE.rst @@ -0,0 +1,3 @@ +Support custom messages for domain errors in the :mod:`math` module +(:func:`math.sqrt`, :func:`math.log` and :func:`math.atanh` were modified as +examples). Patch by Charlie Zhao and Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Library/2023-02-14-17-19-59.gh-issue-72249.fv35wU.rst b/Misc/NEWS.d/next/Library/2023-02-14-17-19-59.gh-issue-72249.fv35wU.rst deleted file mode 100644 index 10cc5a4e7528dd..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-14-17-19-59.gh-issue-72249.fv35wU.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`functools.partial`s of :func:`repr` has been improved to include the -:term:`module` name. Patched by Furkan Onder and Anilyka Barry. diff --git a/Misc/NEWS.d/next/Library/2023-03-03-09-05-42.gh-issue-102389.ucmo0_.rst b/Misc/NEWS.d/next/Library/2023-03-03-09-05-42.gh-issue-102389.ucmo0_.rst deleted file mode 100644 index 8c11567d79ba7b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-03-09-05-42.gh-issue-102389.ucmo0_.rst +++ /dev/null @@ -1 +0,0 @@ -Add ``windows_31j`` to aliases for ``cp932`` codec diff --git a/Misc/NEWS.d/next/Library/2023-04-02-21-20-35.gh-issue-60346.7mjgua.rst b/Misc/NEWS.d/next/Library/2023-04-02-21-20-35.gh-issue-60346.7mjgua.rst deleted file mode 100644 index c15bd6ed11d17f..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-02-21-20-35.gh-issue-60346.7mjgua.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ArgumentParser inconsistent with parse_known_args. diff --git a/Misc/NEWS.d/next/Library/2023-05-01-22-28-57.gh-issue-104061.vxfBXf.rst b/Misc/NEWS.d/next/Library/2023-05-01-22-28-57.gh-issue-104061.vxfBXf.rst deleted file mode 100644 index e15a811f904352..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-01-22-28-57.gh-issue-104061.vxfBXf.rst +++ /dev/null @@ -1 +0,0 @@ -Add :data:`socket.SO_BINDTOIFINDEX` constant. diff --git a/Misc/NEWS.d/next/Library/2023-05-17-21-33-21.gh-issue-69990.Blvz9G.rst b/Misc/NEWS.d/next/Library/2023-05-17-21-33-21.gh-issue-69990.Blvz9G.rst deleted file mode 100644 index b0cdf44f7b9e39..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-17-21-33-21.gh-issue-69990.Blvz9G.rst +++ /dev/null @@ -1 +0,0 @@ -:meth:`Profile.print_stats` has been improved to accept multiple sort arguments. Patched by Chiu-Hsiang Hsu and Furkan Onder. diff --git a/Misc/NEWS.d/next/Library/2023-07-12-14-52-04.gh-issue-57141.L2k8Xb.rst b/Misc/NEWS.d/next/Library/2023-07-12-14-52-04.gh-issue-57141.L2k8Xb.rst deleted file mode 100644 index b8a1236ec3edb0..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-07-12-14-52-04.gh-issue-57141.L2k8Xb.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add option for *non-shallow* comparisons to :class:`filecmp.dircmp` like -:func:`filecmp.cmp`. Original patch by Steven Ward. Enhanced by -Tobias Rautenkranz diff --git a/Misc/NEWS.d/next/Library/2023-08-02-01-17-32.gh-issue-107155.Mj1K9L.rst b/Misc/NEWS.d/next/Library/2023-08-02-01-17-32.gh-issue-107155.Mj1K9L.rst deleted file mode 100644 index 8362dc0fcfaa74..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-08-02-01-17-32.gh-issue-107155.Mj1K9L.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix incorrect output of ``help(x)`` where ``x`` is a :keyword:`lambda` -function, which has an ``__annotations__`` dictionary attribute with a -``"return"`` key. diff --git a/Misc/NEWS.d/next/Library/2023-11-07-10-22-06.gh-issue-111775.IoVxfX.rst b/Misc/NEWS.d/next/Library/2023-11-07-10-22-06.gh-issue-111775.IoVxfX.rst deleted file mode 100644 index 2a3bdd640ea67d..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-07-10-22-06.gh-issue-111775.IoVxfX.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`importlib.resources.simple.ResourceHandle.open` for text mode, -added missed ``stream`` argument. diff --git a/Misc/NEWS.d/next/Library/2023-11-20-16-15-44.gh-issue-112281.gH4EVk.rst b/Misc/NEWS.d/next/Library/2023-11-20-16-15-44.gh-issue-112281.gH4EVk.rst deleted file mode 100644 index 01f6689bb471cd..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-11-20-16-15-44.gh-issue-112281.gH4EVk.rst +++ /dev/null @@ -1,2 +0,0 @@ -Allow creating :ref:`union of types` for -:class:`typing.Annotated` with unhashable metadata. diff --git a/Misc/NEWS.d/next/Library/2024-01-26-16-42-31.gh-issue-114610.S18Vuz.rst b/Misc/NEWS.d/next/Library/2024-01-26-16-42-31.gh-issue-114610.S18Vuz.rst deleted file mode 100644 index 519aede72aaf29..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-26-16-42-31.gh-issue-114610.S18Vuz.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix bug where :meth:`pathlib.PurePath.with_stem` converted a non-empty path -suffix to a stem when given an empty *stem* argument. It now raises -:exc:`ValueError`, just like :meth:`pathlib.PurePath.with_suffix` does when -called on a path with an empty stem, given a non-empty *suffix* argument. diff --git a/Misc/NEWS.d/next/Library/2024-01-29-13-46-41.gh-issue-114709.SQ998l.rst b/Misc/NEWS.d/next/Library/2024-01-29-13-46-41.gh-issue-114709.SQ998l.rst deleted file mode 100644 index ca0d7902c73d1c..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-29-13-46-41.gh-issue-114709.SQ998l.rst +++ /dev/null @@ -1,5 +0,0 @@ -:func:`posixpath.commonpath()` now raises a :exc:`ValueError` exception when -passed an empty iterable. Previously, :exc:`IndexError` was raised. - -:func:`posixpath.commonpath()` now raises a :exc:`TypeError` exception when -passed ``None``. Previously, :exc:`ValueError` was raised. diff --git a/Misc/NEWS.d/next/Library/2024-01-30-23-28-29.gh-issue-114763.BRjKkg.rst b/Misc/NEWS.d/next/Library/2024-01-30-23-28-29.gh-issue-114763.BRjKkg.rst deleted file mode 100644 index e8bdb83dde61fb..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-01-30-23-28-29.gh-issue-114763.BRjKkg.rst +++ /dev/null @@ -1,3 +0,0 @@ -Protect modules loaded with :class:`importlib.util.LazyLoader` from race -conditions when multiple threads try to access attributes before the loading -is complete. diff --git a/Misc/NEWS.d/next/Library/2024-02-09-12-22-47.gh-issue-113812.wOraaG.rst b/Misc/NEWS.d/next/Library/2024-02-09-12-22-47.gh-issue-113812.wOraaG.rst deleted file mode 100644 index 7ef7bc891cd885..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-09-12-22-47.gh-issue-113812.wOraaG.rst +++ /dev/null @@ -1,3 +0,0 @@ -:meth:`DatagramTransport.sendto` will now send zero-length datagrams if -called with an empty bytes object. The transport flow control also now -accounts for the datagram header when calculating the buffer size. diff --git a/Misc/NEWS.d/next/Library/2024-02-09-19-41-48.gh-issue-115197.20wkWH.rst b/Misc/NEWS.d/next/Library/2024-02-09-19-41-48.gh-issue-115197.20wkWH.rst deleted file mode 100644 index e6ca3cc525d74a..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-09-19-41-48.gh-issue-115197.20wkWH.rst +++ /dev/null @@ -1,2 +0,0 @@ -``urllib.request`` no longer resolves the hostname before checking it -against the system's proxy bypass list on macOS and Windows. diff --git a/Misc/NEWS.d/next/Library/2024-02-10-17-18-49.gh-issue-115256.41Fy9P.rst b/Misc/NEWS.d/next/Library/2024-02-10-17-18-49.gh-issue-115256.41Fy9P.rst deleted file mode 100644 index 8cde053d862298..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-10-17-18-49.gh-issue-115256.41Fy9P.rst +++ /dev/null @@ -1,5 +0,0 @@ -Added DeprecationWarning when accessing the tarfile attribute of TarInfo -objects. The attribute is never used internally and is only attached to -TarInfos when the tarfile is opened in write-mode, not read-mode. The -attribute creates an unnecessary reference cycle which may cause -corruption when not closing the handle after writing a tarfile. diff --git a/Misc/NEWS.d/next/Library/2024-02-11-20-12-39.gh-issue-113942.i72sMJ.rst b/Misc/NEWS.d/next/Library/2024-02-11-20-12-39.gh-issue-113942.i72sMJ.rst deleted file mode 100644 index 2da43a43b78798..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-11-20-12-39.gh-issue-113942.i72sMJ.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`pydoc` no longer skips global functions implemented as builtin methods, -such as :class:`~type.MethodDescriptorType` and :class:`~type.WrapperDescriptorType`. diff --git a/Misc/NEWS.d/next/Library/2024-02-12-11-42-48.gh-issue-103092.sGMKr0.rst b/Misc/NEWS.d/next/Library/2024-02-12-11-42-48.gh-issue-103092.sGMKr0.rst deleted file mode 100644 index 47701396c81737..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-12-11-42-48.gh-issue-103092.sGMKr0.rst +++ /dev/null @@ -1 +0,0 @@ -Isolate :mod:`_lsprof` (apply :pep:`687`). diff --git a/Misc/NEWS.d/next/Library/2024-02-15-19-11-49.gh-issue-101293.898b8l.rst b/Misc/NEWS.d/next/Library/2024-02-15-19-11-49.gh-issue-101293.898b8l.rst deleted file mode 100644 index 98365d2edbc4b5..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-15-19-11-49.gh-issue-101293.898b8l.rst +++ /dev/null @@ -1,4 +0,0 @@ -Support callables with the ``__call__()`` method and types with -``__new__()`` and ``__init__()`` methods set to class methods, static -methods, bound methods, partial functions, and other types of methods and -descriptors in :meth:`inspect.Signature.from_callable`. diff --git a/Misc/NEWS.d/next/Library/2024-02-15-23-42-54.gh-issue-112006.4wxcK-.rst b/Misc/NEWS.d/next/Library/2024-02-15-23-42-54.gh-issue-112006.4wxcK-.rst deleted file mode 100644 index 32af2bd24e54f2..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-15-23-42-54.gh-issue-112006.4wxcK-.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :func:`inspect.unwrap` for types with the ``__wrapper__`` data -descriptor. Fix :meth:`inspect.Signature.from_callable` for builtins -:func:`classmethod` and :func:`staticmethod`. diff --git a/Misc/NEWS.d/next/Library/2024-02-16-16-40-10.gh-issue-112720.io6_Ac.rst b/Misc/NEWS.d/next/Library/2024-02-16-16-40-10.gh-issue-112720.io6_Ac.rst deleted file mode 100644 index 32916ede4dee35..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-16-16-40-10.gh-issue-112720.io6_Ac.rst +++ /dev/null @@ -1,2 +0,0 @@ -Refactor :class:`dis.ArgResolver` to make it possible to subclass and change -the way jump args are interpreted. diff --git a/Misc/NEWS.d/next/Library/2024-02-17-18-47-12.gh-issue-115618.napiNp.rst b/Misc/NEWS.d/next/Library/2024-02-17-18-47-12.gh-issue-115618.napiNp.rst deleted file mode 100644 index cb4b147d5dc663..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-17-18-47-12.gh-issue-115618.napiNp.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix improper decreasing the reference count for ``None`` argument in -:class:`property` methods :meth:`~property.getter`, :meth:`~property.setter` -and :meth:`~property.deleter`. diff --git a/Misc/NEWS.d/next/Library/2024-02-18-12-18-12.gh-issue-111358.9yJUMD.rst b/Misc/NEWS.d/next/Library/2024-02-18-12-18-12.gh-issue-111358.9yJUMD.rst deleted file mode 100644 index 2e895f8f181ce7..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-18-12-18-12.gh-issue-111358.9yJUMD.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug in :meth:`asyncio.BaseEventLoop.shutdown_default_executor` to -ensure the timeout passed to the coroutine behaves as expected. diff --git a/Misc/NEWS.d/next/Library/2024-02-19-15-52-30.gh-issue-114914.M5-1d8.rst b/Misc/NEWS.d/next/Library/2024-02-19-15-52-30.gh-issue-114914.M5-1d8.rst deleted file mode 100644 index f7d392c8bceea3..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-19-15-52-30.gh-issue-114914.M5-1d8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix an issue where an abandoned :class:`StreamWriter` would not be garbage -collected. diff --git a/Misc/NEWS.d/next/Library/2024-02-19-16-53-48.gh-issue-112997.sYBXRZ.rst b/Misc/NEWS.d/next/Library/2024-02-19-16-53-48.gh-issue-112997.sYBXRZ.rst deleted file mode 100644 index 4f97b2d6085ba0..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-19-16-53-48.gh-issue-112997.sYBXRZ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Stop logging potentially sensitive callback arguments in :mod:`asyncio` -unless debug mode is active. diff --git a/Misc/NEWS.d/next/Library/2024-02-20-07-38-15.gh-issue-112364.EX7uGI.rst b/Misc/NEWS.d/next/Library/2024-02-20-07-38-15.gh-issue-112364.EX7uGI.rst deleted file mode 100644 index 6af71e60ec2a8e..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-20-07-38-15.gh-issue-112364.EX7uGI.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed :func:`ast.unparse` to handle format_spec with ``"``, ``'`` or ``\\``. Patched by Frank Hoffmann. diff --git a/Misc/NEWS.d/next/Library/2024-02-20-16-42-54.gh-issue-115712.EXVMXw.rst b/Misc/NEWS.d/next/Library/2024-02-20-16-42-54.gh-issue-115712.EXVMXw.rst deleted file mode 100644 index 8b19064dba779d..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-20-16-42-54.gh-issue-115712.EXVMXw.rst +++ /dev/null @@ -1,4 +0,0 @@ -Restore support of space delimiter with ``skipinitialspace=True`` in -:mod:`csv`. :func:`csv.writer()` now quotes empty fields if delimiter is a -space and skipinitialspace is true and raises exception if quoting is not -possible. diff --git a/Misc/NEWS.d/next/Library/2024-02-20-22-02-34.gh-issue-67044.QF9_Ru.rst b/Misc/NEWS.d/next/Library/2024-02-20-22-02-34.gh-issue-67044.QF9_Ru.rst deleted file mode 100644 index 095e69b6cadab6..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-20-22-02-34.gh-issue-67044.QF9_Ru.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`csv.writer` now always quotes or escapes ``'\r'`` and ``'\n'``, -regardless of *lineterminator* value. diff --git a/Misc/NEWS.d/next/Library/2024-02-22-11-29-27.gh-issue-115809.9H1DhB.rst b/Misc/NEWS.d/next/Library/2024-02-22-11-29-27.gh-issue-115809.9H1DhB.rst deleted file mode 100644 index 2a47efbae5c84d..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-22-11-29-27.gh-issue-115809.9H1DhB.rst +++ /dev/null @@ -1,4 +0,0 @@ -Improve algorithm for computing which rolled-over log files to delete in -:class:`logging.TimedRotatingFileHandler`. It is now reliable for handlers -without ``namer`` and with arbitrary deterministic ``namer`` that leaves the -datetime part in the file name unmodified. diff --git a/Misc/NEWS.d/next/Library/2024-02-22-12-10-18.gh-issue-115714.P2JsU1.rst b/Misc/NEWS.d/next/Library/2024-02-22-12-10-18.gh-issue-115714.P2JsU1.rst deleted file mode 100644 index fb626344c87fdb..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-22-12-10-18.gh-issue-115714.P2JsU1.rst +++ /dev/null @@ -1,4 +0,0 @@ -On WASI, the :mod:`time` module no longer get process time using ``times()`` -or ``CLOCK_PROCESS_CPUTIME_ID``, system API is that is unreliable and is -likely to be removed from WASI. The affected clock functions fall back to -calling ``clock()``. diff --git a/Misc/NEWS.d/next/Library/2024-02-23-11-08-31.gh-issue-115532.zVd3gK.rst b/Misc/NEWS.d/next/Library/2024-02-23-11-08-31.gh-issue-115532.zVd3gK.rst deleted file mode 100644 index fb36c0b2a4fabd..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-23-11-08-31.gh-issue-115532.zVd3gK.rst +++ /dev/null @@ -1 +0,0 @@ -Add kernel density estimation to the statistics module. diff --git a/Misc/NEWS.d/next/Library/2024-02-24-18-48-14.gh-issue-115886.rgM6AF.rst b/Misc/NEWS.d/next/Library/2024-02-24-18-48-14.gh-issue-115886.rgM6AF.rst deleted file mode 100644 index 9688f713d5ba7b..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-24-18-48-14.gh-issue-115886.rgM6AF.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix silent truncation of the name with an embedded null character in -:class:`multiprocessing.shared_memory.SharedMemory`. diff --git a/Misc/NEWS.d/next/Library/2024-02-25-19-20-05.gh-issue-115881.ro_Kuw.rst b/Misc/NEWS.d/next/Library/2024-02-25-19-20-05.gh-issue-115881.ro_Kuw.rst deleted file mode 100644 index 99bccb265ff80c..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-25-19-20-05.gh-issue-115881.ro_Kuw.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix issue where :func:`ast.parse` would incorrectly flag conditional context -managers (such as ``with (x() if y else z()): ...``) as invalid syntax if -``feature_version=(3, 8)`` was passed. This reverts changes to the -grammar made as part of gh-94949. diff --git a/Misc/NEWS.d/next/Library/2024-02-27-20-11-29.gh-issue-85644.3rgcBm.rst b/Misc/NEWS.d/next/Library/2024-02-27-20-11-29.gh-issue-85644.3rgcBm.rst deleted file mode 100644 index 818f7046229878..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-27-20-11-29.gh-issue-85644.3rgcBm.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use the ``XDG_CURRENT_DESKTOP`` environment variable in :mod:`webbrowser` to check desktop. -Prefer it to the deprecated ``GNOME_DESKTOP_SESSION_ID`` for GNOME detection. diff --git a/Misc/NEWS.d/next/Library/2024-02-28-12-14-31.gh-issue-115821.YO2vKA.rst b/Misc/NEWS.d/next/Library/2024-02-28-12-14-31.gh-issue-115821.YO2vKA.rst deleted file mode 100644 index 7512a09a37cd46..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-28-12-14-31.gh-issue-115821.YO2vKA.rst +++ /dev/null @@ -1,2 +0,0 @@ -[Enum] Improve error message when calling super().__new__() in custom -__new__. diff --git a/Misc/NEWS.d/next/Library/2024-02-28-17-04-28.gh-issue-65824.gG8KR1.rst b/Misc/NEWS.d/next/Library/2024-02-28-17-04-28.gh-issue-65824.gG8KR1.rst deleted file mode 100644 index 7bc6ced120a7be..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-28-17-04-28.gh-issue-65824.gG8KR1.rst +++ /dev/null @@ -1 +0,0 @@ -Improve the ``less`` prompt in :mod:`pydoc`. diff --git a/Misc/NEWS.d/next/Library/2024-02-29-20-06-06.gh-issue-87115.FVMiOR.rst b/Misc/NEWS.d/next/Library/2024-02-29-20-06-06.gh-issue-87115.FVMiOR.rst deleted file mode 100644 index 844340583cd456..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-29-20-06-06.gh-issue-87115.FVMiOR.rst +++ /dev/null @@ -1 +0,0 @@ -Set ``__main__.__spec__`` to ``None`` when running a script with :mod:`pdb` diff --git a/Misc/NEWS.d/next/Library/2024-03-01-11-57-32.gh-issue-88352.bZ68rw.rst b/Misc/NEWS.d/next/Library/2024-03-01-11-57-32.gh-issue-88352.bZ68rw.rst deleted file mode 100644 index 8ad4ff7cb52414..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-01-11-57-32.gh-issue-88352.bZ68rw.rst +++ /dev/null @@ -1,6 +0,0 @@ -Fix the computation of the next rollover time in the -:class:`logging.TimedRotatingFileHandler` handler. :meth:`!computeRollover` -now always returns a timestamp larger than the specified time and works -correctly during the DST change. :meth:`!doRollover` no longer overwrite the -already rolled over file, saving from data loss when run at midnight or -during repeated time at the DST change. diff --git a/Misc/NEWS.d/next/Library/2024-03-01-14-22-08.gh-issue-115978.r2ePTo.rst b/Misc/NEWS.d/next/Library/2024-03-01-14-22-08.gh-issue-115978.r2ePTo.rst deleted file mode 100644 index 2adac31ac6c21d..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-01-14-22-08.gh-issue-115978.r2ePTo.rst +++ /dev/null @@ -1,4 +0,0 @@ -Disable preadv(), readv(), pwritev(), and writev() on WASI. - -Under wasmtime for WASI 0.2, these functions don't pass test_posix -(https://github.com/bytecodealliance/wasmtime/issues/7830). diff --git a/Misc/NEWS.d/next/Library/2024-07-14-23-19-20.gh-issue-119257.9OEzcN.rst b/Misc/NEWS.d/next/Library/2024-07-14-23-19-20.gh-issue-119257.9OEzcN.rst new file mode 100644 index 00000000000000..8f3f863d93e021 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-14-23-19-20.gh-issue-119257.9OEzcN.rst @@ -0,0 +1,2 @@ +Show tab completions menu below the current line, which results in less +janky behaviour, and fixes a cursor movement bug. Patch by Daniel Hollas diff --git a/Misc/NEWS.d/next/Library/2024-09-12-14-24-25.gh-issue-123987.7_OD1p.rst b/Misc/NEWS.d/next/Library/2024-09-12-14-24-25.gh-issue-123987.7_OD1p.rst new file mode 100644 index 00000000000000..b110900e7efd33 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-12-14-24-25.gh-issue-123987.7_OD1p.rst @@ -0,0 +1,3 @@ +Fixed issue in NamespaceReader where a non-path item in a namespace path, +such as a sentinel added by an editable installer, would break resource +loading. diff --git a/Misc/NEWS.d/next/Library/2024-09-27-19-21-53.gh-issue-124703.lYTLEv.rst b/Misc/NEWS.d/next/Library/2024-09-27-19-21-53.gh-issue-124703.lYTLEv.rst new file mode 100644 index 00000000000000..e55d3539355d73 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-27-19-21-53.gh-issue-124703.lYTLEv.rst @@ -0,0 +1 @@ +Quitting :mod:`pdb` in ``inline`` mode will emit a confirmation prompt and exit gracefully now, instead of printing an exception traceback. diff --git a/Misc/NEWS.d/next/Library/2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst b/Misc/NEWS.d/next/Library/2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst new file mode 100644 index 00000000000000..c2faf470ffc9cf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst @@ -0,0 +1,2 @@ +Add :func:`asyncio.capture_call_graph` and +:func:`asyncio.print_call_graph` functions. diff --git a/Misc/NEWS.d/next/Library/2024-10-26-16-59-02.gh-issue-125553.4pDLzt.rst b/Misc/NEWS.d/next/Library/2024-10-26-16-59-02.gh-issue-125553.4pDLzt.rst new file mode 100644 index 00000000000000..291c5e6f6f2181 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-26-16-59-02.gh-issue-125553.4pDLzt.rst @@ -0,0 +1,2 @@ +Fix round-trip invariance for backslash continuations in +:func:`tokenize.untokenize`. diff --git a/Misc/NEWS.d/next/Library/2024-11-10-19-45-01.gh-issue-126332.WCCKoH.rst b/Misc/NEWS.d/next/Library/2024-11-10-19-45-01.gh-issue-126332.WCCKoH.rst new file mode 100644 index 00000000000000..9277797ddc745e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-10-19-45-01.gh-issue-126332.WCCKoH.rst @@ -0,0 +1 @@ +Fix _pyrepl crash when entering a double CTRL-Z on an overflowing line. diff --git a/Misc/NEWS.d/next/Library/2024-11-24-22-06-42.gh-issue-127096.R7LLpQ.rst b/Misc/NEWS.d/next/Library/2024-11-24-22-06-42.gh-issue-127096.R7LLpQ.rst new file mode 100644 index 00000000000000..8619296143b7d7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-24-22-06-42.gh-issue-127096.R7LLpQ.rst @@ -0,0 +1,2 @@ +Do not recreate unnamed section on every read in +:class:`configparser.ConfigParser`. Patch by Andrey Efremov. diff --git a/Misc/NEWS.d/next/Library/2024-12-12-18-25-50.gh-issue-127873.WJRwfz.rst b/Misc/NEWS.d/next/Library/2024-12-12-18-25-50.gh-issue-127873.WJRwfz.rst new file mode 100644 index 00000000000000..d7575c7efb6e88 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-12-18-25-50.gh-issue-127873.WJRwfz.rst @@ -0,0 +1,3 @@ +When ``-E`` is set, only ignore ``PYTHON_COLORS`` and not +``FORCE_COLOR``/``NO_COLOR``/``TERM`` when colourising output. +Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2024-12-16-22-20-38.gh-issue-121604.m3Xn4G.rst b/Misc/NEWS.d/next/Library/2024-12-16-22-20-38.gh-issue-121604.m3Xn4G.rst new file mode 100644 index 00000000000000..9a6fce8647cc6b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-16-22-20-38.gh-issue-121604.m3Xn4G.rst @@ -0,0 +1 @@ +Add missing Deprecation warnings for :attr:`importlib.machinery.DEBUG_BYTECODE_SUFFIXES`, :attr:`importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES`, :class:`importlib.machinery.WindowsRegistryFinder`, :class:`importlib.abc.ResourceLoader`, :meth:`importlib.abc.SourceLoader.path_mtime`. diff --git a/Misc/NEWS.d/next/Library/2024-12-21-03-20-12.gh-issue-128131.QpPmNt.rst b/Misc/NEWS.d/next/Library/2024-12-21-03-20-12.gh-issue-128131.QpPmNt.rst new file mode 100644 index 00000000000000..f4c4ebce10729c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-21-03-20-12.gh-issue-128131.QpPmNt.rst @@ -0,0 +1,2 @@ +Completely support random access of uncompressed unencrypted read-only +zip files obtained by :meth:`ZipFile.open `. diff --git a/Misc/NEWS.d/next/Library/2024-12-23-02-09-44.gh-issue-58956.4OdMdT.rst b/Misc/NEWS.d/next/Library/2024-12-23-02-09-44.gh-issue-58956.4OdMdT.rst new file mode 100644 index 00000000000000..b78bc5aaf44217 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-23-02-09-44.gh-issue-58956.4OdMdT.rst @@ -0,0 +1 @@ +Fixed a frame reference leak in :mod:`bdb`. diff --git a/Misc/NEWS.d/next/Library/2024-12-26-11-00-03.gh-issue-112064.mCcw3B.rst b/Misc/NEWS.d/next/Library/2024-12-26-11-00-03.gh-issue-112064.mCcw3B.rst new file mode 100644 index 00000000000000..e885add7b68c0f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-26-11-00-03.gh-issue-112064.mCcw3B.rst @@ -0,0 +1,2 @@ +Fix incorrect handling of negative read sizes in :meth:`HTTPResponse.read +`. Patch by Yury Manushkin. diff --git a/Misc/NEWS.d/next/Library/2024-12-30-19-53-14.gh-issue-91279.EeOJk1.rst b/Misc/NEWS.d/next/Library/2024-12-30-19-53-14.gh-issue-91279.EeOJk1.rst new file mode 100644 index 00000000000000..30ee2ea5efd069 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-30-19-53-14.gh-issue-91279.EeOJk1.rst @@ -0,0 +1,3 @@ +:meth:`zipfile.ZipFile.writestr` now respect ``SOURCE_DATE_EPOCH`` that +distributions can set centrally and have build tools consume this in order +to produce reproducible output. diff --git a/Misc/NEWS.d/next/Library/2025-01-02-20-34-04.gh-issue-128427.onPoQZ.rst b/Misc/NEWS.d/next/Library/2025-01-02-20-34-04.gh-issue-128427.onPoQZ.rst new file mode 100644 index 00000000000000..54cae43702ded7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-02-20-34-04.gh-issue-128427.onPoQZ.rst @@ -0,0 +1,2 @@ +:const:`uuid.NIL` and :const:`uuid.MAX` are now available to represent the Nil +and Max UUID formats as defined by :rfc:`9562`. diff --git a/Misc/NEWS.d/next/Library/2025-01-04-11-10-04.gh-issue-128479.jvOrF-.rst b/Misc/NEWS.d/next/Library/2025-01-04-11-10-04.gh-issue-128479.jvOrF-.rst new file mode 100644 index 00000000000000..fc3b4d5a5273a6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-04-11-10-04.gh-issue-128479.jvOrF-.rst @@ -0,0 +1 @@ +Fix :func:`!asyncio.staggered.staggered_race` leaking tasks and issuing an unhandled exception. diff --git a/Misc/NEWS.d/next/Library/2025-01-06-10-37-27.gh-issue-128384.V0xzwH.rst b/Misc/NEWS.d/next/Library/2025-01-06-10-37-27.gh-issue-128384.V0xzwH.rst new file mode 100644 index 00000000000000..2ca592be20b681 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-06-10-37-27.gh-issue-128384.V0xzwH.rst @@ -0,0 +1,5 @@ +Add locking to :mod:`warnings` to avoid some data races when free-threading +is used. Change ``_warnings_runtime_state.mutex`` to be a recursive mutex +and expose it to :mod:`warnings`, via the :func:`!_acquire_lock` and +:func:`!_release_lock` functions. The lock is held when ``filters`` and +``_filters_version`` are updated. diff --git a/Misc/NEWS.d/next/Library/2025-01-07-21-48-32.gh-issue-128498.n6jtlW.rst b/Misc/NEWS.d/next/Library/2025-01-07-21-48-32.gh-issue-128498.n6jtlW.rst new file mode 100644 index 00000000000000..9a241e37c20a44 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-07-21-48-32.gh-issue-128498.n6jtlW.rst @@ -0,0 +1,2 @@ +Default to stdout isatty for color detection instead of stderr. Patch by +Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2025-01-08-22-30-38.gh-issue-128636.jQfWXj.rst b/Misc/NEWS.d/next/Library/2025-01-08-22-30-38.gh-issue-128636.jQfWXj.rst new file mode 100644 index 00000000000000..80c9840b585530 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-08-22-30-38.gh-issue-128636.jQfWXj.rst @@ -0,0 +1,2 @@ +Fix PyREPL failure when :data:`os.environ` is overwritten with an invalid +value. diff --git a/Misc/NEWS.d/next/Library/2025-01-09-16-20-34.gh-issue-128156.GfObBq.rst b/Misc/NEWS.d/next/Library/2025-01-09-16-20-34.gh-issue-128156.GfObBq.rst new file mode 100644 index 00000000000000..ec6a55040ae6cb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-09-16-20-34.gh-issue-128156.GfObBq.rst @@ -0,0 +1,3 @@ +When using macOS system ``libffi``, support for complex types in +:mod:`ctypes` is now checked at runtime (macOS 10.15 or newer). The types +must also be available at build time. diff --git a/Misc/NEWS.d/next/Library/2025-01-10-13-06-54.gh-issue-118761.f8oADD.rst b/Misc/NEWS.d/next/Library/2025-01-10-13-06-54.gh-issue-118761.f8oADD.rst new file mode 100644 index 00000000000000..37c25cb2efd034 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-10-13-06-54.gh-issue-118761.f8oADD.rst @@ -0,0 +1,5 @@ +Improve the performance of :func:`base64.b16decode` by up to ten times +by more efficiently checking the byte-string for hexadecimal digits. +Reduce the import time of :mod:`base64` by up to six times, +by no longer importing :mod:`re`. +Patch by Bénédikt Tran, Chris Markiewicz, and Adam Turner. diff --git a/Misc/NEWS.d/next/Library/2025-01-13-07-54-32.gh-issue-128308.kYSDRF.rst b/Misc/NEWS.d/next/Library/2025-01-13-07-54-32.gh-issue-128308.kYSDRF.rst new file mode 100644 index 00000000000000..efa613876a35fd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-13-07-54-32.gh-issue-128308.kYSDRF.rst @@ -0,0 +1 @@ +Support the *name* keyword argument for eager tasks in :func:`asyncio.loop.create_task`, :func:`asyncio.create_task` and :func:`asyncio.TaskGroup.create_task`, by passing on all *kwargs* to the task factory set by :func:`asyncio.loop.set_task_factory`. diff --git a/Misc/NEWS.d/next/Library/2025-01-15-09-45-43.gh-issue-118761.TvAC8E.rst b/Misc/NEWS.d/next/Library/2025-01-15-09-45-43.gh-issue-118761.TvAC8E.rst new file mode 100644 index 00000000000000..38d18b7f4ca05e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-15-09-45-43.gh-issue-118761.TvAC8E.rst @@ -0,0 +1,3 @@ +Reduce the import time of :mod:`csv` by up to five times, by importing +:mod:`re` on demand. In particular, ``re`` is no more implicitly exposed +as ``csv.re``. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2025-01-15-18-54-48.gh-issue-118761.G1dv6E.rst b/Misc/NEWS.d/next/Library/2025-01-15-18-54-48.gh-issue-118761.G1dv6E.rst new file mode 100644 index 00000000000000..4144ef8f40e6dd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-15-18-54-48.gh-issue-118761.G1dv6E.rst @@ -0,0 +1,2 @@ +Reduce the import time of :mod:`optparse` when no help text is printed. +Patch by Eli Schwartz. diff --git a/Misc/NEWS.d/next/Library/2025-01-15-19-16-50.gh-issue-118761.cbW2ZL.rst b/Misc/NEWS.d/next/Library/2025-01-15-19-16-50.gh-issue-118761.cbW2ZL.rst new file mode 100644 index 00000000000000..0eef8777512dd8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-15-19-16-50.gh-issue-118761.cbW2ZL.rst @@ -0,0 +1,3 @@ +Reduce import time of :mod:`gettext` by up to ten times, by importing +:mod:`re` on demand. In particular, ``re`` is no longer implicitly +exposed as ``gettext.re``. Patch by Eli Schwartz. diff --git a/Misc/NEWS.d/next/Library/2025-01-15-19-32-23.gh-issue-128891.ojUxKo.rst b/Misc/NEWS.d/next/Library/2025-01-15-19-32-23.gh-issue-128891.ojUxKo.rst new file mode 100644 index 00000000000000..79d845bbab7cfc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-15-19-32-23.gh-issue-128891.ojUxKo.rst @@ -0,0 +1 @@ +Add specialized opcodes to ``opcode.opname``. diff --git a/Misc/NEWS.d/next/Library/2025-01-15-21-41-51.gh-issue-128679.tq10F2.rst b/Misc/NEWS.d/next/Library/2025-01-15-21-41-51.gh-issue-128679.tq10F2.rst new file mode 100644 index 00000000000000..5c108da5703c00 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-15-21-41-51.gh-issue-128679.tq10F2.rst @@ -0,0 +1,3 @@ +:mod:`tracemalloc`: Fix race conditions when :func:`tracemalloc.stop` is +called by a thread, while other threads are tracing memory allocations. +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2025-01-16-10-06-40.gh-issue-118761.z100LC.rst b/Misc/NEWS.d/next/Library/2025-01-16-10-06-40.gh-issue-118761.z100LC.rst new file mode 100644 index 00000000000000..ea71ecaaeb2936 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-16-10-06-40.gh-issue-118761.z100LC.rst @@ -0,0 +1,2 @@ +Improve import time of :mod:`tomllib` by removing ``typing``, ``string``, +and ``tomllib._types`` imports. Patch by Taneli Hukkinen. diff --git a/Misc/NEWS.d/next/Library/2025-01-17-11-46-16.gh-issue-128916.GEePbO.rst b/Misc/NEWS.d/next/Library/2025-01-17-11-46-16.gh-issue-128916.GEePbO.rst new file mode 100644 index 00000000000000..f2db341ef81621 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-17-11-46-16.gh-issue-128916.GEePbO.rst @@ -0,0 +1,3 @@ +Do not attempt to set ``SO_REUSEPORT`` on sockets of address families +other than ``AF_INET`` and ``AF_INET6``, as it is meaningless with these +address families, and the call with fail with Linux kernel 6.12.9 and newer. diff --git a/Misc/NEWS.d/next/Library/2025-01-17-17-20-51.gh-issue-128894.gX1-8J.rst b/Misc/NEWS.d/next/Library/2025-01-17-17-20-51.gh-issue-128894.gX1-8J.rst new file mode 100644 index 00000000000000..7e015103a95713 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-17-17-20-51.gh-issue-128894.gX1-8J.rst @@ -0,0 +1,2 @@ +Fix ``traceback.TracebackException._format_syntax_error`` not to fail on +exceptions with custom metadata. diff --git a/Misc/NEWS.d/next/Library/2025-01-17-21-33-11.gh-issue-128961.XwvyIZ.rst b/Misc/NEWS.d/next/Library/2025-01-17-21-33-11.gh-issue-128961.XwvyIZ.rst new file mode 100644 index 00000000000000..9c985df77743da --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-17-21-33-11.gh-issue-128961.XwvyIZ.rst @@ -0,0 +1 @@ +Fix a crash when setting state on an exhausted :class:`array.array` iterator. diff --git a/Misc/NEWS.d/next/Library/2025-01-18-11-04-44.gh-issue-128978.hwg7-w.rst b/Misc/NEWS.d/next/Library/2025-01-18-11-04-44.gh-issue-128978.hwg7-w.rst new file mode 100644 index 00000000000000..521496d6a2f8c2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-18-11-04-44.gh-issue-128978.hwg7-w.rst @@ -0,0 +1,2 @@ +Fix a :exc:`NameError` in :func:`!sysconfig.expand_makefile_vars`. Patch by +Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2025-01-18-11-24-02.gh-issue-118761.G8MmxY.rst b/Misc/NEWS.d/next/Library/2025-01-18-11-24-02.gh-issue-118761.G8MmxY.rst new file mode 100644 index 00000000000000..3b3f3f7d98c5d6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-18-11-24-02.gh-issue-118761.G8MmxY.rst @@ -0,0 +1,2 @@ +Reduce import time of :mod:`pstats` and :mod:`zipfile` by up to 20%, by +removing unnecessary imports to :mod:`typing`. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2025-01-18-16-58-10.gh-issue-128991.EzJit9.rst b/Misc/NEWS.d/next/Library/2025-01-18-16-58-10.gh-issue-128991.EzJit9.rst new file mode 100644 index 00000000000000..64fa04fb53e89c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-18-16-58-10.gh-issue-128991.EzJit9.rst @@ -0,0 +1 @@ +Release the enter frame reference within :mod:`bdb` callback diff --git a/Misc/NEWS.d/next/Library/2025-01-20-13-12-39.gh-issue-128550.AJ5TOL.rst b/Misc/NEWS.d/next/Library/2025-01-20-13-12-39.gh-issue-128550.AJ5TOL.rst new file mode 100644 index 00000000000000..f59feac795ea18 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-20-13-12-39.gh-issue-128550.AJ5TOL.rst @@ -0,0 +1 @@ +Removed an incorrect optimization relating to eager tasks in :class:`asyncio.TaskGroup` that resulted in cancellations being missed. diff --git a/Misc/NEWS.d/next/Library/2025-01-20-16-02-38.gh-issue-129064.JXasgJ.rst b/Misc/NEWS.d/next/Library/2025-01-20-16-02-38.gh-issue-129064.JXasgJ.rst new file mode 100644 index 00000000000000..93a1eda350d1fc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-20-16-02-38.gh-issue-129064.JXasgJ.rst @@ -0,0 +1,2 @@ +Deprecate :func:`!sysconfig.expand_makefile_vars`, in favor of using +:func:`sysconfig.get_paths` with the ``vars`` argument. diff --git a/Misc/NEWS.d/next/Library/2025-01-20-20-59-26.gh-issue-92897.G0xH8o.rst b/Misc/NEWS.d/next/Library/2025-01-20-20-59-26.gh-issue-92897.G0xH8o.rst new file mode 100644 index 00000000000000..632ca03bbf8dd2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-20-20-59-26.gh-issue-92897.G0xH8o.rst @@ -0,0 +1,2 @@ +Scheduled the deprecation of the ``check_home`` argument of +:func:`sysconfig.is_python_build` to Python 3.15. diff --git a/Misc/NEWS.d/next/Library/2025-01-21-18-52-32.gh-issue-129061.4idD_B.rst b/Misc/NEWS.d/next/Library/2025-01-21-18-52-32.gh-issue-129061.4idD_B.rst new file mode 100644 index 00000000000000..5c5c05e1161e86 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-21-18-52-32.gh-issue-129061.4idD_B.rst @@ -0,0 +1 @@ +Fix FORCE_COLOR and NO_COLOR when empty strings. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2025-01-22-16-54-25.gh-issue-129205.FMqrUt.rst b/Misc/NEWS.d/next/Library/2025-01-22-16-54-25.gh-issue-129205.FMqrUt.rst new file mode 100644 index 00000000000000..c4ed76408f32f6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-22-16-54-25.gh-issue-129205.FMqrUt.rst @@ -0,0 +1 @@ +Add :func:`os.readinto` to read into a :ref:`buffer object ` from a file descriptor. diff --git a/Misc/NEWS.d/next/Library/2025-01-24-10-48-32.gh-issue-129195.89d5NU.rst b/Misc/NEWS.d/next/Library/2025-01-24-10-48-32.gh-issue-129195.89d5NU.rst new file mode 100644 index 00000000000000..daf7297387dabd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-24-10-48-32.gh-issue-129195.89d5NU.rst @@ -0,0 +1 @@ +Support reporting call graph information from :func:`!asyncio.staggered.staggered_race`. diff --git a/Misc/NEWS.d/next/Library/2025-01-26-10-01-21.gh-issue-129005.ncpLvw.rst b/Misc/NEWS.d/next/Library/2025-01-26-10-01-21.gh-issue-129005.ncpLvw.rst new file mode 100644 index 00000000000000..a825e9d244d525 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-26-10-01-21.gh-issue-129005.ncpLvw.rst @@ -0,0 +1 @@ +Optimize ``_pyio.FileIO.readinto`` by avoiding unnecessary objects and copies using :func:`os.readinto`. diff --git a/Misc/NEWS.d/next/Library/2025-01-27-14-05-19.gh-issue-129346.gZRd3g.rst b/Misc/NEWS.d/next/Library/2025-01-27-14-05-19.gh-issue-129346.gZRd3g.rst new file mode 100644 index 00000000000000..b5377277f6c51c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-27-14-05-19.gh-issue-129346.gZRd3g.rst @@ -0,0 +1,2 @@ +In :mod:`sqlite3`, handle out-of-memory when creating user-defined SQL +functions. diff --git a/Misc/NEWS.d/next/Library/2025-01-29-10-53-32.gh-issue-118761.i8wjpV.rst b/Misc/NEWS.d/next/Library/2025-01-29-10-53-32.gh-issue-118761.i8wjpV.rst new file mode 100644 index 00000000000000..0762cbe5d63949 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-29-10-53-32.gh-issue-118761.i8wjpV.rst @@ -0,0 +1,2 @@ +Improve import time of :mod:`subprocess` by lazy importing ``locale`` and +``signal``. Patch by Taneli Hukkinen. diff --git a/Misc/NEWS.d/next/Library/2025-01-29-11-14-20.gh-issue-118761.gMZwE1.rst b/Misc/NEWS.d/next/Library/2025-01-29-11-14-20.gh-issue-118761.gMZwE1.rst new file mode 100644 index 00000000000000..c2474795d8233e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-29-11-14-20.gh-issue-118761.gMZwE1.rst @@ -0,0 +1,2 @@ +Always lazy import ``warnings`` in :mod:`threading`. Patch by Taneli +Hukkinen. diff --git a/Misc/NEWS.d/next/Library/2025-01-29-13-37-18.gh-issue-126400.DaBaR3.rst b/Misc/NEWS.d/next/Library/2025-01-29-13-37-18.gh-issue-126400.DaBaR3.rst new file mode 100644 index 00000000000000..1532faf4b7d6f5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-29-13-37-18.gh-issue-126400.DaBaR3.rst @@ -0,0 +1,2 @@ +Add a socket *timeout* keyword argument to +:class:`logging.handlers.SysLogHandler`. diff --git a/Misc/NEWS.d/next/Library/2025-01-29-14-30-54.gh-issue-129409.JZbOE6.rst b/Misc/NEWS.d/next/Library/2025-01-29-14-30-54.gh-issue-129409.JZbOE6.rst new file mode 100644 index 00000000000000..7e00b44c0ef471 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-29-14-30-54.gh-issue-129409.JZbOE6.rst @@ -0,0 +1,2 @@ +Fix an integer overflow in the :mod:`csv` module when writing a data field +larger than 2GB. diff --git a/Misc/NEWS.d/next/Library/2025-01-29-17-10-00.gh-issue-129403.314159.rst b/Misc/NEWS.d/next/Library/2025-01-29-17-10-00.gh-issue-129403.314159.rst new file mode 100644 index 00000000000000..0c2bdd3136e3a3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-29-17-10-00.gh-issue-129403.314159.rst @@ -0,0 +1 @@ +Corrected :exc:`ValueError` message for :class:`asyncio.Barrier` and :class:`threading.Barrier`. diff --git a/Misc/NEWS.d/next/Security/2024-01-26-22-14-09.gh-issue-114572.t1QMQD.rst b/Misc/NEWS.d/next/Security/2024-01-26-22-14-09.gh-issue-114572.t1QMQD.rst deleted file mode 100644 index b4f9fe64db0615..00000000000000 --- a/Misc/NEWS.d/next/Security/2024-01-26-22-14-09.gh-issue-114572.t1QMQD.rst +++ /dev/null @@ -1,4 +0,0 @@ -:meth:`ssl.SSLContext.cert_store_stats` and -:meth:`ssl.SSLContext.get_ca_certs` now correctly lock access to the -certificate store, when the :class:`ssl.SSLContext` is shared across -multiple threads. diff --git a/Misc/NEWS.d/next/Security/2024-02-18-03-14-40.gh-issue-115398.tzvxH8.rst b/Misc/NEWS.d/next/Security/2024-02-18-03-14-40.gh-issue-115398.tzvxH8.rst deleted file mode 100644 index a40fcd35ef99ae..00000000000000 --- a/Misc/NEWS.d/next/Security/2024-02-18-03-14-40.gh-issue-115398.tzvxH8.rst +++ /dev/null @@ -1,8 +0,0 @@ -Allow controlling Expat >=2.6.0 reparse deferral (CVE-2023-52425) by adding -five new methods: - -* :meth:`xml.etree.ElementTree.XMLParser.flush` -* :meth:`xml.etree.ElementTree.XMLPullParser.flush` -* :meth:`xml.parsers.expat.xmlparser.GetReparseDeferralEnabled` -* :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` -* :meth:`xml.sax.expatreader.ExpatParser.flush` diff --git a/Misc/NEWS.d/next/Security/2024-05-24-21-00-52.gh-issue-119511.jKrXQ8.rst b/Misc/NEWS.d/next/Security/2024-05-24-21-00-52.gh-issue-119511.jKrXQ8.rst new file mode 100644 index 00000000000000..f7b4031120e643 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-05-24-21-00-52.gh-issue-119511.jKrXQ8.rst @@ -0,0 +1,7 @@ +Fix a potential denial of service in the :mod:`imaplib` module. When connecting +to a malicious server, it could cause an arbitrary amount of memory to be +allocated. On many systems this is harmless as unused virtual memory is only a +mapping, but if this hit a virtual address size limit it could lead to a +:exc:`MemoryError` or other process crash. On unusual systems or builds where +all allocated memory is touched and backed by actual ram or storage it could've +consumed resources doing so until similarly crashing. diff --git a/Misc/NEWS.d/next/Security/2024-08-06-11-43-08.gh-issue-80222.wfR4BU.rst b/Misc/NEWS.d/next/Security/2024-08-06-11-43-08.gh-issue-80222.wfR4BU.rst new file mode 100644 index 00000000000000..0f0661d0b1cf4a --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-08-06-11-43-08.gh-issue-80222.wfR4BU.rst @@ -0,0 +1,6 @@ +Fix bug in the folding of quoted strings when flattening an email message using +a modern email policy. Previously when a quoted string was folded so that +it spanned more than one line, the surrounding quotes and internal escapes +would be omitted. This could theoretically be used to spoof header lines +using a carefully constructed quoted string if the resulting rendered email +was transmitted or re-parsed. diff --git a/Misc/NEWS.d/next/Tests/2024-02-12-22-35-01.gh-issue-115376.n9vubZ.rst b/Misc/NEWS.d/next/Tests/2024-02-12-22-35-01.gh-issue-115376.n9vubZ.rst deleted file mode 100644 index e09d78a9c4b189..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-02-12-22-35-01.gh-issue-115376.n9vubZ.rst +++ /dev/null @@ -1 +0,0 @@ -Fix segfault in ``_testinternalcapi.compiler_codegen`` on bad input. diff --git a/Misc/NEWS.d/next/Tests/2024-02-13-18-24-04.gh-issue-115420.-dlzfI.rst b/Misc/NEWS.d/next/Tests/2024-02-13-18-24-04.gh-issue-115420.-dlzfI.rst deleted file mode 100644 index 1442ada3490fa0..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-02-13-18-24-04.gh-issue-115420.-dlzfI.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix translation of exception hander targets by -``_testinternalcapi.optimize_cfg``. diff --git a/Misc/NEWS.d/next/Tests/2024-02-16-13-04-28.gh-issue-115556.rjaQ9w.rst b/Misc/NEWS.d/next/Tests/2024-02-16-13-04-28.gh-issue-115556.rjaQ9w.rst deleted file mode 100644 index c2811b133d9314..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-02-16-13-04-28.gh-issue-115556.rjaQ9w.rst +++ /dev/null @@ -1,2 +0,0 @@ -On Windows, commas passed in arguments to ``Tools\buildbot\test.bat`` and -``PCbuild\\rt.bat`` are now properly handled. diff --git a/Misc/NEWS.d/next/Tests/2024-02-17-08-25-01.gh-issue-115596.RGPCrR.rst b/Misc/NEWS.d/next/Tests/2024-02-17-08-25-01.gh-issue-115596.RGPCrR.rst deleted file mode 100644 index 2bcb8b9ac6bcd4..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-02-17-08-25-01.gh-issue-115596.RGPCrR.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix ``ProgramPriorityTests`` in ``test_os`` permanently changing the process -priority. diff --git a/Misc/NEWS.d/next/Tests/2024-02-18-14-20-52.gh-issue-115122.3rGNo9.rst b/Misc/NEWS.d/next/Tests/2024-02-18-14-20-52.gh-issue-115122.3rGNo9.rst deleted file mode 100644 index e187a40a40516b..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-02-18-14-20-52.gh-issue-115122.3rGNo9.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add ``--bisect`` option to regrtest test runner: run failed tests with -``test.bisect_cmd`` to identify failing tests. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2024-02-20-15-47-41.gh-issue-115720.w8i8UG.rst b/Misc/NEWS.d/next/Tests/2024-02-20-15-47-41.gh-issue-115720.w8i8UG.rst deleted file mode 100644 index a03ee11d974251..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-02-20-15-47-41.gh-issue-115720.w8i8UG.rst +++ /dev/null @@ -1,2 +0,0 @@ -Leak tests (``-R``, ``--huntrleaks``) now show a summary of the number of -leaks found in each iteration. diff --git a/Misc/NEWS.d/next/Tests/2024-02-22-00-17-06.gh-issue-115796.d4hpKy.rst b/Misc/NEWS.d/next/Tests/2024-02-22-00-17-06.gh-issue-115796.d4hpKy.rst deleted file mode 100644 index a40be74f73908e..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-02-22-00-17-06.gh-issue-115796.d4hpKy.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make '_testinternalcapi.assemble_code_object' construct the exception table -for the code object. diff --git a/Misc/NEWS.d/next/Tests/2024-02-25-15-58-28.gh-issue-71052.lxBjqY.rst b/Misc/NEWS.d/next/Tests/2024-02-25-15-58-28.gh-issue-71052.lxBjqY.rst deleted file mode 100644 index 8bac68b5aea78e..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-02-25-15-58-28.gh-issue-71052.lxBjqY.rst +++ /dev/null @@ -1,2 +0,0 @@ -Enable ``test_concurrent_futures`` on platforms that support threading but not -multiprocessing. diff --git a/Misc/NEWS.d/next/Tests/2024-02-25-16-28-26.gh-issue-71052.lSb9EC.rst b/Misc/NEWS.d/next/Tests/2024-02-25-16-28-26.gh-issue-71052.lSb9EC.rst deleted file mode 100644 index 9d3467ca7e7d40..00000000000000 --- a/Misc/NEWS.d/next/Tests/2024-02-25-16-28-26.gh-issue-71052.lSb9EC.rst +++ /dev/null @@ -1 +0,0 @@ -Add test exclusions to support running the test suite on Android. diff --git a/Misc/NEWS.d/next/Tests/2025-01-04-02-41-41.gh-issue-128474.0b-tl4.rst b/Misc/NEWS.d/next/Tests/2025-01-04-02-41-41.gh-issue-128474.0b-tl4.rst new file mode 100644 index 00000000000000..7dc807757c5ec0 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2025-01-04-02-41-41.gh-issue-128474.0b-tl4.rst @@ -0,0 +1,2 @@ +Disable ``test_embed`` test cases that segfault on BOLT instrument binaries. +The tests are only disabled when BOLT is enabled. diff --git a/Misc/NEWS.d/next/Tools-Demos/2021-09-05-02-47-48.bpo-45101.60Zqmt.rst b/Misc/NEWS.d/next/Tools-Demos/2021-09-05-02-47-48.bpo-45101.60Zqmt.rst deleted file mode 100644 index 48a09da7822915..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2021-09-05-02-47-48.bpo-45101.60Zqmt.rst +++ /dev/null @@ -1 +0,0 @@ -Add consistency in usage message IO between 2 versions of python-config. diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-02-12-19-28-08.gh-issue-100176.Kzs4Zw.rst b/Misc/NEWS.d/next/Tools-Demos/2023-02-12-19-28-08.gh-issue-100176.Kzs4Zw.rst deleted file mode 100644 index 1a9fc76d93f297..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2023-02-12-19-28-08.gh-issue-100176.Kzs4Zw.rst +++ /dev/null @@ -1 +0,0 @@ -Remove outdated Tools/{io,cc,string}bench diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-01-24-14-49-40.gh-issue-129248.JAapG2.rst b/Misc/NEWS.d/next/Tools-Demos/2025-01-24-14-49-40.gh-issue-129248.JAapG2.rst new file mode 100644 index 00000000000000..e3c781cf10cf50 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-01-24-14-49-40.gh-issue-129248.JAapG2.rst @@ -0,0 +1,2 @@ +The iOS test runner now strips the log prefix from each line output by the +test suite. diff --git a/Misc/NEWS.d/next/Windows/2024-02-15-23-16-31.gh-issue-115543.otrWnw.rst b/Misc/NEWS.d/next/Windows/2024-02-15-23-16-31.gh-issue-115543.otrWnw.rst deleted file mode 100644 index ebd15c83b83491..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-02-15-23-16-31.gh-issue-115543.otrWnw.rst +++ /dev/null @@ -1,3 +0,0 @@ -:ref:`launcher` can now detect Python 3.13 when installed from the Microsoft -Store, and will install Python 3.12 by default when -:envvar:`PYLAUNCHER_ALLOW_INSTALL` is set. diff --git a/Misc/NEWS.d/next/Windows/2024-02-21-23-48-59.gh-issue-115554.02mpQC.rst b/Misc/NEWS.d/next/Windows/2024-02-21-23-48-59.gh-issue-115554.02mpQC.rst deleted file mode 100644 index b3c078b578205e..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-02-21-23-48-59.gh-issue-115554.02mpQC.rst +++ /dev/null @@ -1,6 +0,0 @@ -The installer now has more strict rules about updating the :ref:`launcher`. -In general, most users only have a single launcher installed and will see no -difference. When multiple launchers have been installed, the option to -install the launcher is disabled until all but one have been removed. -Downgrading the launcher (which was never allowed) is now more obviously -blocked. diff --git a/Misc/NEWS.d/next/Windows/2024-02-23-11-43-43.gh-issue-115582.sk1XPi.rst b/Misc/NEWS.d/next/Windows/2024-02-23-11-43-43.gh-issue-115582.sk1XPi.rst deleted file mode 100644 index f2e82bf6a3e028..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-02-23-11-43-43.gh-issue-115582.sk1XPi.rst +++ /dev/null @@ -1,3 +0,0 @@ -Building extensions intended for free-threaded builds of CPython now require -compiling with ``/DPy_GIL_DISABLED`` manually when using a regular install. This -is expected to change in future releases. diff --git a/Misc/NEWS.d/next/Windows/2024-02-27-23-21-55.gh-issue-116012.B9_IwM.rst b/Misc/NEWS.d/next/Windows/2024-02-27-23-21-55.gh-issue-116012.B9_IwM.rst deleted file mode 100644 index a55e5b1c7b566d..00000000000000 --- a/Misc/NEWS.d/next/Windows/2024-02-27-23-21-55.gh-issue-116012.B9_IwM.rst +++ /dev/null @@ -1 +0,0 @@ -Ensure the value of ``GetLastError()`` is preserved across GIL operations. diff --git a/Misc/README b/Misc/README index 3dab768ba1a7a4..cbad9b72dc713c 100644 --- a/Misc/README +++ b/Misc/README @@ -17,7 +17,6 @@ python.man UNIX man page for the python interpreter python.pc.in Package configuration info template for pkg-config README The file you're reading now README.AIX Information about using Python on AIX -README.coverity Information about running Coverity's Prevent on Python README.valgrind Information for Valgrind users, see valgrind-python.supp SpecialBuilds.txt Describes extra symbols you can set for debug builds svnmap.txt Map of old SVN revs and branches to hg changeset ids, diff --git a/Misc/README.coverity b/Misc/README.coverity deleted file mode 100644 index f5e1bf6f28d245..00000000000000 --- a/Misc/README.coverity +++ /dev/null @@ -1,22 +0,0 @@ - -Coverity has a static analysis tool (Prevent) which is similar to Klocwork. -They run their tool on the Python source code (SVN head) on a daily basis. -The results are available at: - - http://scan.coverity.com/ - -About 20 people have access to the analysis reports. Other -people can be added by request. - -Prevent was first run on the Python 2.5 source code in March 2006. -There were originally about 100 defects reported. Some of these -were false positives. Over 70 issues were uncovered. - -Each warning has a unique id and comments that can be made on it. -When checking in changes due to a warning, the unique id -as reported by the tool was added to the SVN commit message. - -False positives were annotated so that the comments can -be reviewed and reversed if the analysis was incorrect. - -Contact python-dev@python.org for more information. diff --git a/Misc/coverity_model.c b/Misc/coverity_model.c deleted file mode 100644 index 8960362a6d74a0..00000000000000 --- a/Misc/coverity_model.c +++ /dev/null @@ -1,179 +0,0 @@ -/* Coverity Scan model - * - * This is a modeling file for Coverity Scan. Modeling helps to avoid false - * positives. - * - * - A model file can't import any header files. - * - Therefore only some built-in primitives like int, char and void are - * available but not wchar_t, NULL etc. - * - Modeling doesn't need full structs and typedefs. Rudimentary structs - * and similar types are sufficient. - * - An uninitialized local pointer is not an error. It signifies that the - * variable could be either NULL or have some data. - * - * Coverity Scan doesn't pick up modifications automatically. The model file - * must be uploaded by an admin in the analysis settings of - * http://scan.coverity.com/projects/200 - */ - -/* dummy definitions, in most cases struct fields aren't required. */ - -#define NULL (void *)0 -#define assert(op) /* empty */ -typedef int sdigit; -typedef long Py_ssize_t; -typedef unsigned short wchar_t; -typedef struct {} PyObject; -typedef struct {} grammar; -typedef struct {} DIR; -typedef struct {} RFILE; - -/* Python/pythonrun.c - * resource leak false positive */ - -void Py_FatalError(const char *msg) { - __coverity_panic__(); -} - -/* Objects/longobject.c - * NEGATIVE_RETURNS false positive */ - -static PyObject *get_small_int(sdigit ival) -{ - /* Never returns NULL */ - PyObject *p; - assert(p != NULL); - return p; -} - -PyObject *PyLong_FromLong(long ival) -{ - PyObject *p; - int maybe; - - if ((ival >= -5) && (ival < 257 + 5)) { - p = get_small_int(ival); - assert(p != NULL); - return p; - } - if (maybe) - return p; - else - return NULL; -} - -PyObject *PyLong_FromLongLong(long long ival) -{ - return PyLong_FromLong((long)ival); -} - -PyObject *PyLong_FromSsize_t(Py_ssize_t ival) -{ - return PyLong_FromLong((long)ival); -} - -/* tainted sinks - * - * Coverity considers argv, environ, read() data etc as tained. - */ - -PyObject *PyErr_SetFromErrnoWithFilename(PyObject *exc, const char *filename) -{ - __coverity_tainted_data_sink__(filename); - return NULL; -} - -/* Python/fileutils.c */ -wchar_t *Py_DecodeLocale(const char* arg, size_t *size) -{ - wchar_t *w; - __coverity_tainted_data_sink__(arg); - __coverity_tainted_data_sink__(size); - return w; -} - -/* Python/marshal.c */ - -static Py_ssize_t r_string(char *s, Py_ssize_t n, RFILE *p) -{ - __coverity_tainted_string_argument__(s); - return 0; -} - -static long r_long(RFILE *p) -{ - long l; - unsigned char buffer[4]; - - r_string((char *)buffer, 4, p); - __coverity_tainted_string_sanitize_content__(buffer); - l = (long)buffer; - return l; -} - -/* Coverity doesn't understand that fdopendir() may take ownership of fd. */ - -DIR *fdopendir(int fd) -{ - DIR *d; - if (d) { - __coverity_close__(fd); - } - return d; -} - -/* Modules/_datetime.c - * - * Coverity thinks that the input values for these function come from a - * tainted source PyDateTime_DATE_GET_* macros use bit shifting. - */ -static PyObject * -build_struct_time(int y, int m, int d, int hh, int mm, int ss, int dstflag) -{ - PyObject *result; - - __coverity_tainted_data_sanitize__(y); - __coverity_tainted_data_sanitize__(m); - __coverity_tainted_data_sanitize__(d); - __coverity_tainted_data_sanitize__(hh); - __coverity_tainted_data_sanitize__(mm); - __coverity_tainted_data_sanitize__(ss); - __coverity_tainted_data_sanitize__(dstflag); - - return result; -} - -static int -ymd_to_ord(int year, int month, int day) -{ - int ord = 0; - - __coverity_tainted_data_sanitize__(year); - __coverity_tainted_data_sanitize__(month); - __coverity_tainted_data_sanitize__(day); - - return ord; -} - -static int -normalize_date(int *year, int *month, int *day) -{ - __coverity_tainted_data_sanitize__(*year); - __coverity_tainted_data_sanitize__(*month); - __coverity_tainted_data_sanitize__(*day); - - return 0; -} - -static int -weekday(int year, int month, int day) -{ - int w = 0; - - __coverity_tainted_data_sanitize__(year); - __coverity_tainted_data_sanitize__(month); - __coverity_tainted_data_sanitize__(day); - - return w; -} - diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 2acfccbb004d6b..d54b1fbe251378 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -43,89 +43,111 @@ "primaryPackagePurpose": "SOURCE", "versionInfo": "3.4.4" }, + { + "SPDXID": "SPDXRef-PACKAGE-mpdecimal", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "338fac3fb8cdd60f406b6326431338756f58a8af94229ffd9bf1e7c2b1ad71ca" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/mpdecimal-4.0.0.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:bytereef:mpdecimal:4.0.0:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "mpdecimal", + "originator": "Organization: bytereef.org", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "4.0.0" + }, { "SPDXID": "SPDXRef-PACKAGE-openssl", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "e6a77c273ebb284fedd8ea19b081fce74a9455936ffd47215f7c24713e2614b2" + "checksumValue": "1550c87996a0858474a9dd179deab2c55eb73726b9a140b32865b02fd3d8a86b" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.0.13.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.0.15.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:openssl:openssl:3.0.13:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:openssl:openssl:3.0.15:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "openssl", "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.0.13" + "versionInfo": "3.0.15" }, { "SPDXID": "SPDXRef-PACKAGE-sqlite", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "6f0364a27375435a34137b138ca4fedef8d23eec6493ca1dfff33bfc0c34fda4" + "checksumValue": "730e4a3efd6a63828bee499940fb13acc2a32c182502ce8a1d970387895d0504" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.45.1.0.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.45.3.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.45.1.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.45.3.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "sqlite", "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.45.1.0" + "versionInfo": "3.45.3.0" }, { "SPDXID": "SPDXRef-PACKAGE-tcl-core", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "1d3f2015e49e269cf681373d433cd54d88d5ef7443fe87f5f50f5fcfe9003e73" + "checksumValue": "4c23f0dd3efcbe6f3a22c503a68d147617bb30c4f5290f1eb3eaacf0b460440b" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-core-8.6.13.1.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-core-8.6.15.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.13.1:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.15.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "tcl-core", "primaryPackagePurpose": "SOURCE", - "versionInfo": "8.6.13.1" + "versionInfo": "8.6.15.0" }, { "SPDXID": "SPDXRef-PACKAGE-tk", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "6056203b8a6aaf6ea89d90a7b55dc7f407e55c093f731a98fd830a712a3c81d3" + "checksumValue": "0ae56d39bca92865f338529557a1e56d110594184b6dc5a91339c5675751e264" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-8.6.13.1.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-8.6.15.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.13.1:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.15.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "tk", "primaryPackagePurpose": "SOURCE", - "versionInfo": "8.6.13.1" + "versionInfo": "8.6.15.0" }, { "SPDXID": "SPDXRef-PACKAGE-xz", @@ -139,7 +161,7 @@ "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:xz_project:xz:5.2.5:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tukaani:xz:5.2.5:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], diff --git a/Misc/platform_triplet.c b/Misc/platform_triplet.c index 0b912e332510a6..ec0857a4a998c0 100644 --- a/Misc/platform_triplet.c +++ b/Misc/platform_triplet.c @@ -12,8 +12,20 @@ #undef powerpc #undef sparc #undef unix + #if defined(__ANDROID__) - # Android is not a multiarch system. +# if defined(__x86_64__) +PLATFORM_TRIPLET=x86_64-linux-android +# elif defined(__i386__) +PLATFORM_TRIPLET=i686-linux-android +# elif defined(__aarch64__) +PLATFORM_TRIPLET=aarch64-linux-android +# elif defined(__arm__) +PLATFORM_TRIPLET=arm-linux-androideabi +# else +# error unknown Android platform +# endif + #elif defined(__linux__) /* * BEGIN of Linux block @@ -234,8 +246,9 @@ PLATFORM_TRIPLET=i386-gnu # endif #elif defined(__APPLE__) # include "TargetConditionals.h" -# if TARGET_OS_IOS -# if TARGET_OS_SIMULATOR +// Older macOS SDKs do not define TARGET_OS_* +# if defined(TARGET_OS_IOS) && TARGET_OS_IOS +# if defined(TARGET_OS_SIMULATOR) && TARGET_OS_SIMULATOR # if __x86_64__ PLATFORM_TRIPLET=x86_64-iphonesimulator # else @@ -244,7 +257,8 @@ PLATFORM_TRIPLET=arm64-iphonesimulator # else PLATFORM_TRIPLET=arm64-iphoneos # endif -# elif TARGET_OS_OSX +// Older macOS SDKs do not define TARGET_OS_OSX +# elif !defined(TARGET_OS_OSX) || TARGET_OS_OSX PLATFORM_TRIPLET=darwin # else # error unknown Apple platform diff --git a/Misc/python-config.sh.in b/Misc/python-config.sh.in index eb02223ddcd2c3..555b0cb6ba2a48 100644 --- a/Misc/python-config.sh.in +++ b/Misc/python-config.sh.in @@ -4,11 +4,12 @@ exit_with_usage () { - local USAGE="Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" - if [[ "$1" -eq 0 ]]; then - echo "$USAGE" + local usage + usage="Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" + if [ "$1" -eq 0 ]; then + echo "$usage" else - echo "$USAGE" >&2 + echo "$usage" >&2 fi exit $1 } diff --git a/Misc/python.man b/Misc/python.man index 0f5dfa2e2289f7..4076b8d3d1ba30 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -251,6 +251,7 @@ emitted by a process (even those that are otherwise ignored by default): -Wdefault # Warn once per call location -Werror # Convert to exceptions -Walways # Warn every time + -Wall # Same as -Walways -Wmodule # Warn once per calling module -Wonce # Warn once per Python process -Wignore # Never warn @@ -607,6 +608,10 @@ output. Setting it to 0 deactivates this behavior. .IP PYTHON_HISTORY This environment variable can be used to set the location of a history file (on Unix, it is \fI~/.python_history\fP by default). +.IP PYTHON_GIL +If this variable is set to 1, the global interpreter lock (GIL) will be forced +on. Setting it to 0 forces the GIL off. Only available in builds configured +with \fB--disable-gil\fP. .SS Debug-mode variables Setting these variables only has an effect in a debug build of Python, that is, if Python was configured with the diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index af82377cf56bfa..316c266b7e4fd6 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -48,11 +48,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "90c06411f131e777e2b5c3d22b7ccf50bc46f617" + "checksumValue": "373cc00d87782a736970644d863ff2ebbd0e4886" }, { "algorithm": "SHA256", - "checksumValue": "3045f9176950aa13a54e53fa096385670c676c492705d636e977f888e4c72d48" + "checksumValue": "0f750bc336e510d14ac9a3e63fc2399f60f3f04f0061c426e86751ed5fba90e4" } ], "fileName": "Modules/expat/expat.h" @@ -62,11 +62,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "b70ce53fdc25ae482681ae2f6623c3c8edc9c1b7" + "checksumValue": "9e615c6e5c3ba00670f674a6b071bb855b0b563d" }, { "algorithm": "SHA256", - "checksumValue": "86afb425ec9999eb4f1ec9ab2fb41c58c4aa5cb9bf934b8c94264670fc5a961d" + "checksumValue": "3d90a4b65c40a3f848c36100f4d73b933a015c7b7cd85c28e4331a6b845c1ad0" } ], "fileName": "Modules/expat/expat_external.h" @@ -90,11 +90,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "9f6d9211a7b627785d5c48d10cc8eda66255113f" + "checksumValue": "e23d160cc33cc2c25a4b48f7b242f906444418e0" }, { "algorithm": "SHA256", - "checksumValue": "9f0bdd346dd94ac4359c636a4e60bc768f4ae53ce0e836eb05fb9246ee36c7f2" + "checksumValue": "f7523357d8009749e7dba94b0bd7d0fa60e011cc254e55c4ebccd6313f031122" } ], "fileName": "Modules/expat/internal.h" @@ -128,29 +128,29 @@ "fileName": "Modules/expat/nametab.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-expat-pyexpatns.h", + "SPDXID": "SPDXRef-FILE-Modules-expat-refresh.sh", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "f50c899172acd93fc539007bfb43315b83d407e4" + "checksumValue": "a9b0a33b8359cfe94b23972a1605daf8dcc605d9" }, { "algorithm": "SHA256", - "checksumValue": "d571b8258cfaa067a20adef553e5fcedd6671ca4a8841483496de031bd904567" + "checksumValue": "19eb541460bc2ca8b87118acd3c048f6af77affbf8719ac29aa7b6c8d70f83fd" } ], - "fileName": "Modules/expat/pyexpatns.h" + "fileName": "Modules/expat/refresh.sh" }, { "SPDXID": "SPDXRef-FILE-Modules-expat-siphash.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "4c49b5df2bc702f663ba3b5a52d1940ec363226b" + "checksumValue": "aca27f46d9fd387b63ce7ff2e4f172cad130b39b" }, { "algorithm": "SHA256", - "checksumValue": "b5ec29f6560acc183f1ee8ab92bb3aea17b87b4c2120cd2e3f78deba7a12491e" + "checksumValue": "f537add526ecda8389503b7ef45fb52b6217e4dc171dcc3a8dc6903ff6134726" } ], "fileName": "Modules/expat/siphash.h" @@ -188,11 +188,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "3b5de0ed1de33cad85b46230707403247f2851df" + "checksumValue": "3199fbd38b6fb158f73d5c8de6b6e6e3812ef803" }, { "algorithm": "SHA256", - "checksumValue": "a03abd531601eef61a87e06113d218ff139b6969e15a3d4668cd85d65fc6f79b" + "checksumValue": "c1518244dd5ea397e345d00e12cc45d42f43453ed208218559c981c97a0583e2" } ], "fileName": "Modules/expat/xmlparse.c" @@ -296,480 +296,536 @@ "fileName": "Modules/expat/xmltok_ns.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-MD5.c", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2b.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "f77449b2b4eb99f1da0938633cc558baf9c444fb" + "checksumValue": "1cd3cda98e0e6882a13a59268b88640c542350fd" }, { "algorithm": "SHA256", - "checksumValue": "0f252967debca5b35362ca53951ea16ca8bb97a19a1d24f6695f44d50010859e" + "checksumValue": "41a420bc9355e451720e60e9536e66f04dc6e416ca9217c4ab18d827887a2e08" } ], - "fileName": "Modules/_hacl/Hacl_Hash_MD5.c" + "fileName": "Modules/_hacl/Hacl_Hash_Blake2b.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-MD5.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2b.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "c24e6779a91c840f3d65d24abbce225b608b676e" + "checksumValue": "b0b3ae92d6aee7b52bacfdf02409d8d7e23701ee" }, { "algorithm": "SHA256", - "checksumValue": "9cd062e782801013e3cacaba583e44e1b5e682e217d20208d5323354d42011f1" + "checksumValue": "95d1dd4097a706b0719610da674297fa253b30d03a6ead4685ed648e20cb51a2" } ], - "fileName": "Modules/_hacl/Hacl_Hash_MD5.h" + "fileName": "Modules/_hacl/Hacl_Hash_Blake2b.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA1.c", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2b-Simd256.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "560f6ff541b5eff480ea047b147f4212bb0db7ed" + "checksumValue": "0ceef306590ec12251db03a31fc08ecba697486d" }, { "algorithm": "SHA256", - "checksumValue": "0ade3ab264e912d7b4e5cdcf773db8c63e4440540d295922d74b06bcfc74c77a" + "checksumValue": "1575a23b21319e55e670f74194fc2dfd1777eb5a3816cad43750e03da6e44db9" } ], - "fileName": "Modules/_hacl/Hacl_Hash_SHA1.c" + "fileName": "Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA1.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2b-Simd256.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "853b77d45379146faaeac5fe899b28db386ad13c" + "checksumValue": "a5011646670c4f51368aca661e458e4c7f1d88e0" }, { "algorithm": "SHA256", - "checksumValue": "b13eb14f91582703819235ea7c8f807bb93e4f1e6b695499dc1d86021dc39e72" + "checksumValue": "f00c1fe8e774c7ec65f6c5a8efa43ce180a17fc80ed6119ada8c4022d058b6e2" } ], - "fileName": "Modules/_hacl/Hacl_Hash_SHA1.h" + "fileName": "Modules/_hacl/Hacl_Hash_Blake2b_Simd256.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA2.c", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2b-Simd256-universal2.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "667120b6100c946cdaa442f1173c723339923071" + "checksumValue": "5afc433179d71abd6649596797a7e8953e89172d" }, { "algorithm": "SHA256", - "checksumValue": "b189459b863341a3a9c5c78c0208b6554a2f2ac26e0748fbd4432a91db21fae6" + "checksumValue": "db42da82d18641d68d3670e6201e0cbb43415daaa84f29770b8f0ebf33562975" } ], - "fileName": "Modules/_hacl/Hacl_Hash_SHA2.c" + "fileName": "Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA2.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2s.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "81db38b0b920e63ec33c7109d1144c35cf091da0" + "checksumValue": "9616a9f8d795d64487bf86a96719f943729621e2" }, { "algorithm": "SHA256", - "checksumValue": "631c9ba19c1c2c835bb63d3f2f22b8d76fb535edfed3c254ff2a52f12af3fe61" + "checksumValue": "5ecde5ddc8ec073cffe64d60e868535d995f33fb0f87f9b50e68bd2a694b7434" } ], - "fileName": "Modules/_hacl/Hacl_Hash_SHA2.h" + "fileName": "Modules/_hacl/Hacl_Hash_Blake2s.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA3.c", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2s.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "9c832b98a2f2a68202d2da016fb718965d7b7602" + "checksumValue": "0328172a62507a051cd60ff9603710ed5aea1bc8" }, { "algorithm": "SHA256", - "checksumValue": "38d350d1184238966cfa821a59ae00343f362182b6c2fbea7f2651763d757fb7" + "checksumValue": "9f3c8ef615c9fbc59ef796d0ad2a7a76a7e55dc8939077b44ca538cbf8889a8c" } ], - "fileName": "Modules/_hacl/Hacl_Hash_SHA3.c" + "fileName": "Modules/_hacl/Hacl_Hash_Blake2s.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA3.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2s-Simd128.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "ecc766fb6f7ee85e902b593b61b41e5a728fca34" + "checksumValue": "5b950ce0a5c8f0c2c56b4ac96e1943b504255d45" }, { "algorithm": "SHA256", - "checksumValue": "bae290a94366a2460f51e8468144baaade91d9048db111e10d2e2ffddc3f98cf" + "checksumValue": "5a5f5d8e376dc30d89fd6c6c435157fe9ffa5308030e7abb1256afaee0765536" } ], - "fileName": "Modules/_hacl/Hacl_Hash_SHA3.h" + "fileName": "Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Streaming-Types.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2s-Simd128.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "ab7b4d9465a2765a07f8d5bccace7182b28ed1b8" + "checksumValue": "32f35c173c10a2c49ac53c839cfbccd8a147274d" }, { "algorithm": "SHA256", - "checksumValue": "26913613f3b4f8ffff0a3e211a5ebc849159094e5e11de0a31fcb95b6105b74c" + "checksumValue": "8734879b551f0fa860002ae81c0d0cfbade561007d9c26ad18c5a221e239237e" } ], - "fileName": "Modules/_hacl/Hacl_Streaming_Types.h" + "fileName": "Modules/_hacl/Hacl_Hash_Blake2s_Simd128.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-FStar-UInt128-Verified.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2s-Simd128-universal2.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "2ea61d6a236147462045f65c20311819d74db80c" + "checksumValue": "d70c6dbcb91d56bbd80f7bf860e508a748042d0d" }, { "algorithm": "SHA256", - "checksumValue": "2c22b4d49ba06d6a3053cdc66405bd5ae953a28fcfed1ab164e8f5e0f6e2fb8b" + "checksumValue": "5b132ab850a5e0fe6f27e08a955f8989ea3aae8e5b3115f0195039034ece8c04" } ], - "fileName": "Modules/_hacl/include/krml/FStar_UInt128_Verified.h" + "fileName": "Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-FStar-UInt-8-16-32-64.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-MD5.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "1a647d841180ac8ca667afa968c353425e81ad0d" + "checksumValue": "f8ba39b46ebdfa7d031d9c33130c6ded680a8120" }, { "algorithm": "SHA256", - "checksumValue": "e5d1c5854833bec7ea02e227ec35bd7b49c5fb9e0f339efa0dd83e1595f722d4" + "checksumValue": "f71cf6a0e8f09354c2af2c785a1d36e0cba7613a589be01ca8a3d8478f4c8874" } ], - "fileName": "Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h" + "fileName": "Modules/_hacl/Hacl_Hash_MD5.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-fstar-uint128-struct-endianness.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-MD5.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "1987119a563a8fdc5966286e274f716dbcea77ee" + "checksumValue": "eaaab54cea2b0bb8ec0eedf0b373d42f1a0f8f6c" }, { "algorithm": "SHA256", - "checksumValue": "fe57e1bc5ce3224d106e36cb8829b5399c63a68a70b0ccd0c91d82a4565c8869" + "checksumValue": "9a02e2a6e163515ea0228a859d5e55c1f57b11fae5908c42f9f9814ce9bca230" } ], - "fileName": "Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h" + "fileName": "Modules/_hacl/Hacl_Hash_MD5.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-internal-target.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA1.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "903c9eb76b01f3a95c04c3bc841c2fb71dea5403" + "checksumValue": "f4f42faf8da78a230199f649c0f2a1b865799a31" }, { "algorithm": "SHA256", - "checksumValue": "08ec602c7f90a1540389c0cfc95769fa7fec251e7ca143ef83c0b9f7afcf89a7" + "checksumValue": "5b29bd9951646861e0e19427be5d923a5bab7a4516824ccc068f696469195eec" } ], - "fileName": "Modules/_hacl/include/krml/internal/target.h" + "fileName": "Modules/_hacl/Hacl_Hash_SHA1.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-lowstar-endianness.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA1.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "964e09bd99ff2366afd6193b59863fc925e7fb05" + "checksumValue": "722b57139737ceeb88e41d3839e6f7d70578741b" }, { "algorithm": "SHA256", - "checksumValue": "3734c7942bec9a434e16df069fa45bdcb84b130f14417bc5f7bfe8546272d9f5" + "checksumValue": "5640295c790d56b1b4df147d6a6c58803b1845cd7d93365bf7cc7b75ba3cacd5" } ], - "fileName": "Modules/_hacl/include/krml/lowstar_endianness.h" + "fileName": "Modules/_hacl/Hacl_Hash_SHA1.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-types.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA2.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "df8e0ed74a5970d09d3cc4c6e7c6c7a4c4e5015c" + "checksumValue": "b0aa8810339adb09623ffa429246b4324fac4565" }, { "algorithm": "SHA256", - "checksumValue": "de7444c345caa4c47902c4380500356a3ee7e199d2aab84fd8c4960410154f3d" + "checksumValue": "2288f8f860efe80eed4f1e14ef570079b7459aeb41f87e94e691d7cf5e0e7adb" } ], - "fileName": "Modules/_hacl/include/krml/types.h" + "fileName": "Modules/_hacl/Hacl_Hash_SHA2.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-MD5.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA2.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "5dd4ee3c835a0d176a6e9fecbe9752fd1474ff41" + "checksumValue": "4903e10291d07367be3bc283935bc52926e57ba1" }, { "algorithm": "SHA256", - "checksumValue": "d82ef594cba44203576d67b047240316bb3c542912ebb7034afa1e07888cec56" + "checksumValue": "093d7693084af0999d2a13d207311d74b5bdfdc9c08447ed4a979e3f7505ae6b" } ], - "fileName": "Modules/_hacl/internal/Hacl_Hash_MD5.h" + "fileName": "Modules/_hacl/Hacl_Hash_SHA2.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA1.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA3.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "515b3082eb7c30597773e1c63ec46688f6da3634" + "checksumValue": "ef374b9d0951ebb38006af944dd4b38a6cf3abb2" }, { "algorithm": "SHA256", - "checksumValue": "10aacf847006b8e0dfb64d5c327443f954db6718b4aec712fb3268230df6a752" + "checksumValue": "164df19f229143006c5f9a3c0bd771415f152bfbc7efb61c337fa0f903003eb3" } ], - "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA1.h" + "fileName": "Modules/_hacl/Hacl_Hash_SHA3.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA2.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA3.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "a044ec12b70ba97b67e9a312827d6270452a20ca" + "checksumValue": "7d78e6844dde1f9b5e68f58ca105a4c330461ff6" }, { "algorithm": "SHA256", - "checksumValue": "a1426b54fa7273ba5b50817c25b2b26fc85c4d1befb14092cd27dc4c99439463" + "checksumValue": "231d9bc13190be4b6821acb518194f32f4a3c04f1c034b3118f6db0bab2debe3" } ], - "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA2.h" + "fileName": "Modules/_hacl/Hacl_Hash_SHA3.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA3.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Hacl-Streaming-Types.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "cfb7b520c39a73cb84c541d370455f92b998781f" + "checksumValue": "ab7b4d9465a2765a07f8d5bccace7182b28ed1b8" }, { "algorithm": "SHA256", - "checksumValue": "fd41997f9e96b3c9a3337b1b51fab965a1e21b0c16f353d156f1a1fa00709fbf" + "checksumValue": "26913613f3b4f8ffff0a3e211a5ebc849159094e5e11de0a31fcb95b6105b74c" } ], - "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA3.h" + "fileName": "Modules/_hacl/Hacl_Streaming_Types.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-hacl-python-hacl-namespaces.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-Lib-Memzero0.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "f5c7b3ed911af6c8d582e8b3714b0c36195dc994" + "checksumValue": "118dc712780ea680affa8d9794470440eb87ff10" }, { "algorithm": "SHA256", - "checksumValue": "07de72398b12957e014e97b9ac197bceef12d6d6505c2bfe8b23ee17b94ec5fa" + "checksumValue": "b017e7d5662a308c938cf4e4b919680c8f3e27f42975ca152b62fe65c5f7fb0c" } ], - "fileName": "Modules/_hacl/python_hacl_namespaces.h" + "fileName": "Modules/_hacl/Lib_Memzero0.c" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2-config.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-FStar-UInt128-Verified.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "ff5e3ae2360adf7279a9c54d12a1d32e16a1f223" + "checksumValue": "7665829b9396f72e7f8098080d6d6773565468e9" }, { "algorithm": "SHA256", - "checksumValue": "1eb919e885244e43cdf7b2104ad30dc9271513478c0026f6bfb4bad6e2f0ab42" + "checksumValue": "ca7357ee70365c690664a44f6522e526636151d9ed2da8d0d29da15bb8556530" } ], - "fileName": "Modules/_blake2/impl/blake2-config.h" + "fileName": "Modules/_hacl/include/krml/FStar_UInt128_Verified.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2-impl.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-FStar-UInt-8-16-32-64.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "28b947b43bdc680b9f4335712bb2a5f2d5d32623" + "checksumValue": "a2db924d0e8f7df3139e9a20355ffa520aded479" }, { "algorithm": "SHA256", - "checksumValue": "4277092643b289f1d36d32cf0fd2efc30ead8bdd99342e5da3b3609dd8ea7d86" + "checksumValue": "f1de79fb4c763b215c823f44471bbae6b65e6bb533eb52a5863d551d5e2e6748" } ], - "fileName": "Modules/_blake2/impl/blake2-impl.h" + "fileName": "Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-fstar-uint128-struct-endianness.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "caa3da7953109d0d2961e3b686d2d285c484b901" + "checksumValue": "7f23693151d5409623cbe886e5b45a0e4f0d3c72" }, { "algorithm": "SHA256", - "checksumValue": "2f6c9d0ecf70be474f2853b52394993625a32960e0a64eae147ef97a3a5c1460" + "checksumValue": "1c9bee7ac4b987c73cc3aba6b7ceed8ec7e75c9a741810e4411f35602490e0d8" } ], - "fileName": "Modules/_blake2/impl/blake2.h" + "fileName": "Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2b-load-sse2.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-internal-target.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "029a98f87a178936d9e5211c7798b3e0fc622f94" + "checksumValue": "9c5cac1582dcd6e0d0a4142e6e8b285b4cb7d9e6" }, { "algorithm": "SHA256", - "checksumValue": "b392a6e7b43813a05609e994db5fc3552c5912bd482efc781daa0778eb56ab4e" + "checksumValue": "b1e32138ac8c262e872f7da43ec80c1e54c08bcbdec4b7be17117aa25807f87e" } ], - "fileName": "Modules/_blake2/impl/blake2b-load-sse2.h" + "fileName": "Modules/_hacl/include/krml/internal/target.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2b-load-sse41.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-lowstar-endianness.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "fb466dd72344170d09e311e5ea12de99ce071357" + "checksumValue": "e18efc9239a5df0f222b5f7b0a65f72509d7e304" }, { "algorithm": "SHA256", - "checksumValue": "cc3072c92164142bf2f9dda4e6c08db61be68ec15a95442415e861090d08f6a2" + "checksumValue": "47dd5a7d21b5302255f9fff28884f65d3056fc3f54471ed62ec85fa1904f8aa5" } ], - "fileName": "Modules/_blake2/impl/blake2b-load-sse41.h" + "fileName": "Modules/_hacl/include/krml/lowstar_endianness.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2b-ref.c", + "SPDXID": "SPDXRef-FILE-Modules-hacl-include-krml-types.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "4c0d79128cf891a95b1f668031d55c0c6d2e0270" + "checksumValue": "df8e0ed74a5970d09d3cc4c6e7c6c7a4c4e5015c" }, { "algorithm": "SHA256", - "checksumValue": "07b257d44e9cc2d95d4911629c92138feafd16d63fef0a5fa7b38914dfd82349" + "checksumValue": "de7444c345caa4c47902c4380500356a3ee7e199d2aab84fd8c4960410154f3d" } ], - "fileName": "Modules/_blake2/impl/blake2b-ref.c" + "fileName": "Modules/_hacl/include/krml/types.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2b-round.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-Blake2b.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "4c7418e2026417c9c6736fcd305a31f23e05a661" + "checksumValue": "31b329bd39ff72ed25086e2afe7875949003c140" }, { "algorithm": "SHA256", - "checksumValue": "fa34a60c2d198a0585033f43fd4003f4ba279c9ebcabdf5d6650def0e6d1e914" + "checksumValue": "16df6cf240ee99aade0fd11d5cc7573c201c7589d8325a5c95c7670c531e1518" } ], - "fileName": "Modules/_blake2/impl/blake2b-round.h" + "fileName": "Modules/_hacl/internal/Hacl_Hash_Blake2b.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2b.c", + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-Blake2b-Simd256.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "f935d64cc633c38e09fc2d89281c95edfbc1fb05" + "checksumValue": "3f4fdfdaef97a2cbac5ec091c91ede18d4b33f92" }, { "algorithm": "SHA256", - "checksumValue": "b932aa273b2504606a48895a50ff08c883f7a68a7e4aced5daa909c43348605a" + "checksumValue": "96b1c77860f12bcadad0caca77a5a1649a840ad9989d97984a3b51bb98c80e2f" } ], - "fileName": "Modules/_blake2/impl/blake2b.c" + "fileName": "Modules/_hacl/internal/Hacl_Hash_Blake2b_Simd256.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-sse2.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-Blake2s.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "ad3f79b6cbe3fd812722114a0d5d08064e69e4d0" + "checksumValue": "9efd61f6ba8d126e98abd83679a5ed5954278c31" }, { "algorithm": "SHA256", - "checksumValue": "57f1ac6c09f4a50d95811529062220eab4f29cec3805bc6081dec00426c6df62" + "checksumValue": "143f58f033786173501a72ac302e435963fdce6c2cc38eef6d6adeb3cdc1bb9c" } ], - "fileName": "Modules/_blake2/impl/blake2s-load-sse2.h" + "fileName": "Modules/_hacl/internal/Hacl_Hash_Blake2s.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-sse41.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-Blake2s-Simd128.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "51c32d79f419f3d2eb9875cd9a7f5c0d7892f8a8" + "checksumValue": "3f984829465285283b03b1111b4918cfb48b8031" }, { "algorithm": "SHA256", - "checksumValue": "ecc9e09adcbe098629eafd305596bed8d7004be1d83f326995def42bbde93b23" + "checksumValue": "cd24038fdd617edc65e472496b0d58f23ff312f81f9244c3e7893fdc9a1b2977" } ], - "fileName": "Modules/_blake2/impl/blake2s-load-sse41.h" + "fileName": "Modules/_hacl/internal/Hacl_Hash_Blake2s_Simd128.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-xop.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-MD5.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "2749a7ba0104b765d4f56f13faf70b6eb89cf203" + "checksumValue": "60f02d21f045c8a4c2b6b84a8f7e023d9490c8e5" }, { "algorithm": "SHA256", - "checksumValue": "8bc95595cec4c50f5d70f2b330d3798de07cc784e8890791b3328890e602d5c5" + "checksumValue": "370d8ef9c48cb55472ece11e12eaf94c58118de3f5515b6df1c130b696597828" } ], - "fileName": "Modules/_blake2/impl/blake2s-load-xop.h" + "fileName": "Modules/_hacl/internal/Hacl_Hash_MD5.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s-ref.c", + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA1.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "883fcfe85f9063819f21b1100296d1f9eb55bac1" + "checksumValue": "6346c30a140e7d3010c98fe19d14fa229a54eb16" }, { "algorithm": "SHA256", - "checksumValue": "9715c00d0f11587a139b07fa26678e6d26e44d3d4910b96158d158da2b022bfb" + "checksumValue": "ab52c6092bdbbfc9884f841bf4824016792ffa96167577cbe0df00dd96f56a34" } ], - "fileName": "Modules/_blake2/impl/blake2s-ref.c" + "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA1.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA2.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "2e9ae174142fc491f20567ab8b5c08cef9b07cfe" + }, + { + "algorithm": "SHA256", + "checksumValue": "07100964adcf4b5f8bd4773e25f475b34cd180b90df8b1c0052e55c008b7cc49" + } + ], + "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA2.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s-round.h", + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA3.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "5d9f69adda40ed163b287b9ed4cedb35b88f2daa" + "checksumValue": "39ba6e8959e44ae956a640d3a1fb3ef60de8a9e5" }, { "algorithm": "SHA256", - "checksumValue": "65d90111c89c43bb18a9e1d1a4fdbd9f85bebd1ff00129335b85995d0f30ee8b" + "checksumValue": "dbf4b86a04b4d8716976f8c023cccbfe174435dbec3bc00fc1f066fb52c4e341" } ], - "fileName": "Modules/_blake2/impl/blake2s-round.h" + "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA3.h" }, { - "SPDXID": "SPDXRef-FILE-Modules-blake2-impl-blake2s.c", + "SPDXID": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Impl-Blake2-Constants.h", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "13ac5bb93578a7ee8f815b4e247e82c849992bbe" + "checksumValue": "c3ae35ed5bf70cf011b2732df011231528b9111c" }, { "algorithm": "SHA256", - "checksumValue": "25ec5dd5c79f916307358059fe9f633781f27df1c0e0962c4fcccdda1feb93a7" + "checksumValue": "c381fea7b8b505a7c7ce27231a36751add6b184b204132935c5faaba4fce8ba1" } ], - "fileName": "Modules/_blake2/impl/blake2s.c" + "fileName": "Modules/_hacl/internal/Hacl_Impl_Blake2_Constants.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-lib-memzero0.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "3d65f95f6f4bbfe980a89b82c55d02d7694a5a79" + }, + { + "algorithm": "SHA256", + "checksumValue": "0f8d744620cf5f6b8450da187484b418d24dec7d8cf72b757b7080e84cb3ae5e" + } + ], + "fileName": "Modules/_hacl/lib_memzero0.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-libintvector.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "f4a33ad535768b860362ab0bd033a70da0b524b7" + }, + { + "algorithm": "SHA256", + "checksumValue": "433cdf4ba80bc72e0cea5d4b420ff18676baeafdb5ba19adf5b7fb33e90b424b" + } + ], + "fileName": "Modules/_hacl/libintvector.h" + }, + { + "SPDXID": "SPDXRef-FILE-Modules-hacl-python-hacl-namespaces.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "37e3eb63c5c6f8ae671748bfde642c180b96d2de" + }, + { + "algorithm": "SHA256", + "checksumValue": "0b5c7892cc25a2b3467936c1f346a6186d9d0a257d1bd5671beda253b66e0f68" + } + ], + "fileName": "Modules/_hacl/python_hacl_namespaces.h" }, { "SPDXID": "SPDXRef-FILE-Lib-ctypes-macholib-init-.py", @@ -1562,14 +1618,14 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "a13447b9aa67d7c860783fdf6820f33ebdea996900d6d8bbc50a628f55f099f7" + "checksumValue": "17aa6cfc5c4c219c09287abfc10bc13f0c06f30bb654b28bfe6f567ca646eb79" } ], - "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_6_0/expat-2.6.0.tar.gz", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_6_3/expat-2.6.3.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.6.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.6.3:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1577,21 +1633,21 @@ "name": "expat", "originator": "Organization: Expat development team", "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.6.0" + "versionInfo": "2.6.3" }, { "SPDXID": "SPDXRef-PACKAGE-hacl-star", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "c23ac158b238c368389dc86bfc315263e5c0e57785da74144aea2cab9a3d51a2" + "checksumValue": "40de5297b032d2676fc0039049b4e8dab1f2730eebb5ecff6a40c04fa0356339" } ], - "downloadLocation": "https://github.com/hacl-star/hacl-star/archive/521af282fdf6d60227335120f18ae9309a4b8e8c.zip", + "downloadLocation": "https://github.com/hacl-star/hacl-star/archive/f218923ef2417d963d7efc7951593ae6aef613f7.zip", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:hacl-star:hacl-star:521af282fdf6d60227335120f18ae9309a4b8e8c:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:hacl-star:hacl-star:f218923ef2417d963d7efc7951593ae6aef613f7:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1599,29 +1655,7 @@ "name": "hacl-star", "originator": "Organization: HACL* Developers", "primaryPackagePurpose": "SOURCE", - "versionInfo": "521af282fdf6d60227335120f18ae9309a4b8e8c" - }, - { - "SPDXID": "SPDXRef-PACKAGE-libb2", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "53626fddce753c454a3fea581cbbc7fe9bbcf0bc70416d48fdbbf5d87ef6c72e" - } - ], - "downloadLocation": "https://github.com/BLAKE2/libb2/releases/download/v0.98.1/libb2-0.98.1.tar.gz", - "externalRefs": [ - { - "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:blake2:libb2:0.98.1:*:*:*:*:*:*:*", - "referenceType": "cpe23Type" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "libb2", - "originator": "Organization: BLAKE2 - fast secure hashing", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "0.98.1" + "versionInfo": "f218923ef2417d963d7efc7951593ae6aef613f7" }, { "SPDXID": "SPDXRef-PACKAGE-macholib", @@ -1715,7 +1749,7 @@ "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-pyexpatns.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-refresh.sh", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-expat" }, @@ -1775,174 +1809,194 @@ "spdxElementId": "SPDXRef-PACKAGE-expat" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-MD5.c", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2b.c", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-MD5.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2b.h", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA1.c", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2b-Simd256.c", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA1.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2b-Simd256.h", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA2.c", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2b-Simd256-universal2.c", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA2.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2s.c", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA3.c", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2s.h", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA3.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2s-Simd128.c", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Streaming-Types.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2s-Simd128.h", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-FStar-UInt128-Verified.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-Blake2s-Simd128-universal2.c", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-FStar-UInt-8-16-32-64.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-MD5.c", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-fstar-uint128-struct-endianness.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-MD5.h", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-internal-target.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA1.c", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-lowstar-endianness.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA1.h", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-types.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA2.c", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-MD5.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA2.h", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA1.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA3.c", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA2.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Hash-SHA3.h", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA3.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Hacl-Streaming-Types.h", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-python-hacl-namespaces.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-Lib-Memzero0.c", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2-config.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-FStar-UInt128-Verified.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2-impl.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-FStar-UInt-8-16-32-64.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-fstar-uint128-struct-endianness.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2b-load-sse2.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-internal-target.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2b-load-sse41.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-lowstar-endianness.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2b-ref.c", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-include-krml-types.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2b-round.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-Blake2b.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2b.c", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-Blake2b-Simd256.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-sse2.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-Blake2s.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-sse41.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-Blake2s-Simd128.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-MD5.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA1.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA2.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s-load-xop.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Hash-SHA3.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s-ref.c", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-internal-Hacl-Impl-Blake2-Constants.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s-round.h", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-lib-memzero0.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { - "relatedSpdxElement": "SPDXRef-FILE-Modules-blake2-impl-blake2s.c", + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-libintvector.h", "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-libb2" + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" + }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-hacl-python-hacl-namespaces.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-hacl-star" }, { "relatedSpdxElement": "SPDXRef-FILE-Lib-ctypes-macholib-init-.py", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index ca7cf02961571e..276526a1b6908e 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -40,7 +40,7 @@ # - struct_abi_kind: for `struct`, defines how much of the struct is exposed: # - 'full-abi': All of the struct is part of the ABI, including the size # (users may define arrays of these structs). -# Typically used for initalization, rather than at runtime. +# Typically used for initialization, rather than at runtime. # - 'opaque': No members are part of the ABI, nor is the size. The Limited # API only handles these via pointers. The C definition should be # incomplete (opaque). @@ -702,7 +702,6 @@ added = '3.2' [function.PyEval_InitThreads] added = '3.2' - abi_only = true [function.PyEval_ReleaseLock] added = '3.2' abi_only = true @@ -1337,10 +1336,8 @@ added = '3.2' [function.PySys_SetArgv] added = '3.2' - abi_only = true [function.PySys_SetArgvEx] added = '3.2' - abi_only = true [function.PySys_SetObject] added = '3.2' [function.PySys_SetPath] @@ -1673,10 +1670,8 @@ added = '3.2' [function.Py_SetProgramName] added = '3.2' - abi_only = true [function.Py_SetPythonHome] added = '3.2' - abi_only = true [function.Py_SetRecursionLimit] added = '3.2' [function.Py_VaBuildValue] @@ -2485,8 +2480,6 @@ [function._Py_SetRefcnt] added = '3.13' abi_only = true -[data.PyExc_IncompleteInputError] - added = '3.13' [function.PyList_GetItemRef] added = '3.13' [typedef.PyCFunctionFast] @@ -2496,3 +2489,58 @@ [typedef.PyCFunctionFastWithKeywords] added = '3.13' # "abi-only" since 3.10. (Same story as PyCFunctionFast.) +[function.PyType_GetFullyQualifiedName] + added = '3.13' +[function.PyType_GetModuleName] + added = '3.13' +[function.Py_GetConstant] + added = '3.13' +[function.Py_GetConstantBorrowed] + added = '3.13' +[function.PyType_GetModuleByDef] + added = '3.13' +[function.PyEval_GetFrameBuiltins] + added = '3.13' +[function.PyEval_GetFrameGlobals] + added = '3.13' +[function.PyEval_GetFrameLocals] + added = '3.13' + +[function.Py_TYPE] + added = '3.14' +[function.Py_REFCNT] + added = '3.14' +[function.PyIter_NextItem] + added = '3.14' +[function.PyLong_FromInt32] + added = '3.14' +[function.PyLong_FromUInt32] + added = '3.14' +[function.PyLong_AsInt32] + added = '3.14' +[function.PyLong_AsUInt32] + added = '3.14' +[function.PyLong_FromInt64] + added = '3.14' +[function.PyLong_FromUInt64] + added = '3.14' +[function.PyLong_AsInt64] + added = '3.14' +[function.PyLong_AsUInt64] + added = '3.14' +[const.Py_tp_vectorcall] + added = '3.14' +[function.PyType_GetBaseByToken] + added = '3.14' +[const.Py_tp_token] + added = '3.14' +[const.Py_TP_USE_SPEC] + added = '3.14' +[function.PyUnicode_Equal] + added = '3.14' +[function.PyType_Freeze] + added = '3.14' +[function.Py_PACK_FULL_VERSION] + added = '3.14' +[function.Py_PACK_VERSION] + added = '3.14' diff --git a/Misc/valgrind-python.supp b/Misc/valgrind-python.supp index 29954c1bbad350..8b2027cd452767 100644 --- a/Misc/valgrind-python.supp +++ b/Misc/valgrind-python.supp @@ -95,6 +95,49 @@ fun:COMMENT_THIS_LINE_TO_DISABLE_LEAK_WARNING } +# +# Leaks: dlopen() called without dlclose() +# + +{ + dlopen() called without dlclose() + Memcheck:Leak + fun:malloc + fun:malloc + fun:strdup + fun:_dl_load_cache_lookup +} +{ + dlopen() called without dlclose() + Memcheck:Leak + fun:malloc + fun:malloc + fun:strdup + fun:_dl_map_object +} +{ + dlopen() called without dlclose() + Memcheck:Leak + fun:malloc + fun:* + fun:_dl_new_object +} +{ + dlopen() called without dlclose() + Memcheck:Leak + fun:calloc + fun:* + fun:_dl_new_object +} +{ + dlopen() called without dlclose() + Memcheck:Leak + fun:calloc + fun:* + fun:_dl_check_map_versions +} + + # # Non-python specific leaks # diff --git a/Modules/Setup b/Modules/Setup index cd1cf24c25d406..ddf39e0b966610 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -137,6 +137,9 @@ PYTHONPATH=$(COREPYTHONPATH) #_datetime _datetimemodule.c #_decimal _decimal/_decimal.c #_heapq _heapqmodule.c +#_interpchannels _interpchannelsmodule.c +#_interpqueues _interpqueuesmodule.c +#_interpreters _interpretersmodule.c #_json _json.c #_lsprof _lsprof.c rotatingtree.c #_multiprocessing -I$(srcdir)/Modules/_multiprocessing _multiprocessing/multiprocessing.c _multiprocessing/semaphore.c @@ -162,7 +165,7 @@ PYTHONPATH=$(COREPYTHONPATH) #pyexpat pyexpat.c # hashing builtins -#_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c +#_blake2 blake2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Hash_Blake2.a #_md5 md5module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_MD5.c -D_BSD_SOURCE -D_DEFAULT_SOURCE #_sha1 sha1module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA1.c -D_BSD_SOURCE -D_DEFAULT_SOURCE #_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Hash_SHA2.a @@ -271,9 +274,6 @@ PYTHONPATH=$(COREPYTHONPATH) # Testing -#_xxsubinterpreters _xxsubinterpretersmodule.c -#_xxinterpchannels _xxinterpchannelsmodule.c -#_xxinterpqueues _xxinterpqueuesmodule.c #_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c #_testbuffer _testbuffer.c #_testinternalcapi _testinternalcapi.c diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index aa4e60e272653b..4dcc0f55176d0e 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -30,6 +30,7 @@ _weakref _weakref.c _abc _abc.c _functools _functoolsmodule.c _locale _localemodule.c +_opcode _opcode.c _operator _operator.c _stat _stat.c _symtable symtablemodule.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 73b082691a3fd4..6b6a8ae57a5119 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -36,16 +36,16 @@ @MODULE__HEAPQ_TRUE@_heapq _heapqmodule.c @MODULE__JSON_TRUE@_json _json.c @MODULE__LSPROF_TRUE@_lsprof _lsprof.c rotatingtree.c -@MODULE__OPCODE_TRUE@_opcode _opcode.c @MODULE__PICKLE_TRUE@_pickle _pickle.c @MODULE__QUEUE_TRUE@_queue _queuemodule.c @MODULE__RANDOM_TRUE@_random _randommodule.c @MODULE__STRUCT_TRUE@_struct _struct.c # build supports subinterpreters -@MODULE__XXSUBINTERPRETERS_TRUE@_xxsubinterpreters _xxsubinterpretersmodule.c -@MODULE__XXINTERPCHANNELS_TRUE@_xxinterpchannels _xxinterpchannelsmodule.c -@MODULE__XXINTERPQUEUES_TRUE@_xxinterpqueues _xxinterpqueuesmodule.c +@MODULE__INTERPRETERS_TRUE@_interpreters _interpretersmodule.c +@MODULE__INTERPCHANNELS_TRUE@_interpchannels _interpchannelsmodule.c +@MODULE__INTERPQUEUES_TRUE@_interpqueues _interpqueuesmodule.c + @MODULE__ZONEINFO_TRUE@_zoneinfo _zoneinfo.c # needs libm @@ -82,7 +82,7 @@ @MODULE__SHA1_TRUE@_sha1 sha1module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA1.c -D_BSD_SOURCE -D_DEFAULT_SOURCE @MODULE__SHA2_TRUE@_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Hash_SHA2.a @MODULE__SHA3_TRUE@_sha3 sha3module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA3.c -D_BSD_SOURCE -D_DEFAULT_SOURCE -@MODULE__BLAKE2_TRUE@_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c +@MODULE__BLAKE2_TRUE@_blake2 blake2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Hash_Blake2.a ############################################################################ # XML and text @@ -162,7 +162,8 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/bytearray.c _testcapi/bytes.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/pyos.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c _testcapi/sys.c _testcapi/hash.c _testcapi/time.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/monitoring.c _testcapi/config.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c @@ -170,7 +171,7 @@ *shared* @MODULE__TESTIMPORTMULTIPLE_TRUE@_testimportmultiple _testimportmultiple.c @MODULE__TESTMULTIPHASE_TRUE@_testmultiphase _testmultiphase.c -@MODULE__TESTMULTIPHASE_TRUE@_testsinglephase _testsinglephase.c +@MODULE__TESTSINGLEPHASE_TRUE@_testsinglephase _testsinglephase.c @MODULE__TESTEXTERNALINSPECTION_TRUE@_testexternalinspection _testexternalinspection.c @MODULE__CTYPES_TEST_TRUE@_ctypes_test _ctypes/_ctypes_test.c diff --git a/Modules/_abc.c b/Modules/_abc.c index 399ecbbd6a2172..d6a953b336025d 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -21,7 +21,7 @@ PyDoc_STRVAR(_abc__doc__, typedef struct { PyTypeObject *_abc_data_type; - unsigned long long abc_invalidation_counter; + uint64_t abc_invalidation_counter; } _abcmodule_state; static inline _abcmodule_state* @@ -32,20 +32,67 @@ get_abc_state(PyObject *module) return (_abcmodule_state *)state; } +static inline uint64_t +get_invalidation_counter(_abcmodule_state *state) +{ +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_uint64(&state->abc_invalidation_counter); +#else + return state->abc_invalidation_counter; +#endif +} + +static inline void +increment_invalidation_counter(_abcmodule_state *state) +{ +#ifdef Py_GIL_DISABLED + _Py_atomic_add_uint64(&state->abc_invalidation_counter, 1); +#else + state->abc_invalidation_counter++; +#endif +} + /* This object stores internal state for ABCs. Note that we can use normal sets for caches, since they are never iterated over. */ typedef struct { PyObject_HEAD + /* These sets of weak references are lazily created. Once created, they + will point to the same sets until the ABCMeta object is destroyed or + cleared, both of which will only happen while the object is visible to a + single thread. */ PyObject *_abc_registry; - PyObject *_abc_cache; /* Normal set of weak references. */ - PyObject *_abc_negative_cache; /* Normal set of weak references. */ - unsigned long long _abc_negative_cache_version; + PyObject *_abc_cache; + PyObject *_abc_negative_cache; + uint64_t _abc_negative_cache_version; } _abc_data; +#define _abc_data_CAST(op) ((_abc_data *)(op)) + +static inline uint64_t +get_cache_version(_abc_data *impl) +{ +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_uint64(&impl->_abc_negative_cache_version); +#else + return impl->_abc_negative_cache_version; +#endif +} + +static inline void +set_cache_version(_abc_data *impl, uint64_t version) +{ +#ifdef Py_GIL_DISABLED + _Py_atomic_store_uint64(&impl->_abc_negative_cache_version, version); +#else + impl->_abc_negative_cache_version = version; +#endif +} + static int -abc_data_traverse(_abc_data *self, visitproc visit, void *arg) +abc_data_traverse(PyObject *op, visitproc visit, void *arg) { + _abc_data *self = _abc_data_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->_abc_registry); Py_VISIT(self->_abc_cache); @@ -54,8 +101,9 @@ abc_data_traverse(_abc_data *self, visitproc visit, void *arg) } static int -abc_data_clear(_abc_data *self) +abc_data_clear(PyObject *op) { + _abc_data *self = _abc_data_CAST(op); Py_CLEAR(self->_abc_registry); Py_CLEAR(self->_abc_cache); Py_CLEAR(self->_abc_negative_cache); @@ -63,7 +111,7 @@ abc_data_clear(_abc_data *self) } static void -abc_data_dealloc(_abc_data *self) +abc_data_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); PyTypeObject *tp = Py_TYPE(self); @@ -90,7 +138,7 @@ abc_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->_abc_registry = NULL; self->_abc_cache = NULL; self->_abc_negative_cache = NULL; - self->_abc_negative_cache_version = state->abc_invalidation_counter; + self->_abc_negative_cache_version = get_invalidation_counter(state); return (PyObject *) self; } @@ -130,8 +178,12 @@ _get_impl(PyObject *module, PyObject *self) } static int -_in_weak_set(PyObject *set, PyObject *obj) +_in_weak_set(_abc_data *impl, PyObject **pset, PyObject *obj) { + PyObject *set; + Py_BEGIN_CRITICAL_SECTION(impl); + set = *pset; + Py_END_CRITICAL_SECTION(); if (set == NULL || PySet_GET_SIZE(set) == 0) { return 0; } @@ -164,20 +216,23 @@ _destroy(PyObject *setweakref, PyObject *objweakref) } static PyMethodDef _destroy_def = { - "_destroy", (PyCFunction) _destroy, METH_O + "_destroy", _destroy, METH_O }; static int -_add_to_weak_set(PyObject **pset, PyObject *obj) +_add_to_weak_set(_abc_data *impl, PyObject **pset, PyObject *obj) { - if (*pset == NULL) { - *pset = PySet_New(NULL); - if (*pset == NULL) { - return -1; - } + PyObject *set; + Py_BEGIN_CRITICAL_SECTION(impl); + set = *pset; + if (set == NULL) { + set = *pset = PySet_New(NULL); + } + Py_END_CRITICAL_SECTION(); + if (set == NULL) { + return -1; } - PyObject *set = *pset; PyObject *ref, *wr; PyObject *destroy_cb; wr = PyWeakref_NewRef(set, NULL); @@ -220,7 +275,11 @@ _abc__reset_registry(PyObject *module, PyObject *self) if (impl == NULL) { return NULL; } - if (impl->_abc_registry != NULL && PySet_Clear(impl->_abc_registry) < 0) { + PyObject *registry; + Py_BEGIN_CRITICAL_SECTION(impl); + registry = impl->_abc_registry; + Py_END_CRITICAL_SECTION(); + if (registry != NULL && PySet_Clear(registry) < 0) { Py_DECREF(impl); return NULL; } @@ -247,13 +306,17 @@ _abc__reset_caches(PyObject *module, PyObject *self) if (impl == NULL) { return NULL; } - if (impl->_abc_cache != NULL && PySet_Clear(impl->_abc_cache) < 0) { + PyObject *cache, *negative_cache; + Py_BEGIN_CRITICAL_SECTION(impl); + cache = impl->_abc_cache; + negative_cache = impl->_abc_negative_cache; + Py_END_CRITICAL_SECTION(); + if (cache != NULL && PySet_Clear(cache) < 0) { Py_DECREF(impl); return NULL; } /* also the second cache */ - if (impl->_abc_negative_cache != NULL && - PySet_Clear(impl->_abc_negative_cache) < 0) { + if (negative_cache != NULL && PySet_Clear(negative_cache) < 0) { Py_DECREF(impl); return NULL; } @@ -282,11 +345,14 @@ _abc__get_dump(PyObject *module, PyObject *self) if (impl == NULL) { return NULL; } - PyObject *res = Py_BuildValue("NNNK", - PySet_New(impl->_abc_registry), - PySet_New(impl->_abc_cache), - PySet_New(impl->_abc_negative_cache), - impl->_abc_negative_cache_version); + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(impl); + res = Py_BuildValue("NNNK", + PySet_New(impl->_abc_registry), + PySet_New(impl->_abc_cache), + PySet_New(impl->_abc_negative_cache), + get_cache_version(impl)); + Py_END_CRITICAL_SECTION(); Py_DECREF(impl); return res; } @@ -453,56 +519,27 @@ _abc__abc_init(PyObject *module, PyObject *self) if (PyType_Check(self)) { PyTypeObject *cls = (PyTypeObject *)self; PyObject *dict = _PyType_GetDict(cls); - PyObject *flags = PyDict_GetItemWithError(dict, - &_Py_ID(__abc_tpflags__)); - if (flags == NULL) { - if (PyErr_Occurred()) { - return NULL; - } + PyObject *flags = NULL; + if (PyDict_Pop(dict, &_Py_ID(__abc_tpflags__), &flags) < 0) { + return NULL; } - else { - if (PyLong_CheckExact(flags)) { - long val = PyLong_AsLong(flags); - if (val == -1 && PyErr_Occurred()) { - return NULL; - } - if ((val & COLLECTION_FLAGS) == COLLECTION_FLAGS) { - PyErr_SetString(PyExc_TypeError, "__abc_tpflags__ cannot be both Py_TPFLAGS_SEQUENCE and Py_TPFLAGS_MAPPING"); - return NULL; - } - ((PyTypeObject *)self)->tp_flags |= (val & COLLECTION_FLAGS); - } - if (PyDict_DelItem(dict, &_Py_ID(__abc_tpflags__)) < 0) { - return NULL; - } + if (flags == NULL || !PyLong_CheckExact(flags)) { + Py_XDECREF(flags); + Py_RETURN_NONE; } - } - Py_RETURN_NONE; -} - -static void -set_collection_flag_recursive(PyTypeObject *child, unsigned long flag) -{ - assert(flag == Py_TPFLAGS_MAPPING || flag == Py_TPFLAGS_SEQUENCE); - if (PyType_HasFeature(child, Py_TPFLAGS_IMMUTABLETYPE) || - (child->tp_flags & COLLECTION_FLAGS) == flag) - { - return; - } - - child->tp_flags &= ~COLLECTION_FLAGS; - child->tp_flags |= flag; - - PyObject *grandchildren = _PyType_GetSubclasses(child); - if (grandchildren == NULL) { - return; - } - for (Py_ssize_t i = 0; i < PyList_GET_SIZE(grandchildren); i++) { - PyObject *grandchild = PyList_GET_ITEM(grandchildren, i); - set_collection_flag_recursive((PyTypeObject *)grandchild, flag); + long val = PyLong_AsLong(flags); + Py_DECREF(flags); + if (val == -1 && PyErr_Occurred()) { + return NULL; + } + if ((val & COLLECTION_FLAGS) == COLLECTION_FLAGS) { + PyErr_SetString(PyExc_TypeError, "__abc_tpflags__ cannot be both Py_TPFLAGS_SEQUENCE and Py_TPFLAGS_MAPPING"); + return NULL; + } + _PyType_SetFlags((PyTypeObject *)self, 0, val & COLLECTION_FLAGS); } - Py_DECREF(grandchildren); + Py_RETURN_NONE; } /*[clinic input] @@ -545,20 +582,23 @@ _abc__abc_register_impl(PyObject *module, PyObject *self, PyObject *subclass) if (impl == NULL) { return NULL; } - if (_add_to_weak_set(&impl->_abc_registry, subclass) < 0) { + if (_add_to_weak_set(impl, &impl->_abc_registry, subclass) < 0) { Py_DECREF(impl); return NULL; } Py_DECREF(impl); /* Invalidate negative cache */ - get_abc_state(module)->abc_invalidation_counter++; + increment_invalidation_counter(get_abc_state(module)); - /* Set Py_TPFLAGS_SEQUENCE or Py_TPFLAGS_MAPPING flag */ + /* Set Py_TPFLAGS_SEQUENCE or Py_TPFLAGS_MAPPING flag */ if (PyType_Check(self)) { - unsigned long collection_flag = ((PyTypeObject *)self)->tp_flags & COLLECTION_FLAGS; + unsigned long collection_flag = + PyType_GetFlags((PyTypeObject *)self) & COLLECTION_FLAGS; if (collection_flag) { - set_collection_flag_recursive((PyTypeObject *)subclass, collection_flag); + _PyType_SetFlagsRecursive((PyTypeObject *)subclass, + COLLECTION_FLAGS, + collection_flag); } } return Py_NewRef(subclass); @@ -592,7 +632,7 @@ _abc__abc_instancecheck_impl(PyObject *module, PyObject *self, return NULL; } /* Inline the cache checking. */ - int incache = _in_weak_set(impl->_abc_cache, subclass); + int incache = _in_weak_set(impl, &impl->_abc_cache, subclass); if (incache < 0) { goto end; } @@ -602,8 +642,8 @@ _abc__abc_instancecheck_impl(PyObject *module, PyObject *self, } subtype = (PyObject *)Py_TYPE(instance); if (subtype == subclass) { - if (impl->_abc_negative_cache_version == get_abc_state(module)->abc_invalidation_counter) { - incache = _in_weak_set(impl->_abc_negative_cache, subclass); + if (get_cache_version(impl) == get_invalidation_counter(get_abc_state(module))) { + incache = _in_weak_set(impl, &impl->_abc_negative_cache, subclass); if (incache < 0) { goto end; } @@ -681,7 +721,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, } /* 1. Check cache. */ - incache = _in_weak_set(impl->_abc_cache, subclass); + incache = _in_weak_set(impl, &impl->_abc_cache, subclass); if (incache < 0) { goto end; } @@ -692,17 +732,20 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, state = get_abc_state(module); /* 2. Check negative cache; may have to invalidate. */ - if (impl->_abc_negative_cache_version < state->abc_invalidation_counter) { + uint64_t invalidation_counter = get_invalidation_counter(state); + if (get_cache_version(impl) < invalidation_counter) { /* Invalidate the negative cache. */ - if (impl->_abc_negative_cache != NULL && - PySet_Clear(impl->_abc_negative_cache) < 0) - { + PyObject *negative_cache; + Py_BEGIN_CRITICAL_SECTION(impl); + negative_cache = impl->_abc_negative_cache; + Py_END_CRITICAL_SECTION(); + if (negative_cache != NULL && PySet_Clear(negative_cache) < 0) { goto end; } - impl->_abc_negative_cache_version = state->abc_invalidation_counter; + set_cache_version(impl, invalidation_counter); } else { - incache = _in_weak_set(impl->_abc_negative_cache, subclass); + incache = _in_weak_set(impl, &impl->_abc_negative_cache, subclass); if (incache < 0) { goto end; } @@ -720,7 +763,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, } if (ok == Py_True) { Py_DECREF(ok); - if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { + if (_add_to_weak_set(impl, &impl->_abc_cache, subclass) < 0) { goto end; } result = Py_True; @@ -728,7 +771,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, } if (ok == Py_False) { Py_DECREF(ok); - if (_add_to_weak_set(&impl->_abc_negative_cache, subclass) < 0) { + if (_add_to_weak_set(impl, &impl->_abc_negative_cache, subclass) < 0) { goto end; } result = Py_False; @@ -744,7 +787,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, /* 4. Check if it's a direct subclass. */ if (PyType_IsSubtype((PyTypeObject *)subclass, (PyTypeObject *)self)) { - if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { + if (_add_to_weak_set(impl, &impl->_abc_cache, subclass) < 0) { goto end; } result = Py_True; @@ -767,12 +810,14 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, goto end; } for (pos = 0; pos < PyList_GET_SIZE(subclasses); pos++) { - PyObject *scls = PyList_GET_ITEM(subclasses, pos); - Py_INCREF(scls); + PyObject *scls = PyList_GetItemRef(subclasses, pos); + if (scls == NULL) { + goto end; + } int r = PyObject_IsSubclass(subclass, scls); Py_DECREF(scls); if (r > 0) { - if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { + if (_add_to_weak_set(impl, &impl->_abc_cache, subclass) < 0) { goto end; } result = Py_True; @@ -784,7 +829,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, } /* No dice; update negative cache. */ - if (_add_to_weak_set(&impl->_abc_negative_cache, subclass) < 0) { + if (_add_to_weak_set(impl, &impl->_abc_negative_cache, subclass) < 0) { goto end; } result = Py_False; @@ -801,7 +846,7 @@ subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, PyObject **result) { // Fast path: check subclass is in weakref directly. - int ret = _in_weak_set(impl->_abc_registry, subclass); + int ret = _in_weak_set(impl, &impl->_abc_registry, subclass); if (ret < 0) { *result = NULL; return -1; @@ -811,33 +856,27 @@ subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, return 1; } - if (impl->_abc_registry == NULL) { + PyObject *registry_shared; + Py_BEGIN_CRITICAL_SECTION(impl); + registry_shared = impl->_abc_registry; + Py_END_CRITICAL_SECTION(); + if (registry_shared == NULL) { return 0; } - Py_ssize_t registry_size = PySet_Size(impl->_abc_registry); - if (registry_size == 0) { - return 0; - } - // Weakref callback may remove entry from set. - // So we take snapshot of registry first. - PyObject **copy = PyMem_Malloc(sizeof(PyObject*) * registry_size); - if (copy == NULL) { - PyErr_NoMemory(); + + // Make a local copy of the registry to protect against concurrent + // modifications of _abc_registry. + PyObject *registry = PyFrozenSet_New(registry_shared); + if (registry == NULL) { return -1; } PyObject *key; Py_ssize_t pos = 0; Py_hash_t hash; - Py_ssize_t i = 0; - while (_PySet_NextEntry(impl->_abc_registry, &pos, &key, &hash)) { - copy[i++] = Py_NewRef(key); - } - assert(i == registry_size); - - for (i = 0; i < registry_size; i++) { + while (_PySet_NextEntry(registry, &pos, &key, &hash)) { PyObject *rkey; - if (PyWeakref_GetRef(copy[i], &rkey) < 0) { + if (PyWeakref_GetRef(key, &rkey) < 0) { // Someone inject non-weakref type in the registry. ret = -1; break; @@ -853,7 +892,7 @@ subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, break; } if (r > 0) { - if (_add_to_weak_set(&impl->_abc_cache, subclass) < 0) { + if (_add_to_weak_set(impl, &impl->_abc_cache, subclass) < 0) { ret = -1; break; } @@ -863,10 +902,7 @@ subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, } } - for (i = 0; i < registry_size; i++) { - Py_DECREF(copy[i]); - } - PyMem_Free(copy); + Py_DECREF(registry); return ret; } @@ -885,7 +921,7 @@ _abc_get_cache_token_impl(PyObject *module) /*[clinic end generated code: output=c7d87841e033dacc input=70413d1c423ad9f9]*/ { _abcmodule_state *state = get_abc_state(module); - return PyLong_FromUnsignedLongLong(state->abc_invalidation_counter); + return PyLong_FromUnsignedLongLong(get_invalidation_counter(state)); } static struct PyMethodDef _abcmodule_methods[] = { @@ -932,12 +968,13 @@ _abcmodule_clear(PyObject *module) static void _abcmodule_free(void *module) { - _abcmodule_clear((PyObject *)module); + (void)_abcmodule_clear((PyObject *)module); } static PyModuleDef_Slot _abcmodule_slots[] = { {Py_mod_exec, _abcmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index c1aa849ecf1aad..d5d49658555f1a 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3,9 +3,13 @@ #endif #include "Python.h" +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION_MUT() #include "pycore_dict.h" // _PyDict_GetItem_KnownHash() +#include "pycore_freelist.h" // _Py_FREELIST_POP() +#include "pycore_llist.h" // struct llist_node #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_object.h" // _PyObject_SetMaybeWeakref #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -19,13 +23,121 @@ module _asyncio [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=8fd17862aa989c69]*/ +typedef enum { + STATE_PENDING, + STATE_CANCELLED, + STATE_FINISHED +} fut_state; + +#define FutureObj_HEAD(prefix) \ + PyObject_HEAD \ + PyObject *prefix##_loop; \ + PyObject *prefix##_callback0; \ + PyObject *prefix##_context0; \ + PyObject *prefix##_callbacks; \ + PyObject *prefix##_exception; \ + PyObject *prefix##_exception_tb; \ + PyObject *prefix##_result; \ + PyObject *prefix##_source_tb; \ + PyObject *prefix##_cancel_msg; \ + PyObject *prefix##_cancelled_exc; \ + PyObject *prefix##_awaited_by; \ + fut_state prefix##_state; \ + /* Used by profilers to make traversing the stack from an external \ + process faster. */ \ + char prefix##_is_task; \ + char prefix##_awaited_by_is_set; \ + /* These bitfields need to be at the end of the struct \ + so that these and bitfields from TaskObj are contiguous. \ + */ \ + unsigned prefix##_log_tb: 1; \ + unsigned prefix##_blocking: 1; \ + +typedef struct { + FutureObj_HEAD(fut) +} FutureObj; + +typedef struct TaskObj { + FutureObj_HEAD(task) + unsigned task_must_cancel: 1; + unsigned task_log_destroy_pending: 1; + int task_num_cancels_requested; + PyObject *task_fut_waiter; + PyObject *task_coro; + PyObject *task_name; + PyObject *task_context; + struct llist_node task_node; +} TaskObj; + +typedef struct { + PyObject_HEAD + TaskObj *sw_task; + PyObject *sw_arg; +} TaskStepMethWrapper; + +#define Future_CheckExact(state, obj) Py_IS_TYPE(obj, state->FutureType) +#define Task_CheckExact(state, obj) Py_IS_TYPE(obj, state->TaskType) -#define FI_FREELIST_MAXLEN 255 +#define Future_Check(state, obj) \ + (Future_CheckExact(state, obj) \ + || PyObject_TypeCheck(obj, state->FutureType)) + +#define Task_Check(state, obj) \ + (Task_CheckExact(state, obj) \ + || PyObject_TypeCheck(obj, state->TaskType)) + +// This macro is optimized to quickly return for native Future *or* Task +// objects by inlining fast "exact" checks to be called first. +#define TaskOrFuture_Check(state, obj) \ + (Task_CheckExact(state, obj) \ + || Future_CheckExact(state, obj) \ + || PyObject_TypeCheck(obj, state->FutureType) \ + || PyObject_TypeCheck(obj, state->TaskType)) + +#ifdef Py_GIL_DISABLED +# define ASYNCIO_STATE_LOCK(state) Py_BEGIN_CRITICAL_SECTION_MUT(&state->mutex) +# define ASYNCIO_STATE_UNLOCK(state) Py_END_CRITICAL_SECTION() +#else +# define ASYNCIO_STATE_LOCK(state) ((void)state) +# define ASYNCIO_STATE_UNLOCK(state) ((void)state) +#endif -typedef struct futureiterobject futureiterobject; +typedef struct _Py_AsyncioModuleDebugOffsets { + struct _asyncio_task_object { + uint64_t size; + uint64_t task_name; + uint64_t task_awaited_by; + uint64_t task_is_task; + uint64_t task_awaited_by_is_set; + uint64_t task_coro; + } asyncio_task_object; + struct _asyncio_thread_state { + uint64_t size; + uint64_t asyncio_running_loop; + uint64_t asyncio_running_task; + } asyncio_thread_state; +} Py_AsyncioModuleDebugOffsets; + +GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets AsyncioDebug) + = {.asyncio_task_object = { + .size = sizeof(TaskObj), + .task_name = offsetof(TaskObj, task_name), + .task_awaited_by = offsetof(TaskObj, task_awaited_by), + .task_is_task = offsetof(TaskObj, task_is_task), + .task_awaited_by_is_set = offsetof(TaskObj, task_awaited_by_is_set), + .task_coro = offsetof(TaskObj, task_coro), + }, + .asyncio_thread_state = { + .size = sizeof(_PyThreadStateImpl), + .asyncio_running_loop = offsetof(_PyThreadStateImpl, asyncio_running_loop), + .asyncio_running_task = offsetof(_PyThreadStateImpl, asyncio_running_task), + }}; /* State of the _asyncio module */ typedef struct { +#ifdef Py_GIL_DISABLED + PyMutex mutex; +#endif PyTypeObject *FutureIterType; PyTypeObject *TaskStepMethWrapper_Type; PyTypeObject *FutureType; @@ -38,8 +150,9 @@ typedef struct { all running event loops. {EventLoop: Task} */ PyObject *current_tasks; - /* WeakSet containing all tasks scheduled to run on event loops. */ - PyObject *scheduled_tasks; + /* WeakSet containing scheduled 3rd party tasks which don't + inherit from native asyncio.Task */ + PyObject *non_asyncio_tasks; /* Set containing all eagerly executing tasks. */ PyObject *eager_tasks; @@ -68,14 +181,14 @@ typedef struct { /* Imports from traceback. */ PyObject *traceback_extract_stack; - PyObject *cached_running_loop; // Borrowed reference - volatile uint64_t cached_running_loop_tsid; - /* Counter for autogenerated Task names */ uint64_t task_name_counter; - futureiterobject *fi_freelist; - Py_ssize_t fi_freelist_len; + /* Head of circular linked-list of all tasks which are instances of `asyncio.Task` + or subclasses of it. Third party tasks implementations which don't inherit from + `asyncio.Task` are tracked separately using the `non_asyncio_tasks` WeakSet. + */ + struct llist_node asyncio_tasks_head; } asyncio_state; static inline asyncio_state * @@ -105,59 +218,6 @@ get_asyncio_state_by_def(PyObject *self) return get_asyncio_state(mod); } -typedef enum { - STATE_PENDING, - STATE_CANCELLED, - STATE_FINISHED -} fut_state; - -#define FutureObj_HEAD(prefix) \ - PyObject_HEAD \ - PyObject *prefix##_loop; \ - PyObject *prefix##_callback0; \ - PyObject *prefix##_context0; \ - PyObject *prefix##_callbacks; \ - PyObject *prefix##_exception; \ - PyObject *prefix##_exception_tb; \ - PyObject *prefix##_result; \ - PyObject *prefix##_source_tb; \ - PyObject *prefix##_cancel_msg; \ - PyObject *prefix##_cancelled_exc; \ - fut_state prefix##_state; \ - /* These bitfields need to be at the end of the struct - so that these and bitfields from TaskObj are contiguous. - */ \ - unsigned prefix##_log_tb: 1; \ - unsigned prefix##_blocking: 1; - -typedef struct { - FutureObj_HEAD(fut) -} FutureObj; - -typedef struct { - FutureObj_HEAD(task) - unsigned task_must_cancel: 1; - unsigned task_log_destroy_pending: 1; - int task_num_cancels_requested; - PyObject *task_fut_waiter; - PyObject *task_coro; - PyObject *task_name; - PyObject *task_context; -} TaskObj; - -typedef struct { - PyObject_HEAD - TaskObj *sw_task; - PyObject *sw_arg; -} TaskStepMethWrapper; - - -#define Future_CheckExact(state, obj) Py_IS_TYPE(obj, state->FutureType) -#define Task_CheckExact(state, obj) Py_IS_TYPE(obj, state->TaskType) - -#define Future_Check(state, obj) PyObject_TypeCheck(obj, state->FutureType) -#define Task_Check(state, obj) PyObject_TypeCheck(obj, state->TaskType) - #include "clinic/_asynciomodule.c.h" @@ -174,6 +234,22 @@ static PyObject * task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *result); +static void +clear_task_coro(TaskObj *task) +{ + Py_CLEAR(task->task_coro); +} + + +static void +set_task_coro(TaskObj *task, PyObject *coro) +{ + assert(coro != NULL); + Py_INCREF(coro); + Py_XSETREF(task->task_coro, coro); +} + + static int _is_coroutine(asyncio_state *state, PyObject *coro) { @@ -262,96 +338,15 @@ get_future_loop(asyncio_state *state, PyObject *fut) return PyObject_GetAttr(fut, &_Py_ID(_loop)); } - -static int -get_running_loop(asyncio_state *state, PyObject **loop) -{ - PyObject *rl; - - PyThreadState *ts = _PyThreadState_GET(); - uint64_t ts_id = PyThreadState_GetID(ts); - if (state->cached_running_loop_tsid == ts_id && - state->cached_running_loop != NULL) - { - // Fast path, check the cache. - rl = state->cached_running_loop; - } - else { - PyObject *ts_dict = _PyThreadState_GetDict(ts); // borrowed - if (ts_dict == NULL) { - goto not_found; - } - - rl = PyDict_GetItemWithError( - ts_dict, &_Py_ID(__asyncio_running_event_loop__)); // borrowed - if (rl == NULL) { - if (PyErr_Occurred()) { - goto error; - } - else { - goto not_found; - } - } - - state->cached_running_loop = rl; - state->cached_running_loop_tsid = ts_id; - } - - - if (rl == Py_None) { - goto not_found; - } - - *loop = Py_NewRef(rl); - return 0; - -not_found: - *loop = NULL; - return 0; - -error: - *loop = NULL; - return -1; -} - - -static int -set_running_loop(asyncio_state *state, PyObject *loop) -{ - PyObject *ts_dict = NULL; - - PyThreadState *tstate = _PyThreadState_GET(); - if (tstate != NULL) { - ts_dict = _PyThreadState_GetDict(tstate); // borrowed - } - - if (ts_dict == NULL) { - PyErr_SetString( - PyExc_RuntimeError, "thread-local storage is not available"); - return -1; - } - if (PyDict_SetItem( - ts_dict, &_Py_ID(__asyncio_running_event_loop__), loop) < 0) - { - return -1; - } - - state->cached_running_loop = loop; // borrowed, kept alive by ts_dict - state->cached_running_loop_tsid = PyThreadState_GetID(tstate); - - return 0; -} - - static PyObject * get_event_loop(asyncio_state *state) { PyObject *loop; PyObject *policy; - if (get_running_loop(state, &loop)) { - return NULL; - } + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + loop = Py_XNewRef(ts->asyncio_running_loop); + if (loop != NULL) { return loop; } @@ -434,18 +429,24 @@ future_ensure_alive(FutureObj *fut) static int future_schedule_callbacks(asyncio_state *state, FutureObj *fut) { - Py_ssize_t len; - Py_ssize_t i; + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); if (fut->fut_callback0 != NULL) { /* There's a 1st callback */ - int ret = call_soon(state, - fut->fut_loop, fut->fut_callback0, - (PyObject *)fut, fut->fut_context0); - - Py_CLEAR(fut->fut_callback0); - Py_CLEAR(fut->fut_context0); + // Beware: An evil call_soon could alter fut_callback0 or fut_context0. + // Since we are anyway clearing them after the call, whether call_soon + // succeeds or not, the idea is to transfer ownership so that external + // code is not able to alter them during the call. + PyObject *fut_callback0 = fut->fut_callback0; + fut->fut_callback0 = NULL; + PyObject *fut_context0 = fut->fut_context0; + fut->fut_context0 = NULL; + + int ret = call_soon(state, fut->fut_loop, fut_callback0, + (PyObject *)fut, fut_context0); + Py_CLEAR(fut_callback0); + Py_CLEAR(fut_context0); if (ret) { /* If an error occurs in pure-Python implementation, all callbacks are cleared. */ @@ -462,27 +463,25 @@ future_schedule_callbacks(asyncio_state *state, FutureObj *fut) return 0; } - len = PyList_GET_SIZE(fut->fut_callbacks); - if (len == 0) { - /* The list of callbacks was empty; clear it and return. */ - Py_CLEAR(fut->fut_callbacks); - return 0; - } - - for (i = 0; i < len; i++) { - PyObject *cb_tup = PyList_GET_ITEM(fut->fut_callbacks, i); + // Beware: An evil call_soon could change fut->fut_callbacks. + // The idea is to transfer the ownership of the callbacks list + // so that external code is not able to mutate the list during + // the iteration. + PyObject *callbacks = fut->fut_callbacks; + fut->fut_callbacks = NULL; + Py_ssize_t n = PyList_GET_SIZE(callbacks); + for (Py_ssize_t i = 0; i < n; i++) { + assert(PyList_GET_SIZE(callbacks) == n); + PyObject *cb_tup = PyList_GET_ITEM(callbacks, i); PyObject *cb = PyTuple_GET_ITEM(cb_tup, 0); PyObject *ctx = PyTuple_GET_ITEM(cb_tup, 1); if (call_soon(state, fut->fut_loop, cb, (PyObject *)fut, ctx)) { - /* If an error occurs in pure-Python implementation, - all callbacks are cleared. */ - Py_CLEAR(fut->fut_callbacks); + Py_DECREF(callbacks); return -1; } } - - Py_CLEAR(fut->fut_callbacks); + Py_DECREF(callbacks); return 0; } @@ -503,10 +502,13 @@ future_init(FutureObj *fut, PyObject *loop) Py_CLEAR(fut->fut_source_tb); Py_CLEAR(fut->fut_cancel_msg); Py_CLEAR(fut->fut_cancelled_exc); + Py_CLEAR(fut->fut_awaited_by); fut->fut_state = STATE_PENDING; fut->fut_log_tb = 0; fut->fut_blocking = 0; + fut->fut_awaited_by_is_set = 0; + fut->fut_is_task = 0; if (loop == Py_None) { asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); @@ -546,9 +548,120 @@ future_init(FutureObj *fut, PyObject *loop) return 0; } +static int +future_awaited_by_add(asyncio_state *state, PyObject *fut, PyObject *thing) +{ + if (!TaskOrFuture_Check(state, fut) || !TaskOrFuture_Check(state, thing)) { + // We only want to support native asyncio Futures. + // For further insight see the comment in the Python + // implementation of "future_add_to_awaited_by()". + return 0; + } + + FutureObj *_fut = (FutureObj *)fut; + + /* Most futures/task are only awaited by one entity, so we want + to avoid always creating a set for `fut_awaited_by`. + */ + if (_fut->fut_awaited_by == NULL) { + assert(!_fut->fut_awaited_by_is_set); + Py_INCREF(thing); + _fut->fut_awaited_by = thing; + return 0; + } + + if (_fut->fut_awaited_by_is_set) { + assert(PySet_CheckExact(_fut->fut_awaited_by)); + return PySet_Add(_fut->fut_awaited_by, thing); + } + + PyObject *set = PySet_New(NULL); + if (set == NULL) { + return -1; + } + if (PySet_Add(set, thing)) { + Py_DECREF(set); + return -1; + } + if (PySet_Add(set, _fut->fut_awaited_by)) { + Py_DECREF(set); + return -1; + } + Py_SETREF(_fut->fut_awaited_by, set); + _fut->fut_awaited_by_is_set = 1; + return 0; +} + +static int +future_awaited_by_discard(asyncio_state *state, PyObject *fut, PyObject *thing) +{ + if (!TaskOrFuture_Check(state, fut) || !TaskOrFuture_Check(state, thing)) { + // We only want to support native asyncio Futures. + // For further insight see the comment in the Python + // implementation of "future_add_to_awaited_by()". + return 0; + } + + FutureObj *_fut = (FutureObj *)fut; + + /* Following the semantics of 'set.discard()' here in not + raising an error if `thing` isn't in the `awaited_by` "set". + */ + if (_fut->fut_awaited_by == NULL) { + return 0; + } + if (_fut->fut_awaited_by == thing) { + Py_CLEAR(_fut->fut_awaited_by); + return 0; + } + if (_fut->fut_awaited_by_is_set) { + assert(PySet_CheckExact(_fut->fut_awaited_by)); + int err = PySet_Discard(_fut->fut_awaited_by, thing); + if (err < 0) { + return -1; + } else { + return 0; + } + } + return 0; +} + +/*[clinic input] +@critical_section +@getter +_asyncio.Future._asyncio_awaited_by +[clinic start generated code]*/ + +static PyObject * +_asyncio_Future__asyncio_awaited_by_get_impl(FutureObj *self) +/*[clinic end generated code: output=932af76d385d2e2a input=64c1783df2d44d2b]*/ +{ + /* Implementation of a Python getter. */ + if (self->fut_awaited_by == NULL) { + Py_RETURN_NONE; + } + if (self->fut_awaited_by_is_set) { + /* Already a set, just wrap it into a frozen set and return. */ + assert(PySet_CheckExact(self->fut_awaited_by)); + return PyFrozenSet_New(self->fut_awaited_by); + } + + PyObject *set = PyFrozenSet_New(NULL); + if (set == NULL) { + return NULL; + } + if (PySet_Add(set, self->fut_awaited_by)) { + Py_DECREF(set); + return NULL; + } + return set; +} + static PyObject * future_set_result(asyncio_state *state, FutureObj *fut, PyObject *res) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + if (future_ensure_alive(fut)) { return NULL; } @@ -571,6 +684,8 @@ future_set_result(asyncio_state *state, FutureObj *fut, PyObject *res) static PyObject * future_set_exception(asyncio_state *state, FutureObj *fut, PyObject *exc) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + PyObject *exc_val = NULL; if (fut->fut_state != STATE_PENDING) { @@ -637,6 +752,8 @@ future_set_exception(asyncio_state *state, FutureObj *fut, PyObject *exc) static PyObject * create_cancelled_error(asyncio_state *state, FutureObj *fut) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + PyObject *exc; if (fut->fut_cancelled_exc != NULL) { /* transfer ownership */ @@ -656,6 +773,8 @@ create_cancelled_error(asyncio_state *state, FutureObj *fut) static void future_set_cancelled_error(asyncio_state *state, FutureObj *fut) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + PyObject *exc = create_cancelled_error(state, fut); if (exc == NULL) { return; @@ -667,6 +786,8 @@ future_set_cancelled_error(asyncio_state *state, FutureObj *fut) static int future_get_result(asyncio_state *state, FutureObj *fut, PyObject **result) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + if (fut->fut_state == STATE_CANCELLED) { future_set_cancelled_error(state, fut); return -1; @@ -700,6 +821,8 @@ static PyObject * future_add_done_callback(asyncio_state *state, FutureObj *fut, PyObject *arg, PyObject *ctx) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + if (!future_is_alive(fut)) { PyErr_SetString(PyExc_RuntimeError, "uninitialized Future object"); return NULL; @@ -774,6 +897,8 @@ future_add_done_callback(asyncio_state *state, FutureObj *fut, PyObject *arg, static PyObject * future_cancel(asyncio_state *state, FutureObj *fut, PyObject *msg) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); + fut->fut_log_tb = 0; if (fut->fut_state != STATE_PENDING) { @@ -832,6 +957,8 @@ FutureObj_clear(FutureObj *fut) Py_CLEAR(fut->fut_source_tb); Py_CLEAR(fut->fut_cancel_msg); Py_CLEAR(fut->fut_cancelled_exc); + Py_CLEAR(fut->fut_awaited_by); + fut->fut_awaited_by_is_set = 0; PyObject_ClearManagedDict((PyObject *)fut); return 0; } @@ -850,11 +977,13 @@ FutureObj_traverse(FutureObj *fut, visitproc visit, void *arg) Py_VISIT(fut->fut_source_tb); Py_VISIT(fut->fut_cancel_msg); Py_VISIT(fut->fut_cancelled_exc); + Py_VISIT(fut->fut_awaited_by); PyObject_VisitManagedDict((PyObject *)fut, visit, arg); return 0; } /*[clinic input] +@critical_section _asyncio.Future.result Return the result this future represents. @@ -866,7 +995,7 @@ the future is done and has an exception set, this exception is raised. static PyObject * _asyncio_Future_result_impl(FutureObj *self) -/*[clinic end generated code: output=f35f940936a4b1e5 input=49ecf9cf5ec50dc5]*/ +/*[clinic end generated code: output=f35f940936a4b1e5 input=61d89f48e4c8b670]*/ { asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); PyObject *result; @@ -895,6 +1024,7 @@ _asyncio_Future_result_impl(FutureObj *self) } /*[clinic input] +@critical_section _asyncio.Future.exception cls: defining_class @@ -910,7 +1040,7 @@ InvalidStateError. static PyObject * _asyncio_Future_exception_impl(FutureObj *self, PyTypeObject *cls) -/*[clinic end generated code: output=ce75576b187c905b input=3faf15c22acdb60d]*/ +/*[clinic end generated code: output=ce75576b187c905b input=647d1fd1fc403301]*/ { if (!future_is_alive(self)) { asyncio_state *state = get_asyncio_state_by_cls(cls); @@ -941,6 +1071,7 @@ _asyncio_Future_exception_impl(FutureObj *self, PyTypeObject *cls) } /*[clinic input] +@critical_section _asyncio.Future.set_result cls: defining_class @@ -956,7 +1087,7 @@ InvalidStateError. static PyObject * _asyncio_Future_set_result_impl(FutureObj *self, PyTypeObject *cls, PyObject *result) -/*[clinic end generated code: output=99afbbe78f99c32d input=d5a41c1e353acc2e]*/ +/*[clinic end generated code: output=99afbbe78f99c32d input=4069306f03a3b6ee]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); ENSURE_FUTURE_ALIVE(state, self) @@ -964,6 +1095,7 @@ _asyncio_Future_set_result_impl(FutureObj *self, PyTypeObject *cls, } /*[clinic input] +@critical_section _asyncio.Future.set_exception cls: defining_class @@ -979,7 +1111,7 @@ InvalidStateError. static PyObject * _asyncio_Future_set_exception_impl(FutureObj *self, PyTypeObject *cls, PyObject *exception) -/*[clinic end generated code: output=0a5e8b5a52f058d6 input=a245cd49d3df939b]*/ +/*[clinic end generated code: output=0a5e8b5a52f058d6 input=b6eab43a389bc966]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); ENSURE_FUTURE_ALIVE(state, self) @@ -987,6 +1119,7 @@ _asyncio_Future_set_exception_impl(FutureObj *self, PyTypeObject *cls, } /*[clinic input] +@critical_section _asyncio.Future.add_done_callback cls: defining_class @@ -1005,7 +1138,7 @@ scheduled with call_soon. static PyObject * _asyncio_Future_add_done_callback_impl(FutureObj *self, PyTypeObject *cls, PyObject *fn, PyObject *context) -/*[clinic end generated code: output=922e9a4cbd601167 input=599261c521458cc2]*/ +/*[clinic end generated code: output=922e9a4cbd601167 input=37d97f941beb7b3e]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); if (context == NULL) { @@ -1021,6 +1154,7 @@ _asyncio_Future_add_done_callback_impl(FutureObj *self, PyTypeObject *cls, } /*[clinic input] +@critical_section _asyncio.Future.remove_done_callback cls: defining_class @@ -1035,7 +1169,7 @@ Returns the number of callbacks removed. static PyObject * _asyncio_Future_remove_done_callback_impl(FutureObj *self, PyTypeObject *cls, PyObject *fn) -/*[clinic end generated code: output=2da35ccabfe41b98 input=c7518709b86fc747]*/ +/*[clinic end generated code: output=2da35ccabfe41b98 input=3afbc9f6a673091b]*/ { PyObject *newlist; Py_ssize_t len, i, j=0; @@ -1045,7 +1179,12 @@ _asyncio_Future_remove_done_callback_impl(FutureObj *self, PyTypeObject *cls, ENSURE_FUTURE_ALIVE(state, self) if (self->fut_callback0 != NULL) { - int cmp = PyObject_RichCompareBool(self->fut_callback0, fn, Py_EQ); + // Beware: An evil PyObject_RichCompareBool could free fut_callback0 + // before a recursive call is made with that same arg. For details, see + // https://github.com/python/cpython/pull/125967#discussion_r1816593340. + PyObject *fut_callback0 = Py_NewRef(self->fut_callback0); + int cmp = PyObject_RichCompareBool(fut_callback0, fn, Py_EQ); + Py_DECREF(fut_callback0); if (cmp == -1) { return NULL; } @@ -1069,8 +1208,10 @@ _asyncio_Future_remove_done_callback_impl(FutureObj *self, PyTypeObject *cls, if (len == 1) { PyObject *cb_tup = PyList_GET_ITEM(self->fut_callbacks, 0); + Py_INCREF(cb_tup); int cmp = PyObject_RichCompareBool( PyTuple_GET_ITEM(cb_tup, 0), fn, Py_EQ); + Py_DECREF(cb_tup); if (cmp == -1) { return NULL; } @@ -1137,6 +1278,7 @@ _asyncio_Future_remove_done_callback_impl(FutureObj *self, PyTypeObject *cls, } /*[clinic input] +@critical_section _asyncio.Future.cancel cls: defining_class @@ -1153,7 +1295,7 @@ return True. static PyObject * _asyncio_Future_cancel_impl(FutureObj *self, PyTypeObject *cls, PyObject *msg) -/*[clinic end generated code: output=074956f35904b034 input=bba8f8b786941a94]*/ +/*[clinic end generated code: output=074956f35904b034 input=44ab4003da839970]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); ENSURE_FUTURE_ALIVE(state, self) @@ -1161,6 +1303,7 @@ _asyncio_Future_cancel_impl(FutureObj *self, PyTypeObject *cls, } /*[clinic input] +@critical_section _asyncio.Future.cancelled Return True if the future was cancelled. @@ -1168,7 +1311,7 @@ Return True if the future was cancelled. static PyObject * _asyncio_Future_cancelled_impl(FutureObj *self) -/*[clinic end generated code: output=145197ced586357d input=943ab8b7b7b17e45]*/ +/*[clinic end generated code: output=145197ced586357d input=9b8644819a675416]*/ { if (future_is_alive(self) && self->fut_state == STATE_CANCELLED) { Py_RETURN_TRUE; @@ -1179,6 +1322,7 @@ _asyncio_Future_cancelled_impl(FutureObj *self) } /*[clinic input] +@critical_section _asyncio.Future.done Return True if the future is done. @@ -1189,7 +1333,7 @@ future was cancelled. static PyObject * _asyncio_Future_done_impl(FutureObj *self) -/*[clinic end generated code: output=244c5ac351145096 input=28d7b23fdb65d2ac]*/ +/*[clinic end generated code: output=244c5ac351145096 input=7204d3cc63bef7f3]*/ { if (!future_is_alive(self) || self->fut_state == STATE_PENDING) { Py_RETURN_FALSE; @@ -1200,6 +1344,7 @@ _asyncio_Future_done_impl(FutureObj *self) } /*[clinic input] +@critical_section _asyncio.Future.get_loop cls: defining_class @@ -1210,17 +1355,24 @@ Return the event loop the Future is bound to. static PyObject * _asyncio_Future_get_loop_impl(FutureObj *self, PyTypeObject *cls) -/*[clinic end generated code: output=f50ea6c374d9ee97 input=163c2c498b45a1f0]*/ +/*[clinic end generated code: output=f50ea6c374d9ee97 input=f3ce629bfd9f45c1]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); ENSURE_FUTURE_ALIVE(state, self) return Py_NewRef(self->fut_loop); } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._asyncio_future_blocking +[clinic start generated code]*/ + static PyObject * -FutureObj_get_blocking(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__asyncio_future_blocking_get_impl(FutureObj *self) +/*[clinic end generated code: output=a558a2c51e38823b input=58da92efc03b617d]*/ { - if (future_is_alive(fut) && fut->fut_blocking) { + if (future_is_alive(self) && self->fut_blocking) { Py_RETURN_TRUE; } else { @@ -1228,31 +1380,47 @@ FutureObj_get_blocking(FutureObj *fut, void *Py_UNUSED(ignored)) } } +/*[clinic input] +@critical_section +@setter +_asyncio.Future._asyncio_future_blocking +[clinic start generated code]*/ + static int -FutureObj_set_blocking(FutureObj *fut, PyObject *val, void *Py_UNUSED(ignored)) +_asyncio_Future__asyncio_future_blocking_set_impl(FutureObj *self, + PyObject *value) +/*[clinic end generated code: output=0686d1cb024a7453 input=3fd4a5f95df788b7]*/ + { - if (future_ensure_alive(fut)) { + if (future_ensure_alive(self)) { return -1; } - if (val == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); return -1; } - int is_true = PyObject_IsTrue(val); + int is_true = PyObject_IsTrue(value); if (is_true < 0) { return -1; } - fut->fut_blocking = is_true; + self->fut_blocking = is_true; return 0; } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._log_traceback +[clinic start generated code]*/ + static PyObject * -FutureObj_get_log_traceback(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__log_traceback_get_impl(FutureObj *self) +/*[clinic end generated code: output=2724433b238593c7 input=91e5144ea4117d8e]*/ { - asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); - ENSURE_FUTURE_ALIVE(state, fut) - if (fut->fut_log_tb) { + asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); + ENSURE_FUTURE_ALIVE(state, self) + if (self->fut_log_tb) { Py_RETURN_TRUE; } else { @@ -1260,14 +1428,21 @@ FutureObj_get_log_traceback(FutureObj *fut, void *Py_UNUSED(ignored)) } } +/*[clinic input] +@critical_section +@setter +_asyncio.Future._log_traceback +[clinic start generated code]*/ + static int -FutureObj_set_log_traceback(FutureObj *fut, PyObject *val, void *Py_UNUSED(ignored)) +_asyncio_Future__log_traceback_set_impl(FutureObj *self, PyObject *value) +/*[clinic end generated code: output=9ce8e19504f42f54 input=30ac8217754b08c2]*/ { - if (val == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); return -1; } - int is_true = PyObject_IsTrue(val); + int is_true = PyObject_IsTrue(value); if (is_true < 0) { return -1; } @@ -1276,133 +1451,185 @@ FutureObj_set_log_traceback(FutureObj *fut, PyObject *val, void *Py_UNUSED(ignor "_log_traceback can only be set to False"); return -1; } - fut->fut_log_tb = is_true; + self->fut_log_tb = is_true; return 0; } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._loop +[clinic start generated code]*/ static PyObject * -FutureObj_get_loop(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__loop_get_impl(FutureObj *self) +/*[clinic end generated code: output=5ba31563eecfeedf input=0337130bc5781670]*/ { - if (!future_is_alive(fut)) { + if (!future_is_alive(self)) { Py_RETURN_NONE; } - return Py_NewRef(fut->fut_loop); + return Py_NewRef(self->fut_loop); } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._callbacks +[clinic start generated code]*/ + static PyObject * -FutureObj_get_callbacks(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__callbacks_get_impl(FutureObj *self) +/*[clinic end generated code: output=b40d360505fcc583 input=7a466649530c01bb]*/ { - asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); - Py_ssize_t i; - - ENSURE_FUTURE_ALIVE(state, fut) - - if (fut->fut_callback0 == NULL) { - if (fut->fut_callbacks == NULL) { - Py_RETURN_NONE; - } + asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); + ENSURE_FUTURE_ALIVE(state, self) - return Py_NewRef(fut->fut_callbacks); + Py_ssize_t len = 0; + if (self->fut_callback0 != NULL) { + len++; } - - Py_ssize_t len = 1; - if (fut->fut_callbacks != NULL) { - len += PyList_GET_SIZE(fut->fut_callbacks); + if (self->fut_callbacks != NULL) { + len += PyList_GET_SIZE(self->fut_callbacks); } - - PyObject *new_list = PyList_New(len); - if (new_list == NULL) { - return NULL; + if (len == 0) { + Py_RETURN_NONE; } - PyObject *tup0 = PyTuple_New(2); - if (tup0 == NULL) { - Py_DECREF(new_list); + PyObject *callbacks = PyList_New(len); + if (callbacks == NULL) { return NULL; } - Py_INCREF(fut->fut_callback0); - PyTuple_SET_ITEM(tup0, 0, fut->fut_callback0); - assert(fut->fut_context0 != NULL); - Py_INCREF(fut->fut_context0); - PyTuple_SET_ITEM(tup0, 1, (PyObject *)fut->fut_context0); - - PyList_SET_ITEM(new_list, 0, tup0); + Py_ssize_t i = 0; + if (self->fut_callback0 != NULL) { + PyObject *tup0 = PyTuple_New(2); + if (tup0 == NULL) { + Py_DECREF(callbacks); + return NULL; + } + PyTuple_SET_ITEM(tup0, 0, Py_NewRef(self->fut_callback0)); + assert(self->fut_context0 != NULL); + PyTuple_SET_ITEM(tup0, 1, Py_NewRef(self->fut_context0)); + PyList_SET_ITEM(callbacks, i, tup0); + i++; + } - if (fut->fut_callbacks != NULL) { - for (i = 0; i < PyList_GET_SIZE(fut->fut_callbacks); i++) { - PyObject *cb = PyList_GET_ITEM(fut->fut_callbacks, i); + if (self->fut_callbacks != NULL) { + for (Py_ssize_t j = 0; j < PyList_GET_SIZE(self->fut_callbacks); j++) { + PyObject *cb = PyList_GET_ITEM(self->fut_callbacks, j); Py_INCREF(cb); - PyList_SET_ITEM(new_list, i + 1, cb); + PyList_SET_ITEM(callbacks, i, cb); + i++; } } - return new_list; + return callbacks; } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._result +[clinic start generated code]*/ + static PyObject * -FutureObj_get_result(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__result_get_impl(FutureObj *self) +/*[clinic end generated code: output=6877e8ce97333873 input=624f8e28e67f2636]*/ + { - asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); - ENSURE_FUTURE_ALIVE(state, fut) - if (fut->fut_result == NULL) { + asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); + ENSURE_FUTURE_ALIVE(state, self) + if (self->fut_result == NULL) { Py_RETURN_NONE; } - return Py_NewRef(fut->fut_result); + return Py_NewRef(self->fut_result); } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._exception +[clinic start generated code]*/ + static PyObject * -FutureObj_get_exception(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__exception_get_impl(FutureObj *self) +/*[clinic end generated code: output=32f2c93b9e021a9b input=1828a1fcac929710]*/ { - asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); - ENSURE_FUTURE_ALIVE(state, fut) - if (fut->fut_exception == NULL) { + asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); + ENSURE_FUTURE_ALIVE(state, self) + if (self->fut_exception == NULL) { Py_RETURN_NONE; } - return Py_NewRef(fut->fut_exception); + return Py_NewRef(self->fut_exception); } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._source_traceback +[clinic start generated code]*/ + static PyObject * -FutureObj_get_source_traceback(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__source_traceback_get_impl(FutureObj *self) +/*[clinic end generated code: output=d4f12b09af22f61b input=3c831fbde5da90d0]*/ { - if (!future_is_alive(fut) || fut->fut_source_tb == NULL) { + if (!future_is_alive(self) || self->fut_source_tb == NULL) { Py_RETURN_NONE; } - return Py_NewRef(fut->fut_source_tb); + return Py_NewRef(self->fut_source_tb); } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._cancel_message +[clinic start generated code]*/ + static PyObject * -FutureObj_get_cancel_message(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__cancel_message_get_impl(FutureObj *self) +/*[clinic end generated code: output=52ef6444f92cedac input=54c12c67082e4eea]*/ { - if (fut->fut_cancel_msg == NULL) { + if (self->fut_cancel_msg == NULL) { Py_RETURN_NONE; } - return Py_NewRef(fut->fut_cancel_msg); + return Py_NewRef(self->fut_cancel_msg); } +/*[clinic input] +@critical_section +@setter +_asyncio.Future._cancel_message +[clinic start generated code]*/ + static int -FutureObj_set_cancel_message(FutureObj *fut, PyObject *msg, - void *Py_UNUSED(ignored)) +_asyncio_Future__cancel_message_set_impl(FutureObj *self, PyObject *value) +/*[clinic end generated code: output=0854b2f77bff2209 input=f461d17f2d891fad]*/ { - if (msg == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); return -1; } - Py_INCREF(msg); - Py_XSETREF(fut->fut_cancel_msg, msg); + Py_INCREF(value); + Py_XSETREF(self->fut_cancel_msg, value); return 0; } +/*[clinic input] +@critical_section +@getter +_asyncio.Future._state +[clinic start generated code]*/ + static PyObject * -FutureObj_get_state(FutureObj *fut, void *Py_UNUSED(ignored)) +_asyncio_Future__state_get_impl(FutureObj *self) +/*[clinic end generated code: output=622f560a3fa69c63 input=7c5ad023a93423ff]*/ { - asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); + asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); PyObject *ret = NULL; - ENSURE_FUTURE_ALIVE(state, fut) + ENSURE_FUTURE_ALIVE(state, self) - switch (fut->fut_state) { + switch (self->fut_state) { case STATE_PENDING: ret = &_Py_ID(PENDING); break; @@ -1428,6 +1655,7 @@ FutureObj_repr(FutureObj *fut) } /*[clinic input] +@critical_section _asyncio.Future._make_cancelled_error Create the CancelledError to raise if the Future is cancelled. @@ -1438,7 +1666,7 @@ it erases the context exception value. static PyObject * _asyncio_Future__make_cancelled_error_impl(FutureObj *self) -/*[clinic end generated code: output=a5df276f6c1213de input=ac6effe4ba795ecc]*/ +/*[clinic end generated code: output=a5df276f6c1213de input=ccb90df8c3c18bcd]*/ { asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); return create_cancelled_error(state, self); @@ -1519,23 +1747,17 @@ static PyMethodDef FutureType_methods[] = { {NULL, NULL} /* Sentinel */ }; -#define FUTURE_COMMON_GETSETLIST \ - {"_state", (getter)FutureObj_get_state, NULL, NULL}, \ - {"_asyncio_future_blocking", (getter)FutureObj_get_blocking, \ - (setter)FutureObj_set_blocking, NULL}, \ - {"_loop", (getter)FutureObj_get_loop, NULL, NULL}, \ - {"_callbacks", (getter)FutureObj_get_callbacks, NULL, NULL}, \ - {"_result", (getter)FutureObj_get_result, NULL, NULL}, \ - {"_exception", (getter)FutureObj_get_exception, NULL, NULL}, \ - {"_log_traceback", (getter)FutureObj_get_log_traceback, \ - (setter)FutureObj_set_log_traceback, NULL}, \ - {"_source_traceback", (getter)FutureObj_get_source_traceback, \ - NULL, NULL}, \ - {"_cancel_message", (getter)FutureObj_get_cancel_message, \ - (setter)FutureObj_set_cancel_message, NULL}, - static PyGetSetDef FutureType_getsetlist[] = { - FUTURE_COMMON_GETSETLIST + _ASYNCIO_FUTURE__STATE_GETSETDEF + _ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF + _ASYNCIO_FUTURE__LOOP_GETSETDEF + _ASYNCIO_FUTURE__CALLBACKS_GETSETDEF + _ASYNCIO_FUTURE__RESULT_GETSETDEF + _ASYNCIO_FUTURE__EXCEPTION_GETSETDEF + _ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF + _ASYNCIO_FUTURE__SOURCE_TRACEBACK_GETSETDEF + _ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF + _ASYNCIO_FUTURE__ASYNCIO_AWAITED_BY_GETSETDEF {NULL} /* Sentinel */ }; @@ -1601,35 +1823,26 @@ static void FutureIter_dealloc(futureiterobject *it) { PyTypeObject *tp = Py_TYPE(it); - asyncio_state *state = get_asyncio_state_by_def((PyObject *)it); + + assert(_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)); + PyObject_GC_UnTrack(it); tp->tp_clear((PyObject *)it); - if (state->fi_freelist_len < FI_FREELIST_MAXLEN) { - state->fi_freelist_len++; - it->future = (FutureObj*) state->fi_freelist; - state->fi_freelist = it; - } - else { + if (!_Py_FREELIST_PUSH(futureiters, it, Py_futureiters_MAXFREELIST)) { PyObject_GC_Del(it); Py_DECREF(tp); } } static PySendResult -FutureIter_am_send(futureiterobject *it, - PyObject *Py_UNUSED(arg), - PyObject **result) +FutureIter_am_send_lock_held(futureiterobject *it, PyObject **result) { - /* arg is unused, see the comment on FutureIter_send for clarification */ - PyObject *res; FutureObj *fut = it->future; + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(fut); *result = NULL; - if (fut == NULL) { - return PYGEN_ERROR; - } if (fut->fut_state == STATE_PENDING) { if (!fut->fut_blocking) { @@ -1642,18 +1855,29 @@ FutureIter_am_send(futureiterobject *it, return PYGEN_ERROR; } - it->future = NULL; res = _asyncio_Future_result_impl(fut); if (res != NULL) { - Py_DECREF(fut); *result = res; return PYGEN_RETURN; } - Py_DECREF(fut); return PYGEN_ERROR; } +static PySendResult +FutureIter_am_send(futureiterobject *it, + PyObject *Py_UNUSED(arg), + PyObject **result) +{ + /* arg is unused, see the comment on FutureIter_send for clarification */ + PySendResult res; + Py_BEGIN_CRITICAL_SECTION(it->future); + res = FutureIter_am_send_lock_held(it, result); + Py_END_CRITICAL_SECTION(); + return res; +} + + static PyObject * FutureIter_iternext(futureiterobject *it) { @@ -1814,14 +2038,8 @@ future_new_iter(PyObject *fut) asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); ENSURE_FUTURE_ALIVE(state, fut) - if (state->fi_freelist_len) { - state->fi_freelist_len--; - it = state->fi_freelist; - state->fi_freelist = (futureiterobject*) it->future; - it->future = NULL; - _Py_NewReference((PyObject*) it); - } - else { + it = _Py_FREELIST_POP(futureiterobject, futureiters); + if (it == NULL) { it = PyObject_GC_New(futureiterobject, state->FutureIterType); if (it == NULL) { return NULL; @@ -1846,6 +2064,8 @@ static int task_call_step_soon(asyncio_state *state, TaskObj *, PyObject *); static PyObject * task_wakeup(TaskObj *, PyObject *); static PyObject * task_step(asyncio_state *, TaskObj *, PyObject *); static int task_eager_start(asyncio_state *state, TaskObj *task); +static inline void clear_ts_asyncio_running_task(PyObject *loop); +static inline void set_ts_asyncio_running_task(PyObject *loop, PyObject *task); /* ----- Task._step wrapper */ @@ -1880,7 +2100,11 @@ TaskStepMethWrapper_call(TaskStepMethWrapper *o, return NULL; } asyncio_state *state = get_asyncio_state_by_def((PyObject *)o); - return task_step(state, o->sw_task, o->sw_arg); + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(o->sw_task); + res = task_step(state, o->sw_task, o->sw_arg); + Py_END_CRITICAL_SECTION(); + return res; } static int @@ -1953,16 +2177,19 @@ static PyMethodDef TaskWakeupDef = { /* ----- Task introspection helpers */ -static int -register_task(asyncio_state *state, PyObject *task) +static void +register_task(asyncio_state *state, TaskObj *task) { - PyObject *res = PyObject_CallMethodOneArg(state->scheduled_tasks, - &_Py_ID(add), task); - if (res == NULL) { - return -1; - } - Py_DECREF(res); - return 0; + ASYNCIO_STATE_LOCK(state); + assert(Task_Check(state, task)); + if (task->task_node.next != NULL) { + // already registered + assert(task->task_node.prev != NULL); + goto exit; + } + llist_insert_tail(&state->asyncio_tasks_head, &task->task_node); +exit: + ASYNCIO_STATE_UNLOCK(state); } static int @@ -1971,16 +2198,19 @@ register_eager_task(asyncio_state *state, PyObject *task) return PySet_Add(state->eager_tasks, task); } -static int -unregister_task(asyncio_state *state, PyObject *task) +static void +unregister_task(asyncio_state *state, TaskObj *task) { - PyObject *res = PyObject_CallMethodOneArg(state->scheduled_tasks, - &_Py_ID(discard), task); - if (res == NULL) { - return -1; - } - Py_DECREF(res); - return 0; + ASYNCIO_STATE_LOCK(state); + assert(Task_Check(state, task)); + if (task->task_node.next == NULL) { + // not registered + assert(task->task_node.prev == NULL); + goto exit; + } + llist_remove(&task->task_node); +exit: + ASYNCIO_STATE_UNLOCK(state); } static int @@ -1993,14 +2223,11 @@ static int enter_task(asyncio_state *state, PyObject *loop, PyObject *task) { PyObject *item; - Py_hash_t hash; - hash = PyObject_Hash(loop); - if (hash == -1) { + int res = PyDict_SetDefaultRef(state->current_tasks, loop, task, &item); + if (res < 0) { return -1; } - item = _PyDict_GetItem_KnownHash(state->current_tasks, loop, hash); - if (item != NULL) { - Py_INCREF(item); + else if (res == 1) { PyErr_Format( PyExc_RuntimeError, "Cannot enter into task %R while another " \ @@ -2009,72 +2236,143 @@ enter_task(asyncio_state *state, PyObject *loop, PyObject *task) Py_DECREF(item); return -1; } - if (PyErr_Occurred()) { - return -1; - } - return _PyDict_SetItem_KnownHash(state->current_tasks, loop, task, hash); + + assert(task == item); + Py_CLEAR(item); + set_ts_asyncio_running_task(loop, task); + return 0; } +static int +err_leave_task(PyObject *item, PyObject *task) +{ + PyErr_Format( + PyExc_RuntimeError, + "Leaving task %R does not match the current task %R.", + task, item); + return -1; +} static int -leave_task(asyncio_state *state, PyObject *loop, PyObject *task) -/*[clinic end generated code: output=0ebf6db4b858fb41 input=51296a46313d1ad8]*/ +leave_task_predicate(PyObject *item, void *task) { - PyObject *item; - Py_hash_t hash; - hash = PyObject_Hash(loop); - if (hash == -1) { - return -1; - } - item = _PyDict_GetItem_KnownHash(state->current_tasks, loop, hash); if (item != task) { - if (item == NULL) { - /* Not entered, replace with None */ - item = Py_None; - } - PyErr_Format( - PyExc_RuntimeError, - "Leaving task %R does not match the current task %R.", - task, item, NULL); - return -1; + return err_leave_task(item, (PyObject *)task); + } + return 1; +} + +static int +leave_task(asyncio_state *state, PyObject *loop, PyObject *task) +{ + int res = _PyDict_DelItemIf(state->current_tasks, loop, + leave_task_predicate, task); + if (res == 0) { + // task was not found + return err_leave_task(Py_None, task); } - return _PyDict_DelItem_KnownHash(state->current_tasks, loop, hash); + clear_ts_asyncio_running_task(loop); + return res; } static PyObject * -swap_current_task(asyncio_state *state, PyObject *loop, PyObject *task) +swap_current_task_lock_held(PyDictObject *current_tasks, PyObject *loop, + Py_hash_t hash, PyObject *task) { PyObject *prev_task; - Py_hash_t hash; - hash = PyObject_Hash(loop); - if (hash == -1) { + if (_PyDict_GetItemRef_KnownHash_LockHeld(current_tasks, loop, hash, &prev_task) < 0) { + return NULL; + } + if (_PyDict_SetItem_KnownHash_LockHeld(current_tasks, loop, task, hash) < 0) { + Py_XDECREF(prev_task); return NULL; } - - prev_task = _PyDict_GetItem_KnownHash(state->current_tasks, loop, hash); if (prev_task == NULL) { - if (PyErr_Occurred()) { - return NULL; - } - prev_task = Py_None; + Py_RETURN_NONE; } - Py_INCREF(prev_task); + return prev_task; +} + +static PyObject * +swap_current_task(asyncio_state *state, PyObject *loop, PyObject *task) +{ + PyObject *prev_task; + clear_ts_asyncio_running_task(loop); if (task == Py_None) { - if (_PyDict_DelItem_KnownHash(state->current_tasks, loop, hash) == -1) { - goto error; + if (PyDict_Pop(state->current_tasks, loop, &prev_task) < 0) { + return NULL; } - } else { - if (_PyDict_SetItem_KnownHash(state->current_tasks, loop, task, hash) == -1) { - goto error; + if (prev_task == NULL) { + Py_RETURN_NONE; } + return prev_task; + } + + Py_hash_t hash = PyObject_Hash(loop); + if (hash == -1) { + return NULL; } + PyDictObject *current_tasks = (PyDictObject *)state->current_tasks; + Py_BEGIN_CRITICAL_SECTION(current_tasks); + prev_task = swap_current_task_lock_held(current_tasks, loop, hash, task); + Py_END_CRITICAL_SECTION(); + set_ts_asyncio_running_task(loop, task); return prev_task; +} -error: - Py_DECREF(prev_task); - return NULL; +static inline void +set_ts_asyncio_running_task(PyObject *loop, PyObject *task) +{ + // We want to enable debuggers and profilers to be able to quickly + // introspect the asyncio running state from another process. + // When we do that, we need to essentially traverse the address space + // of a Python process and understand what every Python thread in it is + // currently doing, mainly: + // + // * current frame + // * current asyncio task + // + // A naive solution would be to require profilers and debuggers to + // find the current task in the "_asynciomodule" module state, but + // unfortunately that would require a lot of complicated remote + // memory reads and logic, as Python's dict is a notoriously complex + // and ever-changing data structure. + // + // So the easier solution is to put a strong reference to the currently + // running `asyncio.Task` on the current thread state (the current loop + // is also stored there.) + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + if (ts->asyncio_running_loop == loop) { + // Protect from a situation when someone calls this method + // from another thread. This shouldn't ever happen though, + // as `enter_task` and `leave_task` can either be called by: + // + // - `asyncio.Task` itself, in `Task.__step()`. That method + // can only be called by the event loop itself. + // + // - third-party Task "from scratch" implementations, that + // our `capture_call_graph` API doesn't support anyway. + // + // That said, we still want to make sure we don't end up in + // a broken state, so we check that we're in the correct thread + // by comparing the *loop* argument to the event loop running + // in the current thread. If they match we know we're in the + // right thread, as asyncio event loops don't change threads. + assert(ts->asyncio_running_task == NULL); + ts->asyncio_running_task = Py_NewRef(task); + } +} + +static inline void +clear_ts_asyncio_running_task(PyObject *loop) +{ + // See comment in set_ts_asyncio_running_task() for details. + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + if (ts->asyncio_running_loop == NULL || ts->asyncio_running_loop == loop) { + Py_CLEAR(ts->asyncio_running_task); + } } /* ----- Task */ @@ -2101,6 +2399,7 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, if (future_init((FutureObj*)self, loop)) { return -1; } + self->task_is_task = 1; asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); int is_coro = is_coroutine(state, coro); @@ -2121,21 +2420,25 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, return -1; } } else { - self->task_context = Py_NewRef(context); + Py_XSETREF(self->task_context, Py_NewRef(context)); } Py_CLEAR(self->task_fut_waiter); self->task_must_cancel = 0; self->task_log_destroy_pending = 1; self->task_num_cancels_requested = 0; - Py_INCREF(coro); - Py_XSETREF(self->task_coro, coro); + set_task_coro(self, coro); if (name == Py_None) { // optimization: defer task name formatting // store the task counter as PyLong in the name // for deferred formatting in get_name - name = PyLong_FromUnsignedLongLong(++state->task_name_counter); +#ifdef Py_GIL_DISABLED + unsigned long long counter = _Py_atomic_add_uint64(&state->task_name_counter, 1) + 1; +#else + unsigned long long counter = ++state->task_name_counter; +#endif + name = PyLong_FromUnsignedLongLong(counter); } else if (!PyUnicode_CheckExact(name)) { name = PyObject_Str(name); } else { @@ -2164,15 +2467,21 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, if (task_call_step_soon(state, self, NULL)) { return -1; } - return register_task(state, (PyObject*)self); +#ifdef Py_GIL_DISABLED + // This is required so that _Py_TryIncref(self) + // works correctly in non-owning threads. + _PyObject_SetMaybeWeakref((PyObject *)self); +#endif + register_task(state, self); + return 0; } static int TaskObj_clear(TaskObj *task) { (void)FutureObj_clear((FutureObj*) task); + clear_task_coro(task); Py_CLEAR(task->task_context); - Py_CLEAR(task->task_coro); Py_CLEAR(task->task_name); Py_CLEAR(task->task_fut_waiter); return 0; @@ -2197,14 +2506,22 @@ TaskObj_traverse(TaskObj *task, visitproc visit, void *arg) Py_VISIT(fut->fut_source_tb); Py_VISIT(fut->fut_cancel_msg); Py_VISIT(fut->fut_cancelled_exc); + Py_VISIT(fut->fut_awaited_by); PyObject_VisitManagedDict((PyObject *)fut, visit, arg); return 0; } +/*[clinic input] +@critical_section +@getter +_asyncio.Task._log_destroy_pending +[clinic start generated code]*/ + static PyObject * -TaskObj_get_log_destroy_pending(TaskObj *task, void *Py_UNUSED(ignored)) +_asyncio_Task__log_destroy_pending_get_impl(TaskObj *self) +/*[clinic end generated code: output=e6c2a47d029ac93b input=17127298cd4c720b]*/ { - if (task->task_log_destroy_pending) { + if (self->task_log_destroy_pending) { Py_RETURN_TRUE; } else { @@ -2212,25 +2529,40 @@ TaskObj_get_log_destroy_pending(TaskObj *task, void *Py_UNUSED(ignored)) } } +/*[clinic input] +@critical_section +@setter +_asyncio.Task._log_destroy_pending +[clinic start generated code]*/ + static int -TaskObj_set_log_destroy_pending(TaskObj *task, PyObject *val, void *Py_UNUSED(ignored)) +_asyncio_Task__log_destroy_pending_set_impl(TaskObj *self, PyObject *value) +/*[clinic end generated code: output=7ebc030bb92ec5ce input=49b759c97d1216a4]*/ { - if (val == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); return -1; } - int is_true = PyObject_IsTrue(val); + int is_true = PyObject_IsTrue(value); if (is_true < 0) { return -1; } - task->task_log_destroy_pending = is_true; + self->task_log_destroy_pending = is_true; return 0; } + +/*[clinic input] +@critical_section +@getter +_asyncio.Task._must_cancel +[clinic start generated code]*/ + static PyObject * -TaskObj_get_must_cancel(TaskObj *task, void *Py_UNUSED(ignored)) +_asyncio_Task__must_cancel_get_impl(TaskObj *self) +/*[clinic end generated code: output=70e79b900996c363 input=2d04529fb23feedf]*/ { - if (task->task_must_cancel) { + if (self->task_must_cancel) { Py_RETURN_TRUE; } else { @@ -2238,21 +2570,36 @@ TaskObj_get_must_cancel(TaskObj *task, void *Py_UNUSED(ignored)) } } +/*[clinic input] +@critical_section +@getter +_asyncio.Task._coro +[clinic start generated code]*/ + static PyObject * -TaskObj_get_coro(TaskObj *task, void *Py_UNUSED(ignored)) +_asyncio_Task__coro_get_impl(TaskObj *self) +/*[clinic end generated code: output=a2726012ab5fd531 input=323c31a272020624]*/ { - if (task->task_coro) { - return Py_NewRef(task->task_coro); + if (self->task_coro) { + return Py_NewRef(self->task_coro); } Py_RETURN_NONE; } + +/*[clinic input] +@critical_section +@getter +_asyncio.Task._fut_waiter +[clinic start generated code]*/ + static PyObject * -TaskObj_get_fut_waiter(TaskObj *task, void *Py_UNUSED(ignored)) +_asyncio_Task__fut_waiter_get_impl(TaskObj *self) +/*[clinic end generated code: output=c4f966b847fefcdf input=4d1005d725e72db7]*/ { - if (task->task_fut_waiter) { - return Py_NewRef(task->task_fut_waiter); + if (self->task_fut_waiter) { + return Py_NewRef(self->task_fut_waiter); } Py_RETURN_NONE; @@ -2268,6 +2615,7 @@ TaskObj_repr(TaskObj *task) /*[clinic input] +@critical_section _asyncio.Task._make_cancelled_error Create the CancelledError to raise if the Task is cancelled. @@ -2278,7 +2626,7 @@ it erases the context exception value. static PyObject * _asyncio_Task__make_cancelled_error_impl(TaskObj *self) -/*[clinic end generated code: output=55a819e8b4276fab input=52c0e32de8e2f840]*/ +/*[clinic end generated code: output=55a819e8b4276fab input=2d3213be0cb02390]*/ { FutureObj *fut = (FutureObj*)self; return _asyncio_Future__make_cancelled_error_impl(fut); @@ -2286,6 +2634,7 @@ _asyncio_Task__make_cancelled_error_impl(TaskObj *self) /*[clinic input] +@critical_section _asyncio.Task.cancel msg: object = None @@ -2314,7 +2663,7 @@ This also increases the task's count of cancellation requests. static PyObject * _asyncio_Task_cancel_impl(TaskObj *self, PyObject *msg) -/*[clinic end generated code: output=c66b60d41c74f9f1 input=7bb51bf25974c783]*/ +/*[clinic end generated code: output=c66b60d41c74f9f1 input=6125d45b9a6a5abd]*/ { self->task_log_tb = 0; @@ -2359,6 +2708,7 @@ _asyncio_Task_cancel_impl(TaskObj *self, PyObject *msg) } /*[clinic input] +@critical_section _asyncio.Task.cancelling Return the count of the task's cancellation requests. @@ -2369,13 +2719,14 @@ and may be decremented using .uncancel(). static PyObject * _asyncio_Task_cancelling_impl(TaskObj *self) -/*[clinic end generated code: output=803b3af96f917d7e input=b625224d310cbb17]*/ +/*[clinic end generated code: output=803b3af96f917d7e input=5ef89b1b38f080ee]*/ /*[clinic end generated code]*/ { return PyLong_FromLong(self->task_num_cancels_requested); } /*[clinic input] +@critical_section _asyncio.Task.uncancel Decrement the task's count of cancellation requests. @@ -2388,11 +2739,14 @@ Returns the remaining number of cancellation requests. static PyObject * _asyncio_Task_uncancel_impl(TaskObj *self) -/*[clinic end generated code: output=58184d236a817d3c input=68f81a4b90b46be2]*/ +/*[clinic end generated code: output=58184d236a817d3c input=cb3220b0e5afd61d]*/ /*[clinic end generated code]*/ { if (self->task_num_cancels_requested > 0) { self->task_num_cancels_requested -= 1; + if (self->task_num_cancels_requested == 0) { + self->task_must_cancel = 0; + } } return PyLong_FromLong(self->task_num_cancels_requested); } @@ -2499,14 +2853,19 @@ _asyncio_Task_set_exception(TaskObj *self, PyObject *exception) } /*[clinic input] +@critical_section _asyncio.Task.get_coro [clinic start generated code]*/ static PyObject * _asyncio_Task_get_coro_impl(TaskObj *self) -/*[clinic end generated code: output=bcac27c8cc6c8073 input=d2e8606c42a7b403]*/ +/*[clinic end generated code: output=bcac27c8cc6c8073 input=a47f81427e39fe0c]*/ { - return Py_NewRef(self->task_coro); + if (self->task_coro) { + return Py_NewRef(self->task_coro); + } + + Py_RETURN_NONE; } /*[clinic input] @@ -2521,12 +2880,13 @@ _asyncio_Task_get_context_impl(TaskObj *self) } /*[clinic input] +@critical_section _asyncio.Task.get_name [clinic start generated code]*/ static PyObject * _asyncio_Task_get_name_impl(TaskObj *self) -/*[clinic end generated code: output=0ecf1570c3b37a8f input=a4a6595d12f4f0f8]*/ +/*[clinic end generated code: output=0ecf1570c3b37a8f input=92a8f30c85034249]*/ { if (self->task_name) { if (PyLong_CheckExact(self->task_name)) { @@ -2543,6 +2903,7 @@ _asyncio_Task_get_name_impl(TaskObj *self) } /*[clinic input] +@critical_section _asyncio.Task.set_name value: object @@ -2550,8 +2911,8 @@ _asyncio.Task.set_name [clinic start generated code]*/ static PyObject * -_asyncio_Task_set_name(TaskObj *self, PyObject *value) -/*[clinic end generated code: output=138a8d51e32057d6 input=a8359b6e65f8fd31]*/ +_asyncio_Task_set_name_impl(TaskObj *self, PyObject *value) +/*[clinic end generated code: output=f88ff4c0d64a9a6f input=e8d400ad64bad799]*/ { if (!PyUnicode_CheckExact(value)) { value = PyObject_Str(value); @@ -2569,6 +2930,15 @@ _asyncio_Task_set_name(TaskObj *self, PyObject *value) static void TaskObj_finalize(TaskObj *task) { + asyncio_state *state = get_asyncio_state_by_def((PyObject *)task); + // Unregister the task from the linked list of tasks. + // Since task is a native task, we directly call the + // unregister_task function. Third party event loops + // should use the asyncio._unregister_task function. + // See https://docs.python.org/3/library/asyncio-extending.html#task-lifetime-support + + unregister_task(state, task); + PyObject *context; PyObject *message = NULL; PyObject *func; @@ -2653,12 +3023,10 @@ static PyMethodDef TaskType_methods[] = { }; static PyGetSetDef TaskType_getsetlist[] = { - FUTURE_COMMON_GETSETLIST - {"_log_destroy_pending", (getter)TaskObj_get_log_destroy_pending, - (setter)TaskObj_set_log_destroy_pending, NULL}, - {"_must_cancel", (getter)TaskObj_get_must_cancel, NULL, NULL}, - {"_coro", (getter)TaskObj_get_coro, NULL, NULL}, - {"_fut_waiter", (getter)TaskObj_get_fut_waiter, NULL, NULL}, + _ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF + _ASYNCIO_TASK__MUST_CANCEL_GETSETDEF + _ASYNCIO_TASK__CORO_GETSETDEF + _ASYNCIO_TASK__FUT_WAITER_GETSETDEF {NULL} /* Sentinel */ }; @@ -2717,7 +3085,11 @@ task_call_step_soon(asyncio_state *state, TaskObj *task, PyObject *arg) return -1; } - int ret = call_soon(state, task->task_loop, cb, NULL, task->task_context); + // Beware: An evil call_soon could alter task_context. + // See: https://github.com/python/cpython/issues/126080. + PyObject *task_context = Py_NewRef(task->task_context); + int ret = call_soon(state, task->task_loop, cb, NULL, task_context); + Py_DECREF(task_context); Py_DECREF(cb); return ret; } @@ -2769,6 +3141,8 @@ gen_status_from_result(PyObject **result) static PyObject * task_step_impl(asyncio_state *state, TaskObj *task, PyObject *exc) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(task); + int clear_exc = 0; PyObject *result = NULL; PyObject *coro; @@ -2776,7 +3150,7 @@ task_step_impl(asyncio_state *state, TaskObj *task, PyObject *exc) if (task->task_state != STATE_PENDING) { PyErr_Format(state->asyncio_InvalidStateError, - "_step(): already done: %R %R", + "__step(): already done: %R %R", task, exc ? exc : Py_None); goto fail; @@ -2898,6 +3272,8 @@ task_step_impl(asyncio_state *state, TaskObj *task, PyObject *exc) static PyObject * task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *result) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(task); + int res; PyObject *o; @@ -2921,6 +3297,10 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu goto yield_insteadof_yf; } + if (future_awaited_by_add(state, result, (PyObject *)task)) { + goto fail; + } + fut->fut_blocking = 0; /* result.add_done_callback(task._wakeup) */ @@ -2928,8 +3308,10 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu if (wrapper == NULL) { goto fail; } + Py_BEGIN_CRITICAL_SECTION(result); tmp = future_add_done_callback(state, (FutureObj*)result, wrapper, task->task_context); + Py_END_CRITICAL_SECTION(); Py_DECREF(wrapper); if (tmp == NULL) { goto fail; @@ -2942,8 +3324,17 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu if (task->task_must_cancel) { PyObject *r; int is_true; + + // Beware: An evil `__getattribute__` could + // prematurely delete task->task_cancel_msg before the + // task is cancelled, thereby causing a UAF crash. + // + // See https://github.com/python/cpython/issues/126138 + PyObject *task_cancel_msg = Py_NewRef(task->task_cancel_msg); r = PyObject_CallMethodOneArg(result, &_Py_ID(cancel), - task->task_cancel_msg); + task_cancel_msg); + Py_DECREF(task_cancel_msg); + if (r == NULL) { return NULL; } @@ -2999,6 +3390,10 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu goto yield_insteadof_yf; } + if (future_awaited_by_add(state, result, (PyObject *)task)) { + goto fail; + } + /* result._asyncio_future_blocking = False */ if (PyObject_SetAttr( result, &_Py_ID(_asyncio_future_blocking), Py_False) == -1) { @@ -3035,8 +3430,17 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu if (task->task_must_cancel) { PyObject *r; int is_true; + + // Beware: An evil `__getattribute__` could + // prematurely delete task->task_cancel_msg before the + // task is cancelled, thereby causing a UAF crash. + // + // See https://github.com/python/cpython/issues/126138 + PyObject *task_cancel_msg = Py_NewRef(task->task_cancel_msg); r = PyObject_CallMethodOneArg(result, &_Py_ID(cancel), - task->task_cancel_msg); + task_cancel_msg); + Py_DECREF(task_cancel_msg); + if (r == NULL) { return NULL; } @@ -3153,7 +3557,10 @@ task_eager_start(asyncio_state *state, TaskObj *task) int retval = 0; - PyObject *stepres = task_step_impl(state, task, NULL); + PyObject *stepres; + Py_BEGIN_CRITICAL_SECTION(task); + stepres = task_step_impl(state, task, NULL); + Py_END_CRITICAL_SECTION(); if (stepres == NULL) { PyObject *exc = PyErr_GetRaisedException(); _PyErr_ChainExceptions1(exc); @@ -3180,28 +3587,35 @@ task_eager_start(asyncio_state *state, TaskObj *task) } if (task->task_state == STATE_PENDING) { - if (register_task(state, (PyObject *)task) == -1) { - retval = -1; - } + register_task(state, task); } else { // This seems to really help performance on pyperformance benchmarks - Py_CLEAR(task->task_coro); + clear_task_coro(task); } return retval; } static PyObject * -task_wakeup(TaskObj *task, PyObject *o) +task_wakeup_lock_held(TaskObj *task, PyObject *o) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(task); + PyObject *result; assert(o); asyncio_state *state = get_asyncio_state_by_def((PyObject *)task); + + if (future_awaited_by_discard(state, o, (PyObject *)task)) { + return NULL; + } + if (Future_CheckExact(state, o) || Task_CheckExact(state, o)) { PyObject *fut_result = NULL; - int res = future_get_result(state, (FutureObj*)o, &fut_result); - + int res; + Py_BEGIN_CRITICAL_SECTION(o); + res = future_get_result(state, (FutureObj*)o, &fut_result); + Py_END_CRITICAL_SECTION(); switch(res) { case -1: assert(fut_result == NULL); @@ -3235,6 +3649,16 @@ task_wakeup(TaskObj *task, PyObject *o) return result; } +static PyObject * +task_wakeup(TaskObj *task, PyObject *o) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(task); + res = task_wakeup_lock_held(task, o); + Py_END_CRITICAL_SECTION(); + return res; +} + /*********************** Functions **************************/ @@ -3253,11 +3677,8 @@ static PyObject * _asyncio__get_running_loop_impl(PyObject *module) /*[clinic end generated code: output=b4390af721411a0a input=0a21627e25a4bd43]*/ { - PyObject *loop; - asyncio_state *state = get_asyncio_state(module); - if (get_running_loop(state, &loop)) { - return NULL; - } + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + PyObject *loop = Py_XNewRef(ts->asyncio_running_loop); if (loop == NULL) { /* There's no currently running event loop */ Py_RETURN_NONE; @@ -3280,10 +3701,11 @@ static PyObject * _asyncio__set_running_loop(PyObject *module, PyObject *loop) /*[clinic end generated code: output=ae56bf7a28ca189a input=4c9720233d606604]*/ { - asyncio_state *state = get_asyncio_state(module); - if (set_running_loop(state, loop)) { - return NULL; + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + if (loop == Py_None) { + loop = NULL; } + Py_XSETREF(ts->asyncio_running_loop, Py_XNewRef(loop)); Py_RETURN_NONE; } @@ -3321,14 +3743,13 @@ _asyncio_get_running_loop_impl(PyObject *module) /*[clinic end generated code: output=c247b5f9e529530e input=2a3bf02ba39f173d]*/ { PyObject *loop; - asyncio_state *state = get_asyncio_state(module); - if (get_running_loop(state, &loop)) { - return NULL; - } + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + loop = Py_XNewRef(ts->asyncio_running_loop); if (loop == NULL) { /* There's no currently running event loop */ PyErr_SetString( PyExc_RuntimeError, "no running event loop"); + return NULL; } return loop; } @@ -3348,9 +3769,20 @@ _asyncio__register_task_impl(PyObject *module, PyObject *task) /*[clinic end generated code: output=8672dadd69a7d4e2 input=21075aaea14dfbad]*/ { asyncio_state *state = get_asyncio_state(module); - if (register_task(state, task) < 0) { + if (Task_Check(state, task)) { + // task is an asyncio.Task instance or subclass, use efficient + // linked-list implementation. + register_task(state, (TaskObj *)task); + Py_RETURN_NONE; + } + // As task does not inherit from asyncio.Task, fallback to less efficient + // weakset implementation. + PyObject *res = PyObject_CallMethodOneArg(state->non_asyncio_tasks, + &_Py_ID(add), task); + if (res == NULL) { return NULL; } + Py_DECREF(res); Py_RETURN_NONE; } @@ -3391,9 +3823,16 @@ _asyncio__unregister_task_impl(PyObject *module, PyObject *task) /*[clinic end generated code: output=6e5585706d568a46 input=28fb98c3975f7bdc]*/ { asyncio_state *state = get_asyncio_state(module); - if (unregister_task(state, task) < 0) { + if (Task_Check(state, task)) { + unregister_task(state, (TaskObj *)task); + Py_RETURN_NONE; + } + PyObject *res = PyObject_CallMethodOneArg(state->non_asyncio_tasks, + &_Py_ID(discard), task); + if (res == NULL) { return NULL; } + Py_DECREF(res); Py_RETURN_NONE; } @@ -3524,26 +3963,178 @@ _asyncio_current_task_impl(PyObject *module, PyObject *loop) } +static inline int +add_one_task(asyncio_state *state, PyObject *tasks, PyObject *task, PyObject *loop) +{ + PyObject *done = PyObject_CallMethodNoArgs(task, &_Py_ID(done)); + if (done == NULL) { + return -1; + } + if (Py_IsTrue(done)) { + return 0; + } + Py_DECREF(done); + PyObject *task_loop = get_future_loop(state, task); + if (task_loop == NULL) { + return -1; + } + if (task_loop == loop) { + if (PySet_Add(tasks, task) < 0) { + Py_DECREF(task_loop); + return -1; + } + } + Py_DECREF(task_loop); + return 0; +} + /*********************** Module **************************/ +/*[clinic input] +_asyncio.all_tasks + + loop: object = None + +Return a set of all tasks for the loop. -static void -module_free_freelists(asyncio_state *state) +[clinic start generated code]*/ + +static PyObject * +_asyncio_all_tasks_impl(PyObject *module, PyObject *loop) +/*[clinic end generated code: output=0e107cbb7f72aa7b input=43a1b423c2d95bfa]*/ +{ + asyncio_state *state = get_asyncio_state(module); + if (loop == Py_None) { + loop = _asyncio_get_running_loop_impl(module); + if (loop == NULL) { + return NULL; + } + } else { + Py_INCREF(loop); + } + // First add eager tasks to the list so that we don't miss + // any tasks which graduates from eager to non-eager + // We first add all the tasks to `tasks` list and then filter + // out the tasks which are done and return it as a set. + PyObject *tasks = PyList_New(0); + if (tasks == NULL) { + Py_DECREF(loop); + return NULL; + } + if (PyList_Extend(tasks, state->eager_tasks) < 0) { + Py_DECREF(tasks); + Py_DECREF(loop); + return NULL; + } + int err = 0; + ASYNCIO_STATE_LOCK(state); + struct llist_node *node; + + llist_for_each_safe(node, &state->asyncio_tasks_head) { + TaskObj *task = llist_data(node, TaskObj, task_node); + // The linked list holds borrowed references to task + // as such it is possible that the task is concurrently + // deallocated while added to this list. + // To protect against concurrent deallocations, + // we first try to incref the task which would fail + // if it is concurrently getting deallocated in another thread, + // otherwise it gets added to the list. + if (_Py_TryIncref((PyObject *)task)) { + if (_PyList_AppendTakeRef((PyListObject *)tasks, (PyObject *)task) < 0) { + Py_DECREF(tasks); + Py_DECREF(loop); + err = 1; + break; + } + } + } + ASYNCIO_STATE_UNLOCK(state); + if (err) { + return NULL; + } + PyObject *scheduled_iter = PyObject_GetIter(state->non_asyncio_tasks); + if (scheduled_iter == NULL) { + Py_DECREF(tasks); + Py_DECREF(loop); + return NULL; + } + PyObject *item; + while ((item = PyIter_Next(scheduled_iter)) != NULL) { + if (PyList_Append(tasks, item) < 0) { + Py_DECREF(tasks); + Py_DECREF(loop); + Py_DECREF(item); + Py_DECREF(scheduled_iter); + return NULL; + } + Py_DECREF(item); + } + Py_DECREF(scheduled_iter); + // All the tasks are now in the list, now filter the tasks which are done + PyObject *res = PySet_New(NULL); + if (res == NULL) { + Py_DECREF(tasks); + Py_DECREF(loop); + return NULL; + } + + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(tasks); i++) { + PyObject *task = PyList_GET_ITEM(tasks, i); + if (add_one_task(state, res, task, loop) < 0) { + Py_DECREF(res); + Py_DECREF(tasks); + Py_DECREF(loop); + return NULL; + } + } + + Py_DECREF(tasks); + Py_DECREF(loop); + return res; +} + +/*[clinic input] +_asyncio.future_add_to_awaited_by + + fut: object + waiter: object + / + +Record that `fut` is awaited on by `waiter`. + +[clinic start generated code]*/ + +static PyObject * +_asyncio_future_add_to_awaited_by_impl(PyObject *module, PyObject *fut, + PyObject *waiter) +/*[clinic end generated code: output=0ab9a1a63389e4df input=06e6eaac51f532b9]*/ { - PyObject *next; - PyObject *current; + asyncio_state *state = get_asyncio_state(module); + if (future_awaited_by_add(state, fut, waiter)) { + return NULL; + } + Py_RETURN_NONE; +} + +/*[clinic input] +_asyncio.future_discard_from_awaited_by - next = (PyObject*) state->fi_freelist; - while (next != NULL) { - assert(state->fi_freelist_len > 0); - state->fi_freelist_len--; + fut: object + waiter: object + / + +[clinic start generated code]*/ - current = next; - next = (PyObject*) ((futureiterobject*) current)->future; - PyObject_GC_Del(current); +static PyObject * +_asyncio_future_discard_from_awaited_by_impl(PyObject *module, PyObject *fut, + PyObject *waiter) +/*[clinic end generated code: output=a03b0b4323b779de input=3833f7639e88e483]*/ +{ + asyncio_state *state = get_asyncio_state(module); + if (future_awaited_by_discard(state, fut, waiter)) { + return NULL; } - assert(state->fi_freelist_len == 0); - state->fi_freelist = NULL; + Py_RETURN_NONE; } static int @@ -3567,20 +4158,13 @@ module_traverse(PyObject *mod, visitproc visit, void *arg) Py_VISIT(state->asyncio_InvalidStateError); Py_VISIT(state->asyncio_CancelledError); - Py_VISIT(state->scheduled_tasks); + Py_VISIT(state->non_asyncio_tasks); Py_VISIT(state->eager_tasks); Py_VISIT(state->current_tasks); Py_VISIT(state->iscoroutine_typecache); Py_VISIT(state->context_kwname); - // Visit freelist. - PyObject *next = (PyObject*) state->fi_freelist; - while (next != NULL) { - PyObject *current = next; - Py_VISIT(current); - next = (PyObject*) ((futureiterobject*) current)->future; - } return 0; } @@ -3605,14 +4189,18 @@ module_clear(PyObject *mod) Py_CLEAR(state->asyncio_InvalidStateError); Py_CLEAR(state->asyncio_CancelledError); - Py_CLEAR(state->scheduled_tasks); + Py_CLEAR(state->non_asyncio_tasks); Py_CLEAR(state->eager_tasks); Py_CLEAR(state->current_tasks); Py_CLEAR(state->iscoroutine_typecache); Py_CLEAR(state->context_kwname); - - module_free_freelists(state); + // Clear the ref to running loop so that finalizers can run early. + // If there are other running loops in different threads, + // those get cleared in PyThreadState_Clear. + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + Py_CLEAR(ts->asyncio_running_loop); + Py_CLEAR(ts->asyncio_running_task); return 0; } @@ -3643,7 +4231,6 @@ module_init(asyncio_state *state) goto fail; } - state->context_kwname = Py_BuildValue("(s)", "context"); if (state->context_kwname == NULL) { goto fail; @@ -3663,7 +4250,7 @@ module_init(asyncio_state *state) } WITH_MOD("asyncio.events") - GET_MOD_ATTR(state->asyncio_get_event_loop_policy, "get_event_loop_policy") + GET_MOD_ATTR(state->asyncio_get_event_loop_policy, "_get_event_loop_policy") WITH_MOD("asyncio.base_futures") GET_MOD_ATTR(state->asyncio_future_repr_func, "_future_repr") @@ -3686,9 +4273,9 @@ module_init(asyncio_state *state) PyObject *weak_set; WITH_MOD("weakref") GET_MOD_ATTR(weak_set, "WeakSet"); - state->scheduled_tasks = PyObject_CallNoArgs(weak_set); + state->non_asyncio_tasks = PyObject_CallNoArgs(weak_set); Py_CLEAR(weak_set); - if (state->scheduled_tasks == NULL) { + if (state->non_asyncio_tasks == NULL) { goto fail; } @@ -3723,6 +4310,9 @@ static PyMethodDef asyncio_methods[] = { _ASYNCIO__ENTER_TASK_METHODDEF _ASYNCIO__LEAVE_TASK_METHODDEF _ASYNCIO__SWAP_CURRENT_TASK_METHODDEF + _ASYNCIO_ALL_TASKS_METHODDEF + _ASYNCIO_FUTURE_ADD_TO_AWAITED_BY_METHODDEF + _ASYNCIO_FUTURE_DISCARD_FROM_AWAITED_BY_METHODDEF {NULL, NULL} }; @@ -3731,6 +4321,8 @@ module_exec(PyObject *mod) { asyncio_state *state = get_asyncio_state(mod); + llist_init(&state->asyncio_tasks_head); + #define CREATE_TYPE(m, tp, spec, base) \ do { \ tp = (PyTypeObject *)PyType_FromMetaclass(NULL, m, spec, \ @@ -3759,7 +4351,7 @@ module_exec(PyObject *mod) return -1; } - if (PyModule_AddObjectRef(mod, "_scheduled_tasks", state->scheduled_tasks) < 0) { + if (PyModule_AddObjectRef(mod, "_scheduled_tasks", state->non_asyncio_tasks) < 0) { return -1; } @@ -3778,6 +4370,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 9e0fd336419b44..9b146265445d9a 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -70,7 +70,7 @@ internal_bisect_right(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t if (sq_item == NULL) { return -1; } - if (Py_EnterRecursiveCall("in _bisect.bisect_right")) { + if (Py_EnterRecursiveCall(" in _bisect.bisect_right")) { return -1; } PyTypeObject *tp = Py_TYPE(item); @@ -254,7 +254,7 @@ internal_bisect_left(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t h if (sq_item == NULL) { return -1; } - if (Py_EnterRecursiveCall("in _bisect.bisect_left")) { + if (Py_EnterRecursiveCall(" in _bisect.bisect_left")) { return -1; } PyTypeObject *tp = Py_TYPE(item); @@ -462,6 +462,7 @@ bisect_modexec(PyObject *m) static PyModuleDef_Slot bisect_slots[] = { {Py_mod_exec, bisect_modexec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_blake2/blake2b2s.py b/Modules/_blake2/blake2b2s.py deleted file mode 100755 index 01cf26521b3779..00000000000000 --- a/Modules/_blake2/blake2b2s.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/python3 - -import os -import re - -HERE = os.path.dirname(os.path.abspath(__file__)) -BLAKE2 = os.path.join(HERE, 'impl') - -PUBLIC_SEARCH = re.compile(r'\ int (blake2[bs]p?[a-z_]*)\(') - - -def getfiles(): - for name in os.listdir(BLAKE2): - name = os.path.join(BLAKE2, name) - if os.path.isfile(name): - yield name - - -def find_public(): - public_funcs = set() - for name in getfiles(): - with open(name) as f: - for line in f: - # find public functions - mo = PUBLIC_SEARCH.search(line) - if mo: - public_funcs.add(mo.group(1)) - - for f in sorted(public_funcs): - print('#define {0:<18} PyBlake2_{0}'.format(f)) - - return public_funcs - - -def main(): - lines = [] - with open(os.path.join(HERE, 'blake2b_impl.c')) as f: - for line in f: - line = line.replace('blake2b', 'blake2s') - line = line.replace('BLAKE2b', 'BLAKE2s') - line = line.replace('BLAKE2B', 'BLAKE2S') - lines.append(line) - with open(os.path.join(HERE, 'blake2s_impl.c'), 'w') as f: - f.write(''.join(lines)) - # find_public() - - -if __name__ == '__main__': - main() diff --git a/Modules/_blake2/blake2b_impl.c b/Modules/_blake2/blake2b_impl.c deleted file mode 100644 index 0c3ae5a2fac275..00000000000000 --- a/Modules/_blake2/blake2b_impl.c +++ /dev/null @@ -1,417 +0,0 @@ -/* - * Written in 2013 by Dmitry Chestnykh - * Modified for CPython by Christian Heimes - * - * To the extent possible under law, the author have dedicated all - * copyright and related and neighboring rights to this software to - * the public domain worldwide. This software is distributed without - * any warranty. http://creativecommons.org/publicdomain/zero/1.0/ - */ - -/* WARNING: autogenerated file! - * - * The blake2s_impl.c is autogenerated from blake2b_impl.c. - */ - -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 -#endif - -#include -#include "Python.h" -#include "pycore_strhex.h" // _Py_strhex() - -#include "../hashlib.h" -#include "blake2module.h" - -#ifndef HAVE_LIBB2 -/* pure SSE2 implementation is very slow, so only use the more optimized SSSE3+ - * https://bugs.python.org/issue31834 */ -#if defined(__SSSE3__) || defined(__SSE4_1__) || defined(__AVX__) || defined(__XOP__) -#include "impl/blake2b.c" -#else -#include "impl/blake2b-ref.c" -#endif -#endif // !HAVE_LIBB2 - -#define HAVE_BLAKE2B 1 - -extern PyType_Spec blake2b_type_spec; - - -typedef struct { - PyObject_HEAD - blake2b_param param; - blake2b_state state; - bool use_mutex; - PyMutex mutex; -} BLAKE2bObject; - -#include "clinic/blake2b_impl.c.h" - -/*[clinic input] -module _blake2 -class _blake2.blake2b "BLAKE2bObject *" "&PyBlake2_BLAKE2bType" -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=d47b0527b39c673f]*/ - - -static BLAKE2bObject * -new_BLAKE2bObject(PyTypeObject *type) -{ - BLAKE2bObject *self; - self = (BLAKE2bObject *)type->tp_alloc(type, 0); - if (self == NULL) { - return NULL; - } - HASHLIB_INIT_MUTEX(self); - - return self; -} - -/*[clinic input] -@classmethod -_blake2.blake2b.__new__ as py_blake2b_new - data: object(c_default="NULL") = b'' - / - * - digest_size: int(c_default="BLAKE2B_OUTBYTES") = _blake2.blake2b.MAX_DIGEST_SIZE - key: Py_buffer(c_default="NULL", py_default="b''") = None - salt: Py_buffer(c_default="NULL", py_default="b''") = None - person: Py_buffer(c_default="NULL", py_default="b''") = None - fanout: int = 1 - depth: int = 1 - leaf_size: unsigned_long = 0 - node_offset: unsigned_long_long = 0 - node_depth: int = 0 - inner_size: int = 0 - last_node: bool = False - usedforsecurity: bool = True - -Return a new BLAKE2b hash object. -[clinic start generated code]*/ - -static PyObject * -py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size, - Py_buffer *key, Py_buffer *salt, Py_buffer *person, - int fanout, int depth, unsigned long leaf_size, - unsigned long long node_offset, int node_depth, - int inner_size, int last_node, int usedforsecurity) -/*[clinic end generated code: output=32bfd8f043c6896f input=b947312abff46977]*/ -{ - BLAKE2bObject *self = NULL; - Py_buffer buf; - - self = new_BLAKE2bObject(type); - if (self == NULL) { - goto error; - } - - /* Zero parameter block. */ - memset(&self->param, 0, sizeof(self->param)); - - /* Set digest size. */ - if (digest_size <= 0 || digest_size > BLAKE2B_OUTBYTES) { - PyErr_Format(PyExc_ValueError, - "digest_size must be between 1 and %d bytes", - BLAKE2B_OUTBYTES); - goto error; - } - self->param.digest_length = digest_size; - - /* Set salt parameter. */ - if ((salt->obj != NULL) && salt->len) { - if (salt->len > BLAKE2B_SALTBYTES) { - PyErr_Format(PyExc_ValueError, - "maximum salt length is %d bytes", - BLAKE2B_SALTBYTES); - goto error; - } - memcpy(self->param.salt, salt->buf, salt->len); - } - - /* Set personalization parameter. */ - if ((person->obj != NULL) && person->len) { - if (person->len > BLAKE2B_PERSONALBYTES) { - PyErr_Format(PyExc_ValueError, - "maximum person length is %d bytes", - BLAKE2B_PERSONALBYTES); - goto error; - } - memcpy(self->param.personal, person->buf, person->len); - } - - /* Set tree parameters. */ - if (fanout < 0 || fanout > 255) { - PyErr_SetString(PyExc_ValueError, - "fanout must be between 0 and 255"); - goto error; - } - self->param.fanout = (uint8_t)fanout; - - if (depth <= 0 || depth > 255) { - PyErr_SetString(PyExc_ValueError, - "depth must be between 1 and 255"); - goto error; - } - self->param.depth = (uint8_t)depth; - - if (leaf_size > 0xFFFFFFFFU) { - PyErr_SetString(PyExc_OverflowError, "leaf_size is too large"); - goto error; - } - // NB: Simple assignment here would be incorrect on big endian platforms. - store32(&(self->param.leaf_length), leaf_size); - -#ifdef HAVE_BLAKE2S - if (node_offset > 0xFFFFFFFFFFFFULL) { - /* maximum 2**48 - 1 */ - PyErr_SetString(PyExc_OverflowError, "node_offset is too large"); - goto error; - } - store48(&(self->param.node_offset), node_offset); -#else - // NB: Simple assignment here would be incorrect on big endian platforms. - store64(&(self->param.node_offset), node_offset); -#endif - - if (node_depth < 0 || node_depth > 255) { - PyErr_SetString(PyExc_ValueError, - "node_depth must be between 0 and 255"); - goto error; - } - self->param.node_depth = node_depth; - - if (inner_size < 0 || inner_size > BLAKE2B_OUTBYTES) { - PyErr_Format(PyExc_ValueError, - "inner_size must be between 0 and is %d", - BLAKE2B_OUTBYTES); - goto error; - } - self->param.inner_length = inner_size; - - /* Set key length. */ - if ((key->obj != NULL) && key->len) { - if (key->len > BLAKE2B_KEYBYTES) { - PyErr_Format(PyExc_ValueError, - "maximum key length is %d bytes", - BLAKE2B_KEYBYTES); - goto error; - } - self->param.key_length = (uint8_t)key->len; - } - - /* Initialize hash state. */ - if (blake2b_init_param(&self->state, &self->param) < 0) { - PyErr_SetString(PyExc_RuntimeError, - "error initializing hash state"); - goto error; - } - - /* Set last node flag (must come after initialization). */ - self->state.last_node = last_node; - - /* Process key block if any. */ - if (self->param.key_length) { - uint8_t block[BLAKE2B_BLOCKBYTES]; - memset(block, 0, sizeof(block)); - memcpy(block, key->buf, key->len); - blake2b_update(&self->state, block, sizeof(block)); - secure_zero_memory(block, sizeof(block)); - } - - /* Process initial data if any. */ - if (data != NULL) { - GET_BUFFER_VIEW_OR_ERROR(data, &buf, goto error); - - if (buf.len >= HASHLIB_GIL_MINSIZE) { - Py_BEGIN_ALLOW_THREADS - blake2b_update(&self->state, buf.buf, buf.len); - Py_END_ALLOW_THREADS - } else { - blake2b_update(&self->state, buf.buf, buf.len); - } - PyBuffer_Release(&buf); - } - - return (PyObject *)self; - - error: - if (self != NULL) { - Py_DECREF(self); - } - return NULL; -} - -/*[clinic input] -_blake2.blake2b.copy - -Return a copy of the hash object. -[clinic start generated code]*/ - -static PyObject * -_blake2_blake2b_copy_impl(BLAKE2bObject *self) -/*[clinic end generated code: output=ff6acee5f93656ae input=e383c2d199fd8a2e]*/ -{ - BLAKE2bObject *cpy; - - if ((cpy = new_BLAKE2bObject(Py_TYPE(self))) == NULL) - return NULL; - - ENTER_HASHLIB(self); - cpy->param = self->param; - cpy->state = self->state; - LEAVE_HASHLIB(self); - return (PyObject *)cpy; -} - -/*[clinic input] -_blake2.blake2b.update - - data: object - / - -Update this hash object's state with the provided bytes-like object. -[clinic start generated code]*/ - -static PyObject * -_blake2_blake2b_update(BLAKE2bObject *self, PyObject *data) -/*[clinic end generated code: output=010dfcbe22654359 input=ffc4aa6a6a225d31]*/ -{ - Py_buffer buf; - - GET_BUFFER_VIEW_OR_ERROUT(data, &buf); - - if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { - self->use_mutex = true; - } - if (self->use_mutex) { - Py_BEGIN_ALLOW_THREADS - PyMutex_Lock(&self->mutex); - blake2b_update(&self->state, buf.buf, buf.len); - PyMutex_Unlock(&self->mutex); - Py_END_ALLOW_THREADS - } else { - blake2b_update(&self->state, buf.buf, buf.len); - } - - PyBuffer_Release(&buf); - - Py_RETURN_NONE; -} - -/*[clinic input] -_blake2.blake2b.digest - -Return the digest value as a bytes object. -[clinic start generated code]*/ - -static PyObject * -_blake2_blake2b_digest_impl(BLAKE2bObject *self) -/*[clinic end generated code: output=a5864660f4bfc61a input=7d21659e9c5fff02]*/ -{ - uint8_t digest[BLAKE2B_OUTBYTES]; - blake2b_state state_cpy; - - ENTER_HASHLIB(self); - state_cpy = self->state; - blake2b_final(&state_cpy, digest, self->param.digest_length); - LEAVE_HASHLIB(self); - return PyBytes_FromStringAndSize((const char *)digest, - self->param.digest_length); -} - -/*[clinic input] -_blake2.blake2b.hexdigest - -Return the digest value as a string of hexadecimal digits. -[clinic start generated code]*/ - -static PyObject * -_blake2_blake2b_hexdigest_impl(BLAKE2bObject *self) -/*[clinic end generated code: output=b5598a87d8794a60 input=76930f6946351f56]*/ -{ - uint8_t digest[BLAKE2B_OUTBYTES]; - blake2b_state state_cpy; - - ENTER_HASHLIB(self); - state_cpy = self->state; - blake2b_final(&state_cpy, digest, self->param.digest_length); - LEAVE_HASHLIB(self); - return _Py_strhex((const char *)digest, self->param.digest_length); -} - - -static PyMethodDef py_blake2b_methods[] = { - _BLAKE2_BLAKE2B_COPY_METHODDEF - _BLAKE2_BLAKE2B_DIGEST_METHODDEF - _BLAKE2_BLAKE2B_HEXDIGEST_METHODDEF - _BLAKE2_BLAKE2B_UPDATE_METHODDEF - {NULL, NULL} -}; - - - -static PyObject * -py_blake2b_get_name(BLAKE2bObject *self, void *closure) -{ - return PyUnicode_FromString("blake2b"); -} - - - -static PyObject * -py_blake2b_get_block_size(BLAKE2bObject *self, void *closure) -{ - return PyLong_FromLong(BLAKE2B_BLOCKBYTES); -} - - - -static PyObject * -py_blake2b_get_digest_size(BLAKE2bObject *self, void *closure) -{ - return PyLong_FromLong(self->param.digest_length); -} - - -static PyGetSetDef py_blake2b_getsetters[] = { - {"name", (getter)py_blake2b_get_name, - NULL, NULL, NULL}, - {"block_size", (getter)py_blake2b_get_block_size, - NULL, NULL, NULL}, - {"digest_size", (getter)py_blake2b_get_digest_size, - NULL, NULL, NULL}, - {NULL} -}; - - -static void -py_blake2b_dealloc(PyObject *self) -{ - BLAKE2bObject *obj = (BLAKE2bObject *)self; - - /* Try not to leave state in memory. */ - secure_zero_memory(&obj->param, sizeof(obj->param)); - secure_zero_memory(&obj->state, sizeof(obj->state)); - - PyTypeObject *type = Py_TYPE(self); - PyObject_Free(self); - Py_DECREF(type); -} - -static PyType_Slot blake2b_type_slots[] = { - {Py_tp_dealloc, py_blake2b_dealloc}, - {Py_tp_doc, (char *)py_blake2b_new__doc__}, - {Py_tp_methods, py_blake2b_methods}, - {Py_tp_getset, py_blake2b_getsetters}, - {Py_tp_new, py_blake2b_new}, - {0,0} -}; - -PyType_Spec blake2b_type_spec = { - .name = "_blake2.blake2b", - .basicsize = sizeof(BLAKE2bObject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, - .slots = blake2b_type_slots -}; diff --git a/Modules/_blake2/blake2module.c b/Modules/_blake2/blake2module.c deleted file mode 100644 index 5df9fd3df493ee..00000000000000 --- a/Modules/_blake2/blake2module.c +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Written in 2013 by Dmitry Chestnykh - * Modified for CPython by Christian Heimes - * - * To the extent possible under law, the author have dedicated all - * copyright and related and neighboring rights to this software to - * the public domain worldwide. This software is distributed without - * any warranty. http://creativecommons.org/publicdomain/zero/1.0/ - */ - -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 -#endif - -#include "Python.h" -#include "blake2module.h" - -extern PyType_Spec blake2b_type_spec; -extern PyType_Spec blake2s_type_spec; - -PyDoc_STRVAR(blake2mod__doc__, -"_blake2b provides BLAKE2b for hashlib\n" -); - -typedef struct { - PyTypeObject* blake2b_type; - PyTypeObject* blake2s_type; -} Blake2State; - -static inline Blake2State* -blake2_get_state(PyObject *module) -{ - void *state = PyModule_GetState(module); - assert(state != NULL); - return (Blake2State *)state; -} - -static struct PyMethodDef blake2mod_functions[] = { - {NULL, NULL} -}; - -static int -_blake2_traverse(PyObject *module, visitproc visit, void *arg) -{ - Blake2State *state = blake2_get_state(module); - Py_VISIT(state->blake2b_type); - Py_VISIT(state->blake2s_type); - return 0; -} - -static int -_blake2_clear(PyObject *module) -{ - Blake2State *state = blake2_get_state(module); - Py_CLEAR(state->blake2b_type); - Py_CLEAR(state->blake2s_type); - return 0; -} - -static void -_blake2_free(void *module) -{ - _blake2_clear((PyObject *)module); -} - -#define ADD_INT(d, name, value) do { \ - PyObject *x = PyLong_FromLong(value); \ - if (!x) \ - return -1; \ - if (PyDict_SetItemString(d, name, x) < 0) { \ - Py_DECREF(x); \ - return -1; \ - } \ - Py_DECREF(x); \ -} while(0) - -#define ADD_INT_CONST(NAME, VALUE) do { \ - if (PyModule_AddIntConstant(m, NAME, VALUE) < 0) { \ - return -1; \ - } \ -} while (0) - -static int -blake2_exec(PyObject *m) -{ - Blake2State* st = blake2_get_state(m); - - st->blake2b_type = (PyTypeObject *)PyType_FromModuleAndSpec( - m, &blake2b_type_spec, NULL); - - if (NULL == st->blake2b_type) - return -1; - /* BLAKE2b */ - if (PyModule_AddType(m, st->blake2b_type) < 0) { - return -1; - } - - PyObject *d = st->blake2b_type->tp_dict; - ADD_INT(d, "SALT_SIZE", BLAKE2B_SALTBYTES); - ADD_INT(d, "PERSON_SIZE", BLAKE2B_PERSONALBYTES); - ADD_INT(d, "MAX_KEY_SIZE", BLAKE2B_KEYBYTES); - ADD_INT(d, "MAX_DIGEST_SIZE", BLAKE2B_OUTBYTES); - - ADD_INT_CONST("BLAKE2B_SALT_SIZE", BLAKE2B_SALTBYTES); - ADD_INT_CONST("BLAKE2B_PERSON_SIZE", BLAKE2B_PERSONALBYTES); - ADD_INT_CONST("BLAKE2B_MAX_KEY_SIZE", BLAKE2B_KEYBYTES); - ADD_INT_CONST("BLAKE2B_MAX_DIGEST_SIZE", BLAKE2B_OUTBYTES); - - /* BLAKE2s */ - st->blake2s_type = (PyTypeObject *)PyType_FromModuleAndSpec( - m, &blake2s_type_spec, NULL); - - if (NULL == st->blake2s_type) - return -1; - - if (PyModule_AddType(m, st->blake2s_type) < 0) { - return -1; - } - - d = st->blake2s_type->tp_dict; - ADD_INT(d, "SALT_SIZE", BLAKE2S_SALTBYTES); - ADD_INT(d, "PERSON_SIZE", BLAKE2S_PERSONALBYTES); - ADD_INT(d, "MAX_KEY_SIZE", BLAKE2S_KEYBYTES); - ADD_INT(d, "MAX_DIGEST_SIZE", BLAKE2S_OUTBYTES); - - ADD_INT_CONST("BLAKE2S_SALT_SIZE", BLAKE2S_SALTBYTES); - ADD_INT_CONST("BLAKE2S_PERSON_SIZE", BLAKE2S_PERSONALBYTES); - ADD_INT_CONST("BLAKE2S_MAX_KEY_SIZE", BLAKE2S_KEYBYTES); - ADD_INT_CONST("BLAKE2S_MAX_DIGEST_SIZE", BLAKE2S_OUTBYTES); - - return 0; -} - -#undef ADD_INT -#undef ADD_INT_CONST - -static PyModuleDef_Slot _blake2_slots[] = { - {Py_mod_exec, blake2_exec}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {0, NULL} -}; - -static struct PyModuleDef blake2_module = { - PyModuleDef_HEAD_INIT, - "_blake2", - .m_doc = blake2mod__doc__, - .m_size = sizeof(Blake2State), - .m_methods = blake2mod_functions, - .m_slots = _blake2_slots, - .m_traverse = _blake2_traverse, - .m_clear = _blake2_clear, - .m_free = _blake2_free, -}; - -PyMODINIT_FUNC -PyInit__blake2(void) -{ - return PyModuleDef_Init(&blake2_module); -} diff --git a/Modules/_blake2/blake2module.h b/Modules/_blake2/blake2module.h deleted file mode 100644 index c8144ec9d48d29..00000000000000 --- a/Modules/_blake2/blake2module.h +++ /dev/null @@ -1,43 +0,0 @@ -#ifndef Py_BLAKE2MODULE_H -#define Py_BLAKE2MODULE_H - -#ifdef HAVE_LIBB2 -#include - -#else -// use vendored copy of blake2 - -// Prefix all public blake2 symbols with PyBlake2_ -#define blake2b PyBlake2_blake2b -#define blake2b_compress PyBlake2_blake2b_compress -#define blake2b_final PyBlake2_blake2b_final -#define blake2b_init PyBlake2_blake2b_init -#define blake2b_init_key PyBlake2_blake2b_init_key -#define blake2b_init_param PyBlake2_blake2b_init_param -#define blake2b_update PyBlake2_blake2b_update -#define blake2bp PyBlake2_blake2bp -#define blake2bp_final PyBlake2_blake2bp_final -#define blake2bp_init PyBlake2_blake2bp_init -#define blake2bp_init_key PyBlake2_blake2bp_init_key -#define blake2bp_update PyBlake2_blake2bp_update -#define blake2s PyBlake2_blake2s -#define blake2s_compress PyBlake2_blake2s_compress -#define blake2s_final PyBlake2_blake2s_final -#define blake2s_init PyBlake2_blake2s_init -#define blake2s_init_key PyBlake2_blake2s_init_key -#define blake2s_init_param PyBlake2_blake2s_init_param -#define blake2s_update PyBlake2_blake2s_update -#define blake2sp PyBlake2_blake2sp -#define blake2sp_final PyBlake2_blake2sp_final -#define blake2sp_init PyBlake2_blake2sp_init -#define blake2sp_init_key PyBlake2_blake2sp_init_key -#define blake2sp_update PyBlake2_blake2sp_update - -#include "impl/blake2.h" - -#endif // HAVE_LIBB2 - -// for secure_zero_memory(), store32(), store48(), and store64() -#include "impl/blake2-impl.h" - -#endif // Py_BLAKE2MODULE_H diff --git a/Modules/_blake2/blake2s_impl.c b/Modules/_blake2/blake2s_impl.c deleted file mode 100644 index 3014773ab52331..00000000000000 --- a/Modules/_blake2/blake2s_impl.c +++ /dev/null @@ -1,417 +0,0 @@ -/* - * Written in 2013 by Dmitry Chestnykh - * Modified for CPython by Christian Heimes - * - * To the extent possible under law, the author have dedicated all - * copyright and related and neighboring rights to this software to - * the public domain worldwide. This software is distributed without - * any warranty. http://creativecommons.org/publicdomain/zero/1.0/ - */ - -/* WARNING: autogenerated file! - * - * The blake2s_impl.c is autogenerated from blake2s_impl.c. - */ - -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 -#endif - -#include -#include "Python.h" -#include "pycore_strhex.h" // _Py_strhex() - -#include "../hashlib.h" -#include "blake2module.h" - -#ifndef HAVE_LIBB2 -/* pure SSE2 implementation is very slow, so only use the more optimized SSSE3+ - * https://bugs.python.org/issue31834 */ -#if defined(__SSSE3__) || defined(__SSE4_1__) || defined(__AVX__) || defined(__XOP__) -#include "impl/blake2s.c" -#else -#include "impl/blake2s-ref.c" -#endif -#endif // !HAVE_LIBB2 - -#define HAVE_BLAKE2S 1 - -extern PyType_Spec blake2s_type_spec; - - -typedef struct { - PyObject_HEAD - blake2s_param param; - blake2s_state state; - bool use_mutex; - PyMutex mutex; -} BLAKE2sObject; - -#include "clinic/blake2s_impl.c.h" - -/*[clinic input] -module _blake2 -class _blake2.blake2s "BLAKE2sObject *" "&PyBlake2_BLAKE2sType" -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=4b79d7ffe07286ce]*/ - - -static BLAKE2sObject * -new_BLAKE2sObject(PyTypeObject *type) -{ - BLAKE2sObject *self; - self = (BLAKE2sObject *)type->tp_alloc(type, 0); - if (self == NULL) { - return NULL; - } - HASHLIB_INIT_MUTEX(self); - - return self; -} - -/*[clinic input] -@classmethod -_blake2.blake2s.__new__ as py_blake2s_new - data: object(c_default="NULL") = b'' - / - * - digest_size: int(c_default="BLAKE2S_OUTBYTES") = _blake2.blake2s.MAX_DIGEST_SIZE - key: Py_buffer(c_default="NULL", py_default="b''") = None - salt: Py_buffer(c_default="NULL", py_default="b''") = None - person: Py_buffer(c_default="NULL", py_default="b''") = None - fanout: int = 1 - depth: int = 1 - leaf_size: unsigned_long = 0 - node_offset: unsigned_long_long = 0 - node_depth: int = 0 - inner_size: int = 0 - last_node: bool = False - usedforsecurity: bool = True - -Return a new BLAKE2s hash object. -[clinic start generated code]*/ - -static PyObject * -py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, - Py_buffer *key, Py_buffer *salt, Py_buffer *person, - int fanout, int depth, unsigned long leaf_size, - unsigned long long node_offset, int node_depth, - int inner_size, int last_node, int usedforsecurity) -/*[clinic end generated code: output=556181f73905c686 input=4dda87723f23abb0]*/ -{ - BLAKE2sObject *self = NULL; - Py_buffer buf; - - self = new_BLAKE2sObject(type); - if (self == NULL) { - goto error; - } - - /* Zero parameter block. */ - memset(&self->param, 0, sizeof(self->param)); - - /* Set digest size. */ - if (digest_size <= 0 || digest_size > BLAKE2S_OUTBYTES) { - PyErr_Format(PyExc_ValueError, - "digest_size must be between 1 and %d bytes", - BLAKE2S_OUTBYTES); - goto error; - } - self->param.digest_length = digest_size; - - /* Set salt parameter. */ - if ((salt->obj != NULL) && salt->len) { - if (salt->len > BLAKE2S_SALTBYTES) { - PyErr_Format(PyExc_ValueError, - "maximum salt length is %d bytes", - BLAKE2S_SALTBYTES); - goto error; - } - memcpy(self->param.salt, salt->buf, salt->len); - } - - /* Set personalization parameter. */ - if ((person->obj != NULL) && person->len) { - if (person->len > BLAKE2S_PERSONALBYTES) { - PyErr_Format(PyExc_ValueError, - "maximum person length is %d bytes", - BLAKE2S_PERSONALBYTES); - goto error; - } - memcpy(self->param.personal, person->buf, person->len); - } - - /* Set tree parameters. */ - if (fanout < 0 || fanout > 255) { - PyErr_SetString(PyExc_ValueError, - "fanout must be between 0 and 255"); - goto error; - } - self->param.fanout = (uint8_t)fanout; - - if (depth <= 0 || depth > 255) { - PyErr_SetString(PyExc_ValueError, - "depth must be between 1 and 255"); - goto error; - } - self->param.depth = (uint8_t)depth; - - if (leaf_size > 0xFFFFFFFFU) { - PyErr_SetString(PyExc_OverflowError, "leaf_size is too large"); - goto error; - } - // NB: Simple assignment here would be incorrect on big endian platforms. - store32(&(self->param.leaf_length), leaf_size); - -#ifdef HAVE_BLAKE2S - if (node_offset > 0xFFFFFFFFFFFFULL) { - /* maximum 2**48 - 1 */ - PyErr_SetString(PyExc_OverflowError, "node_offset is too large"); - goto error; - } - store48(&(self->param.node_offset), node_offset); -#else - // NB: Simple assignment here would be incorrect on big endian platforms. - store64(&(self->param.node_offset), node_offset); -#endif - - if (node_depth < 0 || node_depth > 255) { - PyErr_SetString(PyExc_ValueError, - "node_depth must be between 0 and 255"); - goto error; - } - self->param.node_depth = node_depth; - - if (inner_size < 0 || inner_size > BLAKE2S_OUTBYTES) { - PyErr_Format(PyExc_ValueError, - "inner_size must be between 0 and is %d", - BLAKE2S_OUTBYTES); - goto error; - } - self->param.inner_length = inner_size; - - /* Set key length. */ - if ((key->obj != NULL) && key->len) { - if (key->len > BLAKE2S_KEYBYTES) { - PyErr_Format(PyExc_ValueError, - "maximum key length is %d bytes", - BLAKE2S_KEYBYTES); - goto error; - } - self->param.key_length = (uint8_t)key->len; - } - - /* Initialize hash state. */ - if (blake2s_init_param(&self->state, &self->param) < 0) { - PyErr_SetString(PyExc_RuntimeError, - "error initializing hash state"); - goto error; - } - - /* Set last node flag (must come after initialization). */ - self->state.last_node = last_node; - - /* Process key block if any. */ - if (self->param.key_length) { - uint8_t block[BLAKE2S_BLOCKBYTES]; - memset(block, 0, sizeof(block)); - memcpy(block, key->buf, key->len); - blake2s_update(&self->state, block, sizeof(block)); - secure_zero_memory(block, sizeof(block)); - } - - /* Process initial data if any. */ - if (data != NULL) { - GET_BUFFER_VIEW_OR_ERROR(data, &buf, goto error); - - if (buf.len >= HASHLIB_GIL_MINSIZE) { - Py_BEGIN_ALLOW_THREADS - blake2s_update(&self->state, buf.buf, buf.len); - Py_END_ALLOW_THREADS - } else { - blake2s_update(&self->state, buf.buf, buf.len); - } - PyBuffer_Release(&buf); - } - - return (PyObject *)self; - - error: - if (self != NULL) { - Py_DECREF(self); - } - return NULL; -} - -/*[clinic input] -_blake2.blake2s.copy - -Return a copy of the hash object. -[clinic start generated code]*/ - -static PyObject * -_blake2_blake2s_copy_impl(BLAKE2sObject *self) -/*[clinic end generated code: output=5b90131c4eae275e input=0b9d44942f0fe4b2]*/ -{ - BLAKE2sObject *cpy; - - if ((cpy = new_BLAKE2sObject(Py_TYPE(self))) == NULL) - return NULL; - - ENTER_HASHLIB(self); - cpy->param = self->param; - cpy->state = self->state; - LEAVE_HASHLIB(self); - return (PyObject *)cpy; -} - -/*[clinic input] -_blake2.blake2s.update - - data: object - / - -Update this hash object's state with the provided bytes-like object. -[clinic start generated code]*/ - -static PyObject * -_blake2_blake2s_update(BLAKE2sObject *self, PyObject *data) -/*[clinic end generated code: output=757dc087fec37815 input=97500db2f9de4aaa]*/ -{ - Py_buffer buf; - - GET_BUFFER_VIEW_OR_ERROUT(data, &buf); - - if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { - self->use_mutex = true; - } - if (self->use_mutex) { - Py_BEGIN_ALLOW_THREADS - PyMutex_Lock(&self->mutex); - blake2s_update(&self->state, buf.buf, buf.len); - PyMutex_Unlock(&self->mutex); - Py_END_ALLOW_THREADS - } else { - blake2s_update(&self->state, buf.buf, buf.len); - } - - PyBuffer_Release(&buf); - - Py_RETURN_NONE; -} - -/*[clinic input] -_blake2.blake2s.digest - -Return the digest value as a bytes object. -[clinic start generated code]*/ - -static PyObject * -_blake2_blake2s_digest_impl(BLAKE2sObject *self) -/*[clinic end generated code: output=40c566ca4bc6bc51 input=f41e0b8d6d937454]*/ -{ - uint8_t digest[BLAKE2S_OUTBYTES]; - blake2s_state state_cpy; - - ENTER_HASHLIB(self); - state_cpy = self->state; - blake2s_final(&state_cpy, digest, self->param.digest_length); - LEAVE_HASHLIB(self); - return PyBytes_FromStringAndSize((const char *)digest, - self->param.digest_length); -} - -/*[clinic input] -_blake2.blake2s.hexdigest - -Return the digest value as a string of hexadecimal digits. -[clinic start generated code]*/ - -static PyObject * -_blake2_blake2s_hexdigest_impl(BLAKE2sObject *self) -/*[clinic end generated code: output=15153eb5e59c52eb input=c77a1321567e8952]*/ -{ - uint8_t digest[BLAKE2S_OUTBYTES]; - blake2s_state state_cpy; - - ENTER_HASHLIB(self); - state_cpy = self->state; - blake2s_final(&state_cpy, digest, self->param.digest_length); - LEAVE_HASHLIB(self); - return _Py_strhex((const char *)digest, self->param.digest_length); -} - - -static PyMethodDef py_blake2s_methods[] = { - _BLAKE2_BLAKE2S_COPY_METHODDEF - _BLAKE2_BLAKE2S_DIGEST_METHODDEF - _BLAKE2_BLAKE2S_HEXDIGEST_METHODDEF - _BLAKE2_BLAKE2S_UPDATE_METHODDEF - {NULL, NULL} -}; - - - -static PyObject * -py_blake2s_get_name(BLAKE2sObject *self, void *closure) -{ - return PyUnicode_FromString("blake2s"); -} - - - -static PyObject * -py_blake2s_get_block_size(BLAKE2sObject *self, void *closure) -{ - return PyLong_FromLong(BLAKE2S_BLOCKBYTES); -} - - - -static PyObject * -py_blake2s_get_digest_size(BLAKE2sObject *self, void *closure) -{ - return PyLong_FromLong(self->param.digest_length); -} - - -static PyGetSetDef py_blake2s_getsetters[] = { - {"name", (getter)py_blake2s_get_name, - NULL, NULL, NULL}, - {"block_size", (getter)py_blake2s_get_block_size, - NULL, NULL, NULL}, - {"digest_size", (getter)py_blake2s_get_digest_size, - NULL, NULL, NULL}, - {NULL} -}; - - -static void -py_blake2s_dealloc(PyObject *self) -{ - BLAKE2sObject *obj = (BLAKE2sObject *)self; - - /* Try not to leave state in memory. */ - secure_zero_memory(&obj->param, sizeof(obj->param)); - secure_zero_memory(&obj->state, sizeof(obj->state)); - - PyTypeObject *type = Py_TYPE(self); - PyObject_Free(self); - Py_DECREF(type); -} - -static PyType_Slot blake2s_type_slots[] = { - {Py_tp_dealloc, py_blake2s_dealloc}, - {Py_tp_doc, (char *)py_blake2s_new__doc__}, - {Py_tp_methods, py_blake2s_methods}, - {Py_tp_getset, py_blake2s_getsetters}, - {Py_tp_new, py_blake2s_new}, - {0,0} -}; - -PyType_Spec blake2s_type_spec = { - .name = "_blake2.blake2s", - .basicsize = sizeof(BLAKE2sObject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, - .slots = blake2s_type_slots -}; diff --git a/Modules/_blake2/clinic/blake2s_impl.c.h b/Modules/_blake2/clinic/blake2s_impl.c.h deleted file mode 100644 index 7a0f6eeff5b5b5..00000000000000 --- a/Modules/_blake2/clinic/blake2s_impl.c.h +++ /dev/null @@ -1,268 +0,0 @@ -/*[clinic input] -preserve -[clinic start generated code]*/ - -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif -#include "pycore_long.h" // _PyLong_UnsignedLong_Converter() -#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() - -PyDoc_STRVAR(py_blake2s_new__doc__, -"blake2s(data=b\'\', /, *, digest_size=_blake2.blake2s.MAX_DIGEST_SIZE,\n" -" key=b\'\', salt=b\'\', person=b\'\', fanout=1, depth=1, leaf_size=0,\n" -" node_offset=0, node_depth=0, inner_size=0, last_node=False,\n" -" usedforsecurity=True)\n" -"--\n" -"\n" -"Return a new BLAKE2s hash object."); - -static PyObject * -py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, - Py_buffer *key, Py_buffer *salt, Py_buffer *person, - int fanout, int depth, unsigned long leaf_size, - unsigned long long node_offset, int node_depth, - int inner_size, int last_node, int usedforsecurity); - -static PyObject * -py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 12 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(digest_size), &_Py_ID(key), &_Py_ID(salt), &_Py_ID(person), &_Py_ID(fanout), &_Py_ID(depth), &_Py_ID(leaf_size), &_Py_ID(node_offset), &_Py_ID(node_depth), &_Py_ID(inner_size), &_Py_ID(last_node), &_Py_ID(usedforsecurity), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "blake2s", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[13]; - PyObject * const *fastargs; - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; - PyObject *data = NULL; - int digest_size = BLAKE2S_OUTBYTES; - Py_buffer key = {NULL, NULL}; - Py_buffer salt = {NULL, NULL}; - Py_buffer person = {NULL, NULL}; - int fanout = 1; - int depth = 1; - unsigned long leaf_size = 0; - unsigned long long node_offset = 0; - int node_depth = 0; - int inner_size = 0; - int last_node = 0; - int usedforsecurity = 1; - - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); - if (!fastargs) { - goto exit; - } - if (nargs < 1) { - goto skip_optional_posonly; - } - noptargs--; - data = fastargs[0]; -skip_optional_posonly: - if (!noptargs) { - goto skip_optional_kwonly; - } - if (fastargs[1]) { - digest_size = PyLong_AsInt(fastargs[1]); - if (digest_size == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - if (fastargs[2]) { - if (PyObject_GetBuffer(fastargs[2], &key, PyBUF_SIMPLE) != 0) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - if (fastargs[3]) { - if (PyObject_GetBuffer(fastargs[3], &salt, PyBUF_SIMPLE) != 0) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - if (fastargs[4]) { - if (PyObject_GetBuffer(fastargs[4], &person, PyBUF_SIMPLE) != 0) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - if (fastargs[5]) { - fanout = PyLong_AsInt(fastargs[5]); - if (fanout == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - if (fastargs[6]) { - depth = PyLong_AsInt(fastargs[6]); - if (depth == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - if (fastargs[7]) { - if (!_PyLong_UnsignedLong_Converter(fastargs[7], &leaf_size)) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - if (fastargs[8]) { - if (!_PyLong_UnsignedLongLong_Converter(fastargs[8], &node_offset)) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - if (fastargs[9]) { - node_depth = PyLong_AsInt(fastargs[9]); - if (node_depth == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - if (fastargs[10]) { - inner_size = PyLong_AsInt(fastargs[10]); - if (inner_size == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - if (fastargs[11]) { - last_node = PyObject_IsTrue(fastargs[11]); - if (last_node < 0) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_kwonly; - } - } - usedforsecurity = PyObject_IsTrue(fastargs[12]); - if (usedforsecurity < 0) { - goto exit; - } -skip_optional_kwonly: - return_value = py_blake2s_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); - -exit: - /* Cleanup for key */ - if (key.obj) { - PyBuffer_Release(&key); - } - /* Cleanup for salt */ - if (salt.obj) { - PyBuffer_Release(&salt); - } - /* Cleanup for person */ - if (person.obj) { - PyBuffer_Release(&person); - } - - return return_value; -} - -PyDoc_STRVAR(_blake2_blake2s_copy__doc__, -"copy($self, /)\n" -"--\n" -"\n" -"Return a copy of the hash object."); - -#define _BLAKE2_BLAKE2S_COPY_METHODDEF \ - {"copy", (PyCFunction)_blake2_blake2s_copy, METH_NOARGS, _blake2_blake2s_copy__doc__}, - -static PyObject * -_blake2_blake2s_copy_impl(BLAKE2sObject *self); - -static PyObject * -_blake2_blake2s_copy(BLAKE2sObject *self, PyObject *Py_UNUSED(ignored)) -{ - return _blake2_blake2s_copy_impl(self); -} - -PyDoc_STRVAR(_blake2_blake2s_update__doc__, -"update($self, data, /)\n" -"--\n" -"\n" -"Update this hash object\'s state with the provided bytes-like object."); - -#define _BLAKE2_BLAKE2S_UPDATE_METHODDEF \ - {"update", (PyCFunction)_blake2_blake2s_update, METH_O, _blake2_blake2s_update__doc__}, - -PyDoc_STRVAR(_blake2_blake2s_digest__doc__, -"digest($self, /)\n" -"--\n" -"\n" -"Return the digest value as a bytes object."); - -#define _BLAKE2_BLAKE2S_DIGEST_METHODDEF \ - {"digest", (PyCFunction)_blake2_blake2s_digest, METH_NOARGS, _blake2_blake2s_digest__doc__}, - -static PyObject * -_blake2_blake2s_digest_impl(BLAKE2sObject *self); - -static PyObject * -_blake2_blake2s_digest(BLAKE2sObject *self, PyObject *Py_UNUSED(ignored)) -{ - return _blake2_blake2s_digest_impl(self); -} - -PyDoc_STRVAR(_blake2_blake2s_hexdigest__doc__, -"hexdigest($self, /)\n" -"--\n" -"\n" -"Return the digest value as a string of hexadecimal digits."); - -#define _BLAKE2_BLAKE2S_HEXDIGEST_METHODDEF \ - {"hexdigest", (PyCFunction)_blake2_blake2s_hexdigest, METH_NOARGS, _blake2_blake2s_hexdigest__doc__}, - -static PyObject * -_blake2_blake2s_hexdigest_impl(BLAKE2sObject *self); - -static PyObject * -_blake2_blake2s_hexdigest(BLAKE2sObject *self, PyObject *Py_UNUSED(ignored)) -{ - return _blake2_blake2s_hexdigest_impl(self); -} -/*[clinic end generated code: output=24690e4e2586cafd input=a9049054013a1b77]*/ diff --git a/Modules/_blake2/impl/blake2-config.h b/Modules/_blake2/impl/blake2-config.h deleted file mode 100644 index c09cb4bcf06723..00000000000000 --- a/Modules/_blake2/impl/blake2-config.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ -#pragma once -#ifndef __BLAKE2_CONFIG_H__ -#define __BLAKE2_CONFIG_H__ - -#if defined(__SSE2__) -#define HAVE_SSE2 -#endif - -#if defined(__SSSE3__) -#define HAVE_SSSE3 -#endif - -#if defined(__SSE4_1__) -#define HAVE_SSE4_1 -#endif - -#if defined(__AVX__) -#define HAVE_AVX -#endif - -#if defined(__XOP__) -#define HAVE_XOP -#endif - - -#ifdef HAVE_AVX2 -#ifndef HAVE_AVX -#define HAVE_AVX -#endif -#endif - -#ifdef HAVE_XOP -#ifndef HAVE_AVX -#define HAVE_AVX -#endif -#endif - -#ifdef HAVE_AVX -#ifndef HAVE_SSE4_1 -#define HAVE_SSE4_1 -#endif -#endif - -#ifdef HAVE_SSE4_1 -#ifndef HAVE_SSSE3 -#define HAVE_SSSE3 -#endif -#endif - -#ifdef HAVE_SSSE3 -#define HAVE_SSE2 -#endif - -#if !defined(HAVE_SSE2) -#error "This code requires at least SSE2." -#endif - -#endif - diff --git a/Modules/_blake2/impl/blake2-impl.h b/Modules/_blake2/impl/blake2-impl.h deleted file mode 100644 index 9d2fbb72fc1c03..00000000000000 --- a/Modules/_blake2/impl/blake2-impl.h +++ /dev/null @@ -1,162 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ -#pragma once -#ifndef __BLAKE2_IMPL_H__ -#define __BLAKE2_IMPL_H__ - -#if defined(_WIN32) || defined(WIN32) -#include -#endif - -#include -#include -#include - -#define BLAKE2_IMPL_CAT(x,y) x ## y -#define BLAKE2_IMPL_EVAL(x,y) BLAKE2_IMPL_CAT(x,y) -#define BLAKE2_IMPL_NAME(fun) BLAKE2_IMPL_EVAL(fun, SUFFIX) - -static inline uint32_t load32( const void *src ) -{ -#if defined(NATIVE_LITTLE_ENDIAN) - uint32_t w; - memcpy( &w, src, sizeof( w ) ); - return w; -#else - const uint8_t *p = ( uint8_t * )src; - uint32_t w = *p++; - w |= ( uint32_t )( *p++ ) << 8; - w |= ( uint32_t )( *p++ ) << 16; - w |= ( uint32_t )( *p++ ) << 24; - return w; -#endif -} - -static inline uint64_t load64( const void *src ) -{ -#if defined(NATIVE_LITTLE_ENDIAN) - uint64_t w; - memcpy( &w, src, sizeof( w ) ); - return w; -#else - const uint8_t *p = ( uint8_t * )src; - uint64_t w = *p++; - w |= ( uint64_t )( *p++ ) << 8; - w |= ( uint64_t )( *p++ ) << 16; - w |= ( uint64_t )( *p++ ) << 24; - w |= ( uint64_t )( *p++ ) << 32; - w |= ( uint64_t )( *p++ ) << 40; - w |= ( uint64_t )( *p++ ) << 48; - w |= ( uint64_t )( *p++ ) << 56; - return w; -#endif -} - -static inline void store32( void *dst, uint32_t w ) -{ -#if defined(NATIVE_LITTLE_ENDIAN) - memcpy( dst, &w, sizeof( w ) ); -#else - uint8_t *p = ( uint8_t * )dst; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; -#endif -} - -static inline void store64( void *dst, uint64_t w ) -{ -#if defined(NATIVE_LITTLE_ENDIAN) - memcpy( dst, &w, sizeof( w ) ); -#else - uint8_t *p = ( uint8_t * )dst; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; -#endif -} - -static inline uint64_t load48( const void *src ) -{ - const uint8_t *p = ( const uint8_t * )src; - uint64_t w = *p++; - w |= ( uint64_t )( *p++ ) << 8; - w |= ( uint64_t )( *p++ ) << 16; - w |= ( uint64_t )( *p++ ) << 24; - w |= ( uint64_t )( *p++ ) << 32; - w |= ( uint64_t )( *p++ ) << 40; - return w; -} - -static inline void store48( void *dst, uint64_t w ) -{ - uint8_t *p = ( uint8_t * )dst; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; -} - -static inline uint32_t rotl32( const uint32_t w, const unsigned c ) -{ - return ( w << c ) | ( w >> ( 32 - c ) ); -} - -static inline uint64_t rotl64( const uint64_t w, const unsigned c ) -{ - return ( w << c ) | ( w >> ( 64 - c ) ); -} - -static inline uint32_t rotr32( const uint32_t w, const unsigned c ) -{ - return ( w >> c ) | ( w << ( 32 - c ) ); -} - -static inline uint64_t rotr64( const uint64_t w, const unsigned c ) -{ - return ( w >> c ) | ( w << ( 64 - c ) ); -} - -/* prevents compiler optimizing out memset() */ -static inline void secure_zero_memory(void *v, size_t n) -{ -#if defined(_WIN32) || defined(WIN32) - SecureZeroMemory(v, n); -#elif defined(__hpux) - static void *(*const volatile memset_v)(void *, int, size_t) = &memset; - memset_v(v, 0, n); -#else -// prioritize first the general C11 call -#if defined(HAVE_MEMSET_S) - memset_s(v, n, 0, n); -#elif defined(HAVE_EXPLICIT_BZERO) - explicit_bzero(v, n); -#elif defined(HAVE_EXPLICIT_MEMSET) - explicit_memset(v, 0, n); -#else - memset(v, 0, n); - __asm__ __volatile__("" :: "r"(v) : "memory"); -#endif -#endif -} - -#endif - diff --git a/Modules/_blake2/impl/blake2.h b/Modules/_blake2/impl/blake2.h deleted file mode 100644 index a08d82efefe09f..00000000000000 --- a/Modules/_blake2/impl/blake2.h +++ /dev/null @@ -1,177 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ -#pragma once -#ifndef __BLAKE2_H__ -#define __BLAKE2_H__ - -#include -#include - -#if defined(_WIN32) || defined(__CYGWIN__) - #define BLAKE2_DLL_IMPORT __declspec(dllimport) - #define BLAKE2_DLL_EXPORT __declspec(dllexport) - #define BLAKE2_DLL_PRIVATE -#elif __GNUC__ >= 4 - #define BLAKE2_DLL_IMPORT __attribute__ ((visibility ("default"))) - #define BLAKE2_DLL_EXPORT __attribute__ ((visibility ("default"))) - #define BLAKE2_DLL_PRIVATE __attribute__ ((visibility ("hidden"))) -#else - #define BLAKE2_DLL_IMPORT - #define BLAKE2_DLL_EXPORT - #define BLAKE2_DLL_PRIVATE -#endif - -#if defined(BLAKE2_DLL) - #if defined(BLAKE2_DLL_EXPORTS) // defined if we are building the DLL - #define BLAKE2_API BLAKE2_DLL_EXPORT - #else - #define BLAKE2_API BLAKE2_DLL_IMPORT - #endif - #define BLAKE2_PRIVATE BLAKE2_DLL_PRIVATE // must only be used by hidden logic -#else - #define BLAKE2_API - #define BLAKE2_PRIVATE -#endif - -#if defined(__cplusplus) -extern "C" { -#elif defined(_MSC_VER) && !defined(inline) -#define inline __inline -#endif - - enum blake2s_constant - { - BLAKE2S_BLOCKBYTES = 64, - BLAKE2S_OUTBYTES = 32, - BLAKE2S_KEYBYTES = 32, - BLAKE2S_SALTBYTES = 8, - BLAKE2S_PERSONALBYTES = 8 - }; - - enum blake2b_constant - { - BLAKE2B_BLOCKBYTES = 128, - BLAKE2B_OUTBYTES = 64, - BLAKE2B_KEYBYTES = 64, - BLAKE2B_SALTBYTES = 16, - BLAKE2B_PERSONALBYTES = 16 - }; - -#pragma pack(push, 1) - typedef struct __blake2s_param - { - uint8_t digest_length; // 1 - uint8_t key_length; // 2 - uint8_t fanout; // 3 - uint8_t depth; // 4 - uint32_t leaf_length; // 8 - uint8_t node_offset[6];// 14 - uint8_t node_depth; // 15 - uint8_t inner_length; // 16 - // uint8_t reserved[0]; - uint8_t salt[BLAKE2S_SALTBYTES]; // 24 - uint8_t personal[BLAKE2S_PERSONALBYTES]; // 32 - } blake2s_param; - - typedef struct __blake2s_state - { - uint32_t h[8]; - uint32_t t[2]; - uint32_t f[2]; - uint8_t buf[2 * BLAKE2S_BLOCKBYTES]; - uint32_t buflen; - uint8_t outlen; - uint8_t last_node; - } blake2s_state; - - typedef struct __blake2b_param - { - uint8_t digest_length; // 1 - uint8_t key_length; // 2 - uint8_t fanout; // 3 - uint8_t depth; // 4 - uint32_t leaf_length; // 8 - uint64_t node_offset; // 16 - uint8_t node_depth; // 17 - uint8_t inner_length; // 18 - uint8_t reserved[14]; // 32 - uint8_t salt[BLAKE2B_SALTBYTES]; // 48 - uint8_t personal[BLAKE2B_PERSONALBYTES]; // 64 - } blake2b_param; - - typedef struct __blake2b_state - { - uint64_t h[8]; - uint64_t t[2]; - uint64_t f[2]; - uint8_t buf[2 * BLAKE2B_BLOCKBYTES]; - uint32_t buflen; - uint8_t outlen; - uint8_t last_node; - } blake2b_state; - - typedef struct __blake2sp_state - { - blake2s_state S[8][1]; - blake2s_state R[1]; - uint8_t buf[8 * BLAKE2S_BLOCKBYTES]; - uint32_t buflen; - uint8_t outlen; - } blake2sp_state; - - typedef struct __blake2bp_state - { - blake2b_state S[4][1]; - blake2b_state R[1]; - uint8_t buf[4 * BLAKE2B_BLOCKBYTES]; - uint32_t buflen; - uint8_t outlen; - } blake2bp_state; -#pragma pack(pop) - - // Streaming API - BLAKE2_API int blake2s_init( blake2s_state *S, size_t outlen ); - BLAKE2_API int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ); - BLAKE2_API int blake2s_init_param( blake2s_state *S, const blake2s_param *P ); - BLAKE2_API int blake2s_update( blake2s_state *S, const uint8_t *in, size_t inlen ); - BLAKE2_API int blake2s_final( blake2s_state *S, uint8_t *out, size_t outlen ); - - BLAKE2_API int blake2b_init( blake2b_state *S, size_t outlen ); - BLAKE2_API int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ); - BLAKE2_API int blake2b_init_param( blake2b_state *S, const blake2b_param *P ); - BLAKE2_API int blake2b_update( blake2b_state *S, const uint8_t *in, size_t inlen ); - BLAKE2_API int blake2b_final( blake2b_state *S, uint8_t *out, size_t outlen ); - - BLAKE2_API int blake2sp_init( blake2sp_state *S, size_t outlen ); - BLAKE2_API int blake2sp_init_key( blake2sp_state *S, size_t outlen, const void *key, size_t keylen ); - BLAKE2_API int blake2sp_update( blake2sp_state *S, const uint8_t *in, size_t inlen ); - BLAKE2_API int blake2sp_final( blake2sp_state *S, uint8_t *out, size_t outlen ); - - BLAKE2_API int blake2bp_init( blake2bp_state *S, size_t outlen ); - BLAKE2_API int blake2bp_init_key( blake2bp_state *S, size_t outlen, const void *key, size_t keylen ); - BLAKE2_API int blake2bp_update( blake2bp_state *S, const uint8_t *in, size_t inlen ); - BLAKE2_API int blake2bp_final( blake2bp_state *S, uint8_t *out, size_t outlen ); - - // Simple API - BLAKE2_API int blake2s( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); - BLAKE2_API int blake2b( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); - - BLAKE2_API int blake2sp( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); - BLAKE2_API int blake2bp( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); - -#if defined(__cplusplus) -} -#endif - -#endif - diff --git a/Modules/_blake2/impl/blake2b-load-sse2.h b/Modules/_blake2/impl/blake2b-load-sse2.h deleted file mode 100644 index 1ba153c87d7352..00000000000000 --- a/Modules/_blake2/impl/blake2b-load-sse2.h +++ /dev/null @@ -1,68 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ -#pragma once -#ifndef __BLAKE2B_LOAD_SSE2_H__ -#define __BLAKE2B_LOAD_SSE2_H__ - -#define LOAD_MSG_0_1(b0, b1) b0 = _mm_set_epi64x(m2, m0); b1 = _mm_set_epi64x(m6, m4) -#define LOAD_MSG_0_2(b0, b1) b0 = _mm_set_epi64x(m3, m1); b1 = _mm_set_epi64x(m7, m5) -#define LOAD_MSG_0_3(b0, b1) b0 = _mm_set_epi64x(m10, m8); b1 = _mm_set_epi64x(m14, m12) -#define LOAD_MSG_0_4(b0, b1) b0 = _mm_set_epi64x(m11, m9); b1 = _mm_set_epi64x(m15, m13) -#define LOAD_MSG_1_1(b0, b1) b0 = _mm_set_epi64x(m4, m14); b1 = _mm_set_epi64x(m13, m9) -#define LOAD_MSG_1_2(b0, b1) b0 = _mm_set_epi64x(m8, m10); b1 = _mm_set_epi64x(m6, m15) -#define LOAD_MSG_1_3(b0, b1) b0 = _mm_set_epi64x(m0, m1); b1 = _mm_set_epi64x(m5, m11) -#define LOAD_MSG_1_4(b0, b1) b0 = _mm_set_epi64x(m2, m12); b1 = _mm_set_epi64x(m3, m7) -#define LOAD_MSG_2_1(b0, b1) b0 = _mm_set_epi64x(m12, m11); b1 = _mm_set_epi64x(m15, m5) -#define LOAD_MSG_2_2(b0, b1) b0 = _mm_set_epi64x(m0, m8); b1 = _mm_set_epi64x(m13, m2) -#define LOAD_MSG_2_3(b0, b1) b0 = _mm_set_epi64x(m3, m10); b1 = _mm_set_epi64x(m9, m7) -#define LOAD_MSG_2_4(b0, b1) b0 = _mm_set_epi64x(m6, m14); b1 = _mm_set_epi64x(m4, m1) -#define LOAD_MSG_3_1(b0, b1) b0 = _mm_set_epi64x(m3, m7); b1 = _mm_set_epi64x(m11, m13) -#define LOAD_MSG_3_2(b0, b1) b0 = _mm_set_epi64x(m1, m9); b1 = _mm_set_epi64x(m14, m12) -#define LOAD_MSG_3_3(b0, b1) b0 = _mm_set_epi64x(m5, m2); b1 = _mm_set_epi64x(m15, m4) -#define LOAD_MSG_3_4(b0, b1) b0 = _mm_set_epi64x(m10, m6); b1 = _mm_set_epi64x(m8, m0) -#define LOAD_MSG_4_1(b0, b1) b0 = _mm_set_epi64x(m5, m9); b1 = _mm_set_epi64x(m10, m2) -#define LOAD_MSG_4_2(b0, b1) b0 = _mm_set_epi64x(m7, m0); b1 = _mm_set_epi64x(m15, m4) -#define LOAD_MSG_4_3(b0, b1) b0 = _mm_set_epi64x(m11, m14); b1 = _mm_set_epi64x(m3, m6) -#define LOAD_MSG_4_4(b0, b1) b0 = _mm_set_epi64x(m12, m1); b1 = _mm_set_epi64x(m13, m8) -#define LOAD_MSG_5_1(b0, b1) b0 = _mm_set_epi64x(m6, m2); b1 = _mm_set_epi64x(m8, m0) -#define LOAD_MSG_5_2(b0, b1) b0 = _mm_set_epi64x(m10, m12); b1 = _mm_set_epi64x(m3, m11) -#define LOAD_MSG_5_3(b0, b1) b0 = _mm_set_epi64x(m7, m4); b1 = _mm_set_epi64x(m1, m15) -#define LOAD_MSG_5_4(b0, b1) b0 = _mm_set_epi64x(m5, m13); b1 = _mm_set_epi64x(m9, m14) -#define LOAD_MSG_6_1(b0, b1) b0 = _mm_set_epi64x(m1, m12); b1 = _mm_set_epi64x(m4, m14) -#define LOAD_MSG_6_2(b0, b1) b0 = _mm_set_epi64x(m15, m5); b1 = _mm_set_epi64x(m10, m13) -#define LOAD_MSG_6_3(b0, b1) b0 = _mm_set_epi64x(m6, m0); b1 = _mm_set_epi64x(m8, m9) -#define LOAD_MSG_6_4(b0, b1) b0 = _mm_set_epi64x(m3, m7); b1 = _mm_set_epi64x(m11, m2) -#define LOAD_MSG_7_1(b0, b1) b0 = _mm_set_epi64x(m7, m13); b1 = _mm_set_epi64x(m3, m12) -#define LOAD_MSG_7_2(b0, b1) b0 = _mm_set_epi64x(m14, m11); b1 = _mm_set_epi64x(m9, m1) -#define LOAD_MSG_7_3(b0, b1) b0 = _mm_set_epi64x(m15, m5); b1 = _mm_set_epi64x(m2, m8) -#define LOAD_MSG_7_4(b0, b1) b0 = _mm_set_epi64x(m4, m0); b1 = _mm_set_epi64x(m10, m6) -#define LOAD_MSG_8_1(b0, b1) b0 = _mm_set_epi64x(m14, m6); b1 = _mm_set_epi64x(m0, m11) -#define LOAD_MSG_8_2(b0, b1) b0 = _mm_set_epi64x(m9, m15); b1 = _mm_set_epi64x(m8, m3) -#define LOAD_MSG_8_3(b0, b1) b0 = _mm_set_epi64x(m13, m12); b1 = _mm_set_epi64x(m10, m1) -#define LOAD_MSG_8_4(b0, b1) b0 = _mm_set_epi64x(m7, m2); b1 = _mm_set_epi64x(m5, m4) -#define LOAD_MSG_9_1(b0, b1) b0 = _mm_set_epi64x(m8, m10); b1 = _mm_set_epi64x(m1, m7) -#define LOAD_MSG_9_2(b0, b1) b0 = _mm_set_epi64x(m4, m2); b1 = _mm_set_epi64x(m5, m6) -#define LOAD_MSG_9_3(b0, b1) b0 = _mm_set_epi64x(m9, m15); b1 = _mm_set_epi64x(m13, m3) -#define LOAD_MSG_9_4(b0, b1) b0 = _mm_set_epi64x(m14, m11); b1 = _mm_set_epi64x(m0, m12) -#define LOAD_MSG_10_1(b0, b1) b0 = _mm_set_epi64x(m2, m0); b1 = _mm_set_epi64x(m6, m4) -#define LOAD_MSG_10_2(b0, b1) b0 = _mm_set_epi64x(m3, m1); b1 = _mm_set_epi64x(m7, m5) -#define LOAD_MSG_10_3(b0, b1) b0 = _mm_set_epi64x(m10, m8); b1 = _mm_set_epi64x(m14, m12) -#define LOAD_MSG_10_4(b0, b1) b0 = _mm_set_epi64x(m11, m9); b1 = _mm_set_epi64x(m15, m13) -#define LOAD_MSG_11_1(b0, b1) b0 = _mm_set_epi64x(m4, m14); b1 = _mm_set_epi64x(m13, m9) -#define LOAD_MSG_11_2(b0, b1) b0 = _mm_set_epi64x(m8, m10); b1 = _mm_set_epi64x(m6, m15) -#define LOAD_MSG_11_3(b0, b1) b0 = _mm_set_epi64x(m0, m1); b1 = _mm_set_epi64x(m5, m11) -#define LOAD_MSG_11_4(b0, b1) b0 = _mm_set_epi64x(m2, m12); b1 = _mm_set_epi64x(m3, m7) - - -#endif - diff --git a/Modules/_blake2/impl/blake2b-load-sse41.h b/Modules/_blake2/impl/blake2b-load-sse41.h deleted file mode 100644 index f6c1bc8393f167..00000000000000 --- a/Modules/_blake2/impl/blake2b-load-sse41.h +++ /dev/null @@ -1,402 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ -#pragma once -#ifndef __BLAKE2B_LOAD_SSE41_H__ -#define __BLAKE2B_LOAD_SSE41_H__ - -#define LOAD_MSG_0_1(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m0, m1); \ -b1 = _mm_unpacklo_epi64(m2, m3); \ -} while(0) - - -#define LOAD_MSG_0_2(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m0, m1); \ -b1 = _mm_unpackhi_epi64(m2, m3); \ -} while(0) - - -#define LOAD_MSG_0_3(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m4, m5); \ -b1 = _mm_unpacklo_epi64(m6, m7); \ -} while(0) - - -#define LOAD_MSG_0_4(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m4, m5); \ -b1 = _mm_unpackhi_epi64(m6, m7); \ -} while(0) - - -#define LOAD_MSG_1_1(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m7, m2); \ -b1 = _mm_unpackhi_epi64(m4, m6); \ -} while(0) - - -#define LOAD_MSG_1_2(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m5, m4); \ -b1 = _mm_alignr_epi8(m3, m7, 8); \ -} while(0) - - -#define LOAD_MSG_1_3(b0, b1) \ -do \ -{ \ -b0 = _mm_shuffle_epi32(m0, _MM_SHUFFLE(1,0,3,2)); \ -b1 = _mm_unpackhi_epi64(m5, m2); \ -} while(0) - - -#define LOAD_MSG_1_4(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m6, m1); \ -b1 = _mm_unpackhi_epi64(m3, m1); \ -} while(0) - - -#define LOAD_MSG_2_1(b0, b1) \ -do \ -{ \ -b0 = _mm_alignr_epi8(m6, m5, 8); \ -b1 = _mm_unpackhi_epi64(m2, m7); \ -} while(0) - - -#define LOAD_MSG_2_2(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m4, m0); \ -b1 = _mm_blend_epi16(m1, m6, 0xF0); \ -} while(0) - - -#define LOAD_MSG_2_3(b0, b1) \ -do \ -{ \ -b0 = _mm_blend_epi16(m5, m1, 0xF0); \ -b1 = _mm_unpackhi_epi64(m3, m4); \ -} while(0) - - -#define LOAD_MSG_2_4(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m7, m3); \ -b1 = _mm_alignr_epi8(m2, m0, 8); \ -} while(0) - - -#define LOAD_MSG_3_1(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m3, m1); \ -b1 = _mm_unpackhi_epi64(m6, m5); \ -} while(0) - - -#define LOAD_MSG_3_2(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m4, m0); \ -b1 = _mm_unpacklo_epi64(m6, m7); \ -} while(0) - - -#define LOAD_MSG_3_3(b0, b1) \ -do \ -{ \ -b0 = _mm_blend_epi16(m1, m2, 0xF0); \ -b1 = _mm_blend_epi16(m2, m7, 0xF0); \ -} while(0) - - -#define LOAD_MSG_3_4(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m3, m5); \ -b1 = _mm_unpacklo_epi64(m0, m4); \ -} while(0) - - -#define LOAD_MSG_4_1(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m4, m2); \ -b1 = _mm_unpacklo_epi64(m1, m5); \ -} while(0) - - -#define LOAD_MSG_4_2(b0, b1) \ -do \ -{ \ -b0 = _mm_blend_epi16(m0, m3, 0xF0); \ -b1 = _mm_blend_epi16(m2, m7, 0xF0); \ -} while(0) - - -#define LOAD_MSG_4_3(b0, b1) \ -do \ -{ \ -b0 = _mm_blend_epi16(m7, m5, 0xF0); \ -b1 = _mm_blend_epi16(m3, m1, 0xF0); \ -} while(0) - - -#define LOAD_MSG_4_4(b0, b1) \ -do \ -{ \ -b0 = _mm_alignr_epi8(m6, m0, 8); \ -b1 = _mm_blend_epi16(m4, m6, 0xF0); \ -} while(0) - - -#define LOAD_MSG_5_1(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m1, m3); \ -b1 = _mm_unpacklo_epi64(m0, m4); \ -} while(0) - - -#define LOAD_MSG_5_2(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m6, m5); \ -b1 = _mm_unpackhi_epi64(m5, m1); \ -} while(0) - - -#define LOAD_MSG_5_3(b0, b1) \ -do \ -{ \ -b0 = _mm_blend_epi16(m2, m3, 0xF0); \ -b1 = _mm_unpackhi_epi64(m7, m0); \ -} while(0) - - -#define LOAD_MSG_5_4(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m6, m2); \ -b1 = _mm_blend_epi16(m7, m4, 0xF0); \ -} while(0) - - -#define LOAD_MSG_6_1(b0, b1) \ -do \ -{ \ -b0 = _mm_blend_epi16(m6, m0, 0xF0); \ -b1 = _mm_unpacklo_epi64(m7, m2); \ -} while(0) - - -#define LOAD_MSG_6_2(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m2, m7); \ -b1 = _mm_alignr_epi8(m5, m6, 8); \ -} while(0) - - -#define LOAD_MSG_6_3(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m0, m3); \ -b1 = _mm_shuffle_epi32(m4, _MM_SHUFFLE(1,0,3,2)); \ -} while(0) - - -#define LOAD_MSG_6_4(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m3, m1); \ -b1 = _mm_blend_epi16(m1, m5, 0xF0); \ -} while(0) - - -#define LOAD_MSG_7_1(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m6, m3); \ -b1 = _mm_blend_epi16(m6, m1, 0xF0); \ -} while(0) - - -#define LOAD_MSG_7_2(b0, b1) \ -do \ -{ \ -b0 = _mm_alignr_epi8(m7, m5, 8); \ -b1 = _mm_unpackhi_epi64(m0, m4); \ -} while(0) - - -#define LOAD_MSG_7_3(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m2, m7); \ -b1 = _mm_unpacklo_epi64(m4, m1); \ -} while(0) - - -#define LOAD_MSG_7_4(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m0, m2); \ -b1 = _mm_unpacklo_epi64(m3, m5); \ -} while(0) - - -#define LOAD_MSG_8_1(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m3, m7); \ -b1 = _mm_alignr_epi8(m0, m5, 8); \ -} while(0) - - -#define LOAD_MSG_8_2(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m7, m4); \ -b1 = _mm_alignr_epi8(m4, m1, 8); \ -} while(0) - - -#define LOAD_MSG_8_3(b0, b1) \ -do \ -{ \ -b0 = m6; \ -b1 = _mm_alignr_epi8(m5, m0, 8); \ -} while(0) - - -#define LOAD_MSG_8_4(b0, b1) \ -do \ -{ \ -b0 = _mm_blend_epi16(m1, m3, 0xF0); \ -b1 = m2; \ -} while(0) - - -#define LOAD_MSG_9_1(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m5, m4); \ -b1 = _mm_unpackhi_epi64(m3, m0); \ -} while(0) - - -#define LOAD_MSG_9_2(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m1, m2); \ -b1 = _mm_blend_epi16(m3, m2, 0xF0); \ -} while(0) - - -#define LOAD_MSG_9_3(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m7, m4); \ -b1 = _mm_unpackhi_epi64(m1, m6); \ -} while(0) - - -#define LOAD_MSG_9_4(b0, b1) \ -do \ -{ \ -b0 = _mm_alignr_epi8(m7, m5, 8); \ -b1 = _mm_unpacklo_epi64(m6, m0); \ -} while(0) - - -#define LOAD_MSG_10_1(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m0, m1); \ -b1 = _mm_unpacklo_epi64(m2, m3); \ -} while(0) - - -#define LOAD_MSG_10_2(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m0, m1); \ -b1 = _mm_unpackhi_epi64(m2, m3); \ -} while(0) - - -#define LOAD_MSG_10_3(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m4, m5); \ -b1 = _mm_unpacklo_epi64(m6, m7); \ -} while(0) - - -#define LOAD_MSG_10_4(b0, b1) \ -do \ -{ \ -b0 = _mm_unpackhi_epi64(m4, m5); \ -b1 = _mm_unpackhi_epi64(m6, m7); \ -} while(0) - - -#define LOAD_MSG_11_1(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m7, m2); \ -b1 = _mm_unpackhi_epi64(m4, m6); \ -} while(0) - - -#define LOAD_MSG_11_2(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m5, m4); \ -b1 = _mm_alignr_epi8(m3, m7, 8); \ -} while(0) - - -#define LOAD_MSG_11_3(b0, b1) \ -do \ -{ \ -b0 = _mm_shuffle_epi32(m0, _MM_SHUFFLE(1,0,3,2)); \ -b1 = _mm_unpackhi_epi64(m5, m2); \ -} while(0) - - -#define LOAD_MSG_11_4(b0, b1) \ -do \ -{ \ -b0 = _mm_unpacklo_epi64(m6, m1); \ -b1 = _mm_unpackhi_epi64(m3, m1); \ -} while(0) - - -#endif - diff --git a/Modules/_blake2/impl/blake2b-ref.c b/Modules/_blake2/impl/blake2b-ref.c deleted file mode 100644 index e58c43659d9cc8..00000000000000 --- a/Modules/_blake2/impl/blake2b-ref.c +++ /dev/null @@ -1,379 +0,0 @@ -/* - BLAKE2 reference source code package - reference C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ - -#include -#include -#include - -#include "blake2.h" -#include "blake2-impl.h" - -static const uint64_t blake2b_IV[8] = -{ - 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, - 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, - 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, - 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL -}; - -static const uint8_t blake2b_sigma[12][16] = -{ - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , - { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , - { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , - { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , - { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , - { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , - { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , - { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , - { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } -}; - - -static inline int blake2b_set_lastnode( blake2b_state *S ) -{ - S->f[1] = ~0ULL; - return 0; -} - -static inline int blake2b_clear_lastnode( blake2b_state *S ) -{ - S->f[1] = 0ULL; - return 0; -} - -/* Some helper functions, not necessarily useful */ -static inline int blake2b_set_lastblock( blake2b_state *S ) -{ - if( S->last_node ) blake2b_set_lastnode( S ); - - S->f[0] = ~0ULL; - return 0; -} - -static inline int blake2b_clear_lastblock( blake2b_state *S ) -{ - if( S->last_node ) blake2b_clear_lastnode( S ); - - S->f[0] = 0ULL; - return 0; -} - -static inline int blake2b_increment_counter( blake2b_state *S, const uint64_t inc ) -{ - S->t[0] += inc; - S->t[1] += ( S->t[0] < inc ); - return 0; -} - - - -// Parameter-related functions -static inline int blake2b_param_set_digest_length( blake2b_param *P, const uint8_t digest_length ) -{ - P->digest_length = digest_length; - return 0; -} - -static inline int blake2b_param_set_fanout( blake2b_param *P, const uint8_t fanout ) -{ - P->fanout = fanout; - return 0; -} - -static inline int blake2b_param_set_max_depth( blake2b_param *P, const uint8_t depth ) -{ - P->depth = depth; - return 0; -} - -static inline int blake2b_param_set_leaf_length( blake2b_param *P, const uint32_t leaf_length ) -{ - store32( &P->leaf_length, leaf_length ); - return 0; -} - -static inline int blake2b_param_set_node_offset( blake2b_param *P, const uint64_t node_offset ) -{ - store64( &P->node_offset, node_offset ); - return 0; -} - -static inline int blake2b_param_set_node_depth( blake2b_param *P, const uint8_t node_depth ) -{ - P->node_depth = node_depth; - return 0; -} - -static inline int blake2b_param_set_inner_length( blake2b_param *P, const uint8_t inner_length ) -{ - P->inner_length = inner_length; - return 0; -} - -static inline int blake2b_param_set_salt( blake2b_param *P, const uint8_t salt[BLAKE2B_SALTBYTES] ) -{ - memcpy( P->salt, salt, BLAKE2B_SALTBYTES ); - return 0; -} - -static inline int blake2b_param_set_personal( blake2b_param *P, const uint8_t personal[BLAKE2B_PERSONALBYTES] ) -{ - memcpy( P->personal, personal, BLAKE2B_PERSONALBYTES ); - return 0; -} - -static inline int blake2b_init0( blake2b_state *S ) -{ - memset( S, 0, sizeof( blake2b_state ) ); - - for( int i = 0; i < 8; ++i ) S->h[i] = blake2b_IV[i]; - - return 0; -} - -#if defined(__cplusplus) -extern "C" { -#endif - int blake2b_init( blake2b_state *S, size_t outlen ); - int blake2b_init_param( blake2b_state *S, const blake2b_param *P ); - int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ); - int blake2b_update( blake2b_state *S, const uint8_t *in, size_t inlen ); - int blake2b_final( blake2b_state *S, uint8_t *out, size_t outlen ); - int blake2b( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); -#if defined(__cplusplus) -} -#endif - -/* init xors IV with input parameter block */ -int blake2b_init_param( blake2b_state *S, const blake2b_param *P ) -{ - blake2b_init0( S ); - uint8_t *p = ( uint8_t * )( P ); - - /* IV XOR ParamBlock */ - for( size_t i = 0; i < 8; ++i ) - S->h[i] ^= load64( p + sizeof( S->h[i] ) * i ); - - S->outlen = P->digest_length; - return 0; -} - - - -int blake2b_init( blake2b_state *S, size_t outlen ) -{ - blake2b_param P[1]; - - if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; - - P->digest_length = ( uint8_t ) outlen; - P->key_length = 0; - P->fanout = 1; - P->depth = 1; - store32( &P->leaf_length, 0 ); - store64( &P->node_offset, 0 ); - P->node_depth = 0; - P->inner_length = 0; - memset( P->reserved, 0, sizeof( P->reserved ) ); - memset( P->salt, 0, sizeof( P->salt ) ); - memset( P->personal, 0, sizeof( P->personal ) ); - return blake2b_init_param( S, P ); -} - - -int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ) -{ - blake2b_param P[1]; - - if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; - - if ( !key || !keylen || keylen > BLAKE2B_KEYBYTES ) return -1; - - P->digest_length = ( uint8_t ) outlen; - P->key_length = ( uint8_t ) keylen; - P->fanout = 1; - P->depth = 1; - store32( &P->leaf_length, 0 ); - store64( &P->node_offset, 0 ); - P->node_depth = 0; - P->inner_length = 0; - memset( P->reserved, 0, sizeof( P->reserved ) ); - memset( P->salt, 0, sizeof( P->salt ) ); - memset( P->personal, 0, sizeof( P->personal ) ); - - if( blake2b_init_param( S, P ) < 0 ) return -1; - - { - uint8_t block[BLAKE2B_BLOCKBYTES]; - memset( block, 0, BLAKE2B_BLOCKBYTES ); - memcpy( block, key, keylen ); - blake2b_update( S, block, BLAKE2B_BLOCKBYTES ); - secure_zero_memory( block, BLAKE2B_BLOCKBYTES ); /* Burn the key from stack */ - } - return 0; -} - -static int blake2b_compress( blake2b_state *S, const uint8_t block[BLAKE2B_BLOCKBYTES] ) -{ - uint64_t m[16]; - uint64_t v[16]; - size_t i; - - for( i = 0; i < 16; ++i ) - m[i] = load64( block + i * sizeof( m[i] ) ); - - for( i = 0; i < 8; ++i ) - v[i] = S->h[i]; - - v[ 8] = blake2b_IV[0]; - v[ 9] = blake2b_IV[1]; - v[10] = blake2b_IV[2]; - v[11] = blake2b_IV[3]; - v[12] = S->t[0] ^ blake2b_IV[4]; - v[13] = S->t[1] ^ blake2b_IV[5]; - v[14] = S->f[0] ^ blake2b_IV[6]; - v[15] = S->f[1] ^ blake2b_IV[7]; -#define G(r,i,a,b,c,d) \ - do { \ - a = a + b + m[blake2b_sigma[r][2*i+0]]; \ - d = rotr64(d ^ a, 32); \ - c = c + d; \ - b = rotr64(b ^ c, 24); \ - a = a + b + m[blake2b_sigma[r][2*i+1]]; \ - d = rotr64(d ^ a, 16); \ - c = c + d; \ - b = rotr64(b ^ c, 63); \ - } while(0) -#define ROUND(r) \ - do { \ - G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \ - G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \ - G(r,2,v[ 2],v[ 6],v[10],v[14]); \ - G(r,3,v[ 3],v[ 7],v[11],v[15]); \ - G(r,4,v[ 0],v[ 5],v[10],v[15]); \ - G(r,5,v[ 1],v[ 6],v[11],v[12]); \ - G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \ - G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ - } while(0) - ROUND( 0 ); - ROUND( 1 ); - ROUND( 2 ); - ROUND( 3 ); - ROUND( 4 ); - ROUND( 5 ); - ROUND( 6 ); - ROUND( 7 ); - ROUND( 8 ); - ROUND( 9 ); - ROUND( 10 ); - ROUND( 11 ); - - for( i = 0; i < 8; ++i ) - S->h[i] = S->h[i] ^ v[i] ^ v[i + 8]; - -#undef G -#undef ROUND - return 0; -} - - -int blake2b_update( blake2b_state *S, const uint8_t *in, size_t inlen ) -{ - while( inlen > 0 ) - { - uint32_t left = S->buflen; - uint32_t fill = 2 * BLAKE2B_BLOCKBYTES - left; - - if( inlen > fill ) - { - memcpy( S->buf + left, in, fill ); // Fill buffer - S->buflen += fill; - blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); - blake2b_compress( S, S->buf ); // Compress - memcpy( S->buf, S->buf + BLAKE2B_BLOCKBYTES, BLAKE2B_BLOCKBYTES ); // Shift buffer left - S->buflen -= BLAKE2B_BLOCKBYTES; - in += fill; - inlen -= fill; - } - else // inlen <= fill - { - memcpy( S->buf + left, in, inlen ); - S->buflen += ( uint32_t ) inlen; // Be lazy, do not compress - in += inlen; - inlen -= inlen; - } - } - - return 0; -} - -int blake2b_final( blake2b_state *S, uint8_t *out, size_t outlen ) -{ - uint8_t buffer[BLAKE2B_OUTBYTES]; - size_t i; - - if(S->outlen != outlen) return -1; - - if( S->buflen > BLAKE2B_BLOCKBYTES ) - { - blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); - blake2b_compress( S, S->buf ); - S->buflen -= BLAKE2B_BLOCKBYTES; - memmove( S->buf, S->buf + BLAKE2B_BLOCKBYTES, S->buflen ); - } - - blake2b_increment_counter( S, S->buflen ); - blake2b_set_lastblock( S ); - memset( S->buf + S->buflen, 0, 2 * BLAKE2B_BLOCKBYTES - S->buflen ); /* Padding */ - blake2b_compress( S, S->buf ); - - for( i = 0; i < 8; ++i ) /* Output full hash to temp buffer */ - store64( buffer + sizeof( S->h[i] ) * i, S->h[i] ); - - memcpy( out, buffer, outlen ); - return 0; -} - -int blake2b( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ) -{ - blake2b_state S[1]; - - /* Verify parameters */ - if ( NULL == in && inlen > 0 ) return -1; - - if ( NULL == out ) return -1; - - if( NULL == key && keylen > 0 ) return -1; - - if( !outlen || outlen > BLAKE2B_OUTBYTES ) return -1; - - if( keylen > BLAKE2B_KEYBYTES ) return -1; - - if( keylen > 0 ) - { - if( blake2b_init_key( S, outlen, key, keylen ) < 0 ) return -1; - } - else - { - if( blake2b_init( S, outlen ) < 0 ) return -1; - } - - if( blake2b_update( S, ( uint8_t * )in, inlen ) < 0 ) return -1; - return blake2b_final( S, out, outlen ); -} - - diff --git a/Modules/_blake2/impl/blake2b-round.h b/Modules/_blake2/impl/blake2b-round.h deleted file mode 100644 index 5b452c4d63babe..00000000000000 --- a/Modules/_blake2/impl/blake2b-round.h +++ /dev/null @@ -1,160 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ -#pragma once -#ifndef __BLAKE2B_ROUND_H__ -#define __BLAKE2B_ROUND_H__ - -#define LOAD(p) _mm_load_si128( (__m128i *)(p) ) -#define STORE(p,r) _mm_store_si128((__m128i *)(p), r) - -#define LOADU(p) _mm_loadu_si128( (__m128i *)(p) ) -#define STOREU(p,r) _mm_storeu_si128((__m128i *)(p), r) - -#define TOF(reg) _mm_castsi128_ps((reg)) -#define TOI(reg) _mm_castps_si128((reg)) - -#define LIKELY(x) __builtin_expect((x),1) - - -/* Microarchitecture-specific macros */ -#ifndef HAVE_XOP -#ifdef HAVE_SSSE3 -#define _mm_roti_epi64(x, c) \ - (-(c) == 32) ? _mm_shuffle_epi32((x), _MM_SHUFFLE(2,3,0,1)) \ - : (-(c) == 24) ? _mm_shuffle_epi8((x), r24) \ - : (-(c) == 16) ? _mm_shuffle_epi8((x), r16) \ - : (-(c) == 63) ? _mm_xor_si128(_mm_srli_epi64((x), -(c)), _mm_add_epi64((x), (x))) \ - : _mm_xor_si128(_mm_srli_epi64((x), -(c)), _mm_slli_epi64((x), 64-(-(c)))) -#else -#define _mm_roti_epi64(r, c) _mm_xor_si128(_mm_srli_epi64( (r), -(c) ),_mm_slli_epi64( (r), 64-(-(c)) )) -#endif -#else -/* ... */ -#endif - - - -#define G1(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1) \ - row1l = _mm_add_epi64(_mm_add_epi64(row1l, b0), row2l); \ - row1h = _mm_add_epi64(_mm_add_epi64(row1h, b1), row2h); \ - \ - row4l = _mm_xor_si128(row4l, row1l); \ - row4h = _mm_xor_si128(row4h, row1h); \ - \ - row4l = _mm_roti_epi64(row4l, -32); \ - row4h = _mm_roti_epi64(row4h, -32); \ - \ - row3l = _mm_add_epi64(row3l, row4l); \ - row3h = _mm_add_epi64(row3h, row4h); \ - \ - row2l = _mm_xor_si128(row2l, row3l); \ - row2h = _mm_xor_si128(row2h, row3h); \ - \ - row2l = _mm_roti_epi64(row2l, -24); \ - row2h = _mm_roti_epi64(row2h, -24); \ - -#define G2(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1) \ - row1l = _mm_add_epi64(_mm_add_epi64(row1l, b0), row2l); \ - row1h = _mm_add_epi64(_mm_add_epi64(row1h, b1), row2h); \ - \ - row4l = _mm_xor_si128(row4l, row1l); \ - row4h = _mm_xor_si128(row4h, row1h); \ - \ - row4l = _mm_roti_epi64(row4l, -16); \ - row4h = _mm_roti_epi64(row4h, -16); \ - \ - row3l = _mm_add_epi64(row3l, row4l); \ - row3h = _mm_add_epi64(row3h, row4h); \ - \ - row2l = _mm_xor_si128(row2l, row3l); \ - row2h = _mm_xor_si128(row2h, row3h); \ - \ - row2l = _mm_roti_epi64(row2l, -63); \ - row2h = _mm_roti_epi64(row2h, -63); \ - -#if defined(HAVE_SSSE3) -#define DIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h) \ - t0 = _mm_alignr_epi8(row2h, row2l, 8); \ - t1 = _mm_alignr_epi8(row2l, row2h, 8); \ - row2l = t0; \ - row2h = t1; \ - \ - t0 = row3l; \ - row3l = row3h; \ - row3h = t0; \ - \ - t0 = _mm_alignr_epi8(row4h, row4l, 8); \ - t1 = _mm_alignr_epi8(row4l, row4h, 8); \ - row4l = t1; \ - row4h = t0; - -#define UNDIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h) \ - t0 = _mm_alignr_epi8(row2l, row2h, 8); \ - t1 = _mm_alignr_epi8(row2h, row2l, 8); \ - row2l = t0; \ - row2h = t1; \ - \ - t0 = row3l; \ - row3l = row3h; \ - row3h = t0; \ - \ - t0 = _mm_alignr_epi8(row4l, row4h, 8); \ - t1 = _mm_alignr_epi8(row4h, row4l, 8); \ - row4l = t1; \ - row4h = t0; -#else - -#define DIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h) \ - t0 = row4l;\ - t1 = row2l;\ - row4l = row3l;\ - row3l = row3h;\ - row3h = row4l;\ - row4l = _mm_unpackhi_epi64(row4h, _mm_unpacklo_epi64(t0, t0)); \ - row4h = _mm_unpackhi_epi64(t0, _mm_unpacklo_epi64(row4h, row4h)); \ - row2l = _mm_unpackhi_epi64(row2l, _mm_unpacklo_epi64(row2h, row2h)); \ - row2h = _mm_unpackhi_epi64(row2h, _mm_unpacklo_epi64(t1, t1)) - -#define UNDIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h) \ - t0 = row3l;\ - row3l = row3h;\ - row3h = t0;\ - t0 = row2l;\ - t1 = row4l;\ - row2l = _mm_unpackhi_epi64(row2h, _mm_unpacklo_epi64(row2l, row2l)); \ - row2h = _mm_unpackhi_epi64(t0, _mm_unpacklo_epi64(row2h, row2h)); \ - row4l = _mm_unpackhi_epi64(row4l, _mm_unpacklo_epi64(row4h, row4h)); \ - row4h = _mm_unpackhi_epi64(row4h, _mm_unpacklo_epi64(t1, t1)) - -#endif - -#if defined(HAVE_SSE4_1) -#include "blake2b-load-sse41.h" -#else -#include "blake2b-load-sse2.h" -#endif - -#define ROUND(r) \ - LOAD_MSG_ ##r ##_1(b0, b1); \ - G1(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1); \ - LOAD_MSG_ ##r ##_2(b0, b1); \ - G2(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1); \ - DIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h); \ - LOAD_MSG_ ##r ##_3(b0, b1); \ - G1(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1); \ - LOAD_MSG_ ##r ##_4(b0, b1); \ - G2(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1); \ - UNDIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h); - -#endif - diff --git a/Modules/_blake2/impl/blake2b.c b/Modules/_blake2/impl/blake2b.c deleted file mode 100644 index cef22838917d9d..00000000000000 --- a/Modules/_blake2/impl/blake2b.c +++ /dev/null @@ -1,436 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ - -#include -#include -#include - -#include "blake2.h" -#include "blake2-impl.h" - -#include "blake2-config.h" - -#if defined(_MSC_VER) -#include -#endif - -#if defined(HAVE_SSE2) -#include -// MSVC only defines _mm_set_epi64x for x86_64... -#if defined(_MSC_VER) && !defined(_M_X64) && !defined(__clang__) -static inline __m128i _mm_set_epi64x( const uint64_t u1, const uint64_t u0 ) -{ - return _mm_set_epi32( u1 >> 32, u1, u0 >> 32, u0 ); -} -#endif -#endif - -#if defined(HAVE_SSSE3) -#include -#endif -#if defined(HAVE_SSE4_1) -#include -#endif -#if defined(HAVE_AVX) -#include -#endif -#if defined(HAVE_XOP) && !defined(_MSC_VER) -#include -#endif - - - -#include "blake2b-round.h" - -static const uint64_t blake2b_IV[8] = -{ - 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, - 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, - 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, - 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL -}; - -static const uint8_t blake2b_sigma[12][16] = -{ - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , - { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , - { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , - { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , - { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , - { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , - { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , - { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , - { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } -}; - - -/* Some helper functions, not necessarily useful */ -static inline int blake2b_set_lastnode( blake2b_state *S ) -{ - S->f[1] = ~0ULL; - return 0; -} - -static inline int blake2b_clear_lastnode( blake2b_state *S ) -{ - S->f[1] = 0ULL; - return 0; -} - -static inline int blake2b_set_lastblock( blake2b_state *S ) -{ - if( S->last_node ) blake2b_set_lastnode( S ); - - S->f[0] = ~0ULL; - return 0; -} - -static inline int blake2b_clear_lastblock( blake2b_state *S ) -{ - if( S->last_node ) blake2b_clear_lastnode( S ); - - S->f[0] = 0ULL; - return 0; -} - - -static inline int blake2b_increment_counter( blake2b_state *S, const uint64_t inc ) -{ -#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) - // ADD/ADC chain - __uint128_t t = ( ( __uint128_t )S->t[1] << 64 ) | S->t[0]; - t += inc; - S->t[0] = ( uint64_t )( t >> 0 ); - S->t[1] = ( uint64_t )( t >> 64 ); -#else - S->t[0] += inc; - S->t[1] += ( S->t[0] < inc ); -#endif - return 0; -} - - -// Parameter-related functions -static inline int blake2b_param_set_digest_length( blake2b_param *P, const uint8_t digest_length ) -{ - P->digest_length = digest_length; - return 0; -} - -static inline int blake2b_param_set_fanout( blake2b_param *P, const uint8_t fanout ) -{ - P->fanout = fanout; - return 0; -} - -static inline int blake2b_param_set_max_depth( blake2b_param *P, const uint8_t depth ) -{ - P->depth = depth; - return 0; -} - -static inline int blake2b_param_set_leaf_length( blake2b_param *P, const uint32_t leaf_length ) -{ - P->leaf_length = leaf_length; - return 0; -} - -static inline int blake2b_param_set_node_offset( blake2b_param *P, const uint64_t node_offset ) -{ - P->node_offset = node_offset; - return 0; -} - -static inline int blake2b_param_set_node_depth( blake2b_param *P, const uint8_t node_depth ) -{ - P->node_depth = node_depth; - return 0; -} - -static inline int blake2b_param_set_inner_length( blake2b_param *P, const uint8_t inner_length ) -{ - P->inner_length = inner_length; - return 0; -} - -static inline int blake2b_param_set_salt( blake2b_param *P, const uint8_t salt[BLAKE2B_SALTBYTES] ) -{ - memcpy( P->salt, salt, BLAKE2B_SALTBYTES ); - return 0; -} - -static inline int blake2b_param_set_personal( blake2b_param *P, const uint8_t personal[BLAKE2B_PERSONALBYTES] ) -{ - memcpy( P->personal, personal, BLAKE2B_PERSONALBYTES ); - return 0; -} - -static inline int blake2b_init0( blake2b_state *S ) -{ - memset( S, 0, sizeof( blake2b_state ) ); - - for( int i = 0; i < 8; ++i ) S->h[i] = blake2b_IV[i]; - - return 0; -} - - - -#if defined(__cplusplus) -extern "C" { -#endif - int blake2b_init( blake2b_state *S, size_t outlen ); - int blake2b_init_param( blake2b_state *S, const blake2b_param *P ); - int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ); - int blake2b_update( blake2b_state *S, const uint8_t *in, size_t inlen ); - int blake2b_final( blake2b_state *S, uint8_t *out, size_t outlen ); - int blake2b( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); -#if defined(__cplusplus) -} -#endif - -/* init xors IV with input parameter block */ -int blake2b_init_param( blake2b_state *S, const blake2b_param *P ) -{ - uint8_t *p, *h, *v; - //blake2b_init0( S ); - v = ( uint8_t * )( blake2b_IV ); - h = ( uint8_t * )( S->h ); - p = ( uint8_t * )( P ); - /* IV XOR ParamBlock */ - memset( S, 0, sizeof( blake2b_state ) ); - - for( int i = 0; i < BLAKE2B_OUTBYTES; ++i ) h[i] = v[i] ^ p[i]; - - S->outlen = P->digest_length; - return 0; -} - - -/* Some sort of default parameter block initialization, for sequential blake2b */ - -int blake2b_init( blake2b_state *S, size_t outlen ) -{ - if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; - - const blake2b_param P = - { - ( uint8_t ) outlen, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - {0}, - {0}, - {0} - }; - return blake2b_init_param( S, &P ); -} - -int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ) -{ - if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; - - if ( ( !keylen ) || keylen > BLAKE2B_KEYBYTES ) return -1; - - const blake2b_param P = - { - ( uint8_t ) outlen, - ( uint8_t ) keylen, - 1, - 1, - 0, - 0, - 0, - 0, - {0}, - {0}, - {0} - }; - - if( blake2b_init_param( S, &P ) < 0 ) - return 0; - - { - uint8_t block[BLAKE2B_BLOCKBYTES]; - memset( block, 0, BLAKE2B_BLOCKBYTES ); - memcpy( block, key, keylen ); - blake2b_update( S, block, BLAKE2B_BLOCKBYTES ); - secure_zero_memory( block, BLAKE2B_BLOCKBYTES ); /* Burn the key from stack */ - } - return 0; -} - -static inline int blake2b_compress( blake2b_state *S, const uint8_t block[BLAKE2B_BLOCKBYTES] ) -{ - __m128i row1l, row1h; - __m128i row2l, row2h; - __m128i row3l, row3h; - __m128i row4l, row4h; - __m128i b0, b1; - __m128i t0, t1; -#if defined(HAVE_SSSE3) && !defined(HAVE_XOP) - const __m128i r16 = _mm_setr_epi8( 2, 3, 4, 5, 6, 7, 0, 1, 10, 11, 12, 13, 14, 15, 8, 9 ); - const __m128i r24 = _mm_setr_epi8( 3, 4, 5, 6, 7, 0, 1, 2, 11, 12, 13, 14, 15, 8, 9, 10 ); -#endif -#if defined(HAVE_SSE4_1) - const __m128i m0 = LOADU( block + 00 ); - const __m128i m1 = LOADU( block + 16 ); - const __m128i m2 = LOADU( block + 32 ); - const __m128i m3 = LOADU( block + 48 ); - const __m128i m4 = LOADU( block + 64 ); - const __m128i m5 = LOADU( block + 80 ); - const __m128i m6 = LOADU( block + 96 ); - const __m128i m7 = LOADU( block + 112 ); -#else - const uint64_t m0 = ( ( uint64_t * )block )[ 0]; - const uint64_t m1 = ( ( uint64_t * )block )[ 1]; - const uint64_t m2 = ( ( uint64_t * )block )[ 2]; - const uint64_t m3 = ( ( uint64_t * )block )[ 3]; - const uint64_t m4 = ( ( uint64_t * )block )[ 4]; - const uint64_t m5 = ( ( uint64_t * )block )[ 5]; - const uint64_t m6 = ( ( uint64_t * )block )[ 6]; - const uint64_t m7 = ( ( uint64_t * )block )[ 7]; - const uint64_t m8 = ( ( uint64_t * )block )[ 8]; - const uint64_t m9 = ( ( uint64_t * )block )[ 9]; - const uint64_t m10 = ( ( uint64_t * )block )[10]; - const uint64_t m11 = ( ( uint64_t * )block )[11]; - const uint64_t m12 = ( ( uint64_t * )block )[12]; - const uint64_t m13 = ( ( uint64_t * )block )[13]; - const uint64_t m14 = ( ( uint64_t * )block )[14]; - const uint64_t m15 = ( ( uint64_t * )block )[15]; -#endif - row1l = LOADU( &S->h[0] ); - row1h = LOADU( &S->h[2] ); - row2l = LOADU( &S->h[4] ); - row2h = LOADU( &S->h[6] ); - row3l = LOADU( &blake2b_IV[0] ); - row3h = LOADU( &blake2b_IV[2] ); - row4l = _mm_xor_si128( LOADU( &blake2b_IV[4] ), LOADU( &S->t[0] ) ); - row4h = _mm_xor_si128( LOADU( &blake2b_IV[6] ), LOADU( &S->f[0] ) ); - ROUND( 0 ); - ROUND( 1 ); - ROUND( 2 ); - ROUND( 3 ); - ROUND( 4 ); - ROUND( 5 ); - ROUND( 6 ); - ROUND( 7 ); - ROUND( 8 ); - ROUND( 9 ); - ROUND( 10 ); - ROUND( 11 ); - row1l = _mm_xor_si128( row3l, row1l ); - row1h = _mm_xor_si128( row3h, row1h ); - STOREU( &S->h[0], _mm_xor_si128( LOADU( &S->h[0] ), row1l ) ); - STOREU( &S->h[2], _mm_xor_si128( LOADU( &S->h[2] ), row1h ) ); - row2l = _mm_xor_si128( row4l, row2l ); - row2h = _mm_xor_si128( row4h, row2h ); - STOREU( &S->h[4], _mm_xor_si128( LOADU( &S->h[4] ), row2l ) ); - STOREU( &S->h[6], _mm_xor_si128( LOADU( &S->h[6] ), row2h ) ); - return 0; -} - - -int blake2b_update( blake2b_state *S, const uint8_t *in, size_t inlen ) -{ - while( inlen > 0 ) - { - uint32_t left = S->buflen; - uint32_t fill = 2 * BLAKE2B_BLOCKBYTES - left; - - if( inlen > fill ) - { - memcpy( S->buf + left, in, fill ); // Fill buffer - S->buflen += fill; - blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); - blake2b_compress( S, S->buf ); // Compress - memcpy( S->buf, S->buf + BLAKE2B_BLOCKBYTES, BLAKE2B_BLOCKBYTES ); // Shift buffer left - S->buflen -= BLAKE2B_BLOCKBYTES; - in += fill; - inlen -= fill; - } - else // inlen <= fill - { - memcpy( S->buf + left, in, inlen ); - S->buflen += ( uint32_t ) inlen; // Be lazy, do not compress - in += inlen; - inlen -= inlen; - } - } - - return 0; -} - - -int blake2b_final( blake2b_state *S, uint8_t *out, size_t outlen ) -{ - if(S->outlen != outlen) return -1; - - if( S->buflen > BLAKE2B_BLOCKBYTES ) - { - blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); - blake2b_compress( S, S->buf ); - S->buflen -= BLAKE2B_BLOCKBYTES; - memmove( S->buf, S->buf + BLAKE2B_BLOCKBYTES, S->buflen ); - } - - blake2b_increment_counter( S, S->buflen ); - blake2b_set_lastblock( S ); - memset( S->buf + S->buflen, 0, 2 * BLAKE2B_BLOCKBYTES - S->buflen ); /* Padding */ - blake2b_compress( S, S->buf ); - memcpy( out, &S->h[0], outlen ); - return 0; -} - - -int blake2b( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ) -{ - blake2b_state S[1]; - - /* Verify parameters */ - if ( NULL == in && inlen > 0 ) return -1; - - if ( NULL == out ) return -1; - - if( NULL == key && keylen > 0 ) return -1; - - if( !outlen || outlen > BLAKE2B_OUTBYTES ) return -1; - - if( keylen > BLAKE2B_KEYBYTES ) return -1; - - if( keylen ) - { - if( blake2b_init_key( S, outlen, key, keylen ) < 0 ) return -1; - } - else - { - if( blake2b_init( S, outlen ) < 0 ) return -1; - } - - if( blake2b_update( S, ( uint8_t * )in, inlen ) < 0) return -1; - return blake2b_final( S, out, outlen ); -} - -#if defined(SUPERCOP) -int crypto_hash( unsigned char *out, unsigned char *in, unsigned long long inlen ) -{ - return blake2b( out, in, NULL, BLAKE2B_OUTBYTES, inlen, 0 ); -} -#endif diff --git a/Modules/_blake2/impl/blake2s-load-sse2.h b/Modules/_blake2/impl/blake2s-load-sse2.h deleted file mode 100644 index b24483cf931c1f..00000000000000 --- a/Modules/_blake2/impl/blake2s-load-sse2.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ -#pragma once -#ifndef __BLAKE2S_LOAD_SSE2_H__ -#define __BLAKE2S_LOAD_SSE2_H__ - -#define LOAD_MSG_0_1(buf) buf = _mm_set_epi32(m6,m4,m2,m0) -#define LOAD_MSG_0_2(buf) buf = _mm_set_epi32(m7,m5,m3,m1) -#define LOAD_MSG_0_3(buf) buf = _mm_set_epi32(m14,m12,m10,m8) -#define LOAD_MSG_0_4(buf) buf = _mm_set_epi32(m15,m13,m11,m9) -#define LOAD_MSG_1_1(buf) buf = _mm_set_epi32(m13,m9,m4,m14) -#define LOAD_MSG_1_2(buf) buf = _mm_set_epi32(m6,m15,m8,m10) -#define LOAD_MSG_1_3(buf) buf = _mm_set_epi32(m5,m11,m0,m1) -#define LOAD_MSG_1_4(buf) buf = _mm_set_epi32(m3,m7,m2,m12) -#define LOAD_MSG_2_1(buf) buf = _mm_set_epi32(m15,m5,m12,m11) -#define LOAD_MSG_2_2(buf) buf = _mm_set_epi32(m13,m2,m0,m8) -#define LOAD_MSG_2_3(buf) buf = _mm_set_epi32(m9,m7,m3,m10) -#define LOAD_MSG_2_4(buf) buf = _mm_set_epi32(m4,m1,m6,m14) -#define LOAD_MSG_3_1(buf) buf = _mm_set_epi32(m11,m13,m3,m7) -#define LOAD_MSG_3_2(buf) buf = _mm_set_epi32(m14,m12,m1,m9) -#define LOAD_MSG_3_3(buf) buf = _mm_set_epi32(m15,m4,m5,m2) -#define LOAD_MSG_3_4(buf) buf = _mm_set_epi32(m8,m0,m10,m6) -#define LOAD_MSG_4_1(buf) buf = _mm_set_epi32(m10,m2,m5,m9) -#define LOAD_MSG_4_2(buf) buf = _mm_set_epi32(m15,m4,m7,m0) -#define LOAD_MSG_4_3(buf) buf = _mm_set_epi32(m3,m6,m11,m14) -#define LOAD_MSG_4_4(buf) buf = _mm_set_epi32(m13,m8,m12,m1) -#define LOAD_MSG_5_1(buf) buf = _mm_set_epi32(m8,m0,m6,m2) -#define LOAD_MSG_5_2(buf) buf = _mm_set_epi32(m3,m11,m10,m12) -#define LOAD_MSG_5_3(buf) buf = _mm_set_epi32(m1,m15,m7,m4) -#define LOAD_MSG_5_4(buf) buf = _mm_set_epi32(m9,m14,m5,m13) -#define LOAD_MSG_6_1(buf) buf = _mm_set_epi32(m4,m14,m1,m12) -#define LOAD_MSG_6_2(buf) buf = _mm_set_epi32(m10,m13,m15,m5) -#define LOAD_MSG_6_3(buf) buf = _mm_set_epi32(m8,m9,m6,m0) -#define LOAD_MSG_6_4(buf) buf = _mm_set_epi32(m11,m2,m3,m7) -#define LOAD_MSG_7_1(buf) buf = _mm_set_epi32(m3,m12,m7,m13) -#define LOAD_MSG_7_2(buf) buf = _mm_set_epi32(m9,m1,m14,m11) -#define LOAD_MSG_7_3(buf) buf = _mm_set_epi32(m2,m8,m15,m5) -#define LOAD_MSG_7_4(buf) buf = _mm_set_epi32(m10,m6,m4,m0) -#define LOAD_MSG_8_1(buf) buf = _mm_set_epi32(m0,m11,m14,m6) -#define LOAD_MSG_8_2(buf) buf = _mm_set_epi32(m8,m3,m9,m15) -#define LOAD_MSG_8_3(buf) buf = _mm_set_epi32(m10,m1,m13,m12) -#define LOAD_MSG_8_4(buf) buf = _mm_set_epi32(m5,m4,m7,m2) -#define LOAD_MSG_9_1(buf) buf = _mm_set_epi32(m1,m7,m8,m10) -#define LOAD_MSG_9_2(buf) buf = _mm_set_epi32(m5,m6,m4,m2) -#define LOAD_MSG_9_3(buf) buf = _mm_set_epi32(m13,m3,m9,m15) -#define LOAD_MSG_9_4(buf) buf = _mm_set_epi32(m0,m12,m14,m11) - - -#endif diff --git a/Modules/_blake2/impl/blake2s-load-sse41.h b/Modules/_blake2/impl/blake2s-load-sse41.h deleted file mode 100644 index 3ac12eb6f5d082..00000000000000 --- a/Modules/_blake2/impl/blake2s-load-sse41.h +++ /dev/null @@ -1,229 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ -#pragma once -#ifndef __BLAKE2S_LOAD_SSE41_H__ -#define __BLAKE2S_LOAD_SSE41_H__ - -#define LOAD_MSG_0_1(buf) \ -buf = TOI(_mm_shuffle_ps(TOF(m0), TOF(m1), _MM_SHUFFLE(2,0,2,0))); - -#define LOAD_MSG_0_2(buf) \ -buf = TOI(_mm_shuffle_ps(TOF(m0), TOF(m1), _MM_SHUFFLE(3,1,3,1))); - -#define LOAD_MSG_0_3(buf) \ -buf = TOI(_mm_shuffle_ps(TOF(m2), TOF(m3), _MM_SHUFFLE(2,0,2,0))); - -#define LOAD_MSG_0_4(buf) \ -buf = TOI(_mm_shuffle_ps(TOF(m2), TOF(m3), _MM_SHUFFLE(3,1,3,1))); - -#define LOAD_MSG_1_1(buf) \ -t0 = _mm_blend_epi16(m1, m2, 0x0C); \ -t1 = _mm_slli_si128(m3, 4); \ -t2 = _mm_blend_epi16(t0, t1, 0xF0); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2,1,0,3)); - -#define LOAD_MSG_1_2(buf) \ -t0 = _mm_shuffle_epi32(m2,_MM_SHUFFLE(0,0,2,0)); \ -t1 = _mm_blend_epi16(m1,m3,0xC0); \ -t2 = _mm_blend_epi16(t0, t1, 0xF0); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2,3,0,1)); - -#define LOAD_MSG_1_3(buf) \ -t0 = _mm_slli_si128(m1, 4); \ -t1 = _mm_blend_epi16(m2, t0, 0x30); \ -t2 = _mm_blend_epi16(m0, t1, 0xF0); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2,3,0,1)); - -#define LOAD_MSG_1_4(buf) \ -t0 = _mm_unpackhi_epi32(m0,m1); \ -t1 = _mm_slli_si128(m3, 4); \ -t2 = _mm_blend_epi16(t0, t1, 0x0C); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2,3,0,1)); - -#define LOAD_MSG_2_1(buf) \ -t0 = _mm_unpackhi_epi32(m2,m3); \ -t1 = _mm_blend_epi16(m3,m1,0x0C); \ -t2 = _mm_blend_epi16(t0, t1, 0x0F); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(3,1,0,2)); - -#define LOAD_MSG_2_2(buf) \ -t0 = _mm_unpacklo_epi32(m2,m0); \ -t1 = _mm_blend_epi16(t0, m0, 0xF0); \ -t2 = _mm_slli_si128(m3, 8); \ -buf = _mm_blend_epi16(t1, t2, 0xC0); - -#define LOAD_MSG_2_3(buf) \ -t0 = _mm_blend_epi16(m0, m2, 0x3C); \ -t1 = _mm_srli_si128(m1, 12); \ -t2 = _mm_blend_epi16(t0,t1,0x03); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(1,0,3,2)); - -#define LOAD_MSG_2_4(buf) \ -t0 = _mm_slli_si128(m3, 4); \ -t1 = _mm_blend_epi16(m0, m1, 0x33); \ -t2 = _mm_blend_epi16(t1, t0, 0xC0); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(0,1,2,3)); - -#define LOAD_MSG_3_1(buf) \ -t0 = _mm_unpackhi_epi32(m0,m1); \ -t1 = _mm_unpackhi_epi32(t0, m2); \ -t2 = _mm_blend_epi16(t1, m3, 0x0C); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(3,1,0,2)); - -#define LOAD_MSG_3_2(buf) \ -t0 = _mm_slli_si128(m2, 8); \ -t1 = _mm_blend_epi16(m3,m0,0x0C); \ -t2 = _mm_blend_epi16(t1, t0, 0xC0); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2,0,1,3)); - -#define LOAD_MSG_3_3(buf) \ -t0 = _mm_blend_epi16(m0,m1,0x0F); \ -t1 = _mm_blend_epi16(t0, m3, 0xC0); \ -buf = _mm_shuffle_epi32(t1, _MM_SHUFFLE(3,0,1,2)); - -#define LOAD_MSG_3_4(buf) \ -t0 = _mm_unpacklo_epi32(m0,m2); \ -t1 = _mm_unpackhi_epi32(m1,m2); \ -buf = _mm_unpacklo_epi64(t1,t0); - -#define LOAD_MSG_4_1(buf) \ -t0 = _mm_unpacklo_epi64(m1,m2); \ -t1 = _mm_unpackhi_epi64(m0,m2); \ -t2 = _mm_blend_epi16(t0,t1,0x33); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2,0,1,3)); - -#define LOAD_MSG_4_2(buf) \ -t0 = _mm_unpackhi_epi64(m1,m3); \ -t1 = _mm_unpacklo_epi64(m0,m1); \ -buf = _mm_blend_epi16(t0,t1,0x33); - -#define LOAD_MSG_4_3(buf) \ -t0 = _mm_unpackhi_epi64(m3,m1); \ -t1 = _mm_unpackhi_epi64(m2,m0); \ -buf = _mm_blend_epi16(t1,t0,0x33); - -#define LOAD_MSG_4_4(buf) \ -t0 = _mm_blend_epi16(m0,m2,0x03); \ -t1 = _mm_slli_si128(t0, 8); \ -t2 = _mm_blend_epi16(t1,m3,0x0F); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(1,2,0,3)); - -#define LOAD_MSG_5_1(buf) \ -t0 = _mm_unpackhi_epi32(m0,m1); \ -t1 = _mm_unpacklo_epi32(m0,m2); \ -buf = _mm_unpacklo_epi64(t0,t1); - -#define LOAD_MSG_5_2(buf) \ -t0 = _mm_srli_si128(m2, 4); \ -t1 = _mm_blend_epi16(m0,m3,0x03); \ -buf = _mm_blend_epi16(t1,t0,0x3C); - -#define LOAD_MSG_5_3(buf) \ -t0 = _mm_blend_epi16(m1,m0,0x0C); \ -t1 = _mm_srli_si128(m3, 4); \ -t2 = _mm_blend_epi16(t0,t1,0x30); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(1,2,3,0)); - -#define LOAD_MSG_5_4(buf) \ -t0 = _mm_unpacklo_epi64(m1,m2); \ -t1= _mm_shuffle_epi32(m3, _MM_SHUFFLE(0,2,0,1)); \ -buf = _mm_blend_epi16(t0,t1,0x33); - -#define LOAD_MSG_6_1(buf) \ -t0 = _mm_slli_si128(m1, 12); \ -t1 = _mm_blend_epi16(m0,m3,0x33); \ -buf = _mm_blend_epi16(t1,t0,0xC0); - -#define LOAD_MSG_6_2(buf) \ -t0 = _mm_blend_epi16(m3,m2,0x30); \ -t1 = _mm_srli_si128(m1, 4); \ -t2 = _mm_blend_epi16(t0,t1,0x03); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2,1,3,0)); - -#define LOAD_MSG_6_3(buf) \ -t0 = _mm_unpacklo_epi64(m0,m2); \ -t1 = _mm_srli_si128(m1, 4); \ -buf = _mm_shuffle_epi32(_mm_blend_epi16(t0,t1,0x0C), _MM_SHUFFLE(2,3,1,0)); - -#define LOAD_MSG_6_4(buf) \ -t0 = _mm_unpackhi_epi32(m1,m2); \ -t1 = _mm_unpackhi_epi64(m0,t0); \ -buf = _mm_shuffle_epi32(t1, _MM_SHUFFLE(3,0,1,2)); - -#define LOAD_MSG_7_1(buf) \ -t0 = _mm_unpackhi_epi32(m0,m1); \ -t1 = _mm_blend_epi16(t0,m3,0x0F); \ -buf = _mm_shuffle_epi32(t1,_MM_SHUFFLE(2,0,3,1)); - -#define LOAD_MSG_7_2(buf) \ -t0 = _mm_blend_epi16(m2,m3,0x30); \ -t1 = _mm_srli_si128(m0,4); \ -t2 = _mm_blend_epi16(t0,t1,0x03); \ -buf = _mm_shuffle_epi32(t2, _MM_SHUFFLE(1,0,2,3)); - -#define LOAD_MSG_7_3(buf) \ -t0 = _mm_unpackhi_epi64(m0,m3); \ -t1 = _mm_unpacklo_epi64(m1,m2); \ -t2 = _mm_blend_epi16(t0,t1,0x3C); \ -buf = _mm_shuffle_epi32(t2,_MM_SHUFFLE(0,2,3,1)); - -#define LOAD_MSG_7_4(buf) \ -t0 = _mm_unpacklo_epi32(m0,m1); \ -t1 = _mm_unpackhi_epi32(m1,m2); \ -buf = _mm_unpacklo_epi64(t0,t1); - -#define LOAD_MSG_8_1(buf) \ -t0 = _mm_unpackhi_epi32(m1,m3); \ -t1 = _mm_unpacklo_epi64(t0,m0); \ -t2 = _mm_blend_epi16(t1,m2,0xC0); \ -buf = _mm_shufflehi_epi16(t2,_MM_SHUFFLE(1,0,3,2)); - -#define LOAD_MSG_8_2(buf) \ -t0 = _mm_unpackhi_epi32(m0,m3); \ -t1 = _mm_blend_epi16(m2,t0,0xF0); \ -buf = _mm_shuffle_epi32(t1,_MM_SHUFFLE(0,2,1,3)); - -#define LOAD_MSG_8_3(buf) \ -t0 = _mm_blend_epi16(m2,m0,0x0C); \ -t1 = _mm_slli_si128(t0,4); \ -buf = _mm_blend_epi16(t1,m3,0x0F); - -#define LOAD_MSG_8_4(buf) \ -t0 = _mm_blend_epi16(m1,m0,0x30); \ -buf = _mm_shuffle_epi32(t0,_MM_SHUFFLE(1,0,3,2)); - -#define LOAD_MSG_9_1(buf) \ -t0 = _mm_blend_epi16(m0,m2,0x03); \ -t1 = _mm_blend_epi16(m1,m2,0x30); \ -t2 = _mm_blend_epi16(t1,t0,0x0F); \ -buf = _mm_shuffle_epi32(t2,_MM_SHUFFLE(1,3,0,2)); - -#define LOAD_MSG_9_2(buf) \ -t0 = _mm_slli_si128(m0,4); \ -t1 = _mm_blend_epi16(m1,t0,0xC0); \ -buf = _mm_shuffle_epi32(t1,_MM_SHUFFLE(1,2,0,3)); - -#define LOAD_MSG_9_3(buf) \ -t0 = _mm_unpackhi_epi32(m0,m3); \ -t1 = _mm_unpacklo_epi32(m2,m3); \ -t2 = _mm_unpackhi_epi64(t0,t1); \ -buf = _mm_shuffle_epi32(t2,_MM_SHUFFLE(3,0,2,1)); - -#define LOAD_MSG_9_4(buf) \ -t0 = _mm_blend_epi16(m3,m2,0xC0); \ -t1 = _mm_unpacklo_epi32(m0,m3); \ -t2 = _mm_blend_epi16(t0,t1,0x0F); \ -buf = _mm_shuffle_epi32(t2,_MM_SHUFFLE(0,1,2,3)); - -#endif - diff --git a/Modules/_blake2/impl/blake2s-load-xop.h b/Modules/_blake2/impl/blake2s-load-xop.h deleted file mode 100644 index 14d9e7f7640672..00000000000000 --- a/Modules/_blake2/impl/blake2s-load-xop.h +++ /dev/null @@ -1,189 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ -#pragma once -#ifndef __BLAKE2S_LOAD_XOP_H__ -#define __BLAKE2S_LOAD_XOP_H__ - -#define TOB(x) ((x)*4*0x01010101 + 0x03020100) // ..or not TOB - -/* Basic VPPERM emulation, for testing purposes */ -/*static __m128i _mm_perm_epi8(const __m128i src1, const __m128i src2, const __m128i sel) -{ - const __m128i sixteen = _mm_set1_epi8(16); - const __m128i t0 = _mm_shuffle_epi8(src1, sel); - const __m128i s1 = _mm_shuffle_epi8(src2, _mm_sub_epi8(sel, sixteen)); - const __m128i mask = _mm_or_si128(_mm_cmpeq_epi8(sel, sixteen), - _mm_cmpgt_epi8(sel, sixteen)); // (>=16) = 0xff : 00 - return _mm_blendv_epi8(t0, s1, mask); -}*/ - -#define LOAD_MSG_0_1(buf) \ -buf = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(6),TOB(4),TOB(2),TOB(0)) ); - -#define LOAD_MSG_0_2(buf) \ -buf = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(7),TOB(5),TOB(3),TOB(1)) ); - -#define LOAD_MSG_0_3(buf) \ -buf = _mm_perm_epi8(m2, m3, _mm_set_epi32(TOB(6),TOB(4),TOB(2),TOB(0)) ); - -#define LOAD_MSG_0_4(buf) \ -buf = _mm_perm_epi8(m2, m3, _mm_set_epi32(TOB(7),TOB(5),TOB(3),TOB(1)) ); - -#define LOAD_MSG_1_1(buf) \ -t0 = _mm_perm_epi8(m1, m2, _mm_set_epi32(TOB(0),TOB(5),TOB(0),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(5),TOB(2),TOB(1),TOB(6)) ); - -#define LOAD_MSG_1_2(buf) \ -t1 = _mm_perm_epi8(m1, m2, _mm_set_epi32(TOB(2),TOB(0),TOB(4),TOB(6)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(7),TOB(1),TOB(0)) ); - -#define LOAD_MSG_1_3(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(5),TOB(0),TOB(0),TOB(1)) ); \ -buf = _mm_perm_epi8(t0, m2, _mm_set_epi32(TOB(3),TOB(7),TOB(1),TOB(0)) ); - -#define LOAD_MSG_1_4(buf) \ -t1 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(3),TOB(7),TOB(2),TOB(0)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(1),TOB(4)) ); - -#define LOAD_MSG_2_1(buf) \ -t0 = _mm_perm_epi8(m1, m2, _mm_set_epi32(TOB(0),TOB(1),TOB(0),TOB(7)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(7),TOB(2),TOB(4),TOB(0)) ); - -#define LOAD_MSG_2_2(buf) \ -t1 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(0),TOB(2),TOB(0),TOB(4)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(5),TOB(2),TOB(1),TOB(0)) ); - -#define LOAD_MSG_2_3(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(7),TOB(3),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m2, _mm_set_epi32(TOB(5),TOB(2),TOB(1),TOB(6)) ); - -#define LOAD_MSG_2_4(buf) \ -t1 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(4),TOB(1),TOB(6),TOB(0)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(1),TOB(6)) ); - -#define LOAD_MSG_3_1(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(0),TOB(3),TOB(7)) ); \ -t0 = _mm_perm_epi8(t0, m2, _mm_set_epi32(TOB(7),TOB(2),TOB(1),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(3),TOB(5),TOB(1),TOB(0)) ); - -#define LOAD_MSG_3_2(buf) \ -t1 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(0),TOB(0),TOB(1),TOB(5)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(6),TOB(4),TOB(1),TOB(0)) ); - -#define LOAD_MSG_3_3(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(4),TOB(5),TOB(2)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(7),TOB(2),TOB(1),TOB(0)) ); - -#define LOAD_MSG_3_4(buf) \ -t1 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(0),TOB(0),TOB(6)) ); \ -buf = _mm_perm_epi8(t1, m2, _mm_set_epi32(TOB(4),TOB(2),TOB(6),TOB(0)) ); - -#define LOAD_MSG_4_1(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(2),TOB(5),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m2, _mm_set_epi32(TOB(6),TOB(2),TOB(1),TOB(5)) ); - -#define LOAD_MSG_4_2(buf) \ -t1 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(4),TOB(7),TOB(0)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(7),TOB(2),TOB(1),TOB(0)) ); - -#define LOAD_MSG_4_3(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(3),TOB(6),TOB(0),TOB(0)) ); \ -t0 = _mm_perm_epi8(t0, m2, _mm_set_epi32(TOB(3),TOB(2),TOB(7),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(1),TOB(6)) ); - -#define LOAD_MSG_4_4(buf) \ -t1 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(0),TOB(4),TOB(0),TOB(1)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(5),TOB(2),TOB(4),TOB(0)) ); - -#define LOAD_MSG_5_1(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(0),TOB(6),TOB(2)) ); \ -buf = _mm_perm_epi8(t0, m2, _mm_set_epi32(TOB(4),TOB(2),TOB(1),TOB(0)) ); - -#define LOAD_MSG_5_2(buf) \ -t1 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(3),TOB(7),TOB(6),TOB(0)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(1),TOB(4)) ); - -#define LOAD_MSG_5_3(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(1),TOB(0),TOB(7),TOB(4)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(3),TOB(7),TOB(1),TOB(0)) ); - -#define LOAD_MSG_5_4(buf) \ -t1 = _mm_perm_epi8(m1, m2, _mm_set_epi32(TOB(5),TOB(0),TOB(1),TOB(0)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(6),TOB(1),TOB(5)) ); - -#define LOAD_MSG_6_1(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(4),TOB(0),TOB(1),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(3),TOB(6),TOB(1),TOB(4)) ); - -#define LOAD_MSG_6_2(buf) \ -t1 = _mm_perm_epi8(m1, m2, _mm_set_epi32(TOB(6),TOB(0),TOB(0),TOB(1)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(5),TOB(7),TOB(0)) ); - -#define LOAD_MSG_6_3(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(0),TOB(6),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m2, _mm_set_epi32(TOB(4),TOB(5),TOB(1),TOB(0)) ); - -#define LOAD_MSG_6_4(buf) \ -t1 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(2),TOB(3),TOB(7)) ); \ -buf = _mm_perm_epi8(t1, m2, _mm_set_epi32(TOB(7),TOB(2),TOB(1),TOB(0)) ); - -#define LOAD_MSG_7_1(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(3),TOB(0),TOB(7),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(3),TOB(4),TOB(1),TOB(5)) ); - -#define LOAD_MSG_7_2(buf) \ -t1 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(5),TOB(1),TOB(0),TOB(7)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(6),TOB(0)) ); - -#define LOAD_MSG_7_3(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(2),TOB(0),TOB(0),TOB(5)) ); \ -t0 = _mm_perm_epi8(t0, m2, _mm_set_epi32(TOB(3),TOB(4),TOB(1),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(7),TOB(0)) ); - -#define LOAD_MSG_7_4(buf) \ -t1 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(6),TOB(4),TOB(0)) ); \ -buf = _mm_perm_epi8(t1, m2, _mm_set_epi32(TOB(6),TOB(2),TOB(1),TOB(0)) ); - -#define LOAD_MSG_8_1(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(0),TOB(0),TOB(0),TOB(6)) ); \ -t0 = _mm_perm_epi8(t0, m2, _mm_set_epi32(TOB(3),TOB(7),TOB(1),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(6),TOB(0)) ); - -#define LOAD_MSG_8_2(buf) \ -t1 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(4),TOB(3),TOB(5),TOB(0)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(1),TOB(7)) ); - -#define LOAD_MSG_8_3(buf) \ -t0 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(6),TOB(1),TOB(0),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(5),TOB(4)) ); \ - -#define LOAD_MSG_8_4(buf) \ -buf = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(5),TOB(4),TOB(7),TOB(2)) ); - -#define LOAD_MSG_9_1(buf) \ -t0 = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(1),TOB(7),TOB(0),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m2, _mm_set_epi32(TOB(3),TOB(2),TOB(4),TOB(6)) ); - -#define LOAD_MSG_9_2(buf) \ -buf = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(5),TOB(6),TOB(4),TOB(2)) ); - -#define LOAD_MSG_9_3(buf) \ -t0 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(0),TOB(3),TOB(5),TOB(0)) ); \ -buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(5),TOB(2),TOB(1),TOB(7)) ); - -#define LOAD_MSG_9_4(buf) \ -t1 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(0),TOB(0),TOB(0),TOB(7)) ); \ -buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(4),TOB(6),TOB(0)) ); - -#endif - diff --git a/Modules/_blake2/impl/blake2s-ref.c b/Modules/_blake2/impl/blake2s-ref.c deleted file mode 100644 index ab86cc1b34e67d..00000000000000 --- a/Modules/_blake2/impl/blake2s-ref.c +++ /dev/null @@ -1,368 +0,0 @@ -/* - BLAKE2 reference source code package - reference C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ - -#include -#include -#include - -#include "blake2.h" -#include "blake2-impl.h" - -static const uint32_t blake2s_IV[8] = -{ - 0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, 0xA54FF53AUL, - 0x510E527FUL, 0x9B05688CUL, 0x1F83D9ABUL, 0x5BE0CD19UL -}; - -static const uint8_t blake2s_sigma[10][16] = -{ - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , - { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , - { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , - { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , - { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , - { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , - { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , - { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , - { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , -}; - -static inline int blake2s_set_lastnode( blake2s_state *S ) -{ - S->f[1] = ~0U; - return 0; -} - -static inline int blake2s_clear_lastnode( blake2s_state *S ) -{ - S->f[1] = 0U; - return 0; -} - -/* Some helper functions, not necessarily useful */ -static inline int blake2s_set_lastblock( blake2s_state *S ) -{ - if( S->last_node ) blake2s_set_lastnode( S ); - - S->f[0] = ~0U; - return 0; -} - -static inline int blake2s_clear_lastblock( blake2s_state *S ) -{ - if( S->last_node ) blake2s_clear_lastnode( S ); - - S->f[0] = 0U; - return 0; -} - -static inline int blake2s_increment_counter( blake2s_state *S, const uint32_t inc ) -{ - S->t[0] += inc; - S->t[1] += ( S->t[0] < inc ); - return 0; -} - -// Parameter-related functions -static inline int blake2s_param_set_digest_length( blake2s_param *P, const uint8_t digest_length ) -{ - P->digest_length = digest_length; - return 0; -} - -static inline int blake2s_param_set_fanout( blake2s_param *P, const uint8_t fanout ) -{ - P->fanout = fanout; - return 0; -} - -static inline int blake2s_param_set_max_depth( blake2s_param *P, const uint8_t depth ) -{ - P->depth = depth; - return 0; -} - -static inline int blake2s_param_set_leaf_length( blake2s_param *P, const uint32_t leaf_length ) -{ - store32( &P->leaf_length, leaf_length ); - return 0; -} - -static inline int blake2s_param_set_node_offset( blake2s_param *P, const uint64_t node_offset ) -{ - store48( P->node_offset, node_offset ); - return 0; -} - -static inline int blake2s_param_set_node_depth( blake2s_param *P, const uint8_t node_depth ) -{ - P->node_depth = node_depth; - return 0; -} - -static inline int blake2s_param_set_inner_length( blake2s_param *P, const uint8_t inner_length ) -{ - P->inner_length = inner_length; - return 0; -} - -static inline int blake2s_param_set_salt( blake2s_param *P, const uint8_t salt[BLAKE2S_SALTBYTES] ) -{ - memcpy( P->salt, salt, BLAKE2S_SALTBYTES ); - return 0; -} - -static inline int blake2s_param_set_personal( blake2s_param *P, const uint8_t personal[BLAKE2S_PERSONALBYTES] ) -{ - memcpy( P->personal, personal, BLAKE2S_PERSONALBYTES ); - return 0; -} - -static inline int blake2s_init0( blake2s_state *S ) -{ - memset( S, 0, sizeof( blake2s_state ) ); - - for( int i = 0; i < 8; ++i ) S->h[i] = blake2s_IV[i]; - - return 0; -} - -#if defined(__cplusplus) -extern "C" { -#endif - int blake2s_init( blake2s_state *S, size_t outlen ); - int blake2s_init_param( blake2s_state *S, const blake2s_param *P ); - int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ); - int blake2s_update( blake2s_state *S, const uint8_t *in, size_t inlen ); - int blake2s_final( blake2s_state *S, uint8_t *out, size_t outlen ); - int blake2s( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); -#if defined(__cplusplus) -} -#endif - -/* init2 xors IV with input parameter block */ -int blake2s_init_param( blake2s_state *S, const blake2s_param *P ) -{ - blake2s_init0( S ); - uint32_t *p = ( uint32_t * )( P ); - - /* IV XOR ParamBlock */ - for( size_t i = 0; i < 8; ++i ) - S->h[i] ^= load32( &p[i] ); - - S->outlen = P->digest_length; - return 0; -} - - -// Sequential blake2s initialization -int blake2s_init( blake2s_state *S, size_t outlen ) -{ - blake2s_param P[1]; - - /* Move interval verification here? */ - if ( ( !outlen ) || ( outlen > BLAKE2S_OUTBYTES ) ) return -1; - - P->digest_length = ( uint8_t) outlen; - P->key_length = 0; - P->fanout = 1; - P->depth = 1; - store32( &P->leaf_length, 0 ); - store48( &P->node_offset, 0 ); - P->node_depth = 0; - P->inner_length = 0; - // memset(P->reserved, 0, sizeof(P->reserved) ); - memset( P->salt, 0, sizeof( P->salt ) ); - memset( P->personal, 0, sizeof( P->personal ) ); - return blake2s_init_param( S, P ); -} - -int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ) -{ - blake2s_param P[1]; - - if ( ( !outlen ) || ( outlen > BLAKE2S_OUTBYTES ) ) return -1; - - if ( !key || !keylen || keylen > BLAKE2S_KEYBYTES ) return -1; - - P->digest_length = ( uint8_t ) outlen; - P->key_length = ( uint8_t ) keylen; - P->fanout = 1; - P->depth = 1; - store32( &P->leaf_length, 0 ); - store48( &P->node_offset, 0 ); - P->node_depth = 0; - P->inner_length = 0; - // memset(P->reserved, 0, sizeof(P->reserved) ); - memset( P->salt, 0, sizeof( P->salt ) ); - memset( P->personal, 0, sizeof( P->personal ) ); - - if( blake2s_init_param( S, P ) < 0 ) return -1; - - { - uint8_t block[BLAKE2S_BLOCKBYTES]; - memset( block, 0, BLAKE2S_BLOCKBYTES ); - memcpy( block, key, keylen ); - blake2s_update( S, block, BLAKE2S_BLOCKBYTES ); - secure_zero_memory( block, BLAKE2S_BLOCKBYTES ); /* Burn the key from stack */ - } - return 0; -} - -static int blake2s_compress( blake2s_state *S, const uint8_t block[BLAKE2S_BLOCKBYTES] ) -{ - uint32_t m[16]; - uint32_t v[16]; - - for( size_t i = 0; i < 16; ++i ) - m[i] = load32( block + i * sizeof( m[i] ) ); - - for( size_t i = 0; i < 8; ++i ) - v[i] = S->h[i]; - - v[ 8] = blake2s_IV[0]; - v[ 9] = blake2s_IV[1]; - v[10] = blake2s_IV[2]; - v[11] = blake2s_IV[3]; - v[12] = S->t[0] ^ blake2s_IV[4]; - v[13] = S->t[1] ^ blake2s_IV[5]; - v[14] = S->f[0] ^ blake2s_IV[6]; - v[15] = S->f[1] ^ blake2s_IV[7]; -#define G(r,i,a,b,c,d) \ - do { \ - a = a + b + m[blake2s_sigma[r][2*i+0]]; \ - d = rotr32(d ^ a, 16); \ - c = c + d; \ - b = rotr32(b ^ c, 12); \ - a = a + b + m[blake2s_sigma[r][2*i+1]]; \ - d = rotr32(d ^ a, 8); \ - c = c + d; \ - b = rotr32(b ^ c, 7); \ - } while(0) -#define ROUND(r) \ - do { \ - G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \ - G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \ - G(r,2,v[ 2],v[ 6],v[10],v[14]); \ - G(r,3,v[ 3],v[ 7],v[11],v[15]); \ - G(r,4,v[ 0],v[ 5],v[10],v[15]); \ - G(r,5,v[ 1],v[ 6],v[11],v[12]); \ - G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \ - G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ - } while(0) - ROUND( 0 ); - ROUND( 1 ); - ROUND( 2 ); - ROUND( 3 ); - ROUND( 4 ); - ROUND( 5 ); - ROUND( 6 ); - ROUND( 7 ); - ROUND( 8 ); - ROUND( 9 ); - - for( size_t i = 0; i < 8; ++i ) - S->h[i] = S->h[i] ^ v[i] ^ v[i + 8]; - -#undef G -#undef ROUND - return 0; -} - - -int blake2s_update( blake2s_state *S, const uint8_t *in, size_t inlen ) -{ - while( inlen > 0 ) - { - uint32_t left = S->buflen; - uint32_t fill = 2 * BLAKE2S_BLOCKBYTES - left; - - if( inlen > fill ) - { - memcpy( S->buf + left, in, fill ); // Fill buffer - S->buflen += fill; - blake2s_increment_counter( S, BLAKE2S_BLOCKBYTES ); - blake2s_compress( S, S->buf ); // Compress - memcpy( S->buf, S->buf + BLAKE2S_BLOCKBYTES, BLAKE2S_BLOCKBYTES ); // Shift buffer left - S->buflen -= BLAKE2S_BLOCKBYTES; - in += fill; - inlen -= fill; - } - else // inlen <= fill - { - memcpy( S->buf + left, in, inlen ); - S->buflen += ( uint32_t ) inlen; // Be lazy, do not compress - in += inlen; - inlen -= inlen; - } - } - - return 0; -} - -int blake2s_final( blake2s_state *S, uint8_t *out, size_t outlen ) -{ - uint8_t buffer[BLAKE2S_OUTBYTES]; - size_t i; - - if(S->outlen != outlen) return -1; - - if( S->buflen > BLAKE2S_BLOCKBYTES ) - { - blake2s_increment_counter( S, BLAKE2S_BLOCKBYTES ); - blake2s_compress( S, S->buf ); - S->buflen -= BLAKE2S_BLOCKBYTES; - memmove( S->buf, S->buf + BLAKE2S_BLOCKBYTES, S->buflen ); - } - - blake2s_increment_counter( S, ( uint32_t )S->buflen ); - blake2s_set_lastblock( S ); - memset( S->buf + S->buflen, 0, 2 * BLAKE2S_BLOCKBYTES - S->buflen ); /* Padding */ - blake2s_compress( S, S->buf ); - - for( i = 0; i < 8; ++i ) /* Output full hash to temp buffer */ - store32( buffer + sizeof( S->h[i] ) * i, S->h[i] ); - - memcpy( out, buffer, outlen ); - return 0; -} - -int blake2s( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ) -{ - blake2s_state S[1]; - - /* Verify parameters */ - if ( NULL == in && inlen > 0 ) return -1; - - if ( NULL == out ) return -1; - - if ( NULL == key && keylen > 0 ) return -1; - - if( !outlen || outlen > BLAKE2S_OUTBYTES ) return -1; - - if( keylen > BLAKE2S_KEYBYTES ) return -1; - - if( keylen > 0 ) - { - if( blake2s_init_key( S, outlen, key, keylen ) < 0 ) return -1; - } - else - { - if( blake2s_init( S, outlen ) < 0 ) return -1; - } - - if( blake2s_update( S, ( uint8_t * )in, inlen ) < 0) return -1; - return blake2s_final( S, out, outlen ); -} - diff --git a/Modules/_blake2/impl/blake2s-round.h b/Modules/_blake2/impl/blake2s-round.h deleted file mode 100644 index 3af4be35bee5d4..00000000000000 --- a/Modules/_blake2/impl/blake2s-round.h +++ /dev/null @@ -1,91 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ -#pragma once -#ifndef __BLAKE2S_ROUND_H__ -#define __BLAKE2S_ROUND_H__ - -#define LOAD(p) _mm_load_si128( (__m128i *)(p) ) -#define STORE(p,r) _mm_store_si128((__m128i *)(p), r) - -#define LOADU(p) _mm_loadu_si128( (__m128i *)(p) ) -#define STOREU(p,r) _mm_storeu_si128((__m128i *)(p), r) - -#define TOF(reg) _mm_castsi128_ps((reg)) -#define TOI(reg) _mm_castps_si128((reg)) - -#define LIKELY(x) __builtin_expect((x),1) - - -/* Microarchitecture-specific macros */ -#ifndef HAVE_XOP -#ifdef HAVE_SSSE3 -#define _mm_roti_epi32(r, c) ( \ - (8==-(c)) ? _mm_shuffle_epi8(r,r8) \ - : (16==-(c)) ? _mm_shuffle_epi8(r,r16) \ - : _mm_xor_si128(_mm_srli_epi32( (r), -(c) ),_mm_slli_epi32( (r), 32-(-(c)) )) ) -#else -#define _mm_roti_epi32(r, c) _mm_xor_si128(_mm_srli_epi32( (r), -(c) ),_mm_slli_epi32( (r), 32-(-(c)) )) -#endif -#else -/* ... */ -#endif - - -#define G1(row1,row2,row3,row4,buf) \ - row1 = _mm_add_epi32( _mm_add_epi32( row1, buf), row2 ); \ - row4 = _mm_xor_si128( row4, row1 ); \ - row4 = _mm_roti_epi32(row4, -16); \ - row3 = _mm_add_epi32( row3, row4 ); \ - row2 = _mm_xor_si128( row2, row3 ); \ - row2 = _mm_roti_epi32(row2, -12); - -#define G2(row1,row2,row3,row4,buf) \ - row1 = _mm_add_epi32( _mm_add_epi32( row1, buf), row2 ); \ - row4 = _mm_xor_si128( row4, row1 ); \ - row4 = _mm_roti_epi32(row4, -8); \ - row3 = _mm_add_epi32( row3, row4 ); \ - row2 = _mm_xor_si128( row2, row3 ); \ - row2 = _mm_roti_epi32(row2, -7); - -#define DIAGONALIZE(row1,row2,row3,row4) \ - row4 = _mm_shuffle_epi32( row4, _MM_SHUFFLE(2,1,0,3) ); \ - row3 = _mm_shuffle_epi32( row3, _MM_SHUFFLE(1,0,3,2) ); \ - row2 = _mm_shuffle_epi32( row2, _MM_SHUFFLE(0,3,2,1) ); - -#define UNDIAGONALIZE(row1,row2,row3,row4) \ - row4 = _mm_shuffle_epi32( row4, _MM_SHUFFLE(0,3,2,1) ); \ - row3 = _mm_shuffle_epi32( row3, _MM_SHUFFLE(1,0,3,2) ); \ - row2 = _mm_shuffle_epi32( row2, _MM_SHUFFLE(2,1,0,3) ); - -#if defined(HAVE_XOP) -#include "blake2s-load-xop.h" -#elif defined(HAVE_SSE4_1) -#include "blake2s-load-sse41.h" -#else -#include "blake2s-load-sse2.h" -#endif - -#define ROUND(r) \ - LOAD_MSG_ ##r ##_1(buf1); \ - G1(row1,row2,row3,row4,buf1); \ - LOAD_MSG_ ##r ##_2(buf2); \ - G2(row1,row2,row3,row4,buf2); \ - DIAGONALIZE(row1,row2,row3,row4); \ - LOAD_MSG_ ##r ##_3(buf3); \ - G1(row1,row2,row3,row4,buf3); \ - LOAD_MSG_ ##r ##_4(buf4); \ - G2(row1,row2,row3,row4,buf4); \ - UNDIAGONALIZE(row1,row2,row3,row4); \ - -#endif - diff --git a/Modules/_blake2/impl/blake2s.c b/Modules/_blake2/impl/blake2s.c deleted file mode 100644 index e7f63fd274f212..00000000000000 --- a/Modules/_blake2/impl/blake2s.c +++ /dev/null @@ -1,415 +0,0 @@ -/* - BLAKE2 reference source code package - optimized C implementations - - Written in 2012 by Samuel Neves - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along with - this software. If not, see . -*/ - -#include -#include -#include - -#include "blake2.h" -#include "blake2-impl.h" - -#include "blake2-config.h" - -#if defined(_MSC_VER) -#include -#endif - -#if defined(HAVE_SSE2) -#include -// MSVC only defines _mm_set_epi64x for x86_64... -#if defined(_MSC_VER) && !defined(_M_X64) && !defined(__clang__) -static inline __m128i _mm_set_epi64x( const uint64_t u1, const uint64_t u0 ) -{ - return _mm_set_epi32( u1 >> 32, u1, u0 >> 32, u0 ); -} -#endif -#endif - - -#if defined(HAVE_SSSE3) -#include -#endif -#if defined(HAVE_SSE4_1) -#include -#endif -#if defined(HAVE_AVX) -#include -#endif -#if defined(HAVE_XOP) && !defined(_MSC_VER) -#include -#endif - -#include "blake2s-round.h" - -static const uint32_t blake2s_IV[8] = -{ - 0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, 0xA54FF53AUL, - 0x510E527FUL, 0x9B05688CUL, 0x1F83D9ABUL, 0x5BE0CD19UL -}; - -static const uint8_t blake2s_sigma[10][16] = -{ - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , - { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , - { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , - { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , - { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , - { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , - { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , - { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , - { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , -}; - - -/* Some helper functions, not necessarily useful */ -static inline int blake2s_set_lastnode( blake2s_state *S ) -{ - S->f[1] = ~0U; - return 0; -} - -static inline int blake2s_clear_lastnode( blake2s_state *S ) -{ - S->f[1] = 0U; - return 0; -} - -static inline int blake2s_set_lastblock( blake2s_state *S ) -{ - if( S->last_node ) blake2s_set_lastnode( S ); - - S->f[0] = ~0U; - return 0; -} - -static inline int blake2s_clear_lastblock( blake2s_state *S ) -{ - if( S->last_node ) blake2s_clear_lastnode( S ); - - S->f[0] = 0U; - return 0; -} - -static inline int blake2s_increment_counter( blake2s_state *S, const uint32_t inc ) -{ - uint64_t t = ( ( uint64_t )S->t[1] << 32 ) | S->t[0]; - t += inc; - S->t[0] = ( uint32_t )( t >> 0 ); - S->t[1] = ( uint32_t )( t >> 32 ); - return 0; -} - - -// Parameter-related functions -static inline int blake2s_param_set_digest_length( blake2s_param *P, const uint8_t digest_length ) -{ - P->digest_length = digest_length; - return 0; -} - -static inline int blake2s_param_set_fanout( blake2s_param *P, const uint8_t fanout ) -{ - P->fanout = fanout; - return 0; -} - -static inline int blake2s_param_set_max_depth( blake2s_param *P, const uint8_t depth ) -{ - P->depth = depth; - return 0; -} - -static inline int blake2s_param_set_leaf_length( blake2s_param *P, const uint32_t leaf_length ) -{ - P->leaf_length = leaf_length; - return 0; -} - -static inline int blake2s_param_set_node_offset( blake2s_param *P, const uint64_t node_offset ) -{ - store48( P->node_offset, node_offset ); - return 0; -} - -static inline int blake2s_param_set_node_depth( blake2s_param *P, const uint8_t node_depth ) -{ - P->node_depth = node_depth; - return 0; -} - -static inline int blake2s_param_set_inner_length( blake2s_param *P, const uint8_t inner_length ) -{ - P->inner_length = inner_length; - return 0; -} - -static inline int blake2s_param_set_salt( blake2s_param *P, const uint8_t salt[BLAKE2S_SALTBYTES] ) -{ - memcpy( P->salt, salt, BLAKE2S_SALTBYTES ); - return 0; -} - -static inline int blake2s_param_set_personal( blake2s_param *P, const uint8_t personal[BLAKE2S_PERSONALBYTES] ) -{ - memcpy( P->personal, personal, BLAKE2S_PERSONALBYTES ); - return 0; -} - -static inline int blake2s_init0( blake2s_state *S ) -{ - memset( S, 0, sizeof( blake2s_state ) ); - - for( int i = 0; i < 8; ++i ) S->h[i] = blake2s_IV[i]; - - return 0; -} - -#if defined(__cplusplus) -extern "C" { -#endif - int blake2s_init( blake2s_state *S, size_t outlen ); - int blake2s_init_param( blake2s_state *S, const blake2s_param *P ); - int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ); - int blake2s_update( blake2s_state *S, const uint8_t *in, size_t inlen ); - int blake2s_final( blake2s_state *S, uint8_t *out, size_t outlen ); - int blake2s( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ); -#if defined(__cplusplus) -} -#endif - - -/* init2 xors IV with input parameter block */ -int blake2s_init_param( blake2s_state *S, const blake2s_param *P ) -{ - uint8_t *p, *h, *v; - //blake2s_init0( S ); - v = ( uint8_t * )( blake2s_IV ); - h = ( uint8_t * )( S->h ); - p = ( uint8_t * )( P ); - /* IV XOR ParamBlock */ - memset( S, 0, sizeof( blake2s_state ) ); - - for( int i = 0; i < BLAKE2S_OUTBYTES; ++i ) h[i] = v[i] ^ p[i]; - - S->outlen = P->digest_length; - return 0; -} - - -/* Some sort of default parameter block initialization, for sequential blake2s */ -int blake2s_init( blake2s_state *S, size_t outlen ) -{ - if ( ( !outlen ) || ( outlen > BLAKE2S_OUTBYTES ) ) return -1; - - const blake2s_param P = - { - outlen, - 0, - 1, - 1, - 0, - {0}, - 0, - 0, - {0}, - {0} - }; - return blake2s_init_param( S, &P ); -} - - -int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ) -{ - if ( ( !outlen ) || ( outlen > BLAKE2S_OUTBYTES ) ) return -1; - - if ( ( !key ) || ( !keylen ) || keylen > BLAKE2S_KEYBYTES ) return -1; - - const blake2s_param P = - { - outlen, - keylen, - 1, - 1, - 0, - {0}, - 0, - 0, - {0}, - {0} - }; - - if( blake2s_init_param( S, &P ) < 0 ) - return -1; - - { - uint8_t block[BLAKE2S_BLOCKBYTES]; - memset( block, 0, BLAKE2S_BLOCKBYTES ); - memcpy( block, key, keylen ); - blake2s_update( S, block, BLAKE2S_BLOCKBYTES ); - secure_zero_memory( block, BLAKE2S_BLOCKBYTES ); /* Burn the key from stack */ - } - return 0; -} - - -static inline int blake2s_compress( blake2s_state *S, const uint8_t block[BLAKE2S_BLOCKBYTES] ) -{ - __m128i row1, row2, row3, row4; - __m128i buf1, buf2, buf3, buf4; -#if defined(HAVE_SSE4_1) - __m128i t0, t1; -#if !defined(HAVE_XOP) - __m128i t2; -#endif -#endif - __m128i ff0, ff1; -#if defined(HAVE_SSSE3) && !defined(HAVE_XOP) - const __m128i r8 = _mm_set_epi8( 12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1 ); - const __m128i r16 = _mm_set_epi8( 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2 ); -#endif -#if defined(HAVE_SSE4_1) - const __m128i m0 = LOADU( block + 00 ); - const __m128i m1 = LOADU( block + 16 ); - const __m128i m2 = LOADU( block + 32 ); - const __m128i m3 = LOADU( block + 48 ); -#else - const uint32_t m0 = ( ( uint32_t * )block )[ 0]; - const uint32_t m1 = ( ( uint32_t * )block )[ 1]; - const uint32_t m2 = ( ( uint32_t * )block )[ 2]; - const uint32_t m3 = ( ( uint32_t * )block )[ 3]; - const uint32_t m4 = ( ( uint32_t * )block )[ 4]; - const uint32_t m5 = ( ( uint32_t * )block )[ 5]; - const uint32_t m6 = ( ( uint32_t * )block )[ 6]; - const uint32_t m7 = ( ( uint32_t * )block )[ 7]; - const uint32_t m8 = ( ( uint32_t * )block )[ 8]; - const uint32_t m9 = ( ( uint32_t * )block )[ 9]; - const uint32_t m10 = ( ( uint32_t * )block )[10]; - const uint32_t m11 = ( ( uint32_t * )block )[11]; - const uint32_t m12 = ( ( uint32_t * )block )[12]; - const uint32_t m13 = ( ( uint32_t * )block )[13]; - const uint32_t m14 = ( ( uint32_t * )block )[14]; - const uint32_t m15 = ( ( uint32_t * )block )[15]; -#endif - row1 = ff0 = LOADU( &S->h[0] ); - row2 = ff1 = LOADU( &S->h[4] ); - row3 = _mm_setr_epi32( 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A ); - row4 = _mm_xor_si128( _mm_setr_epi32( 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19 ), LOADU( &S->t[0] ) ); - ROUND( 0 ); - ROUND( 1 ); - ROUND( 2 ); - ROUND( 3 ); - ROUND( 4 ); - ROUND( 5 ); - ROUND( 6 ); - ROUND( 7 ); - ROUND( 8 ); - ROUND( 9 ); - STOREU( &S->h[0], _mm_xor_si128( ff0, _mm_xor_si128( row1, row3 ) ) ); - STOREU( &S->h[4], _mm_xor_si128( ff1, _mm_xor_si128( row2, row4 ) ) ); - return 0; -} - - -int blake2s_update( blake2s_state *S, const uint8_t *in, size_t inlen ) -{ - while( inlen > 0 ) - { - size_t left = S->buflen; - size_t fill = 2 * BLAKE2S_BLOCKBYTES - left; - - if( inlen > fill ) - { - memcpy( S->buf + left, in, fill ); // Fill buffer - S->buflen += fill; - blake2s_increment_counter( S, BLAKE2S_BLOCKBYTES ); - blake2s_compress( S, S->buf ); // Compress - memcpy( S->buf, S->buf + BLAKE2S_BLOCKBYTES, BLAKE2S_BLOCKBYTES ); // Shift buffer left - S->buflen -= BLAKE2S_BLOCKBYTES; - in += fill; - inlen -= fill; - } - else /* inlen <= fill */ - { - memcpy( S->buf + left, in, inlen ); - S->buflen += inlen; // Be lazy, do not compress - in += inlen; - inlen -= inlen; - } - } - - return 0; -} - - -int blake2s_final( blake2s_state *S, uint8_t *out, size_t outlen ) -{ - uint8_t buffer[BLAKE2S_OUTBYTES]; - - if(outlen != S->outlen ) return -1; - - if( S->buflen > BLAKE2S_BLOCKBYTES ) - { - blake2s_increment_counter( S, BLAKE2S_BLOCKBYTES ); - blake2s_compress( S, S->buf ); - S->buflen -= BLAKE2S_BLOCKBYTES; - memmove( S->buf, S->buf + BLAKE2S_BLOCKBYTES, S->buflen ); - } - - blake2s_increment_counter( S, ( uint32_t )S->buflen ); - blake2s_set_lastblock( S ); - memset( S->buf + S->buflen, 0, 2 * BLAKE2S_BLOCKBYTES - S->buflen ); /* Padding */ - blake2s_compress( S, S->buf ); - - for( int i = 0; i < 8; ++i ) /* Output full hash to temp buffer */ - store32( buffer + sizeof( S->h[i] ) * i, S->h[i] ); - - memcpy( out, buffer, outlen ); - return 0; -} - -int blake2s( uint8_t *out, const void *in, const void *key, size_t outlen, size_t inlen, size_t keylen ) -{ - blake2s_state S[1]; - - /* Verify parameters */ - if ( NULL == in && inlen > 0 ) return -1; - - if ( NULL == out ) return -1; - - if ( NULL == key && keylen > 0) return -1; - - if( !outlen || outlen > BLAKE2S_OUTBYTES ) return -1; - - if( keylen > BLAKE2S_KEYBYTES ) return -1; - - if( keylen > 0 ) - { - if( blake2s_init_key( S, outlen, key, keylen ) < 0 ) return -1; - } - else - { - if( blake2s_init( S, outlen ) < 0 ) return -1; - } - - if( blake2s_update( S, ( uint8_t * )in, inlen ) < 0) return -1; - return blake2s_final( S, out, outlen ); -} - -#if defined(SUPERCOP) -int crypto_hash( unsigned char *out, unsigned char *in, unsigned long long inlen ) -{ - return blake2s( out, in, NULL, BLAKE2S_OUTBYTES, (size_t)inlen, 0 ); -} -#endif - diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 3d0d4ee5e79c2b..9e85e0de42cd8d 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -129,6 +129,9 @@ typedef struct { PyThread_type_lock lock; } BZ2Decompressor; +#define _BZ2Compressor_CAST(op) ((BZ2Compressor *)(op)) +#define _BZ2Decompressor_CAST(op) ((BZ2Decompressor *)(op)) + /* Helper functions. */ static int @@ -376,8 +379,9 @@ _bz2_BZ2Compressor_impl(PyTypeObject *type, int compresslevel) } static void -BZ2Compressor_dealloc(BZ2Compressor *self) +BZ2Compressor_dealloc(PyObject *op) { + BZ2Compressor *self = _BZ2Compressor_CAST(op); BZ2_bzCompressEnd(&self->bzs); if (self->lock != NULL) { PyThread_free_lock(self->lock); @@ -388,7 +392,7 @@ BZ2Compressor_dealloc(BZ2Compressor *self) } static int -BZ2Compressor_traverse(BZ2Compressor *self, visitproc visit, void *arg) +BZ2Compressor_traverse(PyObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); return 0; @@ -680,8 +684,10 @@ _bz2_BZ2Decompressor_impl(PyTypeObject *type) } static void -BZ2Decompressor_dealloc(BZ2Decompressor *self) +BZ2Decompressor_dealloc(PyObject *op) { + BZ2Decompressor *self = _BZ2Decompressor_CAST(op); + if(self->input_buffer != NULL) { PyMem_Free(self->input_buffer); } @@ -697,7 +703,7 @@ BZ2Decompressor_dealloc(BZ2Decompressor *self) } static int -BZ2Decompressor_traverse(BZ2Decompressor *self, visitproc visit, void *arg) +BZ2Decompressor_traverse(PyObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); return 0; @@ -802,6 +808,7 @@ _bz2_free(void *module) static struct PyModuleDef_Slot _bz2_slots[] = { {Py_mod_exec, _bz2_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index c31c1b6d6f2bbc..471b42badc8e8c 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -979,6 +979,30 @@ _codecs_register_error_impl(PyObject *module, const char *errors, Py_RETURN_NONE; } +/*[clinic input] +_codecs._unregister_error -> bool + errors: str + / + +Un-register the specified error handler for the error handling `errors'. + +Only custom error handlers can be un-registered. An exception is raised +if the error handling is a built-in one (e.g., 'strict'), or if an error +occurs. + +Otherwise, this returns True if a custom handler has been successfully +un-registered, and False if no custom handler for the specified error +handling exists. + +[clinic start generated code]*/ + +static int +_codecs__unregister_error_impl(PyObject *module, const char *errors) +/*[clinic end generated code: output=28c22be667465503 input=a63ab9e9ce1686d4]*/ +{ + return _PyCodec_UnregisterError(errors); +} + /*[clinic input] _codecs.lookup_error name: str @@ -1044,12 +1068,14 @@ static PyMethodDef _codecs_functions[] = { _CODECS_CODE_PAGE_ENCODE_METHODDEF _CODECS_CODE_PAGE_DECODE_METHODDEF _CODECS_REGISTER_ERROR_METHODDEF + _CODECS__UNREGISTER_ERROR_METHODDEF _CODECS_LOOKUP_ERROR_METHODDEF {NULL, NULL} /* sentinel */ }; static PyModuleDef_Slot _codecs_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 309d63c9bf7cbe..aef04248c7e73c 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1293,7 +1293,7 @@ deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start, index = 0; } } - PyErr_Format(PyExc_ValueError, "%R is not in deque", v); + PyErr_SetString(PyExc_ValueError, "deque.index(x): x not in deque"); return NULL; } @@ -1462,7 +1462,7 @@ deque_remove_impl(dequeobject *deque, PyObject *value) } } if (i == n) { - PyErr_Format(PyExc_ValueError, "%R is not in deque", value); + PyErr_SetString(PyExc_ValueError, "deque.remove(x): x not in deque"); return NULL; } rv = deque_del_item(deque, i); @@ -2179,6 +2179,8 @@ typedef struct { PyObject *default_factory; } defdictobject; +static PyType_Spec defdict_spec; + PyDoc_STRVAR(defdict_missing_doc, "__missing__(key) # Called by __getitem__ for missing key; pseudo-code:\n\ if self.default_factory is None: raise KeyError((key,))\n\ @@ -2358,23 +2360,16 @@ defdict_or(PyObject* left, PyObject* right) { PyObject *self, *other; - // Find module state - PyTypeObject *tp = Py_TYPE(left); - PyObject *mod = PyType_GetModuleByDef(tp, &_collectionsmodule); - if (mod == NULL) { - PyErr_Clear(); - tp = Py_TYPE(right); - mod = PyType_GetModuleByDef(tp, &_collectionsmodule); + int ret = PyType_GetBaseByToken(Py_TYPE(left), &defdict_spec, NULL); + if (ret < 0) { + return NULL; } - assert(mod != NULL); - collections_state *state = get_module_state(mod); - - if (PyObject_TypeCheck(left, state->defdict_type)) { + if (ret) { self = left; other = right; } else { - assert(PyObject_TypeCheck(right, state->defdict_type)); + assert(PyType_GetBaseByToken(Py_TYPE(right), &defdict_spec, NULL) == 1); self = right; other = left; } @@ -2454,6 +2449,7 @@ passed to the dict constructor, including keyword arguments.\n\ #define DEFERRED_ADDRESS(ADDR) 0 static PyType_Slot defdict_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, {Py_tp_dealloc, defdict_dealloc}, {Py_tp_repr, defdict_repr}, {Py_nb_or, defdict_or}, @@ -2511,9 +2507,9 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, /* Only take the fast path when get() and __setitem__() * have not been overridden. */ - mapping_get = _PyType_Lookup(Py_TYPE(mapping), &_Py_ID(get)); + mapping_get = _PyType_LookupRef(Py_TYPE(mapping), &_Py_ID(get)); dict_get = _PyType_Lookup(&PyDict_Type, &_Py_ID(get)); - mapping_setitem = _PyType_Lookup(Py_TYPE(mapping), &_Py_ID(__setitem__)); + mapping_setitem = _PyType_LookupRef(Py_TYPE(mapping), &_Py_ID(__setitem__)); dict_setitem = _PyType_Lookup(&PyDict_Type, &_Py_ID(__setitem__)); if (mapping_get != NULL && mapping_get == dict_get && @@ -2537,12 +2533,9 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, if (key == NULL) break; - if (!PyUnicode_CheckExact(key) || - (hash = _PyASCIIObject_CAST(key)->hash) == -1) - { - hash = PyObject_Hash(key); - if (hash == -1) - goto done; + hash = _PyObject_HashFast(key); + if (hash == -1) { + goto done; } oldval = _PyDict_GetItem_KnownHash(mapping, key, hash); @@ -2575,7 +2568,11 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, oldval = PyObject_CallFunctionObjArgs(bound_get, key, zero, NULL); if (oldval == NULL) break; - newval = PyNumber_Add(oldval, one); + if (oldval == zero) { + newval = Py_NewRef(one); + } else { + newval = PyNumber_Add(oldval, one); + } Py_DECREF(oldval); if (newval == NULL) break; @@ -2587,6 +2584,8 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, } done: + Py_XDECREF(mapping_get); + Py_XDECREF(mapping_setitem); Py_DECREF(it); Py_XDECREF(key); Py_XDECREF(newval); @@ -2817,6 +2816,7 @@ collections_exec(PyObject *module) { static struct PyModuleDef_Slot collections_slots[] = { {Py_mod_exec, collections_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_complex.h b/Modules/_complex.h new file mode 100644 index 00000000000000..28d4a32794b97c --- /dev/null +++ b/Modules/_complex.h @@ -0,0 +1,54 @@ +/* Workarounds for buggy complex number arithmetic implementations. */ + +#ifndef Py_HAVE_C_COMPLEX +# error "this header file should only be included if Py_HAVE_C_COMPLEX is defined" +#endif + +#include + +/* Other compilers (than clang), that claims to + implement C11 *and* define __STDC_IEC_559_COMPLEX__ - don't have + issue with CMPLX(). This is specific to glibc & clang combination: + https://sourceware.org/bugzilla/show_bug.cgi?id=26287 + + Here we fallback to using __builtin_complex(), available in clang + v12+. Else CMPLX implemented following C11 6.2.5p13: "Each complex type + has the same representation and alignment requirements as an array + type containing exactly two elements of the corresponding real type; + the first element is equal to the real part, and the second element + to the imaginary part, of the complex number. + */ +#if !defined(CMPLX) +# if defined(__clang__) && __has_builtin(__builtin_complex) +# define CMPLX(x, y) __builtin_complex ((double) (x), (double) (y)) +# define CMPLXF(x, y) __builtin_complex ((float) (x), (float) (y)) +# define CMPLXL(x, y) __builtin_complex ((long double) (x), (long double) (y)) +# else +static inline double complex +CMPLX(double real, double imag) +{ + double complex z; + ((double *)(&z))[0] = real; + ((double *)(&z))[1] = imag; + return z; +} + +static inline float complex +CMPLXF(float real, float imag) +{ + float complex z; + ((float *)(&z))[0] = real; + ((float *)(&z))[1] = imag; + return z; +} + +static inline long double complex +CMPLXL(long double real, long double imag) +{ + long double complex z; + ((long double *)(&z))[0] = real; + ((long double *)(&z))[1] = imag; + return z; +} +# endif +#endif diff --git a/Modules/_contextvarsmodule.c b/Modules/_contextvarsmodule.c index f621c1de6d42d6..3f96f07909b69a 100644 --- a/Modules/_contextvarsmodule.c +++ b/Modules/_contextvarsmodule.c @@ -45,6 +45,7 @@ _contextvars_exec(PyObject *m) static struct PyModuleDef_Slot _contextvars_slots[] = { {Py_mod_exec, _contextvars_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_csv.c b/Modules/_csv.c index 660c5455af764e..e5ae853590bf2c 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -14,6 +14,7 @@ module instead. #endif #include "Python.h" +#include "pycore_pyatomic_ft_wrappers.h" #include // offsetof() #include @@ -34,7 +35,7 @@ typedef struct { PyTypeObject *dialect_type; PyTypeObject *reader_type; PyTypeObject *writer_type; - long field_limit; /* max parsed field size */ + Py_ssize_t field_limit; /* max parsed field size */ PyObject *str_write; } _csvstate; @@ -76,7 +77,7 @@ _csv_traverse(PyObject *module, visitproc visit, void *arg) static void _csv_free(void *module) { - _csv_clear((PyObject *)module); + (void)_csv_clear((PyObject *)module); } typedef enum { @@ -150,6 +151,10 @@ typedef struct { PyObject *error_obj; /* cached error object */ } WriterObj; +#define _DialectObj_CAST(op) ((DialectObj *)(op)) +#define _ReaderObj_CAST(op) ((ReaderObj *)(op)) +#define _WriterObj_CAST(op) ((WriterObj *)(op)) + /* * DIALECT class */ @@ -175,32 +180,37 @@ get_char_or_None(Py_UCS4 c) } static PyObject * -Dialect_get_lineterminator(DialectObj *self, void *Py_UNUSED(ignored)) +Dialect_get_lineterminator(PyObject *op, void *Py_UNUSED(ignored)) { + DialectObj *self = _DialectObj_CAST(op); return Py_XNewRef(self->lineterminator); } static PyObject * -Dialect_get_delimiter(DialectObj *self, void *Py_UNUSED(ignored)) +Dialect_get_delimiter(PyObject *op, void *Py_UNUSED(ignored)) { + DialectObj *self = _DialectObj_CAST(op); return get_char_or_None(self->delimiter); } static PyObject * -Dialect_get_escapechar(DialectObj *self, void *Py_UNUSED(ignored)) +Dialect_get_escapechar(PyObject *op, void *Py_UNUSED(ignored)) { + DialectObj *self = _DialectObj_CAST(op); return get_char_or_None(self->escapechar); } static PyObject * -Dialect_get_quotechar(DialectObj *self, void *Py_UNUSED(ignored)) +Dialect_get_quotechar(PyObject *op, void *Py_UNUSED(ignored)) { + DialectObj *self = _DialectObj_CAST(op); return get_char_or_None(self->quotechar); } static PyObject * -Dialect_get_quoting(DialectObj *self, void *Py_UNUSED(ignored)) +Dialect_get_quoting(PyObject *op, void *Py_UNUSED(ignored)) { + DialectObj *self = _DialectObj_CAST(op); return PyLong_FromLong(self->quoting); } @@ -367,17 +377,19 @@ static struct PyMemberDef Dialect_memberlist[] = { { NULL } }; +#undef D_OFF + static PyGetSetDef Dialect_getsetlist[] = { - { "delimiter", (getter)Dialect_get_delimiter}, - { "escapechar", (getter)Dialect_get_escapechar}, - { "lineterminator", (getter)Dialect_get_lineterminator}, - { "quotechar", (getter)Dialect_get_quotechar}, - { "quoting", (getter)Dialect_get_quoting}, + {"delimiter", Dialect_get_delimiter}, + {"escapechar", Dialect_get_escapechar}, + {"lineterminator", Dialect_get_lineterminator}, + {"quotechar", Dialect_get_quotechar}, + {"quoting", Dialect_get_quoting}, {NULL}, }; static void -Dialect_dealloc(DialectObj *self) +Dialect_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); @@ -502,6 +514,7 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) DIALECT_GETATTR(skipinitialspace, "skipinitialspace"); DIALECT_GETATTR(strict, "strict"); } +#undef DIALECT_GETATTR /* check types and convert to C values */ #define DIASET(meth, name, target, src, dflt) \ @@ -515,6 +528,7 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) DIASET(_set_int, "quoting", &self->quoting, quoting, QUOTE_MINIMAL); DIASET(_set_bool, "skipinitialspace", &self->skipinitialspace, skipinitialspace, false); DIASET(_set_bool, "strict", &self->strict, strict, false); +#undef DIASET /* validate options */ if (dialect_check_quoting(self->quoting)) @@ -589,15 +603,17 @@ PyDoc_STRVAR(Dialect_Type_doc, "The Dialect type records CSV parsing and generation options.\n"); static int -Dialect_clear(DialectObj *self) +Dialect_clear(PyObject *op) { + DialectObj *self = _DialectObj_CAST(op); Py_CLEAR(self->lineterminator); return 0; } static int -Dialect_traverse(DialectObj *self, visitproc visit, void *arg) +Dialect_traverse(PyObject *op, visitproc visit, void *arg) { + DialectObj *self = _DialectObj_CAST(op); Py_VISIT(self->lineterminator); Py_VISIT(Py_TYPE(self)); return 0; @@ -702,10 +718,11 @@ parse_grow_buff(ReaderObj *self) static int parse_add_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) { - if (self->field_len >= module_state->field_limit) { + Py_ssize_t field_limit = FT_ATOMIC_LOAD_SSIZE_RELAXED(module_state->field_limit); + if (self->field_len >= field_limit) { PyErr_Format(module_state->error_obj, - "field larger than field limit (%ld)", - module_state->field_limit); + "field larger than field limit (%zd)", + field_limit); return -1; } if (self->field_len == self->field_size && !parse_grow_buff(self)) @@ -731,7 +748,7 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) } /* normal character - handle as START_FIELD */ self->state = START_FIELD; - /* fallthru */ + _Py_FALLTHROUGH; case START_FIELD: /* expecting field */ self->unquoted_field = true; @@ -749,7 +766,6 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) } else if (c == dialect->escapechar) { /* possible escaped character */ - self->unquoted_field = false; self->state = ESCAPED_CHAR; } else if (c == ' ' && dialect->skipinitialspace) @@ -785,7 +801,7 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) case AFTER_ESCAPED_CRNL: if (c == EOL) break; - /*fallthru*/ + _Py_FALLTHROUGH; case IN_FIELD: /* in unquoted field */ @@ -911,8 +927,10 @@ parse_reset(ReaderObj *self) } static PyObject * -Reader_iternext(ReaderObj *self) +Reader_iternext(PyObject *op) { + ReaderObj *self = _ReaderObj_CAST(op); + PyObject *fields = NULL; Py_UCS4 c; Py_ssize_t pos, linelen; @@ -977,11 +995,12 @@ Reader_iternext(ReaderObj *self) } static void -Reader_dealloc(ReaderObj *self) +Reader_dealloc(PyObject *op) { + ReaderObj *self = _ReaderObj_CAST(op); PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - tp->tp_clear((PyObject *)self); + (void)tp->tp_clear(op); if (self->field != NULL) { PyMem_Free(self->field); self->field = NULL; @@ -991,8 +1010,9 @@ Reader_dealloc(ReaderObj *self) } static int -Reader_traverse(ReaderObj *self, visitproc visit, void *arg) +Reader_traverse(PyObject *op, visitproc visit, void *arg) { + ReaderObj *self = _ReaderObj_CAST(op); Py_VISIT(self->dialect); Py_VISIT(self->input_iter); Py_VISIT(self->fields); @@ -1001,8 +1021,9 @@ Reader_traverse(ReaderObj *self, visitproc visit, void *arg) } static int -Reader_clear(ReaderObj *self) +Reader_clear(PyObject *op) { + ReaderObj *self = _ReaderObj_CAST(op); Py_CLEAR(self->dialect); Py_CLEAR(self->input_iter); Py_CLEAR(self->fields); @@ -1027,6 +1048,8 @@ static struct PyMemberDef Reader_memberlist[] = { { NULL } }; +#undef R_OFF + static PyType_Slot Reader_Type_slots[] = { {Py_tp_doc, (char*)Reader_Type_doc}, @@ -1073,7 +1096,7 @@ csv_reader(PyObject *module, PyObject *args, PyObject *keyword_args) return NULL; } - if (!PyArg_UnpackTuple(args, "", 1, 2, &iterator, &dialect)) { + if (!PyArg_UnpackTuple(args, "reader", 1, 2, &iterator, &dialect)) { Py_DECREF(self); return NULL; } @@ -1115,7 +1138,7 @@ join_append_data(WriterObj *self, int field_kind, const void *field_data, int copy_phase) { DialectObj *dialect = self->dialect; - int i; + Py_ssize_t i; Py_ssize_t rec_len; #define INCLEN \ @@ -1296,8 +1319,9 @@ PyDoc_STRVAR(csv_writerow_doc, "elements will be converted to string."); static PyObject * -csv_writerow(WriterObj *self, PyObject *seq) +csv_writerow(PyObject *op, PyObject *seq) { + WriterObj *self = _WriterObj_CAST(op); DialectObj *dialect = self->dialect; PyObject *iter, *field, *line, *result; bool null_field = false; @@ -1405,7 +1429,7 @@ PyDoc_STRVAR(csv_writerows_doc, "elements will be converted to string."); static PyObject * -csv_writerows(WriterObj *self, PyObject *seqseq) +csv_writerows(PyObject *self, PyObject *seqseq) { PyObject *row_iter, *row_obj, *result; @@ -1430,9 +1454,9 @@ csv_writerows(WriterObj *self, PyObject *seqseq) } static struct PyMethodDef Writer_methods[] = { - { "writerow", (PyCFunction)csv_writerow, METH_O, csv_writerow_doc}, - { "writerows", (PyCFunction)csv_writerows, METH_O, csv_writerows_doc}, - { NULL, NULL } + {"writerow", csv_writerow, METH_O, csv_writerow_doc}, + {"writerows", csv_writerows, METH_O, csv_writerows_doc}, + {NULL, NULL, 0, NULL} /* sentinel */ }; #define W_OFF(x) offsetof(WriterObj, x) @@ -1442,9 +1466,12 @@ static struct PyMemberDef Writer_memberlist[] = { { NULL } }; +#undef W_OFF + static int -Writer_traverse(WriterObj *self, visitproc visit, void *arg) +Writer_traverse(PyObject *op, visitproc visit, void *arg) { + WriterObj *self = _WriterObj_CAST(op); Py_VISIT(self->dialect); Py_VISIT(self->write); Py_VISIT(self->error_obj); @@ -1453,8 +1480,9 @@ Writer_traverse(WriterObj *self, visitproc visit, void *arg) } static int -Writer_clear(WriterObj *self) +Writer_clear(PyObject *op) { + WriterObj *self = _WriterObj_CAST(op); Py_CLEAR(self->dialect); Py_CLEAR(self->write); Py_CLEAR(self->error_obj); @@ -1462,11 +1490,12 @@ Writer_clear(WriterObj *self) } static void -Writer_dealloc(WriterObj *self) +Writer_dealloc(PyObject *op) { + WriterObj *self = _WriterObj_CAST(op); PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - tp->tp_clear((PyObject *)self); + tp->tp_clear(op); if (self->rec != NULL) { PyMem_Free(self->rec); } @@ -1520,7 +1549,7 @@ csv_writer(PyObject *module, PyObject *args, PyObject *keyword_args) self->error_obj = Py_NewRef(module_state->error_obj); - if (!PyArg_UnpackTuple(args, "", 1, 2, &output_file, &dialect)) { + if (!PyArg_UnpackTuple(args, "writer", 1, 2, &output_file, &dialect)) { Py_DECREF(self); return NULL; } @@ -1572,7 +1601,7 @@ csv_register_dialect(PyObject *module, PyObject *args, PyObject *kwargs) _csvstate *module_state = get_csv_state(module); PyObject *dialect; - if (!PyArg_UnpackTuple(args, "", 1, 2, &name_obj, &dialect_obj)) + if (!PyArg_UnpackTuple(args, "register_dialect", 1, 2, &name_obj, &dialect_obj)) return NULL; if (!PyUnicode_Check(name_obj)) { PyErr_SetString(PyExc_TypeError, @@ -1606,10 +1635,12 @@ _csv_unregister_dialect_impl(PyObject *module, PyObject *name) /*[clinic end generated code: output=0813ebca6c058df4 input=6b5c1557bf60c7e7]*/ { _csvstate *module_state = get_csv_state(module); - if (PyDict_DelItem(module_state->dialects, name) < 0) { - if (PyErr_ExceptionMatches(PyExc_KeyError)) { - PyErr_Format(module_state->error_obj, "unknown dialect"); - } + int rc = PyDict_Pop(module_state->dialects, name, NULL); + if (rc < 0) { + return NULL; + } + if (rc == 0) { + PyErr_Format(module_state->error_obj, "unknown dialect"); return NULL; } Py_RETURN_NONE; @@ -1650,20 +1681,20 @@ _csv_field_size_limit_impl(PyObject *module, PyObject *new_limit) /*[clinic end generated code: output=f2799ecd908e250b input=cec70e9226406435]*/ { _csvstate *module_state = get_csv_state(module); - long old_limit = module_state->field_limit; + Py_ssize_t old_limit = FT_ATOMIC_LOAD_SSIZE_RELAXED(module_state->field_limit); if (new_limit != NULL) { if (!PyLong_CheckExact(new_limit)) { PyErr_Format(PyExc_TypeError, "limit must be an integer"); return NULL; } - module_state->field_limit = PyLong_AsLong(new_limit); - if (module_state->field_limit == -1 && PyErr_Occurred()) { - module_state->field_limit = old_limit; + Py_ssize_t new_limit_value = PyLong_AsSsize_t(new_limit); + if (new_limit_value == -1 && PyErr_Occurred()) { return NULL; } + FT_ATOMIC_STORE_SSIZE_RELAXED(module_state->field_limit, new_limit_value); } - return PyLong_FromLong(old_limit); + return PyLong_FromSsize_t(old_limit); } static PyType_Slot error_slots[] = { @@ -1794,6 +1825,7 @@ csv_exec(PyObject *module) { static PyModuleDef_Slot csv_slots[] = { {Py_mod_exec, csv_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 94245ae41afffc..d86adec0aeca58 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2,7 +2,7 @@ ToDo: Get rid of the checker (and also the converters) field in PyCFuncPtrObject and - StgDictObject, and replace them by slot functions in StgDictObject. + StgInfo, and replace them by slot functions in StgInfo. think about a buffer-like object (memory? bytes?) @@ -36,7 +36,6 @@ PyCData_Type Simple_Type __new__(), __init__(), _as_parameter_ PyCField_Type -PyCStgDict_Type ============================================================================== @@ -82,7 +81,6 @@ bytes(cdata) */ /* - * PyCStgDict_Type * PyCStructType_Type * UnionType_Type * PyCPointerType_Type @@ -128,30 +126,17 @@ bytes(cdata) #include "pycore_long.h" // _PyLong_GetZero() -static PyTypeObject Union_Type; -static PyTypeObject Struct_Type; -static PyTypeObject Simple_Type; - -ctypes_state global_state = { - .PyCStgDict_Type = &PyCStgDict_Type, - .PyCData_Type = &PyCData_Type, - .Struct_Type = &Struct_Type, - .Union_Type = &Union_Type, - .PyCArray_Type = &PyCArray_Type, - .Simple_Type = &Simple_Type, - .PyCPointer_Type = &PyCPointer_Type, - .PyCFuncPtr_Type = &PyCFuncPtr_Type, -}; - -PyObject *PyExc_ArgError = NULL; - -/* This dict maps ctypes types to POINTER types */ -PyObject *_ctypes_ptrtype_cache = NULL; - -/* a callable object used for unpickling: - strong reference to _ctypes._unpickle() function */ -static PyObject *_unpickle; +/*[clinic input] +module _ctypes +class _ctypes.CFuncPtr "PyCFuncPtrObject *" "&PyCFuncPtr_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=58e8c99474bc631e]*/ +#define clinic_state() (get_module_state_by_class(cls)) +#define clinic_state_sub() (get_module_state_by_class(cls->tp_base)) +#include "clinic/_ctypes.c.h" +#undef clinic_state +#undef clinic_state_sub /****************************************************************/ @@ -161,9 +146,12 @@ typedef struct { PyObject *dict; } DictRemoverObject; +#define _DictRemoverObject_CAST(op) ((DictRemoverObject *)(op)) + static int -_DictRemover_traverse(DictRemoverObject *self, visitproc visit, void *arg) +_DictRemover_traverse(PyObject *myself, visitproc visit, void *arg) { + DictRemoverObject *self = _DictRemoverObject_CAST(myself); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->key); Py_VISIT(self->dict); @@ -171,8 +159,9 @@ _DictRemover_traverse(DictRemoverObject *self, visitproc visit, void *arg) } static int -_DictRemover_clear(DictRemoverObject *self) +_DictRemover_clear(PyObject *myself) { + DictRemoverObject *self = _DictRemoverObject_CAST(myself); Py_CLEAR(self->key); Py_CLEAR(self->dict); return 0; @@ -182,9 +171,8 @@ static void _DictRemover_dealloc(PyObject *myself) { PyTypeObject *tp = Py_TYPE(myself); - DictRemoverObject *self = (DictRemoverObject *)myself; PyObject_GC_UnTrack(myself); - (void)_DictRemover_clear(self); + (void)_DictRemover_clear(myself); tp->tp_free(myself); Py_DECREF(tp); } @@ -192,7 +180,7 @@ _DictRemover_dealloc(PyObject *myself) static PyObject * _DictRemover_call(PyObject *myself, PyObject *args, PyObject *kw) { - DictRemoverObject *self = (DictRemoverObject *)myself; + DictRemoverObject *self = _DictRemoverObject_CAST(myself); if (self->key && self->dict) { if (-1 == PyDict_DelItem(self->dict, self->key)) { PyErr_FormatUnraisable("Exception ignored on calling _ctypes.DictRemover"); @@ -223,14 +211,13 @@ static PyType_Spec dictremover_spec = { }; int -PyDict_SetItemProxy(PyObject *dict, PyObject *key, PyObject *item) +PyDict_SetItemProxy(ctypes_state *st, PyObject *dict, PyObject *key, PyObject *item) { PyObject *obj; DictRemoverObject *remover; PyObject *proxy; int result; - ctypes_state *st = GLOBAL_STATE(); obj = _PyObject_CallNoArgs((PyObject *)st->DictRemover_Type); if (obj == NULL) return -1; @@ -337,7 +324,7 @@ _ctypes_alloc_format_string_for_type(char code, int big_endian) indicator set. If called with a suffix of NULL the error indicator must already be set. */ -char * +static char * _ctypes_alloc_format_string(const char *prefix, const char *suffix) { size_t len; @@ -369,7 +356,7 @@ _ctypes_alloc_format_string(const char *prefix, const char *suffix) Returns NULL on failure, with the error indicator set. If called with a suffix of NULL the error indicator must already be set. */ -char * +static char * _ctypes_alloc_format_string_with_shape(int ndim, const Py_ssize_t *shape, const char *prefix, const char *suffix) { @@ -418,16 +405,19 @@ typedef struct { PyObject *keep; // If set, a reference to the original CDataObject. } StructParamObject; +#define _StructParamObject_CAST(op) ((StructParamObject *)(op)) + static int -StructParam_traverse(StructParamObject *self, visitproc visit, void *arg) +StructParam_traverse(PyObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); return 0; } static int -StructParam_clear(StructParamObject *self) +StructParam_clear(PyObject *myself) { + StructParamObject *self = _StructParamObject_CAST(myself); Py_CLEAR(self->keep); return 0; } @@ -435,10 +425,10 @@ StructParam_clear(StructParamObject *self) static void StructParam_dealloc(PyObject *myself) { - StructParamObject *self = (StructParamObject *)myself; + StructParamObject *self = _StructParamObject_CAST(myself); PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(myself); - (void)StructParam_clear(self); + (void)StructParam_clear(myself); PyMem_Free(self->ptr); tp->tp_free(myself); Py_DECREF(tp); @@ -459,20 +449,154 @@ static PyType_Spec structparam_spec = { .slots = structparam_slots, }; +/* + CType_Type - a base metaclass. Its instances (classes) have a StgInfo. + */ + +/*[clinic input] +class _ctypes.CType_Type "PyObject *" "clinic_state()->CType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=8389fc5b74a84f2a]*/ + +static int +CType_Type_traverse(PyObject *self, visitproc visit, void *arg) +{ + StgInfo *info = _PyStgInfo_FromType_NoState(self); + if (!info) { + PyErr_WriteUnraisable(self); + } + if (info) { + Py_VISIT(info->proto); + Py_VISIT(info->argtypes); + Py_VISIT(info->converters); + Py_VISIT(info->restype); + Py_VISIT(info->checker); + Py_VISIT(info->module); + } + Py_VISIT(Py_TYPE(self)); + return PyType_Type.tp_traverse(self, visit, arg); +} + +void +ctype_clear_stginfo(StgInfo *info) +{ + assert(info); + Py_CLEAR(info->proto); + Py_CLEAR(info->argtypes); + Py_CLEAR(info->converters); + Py_CLEAR(info->restype); + Py_CLEAR(info->checker); + Py_CLEAR(info->module); +} + +static int +CType_Type_clear(PyObject *self) +{ + StgInfo *info = _PyStgInfo_FromType_NoState(self); + if (!info) { + PyErr_WriteUnraisable(self); + } + if (info) { + ctype_clear_stginfo(info); + } + return PyType_Type.tp_clear(self); +} + +static void +CType_Type_dealloc(PyObject *self) +{ + StgInfo *info = _PyStgInfo_FromType_NoState(self); + if (!info) { + PyErr_WriteUnraisable(NULL); // NULL avoids segfault here + } + if (info) { + PyMem_Free(info->ffi_type_pointer.elements); + info->ffi_type_pointer.elements = NULL; + PyMem_Free(info->format); + info->format = NULL; + PyMem_Free(info->shape); + info->shape = NULL; + ctype_clear_stginfo(info); + } + + PyTypeObject *tp = Py_TYPE(self); + PyType_Type.tp_dealloc(self); + Py_DECREF(tp); +} + +/*[clinic input] +_ctypes.CType_Type.__sizeof__ + + cls: defining_class + / +Return memory consumption of the type object. +[clinic start generated code]*/ + +static PyObject * +_ctypes_CType_Type___sizeof___impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=c68c235be84d03f3 input=d064433b6110d1ce]*/ +{ + Py_ssize_t size = Py_TYPE(self)->tp_basicsize; + size += Py_TYPE(self)->tp_itemsize * Py_SIZE(self); + + ctypes_state *st = get_module_state_by_class(cls); + StgInfo *info; + if (PyStgInfo_FromType(st, self, &info) < 0) { + return NULL; + } + if (info) { + if (info->format) { + size += strlen(info->format) + 1; + } + if (info->ffi_type_pointer.elements) { + size += (info->length + 1) * sizeof(ffi_type *); + } + size += info->ndim * sizeof(Py_ssize_t); + } + + return PyLong_FromSsize_t(size); +} + +static PyObject * +CType_Type_repeat(PyObject *self, Py_ssize_t length); + + +static PyMethodDef ctype_methods[] = { + _CTYPES_CTYPE_TYPE___SIZEOF___METHODDEF + {0}, +}; + +static PyType_Slot ctype_type_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, + {Py_tp_traverse, CType_Type_traverse}, + {Py_tp_clear, CType_Type_clear}, + {Py_tp_dealloc, CType_Type_dealloc}, + {Py_tp_methods, ctype_methods}, + // Sequence protocol. + {Py_sq_repeat, CType_Type_repeat}, + {0, NULL}, +}; + +PyType_Spec pyctype_type_spec = { + .name = "_ctypes.CType_Type", + .basicsize = -(Py_ssize_t)sizeof(StgInfo), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | + Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_BASETYPE ), + .slots = ctype_type_slots, +}; /* PyCStructType_Type - a meta type/class. Creating a new class using this one as - __metaclass__ will call the constructor StructUnionType_new. It replaces the - tp_dict member with a new instance of StgDict, and initializes the C - accessible fields somehow. + __metaclass__ will call the constructor StructUnionType_new. + It initializes the C accessible fields somehow. */ static PyCArgObject * -StructUnionType_paramfunc(CDataObject *self) +StructUnionType_paramfunc(ctypes_state *st, CDataObject *self) { PyCArgObject *parg; PyObject *obj; - StgDictObject *stgdict; void *ptr; if ((size_t)self->b_size > sizeof(void*)) { @@ -480,12 +604,11 @@ StructUnionType_paramfunc(CDataObject *self) if (ptr == NULL) { return NULL; } - memcpy(ptr, self->b_ptr, self->b_size); + locked_memcpy_from(ptr, self, self->b_size); /* Create a Python object which calls PyMem_Free(ptr) in its deallocator. The object will be destroyed at _ctypes_callproc() cleanup. */ - ctypes_state *st = GLOBAL_STATE(); PyTypeObject *tp = st->StructParam_Type; obj = tp->tp_alloc(tp, 0); if (obj == NULL) { @@ -501,117 +624,131 @@ StructUnionType_paramfunc(CDataObject *self) obj = Py_NewRef(self); } - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) { Py_DECREF(obj); return NULL; } + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + Py_DECREF(obj); + return NULL; + } + assert(stginfo); /* Cannot be NULL for structure/union instances */ + parg->tag = 'V'; - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for structure/union instances */ - parg->pffi_type = &stgdict->ffi_type_pointer; + parg->pffi_type = &stginfo->ffi_type_pointer; parg->value.p = ptr; parg->size = self->b_size; parg->obj = obj; return parg; } -static PyObject * -StructUnionType_new(PyTypeObject *type, PyObject *args, PyObject *kwds, int isStruct) +static int +StructUnionType_init(PyObject *self, PyObject *args, PyObject *kwds, int isStruct) { - PyTypeObject *result; PyObject *fields; - StgDictObject *dict; - /* create the new instance (which is a class, - since we are a metatype!) */ - result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds); - if (!result) - return NULL; + PyObject *attrdict = PyType_GetDict((PyTypeObject *)self); + if (!attrdict) { + return -1; + } /* keep this for bw compatibility */ - int r = PyDict_Contains(result->tp_dict, &_Py_ID(_abstract_)); + int r = PyDict_Contains(attrdict, &_Py_ID(_abstract_)); if (r > 0) { - return (PyObject *)result; + Py_DECREF(attrdict); + return 0; } if (r < 0) { - Py_DECREF(result); - return NULL; + Py_DECREF(attrdict); + return -1; } - ctypes_state *st = GLOBAL_STATE(); - dict = (StgDictObject *)_PyObject_CallNoArgs((PyObject *)st->PyCStgDict_Type); - if (!dict) { - Py_DECREF(result); - return NULL; - } - if (!isStruct) { - dict->flags |= TYPEFLAG_HASUNION; - } - /* replace the class dict by our updated stgdict, which holds info - about storage requirements of the instances */ - if (-1 == PyDict_Update((PyObject *)dict, result->tp_dict)) { - Py_DECREF(result); - Py_DECREF((PyObject *)dict); - return NULL; + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); + StgInfo *info = PyStgInfo_Init(st, (PyTypeObject *)self); + if (!info) { + Py_DECREF(attrdict); + return -1; } - Py_SETREF(result->tp_dict, (PyObject *)dict); - dict->format = _ctypes_alloc_format_string(NULL, "B"); - if (dict->format == NULL) { - Py_DECREF(result); - return NULL; + + info->format = _ctypes_alloc_format_string(NULL, "B"); + if (info->format == NULL) { + Py_DECREF(attrdict); + return -1; } - dict->paramfunc = StructUnionType_paramfunc; + info->paramfunc = StructUnionType_paramfunc; - if (PyDict_GetItemRef((PyObject *)dict, &_Py_ID(_fields_), &fields) < 0) { - Py_DECREF(result); - return NULL; + if (PyDict_GetItemRef(attrdict, &_Py_ID(_fields_), &fields) < 0) { + Py_DECREF(attrdict); + return -1; } + Py_CLEAR(attrdict); if (fields) { - if (PyObject_SetAttr((PyObject *)result, &_Py_ID(_fields_), fields) < 0) { - Py_DECREF(result); + if (PyObject_SetAttr(self, &_Py_ID(_fields_), fields) < 0) { Py_DECREF(fields); - return NULL; + return -1; } Py_DECREF(fields); - return (PyObject *)result; + return 0; } else { - StgDictObject *basedict = PyType_stgdict((PyObject *)result->tp_base); - - if (basedict == NULL) { - return (PyObject *)result; + StgInfo *baseinfo; + if (PyStgInfo_FromType(st, (PyObject *)((PyTypeObject *)self)->tp_base, + &baseinfo) < 0) { + return -1; } - /* copy base dict */ - if (-1 == PyCStgDict_clone(dict, basedict)) { - Py_DECREF(result); - return NULL; + if (baseinfo == NULL) { + return 0; + } + + /* copy base info */ + if (PyCStgInfo_clone(info, baseinfo) < 0) { + return -1; } - dict->flags &= ~DICTFLAG_FINAL; /* clear the 'final' flag in the subclass dict */ - basedict->flags |= DICTFLAG_FINAL; /* set the 'final' flag in the baseclass dict */ - return (PyObject *)result; + info->flags &= ~DICTFLAG_FINAL; /* clear the 'final' flag in the subclass info */ + baseinfo->flags |= DICTFLAG_FINAL; /* set the 'final' flag in the baseclass info */ } + return 0; } -static PyObject * -PyCStructType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +PyCStructType_init(PyObject *self, PyObject *args, PyObject *kwds) { - return StructUnionType_new(type, args, kwds, 1); + return StructUnionType_init(self, args, kwds, 1); } -static PyObject * -UnionType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +UnionType_init(PyObject *self, PyObject *args, PyObject *kwds) { - return StructUnionType_new(type, args, kwds, 0); + return StructUnionType_init(self, args, kwds, 0); } -PyDoc_STRVAR(from_address_doc, -"C.from_address(integer) -> C instance\naccess a C instance at the specified address"); +/*[clinic input] +class _ctypes.CDataType "PyObject *" "clinic_state()->CType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=466a505a93d73156]*/ + + +/*[clinic input] +_ctypes.CDataType.from_address as CDataType_from_address + + type: self + cls: defining_class + value: object + / + +C.from_address(integer) -> C instance + +Access a C instance at the specified address. +[clinic start generated code]*/ static PyObject * -CDataType_from_address(PyObject *type, PyObject *value) +CDataType_from_address_impl(PyObject *type, PyTypeObject *cls, + PyObject *value) +/*[clinic end generated code: output=5be4a7c0d9aa6c74 input=827a22cefe380c01]*/ { void *buf; if (!PyLong_Check(value)) { @@ -622,32 +759,45 @@ CDataType_from_address(PyObject *type, PyObject *value) buf = (void *)PyLong_AsVoidPtr(value); if (PyErr_Occurred()) return NULL; - return PyCData_AtAddress(type, buf); + ctypes_state *st = get_module_state_by_class(cls); + return PyCData_AtAddress(st, type, buf); } -PyDoc_STRVAR(from_buffer_doc, -"C.from_buffer(object, offset=0) -> C instance\ncreate a C instance from a writeable buffer"); - static int KeepRef(CDataObject *target, Py_ssize_t index, PyObject *keep); +/*[clinic input] +_ctypes.CDataType.from_buffer as CDataType_from_buffer + + type: self + cls: defining_class + obj: object + offset: Py_ssize_t = 0 + / + +C.from_buffer(object, offset=0) -> C instance + +Create a C instance from a writeable buffer. +[clinic start generated code]*/ + static PyObject * -CDataType_from_buffer(PyObject *type, PyObject *args) +CDataType_from_buffer_impl(PyObject *type, PyTypeObject *cls, PyObject *obj, + Py_ssize_t offset) +/*[clinic end generated code: output=57604e99635abd31 input=0f36cedd105ca28d]*/ { - PyObject *obj; PyObject *mv; PyObject *result; Py_buffer *buffer; - Py_ssize_t offset = 0; - StgDictObject *dict = PyType_stgdict(type); - if (!dict) { - PyErr_SetString(PyExc_TypeError, "abstract class"); + ctypes_state *st = get_module_state_by_class(cls); + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { return NULL; } - - if (!PyArg_ParseTuple(args, "O|n:from_buffer", &obj, &offset)) + if (!info) { + PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; + } mv = PyMemoryView_FromObject(obj); if (mv == NULL) @@ -676,11 +826,11 @@ CDataType_from_buffer(PyObject *type, PyObject *args) return NULL; } - if (dict->size > buffer->len - offset) { + if (info->size > buffer->len - offset) { PyErr_Format(PyExc_ValueError, "Buffer size too small " "(%zd instead of at least %zd bytes)", - buffer->len, dict->size + offset); + buffer->len, info->size + offset); Py_DECREF(mv); return NULL; } @@ -691,7 +841,7 @@ CDataType_from_buffer(PyObject *type, PyObject *args) return NULL; } - result = PyCData_AtAddress(type, (char *)buffer->buf + offset); + result = PyCData_AtAddress(st, type, (char *)buffer->buf + offset); if (result == NULL) { Py_DECREF(mv); return NULL; @@ -705,72 +855,93 @@ CDataType_from_buffer(PyObject *type, PyObject *args) return result; } -PyDoc_STRVAR(from_buffer_copy_doc, -"C.from_buffer_copy(object, offset=0) -> C instance\ncreate a C instance from a readable buffer"); +static inline PyObject * +generic_pycdata_new(ctypes_state *st, + PyTypeObject *type, PyObject *args, PyObject *kwds); static PyObject * GenericPyCData_new(PyTypeObject *type, PyObject *args, PyObject *kwds); +/*[clinic input] +_ctypes.CDataType.from_buffer_copy as CDataType_from_buffer_copy + + type: self + cls: defining_class + buffer: Py_buffer + offset: Py_ssize_t = 0 + / + +C.from_buffer_copy(object, offset=0) -> C instance + +Create a C instance from a readable buffer. +[clinic start generated code]*/ + static PyObject * -CDataType_from_buffer_copy(PyObject *type, PyObject *args) +CDataType_from_buffer_copy_impl(PyObject *type, PyTypeObject *cls, + Py_buffer *buffer, Py_ssize_t offset) +/*[clinic end generated code: output=c8fc62b03e5cc6fa input=2a81e11b765a6253]*/ { - Py_buffer buffer; - Py_ssize_t offset = 0; PyObject *result; - StgDictObject *dict = PyType_stgdict(type); - if (!dict) { - PyErr_SetString(PyExc_TypeError, "abstract class"); + + ctypes_state *st = get_module_state_by_class(cls); + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { return NULL; } - - if (!PyArg_ParseTuple(args, "y*|n:from_buffer_copy", &buffer, &offset)) + if (!info) { + PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; + } if (offset < 0) { PyErr_SetString(PyExc_ValueError, "offset cannot be negative"); - PyBuffer_Release(&buffer); return NULL; } - if (dict->size > buffer.len - offset) { + if (info->size > buffer->len - offset) { PyErr_Format(PyExc_ValueError, "Buffer size too small (%zd instead of at least %zd bytes)", - buffer.len, dict->size + offset); - PyBuffer_Release(&buffer); + buffer->len, info->size + offset); return NULL; } if (PySys_Audit("ctypes.cdata/buffer", "nnn", - (Py_ssize_t)buffer.buf, buffer.len, offset) < 0) { - PyBuffer_Release(&buffer); + (Py_ssize_t)buffer->buf, buffer->len, offset) < 0) { return NULL; } - result = GenericPyCData_new((PyTypeObject *)type, NULL, NULL); + result = generic_pycdata_new(st, (PyTypeObject *)type, NULL, NULL); if (result != NULL) { - memcpy(((CDataObject *)result)->b_ptr, - (char *)buffer.buf + offset, dict->size); + locked_memcpy_to((CDataObject *) result, (char *)buffer->buf + offset, info->size); } - PyBuffer_Release(&buffer); return result; } -PyDoc_STRVAR(in_dll_doc, -"C.in_dll(dll, name) -> C instance\naccess a C instance in a dll"); +/*[clinic input] +_ctypes.CDataType.in_dll as CDataType_in_dll + + type: self + cls: defining_class + dll: object + name: str + / + +C.in_dll(dll, name) -> C instance + +Access a C instance in a dll. +[clinic start generated code]*/ static PyObject * -CDataType_in_dll(PyObject *type, PyObject *args) +CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, + const char *name) +/*[clinic end generated code: output=d0e5c43b66bfa21f input=f85bf281477042b4]*/ { - PyObject *dll; - char *name; PyObject *obj; void *handle; void *address; - if (!PyArg_ParseTuple(args, "Os:in_dll", &dll, &name)) - return NULL; - if (PySys_Audit("ctypes.dlsym", "O", args) < 0) { + if (PySys_Audit("ctypes.dlsym", "Os", dll, name) < 0) { return NULL; } @@ -791,38 +962,57 @@ CDataType_in_dll(PyObject *type, PyObject *args) return NULL; } +#undef USE_DLERROR #ifdef MS_WIN32 Py_BEGIN_ALLOW_THREADS address = (void *)GetProcAddress(handle, name); Py_END_ALLOW_THREADS - if (!address) { - PyErr_Format(PyExc_ValueError, - "symbol '%s' not found", - name); - return NULL; - } #else + #ifdef __CYGWIN__ + // dlerror() isn't very helpful on cygwin + #else + #define USE_DLERROR + /* dlerror() always returns the latest error. + * + * Clear the previous value before calling dlsym(), + * to ensure we can tell if our call resulted in an error. + */ + (void)dlerror(); + #endif address = (void *)dlsym(handle, name); - if (!address) { -#ifdef __CYGWIN__ -/* dlerror() isn't very helpful on cygwin */ - PyErr_Format(PyExc_ValueError, - "symbol '%s' not found", - name); -#else - PyErr_SetString(PyExc_ValueError, dlerror()); #endif + + if (address) { + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + return PyCData_AtAddress(st, type, address); + } + + #ifdef USE_DLERROR + const char *dlerr = dlerror(); + if (dlerr) { + _PyErr_SetLocaleString(PyExc_ValueError, dlerr); return NULL; } -#endif - return PyCData_AtAddress(type, address); + #endif +#undef USE_DLERROR + PyErr_Format(PyExc_ValueError, "symbol '%s' not found", name); + return NULL; } -PyDoc_STRVAR(from_param_doc, -"Convert a Python object into a function call parameter."); +/*[clinic input] +_ctypes.CDataType.from_param as CDataType_from_param + + type: self + cls: defining_class + value: object + / + +Convert a Python object into a function call parameter. +[clinic start generated code]*/ static PyObject * -CDataType_from_param(PyObject *type, PyObject *value) +CDataType_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) +/*[clinic end generated code: output=8da9e34263309f9e input=275a52c4899ddff0]*/ { PyObject *as_parameter; int res = PyObject_IsInstance(value, type); @@ -831,18 +1021,19 @@ CDataType_from_param(PyObject *type, PyObject *value) if (res) { return Py_NewRef(value); } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(cls); if (PyCArg_CheckExact(st, value)) { PyCArgObject *p = (PyCArgObject *)value; PyObject *ob = p->obj; const char *ob_name; - StgDictObject *dict; - dict = PyType_stgdict(type); - + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } /* If we got a PyCArgObject, we must check if the object packed in it - is an instance of the type's dict->proto */ - if(dict && ob) { - res = PyObject_IsInstance(ob, dict->proto); + is an instance of the type's info->proto */ + if(info && ob) { + res = PyObject_IsInstance(ob, info->proto); if (res == -1) return NULL; if (res) { @@ -860,8 +1051,13 @@ CDataType_from_param(PyObject *type, PyObject *value) return NULL; } if (as_parameter) { - value = CDataType_from_param(type, as_parameter); + if (_Py_EnterRecursiveCall(" while processing _as_parameter_")) { + Py_DECREF(as_parameter); + return NULL; + } + value = CDataType_from_param_impl(type, cls, as_parameter); Py_DECREF(as_parameter); + _Py_LeaveRecursiveCall(); return value; } PyErr_Format(PyExc_TypeError, @@ -872,87 +1068,63 @@ CDataType_from_param(PyObject *type, PyObject *value) } static PyMethodDef CDataType_methods[] = { - { "from_param", CDataType_from_param, METH_O, from_param_doc }, - { "from_address", CDataType_from_address, METH_O, from_address_doc }, - { "from_buffer", CDataType_from_buffer, METH_VARARGS, from_buffer_doc, }, - { "from_buffer_copy", CDataType_from_buffer_copy, METH_VARARGS, from_buffer_copy_doc, }, - { "in_dll", CDataType_in_dll, METH_VARARGS, in_dll_doc }, + CDATATYPE_FROM_PARAM_METHODDEF + CDATATYPE_FROM_ADDRESS_METHODDEF + CDATATYPE_FROM_BUFFER_METHODDEF + CDATATYPE_FROM_BUFFER_COPY_METHODDEF + CDATATYPE_IN_DLL_METHODDEF { NULL, NULL }, }; static PyObject * -CDataType_repeat(PyObject *self, Py_ssize_t length) +CType_Type_repeat(PyObject *self, Py_ssize_t length) { if (length < 0) return PyErr_Format(PyExc_ValueError, "Array length must be >= 0, not %zd", length); - return PyCArrayType_from_ctype(self, length); + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); + return PyCArrayType_from_ctype(st, self, length); } static int -CDataType_clear(PyTypeObject *self) +_structunion_setattro(PyObject *self, PyObject *key, PyObject *value, int is_struct) { - StgDictObject *dict = PyType_stgdict((PyObject *)self); - if (dict) - Py_CLEAR(dict->proto); - return PyType_Type.tp_clear((PyObject *)self); -} - -static int -CDataType_traverse(PyTypeObject *self, visitproc visit, void *arg) -{ - StgDictObject *dict = PyType_stgdict((PyObject *)self); - if (dict) { - Py_VISIT(dict->proto); + /* XXX Should we disallow deleting _fields_? */ + if (PyUnicode_Check(key) + && _PyUnicode_EqualToASCIIString(key, "_fields_")) + { + if (PyCStructUnionType_update_stginfo(self, value, is_struct) < 0) { + return -1; + } } - Py_VISIT(Py_TYPE(self)); - return PyType_Type.tp_traverse((PyObject *)self, visit, arg); + + return PyType_Type.tp_setattro(self, key, value); } static int PyCStructType_setattro(PyObject *self, PyObject *key, PyObject *value) { - /* XXX Should we disallow deleting _fields_? */ - if (-1 == PyType_Type.tp_setattro(self, key, value)) - return -1; - - if (value && PyUnicode_Check(key) && - _PyUnicode_EqualToASCIIString(key, "_fields_")) - return PyCStructUnionType_update_stgdict(self, value, 1); - return 0; + return _structunion_setattro(self, key, value, 1); } - static int UnionType_setattro(PyObject *self, PyObject *key, PyObject *value) { - /* XXX Should we disallow deleting _fields_? */ - if (-1 == PyObject_GenericSetAttr(self, key, value)) - return -1; - - if (PyUnicode_Check(key) && - _PyUnicode_EqualToASCIIString(key, "_fields_")) - return PyCStructUnionType_update_stgdict(self, value, 0); - return 0; + return _structunion_setattro(self, key, value, 0); } static PyType_Slot pycstruct_type_slots[] = { {Py_tp_setattro, PyCStructType_setattro}, {Py_tp_doc, PyDoc_STR("metatype for the CData Objects")}, - {Py_tp_traverse, CDataType_traverse}, - {Py_tp_clear, CDataType_clear}, {Py_tp_methods, CDataType_methods}, - {Py_tp_new, PyCStructType_new}, - - // Sequence protocol. - {Py_sq_repeat, CDataType_repeat}, + {Py_tp_init, PyCStructType_init}, {0, NULL}, }; -PyType_Spec pycstruct_type_spec = { +static PyType_Spec pycstruct_type_spec = { .name = "_ctypes.PyCStructType", - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), .slots = pycstruct_type_slots, }; @@ -960,19 +1132,14 @@ PyType_Spec pycstruct_type_spec = { static PyType_Slot union_type_slots[] = { {Py_tp_setattro, UnionType_setattro}, {Py_tp_doc, PyDoc_STR("metatype for the Union Objects")}, - {Py_tp_traverse, CDataType_traverse}, - {Py_tp_clear, CDataType_clear}, {Py_tp_methods, CDataType_methods}, - {Py_tp_new, UnionType_new}, - - // Sequence protocol. - {Py_sq_repeat, CDataType_repeat}, + {Py_tp_init, UnionType_init}, {0, NULL}, }; static PyType_Spec union_type_spec = { .name = "_ctypes.UnionType", - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), .slots = union_type_slots, }; @@ -993,161 +1160,189 @@ size property/method, and the sequence protocol. */ +/*[clinic input] +class _ctypes.PyCPointerType "PyObject *" "clinic_state()->PyCPointerType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c45e96c1f7645ab7]*/ + + static int -PyCPointerType_SetProto(StgDictObject *stgdict, PyObject *proto) +PyCPointerType_SetProto(ctypes_state *st, StgInfo *stginfo, PyObject *proto) { if (!proto || !PyType_Check(proto)) { PyErr_SetString(PyExc_TypeError, "_type_ must be a type"); return -1; } - if (!PyType_stgdict(proto)) { + StgInfo *info; + if (PyStgInfo_FromType(st, proto, &info) < 0) { + return -1; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "_type_ must have storage info"); return -1; } Py_INCREF(proto); - Py_XSETREF(stgdict->proto, proto); + Py_XSETREF(stginfo->proto, proto); return 0; } static PyCArgObject * -PyCPointerType_paramfunc(CDataObject *self) +PyCPointerType_paramfunc(ctypes_state *st, CDataObject *self) { PyCArgObject *parg; - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->tag = 'P'; parg->pffi_type = &ffi_type_pointer; parg->obj = Py_NewRef(self); - parg->value.p = *(void **)self->b_ptr; + parg->value.p = locked_deref(self); return parg; } -static PyObject * -PyCPointerType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +PyCPointerType_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyTypeObject *result; - StgDictObject *stgdict; PyObject *proto; PyObject *typedict; - typedict = PyTuple_GetItem(args, 2); if (!typedict) { - return NULL; + return -1; } + /* - stgdict items size, align, length contain info about pointers itself, - stgdict->proto has info about the pointed to type! + stginfo items size, align, length contain info about pointers itself, + stginfo->proto has info about the pointed to type! */ - ctypes_state *st = GLOBAL_STATE(); - stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)st->PyCStgDict_Type); - if (!stgdict) { - return NULL; + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); + StgInfo *stginfo = PyStgInfo_Init(st, (PyTypeObject *)self); + if (!stginfo) { + return -1; } - stgdict->size = sizeof(void *); - stgdict->align = _ctypes_get_fielddesc("P")->pffi_type->alignment; - stgdict->length = 1; - stgdict->ffi_type_pointer = ffi_type_pointer; - stgdict->paramfunc = PyCPointerType_paramfunc; - stgdict->flags |= TYPEFLAG_ISPOINTER; + stginfo->size = sizeof(void *); + stginfo->align = _ctypes_get_fielddesc("P")->pffi_type->alignment; + stginfo->length = 1; + stginfo->ffi_type_pointer = ffi_type_pointer; + stginfo->paramfunc = PyCPointerType_paramfunc; + stginfo->flags |= TYPEFLAG_ISPOINTER; if (PyDict_GetItemRef(typedict, &_Py_ID(_type_), &proto) < 0) { - Py_DECREF((PyObject *)stgdict); - return NULL; + return -1; } if (proto) { - StgDictObject *itemdict; const char *current_format; - if (-1 == PyCPointerType_SetProto(stgdict, proto)) { + if (PyCPointerType_SetProto(st, stginfo, proto) < 0) { Py_DECREF(proto); - Py_DECREF((PyObject *)stgdict); - return NULL; + return -1; } - itemdict = PyType_stgdict(proto); - /* PyCPointerType_SetProto has verified proto has a stgdict. */ - assert(itemdict); - /* If itemdict->format is NULL, then this is a pointer to an + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, proto, &iteminfo) < 0) { + Py_DECREF(proto); + return -1; + } + /* PyCPointerType_SetProto has verified proto has a stginfo. */ + assert(iteminfo); + /* If iteminfo->format is NULL, then this is a pointer to an incomplete type. We create a generic format string 'pointer to bytes' in this case. XXX Better would be to fix the format string later... */ - current_format = itemdict->format ? itemdict->format : "B"; - if (itemdict->shape != NULL) { + current_format = iteminfo->format ? iteminfo->format : "B"; + if (iteminfo->shape != NULL) { /* pointer to an array: the shape needs to be prefixed */ - stgdict->format = _ctypes_alloc_format_string_with_shape( - itemdict->ndim, itemdict->shape, "&", current_format); + stginfo->format = _ctypes_alloc_format_string_with_shape( + iteminfo->ndim, iteminfo->shape, "&", current_format); } else { - stgdict->format = _ctypes_alloc_format_string("&", current_format); + stginfo->format = _ctypes_alloc_format_string("&", current_format); } Py_DECREF(proto); - if (stgdict->format == NULL) { - Py_DECREF((PyObject *)stgdict); - return NULL; + if (stginfo->format == NULL) { + return -1; } } - /* create the new instance (which is a class, - since we are a metatype!) */ - result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds); - if (result == NULL) { - Py_DECREF((PyObject *)stgdict); - return NULL; - } - - /* replace the class dict by our updated spam dict */ - if (-1 == PyDict_Update((PyObject *)stgdict, result->tp_dict)) { - Py_DECREF(result); - Py_DECREF((PyObject *)stgdict); - return NULL; - } - Py_SETREF(result->tp_dict, (PyObject *)stgdict); - - return (PyObject *)result; + return 0; } +/*[clinic input] +_ctypes.PyCPointerType.set_type as PyCPointerType_set_type + + self: self(type="PyTypeObject *") + cls: defining_class + type: object + / +[clinic start generated code]*/ static PyObject * -PyCPointerType_set_type(PyTypeObject *self, PyObject *type) +PyCPointerType_set_type_impl(PyTypeObject *self, PyTypeObject *cls, + PyObject *type) +/*[clinic end generated code: output=51459d8f429a70ac input=67e1e8df921f123e]*/ { - StgDictObject *dict; - - - dict = PyType_stgdict((PyObject *)self); - if (!dict) { + PyObject *attrdict = PyType_GetDict(self); + if (!attrdict) { + return NULL; + } + ctypes_state *st = get_module_state_by_class(cls); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)self, &info) < 0) { + Py_DECREF(attrdict); + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); + Py_DECREF(attrdict); return NULL; } - if (-1 == PyCPointerType_SetProto(dict, type)) + if (PyCPointerType_SetProto(st, info, type) < 0) { + Py_DECREF(attrdict); return NULL; + } - if (-1 == PyDict_SetItem((PyObject *)dict, &_Py_ID(_type_), type)) + if (-1 == PyDict_SetItem(attrdict, &_Py_ID(_type_), type)) { + Py_DECREF(attrdict); return NULL; + } + Py_DECREF(attrdict); Py_RETURN_NONE; } -static PyObject *_byref(PyObject *); +static PyObject *_byref(ctypes_state *, PyObject *); + +/*[clinic input] +_ctypes.PyCPointerType.from_param as PyCPointerType_from_param + + type: self + cls: defining_class + value: object + / + +Convert a Python object into a function call parameter. +[clinic start generated code]*/ static PyObject * -PyCPointerType_from_param(PyObject *type, PyObject *value) +PyCPointerType_from_param_impl(PyObject *type, PyTypeObject *cls, + PyObject *value) +/*[clinic end generated code: output=a4b32d929aabaf64 input=6c231276e3997884]*/ { - StgDictObject *typedict; - if (value == Py_None) { /* ConvParam will convert to a NULL pointer later */ return Py_NewRef(value); } - typedict = PyType_stgdict(type); - if (!typedict) { + ctypes_state *st = get_module_state_by_class(cls); + StgInfo *typeinfo; + if (PyStgInfo_FromType(st, type, &typeinfo) < 0) { + return NULL; + } + if (!typeinfo) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; @@ -1156,24 +1351,27 @@ PyCPointerType_from_param(PyObject *type, PyObject *value) /* If we expect POINTER(), but receive a instance, accept it by calling byref(). */ - switch (PyObject_IsInstance(value, typedict->proto)) { + assert(typeinfo->proto); + switch (PyObject_IsInstance(value, typeinfo->proto)) { case 1: Py_INCREF(value); /* _byref steals a refcount */ - return _byref(value); + return _byref(st, value); case -1: return NULL; default: break; } - ctypes_state *st = GLOBAL_STATE(); if (PointerObject_Check(st, value) || ArrayObject_Check(st, value)) { /* Array instances are also pointers when the item types are the same. */ - StgDictObject *v = PyObject_stgdict(value); + StgInfo *v; + if (PyStgInfo_FromObject(st, value, &v) < 0) { + return NULL; + } assert(v); /* Cannot be NULL for pointer or array objects */ - int ret = PyObject_IsSubclass(v->proto, typedict->proto); + int ret = PyObject_IsSubclass(v->proto, typeinfo->proto); if (ret < 0) { return NULL; } @@ -1181,34 +1379,29 @@ PyCPointerType_from_param(PyObject *type, PyObject *value) return Py_NewRef(value); } } - return CDataType_from_param(type, value); + return CDataType_from_param_impl(type, cls, value); } static PyMethodDef PyCPointerType_methods[] = { - { "from_address", CDataType_from_address, METH_O, from_address_doc }, - { "from_buffer", CDataType_from_buffer, METH_VARARGS, from_buffer_doc, }, - { "from_buffer_copy", CDataType_from_buffer_copy, METH_VARARGS, from_buffer_copy_doc, }, - { "in_dll", CDataType_in_dll, METH_VARARGS, in_dll_doc}, - { "from_param", (PyCFunction)PyCPointerType_from_param, METH_O, from_param_doc}, - { "set_type", (PyCFunction)PyCPointerType_set_type, METH_O }, + CDATATYPE_FROM_ADDRESS_METHODDEF + CDATATYPE_FROM_BUFFER_METHODDEF + CDATATYPE_FROM_BUFFER_COPY_METHODDEF + CDATATYPE_IN_DLL_METHODDEF + PYCPOINTERTYPE_FROM_PARAM_METHODDEF + PYCPOINTERTYPE_SET_TYPE_METHODDEF { NULL, NULL }, }; static PyType_Slot pycpointer_type_slots[] = { {Py_tp_doc, PyDoc_STR("metatype for the Pointer Objects")}, - {Py_tp_traverse, CDataType_traverse}, - {Py_tp_clear, CDataType_clear}, {Py_tp_methods, PyCPointerType_methods}, - {Py_tp_new, PyCPointerType_new}, - - // Sequence protocol. - {Py_sq_repeat, CDataType_repeat}, + {Py_tp_init, PyCPointerType_init}, {0, NULL}, }; static PyType_Spec pycpointer_type_spec = { .name = "_ctypes.PyCPointerType", - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), .slots = pycpointer_type_slots, }; @@ -1219,16 +1412,17 @@ static PyType_Spec pycpointer_type_spec = { PyCArrayType_Type */ /* - PyCArrayType_new ensures that the new Array subclass created has a _length_ + PyCArrayType_init ensures that the new Array subclass created has a _length_ attribute, and a _type_ attribute. */ static int -CharArray_set_raw(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored)) +CharArray_set_raw(PyObject *op, PyObject *value, void *Py_UNUSED(ignored)) { char *ptr; Py_ssize_t size; Py_buffer view; + CDataObject *self = _CDataObject_CAST(op); if (value == NULL) { PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); @@ -1244,7 +1438,7 @@ CharArray_set_raw(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored)) goto fail; } - memcpy(self->b_ptr, ptr, size); + locked_memcpy_to(self, ptr, size); PyBuffer_Release(&view); return 0; @@ -1254,27 +1448,38 @@ CharArray_set_raw(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored)) } static PyObject * -CharArray_get_raw(CDataObject *self, void *Py_UNUSED(ignored)) +CharArray_get_raw(PyObject *op, void *Py_UNUSED(ignored)) { - return PyBytes_FromStringAndSize(self->b_ptr, self->b_size); + PyObject *res; + CDataObject *self = _CDataObject_CAST(op); + LOCK_PTR(self); + res = PyBytes_FromStringAndSize(self->b_ptr, self->b_size); + UNLOCK_PTR(self); + return res; } static PyObject * -CharArray_get_value(CDataObject *self, void *Py_UNUSED(ignored)) +CharArray_get_value(PyObject *op, void *Py_UNUSED(ignored)) { Py_ssize_t i; + PyObject *res; + CDataObject *self = _CDataObject_CAST(op); + LOCK_PTR(self); char *ptr = self->b_ptr; for (i = 0; i < self->b_size; ++i) if (*ptr++ == '\0') break; - return PyBytes_FromStringAndSize(self->b_ptr, i); + res = PyBytes_FromStringAndSize(self->b_ptr, i); + UNLOCK_PTR(self); + return res; } static int -CharArray_set_value(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored)) +CharArray_set_value(PyObject *op, PyObject *value, void *Py_UNUSED(ignored)) { const char *ptr; Py_ssize_t size; + CDataObject *self = _CDataObject_CAST(op); if (value == NULL) { PyErr_SetString(PyExc_TypeError, @@ -1298,36 +1503,43 @@ CharArray_set_value(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored) } ptr = PyBytes_AS_STRING(value); + LOCK_PTR(self); memcpy(self->b_ptr, ptr, size); if (size < self->b_size) self->b_ptr[size] = '\0'; + UNLOCK_PTR(self); Py_DECREF(value); return 0; } static PyGetSetDef CharArray_getsets[] = { - { "raw", (getter)CharArray_get_raw, (setter)CharArray_set_raw, - "value", NULL }, - { "value", (getter)CharArray_get_value, (setter)CharArray_set_value, - "string value"}, + { "raw", CharArray_get_raw, CharArray_set_raw, "value", NULL }, + { "value", CharArray_get_value, CharArray_set_value, "string value" }, { NULL, NULL } }; static PyObject * -WCharArray_get_value(CDataObject *self, void *Py_UNUSED(ignored)) +WCharArray_get_value(PyObject *op, void *Py_UNUSED(ignored)) { Py_ssize_t i; + PyObject *res; + CDataObject *self = _CDataObject_CAST(op); wchar_t *ptr = (wchar_t *)self->b_ptr; + LOCK_PTR(self); for (i = 0; i < self->b_size/(Py_ssize_t)sizeof(wchar_t); ++i) if (*ptr++ == (wchar_t)0) break; - return PyUnicode_FromWideChar((wchar_t *)self->b_ptr, i); + res = PyUnicode_FromWideChar((wchar_t *)self->b_ptr, i); + UNLOCK_PTR(self); + return res; } static int -WCharArray_set_value(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored)) +WCharArray_set_value(PyObject *op, PyObject *value, void *Py_UNUSED(ignored)) { + CDataObject *self = _CDataObject_CAST(op); + if (value == NULL) { PyErr_SetString(PyExc_TypeError, "can't delete attribute"); @@ -1352,15 +1564,15 @@ WCharArray_set_value(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored PyErr_SetString(PyExc_ValueError, "string too long"); return -1; } - if (PyUnicode_AsWideChar(value, (wchar_t *)self->b_ptr, size) < 0) { - return -1; - } - return 0; + Py_ssize_t rc; + LOCK_PTR(self); + rc = PyUnicode_AsWideChar(value, (wchar_t *)self->b_ptr, size); + UNLOCK_PTR(self); + return rc < 0 ? -1 : 0; } static PyGetSetDef WCharArray_getsets[] = { - { "value", (getter)WCharArray_get_value, (setter)WCharArray_set_value, - "string value"}, + { "value", WCharArray_get_value, WCharArray_set_value, "string value" }, { NULL, NULL } }; @@ -1391,9 +1603,9 @@ add_getset(PyTypeObject *type, PyGetSetDef *gsp) } static PyCArgObject * -PyCArrayType_paramfunc(CDataObject *self) +PyCArrayType_paramfunc(ctypes_state *st, CDataObject *self) { - PyCArgObject *p = PyCArgObject_new(); + PyCArgObject *p = PyCArgObject_new(st); if (p == NULL) return NULL; p->tag = 'P'; @@ -1403,28 +1615,18 @@ PyCArrayType_paramfunc(CDataObject *self) return p; } -static PyObject * -PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +PyCArrayType_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyTypeObject *result; - StgDictObject *stgdict; - StgDictObject *itemdict; PyObject *length_attr, *type_attr; Py_ssize_t length; Py_ssize_t itemsize, itemalign; - /* create the new instance (which is a class, - since we are a metatype!) */ - result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds); - if (result == NULL) - return NULL; - /* Initialize these variables to NULL so that we can simplify error handling by using Py_XDECREF. */ - stgdict = NULL; type_attr = NULL; - if (PyObject_GetOptionalAttr((PyObject *)result, &_Py_ID(_length_), &length_attr) < 0) { + if (PyObject_GetOptionalAttr(self, &_Py_ID(_length_), &length_attr) < 0) { goto error; } if (!length_attr) { @@ -1440,7 +1642,7 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } - if (_PyLong_Sign(length_attr) == -1) { + if (_PyLong_IsNegative((PyLongObject *)length_attr)) { Py_DECREF(length_attr); PyErr_SetString(PyExc_ValueError, "The '_length_' attribute must not be negative"); @@ -1457,7 +1659,7 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } - if (PyObject_GetOptionalAttr((PyObject *)result, &_Py_ID(_type_), &type_attr) < 0) { + if (PyObject_GetOptionalAttr(self, &_Py_ID(_type_), &type_attr) < 0) { goto error; } if (!type_attr) { @@ -1466,99 +1668,89 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } - ctypes_state *st = GLOBAL_STATE(); - stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)st->PyCStgDict_Type); - if (!stgdict) { + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); + StgInfo *stginfo = PyStgInfo_Init(st, (PyTypeObject*)self); + if (!stginfo) { goto error; } - itemdict = PyType_stgdict(type_attr); - if (!itemdict) { + + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, type_attr, &iteminfo) < 0) { + goto error; + } + if (!iteminfo) { PyErr_SetString(PyExc_TypeError, "_type_ must have storage info"); goto error; } - assert(itemdict->format); - stgdict->format = _ctypes_alloc_format_string(NULL, itemdict->format); - if (stgdict->format == NULL) + assert(iteminfo->format); + stginfo->format = _ctypes_alloc_format_string(NULL, iteminfo->format); + if (stginfo->format == NULL) goto error; - stgdict->ndim = itemdict->ndim + 1; - stgdict->shape = PyMem_Malloc(sizeof(Py_ssize_t) * stgdict->ndim); - if (stgdict->shape == NULL) { + stginfo->ndim = iteminfo->ndim + 1; + stginfo->shape = PyMem_Malloc(sizeof(Py_ssize_t) * stginfo->ndim); + if (stginfo->shape == NULL) { PyErr_NoMemory(); goto error; } - stgdict->shape[0] = length; - if (stgdict->ndim > 1) { - memmove(&stgdict->shape[1], itemdict->shape, - sizeof(Py_ssize_t) * (stgdict->ndim - 1)); + stginfo->shape[0] = length; + if (stginfo->ndim > 1) { + memmove(&stginfo->shape[1], iteminfo->shape, + sizeof(Py_ssize_t) * (stginfo->ndim - 1)); } - itemsize = itemdict->size; + itemsize = iteminfo->size; if (itemsize != 0 && length > PY_SSIZE_T_MAX / itemsize) { PyErr_SetString(PyExc_OverflowError, "array too large"); goto error; } - itemalign = itemdict->align; + itemalign = iteminfo->align; - if (itemdict->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) - stgdict->flags |= TYPEFLAG_HASPOINTER; + if (iteminfo->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) + stginfo->flags |= TYPEFLAG_HASPOINTER; - stgdict->size = itemsize * length; - stgdict->align = itemalign; - stgdict->length = length; - stgdict->proto = type_attr; + stginfo->size = itemsize * length; + stginfo->align = itemalign; + stginfo->length = length; + stginfo->proto = type_attr; type_attr = NULL; - stgdict->paramfunc = &PyCArrayType_paramfunc; + stginfo->paramfunc = &PyCArrayType_paramfunc; /* Arrays are passed as pointers to function calls. */ - stgdict->ffi_type_pointer = ffi_type_pointer; - - /* replace the class dict by our updated spam dict */ - if (-1 == PyDict_Update((PyObject *)stgdict, result->tp_dict)) - goto error; - Py_SETREF(result->tp_dict, (PyObject *)stgdict); /* steal the reference */ - stgdict = NULL; + stginfo->ffi_type_pointer = ffi_type_pointer; /* Special case for character arrays. A permanent annoyance: char arrays are also strings! */ - if (itemdict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { - if (-1 == add_getset(result, CharArray_getsets)) + if (iteminfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { + if (-1 == add_getset((PyTypeObject*)self, CharArray_getsets)) goto error; } - else if (itemdict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { - if (-1 == add_getset(result, WCharArray_getsets)) + else if (iteminfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) { + if (-1 == add_getset((PyTypeObject*)self, WCharArray_getsets)) goto error; } - return (PyObject *)result; + return 0; error: - Py_XDECREF((PyObject*)stgdict); Py_XDECREF(type_attr); - Py_DECREF(result); - return NULL; + return -1; } static PyType_Slot pycarray_type_slots[] = { {Py_tp_doc, PyDoc_STR("metatype for the Array Objects")}, - {Py_tp_traverse, CDataType_traverse}, {Py_tp_methods, CDataType_methods}, - {Py_tp_new, PyCArrayType_new}, - {Py_tp_clear, CDataType_clear}, - - // Sequence protocol. - {Py_sq_repeat, CDataType_repeat}, + {Py_tp_init, PyCArrayType_init}, {0, NULL}, }; static PyType_Spec pycarray_type_spec = { .name = "_ctypes.PyCArrayType", - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), .slots = pycarray_type_slots, }; @@ -1569,26 +1761,56 @@ static PyType_Spec pycarray_type_spec = { */ /* -PyCSimpleType_new ensures that the new Simple_Type subclass created has a valid +PyCSimpleType_init ensures that the new Simple_Type subclass created has a valid _type_ attribute. */ -static const char SIMPLE_TYPE_CHARS[] = "cbBhHiIlLdfuzZqQPXOv?g"; +/*[clinic input] +class _ctypes.PyCSimpleType "PyObject *" "clinic_state()->PyCSimpleType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=d5a45772668e7f49]*/ + +/*[clinic input] +class _ctypes.c_wchar_p "PyObject *" "clinic_state_sub()->PyCSimpleType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=468de7283d622d47]*/ + +/*[clinic input] +class _ctypes.c_char_p "PyObject *" "clinic_state_sub()->PyCSimpleType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e750865616e7dcea]*/ + +/*[clinic input] +class _ctypes.c_void_p "PyObject *" "clinic_state_sub()->PyCSimpleType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=dd4d9646c56f43a9]*/ + + +/*[clinic input] +_ctypes.c_wchar_p.from_param as c_wchar_p_from_param + + type: self + cls: defining_class + value: object + / +[clinic start generated code]*/ static PyObject * -c_wchar_p_from_param(PyObject *type, PyObject *value) +c_wchar_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) +/*[clinic end generated code: output=e453949a2f725a4c input=d322c7237a319607]*/ { PyObject *as_parameter; int res; if (value == Py_None) { Py_RETURN_NONE; } + ctypes_state *st = get_module_state_by_class(cls->tp_base); if (PyUnicode_Check(value)) { PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("Z"); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1606,22 +1828,31 @@ c_wchar_p_from_param(PyObject *type, PyObject *value) if (res) { return Py_NewRef(value); } - ctypes_state *st = GLOBAL_STATE(); if (ArrayObject_Check(st, value) || PointerObject_Check(st, value)) { /* c_wchar array instance or pointer(c_wchar(...)) */ - StgDictObject *dt = PyObject_stgdict(value); - StgDictObject *dict; - assert(dt); /* Cannot be NULL for pointer or array objects */ - dict = dt && dt->proto ? PyType_stgdict(dt->proto) : NULL; - if (dict && (dict->setfunc == _ctypes_get_fielddesc("u")->setfunc)) { + StgInfo *it; + if (PyStgInfo_FromObject(st, value, &it) < 0) { + return NULL; + } + assert(it); /* Cannot be NULL for pointer or array objects */ + StgInfo *info = NULL; + if (it && it->proto) { + if (PyStgInfo_FromType(st, it->proto, &info) < 0) { + return NULL; + } + } + if (info && (info->setfunc == _ctypes_get_fielddesc("u")->setfunc)) { return Py_NewRef(value); } } if (PyCArg_CheckExact(st, value)) { /* byref(c_char(...)) */ PyCArgObject *a = (PyCArgObject *)value; - StgDictObject *dict = PyObject_stgdict(a->obj); - if (dict && (dict->setfunc == _ctypes_get_fielddesc("u")->setfunc)) { + StgInfo *info; + if (PyStgInfo_FromObject(st, a->obj, &info) < 0) { + return NULL; + } + if (info && (info->setfunc == _ctypes_get_fielddesc("u")->setfunc)) { return Py_NewRef(value); } } @@ -1630,8 +1861,13 @@ c_wchar_p_from_param(PyObject *type, PyObject *value) return NULL; } if (as_parameter) { - value = c_wchar_p_from_param(type, as_parameter); + if (_Py_EnterRecursiveCall(" while processing _as_parameter_")) { + Py_DECREF(as_parameter); + return NULL; + } + value = c_wchar_p_from_param_impl(type, cls, as_parameter); Py_DECREF(as_parameter); + _Py_LeaveRecursiveCall(); return value; } PyErr_Format(PyExc_TypeError, @@ -1640,19 +1876,30 @@ c_wchar_p_from_param(PyObject *type, PyObject *value) return NULL; } +/*[clinic input] +_ctypes.c_char_p.from_param as c_char_p_from_param + + type: self + cls: defining_class + value: object + / +[clinic start generated code]*/ + static PyObject * -c_char_p_from_param(PyObject *type, PyObject *value) +c_char_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) +/*[clinic end generated code: output=219652ab7c174aa1 input=6cf0d1b6bb4ede11]*/ { PyObject *as_parameter; int res; if (value == Py_None) { Py_RETURN_NONE; } + ctypes_state *st = get_module_state_by_class(cls->tp_base); if (PyBytes_Check(value)) { PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("z"); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1670,22 +1917,31 @@ c_char_p_from_param(PyObject *type, PyObject *value) if (res) { return Py_NewRef(value); } - ctypes_state *st = GLOBAL_STATE(); if (ArrayObject_Check(st, value) || PointerObject_Check(st, value)) { /* c_char array instance or pointer(c_char(...)) */ - StgDictObject *dt = PyObject_stgdict(value); - StgDictObject *dict; - assert(dt); /* Cannot be NULL for pointer or array objects */ - dict = dt && dt->proto ? PyType_stgdict(dt->proto) : NULL; - if (dict && (dict->setfunc == _ctypes_get_fielddesc("c")->setfunc)) { + StgInfo *it; + if (PyStgInfo_FromObject(st, value, &it) < 0) { + return NULL; + } + assert(it); /* Cannot be NULL for pointer or array objects */ + StgInfo *info = NULL; + if (it && it->proto) { + if (PyStgInfo_FromType(st, it->proto, &info) < 0) { + return NULL; + } + } + if (info && (info->setfunc == _ctypes_get_fielddesc("c")->setfunc)) { return Py_NewRef(value); } } if (PyCArg_CheckExact(st, value)) { /* byref(c_char(...)) */ PyCArgObject *a = (PyCArgObject *)value; - StgDictObject *dict = PyObject_stgdict(a->obj); - if (dict && (dict->setfunc == _ctypes_get_fielddesc("c")->setfunc)) { + StgInfo *info; + if (PyStgInfo_FromObject(st, a->obj, &info) < 0) { + return NULL; + } + if (info && (info->setfunc == _ctypes_get_fielddesc("c")->setfunc)) { return Py_NewRef(value); } } @@ -1694,8 +1950,13 @@ c_char_p_from_param(PyObject *type, PyObject *value) return NULL; } if (as_parameter) { - value = c_char_p_from_param(type, as_parameter); + if (_Py_EnterRecursiveCall(" while processing _as_parameter_")) { + Py_DECREF(as_parameter); + return NULL; + } + value = c_char_p_from_param_impl(type, cls, as_parameter); Py_DECREF(as_parameter); + _Py_LeaveRecursiveCall(); return value; } PyErr_Format(PyExc_TypeError, @@ -1704,10 +1965,19 @@ c_char_p_from_param(PyObject *type, PyObject *value) return NULL; } +/*[clinic input] +_ctypes.c_void_p.from_param as c_void_p_from_param + + type: self + cls: defining_class + value: object + / +[clinic start generated code]*/ + static PyObject * -c_void_p_from_param(PyObject *type, PyObject *value) +c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) +/*[clinic end generated code: output=984d0075b6038cc7 input=0e8b343fc19c77d4]*/ { - StgDictObject *stgd; PyObject *as_parameter; int res; @@ -1715,18 +1985,20 @@ c_void_p_from_param(PyObject *type, PyObject *value) if (value == Py_None) { Py_RETURN_NONE; } + ctypes_state *st = get_module_state_by_class(cls->tp_base); + /* Should probably allow buffer interface as well */ /* int, long */ if (PyLong_Check(value)) { PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("P"); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; parg->tag = 'P'; - parg->obj = fd->setfunc(&parg->value, value, 0); + parg->obj = fd->setfunc(&parg->value, value, sizeof(void*)); if (parg->obj == NULL) { Py_DECREF(parg); return NULL; @@ -1739,7 +2011,7 @@ c_void_p_from_param(PyObject *type, PyObject *value) PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("z"); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1756,7 +2028,7 @@ c_void_p_from_param(PyObject *type, PyObject *value) PyCArgObject *parg; struct fielddesc *fd = _ctypes_get_fielddesc("Z"); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; @@ -1776,7 +2048,6 @@ c_void_p_from_param(PyObject *type, PyObject *value) /* c_void_p instances */ return Py_NewRef(value); } - ctypes_state *st = GLOBAL_STATE(); /* ctypes array or pointer instance */ if (ArrayObject_Check(st, value) || PointerObject_Check(st, value)) { /* Any array or pointer is accepted */ @@ -1795,36 +2066,40 @@ c_void_p_from_param(PyObject *type, PyObject *value) PyCArgObject *parg; PyCFuncPtrObject *func; func = (PyCFuncPtrObject *)value; - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; parg->tag = 'P'; Py_INCREF(value); + // Function pointers don't change their contents, no need to lock parg->value.p = *(void **)func->b_ptr; parg->obj = value; return (PyObject *)parg; } /* c_char_p, c_wchar_p */ - stgd = PyObject_stgdict(value); - if (stgd + StgInfo *stgi; + if (PyStgInfo_FromObject(st, value, &stgi) < 0) { + return NULL; + } + if (stgi && CDataObject_Check(st, value) - && stgd->proto - && PyUnicode_Check(stgd->proto)) + && stgi->proto + && PyUnicode_Check(stgi->proto)) { PyCArgObject *parg; - switch (PyUnicode_AsUTF8(stgd->proto)[0]) { + switch (PyUnicode_AsUTF8(stgi->proto)[0]) { case 'z': /* c_char_p */ case 'Z': /* c_wchar_p */ - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; parg->tag = 'Z'; parg->obj = Py_NewRef(value); /* Remember: b_ptr points to where the pointer is stored! */ - parg->value.p = *(void **)(((CDataObject *)value)->b_ptr); + parg->value.p = locked_deref((CDataObject *)value); return (PyObject *)parg; } } @@ -1833,8 +2108,13 @@ c_void_p_from_param(PyObject *type, PyObject *value) return NULL; } if (as_parameter) { - value = c_void_p_from_param(type, as_parameter); + if (_Py_EnterRecursiveCall(" while processing _as_parameter_")) { + Py_DECREF(as_parameter); + return NULL; + } + value = c_void_p_from_param_impl(type, cls, as_parameter); Py_DECREF(as_parameter); + _Py_LeaveRecursiveCall(); return value; } PyErr_Format(PyExc_TypeError, @@ -1843,37 +2123,37 @@ c_void_p_from_param(PyObject *type, PyObject *value) return NULL; } -static PyMethodDef c_void_p_method = { "from_param", c_void_p_from_param, METH_O }; -static PyMethodDef c_char_p_method = { "from_param", c_char_p_from_param, METH_O }; -static PyMethodDef c_wchar_p_method = { "from_param", c_wchar_p_from_param, METH_O }; +static PyMethodDef c_void_p_methods[] = {C_VOID_P_FROM_PARAM_METHODDEF {0}}; +static PyMethodDef c_char_p_methods[] = {C_CHAR_P_FROM_PARAM_METHODDEF {0}}; +static PyMethodDef c_wchar_p_methods[] = {C_WCHAR_P_FROM_PARAM_METHODDEF {0}}; -static PyObject *CreateSwappedType(PyTypeObject *type, PyObject *args, PyObject *kwds, +static PyObject *CreateSwappedType(ctypes_state *st, PyTypeObject *type, + PyObject *args, PyObject *kwds, PyObject *proto, struct fielddesc *fmt) { PyTypeObject *result; - StgDictObject *stgdict; PyObject *name = PyTuple_GET_ITEM(args, 0); PyObject *newname; PyObject *swapped_args; - static PyObject *suffix; Py_ssize_t i; swapped_args = PyTuple_New(PyTuple_GET_SIZE(args)); if (!swapped_args) return NULL; - if (suffix == NULL) + if (st->swapped_suffix == NULL) { #ifdef WORDS_BIGENDIAN - suffix = PyUnicode_InternFromString("_le"); + st->swapped_suffix = PyUnicode_InternFromString("_le"); #else - suffix = PyUnicode_InternFromString("_be"); + st->swapped_suffix = PyUnicode_InternFromString("_be"); #endif - if (suffix == NULL) { + } + if (st->swapped_suffix == NULL) { Py_DECREF(swapped_args); return NULL; } - newname = PyUnicode_Concat(name, suffix); + newname = PyUnicode_Concat(name, st->swapped_suffix); if (newname == NULL) { Py_DECREF(swapped_args); return NULL; @@ -1893,88 +2173,74 @@ static PyObject *CreateSwappedType(PyTypeObject *type, PyObject *args, PyObject if (result == NULL) return NULL; - ctypes_state *st = GLOBAL_STATE(); - stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)st->PyCStgDict_Type); - if (!stgdict) { + StgInfo *stginfo = PyStgInfo_Init(st, result); + if (!stginfo) { Py_DECREF(result); return NULL; } - stgdict->ffi_type_pointer = *fmt->pffi_type; - stgdict->align = fmt->pffi_type->alignment; - stgdict->length = 0; - stgdict->size = fmt->pffi_type->size; - stgdict->setfunc = fmt->setfunc_swapped; - stgdict->getfunc = fmt->getfunc_swapped; + stginfo->ffi_type_pointer = *fmt->pffi_type; + stginfo->align = fmt->pffi_type->alignment; + stginfo->length = 0; + stginfo->size = fmt->pffi_type->size; + stginfo->setfunc = fmt->setfunc_swapped; + stginfo->getfunc = fmt->getfunc_swapped; - stgdict->proto = Py_NewRef(proto); - - /* replace the class dict by our updated spam dict */ - if (-1 == PyDict_Update((PyObject *)stgdict, result->tp_dict)) { - Py_DECREF(result); - Py_DECREF((PyObject *)stgdict); - return NULL; - } - Py_SETREF(result->tp_dict, (PyObject *)stgdict); + stginfo->proto = Py_NewRef(proto); return (PyObject *)result; } static PyCArgObject * -PyCSimpleType_paramfunc(CDataObject *self) +PyCSimpleType_paramfunc(ctypes_state *st, CDataObject *self) { - StgDictObject *dict; const char *fmt; PyCArgObject *parg; struct fielddesc *fd; - dict = PyObject_stgdict((PyObject *)self); - assert(dict); /* Cannot be NULL for CDataObject instances */ - fmt = PyUnicode_AsUTF8(dict->proto); + StgInfo *info; + if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { + return NULL; + } + assert(info); /* Cannot be NULL for CDataObject instances */ + fmt = PyUnicode_AsUTF8(info->proto); assert(fmt); fd = _ctypes_get_fielddesc(fmt); assert(fd); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->tag = fmt[0]; parg->pffi_type = fd->pffi_type; parg->obj = Py_NewRef(self); - memcpy(&parg->value, self->b_ptr, self->b_size); + locked_memcpy_from(&parg->value, self, self->b_size); return parg; } -static PyObject * -PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +PyCSimpleType_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyTypeObject *result; - StgDictObject *stgdict; PyObject *proto; const char *proto_str; Py_ssize_t proto_len; PyMethodDef *ml; struct fielddesc *fmt; - /* create the new instance (which is a class, - since we are a metatype!) */ - result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds); - if (result == NULL) - return NULL; - - if (PyObject_GetOptionalAttr((PyObject *)result, &_Py_ID(_type_), &proto) < 0) { - return NULL; + if (PyType_Type.tp_init(self, args, kwds) < 0) { + return -1; + } + if (PyObject_GetOptionalAttr(self, &_Py_ID(_type_), &proto) < 0) { + return -1; } if (!proto) { PyErr_SetString(PyExc_AttributeError, "class must define a '_type_' attribute"); error: Py_XDECREF(proto); - Py_DECREF(result); - return NULL; + return -1; } if (PyUnicode_Check(proto)) { proto_str = PyUnicode_AsUTF8AndSize(proto, &proto_len); @@ -1991,85 +2257,82 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) "which must be a string of length 1"); goto error; } - if (!strchr(SIMPLE_TYPE_CHARS, *proto_str)) { + fmt = _ctypes_get_fielddesc(proto_str); + if (!fmt) { PyErr_Format(PyExc_AttributeError, "class must define a '_type_' attribute which must be\n" - "a single character string containing one of '%s'.", - SIMPLE_TYPE_CHARS); + "a single character string containing one of the\n" + "supported types: '%s'.", + _ctypes_get_simple_type_chars()); goto error; } - fmt = _ctypes_get_fielddesc(proto_str); - if (fmt == NULL) { - PyErr_Format(PyExc_ValueError, - "_type_ '%s' not supported", proto_str); + + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); + StgInfo *stginfo = PyStgInfo_Init(st, (PyTypeObject *)self); + if (!stginfo) { goto error; } - ctypes_state *st = GLOBAL_STATE(); - stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)st->PyCStgDict_Type); - if (!stgdict) { - goto error; + if (!fmt->pffi_type->elements) { + stginfo->ffi_type_pointer = *fmt->pffi_type; } - stgdict->ffi_type_pointer = *fmt->pffi_type; - stgdict->align = fmt->pffi_type->alignment; - stgdict->length = 0; - stgdict->size = fmt->pffi_type->size; - stgdict->setfunc = fmt->setfunc; - stgdict->getfunc = fmt->getfunc; + else { + const size_t els_size = sizeof(fmt->pffi_type->elements); + stginfo->ffi_type_pointer.size = fmt->pffi_type->size; + stginfo->ffi_type_pointer.alignment = fmt->pffi_type->alignment; + stginfo->ffi_type_pointer.type = fmt->pffi_type->type; + stginfo->ffi_type_pointer.elements = PyMem_Malloc(els_size); + memcpy(stginfo->ffi_type_pointer.elements, + fmt->pffi_type->elements, els_size); + } + stginfo->align = fmt->pffi_type->alignment; + stginfo->length = 0; + stginfo->size = fmt->pffi_type->size; + stginfo->setfunc = fmt->setfunc; + stginfo->getfunc = fmt->getfunc; #ifdef WORDS_BIGENDIAN - stgdict->format = _ctypes_alloc_format_string_for_type(proto_str[0], 1); + stginfo->format = _ctypes_alloc_format_string_for_type(proto_str[0], 1); #else - stgdict->format = _ctypes_alloc_format_string_for_type(proto_str[0], 0); + stginfo->format = _ctypes_alloc_format_string_for_type(proto_str[0], 0); #endif - if (stgdict->format == NULL) { - Py_DECREF(result); + if (stginfo->format == NULL) { Py_DECREF(proto); - Py_DECREF((PyObject *)stgdict); - return NULL; + return -1; } - stgdict->paramfunc = PyCSimpleType_paramfunc; + stginfo->paramfunc = PyCSimpleType_paramfunc; /* - if (result->tp_base != st->Simple_Type) { - stgdict->setfunc = NULL; - stgdict->getfunc = NULL; + if (self->tp_base != st->Simple_Type) { + stginfo->setfunc = NULL; + stginfo->getfunc = NULL; } */ /* This consumes the refcount on proto which we have */ - stgdict->proto = proto; - - /* replace the class dict by our updated spam dict */ - if (-1 == PyDict_Update((PyObject *)stgdict, result->tp_dict)) { - Py_DECREF(result); - Py_DECREF((PyObject *)stgdict); - return NULL; - } - Py_SETREF(result->tp_dict, (PyObject *)stgdict); + stginfo->proto = proto; /* Install from_param class methods in ctypes base classes. Overrides the PyCSimpleType_from_param generic method. */ - if (result->tp_base == st->Simple_Type) { + if (((PyTypeObject *)self)->tp_base == st->Simple_Type) { switch (*proto_str) { case 'z': /* c_char_p */ - ml = &c_char_p_method; - stgdict->flags |= TYPEFLAG_ISPOINTER; + ml = c_char_p_methods; + stginfo->flags |= TYPEFLAG_ISPOINTER; break; case 'Z': /* c_wchar_p */ - ml = &c_wchar_p_method; - stgdict->flags |= TYPEFLAG_ISPOINTER; + ml = c_wchar_p_methods; + stginfo->flags |= TYPEFLAG_ISPOINTER; break; case 'P': /* c_void_p */ - ml = &c_void_p_method; - stgdict->flags |= TYPEFLAG_ISPOINTER; + ml = c_void_p_methods; + stginfo->flags |= TYPEFLAG_ISPOINTER; break; case 's': case 'X': case 'O': ml = NULL; - stgdict->flags |= TYPEFLAG_ISPOINTER; + stginfo->flags |= TYPEFLAG_ISPOINTER; break; default: ml = NULL; @@ -2079,67 +2342,85 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (ml) { PyObject *meth; int x; - meth = PyDescr_NewClassMethod(result, ml); + meth = PyDescr_NewClassMethod((PyTypeObject*)self, ml); if (!meth) { - Py_DECREF(result); - return NULL; + return -1; + } + PyObject *name = PyUnicode_FromString(ml->ml_name); + if (name == NULL) { + Py_DECREF(meth); + return -1; } - x = PyDict_SetItemString(result->tp_dict, - ml->ml_name, - meth); + PyUnicode_InternInPlace(&name); + x = PyDict_SetItem(((PyTypeObject*)self)->tp_dict, name, meth); + Py_DECREF(name); Py_DECREF(meth); if (x == -1) { - Py_DECREF(result); - return NULL; + return -1; } } } + PyTypeObject *type = Py_TYPE(self); if (type == st->PyCSimpleType_Type && fmt->setfunc_swapped && fmt->getfunc_swapped) { - PyObject *swapped = CreateSwappedType(type, args, kwds, + PyObject *swapped = CreateSwappedType(st, type, args, kwds, proto, fmt); - StgDictObject *sw_dict; if (swapped == NULL) { - Py_DECREF(result); - return NULL; + return -1; } - sw_dict = PyType_stgdict(swapped); + StgInfo *sw_info; + if (PyStgInfo_FromType(st, swapped, &sw_info) < 0) { + return -1; + } + assert(sw_info); #ifdef WORDS_BIGENDIAN - PyObject_SetAttrString((PyObject *)result, "__ctype_le__", swapped); - PyObject_SetAttrString((PyObject *)result, "__ctype_be__", (PyObject *)result); - PyObject_SetAttrString(swapped, "__ctype_be__", (PyObject *)result); + PyObject_SetAttrString(self, "__ctype_le__", swapped); + PyObject_SetAttrString(self, "__ctype_be__", self); + PyObject_SetAttrString(swapped, "__ctype_be__", self); PyObject_SetAttrString(swapped, "__ctype_le__", swapped); /* We are creating the type for the OTHER endian */ - sw_dict->format = _ctypes_alloc_format_string("<", stgdict->format+1); + sw_info->format = _ctypes_alloc_format_string("<", stginfo->format+1); #else - PyObject_SetAttrString((PyObject *)result, "__ctype_be__", swapped); - PyObject_SetAttrString((PyObject *)result, "__ctype_le__", (PyObject *)result); - PyObject_SetAttrString(swapped, "__ctype_le__", (PyObject *)result); + PyObject_SetAttrString(self, "__ctype_be__", swapped); + PyObject_SetAttrString(self, "__ctype_le__", self); + PyObject_SetAttrString(swapped, "__ctype_le__", self); PyObject_SetAttrString(swapped, "__ctype_be__", swapped); /* We are creating the type for the OTHER endian */ - sw_dict->format = _ctypes_alloc_format_string(">", stgdict->format+1); + sw_info->format = _ctypes_alloc_format_string(">", stginfo->format+1); #endif Py_DECREF(swapped); if (PyErr_Occurred()) { - Py_DECREF(result); - return NULL; + return -1; } }; - return (PyObject *)result; + return 0; } /* * This is a *class method*. * Convert a parameter into something that ConvParam can handle. */ + +/*[clinic input] +_ctypes.PyCSimpleType.from_param as PyCSimpleType_from_param + + type: self + cls: defining_class + value: object + / + +Convert a Python object into a function call parameter. +[clinic start generated code]*/ + static PyObject * -PyCSimpleType_from_param(PyObject *type, PyObject *value) +PyCSimpleType_from_param_impl(PyObject *type, PyTypeObject *cls, + PyObject *value) +/*[clinic end generated code: output=8a8453d9663e3a2e input=61cc48ce3a87a570]*/ { - StgDictObject *dict; const char *fmt; PyCArgObject *parg; struct fielddesc *fd; @@ -2155,27 +2436,31 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) return Py_NewRef(value); } - dict = PyType_stgdict(type); - if (!dict) { + ctypes_state *st = get_module_state_by_class(cls); + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; } /* I think we can rely on this being a one-character string */ - fmt = PyUnicode_AsUTF8(dict->proto); + fmt = PyUnicode_AsUTF8(info->proto); assert(fmt); fd = _ctypes_get_fielddesc(fmt); assert(fd); - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->tag = fmt[0]; parg->pffi_type = fd->pffi_type; - parg->obj = fd->setfunc(&parg->value, value, 0); + parg->obj = fd->setfunc(&parg->value, value, info->size); if (parg->obj) return (PyObject *)parg; PyObject *exc = PyErr_GetRaisedException(); @@ -2186,15 +2471,15 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) return NULL; } if (as_parameter) { - if (_Py_EnterRecursiveCall("while processing _as_parameter_")) { + if (_Py_EnterRecursiveCall(" while processing _as_parameter_")) { Py_DECREF(as_parameter); Py_XDECREF(exc); return NULL; } - value = PyCSimpleType_from_param(type, as_parameter); - _Py_LeaveRecursiveCall(); + value = PyCSimpleType_from_param_impl(type, cls, as_parameter); Py_DECREF(as_parameter); Py_XDECREF(exc); + _Py_LeaveRecursiveCall(); return value; } if (exc) { @@ -2207,29 +2492,24 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) } static PyMethodDef PyCSimpleType_methods[] = { - { "from_param", PyCSimpleType_from_param, METH_O, from_param_doc }, - { "from_address", CDataType_from_address, METH_O, from_address_doc }, - { "from_buffer", CDataType_from_buffer, METH_VARARGS, from_buffer_doc, }, - { "from_buffer_copy", CDataType_from_buffer_copy, METH_VARARGS, from_buffer_copy_doc, }, - { "in_dll", CDataType_in_dll, METH_VARARGS, in_dll_doc}, + PYCSIMPLETYPE_FROM_PARAM_METHODDEF + CDATATYPE_FROM_ADDRESS_METHODDEF + CDATATYPE_FROM_BUFFER_METHODDEF + CDATATYPE_FROM_BUFFER_COPY_METHODDEF + CDATATYPE_IN_DLL_METHODDEF { NULL, NULL }, }; static PyType_Slot pycsimple_type_slots[] = { {Py_tp_doc, PyDoc_STR("metatype for the PyCSimpleType Objects")}, {Py_tp_methods, PyCSimpleType_methods}, - {Py_tp_new, PyCSimpleType_new}, - {Py_tp_traverse, CDataType_traverse}, - {Py_tp_clear, CDataType_clear}, - - // Sequence protocol. - {Py_sq_repeat, CDataType_repeat}, + {Py_tp_init, PyCSimpleType_init}, {0, NULL}, }; -PyType_Spec pycsimple_type_spec = { +static PyType_Spec pycsimple_type_spec = { .name = "_ctypes.PyCSimpleType", - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), .slots = pycsimple_type_slots, }; @@ -2240,7 +2520,7 @@ PyType_Spec pycsimple_type_spec = { */ static PyObject * -converters_from_argtypes(PyObject *ob) +converters_from_argtypes(ctypes_state *st, PyObject *ob) { PyObject *converters; Py_ssize_t i; @@ -2255,7 +2535,7 @@ converters_from_argtypes(PyObject *ob) Py_ssize_t nArgs = PyTuple_GET_SIZE(ob); if (nArgs > CTYPES_MAX_ARGCOUNT) { Py_DECREF(ob); - PyErr_Format(PyExc_ArgError, + PyErr_Format(st->PyExc_ArgError, "_argtypes_ has too many arguments (%zi), maximum is %i", nArgs, CTYPES_MAX_ARGCOUNT); return NULL; @@ -2292,10 +2572,17 @@ converters_from_argtypes(PyObject *ob) * not bitfields, the bitfields check is also being disabled as a * precaution. - StgDictObject *stgdict = PyType_stgdict(tp); + StgInfo *stginfo; + if (PyStgInfo_FromType(st, tp, &stginfo) < 0) { + return -1; + } - if (stgdict != NULL) { - if (stgdict->flags & TYPEFLAG_HASUNION) { + // TYPEFLAG_HASUNION and TYPEFLAG_HASBITFIELD used to be set + // if there were any unions/bitfields; + // if the check is re-enabled we either need to loop here or + // restore the flag + if (stginfo != NULL) { + if (stginfo->flags & TYPEFLAG_HASUNION) { Py_DECREF(converters); Py_DECREF(ob); if (!PyErr_Occurred()) { @@ -2306,7 +2593,7 @@ converters_from_argtypes(PyObject *ob) } return NULL; } - if (stgdict->flags & TYPEFLAG_HASBITFIELD) { + if (stginfo->flags & TYPEFLAG_HASBITFIELD) { Py_DECREF(converters); Py_DECREF(ob); if (!PyErr_Occurred()) { @@ -2338,19 +2625,19 @@ converters_from_argtypes(PyObject *ob) } static int -make_funcptrtype_dict(StgDictObject *stgdict) +make_funcptrtype_dict(ctypes_state *st, PyObject *attrdict, StgInfo *stginfo) { PyObject *ob; PyObject *converters = NULL; - stgdict->align = _ctypes_get_fielddesc("P")->pffi_type->alignment; - stgdict->length = 1; - stgdict->size = sizeof(void *); - stgdict->setfunc = NULL; - stgdict->getfunc = NULL; - stgdict->ffi_type_pointer = ffi_type_pointer; + stginfo->align = _ctypes_get_fielddesc("P")->pffi_type->alignment; + stginfo->length = 1; + stginfo->size = sizeof(void *); + stginfo->setfunc = NULL; + stginfo->getfunc = NULL; + stginfo->ffi_type_pointer = ffi_type_pointer; - if (PyDict_GetItemRef((PyObject *)stgdict, &_Py_ID(_flags_), &ob) < 0) { + if (PyDict_GetItemRef(attrdict, &_Py_ID(_flags_), &ob) < 0) { return -1; } if (!ob || !PyLong_Check(ob)) { @@ -2359,42 +2646,46 @@ make_funcptrtype_dict(StgDictObject *stgdict) Py_XDECREF(ob); return -1; } - stgdict->flags = PyLong_AsUnsignedLongMask(ob) | TYPEFLAG_ISPOINTER; + stginfo->flags = PyLong_AsUnsignedLongMask(ob) | TYPEFLAG_ISPOINTER; Py_DECREF(ob); /* _argtypes_ is optional... */ - if (PyDict_GetItemRef((PyObject *)stgdict, &_Py_ID(_argtypes_), &ob) < 0) { + if (PyDict_GetItemRef(attrdict, &_Py_ID(_argtypes_), &ob) < 0) { return -1; } if (ob) { - converters = converters_from_argtypes(ob); + converters = converters_from_argtypes(st, ob); if (!converters) { Py_DECREF(ob); return -1; } - stgdict->argtypes = ob; - stgdict->converters = converters; + stginfo->argtypes = ob; + stginfo->converters = converters; } - if (PyDict_GetItemRef((PyObject *)stgdict, &_Py_ID(_restype_), &ob) < 0) { + if (PyDict_GetItemRef(attrdict, &_Py_ID(_restype_), &ob) < 0) { return -1; } if (ob) { - if (ob != Py_None && !PyType_stgdict(ob) && !PyCallable_Check(ob)) { + StgInfo *info; + if (PyStgInfo_FromType(st, ob, &info) < 0) { + return -1; + } + if (ob != Py_None && !info && !PyCallable_Check(ob)) { PyErr_SetString(PyExc_TypeError, "_restype_ must be a type, a callable, or None"); Py_DECREF(ob); return -1; } - stgdict->restype = ob; + stginfo->restype = ob; if (PyObject_GetOptionalAttr(ob, &_Py_ID(_check_retval_), - &stgdict->checker) < 0) + &stginfo->checker) < 0) { return -1; } } /* XXX later, maybe. - if (PyDict_GetItemRef((PyObject *)stgdict, &_Py _ID(_errcheck_), &ob) < 0) { + if (PyDict_GetItemRef(attrdict, &_Py _ID(_errcheck_), &ob) < 0) { return -1; } if (ob) { @@ -2404,93 +2695,77 @@ make_funcptrtype_dict(StgDictObject *stgdict) Py_DECREF(ob); return -1; } - stgdict->errcheck = ob; + stginfo->errcheck = ob; } */ return 0; } static PyCArgObject * -PyCFuncPtrType_paramfunc(CDataObject *self) +PyCFuncPtrType_paramfunc(ctypes_state *st, CDataObject *self) { PyCArgObject *parg; - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; parg->tag = 'P'; parg->pffi_type = &ffi_type_pointer; parg->obj = Py_NewRef(self); - parg->value.p = *(void **)self->b_ptr; + parg->value.p = locked_deref(self); return parg; } -static PyObject * -PyCFuncPtrType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +PyCFuncPtrType_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyTypeObject *result; - StgDictObject *stgdict; + PyObject *attrdict = PyType_GetDict((PyTypeObject *)self); + if (!attrdict) { + return -1; + } - ctypes_state *st = GLOBAL_STATE(); - stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)st->PyCStgDict_Type); - if (!stgdict) { - return NULL; + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); + StgInfo *stginfo = PyStgInfo_Init(st, (PyTypeObject *)self); + if (!stginfo) { + Py_DECREF(attrdict); + return -1; } - stgdict->paramfunc = PyCFuncPtrType_paramfunc; + + stginfo->paramfunc = PyCFuncPtrType_paramfunc; + /* We do NOT expose the function signature in the format string. It is impossible, generally, because the only requirement for the argtypes items is that they have a .from_param method - we do not know the types of the arguments (although, in practice, most argtypes would be a ctypes type). */ - stgdict->format = _ctypes_alloc_format_string(NULL, "X{}"); - if (stgdict->format == NULL) { - Py_DECREF((PyObject *)stgdict); - return NULL; - } - stgdict->flags |= TYPEFLAG_ISPOINTER; - - /* create the new instance (which is a class, - since we are a metatype!) */ - result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds); - if (result == NULL) { - Py_DECREF((PyObject *)stgdict); - return NULL; - } - - /* replace the class dict by our updated storage dict */ - if (-1 == PyDict_Update((PyObject *)stgdict, result->tp_dict)) { - Py_DECREF(result); - Py_DECREF((PyObject *)stgdict); - return NULL; + stginfo->format = _ctypes_alloc_format_string(NULL, "X{}"); + if (stginfo->format == NULL) { + Py_DECREF(attrdict); + return -1; } - Py_SETREF(result->tp_dict, (PyObject *)stgdict); + stginfo->flags |= TYPEFLAG_ISPOINTER; - if (-1 == make_funcptrtype_dict(stgdict)) { - Py_DECREF(result); - return NULL; + if (make_funcptrtype_dict(st, attrdict, stginfo) < 0) { + Py_DECREF(attrdict); + return -1; } - return (PyObject *)result; + Py_DECREF(attrdict); + return 0; } static PyType_Slot pycfuncptr_type_slots[] = { {Py_tp_doc, PyDoc_STR("metatype for C function pointers")}, - {Py_tp_traverse, CDataType_traverse}, - {Py_tp_clear, CDataType_clear}, {Py_tp_methods, CDataType_methods}, - {Py_tp_new, PyCFuncPtrType_new}, - - // Sequence protocol. - {Py_sq_repeat, CDataType_repeat}, + {Py_tp_init, PyCFuncPtrType_init}, {0, NULL}, }; static PyType_Spec pycfuncptr_type_spec = { .name = "_ctypes.PyCFuncPtrType", - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), .slots = pycfuncptr_type_slots, }; @@ -2604,17 +2879,28 @@ KeepRef(CDataObject *target, Py_ssize_t index, PyObject *keep) /* PyCData_Type */ + +/*[clinic input] +class _ctypes.PyCData "PyObject *" "clinic_state()->PyCData_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=ac13df38dee3c22c]*/ + + static int -PyCData_traverse(CDataObject *self, visitproc visit, void *arg) +PyCData_traverse(PyObject *op, visitproc visit, void *arg) { + CDataObject *self = _CDataObject_CAST(op); Py_VISIT(self->b_objects); Py_VISIT((PyObject *)self->b_base); + PyTypeObject *type = Py_TYPE(self); + Py_VISIT(type); return 0; } static int -PyCData_clear(CDataObject *self) +PyCData_clear(PyObject *op) { + CDataObject *self = _CDataObject_CAST(op); Py_CLEAR(self->b_objects); if ((self->b_needsfree) && _CDataObject_HasExternalBuffer(self)) @@ -2627,8 +2913,11 @@ PyCData_clear(CDataObject *self) static void PyCData_dealloc(PyObject *self) { - PyCData_clear((CDataObject *)self); - Py_TYPE(self)->tp_free(self); + PyTypeObject *type = Py_TYPE(self); + PyObject_GC_UnTrack(self); + (void)PyCData_clear(self); + type->tp_free(self); + Py_DECREF(type); } static PyMemberDef PyCData_members[] = { @@ -2646,19 +2935,20 @@ static PyMemberDef PyCData_members[] = { /* Find the innermost type of an array type, returning a borrowed reference */ static PyObject * -PyCData_item_type(PyObject *type) +PyCData_item_type(ctypes_state *st, PyObject *type) { - ctypes_state *st = GLOBAL_STATE(); if (PyCArrayTypeObject_Check(st, type)) { - StgDictObject *stg_dict; PyObject *elem_type; /* asserts used here as these are all guaranteed by construction */ - stg_dict = PyType_stgdict(type); - assert(stg_dict); - elem_type = stg_dict->proto; + StgInfo *stg_info; + if (PyStgInfo_FromType(st, type, &stg_info) < 0) { + return NULL; + } + assert(stg_info); + elem_type = stg_info->proto; assert(elem_type); - return PyCData_item_type(elem_type); + return PyCData_item_type(st, elem_type); } else { return type; @@ -2668,33 +2958,43 @@ PyCData_item_type(PyObject *type) static int PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) { - CDataObject *self = (CDataObject *)myself; - StgDictObject *dict = PyObject_stgdict(myself); - PyObject *item_type = PyCData_item_type((PyObject*)Py_TYPE(myself)); - StgDictObject *item_dict = PyType_stgdict(item_type); + CDataObject *self = _CDataObject_CAST(myself); + + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(myself))); + StgInfo *info; + if (PyStgInfo_FromObject(st, myself, &info) < 0) { + return -1; + } + assert(info); + + PyObject *item_type = PyCData_item_type(st, (PyObject*)Py_TYPE(myself)); + if (item_type == NULL) { + return 0; + } if (view == NULL) return 0; + StgInfo *item_info; + if (PyStgInfo_FromType(st, item_type, &item_info) < 0) { + return -1; + } + assert(item_info); + view->buf = self->b_ptr; view->obj = Py_NewRef(myself); view->len = self->b_size; view->readonly = 0; /* use default format character if not set */ - view->format = dict->format ? dict->format : "B"; - view->ndim = dict->ndim; - view->shape = dict->shape; - view->itemsize = item_dict->size; + view->format = info->format ? info->format : "B"; + view->ndim = info->ndim; + view->shape = info->shape; + view->itemsize = item_info->size; view->strides = NULL; view->suboffsets = NULL; view->internal = NULL; return 0; } -static PyBufferProcs PyCData_as_buffer = { - PyCData_NewGetBuffer, - NULL, -}; - /* * CData objects are mutable, so they cannot be hashable! */ @@ -2705,12 +3005,28 @@ PyCData_nohash(PyObject *self) return -1; } +/*[clinic input] +_ctypes.PyCData.__reduce__ as PyCData_reduce + + myself: self + cls: defining_class + / +[clinic start generated code]*/ + static PyObject * -PyCData_reduce(PyObject *myself, PyObject *args) +PyCData_reduce_impl(PyObject *myself, PyTypeObject *cls) +/*[clinic end generated code: output=1a025ccfdd8c935d input=34097a5226ea63c1]*/ { - CDataObject *self = (CDataObject *)myself; + CDataObject *self = _CDataObject_CAST(myself); + + ctypes_state *st = get_module_state_by_class(cls); + StgInfo *info; + if (PyStgInfo_FromObject(st, myself, &info) < 0) { + return NULL; + } + assert(info); - if (PyObject_stgdict(myself)->flags & (TYPEFLAG_ISPOINTER|TYPEFLAG_HASPOINTER)) { + if (info->flags & (TYPEFLAG_ISPOINTER|TYPEFLAG_HASPOINTER)) { PyErr_SetString(PyExc_ValueError, "ctypes objects containing pointers cannot be pickled"); return NULL; @@ -2719,8 +3035,12 @@ PyCData_reduce(PyObject *myself, PyObject *args) if (dict == NULL) { return NULL; } - return Py_BuildValue("O(O(NN))", _unpickle, Py_TYPE(myself), dict, - PyBytes_FromStringAndSize(self->b_ptr, self->b_size)); + PyObject *bytes; + LOCK_PTR(self); + bytes = PyBytes_FromStringAndSize(self->b_ptr, self->b_size); + UNLOCK_PTR(self); + return Py_BuildValue("O(O(NN))", st->_unpickle, Py_TYPE(myself), dict, + bytes); } static PyObject * @@ -2730,7 +3050,7 @@ PyCData_setstate(PyObject *myself, PyObject *args) Py_ssize_t len; int res; PyObject *dict, *mydict; - CDataObject *self = (CDataObject *)myself; + CDataObject *self = _CDataObject_CAST(myself); if (!PyArg_ParseTuple(args, "O!s#", &PyDict_Type, &dict, &data, &len)) { @@ -2738,7 +3058,10 @@ PyCData_setstate(PyObject *myself, PyObject *args) } if (len > self->b_size) len = self->b_size; + // XXX Can we use locked_memcpy_to()? + LOCK_PTR(self); memmove(self->b_ptr, data, len); + UNLOCK_PTR(self); mydict = PyObject_GetAttrString(myself, "__dict__"); if (mydict == NULL) { return NULL; @@ -2768,56 +3091,41 @@ PyCData_from_outparam(PyObject *self, PyObject *args) static PyMethodDef PyCData_methods[] = { { "__ctypes_from_outparam__", PyCData_from_outparam, METH_NOARGS, }, - { "__reduce__", PyCData_reduce, METH_NOARGS, }, + PYCDATA_REDUCE_METHODDEF { "__setstate__", PyCData_setstate, METH_VARARGS, }, { NULL, NULL }, }; -PyTypeObject PyCData_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes._CData", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - PyCData_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - PyCData_nohash, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("XXX to be provided"), /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyCData_methods, /* tp_methods */ - PyCData_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycdata_slots[] = { + {Py_tp_dealloc, PyCData_dealloc}, + {Py_tp_hash, PyCData_nohash}, + {Py_tp_doc, PyDoc_STR("XXX to be provided")}, + {Py_tp_traverse, PyCData_traverse}, + {Py_tp_clear, PyCData_clear}, + {Py_tp_methods, PyCData_methods}, + {Py_tp_members, PyCData_members}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {0, NULL}, +}; + +static PyType_Spec pycdata_spec = { + .name = "_ctypes._CData", + .basicsize = sizeof(CDataObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION), + .slots = pycdata_slots, }; -static int PyCData_MallocBuffer(CDataObject *obj, StgDictObject *dict) +static int +PyCData_MallocBuffer(CDataObject *obj, StgInfo *info) { - if ((size_t)dict->size <= sizeof(obj->b_value)) { + /* We don't have to lock in this function, because it's only + * used in constructors and therefore does not have concurrent + * access. + */ + assert (Py_REFCNT(obj) == 1); + + if ((size_t)info->size <= sizeof(obj->b_value)) { /* No need to call malloc, can use the default buffer */ obj->b_ptr = (char *)&obj->b_value; /* The b_needsfree flag does not mean that we actually did @@ -2831,51 +3139,56 @@ static int PyCData_MallocBuffer(CDataObject *obj, StgDictObject *dict) /* In python 2.4, and ctypes 0.9.6, the malloc call took about 33% of the creation time for c_int(). */ - obj->b_ptr = (char *)PyMem_Malloc(dict->size); + obj->b_ptr = (char *)PyMem_Malloc(info->size); if (obj->b_ptr == NULL) { PyErr_NoMemory(); return -1; } obj->b_needsfree = 1; - memset(obj->b_ptr, 0, dict->size); + memset(obj->b_ptr, 0, info->size); } - obj->b_size = dict->size; + obj->b_size = info->size; return 0; } PyObject * -PyCData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr) +PyCData_FromBaseObj(ctypes_state *st, + PyObject *type, PyObject *base, Py_ssize_t index, char *adr) { CDataObject *cmem; - StgDictObject *dict; assert(PyType_Check(type)); - dict = PyType_stgdict(type); - if (!dict) { + + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; } - dict->flags |= DICTFLAG_FINAL; + + info->flags |= DICTFLAG_FINAL; cmem = (CDataObject *)((PyTypeObject *)type)->tp_alloc((PyTypeObject *)type, 0); if (cmem == NULL) { return NULL; } - assert(CDataObject_Check(GLOBAL_STATE(), cmem)); - cmem->b_length = dict->length; - cmem->b_size = dict->size; + assert(CDataObject_Check(st, cmem)); + cmem->b_length = info->length; + cmem->b_size = info->size; if (base) { /* use base's buffer */ - assert(CDataObject_Check(GLOBAL_STATE(), base)); + assert(CDataObject_Check(st, base)); cmem->b_ptr = adr; cmem->b_needsfree = 0; cmem->b_base = (CDataObject *)Py_NewRef(base); cmem->b_index = index; } else { /* copy contents of adr */ - if (-1 == PyCData_MallocBuffer(cmem, dict)) { + if (-1 == PyCData_MallocBuffer(cmem, info)) { Py_DECREF(cmem); return NULL; } - memcpy(cmem->b_ptr, adr, dict->size); + memcpy(cmem->b_ptr, adr, info->size); cmem->b_index = index; } return (PyObject *)cmem; @@ -2885,32 +3198,36 @@ PyCData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr) Box a memory block into a CData instance. */ PyObject * -PyCData_AtAddress(PyObject *type, void *buf) +PyCData_AtAddress(ctypes_state *st, PyObject *type, void *buf) { CDataObject *pd; - StgDictObject *dict; if (PySys_Audit("ctypes.cdata", "n", (Py_ssize_t)buf) < 0) { return NULL; } assert(PyType_Check(type)); - dict = PyType_stgdict(type); - if (!dict) { + + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; } - dict->flags |= DICTFLAG_FINAL; + + info->flags |= DICTFLAG_FINAL; pd = (CDataObject *)((PyTypeObject *)type)->tp_alloc((PyTypeObject *)type, 0); if (!pd) { return NULL; } - assert(CDataObject_Check(GLOBAL_STATE(), pd)); + assert(CDataObject_Check(st, pd)); pd->b_ptr = (char *)buf; - pd->b_length = dict->length; - pd->b_size = dict->size; + pd->b_length = info->length; + pd->b_size = info->size; return (PyObject *)pd; } @@ -2919,10 +3236,9 @@ PyCData_AtAddress(PyObject *type, void *buf) classes. FALSE otherwise FALSE also for subclasses of c_int and such. */ -int _ctypes_simple_instance(PyObject *obj) +int _ctypes_simple_instance(ctypes_state *st, PyObject *obj) { PyTypeObject *type = (PyTypeObject *)obj; - ctypes_state *st = GLOBAL_STATE(); if (PyCSimpleTypeObject_Check(st, type)) { return type->tp_base != st->Simple_Type; @@ -2931,37 +3247,62 @@ int _ctypes_simple_instance(PyObject *obj) } PyObject * -PyCData_get(PyObject *type, GETFUNC getfunc, PyObject *src, +PyCData_get(ctypes_state *st, PyObject *type, GETFUNC getfunc, PyObject *src, Py_ssize_t index, Py_ssize_t size, char *adr) { - StgDictObject *dict; - if (getfunc) - return getfunc(adr, size); + CDataObject *cdata = _CDataObject_CAST(src); + if (getfunc) { + PyObject *res; + LOCK_PTR(cdata); + res = getfunc(adr, size); + UNLOCK_PTR(cdata); + return res; + } assert(type); - dict = PyType_stgdict(type); - if (dict && dict->getfunc && !_ctypes_simple_instance(type)) - return dict->getfunc(adr, size); - return PyCData_FromBaseObj(type, src, index, adr); + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (info && info->getfunc && !_ctypes_simple_instance(st, type)) { + PyObject *res; + LOCK_PTR(cdata); + res = info->getfunc(adr, size); + UNLOCK_PTR(cdata); + return res; + } + return PyCData_FromBaseObj(st, type, src, index, adr); } /* Helper function for PyCData_set below. */ static PyObject * -_PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, +_PyCData_set(ctypes_state *st, + CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, Py_ssize_t size, char *ptr) { CDataObject *src; int err; if (setfunc) { - return setfunc(ptr, value, size); + PyObject *res; + LOCK_PTR(dst); + res = setfunc(ptr, value, size); + UNLOCK_PTR(dst); + return res; } - ctypes_state *st = GLOBAL_STATE(); if (!CDataObject_Check(st, value)) { - StgDictObject *dict = PyType_stgdict(type); - if (dict && dict->setfunc) - return dict->setfunc(ptr, value, size); + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (info && info->setfunc) { + PyObject *res; + LOCK_PTR(dst); + res = info->setfunc(ptr, value, size); + UNLOCK_PTR(dst); + return res; + } /* If value is a tuple, we try to call the type with the tuple and use the result! @@ -2976,12 +3317,14 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, ((PyTypeObject *)type)->tp_name); return NULL; } - result = _PyCData_set(dst, type, setfunc, ob, + result = _PyCData_set(st, dst, type, setfunc, ob, size, ptr); Py_DECREF(ob); return result; } else if (value == Py_None && PyCPointerTypeObject_Check(st, type)) { + LOCK_PTR(dst); *(void **)ptr = NULL; + UNLOCK_PTR(dst); Py_RETURN_NONE; } else { PyErr_Format(PyExc_TypeError, @@ -2997,9 +3340,7 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, if (err == -1) return NULL; if (err) { - memcpy(ptr, - src->b_ptr, - size); + locked_memcpy_from(ptr, src, size); if (PyCPointerTypeObject_Check(st, type)) { /* XXX */ @@ -3014,11 +3355,16 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, if (PyCPointerTypeObject_Check(st, type) && ArrayObject_Check(st, value)) { - StgDictObject *p1, *p2; PyObject *keep; - p1 = PyObject_stgdict(value); + + StgInfo *p1, *p2; + if (PyStgInfo_FromObject(st, value, &p1) < 0) { + return NULL; + } assert(p1); /* Cannot be NULL for array instances */ - p2 = PyType_stgdict(type); + if (PyStgInfo_FromType(st, type, &p2) < 0) { + return NULL; + } assert(p2); /* Cannot be NULL for pointer types */ if (p1->proto != p2->proto) { @@ -3028,7 +3374,9 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, ((PyTypeObject *)type)->tp_name); return NULL; } + LOCK_PTR(src); *(void **)ptr = src->b_ptr; + UNLOCK_PTR(src); keep = GetKeepedObjects(src); if (keep == NULL) @@ -3056,12 +3404,12 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, * to the value 'value'. */ int -PyCData_set(PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, +PyCData_set(ctypes_state *st, + PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, Py_ssize_t index, Py_ssize_t size, char *ptr) { CDataObject *mem = (CDataObject *)dst; PyObject *result; - ctypes_state *st = GLOBAL_STATE(); if (!CDataObject_Check(st, dst)) { PyErr_SetString(PyExc_TypeError, @@ -3069,7 +3417,7 @@ PyCData_set(PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, return -1; } - result = _PyCData_set(mem, type, setfunc, value, + result = _PyCData_set(st, mem, type, setfunc, value, size, ptr); if (result == NULL) return -1; @@ -3084,17 +3432,28 @@ PyCData_set(PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, /******************************************************************/ static PyObject * GenericPyCData_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + return generic_pycdata_new(st, type, args, kwds); +} + +static inline PyObject * +generic_pycdata_new(ctypes_state *st, + PyTypeObject *type, PyObject *args, PyObject *kwds) { CDataObject *obj; - StgDictObject *dict; - dict = PyType_stgdict((PyObject *)type); - if (!dict) { + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; } - dict->flags |= DICTFLAG_FINAL; + + info->flags |= DICTFLAG_FINAL; obj = (CDataObject *)type->tp_alloc(type, 0); if (!obj) @@ -3103,9 +3462,9 @@ GenericPyCData_new(PyTypeObject *type, PyObject *args, PyObject *kwds) obj->b_base = NULL; obj->b_index = 0; obj->b_objects = NULL; - obj->b_length = dict->length; + obj->b_length = info->length; - if (-1 == PyCData_MallocBuffer(obj, dict)) { + if (-1 == PyCData_MallocBuffer(obj, info)) { Py_DECREF(obj); return NULL; } @@ -3116,21 +3475,37 @@ GenericPyCData_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyCFuncPtr_Type */ +/*[clinic input] +@critical_section +@setter +_ctypes.CFuncPtr.errcheck +[clinic start generated code]*/ + static int -PyCFuncPtr_set_errcheck(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ignored)) +_ctypes_CFuncPtr_errcheck_set_impl(PyCFuncPtrObject *self, PyObject *value) +/*[clinic end generated code: output=6580cf1ffdf3b9fb input=84930bb16c490b33]*/ { - if (ob && !PyCallable_Check(ob)) { + if (value && !PyCallable_Check(value)) { PyErr_SetString(PyExc_TypeError, "the errcheck attribute must be callable"); return -1; } - Py_XINCREF(ob); - Py_XSETREF(self->errcheck, ob); + Py_XINCREF(value); + Py_XSETREF(self->errcheck, value); return 0; } +/*[clinic input] +@critical_section +@getter +_ctypes.CFuncPtr.errcheck + +a function to check for errors +[clinic start generated code]*/ + static PyObject * -PyCFuncPtr_get_errcheck(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) +_ctypes_CFuncPtr_errcheck_get_impl(PyCFuncPtrObject *self) +/*[clinic end generated code: output=dfa6fb5c6f90fd14 input=4672135fef37819f]*/ { if (self->errcheck) { return Py_NewRef(self->errcheck); @@ -3138,92 +3513,132 @@ PyCFuncPtr_get_errcheck(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) Py_RETURN_NONE; } +/*[clinic input] +@setter +@critical_section +_ctypes.CFuncPtr.restype +[clinic start generated code]*/ + static int -PyCFuncPtr_set_restype(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ignored)) +_ctypes_CFuncPtr_restype_set_impl(PyCFuncPtrObject *self, PyObject *value) +/*[clinic end generated code: output=0be0a086abbabf18 input=683c3bef4562ccc6]*/ { PyObject *checker, *oldchecker; - if (ob == NULL) { + if (value == NULL) { oldchecker = self->checker; self->checker = NULL; Py_CLEAR(self->restype); Py_XDECREF(oldchecker); return 0; } - if (ob != Py_None && !PyType_stgdict(ob) && !PyCallable_Check(ob)) { + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *info; + if (PyStgInfo_FromType(st, value, &info) < 0) { + return -1; + } + if (value != Py_None && !info && !PyCallable_Check(value)) { PyErr_SetString(PyExc_TypeError, "restype must be a type, a callable, or None"); return -1; } - if (PyObject_GetOptionalAttr(ob, &_Py_ID(_check_retval_), &checker) < 0) { + if (PyObject_GetOptionalAttr(value, &_Py_ID(_check_retval_), &checker) < 0) { return -1; } oldchecker = self->checker; self->checker = checker; - Py_INCREF(ob); - Py_XSETREF(self->restype, ob); + Py_INCREF(value); + Py_XSETREF(self->restype, value); Py_XDECREF(oldchecker); return 0; } +/*[clinic input] +@getter +@critical_section +_ctypes.CFuncPtr.restype + +specify the result type +[clinic start generated code]*/ + static PyObject * -PyCFuncPtr_get_restype(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) +_ctypes_CFuncPtr_restype_get_impl(PyCFuncPtrObject *self) +/*[clinic end generated code: output=c8f44cd16f1dee5e input=5e3ed95116204fd2]*/ { - StgDictObject *dict; if (self->restype) { return Py_NewRef(self->restype); } - dict = PyObject_stgdict((PyObject *)self); - assert(dict); /* Cannot be NULL for PyCFuncPtrObject instances */ - if (dict->restype) { - return Py_NewRef(dict->restype); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *info; + if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { + return NULL; + } + assert(info); /* Cannot be NULL for PyCFuncPtrObject instances */ + if (info->restype) { + return Py_NewRef(info->restype); } else { Py_RETURN_NONE; } } +/*[clinic input] +@setter +@critical_section +_ctypes.CFuncPtr.argtypes +[clinic start generated code]*/ + static int -PyCFuncPtr_set_argtypes(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ignored)) +_ctypes_CFuncPtr_argtypes_set_impl(PyCFuncPtrObject *self, PyObject *value) +/*[clinic end generated code: output=596a36e2ae89d7d1 input=c4627573e980aa8b]*/ { PyObject *converters; - if (ob == NULL || ob == Py_None) { + if (value == NULL || value == Py_None) { Py_CLEAR(self->converters); Py_CLEAR(self->argtypes); } else { - converters = converters_from_argtypes(ob); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + converters = converters_from_argtypes(st, value); if (!converters) return -1; Py_XSETREF(self->converters, converters); - Py_INCREF(ob); - Py_XSETREF(self->argtypes, ob); + Py_INCREF(value); + Py_XSETREF(self->argtypes, value); } return 0; } +/*[clinic input] +@getter +@critical_section +_ctypes.CFuncPtr.argtypes + +specify the argument types +[clinic start generated code]*/ + static PyObject * -PyCFuncPtr_get_argtypes(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) +_ctypes_CFuncPtr_argtypes_get_impl(PyCFuncPtrObject *self) +/*[clinic end generated code: output=c46b05a1b0f99172 input=37a8a545a56f8ae2]*/ { - StgDictObject *dict; if (self->argtypes) { return Py_NewRef(self->argtypes); } - dict = PyObject_stgdict((PyObject *)self); - assert(dict); /* Cannot be NULL for PyCFuncPtrObject instances */ - if (dict->argtypes) { - return Py_NewRef(dict->argtypes); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *info; + if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { + return NULL; + } + assert(info); /* Cannot be NULL for PyCFuncPtrObject instances */ + if (info->argtypes) { + return Py_NewRef(info->argtypes); } else { Py_RETURN_NONE; } } static PyGetSetDef PyCFuncPtr_getsets[] = { - { "errcheck", (getter)PyCFuncPtr_get_errcheck, (setter)PyCFuncPtr_set_errcheck, - "a function to check for errors", NULL }, - { "restype", (getter)PyCFuncPtr_get_restype, (setter)PyCFuncPtr_set_restype, - "specify the result type", NULL }, - { "argtypes", (getter)PyCFuncPtr_get_argtypes, - (setter)PyCFuncPtr_set_argtypes, - "specify the argument types", NULL }, + _CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF + _CTYPES_CFUNCPTR_RESTYPE_GETSETDEF + _CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF { NULL, NULL } }; @@ -3242,7 +3657,6 @@ static PPROC FindAddress(void *handle, const char *name, PyObject *type) #else char *mangled_name; int i; - StgDictObject *dict; Py_BEGIN_ALLOW_THREADS address = (PPROC)GetProcAddress(handle, name); @@ -3253,9 +3667,13 @@ static PPROC FindAddress(void *handle, const char *name, PyObject *type) return NULL; } - dict = PyType_stgdict((PyObject *)type); - /* It should not happen that dict is NULL, but better be safe */ - if (dict==NULL || dict->flags & FUNCFLAG_CDECL) + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return NULL; + } + /* It should not happen that info is NULL, but better be safe */ + if (info==NULL || info->flags & FUNCFLAG_CDECL) return address; /* for stdcall, try mangled names: @@ -3280,23 +3698,23 @@ static PPROC FindAddress(void *handle, const char *name, PyObject *type) /* Return 1 if usable, 0 else and exception set. */ static int -_check_outarg_type(PyObject *arg, Py_ssize_t index) +_check_outarg_type(ctypes_state *st, PyObject *arg, Py_ssize_t index) { - StgDictObject *dict; - ctypes_state *st = GLOBAL_STATE(); - if (PyCPointerTypeObject_Check(st, arg)) { return 1; } if (PyCArrayTypeObject_Check(st, arg)) { return 1; } - dict = PyType_stgdict(arg); - if (dict + StgInfo *info; + if (PyStgInfo_FromType(st, arg, &info) < 0) { + return -1; + } + if (info /* simple pointer types, c_void_p, c_wchar_p, BSTR, ... */ - && PyUnicode_Check(dict->proto) + && PyUnicode_Check(info->proto) /* We only allow c_void_p, c_char_p and c_wchar_p as a simple output parameter type */ - && (strchr("PzZ", PyUnicode_AsUTF8(dict->proto)[0]))) { + && (strchr("PzZ", PyUnicode_AsUTF8(info->proto)[0]))) { return 1; } @@ -3311,21 +3729,23 @@ _check_outarg_type(PyObject *arg, Py_ssize_t index) /* Returns 1 on success, 0 on error */ static int -_validate_paramflags(PyTypeObject *type, PyObject *paramflags) +_validate_paramflags(ctypes_state *st, PyTypeObject *type, PyObject *paramflags) { Py_ssize_t i, len; - StgDictObject *dict; PyObject *argtypes; - dict = PyType_stgdict((PyObject *)type); - if (!dict) { + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return -1; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return 0; } - argtypes = dict->argtypes; + argtypes = info->argtypes; - if (paramflags == NULL || dict->argtypes == NULL) + if (paramflags == NULL || info->argtypes == NULL) return 1; if (!PyTuple_Check(paramflags)) { @@ -3335,7 +3755,7 @@ _validate_paramflags(PyTypeObject *type, PyObject *paramflags) } len = PyTuple_GET_SIZE(paramflags); - if (len != PyTuple_GET_SIZE(dict->argtypes)) { + if (len != PyTuple_GET_SIZE(info->argtypes)) { PyErr_SetString(PyExc_ValueError, "paramflags must have the same length as argtypes"); return 0; @@ -3362,7 +3782,7 @@ _validate_paramflags(PyTypeObject *type, PyObject *paramflags) case PARAMFLAG_FIN | PARAMFLAG_FOUT: break; case PARAMFLAG_FOUT: - if (!_check_outarg_type(typ, i+1)) + if (!_check_outarg_type(st, typ, i+1)) return 0; break; default: @@ -3465,6 +3885,7 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } +#undef USE_DLERROR #ifdef MS_WIN32 address = FindAddress(handle, name, (PyObject *)type); if (!address) { @@ -3480,26 +3901,41 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } #else + #ifdef __CYGWIN__ + //dlerror() isn't very helpful on cygwin */ + #else + #define USE_DLERROR + /* dlerror() always returns the latest error. + * + * Clear the previous value before calling dlsym(), + * to ensure we can tell if our call resulted in an error. + */ + (void)dlerror(); + #endif address = (PPROC)dlsym(handle, name); + if (!address) { -#ifdef __CYGWIN__ -/* dlerror() isn't very helpful on cygwin */ - PyErr_Format(PyExc_AttributeError, - "function '%s' not found", - name); -#else - PyErr_SetString(PyExc_AttributeError, dlerror()); -#endif + #ifdef USE_DLERROR + const char *dlerr = dlerror(); + if (dlerr) { + _PyErr_SetLocaleString(PyExc_AttributeError, dlerr); + Py_DECREF(ftuple); + return NULL; + } + #endif + PyErr_Format(PyExc_AttributeError, "function '%s' not found", name); Py_DECREF(ftuple); return NULL; } #endif - if (!_validate_paramflags(type, paramflags)) { +#undef USE_DLERROR + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + if (!_validate_paramflags(st, type, paramflags)) { Py_DECREF(ftuple); return NULL; } - self = (PyCFuncPtrObject *)GenericPyCData_new(type, args, kwds); + self = (PyCFuncPtrObject *)generic_pycdata_new(st, type, args, kwds); if (!self) { Py_DECREF(ftuple); return NULL; @@ -3507,6 +3943,8 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) self->paramflags = Py_XNewRef(paramflags); + // No other threads can have this object, no need to + // lock it. *(void **)self->b_ptr = address; Py_INCREF(dll); Py_DECREF(ftuple); @@ -3535,10 +3973,11 @@ PyCFuncPtr_FromVtblIndex(PyTypeObject *type, PyObject *args, PyObject *kwds) if (paramflags == Py_None) paramflags = NULL; - if (!_validate_paramflags(type, paramflags)) + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + if (!_validate_paramflags(st, type, paramflags)) { return NULL; - - self = (PyCFuncPtrObject *)GenericPyCData_new(type, args, kwds); + } + self = (PyCFuncPtrObject *)generic_pycdata_new(st, type, args, kwds); self->index = index + 0x1000; self->paramflags = Py_XNewRef(paramflags); if (iid_len == sizeof(GUID)) @@ -3565,7 +4004,6 @@ PyCFuncPtr_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { PyCFuncPtrObject *self; PyObject *callable; - StgDictObject *dict; CThunkObject *thunk; if (PyTuple_GET_SIZE(args) == 0) @@ -3616,23 +4054,28 @@ PyCFuncPtr_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } */ - dict = PyType_stgdict((PyObject *)type); + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return NULL; + } /* XXXX Fails if we do: 'PyCFuncPtr(lambda x: x)' */ - if (!dict || !dict->argtypes) { + if (!info || !info->argtypes) { PyErr_SetString(PyExc_TypeError, "cannot construct instance of this class:" " no argtypes"); return NULL; } - thunk = _ctypes_alloc_callback(callable, - dict->argtypes, - dict->restype, - dict->flags); + thunk = _ctypes_alloc_callback(st, + callable, + info->argtypes, + info->restype, + info->flags); if (!thunk) return NULL; - self = (PyCFuncPtrObject *)GenericPyCData_new(type, args, kwds); + self = (PyCFuncPtrObject *)generic_pycdata_new(st, type, args, kwds); if (self == NULL) { Py_DECREF(thunk); return NULL; @@ -3656,10 +4099,9 @@ PyCFuncPtr_new(PyTypeObject *type, PyObject *args, PyObject *kwds) _byref consumes a refcount to its argument */ static PyObject * -_byref(PyObject *obj) +_byref(ctypes_state *st, PyObject *obj) { PyCArgObject *parg; - ctypes_state *st = GLOBAL_STATE(); if (!CDataObject_Check(st, obj)) { PyErr_SetString(PyExc_TypeError, @@ -3667,7 +4109,7 @@ _byref(PyObject *obj) return NULL; } - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) { Py_DECREF(obj); return NULL; @@ -3731,13 +4173,12 @@ _get_arg(int *pindex, PyObject *name, PyObject *defval, PyObject *inargs, PyObje function. */ static PyObject * -_build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, +_build_callargs(ctypes_state *st, PyCFuncPtrObject *self, PyObject *argtypes, PyObject *inargs, PyObject *kwds, int *poutmask, int *pinoutmask, unsigned int *pnumretvals) { PyObject *paramflags = self->paramflags; PyObject *callargs; - StgDictObject *dict; Py_ssize_t i, len; int inargs_index = 0; /* It's a little bit difficult to determine how many arguments the @@ -3769,7 +4210,6 @@ _build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, inargs_index = 1; } #endif - ctypes_state *st = GLOBAL_STATE(); for (i = 0; i < len; ++i) { PyObject *item = PyTuple_GET_ITEM(paramflags, i); PyObject *ob; @@ -3798,7 +4238,7 @@ _build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, case (PARAMFLAG_FIN | PARAMFLAG_FOUT): *pinoutmask |= (1 << i); /* mark as inout arg */ (*pnumretvals)++; - /* fall through */ + _Py_FALLTHROUGH; case 0: case PARAMFLAG_FIN: /* 'in' parameter. Copy it from inargs. */ @@ -3826,15 +4266,18 @@ _build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, break; } ob = PyTuple_GET_ITEM(argtypes, i); - dict = PyType_stgdict(ob); - if (dict == NULL) { + StgInfo *info; + if (PyStgInfo_FromType(st, ob, &info) < 0) { + goto error; + } + if (info == NULL) { /* Cannot happen: _validate_paramflags() would not accept such an object */ PyErr_Format(PyExc_RuntimeError, - "NULL stgdict unexpected"); + "NULL stginfo unexpected"); goto error; } - if (PyUnicode_Check(dict->proto)) { + if (PyUnicode_Check(info->proto)) { PyErr_Format( PyExc_TypeError, "%s 'out' parameter must be passed as default value", @@ -3846,7 +4289,7 @@ _build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, } else { /* Create an instance of the pointed-to type */ - ob = _PyObject_CallNoArgs(dict->proto); + ob = _PyObject_CallNoArgs(info->proto); } /* XXX Is the following correct any longer? @@ -3960,13 +4403,12 @@ _build_result(PyObject *result, PyObject *callargs, } static PyObject * -PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) +PyCFuncPtr_call(PyObject *op, PyObject *inargs, PyObject *kwds) { PyObject *restype; PyObject *converters; PyObject *checker; PyObject *argtypes; - StgDictObject *dict = PyObject_stgdict((PyObject *)self); PyObject *result; PyObject *callargs; PyObject *errcheck; @@ -3974,18 +4416,25 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) IUnknown *piunk = NULL; #endif void *pProc = NULL; + PyCFuncPtrObject *self = _PyCFuncPtrObject_CAST(op); int inoutmask; int outmask; unsigned int numretvals; - assert(dict); /* Cannot be NULL for PyCFuncPtrObject instances */ - restype = self->restype ? self->restype : dict->restype; - converters = self->converters ? self->converters : dict->converters; - checker = self->checker ? self->checker : dict->checker; - argtypes = self->argtypes ? self->argtypes : dict->argtypes; -/* later, we probably want to have an errcheck field in stgdict */ - errcheck = self->errcheck /* ? self->errcheck : dict->errcheck */; + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *info; + if (PyStgInfo_FromObject(st, op, &info) < 0) { + return NULL; + } + assert(info); /* Cannot be NULL for PyCFuncPtrObject instances */ + + restype = self->restype ? self->restype : info->restype; + converters = self->converters ? self->converters : info->converters; + checker = self->checker ? self->checker : info->checker; + argtypes = self->argtypes ? self->argtypes : info->argtypes; +/* later, we probably want to have an errcheck field in stginfo */ + errcheck = self->errcheck /* ? self->errcheck : info->errcheck */; pProc = *(void **)self->b_ptr; @@ -3999,7 +4448,6 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) "native com method call without 'this' parameter"); return NULL; } - ctypes_state *st = GLOBAL_STATE(); if (!CDataObject_Check(st, this)) { PyErr_SetString(PyExc_TypeError, "Expected a COM this pointer as first argument"); @@ -4021,7 +4469,7 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) pProc = ((void **)piunk->lpVtbl)[self->index - 0x1000]; } #endif - callargs = _build_callargs(self, argtypes, + callargs = _build_callargs(st, self, argtypes, inargs, kwds, &outmask, &inoutmask, &numretvals); if (callargs == NULL) @@ -4033,7 +4481,7 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) int actual = Py_SAFE_DOWNCAST(PyTuple_GET_SIZE(callargs), Py_ssize_t, int); - if ((dict->flags & FUNCFLAG_CDECL) == FUNCFLAG_CDECL) { + if ((info->flags & FUNCFLAG_CDECL) == FUNCFLAG_CDECL) { /* For cdecl functions, we allow more actual arguments than the length of the argtypes tuple. */ @@ -4057,13 +4505,14 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) } } - result = _ctypes_callproc(pProc, + result = _ctypes_callproc(st, + pProc, callargs, #ifdef MS_WIN32 piunk, self->iid, #endif - dict->flags, + info->flags, converters, restype, checker); @@ -4093,8 +4542,9 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) } static int -PyCFuncPtr_traverse(PyCFuncPtrObject *self, visitproc visit, void *arg) +PyCFuncPtr_traverse(PyObject *op, visitproc visit, void *arg) { + PyCFuncPtrObject *self = _PyCFuncPtrObject_CAST(op); Py_VISIT(self->callable); Py_VISIT(self->restype); Py_VISIT(self->checker); @@ -4103,12 +4553,13 @@ PyCFuncPtr_traverse(PyCFuncPtrObject *self, visitproc visit, void *arg) Py_VISIT(self->converters); Py_VISIT(self->paramflags); Py_VISIT(self->thunk); - return PyCData_traverse((CDataObject *)self, visit, arg); + return PyCData_traverse(op, visit, arg); } static int -PyCFuncPtr_clear(PyCFuncPtrObject *self) +PyCFuncPtr_clear(PyObject *op) { + PyCFuncPtrObject *self = _PyCFuncPtrObject_CAST(op); Py_CLEAR(self->callable); Py_CLEAR(self->restype); Py_CLEAR(self->checker); @@ -4117,19 +4568,23 @@ PyCFuncPtr_clear(PyCFuncPtrObject *self) Py_CLEAR(self->converters); Py_CLEAR(self->paramflags); Py_CLEAR(self->thunk); - return PyCData_clear((CDataObject *)self); + return PyCData_clear(op); } static void -PyCFuncPtr_dealloc(PyCFuncPtrObject *self) +PyCFuncPtr_dealloc(PyObject *self) { - PyCFuncPtr_clear(self); - Py_TYPE(self)->tp_free((PyObject *)self); + PyObject_GC_UnTrack(self); + (void)PyCFuncPtr_clear(self); + PyTypeObject *type = Py_TYPE(self); + type->tp_free(self); + Py_DECREF(type); } static PyObject * -PyCFuncPtr_repr(PyCFuncPtrObject *self) +PyCFuncPtr_repr(PyObject *op) { + PyCFuncPtrObject *self = _PyCFuncPtrObject_CAST(op); #ifdef MS_WIN32 if (self->index) return PyUnicode_FromFormat("", @@ -4143,8 +4598,9 @@ PyCFuncPtr_repr(PyCFuncPtrObject *self) } static int -PyCFuncPtr_bool(PyCFuncPtrObject *self) +PyCFuncPtr_bool(PyObject *op) { + PyCFuncPtrObject *self = _PyCFuncPtrObject_CAST(op); return ((*(void **)self->b_ptr != NULL) #ifdef MS_WIN32 || (self->index != 0) @@ -4152,59 +4608,26 @@ PyCFuncPtr_bool(PyCFuncPtrObject *self) ); } -static PyNumberMethods PyCFuncPtr_as_number = { - 0, /* nb_add */ - 0, /* nb_subtract */ - 0, /* nb_multiply */ - 0, /* nb_remainder */ - 0, /* nb_divmod */ - 0, /* nb_power */ - 0, /* nb_negative */ - 0, /* nb_positive */ - 0, /* nb_absolute */ - (inquiry)PyCFuncPtr_bool, /* nb_bool */ +static PyType_Slot pycfuncptr_slots[] = { + {Py_tp_dealloc, PyCFuncPtr_dealloc}, + {Py_tp_repr, PyCFuncPtr_repr}, + {Py_tp_call, PyCFuncPtr_call}, + {Py_tp_doc, PyDoc_STR("Function Pointer")}, + {Py_tp_traverse, PyCFuncPtr_traverse}, + {Py_tp_clear, PyCFuncPtr_clear}, + {Py_tp_getset, PyCFuncPtr_getsets}, + {Py_tp_new, PyCFuncPtr_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {Py_nb_bool, PyCFuncPtr_bool}, + {0, NULL}, }; -PyTypeObject PyCFuncPtr_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.CFuncPtr", - sizeof(PyCFuncPtrObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)PyCFuncPtr_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)PyCFuncPtr_repr, /* tp_repr */ - &PyCFuncPtr_as_number, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - (ternaryfunc)PyCFuncPtr_call, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("Function Pointer"), /* tp_doc */ - (traverseproc)PyCFuncPtr_traverse, /* tp_traverse */ - (inquiry)PyCFuncPtr_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - PyCFuncPtr_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyCFuncPtr_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Spec pycfuncptr_spec = { + .name = "_ctypes.CFuncPtr", + .basicsize = sizeof(PyCFuncPtrObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycfuncptr_slots, }; /*****************************************************************/ @@ -4224,11 +4647,15 @@ _init_pos_args(PyObject *self, PyTypeObject *type, PyObject *args, PyObject *kwds, Py_ssize_t index) { - StgDictObject *dict; PyObject *fields; Py_ssize_t i; - if (PyType_stgdict((PyObject *)type->tp_base)) { + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + StgInfo *baseinfo; + if (PyStgInfo_FromType(st, (PyObject *)type->tp_base, &baseinfo) < 0) { + return -1; + } + if (baseinfo) { index = _init_pos_args(self, type->tp_base, args, kwds, index); @@ -4236,8 +4663,17 @@ _init_pos_args(PyObject *self, PyTypeObject *type, return -1; } - dict = PyType_stgdict((PyObject *)type); - fields = PyDict_GetItemWithError((PyObject *)dict, &_Py_ID(_fields_)); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return -1; + } + assert(info); + + PyObject *attrdict = PyType_GetDict(type); + assert(attrdict); + + fields = PyDict_GetItemWithError((PyObject *)attrdict, &_Py_ID(_fields_)); + Py_CLEAR(attrdict); if (fields == NULL) { if (PyErr_Occurred()) { return -1; @@ -4246,7 +4682,7 @@ _init_pos_args(PyObject *self, PyTypeObject *type, } for (i = index; - i < dict->length && i < PyTuple_GET_SIZE(args); + i < info->length && i < PyTuple_GET_SIZE(args); ++i) { PyObject *pair = PySequence_GetItem(fields, i - index); PyObject *name, *val; @@ -4279,7 +4715,7 @@ _init_pos_args(PyObject *self, PyTypeObject *type, if (res == -1) return -1; } - return dict->length; + return info->length; } static int @@ -4316,88 +4752,34 @@ Struct_init(PyObject *self, PyObject *args, PyObject *kwds) return 0; } -static PyTypeObject Struct_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.Structure", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("Structure base class"), /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - Struct_init, /* tp_init */ - 0, /* tp_alloc */ - GenericPyCData_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycstruct_slots[] = { + {Py_tp_doc, PyDoc_STR("Structure base class")}, + {Py_tp_init, Struct_init}, + {Py_tp_new, GenericPyCData_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {0, NULL}, +}; + +static PyType_Spec pycstruct_spec = { + .name = "_ctypes.Structure", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycstruct_slots, +}; + +static PyType_Slot pycunion_slots[] = { + {Py_tp_doc, PyDoc_STR("Union base class")}, + {Py_tp_init, Struct_init}, + {Py_tp_new, GenericPyCData_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {0, NULL}, }; -static PyTypeObject Union_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.Union", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("Union base class"), /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - Struct_init, /* tp_init */ - 0, /* tp_alloc */ - GenericPyCData_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Spec pycunion_spec = { + .name = "_ctypes.Union", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycunion_slots, }; @@ -4406,7 +4788,7 @@ static PyTypeObject Union_Type = { PyCArray_Type */ static int -Array_init(CDataObject *self, PyObject *args, PyObject *kw) +Array_init(PyObject *self, PyObject *args, PyObject *kw) { Py_ssize_t i; Py_ssize_t n; @@ -4420,7 +4802,7 @@ Array_init(CDataObject *self, PyObject *args, PyObject *kw) for (i = 0; i < n; ++i) { PyObject *v; v = PyTuple_GET_ITEM(args, i); - if (-1 == PySequence_SetItem((PyObject *)self, i, v)) + if (-1 == PySequence_SetItem(self, i, v)) return -1; } return 0; @@ -4429,10 +4811,8 @@ Array_init(CDataObject *self, PyObject *args, PyObject *kw) static PyObject * Array_item(PyObject *myself, Py_ssize_t index) { - CDataObject *self = (CDataObject *)myself; + CDataObject *self = _CDataObject_CAST(myself); Py_ssize_t offset, size; - StgDictObject *stgdict; - if (index < 0 || index >= self->b_length) { PyErr_SetString(PyExc_IndexError, @@ -4440,22 +4820,26 @@ Array_item(PyObject *myself, Py_ssize_t index) return NULL; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for array instances */ + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, myself, &stginfo) < 0) { + return NULL; + } + /* Would it be clearer if we got the item size from - stgdict->proto's stgdict? + stginfo->proto's stginfo? */ - size = stgdict->size / stgdict->length; + size = stginfo->size / stginfo->length; offset = index * size; - return PyCData_get(stgdict->proto, stgdict->getfunc, (PyObject *)self, - index, size, self->b_ptr + offset); + return PyCData_get(st, stginfo->proto, stginfo->getfunc, myself, + index, size, self->b_ptr + offset); } static PyObject * Array_subscript(PyObject *myself, PyObject *item) { - CDataObject *self = (CDataObject *)myself; + CDataObject *self = _CDataObject_CAST(myself); if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); @@ -4467,7 +4851,6 @@ Array_subscript(PyObject *myself, PyObject *item) return Array_item(myself, i); } else if (PySlice_Check(item)) { - StgDictObject *stgdict, *itemdict; PyObject *proto; PyObject *np; Py_ssize_t start, stop, step, slicelen, i; @@ -4478,46 +4861,63 @@ Array_subscript(PyObject *myself, PyObject *item) } slicelen = PySlice_AdjustIndices(self->b_length, &start, &stop, step); - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for array object instances */ - proto = stgdict->proto; - itemdict = PyType_stgdict(proto); - assert(itemdict); /* proto is the item type of the array, a + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, myself, &stginfo) < 0) { + return NULL; + } + assert(stginfo); /* Cannot be NULL for array object instances */ + proto = stginfo->proto; + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, proto, &iteminfo) < 0) { + return NULL; + } + assert(iteminfo); /* proto is the item type of the array, a ctypes type, so this cannot be NULL */ - if (itemdict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { + if (iteminfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { char *ptr = (char *)self->b_ptr; char *dest; if (slicelen <= 0) - return PyBytes_FromStringAndSize("", 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); if (step == 1) { - return PyBytes_FromStringAndSize(ptr + start, - slicelen); + PyObject *res; + LOCK_PTR(self); + res = PyBytes_FromStringAndSize(ptr + start, + slicelen); + UNLOCK_PTR(self); + return res; } dest = (char *)PyMem_Malloc(slicelen); if (dest == NULL) return PyErr_NoMemory(); + LOCK_PTR(self); for (cur = start, i = 0; i < slicelen; cur += step, i++) { dest[i] = ptr[cur]; } + UNLOCK_PTR(self); np = PyBytes_FromStringAndSize(dest, slicelen); PyMem_Free(dest); return np; } - if (itemdict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { + if (iteminfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) { wchar_t *ptr = (wchar_t *)self->b_ptr; wchar_t *dest; if (slicelen <= 0) - return PyUnicode_New(0, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (step == 1) { - return PyUnicode_FromWideChar(ptr + start, - slicelen); + PyObject *res; + LOCK_PTR(self); + res = PyUnicode_FromWideChar(ptr + start, + slicelen); + UNLOCK_PTR(self); + return res; } dest = PyMem_New(wchar_t, slicelen); @@ -4526,10 +4926,12 @@ Array_subscript(PyObject *myself, PyObject *item) return NULL; } + LOCK_PTR(self); for (cur = start, i = 0; i < slicelen; cur += step, i++) { dest[i] = ptr[cur]; } + UNLOCK_PTR(self); np = PyUnicode_FromWideChar(dest, slicelen); PyMem_Free(dest); @@ -4562,9 +4964,8 @@ Array_subscript(PyObject *myself, PyObject *item) static int Array_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) { - CDataObject *self = (CDataObject *)myself; + CDataObject *self = _CDataObject_CAST(myself); Py_ssize_t size, offset; - StgDictObject *stgdict; char *ptr; if (value == NULL) { @@ -4573,25 +4974,30 @@ Array_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) return -1; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for array object instances */ - if (index < 0 || index >= stgdict->length) { + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, myself, &stginfo) < 0) { + return -1; + } + assert(stginfo); /* Cannot be NULL for array object instances */ + + if (index < 0 || index >= stginfo->length) { PyErr_SetString(PyExc_IndexError, "invalid index"); return -1; } - size = stgdict->size / stgdict->length; + size = stginfo->size / stginfo->length; offset = index * size; ptr = self->b_ptr + offset; - return PyCData_set((PyObject *)self, stgdict->proto, stgdict->setfunc, value, - index, size, ptr); + return PyCData_set(st, myself, stginfo->proto, stginfo->setfunc, value, + index, size, ptr); } static int Array_ass_subscript(PyObject *myself, PyObject *item, PyObject *value) { - CDataObject *self = (CDataObject *)myself; + CDataObject *self = _CDataObject_CAST(myself); if (value == NULL) { PyErr_SetString(PyExc_TypeError, @@ -4648,7 +5054,7 @@ Array_ass_subscript(PyObject *myself, PyObject *item, PyObject *value) static Py_ssize_t Array_length(PyObject *myself) { - CDataObject *self = (CDataObject *)myself; + CDataObject *self = _CDataObject_CAST(myself); return self->b_length; } @@ -4658,26 +5064,6 @@ static PyMethodDef Array_methods[] = { { NULL, NULL } }; -static PySequenceMethods Array_as_sequence = { - Array_length, /* sq_length; */ - 0, /* sq_concat; */ - 0, /* sq_repeat; */ - Array_item, /* sq_item; */ - 0, /* sq_slice; */ - Array_ass_item, /* sq_ass_item; */ - 0, /* sq_ass_slice; */ - 0, /* sq_contains; */ - - 0, /* sq_inplace_concat; */ - 0, /* sq_inplace_repeat; */ -}; - -static PyMappingMethods Array_as_mapping = { - Array_length, - Array_subscript, - Array_ass_subscript, -}; - PyDoc_STRVAR(array_doc, "Abstract base class for arrays.\n" "\n" @@ -4688,60 +5074,40 @@ PyDoc_STRVAR(array_doc, "reads, the resulting object is not itself an Array." ); -PyTypeObject PyCArray_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.Array", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - &Array_as_sequence, /* tp_as_sequence */ - &Array_as_mapping, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - array_doc, /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - Array_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)Array_init, /* tp_init */ - 0, /* tp_alloc */ - GenericPyCData_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycarray_slots[] = { + {Py_tp_doc, (char*)array_doc}, + {Py_tp_methods, Array_methods}, + {Py_tp_init, Array_init}, + {Py_tp_new, GenericPyCData_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {Py_sq_length, Array_length}, + {Py_sq_item, Array_item}, + {Py_sq_ass_item, Array_ass_item}, + {Py_mp_length, Array_length}, + {Py_mp_subscript, Array_subscript}, + {Py_mp_ass_subscript, Array_ass_subscript}, + {0, NULL}, +}; + +static PyType_Spec pycarray_spec = { + .name = "_ctypes.Array", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycarray_slots, }; PyObject * -PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length) +PyCArrayType_from_ctype(ctypes_state *st, PyObject *itemtype, Py_ssize_t length) { - static PyObject *cache; PyObject *key; char name[256]; PyObject *len; - if (cache == NULL) { - cache = PyDict_New(); - if (cache == NULL) + if (st->array_cache == NULL) { + st->array_cache = PyDict_New(); + if (st->array_cache == NULL) { return NULL; + } } len = PyLong_FromSsize_t(length); if (len == NULL) @@ -4752,7 +5118,7 @@ PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length) return NULL; PyObject *result; - if (_PyDict_GetItemProxy(cache, key, &result) != 0) { + if (_PyDict_GetItemProxy(st->array_cache, key, &result) != 0) { // found or error Py_DECREF(key); return result; @@ -4771,7 +5137,6 @@ PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length) sprintf(name, "%.200s_Array_%ld", ((PyTypeObject *)itemtype)->tp_name, (long)length); #endif - ctypes_state *st = GLOBAL_STATE(); result = PyObject_CallFunction((PyObject *)st->PyCArrayType_Type, "s(O){s:n,s:O}", name, @@ -4785,7 +5150,7 @@ PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length) Py_DECREF(key); return NULL; } - if (-1 == PyDict_SetItemProxy(cache, key, result)) { + if (PyDict_SetItemProxy(st, st->array_cache, key, result) < 0) { Py_DECREF(key); Py_DECREF(result); return NULL; @@ -4800,20 +5165,34 @@ PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length) Simple_Type */ +/*[clinic input] +class _ctypes.Simple "PyObject *" "clinic_state()->Simple_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=016c476c7aa8b8a8]*/ + static int -Simple_set_value(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored)) +Simple_set_value(PyObject *op, PyObject *value, void *Py_UNUSED(ignored)) { PyObject *result; - StgDictObject *dict = PyObject_stgdict((PyObject *)self); + CDataObject *self = _CDataObject_CAST(op); if (value == NULL) { PyErr_SetString(PyExc_TypeError, "can't delete attribute"); return -1; } - assert(dict); /* Cannot be NULL for CDataObject instances */ - assert(dict->setfunc); - result = dict->setfunc(self->b_ptr, value, dict->size); + + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *info; + if (PyStgInfo_FromObject(st, op, &info) < 0) { + return -1; + } + assert(info); /* Cannot be NULL for CDataObject instances */ + assert(info->setfunc); + + LOCK_PTR(self); + result = info->setfunc(self->b_ptr, value, info->size); + UNLOCK_PTR(self); if (!result) return -1; @@ -4822,7 +5201,7 @@ Simple_set_value(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored)) } static int -Simple_init(CDataObject *self, PyObject *args, PyObject *kw) +Simple_init(PyObject *self, PyObject *args, PyObject *kw) { PyObject *value = NULL; if (!PyArg_UnpackTuple(args, "__init__", 0, 1, &value)) @@ -4833,60 +5212,71 @@ Simple_init(CDataObject *self, PyObject *args, PyObject *kw) } static PyObject * -Simple_get_value(CDataObject *self, void *Py_UNUSED(ignored)) +Simple_get_value(PyObject *op, void *Py_UNUSED(ignored)) { - StgDictObject *dict; - dict = PyObject_stgdict((PyObject *)self); - assert(dict); /* Cannot be NULL for CDataObject instances */ - assert(dict->getfunc); - return dict->getfunc(self->b_ptr, self->b_size); + CDataObject *self = _CDataObject_CAST(op); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *info; + if (PyStgInfo_FromObject(st, op, &info) < 0) { + return NULL; + } + assert(info); /* Cannot be NULL for CDataObject instances */ + assert(info->getfunc); + PyObject *res; + LOCK_PTR(self); + res = info->getfunc(self->b_ptr, self->b_size); + UNLOCK_PTR(self); + return res; } static PyGetSetDef Simple_getsets[] = { - { "value", (getter)Simple_get_value, (setter)Simple_set_value, + { "value", Simple_get_value, Simple_set_value, "current value", NULL }, { NULL, NULL } }; +/*[clinic input] +_ctypes.Simple.__ctypes_from_outparam__ as Simple_from_outparm + + self: self + cls: defining_class + / +[clinic start generated code]*/ + static PyObject * -Simple_from_outparm(PyObject *self, PyObject *args) +Simple_from_outparm_impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=6c61d90da8aa9b4f input=0f362803fb4629d5]*/ { - if (_ctypes_simple_instance((PyObject *)Py_TYPE(self))) { + ctypes_state *st = get_module_state_by_class(cls); + if (_ctypes_simple_instance(st, (PyObject *)Py_TYPE(self))) { return Py_NewRef(self); } - /* call stgdict->getfunc */ - return Simple_get_value((CDataObject *)self, NULL); + /* call stginfo->getfunc */ + return Simple_get_value(self, NULL); } static PyMethodDef Simple_methods[] = { - { "__ctypes_from_outparam__", Simple_from_outparm, METH_NOARGS, }, + SIMPLE_FROM_OUTPARM_METHODDEF { NULL, NULL }, }; -static int Simple_bool(CDataObject *self) +static int +Simple_bool(PyObject *op) { - return memcmp(self->b_ptr, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", self->b_size); + int cmp; + CDataObject *self = _CDataObject_CAST(op); + LOCK_PTR(self); + cmp = memcmp(self->b_ptr, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", self->b_size); + UNLOCK_PTR(self); + return cmp; } -static PyNumberMethods Simple_as_number = { - 0, /* nb_add */ - 0, /* nb_subtract */ - 0, /* nb_multiply */ - 0, /* nb_remainder */ - 0, /* nb_divmod */ - 0, /* nb_power */ - 0, /* nb_negative */ - 0, /* nb_positive */ - 0, /* nb_absolute */ - (inquiry)Simple_bool, /* nb_bool */ -}; - /* "%s(%s)" % (self.__class__.__name__, self.value) */ static PyObject * -Simple_repr(CDataObject *self) +Simple_repr(PyObject *self) { PyObject *val, *result; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); if (Py_TYPE(self)->tp_base != st->Simple_Type) { return PyUnicode_FromFormat("<%s object at %p>", @@ -4903,48 +5293,26 @@ Simple_repr(CDataObject *self) return result; } -static PyTypeObject Simple_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes._SimpleCData", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)&Simple_repr, /* tp_repr */ - &Simple_as_number, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("XXX to be provided"), /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - Simple_methods, /* tp_methods */ - 0, /* tp_members */ - Simple_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)Simple_init, /* tp_init */ - 0, /* tp_alloc */ - GenericPyCData_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycsimple_slots[] = { + {Py_tp_repr, &Simple_repr}, + {Py_tp_doc, PyDoc_STR("XXX to be provided")}, + {Py_tp_methods, Simple_methods}, + {Py_tp_getset, Simple_getsets}, + {Py_tp_init, Simple_init}, + {Py_tp_new, GenericPyCData_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {Py_nb_bool, Simple_bool}, + {0, NULL}, +}; + +static PyType_Spec pycsimple_spec = { + .name = "_ctypes._SimpleCData", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycsimple_slots, }; + /******************************************************************/ /* PyCPointer_Type @@ -4952,41 +5320,48 @@ static PyTypeObject Simple_Type = { static PyObject * Pointer_item(PyObject *myself, Py_ssize_t index) { - CDataObject *self = (CDataObject *)myself; + CDataObject *self = _CDataObject_CAST(myself); Py_ssize_t size; Py_ssize_t offset; - StgDictObject *stgdict, *itemdict; PyObject *proto; + void *deref = locked_deref(self); - if (*(void **)self->b_ptr == NULL) { + if (deref == NULL) { PyErr_SetString(PyExc_ValueError, "NULL pointer access"); return NULL; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for pointer object instances */ + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(myself))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, myself, &stginfo) < 0) { + return NULL; + } + assert(stginfo); /* Cannot be NULL for pointer object instances */ - proto = stgdict->proto; + proto = stginfo->proto; assert(proto); - itemdict = PyType_stgdict(proto); - assert(itemdict); /* proto is the item type of the pointer, a ctypes + + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, proto, &iteminfo) < 0) { + return NULL; + } + assert(iteminfo); /* proto is the item type of the pointer, a ctypes type, so this cannot be NULL */ - size = itemdict->size; - offset = index * itemdict->size; + size = iteminfo->size; + offset = index * iteminfo->size; - return PyCData_get(proto, stgdict->getfunc, (PyObject *)self, - index, size, (*(char **)self->b_ptr) + offset); + return PyCData_get(st, proto, stginfo->getfunc, myself, + index, size, (char *)((char *)deref + offset)); } static int Pointer_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) { - CDataObject *self = (CDataObject *)myself; + CDataObject *self = _CDataObject_CAST(myself); Py_ssize_t size; Py_ssize_t offset; - StgDictObject *stgdict, *itemdict; PyObject *proto; if (value == NULL) { @@ -4995,78 +5370,91 @@ Pointer_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) return -1; } - if (*(void **)self->b_ptr == NULL) { + void *deref = locked_deref(self); + if (deref == NULL) { PyErr_SetString(PyExc_ValueError, "NULL pointer access"); return -1; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for pointer instances */ + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(myself))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, myself, &stginfo) < 0) { + return -1; + } + assert(stginfo); /* Cannot be NULL for pointer instances */ - proto = stgdict->proto; + proto = stginfo->proto; assert(proto); - itemdict = PyType_stgdict(proto); - assert(itemdict); /* Cannot be NULL because the itemtype of a pointer + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, proto, &iteminfo) < 0) { + return -1; + } + assert(iteminfo); /* Cannot be NULL because the itemtype of a pointer is always a ctypes type */ - size = itemdict->size; - offset = index * itemdict->size; + size = iteminfo->size; + offset = index * iteminfo->size; - return PyCData_set((PyObject *)self, proto, stgdict->setfunc, value, - index, size, (*(char **)self->b_ptr) + offset); + return PyCData_set(st, myself, proto, stginfo->setfunc, value, + index, size, ((char *)deref + offset)); } static PyObject * -Pointer_get_contents(CDataObject *self, void *closure) +Pointer_get_contents(PyObject *self, void *closure) { - StgDictObject *stgdict; - - if (*(void **)self->b_ptr == NULL) { + void *deref = locked_deref(_CDataObject_CAST(self)); + if (deref == NULL) { PyErr_SetString(PyExc_ValueError, "NULL pointer access"); return NULL; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for pointer instances */ - return PyCData_FromBaseObj(stgdict->proto, - (PyObject *)self, 0, - *(void **)self->b_ptr); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, self, &stginfo) < 0) { + return NULL; + } + assert(stginfo); /* Cannot be NULL for pointer instances */ + + return PyCData_FromBaseObj(st, stginfo->proto, self, 0, deref); } static int -Pointer_set_contents(CDataObject *self, PyObject *value, void *closure) +Pointer_set_contents(PyObject *op, PyObject *value, void *closure) { - StgDictObject *stgdict; CDataObject *dst; PyObject *keep; + CDataObject *self = _CDataObject_CAST(op); if (value == NULL) { PyErr_SetString(PyExc_TypeError, "Pointer does not support item deletion"); return -1; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for pointer instances */ - assert(stgdict->proto); - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, op, &stginfo) < 0) { + return -1; + } + assert(stginfo); /* Cannot be NULL for pointer instances */ + assert(stginfo->proto); if (!CDataObject_Check(st, value)) { - int res = PyObject_IsInstance(value, stgdict->proto); + int res = PyObject_IsInstance(value, stginfo->proto); if (res == -1) return -1; if (!res) { PyErr_Format(PyExc_TypeError, "expected %s instead of %s", - ((PyTypeObject *)(stgdict->proto))->tp_name, + ((PyTypeObject *)(stginfo->proto))->tp_name, Py_TYPE(value)->tp_name); return -1; } } dst = (CDataObject *)value; - *(void **)self->b_ptr = dst->b_ptr; + locked_deref_assign(self, dst->b_ptr); /* A Pointer instance must keep the value it points to alive. So, a @@ -5086,17 +5474,15 @@ Pointer_set_contents(CDataObject *self, PyObject *value, void *closure) } static PyGetSetDef Pointer_getsets[] = { - { "contents", (getter)Pointer_get_contents, - (setter)Pointer_set_contents, + { "contents", Pointer_get_contents, Pointer_set_contents, "the object this pointer points to (read-write)", NULL }, { NULL, NULL } }; static int -Pointer_init(CDataObject *self, PyObject *args, PyObject *kw) +Pointer_init(PyObject *self, PyObject *args, PyObject *kw) { PyObject *value = NULL; - if (!PyArg_UnpackTuple(args, "POINTER", 0, 1, &value)) return -1; if (value == NULL) @@ -5107,19 +5493,23 @@ Pointer_init(CDataObject *self, PyObject *args, PyObject *kw) static PyObject * Pointer_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - StgDictObject *dict = PyType_stgdict((PyObject *)type); - if (!dict || !dict->proto) { + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return NULL; + } + if (!info || !info->proto) { PyErr_SetString(PyExc_TypeError, "Cannot create instance: has no _type_"); return NULL; } - return GenericPyCData_new(type, args, kw); + return generic_pycdata_new(st, type, args, kw); } static PyObject * Pointer_subscript(PyObject *myself, PyObject *item) { - CDataObject *self = (CDataObject *)myself; + CDataObject *self = _CDataObject_CAST(myself); if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) @@ -5130,7 +5520,6 @@ Pointer_subscript(PyObject *myself, PyObject *item) PySliceObject *slice = (PySliceObject *)item; Py_ssize_t start, stop, step; PyObject *np; - StgDictObject *stgdict, *itemdict; PyObject *proto; Py_ssize_t i, len; size_t cur; @@ -5184,48 +5573,67 @@ Pointer_subscript(PyObject *myself, PyObject *item) else len = (stop - start + 1) / step + 1; - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for pointer instances */ - proto = stgdict->proto; + ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(myself))); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, myself, &stginfo) < 0) { + return NULL; + } + assert(stginfo); /* Cannot be NULL for pointer instances */ + proto = stginfo->proto; assert(proto); - itemdict = PyType_stgdict(proto); - assert(itemdict); - if (itemdict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { - char *ptr = *(char **)self->b_ptr; + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, proto, &iteminfo) < 0) { + return NULL; + } + assert(iteminfo); + if (iteminfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { + char *ptr = locked_deref(self); char *dest; if (len <= 0) - return PyBytes_FromStringAndSize("", 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); if (step == 1) { - return PyBytes_FromStringAndSize(ptr + start, - len); + PyObject *res; + LOCK_PTR(self); + res = PyBytes_FromStringAndSize(ptr + start, + len); + UNLOCK_PTR(self); + return res; } dest = (char *)PyMem_Malloc(len); if (dest == NULL) return PyErr_NoMemory(); + LOCK_PTR(self); for (cur = start, i = 0; i < len; cur += step, i++) { dest[i] = ptr[cur]; } + UNLOCK_PTR(self); np = PyBytes_FromStringAndSize(dest, len); PyMem_Free(dest); return np; } - if (itemdict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { - wchar_t *ptr = *(wchar_t **)self->b_ptr; + if (iteminfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) { + wchar_t *ptr = locked_deref(self); wchar_t *dest; if (len <= 0) - return PyUnicode_New(0, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (step == 1) { - return PyUnicode_FromWideChar(ptr + start, - len); + PyObject *res; + LOCK_PTR(self); + res = PyUnicode_FromWideChar(ptr + start, + len); + UNLOCK_PTR(self); + return res; } dest = PyMem_New(wchar_t, len); if (dest == NULL) return PyErr_NoMemory(); + LOCK_PTR(self); for (cur = start, i = 0; i < len; cur += step, i++) { dest[i] = ptr[cur]; } + UNLOCK_PTR(self); np = PyUnicode_FromWideChar(dest, len); PyMem_Free(dest); return np; @@ -5248,86 +5656,31 @@ Pointer_subscript(PyObject *myself, PyObject *item) } } -static PySequenceMethods Pointer_as_sequence = { - 0, /* inquiry sq_length; */ - 0, /* binaryfunc sq_concat; */ - 0, /* intargfunc sq_repeat; */ - Pointer_item, /* intargfunc sq_item; */ - 0, /* intintargfunc sq_slice; */ - Pointer_ass_item, /* intobjargproc sq_ass_item; */ - 0, /* intintobjargproc sq_ass_slice; */ - 0, /* objobjproc sq_contains; */ - /* Added in release 2.0 */ - 0, /* binaryfunc sq_inplace_concat; */ - 0, /* intargfunc sq_inplace_repeat; */ -}; - -static PyMappingMethods Pointer_as_mapping = { - 0, - Pointer_subscript, -}; - static int -Pointer_bool(CDataObject *self) -{ - return (*(void **)self->b_ptr != NULL); -} - -static PyNumberMethods Pointer_as_number = { - 0, /* nb_add */ - 0, /* nb_subtract */ - 0, /* nb_multiply */ - 0, /* nb_remainder */ - 0, /* nb_divmod */ - 0, /* nb_power */ - 0, /* nb_negative */ - 0, /* nb_positive */ - 0, /* nb_absolute */ - (inquiry)Pointer_bool, /* nb_bool */ -}; +Pointer_bool(PyObject *self) +{ + return locked_deref(_CDataObject_CAST(self)) != NULL; +} -PyTypeObject PyCPointer_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes._Pointer", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - &Pointer_as_number, /* tp_as_number */ - &Pointer_as_sequence, /* tp_as_sequence */ - &Pointer_as_mapping, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("XXX to be provided"), /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - Pointer_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)Pointer_init, /* tp_init */ - 0, /* tp_alloc */ - Pointer_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycpointer_slots[] = { + {Py_tp_doc, (void *)PyDoc_STR("XXX to be provided")}, + {Py_tp_getset, Pointer_getsets}, + {Py_tp_init, Pointer_init}, + {Py_tp_new, Pointer_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {Py_nb_bool, Pointer_bool}, + {Py_mp_subscript, Pointer_subscript}, + {Py_sq_item, Pointer_item}, + {Py_sq_ass_item, Pointer_ass_item}, + {0, NULL}, }; +static PyType_Spec pycpointer_spec = { + .name = "_ctypes._Pointer", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycpointer_slots, +}; /******************************************************************/ /* @@ -5431,21 +5784,21 @@ string_at(const char *ptr, int size) } static int -cast_check_pointertype(PyObject *arg) +cast_check_pointertype(ctypes_state *st, PyObject *arg) { - StgDictObject *dict; - ctypes_state *st = GLOBAL_STATE(); - if (PyCPointerTypeObject_Check(st, arg)) { return 1; } if (PyCFuncPtrTypeObject_Check(st, arg)) { return 1; } - dict = PyType_stgdict(arg); - if (dict != NULL && dict->proto != NULL) { - if (PyUnicode_Check(dict->proto) - && (strchr("sPzUZXO", PyUnicode_AsUTF8(dict->proto)[0]))) { + StgInfo *info; + if (PyStgInfo_FromType(st, arg, &info) < 0) { + return 0; + } + if (info != NULL && info->proto != NULL) { + if (PyUnicode_Check(info->proto) + && (strchr("sPzUZXO", PyUnicode_AsUTF8(info->proto)[0]))) { /* simple pointer types, c_void_p, c_wchar_p, BSTR, ... */ return 1; } @@ -5461,9 +5814,18 @@ cast_check_pointertype(PyObject *arg) static PyObject * cast(void *ptr, PyObject *src, PyObject *ctype) { + PyObject *mod = PyType_GetModuleByDef(Py_TYPE(ctype), &_ctypesmodule); + if (!mod) { + PyErr_SetString(PyExc_TypeError, + "cast() argument 2 must be a pointer type"); + return NULL; + } + ctypes_state *st = get_module_state(mod); + CDataObject *result; - if (0 == cast_check_pointertype(ctype)) + if (cast_check_pointertype(st, ctype) == 0) { return NULL; + } result = (CDataObject *)_PyObject_CallNoArgs(ctype); if (result == NULL) return NULL; @@ -5474,7 +5836,6 @@ cast(void *ptr, PyObject *src, PyObject *ctype) It must certainly contain the source objects one. It must contain the source object itself. */ - ctypes_state *st = GLOBAL_STATE(); if (CDataObject_Check(st, src)) { CDataObject *obj = (CDataObject *)src; CDataObject *container; @@ -5506,7 +5867,7 @@ cast(void *ptr, PyObject *src, PyObject *ctype) } } /* Should we assert that result is a pointer type? */ - memcpy(result->b_ptr, &ptr, sizeof(void *)); + locked_memcpy_to(result, &ptr, sizeof(void *)); return (PyObject *)result; failed: @@ -5527,15 +5888,22 @@ wstring_at(const wchar_t *ptr, int size) return PyUnicode_FromWideChar(ptr, ssize); } - -static struct PyModuleDef _ctypesmodule = { - PyModuleDef_HEAD_INIT, - .m_name = "_ctypes", - .m_doc = _ctypes__doc__, - .m_size = -1, - .m_methods = _ctypes_module_methods, -}; - +static PyObject * +memoryview_at(void *ptr, Py_ssize_t size, int readonly) +{ + if (PySys_Audit("ctypes.memoryview_at", "nni", + (Py_ssize_t)ptr, size, readonly) < 0) { + return NULL; + } + if (size < 0) { + PyErr_Format(PyExc_ValueError, + "memoryview_at: size is negative (or overflowed): %zd", + size); + return NULL; + } + return PyMemoryView_FromMemory(ptr, size, + readonly ? PyBUF_READ : PyBUF_WRITE); +} static int _ctypes_add_types(PyObject *mod) @@ -5544,26 +5912,8 @@ _ctypes_add_types(PyObject *mod) if (PyType_Ready(TYPE) < 0) { \ return -1; \ } - -#define TYPE_READY_BASE(TYPE_EXPR, TP_BASE) \ - do { \ - PyTypeObject *type = (TYPE_EXPR); \ - type->tp_base = (TP_BASE); \ - TYPE_READY(type); \ - } while (0) - -#define MOD_ADD_TYPE(TYPE_EXPR, TP_TYPE, TP_BASE) \ - do { \ - PyTypeObject *type = (TYPE_EXPR); \ - Py_SET_TYPE(type, TP_TYPE); \ - type->tp_base = TP_BASE; \ - if (PyModule_AddType(mod, type) < 0) { \ - return -1; \ - } \ - } while (0) - -#define CREATE_TYPE(MOD, TP, SPEC, BASE) do { \ - PyObject *type = PyType_FromMetaclass(NULL, MOD, SPEC, \ +#define CREATE_TYPE(TP, SPEC, META, BASE) do { \ + PyObject *type = PyType_FromMetaclass(META, mod, SPEC, \ (PyObject *)BASE); \ if (type == NULL) { \ return -1; \ @@ -5571,67 +5921,82 @@ _ctypes_add_types(PyObject *mod) TP = (PyTypeObject *)type; \ } while (0) - ctypes_state *st = GLOBAL_STATE(); +#define MOD_ADD_TYPE(TP, SPEC, META, BASE) do { \ + CREATE_TYPE(TP, SPEC, META, BASE); \ + if (PyModule_AddType(mod, (PyTypeObject *)(TP)) < 0) { \ + return -1; \ + } \ +} while (0) + + ctypes_state *st = get_module_state(mod); /* Note: ob_type is the metatype (the 'type'), defaults to PyType_Type, tp_base is the base type, defaults to 'object' aka PyBaseObject_Type. */ - CREATE_TYPE(mod, st->PyCArg_Type, &carg_spec, NULL); - CREATE_TYPE(mod, st->PyCThunk_Type, &cthunk_spec, NULL); - TYPE_READY(st->PyCData_Type); - /* StgDict is derived from PyDict_Type */ - TYPE_READY_BASE(st->PyCStgDict_Type, &PyDict_Type); + CREATE_TYPE(st->PyCArg_Type, &carg_spec, NULL, NULL); + CREATE_TYPE(st->PyCThunk_Type, &cthunk_spec, NULL, NULL); + CREATE_TYPE(st->PyCData_Type, &pycdata_spec, NULL, NULL); + + // Common Metaclass + CREATE_TYPE(st->PyCType_Type, &pyctype_type_spec, + NULL, &PyType_Type); /************************************************* * * Metaclasses */ - CREATE_TYPE(mod, st->PyCStructType_Type, &pycstruct_type_spec, - &PyType_Type); - CREATE_TYPE(mod, st->UnionType_Type, &union_type_spec, &PyType_Type); - CREATE_TYPE(mod, st->PyCPointerType_Type, &pycpointer_type_spec, - &PyType_Type); - CREATE_TYPE(mod, st->PyCArrayType_Type, &pycarray_type_spec, - &PyType_Type); - CREATE_TYPE(mod, st->PyCSimpleType_Type, &pycsimple_type_spec, - &PyType_Type); - CREATE_TYPE(mod, st->PyCFuncPtrType_Type, &pycfuncptr_type_spec, - &PyType_Type); + CREATE_TYPE(st->PyCStructType_Type, &pycstruct_type_spec, + NULL, st->PyCType_Type); + CREATE_TYPE(st->UnionType_Type, &union_type_spec, + NULL, st->PyCType_Type); + CREATE_TYPE(st->PyCPointerType_Type, &pycpointer_type_spec, + NULL, st->PyCType_Type); + CREATE_TYPE(st->PyCArrayType_Type, &pycarray_type_spec, + NULL, st->PyCType_Type); + CREATE_TYPE(st->PyCSimpleType_Type, &pycsimple_type_spec, + NULL, st->PyCType_Type); + CREATE_TYPE(st->PyCFuncPtrType_Type, &pycfuncptr_type_spec, + NULL, st->PyCType_Type); /************************************************* * * Classes using a custom metaclass */ - MOD_ADD_TYPE(st->Struct_Type, st->PyCStructType_Type, st->PyCData_Type); - MOD_ADD_TYPE(st->Union_Type, st->UnionType_Type, st->PyCData_Type); - MOD_ADD_TYPE(st->PyCPointer_Type, st->PyCPointerType_Type, st->PyCData_Type); - MOD_ADD_TYPE(st->PyCArray_Type, st->PyCArrayType_Type, st->PyCData_Type); - MOD_ADD_TYPE(st->Simple_Type, st->PyCSimpleType_Type, st->PyCData_Type); - MOD_ADD_TYPE(st->PyCFuncPtr_Type, st->PyCFuncPtrType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->Struct_Type, &pycstruct_spec, + st->PyCStructType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->Union_Type, &pycunion_spec, + st->UnionType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->PyCPointer_Type, &pycpointer_spec, + st->PyCPointerType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->PyCArray_Type, &pycarray_spec, + st->PyCArrayType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->Simple_Type, &pycsimple_spec, + st->PyCSimpleType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->PyCFuncPtr_Type, &pycfuncptr_spec, + st->PyCFuncPtrType_Type, st->PyCData_Type); /************************************************* * * Simple classes */ - CREATE_TYPE(mod, st->PyCField_Type, &cfield_spec, NULL); + MOD_ADD_TYPE(st->PyCField_Type, &cfield_spec, NULL, NULL); /************************************************* * * Other stuff */ - CREATE_TYPE(mod, st->DictRemover_Type, &dictremover_spec, NULL); - CREATE_TYPE(mod, st->StructParam_Type, &structparam_spec, NULL); + CREATE_TYPE(st->DictRemover_Type, &dictremover_spec, NULL, NULL); + CREATE_TYPE(st->StructParam_Type, &structparam_spec, NULL, NULL); #ifdef MS_WIN32 - CREATE_TYPE(mod, st->PyComError_Type, &comerror_spec, PyExc_Exception); + CREATE_TYPE(st->PyComError_Type, &comerror_spec, NULL, PyExc_Exception); #endif #undef TYPE_READY -#undef TYPE_READY_BASE #undef MOD_ADD_TYPE #undef CREATE_TYPE return 0; @@ -5648,10 +6013,10 @@ _ctypes_add_objects(PyObject *mod) } \ } while (0) - MOD_ADD("_pointer_type_cache", Py_NewRef(_ctypes_ptrtype_cache)); + ctypes_state *st = get_module_state(mod); + MOD_ADD("_pointer_type_cache", Py_NewRef(st->_ctypes_ptrtype_cache)); #ifdef MS_WIN32 - ctypes_state *st = GLOBAL_STATE(); MOD_ADD("COMError", Py_NewRef(st->PyComError_Type)); MOD_ADD("FUNCFLAG_HRESULT", PyLong_FromLong(FUNCFLAG_HRESULT)); MOD_ADD("FUNCFLAG_STDCALL", PyLong_FromLong(FUNCFLAG_STDCALL)); @@ -5667,6 +6032,7 @@ _ctypes_add_objects(PyObject *mod) MOD_ADD("_string_at_addr", PyLong_FromVoidPtr(string_at)); MOD_ADD("_cast_addr", PyLong_FromVoidPtr(cast)); MOD_ADD("_wstring_at_addr", PyLong_FromVoidPtr(wstring_at)); + MOD_ADD("_memoryview_at_addr", PyLong_FromVoidPtr(memoryview_at)); /* If RTLD_LOCAL is not defined (Windows!), set it to zero. */ #if !HAVE_DECL_RTLD_LOCAL @@ -5681,7 +6047,7 @@ _ctypes_add_objects(PyObject *mod) MOD_ADD("RTLD_LOCAL", PyLong_FromLong(RTLD_LOCAL)); MOD_ADD("RTLD_GLOBAL", PyLong_FromLong(RTLD_GLOBAL)); MOD_ADD("CTYPES_MAX_ARGCOUNT", PyLong_FromLong(CTYPES_MAX_ARGCOUNT)); - MOD_ADD("ArgumentError", Py_NewRef(PyExc_ArgError)); + MOD_ADD("ArgumentError", Py_NewRef(st->PyExc_ArgError)); MOD_ADD("SIZEOF_TIME_T", PyLong_FromSsize_t(SIZEOF_TIME_T)); return 0; #undef MOD_ADD @@ -5691,18 +6057,19 @@ _ctypes_add_objects(PyObject *mod) static int _ctypes_mod_exec(PyObject *mod) { - _unpickle = PyObject_GetAttrString(mod, "_unpickle"); - if (_unpickle == NULL) { + ctypes_state *st = get_module_state(mod); + st->_unpickle = PyObject_GetAttrString(mod, "_unpickle"); + if (st->_unpickle == NULL) { return -1; } - _ctypes_ptrtype_cache = PyDict_New(); - if (_ctypes_ptrtype_cache == NULL) { + st->_ctypes_ptrtype_cache = PyDict_New(); + if (st->_ctypes_ptrtype_cache == NULL) { return -1; } - PyExc_ArgError = PyErr_NewException("ctypes.ArgumentError", NULL, NULL); - if (!PyExc_ArgError) { + st->PyExc_ArgError = PyErr_NewException("ctypes.ArgumentError", NULL, NULL); + if (!st->PyExc_ArgError) { return -1; } @@ -5717,19 +6084,105 @@ _ctypes_mod_exec(PyObject *mod) } +static int +module_traverse(PyObject *module, visitproc visit, void *arg) { + ctypes_state *st = get_module_state(module); + Py_VISIT(st->_ctypes_ptrtype_cache); + Py_VISIT(st->_unpickle); + Py_VISIT(st->array_cache); + Py_VISIT(st->error_object_name); + Py_VISIT(st->PyExc_ArgError); + Py_VISIT(st->swapped_suffix); + + Py_VISIT(st->DictRemover_Type); + Py_VISIT(st->PyCArg_Type); + Py_VISIT(st->PyCField_Type); + Py_VISIT(st->PyCThunk_Type); + Py_VISIT(st->StructParam_Type); + Py_VISIT(st->PyCStructType_Type); + Py_VISIT(st->UnionType_Type); + Py_VISIT(st->PyCPointerType_Type); + Py_VISIT(st->PyCArrayType_Type); + Py_VISIT(st->PyCSimpleType_Type); + Py_VISIT(st->PyCFuncPtrType_Type); + Py_VISIT(st->PyCData_Type); + Py_VISIT(st->Struct_Type); + Py_VISIT(st->Union_Type); + Py_VISIT(st->PyCArray_Type); + Py_VISIT(st->Simple_Type); + Py_VISIT(st->PyCPointer_Type); + Py_VISIT(st->PyCFuncPtr_Type); +#ifdef MS_WIN32 + Py_VISIT(st->PyComError_Type); +#endif + Py_VISIT(st->PyCType_Type); + return 0; +} + +static int +module_clear(PyObject *module) { + ctypes_state *st = get_module_state(module); + Py_CLEAR(st->_ctypes_ptrtype_cache); + Py_CLEAR(st->_unpickle); + Py_CLEAR(st->array_cache); + Py_CLEAR(st->error_object_name); + Py_CLEAR(st->PyExc_ArgError); + Py_CLEAR(st->swapped_suffix); + + Py_CLEAR(st->DictRemover_Type); + Py_CLEAR(st->PyCArg_Type); + Py_CLEAR(st->PyCField_Type); + Py_CLEAR(st->PyCThunk_Type); + Py_CLEAR(st->StructParam_Type); + Py_CLEAR(st->PyCStructType_Type); + Py_CLEAR(st->UnionType_Type); + Py_CLEAR(st->PyCPointerType_Type); + Py_CLEAR(st->PyCArrayType_Type); + Py_CLEAR(st->PyCSimpleType_Type); + Py_CLEAR(st->PyCFuncPtrType_Type); + Py_CLEAR(st->PyCData_Type); + Py_CLEAR(st->Struct_Type); + Py_CLEAR(st->Union_Type); + Py_CLEAR(st->PyCArray_Type); + Py_CLEAR(st->Simple_Type); + Py_CLEAR(st->PyCPointer_Type); + Py_CLEAR(st->PyCFuncPtr_Type); +#ifdef MS_WIN32 + Py_CLEAR(st->PyComError_Type); +#endif + Py_CLEAR(st->PyCType_Type); + return 0; +} + +static void +module_free(void *module) +{ + (void)module_clear((PyObject *)module); +} + +static PyModuleDef_Slot module_slots[] = { + {Py_mod_exec, _ctypes_mod_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL} +}; + +struct PyModuleDef _ctypesmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "_ctypes", + .m_doc = _ctypes__doc__, + .m_size = sizeof(ctypes_state), + .m_methods = _ctypes_module_methods, + .m_slots = module_slots, + .m_traverse = module_traverse, + .m_clear = module_clear, + .m_free = module_free, +}; + PyMODINIT_FUNC PyInit__ctypes(void) { - PyObject *mod = PyModule_Create(&_ctypesmodule); - if (!mod) { - return NULL; - } - - if (_ctypes_mod_exec(mod) < 0) { - Py_DECREF(mod); - return NULL; - } - return mod; + return PyModuleDef_Init(&_ctypesmodule); } /* diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index ecc60417790417..7bac592fd38fb9 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -1,8 +1,7 @@ +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif // gh-85283: On Windows, Py_LIMITED_API requires Py_BUILD_CORE to not attempt @@ -14,6 +13,10 @@ #include +#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) +# include "../_complex.h" // csqrt() +# undef I // for _ctypes_test_generated.c.h +#endif #include // printf() #include // qsort() #include // memset() @@ -23,6 +26,8 @@ #define EXPORT(x) Py_EXPORTED_SYMBOL x +#include "_ctypes_test_generated.c.h" + /* some functions handy for testing */ EXPORT(int) @@ -179,9 +184,9 @@ _testfunc_array_in_struct3B_set_defaults(void) /* * Test3C struct tests the MAX_STRUCT_SIZE 32. Structs containing arrays of up - * to four floating point types are passed in registers on Arm platforms. - * This struct is used for within bounds test on Arm platfroms and for an - * out-of-bounds tests for platfroms where MAX_STRUCT_SIZE is less than 32. + * to four floating-point types are passed in registers on Arm platforms. + * This struct is used for within-bounds tests on Arm platforms and for an + * out-of-bounds test for platforms where MAX_STRUCT_SIZE is less than 32. * See gh-110190. */ typedef struct { @@ -203,9 +208,9 @@ _testfunc_array_in_struct3C_set_defaults(void) /* * Test3D struct tests the MAX_STRUCT_SIZE 64. Structs containing arrays of up - * to eight floating point types are passed in registers on PPC64LE platforms. - * This struct is used for within bounds test on PPC64LE platfroms and for an - * out-of-bounds tests for platfroms where MAX_STRUCT_SIZE is less than 64. + * to eight floating-point types are passed in registers on PPC64LE platforms. + * This struct is used for within bounds test on PPC64LE platforms and for an + * out-of-bounds tests for platforms where MAX_STRUCT_SIZE is less than 64. * See gh-110190. */ typedef struct { @@ -344,6 +349,31 @@ _testfunc_bitfield_by_reference2(Test7 *in) { return result; } +typedef struct{ + uint16_t A ; + uint16_t B : 9; + uint16_t C : 1; + uint16_t D : 1; + uint16_t E : 1; + uint16_t F : 1; + uint16_t G : 3; + uint32_t H : 10; + uint32_t I : 20; + uint32_t J : 2; +} Test9; + +EXPORT(long) +_testfunc_bitfield_by_reference3(Test9 *in, long pos) { + long data[] = {in->A , in->B , in->C , in->D , in->E , in->F , in->G , in->H , in->I , in->J}; + long data_length = (long) (sizeof(data)/sizeof(data[0])); + if(pos < 0) + return -1; + if(pos >= data_length) + return -1; + + return data[pos]; +} + typedef union { signed int A: 1, B:2, C:3, D:2; } Test8; @@ -417,6 +447,23 @@ EXPORT(double) my_sqrt(double a) return sqrt(a); } +#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) +EXPORT(double complex) my_csqrt(double complex a) +{ + return csqrt(a); +} + +EXPORT(float complex) my_csqrtf(float complex a) +{ + return csqrtf(a); +} + +EXPORT(long double complex) my_csqrtl(long double complex a) +{ + return csqrtl(a); +} +#endif + EXPORT(void) my_qsort(void *base, size_t num, size_t width, int(*compare)(const void*, const void*)) { qsort(base, num, width, compare); @@ -705,7 +752,7 @@ struct BITS { */ #ifndef __xlc__ #define SIGNED_SHORT_BITFIELDS - short M: 1, N: 2, O: 3, P: 4, Q: 5, R: 6, S: 7; + signed short M: 1, N: 2, O: 3, P: 4, Q: 5, R: 6, S: 7; #endif }; @@ -735,12 +782,58 @@ EXPORT(int) unpack_bitfields(struct BITS *bits, char name) return 999; } +#if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) +struct +#ifndef MS_WIN32 +__attribute__ ((ms_struct)) +#endif +BITS_msvc +{ + signed int A: 1, B:2, C:3, D:4, E: 5, F: 6, G: 7, H: 8, I: 9; +/* + * The test case needs/uses "signed short" bitfields, but the + * IBM XLC compiler does not support this + */ +#ifndef __xlc__ +#define SIGNED_SHORT_BITFIELDS + signed short M: 1, N: 2, O: 3, P: 4, Q: 5, R: 6, S: 7; +#endif +}; + +EXPORT(int) unpack_bitfields_msvc(struct BITS_msvc *bits, char name) +{ + switch (name) { + case 'A': return bits->A; + case 'B': return bits->B; + case 'C': return bits->C; + case 'D': return bits->D; + case 'E': return bits->E; + case 'F': return bits->F; + case 'G': return bits->G; + case 'H': return bits->H; + case 'I': return bits->I; + +#ifdef SIGNED_SHORT_BITFIELDS + case 'M': return bits->M; + case 'N': return bits->N; + case 'O': return bits->O; + case 'P': return bits->P; + case 'Q': return bits->Q; + case 'R': return bits->R; + case 'S': return bits->S; +#endif + } + return 999; +} +#endif + static PyMethodDef module_methods[] = { /* {"get_last_tf_arg_s", get_last_tf_arg_s, METH_NOARGS}, {"get_last_tf_arg_u", get_last_tf_arg_u, METH_NOARGS}, */ {"func_si", py_func_si, METH_VARARGS}, {"func", py_func, METH_NOARGS}, + {"get_generated_test_data", get_generated_test_data, METH_O}, { NULL, NULL, 0, NULL}, }; @@ -1168,6 +1261,7 @@ _testfunc_pylist_append(PyObject *list, PyObject *item) static struct PyModuleDef_Slot _ctypes_test_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_ctypes/_ctypes_test_generated.c.h b/Modules/_ctypes/_ctypes_test_generated.c.h new file mode 100644 index 00000000000000..d70b33eaa8b515 --- /dev/null +++ b/Modules/_ctypes/_ctypes_test_generated.c.h @@ -0,0 +1,1957 @@ + /* Generated by Lib/test/test_ctypes/test_generated_structs.py */ + + + // Append VALUE to the result. + #define APPEND(ITEM) { \ + PyObject *item = ITEM; \ + if (!item) { \ + Py_DECREF(result); \ + return NULL; \ + } \ + int rv = PyList_Append(result, item); \ + Py_DECREF(item); \ + if (rv < 0) { \ + Py_DECREF(result); \ + return NULL; \ + } \ + } + + // Set TARGET, and append a snapshot of `value`'s + // memory to the result. + #define SET_AND_APPEND(TYPE, TARGET, VAL) { \ + TYPE v = VAL; \ + TARGET = v; \ + APPEND(PyBytes_FromStringAndSize( \ + (char*)&value, sizeof(value))); \ + } + + // Set a field to -1, 1 and 0; append a snapshot of the memory + // after each of the operations. + #define TEST_FIELD(TYPE, TARGET) { \ + SET_AND_APPEND(TYPE, TARGET, -1) \ + SET_AND_APPEND(TYPE, TARGET, 1) \ + SET_AND_APPEND(TYPE, TARGET, 0) \ + } + + #if defined(__GNUC__) || defined(__clang__) + #define GCC_ATTR(X) __attribute__((X)) + #else + #define GCC_ATTR(X) /* */ + #endif + + static PyObject * + get_generated_test_data(PyObject *self, PyObject *name) + { + if (!PyUnicode_Check(name)) { + PyErr_SetString(PyExc_TypeError, "need a string"); + return NULL; + } + PyObject *result = PyList_New(0); + if (!result) { + return NULL; + } + + if (PyUnicode_CompareWithASCIIString(name, "SingleInt") == 0) { + + struct SingleInt { + int a; + }; + struct SingleInt value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("SingleInt")); + APPEND(PyLong_FromLong(sizeof(struct SingleInt))); + APPEND(PyLong_FromLong(_Alignof(struct SingleInt))); + TEST_FIELD(int, value.a); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "SingleInt_Union") == 0) { + + union SingleInt_Union { + int a; + }; + union SingleInt_Union value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("SingleInt_Union")); + APPEND(PyLong_FromLong(sizeof(union SingleInt_Union))); + APPEND(PyLong_FromLong(_Alignof(union SingleInt_Union))); + TEST_FIELD(int, value.a); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "SingleU32") == 0) { + + struct SingleU32 { + uint32_t a; + }; + struct SingleU32 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("SingleU32")); + APPEND(PyLong_FromLong(sizeof(struct SingleU32))); + APPEND(PyLong_FromLong(_Alignof(struct SingleU32))); + TEST_FIELD(uint32_t, value.a); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "SimpleStruct") == 0) { + + struct SimpleStruct { + int32_t x; + int8_t y; + uint16_t z; + }; + struct SimpleStruct value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("SimpleStruct")); + APPEND(PyLong_FromLong(sizeof(struct SimpleStruct))); + APPEND(PyLong_FromLong(_Alignof(struct SimpleStruct))); + TEST_FIELD(int32_t, value.x); + TEST_FIELD(int8_t, value.y); + TEST_FIELD(uint16_t, value.z); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "SimpleUnion") == 0) { + + union SimpleUnion { + int32_t x; + int8_t y; + uint16_t z; + }; + union SimpleUnion value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("SimpleUnion")); + APPEND(PyLong_FromLong(sizeof(union SimpleUnion))); + APPEND(PyLong_FromLong(_Alignof(union SimpleUnion))); + TEST_FIELD(int32_t, value.x); + TEST_FIELD(int8_t, value.y); + TEST_FIELD(uint16_t, value.z); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "ManyTypes") == 0) { + + struct ManyTypes { + int8_t i8; + uint8_t u8; + int16_t i16; + uint16_t u16; + int32_t i32; + uint32_t u32; + int64_t i64; + uint64_t u64; + }; + struct ManyTypes value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("ManyTypes")); + APPEND(PyLong_FromLong(sizeof(struct ManyTypes))); + APPEND(PyLong_FromLong(_Alignof(struct ManyTypes))); + TEST_FIELD(int8_t, value.i8); + TEST_FIELD(uint8_t, value.u8); + TEST_FIELD(int16_t, value.i16); + TEST_FIELD(uint16_t, value.u16); + TEST_FIELD(int32_t, value.i32); + TEST_FIELD(uint32_t, value.u32); + TEST_FIELD(int64_t, value.i64); + TEST_FIELD(uint64_t, value.u64); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "ManyTypesU") == 0) { + + union ManyTypesU { + int8_t i8; + uint8_t u8; + int16_t i16; + uint16_t u16; + int32_t i32; + uint32_t u32; + int64_t i64; + uint64_t u64; + }; + union ManyTypesU value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("ManyTypesU")); + APPEND(PyLong_FromLong(sizeof(union ManyTypesU))); + APPEND(PyLong_FromLong(_Alignof(union ManyTypesU))); + TEST_FIELD(int8_t, value.i8); + TEST_FIELD(uint8_t, value.u8); + TEST_FIELD(int16_t, value.i16); + TEST_FIELD(uint16_t, value.u16); + TEST_FIELD(int32_t, value.i32); + TEST_FIELD(uint32_t, value.u32); + TEST_FIELD(int64_t, value.i64); + TEST_FIELD(uint64_t, value.u64); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Nested") == 0) { + + struct Nested { + struct { + int32_t x; + int8_t y; + uint16_t z; + } a; + union { + int32_t x; + int8_t y; + uint16_t z; + } b; + struct { + int32_t x; + int8_t y; + uint16_t z; + }; + }; + struct Nested value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Nested")); + APPEND(PyLong_FromLong(sizeof(struct Nested))); + APPEND(PyLong_FromLong(_Alignof(struct Nested))); + TEST_FIELD(int32_t, value.a.x); + TEST_FIELD(int8_t, value.a.y); + TEST_FIELD(uint16_t, value.a.z); + TEST_FIELD(int32_t, value.b.x); + TEST_FIELD(int8_t, value.b.y); + TEST_FIELD(uint16_t, value.b.z); + TEST_FIELD(int32_t, value.x); + TEST_FIELD(int8_t, value.y); + TEST_FIELD(uint16_t, value.z); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Packed1") == 0) { + + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Packed1 { + int8_t a; + int64_t b; + }; + #pragma pack(pop) + struct Packed1 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Packed1")); + APPEND(PyLong_FromLong(sizeof(struct Packed1))); + APPEND(PyLong_FromLong(_Alignof(struct Packed1))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int64_t, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Packed2") == 0) { + + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 2) + struct GCC_ATTR(ms_struct) Packed2 { + int8_t a; + int64_t b; + }; + #pragma pack(pop) + struct Packed2 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Packed2")); + APPEND(PyLong_FromLong(sizeof(struct Packed2))); + APPEND(PyLong_FromLong(_Alignof(struct Packed2))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int64_t, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Packed3") == 0) { + + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 4) + struct GCC_ATTR(ms_struct) Packed3 { + int8_t a; + int64_t b; + }; + #pragma pack(pop) + struct Packed3 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Packed3")); + APPEND(PyLong_FromLong(sizeof(struct Packed3))); + APPEND(PyLong_FromLong(_Alignof(struct Packed3))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int64_t, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Packed4") == 0) { + + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 8) + struct GCC_ATTR(ms_struct) Packed4 { + int8_t a; + int64_t b; + }; + #pragma pack(pop) + struct Packed4 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Packed4")); + APPEND(PyLong_FromLong(sizeof(struct Packed4))); + APPEND(PyLong_FromLong(_Alignof(struct Packed4))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int64_t, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "X86_32EdgeCase") == 0) { + + struct X86_32EdgeCase { + int32_t a; + int64_t b; + int32_t c; + }; + struct X86_32EdgeCase value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("X86_32EdgeCase")); + APPEND(PyLong_FromLong(sizeof(struct X86_32EdgeCase))); + APPEND(PyLong_FromLong(_Alignof(struct X86_32EdgeCase))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int32_t, value.c); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "MSBitFieldExample") == 0) { + + struct MSBitFieldExample { + unsigned int a :4; + unsigned int b :5; + unsigned int c :7; + }; + struct MSBitFieldExample value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("MSBitFieldExample")); + APPEND(PyLong_FromLong(sizeof(struct MSBitFieldExample))); + APPEND(PyLong_FromLong(_Alignof(struct MSBitFieldExample))); + TEST_FIELD(unsigned int, value.a); + TEST_FIELD(unsigned int, value.b); + TEST_FIELD(unsigned int, value.c); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "MSStraddlingExample") == 0) { + + struct MSStraddlingExample { + unsigned int first :9; + unsigned int second :7; + unsigned int may_straddle :30; + unsigned int last :18; + }; + struct MSStraddlingExample value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("MSStraddlingExample")); + APPEND(PyLong_FromLong(sizeof(struct MSStraddlingExample))); + APPEND(PyLong_FromLong(_Alignof(struct MSStraddlingExample))); + TEST_FIELD(unsigned int, value.first); + TEST_FIELD(unsigned int, value.second); + TEST_FIELD(unsigned int, value.may_straddle); + TEST_FIELD(unsigned int, value.last); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "IntBits") == 0) { + + struct IntBits { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + }; + struct IntBits value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("IntBits")); + APPEND(PyLong_FromLong(sizeof(struct IntBits))); + APPEND(PyLong_FromLong(_Alignof(struct IntBits))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Bits") == 0) { + + #if (!defined(__xlc__)) + + struct Bits { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + short M :1; + short N :2; + short O :3; + short P :4; + short Q :5; + short R :6; + short S :7; + }; + struct Bits value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Bits")); + APPEND(PyLong_FromLong(sizeof(struct Bits))); + APPEND(PyLong_FromLong(_Alignof(struct Bits))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + TEST_FIELD(short, value.M); + TEST_FIELD(short, value.N); + TEST_FIELD(short, value.O); + TEST_FIELD(short, value.P); + TEST_FIELD(short, value.Q); + TEST_FIELD(short, value.R); + TEST_FIELD(short, value.S); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "IntBits_MSVC") == 0) { + + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + struct GCC_ATTR(ms_struct) IntBits_MSVC { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + }; + struct IntBits_MSVC value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("IntBits_MSVC")); + APPEND(PyLong_FromLong(sizeof(struct IntBits_MSVC))); + APPEND(PyLong_FromLong(_Alignof(struct IntBits_MSVC))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Bits_MSVC") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + struct GCC_ATTR(ms_struct) Bits_MSVC { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + short M :1; + short N :2; + short O :3; + short P :4; + short Q :5; + short R :6; + short S :7; + }; + struct Bits_MSVC value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Bits_MSVC")); + APPEND(PyLong_FromLong(sizeof(struct Bits_MSVC))); + APPEND(PyLong_FromLong(_Alignof(struct Bits_MSVC))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + TEST_FIELD(short, value.M); + TEST_FIELD(short, value.N); + TEST_FIELD(short, value.O); + TEST_FIELD(short, value.P); + TEST_FIELD(short, value.Q); + TEST_FIELD(short, value.R); + TEST_FIELD(short, value.S); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "I64Bits") == 0) { + + #if (!defined(__xlc__)) + + struct I64Bits { + int64_t a :1; + int64_t b :62; + int64_t c :1; + }; + struct I64Bits value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("I64Bits")); + APPEND(PyLong_FromLong(sizeof(struct I64Bits))); + APPEND(PyLong_FromLong(_Alignof(struct I64Bits))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "U64Bits") == 0) { + + #if (!defined(__xlc__)) + + struct U64Bits { + uint64_t a :1; + uint64_t b :62; + uint64_t c :1; + }; + struct U64Bits value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("U64Bits")); + APPEND(PyLong_FromLong(sizeof(struct U64Bits))); + APPEND(PyLong_FromLong(_Alignof(struct U64Bits))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_8 { + int8_t a :3; + int8_t b :3; + int8_t c :1; + }; + struct Struct331_8 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct331_8")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_8))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int8_t, value.b); + TEST_FIELD(int8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_8 { + int8_t a :1; + int8_t b :6; + int8_t c :1; + }; + struct Struct1x1_8 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct1x1_8")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_8))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int8_t, value.b); + TEST_FIELD(int8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_8 { + int8_t a :1; + int8_t full; + int8_t b :6; + int8_t c :1; + }; + struct Struct1nx1_8 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct1nx1_8")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_8))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int8_t, value.full); + TEST_FIELD(int8_t, value.b); + TEST_FIELD(int8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_8 { + int8_t a :3; + int8_t b :6; + int8_t c :6; + }; + struct Struct3xx_8 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct3xx_8")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_8))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int8_t, value.b); + TEST_FIELD(int8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_u8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_u8 { + uint8_t a :3; + uint8_t b :3; + uint8_t c :1; + }; + struct Struct331_u8 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct331_u8")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_u8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_u8))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_u8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_u8 { + uint8_t a :1; + uint8_t b :6; + uint8_t c :1; + }; + struct Struct1x1_u8 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct1x1_u8")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_u8))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_u8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_u8 { + uint8_t a :1; + uint8_t full; + uint8_t b :6; + uint8_t c :1; + }; + struct Struct1nx1_u8 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct1nx1_u8")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_u8))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.full); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_u8") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_u8 { + uint8_t a :3; + uint8_t b :6; + uint8_t c :6; + }; + struct Struct3xx_u8 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct3xx_u8")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_u8))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint8_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_16 { + int16_t a :3; + int16_t b :3; + int16_t c :1; + }; + struct Struct331_16 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct331_16")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_16))); + TEST_FIELD(int16_t, value.a); + TEST_FIELD(int16_t, value.b); + TEST_FIELD(int16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_16 { + int16_t a :1; + int16_t b :14; + int16_t c :1; + }; + struct Struct1x1_16 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct1x1_16")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_16))); + TEST_FIELD(int16_t, value.a); + TEST_FIELD(int16_t, value.b); + TEST_FIELD(int16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_16 { + int16_t a :1; + int16_t full; + int16_t b :14; + int16_t c :1; + }; + struct Struct1nx1_16 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct1nx1_16")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_16))); + TEST_FIELD(int16_t, value.a); + TEST_FIELD(int16_t, value.full); + TEST_FIELD(int16_t, value.b); + TEST_FIELD(int16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_16 { + int16_t a :3; + int16_t b :14; + int16_t c :14; + }; + struct Struct3xx_16 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct3xx_16")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_16))); + TEST_FIELD(int16_t, value.a); + TEST_FIELD(int16_t, value.b); + TEST_FIELD(int16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_u16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_u16 { + uint16_t a :3; + uint16_t b :3; + uint16_t c :1; + }; + struct Struct331_u16 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct331_u16")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_u16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_u16))); + TEST_FIELD(uint16_t, value.a); + TEST_FIELD(uint16_t, value.b); + TEST_FIELD(uint16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_u16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_u16 { + uint16_t a :1; + uint16_t b :14; + uint16_t c :1; + }; + struct Struct1x1_u16 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct1x1_u16")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_u16))); + TEST_FIELD(uint16_t, value.a); + TEST_FIELD(uint16_t, value.b); + TEST_FIELD(uint16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_u16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_u16 { + uint16_t a :1; + uint16_t full; + uint16_t b :14; + uint16_t c :1; + }; + struct Struct1nx1_u16 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct1nx1_u16")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_u16))); + TEST_FIELD(uint16_t, value.a); + TEST_FIELD(uint16_t, value.full); + TEST_FIELD(uint16_t, value.b); + TEST_FIELD(uint16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_u16") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_u16 { + uint16_t a :3; + uint16_t b :14; + uint16_t c :14; + }; + struct Struct3xx_u16 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct3xx_u16")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_u16))); + TEST_FIELD(uint16_t, value.a); + TEST_FIELD(uint16_t, value.b); + TEST_FIELD(uint16_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_32 { + int32_t a :3; + int32_t b :3; + int32_t c :1; + }; + struct Struct331_32 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct331_32")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_32))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int32_t, value.b); + TEST_FIELD(int32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_32 { + int32_t a :1; + int32_t b :30; + int32_t c :1; + }; + struct Struct1x1_32 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct1x1_32")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_32))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int32_t, value.b); + TEST_FIELD(int32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_32 { + int32_t a :1; + int32_t full; + int32_t b :30; + int32_t c :1; + }; + struct Struct1nx1_32 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct1nx1_32")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_32))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int32_t, value.full); + TEST_FIELD(int32_t, value.b); + TEST_FIELD(int32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_32 { + int32_t a :3; + int32_t b :30; + int32_t c :30; + }; + struct Struct3xx_32 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct3xx_32")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_32))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int32_t, value.b); + TEST_FIELD(int32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_u32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_u32 { + uint32_t a :3; + uint32_t b :3; + uint32_t c :1; + }; + struct Struct331_u32 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct331_u32")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_u32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_u32))); + TEST_FIELD(uint32_t, value.a); + TEST_FIELD(uint32_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_u32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_u32 { + uint32_t a :1; + uint32_t b :30; + uint32_t c :1; + }; + struct Struct1x1_u32 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct1x1_u32")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_u32))); + TEST_FIELD(uint32_t, value.a); + TEST_FIELD(uint32_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_u32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_u32 { + uint32_t a :1; + uint32_t full; + uint32_t b :30; + uint32_t c :1; + }; + struct Struct1nx1_u32 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct1nx1_u32")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_u32))); + TEST_FIELD(uint32_t, value.a); + TEST_FIELD(uint32_t, value.full); + TEST_FIELD(uint32_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_u32") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_u32 { + uint32_t a :3; + uint32_t b :30; + uint32_t c :30; + }; + struct Struct3xx_u32 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct3xx_u32")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_u32))); + TEST_FIELD(uint32_t, value.a); + TEST_FIELD(uint32_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_64 { + int64_t a :3; + int64_t b :3; + int64_t c :1; + }; + struct Struct331_64 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct331_64")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_64))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_64 { + int64_t a :1; + int64_t b :62; + int64_t c :1; + }; + struct Struct1x1_64 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct1x1_64")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_64))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_64 { + int64_t a :1; + int64_t full; + int64_t b :62; + int64_t c :1; + }; + struct Struct1nx1_64 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct1nx1_64")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_64))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.full); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_64 { + int64_t a :3; + int64_t b :62; + int64_t c :62; + }; + struct Struct3xx_64 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct3xx_64")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_64))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct331_u64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct331_u64 { + uint64_t a :3; + uint64_t b :3; + uint64_t c :1; + }; + struct Struct331_u64 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct331_u64")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_u64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_u64))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_u64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1x1_u64 { + uint64_t a :1; + uint64_t b :62; + uint64_t c :1; + }; + struct Struct1x1_u64 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct1x1_u64")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_u64))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_u64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct1nx1_u64 { + uint64_t a :1; + uint64_t full; + uint64_t b :62; + uint64_t c :1; + }; + struct Struct1nx1_u64 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct1nx1_u64")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_u64))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.full); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_u64") == 0) { + + #if (!defined(__xlc__)) + + struct Struct3xx_u64 { + uint64_t a :3; + uint64_t b :62; + uint64_t c :62; + }; + struct Struct3xx_u64 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Struct3xx_u64")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_u64))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed1") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed1 { + signed char a :4; + int b :4; + }; + struct Mixed1 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Mixed1")); + APPEND(PyLong_FromLong(sizeof(struct Mixed1))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed1))); + TEST_FIELD(signed char, value.a); + TEST_FIELD(int, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed2") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed2 { + signed char a :4; + int32_t b :32; + }; + struct Mixed2 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Mixed2")); + APPEND(PyLong_FromLong(sizeof(struct Mixed2))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed2))); + TEST_FIELD(signed char, value.a); + TEST_FIELD(int32_t, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed3") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed3 { + signed char a :4; + unsigned char b :4; + }; + struct Mixed3 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Mixed3")); + APPEND(PyLong_FromLong(sizeof(struct Mixed3))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed3))); + TEST_FIELD(signed char, value.a); + TEST_FIELD(unsigned char, value.b); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed4") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed4 { + short a :4; + short b :4; + int c :24; + short d :4; + short e :4; + int f :24; + }; + struct Mixed4 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Mixed4")); + APPEND(PyLong_FromLong(sizeof(struct Mixed4))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed4))); + TEST_FIELD(short, value.a); + TEST_FIELD(short, value.b); + TEST_FIELD(int, value.c); + TEST_FIELD(short, value.d); + TEST_FIELD(short, value.e); + TEST_FIELD(int, value.f); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed5") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed5 { + unsigned int A :1; + unsigned short B :16; + }; + struct Mixed5 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Mixed5")); + APPEND(PyLong_FromLong(sizeof(struct Mixed5))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed5))); + TEST_FIELD(unsigned int, value.A); + TEST_FIELD(unsigned short, value.B); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed6") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed6 { + unsigned long long A :1; + unsigned int B :32; + }; + struct Mixed6 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Mixed6")); + APPEND(PyLong_FromLong(sizeof(struct Mixed6))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed6))); + TEST_FIELD(unsigned long long, value.A); + TEST_FIELD(unsigned int, value.B); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed7") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed7 { + uint32_t A; + uint32_t B :20; + uint64_t C :24; + }; + struct Mixed7 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Mixed7")); + APPEND(PyLong_FromLong(sizeof(struct Mixed7))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed7))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint32_t, value.B); + TEST_FIELD(uint64_t, value.C); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed8_a") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed8_a { + uint32_t A; + uint32_t B :32; + unsigned long long C :1; + }; + struct Mixed8_a value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Mixed8_a")); + APPEND(PyLong_FromLong(sizeof(struct Mixed8_a))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed8_a))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint32_t, value.B); + TEST_FIELD(unsigned long long, value.C); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed8_b") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed8_b { + uint32_t A; + uint32_t B; + unsigned long long C :1; + }; + struct Mixed8_b value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Mixed8_b")); + APPEND(PyLong_FromLong(sizeof(struct Mixed8_b))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed8_b))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint32_t, value.B); + TEST_FIELD(unsigned long long, value.C); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed9") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed9 { + uint8_t A; + uint32_t B :1; + }; + struct Mixed9 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Mixed9")); + APPEND(PyLong_FromLong(sizeof(struct Mixed9))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed9))); + TEST_FIELD(uint8_t, value.A); + TEST_FIELD(uint32_t, value.B); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Mixed10") == 0) { + + #if (!defined(__xlc__)) + + struct Mixed10 { + uint32_t A :1; + uint64_t B :1; + }; + struct Mixed10 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Mixed10")); + APPEND(PyLong_FromLong(sizeof(struct Mixed10))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed10))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint64_t, value.B); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_95496") == 0) { + + #if (!defined(__xlc__)) + + struct Example_gh_95496 { + uint32_t A :1; + uint64_t B :1; + }; + struct Example_gh_95496 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Example_gh_95496")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_95496))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_95496))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint64_t, value.B); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_84039_bad") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_84039_bad { + uint8_t a0 :1; + uint8_t a1 :1; + uint8_t a2 :1; + uint8_t a3 :1; + uint8_t a4 :1; + uint8_t a5 :1; + uint8_t a6 :1; + uint8_t a7 :1; + uint16_t b0 :4; + uint16_t b1 :12; + }; + #pragma pack(pop) + struct Example_gh_84039_bad value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Example_gh_84039_bad")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_84039_bad))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_84039_bad))); + TEST_FIELD(uint8_t, value.a0); + TEST_FIELD(uint8_t, value.a1); + TEST_FIELD(uint8_t, value.a2); + TEST_FIELD(uint8_t, value.a3); + TEST_FIELD(uint8_t, value.a4); + TEST_FIELD(uint8_t, value.a5); + TEST_FIELD(uint8_t, value.a6); + TEST_FIELD(uint8_t, value.a7); + TEST_FIELD(uint16_t, value.b0); + TEST_FIELD(uint16_t, value.b1); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_84039_good_a") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_84039_good_a { + uint8_t a0 :1; + uint8_t a1 :1; + uint8_t a2 :1; + uint8_t a3 :1; + uint8_t a4 :1; + uint8_t a5 :1; + uint8_t a6 :1; + uint8_t a7 :1; + }; + #pragma pack(pop) + struct Example_gh_84039_good_a value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Example_gh_84039_good_a")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_84039_good_a))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_84039_good_a))); + TEST_FIELD(uint8_t, value.a0); + TEST_FIELD(uint8_t, value.a1); + TEST_FIELD(uint8_t, value.a2); + TEST_FIELD(uint8_t, value.a3); + TEST_FIELD(uint8_t, value.a4); + TEST_FIELD(uint8_t, value.a5); + TEST_FIELD(uint8_t, value.a6); + TEST_FIELD(uint8_t, value.a7); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_84039_good") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_84039_good { + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) { + uint8_t a0 :1; + uint8_t a1 :1; + uint8_t a2 :1; + uint8_t a3 :1; + uint8_t a4 :1; + uint8_t a5 :1; + uint8_t a6 :1; + uint8_t a7 :1; + } a; + #pragma pack(pop) + uint16_t b0 :4; + uint16_t b1 :12; + }; + #pragma pack(pop) + struct Example_gh_84039_good value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Example_gh_84039_good")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_84039_good))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_84039_good))); + TEST_FIELD(uint8_t, value.a.a0); + TEST_FIELD(uint8_t, value.a.a1); + TEST_FIELD(uint8_t, value.a.a2); + TEST_FIELD(uint8_t, value.a.a3); + TEST_FIELD(uint8_t, value.a.a4); + TEST_FIELD(uint8_t, value.a.a5); + TEST_FIELD(uint8_t, value.a.a6); + TEST_FIELD(uint8_t, value.a.a7); + TEST_FIELD(uint16_t, value.b0); + TEST_FIELD(uint16_t, value.b1); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_73939") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_73939 { + uint16_t P; + uint16_t L :9; + uint16_t Pro :1; + uint16_t G :1; + uint16_t IB :1; + uint16_t IR :1; + uint16_t R :3; + uint32_t T :10; + uint32_t C :20; + uint32_t R2 :2; + }; + #pragma pack(pop) + struct Example_gh_73939 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Example_gh_73939")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_73939))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_73939))); + TEST_FIELD(uint16_t, value.P); + TEST_FIELD(uint16_t, value.L); + TEST_FIELD(uint16_t, value.Pro); + TEST_FIELD(uint16_t, value.G); + TEST_FIELD(uint16_t, value.IB); + TEST_FIELD(uint16_t, value.IR); + TEST_FIELD(uint16_t, value.R); + TEST_FIELD(uint32_t, value.T); + TEST_FIELD(uint32_t, value.C); + TEST_FIELD(uint32_t, value.R2); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_86098") == 0) { + + #if (!defined(__xlc__)) + + struct Example_gh_86098 { + uint8_t a :8; + uint8_t b :8; + uint32_t c :16; + }; + struct Example_gh_86098 value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Example_gh_86098")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_86098))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_86098))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_86098_pack") == 0) { + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_86098_pack { + uint8_t a :8; + uint8_t b :8; + uint32_t c :16; + }; + #pragma pack(pop) + struct Example_gh_86098_pack value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("Example_gh_86098_pack")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_86098_pack))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_86098_pack))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint32_t, value.c); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "AnonBitfields") == 0) { + + #if (!defined(__xlc__)) + + struct AnonBitfields { + struct { + signed char a :4; + unsigned char b :4; + }; + signed char y; + }; + struct AnonBitfields value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString("AnonBitfields")); + APPEND(PyLong_FromLong(sizeof(struct AnonBitfields))); + APPEND(PyLong_FromLong(_Alignof(struct AnonBitfields))); + TEST_FIELD(signed char, value.a); + TEST_FIELD(unsigned char, value.b); + TEST_FIELD(signed char, value.y); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + return result; + } + + Py_DECREF(result); + PyErr_Format(PyExc_ValueError, "unknown testcase %R", name); + return NULL; + } + + #undef GCC_ATTR + #undef TEST_FIELD + #undef SET_AND_APPEND + #undef APPEND + diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index f70479435915ff..89c0749a093765 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -109,10 +109,14 @@ PrintError(const char *msg, ...) * slower. */ static void -TryAddRef(StgDictObject *dict, CDataObject *obj) +TryAddRef(PyObject *cnv, CDataObject *obj) { IUnknown *punk; - int r = PyDict_Contains((PyObject *)dict, &_Py_ID(_needs_com_addref_)); + PyObject *attrdict = _PyType_GetDict((PyTypeObject *)cnv); + if (!attrdict) { + return; + } + int r = PyDict_Contains(attrdict, &_Py_ID(_needs_com_addref_)); if (r <= 0) { if (r < 0) { PrintError("getting _needs_com_addref_"); @@ -132,7 +136,10 @@ TryAddRef(StgDictObject *dict, CDataObject *obj) * Call the python object with all arguments * */ -static void _CallPythonObject(void *mem, + +// BEWARE: The GIL needs to be held throughout the function +static void _CallPythonObject(ctypes_state *st, + void *mem, ffi_type *restype, SETFUNC setfunc, PyObject *callable, @@ -144,32 +151,34 @@ static void _CallPythonObject(void *mem, Py_ssize_t i = 0, j = 0, nargs = 0; PyObject *error_object = NULL; int *space; - PyGILState_STATE state = PyGILState_Ensure(); assert(PyTuple_Check(converters)); nargs = PyTuple_GET_SIZE(converters); assert(nargs <= CTYPES_MAX_ARGCOUNT); PyObject **args = alloca(nargs * sizeof(PyObject *)); PyObject **cnvs = PySequence_Fast_ITEMS(converters); - ctypes_state *st = GLOBAL_STATE(); for (i = 0; i < nargs; i++) { PyObject *cnv = cnvs[i]; // borrowed ref - StgDictObject *dict; - dict = PyType_stgdict(cnv); - if (dict && dict->getfunc && !_ctypes_simple_instance(cnv)) { - PyObject *v = dict->getfunc(*pArgs, dict->size); + StgInfo *info; + if (PyStgInfo_FromType(st, cnv, &info) < 0) { + goto Done; + } + + if (info && info->getfunc && !_ctypes_simple_instance(st, cnv)) { + PyObject *v = info->getfunc(*pArgs, info->size); if (!v) { PrintError("create argument %zd:\n", i); goto Done; } args[i] = v; /* XXX XXX XX - We have the problem that c_byte or c_short have dict->size of + We have the problem that c_byte or c_short have info->size of 1 resp. 4, but these parameters are pushed as sizeof(int) bytes. BTW, the same problem occurs when they are pushed as parameters */ - } else if (dict) { + } + else if (info) { /* Hm, shouldn't we use PyCData_AtAddress() or something like that instead? */ CDataObject *obj = (CDataObject *)_PyObject_CallNoArgs(cnv); if (!obj) { @@ -181,10 +190,10 @@ static void _CallPythonObject(void *mem, PrintError("unexpected result of create argument %zd:\n", i); goto Done; } - memcpy(obj->b_ptr, *pArgs, dict->size); + memcpy(obj->b_ptr, *pArgs, info->size); args[i] = (PyObject *)obj; #ifdef MS_WIN32 - TryAddRef(dict, obj); + TryAddRef(cnv, obj); #endif } else { PyErr_SetString(PyExc_TypeError, @@ -197,7 +206,7 @@ static void _CallPythonObject(void *mem, } if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) { - error_object = _ctypes_get_errobj(&space); + error_object = _ctypes_get_errobj(st, &space); if (error_object == NULL) goto Done; if (flags & FUNCFLAG_USE_ERRNO) { @@ -255,7 +264,7 @@ static void _CallPythonObject(void *mem, be the result. EXCEPT when restype is py_object - Python itself knows how to manage the refcount of these objects. */ - PyObject *keep = setfunc(mem, result, 0); + PyObject *keep = setfunc(mem, result, restype->size); if (keep == NULL) { /* Could not convert callback result. */ @@ -286,7 +295,6 @@ static void _CallPythonObject(void *mem, for (j = 0; j < i; j++) { Py_DECREF(args[j]); } - PyGILState_Release(state); } static void closure_fcn(ffi_cif *cif, @@ -294,23 +302,28 @@ static void closure_fcn(ffi_cif *cif, void **args, void *userdata) { + PyGILState_STATE state = PyGILState_Ensure(); + CThunkObject *p = (CThunkObject *)userdata; + ctypes_state *st = get_module_state_by_class(Py_TYPE(p)); - _CallPythonObject(resp, + _CallPythonObject(st, + resp, p->ffi_restype, p->setfunc, p->callable, p->converters, p->flags, args); + + PyGILState_Release(state); } -static CThunkObject* CThunkObject_new(Py_ssize_t nargs) +static CThunkObject* CThunkObject_new(ctypes_state *st, Py_ssize_t nargs) { CThunkObject *p; Py_ssize_t i; - ctypes_state *st = GLOBAL_STATE(); p = PyObject_GC_NewVar(CThunkObject, st->PyCThunk_Type, nargs); if (p == NULL) { return NULL; @@ -332,7 +345,8 @@ static CThunkObject* CThunkObject_new(Py_ssize_t nargs) return p; } -CThunkObject *_ctypes_alloc_callback(PyObject *callable, +CThunkObject *_ctypes_alloc_callback(ctypes_state *st, + PyObject *callable, PyObject *converters, PyObject *restype, int flags) @@ -344,14 +358,11 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, assert(PyTuple_Check(converters)); nargs = PyTuple_GET_SIZE(converters); - p = CThunkObject_new(nargs); + p = CThunkObject_new(st, nargs); if (p == NULL) return NULL; -#ifdef Py_DEBUG - ctypes_state *st = GLOBAL_STATE(); assert(CThunk_CheckExact(st, (PyObject *)p)); -#endif p->pcl_write = Py_ffi_closure_alloc(sizeof(ffi_closure), &p->pcl_exec); if (p->pcl_write == NULL) { @@ -363,7 +374,7 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, PyObject **cnvs = PySequence_Fast_ITEMS(converters); for (i = 0; i < nargs; ++i) { PyObject *cnv = cnvs[i]; // borrowed ref - p->atypes[i] = _ctypes_get_ffi_type(cnv); + p->atypes[i] = _ctypes_get_ffi_type(st, cnv); } p->atypes[i] = NULL; @@ -372,14 +383,18 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, p->setfunc = NULL; p->ffi_restype = &ffi_type_void; } else { - StgDictObject *dict = PyType_stgdict(restype); - if (dict == NULL || dict->setfunc == NULL) { + StgInfo *info; + if (PyStgInfo_FromType(st, restype, &info) < 0) { + goto error; + } + + if (info == NULL || info->setfunc == NULL) { PyErr_SetString(PyExc_TypeError, "invalid result type for callback function"); goto error; } - p->setfunc = dict->setfunc; - p->ffi_restype = &dict->ffi_type_pointer; + p->setfunc = info->setfunc; + p->ffi_restype = &info->ffi_type_pointer; } cc = FFI_DEFAULT_ABI; diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 97d1dbaae03d4f..c6b6460126ca90 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -105,6 +105,10 @@ module _ctypes #include "pycore_global_objects.h"// _Py_ID() #include "pycore_traceback.h" // _PyTraceback_Add() +#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) +#include "../_complex.h" // complex +#endif + #include "clinic/callproc.c.h" #define CTYPES_CAPSULE_NAME_PYMEM "_ctypes pymem" @@ -153,22 +157,22 @@ static void pymem_destructor(PyObject *ptr) kept alive in the thread state dictionary as long as the thread itself. */ PyObject * -_ctypes_get_errobj(int **pspace) +_ctypes_get_errobj(ctypes_state *st, int **pspace) { PyObject *dict = PyThreadState_GetDict(); PyObject *errobj; - static PyObject *error_object_name; if (dict == NULL) { PyErr_SetString(PyExc_RuntimeError, "cannot get thread state"); return NULL; } - if (error_object_name == NULL) { - error_object_name = PyUnicode_InternFromString("ctypes.error_object"); - if (error_object_name == NULL) + if (st->error_object_name == NULL) { + st->error_object_name = PyUnicode_InternFromString("ctypes.error_object"); + if (st->error_object_name == NULL) { return NULL; + } } - if (PyDict_GetItemRef(dict, error_object_name, &errobj) < 0) { + if (PyDict_GetItemRef(dict, st->error_object_name, &errobj) < 0) { return NULL; } if (errobj) { @@ -188,8 +192,7 @@ _ctypes_get_errobj(int **pspace) PyMem_Free(space); return NULL; } - if (-1 == PyDict_SetItem(dict, error_object_name, - errobj)) { + if (PyDict_SetItem(dict, st->error_object_name, errobj) < 0) { Py_DECREF(errobj); return NULL; } @@ -202,7 +205,8 @@ static PyObject * get_error_internal(PyObject *self, PyObject *args, int index) { int *space; - PyObject *errobj = _ctypes_get_errobj(&space); + ctypes_state *st = get_module_state(self); + PyObject *errobj = _ctypes_get_errobj(st, &space); PyObject *result; if (errobj == NULL) @@ -222,7 +226,8 @@ set_error_internal(PyObject *self, PyObject *args, int index) if (!PyArg_ParseTuple(args, "i", &new_errno)) { return NULL; } - errobj = _ctypes_get_errobj(&space); + ctypes_state *st = get_module_state(self); + errobj = _ctypes_get_errobj(st, &space); if (errobj == NULL) return NULL; old_errno = space[index]; @@ -473,10 +478,9 @@ check_hresult(PyObject *self, PyObject *args) /**************************************************************/ PyCArgObject * -PyCArgObject_new(void) +PyCArgObject_new(ctypes_state *st) { PyCArgObject *p; - ctypes_state *st = GLOBAL_STATE(); p = PyObject_GC_New(PyCArgObject, st->PyCArg_Type); if (p == NULL) return NULL; @@ -489,27 +493,29 @@ PyCArgObject_new(void) } static int -PyCArg_traverse(PyCArgObject *self, visitproc visit, void *arg) +PyCArg_traverse(PyObject *op, visitproc visit, void *arg) { + PyCArgObject *self = _PyCArgObject_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->obj); return 0; } static int -PyCArg_clear(PyCArgObject *self) +PyCArg_clear(PyObject *op) { + PyCArgObject *self = _PyCArgObject_CAST(op); Py_CLEAR(self->obj); return 0; } static void -PyCArg_dealloc(PyCArgObject *self) +PyCArg_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); (void)PyCArg_clear(self); - tp->tp_free((PyObject *)self); + tp->tp_free(self); Py_DECREF(tp); } @@ -520,8 +526,9 @@ is_literal_char(unsigned char c) } static PyObject * -PyCArg_repr(PyCArgObject *self) +PyCArg_repr(PyObject *op) { + PyCArgObject *self = _PyCArgObject_CAST(op); switch(self->tag) { case 'b': case 'B': @@ -651,6 +658,11 @@ union result { double d; float f; void *p; +#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) + double complex C; + float complex E; + long double complex F; +#endif }; struct argument { @@ -662,17 +674,22 @@ struct argument { /* * Convert a single Python object into a PyCArgObject and return it. */ -static int ConvParam(PyObject *obj, Py_ssize_t index, struct argument *pa) +static int ConvParam(ctypes_state *st, + PyObject *obj, Py_ssize_t index, struct argument *pa) { - StgDictObject *dict; pa->keep = NULL; /* so we cannot forget it later */ - dict = PyObject_stgdict(obj); - if (dict) { + StgInfo *info; + int result = PyStgInfo_FromObject(st, obj, &info); + if (result < 0) { + return -1; + } + if (info) { + assert(info); PyCArgObject *carg; - assert(dict->paramfunc); - /* If it has an stgdict, it is a CDataObject */ - carg = dict->paramfunc((CDataObject *)obj); + assert(info->paramfunc); + /* If it has an stginfo, it is a CDataObject */ + carg = info->paramfunc(st, (CDataObject *)obj); if (carg == NULL) return -1; pa->ffi_type = carg->pffi_type; @@ -681,7 +698,6 @@ static int ConvParam(PyObject *obj, Py_ssize_t index, struct argument *pa) return 0; } - ctypes_state *st = GLOBAL_STATE(); if (PyCArg_CheckExact(st, obj)) { PyCArgObject *carg = (PyCArgObject *)obj; pa->ffi_type = carg->pffi_type; @@ -744,7 +760,7 @@ static int ConvParam(PyObject *obj, Py_ssize_t index, struct argument *pa) */ if (arg) { int result; - result = ConvParam(arg, index, pa); + result = ConvParam(st, arg, index, pa); Py_DECREF(arg); return result; } @@ -778,26 +794,33 @@ int can_return_struct_as_sint64(size_t s) #endif -ffi_type *_ctypes_get_ffi_type(PyObject *obj) +// returns NULL with exception set on error +ffi_type *_ctypes_get_ffi_type(ctypes_state *st, PyObject *obj) { - StgDictObject *dict; - if (obj == NULL) + if (obj == NULL) { return &ffi_type_sint; - dict = PyType_stgdict(obj); - if (dict == NULL) + } + + StgInfo *info; + if (PyStgInfo_FromType(st, obj, &info) < 0) { + return NULL; + } + + if (info == NULL) { return &ffi_type_sint; + } #if defined(MS_WIN32) && !defined(_WIN32_WCE) /* This little trick works correctly with MSVC. It returns small structures in registers */ - if (dict->ffi_type_pointer.type == FFI_TYPE_STRUCT) { - if (can_return_struct_as_int(dict->ffi_type_pointer.size)) + if (info->ffi_type_pointer.type == FFI_TYPE_STRUCT) { + if (can_return_struct_as_int(info->ffi_type_pointer.size)) return &ffi_type_sint32; - else if (can_return_struct_as_sint64 (dict->ffi_type_pointer.size)) + else if (can_return_struct_as_sint64 (info->ffi_type_pointer.size)) return &ffi_type_sint64; } #endif - return &dict->ffi_type_pointer; + return &info->ffi_type_pointer; } @@ -813,7 +836,8 @@ ffi_type *_ctypes_get_ffi_type(PyObject *obj) * * void ffi_call(ffi_cif *cif, void *fn, void *rvalue, void **avalues); */ -static int _call_function_pointer(int flags, +static int _call_function_pointer(ctypes_state *st, + int flags, PPROC pProc, void **avalues, ffi_type **atypes, @@ -914,7 +938,7 @@ static int _call_function_pointer(int flags, } if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) { - error_object = _ctypes_get_errobj(&space); + error_object = _ctypes_get_errobj(st, &space); if (error_object == NULL) return -1; } @@ -981,9 +1005,9 @@ static int _call_function_pointer(int flags, * - If restype is another ctypes type, return an instance of that. * - Otherwise, call restype and return the result. */ -static PyObject *GetResult(PyObject *restype, void *result, PyObject *checker) +static PyObject *GetResult(ctypes_state *st, + PyObject *restype, void *result, PyObject *checker) { - StgDictObject *dict; PyObject *retval, *v; if (restype == NULL) @@ -993,22 +1017,27 @@ static PyObject *GetResult(PyObject *restype, void *result, PyObject *checker) Py_RETURN_NONE; } - dict = PyType_stgdict(restype); - if (dict == NULL) + StgInfo *info; + if (PyStgInfo_FromType(st, restype, &info) < 0) { + return NULL; + } + if (info == NULL) { return PyObject_CallFunction(restype, "i", *(int *)result); + } - if (dict->getfunc && !_ctypes_simple_instance(restype)) { - retval = dict->getfunc(result, dict->size); + if (info->getfunc && !_ctypes_simple_instance(st, restype)) { + retval = info->getfunc(result, info->size); /* If restype is py_object (detected by comparing getfunc with O_get), we have to call Py_DECREF because O_get has already called Py_INCREF. */ - if (dict->getfunc == _ctypes_get_fielddesc("O")->getfunc) { + if (info->getfunc == _ctypes_get_fielddesc("O")->getfunc) { Py_DECREF(retval); } - } else - retval = PyCData_FromBaseObj(restype, NULL, 0, result); - + } + else { + retval = PyCData_FromBaseObj(st, restype, NULL, 0, result); + } if (!checker || !retval) return retval; @@ -1070,7 +1099,7 @@ void _ctypes_extend_error(PyObject *exc_class, const char *fmt, ...) #ifdef MS_WIN32 static PyObject * -GetComError(HRESULT errcode, GUID *riid, IUnknown *pIunk) +GetComError(ctypes_state *st, HRESULT errcode, GUID *riid, IUnknown *pIunk) { HRESULT hr; ISupportErrorInfo *psei = NULL; @@ -1122,7 +1151,6 @@ GetComError(HRESULT errcode, GUID *riid, IUnknown *pIunk) descr, source, helpfile, helpcontext, progid); if (obj) { - ctypes_state *st = GLOBAL_STATE(); PyErr_SetObject((PyObject *)st->PyComError_Type, obj); Py_DECREF(obj); } @@ -1153,7 +1181,8 @@ GetComError(HRESULT errcode, GUID *riid, IUnknown *pIunk) * * - XXX various requirements for restype, not yet collected */ -PyObject *_ctypes_callproc(PPROC pProc, +PyObject *_ctypes_callproc(ctypes_state *st, + PPROC pProc, PyObject *argtuple, #ifdef MS_WIN32 IUnknown *pIunk, @@ -1183,7 +1212,7 @@ PyObject *_ctypes_callproc(PPROC pProc, if (argcount > CTYPES_MAX_ARGCOUNT) { - PyErr_Format(PyExc_ArgError, "too many arguments (%zi), maximum is %i", + PyErr_Format(st->PyExc_ArgError, "too many arguments (%zi), maximum is %i", argcount, CTYPES_MAX_ARGCOUNT); return NULL; } @@ -1216,20 +1245,20 @@ PyObject *_ctypes_callproc(PPROC pProc, converter = PyTuple_GET_ITEM(argtypes, i); v = PyObject_CallOneArg(converter, arg); if (v == NULL) { - _ctypes_extend_error(PyExc_ArgError, "argument %zd: ", i+1); + _ctypes_extend_error(st->PyExc_ArgError, "argument %zd: ", i+1); goto cleanup; } - err = ConvParam(v, i+1, pa); + err = ConvParam(st, v, i+1, pa); Py_DECREF(v); if (-1 == err) { - _ctypes_extend_error(PyExc_ArgError, "argument %zd: ", i+1); + _ctypes_extend_error(st->PyExc_ArgError, "argument %zd: ", i+1); goto cleanup; } } else { - err = ConvParam(arg, i+1, pa); + err = ConvParam(st, arg, i+1, pa); if (-1 == err) { - _ctypes_extend_error(PyExc_ArgError, "argument %zd: ", i+1); + _ctypes_extend_error(st->PyExc_ArgError, "argument %zd: ", i+1); goto cleanup; /* leaking ? */ } } @@ -1238,7 +1267,10 @@ PyObject *_ctypes_callproc(PPROC pProc, if (restype == Py_None) { rtype = &ffi_type_void; } else { - rtype = _ctypes_get_ffi_type(restype); + rtype = _ctypes_get_ffi_type(st, restype); + } + if (!rtype) { + goto cleanup; } resbuf = alloca(max(rtype->size, sizeof(ffi_arg))); @@ -1277,7 +1309,7 @@ PyObject *_ctypes_callproc(PPROC pProc, avalues[i] = (void *)&args[i].value; } - if (-1 == _call_function_pointer(flags, pProc, avalues, atypes, + if (-1 == _call_function_pointer(st, flags, pProc, avalues, atypes, rtype, resbuf, Py_SAFE_DOWNCAST(argcount, Py_ssize_t, int), Py_SAFE_DOWNCAST(argtype_count, Py_ssize_t, int))) @@ -1305,7 +1337,7 @@ PyObject *_ctypes_callproc(PPROC pProc, #ifdef MS_WIN32 if (iid && pIunk) { if (*(int *)resbuf & 0x80000000) - retval = GetComError(*(HRESULT *)resbuf, iid, pIunk); + retval = GetComError(st, *(HRESULT *)resbuf, iid, pIunk); else retval = PyLong_FromLong(*(int *)resbuf); } else if (flags & FUNCFLAG_HRESULT) { @@ -1315,7 +1347,7 @@ PyObject *_ctypes_callproc(PPROC pProc, retval = PyLong_FromLong(*(int *)resbuf); } else #endif - retval = GetResult(restype, resbuf, checker); + retval = GetResult(st, restype, resbuf, checker); cleanup: for (i = 0; i < argcount; ++i) Py_XDECREF(args[i].keep); @@ -1444,8 +1476,10 @@ copy_com_pointer(PyObject *self, PyObject *args) return NULL; a.keep = b.keep = NULL; - if (-1 == ConvParam(p1, 0, &a) || -1 == ConvParam(p2, 1, &b)) + ctypes_state *st = get_module_state(self); + if (ConvParam(st, p1, 0, &a) < 0 || ConvParam(st, p2, 1, &b) < 0) { goto done; + } src = (IUnknown *)a.value.p; pdst = (IUnknown **)b.value.p; @@ -1557,10 +1591,11 @@ static PyObject *py_dl_open(PyObject *self, PyObject *args) Py_XDECREF(name2); if (!handle) { const char *errmsg = dlerror(); - if (!errmsg) - errmsg = "dlopen() error"; - PyErr_SetString(PyExc_OSError, - errmsg); + if (errmsg) { + _PyErr_SetLocaleString(PyExc_OSError, errmsg); + return NULL; + } + PyErr_SetString(PyExc_OSError, "dlopen() error"); return NULL; } return PyLong_FromVoidPtr(handle); @@ -1573,8 +1608,12 @@ static PyObject *py_dl_close(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "O&:dlclose", &_parse_voidp, &handle)) return NULL; if (dlclose(handle)) { - PyErr_SetString(PyExc_OSError, - dlerror()); + const char *errmsg = dlerror(); + if (errmsg) { + _PyErr_SetLocaleString(PyExc_OSError, errmsg); + return NULL; + } + PyErr_SetString(PyExc_OSError, "dlclose() error"); return NULL; } Py_RETURN_NONE; @@ -1592,13 +1631,32 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) if (PySys_Audit("ctypes.dlsym/handle", "O", args) < 0) { return NULL; } +#undef USE_DLERROR + #ifdef __CYGWIN__ + // dlerror() isn't very helpful on cygwin + #else + #define USE_DLERROR + /* dlerror() always returns the latest error. + * + * Clear the previous value before calling dlsym(), + * to ensure we can tell if our call resulted in an error. + */ + (void)dlerror(); + #endif ptr = dlsym((void*)handle, name); - if (!ptr) { - PyErr_SetString(PyExc_OSError, - dlerror()); + if (ptr) { + return PyLong_FromVoidPtr(ptr); + } + #ifdef USE_DLERROR + const char *errmsg = dlerror(); + if (errmsg) { + _PyErr_SetLocaleString(PyExc_OSError, errmsg); return NULL; } - return PyLong_FromVoidPtr(ptr); + #endif + #undef USE_DLERROR + PyErr_Format(PyExc_OSError, "symbol '%s' not found", name); + return NULL; } #endif @@ -1624,7 +1682,9 @@ call_function(PyObject *self, PyObject *args) return NULL; } - result = _ctypes_callproc((PPROC)func, + ctypes_state *st = get_module_state(self); + result = _ctypes_callproc(st, + (PPROC)func, arguments, #ifdef MS_WIN32 NULL, @@ -1659,7 +1719,9 @@ call_cdeclfunction(PyObject *self, PyObject *args) return NULL; } - result = _ctypes_callproc((PPROC)func, + ctypes_state *st = get_module_state(self); + result = _ctypes_callproc(st, + (PPROC)func, arguments, #ifdef MS_WIN32 NULL, @@ -1683,13 +1745,16 @@ PyDoc_STRVAR(sizeof_doc, static PyObject * sizeof_func(PyObject *self, PyObject *obj) { - StgDictObject *dict; + ctypes_state *st = get_module_state(self); - dict = PyType_stgdict(obj); - if (dict) { - return PyLong_FromSsize_t(dict->size); + StgInfo *info; + if (PyStgInfo_FromType(st, obj, &info) < 0) { + return NULL; + } + if (info) { + return PyLong_FromSsize_t(info->size); } - ctypes_state *st = GLOBAL_STATE(); + if (CDataObject_Check(st, obj)) { return PyLong_FromSsize_t(((CDataObject *)obj)->b_size); } @@ -1706,16 +1771,14 @@ PyDoc_STRVAR(alignment_doc, static PyObject * align_func(PyObject *self, PyObject *obj) { - StgDictObject *dict; - - dict = PyType_stgdict(obj); - if (dict) - return PyLong_FromSsize_t(dict->align); - - dict = PyObject_stgdict(obj); - if (dict) - return PyLong_FromSsize_t(dict->align); - + ctypes_state *st = get_module_state(self); + StgInfo *info; + if (PyStgInfo_FromAny(st, obj, &info) < 0) { + return NULL; + } + if (info) { + return PyLong_FromSsize_t(info->align); + } PyErr_SetString(PyExc_TypeError, "no alignment info"); return NULL; @@ -1746,7 +1809,7 @@ byref(PyObject *self, PyObject *args) if (offset == -1 && PyErr_Occurred()) return NULL; } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state(self); if (!CDataObject_Check(st, obj)) { PyErr_Format(PyExc_TypeError, "byref() argument must be a ctypes instance, not '%s'", @@ -1754,7 +1817,7 @@ byref(PyObject *self, PyObject *args) return NULL; } - parg = PyCArgObject_new(); + parg = PyCArgObject_new(st); if (parg == NULL) return NULL; @@ -1772,7 +1835,7 @@ PyDoc_STRVAR(addressof_doc, static PyObject * addressof(PyObject *self, PyObject *obj) { - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state(self); if (!CDataObject_Check(st, obj)) { PyErr_SetString(PyExc_TypeError, "invalid type"); @@ -1824,7 +1887,6 @@ static PyObject * resize(PyObject *self, PyObject *args) { CDataObject *obj; - StgDictObject *dict; Py_ssize_t size; if (!PyArg_ParseTuple(args, @@ -1832,16 +1894,21 @@ resize(PyObject *self, PyObject *args) &obj, &size)) return NULL; - dict = PyObject_stgdict((PyObject *)obj); - if (dict == NULL) { + ctypes_state *st = get_module_state(self); + StgInfo *info; + int result = PyStgInfo_FromObject(st, (PyObject *)obj, &info); + if (result < 0) { + return NULL; + } + if (info == NULL) { PyErr_SetString(PyExc_TypeError, "expected ctypes instance"); return NULL; } - if (size < dict->size) { + if (size < info->size) { PyErr_Format(PyExc_ValueError, "minimum size is %zd", - dict->size); + info->size); return NULL; } if (obj->b_needsfree == 0) { @@ -1925,11 +1992,12 @@ create_pointer_type(PyObject *module, PyObject *cls) PyTypeObject *typ; PyObject *key; - if (PyDict_GetItemRef(_ctypes_ptrtype_cache, cls, &result) != 0) { + assert(module); + ctypes_state *st = get_module_state(module); + if (PyDict_GetItemRef(st->_ctypes_ptrtype_cache, cls, &result) != 0) { // found or error return result; } - ctypes_state *st = GLOBAL_STATE(); // not found if (PyUnicode_CheckExact(cls)) { PyObject *name = PyUnicode_FromFormat("LP_%U", cls); @@ -1959,7 +2027,7 @@ create_pointer_type(PyObject *module, PyObject *cls) PyErr_SetString(PyExc_TypeError, "must be a ctypes type"); return NULL; } - if (-1 == PyDict_SetItem(_ctypes_ptrtype_cache, key, result)) { + if (PyDict_SetItem(st->_ctypes_ptrtype_cache, key, result) < 0) { Py_DECREF(result); Py_DECREF(key); return NULL; @@ -1988,11 +2056,12 @@ create_pointer_inst(PyObject *module, PyObject *arg) PyObject *result; PyObject *typ; - if (PyDict_GetItemRef(_ctypes_ptrtype_cache, (PyObject *)Py_TYPE(arg), &typ) < 0) { + ctypes_state *st = get_module_state(module); + if (PyDict_GetItemRef(st->_ctypes_ptrtype_cache, (PyObject *)Py_TYPE(arg), &typ) < 0) { return NULL; } if (typ == NULL) { - typ = create_pointer_type(NULL, (PyObject *)Py_TYPE(arg)); + typ = create_pointer_type(module, (PyObject *)Py_TYPE(arg)); if (typ == NULL) return NULL; } @@ -2004,28 +2073,30 @@ create_pointer_inst(PyObject *module, PyObject *arg) static PyObject * buffer_info(PyObject *self, PyObject *arg) { - StgDictObject *dict = PyType_stgdict(arg); PyObject *shape; Py_ssize_t i; - if (dict == NULL) - dict = PyObject_stgdict(arg); - if (dict == NULL) { + ctypes_state *st = get_module_state(self); + StgInfo *info; + if (PyStgInfo_FromAny(st, arg, &info) < 0) { + return NULL; + } + if (info == NULL) { PyErr_SetString(PyExc_TypeError, "not a ctypes type or object"); return NULL; } - shape = PyTuple_New(dict->ndim); + shape = PyTuple_New(info->ndim); if (shape == NULL) return NULL; - for (i = 0; i < (int)dict->ndim; ++i) - PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(dict->shape[i])); + for (i = 0; i < (int)info->ndim; ++i) + PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(info->shape[i])); if (PyErr_Occurred()) { Py_DECREF(shape); return NULL; } - return Py_BuildValue("siN", dict->format, dict->ndim, shape); + return Py_BuildValue("siN", info->format, info->ndim, shape); } diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 1d5b0b14bc39e5..9924d62c0881d1 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -10,13 +10,24 @@ #include "pycore_bitutils.h" // _Py_bswap32() #include "pycore_call.h" // _PyObject_CallNoArgs() +#include // bool #include #include "ctypes.h" +#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) +# include "../_complex.h" // complex +#endif #define CTYPES_CFIELD_CAPSULE_NAME_PYMEM "_ctypes/cfield.c pymem" +/*[clinic input] +module _ctypes +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=476a19c49b31a75c]*/ + +#include "clinic/cfield.c.h" + static void pymem_destructor(PyObject *ptr) { void *p = PyCapsule_GetPointer(ptr, CTYPES_CFIELD_CAPSULE_NAME_PYMEM); @@ -30,225 +41,210 @@ static void pymem_destructor(PyObject *ptr) /* PyCField_Type */ +/*[clinic input] +class _ctypes.CField "PyObject *" "PyObject" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=602817ea3ffc709c]*/ -/* - * Expects the size, index and offset for the current field in *psize and - * *poffset, stores the total size so far in *psize, the offset for the next - * field in *poffset, the alignment requirements for the current field in - * *palign, and returns a field descriptor for this field. - */ -/* - * bitfields extension: - * bitsize != 0: this is a bit field. - * pbitofs points to the current bit offset, this will be updated. - * prev_desc points to the type of the previous bitfield, if any. - */ -PyObject * -PyCField_FromDesc(PyObject *desc, Py_ssize_t index, - Py_ssize_t *pfield_size, int bitsize, int *pbitofs, - Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int big_endian) +static inline +Py_ssize_t NUM_BITS(Py_ssize_t bitsize); +static inline +Py_ssize_t LOW_BIT(Py_ssize_t offset); + + +/*[clinic input] +@classmethod +_ctypes.CField.__new__ as PyCField_new + + name: object(subclass_of='&PyUnicode_Type') + type as proto: object + size: Py_ssize_t + offset: Py_ssize_t + index: Py_ssize_t + bit_size as bit_size_obj: object = None + +[clinic start generated code]*/ + +static PyObject * +PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, + Py_ssize_t size, Py_ssize_t offset, Py_ssize_t index, + PyObject *bit_size_obj) +/*[clinic end generated code: output=43649ef9157c5f58 input=3d813f56373c4caa]*/ { - CFieldObject *self; - PyObject *proto; - Py_ssize_t size, align; - SETFUNC setfunc = NULL; - GETFUNC getfunc = NULL; - StgDictObject *dict; - int fieldtype; -#define NO_BITFIELD 0 -#define NEW_BITFIELD 1 -#define CONT_BITFIELD 2 -#define EXPAND_BITFIELD 3 - - ctypes_state *st = GLOBAL_STATE(); - PyTypeObject *tp = st->PyCField_Type; + CFieldObject* self = NULL; + if (size < 0) { + PyErr_Format(PyExc_ValueError, + "size of field %R must not be negative, got %zd", + name, size); + goto error; + } + // assert: no overflow; + if ((unsigned long long int) size + >= (1ULL << (8*sizeof(Py_ssize_t)-1)) / 8) { + PyErr_Format(PyExc_ValueError, + "size of field %R is too big: %zd", name, size); + goto error; + } + + PyTypeObject *tp = type; + ctypes_state *st = get_module_state_by_class(tp); self = (CFieldObject *)tp->tp_alloc(tp, 0); - if (self == NULL) - return NULL; - dict = PyType_stgdict(desc); - if (!dict) { - PyErr_SetString(PyExc_TypeError, - "has no _stginfo_"); - Py_DECREF(self); + if (!self) { return NULL; } - if (bitsize /* this is a bitfield request */ - && *pfield_size /* we have a bitfield open */ -#ifdef MS_WIN32 - /* MSVC, GCC with -mms-bitfields */ - && dict->size * 8 == *pfield_size -#else - /* GCC */ - && dict->size * 8 <= *pfield_size -#endif - && (*pbitofs + bitsize) <= *pfield_size) { - /* continue bit field */ - fieldtype = CONT_BITFIELD; -#ifndef MS_WIN32 - } else if (bitsize /* this is a bitfield request */ - && *pfield_size /* we have a bitfield open */ - && dict->size * 8 >= *pfield_size - && (*pbitofs + bitsize) <= dict->size * 8) { - /* expand bit field */ - fieldtype = EXPAND_BITFIELD; -#endif - } else if (bitsize) { - /* start new bitfield */ - fieldtype = NEW_BITFIELD; - *pbitofs = 0; - *pfield_size = dict->size * 8; + if (PyUnicode_CheckExact(name)) { + self->name = Py_NewRef(name); } else { - /* not a bit field */ - fieldtype = NO_BITFIELD; - *pbitofs = 0; - *pfield_size = 0; + self->name = PyObject_Str(name); + if (!self->name) { + goto error; + } } - size = dict->size; - proto = desc; + StgInfo *info; + if (PyStgInfo_FromType(st, proto, &info) < 0) { + goto error; + } + if (info == NULL) { + PyErr_Format(PyExc_TypeError, + "type of field %R must be a C type", self->name); + goto error; + } + + if (bit_size_obj != Py_None) { +#ifdef Py_DEBUG + Py_ssize_t bit_size = NUM_BITS(size); + assert(bit_size > 0); + assert(bit_size <= info->size * 8); + // Currently, the bit size is specified redundantly + // in NUM_BITS(size) and bit_size_obj. + // Verify that they match. + assert(PyLong_AsSsize_t(bit_size_obj) == bit_size); +#endif + switch(info->ffi_type_pointer.type) { + case FFI_TYPE_UINT8: + case FFI_TYPE_UINT16: + case FFI_TYPE_UINT32: + case FFI_TYPE_SINT64: + case FFI_TYPE_UINT64: + break; + + case FFI_TYPE_SINT8: + case FFI_TYPE_SINT16: + case FFI_TYPE_SINT32: + if (info->getfunc != _ctypes_get_fielddesc("c")->getfunc + && info->getfunc != _ctypes_get_fielddesc("u")->getfunc) + { + break; + } + _Py_FALLTHROUGH; /* else fall through */ + default: + PyErr_Format(PyExc_TypeError, + "bit fields not allowed for type %s", + ((PyTypeObject*)proto)->tp_name); + goto error; + } + } + + self->proto = Py_NewRef(proto); + self->size = size; + self->offset = offset; + + self->index = index; - /* Field descriptors for 'c_char * n' are be scpecial cased to + /* Field descriptors for 'c_char * n' are be special cased to return a Python string instead of an Array object instance... */ + self->setfunc = NULL; + self->getfunc = NULL; if (PyCArrayTypeObject_Check(st, proto)) { - StgDictObject *adict = PyType_stgdict(proto); - StgDictObject *idict; - if (adict && adict->proto) { - idict = PyType_stgdict(adict->proto); - if (!idict) { + StgInfo *ainfo; + if (PyStgInfo_FromType(st, proto, &ainfo) < 0) { + goto error; + } + + if (ainfo && ainfo->proto) { + StgInfo *iinfo; + if (PyStgInfo_FromType(st, ainfo->proto, &iinfo) < 0) { + goto error; + } + if (!iinfo) { PyErr_SetString(PyExc_TypeError, "has no _stginfo_"); - Py_DECREF(self); - return NULL; + goto error; } - if (idict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { + if (iinfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { struct fielddesc *fd = _ctypes_get_fielddesc("s"); - getfunc = fd->getfunc; - setfunc = fd->setfunc; + self->getfunc = fd->getfunc; + self->setfunc = fd->setfunc; } - if (idict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { + if (iinfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) { struct fielddesc *fd = _ctypes_get_fielddesc("U"); - getfunc = fd->getfunc; - setfunc = fd->setfunc; + self->getfunc = fd->getfunc; + self->setfunc = fd->setfunc; } } } - self->setfunc = setfunc; - self->getfunc = getfunc; - self->index = index; - - self->proto = Py_NewRef(proto); - - switch (fieldtype) { - case NEW_BITFIELD: - if (big_endian) - self->size = (bitsize << 16) + *pfield_size - *pbitofs - bitsize; - else - self->size = (bitsize << 16) + *pbitofs; - *pbitofs = bitsize; - /* fall through */ - case NO_BITFIELD: - if (pack) - align = min(pack, dict->align); - else - align = dict->align; - if (align && *poffset % align) { - Py_ssize_t delta = align - (*poffset % align); - *psize += delta; - *poffset += delta; - } - - if (bitsize == 0) - self->size = size; - *psize += size; - - self->offset = *poffset; - *poffset += size; - - *palign = align; - break; - - case EXPAND_BITFIELD: - *poffset += dict->size - *pfield_size/8; - *psize += dict->size - *pfield_size/8; - - *pfield_size = dict->size * 8; - - if (big_endian) - self->size = (bitsize << 16) + *pfield_size - *pbitofs - bitsize; - else - self->size = (bitsize << 16) + *pbitofs; - - self->offset = *poffset - size; /* poffset is already updated for the NEXT field */ - *pbitofs += bitsize; - break; - - case CONT_BITFIELD: - if (big_endian) - self->size = (bitsize << 16) + *pfield_size - *pbitofs - bitsize; - else - self->size = (bitsize << 16) + *pbitofs; - - self->offset = *poffset - size; /* poffset is already updated for the NEXT field */ - *pbitofs += bitsize; - break; - } - return (PyObject *)self; +error: + Py_XDECREF(self); + return NULL; } + static int -PyCField_set(CFieldObject *self, PyObject *inst, PyObject *value) +PyCField_set(PyObject *op, PyObject *inst, PyObject *value) { CDataObject *dst; char *ptr; - ctypes_state *st = GLOBAL_STATE(); + CFieldObject *self = _CFieldObject_CAST(op); + ctypes_state *st = get_module_state_by_class(Py_TYPE(self)); if (!CDataObject_Check(st, inst)) { PyErr_SetString(PyExc_TypeError, "not a ctype instance"); return -1; } - dst = (CDataObject *)inst; + dst = _CDataObject_CAST(inst); ptr = dst->b_ptr + self->offset; if (value == NULL) { PyErr_SetString(PyExc_TypeError, "can't delete attribute"); return -1; } - return PyCData_set(inst, self->proto, self->setfunc, value, - self->index, self->size, ptr); + return PyCData_set(st, inst, self->proto, self->setfunc, value, + self->index, self->size, ptr); } static PyObject * -PyCField_get(CFieldObject *self, PyObject *inst, PyTypeObject *type) +PyCField_get(PyObject *op, PyObject *inst, PyTypeObject *type) { CDataObject *src; + CFieldObject *self = _CFieldObject_CAST(op); if (inst == NULL) { return Py_NewRef(self); } - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(Py_TYPE(self)); if (!CDataObject_Check(st, inst)) { PyErr_SetString(PyExc_TypeError, "not a ctype instance"); return NULL; } - src = (CDataObject *)inst; - return PyCData_get(self->proto, self->getfunc, inst, - self->index, self->size, src->b_ptr + self->offset); + src = _CDataObject_CAST(inst); + return PyCData_get(st, self->proto, self->getfunc, inst, + self->index, self->size, src->b_ptr + self->offset); } static PyObject * PyCField_get_offset(PyObject *self, void *data) { - return PyLong_FromSsize_t(((CFieldObject *)self)->offset); + return PyLong_FromSsize_t(_CFieldObject_CAST(self)->offset); } static PyObject * PyCField_get_size(PyObject *self, void *data) { - return PyLong_FromSsize_t(((CFieldObject *)self)->size); + return PyLong_FromSsize_t(_CFieldObject_CAST(self)->size); } static PyGetSetDef PyCField_getset[] = { @@ -258,17 +254,20 @@ static PyGetSetDef PyCField_getset[] = { }; static int -PyCField_traverse(CFieldObject *self, visitproc visit, void *arg) +PyCField_traverse(PyObject *op, visitproc visit, void *arg) { + CFieldObject *self = _CFieldObject_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->proto); return 0; } static int -PyCField_clear(CFieldObject *self) +PyCField_clear(PyObject *op) { + CFieldObject *self = _CFieldObject_CAST(op); Py_CLEAR(self->proto); + Py_CLEAR(self->name); return 0; } @@ -277,17 +276,18 @@ PyCField_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - (void)PyCField_clear((CFieldObject *)self); - Py_TYPE(self)->tp_free((PyObject *)self); + (void)PyCField_clear(self); + Py_TYPE(self)->tp_free(self); Py_DECREF(tp); } static PyObject * -PyCField_repr(CFieldObject *self) +PyCField_repr(PyObject *op) { PyObject *result; - Py_ssize_t bits = self->size >> 16; - Py_ssize_t size = self->size & 0xFFFF; + CFieldObject *self = _CFieldObject_CAST(op); + Py_ssize_t bits = NUM_BITS(self->size); + Py_ssize_t size = LOW_BIT(self->size); const char *name; name = ((PyTypeObject *)self->proto)->tp_name; @@ -304,6 +304,7 @@ PyCField_repr(CFieldObject *self) } static PyType_Slot cfield_slots[] = { + {Py_tp_new, PyCField_new}, {Py_tp_dealloc, PyCField_dealloc}, {Py_tp_repr, PyCField_repr}, {Py_tp_doc, (void *)PyDoc_STR("Structure/Union member")}, @@ -319,73 +320,24 @@ PyType_Spec cfield_spec = { .name = "_ctypes.CField", .basicsize = sizeof(CFieldObject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION), + Py_TPFLAGS_IMMUTABLETYPE), .slots = cfield_slots, }; -/******************************************************************/ -/* - Accessor functions -*/ - -/* Derived from Modules/structmodule.c: - Helper routine to get a Python integer and raise the appropriate error - if it isn't one */ - -static int -get_long(PyObject *v, long *p) -{ - long x = PyLong_AsUnsignedLongMask(v); - if (x == -1 && PyErr_Occurred()) - return -1; - *p = x; - return 0; -} - -/* Same, but handling unsigned long */ - -static int -get_ulong(PyObject *v, unsigned long *p) -{ - unsigned long x = PyLong_AsUnsignedLongMask(v); - if (x == (unsigned long)-1 && PyErr_Occurred()) - return -1; - *p = x; - return 0; -} - -/* Same, but handling native long long. */ - -static int -get_longlong(PyObject *v, long long *p) -{ - long long x = PyLong_AsUnsignedLongLongMask(v); - if (x == -1 && PyErr_Occurred()) - return -1; - *p = x; - return 0; -} - -/* Same, but handling native unsigned long long. */ - -static int -get_ulonglong(PyObject *v, unsigned long long *p) -{ - unsigned long long x = PyLong_AsUnsignedLongLongMask(v); - if (x == (unsigned long long)-1 && PyErr_Occurred()) - return -1; - *p = x; - return 0; -} - /***************************************************************** * Integer fields, with bitfield support */ /* how to decode the size field, for integer get/set functions */ -#define LOW_BIT(x) ((x) & 0xFFFF) -#define NUM_BITS(x) ((x) >> 16) +static inline +Py_ssize_t LOW_BIT(Py_ssize_t offset) { + return offset & 0xFFFF; +} +static inline +Py_ssize_t NUM_BITS(Py_ssize_t bitsize) { + return bitsize >> 16; +} /* Doesn't work if NUM_BITS(size) == 0, but it never happens in SET() call. */ #define BIT_MASK(type, size) (((((type)1 << (NUM_BITS(size) - 1)) - 1) << 1) + 1) @@ -402,34 +354,8 @@ get_ulonglong(PyObject *v, unsigned long long *p) /* This macro RETURNS the first parameter with the bit field CHANGED. */ #define SET(type, x, v, size) \ (NUM_BITS(size) ? \ - ( ( (type)x & ~(BIT_MASK(type, size) << LOW_BIT(size)) ) | ( ((type)v & BIT_MASK(type, size)) << LOW_BIT(size) ) ) \ - : (type)v) - -#if SIZEOF_SHORT == 2 -# define SWAP_SHORT _Py_bswap16 -#else -# error "unsupported short size" -#endif - -#if SIZEOF_INT == 4 -# define SWAP_INT _Py_bswap32 -#else -# error "unsupported int size" -#endif - -#if SIZEOF_LONG == 4 -# define SWAP_LONG _Py_bswap32 -#elif SIZEOF_LONG == 8 -# define SWAP_LONG _Py_bswap64 -#else -# error "unsupported long size" -#endif - -#if SIZEOF_LONG_LONG == 8 -# define SWAP_LONG_LONG _Py_bswap64 -#else -# error "unsupported long long size" -#endif + ( ( (type)(x) & ~(BIT_MASK(type, size) << LOW_BIT(size)) ) | ( ((type)(v) & BIT_MASK(type, size)) << LOW_BIT(size) ) ) \ + : (type)(v)) /***************************************************************** * The setter methods return an object which must be kept alive, to keep the @@ -452,203 +378,145 @@ get_ulonglong(PyObject *v, unsigned long long *p) #endif /***************************************************************** - * integer accessor methods, supporting bit fields + * accessor methods for fixed-width integers (e.g. int8_t, uint64_t), + * supporting bit fields. + * These are named e.g. `i8_set`/`i8_get` or `u64_set`/`u64_get`, + * and are all alike, so they're defined using a macro. */ -static PyObject * -b_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - long val; - if (get_long(value, &val) < 0) - return NULL; - *(signed char *)ptr = SET(signed char, *(signed char *)ptr, val, size); - _RET(value); -} - - -static PyObject * -b_get(void *ptr, Py_ssize_t size) -{ - signed char val = *(signed char *)ptr; - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - -static PyObject * -B_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long val; - if (get_ulong(value, &val) < 0) - return NULL; - *(unsigned char *)ptr = SET(unsigned char, *(unsigned char*)ptr, val, size); - _RET(value); -} - - -static PyObject * -B_get(void *ptr, Py_ssize_t size) -{ - unsigned char val = *(unsigned char *)ptr; - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - -static PyObject * -h_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - long val; - short x; - if (get_long(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(short, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} - - -static PyObject * -h_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - long val; - short field; - if (get_long(value, &val) < 0) { - return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_SHORT(field); - field = SET(short, field, val, size); - field = SWAP_SHORT(field); - memcpy(ptr, &field, sizeof(field)); - _RET(value); -} - -static PyObject * -h_get(void *ptr, Py_ssize_t size) -{ - short val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromLong((long)val); -} - -static PyObject * -h_get_sw(void *ptr, Py_ssize_t size) -{ - short val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_SHORT(val); - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - -static PyObject * -H_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long val; - unsigned short x; - if (get_ulong(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(unsigned short, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} - -static PyObject * -H_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long val; - unsigned short field; - if (get_ulong(value, &val) < 0) { - return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_SHORT(field); - field = SET(unsigned short, field, val, size); - field = SWAP_SHORT(field); - memcpy(ptr, &field, sizeof(field)); - _RET(value); -} - - -static PyObject * -H_get(void *ptr, Py_ssize_t size) -{ - unsigned short val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - -static PyObject * -H_get_sw(void *ptr, Py_ssize_t size) -{ - unsigned short val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_SHORT(val); - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - -static PyObject * -i_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - long val; - int x; - if (get_long(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(int, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} - -static PyObject * -i_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - long val; - int field; - if (get_long(value, &val) < 0) { - return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_INT(field); - field = SET(int, field, val, size); - field = SWAP_INT(field); - memcpy(ptr, &field, sizeof(field)); - _RET(value); -} - +#define FIXINT_GETSET(TAG, CTYPE, NBITS, PYAPI_FROMFUNC) \ + static PyObject * \ + TAG ## _set(void *ptr, PyObject *value, Py_ssize_t size_arg) \ + { \ + assert(NUM_BITS(size_arg) || (size_arg == (NBITS) / 8)); \ + CTYPE val; \ + if (PyLong_Check(value) \ + && PyUnstable_Long_IsCompact((PyLongObject *)value)) \ + { \ + val = (CTYPE)PyUnstable_Long_CompactValue( \ + (PyLongObject *)value); \ + } \ + else { \ + Py_ssize_t res = PyLong_AsNativeBytes( \ + value, &val, (NBITS) / 8, \ + Py_ASNATIVEBYTES_NATIVE_ENDIAN \ + | Py_ASNATIVEBYTES_ALLOW_INDEX); \ + if (res < 0) { \ + return NULL; \ + } \ + } \ + CTYPE prev; \ + memcpy(&prev, ptr, (NBITS) / 8); \ + val = SET(CTYPE, prev, val, size_arg); \ + memcpy(ptr, &val, (NBITS) / 8); \ + _RET(value); \ + } \ + \ + static PyObject * \ + TAG ## _get(void *ptr, Py_ssize_t size_arg) \ + { \ + assert(NUM_BITS(size_arg) || (size_arg == (NBITS) / 8)); \ + CTYPE val; \ + memcpy(&val, ptr, sizeof(val)); \ + GET_BITFIELD(val, size_arg); \ + return PYAPI_FROMFUNC(val); \ + } \ + /////////////////////////////////////////////////////////////////////////// + +/* Another macro for byte-swapped variants (e.g. `i8_set_sw`/`i8_get_sw`) */ + +#define FIXINT_GETSET_SW(TAG, CTYPE, NBITS, PYAPI_FROMFUNC, PY_SWAPFUNC) \ + static PyObject * \ + TAG ## _set_sw(void *ptr, PyObject *value, Py_ssize_t size_arg) \ + { \ + CTYPE val; \ + PyObject *res = TAG ## _set(&val, value, (NBITS) / 8); \ + if (res == NULL) { \ + return NULL; \ + } \ + Py_DECREF(res); \ + CTYPE field; \ + memcpy(&field, ptr, sizeof(field)); \ + field = PY_SWAPFUNC(field); \ + field = SET(CTYPE, field, val, size_arg); \ + field = PY_SWAPFUNC(field); \ + memcpy(ptr, &field, sizeof(field)); \ + _RET(value); \ + } \ + \ + static PyObject * \ + TAG ## _get_sw(void *ptr, Py_ssize_t size_arg) \ + { \ + assert(NUM_BITS(size_arg) || (size_arg == (NBITS) / 8)); \ + CTYPE val; \ + memcpy(&val, ptr, sizeof(val)); \ + val = PY_SWAPFUNC(val); \ + GET_BITFIELD(val, size_arg); \ + return PYAPI_FROMFUNC(val); \ + } \ + /////////////////////////////////////////////////////////////////////////// + +/* These macros are expanded for all supported combinations of byte sizes + * (1, 2, 4, 8), signed and unsigned, native and swapped byteorder. + * That's a lot, so generate the list with Argument Clinic (`make clinic`). + */ -static PyObject * -i_get(void *ptr, Py_ssize_t size) -{ - int val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} +/*[python input] +for nbits in 8, 16, 32, 64: + for sgn in 'i', 'u': + u = 'u' if sgn == 'u' else '' + U = u.upper() + apibits = max(nbits, 32) + parts = [ + f'{sgn}{nbits}', + f'{u}int{nbits}_t', + f'{nbits}', + f'PyLong_From{U}Int{apibits}', + ] + print(f'FIXINT_GETSET({", ".join(parts)})') + if nbits > 8: + parts.append(f'_Py_bswap{nbits}') + print(f'FIXINT_GETSET_SW({", ".join(parts)})') +[python start generated code]*/ +FIXINT_GETSET(i8, int8_t, 8, PyLong_FromInt32) +FIXINT_GETSET(u8, uint8_t, 8, PyLong_FromUInt32) +FIXINT_GETSET(i16, int16_t, 16, PyLong_FromInt32) +FIXINT_GETSET_SW(i16, int16_t, 16, PyLong_FromInt32, _Py_bswap16) +FIXINT_GETSET(u16, uint16_t, 16, PyLong_FromUInt32) +FIXINT_GETSET_SW(u16, uint16_t, 16, PyLong_FromUInt32, _Py_bswap16) +FIXINT_GETSET(i32, int32_t, 32, PyLong_FromInt32) +FIXINT_GETSET_SW(i32, int32_t, 32, PyLong_FromInt32, _Py_bswap32) +FIXINT_GETSET(u32, uint32_t, 32, PyLong_FromUInt32) +FIXINT_GETSET_SW(u32, uint32_t, 32, PyLong_FromUInt32, _Py_bswap32) +FIXINT_GETSET(i64, int64_t, 64, PyLong_FromInt64) +FIXINT_GETSET_SW(i64, int64_t, 64, PyLong_FromInt64, _Py_bswap64) +FIXINT_GETSET(u64, uint64_t, 64, PyLong_FromUInt64) +FIXINT_GETSET_SW(u64, uint64_t, 64, PyLong_FromUInt64, _Py_bswap64) +/*[python end generated code: output=3d60c96fa58e07d5 input=0b7e166f2ea18e70]*/ + +// For one-byte types, swapped variants are the same as native +#define i8_set_sw i8_set +#define i8_get_sw i8_get +#define u8_set_sw u8_set +#define u8_get_sw u8_get + +#undef FIXINT_GETSET +#undef FIXINT_GETSET_SW -static PyObject * -i_get_sw(void *ptr, Py_ssize_t size) -{ - int val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_INT(val); - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} +/***************************************************************** + * non-integer accessor methods, not supporting bit fields + */ #ifndef MS_WIN32 /* http://msdn.microsoft.com/en-us/library/cc237864.aspx */ #define VARIANT_FALSE 0x0000 #define VARIANT_TRUE 0xFFFF #endif -/* short BOOL - VARIANT_BOOL */ +/* v: short BOOL - VARIANT_BOOL */ static PyObject * -vBOOL_set(void *ptr, PyObject *value, Py_ssize_t size) +v_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(short int))); switch (PyObject_IsTrue(value)) { case -1: return NULL; @@ -662,22 +530,25 @@ vBOOL_set(void *ptr, PyObject *value, Py_ssize_t size) } static PyObject * -vBOOL_get(void *ptr, Py_ssize_t size) +v_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(short int))); return PyBool_FromLong((long)*(short int *)ptr); } +/* bool ('?'): bool (i.e. _Bool) */ static PyObject * bool_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(bool))); switch (PyObject_IsTrue(value)) { case -1: return NULL; case 0: - *(_Bool *)ptr = 0; + *(bool *)ptr = 0; _RET(value); default: - *(_Bool *)ptr = 1; + *(bool *)ptr = 1; _RET(value); } } @@ -685,300 +556,138 @@ bool_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * bool_get(void *ptr, Py_ssize_t size) { - return PyBool_FromLong((long)*(_Bool *)ptr); -} - -static PyObject * -I_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long val; - unsigned int x; - if (get_ulong(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(unsigned int, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} - -static PyObject * -I_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long val; - unsigned int field; - if (get_ulong(value, &val) < 0) { - return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_INT(field); - field = SET(unsigned int, field, (unsigned int)val, size); - field = SWAP_INT(field); - memcpy(ptr, &field, sizeof(field)); - _RET(value); -} - - -static PyObject * -I_get(void *ptr, Py_ssize_t size) -{ - unsigned int val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromUnsignedLong(val); + assert(NUM_BITS(size) || (size == sizeof(bool))); + return PyBool_FromLong((long)*(bool *)ptr); } +/* g: long double */ static PyObject * -I_get_sw(void *ptr, Py_ssize_t size) -{ - unsigned int val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_INT(val); - GET_BITFIELD(val, size); - return PyLong_FromUnsignedLong(val); -} - -static PyObject * -l_set(void *ptr, PyObject *value, Py_ssize_t size) +g_set(void *ptr, PyObject *value, Py_ssize_t size) { - long val; - long x; - if (get_long(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(long, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} + assert(NUM_BITS(size) || (size == sizeof(long double))); + long double x; -static PyObject * -l_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - long val; - long field; - if (get_long(value, &val) < 0) { + x = PyFloat_AsDouble(value); + if (x == -1 && PyErr_Occurred()) return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_LONG(field); - field = SET(long, field, val, size); - field = SWAP_LONG(field); - memcpy(ptr, &field, sizeof(field)); + memcpy(ptr, &x, sizeof(long double)); _RET(value); } - -static PyObject * -l_get(void *ptr, Py_ssize_t size) -{ - long val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromLong(val); -} - static PyObject * -l_get_sw(void *ptr, Py_ssize_t size) +g_get(void *ptr, Py_ssize_t size) { - long val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_LONG(val); - GET_BITFIELD(val, size); - return PyLong_FromLong(val); + assert(NUM_BITS(size) || (size == sizeof(long double))); + long double val; + memcpy(&val, ptr, sizeof(long double)); + return PyFloat_FromDouble(val); } +/* d: double */ static PyObject * -L_set(void *ptr, PyObject *value, Py_ssize_t size) +d_set(void *ptr, PyObject *value, Py_ssize_t size) { - unsigned long val; - unsigned long x; - if (get_ulong(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(unsigned long, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} + assert(NUM_BITS(size) || (size == sizeof(double))); + double x; -static PyObject * -L_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long val; - unsigned long field; - if (get_ulong(value, &val) < 0) { - return NULL; - } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_LONG(field); - field = SET(unsigned long, field, val, size); - field = SWAP_LONG(field); - memcpy(ptr, &field, sizeof(field)); + x = PyFloat_AsDouble(value); + if (x == -1 && PyErr_Occurred()) + return NULL; + memcpy(ptr, &x, sizeof(double)); _RET(value); } - -static PyObject * -L_get(void *ptr, Py_ssize_t size) -{ - unsigned long val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromUnsignedLong(val); -} - static PyObject * -L_get_sw(void *ptr, Py_ssize_t size) +d_get(void *ptr, Py_ssize_t size) { - unsigned long val; + assert(NUM_BITS(size) || (size == sizeof(double))); + double val; memcpy(&val, ptr, sizeof(val)); - val = SWAP_LONG(val); - GET_BITFIELD(val, size); - return PyLong_FromUnsignedLong(val); + return PyFloat_FromDouble(val); } +#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) +/* C: double complex */ static PyObject * -q_set(void *ptr, PyObject *value, Py_ssize_t size) +C_set(void *ptr, PyObject *value, Py_ssize_t size) { - long long val; - long long x; - if (get_longlong(value, &val) < 0) - return NULL; - memcpy(&x, ptr, sizeof(x)); - x = SET(long long, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); -} + assert(NUM_BITS(size) || (size == sizeof(double complex))); + Py_complex c = PyComplex_AsCComplex(value); -static PyObject * -q_set_sw(void *ptr, PyObject *value, Py_ssize_t size) -{ - long long val; - long long field; - if (get_longlong(value, &val) < 0) { + if (c.real == -1 && PyErr_Occurred()) { return NULL; } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_LONG_LONG(field); - field = SET(long long, field, val, size); - field = SWAP_LONG_LONG(field); - memcpy(ptr, &field, sizeof(field)); + double complex x = CMPLX(c.real, c.imag); + memcpy(ptr, &x, sizeof(x)); _RET(value); } static PyObject * -q_get(void *ptr, Py_ssize_t size) +C_get(void *ptr, Py_ssize_t size) { - long long val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromLongLong(val); -} + assert(NUM_BITS(size) || (size == sizeof(double complex))); + double complex x; -static PyObject * -q_get_sw(void *ptr, Py_ssize_t size) -{ - long long val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_LONG_LONG(val); - GET_BITFIELD(val, size); - return PyLong_FromLongLong(val); -} - -static PyObject * -Q_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - unsigned long long val; - unsigned long long x; - if (get_ulonglong(value, &val) < 0) - return NULL; memcpy(&x, ptr, sizeof(x)); - x = SET(long long, x, val, size); - memcpy(ptr, &x, sizeof(x)); - _RET(value); + return PyComplex_FromDoubles(creal(x), cimag(x)); } +/* E: float complex */ static PyObject * -Q_set_sw(void *ptr, PyObject *value, Py_ssize_t size) +E_set(void *ptr, PyObject *value, Py_ssize_t size) { - unsigned long long val; - unsigned long long field; - if (get_ulonglong(value, &val) < 0) { + assert(NUM_BITS(size) || (size == sizeof(float complex))); + Py_complex c = PyComplex_AsCComplex(value); + + if (c.real == -1 && PyErr_Occurred()) { return NULL; } - memcpy(&field, ptr, sizeof(field)); - field = SWAP_LONG_LONG(field); - field = SET(unsigned long long, field, val, size); - field = SWAP_LONG_LONG(field); - memcpy(ptr, &field, sizeof(field)); + float complex x = CMPLXF((float)c.real, (float)c.imag); + memcpy(ptr, &x, sizeof(x)); _RET(value); } static PyObject * -Q_get(void *ptr, Py_ssize_t size) +E_get(void *ptr, Py_ssize_t size) { - unsigned long long val; - memcpy(&val, ptr, sizeof(val)); - GET_BITFIELD(val, size); - return PyLong_FromUnsignedLongLong(val); -} + assert(NUM_BITS(size) || (size == sizeof(float complex))); + float complex x; -static PyObject * -Q_get_sw(void *ptr, Py_ssize_t size) -{ - unsigned long long val; - memcpy(&val, ptr, sizeof(val)); - val = SWAP_LONG_LONG(val); - GET_BITFIELD(val, size); - return PyLong_FromUnsignedLongLong(val); + memcpy(&x, ptr, sizeof(x)); + return PyComplex_FromDoubles(crealf(x), cimagf(x)); } -/***************************************************************** - * non-integer accessor methods, not supporting bit fields - */ - - +/* F: long double complex */ static PyObject * -g_set(void *ptr, PyObject *value, Py_ssize_t size) +F_set(void *ptr, PyObject *value, Py_ssize_t size) { - long double x; + assert(NUM_BITS(size) || (size == sizeof(long double complex))); + Py_complex c = PyComplex_AsCComplex(value); - x = PyFloat_AsDouble(value); - if (x == -1 && PyErr_Occurred()) + if (c.real == -1 && PyErr_Occurred()) { return NULL; - memcpy(ptr, &x, sizeof(long double)); + } + long double complex x = CMPLXL(c.real, c.imag); + memcpy(ptr, &x, sizeof(x)); _RET(value); } static PyObject * -g_get(void *ptr, Py_ssize_t size) +F_get(void *ptr, Py_ssize_t size) { - long double val; - memcpy(&val, ptr, sizeof(long double)); - return PyFloat_FromDouble(val); -} - -static PyObject * -d_set(void *ptr, PyObject *value, Py_ssize_t size) -{ - double x; + assert(NUM_BITS(size) || (size == sizeof(long double complex))); + long double complex x; - x = PyFloat_AsDouble(value); - if (x == -1 && PyErr_Occurred()) - return NULL; - memcpy(ptr, &x, sizeof(double)); - _RET(value); -} - -static PyObject * -d_get(void *ptr, Py_ssize_t size) -{ - double val; - memcpy(&val, ptr, sizeof(val)); - return PyFloat_FromDouble(val); + memcpy(&x, ptr, sizeof(x)); + return PyComplex_FromDoubles((double)creall(x), (double)cimagl(x)); } +#endif +/* d: double */ static PyObject * d_set_sw(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(double))); double x; x = PyFloat_AsDouble(value); @@ -997,6 +706,7 @@ d_set_sw(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * d_get_sw(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(double))); #ifdef WORDS_BIGENDIAN return PyFloat_FromDouble(PyFloat_Unpack8(ptr, 1)); #else @@ -1004,9 +714,11 @@ d_get_sw(void *ptr, Py_ssize_t size) #endif } +/* f: float */ static PyObject * f_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(float))); float x; x = (float)PyFloat_AsDouble(value); @@ -1019,6 +731,7 @@ f_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * f_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(float))); float val; memcpy(&val, ptr, sizeof(val)); return PyFloat_FromDouble(val); @@ -1027,6 +740,7 @@ f_get(void *ptr, Py_ssize_t size) static PyObject * f_set_sw(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(float))); float x; x = (float)PyFloat_AsDouble(value); @@ -1045,6 +759,7 @@ f_set_sw(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * f_get_sw(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(float))); #ifdef WORDS_BIGENDIAN return PyFloat_FromDouble(PyFloat_Unpack4(ptr, 1)); #else @@ -1052,6 +767,7 @@ f_get_sw(void *ptr, Py_ssize_t size) #endif } +/* O: Python object */ /* py_object refcounts: @@ -1065,6 +781,7 @@ f_get_sw(void *ptr, Py_ssize_t size) static PyObject * O_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(PyObject *))); PyObject *ob = *(PyObject **)ptr; if (ob == NULL) { if (!PyErr_Occurred()) @@ -1079,34 +796,57 @@ O_get(void *ptr, Py_ssize_t size) static PyObject * O_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(PyObject *))); /* Hm, does the memory block need it's own refcount or not? */ *(PyObject **)ptr = value; return Py_NewRef(value); } +/* c: a single byte-character */ static PyObject * c_set(void *ptr, PyObject *value, Py_ssize_t size) { - if (PyBytes_Check(value) && PyBytes_GET_SIZE(value) == 1) { + assert(NUM_BITS(size) || (size == sizeof(char))); + if (PyBytes_Check(value)) { + if (PyBytes_GET_SIZE(value) != 1) { + PyErr_Format(PyExc_TypeError, + "one character bytes, bytearray, or an integer " + "in range(256) expected, not bytes of length %zd", + PyBytes_GET_SIZE(value)); + return NULL; + } *(char *)ptr = PyBytes_AS_STRING(value)[0]; _RET(value); } - if (PyByteArray_Check(value) && PyByteArray_GET_SIZE(value) == 1) { + if (PyByteArray_Check(value)) { + if (PyByteArray_GET_SIZE(value) != 1) { + PyErr_Format(PyExc_TypeError, + "one character bytes, bytearray, or an integer " + "in range(256) expected, not bytearray of length %zd", + PyByteArray_GET_SIZE(value)); + return NULL; + } *(char *)ptr = PyByteArray_AS_STRING(value)[0]; _RET(value); } - if (PyLong_Check(value)) - { - long longval = PyLong_AsLong(value); - if (longval < 0 || longval >= 256) - goto error; + if (PyLong_Check(value)) { + int overflow; + long longval = PyLong_AsLongAndOverflow(value, &overflow); + if (longval == -1 && PyErr_Occurred()) { + return NULL; + } + if (overflow || longval < 0 || longval >= 256) { + PyErr_SetString(PyExc_TypeError, "integer not in range(256)"); + return NULL; + } *(char *)ptr = (char)longval; _RET(value); } - error: PyErr_Format(PyExc_TypeError, - "one character bytes, bytearray or integer expected"); + "one character bytes, bytearray, or an integer " + "in range(256) expected, not %T", + value); return NULL; } @@ -1114,33 +854,40 @@ c_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * c_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(char))); return PyBytes_FromStringAndSize((char *)ptr, 1); } -/* u - a single wchar_t character */ +/* u: a single wchar_t character */ static PyObject * u_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(wchar_t))); Py_ssize_t len; wchar_t chars[2]; if (!PyUnicode_Check(value)) { PyErr_Format(PyExc_TypeError, - "unicode string expected instead of %s instance", - Py_TYPE(value)->tp_name); + "a unicode character expected, not instance of %T", + value); return NULL; - } else - Py_INCREF(value); + } len = PyUnicode_AsWideChar(value, chars, 2); if (len != 1) { - Py_DECREF(value); - PyErr_SetString(PyExc_TypeError, - "one character unicode string expected"); + if (PyUnicode_GET_LENGTH(value) != 1) { + PyErr_Format(PyExc_TypeError, + "a unicode character expected, not a string of length %zd", + PyUnicode_GET_LENGTH(value)); + } + else { + PyErr_Format(PyExc_TypeError, + "the string %A cannot be converted to a single wchar_t character", + value); + } return NULL; } *(wchar_t *)ptr = chars[0]; - Py_DECREF(value); _RET(value); } @@ -1149,10 +896,11 @@ u_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * u_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(wchar_t))); return PyUnicode_FromWideChar((wchar_t *)ptr, 1); } -/* U - a unicode string */ +/* U: a wchar_t* unicode string */ static PyObject * U_get(void *ptr, Py_ssize_t size) { @@ -1211,6 +959,7 @@ U_set(void *ptr, PyObject *value, Py_ssize_t length) } +/* s: a byte string */ static PyObject * s_get(void *ptr, Py_ssize_t size) { @@ -1260,6 +1009,7 @@ s_set(void *ptr, PyObject *value, Py_ssize_t length) _RET(value); } +/* z: a byte string, can be set from integer pointer */ static PyObject * z_set(void *ptr, PyObject *value, Py_ssize_t size) { @@ -1296,6 +1046,7 @@ z_get(void *ptr, Py_ssize_t size) } } +/* Z: a wchar* string, can be set from integer pointer */ static PyObject * Z_set(void *ptr, PyObject *value, Py_ssize_t size) { @@ -1350,8 +1101,9 @@ Z_get(void *ptr, Py_ssize_t size) #ifdef MS_WIN32 +/* X: COM BSTR (wide-char string to be handled handled using Windows API) */ static PyObject * -BSTR_set(void *ptr, PyObject *value, Py_ssize_t size) +X_set(void *ptr, PyObject *value, Py_ssize_t size) { BSTR bstr; @@ -1395,7 +1147,7 @@ BSTR_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * -BSTR_get(void *ptr, Py_ssize_t size) +X_get(void *ptr, Py_ssize_t size) { BSTR p; p = *(BSTR *)ptr; @@ -1410,9 +1162,11 @@ BSTR_get(void *ptr, Py_ssize_t size) } #endif +/* P: generic pointer */ static PyObject * P_set(void *ptr, PyObject *value, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(void *))); void *v; if (value == Py_None) { *(void **)ptr = NULL; @@ -1444,144 +1198,399 @@ P_set(void *ptr, PyObject *value, Py_ssize_t size) static PyObject * P_get(void *ptr, Py_ssize_t size) { + assert(NUM_BITS(size) || (size == sizeof(void *))); if (*(void **)ptr == NULL) { Py_RETURN_NONE; } return PyLong_FromVoidPtr(*(void **)ptr); } -static struct fielddesc formattable[] = { - { 's', s_set, s_get, NULL}, - { 'b', b_set, b_get, NULL}, - { 'B', B_set, B_get, NULL}, - { 'c', c_set, c_get, NULL}, - { 'd', d_set, d_get, NULL, d_set_sw, d_get_sw}, - { 'g', g_set, g_get, NULL}, - { 'f', f_set, f_get, NULL, f_set_sw, f_get_sw}, - { 'h', h_set, h_get, NULL, h_set_sw, h_get_sw}, - { 'H', H_set, H_get, NULL, H_set_sw, H_get_sw}, - { 'i', i_set, i_get, NULL, i_set_sw, i_get_sw}, - { 'I', I_set, I_get, NULL, I_set_sw, I_get_sw}, - { 'l', l_set, l_get, NULL, l_set_sw, l_get_sw}, - { 'L', L_set, L_get, NULL, L_set_sw, L_get_sw}, - { 'q', q_set, q_get, NULL, q_set_sw, q_get_sw}, - { 'Q', Q_set, Q_get, NULL, Q_set_sw, Q_get_sw}, - { 'P', P_set, P_get, NULL}, - { 'z', z_set, z_get, NULL}, - { 'u', u_set, u_get, NULL}, - { 'U', U_set, U_get, NULL}, - { 'Z', Z_set, Z_get, NULL}, -#ifdef MS_WIN32 - { 'X', BSTR_set, BSTR_get, NULL}, -#endif - { 'v', vBOOL_set, vBOOL_get, NULL}, -#if SIZEOF__BOOL == SIZEOF_INT - { '?', bool_set, bool_get, NULL, I_set_sw, I_get_sw}, -#elif SIZEOF__BOOL == SIZEOF_LONG - { '?', bool_set, bool_get, NULL, L_set_sw, L_get_sw}, -#elif SIZEOF__BOOL == SIZEOF_LONG_LONG - { '?', bool_set, bool_get, NULL, Q_set_sw, Q_get_sw}, -#else - { '?', bool_set, bool_get, NULL}, -#endif /* SIZEOF__BOOL */ - { 'O', O_set, O_get, NULL}, - { 0, NULL, NULL, NULL}, +/* Table with info about all formats. + * Must be accessed via _ctypes_get_fielddesc, which initializes it on + * first use. After initialization it's treated as constant & read-only. + */ + +struct formattable { +/*[python input] +for nbytes in 8, 16, 32, 64: + for sgn in 'i', 'u': + print(f' struct fielddesc fmt_{sgn}{nbytes};') +for code in 'sbBcdCEFgfhHiIlLqQPzuUZXvO': + print(f' struct fielddesc fmt_{code};') +[python start generated code]*/ + struct fielddesc fmt_i8; + struct fielddesc fmt_u8; + struct fielddesc fmt_i16; + struct fielddesc fmt_u16; + struct fielddesc fmt_i32; + struct fielddesc fmt_u32; + struct fielddesc fmt_i64; + struct fielddesc fmt_u64; + struct fielddesc fmt_s; + struct fielddesc fmt_b; + struct fielddesc fmt_B; + struct fielddesc fmt_c; + struct fielddesc fmt_d; + struct fielddesc fmt_C; + struct fielddesc fmt_E; + struct fielddesc fmt_F; + struct fielddesc fmt_g; + struct fielddesc fmt_f; + struct fielddesc fmt_h; + struct fielddesc fmt_H; + struct fielddesc fmt_i; + struct fielddesc fmt_I; + struct fielddesc fmt_l; + struct fielddesc fmt_L; + struct fielddesc fmt_q; + struct fielddesc fmt_Q; + struct fielddesc fmt_P; + struct fielddesc fmt_z; + struct fielddesc fmt_u; + struct fielddesc fmt_U; + struct fielddesc fmt_Z; + struct fielddesc fmt_X; + struct fielddesc fmt_v; + struct fielddesc fmt_O; +/*[python end generated code: output=fa648744ec7f919d input=087d58357d4bf2c5]*/ + + // bool has code '?': + struct fielddesc fmt_bool; + + // always contains NULLs: + struct fielddesc fmt_nil; + + // Result of _ctypes_get_simple_type_chars. Initialized just after + // the rest of formattable, so we stash it here. + char simple_type_chars[26]; }; -/* - Ideas: Implement VARIANT in this table, using 'V' code. - Use '?' as code for BOOL. -*/ +static struct formattable formattable; + + +/* Get fielddesc info for a fixed-width integer. + * N.B: - must be called after (or from) _ctypes_init_fielddesc! + * - nbytes must be one of the supported values + */ + +static inline struct fielddesc * +_ctypes_fixint_fielddesc(Py_ssize_t nbytes, bool is_signed) +{ +#define _PACK(NBYTES, SGN) ((NBYTES<<2) + (SGN ? 1 : 0)) + switch (_PACK(nbytes, is_signed)) { +/*[python input] +for nbytes in 8, 16, 32, 64: + for sgn in 'i', 'u': + is_signed = sgn == 'i' + print(f' case (_PACK({nbytes // 8}, {int(is_signed)})): ' + + f'return &formattable.fmt_{sgn}{nbytes};') +[python start generated code]*/ + case (_PACK(1, 1)): return &formattable.fmt_i8; + case (_PACK(1, 0)): return &formattable.fmt_u8; + case (_PACK(2, 1)): return &formattable.fmt_i16; + case (_PACK(2, 0)): return &formattable.fmt_u16; + case (_PACK(4, 1)): return &formattable.fmt_i32; + case (_PACK(4, 0)): return &formattable.fmt_u32; + case (_PACK(8, 1)): return &formattable.fmt_i64; + case (_PACK(8, 0)): return &formattable.fmt_u64; +/*[python end generated code: output=0194ba35c4d64ff3 input=ee9f6f5bb872d645]*/ +#undef _PACK + } + /* ctypes currently only supports platforms where the basic integer types + * (`char`, `short`, `int`, `long`, `long long`) have 1, 2, 4, or 8 bytes + * (i.e. 8 to 64 bits). + */ + Py_UNREACHABLE(); +} + + +/* Macro to call _ctypes_fixint_fielddesc for a given C type. */ + +_Py_COMP_DIAG_PUSH +#if defined(__GNUC__) && (__GNUC__ < 14) +/* The signedness check expands to an expression that's always true or false. + * Older GCC gives a '-Wtype-limits' warning for this, which is a GCC bug + * (docs say it should "not warn for constant expressions"): + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86647 + * Silence that warning. + */ +#pragma GCC diagnostic ignored "-Wtype-limits" +#endif + +#define FIXINT_FIELDDESC_FOR(C_TYPE) \ + _ctypes_fixint_fielddesc(sizeof(C_TYPE), (C_TYPE)-1 < 0) + /* Delayed initialization. Windows cannot statically reference dynamically loaded addresses from DLLs. */ -void +static void +_ctypes_init_fielddesc_locked(void) +{ + /* Fixed-width integers */ + +/*[python input] +for nbytes in 8, 16, 32, 64: + for sgn in 'i', 'u': + is_signed = sgn == 'i' + u = 'u' if sgn == 'u' else 's' + parts = [ + f"0", + f'&ffi_type_{u}int{nbytes}', + f'{sgn}{nbytes}_set', + f'{sgn}{nbytes}_get', + f'{sgn}{nbytes}_set_sw', + f'{sgn}{nbytes}_get_sw', + ] + print(f' formattable.fmt_{sgn}{nbytes} = (struct fielddesc){{') + print(f' {', '.join(parts)} }};') +[python start generated code]*/ + formattable.fmt_i8 = (struct fielddesc){ + 0, &ffi_type_sint8, i8_set, i8_get, i8_set_sw, i8_get_sw }; + formattable.fmt_u8 = (struct fielddesc){ + 0, &ffi_type_uint8, u8_set, u8_get, u8_set_sw, u8_get_sw }; + formattable.fmt_i16 = (struct fielddesc){ + 0, &ffi_type_sint16, i16_set, i16_get, i16_set_sw, i16_get_sw }; + formattable.fmt_u16 = (struct fielddesc){ + 0, &ffi_type_uint16, u16_set, u16_get, u16_set_sw, u16_get_sw }; + formattable.fmt_i32 = (struct fielddesc){ + 0, &ffi_type_sint32, i32_set, i32_get, i32_set_sw, i32_get_sw }; + formattable.fmt_u32 = (struct fielddesc){ + 0, &ffi_type_uint32, u32_set, u32_get, u32_set_sw, u32_get_sw }; + formattable.fmt_i64 = (struct fielddesc){ + 0, &ffi_type_sint64, i64_set, i64_get, i64_set_sw, i64_get_sw }; + formattable.fmt_u64 = (struct fielddesc){ + 0, &ffi_type_uint64, u64_set, u64_get, u64_set_sw, u64_get_sw }; +/*[python end generated code: output=16806fe0ca3a9c4c input=850b8dd6388b1b10]*/ + + + /* Native C integers. + * These use getters/setters for fixed-width ints but have their own + * `code` and `pffi_type`. + */ + +/*[python input] +for base_code, base_c_type in [ + ('b', 'char'), + ('h', 'short'), + ('i', 'int'), + ('l', 'long'), + ('q', 'long long'), +]: + for code, c_type, ffi_type in [ + (base_code, 'signed ' + base_c_type, 's' + base_c_type), + (base_code.upper(), 'unsigned ' + base_c_type, 'u' + base_c_type), + ]: + print(f' formattable.fmt_{code} = *FIXINT_FIELDDESC_FOR({c_type});') + print(f" formattable.fmt_{code}.code = '{code}';") + if base_code == 'q': + # ffi doesn't have `long long`; keep use the fixint type + pass + else: + print(f' formattable.fmt_{code}.pffi_type = &ffi_type_{ffi_type};') +[python start generated code]*/ + formattable.fmt_b = *FIXINT_FIELDDESC_FOR(signed char); + formattable.fmt_b.code = 'b'; + formattable.fmt_b.pffi_type = &ffi_type_schar; + formattable.fmt_B = *FIXINT_FIELDDESC_FOR(unsigned char); + formattable.fmt_B.code = 'B'; + formattable.fmt_B.pffi_type = &ffi_type_uchar; + formattable.fmt_h = *FIXINT_FIELDDESC_FOR(signed short); + formattable.fmt_h.code = 'h'; + formattable.fmt_h.pffi_type = &ffi_type_sshort; + formattable.fmt_H = *FIXINT_FIELDDESC_FOR(unsigned short); + formattable.fmt_H.code = 'H'; + formattable.fmt_H.pffi_type = &ffi_type_ushort; + formattable.fmt_i = *FIXINT_FIELDDESC_FOR(signed int); + formattable.fmt_i.code = 'i'; + formattable.fmt_i.pffi_type = &ffi_type_sint; + formattable.fmt_I = *FIXINT_FIELDDESC_FOR(unsigned int); + formattable.fmt_I.code = 'I'; + formattable.fmt_I.pffi_type = &ffi_type_uint; + formattable.fmt_l = *FIXINT_FIELDDESC_FOR(signed long); + formattable.fmt_l.code = 'l'; + formattable.fmt_l.pffi_type = &ffi_type_slong; + formattable.fmt_L = *FIXINT_FIELDDESC_FOR(unsigned long); + formattable.fmt_L.code = 'L'; + formattable.fmt_L.pffi_type = &ffi_type_ulong; + formattable.fmt_q = *FIXINT_FIELDDESC_FOR(signed long long); + formattable.fmt_q.code = 'q'; + formattable.fmt_Q = *FIXINT_FIELDDESC_FOR(unsigned long long); + formattable.fmt_Q.code = 'Q'; +/*[python end generated code: output=873c87a2e6b5075a input=ee814ca263aac18e]*/ + + + /* Other types have bespoke setters and getters named `@_set` and `@_get`, + * where `@` is the type code. + * Some have swapped variants, `@_set_sw` and `@_get_sw` + */ + +#define _TABLE_ENTRY(SYMBOL, FFI_TYPE, ...) \ + formattable.fmt_ ## SYMBOL = \ + (struct fielddesc){(#SYMBOL)[0], (FFI_TYPE), __VA_ARGS__}; \ + /////////////////////////////////////////////////////////////////////////// + +#define TABLE_ENTRY(SYMBOL, FFI_TYPE) \ + _TABLE_ENTRY(SYMBOL, FFI_TYPE, SYMBOL ## _set, SYMBOL ## _get) \ + /////////////////////////////////////////////////////////////////////////// + +#define TABLE_ENTRY_SW(SYMBOL, FFI_TYPE) \ + _TABLE_ENTRY(SYMBOL, FFI_TYPE, SYMBOL ## _set, \ + SYMBOL ## _get, SYMBOL ## _set_sw, SYMBOL ## _get_sw) \ + /////////////////////////////////////////////////////////////////////////// + + TABLE_ENTRY_SW(d, &ffi_type_double); +#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) + if (Py_FFI_COMPLEX_AVAILABLE) { + TABLE_ENTRY(C, &ffi_type_complex_double); + TABLE_ENTRY(E, &ffi_type_complex_float); + TABLE_ENTRY(F, &ffi_type_complex_longdouble); + } +#endif + TABLE_ENTRY(g, &ffi_type_longdouble); + TABLE_ENTRY_SW(f, &ffi_type_float); + TABLE_ENTRY(v, &ffi_type_sshort); /* vBOOL */ + + // ctypes.c_char is signed for FFI, even where C wchar_t is unsigned. + TABLE_ENTRY(c, _ctypes_fixint_fielddesc(sizeof(char), true)->pffi_type); + // ctypes.c_wchar is signed for FFI, even where C wchar_t is unsigned. + TABLE_ENTRY(u, _ctypes_fixint_fielddesc(sizeof(wchar_t), true)->pffi_type); + + TABLE_ENTRY(s, &ffi_type_pointer); + TABLE_ENTRY(P, &ffi_type_pointer); + TABLE_ENTRY(z, &ffi_type_pointer); + TABLE_ENTRY(U, &ffi_type_pointer); + TABLE_ENTRY(Z, &ffi_type_pointer); +#ifdef MS_WIN32 + TABLE_ENTRY(X, &ffi_type_pointer); +#endif + TABLE_ENTRY(O, &ffi_type_pointer); + +#undef TABLE_ENTRY_SW +#undef TABLE_ENTRY +#undef _TABLE_ENTRY + + /* bool has code '?', fill it in manually */ + + // ctypes.c_bool is unsigned for FFI, even where C bool is signed. + formattable.fmt_bool = *_ctypes_fixint_fielddesc(sizeof(bool), false); + formattable.fmt_bool.code = '?'; + formattable.fmt_bool.setfunc = bool_set; + formattable.fmt_bool.getfunc = bool_get; + +/*[python input] +all_chars = "cbBhHiIlLdCEFfuzZqQPXOv?g" +print(f' assert(sizeof(formattable.simple_type_chars) == {len(all_chars)+1});') +print(f' int i = 0;') +for char in all_chars: + ident_char = {'?': 'bool'}.get(char, char) + print(f" if (formattable.fmt_{ident_char}.code) " + + f"formattable.simple_type_chars[i++] = '{char}';") +print(f" formattable.simple_type_chars[i] = 0;") +[python start generated code]*/ + assert(sizeof(formattable.simple_type_chars) == 26); + int i = 0; + if (formattable.fmt_c.code) formattable.simple_type_chars[i++] = 'c'; + if (formattable.fmt_b.code) formattable.simple_type_chars[i++] = 'b'; + if (formattable.fmt_B.code) formattable.simple_type_chars[i++] = 'B'; + if (formattable.fmt_h.code) formattable.simple_type_chars[i++] = 'h'; + if (formattable.fmt_H.code) formattable.simple_type_chars[i++] = 'H'; + if (formattable.fmt_i.code) formattable.simple_type_chars[i++] = 'i'; + if (formattable.fmt_I.code) formattable.simple_type_chars[i++] = 'I'; + if (formattable.fmt_l.code) formattable.simple_type_chars[i++] = 'l'; + if (formattable.fmt_L.code) formattable.simple_type_chars[i++] = 'L'; + if (formattable.fmt_d.code) formattable.simple_type_chars[i++] = 'd'; + if (formattable.fmt_C.code) formattable.simple_type_chars[i++] = 'C'; + if (formattable.fmt_E.code) formattable.simple_type_chars[i++] = 'E'; + if (formattable.fmt_F.code) formattable.simple_type_chars[i++] = 'F'; + if (formattable.fmt_f.code) formattable.simple_type_chars[i++] = 'f'; + if (formattable.fmt_u.code) formattable.simple_type_chars[i++] = 'u'; + if (formattable.fmt_z.code) formattable.simple_type_chars[i++] = 'z'; + if (formattable.fmt_Z.code) formattable.simple_type_chars[i++] = 'Z'; + if (formattable.fmt_q.code) formattable.simple_type_chars[i++] = 'q'; + if (formattable.fmt_Q.code) formattable.simple_type_chars[i++] = 'Q'; + if (formattable.fmt_P.code) formattable.simple_type_chars[i++] = 'P'; + if (formattable.fmt_X.code) formattable.simple_type_chars[i++] = 'X'; + if (formattable.fmt_O.code) formattable.simple_type_chars[i++] = 'O'; + if (formattable.fmt_v.code) formattable.simple_type_chars[i++] = 'v'; + if (formattable.fmt_bool.code) formattable.simple_type_chars[i++] = '?'; + if (formattable.fmt_g.code) formattable.simple_type_chars[i++] = 'g'; + formattable.simple_type_chars[i] = 0; +/*[python end generated code: output=e6e5098a02f4b606 input=72031a625eac00c1]*/ + +} +#undef FIXINT_FIELDDESC_FOR +_Py_COMP_DIAG_POP + +static void _ctypes_init_fielddesc(void) { - struct fielddesc *fd = formattable; - for (; fd->code; ++fd) { - switch (fd->code) { - case 's': fd->pffi_type = &ffi_type_pointer; break; - case 'b': fd->pffi_type = &ffi_type_schar; break; - case 'B': fd->pffi_type = &ffi_type_uchar; break; - case 'c': fd->pffi_type = &ffi_type_schar; break; - case 'd': fd->pffi_type = &ffi_type_double; break; - case 'g': fd->pffi_type = &ffi_type_longdouble; break; - case 'f': fd->pffi_type = &ffi_type_float; break; - case 'h': fd->pffi_type = &ffi_type_sshort; break; - case 'H': fd->pffi_type = &ffi_type_ushort; break; - case 'i': fd->pffi_type = &ffi_type_sint; break; - case 'I': fd->pffi_type = &ffi_type_uint; break; - /* XXX Hm, sizeof(int) == sizeof(long) doesn't hold on every platform */ - /* As soon as we can get rid of the type codes, this is no longer a problem */ - #if SIZEOF_LONG == 4 - case 'l': fd->pffi_type = &ffi_type_sint32; break; - case 'L': fd->pffi_type = &ffi_type_uint32; break; - #elif SIZEOF_LONG == 8 - case 'l': fd->pffi_type = &ffi_type_sint64; break; - case 'L': fd->pffi_type = &ffi_type_uint64; break; - #else - #error - #endif - #if SIZEOF_LONG_LONG == 8 - case 'q': fd->pffi_type = &ffi_type_sint64; break; - case 'Q': fd->pffi_type = &ffi_type_uint64; break; - #else - #error - #endif - case 'P': fd->pffi_type = &ffi_type_pointer; break; - case 'z': fd->pffi_type = &ffi_type_pointer; break; - case 'u': - if (sizeof(wchar_t) == sizeof(short)) - fd->pffi_type = &ffi_type_sshort; - else if (sizeof(wchar_t) == sizeof(int)) - fd->pffi_type = &ffi_type_sint; - else if (sizeof(wchar_t) == sizeof(long)) - fd->pffi_type = &ffi_type_slong; - else - Py_UNREACHABLE(); - break; - case 'U': fd->pffi_type = &ffi_type_pointer; break; - case 'Z': fd->pffi_type = &ffi_type_pointer; break; - #ifdef MS_WIN32 - case 'X': fd->pffi_type = &ffi_type_pointer; break; - #endif - case 'v': fd->pffi_type = &ffi_type_sshort; break; - #if SIZEOF__BOOL == 1 - case '?': fd->pffi_type = &ffi_type_uchar; break; /* Also fallback for no native _Bool support */ - #elif SIZEOF__BOOL == SIZEOF_SHORT - case '?': fd->pffi_type = &ffi_type_ushort; break; - #elif SIZEOF__BOOL == SIZEOF_INT - case '?': fd->pffi_type = &ffi_type_uint; break; - #elif SIZEOF__BOOL == SIZEOF_LONG - case '?': fd->pffi_type = &ffi_type_ulong; break; - #elif SIZEOF__BOOL == SIZEOF_LONG_LONG - case '?': fd->pffi_type = &ffi_type_ulong; break; - #endif /* SIZEOF__BOOL */ - case 'O': fd->pffi_type = &ffi_type_pointer; break; - default: - Py_UNREACHABLE(); - } + static bool initialized = false; + static PyMutex mutex = {0}; + PyMutex_Lock(&mutex); + if (!initialized) { + _ctypes_init_fielddesc_locked(); + initialized = true; } + PyMutex_Unlock(&mutex); +} +char * +_ctypes_get_simple_type_chars(void) { + _ctypes_init_fielddesc(); + return formattable.simple_type_chars; } struct fielddesc * _ctypes_get_fielddesc(const char *fmt) { - static int initialized = 0; - struct fielddesc *table = formattable; - - if (!initialized) { - initialized = 1; - _ctypes_init_fielddesc(); + _ctypes_init_fielddesc(); + + struct fielddesc *result = NULL; + switch(fmt[0]) { +/*[python input] +for code in 'sbBcdCEFgfhHiIlLqQPzuUZXvO': + print(f" case '{code}': result = &formattable.fmt_{code}; break;") +[python start generated code]*/ + case 's': result = &formattable.fmt_s; break; + case 'b': result = &formattable.fmt_b; break; + case 'B': result = &formattable.fmt_B; break; + case 'c': result = &formattable.fmt_c; break; + case 'd': result = &formattable.fmt_d; break; + case 'C': result = &formattable.fmt_C; break; + case 'E': result = &formattable.fmt_E; break; + case 'F': result = &formattable.fmt_F; break; + case 'g': result = &formattable.fmt_g; break; + case 'f': result = &formattable.fmt_f; break; + case 'h': result = &formattable.fmt_h; break; + case 'H': result = &formattable.fmt_H; break; + case 'i': result = &formattable.fmt_i; break; + case 'I': result = &formattable.fmt_I; break; + case 'l': result = &formattable.fmt_l; break; + case 'L': result = &formattable.fmt_L; break; + case 'q': result = &formattable.fmt_q; break; + case 'Q': result = &formattable.fmt_Q; break; + case 'P': result = &formattable.fmt_P; break; + case 'z': result = &formattable.fmt_z; break; + case 'u': result = &formattable.fmt_u; break; + case 'U': result = &formattable.fmt_U; break; + case 'Z': result = &formattable.fmt_Z; break; + case 'X': result = &formattable.fmt_X; break; + case 'v': result = &formattable.fmt_v; break; + case 'O': result = &formattable.fmt_O; break; +/*[python end generated code: output=81a8223dda9f81f7 input=2f59666d3c024edf]*/ + case '?': result = &formattable.fmt_bool; break; } - - for (; table->code; ++table) { - if (table->code == fmt[0]) - return table; + if (!result || !result->code) { + return NULL; } - return NULL; + assert(result->pffi_type); + assert(result->setfunc); + assert(result->getfunc); + return result; } +/* + Ideas: Implement VARIANT in this table, using 'V' code. +*/ + /*---------------- EOF ----------------*/ diff --git a/Modules/_ctypes/clinic/_ctypes.c.h b/Modules/_ctypes/clinic/_ctypes.c.h new file mode 100644 index 00000000000000..1f2e871137ed79 --- /dev/null +++ b/Modules/_ctypes/clinic/_ctypes.c.h @@ -0,0 +1,796 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif +#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +PyDoc_STRVAR(_ctypes_CType_Type___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n" +"Return memory consumption of the type object."); + +#define _CTYPES_CTYPE_TYPE___SIZEOF___METHODDEF \ + {"__sizeof__", _PyCFunction_CAST(_ctypes_CType_Type___sizeof__), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _ctypes_CType_Type___sizeof____doc__}, + +static PyObject * +_ctypes_CType_Type___sizeof___impl(PyObject *self, PyTypeObject *cls); + +static PyObject * +_ctypes_CType_Type___sizeof__(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "__sizeof__() takes no arguments"); + return NULL; + } + return _ctypes_CType_Type___sizeof___impl(self, cls); +} + +PyDoc_STRVAR(CDataType_from_address__doc__, +"from_address($self, value, /)\n" +"--\n" +"\n" +"C.from_address(integer) -> C instance\n" +"\n" +"Access a C instance at the specified address."); + +#define CDATATYPE_FROM_ADDRESS_METHODDEF \ + {"from_address", _PyCFunction_CAST(CDataType_from_address), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, CDataType_from_address__doc__}, + +static PyObject * +CDataType_from_address_impl(PyObject *type, PyTypeObject *cls, + PyObject *value); + +static PyObject * +CDataType_from_address(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_address", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = CDataType_from_address_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(CDataType_from_buffer__doc__, +"from_buffer($self, obj, offset=0, /)\n" +"--\n" +"\n" +"C.from_buffer(object, offset=0) -> C instance\n" +"\n" +"Create a C instance from a writeable buffer."); + +#define CDATATYPE_FROM_BUFFER_METHODDEF \ + {"from_buffer", _PyCFunction_CAST(CDataType_from_buffer), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, CDataType_from_buffer__doc__}, + +static PyObject * +CDataType_from_buffer_impl(PyObject *type, PyTypeObject *cls, PyObject *obj, + Py_ssize_t offset); + +static PyObject * +CDataType_from_buffer(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_buffer", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *obj; + Py_ssize_t offset = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + obj = args[0]; + if (nargs < 2) { + goto skip_optional_posonly; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + offset = ival; + } +skip_optional_posonly: + return_value = CDataType_from_buffer_impl(type, cls, obj, offset); + +exit: + return return_value; +} + +PyDoc_STRVAR(CDataType_from_buffer_copy__doc__, +"from_buffer_copy($self, buffer, offset=0, /)\n" +"--\n" +"\n" +"C.from_buffer_copy(object, offset=0) -> C instance\n" +"\n" +"Create a C instance from a readable buffer."); + +#define CDATATYPE_FROM_BUFFER_COPY_METHODDEF \ + {"from_buffer_copy", _PyCFunction_CAST(CDataType_from_buffer_copy), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, CDataType_from_buffer_copy__doc__}, + +static PyObject * +CDataType_from_buffer_copy_impl(PyObject *type, PyTypeObject *cls, + Py_buffer *buffer, Py_ssize_t offset); + +static PyObject * +CDataType_from_buffer_copy(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_buffer_copy", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_buffer buffer = {NULL, NULL}; + Py_ssize_t offset = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (PyObject_GetBuffer(args[0], &buffer, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (nargs < 2) { + goto skip_optional_posonly; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + offset = ival; + } +skip_optional_posonly: + return_value = CDataType_from_buffer_copy_impl(type, cls, &buffer, offset); + +exit: + /* Cleanup for buffer */ + if (buffer.obj) { + PyBuffer_Release(&buffer); + } + + return return_value; +} + +PyDoc_STRVAR(CDataType_in_dll__doc__, +"in_dll($self, dll, name, /)\n" +"--\n" +"\n" +"C.in_dll(dll, name) -> C instance\n" +"\n" +"Access a C instance in a dll."); + +#define CDATATYPE_IN_DLL_METHODDEF \ + {"in_dll", _PyCFunction_CAST(CDataType_in_dll), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, CDataType_in_dll__doc__}, + +static PyObject * +CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, + const char *name); + +static PyObject * +CDataType_in_dll(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "in_dll", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *dll; + const char *name; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + dll = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("in_dll", "argument 2", "str", args[1]); + goto exit; + } + Py_ssize_t name_length; + name = PyUnicode_AsUTF8AndSize(args[1], &name_length); + if (name == NULL) { + goto exit; + } + if (strlen(name) != (size_t)name_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + return_value = CDataType_in_dll_impl(type, cls, dll, name); + +exit: + return return_value; +} + +PyDoc_STRVAR(CDataType_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n" +"Convert a Python object into a function call parameter."); + +#define CDATATYPE_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(CDataType_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, CDataType_from_param__doc__}, + +static PyObject * +CDataType_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value); + +static PyObject * +CDataType_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = CDataType_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(PyCPointerType_set_type__doc__, +"set_type($self, type, /)\n" +"--\n" +"\n"); + +#define PYCPOINTERTYPE_SET_TYPE_METHODDEF \ + {"set_type", _PyCFunction_CAST(PyCPointerType_set_type), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, PyCPointerType_set_type__doc__}, + +static PyObject * +PyCPointerType_set_type_impl(PyTypeObject *self, PyTypeObject *cls, + PyObject *type); + +static PyObject * +PyCPointerType_set_type(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "set_type", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *type; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + type = args[0]; + return_value = PyCPointerType_set_type_impl((PyTypeObject *)self, cls, type); + +exit: + return return_value; +} + +PyDoc_STRVAR(PyCPointerType_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n" +"Convert a Python object into a function call parameter."); + +#define PYCPOINTERTYPE_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(PyCPointerType_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, PyCPointerType_from_param__doc__}, + +static PyObject * +PyCPointerType_from_param_impl(PyObject *type, PyTypeObject *cls, + PyObject *value); + +static PyObject * +PyCPointerType_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = PyCPointerType_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(c_wchar_p_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n"); + +#define C_WCHAR_P_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(c_wchar_p_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, c_wchar_p_from_param__doc__}, + +static PyObject * +c_wchar_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value); + +static PyObject * +c_wchar_p_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = c_wchar_p_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(c_char_p_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n"); + +#define C_CHAR_P_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(c_char_p_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, c_char_p_from_param__doc__}, + +static PyObject * +c_char_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value); + +static PyObject * +c_char_p_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = c_char_p_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(c_void_p_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n"); + +#define C_VOID_P_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(c_void_p_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, c_void_p_from_param__doc__}, + +static PyObject * +c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value); + +static PyObject * +c_void_p_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = c_void_p_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(PyCSimpleType_from_param__doc__, +"from_param($self, value, /)\n" +"--\n" +"\n" +"Convert a Python object into a function call parameter."); + +#define PYCSIMPLETYPE_FROM_PARAM_METHODDEF \ + {"from_param", _PyCFunction_CAST(PyCSimpleType_from_param), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, PyCSimpleType_from_param__doc__}, + +static PyObject * +PyCSimpleType_from_param_impl(PyObject *type, PyTypeObject *cls, + PyObject *value); + +static PyObject * +PyCSimpleType_from_param(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_param", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + value = args[0]; + return_value = PyCSimpleType_from_param_impl(type, cls, value); + +exit: + return return_value; +} + +PyDoc_STRVAR(PyCData_reduce__doc__, +"__reduce__($self, /)\n" +"--\n" +"\n"); + +#define PYCDATA_REDUCE_METHODDEF \ + {"__reduce__", _PyCFunction_CAST(PyCData_reduce), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, PyCData_reduce__doc__}, + +static PyObject * +PyCData_reduce_impl(PyObject *myself, PyTypeObject *cls); + +static PyObject * +PyCData_reduce(PyObject *myself, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "__reduce__() takes no arguments"); + return NULL; + } + return PyCData_reduce_impl(myself, cls); +} + +#if !defined(_ctypes_CFuncPtr_errcheck_DOCSTR) +# define _ctypes_CFuncPtr_errcheck_DOCSTR NULL +#endif +#if defined(_CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF) +# undef _CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF +# define _CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF {"errcheck", (getter)_ctypes_CFuncPtr_errcheck_get, (setter)_ctypes_CFuncPtr_errcheck_set, _ctypes_CFuncPtr_errcheck_DOCSTR}, +#else +# define _CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF {"errcheck", NULL, (setter)_ctypes_CFuncPtr_errcheck_set, NULL}, +#endif + +static int +_ctypes_CFuncPtr_errcheck_set_impl(PyCFuncPtrObject *self, PyObject *value); + +static int +_ctypes_CFuncPtr_errcheck_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ctypes_CFuncPtr_errcheck_set_impl((PyCFuncPtrObject *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ctypes_CFuncPtr_errcheck__doc__, +"a function to check for errors"); +#if defined(_ctypes_CFuncPtr_errcheck_DOCSTR) +# undef _ctypes_CFuncPtr_errcheck_DOCSTR +#endif +#define _ctypes_CFuncPtr_errcheck_DOCSTR _ctypes_CFuncPtr_errcheck__doc__ + +#if !defined(_ctypes_CFuncPtr_errcheck_DOCSTR) +# define _ctypes_CFuncPtr_errcheck_DOCSTR NULL +#endif +#if defined(_CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF) +# undef _CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF +# define _CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF {"errcheck", (getter)_ctypes_CFuncPtr_errcheck_get, (setter)_ctypes_CFuncPtr_errcheck_set, _ctypes_CFuncPtr_errcheck_DOCSTR}, +#else +# define _CTYPES_CFUNCPTR_ERRCHECK_GETSETDEF {"errcheck", (getter)_ctypes_CFuncPtr_errcheck_get, NULL, _ctypes_CFuncPtr_errcheck_DOCSTR}, +#endif + +static PyObject * +_ctypes_CFuncPtr_errcheck_get_impl(PyCFuncPtrObject *self); + +static PyObject * +_ctypes_CFuncPtr_errcheck_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ctypes_CFuncPtr_errcheck_get_impl((PyCFuncPtrObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ctypes_CFuncPtr_restype_DOCSTR) +# define _ctypes_CFuncPtr_restype_DOCSTR NULL +#endif +#if defined(_CTYPES_CFUNCPTR_RESTYPE_GETSETDEF) +# undef _CTYPES_CFUNCPTR_RESTYPE_GETSETDEF +# define _CTYPES_CFUNCPTR_RESTYPE_GETSETDEF {"restype", (getter)_ctypes_CFuncPtr_restype_get, (setter)_ctypes_CFuncPtr_restype_set, _ctypes_CFuncPtr_restype_DOCSTR}, +#else +# define _CTYPES_CFUNCPTR_RESTYPE_GETSETDEF {"restype", NULL, (setter)_ctypes_CFuncPtr_restype_set, NULL}, +#endif + +static int +_ctypes_CFuncPtr_restype_set_impl(PyCFuncPtrObject *self, PyObject *value); + +static int +_ctypes_CFuncPtr_restype_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ctypes_CFuncPtr_restype_set_impl((PyCFuncPtrObject *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ctypes_CFuncPtr_restype__doc__, +"specify the result type"); +#if defined(_ctypes_CFuncPtr_restype_DOCSTR) +# undef _ctypes_CFuncPtr_restype_DOCSTR +#endif +#define _ctypes_CFuncPtr_restype_DOCSTR _ctypes_CFuncPtr_restype__doc__ + +#if !defined(_ctypes_CFuncPtr_restype_DOCSTR) +# define _ctypes_CFuncPtr_restype_DOCSTR NULL +#endif +#if defined(_CTYPES_CFUNCPTR_RESTYPE_GETSETDEF) +# undef _CTYPES_CFUNCPTR_RESTYPE_GETSETDEF +# define _CTYPES_CFUNCPTR_RESTYPE_GETSETDEF {"restype", (getter)_ctypes_CFuncPtr_restype_get, (setter)_ctypes_CFuncPtr_restype_set, _ctypes_CFuncPtr_restype_DOCSTR}, +#else +# define _CTYPES_CFUNCPTR_RESTYPE_GETSETDEF {"restype", (getter)_ctypes_CFuncPtr_restype_get, NULL, _ctypes_CFuncPtr_restype_DOCSTR}, +#endif + +static PyObject * +_ctypes_CFuncPtr_restype_get_impl(PyCFuncPtrObject *self); + +static PyObject * +_ctypes_CFuncPtr_restype_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ctypes_CFuncPtr_restype_get_impl((PyCFuncPtrObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ctypes_CFuncPtr_argtypes_DOCSTR) +# define _ctypes_CFuncPtr_argtypes_DOCSTR NULL +#endif +#if defined(_CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF) +# undef _CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF +# define _CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF {"argtypes", (getter)_ctypes_CFuncPtr_argtypes_get, (setter)_ctypes_CFuncPtr_argtypes_set, _ctypes_CFuncPtr_argtypes_DOCSTR}, +#else +# define _CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF {"argtypes", NULL, (setter)_ctypes_CFuncPtr_argtypes_set, NULL}, +#endif + +static int +_ctypes_CFuncPtr_argtypes_set_impl(PyCFuncPtrObject *self, PyObject *value); + +static int +_ctypes_CFuncPtr_argtypes_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ctypes_CFuncPtr_argtypes_set_impl((PyCFuncPtrObject *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ctypes_CFuncPtr_argtypes__doc__, +"specify the argument types"); +#if defined(_ctypes_CFuncPtr_argtypes_DOCSTR) +# undef _ctypes_CFuncPtr_argtypes_DOCSTR +#endif +#define _ctypes_CFuncPtr_argtypes_DOCSTR _ctypes_CFuncPtr_argtypes__doc__ + +#if !defined(_ctypes_CFuncPtr_argtypes_DOCSTR) +# define _ctypes_CFuncPtr_argtypes_DOCSTR NULL +#endif +#if defined(_CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF) +# undef _CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF +# define _CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF {"argtypes", (getter)_ctypes_CFuncPtr_argtypes_get, (setter)_ctypes_CFuncPtr_argtypes_set, _ctypes_CFuncPtr_argtypes_DOCSTR}, +#else +# define _CTYPES_CFUNCPTR_ARGTYPES_GETSETDEF {"argtypes", (getter)_ctypes_CFuncPtr_argtypes_get, NULL, _ctypes_CFuncPtr_argtypes_DOCSTR}, +#endif + +static PyObject * +_ctypes_CFuncPtr_argtypes_get_impl(PyCFuncPtrObject *self); + +static PyObject * +_ctypes_CFuncPtr_argtypes_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ctypes_CFuncPtr_argtypes_get_impl((PyCFuncPtrObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(Simple_from_outparm__doc__, +"__ctypes_from_outparam__($self, /)\n" +"--\n" +"\n"); + +#define SIMPLE_FROM_OUTPARM_METHODDEF \ + {"__ctypes_from_outparam__", _PyCFunction_CAST(Simple_from_outparm), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, Simple_from_outparm__doc__}, + +static PyObject * +Simple_from_outparm_impl(PyObject *self, PyTypeObject *cls); + +static PyObject * +Simple_from_outparm(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "__ctypes_from_outparam__() takes no arguments"); + return NULL; + } + return Simple_from_outparm_impl(self, cls); +} +/*[clinic end generated code: output=a18d87239b6fb8ca input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/clinic/cfield.c.h b/Modules/_ctypes/clinic/cfield.c.h new file mode 100644 index 00000000000000..bbf108a1a07b67 --- /dev/null +++ b/Modules/_ctypes/clinic/cfield.c.h @@ -0,0 +1,114 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +static PyObject * +PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, + Py_ssize_t size, Py_ssize_t offset, Py_ssize_t index, + PyObject *bit_size_obj); + +static PyObject * +PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 6 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(index), &_Py_ID(bit_size), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"name", "type", "size", "offset", "index", "bit_size", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "CField", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[6]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 5; + PyObject *name; + PyObject *proto; + Py_ssize_t size; + Py_ssize_t offset; + Py_ssize_t index; + PyObject *bit_size_obj = Py_None; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 5, /*maxpos*/ 6, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!PyUnicode_Check(fastargs[0])) { + _PyArg_BadArgument("CField", "argument 'name'", "str", fastargs[0]); + goto exit; + } + name = fastargs[0]; + proto = fastargs[1]; + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(fastargs[2]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + size = ival; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(fastargs[3]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + offset = ival; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(fastargs[4]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + index = ival; + } + if (!noptargs) { + goto skip_optional_pos; + } + bit_size_obj = fastargs[5]; +skip_optional_pos: + return_value = PyCField_new_impl(type, name, proto, size, offset, index, bit_size_obj); + +exit: + return return_value; +} +/*[clinic end generated code: output=6b450bdd861571e7 input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 02f48a9ed55843..07049d0968c790 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -2,6 +2,26 @@ # include #endif +#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_typeobject.h" // _PyType_GetModuleState() + +// Do we support C99 complex types in ffi? +// For Apple's libffi, this must be determined at runtime (see gh-128156). +#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) +# include "../_complex.h" // complex +# if USING_APPLE_OS_LIBFFI && defined(__has_builtin) +# if __has_builtin(__builtin_available) +# define Py_FFI_COMPLEX_AVAILABLE __builtin_available(macOS 10.15, *) +# else +# define Py_FFI_COMPLEX_AVAILABLE 1 +# endif +# else +# define Py_FFI_COMPLEX_AVAILABLE 1 +# endif +#else +# define Py_FFI_COMPLEX_AVAILABLE 0 +#endif + #ifndef MS_WIN32 #define max(a, b) ((a) > (b) ? (a) : (b)) #define min(a, b) ((a) < (b) ? (a) : (b)) @@ -41,8 +61,8 @@ typedef struct { PyTypeObject *PyCArg_Type; PyTypeObject *PyCField_Type; PyTypeObject *PyCThunk_Type; - PyTypeObject *PyCStgDict_Type; PyTypeObject *StructParam_Type; + PyTypeObject *PyCType_Type; PyTypeObject *PyCStructType_Type; PyTypeObject *UnionType_Type; PyTypeObject *PyCPointerType_Type; @@ -59,21 +79,68 @@ typedef struct { #ifdef MS_WIN32 PyTypeObject *PyComError_Type; #endif + /* This dict maps ctypes types to POINTER types */ + PyObject *_ctypes_ptrtype_cache; + /* a callable object used for unpickling: + strong reference to _ctypes._unpickle() function */ + PyObject *_unpickle; + PyObject *array_cache; + PyObject *error_object_name; // callproc.c + PyObject *PyExc_ArgError; + PyObject *swapped_suffix; } ctypes_state; -extern ctypes_state global_state; -#define GLOBAL_STATE() (&global_state) +extern struct PyModuleDef _ctypesmodule; + + +static inline ctypes_state * +get_module_state(PyObject *module) +{ + void *state = _PyModule_GetState(module); + assert(state != NULL); + return (ctypes_state *)state; +} + +static inline ctypes_state * +get_module_state_by_class(PyTypeObject *cls) +{ + ctypes_state *state = (ctypes_state *)_PyType_GetModuleState(cls); + assert(state != NULL); + return state; +} +static inline ctypes_state * +get_module_state_by_def(PyTypeObject *cls) +{ + PyObject *mod = PyType_GetModuleByDef(cls, &_ctypesmodule); + assert(mod != NULL); + return get_module_state(mod); +} + + +extern PyType_Spec pyctype_type_spec; extern PyType_Spec carg_spec; extern PyType_Spec cfield_spec; extern PyType_Spec cthunk_spec; typedef struct tagPyCArgObject PyCArgObject; +#define _PyCArgObject_CAST(op) ((PyCArgObject *)(op)) + typedef struct tagCDataObject CDataObject; -typedef PyObject *(* GETFUNC)(void *, Py_ssize_t size); -typedef PyObject *(* SETFUNC)(void *, PyObject *value, Py_ssize_t size); -typedef PyCArgObject *(* PARAMFUNC)(CDataObject *obj); +#define _CDataObject_CAST(op) ((CDataObject *)(op)) + +// GETFUNC: convert the C value at *ptr* to Python object, return the object +// SETFUNC: write content of the PyObject *value* to the location at *ptr*; +// return a new reference to either *value*, or None for simple types +// (see _CTYPES_DEBUG_KEEP). +// Note that the *size* arg can have different meanings depending on context: +// for string-like arrays it's the size in bytes +// for int-style fields it's either the type size, or bitfiled info +// that can be unpacked using the LOW_BIT & NUM_BITS macros. +typedef PyObject *(* GETFUNC)(void *ptr, Py_ssize_t size); +typedef PyObject *(* SETFUNC)(void *ptr, PyObject *value, Py_ssize_t size); +typedef PyCArgObject *(* PARAMFUNC)(ctypes_state *st, CDataObject *obj); /* A default buffer in CDataObject, which can be used for small C types. If this buffer is too small, PyMem_Malloc will be called to create a larger one, @@ -125,6 +192,8 @@ typedef struct { ffi_type *ffi_restype; ffi_type *atypes[1]; } CThunkObject; + +#define _CThunkObject_CAST(op) ((CThunkObject *)(op)) #define CThunk_CheckExact(st, v) Py_IS_TYPE(v, st->PyCThunk_Type) typedef struct { @@ -144,7 +213,7 @@ typedef struct { CThunkObject *thunk; PyObject *callable; - /* These two fields will override the ones in the type's stgdict if + /* These two fields will override the ones in the type's stginfo if they are set */ PyObject *converters; PyObject *argtypes; @@ -158,17 +227,14 @@ typedef struct { PyObject *paramflags; } PyCFuncPtrObject; -extern PyTypeObject PyCStgDict_Type; -#define PyCStgDict_CheckExact(st, v) Py_IS_TYPE((v), (st)->PyCStgDict_Type) -#define PyCStgDict_Check(st, v) PyObject_TypeCheck((v), (st)->PyCStgDict_Type) +#define _PyCFuncPtrObject_CAST(op) ((PyCFuncPtrObject *)(op)) -extern int PyCStructUnionType_update_stgdict(PyObject *fields, PyObject *type, int isStruct); +extern int PyCStructUnionType_update_stginfo(PyObject *fields, PyObject *type, int isStruct); extern int PyType_stginfo(PyTypeObject *self, Py_ssize_t *psize, Py_ssize_t *palign, Py_ssize_t *plength); extern int PyObject_stginfo(PyObject *self, Py_ssize_t *psize, Py_ssize_t *palign, Py_ssize_t *plength); -extern PyTypeObject PyCData_Type; #define CDataObject_CheckExact(st, v) Py_IS_TYPE((v), (st)->PyCData_Type) #define CDataObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCData_Type) #define _CDataObject_HasExternalBuffer(v) ((v)->b_ptr != (char *)&(v)->b_value) @@ -178,19 +244,8 @@ extern PyTypeObject PyCData_Type; extern struct fielddesc *_ctypes_get_fielddesc(const char *fmt); - -extern PyObject * -PyCField_FromDesc(PyObject *desc, Py_ssize_t index, - Py_ssize_t *pfield_size, int bitsize, int *pbitofs, - Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int is_big_endian); - -extern PyObject *PyCData_AtAddress(PyObject *type, void *buf); -extern PyObject *PyCData_FromBytes(PyObject *type, char *data, Py_ssize_t length); - -extern PyTypeObject PyCArray_Type; -extern PyTypeObject PyCPointer_Type; -extern PyTypeObject PyCFuncPtr_Type; +extern PyObject *PyCData_AtAddress(ctypes_state *st, PyObject *type, void *buf); +extern PyObject *PyCData_FromBytes(ctypes_state *st, PyObject *type, char *data, Py_ssize_t length); #define PyCArrayTypeObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCArrayType_Type) #define ArrayObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCArray_Type) @@ -201,75 +256,60 @@ extern PyTypeObject PyCFuncPtr_Type; #define PyCStructTypeObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCStructType_Type) extern PyObject * -PyCArrayType_from_ctype(PyObject *itemtype, Py_ssize_t length); +PyCArrayType_from_ctype(ctypes_state *st, PyObject *itemtype, Py_ssize_t length); extern PyMethodDef _ctypes_module_methods[]; -extern CThunkObject *_ctypes_alloc_callback(PyObject *callable, +extern CThunkObject *_ctypes_alloc_callback(ctypes_state *st, + PyObject *callable, PyObject *converters, PyObject *restype, int flags); /* a table entry describing a predefined ctypes type */ struct fielddesc { char code; + ffi_type *pffi_type; /* always statically allocated */ SETFUNC setfunc; GETFUNC getfunc; - ffi_type *pffi_type; /* always statically allocated */ SETFUNC setfunc_swapped; GETFUNC getfunc_swapped; }; -typedef struct { +// Get all single-character type codes (for use in error messages) +extern char *_ctypes_get_simple_type_chars(void); + +typedef struct CFieldObject { PyObject_HEAD Py_ssize_t offset; Py_ssize_t size; Py_ssize_t index; /* Index into CDataObject's object array */ - PyObject *proto; /* a type or NULL */ + PyObject *proto; /* underlying ctype; must have StgInfo */ GETFUNC getfunc; /* getter function if proto is NULL */ SETFUNC setfunc; /* setter function if proto is NULL */ int anonymous; + + PyObject *name; /* exact PyUnicode */ } CFieldObject; -/* A subclass of PyDictObject, used as the instance dictionary of ctypes - metatypes */ -typedef struct { - PyDictObject dict; /* first part identical to PyDictObject */ -/* The size and align fields are unneeded, they are in ffi_type as well. As - an experiment shows, it's trivial to get rid of them, the only thing to - remember is that in PyCArrayType_new the ffi_type fields must be filled in - - so far it was unneeded because libffi doesn't support arrays at all - (because they are passed as pointers to function calls anyway). But it's - too much risk to change that now, and there are other fields which doesn't - belong into this structure anyway. Maybe in ctypes 2.0... (ctypes 2000?) -*/ - Py_ssize_t size; /* number of bytes */ - Py_ssize_t align; /* alignment requirements */ - Py_ssize_t length; /* number of fields */ - ffi_type ffi_type_pointer; - PyObject *proto; /* Only for Pointer/ArrayObject */ - SETFUNC setfunc; /* Only for simple objects */ - GETFUNC getfunc; /* Only for simple objects */ - PARAMFUNC paramfunc; +#define _CFieldObject_CAST(op) ((CFieldObject *)(op)) - /* Following fields only used by PyCFuncPtrType_Type instances */ - PyObject *argtypes; /* tuple of CDataObjects */ - PyObject *converters; /* tuple([t.from_param for t in argtypes]) */ - PyObject *restype; /* CDataObject or NULL */ - PyObject *checker; - int flags; /* calling convention and such */ +/**************************************************************** + StgInfo - /* pep3118 fields, pointers need PyMem_Free */ - char *format; - int ndim; - Py_ssize_t *shape; -/* Py_ssize_t *strides; */ /* unused in ctypes */ -/* Py_ssize_t *suboffsets; */ /* unused in ctypes */ + Since Python 3.13, ctypes-specific type information is stored in the + corresponding type object, in a `StgInfo` struct accessed by the helpers + below. + Before that, each type's `tp_dict` was set to a dict *subclass* that included + the fields that are now in StgInfo. The mechanism was called "StgDict"; a few + references to that name might remain. -} StgDictObject; + Functions for accessing StgInfo are `static inline` for performance; + see later in this file. -/**************************************************************** - StgDictObject fields + **************************************************************** + + StgInfo fields setfunc and getfunc is only set for simple data types, it is copied from the corresponding fielddesc entry. These are functions to set and get the value @@ -280,11 +320,11 @@ typedef struct { object. Probably all the magic ctypes methods (like from_param) should have C - callable wrappers in the StgDictObject. For simple data type, for example, + callable wrappers in the StgInfo. For simple data type, for example, the fielddesc table could have entries for C codec from_param functions or other methods as well, if a subtype overrides this method in Python at construction time, or assigns to it later, tp_setattro should update the - StgDictObject function to a generic one. + StgInfo function to a generic one. Currently, PyCFuncPtr types have 'converters' and 'checker' entries in their type dict. They are only used to cache attributes from other entries, which @@ -308,17 +348,40 @@ typedef struct { *****************************************************************/ -/* May return NULL, but does not set an exception! */ -extern StgDictObject *PyType_stgdict(PyObject *obj); +typedef struct { + int initialized; + Py_ssize_t size; /* number of bytes */ + Py_ssize_t align; /* alignment requirements */ + Py_ssize_t length; /* number of fields */ + ffi_type ffi_type_pointer; + PyObject *proto; /* Only for Pointer/ArrayObject */ + SETFUNC setfunc; /* Only for simple objects */ + GETFUNC getfunc; /* Only for simple objects */ + PARAMFUNC paramfunc; -/* May return NULL, but does not set an exception! */ -extern StgDictObject *PyObject_stgdict(PyObject *self); + /* Following fields only used by PyCFuncPtrType_Type instances */ + PyObject *argtypes; /* tuple of CDataObjects */ + PyObject *converters; /* tuple([t.from_param for t in argtypes]) */ + PyObject *restype; /* CDataObject or NULL */ + PyObject *checker; + PyObject *module; + int flags; /* calling convention and such */ -extern int PyCStgDict_clone(StgDictObject *src, StgDictObject *dst); + /* pep3118 fields, pointers need PyMem_Free */ + char *format; + int ndim; + Py_ssize_t *shape; +/* Py_ssize_t *strides; */ /* unused in ctypes */ +/* Py_ssize_t *suboffsets; */ /* unused in ctypes */ +} StgInfo; + +extern int PyCStgInfo_clone(StgInfo *dst_info, StgInfo *src_info); +extern void ctype_clear_stginfo(StgInfo *info); typedef int(* PPROC)(void); -PyObject *_ctypes_callproc(PPROC pProc, +PyObject *_ctypes_callproc(ctypes_state *st, + PPROC pProc, PyObject *arguments, #ifdef MS_WIN32 IUnknown *pIUnk, @@ -339,8 +402,6 @@ PyObject *_ctypes_callproc(PPROC pProc, #define TYPEFLAG_ISPOINTER 0x100 #define TYPEFLAG_HASPOINTER 0x200 -#define TYPEFLAG_HASUNION 0x400 -#define TYPEFLAG_HASBITFIELD 0x800 #define DICTFLAG_FINAL 0x1000 @@ -359,20 +420,28 @@ struct tagPyCArgObject { double d; float f; void *p; +#if defined(Py_HAVE_C_COMPLEX) && defined(Py_FFI_SUPPORT_C_COMPLEX) + double complex C; + float complex E; + long double complex F; +#endif } value; PyObject *obj; Py_ssize_t size; /* for the 'V' tag */ }; +#define _PyCArgObject_CAST(op) ((PyCArgObject *)(op)) + #define PyCArg_CheckExact(st, v) Py_IS_TYPE(v, st->PyCArg_Type) -extern PyCArgObject *PyCArgObject_new(void); +extern PyCArgObject *PyCArgObject_new(ctypes_state *st); extern PyObject * -PyCData_get(PyObject *type, GETFUNC getfunc, PyObject *src, +PyCData_get(ctypes_state *st, PyObject *type, GETFUNC getfunc, PyObject *src, Py_ssize_t index, Py_ssize_t size, char *ptr); extern int -PyCData_set(PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, +PyCData_set(ctypes_state *st, + PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, Py_ssize_t index, Py_ssize_t size, char *ptr); extern void _ctypes_extend_error(PyObject *exc_class, const char *fmt, ...); @@ -383,30 +452,17 @@ struct basespec { char *adr; }; -extern char basespec_string[]; - -extern ffi_type *_ctypes_get_ffi_type(PyObject *obj); - -/* exception classes */ -extern PyObject *PyExc_ArgError; - -extern char *_ctypes_conversion_encoding; -extern char *_ctypes_conversion_errors; - +extern ffi_type *_ctypes_get_ffi_type(ctypes_state *st, PyObject *obj); extern void _ctypes_free_closure(void *); extern void *_ctypes_alloc_closure(void); -extern PyObject *PyCData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr); -extern char *_ctypes_alloc_format_string(const char *prefix, const char *suffix); -extern char *_ctypes_alloc_format_string_with_shape(int ndim, - const Py_ssize_t *shape, - const char *prefix, const char *suffix); +extern PyObject *PyCData_FromBaseObj(ctypes_state *st, PyObject *type, + PyObject *base, Py_ssize_t index, char *adr); -extern int _ctypes_simple_instance(PyObject *obj); +extern int _ctypes_simple_instance(ctypes_state *st, PyObject *obj); -extern PyObject *_ctypes_ptrtype_cache; -PyObject *_ctypes_get_errobj(int **pspace); +PyObject *_ctypes_get_errobj(ctypes_state *st, int **pspace); #ifdef USING_MALLOC_CLOSURE_DOT_C void Py_ffi_closure_free(void *p); @@ -416,8 +472,149 @@ void *Py_ffi_closure_alloc(size_t size, void** codeloc); #define Py_ffi_closure_alloc ffi_closure_alloc #endif + +/**************************************************************** + * Accessing StgInfo -- these are inlined for performance reasons. + */ + +// `PyStgInfo_From**` functions get a PyCTypeDataObject. +// These return -1 on error, 0 if "not found", 1 on OK. +// (Currently, these do not return -1 in practice. This might change +// in the future.) + +// +// Common helper: +static inline int +_stginfo_from_type(ctypes_state *state, PyTypeObject *type, StgInfo **result) +{ + *result = NULL; + if (!PyObject_IsInstance((PyObject *)type, (PyObject *)state->PyCType_Type)) { + // not a ctypes class. + return 0; + } + StgInfo *info = PyObject_GetTypeData((PyObject *)type, state->PyCType_Type); + assert(info != NULL); + if (!info->initialized) { + // StgInfo is not initialized. This happens in abstract classes. + return 0; + } + *result = info; + return 1; +} +// from a type: +static inline int +PyStgInfo_FromType(ctypes_state *state, PyObject *type, StgInfo **result) +{ + return _stginfo_from_type(state, (PyTypeObject *)type, result); +} +// from an instance: +static inline int +PyStgInfo_FromObject(ctypes_state *state, PyObject *obj, StgInfo **result) +{ + return _stginfo_from_type(state, Py_TYPE(obj), result); +} +// from either a type or an instance: +static inline int +PyStgInfo_FromAny(ctypes_state *state, PyObject *obj, StgInfo **result) +{ + if (PyType_Check(obj)) { + return _stginfo_from_type(state, (PyTypeObject *)obj, result); + } + return _stginfo_from_type(state, Py_TYPE(obj), result); +} + +/* A variant of PyStgInfo_FromType that doesn't need the state, + * so it can be called from finalization functions when the module + * state is torn down. + */ +static inline StgInfo * +_PyStgInfo_FromType_NoState(PyObject *type) +{ + PyTypeObject *PyCType_Type; + if (PyType_GetBaseByToken(Py_TYPE(type), &pyctype_type_spec, &PyCType_Type) < 0) { + return NULL; + } + if (PyCType_Type == NULL) { + PyErr_Format(PyExc_TypeError, "expected a ctypes type, got '%N'", type); + return NULL; + } + + StgInfo *info = PyObject_GetTypeData(type, PyCType_Type); + Py_DECREF(PyCType_Type); + return info; +} + +// Initialize StgInfo on a newly created type +static inline StgInfo * +PyStgInfo_Init(ctypes_state *state, PyTypeObject *type) +{ + if (!PyObject_IsInstance((PyObject *)type, (PyObject *)state->PyCType_Type)) { + PyErr_Format(PyExc_SystemError, + "'%s' is not a ctypes class.", + type->tp_name); + return NULL; + } + StgInfo *info = PyObject_GetTypeData((PyObject *)type, state->PyCType_Type); + if (info->initialized) { + PyErr_Format(PyExc_SystemError, + "StgInfo of '%s' is already initialized.", + type->tp_name); + return NULL; + } + PyObject *module = PyType_GetModule(state->PyCType_Type); + if (!module) { + return NULL; + } + info->module = Py_NewRef(module); + + info->initialized = 1; + return info; +} + +/* See discussion in gh-128490. The plan here is to eventually use a per-object + * lock rather than a critical section, but that work is for later. */ +#ifdef Py_GIL_DISABLED +# define LOCK_PTR(self) Py_BEGIN_CRITICAL_SECTION(self) +# define UNLOCK_PTR(self) Py_END_CRITICAL_SECTION() +#else /* - Local Variables: - compile-command: "python setup.py -q build install --home ~" - End: -*/ + * Dummy functions instead of macros so that 'self' can be + * unused in the caller without triggering a compiler warning. + */ +static inline void LOCK_PTR(CDataObject *Py_UNUSED(self)) {} +static inline void UNLOCK_PTR(CDataObject *Py_UNUSED(self)) {} +#endif + +static inline void +locked_memcpy_to(CDataObject *self, void *buf, Py_ssize_t size) +{ + LOCK_PTR(self); + (void)memcpy(self->b_ptr, buf, size); + UNLOCK_PTR(self); +} + +static inline void +locked_memcpy_from(void *buf, CDataObject *self, Py_ssize_t size) +{ + LOCK_PTR(self); + (void)memcpy(buf, self->b_ptr, size); + UNLOCK_PTR(self); +} + +static inline void * +locked_deref(CDataObject *self) +{ + void *ptr; + LOCK_PTR(self); + ptr = *(void **)self->b_ptr; + UNLOCK_PTR(self); + return ptr; +} + +static inline void +locked_deref_assign(CDataObject *self, void *new_ptr) +{ + LOCK_PTR(self); + *(void **)self->b_ptr = new_ptr; + UNLOCK_PTR(self); +} diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 32ee414a7a0cdd..5ca5b62427600d 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -16,201 +16,64 @@ #endif #include "ctypes.h" -/******************************************************************/ -/* - StdDict - a dictionary subclass, containing additional C accessible fields - - XXX blabla more -*/ - -/* Seems we need this, otherwise we get problems when calling - * PyDict_SetItem() (ma_lookup is NULL) +/* This file relates to StgInfo -- type-specific information for ctypes. + * See ctypes.h for details. */ -static int -PyCStgDict_init(StgDictObject *self, PyObject *args, PyObject *kwds) -{ - if (PyDict_Type.tp_init((PyObject *)self, args, kwds) < 0) - return -1; - self->format = NULL; - self->ndim = 0; - self->shape = NULL; - return 0; -} - -static int -PyCStgDict_clear(StgDictObject *self) -{ - Py_CLEAR(self->proto); - Py_CLEAR(self->argtypes); - Py_CLEAR(self->converters); - Py_CLEAR(self->restype); - Py_CLEAR(self->checker); - return 0; -} - -static void -PyCStgDict_dealloc(StgDictObject *self) -{ - PyCStgDict_clear(self); - PyMem_Free(self->format); - PyMem_Free(self->shape); - PyMem_Free(self->ffi_type_pointer.elements); - PyDict_Type.tp_dealloc((PyObject *)self); -} - -static PyObject * -PyCStgDict_sizeof(StgDictObject *self, void *unused) -{ - Py_ssize_t res; - - res = _PyDict_SizeOf((PyDictObject *)self); - res += sizeof(StgDictObject) - sizeof(PyDictObject); - if (self->format) - res += strlen(self->format) + 1; - res += self->ndim * sizeof(Py_ssize_t); - if (self->ffi_type_pointer.elements) - res += (self->length + 1) * sizeof(ffi_type *); - return PyLong_FromSsize_t(res); -} int -PyCStgDict_clone(StgDictObject *dst, StgDictObject *src) +PyCStgInfo_clone(StgInfo *dst_info, StgInfo *src_info) { - char *d, *s; Py_ssize_t size; - PyCStgDict_clear(dst); - PyMem_Free(dst->ffi_type_pointer.elements); - PyMem_Free(dst->format); - dst->format = NULL; - PyMem_Free(dst->shape); - dst->shape = NULL; - dst->ffi_type_pointer.elements = NULL; - - d = (char *)dst; - s = (char *)src; - memcpy(d + sizeof(PyDictObject), - s + sizeof(PyDictObject), - sizeof(StgDictObject) - sizeof(PyDictObject)); - - Py_XINCREF(dst->proto); - Py_XINCREF(dst->argtypes); - Py_XINCREF(dst->converters); - Py_XINCREF(dst->restype); - Py_XINCREF(dst->checker); - - if (src->format) { - dst->format = PyMem_Malloc(strlen(src->format) + 1); - if (dst->format == NULL) { + ctype_clear_stginfo(dst_info); + PyMem_Free(dst_info->ffi_type_pointer.elements); + PyMem_Free(dst_info->format); + dst_info->format = NULL; + PyMem_Free(dst_info->shape); + dst_info->shape = NULL; + dst_info->ffi_type_pointer.elements = NULL; + + memcpy(dst_info, src_info, sizeof(StgInfo)); + + Py_XINCREF(dst_info->proto); + Py_XINCREF(dst_info->argtypes); + Py_XINCREF(dst_info->converters); + Py_XINCREF(dst_info->restype); + Py_XINCREF(dst_info->checker); + Py_XINCREF(dst_info->module); + + if (src_info->format) { + dst_info->format = PyMem_Malloc(strlen(src_info->format) + 1); + if (dst_info->format == NULL) { PyErr_NoMemory(); return -1; } - strcpy(dst->format, src->format); + strcpy(dst_info->format, src_info->format); } - if (src->shape) { - dst->shape = PyMem_Malloc(sizeof(Py_ssize_t) * src->ndim); - if (dst->shape == NULL) { + if (src_info->shape) { + dst_info->shape = PyMem_Malloc(sizeof(Py_ssize_t) * src_info->ndim); + if (dst_info->shape == NULL) { PyErr_NoMemory(); return -1; } - memcpy(dst->shape, src->shape, - sizeof(Py_ssize_t) * src->ndim); + memcpy(dst_info->shape, src_info->shape, + sizeof(Py_ssize_t) * src_info->ndim); } - if (src->ffi_type_pointer.elements == NULL) + if (src_info->ffi_type_pointer.elements == NULL) return 0; - size = sizeof(ffi_type *) * (src->length + 1); - dst->ffi_type_pointer.elements = PyMem_Malloc(size); - if (dst->ffi_type_pointer.elements == NULL) { + size = sizeof(ffi_type *) * (src_info->length + 1); + dst_info->ffi_type_pointer.elements = PyMem_Malloc(size); + if (dst_info->ffi_type_pointer.elements == NULL) { PyErr_NoMemory(); return -1; } - memcpy(dst->ffi_type_pointer.elements, - src->ffi_type_pointer.elements, + memcpy(dst_info->ffi_type_pointer.elements, + src_info->ffi_type_pointer.elements, size); return 0; } -static struct PyMethodDef PyCStgDict_methods[] = { - {"__sizeof__", (PyCFunction)PyCStgDict_sizeof, METH_NOARGS}, - {NULL, NULL} /* sentinel */ -}; - -PyTypeObject PyCStgDict_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "StgDict", - sizeof(StgDictObject), - 0, - (destructor)PyCStgDict_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyCStgDict_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)PyCStgDict_init, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ -}; - -/* May return NULL, but does not set an exception! */ -StgDictObject * -PyType_stgdict(PyObject *obj) -{ - PyTypeObject *type; - - if (!PyType_Check(obj)) { - return NULL; - } - ctypes_state *st = GLOBAL_STATE(); - type = (PyTypeObject *)obj; - if (!type->tp_dict || !PyCStgDict_CheckExact(st, type->tp_dict)) { - return NULL; - } - return (StgDictObject *)type->tp_dict; -} - -/* May return NULL, but does not set an exception! */ -/* - This function should be as fast as possible, so we don't call PyType_stgdict - above but inline the code, and avoid the PyType_Check(). -*/ -StgDictObject * -PyObject_stgdict(PyObject *self) -{ - PyTypeObject *type = Py_TYPE(self); - ctypes_state *st = GLOBAL_STATE(); - if (!type->tp_dict || !PyCStgDict_CheckExact(st, type->tp_dict)) { - return NULL; - } - return (StgDictObject *)type->tp_dict; -} - /* descr is the descriptor for a field marked as anonymous. Get all the _fields_ descriptors from descr->proto, create new descriptors with offset and index adjusted, and stuff them into type. @@ -231,7 +94,7 @@ MakeFields(PyObject *type, CFieldObject *descr, if (fieldlist == NULL) return -1; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_class(Py_TYPE(descr)); PyTypeObject *cfield_tp = st->PyCField_Type; for (i = 0; i < PySequence_Fast_GET_SIZE(fieldlist); ++i) { PyObject *pair = PySequence_Fast_GET_ITEM(fieldlist, i); /* borrowed */ @@ -312,7 +175,7 @@ MakeAnonFields(PyObject *type) if (anon_names == NULL) return -1; - ctypes_state *st = GLOBAL_STATE(); + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); PyTypeObject *cfield_tp = st->PyCField_Type; for (i = 0; i < PySequence_Fast_GET_SIZE(anon_names); ++i) { PyObject *fname = PySequence_Fast_GET_ITEM(anon_names, i); /* borrowed */ @@ -347,391 +210,230 @@ MakeAnonFields(PyObject *type) return 0; } -/* - Allocate a memory block for a pep3118 format string, copy prefix (if - non-null) into it and append `{padding}x` to the end. - Returns NULL on failure, with the error indicator set. -*/ -char * -_ctypes_alloc_format_padding(const char *prefix, Py_ssize_t padding) -{ - /* int64 decimal characters + x + null */ - char buf[19 + 1 + 1]; - - assert(padding > 0); - - if (padding == 1) { - /* Use x instead of 1x, for brevity */ - return _ctypes_alloc_format_string(prefix, "x"); - } - - int ret = PyOS_snprintf(buf, sizeof(buf), "%zdx", padding); (void)ret; - assert(0 <= ret && ret < (Py_ssize_t)sizeof(buf)); - return _ctypes_alloc_format_string(prefix, buf); -} - /* Retrieve the (optional) _pack_ attribute from a type, the _fields_ attribute, - and create an StgDictObject. Used for Structure and Union subclasses. + and initialize StgInfo. Used for Structure and Union subclasses. */ int -PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct) +PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct) { - StgDictObject *stgdict, *basedict; - Py_ssize_t len, offset, size, align, i; - Py_ssize_t union_size, total_align, aligned_size; - Py_ssize_t field_size = 0; - int bitofs; PyObject *tmp; - int pack; - int forced_alignment = 1; Py_ssize_t ffi_ofs; - int big_endian; int arrays_seen = 0; - if (fields == NULL) + int retval = -1; + // The following are NULL or hold strong references. + // They're cleared on error. + PyObject *layout_fields = NULL; + PyObject *layout = NULL; + PyObject *format_spec_obj = NULL; + + if (fields == NULL) { return 0; + } - int rc = PyObject_HasAttrWithError(type, &_Py_ID(_swappedbytes_)); - if (rc < 0) { - return -1; + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + StgInfo *stginfo; + if (PyStgInfo_FromType(st, type, &stginfo) < 0) { + goto error; } - if (rc) { - big_endian = !PY_BIG_ENDIAN; + if (!stginfo) { + PyErr_SetString(PyExc_TypeError, + "ctypes state is not initialized"); + goto error; } - else { - big_endian = PY_BIG_ENDIAN; + PyObject *base = (PyObject *)((PyTypeObject *)type)->tp_base; + StgInfo *baseinfo; + if (PyStgInfo_FromType(st, base, &baseinfo) < 0) { + goto error; } - if (PyObject_GetOptionalAttr(type, &_Py_ID(_pack_), &tmp) < 0) { - return -1; - } - if (tmp) { - pack = PyLong_AsInt(tmp); - Py_DECREF(tmp); - if (pack < 0) { - if (!PyErr_Occurred() || - PyErr_ExceptionMatches(PyExc_TypeError) || - PyErr_ExceptionMatches(PyExc_OverflowError)) - { - PyErr_SetString(PyExc_ValueError, - "_pack_ must be a non-negative integer"); - } - return -1; + /* If this structure/union is already marked final we cannot assign + _fields_ anymore. */ + + if (stginfo->flags & DICTFLAG_FINAL) {/* is final ? */ + PyErr_SetString(PyExc_AttributeError, + "_fields_ is final"); + goto error; + } + + PyObject *layout_func = _PyImport_GetModuleAttrString("ctypes._layout", + "get_layout"); + if (!layout_func) { + goto error; + } + PyObject *kwnames = PyTuple_Pack( + 2, + &_Py_ID(is_struct), + &_Py_ID(base)); + if (!kwnames) { + goto error; + } + layout = PyObject_Vectorcall( + layout_func, + 1 + (PyObject*[]){ + NULL, + /* positional args */ + type, + fields, + /* keyword args */ + isStruct ? Py_True : Py_False, + baseinfo ? base : Py_None}, + 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames); + Py_DECREF(kwnames); + Py_DECREF(layout_func); + fields = NULL; // a borrowed reference we won't be using again + if (!layout) { + goto error; + } + + tmp = PyObject_GetAttr(layout, &_Py_ID(align)); + if (!tmp) { + goto error; + } + Py_ssize_t total_align = PyLong_AsSsize_t(tmp); + Py_DECREF(tmp); + if (total_align < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "align must be a non-negative integer"); } - } - else { - /* Setting `_pack_ = 0` amounts to using the default alignment */ - pack = 0; + goto error; } - if (PyObject_GetOptionalAttr(type, &_Py_ID(_align_), &tmp) < 0) { - return -1; + tmp = PyObject_GetAttr(layout, &_Py_ID(size)); + if (!tmp) { + goto error; } - if (tmp) { - forced_alignment = PyLong_AsInt(tmp); - Py_DECREF(tmp); - if (forced_alignment < 0) { - if (!PyErr_Occurred() || - PyErr_ExceptionMatches(PyExc_TypeError) || - PyErr_ExceptionMatches(PyExc_OverflowError)) - { - PyErr_SetString(PyExc_ValueError, - "_align_ must be a non-negative integer"); - } - return -1; + Py_ssize_t total_size = PyLong_AsSsize_t(tmp); + Py_DECREF(tmp); + if (total_size < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "size must be a non-negative integer"); } - } - else { - /* Setting `_align_ = 0` amounts to using the default alignment */ - forced_alignment = 1; + goto error; } - len = PySequence_Size(fields); - if (len == -1) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) { - PyErr_SetString(PyExc_TypeError, - "'_fields_' must be a sequence of pairs"); - } - return -1; + format_spec_obj = PyObject_GetAttr(layout, &_Py_ID(format_spec)); + if (!format_spec_obj) { + goto error; } - - stgdict = PyType_stgdict(type); - if (!stgdict) { - PyErr_SetString(PyExc_TypeError, - "ctypes state is not initialized"); - return -1; + Py_ssize_t format_spec_size; + const char *format_spec = PyUnicode_AsUTF8AndSize(format_spec_obj, + &format_spec_size); + if (!format_spec) { + goto error; } - /* If this structure/union is already marked final we cannot assign - _fields_ anymore. */ - if (stgdict->flags & DICTFLAG_FINAL) {/* is final ? */ - PyErr_SetString(PyExc_AttributeError, - "_fields_ is final"); - return -1; + if (stginfo->format) { + PyMem_Free(stginfo->format); + stginfo->format = NULL; + } + stginfo->format = PyMem_Malloc(format_spec_size + 1); + if (!stginfo->format) { + PyErr_NoMemory(); + goto error; } + memcpy(stginfo->format, format_spec, format_spec_size + 1); - if (stgdict->format) { - PyMem_Free(stgdict->format); - stgdict->format = NULL; + PyObject *layout_fields_obj = PyObject_GetAttr(layout, &_Py_ID(fields)); + if (!layout_fields_obj) { + goto error; } + layout_fields = PySequence_Tuple(layout_fields_obj); + Py_DECREF(layout_fields_obj); + if (!layout_fields) { + goto error; + } + Py_CLEAR(layout); - if (stgdict->ffi_type_pointer.elements) - PyMem_Free(stgdict->ffi_type_pointer.elements); + Py_ssize_t len = PyTuple_GET_SIZE(layout_fields); - basedict = PyType_stgdict((PyObject *)((PyTypeObject *)type)->tp_base); - if (basedict) { - stgdict->flags |= (basedict->flags & - (TYPEFLAG_HASUNION | TYPEFLAG_HASBITFIELD)); - } - if (!isStruct) { - stgdict->flags |= TYPEFLAG_HASUNION; + if (stginfo->ffi_type_pointer.elements) { + PyMem_Free(stginfo->ffi_type_pointer.elements); + stginfo->ffi_type_pointer.elements = NULL; } - if (basedict) { - size = offset = basedict->size; - align = basedict->align; - union_size = 0; - total_align = align ? align : 1; - total_align = max(total_align, forced_alignment); - stgdict->ffi_type_pointer.type = FFI_TYPE_STRUCT; - stgdict->ffi_type_pointer.elements = PyMem_New(ffi_type *, basedict->length + len + 1); - if (stgdict->ffi_type_pointer.elements == NULL) { + + if (baseinfo) { + stginfo->ffi_type_pointer.type = FFI_TYPE_STRUCT; + stginfo->ffi_type_pointer.elements = PyMem_New(ffi_type *, baseinfo->length + len + 1); + if (stginfo->ffi_type_pointer.elements == NULL) { PyErr_NoMemory(); - return -1; + goto error; } - memset(stgdict->ffi_type_pointer.elements, 0, - sizeof(ffi_type *) * (basedict->length + len + 1)); - if (basedict->length > 0) { - memcpy(stgdict->ffi_type_pointer.elements, - basedict->ffi_type_pointer.elements, - sizeof(ffi_type *) * (basedict->length)); + memset(stginfo->ffi_type_pointer.elements, 0, + sizeof(ffi_type *) * (baseinfo->length + len + 1)); + if (baseinfo->length > 0) { + memcpy(stginfo->ffi_type_pointer.elements, + baseinfo->ffi_type_pointer.elements, + sizeof(ffi_type *) * (baseinfo->length)); } - ffi_ofs = basedict->length; + ffi_ofs = baseinfo->length; } else { - offset = 0; - size = 0; - align = 0; - union_size = 0; - total_align = forced_alignment; - stgdict->ffi_type_pointer.type = FFI_TYPE_STRUCT; - stgdict->ffi_type_pointer.elements = PyMem_New(ffi_type *, len + 1); - if (stgdict->ffi_type_pointer.elements == NULL) { + stginfo->ffi_type_pointer.type = FFI_TYPE_STRUCT; + stginfo->ffi_type_pointer.elements = PyMem_New(ffi_type *, len + 1); + if (stginfo->ffi_type_pointer.elements == NULL) { PyErr_NoMemory(); - return -1; + goto error; } - memset(stgdict->ffi_type_pointer.elements, 0, + memset(stginfo->ffi_type_pointer.elements, 0, sizeof(ffi_type *) * (len + 1)); ffi_ofs = 0; } - assert(stgdict->format == NULL); - if (isStruct) { - stgdict->format = _ctypes_alloc_format_string(NULL, "T{"); - } else { - /* PEP3118 doesn't support union. Use 'B' for bytes. */ - stgdict->format = _ctypes_alloc_format_string(NULL, "B"); - } - if (stgdict->format == NULL) - return -1; - - ctypes_state *st = GLOBAL_STATE(); - for (i = 0; i < len; ++i) { - PyObject *name = NULL, *desc = NULL; - PyObject *pair = PySequence_GetItem(fields, i); - PyObject *prop; - StgDictObject *dict; - int bitsize = 0; - - if (!pair || !PyArg_ParseTuple(pair, "UO|i", &name, &desc, &bitsize)) { - PyErr_SetString(PyExc_TypeError, - "'_fields_' must be a sequence of (name, C type) pairs"); - Py_XDECREF(pair); - return -1; - } - if (PyCArrayTypeObject_Check(st, desc)) { - arrays_seen = 1; - } - dict = PyType_stgdict(desc); - if (dict == NULL) { - Py_DECREF(pair); + for (Py_ssize_t i = 0; i < len; ++i) { + PyObject *prop_obj = PyTuple_GET_ITEM(layout_fields, i); + assert(prop_obj); + if (!PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type)) { PyErr_Format(PyExc_TypeError, - "second item in _fields_ tuple (index %zd) must be a C type", - i); - return -1; - } - stgdict->ffi_type_pointer.elements[ffi_ofs + i] = &dict->ffi_type_pointer; - if (dict->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) - stgdict->flags |= TYPEFLAG_HASPOINTER; - stgdict->flags |= dict->flags & (TYPEFLAG_HASUNION | TYPEFLAG_HASBITFIELD); - dict->flags |= DICTFLAG_FINAL; /* mark field type final */ - if (PyTuple_Size(pair) == 3) { /* bits specified */ - stgdict->flags |= TYPEFLAG_HASBITFIELD; - switch(dict->ffi_type_pointer.type) { - case FFI_TYPE_UINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT64: - case FFI_TYPE_UINT64: - break; - - case FFI_TYPE_SINT8: - case FFI_TYPE_SINT16: - case FFI_TYPE_SINT32: - if (dict->getfunc != _ctypes_get_fielddesc("c")->getfunc - && dict->getfunc != _ctypes_get_fielddesc("u")->getfunc - ) - break; - /* else fall through */ - default: - PyErr_Format(PyExc_TypeError, - "bit fields not allowed for type %s", - ((PyTypeObject *)desc)->tp_name); - Py_DECREF(pair); - return -1; - } - if (bitsize <= 0 || bitsize > dict->size * 8) { - PyErr_SetString(PyExc_ValueError, - "number of bits invalid for bit field"); - Py_DECREF(pair); - return -1; - } - } else - bitsize = 0; - - if (isStruct) { - const char *fieldfmt = dict->format ? dict->format : "B"; - const char *fieldname = PyUnicode_AsUTF8(name); - char *ptr; - Py_ssize_t len; - char *buf; - Py_ssize_t last_size = size; - Py_ssize_t padding; - - if (fieldname == NULL) - { - Py_DECREF(pair); - return -1; - } - - /* construct the field now, as `prop->offset` is `offset` with - corrected alignment */ - prop = PyCField_FromDesc(desc, i, - &field_size, bitsize, &bitofs, - &size, &offset, &align, - pack, big_endian); - if (prop == NULL) { - Py_DECREF(pair); - return -1; - } + "fields must be of type CField, got %T", prop_obj); + goto error; - /* number of bytes between the end of the last field and the start - of this one */ - padding = ((CFieldObject *)prop)->offset - last_size; - - if (padding > 0) { - ptr = stgdict->format; - stgdict->format = _ctypes_alloc_format_padding(ptr, padding); - PyMem_Free(ptr); - if (stgdict->format == NULL) { - Py_DECREF(pair); - Py_DECREF(prop); - return -1; - } - } - - len = strlen(fieldname) + strlen(fieldfmt); - - buf = PyMem_Malloc(len + 2 + 1); - if (buf == NULL) { - Py_DECREF(pair); - Py_DECREF(prop); - PyErr_NoMemory(); - return -1; - } - sprintf(buf, "%s:%s:", fieldfmt, fieldname); - - ptr = stgdict->format; - if (dict->shape != NULL) { - stgdict->format = _ctypes_alloc_format_string_with_shape( - dict->ndim, dict->shape, stgdict->format, buf); - } else { - stgdict->format = _ctypes_alloc_format_string(stgdict->format, buf); - } - PyMem_Free(ptr); - PyMem_Free(buf); - - if (stgdict->format == NULL) { - Py_DECREF(pair); - Py_DECREF(prop); - return -1; - } - } else /* union */ { - size = 0; - offset = 0; - align = 0; - prop = PyCField_FromDesc(desc, i, - &field_size, bitsize, &bitofs, - &size, &offset, &align, - pack, big_endian); - if (prop == NULL) { - Py_DECREF(pair); - return -1; - } - union_size = max(size, union_size); } - total_align = max(align, total_align); + CFieldObject *prop = (CFieldObject *)prop_obj; // borrow from prop_obj - if (-1 == PyObject_SetAttr(type, name, prop)) { - Py_DECREF(prop); - Py_DECREF(pair); - return -1; + if (prop->index != i) { + PyErr_Format(PyExc_ValueError, + "field %R index mismatch (expected %zd, got %zd)", + prop->name, i, prop->index); + goto error; } - Py_DECREF(pair); - Py_DECREF(prop); - } - if (!isStruct) { - size = union_size; - } + if (PyCArrayTypeObject_Check(st, prop->proto)) { + arrays_seen = 1; + } - /* Adjust the size according to the alignment requirements */ - aligned_size = ((size + total_align - 1) / total_align) * total_align; + StgInfo *info; + if (PyStgInfo_FromType(st, prop->proto, &info) < 0) { + goto error; + } + assert(info); - if (isStruct) { - char *ptr; - Py_ssize_t padding; + stginfo->ffi_type_pointer.elements[ffi_ofs + i] = &info->ffi_type_pointer; + if (info->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) + stginfo->flags |= TYPEFLAG_HASPOINTER; + info->flags |= DICTFLAG_FINAL; /* mark field type final */ - /* Pad up to the full size of the struct */ - padding = aligned_size - size; - if (padding > 0) { - ptr = stgdict->format; - stgdict->format = _ctypes_alloc_format_padding(ptr, padding); - PyMem_Free(ptr); - if (stgdict->format == NULL) { - return -1; - } + if (-1 == PyObject_SetAttr(type, prop->name, prop_obj)) { + goto error; } - - ptr = stgdict->format; - stgdict->format = _ctypes_alloc_format_string(stgdict->format, "}"); - PyMem_Free(ptr); - if (stgdict->format == NULL) - return -1; } - stgdict->ffi_type_pointer.alignment = Py_SAFE_DOWNCAST(total_align, + stginfo->ffi_type_pointer.alignment = Py_SAFE_DOWNCAST(total_align, Py_ssize_t, unsigned short); - stgdict->ffi_type_pointer.size = aligned_size; + stginfo->ffi_type_pointer.size = total_size; - stgdict->size = aligned_size; - stgdict->align = total_align; - stgdict->length = ffi_ofs + len; + stginfo->size = total_size; + stginfo->align = total_align; + stginfo->length = ffi_ofs + len; /* * The value of MAX_STRUCT_SIZE depends on the platform Python is running on. */ -#if defined(__aarch64__) || defined(__arm__) || defined(_M_ARM64) +#if defined(__aarch64__) || defined(__arm__) || defined(_M_ARM64) || defined(__sparc__) # define MAX_STRUCT_SIZE 32 #elif defined(__powerpc64__) # define MAX_STRUCT_SIZE 64 @@ -739,7 +441,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct # define MAX_STRUCT_SIZE 16 #endif - if (arrays_seen && (size <= MAX_STRUCT_SIZE)) { + if (arrays_seen && (total_size <= MAX_STRUCT_SIZE)) { /* * See bpo-22273 and gh-110190. Arrays are normally treated as * pointers, which is fine when an array name is being passed as @@ -814,45 +516,35 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct Py_ssize_t struct_index = 0; /* index into dummy structs */ /* first pass to see how much memory to allocate */ - for (i = 0; i < len; ++i) { - PyObject *name, *desc; - PyObject *pair = PySequence_GetItem(fields, i); - StgDictObject *dict; - int bitsize = 0; - - if (pair == NULL) { - return -1; - } - if (!PyArg_ParseTuple(pair, "UO|i", &name, &desc, &bitsize)) { - PyErr_SetString(PyExc_TypeError, - "'_fields_' must be a sequence of (name, C type) pairs"); - Py_DECREF(pair); - return -1; + for (Py_ssize_t i = 0; i < len; ++i) { + PyObject *prop_obj = PyTuple_GET_ITEM(layout_fields, i); // borrowed + assert(prop_obj); + assert(PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type)); + CFieldObject *prop = (CFieldObject *)prop_obj; // borrowed + + StgInfo *info; + if (PyStgInfo_FromType(st, prop->proto, &info) < 0) { + goto error; } - dict = PyType_stgdict(desc); - if (dict == NULL) { - Py_DECREF(pair); - PyErr_Format(PyExc_TypeError, - "second item in _fields_ tuple (index %zd) must be a C type", - i); - return -1; - } - if (!PyCArrayTypeObject_Check(st, desc)) { + assert(info); + + if (!PyCArrayTypeObject_Check(st, prop->proto)) { /* Not an array. Just need an ffi_type pointer. */ num_ffi_type_pointers++; } else { /* It's an array. */ - Py_ssize_t length = dict->length; - StgDictObject *edict; + Py_ssize_t length = info->length; - edict = PyType_stgdict(dict->proto); - if (edict == NULL) { - Py_DECREF(pair); + StgInfo *einfo; + if (PyStgInfo_FromType(st, info->proto, &einfo) < 0) { + goto error; + } + if (einfo == NULL) { PyErr_Format(PyExc_TypeError, "second item in _fields_ tuple (index %zd) must be a C type", i); - return -1; + goto error; } /* * We need one extra ffi_type to hold the struct, and one @@ -862,7 +554,6 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct num_ffi_types++; num_ffi_type_pointers += length + 1; } - Py_DECREF(pair); } /* @@ -879,7 +570,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct if (type_block == NULL) { PyErr_NoMemory(); - return -1; + goto error; } /* * the first block takes up ffi_ofs + len + 1 which is the pointers * @@ -895,81 +586,61 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct if (num_ffi_types > 0) { memset(structs, 0, num_ffi_types * sizeof(ffi_type)); } - if (ffi_ofs && (basedict != NULL)) { + if (ffi_ofs && (baseinfo != NULL)) { memcpy(element_types, - basedict->ffi_type_pointer.elements, + baseinfo->ffi_type_pointer.elements, ffi_ofs * sizeof(ffi_type *)); } element_index = ffi_ofs; /* second pass to actually set the type pointers */ - for (i = 0; i < len; ++i) { - PyObject *name, *desc; - PyObject *pair = PySequence_GetItem(fields, i); - StgDictObject *dict; - int bitsize = 0; - - if (pair == NULL) { + for (Py_ssize_t i = 0; i < len; ++i) { + PyObject *prop_obj = PyTuple_GET_ITEM(layout_fields, i); // borrowed + assert(prop_obj); + assert(PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type)); + CFieldObject *prop = (CFieldObject *)prop_obj; // borrowed + + StgInfo *info; + if (PyStgInfo_FromType(st, prop->proto, &info) < 0) { PyMem_Free(type_block); - return -1; - } - /* In theory, we made this call in the first pass, so it *shouldn't* - * fail. However, you never know, and the code above might change - * later - keeping the check in here is a tad defensive but it - * will affect program size only slightly and performance hardly at - * all. - */ - if (!PyArg_ParseTuple(pair, "UO|i", &name, &desc, &bitsize)) { - PyErr_SetString(PyExc_TypeError, - "'_fields_' must be a sequence of (name, C type) pairs"); - Py_DECREF(pair); - PyMem_Free(type_block); - return -1; - } - dict = PyType_stgdict(desc); - /* Possibly this check could be avoided, but see above comment. */ - if (dict == NULL) { - Py_DECREF(pair); - PyMem_Free(type_block); - PyErr_Format(PyExc_TypeError, - "second item in _fields_ tuple (index %zd) must be a C type", - i); - return -1; + goto error; } + assert(info); + assert(element_index < (ffi_ofs + len)); /* will be used below */ - if (!PyCArrayTypeObject_Check(st, desc)) { + if (!PyCArrayTypeObject_Check(st, prop->proto)) { /* Not an array. Just copy over the element ffi_type. */ - element_types[element_index++] = &dict->ffi_type_pointer; + element_types[element_index++] = &info->ffi_type_pointer; } else { - Py_ssize_t length = dict->length; - StgDictObject *edict; - - edict = PyType_stgdict(dict->proto); - if (edict == NULL) { - Py_DECREF(pair); + Py_ssize_t length = info->length; + StgInfo *einfo; + if (PyStgInfo_FromType(st, info->proto, &einfo) < 0) { + PyMem_Free(type_block); + goto error; + } + if (einfo == NULL) { PyMem_Free(type_block); PyErr_Format(PyExc_TypeError, "second item in _fields_ tuple (index %zd) must be a C type", i); - return -1; + goto error; } element_types[element_index++] = &structs[struct_index]; - structs[struct_index].size = length * edict->ffi_type_pointer.size; - structs[struct_index].alignment = edict->ffi_type_pointer.alignment; + structs[struct_index].size = length * einfo->ffi_type_pointer.size; + structs[struct_index].alignment = einfo->ffi_type_pointer.alignment; structs[struct_index].type = FFI_TYPE_STRUCT; structs[struct_index].elements = &dummy_types[dummy_index]; ++struct_index; /* Copy over the element's type, length times. */ while (length > 0) { assert(dummy_index < (num_ffi_type_pointers)); - dummy_types[dummy_index++] = &edict->ffi_type_pointer; + dummy_types[dummy_index++] = &einfo->ffi_type_pointer; length--; } assert(dummy_index < (num_ffi_type_pointers)); dummy_types[dummy_index++] = NULL; } - Py_DECREF(pair); } element_types[element_index] = NULL; @@ -977,19 +648,24 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct * Replace the old elements with the new, taking into account * base class elements where necessary. */ - assert(stgdict->ffi_type_pointer.elements); - PyMem_Free(stgdict->ffi_type_pointer.elements); - stgdict->ffi_type_pointer.elements = element_types; + assert(stginfo->ffi_type_pointer.elements); + PyMem_Free(stginfo->ffi_type_pointer.elements); + stginfo->ffi_type_pointer.elements = element_types; } /* We did check that this flag was NOT set above, it must not have been set until now. */ - if (stgdict->flags & DICTFLAG_FINAL) { + if (stginfo->flags & DICTFLAG_FINAL) { PyErr_SetString(PyExc_AttributeError, "Structure or union cannot contain itself"); - return -1; + goto error; } - stgdict->flags |= DICTFLAG_FINAL; + stginfo->flags |= DICTFLAG_FINAL; - return MakeAnonFields(type); + retval = MakeAnonFields(type); +error: + Py_XDECREF(layout_fields); + Py_XDECREF(layout); + Py_XDECREF(format_spec_obj); + return retval; } diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 2ec8f34c5c220b..eecf7a1c8a1e56 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -19,7 +19,13 @@ static const char PyCursesVersion[] = "2.1"; #include "py_curses.h" -#include +#if defined(HAVE_NCURSESW_PANEL_H) +# include +#elif defined(HAVE_NCURSES_PANEL_H) +# include +#elif defined(HAVE_PANEL_H) +# include +#endif typedef struct { PyObject *PyCursesError; @@ -56,7 +62,7 @@ _curses_panel_traverse(PyObject *mod, visitproc visit, void *arg) static void _curses_panel_free(void *mod) { - _curses_panel_clear((PyObject *) mod); + (void)_curses_panel_clear((PyObject *)mod); } /* Utility Functions */ @@ -95,6 +101,8 @@ typedef struct { PyCursesWindowObject *wo; /* for reference counts */ } PyCursesPanelObject; +#define _PyCursesPanelObject_CAST(op) ((PyCursesPanelObject *)(op)) + /* Some helper functions. The problem is that there's always a window associated with a panel. To ensure that Python's GC doesn't pull this window from under our feet we need to keep track of references @@ -271,9 +279,10 @@ PyCursesPanel_New(_curses_panel_state *state, PANEL *pan, } static void -PyCursesPanel_Dealloc(PyCursesPanelObject *po) +PyCursesPanel_Dealloc(PyObject *self) { PyObject *tp, *obj; + PyCursesPanelObject *po = _PyCursesPanelObject_CAST(self); tp = (PyObject *) Py_TYPE(po); obj = (PyObject *) panel_userptr(po->pan); @@ -697,6 +706,7 @@ static PyModuleDef_Slot _curses_slots[] = { // XXX gh-103092: fix isolation. {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index d04d1e973af030..c6835738348ff9 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -105,8 +105,9 @@ static const char PyCursesVersion[] = "2.2"; #endif #include "Python.h" -#include "pycore_long.h" // _PyLong_GetZero() -#include "pycore_structseq.h" // _PyStructSequence_NewType() +#include "pycore_capsule.h" // _PyCapsule_SetTraverse() +#include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_structseq.h" // _PyStructSequence_NewType() #ifdef __hpux #define STRICT_SYSV_CURSES @@ -128,7 +129,7 @@ static const char PyCursesVersion[] = "2.2"; #include #endif -#if !defined(HAVE_NCURSES_H) && (defined(sgi) || defined(__sun) || defined(SCO5)) +#if !defined(NCURSES_VERSION) && (defined(sgi) || defined(__sun) || defined(SCO5)) #define STRICT_SYSV_CURSES /* Don't use ncurses extensions */ typedef chtype attr_t; /* No attr_t type is available */ #endif @@ -137,7 +138,7 @@ typedef chtype attr_t; /* No attr_t type is available */ #define STRICT_SYSV_CURSES #endif -#if NCURSES_EXT_FUNCS+0 >= 20170401 && NCURSES_EXT_COLORS+0 >= 20170401 +#if defined(HAVE_NCURSESW) && NCURSES_EXT_FUNCS+0 >= 20170401 && NCURSES_EXT_COLORS+0 >= 20170401 #define _NCURSES_EXTENDED_COLOR_FUNCS 1 #else #define _NCURSES_EXTENDED_COLOR_FUNCS 0 @@ -159,65 +160,167 @@ typedef chtype attr_t; /* No attr_t type is available */ #define _CURSES_PAIR_CONTENT_FUNC pair_content #endif /* _NCURSES_EXTENDED_COLOR_FUNCS */ +typedef struct { + PyObject *error; // curses exception type + PyTypeObject *window_type; // exposed by PyCursesWindow_Type +} cursesmodule_state; + +static inline cursesmodule_state * +get_cursesmodule_state(PyObject *module) +{ + void *state = PyModule_GetState(module); + assert(state != NULL); + return (cursesmodule_state *)state; +} + +static inline cursesmodule_state * +get_cursesmodule_state_by_cls(PyTypeObject *cls) +{ + void *state = PyType_GetModuleState(cls); + assert(state != NULL); + return (cursesmodule_state *)state; +} + +static inline cursesmodule_state * +get_cursesmodule_state_by_win(PyCursesWindowObject *win) +{ + return get_cursesmodule_state_by_cls(Py_TYPE(win)); +} + +#define _PyCursesWindowObject_CAST(op) ((PyCursesWindowObject *)(op)) + /*[clinic input] module _curses -class _curses.window "PyCursesWindowObject *" "&PyCursesWindow_Type" +class _curses.window "PyCursesWindowObject *" "clinic_state()->window_type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=43265c372c2887d6]*/ - -/* Definition of exception curses.error */ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=ae6cb623018f2cbc]*/ -static PyObject *PyCursesError; +/* Indicate whether the module has already been loaded or not. */ +static int curses_module_loaded = 0; /* Tells whether setupterm() has been called to initialise terminfo. */ -static int initialised_setupterm = FALSE; +static int curses_setupterm_called = FALSE; /* Tells whether initscr() has been called to initialise curses. */ -static int initialised = FALSE; +static int curses_initscr_called = FALSE; /* Tells whether start_color() has been called to initialise color usage. */ -static int initialisedcolors = FALSE; +static int curses_start_color_called = FALSE; -static char *screen_encoding = NULL; +static const char *curses_screen_encoding = NULL; -/* Utility Macros */ -#define PyCursesSetupTermCalled \ - if (initialised_setupterm != TRUE) { \ - PyErr_SetString(PyCursesError, \ - "must call (at least) setupterm() first"); \ - return 0; } +/* Utility Checking Procedures */ -#define PyCursesInitialised \ - if (initialised != TRUE) { \ - PyErr_SetString(PyCursesError, \ - "must call initscr() first"); \ - return 0; } +/* + * Function to check that 'funcname' has been called by testing + * the 'called' boolean. If an error occurs, a PyCursesError is + * set and this returns 0. Otherwise, this returns 1. + * + * Since this function can be called in functions that do not + * have a direct access to the module's state, '_curses.error' + * is imported on demand. + */ +static inline int +_PyCursesCheckFunction(int called, const char *funcname) +{ + if (called == TRUE) { + return 1; + } + PyObject *exc = _PyImport_GetModuleAttrString("_curses", "error"); + if (exc != NULL) { + PyErr_Format(exc, "must call %s() first", funcname); + Py_DECREF(exc); + } + assert(PyErr_Occurred()); + return 0; +} + +/* + * Function to check that 'funcname' has been called by testing + * the 'called'' boolean. If an error occurs, a PyCursesError is + * set and this returns 0. Otherwise this returns 1. + * + * The exception type is obtained from the 'module' state. + */ +static inline int +_PyCursesStatefulCheckFunction(PyObject *module, int called, const char *funcname) +{ + if (called == TRUE) { + return 1; + } + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_Format(state->error, "must call %s() first", funcname); + return 0; +} -#define PyCursesInitialisedColor \ - if (initialisedcolors != TRUE) { \ - PyErr_SetString(PyCursesError, \ - "must call start_color() first"); \ - return 0; } +#define PyCursesStatefulSetupTermCalled(MODULE) \ + do { \ + if (!_PyCursesStatefulCheckFunction(MODULE, \ + curses_setupterm_called, \ + "setupterm")) \ + { \ + return 0; \ + } \ + } while (0) + +#define PyCursesStatefulInitialised(MODULE) \ + do { \ + if (!_PyCursesStatefulCheckFunction(MODULE, \ + curses_initscr_called, \ + "initscr")) \ + { \ + return 0; \ + } \ + } while (0) + +#define PyCursesStatefulInitialisedColor(MODULE) \ + do { \ + if (!_PyCursesStatefulCheckFunction(MODULE, \ + curses_start_color_called, \ + "start_color")) \ + { \ + return 0; \ + } \ + } while (0) /* Utility Functions */ +static inline void +_PyCursesSetError(cursesmodule_state *state, const char *funcname) +{ + if (funcname == NULL) { + PyErr_SetString(state->error, catchall_ERR); + } + else { + PyErr_Format(state->error, "%s() returned ERR", funcname); + } +} + /* * Check the return code from a curses function and return None - * or raise an exception as appropriate. These are exported using the - * capsule API. + * or raise an exception as appropriate. */ static PyObject * -PyCursesCheckERR(int code, const char *fname) +PyCursesCheckERR(PyObject *module, int code, const char *fname) { if (code != ERR) { Py_RETURN_NONE; } else { - if (fname == NULL) { - PyErr_SetString(PyCursesError, catchall_ERR); - } else { - PyErr_Format(PyCursesError, "%s() returned ERR", fname); - } + cursesmodule_state *state = get_cursesmodule_state(module); + _PyCursesSetError(state, fname); + return NULL; + } +} + +static PyObject * +PyCursesCheckERR_ForWin(PyCursesWindowObject *win, int code, const char *fname) +{ + if (code != ERR) { + Py_RETURN_NONE; + } else { + cursesmodule_state *state = get_cursesmodule_state_by_win(win); + _PyCursesSetError(state, fname); return NULL; } } @@ -233,13 +336,20 @@ static int PyCurses_ConvertToChtype(PyCursesWindowObject *win, PyObject *obj, chtype *ch) { long value; - if(PyBytes_Check(obj) && PyBytes_Size(obj) == 1) { + if (PyBytes_Check(obj)) { + if (PyBytes_GET_SIZE(obj) != 1) { + PyErr_Format(PyExc_TypeError, + "expect int or bytes or str of length 1, " + "got a bytes of length %zd", + PyBytes_GET_SIZE(obj)); + return 0; + } value = (unsigned char)PyBytes_AsString(obj)[0]; } else if (PyUnicode_Check(obj)) { - if (PyUnicode_GetLength(obj) != 1) { + if (PyUnicode_GET_LENGTH(obj) != 1) { PyErr_Format(PyExc_TypeError, - "expect bytes or str of length 1, or int, " + "expect int or bytes or str of length 1, " "got a str of length %zi", PyUnicode_GET_LENGTH(obj)); return 0; @@ -251,7 +361,7 @@ PyCurses_ConvertToChtype(PyCursesWindowObject *win, PyObject *obj, chtype *ch) if (win) encoding = win->encoding; else - encoding = screen_encoding; + encoding = curses_screen_encoding; bytes = PyUnicode_AsEncodedString(obj, encoding, NULL); if (bytes == NULL) return 0; @@ -272,7 +382,7 @@ PyCurses_ConvertToChtype(PyCursesWindowObject *win, PyObject *obj, chtype *ch) } else { PyErr_Format(PyExc_TypeError, - "expect bytes or str of length 1, or int, got %s", + "expect int or bytes or str of length 1, got %s", Py_TYPE(obj)->tp_name); return 0; } @@ -315,7 +425,7 @@ PyCurses_ConvertToCchar_t(PyCursesWindowObject *win, PyObject *obj, #ifdef HAVE_NCURSESW if (PyUnicode_AsWideChar(obj, buffer, 2) != 1) { PyErr_Format(PyExc_TypeError, - "expect bytes or str of length 1, or int, " + "expect int or bytes or str of length 1, " "got a str of length %zi", PyUnicode_GET_LENGTH(obj)); return 0; @@ -326,7 +436,14 @@ PyCurses_ConvertToCchar_t(PyCursesWindowObject *win, PyObject *obj, return PyCurses_ConvertToChtype(win, obj, ch); #endif } - else if(PyBytes_Check(obj) && PyBytes_Size(obj) == 1) { + else if (PyBytes_Check(obj)) { + if (PyBytes_GET_SIZE(obj) != 1) { + PyErr_Format(PyExc_TypeError, + "expect int or bytes or str of length 1, " + "got a bytes of length %zd", + PyBytes_GET_SIZE(obj)); + return 0; + } value = (unsigned char)PyBytes_AsString(obj)[0]; } else if (PyLong_CheckExact(obj)) { @@ -340,7 +457,7 @@ PyCurses_ConvertToCchar_t(PyCursesWindowObject *win, PyObject *obj, } else { PyErr_Format(PyExc_TypeError, - "expect bytes or str of length 1, or int, got %s", + "expect int or bytes or str of length 1, got %s", Py_TYPE(obj)->tp_name); return 0; } @@ -527,35 +644,10 @@ class component_converter(CConverter): [python start generated code]*/ /*[python end generated code: output=da39a3ee5e6b4b0d input=38e9be01d33927fb]*/ -/* Function versions of the 3 functions for testing whether curses has been - initialised or not. */ - -static int func_PyCursesSetupTermCalled(void) -{ - PyCursesSetupTermCalled; - return 1; -} - -static int func_PyCursesInitialised(void) -{ - PyCursesInitialised; - return 1; -} - -static int func_PyCursesInitialisedColor(void) -{ - PyCursesInitialisedColor; - return 1; -} - /***************************************************************************** The Window Object ******************************************************************************/ -/* Definition of the window type */ - -PyTypeObject PyCursesWindow_Type; - /* Function prototype macros for Window object X - function name @@ -564,53 +656,80 @@ PyTypeObject PyCursesWindow_Type; PARSESTR - format string for argument parsing */ -#define Window_NoArgNoReturnFunction(X) \ - static PyObject *PyCursesWindow_ ## X \ - (PyCursesWindowObject *self, PyObject *Py_UNUSED(ignored)) \ - { return PyCursesCheckERR(X(self->win), # X); } +#define Window_NoArgNoReturnFunction(X) \ + static PyObject *PyCursesWindow_ ## X \ + (PyObject *op, PyObject *Py_UNUSED(ignored)) \ + { \ + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); \ + int code = X(self->win); \ + return PyCursesCheckERR_ForWin(self, code, # X); \ + } #define Window_NoArgTrueFalseFunction(X) \ static PyObject * PyCursesWindow_ ## X \ - (PyCursesWindowObject *self, PyObject *Py_UNUSED(ignored)) \ + (PyObject *op, PyObject *Py_UNUSED(ignored)) \ { \ - return PyBool_FromLong(X(self->win)); } + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); \ + return PyBool_FromLong(X(self->win)); \ + } -#define Window_NoArgNoReturnVoidFunction(X) \ - static PyObject * PyCursesWindow_ ## X \ - (PyCursesWindowObject *self, PyObject *Py_UNUSED(ignored)) \ - { \ - X(self->win); Py_RETURN_NONE; } +#define Window_NoArgNoReturnVoidFunction(X) \ + static PyObject * PyCursesWindow_ ## X \ + (PyObject *op, PyObject *Py_UNUSED(ignored)) \ + { \ + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); \ + X(self->win); \ + Py_RETURN_NONE; \ + } #define Window_NoArg2TupleReturnFunction(X, TYPE, ERGSTR) \ static PyObject * PyCursesWindow_ ## X \ - (PyCursesWindowObject *self, PyObject *Py_UNUSED(ignored)) \ + (PyObject *op, PyObject *Py_UNUSED(ignored)) \ { \ TYPE arg1, arg2; \ - X(self->win,arg1,arg2); return Py_BuildValue(ERGSTR, arg1, arg2); } + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); \ + X(self->win, arg1, arg2); \ + return Py_BuildValue(ERGSTR, arg1, arg2); \ + } #define Window_OneArgNoReturnVoidFunction(X, TYPE, PARSESTR) \ static PyObject * PyCursesWindow_ ## X \ - (PyCursesWindowObject *self, PyObject *args) \ + (PyObject *op, PyObject *args) \ { \ TYPE arg1; \ - if (!PyArg_ParseTuple(args, PARSESTR, &arg1)) return NULL; \ - X(self->win,arg1); Py_RETURN_NONE; } + if (!PyArg_ParseTuple(args, PARSESTR, &arg1)) { \ + return NULL; \ + } \ + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); \ + X(self->win, arg1); \ + Py_RETURN_NONE; \ + } #define Window_OneArgNoReturnFunction(X, TYPE, PARSESTR) \ static PyObject * PyCursesWindow_ ## X \ - (PyCursesWindowObject *self, PyObject *args) \ + (PyObject *op, PyObject *args) \ { \ TYPE arg1; \ - if (!PyArg_ParseTuple(args,PARSESTR, &arg1)) return NULL; \ - return PyCursesCheckERR(X(self->win, arg1), # X); } + if (!PyArg_ParseTuple(args, PARSESTR, &arg1)) { \ + return NULL; \ + } \ + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); \ + int code = X(self->win, arg1); \ + return PyCursesCheckERR_ForWin(self, code, # X); \ + } #define Window_TwoArgNoReturnFunction(X, TYPE, PARSESTR) \ static PyObject * PyCursesWindow_ ## X \ - (PyCursesWindowObject *self, PyObject *args) \ + (PyObject *op, PyObject *args) \ { \ TYPE arg1, arg2; \ - if (!PyArg_ParseTuple(args,PARSESTR, &arg1, &arg2)) return NULL; \ - return PyCursesCheckERR(X(self->win, arg1, arg2), # X); } + if (!PyArg_ParseTuple(args,PARSESTR, &arg1, &arg2)) { \ + return NULL; \ + } \ + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); \ + int code = X(self->win, arg1, arg2); \ + return PyCursesCheckERR_ForWin(self, code, # X); \ + } /* ------------- WINDOW routines --------------- */ @@ -665,10 +784,9 @@ Window_TwoArgNoReturnFunction(wresize, int, "ii;lines,columns") /* Allocation and deallocation of Window Objects */ static PyObject * -PyCursesWindow_New(WINDOW *win, const char *encoding) +PyCursesWindow_New(cursesmodule_state *state, + WINDOW *win, const char *encoding) { - PyCursesWindowObject *wo; - if (encoding == NULL) { #if defined(MS_WINDOWS) char *buffer[100]; @@ -680,15 +798,20 @@ PyCursesWindow_New(WINDOW *win, const char *encoding) } #elif defined(CODESET) const char *codeset = nl_langinfo(CODESET); - if (codeset != NULL && codeset[0] != 0) + if (codeset != NULL && codeset[0] != 0) { encoding = codeset; + } #endif - if (encoding == NULL) + if (encoding == NULL) { encoding = "utf-8"; + } } - wo = PyObject_New(PyCursesWindowObject, &PyCursesWindow_Type); - if (wo == NULL) return NULL; + PyCursesWindowObject *wo = PyObject_GC_New(PyCursesWindowObject, + state->window_type); + if (wo == NULL) { + return NULL; + } wo->win = win; wo->encoding = _PyMem_Strdup(encoding); if (wo->encoding == NULL) { @@ -696,16 +819,32 @@ PyCursesWindow_New(WINDOW *win, const char *encoding) PyErr_NoMemory(); return NULL; } + PyObject_GC_Track((PyObject *)wo); return (PyObject *)wo; } static void -PyCursesWindow_Dealloc(PyCursesWindowObject *wo) +PyCursesWindow_dealloc(PyObject *self) { - if (wo->win != stdscr) delwin(wo->win); - if (wo->encoding != NULL) + PyTypeObject *window_type = Py_TYPE(self); + PyObject_GC_UnTrack(self); + PyCursesWindowObject *wo = (PyCursesWindowObject *)self; + if (wo->win != stdscr && wo->win != NULL) { + // silently ignore errors in delwin(3) + (void)delwin(wo->win); + } + if (wo->encoding != NULL) { PyMem_Free(wo->encoding); - PyObject_Free(wo); + } + window_type->tp_free(self); + Py_DECREF(window_type); +} + +static int +PyCursesWindow_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; } /* Addch, Addstr, Addnstr */ @@ -780,7 +919,7 @@ _curses_window_addch_impl(PyCursesWindowObject *self, int group_left_1, else { return NULL; } - return PyCursesCheckERR(rtn, funcname); + return PyCursesCheckERR_ForWin(self, rtn, funcname); } /*[clinic input] @@ -860,7 +999,7 @@ _curses_window_addstr_impl(PyCursesWindowObject *self, int group_left_1, } if (use_attr) (void)wattrset(self->win,attr_old); - return PyCursesCheckERR(rtn, funcname); + return PyCursesCheckERR_ForWin(self, rtn, funcname); } /*[clinic input] @@ -943,7 +1082,7 @@ _curses_window_addnstr_impl(PyCursesWindowObject *self, int group_left_1, } if (use_attr) (void)wattrset(self->win,attr_old); - return PyCursesCheckERR(rtn, funcname); + return PyCursesCheckERR_ForWin(self, rtn, funcname); } /*[clinic input] @@ -967,7 +1106,7 @@ _curses_window_bkgd_impl(PyCursesWindowObject *self, PyObject *ch, long attr) if (!PyCurses_ConvertToChtype(self, ch, &bkgd)) return NULL; - return PyCursesCheckERR(wbkgd(self->win, bkgd | attr), "bkgd"); + return PyCursesCheckERR_ForWin(self, wbkgd(self->win, bkgd | attr), "bkgd"); } /*[clinic input] @@ -983,7 +1122,7 @@ static PyObject * _curses_window_attroff_impl(PyCursesWindowObject *self, long attr) /*[clinic end generated code: output=8a2fcd4df682fc64 input=786beedf06a7befe]*/ { - return PyCursesCheckERR(wattroff(self->win, (attr_t)attr), "attroff"); + return PyCursesCheckERR_ForWin(self, wattroff(self->win, (attr_t)attr), "attroff"); } /*[clinic input] @@ -999,7 +1138,7 @@ static PyObject * _curses_window_attron_impl(PyCursesWindowObject *self, long attr) /*[clinic end generated code: output=7afea43b237fa870 input=5a88fba7b1524f32]*/ { - return PyCursesCheckERR(wattron(self->win, (attr_t)attr), "attron"); + return PyCursesCheckERR_ForWin(self, wattron(self->win, (attr_t)attr), "attron"); } /*[clinic input] @@ -1015,7 +1154,7 @@ static PyObject * _curses_window_attrset_impl(PyCursesWindowObject *self, long attr) /*[clinic end generated code: output=84e379bff20c0433 input=42e400c0d0154ab5]*/ { - return PyCursesCheckERR(wattrset(self->win, (attr_t)attr), "attrset"); + return PyCursesCheckERR_ForWin(self, wattrset(self->win, (attr_t)attr), "attrset"); } /*[clinic input] @@ -1041,7 +1180,7 @@ _curses_window_bkgdset_impl(PyCursesWindowObject *self, PyObject *ch, return NULL; wbkgdset(self->win, bkgd | attr); - return PyCursesCheckERR(0, "bkgdset"); + return PyCursesCheckERR_ForWin(self, 0, "bkgdset"); } /*[clinic input] @@ -1156,8 +1295,10 @@ int py_mvwdelch(WINDOW *w, int y, int x) #endif #if defined(HAVE_CURSES_IS_PAD) +// is_pad() is defined, either as a macro or as a function #define py_is_pad(win) is_pad(win) #elif defined(WINDOW_HAS_FLAGS) +// is_pad() is not defined, but we can inspect WINDOW structure members #define py_is_pad(win) ((win) ? ((win)->_flags & _ISPAD) != 0 : FALSE) #endif @@ -1190,8 +1331,10 @@ the touchline() method so that the contents will be redisplayed by the next window refresh. [-clinic start generated code]*/ static PyObject * -PyCursesWindow_ChgAt(PyCursesWindowObject *self, PyObject *args) +PyCursesWindow_ChgAt(PyObject *op, PyObject *args) { + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); + int rtn; int x, y; int num = -1; @@ -1239,7 +1382,7 @@ PyCursesWindow_ChgAt(PyCursesWindowObject *self, PyObject *args) rtn = wchgat(self->win,num,attr,color,NULL); touchline(self->win,y,1); } - return PyCursesCheckERR(rtn, "chgat"); + return PyCursesCheckERR_ForWin(self, rtn, "chgat"); } #endif @@ -1263,10 +1406,10 @@ _curses_window_delch_impl(PyCursesWindowObject *self, int group_right_1, /*[clinic end generated code: output=22e77bb9fa11b461 input=d2f79e630a4fc6d0]*/ { if (!group_right_1) { - return PyCursesCheckERR(wdelch(self->win), "wdelch"); + return PyCursesCheckERR_ForWin(self, wdelch(self->win), "wdelch"); } else { - return PyCursesCheckERR(py_mvwdelch(self->win, y, x), "mvwdelch"); + return PyCursesCheckERR_ForWin(self, py_mvwdelch(self->win, y, x), "mvwdelch"); } } @@ -1302,11 +1445,13 @@ _curses_window_derwin_impl(PyCursesWindowObject *self, int group_left_1, win = derwin(self->win,nlines,ncols,begin_y,begin_x); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, catchall_NULL); return NULL; } - return (PyObject *)PyCursesWindow_New(win, NULL); + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + return PyCursesWindow_New(state, win, NULL); } /*[clinic input] @@ -1334,13 +1479,15 @@ _curses_window_echochar_impl(PyCursesWindowObject *self, PyObject *ch, #ifdef py_is_pad if (py_is_pad(self->win)) { - return PyCursesCheckERR(pechochar(self->win, ch_ | (attr_t)attr), - "echochar"); + return PyCursesCheckERR_ForWin(self, + pechochar(self->win, ch_ | (attr_t)attr), + "echochar"); } else #endif - return PyCursesCheckERR(wechochar(self->win, ch_ | (attr_t)attr), - "echochar"); + return PyCursesCheckERR_ForWin(self, + wechochar(self->win, ch_ | (attr_t)attr), + "echochar"); } #ifdef NCURSES_MOUSE_VERSION @@ -1451,8 +1598,10 @@ _curses_window_getkey_impl(PyCursesWindowObject *self, int group_right_1, if (rtn == ERR) { /* getch() returns ERR in nodelay mode */ PyErr_CheckSignals(); - if (!PyErr_Occurred()) - PyErr_SetString(PyCursesError, "no input"); + if (!PyErr_Occurred()) { + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, "no input"); + } return NULL; } else if (rtn <= 255) { #ifdef NCURSES_VERSION_MAJOR @@ -1510,7 +1659,8 @@ _curses_window_get_wch_impl(PyCursesWindowObject *self, int group_right_1, return NULL; /* get_wch() returns ERR in nodelay mode */ - PyErr_SetString(PyCursesError, "no input"); + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, "no input"); return NULL; } if (ct == KEY_CODE_YES) @@ -1537,8 +1687,10 @@ Read a string from the user, with primitive line editing capacity. [-clinic start generated code]*/ static PyObject * -PyCursesWindow_GetStr(PyCursesWindowObject *self, PyObject *args) +PyCursesWindow_GetStr(PyObject *op, PyObject *args) { + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); + int x, y, n; char rtn[1024]; /* This should be big enough.. I hope */ int rtn2; @@ -1634,10 +1786,10 @@ _curses_window_hline_impl(PyCursesWindowObject *self, int group_left_1, return NULL; if (group_left_1) { if (wmove(self->win, y, x) == ERR) { - return PyCursesCheckERR(ERR, "wmove"); + return PyCursesCheckERR_ForWin(self, ERR, "wmove"); } } - return PyCursesCheckERR(whline(self->win, ch_ | (attr_t)attr, n), "hline"); + return PyCursesCheckERR_ForWin(self, whline(self->win, ch_ | (attr_t)attr, n), "hline"); } /*[clinic input] @@ -1684,7 +1836,7 @@ _curses_window_insch_impl(PyCursesWindowObject *self, int group_left_1, rtn = mvwinsch(self->win, y, x, ch_ | (attr_t)attr); } - return PyCursesCheckERR(rtn, "insch"); + return PyCursesCheckERR_ForWin(self, rtn, "insch"); } /*[clinic input] @@ -1741,8 +1893,10 @@ from the characters. If n is specified, instr() returns a string at most n characters long (exclusive of the trailing NUL). [-clinic start generated code]*/ static PyObject * -PyCursesWindow_InStr(PyCursesWindowObject *self, PyObject *args) +PyCursesWindow_InStr(PyObject *op, PyObject *args) { + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); + int x, y, n; char rtn[1024]; /* This should be big enough.. I hope */ int rtn2; @@ -1861,7 +2015,7 @@ _curses_window_insstr_impl(PyCursesWindowObject *self, int group_left_1, } if (use_attr) (void)wattrset(self->win,attr_old); - return PyCursesCheckERR(rtn, funcname); + return PyCursesCheckERR_ForWin(self, rtn, funcname); } /*[clinic input] @@ -1946,7 +2100,7 @@ _curses_window_insnstr_impl(PyCursesWindowObject *self, int group_left_1, } if (use_attr) (void)wattrset(self->win,attr_old); - return PyCursesCheckERR(rtn, funcname); + return PyCursesCheckERR_ForWin(self, rtn, funcname); } /*[clinic input] @@ -2023,7 +2177,8 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self) #ifdef py_is_pad if (py_is_pad(self->win)) { if (!group_right_1) { - PyErr_SetString(PyCursesError, + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, "noutrefresh() called for a pad " "requires 6 arguments"); return NULL; @@ -2032,7 +2187,7 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self) rtn = pnoutrefresh(self->win, pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol); Py_END_ALLOW_THREADS - return PyCursesCheckERR(rtn, "pnoutrefresh"); + return PyCursesCheckERR_ForWin(self, rtn, "pnoutrefresh"); } if (group_right_1) { PyErr_SetString(PyExc_TypeError, @@ -2043,13 +2198,13 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self) Py_BEGIN_ALLOW_THREADS rtn = wnoutrefresh(self->win); Py_END_ALLOW_THREADS - return PyCursesCheckERR(rtn, "wnoutrefresh"); + return PyCursesCheckERR_ForWin(self, rtn, "wnoutrefresh"); } /*[clinic input] _curses.window.overlay - destwin: object(type="PyCursesWindowObject *", subclass_of="&PyCursesWindow_Type") + destwin: object(type="PyCursesWindowObject *", subclass_of="clinic_state()->window_type") [ sminrow: int @@ -2078,25 +2233,25 @@ _curses_window_overlay_impl(PyCursesWindowObject *self, PyCursesWindowObject *destwin, int group_right_1, int sminrow, int smincol, int dminrow, int dmincol, int dmaxrow, int dmaxcol) -/*[clinic end generated code: output=82bb2c4cb443ca58 input=7edd23ad22cc1984]*/ +/*[clinic end generated code: output=82bb2c4cb443ca58 input=6e4b32a7c627a356]*/ { int rtn; if (group_right_1) { rtn = copywin(self->win, destwin->win, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol, TRUE); - return PyCursesCheckERR(rtn, "copywin"); + return PyCursesCheckERR_ForWin(self, rtn, "copywin"); } else { rtn = overlay(self->win, destwin->win); - return PyCursesCheckERR(rtn, "overlay"); + return PyCursesCheckERR_ForWin(self, rtn, "overlay"); } } /*[clinic input] _curses.window.overwrite - destwin: object(type="PyCursesWindowObject *", subclass_of="&PyCursesWindow_Type") + destwin: object(type="PyCursesWindowObject *", subclass_of="clinic_state()->window_type") [ sminrow: int @@ -2126,18 +2281,18 @@ _curses_window_overwrite_impl(PyCursesWindowObject *self, int group_right_1, int sminrow, int smincol, int dminrow, int dmincol, int dmaxrow, int dmaxcol) -/*[clinic end generated code: output=12ae007d1681be28 input=ea5de1b35cd948e0]*/ +/*[clinic end generated code: output=12ae007d1681be28 input=d83dd8b24ff2bcc9]*/ { int rtn; if (group_right_1) { rtn = copywin(self->win, destwin->win, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol, FALSE); - return PyCursesCheckERR(rtn, "copywin"); + return PyCursesCheckERR_ForWin(self, rtn, "copywin"); } else { rtn = overwrite(self->win, destwin->win); - return PyCursesCheckERR(rtn, "overwrite"); + return PyCursesCheckERR_ForWin(self, rtn, "overwrite"); } } @@ -2166,7 +2321,7 @@ _curses_window_putwin(PyCursesWindowObject *self, PyObject *file) return PyErr_SetFromErrno(PyExc_OSError); if (_Py_set_inheritable(fileno(fp), 0, NULL) < 0) goto exit; - res = PyCursesCheckERR(putwin(self->win, fp), "putwin"); + res = PyCursesCheckERR_ForWin(self, putwin(self->win, fp), "putwin"); if (res == NULL) goto exit; fseek(fp, 0, 0); @@ -2205,7 +2360,7 @@ static PyObject * _curses_window_redrawln_impl(PyCursesWindowObject *self, int beg, int num) /*[clinic end generated code: output=ea216e334f9ce1b4 input=152155e258a77a7a]*/ { - return PyCursesCheckERR(wredrawln(self->win,beg,num), "redrawln"); + return PyCursesCheckERR_ForWin(self, wredrawln(self->win,beg,num), "redrawln"); } /*[clinic input] @@ -2247,7 +2402,8 @@ _curses_window_refresh_impl(PyCursesWindowObject *self, int group_right_1, #ifdef py_is_pad if (py_is_pad(self->win)) { if (!group_right_1) { - PyErr_SetString(PyCursesError, + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, "refresh() for a pad requires 6 arguments"); return NULL; } @@ -2255,7 +2411,7 @@ _curses_window_refresh_impl(PyCursesWindowObject *self, int group_right_1, rtn = prefresh(self->win, pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol); Py_END_ALLOW_THREADS - return PyCursesCheckERR(rtn, "prefresh"); + return PyCursesCheckERR_ForWin(self, rtn, "prefresh"); } #endif if (group_right_1) { @@ -2266,7 +2422,7 @@ _curses_window_refresh_impl(PyCursesWindowObject *self, int group_right_1, Py_BEGIN_ALLOW_THREADS rtn = wrefresh(self->win); Py_END_ALLOW_THREADS - return PyCursesCheckERR(rtn, "prefresh"); + return PyCursesCheckERR_ForWin(self, rtn, "prefresh"); } /*[clinic input] @@ -2288,7 +2444,7 @@ _curses_window_setscrreg_impl(PyCursesWindowObject *self, int top, int bottom) /*[clinic end generated code: output=486ab5db218d2b1a input=1b517b986838bf0e]*/ { - return PyCursesCheckERR(wsetscrreg(self->win, top, bottom), "wsetscrreg"); + return PyCursesCheckERR_ForWin(self, wsetscrreg(self->win, top, bottom), "wsetscrreg"); } /*[clinic input] @@ -2329,11 +2485,13 @@ _curses_window_subwin_impl(PyCursesWindowObject *self, int group_left_1, win = subwin(self->win, nlines, ncols, begin_y, begin_x); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_SetString(state->error, catchall_NULL); return NULL; } - return (PyObject *)PyCursesWindow_New(win, self->encoding); + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + return PyCursesWindow_New(state, win, self->encoding); } /*[clinic input] @@ -2356,10 +2514,10 @@ _curses_window_scroll_impl(PyCursesWindowObject *self, int group_right_1, /*[clinic end generated code: output=4541a8a11852d360 input=c969ca0cfabbdbec]*/ { if (!group_right_1) { - return PyCursesCheckERR(scroll(self->win), "scroll"); + return PyCursesCheckERR_ForWin(self, scroll(self->win), "scroll"); } else { - return PyCursesCheckERR(wscrl(self->win, lines), "scroll"); + return PyCursesCheckERR_ForWin(self, wscrl(self->win, lines), "scroll"); } } @@ -2385,10 +2543,10 @@ _curses_window_touchline_impl(PyCursesWindowObject *self, int start, /*[clinic end generated code: output=65d05b3f7438c61d input=a98aa4f79b6be845]*/ { if (!group_right_1) { - return PyCursesCheckERR(touchline(self->win, start, count), "touchline"); + return PyCursesCheckERR_ForWin(self, touchline(self->win, start, count), "touchline"); } else { - return PyCursesCheckERR(wtouchln(self->win, start, count, changed), "touchline"); + return PyCursesCheckERR_ForWin(self, wtouchln(self->win, start, count, changed), "touchline"); } } @@ -2428,20 +2586,23 @@ _curses_window_vline_impl(PyCursesWindowObject *self, int group_left_1, return NULL; if (group_left_1) { if (wmove(self->win, y, x) == ERR) - return PyCursesCheckERR(ERR, "wmove"); + return PyCursesCheckERR_ForWin(self, ERR, "wmove"); } - return PyCursesCheckERR(wvline(self->win, ch_ | (attr_t)attr, n), "vline"); + return PyCursesCheckERR_ForWin(self, wvline(self->win, ch_ | (attr_t)attr, n), "vline"); } static PyObject * -PyCursesWindow_get_encoding(PyCursesWindowObject *self, void *closure) +PyCursesWindow_get_encoding(PyObject *op, void *closure) { + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); return PyUnicode_FromString(self->encoding); } static int -PyCursesWindow_set_encoding(PyCursesWindowObject *self, PyObject *value, void *Py_UNUSED(ignored)) +PyCursesWindow_set_encoding(PyObject *op, PyObject *value, void *Py_UNUSED(ignored)) { + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); + PyObject *ascii; char *encoding; @@ -2471,9 +2632,11 @@ PyCursesWindow_set_encoding(PyCursesWindowObject *self, PyObject *value, void *P return 0; } +#define clinic_state() (get_cursesmodule_state_by_cls(Py_TYPE(self))) #include "clinic/_cursesmodule.c.h" +#undef clinic_state -static PyMethodDef PyCursesWindow_Methods[] = { +static PyMethodDef PyCursesWindow_methods[] = { _CURSES_WINDOW_ADDCH_METHODDEF _CURSES_WINDOW_ADDNSTR_METHODDEF _CURSES_WINDOW_ADDSTR_METHODDEF @@ -2482,128 +2645,115 @@ static PyMethodDef PyCursesWindow_Methods[] = { _CURSES_WINDOW_ATTRSET_METHODDEF _CURSES_WINDOW_BKGD_METHODDEF #ifdef HAVE_CURSES_WCHGAT - {"chgat", (PyCFunction)PyCursesWindow_ChgAt, METH_VARARGS}, + {"chgat", PyCursesWindow_ChgAt, METH_VARARGS}, #endif _CURSES_WINDOW_BKGDSET_METHODDEF _CURSES_WINDOW_BORDER_METHODDEF _CURSES_WINDOW_BOX_METHODDEF - {"clear", (PyCFunction)PyCursesWindow_wclear, METH_NOARGS}, - {"clearok", (PyCFunction)PyCursesWindow_clearok, METH_VARARGS}, - {"clrtobot", (PyCFunction)PyCursesWindow_wclrtobot, METH_NOARGS}, - {"clrtoeol", (PyCFunction)PyCursesWindow_wclrtoeol, METH_NOARGS}, - {"cursyncup", (PyCFunction)PyCursesWindow_wcursyncup, METH_NOARGS}, + {"clear", PyCursesWindow_wclear, METH_NOARGS}, + {"clearok", PyCursesWindow_clearok, METH_VARARGS}, + {"clrtobot", PyCursesWindow_wclrtobot, METH_NOARGS}, + {"clrtoeol", PyCursesWindow_wclrtoeol, METH_NOARGS}, + {"cursyncup", PyCursesWindow_wcursyncup, METH_NOARGS}, _CURSES_WINDOW_DELCH_METHODDEF - {"deleteln", (PyCFunction)PyCursesWindow_wdeleteln, METH_NOARGS}, + {"deleteln", PyCursesWindow_wdeleteln, METH_NOARGS}, _CURSES_WINDOW_DERWIN_METHODDEF _CURSES_WINDOW_ECHOCHAR_METHODDEF _CURSES_WINDOW_ENCLOSE_METHODDEF - {"erase", (PyCFunction)PyCursesWindow_werase, METH_NOARGS}, - {"getbegyx", (PyCFunction)PyCursesWindow_getbegyx, METH_NOARGS}, + {"erase", PyCursesWindow_werase, METH_NOARGS}, + {"getbegyx", PyCursesWindow_getbegyx, METH_NOARGS}, _CURSES_WINDOW_GETBKGD_METHODDEF _CURSES_WINDOW_GETCH_METHODDEF _CURSES_WINDOW_GETKEY_METHODDEF _CURSES_WINDOW_GET_WCH_METHODDEF - {"getmaxyx", (PyCFunction)PyCursesWindow_getmaxyx, METH_NOARGS}, - {"getparyx", (PyCFunction)PyCursesWindow_getparyx, METH_NOARGS}, - {"getstr", (PyCFunction)PyCursesWindow_GetStr, METH_VARARGS}, - {"getyx", (PyCFunction)PyCursesWindow_getyx, METH_NOARGS}, + {"getmaxyx", PyCursesWindow_getmaxyx, METH_NOARGS}, + {"getparyx", PyCursesWindow_getparyx, METH_NOARGS}, + {"getstr", PyCursesWindow_GetStr, METH_VARARGS}, + {"getyx", PyCursesWindow_getyx, METH_NOARGS}, _CURSES_WINDOW_HLINE_METHODDEF - {"idcok", (PyCFunction)PyCursesWindow_idcok, METH_VARARGS}, - {"idlok", (PyCFunction)PyCursesWindow_idlok, METH_VARARGS}, + {"idcok", PyCursesWindow_idcok, METH_VARARGS}, + {"idlok", PyCursesWindow_idlok, METH_VARARGS}, #ifdef HAVE_CURSES_IMMEDOK - {"immedok", (PyCFunction)PyCursesWindow_immedok, METH_VARARGS}, + {"immedok", PyCursesWindow_immedok, METH_VARARGS}, #endif _CURSES_WINDOW_INCH_METHODDEF _CURSES_WINDOW_INSCH_METHODDEF - {"insdelln", (PyCFunction)PyCursesWindow_winsdelln, METH_VARARGS}, - {"insertln", (PyCFunction)PyCursesWindow_winsertln, METH_NOARGS}, + {"insdelln", PyCursesWindow_winsdelln, METH_VARARGS}, + {"insertln", PyCursesWindow_winsertln, METH_NOARGS}, _CURSES_WINDOW_INSNSTR_METHODDEF _CURSES_WINDOW_INSSTR_METHODDEF - {"instr", (PyCFunction)PyCursesWindow_InStr, METH_VARARGS}, + {"instr", PyCursesWindow_InStr, METH_VARARGS}, _CURSES_WINDOW_IS_LINETOUCHED_METHODDEF - {"is_wintouched", (PyCFunction)PyCursesWindow_is_wintouched, METH_NOARGS}, - {"keypad", (PyCFunction)PyCursesWindow_keypad, METH_VARARGS}, - {"leaveok", (PyCFunction)PyCursesWindow_leaveok, METH_VARARGS}, - {"move", (PyCFunction)PyCursesWindow_wmove, METH_VARARGS}, - {"mvderwin", (PyCFunction)PyCursesWindow_mvderwin, METH_VARARGS}, - {"mvwin", (PyCFunction)PyCursesWindow_mvwin, METH_VARARGS}, - {"nodelay", (PyCFunction)PyCursesWindow_nodelay, METH_VARARGS}, - {"notimeout", (PyCFunction)PyCursesWindow_notimeout, METH_VARARGS}, + {"is_wintouched", PyCursesWindow_is_wintouched, METH_NOARGS}, + {"keypad", PyCursesWindow_keypad, METH_VARARGS}, + {"leaveok", PyCursesWindow_leaveok, METH_VARARGS}, + {"move", PyCursesWindow_wmove, METH_VARARGS}, + {"mvderwin", PyCursesWindow_mvderwin, METH_VARARGS}, + {"mvwin", PyCursesWindow_mvwin, METH_VARARGS}, + {"nodelay", PyCursesWindow_nodelay, METH_VARARGS}, + {"notimeout", PyCursesWindow_notimeout, METH_VARARGS}, _CURSES_WINDOW_NOUTREFRESH_METHODDEF _CURSES_WINDOW_OVERLAY_METHODDEF _CURSES_WINDOW_OVERWRITE_METHODDEF _CURSES_WINDOW_PUTWIN_METHODDEF _CURSES_WINDOW_REDRAWLN_METHODDEF - {"redrawwin", (PyCFunction)PyCursesWindow_redrawwin, METH_NOARGS}, + {"redrawwin", PyCursesWindow_redrawwin, METH_NOARGS}, _CURSES_WINDOW_REFRESH_METHODDEF #ifndef STRICT_SYSV_CURSES - {"resize", (PyCFunction)PyCursesWindow_wresize, METH_VARARGS}, + {"resize", PyCursesWindow_wresize, METH_VARARGS}, #endif _CURSES_WINDOW_SCROLL_METHODDEF - {"scrollok", (PyCFunction)PyCursesWindow_scrollok, METH_VARARGS}, + {"scrollok", PyCursesWindow_scrollok, METH_VARARGS}, _CURSES_WINDOW_SETSCRREG_METHODDEF - {"standend", (PyCFunction)PyCursesWindow_wstandend, METH_NOARGS}, - {"standout", (PyCFunction)PyCursesWindow_wstandout, METH_NOARGS}, + {"standend", PyCursesWindow_wstandend, METH_NOARGS}, + {"standout", PyCursesWindow_wstandout, METH_NOARGS}, {"subpad", (PyCFunction)_curses_window_subwin, METH_VARARGS, _curses_window_subwin__doc__}, _CURSES_WINDOW_SUBWIN_METHODDEF - {"syncdown", (PyCFunction)PyCursesWindow_wsyncdown, METH_NOARGS}, + {"syncdown", PyCursesWindow_wsyncdown, METH_NOARGS}, #ifdef HAVE_CURSES_SYNCOK - {"syncok", (PyCFunction)PyCursesWindow_syncok, METH_VARARGS}, + {"syncok", PyCursesWindow_syncok, METH_VARARGS}, #endif - {"syncup", (PyCFunction)PyCursesWindow_wsyncup, METH_NOARGS}, - {"timeout", (PyCFunction)PyCursesWindow_wtimeout, METH_VARARGS}, + {"syncup", PyCursesWindow_wsyncup, METH_NOARGS}, + {"timeout", PyCursesWindow_wtimeout, METH_VARARGS}, _CURSES_WINDOW_TOUCHLINE_METHODDEF - {"touchwin", (PyCFunction)PyCursesWindow_touchwin, METH_NOARGS}, - {"untouchwin", (PyCFunction)PyCursesWindow_untouchwin, METH_NOARGS}, + {"touchwin", PyCursesWindow_touchwin, METH_NOARGS}, + {"untouchwin", PyCursesWindow_untouchwin, METH_NOARGS}, _CURSES_WINDOW_VLINE_METHODDEF {NULL, NULL} /* sentinel */ }; static PyGetSetDef PyCursesWindow_getsets[] = { - {"encoding", - (getter)PyCursesWindow_get_encoding, - (setter)PyCursesWindow_set_encoding, - "the typecode character used to create the array"}, + { + "encoding", + PyCursesWindow_get_encoding, + PyCursesWindow_set_encoding, + "the typecode character used to create the array" + }, {NULL, NULL, NULL, NULL } /* sentinel */ }; -/* -------------------------------------------------------*/ +static PyType_Slot PyCursesWindow_Type_slots[] = { + {Py_tp_methods, PyCursesWindow_methods}, + {Py_tp_getset, PyCursesWindow_getsets}, + {Py_tp_dealloc, PyCursesWindow_dealloc}, + {Py_tp_traverse, PyCursesWindow_traverse}, + {0, NULL} +}; -PyTypeObject PyCursesWindow_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_curses.window", /*tp_name*/ - sizeof(PyCursesWindowObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - /* methods */ - (destructor)PyCursesWindow_Dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - (getattrfunc)0, /*tp_getattr*/ - (setattrfunc)0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /*tp_flags*/ - 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - PyCursesWindow_Methods, /*tp_methods*/ - 0, /* tp_members */ - PyCursesWindow_getsets, /* tp_getset */ +static PyType_Spec PyCursesWindow_Type_spec = { + .name = "_curses.window", + .basicsize = sizeof(PyCursesWindowObject), + .flags = Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_DISALLOW_INSTANTIATION + | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HEAPTYPE + | Py_TPFLAGS_HAVE_GC, + .slots = PyCursesWindow_Type_slots }; -/* Function Prototype Macros - They are ugly but very, very useful. ;-) +/* -------------------------------------------------------*/ + +/* Function Body Macros - They are ugly but very, very useful. ;-) X - function name TYPE - parameter Type @@ -2613,37 +2763,37 @@ PyTypeObject PyCursesWindow_Type = { #define NoArgNoReturnFunctionBody(X) \ { \ - PyCursesInitialised \ - return PyCursesCheckERR(X(), # X); } + PyCursesStatefulInitialised(module); \ + return PyCursesCheckERR(module, X(), # X); } #define NoArgOrFlagNoReturnFunctionBody(X, flag) \ { \ - PyCursesInitialised \ + PyCursesStatefulInitialised(module); \ if (flag) \ - return PyCursesCheckERR(X(), # X); \ + return PyCursesCheckERR(module, X(), # X); \ else \ - return PyCursesCheckERR(no ## X(), # X); \ + return PyCursesCheckERR(module, no ## X(), # X); \ } #define NoArgReturnIntFunctionBody(X) \ { \ - PyCursesInitialised \ + PyCursesStatefulInitialised(module); \ return PyLong_FromLong((long) X()); } #define NoArgReturnStringFunctionBody(X) \ { \ - PyCursesInitialised \ + PyCursesStatefulInitialised(module); \ return PyBytes_FromString(X()); } #define NoArgTrueFalseFunctionBody(X) \ { \ - PyCursesInitialised \ + PyCursesStatefulInitialised(module); \ return PyBool_FromLong(X()); } #define NoArgNoReturnVoidFunctionBody(X) \ { \ - PyCursesInitialised \ + PyCursesStatefulInitialised(module); \ X(); \ Py_RETURN_NONE; } @@ -2741,12 +2891,13 @@ _curses_color_content_impl(PyObject *module, int color_number) { _CURSES_COLOR_VAL_TYPE r,g,b; - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); if (_COLOR_CONTENT_FUNC(color_number, &r, &g, &b) == ERR) { - PyErr_Format(PyCursesError, "%s() returned ERR", - Py_STRINGIFY(_COLOR_CONTENT_FUNC)); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_Format(state->error, "%s() returned ERR", + Py_STRINGIFY(_COLOR_CONTENT_FUNC)); return NULL; } @@ -2770,8 +2921,8 @@ static PyObject * _curses_color_pair_impl(PyObject *module, int pair_number) /*[clinic end generated code: output=60718abb10ce9feb input=6034e9146f343802]*/ { - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); return PyLong_FromLong(COLOR_PAIR(pair_number)); } @@ -2797,10 +2948,10 @@ _curses_curs_set_impl(PyObject *module, int visibility) { int erg; - PyCursesInitialised; + PyCursesStatefulInitialised(module); erg = curs_set(visibility); - if (erg == ERR) return PyCursesCheckERR(erg, "curs_set"); + if (erg == ERR) return PyCursesCheckERR(module, erg, "curs_set"); return PyLong_FromLong((long) erg); } @@ -2849,9 +3000,9 @@ static PyObject * _curses_delay_output_impl(PyObject *module, int ms) /*[clinic end generated code: output=b6613a67f17fa4f4 input=5316457f5f59196c]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(delay_output(ms), "delay_output"); + return PyCursesCheckERR(module, delay_output(ms), "delay_output"); } /*[clinic input] @@ -2905,7 +3056,7 @@ _curses_erasechar_impl(PyObject *module) { char ch; - PyCursesInitialised; + PyCursesStatefulInitialised(module); ch = erasechar(); @@ -2955,7 +3106,7 @@ _curses_getsyx_impl(PyObject *module) int x = 0; int y = 0; - PyCursesInitialised; + PyCursesStatefulInitialised(module); getsyx(y, x); @@ -2980,11 +3131,12 @@ _curses_getmouse_impl(PyObject *module) int rtn; MEVENT event; - PyCursesInitialised; + PyCursesStatefulInitialised(module); rtn = getmouse( &event ); if (rtn == ERR) { - PyErr_SetString(PyCursesError, "getmouse() returned ERR"); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, "getmouse() returned ERR"); return NULL; } return Py_BuildValue("(hiiik)", @@ -3015,14 +3167,14 @@ _curses_ungetmouse_impl(PyObject *module, short id, int x, int y, int z, { MEVENT event; - PyCursesInitialised; + PyCursesStatefulInitialised(module); event.id = id; event.x = x; event.y = y; event.z = z; event.bstate = bstate; - return PyCursesCheckERR(ungetmouse(&event), "ungetmouse"); + return PyCursesCheckERR(module, ungetmouse(&event), "ungetmouse"); } #endif @@ -3048,7 +3200,7 @@ _curses_getwin(PyObject *module, PyObject *file) WINDOW *win; PyObject *res = NULL; - PyCursesInitialised; + PyCursesStatefulInitialised(module); fp = tmpfile(); if (fp == NULL) @@ -3078,10 +3230,12 @@ _curses_getwin(PyObject *module, PyObject *file) fseek(fp, 0, 0); win = getwin(fp); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, catchall_NULL); goto error; } - res = PyCursesWindow_New(win, NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + res = PyCursesWindow_New(state, win, NULL); error: fclose(fp); @@ -3104,9 +3258,9 @@ static PyObject * _curses_halfdelay_impl(PyObject *module, unsigned char tenths) /*[clinic end generated code: output=e92cdf0ef33c0663 input=e42dce7259c15100]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(halfdelay(tenths), "halfdelay"); + return PyCursesCheckERR(module, halfdelay(tenths), "halfdelay"); } /*[clinic input] @@ -3157,7 +3311,7 @@ static PyObject * _curses_has_key_impl(PyObject *module, int key) /*[clinic end generated code: output=19ad48319414d0b1 input=78bd44acf1a4997c]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); return PyBool_FromLong(has_key(key)); } @@ -3188,10 +3342,11 @@ _curses_init_color_impl(PyObject *module, int color_number, short r, short g, short b) /*[clinic end generated code: output=d7ed71b2d818cdf2 input=ae2b8bea0f152c80]*/ { - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); - return PyCursesCheckERR(_CURSES_INIT_COLOR_FUNC(color_number, r, g, b), + return PyCursesCheckERR(module, + _CURSES_INIT_COLOR_FUNC(color_number, r, g, b), Py_STRINGIFY(_CURSES_INIT_COLOR_FUNC)); } @@ -3216,8 +3371,8 @@ static PyObject * _curses_init_pair_impl(PyObject *module, int pair_number, int fg, int bg) /*[clinic end generated code: output=a0bba03d2bbc3ee6 input=54b421b44c12c389]*/ { - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); if (_CURSES_INIT_PAIR_FUNC(pair_number, fg, bg) == ERR) { if (pair_number >= COLOR_PAIRS) { @@ -3226,7 +3381,8 @@ _curses_init_pair_impl(PyObject *module, int pair_number, int fg, int bg) COLOR_PAIRS - 1); } else { - PyErr_Format(PyCursesError, "%s() returned ERR", + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_Format(state->error, "%s() returned ERR", Py_STRINGIFY(_CURSES_INIT_PAIR_FUNC)); } return NULL; @@ -3235,8 +3391,6 @@ _curses_init_pair_impl(PyObject *module, int pair_number, int fg, int bg) Py_RETURN_NONE; } -static PyObject *ModDict; - /*[clinic input] _curses.initscr @@ -3250,30 +3404,40 @@ _curses_initscr_impl(PyObject *module) /*[clinic end generated code: output=619fb68443810b7b input=514f4bce1821f6b5]*/ { WINDOW *win; - PyCursesWindowObject *winobj; - if (initialised) { + if (curses_initscr_called) { wrefresh(stdscr); - return (PyObject *)PyCursesWindow_New(stdscr, NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + return PyCursesWindow_New(state, stdscr, NULL); } win = initscr(); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, catchall_NULL); return NULL; } - initialised = initialised_setupterm = TRUE; + curses_initscr_called = curses_setupterm_called = TRUE; -/* This was moved from initcurses() because it core dumped on SGI, - where they're not defined until you've called initscr() */ -#define SetDictInt(string,ch) \ - do { \ - PyObject *o = PyLong_FromLong((long) (ch)); \ - if (o && PyDict_SetItemString(ModDict, string, o) == 0) { \ - Py_DECREF(o); \ - } \ + PyObject *module_dict = PyModule_GetDict(module); // borrowed + if (module_dict == NULL) { + return NULL; + } + /* This was moved from initcurses() because it core dumped on SGI, + where they're not defined until you've called initscr() */ +#define SetDictInt(NAME, VALUE) \ + do { \ + PyObject *value = PyLong_FromLong((long)(VALUE)); \ + if (value == NULL) { \ + return NULL; \ + } \ + int rc = PyDict_SetItemString(module_dict, (NAME), value); \ + Py_DECREF(value); \ + if (rc < 0) { \ + return NULL; \ + } \ } while (0) /* Here are some graphic symbols you can use */ @@ -3345,10 +3509,15 @@ _curses_initscr_impl(PyObject *module) SetDictInt("LINES", LINES); SetDictInt("COLS", COLS); +#undef SetDictInt - winobj = (PyCursesWindowObject *)PyCursesWindow_New(win, NULL); - screen_encoding = winobj->encoding; - return (PyObject *)winobj; + cursesmodule_state *state = get_cursesmodule_state(module); + PyObject *winobj = PyCursesWindow_New(state, win, NULL); + if (winobj == NULL) { + return NULL; + } + curses_screen_encoding = ((PyCursesWindowObject *)winobj)->encoding; + return winobj; } /*[clinic input] @@ -3376,9 +3545,8 @@ _curses_setupterm_impl(PyObject *module, const char *term, int fd) sys_stdout = PySys_GetObject("stdout"); if (sys_stdout == NULL || sys_stdout == Py_None) { - PyErr_SetString( - PyCursesError, - "lost sys.stdout"); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, "lost sys.stdout"); return NULL; } @@ -3389,7 +3557,7 @@ _curses_setupterm_impl(PyObject *module, const char *term, int fd) } } - if (!initialised_setupterm && setupterm((char *)term, fd, &err) == ERR) { + if (!curses_setupterm_called && setupterm((char *)term, fd, &err) == ERR) { const char* s = "setupterm: unknown error"; if (err == 0) { @@ -3398,11 +3566,12 @@ _curses_setupterm_impl(PyObject *module, const char *term, int fd) s = "setupterm: could not find terminfo database"; } - PyErr_SetString(PyCursesError,s); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, s); return NULL; } - initialised_setupterm = TRUE; + curses_setupterm_called = TRUE; Py_RETURN_NONE; } @@ -3448,7 +3617,7 @@ _curses_set_escdelay_impl(PyObject *module, int ms) return NULL; } - return PyCursesCheckERR(set_escdelay(ms), "set_escdelay"); + return PyCursesCheckERR(module, set_escdelay(ms), "set_escdelay"); } /*[clinic input] @@ -3487,7 +3656,7 @@ _curses_set_tabsize_impl(PyObject *module, int size) return NULL; } - return PyCursesCheckERR(set_tabsize(size), "set_tabsize"); + return PyCursesCheckERR(module, set_tabsize(size), "set_tabsize"); } #endif @@ -3503,9 +3672,9 @@ static PyObject * _curses_intrflush_impl(PyObject *module, int flag) /*[clinic end generated code: output=c1986df35e999a0f input=c65fe2ef973fe40a]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(intrflush(NULL, flag), "intrflush"); + return PyCursesCheckERR(module, intrflush(NULL, flag), "intrflush"); } /*[clinic input] @@ -3536,7 +3705,7 @@ static PyObject * _curses_is_term_resized_impl(PyObject *module, int nlines, int ncols) /*[clinic end generated code: output=aafe04afe50f1288 input=ca9c0bd0fb8ab444]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); return PyBool_FromLong(is_term_resized(nlines, ncols)); } @@ -3558,7 +3727,7 @@ _curses_keyname_impl(PyObject *module, int key) { const char *knp; - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (key < 0) { PyErr_SetString(PyExc_ValueError, "invalid key number"); @@ -3616,9 +3785,9 @@ static PyObject * _curses_meta_impl(PyObject *module, int yes) /*[clinic end generated code: output=22f5abda46a605d8 input=cfe7da79f51d0e30]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(meta(stdscr, yes), "meta"); + return PyCursesCheckERR(module, meta(stdscr, yes), "meta"); } #ifdef NCURSES_MOUSE_VERSION @@ -3640,9 +3809,9 @@ static PyObject * _curses_mouseinterval_impl(PyObject *module, int interval) /*[clinic end generated code: output=c4f5ff04354634c5 input=75aaa3f0db10ac4e]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(mouseinterval(interval), "mouseinterval"); + return PyCursesCheckERR(module, mouseinterval(interval), "mouseinterval"); } /*[clinic input] @@ -3665,7 +3834,7 @@ _curses_mousemask_impl(PyObject *module, unsigned long newmask) { mmask_t oldmask, availmask; - PyCursesInitialised; + PyCursesStatefulInitialised(module); availmask = mousemask((mmask_t)newmask, &oldmask); return Py_BuildValue("(kk)", (unsigned long)availmask, (unsigned long)oldmask); @@ -3686,8 +3855,11 @@ static int _curses_napms_impl(PyObject *module, int ms) /*[clinic end generated code: output=5f292a6a724491bd input=c6d6e01f2f1df9f7]*/ { - PyCursesInitialised; - + if (!_PyCursesStatefulCheckFunction(module, + curses_initscr_called, + "initscr")) { + return -1; + } return napms(ms); } @@ -3710,16 +3882,18 @@ _curses_newpad_impl(PyObject *module, int nlines, int ncols) { WINDOW *win; - PyCursesInitialised; + PyCursesStatefulInitialised(module); win = newpad(nlines, ncols); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, catchall_NULL); return NULL; } - return (PyObject *)PyCursesWindow_New(win, NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + return PyCursesWindow_New(state, win, NULL); } /*[clinic input] @@ -3750,15 +3924,17 @@ _curses_newwin_impl(PyObject *module, int nlines, int ncols, { WINDOW *win; - PyCursesInitialised; + PyCursesStatefulInitialised(module); win = newwin(nlines,ncols,begin_y,begin_x); if (win == NULL) { - PyErr_SetString(PyCursesError, catchall_NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, catchall_NULL); return NULL; } - return (PyObject *)PyCursesWindow_New(win, NULL); + cursesmodule_state *state = get_cursesmodule_state(module); + return PyCursesWindow_New(state, win, NULL); } /*[clinic input] @@ -3862,8 +4038,8 @@ _curses_pair_content_impl(PyObject *module, int pair_number) { _CURSES_COLOR_NUM_TYPE f, b; - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); if (_CURSES_PAIR_CONTENT_FUNC(pair_number, &f, &b) == ERR) { if (pair_number >= COLOR_PAIRS) { @@ -3872,7 +4048,8 @@ _curses_pair_content_impl(PyObject *module, int pair_number) COLOR_PAIRS - 1); } else { - PyErr_Format(PyCursesError, "%s() returned ERR", + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_Format(state->error, "%s() returned ERR", Py_STRINGIFY(_CURSES_PAIR_CONTENT_FUNC)); } return NULL; @@ -3896,8 +4073,8 @@ static PyObject * _curses_pair_number_impl(PyObject *module, int attr) /*[clinic end generated code: output=85bce7d65c0aa3f4 input=d478548e33f5e61a]*/ { - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); return PyLong_FromLong(PAIR_NUMBER(attr)); } @@ -3917,7 +4094,7 @@ static PyObject * _curses_putp_impl(PyObject *module, const char *string) /*[clinic end generated code: output=e98081d1b8eb5816 input=1601faa828b44cb3]*/ { - return PyCursesCheckERR(putp(string), "putp"); + return PyCursesCheckERR(module, putp(string), "putp"); } /*[clinic input] @@ -3937,7 +4114,7 @@ static PyObject * _curses_qiflush_impl(PyObject *module, int flag) /*[clinic end generated code: output=9167e862f760ea30 input=6ec8b3e2b717ec40]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (flag) { qiflush(); @@ -3948,52 +4125,57 @@ _curses_qiflush_impl(PyObject *module, int flag) Py_RETURN_NONE; } -/* Internal helper used for updating curses.LINES, curses.COLS, _curses.LINES - * and _curses.COLS */ #if defined(HAVE_CURSES_RESIZETERM) || defined(HAVE_CURSES_RESIZE_TERM) +/* Internal helper used for updating curses.LINES, curses.COLS, _curses.LINES + * and _curses.COLS. Returns 1 on success and 0 on failure. */ static int -update_lines_cols(void) +update_lines_cols(PyObject *private_module) { - PyObject *o; - PyObject *m = PyImport_ImportModule("curses"); + PyObject *exposed_module = NULL, *o = NULL; - if (!m) - return 0; + exposed_module = PyImport_ImportModule("curses"); + if (exposed_module == NULL) { + goto error; + } + PyObject *exposed_module_dict = PyModule_GetDict(exposed_module); // borrowed + if (exposed_module_dict == NULL) { + goto error; + } + PyObject *private_module_dict = PyModule_GetDict(private_module); // borrowed + if (private_module_dict == NULL) { + goto error; + } o = PyLong_FromLong(LINES); - if (!o) { - Py_DECREF(m); - return 0; + if (o == NULL) { + goto error; } - if (PyObject_SetAttrString(m, "LINES", o)) { - Py_DECREF(m); - Py_DECREF(o); - return 0; + if (PyDict_SetItemString(exposed_module_dict, "LINES", o) < 0) { + goto error; } - if (PyDict_SetItemString(ModDict, "LINES", o)) { - Py_DECREF(m); - Py_DECREF(o); - return 0; + if (PyDict_SetItemString(private_module_dict, "LINES", o) < 0) { + goto error; } Py_DECREF(o); + o = PyLong_FromLong(COLS); - if (!o) { - Py_DECREF(m); - return 0; + if (o == NULL) { + goto error; } - if (PyObject_SetAttrString(m, "COLS", o)) { - Py_DECREF(m); - Py_DECREF(o); - return 0; + if (PyDict_SetItemString(exposed_module_dict, "COLS", o) < 0) { + goto error; } - if (PyDict_SetItemString(ModDict, "COLS", o)) { - Py_DECREF(m); - Py_DECREF(o); - return 0; + if (PyDict_SetItemString(private_module_dict, "COLS", o) < 0) { + goto error; } Py_DECREF(o); - Py_DECREF(m); + Py_DECREF(exposed_module); return 1; + +error: + Py_XDECREF(o); + Py_XDECREF(exposed_module); + return 0; } /*[clinic input] @@ -4005,7 +4187,7 @@ static PyObject * _curses_update_lines_cols_impl(PyObject *module) /*[clinic end generated code: output=423f2b1e63ed0f75 input=5f065ab7a28a5d90]*/ { - if (!update_lines_cols()) { + if (!update_lines_cols(module)) { return NULL; } Py_RETURN_NONE; @@ -4069,9 +4251,9 @@ NoArgNoReturnFunctionBody(resetty) /*[clinic input] _curses.resizeterm - nlines: int + nlines: short Height. - ncols: int + ncols: short Width. / @@ -4082,17 +4264,17 @@ window dimensions (in particular the SIGWINCH handler). [clinic start generated code]*/ static PyObject * -_curses_resizeterm_impl(PyObject *module, int nlines, int ncols) -/*[clinic end generated code: output=56d6bcc5194ad055 input=0fca02ebad5ffa82]*/ +_curses_resizeterm_impl(PyObject *module, short nlines, short ncols) +/*[clinic end generated code: output=4de3abab50c67f02 input=414e92a63e3e9899]*/ { PyObject *result; - PyCursesInitialised; + PyCursesStatefulInitialised(module); - result = PyCursesCheckERR(resizeterm(nlines, ncols), "resizeterm"); + result = PyCursesCheckERR(module, resizeterm(nlines, ncols), "resizeterm"); if (!result) return NULL; - if (!update_lines_cols()) { + if (!update_lines_cols(module)) { Py_DECREF(result); return NULL; } @@ -4105,9 +4287,9 @@ _curses_resizeterm_impl(PyObject *module, int nlines, int ncols) /*[clinic input] _curses.resize_term - nlines: int + nlines: short Height. - ncols: int + ncols: short Width. / @@ -4121,17 +4303,17 @@ without additional interaction with the application. [clinic start generated code]*/ static PyObject * -_curses_resize_term_impl(PyObject *module, int nlines, int ncols) -/*[clinic end generated code: output=9e26d8b9ea311ed2 input=2197edd05b049ed4]*/ +_curses_resize_term_impl(PyObject *module, short nlines, short ncols) +/*[clinic end generated code: output=46c6d749fa291dbd input=276afa43d8ea7091]*/ { PyObject *result; - PyCursesInitialised; + PyCursesStatefulInitialised(module); - result = PyCursesCheckERR(resize_term(nlines, ncols), "resize_term"); + result = PyCursesCheckERR(module, resize_term(nlines, ncols), "resize_term"); if (!result) return NULL; - if (!update_lines_cols()) { + if (!update_lines_cols(module)) { Py_DECREF(result); return NULL; } @@ -4169,7 +4351,7 @@ static PyObject * _curses_setsyx_impl(PyObject *module, int y, int x) /*[clinic end generated code: output=23dcf753511a2464 input=fa7f2b208e10a557]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); setsyx(y,x); @@ -4194,35 +4376,38 @@ static PyObject * _curses_start_color_impl(PyObject *module) /*[clinic end generated code: output=8b772b41d8090ede input=0ca0ecb2b77e1a12]*/ { - int code; - PyObject *c, *cp; + PyCursesStatefulInitialised(module); - PyCursesInitialised; + if (start_color() == ERR) { + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, "start_color() returned ERR"); + return NULL; + } - code = start_color(); - if (code != ERR) { - initialisedcolors = TRUE; - c = PyLong_FromLong((long) COLORS); - if (c == NULL) - return NULL; - if (PyDict_SetItemString(ModDict, "COLORS", c) < 0) { - Py_DECREF(c); - return NULL; - } - Py_DECREF(c); - cp = PyLong_FromLong((long) COLOR_PAIRS); - if (cp == NULL) - return NULL; - if (PyDict_SetItemString(ModDict, "COLOR_PAIRS", cp) < 0) { - Py_DECREF(cp); - return NULL; - } - Py_DECREF(cp); - Py_RETURN_NONE; - } else { - PyErr_SetString(PyCursesError, "start_color() returned ERR"); + curses_start_color_called = TRUE; + + PyObject *module_dict = PyModule_GetDict(module); // borrowed + if (module_dict == NULL) { return NULL; } +#define DICT_ADD_INT_VALUE(NAME, VALUE) \ + do { \ + PyObject *value = PyLong_FromLong((long)(VALUE)); \ + if (value == NULL) { \ + return NULL; \ + } \ + int rc = PyDict_SetItemString(module_dict, (NAME), value); \ + Py_DECREF(value); \ + if (rc < 0) { \ + return NULL; \ + } \ + } while (0) + + DICT_ADD_INT_VALUE("COLORS", COLORS); + DICT_ADD_INT_VALUE("COLOR_PAIRS", COLOR_PAIRS); +#undef DICT_ADD_INT_VALUE + + Py_RETURN_NONE; } /*[clinic input] @@ -4264,7 +4449,7 @@ static PyObject * _curses_tigetflag_impl(PyObject *module, const char *capname) /*[clinic end generated code: output=8853c0e55542195b input=b0787af9e3e9a6ce]*/ { - PyCursesSetupTermCalled; + PyCursesStatefulSetupTermCalled(module); return PyLong_FromLong( (long) tigetflag( (char *)capname ) ); } @@ -4286,7 +4471,7 @@ static PyObject * _curses_tigetnum_impl(PyObject *module, const char *capname) /*[clinic end generated code: output=46f8b0a1b5dff42f input=5cdf2f410b109720]*/ { - PyCursesSetupTermCalled; + PyCursesStatefulSetupTermCalled(module); return PyLong_FromLong( (long) tigetnum( (char *)capname ) ); } @@ -4308,7 +4493,7 @@ static PyObject * _curses_tigetstr_impl(PyObject *module, const char *capname) /*[clinic end generated code: output=f22b576ad60248f3 input=36644df25c73c0a7]*/ { - PyCursesSetupTermCalled; + PyCursesStatefulSetupTermCalled(module); capname = tigetstr( (char *)capname ); if (capname == NULL || capname == (char*) -1) { @@ -4343,11 +4528,12 @@ _curses_tparm_impl(PyObject *module, const char *str, int i1, int i2, int i3, { char* result = NULL; - PyCursesSetupTermCalled; + PyCursesStatefulSetupTermCalled(module); result = tparm((char *)str,i1,i2,i3,i4,i5,i6,i7,i8,i9); if (!result) { - PyErr_SetString(PyCursesError, "tparm() returned NULL"); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, "tparm() returned NULL"); return NULL; } @@ -4371,9 +4557,9 @@ static PyObject * _curses_typeahead_impl(PyObject *module, int fd) /*[clinic end generated code: output=084bb649d7066583 input=f2968d8e1805051b]*/ { - PyCursesInitialised; + PyCursesStatefulInitialised(module); - return PyCursesCheckERR(typeahead( fd ), "typeahead"); + return PyCursesCheckERR(module, typeahead( fd ), "typeahead"); } #endif @@ -4395,7 +4581,7 @@ _curses_unctrl(PyObject *module, PyObject *ch) { chtype ch_; - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (!PyCurses_ConvertToChtype(NULL, ch, &ch_)) return NULL; @@ -4418,12 +4604,12 @@ _curses_ungetch(PyObject *module, PyObject *ch) { chtype ch_; - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (!PyCurses_ConvertToChtype(NULL, ch, &ch_)) return NULL; - return PyCursesCheckERR(ungetch(ch_), "ungetch"); + return PyCursesCheckERR(module, ungetch(ch_), "ungetch"); } #ifdef HAVE_NCURSESW @@ -4441,7 +4627,7 @@ PyCurses_ConvertToWchar_t(PyObject *obj, wchar_t buffer[2]; if (PyUnicode_AsWideChar(obj, buffer, 2) != 1) { PyErr_Format(PyExc_TypeError, - "expect str of length 1 or int, " + "expect int or str of length 1, " "got a str of length %zi", PyUnicode_GET_LENGTH(obj)); return 0; @@ -4468,7 +4654,7 @@ PyCurses_ConvertToWchar_t(PyObject *obj, } else { PyErr_Format(PyExc_TypeError, - "expect str of length 1 or int, got %s", + "expect int or str of length 1, got %s", Py_TYPE(obj)->tp_name); return 0; } @@ -4489,11 +4675,11 @@ _curses_unget_wch(PyObject *module, PyObject *ch) { wchar_t wch; - PyCursesInitialised; + PyCursesStatefulInitialised(module); if (!PyCurses_ConvertToWchar_t(ch, &wch)) return NULL; - return PyCursesCheckERR(unget_wch(wch), "unget_wch"); + return PyCursesCheckERR(module, unget_wch(wch), "unget_wch"); } #endif @@ -4541,14 +4727,15 @@ _curses_use_default_colors_impl(PyObject *module) { int code; - PyCursesInitialised; - PyCursesInitialisedColor; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); code = use_default_colors(); if (code != ERR) { Py_RETURN_NONE; } else { - PyErr_SetString(PyCursesError, "use_default_colors() returned ERR"); + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, "use_default_colors() returned ERR"); return NULL; } } @@ -4579,26 +4766,32 @@ static PyStructSequence_Desc ncurses_version_desc = { static PyObject * make_ncurses_version(PyTypeObject *type) { - PyObject *ncurses_version; - int pos = 0; - - ncurses_version = PyStructSequence_New(type); + PyObject *ncurses_version = PyStructSequence_New(type); if (ncurses_version == NULL) { return NULL; } - -#define SetIntItem(flag) \ - PyStructSequence_SET_ITEM(ncurses_version, pos++, PyLong_FromLong(flag)); \ - if (PyErr_Occurred()) { \ - Py_CLEAR(ncurses_version); \ - return NULL; \ + const char *str = curses_version(); + unsigned long major = 0, minor = 0, patch = 0; + if (!str || sscanf(str, "%*[^0-9]%lu.%lu.%lu", &major, &minor, &patch) < 3) { + // Fallback to header version, which cannot be that wrong + major = NCURSES_VERSION_MAJOR; + minor = NCURSES_VERSION_MINOR; + patch = NCURSES_VERSION_PATCH; } +#define SET_VERSION_COMPONENT(INDEX, VALUE) \ + do { \ + PyObject *o = PyLong_FromLong(VALUE); \ + if (o == NULL) { \ + Py_DECREF(ncurses_version); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(ncurses_version, INDEX, o); \ + } while (0) - SetIntItem(NCURSES_VERSION_MAJOR) - SetIntItem(NCURSES_VERSION_MINOR) - SetIntItem(NCURSES_VERSION_PATCH) -#undef SetIntItem - + SET_VERSION_COMPONENT(0, major); + SET_VERSION_COMPONENT(1, minor); + SET_VERSION_COMPONENT(2, patch); +#undef SET_VERSION_COMPONENT return ncurses_version; } @@ -4622,7 +4815,7 @@ _curses_has_extended_color_support_impl(PyObject *module) /* List of functions defined in the module */ -static PyMethodDef PyCurses_methods[] = { +static PyMethodDef cursesmodule_methods[] = { _CURSES_BAUDRATE_METHODDEF _CURSES_BEEP_METHODDEF _CURSES_CAN_CHANGE_COLOR_METHODDEF @@ -4707,83 +4900,197 @@ static PyMethodDef PyCurses_methods[] = { {NULL, NULL} /* sentinel */ }; -/* Initialization function for the module */ +/* Module C API */ +/* Function versions of the 3 functions for testing whether curses has been + initialised or not. */ -static struct PyModuleDef _cursesmodule = { - PyModuleDef_HEAD_INIT, - "_curses", - NULL, - -1, - PyCurses_methods, - NULL, - NULL, - NULL, - NULL -}; +static inline int +curses_capi_setupterm_called(void) +{ + return _PyCursesCheckFunction(curses_setupterm_called, "setupterm"); +} -static void -curses_destructor(PyObject *op) +static inline int +curses_capi_initscr_called(void) { - void *ptr = PyCapsule_GetPointer(op, PyCurses_CAPSULE_NAME); - Py_DECREF(*(void **)ptr); - PyMem_Free(ptr); + return _PyCursesCheckFunction(curses_initscr_called, "initscr"); } -PyMODINIT_FUNC -PyInit__curses(void) +static inline int +curses_capi_start_color_called(void) { - PyObject *m, *d, *v, *c_api_object; + return _PyCursesCheckFunction(curses_start_color_called, "start_color"); +} - /* Initialize object type */ - if (PyType_Ready(&PyCursesWindow_Type) < 0) +static void * +curses_capi_new(cursesmodule_state *state) +{ + assert(state->window_type != NULL); + void **capi = (void **)PyMem_Calloc(PyCurses_API_pointers, sizeof(void *)); + if (capi == NULL) { + PyErr_NoMemory(); return NULL; + } + capi[0] = (void *)Py_NewRef(state->window_type); + capi[1] = curses_capi_setupterm_called; + capi[2] = curses_capi_initscr_called; + capi[3] = curses_capi_start_color_called; + return (void *)capi; +} - /* Create the module and add the functions */ - m = PyModule_Create(&_cursesmodule); - if (m == NULL) - return NULL; +static void +curses_capi_free(void *capi) +{ + assert(capi != NULL); + void **capi_ptr = (void **)capi; + // In free-threaded builds, capi_ptr[0] may have been already cleared + // by curses_capi_capsule_destructor(), hence the use of Py_XDECREF(). + Py_XDECREF(capi_ptr[0]); // decref curses window type + PyMem_Free(capi_ptr); +} - /* Add some symbolic constants to the module */ - d = PyModule_GetDict(m); - if (d == NULL) - return NULL; - ModDict = d; /* For PyCurses_InitScr to use later */ +/* Module C API Capsule */ - void **PyCurses_API = PyMem_Calloc(PyCurses_API_pointers, sizeof(void *)); - if (PyCurses_API == NULL) { - PyErr_NoMemory(); +static void +curses_capi_capsule_destructor(PyObject *op) +{ + void *capi = PyCapsule_GetPointer(op, PyCurses_CAPSULE_NAME); + curses_capi_free(capi); +} + +static int +curses_capi_capsule_traverse(PyObject *op, visitproc visit, void *arg) +{ + void **capi_ptr = PyCapsule_GetPointer(op, PyCurses_CAPSULE_NAME); + assert(capi_ptr != NULL); + Py_VISIT(capi_ptr[0]); // visit curses window type + return 0; +} + +static int +curses_capi_capsule_clear(PyObject *op) +{ + void **capi_ptr = PyCapsule_GetPointer(op, PyCurses_CAPSULE_NAME); + assert(capi_ptr != NULL); + Py_CLEAR(capi_ptr[0]); // clear curses window type + return 0; +} + +static PyObject * +curses_capi_capsule_new(void *capi) +{ + PyObject *capsule = PyCapsule_New(capi, PyCurses_CAPSULE_NAME, + curses_capi_capsule_destructor); + if (capsule == NULL) { return NULL; } - /* Initialize the C API pointer array */ - PyCurses_API[0] = (void *)Py_NewRef(&PyCursesWindow_Type); - PyCurses_API[1] = (void *)func_PyCursesSetupTermCalled; - PyCurses_API[2] = (void *)func_PyCursesInitialised; - PyCurses_API[3] = (void *)func_PyCursesInitialisedColor; + if (_PyCapsule_SetTraverse(capsule, + curses_capi_capsule_traverse, + curses_capi_capsule_clear) < 0) + { + Py_DECREF(capsule); + return NULL; + } + return capsule; +} +/* Module initialization and cleanup functions */ + +static int +cursesmodule_traverse(PyObject *mod, visitproc visit, void *arg) +{ + cursesmodule_state *state = get_cursesmodule_state(mod); + Py_VISIT(state->error); + Py_VISIT(state->window_type); + return 0; +} + +static int +cursesmodule_clear(PyObject *mod) +{ + cursesmodule_state *state = get_cursesmodule_state(mod); + Py_CLEAR(state->error); + Py_CLEAR(state->window_type); + return 0; +} + +static void +cursesmodule_free(void *mod) +{ + (void)cursesmodule_clear((PyObject *)mod); + curses_module_loaded = 0; // allow reloading once garbage-collected +} + +static int +cursesmodule_exec(PyObject *module) +{ + if (curses_module_loaded) { + PyErr_SetString(PyExc_ImportError, + "module 'curses' can only be loaded once per process"); + return -1; + } + curses_module_loaded = 1; + + cursesmodule_state *state = get_cursesmodule_state(module); + /* Initialize object type */ + state->window_type = (PyTypeObject *)PyType_FromModuleAndSpec( + module, &PyCursesWindow_Type_spec, NULL); + if (state->window_type == NULL) { + return -1; + } + if (PyModule_AddType(module, state->window_type) < 0) { + return -1; + } + + /* Add some symbolic constants to the module */ + PyObject *module_dict = PyModule_GetDict(module); + if (module_dict == NULL) { + return -1; + } + + /* Create the C API object */ + void *capi = curses_capi_new(state); + if (capi == NULL) { + return -1; + } /* Add a capsule for the C API */ - c_api_object = PyCapsule_New(PyCurses_API, PyCurses_CAPSULE_NAME, - curses_destructor); - if (c_api_object == NULL) { - Py_DECREF(PyCurses_API[0]); - PyMem_Free(PyCurses_API); - return NULL; + PyObject *capi_capsule = curses_capi_capsule_new(capi); + if (capi_capsule == NULL) { + curses_capi_free(capi); + return -1; } - if (PyDict_SetItemString(d, "_C_API", c_api_object) < 0) { - Py_DECREF(c_api_object); - return NULL; + int rc = PyDict_SetItemString(module_dict, "_C_API", capi_capsule); + Py_DECREF(capi_capsule); + if (rc < 0) { + return -1; } - Py_DECREF(c_api_object); /* For exception curses.error */ - PyCursesError = PyErr_NewException("_curses.error", NULL, NULL); - PyDict_SetItemString(d, "error", PyCursesError); + state->error = PyErr_NewException("_curses.error", NULL, NULL); + if (state->error == NULL) { + return -1; + } + rc = PyDict_SetItemString(module_dict, "error", state->error); + if (rc < 0) { + return -1; + } /* Make the version available */ - v = PyBytes_FromString(PyCursesVersion); - PyDict_SetItemString(d, "version", v); - PyDict_SetItemString(d, "__version__", v); - Py_DECREF(v); + PyObject *curses_version = PyBytes_FromString(PyCursesVersion); + if (curses_version == NULL) { + return -1; + } + rc = PyDict_SetItemString(module_dict, "version", curses_version); + if (rc < 0) { + Py_DECREF(curses_version); + return -1; + } + rc = PyDict_SetItemString(module_dict, "__version__", curses_version); + Py_CLEAR(curses_version); + if (rc < 0) { + return -1; + } #ifdef NCURSES_VERSION /* ncurses_version */ @@ -4791,17 +5098,33 @@ PyInit__curses(void) version_type = _PyStructSequence_NewType(&ncurses_version_desc, Py_TPFLAGS_DISALLOW_INSTANTIATION); if (version_type == NULL) { - return NULL; + return -1; } - v = make_ncurses_version(version_type); + PyObject *ncurses_version = make_ncurses_version(version_type); Py_DECREF(version_type); - if (v == NULL) { - return NULL; + if (ncurses_version == NULL) { + return -1; + } + rc = PyDict_SetItemString(module_dict, "ncurses_version", ncurses_version); + Py_CLEAR(ncurses_version); + if (rc < 0) { + return -1; } - PyDict_SetItemString(d, "ncurses_version", v); - Py_DECREF(v); #endif /* NCURSES_VERSION */ +#define SetDictInt(NAME, VALUE) \ + do { \ + PyObject *value = PyLong_FromLong((long)(VALUE)); \ + if (value == NULL) { \ + return -1; \ + } \ + int rc = PyDict_SetItemString(module_dict, (NAME), value); \ + Py_DECREF(value); \ + if (rc < 0) { \ + return -1; \ + } \ + } while (0) + SetDictInt("ERR", ERR); SetDictInt("OK", OK); @@ -4897,43 +5220,71 @@ PyInit__curses(void) SetDictInt("REPORT_MOUSE_POSITION", REPORT_MOUSE_POSITION); #endif /* Now set everything up for KEY_ variables */ - { - int key; - char *key_n; - char *key_n2; - for (key=KEY_MIN;key < KEY_MAX; key++) { - key_n = (char *)keyname(key); - if (key_n == NULL || strcmp(key_n,"UNKNOWN KEY")==0) - continue; - if (strncmp(key_n,"KEY_F(",6)==0) { - char *p1, *p2; - key_n2 = PyMem_Malloc(strlen(key_n)+1); - if (!key_n2) { - PyErr_NoMemory(); - break; - } - p1 = key_n; - p2 = key_n2; - while (*p1) { - if (*p1 != '(' && *p1 != ')') { - *p2 = *p1; - p2++; - } - p1++; + for (int keycode = KEY_MIN; keycode < KEY_MAX; keycode++) { + const char *key_name = keyname(keycode); + if (key_name == NULL || strcmp(key_name, "UNKNOWN KEY") == 0) { + continue; + } + if (strncmp(key_name, "KEY_F(", 6) == 0) { + char *fn_key_name = PyMem_Malloc(strlen(key_name) + 1); + if (!fn_key_name) { + PyErr_NoMemory(); + return -1; + } + const char *p1 = key_name; + char *p2 = fn_key_name; + while (*p1) { + if (*p1 != '(' && *p1 != ')') { + *p2 = *p1; + p2++; } - *p2 = (char)0; - } else - key_n2 = key_n; - SetDictInt(key_n2,key); - if (key_n2 != key_n) - PyMem_Free(key_n2); + p1++; + } + *p2 = (char)0; + PyObject *p_keycode = PyLong_FromLong((long)keycode); + if (p_keycode == NULL) { + PyMem_Free(fn_key_name); + return -1; + } + int rc = PyDict_SetItemString(module_dict, fn_key_name, p_keycode); + Py_DECREF(p_keycode); + PyMem_Free(fn_key_name); + if (rc < 0) { + return -1; + } + } + else { + SetDictInt(key_name, keycode); } - SetDictInt("KEY_MIN", KEY_MIN); - SetDictInt("KEY_MAX", KEY_MAX); } + SetDictInt("KEY_MIN", KEY_MIN); + SetDictInt("KEY_MAX", KEY_MAX); +#undef SetDictInt + return 0; +} - if (PyModule_AddType(m, &PyCursesWindow_Type) < 0) { - return NULL; - } - return m; +/* Initialization function for the module */ + +static PyModuleDef_Slot cursesmodule_slots[] = { + {Py_mod_exec, cursesmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL} +}; + +static struct PyModuleDef cursesmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "_curses", + .m_size = sizeof(cursesmodule_state), + .m_methods = cursesmodule_methods, + .m_slots = cursesmodule_slots, + .m_traverse = cursesmodule_traverse, + .m_clear = cursesmodule_clear, + .m_free = cursesmodule_free +}; + +PyMODINIT_FUNC +PyInit__curses(void) +{ + return PyModuleDef_Init(&cursesmodule); } diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index a626bda2ea9be9..ff2e6d6a098ad9 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -1,6 +1,4 @@ -/* C implementation for the date/time type documented at - * https://www.zope.dev/Members/fdrake/DateTimeWiki/FrontPage - */ +/* C implementation of the datetime module */ /* bpo-35081: Defining this prevents including the C API capsule; * internal versions of the Py*_Check macros which do not require @@ -25,24 +23,20 @@ # include /* struct timeval */ #endif -#define PyDate_Check(op) PyObject_TypeCheck(op, &PyDateTime_DateType) -#define PyDate_CheckExact(op) Py_IS_TYPE(op, &PyDateTime_DateType) - -#define PyDateTime_Check(op) PyObject_TypeCheck(op, &PyDateTime_DateTimeType) -#define PyDateTime_CheckExact(op) Py_IS_TYPE(op, &PyDateTime_DateTimeType) - -#define PyTime_Check(op) PyObject_TypeCheck(op, &PyDateTime_TimeType) -#define PyTime_CheckExact(op) Py_IS_TYPE(op, &PyDateTime_TimeType) -#define PyDelta_Check(op) PyObject_TypeCheck(op, &PyDateTime_DeltaType) -#define PyDelta_CheckExact(op) Py_IS_TYPE(op, &PyDateTime_DeltaType) - -#define PyTZInfo_Check(op) PyObject_TypeCheck(op, &PyDateTime_TZInfoType) -#define PyTZInfo_CheckExact(op) Py_IS_TYPE(op, &PyDateTime_TZInfoType) +/* forward declarations */ +static PyTypeObject PyDateTime_DateType; +static PyTypeObject PyDateTime_DateTimeType; +static PyTypeObject PyDateTime_TimeType; +static PyTypeObject PyDateTime_DeltaType; +static PyTypeObject PyDateTime_TZInfoType; +static PyTypeObject PyDateTime_TimeZoneType; -#define PyTimezone_Check(op) PyObject_TypeCheck(op, &PyDateTime_TimeZoneType) typedef struct { + /* Module heap types. */ + PyTypeObject *isocalendar_date_type; + /* Conversion factors. */ PyObject *us_per_ms; // 1_000 PyObject *us_per_second; // 1_000_000 @@ -52,16 +46,192 @@ typedef struct { PyObject *us_per_week; // 1e6 * 3600 * 24 * 7 as Python int PyObject *seconds_per_day; // 3600 * 24 as Python int - /* The interned UTC timezone instance */ - PyObject *utc; - /* The interned Unix epoch datetime instance */ PyObject *epoch; } datetime_state; -static datetime_state _datetime_global_state; +/* The module has a fixed number of static objects, due to being exposed + * through the datetime C-API. There are five types exposed directly, + * one type exposed indirectly, and one singleton constant (UTC). + * + * Each of these objects is hidden behind a macro in the same way as + * the per-module objects stored in module state. The macros for the + * static objects don't need to be passed a state, but the consistency + * of doing so is more clear. We use a dedicated noop macro, NO_STATE, + * to make the special case obvious. */ + +#define NO_STATE NULL + +#define DATE_TYPE(st) &PyDateTime_DateType +#define DATETIME_TYPE(st) &PyDateTime_DateTimeType +#define TIME_TYPE(st) &PyDateTime_TimeType +#define DELTA_TYPE(st) &PyDateTime_DeltaType +#define TZINFO_TYPE(st) &PyDateTime_TZInfoType +#define TIMEZONE_TYPE(st) &PyDateTime_TimeZoneType +#define ISOCALENDAR_DATE_TYPE(st) st->isocalendar_date_type + +#define PyDate_Check(op) PyObject_TypeCheck(op, DATE_TYPE(NO_STATE)) +#define PyDate_CheckExact(op) Py_IS_TYPE(op, DATE_TYPE(NO_STATE)) + +#define PyDateTime_Check(op) PyObject_TypeCheck(op, DATETIME_TYPE(NO_STATE)) +#define PyDateTime_CheckExact(op) Py_IS_TYPE(op, DATETIME_TYPE(NO_STATE)) + +#define PyTime_Check(op) PyObject_TypeCheck(op, TIME_TYPE(NO_STATE)) +#define PyTime_CheckExact(op) Py_IS_TYPE(op, TIME_TYPE(NO_STATE)) + +#define PyDelta_Check(op) PyObject_TypeCheck(op, DELTA_TYPE(NO_STATE)) +#define PyDelta_CheckExact(op) Py_IS_TYPE(op, DELTA_TYPE(NO_STATE)) + +#define PyTZInfo_Check(op) PyObject_TypeCheck(op, TZINFO_TYPE(NO_STATE)) +#define PyTZInfo_CheckExact(op) Py_IS_TYPE(op, TZINFO_TYPE(NO_STATE)) + +#define PyTimezone_Check(op) PyObject_TypeCheck(op, TIMEZONE_TYPE(NO_STATE)) + +#define CONST_US_PER_MS(st) st->us_per_ms +#define CONST_US_PER_SECOND(st) st->us_per_second +#define CONST_US_PER_MINUTE(st) st->us_per_minute +#define CONST_US_PER_HOUR(st) st->us_per_hour +#define CONST_US_PER_DAY(st) st->us_per_day +#define CONST_US_PER_WEEK(st) st->us_per_week +#define CONST_SEC_PER_DAY(st) st->seconds_per_day +#define CONST_EPOCH(st) st->epoch +#define CONST_UTC(st) ((PyObject *)&utc_timezone) + +static datetime_state * +get_module_state(PyObject *module) +{ + void *state = _PyModule_GetState(module); + assert(state != NULL); + return (datetime_state *)state; +} + + +#define INTERP_KEY ((PyObject *)&_Py_ID(cached_datetime_module)) + +static PyObject * +get_current_module(PyInterpreterState *interp, int *p_reloading) +{ + PyObject *mod = NULL; + int reloading = 0; + + PyObject *dict = PyInterpreterState_GetDict(interp); + if (dict == NULL) { + goto error; + } + PyObject *ref = NULL; + if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { + goto error; + } + if (ref != NULL) { + reloading = 1; + if (ref != Py_None) { + (void)PyWeakref_GetRef(ref, &mod); + if (mod == Py_None) { + Py_CLEAR(mod); + } + Py_DECREF(ref); + } + } + if (p_reloading != NULL) { + *p_reloading = reloading; + } + return mod; + +error: + assert(PyErr_Occurred()); + return NULL; +} + +static PyModuleDef datetimemodule; + +static datetime_state * +_get_current_state(PyObject **p_mod) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + PyObject *mod = get_current_module(interp, NULL); + if (mod == NULL) { + assert(!PyErr_Occurred()); + if (PyErr_Occurred()) { + return NULL; + } + /* The static types can outlive the module, + * so we must re-import the module. */ + mod = PyImport_ImportModule("_datetime"); + if (mod == NULL) { + return NULL; + } + } + datetime_state *st = get_module_state(mod); + *p_mod = mod; + return st; +} + +#define GET_CURRENT_STATE(MOD_VAR) \ + _get_current_state(&MOD_VAR) +#define RELEASE_CURRENT_STATE(ST_VAR, MOD_VAR) \ + Py_DECREF(MOD_VAR) + +static int +set_current_module(PyInterpreterState *interp, PyObject *mod) +{ + assert(mod != NULL); + PyObject *dict = PyInterpreterState_GetDict(interp); + if (dict == NULL) { + return -1; + } + PyObject *ref = PyWeakref_NewRef(mod, NULL); + if (ref == NULL) { + return -1; + } + int rc = PyDict_SetItem(dict, INTERP_KEY, ref); + Py_DECREF(ref); + return rc; +} + +static void +clear_current_module(PyInterpreterState *interp, PyObject *expected) +{ + PyObject *exc = PyErr_GetRaisedException(); + + PyObject *dict = PyInterpreterState_GetDict(interp); + if (dict == NULL) { + goto error; + } + + if (expected != NULL) { + PyObject *ref = NULL; + if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { + goto error; + } + if (ref != NULL) { + PyObject *current = NULL; + int rc = PyWeakref_GetRef(ref, ¤t); + /* We only need "current" for pointer comparison. */ + Py_XDECREF(current); + Py_DECREF(ref); + if (rc < 0) { + goto error; + } + if (current != expected) { + goto finally; + } + } + } + + /* We use None to identify that the module was previously loaded. */ + if (PyDict_SetItem(dict, INTERP_KEY, Py_None) < 0) { + goto error; + } + + goto finally; + +error: + PyErr_WriteUnraisable(NULL); + +finally: + PyErr_SetRaisedException(exc); +} -#define STATIC_STATE() (&_datetime_global_state) /* We require that C int be at least 32 bits, and use int virtually * everywhere. In just a few cases we use a temp long, where a Python @@ -142,25 +312,16 @@ static datetime_state _datetime_global_state; */ #define MONTH_IS_SANE(M) ((unsigned int)(M) - 1 < 12) -/* Forward declarations. */ -static PyTypeObject PyDateTime_DateType; -static PyTypeObject PyDateTime_DateTimeType; -static PyTypeObject PyDateTime_DeltaType; -static PyTypeObject PyDateTime_IsoCalendarDateType; -static PyTypeObject PyDateTime_TimeType; -static PyTypeObject PyDateTime_TZInfoType; -static PyTypeObject PyDateTime_TimeZoneType; - static int check_tzinfo_subclass(PyObject *p); /*[clinic input] module datetime -class datetime.datetime "PyDateTime_DateTime *" "&PyDateTime_DateTimeType" -class datetime.date "PyDateTime_Date *" "&PyDateTime_DateType" -class datetime.time "PyDateTime_Time *" "&PyDateTime_TimeType" -class datetime.IsoCalendarDate "PyDateTime_IsoCalendarDate *" "&PyDateTime_IsoCalendarDateType" +class datetime.datetime "PyDateTime_DateTime *" "get_datetime_state()->datetime_type" +class datetime.date "PyDateTime_Date *" "get_datetime_state()->date_type" +class datetime.time "PyDateTime_Time *" "get_datetime_state()->time_type" +class datetime.IsoCalendarDate "PyDateTime_IsoCalendarDate *" "get_datetime_state()->isocalendar_date_type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6f65a48dd22fa40f]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c8f3d834a860d50a]*/ #include "clinic/_datetimemodule.c.h" @@ -416,6 +577,10 @@ iso_week1_monday(int year) static int iso_to_ymd(const int iso_year, const int iso_week, const int iso_day, int *year, int *month, int *day) { + // Year is bounded to 0 < year < 10000 because 9999-12-31 is (9999, 52, 5) + if (iso_year < MINYEAR || iso_year > MAXYEAR) { + return -4; + } if (iso_week <= 0 || iso_week >= 53) { int out_of_range = 1; if (iso_week == 53) { @@ -762,7 +927,7 @@ parse_isoformat_date(const char *dtstr, const size_t len, int *year, int *month, * -2: Inconsistent date separator usage * -3: Failed to parse ISO week. * -4: Failed to parse ISO day. - * -5, -6: Failure in iso_to_ymd + * -5, -6, -7: Failure in iso_to_ymd */ const char *p = dtstr; p = parse_digits(p, year, 4); @@ -853,6 +1018,9 @@ parse_hh_mm_ss_ff(const char *tstr, const char *tstr_end, int *hour, continue; } else if (c == '.' || c == ',') { + if (i < 2) { + return -3; // Decimal mark on hour or minute + } break; } else if (!has_separator) { --p; @@ -975,7 +1143,7 @@ new_date_ex(int year, int month, int day, PyTypeObject *type) } #define new_date(year, month, day) \ - new_date_ex(year, month, day, &PyDateTime_DateType) + new_date_ex(year, month, day, DATE_TYPE(NO_STATE)) // Forward declaration static PyObject * @@ -987,10 +1155,10 @@ new_date_subclass_ex(int year, int month, int day, PyObject *cls) { PyObject *result; // We have "fast path" constructors for two subclasses: date and datetime - if ((PyTypeObject *)cls == &PyDateTime_DateType) { + if ((PyTypeObject *)cls == DATE_TYPE(NO_STATE)) { result = new_date_ex(year, month, day, (PyTypeObject *)cls); } - else if ((PyTypeObject *)cls == &PyDateTime_DateTimeType) { + else if ((PyTypeObject *)cls == DATETIME_TYPE(NO_STATE)) { result = new_datetime_ex(year, month, day, 0, 0, 0, 0, Py_None, (PyTypeObject *)cls); } @@ -1044,8 +1212,7 @@ new_datetime_ex(int year, int month, int day, int hour, int minute, } #define new_datetime(y, m, d, hh, mm, ss, us, tzinfo, fold) \ - new_datetime_ex2(y, m, d, hh, mm, ss, us, tzinfo, fold, \ - &PyDateTime_DateTimeType) + new_datetime_ex2(y, m, d, hh, mm, ss, us, tzinfo, fold, DATETIME_TYPE(NO_STATE)) static PyObject * call_subclass_fold(PyObject *cls, int fold, const char *format, ...) @@ -1084,9 +1251,10 @@ call_subclass_fold(PyObject *cls, int fold, const char *format, ...) static PyObject * new_datetime_subclass_fold_ex(int year, int month, int day, int hour, int minute, int second, int usecond, PyObject *tzinfo, - int fold, PyObject *cls) { + int fold, PyObject *cls) +{ PyObject* dt; - if ((PyTypeObject*)cls == &PyDateTime_DateTimeType) { + if ((PyTypeObject*)cls == DATETIME_TYPE(NO_STATE)) { // Use the fast path constructor dt = new_datetime(year, month, day, hour, minute, second, usecond, tzinfo, fold); @@ -1147,15 +1315,15 @@ new_time_ex(int hour, int minute, int second, int usecond, return new_time_ex2(hour, minute, second, usecond, tzinfo, 0, type); } -#define new_time(hh, mm, ss, us, tzinfo, fold) \ - new_time_ex2(hh, mm, ss, us, tzinfo, fold, &PyDateTime_TimeType) +#define new_time(hh, mm, ss, us, tzinfo, fold) \ + new_time_ex2(hh, mm, ss, us, tzinfo, fold, TIME_TYPE(NO_STATE)) static PyObject * new_time_subclass_fold_ex(int hour, int minute, int second, int usecond, PyObject *tzinfo, int fold, PyObject *cls) { PyObject *t; - if ((PyTypeObject*)cls == &PyDateTime_TimeType) { + if ((PyTypeObject*)cls == TIME_TYPE(NO_STATE)) { // Use the fast path constructor t = new_time(hour, minute, second, usecond, tzinfo, fold); } @@ -1168,6 +1336,8 @@ new_time_subclass_fold_ex(int hour, int minute, int second, int usecond, return t; } +static PyDateTime_Delta * look_up_delta(int, int, int, PyTypeObject *); + /* Create a timedelta instance. Normalize the members iff normalize is * true. Passing false is a speed optimization, if you know for sure * that seconds and microseconds are already in their proper ranges. In any @@ -1188,6 +1358,12 @@ new_delta_ex(int days, int seconds, int microseconds, int normalize, if (check_delta_day_range(days) < 0) return NULL; + self = look_up_delta(days, seconds, microseconds, type); + if (self != NULL) { + return (PyObject *)self; + } + assert(!PyErr_Occurred()); + self = (PyDateTime_Delta *) (type->tp_alloc(type, 0)); if (self != NULL) { self->hashcode = -1; @@ -1199,7 +1375,7 @@ new_delta_ex(int days, int seconds, int microseconds, int normalize, } #define new_delta(d, s, us, normalize) \ - new_delta_ex(d, s, us, normalize, &PyDateTime_DeltaType) + new_delta_ex(d, s, us, normalize, DELTA_TYPE(NO_STATE)) typedef struct @@ -1209,6 +1385,8 @@ typedef struct PyObject *name; } PyDateTime_TimeZone; +static PyDateTime_TimeZone * look_up_timezone(PyObject *offset, PyObject *name); + /* Create new timezone instance checking offset range. This function does not check the name argument. Caller must assure that offset is a timedelta instance and name is either NULL @@ -1217,12 +1395,18 @@ static PyObject * create_timezone(PyObject *offset, PyObject *name) { PyDateTime_TimeZone *self; - PyTypeObject *type = &PyDateTime_TimeZoneType; + PyTypeObject *type = TIMEZONE_TYPE(NO_STATE); assert(offset != NULL); assert(PyDelta_Check(offset)); assert(name == NULL || PyUnicode_Check(name)); + self = look_up_timezone(offset, name); + if (self != NULL) { + return (PyObject *)self; + } + assert(!PyErr_Occurred()); + self = (PyDateTime_TimeZone *)(type->tp_alloc(type, 0)); if (self == NULL) { return NULL; @@ -1233,6 +1417,7 @@ create_timezone(PyObject *offset, PyObject *name) } static int delta_bool(PyDateTime_Delta *self); +static PyDateTime_TimeZone utc_timezone; static PyObject * new_timezone(PyObject *offset, PyObject *name) @@ -1242,8 +1427,7 @@ new_timezone(PyObject *offset, PyObject *name) assert(name == NULL || PyUnicode_Check(name)); if (name == NULL && delta_bool((PyDateTime_Delta *)offset) == 0) { - datetime_state *st = STATIC_STATE(); - return Py_NewRef(st->utc); + return Py_NewRef(CONST_UTC(NO_STATE)); } if ((GET_TD_DAYS(offset) == -1 && GET_TD_SECONDS(offset) == 0 && @@ -1456,8 +1640,7 @@ tzinfo_from_isoformat_results(int rv, int tzoffset, int tz_useconds) if (rv == 1) { // Create a timezone from offset in seconds (0 returns UTC) if (tzoffset == 0) { - datetime_state *st = STATIC_STATE(); - return Py_NewRef(st->utc); + return Py_NewRef(CONST_UTC(NO_STATE)); } PyObject *delta = new_delta(0, tzoffset, tz_useconds, 1); @@ -1564,7 +1747,7 @@ make_somezreplacement(PyObject *object, char *sep, PyObject *tzinfoarg) PyObject *tzinfo = get_tzinfo_member(object); if (tzinfo == Py_None || tzinfo == NULL) { - return PyBytes_FromStringAndSize(NULL, 0); + return PyUnicode_FromStringAndSize(NULL, 0); } assert(tzinfoarg != NULL); @@ -1575,7 +1758,7 @@ make_somezreplacement(PyObject *object, char *sep, PyObject *tzinfoarg) tzinfoarg) < 0) return NULL; - return PyBytes_FromStringAndSize(buf, strlen(buf)); + return PyUnicode_FromString(buf); } static PyObject * @@ -1583,7 +1766,7 @@ make_Zreplacement(PyObject *object, PyObject *tzinfoarg) { PyObject *temp; PyObject *tzinfo = get_tzinfo_member(object); - PyObject *Zreplacement = PyUnicode_FromStringAndSize(NULL, 0); + PyObject *Zreplacement = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (Zreplacement == NULL) return NULL; @@ -1632,7 +1815,7 @@ make_freplacement(PyObject *object) else sprintf(freplacement, "%06d", 0); - return PyBytes_FromStringAndSize(freplacement, strlen(freplacement)); + return PyUnicode_FromString(freplacement); } /* I sure don't want to reproduce the strftime code from the time module, @@ -1653,79 +1836,61 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, PyObject *Zreplacement = NULL; /* py string, replacement for %Z */ PyObject *freplacement = NULL; /* py string, replacement for %f */ - const char *pin; /* pointer to next char in input format */ - Py_ssize_t flen; /* length of input format */ - char ch; /* next char in input format */ - - PyObject *newfmt = NULL; /* py string, the output format */ - char *pnew; /* pointer to available byte in output format */ - size_t totalnew; /* number bytes total in output format buffer, - exclusive of trailing \0 */ - size_t usednew; /* number bytes used so far in output format buffer */ - - const char *ptoappend; /* ptr to string to append to output buffer */ - Py_ssize_t ntoappend; /* # of bytes to append to output buffer */ - assert(object && format && timetuple); assert(PyUnicode_Check(format)); - /* Convert the input format to a C string and size */ - pin = PyUnicode_AsUTF8AndSize(format, &flen); - if (!pin) + + PyObject *strftime = _PyImport_GetModuleAttrString("time", "strftime"); + if (strftime == NULL) { return NULL; + } /* Scan the input format, looking for %z/%Z/%f escapes, building * a new format. Since computing the replacements for those codes * is expensive, don't unless they're actually used. */ - if (flen > INT_MAX - 1) { - PyErr_NoMemory(); - goto Done; - } - totalnew = flen + 1; /* realistic if no %z/%Z */ - newfmt = PyBytes_FromStringAndSize(NULL, totalnew); - if (newfmt == NULL) goto Done; - pnew = PyBytes_AsString(newfmt); - usednew = 0; + PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); + if (writer == NULL) { + goto Error; + } - while ((ch = *pin++) != '\0') { - if (ch != '%') { - ptoappend = pin - 1; - ntoappend = 1; + Py_ssize_t flen = PyUnicode_GET_LENGTH(format); + Py_ssize_t i = 0; + Py_ssize_t start = 0; + Py_ssize_t end = 0; + while (i != flen) { + i = PyUnicode_FindChar(format, '%', i, flen, 1); + if (i < 0) { + assert(!PyErr_Occurred()); + break; } - else if ((ch = *pin++) == '\0') { - /* Null byte follows %, copy only '%'. - * - * Back the pin up one char so that we catch the null check - * the next time through the loop.*/ - pin--; - ptoappend = pin - 1; - ntoappend = 1; + end = i; + i++; + if (i == flen) { + break; } + Py_UCS4 ch = PyUnicode_READ_CHAR(format, i); + i++; /* A % has been seen and ch is the character after it. */ - else if (ch == 'z') { + PyObject *replacement = NULL; + if (ch == 'z') { /* %z -> +HHMM */ if (zreplacement == NULL) { zreplacement = make_somezreplacement(object, "", tzinfoarg); if (zreplacement == NULL) - goto Done; + goto Error; } - assert(zreplacement != NULL); - assert(PyBytes_Check(zreplacement)); - ptoappend = PyBytes_AS_STRING(zreplacement); - ntoappend = PyBytes_GET_SIZE(zreplacement); + replacement = zreplacement; } - else if (ch == ':' && *pin == 'z' && pin++) { + else if (ch == ':' && i < flen && PyUnicode_READ_CHAR(format, i) == 'z') { /* %:z -> +HH:MM */ + i++; if (colonzreplacement == NULL) { colonzreplacement = make_somezreplacement(object, ":", tzinfoarg); if (colonzreplacement == NULL) - goto Done; + goto Error; } - assert(colonzreplacement != NULL); - assert(PyBytes_Check(colonzreplacement)); - ptoappend = PyBytes_AS_STRING(colonzreplacement); - ntoappend = PyBytes_GET_SIZE(colonzreplacement); + replacement = colonzreplacement; } else if (ch == 'Z') { /* format tzname */ @@ -1733,79 +1898,118 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, Zreplacement = make_Zreplacement(object, tzinfoarg); if (Zreplacement == NULL) - goto Done; + goto Error; } - assert(Zreplacement != NULL); - assert(PyUnicode_Check(Zreplacement)); - ptoappend = PyUnicode_AsUTF8AndSize(Zreplacement, - &ntoappend); - if (ptoappend == NULL) - goto Done; + replacement = Zreplacement; } else if (ch == 'f') { /* format microseconds */ if (freplacement == NULL) { freplacement = make_freplacement(object); if (freplacement == NULL) - goto Done; + goto Error; } - assert(freplacement != NULL); - assert(PyBytes_Check(freplacement)); - ptoappend = PyBytes_AS_STRING(freplacement); - ntoappend = PyBytes_GET_SIZE(freplacement); + replacement = freplacement; } +#ifdef Py_NORMALIZE_CENTURY + else if (ch == 'Y' || ch == 'G' + || ch == 'F' || ch == 'C' + ) { + /* 0-pad year with century as necessary */ + PyObject *item = PySequence_GetItem(timetuple, 0); + if (item == NULL) { + goto Error; + } + long year_long = PyLong_AsLong(item); + Py_DECREF(item); + if (year_long == -1 && PyErr_Occurred()) { + goto Error; + } + /* Note that datetime(1000, 1, 1).strftime('%G') == '1000' so year + 1000 for %G can go on the fast path. */ + if (year_long >= 1000) { + continue; + } + if (ch == 'G') { + PyObject *year_str = PyObject_CallFunction(strftime, "sO", + "%G", timetuple); + if (year_str == NULL) { + goto Error; + } + PyObject *year = PyNumber_Long(year_str); + Py_DECREF(year_str); + if (year == NULL) { + goto Error; + } + year_long = PyLong_AsLong(year); + Py_DECREF(year); + if (year_long == -1 && PyErr_Occurred()) { + goto Error; + } + } + /* Buffer of maximum size of formatted year permitted by long. + * +6 to accommodate dashes, 2-digit month and day for %F. */ + char buf[SIZEOF_LONG * 5 / 2 + 2 + 6]; + Py_ssize_t n = PyOS_snprintf(buf, sizeof(buf), + ch == 'F' ? "%04ld-%%m-%%d" : + "%04ld", year_long); + if (ch == 'C') { + n -= 2; + } + if (PyUnicodeWriter_WriteSubstring(writer, format, start, end) < 0) { + goto Error; + } + start = i; + if (PyUnicodeWriter_WriteUTF8(writer, buf, n) < 0) { + goto Error; + } + continue; + } +#endif else { /* percent followed by something else */ - ptoappend = pin - 2; - ntoappend = 2; - } - - /* Append the ntoappend chars starting at ptoappend to - * the new format. - */ - if (ntoappend == 0) continue; - assert(ptoappend != NULL); - assert(ntoappend > 0); - while (usednew + ntoappend > totalnew) { - if (totalnew > (PY_SSIZE_T_MAX >> 1)) { /* overflow */ - PyErr_NoMemory(); - goto Done; - } - totalnew <<= 1; - if (_PyBytes_Resize(&newfmt, totalnew) < 0) - goto Done; - pnew = PyBytes_AsString(newfmt) + usednew; } - memcpy(pnew, ptoappend, ntoappend); - pnew += ntoappend; - usednew += ntoappend; - assert(usednew <= totalnew); + assert(replacement != NULL); + assert(PyUnicode_Check(replacement)); + if (PyUnicodeWriter_WriteSubstring(writer, format, start, end) < 0) { + goto Error; + } + start = i; + if (PyUnicodeWriter_WriteStr(writer, replacement) < 0) { + goto Error; + } } /* end while() */ - if (_PyBytes_Resize(&newfmt, usednew) < 0) - goto Done; - { - PyObject *format; - PyObject *strftime = _PyImport_GetModuleAttrString("time", "strftime"); - - if (strftime == NULL) + PyObject *newformat; + if (start == 0) { + PyUnicodeWriter_Discard(writer); + newformat = Py_NewRef(format); + } + else { + if (PyUnicodeWriter_WriteSubstring(writer, format, start, flen) < 0) { + goto Error; + } + newformat = PyUnicodeWriter_Finish(writer); + if (newformat == NULL) { goto Done; - format = PyUnicode_FromString(PyBytes_AS_STRING(newfmt)); - if (format != NULL) { - result = PyObject_CallFunctionObjArgs(strftime, - format, timetuple, NULL); - Py_DECREF(format); } - Py_DECREF(strftime); } + result = PyObject_CallFunctionObjArgs(strftime, + newformat, timetuple, NULL); + Py_DECREF(newformat); + Done: Py_XDECREF(freplacement); Py_XDECREF(zreplacement); Py_XDECREF(colonzreplacement); Py_XDECREF(Zreplacement); - Py_XDECREF(newfmt); + Py_XDECREF(strftime); return result; + + Error: + PyUnicodeWriter_Discard(writer); + goto Done; } /* --------------------------------------------------------------------------- @@ -1886,11 +2090,13 @@ delta_to_microseconds(PyDateTime_Delta *self) PyObject *x3 = NULL; PyObject *result = NULL; + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + x1 = PyLong_FromLong(GET_TD_DAYS(self)); if (x1 == NULL) goto Done; - datetime_state *st = STATIC_STATE(); - x2 = PyNumber_Multiply(x1, st->seconds_per_day); /* days in seconds */ + x2 = PyNumber_Multiply(x1, CONST_SEC_PER_DAY(st)); /* days in seconds */ if (x2 == NULL) goto Done; Py_SETREF(x1, NULL); @@ -1907,7 +2113,7 @@ delta_to_microseconds(PyDateTime_Delta *self) /* x1 = */ x2 = NULL; /* x3 has days+seconds in seconds */ - x1 = PyNumber_Multiply(x3, st->us_per_second); /* us */ + x1 = PyNumber_Multiply(x3, CONST_US_PER_SECOND(st)); /* us */ if (x1 == NULL) goto Done; Py_SETREF(x3, NULL); @@ -1923,6 +2129,7 @@ delta_to_microseconds(PyDateTime_Delta *self) Py_XDECREF(x1); Py_XDECREF(x2); Py_XDECREF(x3); + RELEASE_CURRENT_STATE(st, current_mod); return result; } @@ -1962,8 +2169,10 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) PyObject *num = NULL; PyObject *result = NULL; - datetime_state *st = STATIC_STATE(); - tuple = checked_divmod(pyus, st->us_per_second); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + + tuple = checked_divmod(pyus, CONST_US_PER_SECOND(st)); if (tuple == NULL) { goto Done; } @@ -1981,7 +2190,7 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) num = Py_NewRef(PyTuple_GET_ITEM(tuple, 0)); /* leftover seconds */ Py_DECREF(tuple); - tuple = checked_divmod(num, st->seconds_per_day); + tuple = checked_divmod(num, CONST_SEC_PER_DAY(st)); if (tuple == NULL) goto Done; Py_DECREF(num); @@ -2006,6 +2215,7 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) Done: Py_XDECREF(tuple); Py_XDECREF(num); + RELEASE_CURRENT_STATE(st, current_mod); return result; BadDivmod: @@ -2015,7 +2225,7 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) } #define microseconds_to_delta(pymicros) \ - microseconds_to_delta_ex(pymicros, &PyDateTime_DeltaType) + microseconds_to_delta_ex(pymicros, DELTA_TYPE(NO_STATE)) static PyObject * multiply_int_timedelta(PyObject *intobj, PyDateTime_Delta *delta) @@ -2543,6 +2753,9 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) { PyObject *self = NULL; + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + /* Argument objects. */ PyObject *day = NULL; PyObject *second = NULL; @@ -2581,29 +2794,28 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) y = accum("microseconds", x, us, _PyLong_GetOne(), &leftover_us); CLEANUP; } - datetime_state *st = STATIC_STATE(); if (ms) { - y = accum("milliseconds", x, ms, st->us_per_ms, &leftover_us); + y = accum("milliseconds", x, ms, CONST_US_PER_MS(st), &leftover_us); CLEANUP; } if (second) { - y = accum("seconds", x, second, st->us_per_second, &leftover_us); + y = accum("seconds", x, second, CONST_US_PER_SECOND(st), &leftover_us); CLEANUP; } if (minute) { - y = accum("minutes", x, minute, st->us_per_minute, &leftover_us); + y = accum("minutes", x, minute, CONST_US_PER_MINUTE(st), &leftover_us); CLEANUP; } if (hour) { - y = accum("hours", x, hour, st->us_per_hour, &leftover_us); + y = accum("hours", x, hour, CONST_US_PER_HOUR(st), &leftover_us); CLEANUP; } if (day) { - y = accum("days", x, day, st->us_per_day, &leftover_us); + y = accum("days", x, day, CONST_US_PER_DAY(st), &leftover_us); CLEANUP; } if (week) { - y = accum("weeks", x, week, st->us_per_week, &leftover_us); + y = accum("weeks", x, week, CONST_US_PER_WEEK(st), &leftover_us); CLEANUP; } if (leftover_us) { @@ -2645,7 +2857,9 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) self = microseconds_to_delta_ex(x, type); Py_DECREF(x); + Done: + RELEASE_CURRENT_STATE(st, current_mod); return self; #undef CLEANUP @@ -2662,7 +2876,7 @@ delta_bool(PyDateTime_Delta *self) static PyObject * delta_repr(PyDateTime_Delta *self) { - PyObject *args = PyUnicode_FromString(""); + PyObject *args = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (args == NULL) { return NULL; @@ -2758,9 +2972,12 @@ delta_total_seconds(PyObject *self, PyObject *Py_UNUSED(ignored)) if (total_microseconds == NULL) return NULL; - datetime_state *st = STATIC_STATE(); - total_seconds = PyNumber_TrueDivide(total_microseconds, st->us_per_second); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + + total_seconds = PyNumber_TrueDivide(total_microseconds, CONST_US_PER_SECOND(st)); + RELEASE_CURRENT_STATE(st, current_mod); Py_DECREF(total_microseconds); return total_seconds; } @@ -2881,6 +3098,25 @@ static PyTypeObject PyDateTime_DeltaType = { 0, /* tp_free */ }; +// XXX Can we make this const? +static PyDateTime_Delta zero_delta = { + PyObject_HEAD_INIT(&PyDateTime_DeltaType) + /* Letting this be set lazily is a benign race. */ + .hashcode = -1, +}; + +static PyDateTime_Delta * +look_up_delta(int days, int seconds, int microseconds, PyTypeObject *type) +{ + if (days == 0 && seconds == 0 && microseconds == 0 + && type == Py_TYPE(&zero_delta)) + { + return &zero_delta; + } + return NULL; +} + + /* * PyDateTime_Date implementation. */ @@ -3142,15 +3378,13 @@ date_fromisocalendar(PyObject *cls, PyObject *args, PyObject *kw) return NULL; } - // Year is bounded to 0 < year < 10000 because 9999-12-31 is (9999, 52, 5) - if (year < MINYEAR || year > MAXYEAR) { - PyErr_Format(PyExc_ValueError, "Year is out of range: %d", year); - return NULL; - } - int month; int rv = iso_to_ymd(year, week, day, &year, &month, &day); + if (rv == -4) { + PyErr_Format(PyExc_ValueError, "Year is out of range: %d", year); + return NULL; + } if (rv == -2) { PyErr_Format(PyExc_ValueError, "Invalid week: %d", week); @@ -3166,6 +3400,27 @@ date_fromisocalendar(PyObject *cls, PyObject *args, PyObject *kw) return new_date_subclass_ex(year, month, day, cls); } +/* Return new date from _strptime.strptime_datetime_date(). */ +static PyObject * +date_strptime(PyObject *cls, PyObject *args) +{ + PyObject *string, *format, *result; + + if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format)) { + return NULL; + } + + PyObject *module = PyImport_Import(&_Py_ID(_strptime)); + if (module == NULL) { + return NULL; + } + result = PyObject_CallMethodObjArgs(module, + &_Py_ID(_strptime_datetime_date), cls, + string, format, NULL); + Py_DECREF(module); + return result; +} + /* * Date arithmetic. @@ -3413,17 +3668,40 @@ static PyMethodDef iso_calendar_date_methods[] = { {NULL, NULL}, }; -static PyTypeObject PyDateTime_IsoCalendarDateType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "datetime.IsoCalendarDate", - .tp_basicsize = sizeof(PyDateTime_IsoCalendarDate), - .tp_repr = (reprfunc) iso_calendar_date_repr, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = iso_calendar_date__doc__, - .tp_methods = iso_calendar_date_methods, - .tp_getset = iso_calendar_date_getset, - // .tp_base = &PyTuple_Type, // filled in PyInit__datetime - .tp_new = iso_calendar_date_new, +static int +iso_calendar_date_traverse(PyDateTime_IsoCalendarDate *self, visitproc visit, + void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return PyTuple_Type.tp_traverse((PyObject *)self, visit, arg); +} + +static void +iso_calendar_date_dealloc(PyDateTime_IsoCalendarDate *self) +{ + PyTypeObject *tp = Py_TYPE(self); + PyTuple_Type.tp_dealloc((PyObject *)self); // delegate GC-untrack as well + Py_DECREF(tp); +} + +static PyType_Slot isocal_slots[] = { + {Py_tp_repr, iso_calendar_date_repr}, + {Py_tp_doc, (void *)iso_calendar_date__doc__}, + {Py_tp_methods, iso_calendar_date_methods}, + {Py_tp_getset, iso_calendar_date_getset}, + {Py_tp_new, iso_calendar_date_new}, + {Py_tp_dealloc, iso_calendar_date_dealloc}, + {Py_tp_traverse, iso_calendar_date_traverse}, + {0, NULL}, +}; + +static PyType_Spec isocal_spec = { + .name = "datetime.IsoCalendarDate", + .basicsize = sizeof(PyDateTime_IsoCalendarDate), + .flags = (Py_TPFLAGS_DEFAULT | + Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = isocal_slots, }; /*[clinic input] @@ -3473,8 +3751,12 @@ date_isocalendar(PyDateTime_Date *self, PyObject *Py_UNUSED(ignored)) week = 0; } - PyObject* v = iso_calendar_date_new_impl(&PyDateTime_IsoCalendarDateType, - year, week + 1, day + 1); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + + PyObject *v = iso_calendar_date_new_impl(ISOCALENDAR_DATE_TYPE(st), + year, week + 1, day + 1); + RELEASE_CURRENT_STATE(st, current_mod); if (v == NULL) { return NULL; } @@ -3534,7 +3816,7 @@ datetime_date_replace_impl(PyDateTime_Date *self, int year, int month, static Py_hash_t generic_hash(unsigned char *data, int len) { - return _Py_HashBytes(data, len); + return Py_HashBuffer(data, len); } @@ -3604,6 +3886,11 @@ static PyMethodDef date_methods[] = { "number and weekday.\n\n" "This is the inverse of the date.isocalendar() function")}, + {"strptime", (PyCFunction)date_strptime, + METH_VARARGS | METH_CLASS, + PyDoc_STR("string, format -> new date parsed from a string " + "(like time.strptime()).")}, + {"today", (PyCFunction)date_today, METH_NOARGS | METH_CLASS, PyDoc_STR("Current date or datetime: same as " "self.__class__.fromtimestamp(time.time()).")}, @@ -3643,7 +3930,8 @@ static PyMethodDef date_methods[] = { DATETIME_DATE_REPLACE_METHODDEF - {"__replace__", _PyCFunction_CAST(datetime_date_replace), METH_FASTCALL | METH_KEYWORDS}, + {"__replace__", _PyCFunction_CAST(datetime_date_replace), METH_FASTCALL | METH_KEYWORDS, + PyDoc_STR("__replace__($self, /, **changes)\n--\n\nThe same as replace().")}, {"__reduce__", (PyCFunction)date_reduce, METH_NOARGS, PyDoc_STR("__reduce__() -> (cls, state)")}, @@ -3943,7 +4231,7 @@ timezone_new(PyTypeObject *type, PyObject *args, PyObject *kw) PyObject *offset; PyObject *name = NULL; if (PyArg_ParseTupleAndKeywords(args, kw, "O!|U:timezone", timezone_kws, - &PyDateTime_DeltaType, &offset, &name)) + DELTA_TYPE(NO_STATE), &offset, &name)) return new_timezone(offset, name); return NULL; @@ -3996,8 +4284,7 @@ timezone_repr(PyDateTime_TimeZone *self) to use Py_TYPE(self)->tp_name here. */ const char *type_name = Py_TYPE(self)->tp_name; - datetime_state *st = STATIC_STATE(); - if (((PyObject *)self) == st->utc) { + if ((PyObject *)self == CONST_UTC(NO_STATE)) { return PyUnicode_FromFormat("%s.utc", type_name); } @@ -4019,8 +4306,7 @@ timezone_str(PyDateTime_TimeZone *self) if (self->name != NULL) { return Py_NewRef(self->name); } - datetime_state *st = STATIC_STATE(); - if ((PyObject *)self == st->utc || + if ((PyObject *)self == CONST_UTC(NO_STATE) || (GET_TD_DAYS(self->offset) == 0 && GET_TD_SECONDS(self->offset) == 0 && GET_TD_MICROSECONDS(self->offset) == 0)) @@ -4172,6 +4458,23 @@ static PyTypeObject PyDateTime_TimeZoneType = { timezone_new, /* tp_new */ }; +// XXX Can we make this const? +static PyDateTime_TimeZone utc_timezone = { + PyObject_HEAD_INIT(&PyDateTime_TimeZoneType) + .offset = (PyObject *)&zero_delta, + .name = NULL, +}; + +static PyDateTime_TimeZone * +look_up_timezone(PyObject *offset, PyObject *name) +{ + if (offset == utc_timezone.offset && name == NULL) { + return (PyDateTime_TimeZone *)CONST_UTC(NO_STATE); + } + return NULL; +} + + /* * PyDateTime_Time implementation. */ @@ -4322,6 +4625,27 @@ time_new(PyTypeObject *type, PyObject *args, PyObject *kw) return self; } +/* Return new time from _strptime.strptime_datetime_time(). */ +static PyObject * +time_strptime(PyObject *cls, PyObject *args) +{ + PyObject *string, *format, *result; + + if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format)) { + return NULL; + } + + PyObject *module = PyImport_Import(&_Py_ID(_strptime)); + if (module == NULL) { + return NULL; + } + result = PyObject_CallMethodObjArgs(module, + &_Py_ID(_strptime_datetime_time), cls, + string, format, NULL); + Py_DECREF(module); + return result; +} + /* * Destructor. */ @@ -4624,7 +4948,7 @@ datetime.time.replace minute: int(c_default="TIME_GET_MINUTE(self)") = unchanged second: int(c_default="TIME_GET_SECOND(self)") = unchanged microsecond: int(c_default="TIME_GET_MICROSECOND(self)") = unchanged - tzinfo: object(c_default="HASTZINFO(self) ? self->tzinfo : Py_None") = unchanged + tzinfo: object(c_default="HASTZINFO(self) ? ((PyDateTime_Time *)self)->tzinfo : Py_None") = unchanged * fold: int(c_default="TIME_GET_FOLD(self)") = unchanged @@ -4635,7 +4959,7 @@ static PyObject * datetime_time_replace_impl(PyDateTime_Time *self, int hour, int minute, int second, int microsecond, PyObject *tzinfo, int fold) -/*[clinic end generated code: output=0b89a44c299e4f80 input=9b6a35b1e704b0ca]*/ +/*[clinic end generated code: output=0b89a44c299e4f80 input=abf23656e8df4e97]*/ { return new_time_subclass_fold_ex(hour, minute, second, microsecond, tzinfo, fold, (PyObject *)Py_TYPE(self)); @@ -4675,6 +4999,14 @@ time_fromisoformat(PyObject *cls, PyObject *tstr) { goto invalid_string_error; } + if (hour == 24) { + if (minute == 0 && second == 0 && microsecond == 0) { + hour = 0; + } else { + goto invalid_iso_midnight; + } + } + PyObject *tzinfo = tzinfo_from_isoformat_results(rv, tzoffset, tzimicrosecond); @@ -4683,7 +5015,7 @@ time_fromisoformat(PyObject *cls, PyObject *tstr) { } PyObject *t; - if ( (PyTypeObject *)cls == &PyDateTime_TimeType ) { + if ( (PyTypeObject *)cls == TIME_TYPE(NO_STATE)) { t = new_time(hour, minute, second, microsecond, tzinfo, 0); } else { t = PyObject_CallFunction(cls, "iiiiO", @@ -4693,6 +5025,10 @@ time_fromisoformat(PyObject *cls, PyObject *tstr) { Py_DECREF(tzinfo); return t; +invalid_iso_midnight: + PyErr_SetString(PyExc_ValueError, "minute, second, and microsecond must be 0 when hour is 24"); + return NULL; + invalid_string_error: PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", tstr); return NULL; @@ -4745,6 +5081,15 @@ time_reduce(PyDateTime_Time *self, PyObject *arg) static PyMethodDef time_methods[] = { + /* Class method: */ + + {"strptime", (PyCFunction)time_strptime, + METH_VARARGS | METH_CLASS, + PyDoc_STR("string, format -> new time parsed from a string " + "(like time.strptime()).")}, + + /* Instance methods: */ + {"isoformat", _PyCFunction_CAST(time_isoformat), METH_VARARGS | METH_KEYWORDS, PyDoc_STR("Return string in ISO 8601 format, [HH[:MM[:SS[.mmm[uuu]]]]]" "[+HH:MM].\n\n" @@ -4770,7 +5115,8 @@ static PyMethodDef time_methods[] = { DATETIME_TIME_REPLACE_METHODDEF - {"__replace__", _PyCFunction_CAST(datetime_time_replace), METH_FASTCALL | METH_KEYWORDS}, + {"__replace__", _PyCFunction_CAST(datetime_time_replace), METH_FASTCALL | METH_KEYWORDS, + PyDoc_STR("__replace__($self, /, **changes)\n--\n\nThe same as replace().")}, {"fromisoformat", (PyCFunction)time_fromisoformat, METH_O | METH_CLASS, PyDoc_STR("string -> time from a string in ISO 8601 format")}, @@ -5251,23 +5597,24 @@ datetime_utcfromtimestamp(PyObject *cls, PyObject *args) return result; } -/* Return new datetime from _strptime.strptime_datetime(). */ +/* Return new datetime from _strptime.strptime_datetime_datetime(). */ static PyObject * datetime_strptime(PyObject *cls, PyObject *args) { - static PyObject *module = NULL; - PyObject *string, *format; + PyObject *string, *format, *result; if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format)) return NULL; + PyObject *module = PyImport_Import(&_Py_ID(_strptime)); if (module == NULL) { - module = PyImport_ImportModule("_strptime"); - if (module == NULL) - return NULL; + return NULL; } - return PyObject_CallMethodObjArgs(module, &_Py_ID(_strptime_datetime), - cls, string, format, NULL); + result = PyObject_CallMethodObjArgs(module, + &_Py_ID(_strptime_datetime_datetime), + cls, string, format, NULL); + Py_DECREF(module); + return result; } /* Return new datetime from date/datetime and time arguments. */ @@ -5281,8 +5628,8 @@ datetime_combine(PyObject *cls, PyObject *args, PyObject *kw) PyObject *result = NULL; if (PyArg_ParseTupleAndKeywords(args, kw, "O!O!|O:combine", keywords, - &PyDateTime_DateType, &date, - &PyDateTime_TimeType, &time, &tzinfo)) { + DATE_TYPE(NO_STATE), &date, + TIME_TYPE(NO_STATE), &time, &tzinfo)) { if (tzinfo == NULL) { if (HASTZINFO(time)) tzinfo = ((PyDateTime_Time *)time)->tzinfo; @@ -5538,6 +5885,26 @@ datetime_fromisoformat(PyObject *cls, PyObject *dtstr) goto error; } + if ((hour == 24) && (month <= 12)) { + int d_in_month = days_in_month(year, month); + if (day <= d_in_month) { + if (minute == 0 && second == 0 && microsecond == 0) { + // Calculate midnight of the next day + hour = 0; + day += 1; + if (day > d_in_month) { + day = 1; + month += 1; + if (month > 12) { + month = 1; + year += 1; + } + } + } else { + goto invalid_iso_midnight; + } + } + } PyObject *dt = new_datetime_subclass_ex(year, month, day, hour, minute, second, microsecond, tzinfo, cls); @@ -5545,6 +5912,12 @@ datetime_fromisoformat(PyObject *cls, PyObject *dtstr) Py_DECREF(dtstr_clean); return dt; +invalid_iso_midnight: + PyErr_SetString(PyExc_ValueError, "minute, second, and microsecond must be 0 when hour is 24"); + Py_DECREF(tzinfo); + Py_DECREF(dtstr_clean); + return NULL; + invalid_string_error: PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", dtstr); @@ -6077,7 +6450,7 @@ datetime.datetime.replace minute: int(c_default="DATE_GET_MINUTE(self)") = unchanged second: int(c_default="DATE_GET_SECOND(self)") = unchanged microsecond: int(c_default="DATE_GET_MICROSECOND(self)") = unchanged - tzinfo: object(c_default="HASTZINFO(self) ? self->tzinfo : Py_None") = unchanged + tzinfo: object(c_default="HASTZINFO(self) ? ((PyDateTime_DateTime *)self)->tzinfo : Py_None") = unchanged * fold: int(c_default="DATE_GET_FOLD(self)") = unchanged @@ -6089,7 +6462,7 @@ datetime_datetime_replace_impl(PyDateTime_DateTime *self, int year, int month, int day, int hour, int minute, int second, int microsecond, PyObject *tzinfo, int fold) -/*[clinic end generated code: output=00bc96536833fddb input=9b38253d56d9bcad]*/ +/*[clinic end generated code: output=00bc96536833fddb input=fd972762d604d3e7]*/ { return new_datetime_subclass_fold_ex(year, month, day, hour, minute, second, microsecond, tzinfo, fold, @@ -6166,10 +6539,14 @@ local_timezone(PyDateTime_DateTime *utc_time) PyObject *one_second; PyObject *seconds; - datetime_state *st = STATIC_STATE(); - delta = datetime_subtract((PyObject *)utc_time, st->epoch); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + + delta = datetime_subtract((PyObject *)utc_time, CONST_EPOCH(st)); + RELEASE_CURRENT_STATE(st, current_mod); if (delta == NULL) return NULL; + one_second = new_delta(0, 1, 0, 0); if (one_second == NULL) { Py_DECREF(delta); @@ -6279,7 +6656,6 @@ datetime_astimezone(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) if (result == NULL) return NULL; - datetime_state *st = STATIC_STATE(); /* Make sure result is aware and UTC. */ if (!HASTZINFO(result)) { temp = (PyObject *)result; @@ -6291,7 +6667,7 @@ datetime_astimezone(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) DATE_GET_MINUTE(result), DATE_GET_SECOND(result), DATE_GET_MICROSECOND(result), - st->utc, + CONST_UTC(NO_STATE), DATE_GET_FOLD(result), Py_TYPE(result)); Py_DECREF(temp); @@ -6300,7 +6676,7 @@ datetime_astimezone(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) } else { /* Result is already aware - just replace tzinfo. */ - Py_SETREF(result->tzinfo, Py_NewRef(st->utc)); + Py_SETREF(result->tzinfo, Py_NewRef(CONST_UTC(NO_STATE))); } /* Attach new tzinfo and let fromutc() do the rest. */ @@ -6404,9 +6780,12 @@ datetime_timestamp(PyDateTime_DateTime *self, PyObject *Py_UNUSED(ignored)) PyObject *result; if (HASTZINFO(self) && self->tzinfo != Py_None) { - datetime_state *st = STATIC_STATE(); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); + PyObject *delta; - delta = datetime_subtract((PyObject *)self, st->epoch); + delta = datetime_subtract((PyObject *)self, CONST_EPOCH(st)); + RELEASE_CURRENT_STATE(st, current_mod); if (delta == NULL) return NULL; result = delta_total_seconds(delta, NULL); @@ -6617,7 +6996,8 @@ static PyMethodDef datetime_methods[] = { DATETIME_DATETIME_REPLACE_METHODDEF - {"__replace__", _PyCFunction_CAST(datetime_datetime_replace), METH_FASTCALL | METH_KEYWORDS}, + {"__replace__", _PyCFunction_CAST(datetime_datetime_replace), METH_FASTCALL | METH_KEYWORDS, + PyDoc_STR("__replace__($self, /, **changes)\n--\n\nThe same as replace().")}, {"astimezone", _PyCFunction_CAST(datetime_astimezone), METH_VARARGS | METH_KEYWORDS, PyDoc_STR("tz -> convert to local time in new timezone tz\n")}, @@ -6694,11 +7074,45 @@ static PyTypeObject PyDateTime_DateTimeType = { }; /* --------------------------------------------------------------------------- - * Module methods and initialization. + * datetime C-API. */ -static PyMethodDef module_methods[] = { - {NULL, NULL} +static PyTypeObject * const capi_types[] = { + &PyDateTime_DateType, + &PyDateTime_DateTimeType, + &PyDateTime_TimeType, + &PyDateTime_DeltaType, + &PyDateTime_TZInfoType, + /* Indirectly, via the utc object. */ + &PyDateTime_TimeZoneType, +}; + +/* The C-API is process-global. This violates interpreter isolation + * due to the objects stored here. Thus each of those objects must + * be managed carefully. */ +// XXX Can we make this const? +static PyDateTime_CAPI capi = { + /* The classes must be readied before used here. + * That will happen the first time the module is loaded. + * They aren't safe to be shared between interpreters, + * but that's okay as long as the module is single-phase init. */ + .DateType = &PyDateTime_DateType, + .DateTimeType = &PyDateTime_DateTimeType, + .TimeType = &PyDateTime_TimeType, + .DeltaType = &PyDateTime_DeltaType, + .TZInfoType = &PyDateTime_TZInfoType, + + .TimeZone_UTC = (PyObject *)&utc_timezone, + + .Date_FromDate = new_date_ex, + .DateTime_FromDateAndTime = new_datetime_ex, + .Time_FromTime = new_time_ex, + .Delta_FromDelta = new_delta_ex, + .TimeZone_FromTimeZone = new_timezone, + .DateTime_FromTimestamp = datetime_fromtimestamp, + .Date_FromTimestamp = datetime_date_fromtimestamp_capi, + .DateTime_FromDateAndTimeAndFold = new_datetime_ex2, + .Time_FromTimeAndFold = new_time_ex2, }; /* Get a new C API by calling this function. @@ -6707,55 +7121,7 @@ static PyMethodDef module_methods[] = { static inline PyDateTime_CAPI * get_datetime_capi(void) { - PyDateTime_CAPI *capi = PyMem_Malloc(sizeof(PyDateTime_CAPI)); - if (capi == NULL) { - PyErr_NoMemory(); - return NULL; - } - capi->DateType = &PyDateTime_DateType; - capi->DateTimeType = &PyDateTime_DateTimeType; - capi->TimeType = &PyDateTime_TimeType; - capi->DeltaType = &PyDateTime_DeltaType; - capi->TZInfoType = &PyDateTime_TZInfoType; - capi->Date_FromDate = new_date_ex; - capi->DateTime_FromDateAndTime = new_datetime_ex; - capi->Time_FromTime = new_time_ex; - capi->Delta_FromDelta = new_delta_ex; - capi->TimeZone_FromTimeZone = new_timezone; - capi->DateTime_FromTimestamp = datetime_fromtimestamp; - capi->Date_FromTimestamp = datetime_date_fromtimestamp_capi; - capi->DateTime_FromDateAndTimeAndFold = new_datetime_ex2; - capi->Time_FromTimeAndFold = new_time_ex2; - // Make sure this function is called after utc has - // been initialized. - datetime_state *st = STATIC_STATE(); - assert(st->utc != NULL); - capi->TimeZone_UTC = st->utc; // borrowed ref - return capi; -} - -static void -datetime_destructor(PyObject *op) -{ - void *ptr = PyCapsule_GetPointer(op, PyDateTime_CAPSULE_NAME); - PyMem_Free(ptr); -} - -static int -datetime_clear(PyObject *module) -{ - datetime_state *st = STATIC_STATE(); - - Py_CLEAR(st->us_per_ms); - Py_CLEAR(st->us_per_second); - Py_CLEAR(st->us_per_minute); - Py_CLEAR(st->us_per_hour); - Py_CLEAR(st->us_per_day); - Py_CLEAR(st->us_per_week); - Py_CLEAR(st->seconds_per_day); - Py_CLEAR(st->utc); - Py_CLEAR(st->epoch); - return 0; + return &capi; } static PyObject * @@ -6770,9 +7136,45 @@ create_timezone_from_delta(int days, int sec, int ms, int normalize) return tz; } + +/* --------------------------------------------------------------------------- + * Module state lifecycle. + */ + static int -init_state(datetime_state *st) -{ +init_state(datetime_state *st, PyObject *module, PyObject *old_module) +{ + /* Each module gets its own heap types. */ +#define ADD_TYPE(FIELD, SPEC, BASE) \ + do { \ + PyObject *cls = PyType_FromModuleAndSpec( \ + module, SPEC, (PyObject *)BASE); \ + if (cls == NULL) { \ + return -1; \ + } \ + st->FIELD = (PyTypeObject *)cls; \ + } while (0) + + ADD_TYPE(isocalendar_date_type, &isocal_spec, &PyTuple_Type); +#undef ADD_TYPE + + if (old_module != NULL) { + assert(old_module != module); + datetime_state *st_old = get_module_state(old_module); + *st = (datetime_state){ + .isocalendar_date_type = st->isocalendar_date_type, + .us_per_ms = Py_NewRef(st_old->us_per_ms), + .us_per_second = Py_NewRef(st_old->us_per_second), + .us_per_minute = Py_NewRef(st_old->us_per_minute), + .us_per_hour = Py_NewRef(st_old->us_per_hour), + .us_per_day = Py_NewRef(st_old->us_per_day), + .us_per_week = Py_NewRef(st_old->us_per_week), + .seconds_per_day = Py_NewRef(st_old->seconds_per_day), + .epoch = Py_NewRef(st_old->epoch), + }; + return 0; + } + st->us_per_ms = PyLong_FromLong(1000); if (st->us_per_ms == NULL) { return -1; @@ -6806,51 +7208,111 @@ init_state(datetime_state *st) return -1; } - /* Init UTC timezone */ - st->utc = create_timezone_from_delta(0, 0, 0, 0); - if (st->utc == NULL) { - return -1; - } - /* Init Unix epoch */ - st->epoch = new_datetime(1970, 1, 1, 0, 0, 0, 0, st->utc, 0); + st->epoch = new_datetime( + 1970, 1, 1, 0, 0, 0, 0, (PyObject *)&utc_timezone, 0); if (st->epoch == NULL) { return -1; } + return 0; } static int -_datetime_exec(PyObject *module) +traverse_state(datetime_state *st, visitproc visit, void *arg) +{ + /* heap types */ + Py_VISIT(st->isocalendar_date_type); + + return 0; +} + +static int +clear_state(datetime_state *st) +{ + Py_CLEAR(st->isocalendar_date_type); + Py_CLEAR(st->us_per_ms); + Py_CLEAR(st->us_per_second); + Py_CLEAR(st->us_per_minute); + Py_CLEAR(st->us_per_hour); + Py_CLEAR(st->us_per_day); + Py_CLEAR(st->us_per_week); + Py_CLEAR(st->seconds_per_day); + Py_CLEAR(st->epoch); + return 0; +} + + +static int +init_static_types(PyInterpreterState *interp, int reloading) { + if (reloading) { + return 0; + } + // `&...` is not a constant expression according to a strict reading // of C standards. Fill tp_base at run-time rather than statically. // See https://bugs.python.org/issue40777 - PyDateTime_IsoCalendarDateType.tp_base = &PyTuple_Type; PyDateTime_TimeZoneType.tp_base = &PyDateTime_TZInfoType; PyDateTime_DateTimeType.tp_base = &PyDateTime_DateType; - PyTypeObject *types[] = { - &PyDateTime_DateType, - &PyDateTime_DateTimeType, - &PyDateTime_TimeType, - &PyDateTime_DeltaType, - &PyDateTime_TZInfoType, - &PyDateTime_TimeZoneType, - }; + /* Bases classes must be initialized before subclasses, + * so capi_types must have the types in the appropriate order. */ + for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) { + PyTypeObject *type = capi_types[i]; + if (_PyStaticType_InitForExtension(interp, type) < 0) { + return -1; + } + } + + return 0; +} + + +/* --------------------------------------------------------------------------- + * Module methods and initialization. + */ - for (size_t i = 0; i < Py_ARRAY_LENGTH(types); i++) { - if (PyModule_AddType(module, types[i]) < 0) { +static PyMethodDef module_methods[] = { + {NULL, NULL} +}; + + +static int +_datetime_exec(PyObject *module) +{ + int rc = -1; + datetime_state *st = get_module_state(module); + int reloading = 0; + + PyInterpreterState *interp = PyInterpreterState_Get(); + PyObject *old_module = get_current_module(interp, &reloading); + if (PyErr_Occurred()) { + assert(old_module == NULL); + goto error; + } + /* We actually set the "current" module right before a successful return. */ + + if (init_static_types(interp, reloading) < 0) { + goto error; + } + + for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) { + PyTypeObject *type = capi_types[i]; + const char *name = _PyType_Name(type); + assert(name != NULL); + if (PyModule_AddObjectRef(module, name, (PyObject *)type) < 0) { goto error; } } - if (PyType_Ready(&PyDateTime_IsoCalendarDateType) < 0) { + if (init_state(st, module, old_module) < 0) { goto error; } #define DATETIME_ADD_MACRO(dict, c, value_expr) \ do { \ + assert(!PyErr_Occurred()); \ PyObject *value = (value_expr); \ if (value == NULL) { \ goto error; \ @@ -6862,55 +7324,53 @@ _datetime_exec(PyObject *module) Py_DECREF(value); \ } while(0) - /* timedelta values */ - PyObject *d = PyDateTime_DeltaType.tp_dict; - DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - DATETIME_ADD_MACRO(d, "min", new_delta(-MAX_DELTA_DAYS, 0, 0, 0)); - DATETIME_ADD_MACRO(d, "max", - new_delta(MAX_DELTA_DAYS, 24*3600-1, 1000000-1, 0)); - - /* date values */ - d = PyDateTime_DateType.tp_dict; - DATETIME_ADD_MACRO(d, "min", new_date(1, 1, 1)); - DATETIME_ADD_MACRO(d, "max", new_date(MAXYEAR, 12, 31)); - DATETIME_ADD_MACRO(d, "resolution", new_delta(1, 0, 0, 0)); - - /* time values */ - d = PyDateTime_TimeType.tp_dict; - DATETIME_ADD_MACRO(d, "min", new_time(0, 0, 0, 0, Py_None, 0)); - DATETIME_ADD_MACRO(d, "max", new_time(23, 59, 59, 999999, Py_None, 0)); - DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - - /* datetime values */ - d = PyDateTime_DateTimeType.tp_dict; - DATETIME_ADD_MACRO(d, "min", - new_datetime(1, 1, 1, 0, 0, 0, 0, Py_None, 0)); - DATETIME_ADD_MACRO(d, "max", new_datetime(MAXYEAR, 12, 31, 23, 59, 59, - 999999, Py_None, 0)); - DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - - datetime_state *st = STATIC_STATE(); - if (init_state(st) < 0) { - goto error; - } + if (!reloading) { + /* timedelta values */ + PyObject *d = _PyType_GetDict(&PyDateTime_DeltaType); + DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); + DATETIME_ADD_MACRO(d, "min", new_delta(-MAX_DELTA_DAYS, 0, 0, 0)); + DATETIME_ADD_MACRO(d, "max", + new_delta(MAX_DELTA_DAYS, 24*3600-1, 1000000-1, 0)); + + /* date values */ + d = _PyType_GetDict(&PyDateTime_DateType); + DATETIME_ADD_MACRO(d, "min", new_date(1, 1, 1)); + DATETIME_ADD_MACRO(d, "max", new_date(MAXYEAR, 12, 31)); + DATETIME_ADD_MACRO(d, "resolution", new_delta(1, 0, 0, 0)); + + /* time values */ + d = _PyType_GetDict(&PyDateTime_TimeType); + DATETIME_ADD_MACRO(d, "min", new_time(0, 0, 0, 0, Py_None, 0)); + DATETIME_ADD_MACRO(d, "max", new_time(23, 59, 59, 999999, Py_None, 0)); + DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); + + /* datetime values */ + d = _PyType_GetDict(&PyDateTime_DateTimeType); + DATETIME_ADD_MACRO(d, "min", + new_datetime(1, 1, 1, 0, 0, 0, 0, Py_None, 0)); + DATETIME_ADD_MACRO(d, "max", new_datetime(MAXYEAR, 12, 31, 23, 59, 59, + 999999, Py_None, 0)); + DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); + + /* timezone values */ + d = _PyType_GetDict(&PyDateTime_TimeZoneType); + if (PyDict_SetItemString(d, "utc", (PyObject *)&utc_timezone) < 0) { + goto error; + } - /* timezone values */ - d = PyDateTime_TimeZoneType.tp_dict; - if (PyDict_SetItemString(d, "utc", st->utc) < 0) { - goto error; - } + /* bpo-37642: These attributes are rounded to the nearest minute for backwards + * compatibility, even though the constructor will accept a wider range of + * values. This may change in the future.*/ - /* bpo-37642: These attributes are rounded to the nearest minute for backwards - * compatibility, even though the constructor will accept a wider range of - * values. This may change in the future.*/ + /* -23:59 */ + DATETIME_ADD_MACRO(d, "min", create_timezone_from_delta(-1, 60, 0, 1)); - /* -23:59 */ - PyObject *min = create_timezone_from_delta(-1, 60, 0, 1); - DATETIME_ADD_MACRO(d, "min", min); + /* +23:59 */ + DATETIME_ADD_MACRO( + d, "max", create_timezone_from_delta(0, (23 * 60 + 59) * 60, 0, 0)); + } - /* +23:59 */ - PyObject *max = create_timezone_from_delta(0, (23 * 60 + 59) * 60, 0, 0); - DATETIME_ADD_MACRO(d, "max", max); +#undef DATETIME_ADD_MACRO /* Add module level attributes */ if (PyModule_AddIntMacro(module, MINYEAR) < 0) { @@ -6919,7 +7379,7 @@ _datetime_exec(PyObject *module) if (PyModule_AddIntMacro(module, MAXYEAR) < 0) { goto error; } - if (PyModule_AddObjectRef(module, "UTC", st->utc) < 0) { + if (PyModule_AddObjectRef(module, "UTC", (PyObject *)&utc_timezone) < 0) { goto error; } @@ -6928,14 +7388,9 @@ _datetime_exec(PyObject *module) if (capi == NULL) { goto error; } - PyObject *capsule = PyCapsule_New(capi, PyDateTime_CAPSULE_NAME, - datetime_destructor); - if (capsule == NULL) { - PyMem_Free(capi); - goto error; - } + PyObject *capsule = PyCapsule_New(capi, PyDateTime_CAPSULE_NAME, NULL); + // (capsule == NULL) is handled by PyModule_Add if (PyModule_Add(module, "datetime_CAPI", capsule) < 0) { - PyMem_Free(capi); goto error; } @@ -6957,35 +7412,73 @@ _datetime_exec(PyObject *module) static_assert(DI100Y == 25 * DI4Y - 1, "DI100Y"); assert(DI100Y == days_before_year(100+1)); - return 0; + if (set_current_module(interp, module) < 0) { + goto error; + } + + rc = 0; + goto finally; error: - datetime_clear(module); - return -1; + clear_state(st); + +finally: + Py_XDECREF(old_module); + return rc; } -#undef DATETIME_ADD_MACRO -static struct PyModuleDef datetimemodule = { +static PyModuleDef_Slot module_slots[] = { + {Py_mod_exec, _datetime_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + +static int +module_traverse(PyObject *mod, visitproc visit, void *arg) +{ + datetime_state *st = get_module_state(mod); + traverse_state(st, visit, arg); + return 0; +} + +static int +module_clear(PyObject *mod) +{ + datetime_state *st = get_module_state(mod); + clear_state(st); + + PyInterpreterState *interp = PyInterpreterState_Get(); + clear_current_module(interp, mod); + + // The runtime takes care of the static types for us. + // See _PyTypes_FiniExtTypes().. + + return 0; +} + +static void +module_free(void *mod) +{ + (void)module_clear((PyObject *)mod); +} + +static PyModuleDef datetimemodule = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "_datetime", - .m_doc = "Fast implementation of the datetime type.", - .m_size = -1, + .m_doc = "Fast implementation of the datetime module.", + .m_size = sizeof(datetime_state), .m_methods = module_methods, + .m_slots = module_slots, + .m_traverse = module_traverse, + .m_clear = module_clear, + .m_free = module_free, }; PyMODINIT_FUNC PyInit__datetime(void) { - PyObject *mod = PyModule_Create(&datetimemodule); - if (mod == NULL) - return NULL; - - if (_datetime_exec(mod) < 0) { - Py_DECREF(mod); - return NULL; - } - - return mod; + return PyModuleDef_Init(&datetimemodule); } /* --------------------------------------------------------------------------- diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index ee33fe625be3d7..1be4234aad3291 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -616,6 +616,7 @@ _dbm_module_free(void *module) static PyModuleDef_Slot _dbmmodule_slots[] = { {Py_mod_exec, _dbm_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 5b053c73e20bc9..78cf6b1426493b 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -30,10 +30,22 @@ #endif #include -#include "pycore_long.h" // _PyLong_IsZero() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_typeobject.h" #include "complexobject.h" -#include "mpdecimal.h" + +#include + +// Reuse config from mpdecimal.h if present. +#if defined(MPD_CONFIG_64) + #ifndef CONFIG_64 + #define CONFIG_64 MPD_CONFIG_64 + #endif +#elif defined(MPD_CONFIG_32) + #ifndef CONFIG_32 + #define CONFIG_32 MPD_CONFIG_32 + #endif +#endif #include // isascii() #include @@ -63,8 +75,9 @@ typedef struct { #ifndef WITH_DECIMAL_CONTEXTVAR /* Key for thread state dictionary */ PyObject *tls_context_key; - /* Invariant: NULL or the most recently accessed thread local context */ - struct PyDecContextObject *cached_context; + /* Invariant: NULL or a strong reference to the most recently accessed + thread local context. */ + struct PyDecContextObject *cached_context; /* Not borrowed */ #else PyObject *current_context_var; #endif @@ -108,6 +121,8 @@ get_module_state(PyObject *mod) } static struct PyModuleDef _decimal_module; +static PyType_Spec dec_spec; +static PyType_Spec context_spec; static inline decimal_state * get_module_state_by_def(PyTypeObject *tp) @@ -120,13 +135,16 @@ get_module_state_by_def(PyTypeObject *tp) static inline decimal_state * find_state_left_or_right(PyObject *left, PyObject *right) { - PyObject *mod = PyType_GetModuleByDef(Py_TYPE(left), &_decimal_module); - if (mod == NULL) { - PyErr_Clear(); - mod = PyType_GetModuleByDef(Py_TYPE(right), &_decimal_module); + PyTypeObject *base; + if (PyType_GetBaseByToken(Py_TYPE(left), &dec_spec, &base) != 1) { + assert(!PyErr_Occurred()); + PyType_GetBaseByToken(Py_TYPE(right), &dec_spec, &base); } - assert(mod != NULL); - return get_module_state(mod); + assert(base != NULL); + void *state = _PyType_GetModuleState(base); + assert(state != NULL); + Py_DECREF(base); + return (decimal_state *)state; } @@ -150,12 +168,6 @@ find_state_left_or_right(PyObject *left, PyObject *right) #define BOUNDS_CHECK(x, MIN, MAX) x = (x < MIN || MAX < x) ? MAX : x -#if defined(__GNUC__) && !defined(__INTEL_COMPILER) - #define UNUSED __attribute__((unused)) -#else - #define UNUSED -#endif - /* _Py_DEC_MINALLOC >= MPD_MINALLOC */ #define _Py_DEC_MINALLOC 4 @@ -166,11 +178,15 @@ typedef struct { mpd_uint_t data[_Py_DEC_MINALLOC]; } PyDecObject; +#define _PyDecObject_CAST(op) ((PyDecObject *)(op)) + typedef struct { PyObject_HEAD uint32_t *flags; } PyDecSignalDictObject; +#define _PyDecSignalDictObject_CAST(op) ((PyDecSignalDictObject *)(op)) + typedef struct PyDecContextObject { PyObject_HEAD mpd_context_t ctx; @@ -178,25 +194,39 @@ typedef struct PyDecContextObject { PyObject *flags; int capitals; PyThreadState *tstate; + decimal_state *modstate; } PyDecContextObject; +#define _PyDecContextObject_CAST(op) ((PyDecContextObject *)(op)) + typedef struct { PyObject_HEAD PyObject *local; PyObject *global; } PyDecContextManagerObject; +#define _PyDecContextManagerObject_CAST(op) ((PyDecContextManagerObject *)(op)) + #undef MPD #undef CTX #define PyDec_CheckExact(st, v) Py_IS_TYPE(v, (st)->PyDec_Type) #define PyDec_Check(st, v) PyObject_TypeCheck(v, (st)->PyDec_Type) #define PyDecSignalDict_Check(st, v) Py_IS_TYPE(v, (st)->PyDecSignalDict_Type) #define PyDecContext_Check(st, v) PyObject_TypeCheck(v, (st)->PyDecContext_Type) -#define MPD(v) (&((PyDecObject *)v)->dec) -#define SdFlagAddr(v) (((PyDecSignalDictObject *)v)->flags) -#define SdFlags(v) (*((PyDecSignalDictObject *)v)->flags) -#define CTX(v) (&((PyDecContextObject *)v)->ctx) -#define CtxCaps(v) (((PyDecContextObject *)v)->capitals) +#define MPD(v) (&_PyDecObject_CAST(v)->dec) +#define SdFlagAddr(v) (_PyDecSignalDictObject_CAST(v)->flags) +#define SdFlags(v) (*_PyDecSignalDictObject_CAST(v)->flags) +#define CTX(v) (&_PyDecContextObject_CAST(v)->ctx) +#define CtxCaps(v) (_PyDecContextObject_CAST(v)->capitals) + +static inline decimal_state * +get_module_state_from_ctx(PyObject *v) +{ + assert(PyType_GetBaseByToken(Py_TYPE(v), &context_spec, NULL) == 1); + decimal_state *state = ((PyDecContextObject *)v)->modstate; + assert(state != NULL); + return state; +} Py_LOCAL_INLINE(PyObject *) @@ -351,7 +381,7 @@ runtime_error_ptr(const char *mesg) return runtime_error_ptr("internal error in " funcname) static void -dec_traphandler(mpd_context_t *ctx UNUSED) /* GCOV_NOT_REACHED */ +dec_traphandler(mpd_context_t *Py_UNUSED(ctx)) /* GCOV_NOT_REACHED */ { /* GCOV_NOT_REACHED */ return; /* GCOV_NOT_REACHED */ } @@ -552,7 +582,7 @@ static int dec_addstatus(PyObject *context, uint32_t status) { mpd_context_t *ctx = CTX(context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); ctx->status |= status; if (status & (ctx->traps|MPD_Malloc_error)) { @@ -613,7 +643,8 @@ getround(decimal_state *state, PyObject *v) static const char *INVALID_SIGNALDICT_ERROR_MSG = "invalid signal dict"; static int -signaldict_init(PyObject *self, PyObject *args UNUSED, PyObject *kwds UNUSED) +signaldict_init(PyObject *self, + PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwds)) { SdFlagAddr(self) = NULL; return 0; @@ -739,7 +770,7 @@ signaldict_richcompare(PyObject *v, PyObject *w, int op) { PyObject *res = Py_NotImplemented; - decimal_state *state = find_state_left_or_right(v, w); + decimal_state *state = get_module_state_by_def(Py_TYPE(v)); assert(PyDecSignalDict_Check(state, v)); if ((SdFlagAddr(v) == NULL) || (SdFlagAddr(w) == NULL)) { @@ -771,7 +802,7 @@ signaldict_richcompare(PyObject *v, PyObject *w, int op) } static PyObject * -signaldict_copy(PyObject *self, PyObject *args UNUSED) +signaldict_copy(PyObject *self, PyObject *Py_UNUSED(dummy)) { if (SdFlagAddr(self) == NULL) { return value_error_ptr(INVALID_SIGNALDICT_ERROR_MSG); @@ -782,7 +813,7 @@ signaldict_copy(PyObject *self, PyObject *args UNUSED) static PyMethodDef signaldict_methods[] = { - { "copy", (PyCFunction)signaldict_copy, METH_NOARGS, NULL}, + { "copy", signaldict_copy, METH_NOARGS, NULL}, {NULL, NULL} }; @@ -818,18 +849,18 @@ static PyType_Spec signaldict_spec = { /* Context Object, Part 1 */ /******************************************************************************/ -#define Dec_CONTEXT_GET_SSIZE(mem) \ -static PyObject * \ -context_get##mem(PyObject *self, void *closure UNUSED) \ -{ \ - return PyLong_FromSsize_t(mpd_get##mem(CTX(self))); \ +#define Dec_CONTEXT_GET_SSIZE(mem) \ +static PyObject * \ +context_get##mem(PyObject *self, void *Py_UNUSED(closure)) \ +{ \ + return PyLong_FromSsize_t(mpd_get##mem(CTX(self))); \ } -#define Dec_CONTEXT_GET_ULONG(mem) \ -static PyObject * \ -context_get##mem(PyObject *self, void *closure UNUSED) \ -{ \ - return PyLong_FromUnsignedLong(mpd_get##mem(CTX(self))); \ +#define Dec_CONTEXT_GET_ULONG(mem) \ +static PyObject * \ +context_get##mem(PyObject *self, void *Py_UNUSED(closure)) \ +{ \ + return PyLong_FromUnsignedLong(mpd_get##mem(CTX(self))); \ } Dec_CONTEXT_GET_SSIZE(prec) @@ -843,42 +874,42 @@ Dec_CONTEXT_GET_ULONG(status) #endif static PyObject * -context_getround(PyObject *self, void *closure UNUSED) +context_getround(PyObject *self, void *Py_UNUSED(closure)) { int i = mpd_getround(CTX(self)); - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); return Py_NewRef(state->round_map[i]); } static PyObject * -context_getcapitals(PyObject *self, void *closure UNUSED) +context_getcapitals(PyObject *self, void *Py_UNUSED(closure)) { return PyLong_FromLong(CtxCaps(self)); } #ifdef EXTRA_FUNCTIONALITY static PyObject * -context_getallcr(PyObject *self, void *closure UNUSED) +context_getallcr(PyObject *self, void *Py_UNUSED(closure)) { return PyLong_FromLong(mpd_getcr(CTX(self))); } #endif static PyObject * -context_getetiny(PyObject *self, PyObject *dummy UNUSED) +context_getetiny(PyObject *self, PyObject *Py_UNUSED(dummy)) { return PyLong_FromSsize_t(mpd_etiny(CTX(self))); } static PyObject * -context_getetop(PyObject *self, PyObject *dummy UNUSED) +context_getetop(PyObject *self, PyObject *Py_UNUSED(dummy)) { return PyLong_FromSsize_t(mpd_etop(CTX(self))); } static int -context_setprec(PyObject *self, PyObject *value, void *closure UNUSED) +context_setprec(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) { mpd_context_t *ctx; mpd_ssize_t x; @@ -898,7 +929,7 @@ context_setprec(PyObject *self, PyObject *value, void *closure UNUSED) } static int -context_setemin(PyObject *self, PyObject *value, void *closure UNUSED) +context_setemin(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) { mpd_context_t *ctx; mpd_ssize_t x; @@ -918,7 +949,7 @@ context_setemin(PyObject *self, PyObject *value, void *closure UNUSED) } static int -context_setemax(PyObject *self, PyObject *value, void *closure UNUSED) +context_setemax(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) { mpd_context_t *ctx; mpd_ssize_t x; @@ -1000,12 +1031,12 @@ context_unsafe_setemax(PyObject *self, PyObject *value) #endif static int -context_setround(PyObject *self, PyObject *value, void *closure UNUSED) +context_setround(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) { mpd_context_t *ctx; int x; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); x = getround(state, value); if (x == -1) { return -1; @@ -1020,7 +1051,7 @@ context_setround(PyObject *self, PyObject *value, void *closure UNUSED) } static int -context_setcapitals(PyObject *self, PyObject *value, void *closure UNUSED) +context_setcapitals(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) { mpd_ssize_t x; @@ -1040,7 +1071,7 @@ context_setcapitals(PyObject *self, PyObject *value, void *closure UNUSED) #ifdef EXTRA_FUNCTIONALITY static int -context_settraps(PyObject *self, PyObject *value, void *closure UNUSED) +context_settraps(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) { mpd_context_t *ctx; uint32_t flags; @@ -1064,7 +1095,7 @@ context_settraps_list(PyObject *self, PyObject *value) { mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); flags = list_as_flags(state, value); if (flags & DEC_ERRORS) { return -1; @@ -1084,7 +1115,7 @@ context_settraps_dict(PyObject *self, PyObject *value) mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); if (PyDecSignalDict_Check(state, value)) { flags = SdFlags(value); } @@ -1105,7 +1136,7 @@ context_settraps_dict(PyObject *self, PyObject *value) #ifdef EXTRA_FUNCTIONALITY static int -context_setstatus(PyObject *self, PyObject *value, void *closure UNUSED) +context_setstatus(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) { mpd_context_t *ctx; uint32_t flags; @@ -1129,7 +1160,7 @@ context_setstatus_list(PyObject *self, PyObject *value) { mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); flags = list_as_flags(state, value); if (flags & DEC_ERRORS) { @@ -1150,7 +1181,7 @@ context_setstatus_dict(PyObject *self, PyObject *value) mpd_context_t *ctx; uint32_t flags; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); if (PyDecSignalDict_Check(state, value)) { flags = SdFlags(value); } @@ -1170,7 +1201,7 @@ context_setstatus_dict(PyObject *self, PyObject *value) } static int -context_setclamp(PyObject *self, PyObject *value, void *closure UNUSED) +context_setclamp(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) { mpd_context_t *ctx; mpd_ssize_t x; @@ -1191,7 +1222,7 @@ context_setclamp(PyObject *self, PyObject *value, void *closure UNUSED) #ifdef EXTRA_FUNCTIONALITY static int -context_setallcr(PyObject *self, PyObject *value, void *closure UNUSED) +context_setallcr(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) { mpd_context_t *ctx; mpd_ssize_t x; @@ -1313,14 +1344,14 @@ context_setattrs(PyObject *self, PyObject *prec, PyObject *rounding, } static PyObject * -context_clear_traps(PyObject *self, PyObject *dummy UNUSED) +context_clear_traps(PyObject *self, PyObject *Py_UNUSED(dummy)) { CTX(self)->traps = 0; Py_RETURN_NONE; } static PyObject * -context_clear_flags(PyObject *self, PyObject *dummy UNUSED) +context_clear_flags(PyObject *self, PyObject *Py_UNUSED(dummy)) { CTX(self)->status = 0; Py_RETURN_NONE; @@ -1336,7 +1367,8 @@ static mpd_context_t dflt_ctx = { }; static PyObject * -context_new(PyTypeObject *type, PyObject *args UNUSED, PyObject *kwds UNUSED) +context_new(PyTypeObject *type, + PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwds)) { PyDecContextObject *self = NULL; mpd_context_t *ctx; @@ -1379,13 +1411,19 @@ context_new(PyTypeObject *type, PyObject *args UNUSED, PyObject *kwds UNUSED) CtxCaps(self) = 1; self->tstate = NULL; + self->modstate = state; + if (type == state->PyDecContext_Type) { + PyObject_GC_Track(self); + } + assert(PyObject_GC_IsTracked((PyObject *)self)); return (PyObject *)self; } static int -context_traverse(PyDecContextObject *self, visitproc visit, void *arg) +context_traverse(PyObject *op, visitproc visit, void *arg) { + PyDecContextObject *self = _PyDecContextObject_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->traps); Py_VISIT(self->flags); @@ -1393,24 +1431,19 @@ context_traverse(PyDecContextObject *self, visitproc visit, void *arg) } static int -context_clear(PyDecContextObject *self) +context_clear(PyObject *op) { + PyDecContextObject *self = _PyDecContextObject_CAST(op); Py_CLEAR(self->traps); Py_CLEAR(self->flags); return 0; } static void -context_dealloc(PyDecContextObject *self) +context_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); -#ifndef WITH_DECIMAL_CONTEXTVAR - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); - if (self == state->cached_context) { - state->cached_context = NULL; - } -#endif (void)context_clear(self); tp->tp_free(self); Py_DECREF(tp); @@ -1450,7 +1483,7 @@ context_init(PyObject *self, PyObject *args, PyObject *kwds) } static PyObject * -context_repr(PyDecContextObject *self) +context_repr(PyObject *self) { mpd_context_t *ctx; char flags[MPD_MAX_SIGNAL_LIST]; @@ -1458,7 +1491,7 @@ context_repr(PyDecContextObject *self) int n, mem; #ifdef Py_DEBUG - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); assert(PyDecContext_Check(state, self)); #endif ctx = CTX(self); @@ -1478,7 +1511,7 @@ context_repr(PyDecContextObject *self) "Context(prec=%zd, rounding=%s, Emin=%zd, Emax=%zd, " "capitals=%d, clamp=%d, flags=%s, traps=%s)", ctx->prec, mpd_round_string[ctx->round], ctx->emin, ctx->emax, - self->capitals, ctx->clamp, flags, traps); + CtxCaps(self), ctx->clamp, flags, traps); } static void @@ -1509,7 +1542,7 @@ init_extended_context(PyObject *v) #ifdef EXTRA_FUNCTIONALITY /* Factory function for creating IEEE interchange format contexts */ static PyObject * -ieee_context(PyObject *dummy UNUSED, PyObject *v) +ieee_context(PyObject *module, PyObject *v) { PyObject *context; mpd_ssize_t bits; @@ -1526,7 +1559,7 @@ ieee_context(PyObject *dummy UNUSED, PyObject *v) goto error; } - decimal_state *state = get_module_state_by_def(Py_TYPE(v)); + decimal_state *state = get_module_state(module); context = PyObject_CallObject((PyObject *)state->PyDecContext_Type, NULL); if (context == NULL) { return NULL; @@ -1545,11 +1578,11 @@ ieee_context(PyObject *dummy UNUSED, PyObject *v) #endif static PyObject * -context_copy(PyObject *self, PyObject *args UNUSED) +context_copy(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *copy; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); copy = PyObject_CallObject((PyObject *)state->PyDecContext_Type, NULL); if (copy == NULL) { return NULL; @@ -1563,13 +1596,13 @@ context_copy(PyObject *self, PyObject *args UNUSED) } static PyObject * -context_reduce(PyObject *self, PyObject *args UNUSED) +context_reduce(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *flags; PyObject *traps; PyObject *ret; mpd_context_t *ctx; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = get_module_state_from_ctx(self); ctx = CTX(self); @@ -1598,16 +1631,16 @@ context_reduce(PyObject *self, PyObject *args UNUSED) static PyGetSetDef context_getsets [] = { - { "prec", (getter)context_getprec, (setter)context_setprec, NULL, NULL}, - { "Emax", (getter)context_getemax, (setter)context_setemax, NULL, NULL}, - { "Emin", (getter)context_getemin, (setter)context_setemin, NULL, NULL}, - { "rounding", (getter)context_getround, (setter)context_setround, NULL, NULL}, - { "capitals", (getter)context_getcapitals, (setter)context_setcapitals, NULL, NULL}, - { "clamp", (getter)context_getclamp, (setter)context_setclamp, NULL, NULL}, + { "prec", context_getprec, context_setprec, NULL, NULL}, + { "Emax", context_getemax, context_setemax, NULL, NULL}, + { "Emin", context_getemin, context_setemin, NULL, NULL}, + { "rounding", context_getround, context_setround, NULL, NULL}, + { "capitals", context_getcapitals, context_setcapitals, NULL, NULL}, + { "clamp", context_getclamp, context_setclamp, NULL, NULL}, #ifdef EXTRA_FUNCTIONALITY - { "_allcr", (getter)context_getallcr, (setter)context_setallcr, NULL, NULL}, - { "_traps", (getter)context_gettraps, (setter)context_settraps, NULL, NULL}, - { "_flags", (getter)context_getstatus, (setter)context_setstatus, NULL, NULL}, + { "_allcr", context_getallcr, context_setallcr, NULL, NULL}, + { "_traps", context_gettraps, context_settraps, NULL, NULL}, + { "_flags", context_getstatus, context_setstatus, NULL, NULL}, #endif {NULL} }; @@ -1687,7 +1720,8 @@ current_context_from_dict(decimal_state *modstate) /* Cache the context of the current thread, assuming that it * will be accessed several times before a thread switch. */ - modstate->cached_context = (PyDecContextObject *)tl_context; + Py_XSETREF(modstate->cached_context, + (PyDecContextObject *)Py_NewRef(tl_context)); modstate->cached_context->tstate = tstate; /* Borrowed reference with refcount==1 */ @@ -1707,15 +1741,17 @@ current_context(decimal_state *modstate) } /* ctxobj := borrowed reference to the current context */ -#define CURRENT_CONTEXT(state, ctxobj) \ - ctxobj = current_context(state); \ - if (ctxobj == NULL) { \ - return NULL; \ - } +#define CURRENT_CONTEXT(STATE, CTXOBJ) \ + do { \ + CTXOBJ = current_context(STATE); \ + if (CTXOBJ == NULL) { \ + return NULL; \ + } \ + } while (0) /* Return a new reference to the current context */ static PyObject * -PyDec_GetCurrentContext(PyObject *self, PyObject *args UNUSED) +PyDec_GetCurrentContext(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *context; decimal_state *state = get_module_state(self); @@ -1755,7 +1791,7 @@ PyDec_SetCurrentContext(PyObject *self, PyObject *v) Py_INCREF(v); } - state->cached_context = NULL; + Py_CLEAR(state->cached_context); if (PyDict_SetItem(dict, state->tls_context_key, v) < 0) { Py_DECREF(v); return NULL; @@ -1800,16 +1836,18 @@ current_context(decimal_state *state) } /* ctxobj := borrowed reference to the current context */ -#define CURRENT_CONTEXT(state, ctxobj) \ - ctxobj = current_context(state); \ - if (ctxobj == NULL) { \ - return NULL; \ - } \ - Py_DECREF(ctxobj); +#define CURRENT_CONTEXT(STATE, CTXOBJ) \ + do { \ + CTXOBJ = current_context(STATE); \ + if (CTXOBJ == NULL) { \ + return NULL; \ + } \ + Py_DECREF(CTXOBJ); \ + } while (0) /* Return a new reference to the current context */ static PyObject * -PyDec_GetCurrentContext(PyObject *self, PyObject *args UNUSED) +PyDec_GetCurrentContext(PyObject *self, PyObject *Py_UNUSED(dummy)) { decimal_state *state = get_module_state(self); return current_context(state); @@ -1918,9 +1956,9 @@ ctxmanager_new(PyObject *m, PyObject *args, PyObject *kwds) } static int -ctxmanager_traverse(PyDecContextManagerObject *self, visitproc visit, - void *arg) +ctxmanager_traverse(PyObject *op, visitproc visit, void *arg) { + PyDecContextManagerObject *self = _PyDecContextManagerObject_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->local); Py_VISIT(self->global); @@ -1928,28 +1966,29 @@ ctxmanager_traverse(PyDecContextManagerObject *self, visitproc visit, } static int -ctxmanager_clear(PyDecContextManagerObject *self) +ctxmanager_clear(PyObject *op) { + PyDecContextManagerObject *self = _PyDecContextManagerObject_CAST(op); Py_CLEAR(self->local); Py_CLEAR(self->global); return 0; } static void -ctxmanager_dealloc(PyDecContextManagerObject *self) +ctxmanager_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); (void)ctxmanager_clear(self); - tp->tp_free((PyObject *)self); + tp->tp_free(self); Py_DECREF(tp); } static PyObject * -ctxmanager_set_local(PyDecContextManagerObject *self, PyObject *args UNUSED) +ctxmanager_set_local(PyObject *op, PyObject *Py_UNUSED(dummy)) { PyObject *ret; - + PyDecContextManagerObject *self = _PyDecContextManagerObject_CAST(op); ret = PyDec_SetCurrentContext(PyType_GetModule(Py_TYPE(self)), self->local); if (ret == NULL) { return NULL; @@ -1960,11 +1999,10 @@ ctxmanager_set_local(PyDecContextManagerObject *self, PyObject *args UNUSED) } static PyObject * -ctxmanager_restore_global(PyDecContextManagerObject *self, - PyObject *args UNUSED) +ctxmanager_restore_global(PyObject *op, PyObject *Py_UNUSED(args)) { PyObject *ret; - + PyDecContextManagerObject *self = _PyDecContextManagerObject_CAST(op); ret = PyDec_SetCurrentContext(PyType_GetModule(Py_TYPE(self)), self->global); if (ret == NULL) { return NULL; @@ -1976,8 +2014,8 @@ ctxmanager_restore_global(PyDecContextManagerObject *self, static PyMethodDef ctxmanager_methods[] = { - {"__enter__", (PyCFunction)ctxmanager_set_local, METH_NOARGS, NULL}, - {"__exit__", (PyCFunction)ctxmanager_restore_global, METH_VARARGS, NULL}, + {"__enter__", ctxmanager_set_local, METH_NOARGS, NULL}, + {"__exit__", ctxmanager_restore_global, METH_VARARGS, NULL}, {NULL, NULL} }; @@ -2004,11 +2042,10 @@ static PyType_Spec ctxmanager_spec = { /******************************************************************************/ static PyObject * -PyDecType_New(PyTypeObject *type) +PyDecType_New(decimal_state *state, PyTypeObject *type) { PyDecObject *dec; - decimal_state *state = get_module_state_by_def(type); if (type == state->PyDec_Type) { dec = PyObject_GC_New(PyDecObject, state->PyDec_Type); } @@ -2028,9 +2065,13 @@ PyDecType_New(PyTypeObject *type) MPD(dec)->alloc = _Py_DEC_MINALLOC; MPD(dec)->data = dec->data; + if (type == state->PyDec_Type) { + PyObject_GC_Track(dec); + } + assert(PyObject_GC_IsTracked((PyObject *)dec)); return (PyObject *)dec; } -#define dec_alloc(st) PyDecType_New((st)->PyDec_Type) +#define dec_alloc(st) PyDecType_New(st, (st)->PyDec_Type) static int dec_traverse(PyObject *dec, visitproc visit, void *arg) @@ -2133,7 +2174,8 @@ PyDecType_FromCString(PyTypeObject *type, const char *s, PyObject *dec; uint32_t status = 0; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2157,7 +2199,8 @@ PyDecType_FromCStringExact(PyTypeObject *type, const char *s, uint32_t status = 0; mpd_context_t maxctx; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2244,7 +2287,8 @@ PyDecType_FromSsize(PyTypeObject *type, mpd_ssize_t v, PyObject *context) PyObject *dec; uint32_t status = 0; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2265,7 +2309,8 @@ PyDecType_FromSsizeExact(PyTypeObject *type, mpd_ssize_t v, PyObject *context) uint32_t status = 0; mpd_context_t maxctx; - dec = PyDecType_New(type); + decimal_state *state = get_module_state_from_ctx(context); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2283,41 +2328,46 @@ PyDecType_FromSsizeExact(PyTypeObject *type, mpd_ssize_t v, PyObject *context) /* Convert from a PyLongObject. The context is not modified; flags set during conversion are accumulated in the status parameter. */ static PyObject * -dec_from_long(PyTypeObject *type, PyObject *v, +dec_from_long(decimal_state *state, PyTypeObject *type, PyObject *v, const mpd_context_t *ctx, uint32_t *status) { - PyObject *dec; - PyLongObject *l = (PyLongObject *)v; + PyObject *dec = PyDecType_New(state, type); - dec = PyDecType_New(type); if (dec == NULL) { return NULL; } - if (_PyLong_IsZero(l)) { - _dec_settriple(dec, MPD_POS, 0, 0); - return dec; - } - - uint8_t sign = _PyLong_IsNegative(l) ? MPD_NEG : MPD_POS; + PyLongExport export_long; - if (_PyLong_IsCompact(l)) { - _dec_settriple(dec, sign, l->long_value.ob_digit[0], 0); - mpd_qfinalize(MPD(dec), ctx, status); - return dec; + if (PyLong_Export(v, &export_long) == -1) { + Py_DECREF(dec); + return NULL; } - size_t len = _PyLong_DigitCount(l); + if (export_long.digits) { + const PyLongLayout *layout = PyLong_GetNativeLayout(); -#if PYLONG_BITS_IN_DIGIT == 30 - mpd_qimport_u32(MPD(dec), l->long_value.ob_digit, len, sign, PyLong_BASE, - ctx, status); -#elif PYLONG_BITS_IN_DIGIT == 15 - mpd_qimport_u16(MPD(dec), l->long_value.ob_digit, len, sign, PyLong_BASE, - ctx, status); -#else - #error "PYLONG_BITS_IN_DIGIT should be 15 or 30" -#endif + assert(layout->bits_per_digit < 32); + assert(layout->digits_order == -1); + assert(layout->digit_endianness == (PY_LITTLE_ENDIAN ? -1 : 1)); + assert(layout->digit_size == 2 || layout->digit_size == 4); + + uint32_t base = (uint32_t)1 << layout->bits_per_digit; + uint8_t sign = export_long.negative ? MPD_NEG : MPD_POS; + Py_ssize_t len = export_long.ndigits; + if (layout->digit_size == 4) { + mpd_qimport_u32(MPD(dec), export_long.digits, len, sign, + base, ctx, status); + } + else { + mpd_qimport_u16(MPD(dec), export_long.digits, len, sign, + base, ctx, status); + } + PyLong_FreeExport(&export_long); + } + else { + mpd_qset_i64(MPD(dec), export_long.value, ctx, status); + } return dec; } @@ -2334,7 +2384,8 @@ PyDecType_FromLong(PyTypeObject *type, PyObject *v, PyObject *context) return NULL; } - dec = dec_from_long(type, v, CTX(context), &status); + decimal_state *state = get_module_state_from_ctx(context); + dec = dec_from_long(state, type, v, CTX(context), &status); if (dec == NULL) { return NULL; } @@ -2363,7 +2414,8 @@ PyDecType_FromLongExact(PyTypeObject *type, PyObject *v, } mpd_maxcontext(&maxctx); - dec = dec_from_long(type, v, &maxctx, &status); + decimal_state *state = get_module_state_from_ctx(context); + dec = dec_from_long(state, type, v, &maxctx, &status); if (dec == NULL) { return NULL; } @@ -2395,7 +2447,7 @@ PyDecType_FromFloatExact(PyTypeObject *type, PyObject *v, mpd_t *d1, *d2; uint32_t status = 0; mpd_context_t maxctx; - decimal_state *state = get_module_state_by_def(type); + decimal_state *state = get_module_state_from_ctx(context); #ifdef Py_DEBUG assert(PyType_IsSubtype(type, state->PyDec_Type)); @@ -2415,12 +2467,12 @@ PyDecType_FromFloatExact(PyTypeObject *type, PyObject *v, } sign = (copysign(1.0, x) == 1.0) ? 0 : 1; - if (Py_IS_NAN(x) || Py_IS_INFINITY(x)) { - dec = PyDecType_New(type); + if (isnan(x) || isinf(x)) { + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } - if (Py_IS_NAN(x)) { + if (isnan(x)) { /* decimal.py calls repr(float(+-nan)), * which always gives a positive result. */ mpd_setspecial(MPD(dec), MPD_POS, MPD_NAN); @@ -2533,12 +2585,12 @@ PyDecType_FromDecimalExact(PyTypeObject *type, PyObject *v, PyObject *context) PyObject *dec; uint32_t status = 0; - decimal_state *state = get_module_state_by_def(type); + decimal_state *state = get_module_state_from_ctx(context); if (type == state->PyDec_Type && PyDec_CheckExact(state, v)) { return Py_NewRef(v); } - dec = PyDecType_New(type); + dec = PyDecType_New(state, type); if (dec == NULL) { return NULL; } @@ -2818,11 +2870,56 @@ dec_from_float(PyObject *type, PyObject *pyfloat) return result; } +/* 'v' can have any numeric type accepted by the Decimal constructor. Attempt + an exact conversion. If the result does not meet the restrictions + for an mpd_t, fail with InvalidOperation. */ +static PyObject * +PyDecType_FromNumberExact(PyTypeObject *type, PyObject *v, PyObject *context) +{ + decimal_state *state = get_module_state_by_def(type); + assert(v != NULL); + if (PyDec_Check(state, v)) { + return PyDecType_FromDecimalExact(type, v, context); + } + else if (PyLong_Check(v)) { + return PyDecType_FromLongExact(type, v, context); + } + else if (PyFloat_Check(v)) { + if (dec_addstatus(context, MPD_Float_operation)) { + return NULL; + } + return PyDecType_FromFloatExact(type, v, context); + } + else { + PyErr_Format(PyExc_TypeError, + "conversion from %s to Decimal is not supported", + Py_TYPE(v)->tp_name); + return NULL; + } +} + +/* class method */ +static PyObject * +dec_from_number(PyObject *type, PyObject *number) +{ + PyObject *context; + PyObject *result; + + decimal_state *state = get_module_state_by_def((PyTypeObject *)type); + CURRENT_CONTEXT(state, context); + result = PyDecType_FromNumberExact(state->PyDec_Type, number, context); + if (type != (PyObject *)state->PyDec_Type && result != NULL) { + Py_SETREF(result, PyObject_CallFunctionObjArgs(type, result, NULL)); + } + + return result; +} + /* create_decimal_from_float */ static PyObject * ctx_from_float(PyObject *context, PyObject *v) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); return PyDec_FromFloat(state, v, context); } @@ -2833,7 +2930,7 @@ dec_apply(PyObject *v, PyObject *context) PyObject *result; uint32_t status = 0; - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { return NULL; @@ -2860,7 +2957,7 @@ dec_apply(PyObject *v, PyObject *context) static PyObject * PyDecType_FromObjectExact(PyTypeObject *type, PyObject *v, PyObject *context) { - decimal_state *state = get_module_state_by_def(type); + decimal_state *state = get_module_state_from_ctx(context); if (v == NULL) { return PyDecType_FromSsizeExact(type, 0, context); } @@ -2895,7 +2992,7 @@ PyDecType_FromObjectExact(PyTypeObject *type, PyObject *v, PyObject *context) static PyObject * PyDec_FromObject(PyObject *v, PyObject *context) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (v == NULL) { return PyDec_FromSsize(state, 0, context); } @@ -2982,7 +3079,7 @@ ctx_create_decimal(PyObject *context, PyObject *args) Py_LOCAL_INLINE(int) convert_op(int type_err, PyObject **conv, PyObject *v, PyObject *context) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (PyDec_Check(state, v)) { *conv = Py_NewRef(v); return 1; @@ -3085,7 +3182,7 @@ multiply_by_denominator(PyObject *v, PyObject *r, PyObject *context) if (tmp == NULL) { return NULL; } - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); denom = PyDec_FromLongExact(state, tmp, context); Py_DECREF(tmp); if (denom == NULL) { @@ -3140,7 +3237,7 @@ numerator_as_decimal(PyObject *r, PyObject *context) return NULL; } - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); num = PyDec_FromLongExact(state, tmp, context); Py_DECREF(tmp); return num; @@ -3159,7 +3256,7 @@ convert_op_cmp(PyObject **vcmp, PyObject **wcmp, PyObject *v, PyObject *w, *vcmp = v; - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (PyDec_Check(state, w)) { *wcmp = Py_NewRef(w); } @@ -3555,13 +3652,6 @@ dec_format(PyObject *dec, PyObject *args) static PyObject * dec_as_long(PyObject *dec, PyObject *context, int round) { - PyLongObject *pylong; - digit *ob_digit; - size_t n; - mpd_t *x; - mpd_context_t workctx; - uint32_t status = 0; - if (mpd_isspecial(MPD(dec))) { if (mpd_isnan(MPD(dec))) { PyErr_SetString(PyExc_ValueError, @@ -3574,12 +3664,16 @@ dec_as_long(PyObject *dec, PyObject *context, int round) return NULL; } - x = mpd_qnew(); + mpd_t *x = mpd_qnew(); + if (x == NULL) { PyErr_NoMemory(); return NULL; } - workctx = *CTX(context); + + mpd_context_t workctx = *CTX(context); + uint32_t status = 0; + workctx.round = round; mpd_qround_to_int(x, MPD(dec), &workctx, &status); if (dec_addstatus(context, status)) { @@ -3588,39 +3682,61 @@ dec_as_long(PyObject *dec, PyObject *context, int round) } status = 0; - ob_digit = NULL; -#if PYLONG_BITS_IN_DIGIT == 30 - n = mpd_qexport_u32(&ob_digit, 0, PyLong_BASE, x, &status); -#elif PYLONG_BITS_IN_DIGIT == 15 - n = mpd_qexport_u16(&ob_digit, 0, PyLong_BASE, x, &status); -#else - #error "PYLONG_BITS_IN_DIGIT should be 15 or 30" -#endif + int64_t val = mpd_qget_i64(x, &status); + + if (!status) { + mpd_del(x); + return PyLong_FromInt64(val); + } + assert(!mpd_iszero(x)); + + const PyLongLayout *layout = PyLong_GetNativeLayout(); + + assert(layout->bits_per_digit < 32); + assert(layout->digits_order == -1); + assert(layout->digit_endianness == (PY_LITTLE_ENDIAN ? -1 : 1)); + assert(layout->digit_size == 2 || layout->digit_size == 4); + + uint32_t base = (uint32_t)1 << layout->bits_per_digit; + /* We use a temporary buffer for digits for now, as for nonzero rdata + mpd_qexport_u32/u16() require either space "allocated by one of + libmpdec’s allocation functions" or "rlen MUST be correct" (to avoid + reallocation). This can be further optimized by using rlen from + mpd_sizeinbase(). See gh-127925. */ + void *tmp_digits = NULL; + size_t n; + + status = 0; + if (layout->digit_size == 4) { + n = mpd_qexport_u32((uint32_t **)&tmp_digits, 0, base, x, &status); + } + else { + n = mpd_qexport_u16((uint16_t **)&tmp_digits, 0, base, x, &status); + } if (n == SIZE_MAX) { PyErr_NoMemory(); mpd_del(x); + mpd_free(tmp_digits); return NULL; } - if (n == 1) { - sdigit val = mpd_arith_sign(x) * ob_digit[0]; - mpd_free(ob_digit); - mpd_del(x); - return PyLong_FromLong(val); - } + void *digits; + PyLongWriter *writer = PyLongWriter_Create(mpd_isnegative(x), n, &digits); - assert(n > 0); - assert(!mpd_iszero(x)); - pylong = _PyLong_FromDigits(mpd_isnegative(x), n, ob_digit); - mpd_free(ob_digit); mpd_del(x); - return (PyObject *) pylong; + if (writer == NULL) { + mpd_free(tmp_digits); + return NULL; + } + memcpy(digits, tmp_digits, layout->digit_size*n); + mpd_free(tmp_digits); + return PyLongWriter_Finish(writer); } /* Convert a Decimal to its exact integer ratio representation. */ static PyObject * -dec_as_integer_ratio(PyObject *self, PyObject *args UNUSED) +dec_as_integer_ratio(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *numerator = NULL; PyObject *denominator = NULL; @@ -3888,7 +4004,7 @@ PyDec_Round(PyObject *dec, PyObject *args) /* Return the DecimalTuple representation of a PyDecObject. */ static PyObject * -PyDec_AsTuple(PyObject *dec, PyObject *dummy UNUSED) +PyDec_AsTuple(PyObject *dec, PyObject *Py_UNUSED(dummy)) { PyObject *result = NULL; PyObject *sign = NULL; @@ -4044,7 +4160,7 @@ nm_##MPDFUNC(PyObject *self, PyObject *other) \ /* Boolean function without a context arg. */ #define Dec_BoolFunc(MPDFUNC) \ static PyObject * \ -dec_##MPDFUNC(PyObject *self, PyObject *dummy UNUSED) \ +dec_##MPDFUNC(PyObject *self, PyObject *Py_UNUSED(dummy)) \ { \ return MPDFUNC(MPD(self)) ? incr_true() : incr_false(); \ } @@ -4277,7 +4393,7 @@ nm_mpd_qdivmod(PyObject *v, PyObject *w) return NULL; } - ret = Py_BuildValue("(OO)", q, r); + ret = PyTuple_Pack(2, q, r); Py_DECREF(r); Py_DECREF(q); return ret; @@ -4373,7 +4489,7 @@ Dec_BoolFuncVA(mpd_issubnormal) /* Unary functions, no context arg */ static PyObject * -dec_mpd_adjexp(PyObject *self, PyObject *dummy UNUSED) +dec_mpd_adjexp(PyObject *self, PyObject *Py_UNUSED(dummy)) { mpd_ssize_t retval; @@ -4388,23 +4504,22 @@ dec_mpd_adjexp(PyObject *self, PyObject *dummy UNUSED) } static PyObject * -dec_canonical(PyObject *self, PyObject *dummy UNUSED) +dec_canonical(PyObject *self, PyObject *Py_UNUSED(dummy)) { return Py_NewRef(self); } static PyObject * -dec_conjugate(PyObject *self, PyObject *dummy UNUSED) +dec_conjugate(PyObject *self, PyObject *Py_UNUSED(dummy)) { return Py_NewRef(self); } -static PyObject * -dec_mpd_radix(PyObject *self, PyObject *dummy UNUSED) +static inline PyObject * +_dec_mpd_radix(decimal_state *state) { PyObject *result; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); result = dec_alloc(state); if (result == NULL) { return NULL; @@ -4415,7 +4530,14 @@ dec_mpd_radix(PyObject *self, PyObject *dummy UNUSED) } static PyObject * -dec_mpd_qcopy_abs(PyObject *self, PyObject *dummy UNUSED) +dec_mpd_radix(PyObject *self, PyObject *Py_UNUSED(dummy)) +{ + decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + return _dec_mpd_radix(state); +} + +static PyObject * +dec_mpd_qcopy_abs(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *result; uint32_t status = 0; @@ -4436,7 +4558,7 @@ dec_mpd_qcopy_abs(PyObject *self, PyObject *dummy UNUSED) } static PyObject * -dec_mpd_qcopy_negate(PyObject *self, PyObject *dummy UNUSED) +dec_mpd_qcopy_negate(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *result; uint32_t status = 0; @@ -4690,7 +4812,7 @@ dec_richcompare(PyObject *v, PyObject *w, int op) /* __ceil__ */ static PyObject * -dec_ceil(PyObject *self, PyObject *dummy UNUSED) +dec_ceil(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *context; @@ -4701,7 +4823,7 @@ dec_ceil(PyObject *self, PyObject *dummy UNUSED) /* __complex__ */ static PyObject * -dec_complex(PyObject *self, PyObject *dummy UNUSED) +dec_complex(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *f; double x; @@ -4720,16 +4842,16 @@ dec_complex(PyObject *self, PyObject *dummy UNUSED) return PyComplex_FromDoubles(x, 0); } -/* __copy__ and __deepcopy__ */ +/* __copy__ (METH_NOARGS) and __deepcopy__ (METH_O) */ static PyObject * -dec_copy(PyObject *self, PyObject *dummy UNUSED) +dec_copy(PyObject *self, PyObject *Py_UNUSED(dummy)) { return Py_NewRef(self); } /* __floor__ */ static PyObject * -dec_floor(PyObject *self, PyObject *dummy UNUSED) +dec_floor(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *context; @@ -4780,7 +4902,7 @@ _dec_hash(PyDecObject *v) return -1; } else if (mpd_isnan(MPD(v))) { - return _Py_HashPointer(v); + return PyObject_GenericHash((PyObject *)v); } else { return py_hash_inf * mpd_arith_sign(MPD(v)); @@ -4866,7 +4988,7 @@ dec_hash(PyDecObject *self) /* __reduce__ */ static PyObject * -dec_reduce(PyObject *self, PyObject *dummy UNUSED) +dec_reduce(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *result, *str; @@ -4883,7 +5005,7 @@ dec_reduce(PyObject *self, PyObject *dummy UNUSED) /* __sizeof__ */ static PyObject * -dec_sizeof(PyObject *v, PyObject *dummy UNUSED) +dec_sizeof(PyObject *v, PyObject *Py_UNUSED(dummy)) { size_t res = _PyObject_SIZE(Py_TYPE(v)); if (mpd_isdynamic_data(MPD(v))) { @@ -4894,7 +5016,7 @@ dec_sizeof(PyObject *v, PyObject *dummy UNUSED) /* __trunc__ */ static PyObject * -dec_trunc(PyObject *self, PyObject *dummy UNUSED) +dec_trunc(PyObject *self, PyObject *Py_UNUSED(dummy)) { PyObject *context; @@ -4905,13 +5027,13 @@ dec_trunc(PyObject *self, PyObject *dummy UNUSED) /* real and imag */ static PyObject * -dec_real(PyObject *self, void *closure UNUSED) +dec_real(PyObject *self, void *Py_UNUSED(closure)) { return Py_NewRef(self); } static PyObject * -dec_imag(PyObject *self UNUSED, void *closure UNUSED) +dec_imag(PyObject *self, void *Py_UNUSED(closure)) { PyObject *result; @@ -4928,8 +5050,8 @@ dec_imag(PyObject *self UNUSED, void *closure UNUSED) static PyGetSetDef dec_getsets [] = { - { "real", (getter)dec_real, NULL, NULL, NULL}, - { "imag", (getter)dec_imag, NULL, NULL, NULL}, + { "real", dec_real, NULL, NULL, NULL}, + { "imag", dec_imag, NULL, NULL, NULL}, {NULL} }; @@ -5007,6 +5129,7 @@ static PyMethodDef dec_methods [] = /* Miscellaneous */ { "from_float", dec_from_float, METH_O|METH_CLASS, doc_from_float }, + { "from_number", dec_from_number, METH_O|METH_CLASS, doc_from_number }, { "as_tuple", PyDec_AsTuple, METH_NOARGS, doc_as_tuple }, { "as_integer_ratio", dec_as_integer_ratio, METH_NOARGS, doc_as_integer_ratio }, @@ -5026,6 +5149,7 @@ static PyMethodDef dec_methods [] = }; static PyType_Slot dec_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, {Py_tp_dealloc, dec_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_traverse, dec_traverse}, @@ -5115,7 +5239,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *v) \ \ CONVERT_OP_RAISE(&a, v, context); \ decimal_state *state = \ - get_module_state_by_def(Py_TYPE(context)); \ + get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ return NULL; \ @@ -5147,7 +5271,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *args) \ \ CONVERT_BINOP_RAISE(&a, &b, v, w, context); \ decimal_state *state = \ - get_module_state_by_def(Py_TYPE(context)); \ + get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ Py_DECREF(b); \ @@ -5183,7 +5307,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *args) \ \ CONVERT_BINOP_RAISE(&a, &b, v, w, context); \ decimal_state *state = \ - get_module_state_by_def(Py_TYPE(context)); \ + get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ Py_DECREF(b); \ @@ -5212,7 +5336,7 @@ ctx_##MPDFUNC(PyObject *context, PyObject *args) \ } \ \ CONVERT_TERNOP_RAISE(&a, &b, &c, v, w, x, context); \ - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); \ + decimal_state *state = get_module_state_from_ctx(context); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ Py_DECREF(b); \ @@ -5278,7 +5402,7 @@ ctx_mpd_qdivmod(PyObject *context, PyObject *args) } CONVERT_BINOP_RAISE(&a, &b, v, w, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); q = dec_alloc(state); if (q == NULL) { Py_DECREF(a); @@ -5302,7 +5426,7 @@ ctx_mpd_qdivmod(PyObject *context, PyObject *args) return NULL; } - ret = Py_BuildValue("(OO)", q, r); + ret = PyTuple_Pack(2, q, r); Py_DECREF(r); Py_DECREF(q); return ret; @@ -5333,7 +5457,7 @@ ctx_mpd_qpow(PyObject *context, PyObject *args, PyObject *kwds) } } - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5368,7 +5492,8 @@ DecCtx_TernaryFunc(mpd_qfma) static PyObject * ctx_mpd_radix(PyObject *context, PyObject *dummy) { - return dec_mpd_radix(context, dummy); + decimal_state *state = get_module_state_from_ctx(context); + return _dec_mpd_radix(state); } /* Boolean functions: single decimal argument */ @@ -5385,7 +5510,7 @@ DecCtx_BoolFunc_NO_CTX(mpd_iszero) static PyObject * ctx_iscanonical(PyObject *context, PyObject *v) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (!PyDec_Check(state, v)) { PyErr_SetString(PyExc_TypeError, "argument must be a Decimal"); @@ -5411,7 +5536,7 @@ PyDecContext_Apply(PyObject *context, PyObject *v) static PyObject * ctx_canonical(PyObject *context, PyObject *v) { - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); if (!PyDec_Check(state, v)) { PyErr_SetString(PyExc_TypeError, "argument must be a Decimal"); @@ -5428,7 +5553,7 @@ ctx_mpd_qcopy_abs(PyObject *context, PyObject *v) uint32_t status = 0; CONVERT_OP_RAISE(&a, v, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5461,7 +5586,7 @@ ctx_mpd_qcopy_negate(PyObject *context, PyObject *v) uint32_t status = 0; CONVERT_OP_RAISE(&a, v, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5558,7 +5683,7 @@ ctx_mpd_qcopy_sign(PyObject *context, PyObject *args) } CONVERT_BINOP_RAISE(&a, &b, v, w, context); - decimal_state *state = get_module_state_by_def(Py_TYPE(context)); + decimal_state *state = get_module_state_from_ctx(context); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5703,9 +5828,9 @@ static PyMethodDef context_methods [] = #endif /* Miscellaneous */ - { "__copy__", (PyCFunction)context_copy, METH_NOARGS, NULL }, + { "__copy__", context_copy, METH_NOARGS, NULL }, { "__reduce__", context_reduce, METH_NOARGS, NULL }, - { "copy", (PyCFunction)context_copy, METH_NOARGS, doc_ctx_copy }, + { "copy", context_copy, METH_NOARGS, doc_ctx_copy }, { "create_decimal", ctx_create_decimal, METH_VARARGS, doc_ctx_create_decimal }, { "create_decimal_from_float", ctx_from_float, METH_O, doc_ctx_create_decimal_from_float }, @@ -5713,6 +5838,7 @@ static PyMethodDef context_methods [] = }; static PyType_Slot context_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, {Py_tp_dealloc, context_dealloc}, {Py_tp_traverse, context_traverse}, {Py_tp_clear, context_clear}, @@ -5738,11 +5864,11 @@ static PyType_Spec context_spec = { static PyMethodDef _decimal_methods [] = { - { "getcontext", (PyCFunction)PyDec_GetCurrentContext, METH_NOARGS, doc_getcontext}, - { "setcontext", (PyCFunction)PyDec_SetCurrentContext, METH_O, doc_setcontext}, + { "getcontext", PyDec_GetCurrentContext, METH_NOARGS, doc_getcontext}, + { "setcontext", PyDec_SetCurrentContext, METH_O, doc_setcontext}, { "localcontext", _PyCFunction_CAST(ctxmanager_new), METH_VARARGS|METH_KEYWORDS, doc_localcontext}, #ifdef EXTRA_FUNCTIONALITY - { "IEEEContext", (PyCFunction)ieee_context, METH_O, doc_ieee_context}, + { "IEEEContext", ieee_context, METH_O, doc_ieee_context}, #endif { NULL, NULL, 1, NULL } }; @@ -6104,6 +6230,16 @@ decimal_traverse(PyObject *module, visitproc visit, void *arg) Py_VISIT(state->Rational); Py_VISIT(state->SignalTuple); + if (state->signal_map != NULL) { + for (DecCondMap *cm = state->signal_map; cm->name != NULL; cm++) { + Py_VISIT(cm->ex); + } + } + if (state->cond_map != NULL) { + for (DecCondMap *cm = state->cond_map + 1; cm->name != NULL; cm++) { + Py_VISIT(cm->ex); + } + } return 0; } @@ -6133,8 +6269,22 @@ decimal_clear(PyObject *module) Py_CLEAR(state->SignalTuple); Py_CLEAR(state->PyDecimal); - PyMem_Free(state->signal_map); - PyMem_Free(state->cond_map); + if (state->signal_map != NULL) { + for (DecCondMap *cm = state->signal_map; cm->name != NULL; cm++) { + Py_DECREF(cm->ex); + } + PyMem_Free(state->signal_map); + state->signal_map = NULL; + } + + if (state->cond_map != NULL) { + // cond_map[0].ex has borrowed a reference from signal_map[0].ex + for (DecCondMap *cm = state->cond_map + 1; cm->name != NULL; cm++) { + Py_DECREF(cm->ex); + } + PyMem_Free(state->cond_map); + state->cond_map = NULL; + } return 0; } @@ -6147,6 +6297,7 @@ decimal_free(void *module) static struct PyModuleDef_Slot _decimal_slots[] = { {Py_mod_exec, _decimal_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_decimal/docstrings.h b/Modules/_decimal/docstrings.h index a1823cdd32b74c..5abd7b9d807e19 100644 --- a/Modules/_decimal/docstrings.h +++ b/Modules/_decimal/docstrings.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001-2012 Python Software Foundation. All Rights Reserved. + * Copyright (c) 2001 Python Software Foundation. All Rights Reserved. * Modified and extended by Stefan Krah. */ @@ -189,6 +189,19 @@ Decimal.from_float(0.1) is not the same as Decimal('0.1').\n\ \n\ \n"); +PyDoc_STRVAR(doc_from_number, +"from_number($type, number, /)\n--\n\n\ +Class method that converts a real number to a decimal number, exactly.\n\ +\n\ + >>> Decimal.from_number(314) # int\n\ + Decimal('314')\n\ + >>> Decimal.from_number(0.1) # float\n\ + Decimal('0.1000000000000000055511151231257827021181583404541015625')\n\ + >>> Decimal.from_number(Decimal('3.14')) # another decimal instance\n\ + Decimal('3.14')\n\ +\n\ +\n"); + PyDoc_STRVAR(doc_fma, "fma($self, /, other, third, context=None)\n--\n\n\ Fused multiply-add. Return self*other+third with no rounding of the\n\ diff --git a/Modules/_decimal/tests/bench.py b/Modules/_decimal/tests/bench.py index 640290f2ec7962..6605e9a92e2dde 100644 --- a/Modules/_decimal/tests/bench.py +++ b/Modules/_decimal/tests/bench.py @@ -1,5 +1,5 @@ # -# Copyright (C) 2001-2012 Python Software Foundation. All Rights Reserved. +# Copyright (C) 2001 Python Software Foundation. All Rights Reserved. # Modified and extended by Stefan Krah. # diff --git a/Modules/_decimal/windows/mpdecimal.h b/Modules/_decimal/windows/mpdecimal.h new file mode 100644 index 00000000000000..77bc6229fbc119 --- /dev/null +++ b/Modules/_decimal/windows/mpdecimal.h @@ -0,0 +1,17 @@ +/* Windows mpdecimal.h shim + * + * Generally, the mpdecimal library build will copy the correct header into + * place named "mpdecimal.h", but since we're building it ourselves directly + * into _decimal.pyd, we need to pick the right one. + * + * */ + +#if defined(_MSC_VER) + #if defined(CONFIG_64) + #include + #elif defined(CONFIG_32) + #include + #else + #error "Unknown configuration!" + #endif +#endif diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index edd2f88a4881c3..355f322d304c2f 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -184,7 +184,7 @@ elementtree_traverse(PyObject *m, visitproc visit, void *arg) static void elementtree_free(void *m) { - elementtree_clear((PyObject *)m); + (void)elementtree_clear((PyObject *)m); } /* helpers */ @@ -196,7 +196,7 @@ list_join(PyObject* list) PyObject* joiner; PyObject* result; - joiner = PyUnicode_FromStringAndSize("", 0); + joiner = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (!joiner) return NULL; result = PyUnicode_Join(joiner, list); @@ -257,6 +257,7 @@ typedef struct { } ElementObject; +#define _Element_CAST(op) ((ElementObject *)(op)) #define Element_CheckExact(st, op) Py_IS_TYPE(op, (st)->Element_Type) #define Element_Check(st, op) PyObject_TypeCheck(op, (st)->Element_Type) @@ -372,33 +373,27 @@ element_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static PyObject* get_attrib_from_keywords(PyObject *kwds) { - PyObject *attrib_str = PyUnicode_FromString("attrib"); - if (attrib_str == NULL) { + PyObject *attrib; + if (PyDict_PopString(kwds, "attrib", &attrib) < 0) { return NULL; } - PyObject *attrib = PyDict_GetItemWithError(kwds, attrib_str); if (attrib) { /* If attrib was found in kwds, copy its value and remove it from * kwds */ if (!PyDict_Check(attrib)) { - Py_DECREF(attrib_str); PyErr_Format(PyExc_TypeError, "attrib must be dict, not %.100s", Py_TYPE(attrib)->tp_name); + Py_DECREF(attrib); return NULL; } - attrib = PyDict_Copy(attrib); - if (attrib && PyDict_DelItem(kwds, attrib_str) < 0) { - Py_SETREF(attrib, NULL); - } + Py_SETREF(attrib, PyDict_Copy(attrib)); } - else if (!PyErr_Occurred()) { + else { attrib = PyDict_New(); } - Py_DECREF(attrib_str); - if (attrib != NULL && PyDict_Update(attrib, kwds) < 0) { Py_DECREF(attrib); return NULL; @@ -654,8 +649,9 @@ subelement(PyObject *self, PyObject *args, PyObject *kwds) } static int -element_gc_traverse(ElementObject *self, visitproc visit, void *arg) +element_gc_traverse(PyObject *op, visitproc visit, void *arg) { + ElementObject *self = _Element_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->tag); Py_VISIT(JOIN_OBJ(self->text)); @@ -672,8 +668,9 @@ element_gc_traverse(ElementObject *self, visitproc visit, void *arg) } static int -element_gc_clear(ElementObject *self) +element_gc_clear(PyObject *op) { + ElementObject *self = _Element_CAST(op); Py_CLEAR(self->tag); _clear_joined_ptr(&self->text); _clear_joined_ptr(&self->tail); @@ -686,8 +683,9 @@ element_gc_clear(ElementObject *self) } static void -element_dealloc(ElementObject* self) +element_dealloc(PyObject *op) { + ElementObject *self = _Element_CAST(op); PyTypeObject *tp = Py_TYPE(self); /* bpo-31095: UnTrack is needed before calling any callbacks */ @@ -695,13 +693,13 @@ element_dealloc(ElementObject* self) Py_TRASHCAN_BEGIN(self, element_dealloc) if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) self); + PyObject_ClearWeakRefs(op); /* element_gc_clear clears all references and deallocates extra */ - element_gc_clear(self); + (void)element_gc_clear(op); - tp->tp_free((PyObject *)self); + tp->tp_free(self); Py_DECREF(tp); Py_TRASHCAN_END } @@ -1219,12 +1217,8 @@ _elementtree_Element_extend_impl(ElementObject *self, PyTypeObject *cls, PyObject* seq; Py_ssize_t i; - seq = PySequence_Fast(elements, ""); + seq = PySequence_Fast(elements, "'elements' must be an iterable"); if (!seq) { - PyErr_Format( - PyExc_TypeError, - "expected sequence, not \"%.200s\"", Py_TYPE(elements)->tp_name - ); return NULL; } @@ -1327,7 +1321,7 @@ _elementtree_Element_findtext_impl(ElementObject *self, PyTypeObject *cls, PyObject* text = element_get_text((ElementObject*)item); if (text == Py_None) { Py_DECREF(item); - return PyUnicode_New(0, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } Py_XINCREF(text); Py_DECREF(item); @@ -1488,9 +1482,9 @@ _elementtree_Element_itertext_impl(ElementObject *self, PyTypeObject *cls) static PyObject* -element_getitem(PyObject* self_, Py_ssize_t index) +element_getitem(PyObject *op, Py_ssize_t index) { - ElementObject* self = (ElementObject*) self_; + ElementObject *self = _Element_CAST(op); if (!self->extra || index < 0 || index >= self->extra->length) { PyErr_SetString( @@ -1504,11 +1498,11 @@ element_getitem(PyObject* self_, Py_ssize_t index) } static int -element_bool(PyObject* self_) +element_bool(PyObject *op) { - ElementObject* self = (ElementObject*) self_; + ElementObject *self = _Element_CAST(op); if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Testing an element's truth value will raise an exception " + "Testing an element's truth value will always return True " "in future versions. Use specific 'len(elem)' or " "'elem is not None' test instead.", 1) < 0) { @@ -1593,8 +1587,9 @@ _elementtree_Element_keys_impl(ElementObject *self) } static Py_ssize_t -element_length(ElementObject* self) +element_length(PyObject *op) { + ElementObject *self = _Element_CAST(op); if (!self->extra) return 0; @@ -1685,10 +1680,10 @@ _elementtree_Element_remove_impl(ElementObject *self, PyObject *subelement) } static PyObject* -element_repr(ElementObject* self) +element_repr(PyObject *op) { int status; - + ElementObject *self = _Element_CAST(op); if (self->tag == NULL) return PyUnicode_FromFormat("", self); @@ -1738,9 +1733,9 @@ _elementtree_Element_set_impl(ElementObject *self, PyObject *key, } static int -element_setitem(PyObject* self_, Py_ssize_t index, PyObject* item) +element_setitem(PyObject *op, Py_ssize_t index, PyObject* item) { - ElementObject* self = (ElementObject*) self_; + ElementObject *self = _Element_CAST(op); Py_ssize_t i; PyObject* old; @@ -1772,10 +1767,10 @@ element_setitem(PyObject* self_, Py_ssize_t index, PyObject* item) return 0; } -static PyObject* -element_subscr(PyObject* self_, PyObject* item) +static PyObject * +element_subscr(PyObject *op, PyObject *item) { - ElementObject* self = (ElementObject*) self_; + ElementObject *self = _Element_CAST(op); if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); @@ -1785,7 +1780,7 @@ element_subscr(PyObject* self_, PyObject* item) } if (i < 0 && self->extra) i += self->extra->length; - return element_getitem(self_, i); + return element_getitem(op, i); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen, i; @@ -1825,9 +1820,9 @@ element_subscr(PyObject* self_, PyObject* item) } static int -element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) +element_ass_subscr(PyObject *op, PyObject *item, PyObject *value) { - ElementObject* self = (ElementObject*) self_; + ElementObject *self = _Element_CAST(op); if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); @@ -1837,7 +1832,7 @@ element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) } if (i < 0 && self->extra) i += self->extra->length; - return element_setitem(self_, i, value); + return element_setitem(op, i, value); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen, newlen, i; @@ -1924,12 +1919,8 @@ element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) } /* A new slice is actually being assigned */ - seq = PySequence_Fast(value, ""); + seq = PySequence_Fast(value, "assignment expects an iterable"); if (!seq) { - PyErr_Format( - PyExc_TypeError, - "expected sequence, not \"%.200s\"", Py_TYPE(value)->tp_name - ); return -1; } newlen = PySequence_Fast_GET_SIZE(seq); @@ -2012,30 +2003,34 @@ element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) } static PyObject* -element_tag_getter(ElementObject *self, void *closure) +element_tag_getter(PyObject *op, void *closure) { + ElementObject *self = _Element_CAST(op); PyObject *res = self->tag; return Py_NewRef(res); } static PyObject* -element_text_getter(ElementObject *self, void *closure) +element_text_getter(PyObject *op, void *closure) { + ElementObject *self = _Element_CAST(op); PyObject *res = element_get_text(self); return Py_XNewRef(res); } static PyObject* -element_tail_getter(ElementObject *self, void *closure) +element_tail_getter(PyObject *op, void *closure) { + ElementObject *self = _Element_CAST(op); PyObject *res = element_get_tail(self); return Py_XNewRef(res); } static PyObject* -element_attrib_getter(ElementObject *self, void *closure) +element_attrib_getter(PyObject *op, void *closure) { PyObject *res; + ElementObject *self = _Element_CAST(op); if (!self->extra) { if (create_extra(self, NULL) < 0) return NULL; @@ -2054,31 +2049,34 @@ element_attrib_getter(ElementObject *self, void *closure) } static int -element_tag_setter(ElementObject *self, PyObject *value, void *closure) +element_tag_setter(PyObject *op, PyObject *value, void *closure) { _VALIDATE_ATTR_VALUE(value); + ElementObject *self = _Element_CAST(op); Py_SETREF(self->tag, Py_NewRef(value)); return 0; } static int -element_text_setter(ElementObject *self, PyObject *value, void *closure) +element_text_setter(PyObject *op, PyObject *value, void *closure) { _VALIDATE_ATTR_VALUE(value); + ElementObject *self = _Element_CAST(op); _set_joined_ptr(&self->text, Py_NewRef(value)); return 0; } static int -element_tail_setter(ElementObject *self, PyObject *value, void *closure) +element_tail_setter(PyObject *op, PyObject *value, void *closure) { _VALIDATE_ATTR_VALUE(value); + ElementObject *self = _Element_CAST(op); _set_joined_ptr(&self->tail, Py_NewRef(value)); return 0; } static int -element_attrib_setter(ElementObject *self, PyObject *value, void *closure) +element_attrib_setter(PyObject *op, PyObject *value, void *closure) { _VALIDATE_ATTR_VALUE(value); if (!PyDict_Check(value)) { @@ -2087,6 +2085,7 @@ element_attrib_setter(ElementObject *self, PyObject *value, void *closure) Py_TYPE(value)->tp_name); return -1; } + ElementObject *self = _Element_CAST(op); if (!self->extra) { if (create_extra(self, NULL) < 0) return -1; @@ -2120,11 +2119,14 @@ typedef struct { int gettext; } ElementIterObject; +#define _ElementIter_CAST(op) ((ElementIterObject *)(op)) + static void -elementiter_dealloc(ElementIterObject *it) +elementiter_dealloc(PyObject *op) { - PyTypeObject *tp = Py_TYPE(it); + PyTypeObject *tp = Py_TYPE(op); + ElementIterObject *it = _ElementIter_CAST(op); Py_ssize_t i = it->parent_stack_used; it->parent_stack_used = 0; /* bpo-31095: UnTrack is needed before calling any callbacks */ @@ -2141,8 +2143,9 @@ elementiter_dealloc(ElementIterObject *it) } static int -elementiter_traverse(ElementIterObject *it, visitproc visit, void *arg) +elementiter_traverse(PyObject *op, visitproc visit, void *arg) { + ElementIterObject *it = _ElementIter_CAST(op); Py_ssize_t i = it->parent_stack_used; while (i--) Py_VISIT(it->parent_stack[i].parent); @@ -2176,8 +2179,9 @@ parent_stack_push_new(ElementIterObject *it, ElementObject *parent) } static PyObject * -elementiter_next(ElementIterObject *it) +elementiter_next(PyObject *op) { + ElementIterObject *it = _ElementIter_CAST(op); /* Sub-element iterator. * * A short note on gettext: this function serves both the iter() and @@ -2368,6 +2372,8 @@ typedef struct { elementtreestate *state; } TreeBuilderObject; + +#define _TreeBuilder_CAST(op) ((TreeBuilderObject *)(op)) #define TreeBuilder_CheckExact(st, op) Py_IS_TYPE((op), (st)->TreeBuilder_Type) /* -------------------------------------------------------------------- */ @@ -2458,8 +2464,9 @@ _elementtree_TreeBuilder___init___impl(TreeBuilderObject *self, } static int -treebuilder_gc_traverse(TreeBuilderObject *self, visitproc visit, void *arg) +treebuilder_gc_traverse(PyObject *op, visitproc visit, void *arg) { + TreeBuilderObject *self = _TreeBuilder_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->pi_event_obj); Py_VISIT(self->comment_event_obj); @@ -2481,8 +2488,9 @@ treebuilder_gc_traverse(TreeBuilderObject *self, visitproc visit, void *arg) } static int -treebuilder_gc_clear(TreeBuilderObject *self) +treebuilder_gc_clear(PyObject *op) { + TreeBuilderObject *self = _TreeBuilder_CAST(op); Py_CLEAR(self->pi_event_obj); Py_CLEAR(self->comment_event_obj); Py_CLEAR(self->end_ns_event_obj); @@ -2503,11 +2511,11 @@ treebuilder_gc_clear(TreeBuilderObject *self) } static void -treebuilder_dealloc(TreeBuilderObject *self) +treebuilder_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - treebuilder_gc_clear(self); + (void)treebuilder_gc_clear(self); tp->tp_free(self); Py_DECREF(tp); } @@ -3075,6 +3083,9 @@ typedef struct { PyObject *elementtree_module; } XMLParserObject; + +#define _XMLParser_CAST(op) ((XMLParserObject *)(op)) + /* helpers */ LOCAL(PyObject*) @@ -3765,8 +3776,9 @@ _elementtree_XMLParser___init___impl(XMLParserObject *self, PyObject *target, } static int -xmlparser_gc_traverse(XMLParserObject *self, visitproc visit, void *arg) +xmlparser_gc_traverse(PyObject *op, visitproc visit, void *arg) { + XMLParserObject *self = _XMLParser_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->handle_close); Py_VISIT(self->handle_pi); @@ -3786,8 +3798,9 @@ xmlparser_gc_traverse(XMLParserObject *self, visitproc visit, void *arg) } static int -xmlparser_gc_clear(XMLParserObject *self) +xmlparser_gc_clear(PyObject *op) { + XMLParserObject *self = _XMLParser_CAST(op); elementtreestate *st = self->state; if (self->parser != NULL) { XML_Parser parser = self->parser; @@ -3814,11 +3827,11 @@ xmlparser_gc_clear(XMLParserObject *self) } static void -xmlparser_dealloc(XMLParserObject* self) +xmlparser_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - xmlparser_gc_clear(self); + (void)xmlparser_gc_clear(self); tp->tp_free(self); Py_DECREF(tp); } @@ -4186,7 +4199,7 @@ static PyMemberDef xmlparser_members[] = { }; static PyObject* -xmlparser_version_getter(XMLParserObject *self, void *closure) +xmlparser_version_getter(PyObject *op, void *closure) { return PyUnicode_FromFormat( "Expat %d.%d.%d", XML_MAJOR_VERSION, @@ -4194,7 +4207,7 @@ xmlparser_version_getter(XMLParserObject *self, void *closure) } static PyGetSetDef xmlparser_getsetlist[] = { - {"version", (getter)xmlparser_version_getter, NULL, NULL}, + {"version", xmlparser_version_getter, NULL, NULL}, {NULL}, }; @@ -4243,20 +4256,20 @@ static struct PyMemberDef element_members[] = { static PyGetSetDef element_getsetlist[] = { {"tag", - (getter)element_tag_getter, - (setter)element_tag_setter, + element_tag_getter, + element_tag_setter, "A string identifying what kind of data this element represents"}, {"text", - (getter)element_text_getter, - (setter)element_text_setter, + element_text_getter, + element_text_setter, "A string of text directly after the start tag, or None"}, {"tail", - (getter)element_tail_getter, - (setter)element_tail_setter, + element_tail_getter, + element_tail_setter, "A string of text directly after the end tag, or None"}, {"attrib", - (getter)element_attrib_getter, - (setter)element_attrib_setter, + element_attrib_getter, + element_attrib_setter, "A dictionary containing the element's attributes"}, {NULL}, }; @@ -4469,6 +4482,7 @@ module_exec(PyObject *m) static struct PyModuleDef_Slot elementtree_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index d2212d40550a3c..24b38063dde9e5 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -18,13 +18,15 @@ class _functools._lru_cache_wrapper "PyObject *" "&lru_cache_type_spec" /* _functools module written and maintained by Hye-Shik Chang with adaptations by Raymond Hettinger - Copyright (c) 2004, 2005, 2006 Python Software Foundation. + Copyright (c) 2004 Python Software Foundation. All rights reserved. */ typedef struct _functools_state { /* this object is used delimit args and keywords in the cache keys */ PyObject *kwd_mark; + PyTypeObject *placeholder_type; + PyObject *placeholder; // strong reference (singleton) PyTypeObject *partial_type; PyTypeObject *keyobject_type; PyTypeObject *lru_list_elem_type; @@ -41,6 +43,95 @@ get_functools_state(PyObject *module) /* partial object **********************************************************/ + +// The 'Placeholder' singleton indicates which formal positional +// parameters are to be bound first when using a 'partial' object. + +typedef struct { + PyObject_HEAD +} placeholderobject; + +static inline _functools_state * +get_functools_state_by_type(PyTypeObject *type); + +PyDoc_STRVAR(placeholder_doc, +"The type of the Placeholder singleton.\n\n" +"Used as a placeholder for partial arguments."); + +static PyObject * +placeholder_repr(PyObject *op) +{ + return PyUnicode_FromString("Placeholder"); +} + +static PyObject * +placeholder_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("Placeholder"); +} + +static PyMethodDef placeholder_methods[] = { + {"__reduce__", placeholder_reduce, METH_NOARGS, NULL}, + {NULL, NULL} +}; + +static void +placeholder_dealloc(PyObject* self) +{ + PyObject_GC_UnTrack(self); + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free((PyObject*)self); + Py_DECREF(tp); +} + +static PyObject * +placeholder_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + if (PyTuple_GET_SIZE(args) || (kwargs && PyDict_GET_SIZE(kwargs))) { + PyErr_SetString(PyExc_TypeError, "PlaceholderType takes no arguments"); + return NULL; + } + _functools_state *state = get_functools_state_by_type(type); + if (state->placeholder != NULL) { + return Py_NewRef(state->placeholder); + } + + PyObject *placeholder = PyType_GenericNew(type, NULL, NULL); + if (placeholder == NULL) { + return NULL; + } + + if (state->placeholder == NULL) { + state->placeholder = Py_NewRef(placeholder); + } + return placeholder; +} + +static int +placeholder_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + +static PyType_Slot placeholder_type_slots[] = { + {Py_tp_dealloc, placeholder_dealloc}, + {Py_tp_repr, placeholder_repr}, + {Py_tp_doc, (void *)placeholder_doc}, + {Py_tp_methods, placeholder_methods}, + {Py_tp_new, placeholder_new}, + {Py_tp_traverse, placeholder_traverse}, + {0, 0} +}; + +static PyType_Spec placeholder_type_spec = { + .name = "functools._PlaceholderType", + .basicsize = sizeof(placeholderobject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC, + .slots = placeholder_type_slots +}; + + typedef struct { PyObject_HEAD PyObject *fn; @@ -48,13 +139,18 @@ typedef struct { PyObject *kw; PyObject *dict; /* __dict__ */ PyObject *weakreflist; /* List of weak references */ + PyObject *placeholder; /* Placeholder for positional arguments */ + Py_ssize_t phcount; /* Number of placeholders */ vectorcallfunc vectorcall; } partialobject; +// cast a PyObject pointer PTR to a partialobject pointer (no type checks) +#define _PyPartialObject_CAST(PTR) ((partialobject *)(PTR)) + static void partial_setvectorcall(partialobject *pto); static struct PyModuleDef _functools_module; static PyObject * -partial_call(partialobject *pto, PyObject *args, PyObject *kwargs); +partial_call(PyObject *pto, PyObject *args, PyObject *kwargs); static inline _functools_state * get_functools_state_by_type(PyTypeObject *type) @@ -70,36 +166,54 @@ get_functools_state_by_type(PyTypeObject *type) static PyObject * partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - PyObject *func, *pargs, *nargs, *pkw; + PyObject *func, *pto_args, *new_args, *pto_kw, *phold; partialobject *pto; + Py_ssize_t pto_phcount = 0; + Py_ssize_t new_nargs = PyTuple_GET_SIZE(args) - 1; - if (PyTuple_GET_SIZE(args) < 1) { + if (new_nargs < 0) { PyErr_SetString(PyExc_TypeError, "type 'partial' takes at least one argument"); return NULL; } - - pargs = pkw = NULL; func = PyTuple_GET_ITEM(args, 0); - if (Py_TYPE(func)->tp_call == (ternaryfunc)partial_call) { - // The type of "func" might not be exactly the same type object - // as "type", but if it is called using partial_call, it must have the - // same memory layout (fn, args and kw members). + if (!PyCallable_Check(func)) { + PyErr_SetString(PyExc_TypeError, + "the first argument must be callable"); + return NULL; + } + + _functools_state *state = get_functools_state_by_type(type); + if (state == NULL) { + return NULL; + } + phold = state->placeholder; + + /* Placeholder restrictions */ + if (new_nargs && PyTuple_GET_ITEM(args, new_nargs) == phold) { + PyErr_SetString(PyExc_TypeError, + "trailing Placeholders are not allowed"); + return NULL; + } + + /* check wrapped function / object */ + pto_args = pto_kw = NULL; + int res = PyObject_TypeCheck(func, state->partial_type); + if (res == -1) { + return NULL; + } + if (res == 1) { // We can use its underlying function directly and merge the arguments. partialobject *part = (partialobject *)func; if (part->dict == NULL) { - pargs = part->args; - pkw = part->kw; + pto_args = part->args; + pto_kw = part->kw; func = part->fn; - assert(PyTuple_Check(pargs)); - assert(PyDict_Check(pkw)); + pto_phcount = part->phcount; + assert(PyTuple_Check(pto_args)); + assert(PyDict_Check(pto_kw)); } } - if (!PyCallable_Check(func)) { - PyErr_SetString(PyExc_TypeError, - "the first argument must be callable"); - return NULL; - } /* create partialobject structure */ pto = (partialobject *)type->tp_alloc(type, 0); @@ -107,18 +221,58 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; pto->fn = Py_NewRef(func); + pto->placeholder = phold; - nargs = PyTuple_GetSlice(args, 1, PY_SSIZE_T_MAX); - if (nargs == NULL) { + new_args = PyTuple_GetSlice(args, 1, new_nargs + 1); + if (new_args == NULL) { Py_DECREF(pto); return NULL; } - if (pargs == NULL) { - pto->args = nargs; + + /* Count placeholders */ + Py_ssize_t phcount = 0; + for (Py_ssize_t i = 0; i < new_nargs - 1; i++) { + if (PyTuple_GET_ITEM(new_args, i) == phold) { + phcount++; + } + } + /* merge args with args of `func` which is `partial` */ + if (pto_phcount > 0 && new_nargs > 0) { + Py_ssize_t npargs = PyTuple_GET_SIZE(pto_args); + Py_ssize_t tot_nargs = npargs; + if (new_nargs > pto_phcount) { + tot_nargs += new_nargs - pto_phcount; + } + PyObject *item; + PyObject *tot_args = PyTuple_New(tot_nargs); + for (Py_ssize_t i = 0, j = 0; i < tot_nargs; i++) { + if (i < npargs) { + item = PyTuple_GET_ITEM(pto_args, i); + if (j < new_nargs && item == phold) { + item = PyTuple_GET_ITEM(new_args, j); + j++; + pto_phcount--; + } + } + else { + item = PyTuple_GET_ITEM(new_args, j); + j++; + } + Py_INCREF(item); + PyTuple_SET_ITEM(tot_args, i, item); + } + pto->args = tot_args; + pto->phcount = pto_phcount + phcount; + Py_DECREF(new_args); + } + else if (pto_args == NULL) { + pto->args = new_args; + pto->phcount = phcount; } else { - pto->args = PySequence_Concat(pargs, nargs); - Py_DECREF(nargs); + pto->args = PySequence_Concat(pto_args, new_args); + pto->phcount = pto_phcount + phcount; + Py_DECREF(new_args); if (pto->args == NULL) { Py_DECREF(pto); return NULL; @@ -126,7 +280,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) assert(PyTuple_Check(pto->args)); } - if (pkw == NULL || PyDict_GET_SIZE(pkw) == 0) { + if (pto_kw == NULL || PyDict_GET_SIZE(pto_kw) == 0) { if (kw == NULL) { pto->kw = PyDict_New(); } @@ -138,7 +292,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) } } else { - pto->kw = PyDict_Copy(pkw); + pto->kw = PyDict_Copy(pto_kw); if (kw != NULL && pto->kw != NULL) { if (PyDict_Merge(pto->kw, kw, 1) != 0) { Py_DECREF(pto); @@ -156,8 +310,9 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) } static int -partial_clear(partialobject *pto) +partial_clear(PyObject *self) { + partialobject *pto = _PyPartialObject_CAST(self); Py_CLEAR(pto->fn); Py_CLEAR(pto->args); Py_CLEAR(pto->kw); @@ -166,8 +321,9 @@ partial_clear(partialobject *pto) } static int -partial_traverse(partialobject *pto, visitproc visit, void *arg) +partial_traverse(PyObject *self, visitproc visit, void *arg) { + partialobject *pto = _PyPartialObject_CAST(self); Py_VISIT(Py_TYPE(pto)); Py_VISIT(pto->fn); Py_VISIT(pto->args); @@ -177,19 +333,27 @@ partial_traverse(partialobject *pto, visitproc visit, void *arg) } static void -partial_dealloc(partialobject *pto) +partial_dealloc(PyObject *self) { - PyTypeObject *tp = Py_TYPE(pto); + PyTypeObject *tp = Py_TYPE(self); /* bpo-31095: UnTrack is needed before calling any callbacks */ - PyObject_GC_UnTrack(pto); - if (pto->weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *) pto); + PyObject_GC_UnTrack(self); + if (_PyPartialObject_CAST(self)->weakreflist != NULL) { + PyObject_ClearWeakRefs(self); } - (void)partial_clear(pto); - tp->tp_free(pto); + (void)partial_clear(self); + tp->tp_free(self); Py_DECREF(tp); } +static PyObject * +partial_descr_get(PyObject *self, PyObject *obj, PyObject *type) +{ + if (obj == Py_None || obj == NULL) { + return Py_NewRef(self); + } + return PyMethod_New(self, obj); +} /* Merging keyword arguments using the vectorcall convention is messy, so * if we would need to do that, we stop using vectorcall and fall back @@ -201,32 +365,39 @@ partial_vectorcall_fallback(PyThreadState *tstate, partialobject *pto, { pto->vectorcall = NULL; Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - return _PyObject_MakeTpCall(tstate, (PyObject *)pto, - args, nargs, kwnames); + return _PyObject_MakeTpCall(tstate, (PyObject *)pto, args, nargs, kwnames); } static PyObject * -partial_vectorcall(partialobject *pto, PyObject *const *args, +partial_vectorcall(PyObject *self, PyObject *const *args, size_t nargsf, PyObject *kwnames) { + partialobject *pto = _PyPartialObject_CAST(self);; PyThreadState *tstate = _PyThreadState_GET(); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); /* pto->kw is mutable, so need to check every time */ if (PyDict_GET_SIZE(pto->kw)) { return partial_vectorcall_fallback(tstate, pto, args, nargsf, kwnames); } + Py_ssize_t pto_phcount = pto->phcount; + if (nargs < pto_phcount) { + PyErr_Format(PyExc_TypeError, + "missing positional arguments in 'partial' call; " + "expected at least %zd, got %zd", pto_phcount, nargs); + return NULL; + } - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - Py_ssize_t nargs_total = nargs; + Py_ssize_t nargskw = nargs; if (kwnames != NULL) { - nargs_total += PyTuple_GET_SIZE(kwnames); + nargskw += PyTuple_GET_SIZE(kwnames); } PyObject **pto_args = _PyTuple_ITEMS(pto->args); Py_ssize_t pto_nargs = PyTuple_GET_SIZE(pto->args); /* Fast path if we're called without arguments */ - if (nargs_total == 0) { + if (nargskw == 0) { return _PyObject_VectorcallTstate(tstate, pto->fn, pto_args, pto_nargs, NULL); } @@ -243,29 +414,47 @@ partial_vectorcall(partialobject *pto, PyObject *const *args, return ret; } - Py_ssize_t newnargs_total = pto_nargs + nargs_total; - PyObject *small_stack[_PY_FASTCALL_SMALL_STACK]; - PyObject *ret; PyObject **stack; - if (newnargs_total <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { + Py_ssize_t tot_nargskw = pto_nargs + nargskw - pto_phcount; + if (tot_nargskw <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { stack = small_stack; } else { - stack = PyMem_Malloc(newnargs_total * sizeof(PyObject *)); + stack = PyMem_Malloc(tot_nargskw * sizeof(PyObject *)); if (stack == NULL) { PyErr_NoMemory(); return NULL; } } - /* Copy to new stack, using borrowed references */ - memcpy(stack, pto_args, pto_nargs * sizeof(PyObject*)); - memcpy(stack + pto_nargs, args, nargs_total * sizeof(PyObject*)); - - ret = _PyObject_VectorcallTstate(tstate, pto->fn, - stack, pto_nargs + nargs, kwnames); + Py_ssize_t tot_nargs; + if (pto_phcount) { + tot_nargs = pto_nargs + nargs - pto_phcount; + Py_ssize_t j = 0; // New args index + for (Py_ssize_t i = 0; i < pto_nargs; i++) { + if (pto_args[i] == pto->placeholder) { + stack[i] = args[j]; + j += 1; + } + else { + stack[i] = pto_args[i]; + } + } + assert(j == pto_phcount); + if (nargskw > pto_phcount) { + memcpy(stack + pto_nargs, args + j, (nargskw - j) * sizeof(PyObject*)); + } + } + else { + tot_nargs = pto_nargs + nargs; + /* Copy to new stack, using borrowed references */ + memcpy(stack, pto_args, pto_nargs * sizeof(PyObject*)); + memcpy(stack + pto_nargs, args, nargskw * sizeof(PyObject*)); + } + PyObject *ret = _PyObject_VectorcallTstate(tstate, pto->fn, + stack, tot_nargs, kwnames); if (stack != small_stack) { PyMem_Free(stack); } @@ -284,59 +473,102 @@ partial_setvectorcall(partialobject *pto) * but that is unlikely (why use partial without arguments?), * so we don't optimize that */ else { - pto->vectorcall = (vectorcallfunc)partial_vectorcall; + pto->vectorcall = partial_vectorcall; } } // Not converted to argument clinic, because of `*args, **kwargs` arguments. static PyObject * -partial_call(partialobject *pto, PyObject *args, PyObject *kwargs) +partial_call(PyObject *self, PyObject *args, PyObject *kwargs) { + partialobject *pto = _PyPartialObject_CAST(self); assert(PyCallable_Check(pto->fn)); assert(PyTuple_Check(pto->args)); assert(PyDict_Check(pto->kw)); + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t pto_phcount = pto->phcount; + if (nargs < pto_phcount) { + PyErr_Format(PyExc_TypeError, + "missing positional arguments in 'partial' call; " + "expected at least %zd, got %zd", pto_phcount, nargs); + return NULL; + } + /* Merge keywords */ - PyObject *kwargs2; + PyObject *tot_kw; if (PyDict_GET_SIZE(pto->kw) == 0) { /* kwargs can be NULL */ - kwargs2 = Py_XNewRef(kwargs); + tot_kw = Py_XNewRef(kwargs); } else { /* bpo-27840, bpo-29318: dictionary of keyword parameters must be copied, because a function using "**kwargs" can modify the dictionary. */ - kwargs2 = PyDict_Copy(pto->kw); - if (kwargs2 == NULL) { + tot_kw = PyDict_Copy(pto->kw); + if (tot_kw == NULL) { return NULL; } if (kwargs != NULL) { - if (PyDict_Merge(kwargs2, kwargs, 1) != 0) { - Py_DECREF(kwargs2); + if (PyDict_Merge(tot_kw, kwargs, 1) != 0) { + Py_DECREF(tot_kw); return NULL; } } } /* Merge positional arguments */ - /* Note: tupleconcat() is optimized for empty tuples */ - PyObject *args2 = PySequence_Concat(pto->args, args); - if (args2 == NULL) { - Py_XDECREF(kwargs2); - return NULL; + PyObject *tot_args; + if (pto_phcount) { + Py_ssize_t pto_nargs = PyTuple_GET_SIZE(pto->args); + Py_ssize_t tot_nargs = pto_nargs + nargs - pto_phcount; + assert(tot_nargs >= 0); + tot_args = PyTuple_New(tot_nargs); + if (tot_args == NULL) { + Py_XDECREF(tot_kw); + return NULL; + } + PyObject *pto_args = pto->args; + PyObject *item; + Py_ssize_t j = 0; // New args index + for (Py_ssize_t i = 0; i < pto_nargs; i++) { + item = PyTuple_GET_ITEM(pto_args, i); + if (item == pto->placeholder) { + item = PyTuple_GET_ITEM(args, j); + j += 1; + } + Py_INCREF(item); + PyTuple_SET_ITEM(tot_args, i, item); + } + assert(j == pto_phcount); + for (Py_ssize_t i = pto_nargs; i < tot_nargs; i++) { + item = PyTuple_GET_ITEM(args, j); + Py_INCREF(item); + PyTuple_SET_ITEM(tot_args, i, item); + j += 1; + } + } + else { + /* Note: tupleconcat() is optimized for empty tuples */ + tot_args = PySequence_Concat(pto->args, args); + if (tot_args == NULL) { + Py_XDECREF(tot_kw); + return NULL; + } } - PyObject *res = PyObject_Call(pto->fn, args2, kwargs2); - Py_DECREF(args2); - Py_XDECREF(kwargs2); + PyObject *res = PyObject_Call(pto->fn, tot_args, tot_kw); + Py_DECREF(tot_args); + Py_XDECREF(tot_kw); return res; } PyDoc_STRVAR(partial_doc, -"partial(func, *args, **keywords) - new function with partial application\n\ - of the given arguments and keywords.\n"); +"partial(func, /, *args, **keywords)\n--\n\n\ +Create a new function with partial application of the given arguments\n\ +and keywords."); #define OFF(x) offsetof(partialobject, x) static PyMemberDef partial_memberlist[] = { @@ -361,8 +593,9 @@ static PyGetSetDef partial_getsetlist[] = { }; static PyObject * -partial_repr(partialobject *pto) +partial_repr(PyObject *self) { + partialobject *pto = _PyPartialObject_CAST(self); PyObject *result = NULL; PyObject *arglist; PyObject *mod; @@ -371,18 +604,18 @@ partial_repr(partialobject *pto) PyObject *key, *value; int status; - status = Py_ReprEnter((PyObject *)pto); + status = Py_ReprEnter(self); if (status != 0) { if (status < 0) return NULL; return PyUnicode_FromString("..."); } - arglist = PyUnicode_FromString(""); + arglist = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (arglist == NULL) goto done; /* Pack positional arguments */ - assert (PyTuple_Check(pto->args)); + assert(PyTuple_Check(pto->args)); n = PyTuple_GET_SIZE(pto->args); for (i = 0; i < n; i++) { Py_SETREF(arglist, PyUnicode_FromFormat("%U, %R", arglist, @@ -402,7 +635,7 @@ partial_repr(partialobject *pto) goto done; } - mod = _PyType_GetModuleName(Py_TYPE(pto)); + mod = PyType_GetModuleName(Py_TYPE(pto)); if (mod == NULL) { goto error; } @@ -417,11 +650,11 @@ partial_repr(partialobject *pto) Py_DECREF(arglist); done: - Py_ReprLeave((PyObject *)pto); + Py_ReprLeave(self); return result; error: Py_DECREF(arglist); - Py_ReprLeave((PyObject *)pto); + Py_ReprLeave(self); return NULL; } @@ -433,20 +666,25 @@ partial_repr(partialobject *pto) */ static PyObject * -partial_reduce(partialobject *pto, PyObject *unused) +partial_reduce(PyObject *self, PyObject *Py_UNUSED(args)) { + partialobject *pto = _PyPartialObject_CAST(self); return Py_BuildValue("O(O)(OOOO)", Py_TYPE(pto), pto->fn, pto->fn, pto->args, pto->kw, pto->dict ? pto->dict : Py_None); } static PyObject * -partial_setstate(partialobject *pto, PyObject *state) +partial_setstate(PyObject *self, PyObject *state) { + partialobject *pto = _PyPartialObject_CAST(self); PyObject *fn, *fnargs, *kw, *dict; - if (!PyTuple_Check(state) || - !PyArg_ParseTuple(state, "OOOO", &fn, &fnargs, &kw, &dict) || + if (!PyTuple_Check(state)) { + PyErr_SetString(PyExc_TypeError, "invalid partial state"); + return NULL; + } + if (!PyArg_ParseTuple(state, "OOOO", &fn, &fnargs, &kw, &dict) || !PyCallable_Check(fn) || !PyTuple_Check(fnargs) || (kw != Py_None && !PyDict_Check(kw))) @@ -455,6 +693,20 @@ partial_setstate(partialobject *pto, PyObject *state) return NULL; } + Py_ssize_t nargs = PyTuple_GET_SIZE(fnargs); + if (nargs && PyTuple_GET_ITEM(fnargs, nargs - 1) == pto->placeholder) { + PyErr_SetString(PyExc_TypeError, + "trailing Placeholders are not allowed"); + return NULL; + } + /* Count placeholders */ + Py_ssize_t phcount = 0; + for (Py_ssize_t i = 0; i < nargs - 1; i++) { + if (PyTuple_GET_ITEM(fnargs, i) == pto->placeholder) { + phcount++; + } + } + if(!PyTuple_CheckExact(fnargs)) fnargs = PySequence_Tuple(fnargs); else @@ -477,18 +729,18 @@ partial_setstate(partialobject *pto, PyObject *state) dict = NULL; else Py_INCREF(dict); - Py_SETREF(pto->fn, Py_NewRef(fn)); Py_SETREF(pto->args, fnargs); Py_SETREF(pto->kw, kw); + pto->phcount = phcount; Py_XSETREF(pto->dict, dict); partial_setvectorcall(pto); Py_RETURN_NONE; } static PyMethodDef partial_methods[] = { - {"__reduce__", (PyCFunction)partial_reduce, METH_NOARGS}, - {"__setstate__", (PyCFunction)partial_setstate, METH_O}, + {"__reduce__", partial_reduce, METH_NOARGS}, + {"__setstate__", partial_setstate, METH_O}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ @@ -506,6 +758,7 @@ static PyType_Slot partial_type_slots[] = { {Py_tp_methods, partial_methods}, {Py_tp_members, partial_memberlist}, {Py_tp_getset, partial_getsetlist}, + {Py_tp_descr_get, partial_descr_get}, {Py_tp_new, partial_new}, {Py_tp_free, PyObject_GC_Del}, {0, 0} @@ -563,6 +816,17 @@ static PyMemberDef keyobject_members[] = { {NULL} }; +static PyObject * +keyobject_text_signature(PyObject *self, void *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("(obj)"); +} + +static PyGetSetDef keyobject_getset[] = { + {"__text_signature__", keyobject_text_signature, (setter)NULL}, + {NULL} +}; + static PyObject * keyobject_call(keyobject *ko, PyObject *args, PyObject *kwds); @@ -577,6 +841,7 @@ static PyType_Slot keyobject_type_slots[] = { {Py_tp_clear, keyobject_clear}, {Py_tp_richcompare, keyobject_richcompare}, {Py_tp_members, keyobject_members}, + {Py_tp_getset, keyobject_getset}, {0, 0} }; @@ -667,15 +932,31 @@ _functools_cmp_to_key_impl(PyObject *module, PyObject *mycmp) /* reduce (used to be a builtin) ********************************************/ -// Not converted to argument clinic, because of `args` in-place modification. -// AC will affect performance. +/*[clinic input] +_functools.reduce + + function as func: object + iterable as seq: object + / + initial as result: object = NULL + +Apply a function of two arguments cumulatively to the items of an iterable, from left to right. + +This effectively reduces the iterable to a single value. If initial is present, +it is placed before the items of the iterable in the calculation, and serves as +a default when the iterable is empty. + +For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) +calculates ((((1 + 2) + 3) + 4) + 5). +[clinic start generated code]*/ + static PyObject * -functools_reduce(PyObject *self, PyObject *args) +_functools_reduce_impl(PyObject *module, PyObject *func, PyObject *seq, + PyObject *result) +/*[clinic end generated code: output=30d898fe1267c79d input=1511e9a8c38581ac]*/ { - PyObject *seq, *func, *result = NULL, *it; + PyObject *args, *it; - if (!PyArg_UnpackTuple(args, "reduce", 2, 3, &func, &seq, &result)) - return NULL; if (result != NULL) Py_INCREF(result); @@ -741,16 +1022,6 @@ functools_reduce(PyObject *self, PyObject *args) return NULL; } -PyDoc_STRVAR(functools_reduce_doc, -"reduce(function, iterable[, initial], /) -> value\n\ -\n\ -Apply a function of two arguments cumulatively to the items of a sequence\n\ -or iterable, from left to right, so as to reduce the iterable to a single\n\ -value. For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates\n\ -((((1+2)+3)+4)+5). If initial is present, it is placed before the items\n\ -of the iterable in the calculation, and serves as a default when the\n\ -iterable is empty."); - /* lru_cache object **********************************************************/ /* There are four principal algorithmic differences from the pure python version: @@ -1455,7 +1726,7 @@ PyDoc_STRVAR(_functools_doc, "Tools that operate on functions."); static PyMethodDef _functools_methods[] = { - {"reduce", functools_reduce, METH_VARARGS, functools_reduce_doc}, + _FUNCTOOLS_REDUCE_METHODDEF _FUNCTOOLS_CMP_TO_KEY_METHODDEF {NULL, NULL} /* sentinel */ }; @@ -1469,6 +1740,25 @@ _functools_exec(PyObject *module) return -1; } + state->placeholder_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, + &placeholder_type_spec, NULL); + if (state->placeholder_type == NULL) { + return -1; + } + if (PyModule_AddType(module, state->placeholder_type) < 0) { + return -1; + } + + PyObject *placeholder = PyObject_CallNoArgs((PyObject *)state->placeholder_type); + if (placeholder == NULL) { + return -1; + } + if (PyModule_AddObjectRef(module, "Placeholder", placeholder) < 0) { + Py_DECREF(placeholder); + return -1; + } + Py_DECREF(placeholder); + state->partial_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, &partial_type_spec, NULL); if (state->partial_type == NULL) { @@ -1513,6 +1803,8 @@ _functools_traverse(PyObject *module, visitproc visit, void *arg) { _functools_state *state = get_functools_state(module); Py_VISIT(state->kwd_mark); + Py_VISIT(state->placeholder_type); + Py_VISIT(state->placeholder); Py_VISIT(state->partial_type); Py_VISIT(state->keyobject_type); Py_VISIT(state->lru_list_elem_type); @@ -1524,6 +1816,8 @@ _functools_clear(PyObject *module) { _functools_state *state = get_functools_state(module); Py_CLEAR(state->kwd_mark); + Py_CLEAR(state->placeholder_type); + Py_CLEAR(state->placeholder); Py_CLEAR(state->partial_type); Py_CLEAR(state->keyobject_type); Py_CLEAR(state->lru_list_elem_type); @@ -1539,6 +1833,7 @@ _functools_free(void *module) static struct PyModuleDef_Slot _functools_slots[] = { {Py_mod_exec, _functools_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index db868c18160fda..ab2ebdba9249bf 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -8,10 +8,11 @@ #endif #include "Python.h" +#include "pycore_pyerrors.h" // _PyErr_SetLocaleString() #include "gdbm.h" #include -#include // free() +#include // free() #include #include @@ -33,6 +34,24 @@ get_gdbm_state(PyObject *module) return (_gdbm_state *)state; } +/* + * Set the gdbm error obtained by gdbm_strerror(gdbm_errno). + * + * If no error message exists, a generic (UTF-8) error message + * is used instead. + */ +static void +set_gdbm_error(_gdbm_state *state, const char *generic_error) +{ + const char *gdbm_errmsg = gdbm_strerror(gdbm_errno); + if (gdbm_errmsg) { + _PyErr_SetLocaleString(state->gdbm_error, gdbm_errmsg); + } + else { + PyErr_SetString(state->gdbm_error, generic_error); + } +} + /*[clinic input] module _gdbm class _gdbm.gdbm "gdbmobject *" "&Gdbmtype" @@ -57,6 +76,8 @@ typedef struct { GDBM_FILE di_dbm; } gdbmobject; +#define _gdbmobject_CAST(op) ((gdbmobject *)(op)) + #include "clinic/_gdbmmodule.c.h" #define check_gdbmobject_open(v, err) \ @@ -91,7 +112,7 @@ newgdbmobject(_gdbm_state *state, const char *file, int flags, int mode) PyErr_SetFromErrnoWithFilename(state->gdbm_error, file); } else { - PyErr_SetString(state->gdbm_error, gdbm_strerror(gdbm_errno)); + set_gdbm_error(state, "gdbm_open() error"); } Py_DECREF(dp); return NULL; @@ -101,27 +122,29 @@ newgdbmobject(_gdbm_state *state, const char *file, int flags, int mode) /* Methods */ static int -gdbm_traverse(gdbmobject *dp, visitproc visit, void *arg) +gdbm_traverse(PyObject *op, visitproc visit, void *arg) { - Py_VISIT(Py_TYPE(dp)); + Py_VISIT(Py_TYPE(op)); return 0; } static void -gdbm_dealloc(gdbmobject *dp) +gdbm_dealloc(PyObject *op) { + gdbmobject *dp = _gdbmobject_CAST(op); + PyTypeObject *tp = Py_TYPE(dp); PyObject_GC_UnTrack(dp); if (dp->di_dbm) { gdbm_close(dp->di_dbm); } - PyTypeObject *tp = Py_TYPE(dp); tp->tp_free(dp); Py_DECREF(tp); } static Py_ssize_t -gdbm_length(gdbmobject *dp) +gdbm_length(PyObject *op) { + gdbmobject *dp = _gdbmobject_CAST(op); _gdbm_state *state = PyType_GetModuleState(Py_TYPE(dp)); if (dp->di_dbm == NULL) { PyErr_SetString(state->gdbm_error, "GDBM object has already been closed"); @@ -136,7 +159,7 @@ gdbm_length(gdbmobject *dp) PyErr_SetFromErrno(state->gdbm_error); } else { - PyErr_SetString(state->gdbm_error, gdbm_strerror(gdbm_errno)); + set_gdbm_error(state, "gdbm_count() error"); } return -1; } @@ -166,8 +189,9 @@ gdbm_length(gdbmobject *dp) } static int -gdbm_bool(gdbmobject *dp) +gdbm_bool(PyObject *op) { + gdbmobject *dp = _gdbmobject_CAST(op); _gdbm_state *state = PyType_GetModuleState(Py_TYPE(dp)); if (dp->di_dbm == NULL) { PyErr_SetString(state->gdbm_error, "GDBM object has already been closed"); @@ -216,10 +240,11 @@ parse_datum(PyObject *o, datum *d, const char *failmsg) } static PyObject * -gdbm_subscript(gdbmobject *dp, PyObject *key) +gdbm_subscript(PyObject *op, PyObject *key) { PyObject *v; datum drec, krec; + gdbmobject *dp = _gdbmobject_CAST(op); _gdbm_state *state = PyType_GetModuleState(Py_TYPE(dp)); if (!parse_datum(key, &krec, NULL)) { @@ -256,7 +281,7 @@ _gdbm_gdbm_get_impl(gdbmobject *self, PyObject *key, PyObject *default_value) { PyObject *res; - res = gdbm_subscript(self, key); + res = gdbm_subscript((PyObject *)self, key); if (res == NULL && PyErr_ExceptionMatches(PyExc_KeyError)) { PyErr_Clear(); return Py_NewRef(default_value); @@ -265,10 +290,11 @@ _gdbm_gdbm_get_impl(gdbmobject *self, PyObject *key, PyObject *default_value) } static int -gdbm_ass_sub(gdbmobject *dp, PyObject *v, PyObject *w) +gdbm_ass_sub(PyObject *op, PyObject *v, PyObject *w) { datum krec, drec; const char *failmsg = "gdbm mappings have bytes or string indices only"; + gdbmobject *dp = _gdbmobject_CAST(op); _gdbm_state *state = PyType_GetModuleState(Py_TYPE(dp)); if (!parse_datum(v, &krec, failmsg)) { @@ -286,7 +312,7 @@ gdbm_ass_sub(gdbmobject *dp, PyObject *v, PyObject *w) PyErr_SetObject(PyExc_KeyError, v); } else { - PyErr_SetString(state->gdbm_error, gdbm_strerror(gdbm_errno)); + set_gdbm_error(state, "gdbm_delete() error"); } return -1; } @@ -297,11 +323,12 @@ gdbm_ass_sub(gdbmobject *dp, PyObject *v, PyObject *w) } errno = 0; if (gdbm_store(dp->di_dbm, krec, drec, GDBM_REPLACE) < 0) { - if (errno != 0) + if (errno != 0) { PyErr_SetFromErrno(state->gdbm_error); - else - PyErr_SetString(state->gdbm_error, - gdbm_strerror(gdbm_errno)); + } + else { + set_gdbm_error(state, "gdbm_store() error"); + } return -1; } } @@ -325,12 +352,12 @@ _gdbm_gdbm_setdefault_impl(gdbmobject *self, PyObject *key, { PyObject *res; - res = gdbm_subscript(self, key); + res = gdbm_subscript((PyObject *)self, key); if (res == NULL && PyErr_ExceptionMatches(PyExc_KeyError)) { PyErr_Clear(); - if (gdbm_ass_sub(self, key, default_value) < 0) + if (gdbm_ass_sub((PyObject *)self, key, default_value) < 0) return NULL; - return gdbm_subscript(self, key); + return gdbm_subscript((PyObject *)self, key); } return res; } @@ -534,10 +561,12 @@ _gdbm_gdbm_reorganize_impl(gdbmobject *self, PyTypeObject *cls) check_gdbmobject_open(self, state->gdbm_error); errno = 0; if (gdbm_reorganize(self->di_dbm) < 0) { - if (errno != 0) + if (errno != 0) { PyErr_SetFromErrno(state->gdbm_error); - else - PyErr_SetString(state->gdbm_error, gdbm_strerror(gdbm_errno)); + } + else { + set_gdbm_error(state, "gdbm_reorganize() error"); + } return NULL; } Py_RETURN_NONE; @@ -819,12 +848,13 @@ _gdbm_module_clear(PyObject *module) static void _gdbm_module_free(void *module) { - _gdbm_module_clear((PyObject *)module); + (void)_gdbm_module_clear((PyObject *)module); } static PyModuleDef_Slot _gdbm_module_slots[] = { {Py_mod_exec, _gdbm_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_hacl/Hacl_Hash_Blake2b.c b/Modules/_hacl/Hacl_Hash_Blake2b.c new file mode 100644 index 00000000000000..1bab75e6aaf2ab --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_Blake2b.c @@ -0,0 +1,1566 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#include "internal/Hacl_Hash_Blake2b.h" + +#include "internal/Hacl_Impl_Blake2_Constants.h" +#include "lib_memzero0.h" + +static void +update_block( + uint64_t *wv, + uint64_t *hash, + bool flag, + bool last_node, + FStar_UInt128_uint128 totlen, + uint8_t *d +) +{ + uint64_t m_w[16U] = { 0U }; + KRML_MAYBE_FOR16(i, + 0U, + 16U, + 1U, + uint64_t *os = m_w; + uint8_t *bj = d + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + uint64_t mask[4U] = { 0U }; + uint64_t wv_14; + if (flag) + { + wv_14 = 0xFFFFFFFFFFFFFFFFULL; + } + else + { + wv_14 = 0ULL; + } + uint64_t wv_15; + if (last_node) + { + wv_15 = 0xFFFFFFFFFFFFFFFFULL; + } + else + { + wv_15 = 0ULL; + } + mask[0U] = FStar_UInt128_uint128_to_uint64(totlen); + mask[1U] = FStar_UInt128_uint128_to_uint64(FStar_UInt128_shift_right(totlen, 64U)); + mask[2U] = wv_14; + mask[3U] = wv_15; + memcpy(wv, hash, 16U * sizeof (uint64_t)); + uint64_t *wv3 = wv + 12U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv3; + uint64_t x = wv3[i] ^ mask[i]; + os[i] = x;); + KRML_MAYBE_FOR12(i0, + 0U, + 12U, + 1U, + uint32_t start_idx = i0 % 10U * 16U; + uint64_t m_st[16U] = { 0U }; + uint64_t *r0 = m_st; + uint64_t *r1 = m_st + 4U; + uint64_t *r20 = m_st + 8U; + uint64_t *r30 = m_st + 12U; + uint32_t s0 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 0U]; + uint32_t s1 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 1U]; + uint32_t s2 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 2U]; + uint32_t s3 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 3U]; + uint32_t s4 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 4U]; + uint32_t s5 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 5U]; + uint32_t s6 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 6U]; + uint32_t s7 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 7U]; + uint32_t s8 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 8U]; + uint32_t s9 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 9U]; + uint32_t s10 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 10U]; + uint32_t s11 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 11U]; + uint32_t s12 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 12U]; + uint32_t s13 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 13U]; + uint32_t s14 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 14U]; + uint32_t s15 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 15U]; + uint64_t uu____0 = m_w[s2]; + uint64_t uu____1 = m_w[s4]; + uint64_t uu____2 = m_w[s6]; + r0[0U] = m_w[s0]; + r0[1U] = uu____0; + r0[2U] = uu____1; + r0[3U] = uu____2; + uint64_t uu____3 = m_w[s3]; + uint64_t uu____4 = m_w[s5]; + uint64_t uu____5 = m_w[s7]; + r1[0U] = m_w[s1]; + r1[1U] = uu____3; + r1[2U] = uu____4; + r1[3U] = uu____5; + uint64_t uu____6 = m_w[s10]; + uint64_t uu____7 = m_w[s12]; + uint64_t uu____8 = m_w[s14]; + r20[0U] = m_w[s8]; + r20[1U] = uu____6; + r20[2U] = uu____7; + r20[3U] = uu____8; + uint64_t uu____9 = m_w[s11]; + uint64_t uu____10 = m_w[s13]; + uint64_t uu____11 = m_w[s15]; + r30[0U] = m_w[s9]; + r30[1U] = uu____9; + r30[2U] = uu____10; + r30[3U] = uu____11; + uint64_t *x = m_st; + uint64_t *y = m_st + 4U; + uint64_t *z = m_st + 8U; + uint64_t *w = m_st + 12U; + uint32_t a = 0U; + uint32_t b0 = 1U; + uint32_t c0 = 2U; + uint32_t d10 = 3U; + uint64_t *wv_a0 = wv + a * 4U; + uint64_t *wv_b0 = wv + b0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a0; + uint64_t x1 = wv_a0[i] + wv_b0[i]; + os[i] = x1;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a0; + uint64_t x1 = wv_a0[i] + x[i]; + os[i] = x1;); + uint64_t *wv_a1 = wv + d10 * 4U; + uint64_t *wv_b1 = wv + a * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a1; + uint64_t x1 = wv_a1[i] ^ wv_b1[i]; + os[i] = x1;); + uint64_t *r10 = wv_a1; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = r10; + uint64_t x1 = r10[i]; + uint64_t x10 = x1 >> 32U | x1 << 32U; + os[i] = x10;); + uint64_t *wv_a2 = wv + c0 * 4U; + uint64_t *wv_b2 = wv + d10 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a2; + uint64_t x1 = wv_a2[i] + wv_b2[i]; + os[i] = x1;); + uint64_t *wv_a3 = wv + b0 * 4U; + uint64_t *wv_b3 = wv + c0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a3; + uint64_t x1 = wv_a3[i] ^ wv_b3[i]; + os[i] = x1;); + uint64_t *r12 = wv_a3; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = r12; + uint64_t x1 = r12[i]; + uint64_t x10 = x1 >> 24U | x1 << 40U; + os[i] = x10;); + uint64_t *wv_a4 = wv + a * 4U; + uint64_t *wv_b4 = wv + b0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a4; + uint64_t x1 = wv_a4[i] + wv_b4[i]; + os[i] = x1;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a4; + uint64_t x1 = wv_a4[i] + y[i]; + os[i] = x1;); + uint64_t *wv_a5 = wv + d10 * 4U; + uint64_t *wv_b5 = wv + a * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a5; + uint64_t x1 = wv_a5[i] ^ wv_b5[i]; + os[i] = x1;); + uint64_t *r13 = wv_a5; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = r13; + uint64_t x1 = r13[i]; + uint64_t x10 = x1 >> 16U | x1 << 48U; + os[i] = x10;); + uint64_t *wv_a6 = wv + c0 * 4U; + uint64_t *wv_b6 = wv + d10 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a6; + uint64_t x1 = wv_a6[i] + wv_b6[i]; + os[i] = x1;); + uint64_t *wv_a7 = wv + b0 * 4U; + uint64_t *wv_b7 = wv + c0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a7; + uint64_t x1 = wv_a7[i] ^ wv_b7[i]; + os[i] = x1;); + uint64_t *r14 = wv_a7; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = r14; + uint64_t x1 = r14[i]; + uint64_t x10 = x1 >> 63U | x1 << 1U; + os[i] = x10;); + uint64_t *r15 = wv + 4U; + uint64_t *r21 = wv + 8U; + uint64_t *r31 = wv + 12U; + uint64_t *r110 = r15; + uint64_t x00 = r110[1U]; + uint64_t x10 = r110[2U]; + uint64_t x20 = r110[3U]; + uint64_t x30 = r110[0U]; + r110[0U] = x00; + r110[1U] = x10; + r110[2U] = x20; + r110[3U] = x30; + uint64_t *r111 = r21; + uint64_t x01 = r111[2U]; + uint64_t x11 = r111[3U]; + uint64_t x21 = r111[0U]; + uint64_t x31 = r111[1U]; + r111[0U] = x01; + r111[1U] = x11; + r111[2U] = x21; + r111[3U] = x31; + uint64_t *r112 = r31; + uint64_t x02 = r112[3U]; + uint64_t x12 = r112[0U]; + uint64_t x22 = r112[1U]; + uint64_t x32 = r112[2U]; + r112[0U] = x02; + r112[1U] = x12; + r112[2U] = x22; + r112[3U] = x32; + uint32_t a0 = 0U; + uint32_t b = 1U; + uint32_t c = 2U; + uint32_t d1 = 3U; + uint64_t *wv_a = wv + a0 * 4U; + uint64_t *wv_b8 = wv + b * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a; + uint64_t x1 = wv_a[i] + wv_b8[i]; + os[i] = x1;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a; + uint64_t x1 = wv_a[i] + z[i]; + os[i] = x1;); + uint64_t *wv_a8 = wv + d1 * 4U; + uint64_t *wv_b9 = wv + a0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a8; + uint64_t x1 = wv_a8[i] ^ wv_b9[i]; + os[i] = x1;); + uint64_t *r16 = wv_a8; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = r16; + uint64_t x1 = r16[i]; + uint64_t x13 = x1 >> 32U | x1 << 32U; + os[i] = x13;); + uint64_t *wv_a9 = wv + c * 4U; + uint64_t *wv_b10 = wv + d1 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a9; + uint64_t x1 = wv_a9[i] + wv_b10[i]; + os[i] = x1;); + uint64_t *wv_a10 = wv + b * 4U; + uint64_t *wv_b11 = wv + c * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a10; + uint64_t x1 = wv_a10[i] ^ wv_b11[i]; + os[i] = x1;); + uint64_t *r17 = wv_a10; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = r17; + uint64_t x1 = r17[i]; + uint64_t x13 = x1 >> 24U | x1 << 40U; + os[i] = x13;); + uint64_t *wv_a11 = wv + a0 * 4U; + uint64_t *wv_b12 = wv + b * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a11; + uint64_t x1 = wv_a11[i] + wv_b12[i]; + os[i] = x1;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a11; + uint64_t x1 = wv_a11[i] + w[i]; + os[i] = x1;); + uint64_t *wv_a12 = wv + d1 * 4U; + uint64_t *wv_b13 = wv + a0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a12; + uint64_t x1 = wv_a12[i] ^ wv_b13[i]; + os[i] = x1;); + uint64_t *r18 = wv_a12; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = r18; + uint64_t x1 = r18[i]; + uint64_t x13 = x1 >> 16U | x1 << 48U; + os[i] = x13;); + uint64_t *wv_a13 = wv + c * 4U; + uint64_t *wv_b14 = wv + d1 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a13; + uint64_t x1 = wv_a13[i] + wv_b14[i]; + os[i] = x1;); + uint64_t *wv_a14 = wv + b * 4U; + uint64_t *wv_b = wv + c * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = wv_a14; + uint64_t x1 = wv_a14[i] ^ wv_b[i]; + os[i] = x1;); + uint64_t *r19 = wv_a14; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = r19; + uint64_t x1 = r19[i]; + uint64_t x13 = x1 >> 63U | x1 << 1U; + os[i] = x13;); + uint64_t *r113 = wv + 4U; + uint64_t *r2 = wv + 8U; + uint64_t *r3 = wv + 12U; + uint64_t *r11 = r113; + uint64_t x03 = r11[3U]; + uint64_t x13 = r11[0U]; + uint64_t x23 = r11[1U]; + uint64_t x33 = r11[2U]; + r11[0U] = x03; + r11[1U] = x13; + r11[2U] = x23; + r11[3U] = x33; + uint64_t *r114 = r2; + uint64_t x04 = r114[2U]; + uint64_t x14 = r114[3U]; + uint64_t x24 = r114[0U]; + uint64_t x34 = r114[1U]; + r114[0U] = x04; + r114[1U] = x14; + r114[2U] = x24; + r114[3U] = x34; + uint64_t *r115 = r3; + uint64_t x0 = r115[1U]; + uint64_t x1 = r115[2U]; + uint64_t x2 = r115[3U]; + uint64_t x3 = r115[0U]; + r115[0U] = x0; + r115[1U] = x1; + r115[2U] = x2; + r115[3U] = x3;); + uint64_t *s0 = hash; + uint64_t *s1 = hash + 4U; + uint64_t *r0 = wv; + uint64_t *r1 = wv + 4U; + uint64_t *r2 = wv + 8U; + uint64_t *r3 = wv + 12U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = s0; + uint64_t x = s0[i] ^ r0[i]; + os[i] = x;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = s0; + uint64_t x = s0[i] ^ r2[i]; + os[i] = x;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = s1; + uint64_t x = s1[i] ^ r1[i]; + os[i] = x;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = s1; + uint64_t x = s1[i] ^ r3[i]; + os[i] = x;); +} + +void Hacl_Hash_Blake2b_init(uint64_t *hash, uint32_t kk, uint32_t nn) +{ + uint8_t salt[16U] = { 0U }; + uint8_t personal[16U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = 64U, .key_length = 0U, .fanout = 1U, .depth = 1U, .leaf_length = 0U, + .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, .personal = personal + }; + uint64_t tmp[8U] = { 0U }; + uint64_t *r0 = hash; + uint64_t *r1 = hash + 4U; + uint64_t *r2 = hash + 8U; + uint64_t *r3 = hash + 12U; + uint64_t iv0 = Hacl_Hash_Blake2b_ivTable_B[0U]; + uint64_t iv1 = Hacl_Hash_Blake2b_ivTable_B[1U]; + uint64_t iv2 = Hacl_Hash_Blake2b_ivTable_B[2U]; + uint64_t iv3 = Hacl_Hash_Blake2b_ivTable_B[3U]; + uint64_t iv4 = Hacl_Hash_Blake2b_ivTable_B[4U]; + uint64_t iv5 = Hacl_Hash_Blake2b_ivTable_B[5U]; + uint64_t iv6 = Hacl_Hash_Blake2b_ivTable_B[6U]; + uint64_t iv7 = Hacl_Hash_Blake2b_ivTable_B[7U]; + r2[0U] = iv0; + r2[1U] = iv1; + r2[2U] = iv2; + r2[3U] = iv3; + r3[0U] = iv4; + r3[1U] = iv5; + r3[2U] = iv6; + r3[3U] = iv7; + uint8_t kk1 = (uint8_t)kk; + uint8_t nn1 = (uint8_t)nn; + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 4U; + uint8_t *bj = p.salt + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 6U; + uint8_t *bj = p.personal + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + tmp[0U] = + (uint64_t)nn1 + ^ + ((uint64_t)kk1 + << 8U + ^ ((uint64_t)p.fanout << 16U ^ ((uint64_t)p.depth << 24U ^ (uint64_t)p.leaf_length << 32U))); + tmp[1U] = p.node_offset; + tmp[2U] = (uint64_t)p.node_depth ^ (uint64_t)p.inner_length << 8U; + tmp[3U] = 0ULL; + uint64_t tmp0 = tmp[0U]; + uint64_t tmp1 = tmp[1U]; + uint64_t tmp2 = tmp[2U]; + uint64_t tmp3 = tmp[3U]; + uint64_t tmp4 = tmp[4U]; + uint64_t tmp5 = tmp[5U]; + uint64_t tmp6 = tmp[6U]; + uint64_t tmp7 = tmp[7U]; + uint64_t iv0_ = iv0 ^ tmp0; + uint64_t iv1_ = iv1 ^ tmp1; + uint64_t iv2_ = iv2 ^ tmp2; + uint64_t iv3_ = iv3 ^ tmp3; + uint64_t iv4_ = iv4 ^ tmp4; + uint64_t iv5_ = iv5 ^ tmp5; + uint64_t iv6_ = iv6 ^ tmp6; + uint64_t iv7_ = iv7 ^ tmp7; + r0[0U] = iv0_; + r0[1U] = iv1_; + r0[2U] = iv2_; + r0[3U] = iv3_; + r1[0U] = iv4_; + r1[1U] = iv5_; + r1[2U] = iv6_; + r1[3U] = iv7_; +} + +static void update_key(uint64_t *wv, uint64_t *hash, uint32_t kk, uint8_t *k, uint32_t ll) +{ + FStar_UInt128_uint128 lb = FStar_UInt128_uint64_to_uint128((uint64_t)128U); + uint8_t b[128U] = { 0U }; + memcpy(b, k, kk * sizeof (uint8_t)); + if (ll == 0U) + { + update_block(wv, hash, true, false, lb, b); + } + else + { + update_block(wv, hash, false, false, lb, b); + } + Lib_Memzero0_memzero(b, 128U, uint8_t, void *); +} + +void +Hacl_Hash_Blake2b_update_multi( + uint32_t len, + uint64_t *wv, + uint64_t *hash, + FStar_UInt128_uint128 prev, + uint8_t *blocks, + uint32_t nb +) +{ + KRML_MAYBE_UNUSED_VAR(len); + for (uint32_t i = 0U; i < nb; i++) + { + FStar_UInt128_uint128 + totlen = + FStar_UInt128_add_mod(prev, + FStar_UInt128_uint64_to_uint128((uint64_t)((i + 1U) * 128U))); + uint8_t *b = blocks + i * 128U; + update_block(wv, hash, false, false, totlen, b); + } +} + +void +Hacl_Hash_Blake2b_update_last( + uint32_t len, + uint64_t *wv, + uint64_t *hash, + bool last_node, + FStar_UInt128_uint128 prev, + uint32_t rem, + uint8_t *d +) +{ + uint8_t b[128U] = { 0U }; + uint8_t *last = d + len - rem; + memcpy(b, last, rem * sizeof (uint8_t)); + FStar_UInt128_uint128 + totlen = FStar_UInt128_add_mod(prev, FStar_UInt128_uint64_to_uint128((uint64_t)len)); + update_block(wv, hash, true, last_node, totlen, b); + Lib_Memzero0_memzero(b, 128U, uint8_t, void *); +} + +static void +update_blocks( + uint32_t len, + uint64_t *wv, + uint64_t *hash, + FStar_UInt128_uint128 prev, + uint8_t *blocks +) +{ + uint32_t nb0 = len / 128U; + uint32_t rem0 = len % 128U; + uint32_t nb; + if (rem0 == 0U && nb0 > 0U) + { + nb = nb0 - 1U; + } + else + { + nb = nb0; + } + uint32_t rem; + if (rem0 == 0U && nb0 > 0U) + { + rem = 128U; + } + else + { + rem = rem0; + } + Hacl_Hash_Blake2b_update_multi(len, wv, hash, prev, blocks, nb); + Hacl_Hash_Blake2b_update_last(len, wv, hash, false, prev, rem, blocks); +} + +static inline void +update(uint64_t *wv, uint64_t *hash, uint32_t kk, uint8_t *k, uint32_t ll, uint8_t *d) +{ + FStar_UInt128_uint128 lb = FStar_UInt128_uint64_to_uint128((uint64_t)128U); + if (kk > 0U) + { + update_key(wv, hash, kk, k, ll); + if (!(ll == 0U)) + { + update_blocks(ll, wv, hash, lb, d); + return; + } + return; + } + update_blocks(ll, wv, hash, FStar_UInt128_uint64_to_uint128((uint64_t)0U), d); +} + +void Hacl_Hash_Blake2b_finish(uint32_t nn, uint8_t *output, uint64_t *hash) +{ + uint8_t b[64U] = { 0U }; + uint8_t *first = b; + uint8_t *second = b + 32U; + uint64_t *row0 = hash; + uint64_t *row1 = hash + 4U; + KRML_MAYBE_FOR4(i, 0U, 4U, 1U, store64_le(first + i * 8U, row0[i]);); + KRML_MAYBE_FOR4(i, 0U, 4U, 1U, store64_le(second + i * 8U, row1[i]);); + uint8_t *final = b; + memcpy(output, final, nn * sizeof (uint8_t)); + Lib_Memzero0_memzero(b, 64U, uint8_t, void *); +} + +static Hacl_Hash_Blake2b_state_t +*malloc_raw(Hacl_Hash_Blake2b_index kk, Hacl_Hash_Blake2b_params_and_key key) +{ + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(128U, sizeof (uint8_t)); + uint64_t *wv = (uint64_t *)KRML_HOST_CALLOC(16U, sizeof (uint64_t)); + uint64_t *b = (uint64_t *)KRML_HOST_CALLOC(16U, sizeof (uint64_t)); + Hacl_Hash_Blake2b_block_state_t + block_state = + { + .fst = kk.key_length, + .snd = kk.digest_length, + .thd = kk.last_node, + .f3 = { .fst = wv, .snd = b } + }; + uint8_t kk10 = kk.key_length; + uint32_t ite; + if (kk10 != 0U) + { + ite = 128U; + } + else + { + ite = 0U; + } + Hacl_Hash_Blake2b_state_t + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)ite }; + Hacl_Hash_Blake2b_state_t + *p = (Hacl_Hash_Blake2b_state_t *)KRML_HOST_MALLOC(sizeof (Hacl_Hash_Blake2b_state_t)); + p[0U] = s; + Hacl_Hash_Blake2b_blake2_params *p1 = key.fst; + uint8_t kk1 = p1->key_length; + uint8_t nn = p1->digest_length; + bool last_node = block_state.thd; + Hacl_Hash_Blake2b_index i = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + uint64_t *h = block_state.f3.snd; + uint32_t kk20 = (uint32_t)i.key_length; + uint8_t *k_1 = key.snd; + if (!(kk20 == 0U)) + { + uint8_t *sub_b = buf + kk20; + memset(sub_b, 0U, (128U - kk20) * sizeof (uint8_t)); + memcpy(buf, k_1, kk20 * sizeof (uint8_t)); + } + Hacl_Hash_Blake2b_blake2_params pv = p1[0U]; + uint64_t tmp[8U] = { 0U }; + uint64_t *r0 = h; + uint64_t *r1 = h + 4U; + uint64_t *r2 = h + 8U; + uint64_t *r3 = h + 12U; + uint64_t iv0 = Hacl_Hash_Blake2b_ivTable_B[0U]; + uint64_t iv1 = Hacl_Hash_Blake2b_ivTable_B[1U]; + uint64_t iv2 = Hacl_Hash_Blake2b_ivTable_B[2U]; + uint64_t iv3 = Hacl_Hash_Blake2b_ivTable_B[3U]; + uint64_t iv4 = Hacl_Hash_Blake2b_ivTable_B[4U]; + uint64_t iv5 = Hacl_Hash_Blake2b_ivTable_B[5U]; + uint64_t iv6 = Hacl_Hash_Blake2b_ivTable_B[6U]; + uint64_t iv7 = Hacl_Hash_Blake2b_ivTable_B[7U]; + r2[0U] = iv0; + r2[1U] = iv1; + r2[2U] = iv2; + r2[3U] = iv3; + r3[0U] = iv4; + r3[1U] = iv5; + r3[2U] = iv6; + r3[3U] = iv7; + uint8_t kk2 = pv.key_length; + uint8_t nn1 = pv.digest_length; + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 4U; + uint8_t *bj = pv.salt + i0 * 8U; + uint64_t u = load64_le(bj); + uint64_t r4 = u; + uint64_t x = r4; + os[i0] = x;); + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 6U; + uint8_t *bj = pv.personal + i0 * 8U; + uint64_t u = load64_le(bj); + uint64_t r4 = u; + uint64_t x = r4; + os[i0] = x;); + tmp[0U] = + (uint64_t)nn1 + ^ + ((uint64_t)kk2 + << 8U + ^ ((uint64_t)pv.fanout << 16U ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); + tmp[1U] = pv.node_offset; + tmp[2U] = (uint64_t)pv.node_depth ^ (uint64_t)pv.inner_length << 8U; + tmp[3U] = 0ULL; + uint64_t tmp0 = tmp[0U]; + uint64_t tmp1 = tmp[1U]; + uint64_t tmp2 = tmp[2U]; + uint64_t tmp3 = tmp[3U]; + uint64_t tmp4 = tmp[4U]; + uint64_t tmp5 = tmp[5U]; + uint64_t tmp6 = tmp[6U]; + uint64_t tmp7 = tmp[7U]; + uint64_t iv0_ = iv0 ^ tmp0; + uint64_t iv1_ = iv1 ^ tmp1; + uint64_t iv2_ = iv2 ^ tmp2; + uint64_t iv3_ = iv3 ^ tmp3; + uint64_t iv4_ = iv4 ^ tmp4; + uint64_t iv5_ = iv5 ^ tmp5; + uint64_t iv6_ = iv6 ^ tmp6; + uint64_t iv7_ = iv7 ^ tmp7; + r0[0U] = iv0_; + r0[1U] = iv1_; + r0[2U] = iv2_; + r0[3U] = iv3_; + r1[0U] = iv4_; + r1[1U] = iv5_; + r1[2U] = iv6_; + r1[3U] = iv7_; + return p; +} + +/** + General-purpose allocation function that gives control over all +Blake2 parameters, including the key. Further resettings of the state SHALL be +done with `reset_with_params_and_key`, and SHALL feature the exact same values +for the `key_length` and `digest_length` fields as passed here. In other words, +once you commit to a digest and key length, the only way to change these +parameters is to allocate a new object. + +The caller must satisfy the following requirements. +- The length of the key k MUST match the value of the field key_length in the + parameters. +- The key_length must not exceed 32 for S, 64 for B. +- The digest_length must not exceed 32 for S, 64 for B. + +*/ +Hacl_Hash_Blake2b_state_t +*Hacl_Hash_Blake2b_malloc_with_params_and_key( + Hacl_Hash_Blake2b_blake2_params *p, + bool last_node, + uint8_t *k +) +{ + Hacl_Hash_Blake2b_blake2_params pv = p[0U]; + Hacl_Hash_Blake2b_index + i1 = { .key_length = pv.key_length, .digest_length = pv.digest_length, .last_node = last_node }; + return malloc_raw(i1, ((Hacl_Hash_Blake2b_params_and_key){ .fst = p, .snd = k })); +} + +/** + Specialized allocation function that picks default values for all +parameters, except for the key_length. Further resettings of the state SHALL be +done with `reset_with_key`, and SHALL feature the exact same key length `kk` as +passed here. In other words, once you commit to a key length, the only way to +change this parameter is to allocate a new object. + +The caller must satisfy the following requirements. +- The key_length must not exceed 32 for S, 64 for B. + +*/ +Hacl_Hash_Blake2b_state_t *Hacl_Hash_Blake2b_malloc_with_key(uint8_t *k, uint8_t kk) +{ + uint8_t nn = 64U; + Hacl_Hash_Blake2b_index i = { .key_length = kk, .digest_length = nn, .last_node = false }; + uint8_t salt[16U] = { 0U }; + uint8_t personal[16U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = i.digest_length, .key_length = i.key_length, .fanout = 1U, .depth = 1U, + .leaf_length = 0U, .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, + .personal = personal + }; + Hacl_Hash_Blake2b_blake2_params p0 = p; + Hacl_Hash_Blake2b_state_t *s = Hacl_Hash_Blake2b_malloc_with_params_and_key(&p0, false, k); + return s; +} + +/** + Specialized allocation function that picks default values for all +parameters, and has no key. Effectively, this is what you want if you intend to +use Blake2 as a hash function. Further resettings of the state SHALL be done with `reset`. +*/ +Hacl_Hash_Blake2b_state_t *Hacl_Hash_Blake2b_malloc(void) +{ + return Hacl_Hash_Blake2b_malloc_with_key(NULL, 0U); +} + +static Hacl_Hash_Blake2b_index index_of_state(Hacl_Hash_Blake2b_state_t *s) +{ + Hacl_Hash_Blake2b_block_state_t block_state = (*s).block_state; + bool last_node = block_state.thd; + uint8_t nn = block_state.snd; + uint8_t kk1 = block_state.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk1, .digest_length = nn, .last_node = last_node }); +} + +static void reset_raw(Hacl_Hash_Blake2b_state_t *state, Hacl_Hash_Blake2b_params_and_key key) +{ + Hacl_Hash_Blake2b_state_t scrut = *state; + uint8_t *buf = scrut.buf; + Hacl_Hash_Blake2b_block_state_t block_state = scrut.block_state; + bool last_node0 = block_state.thd; + uint8_t nn0 = block_state.snd; + uint8_t kk10 = block_state.fst; + Hacl_Hash_Blake2b_index + i = { .key_length = kk10, .digest_length = nn0, .last_node = last_node0 }; + KRML_MAYBE_UNUSED_VAR(i); + Hacl_Hash_Blake2b_blake2_params *p = key.fst; + uint8_t kk1 = p->key_length; + uint8_t nn = p->digest_length; + bool last_node = block_state.thd; + Hacl_Hash_Blake2b_index + i1 = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + uint64_t *h = block_state.f3.snd; + uint32_t kk20 = (uint32_t)i1.key_length; + uint8_t *k_1 = key.snd; + if (!(kk20 == 0U)) + { + uint8_t *sub_b = buf + kk20; + memset(sub_b, 0U, (128U - kk20) * sizeof (uint8_t)); + memcpy(buf, k_1, kk20 * sizeof (uint8_t)); + } + Hacl_Hash_Blake2b_blake2_params pv = p[0U]; + uint64_t tmp[8U] = { 0U }; + uint64_t *r0 = h; + uint64_t *r1 = h + 4U; + uint64_t *r2 = h + 8U; + uint64_t *r3 = h + 12U; + uint64_t iv0 = Hacl_Hash_Blake2b_ivTable_B[0U]; + uint64_t iv1 = Hacl_Hash_Blake2b_ivTable_B[1U]; + uint64_t iv2 = Hacl_Hash_Blake2b_ivTable_B[2U]; + uint64_t iv3 = Hacl_Hash_Blake2b_ivTable_B[3U]; + uint64_t iv4 = Hacl_Hash_Blake2b_ivTable_B[4U]; + uint64_t iv5 = Hacl_Hash_Blake2b_ivTable_B[5U]; + uint64_t iv6 = Hacl_Hash_Blake2b_ivTable_B[6U]; + uint64_t iv7 = Hacl_Hash_Blake2b_ivTable_B[7U]; + r2[0U] = iv0; + r2[1U] = iv1; + r2[2U] = iv2; + r2[3U] = iv3; + r3[0U] = iv4; + r3[1U] = iv5; + r3[2U] = iv6; + r3[3U] = iv7; + uint8_t kk2 = pv.key_length; + uint8_t nn1 = pv.digest_length; + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 4U; + uint8_t *bj = pv.salt + i0 * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i0] = x;); + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 6U; + uint8_t *bj = pv.personal + i0 * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i0] = x;); + tmp[0U] = + (uint64_t)nn1 + ^ + ((uint64_t)kk2 + << 8U + ^ ((uint64_t)pv.fanout << 16U ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); + tmp[1U] = pv.node_offset; + tmp[2U] = (uint64_t)pv.node_depth ^ (uint64_t)pv.inner_length << 8U; + tmp[3U] = 0ULL; + uint64_t tmp0 = tmp[0U]; + uint64_t tmp1 = tmp[1U]; + uint64_t tmp2 = tmp[2U]; + uint64_t tmp3 = tmp[3U]; + uint64_t tmp4 = tmp[4U]; + uint64_t tmp5 = tmp[5U]; + uint64_t tmp6 = tmp[6U]; + uint64_t tmp7 = tmp[7U]; + uint64_t iv0_ = iv0 ^ tmp0; + uint64_t iv1_ = iv1 ^ tmp1; + uint64_t iv2_ = iv2 ^ tmp2; + uint64_t iv3_ = iv3 ^ tmp3; + uint64_t iv4_ = iv4 ^ tmp4; + uint64_t iv5_ = iv5 ^ tmp5; + uint64_t iv6_ = iv6 ^ tmp6; + uint64_t iv7_ = iv7 ^ tmp7; + r0[0U] = iv0_; + r0[1U] = iv1_; + r0[2U] = iv2_; + r0[3U] = iv3_; + r1[0U] = iv4_; + r1[1U] = iv5_; + r1[2U] = iv6_; + r1[3U] = iv7_; + uint8_t kk11 = i.key_length; + uint32_t ite; + if (kk11 != 0U) + { + ite = 128U; + } + else + { + ite = 0U; + } + Hacl_Hash_Blake2b_state_t + tmp8 = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)ite }; + state[0U] = tmp8; +} + +/** + General-purpose re-initialization function with parameters and +key. You cannot change digest_length, key_length, or last_node, meaning those values in +the parameters object must be the same as originally decided via one of the +malloc functions. All other values of the parameter can be changed. The behavior +is unspecified if you violate this precondition. +*/ +void +Hacl_Hash_Blake2b_reset_with_key_and_params( + Hacl_Hash_Blake2b_state_t *s, + Hacl_Hash_Blake2b_blake2_params *p, + uint8_t *k +) +{ + Hacl_Hash_Blake2b_index i1 = index_of_state(s); + KRML_MAYBE_UNUSED_VAR(i1); + reset_raw(s, ((Hacl_Hash_Blake2b_params_and_key){ .fst = p, .snd = k })); +} + +/** + Specialized-purpose re-initialization function with no parameters, +and a key. The key length must be the same as originally decided via your choice +of malloc function. All other parameters are reset to their default values. The +original call to malloc MUST have set digest_length to the default value. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2b_reset_with_key(Hacl_Hash_Blake2b_state_t *s, uint8_t *k) +{ + Hacl_Hash_Blake2b_index idx = index_of_state(s); + uint8_t salt[16U] = { 0U }; + uint8_t personal[16U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = idx.digest_length, .key_length = idx.key_length, .fanout = 1U, .depth = 1U, + .leaf_length = 0U, .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, + .personal = personal + }; + Hacl_Hash_Blake2b_blake2_params p0 = p; + reset_raw(s, ((Hacl_Hash_Blake2b_params_and_key){ .fst = &p0, .snd = k })); +} + +/** + Specialized-purpose re-initialization function with no parameters +and no key. This is what you want if you intend to use Blake2 as a hash +function. The key length and digest length must have been set to their +respective default values via your choice of malloc function (always true if you +used `malloc`). All other parameters are reset to their default values. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2b_reset(Hacl_Hash_Blake2b_state_t *s) +{ + Hacl_Hash_Blake2b_reset_with_key(s, NULL); +} + +/** + Update function; 0 = success, 1 = max length exceeded +*/ +Hacl_Streaming_Types_error_code +Hacl_Hash_Blake2b_update(Hacl_Hash_Blake2b_state_t *state, uint8_t *chunk, uint32_t chunk_len) +{ + Hacl_Hash_Blake2b_state_t s = *state; + uint64_t total_len = s.total_len; + if ((uint64_t)chunk_len > 0xffffffffffffffffULL - total_len) + { + return Hacl_Streaming_Types_MaximumLengthExceeded; + } + uint32_t sz; + if (total_len % (uint64_t)128U == 0ULL && total_len > 0ULL) + { + sz = 128U; + } + else + { + sz = (uint32_t)(total_len % (uint64_t)128U); + } + if (chunk_len <= 128U - sz) + { + Hacl_Hash_Blake2b_state_t s1 = *state; + Hacl_Hash_Blake2b_block_state_t block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)128U == 0ULL && total_len1 > 0ULL) + { + sz1 = 128U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)128U); + } + uint8_t *buf2 = buf + sz1; + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state + = + ( + (Hacl_Hash_Blake2b_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len2 + } + ); + } + else if (sz == 0U) + { + Hacl_Hash_Blake2b_state_t s1 = *state; + Hacl_Hash_Blake2b_block_state_t block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)128U == 0ULL && total_len1 > 0ULL) + { + sz1 = 128U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)128U); + } + if (!(sz1 == 0U)) + { + uint64_t prevlen = total_len1 - (uint64_t)sz1; + K____uint64_t___uint64_t_ acc = block_state1.f3; + uint64_t *wv = acc.fst; + uint64_t *hash = acc.snd; + uint32_t nb = 1U; + Hacl_Hash_Blake2b_update_multi(128U, + wv, + hash, + FStar_UInt128_uint64_to_uint128(prevlen), + buf, + nb); + } + uint32_t ite; + if ((uint64_t)chunk_len % (uint64_t)128U == 0ULL && (uint64_t)chunk_len > 0ULL) + { + ite = 128U; + } + else + { + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)128U); + } + uint32_t n_blocks = (chunk_len - ite) / 128U; + uint32_t data1_len = n_blocks * 128U; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; + K____uint64_t___uint64_t_ acc = block_state1.f3; + uint64_t *wv = acc.fst; + uint64_t *hash = acc.snd; + uint32_t nb = data1_len / 128U; + Hacl_Hash_Blake2b_update_multi(data1_len, + wv, + hash, + FStar_UInt128_uint64_to_uint128(total_len1), + data1, + nb); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state + = + ( + (Hacl_Hash_Blake2b_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)chunk_len + } + ); + } + else + { + uint32_t diff = 128U - sz; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Hash_Blake2b_state_t s1 = *state; + Hacl_Hash_Blake2b_block_state_t block_state10 = s1.block_state; + uint8_t *buf0 = s1.buf; + uint64_t total_len10 = s1.total_len; + uint32_t sz10; + if (total_len10 % (uint64_t)128U == 0ULL && total_len10 > 0ULL) + { + sz10 = 128U; + } + else + { + sz10 = (uint32_t)(total_len10 % (uint64_t)128U); + } + uint8_t *buf2 = buf0 + sz10; + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); + uint64_t total_len2 = total_len10 + (uint64_t)diff; + *state + = + ( + (Hacl_Hash_Blake2b_state_t){ + .block_state = block_state10, + .buf = buf0, + .total_len = total_len2 + } + ); + Hacl_Hash_Blake2b_state_t s10 = *state; + Hacl_Hash_Blake2b_block_state_t block_state1 = s10.block_state; + uint8_t *buf = s10.buf; + uint64_t total_len1 = s10.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)128U == 0ULL && total_len1 > 0ULL) + { + sz1 = 128U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)128U); + } + if (!(sz1 == 0U)) + { + uint64_t prevlen = total_len1 - (uint64_t)sz1; + K____uint64_t___uint64_t_ acc = block_state1.f3; + uint64_t *wv = acc.fst; + uint64_t *hash = acc.snd; + uint32_t nb = 1U; + Hacl_Hash_Blake2b_update_multi(128U, + wv, + hash, + FStar_UInt128_uint64_to_uint128(prevlen), + buf, + nb); + } + uint32_t ite; + if + ((uint64_t)(chunk_len - diff) % (uint64_t)128U == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL) + { + ite = 128U; + } + else + { + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)128U); + } + uint32_t n_blocks = (chunk_len - diff - ite) / 128U; + uint32_t data1_len = n_blocks * 128U; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; + K____uint64_t___uint64_t_ acc = block_state1.f3; + uint64_t *wv = acc.fst; + uint64_t *hash = acc.snd; + uint32_t nb = data1_len / 128U; + Hacl_Hash_Blake2b_update_multi(data1_len, + wv, + hash, + FStar_UInt128_uint64_to_uint128(total_len1), + data1, + nb); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state + = + ( + (Hacl_Hash_Blake2b_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)(chunk_len - diff) + } + ); + } + return Hacl_Streaming_Types_Success; +} + +/** + Digest function. This function expects the `output` array to hold +at least `digest_length` bytes, where `digest_length` was determined by your +choice of `malloc` function. Concretely, if you used `malloc` or +`malloc_with_key`, then the expected length is 32 for S, or 64 for B (default +digest length). If you used `malloc_with_params_and_key`, then the expected +length is whatever you chose for the `digest_length` field of your parameters. +For convenience, this function returns `digest_length`. When in doubt, callers +can pass an array of size HACL_BLAKE2B_32_OUT_BYTES, then use the return value +to see how many bytes were actually written. +*/ +uint8_t Hacl_Hash_Blake2b_digest(Hacl_Hash_Blake2b_state_t *s, uint8_t *dst) +{ + Hacl_Hash_Blake2b_block_state_t block_state0 = (*s).block_state; + bool last_node0 = block_state0.thd; + uint8_t nn0 = block_state0.snd; + uint8_t kk0 = block_state0.fst; + Hacl_Hash_Blake2b_index + i1 = { .key_length = kk0, .digest_length = nn0, .last_node = last_node0 }; + Hacl_Hash_Blake2b_state_t scrut = *s; + Hacl_Hash_Blake2b_block_state_t block_state = scrut.block_state; + uint8_t *buf_ = scrut.buf; + uint64_t total_len = scrut.total_len; + uint32_t r; + if (total_len % (uint64_t)128U == 0ULL && total_len > 0ULL) + { + r = 128U; + } + else + { + r = (uint32_t)(total_len % (uint64_t)128U); + } + uint8_t *buf_1 = buf_; + uint64_t wv0[16U] = { 0U }; + uint64_t b[16U] = { 0U }; + Hacl_Hash_Blake2b_block_state_t + tmp_block_state = + { + .fst = i1.key_length, + .snd = i1.digest_length, + .thd = i1.last_node, + .f3 = { .fst = wv0, .snd = b } + }; + uint64_t *src_b = block_state.f3.snd; + uint64_t *dst_b = tmp_block_state.f3.snd; + memcpy(dst_b, src_b, 16U * sizeof (uint64_t)); + uint64_t prev_len = total_len - (uint64_t)r; + uint32_t ite; + if (r % 128U == 0U && r > 0U) + { + ite = 128U; + } + else + { + ite = r % 128U; + } + uint8_t *buf_last = buf_1 + r - ite; + uint8_t *buf_multi = buf_1; + K____uint64_t___uint64_t_ acc0 = tmp_block_state.f3; + uint64_t *wv1 = acc0.fst; + uint64_t *hash0 = acc0.snd; + uint32_t nb = 0U; + Hacl_Hash_Blake2b_update_multi(0U, + wv1, + hash0, + FStar_UInt128_uint64_to_uint128(prev_len), + buf_multi, + nb); + uint64_t prev_len_last = total_len - (uint64_t)r; + K____uint64_t___uint64_t_ acc = tmp_block_state.f3; + bool last_node1 = tmp_block_state.thd; + uint64_t *wv = acc.fst; + uint64_t *hash = acc.snd; + Hacl_Hash_Blake2b_update_last(r, + wv, + hash, + last_node1, + FStar_UInt128_uint64_to_uint128(prev_len_last), + r, + buf_last); + uint8_t nn1 = tmp_block_state.snd; + Hacl_Hash_Blake2b_finish((uint32_t)nn1, dst, tmp_block_state.f3.snd); + Hacl_Hash_Blake2b_block_state_t block_state1 = (*s).block_state; + bool last_node = block_state1.thd; + uint8_t nn = block_state1.snd; + uint8_t kk = block_state1.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk, .digest_length = nn, .last_node = last_node }).digest_length; +} + +Hacl_Hash_Blake2b_index Hacl_Hash_Blake2b_info(Hacl_Hash_Blake2b_state_t *s) +{ + Hacl_Hash_Blake2b_block_state_t block_state = (*s).block_state; + bool last_node = block_state.thd; + uint8_t nn = block_state.snd; + uint8_t kk = block_state.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk, .digest_length = nn, .last_node = last_node }); +} + +/** + Free state function when there is no key +*/ +void Hacl_Hash_Blake2b_free(Hacl_Hash_Blake2b_state_t *state) +{ + Hacl_Hash_Blake2b_state_t scrut = *state; + uint8_t *buf = scrut.buf; + Hacl_Hash_Blake2b_block_state_t block_state = scrut.block_state; + uint64_t *b = block_state.f3.snd; + uint64_t *wv = block_state.f3.fst; + KRML_HOST_FREE(wv); + KRML_HOST_FREE(b); + KRML_HOST_FREE(buf); + KRML_HOST_FREE(state); +} + +/** + Copying. This preserves all parameters. +*/ +Hacl_Hash_Blake2b_state_t *Hacl_Hash_Blake2b_copy(Hacl_Hash_Blake2b_state_t *state) +{ + Hacl_Hash_Blake2b_state_t scrut = *state; + Hacl_Hash_Blake2b_block_state_t block_state0 = scrut.block_state; + uint8_t *buf0 = scrut.buf; + uint64_t total_len0 = scrut.total_len; + bool last_node = block_state0.thd; + uint8_t nn = block_state0.snd; + uint8_t kk1 = block_state0.fst; + Hacl_Hash_Blake2b_index i = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(128U, sizeof (uint8_t)); + memcpy(buf, buf0, 128U * sizeof (uint8_t)); + uint64_t *wv = (uint64_t *)KRML_HOST_CALLOC(16U, sizeof (uint64_t)); + uint64_t *b = (uint64_t *)KRML_HOST_CALLOC(16U, sizeof (uint64_t)); + Hacl_Hash_Blake2b_block_state_t + block_state = + { + .fst = i.key_length, + .snd = i.digest_length, + .thd = i.last_node, + .f3 = { .fst = wv, .snd = b } + }; + uint64_t *src_b = block_state0.f3.snd; + uint64_t *dst_b = block_state.f3.snd; + memcpy(dst_b, src_b, 16U * sizeof (uint64_t)); + Hacl_Hash_Blake2b_state_t + s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; + Hacl_Hash_Blake2b_state_t + *p = (Hacl_Hash_Blake2b_state_t *)KRML_HOST_MALLOC(sizeof (Hacl_Hash_Blake2b_state_t)); + p[0U] = s; + return p; +} + +/** +Write the BLAKE2b digest of message `input` using key `key` into `output`. + +@param output Pointer to `output_len` bytes of memory where the digest is written to. +@param output_len Length of the to-be-generated digest with 1 <= `output_len` <= 64. +@param input Pointer to `input_len` bytes of memory where the input message is read from. +@param input_len Length of the input message. +@param key Pointer to `key_len` bytes of memory where the key is read from. +@param key_len Length of the key. Can be 0. +*/ +void +Hacl_Hash_Blake2b_hash_with_key( + uint8_t *output, + uint32_t output_len, + uint8_t *input, + uint32_t input_len, + uint8_t *key, + uint32_t key_len +) +{ + uint64_t b[16U] = { 0U }; + uint64_t b1[16U] = { 0U }; + Hacl_Hash_Blake2b_init(b, key_len, output_len); + update(b1, b, key_len, key, input_len, input); + Hacl_Hash_Blake2b_finish(output_len, output, b); + Lib_Memzero0_memzero(b1, 16U, uint64_t, void *); + Lib_Memzero0_memzero(b, 16U, uint64_t, void *); +} + +/** +Write the BLAKE2b digest of message `input` using key `key` and +parameters `params` into `output`. The `key` array must be of length +`params.key_length`. The `output` array must be of length +`params.digest_length`. +*/ +void +Hacl_Hash_Blake2b_hash_with_key_and_params( + uint8_t *output, + uint8_t *input, + uint32_t input_len, + Hacl_Hash_Blake2b_blake2_params params, + uint8_t *key +) +{ + uint64_t b[16U] = { 0U }; + uint64_t b1[16U] = { 0U }; + uint64_t tmp[8U] = { 0U }; + uint64_t *r0 = b; + uint64_t *r1 = b + 4U; + uint64_t *r2 = b + 8U; + uint64_t *r3 = b + 12U; + uint64_t iv0 = Hacl_Hash_Blake2b_ivTable_B[0U]; + uint64_t iv1 = Hacl_Hash_Blake2b_ivTable_B[1U]; + uint64_t iv2 = Hacl_Hash_Blake2b_ivTable_B[2U]; + uint64_t iv3 = Hacl_Hash_Blake2b_ivTable_B[3U]; + uint64_t iv4 = Hacl_Hash_Blake2b_ivTable_B[4U]; + uint64_t iv5 = Hacl_Hash_Blake2b_ivTable_B[5U]; + uint64_t iv6 = Hacl_Hash_Blake2b_ivTable_B[6U]; + uint64_t iv7 = Hacl_Hash_Blake2b_ivTable_B[7U]; + r2[0U] = iv0; + r2[1U] = iv1; + r2[2U] = iv2; + r2[3U] = iv3; + r3[0U] = iv4; + r3[1U] = iv5; + r3[2U] = iv6; + r3[3U] = iv7; + uint8_t kk = params.key_length; + uint8_t nn = params.digest_length; + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 4U; + uint8_t *bj = params.salt + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 6U; + uint8_t *bj = params.personal + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + tmp[0U] = + (uint64_t)nn + ^ + ((uint64_t)kk + << 8U + ^ + ((uint64_t)params.fanout + << 16U + ^ ((uint64_t)params.depth << 24U ^ (uint64_t)params.leaf_length << 32U))); + tmp[1U] = params.node_offset; + tmp[2U] = (uint64_t)params.node_depth ^ (uint64_t)params.inner_length << 8U; + tmp[3U] = 0ULL; + uint64_t tmp0 = tmp[0U]; + uint64_t tmp1 = tmp[1U]; + uint64_t tmp2 = tmp[2U]; + uint64_t tmp3 = tmp[3U]; + uint64_t tmp4 = tmp[4U]; + uint64_t tmp5 = tmp[5U]; + uint64_t tmp6 = tmp[6U]; + uint64_t tmp7 = tmp[7U]; + uint64_t iv0_ = iv0 ^ tmp0; + uint64_t iv1_ = iv1 ^ tmp1; + uint64_t iv2_ = iv2 ^ tmp2; + uint64_t iv3_ = iv3 ^ tmp3; + uint64_t iv4_ = iv4 ^ tmp4; + uint64_t iv5_ = iv5 ^ tmp5; + uint64_t iv6_ = iv6 ^ tmp6; + uint64_t iv7_ = iv7 ^ tmp7; + r0[0U] = iv0_; + r0[1U] = iv1_; + r0[2U] = iv2_; + r0[3U] = iv3_; + r1[0U] = iv4_; + r1[1U] = iv5_; + r1[2U] = iv6_; + r1[3U] = iv7_; + update(b1, b, (uint32_t)params.key_length, key, input_len, input); + Hacl_Hash_Blake2b_finish((uint32_t)params.digest_length, output, b); + Lib_Memzero0_memzero(b1, 16U, uint64_t, void *); + Lib_Memzero0_memzero(b, 16U, uint64_t, void *); +} + diff --git a/Modules/_hacl/Hacl_Hash_Blake2b.h b/Modules/_hacl/Hacl_Hash_Blake2b.h new file mode 100644 index 00000000000000..5b5b037bcdc8a4 --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_Blake2b.h @@ -0,0 +1,245 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __Hacl_Hash_Blake2b_H +#define __Hacl_Hash_Blake2b_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "python_hacl_namespaces.h" +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "Hacl_Streaming_Types.h" + + +typedef struct Hacl_Hash_Blake2b_blake2_params_s +{ + uint8_t digest_length; + uint8_t key_length; + uint8_t fanout; + uint8_t depth; + uint32_t leaf_length; + uint64_t node_offset; + uint8_t node_depth; + uint8_t inner_length; + uint8_t *salt; + uint8_t *personal; +} +Hacl_Hash_Blake2b_blake2_params; + +typedef struct Hacl_Hash_Blake2b_index_s +{ + uint8_t key_length; + uint8_t digest_length; + bool last_node; +} +Hacl_Hash_Blake2b_index; + +#define HACL_HASH_BLAKE2B_BLOCK_BYTES (128U) + +#define HACL_HASH_BLAKE2B_OUT_BYTES (64U) + +#define HACL_HASH_BLAKE2B_KEY_BYTES (64U) + +#define HACL_HASH_BLAKE2B_SALT_BYTES (16U) + +#define HACL_HASH_BLAKE2B_PERSONAL_BYTES (16U) + +typedef struct K____uint64_t___uint64_t__s +{ + uint64_t *fst; + uint64_t *snd; +} +K____uint64_t___uint64_t_; + +typedef struct Hacl_Hash_Blake2b_block_state_t_s +{ + uint8_t fst; + uint8_t snd; + bool thd; + K____uint64_t___uint64_t_ f3; +} +Hacl_Hash_Blake2b_block_state_t; + +typedef struct Hacl_Hash_Blake2b_state_t_s +{ + Hacl_Hash_Blake2b_block_state_t block_state; + uint8_t *buf; + uint64_t total_len; +} +Hacl_Hash_Blake2b_state_t; + +/** + General-purpose allocation function that gives control over all +Blake2 parameters, including the key. Further resettings of the state SHALL be +done with `reset_with_params_and_key`, and SHALL feature the exact same values +for the `key_length` and `digest_length` fields as passed here. In other words, +once you commit to a digest and key length, the only way to change these +parameters is to allocate a new object. + +The caller must satisfy the following requirements. +- The length of the key k MUST match the value of the field key_length in the + parameters. +- The key_length must not exceed 32 for S, 64 for B. +- The digest_length must not exceed 32 for S, 64 for B. + +*/ +Hacl_Hash_Blake2b_state_t +*Hacl_Hash_Blake2b_malloc_with_params_and_key( + Hacl_Hash_Blake2b_blake2_params *p, + bool last_node, + uint8_t *k +); + +/** + Specialized allocation function that picks default values for all +parameters, except for the key_length. Further resettings of the state SHALL be +done with `reset_with_key`, and SHALL feature the exact same key length `kk` as +passed here. In other words, once you commit to a key length, the only way to +change this parameter is to allocate a new object. + +The caller must satisfy the following requirements. +- The key_length must not exceed 32 for S, 64 for B. + +*/ +Hacl_Hash_Blake2b_state_t *Hacl_Hash_Blake2b_malloc_with_key(uint8_t *k, uint8_t kk); + +/** + Specialized allocation function that picks default values for all +parameters, and has no key. Effectively, this is what you want if you intend to +use Blake2 as a hash function. Further resettings of the state SHALL be done with `reset`. +*/ +Hacl_Hash_Blake2b_state_t *Hacl_Hash_Blake2b_malloc(void); + +/** + General-purpose re-initialization function with parameters and +key. You cannot change digest_length, key_length, or last_node, meaning those values in +the parameters object must be the same as originally decided via one of the +malloc functions. All other values of the parameter can be changed. The behavior +is unspecified if you violate this precondition. +*/ +void +Hacl_Hash_Blake2b_reset_with_key_and_params( + Hacl_Hash_Blake2b_state_t *s, + Hacl_Hash_Blake2b_blake2_params *p, + uint8_t *k +); + +/** + Specialized-purpose re-initialization function with no parameters, +and a key. The key length must be the same as originally decided via your choice +of malloc function. All other parameters are reset to their default values. The +original call to malloc MUST have set digest_length to the default value. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2b_reset_with_key(Hacl_Hash_Blake2b_state_t *s, uint8_t *k); + +/** + Specialized-purpose re-initialization function with no parameters +and no key. This is what you want if you intend to use Blake2 as a hash +function. The key length and digest length must have been set to their +respective default values via your choice of malloc function (always true if you +used `malloc`). All other parameters are reset to their default values. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2b_reset(Hacl_Hash_Blake2b_state_t *s); + +/** + Update function; 0 = success, 1 = max length exceeded +*/ +Hacl_Streaming_Types_error_code +Hacl_Hash_Blake2b_update(Hacl_Hash_Blake2b_state_t *state, uint8_t *chunk, uint32_t chunk_len); + +/** + Digest function. This function expects the `output` array to hold +at least `digest_length` bytes, where `digest_length` was determined by your +choice of `malloc` function. Concretely, if you used `malloc` or +`malloc_with_key`, then the expected length is 32 for S, or 64 for B (default +digest length). If you used `malloc_with_params_and_key`, then the expected +length is whatever you chose for the `digest_length` field of your parameters. +For convenience, this function returns `digest_length`. When in doubt, callers +can pass an array of size HACL_BLAKE2B_32_OUT_BYTES, then use the return value +to see how many bytes were actually written. +*/ +uint8_t Hacl_Hash_Blake2b_digest(Hacl_Hash_Blake2b_state_t *s, uint8_t *dst); + +Hacl_Hash_Blake2b_index Hacl_Hash_Blake2b_info(Hacl_Hash_Blake2b_state_t *s); + +/** + Free state function when there is no key +*/ +void Hacl_Hash_Blake2b_free(Hacl_Hash_Blake2b_state_t *state); + +/** + Copying. This preserves all parameters. +*/ +Hacl_Hash_Blake2b_state_t *Hacl_Hash_Blake2b_copy(Hacl_Hash_Blake2b_state_t *state); + +/** +Write the BLAKE2b digest of message `input` using key `key` into `output`. + +@param output Pointer to `output_len` bytes of memory where the digest is written to. +@param output_len Length of the to-be-generated digest with 1 <= `output_len` <= 64. +@param input Pointer to `input_len` bytes of memory where the input message is read from. +@param input_len Length of the input message. +@param key Pointer to `key_len` bytes of memory where the key is read from. +@param key_len Length of the key. Can be 0. +*/ +void +Hacl_Hash_Blake2b_hash_with_key( + uint8_t *output, + uint32_t output_len, + uint8_t *input, + uint32_t input_len, + uint8_t *key, + uint32_t key_len +); + +/** +Write the BLAKE2b digest of message `input` using key `key` and +parameters `params` into `output`. The `key` array must be of length +`params.key_length`. The `output` array must be of length +`params.digest_length`. +*/ +void +Hacl_Hash_Blake2b_hash_with_key_and_params( + uint8_t *output, + uint8_t *input, + uint32_t input_len, + Hacl_Hash_Blake2b_blake2_params params, + uint8_t *key +); + +#if defined(__cplusplus) +} +#endif + +#define __Hacl_Hash_Blake2b_H_DEFINED +#endif diff --git a/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c new file mode 100644 index 00000000000000..19234ab9d7f9b2 --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c @@ -0,0 +1,1398 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#include "internal/Hacl_Hash_Blake2b_Simd256.h" + +#include "internal/Hacl_Impl_Blake2_Constants.h" +#include "internal/Hacl_Hash_Blake2b.h" +#include "lib_memzero0.h" + +static inline void +update_block( + Lib_IntVector_Intrinsics_vec256 *wv, + Lib_IntVector_Intrinsics_vec256 *hash, + bool flag, + bool last_node, + FStar_UInt128_uint128 totlen, + uint8_t *d +) +{ + uint64_t m_w[16U] = { 0U }; + KRML_MAYBE_FOR16(i, + 0U, + 16U, + 1U, + uint64_t *os = m_w; + uint8_t *bj = d + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + Lib_IntVector_Intrinsics_vec256 mask = Lib_IntVector_Intrinsics_vec256_zero; + uint64_t wv_14; + if (flag) + { + wv_14 = 0xFFFFFFFFFFFFFFFFULL; + } + else + { + wv_14 = 0ULL; + } + uint64_t wv_15; + if (last_node) + { + wv_15 = 0xFFFFFFFFFFFFFFFFULL; + } + else + { + wv_15 = 0ULL; + } + mask = + Lib_IntVector_Intrinsics_vec256_load64s(FStar_UInt128_uint128_to_uint64(totlen), + FStar_UInt128_uint128_to_uint64(FStar_UInt128_shift_right(totlen, 64U)), + wv_14, + wv_15); + memcpy(wv, hash, 4U * sizeof (Lib_IntVector_Intrinsics_vec256)); + Lib_IntVector_Intrinsics_vec256 *wv3 = wv + 3U; + wv3[0U] = Lib_IntVector_Intrinsics_vec256_xor(wv3[0U], mask); + KRML_MAYBE_FOR12(i, + 0U, + 12U, + 1U, + uint32_t start_idx = i % 10U * 16U; + KRML_PRE_ALIGN(32) Lib_IntVector_Intrinsics_vec256 m_st[4U] KRML_POST_ALIGN(32) = { 0U }; + Lib_IntVector_Intrinsics_vec256 *r0 = m_st; + Lib_IntVector_Intrinsics_vec256 *r1 = m_st + 1U; + Lib_IntVector_Intrinsics_vec256 *r20 = m_st + 2U; + Lib_IntVector_Intrinsics_vec256 *r30 = m_st + 3U; + uint32_t s0 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 0U]; + uint32_t s1 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 1U]; + uint32_t s2 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 2U]; + uint32_t s3 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 3U]; + uint32_t s4 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 4U]; + uint32_t s5 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 5U]; + uint32_t s6 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 6U]; + uint32_t s7 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 7U]; + uint32_t s8 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 8U]; + uint32_t s9 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 9U]; + uint32_t s10 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 10U]; + uint32_t s11 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 11U]; + uint32_t s12 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 12U]; + uint32_t s13 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 13U]; + uint32_t s14 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 14U]; + uint32_t s15 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 15U]; + r0[0U] = Lib_IntVector_Intrinsics_vec256_load64s(m_w[s0], m_w[s2], m_w[s4], m_w[s6]); + r1[0U] = Lib_IntVector_Intrinsics_vec256_load64s(m_w[s1], m_w[s3], m_w[s5], m_w[s7]); + r20[0U] = Lib_IntVector_Intrinsics_vec256_load64s(m_w[s8], m_w[s10], m_w[s12], m_w[s14]); + r30[0U] = Lib_IntVector_Intrinsics_vec256_load64s(m_w[s9], m_w[s11], m_w[s13], m_w[s15]); + Lib_IntVector_Intrinsics_vec256 *x = m_st; + Lib_IntVector_Intrinsics_vec256 *y = m_st + 1U; + Lib_IntVector_Intrinsics_vec256 *z = m_st + 2U; + Lib_IntVector_Intrinsics_vec256 *w = m_st + 3U; + uint32_t a = 0U; + uint32_t b0 = 1U; + uint32_t c0 = 2U; + uint32_t d10 = 3U; + Lib_IntVector_Intrinsics_vec256 *wv_a0 = wv + a * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b0 = wv + b0 * 1U; + wv_a0[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a0[0U], wv_b0[0U]); + wv_a0[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a0[0U], x[0U]); + Lib_IntVector_Intrinsics_vec256 *wv_a1 = wv + d10 * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b1 = wv + a * 1U; + wv_a1[0U] = Lib_IntVector_Intrinsics_vec256_xor(wv_a1[0U], wv_b1[0U]); + wv_a1[0U] = Lib_IntVector_Intrinsics_vec256_rotate_right64(wv_a1[0U], 32U); + Lib_IntVector_Intrinsics_vec256 *wv_a2 = wv + c0 * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b2 = wv + d10 * 1U; + wv_a2[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a2[0U], wv_b2[0U]); + Lib_IntVector_Intrinsics_vec256 *wv_a3 = wv + b0 * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b3 = wv + c0 * 1U; + wv_a3[0U] = Lib_IntVector_Intrinsics_vec256_xor(wv_a3[0U], wv_b3[0U]); + wv_a3[0U] = Lib_IntVector_Intrinsics_vec256_rotate_right64(wv_a3[0U], 24U); + Lib_IntVector_Intrinsics_vec256 *wv_a4 = wv + a * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b4 = wv + b0 * 1U; + wv_a4[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a4[0U], wv_b4[0U]); + wv_a4[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a4[0U], y[0U]); + Lib_IntVector_Intrinsics_vec256 *wv_a5 = wv + d10 * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b5 = wv + a * 1U; + wv_a5[0U] = Lib_IntVector_Intrinsics_vec256_xor(wv_a5[0U], wv_b5[0U]); + wv_a5[0U] = Lib_IntVector_Intrinsics_vec256_rotate_right64(wv_a5[0U], 16U); + Lib_IntVector_Intrinsics_vec256 *wv_a6 = wv + c0 * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b6 = wv + d10 * 1U; + wv_a6[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a6[0U], wv_b6[0U]); + Lib_IntVector_Intrinsics_vec256 *wv_a7 = wv + b0 * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b7 = wv + c0 * 1U; + wv_a7[0U] = Lib_IntVector_Intrinsics_vec256_xor(wv_a7[0U], wv_b7[0U]); + wv_a7[0U] = Lib_IntVector_Intrinsics_vec256_rotate_right64(wv_a7[0U], 63U); + Lib_IntVector_Intrinsics_vec256 *r10 = wv + 1U; + Lib_IntVector_Intrinsics_vec256 *r21 = wv + 2U; + Lib_IntVector_Intrinsics_vec256 *r31 = wv + 3U; + Lib_IntVector_Intrinsics_vec256 v00 = r10[0U]; + Lib_IntVector_Intrinsics_vec256 + v1 = Lib_IntVector_Intrinsics_vec256_rotate_right_lanes64(v00, 1U); + r10[0U] = v1; + Lib_IntVector_Intrinsics_vec256 v01 = r21[0U]; + Lib_IntVector_Intrinsics_vec256 + v10 = Lib_IntVector_Intrinsics_vec256_rotate_right_lanes64(v01, 2U); + r21[0U] = v10; + Lib_IntVector_Intrinsics_vec256 v02 = r31[0U]; + Lib_IntVector_Intrinsics_vec256 + v11 = Lib_IntVector_Intrinsics_vec256_rotate_right_lanes64(v02, 3U); + r31[0U] = v11; + uint32_t a0 = 0U; + uint32_t b = 1U; + uint32_t c = 2U; + uint32_t d1 = 3U; + Lib_IntVector_Intrinsics_vec256 *wv_a = wv + a0 * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b8 = wv + b * 1U; + wv_a[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a[0U], wv_b8[0U]); + wv_a[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a[0U], z[0U]); + Lib_IntVector_Intrinsics_vec256 *wv_a8 = wv + d1 * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b9 = wv + a0 * 1U; + wv_a8[0U] = Lib_IntVector_Intrinsics_vec256_xor(wv_a8[0U], wv_b9[0U]); + wv_a8[0U] = Lib_IntVector_Intrinsics_vec256_rotate_right64(wv_a8[0U], 32U); + Lib_IntVector_Intrinsics_vec256 *wv_a9 = wv + c * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b10 = wv + d1 * 1U; + wv_a9[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a9[0U], wv_b10[0U]); + Lib_IntVector_Intrinsics_vec256 *wv_a10 = wv + b * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b11 = wv + c * 1U; + wv_a10[0U] = Lib_IntVector_Intrinsics_vec256_xor(wv_a10[0U], wv_b11[0U]); + wv_a10[0U] = Lib_IntVector_Intrinsics_vec256_rotate_right64(wv_a10[0U], 24U); + Lib_IntVector_Intrinsics_vec256 *wv_a11 = wv + a0 * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b12 = wv + b * 1U; + wv_a11[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a11[0U], wv_b12[0U]); + wv_a11[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a11[0U], w[0U]); + Lib_IntVector_Intrinsics_vec256 *wv_a12 = wv + d1 * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b13 = wv + a0 * 1U; + wv_a12[0U] = Lib_IntVector_Intrinsics_vec256_xor(wv_a12[0U], wv_b13[0U]); + wv_a12[0U] = Lib_IntVector_Intrinsics_vec256_rotate_right64(wv_a12[0U], 16U); + Lib_IntVector_Intrinsics_vec256 *wv_a13 = wv + c * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b14 = wv + d1 * 1U; + wv_a13[0U] = Lib_IntVector_Intrinsics_vec256_add64(wv_a13[0U], wv_b14[0U]); + Lib_IntVector_Intrinsics_vec256 *wv_a14 = wv + b * 1U; + Lib_IntVector_Intrinsics_vec256 *wv_b = wv + c * 1U; + wv_a14[0U] = Lib_IntVector_Intrinsics_vec256_xor(wv_a14[0U], wv_b[0U]); + wv_a14[0U] = Lib_IntVector_Intrinsics_vec256_rotate_right64(wv_a14[0U], 63U); + Lib_IntVector_Intrinsics_vec256 *r11 = wv + 1U; + Lib_IntVector_Intrinsics_vec256 *r2 = wv + 2U; + Lib_IntVector_Intrinsics_vec256 *r3 = wv + 3U; + Lib_IntVector_Intrinsics_vec256 v0 = r11[0U]; + Lib_IntVector_Intrinsics_vec256 + v12 = Lib_IntVector_Intrinsics_vec256_rotate_right_lanes64(v0, 3U); + r11[0U] = v12; + Lib_IntVector_Intrinsics_vec256 v03 = r2[0U]; + Lib_IntVector_Intrinsics_vec256 + v13 = Lib_IntVector_Intrinsics_vec256_rotate_right_lanes64(v03, 2U); + r2[0U] = v13; + Lib_IntVector_Intrinsics_vec256 v04 = r3[0U]; + Lib_IntVector_Intrinsics_vec256 + v14 = Lib_IntVector_Intrinsics_vec256_rotate_right_lanes64(v04, 1U); + r3[0U] = v14;); + Lib_IntVector_Intrinsics_vec256 *s0 = hash; + Lib_IntVector_Intrinsics_vec256 *s1 = hash + 1U; + Lib_IntVector_Intrinsics_vec256 *r0 = wv; + Lib_IntVector_Intrinsics_vec256 *r1 = wv + 1U; + Lib_IntVector_Intrinsics_vec256 *r2 = wv + 2U; + Lib_IntVector_Intrinsics_vec256 *r3 = wv + 3U; + s0[0U] = Lib_IntVector_Intrinsics_vec256_xor(s0[0U], r0[0U]); + s0[0U] = Lib_IntVector_Intrinsics_vec256_xor(s0[0U], r2[0U]); + s1[0U] = Lib_IntVector_Intrinsics_vec256_xor(s1[0U], r1[0U]); + s1[0U] = Lib_IntVector_Intrinsics_vec256_xor(s1[0U], r3[0U]); +} + +void +Hacl_Hash_Blake2b_Simd256_init(Lib_IntVector_Intrinsics_vec256 *hash, uint32_t kk, uint32_t nn) +{ + uint8_t salt[16U] = { 0U }; + uint8_t personal[16U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = 64U, .key_length = 0U, .fanout = 1U, .depth = 1U, .leaf_length = 0U, + .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, .personal = personal + }; + uint64_t tmp[8U] = { 0U }; + Lib_IntVector_Intrinsics_vec256 *r0 = hash; + Lib_IntVector_Intrinsics_vec256 *r1 = hash + 1U; + Lib_IntVector_Intrinsics_vec256 *r2 = hash + 2U; + Lib_IntVector_Intrinsics_vec256 *r3 = hash + 3U; + uint64_t iv0 = Hacl_Hash_Blake2b_ivTable_B[0U]; + uint64_t iv1 = Hacl_Hash_Blake2b_ivTable_B[1U]; + uint64_t iv2 = Hacl_Hash_Blake2b_ivTable_B[2U]; + uint64_t iv3 = Hacl_Hash_Blake2b_ivTable_B[3U]; + uint64_t iv4 = Hacl_Hash_Blake2b_ivTable_B[4U]; + uint64_t iv5 = Hacl_Hash_Blake2b_ivTable_B[5U]; + uint64_t iv6 = Hacl_Hash_Blake2b_ivTable_B[6U]; + uint64_t iv7 = Hacl_Hash_Blake2b_ivTable_B[7U]; + r2[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv0, iv1, iv2, iv3); + r3[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv4, iv5, iv6, iv7); + uint8_t kk1 = (uint8_t)kk; + uint8_t nn1 = (uint8_t)nn; + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 4U; + uint8_t *bj = p.salt + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 6U; + uint8_t *bj = p.personal + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + tmp[0U] = + (uint64_t)nn1 + ^ + ((uint64_t)kk1 + << 8U + ^ ((uint64_t)p.fanout << 16U ^ ((uint64_t)p.depth << 24U ^ (uint64_t)p.leaf_length << 32U))); + tmp[1U] = p.node_offset; + tmp[2U] = (uint64_t)p.node_depth ^ (uint64_t)p.inner_length << 8U; + tmp[3U] = 0ULL; + uint64_t tmp0 = tmp[0U]; + uint64_t tmp1 = tmp[1U]; + uint64_t tmp2 = tmp[2U]; + uint64_t tmp3 = tmp[3U]; + uint64_t tmp4 = tmp[4U]; + uint64_t tmp5 = tmp[5U]; + uint64_t tmp6 = tmp[6U]; + uint64_t tmp7 = tmp[7U]; + uint64_t iv0_ = iv0 ^ tmp0; + uint64_t iv1_ = iv1 ^ tmp1; + uint64_t iv2_ = iv2 ^ tmp2; + uint64_t iv3_ = iv3 ^ tmp3; + uint64_t iv4_ = iv4 ^ tmp4; + uint64_t iv5_ = iv5 ^ tmp5; + uint64_t iv6_ = iv6 ^ tmp6; + uint64_t iv7_ = iv7 ^ tmp7; + r0[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv0_, iv1_, iv2_, iv3_); + r1[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv4_, iv5_, iv6_, iv7_); +} + +static void +update_key( + Lib_IntVector_Intrinsics_vec256 *wv, + Lib_IntVector_Intrinsics_vec256 *hash, + uint32_t kk, + uint8_t *k, + uint32_t ll +) +{ + FStar_UInt128_uint128 lb = FStar_UInt128_uint64_to_uint128((uint64_t)128U); + uint8_t b[128U] = { 0U }; + memcpy(b, k, kk * sizeof (uint8_t)); + if (ll == 0U) + { + update_block(wv, hash, true, false, lb, b); + } + else + { + update_block(wv, hash, false, false, lb, b); + } + Lib_Memzero0_memzero(b, 128U, uint8_t, void *); +} + +void +Hacl_Hash_Blake2b_Simd256_update_multi( + uint32_t len, + Lib_IntVector_Intrinsics_vec256 *wv, + Lib_IntVector_Intrinsics_vec256 *hash, + FStar_UInt128_uint128 prev, + uint8_t *blocks, + uint32_t nb +) +{ + KRML_MAYBE_UNUSED_VAR(len); + for (uint32_t i = 0U; i < nb; i++) + { + FStar_UInt128_uint128 + totlen = + FStar_UInt128_add_mod(prev, + FStar_UInt128_uint64_to_uint128((uint64_t)((i + 1U) * 128U))); + uint8_t *b = blocks + i * 128U; + update_block(wv, hash, false, false, totlen, b); + } +} + +void +Hacl_Hash_Blake2b_Simd256_update_last( + uint32_t len, + Lib_IntVector_Intrinsics_vec256 *wv, + Lib_IntVector_Intrinsics_vec256 *hash, + bool last_node, + FStar_UInt128_uint128 prev, + uint32_t rem, + uint8_t *d +) +{ + uint8_t b[128U] = { 0U }; + uint8_t *last = d + len - rem; + memcpy(b, last, rem * sizeof (uint8_t)); + FStar_UInt128_uint128 + totlen = FStar_UInt128_add_mod(prev, FStar_UInt128_uint64_to_uint128((uint64_t)len)); + update_block(wv, hash, true, last_node, totlen, b); + Lib_Memzero0_memzero(b, 128U, uint8_t, void *); +} + +static inline void +update_blocks( + uint32_t len, + Lib_IntVector_Intrinsics_vec256 *wv, + Lib_IntVector_Intrinsics_vec256 *hash, + FStar_UInt128_uint128 prev, + uint8_t *blocks +) +{ + uint32_t nb0 = len / 128U; + uint32_t rem0 = len % 128U; + uint32_t nb; + if (rem0 == 0U && nb0 > 0U) + { + nb = nb0 - 1U; + } + else + { + nb = nb0; + } + uint32_t rem; + if (rem0 == 0U && nb0 > 0U) + { + rem = 128U; + } + else + { + rem = rem0; + } + Hacl_Hash_Blake2b_Simd256_update_multi(len, wv, hash, prev, blocks, nb); + Hacl_Hash_Blake2b_Simd256_update_last(len, wv, hash, false, prev, rem, blocks); +} + +static inline void +update( + Lib_IntVector_Intrinsics_vec256 *wv, + Lib_IntVector_Intrinsics_vec256 *hash, + uint32_t kk, + uint8_t *k, + uint32_t ll, + uint8_t *d +) +{ + FStar_UInt128_uint128 lb = FStar_UInt128_uint64_to_uint128((uint64_t)128U); + if (kk > 0U) + { + update_key(wv, hash, kk, k, ll); + if (!(ll == 0U)) + { + update_blocks(ll, wv, hash, lb, d); + return; + } + return; + } + update_blocks(ll, wv, hash, FStar_UInt128_uint64_to_uint128((uint64_t)0U), d); +} + +void +Hacl_Hash_Blake2b_Simd256_finish( + uint32_t nn, + uint8_t *output, + Lib_IntVector_Intrinsics_vec256 *hash +) +{ + uint8_t b[64U] = { 0U }; + uint8_t *first = b; + uint8_t *second = b + 32U; + Lib_IntVector_Intrinsics_vec256 *row0 = hash; + Lib_IntVector_Intrinsics_vec256 *row1 = hash + 1U; + Lib_IntVector_Intrinsics_vec256_store64_le(first, row0[0U]); + Lib_IntVector_Intrinsics_vec256_store64_le(second, row1[0U]); + uint8_t *final = b; + memcpy(output, final, nn * sizeof (uint8_t)); + Lib_Memzero0_memzero(b, 64U, uint8_t, void *); +} + +void +Hacl_Hash_Blake2b_Simd256_load_state256b_from_state32( + Lib_IntVector_Intrinsics_vec256 *st, + uint64_t *st32 +) +{ + Lib_IntVector_Intrinsics_vec256 *r0 = st; + Lib_IntVector_Intrinsics_vec256 *r1 = st + 1U; + Lib_IntVector_Intrinsics_vec256 *r2 = st + 2U; + Lib_IntVector_Intrinsics_vec256 *r3 = st + 3U; + uint64_t *b0 = st32; + uint64_t *b1 = st32 + 4U; + uint64_t *b2 = st32 + 8U; + uint64_t *b3 = st32 + 12U; + r0[0U] = Lib_IntVector_Intrinsics_vec256_load64s(b0[0U], b0[1U], b0[2U], b0[3U]); + r1[0U] = Lib_IntVector_Intrinsics_vec256_load64s(b1[0U], b1[1U], b1[2U], b1[3U]); + r2[0U] = Lib_IntVector_Intrinsics_vec256_load64s(b2[0U], b2[1U], b2[2U], b2[3U]); + r3[0U] = Lib_IntVector_Intrinsics_vec256_load64s(b3[0U], b3[1U], b3[2U], b3[3U]); +} + +void +Hacl_Hash_Blake2b_Simd256_store_state256b_to_state32( + uint64_t *st32, + Lib_IntVector_Intrinsics_vec256 *st +) +{ + Lib_IntVector_Intrinsics_vec256 *r0 = st; + Lib_IntVector_Intrinsics_vec256 *r1 = st + 1U; + Lib_IntVector_Intrinsics_vec256 *r2 = st + 2U; + Lib_IntVector_Intrinsics_vec256 *r3 = st + 3U; + uint64_t *b0 = st32; + uint64_t *b1 = st32 + 4U; + uint64_t *b2 = st32 + 8U; + uint64_t *b3 = st32 + 12U; + uint8_t b8[32U] = { 0U }; + Lib_IntVector_Intrinsics_vec256_store64_le(b8, r0[0U]); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = b0; + uint8_t *bj = b8 + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + uint8_t b80[32U] = { 0U }; + Lib_IntVector_Intrinsics_vec256_store64_le(b80, r1[0U]); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = b1; + uint8_t *bj = b80 + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + uint8_t b81[32U] = { 0U }; + Lib_IntVector_Intrinsics_vec256_store64_le(b81, r2[0U]); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = b2; + uint8_t *bj = b81 + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + uint8_t b82[32U] = { 0U }; + Lib_IntVector_Intrinsics_vec256_store64_le(b82, r3[0U]); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint64_t *os = b3; + uint8_t *bj = b82 + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); +} + +Lib_IntVector_Intrinsics_vec256 *Hacl_Hash_Blake2b_Simd256_malloc_with_key(void) +{ + Lib_IntVector_Intrinsics_vec256 + *buf = + (Lib_IntVector_Intrinsics_vec256 *)KRML_ALIGNED_MALLOC(32, + sizeof (Lib_IntVector_Intrinsics_vec256) * 4U); + memset(buf, 0U, 4U * sizeof (Lib_IntVector_Intrinsics_vec256)); + return buf; +} + +static Hacl_Hash_Blake2b_Simd256_state_t +*malloc_raw(Hacl_Hash_Blake2b_index kk, Hacl_Hash_Blake2b_params_and_key key) +{ + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(128U, sizeof (uint8_t)); + Lib_IntVector_Intrinsics_vec256 + *wv = + (Lib_IntVector_Intrinsics_vec256 *)KRML_ALIGNED_MALLOC(32, + sizeof (Lib_IntVector_Intrinsics_vec256) * 4U); + memset(wv, 0U, 4U * sizeof (Lib_IntVector_Intrinsics_vec256)); + Lib_IntVector_Intrinsics_vec256 + *b = + (Lib_IntVector_Intrinsics_vec256 *)KRML_ALIGNED_MALLOC(32, + sizeof (Lib_IntVector_Intrinsics_vec256) * 4U); + memset(b, 0U, 4U * sizeof (Lib_IntVector_Intrinsics_vec256)); + Hacl_Hash_Blake2b_Simd256_block_state_t + block_state = + { + .fst = kk.key_length, + .snd = kk.digest_length, + .thd = kk.last_node, + .f3 = { .fst = wv, .snd = b } + }; + uint8_t kk10 = kk.key_length; + uint32_t ite; + if (kk10 != 0U) + { + ite = 128U; + } + else + { + ite = 0U; + } + Hacl_Hash_Blake2b_Simd256_state_t + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)ite }; + Hacl_Hash_Blake2b_Simd256_state_t + *p = + (Hacl_Hash_Blake2b_Simd256_state_t *)KRML_HOST_MALLOC(sizeof ( + Hacl_Hash_Blake2b_Simd256_state_t + )); + p[0U] = s; + Hacl_Hash_Blake2b_blake2_params *p1 = key.fst; + uint8_t kk1 = p1->key_length; + uint8_t nn = p1->digest_length; + bool last_node = block_state.thd; + Hacl_Hash_Blake2b_index i = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + Lib_IntVector_Intrinsics_vec256 *h = block_state.f3.snd; + uint32_t kk20 = (uint32_t)i.key_length; + uint8_t *k_1 = key.snd; + if (!(kk20 == 0U)) + { + uint8_t *sub_b = buf + kk20; + memset(sub_b, 0U, (128U - kk20) * sizeof (uint8_t)); + memcpy(buf, k_1, kk20 * sizeof (uint8_t)); + } + Hacl_Hash_Blake2b_blake2_params pv = p1[0U]; + uint64_t tmp[8U] = { 0U }; + Lib_IntVector_Intrinsics_vec256 *r0 = h; + Lib_IntVector_Intrinsics_vec256 *r1 = h + 1U; + Lib_IntVector_Intrinsics_vec256 *r2 = h + 2U; + Lib_IntVector_Intrinsics_vec256 *r3 = h + 3U; + uint64_t iv0 = Hacl_Hash_Blake2b_ivTable_B[0U]; + uint64_t iv1 = Hacl_Hash_Blake2b_ivTable_B[1U]; + uint64_t iv2 = Hacl_Hash_Blake2b_ivTable_B[2U]; + uint64_t iv3 = Hacl_Hash_Blake2b_ivTable_B[3U]; + uint64_t iv4 = Hacl_Hash_Blake2b_ivTable_B[4U]; + uint64_t iv5 = Hacl_Hash_Blake2b_ivTable_B[5U]; + uint64_t iv6 = Hacl_Hash_Blake2b_ivTable_B[6U]; + uint64_t iv7 = Hacl_Hash_Blake2b_ivTable_B[7U]; + r2[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv0, iv1, iv2, iv3); + r3[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv4, iv5, iv6, iv7); + uint8_t kk2 = pv.key_length; + uint8_t nn1 = pv.digest_length; + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 4U; + uint8_t *bj = pv.salt + i0 * 8U; + uint64_t u = load64_le(bj); + uint64_t r4 = u; + uint64_t x = r4; + os[i0] = x;); + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 6U; + uint8_t *bj = pv.personal + i0 * 8U; + uint64_t u = load64_le(bj); + uint64_t r4 = u; + uint64_t x = r4; + os[i0] = x;); + tmp[0U] = + (uint64_t)nn1 + ^ + ((uint64_t)kk2 + << 8U + ^ ((uint64_t)pv.fanout << 16U ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); + tmp[1U] = pv.node_offset; + tmp[2U] = (uint64_t)pv.node_depth ^ (uint64_t)pv.inner_length << 8U; + tmp[3U] = 0ULL; + uint64_t tmp0 = tmp[0U]; + uint64_t tmp1 = tmp[1U]; + uint64_t tmp2 = tmp[2U]; + uint64_t tmp3 = tmp[3U]; + uint64_t tmp4 = tmp[4U]; + uint64_t tmp5 = tmp[5U]; + uint64_t tmp6 = tmp[6U]; + uint64_t tmp7 = tmp[7U]; + uint64_t iv0_ = iv0 ^ tmp0; + uint64_t iv1_ = iv1 ^ tmp1; + uint64_t iv2_ = iv2 ^ tmp2; + uint64_t iv3_ = iv3 ^ tmp3; + uint64_t iv4_ = iv4 ^ tmp4; + uint64_t iv5_ = iv5 ^ tmp5; + uint64_t iv6_ = iv6 ^ tmp6; + uint64_t iv7_ = iv7 ^ tmp7; + r0[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv0_, iv1_, iv2_, iv3_); + r1[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv4_, iv5_, iv6_, iv7_); + return p; +} + +/** + General-purpose allocation function that gives control over all +Blake2 parameters, including the key. Further resettings of the state SHALL be +done with `reset_with_params_and_key`, and SHALL feature the exact same values +for the `key_length` and `digest_length` fields as passed here. In other words, +once you commit to a digest and key length, the only way to change these +parameters is to allocate a new object. + +The caller must satisfy the following requirements. +- The length of the key k MUST match the value of the field key_length in the + parameters. +- The key_length must not exceed 256 for S, 64 for B. +- The digest_length must not exceed 256 for S, 64 for B. + +*/ +Hacl_Hash_Blake2b_Simd256_state_t +*Hacl_Hash_Blake2b_Simd256_malloc_with_params_and_key( + Hacl_Hash_Blake2b_blake2_params *p, + bool last_node, + uint8_t *k +) +{ + Hacl_Hash_Blake2b_blake2_params pv = p[0U]; + Hacl_Hash_Blake2b_index + i1 = { .key_length = pv.key_length, .digest_length = pv.digest_length, .last_node = last_node }; + return malloc_raw(i1, ((Hacl_Hash_Blake2b_params_and_key){ .fst = p, .snd = k })); +} + +/** + Specialized allocation function that picks default values for all +parameters, except for the key_length. Further resettings of the state SHALL be +done with `reset_with_key`, and SHALL feature the exact same key length `kk` as +passed here. In other words, once you commit to a key length, the only way to +change this parameter is to allocate a new object. + +The caller must satisfy the following requirements. +- The key_length must not exceed 256 for S, 64 for B. + +*/ +Hacl_Hash_Blake2b_Simd256_state_t +*Hacl_Hash_Blake2b_Simd256_malloc_with_key0(uint8_t *k, uint8_t kk) +{ + uint8_t nn = 64U; + Hacl_Hash_Blake2b_index i = { .key_length = kk, .digest_length = nn, .last_node = false }; + uint8_t salt[16U] = { 0U }; + uint8_t personal[16U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = i.digest_length, .key_length = i.key_length, .fanout = 1U, .depth = 1U, + .leaf_length = 0U, .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, + .personal = personal + }; + Hacl_Hash_Blake2b_blake2_params p0 = p; + Hacl_Hash_Blake2b_Simd256_state_t + *s = Hacl_Hash_Blake2b_Simd256_malloc_with_params_and_key(&p0, false, k); + return s; +} + +/** + Specialized allocation function that picks default values for all +parameters, and has no key. Effectively, this is what you want if you intend to +use Blake2 as a hash function. Further resettings of the state SHALL be done with `reset`. +*/ +Hacl_Hash_Blake2b_Simd256_state_t *Hacl_Hash_Blake2b_Simd256_malloc(void) +{ + return Hacl_Hash_Blake2b_Simd256_malloc_with_key0(NULL, 0U); +} + +static Hacl_Hash_Blake2b_index index_of_state(Hacl_Hash_Blake2b_Simd256_state_t *s) +{ + Hacl_Hash_Blake2b_Simd256_block_state_t block_state = (*s).block_state; + bool last_node = block_state.thd; + uint8_t nn = block_state.snd; + uint8_t kk1 = block_state.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk1, .digest_length = nn, .last_node = last_node }); +} + +static void +reset_raw(Hacl_Hash_Blake2b_Simd256_state_t *state, Hacl_Hash_Blake2b_params_and_key key) +{ + Hacl_Hash_Blake2b_Simd256_state_t scrut = *state; + uint8_t *buf = scrut.buf; + Hacl_Hash_Blake2b_Simd256_block_state_t block_state = scrut.block_state; + bool last_node0 = block_state.thd; + uint8_t nn0 = block_state.snd; + uint8_t kk10 = block_state.fst; + Hacl_Hash_Blake2b_index + i = { .key_length = kk10, .digest_length = nn0, .last_node = last_node0 }; + KRML_MAYBE_UNUSED_VAR(i); + Hacl_Hash_Blake2b_blake2_params *p = key.fst; + uint8_t kk1 = p->key_length; + uint8_t nn = p->digest_length; + bool last_node = block_state.thd; + Hacl_Hash_Blake2b_index + i1 = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + Lib_IntVector_Intrinsics_vec256 *h = block_state.f3.snd; + uint32_t kk20 = (uint32_t)i1.key_length; + uint8_t *k_1 = key.snd; + if (!(kk20 == 0U)) + { + uint8_t *sub_b = buf + kk20; + memset(sub_b, 0U, (128U - kk20) * sizeof (uint8_t)); + memcpy(buf, k_1, kk20 * sizeof (uint8_t)); + } + Hacl_Hash_Blake2b_blake2_params pv = p[0U]; + uint64_t tmp[8U] = { 0U }; + Lib_IntVector_Intrinsics_vec256 *r0 = h; + Lib_IntVector_Intrinsics_vec256 *r1 = h + 1U; + Lib_IntVector_Intrinsics_vec256 *r2 = h + 2U; + Lib_IntVector_Intrinsics_vec256 *r3 = h + 3U; + uint64_t iv0 = Hacl_Hash_Blake2b_ivTable_B[0U]; + uint64_t iv1 = Hacl_Hash_Blake2b_ivTable_B[1U]; + uint64_t iv2 = Hacl_Hash_Blake2b_ivTable_B[2U]; + uint64_t iv3 = Hacl_Hash_Blake2b_ivTable_B[3U]; + uint64_t iv4 = Hacl_Hash_Blake2b_ivTable_B[4U]; + uint64_t iv5 = Hacl_Hash_Blake2b_ivTable_B[5U]; + uint64_t iv6 = Hacl_Hash_Blake2b_ivTable_B[6U]; + uint64_t iv7 = Hacl_Hash_Blake2b_ivTable_B[7U]; + r2[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv0, iv1, iv2, iv3); + r3[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv4, iv5, iv6, iv7); + uint8_t kk2 = pv.key_length; + uint8_t nn1 = pv.digest_length; + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 4U; + uint8_t *bj = pv.salt + i0 * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i0] = x;); + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 6U; + uint8_t *bj = pv.personal + i0 * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i0] = x;); + tmp[0U] = + (uint64_t)nn1 + ^ + ((uint64_t)kk2 + << 8U + ^ ((uint64_t)pv.fanout << 16U ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); + tmp[1U] = pv.node_offset; + tmp[2U] = (uint64_t)pv.node_depth ^ (uint64_t)pv.inner_length << 8U; + tmp[3U] = 0ULL; + uint64_t tmp0 = tmp[0U]; + uint64_t tmp1 = tmp[1U]; + uint64_t tmp2 = tmp[2U]; + uint64_t tmp3 = tmp[3U]; + uint64_t tmp4 = tmp[4U]; + uint64_t tmp5 = tmp[5U]; + uint64_t tmp6 = tmp[6U]; + uint64_t tmp7 = tmp[7U]; + uint64_t iv0_ = iv0 ^ tmp0; + uint64_t iv1_ = iv1 ^ tmp1; + uint64_t iv2_ = iv2 ^ tmp2; + uint64_t iv3_ = iv3 ^ tmp3; + uint64_t iv4_ = iv4 ^ tmp4; + uint64_t iv5_ = iv5 ^ tmp5; + uint64_t iv6_ = iv6 ^ tmp6; + uint64_t iv7_ = iv7 ^ tmp7; + r0[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv0_, iv1_, iv2_, iv3_); + r1[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv4_, iv5_, iv6_, iv7_); + uint8_t kk11 = i.key_length; + uint32_t ite; + if (kk11 != 0U) + { + ite = 128U; + } + else + { + ite = 0U; + } + Hacl_Hash_Blake2b_Simd256_state_t + tmp8 = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)ite }; + state[0U] = tmp8; +} + +/** + General-purpose re-initialization function with parameters and +key. You cannot change digest_length, key_length, or last_node, meaning those values in +the parameters object must be the same as originally decided via one of the +malloc functions. All other values of the parameter can be changed. The behavior +is unspecified if you violate this precondition. +*/ +void +Hacl_Hash_Blake2b_Simd256_reset_with_key_and_params( + Hacl_Hash_Blake2b_Simd256_state_t *s, + Hacl_Hash_Blake2b_blake2_params *p, + uint8_t *k +) +{ + Hacl_Hash_Blake2b_index i1 = index_of_state(s); + KRML_MAYBE_UNUSED_VAR(i1); + reset_raw(s, ((Hacl_Hash_Blake2b_params_and_key){ .fst = p, .snd = k })); +} + +/** + Specialized-purpose re-initialization function with no parameters, +and a key. The key length must be the same as originally decided via your choice +of malloc function. All other parameters are reset to their default values. The +original call to malloc MUST have set digest_length to the default value. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2b_Simd256_reset_with_key(Hacl_Hash_Blake2b_Simd256_state_t *s, uint8_t *k) +{ + Hacl_Hash_Blake2b_index idx = index_of_state(s); + uint8_t salt[16U] = { 0U }; + uint8_t personal[16U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = idx.digest_length, .key_length = idx.key_length, .fanout = 1U, .depth = 1U, + .leaf_length = 0U, .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, + .personal = personal + }; + Hacl_Hash_Blake2b_blake2_params p0 = p; + reset_raw(s, ((Hacl_Hash_Blake2b_params_and_key){ .fst = &p0, .snd = k })); +} + +/** + Specialized-purpose re-initialization function with no parameters +and no key. This is what you want if you intend to use Blake2 as a hash +function. The key length and digest length must have been set to their +respective default values via your choice of malloc function (always true if you +used `malloc`). All other parameters are reset to their default values. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2b_Simd256_reset(Hacl_Hash_Blake2b_Simd256_state_t *s) +{ + Hacl_Hash_Blake2b_Simd256_reset_with_key(s, NULL); +} + +/** + Update function; 0 = success, 1 = max length exceeded +*/ +Hacl_Streaming_Types_error_code +Hacl_Hash_Blake2b_Simd256_update( + Hacl_Hash_Blake2b_Simd256_state_t *state, + uint8_t *chunk, + uint32_t chunk_len +) +{ + Hacl_Hash_Blake2b_Simd256_state_t s = *state; + uint64_t total_len = s.total_len; + if ((uint64_t)chunk_len > 0xffffffffffffffffULL - total_len) + { + return Hacl_Streaming_Types_MaximumLengthExceeded; + } + uint32_t sz; + if (total_len % (uint64_t)128U == 0ULL && total_len > 0ULL) + { + sz = 128U; + } + else + { + sz = (uint32_t)(total_len % (uint64_t)128U); + } + if (chunk_len <= 128U - sz) + { + Hacl_Hash_Blake2b_Simd256_state_t s1 = *state; + Hacl_Hash_Blake2b_Simd256_block_state_t block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)128U == 0ULL && total_len1 > 0ULL) + { + sz1 = 128U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)128U); + } + uint8_t *buf2 = buf + sz1; + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state + = + ( + (Hacl_Hash_Blake2b_Simd256_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len2 + } + ); + } + else if (sz == 0U) + { + Hacl_Hash_Blake2b_Simd256_state_t s1 = *state; + Hacl_Hash_Blake2b_Simd256_block_state_t block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)128U == 0ULL && total_len1 > 0ULL) + { + sz1 = 128U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)128U); + } + if (!(sz1 == 0U)) + { + uint64_t prevlen = total_len1 - (uint64_t)sz1; + K____Lib_IntVector_Intrinsics_vec256___Lib_IntVector_Intrinsics_vec256_ acc = block_state1.f3; + Lib_IntVector_Intrinsics_vec256 *wv = acc.fst; + Lib_IntVector_Intrinsics_vec256 *hash = acc.snd; + uint32_t nb = 1U; + Hacl_Hash_Blake2b_Simd256_update_multi(128U, + wv, + hash, + FStar_UInt128_uint64_to_uint128(prevlen), + buf, + nb); + } + uint32_t ite; + if ((uint64_t)chunk_len % (uint64_t)128U == 0ULL && (uint64_t)chunk_len > 0ULL) + { + ite = 128U; + } + else + { + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)128U); + } + uint32_t n_blocks = (chunk_len - ite) / 128U; + uint32_t data1_len = n_blocks * 128U; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; + K____Lib_IntVector_Intrinsics_vec256___Lib_IntVector_Intrinsics_vec256_ acc = block_state1.f3; + Lib_IntVector_Intrinsics_vec256 *wv = acc.fst; + Lib_IntVector_Intrinsics_vec256 *hash = acc.snd; + uint32_t nb = data1_len / 128U; + Hacl_Hash_Blake2b_Simd256_update_multi(data1_len, + wv, + hash, + FStar_UInt128_uint64_to_uint128(total_len1), + data1, + nb); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state + = + ( + (Hacl_Hash_Blake2b_Simd256_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)chunk_len + } + ); + } + else + { + uint32_t diff = 128U - sz; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Hash_Blake2b_Simd256_state_t s1 = *state; + Hacl_Hash_Blake2b_Simd256_block_state_t block_state10 = s1.block_state; + uint8_t *buf0 = s1.buf; + uint64_t total_len10 = s1.total_len; + uint32_t sz10; + if (total_len10 % (uint64_t)128U == 0ULL && total_len10 > 0ULL) + { + sz10 = 128U; + } + else + { + sz10 = (uint32_t)(total_len10 % (uint64_t)128U); + } + uint8_t *buf2 = buf0 + sz10; + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); + uint64_t total_len2 = total_len10 + (uint64_t)diff; + *state + = + ( + (Hacl_Hash_Blake2b_Simd256_state_t){ + .block_state = block_state10, + .buf = buf0, + .total_len = total_len2 + } + ); + Hacl_Hash_Blake2b_Simd256_state_t s10 = *state; + Hacl_Hash_Blake2b_Simd256_block_state_t block_state1 = s10.block_state; + uint8_t *buf = s10.buf; + uint64_t total_len1 = s10.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)128U == 0ULL && total_len1 > 0ULL) + { + sz1 = 128U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)128U); + } + if (!(sz1 == 0U)) + { + uint64_t prevlen = total_len1 - (uint64_t)sz1; + K____Lib_IntVector_Intrinsics_vec256___Lib_IntVector_Intrinsics_vec256_ acc = block_state1.f3; + Lib_IntVector_Intrinsics_vec256 *wv = acc.fst; + Lib_IntVector_Intrinsics_vec256 *hash = acc.snd; + uint32_t nb = 1U; + Hacl_Hash_Blake2b_Simd256_update_multi(128U, + wv, + hash, + FStar_UInt128_uint64_to_uint128(prevlen), + buf, + nb); + } + uint32_t ite; + if + ((uint64_t)(chunk_len - diff) % (uint64_t)128U == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL) + { + ite = 128U; + } + else + { + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)128U); + } + uint32_t n_blocks = (chunk_len - diff - ite) / 128U; + uint32_t data1_len = n_blocks * 128U; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; + K____Lib_IntVector_Intrinsics_vec256___Lib_IntVector_Intrinsics_vec256_ acc = block_state1.f3; + Lib_IntVector_Intrinsics_vec256 *wv = acc.fst; + Lib_IntVector_Intrinsics_vec256 *hash = acc.snd; + uint32_t nb = data1_len / 128U; + Hacl_Hash_Blake2b_Simd256_update_multi(data1_len, + wv, + hash, + FStar_UInt128_uint64_to_uint128(total_len1), + data1, + nb); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state + = + ( + (Hacl_Hash_Blake2b_Simd256_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)(chunk_len - diff) + } + ); + } + return Hacl_Streaming_Types_Success; +} + +/** + Digest function. This function expects the `output` array to hold +at least `digest_length` bytes, where `digest_length` was determined by your +choice of `malloc` function. Concretely, if you used `malloc` or +`malloc_with_key`, then the expected length is 256 for S, or 64 for B (default +digest length). If you used `malloc_with_params_and_key`, then the expected +length is whatever you chose for the `digest_length` field of your parameters. +For convenience, this function returns `digest_length`. When in doubt, callers +can pass an array of size HACL_BLAKE2B_256_OUT_BYTES, then use the return value +to see how many bytes were actually written. +*/ +uint8_t Hacl_Hash_Blake2b_Simd256_digest(Hacl_Hash_Blake2b_Simd256_state_t *s, uint8_t *dst) +{ + Hacl_Hash_Blake2b_Simd256_block_state_t block_state0 = (*s).block_state; + bool last_node0 = block_state0.thd; + uint8_t nn0 = block_state0.snd; + uint8_t kk0 = block_state0.fst; + Hacl_Hash_Blake2b_index + i1 = { .key_length = kk0, .digest_length = nn0, .last_node = last_node0 }; + Hacl_Hash_Blake2b_Simd256_state_t scrut = *s; + Hacl_Hash_Blake2b_Simd256_block_state_t block_state = scrut.block_state; + uint8_t *buf_ = scrut.buf; + uint64_t total_len = scrut.total_len; + uint32_t r; + if (total_len % (uint64_t)128U == 0ULL && total_len > 0ULL) + { + r = 128U; + } + else + { + r = (uint32_t)(total_len % (uint64_t)128U); + } + uint8_t *buf_1 = buf_; + KRML_PRE_ALIGN(32) Lib_IntVector_Intrinsics_vec256 wv0[4U] KRML_POST_ALIGN(32) = { 0U }; + KRML_PRE_ALIGN(32) Lib_IntVector_Intrinsics_vec256 b[4U] KRML_POST_ALIGN(32) = { 0U }; + Hacl_Hash_Blake2b_Simd256_block_state_t + tmp_block_state = + { + .fst = i1.key_length, + .snd = i1.digest_length, + .thd = i1.last_node, + .f3 = { .fst = wv0, .snd = b } + }; + Lib_IntVector_Intrinsics_vec256 *src_b = block_state.f3.snd; + Lib_IntVector_Intrinsics_vec256 *dst_b = tmp_block_state.f3.snd; + memcpy(dst_b, src_b, 4U * sizeof (Lib_IntVector_Intrinsics_vec256)); + uint64_t prev_len = total_len - (uint64_t)r; + uint32_t ite; + if (r % 128U == 0U && r > 0U) + { + ite = 128U; + } + else + { + ite = r % 128U; + } + uint8_t *buf_last = buf_1 + r - ite; + uint8_t *buf_multi = buf_1; + K____Lib_IntVector_Intrinsics_vec256___Lib_IntVector_Intrinsics_vec256_ + acc0 = tmp_block_state.f3; + Lib_IntVector_Intrinsics_vec256 *wv1 = acc0.fst; + Lib_IntVector_Intrinsics_vec256 *hash0 = acc0.snd; + uint32_t nb = 0U; + Hacl_Hash_Blake2b_Simd256_update_multi(0U, + wv1, + hash0, + FStar_UInt128_uint64_to_uint128(prev_len), + buf_multi, + nb); + uint64_t prev_len_last = total_len - (uint64_t)r; + K____Lib_IntVector_Intrinsics_vec256___Lib_IntVector_Intrinsics_vec256_ + acc = tmp_block_state.f3; + bool last_node1 = tmp_block_state.thd; + Lib_IntVector_Intrinsics_vec256 *wv = acc.fst; + Lib_IntVector_Intrinsics_vec256 *hash = acc.snd; + Hacl_Hash_Blake2b_Simd256_update_last(r, + wv, + hash, + last_node1, + FStar_UInt128_uint64_to_uint128(prev_len_last), + r, + buf_last); + uint8_t nn1 = tmp_block_state.snd; + Hacl_Hash_Blake2b_Simd256_finish((uint32_t)nn1, dst, tmp_block_state.f3.snd); + Hacl_Hash_Blake2b_Simd256_block_state_t block_state1 = (*s).block_state; + bool last_node = block_state1.thd; + uint8_t nn = block_state1.snd; + uint8_t kk = block_state1.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk, .digest_length = nn, .last_node = last_node }).digest_length; +} + +Hacl_Hash_Blake2b_index Hacl_Hash_Blake2b_Simd256_info(Hacl_Hash_Blake2b_Simd256_state_t *s) +{ + Hacl_Hash_Blake2b_Simd256_block_state_t block_state = (*s).block_state; + bool last_node = block_state.thd; + uint8_t nn = block_state.snd; + uint8_t kk = block_state.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk, .digest_length = nn, .last_node = last_node }); +} + +/** + Free state function when there is no key +*/ +void Hacl_Hash_Blake2b_Simd256_free(Hacl_Hash_Blake2b_Simd256_state_t *state) +{ + Hacl_Hash_Blake2b_Simd256_state_t scrut = *state; + uint8_t *buf = scrut.buf; + Hacl_Hash_Blake2b_Simd256_block_state_t block_state = scrut.block_state; + Lib_IntVector_Intrinsics_vec256 *b = block_state.f3.snd; + Lib_IntVector_Intrinsics_vec256 *wv = block_state.f3.fst; + KRML_ALIGNED_FREE(wv); + KRML_ALIGNED_FREE(b); + KRML_HOST_FREE(buf); + KRML_HOST_FREE(state); +} + +/** + Copying. This preserves all parameters. +*/ +Hacl_Hash_Blake2b_Simd256_state_t +*Hacl_Hash_Blake2b_Simd256_copy(Hacl_Hash_Blake2b_Simd256_state_t *state) +{ + Hacl_Hash_Blake2b_Simd256_state_t scrut = *state; + Hacl_Hash_Blake2b_Simd256_block_state_t block_state0 = scrut.block_state; + uint8_t *buf0 = scrut.buf; + uint64_t total_len0 = scrut.total_len; + bool last_node = block_state0.thd; + uint8_t nn = block_state0.snd; + uint8_t kk1 = block_state0.fst; + Hacl_Hash_Blake2b_index i = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(128U, sizeof (uint8_t)); + memcpy(buf, buf0, 128U * sizeof (uint8_t)); + Lib_IntVector_Intrinsics_vec256 + *wv = + (Lib_IntVector_Intrinsics_vec256 *)KRML_ALIGNED_MALLOC(32, + sizeof (Lib_IntVector_Intrinsics_vec256) * 4U); + memset(wv, 0U, 4U * sizeof (Lib_IntVector_Intrinsics_vec256)); + Lib_IntVector_Intrinsics_vec256 + *b = + (Lib_IntVector_Intrinsics_vec256 *)KRML_ALIGNED_MALLOC(32, + sizeof (Lib_IntVector_Intrinsics_vec256) * 4U); + memset(b, 0U, 4U * sizeof (Lib_IntVector_Intrinsics_vec256)); + Hacl_Hash_Blake2b_Simd256_block_state_t + block_state = + { + .fst = i.key_length, + .snd = i.digest_length, + .thd = i.last_node, + .f3 = { .fst = wv, .snd = b } + }; + Lib_IntVector_Intrinsics_vec256 *src_b = block_state0.f3.snd; + Lib_IntVector_Intrinsics_vec256 *dst_b = block_state.f3.snd; + memcpy(dst_b, src_b, 4U * sizeof (Lib_IntVector_Intrinsics_vec256)); + Hacl_Hash_Blake2b_Simd256_state_t + s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; + Hacl_Hash_Blake2b_Simd256_state_t + *p = + (Hacl_Hash_Blake2b_Simd256_state_t *)KRML_HOST_MALLOC(sizeof ( + Hacl_Hash_Blake2b_Simd256_state_t + )); + p[0U] = s; + return p; +} + +/** +Write the BLAKE2b digest of message `input` using key `key` into `output`. + +@param output Pointer to `output_len` bytes of memory where the digest is written to. +@param output_len Length of the to-be-generated digest with 1 <= `output_len` <= 64. +@param input Pointer to `input_len` bytes of memory where the input message is read from. +@param input_len Length of the input message. +@param key Pointer to `key_len` bytes of memory where the key is read from. +@param key_len Length of the key. Can be 0. +*/ +void +Hacl_Hash_Blake2b_Simd256_hash_with_key( + uint8_t *output, + uint32_t output_len, + uint8_t *input, + uint32_t input_len, + uint8_t *key, + uint32_t key_len +) +{ + KRML_PRE_ALIGN(32) Lib_IntVector_Intrinsics_vec256 b[4U] KRML_POST_ALIGN(32) = { 0U }; + KRML_PRE_ALIGN(32) Lib_IntVector_Intrinsics_vec256 b1[4U] KRML_POST_ALIGN(32) = { 0U }; + Hacl_Hash_Blake2b_Simd256_init(b, key_len, output_len); + update(b1, b, key_len, key, input_len, input); + Hacl_Hash_Blake2b_Simd256_finish(output_len, output, b); + Lib_Memzero0_memzero(b1, 4U, Lib_IntVector_Intrinsics_vec256, void *); + Lib_Memzero0_memzero(b, 4U, Lib_IntVector_Intrinsics_vec256, void *); +} + +/** +Write the BLAKE2b digest of message `input` using key `key` and +parameters `params` into `output`. The `key` array must be of length +`params.key_length`. The `output` array must be of length +`params.digest_length`. +*/ +void +Hacl_Hash_Blake2b_Simd256_hash_with_key_and_params( + uint8_t *output, + uint8_t *input, + uint32_t input_len, + Hacl_Hash_Blake2b_blake2_params params, + uint8_t *key +) +{ + KRML_PRE_ALIGN(32) Lib_IntVector_Intrinsics_vec256 b[4U] KRML_POST_ALIGN(32) = { 0U }; + KRML_PRE_ALIGN(32) Lib_IntVector_Intrinsics_vec256 b1[4U] KRML_POST_ALIGN(32) = { 0U }; + uint64_t tmp[8U] = { 0U }; + Lib_IntVector_Intrinsics_vec256 *r0 = b; + Lib_IntVector_Intrinsics_vec256 *r1 = b + 1U; + Lib_IntVector_Intrinsics_vec256 *r2 = b + 2U; + Lib_IntVector_Intrinsics_vec256 *r3 = b + 3U; + uint64_t iv0 = Hacl_Hash_Blake2b_ivTable_B[0U]; + uint64_t iv1 = Hacl_Hash_Blake2b_ivTable_B[1U]; + uint64_t iv2 = Hacl_Hash_Blake2b_ivTable_B[2U]; + uint64_t iv3 = Hacl_Hash_Blake2b_ivTable_B[3U]; + uint64_t iv4 = Hacl_Hash_Blake2b_ivTable_B[4U]; + uint64_t iv5 = Hacl_Hash_Blake2b_ivTable_B[5U]; + uint64_t iv6 = Hacl_Hash_Blake2b_ivTable_B[6U]; + uint64_t iv7 = Hacl_Hash_Blake2b_ivTable_B[7U]; + r2[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv0, iv1, iv2, iv3); + r3[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv4, iv5, iv6, iv7); + uint8_t kk = params.key_length; + uint8_t nn = params.digest_length; + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 4U; + uint8_t *bj = params.salt + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint64_t *os = tmp + 6U; + uint8_t *bj = params.personal + i * 8U; + uint64_t u = load64_le(bj); + uint64_t r = u; + uint64_t x = r; + os[i] = x;); + tmp[0U] = + (uint64_t)nn + ^ + ((uint64_t)kk + << 8U + ^ + ((uint64_t)params.fanout + << 16U + ^ ((uint64_t)params.depth << 24U ^ (uint64_t)params.leaf_length << 32U))); + tmp[1U] = params.node_offset; + tmp[2U] = (uint64_t)params.node_depth ^ (uint64_t)params.inner_length << 8U; + tmp[3U] = 0ULL; + uint64_t tmp0 = tmp[0U]; + uint64_t tmp1 = tmp[1U]; + uint64_t tmp2 = tmp[2U]; + uint64_t tmp3 = tmp[3U]; + uint64_t tmp4 = tmp[4U]; + uint64_t tmp5 = tmp[5U]; + uint64_t tmp6 = tmp[6U]; + uint64_t tmp7 = tmp[7U]; + uint64_t iv0_ = iv0 ^ tmp0; + uint64_t iv1_ = iv1 ^ tmp1; + uint64_t iv2_ = iv2 ^ tmp2; + uint64_t iv3_ = iv3 ^ tmp3; + uint64_t iv4_ = iv4 ^ tmp4; + uint64_t iv5_ = iv5 ^ tmp5; + uint64_t iv6_ = iv6 ^ tmp6; + uint64_t iv7_ = iv7 ^ tmp7; + r0[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv0_, iv1_, iv2_, iv3_); + r1[0U] = Lib_IntVector_Intrinsics_vec256_load64s(iv4_, iv5_, iv6_, iv7_); + update(b1, b, (uint32_t)params.key_length, key, input_len, input); + Hacl_Hash_Blake2b_Simd256_finish((uint32_t)params.digest_length, output, b); + Lib_Memzero0_memzero(b1, 4U, Lib_IntVector_Intrinsics_vec256, void *); + Lib_Memzero0_memzero(b, 4U, Lib_IntVector_Intrinsics_vec256, void *); +} + diff --git a/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.h b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.h new file mode 100644 index 00000000000000..6c11a4ba32134a --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.h @@ -0,0 +1,231 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __Hacl_Hash_Blake2b_Simd256_H +#define __Hacl_Hash_Blake2b_Simd256_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "python_hacl_namespaces.h" +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "Hacl_Streaming_Types.h" + +#include "Hacl_Hash_Blake2b.h" +#include "libintvector.h" + +#define HACL_HASH_BLAKE2B_SIMD256_BLOCK_BYTES (128U) + +#define HACL_HASH_BLAKE2B_SIMD256_OUT_BYTES (64U) + +#define HACL_HASH_BLAKE2B_SIMD256_KEY_BYTES (64U) + +#define HACL_HASH_BLAKE2B_SIMD256_SALT_BYTES (16U) + +#define HACL_HASH_BLAKE2B_SIMD256_PERSONAL_BYTES (16U) + +typedef struct K____Lib_IntVector_Intrinsics_vec256___Lib_IntVector_Intrinsics_vec256__s +{ + Lib_IntVector_Intrinsics_vec256 *fst; + Lib_IntVector_Intrinsics_vec256 *snd; +} +K____Lib_IntVector_Intrinsics_vec256___Lib_IntVector_Intrinsics_vec256_; + +typedef struct Hacl_Hash_Blake2b_Simd256_block_state_t_s +{ + uint8_t fst; + uint8_t snd; + bool thd; + K____Lib_IntVector_Intrinsics_vec256___Lib_IntVector_Intrinsics_vec256_ f3; +} +Hacl_Hash_Blake2b_Simd256_block_state_t; + +typedef struct Hacl_Hash_Blake2b_Simd256_state_t_s +{ + Hacl_Hash_Blake2b_Simd256_block_state_t block_state; + uint8_t *buf; + uint64_t total_len; +} +Hacl_Hash_Blake2b_Simd256_state_t; + +/** + General-purpose allocation function that gives control over all +Blake2 parameters, including the key. Further resettings of the state SHALL be +done with `reset_with_params_and_key`, and SHALL feature the exact same values +for the `key_length` and `digest_length` fields as passed here. In other words, +once you commit to a digest and key length, the only way to change these +parameters is to allocate a new object. + +The caller must satisfy the following requirements. +- The length of the key k MUST match the value of the field key_length in the + parameters. +- The key_length must not exceed 256 for S, 64 for B. +- The digest_length must not exceed 256 for S, 64 for B. + +*/ +Hacl_Hash_Blake2b_Simd256_state_t +*Hacl_Hash_Blake2b_Simd256_malloc_with_params_and_key( + Hacl_Hash_Blake2b_blake2_params *p, + bool last_node, + uint8_t *k +); + +/** + Specialized allocation function that picks default values for all +parameters, except for the key_length. Further resettings of the state SHALL be +done with `reset_with_key`, and SHALL feature the exact same key length `kk` as +passed here. In other words, once you commit to a key length, the only way to +change this parameter is to allocate a new object. + +The caller must satisfy the following requirements. +- The key_length must not exceed 256 for S, 64 for B. + +*/ +Hacl_Hash_Blake2b_Simd256_state_t +*Hacl_Hash_Blake2b_Simd256_malloc_with_key0(uint8_t *k, uint8_t kk); + +/** + Specialized allocation function that picks default values for all +parameters, and has no key. Effectively, this is what you want if you intend to +use Blake2 as a hash function. Further resettings of the state SHALL be done with `reset`. +*/ +Hacl_Hash_Blake2b_Simd256_state_t *Hacl_Hash_Blake2b_Simd256_malloc(void); + +/** + General-purpose re-initialization function with parameters and +key. You cannot change digest_length, key_length, or last_node, meaning those values in +the parameters object must be the same as originally decided via one of the +malloc functions. All other values of the parameter can be changed. The behavior +is unspecified if you violate this precondition. +*/ +void +Hacl_Hash_Blake2b_Simd256_reset_with_key_and_params( + Hacl_Hash_Blake2b_Simd256_state_t *s, + Hacl_Hash_Blake2b_blake2_params *p, + uint8_t *k +); + +/** + Specialized-purpose re-initialization function with no parameters, +and a key. The key length must be the same as originally decided via your choice +of malloc function. All other parameters are reset to their default values. The +original call to malloc MUST have set digest_length to the default value. The +behavior is unspecified if you violate this precondition. +*/ +void +Hacl_Hash_Blake2b_Simd256_reset_with_key(Hacl_Hash_Blake2b_Simd256_state_t *s, uint8_t *k); + +/** + Specialized-purpose re-initialization function with no parameters +and no key. This is what you want if you intend to use Blake2 as a hash +function. The key length and digest length must have been set to their +respective default values via your choice of malloc function (always true if you +used `malloc`). All other parameters are reset to their default values. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2b_Simd256_reset(Hacl_Hash_Blake2b_Simd256_state_t *s); + +/** + Update function; 0 = success, 1 = max length exceeded +*/ +Hacl_Streaming_Types_error_code +Hacl_Hash_Blake2b_Simd256_update( + Hacl_Hash_Blake2b_Simd256_state_t *state, + uint8_t *chunk, + uint32_t chunk_len +); + +/** + Digest function. This function expects the `output` array to hold +at least `digest_length` bytes, where `digest_length` was determined by your +choice of `malloc` function. Concretely, if you used `malloc` or +`malloc_with_key`, then the expected length is 256 for S, or 64 for B (default +digest length). If you used `malloc_with_params_and_key`, then the expected +length is whatever you chose for the `digest_length` field of your parameters. +For convenience, this function returns `digest_length`. When in doubt, callers +can pass an array of size HACL_BLAKE2B_256_OUT_BYTES, then use the return value +to see how many bytes were actually written. +*/ +uint8_t Hacl_Hash_Blake2b_Simd256_digest(Hacl_Hash_Blake2b_Simd256_state_t *s, uint8_t *dst); + +Hacl_Hash_Blake2b_index Hacl_Hash_Blake2b_Simd256_info(Hacl_Hash_Blake2b_Simd256_state_t *s); + +/** + Free state function when there is no key +*/ +void Hacl_Hash_Blake2b_Simd256_free(Hacl_Hash_Blake2b_Simd256_state_t *state); + +/** + Copying. This preserves all parameters. +*/ +Hacl_Hash_Blake2b_Simd256_state_t +*Hacl_Hash_Blake2b_Simd256_copy(Hacl_Hash_Blake2b_Simd256_state_t *state); + +/** +Write the BLAKE2b digest of message `input` using key `key` into `output`. + +@param output Pointer to `output_len` bytes of memory where the digest is written to. +@param output_len Length of the to-be-generated digest with 1 <= `output_len` <= 64. +@param input Pointer to `input_len` bytes of memory where the input message is read from. +@param input_len Length of the input message. +@param key Pointer to `key_len` bytes of memory where the key is read from. +@param key_len Length of the key. Can be 0. +*/ +void +Hacl_Hash_Blake2b_Simd256_hash_with_key( + uint8_t *output, + uint32_t output_len, + uint8_t *input, + uint32_t input_len, + uint8_t *key, + uint32_t key_len +); + +/** +Write the BLAKE2b digest of message `input` using key `key` and +parameters `params` into `output`. The `key` array must be of length +`params.key_length`. The `output` array must be of length +`params.digest_length`. +*/ +void +Hacl_Hash_Blake2b_Simd256_hash_with_key_and_params( + uint8_t *output, + uint8_t *input, + uint32_t input_len, + Hacl_Hash_Blake2b_blake2_params params, + uint8_t *key +); + +#if defined(__cplusplus) +} +#endif + +#define __Hacl_Hash_Blake2b_Simd256_H_DEFINED +#endif diff --git a/Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.c b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.c new file mode 100644 index 00000000000000..116499fedb3bd0 --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.c @@ -0,0 +1,15 @@ +// This file isn't part of a standard HACL source tree. +// +// It is required for compatibility with universal2 macOS builds. The code in +// Hacl_Hash_Blake2b_Simd256.c *will* compile on macOS x86_64, but *won't* +// compile on ARM64. However, because universal2 builds are compiled in a +// single pass, autoconf detects that the required compiler features *are* +// available, and tries to compile this file, which then fails because of the +// lack of support on ARM64. +// +// To compensate for this, autoconf will include *this* file instead of +// Hacl_Hash_Blake2b_Simd256.c when compiling for universal. This allows the +// underlying source code of HACL to remain unmodified. +#if !(defined(__APPLE__) && defined(__arm64__)) +#include "Hacl_Hash_Blake2b_Simd256.c" +#endif diff --git a/Modules/_hacl/Hacl_Hash_Blake2s.c b/Modules/_hacl/Hacl_Hash_Blake2s.c new file mode 100644 index 00000000000000..ceb7385072e048 --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_Blake2s.c @@ -0,0 +1,1514 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#include "internal/Hacl_Hash_Blake2s.h" + +#include "internal/Hacl_Impl_Blake2_Constants.h" +#include "internal/Hacl_Hash_Blake2b.h" +#include "lib_memzero0.h" + +static inline void +update_block( + uint32_t *wv, + uint32_t *hash, + bool flag, + bool last_node, + uint64_t totlen, + uint8_t *d +) +{ + uint32_t m_w[16U] = { 0U }; + KRML_MAYBE_FOR16(i, + 0U, + 16U, + 1U, + uint32_t *os = m_w; + uint8_t *bj = d + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + uint32_t mask[4U] = { 0U }; + uint32_t wv_14; + if (flag) + { + wv_14 = 0xFFFFFFFFU; + } + else + { + wv_14 = 0U; + } + uint32_t wv_15; + if (last_node) + { + wv_15 = 0xFFFFFFFFU; + } + else + { + wv_15 = 0U; + } + mask[0U] = (uint32_t)totlen; + mask[1U] = (uint32_t)(totlen >> 32U); + mask[2U] = wv_14; + mask[3U] = wv_15; + memcpy(wv, hash, 16U * sizeof (uint32_t)); + uint32_t *wv3 = wv + 12U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv3; + uint32_t x = wv3[i] ^ mask[i]; + os[i] = x;); + KRML_MAYBE_FOR10(i0, + 0U, + 10U, + 1U, + uint32_t start_idx = i0 % 10U * 16U; + uint32_t m_st[16U] = { 0U }; + uint32_t *r0 = m_st; + uint32_t *r1 = m_st + 4U; + uint32_t *r20 = m_st + 8U; + uint32_t *r30 = m_st + 12U; + uint32_t s0 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 0U]; + uint32_t s1 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 1U]; + uint32_t s2 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 2U]; + uint32_t s3 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 3U]; + uint32_t s4 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 4U]; + uint32_t s5 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 5U]; + uint32_t s6 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 6U]; + uint32_t s7 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 7U]; + uint32_t s8 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 8U]; + uint32_t s9 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 9U]; + uint32_t s10 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 10U]; + uint32_t s11 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 11U]; + uint32_t s12 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 12U]; + uint32_t s13 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 13U]; + uint32_t s14 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 14U]; + uint32_t s15 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 15U]; + uint32_t uu____0 = m_w[s2]; + uint32_t uu____1 = m_w[s4]; + uint32_t uu____2 = m_w[s6]; + r0[0U] = m_w[s0]; + r0[1U] = uu____0; + r0[2U] = uu____1; + r0[3U] = uu____2; + uint32_t uu____3 = m_w[s3]; + uint32_t uu____4 = m_w[s5]; + uint32_t uu____5 = m_w[s7]; + r1[0U] = m_w[s1]; + r1[1U] = uu____3; + r1[2U] = uu____4; + r1[3U] = uu____5; + uint32_t uu____6 = m_w[s10]; + uint32_t uu____7 = m_w[s12]; + uint32_t uu____8 = m_w[s14]; + r20[0U] = m_w[s8]; + r20[1U] = uu____6; + r20[2U] = uu____7; + r20[3U] = uu____8; + uint32_t uu____9 = m_w[s11]; + uint32_t uu____10 = m_w[s13]; + uint32_t uu____11 = m_w[s15]; + r30[0U] = m_w[s9]; + r30[1U] = uu____9; + r30[2U] = uu____10; + r30[3U] = uu____11; + uint32_t *x = m_st; + uint32_t *y = m_st + 4U; + uint32_t *z = m_st + 8U; + uint32_t *w = m_st + 12U; + uint32_t a = 0U; + uint32_t b0 = 1U; + uint32_t c0 = 2U; + uint32_t d10 = 3U; + uint32_t *wv_a0 = wv + a * 4U; + uint32_t *wv_b0 = wv + b0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a0; + uint32_t x1 = wv_a0[i] + wv_b0[i]; + os[i] = x1;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a0; + uint32_t x1 = wv_a0[i] + x[i]; + os[i] = x1;); + uint32_t *wv_a1 = wv + d10 * 4U; + uint32_t *wv_b1 = wv + a * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a1; + uint32_t x1 = wv_a1[i] ^ wv_b1[i]; + os[i] = x1;); + uint32_t *r10 = wv_a1; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = r10; + uint32_t x1 = r10[i]; + uint32_t x10 = x1 >> 16U | x1 << 16U; + os[i] = x10;); + uint32_t *wv_a2 = wv + c0 * 4U; + uint32_t *wv_b2 = wv + d10 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a2; + uint32_t x1 = wv_a2[i] + wv_b2[i]; + os[i] = x1;); + uint32_t *wv_a3 = wv + b0 * 4U; + uint32_t *wv_b3 = wv + c0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a3; + uint32_t x1 = wv_a3[i] ^ wv_b3[i]; + os[i] = x1;); + uint32_t *r12 = wv_a3; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = r12; + uint32_t x1 = r12[i]; + uint32_t x10 = x1 >> 12U | x1 << 20U; + os[i] = x10;); + uint32_t *wv_a4 = wv + a * 4U; + uint32_t *wv_b4 = wv + b0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a4; + uint32_t x1 = wv_a4[i] + wv_b4[i]; + os[i] = x1;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a4; + uint32_t x1 = wv_a4[i] + y[i]; + os[i] = x1;); + uint32_t *wv_a5 = wv + d10 * 4U; + uint32_t *wv_b5 = wv + a * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a5; + uint32_t x1 = wv_a5[i] ^ wv_b5[i]; + os[i] = x1;); + uint32_t *r13 = wv_a5; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = r13; + uint32_t x1 = r13[i]; + uint32_t x10 = x1 >> 8U | x1 << 24U; + os[i] = x10;); + uint32_t *wv_a6 = wv + c0 * 4U; + uint32_t *wv_b6 = wv + d10 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a6; + uint32_t x1 = wv_a6[i] + wv_b6[i]; + os[i] = x1;); + uint32_t *wv_a7 = wv + b0 * 4U; + uint32_t *wv_b7 = wv + c0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a7; + uint32_t x1 = wv_a7[i] ^ wv_b7[i]; + os[i] = x1;); + uint32_t *r14 = wv_a7; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = r14; + uint32_t x1 = r14[i]; + uint32_t x10 = x1 >> 7U | x1 << 25U; + os[i] = x10;); + uint32_t *r15 = wv + 4U; + uint32_t *r21 = wv + 8U; + uint32_t *r31 = wv + 12U; + uint32_t *r110 = r15; + uint32_t x00 = r110[1U]; + uint32_t x10 = r110[2U]; + uint32_t x20 = r110[3U]; + uint32_t x30 = r110[0U]; + r110[0U] = x00; + r110[1U] = x10; + r110[2U] = x20; + r110[3U] = x30; + uint32_t *r111 = r21; + uint32_t x01 = r111[2U]; + uint32_t x11 = r111[3U]; + uint32_t x21 = r111[0U]; + uint32_t x31 = r111[1U]; + r111[0U] = x01; + r111[1U] = x11; + r111[2U] = x21; + r111[3U] = x31; + uint32_t *r112 = r31; + uint32_t x02 = r112[3U]; + uint32_t x12 = r112[0U]; + uint32_t x22 = r112[1U]; + uint32_t x32 = r112[2U]; + r112[0U] = x02; + r112[1U] = x12; + r112[2U] = x22; + r112[3U] = x32; + uint32_t a0 = 0U; + uint32_t b = 1U; + uint32_t c = 2U; + uint32_t d1 = 3U; + uint32_t *wv_a = wv + a0 * 4U; + uint32_t *wv_b8 = wv + b * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a; + uint32_t x1 = wv_a[i] + wv_b8[i]; + os[i] = x1;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a; + uint32_t x1 = wv_a[i] + z[i]; + os[i] = x1;); + uint32_t *wv_a8 = wv + d1 * 4U; + uint32_t *wv_b9 = wv + a0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a8; + uint32_t x1 = wv_a8[i] ^ wv_b9[i]; + os[i] = x1;); + uint32_t *r16 = wv_a8; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = r16; + uint32_t x1 = r16[i]; + uint32_t x13 = x1 >> 16U | x1 << 16U; + os[i] = x13;); + uint32_t *wv_a9 = wv + c * 4U; + uint32_t *wv_b10 = wv + d1 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a9; + uint32_t x1 = wv_a9[i] + wv_b10[i]; + os[i] = x1;); + uint32_t *wv_a10 = wv + b * 4U; + uint32_t *wv_b11 = wv + c * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a10; + uint32_t x1 = wv_a10[i] ^ wv_b11[i]; + os[i] = x1;); + uint32_t *r17 = wv_a10; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = r17; + uint32_t x1 = r17[i]; + uint32_t x13 = x1 >> 12U | x1 << 20U; + os[i] = x13;); + uint32_t *wv_a11 = wv + a0 * 4U; + uint32_t *wv_b12 = wv + b * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a11; + uint32_t x1 = wv_a11[i] + wv_b12[i]; + os[i] = x1;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a11; + uint32_t x1 = wv_a11[i] + w[i]; + os[i] = x1;); + uint32_t *wv_a12 = wv + d1 * 4U; + uint32_t *wv_b13 = wv + a0 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a12; + uint32_t x1 = wv_a12[i] ^ wv_b13[i]; + os[i] = x1;); + uint32_t *r18 = wv_a12; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = r18; + uint32_t x1 = r18[i]; + uint32_t x13 = x1 >> 8U | x1 << 24U; + os[i] = x13;); + uint32_t *wv_a13 = wv + c * 4U; + uint32_t *wv_b14 = wv + d1 * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a13; + uint32_t x1 = wv_a13[i] + wv_b14[i]; + os[i] = x1;); + uint32_t *wv_a14 = wv + b * 4U; + uint32_t *wv_b = wv + c * 4U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = wv_a14; + uint32_t x1 = wv_a14[i] ^ wv_b[i]; + os[i] = x1;); + uint32_t *r19 = wv_a14; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = r19; + uint32_t x1 = r19[i]; + uint32_t x13 = x1 >> 7U | x1 << 25U; + os[i] = x13;); + uint32_t *r113 = wv + 4U; + uint32_t *r2 = wv + 8U; + uint32_t *r3 = wv + 12U; + uint32_t *r11 = r113; + uint32_t x03 = r11[3U]; + uint32_t x13 = r11[0U]; + uint32_t x23 = r11[1U]; + uint32_t x33 = r11[2U]; + r11[0U] = x03; + r11[1U] = x13; + r11[2U] = x23; + r11[3U] = x33; + uint32_t *r114 = r2; + uint32_t x04 = r114[2U]; + uint32_t x14 = r114[3U]; + uint32_t x24 = r114[0U]; + uint32_t x34 = r114[1U]; + r114[0U] = x04; + r114[1U] = x14; + r114[2U] = x24; + r114[3U] = x34; + uint32_t *r115 = r3; + uint32_t x0 = r115[1U]; + uint32_t x1 = r115[2U]; + uint32_t x2 = r115[3U]; + uint32_t x3 = r115[0U]; + r115[0U] = x0; + r115[1U] = x1; + r115[2U] = x2; + r115[3U] = x3;); + uint32_t *s0 = hash; + uint32_t *s1 = hash + 4U; + uint32_t *r0 = wv; + uint32_t *r1 = wv + 4U; + uint32_t *r2 = wv + 8U; + uint32_t *r3 = wv + 12U; + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = s0; + uint32_t x = s0[i] ^ r0[i]; + os[i] = x;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = s0; + uint32_t x = s0[i] ^ r2[i]; + os[i] = x;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = s1; + uint32_t x = s1[i] ^ r1[i]; + os[i] = x;); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = s1; + uint32_t x = s1[i] ^ r3[i]; + os[i] = x;); +} + +void Hacl_Hash_Blake2s_init(uint32_t *hash, uint32_t kk, uint32_t nn) +{ + uint8_t salt[8U] = { 0U }; + uint8_t personal[8U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = 32U, .key_length = 0U, .fanout = 1U, .depth = 1U, .leaf_length = 0U, + .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, .personal = personal + }; + uint32_t tmp[8U] = { 0U }; + uint32_t *r0 = hash; + uint32_t *r1 = hash + 4U; + uint32_t *r2 = hash + 8U; + uint32_t *r3 = hash + 12U; + uint32_t iv0 = Hacl_Hash_Blake2b_ivTable_S[0U]; + uint32_t iv1 = Hacl_Hash_Blake2b_ivTable_S[1U]; + uint32_t iv2 = Hacl_Hash_Blake2b_ivTable_S[2U]; + uint32_t iv3 = Hacl_Hash_Blake2b_ivTable_S[3U]; + uint32_t iv4 = Hacl_Hash_Blake2b_ivTable_S[4U]; + uint32_t iv5 = Hacl_Hash_Blake2b_ivTable_S[5U]; + uint32_t iv6 = Hacl_Hash_Blake2b_ivTable_S[6U]; + uint32_t iv7 = Hacl_Hash_Blake2b_ivTable_S[7U]; + r2[0U] = iv0; + r2[1U] = iv1; + r2[2U] = iv2; + r2[3U] = iv3; + r3[0U] = iv4; + r3[1U] = iv5; + r3[2U] = iv6; + r3[3U] = iv7; + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 4U; + uint8_t *bj = p.salt + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 6U; + uint8_t *bj = p.personal + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + tmp[0U] = + (uint32_t)(uint8_t)nn + ^ ((uint32_t)(uint8_t)kk << 8U ^ ((uint32_t)p.fanout << 16U ^ (uint32_t)p.depth << 24U)); + tmp[1U] = p.leaf_length; + tmp[2U] = (uint32_t)p.node_offset; + tmp[3U] = + (uint32_t)(p.node_offset >> 32U) + ^ ((uint32_t)p.node_depth << 16U ^ (uint32_t)p.inner_length << 24U); + uint32_t tmp0 = tmp[0U]; + uint32_t tmp1 = tmp[1U]; + uint32_t tmp2 = tmp[2U]; + uint32_t tmp3 = tmp[3U]; + uint32_t tmp4 = tmp[4U]; + uint32_t tmp5 = tmp[5U]; + uint32_t tmp6 = tmp[6U]; + uint32_t tmp7 = tmp[7U]; + uint32_t iv0_ = iv0 ^ tmp0; + uint32_t iv1_ = iv1 ^ tmp1; + uint32_t iv2_ = iv2 ^ tmp2; + uint32_t iv3_ = iv3 ^ tmp3; + uint32_t iv4_ = iv4 ^ tmp4; + uint32_t iv5_ = iv5 ^ tmp5; + uint32_t iv6_ = iv6 ^ tmp6; + uint32_t iv7_ = iv7 ^ tmp7; + r0[0U] = iv0_; + r0[1U] = iv1_; + r0[2U] = iv2_; + r0[3U] = iv3_; + r1[0U] = iv4_; + r1[1U] = iv5_; + r1[2U] = iv6_; + r1[3U] = iv7_; +} + +static void update_key(uint32_t *wv, uint32_t *hash, uint32_t kk, uint8_t *k, uint32_t ll) +{ + uint64_t lb = (uint64_t)64U; + uint8_t b[64U] = { 0U }; + memcpy(b, k, kk * sizeof (uint8_t)); + if (ll == 0U) + { + update_block(wv, hash, true, false, lb, b); + } + else + { + update_block(wv, hash, false, false, lb, b); + } + Lib_Memzero0_memzero(b, 64U, uint8_t, void *); +} + +void +Hacl_Hash_Blake2s_update_multi( + uint32_t len, + uint32_t *wv, + uint32_t *hash, + uint64_t prev, + uint8_t *blocks, + uint32_t nb +) +{ + KRML_MAYBE_UNUSED_VAR(len); + for (uint32_t i = 0U; i < nb; i++) + { + uint64_t totlen = prev + (uint64_t)((i + 1U) * 64U); + uint8_t *b = blocks + i * 64U; + update_block(wv, hash, false, false, totlen, b); + } +} + +void +Hacl_Hash_Blake2s_update_last( + uint32_t len, + uint32_t *wv, + uint32_t *hash, + bool last_node, + uint64_t prev, + uint32_t rem, + uint8_t *d +) +{ + uint8_t b[64U] = { 0U }; + uint8_t *last = d + len - rem; + memcpy(b, last, rem * sizeof (uint8_t)); + uint64_t totlen = prev + (uint64_t)len; + update_block(wv, hash, true, last_node, totlen, b); + Lib_Memzero0_memzero(b, 64U, uint8_t, void *); +} + +static void +update_blocks(uint32_t len, uint32_t *wv, uint32_t *hash, uint64_t prev, uint8_t *blocks) +{ + uint32_t nb0 = len / 64U; + uint32_t rem0 = len % 64U; + uint32_t nb; + if (rem0 == 0U && nb0 > 0U) + { + nb = nb0 - 1U; + } + else + { + nb = nb0; + } + uint32_t rem; + if (rem0 == 0U && nb0 > 0U) + { + rem = 64U; + } + else + { + rem = rem0; + } + Hacl_Hash_Blake2s_update_multi(len, wv, hash, prev, blocks, nb); + Hacl_Hash_Blake2s_update_last(len, wv, hash, false, prev, rem, blocks); +} + +static inline void +update(uint32_t *wv, uint32_t *hash, uint32_t kk, uint8_t *k, uint32_t ll, uint8_t *d) +{ + uint64_t lb = (uint64_t)64U; + if (kk > 0U) + { + update_key(wv, hash, kk, k, ll); + if (!(ll == 0U)) + { + update_blocks(ll, wv, hash, lb, d); + return; + } + return; + } + update_blocks(ll, wv, hash, (uint64_t)0U, d); +} + +void Hacl_Hash_Blake2s_finish(uint32_t nn, uint8_t *output, uint32_t *hash) +{ + uint8_t b[32U] = { 0U }; + uint8_t *first = b; + uint8_t *second = b + 16U; + uint32_t *row0 = hash; + uint32_t *row1 = hash + 4U; + KRML_MAYBE_FOR4(i, 0U, 4U, 1U, store32_le(first + i * 4U, row0[i]);); + KRML_MAYBE_FOR4(i, 0U, 4U, 1U, store32_le(second + i * 4U, row1[i]);); + uint8_t *final = b; + memcpy(output, final, nn * sizeof (uint8_t)); + Lib_Memzero0_memzero(b, 32U, uint8_t, void *); +} + +static Hacl_Hash_Blake2s_state_t +*malloc_raw(Hacl_Hash_Blake2b_index kk, Hacl_Hash_Blake2b_params_and_key key) +{ + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + uint32_t *wv = (uint32_t *)KRML_HOST_CALLOC(16U, sizeof (uint32_t)); + uint32_t *b = (uint32_t *)KRML_HOST_CALLOC(16U, sizeof (uint32_t)); + Hacl_Hash_Blake2s_block_state_t + block_state = + { + .fst = kk.key_length, + .snd = kk.digest_length, + .thd = kk.last_node, + .f3 = { .fst = wv, .snd = b } + }; + uint8_t kk10 = kk.key_length; + uint32_t ite; + if (kk10 != 0U) + { + ite = 64U; + } + else + { + ite = 0U; + } + Hacl_Hash_Blake2s_state_t + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)ite }; + Hacl_Hash_Blake2s_state_t + *p = (Hacl_Hash_Blake2s_state_t *)KRML_HOST_MALLOC(sizeof (Hacl_Hash_Blake2s_state_t)); + p[0U] = s; + Hacl_Hash_Blake2b_blake2_params *p1 = key.fst; + uint8_t kk1 = p1->key_length; + uint8_t nn = p1->digest_length; + bool last_node = block_state.thd; + Hacl_Hash_Blake2b_index i = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + uint32_t *h = block_state.f3.snd; + uint32_t kk2 = (uint32_t)i.key_length; + uint8_t *k_1 = key.snd; + if (!(kk2 == 0U)) + { + uint8_t *sub_b = buf + kk2; + memset(sub_b, 0U, (64U - kk2) * sizeof (uint8_t)); + memcpy(buf, k_1, kk2 * sizeof (uint8_t)); + } + Hacl_Hash_Blake2b_blake2_params pv = p1[0U]; + uint32_t tmp[8U] = { 0U }; + uint32_t *r0 = h; + uint32_t *r1 = h + 4U; + uint32_t *r2 = h + 8U; + uint32_t *r3 = h + 12U; + uint32_t iv0 = Hacl_Hash_Blake2b_ivTable_S[0U]; + uint32_t iv1 = Hacl_Hash_Blake2b_ivTable_S[1U]; + uint32_t iv2 = Hacl_Hash_Blake2b_ivTable_S[2U]; + uint32_t iv3 = Hacl_Hash_Blake2b_ivTable_S[3U]; + uint32_t iv4 = Hacl_Hash_Blake2b_ivTable_S[4U]; + uint32_t iv5 = Hacl_Hash_Blake2b_ivTable_S[5U]; + uint32_t iv6 = Hacl_Hash_Blake2b_ivTable_S[6U]; + uint32_t iv7 = Hacl_Hash_Blake2b_ivTable_S[7U]; + r2[0U] = iv0; + r2[1U] = iv1; + r2[2U] = iv2; + r2[3U] = iv3; + r3[0U] = iv4; + r3[1U] = iv5; + r3[2U] = iv6; + r3[3U] = iv7; + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 4U; + uint8_t *bj = pv.salt + i0 * 4U; + uint32_t u = load32_le(bj); + uint32_t r4 = u; + uint32_t x = r4; + os[i0] = x;); + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 6U; + uint8_t *bj = pv.personal + i0 * 4U; + uint32_t u = load32_le(bj); + uint32_t r4 = u; + uint32_t x = r4; + os[i0] = x;); + tmp[0U] = + (uint32_t)pv.digest_length + ^ ((uint32_t)pv.key_length << 8U ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); + tmp[1U] = pv.leaf_length; + tmp[2U] = (uint32_t)pv.node_offset; + tmp[3U] = + (uint32_t)(pv.node_offset >> 32U) + ^ ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); + uint32_t tmp0 = tmp[0U]; + uint32_t tmp1 = tmp[1U]; + uint32_t tmp2 = tmp[2U]; + uint32_t tmp3 = tmp[3U]; + uint32_t tmp4 = tmp[4U]; + uint32_t tmp5 = tmp[5U]; + uint32_t tmp6 = tmp[6U]; + uint32_t tmp7 = tmp[7U]; + uint32_t iv0_ = iv0 ^ tmp0; + uint32_t iv1_ = iv1 ^ tmp1; + uint32_t iv2_ = iv2 ^ tmp2; + uint32_t iv3_ = iv3 ^ tmp3; + uint32_t iv4_ = iv4 ^ tmp4; + uint32_t iv5_ = iv5 ^ tmp5; + uint32_t iv6_ = iv6 ^ tmp6; + uint32_t iv7_ = iv7 ^ tmp7; + r0[0U] = iv0_; + r0[1U] = iv1_; + r0[2U] = iv2_; + r0[3U] = iv3_; + r1[0U] = iv4_; + r1[1U] = iv5_; + r1[2U] = iv6_; + r1[3U] = iv7_; + return p; +} + +/** + General-purpose allocation function that gives control over all +Blake2 parameters, including the key. Further resettings of the state SHALL be +done with `reset_with_params_and_key`, and SHALL feature the exact same values +for the `key_length` and `digest_length` fields as passed here. In other words, +once you commit to a digest and key length, the only way to change these +parameters is to allocate a new object. + +The caller must satisfy the following requirements. +- The length of the key k MUST match the value of the field key_length in the + parameters. +- The key_length must not exceed 32 for S, 64 for B. +- The digest_length must not exceed 32 for S, 64 for B. + +*/ +Hacl_Hash_Blake2s_state_t +*Hacl_Hash_Blake2s_malloc_with_params_and_key( + Hacl_Hash_Blake2b_blake2_params *p, + bool last_node, + uint8_t *k +) +{ + Hacl_Hash_Blake2b_blake2_params pv = p[0U]; + Hacl_Hash_Blake2b_index + i1 = { .key_length = pv.key_length, .digest_length = pv.digest_length, .last_node = last_node }; + return malloc_raw(i1, ((Hacl_Hash_Blake2b_params_and_key){ .fst = p, .snd = k })); +} + +/** + Specialized allocation function that picks default values for all +parameters, except for the key_length. Further resettings of the state SHALL be +done with `reset_with_key`, and SHALL feature the exact same key length `kk` as +passed here. In other words, once you commit to a key length, the only way to +change this parameter is to allocate a new object. + +The caller must satisfy the following requirements. +- The key_length must not exceed 32 for S, 64 for B. + +*/ +Hacl_Hash_Blake2s_state_t *Hacl_Hash_Blake2s_malloc_with_key(uint8_t *k, uint8_t kk) +{ + uint8_t nn = 32U; + Hacl_Hash_Blake2b_index i = { .key_length = kk, .digest_length = nn, .last_node = false }; + uint8_t salt[8U] = { 0U }; + uint8_t personal[8U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = i.digest_length, .key_length = i.key_length, .fanout = 1U, .depth = 1U, + .leaf_length = 0U, .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, + .personal = personal + }; + Hacl_Hash_Blake2b_blake2_params p0 = p; + Hacl_Hash_Blake2s_state_t *s = Hacl_Hash_Blake2s_malloc_with_params_and_key(&p0, false, k); + return s; +} + +/** + Specialized allocation function that picks default values for all +parameters, and has no key. Effectively, this is what you want if you intend to +use Blake2 as a hash function. Further resettings of the state SHALL be done with `reset`. +*/ +Hacl_Hash_Blake2s_state_t *Hacl_Hash_Blake2s_malloc(void) +{ + return Hacl_Hash_Blake2s_malloc_with_key(NULL, 0U); +} + +static Hacl_Hash_Blake2b_index index_of_state(Hacl_Hash_Blake2s_state_t *s) +{ + Hacl_Hash_Blake2s_block_state_t block_state = (*s).block_state; + bool last_node = block_state.thd; + uint8_t nn = block_state.snd; + uint8_t kk1 = block_state.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk1, .digest_length = nn, .last_node = last_node }); +} + +static void reset_raw(Hacl_Hash_Blake2s_state_t *state, Hacl_Hash_Blake2b_params_and_key key) +{ + Hacl_Hash_Blake2s_state_t scrut = *state; + uint8_t *buf = scrut.buf; + Hacl_Hash_Blake2s_block_state_t block_state = scrut.block_state; + bool last_node0 = block_state.thd; + uint8_t nn0 = block_state.snd; + uint8_t kk10 = block_state.fst; + Hacl_Hash_Blake2b_index + i = { .key_length = kk10, .digest_length = nn0, .last_node = last_node0 }; + KRML_MAYBE_UNUSED_VAR(i); + Hacl_Hash_Blake2b_blake2_params *p = key.fst; + uint8_t kk1 = p->key_length; + uint8_t nn = p->digest_length; + bool last_node = block_state.thd; + Hacl_Hash_Blake2b_index + i1 = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + uint32_t *h = block_state.f3.snd; + uint32_t kk2 = (uint32_t)i1.key_length; + uint8_t *k_1 = key.snd; + if (!(kk2 == 0U)) + { + uint8_t *sub_b = buf + kk2; + memset(sub_b, 0U, (64U - kk2) * sizeof (uint8_t)); + memcpy(buf, k_1, kk2 * sizeof (uint8_t)); + } + Hacl_Hash_Blake2b_blake2_params pv = p[0U]; + uint32_t tmp[8U] = { 0U }; + uint32_t *r0 = h; + uint32_t *r1 = h + 4U; + uint32_t *r2 = h + 8U; + uint32_t *r3 = h + 12U; + uint32_t iv0 = Hacl_Hash_Blake2b_ivTable_S[0U]; + uint32_t iv1 = Hacl_Hash_Blake2b_ivTable_S[1U]; + uint32_t iv2 = Hacl_Hash_Blake2b_ivTable_S[2U]; + uint32_t iv3 = Hacl_Hash_Blake2b_ivTable_S[3U]; + uint32_t iv4 = Hacl_Hash_Blake2b_ivTable_S[4U]; + uint32_t iv5 = Hacl_Hash_Blake2b_ivTable_S[5U]; + uint32_t iv6 = Hacl_Hash_Blake2b_ivTable_S[6U]; + uint32_t iv7 = Hacl_Hash_Blake2b_ivTable_S[7U]; + r2[0U] = iv0; + r2[1U] = iv1; + r2[2U] = iv2; + r2[3U] = iv3; + r3[0U] = iv4; + r3[1U] = iv5; + r3[2U] = iv6; + r3[3U] = iv7; + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 4U; + uint8_t *bj = pv.salt + i0 * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i0] = x;); + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 6U; + uint8_t *bj = pv.personal + i0 * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i0] = x;); + tmp[0U] = + (uint32_t)pv.digest_length + ^ ((uint32_t)pv.key_length << 8U ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); + tmp[1U] = pv.leaf_length; + tmp[2U] = (uint32_t)pv.node_offset; + tmp[3U] = + (uint32_t)(pv.node_offset >> 32U) + ^ ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); + uint32_t tmp0 = tmp[0U]; + uint32_t tmp1 = tmp[1U]; + uint32_t tmp2 = tmp[2U]; + uint32_t tmp3 = tmp[3U]; + uint32_t tmp4 = tmp[4U]; + uint32_t tmp5 = tmp[5U]; + uint32_t tmp6 = tmp[6U]; + uint32_t tmp7 = tmp[7U]; + uint32_t iv0_ = iv0 ^ tmp0; + uint32_t iv1_ = iv1 ^ tmp1; + uint32_t iv2_ = iv2 ^ tmp2; + uint32_t iv3_ = iv3 ^ tmp3; + uint32_t iv4_ = iv4 ^ tmp4; + uint32_t iv5_ = iv5 ^ tmp5; + uint32_t iv6_ = iv6 ^ tmp6; + uint32_t iv7_ = iv7 ^ tmp7; + r0[0U] = iv0_; + r0[1U] = iv1_; + r0[2U] = iv2_; + r0[3U] = iv3_; + r1[0U] = iv4_; + r1[1U] = iv5_; + r1[2U] = iv6_; + r1[3U] = iv7_; + uint8_t kk11 = i.key_length; + uint32_t ite; + if (kk11 != 0U) + { + ite = 64U; + } + else + { + ite = 0U; + } + Hacl_Hash_Blake2s_state_t + tmp8 = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)ite }; + state[0U] = tmp8; +} + +/** + General-purpose re-initialization function with parameters and +key. You cannot change digest_length, key_length, or last_node, meaning those values in +the parameters object must be the same as originally decided via one of the +malloc functions. All other values of the parameter can be changed. The behavior +is unspecified if you violate this precondition. +*/ +void +Hacl_Hash_Blake2s_reset_with_key_and_params( + Hacl_Hash_Blake2s_state_t *s, + Hacl_Hash_Blake2b_blake2_params *p, + uint8_t *k +) +{ + Hacl_Hash_Blake2b_index i1 = index_of_state(s); + KRML_MAYBE_UNUSED_VAR(i1); + reset_raw(s, ((Hacl_Hash_Blake2b_params_and_key){ .fst = p, .snd = k })); +} + +/** + Specialized-purpose re-initialization function with no parameters, +and a key. The key length must be the same as originally decided via your choice +of malloc function. All other parameters are reset to their default values. The +original call to malloc MUST have set digest_length to the default value. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2s_reset_with_key(Hacl_Hash_Blake2s_state_t *s, uint8_t *k) +{ + Hacl_Hash_Blake2b_index idx = index_of_state(s); + uint8_t salt[8U] = { 0U }; + uint8_t personal[8U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = idx.digest_length, .key_length = idx.key_length, .fanout = 1U, .depth = 1U, + .leaf_length = 0U, .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, + .personal = personal + }; + Hacl_Hash_Blake2b_blake2_params p0 = p; + reset_raw(s, ((Hacl_Hash_Blake2b_params_and_key){ .fst = &p0, .snd = k })); +} + +/** + Specialized-purpose re-initialization function with no parameters +and no key. This is what you want if you intend to use Blake2 as a hash +function. The key length and digest length must have been set to their +respective default values via your choice of malloc function (always true if you +used `malloc`). All other parameters are reset to their default values. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2s_reset(Hacl_Hash_Blake2s_state_t *s) +{ + Hacl_Hash_Blake2s_reset_with_key(s, NULL); +} + +/** + Update function; 0 = success, 1 = max length exceeded +*/ +Hacl_Streaming_Types_error_code +Hacl_Hash_Blake2s_update(Hacl_Hash_Blake2s_state_t *state, uint8_t *chunk, uint32_t chunk_len) +{ + Hacl_Hash_Blake2s_state_t s = *state; + uint64_t total_len = s.total_len; + if ((uint64_t)chunk_len > 0xffffffffffffffffULL - total_len) + { + return Hacl_Streaming_Types_MaximumLengthExceeded; + } + uint32_t sz; + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) + { + sz = 64U; + } + else + { + sz = (uint32_t)(total_len % (uint64_t)64U); + } + if (chunk_len <= 64U - sz) + { + Hacl_Hash_Blake2s_state_t s1 = *state; + Hacl_Hash_Blake2s_block_state_t block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) + { + sz1 = 64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); + } + uint8_t *buf2 = buf + sz1; + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state + = + ( + (Hacl_Hash_Blake2s_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len2 + } + ); + } + else if (sz == 0U) + { + Hacl_Hash_Blake2s_state_t s1 = *state; + Hacl_Hash_Blake2s_block_state_t block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) + { + sz1 = 64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); + } + if (!(sz1 == 0U)) + { + uint64_t prevlen = total_len1 - (uint64_t)sz1; + K____uint32_t___uint32_t_ acc = block_state1.f3; + uint32_t *wv = acc.fst; + uint32_t *hash = acc.snd; + uint32_t nb = 1U; + Hacl_Hash_Blake2s_update_multi(64U, wv, hash, prevlen, buf, nb); + } + uint32_t ite; + if ((uint64_t)chunk_len % (uint64_t)64U == 0ULL && (uint64_t)chunk_len > 0ULL) + { + ite = 64U; + } + else + { + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)64U); + } + uint32_t n_blocks = (chunk_len - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; + K____uint32_t___uint32_t_ acc = block_state1.f3; + uint32_t *wv = acc.fst; + uint32_t *hash = acc.snd; + uint32_t nb = data1_len / 64U; + Hacl_Hash_Blake2s_update_multi(data1_len, wv, hash, total_len1, data1, nb); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state + = + ( + (Hacl_Hash_Blake2s_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)chunk_len + } + ); + } + else + { + uint32_t diff = 64U - sz; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Hash_Blake2s_state_t s1 = *state; + Hacl_Hash_Blake2s_block_state_t block_state10 = s1.block_state; + uint8_t *buf0 = s1.buf; + uint64_t total_len10 = s1.total_len; + uint32_t sz10; + if (total_len10 % (uint64_t)64U == 0ULL && total_len10 > 0ULL) + { + sz10 = 64U; + } + else + { + sz10 = (uint32_t)(total_len10 % (uint64_t)64U); + } + uint8_t *buf2 = buf0 + sz10; + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); + uint64_t total_len2 = total_len10 + (uint64_t)diff; + *state + = + ( + (Hacl_Hash_Blake2s_state_t){ + .block_state = block_state10, + .buf = buf0, + .total_len = total_len2 + } + ); + Hacl_Hash_Blake2s_state_t s10 = *state; + Hacl_Hash_Blake2s_block_state_t block_state1 = s10.block_state; + uint8_t *buf = s10.buf; + uint64_t total_len1 = s10.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) + { + sz1 = 64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); + } + if (!(sz1 == 0U)) + { + uint64_t prevlen = total_len1 - (uint64_t)sz1; + K____uint32_t___uint32_t_ acc = block_state1.f3; + uint32_t *wv = acc.fst; + uint32_t *hash = acc.snd; + uint32_t nb = 1U; + Hacl_Hash_Blake2s_update_multi(64U, wv, hash, prevlen, buf, nb); + } + uint32_t ite; + if + ((uint64_t)(chunk_len - diff) % (uint64_t)64U == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL) + { + ite = 64U; + } + else + { + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)64U); + } + uint32_t n_blocks = (chunk_len - diff - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; + K____uint32_t___uint32_t_ acc = block_state1.f3; + uint32_t *wv = acc.fst; + uint32_t *hash = acc.snd; + uint32_t nb = data1_len / 64U; + Hacl_Hash_Blake2s_update_multi(data1_len, wv, hash, total_len1, data1, nb); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state + = + ( + (Hacl_Hash_Blake2s_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)(chunk_len - diff) + } + ); + } + return Hacl_Streaming_Types_Success; +} + +/** + Digest function. This function expects the `output` array to hold +at least `digest_length` bytes, where `digest_length` was determined by your +choice of `malloc` function. Concretely, if you used `malloc` or +`malloc_with_key`, then the expected length is 32 for S, or 64 for B (default +digest length). If you used `malloc_with_params_and_key`, then the expected +length is whatever you chose for the `digest_length` field of your parameters. +For convenience, this function returns `digest_length`. When in doubt, callers +can pass an array of size HACL_BLAKE2S_32_OUT_BYTES, then use the return value +to see how many bytes were actually written. +*/ +uint8_t Hacl_Hash_Blake2s_digest(Hacl_Hash_Blake2s_state_t *s, uint8_t *dst) +{ + Hacl_Hash_Blake2s_block_state_t block_state0 = (*s).block_state; + bool last_node0 = block_state0.thd; + uint8_t nn0 = block_state0.snd; + uint8_t kk0 = block_state0.fst; + Hacl_Hash_Blake2b_index + i1 = { .key_length = kk0, .digest_length = nn0, .last_node = last_node0 }; + Hacl_Hash_Blake2s_state_t scrut = *s; + Hacl_Hash_Blake2s_block_state_t block_state = scrut.block_state; + uint8_t *buf_ = scrut.buf; + uint64_t total_len = scrut.total_len; + uint32_t r; + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) + { + r = 64U; + } + else + { + r = (uint32_t)(total_len % (uint64_t)64U); + } + uint8_t *buf_1 = buf_; + uint32_t wv0[16U] = { 0U }; + uint32_t b[16U] = { 0U }; + Hacl_Hash_Blake2s_block_state_t + tmp_block_state = + { + .fst = i1.key_length, + .snd = i1.digest_length, + .thd = i1.last_node, + .f3 = { .fst = wv0, .snd = b } + }; + uint32_t *src_b = block_state.f3.snd; + uint32_t *dst_b = tmp_block_state.f3.snd; + memcpy(dst_b, src_b, 16U * sizeof (uint32_t)); + uint64_t prev_len = total_len - (uint64_t)r; + uint32_t ite; + if (r % 64U == 0U && r > 0U) + { + ite = 64U; + } + else + { + ite = r % 64U; + } + uint8_t *buf_last = buf_1 + r - ite; + uint8_t *buf_multi = buf_1; + K____uint32_t___uint32_t_ acc0 = tmp_block_state.f3; + uint32_t *wv1 = acc0.fst; + uint32_t *hash0 = acc0.snd; + uint32_t nb = 0U; + Hacl_Hash_Blake2s_update_multi(0U, wv1, hash0, prev_len, buf_multi, nb); + uint64_t prev_len_last = total_len - (uint64_t)r; + K____uint32_t___uint32_t_ acc = tmp_block_state.f3; + bool last_node1 = tmp_block_state.thd; + uint32_t *wv = acc.fst; + uint32_t *hash = acc.snd; + Hacl_Hash_Blake2s_update_last(r, wv, hash, last_node1, prev_len_last, r, buf_last); + uint8_t nn1 = tmp_block_state.snd; + Hacl_Hash_Blake2s_finish((uint32_t)nn1, dst, tmp_block_state.f3.snd); + Hacl_Hash_Blake2s_block_state_t block_state1 = (*s).block_state; + bool last_node = block_state1.thd; + uint8_t nn = block_state1.snd; + uint8_t kk = block_state1.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk, .digest_length = nn, .last_node = last_node }).digest_length; +} + +Hacl_Hash_Blake2b_index Hacl_Hash_Blake2s_info(Hacl_Hash_Blake2s_state_t *s) +{ + Hacl_Hash_Blake2s_block_state_t block_state = (*s).block_state; + bool last_node = block_state.thd; + uint8_t nn = block_state.snd; + uint8_t kk = block_state.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk, .digest_length = nn, .last_node = last_node }); +} + +/** + Free state function when there is no key +*/ +void Hacl_Hash_Blake2s_free(Hacl_Hash_Blake2s_state_t *state) +{ + Hacl_Hash_Blake2s_state_t scrut = *state; + uint8_t *buf = scrut.buf; + Hacl_Hash_Blake2s_block_state_t block_state = scrut.block_state; + uint32_t *b = block_state.f3.snd; + uint32_t *wv = block_state.f3.fst; + KRML_HOST_FREE(wv); + KRML_HOST_FREE(b); + KRML_HOST_FREE(buf); + KRML_HOST_FREE(state); +} + +/** + Copying. This preserves all parameters. +*/ +Hacl_Hash_Blake2s_state_t *Hacl_Hash_Blake2s_copy(Hacl_Hash_Blake2s_state_t *state) +{ + Hacl_Hash_Blake2s_state_t scrut = *state; + Hacl_Hash_Blake2s_block_state_t block_state0 = scrut.block_state; + uint8_t *buf0 = scrut.buf; + uint64_t total_len0 = scrut.total_len; + bool last_node = block_state0.thd; + uint8_t nn = block_state0.snd; + uint8_t kk1 = block_state0.fst; + Hacl_Hash_Blake2b_index i = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + memcpy(buf, buf0, 64U * sizeof (uint8_t)); + uint32_t *wv = (uint32_t *)KRML_HOST_CALLOC(16U, sizeof (uint32_t)); + uint32_t *b = (uint32_t *)KRML_HOST_CALLOC(16U, sizeof (uint32_t)); + Hacl_Hash_Blake2s_block_state_t + block_state = + { + .fst = i.key_length, + .snd = i.digest_length, + .thd = i.last_node, + .f3 = { .fst = wv, .snd = b } + }; + uint32_t *src_b = block_state0.f3.snd; + uint32_t *dst_b = block_state.f3.snd; + memcpy(dst_b, src_b, 16U * sizeof (uint32_t)); + Hacl_Hash_Blake2s_state_t + s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; + Hacl_Hash_Blake2s_state_t + *p = (Hacl_Hash_Blake2s_state_t *)KRML_HOST_MALLOC(sizeof (Hacl_Hash_Blake2s_state_t)); + p[0U] = s; + return p; +} + +/** +Write the BLAKE2s digest of message `input` using key `key` into `output`. + +@param output Pointer to `output_len` bytes of memory where the digest is written to. +@param output_len Length of the to-be-generated digest with 1 <= `output_len` <= 64. +@param input Pointer to `input_len` bytes of memory where the input message is read from. +@param input_len Length of the input message. +@param key Pointer to `key_len` bytes of memory where the key is read from. +@param key_len Length of the key. Can be 0. +*/ +void +Hacl_Hash_Blake2s_hash_with_key( + uint8_t *output, + uint32_t output_len, + uint8_t *input, + uint32_t input_len, + uint8_t *key, + uint32_t key_len +) +{ + uint32_t b[16U] = { 0U }; + uint32_t b1[16U] = { 0U }; + Hacl_Hash_Blake2s_init(b, key_len, output_len); + update(b1, b, key_len, key, input_len, input); + Hacl_Hash_Blake2s_finish(output_len, output, b); + Lib_Memzero0_memzero(b1, 16U, uint32_t, void *); + Lib_Memzero0_memzero(b, 16U, uint32_t, void *); +} + +/** +Write the BLAKE2s digest of message `input` using key `key` and +parameters `params` into `output`. The `key` array must be of length +`params.key_length`. The `output` array must be of length +`params.digest_length`. +*/ +void +Hacl_Hash_Blake2s_hash_with_key_and_params( + uint8_t *output, + uint8_t *input, + uint32_t input_len, + Hacl_Hash_Blake2b_blake2_params params, + uint8_t *key +) +{ + uint32_t b[16U] = { 0U }; + uint32_t b1[16U] = { 0U }; + uint32_t tmp[8U] = { 0U }; + uint32_t *r0 = b; + uint32_t *r1 = b + 4U; + uint32_t *r2 = b + 8U; + uint32_t *r3 = b + 12U; + uint32_t iv0 = Hacl_Hash_Blake2b_ivTable_S[0U]; + uint32_t iv1 = Hacl_Hash_Blake2b_ivTable_S[1U]; + uint32_t iv2 = Hacl_Hash_Blake2b_ivTable_S[2U]; + uint32_t iv3 = Hacl_Hash_Blake2b_ivTable_S[3U]; + uint32_t iv4 = Hacl_Hash_Blake2b_ivTable_S[4U]; + uint32_t iv5 = Hacl_Hash_Blake2b_ivTable_S[5U]; + uint32_t iv6 = Hacl_Hash_Blake2b_ivTable_S[6U]; + uint32_t iv7 = Hacl_Hash_Blake2b_ivTable_S[7U]; + r2[0U] = iv0; + r2[1U] = iv1; + r2[2U] = iv2; + r2[3U] = iv3; + r3[0U] = iv4; + r3[1U] = iv5; + r3[2U] = iv6; + r3[3U] = iv7; + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 4U; + uint8_t *bj = params.salt + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 6U; + uint8_t *bj = params.personal + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + tmp[0U] = + (uint32_t)params.digest_length + ^ + ((uint32_t)params.key_length + << 8U + ^ ((uint32_t)params.fanout << 16U ^ (uint32_t)params.depth << 24U)); + tmp[1U] = params.leaf_length; + tmp[2U] = (uint32_t)params.node_offset; + tmp[3U] = + (uint32_t)(params.node_offset >> 32U) + ^ ((uint32_t)params.node_depth << 16U ^ (uint32_t)params.inner_length << 24U); + uint32_t tmp0 = tmp[0U]; + uint32_t tmp1 = tmp[1U]; + uint32_t tmp2 = tmp[2U]; + uint32_t tmp3 = tmp[3U]; + uint32_t tmp4 = tmp[4U]; + uint32_t tmp5 = tmp[5U]; + uint32_t tmp6 = tmp[6U]; + uint32_t tmp7 = tmp[7U]; + uint32_t iv0_ = iv0 ^ tmp0; + uint32_t iv1_ = iv1 ^ tmp1; + uint32_t iv2_ = iv2 ^ tmp2; + uint32_t iv3_ = iv3 ^ tmp3; + uint32_t iv4_ = iv4 ^ tmp4; + uint32_t iv5_ = iv5 ^ tmp5; + uint32_t iv6_ = iv6 ^ tmp6; + uint32_t iv7_ = iv7 ^ tmp7; + r0[0U] = iv0_; + r0[1U] = iv1_; + r0[2U] = iv2_; + r0[3U] = iv3_; + r1[0U] = iv4_; + r1[1U] = iv5_; + r1[2U] = iv6_; + r1[3U] = iv7_; + update(b1, b, (uint32_t)params.key_length, key, input_len, input); + Hacl_Hash_Blake2s_finish((uint32_t)params.digest_length, output, b); + Lib_Memzero0_memzero(b1, 16U, uint32_t, void *); + Lib_Memzero0_memzero(b, 16U, uint32_t, void *); +} + diff --git a/Modules/_hacl/Hacl_Hash_Blake2s.h b/Modules/_hacl/Hacl_Hash_Blake2s.h new file mode 100644 index 00000000000000..5c01da144018e3 --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_Blake2s.h @@ -0,0 +1,222 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __Hacl_Hash_Blake2s_H +#define __Hacl_Hash_Blake2s_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "python_hacl_namespaces.h" +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "Hacl_Streaming_Types.h" +#include "Hacl_Hash_Blake2b.h" + +#define HACL_HASH_BLAKE2S_BLOCK_BYTES (64U) + +#define HACL_HASH_BLAKE2S_OUT_BYTES (32U) + +#define HACL_HASH_BLAKE2S_KEY_BYTES (32U) + +#define HACL_HASH_BLAKE2S_SALT_BYTES (8U) + +#define HACL_HASH_BLAKE2S_PERSONAL_BYTES (8U) + +typedef struct K____uint32_t___uint32_t__s +{ + uint32_t *fst; + uint32_t *snd; +} +K____uint32_t___uint32_t_; + +typedef struct Hacl_Hash_Blake2s_block_state_t_s +{ + uint8_t fst; + uint8_t snd; + bool thd; + K____uint32_t___uint32_t_ f3; +} +Hacl_Hash_Blake2s_block_state_t; + +typedef struct Hacl_Hash_Blake2s_state_t_s +{ + Hacl_Hash_Blake2s_block_state_t block_state; + uint8_t *buf; + uint64_t total_len; +} +Hacl_Hash_Blake2s_state_t; + +/** + General-purpose allocation function that gives control over all +Blake2 parameters, including the key. Further resettings of the state SHALL be +done with `reset_with_params_and_key`, and SHALL feature the exact same values +for the `key_length` and `digest_length` fields as passed here. In other words, +once you commit to a digest and key length, the only way to change these +parameters is to allocate a new object. + +The caller must satisfy the following requirements. +- The length of the key k MUST match the value of the field key_length in the + parameters. +- The key_length must not exceed 32 for S, 64 for B. +- The digest_length must not exceed 32 for S, 64 for B. + +*/ +Hacl_Hash_Blake2s_state_t +*Hacl_Hash_Blake2s_malloc_with_params_and_key( + Hacl_Hash_Blake2b_blake2_params *p, + bool last_node, + uint8_t *k +); + +/** + Specialized allocation function that picks default values for all +parameters, except for the key_length. Further resettings of the state SHALL be +done with `reset_with_key`, and SHALL feature the exact same key length `kk` as +passed here. In other words, once you commit to a key length, the only way to +change this parameter is to allocate a new object. + +The caller must satisfy the following requirements. +- The key_length must not exceed 32 for S, 64 for B. + +*/ +Hacl_Hash_Blake2s_state_t *Hacl_Hash_Blake2s_malloc_with_key(uint8_t *k, uint8_t kk); + +/** + Specialized allocation function that picks default values for all +parameters, and has no key. Effectively, this is what you want if you intend to +use Blake2 as a hash function. Further resettings of the state SHALL be done with `reset`. +*/ +Hacl_Hash_Blake2s_state_t *Hacl_Hash_Blake2s_malloc(void); + +/** + General-purpose re-initialization function with parameters and +key. You cannot change digest_length, key_length, or last_node, meaning those values in +the parameters object must be the same as originally decided via one of the +malloc functions. All other values of the parameter can be changed. The behavior +is unspecified if you violate this precondition. +*/ +void +Hacl_Hash_Blake2s_reset_with_key_and_params( + Hacl_Hash_Blake2s_state_t *s, + Hacl_Hash_Blake2b_blake2_params *p, + uint8_t *k +); + +/** + Specialized-purpose re-initialization function with no parameters, +and a key. The key length must be the same as originally decided via your choice +of malloc function. All other parameters are reset to their default values. The +original call to malloc MUST have set digest_length to the default value. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2s_reset_with_key(Hacl_Hash_Blake2s_state_t *s, uint8_t *k); + +/** + Specialized-purpose re-initialization function with no parameters +and no key. This is what you want if you intend to use Blake2 as a hash +function. The key length and digest length must have been set to their +respective default values via your choice of malloc function (always true if you +used `malloc`). All other parameters are reset to their default values. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2s_reset(Hacl_Hash_Blake2s_state_t *s); + +/** + Update function; 0 = success, 1 = max length exceeded +*/ +Hacl_Streaming_Types_error_code +Hacl_Hash_Blake2s_update(Hacl_Hash_Blake2s_state_t *state, uint8_t *chunk, uint32_t chunk_len); + +/** + Digest function. This function expects the `output` array to hold +at least `digest_length` bytes, where `digest_length` was determined by your +choice of `malloc` function. Concretely, if you used `malloc` or +`malloc_with_key`, then the expected length is 32 for S, or 64 for B (default +digest length). If you used `malloc_with_params_and_key`, then the expected +length is whatever you chose for the `digest_length` field of your parameters. +For convenience, this function returns `digest_length`. When in doubt, callers +can pass an array of size HACL_BLAKE2S_32_OUT_BYTES, then use the return value +to see how many bytes were actually written. +*/ +uint8_t Hacl_Hash_Blake2s_digest(Hacl_Hash_Blake2s_state_t *s, uint8_t *dst); + +Hacl_Hash_Blake2b_index Hacl_Hash_Blake2s_info(Hacl_Hash_Blake2s_state_t *s); + +/** + Free state function when there is no key +*/ +void Hacl_Hash_Blake2s_free(Hacl_Hash_Blake2s_state_t *state); + +/** + Copying. This preserves all parameters. +*/ +Hacl_Hash_Blake2s_state_t *Hacl_Hash_Blake2s_copy(Hacl_Hash_Blake2s_state_t *state); + +/** +Write the BLAKE2s digest of message `input` using key `key` into `output`. + +@param output Pointer to `output_len` bytes of memory where the digest is written to. +@param output_len Length of the to-be-generated digest with 1 <= `output_len` <= 64. +@param input Pointer to `input_len` bytes of memory where the input message is read from. +@param input_len Length of the input message. +@param key Pointer to `key_len` bytes of memory where the key is read from. +@param key_len Length of the key. Can be 0. +*/ +void +Hacl_Hash_Blake2s_hash_with_key( + uint8_t *output, + uint32_t output_len, + uint8_t *input, + uint32_t input_len, + uint8_t *key, + uint32_t key_len +); + +/** +Write the BLAKE2s digest of message `input` using key `key` and +parameters `params` into `output`. The `key` array must be of length +`params.key_length`. The `output` array must be of length +`params.digest_length`. +*/ +void +Hacl_Hash_Blake2s_hash_with_key_and_params( + uint8_t *output, + uint8_t *input, + uint32_t input_len, + Hacl_Hash_Blake2b_blake2_params params, + uint8_t *key +); + +#if defined(__cplusplus) +} +#endif + +#define __Hacl_Hash_Blake2s_H_DEFINED +#endif diff --git a/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c new file mode 100644 index 00000000000000..3b68783bfad9b4 --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c @@ -0,0 +1,1351 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#include "internal/Hacl_Hash_Blake2s_Simd128.h" + +#include "internal/Hacl_Impl_Blake2_Constants.h" +#include "internal/Hacl_Hash_Blake2b.h" +#include "lib_memzero0.h" + +static inline void +update_block( + Lib_IntVector_Intrinsics_vec128 *wv, + Lib_IntVector_Intrinsics_vec128 *hash, + bool flag, + bool last_node, + uint64_t totlen, + uint8_t *d +) +{ + uint32_t m_w[16U] = { 0U }; + KRML_MAYBE_FOR16(i, + 0U, + 16U, + 1U, + uint32_t *os = m_w; + uint8_t *bj = d + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + Lib_IntVector_Intrinsics_vec128 mask = Lib_IntVector_Intrinsics_vec128_zero; + uint32_t wv_14; + if (flag) + { + wv_14 = 0xFFFFFFFFU; + } + else + { + wv_14 = 0U; + } + uint32_t wv_15; + if (last_node) + { + wv_15 = 0xFFFFFFFFU; + } + else + { + wv_15 = 0U; + } + mask = + Lib_IntVector_Intrinsics_vec128_load32s((uint32_t)totlen, + (uint32_t)(totlen >> 32U), + wv_14, + wv_15); + memcpy(wv, hash, 4U * sizeof (Lib_IntVector_Intrinsics_vec128)); + Lib_IntVector_Intrinsics_vec128 *wv3 = wv + 3U; + wv3[0U] = Lib_IntVector_Intrinsics_vec128_xor(wv3[0U], mask); + KRML_MAYBE_FOR10(i, + 0U, + 10U, + 1U, + uint32_t start_idx = i % 10U * 16U; + KRML_PRE_ALIGN(16) Lib_IntVector_Intrinsics_vec128 m_st[4U] KRML_POST_ALIGN(16) = { 0U }; + Lib_IntVector_Intrinsics_vec128 *r0 = m_st; + Lib_IntVector_Intrinsics_vec128 *r1 = m_st + 1U; + Lib_IntVector_Intrinsics_vec128 *r20 = m_st + 2U; + Lib_IntVector_Intrinsics_vec128 *r30 = m_st + 3U; + uint32_t s0 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 0U]; + uint32_t s1 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 1U]; + uint32_t s2 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 2U]; + uint32_t s3 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 3U]; + uint32_t s4 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 4U]; + uint32_t s5 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 5U]; + uint32_t s6 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 6U]; + uint32_t s7 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 7U]; + uint32_t s8 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 8U]; + uint32_t s9 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 9U]; + uint32_t s10 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 10U]; + uint32_t s11 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 11U]; + uint32_t s12 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 12U]; + uint32_t s13 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 13U]; + uint32_t s14 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 14U]; + uint32_t s15 = Hacl_Hash_Blake2b_sigmaTable[start_idx + 15U]; + r0[0U] = Lib_IntVector_Intrinsics_vec128_load32s(m_w[s0], m_w[s2], m_w[s4], m_w[s6]); + r1[0U] = Lib_IntVector_Intrinsics_vec128_load32s(m_w[s1], m_w[s3], m_w[s5], m_w[s7]); + r20[0U] = Lib_IntVector_Intrinsics_vec128_load32s(m_w[s8], m_w[s10], m_w[s12], m_w[s14]); + r30[0U] = Lib_IntVector_Intrinsics_vec128_load32s(m_w[s9], m_w[s11], m_w[s13], m_w[s15]); + Lib_IntVector_Intrinsics_vec128 *x = m_st; + Lib_IntVector_Intrinsics_vec128 *y = m_st + 1U; + Lib_IntVector_Intrinsics_vec128 *z = m_st + 2U; + Lib_IntVector_Intrinsics_vec128 *w = m_st + 3U; + uint32_t a = 0U; + uint32_t b0 = 1U; + uint32_t c0 = 2U; + uint32_t d10 = 3U; + Lib_IntVector_Intrinsics_vec128 *wv_a0 = wv + a * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b0 = wv + b0 * 1U; + wv_a0[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a0[0U], wv_b0[0U]); + wv_a0[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a0[0U], x[0U]); + Lib_IntVector_Intrinsics_vec128 *wv_a1 = wv + d10 * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b1 = wv + a * 1U; + wv_a1[0U] = Lib_IntVector_Intrinsics_vec128_xor(wv_a1[0U], wv_b1[0U]); + wv_a1[0U] = Lib_IntVector_Intrinsics_vec128_rotate_right32(wv_a1[0U], 16U); + Lib_IntVector_Intrinsics_vec128 *wv_a2 = wv + c0 * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b2 = wv + d10 * 1U; + wv_a2[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a2[0U], wv_b2[0U]); + Lib_IntVector_Intrinsics_vec128 *wv_a3 = wv + b0 * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b3 = wv + c0 * 1U; + wv_a3[0U] = Lib_IntVector_Intrinsics_vec128_xor(wv_a3[0U], wv_b3[0U]); + wv_a3[0U] = Lib_IntVector_Intrinsics_vec128_rotate_right32(wv_a3[0U], 12U); + Lib_IntVector_Intrinsics_vec128 *wv_a4 = wv + a * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b4 = wv + b0 * 1U; + wv_a4[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a4[0U], wv_b4[0U]); + wv_a4[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a4[0U], y[0U]); + Lib_IntVector_Intrinsics_vec128 *wv_a5 = wv + d10 * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b5 = wv + a * 1U; + wv_a5[0U] = Lib_IntVector_Intrinsics_vec128_xor(wv_a5[0U], wv_b5[0U]); + wv_a5[0U] = Lib_IntVector_Intrinsics_vec128_rotate_right32(wv_a5[0U], 8U); + Lib_IntVector_Intrinsics_vec128 *wv_a6 = wv + c0 * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b6 = wv + d10 * 1U; + wv_a6[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a6[0U], wv_b6[0U]); + Lib_IntVector_Intrinsics_vec128 *wv_a7 = wv + b0 * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b7 = wv + c0 * 1U; + wv_a7[0U] = Lib_IntVector_Intrinsics_vec128_xor(wv_a7[0U], wv_b7[0U]); + wv_a7[0U] = Lib_IntVector_Intrinsics_vec128_rotate_right32(wv_a7[0U], 7U); + Lib_IntVector_Intrinsics_vec128 *r10 = wv + 1U; + Lib_IntVector_Intrinsics_vec128 *r21 = wv + 2U; + Lib_IntVector_Intrinsics_vec128 *r31 = wv + 3U; + Lib_IntVector_Intrinsics_vec128 v00 = r10[0U]; + Lib_IntVector_Intrinsics_vec128 + v1 = Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(v00, 1U); + r10[0U] = v1; + Lib_IntVector_Intrinsics_vec128 v01 = r21[0U]; + Lib_IntVector_Intrinsics_vec128 + v10 = Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(v01, 2U); + r21[0U] = v10; + Lib_IntVector_Intrinsics_vec128 v02 = r31[0U]; + Lib_IntVector_Intrinsics_vec128 + v11 = Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(v02, 3U); + r31[0U] = v11; + uint32_t a0 = 0U; + uint32_t b = 1U; + uint32_t c = 2U; + uint32_t d1 = 3U; + Lib_IntVector_Intrinsics_vec128 *wv_a = wv + a0 * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b8 = wv + b * 1U; + wv_a[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a[0U], wv_b8[0U]); + wv_a[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a[0U], z[0U]); + Lib_IntVector_Intrinsics_vec128 *wv_a8 = wv + d1 * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b9 = wv + a0 * 1U; + wv_a8[0U] = Lib_IntVector_Intrinsics_vec128_xor(wv_a8[0U], wv_b9[0U]); + wv_a8[0U] = Lib_IntVector_Intrinsics_vec128_rotate_right32(wv_a8[0U], 16U); + Lib_IntVector_Intrinsics_vec128 *wv_a9 = wv + c * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b10 = wv + d1 * 1U; + wv_a9[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a9[0U], wv_b10[0U]); + Lib_IntVector_Intrinsics_vec128 *wv_a10 = wv + b * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b11 = wv + c * 1U; + wv_a10[0U] = Lib_IntVector_Intrinsics_vec128_xor(wv_a10[0U], wv_b11[0U]); + wv_a10[0U] = Lib_IntVector_Intrinsics_vec128_rotate_right32(wv_a10[0U], 12U); + Lib_IntVector_Intrinsics_vec128 *wv_a11 = wv + a0 * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b12 = wv + b * 1U; + wv_a11[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a11[0U], wv_b12[0U]); + wv_a11[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a11[0U], w[0U]); + Lib_IntVector_Intrinsics_vec128 *wv_a12 = wv + d1 * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b13 = wv + a0 * 1U; + wv_a12[0U] = Lib_IntVector_Intrinsics_vec128_xor(wv_a12[0U], wv_b13[0U]); + wv_a12[0U] = Lib_IntVector_Intrinsics_vec128_rotate_right32(wv_a12[0U], 8U); + Lib_IntVector_Intrinsics_vec128 *wv_a13 = wv + c * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b14 = wv + d1 * 1U; + wv_a13[0U] = Lib_IntVector_Intrinsics_vec128_add32(wv_a13[0U], wv_b14[0U]); + Lib_IntVector_Intrinsics_vec128 *wv_a14 = wv + b * 1U; + Lib_IntVector_Intrinsics_vec128 *wv_b = wv + c * 1U; + wv_a14[0U] = Lib_IntVector_Intrinsics_vec128_xor(wv_a14[0U], wv_b[0U]); + wv_a14[0U] = Lib_IntVector_Intrinsics_vec128_rotate_right32(wv_a14[0U], 7U); + Lib_IntVector_Intrinsics_vec128 *r11 = wv + 1U; + Lib_IntVector_Intrinsics_vec128 *r2 = wv + 2U; + Lib_IntVector_Intrinsics_vec128 *r3 = wv + 3U; + Lib_IntVector_Intrinsics_vec128 v0 = r11[0U]; + Lib_IntVector_Intrinsics_vec128 + v12 = Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(v0, 3U); + r11[0U] = v12; + Lib_IntVector_Intrinsics_vec128 v03 = r2[0U]; + Lib_IntVector_Intrinsics_vec128 + v13 = Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(v03, 2U); + r2[0U] = v13; + Lib_IntVector_Intrinsics_vec128 v04 = r3[0U]; + Lib_IntVector_Intrinsics_vec128 + v14 = Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(v04, 1U); + r3[0U] = v14;); + Lib_IntVector_Intrinsics_vec128 *s0 = hash; + Lib_IntVector_Intrinsics_vec128 *s1 = hash + 1U; + Lib_IntVector_Intrinsics_vec128 *r0 = wv; + Lib_IntVector_Intrinsics_vec128 *r1 = wv + 1U; + Lib_IntVector_Intrinsics_vec128 *r2 = wv + 2U; + Lib_IntVector_Intrinsics_vec128 *r3 = wv + 3U; + s0[0U] = Lib_IntVector_Intrinsics_vec128_xor(s0[0U], r0[0U]); + s0[0U] = Lib_IntVector_Intrinsics_vec128_xor(s0[0U], r2[0U]); + s1[0U] = Lib_IntVector_Intrinsics_vec128_xor(s1[0U], r1[0U]); + s1[0U] = Lib_IntVector_Intrinsics_vec128_xor(s1[0U], r3[0U]); +} + +void +Hacl_Hash_Blake2s_Simd128_init(Lib_IntVector_Intrinsics_vec128 *hash, uint32_t kk, uint32_t nn) +{ + uint8_t salt[8U] = { 0U }; + uint8_t personal[8U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = 32U, .key_length = 0U, .fanout = 1U, .depth = 1U, .leaf_length = 0U, + .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, .personal = personal + }; + uint32_t tmp[8U] = { 0U }; + Lib_IntVector_Intrinsics_vec128 *r0 = hash; + Lib_IntVector_Intrinsics_vec128 *r1 = hash + 1U; + Lib_IntVector_Intrinsics_vec128 *r2 = hash + 2U; + Lib_IntVector_Intrinsics_vec128 *r3 = hash + 3U; + uint32_t iv0 = Hacl_Hash_Blake2b_ivTable_S[0U]; + uint32_t iv1 = Hacl_Hash_Blake2b_ivTable_S[1U]; + uint32_t iv2 = Hacl_Hash_Blake2b_ivTable_S[2U]; + uint32_t iv3 = Hacl_Hash_Blake2b_ivTable_S[3U]; + uint32_t iv4 = Hacl_Hash_Blake2b_ivTable_S[4U]; + uint32_t iv5 = Hacl_Hash_Blake2b_ivTable_S[5U]; + uint32_t iv6 = Hacl_Hash_Blake2b_ivTable_S[6U]; + uint32_t iv7 = Hacl_Hash_Blake2b_ivTable_S[7U]; + r2[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv0, iv1, iv2, iv3); + r3[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv4, iv5, iv6, iv7); + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 4U; + uint8_t *bj = p.salt + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 6U; + uint8_t *bj = p.personal + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + tmp[0U] = + (uint32_t)(uint8_t)nn + ^ ((uint32_t)(uint8_t)kk << 8U ^ ((uint32_t)p.fanout << 16U ^ (uint32_t)p.depth << 24U)); + tmp[1U] = p.leaf_length; + tmp[2U] = (uint32_t)p.node_offset; + tmp[3U] = + (uint32_t)(p.node_offset >> 32U) + ^ ((uint32_t)p.node_depth << 16U ^ (uint32_t)p.inner_length << 24U); + uint32_t tmp0 = tmp[0U]; + uint32_t tmp1 = tmp[1U]; + uint32_t tmp2 = tmp[2U]; + uint32_t tmp3 = tmp[3U]; + uint32_t tmp4 = tmp[4U]; + uint32_t tmp5 = tmp[5U]; + uint32_t tmp6 = tmp[6U]; + uint32_t tmp7 = tmp[7U]; + uint32_t iv0_ = iv0 ^ tmp0; + uint32_t iv1_ = iv1 ^ tmp1; + uint32_t iv2_ = iv2 ^ tmp2; + uint32_t iv3_ = iv3 ^ tmp3; + uint32_t iv4_ = iv4 ^ tmp4; + uint32_t iv5_ = iv5 ^ tmp5; + uint32_t iv6_ = iv6 ^ tmp6; + uint32_t iv7_ = iv7 ^ tmp7; + r0[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv0_, iv1_, iv2_, iv3_); + r1[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv4_, iv5_, iv6_, iv7_); +} + +static void +update_key( + Lib_IntVector_Intrinsics_vec128 *wv, + Lib_IntVector_Intrinsics_vec128 *hash, + uint32_t kk, + uint8_t *k, + uint32_t ll +) +{ + uint64_t lb = (uint64_t)64U; + uint8_t b[64U] = { 0U }; + memcpy(b, k, kk * sizeof (uint8_t)); + if (ll == 0U) + { + update_block(wv, hash, true, false, lb, b); + } + else + { + update_block(wv, hash, false, false, lb, b); + } + Lib_Memzero0_memzero(b, 64U, uint8_t, void *); +} + +void +Hacl_Hash_Blake2s_Simd128_update_multi( + uint32_t len, + Lib_IntVector_Intrinsics_vec128 *wv, + Lib_IntVector_Intrinsics_vec128 *hash, + uint64_t prev, + uint8_t *blocks, + uint32_t nb +) +{ + KRML_MAYBE_UNUSED_VAR(len); + for (uint32_t i = 0U; i < nb; i++) + { + uint64_t totlen = prev + (uint64_t)((i + 1U) * 64U); + uint8_t *b = blocks + i * 64U; + update_block(wv, hash, false, false, totlen, b); + } +} + +void +Hacl_Hash_Blake2s_Simd128_update_last( + uint32_t len, + Lib_IntVector_Intrinsics_vec128 *wv, + Lib_IntVector_Intrinsics_vec128 *hash, + bool last_node, + uint64_t prev, + uint32_t rem, + uint8_t *d +) +{ + uint8_t b[64U] = { 0U }; + uint8_t *last = d + len - rem; + memcpy(b, last, rem * sizeof (uint8_t)); + uint64_t totlen = prev + (uint64_t)len; + update_block(wv, hash, true, last_node, totlen, b); + Lib_Memzero0_memzero(b, 64U, uint8_t, void *); +} + +static inline void +update_blocks( + uint32_t len, + Lib_IntVector_Intrinsics_vec128 *wv, + Lib_IntVector_Intrinsics_vec128 *hash, + uint64_t prev, + uint8_t *blocks +) +{ + uint32_t nb0 = len / 64U; + uint32_t rem0 = len % 64U; + uint32_t nb; + if (rem0 == 0U && nb0 > 0U) + { + nb = nb0 - 1U; + } + else + { + nb = nb0; + } + uint32_t rem; + if (rem0 == 0U && nb0 > 0U) + { + rem = 64U; + } + else + { + rem = rem0; + } + Hacl_Hash_Blake2s_Simd128_update_multi(len, wv, hash, prev, blocks, nb); + Hacl_Hash_Blake2s_Simd128_update_last(len, wv, hash, false, prev, rem, blocks); +} + +static inline void +update( + Lib_IntVector_Intrinsics_vec128 *wv, + Lib_IntVector_Intrinsics_vec128 *hash, + uint32_t kk, + uint8_t *k, + uint32_t ll, + uint8_t *d +) +{ + uint64_t lb = (uint64_t)64U; + if (kk > 0U) + { + update_key(wv, hash, kk, k, ll); + if (!(ll == 0U)) + { + update_blocks(ll, wv, hash, lb, d); + return; + } + return; + } + update_blocks(ll, wv, hash, (uint64_t)0U, d); +} + +void +Hacl_Hash_Blake2s_Simd128_finish( + uint32_t nn, + uint8_t *output, + Lib_IntVector_Intrinsics_vec128 *hash +) +{ + uint8_t b[32U] = { 0U }; + uint8_t *first = b; + uint8_t *second = b + 16U; + Lib_IntVector_Intrinsics_vec128 *row0 = hash; + Lib_IntVector_Intrinsics_vec128 *row1 = hash + 1U; + Lib_IntVector_Intrinsics_vec128_store32_le(first, row0[0U]); + Lib_IntVector_Intrinsics_vec128_store32_le(second, row1[0U]); + uint8_t *final = b; + memcpy(output, final, nn * sizeof (uint8_t)); + Lib_Memzero0_memzero(b, 32U, uint8_t, void *); +} + +void +Hacl_Hash_Blake2s_Simd128_store_state128s_to_state32( + uint32_t *st32, + Lib_IntVector_Intrinsics_vec128 *st +) +{ + Lib_IntVector_Intrinsics_vec128 *r0 = st; + Lib_IntVector_Intrinsics_vec128 *r1 = st + 1U; + Lib_IntVector_Intrinsics_vec128 *r2 = st + 2U; + Lib_IntVector_Intrinsics_vec128 *r3 = st + 3U; + uint32_t *b0 = st32; + uint32_t *b1 = st32 + 4U; + uint32_t *b2 = st32 + 8U; + uint32_t *b3 = st32 + 12U; + uint8_t b8[16U] = { 0U }; + Lib_IntVector_Intrinsics_vec128_store32_le(b8, r0[0U]); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = b0; + uint8_t *bj = b8 + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + uint8_t b80[16U] = { 0U }; + Lib_IntVector_Intrinsics_vec128_store32_le(b80, r1[0U]); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = b1; + uint8_t *bj = b80 + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + uint8_t b81[16U] = { 0U }; + Lib_IntVector_Intrinsics_vec128_store32_le(b81, r2[0U]); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = b2; + uint8_t *bj = b81 + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + uint8_t b82[16U] = { 0U }; + Lib_IntVector_Intrinsics_vec128_store32_le(b82, r3[0U]); + KRML_MAYBE_FOR4(i, + 0U, + 4U, + 1U, + uint32_t *os = b3; + uint8_t *bj = b82 + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); +} + +void +Hacl_Hash_Blake2s_Simd128_load_state128s_from_state32( + Lib_IntVector_Intrinsics_vec128 *st, + uint32_t *st32 +) +{ + Lib_IntVector_Intrinsics_vec128 *r0 = st; + Lib_IntVector_Intrinsics_vec128 *r1 = st + 1U; + Lib_IntVector_Intrinsics_vec128 *r2 = st + 2U; + Lib_IntVector_Intrinsics_vec128 *r3 = st + 3U; + uint32_t *b0 = st32; + uint32_t *b1 = st32 + 4U; + uint32_t *b2 = st32 + 8U; + uint32_t *b3 = st32 + 12U; + r0[0U] = Lib_IntVector_Intrinsics_vec128_load32s(b0[0U], b0[1U], b0[2U], b0[3U]); + r1[0U] = Lib_IntVector_Intrinsics_vec128_load32s(b1[0U], b1[1U], b1[2U], b1[3U]); + r2[0U] = Lib_IntVector_Intrinsics_vec128_load32s(b2[0U], b2[1U], b2[2U], b2[3U]); + r3[0U] = Lib_IntVector_Intrinsics_vec128_load32s(b3[0U], b3[1U], b3[2U], b3[3U]); +} + +Lib_IntVector_Intrinsics_vec128 *Hacl_Hash_Blake2s_Simd128_malloc_with_key(void) +{ + Lib_IntVector_Intrinsics_vec128 + *buf = + (Lib_IntVector_Intrinsics_vec128 *)KRML_ALIGNED_MALLOC(16, + sizeof (Lib_IntVector_Intrinsics_vec128) * 4U); + memset(buf, 0U, 4U * sizeof (Lib_IntVector_Intrinsics_vec128)); + return buf; +} + +static Hacl_Hash_Blake2s_Simd128_state_t +*malloc_raw(Hacl_Hash_Blake2b_index kk, Hacl_Hash_Blake2b_params_and_key key) +{ + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + Lib_IntVector_Intrinsics_vec128 + *wv = + (Lib_IntVector_Intrinsics_vec128 *)KRML_ALIGNED_MALLOC(16, + sizeof (Lib_IntVector_Intrinsics_vec128) * 4U); + memset(wv, 0U, 4U * sizeof (Lib_IntVector_Intrinsics_vec128)); + Lib_IntVector_Intrinsics_vec128 + *b = + (Lib_IntVector_Intrinsics_vec128 *)KRML_ALIGNED_MALLOC(16, + sizeof (Lib_IntVector_Intrinsics_vec128) * 4U); + memset(b, 0U, 4U * sizeof (Lib_IntVector_Intrinsics_vec128)); + Hacl_Hash_Blake2s_Simd128_block_state_t + block_state = + { + .fst = kk.key_length, + .snd = kk.digest_length, + .thd = kk.last_node, + .f3 = { .fst = wv, .snd = b } + }; + uint8_t kk10 = kk.key_length; + uint32_t ite; + if (kk10 != 0U) + { + ite = 64U; + } + else + { + ite = 0U; + } + Hacl_Hash_Blake2s_Simd128_state_t + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)ite }; + Hacl_Hash_Blake2s_Simd128_state_t + *p = + (Hacl_Hash_Blake2s_Simd128_state_t *)KRML_HOST_MALLOC(sizeof ( + Hacl_Hash_Blake2s_Simd128_state_t + )); + p[0U] = s; + Hacl_Hash_Blake2b_blake2_params *p1 = key.fst; + uint8_t kk1 = p1->key_length; + uint8_t nn = p1->digest_length; + bool last_node = block_state.thd; + Hacl_Hash_Blake2b_index i = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + Lib_IntVector_Intrinsics_vec128 *h = block_state.f3.snd; + uint32_t kk2 = (uint32_t)i.key_length; + uint8_t *k_1 = key.snd; + if (!(kk2 == 0U)) + { + uint8_t *sub_b = buf + kk2; + memset(sub_b, 0U, (64U - kk2) * sizeof (uint8_t)); + memcpy(buf, k_1, kk2 * sizeof (uint8_t)); + } + Hacl_Hash_Blake2b_blake2_params pv = p1[0U]; + uint32_t tmp[8U] = { 0U }; + Lib_IntVector_Intrinsics_vec128 *r0 = h; + Lib_IntVector_Intrinsics_vec128 *r1 = h + 1U; + Lib_IntVector_Intrinsics_vec128 *r2 = h + 2U; + Lib_IntVector_Intrinsics_vec128 *r3 = h + 3U; + uint32_t iv0 = Hacl_Hash_Blake2b_ivTable_S[0U]; + uint32_t iv1 = Hacl_Hash_Blake2b_ivTable_S[1U]; + uint32_t iv2 = Hacl_Hash_Blake2b_ivTable_S[2U]; + uint32_t iv3 = Hacl_Hash_Blake2b_ivTable_S[3U]; + uint32_t iv4 = Hacl_Hash_Blake2b_ivTable_S[4U]; + uint32_t iv5 = Hacl_Hash_Blake2b_ivTable_S[5U]; + uint32_t iv6 = Hacl_Hash_Blake2b_ivTable_S[6U]; + uint32_t iv7 = Hacl_Hash_Blake2b_ivTable_S[7U]; + r2[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv0, iv1, iv2, iv3); + r3[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv4, iv5, iv6, iv7); + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 4U; + uint8_t *bj = pv.salt + i0 * 4U; + uint32_t u = load32_le(bj); + uint32_t r4 = u; + uint32_t x = r4; + os[i0] = x;); + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 6U; + uint8_t *bj = pv.personal + i0 * 4U; + uint32_t u = load32_le(bj); + uint32_t r4 = u; + uint32_t x = r4; + os[i0] = x;); + tmp[0U] = + (uint32_t)pv.digest_length + ^ ((uint32_t)pv.key_length << 8U ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); + tmp[1U] = pv.leaf_length; + tmp[2U] = (uint32_t)pv.node_offset; + tmp[3U] = + (uint32_t)(pv.node_offset >> 32U) + ^ ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); + uint32_t tmp0 = tmp[0U]; + uint32_t tmp1 = tmp[1U]; + uint32_t tmp2 = tmp[2U]; + uint32_t tmp3 = tmp[3U]; + uint32_t tmp4 = tmp[4U]; + uint32_t tmp5 = tmp[5U]; + uint32_t tmp6 = tmp[6U]; + uint32_t tmp7 = tmp[7U]; + uint32_t iv0_ = iv0 ^ tmp0; + uint32_t iv1_ = iv1 ^ tmp1; + uint32_t iv2_ = iv2 ^ tmp2; + uint32_t iv3_ = iv3 ^ tmp3; + uint32_t iv4_ = iv4 ^ tmp4; + uint32_t iv5_ = iv5 ^ tmp5; + uint32_t iv6_ = iv6 ^ tmp6; + uint32_t iv7_ = iv7 ^ tmp7; + r0[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv0_, iv1_, iv2_, iv3_); + r1[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv4_, iv5_, iv6_, iv7_); + return p; +} + +/** + General-purpose allocation function that gives control over all +Blake2 parameters, including the key. Further resettings of the state SHALL be +done with `reset_with_params_and_key`, and SHALL feature the exact same values +for the `key_length` and `digest_length` fields as passed here. In other words, +once you commit to a digest and key length, the only way to change these +parameters is to allocate a new object. + +The caller must satisfy the following requirements. +- The length of the key k MUST match the value of the field key_length in the + parameters. +- The key_length must not exceed 128 for S, 64 for B. +- The digest_length must not exceed 128 for S, 64 for B. + +*/ +Hacl_Hash_Blake2s_Simd128_state_t +*Hacl_Hash_Blake2s_Simd128_malloc_with_params_and_key( + Hacl_Hash_Blake2b_blake2_params *p, + bool last_node, + uint8_t *k +) +{ + Hacl_Hash_Blake2b_blake2_params pv = p[0U]; + Hacl_Hash_Blake2b_index + i1 = { .key_length = pv.key_length, .digest_length = pv.digest_length, .last_node = last_node }; + return malloc_raw(i1, ((Hacl_Hash_Blake2b_params_and_key){ .fst = p, .snd = k })); +} + +/** + Specialized allocation function that picks default values for all +parameters, except for the key_length. Further resettings of the state SHALL be +done with `reset_with_key`, and SHALL feature the exact same key length `kk` as +passed here. In other words, once you commit to a key length, the only way to +change this parameter is to allocate a new object. + +The caller must satisfy the following requirements. +- The key_length must not exceed 128 for S, 64 for B. + +*/ +Hacl_Hash_Blake2s_Simd128_state_t +*Hacl_Hash_Blake2s_Simd128_malloc_with_key0(uint8_t *k, uint8_t kk) +{ + uint8_t nn = 32U; + Hacl_Hash_Blake2b_index i = { .key_length = kk, .digest_length = nn, .last_node = false }; + uint8_t salt[8U] = { 0U }; + uint8_t personal[8U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = i.digest_length, .key_length = i.key_length, .fanout = 1U, .depth = 1U, + .leaf_length = 0U, .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, + .personal = personal + }; + Hacl_Hash_Blake2b_blake2_params p0 = p; + Hacl_Hash_Blake2s_Simd128_state_t + *s = Hacl_Hash_Blake2s_Simd128_malloc_with_params_and_key(&p0, false, k); + return s; +} + +/** + Specialized allocation function that picks default values for all +parameters, and has no key. Effectively, this is what you want if you intend to +use Blake2 as a hash function. Further resettings of the state SHALL be done with `reset`. +*/ +Hacl_Hash_Blake2s_Simd128_state_t *Hacl_Hash_Blake2s_Simd128_malloc(void) +{ + return Hacl_Hash_Blake2s_Simd128_malloc_with_key0(NULL, 0U); +} + +static Hacl_Hash_Blake2b_index index_of_state(Hacl_Hash_Blake2s_Simd128_state_t *s) +{ + Hacl_Hash_Blake2s_Simd128_block_state_t block_state = (*s).block_state; + bool last_node = block_state.thd; + uint8_t nn = block_state.snd; + uint8_t kk1 = block_state.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk1, .digest_length = nn, .last_node = last_node }); +} + +static void +reset_raw(Hacl_Hash_Blake2s_Simd128_state_t *state, Hacl_Hash_Blake2b_params_and_key key) +{ + Hacl_Hash_Blake2s_Simd128_state_t scrut = *state; + uint8_t *buf = scrut.buf; + Hacl_Hash_Blake2s_Simd128_block_state_t block_state = scrut.block_state; + bool last_node0 = block_state.thd; + uint8_t nn0 = block_state.snd; + uint8_t kk10 = block_state.fst; + Hacl_Hash_Blake2b_index + i = { .key_length = kk10, .digest_length = nn0, .last_node = last_node0 }; + KRML_MAYBE_UNUSED_VAR(i); + Hacl_Hash_Blake2b_blake2_params *p = key.fst; + uint8_t kk1 = p->key_length; + uint8_t nn = p->digest_length; + bool last_node = block_state.thd; + Hacl_Hash_Blake2b_index + i1 = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + Lib_IntVector_Intrinsics_vec128 *h = block_state.f3.snd; + uint32_t kk2 = (uint32_t)i1.key_length; + uint8_t *k_1 = key.snd; + if (!(kk2 == 0U)) + { + uint8_t *sub_b = buf + kk2; + memset(sub_b, 0U, (64U - kk2) * sizeof (uint8_t)); + memcpy(buf, k_1, kk2 * sizeof (uint8_t)); + } + Hacl_Hash_Blake2b_blake2_params pv = p[0U]; + uint32_t tmp[8U] = { 0U }; + Lib_IntVector_Intrinsics_vec128 *r0 = h; + Lib_IntVector_Intrinsics_vec128 *r1 = h + 1U; + Lib_IntVector_Intrinsics_vec128 *r2 = h + 2U; + Lib_IntVector_Intrinsics_vec128 *r3 = h + 3U; + uint32_t iv0 = Hacl_Hash_Blake2b_ivTable_S[0U]; + uint32_t iv1 = Hacl_Hash_Blake2b_ivTable_S[1U]; + uint32_t iv2 = Hacl_Hash_Blake2b_ivTable_S[2U]; + uint32_t iv3 = Hacl_Hash_Blake2b_ivTable_S[3U]; + uint32_t iv4 = Hacl_Hash_Blake2b_ivTable_S[4U]; + uint32_t iv5 = Hacl_Hash_Blake2b_ivTable_S[5U]; + uint32_t iv6 = Hacl_Hash_Blake2b_ivTable_S[6U]; + uint32_t iv7 = Hacl_Hash_Blake2b_ivTable_S[7U]; + r2[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv0, iv1, iv2, iv3); + r3[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv4, iv5, iv6, iv7); + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 4U; + uint8_t *bj = pv.salt + i0 * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i0] = x;); + KRML_MAYBE_FOR2(i0, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 6U; + uint8_t *bj = pv.personal + i0 * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i0] = x;); + tmp[0U] = + (uint32_t)pv.digest_length + ^ ((uint32_t)pv.key_length << 8U ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); + tmp[1U] = pv.leaf_length; + tmp[2U] = (uint32_t)pv.node_offset; + tmp[3U] = + (uint32_t)(pv.node_offset >> 32U) + ^ ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); + uint32_t tmp0 = tmp[0U]; + uint32_t tmp1 = tmp[1U]; + uint32_t tmp2 = tmp[2U]; + uint32_t tmp3 = tmp[3U]; + uint32_t tmp4 = tmp[4U]; + uint32_t tmp5 = tmp[5U]; + uint32_t tmp6 = tmp[6U]; + uint32_t tmp7 = tmp[7U]; + uint32_t iv0_ = iv0 ^ tmp0; + uint32_t iv1_ = iv1 ^ tmp1; + uint32_t iv2_ = iv2 ^ tmp2; + uint32_t iv3_ = iv3 ^ tmp3; + uint32_t iv4_ = iv4 ^ tmp4; + uint32_t iv5_ = iv5 ^ tmp5; + uint32_t iv6_ = iv6 ^ tmp6; + uint32_t iv7_ = iv7 ^ tmp7; + r0[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv0_, iv1_, iv2_, iv3_); + r1[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv4_, iv5_, iv6_, iv7_); + uint8_t kk11 = i.key_length; + uint32_t ite; + if (kk11 != 0U) + { + ite = 64U; + } + else + { + ite = 0U; + } + Hacl_Hash_Blake2s_Simd128_state_t + tmp8 = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)ite }; + state[0U] = tmp8; +} + +/** + General-purpose re-initialization function with parameters and +key. You cannot change digest_length, key_length, or last_node, meaning those values in +the parameters object must be the same as originally decided via one of the +malloc functions. All other values of the parameter can be changed. The behavior +is unspecified if you violate this precondition. +*/ +void +Hacl_Hash_Blake2s_Simd128_reset_with_key_and_params( + Hacl_Hash_Blake2s_Simd128_state_t *s, + Hacl_Hash_Blake2b_blake2_params *p, + uint8_t *k +) +{ + Hacl_Hash_Blake2b_index i1 = index_of_state(s); + KRML_MAYBE_UNUSED_VAR(i1); + reset_raw(s, ((Hacl_Hash_Blake2b_params_and_key){ .fst = p, .snd = k })); +} + +/** + Specialized-purpose re-initialization function with no parameters, +and a key. The key length must be the same as originally decided via your choice +of malloc function. All other parameters are reset to their default values. The +original call to malloc MUST have set digest_length to the default value. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2s_Simd128_reset_with_key(Hacl_Hash_Blake2s_Simd128_state_t *s, uint8_t *k) +{ + Hacl_Hash_Blake2b_index idx = index_of_state(s); + uint8_t salt[8U] = { 0U }; + uint8_t personal[8U] = { 0U }; + Hacl_Hash_Blake2b_blake2_params + p = + { + .digest_length = idx.digest_length, .key_length = idx.key_length, .fanout = 1U, .depth = 1U, + .leaf_length = 0U, .node_offset = 0ULL, .node_depth = 0U, .inner_length = 0U, .salt = salt, + .personal = personal + }; + Hacl_Hash_Blake2b_blake2_params p0 = p; + reset_raw(s, ((Hacl_Hash_Blake2b_params_and_key){ .fst = &p0, .snd = k })); +} + +/** + Specialized-purpose re-initialization function with no parameters +and no key. This is what you want if you intend to use Blake2 as a hash +function. The key length and digest length must have been set to their +respective default values via your choice of malloc function (always true if you +used `malloc`). All other parameters are reset to their default values. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2s_Simd128_reset(Hacl_Hash_Blake2s_Simd128_state_t *s) +{ + Hacl_Hash_Blake2s_Simd128_reset_with_key(s, NULL); +} + +/** + Update function; 0 = success, 1 = max length exceeded +*/ +Hacl_Streaming_Types_error_code +Hacl_Hash_Blake2s_Simd128_update( + Hacl_Hash_Blake2s_Simd128_state_t *state, + uint8_t *chunk, + uint32_t chunk_len +) +{ + Hacl_Hash_Blake2s_Simd128_state_t s = *state; + uint64_t total_len = s.total_len; + if ((uint64_t)chunk_len > 0xffffffffffffffffULL - total_len) + { + return Hacl_Streaming_Types_MaximumLengthExceeded; + } + uint32_t sz; + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) + { + sz = 64U; + } + else + { + sz = (uint32_t)(total_len % (uint64_t)64U); + } + if (chunk_len <= 64U - sz) + { + Hacl_Hash_Blake2s_Simd128_state_t s1 = *state; + Hacl_Hash_Blake2s_Simd128_block_state_t block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) + { + sz1 = 64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); + } + uint8_t *buf2 = buf + sz1; + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state + = + ( + (Hacl_Hash_Blake2s_Simd128_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len2 + } + ); + } + else if (sz == 0U) + { + Hacl_Hash_Blake2s_Simd128_state_t s1 = *state; + Hacl_Hash_Blake2s_Simd128_block_state_t block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) + { + sz1 = 64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); + } + if (!(sz1 == 0U)) + { + uint64_t prevlen = total_len1 - (uint64_t)sz1; + K____Lib_IntVector_Intrinsics_vec128___Lib_IntVector_Intrinsics_vec128_ acc = block_state1.f3; + Lib_IntVector_Intrinsics_vec128 *wv = acc.fst; + Lib_IntVector_Intrinsics_vec128 *hash = acc.snd; + uint32_t nb = 1U; + Hacl_Hash_Blake2s_Simd128_update_multi(64U, wv, hash, prevlen, buf, nb); + } + uint32_t ite; + if ((uint64_t)chunk_len % (uint64_t)64U == 0ULL && (uint64_t)chunk_len > 0ULL) + { + ite = 64U; + } + else + { + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)64U); + } + uint32_t n_blocks = (chunk_len - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; + K____Lib_IntVector_Intrinsics_vec128___Lib_IntVector_Intrinsics_vec128_ acc = block_state1.f3; + Lib_IntVector_Intrinsics_vec128 *wv = acc.fst; + Lib_IntVector_Intrinsics_vec128 *hash = acc.snd; + uint32_t nb = data1_len / 64U; + Hacl_Hash_Blake2s_Simd128_update_multi(data1_len, wv, hash, total_len1, data1, nb); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state + = + ( + (Hacl_Hash_Blake2s_Simd128_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)chunk_len + } + ); + } + else + { + uint32_t diff = 64U - sz; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Hash_Blake2s_Simd128_state_t s1 = *state; + Hacl_Hash_Blake2s_Simd128_block_state_t block_state10 = s1.block_state; + uint8_t *buf0 = s1.buf; + uint64_t total_len10 = s1.total_len; + uint32_t sz10; + if (total_len10 % (uint64_t)64U == 0ULL && total_len10 > 0ULL) + { + sz10 = 64U; + } + else + { + sz10 = (uint32_t)(total_len10 % (uint64_t)64U); + } + uint8_t *buf2 = buf0 + sz10; + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); + uint64_t total_len2 = total_len10 + (uint64_t)diff; + *state + = + ( + (Hacl_Hash_Blake2s_Simd128_state_t){ + .block_state = block_state10, + .buf = buf0, + .total_len = total_len2 + } + ); + Hacl_Hash_Blake2s_Simd128_state_t s10 = *state; + Hacl_Hash_Blake2s_Simd128_block_state_t block_state1 = s10.block_state; + uint8_t *buf = s10.buf; + uint64_t total_len1 = s10.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) + { + sz1 = 64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); + } + if (!(sz1 == 0U)) + { + uint64_t prevlen = total_len1 - (uint64_t)sz1; + K____Lib_IntVector_Intrinsics_vec128___Lib_IntVector_Intrinsics_vec128_ acc = block_state1.f3; + Lib_IntVector_Intrinsics_vec128 *wv = acc.fst; + Lib_IntVector_Intrinsics_vec128 *hash = acc.snd; + uint32_t nb = 1U; + Hacl_Hash_Blake2s_Simd128_update_multi(64U, wv, hash, prevlen, buf, nb); + } + uint32_t ite; + if + ((uint64_t)(chunk_len - diff) % (uint64_t)64U == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL) + { + ite = 64U; + } + else + { + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)64U); + } + uint32_t n_blocks = (chunk_len - diff - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; + K____Lib_IntVector_Intrinsics_vec128___Lib_IntVector_Intrinsics_vec128_ acc = block_state1.f3; + Lib_IntVector_Intrinsics_vec128 *wv = acc.fst; + Lib_IntVector_Intrinsics_vec128 *hash = acc.snd; + uint32_t nb = data1_len / 64U; + Hacl_Hash_Blake2s_Simd128_update_multi(data1_len, wv, hash, total_len1, data1, nb); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state + = + ( + (Hacl_Hash_Blake2s_Simd128_state_t){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)(chunk_len - diff) + } + ); + } + return Hacl_Streaming_Types_Success; +} + +/** + Digest function. This function expects the `output` array to hold +at least `digest_length` bytes, where `digest_length` was determined by your +choice of `malloc` function. Concretely, if you used `malloc` or +`malloc_with_key`, then the expected length is 128 for S, or 64 for B (default +digest length). If you used `malloc_with_params_and_key`, then the expected +length is whatever you chose for the `digest_length` field of your parameters. +For convenience, this function returns `digest_length`. When in doubt, callers +can pass an array of size HACL_BLAKE2S_128_OUT_BYTES, then use the return value +to see how many bytes were actually written. +*/ +uint8_t Hacl_Hash_Blake2s_Simd128_digest(Hacl_Hash_Blake2s_Simd128_state_t *s, uint8_t *dst) +{ + Hacl_Hash_Blake2s_Simd128_block_state_t block_state0 = (*s).block_state; + bool last_node0 = block_state0.thd; + uint8_t nn0 = block_state0.snd; + uint8_t kk0 = block_state0.fst; + Hacl_Hash_Blake2b_index + i1 = { .key_length = kk0, .digest_length = nn0, .last_node = last_node0 }; + Hacl_Hash_Blake2s_Simd128_state_t scrut = *s; + Hacl_Hash_Blake2s_Simd128_block_state_t block_state = scrut.block_state; + uint8_t *buf_ = scrut.buf; + uint64_t total_len = scrut.total_len; + uint32_t r; + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) + { + r = 64U; + } + else + { + r = (uint32_t)(total_len % (uint64_t)64U); + } + uint8_t *buf_1 = buf_; + KRML_PRE_ALIGN(16) Lib_IntVector_Intrinsics_vec128 wv0[4U] KRML_POST_ALIGN(16) = { 0U }; + KRML_PRE_ALIGN(16) Lib_IntVector_Intrinsics_vec128 b[4U] KRML_POST_ALIGN(16) = { 0U }; + Hacl_Hash_Blake2s_Simd128_block_state_t + tmp_block_state = + { + .fst = i1.key_length, + .snd = i1.digest_length, + .thd = i1.last_node, + .f3 = { .fst = wv0, .snd = b } + }; + Lib_IntVector_Intrinsics_vec128 *src_b = block_state.f3.snd; + Lib_IntVector_Intrinsics_vec128 *dst_b = tmp_block_state.f3.snd; + memcpy(dst_b, src_b, 4U * sizeof (Lib_IntVector_Intrinsics_vec128)); + uint64_t prev_len = total_len - (uint64_t)r; + uint32_t ite; + if (r % 64U == 0U && r > 0U) + { + ite = 64U; + } + else + { + ite = r % 64U; + } + uint8_t *buf_last = buf_1 + r - ite; + uint8_t *buf_multi = buf_1; + K____Lib_IntVector_Intrinsics_vec128___Lib_IntVector_Intrinsics_vec128_ + acc0 = tmp_block_state.f3; + Lib_IntVector_Intrinsics_vec128 *wv1 = acc0.fst; + Lib_IntVector_Intrinsics_vec128 *hash0 = acc0.snd; + uint32_t nb = 0U; + Hacl_Hash_Blake2s_Simd128_update_multi(0U, wv1, hash0, prev_len, buf_multi, nb); + uint64_t prev_len_last = total_len - (uint64_t)r; + K____Lib_IntVector_Intrinsics_vec128___Lib_IntVector_Intrinsics_vec128_ + acc = tmp_block_state.f3; + bool last_node1 = tmp_block_state.thd; + Lib_IntVector_Intrinsics_vec128 *wv = acc.fst; + Lib_IntVector_Intrinsics_vec128 *hash = acc.snd; + Hacl_Hash_Blake2s_Simd128_update_last(r, wv, hash, last_node1, prev_len_last, r, buf_last); + uint8_t nn1 = tmp_block_state.snd; + Hacl_Hash_Blake2s_Simd128_finish((uint32_t)nn1, dst, tmp_block_state.f3.snd); + Hacl_Hash_Blake2s_Simd128_block_state_t block_state1 = (*s).block_state; + bool last_node = block_state1.thd; + uint8_t nn = block_state1.snd; + uint8_t kk = block_state1.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk, .digest_length = nn, .last_node = last_node }).digest_length; +} + +Hacl_Hash_Blake2b_index Hacl_Hash_Blake2s_Simd128_info(Hacl_Hash_Blake2s_Simd128_state_t *s) +{ + Hacl_Hash_Blake2s_Simd128_block_state_t block_state = (*s).block_state; + bool last_node = block_state.thd; + uint8_t nn = block_state.snd; + uint8_t kk = block_state.fst; + return + ((Hacl_Hash_Blake2b_index){ .key_length = kk, .digest_length = nn, .last_node = last_node }); +} + +/** + Free state function when there is no key +*/ +void Hacl_Hash_Blake2s_Simd128_free(Hacl_Hash_Blake2s_Simd128_state_t *state) +{ + Hacl_Hash_Blake2s_Simd128_state_t scrut = *state; + uint8_t *buf = scrut.buf; + Hacl_Hash_Blake2s_Simd128_block_state_t block_state = scrut.block_state; + Lib_IntVector_Intrinsics_vec128 *b = block_state.f3.snd; + Lib_IntVector_Intrinsics_vec128 *wv = block_state.f3.fst; + KRML_ALIGNED_FREE(wv); + KRML_ALIGNED_FREE(b); + KRML_HOST_FREE(buf); + KRML_HOST_FREE(state); +} + +/** + Copying. This preserves all parameters. +*/ +Hacl_Hash_Blake2s_Simd128_state_t +*Hacl_Hash_Blake2s_Simd128_copy(Hacl_Hash_Blake2s_Simd128_state_t *state) +{ + Hacl_Hash_Blake2s_Simd128_state_t scrut = *state; + Hacl_Hash_Blake2s_Simd128_block_state_t block_state0 = scrut.block_state; + uint8_t *buf0 = scrut.buf; + uint64_t total_len0 = scrut.total_len; + bool last_node = block_state0.thd; + uint8_t nn = block_state0.snd; + uint8_t kk1 = block_state0.fst; + Hacl_Hash_Blake2b_index i = { .key_length = kk1, .digest_length = nn, .last_node = last_node }; + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + memcpy(buf, buf0, 64U * sizeof (uint8_t)); + Lib_IntVector_Intrinsics_vec128 + *wv = + (Lib_IntVector_Intrinsics_vec128 *)KRML_ALIGNED_MALLOC(16, + sizeof (Lib_IntVector_Intrinsics_vec128) * 4U); + memset(wv, 0U, 4U * sizeof (Lib_IntVector_Intrinsics_vec128)); + Lib_IntVector_Intrinsics_vec128 + *b = + (Lib_IntVector_Intrinsics_vec128 *)KRML_ALIGNED_MALLOC(16, + sizeof (Lib_IntVector_Intrinsics_vec128) * 4U); + memset(b, 0U, 4U * sizeof (Lib_IntVector_Intrinsics_vec128)); + Hacl_Hash_Blake2s_Simd128_block_state_t + block_state = + { + .fst = i.key_length, + .snd = i.digest_length, + .thd = i.last_node, + .f3 = { .fst = wv, .snd = b } + }; + Lib_IntVector_Intrinsics_vec128 *src_b = block_state0.f3.snd; + Lib_IntVector_Intrinsics_vec128 *dst_b = block_state.f3.snd; + memcpy(dst_b, src_b, 4U * sizeof (Lib_IntVector_Intrinsics_vec128)); + Hacl_Hash_Blake2s_Simd128_state_t + s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; + Hacl_Hash_Blake2s_Simd128_state_t + *p = + (Hacl_Hash_Blake2s_Simd128_state_t *)KRML_HOST_MALLOC(sizeof ( + Hacl_Hash_Blake2s_Simd128_state_t + )); + p[0U] = s; + return p; +} + +/** +Write the BLAKE2s digest of message `input` using key `key` into `output`. + +@param output Pointer to `output_len` bytes of memory where the digest is written to. +@param output_len Length of the to-be-generated digest with 1 <= `output_len` <= 64. +@param input Pointer to `input_len` bytes of memory where the input message is read from. +@param input_len Length of the input message. +@param key Pointer to `key_len` bytes of memory where the key is read from. +@param key_len Length of the key. Can be 0. +*/ +void +Hacl_Hash_Blake2s_Simd128_hash_with_key( + uint8_t *output, + uint32_t output_len, + uint8_t *input, + uint32_t input_len, + uint8_t *key, + uint32_t key_len +) +{ + KRML_PRE_ALIGN(16) Lib_IntVector_Intrinsics_vec128 b[4U] KRML_POST_ALIGN(16) = { 0U }; + KRML_PRE_ALIGN(16) Lib_IntVector_Intrinsics_vec128 b1[4U] KRML_POST_ALIGN(16) = { 0U }; + Hacl_Hash_Blake2s_Simd128_init(b, key_len, output_len); + update(b1, b, key_len, key, input_len, input); + Hacl_Hash_Blake2s_Simd128_finish(output_len, output, b); + Lib_Memzero0_memzero(b1, 4U, Lib_IntVector_Intrinsics_vec128, void *); + Lib_Memzero0_memzero(b, 4U, Lib_IntVector_Intrinsics_vec128, void *); +} + +/** +Write the BLAKE2s digest of message `input` using key `key` and +parameters `params` into `output`. The `key` array must be of length +`params.key_length`. The `output` array must be of length +`params.digest_length`. +*/ +void +Hacl_Hash_Blake2s_Simd128_hash_with_key_and_params( + uint8_t *output, + uint8_t *input, + uint32_t input_len, + Hacl_Hash_Blake2b_blake2_params params, + uint8_t *key +) +{ + KRML_PRE_ALIGN(16) Lib_IntVector_Intrinsics_vec128 b[4U] KRML_POST_ALIGN(16) = { 0U }; + KRML_PRE_ALIGN(16) Lib_IntVector_Intrinsics_vec128 b1[4U] KRML_POST_ALIGN(16) = { 0U }; + uint32_t tmp[8U] = { 0U }; + Lib_IntVector_Intrinsics_vec128 *r0 = b; + Lib_IntVector_Intrinsics_vec128 *r1 = b + 1U; + Lib_IntVector_Intrinsics_vec128 *r2 = b + 2U; + Lib_IntVector_Intrinsics_vec128 *r3 = b + 3U; + uint32_t iv0 = Hacl_Hash_Blake2b_ivTable_S[0U]; + uint32_t iv1 = Hacl_Hash_Blake2b_ivTable_S[1U]; + uint32_t iv2 = Hacl_Hash_Blake2b_ivTable_S[2U]; + uint32_t iv3 = Hacl_Hash_Blake2b_ivTable_S[3U]; + uint32_t iv4 = Hacl_Hash_Blake2b_ivTable_S[4U]; + uint32_t iv5 = Hacl_Hash_Blake2b_ivTable_S[5U]; + uint32_t iv6 = Hacl_Hash_Blake2b_ivTable_S[6U]; + uint32_t iv7 = Hacl_Hash_Blake2b_ivTable_S[7U]; + r2[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv0, iv1, iv2, iv3); + r3[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv4, iv5, iv6, iv7); + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 4U; + uint8_t *bj = params.salt + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + KRML_MAYBE_FOR2(i, + 0U, + 2U, + 1U, + uint32_t *os = tmp + 6U; + uint8_t *bj = params.personal + i * 4U; + uint32_t u = load32_le(bj); + uint32_t r = u; + uint32_t x = r; + os[i] = x;); + tmp[0U] = + (uint32_t)params.digest_length + ^ + ((uint32_t)params.key_length + << 8U + ^ ((uint32_t)params.fanout << 16U ^ (uint32_t)params.depth << 24U)); + tmp[1U] = params.leaf_length; + tmp[2U] = (uint32_t)params.node_offset; + tmp[3U] = + (uint32_t)(params.node_offset >> 32U) + ^ ((uint32_t)params.node_depth << 16U ^ (uint32_t)params.inner_length << 24U); + uint32_t tmp0 = tmp[0U]; + uint32_t tmp1 = tmp[1U]; + uint32_t tmp2 = tmp[2U]; + uint32_t tmp3 = tmp[3U]; + uint32_t tmp4 = tmp[4U]; + uint32_t tmp5 = tmp[5U]; + uint32_t tmp6 = tmp[6U]; + uint32_t tmp7 = tmp[7U]; + uint32_t iv0_ = iv0 ^ tmp0; + uint32_t iv1_ = iv1 ^ tmp1; + uint32_t iv2_ = iv2 ^ tmp2; + uint32_t iv3_ = iv3 ^ tmp3; + uint32_t iv4_ = iv4 ^ tmp4; + uint32_t iv5_ = iv5 ^ tmp5; + uint32_t iv6_ = iv6 ^ tmp6; + uint32_t iv7_ = iv7 ^ tmp7; + r0[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv0_, iv1_, iv2_, iv3_); + r1[0U] = Lib_IntVector_Intrinsics_vec128_load32s(iv4_, iv5_, iv6_, iv7_); + update(b1, b, (uint32_t)params.key_length, key, input_len, input); + Hacl_Hash_Blake2s_Simd128_finish((uint32_t)params.digest_length, output, b); + Lib_Memzero0_memzero(b1, 4U, Lib_IntVector_Intrinsics_vec128, void *); + Lib_Memzero0_memzero(b, 4U, Lib_IntVector_Intrinsics_vec128, void *); +} + diff --git a/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.h b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.h new file mode 100644 index 00000000000000..cd1654c9726dc0 --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.h @@ -0,0 +1,230 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __Hacl_Hash_Blake2s_Simd128_H +#define __Hacl_Hash_Blake2s_Simd128_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "python_hacl_namespaces.h" +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "Hacl_Streaming_Types.h" +#include "Hacl_Hash_Blake2b.h" +#include "libintvector.h" + +#define HACL_HASH_BLAKE2S_SIMD128_BLOCK_BYTES (64U) + +#define HACL_HASH_BLAKE2S_SIMD128_OUT_BYTES (32U) + +#define HACL_HASH_BLAKE2S_SIMD128_KEY_BYTES (32U) + +#define HACL_HASH_BLAKE2S_SIMD128_SALT_BYTES (8U) + +#define HACL_HASH_BLAKE2S_SIMD128_PERSONAL_BYTES (8U) + +typedef struct K____Lib_IntVector_Intrinsics_vec128___Lib_IntVector_Intrinsics_vec128__s +{ + Lib_IntVector_Intrinsics_vec128 *fst; + Lib_IntVector_Intrinsics_vec128 *snd; +} +K____Lib_IntVector_Intrinsics_vec128___Lib_IntVector_Intrinsics_vec128_; + +typedef struct Hacl_Hash_Blake2s_Simd128_block_state_t_s +{ + uint8_t fst; + uint8_t snd; + bool thd; + K____Lib_IntVector_Intrinsics_vec128___Lib_IntVector_Intrinsics_vec128_ f3; +} +Hacl_Hash_Blake2s_Simd128_block_state_t; + +typedef struct Hacl_Hash_Blake2s_Simd128_state_t_s +{ + Hacl_Hash_Blake2s_Simd128_block_state_t block_state; + uint8_t *buf; + uint64_t total_len; +} +Hacl_Hash_Blake2s_Simd128_state_t; + +/** + General-purpose allocation function that gives control over all +Blake2 parameters, including the key. Further resettings of the state SHALL be +done with `reset_with_params_and_key`, and SHALL feature the exact same values +for the `key_length` and `digest_length` fields as passed here. In other words, +once you commit to a digest and key length, the only way to change these +parameters is to allocate a new object. + +The caller must satisfy the following requirements. +- The length of the key k MUST match the value of the field key_length in the + parameters. +- The key_length must not exceed 128 for S, 64 for B. +- The digest_length must not exceed 128 for S, 64 for B. + +*/ +Hacl_Hash_Blake2s_Simd128_state_t +*Hacl_Hash_Blake2s_Simd128_malloc_with_params_and_key( + Hacl_Hash_Blake2b_blake2_params *p, + bool last_node, + uint8_t *k +); + +/** + Specialized allocation function that picks default values for all +parameters, except for the key_length. Further resettings of the state SHALL be +done with `reset_with_key`, and SHALL feature the exact same key length `kk` as +passed here. In other words, once you commit to a key length, the only way to +change this parameter is to allocate a new object. + +The caller must satisfy the following requirements. +- The key_length must not exceed 128 for S, 64 for B. + +*/ +Hacl_Hash_Blake2s_Simd128_state_t +*Hacl_Hash_Blake2s_Simd128_malloc_with_key0(uint8_t *k, uint8_t kk); + +/** + Specialized allocation function that picks default values for all +parameters, and has no key. Effectively, this is what you want if you intend to +use Blake2 as a hash function. Further resettings of the state SHALL be done with `reset`. +*/ +Hacl_Hash_Blake2s_Simd128_state_t *Hacl_Hash_Blake2s_Simd128_malloc(void); + +/** + General-purpose re-initialization function with parameters and +key. You cannot change digest_length, key_length, or last_node, meaning those values in +the parameters object must be the same as originally decided via one of the +malloc functions. All other values of the parameter can be changed. The behavior +is unspecified if you violate this precondition. +*/ +void +Hacl_Hash_Blake2s_Simd128_reset_with_key_and_params( + Hacl_Hash_Blake2s_Simd128_state_t *s, + Hacl_Hash_Blake2b_blake2_params *p, + uint8_t *k +); + +/** + Specialized-purpose re-initialization function with no parameters, +and a key. The key length must be the same as originally decided via your choice +of malloc function. All other parameters are reset to their default values. The +original call to malloc MUST have set digest_length to the default value. The +behavior is unspecified if you violate this precondition. +*/ +void +Hacl_Hash_Blake2s_Simd128_reset_with_key(Hacl_Hash_Blake2s_Simd128_state_t *s, uint8_t *k); + +/** + Specialized-purpose re-initialization function with no parameters +and no key. This is what you want if you intend to use Blake2 as a hash +function. The key length and digest length must have been set to their +respective default values via your choice of malloc function (always true if you +used `malloc`). All other parameters are reset to their default values. The +behavior is unspecified if you violate this precondition. +*/ +void Hacl_Hash_Blake2s_Simd128_reset(Hacl_Hash_Blake2s_Simd128_state_t *s); + +/** + Update function; 0 = success, 1 = max length exceeded +*/ +Hacl_Streaming_Types_error_code +Hacl_Hash_Blake2s_Simd128_update( + Hacl_Hash_Blake2s_Simd128_state_t *state, + uint8_t *chunk, + uint32_t chunk_len +); + +/** + Digest function. This function expects the `output` array to hold +at least `digest_length` bytes, where `digest_length` was determined by your +choice of `malloc` function. Concretely, if you used `malloc` or +`malloc_with_key`, then the expected length is 128 for S, or 64 for B (default +digest length). If you used `malloc_with_params_and_key`, then the expected +length is whatever you chose for the `digest_length` field of your parameters. +For convenience, this function returns `digest_length`. When in doubt, callers +can pass an array of size HACL_BLAKE2S_128_OUT_BYTES, then use the return value +to see how many bytes were actually written. +*/ +uint8_t Hacl_Hash_Blake2s_Simd128_digest(Hacl_Hash_Blake2s_Simd128_state_t *s, uint8_t *dst); + +Hacl_Hash_Blake2b_index Hacl_Hash_Blake2s_Simd128_info(Hacl_Hash_Blake2s_Simd128_state_t *s); + +/** + Free state function when there is no key +*/ +void Hacl_Hash_Blake2s_Simd128_free(Hacl_Hash_Blake2s_Simd128_state_t *state); + +/** + Copying. This preserves all parameters. +*/ +Hacl_Hash_Blake2s_Simd128_state_t +*Hacl_Hash_Blake2s_Simd128_copy(Hacl_Hash_Blake2s_Simd128_state_t *state); + +/** +Write the BLAKE2s digest of message `input` using key `key` into `output`. + +@param output Pointer to `output_len` bytes of memory where the digest is written to. +@param output_len Length of the to-be-generated digest with 1 <= `output_len` <= 64. +@param input Pointer to `input_len` bytes of memory where the input message is read from. +@param input_len Length of the input message. +@param key Pointer to `key_len` bytes of memory where the key is read from. +@param key_len Length of the key. Can be 0. +*/ +void +Hacl_Hash_Blake2s_Simd128_hash_with_key( + uint8_t *output, + uint32_t output_len, + uint8_t *input, + uint32_t input_len, + uint8_t *key, + uint32_t key_len +); + +/** +Write the BLAKE2s digest of message `input` using key `key` and +parameters `params` into `output`. The `key` array must be of length +`params.key_length`. The `output` array must be of length +`params.digest_length`. +*/ +void +Hacl_Hash_Blake2s_Simd128_hash_with_key_and_params( + uint8_t *output, + uint8_t *input, + uint32_t input_len, + Hacl_Hash_Blake2b_blake2_params params, + uint8_t *key +); + +#if defined(__cplusplus) +} +#endif + +#define __Hacl_Hash_Blake2s_Simd128_H_DEFINED +#endif diff --git a/Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.c b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.c new file mode 100644 index 00000000000000..951306db494833 --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.c @@ -0,0 +1,14 @@ +// This file isn't part of a standard HACL source tree. +// +// It is required for compatibility with universal2 macOS builds. The code in +// Hacl_Hash_Blake2s_Simd128.c will compile on macOS ARM64, but performance +// isn't great, so it's disabled. However, because universal2 builds are +// compiled in a single pass, autoconf detects that the required compiler +// features *are* available, and tries to include this file. +// +// To compensate for this, autoconf will include *this* file instead of +// Hacl_Hash_Blake2s_Simd128.c when compiling for universal. This allows the +// underlying source code of HACL to remain unmodified. +#if !(defined(__APPLE__) && defined(__arm64__)) +#include "Hacl_Hash_Blake2s_Simd128.c" +#endif diff --git a/Modules/_hacl/Hacl_Hash_MD5.c b/Modules/_hacl/Hacl_Hash_MD5.c index 222ac824f01961..ed294839ed8dc0 100644 --- a/Modules/_hacl/Hacl_Hash_MD5.c +++ b/Modules/_hacl/Hacl_Hash_MD5.c @@ -25,37 +25,29 @@ #include "internal/Hacl_Hash_MD5.h" -static uint32_t -_h0[4U] = - { (uint32_t)0x67452301U, (uint32_t)0xefcdab89U, (uint32_t)0x98badcfeU, (uint32_t)0x10325476U }; +static uint32_t _h0[4U] = { 0x67452301U, 0xefcdab89U, 0x98badcfeU, 0x10325476U }; static uint32_t _t[64U] = { - (uint32_t)0xd76aa478U, (uint32_t)0xe8c7b756U, (uint32_t)0x242070dbU, (uint32_t)0xc1bdceeeU, - (uint32_t)0xf57c0fafU, (uint32_t)0x4787c62aU, (uint32_t)0xa8304613U, (uint32_t)0xfd469501U, - (uint32_t)0x698098d8U, (uint32_t)0x8b44f7afU, (uint32_t)0xffff5bb1U, (uint32_t)0x895cd7beU, - (uint32_t)0x6b901122U, (uint32_t)0xfd987193U, (uint32_t)0xa679438eU, (uint32_t)0x49b40821U, - (uint32_t)0xf61e2562U, (uint32_t)0xc040b340U, (uint32_t)0x265e5a51U, (uint32_t)0xe9b6c7aaU, - (uint32_t)0xd62f105dU, (uint32_t)0x02441453U, (uint32_t)0xd8a1e681U, (uint32_t)0xe7d3fbc8U, - (uint32_t)0x21e1cde6U, (uint32_t)0xc33707d6U, (uint32_t)0xf4d50d87U, (uint32_t)0x455a14edU, - (uint32_t)0xa9e3e905U, (uint32_t)0xfcefa3f8U, (uint32_t)0x676f02d9U, (uint32_t)0x8d2a4c8aU, - (uint32_t)0xfffa3942U, (uint32_t)0x8771f681U, (uint32_t)0x6d9d6122U, (uint32_t)0xfde5380cU, - (uint32_t)0xa4beea44U, (uint32_t)0x4bdecfa9U, (uint32_t)0xf6bb4b60U, (uint32_t)0xbebfbc70U, - (uint32_t)0x289b7ec6U, (uint32_t)0xeaa127faU, (uint32_t)0xd4ef3085U, (uint32_t)0x4881d05U, - (uint32_t)0xd9d4d039U, (uint32_t)0xe6db99e5U, (uint32_t)0x1fa27cf8U, (uint32_t)0xc4ac5665U, - (uint32_t)0xf4292244U, (uint32_t)0x432aff97U, (uint32_t)0xab9423a7U, (uint32_t)0xfc93a039U, - (uint32_t)0x655b59c3U, (uint32_t)0x8f0ccc92U, (uint32_t)0xffeff47dU, (uint32_t)0x85845dd1U, - (uint32_t)0x6fa87e4fU, (uint32_t)0xfe2ce6e0U, (uint32_t)0xa3014314U, (uint32_t)0x4e0811a1U, - (uint32_t)0xf7537e82U, (uint32_t)0xbd3af235U, (uint32_t)0x2ad7d2bbU, (uint32_t)0xeb86d391U + 0xd76aa478U, 0xe8c7b756U, 0x242070dbU, 0xc1bdceeeU, 0xf57c0fafU, 0x4787c62aU, 0xa8304613U, + 0xfd469501U, 0x698098d8U, 0x8b44f7afU, 0xffff5bb1U, 0x895cd7beU, 0x6b901122U, 0xfd987193U, + 0xa679438eU, 0x49b40821U, 0xf61e2562U, 0xc040b340U, 0x265e5a51U, 0xe9b6c7aaU, 0xd62f105dU, + 0x02441453U, 0xd8a1e681U, 0xe7d3fbc8U, 0x21e1cde6U, 0xc33707d6U, 0xf4d50d87U, 0x455a14edU, + 0xa9e3e905U, 0xfcefa3f8U, 0x676f02d9U, 0x8d2a4c8aU, 0xfffa3942U, 0x8771f681U, 0x6d9d6122U, + 0xfde5380cU, 0xa4beea44U, 0x4bdecfa9U, 0xf6bb4b60U, 0xbebfbc70U, 0x289b7ec6U, 0xeaa127faU, + 0xd4ef3085U, 0x4881d05U, 0xd9d4d039U, 0xe6db99e5U, 0x1fa27cf8U, 0xc4ac5665U, 0xf4292244U, + 0x432aff97U, 0xab9423a7U, 0xfc93a039U, 0x655b59c3U, 0x8f0ccc92U, 0xffeff47dU, 0x85845dd1U, + 0x6fa87e4fU, 0xfe2ce6e0U, 0xa3014314U, 0x4e0811a1U, 0xf7537e82U, 0xbd3af235U, 0x2ad7d2bbU, + 0xeb86d391U }; -void Hacl_Hash_Core_MD5_legacy_init(uint32_t *s) +void Hacl_Hash_MD5_init(uint32_t *s) { - KRML_MAYBE_FOR4(i, (uint32_t)0U, (uint32_t)4U, (uint32_t)1U, s[i] = _h0[i];); + KRML_MAYBE_FOR4(i, 0U, 4U, 1U, s[i] = _h0[i];); } -static void legacy_update(uint32_t *abcd, uint8_t *x) +static void update(uint32_t *abcd, uint8_t *x) { uint32_t aa = abcd[0U]; uint32_t bb = abcd[1U]; @@ -74,14 +66,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb0 + ((va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) - << (uint32_t)7U - | (va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) >> (uint32_t)25U); + << 7U + | (va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) >> 25U); abcd[0U] = v; uint32_t va0 = abcd[3U]; uint32_t vb1 = abcd[0U]; uint32_t vc1 = abcd[1U]; uint32_t vd1 = abcd[2U]; - uint8_t *b1 = x + (uint32_t)4U; + uint8_t *b1 = x + 4U; uint32_t u0 = load32_le(b1); uint32_t xk0 = u0; uint32_t ti1 = _t[1U]; @@ -90,14 +82,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb1 + ((va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) - << (uint32_t)12U - | (va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) >> (uint32_t)20U); + << 12U + | (va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) >> 20U); abcd[3U] = v0; uint32_t va1 = abcd[2U]; uint32_t vb2 = abcd[3U]; uint32_t vc2 = abcd[0U]; uint32_t vd2 = abcd[1U]; - uint8_t *b2 = x + (uint32_t)8U; + uint8_t *b2 = x + 8U; uint32_t u1 = load32_le(b2); uint32_t xk1 = u1; uint32_t ti2 = _t[2U]; @@ -106,14 +98,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb2 + ((va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) - << (uint32_t)17U - | (va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) >> (uint32_t)15U); + << 17U + | (va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) >> 15U); abcd[2U] = v1; uint32_t va2 = abcd[1U]; uint32_t vb3 = abcd[2U]; uint32_t vc3 = abcd[3U]; uint32_t vd3 = abcd[0U]; - uint8_t *b3 = x + (uint32_t)12U; + uint8_t *b3 = x + 12U; uint32_t u2 = load32_le(b3); uint32_t xk2 = u2; uint32_t ti3 = _t[3U]; @@ -122,14 +114,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb3 + ((va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) - << (uint32_t)22U - | (va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) >> (uint32_t)10U); + << 22U + | (va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) >> 10U); abcd[1U] = v2; uint32_t va3 = abcd[0U]; uint32_t vb4 = abcd[1U]; uint32_t vc4 = abcd[2U]; uint32_t vd4 = abcd[3U]; - uint8_t *b4 = x + (uint32_t)16U; + uint8_t *b4 = x + 16U; uint32_t u3 = load32_le(b4); uint32_t xk3 = u3; uint32_t ti4 = _t[4U]; @@ -138,14 +130,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb4 + ((va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) - << (uint32_t)7U - | (va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) >> (uint32_t)25U); + << 7U + | (va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) >> 25U); abcd[0U] = v3; uint32_t va4 = abcd[3U]; uint32_t vb5 = abcd[0U]; uint32_t vc5 = abcd[1U]; uint32_t vd5 = abcd[2U]; - uint8_t *b5 = x + (uint32_t)20U; + uint8_t *b5 = x + 20U; uint32_t u4 = load32_le(b5); uint32_t xk4 = u4; uint32_t ti5 = _t[5U]; @@ -154,14 +146,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb5 + ((va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) - << (uint32_t)12U - | (va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) >> (uint32_t)20U); + << 12U + | (va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) >> 20U); abcd[3U] = v4; uint32_t va5 = abcd[2U]; uint32_t vb6 = abcd[3U]; uint32_t vc6 = abcd[0U]; uint32_t vd6 = abcd[1U]; - uint8_t *b6 = x + (uint32_t)24U; + uint8_t *b6 = x + 24U; uint32_t u5 = load32_le(b6); uint32_t xk5 = u5; uint32_t ti6 = _t[6U]; @@ -170,14 +162,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb6 + ((va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) - << (uint32_t)17U - | (va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) >> (uint32_t)15U); + << 17U + | (va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) >> 15U); abcd[2U] = v5; uint32_t va6 = abcd[1U]; uint32_t vb7 = abcd[2U]; uint32_t vc7 = abcd[3U]; uint32_t vd7 = abcd[0U]; - uint8_t *b7 = x + (uint32_t)28U; + uint8_t *b7 = x + 28U; uint32_t u6 = load32_le(b7); uint32_t xk6 = u6; uint32_t ti7 = _t[7U]; @@ -186,14 +178,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb7 + ((va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) - << (uint32_t)22U - | (va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) >> (uint32_t)10U); + << 22U + | (va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) >> 10U); abcd[1U] = v6; uint32_t va7 = abcd[0U]; uint32_t vb8 = abcd[1U]; uint32_t vc8 = abcd[2U]; uint32_t vd8 = abcd[3U]; - uint8_t *b8 = x + (uint32_t)32U; + uint8_t *b8 = x + 32U; uint32_t u7 = load32_le(b8); uint32_t xk7 = u7; uint32_t ti8 = _t[8U]; @@ -202,14 +194,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb8 + ((va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) - << (uint32_t)7U - | (va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) >> (uint32_t)25U); + << 7U + | (va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) >> 25U); abcd[0U] = v7; uint32_t va8 = abcd[3U]; uint32_t vb9 = abcd[0U]; uint32_t vc9 = abcd[1U]; uint32_t vd9 = abcd[2U]; - uint8_t *b9 = x + (uint32_t)36U; + uint8_t *b9 = x + 36U; uint32_t u8 = load32_le(b9); uint32_t xk8 = u8; uint32_t ti9 = _t[9U]; @@ -218,14 +210,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb9 + ((va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) - << (uint32_t)12U - | (va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) >> (uint32_t)20U); + << 12U + | (va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) >> 20U); abcd[3U] = v8; uint32_t va9 = abcd[2U]; uint32_t vb10 = abcd[3U]; uint32_t vc10 = abcd[0U]; uint32_t vd10 = abcd[1U]; - uint8_t *b10 = x + (uint32_t)40U; + uint8_t *b10 = x + 40U; uint32_t u9 = load32_le(b10); uint32_t xk9 = u9; uint32_t ti10 = _t[10U]; @@ -234,14 +226,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb10 + ((va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) - << (uint32_t)17U - | (va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) >> (uint32_t)15U); + << 17U + | (va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) >> 15U); abcd[2U] = v9; uint32_t va10 = abcd[1U]; uint32_t vb11 = abcd[2U]; uint32_t vc11 = abcd[3U]; uint32_t vd11 = abcd[0U]; - uint8_t *b11 = x + (uint32_t)44U; + uint8_t *b11 = x + 44U; uint32_t u10 = load32_le(b11); uint32_t xk10 = u10; uint32_t ti11 = _t[11U]; @@ -250,14 +242,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb11 + ((va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) - << (uint32_t)22U - | (va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) >> (uint32_t)10U); + << 22U + | (va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) >> 10U); abcd[1U] = v10; uint32_t va11 = abcd[0U]; uint32_t vb12 = abcd[1U]; uint32_t vc12 = abcd[2U]; uint32_t vd12 = abcd[3U]; - uint8_t *b12 = x + (uint32_t)48U; + uint8_t *b12 = x + 48U; uint32_t u11 = load32_le(b12); uint32_t xk11 = u11; uint32_t ti12 = _t[12U]; @@ -266,14 +258,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb12 + ((va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) - << (uint32_t)7U - | (va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) >> (uint32_t)25U); + << 7U + | (va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) >> 25U); abcd[0U] = v11; uint32_t va12 = abcd[3U]; uint32_t vb13 = abcd[0U]; uint32_t vc13 = abcd[1U]; uint32_t vd13 = abcd[2U]; - uint8_t *b13 = x + (uint32_t)52U; + uint8_t *b13 = x + 52U; uint32_t u12 = load32_le(b13); uint32_t xk12 = u12; uint32_t ti13 = _t[13U]; @@ -282,14 +274,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb13 + ((va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) - << (uint32_t)12U - | (va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) >> (uint32_t)20U); + << 12U + | (va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) >> 20U); abcd[3U] = v12; uint32_t va13 = abcd[2U]; uint32_t vb14 = abcd[3U]; uint32_t vc14 = abcd[0U]; uint32_t vd14 = abcd[1U]; - uint8_t *b14 = x + (uint32_t)56U; + uint8_t *b14 = x + 56U; uint32_t u13 = load32_le(b14); uint32_t xk13 = u13; uint32_t ti14 = _t[14U]; @@ -298,14 +290,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb14 + ((va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) - << (uint32_t)17U - | (va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) >> (uint32_t)15U); + << 17U + | (va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) >> 15U); abcd[2U] = v13; uint32_t va14 = abcd[1U]; uint32_t vb15 = abcd[2U]; uint32_t vc15 = abcd[3U]; uint32_t vd15 = abcd[0U]; - uint8_t *b15 = x + (uint32_t)60U; + uint8_t *b15 = x + 60U; uint32_t u14 = load32_le(b15); uint32_t xk14 = u14; uint32_t ti15 = _t[15U]; @@ -314,14 +306,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb15 + ((va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) - << (uint32_t)22U - | (va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) >> (uint32_t)10U); + << 22U + | (va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) >> 10U); abcd[1U] = v14; uint32_t va15 = abcd[0U]; uint32_t vb16 = abcd[1U]; uint32_t vc16 = abcd[2U]; uint32_t vd16 = abcd[3U]; - uint8_t *b16 = x + (uint32_t)4U; + uint8_t *b16 = x + 4U; uint32_t u15 = load32_le(b16); uint32_t xk15 = u15; uint32_t ti16 = _t[16U]; @@ -330,14 +322,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb16 + ((va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) - << (uint32_t)5U - | (va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) >> (uint32_t)27U); + << 5U + | (va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) >> 27U); abcd[0U] = v15; uint32_t va16 = abcd[3U]; uint32_t vb17 = abcd[0U]; uint32_t vc17 = abcd[1U]; uint32_t vd17 = abcd[2U]; - uint8_t *b17 = x + (uint32_t)24U; + uint8_t *b17 = x + 24U; uint32_t u16 = load32_le(b17); uint32_t xk16 = u16; uint32_t ti17 = _t[17U]; @@ -346,14 +338,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb17 + ((va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) - << (uint32_t)9U - | (va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) >> (uint32_t)23U); + << 9U + | (va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) >> 23U); abcd[3U] = v16; uint32_t va17 = abcd[2U]; uint32_t vb18 = abcd[3U]; uint32_t vc18 = abcd[0U]; uint32_t vd18 = abcd[1U]; - uint8_t *b18 = x + (uint32_t)44U; + uint8_t *b18 = x + 44U; uint32_t u17 = load32_le(b18); uint32_t xk17 = u17; uint32_t ti18 = _t[18U]; @@ -362,8 +354,8 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb18 + ((va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) - << (uint32_t)14U - | (va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) >> (uint32_t)18U); + << 14U + | (va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) >> 18U); abcd[2U] = v17; uint32_t va18 = abcd[1U]; uint32_t vb19 = abcd[2U]; @@ -378,14 +370,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb19 + ((va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) - << (uint32_t)20U - | (va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) >> (uint32_t)12U); + << 20U + | (va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) >> 12U); abcd[1U] = v18; uint32_t va19 = abcd[0U]; uint32_t vb20 = abcd[1U]; uint32_t vc20 = abcd[2U]; uint32_t vd20 = abcd[3U]; - uint8_t *b20 = x + (uint32_t)20U; + uint8_t *b20 = x + 20U; uint32_t u19 = load32_le(b20); uint32_t xk19 = u19; uint32_t ti20 = _t[20U]; @@ -394,14 +386,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb20 + ((va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) - << (uint32_t)5U - | (va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) >> (uint32_t)27U); + << 5U + | (va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) >> 27U); abcd[0U] = v19; uint32_t va20 = abcd[3U]; uint32_t vb21 = abcd[0U]; uint32_t vc21 = abcd[1U]; uint32_t vd21 = abcd[2U]; - uint8_t *b21 = x + (uint32_t)40U; + uint8_t *b21 = x + 40U; uint32_t u20 = load32_le(b21); uint32_t xk20 = u20; uint32_t ti21 = _t[21U]; @@ -410,14 +402,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb21 + ((va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) - << (uint32_t)9U - | (va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) >> (uint32_t)23U); + << 9U + | (va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) >> 23U); abcd[3U] = v20; uint32_t va21 = abcd[2U]; uint32_t vb22 = abcd[3U]; uint32_t vc22 = abcd[0U]; uint32_t vd22 = abcd[1U]; - uint8_t *b22 = x + (uint32_t)60U; + uint8_t *b22 = x + 60U; uint32_t u21 = load32_le(b22); uint32_t xk21 = u21; uint32_t ti22 = _t[22U]; @@ -426,14 +418,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb22 + ((va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) - << (uint32_t)14U - | (va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) >> (uint32_t)18U); + << 14U + | (va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) >> 18U); abcd[2U] = v21; uint32_t va22 = abcd[1U]; uint32_t vb23 = abcd[2U]; uint32_t vc23 = abcd[3U]; uint32_t vd23 = abcd[0U]; - uint8_t *b23 = x + (uint32_t)16U; + uint8_t *b23 = x + 16U; uint32_t u22 = load32_le(b23); uint32_t xk22 = u22; uint32_t ti23 = _t[23U]; @@ -442,14 +434,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb23 + ((va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) - << (uint32_t)20U - | (va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) >> (uint32_t)12U); + << 20U + | (va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) >> 12U); abcd[1U] = v22; uint32_t va23 = abcd[0U]; uint32_t vb24 = abcd[1U]; uint32_t vc24 = abcd[2U]; uint32_t vd24 = abcd[3U]; - uint8_t *b24 = x + (uint32_t)36U; + uint8_t *b24 = x + 36U; uint32_t u23 = load32_le(b24); uint32_t xk23 = u23; uint32_t ti24 = _t[24U]; @@ -458,14 +450,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb24 + ((va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) - << (uint32_t)5U - | (va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) >> (uint32_t)27U); + << 5U + | (va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) >> 27U); abcd[0U] = v23; uint32_t va24 = abcd[3U]; uint32_t vb25 = abcd[0U]; uint32_t vc25 = abcd[1U]; uint32_t vd25 = abcd[2U]; - uint8_t *b25 = x + (uint32_t)56U; + uint8_t *b25 = x + 56U; uint32_t u24 = load32_le(b25); uint32_t xk24 = u24; uint32_t ti25 = _t[25U]; @@ -474,14 +466,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb25 + ((va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) - << (uint32_t)9U - | (va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) >> (uint32_t)23U); + << 9U + | (va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) >> 23U); abcd[3U] = v24; uint32_t va25 = abcd[2U]; uint32_t vb26 = abcd[3U]; uint32_t vc26 = abcd[0U]; uint32_t vd26 = abcd[1U]; - uint8_t *b26 = x + (uint32_t)12U; + uint8_t *b26 = x + 12U; uint32_t u25 = load32_le(b26); uint32_t xk25 = u25; uint32_t ti26 = _t[26U]; @@ -490,14 +482,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb26 + ((va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) - << (uint32_t)14U - | (va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) >> (uint32_t)18U); + << 14U + | (va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) >> 18U); abcd[2U] = v25; uint32_t va26 = abcd[1U]; uint32_t vb27 = abcd[2U]; uint32_t vc27 = abcd[3U]; uint32_t vd27 = abcd[0U]; - uint8_t *b27 = x + (uint32_t)32U; + uint8_t *b27 = x + 32U; uint32_t u26 = load32_le(b27); uint32_t xk26 = u26; uint32_t ti27 = _t[27U]; @@ -506,14 +498,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb27 + ((va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) - << (uint32_t)20U - | (va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) >> (uint32_t)12U); + << 20U + | (va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) >> 12U); abcd[1U] = v26; uint32_t va27 = abcd[0U]; uint32_t vb28 = abcd[1U]; uint32_t vc28 = abcd[2U]; uint32_t vd28 = abcd[3U]; - uint8_t *b28 = x + (uint32_t)52U; + uint8_t *b28 = x + 52U; uint32_t u27 = load32_le(b28); uint32_t xk27 = u27; uint32_t ti28 = _t[28U]; @@ -522,14 +514,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb28 + ((va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) - << (uint32_t)5U - | (va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) >> (uint32_t)27U); + << 5U + | (va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) >> 27U); abcd[0U] = v27; uint32_t va28 = abcd[3U]; uint32_t vb29 = abcd[0U]; uint32_t vc29 = abcd[1U]; uint32_t vd29 = abcd[2U]; - uint8_t *b29 = x + (uint32_t)8U; + uint8_t *b29 = x + 8U; uint32_t u28 = load32_le(b29); uint32_t xk28 = u28; uint32_t ti29 = _t[29U]; @@ -538,14 +530,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb29 + ((va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) - << (uint32_t)9U - | (va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) >> (uint32_t)23U); + << 9U + | (va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) >> 23U); abcd[3U] = v28; uint32_t va29 = abcd[2U]; uint32_t vb30 = abcd[3U]; uint32_t vc30 = abcd[0U]; uint32_t vd30 = abcd[1U]; - uint8_t *b30 = x + (uint32_t)28U; + uint8_t *b30 = x + 28U; uint32_t u29 = load32_le(b30); uint32_t xk29 = u29; uint32_t ti30 = _t[30U]; @@ -554,14 +546,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb30 + ((va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) - << (uint32_t)14U - | (va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) >> (uint32_t)18U); + << 14U + | (va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) >> 18U); abcd[2U] = v29; uint32_t va30 = abcd[1U]; uint32_t vb31 = abcd[2U]; uint32_t vc31 = abcd[3U]; uint32_t vd31 = abcd[0U]; - uint8_t *b31 = x + (uint32_t)48U; + uint8_t *b31 = x + 48U; uint32_t u30 = load32_le(b31); uint32_t xk30 = u30; uint32_t ti31 = _t[31U]; @@ -570,14 +562,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb31 + ((va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) - << (uint32_t)20U - | (va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) >> (uint32_t)12U); + << 20U + | (va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) >> 12U); abcd[1U] = v30; uint32_t va31 = abcd[0U]; uint32_t vb32 = abcd[1U]; uint32_t vc32 = abcd[2U]; uint32_t vd32 = abcd[3U]; - uint8_t *b32 = x + (uint32_t)20U; + uint8_t *b32 = x + 20U; uint32_t u31 = load32_le(b32); uint32_t xk31 = u31; uint32_t ti32 = _t[32U]; @@ -586,14 +578,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb32 + ((va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) - << (uint32_t)4U - | (va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) >> (uint32_t)28U); + << 4U + | (va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) >> 28U); abcd[0U] = v31; uint32_t va32 = abcd[3U]; uint32_t vb33 = abcd[0U]; uint32_t vc33 = abcd[1U]; uint32_t vd33 = abcd[2U]; - uint8_t *b33 = x + (uint32_t)32U; + uint8_t *b33 = x + 32U; uint32_t u32 = load32_le(b33); uint32_t xk32 = u32; uint32_t ti33 = _t[33U]; @@ -602,14 +594,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb33 + ((va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) - << (uint32_t)11U - | (va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) >> (uint32_t)21U); + << 11U + | (va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) >> 21U); abcd[3U] = v32; uint32_t va33 = abcd[2U]; uint32_t vb34 = abcd[3U]; uint32_t vc34 = abcd[0U]; uint32_t vd34 = abcd[1U]; - uint8_t *b34 = x + (uint32_t)44U; + uint8_t *b34 = x + 44U; uint32_t u33 = load32_le(b34); uint32_t xk33 = u33; uint32_t ti34 = _t[34U]; @@ -618,14 +610,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb34 + ((va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) - << (uint32_t)16U - | (va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) >> (uint32_t)16U); + << 16U + | (va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) >> 16U); abcd[2U] = v33; uint32_t va34 = abcd[1U]; uint32_t vb35 = abcd[2U]; uint32_t vc35 = abcd[3U]; uint32_t vd35 = abcd[0U]; - uint8_t *b35 = x + (uint32_t)56U; + uint8_t *b35 = x + 56U; uint32_t u34 = load32_le(b35); uint32_t xk34 = u34; uint32_t ti35 = _t[35U]; @@ -634,14 +626,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb35 + ((va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) - << (uint32_t)23U - | (va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) >> (uint32_t)9U); + << 23U + | (va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) >> 9U); abcd[1U] = v34; uint32_t va35 = abcd[0U]; uint32_t vb36 = abcd[1U]; uint32_t vc36 = abcd[2U]; uint32_t vd36 = abcd[3U]; - uint8_t *b36 = x + (uint32_t)4U; + uint8_t *b36 = x + 4U; uint32_t u35 = load32_le(b36); uint32_t xk35 = u35; uint32_t ti36 = _t[36U]; @@ -650,14 +642,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb36 + ((va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) - << (uint32_t)4U - | (va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) >> (uint32_t)28U); + << 4U + | (va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) >> 28U); abcd[0U] = v35; uint32_t va36 = abcd[3U]; uint32_t vb37 = abcd[0U]; uint32_t vc37 = abcd[1U]; uint32_t vd37 = abcd[2U]; - uint8_t *b37 = x + (uint32_t)16U; + uint8_t *b37 = x + 16U; uint32_t u36 = load32_le(b37); uint32_t xk36 = u36; uint32_t ti37 = _t[37U]; @@ -666,14 +658,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb37 + ((va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) - << (uint32_t)11U - | (va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) >> (uint32_t)21U); + << 11U + | (va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) >> 21U); abcd[3U] = v36; uint32_t va37 = abcd[2U]; uint32_t vb38 = abcd[3U]; uint32_t vc38 = abcd[0U]; uint32_t vd38 = abcd[1U]; - uint8_t *b38 = x + (uint32_t)28U; + uint8_t *b38 = x + 28U; uint32_t u37 = load32_le(b38); uint32_t xk37 = u37; uint32_t ti38 = _t[38U]; @@ -682,14 +674,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb38 + ((va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) - << (uint32_t)16U - | (va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) >> (uint32_t)16U); + << 16U + | (va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) >> 16U); abcd[2U] = v37; uint32_t va38 = abcd[1U]; uint32_t vb39 = abcd[2U]; uint32_t vc39 = abcd[3U]; uint32_t vd39 = abcd[0U]; - uint8_t *b39 = x + (uint32_t)40U; + uint8_t *b39 = x + 40U; uint32_t u38 = load32_le(b39); uint32_t xk38 = u38; uint32_t ti39 = _t[39U]; @@ -698,14 +690,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb39 + ((va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) - << (uint32_t)23U - | (va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) >> (uint32_t)9U); + << 23U + | (va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) >> 9U); abcd[1U] = v38; uint32_t va39 = abcd[0U]; uint32_t vb40 = abcd[1U]; uint32_t vc40 = abcd[2U]; uint32_t vd40 = abcd[3U]; - uint8_t *b40 = x + (uint32_t)52U; + uint8_t *b40 = x + 52U; uint32_t u39 = load32_le(b40); uint32_t xk39 = u39; uint32_t ti40 = _t[40U]; @@ -714,8 +706,8 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb40 + ((va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) - << (uint32_t)4U - | (va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) >> (uint32_t)28U); + << 4U + | (va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) >> 28U); abcd[0U] = v39; uint32_t va40 = abcd[3U]; uint32_t vb41 = abcd[0U]; @@ -730,14 +722,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb41 + ((va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) - << (uint32_t)11U - | (va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) >> (uint32_t)21U); + << 11U + | (va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) >> 21U); abcd[3U] = v40; uint32_t va41 = abcd[2U]; uint32_t vb42 = abcd[3U]; uint32_t vc42 = abcd[0U]; uint32_t vd42 = abcd[1U]; - uint8_t *b42 = x + (uint32_t)12U; + uint8_t *b42 = x + 12U; uint32_t u41 = load32_le(b42); uint32_t xk41 = u41; uint32_t ti42 = _t[42U]; @@ -746,14 +738,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb42 + ((va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) - << (uint32_t)16U - | (va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) >> (uint32_t)16U); + << 16U + | (va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) >> 16U); abcd[2U] = v41; uint32_t va42 = abcd[1U]; uint32_t vb43 = abcd[2U]; uint32_t vc43 = abcd[3U]; uint32_t vd43 = abcd[0U]; - uint8_t *b43 = x + (uint32_t)24U; + uint8_t *b43 = x + 24U; uint32_t u42 = load32_le(b43); uint32_t xk42 = u42; uint32_t ti43 = _t[43U]; @@ -762,14 +754,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb43 + ((va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) - << (uint32_t)23U - | (va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) >> (uint32_t)9U); + << 23U + | (va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) >> 9U); abcd[1U] = v42; uint32_t va43 = abcd[0U]; uint32_t vb44 = abcd[1U]; uint32_t vc44 = abcd[2U]; uint32_t vd44 = abcd[3U]; - uint8_t *b44 = x + (uint32_t)36U; + uint8_t *b44 = x + 36U; uint32_t u43 = load32_le(b44); uint32_t xk43 = u43; uint32_t ti44 = _t[44U]; @@ -778,14 +770,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb44 + ((va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) - << (uint32_t)4U - | (va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) >> (uint32_t)28U); + << 4U + | (va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) >> 28U); abcd[0U] = v43; uint32_t va44 = abcd[3U]; uint32_t vb45 = abcd[0U]; uint32_t vc45 = abcd[1U]; uint32_t vd45 = abcd[2U]; - uint8_t *b45 = x + (uint32_t)48U; + uint8_t *b45 = x + 48U; uint32_t u44 = load32_le(b45); uint32_t xk44 = u44; uint32_t ti45 = _t[45U]; @@ -794,14 +786,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb45 + ((va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) - << (uint32_t)11U - | (va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) >> (uint32_t)21U); + << 11U + | (va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) >> 21U); abcd[3U] = v44; uint32_t va45 = abcd[2U]; uint32_t vb46 = abcd[3U]; uint32_t vc46 = abcd[0U]; uint32_t vd46 = abcd[1U]; - uint8_t *b46 = x + (uint32_t)60U; + uint8_t *b46 = x + 60U; uint32_t u45 = load32_le(b46); uint32_t xk45 = u45; uint32_t ti46 = _t[46U]; @@ -810,14 +802,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb46 + ((va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) - << (uint32_t)16U - | (va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) >> (uint32_t)16U); + << 16U + | (va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) >> 16U); abcd[2U] = v45; uint32_t va46 = abcd[1U]; uint32_t vb47 = abcd[2U]; uint32_t vc47 = abcd[3U]; uint32_t vd47 = abcd[0U]; - uint8_t *b47 = x + (uint32_t)8U; + uint8_t *b47 = x + 8U; uint32_t u46 = load32_le(b47); uint32_t xk46 = u46; uint32_t ti47 = _t[47U]; @@ -826,8 +818,8 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb47 + ((va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) - << (uint32_t)23U - | (va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) >> (uint32_t)9U); + << 23U + | (va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) >> 9U); abcd[1U] = v46; uint32_t va47 = abcd[0U]; uint32_t vb48 = abcd[1U]; @@ -842,14 +834,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb48 + ((va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) - << (uint32_t)6U - | (va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) >> (uint32_t)26U); + << 6U + | (va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) >> 26U); abcd[0U] = v47; uint32_t va48 = abcd[3U]; uint32_t vb49 = abcd[0U]; uint32_t vc49 = abcd[1U]; uint32_t vd49 = abcd[2U]; - uint8_t *b49 = x + (uint32_t)28U; + uint8_t *b49 = x + 28U; uint32_t u48 = load32_le(b49); uint32_t xk48 = u48; uint32_t ti49 = _t[49U]; @@ -858,14 +850,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb49 + ((va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) - << (uint32_t)10U - | (va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) >> (uint32_t)22U); + << 10U + | (va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) >> 22U); abcd[3U] = v48; uint32_t va49 = abcd[2U]; uint32_t vb50 = abcd[3U]; uint32_t vc50 = abcd[0U]; uint32_t vd50 = abcd[1U]; - uint8_t *b50 = x + (uint32_t)56U; + uint8_t *b50 = x + 56U; uint32_t u49 = load32_le(b50); uint32_t xk49 = u49; uint32_t ti50 = _t[50U]; @@ -874,14 +866,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb50 + ((va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) - << (uint32_t)15U - | (va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) >> (uint32_t)17U); + << 15U + | (va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) >> 17U); abcd[2U] = v49; uint32_t va50 = abcd[1U]; uint32_t vb51 = abcd[2U]; uint32_t vc51 = abcd[3U]; uint32_t vd51 = abcd[0U]; - uint8_t *b51 = x + (uint32_t)20U; + uint8_t *b51 = x + 20U; uint32_t u50 = load32_le(b51); uint32_t xk50 = u50; uint32_t ti51 = _t[51U]; @@ -890,14 +882,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb51 + ((va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) - << (uint32_t)21U - | (va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) >> (uint32_t)11U); + << 21U + | (va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) >> 11U); abcd[1U] = v50; uint32_t va51 = abcd[0U]; uint32_t vb52 = abcd[1U]; uint32_t vc52 = abcd[2U]; uint32_t vd52 = abcd[3U]; - uint8_t *b52 = x + (uint32_t)48U; + uint8_t *b52 = x + 48U; uint32_t u51 = load32_le(b52); uint32_t xk51 = u51; uint32_t ti52 = _t[52U]; @@ -906,14 +898,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb52 + ((va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) - << (uint32_t)6U - | (va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) >> (uint32_t)26U); + << 6U + | (va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) >> 26U); abcd[0U] = v51; uint32_t va52 = abcd[3U]; uint32_t vb53 = abcd[0U]; uint32_t vc53 = abcd[1U]; uint32_t vd53 = abcd[2U]; - uint8_t *b53 = x + (uint32_t)12U; + uint8_t *b53 = x + 12U; uint32_t u52 = load32_le(b53); uint32_t xk52 = u52; uint32_t ti53 = _t[53U]; @@ -922,14 +914,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb53 + ((va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) - << (uint32_t)10U - | (va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) >> (uint32_t)22U); + << 10U + | (va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) >> 22U); abcd[3U] = v52; uint32_t va53 = abcd[2U]; uint32_t vb54 = abcd[3U]; uint32_t vc54 = abcd[0U]; uint32_t vd54 = abcd[1U]; - uint8_t *b54 = x + (uint32_t)40U; + uint8_t *b54 = x + 40U; uint32_t u53 = load32_le(b54); uint32_t xk53 = u53; uint32_t ti54 = _t[54U]; @@ -938,14 +930,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb54 + ((va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) - << (uint32_t)15U - | (va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) >> (uint32_t)17U); + << 15U + | (va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) >> 17U); abcd[2U] = v53; uint32_t va54 = abcd[1U]; uint32_t vb55 = abcd[2U]; uint32_t vc55 = abcd[3U]; uint32_t vd55 = abcd[0U]; - uint8_t *b55 = x + (uint32_t)4U; + uint8_t *b55 = x + 4U; uint32_t u54 = load32_le(b55); uint32_t xk54 = u54; uint32_t ti55 = _t[55U]; @@ -954,14 +946,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb55 + ((va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) - << (uint32_t)21U - | (va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) >> (uint32_t)11U); + << 21U + | (va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) >> 11U); abcd[1U] = v54; uint32_t va55 = abcd[0U]; uint32_t vb56 = abcd[1U]; uint32_t vc56 = abcd[2U]; uint32_t vd56 = abcd[3U]; - uint8_t *b56 = x + (uint32_t)32U; + uint8_t *b56 = x + 32U; uint32_t u55 = load32_le(b56); uint32_t xk55 = u55; uint32_t ti56 = _t[56U]; @@ -970,14 +962,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb56 + ((va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) - << (uint32_t)6U - | (va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) >> (uint32_t)26U); + << 6U + | (va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) >> 26U); abcd[0U] = v55; uint32_t va56 = abcd[3U]; uint32_t vb57 = abcd[0U]; uint32_t vc57 = abcd[1U]; uint32_t vd57 = abcd[2U]; - uint8_t *b57 = x + (uint32_t)60U; + uint8_t *b57 = x + 60U; uint32_t u56 = load32_le(b57); uint32_t xk56 = u56; uint32_t ti57 = _t[57U]; @@ -986,14 +978,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb57 + ((va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) - << (uint32_t)10U - | (va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) >> (uint32_t)22U); + << 10U + | (va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) >> 22U); abcd[3U] = v56; uint32_t va57 = abcd[2U]; uint32_t vb58 = abcd[3U]; uint32_t vc58 = abcd[0U]; uint32_t vd58 = abcd[1U]; - uint8_t *b58 = x + (uint32_t)24U; + uint8_t *b58 = x + 24U; uint32_t u57 = load32_le(b58); uint32_t xk57 = u57; uint32_t ti58 = _t[58U]; @@ -1002,14 +994,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb58 + ((va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) - << (uint32_t)15U - | (va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) >> (uint32_t)17U); + << 15U + | (va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) >> 17U); abcd[2U] = v57; uint32_t va58 = abcd[1U]; uint32_t vb59 = abcd[2U]; uint32_t vc59 = abcd[3U]; uint32_t vd59 = abcd[0U]; - uint8_t *b59 = x + (uint32_t)52U; + uint8_t *b59 = x + 52U; uint32_t u58 = load32_le(b59); uint32_t xk58 = u58; uint32_t ti59 = _t[59U]; @@ -1018,14 +1010,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb59 + ((va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) - << (uint32_t)21U - | (va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) >> (uint32_t)11U); + << 21U + | (va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) >> 11U); abcd[1U] = v58; uint32_t va59 = abcd[0U]; uint32_t vb60 = abcd[1U]; uint32_t vc60 = abcd[2U]; uint32_t vd60 = abcd[3U]; - uint8_t *b60 = x + (uint32_t)16U; + uint8_t *b60 = x + 16U; uint32_t u59 = load32_le(b60); uint32_t xk59 = u59; uint32_t ti60 = _t[60U]; @@ -1034,14 +1026,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb60 + ((va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) - << (uint32_t)6U - | (va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) >> (uint32_t)26U); + << 6U + | (va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) >> 26U); abcd[0U] = v59; uint32_t va60 = abcd[3U]; uint32_t vb61 = abcd[0U]; uint32_t vc61 = abcd[1U]; uint32_t vd61 = abcd[2U]; - uint8_t *b61 = x + (uint32_t)44U; + uint8_t *b61 = x + 44U; uint32_t u60 = load32_le(b61); uint32_t xk60 = u60; uint32_t ti61 = _t[61U]; @@ -1050,14 +1042,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb61 + ((va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) - << (uint32_t)10U - | (va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) >> (uint32_t)22U); + << 10U + | (va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) >> 22U); abcd[3U] = v60; uint32_t va61 = abcd[2U]; uint32_t vb62 = abcd[3U]; uint32_t vc62 = abcd[0U]; uint32_t vd62 = abcd[1U]; - uint8_t *b62 = x + (uint32_t)8U; + uint8_t *b62 = x + 8U; uint32_t u61 = load32_le(b62); uint32_t xk61 = u61; uint32_t ti62 = _t[62U]; @@ -1066,14 +1058,14 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb62 + ((va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) - << (uint32_t)15U - | (va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) >> (uint32_t)17U); + << 15U + | (va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) >> 17U); abcd[2U] = v61; uint32_t va62 = abcd[1U]; uint32_t vb = abcd[2U]; uint32_t vc = abcd[3U]; uint32_t vd = abcd[0U]; - uint8_t *b63 = x + (uint32_t)36U; + uint8_t *b63 = x + 36U; uint32_t u62 = load32_le(b63); uint32_t xk62 = u62; uint32_t ti = _t[63U]; @@ -1082,8 +1074,8 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) vb + ((va62 + (vc ^ (vb | ~vd)) + xk62 + ti) - << (uint32_t)21U - | (va62 + (vc ^ (vb | ~vd)) + xk62 + ti) >> (uint32_t)11U); + << 21U + | (va62 + (vc ^ (vb | ~vd)) + xk62 + ti) >> 11U); abcd[1U] = v62; uint32_t a = abcd[0U]; uint32_t b = abcd[1U]; @@ -1095,98 +1087,69 @@ static void legacy_update(uint32_t *abcd, uint8_t *x) abcd[3U] = d + dd; } -static void legacy_pad(uint64_t len, uint8_t *dst) +static void pad(uint64_t len, uint8_t *dst) { uint8_t *dst1 = dst; - dst1[0U] = (uint8_t)0x80U; - uint8_t *dst2 = dst + (uint32_t)1U; - for - (uint32_t - i = (uint32_t)0U; - i - < ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(len % (uint64_t)(uint32_t)64U))) % (uint32_t)64U; - i++) + dst1[0U] = 0x80U; + uint8_t *dst2 = dst + 1U; + for (uint32_t i = 0U; i < (128U - (9U + (uint32_t)(len % (uint64_t)64U))) % 64U; i++) { - dst2[i] = (uint8_t)0U; + dst2[i] = 0U; } - uint8_t - *dst3 = - dst - + - (uint32_t)1U - + - ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(len % (uint64_t)(uint32_t)64U))) - % (uint32_t)64U; - store64_le(dst3, len << (uint32_t)3U); + uint8_t *dst3 = dst + 1U + (128U - (9U + (uint32_t)(len % (uint64_t)64U))) % 64U; + store64_le(dst3, len << 3U); } -void Hacl_Hash_Core_MD5_legacy_finish(uint32_t *s, uint8_t *dst) +void Hacl_Hash_MD5_finish(uint32_t *s, uint8_t *dst) { - KRML_MAYBE_FOR4(i, - (uint32_t)0U, - (uint32_t)4U, - (uint32_t)1U, - store32_le(dst + i * (uint32_t)4U, s[i]);); + KRML_MAYBE_FOR4(i, 0U, 4U, 1U, store32_le(dst + i * 4U, s[i]);); } -void Hacl_Hash_MD5_legacy_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks) +void Hacl_Hash_MD5_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks) { - for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + for (uint32_t i = 0U; i < n_blocks; i++) { - uint32_t sz = (uint32_t)64U; + uint32_t sz = 64U; uint8_t *block = blocks + sz * i; - legacy_update(s, block); + update(s, block); } } void -Hacl_Hash_MD5_legacy_update_last( - uint32_t *s, - uint64_t prev_len, - uint8_t *input, - uint32_t input_len -) +Hacl_Hash_MD5_update_last(uint32_t *s, uint64_t prev_len, uint8_t *input, uint32_t input_len) { - uint32_t blocks_n = input_len / (uint32_t)64U; - uint32_t blocks_len = blocks_n * (uint32_t)64U; + uint32_t blocks_n = input_len / 64U; + uint32_t blocks_len = blocks_n * 64U; uint8_t *blocks = input; uint32_t rest_len = input_len - blocks_len; uint8_t *rest = input + blocks_len; - Hacl_Hash_MD5_legacy_update_multi(s, blocks, blocks_n); + Hacl_Hash_MD5_update_multi(s, blocks, blocks_n); uint64_t total_input_len = prev_len + (uint64_t)input_len; - uint32_t - pad_len = - (uint32_t)1U - + - ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(total_input_len % (uint64_t)(uint32_t)64U))) - % (uint32_t)64U - + (uint32_t)8U; + uint32_t pad_len = 1U + (128U - (9U + (uint32_t)(total_input_len % (uint64_t)64U))) % 64U + 8U; uint32_t tmp_len = rest_len + pad_len; uint8_t tmp_twoblocks[128U] = { 0U }; uint8_t *tmp = tmp_twoblocks; uint8_t *tmp_rest = tmp; uint8_t *tmp_pad = tmp + rest_len; memcpy(tmp_rest, rest, rest_len * sizeof (uint8_t)); - legacy_pad(total_input_len, tmp_pad); - Hacl_Hash_MD5_legacy_update_multi(s, tmp, tmp_len / (uint32_t)64U); + pad(total_input_len, tmp_pad); + Hacl_Hash_MD5_update_multi(s, tmp, tmp_len / 64U); } -void Hacl_Hash_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_MD5_hash_oneshot(uint8_t *output, uint8_t *input, uint32_t input_len) { - uint32_t - s[4U] = - { (uint32_t)0x67452301U, (uint32_t)0xefcdab89U, (uint32_t)0x98badcfeU, (uint32_t)0x10325476U }; - uint32_t blocks_n0 = input_len / (uint32_t)64U; + uint32_t s[4U] = { 0x67452301U, 0xefcdab89U, 0x98badcfeU, 0x10325476U }; + uint32_t blocks_n0 = input_len / 64U; uint32_t blocks_n1; - if (input_len % (uint32_t)64U == (uint32_t)0U && blocks_n0 > (uint32_t)0U) + if (input_len % 64U == 0U && blocks_n0 > 0U) { - blocks_n1 = blocks_n0 - (uint32_t)1U; + blocks_n1 = blocks_n0 - 1U; } else { blocks_n1 = blocks_n0; } - uint32_t blocks_len0 = blocks_n1 * (uint32_t)64U; + uint32_t blocks_len0 = blocks_n1 * 64U; uint8_t *blocks0 = input; uint32_t rest_len0 = input_len - blocks_len0; uint8_t *rest0 = input + blocks_len0; @@ -1195,75 +1158,75 @@ void Hacl_Hash_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) uint8_t *blocks = blocks0; uint32_t rest_len = rest_len0; uint8_t *rest = rest0; - Hacl_Hash_MD5_legacy_update_multi(s, blocks, blocks_n); - Hacl_Hash_MD5_legacy_update_last(s, (uint64_t)blocks_len, rest, rest_len); - Hacl_Hash_Core_MD5_legacy_finish(s, dst); + Hacl_Hash_MD5_update_multi(s, blocks, blocks_n); + Hacl_Hash_MD5_update_last(s, (uint64_t)blocks_len, rest, rest_len); + Hacl_Hash_MD5_finish(s, output); } -Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_create_in(void) +Hacl_Streaming_MD_state_32 *Hacl_Hash_MD5_malloc(void) { - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); - uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)4U, sizeof (uint32_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC(4U, sizeof (uint32_t)); Hacl_Streaming_MD_state_32 - s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; Hacl_Streaming_MD_state_32 *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); p[0U] = s; - Hacl_Hash_Core_MD5_legacy_init(block_state); + Hacl_Hash_MD5_init(block_state); return p; } -void Hacl_Streaming_MD5_legacy_init(Hacl_Streaming_MD_state_32 *s) +void Hacl_Hash_MD5_reset(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *state; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; - Hacl_Hash_Core_MD5_legacy_init(block_state); + Hacl_Hash_MD5_init(block_state); Hacl_Streaming_MD_state_32 - tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - s[0U] = tmp; + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; + state[0U] = tmp; } /** 0 = success, 1 = max length exceeded */ Hacl_Streaming_Types_error_code -Hacl_Streaming_MD5_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) +Hacl_Hash_MD5_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk_len) { - Hacl_Streaming_MD_state_32 s = *p; + Hacl_Streaming_MD_state_32 s = *state; uint64_t total_len = s.total_len; - if ((uint64_t)len > (uint64_t)2305843009213693951U - total_len) + if ((uint64_t)chunk_len > 2305843009213693951ULL - total_len) { return Hacl_Streaming_Types_MaximumLengthExceeded; } uint32_t sz; - if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) { - sz = (uint32_t)64U; + sz = 64U; } else { - sz = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + sz = (uint32_t)(total_len % (uint64_t)64U); } - if (len <= (uint32_t)64U - sz) + if (chunk_len <= 64U - sz) { - Hacl_Streaming_MD_state_32 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } uint8_t *buf2 = buf + sz1; - memcpy(buf2, data, len * sizeof (uint8_t)); - uint64_t total_len2 = total_len1 + (uint64_t)len; - *p + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state = ( (Hacl_Streaming_MD_state_32){ @@ -1273,74 +1236,74 @@ Hacl_Streaming_MD5_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, u } ); } - else if (sz == (uint32_t)0U) + else if (sz == 0U) { - Hacl_Streaming_MD_state_32 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_Hash_MD5_legacy_update_multi(block_state1, buf, (uint32_t)1U); + Hacl_Hash_MD5_update_multi(block_state1, buf, 1U); } uint32_t ite; - if ((uint64_t)len % (uint64_t)(uint32_t)64U == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + if ((uint64_t)chunk_len % (uint64_t)64U == 0ULL && (uint64_t)chunk_len > 0ULL) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = (uint32_t)((uint64_t)len % (uint64_t)(uint32_t)64U); + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)64U); } - uint32_t n_blocks = (len - ite) / (uint32_t)64U; - uint32_t data1_len = n_blocks * (uint32_t)64U; - uint32_t data2_len = len - data1_len; - uint8_t *data1 = data; - uint8_t *data2 = data + data1_len; - Hacl_Hash_MD5_legacy_update_multi(block_state1, data1, data1_len / (uint32_t)64U); + uint32_t n_blocks = (chunk_len - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; + Hacl_Hash_MD5_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *p + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)len + .total_len = total_len1 + (uint64_t)chunk_len } ); } else { - uint32_t diff = (uint32_t)64U - sz; - uint8_t *data1 = data; - uint8_t *data2 = data + diff; - Hacl_Streaming_MD_state_32 s1 = *p; + uint32_t diff = 64U - sz; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state10 = s1.block_state; uint8_t *buf0 = s1.buf; uint64_t total_len10 = s1.total_len; uint32_t sz10; - if (total_len10 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len10 > (uint64_t)0U) + if (total_len10 % (uint64_t)64U == 0ULL && total_len10 > 0ULL) { - sz10 = (uint32_t)64U; + sz10 = 64U; } else { - sz10 = (uint32_t)(total_len10 % (uint64_t)(uint32_t)64U); + sz10 = (uint32_t)(total_len10 % (uint64_t)64U); } uint8_t *buf2 = buf0 + sz10; - memcpy(buf2, data1, diff * sizeof (uint8_t)); + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *p + *state = ( (Hacl_Streaming_MD_state_32){ @@ -1349,114 +1312,109 @@ Hacl_Streaming_MD5_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, u .total_len = total_len2 } ); - Hacl_Streaming_MD_state_32 s10 = *p; + Hacl_Streaming_MD_state_32 s10 = *state; uint32_t *block_state1 = s10.block_state; uint8_t *buf = s10.buf; uint64_t total_len1 = s10.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_Hash_MD5_legacy_update_multi(block_state1, buf, (uint32_t)1U); + Hacl_Hash_MD5_update_multi(block_state1, buf, 1U); } uint32_t ite; if - ( - (uint64_t)(len - diff) - % (uint64_t)(uint32_t)64U - == (uint64_t)0U - && (uint64_t)(len - diff) > (uint64_t)0U - ) + ((uint64_t)(chunk_len - diff) % (uint64_t)64U == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)(uint32_t)64U); + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)64U); } - uint32_t n_blocks = (len - diff - ite) / (uint32_t)64U; - uint32_t data1_len = n_blocks * (uint32_t)64U; - uint32_t data2_len = len - diff - data1_len; - uint8_t *data11 = data2; - uint8_t *data21 = data2 + data1_len; - Hacl_Hash_MD5_legacy_update_multi(block_state1, data11, data1_len / (uint32_t)64U); + uint32_t n_blocks = (chunk_len - diff - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; + Hacl_Hash_MD5_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; - memcpy(dst, data21, data2_len * sizeof (uint8_t)); - *p + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)(len - diff) + .total_len = total_len1 + (uint64_t)(chunk_len - diff) } ); } return Hacl_Streaming_Types_Success; } -void Hacl_Streaming_MD5_legacy_finish(Hacl_Streaming_MD_state_32 *p, uint8_t *dst) +void Hacl_Hash_MD5_digest(Hacl_Streaming_MD_state_32 *state, uint8_t *output) { - Hacl_Streaming_MD_state_32 scrut = *p; + Hacl_Streaming_MD_state_32 scrut = *state; uint32_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; uint32_t r; - if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) { - r = (uint32_t)64U; + r = 64U; } else { - r = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + r = (uint32_t)(total_len % (uint64_t)64U); } uint8_t *buf_1 = buf_; uint32_t tmp_block_state[4U] = { 0U }; - memcpy(tmp_block_state, block_state, (uint32_t)4U * sizeof (uint32_t)); + memcpy(tmp_block_state, block_state, 4U * sizeof (uint32_t)); uint32_t ite; - if (r % (uint32_t)64U == (uint32_t)0U && r > (uint32_t)0U) + if (r % 64U == 0U && r > 0U) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = r % (uint32_t)64U; + ite = r % 64U; } uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; - Hacl_Hash_MD5_legacy_update_multi(tmp_block_state, buf_multi, (uint32_t)0U); + Hacl_Hash_MD5_update_multi(tmp_block_state, buf_multi, 0U); uint64_t prev_len_last = total_len - (uint64_t)r; - Hacl_Hash_MD5_legacy_update_last(tmp_block_state, prev_len_last, buf_last, r); - Hacl_Hash_Core_MD5_legacy_finish(tmp_block_state, dst); + Hacl_Hash_MD5_update_last(tmp_block_state, prev_len_last, buf_last, r); + Hacl_Hash_MD5_finish(tmp_block_state, output); } -void Hacl_Streaming_MD5_legacy_free(Hacl_Streaming_MD_state_32 *s) +void Hacl_Hash_MD5_free(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *state; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; KRML_HOST_FREE(block_state); KRML_HOST_FREE(buf); - KRML_HOST_FREE(s); + KRML_HOST_FREE(state); } -Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_copy(Hacl_Streaming_MD_state_32 *s0) +Hacl_Streaming_MD_state_32 *Hacl_Hash_MD5_copy(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s0; + Hacl_Streaming_MD_state_32 scrut = *state; uint32_t *block_state0 = scrut.block_state; uint8_t *buf0 = scrut.buf; uint64_t total_len0 = scrut.total_len; - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); - memcpy(buf, buf0, (uint32_t)64U * sizeof (uint8_t)); - uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)4U, sizeof (uint32_t)); - memcpy(block_state, block_state0, (uint32_t)4U * sizeof (uint32_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + memcpy(buf, buf0, 64U * sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC(4U, sizeof (uint32_t)); + memcpy(block_state, block_state0, 4U * sizeof (uint32_t)); Hacl_Streaming_MD_state_32 s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; Hacl_Streaming_MD_state_32 @@ -1465,8 +1423,8 @@ Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_copy(Hacl_Streaming_MD_sta return p; } -void Hacl_Streaming_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_MD5_hash(uint8_t *output, uint8_t *input, uint32_t input_len) { - Hacl_Hash_MD5_legacy_hash(input, input_len, dst); + Hacl_Hash_MD5_hash_oneshot(output, input, input_len); } diff --git a/Modules/_hacl/Hacl_Hash_MD5.h b/Modules/_hacl/Hacl_Hash_MD5.h index 13c19fd40f4d12..f69d6e5a81d63a 100644 --- a/Modules/_hacl/Hacl_Hash_MD5.h +++ b/Modules/_hacl/Hacl_Hash_MD5.h @@ -31,31 +31,32 @@ extern "C" { #endif #include +#include "python_hacl_namespaces.h" #include "krml/types.h" #include "krml/lowstar_endianness.h" #include "krml/internal/target.h" #include "Hacl_Streaming_Types.h" -typedef Hacl_Streaming_MD_state_32 Hacl_Streaming_MD5_state; +typedef Hacl_Streaming_MD_state_32 Hacl_Hash_MD5_state_t; -Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_create_in(void); +Hacl_Streaming_MD_state_32 *Hacl_Hash_MD5_malloc(void); -void Hacl_Streaming_MD5_legacy_init(Hacl_Streaming_MD_state_32 *s); +void Hacl_Hash_MD5_reset(Hacl_Streaming_MD_state_32 *state); /** 0 = success, 1 = max length exceeded */ Hacl_Streaming_Types_error_code -Hacl_Streaming_MD5_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len); +Hacl_Hash_MD5_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk_len); -void Hacl_Streaming_MD5_legacy_finish(Hacl_Streaming_MD_state_32 *p, uint8_t *dst); +void Hacl_Hash_MD5_digest(Hacl_Streaming_MD_state_32 *state, uint8_t *output); -void Hacl_Streaming_MD5_legacy_free(Hacl_Streaming_MD_state_32 *s); +void Hacl_Hash_MD5_free(Hacl_Streaming_MD_state_32 *state); -Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_copy(Hacl_Streaming_MD_state_32 *s0); +Hacl_Streaming_MD_state_32 *Hacl_Hash_MD5_copy(Hacl_Streaming_MD_state_32 *state); -void Hacl_Streaming_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst); +void Hacl_Hash_MD5_hash(uint8_t *output, uint8_t *input, uint32_t input_len); #if defined(__cplusplus) } diff --git a/Modules/_hacl/Hacl_Hash_SHA1.c b/Modules/_hacl/Hacl_Hash_SHA1.c index 5ecb3c0b3a56e0..1a8b09b1711894 100644 --- a/Modules/_hacl/Hacl_Hash_SHA1.c +++ b/Modules/_hacl/Hacl_Hash_SHA1.c @@ -25,19 +25,14 @@ #include "internal/Hacl_Hash_SHA1.h" -static uint32_t -_h0[5U] = - { - (uint32_t)0x67452301U, (uint32_t)0xefcdab89U, (uint32_t)0x98badcfeU, (uint32_t)0x10325476U, - (uint32_t)0xc3d2e1f0U - }; +static uint32_t _h0[5U] = { 0x67452301U, 0xefcdab89U, 0x98badcfeU, 0x10325476U, 0xc3d2e1f0U }; -void Hacl_Hash_Core_SHA1_legacy_init(uint32_t *s) +void Hacl_Hash_SHA1_init(uint32_t *s) { - KRML_MAYBE_FOR5(i, (uint32_t)0U, (uint32_t)5U, (uint32_t)1U, s[i] = _h0[i];); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i] = _h0[i];); } -static void legacy_update(uint32_t *h, uint8_t *l) +static void update(uint32_t *h, uint8_t *l) { uint32_t ha = h[0U]; uint32_t hb = h[1U]; @@ -45,29 +40,26 @@ static void legacy_update(uint32_t *h, uint8_t *l) uint32_t hd = h[3U]; uint32_t he = h[4U]; uint32_t _w[80U] = { 0U }; - for (uint32_t i = (uint32_t)0U; i < (uint32_t)80U; i++) + for (uint32_t i = 0U; i < 80U; i++) { uint32_t v; - if (i < (uint32_t)16U) + if (i < 16U) { - uint8_t *b = l + i * (uint32_t)4U; + uint8_t *b = l + i * 4U; uint32_t u = load32_be(b); v = u; } else { - uint32_t wmit3 = _w[i - (uint32_t)3U]; - uint32_t wmit8 = _w[i - (uint32_t)8U]; - uint32_t wmit14 = _w[i - (uint32_t)14U]; - uint32_t wmit16 = _w[i - (uint32_t)16U]; - v = - (wmit3 ^ (wmit8 ^ (wmit14 ^ wmit16))) - << (uint32_t)1U - | (wmit3 ^ (wmit8 ^ (wmit14 ^ wmit16))) >> (uint32_t)31U; + uint32_t wmit3 = _w[i - 3U]; + uint32_t wmit8 = _w[i - 8U]; + uint32_t wmit14 = _w[i - 14U]; + uint32_t wmit16 = _w[i - 16U]; + v = (wmit3 ^ (wmit8 ^ (wmit14 ^ wmit16))) << 1U | (wmit3 ^ (wmit8 ^ (wmit14 ^ wmit16))) >> 31U; } _w[i] = v; } - for (uint32_t i = (uint32_t)0U; i < (uint32_t)80U; i++) + for (uint32_t i = 0U; i < 80U; i++) { uint32_t _a = h[0U]; uint32_t _b = h[1U]; @@ -76,11 +68,11 @@ static void legacy_update(uint32_t *h, uint8_t *l) uint32_t _e = h[4U]; uint32_t wmit = _w[i]; uint32_t ite0; - if (i < (uint32_t)20U) + if (i < 20U) { ite0 = (_b & _c) ^ (~_b & _d); } - else if ((uint32_t)39U < i && i < (uint32_t)60U) + else if (39U < i && i < 60U) { ite0 = (_b & _c) ^ ((_b & _d) ^ (_c & _d)); } @@ -89,32 +81,32 @@ static void legacy_update(uint32_t *h, uint8_t *l) ite0 = _b ^ (_c ^ _d); } uint32_t ite; - if (i < (uint32_t)20U) + if (i < 20U) { - ite = (uint32_t)0x5a827999U; + ite = 0x5a827999U; } - else if (i < (uint32_t)40U) + else if (i < 40U) { - ite = (uint32_t)0x6ed9eba1U; + ite = 0x6ed9eba1U; } - else if (i < (uint32_t)60U) + else if (i < 60U) { - ite = (uint32_t)0x8f1bbcdcU; + ite = 0x8f1bbcdcU; } else { - ite = (uint32_t)0xca62c1d6U; + ite = 0xca62c1d6U; } - uint32_t _T = (_a << (uint32_t)5U | _a >> (uint32_t)27U) + ite0 + _e + ite + wmit; + uint32_t _T = (_a << 5U | _a >> 27U) + ite0 + _e + ite + wmit; h[0U] = _T; h[1U] = _a; - h[2U] = _b << (uint32_t)30U | _b >> (uint32_t)2U; + h[2U] = _b << 30U | _b >> 2U; h[3U] = _c; h[4U] = _d; } - for (uint32_t i = (uint32_t)0U; i < (uint32_t)80U; i++) + for (uint32_t i = 0U; i < 80U; i++) { - _w[i] = (uint32_t)0U; + _w[i] = 0U; } uint32_t sta = h[0U]; uint32_t stb = h[1U]; @@ -128,101 +120,69 @@ static void legacy_update(uint32_t *h, uint8_t *l) h[4U] = ste + he; } -static void legacy_pad(uint64_t len, uint8_t *dst) +static void pad(uint64_t len, uint8_t *dst) { uint8_t *dst1 = dst; - dst1[0U] = (uint8_t)0x80U; - uint8_t *dst2 = dst + (uint32_t)1U; - for - (uint32_t - i = (uint32_t)0U; - i - < ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(len % (uint64_t)(uint32_t)64U))) % (uint32_t)64U; - i++) + dst1[0U] = 0x80U; + uint8_t *dst2 = dst + 1U; + for (uint32_t i = 0U; i < (128U - (9U + (uint32_t)(len % (uint64_t)64U))) % 64U; i++) { - dst2[i] = (uint8_t)0U; + dst2[i] = 0U; } - uint8_t - *dst3 = - dst - + - (uint32_t)1U - + - ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(len % (uint64_t)(uint32_t)64U))) - % (uint32_t)64U; - store64_be(dst3, len << (uint32_t)3U); + uint8_t *dst3 = dst + 1U + (128U - (9U + (uint32_t)(len % (uint64_t)64U))) % 64U; + store64_be(dst3, len << 3U); } -void Hacl_Hash_Core_SHA1_legacy_finish(uint32_t *s, uint8_t *dst) +void Hacl_Hash_SHA1_finish(uint32_t *s, uint8_t *dst) { - KRML_MAYBE_FOR5(i, - (uint32_t)0U, - (uint32_t)5U, - (uint32_t)1U, - store32_be(dst + i * (uint32_t)4U, s[i]);); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, store32_be(dst + i * 4U, s[i]);); } -void Hacl_Hash_SHA1_legacy_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks) +void Hacl_Hash_SHA1_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks) { - for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + for (uint32_t i = 0U; i < n_blocks; i++) { - uint32_t sz = (uint32_t)64U; + uint32_t sz = 64U; uint8_t *block = blocks + sz * i; - legacy_update(s, block); + update(s, block); } } void -Hacl_Hash_SHA1_legacy_update_last( - uint32_t *s, - uint64_t prev_len, - uint8_t *input, - uint32_t input_len -) +Hacl_Hash_SHA1_update_last(uint32_t *s, uint64_t prev_len, uint8_t *input, uint32_t input_len) { - uint32_t blocks_n = input_len / (uint32_t)64U; - uint32_t blocks_len = blocks_n * (uint32_t)64U; + uint32_t blocks_n = input_len / 64U; + uint32_t blocks_len = blocks_n * 64U; uint8_t *blocks = input; uint32_t rest_len = input_len - blocks_len; uint8_t *rest = input + blocks_len; - Hacl_Hash_SHA1_legacy_update_multi(s, blocks, blocks_n); + Hacl_Hash_SHA1_update_multi(s, blocks, blocks_n); uint64_t total_input_len = prev_len + (uint64_t)input_len; - uint32_t - pad_len = - (uint32_t)1U - + - ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(total_input_len % (uint64_t)(uint32_t)64U))) - % (uint32_t)64U - + (uint32_t)8U; + uint32_t pad_len = 1U + (128U - (9U + (uint32_t)(total_input_len % (uint64_t)64U))) % 64U + 8U; uint32_t tmp_len = rest_len + pad_len; uint8_t tmp_twoblocks[128U] = { 0U }; uint8_t *tmp = tmp_twoblocks; uint8_t *tmp_rest = tmp; uint8_t *tmp_pad = tmp + rest_len; memcpy(tmp_rest, rest, rest_len * sizeof (uint8_t)); - legacy_pad(total_input_len, tmp_pad); - Hacl_Hash_SHA1_legacy_update_multi(s, tmp, tmp_len / (uint32_t)64U); + pad(total_input_len, tmp_pad); + Hacl_Hash_SHA1_update_multi(s, tmp, tmp_len / 64U); } -void Hacl_Hash_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_SHA1_hash_oneshot(uint8_t *output, uint8_t *input, uint32_t input_len) { - uint32_t - s[5U] = - { - (uint32_t)0x67452301U, (uint32_t)0xefcdab89U, (uint32_t)0x98badcfeU, (uint32_t)0x10325476U, - (uint32_t)0xc3d2e1f0U - }; - uint32_t blocks_n0 = input_len / (uint32_t)64U; + uint32_t s[5U] = { 0x67452301U, 0xefcdab89U, 0x98badcfeU, 0x10325476U, 0xc3d2e1f0U }; + uint32_t blocks_n0 = input_len / 64U; uint32_t blocks_n1; - if (input_len % (uint32_t)64U == (uint32_t)0U && blocks_n0 > (uint32_t)0U) + if (input_len % 64U == 0U && blocks_n0 > 0U) { - blocks_n1 = blocks_n0 - (uint32_t)1U; + blocks_n1 = blocks_n0 - 1U; } else { blocks_n1 = blocks_n0; } - uint32_t blocks_len0 = blocks_n1 * (uint32_t)64U; + uint32_t blocks_len0 = blocks_n1 * 64U; uint8_t *blocks0 = input; uint32_t rest_len0 = input_len - blocks_len0; uint8_t *rest0 = input + blocks_len0; @@ -231,75 +191,75 @@ void Hacl_Hash_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst uint8_t *blocks = blocks0; uint32_t rest_len = rest_len0; uint8_t *rest = rest0; - Hacl_Hash_SHA1_legacy_update_multi(s, blocks, blocks_n); - Hacl_Hash_SHA1_legacy_update_last(s, (uint64_t)blocks_len, rest, rest_len); - Hacl_Hash_Core_SHA1_legacy_finish(s, dst); + Hacl_Hash_SHA1_update_multi(s, blocks, blocks_n); + Hacl_Hash_SHA1_update_last(s, (uint64_t)blocks_len, rest, rest_len); + Hacl_Hash_SHA1_finish(s, output); } -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_create_in(void) +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA1_malloc(void) { - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); - uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)5U, sizeof (uint32_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC(5U, sizeof (uint32_t)); Hacl_Streaming_MD_state_32 - s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; Hacl_Streaming_MD_state_32 *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); p[0U] = s; - Hacl_Hash_Core_SHA1_legacy_init(block_state); + Hacl_Hash_SHA1_init(block_state); return p; } -void Hacl_Streaming_SHA1_legacy_init(Hacl_Streaming_MD_state_32 *s) +void Hacl_Hash_SHA1_reset(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *state; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; - Hacl_Hash_Core_SHA1_legacy_init(block_state); + Hacl_Hash_SHA1_init(block_state); Hacl_Streaming_MD_state_32 - tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - s[0U] = tmp; + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; + state[0U] = tmp; } /** 0 = success, 1 = max length exceeded */ Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA1_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) +Hacl_Hash_SHA1_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk_len) { - Hacl_Streaming_MD_state_32 s = *p; + Hacl_Streaming_MD_state_32 s = *state; uint64_t total_len = s.total_len; - if ((uint64_t)len > (uint64_t)2305843009213693951U - total_len) + if ((uint64_t)chunk_len > 2305843009213693951ULL - total_len) { return Hacl_Streaming_Types_MaximumLengthExceeded; } uint32_t sz; - if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) { - sz = (uint32_t)64U; + sz = 64U; } else { - sz = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + sz = (uint32_t)(total_len % (uint64_t)64U); } - if (len <= (uint32_t)64U - sz) + if (chunk_len <= 64U - sz) { - Hacl_Streaming_MD_state_32 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } uint8_t *buf2 = buf + sz1; - memcpy(buf2, data, len * sizeof (uint8_t)); - uint64_t total_len2 = total_len1 + (uint64_t)len; - *p + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state = ( (Hacl_Streaming_MD_state_32){ @@ -309,74 +269,74 @@ Hacl_Streaming_SHA1_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, } ); } - else if (sz == (uint32_t)0U) + else if (sz == 0U) { - Hacl_Streaming_MD_state_32 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_Hash_SHA1_legacy_update_multi(block_state1, buf, (uint32_t)1U); + Hacl_Hash_SHA1_update_multi(block_state1, buf, 1U); } uint32_t ite; - if ((uint64_t)len % (uint64_t)(uint32_t)64U == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + if ((uint64_t)chunk_len % (uint64_t)64U == 0ULL && (uint64_t)chunk_len > 0ULL) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = (uint32_t)((uint64_t)len % (uint64_t)(uint32_t)64U); + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)64U); } - uint32_t n_blocks = (len - ite) / (uint32_t)64U; - uint32_t data1_len = n_blocks * (uint32_t)64U; - uint32_t data2_len = len - data1_len; - uint8_t *data1 = data; - uint8_t *data2 = data + data1_len; - Hacl_Hash_SHA1_legacy_update_multi(block_state1, data1, data1_len / (uint32_t)64U); + uint32_t n_blocks = (chunk_len - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; + Hacl_Hash_SHA1_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *p + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)len + .total_len = total_len1 + (uint64_t)chunk_len } ); } else { - uint32_t diff = (uint32_t)64U - sz; - uint8_t *data1 = data; - uint8_t *data2 = data + diff; - Hacl_Streaming_MD_state_32 s1 = *p; + uint32_t diff = 64U - sz; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state10 = s1.block_state; uint8_t *buf0 = s1.buf; uint64_t total_len10 = s1.total_len; uint32_t sz10; - if (total_len10 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len10 > (uint64_t)0U) + if (total_len10 % (uint64_t)64U == 0ULL && total_len10 > 0ULL) { - sz10 = (uint32_t)64U; + sz10 = 64U; } else { - sz10 = (uint32_t)(total_len10 % (uint64_t)(uint32_t)64U); + sz10 = (uint32_t)(total_len10 % (uint64_t)64U); } uint8_t *buf2 = buf0 + sz10; - memcpy(buf2, data1, diff * sizeof (uint8_t)); + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *p + *state = ( (Hacl_Streaming_MD_state_32){ @@ -385,114 +345,109 @@ Hacl_Streaming_SHA1_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, .total_len = total_len2 } ); - Hacl_Streaming_MD_state_32 s10 = *p; + Hacl_Streaming_MD_state_32 s10 = *state; uint32_t *block_state1 = s10.block_state; uint8_t *buf = s10.buf; uint64_t total_len1 = s10.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_Hash_SHA1_legacy_update_multi(block_state1, buf, (uint32_t)1U); + Hacl_Hash_SHA1_update_multi(block_state1, buf, 1U); } uint32_t ite; if - ( - (uint64_t)(len - diff) - % (uint64_t)(uint32_t)64U - == (uint64_t)0U - && (uint64_t)(len - diff) > (uint64_t)0U - ) + ((uint64_t)(chunk_len - diff) % (uint64_t)64U == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)(uint32_t)64U); + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)64U); } - uint32_t n_blocks = (len - diff - ite) / (uint32_t)64U; - uint32_t data1_len = n_blocks * (uint32_t)64U; - uint32_t data2_len = len - diff - data1_len; - uint8_t *data11 = data2; - uint8_t *data21 = data2 + data1_len; - Hacl_Hash_SHA1_legacy_update_multi(block_state1, data11, data1_len / (uint32_t)64U); + uint32_t n_blocks = (chunk_len - diff - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; + Hacl_Hash_SHA1_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; - memcpy(dst, data21, data2_len * sizeof (uint8_t)); - *p + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)(len - diff) + .total_len = total_len1 + (uint64_t)(chunk_len - diff) } ); } return Hacl_Streaming_Types_Success; } -void Hacl_Streaming_SHA1_legacy_finish(Hacl_Streaming_MD_state_32 *p, uint8_t *dst) +void Hacl_Hash_SHA1_digest(Hacl_Streaming_MD_state_32 *state, uint8_t *output) { - Hacl_Streaming_MD_state_32 scrut = *p; + Hacl_Streaming_MD_state_32 scrut = *state; uint32_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; uint32_t r; - if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) { - r = (uint32_t)64U; + r = 64U; } else { - r = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + r = (uint32_t)(total_len % (uint64_t)64U); } uint8_t *buf_1 = buf_; uint32_t tmp_block_state[5U] = { 0U }; - memcpy(tmp_block_state, block_state, (uint32_t)5U * sizeof (uint32_t)); + memcpy(tmp_block_state, block_state, 5U * sizeof (uint32_t)); uint32_t ite; - if (r % (uint32_t)64U == (uint32_t)0U && r > (uint32_t)0U) + if (r % 64U == 0U && r > 0U) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = r % (uint32_t)64U; + ite = r % 64U; } uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; - Hacl_Hash_SHA1_legacy_update_multi(tmp_block_state, buf_multi, (uint32_t)0U); + Hacl_Hash_SHA1_update_multi(tmp_block_state, buf_multi, 0U); uint64_t prev_len_last = total_len - (uint64_t)r; - Hacl_Hash_SHA1_legacy_update_last(tmp_block_state, prev_len_last, buf_last, r); - Hacl_Hash_Core_SHA1_legacy_finish(tmp_block_state, dst); + Hacl_Hash_SHA1_update_last(tmp_block_state, prev_len_last, buf_last, r); + Hacl_Hash_SHA1_finish(tmp_block_state, output); } -void Hacl_Streaming_SHA1_legacy_free(Hacl_Streaming_MD_state_32 *s) +void Hacl_Hash_SHA1_free(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *state; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; KRML_HOST_FREE(block_state); KRML_HOST_FREE(buf); - KRML_HOST_FREE(s); + KRML_HOST_FREE(state); } -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_copy(Hacl_Streaming_MD_state_32 *s0) +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA1_copy(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s0; + Hacl_Streaming_MD_state_32 scrut = *state; uint32_t *block_state0 = scrut.block_state; uint8_t *buf0 = scrut.buf; uint64_t total_len0 = scrut.total_len; - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); - memcpy(buf, buf0, (uint32_t)64U * sizeof (uint8_t)); - uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)5U, sizeof (uint32_t)); - memcpy(block_state, block_state0, (uint32_t)5U * sizeof (uint32_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + memcpy(buf, buf0, 64U * sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC(5U, sizeof (uint32_t)); + memcpy(block_state, block_state0, 5U * sizeof (uint32_t)); Hacl_Streaming_MD_state_32 s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; Hacl_Streaming_MD_state_32 @@ -501,8 +456,8 @@ Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_copy(Hacl_Streaming_MD_st return p; } -void Hacl_Streaming_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_SHA1_hash(uint8_t *output, uint8_t *input, uint32_t input_len) { - Hacl_Hash_SHA1_legacy_hash(input, input_len, dst); + Hacl_Hash_SHA1_hash_oneshot(output, input, input_len); } diff --git a/Modules/_hacl/Hacl_Hash_SHA1.h b/Modules/_hacl/Hacl_Hash_SHA1.h index dc50aa6f6d3902..ad1e8e72a739ec 100644 --- a/Modules/_hacl/Hacl_Hash_SHA1.h +++ b/Modules/_hacl/Hacl_Hash_SHA1.h @@ -31,31 +31,32 @@ extern "C" { #endif #include +#include "python_hacl_namespaces.h" #include "krml/types.h" #include "krml/lowstar_endianness.h" #include "krml/internal/target.h" #include "Hacl_Streaming_Types.h" -typedef Hacl_Streaming_MD_state_32 Hacl_Streaming_SHA1_state; +typedef Hacl_Streaming_MD_state_32 Hacl_Hash_SHA1_state_t; -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_create_in(void); +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA1_malloc(void); -void Hacl_Streaming_SHA1_legacy_init(Hacl_Streaming_MD_state_32 *s); +void Hacl_Hash_SHA1_reset(Hacl_Streaming_MD_state_32 *state); /** 0 = success, 1 = max length exceeded */ Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA1_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len); +Hacl_Hash_SHA1_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk_len); -void Hacl_Streaming_SHA1_legacy_finish(Hacl_Streaming_MD_state_32 *p, uint8_t *dst); +void Hacl_Hash_SHA1_digest(Hacl_Streaming_MD_state_32 *state, uint8_t *output); -void Hacl_Streaming_SHA1_legacy_free(Hacl_Streaming_MD_state_32 *s); +void Hacl_Hash_SHA1_free(Hacl_Streaming_MD_state_32 *state); -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_copy(Hacl_Streaming_MD_state_32 *s0); +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA1_copy(Hacl_Streaming_MD_state_32 *state); -void Hacl_Streaming_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst); +void Hacl_Hash_SHA1_hash(uint8_t *output, uint8_t *input, uint32_t input_len); #if defined(__cplusplus) } diff --git a/Modules/_hacl/Hacl_Hash_SHA2.c b/Modules/_hacl/Hacl_Hash_SHA2.c index 08e3f7edbf4ede..cc930bbc89e8ad 100644 --- a/Modules/_hacl/Hacl_Hash_SHA2.c +++ b/Modules/_hacl/Hacl_Hash_SHA2.c @@ -27,14 +27,14 @@ -void Hacl_SHA2_Scalar32_sha256_init(uint32_t *hash) +void Hacl_Hash_SHA2_sha256_init(uint32_t *hash) { KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, + 0U, + 8U, + 1U, uint32_t *os = hash; - uint32_t x = Hacl_Impl_SHA2_Generic_h256[i]; + uint32_t x = Hacl_Hash_SHA2_h256[i]; os[i] = x;); } @@ -42,49 +42,49 @@ static inline void sha256_update(uint8_t *b, uint32_t *hash) { uint32_t hash_old[8U] = { 0U }; uint32_t ws[16U] = { 0U }; - memcpy(hash_old, hash, (uint32_t)8U * sizeof (uint32_t)); + memcpy(hash_old, hash, 8U * sizeof (uint32_t)); uint8_t *b10 = b; uint32_t u = load32_be(b10); ws[0U] = u; - uint32_t u0 = load32_be(b10 + (uint32_t)4U); + uint32_t u0 = load32_be(b10 + 4U); ws[1U] = u0; - uint32_t u1 = load32_be(b10 + (uint32_t)8U); + uint32_t u1 = load32_be(b10 + 8U); ws[2U] = u1; - uint32_t u2 = load32_be(b10 + (uint32_t)12U); + uint32_t u2 = load32_be(b10 + 12U); ws[3U] = u2; - uint32_t u3 = load32_be(b10 + (uint32_t)16U); + uint32_t u3 = load32_be(b10 + 16U); ws[4U] = u3; - uint32_t u4 = load32_be(b10 + (uint32_t)20U); + uint32_t u4 = load32_be(b10 + 20U); ws[5U] = u4; - uint32_t u5 = load32_be(b10 + (uint32_t)24U); + uint32_t u5 = load32_be(b10 + 24U); ws[6U] = u5; - uint32_t u6 = load32_be(b10 + (uint32_t)28U); + uint32_t u6 = load32_be(b10 + 28U); ws[7U] = u6; - uint32_t u7 = load32_be(b10 + (uint32_t)32U); + uint32_t u7 = load32_be(b10 + 32U); ws[8U] = u7; - uint32_t u8 = load32_be(b10 + (uint32_t)36U); + uint32_t u8 = load32_be(b10 + 36U); ws[9U] = u8; - uint32_t u9 = load32_be(b10 + (uint32_t)40U); + uint32_t u9 = load32_be(b10 + 40U); ws[10U] = u9; - uint32_t u10 = load32_be(b10 + (uint32_t)44U); + uint32_t u10 = load32_be(b10 + 44U); ws[11U] = u10; - uint32_t u11 = load32_be(b10 + (uint32_t)48U); + uint32_t u11 = load32_be(b10 + 48U); ws[12U] = u11; - uint32_t u12 = load32_be(b10 + (uint32_t)52U); + uint32_t u12 = load32_be(b10 + 52U); ws[13U] = u12; - uint32_t u13 = load32_be(b10 + (uint32_t)56U); + uint32_t u13 = load32_be(b10 + 56U); ws[14U] = u13; - uint32_t u14 = load32_be(b10 + (uint32_t)60U); + uint32_t u14 = load32_be(b10 + 60U); ws[15U] = u14; KRML_MAYBE_FOR4(i0, - (uint32_t)0U, - (uint32_t)4U, - (uint32_t)1U, + 0U, + 4U, + 1U, KRML_MAYBE_FOR16(i, - (uint32_t)0U, - (uint32_t)16U, - (uint32_t)1U, - uint32_t k_t = Hacl_Impl_SHA2_Generic_k224_256[(uint32_t)16U * i0 + i]; + 0U, + 16U, + 1U, + uint32_t k_t = Hacl_Hash_SHA2_k224_256[16U * i0 + i]; uint32_t ws_t = ws[i]; uint32_t a0 = hash[0U]; uint32_t b0 = hash[1U]; @@ -98,20 +98,13 @@ static inline void sha256_update(uint8_t *b, uint32_t *hash) uint32_t t1 = h02 - + - ((e0 << (uint32_t)26U | e0 >> (uint32_t)6U) - ^ - ((e0 << (uint32_t)21U | e0 >> (uint32_t)11U) - ^ (e0 << (uint32_t)7U | e0 >> (uint32_t)25U))) + + ((e0 << 26U | e0 >> 6U) ^ ((e0 << 21U | e0 >> 11U) ^ (e0 << 7U | e0 >> 25U))) + ((e0 & f0) ^ (~e0 & g0)) + k_e_t + ws_t; uint32_t t2 = - ((a0 << (uint32_t)30U | a0 >> (uint32_t)2U) - ^ - ((a0 << (uint32_t)19U | a0 >> (uint32_t)13U) - ^ (a0 << (uint32_t)10U | a0 >> (uint32_t)22U))) + ((a0 << 30U | a0 >> 2U) ^ ((a0 << 19U | a0 >> 13U) ^ (a0 << 10U | a0 >> 22U))) + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); uint32_t a1 = t1 + t2; uint32_t b1 = a0; @@ -129,74 +122,63 @@ static inline void sha256_update(uint8_t *b, uint32_t *hash) hash[5U] = f1; hash[6U] = g1; hash[7U] = h12;); - if (i0 < (uint32_t)3U) + if (i0 < 3U) { KRML_MAYBE_FOR16(i, - (uint32_t)0U, - (uint32_t)16U, - (uint32_t)1U, + 0U, + 16U, + 1U, uint32_t t16 = ws[i]; - uint32_t t15 = ws[(i + (uint32_t)1U) % (uint32_t)16U]; - uint32_t t7 = ws[(i + (uint32_t)9U) % (uint32_t)16U]; - uint32_t t2 = ws[(i + (uint32_t)14U) % (uint32_t)16U]; - uint32_t - s1 = - (t2 << (uint32_t)15U | t2 >> (uint32_t)17U) - ^ ((t2 << (uint32_t)13U | t2 >> (uint32_t)19U) ^ t2 >> (uint32_t)10U); - uint32_t - s0 = - (t15 << (uint32_t)25U | t15 >> (uint32_t)7U) - ^ ((t15 << (uint32_t)14U | t15 >> (uint32_t)18U) ^ t15 >> (uint32_t)3U); + uint32_t t15 = ws[(i + 1U) % 16U]; + uint32_t t7 = ws[(i + 9U) % 16U]; + uint32_t t2 = ws[(i + 14U) % 16U]; + uint32_t s1 = (t2 << 15U | t2 >> 17U) ^ ((t2 << 13U | t2 >> 19U) ^ t2 >> 10U); + uint32_t s0 = (t15 << 25U | t15 >> 7U) ^ ((t15 << 14U | t15 >> 18U) ^ t15 >> 3U); ws[i] = s1 + t7 + s0 + t16;); }); KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, + 0U, + 8U, + 1U, uint32_t *os = hash; uint32_t x = hash[i] + hash_old[i]; os[i] = x;); } -void Hacl_SHA2_Scalar32_sha256_update_nblocks(uint32_t len, uint8_t *b, uint32_t *st) +void Hacl_Hash_SHA2_sha256_update_nblocks(uint32_t len, uint8_t *b, uint32_t *st) { - uint32_t blocks = len / (uint32_t)64U; - for (uint32_t i = (uint32_t)0U; i < blocks; i++) + uint32_t blocks = len / 64U; + for (uint32_t i = 0U; i < blocks; i++) { uint8_t *b0 = b; - uint8_t *mb = b0 + i * (uint32_t)64U; + uint8_t *mb = b0 + i * 64U; sha256_update(mb, st); } } void -Hacl_SHA2_Scalar32_sha256_update_last( - uint64_t totlen, - uint32_t len, - uint8_t *b, - uint32_t *hash -) +Hacl_Hash_SHA2_sha256_update_last(uint64_t totlen, uint32_t len, uint8_t *b, uint32_t *hash) { uint32_t blocks; - if (len + (uint32_t)8U + (uint32_t)1U <= (uint32_t)64U) + if (len + 8U + 1U <= 64U) { - blocks = (uint32_t)1U; + blocks = 1U; } else { - blocks = (uint32_t)2U; + blocks = 2U; } - uint32_t fin = blocks * (uint32_t)64U; + uint32_t fin = blocks * 64U; uint8_t last[128U] = { 0U }; uint8_t totlen_buf[8U] = { 0U }; - uint64_t total_len_bits = totlen << (uint32_t)3U; + uint64_t total_len_bits = totlen << 3U; store64_be(totlen_buf, total_len_bits); uint8_t *b0 = b; memcpy(last, b0, len * sizeof (uint8_t)); - last[len] = (uint8_t)0x80U; - memcpy(last + fin - (uint32_t)8U, totlen_buf, (uint32_t)8U * sizeof (uint8_t)); + last[len] = 0x80U; + memcpy(last + fin - 8U, totlen_buf, 8U * sizeof (uint8_t)); uint8_t *last00 = last; - uint8_t *last10 = last + (uint32_t)64U; + uint8_t *last10 = last + 64U; uint8_t *l0 = last00; uint8_t *l1 = last10; uint8_t *lb0 = l0; @@ -204,65 +186,56 @@ Hacl_SHA2_Scalar32_sha256_update_last( uint8_t *last0 = lb0; uint8_t *last1 = lb1; sha256_update(last0, hash); - if (blocks > (uint32_t)1U) + if (blocks > 1U) { sha256_update(last1, hash); return; } } -void Hacl_SHA2_Scalar32_sha256_finish(uint32_t *st, uint8_t *h) +void Hacl_Hash_SHA2_sha256_finish(uint32_t *st, uint8_t *h) { uint8_t hbuf[32U] = { 0U }; - KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, - store32_be(hbuf + i * (uint32_t)4U, st[i]);); - memcpy(h, hbuf, (uint32_t)32U * sizeof (uint8_t)); + KRML_MAYBE_FOR8(i, 0U, 8U, 1U, store32_be(hbuf + i * 4U, st[i]);); + memcpy(h, hbuf, 32U * sizeof (uint8_t)); } -void Hacl_SHA2_Scalar32_sha224_init(uint32_t *hash) +void Hacl_Hash_SHA2_sha224_init(uint32_t *hash) { KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, + 0U, + 8U, + 1U, uint32_t *os = hash; - uint32_t x = Hacl_Impl_SHA2_Generic_h224[i]; + uint32_t x = Hacl_Hash_SHA2_h224[i]; os[i] = x;); } -static inline void sha224_update_nblocks(uint32_t len, uint8_t *b, uint32_t *st) +void Hacl_Hash_SHA2_sha224_update_nblocks(uint32_t len, uint8_t *b, uint32_t *st) { - Hacl_SHA2_Scalar32_sha256_update_nblocks(len, b, st); + Hacl_Hash_SHA2_sha256_update_nblocks(len, b, st); } -void -Hacl_SHA2_Scalar32_sha224_update_last(uint64_t totlen, uint32_t len, uint8_t *b, uint32_t *st) +void Hacl_Hash_SHA2_sha224_update_last(uint64_t totlen, uint32_t len, uint8_t *b, uint32_t *st) { - Hacl_SHA2_Scalar32_sha256_update_last(totlen, len, b, st); + Hacl_Hash_SHA2_sha256_update_last(totlen, len, b, st); } -void Hacl_SHA2_Scalar32_sha224_finish(uint32_t *st, uint8_t *h) +void Hacl_Hash_SHA2_sha224_finish(uint32_t *st, uint8_t *h) { uint8_t hbuf[32U] = { 0U }; - KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, - store32_be(hbuf + i * (uint32_t)4U, st[i]);); - memcpy(h, hbuf, (uint32_t)28U * sizeof (uint8_t)); + KRML_MAYBE_FOR8(i, 0U, 8U, 1U, store32_be(hbuf + i * 4U, st[i]);); + memcpy(h, hbuf, 28U * sizeof (uint8_t)); } -void Hacl_SHA2_Scalar32_sha512_init(uint64_t *hash) +void Hacl_Hash_SHA2_sha512_init(uint64_t *hash) { KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, + 0U, + 8U, + 1U, uint64_t *os = hash; - uint64_t x = Hacl_Impl_SHA2_Generic_h512[i]; + uint64_t x = Hacl_Hash_SHA2_h512[i]; os[i] = x;); } @@ -270,49 +243,49 @@ static inline void sha512_update(uint8_t *b, uint64_t *hash) { uint64_t hash_old[8U] = { 0U }; uint64_t ws[16U] = { 0U }; - memcpy(hash_old, hash, (uint32_t)8U * sizeof (uint64_t)); + memcpy(hash_old, hash, 8U * sizeof (uint64_t)); uint8_t *b10 = b; uint64_t u = load64_be(b10); ws[0U] = u; - uint64_t u0 = load64_be(b10 + (uint32_t)8U); + uint64_t u0 = load64_be(b10 + 8U); ws[1U] = u0; - uint64_t u1 = load64_be(b10 + (uint32_t)16U); + uint64_t u1 = load64_be(b10 + 16U); ws[2U] = u1; - uint64_t u2 = load64_be(b10 + (uint32_t)24U); + uint64_t u2 = load64_be(b10 + 24U); ws[3U] = u2; - uint64_t u3 = load64_be(b10 + (uint32_t)32U); + uint64_t u3 = load64_be(b10 + 32U); ws[4U] = u3; - uint64_t u4 = load64_be(b10 + (uint32_t)40U); + uint64_t u4 = load64_be(b10 + 40U); ws[5U] = u4; - uint64_t u5 = load64_be(b10 + (uint32_t)48U); + uint64_t u5 = load64_be(b10 + 48U); ws[6U] = u5; - uint64_t u6 = load64_be(b10 + (uint32_t)56U); + uint64_t u6 = load64_be(b10 + 56U); ws[7U] = u6; - uint64_t u7 = load64_be(b10 + (uint32_t)64U); + uint64_t u7 = load64_be(b10 + 64U); ws[8U] = u7; - uint64_t u8 = load64_be(b10 + (uint32_t)72U); + uint64_t u8 = load64_be(b10 + 72U); ws[9U] = u8; - uint64_t u9 = load64_be(b10 + (uint32_t)80U); + uint64_t u9 = load64_be(b10 + 80U); ws[10U] = u9; - uint64_t u10 = load64_be(b10 + (uint32_t)88U); + uint64_t u10 = load64_be(b10 + 88U); ws[11U] = u10; - uint64_t u11 = load64_be(b10 + (uint32_t)96U); + uint64_t u11 = load64_be(b10 + 96U); ws[12U] = u11; - uint64_t u12 = load64_be(b10 + (uint32_t)104U); + uint64_t u12 = load64_be(b10 + 104U); ws[13U] = u12; - uint64_t u13 = load64_be(b10 + (uint32_t)112U); + uint64_t u13 = load64_be(b10 + 112U); ws[14U] = u13; - uint64_t u14 = load64_be(b10 + (uint32_t)120U); + uint64_t u14 = load64_be(b10 + 120U); ws[15U] = u14; KRML_MAYBE_FOR5(i0, - (uint32_t)0U, - (uint32_t)5U, - (uint32_t)1U, + 0U, + 5U, + 1U, KRML_MAYBE_FOR16(i, - (uint32_t)0U, - (uint32_t)16U, - (uint32_t)1U, - uint64_t k_t = Hacl_Impl_SHA2_Generic_k384_512[(uint32_t)16U * i0 + i]; + 0U, + 16U, + 1U, + uint64_t k_t = Hacl_Hash_SHA2_k384_512[16U * i0 + i]; uint64_t ws_t = ws[i]; uint64_t a0 = hash[0U]; uint64_t b0 = hash[1U]; @@ -326,20 +299,13 @@ static inline void sha512_update(uint8_t *b, uint64_t *hash) uint64_t t1 = h02 - + - ((e0 << (uint32_t)50U | e0 >> (uint32_t)14U) - ^ - ((e0 << (uint32_t)46U | e0 >> (uint32_t)18U) - ^ (e0 << (uint32_t)23U | e0 >> (uint32_t)41U))) + + ((e0 << 50U | e0 >> 14U) ^ ((e0 << 46U | e0 >> 18U) ^ (e0 << 23U | e0 >> 41U))) + ((e0 & f0) ^ (~e0 & g0)) + k_e_t + ws_t; uint64_t t2 = - ((a0 << (uint32_t)36U | a0 >> (uint32_t)28U) - ^ - ((a0 << (uint32_t)30U | a0 >> (uint32_t)34U) - ^ (a0 << (uint32_t)25U | a0 >> (uint32_t)39U))) + ((a0 << 36U | a0 >> 28U) ^ ((a0 << 30U | a0 >> 34U) ^ (a0 << 25U | a0 >> 39U))) + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); uint64_t a1 = t1 + t2; uint64_t b1 = a0; @@ -357,48 +323,42 @@ static inline void sha512_update(uint8_t *b, uint64_t *hash) hash[5U] = f1; hash[6U] = g1; hash[7U] = h12;); - if (i0 < (uint32_t)4U) + if (i0 < 4U) { KRML_MAYBE_FOR16(i, - (uint32_t)0U, - (uint32_t)16U, - (uint32_t)1U, + 0U, + 16U, + 1U, uint64_t t16 = ws[i]; - uint64_t t15 = ws[(i + (uint32_t)1U) % (uint32_t)16U]; - uint64_t t7 = ws[(i + (uint32_t)9U) % (uint32_t)16U]; - uint64_t t2 = ws[(i + (uint32_t)14U) % (uint32_t)16U]; - uint64_t - s1 = - (t2 << (uint32_t)45U | t2 >> (uint32_t)19U) - ^ ((t2 << (uint32_t)3U | t2 >> (uint32_t)61U) ^ t2 >> (uint32_t)6U); - uint64_t - s0 = - (t15 << (uint32_t)63U | t15 >> (uint32_t)1U) - ^ ((t15 << (uint32_t)56U | t15 >> (uint32_t)8U) ^ t15 >> (uint32_t)7U); + uint64_t t15 = ws[(i + 1U) % 16U]; + uint64_t t7 = ws[(i + 9U) % 16U]; + uint64_t t2 = ws[(i + 14U) % 16U]; + uint64_t s1 = (t2 << 45U | t2 >> 19U) ^ ((t2 << 3U | t2 >> 61U) ^ t2 >> 6U); + uint64_t s0 = (t15 << 63U | t15 >> 1U) ^ ((t15 << 56U | t15 >> 8U) ^ t15 >> 7U); ws[i] = s1 + t7 + s0 + t16;); }); KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, + 0U, + 8U, + 1U, uint64_t *os = hash; uint64_t x = hash[i] + hash_old[i]; os[i] = x;); } -void Hacl_SHA2_Scalar32_sha512_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st) +void Hacl_Hash_SHA2_sha512_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st) { - uint32_t blocks = len / (uint32_t)128U; - for (uint32_t i = (uint32_t)0U; i < blocks; i++) + uint32_t blocks = len / 128U; + for (uint32_t i = 0U; i < blocks; i++) { uint8_t *b0 = b; - uint8_t *mb = b0 + i * (uint32_t)128U; + uint8_t *mb = b0 + i * 128U; sha512_update(mb, st); } } void -Hacl_SHA2_Scalar32_sha512_update_last( +Hacl_Hash_SHA2_sha512_update_last( FStar_UInt128_uint128 totlen, uint32_t len, uint8_t *b, @@ -406,25 +366,25 @@ Hacl_SHA2_Scalar32_sha512_update_last( ) { uint32_t blocks; - if (len + (uint32_t)16U + (uint32_t)1U <= (uint32_t)128U) + if (len + 16U + 1U <= 128U) { - blocks = (uint32_t)1U; + blocks = 1U; } else { - blocks = (uint32_t)2U; + blocks = 2U; } - uint32_t fin = blocks * (uint32_t)128U; + uint32_t fin = blocks * 128U; uint8_t last[256U] = { 0U }; uint8_t totlen_buf[16U] = { 0U }; - FStar_UInt128_uint128 total_len_bits = FStar_UInt128_shift_left(totlen, (uint32_t)3U); + FStar_UInt128_uint128 total_len_bits = FStar_UInt128_shift_left(totlen, 3U); store128_be(totlen_buf, total_len_bits); uint8_t *b0 = b; memcpy(last, b0, len * sizeof (uint8_t)); - last[len] = (uint8_t)0x80U; - memcpy(last + fin - (uint32_t)16U, totlen_buf, (uint32_t)16U * sizeof (uint8_t)); + last[len] = 0x80U; + memcpy(last + fin - 16U, totlen_buf, 16U * sizeof (uint8_t)); uint8_t *last00 = last; - uint8_t *last10 = last + (uint32_t)128U; + uint8_t *last10 = last + 128U; uint8_t *l0 = last00; uint8_t *l1 = last10; uint8_t *lb0 = l0; @@ -432,76 +392,68 @@ Hacl_SHA2_Scalar32_sha512_update_last( uint8_t *last0 = lb0; uint8_t *last1 = lb1; sha512_update(last0, hash); - if (blocks > (uint32_t)1U) + if (blocks > 1U) { sha512_update(last1, hash); return; } } -void Hacl_SHA2_Scalar32_sha512_finish(uint64_t *st, uint8_t *h) +void Hacl_Hash_SHA2_sha512_finish(uint64_t *st, uint8_t *h) { uint8_t hbuf[64U] = { 0U }; - KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, - store64_be(hbuf + i * (uint32_t)8U, st[i]);); - memcpy(h, hbuf, (uint32_t)64U * sizeof (uint8_t)); + KRML_MAYBE_FOR8(i, 0U, 8U, 1U, store64_be(hbuf + i * 8U, st[i]);); + memcpy(h, hbuf, 64U * sizeof (uint8_t)); } -void Hacl_SHA2_Scalar32_sha384_init(uint64_t *hash) +void Hacl_Hash_SHA2_sha384_init(uint64_t *hash) { KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, + 0U, + 8U, + 1U, uint64_t *os = hash; - uint64_t x = Hacl_Impl_SHA2_Generic_h384[i]; + uint64_t x = Hacl_Hash_SHA2_h384[i]; os[i] = x;); } -void Hacl_SHA2_Scalar32_sha384_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st) +void Hacl_Hash_SHA2_sha384_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st) { - Hacl_SHA2_Scalar32_sha512_update_nblocks(len, b, st); + Hacl_Hash_SHA2_sha512_update_nblocks(len, b, st); } void -Hacl_SHA2_Scalar32_sha384_update_last( +Hacl_Hash_SHA2_sha384_update_last( FStar_UInt128_uint128 totlen, uint32_t len, uint8_t *b, uint64_t *st ) { - Hacl_SHA2_Scalar32_sha512_update_last(totlen, len, b, st); + Hacl_Hash_SHA2_sha512_update_last(totlen, len, b, st); } -void Hacl_SHA2_Scalar32_sha384_finish(uint64_t *st, uint8_t *h) +void Hacl_Hash_SHA2_sha384_finish(uint64_t *st, uint8_t *h) { uint8_t hbuf[64U] = { 0U }; - KRML_MAYBE_FOR8(i, - (uint32_t)0U, - (uint32_t)8U, - (uint32_t)1U, - store64_be(hbuf + i * (uint32_t)8U, st[i]);); - memcpy(h, hbuf, (uint32_t)48U * sizeof (uint8_t)); + KRML_MAYBE_FOR8(i, 0U, 8U, 1U, store64_be(hbuf + i * 8U, st[i]);); + memcpy(h, hbuf, 48U * sizeof (uint8_t)); } /** Allocate initial state for the SHA2_256 hash. The state is to be freed by calling `free_256`. */ -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_create_in_256(void) +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA2_malloc_256(void) { - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); - uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint32_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC(8U, sizeof (uint32_t)); Hacl_Streaming_MD_state_32 - s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; Hacl_Streaming_MD_state_32 *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); p[0U] = s; - Hacl_SHA2_Scalar32_sha256_init(block_state); + Hacl_Hash_SHA2_sha256_init(block_state); return p; } @@ -511,16 +463,16 @@ The state is to be freed by calling `free_256`. Cloning the state this way is useful, for instance, if your control-flow diverges and you need to feed more (different) data into the hash in each branch. */ -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_copy_256(Hacl_Streaming_MD_state_32 *s0) +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA2_copy_256(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s0; + Hacl_Streaming_MD_state_32 scrut = *state; uint32_t *block_state0 = scrut.block_state; uint8_t *buf0 = scrut.buf; uint64_t total_len0 = scrut.total_len; - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); - memcpy(buf, buf0, (uint32_t)64U * sizeof (uint8_t)); - uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint32_t)); - memcpy(block_state, block_state0, (uint32_t)8U * sizeof (uint32_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + memcpy(buf, buf0, 64U * sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC(8U, sizeof (uint32_t)); + memcpy(block_state, block_state0, 8U * sizeof (uint32_t)); Hacl_Streaming_MD_state_32 s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; Hacl_Streaming_MD_state_32 @@ -532,54 +484,54 @@ Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_copy_256(Hacl_Streaming_MD_state /** Reset an existing state to the initial hash state with empty data. */ -void Hacl_Streaming_SHA2_init_256(Hacl_Streaming_MD_state_32 *s) +void Hacl_Hash_SHA2_reset_256(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *state; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; - Hacl_SHA2_Scalar32_sha256_init(block_state); + Hacl_Hash_SHA2_sha256_init(block_state); Hacl_Streaming_MD_state_32 - tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - s[0U] = tmp; + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; + state[0U] = tmp; } static inline Hacl_Streaming_Types_error_code -update_224_256(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) +update_224_256(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk_len) { - Hacl_Streaming_MD_state_32 s = *p; + Hacl_Streaming_MD_state_32 s = *state; uint64_t total_len = s.total_len; - if ((uint64_t)len > (uint64_t)2305843009213693951U - total_len) + if ((uint64_t)chunk_len > 2305843009213693951ULL - total_len) { return Hacl_Streaming_Types_MaximumLengthExceeded; } uint32_t sz; - if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) { - sz = (uint32_t)64U; + sz = 64U; } else { - sz = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + sz = (uint32_t)(total_len % (uint64_t)64U); } - if (len <= (uint32_t)64U - sz) + if (chunk_len <= 64U - sz) { - Hacl_Streaming_MD_state_32 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } uint8_t *buf2 = buf + sz1; - memcpy(buf2, data, len * sizeof (uint8_t)); - uint64_t total_len2 = total_len1 + (uint64_t)len; - *p + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state = ( (Hacl_Streaming_MD_state_32){ @@ -589,76 +541,74 @@ update_224_256(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) } ); } - else if (sz == (uint32_t)0U) + else if (sz == 0U) { - Hacl_Streaming_MD_state_32 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_SHA2_Scalar32_sha256_update_nblocks((uint32_t)64U, buf, block_state1); + Hacl_Hash_SHA2_sha256_update_nblocks(64U, buf, block_state1); } uint32_t ite; - if ((uint64_t)len % (uint64_t)(uint32_t)64U == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + if ((uint64_t)chunk_len % (uint64_t)64U == 0ULL && (uint64_t)chunk_len > 0ULL) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = (uint32_t)((uint64_t)len % (uint64_t)(uint32_t)64U); + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)64U); } - uint32_t n_blocks = (len - ite) / (uint32_t)64U; - uint32_t data1_len = n_blocks * (uint32_t)64U; - uint32_t data2_len = len - data1_len; - uint8_t *data1 = data; - uint8_t *data2 = data + data1_len; - Hacl_SHA2_Scalar32_sha256_update_nblocks(data1_len / (uint32_t)64U * (uint32_t)64U, - data1, - block_state1); + uint32_t n_blocks = (chunk_len - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; + Hacl_Hash_SHA2_sha256_update_nblocks(data1_len / 64U * 64U, data1, block_state1); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *p + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)len + .total_len = total_len1 + (uint64_t)chunk_len } ); } else { - uint32_t diff = (uint32_t)64U - sz; - uint8_t *data1 = data; - uint8_t *data2 = data + diff; - Hacl_Streaming_MD_state_32 s1 = *p; + uint32_t diff = 64U - sz; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Streaming_MD_state_32 s1 = *state; uint32_t *block_state10 = s1.block_state; uint8_t *buf0 = s1.buf; uint64_t total_len10 = s1.total_len; uint32_t sz10; - if (total_len10 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len10 > (uint64_t)0U) + if (total_len10 % (uint64_t)64U == 0ULL && total_len10 > 0ULL) { - sz10 = (uint32_t)64U; + sz10 = 64U; } else { - sz10 = (uint32_t)(total_len10 % (uint64_t)(uint32_t)64U); + sz10 = (uint32_t)(total_len10 % (uint64_t)64U); } uint8_t *buf2 = buf0 + sz10; - memcpy(buf2, data1, diff * sizeof (uint8_t)); + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *p + *state = ( (Hacl_Streaming_MD_state_32){ @@ -667,55 +617,48 @@ update_224_256(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) .total_len = total_len2 } ); - Hacl_Streaming_MD_state_32 s10 = *p; + Hacl_Streaming_MD_state_32 s10 = *state; uint32_t *block_state1 = s10.block_state; uint8_t *buf = s10.buf; uint64_t total_len1 = s10.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)64U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)64U; + sz1 = 64U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + sz1 = (uint32_t)(total_len1 % (uint64_t)64U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_SHA2_Scalar32_sha256_update_nblocks((uint32_t)64U, buf, block_state1); + Hacl_Hash_SHA2_sha256_update_nblocks(64U, buf, block_state1); } uint32_t ite; if - ( - (uint64_t)(len - diff) - % (uint64_t)(uint32_t)64U - == (uint64_t)0U - && (uint64_t)(len - diff) > (uint64_t)0U - ) + ((uint64_t)(chunk_len - diff) % (uint64_t)64U == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)(uint32_t)64U); + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)64U); } - uint32_t n_blocks = (len - diff - ite) / (uint32_t)64U; - uint32_t data1_len = n_blocks * (uint32_t)64U; - uint32_t data2_len = len - diff - data1_len; - uint8_t *data11 = data2; - uint8_t *data21 = data2 + data1_len; - Hacl_SHA2_Scalar32_sha256_update_nblocks(data1_len / (uint32_t)64U * (uint32_t)64U, - data11, - block_state1); + uint32_t n_blocks = (chunk_len - diff - ite) / 64U; + uint32_t data1_len = n_blocks * 64U; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; + Hacl_Hash_SHA2_sha256_update_nblocks(data1_len / 64U * 64U, data1, block_state1); uint8_t *dst = buf; - memcpy(dst, data21, data2_len * sizeof (uint8_t)); - *p + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)(len - diff) + .total_len = total_len1 + (uint64_t)(chunk_len - diff) } ); } @@ -725,209 +668,203 @@ update_224_256(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) /** Feed an arbitrary amount of data into the hash. This function returns 0 for success, or 1 if the combined length of all of the data passed to `update_256` -(since the last call to `init_256`) exceeds 2^61-1 bytes. +(since the last call to `reset_256`) exceeds 2^61-1 bytes. This function is identical to the update function for SHA2_224. */ Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_256( - Hacl_Streaming_MD_state_32 *p, +Hacl_Hash_SHA2_update_256( + Hacl_Streaming_MD_state_32 *state, uint8_t *input, uint32_t input_len ) { - return update_224_256(p, input, input_len); + return update_224_256(state, input, input_len); } /** -Write the resulting hash into `dst`, an array of 32 bytes. The state remains -valid after a call to `finish_256`, meaning the user may feed more data into -the hash via `update_256`. (The finish_256 function operates on an internal copy of +Write the resulting hash into `output`, an array of 32 bytes. The state remains +valid after a call to `digest_256`, meaning the user may feed more data into +the hash via `update_256`. (The digest_256 function operates on an internal copy of the state and therefore does not invalidate the client-held state `p`.) */ -void Hacl_Streaming_SHA2_finish_256(Hacl_Streaming_MD_state_32 *p, uint8_t *dst) +void Hacl_Hash_SHA2_digest_256(Hacl_Streaming_MD_state_32 *state, uint8_t *output) { - Hacl_Streaming_MD_state_32 scrut = *p; + Hacl_Streaming_MD_state_32 scrut = *state; uint32_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; uint32_t r; - if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) { - r = (uint32_t)64U; + r = 64U; } else { - r = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + r = (uint32_t)(total_len % (uint64_t)64U); } uint8_t *buf_1 = buf_; uint32_t tmp_block_state[8U] = { 0U }; - memcpy(tmp_block_state, block_state, (uint32_t)8U * sizeof (uint32_t)); + memcpy(tmp_block_state, block_state, 8U * sizeof (uint32_t)); uint32_t ite; - if (r % (uint32_t)64U == (uint32_t)0U && r > (uint32_t)0U) + if (r % 64U == 0U && r > 0U) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = r % (uint32_t)64U; + ite = r % 64U; } uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; - Hacl_SHA2_Scalar32_sha256_update_nblocks((uint32_t)0U, buf_multi, tmp_block_state); + Hacl_Hash_SHA2_sha256_update_nblocks(0U, buf_multi, tmp_block_state); uint64_t prev_len_last = total_len - (uint64_t)r; - Hacl_SHA2_Scalar32_sha256_update_last(prev_len_last + (uint64_t)r, - r, - buf_last, - tmp_block_state); - Hacl_SHA2_Scalar32_sha256_finish(tmp_block_state, dst); + Hacl_Hash_SHA2_sha256_update_last(prev_len_last + (uint64_t)r, r, buf_last, tmp_block_state); + Hacl_Hash_SHA2_sha256_finish(tmp_block_state, output); } /** -Free a state allocated with `create_in_256`. +Free a state allocated with `malloc_256`. This function is identical to the free function for SHA2_224. */ -void Hacl_Streaming_SHA2_free_256(Hacl_Streaming_MD_state_32 *s) +void Hacl_Hash_SHA2_free_256(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *state; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; KRML_HOST_FREE(block_state); KRML_HOST_FREE(buf); - KRML_HOST_FREE(s); + KRML_HOST_FREE(state); } /** -Hash `input`, of len `input_len`, into `dst`, an array of 32 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 32 bytes. */ -void Hacl_Streaming_SHA2_hash_256(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_SHA2_hash_256(uint8_t *output, uint8_t *input, uint32_t input_len) { uint8_t *ib = input; - uint8_t *rb = dst; + uint8_t *rb = output; uint32_t st[8U] = { 0U }; - Hacl_SHA2_Scalar32_sha256_init(st); - uint32_t rem = input_len % (uint32_t)64U; + Hacl_Hash_SHA2_sha256_init(st); + uint32_t rem = input_len % 64U; uint64_t len_ = (uint64_t)input_len; - Hacl_SHA2_Scalar32_sha256_update_nblocks(input_len, ib, st); - uint32_t rem1 = input_len % (uint32_t)64U; + Hacl_Hash_SHA2_sha256_update_nblocks(input_len, ib, st); + uint32_t rem1 = input_len % 64U; uint8_t *b0 = ib; uint8_t *lb = b0 + input_len - rem1; - Hacl_SHA2_Scalar32_sha256_update_last(len_, rem, lb, st); - Hacl_SHA2_Scalar32_sha256_finish(st, rb); + Hacl_Hash_SHA2_sha256_update_last(len_, rem, lb, st); + Hacl_Hash_SHA2_sha256_finish(st, rb); } -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_create_in_224(void) +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA2_malloc_224(void) { - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); - uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint32_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(64U, sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC(8U, sizeof (uint32_t)); Hacl_Streaming_MD_state_32 - s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; Hacl_Streaming_MD_state_32 *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); p[0U] = s; - Hacl_SHA2_Scalar32_sha224_init(block_state); + Hacl_Hash_SHA2_sha224_init(block_state); return p; } -void Hacl_Streaming_SHA2_init_224(Hacl_Streaming_MD_state_32 *s) +void Hacl_Hash_SHA2_reset_224(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_MD_state_32 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *state; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; - Hacl_SHA2_Scalar32_sha224_init(block_state); + Hacl_Hash_SHA2_sha224_init(block_state); Hacl_Streaming_MD_state_32 - tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - s[0U] = tmp; + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; + state[0U] = tmp; } Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_224( - Hacl_Streaming_MD_state_32 *p, +Hacl_Hash_SHA2_update_224( + Hacl_Streaming_MD_state_32 *state, uint8_t *input, uint32_t input_len ) { - return update_224_256(p, input, input_len); + return update_224_256(state, input, input_len); } /** -Write the resulting hash into `dst`, an array of 28 bytes. The state remains -valid after a call to `finish_224`, meaning the user may feed more data into +Write the resulting hash into `output`, an array of 28 bytes. The state remains +valid after a call to `digest_224`, meaning the user may feed more data into the hash via `update_224`. */ -void Hacl_Streaming_SHA2_finish_224(Hacl_Streaming_MD_state_32 *p, uint8_t *dst) +void Hacl_Hash_SHA2_digest_224(Hacl_Streaming_MD_state_32 *state, uint8_t *output) { - Hacl_Streaming_MD_state_32 scrut = *p; + Hacl_Streaming_MD_state_32 scrut = *state; uint32_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; uint32_t r; - if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)64U == 0ULL && total_len > 0ULL) { - r = (uint32_t)64U; + r = 64U; } else { - r = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + r = (uint32_t)(total_len % (uint64_t)64U); } uint8_t *buf_1 = buf_; uint32_t tmp_block_state[8U] = { 0U }; - memcpy(tmp_block_state, block_state, (uint32_t)8U * sizeof (uint32_t)); + memcpy(tmp_block_state, block_state, 8U * sizeof (uint32_t)); uint32_t ite; - if (r % (uint32_t)64U == (uint32_t)0U && r > (uint32_t)0U) + if (r % 64U == 0U && r > 0U) { - ite = (uint32_t)64U; + ite = 64U; } else { - ite = r % (uint32_t)64U; + ite = r % 64U; } uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; - sha224_update_nblocks((uint32_t)0U, buf_multi, tmp_block_state); + Hacl_Hash_SHA2_sha224_update_nblocks(0U, buf_multi, tmp_block_state); uint64_t prev_len_last = total_len - (uint64_t)r; - Hacl_SHA2_Scalar32_sha224_update_last(prev_len_last + (uint64_t)r, - r, - buf_last, - tmp_block_state); - Hacl_SHA2_Scalar32_sha224_finish(tmp_block_state, dst); + Hacl_Hash_SHA2_sha224_update_last(prev_len_last + (uint64_t)r, r, buf_last, tmp_block_state); + Hacl_Hash_SHA2_sha224_finish(tmp_block_state, output); } -void Hacl_Streaming_SHA2_free_224(Hacl_Streaming_MD_state_32 *p) +void Hacl_Hash_SHA2_free_224(Hacl_Streaming_MD_state_32 *state) { - Hacl_Streaming_SHA2_free_256(p); + Hacl_Hash_SHA2_free_256(state); } /** -Hash `input`, of len `input_len`, into `dst`, an array of 28 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 28 bytes. */ -void Hacl_Streaming_SHA2_hash_224(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_SHA2_hash_224(uint8_t *output, uint8_t *input, uint32_t input_len) { uint8_t *ib = input; - uint8_t *rb = dst; + uint8_t *rb = output; uint32_t st[8U] = { 0U }; - Hacl_SHA2_Scalar32_sha224_init(st); - uint32_t rem = input_len % (uint32_t)64U; + Hacl_Hash_SHA2_sha224_init(st); + uint32_t rem = input_len % 64U; uint64_t len_ = (uint64_t)input_len; - sha224_update_nblocks(input_len, ib, st); - uint32_t rem1 = input_len % (uint32_t)64U; + Hacl_Hash_SHA2_sha224_update_nblocks(input_len, ib, st); + uint32_t rem1 = input_len % 64U; uint8_t *b0 = ib; uint8_t *lb = b0 + input_len - rem1; - Hacl_SHA2_Scalar32_sha224_update_last(len_, rem, lb, st); - Hacl_SHA2_Scalar32_sha224_finish(st, rb); + Hacl_Hash_SHA2_sha224_update_last(len_, rem, lb, st); + Hacl_Hash_SHA2_sha224_finish(st, rb); } -Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_create_in_512(void) +Hacl_Streaming_MD_state_64 *Hacl_Hash_SHA2_malloc_512(void) { - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)128U, sizeof (uint8_t)); - uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint64_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(128U, sizeof (uint8_t)); + uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC(8U, sizeof (uint64_t)); Hacl_Streaming_MD_state_64 - s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; Hacl_Streaming_MD_state_64 *p = (Hacl_Streaming_MD_state_64 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_64)); p[0U] = s; - Hacl_SHA2_Scalar32_sha512_init(block_state); + Hacl_Hash_SHA2_sha512_init(block_state); return p; } @@ -937,16 +874,16 @@ The state is to be freed by calling `free_512`. Cloning the state this way is useful, for instance, if your control-flow diverges and you need to feed more (different) data into the hash in each branch. */ -Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_copy_512(Hacl_Streaming_MD_state_64 *s0) +Hacl_Streaming_MD_state_64 *Hacl_Hash_SHA2_copy_512(Hacl_Streaming_MD_state_64 *state) { - Hacl_Streaming_MD_state_64 scrut = *s0; + Hacl_Streaming_MD_state_64 scrut = *state; uint64_t *block_state0 = scrut.block_state; uint8_t *buf0 = scrut.buf; uint64_t total_len0 = scrut.total_len; - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)128U, sizeof (uint8_t)); - memcpy(buf, buf0, (uint32_t)128U * sizeof (uint8_t)); - uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint64_t)); - memcpy(block_state, block_state0, (uint32_t)8U * sizeof (uint64_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(128U, sizeof (uint8_t)); + memcpy(buf, buf0, 128U * sizeof (uint8_t)); + uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC(8U, sizeof (uint64_t)); + memcpy(block_state, block_state0, 8U * sizeof (uint64_t)); Hacl_Streaming_MD_state_64 s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; Hacl_Streaming_MD_state_64 @@ -955,54 +892,54 @@ Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_copy_512(Hacl_Streaming_MD_state return p; } -void Hacl_Streaming_SHA2_init_512(Hacl_Streaming_MD_state_64 *s) +void Hacl_Hash_SHA2_reset_512(Hacl_Streaming_MD_state_64 *state) { - Hacl_Streaming_MD_state_64 scrut = *s; + Hacl_Streaming_MD_state_64 scrut = *state; uint8_t *buf = scrut.buf; uint64_t *block_state = scrut.block_state; - Hacl_SHA2_Scalar32_sha512_init(block_state); + Hacl_Hash_SHA2_sha512_init(block_state); Hacl_Streaming_MD_state_64 - tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - s[0U] = tmp; + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; + state[0U] = tmp; } static inline Hacl_Streaming_Types_error_code -update_384_512(Hacl_Streaming_MD_state_64 *p, uint8_t *data, uint32_t len) +update_384_512(Hacl_Streaming_MD_state_64 *state, uint8_t *chunk, uint32_t chunk_len) { - Hacl_Streaming_MD_state_64 s = *p; + Hacl_Streaming_MD_state_64 s = *state; uint64_t total_len = s.total_len; - if ((uint64_t)len > (uint64_t)18446744073709551615U - total_len) + if ((uint64_t)chunk_len > 18446744073709551615ULL - total_len) { return Hacl_Streaming_Types_MaximumLengthExceeded; } uint32_t sz; - if (total_len % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)128U == 0ULL && total_len > 0ULL) { - sz = (uint32_t)128U; + sz = 128U; } else { - sz = (uint32_t)(total_len % (uint64_t)(uint32_t)128U); + sz = (uint32_t)(total_len % (uint64_t)128U); } - if (len <= (uint32_t)128U - sz) + if (chunk_len <= 128U - sz) { - Hacl_Streaming_MD_state_64 s1 = *p; + Hacl_Streaming_MD_state_64 s1 = *state; uint64_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)128U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)128U; + sz1 = 128U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)128U); + sz1 = (uint32_t)(total_len1 % (uint64_t)128U); } uint8_t *buf2 = buf + sz1; - memcpy(buf2, data, len * sizeof (uint8_t)); - uint64_t total_len2 = total_len1 + (uint64_t)len; - *p + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state = ( (Hacl_Streaming_MD_state_64){ @@ -1012,76 +949,74 @@ update_384_512(Hacl_Streaming_MD_state_64 *p, uint8_t *data, uint32_t len) } ); } - else if (sz == (uint32_t)0U) + else if (sz == 0U) { - Hacl_Streaming_MD_state_64 s1 = *p; + Hacl_Streaming_MD_state_64 s1 = *state; uint64_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)128U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)128U; + sz1 = 128U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)128U); + sz1 = (uint32_t)(total_len1 % (uint64_t)128U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_SHA2_Scalar32_sha512_update_nblocks((uint32_t)128U, buf, block_state1); + Hacl_Hash_SHA2_sha512_update_nblocks(128U, buf, block_state1); } uint32_t ite; - if ((uint64_t)len % (uint64_t)(uint32_t)128U == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + if ((uint64_t)chunk_len % (uint64_t)128U == 0ULL && (uint64_t)chunk_len > 0ULL) { - ite = (uint32_t)128U; + ite = 128U; } else { - ite = (uint32_t)((uint64_t)len % (uint64_t)(uint32_t)128U); + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)128U); } - uint32_t n_blocks = (len - ite) / (uint32_t)128U; - uint32_t data1_len = n_blocks * (uint32_t)128U; - uint32_t data2_len = len - data1_len; - uint8_t *data1 = data; - uint8_t *data2 = data + data1_len; - Hacl_SHA2_Scalar32_sha512_update_nblocks(data1_len / (uint32_t)128U * (uint32_t)128U, - data1, - block_state1); + uint32_t n_blocks = (chunk_len - ite) / 128U; + uint32_t data1_len = n_blocks * 128U; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; + Hacl_Hash_SHA2_sha512_update_nblocks(data1_len / 128U * 128U, data1, block_state1); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *p + *state = ( (Hacl_Streaming_MD_state_64){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)len + .total_len = total_len1 + (uint64_t)chunk_len } ); } else { - uint32_t diff = (uint32_t)128U - sz; - uint8_t *data1 = data; - uint8_t *data2 = data + diff; - Hacl_Streaming_MD_state_64 s1 = *p; + uint32_t diff = 128U - sz; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Streaming_MD_state_64 s1 = *state; uint64_t *block_state10 = s1.block_state; uint8_t *buf0 = s1.buf; uint64_t total_len10 = s1.total_len; uint32_t sz10; - if (total_len10 % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len10 > (uint64_t)0U) + if (total_len10 % (uint64_t)128U == 0ULL && total_len10 > 0ULL) { - sz10 = (uint32_t)128U; + sz10 = 128U; } else { - sz10 = (uint32_t)(total_len10 % (uint64_t)(uint32_t)128U); + sz10 = (uint32_t)(total_len10 % (uint64_t)128U); } uint8_t *buf2 = buf0 + sz10; - memcpy(buf2, data1, diff * sizeof (uint8_t)); + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *p + *state = ( (Hacl_Streaming_MD_state_64){ @@ -1090,55 +1025,48 @@ update_384_512(Hacl_Streaming_MD_state_64 *p, uint8_t *data, uint32_t len) .total_len = total_len2 } ); - Hacl_Streaming_MD_state_64 s10 = *p; + Hacl_Streaming_MD_state_64 s10 = *state; uint64_t *block_state1 = s10.block_state; uint8_t *buf = s10.buf; uint64_t total_len1 = s10.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)128U == 0ULL && total_len1 > 0ULL) { - sz1 = (uint32_t)128U; + sz1 = 128U; } else { - sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)128U); + sz1 = (uint32_t)(total_len1 % (uint64_t)128U); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { - Hacl_SHA2_Scalar32_sha512_update_nblocks((uint32_t)128U, buf, block_state1); + Hacl_Hash_SHA2_sha512_update_nblocks(128U, buf, block_state1); } uint32_t ite; if - ( - (uint64_t)(len - diff) - % (uint64_t)(uint32_t)128U - == (uint64_t)0U - && (uint64_t)(len - diff) > (uint64_t)0U - ) + ((uint64_t)(chunk_len - diff) % (uint64_t)128U == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL) { - ite = (uint32_t)128U; + ite = 128U; } else { - ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)(uint32_t)128U); + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)128U); } - uint32_t n_blocks = (len - diff - ite) / (uint32_t)128U; - uint32_t data1_len = n_blocks * (uint32_t)128U; - uint32_t data2_len = len - diff - data1_len; - uint8_t *data11 = data2; - uint8_t *data21 = data2 + data1_len; - Hacl_SHA2_Scalar32_sha512_update_nblocks(data1_len / (uint32_t)128U * (uint32_t)128U, - data11, - block_state1); + uint32_t n_blocks = (chunk_len - diff - ite) / 128U; + uint32_t data1_len = n_blocks * 128U; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; + Hacl_Hash_SHA2_sha512_update_nblocks(data1_len / 128U * 128U, data1, block_state1); uint8_t *dst = buf; - memcpy(dst, data21, data2_len * sizeof (uint8_t)); - *p + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state = ( (Hacl_Streaming_MD_state_64){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)(len - diff) + .total_len = total_len1 + (uint64_t)(chunk_len - diff) } ); } @@ -1148,198 +1076,198 @@ update_384_512(Hacl_Streaming_MD_state_64 *p, uint8_t *data, uint32_t len) /** Feed an arbitrary amount of data into the hash. This function returns 0 for success, or 1 if the combined length of all of the data passed to `update_512` -(since the last call to `init_512`) exceeds 2^125-1 bytes. +(since the last call to `reset_512`) exceeds 2^125-1 bytes. This function is identical to the update function for SHA2_384. */ Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_512( - Hacl_Streaming_MD_state_64 *p, +Hacl_Hash_SHA2_update_512( + Hacl_Streaming_MD_state_64 *state, uint8_t *input, uint32_t input_len ) { - return update_384_512(p, input, input_len); + return update_384_512(state, input, input_len); } /** -Write the resulting hash into `dst`, an array of 64 bytes. The state remains -valid after a call to `finish_512`, meaning the user may feed more data into -the hash via `update_512`. (The finish_512 function operates on an internal copy of +Write the resulting hash into `output`, an array of 64 bytes. The state remains +valid after a call to `digest_512`, meaning the user may feed more data into +the hash via `update_512`. (The digest_512 function operates on an internal copy of the state and therefore does not invalidate the client-held state `p`.) */ -void Hacl_Streaming_SHA2_finish_512(Hacl_Streaming_MD_state_64 *p, uint8_t *dst) +void Hacl_Hash_SHA2_digest_512(Hacl_Streaming_MD_state_64 *state, uint8_t *output) { - Hacl_Streaming_MD_state_64 scrut = *p; + Hacl_Streaming_MD_state_64 scrut = *state; uint64_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; uint32_t r; - if (total_len % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)128U == 0ULL && total_len > 0ULL) { - r = (uint32_t)128U; + r = 128U; } else { - r = (uint32_t)(total_len % (uint64_t)(uint32_t)128U); + r = (uint32_t)(total_len % (uint64_t)128U); } uint8_t *buf_1 = buf_; uint64_t tmp_block_state[8U] = { 0U }; - memcpy(tmp_block_state, block_state, (uint32_t)8U * sizeof (uint64_t)); + memcpy(tmp_block_state, block_state, 8U * sizeof (uint64_t)); uint32_t ite; - if (r % (uint32_t)128U == (uint32_t)0U && r > (uint32_t)0U) + if (r % 128U == 0U && r > 0U) { - ite = (uint32_t)128U; + ite = 128U; } else { - ite = r % (uint32_t)128U; + ite = r % 128U; } uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; - Hacl_SHA2_Scalar32_sha512_update_nblocks((uint32_t)0U, buf_multi, tmp_block_state); + Hacl_Hash_SHA2_sha512_update_nblocks(0U, buf_multi, tmp_block_state); uint64_t prev_len_last = total_len - (uint64_t)r; - Hacl_SHA2_Scalar32_sha512_update_last(FStar_UInt128_add(FStar_UInt128_uint64_to_uint128(prev_len_last), + Hacl_Hash_SHA2_sha512_update_last(FStar_UInt128_add(FStar_UInt128_uint64_to_uint128(prev_len_last), FStar_UInt128_uint64_to_uint128((uint64_t)r)), r, buf_last, tmp_block_state); - Hacl_SHA2_Scalar32_sha512_finish(tmp_block_state, dst); + Hacl_Hash_SHA2_sha512_finish(tmp_block_state, output); } /** -Free a state allocated with `create_in_512`. +Free a state allocated with `malloc_512`. This function is identical to the free function for SHA2_384. */ -void Hacl_Streaming_SHA2_free_512(Hacl_Streaming_MD_state_64 *s) +void Hacl_Hash_SHA2_free_512(Hacl_Streaming_MD_state_64 *state) { - Hacl_Streaming_MD_state_64 scrut = *s; + Hacl_Streaming_MD_state_64 scrut = *state; uint8_t *buf = scrut.buf; uint64_t *block_state = scrut.block_state; KRML_HOST_FREE(block_state); KRML_HOST_FREE(buf); - KRML_HOST_FREE(s); + KRML_HOST_FREE(state); } /** -Hash `input`, of len `input_len`, into `dst`, an array of 64 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 64 bytes. */ -void Hacl_Streaming_SHA2_hash_512(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_SHA2_hash_512(uint8_t *output, uint8_t *input, uint32_t input_len) { uint8_t *ib = input; - uint8_t *rb = dst; + uint8_t *rb = output; uint64_t st[8U] = { 0U }; - Hacl_SHA2_Scalar32_sha512_init(st); - uint32_t rem = input_len % (uint32_t)128U; + Hacl_Hash_SHA2_sha512_init(st); + uint32_t rem = input_len % 128U; FStar_UInt128_uint128 len_ = FStar_UInt128_uint64_to_uint128((uint64_t)input_len); - Hacl_SHA2_Scalar32_sha512_update_nblocks(input_len, ib, st); - uint32_t rem1 = input_len % (uint32_t)128U; + Hacl_Hash_SHA2_sha512_update_nblocks(input_len, ib, st); + uint32_t rem1 = input_len % 128U; uint8_t *b0 = ib; uint8_t *lb = b0 + input_len - rem1; - Hacl_SHA2_Scalar32_sha512_update_last(len_, rem, lb, st); - Hacl_SHA2_Scalar32_sha512_finish(st, rb); + Hacl_Hash_SHA2_sha512_update_last(len_, rem, lb, st); + Hacl_Hash_SHA2_sha512_finish(st, rb); } -Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_create_in_384(void) +Hacl_Streaming_MD_state_64 *Hacl_Hash_SHA2_malloc_384(void) { - uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)128U, sizeof (uint8_t)); - uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint64_t)); + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC(128U, sizeof (uint8_t)); + uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC(8U, sizeof (uint64_t)); Hacl_Streaming_MD_state_64 - s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; Hacl_Streaming_MD_state_64 *p = (Hacl_Streaming_MD_state_64 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_64)); p[0U] = s; - Hacl_SHA2_Scalar32_sha384_init(block_state); + Hacl_Hash_SHA2_sha384_init(block_state); return p; } -void Hacl_Streaming_SHA2_init_384(Hacl_Streaming_MD_state_64 *s) +void Hacl_Hash_SHA2_reset_384(Hacl_Streaming_MD_state_64 *state) { - Hacl_Streaming_MD_state_64 scrut = *s; + Hacl_Streaming_MD_state_64 scrut = *state; uint8_t *buf = scrut.buf; uint64_t *block_state = scrut.block_state; - Hacl_SHA2_Scalar32_sha384_init(block_state); + Hacl_Hash_SHA2_sha384_init(block_state); Hacl_Streaming_MD_state_64 - tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - s[0U] = tmp; + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; + state[0U] = tmp; } Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_384( - Hacl_Streaming_MD_state_64 *p, +Hacl_Hash_SHA2_update_384( + Hacl_Streaming_MD_state_64 *state, uint8_t *input, uint32_t input_len ) { - return update_384_512(p, input, input_len); + return update_384_512(state, input, input_len); } /** -Write the resulting hash into `dst`, an array of 48 bytes. The state remains -valid after a call to `finish_384`, meaning the user may feed more data into +Write the resulting hash into `output`, an array of 48 bytes. The state remains +valid after a call to `digest_384`, meaning the user may feed more data into the hash via `update_384`. */ -void Hacl_Streaming_SHA2_finish_384(Hacl_Streaming_MD_state_64 *p, uint8_t *dst) +void Hacl_Hash_SHA2_digest_384(Hacl_Streaming_MD_state_64 *state, uint8_t *output) { - Hacl_Streaming_MD_state_64 scrut = *p; + Hacl_Streaming_MD_state_64 scrut = *state; uint64_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; uint32_t r; - if (total_len % (uint64_t)(uint32_t)128U == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)128U == 0ULL && total_len > 0ULL) { - r = (uint32_t)128U; + r = 128U; } else { - r = (uint32_t)(total_len % (uint64_t)(uint32_t)128U); + r = (uint32_t)(total_len % (uint64_t)128U); } uint8_t *buf_1 = buf_; uint64_t tmp_block_state[8U] = { 0U }; - memcpy(tmp_block_state, block_state, (uint32_t)8U * sizeof (uint64_t)); + memcpy(tmp_block_state, block_state, 8U * sizeof (uint64_t)); uint32_t ite; - if (r % (uint32_t)128U == (uint32_t)0U && r > (uint32_t)0U) + if (r % 128U == 0U && r > 0U) { - ite = (uint32_t)128U; + ite = 128U; } else { - ite = r % (uint32_t)128U; + ite = r % 128U; } uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; - Hacl_SHA2_Scalar32_sha384_update_nblocks((uint32_t)0U, buf_multi, tmp_block_state); + Hacl_Hash_SHA2_sha384_update_nblocks(0U, buf_multi, tmp_block_state); uint64_t prev_len_last = total_len - (uint64_t)r; - Hacl_SHA2_Scalar32_sha384_update_last(FStar_UInt128_add(FStar_UInt128_uint64_to_uint128(prev_len_last), + Hacl_Hash_SHA2_sha384_update_last(FStar_UInt128_add(FStar_UInt128_uint64_to_uint128(prev_len_last), FStar_UInt128_uint64_to_uint128((uint64_t)r)), r, buf_last, tmp_block_state); - Hacl_SHA2_Scalar32_sha384_finish(tmp_block_state, dst); + Hacl_Hash_SHA2_sha384_finish(tmp_block_state, output); } -void Hacl_Streaming_SHA2_free_384(Hacl_Streaming_MD_state_64 *p) +void Hacl_Hash_SHA2_free_384(Hacl_Streaming_MD_state_64 *state) { - Hacl_Streaming_SHA2_free_512(p); + Hacl_Hash_SHA2_free_512(state); } /** -Hash `input`, of len `input_len`, into `dst`, an array of 48 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 48 bytes. */ -void Hacl_Streaming_SHA2_hash_384(uint8_t *input, uint32_t input_len, uint8_t *dst) +void Hacl_Hash_SHA2_hash_384(uint8_t *output, uint8_t *input, uint32_t input_len) { uint8_t *ib = input; - uint8_t *rb = dst; + uint8_t *rb = output; uint64_t st[8U] = { 0U }; - Hacl_SHA2_Scalar32_sha384_init(st); - uint32_t rem = input_len % (uint32_t)128U; + Hacl_Hash_SHA2_sha384_init(st); + uint32_t rem = input_len % 128U; FStar_UInt128_uint128 len_ = FStar_UInt128_uint64_to_uint128((uint64_t)input_len); - Hacl_SHA2_Scalar32_sha384_update_nblocks(input_len, ib, st); - uint32_t rem1 = input_len % (uint32_t)128U; + Hacl_Hash_SHA2_sha384_update_nblocks(input_len, ib, st); + uint32_t rem1 = input_len % 128U; uint8_t *b0 = ib; uint8_t *lb = b0 + input_len - rem1; - Hacl_SHA2_Scalar32_sha384_update_last(len_, rem, lb, st); - Hacl_SHA2_Scalar32_sha384_finish(st, rb); + Hacl_Hash_SHA2_sha384_update_last(len_, rem, lb, st); + Hacl_Hash_SHA2_sha384_finish(st, rb); } diff --git a/Modules/_hacl/Hacl_Hash_SHA2.h b/Modules/_hacl/Hacl_Hash_SHA2.h index a0e731094dfaa5..d8204b504baf82 100644 --- a/Modules/_hacl/Hacl_Hash_SHA2.h +++ b/Modules/_hacl/Hacl_Hash_SHA2.h @@ -39,19 +39,19 @@ extern "C" { #include "Hacl_Streaming_Types.h" -typedef Hacl_Streaming_MD_state_32 Hacl_Streaming_SHA2_state_sha2_224; +typedef Hacl_Streaming_MD_state_32 Hacl_Hash_SHA2_state_t_224; -typedef Hacl_Streaming_MD_state_32 Hacl_Streaming_SHA2_state_sha2_256; +typedef Hacl_Streaming_MD_state_32 Hacl_Hash_SHA2_state_t_256; -typedef Hacl_Streaming_MD_state_64 Hacl_Streaming_SHA2_state_sha2_384; +typedef Hacl_Streaming_MD_state_64 Hacl_Hash_SHA2_state_t_384; -typedef Hacl_Streaming_MD_state_64 Hacl_Streaming_SHA2_state_sha2_512; +typedef Hacl_Streaming_MD_state_64 Hacl_Hash_SHA2_state_t_512; /** Allocate initial state for the SHA2_256 hash. The state is to be freed by calling `free_256`. */ -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_create_in_256(void); +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA2_malloc_256(void); /** Copies the state passed as argument into a newly allocated state (deep copy). @@ -59,73 +59,73 @@ The state is to be freed by calling `free_256`. Cloning the state this way is useful, for instance, if your control-flow diverges and you need to feed more (different) data into the hash in each branch. */ -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_copy_256(Hacl_Streaming_MD_state_32 *s0); +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA2_copy_256(Hacl_Streaming_MD_state_32 *state); /** Reset an existing state to the initial hash state with empty data. */ -void Hacl_Streaming_SHA2_init_256(Hacl_Streaming_MD_state_32 *s); +void Hacl_Hash_SHA2_reset_256(Hacl_Streaming_MD_state_32 *state); /** Feed an arbitrary amount of data into the hash. This function returns 0 for success, or 1 if the combined length of all of the data passed to `update_256` -(since the last call to `init_256`) exceeds 2^61-1 bytes. +(since the last call to `reset_256`) exceeds 2^61-1 bytes. This function is identical to the update function for SHA2_224. */ Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_256( - Hacl_Streaming_MD_state_32 *p, +Hacl_Hash_SHA2_update_256( + Hacl_Streaming_MD_state_32 *state, uint8_t *input, uint32_t input_len ); /** -Write the resulting hash into `dst`, an array of 32 bytes. The state remains -valid after a call to `finish_256`, meaning the user may feed more data into -the hash via `update_256`. (The finish_256 function operates on an internal copy of +Write the resulting hash into `output`, an array of 32 bytes. The state remains +valid after a call to `digest_256`, meaning the user may feed more data into +the hash via `update_256`. (The digest_256 function operates on an internal copy of the state and therefore does not invalidate the client-held state `p`.) */ -void Hacl_Streaming_SHA2_finish_256(Hacl_Streaming_MD_state_32 *p, uint8_t *dst); +void Hacl_Hash_SHA2_digest_256(Hacl_Streaming_MD_state_32 *state, uint8_t *output); /** -Free a state allocated with `create_in_256`. +Free a state allocated with `malloc_256`. This function is identical to the free function for SHA2_224. */ -void Hacl_Streaming_SHA2_free_256(Hacl_Streaming_MD_state_32 *s); +void Hacl_Hash_SHA2_free_256(Hacl_Streaming_MD_state_32 *state); /** -Hash `input`, of len `input_len`, into `dst`, an array of 32 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 32 bytes. */ -void Hacl_Streaming_SHA2_hash_256(uint8_t *input, uint32_t input_len, uint8_t *dst); +void Hacl_Hash_SHA2_hash_256(uint8_t *output, uint8_t *input, uint32_t input_len); -Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_create_in_224(void); +Hacl_Streaming_MD_state_32 *Hacl_Hash_SHA2_malloc_224(void); -void Hacl_Streaming_SHA2_init_224(Hacl_Streaming_MD_state_32 *s); +void Hacl_Hash_SHA2_reset_224(Hacl_Streaming_MD_state_32 *state); Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_224( - Hacl_Streaming_MD_state_32 *p, +Hacl_Hash_SHA2_update_224( + Hacl_Streaming_MD_state_32 *state, uint8_t *input, uint32_t input_len ); /** -Write the resulting hash into `dst`, an array of 28 bytes. The state remains -valid after a call to `finish_224`, meaning the user may feed more data into +Write the resulting hash into `output`, an array of 28 bytes. The state remains +valid after a call to `digest_224`, meaning the user may feed more data into the hash via `update_224`. */ -void Hacl_Streaming_SHA2_finish_224(Hacl_Streaming_MD_state_32 *p, uint8_t *dst); +void Hacl_Hash_SHA2_digest_224(Hacl_Streaming_MD_state_32 *state, uint8_t *output); -void Hacl_Streaming_SHA2_free_224(Hacl_Streaming_MD_state_32 *p); +void Hacl_Hash_SHA2_free_224(Hacl_Streaming_MD_state_32 *state); /** -Hash `input`, of len `input_len`, into `dst`, an array of 28 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 28 bytes. */ -void Hacl_Streaming_SHA2_hash_224(uint8_t *input, uint32_t input_len, uint8_t *dst); +void Hacl_Hash_SHA2_hash_224(uint8_t *output, uint8_t *input, uint32_t input_len); -Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_create_in_512(void); +Hacl_Streaming_MD_state_64 *Hacl_Hash_SHA2_malloc_512(void); /** Copies the state passed as argument into a newly allocated state (deep copy). @@ -133,68 +133,68 @@ The state is to be freed by calling `free_512`. Cloning the state this way is useful, for instance, if your control-flow diverges and you need to feed more (different) data into the hash in each branch. */ -Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_copy_512(Hacl_Streaming_MD_state_64 *s0); +Hacl_Streaming_MD_state_64 *Hacl_Hash_SHA2_copy_512(Hacl_Streaming_MD_state_64 *state); -void Hacl_Streaming_SHA2_init_512(Hacl_Streaming_MD_state_64 *s); +void Hacl_Hash_SHA2_reset_512(Hacl_Streaming_MD_state_64 *state); /** Feed an arbitrary amount of data into the hash. This function returns 0 for success, or 1 if the combined length of all of the data passed to `update_512` -(since the last call to `init_512`) exceeds 2^125-1 bytes. +(since the last call to `reset_512`) exceeds 2^125-1 bytes. This function is identical to the update function for SHA2_384. */ Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_512( - Hacl_Streaming_MD_state_64 *p, +Hacl_Hash_SHA2_update_512( + Hacl_Streaming_MD_state_64 *state, uint8_t *input, uint32_t input_len ); /** -Write the resulting hash into `dst`, an array of 64 bytes. The state remains -valid after a call to `finish_512`, meaning the user may feed more data into -the hash via `update_512`. (The finish_512 function operates on an internal copy of +Write the resulting hash into `output`, an array of 64 bytes. The state remains +valid after a call to `digest_512`, meaning the user may feed more data into +the hash via `update_512`. (The digest_512 function operates on an internal copy of the state and therefore does not invalidate the client-held state `p`.) */ -void Hacl_Streaming_SHA2_finish_512(Hacl_Streaming_MD_state_64 *p, uint8_t *dst); +void Hacl_Hash_SHA2_digest_512(Hacl_Streaming_MD_state_64 *state, uint8_t *output); /** -Free a state allocated with `create_in_512`. +Free a state allocated with `malloc_512`. This function is identical to the free function for SHA2_384. */ -void Hacl_Streaming_SHA2_free_512(Hacl_Streaming_MD_state_64 *s); +void Hacl_Hash_SHA2_free_512(Hacl_Streaming_MD_state_64 *state); /** -Hash `input`, of len `input_len`, into `dst`, an array of 64 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 64 bytes. */ -void Hacl_Streaming_SHA2_hash_512(uint8_t *input, uint32_t input_len, uint8_t *dst); +void Hacl_Hash_SHA2_hash_512(uint8_t *output, uint8_t *input, uint32_t input_len); -Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_create_in_384(void); +Hacl_Streaming_MD_state_64 *Hacl_Hash_SHA2_malloc_384(void); -void Hacl_Streaming_SHA2_init_384(Hacl_Streaming_MD_state_64 *s); +void Hacl_Hash_SHA2_reset_384(Hacl_Streaming_MD_state_64 *state); Hacl_Streaming_Types_error_code -Hacl_Streaming_SHA2_update_384( - Hacl_Streaming_MD_state_64 *p, +Hacl_Hash_SHA2_update_384( + Hacl_Streaming_MD_state_64 *state, uint8_t *input, uint32_t input_len ); /** -Write the resulting hash into `dst`, an array of 48 bytes. The state remains -valid after a call to `finish_384`, meaning the user may feed more data into +Write the resulting hash into `output`, an array of 48 bytes. The state remains +valid after a call to `digest_384`, meaning the user may feed more data into the hash via `update_384`. */ -void Hacl_Streaming_SHA2_finish_384(Hacl_Streaming_MD_state_64 *p, uint8_t *dst); +void Hacl_Hash_SHA2_digest_384(Hacl_Streaming_MD_state_64 *state, uint8_t *output); -void Hacl_Streaming_SHA2_free_384(Hacl_Streaming_MD_state_64 *p); +void Hacl_Hash_SHA2_free_384(Hacl_Streaming_MD_state_64 *state); /** -Hash `input`, of len `input_len`, into `dst`, an array of 48 bytes. +Hash `input`, of len `input_len`, into `output`, an array of 48 bytes. */ -void Hacl_Streaming_SHA2_hash_384(uint8_t *input, uint32_t input_len, uint8_t *dst); +void Hacl_Hash_SHA2_hash_384(uint8_t *output, uint8_t *input, uint32_t input_len); #if defined(__cplusplus) } diff --git a/Modules/_hacl/Hacl_Hash_SHA3.c b/Modules/_hacl/Hacl_Hash_SHA3.c index b3febdfeb2b221..b964e1d9c0aa69 100644 --- a/Modules/_hacl/Hacl_Hash_SHA3.c +++ b/Modules/_hacl/Hacl_Hash_SHA3.c @@ -25,33 +25,178 @@ #include "internal/Hacl_Hash_SHA3.h" +const +uint32_t +Hacl_Hash_SHA3_keccak_rotc[24U] = + { + 1U, 3U, 6U, 10U, 15U, 21U, 28U, 36U, 45U, 55U, 2U, 14U, 27U, 41U, 56U, 8U, 25U, 43U, 62U, 18U, + 39U, 61U, 20U, 44U + }; + +const +uint32_t +Hacl_Hash_SHA3_keccak_piln[24U] = + { + 10U, 7U, 11U, 17U, 18U, 3U, 5U, 16U, 8U, 21U, 24U, 4U, 15U, 23U, 19U, 13U, 12U, 2U, 20U, 14U, + 22U, 9U, 6U, 1U + }; + +const +uint64_t +Hacl_Hash_SHA3_keccak_rndc[24U] = + { + 0x0000000000000001ULL, 0x0000000000008082ULL, 0x800000000000808aULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x0000000080000001ULL, 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x0000000000000088ULL, 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, 0x000000000000800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, 0x0000000080000001ULL, 0x8000000080008008ULL + }; + +static void absorb_inner_32(uint8_t *b, uint64_t *s) +{ + uint64_t ws[32U] = { 0U }; + uint8_t *b1 = b; + uint64_t u = load64_le(b1); + ws[0U] = u; + uint64_t u0 = load64_le(b1 + 8U); + ws[1U] = u0; + uint64_t u1 = load64_le(b1 + 16U); + ws[2U] = u1; + uint64_t u2 = load64_le(b1 + 24U); + ws[3U] = u2; + uint64_t u3 = load64_le(b1 + 32U); + ws[4U] = u3; + uint64_t u4 = load64_le(b1 + 40U); + ws[5U] = u4; + uint64_t u5 = load64_le(b1 + 48U); + ws[6U] = u5; + uint64_t u6 = load64_le(b1 + 56U); + ws[7U] = u6; + uint64_t u7 = load64_le(b1 + 64U); + ws[8U] = u7; + uint64_t u8 = load64_le(b1 + 72U); + ws[9U] = u8; + uint64_t u9 = load64_le(b1 + 80U); + ws[10U] = u9; + uint64_t u10 = load64_le(b1 + 88U); + ws[11U] = u10; + uint64_t u11 = load64_le(b1 + 96U); + ws[12U] = u11; + uint64_t u12 = load64_le(b1 + 104U); + ws[13U] = u12; + uint64_t u13 = load64_le(b1 + 112U); + ws[14U] = u13; + uint64_t u14 = load64_le(b1 + 120U); + ws[15U] = u14; + uint64_t u15 = load64_le(b1 + 128U); + ws[16U] = u15; + uint64_t u16 = load64_le(b1 + 136U); + ws[17U] = u16; + uint64_t u17 = load64_le(b1 + 144U); + ws[18U] = u17; + uint64_t u18 = load64_le(b1 + 152U); + ws[19U] = u18; + uint64_t u19 = load64_le(b1 + 160U); + ws[20U] = u19; + uint64_t u20 = load64_le(b1 + 168U); + ws[21U] = u20; + uint64_t u21 = load64_le(b1 + 176U); + ws[22U] = u21; + uint64_t u22 = load64_le(b1 + 184U); + ws[23U] = u22; + uint64_t u23 = load64_le(b1 + 192U); + ws[24U] = u23; + uint64_t u24 = load64_le(b1 + 200U); + ws[25U] = u24; + uint64_t u25 = load64_le(b1 + 208U); + ws[26U] = u25; + uint64_t u26 = load64_le(b1 + 216U); + ws[27U] = u26; + uint64_t u27 = load64_le(b1 + 224U); + ws[28U] = u27; + uint64_t u28 = load64_le(b1 + 232U); + ws[29U] = u28; + uint64_t u29 = load64_le(b1 + 240U); + ws[30U] = u29; + uint64_t u30 = load64_le(b1 + 248U); + ws[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) + { + s[i] = s[i] ^ ws[i]; + } + for (uint32_t i0 = 0U; i0 < 24U; i0++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i1, + 0U, + 5U, + 1U, + uint64_t uu____0 = _C[(i1 + 1U) % 5U]; + uint64_t _D = _C[(i1 + 4U) % 5U] ^ (uu____0 << 1U | uu____0 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i1 + 5U * i] = s[i1 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i0]; + s[0U] = s[0U] ^ c; + } +} + static uint32_t block_len(Spec_Hash_Definitions_hash_alg a) { switch (a) { case Spec_Hash_Definitions_SHA3_224: { - return (uint32_t)144U; + return 144U; } case Spec_Hash_Definitions_SHA3_256: { - return (uint32_t)136U; + return 136U; } case Spec_Hash_Definitions_SHA3_384: { - return (uint32_t)104U; + return 104U; } case Spec_Hash_Definitions_SHA3_512: { - return (uint32_t)72U; + return 72U; } case Spec_Hash_Definitions_Shake128: { - return (uint32_t)168U; + return 168U; } case Spec_Hash_Definitions_Shake256: { - return (uint32_t)136U; + return 136U; } default: { @@ -67,19 +212,19 @@ static uint32_t hash_len(Spec_Hash_Definitions_hash_alg a) { case Spec_Hash_Definitions_SHA3_224: { - return (uint32_t)28U; + return 28U; } case Spec_Hash_Definitions_SHA3_256: { - return (uint32_t)32U; + return 32U; } case Spec_Hash_Definitions_SHA3_384: { - return (uint32_t)48U; + return 48U; } case Spec_Hash_Definitions_SHA3_512: { - return (uint32_t)64U; + return 64U; } default: { @@ -97,10 +242,18 @@ Hacl_Hash_SHA3_update_multi_sha3( uint32_t n_blocks ) { - for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + uint32_t l = block_len(a) * n_blocks; + for (uint32_t i = 0U; i < l / block_len(a); i++) { - uint8_t *block = blocks + i * block_len(a); - Hacl_Impl_SHA3_absorb_inner(block_len(a), block, s); + uint8_t b[256U] = { 0U }; + uint8_t *b_ = b; + uint8_t *b0 = blocks; + uint8_t *bl0 = b_; + uint8_t *uu____0 = b0 + i * block_len(a); + memcpy(bl0, uu____0, block_len(a) * sizeof (uint8_t)); + uint32_t unused = block_len(a); + KRML_MAYBE_UNUSED_VAR(unused); + absorb_inner_32(b_, s); } } @@ -115,139 +268,374 @@ Hacl_Hash_SHA3_update_last_sha3( uint8_t suffix; if (a == Spec_Hash_Definitions_Shake128 || a == Spec_Hash_Definitions_Shake256) { - suffix = (uint8_t)0x1fU; + suffix = 0x1fU; } else { - suffix = (uint8_t)0x06U; + suffix = 0x06U; } uint32_t len = block_len(a); if (input_len == len) { - Hacl_Impl_SHA3_absorb_inner(len, input, s); - uint8_t *uu____0 = input + input_len; - uint8_t lastBlock_[200U] = { 0U }; - uint8_t *lastBlock = lastBlock_; - memcpy(lastBlock, uu____0, (uint32_t)0U * sizeof (uint8_t)); - lastBlock[0U] = suffix; - Hacl_Impl_SHA3_loadState(len, lastBlock, s); - if (!((suffix & (uint8_t)0x80U) == (uint8_t)0U) && (uint32_t)0U == len - (uint32_t)1U) - { - Hacl_Impl_SHA3_state_permute(s); - } - uint8_t nextBlock_[200U] = { 0U }; - uint8_t *nextBlock = nextBlock_; - nextBlock[len - (uint32_t)1U] = (uint8_t)0x80U; - Hacl_Impl_SHA3_loadState(len, nextBlock, s); - Hacl_Impl_SHA3_state_permute(s); + uint8_t b1[256U] = { 0U }; + uint8_t *b_ = b1; + uint8_t *b00 = input; + uint8_t *bl00 = b_; + memcpy(bl00, b00 + 0U * len, len * sizeof (uint8_t)); + absorb_inner_32(b_, s); + uint8_t b2[256U] = { 0U }; + uint8_t *b_0 = b2; + uint32_t rem = 0U % len; + uint8_t *b01 = input + input_len; + uint8_t *bl0 = b_0; + memcpy(bl0, b01 + 0U - rem, rem * sizeof (uint8_t)); + uint8_t *b02 = b_0; + b02[0U % len] = suffix; + uint64_t ws[32U] = { 0U }; + uint8_t *b = b_0; + uint64_t u = load64_le(b); + ws[0U] = u; + uint64_t u0 = load64_le(b + 8U); + ws[1U] = u0; + uint64_t u1 = load64_le(b + 16U); + ws[2U] = u1; + uint64_t u2 = load64_le(b + 24U); + ws[3U] = u2; + uint64_t u3 = load64_le(b + 32U); + ws[4U] = u3; + uint64_t u4 = load64_le(b + 40U); + ws[5U] = u4; + uint64_t u5 = load64_le(b + 48U); + ws[6U] = u5; + uint64_t u6 = load64_le(b + 56U); + ws[7U] = u6; + uint64_t u7 = load64_le(b + 64U); + ws[8U] = u7; + uint64_t u8 = load64_le(b + 72U); + ws[9U] = u8; + uint64_t u9 = load64_le(b + 80U); + ws[10U] = u9; + uint64_t u10 = load64_le(b + 88U); + ws[11U] = u10; + uint64_t u11 = load64_le(b + 96U); + ws[12U] = u11; + uint64_t u12 = load64_le(b + 104U); + ws[13U] = u12; + uint64_t u13 = load64_le(b + 112U); + ws[14U] = u13; + uint64_t u14 = load64_le(b + 120U); + ws[15U] = u14; + uint64_t u15 = load64_le(b + 128U); + ws[16U] = u15; + uint64_t u16 = load64_le(b + 136U); + ws[17U] = u16; + uint64_t u17 = load64_le(b + 144U); + ws[18U] = u17; + uint64_t u18 = load64_le(b + 152U); + ws[19U] = u18; + uint64_t u19 = load64_le(b + 160U); + ws[20U] = u19; + uint64_t u20 = load64_le(b + 168U); + ws[21U] = u20; + uint64_t u21 = load64_le(b + 176U); + ws[22U] = u21; + uint64_t u22 = load64_le(b + 184U); + ws[23U] = u22; + uint64_t u23 = load64_le(b + 192U); + ws[24U] = u23; + uint64_t u24 = load64_le(b + 200U); + ws[25U] = u24; + uint64_t u25 = load64_le(b + 208U); + ws[26U] = u25; + uint64_t u26 = load64_le(b + 216U); + ws[27U] = u26; + uint64_t u27 = load64_le(b + 224U); + ws[28U] = u27; + uint64_t u28 = load64_le(b + 232U); + ws[29U] = u28; + uint64_t u29 = load64_le(b + 240U); + ws[30U] = u29; + uint64_t u30 = load64_le(b + 248U); + ws[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) + { + s[i] = s[i] ^ ws[i]; + } + if (!(((uint32_t)suffix & 0x80U) == 0U) && 0U % len == len - 1U) + { + for (uint32_t i0 = 0U; i0 < 24U; i0++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i1, + 0U, + 5U, + 1U, + uint64_t uu____0 = _C[(i1 + 1U) % 5U]; + uint64_t _D = _C[(i1 + 4U) % 5U] ^ (uu____0 << 1U | uu____0 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i1 + 5U * i] = s[i1 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i0]; + s[0U] = s[0U] ^ c; + } + } + uint8_t b3[256U] = { 0U }; + uint8_t *b4 = b3; + uint8_t *b0 = b4; + b0[len - 1U] = 0x80U; + absorb_inner_32(b4, s); return; } - uint8_t lastBlock_[200U] = { 0U }; - uint8_t *lastBlock = lastBlock_; - memcpy(lastBlock, input, input_len * sizeof (uint8_t)); - lastBlock[input_len] = suffix; - Hacl_Impl_SHA3_loadState(len, lastBlock, s); - if (!((suffix & (uint8_t)0x80U) == (uint8_t)0U) && input_len == len - (uint32_t)1U) + uint8_t b1[256U] = { 0U }; + uint8_t *b_ = b1; + uint32_t rem = input_len % len; + uint8_t *b00 = input; + uint8_t *bl0 = b_; + memcpy(bl0, b00 + input_len - rem, rem * sizeof (uint8_t)); + uint8_t *b01 = b_; + b01[input_len % len] = suffix; + uint64_t ws[32U] = { 0U }; + uint8_t *b = b_; + uint64_t u = load64_le(b); + ws[0U] = u; + uint64_t u0 = load64_le(b + 8U); + ws[1U] = u0; + uint64_t u1 = load64_le(b + 16U); + ws[2U] = u1; + uint64_t u2 = load64_le(b + 24U); + ws[3U] = u2; + uint64_t u3 = load64_le(b + 32U); + ws[4U] = u3; + uint64_t u4 = load64_le(b + 40U); + ws[5U] = u4; + uint64_t u5 = load64_le(b + 48U); + ws[6U] = u5; + uint64_t u6 = load64_le(b + 56U); + ws[7U] = u6; + uint64_t u7 = load64_le(b + 64U); + ws[8U] = u7; + uint64_t u8 = load64_le(b + 72U); + ws[9U] = u8; + uint64_t u9 = load64_le(b + 80U); + ws[10U] = u9; + uint64_t u10 = load64_le(b + 88U); + ws[11U] = u10; + uint64_t u11 = load64_le(b + 96U); + ws[12U] = u11; + uint64_t u12 = load64_le(b + 104U); + ws[13U] = u12; + uint64_t u13 = load64_le(b + 112U); + ws[14U] = u13; + uint64_t u14 = load64_le(b + 120U); + ws[15U] = u14; + uint64_t u15 = load64_le(b + 128U); + ws[16U] = u15; + uint64_t u16 = load64_le(b + 136U); + ws[17U] = u16; + uint64_t u17 = load64_le(b + 144U); + ws[18U] = u17; + uint64_t u18 = load64_le(b + 152U); + ws[19U] = u18; + uint64_t u19 = load64_le(b + 160U); + ws[20U] = u19; + uint64_t u20 = load64_le(b + 168U); + ws[21U] = u20; + uint64_t u21 = load64_le(b + 176U); + ws[22U] = u21; + uint64_t u22 = load64_le(b + 184U); + ws[23U] = u22; + uint64_t u23 = load64_le(b + 192U); + ws[24U] = u23; + uint64_t u24 = load64_le(b + 200U); + ws[25U] = u24; + uint64_t u25 = load64_le(b + 208U); + ws[26U] = u25; + uint64_t u26 = load64_le(b + 216U); + ws[27U] = u26; + uint64_t u27 = load64_le(b + 224U); + ws[28U] = u27; + uint64_t u28 = load64_le(b + 232U); + ws[29U] = u28; + uint64_t u29 = load64_le(b + 240U); + ws[30U] = u29; + uint64_t u30 = load64_le(b + 248U); + ws[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) + { + s[i] = s[i] ^ ws[i]; + } + if (!(((uint32_t)suffix & 0x80U) == 0U) && input_len % len == len - 1U) { - Hacl_Impl_SHA3_state_permute(s); + for (uint32_t i0 = 0U; i0 < 24U; i0++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i1, + 0U, + 5U, + 1U, + uint64_t uu____2 = _C[(i1 + 1U) % 5U]; + uint64_t _D = _C[(i1 + 4U) % 5U] ^ (uu____2 << 1U | uu____2 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i1 + 5U * i] = s[i1 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____3 = current; + s[_Y] = uu____3 << r | uu____3 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i0]; + s[0U] = s[0U] ^ c; + } } - uint8_t nextBlock_[200U] = { 0U }; - uint8_t *nextBlock = nextBlock_; - nextBlock[len - (uint32_t)1U] = (uint8_t)0x80U; - Hacl_Impl_SHA3_loadState(len, nextBlock, s); - Hacl_Impl_SHA3_state_permute(s); + uint8_t b2[256U] = { 0U }; + uint8_t *b3 = b2; + uint8_t *b0 = b3; + b0[len - 1U] = 0x80U; + absorb_inner_32(b3, s); } typedef struct hash_buf2_s { - Hacl_Streaming_Keccak_hash_buf fst; - Hacl_Streaming_Keccak_hash_buf snd; + Hacl_Hash_SHA3_hash_buf fst; + Hacl_Hash_SHA3_hash_buf snd; } hash_buf2; -Spec_Hash_Definitions_hash_alg Hacl_Streaming_Keccak_get_alg(Hacl_Streaming_Keccak_state *s) +Spec_Hash_Definitions_hash_alg Hacl_Hash_SHA3_get_alg(Hacl_Hash_SHA3_state_t *s) { - Hacl_Streaming_Keccak_state scrut = *s; - Hacl_Streaming_Keccak_hash_buf block_state = scrut.block_state; + Hacl_Hash_SHA3_hash_buf block_state = (*s).block_state; return block_state.fst; } -Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_hash_alg a) +Hacl_Hash_SHA3_state_t *Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_hash_alg a) { KRML_CHECK_SIZE(sizeof (uint8_t), block_len(a)); uint8_t *buf0 = (uint8_t *)KRML_HOST_CALLOC(block_len(a), sizeof (uint8_t)); - uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC((uint32_t)25U, sizeof (uint64_t)); - Hacl_Streaming_Keccak_hash_buf block_state = { .fst = a, .snd = buf }; - Hacl_Streaming_Keccak_state - s = { .block_state = block_state, .buf = buf0, .total_len = (uint64_t)(uint32_t)0U }; - Hacl_Streaming_Keccak_state - *p = (Hacl_Streaming_Keccak_state *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_Keccak_state)); + uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC(25U, sizeof (uint64_t)); + Hacl_Hash_SHA3_hash_buf block_state = { .fst = a, .snd = buf }; + Hacl_Hash_SHA3_state_t + s = { .block_state = block_state, .buf = buf0, .total_len = (uint64_t)0U }; + Hacl_Hash_SHA3_state_t + *p = (Hacl_Hash_SHA3_state_t *)KRML_HOST_MALLOC(sizeof (Hacl_Hash_SHA3_state_t)); p[0U] = s; uint64_t *s1 = block_state.snd; - memset(s1, 0U, (uint32_t)25U * sizeof (uint64_t)); + memset(s1, 0U, 25U * sizeof (uint64_t)); return p; } -void Hacl_Streaming_Keccak_free(Hacl_Streaming_Keccak_state *s) +void Hacl_Hash_SHA3_free(Hacl_Hash_SHA3_state_t *state) { - Hacl_Streaming_Keccak_state scrut = *s; + Hacl_Hash_SHA3_state_t scrut = *state; uint8_t *buf = scrut.buf; - Hacl_Streaming_Keccak_hash_buf block_state = scrut.block_state; - uint64_t *s1 = block_state.snd; - KRML_HOST_FREE(s1); - KRML_HOST_FREE(buf); + Hacl_Hash_SHA3_hash_buf block_state = scrut.block_state; + uint64_t *s = block_state.snd; KRML_HOST_FREE(s); + KRML_HOST_FREE(buf); + KRML_HOST_FREE(state); } -Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_copy(Hacl_Streaming_Keccak_state *s0) +Hacl_Hash_SHA3_state_t *Hacl_Hash_SHA3_copy(Hacl_Hash_SHA3_state_t *state) { - Hacl_Streaming_Keccak_state scrut0 = *s0; - Hacl_Streaming_Keccak_hash_buf block_state0 = scrut0.block_state; + Hacl_Hash_SHA3_state_t scrut0 = *state; + Hacl_Hash_SHA3_hash_buf block_state0 = scrut0.block_state; uint8_t *buf0 = scrut0.buf; uint64_t total_len0 = scrut0.total_len; Spec_Hash_Definitions_hash_alg i = block_state0.fst; KRML_CHECK_SIZE(sizeof (uint8_t), block_len(i)); uint8_t *buf1 = (uint8_t *)KRML_HOST_CALLOC(block_len(i), sizeof (uint8_t)); memcpy(buf1, buf0, block_len(i) * sizeof (uint8_t)); - uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC((uint32_t)25U, sizeof (uint64_t)); - Hacl_Streaming_Keccak_hash_buf block_state = { .fst = i, .snd = buf }; + uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC(25U, sizeof (uint64_t)); + Hacl_Hash_SHA3_hash_buf block_state = { .fst = i, .snd = buf }; hash_buf2 scrut = { .fst = block_state0, .snd = block_state }; uint64_t *s_dst = scrut.snd.snd; uint64_t *s_src = scrut.fst.snd; - memcpy(s_dst, s_src, (uint32_t)25U * sizeof (uint64_t)); - Hacl_Streaming_Keccak_state + memcpy(s_dst, s_src, 25U * sizeof (uint64_t)); + Hacl_Hash_SHA3_state_t s = { .block_state = block_state, .buf = buf1, .total_len = total_len0 }; - Hacl_Streaming_Keccak_state - *p = (Hacl_Streaming_Keccak_state *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_Keccak_state)); + Hacl_Hash_SHA3_state_t + *p = (Hacl_Hash_SHA3_state_t *)KRML_HOST_MALLOC(sizeof (Hacl_Hash_SHA3_state_t)); p[0U] = s; return p; } -void Hacl_Streaming_Keccak_reset(Hacl_Streaming_Keccak_state *s) +void Hacl_Hash_SHA3_reset(Hacl_Hash_SHA3_state_t *state) { - Hacl_Streaming_Keccak_state scrut = *s; + Hacl_Hash_SHA3_state_t scrut = *state; uint8_t *buf = scrut.buf; - Hacl_Streaming_Keccak_hash_buf block_state = scrut.block_state; - uint64_t *s1 = block_state.snd; - memset(s1, 0U, (uint32_t)25U * sizeof (uint64_t)); - Hacl_Streaming_Keccak_state - tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - s[0U] = tmp; + Hacl_Hash_SHA3_hash_buf block_state = scrut.block_state; + Spec_Hash_Definitions_hash_alg i = block_state.fst; + KRML_MAYBE_UNUSED_VAR(i); + uint64_t *s = block_state.snd; + memset(s, 0U, 25U * sizeof (uint64_t)); + Hacl_Hash_SHA3_state_t + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)0U }; + state[0U] = tmp; } Hacl_Streaming_Types_error_code -Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint32_t len) +Hacl_Hash_SHA3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *chunk, uint32_t chunk_len) { - Hacl_Streaming_Keccak_state s = *p; - Hacl_Streaming_Keccak_hash_buf block_state = s.block_state; + Hacl_Hash_SHA3_state_t s = *state; + Hacl_Hash_SHA3_hash_buf block_state = s.block_state; uint64_t total_len = s.total_len; Spec_Hash_Definitions_hash_alg i = block_state.fst; - if ((uint64_t)len > (uint64_t)0xFFFFFFFFFFFFFFFFU - total_len) + if ((uint64_t)chunk_len > 0xFFFFFFFFFFFFFFFFULL - total_len) { return Hacl_Streaming_Types_MaximumLengthExceeded; } uint32_t sz; - if (total_len % (uint64_t)block_len(i) == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)block_len(i) == 0ULL && total_len > 0ULL) { sz = block_len(i); } @@ -255,14 +643,14 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint { sz = (uint32_t)(total_len % (uint64_t)block_len(i)); } - if (len <= block_len(i) - sz) + if (chunk_len <= block_len(i) - sz) { - Hacl_Streaming_Keccak_state s1 = *p; - Hacl_Streaming_Keccak_hash_buf block_state1 = s1.block_state; + Hacl_Hash_SHA3_state_t s1 = *state; + Hacl_Hash_SHA3_hash_buf block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)block_len(i) == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)block_len(i) == 0ULL && total_len1 > 0ULL) { sz1 = block_len(i); } @@ -271,26 +659,20 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint sz1 = (uint32_t)(total_len1 % (uint64_t)block_len(i)); } uint8_t *buf2 = buf + sz1; - memcpy(buf2, data, len * sizeof (uint8_t)); - uint64_t total_len2 = total_len1 + (uint64_t)len; - *p + memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; + *state = - ( - (Hacl_Streaming_Keccak_state){ - .block_state = block_state1, - .buf = buf, - .total_len = total_len2 - } - ); + ((Hacl_Hash_SHA3_state_t){ .block_state = block_state1, .buf = buf, .total_len = total_len2 }); } - else if (sz == (uint32_t)0U) + else if (sz == 0U) { - Hacl_Streaming_Keccak_state s1 = *p; - Hacl_Streaming_Keccak_hash_buf block_state1 = s1.block_state; + Hacl_Hash_SHA3_state_t s1 = *state; + Hacl_Hash_SHA3_hash_buf block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)block_len(i) == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)block_len(i) == 0ULL && total_len1 > 0ULL) { sz1 = block_len(i); } @@ -298,52 +680,52 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint { sz1 = (uint32_t)(total_len1 % (uint64_t)block_len(i)); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; uint64_t *s2 = block_state1.snd; Hacl_Hash_SHA3_update_multi_sha3(a1, s2, buf, block_len(i) / block_len(a1)); } uint32_t ite; - if ((uint64_t)len % (uint64_t)block_len(i) == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + if ((uint64_t)chunk_len % (uint64_t)block_len(i) == 0ULL && (uint64_t)chunk_len > 0ULL) { ite = block_len(i); } else { - ite = (uint32_t)((uint64_t)len % (uint64_t)block_len(i)); + ite = (uint32_t)((uint64_t)chunk_len % (uint64_t)block_len(i)); } - uint32_t n_blocks = (len - ite) / block_len(i); + uint32_t n_blocks = (chunk_len - ite) / block_len(i); uint32_t data1_len = n_blocks * block_len(i); - uint32_t data2_len = len - data1_len; - uint8_t *data1 = data; - uint8_t *data2 = data + data1_len; + uint32_t data2_len = chunk_len - data1_len; + uint8_t *data1 = chunk; + uint8_t *data2 = chunk + data1_len; Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; uint64_t *s2 = block_state1.snd; Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data1, data1_len / block_len(a1)); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *p + *state = ( - (Hacl_Streaming_Keccak_state){ + (Hacl_Hash_SHA3_state_t){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)len + .total_len = total_len1 + (uint64_t)chunk_len } ); } else { uint32_t diff = block_len(i) - sz; - uint8_t *data1 = data; - uint8_t *data2 = data + diff; - Hacl_Streaming_Keccak_state s1 = *p; - Hacl_Streaming_Keccak_hash_buf block_state10 = s1.block_state; + uint8_t *chunk1 = chunk; + uint8_t *chunk2 = chunk + diff; + Hacl_Hash_SHA3_state_t s1 = *state; + Hacl_Hash_SHA3_hash_buf block_state10 = s1.block_state; uint8_t *buf0 = s1.buf; uint64_t total_len10 = s1.total_len; uint32_t sz10; - if (total_len10 % (uint64_t)block_len(i) == (uint64_t)0U && total_len10 > (uint64_t)0U) + if (total_len10 % (uint64_t)block_len(i) == 0ULL && total_len10 > 0ULL) { sz10 = block_len(i); } @@ -352,23 +734,23 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint sz10 = (uint32_t)(total_len10 % (uint64_t)block_len(i)); } uint8_t *buf2 = buf0 + sz10; - memcpy(buf2, data1, diff * sizeof (uint8_t)); + memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *p + *state = ( - (Hacl_Streaming_Keccak_state){ + (Hacl_Hash_SHA3_state_t){ .block_state = block_state10, .buf = buf0, .total_len = total_len2 } ); - Hacl_Streaming_Keccak_state s10 = *p; - Hacl_Streaming_Keccak_hash_buf block_state1 = s10.block_state; + Hacl_Hash_SHA3_state_t s10 = *state; + Hacl_Hash_SHA3_hash_buf block_state1 = s10.block_state; uint8_t *buf = s10.buf; uint64_t total_len1 = s10.total_len; uint32_t sz1; - if (total_len1 % (uint64_t)block_len(i) == (uint64_t)0U && total_len1 > (uint64_t)0U) + if (total_len1 % (uint64_t)block_len(i) == 0ULL && total_len1 > 0ULL) { sz1 = block_len(i); } @@ -376,7 +758,7 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint { sz1 = (uint32_t)(total_len1 % (uint64_t)block_len(i)); } - if (!(sz1 == (uint32_t)0U)) + if (!(sz1 == 0U)) { Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; uint64_t *s2 = block_state1.snd; @@ -385,35 +767,35 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint uint32_t ite; if ( - (uint64_t)(len - diff) + (uint64_t)(chunk_len - diff) % (uint64_t)block_len(i) - == (uint64_t)0U - && (uint64_t)(len - diff) > (uint64_t)0U + == 0ULL + && (uint64_t)(chunk_len - diff) > 0ULL ) { ite = block_len(i); } else { - ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)block_len(i)); + ite = (uint32_t)((uint64_t)(chunk_len - diff) % (uint64_t)block_len(i)); } - uint32_t n_blocks = (len - diff - ite) / block_len(i); + uint32_t n_blocks = (chunk_len - diff - ite) / block_len(i); uint32_t data1_len = n_blocks * block_len(i); - uint32_t data2_len = len - diff - data1_len; - uint8_t *data11 = data2; - uint8_t *data21 = data2 + data1_len; + uint32_t data2_len = chunk_len - diff - data1_len; + uint8_t *data1 = chunk2; + uint8_t *data2 = chunk2 + data1_len; Spec_Hash_Definitions_hash_alg a1 = block_state1.fst; uint64_t *s2 = block_state1.snd; - Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data11, data1_len / block_len(a1)); + Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data1, data1_len / block_len(a1)); uint8_t *dst = buf; - memcpy(dst, data21, data2_len * sizeof (uint8_t)); - *p + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *state = ( - (Hacl_Streaming_Keccak_state){ + (Hacl_Hash_SHA3_state_t){ .block_state = block_state1, .buf = buf, - .total_len = total_len1 + (uint64_t)(len - diff) + .total_len = total_len1 + (uint64_t)(chunk_len - diff) } ); } @@ -421,19 +803,19 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint } static void -finish_( +digest_( Spec_Hash_Definitions_hash_alg a, - Hacl_Streaming_Keccak_state *p, - uint8_t *dst, + Hacl_Hash_SHA3_state_t *state, + uint8_t *output, uint32_t l ) { - Hacl_Streaming_Keccak_state scrut0 = *p; - Hacl_Streaming_Keccak_hash_buf block_state = scrut0.block_state; + Hacl_Hash_SHA3_state_t scrut0 = *state; + Hacl_Hash_SHA3_hash_buf block_state = scrut0.block_state; uint8_t *buf_ = scrut0.buf; uint64_t total_len = scrut0.total_len; uint32_t r; - if (total_len % (uint64_t)block_len(a) == (uint64_t)0U && total_len > (uint64_t)0U) + if (total_len % (uint64_t)block_len(a) == 0ULL && total_len > 0ULL) { r = block_len(a); } @@ -443,25 +825,25 @@ finish_( } uint8_t *buf_1 = buf_; uint64_t buf[25U] = { 0U }; - Hacl_Streaming_Keccak_hash_buf tmp_block_state = { .fst = a, .snd = buf }; + Hacl_Hash_SHA3_hash_buf tmp_block_state = { .fst = a, .snd = buf }; hash_buf2 scrut = { .fst = block_state, .snd = tmp_block_state }; uint64_t *s_dst = scrut.snd.snd; uint64_t *s_src = scrut.fst.snd; - memcpy(s_dst, s_src, (uint32_t)25U * sizeof (uint64_t)); - uint32_t ite0; - if (r % block_len(a) == (uint32_t)0U && r > (uint32_t)0U) + memcpy(s_dst, s_src, 25U * sizeof (uint64_t)); + uint32_t ite; + if (r % block_len(a) == 0U && r > 0U) { - ite0 = block_len(a); + ite = block_len(a); } else { - ite0 = r % block_len(a); + ite = r % block_len(a); } - uint8_t *buf_last = buf_1 + r - ite0; + uint8_t *buf_last = buf_1 + r - ite; uint8_t *buf_multi = buf_1; Spec_Hash_Definitions_hash_alg a1 = tmp_block_state.fst; uint64_t *s0 = tmp_block_state.snd; - Hacl_Hash_SHA3_update_multi_sha3(a1, s0, buf_multi, (uint32_t)0U / block_len(a1)); + Hacl_Hash_SHA3_update_multi_sha3(a1, s0, buf_multi, 0U / block_len(a1)); Spec_Hash_Definitions_hash_alg a10 = tmp_block_state.fst; uint64_t *s1 = tmp_block_state.snd; Hacl_Hash_SHA3_update_last_sha3(a10, s1, buf_last, r); @@ -469,356 +851,1521 @@ finish_( uint64_t *s = tmp_block_state.snd; if (a11 == Spec_Hash_Definitions_Shake128 || a11 == Spec_Hash_Definitions_Shake256) { - uint32_t ite; - if (a11 == Spec_Hash_Definitions_Shake128 || a11 == Spec_Hash_Definitions_Shake256) + for (uint32_t i0 = 0U; i0 < l / block_len(a11); i0++) { - ite = l; + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + uint8_t *b0 = output; + uint8_t *uu____0 = hbuf; + memcpy(b0 + i0 * block_len(a11), uu____0, block_len(a11) * sizeof (uint8_t)); + for (uint32_t i1 = 0U; i1 < 24U; i1++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i2, + 0U, + 5U, + 1U, + uint64_t uu____1 = _C[(i2 + 1U) % 5U]; + uint64_t _D = _C[(i2 + 4U) % 5U] ^ (uu____1 << 1U | uu____1 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i2 + 5U * i] = s[i2 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r1 = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____2 = current; + s[_Y] = uu____2 << r1 | uu____2 >> (64U - r1); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i1]; + s[0U] = s[0U] ^ c; + } } - else + uint32_t remOut = l % block_len(a11); + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) { - ite = hash_len(a11); + store64_le(hbuf + i * 8U, ws[i]); } - Hacl_Impl_SHA3_squeeze(s, block_len(a11), ite, dst); + memcpy(output + l - remOut, hbuf, remOut * sizeof (uint8_t)); return; } - Hacl_Impl_SHA3_squeeze(s, block_len(a11), hash_len(a11), dst); + for (uint32_t i0 = 0U; i0 < hash_len(a11) / block_len(a11); i0++) + { + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + uint8_t *b0 = output; + uint8_t *uu____3 = hbuf; + memcpy(b0 + i0 * block_len(a11), uu____3, block_len(a11) * sizeof (uint8_t)); + for (uint32_t i1 = 0U; i1 < 24U; i1++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i2, + 0U, + 5U, + 1U, + uint64_t uu____4 = _C[(i2 + 1U) % 5U]; + uint64_t _D = _C[(i2 + 4U) % 5U] ^ (uu____4 << 1U | uu____4 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i2 + 5U * i] = s[i2 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r1 = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____5 = current; + s[_Y] = uu____5 << r1 | uu____5 >> (64U - r1); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i1]; + s[0U] = s[0U] ^ c; + } + } + uint32_t remOut = hash_len(a11) % block_len(a11); + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + uint8_t *uu____6 = hbuf; + memcpy(output + hash_len(a11) - remOut, uu____6, remOut * sizeof (uint8_t)); } Hacl_Streaming_Types_error_code -Hacl_Streaming_Keccak_finish(Hacl_Streaming_Keccak_state *s, uint8_t *dst) +Hacl_Hash_SHA3_digest(Hacl_Hash_SHA3_state_t *state, uint8_t *output) { - Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + Spec_Hash_Definitions_hash_alg a1 = Hacl_Hash_SHA3_get_alg(state); if (a1 == Spec_Hash_Definitions_Shake128 || a1 == Spec_Hash_Definitions_Shake256) { return Hacl_Streaming_Types_InvalidAlgorithm; } - finish_(a1, s, dst, hash_len(a1)); + digest_(a1, state, output, hash_len(a1)); return Hacl_Streaming_Types_Success; } Hacl_Streaming_Types_error_code -Hacl_Streaming_Keccak_squeeze(Hacl_Streaming_Keccak_state *s, uint8_t *dst, uint32_t l) +Hacl_Hash_SHA3_squeeze(Hacl_Hash_SHA3_state_t *s, uint8_t *dst, uint32_t l) { - Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + Spec_Hash_Definitions_hash_alg a1 = Hacl_Hash_SHA3_get_alg(s); if (!(a1 == Spec_Hash_Definitions_Shake128 || a1 == Spec_Hash_Definitions_Shake256)) { return Hacl_Streaming_Types_InvalidAlgorithm; } - if (l == (uint32_t)0U) + if (l == 0U) { return Hacl_Streaming_Types_InvalidLength; } - finish_(a1, s, dst, l); + digest_(a1, s, dst, l); return Hacl_Streaming_Types_Success; } -uint32_t Hacl_Streaming_Keccak_block_len(Hacl_Streaming_Keccak_state *s) +uint32_t Hacl_Hash_SHA3_block_len(Hacl_Hash_SHA3_state_t *s) { - Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + Spec_Hash_Definitions_hash_alg a1 = Hacl_Hash_SHA3_get_alg(s); return block_len(a1); } -uint32_t Hacl_Streaming_Keccak_hash_len(Hacl_Streaming_Keccak_state *s) +uint32_t Hacl_Hash_SHA3_hash_len(Hacl_Hash_SHA3_state_t *s) { - Spec_Hash_Definitions_hash_alg a1 = Hacl_Streaming_Keccak_get_alg(s); + Spec_Hash_Definitions_hash_alg a1 = Hacl_Hash_SHA3_get_alg(s); return hash_len(a1); } -bool Hacl_Streaming_Keccak_is_shake(Hacl_Streaming_Keccak_state *s) +bool Hacl_Hash_SHA3_is_shake(Hacl_Hash_SHA3_state_t *s) { - Spec_Hash_Definitions_hash_alg uu____0 = Hacl_Streaming_Keccak_get_alg(s); + Spec_Hash_Definitions_hash_alg uu____0 = Hacl_Hash_SHA3_get_alg(s); return uu____0 == Spec_Hash_Definitions_Shake128 || uu____0 == Spec_Hash_Definitions_Shake256; } -void -Hacl_SHA3_shake128_hacl( - uint32_t inputByteLen, - uint8_t *input, - uint32_t outputByteLen, - uint8_t *output -) +void Hacl_Hash_SHA3_absorb_inner_32(uint32_t rateInBytes, uint8_t *b, uint64_t *s) { - Hacl_Impl_SHA3_keccak((uint32_t)1344U, - (uint32_t)256U, - inputByteLen, - input, - (uint8_t)0x1FU, - outputByteLen, - output); + KRML_MAYBE_UNUSED_VAR(rateInBytes); + uint64_t ws[32U] = { 0U }; + uint8_t *b1 = b; + uint64_t u = load64_le(b1); + ws[0U] = u; + uint64_t u0 = load64_le(b1 + 8U); + ws[1U] = u0; + uint64_t u1 = load64_le(b1 + 16U); + ws[2U] = u1; + uint64_t u2 = load64_le(b1 + 24U); + ws[3U] = u2; + uint64_t u3 = load64_le(b1 + 32U); + ws[4U] = u3; + uint64_t u4 = load64_le(b1 + 40U); + ws[5U] = u4; + uint64_t u5 = load64_le(b1 + 48U); + ws[6U] = u5; + uint64_t u6 = load64_le(b1 + 56U); + ws[7U] = u6; + uint64_t u7 = load64_le(b1 + 64U); + ws[8U] = u7; + uint64_t u8 = load64_le(b1 + 72U); + ws[9U] = u8; + uint64_t u9 = load64_le(b1 + 80U); + ws[10U] = u9; + uint64_t u10 = load64_le(b1 + 88U); + ws[11U] = u10; + uint64_t u11 = load64_le(b1 + 96U); + ws[12U] = u11; + uint64_t u12 = load64_le(b1 + 104U); + ws[13U] = u12; + uint64_t u13 = load64_le(b1 + 112U); + ws[14U] = u13; + uint64_t u14 = load64_le(b1 + 120U); + ws[15U] = u14; + uint64_t u15 = load64_le(b1 + 128U); + ws[16U] = u15; + uint64_t u16 = load64_le(b1 + 136U); + ws[17U] = u16; + uint64_t u17 = load64_le(b1 + 144U); + ws[18U] = u17; + uint64_t u18 = load64_le(b1 + 152U); + ws[19U] = u18; + uint64_t u19 = load64_le(b1 + 160U); + ws[20U] = u19; + uint64_t u20 = load64_le(b1 + 168U); + ws[21U] = u20; + uint64_t u21 = load64_le(b1 + 176U); + ws[22U] = u21; + uint64_t u22 = load64_le(b1 + 184U); + ws[23U] = u22; + uint64_t u23 = load64_le(b1 + 192U); + ws[24U] = u23; + uint64_t u24 = load64_le(b1 + 200U); + ws[25U] = u24; + uint64_t u25 = load64_le(b1 + 208U); + ws[26U] = u25; + uint64_t u26 = load64_le(b1 + 216U); + ws[27U] = u26; + uint64_t u27 = load64_le(b1 + 224U); + ws[28U] = u27; + uint64_t u28 = load64_le(b1 + 232U); + ws[29U] = u28; + uint64_t u29 = load64_le(b1 + 240U); + ws[30U] = u29; + uint64_t u30 = load64_le(b1 + 248U); + ws[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) + { + s[i] = s[i] ^ ws[i]; + } + for (uint32_t i0 = 0U; i0 < 24U; i0++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i1, + 0U, + 5U, + 1U, + uint64_t uu____0 = _C[(i1 + 1U) % 5U]; + uint64_t _D = _C[(i1 + 4U) % 5U] ^ (uu____0 << 1U | uu____0 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i1 + 5U * i] = s[i1 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i0]; + s[0U] = s[0U] ^ c; + } } void -Hacl_SHA3_shake256_hacl( - uint32_t inputByteLen, - uint8_t *input, +Hacl_Hash_SHA3_shake128( + uint8_t *output, uint32_t outputByteLen, - uint8_t *output + uint8_t *input, + uint32_t inputByteLen ) { - Hacl_Impl_SHA3_keccak((uint32_t)1088U, - (uint32_t)512U, - inputByteLen, - input, - (uint8_t)0x1FU, - outputByteLen, - output); + uint8_t *ib = input; + uint8_t *rb = output; + uint64_t s[25U] = { 0U }; + uint32_t rateInBytes1 = 168U; + for (uint32_t i = 0U; i < inputByteLen / rateInBytes1; i++) + { + uint8_t b[256U] = { 0U }; + uint8_t *b_ = b; + uint8_t *b0 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b0 + i * rateInBytes1, rateInBytes1 * sizeof (uint8_t)); + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b_, s); + } + uint8_t b1[256U] = { 0U }; + uint8_t *b_ = b1; + uint32_t rem = inputByteLen % rateInBytes1; + uint8_t *b00 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b00 + inputByteLen - rem, rem * sizeof (uint8_t)); + uint8_t *b01 = b_; + b01[inputByteLen % rateInBytes1] = 0x1FU; + uint64_t ws0[32U] = { 0U }; + uint8_t *b = b_; + uint64_t u = load64_le(b); + ws0[0U] = u; + uint64_t u0 = load64_le(b + 8U); + ws0[1U] = u0; + uint64_t u1 = load64_le(b + 16U); + ws0[2U] = u1; + uint64_t u2 = load64_le(b + 24U); + ws0[3U] = u2; + uint64_t u3 = load64_le(b + 32U); + ws0[4U] = u3; + uint64_t u4 = load64_le(b + 40U); + ws0[5U] = u4; + uint64_t u5 = load64_le(b + 48U); + ws0[6U] = u5; + uint64_t u6 = load64_le(b + 56U); + ws0[7U] = u6; + uint64_t u7 = load64_le(b + 64U); + ws0[8U] = u7; + uint64_t u8 = load64_le(b + 72U); + ws0[9U] = u8; + uint64_t u9 = load64_le(b + 80U); + ws0[10U] = u9; + uint64_t u10 = load64_le(b + 88U); + ws0[11U] = u10; + uint64_t u11 = load64_le(b + 96U); + ws0[12U] = u11; + uint64_t u12 = load64_le(b + 104U); + ws0[13U] = u12; + uint64_t u13 = load64_le(b + 112U); + ws0[14U] = u13; + uint64_t u14 = load64_le(b + 120U); + ws0[15U] = u14; + uint64_t u15 = load64_le(b + 128U); + ws0[16U] = u15; + uint64_t u16 = load64_le(b + 136U); + ws0[17U] = u16; + uint64_t u17 = load64_le(b + 144U); + ws0[18U] = u17; + uint64_t u18 = load64_le(b + 152U); + ws0[19U] = u18; + uint64_t u19 = load64_le(b + 160U); + ws0[20U] = u19; + uint64_t u20 = load64_le(b + 168U); + ws0[21U] = u20; + uint64_t u21 = load64_le(b + 176U); + ws0[22U] = u21; + uint64_t u22 = load64_le(b + 184U); + ws0[23U] = u22; + uint64_t u23 = load64_le(b + 192U); + ws0[24U] = u23; + uint64_t u24 = load64_le(b + 200U); + ws0[25U] = u24; + uint64_t u25 = load64_le(b + 208U); + ws0[26U] = u25; + uint64_t u26 = load64_le(b + 216U); + ws0[27U] = u26; + uint64_t u27 = load64_le(b + 224U); + ws0[28U] = u27; + uint64_t u28 = load64_le(b + 232U); + ws0[29U] = u28; + uint64_t u29 = load64_le(b + 240U); + ws0[30U] = u29; + uint64_t u30 = load64_le(b + 248U); + ws0[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) + { + s[i] = s[i] ^ ws0[i]; + } + uint8_t b2[256U] = { 0U }; + uint8_t *b3 = b2; + uint8_t *b0 = b3; + b0[rateInBytes1 - 1U] = 0x80U; + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b3, s); + for (uint32_t i0 = 0U; i0 < outputByteLen / rateInBytes1; i0++) + { + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + uint8_t *b02 = rb; + memcpy(b02 + i0 * rateInBytes1, hbuf, rateInBytes1 * sizeof (uint8_t)); + for (uint32_t i1 = 0U; i1 < 24U; i1++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i2, + 0U, + 5U, + 1U, + uint64_t uu____0 = _C[(i2 + 1U) % 5U]; + uint64_t _D = _C[(i2 + 4U) % 5U] ^ (uu____0 << 1U | uu____0 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i2 + 5U * i] = s[i2 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i1]; + s[0U] = s[0U] ^ c; + } + } + uint32_t remOut = outputByteLen % rateInBytes1; + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + memcpy(rb + outputByteLen - remOut, hbuf, remOut * sizeof (uint8_t)); } -void Hacl_SHA3_sha3_224(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +void +Hacl_Hash_SHA3_shake256( + uint8_t *output, + uint32_t outputByteLen, + uint8_t *input, + uint32_t inputByteLen +) { - Hacl_Impl_SHA3_keccak((uint32_t)1152U, - (uint32_t)448U, - inputByteLen, - input, - (uint8_t)0x06U, - (uint32_t)28U, - output); + uint8_t *ib = input; + uint8_t *rb = output; + uint64_t s[25U] = { 0U }; + uint32_t rateInBytes1 = 136U; + for (uint32_t i = 0U; i < inputByteLen / rateInBytes1; i++) + { + uint8_t b[256U] = { 0U }; + uint8_t *b_ = b; + uint8_t *b0 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b0 + i * rateInBytes1, rateInBytes1 * sizeof (uint8_t)); + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b_, s); + } + uint8_t b1[256U] = { 0U }; + uint8_t *b_ = b1; + uint32_t rem = inputByteLen % rateInBytes1; + uint8_t *b00 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b00 + inputByteLen - rem, rem * sizeof (uint8_t)); + uint8_t *b01 = b_; + b01[inputByteLen % rateInBytes1] = 0x1FU; + uint64_t ws0[32U] = { 0U }; + uint8_t *b = b_; + uint64_t u = load64_le(b); + ws0[0U] = u; + uint64_t u0 = load64_le(b + 8U); + ws0[1U] = u0; + uint64_t u1 = load64_le(b + 16U); + ws0[2U] = u1; + uint64_t u2 = load64_le(b + 24U); + ws0[3U] = u2; + uint64_t u3 = load64_le(b + 32U); + ws0[4U] = u3; + uint64_t u4 = load64_le(b + 40U); + ws0[5U] = u4; + uint64_t u5 = load64_le(b + 48U); + ws0[6U] = u5; + uint64_t u6 = load64_le(b + 56U); + ws0[7U] = u6; + uint64_t u7 = load64_le(b + 64U); + ws0[8U] = u7; + uint64_t u8 = load64_le(b + 72U); + ws0[9U] = u8; + uint64_t u9 = load64_le(b + 80U); + ws0[10U] = u9; + uint64_t u10 = load64_le(b + 88U); + ws0[11U] = u10; + uint64_t u11 = load64_le(b + 96U); + ws0[12U] = u11; + uint64_t u12 = load64_le(b + 104U); + ws0[13U] = u12; + uint64_t u13 = load64_le(b + 112U); + ws0[14U] = u13; + uint64_t u14 = load64_le(b + 120U); + ws0[15U] = u14; + uint64_t u15 = load64_le(b + 128U); + ws0[16U] = u15; + uint64_t u16 = load64_le(b + 136U); + ws0[17U] = u16; + uint64_t u17 = load64_le(b + 144U); + ws0[18U] = u17; + uint64_t u18 = load64_le(b + 152U); + ws0[19U] = u18; + uint64_t u19 = load64_le(b + 160U); + ws0[20U] = u19; + uint64_t u20 = load64_le(b + 168U); + ws0[21U] = u20; + uint64_t u21 = load64_le(b + 176U); + ws0[22U] = u21; + uint64_t u22 = load64_le(b + 184U); + ws0[23U] = u22; + uint64_t u23 = load64_le(b + 192U); + ws0[24U] = u23; + uint64_t u24 = load64_le(b + 200U); + ws0[25U] = u24; + uint64_t u25 = load64_le(b + 208U); + ws0[26U] = u25; + uint64_t u26 = load64_le(b + 216U); + ws0[27U] = u26; + uint64_t u27 = load64_le(b + 224U); + ws0[28U] = u27; + uint64_t u28 = load64_le(b + 232U); + ws0[29U] = u28; + uint64_t u29 = load64_le(b + 240U); + ws0[30U] = u29; + uint64_t u30 = load64_le(b + 248U); + ws0[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) + { + s[i] = s[i] ^ ws0[i]; + } + uint8_t b2[256U] = { 0U }; + uint8_t *b3 = b2; + uint8_t *b0 = b3; + b0[rateInBytes1 - 1U] = 0x80U; + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b3, s); + for (uint32_t i0 = 0U; i0 < outputByteLen / rateInBytes1; i0++) + { + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + uint8_t *b02 = rb; + memcpy(b02 + i0 * rateInBytes1, hbuf, rateInBytes1 * sizeof (uint8_t)); + for (uint32_t i1 = 0U; i1 < 24U; i1++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i2, + 0U, + 5U, + 1U, + uint64_t uu____0 = _C[(i2 + 1U) % 5U]; + uint64_t _D = _C[(i2 + 4U) % 5U] ^ (uu____0 << 1U | uu____0 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i2 + 5U * i] = s[i2 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i1]; + s[0U] = s[0U] ^ c; + } + } + uint32_t remOut = outputByteLen % rateInBytes1; + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + memcpy(rb + outputByteLen - remOut, hbuf, remOut * sizeof (uint8_t)); } -void Hacl_SHA3_sha3_256(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +void Hacl_Hash_SHA3_sha3_224(uint8_t *output, uint8_t *input, uint32_t inputByteLen) { - Hacl_Impl_SHA3_keccak((uint32_t)1088U, - (uint32_t)512U, - inputByteLen, - input, - (uint8_t)0x06U, - (uint32_t)32U, - output); + uint8_t *ib = input; + uint8_t *rb = output; + uint64_t s[25U] = { 0U }; + uint32_t rateInBytes1 = 144U; + for (uint32_t i = 0U; i < inputByteLen / rateInBytes1; i++) + { + uint8_t b[256U] = { 0U }; + uint8_t *b_ = b; + uint8_t *b0 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b0 + i * rateInBytes1, rateInBytes1 * sizeof (uint8_t)); + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b_, s); + } + uint8_t b1[256U] = { 0U }; + uint8_t *b_ = b1; + uint32_t rem = inputByteLen % rateInBytes1; + uint8_t *b00 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b00 + inputByteLen - rem, rem * sizeof (uint8_t)); + uint8_t *b01 = b_; + b01[inputByteLen % rateInBytes1] = 0x06U; + uint64_t ws0[32U] = { 0U }; + uint8_t *b = b_; + uint64_t u = load64_le(b); + ws0[0U] = u; + uint64_t u0 = load64_le(b + 8U); + ws0[1U] = u0; + uint64_t u1 = load64_le(b + 16U); + ws0[2U] = u1; + uint64_t u2 = load64_le(b + 24U); + ws0[3U] = u2; + uint64_t u3 = load64_le(b + 32U); + ws0[4U] = u3; + uint64_t u4 = load64_le(b + 40U); + ws0[5U] = u4; + uint64_t u5 = load64_le(b + 48U); + ws0[6U] = u5; + uint64_t u6 = load64_le(b + 56U); + ws0[7U] = u6; + uint64_t u7 = load64_le(b + 64U); + ws0[8U] = u7; + uint64_t u8 = load64_le(b + 72U); + ws0[9U] = u8; + uint64_t u9 = load64_le(b + 80U); + ws0[10U] = u9; + uint64_t u10 = load64_le(b + 88U); + ws0[11U] = u10; + uint64_t u11 = load64_le(b + 96U); + ws0[12U] = u11; + uint64_t u12 = load64_le(b + 104U); + ws0[13U] = u12; + uint64_t u13 = load64_le(b + 112U); + ws0[14U] = u13; + uint64_t u14 = load64_le(b + 120U); + ws0[15U] = u14; + uint64_t u15 = load64_le(b + 128U); + ws0[16U] = u15; + uint64_t u16 = load64_le(b + 136U); + ws0[17U] = u16; + uint64_t u17 = load64_le(b + 144U); + ws0[18U] = u17; + uint64_t u18 = load64_le(b + 152U); + ws0[19U] = u18; + uint64_t u19 = load64_le(b + 160U); + ws0[20U] = u19; + uint64_t u20 = load64_le(b + 168U); + ws0[21U] = u20; + uint64_t u21 = load64_le(b + 176U); + ws0[22U] = u21; + uint64_t u22 = load64_le(b + 184U); + ws0[23U] = u22; + uint64_t u23 = load64_le(b + 192U); + ws0[24U] = u23; + uint64_t u24 = load64_le(b + 200U); + ws0[25U] = u24; + uint64_t u25 = load64_le(b + 208U); + ws0[26U] = u25; + uint64_t u26 = load64_le(b + 216U); + ws0[27U] = u26; + uint64_t u27 = load64_le(b + 224U); + ws0[28U] = u27; + uint64_t u28 = load64_le(b + 232U); + ws0[29U] = u28; + uint64_t u29 = load64_le(b + 240U); + ws0[30U] = u29; + uint64_t u30 = load64_le(b + 248U); + ws0[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) + { + s[i] = s[i] ^ ws0[i]; + } + uint8_t b2[256U] = { 0U }; + uint8_t *b3 = b2; + uint8_t *b0 = b3; + b0[rateInBytes1 - 1U] = 0x80U; + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b3, s); + for (uint32_t i0 = 0U; i0 < 28U / rateInBytes1; i0++) + { + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + uint8_t *b02 = rb; + memcpy(b02 + i0 * rateInBytes1, hbuf, rateInBytes1 * sizeof (uint8_t)); + for (uint32_t i1 = 0U; i1 < 24U; i1++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i2, + 0U, + 5U, + 1U, + uint64_t uu____0 = _C[(i2 + 1U) % 5U]; + uint64_t _D = _C[(i2 + 4U) % 5U] ^ (uu____0 << 1U | uu____0 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i2 + 5U * i] = s[i2 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i1]; + s[0U] = s[0U] ^ c; + } + } + uint32_t remOut = 28U % rateInBytes1; + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + memcpy(rb + 28U - remOut, hbuf, remOut * sizeof (uint8_t)); } -void Hacl_SHA3_sha3_384(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +void Hacl_Hash_SHA3_sha3_256(uint8_t *output, uint8_t *input, uint32_t inputByteLen) { - Hacl_Impl_SHA3_keccak((uint32_t)832U, - (uint32_t)768U, - inputByteLen, - input, - (uint8_t)0x06U, - (uint32_t)48U, - output); + uint8_t *ib = input; + uint8_t *rb = output; + uint64_t s[25U] = { 0U }; + uint32_t rateInBytes1 = 136U; + for (uint32_t i = 0U; i < inputByteLen / rateInBytes1; i++) + { + uint8_t b[256U] = { 0U }; + uint8_t *b_ = b; + uint8_t *b0 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b0 + i * rateInBytes1, rateInBytes1 * sizeof (uint8_t)); + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b_, s); + } + uint8_t b1[256U] = { 0U }; + uint8_t *b_ = b1; + uint32_t rem = inputByteLen % rateInBytes1; + uint8_t *b00 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b00 + inputByteLen - rem, rem * sizeof (uint8_t)); + uint8_t *b01 = b_; + b01[inputByteLen % rateInBytes1] = 0x06U; + uint64_t ws0[32U] = { 0U }; + uint8_t *b = b_; + uint64_t u = load64_le(b); + ws0[0U] = u; + uint64_t u0 = load64_le(b + 8U); + ws0[1U] = u0; + uint64_t u1 = load64_le(b + 16U); + ws0[2U] = u1; + uint64_t u2 = load64_le(b + 24U); + ws0[3U] = u2; + uint64_t u3 = load64_le(b + 32U); + ws0[4U] = u3; + uint64_t u4 = load64_le(b + 40U); + ws0[5U] = u4; + uint64_t u5 = load64_le(b + 48U); + ws0[6U] = u5; + uint64_t u6 = load64_le(b + 56U); + ws0[7U] = u6; + uint64_t u7 = load64_le(b + 64U); + ws0[8U] = u7; + uint64_t u8 = load64_le(b + 72U); + ws0[9U] = u8; + uint64_t u9 = load64_le(b + 80U); + ws0[10U] = u9; + uint64_t u10 = load64_le(b + 88U); + ws0[11U] = u10; + uint64_t u11 = load64_le(b + 96U); + ws0[12U] = u11; + uint64_t u12 = load64_le(b + 104U); + ws0[13U] = u12; + uint64_t u13 = load64_le(b + 112U); + ws0[14U] = u13; + uint64_t u14 = load64_le(b + 120U); + ws0[15U] = u14; + uint64_t u15 = load64_le(b + 128U); + ws0[16U] = u15; + uint64_t u16 = load64_le(b + 136U); + ws0[17U] = u16; + uint64_t u17 = load64_le(b + 144U); + ws0[18U] = u17; + uint64_t u18 = load64_le(b + 152U); + ws0[19U] = u18; + uint64_t u19 = load64_le(b + 160U); + ws0[20U] = u19; + uint64_t u20 = load64_le(b + 168U); + ws0[21U] = u20; + uint64_t u21 = load64_le(b + 176U); + ws0[22U] = u21; + uint64_t u22 = load64_le(b + 184U); + ws0[23U] = u22; + uint64_t u23 = load64_le(b + 192U); + ws0[24U] = u23; + uint64_t u24 = load64_le(b + 200U); + ws0[25U] = u24; + uint64_t u25 = load64_le(b + 208U); + ws0[26U] = u25; + uint64_t u26 = load64_le(b + 216U); + ws0[27U] = u26; + uint64_t u27 = load64_le(b + 224U); + ws0[28U] = u27; + uint64_t u28 = load64_le(b + 232U); + ws0[29U] = u28; + uint64_t u29 = load64_le(b + 240U); + ws0[30U] = u29; + uint64_t u30 = load64_le(b + 248U); + ws0[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) + { + s[i] = s[i] ^ ws0[i]; + } + uint8_t b2[256U] = { 0U }; + uint8_t *b3 = b2; + uint8_t *b0 = b3; + b0[rateInBytes1 - 1U] = 0x80U; + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b3, s); + for (uint32_t i0 = 0U; i0 < 32U / rateInBytes1; i0++) + { + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + uint8_t *b02 = rb; + memcpy(b02 + i0 * rateInBytes1, hbuf, rateInBytes1 * sizeof (uint8_t)); + for (uint32_t i1 = 0U; i1 < 24U; i1++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i2, + 0U, + 5U, + 1U, + uint64_t uu____0 = _C[(i2 + 1U) % 5U]; + uint64_t _D = _C[(i2 + 4U) % 5U] ^ (uu____0 << 1U | uu____0 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i2 + 5U * i] = s[i2 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i1]; + s[0U] = s[0U] ^ c; + } + } + uint32_t remOut = 32U % rateInBytes1; + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + memcpy(rb + 32U - remOut, hbuf, remOut * sizeof (uint8_t)); } -void Hacl_SHA3_sha3_512(uint32_t inputByteLen, uint8_t *input, uint8_t *output) +void Hacl_Hash_SHA3_sha3_384(uint8_t *output, uint8_t *input, uint32_t inputByteLen) { - Hacl_Impl_SHA3_keccak((uint32_t)576U, - (uint32_t)1024U, - inputByteLen, - input, - (uint8_t)0x06U, - (uint32_t)64U, - output); -} - -static const -uint32_t -keccak_rotc[24U] = + uint8_t *ib = input; + uint8_t *rb = output; + uint64_t s[25U] = { 0U }; + uint32_t rateInBytes1 = 104U; + for (uint32_t i = 0U; i < inputByteLen / rateInBytes1; i++) { - (uint32_t)1U, (uint32_t)3U, (uint32_t)6U, (uint32_t)10U, (uint32_t)15U, (uint32_t)21U, - (uint32_t)28U, (uint32_t)36U, (uint32_t)45U, (uint32_t)55U, (uint32_t)2U, (uint32_t)14U, - (uint32_t)27U, (uint32_t)41U, (uint32_t)56U, (uint32_t)8U, (uint32_t)25U, (uint32_t)43U, - (uint32_t)62U, (uint32_t)18U, (uint32_t)39U, (uint32_t)61U, (uint32_t)20U, (uint32_t)44U - }; - -static const -uint32_t -keccak_piln[24U] = + uint8_t b[256U] = { 0U }; + uint8_t *b_ = b; + uint8_t *b0 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b0 + i * rateInBytes1, rateInBytes1 * sizeof (uint8_t)); + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b_, s); + } + uint8_t b1[256U] = { 0U }; + uint8_t *b_ = b1; + uint32_t rem = inputByteLen % rateInBytes1; + uint8_t *b00 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b00 + inputByteLen - rem, rem * sizeof (uint8_t)); + uint8_t *b01 = b_; + b01[inputByteLen % rateInBytes1] = 0x06U; + uint64_t ws0[32U] = { 0U }; + uint8_t *b = b_; + uint64_t u = load64_le(b); + ws0[0U] = u; + uint64_t u0 = load64_le(b + 8U); + ws0[1U] = u0; + uint64_t u1 = load64_le(b + 16U); + ws0[2U] = u1; + uint64_t u2 = load64_le(b + 24U); + ws0[3U] = u2; + uint64_t u3 = load64_le(b + 32U); + ws0[4U] = u3; + uint64_t u4 = load64_le(b + 40U); + ws0[5U] = u4; + uint64_t u5 = load64_le(b + 48U); + ws0[6U] = u5; + uint64_t u6 = load64_le(b + 56U); + ws0[7U] = u6; + uint64_t u7 = load64_le(b + 64U); + ws0[8U] = u7; + uint64_t u8 = load64_le(b + 72U); + ws0[9U] = u8; + uint64_t u9 = load64_le(b + 80U); + ws0[10U] = u9; + uint64_t u10 = load64_le(b + 88U); + ws0[11U] = u10; + uint64_t u11 = load64_le(b + 96U); + ws0[12U] = u11; + uint64_t u12 = load64_le(b + 104U); + ws0[13U] = u12; + uint64_t u13 = load64_le(b + 112U); + ws0[14U] = u13; + uint64_t u14 = load64_le(b + 120U); + ws0[15U] = u14; + uint64_t u15 = load64_le(b + 128U); + ws0[16U] = u15; + uint64_t u16 = load64_le(b + 136U); + ws0[17U] = u16; + uint64_t u17 = load64_le(b + 144U); + ws0[18U] = u17; + uint64_t u18 = load64_le(b + 152U); + ws0[19U] = u18; + uint64_t u19 = load64_le(b + 160U); + ws0[20U] = u19; + uint64_t u20 = load64_le(b + 168U); + ws0[21U] = u20; + uint64_t u21 = load64_le(b + 176U); + ws0[22U] = u21; + uint64_t u22 = load64_le(b + 184U); + ws0[23U] = u22; + uint64_t u23 = load64_le(b + 192U); + ws0[24U] = u23; + uint64_t u24 = load64_le(b + 200U); + ws0[25U] = u24; + uint64_t u25 = load64_le(b + 208U); + ws0[26U] = u25; + uint64_t u26 = load64_le(b + 216U); + ws0[27U] = u26; + uint64_t u27 = load64_le(b + 224U); + ws0[28U] = u27; + uint64_t u28 = load64_le(b + 232U); + ws0[29U] = u28; + uint64_t u29 = load64_le(b + 240U); + ws0[30U] = u29; + uint64_t u30 = load64_le(b + 248U); + ws0[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) { - (uint32_t)10U, (uint32_t)7U, (uint32_t)11U, (uint32_t)17U, (uint32_t)18U, (uint32_t)3U, - (uint32_t)5U, (uint32_t)16U, (uint32_t)8U, (uint32_t)21U, (uint32_t)24U, (uint32_t)4U, - (uint32_t)15U, (uint32_t)23U, (uint32_t)19U, (uint32_t)13U, (uint32_t)12U, (uint32_t)2U, - (uint32_t)20U, (uint32_t)14U, (uint32_t)22U, (uint32_t)9U, (uint32_t)6U, (uint32_t)1U - }; - -static const -uint64_t -keccak_rndc[24U] = - { - (uint64_t)0x0000000000000001U, (uint64_t)0x0000000000008082U, (uint64_t)0x800000000000808aU, - (uint64_t)0x8000000080008000U, (uint64_t)0x000000000000808bU, (uint64_t)0x0000000080000001U, - (uint64_t)0x8000000080008081U, (uint64_t)0x8000000000008009U, (uint64_t)0x000000000000008aU, - (uint64_t)0x0000000000000088U, (uint64_t)0x0000000080008009U, (uint64_t)0x000000008000000aU, - (uint64_t)0x000000008000808bU, (uint64_t)0x800000000000008bU, (uint64_t)0x8000000000008089U, - (uint64_t)0x8000000000008003U, (uint64_t)0x8000000000008002U, (uint64_t)0x8000000000000080U, - (uint64_t)0x000000000000800aU, (uint64_t)0x800000008000000aU, (uint64_t)0x8000000080008081U, - (uint64_t)0x8000000000008080U, (uint64_t)0x0000000080000001U, (uint64_t)0x8000000080008008U - }; - -void Hacl_Impl_SHA3_state_permute(uint64_t *s) -{ - for (uint32_t i0 = (uint32_t)0U; i0 < (uint32_t)24U; i0++) + s[i] = s[i] ^ ws0[i]; + } + uint8_t b2[256U] = { 0U }; + uint8_t *b3 = b2; + uint8_t *b0 = b3; + b0[rateInBytes1 - 1U] = 0x80U; + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b3, s); + for (uint32_t i0 = 0U; i0 < 48U / rateInBytes1; i0++) { - uint64_t _C[5U] = { 0U }; - KRML_MAYBE_FOR5(i, - (uint32_t)0U, - (uint32_t)5U, - (uint32_t)1U, - _C[i] = - s[i - + (uint32_t)0U] - ^ - (s[i - + (uint32_t)5U] - ^ (s[i + (uint32_t)10U] ^ (s[i + (uint32_t)15U] ^ s[i + (uint32_t)20U])));); - KRML_MAYBE_FOR5(i1, - (uint32_t)0U, - (uint32_t)5U, - (uint32_t)1U, - uint64_t uu____0 = _C[(i1 + (uint32_t)1U) % (uint32_t)5U]; - uint64_t - _D = - _C[(i1 + (uint32_t)4U) - % (uint32_t)5U] - ^ (uu____0 << (uint32_t)1U | uu____0 >> (uint32_t)63U); - KRML_MAYBE_FOR5(i, - (uint32_t)0U, - (uint32_t)5U, - (uint32_t)1U, - s[i1 + (uint32_t)5U * i] = s[i1 + (uint32_t)5U * i] ^ _D;);); - uint64_t x = s[1U]; - uint64_t current = x; - for (uint32_t i = (uint32_t)0U; i < (uint32_t)24U; i++) + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) { - uint32_t _Y = keccak_piln[i]; - uint32_t r = keccak_rotc[i]; - uint64_t temp = s[_Y]; - uint64_t uu____1 = current; - s[_Y] = uu____1 << r | uu____1 >> ((uint32_t)64U - r); - current = temp; + store64_le(hbuf + i * 8U, ws[i]); } - KRML_MAYBE_FOR5(i, - (uint32_t)0U, - (uint32_t)5U, - (uint32_t)1U, - uint64_t - v0 = - s[(uint32_t)0U - + (uint32_t)5U * i] - ^ (~s[(uint32_t)1U + (uint32_t)5U * i] & s[(uint32_t)2U + (uint32_t)5U * i]); - uint64_t - v1 = - s[(uint32_t)1U - + (uint32_t)5U * i] - ^ (~s[(uint32_t)2U + (uint32_t)5U * i] & s[(uint32_t)3U + (uint32_t)5U * i]); - uint64_t - v2 = - s[(uint32_t)2U - + (uint32_t)5U * i] - ^ (~s[(uint32_t)3U + (uint32_t)5U * i] & s[(uint32_t)4U + (uint32_t)5U * i]); - uint64_t - v3 = - s[(uint32_t)3U - + (uint32_t)5U * i] - ^ (~s[(uint32_t)4U + (uint32_t)5U * i] & s[(uint32_t)0U + (uint32_t)5U * i]); - uint64_t - v4 = - s[(uint32_t)4U - + (uint32_t)5U * i] - ^ (~s[(uint32_t)0U + (uint32_t)5U * i] & s[(uint32_t)1U + (uint32_t)5U * i]); - s[(uint32_t)0U + (uint32_t)5U * i] = v0; - s[(uint32_t)1U + (uint32_t)5U * i] = v1; - s[(uint32_t)2U + (uint32_t)5U * i] = v2; - s[(uint32_t)3U + (uint32_t)5U * i] = v3; - s[(uint32_t)4U + (uint32_t)5U * i] = v4;); - uint64_t c = keccak_rndc[i0]; - s[0U] = s[0U] ^ c; + uint8_t *b02 = rb; + memcpy(b02 + i0 * rateInBytes1, hbuf, rateInBytes1 * sizeof (uint8_t)); + for (uint32_t i1 = 0U; i1 < 24U; i1++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i2, + 0U, + 5U, + 1U, + uint64_t uu____0 = _C[(i2 + 1U) % 5U]; + uint64_t _D = _C[(i2 + 4U) % 5U] ^ (uu____0 << 1U | uu____0 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i2 + 5U * i] = s[i2 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i1]; + s[0U] = s[0U] ^ c; + } + } + uint32_t remOut = 48U % rateInBytes1; + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); } + memcpy(rb + 48U - remOut, hbuf, remOut * sizeof (uint8_t)); } -void Hacl_Impl_SHA3_loadState(uint32_t rateInBytes, uint8_t *input, uint64_t *s) +void Hacl_Hash_SHA3_sha3_512(uint8_t *output, uint8_t *input, uint32_t inputByteLen) { - uint8_t block[200U] = { 0U }; - memcpy(block, input, rateInBytes * sizeof (uint8_t)); - for (uint32_t i = (uint32_t)0U; i < (uint32_t)25U; i++) + uint8_t *ib = input; + uint8_t *rb = output; + uint64_t s[25U] = { 0U }; + uint32_t rateInBytes1 = 72U; + for (uint32_t i = 0U; i < inputByteLen / rateInBytes1; i++) { - uint64_t u = load64_le(block + i * (uint32_t)8U); - uint64_t x = u; - s[i] = s[i] ^ x; + uint8_t b[256U] = { 0U }; + uint8_t *b_ = b; + uint8_t *b0 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b0 + i * rateInBytes1, rateInBytes1 * sizeof (uint8_t)); + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b_, s); } + uint8_t b1[256U] = { 0U }; + uint8_t *b_ = b1; + uint32_t rem = inputByteLen % rateInBytes1; + uint8_t *b00 = ib; + uint8_t *bl0 = b_; + memcpy(bl0, b00 + inputByteLen - rem, rem * sizeof (uint8_t)); + uint8_t *b01 = b_; + b01[inputByteLen % rateInBytes1] = 0x06U; + uint64_t ws0[32U] = { 0U }; + uint8_t *b = b_; + uint64_t u = load64_le(b); + ws0[0U] = u; + uint64_t u0 = load64_le(b + 8U); + ws0[1U] = u0; + uint64_t u1 = load64_le(b + 16U); + ws0[2U] = u1; + uint64_t u2 = load64_le(b + 24U); + ws0[3U] = u2; + uint64_t u3 = load64_le(b + 32U); + ws0[4U] = u3; + uint64_t u4 = load64_le(b + 40U); + ws0[5U] = u4; + uint64_t u5 = load64_le(b + 48U); + ws0[6U] = u5; + uint64_t u6 = load64_le(b + 56U); + ws0[7U] = u6; + uint64_t u7 = load64_le(b + 64U); + ws0[8U] = u7; + uint64_t u8 = load64_le(b + 72U); + ws0[9U] = u8; + uint64_t u9 = load64_le(b + 80U); + ws0[10U] = u9; + uint64_t u10 = load64_le(b + 88U); + ws0[11U] = u10; + uint64_t u11 = load64_le(b + 96U); + ws0[12U] = u11; + uint64_t u12 = load64_le(b + 104U); + ws0[13U] = u12; + uint64_t u13 = load64_le(b + 112U); + ws0[14U] = u13; + uint64_t u14 = load64_le(b + 120U); + ws0[15U] = u14; + uint64_t u15 = load64_le(b + 128U); + ws0[16U] = u15; + uint64_t u16 = load64_le(b + 136U); + ws0[17U] = u16; + uint64_t u17 = load64_le(b + 144U); + ws0[18U] = u17; + uint64_t u18 = load64_le(b + 152U); + ws0[19U] = u18; + uint64_t u19 = load64_le(b + 160U); + ws0[20U] = u19; + uint64_t u20 = load64_le(b + 168U); + ws0[21U] = u20; + uint64_t u21 = load64_le(b + 176U); + ws0[22U] = u21; + uint64_t u22 = load64_le(b + 184U); + ws0[23U] = u22; + uint64_t u23 = load64_le(b + 192U); + ws0[24U] = u23; + uint64_t u24 = load64_le(b + 200U); + ws0[25U] = u24; + uint64_t u25 = load64_le(b + 208U); + ws0[26U] = u25; + uint64_t u26 = load64_le(b + 216U); + ws0[27U] = u26; + uint64_t u27 = load64_le(b + 224U); + ws0[28U] = u27; + uint64_t u28 = load64_le(b + 232U); + ws0[29U] = u28; + uint64_t u29 = load64_le(b + 240U); + ws0[30U] = u29; + uint64_t u30 = load64_le(b + 248U); + ws0[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) + { + s[i] = s[i] ^ ws0[i]; + } + uint8_t b2[256U] = { 0U }; + uint8_t *b3 = b2; + uint8_t *b0 = b3; + b0[rateInBytes1 - 1U] = 0x80U; + Hacl_Hash_SHA3_absorb_inner_32(rateInBytes1, b3, s); + for (uint32_t i0 = 0U; i0 < 64U / rateInBytes1; i0++) + { + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + uint8_t *b02 = rb; + memcpy(b02 + i0 * rateInBytes1, hbuf, rateInBytes1 * sizeof (uint8_t)); + for (uint32_t i1 = 0U; i1 < 24U; i1++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = s[i + 0U] ^ (s[i + 5U] ^ (s[i + 10U] ^ (s[i + 15U] ^ s[i + 20U])));); + KRML_MAYBE_FOR5(i2, + 0U, + 5U, + 1U, + uint64_t uu____0 = _C[(i2 + 1U) % 5U]; + uint64_t _D = _C[(i2 + 4U) % 5U] ^ (uu____0 << 1U | uu____0 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, s[i2 + 5U * i] = s[i2 + 5U * i] ^ _D;);); + uint64_t x = s[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = s[_Y]; + uint64_t uu____1 = current; + s[_Y] = uu____1 << r | uu____1 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = s[0U + 5U * i] ^ (~s[1U + 5U * i] & s[2U + 5U * i]); + uint64_t v1 = s[1U + 5U * i] ^ (~s[2U + 5U * i] & s[3U + 5U * i]); + uint64_t v2 = s[2U + 5U * i] ^ (~s[3U + 5U * i] & s[4U + 5U * i]); + uint64_t v3 = s[3U + 5U * i] ^ (~s[4U + 5U * i] & s[0U + 5U * i]); + uint64_t v4 = s[4U + 5U * i] ^ (~s[0U + 5U * i] & s[1U + 5U * i]); + s[0U + 5U * i] = v0; + s[1U + 5U * i] = v1; + s[2U + 5U * i] = v2; + s[3U + 5U * i] = v3; + s[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i1]; + s[0U] = s[0U] ^ c; + } + } + uint32_t remOut = 64U % rateInBytes1; + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, s, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + memcpy(rb + 64U - remOut, hbuf, remOut * sizeof (uint8_t)); } -static void storeState(uint32_t rateInBytes, uint64_t *s, uint8_t *res) +/** +Allocate state buffer of 200-bytes +*/ +uint64_t *Hacl_Hash_SHA3_state_malloc(void) { - uint8_t block[200U] = { 0U }; - for (uint32_t i = (uint32_t)0U; i < (uint32_t)25U; i++) - { - uint64_t sj = s[i]; - store64_le(block + i * (uint32_t)8U, sj); - } - memcpy(res, block, rateInBytes * sizeof (uint8_t)); + uint64_t *buf = (uint64_t *)KRML_HOST_CALLOC(25U, sizeof (uint64_t)); + return buf; } -void Hacl_Impl_SHA3_absorb_inner(uint32_t rateInBytes, uint8_t *block, uint64_t *s) +/** +Free state buffer +*/ +void Hacl_Hash_SHA3_state_free(uint64_t *s) { - Hacl_Impl_SHA3_loadState(rateInBytes, block, s); - Hacl_Impl_SHA3_state_permute(s); + KRML_HOST_FREE(s); } -static void -absorb( - uint64_t *s, - uint32_t rateInBytes, - uint32_t inputByteLen, - uint8_t *input, - uint8_t delimitedSuffix -) +/** +Absorb number of input blocks and write the output state + + This function is intended to receive a hash state and input buffer. + It processes an input of multiple of 168-bytes (SHAKE128 block size), + any additional bytes of final partial block are ignored. + + The argument `state` (IN/OUT) points to hash state, i.e., uint64_t[25] + The argument `input` (IN) points to `inputByteLen` bytes of valid memory, + i.e., uint8_t[inputByteLen] +*/ +void +Hacl_Hash_SHA3_shake128_absorb_nblocks(uint64_t *state, uint8_t *input, uint32_t inputByteLen) { - uint32_t n_blocks = inputByteLen / rateInBytes; - uint32_t rem = inputByteLen % rateInBytes; - for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) - { - uint8_t *block = input + i * rateInBytes; - Hacl_Impl_SHA3_absorb_inner(rateInBytes, block, s); - } - uint8_t *last = input + n_blocks * rateInBytes; - uint8_t lastBlock_[200U] = { 0U }; - uint8_t *lastBlock = lastBlock_; - memcpy(lastBlock, last, rem * sizeof (uint8_t)); - lastBlock[rem] = delimitedSuffix; - Hacl_Impl_SHA3_loadState(rateInBytes, lastBlock, s); - if (!((delimitedSuffix & (uint8_t)0x80U) == (uint8_t)0U) && rem == rateInBytes - (uint32_t)1U) - { - Hacl_Impl_SHA3_state_permute(s); - } - uint8_t nextBlock_[200U] = { 0U }; - uint8_t *nextBlock = nextBlock_; - nextBlock[rateInBytes - (uint32_t)1U] = (uint8_t)0x80U; - Hacl_Impl_SHA3_loadState(rateInBytes, nextBlock, s); - Hacl_Impl_SHA3_state_permute(s); + for (uint32_t i = 0U; i < inputByteLen / 168U; i++) + { + uint8_t b[256U] = { 0U }; + uint8_t *b_ = b; + uint8_t *b0 = input; + uint8_t *bl0 = b_; + memcpy(bl0, b0 + i * 168U, 168U * sizeof (uint8_t)); + Hacl_Hash_SHA3_absorb_inner_32(168U, b_, state); + } } +/** +Absorb a final partial block of input and write the output state + + This function is intended to receive a hash state and input buffer. + It processes a sequence of bytes at end of input buffer that is less + than 168-bytes (SHAKE128 block size), + any bytes of full blocks at start of input buffer are ignored. + + The argument `state` (IN/OUT) points to hash state, i.e., uint64_t[25] + The argument `input` (IN) points to `inputByteLen` bytes of valid memory, + i.e., uint8_t[inputByteLen] + + Note: Full size of input buffer must be passed to `inputByteLen` including + the number of full-block bytes at start of input buffer that are ignored +*/ void -Hacl_Impl_SHA3_squeeze( - uint64_t *s, - uint32_t rateInBytes, - uint32_t outputByteLen, - uint8_t *output -) +Hacl_Hash_SHA3_shake128_absorb_final(uint64_t *state, uint8_t *input, uint32_t inputByteLen) { - uint32_t outBlocks = outputByteLen / rateInBytes; - uint32_t remOut = outputByteLen % rateInBytes; - uint8_t *last = output + outputByteLen - remOut; - uint8_t *blocks = output; - for (uint32_t i = (uint32_t)0U; i < outBlocks; i++) + uint8_t b1[256U] = { 0U }; + uint8_t *b_ = b1; + uint32_t rem = inputByteLen % 168U; + uint8_t *b00 = input; + uint8_t *bl0 = b_; + memcpy(bl0, b00 + inputByteLen - rem, rem * sizeof (uint8_t)); + uint8_t *b01 = b_; + b01[inputByteLen % 168U] = 0x1FU; + uint64_t ws[32U] = { 0U }; + uint8_t *b = b_; + uint64_t u = load64_le(b); + ws[0U] = u; + uint64_t u0 = load64_le(b + 8U); + ws[1U] = u0; + uint64_t u1 = load64_le(b + 16U); + ws[2U] = u1; + uint64_t u2 = load64_le(b + 24U); + ws[3U] = u2; + uint64_t u3 = load64_le(b + 32U); + ws[4U] = u3; + uint64_t u4 = load64_le(b + 40U); + ws[5U] = u4; + uint64_t u5 = load64_le(b + 48U); + ws[6U] = u5; + uint64_t u6 = load64_le(b + 56U); + ws[7U] = u6; + uint64_t u7 = load64_le(b + 64U); + ws[8U] = u7; + uint64_t u8 = load64_le(b + 72U); + ws[9U] = u8; + uint64_t u9 = load64_le(b + 80U); + ws[10U] = u9; + uint64_t u10 = load64_le(b + 88U); + ws[11U] = u10; + uint64_t u11 = load64_le(b + 96U); + ws[12U] = u11; + uint64_t u12 = load64_le(b + 104U); + ws[13U] = u12; + uint64_t u13 = load64_le(b + 112U); + ws[14U] = u13; + uint64_t u14 = load64_le(b + 120U); + ws[15U] = u14; + uint64_t u15 = load64_le(b + 128U); + ws[16U] = u15; + uint64_t u16 = load64_le(b + 136U); + ws[17U] = u16; + uint64_t u17 = load64_le(b + 144U); + ws[18U] = u17; + uint64_t u18 = load64_le(b + 152U); + ws[19U] = u18; + uint64_t u19 = load64_le(b + 160U); + ws[20U] = u19; + uint64_t u20 = load64_le(b + 168U); + ws[21U] = u20; + uint64_t u21 = load64_le(b + 176U); + ws[22U] = u21; + uint64_t u22 = load64_le(b + 184U); + ws[23U] = u22; + uint64_t u23 = load64_le(b + 192U); + ws[24U] = u23; + uint64_t u24 = load64_le(b + 200U); + ws[25U] = u24; + uint64_t u25 = load64_le(b + 208U); + ws[26U] = u25; + uint64_t u26 = load64_le(b + 216U); + ws[27U] = u26; + uint64_t u27 = load64_le(b + 224U); + ws[28U] = u27; + uint64_t u28 = load64_le(b + 232U); + ws[29U] = u28; + uint64_t u29 = load64_le(b + 240U); + ws[30U] = u29; + uint64_t u30 = load64_le(b + 248U); + ws[31U] = u30; + for (uint32_t i = 0U; i < 25U; i++) { - storeState(rateInBytes, s, blocks + i * rateInBytes); - Hacl_Impl_SHA3_state_permute(s); + state[i] = state[i] ^ ws[i]; } - storeState(remOut, s, last); + uint8_t b2[256U] = { 0U }; + uint8_t *b3 = b2; + uint8_t *b0 = b3; + b0[167U] = 0x80U; + Hacl_Hash_SHA3_absorb_inner_32(168U, b3, state); } +/** +Squeeze a hash state to output buffer + + This function is intended to receive a hash state and output buffer. + It produces an output of multiple of 168-bytes (SHAKE128 block size), + any additional bytes of final partial block are ignored. + + The argument `state` (IN) points to hash state, i.e., uint64_t[25] + The argument `output` (OUT) points to `outputByteLen` bytes of valid memory, + i.e., uint8_t[outputByteLen] +*/ void -Hacl_Impl_SHA3_keccak( - uint32_t rate, - uint32_t capacity, - uint32_t inputByteLen, - uint8_t *input, - uint8_t delimitedSuffix, - uint32_t outputByteLen, - uint8_t *output +Hacl_Hash_SHA3_shake128_squeeze_nblocks( + uint64_t *state, + uint8_t *output, + uint32_t outputByteLen ) { - uint32_t rateInBytes = rate / (uint32_t)8U; - uint64_t s[25U] = { 0U }; - absorb(s, rateInBytes, inputByteLen, input, delimitedSuffix); - Hacl_Impl_SHA3_squeeze(s, rateInBytes, outputByteLen, output); + for (uint32_t i0 = 0U; i0 < outputByteLen / 168U; i0++) + { + uint8_t hbuf[256U] = { 0U }; + uint64_t ws[32U] = { 0U }; + memcpy(ws, state, 25U * sizeof (uint64_t)); + for (uint32_t i = 0U; i < 32U; i++) + { + store64_le(hbuf + i * 8U, ws[i]); + } + uint8_t *b0 = output; + memcpy(b0 + i0 * 168U, hbuf, 168U * sizeof (uint8_t)); + for (uint32_t i1 = 0U; i1 < 24U; i1++) + { + uint64_t _C[5U] = { 0U }; + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + _C[i] = + state[i + + 0U] + ^ (state[i + 5U] ^ (state[i + 10U] ^ (state[i + 15U] ^ state[i + 20U])));); + KRML_MAYBE_FOR5(i2, + 0U, + 5U, + 1U, + uint64_t uu____0 = _C[(i2 + 1U) % 5U]; + uint64_t _D = _C[(i2 + 4U) % 5U] ^ (uu____0 << 1U | uu____0 >> 63U); + KRML_MAYBE_FOR5(i, 0U, 5U, 1U, state[i2 + 5U * i] = state[i2 + 5U * i] ^ _D;);); + uint64_t x = state[1U]; + uint64_t current = x; + for (uint32_t i = 0U; i < 24U; i++) + { + uint32_t _Y = Hacl_Hash_SHA3_keccak_piln[i]; + uint32_t r = Hacl_Hash_SHA3_keccak_rotc[i]; + uint64_t temp = state[_Y]; + uint64_t uu____1 = current; + state[_Y] = uu____1 << r | uu____1 >> (64U - r); + current = temp; + } + KRML_MAYBE_FOR5(i, + 0U, + 5U, + 1U, + uint64_t v0 = state[0U + 5U * i] ^ (~state[1U + 5U * i] & state[2U + 5U * i]); + uint64_t v1 = state[1U + 5U * i] ^ (~state[2U + 5U * i] & state[3U + 5U * i]); + uint64_t v2 = state[2U + 5U * i] ^ (~state[3U + 5U * i] & state[4U + 5U * i]); + uint64_t v3 = state[3U + 5U * i] ^ (~state[4U + 5U * i] & state[0U + 5U * i]); + uint64_t v4 = state[4U + 5U * i] ^ (~state[0U + 5U * i] & state[1U + 5U * i]); + state[0U + 5U * i] = v0; + state[1U + 5U * i] = v1; + state[2U + 5U * i] = v2; + state[3U + 5U * i] = v3; + state[4U + 5U * i] = v4;); + uint64_t c = Hacl_Hash_SHA3_keccak_rndc[i1]; + state[0U] = state[0U] ^ c; + } + } } diff --git a/Modules/_hacl/Hacl_Hash_SHA3.h b/Modules/_hacl/Hacl_Hash_SHA3.h index 681b6af4a80e77..4d85bb6902cc5b 100644 --- a/Modules/_hacl/Hacl_Hash_SHA3.h +++ b/Modules/_hacl/Hacl_Hash_SHA3.h @@ -31,95 +31,137 @@ extern "C" { #endif #include +#include "python_hacl_namespaces.h" #include "krml/types.h" #include "krml/lowstar_endianness.h" #include "krml/internal/target.h" #include "Hacl_Streaming_Types.h" -typedef struct Hacl_Streaming_Keccak_hash_buf_s +typedef struct Hacl_Hash_SHA3_hash_buf_s { Spec_Hash_Definitions_hash_alg fst; uint64_t *snd; } -Hacl_Streaming_Keccak_hash_buf; +Hacl_Hash_SHA3_hash_buf; -typedef struct Hacl_Streaming_Keccak_state_s +typedef struct Hacl_Hash_SHA3_state_t_s { - Hacl_Streaming_Keccak_hash_buf block_state; + Hacl_Hash_SHA3_hash_buf block_state; uint8_t *buf; uint64_t total_len; } -Hacl_Streaming_Keccak_state; +Hacl_Hash_SHA3_state_t; -Spec_Hash_Definitions_hash_alg Hacl_Streaming_Keccak_get_alg(Hacl_Streaming_Keccak_state *s); +Spec_Hash_Definitions_hash_alg Hacl_Hash_SHA3_get_alg(Hacl_Hash_SHA3_state_t *s); -Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_hash_alg a); +Hacl_Hash_SHA3_state_t *Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_hash_alg a); -void Hacl_Streaming_Keccak_free(Hacl_Streaming_Keccak_state *s); +void Hacl_Hash_SHA3_free(Hacl_Hash_SHA3_state_t *state); -Hacl_Streaming_Keccak_state *Hacl_Streaming_Keccak_copy(Hacl_Streaming_Keccak_state *s0); +Hacl_Hash_SHA3_state_t *Hacl_Hash_SHA3_copy(Hacl_Hash_SHA3_state_t *state); -void Hacl_Streaming_Keccak_reset(Hacl_Streaming_Keccak_state *s); +void Hacl_Hash_SHA3_reset(Hacl_Hash_SHA3_state_t *state); Hacl_Streaming_Types_error_code -Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint32_t len); +Hacl_Hash_SHA3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *chunk, uint32_t chunk_len); Hacl_Streaming_Types_error_code -Hacl_Streaming_Keccak_finish(Hacl_Streaming_Keccak_state *s, uint8_t *dst); +Hacl_Hash_SHA3_digest(Hacl_Hash_SHA3_state_t *state, uint8_t *output); Hacl_Streaming_Types_error_code -Hacl_Streaming_Keccak_squeeze(Hacl_Streaming_Keccak_state *s, uint8_t *dst, uint32_t l); +Hacl_Hash_SHA3_squeeze(Hacl_Hash_SHA3_state_t *s, uint8_t *dst, uint32_t l); -uint32_t Hacl_Streaming_Keccak_block_len(Hacl_Streaming_Keccak_state *s); +uint32_t Hacl_Hash_SHA3_block_len(Hacl_Hash_SHA3_state_t *s); -uint32_t Hacl_Streaming_Keccak_hash_len(Hacl_Streaming_Keccak_state *s); +uint32_t Hacl_Hash_SHA3_hash_len(Hacl_Hash_SHA3_state_t *s); -bool Hacl_Streaming_Keccak_is_shake(Hacl_Streaming_Keccak_state *s); +bool Hacl_Hash_SHA3_is_shake(Hacl_Hash_SHA3_state_t *s); + +void Hacl_Hash_SHA3_absorb_inner_32(uint32_t rateInBytes, uint8_t *b, uint64_t *s); void -Hacl_SHA3_shake128_hacl( - uint32_t inputByteLen, - uint8_t *input, +Hacl_Hash_SHA3_shake128( + uint8_t *output, uint32_t outputByteLen, - uint8_t *output + uint8_t *input, + uint32_t inputByteLen ); void -Hacl_SHA3_shake256_hacl( - uint32_t inputByteLen, - uint8_t *input, +Hacl_Hash_SHA3_shake256( + uint8_t *output, uint32_t outputByteLen, - uint8_t *output + uint8_t *input, + uint32_t inputByteLen ); -void Hacl_SHA3_sha3_224(uint32_t inputByteLen, uint8_t *input, uint8_t *output); +void Hacl_Hash_SHA3_sha3_224(uint8_t *output, uint8_t *input, uint32_t inputByteLen); -void Hacl_SHA3_sha3_256(uint32_t inputByteLen, uint8_t *input, uint8_t *output); +void Hacl_Hash_SHA3_sha3_256(uint8_t *output, uint8_t *input, uint32_t inputByteLen); -void Hacl_SHA3_sha3_384(uint32_t inputByteLen, uint8_t *input, uint8_t *output); +void Hacl_Hash_SHA3_sha3_384(uint8_t *output, uint8_t *input, uint32_t inputByteLen); -void Hacl_SHA3_sha3_512(uint32_t inputByteLen, uint8_t *input, uint8_t *output); +void Hacl_Hash_SHA3_sha3_512(uint8_t *output, uint8_t *input, uint32_t inputByteLen); -void Hacl_Impl_SHA3_absorb_inner(uint32_t rateInBytes, uint8_t *block, uint64_t *s); +/** +Allocate state buffer of 200-bytes +*/ +uint64_t *Hacl_Hash_SHA3_state_malloc(void); +/** +Free state buffer +*/ +void Hacl_Hash_SHA3_state_free(uint64_t *s); + +/** +Absorb number of input blocks and write the output state + + This function is intended to receive a hash state and input buffer. + It processes an input of multiple of 168-bytes (SHAKE128 block size), + any additional bytes of final partial block are ignored. + + The argument `state` (IN/OUT) points to hash state, i.e., uint64_t[25] + The argument `input` (IN) points to `inputByteLen` bytes of valid memory, + i.e., uint8_t[inputByteLen] +*/ void -Hacl_Impl_SHA3_squeeze( - uint64_t *s, - uint32_t rateInBytes, - uint32_t outputByteLen, - uint8_t *output -); +Hacl_Hash_SHA3_shake128_absorb_nblocks(uint64_t *state, uint8_t *input, uint32_t inputByteLen); +/** +Absorb a final partial block of input and write the output state + + This function is intended to receive a hash state and input buffer. + It processes a sequence of bytes at end of input buffer that is less + than 168-bytes (SHAKE128 block size), + any bytes of full blocks at start of input buffer are ignored. + + The argument `state` (IN/OUT) points to hash state, i.e., uint64_t[25] + The argument `input` (IN) points to `inputByteLen` bytes of valid memory, + i.e., uint8_t[inputByteLen] + + Note: Full size of input buffer must be passed to `inputByteLen` including + the number of full-block bytes at start of input buffer that are ignored +*/ void -Hacl_Impl_SHA3_keccak( - uint32_t rate, - uint32_t capacity, - uint32_t inputByteLen, - uint8_t *input, - uint8_t delimitedSuffix, - uint32_t outputByteLen, - uint8_t *output +Hacl_Hash_SHA3_shake128_absorb_final(uint64_t *state, uint8_t *input, uint32_t inputByteLen); + +/** +Squeeze a hash state to output buffer + + This function is intended to receive a hash state and output buffer. + It produces an output of multiple of 168-bytes (SHAKE128 block size), + any additional bytes of final partial block are ignored. + + The argument `state` (IN) points to hash state, i.e., uint64_t[25] + The argument `output` (OUT) points to `outputByteLen` bytes of valid memory, + i.e., uint8_t[outputByteLen] +*/ +void +Hacl_Hash_SHA3_shake128_squeeze_nblocks( + uint64_t *state, + uint8_t *output, + uint32_t outputByteLen ); #if defined(__cplusplus) diff --git a/Modules/_hacl/Lib_Memzero0.c b/Modules/_hacl/Lib_Memzero0.c new file mode 100644 index 00000000000000..f01568a138648f --- /dev/null +++ b/Modules/_hacl/Lib_Memzero0.c @@ -0,0 +1,58 @@ +#if defined(__has_include) +#if __has_include("config.h") +#include "config.h" +#endif +#endif + +#ifdef _WIN32 +#include +#endif + +#if defined(__APPLE__) && defined(__MACH__) +#include +#endif + +#if (defined(__APPLE__) && defined(__MACH__)) || defined(__linux__) +#define __STDC_WANT_LIB_EXT1__ 1 +#include +#endif + +#if defined(__FreeBSD__) || defined(__NetBSD__) +#include +#endif + +#include +#include +#include +#include + +/* This is now a hand-written header */ +#include "lib_memzero0.h" +#include "krml/internal/target.h" + +/* The F* formalization talks about the number of elements in the array. The C + implementation wants a number of bytes in the array. KaRaMeL is aware of this + and inserts a sizeof multiplication. */ +void Lib_Memzero0_memzero0(void *dst, uint64_t len) { + /* This is safe: karamel checks at run-time (if needed) that all object sizes + fit within a size_t, so the size we receive has been checked at + allocation-time, possibly via KRML_CHECK_SIZE, to fit in a size_t. */ + size_t len_ = (size_t) len; + + #ifdef _WIN32 + SecureZeroMemory(dst, len_); + #elif defined(__APPLE__) && defined(__MACH__) && defined(MAC_OS_X_VERSION_MIN_REQUIRED) && (MAC_OS_X_VERSION_MIN_REQUIRED >= 1090) + memset_s(dst, len_, 0, len_); + #elif (defined(__linux__) && !defined(LINUX_NO_EXPLICIT_BZERO)) || defined(__FreeBSD__) + explicit_bzero(dst, len_); + #elif defined(__NetBSD__) + explicit_memset(dst, 0, len_); + #else + /* Default implementation for platforms with no particular support. */ + #warning "Your platform does not support any safe implementation of memzero -- consider a pull request!" + volatile unsigned char *volatile dst_ = (volatile unsigned char *volatile) dst; + size_t i = 0U; + while (i < len) + dst_[i++] = 0U; + #endif +} diff --git a/Modules/_hacl/include/krml/FStar_UInt128_Verified.h b/Modules/_hacl/include/krml/FStar_UInt128_Verified.h index 3d36d440735530..659745b24265cb 100644 --- a/Modules/_hacl/include/krml/FStar_UInt128_Verified.h +++ b/Modules/_hacl/include/krml/FStar_UInt128_Verified.h @@ -1,6 +1,6 @@ /* Copyright (c) INRIA and Microsoft Corporation. All rights reserved. - Licensed under the Apache 2.0 License. + Licensed under the Apache 2.0 and MIT Licenses. */ @@ -15,7 +15,7 @@ static inline uint64_t FStar_UInt128_constant_time_carry(uint64_t a, uint64_t b) { - return (a ^ ((a ^ b) | ((a - b) ^ b))) >> (uint32_t)63U; + return (a ^ ((a ^ b) | ((a - b) ^ b))) >> 63U; } static inline uint64_t FStar_UInt128_carry(uint64_t a, uint64_t b) @@ -118,7 +118,7 @@ static inline FStar_UInt128_uint128 FStar_UInt128_lognot(FStar_UInt128_uint128 a return lit; } -static uint32_t FStar_UInt128_u32_64 = (uint32_t)64U; +static uint32_t FStar_UInt128_u32_64 = 64U; static inline uint64_t FStar_UInt128_add_u64_shift_left(uint64_t hi, uint64_t lo, uint32_t s) { @@ -134,7 +134,7 @@ FStar_UInt128_add_u64_shift_left_respec(uint64_t hi, uint64_t lo, uint32_t s) static inline FStar_UInt128_uint128 FStar_UInt128_shift_left_small(FStar_UInt128_uint128 a, uint32_t s) { - if (s == (uint32_t)0U) + if (s == 0U) { return a; } @@ -151,7 +151,7 @@ static inline FStar_UInt128_uint128 FStar_UInt128_shift_left_large(FStar_UInt128_uint128 a, uint32_t s) { FStar_UInt128_uint128 lit; - lit.low = (uint64_t)0U; + lit.low = 0ULL; lit.high = a.low << (s - FStar_UInt128_u32_64); return lit; } @@ -183,7 +183,7 @@ FStar_UInt128_add_u64_shift_right_respec(uint64_t hi, uint64_t lo, uint32_t s) static inline FStar_UInt128_uint128 FStar_UInt128_shift_right_small(FStar_UInt128_uint128 a, uint32_t s) { - if (s == (uint32_t)0U) + if (s == 0U) { return a; } @@ -201,7 +201,7 @@ FStar_UInt128_shift_right_large(FStar_UInt128_uint128 a, uint32_t s) { FStar_UInt128_uint128 lit; lit.low = a.high >> (s - FStar_UInt128_u32_64); - lit.high = (uint64_t)0U; + lit.high = 0ULL; return lit; } @@ -269,7 +269,7 @@ static inline FStar_UInt128_uint128 FStar_UInt128_uint64_to_uint128(uint64_t a) { FStar_UInt128_uint128 lit; lit.low = a; - lit.high = (uint64_t)0U; + lit.high = 0ULL; return lit; } @@ -280,10 +280,10 @@ static inline uint64_t FStar_UInt128_uint128_to_uint64(FStar_UInt128_uint128 a) static inline uint64_t FStar_UInt128_u64_mod_32(uint64_t a) { - return a & (uint64_t)0xffffffffU; + return a & 0xffffffffULL; } -static uint32_t FStar_UInt128_u32_32 = (uint32_t)32U; +static uint32_t FStar_UInt128_u32_32 = 32U; static inline uint64_t FStar_UInt128_u32_combine(uint64_t hi, uint64_t lo) { diff --git a/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h b/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h index a56c7d613498b7..68bac0b3f0aab1 100644 --- a/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h +++ b/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h @@ -1,6 +1,6 @@ /* Copyright (c) INRIA and Microsoft Corporation. All rights reserved. - Licensed under the Apache 2.0 License. + Licensed under the Apache 2.0 and MIT Licenses. */ @@ -14,16 +14,16 @@ #include "krml/types.h" #include "krml/internal/target.h" -static inline uint64_t FStar_UInt64_eq_mask(uint64_t a, uint64_t b) +static KRML_NOINLINE uint64_t FStar_UInt64_eq_mask(uint64_t a, uint64_t b) { uint64_t x = a ^ b; - uint64_t minus_x = ~x + (uint64_t)1U; + uint64_t minus_x = ~x + 1ULL; uint64_t x_or_minus_x = x | minus_x; - uint64_t xnx = x_or_minus_x >> (uint32_t)63U; - return xnx - (uint64_t)1U; + uint64_t xnx = x_or_minus_x >> 63U; + return xnx - 1ULL; } -static inline uint64_t FStar_UInt64_gte_mask(uint64_t a, uint64_t b) +static KRML_NOINLINE uint64_t FStar_UInt64_gte_mask(uint64_t a, uint64_t b) { uint64_t x = a; uint64_t y = b; @@ -32,20 +32,20 @@ static inline uint64_t FStar_UInt64_gte_mask(uint64_t a, uint64_t b) uint64_t x_sub_y_xor_y = x_sub_y ^ y; uint64_t q = x_xor_y | x_sub_y_xor_y; uint64_t x_xor_q = x ^ q; - uint64_t x_xor_q_ = x_xor_q >> (uint32_t)63U; - return x_xor_q_ - (uint64_t)1U; + uint64_t x_xor_q_ = x_xor_q >> 63U; + return x_xor_q_ - 1ULL; } -static inline uint32_t FStar_UInt32_eq_mask(uint32_t a, uint32_t b) +static KRML_NOINLINE uint32_t FStar_UInt32_eq_mask(uint32_t a, uint32_t b) { uint32_t x = a ^ b; - uint32_t minus_x = ~x + (uint32_t)1U; + uint32_t minus_x = ~x + 1U; uint32_t x_or_minus_x = x | minus_x; - uint32_t xnx = x_or_minus_x >> (uint32_t)31U; - return xnx - (uint32_t)1U; + uint32_t xnx = x_or_minus_x >> 31U; + return xnx - 1U; } -static inline uint32_t FStar_UInt32_gte_mask(uint32_t a, uint32_t b) +static KRML_NOINLINE uint32_t FStar_UInt32_gte_mask(uint32_t a, uint32_t b) { uint32_t x = a; uint32_t y = b; @@ -54,52 +54,52 @@ static inline uint32_t FStar_UInt32_gte_mask(uint32_t a, uint32_t b) uint32_t x_sub_y_xor_y = x_sub_y ^ y; uint32_t q = x_xor_y | x_sub_y_xor_y; uint32_t x_xor_q = x ^ q; - uint32_t x_xor_q_ = x_xor_q >> (uint32_t)31U; - return x_xor_q_ - (uint32_t)1U; + uint32_t x_xor_q_ = x_xor_q >> 31U; + return x_xor_q_ - 1U; } -static inline uint16_t FStar_UInt16_eq_mask(uint16_t a, uint16_t b) +static KRML_NOINLINE uint16_t FStar_UInt16_eq_mask(uint16_t a, uint16_t b) { - uint16_t x = a ^ b; - uint16_t minus_x = ~x + (uint16_t)1U; - uint16_t x_or_minus_x = x | minus_x; - uint16_t xnx = x_or_minus_x >> (uint32_t)15U; - return xnx - (uint16_t)1U; + uint16_t x = (uint32_t)a ^ (uint32_t)b; + uint16_t minus_x = (uint32_t)~x + 1U; + uint16_t x_or_minus_x = (uint32_t)x | (uint32_t)minus_x; + uint16_t xnx = (uint32_t)x_or_minus_x >> 15U; + return (uint32_t)xnx - 1U; } -static inline uint16_t FStar_UInt16_gte_mask(uint16_t a, uint16_t b) +static KRML_NOINLINE uint16_t FStar_UInt16_gte_mask(uint16_t a, uint16_t b) { uint16_t x = a; uint16_t y = b; - uint16_t x_xor_y = x ^ y; - uint16_t x_sub_y = x - y; - uint16_t x_sub_y_xor_y = x_sub_y ^ y; - uint16_t q = x_xor_y | x_sub_y_xor_y; - uint16_t x_xor_q = x ^ q; - uint16_t x_xor_q_ = x_xor_q >> (uint32_t)15U; - return x_xor_q_ - (uint16_t)1U; + uint16_t x_xor_y = (uint32_t)x ^ (uint32_t)y; + uint16_t x_sub_y = (uint32_t)x - (uint32_t)y; + uint16_t x_sub_y_xor_y = (uint32_t)x_sub_y ^ (uint32_t)y; + uint16_t q = (uint32_t)x_xor_y | (uint32_t)x_sub_y_xor_y; + uint16_t x_xor_q = (uint32_t)x ^ (uint32_t)q; + uint16_t x_xor_q_ = (uint32_t)x_xor_q >> 15U; + return (uint32_t)x_xor_q_ - 1U; } -static inline uint8_t FStar_UInt8_eq_mask(uint8_t a, uint8_t b) +static KRML_NOINLINE uint8_t FStar_UInt8_eq_mask(uint8_t a, uint8_t b) { - uint8_t x = a ^ b; - uint8_t minus_x = ~x + (uint8_t)1U; - uint8_t x_or_minus_x = x | minus_x; - uint8_t xnx = x_or_minus_x >> (uint32_t)7U; - return xnx - (uint8_t)1U; + uint8_t x = (uint32_t)a ^ (uint32_t)b; + uint8_t minus_x = (uint32_t)~x + 1U; + uint8_t x_or_minus_x = (uint32_t)x | (uint32_t)minus_x; + uint8_t xnx = (uint32_t)x_or_minus_x >> 7U; + return (uint32_t)xnx - 1U; } -static inline uint8_t FStar_UInt8_gte_mask(uint8_t a, uint8_t b) +static KRML_NOINLINE uint8_t FStar_UInt8_gte_mask(uint8_t a, uint8_t b) { uint8_t x = a; uint8_t y = b; - uint8_t x_xor_y = x ^ y; - uint8_t x_sub_y = x - y; - uint8_t x_sub_y_xor_y = x_sub_y ^ y; - uint8_t q = x_xor_y | x_sub_y_xor_y; - uint8_t x_xor_q = x ^ q; - uint8_t x_xor_q_ = x_xor_q >> (uint32_t)7U; - return x_xor_q_ - (uint8_t)1U; + uint8_t x_xor_y = (uint32_t)x ^ (uint32_t)y; + uint8_t x_sub_y = (uint32_t)x - (uint32_t)y; + uint8_t x_sub_y_xor_y = (uint32_t)x_sub_y ^ (uint32_t)y; + uint8_t q = (uint32_t)x_xor_y | (uint32_t)x_sub_y_xor_y; + uint8_t x_xor_q = (uint32_t)x ^ (uint32_t)q; + uint8_t x_xor_q_ = (uint32_t)x_xor_q >> 7U; + return (uint32_t)x_xor_q_ - 1U; } diff --git a/Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h b/Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h index e2b6d62859a5f1..bb736add318aa8 100644 --- a/Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h +++ b/Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h @@ -1,5 +1,5 @@ /* Copyright (c) INRIA and Microsoft Corporation. All rights reserved. - Licensed under the Apache 2.0 License. */ + Licensed under the Apache 2.0 and MIT Licenses. */ #ifndef FSTAR_UINT128_STRUCT_ENDIANNESS_H #define FSTAR_UINT128_STRUCT_ENDIANNESS_H diff --git a/Modules/_hacl/include/krml/internal/target.h b/Modules/_hacl/include/krml/internal/target.h index 5a2f94eb2ec8da..9b403c36ceca19 100644 --- a/Modules/_hacl/include/krml/internal/target.h +++ b/Modules/_hacl/include/krml/internal/target.h @@ -1,16 +1,16 @@ /* Copyright (c) INRIA and Microsoft Corporation. All rights reserved. - Licensed under the Apache 2.0 License. */ + Licensed under the Apache 2.0 and MIT Licenses. */ #ifndef __KRML_TARGET_H #define __KRML_TARGET_H -#include -#include -#include -#include +#include #include #include -#include +#include +#include +#include +#include /* Since KaRaMeL emits the inline keyword unconditionally, we follow the * guidelines at https://gcc.gnu.org/onlinedocs/gcc/Inline.html and make this @@ -19,6 +19,20 @@ # define inline __inline__ #endif +/* There is no support for aligned_alloc() in macOS before Catalina, so + * let's make a macro to use _mm_malloc() and _mm_free() functions + * from mm_malloc.h. */ +#if defined(__APPLE__) && defined(__MACH__) +# include +# if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && \ + (MAC_OS_X_VERSION_MIN_REQUIRED < 101500) +# include +# define LEGACY_MACOS +# else +# undef LEGACY_MACOS +#endif +#endif + /******************************************************************************/ /* Macros that KaRaMeL will generate. */ /******************************************************************************/ @@ -57,6 +71,108 @@ # define KRML_HOST_IGNORE(x) (void)(x) #endif +#ifndef KRML_MAYBE_UNUSED_VAR +# define KRML_MAYBE_UNUSED_VAR(x) KRML_HOST_IGNORE(x) +#endif + +#ifndef KRML_MAYBE_UNUSED +# if defined(__GNUC__) +# define KRML_MAYBE_UNUSED __attribute__((unused)) +# else +# define KRML_MAYBE_UNUSED +# endif +#endif + +#ifndef KRML_ATTRIBUTE_TARGET +# if defined(__GNUC__) +# define KRML_ATTRIBUTE_TARGET(x) __attribute__((target(x))) +# else +# define KRML_ATTRIBUTE_TARGET(x) +# endif +#endif + +#ifndef KRML_NOINLINE +# if defined(_MSC_VER) +# define KRML_NOINLINE __declspec(noinline) +# elif defined (__GNUC__) +# define KRML_NOINLINE __attribute__((noinline,unused)) +# elif defined (__SUNPRO_C) +# define KRML_NOINLINE __attribute__((noinline)) +# else +# define KRML_NOINLINE +# warning "The KRML_NOINLINE macro is not defined for this toolchain!" +# warning "The compiler may defeat side-channel resistance with optimizations." +# warning "Please locate target.h and try to fill it out with a suitable definition for this compiler." +# endif +#endif + +#ifndef KRML_MUSTINLINE +# if defined(_MSC_VER) +# define KRML_MUSTINLINE inline __forceinline +# elif defined (__GNUC__) +# define KRML_MUSTINLINE inline __attribute__((always_inline)) +# elif defined (__SUNPRO_C) +# define KRML_MUSTINLINE inline __attribute__((always_inline)) +# else +# define KRML_MUSTINLINE inline +# warning "The KRML_MUSTINLINE macro defaults to plain inline for this toolchain!" +# warning "Please locate target.h and try to fill it out with a suitable definition for this compiler." +# endif +#endif + +#ifndef KRML_PRE_ALIGN +# ifdef _MSC_VER +# define KRML_PRE_ALIGN(X) __declspec(align(X)) +# else +# define KRML_PRE_ALIGN(X) +# endif +#endif + +#ifndef KRML_POST_ALIGN +# ifdef _MSC_VER +# define KRML_POST_ALIGN(X) +# else +# define KRML_POST_ALIGN(X) __attribute__((aligned(X))) +# endif +#endif + +/* MinGW-W64 does not support C11 aligned_alloc, but it supports + * MSVC's _aligned_malloc. + */ +#ifndef KRML_ALIGNED_MALLOC +# ifdef __MINGW32__ +# include <_mingw.h> +# endif +# if ( \ + defined(_MSC_VER) || \ + (defined(__MINGW32__) && defined(__MINGW64_VERSION_MAJOR))) +# define KRML_ALIGNED_MALLOC(X, Y) _aligned_malloc(Y, X) +# elif defined(LEGACY_MACOS) +# define KRML_ALIGNED_MALLOC(X, Y) _mm_malloc(Y, X) +# else +# define KRML_ALIGNED_MALLOC(X, Y) aligned_alloc(X, Y) +# endif +#endif + +/* Since aligned allocations with MinGW-W64 are done with + * _aligned_malloc (see above), such pointers must be freed with + * _aligned_free. + */ +#ifndef KRML_ALIGNED_FREE +# ifdef __MINGW32__ +# include <_mingw.h> +# endif +# if ( \ + defined(_MSC_VER) || \ + (defined(__MINGW32__) && defined(__MINGW64_VERSION_MAJOR))) +# define KRML_ALIGNED_FREE(X) _aligned_free(X) +# elif defined(LEGACY_MACOS) +# define KRML_ALIGNED_FREE(X) _mm_free(X) +# else +# define KRML_ALIGNED_FREE(X) free(X) +# endif +#endif + /* In FStar.Buffer.fst, the size of arrays is uint32_t, but it's a number of * *elements*. Do an ugly, run-time check (some of which KaRaMeL can eliminate). */ @@ -83,184 +199,186 @@ #define KRML_LOOP1(i, n, x) { \ x \ i += n; \ + (void) i; \ } -#define KRML_LOOP2(i, n, x) \ - KRML_LOOP1(i, n, x) \ +#define KRML_LOOP2(i, n, x) \ + KRML_LOOP1(i, n, x) \ KRML_LOOP1(i, n, x) -#define KRML_LOOP3(i, n, x) \ - KRML_LOOP2(i, n, x) \ +#define KRML_LOOP3(i, n, x) \ + KRML_LOOP2(i, n, x) \ KRML_LOOP1(i, n, x) -#define KRML_LOOP4(i, n, x) \ - KRML_LOOP2(i, n, x) \ +#define KRML_LOOP4(i, n, x) \ + KRML_LOOP2(i, n, x) \ KRML_LOOP2(i, n, x) -#define KRML_LOOP5(i, n, x) \ - KRML_LOOP4(i, n, x) \ +#define KRML_LOOP5(i, n, x) \ + KRML_LOOP4(i, n, x) \ KRML_LOOP1(i, n, x) -#define KRML_LOOP6(i, n, x) \ - KRML_LOOP4(i, n, x) \ +#define KRML_LOOP6(i, n, x) \ + KRML_LOOP4(i, n, x) \ KRML_LOOP2(i, n, x) -#define KRML_LOOP7(i, n, x) \ - KRML_LOOP4(i, n, x) \ +#define KRML_LOOP7(i, n, x) \ + KRML_LOOP4(i, n, x) \ KRML_LOOP3(i, n, x) -#define KRML_LOOP8(i, n, x) \ - KRML_LOOP4(i, n, x) \ +#define KRML_LOOP8(i, n, x) \ + KRML_LOOP4(i, n, x) \ KRML_LOOP4(i, n, x) -#define KRML_LOOP9(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP9(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP1(i, n, x) -#define KRML_LOOP10(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP10(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP2(i, n, x) -#define KRML_LOOP11(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP11(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP3(i, n, x) -#define KRML_LOOP12(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP12(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP4(i, n, x) -#define KRML_LOOP13(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP13(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP5(i, n, x) -#define KRML_LOOP14(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP14(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP6(i, n, x) -#define KRML_LOOP15(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP15(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP7(i, n, x) -#define KRML_LOOP16(i, n, x) \ - KRML_LOOP8(i, n, x) \ +#define KRML_LOOP16(i, n, x) \ + KRML_LOOP8(i, n, x) \ KRML_LOOP8(i, n, x) -#define KRML_UNROLL_FOR(i, z, n, k, x) do { \ - uint32_t i = z; \ - KRML_LOOP##n(i, k, x) \ -} while (0) +#define KRML_UNROLL_FOR(i, z, n, k, x) \ + do { \ + uint32_t i = z; \ + KRML_LOOP##n(i, k, x) \ + } while (0) -#define KRML_ACTUAL_FOR(i, z, n, k, x) \ - do { \ - for (uint32_t i = z; i < n; i += k) { \ - x \ - } \ +#define KRML_ACTUAL_FOR(i, z, n, k, x) \ + do { \ + for (uint32_t i = z; i < n; i += k) { \ + x \ + } \ } while (0) #ifndef KRML_UNROLL_MAX -#define KRML_UNROLL_MAX 16 +# define KRML_UNROLL_MAX 16 #endif /* 1 is the number of loop iterations, i.e. (n - z)/k as evaluated by krml */ #if 0 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR0(i, z, n, k, x) +# define KRML_MAYBE_FOR0(i, z, n, k, x) #else -#define KRML_MAYBE_FOR0(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR0(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 1 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR1(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 1, k, x) +# define KRML_MAYBE_FOR1(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 1, k, x) #else -#define KRML_MAYBE_FOR1(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR1(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 2 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR2(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 2, k, x) +# define KRML_MAYBE_FOR2(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 2, k, x) #else -#define KRML_MAYBE_FOR2(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR2(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 3 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR3(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 3, k, x) +# define KRML_MAYBE_FOR3(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 3, k, x) #else -#define KRML_MAYBE_FOR3(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR3(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 4 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR4(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 4, k, x) +# define KRML_MAYBE_FOR4(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 4, k, x) #else -#define KRML_MAYBE_FOR4(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR4(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 5 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR5(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 5, k, x) +# define KRML_MAYBE_FOR5(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 5, k, x) #else -#define KRML_MAYBE_FOR5(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR5(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 6 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR6(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 6, k, x) +# define KRML_MAYBE_FOR6(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 6, k, x) #else -#define KRML_MAYBE_FOR6(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR6(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 7 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR7(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 7, k, x) +# define KRML_MAYBE_FOR7(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 7, k, x) #else -#define KRML_MAYBE_FOR7(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR7(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 8 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR8(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 8, k, x) +# define KRML_MAYBE_FOR8(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 8, k, x) #else -#define KRML_MAYBE_FOR8(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR8(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 9 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR9(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 9, k, x) +# define KRML_MAYBE_FOR9(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 9, k, x) #else -#define KRML_MAYBE_FOR9(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR9(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 10 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR10(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 10, k, x) +# define KRML_MAYBE_FOR10(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 10, k, x) #else -#define KRML_MAYBE_FOR10(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR10(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 11 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR11(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 11, k, x) +# define KRML_MAYBE_FOR11(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 11, k, x) #else -#define KRML_MAYBE_FOR11(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR11(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 12 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR12(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 12, k, x) +# define KRML_MAYBE_FOR12(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 12, k, x) #else -#define KRML_MAYBE_FOR12(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR12(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 13 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR13(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 13, k, x) +# define KRML_MAYBE_FOR13(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 13, k, x) #else -#define KRML_MAYBE_FOR13(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR13(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 14 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR14(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 14, k, x) +# define KRML_MAYBE_FOR14(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 14, k, x) #else -#define KRML_MAYBE_FOR14(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR14(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 15 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR15(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 15, k, x) +# define KRML_MAYBE_FOR15(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 15, k, x) #else -#define KRML_MAYBE_FOR15(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR15(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #if 16 <= KRML_UNROLL_MAX -#define KRML_MAYBE_FOR16(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 16, k, x) +# define KRML_MAYBE_FOR16(i, z, n, k, x) KRML_UNROLL_FOR(i, z, 16, k, x) #else -#define KRML_MAYBE_FOR16(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) +# define KRML_MAYBE_FOR16(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif #endif diff --git a/Modules/_hacl/include/krml/lowstar_endianness.h b/Modules/_hacl/include/krml/lowstar_endianness.h index 1aa2ccd644c06f..af6b882cf259cc 100644 --- a/Modules/_hacl/include/krml/lowstar_endianness.h +++ b/Modules/_hacl/include/krml/lowstar_endianness.h @@ -1,5 +1,5 @@ /* Copyright (c) INRIA and Microsoft Corporation. All rights reserved. - Licensed under the Apache 2.0 License. */ + Licensed under the Apache 2.0 and MIT Licenses. */ #ifndef __LOWSTAR_ENDIANNESS_H #define __LOWSTAR_ENDIANNESS_H diff --git a/Modules/_hacl/internal/Hacl_Hash_Blake2b.h b/Modules/_hacl/internal/Hacl_Hash_Blake2b.h new file mode 100644 index 00000000000000..8ee70282f4e4de --- /dev/null +++ b/Modules/_hacl/internal/Hacl_Hash_Blake2b.h @@ -0,0 +1,78 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __internal_Hacl_Hash_Blake2b_H +#define __internal_Hacl_Hash_Blake2b_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "internal/Hacl_Impl_Blake2_Constants.h" +#include "../Hacl_Hash_Blake2b.h" + +typedef struct Hacl_Hash_Blake2b_params_and_key_s +{ + Hacl_Hash_Blake2b_blake2_params *fst; + uint8_t *snd; +} +Hacl_Hash_Blake2b_params_and_key; + +void Hacl_Hash_Blake2b_init(uint64_t *hash, uint32_t kk, uint32_t nn); + +void +Hacl_Hash_Blake2b_update_multi( + uint32_t len, + uint64_t *wv, + uint64_t *hash, + FStar_UInt128_uint128 prev, + uint8_t *blocks, + uint32_t nb +); + +void +Hacl_Hash_Blake2b_update_last( + uint32_t len, + uint64_t *wv, + uint64_t *hash, + bool last_node, + FStar_UInt128_uint128 prev, + uint32_t rem, + uint8_t *d +); + +void Hacl_Hash_Blake2b_finish(uint32_t nn, uint8_t *output, uint64_t *hash); + +#if defined(__cplusplus) +} +#endif + +#define __internal_Hacl_Hash_Blake2b_H_DEFINED +#endif diff --git a/Modules/_hacl/internal/Hacl_Hash_Blake2b_Simd256.h b/Modules/_hacl/internal/Hacl_Hash_Blake2b_Simd256.h new file mode 100644 index 00000000000000..ab329b92c3630c --- /dev/null +++ b/Modules/_hacl/internal/Hacl_Hash_Blake2b_Simd256.h @@ -0,0 +1,93 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __internal_Hacl_Hash_Blake2b_Simd256_H +#define __internal_Hacl_Hash_Blake2b_Simd256_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "internal/Hacl_Impl_Blake2_Constants.h" +#include "internal/Hacl_Hash_Blake2b.h" +#include "../Hacl_Hash_Blake2b_Simd256.h" +#include "libintvector.h" + +void +Hacl_Hash_Blake2b_Simd256_init(Lib_IntVector_Intrinsics_vec256 *hash, uint32_t kk, uint32_t nn); + +void +Hacl_Hash_Blake2b_Simd256_update_multi( + uint32_t len, + Lib_IntVector_Intrinsics_vec256 *wv, + Lib_IntVector_Intrinsics_vec256 *hash, + FStar_UInt128_uint128 prev, + uint8_t *blocks, + uint32_t nb +); + +void +Hacl_Hash_Blake2b_Simd256_update_last( + uint32_t len, + Lib_IntVector_Intrinsics_vec256 *wv, + Lib_IntVector_Intrinsics_vec256 *hash, + bool last_node, + FStar_UInt128_uint128 prev, + uint32_t rem, + uint8_t *d +); + +void +Hacl_Hash_Blake2b_Simd256_finish( + uint32_t nn, + uint8_t *output, + Lib_IntVector_Intrinsics_vec256 *hash +); + +void +Hacl_Hash_Blake2b_Simd256_load_state256b_from_state32( + Lib_IntVector_Intrinsics_vec256 *st, + uint64_t *st32 +); + +void +Hacl_Hash_Blake2b_Simd256_store_state256b_to_state32( + uint64_t *st32, + Lib_IntVector_Intrinsics_vec256 *st +); + +Lib_IntVector_Intrinsics_vec256 *Hacl_Hash_Blake2b_Simd256_malloc_with_key(void); + +#if defined(__cplusplus) +} +#endif + +#define __internal_Hacl_Hash_Blake2b_Simd256_H_DEFINED +#endif diff --git a/Modules/_hacl/internal/Hacl_Hash_Blake2s.h b/Modules/_hacl/internal/Hacl_Hash_Blake2s.h new file mode 100644 index 00000000000000..6494075b60a25b --- /dev/null +++ b/Modules/_hacl/internal/Hacl_Hash_Blake2s.h @@ -0,0 +1,72 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __internal_Hacl_Hash_Blake2s_H +#define __internal_Hacl_Hash_Blake2s_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "internal/Hacl_Impl_Blake2_Constants.h" +#include "internal/Hacl_Hash_Blake2b.h" +#include "../Hacl_Hash_Blake2s.h" + +void Hacl_Hash_Blake2s_init(uint32_t *hash, uint32_t kk, uint32_t nn); + +void +Hacl_Hash_Blake2s_update_multi( + uint32_t len, + uint32_t *wv, + uint32_t *hash, + uint64_t prev, + uint8_t *blocks, + uint32_t nb +); + +void +Hacl_Hash_Blake2s_update_last( + uint32_t len, + uint32_t *wv, + uint32_t *hash, + bool last_node, + uint64_t prev, + uint32_t rem, + uint8_t *d +); + +void Hacl_Hash_Blake2s_finish(uint32_t nn, uint8_t *output, uint32_t *hash); + +#if defined(__cplusplus) +} +#endif + +#define __internal_Hacl_Hash_Blake2s_H_DEFINED +#endif diff --git a/Modules/_hacl/internal/Hacl_Hash_Blake2s_Simd128.h b/Modules/_hacl/internal/Hacl_Hash_Blake2s_Simd128.h new file mode 100644 index 00000000000000..60c09a67b445b6 --- /dev/null +++ b/Modules/_hacl/internal/Hacl_Hash_Blake2s_Simd128.h @@ -0,0 +1,93 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __internal_Hacl_Hash_Blake2s_Simd128_H +#define __internal_Hacl_Hash_Blake2s_Simd128_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "internal/Hacl_Impl_Blake2_Constants.h" +#include "internal/Hacl_Hash_Blake2b.h" +#include "../Hacl_Hash_Blake2s_Simd128.h" +#include "libintvector.h" + +void +Hacl_Hash_Blake2s_Simd128_init(Lib_IntVector_Intrinsics_vec128 *hash, uint32_t kk, uint32_t nn); + +void +Hacl_Hash_Blake2s_Simd128_update_multi( + uint32_t len, + Lib_IntVector_Intrinsics_vec128 *wv, + Lib_IntVector_Intrinsics_vec128 *hash, + uint64_t prev, + uint8_t *blocks, + uint32_t nb +); + +void +Hacl_Hash_Blake2s_Simd128_update_last( + uint32_t len, + Lib_IntVector_Intrinsics_vec128 *wv, + Lib_IntVector_Intrinsics_vec128 *hash, + bool last_node, + uint64_t prev, + uint32_t rem, + uint8_t *d +); + +void +Hacl_Hash_Blake2s_Simd128_finish( + uint32_t nn, + uint8_t *output, + Lib_IntVector_Intrinsics_vec128 *hash +); + +void +Hacl_Hash_Blake2s_Simd128_store_state128s_to_state32( + uint32_t *st32, + Lib_IntVector_Intrinsics_vec128 *st +); + +void +Hacl_Hash_Blake2s_Simd128_load_state128s_from_state32( + Lib_IntVector_Intrinsics_vec128 *st, + uint32_t *st32 +); + +Lib_IntVector_Intrinsics_vec128 *Hacl_Hash_Blake2s_Simd128_malloc_with_key(void); + +#if defined(__cplusplus) +} +#endif + +#define __internal_Hacl_Hash_Blake2s_Simd128_H_DEFINED +#endif diff --git a/Modules/_hacl/internal/Hacl_Hash_MD5.h b/Modules/_hacl/internal/Hacl_Hash_MD5.h index 87ad4cf228d91b..a50ec407f53e39 100644 --- a/Modules/_hacl/internal/Hacl_Hash_MD5.h +++ b/Modules/_hacl/internal/Hacl_Hash_MD5.h @@ -37,21 +37,16 @@ extern "C" { #include "../Hacl_Hash_MD5.h" -void Hacl_Hash_Core_MD5_legacy_init(uint32_t *s); +void Hacl_Hash_MD5_init(uint32_t *s); -void Hacl_Hash_Core_MD5_legacy_finish(uint32_t *s, uint8_t *dst); +void Hacl_Hash_MD5_finish(uint32_t *s, uint8_t *dst); -void Hacl_Hash_MD5_legacy_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks); +void Hacl_Hash_MD5_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks); void -Hacl_Hash_MD5_legacy_update_last( - uint32_t *s, - uint64_t prev_len, - uint8_t *input, - uint32_t input_len -); - -void Hacl_Hash_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst); +Hacl_Hash_MD5_update_last(uint32_t *s, uint64_t prev_len, uint8_t *input, uint32_t input_len); + +void Hacl_Hash_MD5_hash_oneshot(uint8_t *output, uint8_t *input, uint32_t input_len); #if defined(__cplusplus) } diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA1.h b/Modules/_hacl/internal/Hacl_Hash_SHA1.h index d2d9df44c6c14c..b39bad3f3b93e8 100644 --- a/Modules/_hacl/internal/Hacl_Hash_SHA1.h +++ b/Modules/_hacl/internal/Hacl_Hash_SHA1.h @@ -37,21 +37,16 @@ extern "C" { #include "../Hacl_Hash_SHA1.h" -void Hacl_Hash_Core_SHA1_legacy_init(uint32_t *s); +void Hacl_Hash_SHA1_init(uint32_t *s); -void Hacl_Hash_Core_SHA1_legacy_finish(uint32_t *s, uint8_t *dst); +void Hacl_Hash_SHA1_finish(uint32_t *s, uint8_t *dst); -void Hacl_Hash_SHA1_legacy_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks); +void Hacl_Hash_SHA1_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks); void -Hacl_Hash_SHA1_legacy_update_last( - uint32_t *s, - uint64_t prev_len, - uint8_t *input, - uint32_t input_len -); - -void Hacl_Hash_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst); +Hacl_Hash_SHA1_update_last(uint32_t *s, uint64_t prev_len, uint8_t *input, uint32_t input_len); + +void Hacl_Hash_SHA1_hash_oneshot(uint8_t *output, uint8_t *input, uint32_t input_len); #if defined(__cplusplus) } diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA2.h b/Modules/_hacl/internal/Hacl_Hash_SHA2.h index 851f7dc60c94c2..cb60f9e9bd4df6 100644 --- a/Modules/_hacl/internal/Hacl_Hash_SHA2.h +++ b/Modules/_hacl/internal/Hacl_Hash_SHA2.h @@ -40,141 +40,123 @@ extern "C" { static const uint32_t -Hacl_Impl_SHA2_Generic_h224[8U] = +Hacl_Hash_SHA2_h224[8U] = { - (uint32_t)0xc1059ed8U, (uint32_t)0x367cd507U, (uint32_t)0x3070dd17U, (uint32_t)0xf70e5939U, - (uint32_t)0xffc00b31U, (uint32_t)0x68581511U, (uint32_t)0x64f98fa7U, (uint32_t)0xbefa4fa4U + 0xc1059ed8U, 0x367cd507U, 0x3070dd17U, 0xf70e5939U, 0xffc00b31U, 0x68581511U, 0x64f98fa7U, + 0xbefa4fa4U }; static const uint32_t -Hacl_Impl_SHA2_Generic_h256[8U] = +Hacl_Hash_SHA2_h256[8U] = { - (uint32_t)0x6a09e667U, (uint32_t)0xbb67ae85U, (uint32_t)0x3c6ef372U, (uint32_t)0xa54ff53aU, - (uint32_t)0x510e527fU, (uint32_t)0x9b05688cU, (uint32_t)0x1f83d9abU, (uint32_t)0x5be0cd19U + 0x6a09e667U, 0xbb67ae85U, 0x3c6ef372U, 0xa54ff53aU, 0x510e527fU, 0x9b05688cU, 0x1f83d9abU, + 0x5be0cd19U }; static const uint64_t -Hacl_Impl_SHA2_Generic_h384[8U] = +Hacl_Hash_SHA2_h384[8U] = { - (uint64_t)0xcbbb9d5dc1059ed8U, (uint64_t)0x629a292a367cd507U, (uint64_t)0x9159015a3070dd17U, - (uint64_t)0x152fecd8f70e5939U, (uint64_t)0x67332667ffc00b31U, (uint64_t)0x8eb44a8768581511U, - (uint64_t)0xdb0c2e0d64f98fa7U, (uint64_t)0x47b5481dbefa4fa4U + 0xcbbb9d5dc1059ed8ULL, 0x629a292a367cd507ULL, 0x9159015a3070dd17ULL, 0x152fecd8f70e5939ULL, + 0x67332667ffc00b31ULL, 0x8eb44a8768581511ULL, 0xdb0c2e0d64f98fa7ULL, 0x47b5481dbefa4fa4ULL }; static const uint64_t -Hacl_Impl_SHA2_Generic_h512[8U] = +Hacl_Hash_SHA2_h512[8U] = { - (uint64_t)0x6a09e667f3bcc908U, (uint64_t)0xbb67ae8584caa73bU, (uint64_t)0x3c6ef372fe94f82bU, - (uint64_t)0xa54ff53a5f1d36f1U, (uint64_t)0x510e527fade682d1U, (uint64_t)0x9b05688c2b3e6c1fU, - (uint64_t)0x1f83d9abfb41bd6bU, (uint64_t)0x5be0cd19137e2179U + 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, + 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL }; static const uint32_t -Hacl_Impl_SHA2_Generic_k224_256[64U] = +Hacl_Hash_SHA2_k224_256[64U] = { - (uint32_t)0x428a2f98U, (uint32_t)0x71374491U, (uint32_t)0xb5c0fbcfU, (uint32_t)0xe9b5dba5U, - (uint32_t)0x3956c25bU, (uint32_t)0x59f111f1U, (uint32_t)0x923f82a4U, (uint32_t)0xab1c5ed5U, - (uint32_t)0xd807aa98U, (uint32_t)0x12835b01U, (uint32_t)0x243185beU, (uint32_t)0x550c7dc3U, - (uint32_t)0x72be5d74U, (uint32_t)0x80deb1feU, (uint32_t)0x9bdc06a7U, (uint32_t)0xc19bf174U, - (uint32_t)0xe49b69c1U, (uint32_t)0xefbe4786U, (uint32_t)0x0fc19dc6U, (uint32_t)0x240ca1ccU, - (uint32_t)0x2de92c6fU, (uint32_t)0x4a7484aaU, (uint32_t)0x5cb0a9dcU, (uint32_t)0x76f988daU, - (uint32_t)0x983e5152U, (uint32_t)0xa831c66dU, (uint32_t)0xb00327c8U, (uint32_t)0xbf597fc7U, - (uint32_t)0xc6e00bf3U, (uint32_t)0xd5a79147U, (uint32_t)0x06ca6351U, (uint32_t)0x14292967U, - (uint32_t)0x27b70a85U, (uint32_t)0x2e1b2138U, (uint32_t)0x4d2c6dfcU, (uint32_t)0x53380d13U, - (uint32_t)0x650a7354U, (uint32_t)0x766a0abbU, (uint32_t)0x81c2c92eU, (uint32_t)0x92722c85U, - (uint32_t)0xa2bfe8a1U, (uint32_t)0xa81a664bU, (uint32_t)0xc24b8b70U, (uint32_t)0xc76c51a3U, - (uint32_t)0xd192e819U, (uint32_t)0xd6990624U, (uint32_t)0xf40e3585U, (uint32_t)0x106aa070U, - (uint32_t)0x19a4c116U, (uint32_t)0x1e376c08U, (uint32_t)0x2748774cU, (uint32_t)0x34b0bcb5U, - (uint32_t)0x391c0cb3U, (uint32_t)0x4ed8aa4aU, (uint32_t)0x5b9cca4fU, (uint32_t)0x682e6ff3U, - (uint32_t)0x748f82eeU, (uint32_t)0x78a5636fU, (uint32_t)0x84c87814U, (uint32_t)0x8cc70208U, - (uint32_t)0x90befffaU, (uint32_t)0xa4506cebU, (uint32_t)0xbef9a3f7U, (uint32_t)0xc67178f2U + 0x428a2f98U, 0x71374491U, 0xb5c0fbcfU, 0xe9b5dba5U, 0x3956c25bU, 0x59f111f1U, 0x923f82a4U, + 0xab1c5ed5U, 0xd807aa98U, 0x12835b01U, 0x243185beU, 0x550c7dc3U, 0x72be5d74U, 0x80deb1feU, + 0x9bdc06a7U, 0xc19bf174U, 0xe49b69c1U, 0xefbe4786U, 0x0fc19dc6U, 0x240ca1ccU, 0x2de92c6fU, + 0x4a7484aaU, 0x5cb0a9dcU, 0x76f988daU, 0x983e5152U, 0xa831c66dU, 0xb00327c8U, 0xbf597fc7U, + 0xc6e00bf3U, 0xd5a79147U, 0x06ca6351U, 0x14292967U, 0x27b70a85U, 0x2e1b2138U, 0x4d2c6dfcU, + 0x53380d13U, 0x650a7354U, 0x766a0abbU, 0x81c2c92eU, 0x92722c85U, 0xa2bfe8a1U, 0xa81a664bU, + 0xc24b8b70U, 0xc76c51a3U, 0xd192e819U, 0xd6990624U, 0xf40e3585U, 0x106aa070U, 0x19a4c116U, + 0x1e376c08U, 0x2748774cU, 0x34b0bcb5U, 0x391c0cb3U, 0x4ed8aa4aU, 0x5b9cca4fU, 0x682e6ff3U, + 0x748f82eeU, 0x78a5636fU, 0x84c87814U, 0x8cc70208U, 0x90befffaU, 0xa4506cebU, 0xbef9a3f7U, + 0xc67178f2U }; static const uint64_t -Hacl_Impl_SHA2_Generic_k384_512[80U] = +Hacl_Hash_SHA2_k384_512[80U] = { - (uint64_t)0x428a2f98d728ae22U, (uint64_t)0x7137449123ef65cdU, (uint64_t)0xb5c0fbcfec4d3b2fU, - (uint64_t)0xe9b5dba58189dbbcU, (uint64_t)0x3956c25bf348b538U, (uint64_t)0x59f111f1b605d019U, - (uint64_t)0x923f82a4af194f9bU, (uint64_t)0xab1c5ed5da6d8118U, (uint64_t)0xd807aa98a3030242U, - (uint64_t)0x12835b0145706fbeU, (uint64_t)0x243185be4ee4b28cU, (uint64_t)0x550c7dc3d5ffb4e2U, - (uint64_t)0x72be5d74f27b896fU, (uint64_t)0x80deb1fe3b1696b1U, (uint64_t)0x9bdc06a725c71235U, - (uint64_t)0xc19bf174cf692694U, (uint64_t)0xe49b69c19ef14ad2U, (uint64_t)0xefbe4786384f25e3U, - (uint64_t)0x0fc19dc68b8cd5b5U, (uint64_t)0x240ca1cc77ac9c65U, (uint64_t)0x2de92c6f592b0275U, - (uint64_t)0x4a7484aa6ea6e483U, (uint64_t)0x5cb0a9dcbd41fbd4U, (uint64_t)0x76f988da831153b5U, - (uint64_t)0x983e5152ee66dfabU, (uint64_t)0xa831c66d2db43210U, (uint64_t)0xb00327c898fb213fU, - (uint64_t)0xbf597fc7beef0ee4U, (uint64_t)0xc6e00bf33da88fc2U, (uint64_t)0xd5a79147930aa725U, - (uint64_t)0x06ca6351e003826fU, (uint64_t)0x142929670a0e6e70U, (uint64_t)0x27b70a8546d22ffcU, - (uint64_t)0x2e1b21385c26c926U, (uint64_t)0x4d2c6dfc5ac42aedU, (uint64_t)0x53380d139d95b3dfU, - (uint64_t)0x650a73548baf63deU, (uint64_t)0x766a0abb3c77b2a8U, (uint64_t)0x81c2c92e47edaee6U, - (uint64_t)0x92722c851482353bU, (uint64_t)0xa2bfe8a14cf10364U, (uint64_t)0xa81a664bbc423001U, - (uint64_t)0xc24b8b70d0f89791U, (uint64_t)0xc76c51a30654be30U, (uint64_t)0xd192e819d6ef5218U, - (uint64_t)0xd69906245565a910U, (uint64_t)0xf40e35855771202aU, (uint64_t)0x106aa07032bbd1b8U, - (uint64_t)0x19a4c116b8d2d0c8U, (uint64_t)0x1e376c085141ab53U, (uint64_t)0x2748774cdf8eeb99U, - (uint64_t)0x34b0bcb5e19b48a8U, (uint64_t)0x391c0cb3c5c95a63U, (uint64_t)0x4ed8aa4ae3418acbU, - (uint64_t)0x5b9cca4f7763e373U, (uint64_t)0x682e6ff3d6b2b8a3U, (uint64_t)0x748f82ee5defb2fcU, - (uint64_t)0x78a5636f43172f60U, (uint64_t)0x84c87814a1f0ab72U, (uint64_t)0x8cc702081a6439ecU, - (uint64_t)0x90befffa23631e28U, (uint64_t)0xa4506cebde82bde9U, (uint64_t)0xbef9a3f7b2c67915U, - (uint64_t)0xc67178f2e372532bU, (uint64_t)0xca273eceea26619cU, (uint64_t)0xd186b8c721c0c207U, - (uint64_t)0xeada7dd6cde0eb1eU, (uint64_t)0xf57d4f7fee6ed178U, (uint64_t)0x06f067aa72176fbaU, - (uint64_t)0x0a637dc5a2c898a6U, (uint64_t)0x113f9804bef90daeU, (uint64_t)0x1b710b35131c471bU, - (uint64_t)0x28db77f523047d84U, (uint64_t)0x32caab7b40c72493U, (uint64_t)0x3c9ebe0a15c9bebcU, - (uint64_t)0x431d67c49c100d4cU, (uint64_t)0x4cc5d4becb3e42b6U, (uint64_t)0x597f299cfc657e2aU, - (uint64_t)0x5fcb6fab3ad6faecU, (uint64_t)0x6c44198c4a475817U + 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, 0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL, + 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, + 0xd807aa98a3030242ULL, 0x12835b0145706fbeULL, 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, 0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL, + 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, + 0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL, + 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, + 0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL, 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, 0x81c2c92e47edaee6ULL, 0x92722c851482353bULL, + 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, + 0xd192e819d6ef5218ULL, 0xd69906245565a910ULL, 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, 0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, + 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, + 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL, 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, 0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL, + 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, + 0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL, 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, 0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL, + 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL }; -void Hacl_SHA2_Scalar32_sha256_init(uint32_t *hash); +void Hacl_Hash_SHA2_sha256_init(uint32_t *hash); -void Hacl_SHA2_Scalar32_sha256_update_nblocks(uint32_t len, uint8_t *b, uint32_t *st); +void Hacl_Hash_SHA2_sha256_update_nblocks(uint32_t len, uint8_t *b, uint32_t *st); void -Hacl_SHA2_Scalar32_sha256_update_last( - uint64_t totlen, - uint32_t len, - uint8_t *b, - uint32_t *hash -); +Hacl_Hash_SHA2_sha256_update_last(uint64_t totlen, uint32_t len, uint8_t *b, uint32_t *hash); + +void Hacl_Hash_SHA2_sha256_finish(uint32_t *st, uint8_t *h); -void Hacl_SHA2_Scalar32_sha256_finish(uint32_t *st, uint8_t *h); +void Hacl_Hash_SHA2_sha224_init(uint32_t *hash); -void Hacl_SHA2_Scalar32_sha224_init(uint32_t *hash); +void Hacl_Hash_SHA2_sha224_update_nblocks(uint32_t len, uint8_t *b, uint32_t *st); void -Hacl_SHA2_Scalar32_sha224_update_last(uint64_t totlen, uint32_t len, uint8_t *b, uint32_t *st); +Hacl_Hash_SHA2_sha224_update_last(uint64_t totlen, uint32_t len, uint8_t *b, uint32_t *st); -void Hacl_SHA2_Scalar32_sha224_finish(uint32_t *st, uint8_t *h); +void Hacl_Hash_SHA2_sha224_finish(uint32_t *st, uint8_t *h); -void Hacl_SHA2_Scalar32_sha512_init(uint64_t *hash); +void Hacl_Hash_SHA2_sha512_init(uint64_t *hash); -void Hacl_SHA2_Scalar32_sha512_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st); +void Hacl_Hash_SHA2_sha512_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st); void -Hacl_SHA2_Scalar32_sha512_update_last( +Hacl_Hash_SHA2_sha512_update_last( FStar_UInt128_uint128 totlen, uint32_t len, uint8_t *b, uint64_t *hash ); -void Hacl_SHA2_Scalar32_sha512_finish(uint64_t *st, uint8_t *h); +void Hacl_Hash_SHA2_sha512_finish(uint64_t *st, uint8_t *h); -void Hacl_SHA2_Scalar32_sha384_init(uint64_t *hash); +void Hacl_Hash_SHA2_sha384_init(uint64_t *hash); -void Hacl_SHA2_Scalar32_sha384_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st); +void Hacl_Hash_SHA2_sha384_update_nblocks(uint32_t len, uint8_t *b, uint64_t *st); void -Hacl_SHA2_Scalar32_sha384_update_last( +Hacl_Hash_SHA2_sha384_update_last( FStar_UInt128_uint128 totlen, uint32_t len, uint8_t *b, uint64_t *st ); -void Hacl_SHA2_Scalar32_sha384_finish(uint64_t *st, uint8_t *h); +void Hacl_Hash_SHA2_sha384_finish(uint64_t *st, uint8_t *h); #if defined(__cplusplus) } diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA3.h b/Modules/_hacl/internal/Hacl_Hash_SHA3.h index 1c9808b8dd497c..0a152b4c622533 100644 --- a/Modules/_hacl/internal/Hacl_Hash_SHA3.h +++ b/Modules/_hacl/internal/Hacl_Hash_SHA3.h @@ -37,6 +37,12 @@ extern "C" { #include "../Hacl_Hash_SHA3.h" +extern const uint32_t Hacl_Hash_SHA3_keccak_rotc[24U]; + +extern const uint32_t Hacl_Hash_SHA3_keccak_piln[24U]; + +extern const uint64_t Hacl_Hash_SHA3_keccak_rndc[24U]; + void Hacl_Hash_SHA3_update_multi_sha3( Spec_Hash_Definitions_hash_alg a, @@ -53,10 +59,6 @@ Hacl_Hash_SHA3_update_last_sha3( uint32_t input_len ); -void Hacl_Impl_SHA3_state_permute(uint64_t *s); - -void Hacl_Impl_SHA3_loadState(uint32_t rateInBytes, uint8_t *input, uint64_t *s); - #if defined(__cplusplus) } #endif diff --git a/Modules/_hacl/internal/Hacl_Impl_Blake2_Constants.h b/Modules/_hacl/internal/Hacl_Impl_Blake2_Constants.h new file mode 100644 index 00000000000000..f4cf516124aabb --- /dev/null +++ b/Modules/_hacl/internal/Hacl_Impl_Blake2_Constants.h @@ -0,0 +1,73 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __internal_Hacl_Impl_Blake2_Constants_H +#define __internal_Hacl_Impl_Blake2_Constants_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +static const +uint32_t +Hacl_Hash_Blake2b_sigmaTable[160U] = + { + 0U, 1U, 2U, 3U, 4U, 5U, 6U, 7U, 8U, 9U, 10U, 11U, 12U, 13U, 14U, 15U, 14U, 10U, 4U, 8U, 9U, 15U, + 13U, 6U, 1U, 12U, 0U, 2U, 11U, 7U, 5U, 3U, 11U, 8U, 12U, 0U, 5U, 2U, 15U, 13U, 10U, 14U, 3U, 6U, + 7U, 1U, 9U, 4U, 7U, 9U, 3U, 1U, 13U, 12U, 11U, 14U, 2U, 6U, 5U, 10U, 4U, 0U, 15U, 8U, 9U, 0U, + 5U, 7U, 2U, 4U, 10U, 15U, 14U, 1U, 11U, 12U, 6U, 8U, 3U, 13U, 2U, 12U, 6U, 10U, 0U, 11U, 8U, 3U, + 4U, 13U, 7U, 5U, 15U, 14U, 1U, 9U, 12U, 5U, 1U, 15U, 14U, 13U, 4U, 10U, 0U, 7U, 6U, 3U, 9U, 2U, + 8U, 11U, 13U, 11U, 7U, 14U, 12U, 1U, 3U, 9U, 5U, 0U, 15U, 4U, 8U, 6U, 2U, 10U, 6U, 15U, 14U, 9U, + 11U, 3U, 0U, 8U, 12U, 2U, 13U, 7U, 1U, 4U, 10U, 5U, 10U, 2U, 8U, 4U, 7U, 6U, 1U, 5U, 15U, 11U, + 9U, 14U, 3U, 12U, 13U + }; + +static const +uint32_t +Hacl_Hash_Blake2b_ivTable_S[8U] = + { + 0x6A09E667U, 0xBB67AE85U, 0x3C6EF372U, 0xA54FF53AU, 0x510E527FU, 0x9B05688CU, 0x1F83D9ABU, + 0x5BE0CD19U + }; + +static const +uint64_t +Hacl_Hash_Blake2b_ivTable_B[8U] = + { + 0x6A09E667F3BCC908ULL, 0xBB67AE8584CAA73BULL, 0x3C6EF372FE94F82BULL, 0xA54FF53A5F1D36F1ULL, + 0x510E527FADE682D1ULL, 0x9B05688C2B3E6C1FULL, 0x1F83D9ABFB41BD6BULL, 0x5BE0CD19137E2179ULL + }; + +#if defined(__cplusplus) +} +#endif + +#define __internal_Hacl_Impl_Blake2_Constants_H_DEFINED +#endif diff --git a/Modules/_hacl/lib_memzero0.h b/Modules/_hacl/lib_memzero0.h new file mode 100644 index 00000000000000..fea3e41c907f44 --- /dev/null +++ b/Modules/_hacl/lib_memzero0.h @@ -0,0 +1,5 @@ +#include + +void Lib_Memzero0_memzero0(void *dst, uint64_t len); + +#define Lib_Memzero0_memzero(dst, len, t, _ret_t) Lib_Memzero0_memzero0(dst, len * sizeof(t)) diff --git a/Modules/_hacl/libintvector.h b/Modules/_hacl/libintvector.h new file mode 100644 index 00000000000000..11e914f7e1650a --- /dev/null +++ b/Modules/_hacl/libintvector.h @@ -0,0 +1,936 @@ +#ifndef __Vec_Intrin_H +#define __Vec_Intrin_H + +#include + +/* We include config.h here to ensure that the various feature-flags are + * properly brought into scope. Users can either run the configure script, or + * write a config.h themselves and put it under version control. */ +#if defined(__has_include) +#if __has_include("config.h") +#include "config.h" +#endif +#endif + +/* # DEBUGGING: + * ============ + * It is possible to debug the current definitions by using libintvector_debug.h + * See the include at the bottom of the file. */ + +#define Lib_IntVector_Intrinsics_bit_mask64(x) -((x) & 1) + +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) + +#if defined(HACL_CAN_COMPILE_VEC128) + +#include +#include +#include + +typedef __m128i Lib_IntVector_Intrinsics_vec128; + +#define Lib_IntVector_Intrinsics_ni_aes_enc(x0, x1) \ + (_mm_aesenc_si128(x0, x1)) + +#define Lib_IntVector_Intrinsics_ni_aes_enc_last(x0, x1) \ + (_mm_aesenclast_si128(x0, x1)) + +#define Lib_IntVector_Intrinsics_ni_aes_keygen_assist(x0, x1) \ + (_mm_aeskeygenassist_si128(x0, x1)) + +#define Lib_IntVector_Intrinsics_ni_clmul(x0, x1, x2) \ + (_mm_clmulepi64_si128(x0, x1, x2)) + + +#define Lib_IntVector_Intrinsics_vec128_xor(x0, x1) \ + (_mm_xor_si128(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_eq64(x0, x1) \ + (_mm_cmpeq_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_eq32(x0, x1) \ + (_mm_cmpeq_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_gt64(x0, x1) \ + (_mm_cmpgt_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_gt32(x0, x1) \ + (_mm_cmpgt_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_or(x0, x1) \ + (_mm_or_si128(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_and(x0, x1) \ + (_mm_and_si128(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_lognot(x0) \ + (_mm_xor_si128(x0, _mm_set1_epi32(-1))) + + +#define Lib_IntVector_Intrinsics_vec128_shift_left(x0, x1) \ + (_mm_slli_si128(x0, (x1)/8)) + +#define Lib_IntVector_Intrinsics_vec128_shift_right(x0, x1) \ + (_mm_srli_si128(x0, (x1)/8)) + +#define Lib_IntVector_Intrinsics_vec128_shift_left64(x0, x1) \ + (_mm_slli_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_shift_right64(x0, x1) \ + (_mm_srli_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_shift_left32(x0, x1) \ + (_mm_slli_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_shift_right32(x0, x1) \ + (_mm_srli_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_rotate_left32_8(x0) \ + (_mm_shuffle_epi8(x0, _mm_set_epi8(14,13,12,15,10,9,8,11,6,5,4,7,2,1,0,3))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_left32_16(x0) \ + (_mm_shuffle_epi8(x0, _mm_set_epi8(13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_left32_24(x0) \ + (_mm_shuffle_epi8(x0, _mm_set_epi8(12,15,14,13,8,11,10,9,4,7,6,5,0,3,2,1))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_left32(x0,x1) \ + (((x1) == 8? Lib_IntVector_Intrinsics_vec128_rotate_left32_8(x0) : \ + ((x1) == 16? Lib_IntVector_Intrinsics_vec128_rotate_left32_16(x0) : \ + ((x1) == 24? Lib_IntVector_Intrinsics_vec128_rotate_left32_24(x0) : \ + _mm_xor_si128(_mm_slli_epi32(x0,x1),_mm_srli_epi32(x0,32-(x1))))))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right32(x0,x1) \ + (Lib_IntVector_Intrinsics_vec128_rotate_left32(x0,32-(x1))) + +#define Lib_IntVector_Intrinsics_vec128_shuffle32(x0, x1, x2, x3, x4) \ + (_mm_shuffle_epi32(x0, _MM_SHUFFLE(x4,x3,x2,x1))) + +#define Lib_IntVector_Intrinsics_vec128_shuffle64(x0, x1, x2) \ + (_mm_shuffle_epi32(x0, _MM_SHUFFLE(2*x1+1,2*x1,2*x2+1,2*x2))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(x0, x1) \ + (_mm_shuffle_epi32(x0, _MM_SHUFFLE((x1+3)%4,(x1+2)%4,(x1+1)%4,x1%4))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes64(x0, x1) \ + (_mm_shuffle_epi32(x0, _MM_SHUFFLE((2*x1+3)%4,(2*x1+2)%4,(2*x1+1)%4,(2*x1)%4))) + +#define Lib_IntVector_Intrinsics_vec128_load32_le(x0) \ + (_mm_loadu_si128((__m128i*)(x0))) + +#define Lib_IntVector_Intrinsics_vec128_load64_le(x0) \ + (_mm_loadu_si128((__m128i*)(x0))) + +#define Lib_IntVector_Intrinsics_vec128_store32_le(x0, x1) \ + (_mm_storeu_si128((__m128i*)(x0), x1)) + +#define Lib_IntVector_Intrinsics_vec128_store64_le(x0, x1) \ + (_mm_storeu_si128((__m128i*)(x0), x1)) + +#define Lib_IntVector_Intrinsics_vec128_load_be(x0) \ + (_mm_shuffle_epi8(_mm_loadu_si128((__m128i*)(x0)), _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15))) + +#define Lib_IntVector_Intrinsics_vec128_load32_be(x0) \ + (_mm_shuffle_epi8(_mm_loadu_si128((__m128i*)(x0)), _mm_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3))) + +#define Lib_IntVector_Intrinsics_vec128_load64_be(x0) \ + (_mm_shuffle_epi8(_mm_loadu_si128((__m128i*)(x0)), _mm_set_epi8(8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7))) + +#define Lib_IntVector_Intrinsics_vec128_store_be(x0, x1) \ + (_mm_storeu_si128((__m128i*)(x0), _mm_shuffle_epi8(x1, _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)))) + + +#define Lib_IntVector_Intrinsics_vec128_store32_be(x0, x1) \ + (_mm_storeu_si128((__m128i*)(x0), _mm_shuffle_epi8(x1, _mm_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3)))) + +#define Lib_IntVector_Intrinsics_vec128_store64_be(x0, x1) \ + (_mm_storeu_si128((__m128i*)(x0), _mm_shuffle_epi8(x1, _mm_set_epi8(8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7)))) + + + +#define Lib_IntVector_Intrinsics_vec128_insert8(x0, x1, x2) \ + (_mm_insert_epi8(x0, x1, x2)) + +#define Lib_IntVector_Intrinsics_vec128_insert32(x0, x1, x2) \ + (_mm_insert_epi32(x0, x1, x2)) + +#define Lib_IntVector_Intrinsics_vec128_insert64(x0, x1, x2) \ + (_mm_insert_epi64(x0, x1, x2)) + +#define Lib_IntVector_Intrinsics_vec128_extract8(x0, x1) \ + (_mm_extract_epi8(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_extract32(x0, x1) \ + (_mm_extract_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_extract64(x0, x1) \ + (_mm_extract_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_zero \ + (_mm_setzero_si128()) + + +#define Lib_IntVector_Intrinsics_vec128_add64(x0, x1) \ + (_mm_add_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_sub64(x0, x1) \ + (_mm_sub_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_mul64(x0, x1) \ + (_mm_mul_epu32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_smul64(x0, x1) \ + (_mm_mul_epu32(x0, _mm_set1_epi64x(x1))) + +#define Lib_IntVector_Intrinsics_vec128_add32(x0, x1) \ + (_mm_add_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_sub32(x0, x1) \ + (_mm_sub_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_mul32(x0, x1) \ + (_mm_mullo_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_smul32(x0, x1) \ + (_mm_mullo_epi32(x0, _mm_set1_epi32(x1))) + +#define Lib_IntVector_Intrinsics_vec128_load128(x) \ + ((__m128i)x) + +#define Lib_IntVector_Intrinsics_vec128_load64(x) \ + (_mm_set1_epi64x(x)) /* hi lo */ + +#define Lib_IntVector_Intrinsics_vec128_load64s(x0, x1) \ + (_mm_set_epi64x(x1, x0)) /* hi lo */ + +#define Lib_IntVector_Intrinsics_vec128_load32(x) \ + (_mm_set1_epi32(x)) + +#define Lib_IntVector_Intrinsics_vec128_load32s(x0, x1, x2, x3) \ + (_mm_set_epi32(x3, x2, x1, x0)) /* hi lo */ + +#define Lib_IntVector_Intrinsics_vec128_interleave_low32(x1, x2) \ + (_mm_unpacklo_epi32(x1, x2)) + +#define Lib_IntVector_Intrinsics_vec128_interleave_high32(x1, x2) \ + (_mm_unpackhi_epi32(x1, x2)) + +#define Lib_IntVector_Intrinsics_vec128_interleave_low64(x1, x2) \ + (_mm_unpacklo_epi64(x1, x2)) + +#define Lib_IntVector_Intrinsics_vec128_interleave_high64(x1, x2) \ + (_mm_unpackhi_epi64(x1, x2)) + +#endif /* HACL_CAN_COMPILE_VEC128 */ + +#if defined(HACL_CAN_COMPILE_VEC256) + +#include + +typedef __m256i Lib_IntVector_Intrinsics_vec256; + + +#define Lib_IntVector_Intrinsics_vec256_eq64(x0, x1) \ + (_mm256_cmpeq_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_eq32(x0, x1) \ + (_mm256_cmpeq_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_gt64(x0, x1) \ + (_mm256_cmpgt_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_gt32(x0, x1) \ + (_mm256_cmpgt_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_xor(x0, x1) \ + (_mm256_xor_si256(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_or(x0, x1) \ + (_mm256_or_si256(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_and(x0, x1) \ + (_mm256_and_si256(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_lognot(x0) \ + (_mm256_xor_si256(x0, _mm256_set1_epi32(-1))) + +#define Lib_IntVector_Intrinsics_vec256_shift_left(x0, x1) \ + (_mm256_slli_si256(x0, (x1)/8)) + +#define Lib_IntVector_Intrinsics_vec256_shift_right(x0, x1) \ + (_mm256_srli_si256(x0, (x1)/8)) + +#define Lib_IntVector_Intrinsics_vec256_shift_left64(x0, x1) \ + (_mm256_slli_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_shift_right64(x0, x1) \ + (_mm256_srli_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_shift_left32(x0, x1) \ + (_mm256_slli_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_shift_right32(x0, x1) \ + (_mm256_srli_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_rotate_left32_8(x0) \ + (_mm256_shuffle_epi8(x0, _mm256_set_epi8(14,13,12,15,10,9,8,11,6,5,4,7,2,1,0,3,14,13,12,15,10,9,8,11,6,5,4,7,2,1,0,3))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_left32_16(x0) \ + (_mm256_shuffle_epi8(x0, _mm256_set_epi8(13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_left32_24(x0) \ + (_mm256_shuffle_epi8(x0, _mm256_set_epi8(12,15,14,13,8,11,10,9,4,7,6,5,0,3,2,1,12,15,14,13,8,11,10,9,4,7,6,5,0,3,2,1))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_left32(x0,x1) \ + ((x1 == 8? Lib_IntVector_Intrinsics_vec256_rotate_left32_8(x0) : \ + (x1 == 16? Lib_IntVector_Intrinsics_vec256_rotate_left32_16(x0) : \ + (x1 == 24? Lib_IntVector_Intrinsics_vec256_rotate_left32_24(x0) : \ + _mm256_or_si256(_mm256_slli_epi32(x0,x1),_mm256_srli_epi32(x0,32-(x1))))))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right32(x0,x1) \ + (Lib_IntVector_Intrinsics_vec256_rotate_left32(x0,32-(x1))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right64_8(x0) \ + (_mm256_shuffle_epi8(x0, _mm256_set_epi8(8,15,14,13,12,11,10,9,0,7,6,5,4,3,2,1,8,15,14,13,12,11,10,9,0,7,6,5,4,3,2,1))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right64_16(x0) \ + (_mm256_shuffle_epi8(x0, _mm256_set_epi8(9,8,15,14,13,12,11,10,1,0,7,6,5,4,3,2,9,8,15,14,13,12,11,10,1,0,7,6,5,4,3,2))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right64_24(x0) \ + (_mm256_shuffle_epi8(x0, _mm256_set_epi8(10,9,8,15,14,13,12,11,2,1,0,7,6,5,4,3,10,9,8,15,14,13,12,11,2,1,0,7,6,5,4,3))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right64_32(x0) \ + (_mm256_shuffle_epi8(x0, _mm256_set_epi8(11,10,9,8,15,14,13,12,3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12,3,2,1,0,7,6,5,4))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right64_40(x0) \ + (_mm256_shuffle_epi8(x0, _mm256_set_epi8(12,11,10,9,8,15,14,13,4,3,2,1,0,7,6,5,12,11,10,9,8,15,14,13,4,3,2,1,0,7,6,5))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right64_48(x0) \ + (_mm256_shuffle_epi8(x0, _mm256_set_epi8(13,12,11,10,9,8,15,14,5,4,3,2,1,0,7,6,13,12,11,10,9,8,15,14,5,4,3,2,1,0,7,6))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right64_56(x0) \ + (_mm256_shuffle_epi8(x0, _mm256_set_epi8(14,13,12,11,10,9,8,15,6,5,4,3,2,1,0,7,14,13,12,11,10,9,8,15,6,5,4,3,2,1,0,7))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right64(x0,x1) \ + ((x1 == 8? Lib_IntVector_Intrinsics_vec256_rotate_right64_8(x0) : \ + (x1 == 16? Lib_IntVector_Intrinsics_vec256_rotate_right64_16(x0) : \ + (x1 == 24? Lib_IntVector_Intrinsics_vec256_rotate_right64_24(x0) : \ + (x1 == 32? Lib_IntVector_Intrinsics_vec256_rotate_right64_32(x0) : \ + (x1 == 40? Lib_IntVector_Intrinsics_vec256_rotate_right64_40(x0) : \ + (x1 == 48? Lib_IntVector_Intrinsics_vec256_rotate_right64_48(x0) : \ + (x1 == 56? Lib_IntVector_Intrinsics_vec256_rotate_right64_56(x0) : \ + _mm256_xor_si256(_mm256_srli_epi64((x0),(x1)),_mm256_slli_epi64((x0),(64-(x1)))))))))))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_left64(x0,x1) \ + (Lib_IntVector_Intrinsics_vec256_rotate_right64(x0,64-(x1))) + +#define Lib_IntVector_Intrinsics_vec256_shuffle64(x0, x1, x2, x3, x4) \ + (_mm256_permute4x64_epi64(x0, _MM_SHUFFLE(x4,x3,x2,x1))) + +#define Lib_IntVector_Intrinsics_vec256_shuffle32(x0, x1, x2, x3, x4, x5, x6, x7, x8) \ + (_mm256_permutevar8x32_epi32(x0, _mm256_set_epi32(x8,x7,x6,x5,x4,x3,x2,x1))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right_lanes32(x0, x1) \ + (_mm256_permutevar8x32_epi32(x0, _mm256_set_epi32((x1+7)%8,(x1+6)%8,(x1+5)%8,(x1+4)%8,(x1+3%8),(x1+2)%8,(x1+1)%8,x1%8))) + +#define Lib_IntVector_Intrinsics_vec256_rotate_right_lanes64(x0, x1) \ + (_mm256_permute4x64_epi64(x0, _MM_SHUFFLE((x1+3)%4,(x1+2)%4,(x1+1)%4,x1%4))) + +#define Lib_IntVector_Intrinsics_vec256_load32_le(x0) \ + (_mm256_loadu_si256((__m256i*)(x0))) + +#define Lib_IntVector_Intrinsics_vec256_load64_le(x0) \ + (_mm256_loadu_si256((__m256i*)(x0))) + +#define Lib_IntVector_Intrinsics_vec256_load32_be(x0) \ + (_mm256_shuffle_epi8(_mm256_loadu_si256((__m256i*)(x0)), _mm256_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3))) + +#define Lib_IntVector_Intrinsics_vec256_load64_be(x0) \ + (_mm256_shuffle_epi8(_mm256_loadu_si256((__m256i*)(x0)), _mm256_set_epi8(8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7))) + + +#define Lib_IntVector_Intrinsics_vec256_store32_le(x0, x1) \ + (_mm256_storeu_si256((__m256i*)(x0), x1)) + +#define Lib_IntVector_Intrinsics_vec256_store64_le(x0, x1) \ + (_mm256_storeu_si256((__m256i*)(x0), x1)) + +#define Lib_IntVector_Intrinsics_vec256_store32_be(x0, x1) \ + (_mm256_storeu_si256((__m256i*)(x0), _mm256_shuffle_epi8(x1, _mm256_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3)))) + +#define Lib_IntVector_Intrinsics_vec256_store64_be(x0, x1) \ + (_mm256_storeu_si256((__m256i*)(x0), _mm256_shuffle_epi8(x1, _mm256_set_epi8(8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7)))) + + +#define Lib_IntVector_Intrinsics_vec256_insert8(x0, x1, x2) \ + (_mm256_insert_epi8(x0, x1, x2)) + +#define Lib_IntVector_Intrinsics_vec256_insert32(x0, x1, x2) \ + (_mm256_insert_epi32(x0, x1, x2)) + +#define Lib_IntVector_Intrinsics_vec256_insert64(x0, x1, x2) \ + (_mm256_insert_epi64(x0, x1, x2)) + +#define Lib_IntVector_Intrinsics_vec256_extract8(x0, x1) \ + (_mm256_extract_epi8(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_extract32(x0, x1) \ + (_mm256_extract_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_extract64(x0, x1) \ + (_mm256_extract_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_zero \ + (_mm256_setzero_si256()) + +#define Lib_IntVector_Intrinsics_vec256_add64(x0, x1) \ + (_mm256_add_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_sub64(x0, x1) \ + (_mm256_sub_epi64(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_mul64(x0, x1) \ + (_mm256_mul_epu32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_smul64(x0, x1) \ + (_mm256_mul_epu32(x0, _mm256_set1_epi64x(x1))) + + +#define Lib_IntVector_Intrinsics_vec256_add32(x0, x1) \ + (_mm256_add_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_sub32(x0, x1) \ + (_mm256_sub_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_mul32(x0, x1) \ + (_mm256_mullo_epi32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec256_smul32(x0, x1) \ + (_mm256_mullo_epi32(x0, _mm256_set1_epi32(x1))) + + +#define Lib_IntVector_Intrinsics_vec256_load64(x1) \ + (_mm256_set1_epi64x(x1)) /* hi lo */ + +#define Lib_IntVector_Intrinsics_vec256_load64s(x0, x1, x2, x3) \ + (_mm256_set_epi64x(x3,x2,x1,x0)) /* hi lo */ + +#define Lib_IntVector_Intrinsics_vec256_load32(x) \ + (_mm256_set1_epi32(x)) + +#define Lib_IntVector_Intrinsics_vec256_load32s(x0,x1,x2,x3,x4, x5, x6, x7) \ + (_mm256_set_epi32(x7, x6, x5, x4, x3, x2, x1, x0)) /* hi lo */ + +#define Lib_IntVector_Intrinsics_vec256_load128(x) \ + (_mm256_set_m128i((__m128i)x)) + +#define Lib_IntVector_Intrinsics_vec256_load128s(x0,x1) \ + (_mm256_set_m128i((__m128i)x1,(__m128i)x0)) + +#define Lib_IntVector_Intrinsics_vec256_interleave_low32(x1, x2) \ + (_mm256_unpacklo_epi32(x1, x2)) + +#define Lib_IntVector_Intrinsics_vec256_interleave_high32(x1, x2) \ + (_mm256_unpackhi_epi32(x1, x2)) + +#define Lib_IntVector_Intrinsics_vec256_interleave_low64(x1, x2) \ + (_mm256_unpacklo_epi64(x1, x2)) + +#define Lib_IntVector_Intrinsics_vec256_interleave_high64(x1, x2) \ + (_mm256_unpackhi_epi64(x1, x2)) + +#define Lib_IntVector_Intrinsics_vec256_interleave_low128(x1, x2) \ + (_mm256_permute2x128_si256(x1, x2, 0x20)) + +#define Lib_IntVector_Intrinsics_vec256_interleave_high128(x1, x2) \ + (_mm256_permute2x128_si256(x1, x2, 0x31)) + +#endif /* HACL_CAN_COMPILE_VEC256 */ + +#elif (defined(__aarch64__) || defined(_M_ARM64) || defined(__arm__) || defined(_M_ARM)) \ + && !defined(__ARM_32BIT_STATE) + +#if defined(HACL_CAN_COMPILE_VEC128) + +#include + +typedef uint32x4_t Lib_IntVector_Intrinsics_vec128; + +#define Lib_IntVector_Intrinsics_vec128_xor(x0, x1) \ + (veorq_u32(x0,x1)) + +#define Lib_IntVector_Intrinsics_vec128_eq64(x0, x1) \ + (vceqq_u32(x0,x1)) + +#define Lib_IntVector_Intrinsics_vec128_eq32(x0, x1) \ + (vceqq_u32(x0,x1)) + +#define Lib_IntVector_Intrinsics_vec128_gt32(x0, x1) \ + (vcgtq_u32(x0, x1)) + +#define high32(x0) \ + (vmovn_u64(vshrq_n_u64(vreinterpretq_u64_u32(x0),32))) + +#define low32(x0) \ + (vmovn_u64(vreinterpretq_u64_u32(x0))) + +#define Lib_IntVector_Intrinsics_vec128_gt64(x0, x1) \ + (vreinterpretq_u32_u64(vmovl_u32(vorr_u32(vcgt_u32(high32(x0),high32(x1)),vand_u32(vceq_u32(high32(x0),high32(x1)),vcgt_u32(low32(x0),low32(x1))))))) + +#define Lib_IntVector_Intrinsics_vec128_or(x0, x1) \ + (vorrq_u32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_and(x0, x1) \ + (vandq_u32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_lognot(x0) \ + (vmvnq_u32(x0)) + + +#define Lib_IntVector_Intrinsics_vec128_shift_left(x0, x1) \ + (vextq_u32(x0, vdupq_n_u8(0), 16-(x1)/8)) + +#define Lib_IntVector_Intrinsics_vec128_shift_right(x0, x1) \ + (vextq_u32(x0, vdupq_n_u8(0), (x1)/8)) + +#define Lib_IntVector_Intrinsics_vec128_shift_left64(x0, x1) \ + (vreinterpretq_u32_u64(vshlq_n_u64(vreinterpretq_u64_u32(x0), x1))) + +#define Lib_IntVector_Intrinsics_vec128_shift_right64(x0, x1) \ + (vreinterpretq_u32_u64(vshrq_n_u64(vreinterpretq_u64_u32(x0), x1))) + +#define Lib_IntVector_Intrinsics_vec128_shift_left32(x0, x1) \ + (vshlq_n_u32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_shift_right32(x0, x1) \ + (vshrq_n_u32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_rotate_left32_16(x1) \ + (vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_left32(x0,x1) \ + (((x1) == 16? Lib_IntVector_Intrinsics_vec128_rotate_left32_16(x0) : \ + vsriq_n_u32(vshlq_n_u32((x0),(x1)),(x0),32-(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right32_16(x1) \ + (vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right32(x0,x1) \ + (((x1) == 16? Lib_IntVector_Intrinsics_vec128_rotate_right32_16(x0) : \ + vsriq_n_u32(vshlq_n_u32((x0),32-(x1)),(x0),(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(x0, x1) \ + (vextq_u32(x0,x0,x1)) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes64(x0, x1) \ + (vextq_u64(x0,x0,x1)) + + +/* +#define Lib_IntVector_Intrinsics_vec128_shuffle32(x0, x1, x2, x3, x4) \ + (_mm_shuffle_epi32(x0, _MM_SHUFFLE(x1,x2,x3,x4))) + +#define Lib_IntVector_Intrinsics_vec128_shuffle64(x0, x1, x2) \ + (_mm_shuffle_epi32(x0, _MM_SHUFFLE(2*x1+1,2*x1,2*x2+1,2*x2))) +*/ + +#define Lib_IntVector_Intrinsics_vec128_load32_le(x0) \ + (vld1q_u32((const uint32_t*) (x0))) + +#define Lib_IntVector_Intrinsics_vec128_load64_le(x0) \ + (vld1q_u32((const uint32_t*) (x0))) + +#define Lib_IntVector_Intrinsics_vec128_store32_le(x0, x1) \ + (vst1q_u32((uint32_t*)(x0),(x1))) + +#define Lib_IntVector_Intrinsics_vec128_store64_le(x0, x1) \ + (vst1q_u32((uint32_t*)(x0),(x1))) + +/* +#define Lib_IntVector_Intrinsics_vec128_load_be(x0) \ + ( Lib_IntVector_Intrinsics_vec128 l = vrev64q_u8(vld1q_u32((uint32_t*)(x0))); + +*/ + +#define Lib_IntVector_Intrinsics_vec128_load32_be(x0) \ + (vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(vld1q_u32((const uint32_t*)(x0)))))) + +#define Lib_IntVector_Intrinsics_vec128_load64_be(x0) \ + (vreinterpretq_u32_u8(vrev64q_u8(vreinterpretq_u8_u32(vld1q_u32((const uint32_t*)(x0)))))) + +/* +#define Lib_IntVector_Intrinsics_vec128_store_be(x0, x1) \ + (_mm_storeu_si128((__m128i*)(x0), _mm_shuffle_epi8(x1, _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)))) +*/ + +#define Lib_IntVector_Intrinsics_vec128_store32_be(x0, x1) \ + (vst1q_u32((uint32_t*)(x0),(vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(x1)))))) + +#define Lib_IntVector_Intrinsics_vec128_store64_be(x0, x1) \ + (vst1q_u32((uint32_t*)(x0),(vreinterpretq_u32_u8(vrev64q_u8(vreinterpretq_u8_u32(x1)))))) + +#define Lib_IntVector_Intrinsics_vec128_insert8(x0, x1, x2) \ + (vsetq_lane_u8(x1,x0,x2)) + +#define Lib_IntVector_Intrinsics_vec128_insert32(x0, x1, x2) \ + (vsetq_lane_u32(x1,x0,x2)) + +#define Lib_IntVector_Intrinsics_vec128_insert64(x0, x1, x2) \ + (vreinterpretq_u32_u64(vsetq_lane_u64(x1,vreinterpretq_u64_u32(x0),x2))) + +#define Lib_IntVector_Intrinsics_vec128_extract8(x0, x1) \ + (vgetq_lane_u8(x0,x1)) + +#define Lib_IntVector_Intrinsics_vec128_extract32(x0, x1) \ + (vgetq_lane_u32(x0,x1)) + +#define Lib_IntVector_Intrinsics_vec128_extract64(x0, x1) \ + (vgetq_lane_u64(vreinterpretq_u64_u32(x0),x1)) + +#define Lib_IntVector_Intrinsics_vec128_zero \ + (vdupq_n_u32(0)) + +#define Lib_IntVector_Intrinsics_vec128_add64(x0, x1) \ + (vreinterpretq_u32_u64(vaddq_u64(vreinterpretq_u64_u32(x0), vreinterpretq_u64_u32(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_sub64(x0, x1) \ + (vreinterpretq_u32_u64(vsubq_u64(vreinterpretq_u64_u32(x0), vreinterpretq_u64_u32(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_mul64(x0, x1) \ + (vreinterpretq_u32_u64(vmull_u32(vmovn_u64(vreinterpretq_u64_u32(x0)), vmovn_u64(vreinterpretq_u64_u32(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_smul64(x0, x1) \ + (vreinterpretq_u32_u64(vmull_n_u32(vmovn_u64(vreinterpretq_u64_u32(x0)), (uint32_t)x1))) + +#define Lib_IntVector_Intrinsics_vec128_add32(x0, x1) \ + (vaddq_u32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_sub32(x0, x1) \ + (vsubq_u32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_mul32(x0, x1) \ + (vmulq_lane_u32(x0, x1)) + +#define Lib_IntVector_Intrinsics_vec128_smul32(x0, x1) \ + (vmulq_lane_u32(x0, vdupq_n_u32(x1))) + +#define Lib_IntVector_Intrinsics_vec128_load128(x) \ + ((uint32x4_t)(x)) + +#define Lib_IntVector_Intrinsics_vec128_load64(x) \ + (vreinterpretq_u32_u64(vdupq_n_u64(x))) /* hi lo */ + +#define Lib_IntVector_Intrinsics_vec128_load32(x) \ + (vdupq_n_u32(x)) /* hi lo */ + +static inline Lib_IntVector_Intrinsics_vec128 Lib_IntVector_Intrinsics_vec128_load64s(uint64_t x1, uint64_t x2){ + const uint64_t a[2] = {x1,x2}; + return vreinterpretq_u32_u64(vld1q_u64(a)); +} + +static inline Lib_IntVector_Intrinsics_vec128 Lib_IntVector_Intrinsics_vec128_load32s(uint32_t x1, uint32_t x2, uint32_t x3, uint32_t x4){ + const uint32_t a[4] = {x1,x2,x3,x4}; + return vld1q_u32(a); +} + +#define Lib_IntVector_Intrinsics_vec128_interleave_low32(x1, x2) \ + (vzip1q_u32(x1,x2)) + +#define Lib_IntVector_Intrinsics_vec128_interleave_high32(x1, x2) \ + (vzip2q_u32(x1,x2)) + +#define Lib_IntVector_Intrinsics_vec128_interleave_low64(x1,x2) \ + (vreinterpretq_u32_u64(vzip1q_u64(vreinterpretq_u64_u32(x1),vreinterpretq_u64_u32(x2)))) + +#define Lib_IntVector_Intrinsics_vec128_interleave_high64(x1,x2) \ + (vreinterpretq_u32_u64(vzip2q_u64(vreinterpretq_u64_u32(x1),vreinterpretq_u64_u32(x2)))) + +#endif /* HACL_CAN_COMPILE_VEC128 */ + +/* IBM z architecture */ +#elif defined(__s390x__) /* this flag is for GCC only */ + +#if defined(HACL_CAN_COMPILE_VEC128) + +#include +#include + +/* The main vector 128 type + * We can't use uint8_t, uint32_t, uint64_t... instead of unsigned char, + * unsigned int, unsigned long long: the compiler complains that the parameter + * combination is invalid. */ +typedef unsigned char vector128_8 __attribute__ ((vector_size(16))); +typedef unsigned int vector128_32 __attribute__ ((vector_size(16))); +typedef unsigned long long vector128_64 __attribute__ ((vector_size(16))); + +typedef vector128_8 Lib_IntVector_Intrinsics_vec128; +typedef vector128_8 vector128; + +#define Lib_IntVector_Intrinsics_vec128_load32_le(x) \ + (vector128) ((vector128_32) vec_revb(*((vector128_32*) (const uint8_t*)(x)))) + +#define Lib_IntVector_Intrinsics_vec128_load32_be(x) \ + (vector128) (*((vector128_32*) (const uint8_t*)(x))) + +#define Lib_IntVector_Intrinsics_vec128_load64_le(x) \ + (vector128) ((vector128_64) vec_revb(*((vector128_64*) (const uint8_t*)(x)))) + +static inline +void Lib_IntVector_Intrinsics_vec128_store32_le(const uint8_t *x0, vector128 x1) { + *((vector128_32*)x0) = vec_revb((vector128_32) x1); +} + +static inline +void Lib_IntVector_Intrinsics_vec128_store32_be(const uint8_t *x0, vector128 x1) { + *((vector128_32*)x0) = (vector128_32) x1; +} + +static inline +void Lib_IntVector_Intrinsics_vec128_store64_le(const uint8_t *x0, vector128 x1) { + *((vector128_64*)x0) = vec_revb((vector128_64) x1); +} + +#define Lib_IntVector_Intrinsics_vec128_add32(x0,x1) \ + ((vector128)((vector128_32)(((vector128_32)(x0)) + ((vector128_32)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_add64(x0, x1) \ + ((vector128)((vector128_64)(((vector128_64)(x0)) + ((vector128_64)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_and(x0, x1) \ + ((vector128)(vec_and((vector128)(x0),(vector128)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_eq32(x0, x1) \ + ((vector128)(vec_cmpeq(((vector128_32)(x0)),((vector128_32)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_eq64(x0, x1) \ + ((vector128)(vec_cmpeq(((vector128_64)(x0)),((vector128_64)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_extract32(x0, x1) \ + ((unsigned int)(vec_extract((vector128_32)(x0), x1))) + +#define Lib_IntVector_Intrinsics_vec128_extract64(x0, x1) \ + ((unsigned long long)(vec_extract((vector128_64)(x0), x1))) + +#define Lib_IntVector_Intrinsics_vec128_gt32(x0, x1) \ + ((vector128)((vector128_32)(((vector128_32)(x0)) > ((vector128_32)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_gt64(x0, x1) \ + ((vector128)((vector128_64)(((vector128_64)(x0)) > ((vector128_64)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_insert32(x0, x1, x2) \ + ((vector128)((vector128_32)vec_insert((unsigned int)(x1), (vector128_32)(x0), x2))) + +#define Lib_IntVector_Intrinsics_vec128_insert64(x0, x1, x2) \ + ((vector128)((vector128_64)vec_insert((unsigned long long)(x1), (vector128_64)(x0), x2))) + +#define Lib_IntVector_Intrinsics_vec128_interleave_high32(x0, x1) \ + ((vector128)((vector128_32)vec_mergel((vector128_32)(x0), (vector128_32)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_interleave_high64(x0, x1) \ + ((vector128)((vector128_64)vec_mergel((vector128_64)(x0), (vector128_64)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_interleave_low32(x0, x1) \ + ((vector128)((vector128_32)vec_mergeh((vector128_32)(x0), (vector128_32)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_interleave_low64(x0, x1) \ + ((vector128)((vector128_64)vec_mergeh((vector128_64)(x0), (vector128_64)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_load32(x) \ + ((vector128)((vector128_32){(unsigned int)(x), (unsigned int)(x), \ + (unsigned int)(x), (unsigned int)(x)})) + +#define Lib_IntVector_Intrinsics_vec128_load32s(x0, x1, x2, x3) \ + ((vector128)((vector128_32){(unsigned int)(x0),(unsigned int)(x1),(unsigned int)(x2),(unsigned int)(x3)})) + +#define Lib_IntVector_Intrinsics_vec128_load64(x) \ + ((vector128)((vector128_64)vec_load_pair((unsigned long long)(x),(unsigned long long)(x)))) + +#define Lib_IntVector_Intrinsics_vec128_lognot(x0) \ + ((vector128)(vec_xor((vector128)(x0), (vector128)vec_splat_u32(-1)))) + +#define Lib_IntVector_Intrinsics_vec128_mul64(x0, x1) \ + ((vector128)(vec_mulo((vector128_32)(x0), \ + (vector128_32)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_or(x0, x1) \ + ((vector128)(vec_or((vector128)(x0),(vector128)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_left32(x0, x1) \ + ((vector128)(vec_rli((vector128_32)(x0), (unsigned long)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right32(x0, x1) \ + (Lib_IntVector_Intrinsics_vec128_rotate_left32(x0,(uint32_t)(32-(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(x0, x1) \ + ((vector128)(vec_sld((vector128)(x0), (vector128)(x0), (x1%4)*4))) + +#define Lib_IntVector_Intrinsics_vec128_shift_left64(x0, x1) \ + (((vector128)((vector128_64)vec_rli((vector128_64)(x0), (unsigned long)(x1)))) & \ + ((vector128)((vector128_64){0xffffffffffffffff << (x1), 0xffffffffffffffff << (x1)}))) + +#define Lib_IntVector_Intrinsics_vec128_shift_right64(x0, x1) \ + (((vector128)((vector128_64)vec_rli((vector128_64)(x0), (unsigned long)(64-(x1))))) & \ + ((vector128)((vector128_64){0xffffffffffffffff >> (x1), 0xffffffffffffffff >> (x1)}))) + +#define Lib_IntVector_Intrinsics_vec128_shift_right32(x0, x1) \ + (((vector128)((vector128_32)vec_rli((vector128_32)(x0), (unsigned int)(32-(x1))))) & \ + ((vector128)((vector128_32){0xffffffff >> (x1), 0xffffffff >> (x1), \ + 0xffffffff >> (x1), 0xffffffff >> (x1)}))) + +/* Doesn't work with vec_splat_u64 */ +#define Lib_IntVector_Intrinsics_vec128_smul64(x0, x1) \ + ((vector128)(Lib_IntVector_Intrinsics_vec128_mul64(x0,((vector128_64){(unsigned long long)(x1),(unsigned long long)(x1)})))) + +#define Lib_IntVector_Intrinsics_vec128_sub64(x0, x1) \ + ((vector128)((vector128_64)(x0) - (vector128_64)(x1))) + +static inline +vector128 Lib_IntVector_Intrinsics_vec128_xor(vector128 x0, vector128 x1) { + return ((vector128)(vec_xor((vector128)(x0), (vector128)(x1)))); +} + + +#define Lib_IntVector_Intrinsics_vec128_zero \ + ((vector128){}) + +#endif /* HACL_CAN_COMPILE_VEC128 */ + +#elif defined(__powerpc64__) // PowerPC 64 - this flag is for GCC only + +#if defined(HACL_CAN_COMPILE_VEC128) + +#include +#include // for memcpy +#include + +// The main vector 128 type +// We can't use uint8_t, uint32_t, uint64_t... instead of unsigned char, +// unsigned int, unsigned long long: the compiler complains that the parameter +// combination is invalid. +typedef vector unsigned char vector128_8; +typedef vector unsigned int vector128_32; +typedef vector unsigned long long vector128_64; + +typedef vector128_8 Lib_IntVector_Intrinsics_vec128; +typedef vector128_8 vector128; + +#define Lib_IntVector_Intrinsics_vec128_load32_le(x) \ + ((vector128)((vector128_32)(vec_xl(0, (const unsigned int*) ((const uint8_t*)(x)))))) + +#define Lib_IntVector_Intrinsics_vec128_load64_le(x) \ + ((vector128)((vector128_64)(vec_xl(0, (const unsigned long long*) ((const uint8_t*)(x)))))) + +#define Lib_IntVector_Intrinsics_vec128_store32_le(x0, x1) \ + (vec_xst((vector128_32)(x1), 0, (unsigned int*) ((uint8_t*)(x0)))) + +#define Lib_IntVector_Intrinsics_vec128_store64_le(x0, x1) \ + (vec_xst((vector128_64)(x1), 0, (unsigned long long*) ((uint8_t*)(x0)))) + +#define Lib_IntVector_Intrinsics_vec128_add32(x0,x1) \ + ((vector128)((vector128_32)(((vector128_32)(x0)) + ((vector128_32)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_add64(x0, x1) \ + ((vector128)((vector128_64)(((vector128_64)(x0)) + ((vector128_64)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_and(x0, x1) \ + ((vector128)(vec_and((vector128)(x0),(vector128)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_eq32(x0, x1) \ + ((vector128)(vec_cmpeq(((vector128_32)(x0)),((vector128_32)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_eq64(x0, x1) \ + ((vector128)(vec_cmpeq(((vector128_64)(x0)),((vector128_64)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_extract32(x0, x1) \ + ((unsigned int)(vec_extract((vector128_32)(x0), x1))) + +#define Lib_IntVector_Intrinsics_vec128_extract64(x0, x1) \ + ((unsigned long long)(vec_extract((vector128_64)(x0), x1))) + +#define Lib_IntVector_Intrinsics_vec128_gt32(x0, x1) \ + ((vector128)((vector128_32)(((vector128_32)(x0)) > ((vector128_32)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_gt64(x0, x1) \ + ((vector128)((vector128_64)(((vector128_64)(x0)) > ((vector128_64)(x1))))) + +#define Lib_IntVector_Intrinsics_vec128_insert32(x0, x1, x2) \ + ((vector128)((vector128_32)vec_insert((unsigned int)(x1), (vector128_32)(x0), x2))) + +#define Lib_IntVector_Intrinsics_vec128_insert64(x0, x1, x2) \ + ((vector128)((vector128_64)vec_insert((unsigned long long)(x1), (vector128_64)(x0), x2))) + +#define Lib_IntVector_Intrinsics_vec128_interleave_high32(x0, x1) \ + ((vector128)((vector128_32)vec_mergel((vector128_32)(x0), (vector128_32)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_interleave_high64(x0, x1) \ + ((vector128)((vector128_64)vec_mergel((vector128_64)(x0), (vector128_64)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_interleave_low32(x0, x1) \ + ((vector128)((vector128_32)vec_mergeh((vector128_32)(x0), (vector128_32)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_interleave_low64(x0, x1) \ + ((vector128)((vector128_64)vec_mergeh((vector128_64)(x0), (vector128_64)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_load32(x) \ + ((vector128)((vector128_32){(unsigned int)(x), (unsigned int)(x), \ + (unsigned int)(x), (unsigned int)(x)})) + +#define Lib_IntVector_Intrinsics_vec128_load32s(x0, x1, x2, x3) \ + ((vector128)((vector128_32){(unsigned int)(x0),(unsigned int)(x1),(unsigned int)(x2),(unsigned int)(x3)})) + +#define Lib_IntVector_Intrinsics_vec128_load64(x) \ + ((vector128)((vector128_64){(unsigned long long)(x),(unsigned long long)(x)})) + +#define Lib_IntVector_Intrinsics_vec128_lognot(x0) \ + ((vector128)(vec_xor((vector128)(x0), (vector128)vec_splat_u32(-1)))) + +#define Lib_IntVector_Intrinsics_vec128_mul64(x0, x1) \ + ((vector128)(vec_mule((vector128_32)(x0), \ + (vector128_32)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_or(x0, x1) \ + ((vector128)(vec_or((vector128)(x0),(vector128)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_left32(x0, x1) \ + ((vector128)(vec_rl((vector128_32)(x0), (vector128_32){(unsigned int)(x1),(unsigned int)(x1),(unsigned int)(x1),(unsigned int)(x1)}))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right32(x0, x1) \ + (Lib_IntVector_Intrinsics_vec128_rotate_left32(x0,(uint32_t)(32-(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(x0, x1) \ + ((vector128)(vec_sld((vector128)(x0), (vector128)(x0), ((4-(x1))%4)*4))) + +#define Lib_IntVector_Intrinsics_vec128_shift_left64(x0, x1) \ + ((vector128)((vector128_64)vec_sl((vector128_64)(x0), (vector128_64){(unsigned long)(x1),(unsigned long)(x1)}))) + +#define Lib_IntVector_Intrinsics_vec128_shift_right64(x0, x1) \ + ((vector128)((vector128_64)vec_sr((vector128_64)(x0), (vector128_64){(unsigned long)(x1),(unsigned long)(x1)}))) + +// Doesn't work with vec_splat_u64 +#define Lib_IntVector_Intrinsics_vec128_smul64(x0, x1) \ + ((vector128)(Lib_IntVector_Intrinsics_vec128_mul64(x0,((vector128_64){(unsigned long long)(x1),(unsigned long long)(x1)})))) + +#define Lib_IntVector_Intrinsics_vec128_sub64(x0, x1) \ + ((vector128)((vector128_64)(x0) - (vector128_64)(x1))) + +#define Lib_IntVector_Intrinsics_vec128_xor(x0, x1) \ + ((vector128)(vec_xor((vector128)(x0), (vector128)(x1)))) + +#define Lib_IntVector_Intrinsics_vec128_zero \ + ((vector128){}) + +#endif /* HACL_CAN_COMPILE_VEC128 */ + +#endif // PowerPC64 + +// DEBUGGING: +// If libintvector_debug.h exists, use it to debug the current implementations. +// Note that some flags must be enabled for the debugging to be effective: +// see libintvector_debug.h for more details. +#if defined(__has_include) +#if __has_include("libintvector_debug.h") +#include "libintvector_debug.h" +#endif +#endif + +#endif // __Vec_Intrin_H diff --git a/Modules/_hacl/python_hacl_namespaces.h b/Modules/_hacl/python_hacl_namespaces.h index 0df236282ac509..8a1f4aef384d62 100644 --- a/Modules/_hacl/python_hacl_namespaces.h +++ b/Modules/_hacl/python_hacl_namespaces.h @@ -5,59 +5,61 @@ * C's excuse for namespaces: Use globally unique names to avoid linkage * conflicts with builds linking or dynamically loading other code potentially * using HACL* libraries. + * + * Something like this to generate new entries for the list: nm *.o | grep Hacl | cut -c 20- | sort | uniq | grep -v python_hashlib | egrep ^_ | gsed 's/_\(.*\)/#define \1 python_hashlib_\1/' */ -#define Hacl_Streaming_SHA2_state_sha2_224_s python_hashlib_Hacl_Streaming_SHA2_state_sha2_224_s -#define Hacl_Streaming_SHA2_state_sha2_224 python_hashlib_Hacl_Streaming_SHA2_state_sha2_224 -#define Hacl_Streaming_SHA2_state_sha2_256 python_hashlib_Hacl_Streaming_SHA2_state_sha2_256 -#define Hacl_Streaming_SHA2_state_sha2_384_s python_hashlib_Hacl_Streaming_SHA2_state_sha2_384_s -#define Hacl_Streaming_SHA2_state_sha2_384 python_hashlib_Hacl_Streaming_SHA2_state_sha2_384 -#define Hacl_Streaming_SHA2_state_sha2_512 python_hashlib_Hacl_Streaming_SHA2_state_sha2_512 -#define Hacl_Streaming_SHA2_create_in_256 python_hashlib_Hacl_Streaming_SHA2_create_in_256 -#define Hacl_Streaming_SHA2_create_in_224 python_hashlib_Hacl_Streaming_SHA2_create_in_224 -#define Hacl_Streaming_SHA2_create_in_512 python_hashlib_Hacl_Streaming_SHA2_create_in_512 -#define Hacl_Streaming_SHA2_create_in_384 python_hashlib_Hacl_Streaming_SHA2_create_in_384 -#define Hacl_Streaming_SHA2_copy_256 python_hashlib_Hacl_Streaming_SHA2_copy_256 -#define Hacl_Streaming_SHA2_copy_224 python_hashlib_Hacl_Streaming_SHA2_copy_224 -#define Hacl_Streaming_SHA2_copy_512 python_hashlib_Hacl_Streaming_SHA2_copy_512 -#define Hacl_Streaming_SHA2_copy_384 python_hashlib_Hacl_Streaming_SHA2_copy_384 -#define Hacl_Streaming_SHA2_init_256 python_hashlib_Hacl_Streaming_SHA2_init_256 -#define Hacl_Streaming_SHA2_init_224 python_hashlib_Hacl_Streaming_SHA2_init_224 -#define Hacl_Streaming_SHA2_init_512 python_hashlib_Hacl_Streaming_SHA2_init_512 -#define Hacl_Streaming_SHA2_init_384 python_hashlib_Hacl_Streaming_SHA2_init_384 +#define Hacl_Hash_SHA2_state_sha2_224_s python_hashlib_Hacl_Hash_SHA2_state_sha2_224_s +#define Hacl_Hash_SHA2_state_sha2_224 python_hashlib_Hacl_Hash_SHA2_state_sha2_224 +#define Hacl_Hash_SHA2_state_sha2_256 python_hashlib_Hacl_Hash_SHA2_state_sha2_256 +#define Hacl_Hash_SHA2_state_sha2_384_s python_hashlib_Hacl_Hash_SHA2_state_sha2_384_s +#define Hacl_Hash_SHA2_state_sha2_384 python_hashlib_Hacl_Hash_SHA2_state_sha2_384 +#define Hacl_Hash_SHA2_state_sha2_512 python_hashlib_Hacl_Hash_SHA2_state_sha2_512 +#define Hacl_Hash_SHA2_malloc_256 python_hashlib_Hacl_Hash_SHA2_malloc_256 +#define Hacl_Hash_SHA2_malloc_224 python_hashlib_Hacl_Hash_SHA2_malloc_224 +#define Hacl_Hash_SHA2_malloc_512 python_hashlib_Hacl_Hash_SHA2_malloc_512 +#define Hacl_Hash_SHA2_malloc_384 python_hashlib_Hacl_Hash_SHA2_malloc_384 +#define Hacl_Hash_SHA2_copy_256 python_hashlib_Hacl_Hash_SHA2_copy_256 +#define Hacl_Hash_SHA2_copy_224 python_hashlib_Hacl_Hash_SHA2_copy_224 +#define Hacl_Hash_SHA2_copy_512 python_hashlib_Hacl_Hash_SHA2_copy_512 +#define Hacl_Hash_SHA2_copy_384 python_hashlib_Hacl_Hash_SHA2_copy_384 +#define Hacl_Hash_SHA2_init_256 python_hashlib_Hacl_Hash_SHA2_init_256 +#define Hacl_Hash_SHA2_init_224 python_hashlib_Hacl_Hash_SHA2_init_224 +#define Hacl_Hash_SHA2_init_512 python_hashlib_Hacl_Hash_SHA2_init_512 +#define Hacl_Hash_SHA2_init_384 python_hashlib_Hacl_Hash_SHA2_init_384 #define Hacl_SHA2_Scalar32_sha512_init python_hashlib_Hacl_SHA2_Scalar32_sha512_init -#define Hacl_Streaming_SHA2_update_256 python_hashlib_Hacl_Streaming_SHA2_update_256 -#define Hacl_Streaming_SHA2_update_224 python_hashlib_Hacl_Streaming_SHA2_update_224 -#define Hacl_Streaming_SHA2_update_512 python_hashlib_Hacl_Streaming_SHA2_update_512 -#define Hacl_Streaming_SHA2_update_384 python_hashlib_Hacl_Streaming_SHA2_update_384 -#define Hacl_Streaming_SHA2_finish_256 python_hashlib_Hacl_Streaming_SHA2_finish_256 -#define Hacl_Streaming_SHA2_finish_224 python_hashlib_Hacl_Streaming_SHA2_finish_224 -#define Hacl_Streaming_SHA2_finish_512 python_hashlib_Hacl_Streaming_SHA2_finish_512 -#define Hacl_Streaming_SHA2_finish_384 python_hashlib_Hacl_Streaming_SHA2_finish_384 -#define Hacl_Streaming_SHA2_free_256 python_hashlib_Hacl_Streaming_SHA2_free_256 -#define Hacl_Streaming_SHA2_free_224 python_hashlib_Hacl_Streaming_SHA2_free_224 -#define Hacl_Streaming_SHA2_free_512 python_hashlib_Hacl_Streaming_SHA2_free_512 -#define Hacl_Streaming_SHA2_free_384 python_hashlib_Hacl_Streaming_SHA2_free_384 -#define Hacl_Streaming_SHA2_sha256 python_hashlib_Hacl_Streaming_SHA2_sha256 -#define Hacl_Streaming_SHA2_sha224 python_hashlib_Hacl_Streaming_SHA2_sha224 -#define Hacl_Streaming_SHA2_sha512 python_hashlib_Hacl_Streaming_SHA2_sha512 -#define Hacl_Streaming_SHA2_sha384 python_hashlib_Hacl_Streaming_SHA2_sha384 +#define Hacl_Hash_SHA2_update_256 python_hashlib_Hacl_Hash_SHA2_update_256 +#define Hacl_Hash_SHA2_update_224 python_hashlib_Hacl_Hash_SHA2_update_224 +#define Hacl_Hash_SHA2_update_512 python_hashlib_Hacl_Hash_SHA2_update_512 +#define Hacl_Hash_SHA2_update_384 python_hashlib_Hacl_Hash_SHA2_update_384 +#define Hacl_Hash_SHA2_digest_256 python_hashlib_Hacl_Hash_SHA2_digest_256 +#define Hacl_Hash_SHA2_digest_224 python_hashlib_Hacl_Hash_SHA2_digest_224 +#define Hacl_Hash_SHA2_digest_512 python_hashlib_Hacl_Hash_SHA2_digest_512 +#define Hacl_Hash_SHA2_digest_384 python_hashlib_Hacl_Hash_SHA2_digest_384 +#define Hacl_Hash_SHA2_free_256 python_hashlib_Hacl_Hash_SHA2_free_256 +#define Hacl_Hash_SHA2_free_224 python_hashlib_Hacl_Hash_SHA2_free_224 +#define Hacl_Hash_SHA2_free_512 python_hashlib_Hacl_Hash_SHA2_free_512 +#define Hacl_Hash_SHA2_free_384 python_hashlib_Hacl_Hash_SHA2_free_384 +#define Hacl_Hash_SHA2_sha256 python_hashlib_Hacl_Hash_SHA2_sha256 +#define Hacl_Hash_SHA2_sha224 python_hashlib_Hacl_Hash_SHA2_sha224 +#define Hacl_Hash_SHA2_sha512 python_hashlib_Hacl_Hash_SHA2_sha512 +#define Hacl_Hash_SHA2_sha384 python_hashlib_Hacl_Hash_SHA2_sha384 -#define Hacl_Streaming_MD5_legacy_create_in python_hashlib_Hacl_Streaming_MD5_legacy_create_in -#define Hacl_Streaming_MD5_legacy_init python_hashlib_Hacl_Streaming_MD5_legacy_init -#define Hacl_Streaming_MD5_legacy_update python_hashlib_Hacl_Streaming_MD5_legacy_update -#define Hacl_Streaming_MD5_legacy_finish python_hashlib_Hacl_Streaming_MD5_legacy_finish -#define Hacl_Streaming_MD5_legacy_free python_hashlib_Hacl_Streaming_MD5_legacy_free -#define Hacl_Streaming_MD5_legacy_copy python_hashlib_Hacl_Streaming_MD5_legacy_copy -#define Hacl_Streaming_MD5_legacy_hash python_hashlib_Hacl_Streaming_MD5_legacy_hash +#define Hacl_Hash_MD5_malloc python_hashlib_Hacl_Hash_MD5_malloc +#define Hacl_Hash_MD5_init python_hashlib_Hacl_Hash_MD5_init +#define Hacl_Hash_MD5_update python_hashlib_Hacl_Hash_MD5_update +#define Hacl_Hash_MD5_digest python_hashlib_Hacl_Hash_MD5_digest +#define Hacl_Hash_MD5_free python_hashlib_Hacl_Hash_MD5_free +#define Hacl_Hash_MD5_copy python_hashlib_Hacl_Hash_MD5_copy +#define Hacl_Hash_MD5_hash python_hashlib_Hacl_Hash_MD5_hash -#define Hacl_Streaming_SHA1_legacy_create_in python_hashlib_Hacl_Streaming_SHA1_legacy_create_in -#define Hacl_Streaming_SHA1_legacy_init python_hashlib_Hacl_Streaming_SHA1_legacy_init -#define Hacl_Streaming_SHA1_legacy_update python_hashlib_Hacl_Streaming_SHA1_legacy_update -#define Hacl_Streaming_SHA1_legacy_finish python_hashlib_Hacl_Streaming_SHA1_legacy_finish -#define Hacl_Streaming_SHA1_legacy_free python_hashlib_Hacl_Streaming_SHA1_legacy_free -#define Hacl_Streaming_SHA1_legacy_copy python_hashlib_Hacl_Streaming_SHA1_legacy_copy -#define Hacl_Streaming_SHA1_legacy_hash python_hashlib_Hacl_Streaming_SHA1_legacy_hash +#define Hacl_Hash_SHA1_malloc python_hashlib_Hacl_Hash_SHA1_malloc +#define Hacl_Hash_SHA1_init python_hashlib_Hacl_Hash_SHA1_init +#define Hacl_Hash_SHA1_update python_hashlib_Hacl_Hash_SHA1_update +#define Hacl_Hash_SHA1_digest python_hashlib_Hacl_Hash_SHA1_digest +#define Hacl_Hash_SHA1_free python_hashlib_Hacl_Hash_SHA1_free +#define Hacl_Hash_SHA1_copy python_hashlib_Hacl_Hash_SHA1_copy +#define Hacl_Hash_SHA1_hash python_hashlib_Hacl_Hash_SHA1_hash #define Hacl_Hash_SHA3_update_last_sha3 python_hashlib_Hacl_Hash_SHA3_update_last_sha3 #define Hacl_Hash_SHA3_update_multi_sha3 python_hashlib_Hacl_Hash_SHA3_update_multi_sha3 @@ -72,15 +74,139 @@ #define Hacl_SHA3_sha3_512 python_hashlib_Hacl_SHA3_sha3_512 #define Hacl_SHA3_shake128_hacl python_hashlib_Hacl_SHA3_shake128_hacl #define Hacl_SHA3_shake256_hacl python_hashlib_Hacl_SHA3_shake256_hacl -#define Hacl_Streaming_Keccak_block_len python_hashlib_Hacl_Streaming_Keccak_block_len -#define Hacl_Streaming_Keccak_copy python_hashlib_Hacl_Streaming_Keccak_copy -#define Hacl_Streaming_Keccak_finish python_hashlib_Hacl_Streaming_Keccak_finish -#define Hacl_Streaming_Keccak_free python_hashlib_Hacl_Streaming_Keccak_free -#define Hacl_Streaming_Keccak_get_alg python_hashlib_Hacl_Streaming_Keccak_get_alg -#define Hacl_Streaming_Keccak_hash_len python_hashlib_Hacl_Streaming_Keccak_hash_len -#define Hacl_Streaming_Keccak_is_shake python_hashlib_Hacl_Streaming_Keccak_is_shake -#define Hacl_Streaming_Keccak_malloc python_hashlib_Hacl_Streaming_Keccak_malloc -#define Hacl_Streaming_Keccak_reset python_hashlib_Hacl_Streaming_Keccak_reset -#define Hacl_Streaming_Keccak_update python_hashlib_Hacl_Streaming_Keccak_update +#define Hacl_Hash_SHA3_block_len python_hashlib_Hacl_Hash_SHA3_block_len +#define Hacl_Hash_SHA3_copy python_hashlib_Hacl_Hash_SHA3_copy +#define Hacl_Hash_SHA3_digest python_hashlib_Hacl_Hash_SHA3_digest +#define Hacl_Hash_SHA3_free python_hashlib_Hacl_Hash_SHA3_free +#define Hacl_Hash_SHA3_get_alg python_hashlib_Hacl_Hash_SHA3_get_alg +#define Hacl_Hash_SHA3_hash_len python_hashlib_Hacl_Hash_SHA3_hash_len +#define Hacl_Hash_SHA3_is_shake python_hashlib_Hacl_Hash_SHA3_is_shake +#define Hacl_Hash_SHA3_malloc python_hashlib_Hacl_Hash_SHA3_malloc +#define Hacl_Hash_SHA3_reset python_hashlib_Hacl_Hash_SHA3_reset +#define Hacl_Hash_SHA3_update python_hashlib_Hacl_Hash_SHA3_update +#define Hacl_Hash_SHA3_squeeze python_hashlib_Hacl_Hash_SHA3_squeeze + +#define Hacl_Hash_Blake2b_Simd256_copy python_hashlib_Hacl_Hash_Blake2b_Simd256_copy +#define Hacl_Hash_Blake2b_Simd256_digest python_hashlib_Hacl_Hash_Blake2b_Simd256_digest +#define Hacl_Hash_Blake2b_Simd256_finish python_hashlib_Hacl_Hash_Blake2b_Simd256_finish +#define Hacl_Hash_Blake2b_Simd256_free python_hashlib_Hacl_Hash_Blake2b_Simd256_free +#define Hacl_Hash_Blake2b_Simd256_hash_with_key python_hashlib_Hacl_Hash_Blake2b_Simd256_hash_with_key +#define Hacl_Hash_Blake2b_Simd256_hash_with_key_and_params python_hashlib_Hacl_Hash_Blake2b_Simd256_hash_with_key_and_params +#define Hacl_Hash_Blake2b_Simd256_info python_hashlib_Hacl_Hash_Blake2b_Simd256_info +#define Hacl_Hash_Blake2b_Simd256_init python_hashlib_Hacl_Hash_Blake2b_Simd256_init +#define Hacl_Hash_Blake2b_Simd256_load_state256b_from_state32 python_hashlib_Hacl_Hash_Blake2b_Simd256_load_state256b_from_state32 +#define Hacl_Hash_Blake2b_Simd256_malloc python_hashlib_Hacl_Hash_Blake2b_Simd256_malloc +#define Hacl_Hash_Blake2b_Simd256_malloc_with_key python_hashlib_Hacl_Hash_Blake2b_Simd256_malloc_with_key +#define Hacl_Hash_Blake2b_Simd256_malloc_with_key0 python_hashlib_Hacl_Hash_Blake2b_Simd256_malloc_with_key0 +#define Hacl_Hash_Blake2b_Simd256_malloc_with_params_and_key python_hashlib_Hacl_Hash_Blake2b_Simd256_malloc_with_params_and_key +#define Hacl_Hash_Blake2b_Simd256_reset python_hashlib_Hacl_Hash_Blake2b_Simd256_reset +#define Hacl_Hash_Blake2b_Simd256_reset_with_key python_hashlib_Hacl_Hash_Blake2b_Simd256_reset_with_key +#define Hacl_Hash_Blake2b_Simd256_reset_with_key_and_params python_hashlib_Hacl_Hash_Blake2b_Simd256_reset_with_key_and_params +#define Hacl_Hash_Blake2b_Simd256_store_state256b_to_state32 python_hashlib_Hacl_Hash_Blake2b_Simd256_store_state256b_to_state32 +#define Hacl_Hash_Blake2b_Simd256_update python_hashlib_Hacl_Hash_Blake2b_Simd256_update +#define Hacl_Hash_Blake2b_Simd256_update_last python_hashlib_Hacl_Hash_Blake2b_Simd256_update_last +#define Hacl_Hash_Blake2b_Simd256_update_multi python_hashlib_Hacl_Hash_Blake2b_Simd256_update_multi +#define Hacl_Hash_Blake2b_copy python_hashlib_Hacl_Hash_Blake2b_copy +#define Hacl_Hash_Blake2b_digest python_hashlib_Hacl_Hash_Blake2b_digest +#define Hacl_Hash_Blake2b_finish python_hashlib_Hacl_Hash_Blake2b_finish +#define Hacl_Hash_Blake2b_free python_hashlib_Hacl_Hash_Blake2b_free +#define Hacl_Hash_Blake2b_hash_with_key python_hashlib_Hacl_Hash_Blake2b_hash_with_key +#define Hacl_Hash_Blake2b_hash_with_key_and_params python_hashlib_Hacl_Hash_Blake2b_hash_with_key_and_params +#define Hacl_Hash_Blake2b_info python_hashlib_Hacl_Hash_Blake2b_info +#define Hacl_Hash_Blake2b_init python_hashlib_Hacl_Hash_Blake2b_init +#define Hacl_Hash_Blake2b_malloc python_hashlib_Hacl_Hash_Blake2b_malloc +#define Hacl_Hash_Blake2b_malloc_with_key python_hashlib_Hacl_Hash_Blake2b_malloc_with_key +#define Hacl_Hash_Blake2b_malloc_with_params_and_key python_hashlib_Hacl_Hash_Blake2b_malloc_with_params_and_key +#define Hacl_Hash_Blake2b_reset python_hashlib_Hacl_Hash_Blake2b_reset +#define Hacl_Hash_Blake2b_reset_with_key python_hashlib_Hacl_Hash_Blake2b_reset_with_key +#define Hacl_Hash_Blake2b_reset_with_key_and_params python_hashlib_Hacl_Hash_Blake2b_reset_with_key_and_params +#define Hacl_Hash_Blake2b_update python_hashlib_Hacl_Hash_Blake2b_update +#define Hacl_Hash_Blake2b_update_last python_hashlib_Hacl_Hash_Blake2b_update_last +#define Hacl_Hash_Blake2b_update_multi python_hashlib_Hacl_Hash_Blake2b_update_multi +#define Hacl_Hash_Blake2s_Simd128_copy python_hashlib_Hacl_Hash_Blake2s_Simd128_copy +#define Hacl_Hash_Blake2s_Simd128_digest python_hashlib_Hacl_Hash_Blake2s_Simd128_digest +#define Hacl_Hash_Blake2s_Simd128_finish python_hashlib_Hacl_Hash_Blake2s_Simd128_finish +#define Hacl_Hash_Blake2s_Simd128_free python_hashlib_Hacl_Hash_Blake2s_Simd128_free +#define Hacl_Hash_Blake2s_Simd128_hash_with_key python_hashlib_Hacl_Hash_Blake2s_Simd128_hash_with_key +#define Hacl_Hash_Blake2s_Simd128_hash_with_key_and_params python_hashlib_Hacl_Hash_Blake2s_Simd128_hash_with_key_and_params +#define Hacl_Hash_Blake2s_Simd128_info python_hashlib_Hacl_Hash_Blake2s_Simd128_info +#define Hacl_Hash_Blake2s_Simd128_init python_hashlib_Hacl_Hash_Blake2s_Simd128_init +#define Hacl_Hash_Blake2s_Simd128_load_state128s_from_state32 python_hashlib_Hacl_Hash_Blake2s_Simd128_load_state128s_from_state32 +#define Hacl_Hash_Blake2s_Simd128_malloc python_hashlib_Hacl_Hash_Blake2s_Simd128_malloc +#define Hacl_Hash_Blake2s_Simd128_malloc_with_key python_hashlib_Hacl_Hash_Blake2s_Simd128_malloc_with_key +#define Hacl_Hash_Blake2s_Simd128_malloc_with_key0 python_hashlib_Hacl_Hash_Blake2s_Simd128_malloc_with_key0 +#define Hacl_Hash_Blake2s_Simd128_malloc_with_params_and_key python_hashlib_Hacl_Hash_Blake2s_Simd128_malloc_with_params_and_key +#define Hacl_Hash_Blake2s_Simd128_reset python_hashlib_Hacl_Hash_Blake2s_Simd128_reset +#define Hacl_Hash_Blake2s_Simd128_reset_with_key python_hashlib_Hacl_Hash_Blake2s_Simd128_reset_with_key +#define Hacl_Hash_Blake2s_Simd128_reset_with_key_and_params python_hashlib_Hacl_Hash_Blake2s_Simd128_reset_with_key_and_params +#define Hacl_Hash_Blake2s_Simd128_store_state128s_to_state32 python_hashlib_Hacl_Hash_Blake2s_Simd128_store_state128s_to_state32 +#define Hacl_Hash_Blake2s_Simd128_update python_hashlib_Hacl_Hash_Blake2s_Simd128_update +#define Hacl_Hash_Blake2s_Simd128_update_last python_hashlib_Hacl_Hash_Blake2s_Simd128_update_last +#define Hacl_Hash_Blake2s_Simd128_update_multi python_hashlib_Hacl_Hash_Blake2s_Simd128_update_multi +#define Hacl_Hash_Blake2s_copy python_hashlib_Hacl_Hash_Blake2s_copy +#define Hacl_Hash_Blake2s_digest python_hashlib_Hacl_Hash_Blake2s_digest +#define Hacl_Hash_Blake2s_finish python_hashlib_Hacl_Hash_Blake2s_finish +#define Hacl_Hash_Blake2s_free python_hashlib_Hacl_Hash_Blake2s_free +#define Hacl_Hash_Blake2s_hash_with_key python_hashlib_Hacl_Hash_Blake2s_hash_with_key +#define Hacl_Hash_Blake2s_hash_with_key_and_params python_hashlib_Hacl_Hash_Blake2s_hash_with_key_and_params +#define Hacl_Hash_Blake2s_info python_hashlib_Hacl_Hash_Blake2s_info +#define Hacl_Hash_Blake2s_init python_hashlib_Hacl_Hash_Blake2s_init +#define Hacl_Hash_Blake2s_malloc python_hashlib_Hacl_Hash_Blake2s_malloc +#define Hacl_Hash_Blake2s_malloc_with_key python_hashlib_Hacl_Hash_Blake2s_malloc_with_key +#define Hacl_Hash_Blake2s_malloc_with_params_and_key python_hashlib_Hacl_Hash_Blake2s_malloc_with_params_and_key +#define Hacl_Hash_Blake2s_reset python_hashlib_Hacl_Hash_Blake2s_reset +#define Hacl_Hash_Blake2s_reset_with_key python_hashlib_Hacl_Hash_Blake2s_reset_with_key +#define Hacl_Hash_Blake2s_reset_with_key_and_params python_hashlib_Hacl_Hash_Blake2s_reset_with_key_and_params +#define Hacl_Hash_Blake2s_update python_hashlib_Hacl_Hash_Blake2s_update +#define Hacl_Hash_Blake2s_update_last python_hashlib_Hacl_Hash_Blake2s_update_last +#define Hacl_Hash_Blake2s_update_multi python_hashlib_Hacl_Hash_Blake2s_update_multi +#define Hacl_Hash_MD5_finish python_hashlib_Hacl_Hash_MD5_finish +#define Hacl_Hash_MD5_hash_oneshot python_hashlib_Hacl_Hash_MD5_hash_oneshot +#define Hacl_Hash_MD5_reset python_hashlib_Hacl_Hash_MD5_reset +#define Hacl_Hash_MD5_update_last python_hashlib_Hacl_Hash_MD5_update_last +#define Hacl_Hash_MD5_update_multi python_hashlib_Hacl_Hash_MD5_update_multi +#define Hacl_Hash_SHA1_finish python_hashlib_Hacl_Hash_SHA1_finish +#define Hacl_Hash_SHA1_hash_oneshot python_hashlib_Hacl_Hash_SHA1_hash_oneshot +#define Hacl_Hash_SHA1_reset python_hashlib_Hacl_Hash_SHA1_reset +#define Hacl_Hash_SHA1_update_last python_hashlib_Hacl_Hash_SHA1_update_last +#define Hacl_Hash_SHA1_update_multi python_hashlib_Hacl_Hash_SHA1_update_multi +#define Hacl_Hash_SHA2_hash_224 python_hashlib_Hacl_Hash_SHA2_hash_224 +#define Hacl_Hash_SHA2_hash_256 python_hashlib_Hacl_Hash_SHA2_hash_256 +#define Hacl_Hash_SHA2_hash_384 python_hashlib_Hacl_Hash_SHA2_hash_384 +#define Hacl_Hash_SHA2_hash_512 python_hashlib_Hacl_Hash_SHA2_hash_512 +#define Hacl_Hash_SHA2_reset_224 python_hashlib_Hacl_Hash_SHA2_reset_224 +#define Hacl_Hash_SHA2_reset_256 python_hashlib_Hacl_Hash_SHA2_reset_256 +#define Hacl_Hash_SHA2_reset_384 python_hashlib_Hacl_Hash_SHA2_reset_384 +#define Hacl_Hash_SHA2_reset_512 python_hashlib_Hacl_Hash_SHA2_reset_512 +#define Hacl_Hash_SHA2_sha224_finish python_hashlib_Hacl_Hash_SHA2_sha224_finish +#define Hacl_Hash_SHA2_sha224_init python_hashlib_Hacl_Hash_SHA2_sha224_init +#define Hacl_Hash_SHA2_sha224_update_last python_hashlib_Hacl_Hash_SHA2_sha224_update_last +#define Hacl_Hash_SHA2_sha256_finish python_hashlib_Hacl_Hash_SHA2_sha256_finish +#define Hacl_Hash_SHA2_sha256_init python_hashlib_Hacl_Hash_SHA2_sha256_init +#define Hacl_Hash_SHA2_sha256_update_last python_hashlib_Hacl_Hash_SHA2_sha256_update_last +#define Hacl_Hash_SHA2_sha256_update_nblocks python_hashlib_Hacl_Hash_SHA2_sha256_update_nblocks +#define Hacl_Hash_SHA2_sha384_finish python_hashlib_Hacl_Hash_SHA2_sha384_finish +#define Hacl_Hash_SHA2_sha384_init python_hashlib_Hacl_Hash_SHA2_sha384_init +#define Hacl_Hash_SHA2_sha384_update_last python_hashlib_Hacl_Hash_SHA2_sha384_update_last +#define Hacl_Hash_SHA2_sha384_update_nblocks python_hashlib_Hacl_Hash_SHA2_sha384_update_nblocks +#define Hacl_Hash_SHA2_sha512_finish python_hashlib_Hacl_Hash_SHA2_sha512_finish +#define Hacl_Hash_SHA2_sha512_init python_hashlib_Hacl_Hash_SHA2_sha512_init +#define Hacl_Hash_SHA2_sha512_update_last python_hashlib_Hacl_Hash_SHA2_sha512_update_last +#define Hacl_Hash_SHA2_sha512_update_nblocks python_hashlib_Hacl_Hash_SHA2_sha512_update_nblocks +#define Hacl_Hash_SHA3_absorb_inner_32 python_hashlib_Hacl_Hash_SHA3_absorb_inner_32 +#define Hacl_Hash_SHA3_keccak_piln python_hashlib_Hacl_Hash_SHA3_keccak_piln +#define Hacl_Hash_SHA3_keccak_rndc python_hashlib_Hacl_Hash_SHA3_keccak_rndc +#define Hacl_Hash_SHA3_keccak_rotc python_hashlib_Hacl_Hash_SHA3_keccak_rotc +#define Hacl_Hash_SHA3_sha3_224 python_hashlib_Hacl_Hash_SHA3_sha3_224 +#define Hacl_Hash_SHA3_sha3_256 python_hashlib_Hacl_Hash_SHA3_sha3_256 +#define Hacl_Hash_SHA3_sha3_384 python_hashlib_Hacl_Hash_SHA3_sha3_384 +#define Hacl_Hash_SHA3_sha3_512 python_hashlib_Hacl_Hash_SHA3_sha3_512 +#define Hacl_Hash_SHA3_shake128 python_hashlib_Hacl_Hash_SHA3_shake128 +#define Hacl_Hash_SHA3_shake128_absorb_final python_hashlib_Hacl_Hash_SHA3_shake128_absorb_final +#define Hacl_Hash_SHA3_shake128_absorb_nblocks python_hashlib_Hacl_Hash_SHA3_shake128_absorb_nblocks +#define Hacl_Hash_SHA3_shake128_squeeze_nblocks python_hashlib_Hacl_Hash_SHA3_shake128_squeeze_nblocks +#define Hacl_Hash_SHA3_shake256 python_hashlib_Hacl_Hash_SHA3_shake256 +#define Hacl_Hash_SHA3_state_free python_hashlib_Hacl_Hash_SHA3_state_free +#define Hacl_Hash_SHA3_state_malloc python_hashlib_Hacl_Hash_SHA3_state_malloc #endif // _PYTHON_HACL_NAMESPACES_H diff --git a/Modules/_hacl/refresh.sh b/Modules/_hacl/refresh.sh index c1b3e37f3afb9d..4147ab302fe146 100755 --- a/Modules/_hacl/refresh.sh +++ b/Modules/_hacl/refresh.sh @@ -22,7 +22,7 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. -expected_hacl_star_rev=521af282fdf6d60227335120f18ae9309a4b8e8c +expected_hacl_star_rev=f218923ef2417d963d7efc7951593ae6aef613f7 hacl_dir="$(realpath "$1")" cd "$(dirname "$0")" @@ -40,19 +40,35 @@ fi declare -a dist_files dist_files=( - Hacl_Hash_SHA2.h Hacl_Streaming_Types.h - Hacl_Hash_SHA1.h - internal/Hacl_Hash_SHA1.h Hacl_Hash_MD5.h + Hacl_Hash_SHA1.h + Hacl_Hash_SHA2.h Hacl_Hash_SHA3.h + Hacl_Hash_Blake2b.h + Hacl_Hash_Blake2s.h + Hacl_Hash_Blake2b_Simd256.h + Hacl_Hash_Blake2s_Simd128.h internal/Hacl_Hash_MD5.h - internal/Hacl_Hash_SHA3.h - Hacl_Hash_SHA2.c + internal/Hacl_Hash_SHA1.h internal/Hacl_Hash_SHA2.h - Hacl_Hash_SHA1.c + internal/Hacl_Hash_SHA3.h + internal/Hacl_Hash_Blake2b.h + internal/Hacl_Hash_Blake2s.h + internal/Hacl_Hash_Blake2b_Simd256.h + internal/Hacl_Hash_Blake2s_Simd128.h + internal/Hacl_Impl_Blake2_Constants.h Hacl_Hash_MD5.c + Hacl_Hash_SHA1.c + Hacl_Hash_SHA2.c Hacl_Hash_SHA3.c + Hacl_Hash_Blake2b.c + Hacl_Hash_Blake2s.c + Hacl_Hash_Blake2b_Simd256.c + Hacl_Hash_Blake2s_Simd128.c + libintvector.h + lib_memzero0.h + Lib_Memzero0.c ) declare -a include_files @@ -127,13 +143,17 @@ $sed -i -z 's!\(extern\|typedef\)[^;]*;\n\n!!g' include/krml/FStar_UInt_8_16_32_ $sed -i 's!#include.*Hacl_Krmllib.h"!!g' "${all_files[@]}" # Use globally unique names for the Hacl_ C APIs to avoid linkage conflicts. -$sed -i -z 's!#include \n!#include \n#include "python_hacl_namespaces.h"\n!' Hacl_Hash_SHA2.h +$sed -i -z 's!#include \n!#include \n#include "python_hacl_namespaces.h"\n!' Hacl_Hash_*.h # Finally, we remove a bunch of ifdefs from target.h that are, again, useful in # the general case, but not exercised by the subset of HACL* that we vendor. -$sed -z -i 's!#ifndef KRML_\(PRE_ALIGN\|POST_ALIGN\|ALIGNED_MALLOC\|ALIGNED_FREE\|HOST_TIME\)\n\(\n\|# [^\n]*\n\|[^#][^\n]*\n\)*#endif\n\n!!g' include/krml/internal/target.h +$sed -z -i 's!#ifndef KRML_\(HOST_TIME\)\n\(\n\|# [^\n]*\n\|[^#][^\n]*\n\)*#endif\n\n!!g' include/krml/internal/target.h $sed -z -i 's!\n\n\([^#][^\n]*\n\)*#define KRML_\(EABORT\|EXIT\)[^\n]*\(\n [^\n]*\)*!!g' include/krml/internal/target.h $sed -z -i 's!\n\n\([^#][^\n]*\n\)*#if [^\n]*\n\( [^\n]*\n\)*#define KRML_\(EABORT\|EXIT\|CHECK_SIZE\)[^\n]*\(\n [^\n]*\)*!!g' include/krml/internal/target.h $sed -z -i 's!\n\n\([^#][^\n]*\n\)*#if [^\n]*\n\( [^\n]*\n\)*# define _\?KRML_\(DEPRECATED\|HOST_SNPRINTF\)[^\n]*\n\([^#][^\n]*\n\|#el[^\n]*\n\|# [^\n]*\n\)*#endif!!g' include/krml/internal/target.h +# Step 3: trim whitespace (for the linter) + +find . -name '*.c' -or -name '*.h' | xargs $sed -i 's![[:space:]]\+$!!' + echo "Updated; verify all is okay using git diff and git status." diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 0e230f332ff6cb..082929be3c77b7 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -25,7 +25,6 @@ #include #include "Python.h" #include "pycore_hashtable.h" -#include "pycore_pyhash.h" // _Py_HashBytes() #include "pycore_strhex.h" // _Py_strhex() #include "hashlib.h" @@ -45,9 +44,15 @@ #define MUNCH_SIZE INT_MAX #define PY_OPENSSL_HAS_SCRYPT 1 +#if defined(NID_sha3_224) && defined(NID_sha3_256) && defined(NID_sha3_384) && defined(NID_sha3_512) #define PY_OPENSSL_HAS_SHA3 1 +#endif +#if defined(NID_shake128) || defined(NID_shake256) #define PY_OPENSSL_HAS_SHAKE 1 +#endif +#if defined(NID_blake2s256) || defined(NID_blake2b512) #define PY_OPENSSL_HAS_BLAKE2 1 +#endif #if OPENSSL_VERSION_NUMBER >= 0x30000000L #define PY_EVP_MD EVP_MD @@ -88,22 +93,45 @@ typedef struct { PY_EVP_MD *evp_nosecurity; } py_hashentry_t; +// Fundamental to TLS, assumed always present in any libcrypto: #define Py_hash_md5 "md5" #define Py_hash_sha1 "sha1" #define Py_hash_sha224 "sha224" #define Py_hash_sha256 "sha256" #define Py_hash_sha384 "sha384" #define Py_hash_sha512 "sha512" -#define Py_hash_sha512_224 "sha512_224" -#define Py_hash_sha512_256 "sha512_256" -#define Py_hash_sha3_224 "sha3_224" -#define Py_hash_sha3_256 "sha3_256" -#define Py_hash_sha3_384 "sha3_384" -#define Py_hash_sha3_512 "sha3_512" -#define Py_hash_shake_128 "shake_128" -#define Py_hash_shake_256 "shake_256" -#define Py_hash_blake2s "blake2s" -#define Py_hash_blake2b "blake2b" + +// Not all OpenSSL-like libcrypto libraries provide these: +#if defined(NID_sha512_224) +# define Py_hash_sha512_224 "sha512_224" +#endif +#if defined(NID_sha512_256) +# define Py_hash_sha512_256 "sha512_256" +#endif +#if defined(NID_sha3_224) +# define Py_hash_sha3_224 "sha3_224" +#endif +#if defined(NID_sha3_256) +# define Py_hash_sha3_256 "sha3_256" +#endif +#if defined(NID_sha3_384) +# define Py_hash_sha3_384 "sha3_384" +#endif +#if defined(NID_sha3_512) +# define Py_hash_sha3_512 "sha3_512" +#endif +#if defined(NID_shake128) +# define Py_hash_shake_128 "shake_128" +#endif +#if defined(NID_shake256) +# define Py_hash_shake_256 "shake_256" +#endif +#if defined(NID_blake2s256) +# define Py_hash_blake2s "blake2s" +#endif +#if defined(NID_blake2b512) +# define Py_hash_blake2b "blake2b" +#endif #define PY_HASH_ENTRY(py_name, py_alias, ossl_name, ossl_nid) \ {py_name, py_alias, ossl_name, ossl_nid, 0, NULL, NULL} @@ -119,25 +147,45 @@ static const py_hashentry_t py_hashes[] = { PY_HASH_ENTRY(Py_hash_sha384, "SHA384", SN_sha384, NID_sha384), PY_HASH_ENTRY(Py_hash_sha512, "SHA512", SN_sha512, NID_sha512), /* truncated sha2 */ +#ifdef Py_hash_sha512_224 PY_HASH_ENTRY(Py_hash_sha512_224, "SHA512_224", SN_sha512_224, NID_sha512_224), +#endif +#ifdef Py_hash_sha512_256 PY_HASH_ENTRY(Py_hash_sha512_256, "SHA512_256", SN_sha512_256, NID_sha512_256), +#endif /* sha3 */ +#ifdef Py_hash_sha3_224 PY_HASH_ENTRY(Py_hash_sha3_224, NULL, SN_sha3_224, NID_sha3_224), +#endif +#ifdef Py_hash_sha3_256 PY_HASH_ENTRY(Py_hash_sha3_256, NULL, SN_sha3_256, NID_sha3_256), +#endif +#ifdef Py_hash_sha3_384 PY_HASH_ENTRY(Py_hash_sha3_384, NULL, SN_sha3_384, NID_sha3_384), +#endif +#ifdef Py_hash_sha3_512 PY_HASH_ENTRY(Py_hash_sha3_512, NULL, SN_sha3_512, NID_sha3_512), +#endif /* sha3 shake */ +#ifdef Py_hash_shake_128 PY_HASH_ENTRY(Py_hash_shake_128, NULL, SN_shake128, NID_shake128), +#endif +#ifdef Py_hash_shake_256 PY_HASH_ENTRY(Py_hash_shake_256, NULL, SN_shake256, NID_shake256), +#endif /* blake2 digest */ +#ifdef Py_hash_blake2s PY_HASH_ENTRY(Py_hash_blake2s, "blake2s256", SN_blake2s256, NID_blake2s256), +#endif +#ifdef Py_hash_blake2b PY_HASH_ENTRY(Py_hash_blake2b, "blake2b512", SN_blake2b512, NID_blake2b512), +#endif PY_HASH_ENTRY(NULL, NULL, NULL, 0), }; static Py_uhash_t py_hashentry_t_hash_name(const void *key) { - return _Py_HashBytes(key, strlen((const char *)key)); + return Py_HashBuffer(key, strlen((const char *)key)); } static int @@ -271,6 +319,7 @@ _setException(PyObject *exc, const char* altmsg, ...) va_end(vargs); ERR_clear_error(); + /* ERR_ERROR_STRING(3) ensures that the messages below are ASCII */ lib = ERR_lib_error_string(errcode); func = ERR_func_error_string(errcode); reason = ERR_reason_error_string(errcode); @@ -2240,6 +2289,7 @@ static PyModuleDef_Slot hashlib_slots[] = { {Py_mod_exec, hashlib_init_constructors}, {Py_mod_exec, hashlib_exception}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 9d4ec256ee9e3e..80fe9cff98509d 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -585,7 +585,7 @@ non-existing elements are considered to be infinite. The interesting\n\ property of a heap is that a[0] is always its smallest element.\n" "\n\ The strange invariant above is meant to be an efficient memory\n\ -representation for a tournament. The numbers below are `k', not a[k]:\n\ +representation for a tournament. The numbers below are 'k', not a[k]:\n\ \n\ 0\n\ \n\ @@ -598,7 +598,7 @@ representation for a tournament. The numbers below are `k', not a[k]:\n\ 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30\n\ \n\ \n\ -In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'. In\n\ +In the tree above, each cell 'k' is topping '2*k+1' and '2*k+2'. In\n\ a usual binary tournament we see in sports, each cell is the winner\n\ over the two cells it tops, and we can trace the winner down the tree\n\ to see all opponents s/he had. However, in many computer applications\n\ @@ -653,7 +653,7 @@ vanishes, you switch heaps and start a new run. Clever and quite\n\ effective!\n\ \n\ In a word, heaps are useful memory structures to know. I use them in\n\ -a few applications, and I think it is good to keep a `heap' module\n\ +a few applications, and I think it is good to keep a 'heap' module\n\ around. :-)\n" "\n\ --------------------\n\ @@ -681,6 +681,7 @@ heapq_exec(PyObject *m) static struct PyModuleDef_Slot heapq_slots[] = { {Py_mod_exec, heapq_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_interpchannelsmodule.c similarity index 86% rename from Modules/_xxinterpchannelsmodule.c rename to Modules/_interpchannelsmodule.c index 0ad184a78e8c1a..75d69ade1d3c9b 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -6,9 +6,9 @@ #endif #include "Python.h" -#include "interpreteridobject.h" -#include "pycore_crossinterp.h" // struct _xid +#include "pycore_crossinterp.h" // _PyXIData_t #include "pycore_interp.h" // _PyInterpreterState_LookUpID() +#include "pycore_pystate.h" // _PyInterpreterState_GetIDObject() #ifdef MS_WINDOWS #define WIN32_LEAN_AND_MEAN @@ -17,13 +17,18 @@ #include // sched_yield() #endif +#define REGISTERS_HEAP_TYPES +#define HAS_UNBOUND_ITEMS #include "_interpreters_common.h" +#undef HAS_UNBOUND_ITEMS +#undef REGISTERS_HEAP_TYPES /* This module has the following process-global state: _globals (static struct globals): + mutex (PyMutex) module_count (int) channels (struct _channels): numopen (int64_t) @@ -54,11 +59,11 @@ _globals (static struct globals): first (struct _channelitem *): next (struct _channelitem *): ... - data (_PyCrossInterpreterData *): + data (_PyXIData_t *): data (void *) obj (PyObject *) interpid (int64_t) - new_object (xid_newobjectfunc) + new_object (xid_newobjfunc) free (xid_freefunc) last (struct _channelitem *): ... @@ -75,14 +80,14 @@ The above state includes the following allocations by the module: * 1 struct _channelqueue * for each item in each channel: * 1 struct _channelitem - * 1 _PyCrossInterpreterData + * 1 _PyXIData_t The only objects in that global state are the references held by each -channel's queue, which are safely managed via the _PyCrossInterpreterData_*() +channel's queue, which are safely managed via the _PyXIData_*() API.. The module does not create any objects that are shared globally. */ -#define MODULE_NAME _xxinterpchannels +#define MODULE_NAME _interpchannels #define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME) #define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME) @@ -97,7 +102,7 @@ API.. The module does not create any objects that are shared globally. #define XID_FREE 2 static int -_release_xid_data(_PyCrossInterpreterData *data, int flags) +_release_xid_data(_PyXIData_t *data, int flags) { int ignoreexc = flags & XID_IGNORE_EXC; PyObject *exc; @@ -106,10 +111,10 @@ _release_xid_data(_PyCrossInterpreterData *data, int flags) } int res; if (flags & XID_FREE) { - res = _PyCrossInterpreterData_ReleaseAndRawFree(data); + res = _PyXIData_ReleaseAndRawFree(data); } else { - res = _PyCrossInterpreterData_Release(data); + res = _PyXIData_Release(data); } if (res < 0) { /* The owning interpreter is already destroyed. */ @@ -281,17 +286,17 @@ clear_xid_types(module_state *state) { /* external types */ if (state->send_channel_type != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass(state->send_channel_type); + (void)clear_xid_class(state->send_channel_type); Py_CLEAR(state->send_channel_type); } if (state->recv_channel_type != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass(state->recv_channel_type); + (void)clear_xid_class(state->recv_channel_type); Py_CLEAR(state->recv_channel_type); } /* heap types */ if (state->ChannelIDType != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass(state->ChannelIDType); + (void)clear_xid_class(state->ChannelIDType); Py_CLEAR(state->ChannelIDType); } } @@ -509,8 +514,14 @@ _waiting_finish_releasing(_waiting_t *waiting) struct _channelitem; typedef struct _channelitem { - _PyCrossInterpreterData *data; + /* The interpreter that added the item to the queue. + The actual bound interpid is found in item->data. + This is necessary because item->data might be NULL, + meaning the interpreter has been destroyed. */ + int64_t interpid; + _PyXIData_t *data; _waiting_t *waiting; + int unboundop; struct _channelitem *next; } _channelitem; @@ -522,11 +533,22 @@ _channelitem_ID(_channelitem *item) static void _channelitem_init(_channelitem *item, - _PyCrossInterpreterData *data, _waiting_t *waiting) + int64_t interpid, _PyXIData_t *data, + _waiting_t *waiting, int unboundop) { + if (interpid < 0) { + interpid = _get_interpid(data); + } + else { + assert(data == NULL + || _PyXIData_INTERPID(data) < 0 + || interpid == _PyXIData_INTERPID(data)); + } *item = (_channelitem){ + .interpid = interpid, .data = data, .waiting = waiting, + .unboundop = unboundop, }; if (waiting != NULL) { waiting->itemid = _channelitem_ID(item); @@ -534,17 +556,15 @@ _channelitem_init(_channelitem *item, } static void -_channelitem_clear(_channelitem *item) +_channelitem_clear_data(_channelitem *item, int removed) { - item->next = NULL; - if (item->data != NULL) { // It was allocated in channel_send(). (void)_release_xid_data(item->data, XID_IGNORE_EXC & XID_FREE); item->data = NULL; } - if (item->waiting != NULL) { + if (item->waiting != NULL && removed) { if (item->waiting->status == WAITING_ACQUIRED) { _waiting_release(item->waiting, 0); } @@ -552,15 +572,23 @@ _channelitem_clear(_channelitem *item) } } +static void +_channelitem_clear(_channelitem *item) +{ + item->next = NULL; + _channelitem_clear_data(item, 1); +} + static _channelitem * -_channelitem_new(_PyCrossInterpreterData *data, _waiting_t *waiting) +_channelitem_new(int64_t interpid, _PyXIData_t *data, + _waiting_t *waiting, int unboundop) { _channelitem *item = GLOBAL_MALLOC(_channelitem); if (item == NULL) { PyErr_NoMemory(); return NULL; } - _channelitem_init(item, data, waiting); + _channelitem_init(item, interpid, data, waiting, unboundop); return item; } @@ -583,17 +611,48 @@ _channelitem_free_all(_channelitem *item) static void _channelitem_popped(_channelitem *item, - _PyCrossInterpreterData **p_data, _waiting_t **p_waiting) + _PyXIData_t **p_data, _waiting_t **p_waiting, + int *p_unboundop) { assert(item->waiting == NULL || item->waiting->status == WAITING_ACQUIRED); *p_data = item->data; *p_waiting = item->waiting; + *p_unboundop = item->unboundop; // We clear them here, so they won't be released in _channelitem_clear(). item->data = NULL; item->waiting = NULL; _channelitem_free(item); } +static int +_channelitem_clear_interpreter(_channelitem *item) +{ + assert(item->interpid >= 0); + if (item->data == NULL) { + // Its interpreter was already cleared (or it was never bound). + // For UNBOUND_REMOVE it should have been freed at that time. + assert(item->unboundop != UNBOUND_REMOVE); + return 0; + } + assert(_PyXIData_INTERPID(item->data) == item->interpid); + + switch (item->unboundop) { + case UNBOUND_REMOVE: + // The caller must free/clear it. + return 1; + case UNBOUND_ERROR: + case UNBOUND_REPLACE: + // We won't need the cross-interpreter data later + // so we completely throw it away. + _channelitem_clear_data(item, 0); + return 0; + default: + Py_FatalError("not reachable"); + return -1; + } +} + + typedef struct _channelqueue { int64_t count; _channelitem *first; @@ -632,9 +691,10 @@ _channelqueue_free(_channelqueue *queue) static int _channelqueue_put(_channelqueue *queue, - _PyCrossInterpreterData *data, _waiting_t *waiting) + int64_t interpid, _PyXIData_t *data, + _waiting_t *waiting, int unboundop) { - _channelitem *item = _channelitem_new(data, waiting); + _channelitem *item = _channelitem_new(interpid, data, waiting, unboundop); if (item == NULL) { return -1; } @@ -657,7 +717,8 @@ _channelqueue_put(_channelqueue *queue, static int _channelqueue_get(_channelqueue *queue, - _PyCrossInterpreterData **p_data, _waiting_t **p_waiting) + _PyXIData_t **p_data, _waiting_t **p_waiting, + int *p_unboundop) { _channelitem *item = queue->first; if (item == NULL) { @@ -669,7 +730,7 @@ _channelqueue_get(_channelqueue *queue, } queue->count -= 1; - _channelitem_popped(item, p_data, p_waiting); + _channelitem_popped(item, p_data, p_waiting, p_unboundop); return 0; } @@ -708,7 +769,7 @@ _channelqueue_find(_channelqueue *queue, _channelitem_id_t itemid, static void _channelqueue_remove(_channelqueue *queue, _channelitem_id_t itemid, - _PyCrossInterpreterData **p_data, _waiting_t **p_waiting) + _PyXIData_t **p_data, _waiting_t **p_waiting) { _channelitem *prev = NULL; _channelitem *item = NULL; @@ -735,7 +796,8 @@ _channelqueue_remove(_channelqueue *queue, _channelitem_id_t itemid, } queue->count -= 1; - _channelitem_popped(item, p_data, p_waiting); + int unboundop; + _channelitem_popped(item, p_data, p_waiting, &unboundop); } static void @@ -746,14 +808,17 @@ _channelqueue_clear_interpreter(_channelqueue *queue, int64_t interpid) while (next != NULL) { _channelitem *item = next; next = item->next; - if (_PyCrossInterpreterData_INTERPID(item->data) == interpid) { + int remove = (item->interpid == interpid) + ? _channelitem_clear_interpreter(item) + : 0; + if (remove) { + _channelitem_free(item); if (prev == NULL) { - queue->first = item->next; + queue->first = next; } else { - prev->next = item->next; + prev->next = next; } - _channelitem_free(item); queue->count -= 1; } else { @@ -1016,12 +1081,15 @@ typedef struct _channel { PyThread_type_lock mutex; _channelqueue *queue; _channelends *ends; + struct { + int unboundop; + } defaults; int open; struct _channel_closing *closing; } _channel_state; static _channel_state * -_channel_new(PyThread_type_lock mutex) +_channel_new(PyThread_type_lock mutex, int unboundop) { _channel_state *chan = GLOBAL_MALLOC(_channel_state); if (chan == NULL) { @@ -1039,6 +1107,7 @@ _channel_new(PyThread_type_lock mutex) GLOBAL_FREE(chan); return NULL; } + chan->defaults.unboundop = unboundop; chan->open = 1; chan->closing = NULL; return chan; @@ -1059,7 +1128,7 @@ _channel_free(_channel_state *chan) static int _channel_add(_channel_state *chan, int64_t interpid, - _PyCrossInterpreterData *data, _waiting_t *waiting) + _PyXIData_t *data, _waiting_t *waiting, int unboundop) { int res = -1; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); @@ -1073,7 +1142,7 @@ _channel_add(_channel_state *chan, int64_t interpid, goto done; } - if (_channelqueue_put(chan->queue, data, waiting) != 0) { + if (_channelqueue_put(chan->queue, interpid, data, waiting, unboundop) != 0) { goto done; } // Any errors past this point must cause a _waiting_release() call. @@ -1086,7 +1155,7 @@ _channel_add(_channel_state *chan, int64_t interpid, static int _channel_next(_channel_state *chan, int64_t interpid, - _PyCrossInterpreterData **p_data, _waiting_t **p_waiting) + _PyXIData_t **p_data, _waiting_t **p_waiting, int *p_unboundop) { int err = 0; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); @@ -1100,11 +1169,15 @@ _channel_next(_channel_state *chan, int64_t interpid, goto done; } - int empty = _channelqueue_get(chan->queue, p_data, p_waiting); - assert(empty == 0 || empty == ERR_CHANNEL_EMPTY); + int empty = _channelqueue_get(chan->queue, p_data, p_waiting, p_unboundop); assert(!PyErr_Occurred()); - if (empty && chan->closing != NULL) { - chan->open = 0; + if (empty) { + assert(empty == ERR_CHANNEL_EMPTY); + if (chan->closing != NULL) { + chan->open = 0; + } + err = ERR_CHANNEL_EMPTY; + goto done; } done: @@ -1118,7 +1191,7 @@ _channel_next(_channel_state *chan, int64_t interpid, static void _channel_remove(_channel_state *chan, _channelitem_id_t itemid) { - _PyCrossInterpreterData *data = NULL; + _PyXIData_t *data = NULL; _waiting_t *waiting = NULL; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); @@ -1275,21 +1348,29 @@ typedef struct _channels { static void _channels_init(_channels *channels, PyThread_type_lock mutex) { - channels->mutex = mutex; - channels->head = NULL; - channels->numopen = 0; - channels->next_id = 0; + assert(mutex != NULL); + assert(channels->mutex == NULL); + *channels = (_channels){ + .mutex = mutex, + .head = NULL, + .numopen = 0, + .next_id = 0, + }; } static void -_channels_fini(_channels *channels) +_channels_fini(_channels *channels, PyThread_type_lock *p_mutex) { + PyThread_type_lock mutex = channels->mutex; + assert(mutex != NULL); + + PyThread_acquire_lock(mutex, WAIT_LOCK); assert(channels->numopen == 0); assert(channels->head == NULL); - if (channels->mutex != NULL) { - PyThread_free_lock(channels->mutex); - channels->mutex = NULL; - } + *channels = (_channels){0}; + PyThread_release_lock(mutex); + + *p_mutex = mutex; } static int64_t @@ -1526,18 +1607,27 @@ _channels_release_cid_object(_channels *channels, int64_t cid) PyThread_release_lock(channels->mutex); } -static int64_t * +struct channel_id_and_info { + int64_t id; + int unboundop; +}; + +static struct channel_id_and_info * _channels_list_all(_channels *channels, int64_t *count) { - int64_t *cids = NULL; + struct channel_id_and_info *cids = NULL; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); - int64_t *ids = PyMem_NEW(int64_t, (Py_ssize_t)(channels->numopen)); + struct channel_id_and_info *ids = + PyMem_NEW(struct channel_id_and_info, (Py_ssize_t)(channels->numopen)); if (ids == NULL) { goto done; } _channelref *ref = channels->head; for (int64_t i=0; ref != NULL; ref = ref->next, i++) { - ids[i] = ref->cid; + ids[i] = (struct channel_id_and_info){ + .id = ref->cid, + .unboundop = ref->chan->defaults.unboundop, + }; } *count = channels->numopen; @@ -1622,13 +1712,13 @@ _channel_finish_closing(_channel_state *chan) { // Create a new channel. static int64_t -channel_create(_channels *channels) +channel_create(_channels *channels, int unboundop) { PyThread_type_lock mutex = PyThread_allocate_lock(); if (mutex == NULL) { return ERR_CHANNEL_MUTEX_INIT; } - _channel_state *chan = _channel_new(mutex); + _channel_state *chan = _channel_new(mutex, unboundop); if (chan == NULL) { PyThread_free_lock(mutex); return -1; @@ -1660,7 +1750,7 @@ channel_destroy(_channels *channels, int64_t cid) // Optionally request to be notified when it is received. static int channel_send(_channels *channels, int64_t cid, PyObject *obj, - _waiting_t *waiting) + _waiting_t *waiting, int unboundop) { PyInterpreterState *interp = _get_current_interp(); if (interp == NULL) { @@ -1668,6 +1758,11 @@ channel_send(_channels *channels, int64_t cid, PyObject *obj, } int64_t interpid = PyInterpreterState_GetID(interp); + _PyXIData_lookup_context_t ctx; + if (_PyXIData_GetLookupContext(interp, &ctx) < 0) { + return -1; + } + // Look up the channel. PyThread_type_lock mutex = NULL; _channel_state *chan = NULL; @@ -1684,19 +1779,19 @@ channel_send(_channels *channels, int64_t cid, PyObject *obj, } // Convert the object to cross-interpreter data. - _PyCrossInterpreterData *data = GLOBAL_MALLOC(_PyCrossInterpreterData); + _PyXIData_t *data = GLOBAL_MALLOC(_PyXIData_t); if (data == NULL) { PyThread_release_lock(mutex); return -1; } - if (_PyObject_GetCrossInterpreterData(obj, data) != 0) { + if (_PyObject_GetXIData(&ctx, obj, data) != 0) { PyThread_release_lock(mutex); GLOBAL_FREE(data); return -1; } // Add the data to the channel. - int res = _channel_add(chan, interpid, data, waiting); + int res = _channel_add(chan, interpid, data, waiting, unboundop); PyThread_release_lock(mutex); if (res != 0) { // We may chain an exception here: @@ -1733,7 +1828,7 @@ channel_clear_sent(_channels *channels, int64_t cid, _waiting_t *waiting) // Like channel_send(), but strictly wait for the object to be received. static int channel_send_wait(_channels *channels, int64_t cid, PyObject *obj, - PY_TIMEOUT_T timeout) + int unboundop, PY_TIMEOUT_T timeout) { // We use a stack variable here, so we must ensure that &waiting // is not held by any channel item at the point this function exits. @@ -1744,7 +1839,7 @@ channel_send_wait(_channels *channels, int64_t cid, PyObject *obj, } /* Queue up the object. */ - int res = channel_send(channels, cid, obj, &waiting); + int res = channel_send(channels, cid, obj, &waiting, unboundop); if (res < 0) { assert(waiting.status == WAITING_NO_STATUS); goto finally; @@ -1786,7 +1881,7 @@ channel_send_wait(_channels *channels, int64_t cid, PyObject *obj, // The current interpreter gets associated with the recv end of the channel. // XXX Support a "wait" mutex? static int -channel_recv(_channels *channels, int64_t cid, PyObject **res) +channel_recv(_channels *channels, int64_t cid, PyObject **res, int *p_unboundop) { int err; *res = NULL; @@ -1812,20 +1907,22 @@ channel_recv(_channels *channels, int64_t cid, PyObject **res) // Past this point we are responsible for releasing the mutex. // Pop off the next item from the channel. - _PyCrossInterpreterData *data = NULL; + _PyXIData_t *data = NULL; _waiting_t *waiting = NULL; - err = _channel_next(chan, interpid, &data, &waiting); + err = _channel_next(chan, interpid, &data, &waiting, p_unboundop); PyThread_release_lock(mutex); if (err != 0) { return err; } else if (data == NULL) { + // The item was unbound. assert(!PyErr_Occurred()); + *res = NULL; return 0; } // Convert the data back to an object. - PyObject *obj = _PyCrossInterpreterData_NewObject(data); + PyObject *obj = _PyXIData_NewObject(data); if (obj == NULL) { assert(PyErr_Occurred()); // It was allocated in channel_send(), so we free it. @@ -1913,6 +2010,23 @@ channel_is_associated(_channels *channels, int64_t cid, int64_t interpid, return (end != NULL && end->open); } +static int +_channel_get_count(_channels *channels, int64_t cid, Py_ssize_t *p_count) +{ + PyThread_type_lock mutex = NULL; + _channel_state *chan = NULL; + int err = _channels_lookup(channels, cid, &mutex, &chan); + if (err != 0) { + return err; + } + assert(chan != NULL); + int64_t count = chan->queue->count; + PyThread_release_lock(mutex); + + *p_count = (Py_ssize_t)count; + return 0; +} + /* channel info */ @@ -1936,7 +2050,7 @@ struct channel_info { int recv; } cur; } status; - Py_ssize_t count; + int64_t count; }; static int @@ -1950,7 +2064,7 @@ _channel_get_info(_channels *channels, int64_t cid, struct channel_info *info) if (interp == NULL) { return -1; } - Py_ssize_t interpid = PyInterpreterState_GetID(interp); + int64_t interpid = PyInterpreterState_GetID(interp); // Hold the global lock until we're done. PyThread_acquire_lock(channels->mutex, WAIT_LOCK); @@ -2434,10 +2548,9 @@ struct _channelid_xid { }; static PyObject * -_channelid_from_xid(_PyCrossInterpreterData *data) +_channelid_from_xid(_PyXIData_t *data) { - struct _channelid_xid *xid = \ - (struct _channelid_xid *)_PyCrossInterpreterData_DATA(data); + struct _channelid_xid *xid = (struct _channelid_xid *)_PyXIData_DATA(data); // It might not be imported yet, so we can't use _get_current_module(). PyObject *mod = PyImport_ImportModule(MODULE_NAME_STR); @@ -2483,18 +2596,16 @@ _channelid_from_xid(_PyCrossInterpreterData *data) } static int -_channelid_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) +_channelid_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { - if (_PyCrossInterpreterData_InitWithSize( + if (_PyXIData_InitWithSize( data, tstate->interp, sizeof(struct _channelid_xid), obj, _channelid_from_xid ) < 0) { return -1; } - struct _channelid_xid *xid = \ - (struct _channelid_xid *)_PyCrossInterpreterData_DATA(data); + struct _channelid_xid *xid = (struct _channelid_xid *)_PyXIData_DATA(data); xid->cid = ((channelid *)obj)->cid; xid->end = ((channelid *)obj)->end; xid->resolve = ((channelid *)obj)->resolve; @@ -2613,10 +2724,10 @@ _get_current_channelend_type(int end) } if (cls == NULL) { // Force the module to be loaded, to register the type. - PyObject *highlevel = PyImport_ImportModule("interpreters.channel"); + PyObject *highlevel = PyImport_ImportModule("interpreters.channels"); if (highlevel == NULL) { PyErr_Clear(); - highlevel = PyImport_ImportModule("test.support.interpreters.channel"); + highlevel = PyImport_ImportModule("test.support.interpreters.channels"); if (highlevel == NULL) { return NULL; } @@ -2634,7 +2745,7 @@ _get_current_channelend_type(int end) } static PyObject * -_channelend_from_xid(_PyCrossInterpreterData *data) +_channelend_from_xid(_PyXIData_t *data) { channelid *cidobj = (channelid *)_channelid_from_xid(data); if (cidobj == NULL) { @@ -2651,8 +2762,7 @@ _channelend_from_xid(_PyCrossInterpreterData *data) } static int -_channelend_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) +_channelend_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { PyObject *cidobj = PyObject_GetAttrString(obj, "_id"); if (cidobj == NULL) { @@ -2663,7 +2773,7 @@ _channelend_shared(PyThreadState *tstate, PyObject *obj, if (res < 0) { return -1; } - _PyCrossInterpreterData_SET_NEW_OBJECT(data, _channelend_from_xid); + _PyXIData_SET_NEW_OBJECT(data, _channelend_from_xid); return 0; } @@ -2677,11 +2787,11 @@ set_channelend_types(PyObject *mod, PyTypeObject *send, PyTypeObject *recv) // Clear the old values if the .py module was reloaded. if (state->send_channel_type != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass(state->send_channel_type); + (void)clear_xid_class(state->send_channel_type); Py_CLEAR(state->send_channel_type); } if (state->recv_channel_type != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass(state->recv_channel_type); + (void)clear_xid_class(state->recv_channel_type); Py_CLEAR(state->recv_channel_type); } @@ -2694,7 +2804,7 @@ set_channelend_types(PyObject *mod, PyTypeObject *send, PyTypeObject *recv) return -1; } if (ensure_xid_class(recv, _channelend_shared) < 0) { - (void)_PyCrossInterpreterData_UnregisterClass(state->send_channel_type); + (void)clear_xid_class(state->send_channel_type); Py_CLEAR(state->send_channel_type); Py_CLEAR(state->recv_channel_type); return -1; @@ -2710,6 +2820,7 @@ set_channelend_types(PyObject *mod, PyTypeObject *send, PyTypeObject *recv) the data that we need to share between interpreters, so it cannot hold PyObject values. */ static struct globals { + PyMutex mutex; int module_count; _channels channels; } _globals = {0}; @@ -2717,32 +2828,36 @@ static struct globals { static int _globals_init(void) { - // XXX This isn't thread-safe. + PyMutex_Lock(&_globals.mutex); + assert(_globals.module_count >= 0); _globals.module_count++; - if (_globals.module_count > 1) { - // Already initialized. - return 0; - } - - assert(_globals.channels.mutex == NULL); - PyThread_type_lock mutex = PyThread_allocate_lock(); - if (mutex == NULL) { - return ERR_CHANNELS_MUTEX_INIT; + if (_globals.module_count == 1) { + // Called for the first time. + PyThread_type_lock mutex = PyThread_allocate_lock(); + if (mutex == NULL) { + _globals.module_count--; + PyMutex_Unlock(&_globals.mutex); + return ERR_CHANNELS_MUTEX_INIT; + } + _channels_init(&_globals.channels, mutex); } - _channels_init(&_globals.channels, mutex); + PyMutex_Unlock(&_globals.mutex); return 0; } static void _globals_fini(void) { - // XXX This isn't thread-safe. + PyMutex_Lock(&_globals.mutex); + assert(_globals.module_count > 0); _globals.module_count--; - if (_globals.module_count > 0) { - return; + if (_globals.module_count == 0) { + PyThread_type_lock mutex; + _channels_fini(&_globals.channels, &mutex); + assert(mutex != NULL); + PyThread_free_lock(mutex); } - - _channels_fini(&_globals.channels); + PyMutex_Unlock(&_globals.mutex); } static _channels * @@ -2765,9 +2880,22 @@ clear_interpreter(void *data) static PyObject * -channelsmod_create(PyObject *self, PyObject *Py_UNUSED(ignored)) +channelsmod_create(PyObject *self, PyObject *args, PyObject *kwds) { - int64_t cid = channel_create(&_globals.channels); + static char *kwlist[] = {"unboundop", NULL}; + int unboundop; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "i:create", kwlist, + &unboundop)) + { + return NULL; + } + if (!check_unbound(unboundop)) { + PyErr_Format(PyExc_ValueError, + "unsupported unboundop %d", unboundop); + return NULL; + } + + int64_t cid = channel_create(&_globals.channels, unboundop); if (cid < 0) { (void)handle_channel_error(-1, self, cid); return NULL; @@ -2794,7 +2922,7 @@ channelsmod_create(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(channelsmod_create_doc, -"channel_create() -> cid\n\ +"channel_create(unboundop) -> cid\n\ \n\ Create a new cross-interpreter channel and return a unique generated ID."); @@ -2829,7 +2957,8 @@ static PyObject * channelsmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) { int64_t count = 0; - int64_t *cids = _channels_list_all(&_globals.channels, &count); + struct channel_id_and_info *cids = + _channels_list_all(&_globals.channels, &count); if (cids == NULL) { if (count == 0) { return PyList_New(0); @@ -2846,19 +2975,26 @@ channelsmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) ids = NULL; goto finally; } - int64_t *cur = cids; + struct channel_id_and_info *cur = cids; for (int64_t i=0; i < count; cur++, i++) { PyObject *cidobj = NULL; - int err = newchannelid(state->ChannelIDType, *cur, 0, + int err = newchannelid(state->ChannelIDType, cur->id, 0, &_globals.channels, 0, 0, (channelid **)&cidobj); - if (handle_channel_error(err, self, *cur)) { + if (handle_channel_error(err, self, cur->id)) { assert(cidobj == NULL); Py_SETREF(ids, NULL); break; } assert(cidobj != NULL); - PyList_SET_ITEM(ids, (Py_ssize_t)i, cidobj); + + PyObject *item = Py_BuildValue("Oi", cidobj, cur->unboundop); + Py_DECREF(cidobj); + if (item == NULL) { + Py_SETREF(ids, NULL); + break; + } + PyList_SET_ITEM(ids, (Py_ssize_t)i, item); } finally: @@ -2906,7 +3042,7 @@ channelsmod_list_interpreters(PyObject *self, PyObject *args, PyObject *kwds) goto except; } if (res) { - interpid_obj = PyInterpreterState_GetIDObject(interp); + interpid_obj = _PyInterpreterState_GetIDObject(interp); if (interpid_obj == NULL) { goto except; } @@ -2940,16 +3076,24 @@ receive end."); static PyObject * channelsmod_send(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"cid", "obj", "blocking", "timeout", NULL}; + static char *kwlist[] = {"cid", "obj", "unboundop", "blocking", "timeout", + NULL}; struct channel_id_converter_data cid_data = { .module = self, }; PyObject *obj; + int unboundop = UNBOUND_REPLACE; int blocking = 1; PyObject *timeout_obj = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O|$pO:channel_send", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O|i$pO:channel_send", kwlist, channel_id_converter, &cid_data, &obj, - &blocking, &timeout_obj)) { + &unboundop, &blocking, &timeout_obj)) + { + return NULL; + } + if (!check_unbound(unboundop)) { + PyErr_Format(PyExc_ValueError, + "unsupported unboundop %d", unboundop); return NULL; } @@ -2962,10 +3106,10 @@ channelsmod_send(PyObject *self, PyObject *args, PyObject *kwds) /* Queue up the object. */ int err = 0; if (blocking) { - err = channel_send_wait(&_globals.channels, cid, obj, timeout); + err = channel_send_wait(&_globals.channels, cid, obj, unboundop, timeout); } else { - err = channel_send(&_globals.channels, cid, obj, NULL); + err = channel_send(&_globals.channels, cid, obj, NULL, unboundop); } if (handle_channel_error(err, self, cid)) { return NULL; @@ -2975,7 +3119,7 @@ channelsmod_send(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(channelsmod_send_doc, -"channel_send(cid, obj, blocking=True)\n\ +"channel_send(cid, obj, *, blocking=True, timeout=None)\n\ \n\ Add the object's data to the channel's queue.\n\ By default this waits for the object to be received."); @@ -2983,17 +3127,24 @@ By default this waits for the object to be received."); static PyObject * channelsmod_send_buffer(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"cid", "obj", "blocking", "timeout", NULL}; + static char *kwlist[] = {"cid", "obj", "unboundop", "blocking", "timeout", + NULL}; struct channel_id_converter_data cid_data = { .module = self, }; PyObject *obj; + int unboundop = UNBOUND_REPLACE; int blocking = 1; PyObject *timeout_obj = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O&O|$pO:channel_send_buffer", kwlist, + "O&O|i$pO:channel_send_buffer", kwlist, channel_id_converter, &cid_data, &obj, - &blocking, &timeout_obj)) { + &unboundop, &blocking, &timeout_obj)) { + return NULL; + } + if (!check_unbound(unboundop)) { + PyErr_Format(PyExc_ValueError, + "unsupported unboundop %d", unboundop); return NULL; } @@ -3011,10 +3162,11 @@ channelsmod_send_buffer(PyObject *self, PyObject *args, PyObject *kwds) /* Queue up the object. */ int err = 0; if (blocking) { - err = channel_send_wait(&_globals.channels, cid, tempobj, timeout); + err = channel_send_wait( + &_globals.channels, cid, tempobj, unboundop, timeout); } else { - err = channel_send(&_globals.channels, cid, tempobj, NULL); + err = channel_send(&_globals.channels, cid, tempobj, NULL, unboundop); } Py_DECREF(tempobj); if (handle_channel_error(err, self, cid)) { @@ -3025,7 +3177,7 @@ channelsmod_send_buffer(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(channelsmod_send_buffer_doc, -"channel_send_buffer(cid, obj, blocking=True)\n\ +"channel_send_buffer(cid, obj, *, blocking=True, timeout=None)\n\ \n\ Add the object's buffer to the channel's queue.\n\ By default this waits for the object to be received."); @@ -3046,25 +3198,28 @@ channelsmod_recv(PyObject *self, PyObject *args, PyObject *kwds) cid = cid_data.cid; PyObject *obj = NULL; - int err = channel_recv(&_globals.channels, cid, &obj); - if (handle_channel_error(err, self, cid)) { - return NULL; - } - Py_XINCREF(dflt); - if (obj == NULL) { + int unboundop = 0; + int err = channel_recv(&_globals.channels, cid, &obj, &unboundop); + if (err == ERR_CHANNEL_EMPTY && dflt != NULL) { // Use the default. - if (dflt == NULL) { - (void)handle_channel_error(ERR_CHANNEL_EMPTY, self, cid); - return NULL; - } obj = Py_NewRef(dflt); + err = 0; } - Py_XDECREF(dflt); - return obj; + else if (handle_channel_error(err, self, cid)) { + return NULL; + } + else if (obj == NULL) { + // The item was unbound. + return Py_BuildValue("Oi", Py_None, unboundop); + } + + PyObject *res = Py_BuildValue("OO", obj, Py_None); + Py_DECREF(obj); + return res; } PyDoc_STRVAR(channelsmod_recv_doc, -"channel_recv(cid, [default]) -> obj\n\ +"channel_recv(cid, [default]) -> (obj, unboundop)\n\ \n\ Return a new object from the data at the front of the channel's queue.\n\ \n\ @@ -3165,6 +3320,34 @@ Close the channel for the current interpreter. 'send' and 'recv'\n\ (bool) may be used to indicate the ends to close. By default both\n\ ends are closed. Closing an already closed end is a noop."); +static PyObject * +channelsmod_get_count(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"cid", NULL}; + struct channel_id_converter_data cid_data = { + .module = self, + }; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:get_count", kwlist, + channel_id_converter, &cid_data)) { + return NULL; + } + int64_t cid = cid_data.cid; + + Py_ssize_t count = -1; + int err = _channel_get_count(&_globals.channels, cid, &count); + if (handle_channel_error(err, self, cid)) { + return NULL; + } + assert(count >= 0); + return PyLong_FromSsize_t(count); +} + +PyDoc_STRVAR(channelsmod_get_count_doc, +"get_count(cid)\n\ +\n\ +Return the number of items in the channel."); + static PyObject * channelsmod_get_info(PyObject *self, PyObject *args, PyObject *kwds) { @@ -3192,6 +3375,38 @@ PyDoc_STRVAR(channelsmod_get_info_doc, \n\ Return details about the channel."); +static PyObject * +channelsmod_get_channel_defaults(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"cid", NULL}; + struct channel_id_converter_data cid_data = { + .module = self, + }; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:get_channel_defaults", kwlist, + channel_id_converter, &cid_data)) { + return NULL; + } + int64_t cid = cid_data.cid; + + PyThread_type_lock mutex = NULL; + _channel_state *channel = NULL; + int err = _channels_lookup(&_globals.channels, cid, &mutex, &channel); + if (handle_channel_error(err, self, cid)) { + return NULL; + } + int unboundop = channel->defaults.unboundop; + PyThread_release_lock(mutex); + + PyObject *defaults = Py_BuildValue("i", unboundop); + return defaults; +} + +PyDoc_STRVAR(channelsmod_get_channel_defaults_doc, +"get_channel_defaults(cid)\n\ +\n\ +Return the channel's default values, set when it was created."); + static PyObject * channelsmod__channel_id(PyObject *self, PyObject *args, PyObject *kwds) { @@ -3238,8 +3453,8 @@ channelsmod__register_end_types(PyObject *self, PyObject *args, PyObject *kwds) } static PyMethodDef module_functions[] = { - {"create", channelsmod_create, - METH_NOARGS, channelsmod_create_doc}, + {"create", _PyCFunction_CAST(channelsmod_create), + METH_VARARGS | METH_KEYWORDS, channelsmod_create_doc}, {"destroy", _PyCFunction_CAST(channelsmod_destroy), METH_VARARGS | METH_KEYWORDS, channelsmod_destroy_doc}, {"list_all", channelsmod_list_all, @@ -3256,8 +3471,12 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, channelsmod_close_doc}, {"release", _PyCFunction_CAST(channelsmod_release), METH_VARARGS | METH_KEYWORDS, channelsmod_release_doc}, + {"get_count", _PyCFunction_CAST(channelsmod_get_count), + METH_VARARGS | METH_KEYWORDS, channelsmod_get_count_doc}, {"get_info", _PyCFunction_CAST(channelsmod_get_info), METH_VARARGS | METH_KEYWORDS, channelsmod_get_info_doc}, + {"get_channel_defaults", _PyCFunction_CAST(channelsmod_get_channel_defaults), + METH_VARARGS | METH_KEYWORDS, channelsmod_get_channel_defaults_doc}, {"_channel_id", _PyCFunction_CAST(channelsmod__channel_id), METH_VARARGS | METH_KEYWORDS, NULL}, {"_register_end_types", _PyCFunction_CAST(channelsmod__register_end_types), @@ -3276,7 +3495,8 @@ The 'interpreters' module provides a more convenient interface."); static int module_exec(PyObject *mod) { - if (_globals_init() != 0) { + int err = _globals_init(); + if (handle_channel_error(err, mod, -1)) { return -1; } @@ -3324,6 +3544,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_interpqueuesmodule.c similarity index 72% rename from Modules/_xxinterpqueuesmodule.c rename to Modules/_interpqueuesmodule.c index 1b76b6963ae0f1..808938a9e8cd16 100644 --- a/Modules/_xxinterpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -6,12 +6,16 @@ #endif #include "Python.h" -#include "pycore_crossinterp.h" // struct _xid +#include "pycore_crossinterp.h" // _PyXIData_t +#define REGISTERS_HEAP_TYPES +#define HAS_UNBOUND_ITEMS #include "_interpreters_common.h" +#undef HAS_UNBOUND_ITEMS +#undef REGISTERS_HEAP_TYPES -#define MODULE_NAME _xxinterpqueues +#define MODULE_NAME _interpqueues #define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME) #define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME) @@ -26,7 +30,7 @@ #define XID_FREE 2 static int -_release_xid_data(_PyCrossInterpreterData *data, int flags) +_release_xid_data(_PyXIData_t *data, int flags) { int ignoreexc = flags & XID_IGNORE_EXC; PyObject *exc; @@ -35,10 +39,10 @@ _release_xid_data(_PyCrossInterpreterData *data, int flags) } int res; if (flags & XID_FREE) { - res = _PyCrossInterpreterData_ReleaseAndRawFree(data); + res = _PyXIData_ReleaseAndRawFree(data); } else { - res = _PyCrossInterpreterData_Release(data); + res = _PyXIData_Release(data); } if (res < 0) { /* The owning interpreter is already destroyed. */ @@ -56,7 +60,6 @@ _release_xid_data(_PyCrossInterpreterData *data, int flags) return res; } - static PyInterpreterState * _get_current_interp(void) { @@ -128,6 +131,22 @@ idarg_int64_converter(PyObject *arg, void *ptr) } +static int +ensure_highlevel_module_loaded(void) +{ + PyObject *highlevel = PyImport_ImportModule("interpreters.queues"); + if (highlevel == NULL) { + PyErr_Clear(); + highlevel = PyImport_ImportModule("test.support.interpreters.queues"); + if (highlevel == NULL) { + return -1; + } + } + Py_DECREF(highlevel); + return 0; +} + + /* module state *************************************************************/ typedef struct { @@ -170,7 +189,7 @@ clear_module_state(module_state *state) { /* external types */ if (state->queue_type != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass(state->queue_type); + (void)clear_xid_class(state->queue_type); } Py_CLEAR(state->queue_type); @@ -195,6 +214,9 @@ clear_module_state(module_state *state) // single-queue errors #define ERR_QUEUE_EMPTY (-21) #define ERR_QUEUE_FULL (-22) +#define ERR_QUEUE_NEVER_BOUND (-23) + +static int ensure_external_exc_types(module_state *); static int resolve_module_errcode(module_state *state, int errcode, int64_t qid, @@ -212,13 +234,23 @@ resolve_module_errcode(module_state *state, int errcode, int64_t qid, msg = PyUnicode_FromFormat("queue %" PRId64 " not found", qid); break; case ERR_QUEUE_EMPTY: + if (ensure_external_exc_types(state) < 0) { + return -1; + } exctype = state->QueueEmpty; msg = PyUnicode_FromFormat("queue %" PRId64 " is empty", qid); break; case ERR_QUEUE_FULL: + if (ensure_external_exc_types(state) < 0) { + return -1; + } exctype = state->QueueFull; msg = PyUnicode_FromFormat("queue %" PRId64 " is full", qid); break; + case ERR_QUEUE_NEVER_BOUND: + exctype = state->QueueError; + msg = PyUnicode_FromFormat("queue %" PRId64 " never bound", qid); + break; default: PyErr_Format(PyExc_ValueError, "unsupported error code %d", errcode); @@ -267,20 +299,59 @@ add_QueueError(PyObject *mod) #define PREFIX "test.support.interpreters." #define ADD_EXCTYPE(NAME, BASE, DOC) \ + assert(state->NAME == NULL); \ if (add_exctype(mod, &state->NAME, PREFIX #NAME, DOC, BASE) < 0) { \ return -1; \ } ADD_EXCTYPE(QueueError, PyExc_RuntimeError, "Indicates that a queue-related error happened.") ADD_EXCTYPE(QueueNotFoundError, state->QueueError, NULL) - ADD_EXCTYPE(QueueEmpty, state->QueueError, NULL) - ADD_EXCTYPE(QueueFull, state->QueueError, NULL) + // QueueEmpty and QueueFull are set by set_external_exc_types(). + state->QueueEmpty = NULL; + state->QueueFull = NULL; #undef ADD_EXCTYPE #undef PREFIX return 0; } +static int +set_external_exc_types(module_state *state, + PyObject *emptyerror, PyObject *fullerror) +{ + if (state->QueueEmpty != NULL) { + assert(state->QueueFull != NULL); + Py_CLEAR(state->QueueEmpty); + Py_CLEAR(state->QueueFull); + } + else { + assert(state->QueueFull == NULL); + } + assert(PyObject_IsSubclass(emptyerror, state->QueueError)); + assert(PyObject_IsSubclass(fullerror, state->QueueError)); + state->QueueEmpty = Py_NewRef(emptyerror); + state->QueueFull = Py_NewRef(fullerror); + return 0; +} + +static int +ensure_external_exc_types(module_state *state) +{ + if (state->QueueEmpty != NULL) { + assert(state->QueueFull != NULL); + return 0; + } + assert(state->QueueFull == NULL); + + // Force the module to be loaded, to register the type. + if (ensure_highlevel_module_loaded() < 0) { + return -1; + } + assert(state->QueueEmpty != NULL); + assert(state->QueueFull != NULL); + return 0; +} + static int handle_queue_error(int err, PyObject *mod, int64_t qid) { @@ -293,7 +364,7 @@ handle_queue_error(int err, PyObject *mod, int64_t qid) module_state *state; switch (err) { - case ERR_QUEUE_ALLOC: // fall through + case ERR_QUEUE_ALLOC: _Py_FALLTHROUGH; case ERR_QUEUES_ALLOC: PyErr_NoMemory(); break; @@ -324,42 +395,65 @@ handle_queue_error(int err, PyObject *mod, int64_t qid) struct _queueitem; typedef struct _queueitem { - _PyCrossInterpreterData *data; + /* The interpreter that added the item to the queue. + The actual bound interpid is found in item->data. + This is necessary because item->data might be NULL, + meaning the interpreter has been destroyed. */ + int64_t interpid; + _PyXIData_t *data; int fmt; + int unboundop; struct _queueitem *next; } _queueitem; static void _queueitem_init(_queueitem *item, - _PyCrossInterpreterData *data, int fmt) + int64_t interpid, _PyXIData_t *data, int fmt, int unboundop) { + if (interpid < 0) { + interpid = _get_interpid(data); + } + else { + assert(data == NULL + || _PyXIData_INTERPID(data) < 0 + || interpid == _PyXIData_INTERPID(data)); + } + assert(check_unbound(unboundop)); *item = (_queueitem){ + .interpid = interpid, .data = data, .fmt = fmt, + .unboundop = unboundop, }; } +static void +_queueitem_clear_data(_queueitem *item) +{ + if (item->data == NULL) { + return; + } + // It was allocated in queue_put(). + (void)_release_xid_data(item->data, XID_IGNORE_EXC & XID_FREE); + item->data = NULL; +} + static void _queueitem_clear(_queueitem *item) { item->next = NULL; - - if (item->data != NULL) { - // It was allocated in queue_put(). - (void)_release_xid_data(item->data, XID_IGNORE_EXC & XID_FREE); - item->data = NULL; - } + _queueitem_clear_data(item); } static _queueitem * -_queueitem_new(_PyCrossInterpreterData *data, int fmt) +_queueitem_new(int64_t interpid, _PyXIData_t *data, int fmt, int unboundop) { _queueitem *item = GLOBAL_MALLOC(_queueitem); if (item == NULL) { PyErr_NoMemory(); return NULL; } - _queueitem_init(item, data, fmt); + _queueitem_init(item, interpid, data, fmt, unboundop); return item; } @@ -382,17 +476,47 @@ _queueitem_free_all(_queueitem *item) static void _queueitem_popped(_queueitem *item, - _PyCrossInterpreterData **p_data, int *p_fmt) + _PyXIData_t **p_data, int *p_fmt, int *p_unboundop) { *p_data = item->data; *p_fmt = item->fmt; + *p_unboundop = item->unboundop; // We clear them here, so they won't be released in _queueitem_clear(). item->data = NULL; _queueitem_free(item); } +static int +_queueitem_clear_interpreter(_queueitem *item) +{ + assert(item->interpid >= 0); + if (item->data == NULL) { + // Its interpreter was already cleared (or it was never bound). + // For UNBOUND_REMOVE it should have been freed at that time. + assert(item->unboundop != UNBOUND_REMOVE); + return 0; + } + assert(_PyXIData_INTERPID(item->data) == item->interpid); + + switch (item->unboundop) { + case UNBOUND_REMOVE: + // The caller must free/clear it. + return 1; + case UNBOUND_ERROR: + case UNBOUND_REPLACE: + // We won't need the cross-interpreter data later + // so we completely throw it away. + _queueitem_clear_data(item); + return 0; + default: + Py_FatalError("not reachable"); + return -1; + } +} + /* the queue */ + typedef struct _queue { Py_ssize_t num_waiters; // protected by global lock PyThread_type_lock mutex; @@ -403,12 +527,16 @@ typedef struct _queue { _queueitem *first; _queueitem *last; } items; - int fmt; + struct { + int fmt; + int unboundop; + } defaults; } _queue; static int -_queue_init(_queue *queue, Py_ssize_t maxsize, int fmt) +_queue_init(_queue *queue, Py_ssize_t maxsize, int fmt, int unboundop) { + assert(check_unbound(unboundop)); PyThread_type_lock mutex = PyThread_allocate_lock(); if (mutex == NULL) { return ERR_QUEUE_ALLOC; @@ -419,7 +547,10 @@ _queue_init(_queue *queue, Py_ssize_t maxsize, int fmt) .items = { .maxsize = maxsize, }, - .fmt = fmt, + .defaults = { + .fmt = fmt, + .unboundop = unboundop, + }, }; return 0; } @@ -435,6 +566,8 @@ _queue_clear(_queue *queue) *queue = (_queue){0}; } +static void _queue_free(_queue *); + static void _queue_kill_and_wait(_queue *queue) { @@ -498,7 +631,8 @@ _queue_unlock(_queue *queue) } static int -_queue_add(_queue *queue, _PyCrossInterpreterData *data, int fmt) +_queue_add(_queue *queue, int64_t interpid, _PyXIData_t *data, + int fmt, int unboundop) { int err = _queue_lock(queue); if (err < 0) { @@ -514,7 +648,7 @@ _queue_add(_queue *queue, _PyCrossInterpreterData *data, int fmt) return ERR_QUEUE_FULL; } - _queueitem *item = _queueitem_new(data, fmt); + _queueitem *item = _queueitem_new(interpid, data, fmt, unboundop); if (item == NULL) { _queue_unlock(queue); return -1; @@ -535,7 +669,7 @@ _queue_add(_queue *queue, _PyCrossInterpreterData *data, int fmt) static int _queue_next(_queue *queue, - _PyCrossInterpreterData **p_data, int *p_fmt) + _PyXIData_t **p_data, int *p_fmt, int *p_unboundop) { int err = _queue_lock(queue); if (err < 0) { @@ -554,7 +688,7 @@ _queue_next(_queue *queue, } queue->items.count -= 1; - _queueitem_popped(item, p_data, p_fmt); + _queueitem_popped(item, p_data, p_fmt, p_unboundop); _queue_unlock(queue); return 0; @@ -619,14 +753,17 @@ _queue_clear_interpreter(_queue *queue, int64_t interpid) while (next != NULL) { _queueitem *item = next; next = item->next; - if (_PyCrossInterpreterData_INTERPID(item->data) == interpid) { + int remove = (item->interpid == interpid) + ? _queueitem_clear_interpreter(item) + : 0; + if (remove) { + _queueitem_free(item); if (prev == NULL) { - queue->items.first = item->next; + queue->items.first = next; } else { - prev->next = item->next; + prev->next = next; } - _queueitem_free(item); queue->items.count -= 1; } else { @@ -667,6 +804,32 @@ _queuerefs_find(_queueref *first, int64_t qid, _queueref **pprev) return ref; } +static void +_queuerefs_clear(_queueref *head) +{ + _queueref *next = head; + while (next != NULL) { + _queueref *ref = next; + next = ref->next; + +#ifdef Py_DEBUG + int64_t qid = ref->qid; + fprintf(stderr, "queue %" PRId64 " still exists\n", qid); +#endif + _queue *queue = ref->queue; + GLOBAL_FREE(ref); + + _queue_kill_and_wait(queue); +#ifdef Py_DEBUG + if (queue->items.count > 0) { + fprintf(stderr, "queue %" PRId64 " still holds %zd items\n", + qid, queue->items.count); + } +#endif + _queue_free(queue); + } +} + /* a collection of queues ***************************************************/ @@ -680,21 +843,31 @@ typedef struct _queues { static void _queues_init(_queues *queues, PyThread_type_lock mutex) { - queues->mutex = mutex; - queues->head = NULL; - queues->count = 0; - queues->next_id = 1; + assert(mutex != NULL); + assert(queues->mutex == NULL); + *queues = (_queues){ + .mutex = mutex, + .head = NULL, + .count = 0, + .next_id = 1, + }; } static void -_queues_fini(_queues *queues) +_queues_fini(_queues *queues, PyThread_type_lock *p_mutex) { - assert(queues->count == 0); - assert(queues->head == NULL); - if (queues->mutex != NULL) { - PyThread_free_lock(queues->mutex); - queues->mutex = NULL; + PyThread_type_lock mutex = queues->mutex; + assert(mutex != NULL); + + PyThread_acquire_lock(mutex, WAIT_LOCK); + if (queues->count > 0) { + assert(queues->head != NULL); + _queuerefs_clear(queues->head); } + *queues = (_queues){0}; + PyThread_release_lock(mutex); + + *p_mutex = mutex; } static int64_t @@ -822,19 +995,21 @@ _queues_incref(_queues *queues, int64_t qid) return res; } -static void _queue_free(_queue *); - -static void +static int _queues_decref(_queues *queues, int64_t qid) { + int res = -1; PyThread_acquire_lock(queues->mutex, WAIT_LOCK); _queueref *prev = NULL; _queueref *ref = _queuerefs_find(queues->head, qid, &prev); if (ref == NULL) { assert(!PyErr_Occurred()); - // Already destroyed. - // XXX Warn? + res = ERR_QUEUE_NOT_FOUND; + goto finally; + } + if (ref->refcount == 0) { + res = ERR_QUEUE_NEVER_BOUND; goto finally; } assert(ref->refcount > 0); @@ -849,25 +1024,28 @@ _queues_decref(_queues *queues, int64_t qid) _queue_kill_and_wait(queue); _queue_free(queue); - return; + return 0; } + res = 0; finally: PyThread_release_lock(queues->mutex); + return res; } -struct queue_id_and_fmt { +struct queue_id_and_info { int64_t id; int fmt; + int unboundop; }; -static struct queue_id_and_fmt * -_queues_list_all(_queues *queues, int64_t *count) +static struct queue_id_and_info * +_queues_list_all(_queues *queues, int64_t *p_count) { - struct queue_id_and_fmt *qids = NULL; + struct queue_id_and_info *qids = NULL; PyThread_acquire_lock(queues->mutex, WAIT_LOCK); - struct queue_id_and_fmt *ids = PyMem_NEW(struct queue_id_and_fmt, - (Py_ssize_t)(queues->count)); + struct queue_id_and_info *ids = PyMem_NEW(struct queue_id_and_info, + (Py_ssize_t)(queues->count)); if (ids == NULL) { goto done; } @@ -875,9 +1053,10 @@ _queues_list_all(_queues *queues, int64_t *count) for (int64_t i=0; ref != NULL; ref = ref->next, i++) { ids[i].id = ref->qid; assert(ref->queue != NULL); - ids[i].fmt = ref->queue->fmt; + ids[i].fmt = ref->queue->defaults.fmt; + ids[i].unboundop = ref->queue->defaults.unboundop; } - *count = queues->count; + *p_count = queues->count; qids = ids; done: @@ -911,13 +1090,13 @@ _queue_free(_queue *queue) // Create a new queue. static int64_t -queue_create(_queues *queues, Py_ssize_t maxsize, int fmt) +queue_create(_queues *queues, Py_ssize_t maxsize, int fmt, int unboundop) { _queue *queue = GLOBAL_MALLOC(_queue); if (queue == NULL) { return ERR_QUEUE_ALLOC; } - int err = _queue_init(queue, maxsize, fmt); + int err = _queue_init(queue, maxsize, fmt, unboundop); if (err < 0) { GLOBAL_FREE(queue); return (int64_t)err; @@ -946,8 +1125,14 @@ queue_destroy(_queues *queues, int64_t qid) // Push an object onto the queue. static int -queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt) +queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt, int unboundop) { + PyInterpreterState *interp = PyInterpreterState_Get(); + _PyXIData_lookup_context_t ctx; + if (_PyXIData_GetLookupContext(interp, &ctx) < 0) { + return -1; + } + // Look up the queue. _queue *queue = NULL; int err = _queues_lookup(queues, qid, &queue); @@ -957,19 +1142,21 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt) assert(queue != NULL); // Convert the object to cross-interpreter data. - _PyCrossInterpreterData *data = GLOBAL_MALLOC(_PyCrossInterpreterData); + _PyXIData_t *data = GLOBAL_MALLOC(_PyXIData_t); if (data == NULL) { _queue_unmark_waiter(queue, queues->mutex); return -1; } - if (_PyObject_GetCrossInterpreterData(obj, data) != 0) { + if (_PyObject_GetXIData(&ctx, obj, data) != 0) { _queue_unmark_waiter(queue, queues->mutex); GLOBAL_FREE(data); return -1; } + assert(_PyXIData_INTERPID(data) == PyInterpreterState_GetID(interp)); // Add the data to the queue. - int res = _queue_add(queue, data, fmt); + int64_t interpid = -1; // _queueitem_init() will set it. + int res = _queue_add(queue, interpid, data, fmt, unboundop); _queue_unmark_waiter(queue, queues->mutex); if (res != 0) { // We may chain an exception here: @@ -984,7 +1171,8 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt) // Pop the next object off the queue. Fail if empty. // XXX Support a "wait" mutex? static int -queue_get(_queues *queues, int64_t qid, PyObject **res, int *p_fmt) +queue_get(_queues *queues, int64_t qid, + PyObject **res, int *p_fmt, int *p_unboundop) { int err; *res = NULL; @@ -999,8 +1187,8 @@ queue_get(_queues *queues, int64_t qid, PyObject **res, int *p_fmt) assert(queue != NULL); // Pop off the next item from the queue. - _PyCrossInterpreterData *data = NULL; - err = _queue_next(queue, &data, p_fmt); + _PyXIData_t *data = NULL; + err = _queue_next(queue, &data, p_fmt, p_unboundop); _queue_unmark_waiter(queue, queues->mutex); if (err != 0) { return err; @@ -1011,7 +1199,7 @@ queue_get(_queues *queues, int64_t qid, PyObject **res, int *p_fmt) } // Convert the data back to an object. - PyObject *obj = _PyCrossInterpreterData_NewObject(data); + PyObject *obj = _PyXIData_NewObject(data); if (obj == NULL) { assert(PyErr_Occurred()); // It was allocated in queue_put(), so we free it. @@ -1073,18 +1261,14 @@ queue_get_count(_queues *queues, int64_t qid, Py_ssize_t *p_count) /* external Queue objects ***************************************************/ -static int _queueobj_shared(PyThreadState *, - PyObject *, _PyCrossInterpreterData *); +static int _queueobj_shared(PyThreadState *, PyObject *, _PyXIData_t *); static int -set_external_queue_type(PyObject *module, PyTypeObject *queue_type) +set_external_queue_type(module_state *state, PyTypeObject *queue_type) { - module_state *state = get_module_state(module); - // Clear the old value if the .py module was reloaded. if (state->queue_type != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass( - state->queue_type); + (void)clear_xid_class(state->queue_type); Py_CLEAR(state->queue_type); } @@ -1105,15 +1289,9 @@ get_external_queue_type(PyObject *module) PyTypeObject *cls = state->queue_type; if (cls == NULL) { // Force the module to be loaded, to register the type. - PyObject *highlevel = PyImport_ImportModule("interpreters.queue"); - if (highlevel == NULL) { - PyErr_Clear(); - highlevel = PyImport_ImportModule("test.support.interpreters.queue"); - if (highlevel == NULL) { - return NULL; - } + if (ensure_highlevel_module_loaded() < 0) { + return NULL; } - Py_DECREF(highlevel); cls = state->queue_type; assert(cls != NULL); } @@ -1139,7 +1317,7 @@ _queueid_xid_new(int64_t qid) struct _queueid_xid *data = PyMem_RawMalloc(sizeof(struct _queueid_xid)); if (data == NULL) { - _queues_incref(queues, qid); + _queues_decref(queues, qid); return NULL; } data->qid = qid; @@ -1152,13 +1330,20 @@ _queueid_xid_free(void *data) int64_t qid = ((struct _queueid_xid *)data)->qid; PyMem_RawFree(data); _queues *queues = _get_global_queues(); - _queues_decref(queues, qid); + int res = _queues_decref(queues, qid); + if (res == ERR_QUEUE_NOT_FOUND) { + // Already destroyed. + // XXX Warn? + } + else { + assert(res == 0); + } } static PyObject * -_queueobj_from_xid(_PyCrossInterpreterData *data) +_queueobj_from_xid(_PyXIData_t *data) { - int64_t qid = *(int64_t *)_PyCrossInterpreterData_DATA(data); + int64_t qid = *(int64_t *)_PyXIData_DATA(data); PyObject *qidobj = PyLong_FromLongLong(qid); if (qidobj == NULL) { return NULL; @@ -1184,8 +1369,7 @@ _queueobj_from_xid(_PyCrossInterpreterData *data) } static int -_queueobj_shared(PyThreadState *tstate, PyObject *queueobj, - _PyCrossInterpreterData *data) +_queueobj_shared(PyThreadState *tstate, PyObject *queueobj, _PyXIData_t *data) { PyObject *qidobj = PyObject_GetAttrString(queueobj, "_id"); if (qidobj == NULL) { @@ -1205,9 +1389,8 @@ _queueobj_shared(PyThreadState *tstate, PyObject *queueobj, if (raw == NULL) { return -1; } - _PyCrossInterpreterData_Init(data, tstate->interp, raw, NULL, - _queueobj_from_xid); - _PyCrossInterpreterData_SET_FREE(data, _queueid_xid_free); + _PyXIData_Init(data, tstate->interp, raw, NULL, _queueobj_from_xid); + _PyXIData_SET_FREE(data, _queueid_xid_free); return 0; } @@ -1218,6 +1401,7 @@ _queueobj_shared(PyThreadState *tstate, PyObject *queueobj, the data that we need to share between interpreters, so it cannot hold PyObject values. */ static struct globals { + PyMutex mutex; int module_count; _queues queues; } _globals = {0}; @@ -1225,32 +1409,36 @@ static struct globals { static int _globals_init(void) { - // XXX This isn't thread-safe. + PyMutex_Lock(&_globals.mutex); + assert(_globals.module_count >= 0); _globals.module_count++; - if (_globals.module_count > 1) { - // Already initialized. - return 0; - } - - assert(_globals.queues.mutex == NULL); - PyThread_type_lock mutex = PyThread_allocate_lock(); - if (mutex == NULL) { - return ERR_QUEUES_ALLOC; + if (_globals.module_count == 1) { + // Called for the first time. + PyThread_type_lock mutex = PyThread_allocate_lock(); + if (mutex == NULL) { + _globals.module_count--; + PyMutex_Unlock(&_globals.mutex); + return ERR_QUEUES_ALLOC; + } + _queues_init(&_globals.queues, mutex); } - _queues_init(&_globals.queues, mutex); + PyMutex_Unlock(&_globals.mutex); return 0; } static void _globals_fini(void) { - // XXX This isn't thread-safe. + PyMutex_Lock(&_globals.mutex); + assert(_globals.module_count > 0); _globals.module_count--; - if (_globals.module_count > 0) { - return; + if (_globals.module_count == 0) { + PyThread_type_lock mutex; + _queues_fini(&_globals.queues, &mutex); + assert(mutex != NULL); + PyThread_free_lock(mutex); } - - _queues_fini(&_globals.queues); + PyMutex_Unlock(&_globals.mutex); } static _queues * @@ -1289,15 +1477,22 @@ qidarg_converter(PyObject *arg, void *ptr) static PyObject * queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"maxsize", "fmt", NULL}; + static char *kwlist[] = {"maxsize", "fmt", "unboundop", NULL}; Py_ssize_t maxsize; int fmt; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "ni:create", kwlist, - &maxsize, &fmt)) { + int unboundop; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "nii:create", kwlist, + &maxsize, &fmt, &unboundop)) + { + return NULL; + } + if (!check_unbound(unboundop)) { + PyErr_Format(PyExc_ValueError, + "unsupported unboundop %d", unboundop); return NULL; } - int64_t qid = queue_create(&_globals.queues, maxsize, fmt); + int64_t qid = queue_create(&_globals.queues, maxsize, fmt, unboundop); if (qid < 0) { (void)handle_queue_error((int)qid, self, qid); return NULL; @@ -1319,16 +1514,19 @@ queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(queuesmod_create_doc, -"create() -> qid\n\ +"create(maxsize, fmt, unboundop) -> qid\n\ \n\ Create a new cross-interpreter queue and return its unique generated ID.\n\ -It is a new reference as though bind() had been called on the queue."); +It is a new reference as though bind() had been called on the queue.\n\ +\n\ +The caller is responsible for calling destroy() for the new queue\n\ +before the runtime is finalized."); static PyObject * queuesmod_destroy(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:destroy", kwlist, qidarg_converter, &qidarg)) { return NULL; @@ -1352,9 +1550,9 @@ static PyObject * queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) { int64_t count = 0; - struct queue_id_and_fmt *qids = _queues_list_all(&_globals.queues, &count); + struct queue_id_and_info *qids = _queues_list_all(&_globals.queues, &count); if (qids == NULL) { - if (count == 0) { + if (!PyErr_Occurred() && count == 0) { return PyList_New(0); } return NULL; @@ -1363,9 +1561,10 @@ queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) if (ids == NULL) { goto finally; } - struct queue_id_and_fmt *cur = qids; + struct queue_id_and_info *cur = qids; for (int64_t i=0; i < count; cur++, i++) { - PyObject *item = Py_BuildValue("Li", cur->id, cur->fmt); + PyObject *item = Py_BuildValue("Lii", cur->id, cur->fmt, + cur->unboundop); if (item == NULL) { Py_SETREF(ids, NULL); break; @@ -1379,25 +1578,35 @@ queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(queuesmod_list_all_doc, -"list_all() -> [qid]\n\ +"list_all() -> [(qid, fmt)]\n\ \n\ -Return the list of IDs for all queues."); +Return the list of IDs for all queues.\n\ +Each corresponding default format is also included."); static PyObject * queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"qid", "obj", "fmt", NULL}; - qidarg_converter_data qidarg; + static char *kwlist[] = {"qid", "obj", "fmt", "unboundop", NULL}; + qidarg_converter_data qidarg = {0}; PyObject *obj; int fmt; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&Oi:put", kwlist, - qidarg_converter, &qidarg, &obj, &fmt)) { + int unboundop; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&Oii:put", kwlist, + qidarg_converter, &qidarg, &obj, &fmt, + &unboundop)) + { return NULL; } int64_t qid = qidarg.id; + if (!check_unbound(unboundop)) { + PyErr_Format(PyExc_ValueError, + "unsupported unboundop %d", unboundop); + return NULL; + } /* Queue up the object. */ - int err = queue_put(&_globals.queues, qid, obj, fmt); + int err = queue_put(&_globals.queues, qid, obj, fmt, unboundop); + // This is the only place that raises QueueFull. if (handle_queue_error(err, self, qid)) { return NULL; } @@ -1406,51 +1615,51 @@ queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(queuesmod_put_doc, -"put(qid, obj, sharedonly=False)\n\ +"put(qid, obj, fmt)\n\ \n\ Add the object's data to the queue."); static PyObject * queuesmod_get(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"qid", "default", NULL}; - qidarg_converter_data qidarg; - PyObject *dflt = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O:get", kwlist, - qidarg_converter, &qidarg, &dflt)) { + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg = {0}; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:get", kwlist, + qidarg_converter, &qidarg)) { return NULL; } int64_t qid = qidarg.id; PyObject *obj = NULL; int fmt = 0; - int err = queue_get(&_globals.queues, qid, &obj, &fmt); - if (err == ERR_QUEUE_EMPTY && dflt != NULL) { - assert(obj == NULL); - obj = Py_NewRef(dflt); - } - else if (handle_queue_error(err, self, qid)) { + int unboundop = 0; + int err = queue_get(&_globals.queues, qid, &obj, &fmt, &unboundop); + // This is the only place that raises QueueEmpty. + if (handle_queue_error(err, self, qid)) { return NULL; } - PyObject *res = Py_BuildValue("Oi", obj, fmt); + if (obj == NULL) { + return Py_BuildValue("Oii", Py_None, fmt, unboundop); + } + PyObject *res = Py_BuildValue("OiO", obj, fmt, Py_None); Py_DECREF(obj); return res; } PyDoc_STRVAR(queuesmod_get_doc, -"get(qid, [default]) -> obj\n\ +"get(qid) -> (obj, fmt)\n\ \n\ Return a new object from the data at the front of the queue.\n\ +The object's format is also returned.\n\ \n\ -If there is nothing to receive then raise QueueEmpty, unless\n\ -a default value is provided. In that case return it."); +If there is nothing to receive then raise QueueEmpty."); static PyObject * queuesmod_bind(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:bind", kwlist, qidarg_converter, &qidarg)) { return NULL; @@ -1480,7 +1689,7 @@ queuesmod_release(PyObject *self, PyObject *args, PyObject *kwds) { // Note that only the current interpreter is affected. static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:release", kwlist, qidarg_converter, &qidarg)) { @@ -1491,7 +1700,10 @@ queuesmod_release(PyObject *self, PyObject *args, PyObject *kwds) // XXX Check module state if bound already. // XXX Update module state. - _queues_decref(&_globals.queues, qid); + int err = _queues_decref(&_globals.queues, qid); + if (handle_queue_error(err, self, qid)) { + return NULL; + } Py_RETURN_NONE; } @@ -1506,7 +1718,7 @@ static PyObject * queuesmod_get_maxsize(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:get_maxsize", kwlist, qidarg_converter, &qidarg)) { @@ -1528,12 +1740,12 @@ PyDoc_STRVAR(queuesmod_get_maxsize_doc, Return the maximum number of items in the queue."); static PyObject * -queuesmod_get_default_fmt(PyObject *self, PyObject *args, PyObject *kwds) +queuesmod_get_queue_defaults(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O&:get_default_fmt", kwlist, + "O&:get_queue_defaults", kwlist, qidarg_converter, &qidarg)) { return NULL; } @@ -1544,21 +1756,24 @@ queuesmod_get_default_fmt(PyObject *self, PyObject *args, PyObject *kwds) if (handle_queue_error(err, self, qid)) { return NULL; } - int fmt = queue->fmt; + int fmt = queue->defaults.fmt; + int unboundop = queue->defaults.unboundop; _queue_unmark_waiter(queue, _globals.queues.mutex); - return PyLong_FromLong(fmt); + + PyObject *defaults = Py_BuildValue("ii", fmt, unboundop); + return defaults; } -PyDoc_STRVAR(queuesmod_get_default_fmt_doc, -"get_default_fmt(qid)\n\ +PyDoc_STRVAR(queuesmod_get_queue_defaults_doc, +"get_queue_defaults(qid)\n\ \n\ -Return the default format to use for the queue."); +Return the queue's default values, set when it was created."); static PyObject * queuesmod_is_full(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:is_full", kwlist, qidarg_converter, &qidarg)) { @@ -1586,7 +1801,7 @@ static PyObject * queuesmod_get_count(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg; + qidarg_converter_data qidarg = {0}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:get_count", kwlist, qidarg_converter, &qidarg)) { @@ -1609,22 +1824,39 @@ PyDoc_STRVAR(queuesmod_get_count_doc, Return the number of items in the queue."); static PyObject * -queuesmod__register_queue_type(PyObject *self, PyObject *args, PyObject *kwds) +queuesmod__register_heap_types(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"queuetype", NULL}; + static char *kwlist[] = {"queuetype", "emptyerror", "fullerror", NULL}; PyObject *queuetype; + PyObject *emptyerror; + PyObject *fullerror; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:_register_queue_type", kwlist, - &queuetype)) { + "OOO:_register_heap_types", kwlist, + &queuetype, &emptyerror, &fullerror)) { return NULL; } if (!PyType_Check(queuetype)) { - PyErr_SetString(PyExc_TypeError, "expected a type for 'queuetype'"); + PyErr_SetString(PyExc_TypeError, + "expected a type for 'queuetype'"); return NULL; } - PyTypeObject *cls_queue = (PyTypeObject *)queuetype; + if (!PyExceptionClass_Check(emptyerror)) { + PyErr_SetString(PyExc_TypeError, + "expected an exception type for 'emptyerror'"); + return NULL; + } + if (!PyExceptionClass_Check(fullerror)) { + PyErr_SetString(PyExc_TypeError, + "expected an exception type for 'fullerror'"); + return NULL; + } + + module_state *state = get_module_state(self); - if (set_external_queue_type(self, cls_queue) < 0) { + if (set_external_queue_type(state, (PyTypeObject *)queuetype) < 0) { + return NULL; + } + if (set_external_exc_types(state, emptyerror, fullerror) < 0) { return NULL; } @@ -1638,23 +1870,23 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, queuesmod_destroy_doc}, {"list_all", queuesmod_list_all, METH_NOARGS, queuesmod_list_all_doc}, - {"put", _PyCFunction_CAST(queuesmod_put), + {"put", _PyCFunction_CAST(queuesmod_put), METH_VARARGS | METH_KEYWORDS, queuesmod_put_doc}, - {"get", _PyCFunction_CAST(queuesmod_get), + {"get", _PyCFunction_CAST(queuesmod_get), METH_VARARGS | METH_KEYWORDS, queuesmod_get_doc}, - {"bind", _PyCFunction_CAST(queuesmod_bind), + {"bind", _PyCFunction_CAST(queuesmod_bind), METH_VARARGS | METH_KEYWORDS, queuesmod_bind_doc}, {"release", _PyCFunction_CAST(queuesmod_release), METH_VARARGS | METH_KEYWORDS, queuesmod_release_doc}, {"get_maxsize", _PyCFunction_CAST(queuesmod_get_maxsize), METH_VARARGS | METH_KEYWORDS, queuesmod_get_maxsize_doc}, - {"get_default_fmt", _PyCFunction_CAST(queuesmod_get_default_fmt), - METH_VARARGS | METH_KEYWORDS, queuesmod_get_default_fmt_doc}, + {"get_queue_defaults", _PyCFunction_CAST(queuesmod_get_queue_defaults), + METH_VARARGS | METH_KEYWORDS, queuesmod_get_queue_defaults_doc}, {"is_full", _PyCFunction_CAST(queuesmod_is_full), METH_VARARGS | METH_KEYWORDS, queuesmod_is_full_doc}, {"get_count", _PyCFunction_CAST(queuesmod_get_count), METH_VARARGS | METH_KEYWORDS, queuesmod_get_count_doc}, - {"_register_queue_type", _PyCFunction_CAST(queuesmod__register_queue_type), + {"_register_heap_types", _PyCFunction_CAST(queuesmod__register_heap_types), METH_VARARGS | METH_KEYWORDS, NULL}, {NULL, NULL} /* sentinel */ @@ -1670,7 +1902,8 @@ The 'interpreters' module provides a more convenient interface."); static int module_exec(PyObject *mod) { - if (_globals_init() != 0) { + int err = _globals_init(); + if (handle_queue_error(err, mod, -1)) { return -1; } @@ -1693,6 +1926,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_interpreters_common.h b/Modules/_interpreters_common.h index 5661a26d8790d1..a6c639feea5d14 100644 --- a/Modules/_interpreters_common.h +++ b/Modules/_interpreters_common.h @@ -6,8 +6,70 @@ static int -ensure_xid_class(PyTypeObject *cls, crossinterpdatafunc getdata) +ensure_xid_class(PyTypeObject *cls, xidatafunc getdata) { - //assert(cls->tp_flags & Py_TPFLAGS_HEAPTYPE); - return _PyCrossInterpreterData_RegisterClass(cls, getdata); + PyInterpreterState *interp = PyInterpreterState_Get(); + _PyXIData_lookup_context_t ctx; + if (_PyXIData_GetLookupContext(interp, &ctx) < 0) { + return -1; + } + return _PyXIData_RegisterClass(&ctx, cls, getdata); } + +#ifdef REGISTERS_HEAP_TYPES +static int +clear_xid_class(PyTypeObject *cls) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + _PyXIData_lookup_context_t ctx; + if (_PyXIData_GetLookupContext(interp, &ctx) < 0) { + return -1; + } + return _PyXIData_UnregisterClass(&ctx, cls); +} +#endif + + +static inline int64_t +_get_interpid(_PyXIData_t *data) +{ + int64_t interpid; + if (data != NULL) { + interpid = _PyXIData_INTERPID(data); + assert(!PyErr_Occurred()); + } + else { + interpid = PyInterpreterState_GetID(PyInterpreterState_Get()); + } + return interpid; +} + + +/* unbound items ************************************************************/ + +#ifdef HAS_UNBOUND_ITEMS + +#define UNBOUND_REMOVE 1 +#define UNBOUND_ERROR 2 +#define UNBOUND_REPLACE 3 + +// It would also be possible to add UNBOUND_REPLACE where the replacement +// value is user-provided. There would be some limitations there, though. +// Another possibility would be something like UNBOUND_COPY, where the +// object is released but the underlying data is copied (with the "raw" +// allocator) and used when the item is popped off the queue. + +static int +check_unbound(int unboundop) +{ + switch (unboundop) { + case UNBOUND_REMOVE: + case UNBOUND_ERROR: + case UNBOUND_REPLACE: + return 1; + default: + return 0; + } +} + +#endif diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_interpretersmodule.c similarity index 59% rename from Modules/_xxsubinterpretersmodule.c rename to Modules/_interpretersmodule.c index 28c2f9c08bc0da..fcd0baf696f943 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -7,22 +7,22 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() -#include "pycore_crossinterp.h" // struct _xid +#include "pycore_crossinterp.h" // _PyXIData_t #include "pycore_interp.h" // _PyInterpreterState_IDIncref() #include "pycore_initconfig.h" // _PyErr_SetFromPyStatus() -#include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_modsupport.h" // _PyArg_BadArgument() +#include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree() #include "pycore_pyerrors.h" // _Py_excinfo +#include "pycore_pylifecycle.h" // _PyInterpreterConfig_AsDict() #include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() -#include "interpreteridobject.h" #include "marshal.h" // PyMarshal_ReadObjectFromString() #include "_interpreters_common.h" -#define MODULE_NAME _xxsubinterpreters +#define MODULE_NAME _interpreters #define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME) #define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME) @@ -35,96 +35,8 @@ _get_current_interp(void) return PyInterpreterState_Get(); } -static int64_t -pylong_to_interpid(PyObject *idobj) -{ - assert(PyLong_CheckExact(idobj)); - - if (_PyLong_IsNegative((PyLongObject *)idobj)) { - PyErr_Format(PyExc_ValueError, - "interpreter ID must be a non-negative int, got %R", - idobj); - return -1; - } - - int overflow; - long long id = PyLong_AsLongLongAndOverflow(idobj, &overflow); - if (id == -1) { - if (!overflow) { - assert(PyErr_Occurred()); - return -1; - } - assert(!PyErr_Occurred()); - // For now, we don't worry about if LLONG_MAX < INT64_MAX. - goto bad_id; - } -#if LLONG_MAX > INT64_MAX - if (id > INT64_MAX) { - goto bad_id; - } -#endif - return (int64_t)id; - -bad_id: - PyErr_Format(PyExc_RuntimeError, - "unrecognized interpreter ID %O", idobj); - return -1; -} - -static int64_t -convert_interpid_obj(PyObject *arg) -{ - int64_t id = -1; - if (_PyIndex_Check(arg)) { - PyObject *idobj = PyNumber_Long(arg); - if (idobj == NULL) { - return -1; - } - id = pylong_to_interpid(idobj); - Py_DECREF(idobj); - if (id < 0) { - return -1; - } - } - else { - PyErr_Format(PyExc_TypeError, - "interpreter ID must be an int, got %.100s", - Py_TYPE(arg)->tp_name); - return -1; - } - return id; -} - -static PyInterpreterState * -look_up_interp(PyObject *arg) -{ - int64_t id = convert_interpid_obj(arg); - if (id < 0) { - return NULL; - } - return _PyInterpreterState_LookUpID(id); -} - - -static PyObject * -interpid_to_pylong(int64_t id) -{ - assert(id < LLONG_MAX); - return PyLong_FromLongLong(id); -} +#define look_up_interp _PyInterpreterState_LookUpIDObject -static PyObject * -get_interpid_obj(PyInterpreterState *interp) -{ - if (_PyInterpreterState_IDInitref(interp) != 0) { - return NULL; - }; - int64_t id = PyInterpreterState_GetID(interp); - if (id < 0) { - return NULL; - } - return interpid_to_pylong(id); -} static PyObject * _get_current_module(void) @@ -143,6 +55,24 @@ _get_current_module(void) } +static int +is_running_main(PyInterpreterState *interp) +{ + if (_PyInterpreterState_IsRunningMain(interp)) { + return 1; + } + // Unlike with the general C-API, we can be confident that someone + // using this module for the main interpreter is doing so through + // the main program. Thus we can make this extra check. This benefits + // applications that embed Python but haven't been updated yet + // to call_PyInterpreterState_SetRunningMain(). + if (_Py_IsMainInterpreter(interp)) { + return 1; + } + return 0; +} + + /* Cross-interpreter Buffer Views *******************************************/ // XXX Release when the original interpreter is destroyed. @@ -154,18 +84,18 @@ typedef struct { } XIBufferViewObject; static PyObject * -xibufferview_from_xid(PyTypeObject *cls, _PyCrossInterpreterData *data) +xibufferview_from_xid(PyTypeObject *cls, _PyXIData_t *data) { - assert(_PyCrossInterpreterData_DATA(data) != NULL); - assert(_PyCrossInterpreterData_OBJ(data) == NULL); - assert(_PyCrossInterpreterData_INTERPID(data) >= 0); + assert(_PyXIData_DATA(data) != NULL); + assert(_PyXIData_OBJ(data) == NULL); + assert(_PyXIData_INTERPID(data) >= 0); XIBufferViewObject *self = PyObject_Malloc(sizeof(XIBufferViewObject)); if (self == NULL) { return NULL; } PyObject_Init((PyObject *)self, cls); - self->view = (Py_buffer *)_PyCrossInterpreterData_DATA(data); - self->interpid = _PyCrossInterpreterData_INTERPID(data); + self->view = (Py_buffer *)_PyXIData_DATA(data); + self->interpid = _PyXIData_INTERPID(data); return (PyObject *)self; } @@ -224,7 +154,7 @@ static PyType_Spec XIBufferViewType_spec = { static PyTypeObject * _get_current_xibufferview_type(void); static PyObject * -_memoryview_from_xid(_PyCrossInterpreterData *data) +_memoryview_from_xid(_PyXIData_t *data) { PyTypeObject *cls = _get_current_xibufferview_type(); if (cls == NULL) { @@ -238,8 +168,7 @@ _memoryview_from_xid(_PyCrossInterpreterData *data) } static int -_memoryview_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) +_memoryview_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { Py_buffer *view = PyMem_RawMalloc(sizeof(Py_buffer)); if (view == NULL) { @@ -249,8 +178,7 @@ _memoryview_shared(PyThreadState *tstate, PyObject *obj, PyMem_RawFree(view); return -1; } - _PyCrossInterpreterData_Init(data, tstate->interp, view, NULL, - _memoryview_from_xid); + _PyXIData_Init(data, tstate->interp, view, NULL, _memoryview_from_xid); return 0; } @@ -436,6 +364,66 @@ get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p) /* interpreter-specific code ************************************************/ +static int +init_named_config(PyInterpreterConfig *config, const char *name) +{ + if (name == NULL + || strcmp(name, "") == 0 + || strcmp(name, "default") == 0) + { + name = "isolated"; + } + + if (strcmp(name, "isolated") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; + } + else if (strcmp(name, "legacy") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; + } + else if (strcmp(name, "empty") == 0) { + *config = (PyInterpreterConfig){0}; + } + else { + PyErr_Format(PyExc_ValueError, + "unsupported config name '%s'", name); + return -1; + } + return 0; +} + +static int +config_from_object(PyObject *configobj, PyInterpreterConfig *config) +{ + if (configobj == NULL || configobj == Py_None) { + if (init_named_config(config, NULL) < 0) { + return -1; + } + } + else if (PyUnicode_Check(configobj)) { + const char *utf8name = PyUnicode_AsUTF8(configobj); + if (utf8name == NULL) { + return -1; + } + if (init_named_config(config, utf8name) < 0) { + return -1; + } + } + else { + PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); + if (dict == NULL) { + PyErr_Format(PyExc_TypeError, "bad config %R", configobj); + return -1; + } + int res = _PyInterpreterConfig_InitFromDict(config, dict); + Py_DECREF(dict); + if (res < 0) { + return -1; + } + } + return 0; +} + + static int _run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags) { @@ -471,7 +459,12 @@ _run_in_interpreter(PyInterpreterState *interp, // Prep and switch interpreters. if (_PyXI_Enter(&session, interp, shareables) < 0) { - assert(!PyErr_Occurred()); + if (PyErr_Occurred()) { + // If an error occured at this step, it means that interp + // was not prepared and switched. + return -1; + } + // Now, apply the error from another interpreter: PyObject *excinfo = _PyXI_ApplyError(session.error); if (excinfo != NULL) { *p_excinfo = excinfo; @@ -504,80 +497,195 @@ _run_in_interpreter(PyInterpreterState *interp, /* module level code ********************************************************/ +static long +get_whence(PyInterpreterState *interp) +{ + return _PyInterpreterState_GetWhence(interp); +} + + +static PyInterpreterState * +resolve_interp(PyObject *idobj, int restricted, int reqready, const char *op) +{ + PyInterpreterState *interp; + if (idobj == NULL) { + interp = PyInterpreterState_Get(); + } + else { + interp = look_up_interp(idobj); + if (interp == NULL) { + return NULL; + } + } + + if (reqready && !_PyInterpreterState_IsReady(interp)) { + if (idobj == NULL) { + PyErr_Format(PyExc_InterpreterError, + "cannot %s current interpreter (not ready)", op); + } + else { + PyErr_Format(PyExc_InterpreterError, + "cannot %s interpreter %R (not ready)", op, idobj); + } + return NULL; + } + + if (restricted && get_whence(interp) != _PyInterpreterState_WHENCE_STDLIB) { + if (idobj == NULL) { + PyErr_Format(PyExc_InterpreterError, + "cannot %s unrecognized current interpreter", op); + } + else { + PyErr_Format(PyExc_InterpreterError, + "cannot %s unrecognized interpreter %R", op, idobj); + } + return NULL; + } + + return interp; +} + + +static PyObject * +get_summary(PyInterpreterState *interp) +{ + PyObject *idobj = _PyInterpreterState_GetIDObject(interp); + if (idobj == NULL) { + return NULL; + } + PyObject *whenceobj = PyLong_FromLong( + get_whence(interp)); + if (whenceobj == NULL) { + Py_DECREF(idobj); + return NULL; + } + PyObject *res = PyTuple_Pack(2, idobj, whenceobj); + Py_DECREF(idobj); + Py_DECREF(whenceobj); + return res; +} + + +static PyObject * +interp_new_config(PyObject *self, PyObject *args, PyObject *kwds) +{ + const char *name = NULL; + if (!PyArg_ParseTuple(args, "|s:" MODULE_NAME_STR ".new_config", + &name)) + { + return NULL; + } + PyObject *overrides = kwds; + + PyInterpreterConfig config; + if (init_named_config(&config, name) < 0) { + return NULL; + } + + if (overrides != NULL && PyDict_GET_SIZE(overrides) > 0) { + if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) { + return NULL; + } + } + + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +PyDoc_STRVAR(new_config_doc, +"new_config(name='isolated', /, **overrides) -> type.SimpleNamespace\n\ +\n\ +Return a representation of a new PyInterpreterConfig.\n\ +\n\ +The name determines the initial values of the config. Supported named\n\ +configs are: default, isolated, legacy, and empty.\n\ +\n\ +Any keyword arguments are set on the corresponding config fields,\n\ +overriding the initial values."); + + static PyObject * interp_create(PyObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"config", "reqrefs", NULL}; + PyObject *configobj = NULL; + int reqrefs = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O$p:create", kwlist, + &configobj, &reqrefs)) { + return NULL; + } - static char *kwlist[] = {"isolated", NULL}; - int isolated = 1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|$i:create", kwlist, - &isolated)) { + PyInterpreterConfig config; + if (config_from_object(configobj, &config) < 0) { return NULL; } - // Create and initialize the new interpreter. - PyThreadState *save_tstate = PyThreadState_Get(); - assert(save_tstate != NULL); - const PyInterpreterConfig config = isolated - ? (PyInterpreterConfig)_PyInterpreterConfig_INIT - : (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; - - // XXX Possible GILState issues? - PyThreadState *tstate = NULL; - PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); - PyThreadState_Swap(save_tstate); - if (PyStatus_Exception(status)) { - /* Since no new thread state was created, there is no exception to - propagate; raise a fresh one after swapping in the old thread - state. */ - _PyErr_SetFromPyStatus(status); + long whence = _PyInterpreterState_WHENCE_STDLIB; + PyInterpreterState *interp = \ + _PyXI_NewInterpreter(&config, &whence, NULL, NULL); + if (interp == NULL) { + // XXX Move the chained exception to interpreters.create()? PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(PyExc_RuntimeError, "interpreter creation failed"); + assert(exc != NULL); + PyErr_SetString(PyExc_InterpreterError, "interpreter creation failed"); _PyErr_ChainExceptions1(exc); return NULL; } - assert(tstate != NULL); + assert(_PyInterpreterState_IsReady(interp)); - PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); - PyObject *idobj = get_interpid_obj(interp); + PyObject *idobj = _PyInterpreterState_GetIDObject(interp); if (idobj == NULL) { - // XXX Possible GILState issues? - save_tstate = PyThreadState_Swap(tstate); - Py_EndInterpreter(tstate); - PyThreadState_Swap(save_tstate); + _PyXI_EndInterpreter(interp, NULL, NULL); return NULL; } - PyThreadState_Swap(tstate); - PyThreadState_Clear(tstate); - PyThreadState_Swap(save_tstate); - PyThreadState_Delete(tstate); + if (reqrefs) { + // Decref to 0 will destroy the interpreter. + _PyInterpreterState_RequireIDRef(interp, 1); + } - _PyInterpreterState_RequireIDRef(interp, 1); return idobj; } + PyDoc_STRVAR(create_doc, -"create() -> ID\n\ +"create([config], *, reqrefs=False) -> ID\n\ \n\ Create a new interpreter and return a unique generated ID.\n\ \n\ -The caller is responsible for destroying the interpreter before exiting."); +The caller is responsible for destroying the interpreter before exiting,\n\ +typically by using _interpreters.destroy(). This can be managed \n\ +automatically by passing \"reqrefs=True\" and then using _incref() and\n\ +_decref() appropriately.\n\ +\n\ +\"config\" must be a valid interpreter config or the name of a\n\ +predefined config (\"isolated\" or \"legacy\"). The default\n\ +is \"isolated\"."); static PyObject * interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", NULL}; + static char *kwlist[] = {"id", "restrict", NULL}; PyObject *id; + int restricted = 0; // XXX Use "L" for id? if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:destroy", kwlist, &id)) { + "O|$p:destroy", kwlist, &id, &restricted)) + { return NULL; } // Look up the interpreter. - PyInterpreterState *interp = look_up_interp(id); + int reqready = 0; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "destroy"); if (interp == NULL) { return NULL; } @@ -588,7 +696,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } if (interp == current) { - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_InterpreterError, "cannot destroy the current interpreter"); return NULL; } @@ -596,57 +704,60 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) // Ensure the interpreter isn't running. /* XXX We *could* support destroying a running interpreter but aren't going to worry about it for now. */ - if (_PyInterpreterState_IsRunningMain(interp)) { - PyErr_Format(PyExc_RuntimeError, "interpreter running"); + if (is_running_main(interp)) { + PyErr_Format(PyExc_InterpreterError, "interpreter running"); return NULL; } // Destroy the interpreter. - PyThreadState *tstate = PyThreadState_New(interp); - _PyThreadState_SetWhence(tstate, _PyThreadState_WHENCE_INTERP); - // XXX Possible GILState issues? - PyThreadState *save_tstate = PyThreadState_Swap(tstate); - Py_EndInterpreter(tstate); - PyThreadState_Swap(save_tstate); + _PyXI_EndInterpreter(interp, NULL, NULL); Py_RETURN_NONE; } PyDoc_STRVAR(destroy_doc, -"destroy(id)\n\ +"destroy(id, *, restrict=False)\n\ \n\ Destroy the identified interpreter.\n\ \n\ -Attempting to destroy the current interpreter results in a RuntimeError.\n\ +Attempting to destroy the current interpreter raises InterpreterError.\n\ So does an unrecognized ID."); static PyObject * -interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) +interp_list_all(PyObject *self, PyObject *args, PyObject *kwargs) { - PyObject *ids, *id; - PyInterpreterState *interp; + static char *kwlist[] = {"require_ready", NULL}; + int reqready = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "|$p:" MODULE_NAME_STR ".list_all", + kwlist, &reqready)) + { + return NULL; + } - ids = PyList_New(0); + PyObject *ids = PyList_New(0); if (ids == NULL) { return NULL; } - interp = PyInterpreterState_Head(); + PyInterpreterState *interp = PyInterpreterState_Head(); while (interp != NULL) { - id = get_interpid_obj(interp); - if (id == NULL) { - Py_DECREF(ids); - return NULL; - } - // insert at front of list - int res = PyList_Insert(ids, 0, id); - Py_DECREF(id); - if (res < 0) { - Py_DECREF(ids); - return NULL; - } + if (!reqready || _PyInterpreterState_IsReady(interp)) { + PyObject *item = get_summary(interp); + if (item == NULL) { + Py_DECREF(ids); + return NULL; + } + // insert at front of list + int res = PyList_Insert(ids, 0, item); + Py_DECREF(item); + if (res < 0) { + Py_DECREF(ids); + return NULL; + } + } interp = PyInterpreterState_Next(interp); } @@ -654,7 +765,7 @@ interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(list_all_doc, -"list_all() -> [ID]\n\ +"list_all() -> [(ID, whence)]\n\ \n\ Return a list containing the ID of every existing interpreter."); @@ -666,11 +777,12 @@ interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored)) if (interp == NULL) { return NULL; } - return get_interpid_obj(interp); + assert(_PyInterpreterState_IsReady(interp)); + return get_summary(interp); } PyDoc_STRVAR(get_current_doc, -"get_current() -> ID\n\ +"get_current() -> (ID, whence)\n\ \n\ Return the ID of current interpreter."); @@ -678,28 +790,34 @@ Return the ID of current interpreter."); static PyObject * interp_get_main(PyObject *self, PyObject *Py_UNUSED(ignored)) { - // Currently, 0 is always the main interpreter. - int64_t id = 0; - return PyLong_FromLongLong(id); + PyInterpreterState *interp = _PyInterpreterState_Main(); + assert(_PyInterpreterState_IsReady(interp)); + return get_summary(interp); } PyDoc_STRVAR(get_main_doc, -"get_main() -> ID\n\ +"get_main() -> (ID, whence)\n\ \n\ Return the ID of main interpreter."); + static PyObject * -interp_set___main___attrs(PyObject *self, PyObject *args) +interp_set___main___attrs(PyObject *self, PyObject *args, PyObject *kwargs) { + static char *kwlist[] = {"id", "updates", "restrict", NULL}; PyObject *id, *updates; - if (!PyArg_ParseTuple(args, "OO:" MODULE_NAME_STR ".set___main___attrs", - &id, &updates)) + int restricted = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "OO|$p:" MODULE_NAME_STR ".set___main___attrs", + kwlist, &id, &updates, &restricted)) { return NULL; } // Look up the interpreter. - PyInterpreterState *interp = PyInterpreterID_LookUp(id); + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "update __main__ for"); if (interp == NULL) { return NULL; } @@ -738,10 +856,11 @@ interp_set___main___attrs(PyObject *self, PyObject *args) } PyDoc_STRVAR(set___main___attrs_doc, -"set___main___attrs(id, ns)\n\ +"set___main___attrs(id, ns, *, restrict=False)\n\ \n\ Bind the given attributes in the interpreter's __main__ module."); + static PyUnicodeObject * convert_script_arg(PyObject *arg, const char *fname, const char *displayname, const char *expected) @@ -819,13 +938,11 @@ convert_code_arg(PyObject *arg, const char *fname, const char *displayname, } static int -_interp_exec(PyObject *self, - PyObject *id_arg, PyObject *code_arg, PyObject *shared_arg, - PyObject **p_excinfo) +_interp_exec(PyObject *self, PyInterpreterState *interp, + PyObject *code_arg, PyObject *shared_arg, PyObject **p_excinfo) { - // Look up the interpreter. - PyInterpreterState *interp = look_up_interp(id_arg); - if (interp == NULL) { + if (shared_arg != NULL && !PyDict_CheckExact(shared_arg)) { + PyErr_SetString(PyExc_TypeError, "expected 'shared' to be a dict"); return -1; } @@ -853,12 +970,21 @@ _interp_exec(PyObject *self, static PyObject * interp_exec(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", "code", "shared", NULL}; + static char *kwlist[] = {"id", "code", "shared", "restrict", NULL}; PyObject *id, *code; PyObject *shared = NULL; + int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O:" MODULE_NAME_STR ".exec", kwlist, - &id, &code, &shared)) { + "OO|O$p:" MODULE_NAME_STR ".exec", kwlist, + &id, &code, &shared, &restricted)) + { + return NULL; + } + + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "exec code for"); + if (interp == NULL) { return NULL; } @@ -876,7 +1002,7 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) } PyObject *excinfo = NULL; - int res = _interp_exec(self, id, code, shared, &excinfo); + int res = _interp_exec(self, interp, code, shared, &excinfo); Py_DECREF(code); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); @@ -886,7 +1012,7 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(exec_doc, -"exec(id, code, shared=None)\n\ +"exec(id, code, shared=None, *, restrict=False)\n\ \n\ Execute the provided code in the identified interpreter.\n\ This is equivalent to running the builtin exec() under the target\n\ @@ -905,13 +1031,24 @@ is ignored, including its __globals__ dict."); static PyObject * interp_call(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", "callable", "args", "kwargs", NULL}; + static char *kwlist[] = {"id", "callable", "args", "kwargs", + "restrict", NULL}; PyObject *id, *callable; PyObject *args_obj = NULL; PyObject *kwargs_obj = NULL; + int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|OO:" MODULE_NAME_STR ".call", kwlist, - &id, &callable, &args_obj, &kwargs_obj)) { + "OO|OO$p:" MODULE_NAME_STR ".call", kwlist, + &id, &callable, &args_obj, &kwargs_obj, + &restricted)) + { + return NULL; + } + + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "make a call in"); + if (interp == NULL) { return NULL; } @@ -931,7 +1068,7 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) } PyObject *excinfo = NULL; - int res = _interp_exec(self, id, code, NULL, &excinfo); + int res = _interp_exec(self, interp, code, NULL, &excinfo); Py_DECREF(code); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); @@ -941,7 +1078,7 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(call_doc, -"call(id, callable, args=None, kwargs=None)\n\ +"call(id, callable, args=None, kwargs=None, *, restrict=False)\n\ \n\ Call the provided object in the identified interpreter.\n\ Pass the given args and kwargs, if possible.\n\ @@ -955,12 +1092,21 @@ is ignored, including its __globals__ dict."); static PyObject * interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", "script", "shared", NULL}; + static char *kwlist[] = {"id", "script", "shared", "restrict", NULL}; PyObject *id, *script; PyObject *shared = NULL; + int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OU|O:" MODULE_NAME_STR ".run_string", kwlist, - &id, &script, &shared)) { + "OU|O$p:" MODULE_NAME_STR ".run_string", + kwlist, &id, &script, &shared, &restricted)) + { + return NULL; + } + + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "run a string in"); + if (interp == NULL) { return NULL; } @@ -971,7 +1117,7 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) } PyObject *excinfo = NULL; - int res = _interp_exec(self, id, script, shared, &excinfo); + int res = _interp_exec(self, interp, script, shared, &excinfo); Py_DECREF(script); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); @@ -981,7 +1127,7 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(run_string_doc, -"run_string(id, script, shared=None)\n\ +"run_string(id, script, shared=None, *, restrict=False)\n\ \n\ Execute the provided string in the identified interpreter.\n\ \n\ @@ -990,12 +1136,21 @@ Execute the provided string in the identified interpreter.\n\ static PyObject * interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", "func", "shared", NULL}; + static char *kwlist[] = {"id", "func", "shared", "restrict", NULL}; PyObject *id, *func; PyObject *shared = NULL; + int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O:" MODULE_NAME_STR ".run_func", kwlist, - &id, &func, &shared)) { + "OO|O$p:" MODULE_NAME_STR ".run_func", + kwlist, &id, &func, &shared, &restricted)) + { + return NULL; + } + + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "run a function in"); + if (interp == NULL) { return NULL; } @@ -1007,7 +1162,7 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) } PyObject *excinfo = NULL; - int res = _interp_exec(self, id, (PyObject *)code, shared, &excinfo); + int res = _interp_exec(self, interp, (PyObject *)code, shared, &excinfo); Py_DECREF(code); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); @@ -1017,7 +1172,7 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(run_func_doc, -"run_func(id, func, shared=None)\n\ +"run_func(id, func, shared=None, *, restrict=False)\n\ \n\ Execute the body of the provided function in the identified interpreter.\n\ Code objects are also supported. In both cases, closures and args\n\ @@ -1036,7 +1191,13 @@ object_is_shareable(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - if (_PyObject_CheckCrossInterpreterData(obj) == 0) { + PyInterpreterState *interp = PyInterpreterState_Get(); + _PyXIData_lookup_context_t ctx; + if (_PyXIData_GetLookupContext(interp, &ctx) < 0) { + return NULL; + } + + if (_PyObject_CheckXIData(&ctx, obj) == 0) { Py_RETURN_TRUE; } PyErr_Clear(); @@ -1053,36 +1214,86 @@ False otherwise."); static PyObject * interp_is_running(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", NULL}; + static char *kwlist[] = {"id", "restrict", NULL}; PyObject *id; + int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:is_running", kwlist, &id)) { + "O|$p:is_running", kwlist, + &id, &restricted)) + { return NULL; } - PyInterpreterState *interp = look_up_interp(id); + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "check if running for"); if (interp == NULL) { return NULL; } - if (_PyInterpreterState_IsRunningMain(interp)) { + + if (is_running_main(interp)) { Py_RETURN_TRUE; } Py_RETURN_FALSE; } PyDoc_STRVAR(is_running_doc, -"is_running(id) -> bool\n\ +"is_running(id, *, restrict=False) -> bool\n\ \n\ Return whether or not the identified interpreter is running."); static PyObject * -interp_incref(PyObject *self, PyObject *args, PyObject *kwds) +interp_get_config(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "restrict", NULL}; + PyObject *idobj = NULL; + int restricted = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O|$p:get_config", kwlist, + &idobj, &restricted)) + { + return NULL; + } + if (idobj == Py_None) { + idobj = NULL; + } + + int reqready = 0; + PyInterpreterState *interp = \ + resolve_interp(idobj, restricted, reqready, "get the config of"); + if (interp == NULL) { + return NULL; + } + + PyInterpreterConfig config; + if (_PyInterpreterConfig_InitFromState(&config, interp) < 0) { + return NULL; + } + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +PyDoc_STRVAR(get_config_doc, +"get_config(id, *, restrict=False) -> types.SimpleNamespace\n\ +\n\ +Return a representation of the config used to initialize the interpreter."); + + +static PyObject * +interp_whence(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"id", NULL}; PyObject *id; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:_incref", kwlist, &id)) { + "O:whence", kwlist, &id)) + { return NULL; } @@ -1090,9 +1301,42 @@ interp_incref(PyObject *self, PyObject *args, PyObject *kwds) if (interp == NULL) { return NULL; } - if (_PyInterpreterState_IDInitref(interp) < 0) { + + long whence = get_whence(interp); + return PyLong_FromLong(whence); +} + +PyDoc_STRVAR(whence_doc, +"whence(id) -> int\n\ +\n\ +Return an identifier for where the interpreter was created."); + + +static PyObject * +interp_incref(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "implieslink", "restrict", NULL}; + PyObject *id; + int implieslink = 0; + int restricted = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O|$pp:incref", kwlist, + &id, &implieslink, &restricted)) + { + return NULL; + } + + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "incref"); + if (interp == NULL) { return NULL; } + + if (implieslink) { + // Decref to 0 will destroy the interpreter. + _PyInterpreterState_RequireIDRef(interp, 1); + } _PyInterpreterState_IDIncref(interp); Py_RETURN_NONE; @@ -1102,30 +1346,106 @@ interp_incref(PyObject *self, PyObject *args, PyObject *kwds) static PyObject * interp_decref(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", NULL}; + static char *kwlist[] = {"id", "restrict", NULL}; PyObject *id; + int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:_incref", kwlist, &id)) { + "O|$p:decref", kwlist, &id, &restricted)) + { return NULL; } - PyInterpreterState *interp = look_up_interp(id); + int reqready = 1; + PyInterpreterState *interp = \ + resolve_interp(id, restricted, reqready, "decref"); if (interp == NULL) { return NULL; } + _PyInterpreterState_IDDecref(interp); Py_RETURN_NONE; } +static PyObject * +capture_exception(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"exc", NULL}; + PyObject *exc_arg = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "|O:capture_exception", kwlist, + &exc_arg)) + { + return NULL; + } + + PyObject *exc = exc_arg; + if (exc == NULL || exc == Py_None) { + exc = PyErr_GetRaisedException(); + if (exc == NULL) { + Py_RETURN_NONE; + } + } + else if (!PyExceptionInstance_Check(exc)) { + PyErr_Format(PyExc_TypeError, "expected exception, got %R", exc); + return NULL; + } + PyObject *captured = NULL; + + _PyXI_excinfo info = {0}; + if (_PyXI_InitExcInfo(&info, exc) < 0) { + goto finally; + } + captured = _PyXI_ExcInfoAsObject(&info); + if (captured == NULL) { + goto finally; + } + + PyObject *formatted = _PyXI_FormatExcInfo(&info); + if (formatted == NULL) { + Py_CLEAR(captured); + goto finally; + } + int res = PyObject_SetAttrString(captured, "formatted", formatted); + Py_DECREF(formatted); + if (res < 0) { + Py_CLEAR(captured); + goto finally; + } + +finally: + _PyXI_ClearExcInfo(&info); + if (exc != exc_arg) { + if (PyErr_Occurred()) { + PyErr_SetRaisedException(exc); + } + else { + _PyErr_ChainExceptions1(exc); + } + } + return captured; +} + +PyDoc_STRVAR(capture_exception_doc, +"capture_exception(exc=None) -> types.SimpleNamespace\n\ +\n\ +Return a snapshot of an exception. If \"exc\" is None\n\ +then the current exception, if any, is used (but not cleared).\n\ +\n\ +The returned snapshot is the same as what _interpreters.exec() returns."); + + static PyMethodDef module_functions[] = { + {"new_config", _PyCFunction_CAST(interp_new_config), + METH_VARARGS | METH_KEYWORDS, new_config_doc}, + {"create", _PyCFunction_CAST(interp_create), METH_VARARGS | METH_KEYWORDS, create_doc}, {"destroy", _PyCFunction_CAST(interp_destroy), METH_VARARGS | METH_KEYWORDS, destroy_doc}, - {"list_all", interp_list_all, - METH_NOARGS, list_all_doc}, + {"list_all", _PyCFunction_CAST(interp_list_all), + METH_VARARGS | METH_KEYWORDS, list_all_doc}, {"get_current", interp_get_current, METH_NOARGS, get_current_doc}, {"get_main", interp_get_main, @@ -1133,6 +1453,10 @@ static PyMethodDef module_functions[] = { {"is_running", _PyCFunction_CAST(interp_is_running), METH_VARARGS | METH_KEYWORDS, is_running_doc}, + {"get_config", _PyCFunction_CAST(interp_get_config), + METH_VARARGS | METH_KEYWORDS, get_config_doc}, + {"whence", _PyCFunction_CAST(interp_whence), + METH_VARARGS | METH_KEYWORDS, whence_doc}, {"exec", _PyCFunction_CAST(interp_exec), METH_VARARGS | METH_KEYWORDS, exec_doc}, {"call", _PyCFunction_CAST(interp_call), @@ -1143,15 +1467,19 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, run_func_doc}, {"set___main___attrs", _PyCFunction_CAST(interp_set___main___attrs), - METH_VARARGS, set___main___attrs_doc}, - {"is_shareable", _PyCFunction_CAST(object_is_shareable), - METH_VARARGS | METH_KEYWORDS, is_shareable_doc}, + METH_VARARGS | METH_KEYWORDS, set___main___attrs_doc}, - {"_incref", _PyCFunction_CAST(interp_incref), + {"incref", _PyCFunction_CAST(interp_incref), METH_VARARGS | METH_KEYWORDS, NULL}, - {"_decref", _PyCFunction_CAST(interp_decref), + {"decref", _PyCFunction_CAST(interp_decref), METH_VARARGS | METH_KEYWORDS, NULL}, + {"is_shareable", _PyCFunction_CAST(object_is_shareable), + METH_VARARGS | METH_KEYWORDS, is_shareable_doc}, + + {"capture_exception", _PyCFunction_CAST(capture_exception), + METH_VARARGS | METH_KEYWORDS, capture_exception_doc}, + {NULL, NULL} /* sentinel */ }; @@ -1168,6 +1496,25 @@ module_exec(PyObject *mod) PyInterpreterState *interp = PyInterpreterState_Get(); module_state *state = get_module_state(mod); + _PyXIData_lookup_context_t ctx; + if (_PyXIData_GetLookupContext(interp, &ctx) < 0) { + return -1; + } + +#define ADD_WHENCE(NAME) \ + if (PyModule_AddIntConstant(mod, "WHENCE_" #NAME, \ + _PyInterpreterState_WHENCE_##NAME) < 0) \ + { \ + goto error; \ + } + ADD_WHENCE(UNKNOWN) + ADD_WHENCE(RUNTIME) + ADD_WHENCE(LEGACY_CAPI) + ADD_WHENCE(CAPI) + ADD_WHENCE(XI) + ADD_WHENCE(STDLIB) +#undef ADD_WHENCE + // exceptions if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterError) < 0) { goto error; @@ -1175,9 +1522,7 @@ module_exec(PyObject *mod) if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterNotFoundError) < 0) { goto error; } - PyObject *PyExc_NotShareableError = \ - _PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError; - if (PyModule_AddType(mod, (PyTypeObject *)PyExc_NotShareableError) < 0) { + if (PyModule_AddType(mod, (PyTypeObject *)ctx.PyExc_NotShareableError) < 0) { goto error; } @@ -1194,6 +1539,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 173f5b55e5f732..6622f2cabb908b 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -10,7 +10,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_initconfig.h" // _PyStatus_OK() -#include "pycore_long.h" // _PyLong_Sign() +#include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -202,7 +202,7 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, const char *newline, int closefd, PyObject *opener) /*[clinic end generated code: output=aefafc4ce2b46dc0 input=cd034e7cdfbf4e78]*/ { - unsigned i; + size_t i; int creating = 0, reading = 0, writing = 0, appending = 0, updating = 0; int text = 0, binary = 0; @@ -346,7 +346,7 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, /* buffering */ if (buffering < 0) { - PyObject *res = PyObject_CallMethodNoArgs(raw, &_Py_ID(isatty)); + PyObject *res = PyObject_CallMethodNoArgs(raw, &_Py_ID(_isatty_open_only)); if (res == NULL) goto error; isatty = PyObject_IsTrue(res); @@ -544,10 +544,7 @@ PyNumber_AsOff_t(PyObject *item, PyObject *err) */ if (!err) { assert(PyLong_Check(value)); - /* Whether or not it is less than or equal to - zero is determined by the sign of ob_size - */ - if (_PyLong_Sign(value) < 0) + if (_PyLong_IsNegative((PyLongObject *)value)) result = PY_OFF_T_MIN; else result = PY_OFF_T_MAX; @@ -720,6 +717,7 @@ iomodule_exec(PyObject *m) static struct PyModuleDef_Slot iomodule_slots[] = { {Py_mod_exec, iomodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index b3450eeaf99401..bc5fff54a62b6d 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -8,7 +8,6 @@ */ #include "Python.h" -#include "pycore_bytesobject.h" // _PyBytes_Join() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _Py_FatalErrorFormat() @@ -1284,7 +1283,7 @@ _buffered_readline(buffered *self, Py_ssize_t limit) Py_CLEAR(res); goto end; } - Py_XSETREF(res, _PyBytes_Join((PyObject *)&_Py_SINGLETON(bytes_empty), chunks)); + Py_XSETREF(res, PyBytes_Join((PyObject *)&_Py_SINGLETON(bytes_empty), chunks)); end: LEAVE_BUFFERED(self) @@ -1737,7 +1736,7 @@ _bufferedreader_read_all(buffered *self) goto cleanup; } else { - tmp = _PyBytes_Join((PyObject *)&_Py_SINGLETON(bytes_empty), chunks); + tmp = PyBytes_Join((PyObject *)&_Py_SINGLETON(bytes_empty), chunks); res = tmp; goto cleanup; } @@ -2092,7 +2091,7 @@ _io_BufferedWriter_write_impl(buffered *self, Py_buffer *buffer) self->raw_pos = 0; } avail = Py_SAFE_DOWNCAST(self->buffer_size - self->pos, Py_off_t, Py_ssize_t); - if (buffer->len <= avail) { + if (buffer->len <= avail && buffer->len < self->buffer_size) { memcpy(self->buffer + self->pos, buffer->buf, buffer->len); if (!VALID_WRITE_BUFFER(self) || self->write_pos > self->pos) { self->write_pos = self->pos; @@ -2161,7 +2160,7 @@ _io_BufferedWriter_write_impl(buffered *self, Py_buffer *buffer) /* Then write buf itself. At this point the buffer has been emptied. */ remaining = buffer->len; written = 0; - while (remaining > self->buffer_size) { + while (remaining >= self->buffer_size) { Py_ssize_t n = _bufferedwriter_raw_write( self, (char *) buffer->buf + written, buffer->len - written); if (n == -1) { @@ -2531,8 +2530,8 @@ static PyMethodDef bufferedreader_methods[] = { _IO__BUFFERED_TRUNCATE_METHODDEF _IO__BUFFERED___SIZEOF___METHODDEF - {"__reduce__", _PyIOBase_cannot_pickle, METH_VARARGS}, - {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_VARARGS}, + {"__reduce__", _PyIOBase_cannot_pickle, METH_NOARGS}, + {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_O}, {NULL, NULL} }; @@ -2591,8 +2590,8 @@ static PyMethodDef bufferedwriter_methods[] = { _IO__BUFFERED_TELL_METHODDEF _IO__BUFFERED___SIZEOF___METHODDEF - {"__reduce__", _PyIOBase_cannot_pickle, METH_VARARGS}, - {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_VARARGS}, + {"__reduce__", _PyIOBase_cannot_pickle, METH_NOARGS}, + {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_O}, {NULL, NULL} }; @@ -2709,8 +2708,8 @@ static PyMethodDef bufferedrandom_methods[] = { _IO_BUFFEREDWRITER_WRITE_METHODDEF _IO__BUFFERED___SIZEOF___METHODDEF - {"__reduce__", _PyIOBase_cannot_pickle, METH_VARARGS}, - {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_VARARGS}, + {"__reduce__", _PyIOBase_cannot_pickle, METH_NOARGS}, + {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_O}, {NULL, NULL} }; diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 4a15c8e841f25f..16095333db6638 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -155,9 +155,6 @@ resize_buffer(bytesio *self, size_t size) alloc = size + 1; } - if (alloc > ((size_t)-1) / sizeof(char)) - goto overflow; - if (SHARED_BUF(self)) { if (unshare_buffer(self, alloc) < 0) return -1; @@ -591,7 +588,7 @@ _io_BytesIO_readinto_impl(bytesio *self, Py_buffer *buffer) /*[clinic input] _io.BytesIO.truncate - size: Py_ssize_t(accept={int, NoneType}, c_default="self->pos") = None + size: Py_ssize_t(accept={int, NoneType}, c_default="((bytesio *)self)->pos") = None / Truncate the file to at most size bytes. @@ -602,7 +599,7 @@ The current file position is unchanged. Returns the new size. static PyObject * _io_BytesIO_truncate_impl(bytesio *self, Py_ssize_t size) -/*[clinic end generated code: output=9ad17650c15fa09b input=423759dd42d2f7c1]*/ +/*[clinic end generated code: output=9ad17650c15fa09b input=dae4295e11c1bbb4]*/ { CHECK_CLOSED(self); CHECK_EXPORTS(self); diff --git a/Modules/_io/clinic/_iomodule.c.h b/Modules/_io/clinic/_iomodule.c.h index 112408a95df036..82932a23331ab6 100644 --- a/Modules/_io/clinic/_iomodule.c.h +++ b/Modules/_io/clinic/_iomodule.c.h @@ -175,7 +175,8 @@ _io_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw int closefd = 1; PyObject *opener = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 8, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 8, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -390,7 +391,8 @@ _io_open_code(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec PyObject *argsbuf[1]; PyObject *path; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -404,4 +406,4 @@ _io_open_code(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=5d60f4e778a600a4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ec1df2ff5265ab16 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index 64eddcd314a803..8ab8000fafee02 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -4,7 +4,7 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() @@ -151,7 +151,8 @@ _io__BufferedIOBase_read(PyObject *self, PyTypeObject *cls, PyObject *const *arg PyObject *argsbuf[1]; int size = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -205,7 +206,8 @@ _io__BufferedIOBase_read1(PyObject *self, PyTypeObject *cls, PyObject *const *ar PyObject *argsbuf[1]; int size = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -262,7 +264,8 @@ _io__BufferedIOBase_write(PyObject *self, PyTypeObject *cls, PyObject *const *ar PyObject *argsbuf[1]; PyObject *b; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -285,12 +288,12 @@ static PyObject * _io__Buffered___sizeof___impl(buffered *self); static PyObject * -_io__Buffered___sizeof__(buffered *self, PyObject *Py_UNUSED(ignored)) +_io__Buffered___sizeof__(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered___sizeof___impl(self); + return_value = _io__Buffered___sizeof___impl((buffered *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -316,20 +319,18 @@ static PyObject * _io__Buffered_simple_flush_impl(buffered *self); static PyObject * -_io__Buffered_simple_flush(buffered *self, PyObject *Py_UNUSED(ignored)) +_io__Buffered_simple_flush(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered_simple_flush_impl(self); + return_value = _io__Buffered_simple_flush_impl((buffered *)self); Py_END_CRITICAL_SECTION(); return return_value; } -#if defined(_io__Buffered_closed_HAS_DOCSTR) -# define _io__Buffered_closed_DOCSTR _io__Buffered_closed__doc__ -#else +#if !defined(_io__Buffered_closed_DOCSTR) # define _io__Buffered_closed_DOCSTR NULL #endif #if defined(_IO__BUFFERED_CLOSED_GETSETDEF) @@ -343,12 +344,12 @@ static PyObject * _io__Buffered_closed_get_impl(buffered *self); static PyObject * -_io__Buffered_closed_get(buffered *self, void *Py_UNUSED(context)) +_io__Buffered_closed_get(PyObject *self, void *Py_UNUSED(context)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered_closed_get_impl(self); + return_value = _io__Buffered_closed_get_impl((buffered *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -366,12 +367,12 @@ static PyObject * _io__Buffered_close_impl(buffered *self); static PyObject * -_io__Buffered_close(buffered *self, PyObject *Py_UNUSED(ignored)) +_io__Buffered_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered_close_impl(self); + return_value = _io__Buffered_close_impl((buffered *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -389,12 +390,12 @@ static PyObject * _io__Buffered_detach_impl(buffered *self); static PyObject * -_io__Buffered_detach(buffered *self, PyObject *Py_UNUSED(ignored)) +_io__Buffered_detach(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered_detach_impl(self); + return_value = _io__Buffered_detach_impl((buffered *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -412,12 +413,12 @@ static PyObject * _io__Buffered_seekable_impl(buffered *self); static PyObject * -_io__Buffered_seekable(buffered *self, PyObject *Py_UNUSED(ignored)) +_io__Buffered_seekable(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered_seekable_impl(self); + return_value = _io__Buffered_seekable_impl((buffered *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -435,12 +436,12 @@ static PyObject * _io__Buffered_readable_impl(buffered *self); static PyObject * -_io__Buffered_readable(buffered *self, PyObject *Py_UNUSED(ignored)) +_io__Buffered_readable(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered_readable_impl(self); + return_value = _io__Buffered_readable_impl((buffered *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -458,20 +459,18 @@ static PyObject * _io__Buffered_writable_impl(buffered *self); static PyObject * -_io__Buffered_writable(buffered *self, PyObject *Py_UNUSED(ignored)) +_io__Buffered_writable(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered_writable_impl(self); + return_value = _io__Buffered_writable_impl((buffered *)self); Py_END_CRITICAL_SECTION(); return return_value; } -#if defined(_io__Buffered_name_HAS_DOCSTR) -# define _io__Buffered_name_DOCSTR _io__Buffered_name__doc__ -#else +#if !defined(_io__Buffered_name_DOCSTR) # define _io__Buffered_name_DOCSTR NULL #endif #if defined(_IO__BUFFERED_NAME_GETSETDEF) @@ -485,20 +484,18 @@ static PyObject * _io__Buffered_name_get_impl(buffered *self); static PyObject * -_io__Buffered_name_get(buffered *self, void *Py_UNUSED(context)) +_io__Buffered_name_get(PyObject *self, void *Py_UNUSED(context)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered_name_get_impl(self); + return_value = _io__Buffered_name_get_impl((buffered *)self); Py_END_CRITICAL_SECTION(); return return_value; } -#if defined(_io__Buffered_mode_HAS_DOCSTR) -# define _io__Buffered_mode_DOCSTR _io__Buffered_mode__doc__ -#else +#if !defined(_io__Buffered_mode_DOCSTR) # define _io__Buffered_mode_DOCSTR NULL #endif #if defined(_IO__BUFFERED_MODE_GETSETDEF) @@ -512,12 +509,12 @@ static PyObject * _io__Buffered_mode_get_impl(buffered *self); static PyObject * -_io__Buffered_mode_get(buffered *self, void *Py_UNUSED(context)) +_io__Buffered_mode_get(PyObject *self, void *Py_UNUSED(context)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered_mode_get_impl(self); + return_value = _io__Buffered_mode_get_impl((buffered *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -535,12 +532,12 @@ static PyObject * _io__Buffered_fileno_impl(buffered *self); static PyObject * -_io__Buffered_fileno(buffered *self, PyObject *Py_UNUSED(ignored)) +_io__Buffered_fileno(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered_fileno_impl(self); + return_value = _io__Buffered_fileno_impl((buffered *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -558,12 +555,12 @@ static PyObject * _io__Buffered_isatty_impl(buffered *self); static PyObject * -_io__Buffered_isatty(buffered *self, PyObject *Py_UNUSED(ignored)) +_io__Buffered_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered_isatty_impl(self); + return_value = _io__Buffered_isatty_impl((buffered *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -581,12 +578,12 @@ static PyObject * _io__Buffered_flush_impl(buffered *self); static PyObject * -_io__Buffered_flush(buffered *self, PyObject *Py_UNUSED(ignored)) +_io__Buffered_flush(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered_flush_impl(self); + return_value = _io__Buffered_flush_impl((buffered *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -604,7 +601,7 @@ static PyObject * _io__Buffered_peek_impl(buffered *self, Py_ssize_t size); static PyObject * -_io__Buffered_peek(buffered *self, PyObject *const *args, Py_ssize_t nargs) +_io__Buffered_peek(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t size = 0; @@ -629,7 +626,7 @@ _io__Buffered_peek(buffered *self, PyObject *const *args, Py_ssize_t nargs) } skip_optional: Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered_peek_impl(self, size); + return_value = _io__Buffered_peek_impl((buffered *)self, size); Py_END_CRITICAL_SECTION(); exit: @@ -648,7 +645,7 @@ static PyObject * _io__Buffered_read_impl(buffered *self, Py_ssize_t n); static PyObject * -_io__Buffered_read(buffered *self, PyObject *const *args, Py_ssize_t nargs) +_io__Buffered_read(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t n = -1; @@ -664,7 +661,7 @@ _io__Buffered_read(buffered *self, PyObject *const *args, Py_ssize_t nargs) } skip_optional: Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered_read_impl(self, n); + return_value = _io__Buffered_read_impl((buffered *)self, n); Py_END_CRITICAL_SECTION(); exit: @@ -683,7 +680,7 @@ static PyObject * _io__Buffered_read1_impl(buffered *self, Py_ssize_t n); static PyObject * -_io__Buffered_read1(buffered *self, PyObject *const *args, Py_ssize_t nargs) +_io__Buffered_read1(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t n = -1; @@ -708,7 +705,7 @@ _io__Buffered_read1(buffered *self, PyObject *const *args, Py_ssize_t nargs) } skip_optional: Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered_read1_impl(self, n); + return_value = _io__Buffered_read1_impl((buffered *)self, n); Py_END_CRITICAL_SECTION(); exit: @@ -727,7 +724,7 @@ static PyObject * _io__Buffered_readinto_impl(buffered *self, Py_buffer *buffer); static PyObject * -_io__Buffered_readinto(buffered *self, PyObject *arg) +_io__Buffered_readinto(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; Py_buffer buffer = {NULL, NULL}; @@ -737,7 +734,7 @@ _io__Buffered_readinto(buffered *self, PyObject *arg) goto exit; } Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered_readinto_impl(self, &buffer); + return_value = _io__Buffered_readinto_impl((buffered *)self, &buffer); Py_END_CRITICAL_SECTION(); exit: @@ -761,7 +758,7 @@ static PyObject * _io__Buffered_readinto1_impl(buffered *self, Py_buffer *buffer); static PyObject * -_io__Buffered_readinto1(buffered *self, PyObject *arg) +_io__Buffered_readinto1(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; Py_buffer buffer = {NULL, NULL}; @@ -771,7 +768,7 @@ _io__Buffered_readinto1(buffered *self, PyObject *arg) goto exit; } Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered_readinto1_impl(self, &buffer); + return_value = _io__Buffered_readinto1_impl((buffered *)self, &buffer); Py_END_CRITICAL_SECTION(); exit: @@ -795,7 +792,7 @@ static PyObject * _io__Buffered_readline_impl(buffered *self, Py_ssize_t size); static PyObject * -_io__Buffered_readline(buffered *self, PyObject *const *args, Py_ssize_t nargs) +_io__Buffered_readline(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t size = -1; @@ -811,7 +808,7 @@ _io__Buffered_readline(buffered *self, PyObject *const *args, Py_ssize_t nargs) } skip_optional: Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered_readline_impl(self, size); + return_value = _io__Buffered_readline_impl((buffered *)self, size); Py_END_CRITICAL_SECTION(); exit: @@ -830,12 +827,12 @@ static PyObject * _io__Buffered_tell_impl(buffered *self); static PyObject * -_io__Buffered_tell(buffered *self, PyObject *Py_UNUSED(ignored)) +_io__Buffered_tell(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered_tell_impl(self); + return_value = _io__Buffered_tell_impl((buffered *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -853,7 +850,7 @@ static PyObject * _io__Buffered_seek_impl(buffered *self, PyObject *targetobj, int whence); static PyObject * -_io__Buffered_seek(buffered *self, PyObject *const *args, Py_ssize_t nargs) +_io__Buffered_seek(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *targetobj; @@ -872,7 +869,7 @@ _io__Buffered_seek(buffered *self, PyObject *const *args, Py_ssize_t nargs) } skip_optional: Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered_seek_impl(self, targetobj, whence); + return_value = _io__Buffered_seek_impl((buffered *)self, targetobj, whence); Py_END_CRITICAL_SECTION(); exit: @@ -891,7 +888,7 @@ static PyObject * _io__Buffered_truncate_impl(buffered *self, PyTypeObject *cls, PyObject *pos); static PyObject * -_io__Buffered_truncate(buffered *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_io__Buffered_truncate(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -910,7 +907,8 @@ _io__Buffered_truncate(buffered *self, PyTypeObject *cls, PyObject *const *args, PyObject *argsbuf[1]; PyObject *pos = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -920,7 +918,7 @@ _io__Buffered_truncate(buffered *self, PyTypeObject *cls, PyObject *const *args, pos = args[0]; skip_optional_posonly: Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io__Buffered_truncate_impl(self, cls, pos); + return_value = _io__Buffered_truncate_impl((buffered *)self, cls, pos); Py_END_CRITICAL_SECTION(); exit: @@ -973,7 +971,8 @@ _io_BufferedReader___init__(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *raw; Py_ssize_t buffer_size = DEFAULT_BUFFER_SIZE; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -1050,7 +1049,8 @@ _io_BufferedWriter___init__(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *raw; Py_ssize_t buffer_size = DEFAULT_BUFFER_SIZE; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -1089,7 +1089,7 @@ static PyObject * _io_BufferedWriter_write_impl(buffered *self, Py_buffer *buffer); static PyObject * -_io_BufferedWriter_write(buffered *self, PyObject *arg) +_io_BufferedWriter_write(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; Py_buffer buffer = {NULL, NULL}; @@ -1098,7 +1098,7 @@ _io_BufferedWriter_write(buffered *self, PyObject *arg) goto exit; } Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_BufferedWriter_write_impl(self, &buffer); + return_value = _io_BufferedWriter_write_impl((buffered *)self, &buffer); Py_END_CRITICAL_SECTION(); exit: @@ -1219,7 +1219,8 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *raw; Py_ssize_t buffer_size = DEFAULT_BUFFER_SIZE; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -1245,4 +1246,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=4249187a725a3b3e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f019d29701ba2556 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index 620e9e3b84ea19..5528df952c33fb 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -22,9 +22,9 @@ static PyObject * _io_BytesIO_readable_impl(bytesio *self); static PyObject * -_io_BytesIO_readable(bytesio *self, PyObject *Py_UNUSED(ignored)) +_io_BytesIO_readable(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_readable_impl(self); + return _io_BytesIO_readable_impl((bytesio *)self); } PyDoc_STRVAR(_io_BytesIO_writable__doc__, @@ -40,9 +40,9 @@ static PyObject * _io_BytesIO_writable_impl(bytesio *self); static PyObject * -_io_BytesIO_writable(bytesio *self, PyObject *Py_UNUSED(ignored)) +_io_BytesIO_writable(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_writable_impl(self); + return _io_BytesIO_writable_impl((bytesio *)self); } PyDoc_STRVAR(_io_BytesIO_seekable__doc__, @@ -58,9 +58,9 @@ static PyObject * _io_BytesIO_seekable_impl(bytesio *self); static PyObject * -_io_BytesIO_seekable(bytesio *self, PyObject *Py_UNUSED(ignored)) +_io_BytesIO_seekable(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_seekable_impl(self); + return _io_BytesIO_seekable_impl((bytesio *)self); } PyDoc_STRVAR(_io_BytesIO_flush__doc__, @@ -76,9 +76,9 @@ static PyObject * _io_BytesIO_flush_impl(bytesio *self); static PyObject * -_io_BytesIO_flush(bytesio *self, PyObject *Py_UNUSED(ignored)) +_io_BytesIO_flush(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_flush_impl(self); + return _io_BytesIO_flush_impl((bytesio *)self); } PyDoc_STRVAR(_io_BytesIO_getbuffer__doc__, @@ -94,13 +94,13 @@ static PyObject * _io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls); static PyObject * -_io_BytesIO_getbuffer(bytesio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_io_BytesIO_getbuffer(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "getbuffer() takes no arguments"); return NULL; } - return _io_BytesIO_getbuffer_impl(self, cls); + return _io_BytesIO_getbuffer_impl((bytesio *)self, cls); } PyDoc_STRVAR(_io_BytesIO_getvalue__doc__, @@ -116,9 +116,9 @@ static PyObject * _io_BytesIO_getvalue_impl(bytesio *self); static PyObject * -_io_BytesIO_getvalue(bytesio *self, PyObject *Py_UNUSED(ignored)) +_io_BytesIO_getvalue(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_getvalue_impl(self); + return _io_BytesIO_getvalue_impl((bytesio *)self); } PyDoc_STRVAR(_io_BytesIO_isatty__doc__, @@ -136,9 +136,9 @@ static PyObject * _io_BytesIO_isatty_impl(bytesio *self); static PyObject * -_io_BytesIO_isatty(bytesio *self, PyObject *Py_UNUSED(ignored)) +_io_BytesIO_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_isatty_impl(self); + return _io_BytesIO_isatty_impl((bytesio *)self); } PyDoc_STRVAR(_io_BytesIO_tell__doc__, @@ -154,9 +154,9 @@ static PyObject * _io_BytesIO_tell_impl(bytesio *self); static PyObject * -_io_BytesIO_tell(bytesio *self, PyObject *Py_UNUSED(ignored)) +_io_BytesIO_tell(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_tell_impl(self); + return _io_BytesIO_tell_impl((bytesio *)self); } PyDoc_STRVAR(_io_BytesIO_read__doc__, @@ -175,7 +175,7 @@ static PyObject * _io_BytesIO_read_impl(bytesio *self, Py_ssize_t size); static PyObject * -_io_BytesIO_read(bytesio *self, PyObject *const *args, Py_ssize_t nargs) +_io_BytesIO_read(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t size = -1; @@ -190,7 +190,7 @@ _io_BytesIO_read(bytesio *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: - return_value = _io_BytesIO_read_impl(self, size); + return_value = _io_BytesIO_read_impl((bytesio *)self, size); exit: return return_value; @@ -212,7 +212,7 @@ static PyObject * _io_BytesIO_read1_impl(bytesio *self, Py_ssize_t size); static PyObject * -_io_BytesIO_read1(bytesio *self, PyObject *const *args, Py_ssize_t nargs) +_io_BytesIO_read1(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t size = -1; @@ -227,7 +227,7 @@ _io_BytesIO_read1(bytesio *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: - return_value = _io_BytesIO_read1_impl(self, size); + return_value = _io_BytesIO_read1_impl((bytesio *)self, size); exit: return return_value; @@ -250,7 +250,7 @@ static PyObject * _io_BytesIO_readline_impl(bytesio *self, Py_ssize_t size); static PyObject * -_io_BytesIO_readline(bytesio *self, PyObject *const *args, Py_ssize_t nargs) +_io_BytesIO_readline(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t size = -1; @@ -265,7 +265,7 @@ _io_BytesIO_readline(bytesio *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: - return_value = _io_BytesIO_readline_impl(self, size); + return_value = _io_BytesIO_readline_impl((bytesio *)self, size); exit: return return_value; @@ -288,7 +288,7 @@ static PyObject * _io_BytesIO_readlines_impl(bytesio *self, PyObject *arg); static PyObject * -_io_BytesIO_readlines(bytesio *self, PyObject *const *args, Py_ssize_t nargs) +_io_BytesIO_readlines(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *arg = Py_None; @@ -301,7 +301,7 @@ _io_BytesIO_readlines(bytesio *self, PyObject *const *args, Py_ssize_t nargs) } arg = args[0]; skip_optional: - return_value = _io_BytesIO_readlines_impl(self, arg); + return_value = _io_BytesIO_readlines_impl((bytesio *)self, arg); exit: return return_value; @@ -323,7 +323,7 @@ static PyObject * _io_BytesIO_readinto_impl(bytesio *self, Py_buffer *buffer); static PyObject * -_io_BytesIO_readinto(bytesio *self, PyObject *arg) +_io_BytesIO_readinto(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; Py_buffer buffer = {NULL, NULL}; @@ -332,7 +332,7 @@ _io_BytesIO_readinto(bytesio *self, PyObject *arg) _PyArg_BadArgument("readinto", "argument", "read-write bytes-like object", arg); goto exit; } - return_value = _io_BytesIO_readinto_impl(self, &buffer); + return_value = _io_BytesIO_readinto_impl((bytesio *)self, &buffer); exit: /* Cleanup for buffer */ @@ -359,10 +359,10 @@ static PyObject * _io_BytesIO_truncate_impl(bytesio *self, Py_ssize_t size); static PyObject * -_io_BytesIO_truncate(bytesio *self, PyObject *const *args, Py_ssize_t nargs) +_io_BytesIO_truncate(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_ssize_t size = self->pos; + Py_ssize_t size = ((bytesio *)self)->pos; if (!_PyArg_CheckPositional("truncate", nargs, 0, 1)) { goto exit; @@ -374,7 +374,7 @@ _io_BytesIO_truncate(bytesio *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: - return_value = _io_BytesIO_truncate_impl(self, size); + return_value = _io_BytesIO_truncate_impl((bytesio *)self, size); exit: return return_value; @@ -399,7 +399,7 @@ static PyObject * _io_BytesIO_seek_impl(bytesio *self, Py_ssize_t pos, int whence); static PyObject * -_io_BytesIO_seek(bytesio *self, PyObject *const *args, Py_ssize_t nargs) +_io_BytesIO_seek(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t pos; @@ -428,7 +428,7 @@ _io_BytesIO_seek(bytesio *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: - return_value = _io_BytesIO_seek_impl(self, pos, whence); + return_value = _io_BytesIO_seek_impl((bytesio *)self, pos, whence); exit: return return_value; @@ -471,9 +471,9 @@ static PyObject * _io_BytesIO_close_impl(bytesio *self); static PyObject * -_io_BytesIO_close(bytesio *self, PyObject *Py_UNUSED(ignored)) +_io_BytesIO_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_close_impl(self); + return _io_BytesIO_close_impl((bytesio *)self); } PyDoc_STRVAR(_io_BytesIO___init____doc__, @@ -520,7 +520,8 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; PyObject *initvalue = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -534,4 +535,4 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=ef116925b8b9e535 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8a5e153bc7584b55 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index 5b5487d63eba90..22d27bce67799e 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -25,13 +25,13 @@ static PyObject * _io_FileIO_close_impl(fileio *self, PyTypeObject *cls); static PyObject * -_io_FileIO_close(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_io_FileIO_close(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "close() takes no arguments"); return NULL; } - return _io_FileIO_close_impl(self, cls); + return _io_FileIO_close_impl((fileio *)self, cls); } PyDoc_STRVAR(_io_FileIO___init____doc__, @@ -94,7 +94,8 @@ _io_FileIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) int closefd = 1; PyObject *opener = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 4, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -150,9 +151,9 @@ static PyObject * _io_FileIO_fileno_impl(fileio *self); static PyObject * -_io_FileIO_fileno(fileio *self, PyObject *Py_UNUSED(ignored)) +_io_FileIO_fileno(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_FileIO_fileno_impl(self); + return _io_FileIO_fileno_impl((fileio *)self); } PyDoc_STRVAR(_io_FileIO_readable__doc__, @@ -168,9 +169,9 @@ static PyObject * _io_FileIO_readable_impl(fileio *self); static PyObject * -_io_FileIO_readable(fileio *self, PyObject *Py_UNUSED(ignored)) +_io_FileIO_readable(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_FileIO_readable_impl(self); + return _io_FileIO_readable_impl((fileio *)self); } PyDoc_STRVAR(_io_FileIO_writable__doc__, @@ -186,9 +187,9 @@ static PyObject * _io_FileIO_writable_impl(fileio *self); static PyObject * -_io_FileIO_writable(fileio *self, PyObject *Py_UNUSED(ignored)) +_io_FileIO_writable(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_FileIO_writable_impl(self); + return _io_FileIO_writable_impl((fileio *)self); } PyDoc_STRVAR(_io_FileIO_seekable__doc__, @@ -204,9 +205,9 @@ static PyObject * _io_FileIO_seekable_impl(fileio *self); static PyObject * -_io_FileIO_seekable(fileio *self, PyObject *Py_UNUSED(ignored)) +_io_FileIO_seekable(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_FileIO_seekable_impl(self); + return _io_FileIO_seekable_impl((fileio *)self); } PyDoc_STRVAR(_io_FileIO_readinto__doc__, @@ -222,7 +223,7 @@ static PyObject * _io_FileIO_readinto_impl(fileio *self, PyTypeObject *cls, Py_buffer *buffer); static PyObject * -_io_FileIO_readinto(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_io_FileIO_readinto(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -241,7 +242,8 @@ _io_FileIO_readinto(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_s PyObject *argsbuf[1]; Py_buffer buffer = {NULL, NULL}; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -249,7 +251,7 @@ _io_FileIO_readinto(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_s _PyArg_BadArgument("readinto", "argument 1", "read-write bytes-like object", args[0]); goto exit; } - return_value = _io_FileIO_readinto_impl(self, cls, &buffer); + return_value = _io_FileIO_readinto_impl((fileio *)self, cls, &buffer); exit: /* Cleanup for buffer */ @@ -276,9 +278,9 @@ static PyObject * _io_FileIO_readall_impl(fileio *self); static PyObject * -_io_FileIO_readall(fileio *self, PyObject *Py_UNUSED(ignored)) +_io_FileIO_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_FileIO_readall_impl(self); + return _io_FileIO_readall_impl((fileio *)self); } PyDoc_STRVAR(_io_FileIO_read__doc__, @@ -298,7 +300,7 @@ static PyObject * _io_FileIO_read_impl(fileio *self, PyTypeObject *cls, Py_ssize_t size); static PyObject * -_io_FileIO_read(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_io_FileIO_read(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -317,7 +319,8 @@ _io_FileIO_read(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize PyObject *argsbuf[1]; Py_ssize_t size = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -328,7 +331,7 @@ _io_FileIO_read(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize goto exit; } skip_optional_posonly: - return_value = _io_FileIO_read_impl(self, cls, size); + return_value = _io_FileIO_read_impl((fileio *)self, cls, size); exit: return return_value; @@ -351,7 +354,7 @@ static PyObject * _io_FileIO_write_impl(fileio *self, PyTypeObject *cls, Py_buffer *b); static PyObject * -_io_FileIO_write(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_io_FileIO_write(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -370,14 +373,15 @@ _io_FileIO_write(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssiz PyObject *argsbuf[1]; Py_buffer b = {NULL, NULL}; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } if (PyObject_GetBuffer(args[0], &b, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = _io_FileIO_write_impl(self, cls, &b); + return_value = _io_FileIO_write_impl((fileio *)self, cls, &b); exit: /* Cleanup for b */ @@ -409,7 +413,7 @@ static PyObject * _io_FileIO_seek_impl(fileio *self, PyObject *pos, int whence); static PyObject * -_io_FileIO_seek(fileio *self, PyObject *const *args, Py_ssize_t nargs) +_io_FileIO_seek(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *pos; @@ -427,7 +431,7 @@ _io_FileIO_seek(fileio *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: - return_value = _io_FileIO_seek_impl(self, pos, whence); + return_value = _io_FileIO_seek_impl((fileio *)self, pos, whence); exit: return return_value; @@ -448,9 +452,9 @@ static PyObject * _io_FileIO_tell_impl(fileio *self); static PyObject * -_io_FileIO_tell(fileio *self, PyObject *Py_UNUSED(ignored)) +_io_FileIO_tell(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_FileIO_tell_impl(self); + return _io_FileIO_tell_impl((fileio *)self); } #if defined(HAVE_FTRUNCATE) @@ -471,7 +475,7 @@ static PyObject * _io_FileIO_truncate_impl(fileio *self, PyTypeObject *cls, PyObject *posobj); static PyObject * -_io_FileIO_truncate(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_io_FileIO_truncate(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -490,7 +494,8 @@ _io_FileIO_truncate(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_s PyObject *argsbuf[1]; PyObject *posobj = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -499,7 +504,7 @@ _io_FileIO_truncate(fileio *self, PyTypeObject *cls, PyObject *const *args, Py_s } posobj = args[0]; skip_optional_posonly: - return_value = _io_FileIO_truncate_impl(self, cls, posobj); + return_value = _io_FileIO_truncate_impl((fileio *)self, cls, posobj); exit: return return_value; @@ -520,12 +525,12 @@ static PyObject * _io_FileIO_isatty_impl(fileio *self); static PyObject * -_io_FileIO_isatty(fileio *self, PyObject *Py_UNUSED(ignored)) +_io_FileIO_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_FileIO_isatty_impl(self); + return _io_FileIO_isatty_impl((fileio *)self); } #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=e3d9446b4087020e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dcbeb6a0b13e4b1f input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/iobase.c.h b/Modules/_io/clinic/iobase.c.h index bae80a265fab07..402448545dfc51 100644 --- a/Modules/_io/clinic/iobase.c.h +++ b/Modules/_io/clinic/iobase.c.h @@ -2,6 +2,9 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() @@ -53,7 +56,8 @@ _io__IOBase_seek(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ss int offset; int whence = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -129,7 +133,8 @@ _io__IOBase_truncate(PyObject *self, PyTypeObject *cls, PyObject *const *args, P PyObject *argsbuf[1]; PyObject *size = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -438,4 +443,4 @@ _io__RawIOBase_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _io__RawIOBase_readall_impl(self); } -/*[clinic end generated code: output=e7326fbefc52bfba input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9359e74d95534bef input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/stringio.c.h b/Modules/_io/clinic/stringio.c.h index 6bdb2181985f7d..bc571698806bde 100644 --- a/Modules/_io/clinic/stringio.c.h +++ b/Modules/_io/clinic/stringio.c.h @@ -23,12 +23,12 @@ static PyObject * _io_StringIO_getvalue_impl(stringio *self); static PyObject * -_io_StringIO_getvalue(stringio *self, PyObject *Py_UNUSED(ignored)) +_io_StringIO_getvalue(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_StringIO_getvalue_impl(self); + return_value = _io_StringIO_getvalue_impl((stringio *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -47,12 +47,12 @@ static PyObject * _io_StringIO_tell_impl(stringio *self); static PyObject * -_io_StringIO_tell(stringio *self, PyObject *Py_UNUSED(ignored)) +_io_StringIO_tell(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_StringIO_tell_impl(self); + return_value = _io_StringIO_tell_impl((stringio *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -74,7 +74,7 @@ static PyObject * _io_StringIO_read_impl(stringio *self, Py_ssize_t size); static PyObject * -_io_StringIO_read(stringio *self, PyObject *const *args, Py_ssize_t nargs) +_io_StringIO_read(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t size = -1; @@ -90,7 +90,7 @@ _io_StringIO_read(stringio *self, PyObject *const *args, Py_ssize_t nargs) } skip_optional: Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_StringIO_read_impl(self, size); + return_value = _io_StringIO_read_impl((stringio *)self, size); Py_END_CRITICAL_SECTION(); exit: @@ -112,7 +112,7 @@ static PyObject * _io_StringIO_readline_impl(stringio *self, Py_ssize_t size); static PyObject * -_io_StringIO_readline(stringio *self, PyObject *const *args, Py_ssize_t nargs) +_io_StringIO_readline(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t size = -1; @@ -128,7 +128,7 @@ _io_StringIO_readline(stringio *self, PyObject *const *args, Py_ssize_t nargs) } skip_optional: Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_StringIO_readline_impl(self, size); + return_value = _io_StringIO_readline_impl((stringio *)self, size); Py_END_CRITICAL_SECTION(); exit: @@ -152,10 +152,10 @@ static PyObject * _io_StringIO_truncate_impl(stringio *self, Py_ssize_t size); static PyObject * -_io_StringIO_truncate(stringio *self, PyObject *const *args, Py_ssize_t nargs) +_io_StringIO_truncate(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_ssize_t size = self->pos; + Py_ssize_t size = ((stringio *)self)->pos; if (!_PyArg_CheckPositional("truncate", nargs, 0, 1)) { goto exit; @@ -168,7 +168,7 @@ _io_StringIO_truncate(stringio *self, PyObject *const *args, Py_ssize_t nargs) } skip_optional: Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_StringIO_truncate_impl(self, size); + return_value = _io_StringIO_truncate_impl((stringio *)self, size); Py_END_CRITICAL_SECTION(); exit: @@ -194,7 +194,7 @@ static PyObject * _io_StringIO_seek_impl(stringio *self, Py_ssize_t pos, int whence); static PyObject * -_io_StringIO_seek(stringio *self, PyObject *const *args, Py_ssize_t nargs) +_io_StringIO_seek(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t pos; @@ -224,7 +224,7 @@ _io_StringIO_seek(stringio *self, PyObject *const *args, Py_ssize_t nargs) } skip_optional: Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_StringIO_seek_impl(self, pos, whence); + return_value = _io_StringIO_seek_impl((stringio *)self, pos, whence); Py_END_CRITICAL_SECTION(); exit: @@ -252,7 +252,7 @@ _io_StringIO_write(stringio *self, PyObject *obj) PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_StringIO_write_impl(self, obj); + return_value = _io_StringIO_write_impl((stringio *)self, obj); Py_END_CRITICAL_SECTION(); return return_value; @@ -276,12 +276,12 @@ static PyObject * _io_StringIO_close_impl(stringio *self); static PyObject * -_io_StringIO_close(stringio *self, PyObject *Py_UNUSED(ignored)) +_io_StringIO_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_StringIO_close_impl(self); + return_value = _io_StringIO_close_impl((stringio *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -336,7 +336,8 @@ _io_StringIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *value = NULL; PyObject *newline_obj = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -370,12 +371,12 @@ static PyObject * _io_StringIO_readable_impl(stringio *self); static PyObject * -_io_StringIO_readable(stringio *self, PyObject *Py_UNUSED(ignored)) +_io_StringIO_readable(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_StringIO_readable_impl(self); + return_value = _io_StringIO_readable_impl((stringio *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -394,12 +395,12 @@ static PyObject * _io_StringIO_writable_impl(stringio *self); static PyObject * -_io_StringIO_writable(stringio *self, PyObject *Py_UNUSED(ignored)) +_io_StringIO_writable(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_StringIO_writable_impl(self); + return_value = _io_StringIO_writable_impl((stringio *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -418,12 +419,12 @@ static PyObject * _io_StringIO_seekable_impl(stringio *self); static PyObject * -_io_StringIO_seekable(stringio *self, PyObject *Py_UNUSED(ignored)) +_io_StringIO_seekable(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_StringIO_seekable_impl(self); + return_value = _io_StringIO_seekable_impl((stringio *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -441,12 +442,12 @@ static PyObject * _io_StringIO___getstate___impl(stringio *self); static PyObject * -_io_StringIO___getstate__(stringio *self, PyObject *Py_UNUSED(ignored)) +_io_StringIO___getstate__(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_StringIO___getstate___impl(self); + return_value = _io_StringIO___getstate___impl((stringio *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -469,15 +470,13 @@ _io_StringIO___setstate__(stringio *self, PyObject *state) PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_StringIO___setstate___impl(self, state); + return_value = _io_StringIO___setstate___impl((stringio *)self, state); Py_END_CRITICAL_SECTION(); return return_value; } -#if defined(_io_StringIO_closed_HAS_DOCSTR) -# define _io_StringIO_closed_DOCSTR _io_StringIO_closed__doc__ -#else +#if !defined(_io_StringIO_closed_DOCSTR) # define _io_StringIO_closed_DOCSTR NULL #endif #if defined(_IO_STRINGIO_CLOSED_GETSETDEF) @@ -491,20 +490,18 @@ static PyObject * _io_StringIO_closed_get_impl(stringio *self); static PyObject * -_io_StringIO_closed_get(stringio *self, void *Py_UNUSED(context)) +_io_StringIO_closed_get(PyObject *self, void *Py_UNUSED(context)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_StringIO_closed_get_impl(self); + return_value = _io_StringIO_closed_get_impl((stringio *)self); Py_END_CRITICAL_SECTION(); return return_value; } -#if defined(_io_StringIO_line_buffering_HAS_DOCSTR) -# define _io_StringIO_line_buffering_DOCSTR _io_StringIO_line_buffering__doc__ -#else +#if !defined(_io_StringIO_line_buffering_DOCSTR) # define _io_StringIO_line_buffering_DOCSTR NULL #endif #if defined(_IO_STRINGIO_LINE_BUFFERING_GETSETDEF) @@ -518,20 +515,18 @@ static PyObject * _io_StringIO_line_buffering_get_impl(stringio *self); static PyObject * -_io_StringIO_line_buffering_get(stringio *self, void *Py_UNUSED(context)) +_io_StringIO_line_buffering_get(PyObject *self, void *Py_UNUSED(context)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_StringIO_line_buffering_get_impl(self); + return_value = _io_StringIO_line_buffering_get_impl((stringio *)self); Py_END_CRITICAL_SECTION(); return return_value; } -#if defined(_io_StringIO_newlines_HAS_DOCSTR) -# define _io_StringIO_newlines_DOCSTR _io_StringIO_newlines__doc__ -#else +#if !defined(_io_StringIO_newlines_DOCSTR) # define _io_StringIO_newlines_DOCSTR NULL #endif #if defined(_IO_STRINGIO_NEWLINES_GETSETDEF) @@ -545,14 +540,14 @@ static PyObject * _io_StringIO_newlines_get_impl(stringio *self); static PyObject * -_io_StringIO_newlines_get(stringio *self, void *Py_UNUSED(context)) +_io_StringIO_newlines_get(PyObject *self, void *Py_UNUSED(context)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_StringIO_newlines_get_impl(self); + return_value = _io_StringIO_newlines_get_impl((stringio *)self); Py_END_CRITICAL_SECTION(); return return_value; } -/*[clinic end generated code: output=9ffea20cd32d4cd8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7796e223e778a214 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index f04ee729abc9ed..9ce1d70ad71052 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -4,7 +4,7 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() @@ -70,7 +70,8 @@ _io__TextIOBase_read(PyObject *self, PyTypeObject *cls, PyObject *const *args, P PyObject *argsbuf[1]; int size = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -124,7 +125,8 @@ _io__TextIOBase_readline(PyObject *self, PyTypeObject *cls, PyObject *const *arg PyObject *argsbuf[1]; int size = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -178,7 +180,8 @@ _io__TextIOBase_write(PyObject *self, PyTypeObject *cls, PyObject *const *args, PyObject *argsbuf[1]; const char *s; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -205,11 +208,12 @@ PyDoc_STRVAR(_io__TextIOBase_encoding__doc__, "Encoding of the text stream.\n" "\n" "Subclasses should override."); -#define _io__TextIOBase_encoding_HAS_DOCSTR +#if defined(_io__TextIOBase_encoding_DOCSTR) +# undef _io__TextIOBase_encoding_DOCSTR +#endif +#define _io__TextIOBase_encoding_DOCSTR _io__TextIOBase_encoding__doc__ -#if defined(_io__TextIOBase_encoding_HAS_DOCSTR) -# define _io__TextIOBase_encoding_DOCSTR _io__TextIOBase_encoding__doc__ -#else +#if !defined(_io__TextIOBase_encoding_DOCSTR) # define _io__TextIOBase_encoding_DOCSTR NULL #endif #if defined(_IO__TEXTIOBASE_ENCODING_GETSETDEF) @@ -234,11 +238,12 @@ PyDoc_STRVAR(_io__TextIOBase_newlines__doc__, "Only line endings translated during reading are considered.\n" "\n" "Subclasses should override."); -#define _io__TextIOBase_newlines_HAS_DOCSTR +#if defined(_io__TextIOBase_newlines_DOCSTR) +# undef _io__TextIOBase_newlines_DOCSTR +#endif +#define _io__TextIOBase_newlines_DOCSTR _io__TextIOBase_newlines__doc__ -#if defined(_io__TextIOBase_newlines_HAS_DOCSTR) -# define _io__TextIOBase_newlines_DOCSTR _io__TextIOBase_newlines__doc__ -#else +#if !defined(_io__TextIOBase_newlines_DOCSTR) # define _io__TextIOBase_newlines_DOCSTR NULL #endif #if defined(_IO__TEXTIOBASE_NEWLINES_GETSETDEF) @@ -261,11 +266,12 @@ PyDoc_STRVAR(_io__TextIOBase_errors__doc__, "The error setting of the decoder or encoder.\n" "\n" "Subclasses should override."); -#define _io__TextIOBase_errors_HAS_DOCSTR +#if defined(_io__TextIOBase_errors_DOCSTR) +# undef _io__TextIOBase_errors_DOCSTR +#endif +#define _io__TextIOBase_errors_DOCSTR _io__TextIOBase_errors__doc__ -#if defined(_io__TextIOBase_errors_HAS_DOCSTR) -# define _io__TextIOBase_errors_DOCSTR _io__TextIOBase_errors__doc__ -#else +#if !defined(_io__TextIOBase_errors_DOCSTR) # define _io__TextIOBase_errors_DOCSTR NULL #endif #if defined(_IO__TEXTIOBASE_ERRORS_GETSETDEF) @@ -339,7 +345,8 @@ _io_IncrementalNewlineDecoder___init__(PyObject *self, PyObject *args, PyObject int translate; PyObject *errors = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 2, 3, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -372,7 +379,7 @@ _io_IncrementalNewlineDecoder_decode_impl(nldecoder_object *self, PyObject *input, int final); static PyObject * -_io_IncrementalNewlineDecoder_decode(nldecoder_object *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_io_IncrementalNewlineDecoder_decode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -405,7 +412,8 @@ _io_IncrementalNewlineDecoder_decode(nldecoder_object *self, PyObject *const *ar PyObject *input; int final = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -418,7 +426,7 @@ _io_IncrementalNewlineDecoder_decode(nldecoder_object *self, PyObject *const *ar goto exit; } skip_optional_pos: - return_value = _io_IncrementalNewlineDecoder_decode_impl(self, input, final); + return_value = _io_IncrementalNewlineDecoder_decode_impl((nldecoder_object *)self, input, final); exit: return return_value; @@ -436,9 +444,9 @@ static PyObject * _io_IncrementalNewlineDecoder_getstate_impl(nldecoder_object *self); static PyObject * -_io_IncrementalNewlineDecoder_getstate(nldecoder_object *self, PyObject *Py_UNUSED(ignored)) +_io_IncrementalNewlineDecoder_getstate(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_IncrementalNewlineDecoder_getstate_impl(self); + return _io_IncrementalNewlineDecoder_getstate_impl((nldecoder_object *)self); } PyDoc_STRVAR(_io_IncrementalNewlineDecoder_setstate__doc__, @@ -461,9 +469,9 @@ static PyObject * _io_IncrementalNewlineDecoder_reset_impl(nldecoder_object *self); static PyObject * -_io_IncrementalNewlineDecoder_reset(nldecoder_object *self, PyObject *Py_UNUSED(ignored)) +_io_IncrementalNewlineDecoder_reset(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_IncrementalNewlineDecoder_reset_impl(self); + return _io_IncrementalNewlineDecoder_reset_impl((nldecoder_object *)self); } PyDoc_STRVAR(_io_TextIOWrapper___init____doc__, @@ -546,7 +554,8 @@ _io_TextIOWrapper___init__(PyObject *self, PyObject *args, PyObject *kwargs) int line_buffering = 0; int write_through = 0; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 6, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 6, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -645,7 +654,7 @@ _io_TextIOWrapper_reconfigure_impl(textio *self, PyObject *encoding, PyObject *write_through_obj); static PyObject * -_io_TextIOWrapper_reconfigure(textio *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_io_TextIOWrapper_reconfigure(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -681,7 +690,8 @@ _io_TextIOWrapper_reconfigure(textio *self, PyObject *const *args, Py_ssize_t na PyObject *line_buffering_obj = Py_None; PyObject *write_through_obj = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -715,7 +725,7 @@ _io_TextIOWrapper_reconfigure(textio *self, PyObject *const *args, Py_ssize_t na write_through_obj = args[4]; skip_optional_kwonly: Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper_reconfigure_impl(self, encoding, errors, newline_obj, line_buffering_obj, write_through_obj); + return_value = _io_TextIOWrapper_reconfigure_impl((textio *)self, encoding, errors, newline_obj, line_buffering_obj, write_through_obj); Py_END_CRITICAL_SECTION(); exit: @@ -734,12 +744,12 @@ static PyObject * _io_TextIOWrapper_detach_impl(textio *self); static PyObject * -_io_TextIOWrapper_detach(textio *self, PyObject *Py_UNUSED(ignored)) +_io_TextIOWrapper_detach(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper_detach_impl(self); + return_value = _io_TextIOWrapper_detach_impl((textio *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -757,7 +767,7 @@ static PyObject * _io_TextIOWrapper_write_impl(textio *self, PyObject *text); static PyObject * -_io_TextIOWrapper_write(textio *self, PyObject *arg) +_io_TextIOWrapper_write(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; PyObject *text; @@ -768,7 +778,7 @@ _io_TextIOWrapper_write(textio *self, PyObject *arg) } text = arg; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper_write_impl(self, text); + return_value = _io_TextIOWrapper_write_impl((textio *)self, text); Py_END_CRITICAL_SECTION(); exit: @@ -787,7 +797,7 @@ static PyObject * _io_TextIOWrapper_read_impl(textio *self, Py_ssize_t n); static PyObject * -_io_TextIOWrapper_read(textio *self, PyObject *const *args, Py_ssize_t nargs) +_io_TextIOWrapper_read(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t n = -1; @@ -803,7 +813,7 @@ _io_TextIOWrapper_read(textio *self, PyObject *const *args, Py_ssize_t nargs) } skip_optional: Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper_read_impl(self, n); + return_value = _io_TextIOWrapper_read_impl((textio *)self, n); Py_END_CRITICAL_SECTION(); exit: @@ -822,7 +832,7 @@ static PyObject * _io_TextIOWrapper_readline_impl(textio *self, Py_ssize_t size); static PyObject * -_io_TextIOWrapper_readline(textio *self, PyObject *const *args, Py_ssize_t nargs) +_io_TextIOWrapper_readline(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t size = -1; @@ -847,7 +857,7 @@ _io_TextIOWrapper_readline(textio *self, PyObject *const *args, Py_ssize_t nargs } skip_optional: Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper_readline_impl(self, size); + return_value = _io_TextIOWrapper_readline_impl((textio *)self, size); Py_END_CRITICAL_SECTION(); exit: @@ -884,7 +894,7 @@ static PyObject * _io_TextIOWrapper_seek_impl(textio *self, PyObject *cookieObj, int whence); static PyObject * -_io_TextIOWrapper_seek(textio *self, PyObject *const *args, Py_ssize_t nargs) +_io_TextIOWrapper_seek(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *cookieObj; @@ -903,7 +913,7 @@ _io_TextIOWrapper_seek(textio *self, PyObject *const *args, Py_ssize_t nargs) } skip_optional: Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper_seek_impl(self, cookieObj, whence); + return_value = _io_TextIOWrapper_seek_impl((textio *)self, cookieObj, whence); Py_END_CRITICAL_SECTION(); exit: @@ -926,12 +936,12 @@ static PyObject * _io_TextIOWrapper_tell_impl(textio *self); static PyObject * -_io_TextIOWrapper_tell(textio *self, PyObject *Py_UNUSED(ignored)) +_io_TextIOWrapper_tell(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper_tell_impl(self); + return_value = _io_TextIOWrapper_tell_impl((textio *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -949,7 +959,7 @@ static PyObject * _io_TextIOWrapper_truncate_impl(textio *self, PyObject *pos); static PyObject * -_io_TextIOWrapper_truncate(textio *self, PyObject *const *args, Py_ssize_t nargs) +_io_TextIOWrapper_truncate(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *pos = Py_None; @@ -963,7 +973,7 @@ _io_TextIOWrapper_truncate(textio *self, PyObject *const *args, Py_ssize_t nargs pos = args[0]; skip_optional: Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper_truncate_impl(self, pos); + return_value = _io_TextIOWrapper_truncate_impl((textio *)self, pos); Py_END_CRITICAL_SECTION(); exit: @@ -982,12 +992,12 @@ static PyObject * _io_TextIOWrapper_fileno_impl(textio *self); static PyObject * -_io_TextIOWrapper_fileno(textio *self, PyObject *Py_UNUSED(ignored)) +_io_TextIOWrapper_fileno(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper_fileno_impl(self); + return_value = _io_TextIOWrapper_fileno_impl((textio *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -1005,12 +1015,12 @@ static PyObject * _io_TextIOWrapper_seekable_impl(textio *self); static PyObject * -_io_TextIOWrapper_seekable(textio *self, PyObject *Py_UNUSED(ignored)) +_io_TextIOWrapper_seekable(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper_seekable_impl(self); + return_value = _io_TextIOWrapper_seekable_impl((textio *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -1028,12 +1038,12 @@ static PyObject * _io_TextIOWrapper_readable_impl(textio *self); static PyObject * -_io_TextIOWrapper_readable(textio *self, PyObject *Py_UNUSED(ignored)) +_io_TextIOWrapper_readable(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper_readable_impl(self); + return_value = _io_TextIOWrapper_readable_impl((textio *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -1051,12 +1061,12 @@ static PyObject * _io_TextIOWrapper_writable_impl(textio *self); static PyObject * -_io_TextIOWrapper_writable(textio *self, PyObject *Py_UNUSED(ignored)) +_io_TextIOWrapper_writable(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper_writable_impl(self); + return_value = _io_TextIOWrapper_writable_impl((textio *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -1074,12 +1084,12 @@ static PyObject * _io_TextIOWrapper_isatty_impl(textio *self); static PyObject * -_io_TextIOWrapper_isatty(textio *self, PyObject *Py_UNUSED(ignored)) +_io_TextIOWrapper_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper_isatty_impl(self); + return_value = _io_TextIOWrapper_isatty_impl((textio *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -1097,12 +1107,12 @@ static PyObject * _io_TextIOWrapper_flush_impl(textio *self); static PyObject * -_io_TextIOWrapper_flush(textio *self, PyObject *Py_UNUSED(ignored)) +_io_TextIOWrapper_flush(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper_flush_impl(self); + return_value = _io_TextIOWrapper_flush_impl((textio *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -1120,20 +1130,18 @@ static PyObject * _io_TextIOWrapper_close_impl(textio *self); static PyObject * -_io_TextIOWrapper_close(textio *self, PyObject *Py_UNUSED(ignored)) +_io_TextIOWrapper_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper_close_impl(self); + return_value = _io_TextIOWrapper_close_impl((textio *)self); Py_END_CRITICAL_SECTION(); return return_value; } -#if defined(_io_TextIOWrapper_name_HAS_DOCSTR) -# define _io_TextIOWrapper_name_DOCSTR _io_TextIOWrapper_name__doc__ -#else +#if !defined(_io_TextIOWrapper_name_DOCSTR) # define _io_TextIOWrapper_name_DOCSTR NULL #endif #if defined(_IO_TEXTIOWRAPPER_NAME_GETSETDEF) @@ -1147,20 +1155,18 @@ static PyObject * _io_TextIOWrapper_name_get_impl(textio *self); static PyObject * -_io_TextIOWrapper_name_get(textio *self, void *Py_UNUSED(context)) +_io_TextIOWrapper_name_get(PyObject *self, void *Py_UNUSED(context)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper_name_get_impl(self); + return_value = _io_TextIOWrapper_name_get_impl((textio *)self); Py_END_CRITICAL_SECTION(); return return_value; } -#if defined(_io_TextIOWrapper_closed_HAS_DOCSTR) -# define _io_TextIOWrapper_closed_DOCSTR _io_TextIOWrapper_closed__doc__ -#else +#if !defined(_io_TextIOWrapper_closed_DOCSTR) # define _io_TextIOWrapper_closed_DOCSTR NULL #endif #if defined(_IO_TEXTIOWRAPPER_CLOSED_GETSETDEF) @@ -1174,20 +1180,18 @@ static PyObject * _io_TextIOWrapper_closed_get_impl(textio *self); static PyObject * -_io_TextIOWrapper_closed_get(textio *self, void *Py_UNUSED(context)) +_io_TextIOWrapper_closed_get(PyObject *self, void *Py_UNUSED(context)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper_closed_get_impl(self); + return_value = _io_TextIOWrapper_closed_get_impl((textio *)self); Py_END_CRITICAL_SECTION(); return return_value; } -#if defined(_io_TextIOWrapper_newlines_HAS_DOCSTR) -# define _io_TextIOWrapper_newlines_DOCSTR _io_TextIOWrapper_newlines__doc__ -#else +#if !defined(_io_TextIOWrapper_newlines_DOCSTR) # define _io_TextIOWrapper_newlines_DOCSTR NULL #endif #if defined(_IO_TEXTIOWRAPPER_NEWLINES_GETSETDEF) @@ -1201,20 +1205,18 @@ static PyObject * _io_TextIOWrapper_newlines_get_impl(textio *self); static PyObject * -_io_TextIOWrapper_newlines_get(textio *self, void *Py_UNUSED(context)) +_io_TextIOWrapper_newlines_get(PyObject *self, void *Py_UNUSED(context)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper_newlines_get_impl(self); + return_value = _io_TextIOWrapper_newlines_get_impl((textio *)self); Py_END_CRITICAL_SECTION(); return return_value; } -#if defined(_io_TextIOWrapper_errors_HAS_DOCSTR) -# define _io_TextIOWrapper_errors_DOCSTR _io_TextIOWrapper_errors__doc__ -#else +#if !defined(_io_TextIOWrapper_errors_DOCSTR) # define _io_TextIOWrapper_errors_DOCSTR NULL #endif #if defined(_IO_TEXTIOWRAPPER_ERRORS_GETSETDEF) @@ -1228,20 +1230,18 @@ static PyObject * _io_TextIOWrapper_errors_get_impl(textio *self); static PyObject * -_io_TextIOWrapper_errors_get(textio *self, void *Py_UNUSED(context)) +_io_TextIOWrapper_errors_get(PyObject *self, void *Py_UNUSED(context)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper_errors_get_impl(self); + return_value = _io_TextIOWrapper_errors_get_impl((textio *)self); Py_END_CRITICAL_SECTION(); return return_value; } -#if defined(_io_TextIOWrapper__CHUNK_SIZE_HAS_DOCSTR) -# define _io_TextIOWrapper__CHUNK_SIZE_DOCSTR _io_TextIOWrapper__CHUNK_SIZE__doc__ -#else +#if !defined(_io_TextIOWrapper__CHUNK_SIZE_DOCSTR) # define _io_TextIOWrapper__CHUNK_SIZE_DOCSTR NULL #endif #if defined(_IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF) @@ -1255,20 +1255,18 @@ static PyObject * _io_TextIOWrapper__CHUNK_SIZE_get_impl(textio *self); static PyObject * -_io_TextIOWrapper__CHUNK_SIZE_get(textio *self, void *Py_UNUSED(context)) +_io_TextIOWrapper__CHUNK_SIZE_get(PyObject *self, void *Py_UNUSED(context)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper__CHUNK_SIZE_get_impl(self); + return_value = _io_TextIOWrapper__CHUNK_SIZE_get_impl((textio *)self); Py_END_CRITICAL_SECTION(); return return_value; } -#if defined(_IO_TEXTIOWRAPPER__CHUNK_SIZE_HAS_DOCSTR) -# define _io_TextIOWrapper__CHUNK_SIZE_DOCSTR _io_TextIOWrapper__CHUNK_SIZE__doc__ -#else +#if !defined(_io_TextIOWrapper__CHUNK_SIZE_DOCSTR) # define _io_TextIOWrapper__CHUNK_SIZE_DOCSTR NULL #endif #if defined(_IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF) @@ -1282,14 +1280,14 @@ static int _io_TextIOWrapper__CHUNK_SIZE_set_impl(textio *self, PyObject *value); static int -_io_TextIOWrapper__CHUNK_SIZE_set(textio *self, PyObject *value, void *Py_UNUSED(context)) +_io_TextIOWrapper__CHUNK_SIZE_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) { int return_value; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_TextIOWrapper__CHUNK_SIZE_set_impl(self, value); + return_value = _io_TextIOWrapper__CHUNK_SIZE_set_impl((textio *)self, value); Py_END_CRITICAL_SECTION(); return return_value; } -/*[clinic end generated code: output=93a5a91a22100a28 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6e64e43113a97340 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index 4696ecc5c843e6..ba6dcde6e01064 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -27,13 +27,13 @@ static PyObject * _io__WindowsConsoleIO_close_impl(winconsoleio *self, PyTypeObject *cls); static PyObject * -_io__WindowsConsoleIO_close(winconsoleio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_io__WindowsConsoleIO_close(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "close() takes no arguments"); return NULL; } - return _io__WindowsConsoleIO_close_impl(self, cls); + return _io__WindowsConsoleIO_close_impl((winconsoleio *)self, cls); } #endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ @@ -93,7 +93,8 @@ _io__WindowsConsoleIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) int closefd = 1; PyObject *opener = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 4, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -153,9 +154,9 @@ static PyObject * _io__WindowsConsoleIO_fileno_impl(winconsoleio *self); static PyObject * -_io__WindowsConsoleIO_fileno(winconsoleio *self, PyObject *Py_UNUSED(ignored)) +_io__WindowsConsoleIO_fileno(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io__WindowsConsoleIO_fileno_impl(self); + return _io__WindowsConsoleIO_fileno_impl((winconsoleio *)self); } #endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ @@ -175,9 +176,9 @@ static PyObject * _io__WindowsConsoleIO_readable_impl(winconsoleio *self); static PyObject * -_io__WindowsConsoleIO_readable(winconsoleio *self, PyObject *Py_UNUSED(ignored)) +_io__WindowsConsoleIO_readable(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io__WindowsConsoleIO_readable_impl(self); + return _io__WindowsConsoleIO_readable_impl((winconsoleio *)self); } #endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ @@ -197,9 +198,9 @@ static PyObject * _io__WindowsConsoleIO_writable_impl(winconsoleio *self); static PyObject * -_io__WindowsConsoleIO_writable(winconsoleio *self, PyObject *Py_UNUSED(ignored)) +_io__WindowsConsoleIO_writable(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io__WindowsConsoleIO_writable_impl(self); + return _io__WindowsConsoleIO_writable_impl((winconsoleio *)self); } #endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ @@ -220,7 +221,7 @@ _io__WindowsConsoleIO_readinto_impl(winconsoleio *self, PyTypeObject *cls, Py_buffer *buffer); static PyObject * -_io__WindowsConsoleIO_readinto(winconsoleio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_io__WindowsConsoleIO_readinto(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -239,7 +240,8 @@ _io__WindowsConsoleIO_readinto(winconsoleio *self, PyTypeObject *cls, PyObject * PyObject *argsbuf[1]; Py_buffer buffer = {NULL, NULL}; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -247,7 +249,7 @@ _io__WindowsConsoleIO_readinto(winconsoleio *self, PyTypeObject *cls, PyObject * _PyArg_BadArgument("readinto", "argument 1", "read-write bytes-like object", args[0]); goto exit; } - return_value = _io__WindowsConsoleIO_readinto_impl(self, cls, &buffer); + return_value = _io__WindowsConsoleIO_readinto_impl((winconsoleio *)self, cls, &buffer); exit: /* Cleanup for buffer */ @@ -277,9 +279,9 @@ static PyObject * _io__WindowsConsoleIO_readall_impl(winconsoleio *self); static PyObject * -_io__WindowsConsoleIO_readall(winconsoleio *self, PyObject *Py_UNUSED(ignored)) +_io__WindowsConsoleIO_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io__WindowsConsoleIO_readall_impl(self); + return _io__WindowsConsoleIO_readall_impl((winconsoleio *)self); } #endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ @@ -304,7 +306,7 @@ _io__WindowsConsoleIO_read_impl(winconsoleio *self, PyTypeObject *cls, Py_ssize_t size); static PyObject * -_io__WindowsConsoleIO_read(winconsoleio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_io__WindowsConsoleIO_read(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -323,7 +325,8 @@ _io__WindowsConsoleIO_read(winconsoleio *self, PyTypeObject *cls, PyObject *cons PyObject *argsbuf[1]; Py_ssize_t size = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -334,7 +337,7 @@ _io__WindowsConsoleIO_read(winconsoleio *self, PyTypeObject *cls, PyObject *cons goto exit; } skip_optional_posonly: - return_value = _io__WindowsConsoleIO_read_impl(self, cls, size); + return_value = _io__WindowsConsoleIO_read_impl((winconsoleio *)self, cls, size); exit: return return_value; @@ -361,7 +364,7 @@ _io__WindowsConsoleIO_write_impl(winconsoleio *self, PyTypeObject *cls, Py_buffer *b); static PyObject * -_io__WindowsConsoleIO_write(winconsoleio *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_io__WindowsConsoleIO_write(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -380,14 +383,15 @@ _io__WindowsConsoleIO_write(winconsoleio *self, PyTypeObject *cls, PyObject *con PyObject *argsbuf[1]; Py_buffer b = {NULL, NULL}; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } if (PyObject_GetBuffer(args[0], &b, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = _io__WindowsConsoleIO_write_impl(self, cls, &b); + return_value = _io__WindowsConsoleIO_write_impl((winconsoleio *)self, cls, &b); exit: /* Cleanup for b */ @@ -415,9 +419,9 @@ static PyObject * _io__WindowsConsoleIO_isatty_impl(winconsoleio *self); static PyObject * -_io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) +_io__WindowsConsoleIO_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io__WindowsConsoleIO_isatty_impl(self); + return _io__WindowsConsoleIO_isatty_impl((winconsoleio *)self); } #endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ @@ -457,4 +461,4 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=2c2bc86713b21dd6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=edc47f5c49589045 input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 6bb156e41fe43c..cf0f1d671b507a 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -12,9 +12,6 @@ #ifdef HAVE_SYS_TYPES_H # include #endif -#ifdef HAVE_SYS_STAT_H -# include -#endif #ifdef HAVE_IO_H # include #endif @@ -54,6 +51,9 @@ # define SMALLCHUNK BUFSIZ #endif +/* Size at which a buffer is considered "large" and behavior should change to + avoid excessive memory allocation */ +#define LARGE_BUFFER_CUTOFF_SIZE 65536 /*[clinic input] module _io @@ -71,12 +71,19 @@ typedef struct { signed int seekable : 2; /* -1 means unknown */ unsigned int closefd : 1; char finalizing; - unsigned int blksize; + /* Stat result which was grabbed at file open, useful for optimizing common + File I/O patterns to be more efficient. This is only guidance / an + estimate, as it is subject to Time-Of-Check to Time-Of-Use (TOCTOU) + issues / bugs. Both the underlying file descriptor and file may be + modified outside of the fileio object / Python (ex. gh-90102, GH-121941, + gh-109523). */ + struct _Py_stat_struct *stat_atopen; PyObject *weakreflist; PyObject *dict; } fileio; #define PyFileIO_Check(state, op) (PyObject_TypeCheck((op), state->PyFileIO_Type)) +#define _PyFileIO_CAST(op) _Py_CAST(fileio*, (op)) /* Forward declarations */ static PyObject* portable_lseek(fileio *self, PyObject *posobj, int whence, bool suppress_pipe_error); @@ -84,15 +91,16 @@ static PyObject* portable_lseek(fileio *self, PyObject *posobj, int whence, bool int _PyFileIO_closed(PyObject *self) { - return ((fileio *)self)->fd < 0; + return (_PyFileIO_CAST(self)->fd < 0); } /* Because this can call arbitrary code, it shouldn't be called when the refcount is 0 (that is, not directly from tp_dealloc unless the refcount has been temporarily re-incremented). */ static PyObject * -fileio_dealloc_warn(fileio *self, PyObject *source) +fileio_dealloc_warn(PyObject *op, PyObject *source) { + fileio *self = _PyFileIO_CAST(op); if (self->fd >= 0 && self->closefd) { PyObject *exc = PyErr_GetRaisedException(); if (PyErr_ResourceWarning(source, 1, "unclosed file %R", source)) { @@ -123,6 +131,8 @@ internal_close(fileio *self) _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS } + PyMem_Free(self->stat_atopen); + self->stat_atopen = NULL; if (err < 0) { errno = save_errno; PyErr_SetFromErrno(PyExc_OSError); @@ -162,7 +172,7 @@ _io_FileIO_close_impl(fileio *self, PyTypeObject *cls) exc = PyErr_GetRaisedException(); } if (self->finalizing) { - PyObject *r = fileio_dealloc_warn(self, (PyObject *) self); + PyObject *r = fileio_dealloc_warn((PyObject*)self, (PyObject *) self); if (r) { Py_DECREF(r); } @@ -183,23 +193,22 @@ _io_FileIO_close_impl(fileio *self, PyTypeObject *cls) static PyObject * fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - fileio *self; - assert(type != NULL && type->tp_alloc != NULL); - self = (fileio *) type->tp_alloc(type, 0); - if (self != NULL) { - self->fd = -1; - self->created = 0; - self->readable = 0; - self->writable = 0; - self->appending = 0; - self->seekable = -1; - self->blksize = 0; - self->closefd = 1; - self->weakreflist = NULL; + fileio *self = (fileio *) type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; } + self->fd = -1; + self->created = 0; + self->readable = 0; + self->writable = 0; + self->appending = 0; + self->seekable = -1; + self->stat_atopen = NULL; + self->closefd = 1; + self->weakreflist = NULL; return (PyObject *) self; } @@ -251,7 +260,6 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, #elif !defined(MS_WINDOWS) int *atomic_flag_works = NULL; #endif - struct _Py_stat_struct fdfstat; int fstat_result; int async_err = 0; @@ -262,8 +270,9 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, if (self->fd >= 0) { if (self->closefd) { /* Have to close the existing file first. */ - if (internal_close(self) < 0) + if (internal_close(self) < 0) { return -1; + } } else self->fd = -1; @@ -449,9 +458,14 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, #endif } - self->blksize = DEFAULT_BUFFER_SIZE; + PyMem_Free(self->stat_atopen); + self->stat_atopen = PyMem_New(struct _Py_stat_struct, 1); + if (self->stat_atopen == NULL) { + PyErr_NoMemory(); + goto error; + } Py_BEGIN_ALLOW_THREADS - fstat_result = _Py_fstat_noraise(self->fd, &fdfstat); + fstat_result = _Py_fstat_noraise(self->fd, self->stat_atopen); Py_END_ALLOW_THREADS if (fstat_result < 0) { /* Tolerate fstat() errors other than EBADF. See Issue #25717, where @@ -466,22 +480,21 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, #endif goto error; } + + PyMem_Free(self->stat_atopen); + self->stat_atopen = NULL; } else { #if defined(S_ISDIR) && defined(EISDIR) /* On Unix, open will succeed for directories. In Python, there should be no file objects referring to directories, so we need a check. */ - if (S_ISDIR(fdfstat.st_mode)) { + if (S_ISDIR(self->stat_atopen->st_mode)) { errno = EISDIR; PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, nameobj); goto error; } #endif /* defined(S_ISDIR) */ -#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE - if (fdfstat.st_blksize > 1) - self->blksize = fdfstat.st_blksize; -#endif /* HAVE_STRUCT_STAT_ST_BLKSIZE */ } #if defined(MS_WINDOWS) || defined(__CYGWIN__) @@ -513,6 +526,8 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, internal_close(self); _PyErr_ChainExceptions1(exc); } + PyMem_Free(self->stat_atopen); + self->stat_atopen = NULL; done: #ifdef MS_WINDOWS @@ -523,32 +538,43 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, } static int -fileio_traverse(fileio *self, visitproc visit, void *arg) +fileio_traverse(PyObject *op, visitproc visit, void *arg) { + fileio *self = _PyFileIO_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->dict); return 0; } static int -fileio_clear(fileio *self) +fileio_clear(PyObject *op) { + fileio *self = _PyFileIO_CAST(op); Py_CLEAR(self->dict); return 0; } static void -fileio_dealloc(fileio *self) +fileio_dealloc(PyObject *op) { - PyTypeObject *tp = Py_TYPE(self); + fileio *self = _PyFileIO_CAST(op); self->finalizing = 1; - if (_PyIOBase_finalize((PyObject *) self) < 0) + if (_PyIOBase_finalize(op) < 0) { return; + } + _PyObject_GC_UNTRACK(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) self); - (void)fileio_clear(self); - tp->tp_free((PyObject *)self); + if (self->stat_atopen != NULL) { + PyMem_Free(self->stat_atopen); + self->stat_atopen = NULL; + } + if (self->weakreflist != NULL) { + PyObject_ClearWeakRefs(op); + } + (void)fileio_clear(op); + + PyTypeObject *tp = Py_TYPE(op); + tp->tp_free(op); Py_DECREF(tp); } @@ -684,7 +710,7 @@ new_buffersize(fileio *self, size_t currentsize) giving us amortized linear-time behavior. For bigger sizes, use a less-than-double growth factor to avoid excessive allocation. */ assert(currentsize <= PY_SSIZE_T_MAX); - if (currentsize > 65536) + if (currentsize > LARGE_BUFFER_CUTOFF_SIZE) addend = currentsize >> 3; else addend = 256 + currentsize; @@ -707,43 +733,63 @@ static PyObject * _io_FileIO_readall_impl(fileio *self) /*[clinic end generated code: output=faa0292b213b4022 input=dbdc137f55602834]*/ { - struct _Py_stat_struct status; Py_off_t pos, end; PyObject *result; Py_ssize_t bytes_read = 0; Py_ssize_t n; size_t bufsize; - int fstat_result; - if (self->fd < 0) + if (self->fd < 0) { return err_closed(); + } - Py_BEGIN_ALLOW_THREADS - _Py_BEGIN_SUPPRESS_IPH + if (self->stat_atopen != NULL && self->stat_atopen->st_size < _PY_READ_MAX) { + end = (Py_off_t)self->stat_atopen->st_size; + } + else { + end = -1; + } + if (end <= 0) { + /* Use a default size and resize as needed. */ + bufsize = SMALLCHUNK; + } + else { + /* This is probably a real file. */ + if (end > _PY_READ_MAX - 1) { + bufsize = _PY_READ_MAX; + } + else { + /* In order to detect end of file, need a read() of at + least 1 byte which returns size 0. Oversize the buffer + by 1 byte so the I/O can be completed with two read() + calls (one for all data, one for EOF) without needing + to resize the buffer. */ + bufsize = (size_t)end + 1; + } + + /* While a lot of code does open().read() to get the whole contents + of a file it is possible a caller seeks/reads a ways into the file + then calls readall() to get the rest, which would result in allocating + more than required. Guard against that for larger files where we expect + the I/O time to dominate anyways while keeping small files fast. */ + if (bufsize > LARGE_BUFFER_CUTOFF_SIZE) { + Py_BEGIN_ALLOW_THREADS + _Py_BEGIN_SUPPRESS_IPH #ifdef MS_WINDOWS - pos = _lseeki64(self->fd, 0L, SEEK_CUR); + pos = _lseeki64(self->fd, 0L, SEEK_CUR); #else - pos = lseek(self->fd, 0L, SEEK_CUR); + pos = lseek(self->fd, 0L, SEEK_CUR); #endif - _Py_END_SUPPRESS_IPH - fstat_result = _Py_fstat_noraise(self->fd, &status); - Py_END_ALLOW_THREADS + _Py_END_SUPPRESS_IPH + Py_END_ALLOW_THREADS - if (fstat_result == 0) - end = status.st_size; - else - end = (Py_off_t)-1; - - if (end > 0 && end >= pos && pos >= 0 && end - pos < PY_SSIZE_T_MAX) { - /* This is probably a real file, so we try to allocate a - buffer one byte larger than the rest of the file. If the - calculation is right then we should get EOF without having - to enlarge the buffer. */ - bufsize = (size_t)(end - pos + 1); - } else { - bufsize = SMALLCHUNK; + if (end >= pos && pos >= 0 && (end - pos) < (_PY_READ_MAX - 1)) { + bufsize = (size_t)(end - pos) + 1; + } + } } + result = PyBytes_FromStringAndSize(NULL, bufsize); if (result == NULL) return NULL; @@ -783,7 +829,6 @@ _io_FileIO_readall_impl(fileio *self) return NULL; } bytes_read += n; - pos += n; } if (PyBytes_GET_SIZE(result) > bytes_read) { @@ -1074,6 +1119,14 @@ _io_FileIO_truncate_impl(fileio *self, PyTypeObject *cls, PyObject *posobj) return NULL; } + /* Since the file was truncated, its size at open is no longer accurate + as an estimate. Clear out the stat result, and rely on dynamic resize + code if a readall is requested. */ + if (self->stat_atopen != NULL) { + PyMem_Free(self->stat_atopen); + self->stat_atopen = NULL; + } + return posobj; } #endif /* HAVE_FTRUNCATE */ @@ -1104,18 +1157,20 @@ mode_string(fileio *self) } static PyObject * -fileio_repr(fileio *self) +fileio_repr(PyObject *op) { - PyObject *nameobj, *res; - const char *type_name = Py_TYPE((PyObject *) self)->tp_name; + fileio *self = _PyFileIO_CAST(op); + const char *type_name = Py_TYPE(self)->tp_name; if (self->fd < 0) { return PyUnicode_FromFormat("<%.100s [closed]>", type_name); } + PyObject *nameobj; if (PyObject_GetOptionalAttr((PyObject *) self, &_Py_ID(name), &nameobj) < 0) { return NULL; } + PyObject *res; if (nameobj == NULL) { res = PyUnicode_FromFormat( "<%.100s fd=%d mode='%s' closefd=%s>", @@ -1161,6 +1216,24 @@ _io_FileIO_isatty_impl(fileio *self) return PyBool_FromLong(res); } +/* Checks whether the file is a TTY using an open-only optimization. + + TTYs are always character devices. If the interpreter knows a file is + not a character device when it would call ``isatty``, can skip that + call. Inside ``open()`` there is a fresh stat result that contains that + information. Use the stat result to skip a system call. Outside of that + context TOCTOU issues (the fd could be arbitrarily modified by + surrounding code). */ +static PyObject * +_io_FileIO_isatty_open_only(PyObject *op, PyObject *Py_UNUSED(ignored)) +{ + fileio *self = _PyFileIO_CAST(op); + if (self->stat_atopen != NULL && !S_ISCHR(self->stat_atopen->st_mode)) { + Py_RETURN_FALSE; + } + return _io_FileIO_isatty_impl(self); +} + #include "clinic/fileio.c.h" static PyMethodDef fileio_methods[] = { @@ -1177,42 +1250,58 @@ static PyMethodDef fileio_methods[] = { _IO_FILEIO_WRITABLE_METHODDEF _IO_FILEIO_FILENO_METHODDEF _IO_FILEIO_ISATTY_METHODDEF - {"_dealloc_warn", (PyCFunction)fileio_dealloc_warn, METH_O, NULL}, - {"__reduce__", _PyIOBase_cannot_pickle, METH_VARARGS}, - {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_VARARGS}, + {"_isatty_open_only", _io_FileIO_isatty_open_only, METH_NOARGS}, + {"_dealloc_warn", fileio_dealloc_warn, METH_O, NULL}, + {"__reduce__", _PyIOBase_cannot_pickle, METH_NOARGS}, + {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_O}, {NULL, NULL} /* sentinel */ }; /* 'closed' and 'mode' are attributes for backwards compatibility reasons. */ static PyObject * -get_closed(fileio *self, void *closure) +fileio_get_closed(PyObject *op, void *closure) { + fileio *self = _PyFileIO_CAST(op); return PyBool_FromLong((long)(self->fd < 0)); } static PyObject * -get_closefd(fileio *self, void *closure) +fileio_get_closefd(PyObject *op, void *closure) { + fileio *self = _PyFileIO_CAST(op); return PyBool_FromLong((long)(self->closefd)); } static PyObject * -get_mode(fileio *self, void *closure) +fileio_get_mode(PyObject *op, void *closure) { + fileio *self = _PyFileIO_CAST(op); return PyUnicode_FromString(mode_string(self)); } +static PyObject * +fileio_get_blksize(PyObject *op, void *closure) +{ + fileio *self = _PyFileIO_CAST(op); +#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE + if (self->stat_atopen != NULL && self->stat_atopen->st_blksize > 1) { + return PyLong_FromLong(self->stat_atopen->st_blksize); + } +#endif /* HAVE_STRUCT_STAT_ST_BLKSIZE */ + return PyLong_FromLong(DEFAULT_BUFFER_SIZE); +} + static PyGetSetDef fileio_getsetlist[] = { - {"closed", (getter)get_closed, NULL, "True if the file is closed"}, - {"closefd", (getter)get_closefd, NULL, + {"closed", fileio_get_closed, NULL, "True if the file is closed"}, + {"closefd", fileio_get_closefd, NULL, "True if the file descriptor will be closed by close()."}, - {"mode", (getter)get_mode, NULL, "String giving the file mode"}, + {"mode", fileio_get_mode, NULL, "String giving the file mode"}, + {"_blksize", fileio_get_blksize, NULL, "Stat st_blksize if available"}, {NULL}, }; static PyMemberDef fileio_members[] = { - {"_blksize", Py_T_UINT, offsetof(fileio, blksize), 0}, {"_finalizing", Py_T_BOOL, offsetof(fileio, finalizing), 0}, {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(fileio, weakreflist), Py_READONLY}, {"__dictoffset__", Py_T_PYSSIZET, offsetof(fileio, dict), Py_READONLY}, diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 184e0b7d1aa7f1..419e5516b5c11e 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -999,7 +999,7 @@ _io__RawIOBase_readall_impl(PyObject *self) return NULL; } } - result = _PyBytes_Join((PyObject *)&_Py_SINGLETON(bytes_empty), chunks); + result = PyBytes_Join((PyObject *)&_Py_SINGLETON(bytes_empty), chunks); Py_DECREF(chunks); return result; } diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 06bc2679e8e227..d1a71298a9087f 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -30,7 +30,7 @@ typedef struct { _PyUnicodeWriter is destroyed. */ int state; - _PyUnicodeWriter writer; + PyUnicodeWriter *writer; char ok; /* initialized? */ char closed; @@ -129,14 +129,18 @@ resize_buffer(stringio *self, size_t size) static PyObject * make_intermediate(stringio *self) { - PyObject *intermediate = _PyUnicodeWriter_Finish(&self->writer); + PyObject *intermediate = PyUnicodeWriter_Finish(self->writer); + self->writer = NULL; self->state = STATE_REALIZED; if (intermediate == NULL) return NULL; - _PyUnicodeWriter_Init(&self->writer); - self->writer.overallocate = 1; - if (_PyUnicodeWriter_WriteStr(&self->writer, intermediate)) { + self->writer = PyUnicodeWriter_Create(0); + if (self->writer == NULL) { + Py_DECREF(intermediate); + return NULL; + } + if (PyUnicodeWriter_WriteStr(self->writer, intermediate)) { Py_DECREF(intermediate); return NULL; } @@ -155,7 +159,8 @@ realize(stringio *self) assert(self->state == STATE_ACCUMULATING); self->state = STATE_REALIZED; - intermediate = _PyUnicodeWriter_Finish(&self->writer); + intermediate = PyUnicodeWriter_Finish(self->writer); + self->writer = NULL; if (intermediate == NULL) return -1; @@ -196,7 +201,7 @@ write_str(stringio *self, PyObject *obj) } if (self->writenl) { PyObject *translated = PyUnicode_Replace( - decoded, &_Py_STR(newline), self->writenl, -1); + decoded, _Py_LATIN1_CHR('\n'), self->writenl, -1); Py_SETREF(decoded, translated); } if (decoded == NULL) @@ -217,7 +222,7 @@ write_str(stringio *self, PyObject *obj) if (self->state == STATE_ACCUMULATING) { if (self->string_size == self->pos) { - if (_PyUnicodeWriter_WriteStr(&self->writer, decoded)) + if (PyUnicodeWriter_WriteStr(self->writer, decoded)) goto fail; goto success; } @@ -353,7 +358,7 @@ _stringio_readline(stringio *self, Py_ssize_t limit) /* In case of overseek, return the empty string */ if (self->pos >= self->string_size) - return PyUnicode_New(0, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); start = self->buf + self->pos; if (limit < 0 || limit > self->string_size - self->pos) @@ -437,7 +442,7 @@ stringio_iternext(stringio *self) /*[clinic input] @critical_section _io.StringIO.truncate - pos as size: Py_ssize_t(accept={int, NoneType}, c_default="self->pos") = None + pos as size: Py_ssize_t(accept={int, NoneType}, c_default="((stringio *)self)->pos") = None / Truncate size to pos. @@ -449,7 +454,7 @@ Returns the new absolute position. static PyObject * _io_StringIO_truncate_impl(stringio *self, Py_ssize_t size) -/*[clinic end generated code: output=eb3aef8e06701365 input=461b872dce238452]*/ +/*[clinic end generated code: output=eb3aef8e06701365 input=fa8a6c98bb2ba780]*/ { CHECK_INITIALIZED(self); CHECK_CLOSED(self); @@ -577,7 +582,8 @@ _io_StringIO_close_impl(stringio *self) /* Free up some memory */ if (resize_buffer(self, 0) < 0) return NULL; - _PyUnicodeWriter_Dealloc(&self->writer); + PyUnicodeWriter_Discard(self->writer); + self->writer = NULL; Py_CLEAR(self->readnl); Py_CLEAR(self->writenl); Py_CLEAR(self->decoder); @@ -615,7 +621,7 @@ stringio_dealloc(stringio *self) PyMem_Free(self->buf); self->buf = NULL; } - _PyUnicodeWriter_Dealloc(&self->writer); + PyUnicodeWriter_Discard(self->writer); (void)stringio_clear(self); if (self->weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *) self); @@ -699,7 +705,8 @@ _io_StringIO___init___impl(stringio *self, PyObject *value, self->ok = 0; - _PyUnicodeWriter_Dealloc(&self->writer); + PyUnicodeWriter_Discard(self->writer); + self->writer = NULL; Py_CLEAR(self->readnl); Py_CLEAR(self->writenl); Py_CLEAR(self->decoder); @@ -754,8 +761,10 @@ _io_StringIO___init___impl(stringio *self, PyObject *value, /* Empty stringio object, we can start by accumulating */ if (resize_buffer(self, 0) < 0) return -1; - _PyUnicodeWriter_Init(&self->writer); - self->writer.overallocate = 1; + self->writer = PyUnicodeWriter_Create(0); + if (self->writer == NULL) { + return -1; + } self->state = STATE_ACCUMULATING; } self->pos = 0; @@ -908,23 +917,25 @@ _io_StringIO___setstate___impl(stringio *self, PyObject *state) once by __init__. So we do not take any chance and replace object's buffer completely. */ { - PyObject *item; - Py_UCS4 *buf; - Py_ssize_t bufsize; - - item = PyTuple_GET_ITEM(state, 0); - buf = PyUnicode_AsUCS4Copy(item); - if (buf == NULL) - return NULL; - bufsize = PyUnicode_GET_LENGTH(item); + PyObject *item = PyTuple_GET_ITEM(state, 0); + if (PyUnicode_Check(item)) { + Py_UCS4 *buf = PyUnicode_AsUCS4Copy(item); + if (buf == NULL) + return NULL; + Py_ssize_t bufsize = PyUnicode_GET_LENGTH(item); - if (resize_buffer(self, bufsize) < 0) { + if (resize_buffer(self, bufsize) < 0) { + PyMem_Free(buf); + return NULL; + } + memcpy(self->buf, buf, bufsize * sizeof(Py_UCS4)); PyMem_Free(buf); - return NULL; + self->string_size = bufsize; + } + else { + assert(item == Py_None); + self->string_size = 0; } - memcpy(self->buf, buf, bufsize * sizeof(Py_UCS4)); - PyMem_Free(buf); - self->string_size = bufsize; } /* Set carefully the position value. Alternatively, we could use the seek diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index a3239ec0f52960..791ee070401fe5 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -559,7 +559,7 @@ _io_IncrementalNewlineDecoder_getstate_impl(nldecoder_object *self) Py_DECREF(state); } else { - buffer = PyBytes_FromString(""); + buffer = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); flag = 0; } flag <<= 1; @@ -1719,16 +1719,26 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text) bytes_len = PyBytes_GET_SIZE(b); } - if (self->pending_bytes == NULL) { - self->pending_bytes_count = 0; - self->pending_bytes = b; - } - else if (self->pending_bytes_count + bytes_len > self->chunk_size) { - // Prevent to concatenate more than chunk_size data. - if (_textiowrapper_writeflush(self) < 0) { - Py_DECREF(b); - return NULL; + // We should avoid concatenating huge data. + // Flush the buffer before adding b to the buffer if b is not small. + // https://github.com/python/cpython/issues/87426 + if (bytes_len >= self->chunk_size) { + // _textiowrapper_writeflush() calls buffer.write(). + // self->pending_bytes can be appended during buffer->write() + // or other thread. + // We need to loop until buffer becomes empty. + // https://github.com/python/cpython/issues/118138 + // https://github.com/python/cpython/issues/119506 + while (self->pending_bytes != NULL) { + if (_textiowrapper_writeflush(self) < 0) { + Py_DECREF(b); + return NULL; + } } + } + + if (self->pending_bytes == NULL) { + assert(self->pending_bytes_count == 0); self->pending_bytes = b; } else if (!PyList_CheckExact(self->pending_bytes)) { @@ -1737,6 +1747,9 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text) Py_DECREF(b); return NULL; } + // Since Python 3.12, allocating GC object won't trigger GC and release + // GIL. See https://github.com/python/cpython/issues/97922 + assert(!PyList_CheckExact(self->pending_bytes)); PyList_SET_ITEM(list, 0, self->pending_bytes); PyList_SET_ITEM(list, 1, b); self->pending_bytes = list; @@ -1793,7 +1806,7 @@ textiowrapper_get_decoded_chars(textio *self, Py_ssize_t n) Py_ssize_t avail; if (self->decoded_chars == NULL) - return PyUnicode_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); /* decoded_chars is guaranteed to be "ready". */ avail = (PyUnicode_GET_LENGTH(self->decoded_chars) @@ -1979,6 +1992,12 @@ _io_TextIOWrapper_read_impl(textio *self, Py_ssize_t n) if (bytes == NULL) goto fail; + if (bytes == Py_None){ + Py_DECREF(bytes); + PyErr_SetString(PyExc_BlockingIOError, "Read returned None."); + return NULL; + } + _PyIO_State *state = self->state; if (Py_IS_TYPE(self->decoder, state->PyIncrementalNewlineDecoder_Type)) decoded = _PyIncrementalNewlineDecoder_decode(self->decoder, @@ -3337,8 +3356,8 @@ static PyMethodDef textiowrapper_methods[] = { _IO_TEXTIOWRAPPER_TELL_METHODDEF _IO_TEXTIOWRAPPER_TRUNCATE_METHODDEF - {"__reduce__", _PyIOBase_cannot_pickle, METH_VARARGS}, - {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_VARARGS}, + {"__reduce__", _PyIOBase_cannot_pickle, METH_NOARGS}, + {"__reduce_ex__", _PyIOBase_cannot_pickle, METH_O}, {NULL, NULL} }; diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 54e15555417287..3fa0301e337991 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -135,19 +135,67 @@ char _PyIO_get_console_type(PyObject *path_or_fd) { } static DWORD -_find_last_utf8_boundary(const char *buf, DWORD len) +_find_last_utf8_boundary(const unsigned char *buf, DWORD len) { - /* This function never returns 0, returns the original len instead */ - DWORD count = 1; - if (len == 0 || (buf[len - 1] & 0x80) == 0) { - return len; - } - for (;; count++) { - if (count > 3 || count >= len) { + for (DWORD count = 1; count < 4 && count <= len; count++) { + unsigned char c = buf[len - count]; + if (c < 0x80) { + /* No starting byte found. */ return len; } - if ((buf[len - count] & 0xc0) != 0x80) { - return len - count; + if (c >= 0xc0) { + if (c < 0xe0 /* 2-bytes sequence */ ? count < 2 : + c < 0xf0 /* 3-bytes sequence */ ? count < 3 : + c < 0xf8 /* 4-bytes sequence */) + { + /* Incomplete multibyte sequence. */ + return len - count; + } + /* Either complete or invalid sequence. */ + return len; + } + } + /* Either complete 4-bytes sequence or invalid sequence. */ + return len; +} + +/* Find the number of UTF-8 bytes that corresponds to the specified number of + * wchars. + * I.e. find x <= len so that MultiByteToWideChar(CP_UTF8, 0, s, x, NULL, 0) == n. + * + * WideCharToMultiByte() cannot be used for this, because the UTF-8 -> wchar + * conversion is not reversible (invalid UTF-8 byte produces \ufffd which + * will be converted back to 3-bytes UTF-8 sequence \xef\xbf\xbd). + * So we need to use binary search. + */ +static DWORD +_wchar_to_utf8_count(const unsigned char *s, DWORD len, DWORD n) +{ + DWORD start = 0; + while (1) { + DWORD mid = 0; + for (DWORD i = len / 2; i <= len; i++) { + mid = _find_last_utf8_boundary(s, i); + if (mid != 0) { + break; + } + /* The middle could split the first multibytes sequence. */ + } + if (mid == len) { + return start + len; + } + if (mid == 0) { + mid = len > 1 ? len - 1 : 1; + } + DWORD wlen = MultiByteToWideChar(CP_UTF8, 0, s, mid, NULL, 0); + if (wlen <= n) { + s += mid; + start += mid; + len -= mid; + n -= wlen; + } + else { + len = mid; } } } @@ -298,6 +346,13 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, self->fd = -1; } + if (PyBool_Check(nameobj)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "bool is used as a file descriptor", 1)) + { + return -1; + } + } fd = PyLong_AsInt(nameobj); if (fd < 0) { if (!PyErr_Occurred()) { @@ -556,8 +611,10 @@ read_console_w(HANDLE handle, DWORD maxlen, DWORD *readlen) { int err = 0, sig = 0; wchar_t *buf = (wchar_t*)PyMem_Malloc(maxlen * sizeof(wchar_t)); - if (!buf) + if (!buf) { + PyErr_NoMemory(); goto error; + } *readlen = 0; @@ -615,6 +672,7 @@ read_console_w(HANDLE handle, DWORD maxlen, DWORD *readlen) { Py_UNBLOCK_THREADS if (!newbuf) { sig = -1; + PyErr_NoMemory(); break; } buf = newbuf; @@ -638,8 +696,10 @@ read_console_w(HANDLE handle, DWORD maxlen, DWORD *readlen) { if (*readlen > 0 && buf[0] == L'\x1a') { PyMem_Free(buf); buf = (wchar_t *)PyMem_Malloc(sizeof(wchar_t)); - if (!buf) + if (!buf) { + PyErr_NoMemory(); goto error; + } buf[0] = L'\0'; *readlen = 0; } @@ -817,8 +877,10 @@ _io__WindowsConsoleIO_readall_impl(winconsoleio *self) bufsize = BUFSIZ; buf = (wchar_t*)PyMem_Malloc((bufsize + 1) * sizeof(wchar_t)); - if (buf == NULL) + if (buf == NULL) { + PyErr_NoMemory(); return NULL; + } while (1) { wchar_t *subbuf; @@ -840,6 +902,7 @@ _io__WindowsConsoleIO_readall_impl(winconsoleio *self) (bufsize + 1) * sizeof(wchar_t)); if (tmp == NULL) { PyMem_Free(buf); + PyErr_NoMemory(); return NULL; } buf = tmp; @@ -1015,43 +1078,49 @@ _io__WindowsConsoleIO_write_impl(winconsoleio *self, PyTypeObject *cls, len = (DWORD)b->len; Py_BEGIN_ALLOW_THREADS - wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, NULL, 0); - /* issue11395 there is an unspecified upper bound on how many bytes can be written at once. We cap at 32k - the caller will have to handle partial writes. Since we don't know how many input bytes are being ignored, we have to reduce and recalculate. */ - while (wlen > 32766 / sizeof(wchar_t)) { - len /= 2; + const DWORD max_wlen = 32766 / sizeof(wchar_t); + /* UTF-8 to wchar ratio is at most 3:1. */ + len = Py_MIN(len, max_wlen * 3); + while (1) { /* Fix for github issues gh-110913 and gh-82052. */ len = _find_last_utf8_boundary(b->buf, len); wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, NULL, 0); + if (wlen <= max_wlen) { + break; + } + len /= 2; } Py_END_ALLOW_THREADS - if (!wlen) - return PyErr_SetFromWindowsErr(0); + if (!wlen) { + return PyLong_FromLong(0); + } wbuf = (wchar_t*)PyMem_Malloc(wlen * sizeof(wchar_t)); + if (!wbuf) { + PyErr_NoMemory(); + return NULL; + } Py_BEGIN_ALLOW_THREADS wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, wbuf, wlen); if (wlen) { res = WriteConsoleW(handle, wbuf, wlen, &n, NULL); +#ifdef Py_DEBUG + if (res) { +#else if (res && n < wlen) { +#endif /* Wrote fewer characters than expected, which means our * len value may be wrong. So recalculate it from the - * characters that were written. As this could potentially - * result in a different value, we also validate that value. + * characters that were written. */ - len = WideCharToMultiByte(CP_UTF8, 0, wbuf, n, - NULL, 0, NULL, NULL); - if (len) { - wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, - NULL, 0); - assert(wlen == len); - } + len = _wchar_to_utf8_count(b->buf, len, n); } } else res = 0; @@ -1121,6 +1190,7 @@ static PyMethodDef winconsoleio_methods[] = { _IO__WINDOWSCONSOLEIO_WRITABLE_METHODDEF _IO__WINDOWSCONSOLEIO_FILENO_METHODDEF _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF + {"_isatty_open_only", (PyCFunction)_io__WindowsConsoleIO_isatty, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_json.c b/Modules/_json.c index c55299899e77fe..31a5e935e13ad9 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -11,6 +11,7 @@ #include "Python.h" #include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_runtime.h" // _PyRuntime +#include "pycore_pyerrors.h" // _PyErr_FormatNote #include "pycore_global_strings.h" // _Py_ID() #include // bool @@ -85,11 +86,11 @@ encoder_dealloc(PyObject *self); static int encoder_clear(PyEncoderObject *self); static int -encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *seq, Py_ssize_t indent_level); +encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *seq, Py_ssize_t indent_level, PyObject *indent_cache); static int -encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *obj, Py_ssize_t indent_level); +encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *obj, Py_ssize_t indent_level, PyObject *indent_cache); static int -encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *dct, Py_ssize_t indent_level); +encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *dct, Py_ssize_t indent_level, PyObject *indent_cache); static PyObject * _encoded_const(PyObject *obj); static void @@ -352,6 +353,13 @@ _build_rval_index_tuple(PyObject *rval, Py_ssize_t idx) { return tpl; } +static inline int +_PyUnicodeWriter_IsEmpty(PyUnicodeWriter *writer_pub) +{ + _PyUnicodeWriter *writer = (_PyUnicodeWriter*)writer_pub; + return (writer->pos == 0); +} + static PyObject * scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next_end_ptr) { @@ -370,9 +378,10 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next const void *buf; int kind; - _PyUnicodeWriter writer; - _PyUnicodeWriter_Init(&writer); - writer.overallocate = 1; + PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); + if (writer == NULL) { + goto bail; + } len = PyUnicode_GET_LENGTH(pystr); buf = PyUnicode_DATA(pystr); @@ -403,11 +412,12 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next if (c == '"') { // Fast path for simple case. - if (writer.buffer == NULL) { + if (_PyUnicodeWriter_IsEmpty(writer)) { PyObject *ret = PyUnicode_Substring(pystr, end, next); if (ret == NULL) { goto bail; } + PyUnicodeWriter_Discard(writer); *next_end_ptr = next + 1;; return ret; } @@ -419,7 +429,7 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next /* Pick up this chunk if it's not zero length */ if (next != end) { - if (_PyUnicodeWriter_WriteSubstring(&writer, pystr, end, next) < 0) { + if (PyUnicodeWriter_WriteSubstring(writer, pystr, end, next) < 0) { goto bail; } } @@ -510,18 +520,18 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next end -= 6; } } - if (_PyUnicodeWriter_WriteChar(&writer, c) < 0) { + if (PyUnicodeWriter_WriteChar(writer, c) < 0) { goto bail; } } - rval = _PyUnicodeWriter_Finish(&writer); + rval = PyUnicodeWriter_Finish(writer); *next_end_ptr = end; return rval; bail: *next_end_ptr = -1; - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); return NULL; } @@ -1251,33 +1261,130 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *)s; } + +/* indent_cache is a list that contains intermixed values at even and odd + * positions: + * + * 2*k : '\n' + indent * (k + initial_indent_level) + * strings written after opening and before closing brackets + * 2*k-1 : item_separator + '\n' + indent * (k + initial_indent_level) + * strings written between items + * + * Its size is always an odd number. + */ +static PyObject * +create_indent_cache(PyEncoderObject *s, Py_ssize_t indent_level) +{ + PyObject *newline_indent = PyUnicode_FromOrdinal('\n'); + if (newline_indent != NULL && indent_level) { + PyUnicode_AppendAndDel(&newline_indent, + PySequence_Repeat(s->indent, indent_level)); + } + if (newline_indent == NULL) { + return NULL; + } + PyObject *indent_cache = PyList_New(1); + if (indent_cache == NULL) { + Py_DECREF(newline_indent); + return NULL; + } + PyList_SET_ITEM(indent_cache, 0, newline_indent); + return indent_cache; +} + +/* Extend indent_cache by adding values for the next level. + * It should have values for the indent_level-1 level before the call. + */ +static int +update_indent_cache(PyEncoderObject *s, + Py_ssize_t indent_level, PyObject *indent_cache) +{ + assert(indent_level * 2 == PyList_GET_SIZE(indent_cache) + 1); + assert(indent_level > 0); + PyObject *newline_indent = PyList_GET_ITEM(indent_cache, (indent_level - 1)*2); + newline_indent = PyUnicode_Concat(newline_indent, s->indent); + if (newline_indent == NULL) { + return -1; + } + PyObject *separator_indent = PyUnicode_Concat(s->item_separator, newline_indent); + if (separator_indent == NULL) { + Py_DECREF(newline_indent); + return -1; + } + + if (PyList_Append(indent_cache, separator_indent) < 0 || + PyList_Append(indent_cache, newline_indent) < 0) + { + Py_DECREF(separator_indent); + Py_DECREF(newline_indent); + return -1; + } + Py_DECREF(separator_indent); + Py_DECREF(newline_indent); + return 0; +} + +static PyObject * +get_item_separator(PyEncoderObject *s, + Py_ssize_t indent_level, PyObject *indent_cache) +{ + assert(indent_level > 0); + if (indent_level * 2 > PyList_GET_SIZE(indent_cache)) { + if (update_indent_cache(s, indent_level, indent_cache) < 0) { + return NULL; + } + } + assert(indent_level * 2 < PyList_GET_SIZE(indent_cache)); + return PyList_GET_ITEM(indent_cache, indent_level * 2 - 1); +} + +static int +write_newline_indent(PyUnicodeWriter *writer, + Py_ssize_t indent_level, PyObject *indent_cache) +{ + PyObject *newline_indent = PyList_GET_ITEM(indent_cache, indent_level * 2); + return PyUnicodeWriter_WriteStr(writer, newline_indent); +} + + static PyObject * encoder_call(PyEncoderObject *self, PyObject *args, PyObject *kwds) { /* Python callable interface to encode_listencode_obj */ static char *kwlist[] = {"obj", "_current_indent_level", NULL}; - PyObject *obj, *result; + PyObject *obj; Py_ssize_t indent_level; - _PyUnicodeWriter writer; if (!PyArg_ParseTupleAndKeywords(args, kwds, "On:_iterencode", kwlist, - &obj, &indent_level)) + &obj, &indent_level)) return NULL; - _PyUnicodeWriter_Init(&writer); - writer.overallocate = 1; + PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); + if (writer == NULL) { + return NULL; + } - if (encoder_listencode_obj(self, &writer, obj, indent_level)) { - _PyUnicodeWriter_Dealloc(&writer); + PyObject *indent_cache = NULL; + if (self->indent != Py_None) { + indent_cache = create_indent_cache(self, indent_level); + if (indent_cache == NULL) { + PyUnicodeWriter_Discard(writer); + return NULL; + } + } + if (encoder_listencode_obj(self, writer, obj, indent_level, indent_cache)) { + PyUnicodeWriter_Discard(writer); + Py_XDECREF(indent_cache); return NULL; } + Py_XDECREF(indent_cache); - result = PyTuple_New(1); - if (result == NULL || - PyTuple_SetItem(result, 0, _PyUnicodeWriter_Finish(&writer)) < 0) { - Py_XDECREF(result); + PyObject *str = PyUnicodeWriter_Finish(writer); + if (str == NULL) { return NULL; } + PyObject *result = PyTuple_Pack(1, str); + Py_DECREF(str); return result; } @@ -1305,7 +1412,7 @@ encoder_encode_float(PyEncoderObject *s, PyObject *obj) { /* Return the JSON representation of a PyFloat. */ double i = PyFloat_AS_DOUBLE(obj); - if (!Py_IS_FINITE(i)) { + if (!isfinite(i)) { if (!s->allow_nan) { PyErr_Format( PyExc_ValueError, @@ -1348,30 +1455,31 @@ encoder_encode_string(PyEncoderObject *s, PyObject *obj) } static int -_steal_accumulate(_PyUnicodeWriter *writer, PyObject *stolen) +_steal_accumulate(PyUnicodeWriter *writer, PyObject *stolen) { /* Append stolen and then decrement its reference count */ - int rval = _PyUnicodeWriter_WriteStr(writer, stolen); + int rval = PyUnicodeWriter_WriteStr(writer, stolen); Py_DECREF(stolen); return rval; } static int -encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, - PyObject *obj, Py_ssize_t indent_level) +encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, + PyObject *obj, + Py_ssize_t indent_level, PyObject *indent_cache) { /* Encode Python object obj to a JSON term */ PyObject *newobj; int rv; if (obj == Py_None) { - return _PyUnicodeWriter_WriteASCIIString(writer, "null", 4); + return PyUnicodeWriter_WriteUTF8(writer, "null", 4); } else if (obj == Py_True) { - return _PyUnicodeWriter_WriteASCIIString(writer, "true", 4); + return PyUnicodeWriter_WriteUTF8(writer, "true", 4); } else if (obj == Py_False) { - return _PyUnicodeWriter_WriteASCIIString(writer, "false", 5); + return PyUnicodeWriter_WriteUTF8(writer, "false", 5); } else if (PyUnicode_Check(obj)) { PyObject *encoded = encoder_encode_string(s, obj); @@ -1380,6 +1488,10 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, return _steal_accumulate(writer, encoded); } else if (PyLong_Check(obj)) { + if (PyLong_CheckExact(obj)) { + // Fast-path for exact integers + return PyUnicodeWriter_WriteRepr(writer, obj); + } PyObject *encoded = PyLong_Type.tp_repr(obj); if (encoded == NULL) return -1; @@ -1394,14 +1506,14 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, else if (PyList_Check(obj) || PyTuple_Check(obj)) { if (_Py_EnterRecursiveCall(" while encoding a JSON object")) return -1; - rv = encoder_listencode_list(s, writer, obj, indent_level); + rv = encoder_listencode_list(s, writer, obj, indent_level, indent_cache); _Py_LeaveRecursiveCall(); return rv; } else if (PyDict_Check(obj)) { if (_Py_EnterRecursiveCall(" while encoding a JSON object")) return -1; - rv = encoder_listencode_dict(s, writer, obj, indent_level); + rv = encoder_listencode_dict(s, writer, obj, indent_level, indent_cache); _Py_LeaveRecursiveCall(); return rv; } @@ -1435,11 +1547,12 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, Py_XDECREF(ident); return -1; } - rv = encoder_listencode_obj(s, writer, newobj, indent_level); + rv = encoder_listencode_obj(s, writer, newobj, indent_level, indent_cache); _Py_LeaveRecursiveCall(); Py_DECREF(newobj); if (rv) { + _PyErr_FormatNote("when serializing %T object", obj); Py_XDECREF(ident); return -1; } @@ -1455,8 +1568,10 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, } static int -encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *first, - PyObject *key, PyObject *value, Py_ssize_t indent_level) +encoder_encode_key_value(PyEncoderObject *s, PyUnicodeWriter *writer, bool *first, + PyObject *dct, PyObject *key, PyObject *value, + Py_ssize_t indent_level, PyObject *indent_cache, + PyObject *item_separator) { PyObject *keystr = NULL; PyObject *encoded; @@ -1493,7 +1608,7 @@ encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *fir *first = false; } else { - if (_PyUnicodeWriter_WriteStr(writer, s->item_separator) < 0) { + if (PyUnicodeWriter_WriteStr(writer, item_separator) < 0) { Py_DECREF(keystr); return -1; } @@ -1508,18 +1623,20 @@ encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *fir if (_steal_accumulate(writer, encoded) < 0) { return -1; } - if (_PyUnicodeWriter_WriteStr(writer, s->key_separator) < 0) { + if (PyUnicodeWriter_WriteStr(writer, s->key_separator) < 0) { return -1; } - if (encoder_listencode_obj(s, writer, value, indent_level) < 0) { + if (encoder_listencode_obj(s, writer, value, indent_level, indent_cache) < 0) { + _PyErr_FormatNote("when serializing %T item %R", dct, key); return -1; } return 0; } static int -encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, - PyObject *dct, Py_ssize_t indent_level) +encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, + PyObject *dct, + Py_ssize_t indent_level, PyObject *indent_cache) { /* Encode Python dict dct a JSON term */ PyObject *ident = NULL; @@ -1527,8 +1644,10 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *key, *value; bool first = true; - if (PyDict_GET_SIZE(dct) == 0) /* Fast path */ - return _PyUnicodeWriter_WriteASCIIString(writer, "{}", 2); + if (PyDict_GET_SIZE(dct) == 0) { + /* Fast path */ + return PyUnicodeWriter_WriteUTF8(writer, "{}", 2); + } if (s->markers != Py_None) { int has_key; @@ -1546,17 +1665,19 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, } } - if (_PyUnicodeWriter_WriteChar(writer, '{')) + if (PyUnicodeWriter_WriteChar(writer, '{')) { goto bail; + } + PyObject *separator = s->item_separator; // borrowed reference if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) - separator = _item_separator + newline_indent - buf += newline_indent - */ + indent_level++; + separator = get_item_separator(s, indent_level, indent_cache); + if (separator == NULL || + write_newline_indent(writer, indent_level, indent_cache) < 0) + { + goto bail; + } } if (s->sort_keys || !PyDict_CheckExact(dct)) { @@ -1574,7 +1695,9 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, key = PyTuple_GET_ITEM(item, 0); value = PyTuple_GET_ITEM(item, 1); - if (encoder_encode_key_value(s, writer, &first, key, value, indent_level) < 0) + if (encoder_encode_key_value(s, writer, &first, dct, key, value, + indent_level, indent_cache, + separator) < 0) goto bail; } Py_CLEAR(items); @@ -1582,7 +1705,9 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, } else { Py_ssize_t pos = 0; while (PyDict_Next(dct, &pos, &key, &value)) { - if (encoder_encode_key_value(s, writer, &first, key, value, indent_level) < 0) + if (encoder_encode_key_value(s, writer, &first, dct, key, value, + indent_level, indent_cache, + separator) < 0) goto bail; } } @@ -1592,14 +1717,16 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, goto bail; Py_CLEAR(ident); } - /* TODO DOES NOT RUN; dead code if (s->indent != Py_None) { - indent_level -= 1; + indent_level--; + if (write_newline_indent(writer, indent_level, indent_cache) < 0) { + goto bail; + } + } - yield '\n' + (' ' * (_indent * _current_indent_level)) - }*/ - if (_PyUnicodeWriter_WriteChar(writer, '}')) + if (PyUnicodeWriter_WriteChar(writer, '}')) { goto bail; + } return 0; bail: @@ -1609,8 +1736,9 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, } static int -encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, - PyObject *seq, Py_ssize_t indent_level) +encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer, + PyObject *seq, + Py_ssize_t indent_level, PyObject *indent_cache) { PyObject *ident = NULL; PyObject *s_fast = NULL; @@ -1622,7 +1750,7 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, return -1; if (PySequence_Fast_GET_SIZE(s_fast) == 0) { Py_DECREF(s_fast); - return _PyUnicodeWriter_WriteASCIIString(writer, "[]", 2); + return PyUnicodeWriter_WriteUTF8(writer, "[]", 2); } if (s->markers != Py_None) { @@ -1641,25 +1769,30 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, } } - if (_PyUnicodeWriter_WriteChar(writer, '[')) + if (PyUnicodeWriter_WriteChar(writer, '[')) { goto bail; + } + + PyObject *separator = s->item_separator; // borrowed reference if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) - separator = _item_separator + newline_indent - buf += newline_indent - */ + indent_level++; + separator = get_item_separator(s, indent_level, indent_cache); + if (separator == NULL || + write_newline_indent(writer, indent_level, indent_cache) < 0) + { + goto bail; + } } for (i = 0; i < PySequence_Fast_GET_SIZE(s_fast); i++) { PyObject *obj = PySequence_Fast_GET_ITEM(s_fast, i); if (i) { - if (_PyUnicodeWriter_WriteStr(writer, s->item_separator)) + if (PyUnicodeWriter_WriteStr(writer, separator) < 0) goto bail; } - if (encoder_listencode_obj(s, writer, obj, indent_level)) + if (encoder_listencode_obj(s, writer, obj, indent_level, indent_cache)) { + _PyErr_FormatNote("when serializing %T item %zd", seq, i); goto bail; + } } if (ident != NULL) { if (PyDict_DelItem(s->markers, ident)) @@ -1667,14 +1800,16 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, Py_CLEAR(ident); } - /* TODO: DOES NOT RUN if (s->indent != Py_None) { - indent_level -= 1; + indent_level--; + if (write_newline_indent(writer, indent_level, indent_cache) < 0) { + goto bail; + } + } - yield '\n' + (' ' * (_indent * _current_indent_level)) - }*/ - if (_PyUnicodeWriter_WriteChar(writer, ']')) + if (PyUnicodeWriter_WriteChar(writer, ']')) { goto bail; + } Py_DECREF(s_fast); return 0; @@ -1721,7 +1856,7 @@ encoder_clear(PyEncoderObject *self) return 0; } -PyDoc_STRVAR(encoder_doc, "_iterencode(obj, _current_indent_level) -> iterable"); +PyDoc_STRVAR(encoder_doc, "Encoder(markers, default, encoder, indent, key_separator, item_separator, sort_keys, skipkeys, allow_nan)"); static PyType_Slot PyEncoderType_slots[] = { {Py_tp_doc, (void *)encoder_doc}, @@ -1780,6 +1915,7 @@ _json_exec(PyObject *module) static PyModuleDef_Slot _json_slots[] = { {Py_mod_exec, _json_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index fe8e4c5e30035b..876627bcf5fad8 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -52,7 +52,7 @@ module _locale [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=ed98569b726feada]*/ -/* support functions for formatting floating point numbers */ +/* support functions for formatting floating-point numbers */ /* the grouping is terminated by either 0 or CHAR_MAX */ static PyObject* @@ -144,6 +144,17 @@ locale_is_ascii(const char *str) return (strlen(str) == 1 && ((unsigned char)str[0]) <= 127); } +static int +is_all_ascii(const char *str) +{ + for (; *str; str++) { + if ((unsigned char)*str > 127) { + return 0; + } + } + return 1; +} + static int locale_decode_monetary(PyObject *dict, struct lconv *lc) { @@ -478,113 +489,184 @@ _locale__getdefaultlocale_impl(PyObject *module) #endif #ifdef HAVE_LANGINFO_H -#define LANGINFO(X) {#X, X} +#define LANGINFO(X, Y) {#X, X, Y} static struct langinfo_constant{ - char* name; + const char *name; int value; + int category; } langinfo_constants[] = { /* These constants should exist on any langinfo implementation */ - LANGINFO(DAY_1), - LANGINFO(DAY_2), - LANGINFO(DAY_3), - LANGINFO(DAY_4), - LANGINFO(DAY_5), - LANGINFO(DAY_6), - LANGINFO(DAY_7), - - LANGINFO(ABDAY_1), - LANGINFO(ABDAY_2), - LANGINFO(ABDAY_3), - LANGINFO(ABDAY_4), - LANGINFO(ABDAY_5), - LANGINFO(ABDAY_6), - LANGINFO(ABDAY_7), - - LANGINFO(MON_1), - LANGINFO(MON_2), - LANGINFO(MON_3), - LANGINFO(MON_4), - LANGINFO(MON_5), - LANGINFO(MON_6), - LANGINFO(MON_7), - LANGINFO(MON_8), - LANGINFO(MON_9), - LANGINFO(MON_10), - LANGINFO(MON_11), - LANGINFO(MON_12), - - LANGINFO(ABMON_1), - LANGINFO(ABMON_2), - LANGINFO(ABMON_3), - LANGINFO(ABMON_4), - LANGINFO(ABMON_5), - LANGINFO(ABMON_6), - LANGINFO(ABMON_7), - LANGINFO(ABMON_8), - LANGINFO(ABMON_9), - LANGINFO(ABMON_10), - LANGINFO(ABMON_11), - LANGINFO(ABMON_12), + LANGINFO(DAY_1, LC_TIME), + LANGINFO(DAY_2, LC_TIME), + LANGINFO(DAY_3, LC_TIME), + LANGINFO(DAY_4, LC_TIME), + LANGINFO(DAY_5, LC_TIME), + LANGINFO(DAY_6, LC_TIME), + LANGINFO(DAY_7, LC_TIME), + + LANGINFO(ABDAY_1, LC_TIME), + LANGINFO(ABDAY_2, LC_TIME), + LANGINFO(ABDAY_3, LC_TIME), + LANGINFO(ABDAY_4, LC_TIME), + LANGINFO(ABDAY_5, LC_TIME), + LANGINFO(ABDAY_6, LC_TIME), + LANGINFO(ABDAY_7, LC_TIME), + + LANGINFO(MON_1, LC_TIME), + LANGINFO(MON_2, LC_TIME), + LANGINFO(MON_3, LC_TIME), + LANGINFO(MON_4, LC_TIME), + LANGINFO(MON_5, LC_TIME), + LANGINFO(MON_6, LC_TIME), + LANGINFO(MON_7, LC_TIME), + LANGINFO(MON_8, LC_TIME), + LANGINFO(MON_9, LC_TIME), + LANGINFO(MON_10, LC_TIME), + LANGINFO(MON_11, LC_TIME), + LANGINFO(MON_12, LC_TIME), + + LANGINFO(ABMON_1, LC_TIME), + LANGINFO(ABMON_2, LC_TIME), + LANGINFO(ABMON_3, LC_TIME), + LANGINFO(ABMON_4, LC_TIME), + LANGINFO(ABMON_5, LC_TIME), + LANGINFO(ABMON_6, LC_TIME), + LANGINFO(ABMON_7, LC_TIME), + LANGINFO(ABMON_8, LC_TIME), + LANGINFO(ABMON_9, LC_TIME), + LANGINFO(ABMON_10, LC_TIME), + LANGINFO(ABMON_11, LC_TIME), + LANGINFO(ABMON_12, LC_TIME), #ifdef RADIXCHAR /* The following are not available with glibc 2.0 */ - LANGINFO(RADIXCHAR), - LANGINFO(THOUSEP), + LANGINFO(RADIXCHAR, LC_NUMERIC), + LANGINFO(THOUSEP, LC_NUMERIC), /* YESSTR and NOSTR are deprecated in glibc, since they are a special case of message translation, which should be rather done using gettext. So we don't expose it to Python in the first place. - LANGINFO(YESSTR), - LANGINFO(NOSTR), + LANGINFO(YESSTR, LC_MESSAGES), + LANGINFO(NOSTR, LC_MESSAGES), */ - LANGINFO(CRNCYSTR), + LANGINFO(CRNCYSTR, LC_MONETARY), #endif - LANGINFO(D_T_FMT), - LANGINFO(D_FMT), - LANGINFO(T_FMT), - LANGINFO(AM_STR), - LANGINFO(PM_STR), + LANGINFO(D_T_FMT, LC_TIME), + LANGINFO(D_FMT, LC_TIME), + LANGINFO(T_FMT, LC_TIME), + LANGINFO(AM_STR, LC_TIME), + LANGINFO(PM_STR, LC_TIME), /* The following constants are available only with XPG4, but... OpenBSD doesn't have CODESET but has T_FMT_AMPM, and doesn't have a few of the others. Solution: ifdef-test them all. */ #ifdef CODESET - LANGINFO(CODESET), + LANGINFO(CODESET, LC_CTYPE), #endif #ifdef T_FMT_AMPM - LANGINFO(T_FMT_AMPM), + LANGINFO(T_FMT_AMPM, LC_TIME), #endif #ifdef ERA - LANGINFO(ERA), + LANGINFO(ERA, LC_TIME), #endif #ifdef ERA_D_FMT - LANGINFO(ERA_D_FMT), + LANGINFO(ERA_D_FMT, LC_TIME), #endif #ifdef ERA_D_T_FMT - LANGINFO(ERA_D_T_FMT), + LANGINFO(ERA_D_T_FMT, LC_TIME), #endif #ifdef ERA_T_FMT - LANGINFO(ERA_T_FMT), + LANGINFO(ERA_T_FMT, LC_TIME), #endif #ifdef ALT_DIGITS - LANGINFO(ALT_DIGITS), + LANGINFO(ALT_DIGITS, LC_TIME), #endif #ifdef YESEXPR - LANGINFO(YESEXPR), + LANGINFO(YESEXPR, LC_MESSAGES), #endif #ifdef NOEXPR - LANGINFO(NOEXPR), + LANGINFO(NOEXPR, LC_MESSAGES), #endif #ifdef _DATE_FMT /* This is not available in all glibc versions that have CODESET. */ - LANGINFO(_DATE_FMT), + LANGINFO(_DATE_FMT, LC_TIME), #endif - {0, 0} + {0, 0, 0} }; +/* Temporary make the LC_CTYPE locale to be the same as + * the locale of the specified category. */ +static int +change_locale(int category, char **oldloc) +{ + /* Keep a copy of the LC_CTYPE locale */ + *oldloc = setlocale(LC_CTYPE, NULL); + if (!*oldloc) { + PyErr_SetString(PyExc_RuntimeError, "faild to get LC_CTYPE locale"); + return -1; + } + *oldloc = _PyMem_Strdup(*oldloc); + if (!*oldloc) { + PyErr_NoMemory(); + return -1; + } + + /* Set a new locale if it is different. */ + char *loc = setlocale(category, NULL); + if (loc == NULL || strcmp(loc, *oldloc) == 0) { + PyMem_Free(*oldloc); + *oldloc = NULL; + return 0; + } + + setlocale(LC_CTYPE, loc); + return 1; +} + +/* Restore the old LC_CTYPE locale. */ +static void +restore_locale(char *oldloc) +{ + if (oldloc != NULL) { + setlocale(LC_CTYPE, oldloc); + PyMem_Free(oldloc); + } +} + +#ifdef __GLIBC__ +#if defined(ALT_DIGITS) || defined(ERA) +static PyObject * +decode_strings(const char *result, size_t max_count) +{ + /* Convert a sequence of NUL-separated C strings to a Python string + * containing semicolon separated items. */ + size_t i = 0; + size_t count = 0; + for (; count < max_count && result[i]; count++) { + i += strlen(result + i) + 1; + } + char *buf = PyMem_Malloc(i); + if (buf == NULL) { + PyErr_NoMemory(); + return NULL; + } + memcpy(buf, result, i); + /* Replace all NULs with semicolons. */ + i = 0; + while (--count) { + i += strlen(buf + i); + buf[i++] = ';'; + } + PyObject *pyresult = PyUnicode_DecodeLocale(buf, NULL); + PyMem_Free(buf); + return pyresult; +} +#endif +#endif + /*[clinic input] _locale.nl_langinfo @@ -602,14 +684,44 @@ _locale_nl_langinfo_impl(PyObject *module, int item) /* Check whether this is a supported constant. GNU libc sometimes returns numeric values in the char* return value, which would crash PyUnicode_FromString. */ - for (i = 0; langinfo_constants[i].name; i++) + for (i = 0; langinfo_constants[i].name; i++) { if (langinfo_constants[i].value == item) { /* Check NULL as a workaround for GNU libc's returning NULL instead of an empty string for nl_langinfo(ERA). */ const char *result = nl_langinfo(item); result = result != NULL ? result : ""; - return PyUnicode_DecodeLocale(result, NULL); + char *oldloc = NULL; + if (langinfo_constants[i].category != LC_CTYPE + && !is_all_ascii(result) + && change_locale(langinfo_constants[i].category, &oldloc) < 0) + { + return NULL; + } + PyObject *pyresult; +#ifdef __GLIBC__ + /* According to the POSIX specification the result must be + * a sequence of semicolon-separated strings. + * But in Glibc they are NUL-separated. */ +#ifdef ALT_DIGITS + if (item == ALT_DIGITS && *result) { + pyresult = decode_strings(result, 100); + } + else +#endif +#ifdef ERA + if (item == ERA && *result) { + pyresult = decode_strings(result, SIZE_MAX); + } + else +#endif +#endif + { + pyresult = PyUnicode_DecodeLocale(result, NULL); + } + restore_locale(oldloc); + return pyresult; } + } PyErr_SetString(PyExc_ValueError, "unsupported langinfo constant"); return NULL; } @@ -860,6 +972,7 @@ _locale_exec(PyObject *module) static struct PyModuleDef_Slot _locale_slots[] = { {Py_mod_exec, _locale_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; @@ -880,9 +993,9 @@ locale_clear(PyObject *module) } static void -locale_free(PyObject *module) +locale_free(void *module) { - locale_clear(module); + locale_clear((PyObject*)module); } static struct PyModuleDef _localemodule = { @@ -894,7 +1007,7 @@ static struct PyModuleDef _localemodule = { _locale_slots, locale_traverse, locale_clear, - (freefunc)locale_free, + locale_free, }; PyMODINIT_FUNC diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index a76c3dea555783..51ad9fc7da8492 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -59,6 +59,7 @@ typedef struct { #define POF_ENABLED 0x001 #define POF_SUBCALLS 0x002 #define POF_BUILTINS 0x004 +#define POF_EXT_TIMER 0x008 #define POF_NOMEMORY 0x100 /*[clinic input] @@ -87,7 +88,14 @@ _lsprof_get_state(PyObject *module) static PyTime_t CallExternalTimer(ProfilerObject *pObj) { - PyObject *o = _PyObject_CallNoArgs(pObj->externalTimer); + PyObject *o = NULL; + + // External timer can do arbitrary things so we need a flag to prevent + // horrible things to happen + pObj->flags |= POF_EXT_TIMER; + o = _PyObject_CallNoArgs(pObj->externalTimer); + pObj->flags &= ~POF_EXT_TIMER; + if (o == NULL) { PyErr_WriteUnraisable(pObj->externalTimer); return 0; @@ -121,7 +129,9 @@ call_timer(ProfilerObject *pObj) return CallExternalTimer(pObj); } else { - return _PyTime_PerfCounterUnchecked(); + PyTime_t t; + (void)PyTime_PerfCounterRaw(&t); + return t; } } @@ -175,8 +185,7 @@ normalizeUserObj(PyObject *obj) PyObject *modname = fn->m_module; if (name != NULL) { - PyObject *mo = _PyType_Lookup(Py_TYPE(self), name); - Py_XINCREF(mo); + PyObject *mo = _PyType_LookupRef(Py_TYPE(self), name); Py_DECREF(name); if (mo != NULL) { PyObject *res = PyObject_Repr(mo); @@ -597,17 +606,42 @@ setBuiltins(ProfilerObject *pObj, int nvalue) return 0; } -PyObject* pystart_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +/*[clinic input] +_lsprof.Profiler._pystart_callback + + code: object + instruction_offset: object + / + +[clinic start generated code]*/ + +static PyObject * +_lsprof_Profiler__pystart_callback_impl(ProfilerObject *self, PyObject *code, + PyObject *instruction_offset) +/*[clinic end generated code: output=5fec8b7ad5ed25e8 input=b166e6953c579cda]*/ { - PyObject* code = args[0]; - ptrace_enter_call((PyObject*)self, (void *)code, (PyObject *)code); + ptrace_enter_call((PyObject*)self, (void *)code, code); Py_RETURN_NONE; } -PyObject* pyreturn_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +/*[clinic input] +_lsprof.Profiler._pyreturn_callback + + code: object + instruction_offset: object + retval: object + / + +[clinic start generated code]*/ + +static PyObject * +_lsprof_Profiler__pyreturn_callback_impl(ProfilerObject *self, + PyObject *code, + PyObject *instruction_offset, + PyObject *retval) +/*[clinic end generated code: output=9e2f6fc1b882c51e input=667ffaeb2fa6fd1f]*/ { - PyObject* code = args[0]; ptrace_leave_call((PyObject*)self, (void *)code); Py_RETURN_NONE; @@ -640,12 +674,24 @@ PyObject* get_cfunc_from_callable(PyObject* callable, PyObject* self_arg, PyObje return NULL; } -PyObject* ccall_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +/*[clinic input] +_lsprof.Profiler._ccall_callback + + code: object + instruction_offset: object + callable: object + self_arg: object + / + +[clinic start generated code]*/ + +static PyObject * +_lsprof_Profiler__ccall_callback_impl(ProfilerObject *self, PyObject *code, + PyObject *instruction_offset, + PyObject *callable, PyObject *self_arg) +/*[clinic end generated code: output=152db83cabd18cad input=0e66687cfb95c001]*/ { if (self->flags & POF_BUILTINS) { - PyObject* callable = args[2]; - PyObject* self_arg = args[3]; - PyObject* cfunc = get_cfunc_from_callable(callable, self_arg, self->missing); if (cfunc) { @@ -658,12 +704,25 @@ PyObject* ccall_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t Py_RETURN_NONE; } -PyObject* creturn_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +/*[clinic input] +_lsprof.Profiler._creturn_callback + + code: object + instruction_offset: object + callable: object + self_arg: object + / + +[clinic start generated code]*/ + +static PyObject * +_lsprof_Profiler__creturn_callback_impl(ProfilerObject *self, PyObject *code, + PyObject *instruction_offset, + PyObject *callable, + PyObject *self_arg) +/*[clinic end generated code: output=1e886dde8fed8fb0 input=b18afe023746923a]*/ { if (self->flags & POF_BUILTINS) { - PyObject* callable = args[2]; - PyObject* self_arg = args[3]; - PyObject* cfunc = get_cfunc_from_callable(callable, self_arg, self->missing); if (cfunc) { @@ -691,27 +750,27 @@ static const struct { {0, NULL} }; -PyDoc_STRVAR(enable_doc, "\ -enable(subcalls=True, builtins=True)\n\ -\n\ -Start collecting profiling information.\n\ -If 'subcalls' is True, also records for each function\n\ -statistics separated according to its current caller.\n\ -If 'builtins' is True, records the time spent in\n\ -built-in functions separately from their caller.\n\ -"); - -static PyObject* -profiler_enable(ProfilerObject *self, PyObject *args, PyObject *kwds) + +/*[clinic input] +_lsprof.Profiler.enable + + subcalls: bool = True + If True, also records for each function + statistics separated according to its current caller. + + builtins: bool = True + If True, records the time spent in + built-in functions separately from their caller. + +Start collecting profiling information. +[clinic start generated code]*/ + +static PyObject * +_lsprof_Profiler_enable_impl(ProfilerObject *self, int subcalls, + int builtins) +/*[clinic end generated code: output=1e747f9dc1edd571 input=9ab81405107ab7f1]*/ { - int subcalls = -1; - int builtins = -1; - static char *kwlist[] = {"subcalls", "builtins", 0}; int all_events = 0; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|pp:enable", - kwlist, &subcalls, &builtins)) - return NULL; if (setSubcalls(self, subcalls) < 0 || setBuiltins(self, builtins) < 0) { return NULL; } @@ -721,34 +780,47 @@ profiler_enable(ProfilerObject *self, PyObject *args, PyObject *kwds) return NULL; } - if (PyObject_CallMethod(monitoring, "use_tool_id", "is", self->tool_id, "cProfile") == NULL) { + PyObject *check = PyObject_CallMethod(monitoring, + "use_tool_id", "is", + self->tool_id, "cProfile"); + if (check == NULL) { PyErr_Format(PyExc_ValueError, "Another profiling tool is already active"); - Py_DECREF(monitoring); - return NULL; + goto error; } + Py_DECREF(check); for (int i = 0; callback_table[i].callback_method; i++) { + int event = (1 << callback_table[i].event); PyObject* callback = PyObject_GetAttrString((PyObject*)self, callback_table[i].callback_method); if (!callback) { - Py_DECREF(monitoring); - return NULL; + goto error; } - Py_XDECREF(PyObject_CallMethod(monitoring, "register_callback", "iiO", self->tool_id, - (1 << callback_table[i].event), - callback)); + PyObject *register_result = PyObject_CallMethod(monitoring, "register_callback", + "iiO", self->tool_id, + event, callback); Py_DECREF(callback); - all_events |= (1 << callback_table[i].event); + if (register_result == NULL) { + goto error; + } + Py_DECREF(register_result); + all_events |= event; } - if (!PyObject_CallMethod(monitoring, "set_events", "ii", self->tool_id, all_events)) { - Py_DECREF(monitoring); - return NULL; + PyObject *event_result = PyObject_CallMethod(monitoring, "set_events", "ii", + self->tool_id, all_events); + if (event_result == NULL) { + goto error; } + Py_DECREF(event_result); Py_DECREF(monitoring); self->flags |= POF_ENABLED; Py_RETURN_NONE; + +error: + Py_DECREF(monitoring); + return NULL; } static void @@ -767,15 +839,22 @@ flush_unmatched(ProfilerObject *pObj) } -PyDoc_STRVAR(disable_doc, "\ -disable()\n\ -\n\ -Stop collecting profiling information.\n\ -"); -static PyObject* -profiler_disable(ProfilerObject *self, PyObject* noarg) +/*[clinic input] +_lsprof.Profiler.disable + +Stop collecting profiling information. +[clinic start generated code]*/ + +static PyObject * +_lsprof_Profiler_disable_impl(ProfilerObject *self) +/*[clinic end generated code: output=838cffef7f651870 input=05700b3fc68d1f50]*/ { + if (self->flags & POF_EXT_TIMER) { + PyErr_SetString(PyExc_RuntimeError, + "cannot disable profiler in external timer"); + return NULL; + } if (self->flags & POF_ENABLED) { PyObject* result = NULL; PyObject* monitoring = _PyImport_GetModuleAttrString("sys", "monitoring"); @@ -820,16 +899,22 @@ profiler_disable(ProfilerObject *self, PyObject* noarg) Py_RETURN_NONE; } -PyDoc_STRVAR(clear_doc, "\ -clear()\n\ -\n\ -Clear all profiling information collected so far.\n\ -"); +/*[clinic input] +_lsprof.Profiler.clear -static PyObject* -profiler_clear(ProfilerObject *pObj, PyObject* noarg) +Clear all profiling information collected so far. +[clinic start generated code]*/ + +static PyObject * +_lsprof_Profiler_clear_impl(ProfilerObject *self) +/*[clinic end generated code: output=dd1c668fb84b1335 input=fbe1f88c28be4f98]*/ { - clearEntries(pObj); + if (self->flags & POF_EXT_TIMER) { + PyErr_SetString(PyExc_RuntimeError, + "cannot clear profiler in external timer"); + return NULL; + } + clearEntries(self); Py_RETURN_NONE; } @@ -837,6 +922,7 @@ static int profiler_traverse(ProfilerObject *op, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(op)); + Py_VISIT(op->externalTimer); return 0; } @@ -859,33 +945,40 @@ profiler_dealloc(ProfilerObject *op) Py_DECREF(tp); } +/*[clinic input] +_lsprof.Profiler.__init__ as profiler_init + + timer: object(c_default='NULL') = None + timeunit: double = 0.0 + subcalls: bool = True + builtins: bool = True + +Build a profiler object using the specified timer function. + +The default timer is a fast built-in one based on real time. +For custom timer functions returning integers, 'timeunit' can +be a float specifying a scale (that is, how long each integer unit +is, in seconds). +[clinic start generated code]*/ + static int -profiler_init(ProfilerObject *pObj, PyObject *args, PyObject *kw) +profiler_init_impl(ProfilerObject *self, PyObject *timer, double timeunit, + int subcalls, int builtins) +/*[clinic end generated code: output=ac523803ec9f9df2 input=8285ca746f96a414]*/ { - PyObject *timer = NULL; - double timeunit = 0.0; - int subcalls = 1; - int builtins = 1; - static char *kwlist[] = {"timer", "timeunit", - "subcalls", "builtins", 0}; - - if (!PyArg_ParseTupleAndKeywords(args, kw, "|Odpp:Profiler", kwlist, - &timer, &timeunit, - &subcalls, &builtins)) - return -1; - - if (setSubcalls(pObj, subcalls) < 0 || setBuiltins(pObj, builtins) < 0) + if (setSubcalls(self, subcalls) < 0 || setBuiltins(self, builtins) < 0) { return -1; - pObj->externalTimerUnit = timeunit; - Py_XSETREF(pObj->externalTimer, Py_XNewRef(timer)); - pObj->tool_id = PY_MONITORING_PROFILER_ID; + } + self->externalTimerUnit = timeunit; + Py_XSETREF(self->externalTimer, Py_XNewRef(timer)); + self->tool_id = PY_MONITORING_PROFILER_ID; PyObject* monitoring = _PyImport_GetModuleAttrString("sys", "monitoring"); if (!monitoring) { return -1; } - pObj->missing = PyObject_GetAttrString(monitoring, "MISSING"); - if (!pObj->missing) { + self->missing = PyObject_GetAttrString(monitoring, "MISSING"); + if (!self->missing) { Py_DECREF(monitoring); return -1; } @@ -895,35 +988,18 @@ profiler_init(ProfilerObject *pObj, PyObject *args, PyObject *kw) static PyMethodDef profiler_methods[] = { _LSPROF_PROFILER_GETSTATS_METHODDEF - {"enable", _PyCFunction_CAST(profiler_enable), - METH_VARARGS | METH_KEYWORDS, enable_doc}, - {"disable", (PyCFunction)profiler_disable, - METH_NOARGS, disable_doc}, - {"clear", (PyCFunction)profiler_clear, - METH_NOARGS, clear_doc}, - {"_pystart_callback", _PyCFunction_CAST(pystart_callback), - METH_FASTCALL, NULL}, - {"_pyreturn_callback", _PyCFunction_CAST(pyreturn_callback), - METH_FASTCALL, NULL}, - {"_ccall_callback", _PyCFunction_CAST(ccall_callback), - METH_FASTCALL, NULL}, - {"_creturn_callback", _PyCFunction_CAST(creturn_callback), - METH_FASTCALL, NULL}, + _LSPROF_PROFILER_ENABLE_METHODDEF + _LSPROF_PROFILER_DISABLE_METHODDEF + _LSPROF_PROFILER_CLEAR_METHODDEF + _LSPROF_PROFILER__PYSTART_CALLBACK_METHODDEF + _LSPROF_PROFILER__PYRETURN_CALLBACK_METHODDEF + _LSPROF_PROFILER__CCALL_CALLBACK_METHODDEF + _LSPROF_PROFILER__CRETURN_CALLBACK_METHODDEF {NULL, NULL} }; -PyDoc_STRVAR(profiler_doc, "\ -Profiler(timer=None, timeunit=None, subcalls=True, builtins=True)\n\ -\n\ - Builds a profiler object using the specified timer function.\n\ - The default timer is a fast built-in one based on real time.\n\ - For custom timer functions returning integers, timeunit can\n\ - be a float specifying a scale (i.e. how long each integer unit\n\ - is, in seconds).\n\ -"); - static PyType_Slot _lsprof_profiler_type_spec_slots[] = { - {Py_tp_doc, (void *)profiler_doc}, + {Py_tp_doc, (void *)profiler_init__doc__}, {Py_tp_methods, profiler_methods}, {Py_tp_dealloc, profiler_dealloc}, {Py_tp_init, profiler_init}, @@ -1006,6 +1082,7 @@ _lsprof_exec(PyObject *module) static PyModuleDef_Slot _lsprofslots[] = { {Py_mod_exec, _lsprof_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index f6bfbfa62687b8..97f3a8f03da9a8 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -1604,6 +1604,7 @@ static PyMethodDef lzma_methods[] = { static PyModuleDef_Slot lzma_slots[] = { {Py_mod_exec, lzma_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_multiprocessing/clinic/posixshmem.c.h b/Modules/_multiprocessing/clinic/posixshmem.c.h index 1b894ea4c67adc..a545ff4d80f067 100644 --- a/Modules/_multiprocessing/clinic/posixshmem.c.h +++ b/Modules/_multiprocessing/clinic/posixshmem.c.h @@ -45,7 +45,7 @@ _posixshmem_shm_open(PyObject *module, PyObject *args, PyObject *kwargs) #if defined(HAVE_SHM_UNLINK) PyDoc_STRVAR(_posixshmem_shm_unlink__doc__, -"shm_unlink($module, /, path)\n" +"shm_unlink($module, path, /)\n" "--\n" "\n" "Remove a shared memory object (similar to unlink()).\n" @@ -55,21 +55,22 @@ PyDoc_STRVAR(_posixshmem_shm_unlink__doc__, "region."); #define _POSIXSHMEM_SHM_UNLINK_METHODDEF \ - {"shm_unlink", (PyCFunction)(void(*)(void))_posixshmem_shm_unlink, METH_VARARGS|METH_KEYWORDS, _posixshmem_shm_unlink__doc__}, + {"shm_unlink", (PyCFunction)_posixshmem_shm_unlink, METH_O, _posixshmem_shm_unlink__doc__}, static PyObject * _posixshmem_shm_unlink_impl(PyObject *module, PyObject *path); static PyObject * -_posixshmem_shm_unlink(PyObject *module, PyObject *args, PyObject *kwargs) +_posixshmem_shm_unlink(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - static char *_keywords[] = {"path", NULL}; PyObject *path; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "U:shm_unlink", _keywords, - &path)) + if (!PyUnicode_Check(arg)) { + PyErr_Format(PyExc_TypeError, "shm_unlink() argument must be str, not %T", arg); goto exit; + } + path = arg; return_value = _posixshmem_shm_unlink_impl(module, path); exit: @@ -85,4 +86,4 @@ _posixshmem_shm_unlink(PyObject *module, PyObject *args, PyObject *kwargs) #ifndef _POSIXSHMEM_SHM_UNLINK_METHODDEF #define _POSIXSHMEM_SHM_UNLINK_METHODDEF #endif /* !defined(_POSIXSHMEM_SHM_UNLINK_METHODDEF) */ -/*[clinic end generated code: output=be0661dbed83ea23 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=74588a5abba6e36c input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/clinic/semaphore.c.h b/Modules/_multiprocessing/clinic/semaphore.c.h index 7c855113113c20..e789137ec1e013 100644 --- a/Modules/_multiprocessing/clinic/semaphore.c.h +++ b/Modules/_multiprocessing/clinic/semaphore.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() #if defined(HAVE_MP_SEMAPHORE) && defined(MS_WINDOWS) @@ -24,7 +25,7 @@ _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, PyObject *timeout_obj); static PyObject * -_multiprocessing_SemLock_acquire(SemLockObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_multiprocessing_SemLock_acquire(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -57,7 +58,8 @@ _multiprocessing_SemLock_acquire(SemLockObject *self, PyObject *const *args, Py_ int blocking = 1; PyObject *timeout_obj = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -75,7 +77,9 @@ _multiprocessing_SemLock_acquire(SemLockObject *self, PyObject *const *args, Py_ } timeout_obj = args[1]; skip_optional_pos: - return_value = _multiprocessing_SemLock_acquire_impl(self, blocking, timeout_obj); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock_acquire_impl((SemLockObject *)self, blocking, timeout_obj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -98,9 +102,15 @@ static PyObject * _multiprocessing_SemLock_release_impl(SemLockObject *self); static PyObject * -_multiprocessing_SemLock_release(SemLockObject *self, PyObject *Py_UNUSED(ignored)) +_multiprocessing_SemLock_release(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock_release_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock_release_impl((SemLockObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_MP_SEMAPHORE) && defined(MS_WINDOWS) */ @@ -121,7 +131,7 @@ _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, PyObject *timeout_obj); static PyObject * -_multiprocessing_SemLock_acquire(SemLockObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_multiprocessing_SemLock_acquire(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -154,7 +164,8 @@ _multiprocessing_SemLock_acquire(SemLockObject *self, PyObject *const *args, Py_ int blocking = 1; PyObject *timeout_obj = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -172,7 +183,9 @@ _multiprocessing_SemLock_acquire(SemLockObject *self, PyObject *const *args, Py_ } timeout_obj = args[1]; skip_optional_pos: - return_value = _multiprocessing_SemLock_acquire_impl(self, blocking, timeout_obj); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock_acquire_impl((SemLockObject *)self, blocking, timeout_obj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -195,9 +208,15 @@ static PyObject * _multiprocessing_SemLock_release_impl(SemLockObject *self); static PyObject * -_multiprocessing_SemLock_release(SemLockObject *self, PyObject *Py_UNUSED(ignored)) +_multiprocessing_SemLock_release(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock_release_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock_release_impl((SemLockObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_MP_SEMAPHORE) && !defined(MS_WINDOWS) */ @@ -246,7 +265,8 @@ _multiprocessing_SemLock(PyTypeObject *type, PyObject *args, PyObject *kwargs) const char *name; int unlink; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 5, 5, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 5, /*maxpos*/ 5, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -338,9 +358,15 @@ static PyObject * _multiprocessing_SemLock__count_impl(SemLockObject *self); static PyObject * -_multiprocessing_SemLock__count(SemLockObject *self, PyObject *Py_UNUSED(ignored)) +_multiprocessing_SemLock__count(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock__count_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock__count_impl((SemLockObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_MP_SEMAPHORE) */ @@ -360,9 +386,9 @@ static PyObject * _multiprocessing_SemLock__is_mine_impl(SemLockObject *self); static PyObject * -_multiprocessing_SemLock__is_mine(SemLockObject *self, PyObject *Py_UNUSED(ignored)) +_multiprocessing_SemLock__is_mine(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock__is_mine_impl(self); + return _multiprocessing_SemLock__is_mine_impl((SemLockObject *)self); } #endif /* defined(HAVE_MP_SEMAPHORE) */ @@ -382,9 +408,9 @@ static PyObject * _multiprocessing_SemLock__get_value_impl(SemLockObject *self); static PyObject * -_multiprocessing_SemLock__get_value(SemLockObject *self, PyObject *Py_UNUSED(ignored)) +_multiprocessing_SemLock__get_value(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock__get_value_impl(self); + return _multiprocessing_SemLock__get_value_impl((SemLockObject *)self); } #endif /* defined(HAVE_MP_SEMAPHORE) */ @@ -404,9 +430,9 @@ static PyObject * _multiprocessing_SemLock__is_zero_impl(SemLockObject *self); static PyObject * -_multiprocessing_SemLock__is_zero(SemLockObject *self, PyObject *Py_UNUSED(ignored)) +_multiprocessing_SemLock__is_zero(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock__is_zero_impl(self); + return _multiprocessing_SemLock__is_zero_impl((SemLockObject *)self); } #endif /* defined(HAVE_MP_SEMAPHORE) */ @@ -426,9 +452,9 @@ static PyObject * _multiprocessing_SemLock__after_fork_impl(SemLockObject *self); static PyObject * -_multiprocessing_SemLock__after_fork(SemLockObject *self, PyObject *Py_UNUSED(ignored)) +_multiprocessing_SemLock__after_fork(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock__after_fork_impl(self); + return _multiprocessing_SemLock__after_fork_impl((SemLockObject *)self); } #endif /* defined(HAVE_MP_SEMAPHORE) */ @@ -448,9 +474,15 @@ static PyObject * _multiprocessing_SemLock___enter___impl(SemLockObject *self); static PyObject * -_multiprocessing_SemLock___enter__(SemLockObject *self, PyObject *Py_UNUSED(ignored)) +_multiprocessing_SemLock___enter__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock___enter___impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock___enter___impl((SemLockObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_MP_SEMAPHORE) */ @@ -472,7 +504,7 @@ _multiprocessing_SemLock___exit___impl(SemLockObject *self, PyObject *exc_value, PyObject *exc_tb); static PyObject * -_multiprocessing_SemLock___exit__(SemLockObject *self, PyObject *const *args, Py_ssize_t nargs) +_multiprocessing_SemLock___exit__(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *exc_type = Py_None; @@ -495,7 +527,9 @@ _multiprocessing_SemLock___exit__(SemLockObject *self, PyObject *const *args, Py } exc_tb = args[2]; skip_optional: - return_value = _multiprocessing_SemLock___exit___impl(self, exc_type, exc_value, exc_tb); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock___exit___impl((SemLockObject *)self, exc_type, exc_value, exc_tb); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -542,4 +576,4 @@ _multiprocessing_SemLock___exit__(SemLockObject *self, PyObject *const *args, Py #ifndef _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #define _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #endif /* !defined(_MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF) */ -/*[clinic end generated code: output=d57992037e6770b6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e28d0fdbfefd1235 input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 1f6ab718a36984..cee8cf7b9a83c0 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -277,6 +277,7 @@ multiprocessing_exec(PyObject *module) static PyModuleDef_Slot multiprocessing_slots[] = { {Py_mod_exec, multiprocessing_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index 4ab15fa6573665..ab45e4136c7d46 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -2,11 +2,10 @@ posixshmem - A Python extension that provides shm_open() and shm_unlink() */ +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include @@ -78,6 +77,7 @@ _posixshmem_shm_open_impl(PyObject *module, PyObject *path, int flags, /*[clinic input] _posixshmem.shm_unlink path: unicode + / Remove a shared memory object (similar to unlink()). @@ -89,7 +89,7 @@ region. static PyObject * _posixshmem_shm_unlink_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=42f8b23d134b9ff5 input=8dc0f87143e3b300]*/ +/*[clinic end generated code: output=42f8b23d134b9ff5 input=298369d013dcad63]*/ { int rv; int async_err = 0; @@ -129,6 +129,7 @@ static PyMethodDef module_methods[ ] = { static PyModuleDef_Slot module_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_multiprocessing/semaphore.c b/Modules/_multiprocessing/semaphore.c index f8f2afda28d06d..036db2cd4c6c85 100644 --- a/Modules/_multiprocessing/semaphore.c +++ b/Modules/_multiprocessing/semaphore.c @@ -15,6 +15,7 @@ #ifdef HAVE_MP_SEMAPHORE +// These match the values in Lib/multiprocessing/synchronize.py enum { RECURSIVE_MUTEX, SEMAPHORE }; typedef struct { @@ -27,6 +28,8 @@ typedef struct { char *name; } SemLockObject; +#define _SemLockObject_CAST(op) ((SemLockObject *)(op)) + /*[python input] class SEM_HANDLE_converter(CConverter): type = "SEM_HANDLE" @@ -81,6 +84,7 @@ _GetSemaphoreValue(HANDLE handle, long *value) } /*[clinic input] +@critical_section _multiprocessing.SemLock.acquire block as blocking: bool = True @@ -92,7 +96,7 @@ Acquire the semaphore/lock. static PyObject * _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, PyObject *timeout_obj) -/*[clinic end generated code: output=f9998f0b6b0b0872 input=e5b45f5cbb775166]*/ +/*[clinic end generated code: output=f9998f0b6b0b0872 input=079ca779975f3ad6]*/ { double timeout; DWORD res, full_msecs, nhandles; @@ -172,6 +176,7 @@ _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, } /*[clinic input] +@critical_section _multiprocessing.SemLock.release Release the semaphore/lock. @@ -179,7 +184,7 @@ Release the semaphore/lock. static PyObject * _multiprocessing_SemLock_release_impl(SemLockObject *self) -/*[clinic end generated code: output=b22f53ba96b0d1db input=ba7e63a961885d3d]*/ +/*[clinic end generated code: output=b22f53ba96b0d1db input=9bd62d3645e7a531]*/ { if (self->kind == RECURSIVE_MUTEX) { if (!ISMINE(self)) { @@ -297,6 +302,7 @@ sem_timedwait_save(sem_t *sem, struct timespec *deadline, PyThreadState *_save) #endif /* !HAVE_SEM_TIMEDWAIT */ /*[clinic input] +@critical_section _multiprocessing.SemLock.acquire block as blocking: bool = True @@ -308,7 +314,7 @@ Acquire the semaphore/lock. static PyObject * _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, PyObject *timeout_obj) -/*[clinic end generated code: output=f9998f0b6b0b0872 input=e5b45f5cbb775166]*/ +/*[clinic end generated code: output=f9998f0b6b0b0872 input=079ca779975f3ad6]*/ { int res, err = 0; struct timespec deadline = {0}; @@ -382,6 +388,7 @@ _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, } /*[clinic input] +@critical_section _multiprocessing.SemLock.release Release the semaphore/lock. @@ -389,7 +396,7 @@ Release the semaphore/lock. static PyObject * _multiprocessing_SemLock_release_impl(SemLockObject *self) -/*[clinic end generated code: output=b22f53ba96b0d1db input=ba7e63a961885d3d]*/ +/*[clinic end generated code: output=b22f53ba96b0d1db input=9bd62d3645e7a531]*/ { if (self->kind == RECURSIVE_MUTEX) { if (!ISMINE(self)) { @@ -571,8 +578,9 @@ _multiprocessing_SemLock__rebuild_impl(PyTypeObject *type, SEM_HANDLE handle, } static void -semlock_dealloc(SemLockObject* self) +semlock_dealloc(PyObject *op) { + SemLockObject *self = _SemLockObject_CAST(op); PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); if (self->handle != SEM_FAILED) @@ -583,6 +591,7 @@ semlock_dealloc(SemLockObject* self) } /*[clinic input] +@critical_section _multiprocessing.SemLock._count Num of `acquire()`s minus num of `release()`s for this process. @@ -590,7 +599,7 @@ Num of `acquire()`s minus num of `release()`s for this process. static PyObject * _multiprocessing_SemLock__count_impl(SemLockObject *self) -/*[clinic end generated code: output=5ba8213900e517bb input=36fc59b1cd1025ab]*/ +/*[clinic end generated code: output=5ba8213900e517bb input=9fa6e0b321b16935]*/ { return PyLong_FromLong((long)self->count); } @@ -677,6 +686,7 @@ _multiprocessing_SemLock__after_fork_impl(SemLockObject *self) } /*[clinic input] +@critical_section _multiprocessing.SemLock.__enter__ Enter the semaphore/lock. @@ -684,12 +694,13 @@ Enter the semaphore/lock. static PyObject * _multiprocessing_SemLock___enter___impl(SemLockObject *self) -/*[clinic end generated code: output=beeb2f07c858511f input=c5e27d594284690b]*/ +/*[clinic end generated code: output=beeb2f07c858511f input=d35c9860992ee790]*/ { return _multiprocessing_SemLock_acquire_impl(self, 1, Py_None); } /*[clinic input] +@critical_section _multiprocessing.SemLock.__exit__ exc_type: object = None @@ -704,13 +715,13 @@ static PyObject * _multiprocessing_SemLock___exit___impl(SemLockObject *self, PyObject *exc_type, PyObject *exc_value, PyObject *exc_tb) -/*[clinic end generated code: output=3b37c1a9f8b91a03 input=7d644b64a89903f8]*/ +/*[clinic end generated code: output=3b37c1a9f8b91a03 input=1610c8cc3e0e337e]*/ { return _multiprocessing_SemLock_release_impl(self); } static int -semlock_traverse(SemLockObject *s, visitproc visit, void *arg) +semlock_traverse(PyObject *s, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(s)); return 0; diff --git a/Modules/_opcode.c b/Modules/_opcode.c index 93c71377f03a76..7ccf7af6bf908f 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -5,9 +5,13 @@ #include "Python.h" #include "compile.h" #include "opcode.h" -#include "internal/pycore_code.h" -#include "internal/pycore_compile.h" -#include "internal/pycore_intrinsics.h" +#include "pycore_ceval.h" +#include "pycore_code.h" +#include "pycore_compile.h" +#include "pycore_intrinsics.h" +#include "pycore_optimizer.h" // _Py_GetExecutor() +#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE, OPCODE_HAS_*, etc +#include "pycore_opcode_utils.h" /*[clinic input] module _opcode @@ -79,7 +83,7 @@ static int _opcode_is_valid_impl(PyObject *module, int opcode) /*[clinic end generated code: output=b0d918ea1d073f65 input=fe23e0aa194ddae0]*/ { - return _PyCompile_OpcodeIsValid(opcode); + return IS_VALID_OPCODE(opcode); } /*[clinic input] @@ -95,8 +99,7 @@ static int _opcode_has_arg_impl(PyObject *module, int opcode) /*[clinic end generated code: output=7a062d3b2dcc0815 input=93d878ba6361db5f]*/ { - return _PyCompile_OpcodeIsValid(opcode) && - _PyCompile_OpcodeHasArg(opcode); + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_ARG(opcode); } /*[clinic input] @@ -112,8 +115,7 @@ static int _opcode_has_const_impl(PyObject *module, int opcode) /*[clinic end generated code: output=c646d5027c634120 input=a6999e4cf13f9410]*/ { - return _PyCompile_OpcodeIsValid(opcode) && - _PyCompile_OpcodeHasConst(opcode); + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_CONST(opcode); } /*[clinic input] @@ -129,8 +131,7 @@ static int _opcode_has_name_impl(PyObject *module, int opcode) /*[clinic end generated code: output=b49a83555c2fa517 input=448aa5e4bcc947ba]*/ { - return _PyCompile_OpcodeIsValid(opcode) && - _PyCompile_OpcodeHasName(opcode); + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_NAME(opcode); } /*[clinic input] @@ -146,9 +147,7 @@ static int _opcode_has_jump_impl(PyObject *module, int opcode) /*[clinic end generated code: output=e9c583c669f1c46a input=35f711274357a0c3]*/ { - return _PyCompile_OpcodeIsValid(opcode) && - _PyCompile_OpcodeHasJump(opcode); - + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_JUMP(opcode); } /*[clinic input] @@ -169,9 +168,7 @@ static int _opcode_has_free_impl(PyObject *module, int opcode) /*[clinic end generated code: output=d81ae4d79af0ee26 input=117dcd5c19c1139b]*/ { - return _PyCompile_OpcodeIsValid(opcode) && - _PyCompile_OpcodeHasFree(opcode); - + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_FREE(opcode); } /*[clinic input] @@ -187,8 +184,7 @@ static int _opcode_has_local_impl(PyObject *module, int opcode) /*[clinic end generated code: output=da5a8616b7a5097b input=9a798ee24aaef49d]*/ { - return _PyCompile_OpcodeIsValid(opcode) && - _PyCompile_OpcodeHasLocal(opcode); + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_LOCAL(opcode); } /*[clinic input] @@ -204,8 +200,7 @@ static int _opcode_has_exc_impl(PyObject *module, int opcode) /*[clinic end generated code: output=41b68dff0ec82a52 input=db0e4bdb9bf13fa5]*/ { - return _PyCompile_OpcodeIsValid(opcode) && - _PyCompile_OpcodeHasExc(opcode); + return IS_VALID_OPCODE(opcode) && IS_BLOCK_PUSH_OPCODE(opcode); } /*[clinic input] @@ -347,6 +342,60 @@ _opcode_get_intrinsic2_descs_impl(PyObject *module) return list; } +/*[clinic input] + +_opcode.get_special_method_names + +Return a list of special method names. +[clinic start generated code]*/ + +static PyObject * +_opcode_get_special_method_names_impl(PyObject *module) +/*[clinic end generated code: output=fce72614cd988d17 input=25f2115560bdf163]*/ +{ + PyObject *list = PyList_New(SPECIAL_MAX + 1); + if (list == NULL) { + return NULL; + } + for (int i=0; i <= SPECIAL_MAX; i++) { + PyObject *name = _Py_SpecialMethods[i].name; + if (name == NULL) { + Py_DECREF(list); + return NULL; + } + PyList_SET_ITEM(list, i, name); + } + return list; +} + +/*[clinic input] + +_opcode.get_executor + + code: object + offset: int + +Return the executor object at offset in code if exists, None otherwise. +[clinic start generated code]*/ + +static PyObject * +_opcode_get_executor_impl(PyObject *module, PyObject *code, int offset) +/*[clinic end generated code: output=c035c7a47b16648f input=85eff93ea7aac282]*/ +{ + if (!PyCode_Check(code)) { + PyErr_Format(PyExc_TypeError, + "expected a code object, not '%.100s'", + Py_TYPE(code)->tp_name); + return NULL; + } +#ifdef _Py_TIER2 + return (PyObject *)_Py_GetExecutor((PyCodeObject *)code, offset); +#else + PyErr_Format(PyExc_RuntimeError, + "Executors are not available in this build"); + return NULL; +#endif +} static PyMethodDef opcode_functions[] = { @@ -363,20 +412,26 @@ opcode_functions[] = { _OPCODE_GET_NB_OPS_METHODDEF _OPCODE_GET_INTRINSIC1_DESCS_METHODDEF _OPCODE_GET_INTRINSIC2_DESCS_METHODDEF + _OPCODE_GET_EXECUTOR_METHODDEF + _OPCODE_GET_SPECIAL_METHOD_NAMES_METHODDEF {NULL, NULL, 0, NULL} }; -int +static int _opcode_exec(PyObject *m) { if (PyModule_AddIntMacro(m, ENABLE_SPECIALIZATION) < 0) { return -1; } + if (PyModule_AddIntMacro(m, ENABLE_SPECIALIZATION_FT) < 0) { + return -1; + } return 0; } static PyModuleDef_Slot module_slots[] = { {Py_mod_exec, _opcode_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_operator.c b/Modules/_operator.c index 1f6496d381adac..ce3ef015710223 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -2,6 +2,7 @@ #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_runtime.h" // _Py_ID() +#include "pycore_pystate.h" // _PyInterpreterState_GET() #include "clinic/_operator.c.h" @@ -727,6 +728,34 @@ _operator_is_not_impl(PyObject *module, PyObject *a, PyObject *b) return Py_NewRef(result); } +/*[clinic input] +_operator.is_none = _operator.neg + +Same as a is None. +[clinic start generated code]*/ + +static PyObject * +_operator_is_none(PyObject *module, PyObject *a) +/*[clinic end generated code: output=07159cc102261dec input=0448b38af7b8533d]*/ +{ + PyObject *result = Py_IsNone(a) ? Py_True : Py_False; + return Py_NewRef(result); +} + +/*[clinic input] +_operator.is_not_none = _operator.neg + +Same as a is not None. +[clinic start generated code]*/ + +static PyObject * +_operator_is_not_none(PyObject *module, PyObject *a) +/*[clinic end generated code: output=b0168a51451d9140 input=7587f38ebac51688]*/ +{ + PyObject *result = Py_IsNone(a) ? Py_False : Py_True; + return Py_NewRef(result); +} + /* compare_digest **********************************************************/ /* @@ -915,6 +944,8 @@ static struct PyMethodDef operator_methods[] = { _OPERATOR_COUNTOF_METHODDEF _OPERATOR_IS__METHODDEF _OPERATOR_IS_NOT_METHODDEF + _OPERATOR_IS_NONE_METHODDEF + _OPERATOR_IS_NOT_NONE_METHODDEF _OPERATOR_INDEX_METHODDEF _OPERATOR_ADD_METHODDEF _OPERATOR_SUB_METHODDEF @@ -966,6 +997,18 @@ static struct PyMethodDef operator_methods[] = { }; + +static PyObject * +text_signature(PyObject *self, void *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("(obj, /)"); +} + +static PyGetSetDef common_getset[] = { + {"__text_signature__", text_signature, (setter)NULL}, + {NULL} +}; + /* itemgetter object **********************************************************/ typedef struct { @@ -1171,6 +1214,7 @@ static PyType_Slot itemgetter_type_slots[] = { {Py_tp_clear, itemgetter_clear}, {Py_tp_methods, itemgetter_methods}, {Py_tp_members, itemgetter_members}, + {Py_tp_getset, common_getset}, {Py_tp_new, itemgetter_new}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_repr, itemgetter_repr}, @@ -1223,6 +1267,7 @@ attrgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; /* prepare attr while checking args */ + PyInterpreterState *interp = _PyInterpreterState_GET(); for (idx = 0; idx < nattrs; ++idx) { PyObject *item = PyTuple_GET_ITEM(args, idx); int dot_count; @@ -1246,7 +1291,7 @@ attrgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (dot_count == 0) { Py_INCREF(item); - PyUnicode_InternInPlace(&item); + _PyUnicode_InternMortal(interp, &item); PyTuple_SET_ITEM(attr, idx, item); } else { /* make it a tuple of non-dotted attrnames */ PyObject *attr_chain = PyTuple_New(dot_count + 1); @@ -1272,7 +1317,7 @@ attrgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(attr); return NULL; } - PyUnicode_InternInPlace(&attr_chain_item); + _PyUnicode_InternMortal(interp, &attr_chain_item); PyTuple_SET_ITEM(attr_chain, attr_chain_idx, attr_chain_item); ++attr_chain_idx; unibuff_till = unibuff_from = unibuff_till + 1; @@ -1286,7 +1331,7 @@ attrgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(attr); return NULL; } - PyUnicode_InternInPlace(&attr_chain_item); + _PyUnicode_InternMortal(interp, &attr_chain_item); PyTuple_SET_ITEM(attr_chain, attr_chain_idx, attr_chain_item); PyTuple_SET_ITEM(attr, idx, attr_chain); @@ -1528,6 +1573,7 @@ static PyType_Slot attrgetter_type_slots[] = { {Py_tp_clear, attrgetter_clear}, {Py_tp_methods, attrgetter_methods}, {Py_tp_members, attrgetter_members}, + {Py_tp_getset, common_getset}, {Py_tp_new, attrgetter_new}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_repr, attrgetter_repr}, @@ -1549,76 +1595,75 @@ static PyType_Spec attrgetter_type_spec = { typedef struct { PyObject_HEAD PyObject *name; - PyObject *xargs; // reference to arguments passed in constructor + PyObject *args; PyObject *kwds; - PyObject **vectorcall_args; /* Borrowed references */ + PyObject *vectorcall_args; PyObject *vectorcall_kwnames; vectorcallfunc vectorcall; } methodcallerobject; -static int _methodcaller_initialize_vectorcall(methodcallerobject* mc) -{ - PyObject* args = mc->xargs; - PyObject* kwds = mc->kwds; - - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - assert(nargs > 0); - mc->vectorcall_args = PyMem_Calloc( - nargs + (kwds ? PyDict_Size(kwds) : 0), - sizeof(PyObject*)); - if (!mc->vectorcall_args) { - PyErr_NoMemory(); - return -1; - } - /* The first item of vectorcall_args will be filled with obj later */ - if (nargs > 1) { - memcpy(mc->vectorcall_args, PySequence_Fast_ITEMS(args), - nargs * sizeof(PyObject*)); - } - if (kwds) { - const Py_ssize_t nkwds = PyDict_Size(kwds); - - mc->vectorcall_kwnames = PyTuple_New(nkwds); - if (!mc->vectorcall_kwnames) { - return -1; - } - Py_ssize_t i = 0, ppos = 0; - PyObject* key, * value; - while (PyDict_Next(kwds, &ppos, &key, &value)) { - PyTuple_SET_ITEM(mc->vectorcall_kwnames, i, Py_NewRef(key)); - mc->vectorcall_args[nargs + i] = value; // borrowed reference - ++i; - } - } - else { - mc->vectorcall_kwnames = NULL; - } - return 1; -} +#define _METHODCALLER_MAX_ARGS 8 static PyObject * -methodcaller_vectorcall( - methodcallerobject *mc, PyObject *const *args, size_t nargsf, PyObject* kwnames) +methodcaller_vectorcall(methodcallerobject *mc, PyObject *const *args, + size_t nargsf, PyObject* kwnames) { if (!_PyArg_CheckPositional("methodcaller", PyVectorcall_NARGS(nargsf), 1, 1) || !_PyArg_NoKwnames("methodcaller", kwnames)) { return NULL; } - if (mc->vectorcall_args == NULL) { - if (_methodcaller_initialize_vectorcall(mc) < 0) { - return NULL; - } - } + assert(mc->vectorcall_args != NULL); - assert(mc->vectorcall_args != 0); - mc->vectorcall_args[0] = args[0]; - return PyObject_VectorcallMethod( - mc->name, mc->vectorcall_args, - (PyTuple_GET_SIZE(mc->xargs)) | PY_VECTORCALL_ARGUMENTS_OFFSET, + PyObject *tmp_args[_METHODCALLER_MAX_ARGS]; + tmp_args[0] = args[0]; + assert(1 + PyTuple_GET_SIZE(mc->vectorcall_args) <= _METHODCALLER_MAX_ARGS); + memcpy(tmp_args + 1, _PyTuple_ITEMS(mc->vectorcall_args), sizeof(PyObject *) * PyTuple_GET_SIZE(mc->vectorcall_args)); + + return PyObject_VectorcallMethod(mc->name, tmp_args, + (1 + PyTuple_GET_SIZE(mc->args)) | PY_VECTORCALL_ARGUMENTS_OFFSET, mc->vectorcall_kwnames); } +static int +_methodcaller_initialize_vectorcall(methodcallerobject* mc) +{ + PyObject* args = mc->args; + PyObject* kwds = mc->kwds; + + if (kwds && PyDict_Size(kwds)) { + PyObject *values = PyDict_Values(kwds); + if (!values) { + return -1; + } + PyObject *values_tuple = PySequence_Tuple(values); + Py_DECREF(values); + if (!values_tuple) { + return -1; + } + if (PyTuple_GET_SIZE(args)) { + mc->vectorcall_args = PySequence_Concat(args, values_tuple); + Py_DECREF(values_tuple); + if (mc->vectorcall_args == NULL) { + return -1; + } + } + else { + mc->vectorcall_args = values_tuple; + } + mc->vectorcall_kwnames = PySequence_Tuple(kwds); + if (!mc->vectorcall_kwnames) { + return -1; + } + } + else { + mc->vectorcall_args = Py_NewRef(args); + mc->vectorcall_kwnames = NULL; + } + + mc->vectorcall = (vectorcallfunc)methodcaller_vectorcall; + return 0; +} /* AC 3.5: variable number of arguments, not currently support by AC */ static PyObject * @@ -1646,17 +1691,30 @@ methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (mc == NULL) { return NULL; } + mc->vectorcall = NULL; + mc->vectorcall_args = NULL; + mc->vectorcall_kwnames = NULL; + mc->kwds = Py_XNewRef(kwds); Py_INCREF(name); - PyUnicode_InternInPlace(&name); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternMortal(interp, &name); mc->name = name; - mc->xargs = Py_XNewRef(args); // allows us to use borrowed references - mc->kwds = Py_XNewRef(kwds); - mc->vectorcall_args = 0; - + mc->args = PyTuple_GetSlice(args, 1, PyTuple_GET_SIZE(args)); + if (mc->args == NULL) { + Py_DECREF(mc); + return NULL; + } - mc->vectorcall = (vectorcallfunc)methodcaller_vectorcall; + Py_ssize_t vectorcall_size = PyTuple_GET_SIZE(args) + + (kwds ? PyDict_Size(kwds) : 0); + if (vectorcall_size < (_METHODCALLER_MAX_ARGS)) { + if (_methodcaller_initialize_vectorcall(mc) < 0) { + Py_DECREF(mc); + return NULL; + } + } PyObject_GC_Track(mc); return (PyObject *)mc; @@ -1666,13 +1724,10 @@ static void methodcaller_clear(methodcallerobject *mc) { Py_CLEAR(mc->name); - Py_CLEAR(mc->xargs); + Py_CLEAR(mc->args); Py_CLEAR(mc->kwds); - if (mc->vectorcall_args != NULL) { - PyMem_Free(mc->vectorcall_args); - mc->vectorcall_args = 0; - Py_CLEAR(mc->vectorcall_kwnames); - } + Py_CLEAR(mc->vectorcall_args); + Py_CLEAR(mc->vectorcall_kwnames); } static void @@ -1689,8 +1744,10 @@ static int methodcaller_traverse(methodcallerobject *mc, visitproc visit, void *arg) { Py_VISIT(mc->name); - Py_VISIT(mc->xargs); + Py_VISIT(mc->args); Py_VISIT(mc->kwds); + Py_VISIT(mc->vectorcall_args); + Py_VISIT(mc->vectorcall_kwnames); Py_VISIT(Py_TYPE(mc)); return 0; } @@ -1709,15 +1766,7 @@ methodcaller_call(methodcallerobject *mc, PyObject *args, PyObject *kw) if (method == NULL) return NULL; - - PyObject *cargs = PyTuple_GetSlice(mc->xargs, 1, PyTuple_GET_SIZE(mc->xargs)); - if (cargs == NULL) { - Py_DECREF(method); - return NULL; - } - - result = PyObject_Call(method, cargs, mc->kwds); - Py_DECREF(cargs); + result = PyObject_Call(method, mc->args, mc->kwds); Py_DECREF(method); return result; } @@ -1735,7 +1784,7 @@ methodcaller_repr(methodcallerobject *mc) } numkwdargs = mc->kwds != NULL ? PyDict_GET_SIZE(mc->kwds) : 0; - numposargs = PyTuple_GET_SIZE(mc->xargs) - 1; + numposargs = PyTuple_GET_SIZE(mc->args); numtotalargs = numposargs + numkwdargs; if (numtotalargs == 0) { @@ -1751,7 +1800,7 @@ methodcaller_repr(methodcallerobject *mc) } for (i = 0; i < numposargs; ++i) { - PyObject *onerepr = PyObject_Repr(PyTuple_GET_ITEM(mc->xargs, i+1)); + PyObject *onerepr = PyObject_Repr(PyTuple_GET_ITEM(mc->args, i)); if (onerepr == NULL) goto done; PyTuple_SET_ITEM(argreprs, i, onerepr); @@ -1803,14 +1852,14 @@ methodcaller_reduce(methodcallerobject *mc, PyObject *Py_UNUSED(ignored)) { if (!mc->kwds || PyDict_GET_SIZE(mc->kwds) == 0) { Py_ssize_t i; - Py_ssize_t newarg_size = PyTuple_GET_SIZE(mc->xargs); - PyObject *newargs = PyTuple_New(newarg_size); + Py_ssize_t callargcount = PyTuple_GET_SIZE(mc->args); + PyObject *newargs = PyTuple_New(1 + callargcount); if (newargs == NULL) return NULL; PyTuple_SET_ITEM(newargs, 0, Py_NewRef(mc->name)); - for (i = 1; i < newarg_size; ++i) { - PyObject *arg = PyTuple_GET_ITEM(mc->xargs, i); - PyTuple_SET_ITEM(newargs, i, Py_NewRef(arg)); + for (i = 0; i < callargcount; ++i) { + PyObject *arg = PyTuple_GET_ITEM(mc->args, i); + PyTuple_SET_ITEM(newargs, i + 1, Py_NewRef(arg)); } return Py_BuildValue("ON", Py_TYPE(mc), newargs); } @@ -1828,12 +1877,7 @@ methodcaller_reduce(methodcallerobject *mc, PyObject *Py_UNUSED(ignored)) constructor = PyObject_VectorcallDict(partial, newargs, 2, mc->kwds); Py_DECREF(partial); - PyObject *args = PyTuple_GetSlice(mc->xargs, 1, PyTuple_GET_SIZE(mc->xargs)); - if (!args) { - Py_DECREF(constructor); - return NULL; - } - return Py_BuildValue("NO", constructor, args); + return Py_BuildValue("NO", constructor, mc->args); } } @@ -1863,6 +1907,7 @@ static PyType_Slot methodcaller_type_slots[] = { {Py_tp_clear, methodcaller_clear}, {Py_tp_methods, methodcaller_methods}, {Py_tp_members, methodcaller_members}, + {Py_tp_getset, common_getset}, {Py_tp_new, methodcaller_new}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_repr, methodcaller_repr}, @@ -1913,6 +1958,7 @@ operator_exec(PyObject *module) static struct PyModuleDef_Slot operator_slots[] = { {Py_mod_exec, operator_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 0d83261168185d..a6cfb2deeb23c1 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -9,15 +9,17 @@ #endif #include "Python.h" -#include "pycore_bytesobject.h" // _PyBytesWriter -#include "pycore_ceval.h" // _Py_EnterRecursiveCall() -#include "pycore_long.h" // _PyLong_AsByteArray() -#include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_object.h" // _PyNone_Type -#include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_runtime.h" // _Py_ID() -#include "pycore_setobject.h" // _PySet_NextEntry() -#include "pycore_sysmodule.h" // _PySys_GetAttr() +#include "pycore_bytesobject.h" // _PyBytesWriter +#include "pycore_ceval.h" // _Py_EnterRecursiveCall() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() +#include "pycore_long.h" // _PyLong_AsByteArray() +#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_object.h" // _PyNone_Type +#include "pycore_pyerrors.h" // _PyErr_FormatNote +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_runtime.h" // _Py_ID() +#include "pycore_setobject.h" // _PySet_NextEntry() +#include "pycore_sysmodule.h" // _PySys_GetAttr() #include // strtol() @@ -39,7 +41,7 @@ class _pickle.UnpicklerMemoProxy "UnpicklerMemoProxyObject *" "" already includes it. */ enum { HIGHEST_PROTOCOL = 5, - DEFAULT_PROTOCOL = 4 + DEFAULT_PROTOCOL = 5 }; #ifdef MS_WINDOWS @@ -611,6 +613,7 @@ typedef struct PicklerObject { objects to support self-referential objects pickling. */ PyObject *persistent_id; /* persistent_id() method, can be NULL */ + PyObject *persistent_id_attr; /* instance attribute, can be NULL */ PyObject *dispatch_table; /* private dispatch_table, can be NULL */ PyObject *reducer_override; /* hook for invoking user-defined callbacks instead of save_global when pickling @@ -653,6 +656,7 @@ typedef struct UnpicklerObject { size_t memo_len; /* Number of objects in the memo */ PyObject *persistent_load; /* persistent_load() method, can be NULL. */ + PyObject *persistent_load_attr; /* instance attribute, can be NULL. */ Py_buffer buffer; char *input_buffer; @@ -1106,6 +1110,7 @@ _Pickler_New(PickleState *st) self->memo = memo; self->persistent_id = NULL; + self->persistent_id_attr = NULL; self->dispatch_table = NULL; self->reducer_override = NULL; self->write = NULL; @@ -1286,6 +1291,10 @@ _Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n) else { read_size = _Unpickler_SetStringInput(self, data); Py_DECREF(data); + if (read_size < 0) { + return -1; + } + self->prefetched_idx = 0; if (n <= read_size) return n; @@ -1600,6 +1609,7 @@ _Unpickler_New(PyObject *module) self->memo_size = MEMO_SIZE; self->memo_len = 0; self->persistent_load = NULL; + self->persistent_load_attr = NULL; memset(&self->buffer, 0, sizeof(Py_buffer)); self->input_buffer = NULL; self->input_line = NULL; @@ -1802,80 +1812,53 @@ memo_put(PickleState *st, PicklerObject *self, PyObject *obj) } static PyObject * -get_dotted_path(PyObject *obj, PyObject *name) +get_dotted_path(PyObject *name) +{ + return PyUnicode_Split(name, _Py_LATIN1_CHR('.'), -1); +} + +static int +check_dotted_path(PickleState *st, PyObject *obj, PyObject *dotted_path) { - PyObject *dotted_path; Py_ssize_t i, n; - _Py_DECLARE_STR(dot, "."); - dotted_path = PyUnicode_Split(name, &_Py_STR(dot), -1); - if (dotted_path == NULL) - return NULL; n = PyList_GET_SIZE(dotted_path); assert(n >= 1); for (i = 0; i < n; i++) { PyObject *subpath = PyList_GET_ITEM(dotted_path, i); if (_PyUnicode_EqualToASCIIString(subpath, "")) { - if (obj == NULL) - PyErr_Format(PyExc_AttributeError, - "Can't pickle local object %R", name); - else - PyErr_Format(PyExc_AttributeError, - "Can't pickle local attribute %R on %R", name, obj); - Py_DECREF(dotted_path); - return NULL; + PyErr_Format(st->PicklingError, + "Can't pickle local object %R", obj); + return -1; } } - return dotted_path; + return 0; } static PyObject * -get_deep_attribute(PyObject *obj, PyObject *names, PyObject **pparent) +getattribute(PyObject *obj, PyObject *names, int raises) { Py_ssize_t i, n; - PyObject *parent = NULL; assert(PyList_CheckExact(names)); Py_INCREF(obj); n = PyList_GET_SIZE(names); for (i = 0; i < n; i++) { PyObject *name = PyList_GET_ITEM(names, i); - Py_XSETREF(parent, obj); - (void)PyObject_GetOptionalAttr(parent, name, &obj); + PyObject *parent = obj; + if (raises) { + obj = PyObject_GetAttr(parent, name); + } + else { + (void)PyObject_GetOptionalAttr(parent, name, &obj); + } + Py_DECREF(parent); if (obj == NULL) { - Py_DECREF(parent); return NULL; } } - if (pparent != NULL) - *pparent = parent; - else - Py_XDECREF(parent); return obj; } - -static PyObject * -getattribute(PyObject *obj, PyObject *name, int allow_qualname) -{ - PyObject *dotted_path, *attr; - - if (allow_qualname) { - dotted_path = get_dotted_path(obj, name); - if (dotted_path == NULL) - return NULL; - attr = get_deep_attribute(obj, dotted_path, NULL); - Py_DECREF(dotted_path); - } - else { - (void)PyObject_GetOptionalAttr(obj, name, &attr); - } - if (attr == NULL && !PyErr_Occurred()) { - PyErr_Format(PyExc_AttributeError, - "Can't get attribute %R on %R", name, obj); - } - return attr; -} - static int _checkmodule(PyObject *module_name, PyObject *module, PyObject *global, PyObject *dotted_path) @@ -1888,7 +1871,7 @@ _checkmodule(PyObject *module_name, PyObject *module, return -1; } - PyObject *candidate = get_deep_attribute(module, dotted_path, NULL); + PyObject *candidate = getattribute(module, dotted_path, 0); if (candidate == NULL) { return -1; } @@ -1901,73 +1884,125 @@ _checkmodule(PyObject *module_name, PyObject *module, } static PyObject * -whichmodule(PyObject *global, PyObject *dotted_path) +whichmodule(PickleState *st, PyObject *global, PyObject *global_name, PyObject *dotted_path) { PyObject *module_name; PyObject *module = NULL; Py_ssize_t i; PyObject *modules; + if (check_dotted_path(st, global, dotted_path) < 0) { + return NULL; + } if (PyObject_GetOptionalAttr(global, &_Py_ID(__module__), &module_name) < 0) { return NULL; } - if (module_name) { + if (module_name == NULL || module_name == Py_None) { /* In some rare cases (e.g., bound methods of extension types), __module__ can be None. If it is so, then search sys.modules for the module of global. */ - if (module_name != Py_None) - return module_name; Py_CLEAR(module_name); - } - assert(module_name == NULL); - - /* Fallback on walking sys.modules */ - PyThreadState *tstate = _PyThreadState_GET(); - modules = _PySys_GetAttr(tstate, &_Py_ID(modules)); - if (modules == NULL) { - PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules"); - return NULL; - } - if (PyDict_CheckExact(modules)) { - i = 0; - while (PyDict_Next(modules, &i, &module_name, &module)) { - if (_checkmodule(module_name, module, global, dotted_path) == 0) { - return Py_NewRef(module_name); - } - if (PyErr_Occurred()) { - return NULL; - } - } - } - else { - PyObject *iterator = PyObject_GetIter(modules); - if (iterator == NULL) { + PyThreadState *tstate = _PyThreadState_GET(); + modules = _PySys_GetAttr(tstate, &_Py_ID(modules)); + if (modules == NULL) { + PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules"); return NULL; } - while ((module_name = PyIter_Next(iterator))) { - module = PyObject_GetItem(modules, module_name); - if (module == NULL) { + if (PyDict_CheckExact(modules)) { + i = 0; + while (PyDict_Next(modules, &i, &module_name, &module)) { + Py_INCREF(module_name); + Py_INCREF(module); + if (_checkmodule(module_name, module, global, dotted_path) == 0) { + Py_DECREF(module); + return module_name; + } + Py_DECREF(module); Py_DECREF(module_name); - Py_DECREF(iterator); + if (PyErr_Occurred()) { + return NULL; + } + } + } + else { + PyObject *iterator = PyObject_GetIter(modules); + if (iterator == NULL) { return NULL; } - if (_checkmodule(module_name, module, global, dotted_path) == 0) { + while ((module_name = PyIter_Next(iterator))) { + module = PyObject_GetItem(modules, module_name); + if (module == NULL) { + Py_DECREF(module_name); + Py_DECREF(iterator); + return NULL; + } + if (_checkmodule(module_name, module, global, dotted_path) == 0) { + Py_DECREF(module); + Py_DECREF(iterator); + return module_name; + } Py_DECREF(module); - Py_DECREF(iterator); - return module_name; - } - Py_DECREF(module); - Py_DECREF(module_name); - if (PyErr_Occurred()) { - Py_DECREF(iterator); - return NULL; + Py_DECREF(module_name); + if (PyErr_Occurred()) { + Py_DECREF(iterator); + return NULL; + } } + Py_DECREF(iterator); } - Py_DECREF(iterator); + if (PyErr_Occurred()) { + return NULL; + } + + /* If no module is found, use __main__. */ + module_name = Py_NewRef(&_Py_ID(__main__)); } - /* If no module is found, use __main__. */ - return &_Py_ID(__main__); + /* XXX: Change to use the import C API directly with level=0 to disallow + relative imports. + + XXX: PyImport_ImportModuleLevel could be used. However, this bypasses + builtins.__import__. Therefore, _pickle, unlike pickle.py, will ignore + custom import functions (IMHO, this would be a nice security + feature). The import C API would need to be extended to support the + extra parameters of __import__ to fix that. */ + module = PyImport_Import(module_name); + if (module == NULL) { + if (PyErr_ExceptionMatches(PyExc_ImportError) || + PyErr_ExceptionMatches(PyExc_ValueError)) + { + PyObject *exc = PyErr_GetRaisedException(); + PyErr_Format(st->PicklingError, + "Can't pickle %R: %S", global, exc); + _PyErr_ChainExceptions1(exc); + } + Py_DECREF(module_name); + return NULL; + } + PyObject *actual = getattribute(module, dotted_path, 1); + Py_DECREF(module); + if (actual == NULL) { + assert(PyErr_Occurred()); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyObject *exc = PyErr_GetRaisedException(); + PyErr_Format(st->PicklingError, + "Can't pickle %R: it's not found as %S.%S", + global, module_name, global_name); + _PyErr_ChainExceptions1(exc); + } + Py_DECREF(module_name); + return NULL; + } + if (actual != global) { + Py_DECREF(actual); + PyErr_Format(st->PicklingError, + "Can't pickle %R: it's not the same object as %S.%S", + global, module_name, global_name); + Py_DECREF(module_name); + return NULL; + } + Py_DECREF(actual); + return module_name; } /* fast_save_enter() and fast_save_leave() are guards against recursive @@ -2119,13 +2154,15 @@ save_long(PicklerObject *self, PyObject *obj) if (self->proto >= 2) { /* Linear-time pickling. */ - size_t nbits; + int64_t nbits; size_t nbytes; unsigned char *pdata; char header[5]; int i; - int sign = _PyLong_Sign(obj); + int sign; + assert(PyLong_Check(obj)); + (void)PyLong_GetSign(obj, &sign); if (sign == 0) { header[0] = LONG1; header[1] = 0; /* It's 0 -- an empty bytestring. */ @@ -2134,8 +2171,8 @@ save_long(PicklerObject *self, PyObject *obj) return 0; } nbits = _PyLong_NumBits(obj); - if (nbits == (size_t)-1 && PyErr_Occurred()) - goto error; + assert(nbits >= 0); + assert(!PyErr_Occurred()); /* How many bytes do we need? There are nbits >> 3 full * bytes of data, and nbits & 7 leftover bits. If there * are any leftover bits, then we clearly need another @@ -2150,7 +2187,7 @@ save_long(PicklerObject *self, PyObject *obj) * for in advance, though, so we always grab an extra * byte at the start, and cut it back later if possible. */ - nbytes = (nbits >> 3) + 1; + nbytes = (size_t)((nbits >> 3) + 1); if (nbytes > 0x7fffffffL) { PyErr_SetString(PyExc_OverflowError, "int too large to pickle"); @@ -2507,7 +2544,7 @@ save_picklebuffer(PickleState *st, PicklerObject *self, PyObject *obj) { if (self->proto < 5) { PyErr_SetString(st->PicklingError, - "PickleBuffer can only pickled with protocol >= 5"); + "PickleBuffer can only be pickled with protocol >= 5"); return -1; } const Py_buffer* view = PyPickleBuffer_GetBuffer(obj); @@ -2738,8 +2775,10 @@ store_tuple_elements(PickleState *state, PicklerObject *self, PyObject *t, if (element == NULL) return -1; - if (save(state, self, element, 0) < 0) + if (save(state, self, element, 0) < 0) { + _PyErr_FormatNote("when serializing %T item %zd", t, i); return -1; + } } return 0; @@ -2858,11 +2897,12 @@ save_tuple(PickleState *state, PicklerObject *self, PyObject *obj) * Returns 0 on success, <0 on error. */ static int -batch_list(PickleState *state, PicklerObject *self, PyObject *iter) +batch_list(PickleState *state, PicklerObject *self, PyObject *iter, PyObject *origobj) { PyObject *obj = NULL; PyObject *firstitem = NULL; int i, n; + Py_ssize_t total = 0; const char mark_op = MARK; const char append_op = APPEND; @@ -2877,7 +2917,7 @@ batch_list(PickleState *state, PicklerObject *self, PyObject *iter) if (self->proto == 0) { /* APPENDS isn't available; do one at a time. */ - for (;;) { + for (;; total++) { obj = PyIter_Next(iter); if (obj == NULL) { if (PyErr_Occurred()) @@ -2886,8 +2926,10 @@ batch_list(PickleState *state, PicklerObject *self, PyObject *iter) } i = save(state, self, obj, 0); Py_DECREF(obj); - if (i < 0) + if (i < 0) { + _PyErr_FormatNote("when serializing %T item %zd", origobj, total); return -1; + } if (_Pickler_Write(self, &append_op, 1) < 0) return -1; } @@ -2913,8 +2955,10 @@ batch_list(PickleState *state, PicklerObject *self, PyObject *iter) goto error; /* Only one item to write */ - if (save(state, self, firstitem, 0) < 0) + if (save(state, self, firstitem, 0) < 0) { + _PyErr_FormatNote("when serializing %T item %zd", origobj, total); goto error; + } if (_Pickler_Write(self, &append_op, 1) < 0) goto error; Py_CLEAR(firstitem); @@ -2927,16 +2971,22 @@ batch_list(PickleState *state, PicklerObject *self, PyObject *iter) if (_Pickler_Write(self, &mark_op, 1) < 0) goto error; - if (save(state, self, firstitem, 0) < 0) + if (save(state, self, firstitem, 0) < 0) { + _PyErr_FormatNote("when serializing %T item %zd", origobj, total); goto error; + } Py_CLEAR(firstitem); + total++; n = 1; /* Fetch and save up to BATCHSIZE items */ while (obj) { - if (save(state, self, obj, 0) < 0) + if (save(state, self, obj, 0) < 0) { + _PyErr_FormatNote("when serializing %T item %zd", origobj, total); goto error; + } Py_CLEAR(obj); + total++; n += 1; if (n == BATCHSIZE) @@ -2992,8 +3042,10 @@ batch_list_exact(PickleState *state, PicklerObject *self, PyObject *obj) Py_INCREF(item); int err = save(state, self, item, 0); Py_DECREF(item); - if (err < 0) + if (err < 0) { + _PyErr_FormatNote("when serializing %T item 0", obj); return -1; + } if (_Pickler_Write(self, &append_op, 1) < 0) return -1; return 0; @@ -3010,8 +3062,10 @@ batch_list_exact(PickleState *state, PicklerObject *self, PyObject *obj) Py_INCREF(item); int err = save(state, self, item, 0); Py_DECREF(item); - if (err < 0) + if (err < 0) { + _PyErr_FormatNote("when serializing %T item %zd", obj, total); return -1; + } total++; if (++this_batch == BATCHSIZE) break; @@ -3071,7 +3125,7 @@ save_list(PickleState *state, PicklerObject *self, PyObject *obj) Py_DECREF(iter); goto error; } - status = batch_list(state, self, iter); + status = batch_list(state, self, iter, obj); _Py_LeaveRecursiveCall(); Py_DECREF(iter); } @@ -3099,7 +3153,7 @@ save_list(PickleState *state, PicklerObject *self, PyObject *obj) * ugly to bear. */ static int -batch_dict(PickleState *state, PicklerObject *self, PyObject *iter) +batch_dict(PickleState *state, PicklerObject *self, PyObject *iter, PyObject *origobj) { PyObject *obj = NULL; PyObject *firstitem = NULL; @@ -3123,11 +3177,17 @@ batch_dict(PickleState *state, PicklerObject *self, PyObject *iter) if (!PyTuple_Check(obj) || PyTuple_Size(obj) != 2) { PyErr_SetString(PyExc_TypeError, "dict items " "iterator must return 2-tuples"); + Py_DECREF(obj); return -1; } i = save(state, self, PyTuple_GET_ITEM(obj, 0), 0); - if (i >= 0) + if (i >= 0) { i = save(state, self, PyTuple_GET_ITEM(obj, 1), 0); + if (i < 0) { + _PyErr_FormatNote("when serializing %T item %R", + origobj, PyTuple_GET_ITEM(obj, 0)); + } + } Py_DECREF(obj); if (i < 0) return -1; @@ -3163,8 +3223,11 @@ batch_dict(PickleState *state, PicklerObject *self, PyObject *iter) /* Only one item to write */ if (save(state, self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0) goto error; - if (save(state, self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0) + if (save(state, self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0) { + _PyErr_FormatNote("when serializing %T item %R", + origobj, PyTuple_GET_ITEM(firstitem, 0)); goto error; + } if (_Pickler_Write(self, &setitem_op, 1) < 0) goto error; Py_CLEAR(firstitem); @@ -3179,8 +3242,11 @@ batch_dict(PickleState *state, PicklerObject *self, PyObject *iter) if (save(state, self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0) goto error; - if (save(state, self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0) + if (save(state, self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0) { + _PyErr_FormatNote("when serializing %T item %R", + origobj, PyTuple_GET_ITEM(firstitem, 0)); goto error; + } Py_CLEAR(firstitem); n = 1; @@ -3191,9 +3257,13 @@ batch_dict(PickleState *state, PicklerObject *self, PyObject *iter) "iterator must return 2-tuples"); goto error; } - if (save(state, self, PyTuple_GET_ITEM(obj, 0), 0) < 0 || - save(state, self, PyTuple_GET_ITEM(obj, 1), 0) < 0) + if (save(state, self, PyTuple_GET_ITEM(obj, 0), 0) < 0) + goto error; + if (save(state, self, PyTuple_GET_ITEM(obj, 1), 0) < 0) { + _PyErr_FormatNote("when serializing %T item %R", + origobj, PyTuple_GET_ITEM(obj, 0)); goto error; + } Py_CLEAR(obj); n += 1; @@ -3254,6 +3324,7 @@ batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) goto error; } if (save(state, self, value, 0) < 0) { + _PyErr_FormatNote("when serializing %T item %R", obj, key); goto error; } Py_CLEAR(key); @@ -3275,6 +3346,7 @@ batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) goto error; } if (save(state, self, value, 0) < 0) { + _PyErr_FormatNote("when serializing %T item %R", obj, key); goto error; } Py_CLEAR(key); @@ -3349,7 +3421,7 @@ save_dict(PickleState *state, PicklerObject *self, PyObject *obj) Py_DECREF(iter); goto error; } - status = batch_dict(state, self, iter); + status = batch_dict(state, self, iter, obj); _Py_LeaveRecursiveCall(); Py_DECREF(iter); } @@ -3413,15 +3485,23 @@ save_set(PickleState *state, PicklerObject *self, PyObject *obj) i = 0; if (_Pickler_Write(self, &mark_op, 1) < 0) return -1; - while (_PySet_NextEntry(obj, &ppos, &item, &hash)) { - Py_INCREF(item); - int err = save(state, self, item, 0); + + int err = 0; + Py_BEGIN_CRITICAL_SECTION(obj); + while (_PySet_NextEntryRef(obj, &ppos, &item, &hash)) { + err = save(state, self, item, 0); Py_CLEAR(item); - if (err < 0) - return -1; + if (err < 0) { + _PyErr_FormatNote("when serializing %T element", obj); + break; + } if (++i == BATCHSIZE) break; } + Py_END_CRITICAL_SECTION(); + if (err < 0) { + return -1; + } if (_Pickler_Write(self, &additems_op, 1) < 0) return -1; if (PySet_GET_SIZE(obj) != set_size) { @@ -3486,6 +3566,7 @@ save_frozenset(PickleState *state, PicklerObject *self, PyObject *obj) break; } if (save(state, self, item, 0) < 0) { + _PyErr_FormatNote("when serializing %T element", obj); Py_DECREF(item); Py_DECREF(iter); return -1; @@ -3583,11 +3664,7 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, { PyObject *global_name = NULL; PyObject *module_name = NULL; - PyObject *module = NULL; - PyObject *parent = NULL; PyObject *dotted_path = NULL; - PyObject *lastname = NULL; - PyObject *cls; int status = 0; const char global_op = GLOBAL; @@ -3605,47 +3682,13 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, } } - dotted_path = get_dotted_path(module, global_name); + dotted_path = get_dotted_path(global_name); if (dotted_path == NULL) goto error; - module_name = whichmodule(obj, dotted_path); + module_name = whichmodule(st, obj, global_name, dotted_path); if (module_name == NULL) goto error; - /* XXX: Change to use the import C API directly with level=0 to disallow - relative imports. - - XXX: PyImport_ImportModuleLevel could be used. However, this bypasses - builtins.__import__. Therefore, _pickle, unlike pickle.py, will ignore - custom import functions (IMHO, this would be a nice security - feature). The import C API would need to be extended to support the - extra parameters of __import__ to fix that. */ - module = PyImport_Import(module_name); - if (module == NULL) { - PyErr_Format(st->PicklingError, - "Can't pickle %R: import of module %R failed", - obj, module_name); - goto error; - } - lastname = Py_NewRef(PyList_GET_ITEM(dotted_path, - PyList_GET_SIZE(dotted_path) - 1)); - cls = get_deep_attribute(module, dotted_path, &parent); - Py_CLEAR(dotted_path); - if (cls == NULL) { - PyErr_Format(st->PicklingError, - "Can't pickle %R: attribute lookup %S on %S failed", - obj, global_name, module_name); - goto error; - } - if (cls != obj) { - Py_DECREF(cls); - PyErr_Format(st->PicklingError, - "Can't pickle %R: it's not the same object as %S.%S", - obj, module_name, global_name); - goto error; - } - Py_DECREF(cls); - if (self->proto >= 2) { /* See whether this is in the extension registry, and if * so generate an EXT opcode. @@ -3660,34 +3703,24 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, if (extension_key == NULL) { goto error; } - code_obj = PyDict_GetItemWithError(st->extension_registry, - extension_key); + if (PyDict_GetItemRef(st->extension_registry, extension_key, &code_obj) < 0) { + Py_DECREF(extension_key); + goto error; + } Py_DECREF(extension_key); - /* The object is not registered in the extension registry. - This is the most likely code path. */ if (code_obj == NULL) { - if (PyErr_Occurred()) { - goto error; - } + /* The object is not registered in the extension registry. + This is the most likely code path. */ goto gen_global; } - /* XXX: pickle.py doesn't check neither the type, nor the range - of the value returned by the extension_registry. It should for - consistency. */ - - /* Verify code_obj has the right type and value. */ - if (!PyLong_Check(code_obj)) { - PyErr_Format(st->PicklingError, - "Can't pickle %R: extension code %R isn't an integer", - obj, code_obj); - goto error; - } - code = PyLong_AS_LONG(code_obj); + code = PyLong_AsLong(code_obj); + Py_DECREF(code_obj); if (code <= 0 || code > 0x7fffffffL) { + /* Should never happen in normal circumstances, since the type and + the value of the code are checked in copyreg.add_extension(). */ if (!PyErr_Occurred()) - PyErr_Format(st->PicklingError, "Can't pickle %R: extension " - "code %ld is out of range", obj, code); + PyErr_Format(PyExc_RuntimeError, "extension code %ld is out of range", code); goto error; } @@ -3717,9 +3750,6 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, } else { gen_global: - if (parent == module) { - Py_SETREF(global_name, Py_NewRef(lastname)); - } if (self->proto >= 4) { const char stack_global_op = STACK_GLOBAL; @@ -3731,20 +3761,30 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, if (_Pickler_Write(self, &stack_global_op, 1) < 0) goto error; } - else if (parent != module) { - PyObject *reduce_value = Py_BuildValue("(O(OO))", - st->getattr, parent, lastname); - if (reduce_value == NULL) - goto error; - status = save_reduce(st, self, reduce_value, NULL); - Py_DECREF(reduce_value); - if (status < 0) - goto error; - } else { /* Generate a normal global opcode if we are using a pickle protocol < 4, or if the object is not registered in the - extension registry. */ + extension registry. + + Objects with multi-part __qualname__ are represented as + getattr(getattr(..., attrname1), attrname2). */ + const char mark_op = MARK; + const char tupletwo_op = (self->proto < 2) ? TUPLE : TUPLE2; + const char reduce_op = REDUCE; + Py_ssize_t i; + if (dotted_path) { + if (PyList_GET_SIZE(dotted_path) > 1) { + Py_SETREF(global_name, Py_NewRef(PyList_GET_ITEM(dotted_path, 0))); + } + for (i = 1; i < PyList_GET_SIZE(dotted_path); i++) { + if (save(st, self, st->getattr, 0) < 0 || + (self->proto < 2 && _Pickler_Write(self, &mark_op, 1) < 0)) + { + goto error; + } + } + } + PyObject *encoded; PyObject *(*unicode_encoder)(PyObject *); @@ -3772,11 +3812,14 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, } encoded = unicode_encoder(module_name); if (encoded == NULL) { - if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) + if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) { + PyObject *exc = PyErr_GetRaisedException(); PyErr_Format(st->PicklingError, - "can't pickle module identifier '%S' using " + "can't pickle module identifier %R using " "pickle protocol %i", module_name, self->proto); + _PyErr_ChainExceptions1(exc); + } goto error; } if (_Pickler_Write(self, PyBytes_AS_STRING(encoded), @@ -3791,11 +3834,14 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, /* Save the name of the module. */ encoded = unicode_encoder(global_name); if (encoded == NULL) { - if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) + if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) { + PyObject *exc = PyErr_GetRaisedException(); PyErr_Format(st->PicklingError, - "can't pickle global identifier '%S' using " + "can't pickle global identifier %R using " "pickle protocol %i", global_name, self->proto); + _PyErr_ChainExceptions1(exc); + } goto error; } if (_Pickler_Write(self, PyBytes_AS_STRING(encoded), @@ -3806,6 +3852,17 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, Py_DECREF(encoded); if (_Pickler_Write(self, "\n", 1) < 0) goto error; + + if (dotted_path) { + for (i = 1; i < PyList_GET_SIZE(dotted_path); i++) { + if (save(st, self, PyList_GET_ITEM(dotted_path, i), 0) < 0 || + _Pickler_Write(self, &tupletwo_op, 1) < 0 || + _Pickler_Write(self, &reduce_op, 1) < 0) + { + goto error; + } + } + } } /* Memoize the object. */ if (memo_put(st, self, obj) < 0) @@ -3818,10 +3875,7 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, } Py_XDECREF(module_name); Py_XDECREF(global_name); - Py_XDECREF(module); - Py_XDECREF(parent); Py_XDECREF(dotted_path); - Py_XDECREF(lastname); return status; } @@ -3948,8 +4002,9 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, size = PyTuple_Size(args); if (size < 2 || size > 6) { - PyErr_SetString(st->PicklingError, "tuple returned by " - "__reduce__ must contain 2 through 6 elements"); + PyErr_SetString(st->PicklingError, + "tuple returned by __reduce__ " + "must contain 2 through 6 elements"); return -1; } @@ -3959,13 +4014,15 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, return -1; if (!PyCallable_Check(callable)) { - PyErr_SetString(st->PicklingError, "first item of the tuple " - "returned by __reduce__ must be callable"); + PyErr_Format(st->PicklingError, + "first item of the tuple returned by __reduce__ " + "must be callable, not %T", callable); return -1; } if (!PyTuple_Check(argtup)) { - PyErr_SetString(st->PicklingError, "second item of the tuple " - "returned by __reduce__ must be a tuple"); + PyErr_Format(st->PicklingError, + "second item of the tuple returned by __reduce__ " + "must be a tuple, not %T", argtup); return -1; } @@ -3975,27 +4032,27 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, if (listitems == Py_None) listitems = NULL; else if (!PyIter_Check(listitems)) { - PyErr_Format(st->PicklingError, "fourth element of the tuple " - "returned by __reduce__ must be an iterator, not %s", - Py_TYPE(listitems)->tp_name); + PyErr_Format(st->PicklingError, + "fourth item of the tuple returned by __reduce__ " + "must be an iterator, not %T", listitems); return -1; } if (dictitems == Py_None) dictitems = NULL; else if (!PyIter_Check(dictitems)) { - PyErr_Format(st->PicklingError, "fifth element of the tuple " - "returned by __reduce__ must be an iterator, not %s", - Py_TYPE(dictitems)->tp_name); + PyErr_Format(st->PicklingError, + "fifth item of the tuple returned by __reduce__ " + "must be an iterator, not %T", dictitems); return -1; } if (state_setter == Py_None) state_setter = NULL; else if (!PyCallable_Check(state_setter)) { - PyErr_Format(st->PicklingError, "sixth element of the tuple " - "returned by __reduce__ must be a function, not %s", - Py_TYPE(state_setter)->tp_name); + PyErr_Format(st->PicklingError, + "sixth item of the tuple returned by __reduce__ " + "must be callable, not %T", state_setter); return -1; } @@ -4021,38 +4078,45 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, if (PyTuple_GET_SIZE(argtup) != 3) { PyErr_Format(st->PicklingError, - "length of the NEWOBJ_EX argument tuple must be " - "exactly 3, not %zd", PyTuple_GET_SIZE(argtup)); + "__newobj_ex__ expected 3 arguments, got %zd", + PyTuple_GET_SIZE(argtup)); return -1; } cls = PyTuple_GET_ITEM(argtup, 0); if (!PyType_Check(cls)) { PyErr_Format(st->PicklingError, - "first item from NEWOBJ_EX argument tuple must " - "be a class, not %.200s", Py_TYPE(cls)->tp_name); + "first argument to __newobj_ex__() " + "must be a class, not %T", cls); return -1; } args = PyTuple_GET_ITEM(argtup, 1); if (!PyTuple_Check(args)) { PyErr_Format(st->PicklingError, - "second item from NEWOBJ_EX argument tuple must " - "be a tuple, not %.200s", Py_TYPE(args)->tp_name); + "second argument to __newobj_ex__() " + "must be a tuple, not %T", args); return -1; } kwargs = PyTuple_GET_ITEM(argtup, 2); if (!PyDict_Check(kwargs)) { PyErr_Format(st->PicklingError, - "third item from NEWOBJ_EX argument tuple must " - "be a dict, not %.200s", Py_TYPE(kwargs)->tp_name); + "third argument to __newobj_ex__() " + "must be a dict, not %T", kwargs); return -1; } if (self->proto >= 4) { - if (save(st, self, cls, 0) < 0 || - save(st, self, args, 0) < 0 || - save(st, self, kwargs, 0) < 0 || - _Pickler_Write(self, &newobj_ex_op, 1) < 0) { + if (save(st, self, cls, 0) < 0) { + _PyErr_FormatNote("when serializing %T class", obj); + return -1; + } + if (save(st, self, args, 0) < 0 || + save(st, self, kwargs, 0) < 0) + { + _PyErr_FormatNote("when serializing %T __new__ arguments", obj); + return -1; + } + if (_Pickler_Write(self, &newobj_ex_op, 1) < 0) { return -1; } } @@ -4089,14 +4153,18 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, } if (save(st, self, callable, 0) < 0 || - save(st, self, newargs, 0) < 0 || - _Pickler_Write(self, &reduce_op, 1) < 0) { + save(st, self, newargs, 0) < 0) + { + _PyErr_FormatNote("when serializing %T reconstructor", obj); Py_DECREF(newargs); Py_DECREF(callable); return -1; } Py_DECREF(newargs); Py_DECREF(callable); + if (_Pickler_Write(self, &reduce_op, 1) < 0) { + return -1; + } } } else if (use_newobj) { @@ -4107,14 +4175,17 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, /* Sanity checks. */ if (PyTuple_GET_SIZE(argtup) < 1) { - PyErr_SetString(st->PicklingError, "__newobj__ arglist is empty"); + PyErr_Format(st->PicklingError, + "__newobj__ expected at least 1 argument, got %zd", + PyTuple_GET_SIZE(argtup)); return -1; } cls = PyTuple_GET_ITEM(argtup, 0); if (!PyType_Check(cls)) { - PyErr_SetString(st->PicklingError, "args[0] from " - "__newobj__ args is not a type"); + PyErr_Format(st->PicklingError, + "first argument to __newobj__() " + "must be a class, not %T", cls); return -1; } @@ -4123,13 +4194,14 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, if (obj_class == NULL) { return -1; } - p = obj_class != cls; - Py_DECREF(obj_class); - if (p) { - PyErr_SetString(st->PicklingError, "args[0] from " - "__newobj__ args has the wrong class"); + if (obj_class != cls) { + PyErr_Format(st->PicklingError, + "first argument to __newobj__() " + "must be %R, not %R", obj_class, cls); + Py_DECREF(obj_class); return -1; } + Py_DECREF(obj_class); } /* XXX: These calls save() are prone to infinite recursion. Imagine what happen if the value returned by the __reduce__() method of @@ -4160,6 +4232,7 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, /* Save the class and its __new__ arguments. */ if (save(st, self, cls, 0) < 0) { + _PyErr_FormatNote("when serializing %T class", obj); return -1; } @@ -4169,18 +4242,27 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, p = save(st, self, newargtup, 0); Py_DECREF(newargtup); - if (p < 0) + if (p < 0) { + _PyErr_FormatNote("when serializing %T __new__ arguments", obj); return -1; + } /* Add NEWOBJ opcode. */ if (_Pickler_Write(self, &newobj_op, 1) < 0) return -1; } else { /* Not using NEWOBJ. */ - if (save(st, self, callable, 0) < 0 || - save(st, self, argtup, 0) < 0 || - _Pickler_Write(self, &reduce_op, 1) < 0) + if (save(st, self, callable, 0) < 0) { + _PyErr_FormatNote("when serializing %T reconstructor", obj); + return -1; + } + if (save(st, self, argtup, 0) < 0) { + _PyErr_FormatNote("when serializing %T reconstructor arguments", obj); + return -1; + } + if (_Pickler_Write(self, &reduce_op, 1) < 0) { return -1; + } } /* obj can be NULL when save_reduce() is used directly. A NULL obj means @@ -4205,16 +4287,19 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, return -1; } - if (listitems && batch_list(st, self, listitems) < 0) + if (listitems && batch_list(st, self, listitems, obj) < 0) return -1; - if (dictitems && batch_dict(st, self, dictitems) < 0) + if (dictitems && batch_dict(st, self, dictitems, obj) < 0) return -1; if (state) { if (state_setter == NULL) { - if (save(st, self, state, 0) < 0 || - _Pickler_Write(self, &build_op, 1) < 0) + if (save(st, self, state, 0) < 0) { + _PyErr_FormatNote("when serializing %T state", obj); + return -1; + } + if (_Pickler_Write(self, &build_op, 1) < 0) return -1; } else { @@ -4230,9 +4315,18 @@ save_reduce(PickleState *st, PicklerObject *self, PyObject *args, const char tupletwo_op = TUPLE2; const char pop_op = POP; - if (save(st, self, state_setter, 0) < 0 || - save(st, self, obj, 0) < 0 || save(st, self, state, 0) < 0 || - _Pickler_Write(self, &tupletwo_op, 1) < 0 || + if (save(st, self, state_setter, 0) < 0) { + _PyErr_FormatNote("when serializing %T state setter", obj); + return -1; + } + if (save(st, self, obj, 0) < 0) { + return -1; + } + if (save(st, self, state, 0) < 0) { + _PyErr_FormatNote("when serializing %T state", obj); + return -1; + } + if (_Pickler_Write(self, &tupletwo_op, 1) < 0 || _Pickler_Write(self, &reduce_op, 1) < 0 || _Pickler_Write(self, &pop_op, 1) < 0) return -1; @@ -4422,8 +4516,7 @@ save(PickleState *st, PicklerObject *self, PyObject *obj, int pers_save) } else { PyErr_Format(st->PicklingError, - "can't pickle '%.200s' object: %R", - type->tp_name, obj); + "Can't pickle %T object", obj); goto error; } } @@ -4439,12 +4532,16 @@ save(PickleState *st, PicklerObject *self, PyObject *obj, int pers_save) } if (!PyTuple_Check(reduce_value)) { - PyErr_SetString(st->PicklingError, - "__reduce__ must return a string or tuple"); + PyErr_Format(st->PicklingError, + "__reduce__ must return a string or tuple, not %T", reduce_value); + _PyErr_FormatNote("when serializing %T object", obj); goto error; } status = save_reduce(st, self, reduce_value, obj); + if (status < 0) { + _PyErr_FormatNote("when serializing %T object", obj); + } if (0) { error: @@ -4686,7 +4783,7 @@ This takes a binary file for writing a pickle data stream. The optional *protocol* argument tells the pickler to use the given protocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default -protocol is 4. It was introduced in Python 3.4, and is incompatible +protocol is 5. It was introduced in Python 3.8, and is incompatible with previous versions. Specifying a negative protocol version selects the highest protocol @@ -4719,7 +4816,7 @@ static int _pickle_Pickler___init___impl(PicklerObject *self, PyObject *file, PyObject *protocol, int fix_imports, PyObject *buffer_callback) -/*[clinic end generated code: output=0abedc50590d259b input=a7c969699bf5dad3]*/ +/*[clinic end generated code: output=0abedc50590d259b input=cddc50f66b770002]*/ { /* In case of multiple __init__() calls, clear previous content. */ if (self->write != NULL) @@ -5001,6 +5098,33 @@ Pickler_set_memo(PicklerObject *self, PyObject *obj, void *Py_UNUSED(ignored)) return -1; } +static PyObject * +Pickler_getattr(PyObject *self, PyObject *name) +{ + if (PyUnicode_Check(name) + && PyUnicode_EqualToUTF8(name, "persistent_id") + && ((PicklerObject *)self)->persistent_id_attr) + { + return Py_NewRef(((PicklerObject *)self)->persistent_id_attr); + } + + return PyObject_GenericGetAttr(self, name); +} + +static int +Pickler_setattr(PyObject *self, PyObject *name, PyObject *value) +{ + if (PyUnicode_Check(name) + && PyUnicode_EqualToUTF8(name, "persistent_id")) + { + Py_XINCREF(value); + Py_XSETREF(((PicklerObject *)self)->persistent_id_attr, value); + return 0; + } + + return PyObject_GenericSetAttr(self, name, value); +} + static PyMemberDef Pickler_members[] = { {"bin", Py_T_INT, offsetof(PicklerObject, bin)}, {"fast", Py_T_INT, offsetof(PicklerObject, fast)}, @@ -5016,6 +5140,8 @@ static PyGetSetDef Pickler_getsets[] = { static PyType_Slot pickler_type_slots[] = { {Py_tp_dealloc, Pickler_dealloc}, + {Py_tp_getattro, Pickler_getattr}, + {Py_tp_setattro, Pickler_setattr}, {Py_tp_methods, Pickler_methods}, {Py_tp_members, Pickler_members}, {Py_tp_getset, Pickler_getsets}, @@ -5087,16 +5213,14 @@ load_int(PickleState *state, UnpicklerObject *self) return bad_readline(state); errno = 0; - /* XXX: Should the base argument of strtol() be explicitly set to 10? - XXX(avassalotti): Should this uses PyOS_strtol()? */ - x = strtol(s, &endptr, 0); + /* XXX(avassalotti): Should this uses PyOS_strtol()? */ + x = strtol(s, &endptr, 10); if (errno || (*endptr != '\n' && *endptr != '\0')) { /* Hm, maybe we've got something long. Let's try reading * it as a Python int object. */ errno = 0; - /* XXX: Same thing about the base here. */ - value = PyLong_FromString(s, NULL, 0); + value = PyLong_FromString(s, NULL, 10); if (value == NULL) { PyErr_SetString(PyExc_ValueError, "could not convert string to int"); @@ -5246,8 +5370,7 @@ load_long(PickleState *state, UnpicklerObject *self) the 'L' to be present. */ if (s[len-2] == 'L') s[len-2] = '\0'; - /* XXX: Should the base argument explicitly set to 10? */ - value = PyLong_FromString(s, NULL, 0); + value = PyLong_FromString(s, NULL, 10); if (value == NULL) return -1; @@ -6518,11 +6641,13 @@ load_additems(PickleState *state, UnpicklerObject *self) if (result == NULL) { Pdata_clear(self->stack, i + 1); Py_SET_SIZE(self->stack, mark); + Py_DECREF(add_func); return -1; } Py_DECREF(result); } Py_SET_SIZE(self->stack, mark); + Py_DECREF(add_func); } return 0; @@ -6598,10 +6723,13 @@ load_build(PickleState *st, UnpicklerObject *self) /* normally the keys for instance attributes are interned. we should try to do that here. */ Py_INCREF(d_key); - if (PyUnicode_CheckExact(d_key)) - PyUnicode_InternInPlace(&d_key); + if (PyUnicode_CheckExact(d_key)) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternMortal(interp, &d_key); + } if (PyObject_SetItem(dict, d_key, d_value) < 0) { Py_DECREF(d_key); + Py_DECREF(dict); goto error; } Py_DECREF(d_key); @@ -7033,7 +7161,26 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self, PyTypeObject *cls, if (module == NULL) { return NULL; } - global = getattribute(module, global_name, self->proto >= 4); + if (self->proto >= 4) { + PyObject *dotted_path = get_dotted_path(global_name); + if (dotted_path == NULL) { + Py_DECREF(module); + return NULL; + } + global = getattribute(module, dotted_path, 1); + assert(global != NULL || PyErr_Occurred()); + if (global == NULL && PyList_GET_SIZE(dotted_path) > 1) { + PyObject *exc = PyErr_GetRaisedException(); + PyErr_Format(PyExc_AttributeError, + "Can't resolve path %R on module %R", + global_name, module_name); + _PyErr_ChainExceptions1(exc); + } + Py_DECREF(dotted_path); + } + else { + global = PyObject_GetAttr(module, global_name); + } Py_DECREF(module); return global; } @@ -7452,6 +7599,33 @@ Unpickler_set_memo(UnpicklerObject *self, PyObject *obj, void *Py_UNUSED(ignored return -1; } +static PyObject * +Unpickler_getattr(PyObject *self, PyObject *name) +{ + if (PyUnicode_Check(name) + && PyUnicode_EqualToUTF8(name, "persistent_load") + && ((UnpicklerObject *)self)->persistent_load_attr) + { + return Py_NewRef(((UnpicklerObject *)self)->persistent_load_attr); + } + + return PyObject_GenericGetAttr(self, name); +} + +static int +Unpickler_setattr(PyObject *self, PyObject *name, PyObject *value) +{ + if (PyUnicode_Check(name) + && PyUnicode_EqualToUTF8(name, "persistent_load")) + { + Py_XINCREF(value); + Py_XSETREF(((UnpicklerObject *)self)->persistent_load_attr, value); + return 0; + } + + return PyObject_GenericSetAttr(self, name, value); +} + static PyGetSetDef Unpickler_getsets[] = { {"memo", (getter)Unpickler_get_memo, (setter)Unpickler_set_memo}, {NULL} @@ -7460,6 +7634,8 @@ static PyGetSetDef Unpickler_getsets[] = { static PyType_Slot unpickler_type_slots[] = { {Py_tp_dealloc, Unpickler_dealloc}, {Py_tp_doc, (char *)_pickle_Unpickler___init____doc__}, + {Py_tp_getattro, Unpickler_getattr}, + {Py_tp_setattro, Unpickler_setattr}, {Py_tp_traverse, Unpickler_traverse}, {Py_tp_clear, Unpickler_clear}, {Py_tp_methods, Unpickler_methods}, @@ -7497,7 +7673,7 @@ be more efficient. The optional *protocol* argument tells the pickler to use the given protocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default -protocol is 4. It was introduced in Python 3.4, and is incompatible +protocol is 5. It was introduced in Python 3.8, and is incompatible with previous versions. Specifying a negative protocol version selects the highest protocol @@ -7523,7 +7699,7 @@ static PyObject * _pickle_dump_impl(PyObject *module, PyObject *obj, PyObject *file, PyObject *protocol, int fix_imports, PyObject *buffer_callback) -/*[clinic end generated code: output=706186dba996490c input=5ed6653da99cd97c]*/ +/*[clinic end generated code: output=706186dba996490c input=b89ce8d0e911fd46]*/ { PickleState *state = _Pickle_GetState(module); PicklerObject *pickler = _Pickler_New(state); @@ -7568,7 +7744,7 @@ Return the pickled representation of the object as a bytes object. The optional *protocol* argument tells the pickler to use the given protocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default -protocol is 4. It was introduced in Python 3.4, and is incompatible +protocol is 5. It was introduced in Python 3.8, and is incompatible with previous versions. Specifying a negative protocol version selects the highest protocol @@ -7588,7 +7764,7 @@ into *file* as part of the pickle stream. It is an error if static PyObject * _pickle_dumps_impl(PyObject *module, PyObject *obj, PyObject *protocol, int fix_imports, PyObject *buffer_callback) -/*[clinic end generated code: output=fbab0093a5580fdf input=e543272436c6f987]*/ +/*[clinic end generated code: output=fbab0093a5580fdf input=139fc546886c63ac]*/ { PyObject *result; PickleState *state = _Pickle_GetState(module); @@ -7856,6 +8032,7 @@ _pickle_exec(PyObject *m) static PyModuleDef_Slot pickle_slots[] = { {Py_mod_exec, _pickle_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index bcbbe70680b8e7..ad6d7ceda84e37 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -977,7 +977,6 @@ _posixsubprocess.fork_exec as subprocess_fork_exec uid as uid_object: object child_umask: int preexec_fn: object - allow_vfork: bool / Spawn a fresh new child process. @@ -1014,8 +1013,8 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, pid_t pgid_to_set, PyObject *gid_object, PyObject *extra_groups_packed, PyObject *uid_object, int child_umask, - PyObject *preexec_fn, int allow_vfork) -/*[clinic end generated code: output=7ee4f6ee5cf22b5b input=51757287ef266ffa]*/ + PyObject *preexec_fn) +/*[clinic end generated code: output=288464dc56e373c7 input=f311c3bcb5dd55c8]*/ { PyObject *converted_args = NULL, *fast_args = NULL; PyObject *preexec_fn_args_tuple = NULL; @@ -1031,7 +1030,9 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, Py_ssize_t fds_to_keep_len = PyTuple_GET_SIZE(py_fds_to_keep); PyInterpreterState *interp = _PyInterpreterState_GET(); - if ((preexec_fn != Py_None) && interp->finalizing) { + if ((preexec_fn != Py_None) && + _PyInterpreterState_GetFinalizing(interp) != NULL) + { PyErr_SetString(PyExc_PythonFinalizationError, "preexec_fn not supported at interpreter shutdown"); return NULL; @@ -1216,7 +1217,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, #ifdef VFORK_USABLE /* Use vfork() only if it's safe. See the comment above child_exec(). */ sigset_t old_sigs; - if (preexec_fn == Py_None && allow_vfork && + if (preexec_fn == Py_None && uid == (uid_t)-1 && gid == (gid_t)-1 && extra_group_size < 0) { /* Block all signals to ensure that no signal handlers are run in the * child process while it shares memory with us. Note that signals @@ -1315,6 +1316,7 @@ static PyMethodDef module_methods[] = { static PyModuleDef_Slot _posixsubprocess_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 5db9b645849fcd..aee8db802d8c3f 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -594,6 +594,7 @@ queuemodule_exec(PyObject *module) static PyModuleDef_Slot queuemodule_slots[] = { {Py_mod_exec, queuemodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 56b891dfe0f85f..ad66df47349db0 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -295,7 +295,8 @@ random_seed(RandomObject *self, PyObject *arg) int result = -1; /* guilty until proved innocent */ PyObject *n = NULL; uint32_t *key = NULL; - size_t bits, keyused; + int64_t bits; + size_t keyused; int res; if (arg == NULL || arg == Py_None) { @@ -334,11 +335,11 @@ random_seed(RandomObject *self, PyObject *arg) /* Now split n into 32-bit chunks, from the right. */ bits = _PyLong_NumBits(n); - if (bits == (size_t)-1 && PyErr_Occurred()) - goto Done; + assert(bits >= 0); + assert(!PyErr_Occurred()); /* Figure out how many 32-bit chunks this gives us. */ - keyused = bits == 0 ? 1 : (bits - 1) / 32 + 1; + keyused = bits == 0 ? 1 : (size_t)((bits - 1) / 32 + 1); /* Convert seed to byte sequence. */ key = (uint32_t *)PyMem_Malloc((size_t)4 * keyused); @@ -642,6 +643,7 @@ _random_exec(PyObject *module) static PyModuleDef_Slot _random_slots[] = { {Py_mod_exec, _random_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_scproxy.c b/Modules/_scproxy.c index fe82e918677f9a..e9170f2ce1ae87 100644 --- a/Modules/_scproxy.c +++ b/Modules/_scproxy.c @@ -3,11 +3,10 @@ * using the SystemConfiguration framework. */ +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include @@ -240,6 +239,7 @@ static PyMethodDef mod_methods[] = { static PyModuleDef_Slot _scproxy_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index f099020c5f4e6f..d1a549a971c24a 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -4,7 +4,6 @@ #include "blob.h" #include "util.h" -#include "pycore_weakref.h" // _PyWeakref_GET_REF() #define clinic_state() (pysqlite_get_state_by_type(Py_TYPE(self))) #include "clinic/blob.c.h" @@ -100,10 +99,10 @@ blob_close_impl(pysqlite_Blob *self) void pysqlite_close_all_blobs(pysqlite_Connection *self) { - for (int i = 0; i < PyList_GET_SIZE(self->blobs); i++) { + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(self->blobs); i++) { PyObject *weakref = PyList_GET_ITEM(self->blobs, i); - PyObject *blob = _PyWeakref_GET_REF(weakref); - if (blob == NULL) { + PyObject *blob; + if (!PyWeakref_GetRef(weakref, &blob)) { continue; } close_blob((pysqlite_Blob *)blob); diff --git a/Modules/_sqlite/clinic/blob.c.h b/Modules/_sqlite/clinic/blob.c.h index b95ba948aaf97f..921e7cbd7ffcab 100644 --- a/Modules/_sqlite/clinic/blob.c.h +++ b/Modules/_sqlite/clinic/blob.c.h @@ -17,9 +17,9 @@ static PyObject * blob_close_impl(pysqlite_Blob *self); static PyObject * -blob_close(pysqlite_Blob *self, PyObject *Py_UNUSED(ignored)) +blob_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return blob_close_impl(self); + return blob_close_impl((pysqlite_Blob *)self); } PyDoc_STRVAR(blob_read__doc__, @@ -42,7 +42,7 @@ static PyObject * blob_read_impl(pysqlite_Blob *self, int length); static PyObject * -blob_read(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) +blob_read(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int length = -1; @@ -58,7 +58,7 @@ blob_read(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: - return_value = blob_read_impl(self, length); + return_value = blob_read_impl((pysqlite_Blob *)self, length); exit: return return_value; @@ -80,7 +80,7 @@ static PyObject * blob_write_impl(pysqlite_Blob *self, Py_buffer *data); static PyObject * -blob_write(pysqlite_Blob *self, PyObject *arg) +blob_write(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; Py_buffer data = {NULL, NULL}; @@ -88,7 +88,7 @@ blob_write(pysqlite_Blob *self, PyObject *arg) if (PyObject_GetBuffer(arg, &data, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = blob_write_impl(self, &data); + return_value = blob_write_impl((pysqlite_Blob *)self, &data); exit: /* Cleanup for data */ @@ -116,7 +116,7 @@ static PyObject * blob_seek_impl(pysqlite_Blob *self, int offset, int origin); static PyObject * -blob_seek(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) +blob_seek(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int offset; @@ -137,7 +137,7 @@ blob_seek(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: - return_value = blob_seek_impl(self, offset, origin); + return_value = blob_seek_impl((pysqlite_Blob *)self, offset, origin); exit: return return_value; @@ -156,9 +156,9 @@ static PyObject * blob_tell_impl(pysqlite_Blob *self); static PyObject * -blob_tell(pysqlite_Blob *self, PyObject *Py_UNUSED(ignored)) +blob_tell(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return blob_tell_impl(self); + return blob_tell_impl((pysqlite_Blob *)self); } PyDoc_STRVAR(blob_enter__doc__, @@ -174,9 +174,9 @@ static PyObject * blob_enter_impl(pysqlite_Blob *self); static PyObject * -blob_enter(pysqlite_Blob *self, PyObject *Py_UNUSED(ignored)) +blob_enter(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return blob_enter_impl(self); + return blob_enter_impl((pysqlite_Blob *)self); } PyDoc_STRVAR(blob_exit__doc__, @@ -193,7 +193,7 @@ blob_exit_impl(pysqlite_Blob *self, PyObject *type, PyObject *val, PyObject *tb); static PyObject * -blob_exit(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) +blob_exit(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *type; @@ -206,9 +206,9 @@ blob_exit(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) type = args[0]; val = args[1]; tb = args[2]; - return_value = blob_exit_impl(self, type, val, tb); + return_value = blob_exit_impl((pysqlite_Blob *)self, type, val, tb); exit: return return_value; } -/*[clinic end generated code: output=31abd55660e0c5af input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f03f4ba622b67ae0 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index 811314b5cd8aed..82fba44eb1b074 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -81,7 +81,8 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) goto exit; } } - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 8, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 8, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -181,7 +182,7 @@ static PyObject * pysqlite_connection_cursor_impl(pysqlite_Connection *self, PyObject *factory); static PyObject * -pysqlite_connection_cursor(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +pysqlite_connection_cursor(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -213,7 +214,8 @@ pysqlite_connection_cursor(pysqlite_Connection *self, PyObject *const *args, Py_ Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *factory = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -222,7 +224,7 @@ pysqlite_connection_cursor(pysqlite_Connection *self, PyObject *const *args, Py_ } factory = args[0]; skip_optional_pos: - return_value = pysqlite_connection_cursor_impl(self, factory); + return_value = pysqlite_connection_cursor_impl((pysqlite_Connection *)self, factory); exit: return return_value; @@ -253,7 +255,7 @@ blobopen_impl(pysqlite_Connection *self, const char *table, const char *col, sqlite3_int64 row, int readonly, const char *name); static PyObject * -blobopen(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +blobopen(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -289,7 +291,8 @@ blobopen(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyO int readonly = 0; const char *name = "main"; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -348,7 +351,7 @@ blobopen(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyO goto exit; } skip_optional_kwonly: - return_value = blobopen_impl(self, table, col, row, readonly, name); + return_value = blobopen_impl((pysqlite_Connection *)self, table, col, row, readonly, name); exit: return return_value; @@ -369,9 +372,9 @@ static PyObject * pysqlite_connection_close_impl(pysqlite_Connection *self); static PyObject * -pysqlite_connection_close(pysqlite_Connection *self, PyObject *Py_UNUSED(ignored)) +pysqlite_connection_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return pysqlite_connection_close_impl(self); + return pysqlite_connection_close_impl((pysqlite_Connection *)self); } PyDoc_STRVAR(pysqlite_connection_commit__doc__, @@ -389,9 +392,9 @@ static PyObject * pysqlite_connection_commit_impl(pysqlite_Connection *self); static PyObject * -pysqlite_connection_commit(pysqlite_Connection *self, PyObject *Py_UNUSED(ignored)) +pysqlite_connection_commit(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return pysqlite_connection_commit_impl(self); + return pysqlite_connection_commit_impl((pysqlite_Connection *)self); } PyDoc_STRVAR(pysqlite_connection_rollback__doc__, @@ -409,9 +412,9 @@ static PyObject * pysqlite_connection_rollback_impl(pysqlite_Connection *self); static PyObject * -pysqlite_connection_rollback(pysqlite_Connection *self, PyObject *Py_UNUSED(ignored)) +pysqlite_connection_rollback(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return pysqlite_connection_rollback_impl(self); + return pysqlite_connection_rollback_impl((pysqlite_Connection *)self); } PyDoc_STRVAR(pysqlite_connection_create_function__doc__, @@ -446,7 +449,7 @@ pysqlite_connection_create_function_impl(pysqlite_Connection *self, #endif static PyObject * -pysqlite_connection_create_function(pysqlite_Connection *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +pysqlite_connection_create_function(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -481,7 +484,8 @@ pysqlite_connection_create_function(pysqlite_Connection *self, PyTypeObject *cls PyObject *func; int deterministic = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -521,7 +525,7 @@ pysqlite_connection_create_function(pysqlite_Connection *self, PyTypeObject *cls goto exit; } skip_optional_kwonly: - return_value = pysqlite_connection_create_function_impl(self, cls, name, narg, func, deterministic); + return_value = pysqlite_connection_create_function_impl((pysqlite_Connection *)self, cls, name, narg, func, deterministic); exit: return return_value; @@ -553,7 +557,7 @@ create_window_function_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *aggregate_class); static PyObject * -create_window_function(pysqlite_Connection *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +create_window_function(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -574,7 +578,8 @@ create_window_function(pysqlite_Connection *self, PyTypeObject *cls, PyObject *c int num_params; PyObject *aggregate_class; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -596,7 +601,7 @@ create_window_function(pysqlite_Connection *self, PyTypeObject *cls, PyObject *c goto exit; } aggregate_class = args[2]; - return_value = create_window_function_impl(self, cls, name, num_params, aggregate_class); + return_value = create_window_function_impl((pysqlite_Connection *)self, cls, name, num_params, aggregate_class); exit: return return_value; @@ -637,7 +642,7 @@ pysqlite_connection_create_aggregate_impl(pysqlite_Connection *self, #endif static PyObject * -pysqlite_connection_create_aggregate(pysqlite_Connection *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +pysqlite_connection_create_aggregate(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -670,7 +675,8 @@ pysqlite_connection_create_aggregate(pysqlite_Connection *self, PyTypeObject *cl int n_arg; PyObject *aggregate_class; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -702,7 +708,7 @@ pysqlite_connection_create_aggregate(pysqlite_Connection *self, PyTypeObject *cl goto exit; } aggregate_class = args[2]; - return_value = pysqlite_connection_create_aggregate_impl(self, cls, name, n_arg, aggregate_class); + return_value = pysqlite_connection_create_aggregate_impl((pysqlite_Connection *)self, cls, name, n_arg, aggregate_class); exit: return return_value; @@ -739,12 +745,12 @@ pysqlite_connection_set_authorizer_impl(pysqlite_Connection *self, #endif static PyObject * -pysqlite_connection_set_authorizer(pysqlite_Connection *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +pysqlite_connection_set_authorizer(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 1 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -770,7 +776,8 @@ pysqlite_connection_set_authorizer(pysqlite_Connection *self, PyTypeObject *cls, PyObject *argsbuf[1]; PyObject *callable; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -785,7 +792,7 @@ pysqlite_connection_set_authorizer(pysqlite_Connection *self, PyTypeObject *cls, } } callable = args[0]; - return_value = pysqlite_connection_set_authorizer_impl(self, cls, callable); + return_value = pysqlite_connection_set_authorizer_impl((pysqlite_Connection *)self, cls, callable); exit: return return_value; @@ -832,19 +839,19 @@ pysqlite_connection_set_progress_handler_impl(pysqlite_Connection *self, #endif static PyObject * -pysqlite_connection_set_progress_handler(pysqlite_Connection *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +pysqlite_connection_set_progress_handler(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 3 + #define NUM_KEYWORDS 2 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(progress_handler), &_Py_ID(n), }, + .ob_item = { &_Py_ID(progress_handler), _Py_LATIN1_CHR('n'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -864,7 +871,8 @@ pysqlite_connection_set_progress_handler(pysqlite_Connection *self, PyTypeObject PyObject *callable; int n; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -883,7 +891,7 @@ pysqlite_connection_set_progress_handler(pysqlite_Connection *self, PyTypeObject if (n == -1 && PyErr_Occurred()) { goto exit; } - return_value = pysqlite_connection_set_progress_handler_impl(self, cls, callable, n); + return_value = pysqlite_connection_set_progress_handler_impl((pysqlite_Connection *)self, cls, callable, n); exit: return return_value; @@ -920,12 +928,12 @@ pysqlite_connection_set_trace_callback_impl(pysqlite_Connection *self, #endif static PyObject * -pysqlite_connection_set_trace_callback(pysqlite_Connection *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +pysqlite_connection_set_trace_callback(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 1 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -951,7 +959,8 @@ pysqlite_connection_set_trace_callback(pysqlite_Connection *self, PyTypeObject * PyObject *argsbuf[1]; PyObject *callable; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -966,7 +975,7 @@ pysqlite_connection_set_trace_callback(pysqlite_Connection *self, PyTypeObject * } } callable = args[0]; - return_value = pysqlite_connection_set_trace_callback_impl(self, cls, callable); + return_value = pysqlite_connection_set_trace_callback_impl((pysqlite_Connection *)self, cls, callable); exit: return return_value; @@ -988,7 +997,7 @@ pysqlite_connection_enable_load_extension_impl(pysqlite_Connection *self, int onoff); static PyObject * -pysqlite_connection_enable_load_extension(pysqlite_Connection *self, PyObject *arg) +pysqlite_connection_enable_load_extension(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; int onoff; @@ -997,7 +1006,7 @@ pysqlite_connection_enable_load_extension(pysqlite_Connection *self, PyObject *a if (onoff < 0) { goto exit; } - return_value = pysqlite_connection_enable_load_extension_impl(self, onoff); + return_value = pysqlite_connection_enable_load_extension_impl((pysqlite_Connection *)self, onoff); exit: return return_value; @@ -1022,7 +1031,7 @@ pysqlite_connection_load_extension_impl(pysqlite_Connection *self, const char *entrypoint); static PyObject * -pysqlite_connection_load_extension(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +pysqlite_connection_load_extension(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -1055,7 +1064,8 @@ pysqlite_connection_load_extension(pysqlite_Connection *self, PyObject *const *a const char *extension_name; const char *entrypoint = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1094,7 +1104,7 @@ pysqlite_connection_load_extension(pysqlite_Connection *self, PyObject *const *a goto exit; } skip_optional_kwonly: - return_value = pysqlite_connection_load_extension_impl(self, extension_name, entrypoint); + return_value = pysqlite_connection_load_extension_impl((pysqlite_Connection *)self, extension_name, entrypoint); exit: return return_value; @@ -1116,7 +1126,7 @@ pysqlite_connection_execute_impl(pysqlite_Connection *self, PyObject *sql, PyObject *parameters); static PyObject * -pysqlite_connection_execute(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs) +pysqlite_connection_execute(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *sql; @@ -1135,7 +1145,7 @@ pysqlite_connection_execute(pysqlite_Connection *self, PyObject *const *args, Py } parameters = args[1]; skip_optional: - return_value = pysqlite_connection_execute_impl(self, sql, parameters); + return_value = pysqlite_connection_execute_impl((pysqlite_Connection *)self, sql, parameters); exit: return return_value; @@ -1155,7 +1165,7 @@ pysqlite_connection_executemany_impl(pysqlite_Connection *self, PyObject *sql, PyObject *parameters); static PyObject * -pysqlite_connection_executemany(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs) +pysqlite_connection_executemany(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *sql; @@ -1170,7 +1180,7 @@ pysqlite_connection_executemany(pysqlite_Connection *self, PyObject *const *args } sql = args[0]; parameters = args[1]; - return_value = pysqlite_connection_executemany_impl(self, sql, parameters); + return_value = pysqlite_connection_executemany_impl((pysqlite_Connection *)self, sql, parameters); exit: return return_value; @@ -1198,9 +1208,9 @@ static PyObject * pysqlite_connection_interrupt_impl(pysqlite_Connection *self); static PyObject * -pysqlite_connection_interrupt(pysqlite_Connection *self, PyObject *Py_UNUSED(ignored)) +pysqlite_connection_interrupt(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return pysqlite_connection_interrupt_impl(self); + return pysqlite_connection_interrupt_impl((pysqlite_Connection *)self); } PyDoc_STRVAR(pysqlite_connection_iterdump__doc__, @@ -1220,7 +1230,7 @@ pysqlite_connection_iterdump_impl(pysqlite_Connection *self, PyObject *filter); static PyObject * -pysqlite_connection_iterdump(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +pysqlite_connection_iterdump(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -1252,7 +1262,8 @@ pysqlite_connection_iterdump(pysqlite_Connection *self, PyObject *const *args, P Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *filter = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1261,7 +1272,7 @@ pysqlite_connection_iterdump(pysqlite_Connection *self, PyObject *const *args, P } filter = args[0]; skip_optional_kwonly: - return_value = pysqlite_connection_iterdump_impl(self, filter); + return_value = pysqlite_connection_iterdump_impl((pysqlite_Connection *)self, filter); exit: return return_value; @@ -1284,7 +1295,7 @@ pysqlite_connection_backup_impl(pysqlite_Connection *self, double sleep); static PyObject * -pysqlite_connection_backup(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +pysqlite_connection_backup(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -1320,7 +1331,8 @@ pysqlite_connection_backup(pysqlite_Connection *self, PyObject *const *args, Py_ const char *name = "main"; double sleep = 0.25; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1376,7 +1388,7 @@ pysqlite_connection_backup(pysqlite_Connection *self, PyObject *const *args, Py_ } } skip_optional_kwonly: - return_value = pysqlite_connection_backup_impl(self, target, pages, progress, name, sleep); + return_value = pysqlite_connection_backup_impl((pysqlite_Connection *)self, target, pages, progress, name, sleep); exit: return return_value; @@ -1398,7 +1410,7 @@ pysqlite_connection_create_collation_impl(pysqlite_Connection *self, PyObject *callable); static PyObject * -pysqlite_connection_create_collation(pysqlite_Connection *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +pysqlite_connection_create_collation(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -1418,7 +1430,8 @@ pysqlite_connection_create_collation(pysqlite_Connection *self, PyTypeObject *cl const char *name; PyObject *callable; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1436,7 +1449,7 @@ pysqlite_connection_create_collation(pysqlite_Connection *self, PyTypeObject *cl goto exit; } callable = args[1]; - return_value = pysqlite_connection_create_collation_impl(self, cls, name, callable); + return_value = pysqlite_connection_create_collation_impl((pysqlite_Connection *)self, cls, name, callable); exit: return return_value; @@ -1465,7 +1478,7 @@ static PyObject * serialize_impl(pysqlite_Connection *self, const char *name); static PyObject * -serialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +serialize(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -1497,7 +1510,8 @@ serialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, Py Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; const char *name = "main"; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1518,7 +1532,7 @@ serialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, Py goto exit; } skip_optional_kwonly: - return_value = serialize_impl(self, name); + return_value = serialize_impl((pysqlite_Connection *)self, name); exit: return return_value; @@ -1554,7 +1568,7 @@ deserialize_impl(pysqlite_Connection *self, Py_buffer *data, const char *name); static PyObject * -deserialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +deserialize(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -1587,7 +1601,8 @@ deserialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, Py_buffer data = {NULL, NULL}; const char *name = "main"; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1623,7 +1638,7 @@ deserialize(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, goto exit; } skip_optional_kwonly: - return_value = deserialize_impl(self, &data, name); + return_value = deserialize_impl((pysqlite_Connection *)self, &data, name); exit: /* Cleanup for data */ @@ -1651,9 +1666,9 @@ static PyObject * pysqlite_connection_enter_impl(pysqlite_Connection *self); static PyObject * -pysqlite_connection_enter(pysqlite_Connection *self, PyObject *Py_UNUSED(ignored)) +pysqlite_connection_enter(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return pysqlite_connection_enter_impl(self); + return pysqlite_connection_enter_impl((pysqlite_Connection *)self); } PyDoc_STRVAR(pysqlite_connection_exit__doc__, @@ -1672,7 +1687,7 @@ pysqlite_connection_exit_impl(pysqlite_Connection *self, PyObject *exc_type, PyObject *exc_value, PyObject *exc_tb); static PyObject * -pysqlite_connection_exit(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs) +pysqlite_connection_exit(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *exc_type; @@ -1685,7 +1700,7 @@ pysqlite_connection_exit(pysqlite_Connection *self, PyObject *const *args, Py_ss exc_type = args[0]; exc_value = args[1]; exc_tb = args[2]; - return_value = pysqlite_connection_exit_impl(self, exc_type, exc_value, exc_tb); + return_value = pysqlite_connection_exit_impl((pysqlite_Connection *)self, exc_type, exc_value, exc_tb); exit: return return_value; @@ -1714,7 +1729,7 @@ static PyObject * setlimit_impl(pysqlite_Connection *self, int category, int limit); static PyObject * -setlimit(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs) +setlimit(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int category; @@ -1731,7 +1746,7 @@ setlimit(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs) if (limit == -1 && PyErr_Occurred()) { goto exit; } - return_value = setlimit_impl(self, category, limit); + return_value = setlimit_impl((pysqlite_Connection *)self, category, limit); exit: return return_value; @@ -1753,7 +1768,7 @@ static PyObject * getlimit_impl(pysqlite_Connection *self, int category); static PyObject * -getlimit(pysqlite_Connection *self, PyObject *arg) +getlimit(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; int category; @@ -1762,7 +1777,7 @@ getlimit(pysqlite_Connection *self, PyObject *arg) if (category == -1 && PyErr_Occurred()) { goto exit; } - return_value = getlimit_impl(self, category); + return_value = getlimit_impl((pysqlite_Connection *)self, category); exit: return return_value; @@ -1784,7 +1799,7 @@ static PyObject * setconfig_impl(pysqlite_Connection *self, int op, int enable); static PyObject * -setconfig(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs) +setconfig(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int op; @@ -1805,7 +1820,7 @@ setconfig(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: - return_value = setconfig_impl(self, op, enable); + return_value = setconfig_impl((pysqlite_Connection *)self, op, enable); exit: return return_value; @@ -1827,7 +1842,7 @@ static int getconfig_impl(pysqlite_Connection *self, int op); static PyObject * -getconfig(pysqlite_Connection *self, PyObject *arg) +getconfig(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; int op; @@ -1837,7 +1852,7 @@ getconfig(pysqlite_Connection *self, PyObject *arg) if (op == -1 && PyErr_Occurred()) { goto exit; } - _return_value = getconfig_impl(self, op); + _return_value = getconfig_impl((pysqlite_Connection *)self, op); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -1866,4 +1881,4 @@ getconfig(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=3c6d0b748fac016f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c59effb407b8ea4d input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/cursor.c.h b/Modules/_sqlite/clinic/cursor.c.h index a13e0d0745b58d..590e429e9139f1 100644 --- a/Modules/_sqlite/clinic/cursor.c.h +++ b/Modules/_sqlite/clinic/cursor.c.h @@ -52,7 +52,7 @@ pysqlite_cursor_execute_impl(pysqlite_Cursor *self, PyObject *sql, PyObject *parameters); static PyObject * -pysqlite_cursor_execute(pysqlite_Cursor *self, PyObject *const *args, Py_ssize_t nargs) +pysqlite_cursor_execute(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *sql; @@ -71,7 +71,7 @@ pysqlite_cursor_execute(pysqlite_Cursor *self, PyObject *const *args, Py_ssize_t } parameters = args[1]; skip_optional: - return_value = pysqlite_cursor_execute_impl(self, sql, parameters); + return_value = pysqlite_cursor_execute_impl((pysqlite_Cursor *)self, sql, parameters); exit: return return_value; @@ -91,7 +91,7 @@ pysqlite_cursor_executemany_impl(pysqlite_Cursor *self, PyObject *sql, PyObject *seq_of_parameters); static PyObject * -pysqlite_cursor_executemany(pysqlite_Cursor *self, PyObject *const *args, Py_ssize_t nargs) +pysqlite_cursor_executemany(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *sql; @@ -106,7 +106,7 @@ pysqlite_cursor_executemany(pysqlite_Cursor *self, PyObject *const *args, Py_ssi } sql = args[0]; seq_of_parameters = args[1]; - return_value = pysqlite_cursor_executemany_impl(self, sql, seq_of_parameters); + return_value = pysqlite_cursor_executemany_impl((pysqlite_Cursor *)self, sql, seq_of_parameters); exit: return return_value; @@ -126,7 +126,7 @@ pysqlite_cursor_executescript_impl(pysqlite_Cursor *self, const char *sql_script); static PyObject * -pysqlite_cursor_executescript(pysqlite_Cursor *self, PyObject *arg) +pysqlite_cursor_executescript(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; const char *sql_script; @@ -144,7 +144,7 @@ pysqlite_cursor_executescript(pysqlite_Cursor *self, PyObject *arg) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - return_value = pysqlite_cursor_executescript_impl(self, sql_script); + return_value = pysqlite_cursor_executescript_impl((pysqlite_Cursor *)self, sql_script); exit: return return_value; @@ -163,9 +163,9 @@ static PyObject * pysqlite_cursor_fetchone_impl(pysqlite_Cursor *self); static PyObject * -pysqlite_cursor_fetchone(pysqlite_Cursor *self, PyObject *Py_UNUSED(ignored)) +pysqlite_cursor_fetchone(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return pysqlite_cursor_fetchone_impl(self); + return pysqlite_cursor_fetchone_impl((pysqlite_Cursor *)self); } PyDoc_STRVAR(pysqlite_cursor_fetchmany__doc__, @@ -184,7 +184,7 @@ static PyObject * pysqlite_cursor_fetchmany_impl(pysqlite_Cursor *self, int maxrows); static PyObject * -pysqlite_cursor_fetchmany(pysqlite_Cursor *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +pysqlite_cursor_fetchmany(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -214,9 +214,10 @@ pysqlite_cursor_fetchmany(pysqlite_Cursor *self, PyObject *const *args, Py_ssize #undef KWTUPLE PyObject *argsbuf[1]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - int maxrows = self->arraysize; + int maxrows = ((pysqlite_Cursor *)self)->arraysize; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -228,7 +229,7 @@ pysqlite_cursor_fetchmany(pysqlite_Cursor *self, PyObject *const *args, Py_ssize goto exit; } skip_optional_pos: - return_value = pysqlite_cursor_fetchmany_impl(self, maxrows); + return_value = pysqlite_cursor_fetchmany_impl((pysqlite_Cursor *)self, maxrows); exit: return return_value; @@ -247,9 +248,9 @@ static PyObject * pysqlite_cursor_fetchall_impl(pysqlite_Cursor *self); static PyObject * -pysqlite_cursor_fetchall(pysqlite_Cursor *self, PyObject *Py_UNUSED(ignored)) +pysqlite_cursor_fetchall(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return pysqlite_cursor_fetchall_impl(self); + return pysqlite_cursor_fetchall_impl((pysqlite_Cursor *)self); } PyDoc_STRVAR(pysqlite_cursor_setinputsizes__doc__, @@ -275,7 +276,7 @@ pysqlite_cursor_setoutputsize_impl(pysqlite_Cursor *self, PyObject *size, PyObject *column); static PyObject * -pysqlite_cursor_setoutputsize(pysqlite_Cursor *self, PyObject *const *args, Py_ssize_t nargs) +pysqlite_cursor_setoutputsize(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *size; @@ -290,7 +291,7 @@ pysqlite_cursor_setoutputsize(pysqlite_Cursor *self, PyObject *const *args, Py_s } column = args[1]; skip_optional: - return_value = pysqlite_cursor_setoutputsize_impl(self, size, column); + return_value = pysqlite_cursor_setoutputsize_impl((pysqlite_Cursor *)self, size, column); exit: return return_value; @@ -309,8 +310,8 @@ static PyObject * pysqlite_cursor_close_impl(pysqlite_Cursor *self); static PyObject * -pysqlite_cursor_close(pysqlite_Cursor *self, PyObject *Py_UNUSED(ignored)) +pysqlite_cursor_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return pysqlite_cursor_close_impl(self); + return pysqlite_cursor_close_impl((pysqlite_Cursor *)self); } -/*[clinic end generated code: output=a8ce095c3c80cf65 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=82620ca7622b547c input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/module.c.h b/Modules/_sqlite/clinic/module.c.h index 529dc4e281e0eb..5569c45451647f 100644 --- a/Modules/_sqlite/clinic/module.c.h +++ b/Modules/_sqlite/clinic/module.c.h @@ -52,7 +52,8 @@ pysqlite_complete_statement(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *argsbuf[1]; const char *statement; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -208,4 +209,4 @@ pysqlite_adapt(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=457ab0fdbb9e1880 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=db9bf7ecad197343 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/row.c.h b/Modules/_sqlite/clinic/row.c.h index e8d1dbf2ba8bc9..068906744e445f 100644 --- a/Modules/_sqlite/clinic/row.c.h +++ b/Modules/_sqlite/clinic/row.c.h @@ -52,8 +52,8 @@ static PyObject * pysqlite_row_keys_impl(pysqlite_Row *self); static PyObject * -pysqlite_row_keys(pysqlite_Row *self, PyObject *Py_UNUSED(ignored)) +pysqlite_row_keys(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return pysqlite_row_keys_impl(self); + return pysqlite_row_keys_impl((pysqlite_Row *)self); } -/*[clinic end generated code: output=788bf817acc02b8e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6c1acbb48f386468 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index f97afcf5fcf16e..62598ecc864120 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -38,7 +38,7 @@ #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() -#include "pycore_weakref.h" // _PyWeakref_IS_DEAD() +#include "pycore_weakref.h" #include @@ -958,6 +958,11 @@ step_callback(sqlite3_context *context, int argc, sqlite3_value **params) assert(ctx != NULL); aggregate_instance = (PyObject**)sqlite3_aggregate_context(context, sizeof(PyObject*)); + if (aggregate_instance == NULL) { + (void)PyErr_NoMemory(); + set_sqlite_error(context, "unable to allocate SQLite aggregate context"); + goto error; + } if (*aggregate_instance == NULL) { *aggregate_instance = PyObject_CallNoArgs(ctx->callable); if (!*aggregate_instance) { @@ -1065,7 +1070,7 @@ static void _pysqlite_drop_unused_cursor_references(pysqlite_Connection* self) for (Py_ssize_t i = 0; i < PyList_Size(self->cursors); i++) { PyObject* weakref = PyList_GetItem(self->cursors, i); - if (_PyWeakref_IS_DEAD(weakref)) { + if (_PyWeakref_IsDead(weakref)) { continue; } if (PyList_Append(new_list, weakref) != 0) { @@ -2561,6 +2566,12 @@ set_autocommit(pysqlite_Connection *self, PyObject *val, void *Py_UNUSED(ctx)) return 0; } +static PyObject * +get_sig(PyObject *self, void *Py_UNUSED(ctx)) +{ + return PyUnicode_FromString("(sql, /)"); +} + static const char connection_doc[] = PyDoc_STR("SQLite database connection object."); @@ -2570,6 +2581,7 @@ static PyGetSetDef connection_getset[] = { {"total_changes", (getter)pysqlite_connection_get_total_changes, (setter)0}, {"in_transaction", (getter)pysqlite_connection_get_in_transaction, (setter)0}, {"autocommit", (getter)get_autocommit, (setter)set_autocommit}, + {"__text_signature__", get_sig, (setter)0}, {NULL} }; diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index f95df612328e57..24e97fcf1897e9 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -669,16 +669,13 @@ bind_parameters(pysqlite_state *state, pysqlite_Statement *self, } for (i = 0; i < num_params; i++) { const char *name = sqlite3_bind_parameter_name(self->st, i+1); - if (name != NULL) { - int ret = PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + if (name != NULL && name[0] != '?') { + PyErr_Format(state->ProgrammingError, "Binding %d ('%s') is a named parameter, but you " "supplied a sequence which requires nameless (qmark) " - "placeholders. Starting with Python 3.14 an " - "sqlite3.ProgrammingError will be raised.", + "placeholders.", i+1, name); - if (ret < 0) { - return; - } + return; } if (PyTuple_CheckExact(parameters)) { @@ -1158,7 +1155,7 @@ pysqlite_cursor_fetchone_impl(pysqlite_Cursor *self) /*[clinic input] _sqlite3.Cursor.fetchmany as pysqlite_cursor_fetchmany - size as maxrows: int(c_default='self->arraysize') = 1 + size as maxrows: int(c_default='((pysqlite_Cursor *)self)->arraysize') = 1 The default value is set by the Cursor.arraysize attribute. Fetches several rows from the resultset. @@ -1166,7 +1163,7 @@ Fetches several rows from the resultset. static PyObject * pysqlite_cursor_fetchmany_impl(pysqlite_Cursor *self, int maxrows) -/*[clinic end generated code: output=a8ef31fea64d0906 input=c26e6ca3f34debd0]*/ +/*[clinic end generated code: output=a8ef31fea64d0906 input=035dbe44a1005bf2]*/ { PyObject* row; PyObject* list; diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 46fed9f13281f3..698e81d9b897d0 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -714,10 +714,6 @@ module_exec(PyObject *module) goto error; } - if (PyModule_AddStringConstant(module, "_deprecated_version", PYSQLITE_VERSION) < 0) { - goto error; - } - if (PyModule_AddStringConstant(module, "sqlite_version", sqlite3_libversion())) { goto error; } @@ -758,6 +754,7 @@ module_exec(PyObject *module) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_sqlite/util.c b/Modules/_sqlite/util.c index 9e8613ef67916e..b0622e66928f47 100644 --- a/Modules/_sqlite/util.c +++ b/Modules/_sqlite/util.c @@ -134,6 +134,7 @@ _pysqlite_seterror(pysqlite_state *state, sqlite3 *db) /* Create and set the exception. */ int extended_errcode = sqlite3_extended_errcode(db); + // sqlite3_errmsg() always returns an UTF-8 encoded message const char *errmsg = sqlite3_errmsg(db); raise_exception(exc_class, extended_errcode, errmsg); return extended_errcode; diff --git a/Modules/_sre/clinic/sre.c.h b/Modules/_sre/clinic/sre.c.h index 48336c7a2fca26..cfc6813f37f012 100644 --- a/Modules/_sre/clinic/sre.c.h +++ b/Modules/_sre/clinic/sre.c.h @@ -179,7 +179,7 @@ _sre_SRE_Pattern_match_impl(PatternObject *self, PyTypeObject *cls, Py_ssize_t endpos); static PyObject * -_sre_SRE_Pattern_match(PatternObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_sre_SRE_Pattern_match(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -213,7 +213,8 @@ _sre_SRE_Pattern_match(PatternObject *self, PyTypeObject *cls, PyObject *const * Py_ssize_t pos = 0; Py_ssize_t endpos = PY_SSIZE_T_MAX; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -251,7 +252,7 @@ _sre_SRE_Pattern_match(PatternObject *self, PyTypeObject *cls, PyObject *const * endpos = ival; } skip_optional_pos: - return_value = _sre_SRE_Pattern_match_impl(self, cls, string, pos, endpos); + return_value = _sre_SRE_Pattern_match_impl((PatternObject *)self, cls, string, pos, endpos); exit: return return_value; @@ -272,7 +273,7 @@ _sre_SRE_Pattern_fullmatch_impl(PatternObject *self, PyTypeObject *cls, Py_ssize_t endpos); static PyObject * -_sre_SRE_Pattern_fullmatch(PatternObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_sre_SRE_Pattern_fullmatch(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -306,7 +307,8 @@ _sre_SRE_Pattern_fullmatch(PatternObject *self, PyTypeObject *cls, PyObject *con Py_ssize_t pos = 0; Py_ssize_t endpos = PY_SSIZE_T_MAX; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -344,7 +346,7 @@ _sre_SRE_Pattern_fullmatch(PatternObject *self, PyTypeObject *cls, PyObject *con endpos = ival; } skip_optional_pos: - return_value = _sre_SRE_Pattern_fullmatch_impl(self, cls, string, pos, endpos); + return_value = _sre_SRE_Pattern_fullmatch_impl((PatternObject *)self, cls, string, pos, endpos); exit: return return_value; @@ -367,7 +369,7 @@ _sre_SRE_Pattern_search_impl(PatternObject *self, PyTypeObject *cls, Py_ssize_t endpos); static PyObject * -_sre_SRE_Pattern_search(PatternObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_sre_SRE_Pattern_search(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -401,7 +403,8 @@ _sre_SRE_Pattern_search(PatternObject *self, PyTypeObject *cls, PyObject *const Py_ssize_t pos = 0; Py_ssize_t endpos = PY_SSIZE_T_MAX; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -439,7 +442,7 @@ _sre_SRE_Pattern_search(PatternObject *self, PyTypeObject *cls, PyObject *const endpos = ival; } skip_optional_pos: - return_value = _sre_SRE_Pattern_search_impl(self, cls, string, pos, endpos); + return_value = _sre_SRE_Pattern_search_impl((PatternObject *)self, cls, string, pos, endpos); exit: return return_value; @@ -459,7 +462,7 @@ _sre_SRE_Pattern_findall_impl(PatternObject *self, PyObject *string, Py_ssize_t pos, Py_ssize_t endpos); static PyObject * -_sre_SRE_Pattern_findall(PatternObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_sre_SRE_Pattern_findall(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -493,7 +496,8 @@ _sre_SRE_Pattern_findall(PatternObject *self, PyObject *const *args, Py_ssize_t Py_ssize_t pos = 0; Py_ssize_t endpos = PY_SSIZE_T_MAX; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -531,7 +535,7 @@ _sre_SRE_Pattern_findall(PatternObject *self, PyObject *const *args, Py_ssize_t endpos = ival; } skip_optional_pos: - return_value = _sre_SRE_Pattern_findall_impl(self, string, pos, endpos); + return_value = _sre_SRE_Pattern_findall_impl((PatternObject *)self, string, pos, endpos); exit: return return_value; @@ -554,7 +558,7 @@ _sre_SRE_Pattern_finditer_impl(PatternObject *self, PyTypeObject *cls, Py_ssize_t endpos); static PyObject * -_sre_SRE_Pattern_finditer(PatternObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_sre_SRE_Pattern_finditer(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -588,7 +592,8 @@ _sre_SRE_Pattern_finditer(PatternObject *self, PyTypeObject *cls, PyObject *cons Py_ssize_t pos = 0; Py_ssize_t endpos = PY_SSIZE_T_MAX; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -626,7 +631,7 @@ _sre_SRE_Pattern_finditer(PatternObject *self, PyTypeObject *cls, PyObject *cons endpos = ival; } skip_optional_pos: - return_value = _sre_SRE_Pattern_finditer_impl(self, cls, string, pos, endpos); + return_value = _sre_SRE_Pattern_finditer_impl((PatternObject *)self, cls, string, pos, endpos); exit: return return_value; @@ -646,7 +651,7 @@ _sre_SRE_Pattern_scanner_impl(PatternObject *self, PyTypeObject *cls, Py_ssize_t endpos); static PyObject * -_sre_SRE_Pattern_scanner(PatternObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_sre_SRE_Pattern_scanner(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -680,7 +685,8 @@ _sre_SRE_Pattern_scanner(PatternObject *self, PyTypeObject *cls, PyObject *const Py_ssize_t pos = 0; Py_ssize_t endpos = PY_SSIZE_T_MAX; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -718,7 +724,7 @@ _sre_SRE_Pattern_scanner(PatternObject *self, PyTypeObject *cls, PyObject *const endpos = ival; } skip_optional_pos: - return_value = _sre_SRE_Pattern_scanner_impl(self, cls, string, pos, endpos); + return_value = _sre_SRE_Pattern_scanner_impl((PatternObject *)self, cls, string, pos, endpos); exit: return return_value; @@ -738,7 +744,7 @@ _sre_SRE_Pattern_split_impl(PatternObject *self, PyObject *string, Py_ssize_t maxsplit); static PyObject * -_sre_SRE_Pattern_split(PatternObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_sre_SRE_Pattern_split(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -771,7 +777,8 @@ _sre_SRE_Pattern_split(PatternObject *self, PyObject *const *args, Py_ssize_t na PyObject *string; Py_ssize_t maxsplit = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -792,7 +799,7 @@ _sre_SRE_Pattern_split(PatternObject *self, PyObject *const *args, Py_ssize_t na maxsplit = ival; } skip_optional_pos: - return_value = _sre_SRE_Pattern_split_impl(self, string, maxsplit); + return_value = _sre_SRE_Pattern_split_impl((PatternObject *)self, string, maxsplit); exit: return return_value; @@ -812,7 +819,7 @@ _sre_SRE_Pattern_sub_impl(PatternObject *self, PyTypeObject *cls, PyObject *repl, PyObject *string, Py_ssize_t count); static PyObject * -_sre_SRE_Pattern_sub(PatternObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_sre_SRE_Pattern_sub(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -846,7 +853,8 @@ _sre_SRE_Pattern_sub(PatternObject *self, PyTypeObject *cls, PyObject *const *ar PyObject *string; Py_ssize_t count = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -868,7 +876,7 @@ _sre_SRE_Pattern_sub(PatternObject *self, PyTypeObject *cls, PyObject *const *ar count = ival; } skip_optional_pos: - return_value = _sre_SRE_Pattern_sub_impl(self, cls, repl, string, count); + return_value = _sre_SRE_Pattern_sub_impl((PatternObject *)self, cls, repl, string, count); exit: return return_value; @@ -889,7 +897,7 @@ _sre_SRE_Pattern_subn_impl(PatternObject *self, PyTypeObject *cls, Py_ssize_t count); static PyObject * -_sre_SRE_Pattern_subn(PatternObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_sre_SRE_Pattern_subn(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -923,7 +931,8 @@ _sre_SRE_Pattern_subn(PatternObject *self, PyTypeObject *cls, PyObject *const *a PyObject *string; Py_ssize_t count = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -945,7 +954,7 @@ _sre_SRE_Pattern_subn(PatternObject *self, PyTypeObject *cls, PyObject *const *a count = ival; } skip_optional_pos: - return_value = _sre_SRE_Pattern_subn_impl(self, cls, repl, string, count); + return_value = _sre_SRE_Pattern_subn_impl((PatternObject *)self, cls, repl, string, count); exit: return return_value; @@ -963,9 +972,9 @@ static PyObject * _sre_SRE_Pattern___copy___impl(PatternObject *self); static PyObject * -_sre_SRE_Pattern___copy__(PatternObject *self, PyObject *Py_UNUSED(ignored)) +_sre_SRE_Pattern___copy__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _sre_SRE_Pattern___copy___impl(self); + return _sre_SRE_Pattern___copy___impl((PatternObject *)self); } PyDoc_STRVAR(_sre_SRE_Pattern___deepcopy____doc__, @@ -976,6 +985,44 @@ PyDoc_STRVAR(_sre_SRE_Pattern___deepcopy____doc__, #define _SRE_SRE_PATTERN___DEEPCOPY___METHODDEF \ {"__deepcopy__", (PyCFunction)_sre_SRE_Pattern___deepcopy__, METH_O, _sre_SRE_Pattern___deepcopy____doc__}, +#if defined(Py_DEBUG) + +PyDoc_STRVAR(_sre_SRE_Pattern__fail_after__doc__, +"_fail_after($self, count, exception, /)\n" +"--\n" +"\n" +"For debugging."); + +#define _SRE_SRE_PATTERN__FAIL_AFTER_METHODDEF \ + {"_fail_after", _PyCFunction_CAST(_sre_SRE_Pattern__fail_after), METH_FASTCALL, _sre_SRE_Pattern__fail_after__doc__}, + +static PyObject * +_sre_SRE_Pattern__fail_after_impl(PatternObject *self, int count, + PyObject *exception); + +static PyObject * +_sre_SRE_Pattern__fail_after(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int count; + PyObject *exception; + + if (!_PyArg_CheckPositional("_fail_after", nargs, 2, 2)) { + goto exit; + } + count = PyLong_AsInt(args[0]); + if (count == -1 && PyErr_Occurred()) { + goto exit; + } + exception = args[1]; + return_value = _sre_SRE_Pattern__fail_after_impl((PatternObject *)self, count, exception); + +exit: + return return_value; +} + +#endif /* defined(Py_DEBUG) */ + PyDoc_STRVAR(_sre_compile__doc__, "compile($module, /, pattern, flags, code, groups, groupindex,\n" " indexgroup)\n" @@ -1027,7 +1074,8 @@ _sre_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *groupindex; PyObject *indexgroup; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 6, 6, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 6, /*maxpos*/ 6, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1121,7 +1169,7 @@ static PyObject * _sre_SRE_Match_expand_impl(MatchObject *self, PyObject *template); static PyObject * -_sre_SRE_Match_expand(MatchObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_sre_SRE_Match_expand(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -1152,12 +1200,13 @@ _sre_SRE_Match_expand(MatchObject *self, PyObject *const *args, Py_ssize_t nargs PyObject *argsbuf[1]; PyObject *template; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } template = args[0]; - return_value = _sre_SRE_Match_expand_impl(self, template); + return_value = _sre_SRE_Match_expand_impl((MatchObject *)self, template); exit: return return_value; @@ -1179,7 +1228,7 @@ static PyObject * _sre_SRE_Match_groups_impl(MatchObject *self, PyObject *default_value); static PyObject * -_sre_SRE_Match_groups(MatchObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_sre_SRE_Match_groups(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -1211,7 +1260,8 @@ _sre_SRE_Match_groups(MatchObject *self, PyObject *const *args, Py_ssize_t nargs Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *default_value = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1220,7 +1270,7 @@ _sre_SRE_Match_groups(MatchObject *self, PyObject *const *args, Py_ssize_t nargs } default_value = args[0]; skip_optional_pos: - return_value = _sre_SRE_Match_groups_impl(self, default_value); + return_value = _sre_SRE_Match_groups_impl((MatchObject *)self, default_value); exit: return return_value; @@ -1242,7 +1292,7 @@ static PyObject * _sre_SRE_Match_groupdict_impl(MatchObject *self, PyObject *default_value); static PyObject * -_sre_SRE_Match_groupdict(MatchObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_sre_SRE_Match_groupdict(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -1274,7 +1324,8 @@ _sre_SRE_Match_groupdict(MatchObject *self, PyObject *const *args, Py_ssize_t na Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *default_value = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1283,7 +1334,7 @@ _sre_SRE_Match_groupdict(MatchObject *self, PyObject *const *args, Py_ssize_t na } default_value = args[0]; skip_optional_pos: - return_value = _sre_SRE_Match_groupdict_impl(self, default_value); + return_value = _sre_SRE_Match_groupdict_impl((MatchObject *)self, default_value); exit: return return_value; @@ -1302,7 +1353,7 @@ static Py_ssize_t _sre_SRE_Match_start_impl(MatchObject *self, PyObject *group); static PyObject * -_sre_SRE_Match_start(MatchObject *self, PyObject *const *args, Py_ssize_t nargs) +_sre_SRE_Match_start(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *group = NULL; @@ -1316,7 +1367,7 @@ _sre_SRE_Match_start(MatchObject *self, PyObject *const *args, Py_ssize_t nargs) } group = args[0]; skip_optional: - _return_value = _sre_SRE_Match_start_impl(self, group); + _return_value = _sre_SRE_Match_start_impl((MatchObject *)self, group); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -1339,7 +1390,7 @@ static Py_ssize_t _sre_SRE_Match_end_impl(MatchObject *self, PyObject *group); static PyObject * -_sre_SRE_Match_end(MatchObject *self, PyObject *const *args, Py_ssize_t nargs) +_sre_SRE_Match_end(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *group = NULL; @@ -1353,7 +1404,7 @@ _sre_SRE_Match_end(MatchObject *self, PyObject *const *args, Py_ssize_t nargs) } group = args[0]; skip_optional: - _return_value = _sre_SRE_Match_end_impl(self, group); + _return_value = _sre_SRE_Match_end_impl((MatchObject *)self, group); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -1376,7 +1427,7 @@ static PyObject * _sre_SRE_Match_span_impl(MatchObject *self, PyObject *group); static PyObject * -_sre_SRE_Match_span(MatchObject *self, PyObject *const *args, Py_ssize_t nargs) +_sre_SRE_Match_span(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *group = NULL; @@ -1389,7 +1440,7 @@ _sre_SRE_Match_span(MatchObject *self, PyObject *const *args, Py_ssize_t nargs) } group = args[0]; skip_optional: - return_value = _sre_SRE_Match_span_impl(self, group); + return_value = _sre_SRE_Match_span_impl((MatchObject *)self, group); exit: return return_value; @@ -1407,9 +1458,9 @@ static PyObject * _sre_SRE_Match___copy___impl(MatchObject *self); static PyObject * -_sre_SRE_Match___copy__(MatchObject *self, PyObject *Py_UNUSED(ignored)) +_sre_SRE_Match___copy__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _sre_SRE_Match___copy___impl(self); + return _sre_SRE_Match___copy___impl((MatchObject *)self); } PyDoc_STRVAR(_sre_SRE_Match___deepcopy____doc__, @@ -1432,13 +1483,13 @@ static PyObject * _sre_SRE_Scanner_match_impl(ScannerObject *self, PyTypeObject *cls); static PyObject * -_sre_SRE_Scanner_match(ScannerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_sre_SRE_Scanner_match(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "match() takes no arguments"); return NULL; } - return _sre_SRE_Scanner_match_impl(self, cls); + return _sre_SRE_Scanner_match_impl((ScannerObject *)self, cls); } PyDoc_STRVAR(_sre_SRE_Scanner_search__doc__, @@ -1453,12 +1504,16 @@ static PyObject * _sre_SRE_Scanner_search_impl(ScannerObject *self, PyTypeObject *cls); static PyObject * -_sre_SRE_Scanner_search(ScannerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_sre_SRE_Scanner_search(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "search() takes no arguments"); return NULL; } - return _sre_SRE_Scanner_search_impl(self, cls); + return _sre_SRE_Scanner_search_impl((ScannerObject *)self, cls); } -/*[clinic end generated code: output=c3e711f0b2f43d66 input=a9049054013a1b77]*/ + +#ifndef _SRE_SRE_PATTERN__FAIL_AFTER_METHODDEF + #define _SRE_SRE_PATTERN__FAIL_AFTER_METHODDEF +#endif /* !defined(_SRE_SRE_PATTERN__FAIL_AFTER_METHODDEF) */ +/*[clinic end generated code: output=3654103c87eb4830 input=a9049054013a1b77]*/ diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 00fbd9674b8cdd..d0025dd21e045b 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -267,6 +267,85 @@ data_stack_grow(SRE_STATE* state, Py_ssize_t size) return 0; } +/* memory pool functions for SRE_REPEAT, this can avoid memory + leak when SRE(match) function terminates abruptly. + state->repeat_pool_used is a doubly-linked list, so that we + can remove a SRE_REPEAT node from it. + state->repeat_pool_unused is a singly-linked list, we put/get + node at the head. */ +static SRE_REPEAT * +repeat_pool_malloc(SRE_STATE *state) +{ + SRE_REPEAT *repeat; + + if (state->repeat_pool_unused) { + /* remove from unused pool (singly-linked list) */ + repeat = state->repeat_pool_unused; + state->repeat_pool_unused = repeat->pool_next; + } + else { + repeat = PyMem_Malloc(sizeof(SRE_REPEAT)); + if (!repeat) { + return NULL; + } + } + + /* add to used pool (doubly-linked list) */ + SRE_REPEAT *temp = state->repeat_pool_used; + if (temp) { + temp->pool_prev = repeat; + } + repeat->pool_prev = NULL; + repeat->pool_next = temp; + state->repeat_pool_used = repeat; + + return repeat; +} + +static void +repeat_pool_free(SRE_STATE *state, SRE_REPEAT *repeat) +{ + SRE_REPEAT *prev = repeat->pool_prev; + SRE_REPEAT *next = repeat->pool_next; + + /* remove from used pool (doubly-linked list) */ + if (prev) { + prev->pool_next = next; + } + else { + state->repeat_pool_used = next; + } + if (next) { + next->pool_prev = prev; + } + + /* add to unused pool (singly-linked list) */ + repeat->pool_next = state->repeat_pool_unused; + state->repeat_pool_unused = repeat; +} + +static void +repeat_pool_clear(SRE_STATE *state) +{ + /* clear used pool */ + SRE_REPEAT *next = state->repeat_pool_used; + state->repeat_pool_used = NULL; + while (next) { + SRE_REPEAT *temp = next; + next = temp->pool_next; + PyMem_Free(temp); + } + + /* clear unused pool */ + next = state->repeat_pool_unused; + state->repeat_pool_unused = NULL; + while (next) { + SRE_REPEAT *temp = next; + next = temp->pool_next; + PyMem_Free(temp); + } +} + /* generate 8-bit version */ #define SRE_CHAR Py_UCS1 @@ -316,6 +395,11 @@ static struct PyModuleDef sremodule; static PyObject*pattern_new_match(_sremodulestate *, PatternObject*, SRE_STATE*, Py_ssize_t); static PyObject *pattern_scanner(_sremodulestate *, PatternObject *, PyObject *, Py_ssize_t, Py_ssize_t); +#define _PatternObject_CAST(op) ((PatternObject *)(op)) +#define _MatchObject_CAST(op) ((MatchObject *)(op)) +#define _TemplateObject_CAST(op) ((TemplateObject *)(op)) +#define _ScannerObject_CAST(op) ((ScannerObject *)(op)) + /*[clinic input] module _sre class _sre.SRE_Pattern "PatternObject *" "get_sre_module_state_by_class(tp)->Pattern_Type" @@ -511,6 +595,11 @@ state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string, state->pos = start; state->endpos = end; +#ifdef Py_DEBUG + state->fail_after_count = pattern->fail_after_count; + state->fail_after_exc = pattern->fail_after_exc; // borrowed ref +#endif + return string; err: /* We add an explicit cast here because MSVC has a bug when @@ -530,9 +619,11 @@ state_fini(SRE_STATE* state) PyBuffer_Release(&state->buffer); Py_XDECREF(state->string); data_stack_dealloc(state); - /* See above PyMem_Del for why we explicitly cast here. */ + /* See above PyMem_Free() for why we explicitly cast here. */ PyMem_Free((void*) state->mark); state->mark = NULL; + /* SRE_REPEAT pool */ + repeat_pool_clear(state); } /* calculate offset from start of string */ @@ -613,32 +704,40 @@ pattern_error(Py_ssize_t status) } static int -pattern_traverse(PatternObject *self, visitproc visit, void *arg) +pattern_traverse(PyObject *op, visitproc visit, void *arg) { + PatternObject *self = _PatternObject_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->groupindex); Py_VISIT(self->indexgroup); Py_VISIT(self->pattern); +#ifdef Py_DEBUG + Py_VISIT(self->fail_after_exc); +#endif return 0; } static int -pattern_clear(PatternObject *self) +pattern_clear(PyObject *op) { + PatternObject *self = _PatternObject_CAST(op); Py_CLEAR(self->groupindex); Py_CLEAR(self->indexgroup); Py_CLEAR(self->pattern); +#ifdef Py_DEBUG + Py_CLEAR(self->fail_after_exc); +#endif return 0; } static void -pattern_dealloc(PatternObject* self) +pattern_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_UnTrack(self); - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *) self); + PatternObject *obj = _PatternObject_CAST(self); + if (obj->weakreflist != NULL) { + PyObject_ClearWeakRefs(self); } (void)pattern_clear(self); tp->tp_free(self); @@ -690,7 +789,7 @@ _sre_SRE_Pattern_match_impl(PatternObject *self, PyTypeObject *cls, Py_ssize_t status; PyObject *match; - if (!state_init(&state, (PatternObject *)self, string, pos, endpos)) + if (!state_init(&state, self, string, pos, endpos)) return NULL; INIT_TRACE(&state); @@ -1287,7 +1386,7 @@ pattern_subx(_sremodulestate* module_state, } else { if (state.isbytes) - item = _PyBytes_Join(joiner, list); + item = PyBytes_Join(joiner, list); else item = PyUnicode_Join(joiner, list); Py_DECREF(joiner); @@ -1381,8 +1480,31 @@ _sre_SRE_Pattern___deepcopy__(PatternObject *self, PyObject *memo) return Py_NewRef(self); } +#ifdef Py_DEBUG +/*[clinic input] +_sre.SRE_Pattern._fail_after + + count: int + exception: object + / + +For debugging. +[clinic start generated code]*/ + +static PyObject * +_sre_SRE_Pattern__fail_after_impl(PatternObject *self, int count, + PyObject *exception) +/*[clinic end generated code: output=9a6bf12135ac50c2 input=ef80a45c66c5499d]*/ +{ + self->fail_after_count = count; + Py_INCREF(exception); + Py_XSETREF(self->fail_after_exc, exception); + Py_RETURN_NONE; +} +#endif /* Py_DEBUG */ + static PyObject * -pattern_repr(PatternObject *obj) +pattern_repr(PyObject *self) { static const struct { const char *name; @@ -1397,6 +1519,8 @@ pattern_repr(PatternObject *obj) {"re.DEBUG", SRE_FLAG_DEBUG}, {"re.ASCII", SRE_FLAG_ASCII}, }; + + PatternObject *obj = _PatternObject_CAST(self); PyObject *result = NULL; PyObject *flag_items; size_t i; @@ -1464,8 +1588,9 @@ PyDoc_STRVAR(pattern_doc, "Compiled regular expression object."); /* PatternObject's 'groupindex' method. */ static PyObject * -pattern_groupindex(PatternObject *self, void *Py_UNUSED(ignored)) +pattern_groupindex(PyObject *op, void *Py_UNUSED(ignored)) { + PatternObject *self = _PatternObject_CAST(op); if (self->groupindex == NULL) return PyDict_New(); return PyDictProxy_New(self->groupindex); @@ -1506,6 +1631,10 @@ _sre_compile_impl(PyObject *module, PyObject *pattern, int flags, self->pattern = NULL; self->groupindex = NULL; self->indexgroup = NULL; +#ifdef Py_DEBUG + self->fail_after_count = -1; + self->fail_after_exc = NULL; +#endif self->codesize = n; @@ -1622,6 +1751,7 @@ _sre_template_impl(PyObject *module, PyObject *pattern, PyObject *template) } self->items[i].literal = Py_XNewRef(literal); } + PyObject_GC_Track(self); return (PyObject*) self; bad_template: @@ -2125,8 +2255,9 @@ _validate(PatternObject *self) /* match methods */ static int -match_traverse(MatchObject *self, visitproc visit, void *arg) +match_traverse(PyObject *op, visitproc visit, void *arg) { + MatchObject *self = _MatchObject_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->string); Py_VISIT(self->regs); @@ -2135,8 +2266,9 @@ match_traverse(MatchObject *self, visitproc visit, void *arg) } static int -match_clear(MatchObject *self) +match_clear(PyObject *op) { + MatchObject *self = _MatchObject_CAST(op); Py_CLEAR(self->string); Py_CLEAR(self->regs); Py_CLEAR(self->pattern); @@ -2144,10 +2276,9 @@ match_clear(MatchObject *self) } static void -match_dealloc(MatchObject* self) +match_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_UnTrack(self); (void)match_clear(self); tp->tp_free(self); @@ -2216,6 +2347,8 @@ match_getindex(MatchObject* self, PyObject* index) return -1; } + // Check that i*2 cannot overflow to make static analyzers happy + assert(i <= SRE_MAXGROUPS); return i; } @@ -2254,8 +2387,9 @@ _sre_SRE_Match_expand_impl(MatchObject *self, PyObject *template) } static PyObject* -match_group(MatchObject* self, PyObject* args) +match_group(PyObject *op, PyObject* args) { + MatchObject *self = _MatchObject_CAST(op); PyObject* result; Py_ssize_t i, size; @@ -2289,8 +2423,9 @@ match_group(MatchObject* self, PyObject* args) } static PyObject* -match_getitem(MatchObject* self, PyObject* name) +match_getitem(PyObject *op, PyObject* name) { + MatchObject *self = _MatchObject_CAST(op); return match_getslice(self, name, Py_None); } @@ -2368,7 +2503,7 @@ _sre_SRE_Match_groupdict_impl(MatchObject *self, PyObject *default_value) goto exit; } } -exit: +exit:; Py_END_CRITICAL_SECTION(); return result; @@ -2532,16 +2667,18 @@ PyDoc_STRVAR(match_group_doc, For 0 returns the entire match."); static PyObject * -match_lastindex_get(MatchObject *self, void *Py_UNUSED(ignored)) +match_lastindex_get(PyObject *op, void *Py_UNUSED(ignored)) { + MatchObject *self = _MatchObject_CAST(op); if (self->lastindex >= 0) return PyLong_FromSsize_t(self->lastindex); Py_RETURN_NONE; } static PyObject * -match_lastgroup_get(MatchObject *self, void *Py_UNUSED(ignored)) +match_lastgroup_get(PyObject *op, void *Py_UNUSED(ignored)) { + MatchObject *self = _MatchObject_CAST(op); if (self->pattern->indexgroup && self->lastindex >= 0 && self->lastindex < PyTuple_GET_SIZE(self->pattern->indexgroup)) @@ -2554,8 +2691,9 @@ match_lastgroup_get(MatchObject *self, void *Py_UNUSED(ignored)) } static PyObject * -match_regs_get(MatchObject *self, void *Py_UNUSED(ignored)) +match_regs_get(PyObject *op, void *Py_UNUSED(ignored)) { + MatchObject *self = _MatchObject_CAST(op); if (self->regs) { return Py_NewRef(self->regs); } else @@ -2601,7 +2739,8 @@ pattern_new_match(_sremodulestate* module_state, if (!match) return NULL; - match->pattern = (PatternObject*)Py_NewRef(pattern); + Py_INCREF(pattern); + match->pattern = pattern; match->string = Py_NewRef(state->string); @@ -2657,27 +2796,29 @@ pattern_new_match(_sremodulestate* module_state, /* scanner methods (experimental) */ static int -scanner_traverse(ScannerObject *self, visitproc visit, void *arg) +scanner_traverse(PyObject *op, visitproc visit, void *arg) { + ScannerObject *self = _ScannerObject_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->pattern); return 0; } static int -scanner_clear(ScannerObject *self) +scanner_clear(PyObject *op) { + ScannerObject *self = _ScannerObject_CAST(op); Py_CLEAR(self->pattern); return 0; } static void -scanner_dealloc(ScannerObject* self) +scanner_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_UnTrack(self); - state_fini(&self->state); + ScannerObject *scanner = _ScannerObject_CAST(self); + state_fini(&scanner->state); (void)scanner_clear(self); tp->tp_free(self); Py_DECREF(tp); @@ -2737,7 +2878,7 @@ _sre_SRE_Scanner_match_impl(ScannerObject *self, PyTypeObject *cls) return NULL; } - match = pattern_new_match(module_state, (PatternObject*) self->pattern, + match = pattern_new_match(module_state, self->pattern, state, status); if (status == 0) @@ -2787,7 +2928,7 @@ _sre_SRE_Scanner_search_impl(ScannerObject *self, PyTypeObject *cls) return NULL; } - match = pattern_new_match(module_state, (PatternObject*) self->pattern, + match = pattern_new_match(module_state, self->pattern, state, status); if (status == 0) @@ -2823,7 +2964,8 @@ pattern_scanner(_sremodulestate *module_state, return NULL; } - scanner->pattern = Py_NewRef(self); + Py_INCREF(self); + scanner->pattern = self; PyObject_GC_Track(scanner); return (PyObject*) scanner; @@ -2833,8 +2975,9 @@ pattern_scanner(_sremodulestate *module_state, /* template methods */ static int -template_traverse(TemplateObject *self, visitproc visit, void *arg) +template_traverse(PyObject *op, visitproc visit, void *arg) { + TemplateObject *self = _TemplateObject_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->literal); for (Py_ssize_t i = 0, n = Py_SIZE(self); i < n; i++) { @@ -2844,8 +2987,9 @@ template_traverse(TemplateObject *self, visitproc visit, void *arg) } static int -template_clear(TemplateObject *self) +template_clear(PyObject *op) { + TemplateObject *self = _TemplateObject_CAST(op); Py_CLEAR(self->literal); for (Py_ssize_t i = 0, n = Py_SIZE(self); i < n; i++) { Py_CLEAR(self->items[i].literal); @@ -2854,10 +2998,9 @@ template_clear(TemplateObject *self) } static void -template_dealloc(TemplateObject *self) +template_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_UnTrack(self); (void)template_clear(self); tp->tp_free(self); @@ -2915,7 +3058,7 @@ expand_template(TemplateObject *self, MatchObject *match) } else { Py_SET_SIZE(list, count); - result = _PyBytes_Join((PyObject *)&_Py_SINGLETON(bytes_empty), list); + result = PyBytes_Join((PyObject *)&_Py_SINGLETON(bytes_empty), list); } cleanup: @@ -2932,8 +3075,10 @@ expand_template(TemplateObject *self, MatchObject *match) static Py_hash_t -pattern_hash(PatternObject *self) +pattern_hash(PyObject *op) { + PatternObject *self = _PatternObject_CAST(op); + Py_hash_t hash, hash2; hash = PyObject_Hash(self->pattern); @@ -2941,7 +3086,7 @@ pattern_hash(PatternObject *self) return -1; } - hash2 = _Py_HashBytes(self->code, sizeof(self->code[0]) * self->codesize); + hash2 = Py_HashBuffer(self->code, sizeof(self->code[0]) * self->codesize); hash ^= hash2; hash ^= self->flags; @@ -3017,13 +3162,14 @@ static PyMethodDef pattern_methods[] = { _SRE_SRE_PATTERN_SCANNER_METHODDEF _SRE_SRE_PATTERN___COPY___METHODDEF _SRE_SRE_PATTERN___DEEPCOPY___METHODDEF + _SRE_SRE_PATTERN__FAIL_AFTER_METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} }; static PyGetSetDef pattern_getset[] = { - {"groupindex", (getter)pattern_groupindex, (setter)NULL, + {"groupindex", pattern_groupindex, NULL, "A dictionary mapping group names to group numbers."}, {NULL} /* Sentinel */ }; @@ -3041,9 +3187,9 @@ static PyMemberDef pattern_members[] = { }; static PyType_Slot pattern_slots[] = { - {Py_tp_dealloc, (destructor)pattern_dealloc}, - {Py_tp_repr, (reprfunc)pattern_repr}, - {Py_tp_hash, (hashfunc)pattern_hash}, + {Py_tp_dealloc, pattern_dealloc}, + {Py_tp_repr, pattern_repr}, + {Py_tp_hash, pattern_hash}, {Py_tp_doc, (void *)pattern_doc}, {Py_tp_richcompare, pattern_richcompare}, {Py_tp_methods, pattern_methods}, @@ -3064,7 +3210,7 @@ static PyType_Spec pattern_spec = { }; static PyMethodDef match_methods[] = { - {"group", (PyCFunction) match_group, METH_VARARGS, match_group_doc}, + {"group", match_group, METH_VARARGS, match_group_doc}, _SRE_SRE_MATCH_START_METHODDEF _SRE_SRE_MATCH_END_METHODDEF _SRE_SRE_MATCH_SPAN_METHODDEF @@ -3079,11 +3225,11 @@ static PyMethodDef match_methods[] = { }; static PyGetSetDef match_getset[] = { - {"lastindex", (getter)match_lastindex_get, (setter)NULL, + {"lastindex", match_lastindex_get, NULL, "The integer index of the last matched capturing group."}, - {"lastgroup", (getter)match_lastgroup_get, (setter)NULL, + {"lastgroup", match_lastgroup_get, NULL, "The name of the last matched capturing group."}, - {"regs", (getter)match_regs_get, (setter)NULL}, + {"regs", match_regs_get, NULL, NULL}, {NULL} }; @@ -3272,6 +3418,7 @@ sre_exec(PyObject *m) static PyModuleDef_Slot sre_slots[] = { {Py_mod_exec, sre_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_sre/sre.h b/Modules/_sre/sre.h index 83d89d57b11199..42681c2addf3c2 100644 --- a/Modules/_sre/sre.h +++ b/Modules/_sre/sre.h @@ -34,6 +34,11 @@ typedef struct { int flags; /* flags used when compiling pattern source */ PyObject *weakreflist; /* List of weak references */ int isbytes; /* pattern type (1 - bytes, 0 - string, -1 - None) */ +#ifdef Py_DEBUG + /* for simulation of user interruption */ + int fail_after_count; + PyObject *fail_after_exc; +#endif /* pattern code */ Py_ssize_t codesize; SRE_CODE code[1]; @@ -68,6 +73,9 @@ typedef struct SRE_REPEAT_T { const SRE_CODE* pattern; /* points to REPEAT operator arguments */ const void* last_ptr; /* helper to check for infinite loops */ struct SRE_REPEAT_T *prev; /* points to previous repeat context */ + /* for SRE_REPEAT pool */ + struct SRE_REPEAT_T *pool_prev; + struct SRE_REPEAT_T *pool_next; } SRE_REPEAT; typedef struct { @@ -95,12 +103,19 @@ typedef struct { size_t data_stack_base; /* current repeat context */ SRE_REPEAT *repeat; + /* SRE_REPEAT pool */ + SRE_REPEAT *repeat_pool_used; + SRE_REPEAT *repeat_pool_unused; unsigned int sigcount; +#ifdef Py_DEBUG + int fail_after_count; + PyObject *fail_after_exc; +#endif } SRE_STATE; typedef struct { PyObject_HEAD - PyObject* pattern; + PatternObject* pattern; SRE_STATE state; int executing; } ScannerObject; diff --git a/Modules/_sre/sre_lib.h b/Modules/_sre/sre_lib.h index 97fbb0a75e54b6..df377905bfae0d 100644 --- a/Modules/_sre/sre_lib.h +++ b/Modules/_sre/sre_lib.h @@ -42,8 +42,6 @@ SRE(at)(SRE_STATE* state, const SRE_CHAR* ptr, SRE_CODE at) return ((void*) ptr == state->end); case SRE_AT_BOUNDARY: - if (state->beginning == state->end) - return 0; thatp = ((void*) ptr > state->beginning) ? SRE_IS_WORD((int) ptr[-1]) : 0; thisp = ((void*) ptr < state->end) ? @@ -51,8 +49,6 @@ SRE(at)(SRE_STATE* state, const SRE_CHAR* ptr, SRE_CODE at) return thisp != thatp; case SRE_AT_NON_BOUNDARY: - if (state->beginning == state->end) - return 0; thatp = ((void*) ptr > state->beginning) ? SRE_IS_WORD((int) ptr[-1]) : 0; thisp = ((void*) ptr < state->end) ? @@ -60,8 +56,6 @@ SRE(at)(SRE_STATE* state, const SRE_CHAR* ptr, SRE_CODE at) return thisp == thatp; case SRE_AT_LOC_BOUNDARY: - if (state->beginning == state->end) - return 0; thatp = ((void*) ptr > state->beginning) ? SRE_LOC_IS_WORD((int) ptr[-1]) : 0; thisp = ((void*) ptr < state->end) ? @@ -69,8 +63,6 @@ SRE(at)(SRE_STATE* state, const SRE_CHAR* ptr, SRE_CODE at) return thisp != thatp; case SRE_AT_LOC_NON_BOUNDARY: - if (state->beginning == state->end) - return 0; thatp = ((void*) ptr > state->beginning) ? SRE_LOC_IS_WORD((int) ptr[-1]) : 0; thisp = ((void*) ptr < state->end) ? @@ -78,8 +70,6 @@ SRE(at)(SRE_STATE* state, const SRE_CHAR* ptr, SRE_CODE at) return thisp == thatp; case SRE_AT_UNI_BOUNDARY: - if (state->beginning == state->end) - return 0; thatp = ((void*) ptr > state->beginning) ? SRE_UNI_IS_WORD((int) ptr[-1]) : 0; thisp = ((void*) ptr < state->end) ? @@ -87,8 +77,6 @@ SRE(at)(SRE_STATE* state, const SRE_CHAR* ptr, SRE_CODE at) return thisp != thatp; case SRE_AT_UNI_NON_BOUNDARY: - if (state->beginning == state->end) - return 0; thatp = ((void*) ptr > state->beginning) ? SRE_UNI_IS_WORD((int) ptr[-1]) : 0; thisp = ((void*) ptr < state->end) ? @@ -560,13 +548,28 @@ typedef struct { Py_ssize_t last_ctx_pos; } SRE(match_context); -#define MAYBE_CHECK_SIGNALS \ +#define _MAYBE_CHECK_SIGNALS \ do { \ if ((0 == (++sigcount & 0xfff)) && PyErr_CheckSignals()) { \ RETURN_ERROR(SRE_ERROR_INTERRUPTED); \ } \ } while (0) +#ifdef Py_DEBUG +# define MAYBE_CHECK_SIGNALS \ + do { \ + _MAYBE_CHECK_SIGNALS; \ + if (state->fail_after_count >= 0) { \ + if (state->fail_after_count-- == 0) { \ + PyErr_SetNone(state->fail_after_exc); \ + RETURN_ERROR(SRE_ERROR_INTERRUPTED); \ + } \ + } \ + } while (0) +#else +# define MAYBE_CHECK_SIGNALS _MAYBE_CHECK_SIGNALS +#endif /* Py_DEBUG */ + #ifdef HAVE_COMPUTED_GOTOS #ifndef USE_COMPUTED_GOTOS #define USE_COMPUTED_GOTOS 1 @@ -1120,12 +1123,9 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) pattern[1], pattern[2])); /* install new repeat context */ - /* TODO(https://github.com/python/cpython/issues/67877): Fix this - * potential memory leak. */ - ctx->u.rep = (SRE_REPEAT*) PyMem_Malloc(sizeof(*ctx->u.rep)); + ctx->u.rep = repeat_pool_malloc(state); if (!ctx->u.rep) { - PyErr_NoMemory(); - RETURN_FAILURE; + RETURN_ERROR(SRE_ERROR_MEMORY); } ctx->u.rep->count = -1; ctx->u.rep->pattern = pattern; @@ -1136,7 +1136,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) state->ptr = ptr; DO_JUMP(JUMP_REPEAT, jump_repeat, pattern+pattern[0]); state->repeat = ctx->u.rep->prev; - PyMem_Free(ctx->u.rep); + repeat_pool_free(state, ctx->u.rep); if (ret) { RETURN_ON_ERROR(ret); @@ -1294,6 +1294,17 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) pointer */ state->ptr = ptr; + /* Set state->repeat to non-NULL */ + ctx->u.rep = repeat_pool_malloc(state); + if (!ctx->u.rep) { + RETURN_ERROR(SRE_ERROR_MEMORY); + } + ctx->u.rep->count = -1; + ctx->u.rep->pattern = NULL; + ctx->u.rep->prev = state->repeat; + ctx->u.rep->last_ptr = NULL; + state->repeat = ctx->u.rep; + /* Initialize Count to 0 */ ctx->count = 0; @@ -1308,6 +1319,9 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) } else { state->ptr = ptr; + /* Restore state->repeat */ + state->repeat = ctx->u.rep->prev; + repeat_pool_free(state, ctx->u.rep); RETURN_FAILURE; } } @@ -1380,6 +1394,10 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) } } + /* Restore state->repeat */ + state->repeat = ctx->u.rep->prev; + repeat_pool_free(state, ctx->u.rep); + /* Evaluate Tail */ /* Jump to end of pattern indicated by skip, and then skip the SUCCESS op code that follows it. */ diff --git a/Modules/_ssl.c b/Modules/_ssl.c index d00f407b569fb6..c15a582a92aa4a 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -29,7 +29,6 @@ #include "pycore_fileutils.h" // _PyIsSelectable_fd() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_time.h" // _PyDeadline_Init() -#include "pycore_weakref.h" // _PyWeakref_GET_REF() /* Include symbols from _socket module */ #include "socketmodule.h" @@ -52,7 +51,6 @@ PySSL_BEGIN_ALLOW_THREADS_S(_save); #define PySSL_END_ALLOW_THREADS PySSL_END_ALLOW_THREADS_S(_save); } - #if defined(HAVE_POLL_H) #include #elif defined(HAVE_SYS_POLL_H) @@ -122,8 +120,9 @@ static void _PySSLFixErrno(void) { #endif /* Include generated data (error codes) */ +/* See make_ssl_data.h for notes on adding a new version. */ #if (OPENSSL_VERSION_NUMBER >= 0x30100000L) -#include "_ssl_data_31.h" +#include "_ssl_data_34.h" #elif (OPENSSL_VERSION_NUMBER >= 0x30000000L) #include "_ssl_data_300.h" #elif (OPENSSL_VERSION_NUMBER >= 0x10101000L) @@ -188,6 +187,11 @@ extern const SSL_METHOD *TLSv1_2_method(void); #endif +#if defined(SSL_VERIFY_POST_HANDSHAKE) && defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) + #define PySSL_HAVE_POST_HS_AUTH +#endif + + enum py_ssl_error { /* these mirror ssl.h */ PY_SSL_ERROR_NONE, @@ -232,7 +236,7 @@ enum py_proto_version { PY_PROTO_TLSv1 = TLS1_VERSION, PY_PROTO_TLSv1_1 = TLS1_1_VERSION, PY_PROTO_TLSv1_2 = TLS1_2_VERSION, -#ifdef TLS1_3_VERSION +#if defined(TLS1_3_VERSION) PY_PROTO_TLSv1_3 = TLS1_3_VERSION, #else PY_PROTO_TLSv1_3 = 0x304, @@ -294,7 +298,7 @@ typedef struct { */ unsigned int hostflags; int protocol; -#ifdef TLS1_3_VERSION +#if defined(PySSL_HAVE_POST_HS_AUTH) int post_handshake_auth; #endif PyObject *msg_cb; @@ -372,9 +376,6 @@ class _ssl.SSLSession "PySSLSession *" "get_state_type(type)->PySSLSession_Type" static int PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout); -static int PySSL_set_owner(PySSLSocket *, PyObject *, void *); -static int PySSL_set_session(PySSLSocket *, PyObject *, void *); - typedef enum { SOCKET_IS_NONBLOCKING, SOCKET_IS_BLOCKING, @@ -392,8 +393,8 @@ typedef enum { // Return a borrowed reference. static inline PySocketSockObject* GET_SOCKET(PySSLSocket *obj) { if (obj->Socket) { - PyObject *sock = _PyWeakref_GET_REF(obj->Socket); - if (sock != NULL) { + PyObject *sock; + if (PyWeakref_GetRef(obj->Socket, &sock)) { // GET_SOCKET() returns a borrowed reference Py_DECREF(sock); } @@ -472,6 +473,7 @@ fill_and_set_sslerror(_sslmodulestate *state, PyObject *err_value = NULL, *reason_obj = NULL, *lib_obj = NULL; PyObject *verify_obj = NULL, *verify_code_obj = NULL; PyObject *init_value, *msg, *key; + PyUnicodeWriter *writer = NULL; if (errcode != 0) { int lib, reason; @@ -494,11 +496,10 @@ fill_and_set_sslerror(_sslmodulestate *state, if (lib_obj == NULL && PyErr_Occurred()) { goto fail; } - if (errstr == NULL) + if (errstr == NULL) { errstr = ERR_reason_error_string(errcode); + } } - if (errstr == NULL) - errstr = "unknown error"; /* verify code for cert validation error */ if ((sslsock != NULL) && (type == state->PySSLCertVerificationErrorObject)) { @@ -538,20 +539,50 @@ fill_and_set_sslerror(_sslmodulestate *state, } } - if (verify_obj && reason_obj && lib_obj) - msg = PyUnicode_FromFormat("[%S: %S] %s: %S (_ssl.c:%d)", - lib_obj, reason_obj, errstr, verify_obj, - lineno); - else if (reason_obj && lib_obj) - msg = PyUnicode_FromFormat("[%S: %S] %s (_ssl.c:%d)", - lib_obj, reason_obj, errstr, lineno); - else if (lib_obj) - msg = PyUnicode_FromFormat("[%S] %s (_ssl.c:%d)", - lib_obj, errstr, lineno); - else - msg = PyUnicode_FromFormat("%s (_ssl.c:%d)", errstr, lineno); - if (msg == NULL) + // Format message roughly as: + // [lib_obj: reason_obj] errstr: verify_obj (_ssl.c:lineno) + // with parts missing/replaced if unavailable + writer = PyUnicodeWriter_Create(64); + if (!writer) { + goto fail; + } + if (lib_obj) { + if (PyUnicodeWriter_Format(writer, "[%S", lib_obj) < 0) { + goto fail; + } + if (reason_obj) { + if (PyUnicodeWriter_Format(writer, ": %S", reason_obj) < 0) { + goto fail; + } + } + if (PyUnicodeWriter_WriteUTF8(writer, "] ", 2) < 0) { + goto fail; + } + } + if (errstr) { + if (PyUnicodeWriter_Format(writer, "%s", errstr) < 0) { + goto fail; + } + } + else { + if (PyUnicodeWriter_Format( + writer, "unknown error (0x%x)", errcode) < 0) { + goto fail; + } + } + if (verify_obj) { + if (PyUnicodeWriter_Format(writer, ": %S", verify_obj) < 0) { + goto fail; + } + } + if (PyUnicodeWriter_Format(writer, " (_ssl.c:%d)", lineno) < 0) { + goto fail; + } + msg = PyUnicodeWriter_Finish(writer); + writer = NULL; + if (!msg) { goto fail; + } init_value = Py_BuildValue("iN", ERR_GET_REASON(ssl_errno), msg); if (init_value == NULL) @@ -586,6 +617,7 @@ fill_and_set_sslerror(_sslmodulestate *state, Py_XDECREF(err_value); Py_XDECREF(verify_code_obj); Py_XDECREF(verify_obj); + PyUnicodeWriter_Discard(writer); } static int @@ -599,7 +631,7 @@ PySSL_ChainExceptions(PySSLSocket *sslsock) { } static PyObject * -PySSL_SetError(PySSLSocket *sslsock, int ret, const char *filename, int lineno) +PySSL_SetError(PySSLSocket *sslsock, const char *filename, int lineno) { PyObject *type; char *errstr = NULL; @@ -612,7 +644,7 @@ PySSL_SetError(PySSLSocket *sslsock, int ret, const char *filename, int lineno) _sslmodulestate *state = get_state_sock(sslsock); type = state->PySSLErrorObject; - assert(ret <= 0); + // ERR functions are thread local, no need to lock them. e = ERR_peek_last_error(); if (sslsock->ssl != NULL) { @@ -645,38 +677,32 @@ PySSL_SetError(PySSLSocket *sslsock, int ret, const char *filename, int lineno) case SSL_ERROR_SYSCALL: { if (e == 0) { - PySocketSockObject *s = GET_SOCKET(sslsock); - if (ret == 0 || (((PyObject *)s) == Py_None)) { + /* underlying BIO reported an I/O error */ + ERR_clear_error(); +#ifdef MS_WINDOWS + if (err.ws) { + return PyErr_SetFromWindowsErr(err.ws); + } +#endif + if (err.c) { + errno = err.c; + return PyErr_SetFromErrno(PyExc_OSError); + } + else { p = PY_SSL_ERROR_EOF; type = state->PySSLEOFErrorObject; errstr = "EOF occurred in violation of protocol"; - } else if (s && ret == -1) { - /* underlying BIO reported an I/O error */ - ERR_clear_error(); -#ifdef MS_WINDOWS - if (err.ws) { - return PyErr_SetFromWindowsErr(err.ws); - } -#endif - if (err.c) { - errno = err.c; - return PyErr_SetFromErrno(PyExc_OSError); - } - else { - p = PY_SSL_ERROR_EOF; - type = state->PySSLEOFErrorObject; - errstr = "EOF occurred in violation of protocol"; - } - } else { /* possible? */ - p = PY_SSL_ERROR_SYSCALL; - type = state->PySSLSyscallErrorObject; - errstr = "Some I/O error occurred"; } } else { if (ERR_GET_LIB(e) == ERR_LIB_SSL && ERR_GET_REASON(e) == SSL_R_CERTIFICATE_VERIFY_FAILED) { type = state->PySSLCertVerificationErrorObject; } + if (ERR_GET_LIB(e) == ERR_LIB_SYS) { + // A system error is being reported; reason is set to errno + errno = ERR_GET_REASON(e); + return PyErr_SetFromErrno(PyExc_OSError); + } p = PY_SSL_ERROR_SYSCALL; } break; @@ -702,6 +728,11 @@ PySSL_SetError(PySSLSocket *sslsock, int ret, const char *filename, int lineno) errstr = "EOF occurred in violation of protocol"; } #endif + if (ERR_GET_LIB(e) == ERR_LIB_SYS) { + // A system error is being reported; reason is set to errno + errno = ERR_GET_REASON(e); + return PyErr_SetFromErrno(PyExc_OSError); + } break; } default: @@ -886,7 +917,7 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, SSL_set_mode(self->ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER | SSL_MODE_AUTO_RETRY); -#ifdef TLS1_3_VERSION +#if defined(PySSL_HAVE_POST_HS_AUTH) if (sslctx->post_handshake_auth == 1) { if (socket_type == PY_SSL_SERVER) { /* bpo-37428: OpenSSL does not ignore SSL_VERIFY_POST_HANDSHAKE. @@ -934,13 +965,13 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, } } if (owner && owner != Py_None) { - if (PySSL_set_owner(self, owner, NULL) == -1) { + if (_ssl__SSLSocket_owner_set((PyObject *)self, owner, NULL) < 0) { Py_DECREF(self); return NULL; } } if (session && session != Py_None) { - if (PySSL_set_session(self, session, NULL) == -1) { + if (_ssl__SSLSocket_session_set((PyObject *)self, session, NULL) < 0) { Py_DECREF(self); return NULL; } @@ -953,12 +984,13 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, /* SSL object methods */ /*[clinic input] +@critical_section _ssl._SSLSocket.do_handshake [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) -/*[clinic end generated code: output=6c0898a8936548f6 input=d2d737de3df018c8]*/ +/*[clinic end generated code: output=6c0898a8936548f6 input=65619a7a4bea3176]*/ { int ret; _PySSLError err; @@ -1029,8 +1061,9 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) } while (err.ssl == SSL_ERROR_WANT_READ || err.ssl == SSL_ERROR_WANT_WRITE); Py_XDECREF(sock); + if (ret < 1) - return PySSL_SetError(self, ret, __FILE__, __LINE__); + return PySSL_SetError(self, __FILE__, __LINE__); if (PySSL_ChainExceptions(self) < 0) return NULL; Py_RETURN_NONE; @@ -1827,6 +1860,7 @@ _ssl__test_decode_cert_impl(PyObject *module, PyObject *path) /*[clinic input] +@critical_section _ssl._SSLSocket.getpeercert der as binary_mode: bool = False / @@ -1844,7 +1878,7 @@ return the certificate even if it wasn't validated. static PyObject * _ssl__SSLSocket_getpeercert_impl(PySSLSocket *self, int binary_mode) -/*[clinic end generated code: output=1f0ab66dfb693c88 input=c0fbe802e57629b7]*/ +/*[clinic end generated code: output=1f0ab66dfb693c88 input=e35af55fa5f9bab8]*/ { int verification; X509 *peer_cert; @@ -1874,13 +1908,14 @@ _ssl__SSLSocket_getpeercert_impl(PySSLSocket *self, int binary_mode) } /*[clinic input] +@critical_section _ssl._SSLSocket.get_verified_chain [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_get_verified_chain_impl(PySSLSocket *self) -/*[clinic end generated code: output=802421163cdc3110 input=5fb0714f77e2bd51]*/ +/*[clinic end generated code: output=802421163cdc3110 input=83035fe238ec057b]*/ { /* borrowed reference */ STACK_OF(X509) *chain = SSL_get0_verified_chain(self->ssl); @@ -1891,13 +1926,14 @@ _ssl__SSLSocket_get_verified_chain_impl(PySSLSocket *self) } /*[clinic input] +@critical_section _ssl._SSLSocket.get_unverified_chain [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_get_unverified_chain_impl(PySSLSocket *self) -/*[clinic end generated code: output=5acdae414e13f913 input=78c33c360c635cb5]*/ +/*[clinic end generated code: output=5acdae414e13f913 input=079f8ff5c205cb3b]*/ { PyObject *retval; /* borrowed reference */ @@ -2027,12 +2063,13 @@ cipher_to_dict(const SSL_CIPHER *cipher) } /*[clinic input] +@critical_section _ssl._SSLSocket.shared_ciphers [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_shared_ciphers_impl(PySSLSocket *self) -/*[clinic end generated code: output=3d174ead2e42c4fd input=0bfe149da8fe6306]*/ +/*[clinic end generated code: output=3d174ead2e42c4fd input=869645271e3bc6d0]*/ { STACK_OF(SSL_CIPHER) *server_ciphers; STACK_OF(SSL_CIPHER) *client_ciphers; @@ -2076,12 +2113,13 @@ _ssl__SSLSocket_shared_ciphers_impl(PySSLSocket *self) } /*[clinic input] +@critical_section _ssl._SSLSocket.cipher [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_cipher_impl(PySSLSocket *self) -/*[clinic end generated code: output=376417c16d0e5815 input=548fb0e27243796d]*/ +/*[clinic end generated code: output=376417c16d0e5815 input=39e180269a36f486]*/ { const SSL_CIPHER *current; @@ -2094,12 +2132,13 @@ _ssl__SSLSocket_cipher_impl(PySSLSocket *self) } /*[clinic input] +@critical_section _ssl._SSLSocket.version [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_version_impl(PySSLSocket *self) -/*[clinic end generated code: output=178aed33193b2cdb input=900186a503436fd6]*/ +/*[clinic end generated code: output=178aed33193b2cdb input=2732bc3f7f597d09]*/ { const char *version; @@ -2116,12 +2155,13 @@ _ssl__SSLSocket_version_impl(PySSLSocket *self) } /*[clinic input] +@critical_section _ssl._SSLSocket.selected_alpn_protocol [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_selected_alpn_protocol_impl(PySSLSocket *self) -/*[clinic end generated code: output=ec33688b303d250f input=442de30e35bc2913]*/ +/*[clinic end generated code: output=ec33688b303d250f input=f0b53506c9acdf8c]*/ { const unsigned char *out; unsigned int outlen; @@ -2159,13 +2199,35 @@ _ssl__SSLSocket_compression_impl(PySSLSocket *self) #endif } -static PySSLContext *PySSL_get_context(PySSLSocket *self, void *closure) { - return (PySSLContext*)Py_NewRef(self->ctx); +/*[clinic input] +@critical_section +@getter +_ssl._SSLSocket.context + +This changes the context associated with the SSLSocket. + +This is typically used from within a callback function set by the sni_callback +on the SSLContext to change the certificate information associated with the +SSLSocket before the cryptographic exchange handshake messages. +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLSocket_context_get_impl(PySSLSocket *self) +/*[clinic end generated code: output=d23e82f72f32e3d7 input=7cbb97407c2ace30]*/ +{ + return Py_NewRef(self->ctx); } -static int PySSL_set_context(PySSLSocket *self, PyObject *value, - void *closure) { +/*[clinic input] +@critical_section +@setter +_ssl._SSLSocket.context +[clinic start generated code]*/ +static int +_ssl__SSLSocket_context_set_impl(PySSLSocket *self, PyObject *value) +/*[clinic end generated code: output=6b0a6cc5cf33d9fe input=48ece77724fd9dd4]*/ +{ if (PyObject_TypeCheck(value, self->ctx->state->PySSLContext_Type)) { Py_SETREF(self->ctx, (PySSLContext *)Py_NewRef(value)); SSL_set_SSL_CTX(self->ssl, self->ctx->ctx); @@ -2182,50 +2244,71 @@ static int PySSL_set_context(PySSLSocket *self, PyObject *value, return 0; } -PyDoc_STRVAR(PySSL_set_context_doc, -"_setter_context(ctx)\n\ -\ -This changes the context associated with the SSLSocket. This is typically\n\ -used from within a callback function set by the sni_callback\n\ -on the SSLContext to change the certificate information associated with the\n\ -SSLSocket before the cryptographic exchange handshake messages\n"); +/*[clinic input] +@critical_section +@getter +_ssl._SSLSocket.server_side +Whether this is a server-side socket. +[clinic start generated code]*/ static PyObject * -PySSL_get_server_side(PySSLSocket *self, void *c) +_ssl__SSLSocket_server_side_get_impl(PySSLSocket *self) +/*[clinic end generated code: output=ae51e372489148e3 input=b09b320510bc7cae]*/ { return PyBool_FromLong(self->socket_type == PY_SSL_SERVER); } -PyDoc_STRVAR(PySSL_get_server_side_doc, -"Whether this is a server-side socket."); +/*[clinic input] +@critical_section +@getter +_ssl._SSLSocket.server_hostname + +The currently set server hostname (for SNI). +[clinic start generated code]*/ static PyObject * -PySSL_get_server_hostname(PySSLSocket *self, void *c) +_ssl__SSLSocket_server_hostname_get_impl(PySSLSocket *self) +/*[clinic end generated code: output=1f40ea5a076de8e7 input=55d12a1dc6634b08]*/ { if (self->server_hostname == NULL) Py_RETURN_NONE; return Py_NewRef(self->server_hostname); } -PyDoc_STRVAR(PySSL_get_server_hostname_doc, -"The currently set server hostname (for SNI)."); +/*[clinic input] +@critical_section +@getter +_ssl._SSLSocket.owner + +The Python-level owner of this object. + +Passed as "self" in servername callback. +[clinic start generated code]*/ static PyObject * -PySSL_get_owner(PySSLSocket *self, void *c) +_ssl__SSLSocket_owner_get_impl(PySSLSocket *self) +/*[clinic end generated code: output=1f278cb930382927 input=978a8382d9c25c92]*/ { if (self->owner == NULL) { Py_RETURN_NONE; } - PyObject *owner = _PyWeakref_GET_REF(self->owner); - if (owner == NULL) { + PyObject *owner; + if (!PyWeakref_GetRef(self->owner, &owner)) { Py_RETURN_NONE; } return owner; } +/*[clinic input] +@critical_section +@setter +_ssl._SSLSocket.owner +[clinic start generated code]*/ + static int -PySSL_set_owner(PySSLSocket *self, PyObject *value, void *c) +_ssl__SSLSocket_owner_set_impl(PySSLSocket *self, PyObject *value) +/*[clinic end generated code: output=2e3924498f2b6cde input=875666fd32367a73]*/ { Py_XSETREF(self->owner, PyWeakref_NewRef(value, NULL)); if (self->owner == NULL) @@ -2233,10 +2316,6 @@ PySSL_set_owner(PySSLSocket *self, PyObject *value, void *c) return 0; } -PyDoc_STRVAR(PySSL_get_owner_doc, -"The Python-level owner of this object.\ -Passed as \"self\" in servername callback."); - static int PySSL_traverse(PySSLSocket *self, visitproc visit, void *arg) { @@ -2258,6 +2337,17 @@ PySSL_dealloc(PySSLSocket *self) PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); if (self->ssl) { + // If we free the SSL socket object without having called SSL_shutdown, + // OpenSSL will invalidate the linked SSL session object. While this + // behavior is strictly RFC-compliant, it makes session reuse less + // likely and it would also break compatibility with older stdlib + // versions (which used an ugly workaround of duplicating the + // SSL_SESSION object). + // Therefore, we ensure the socket is marked as shutdown in any case. + // + // See elaborate explanation at + // https://github.com/python/cpython/pull/123249#discussion_r1766164530 + SSL_set_shutdown(self->ssl, SSL_SENT_SHUTDOWN | SSL_get_shutdown(self->ssl)); SSL_free(self->ssl); } Py_XDECREF(self->Socket); @@ -2339,6 +2429,7 @@ PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout) } /*[clinic input] +@critical_section _ssl._SSLSocket.write b: Py_buffer / @@ -2350,7 +2441,7 @@ Returns the number of bytes written. static PyObject * _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) -/*[clinic end generated code: output=aa7a6be5527358d8 input=77262d994fe5100a]*/ +/*[clinic end generated code: output=aa7a6be5527358d8 input=967b5feeae641a26]*/ { size_t count = 0; int retval; @@ -2437,7 +2528,7 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) Py_XDECREF(sock); if (retval == 0) - return PySSL_SetError(self, retval, __FILE__, __LINE__); + return PySSL_SetError(self, __FILE__, __LINE__); if (PySSL_ChainExceptions(self) < 0) return NULL; return PyLong_FromSize_t(count); @@ -2448,6 +2539,7 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) } /*[clinic input] +@critical_section _ssl._SSLSocket.pending Returns the number of already decrypted bytes available for read, pending on the connection. @@ -2455,7 +2547,7 @@ Returns the number of already decrypted bytes available for read, pending on the static PyObject * _ssl__SSLSocket_pending_impl(PySSLSocket *self) -/*[clinic end generated code: output=983d9fecdc308a83 input=2b77487d6dfd597f]*/ +/*[clinic end generated code: output=983d9fecdc308a83 input=32ab982a254e8866]*/ { int count = 0; _PySSLError err; @@ -2467,12 +2559,13 @@ _ssl__SSLSocket_pending_impl(PySSLSocket *self) self->err = err; if (count < 0) - return PySSL_SetError(self, count, __FILE__, __LINE__); + return PySSL_SetError(self, __FILE__, __LINE__); else return PyLong_FromLong(count); } /*[clinic input] +@critical_section _ssl._SSLSocket.read size as len: Py_ssize_t [ @@ -2486,7 +2579,7 @@ Read up to size bytes from the SSL socket. static PyObject * _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, int group_right_1, Py_buffer *buffer) -/*[clinic end generated code: output=49b16e6406023734 input=ec48bf622be1c4a1]*/ +/*[clinic end generated code: output=49b16e6406023734 input=80ed30436df01a71]*/ { PyObject *dest = NULL; char *mem; @@ -2590,7 +2683,7 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, err.ssl == SSL_ERROR_WANT_WRITE); if (retval == 0) { - PySSL_SetError(self, retval, __FILE__, __LINE__); + PySSL_SetError(self, __FILE__, __LINE__); goto error; } if (self->exc != NULL) @@ -2615,6 +2708,7 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, } /*[clinic input] +@critical_section _ssl._SSLSocket.shutdown Does the SSL shutdown handshake with the remote end. @@ -2622,7 +2716,7 @@ Does the SSL shutdown handshake with the remote end. static PyObject * _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) -/*[clinic end generated code: output=ca1aa7ed9d25ca42 input=11d39e69b0a2bf4a]*/ +/*[clinic end generated code: output=ca1aa7ed9d25ca42 input=98d9635cd4e16514]*/ { _PySSLError err; int sockstate, nonblocking, ret; @@ -2716,7 +2810,7 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) } if (ret < 0) { Py_XDECREF(sock); - PySSL_SetError(self, ret, __FILE__, __LINE__); + PySSL_SetError(self, __FILE__, __LINE__); return NULL; } if (self->exc != NULL) @@ -2734,6 +2828,7 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) } /*[clinic input] +@critical_section _ssl._SSLSocket.get_channel_binding cb_type: str = "tls-unique" @@ -2747,7 +2842,7 @@ Only 'tls-unique' channel binding data from RFC 5929 is supported. static PyObject * _ssl__SSLSocket_get_channel_binding_impl(PySSLSocket *self, const char *cb_type) -/*[clinic end generated code: output=34bac9acb6a61d31 input=08b7e43b99c17d41]*/ +/*[clinic end generated code: output=34bac9acb6a61d31 input=e008004fc08744db]*/ { char buf[PySSL_CB_MAXLEN]; size_t len; @@ -2779,6 +2874,7 @@ _ssl__SSLSocket_get_channel_binding_impl(PySSLSocket *self, } /*[clinic input] +@critical_section _ssl._SSLSocket.verify_client_post_handshake Initiate TLS 1.3 post-handshake authentication @@ -2786,9 +2882,9 @@ Initiate TLS 1.3 post-handshake authentication static PyObject * _ssl__SSLSocket_verify_client_post_handshake_impl(PySSLSocket *self) -/*[clinic end generated code: output=532147f3b1341425 input=6bfa874810a3d889]*/ +/*[clinic end generated code: output=532147f3b1341425 input=42b5bb1f0981eda1]*/ { -#ifdef TLS1_3_VERSION +#if defined(PySSL_HAVE_POST_HS_AUTH) int err = SSL_verify_client_post_handshake(self->ssl); if (err == 0) return _setSSLError(get_state_sock(self), NULL, 0, __FILE__, __LINE__); @@ -2802,70 +2898,31 @@ _ssl__SSLSocket_verify_client_post_handshake_impl(PySSLSocket *self) #endif } -static SSL_SESSION* -_ssl_session_dup(SSL_SESSION *session) { - SSL_SESSION *newsession = NULL; - int slen; - unsigned char *senc = NULL, *p; - const unsigned char *const_p; - - if (session == NULL) { - PyErr_SetString(PyExc_ValueError, "Invalid session"); - goto error; - } +/*[clinic input] +@critical_section +@getter +_ssl._SSLSocket.session - /* get length */ - slen = i2d_SSL_SESSION(session, NULL); - if (slen == 0 || slen > 0xFF00) { - PyErr_SetString(PyExc_ValueError, "i2d() failed"); - goto error; - } - if ((senc = PyMem_Malloc(slen)) == NULL) { - PyErr_NoMemory(); - goto error; - } - p = senc; - if (!i2d_SSL_SESSION(session, &p)) { - PyErr_SetString(PyExc_ValueError, "i2d() failed"); - goto error; - } - const_p = senc; - newsession = d2i_SSL_SESSION(NULL, &const_p, slen); - if (newsession == NULL) { - PyErr_SetString(PyExc_ValueError, "d2i() failed"); - goto error; - } - PyMem_Free(senc); - return newsession; - error: - if (senc != NULL) { - PyMem_Free(senc); - } - return NULL; -} +The underlying SSLSession object. +[clinic start generated code]*/ static PyObject * -PySSL_get_session(PySSLSocket *self, void *closure) { +_ssl__SSLSocket_session_get_impl(PySSLSocket *self) +/*[clinic end generated code: output=a5cd5755b35da670 input=d427318604244bf8]*/ +{ /* get_session can return sessions from a server-side connection, * it does not check for handshake done or client socket. */ PySSLSession *pysess; SSL_SESSION *session; - /* duplicate session as workaround for session bug in OpenSSL 1.1.0, - * https://github.com/openssl/openssl/issues/1550 */ - session = SSL_get0_session(self->ssl); /* borrowed reference */ - if (session == NULL) { - Py_RETURN_NONE; - } - if ((session = _ssl_session_dup(session)) == NULL) { - return NULL; - } session = SSL_get1_session(self->ssl); if (session == NULL) { Py_RETURN_NONE; } pysess = PyObject_GC_New(PySSLSession, self->ctx->state->PySSLSession_Type); if (pysess == NULL) { + // It's not possible for another thread to access this, so + // we don't need to lock it. SSL_SESSION_free(session); return NULL; } @@ -2877,12 +2934,17 @@ PySSL_get_session(PySSLSocket *self, void *closure) { return (PyObject *)pysess; } -static int PySSL_set_session(PySSLSocket *self, PyObject *value, - void *closure) - { +/*[clinic input] +@critical_section +@setter +_ssl._SSLSocket.session +[clinic start generated code]*/ + +static int +_ssl__SSLSocket_session_set_impl(PySSLSocket *self, PyObject *value) +/*[clinic end generated code: output=a3fa2ddd7c2d54a2 input=5fa5f921640db98b]*/ +{ PySSLSession *pysess; - SSL_SESSION *session; - int result; if (!Py_IS_TYPE(value, get_state_sock(self)->PySSLSession_Type)) { PyErr_SetString(PyExc_TypeError, "Value is not a SSLSession."); @@ -2905,50 +2967,37 @@ static int PySSL_set_session(PySSLSocket *self, PyObject *value, "Cannot set session after handshake."); return -1; } - /* duplicate session */ - if ((session = _ssl_session_dup(pysess->session)) == NULL) { - return -1; - } - result = SSL_set_session(self->ssl, session); - /* free duplicate, SSL_set_session() bumps ref count */ - SSL_SESSION_free(session); - if (result == 0) { + + if (SSL_set_session(self->ssl, pysess->session) == 0) { _setSSLError(get_state_sock(self), NULL, 0, __FILE__, __LINE__); return -1; } return 0; } -PyDoc_STRVAR(PySSL_set_session_doc, -"_setter_session(session)\n\ -\ -Get / set SSLSession."); +/*[clinic input] +@critical_section +@getter +_ssl._SSLSocket.session_reused + +Was the client session reused during handshake? +[clinic start generated code]*/ static PyObject * -PySSL_get_session_reused(PySSLSocket *self, void *closure) { - if (SSL_session_reused(self->ssl)) { - Py_RETURN_TRUE; - } else { - Py_RETURN_FALSE; - } +_ssl__SSLSocket_session_reused_get_impl(PySSLSocket *self) +/*[clinic end generated code: output=c8916909bcb80893 input=cec8bfec73a4461e]*/ +{ + int res = SSL_session_reused(self->ssl); + return res ? Py_True : Py_False; } -PyDoc_STRVAR(PySSL_get_session_reused_doc, -"Was the client session reused during handshake?"); - static PyGetSetDef ssl_getsetlist[] = { - {"context", (getter) PySSL_get_context, - (setter) PySSL_set_context, PySSL_set_context_doc}, - {"server_side", (getter) PySSL_get_server_side, NULL, - PySSL_get_server_side_doc}, - {"server_hostname", (getter) PySSL_get_server_hostname, NULL, - PySSL_get_server_hostname_doc}, - {"owner", (getter) PySSL_get_owner, (setter) PySSL_set_owner, - PySSL_get_owner_doc}, - {"session", (getter) PySSL_get_session, - (setter) PySSL_set_session, PySSL_set_session_doc}, - {"session_reused", (getter) PySSL_get_session_reused, NULL, - PySSL_get_session_reused_doc}, + _SSL__SSLSOCKET_CONTEXT_GETSETDEF + _SSL__SSLSOCKET_SERVER_SIDE_GETSETDEF + _SSL__SSLSOCKET_SERVER_HOSTNAME_GETSETDEF + _SSL__SSLSOCKET_OWNER_GETSETDEF + _SSL__SSLSOCKET_SESSION_GETSETDEF + _SSL__SSLSOCKET_SESSION_REUSED_GETSETDEF {NULL}, /* sentinel */ }; @@ -2984,7 +3033,7 @@ static PyType_Spec PySSLSocket_spec = { .name = "_ssl._SSLSocket", .basicsize = sizeof(PySSLSocket), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | - Py_TPFLAGS_HAVE_GC), + Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_DISALLOW_INSTANTIATION), .slots = PySSLSocket_slots, }; @@ -3021,6 +3070,7 @@ _set_verify_mode(PySSLContext *self, enum py_ssl_cert_requirements n) } /*[clinic input] +@critical_section @classmethod _ssl._SSLContext.__new__ protocol as proto_version: int @@ -3029,7 +3079,7 @@ _ssl._SSLContext.__new__ static PyObject * _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) -/*[clinic end generated code: output=2cf0d7a0741b6bd1 input=8d58a805b95fc534]*/ +/*[clinic end generated code: output=2cf0d7a0741b6bd1 input=6fc79e62ae9d143c]*/ { PySSLContext *self; uint64_t options; @@ -3099,6 +3149,10 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) return NULL; } + // This is the constructor, no need to lock it, because + // no other thread can be touching this object yet. + // (Technically, we can't even lock if we wanted to, as the + // lock hasn't been initialized yet.) PySSL_BEGIN_ALLOW_THREADS ctx = SSL_CTX_new(method); PySSL_END_ALLOW_THREADS @@ -3178,7 +3232,6 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) result = SSL_CTX_set_cipher_list(ctx, "HIGH:!aNULL:!eNULL"); } if (result == 0) { - Py_DECREF(self); ERR_clear_error(); PyErr_SetString(get_state_ctx(self)->PySSLErrorObject, "No cipher can be selected."); @@ -3212,7 +3265,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) X509_VERIFY_PARAM_set_flags(params, X509_V_FLAG_TRUSTED_FIRST); X509_VERIFY_PARAM_set_hostflags(params, self->hostflags); -#ifdef TLS1_3_VERSION +#if defined(PySSL_HAVE_POST_HS_AUTH) self->post_handshake_auth = 0; SSL_CTX_set_post_handshake_auth(self->ctx, self->post_handshake_auth); #endif @@ -3266,6 +3319,7 @@ context_dealloc(PySSLContext *self) } /*[clinic input] +@critical_section _ssl._SSLContext.set_ciphers cipherlist: str / @@ -3273,7 +3327,7 @@ _ssl._SSLContext.set_ciphers static PyObject * _ssl__SSLContext_set_ciphers_impl(PySSLContext *self, const char *cipherlist) -/*[clinic end generated code: output=3a3162f3557c0f3f input=a7ac931b9f3ca7fc]*/ +/*[clinic end generated code: output=3a3162f3557c0f3f input=40b583cded5c6ff9]*/ { int ret = SSL_CTX_set_cipher_list(self->ctx, cipherlist); if (ret == 0) { @@ -3289,12 +3343,13 @@ _ssl__SSLContext_set_ciphers_impl(PySSLContext *self, const char *cipherlist) } /*[clinic input] +@critical_section _ssl._SSLContext.get_ciphers [clinic start generated code]*/ static PyObject * _ssl__SSLContext_get_ciphers_impl(PySSLContext *self) -/*[clinic end generated code: output=a56e4d68a406dfc4 input=a2aadc9af89b79c5]*/ +/*[clinic end generated code: output=a56e4d68a406dfc4 input=d7fff51631a260ae]*/ { SSL *ssl = NULL; STACK_OF(SSL_CIPHER) *sk = NULL; @@ -3369,6 +3424,7 @@ _selectALPN_cb(SSL *s, } /*[clinic input] +@critical_section _ssl._SSLContext._set_alpn_protocols protos: Py_buffer / @@ -3377,7 +3433,7 @@ _ssl._SSLContext._set_alpn_protocols static PyObject * _ssl__SSLContext__set_alpn_protocols_impl(PySSLContext *self, Py_buffer *protos) -/*[clinic end generated code: output=87599a7f76651a9b input=9bba964595d519be]*/ +/*[clinic end generated code: output=87599a7f76651a9b input=b5096b186e49287d]*/ { if ((size_t)protos->len > UINT_MAX) { PyErr_Format(PyExc_OverflowError, @@ -3387,25 +3443,34 @@ _ssl__SSLContext__set_alpn_protocols_impl(PySSLContext *self, PyMem_Free(self->alpn_protocols); self->alpn_protocols = PyMem_Malloc(protos->len); - if (!self->alpn_protocols) + if (!self->alpn_protocols) { return PyErr_NoMemory(); + } memcpy(self->alpn_protocols, protos->buf, protos->len); self->alpn_protocols_len = (unsigned int)protos->len; - - if (SSL_CTX_set_alpn_protos(self->ctx, self->alpn_protocols, self->alpn_protocols_len)) + if (SSL_CTX_set_alpn_protos(self->ctx, self->alpn_protocols, self->alpn_protocols_len)) { return PyErr_NoMemory(); + } SSL_CTX_set_alpn_select_cb(self->ctx, _selectALPN_cb, self); Py_RETURN_NONE; } +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext.verify_mode +[clinic start generated code]*/ + static PyObject * -get_verify_mode(PySSLContext *self, void *c) +_ssl__SSLContext_verify_mode_get_impl(PySSLContext *self) +/*[clinic end generated code: output=3e788736cc7229bc input=7e3c7f4454121d0a]*/ { /* ignore SSL_VERIFY_CLIENT_ONCE and SSL_VERIFY_POST_HANDSHAKE */ int mask = (SSL_VERIFY_NONE | SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT); - switch (SSL_CTX_get_verify_mode(self->ctx) & mask) { + int verify_mode = SSL_CTX_get_verify_mode(self->ctx); + switch (verify_mode & mask) { case SSL_VERIFY_NONE: return PyLong_FromLong(PY_SSL_CERT_NONE); case SSL_VERIFY_PEER: @@ -3418,11 +3483,18 @@ get_verify_mode(PySSLContext *self, void *c) return NULL; } +/*[clinic input] +@critical_section +@setter +_ssl._SSLContext.verify_mode +[clinic start generated code]*/ + static int -set_verify_mode(PySSLContext *self, PyObject *arg, void *c) +_ssl__SSLContext_verify_mode_set_impl(PySSLContext *self, PyObject *value) +/*[clinic end generated code: output=d698e16c58db3118 input=3ee60057c3a22378]*/ { int n; - if (!PyArg_Parse(arg, "i", &n)) + if (!PyArg_Parse(value, "i", &n)) return -1; if (n == PY_SSL_CERT_NONE && self->check_hostname) { PyErr_SetString(PyExc_ValueError, @@ -3433,8 +3505,15 @@ set_verify_mode(PySSLContext *self, PyObject *arg, void *c) return _set_verify_mode(self, n); } +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext.verify_flags +[clinic start generated code]*/ + static PyObject * -get_verify_flags(PySSLContext *self, void *c) +_ssl__SSLContext_verify_flags_get_impl(PySSLContext *self) +/*[clinic end generated code: output=fbbf8ba28ad6e56e input=c1ec36d610b3f391]*/ { X509_VERIFY_PARAM *param; unsigned long flags; @@ -3444,13 +3523,20 @@ get_verify_flags(PySSLContext *self, void *c) return PyLong_FromUnsignedLong(flags); } +/*[clinic input] +@critical_section +@setter +_ssl._SSLContext.verify_flags +[clinic start generated code]*/ + static int -set_verify_flags(PySSLContext *self, PyObject *arg, void *c) +_ssl__SSLContext_verify_flags_set_impl(PySSLContext *self, PyObject *value) +/*[clinic end generated code: output=a3e3b2a0ce6c2e99 input=b2a0c42583d4f34e]*/ { X509_VERIFY_PARAM *param; unsigned long new_flags, flags, set, clear; - if (!PyArg_Parse(arg, "k", &new_flags)) + if (!PyArg_Parse(value, "k", &new_flags)) return -1; param = SSL_CTX_get0_param(self->ctx); flags = X509_VERIFY_PARAM_get_flags(param); @@ -3486,8 +3572,8 @@ set_min_max_proto_version(PySSLContext *self, PyObject *arg, int what) } switch(self->protocol) { - case PY_SSL_VERSION_TLS_CLIENT: /* fall through */ - case PY_SSL_VERSION_TLS_SERVER: /* fall through */ + case PY_SSL_VERSION_TLS_CLIENT: _Py_FALLTHROUGH; + case PY_SSL_VERSION_TLS_SERVER: _Py_FALLTHROUGH; case PY_SSL_VERSION_TLS: break; default: @@ -3558,8 +3644,15 @@ set_min_max_proto_version(PySSLContext *self, PyObject *arg, int what) return 0; } +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext.minimum_version +[clinic start generated code]*/ + static PyObject * -get_minimum_version(PySSLContext *self, void *c) +_ssl__SSLContext_minimum_version_get_impl(PySSLContext *self) +/*[clinic end generated code: output=27fa8382276635ed input=6832821e7e974d40]*/ { int v = SSL_CTX_get_min_proto_version(self->ctx); if (v == 0) { @@ -3568,14 +3661,29 @@ get_minimum_version(PySSLContext *self, void *c) return PyLong_FromLong(v); } +/*[clinic input] +@critical_section +@setter +_ssl._SSLContext.minimum_version +[clinic start generated code]*/ + static int -set_minimum_version(PySSLContext *self, PyObject *arg, void *c) +_ssl__SSLContext_minimum_version_set_impl(PySSLContext *self, + PyObject *value) +/*[clinic end generated code: output=482e82f7372afb78 input=2c64724901a514b3]*/ { - return set_min_max_proto_version(self, arg, 0); + return set_min_max_proto_version(self, value, 0); } +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext.maximum_version +[clinic start generated code]*/ + static PyObject * -get_maximum_version(PySSLContext *self, void *c) +_ssl__SSLContext_maximum_version_get_impl(PySSLContext *self) +/*[clinic end generated code: output=889249475112826a input=2b9e4c2d45f16b14]*/ { int v = SSL_CTX_get_max_proto_version(self->ctx); if (v == 0) { @@ -3584,24 +3692,53 @@ get_maximum_version(PySSLContext *self, void *c) return PyLong_FromLong(v); } +/*[clinic input] +@critical_section +@setter +_ssl._SSLContext.maximum_version +[clinic start generated code]*/ + static int -set_maximum_version(PySSLContext *self, PyObject *arg, void *c) +_ssl__SSLContext_maximum_version_set_impl(PySSLContext *self, + PyObject *value) +/*[clinic end generated code: output=4c0eed3042ca20d5 input=fe27e9fbbeb73c89]*/ { - return set_min_max_proto_version(self, arg, 1); + return set_min_max_proto_version(self, value, 1); } -#ifdef TLS1_3_VERSION +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext.num_tickets + +Control the number of TLSv1.3 session tickets. +[clinic start generated code]*/ + static PyObject * -get_num_tickets(PySSLContext *self, void *c) +_ssl__SSLContext_num_tickets_get_impl(PySSLContext *self) +/*[clinic end generated code: output=3d06d016318846c9 input=1e2599a2e22564ff]*/ { - return PyLong_FromSize_t(SSL_CTX_get_num_tickets(self->ctx)); + // Clinic seems to be misbehaving when the comment is wrapped with in directive +#if defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) + PyObject *res = PyLong_FromSize_t(SSL_CTX_get_num_tickets(self->ctx)); + return res; +#else + return 0; +#endif } +/*[clinic input] +@critical_section +@setter +_ssl._SSLContext.num_tickets +[clinic start generated code]*/ + static int -set_num_tickets(PySSLContext *self, PyObject *arg, void *c) +_ssl__SSLContext_num_tickets_set_impl(PySSLContext *self, PyObject *value) +/*[clinic end generated code: output=ced81b46f3beab09 input=6ef8067ac55607e7]*/ { long num; - if (!PyArg_Parse(arg, "l", &num)) + if (!PyArg_Parse(value, "l", &num)) return -1; if (num < 0) { PyErr_SetString(PyExc_ValueError, "value must be non-negative"); @@ -3619,27 +3756,46 @@ set_num_tickets(PySSLContext *self, PyObject *arg, void *c) return 0; } -PyDoc_STRVAR(PySSLContext_num_tickets_doc, -"Control the number of TLSv1.3 session tickets"); -#endif /* TLS1_3_VERSION */ +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext.security_level + +The current security level. +[clinic start generated code]*/ static PyObject * -get_security_level(PySSLContext *self, void *c) +_ssl__SSLContext_security_level_get_impl(PySSLContext *self) +/*[clinic end generated code: output=56ece09e6a9572d0 input=2bdeecb57bb86e3f]*/ { - return PyLong_FromLong(SSL_CTX_get_security_level(self->ctx)); + PyObject *res = PyLong_FromLong(SSL_CTX_get_security_level(self->ctx)); + return res; } -PyDoc_STRVAR(PySSLContext_security_level_doc, "The current security level"); + +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext.options +[clinic start generated code]*/ static PyObject * -get_options(PySSLContext *self, void *c) +_ssl__SSLContext_options_get_impl(PySSLContext *self) +/*[clinic end generated code: output=3dfa6a74837f525b input=f5a2805c7cda6f25]*/ { uint64_t options = SSL_CTX_get_options(self->ctx); Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(options)); return PyLong_FromUnsignedLongLong(options); } +/*[clinic input] +@critical_section +@setter +_ssl._SSLContext.options +[clinic start generated code]*/ + static int -set_options(PySSLContext *self, PyObject *arg, void *c) +_ssl__SSLContext_options_set_impl(PySSLContext *self, PyObject *value) +/*[clinic end generated code: output=92ca34731ece5dbb input=2b94bf789e9ae5dd]*/ { PyObject *new_opts_obj; unsigned long long new_opts_arg; @@ -3649,7 +3805,7 @@ set_options(PySSLContext *self, PyObject *arg, void *c) SSL_OP_NO_TLSv1_1 | SSL_OP_NO_TLSv1_2 | SSL_OP_NO_TLSv1_3 ); - if (!PyArg_Parse(arg, "O!", &PyLong_Type, &new_opts_obj)) { + if (!PyArg_Parse(value, "O!", &PyLong_Type, &new_opts_obj)) { return -1; } new_opts_arg = PyLong_AsUnsignedLongLong(new_opts_obj); @@ -3678,19 +3834,33 @@ set_options(PySSLContext *self, PyObject *arg, void *c) return 0; } +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext._host_flags +[clinic start generated code]*/ + static PyObject * -get_host_flags(PySSLContext *self, void *c) +_ssl__SSLContext__host_flags_get_impl(PySSLContext *self) +/*[clinic end generated code: output=0f9db6654ce32582 input=8e3c49499eefd0e5]*/ { return PyLong_FromUnsignedLong(self->hostflags); } +/*[clinic input] +@critical_section +@setter +_ssl._SSLContext._host_flags +[clinic start generated code]*/ + static int -set_host_flags(PySSLContext *self, PyObject *arg, void *c) +_ssl__SSLContext__host_flags_set_impl(PySSLContext *self, PyObject *value) +/*[clinic end generated code: output=1ed6f4027aaf2e3e input=28caf1fb9c32f6cb]*/ { X509_VERIFY_PARAM *param; unsigned int new_flags = 0; - if (!PyArg_Parse(arg, "I", &new_flags)) + if (!PyArg_Parse(value, "I", &new_flags)) return -1; param = SSL_CTX_get0_param(self->ctx); @@ -3699,20 +3869,35 @@ set_host_flags(PySSLContext *self, PyObject *arg, void *c) return 0; } +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext.check_hostname +[clinic start generated code]*/ + static PyObject * -get_check_hostname(PySSLContext *self, void *c) +_ssl__SSLContext_check_hostname_get_impl(PySSLContext *self) +/*[clinic end generated code: output=e046d6eeefc76063 input=1b8341e705f9ecf5]*/ { return PyBool_FromLong(self->check_hostname); } +/*[clinic input] +@critical_section +@setter +_ssl._SSLContext.check_hostname +[clinic start generated code]*/ + static int -set_check_hostname(PySSLContext *self, PyObject *arg, void *c) +_ssl__SSLContext_check_hostname_set_impl(PySSLContext *self, PyObject *value) +/*[clinic end generated code: output=0e767b4784e7dc3f input=e6a771cb5919f74d]*/ { int check_hostname; - if (!PyArg_Parse(arg, "p", &check_hostname)) + if (!PyArg_Parse(value, "p", &check_hostname)) return -1; + int verify_mode = check_hostname ? SSL_CTX_get_verify_mode(self->ctx) : 0; if (check_hostname && - SSL_CTX_get_verify_mode(self->ctx) == SSL_VERIFY_NONE) { + verify_mode == SSL_VERIFY_NONE) { /* check_hostname = True sets verify_mode = CERT_REQUIRED */ if (_set_verify_mode(self, PY_SSL_CERT_REQUIRED) == -1) { return -1; @@ -3724,14 +3909,14 @@ set_check_hostname(PySSLContext *self, PyObject *arg, void *c) static PyObject * get_post_handshake_auth(PySSLContext *self, void *c) { -#if TLS1_3_VERSION +#if defined(PySSL_HAVE_POST_HS_AUTH) return PyBool_FromLong(self->post_handshake_auth); #else Py_RETURN_NONE; #endif } -#if TLS1_3_VERSION +#if defined(PySSL_HAVE_POST_HS_AUTH) static int set_post_handshake_auth(PySSLContext *self, PyObject *arg, void *c) { if (arg == NULL) { @@ -3752,8 +3937,16 @@ set_post_handshake_auth(PySSLContext *self, PyObject *arg, void *c) { } #endif +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext.protocol +[clinic start generated code]*/ + static PyObject * -get_protocol(PySSLContext *self, void *c) { +_ssl__SSLContext_protocol_get_impl(PySSLContext *self) +/*[clinic end generated code: output=a9a48febc16cee22 input=c9f5fa1a2bd4b8a8]*/ +{ return PyLong_FromLong(self->protocol); } @@ -3867,6 +4060,7 @@ _password_callback(char *buf, int size, int rwflag, void *userdata) } /*[clinic input] +@critical_section _ssl._SSLContext.load_cert_chain certfile: object keyfile: object = None @@ -3877,7 +4071,7 @@ _ssl._SSLContext.load_cert_chain static PyObject * _ssl__SSLContext_load_cert_chain_impl(PySSLContext *self, PyObject *certfile, PyObject *keyfile, PyObject *password) -/*[clinic end generated code: output=9480bc1c380e2095 input=30bc7e967ea01a58]*/ +/*[clinic end generated code: output=9480bc1c380e2095 input=6c7c5e8b73e4264b]*/ { PyObject *certfile_bytes = NULL, *keyfile_bytes = NULL; pem_password_cb *orig_passwd_cb = SSL_CTX_get_default_passwd_cb(self->ctx); @@ -4071,6 +4265,7 @@ _add_ca_certs(PySSLContext *self, const void *data, Py_ssize_t len, /*[clinic input] +@critical_section _ssl._SSLContext.load_verify_locations cafile: object = None capath: object = None @@ -4083,7 +4278,7 @@ _ssl__SSLContext_load_verify_locations_impl(PySSLContext *self, PyObject *cafile, PyObject *capath, PyObject *cadata) -/*[clinic end generated code: output=454c7e41230ca551 input=42ecfe258233e194]*/ +/*[clinic end generated code: output=454c7e41230ca551 input=b178852b41618414]*/ { PyObject *cafile_bytes = NULL, *capath_bytes = NULL; const char *cafile_buf = NULL, *capath_buf = NULL; @@ -4199,6 +4394,7 @@ _ssl__SSLContext_load_verify_locations_impl(PySSLContext *self, } /*[clinic input] +@critical_section _ssl._SSLContext.load_dh_params path as filepath: object / @@ -4206,13 +4402,13 @@ _ssl._SSLContext.load_dh_params [clinic start generated code]*/ static PyObject * -_ssl__SSLContext_load_dh_params(PySSLContext *self, PyObject *filepath) -/*[clinic end generated code: output=1c8e57a38e055af0 input=c8871f3c796ae1d6]*/ +_ssl__SSLContext_load_dh_params_impl(PySSLContext *self, PyObject *filepath) +/*[clinic end generated code: output=dd74b3c524dd2723 input=832769a0734b8c4d]*/ { FILE *f; DH *dh; - f = _Py_fopen_obj(filepath, "rb"); + f = Py_fopen(filepath, "rb"); if (f == NULL) return NULL; @@ -4240,6 +4436,7 @@ _ssl__SSLContext_load_dh_params(PySSLContext *self, PyObject *filepath) } /*[clinic input] +@critical_section _ssl._SSLContext._wrap_socket sock: object(subclass_of="get_state_ctx(self)->Sock_Type") server_side: bool @@ -4254,7 +4451,7 @@ static PyObject * _ssl__SSLContext__wrap_socket_impl(PySSLContext *self, PyObject *sock, int server_side, PyObject *hostname_obj, PyObject *owner, PyObject *session) -/*[clinic end generated code: output=f103f238633940b4 input=700ca8fedff53994]*/ +/*[clinic end generated code: output=f103f238633940b4 input=eceadcee4434a06f]*/ { char *hostname = NULL; PyObject *res; @@ -4276,6 +4473,7 @@ _ssl__SSLContext__wrap_socket_impl(PySSLContext *self, PyObject *sock, } /*[clinic input] +@critical_section _ssl._SSLContext._wrap_bio incoming: object(subclass_of="get_state_ctx(self)->PySSLMemoryBIO_Type", type="PySSLMemoryBIO *") outgoing: object(subclass_of="get_state_ctx(self)->PySSLMemoryBIO_Type", type="PySSLMemoryBIO *") @@ -4292,7 +4490,7 @@ _ssl__SSLContext__wrap_bio_impl(PySSLContext *self, PySSLMemoryBIO *incoming, PySSLMemoryBIO *outgoing, int server_side, PyObject *hostname_obj, PyObject *owner, PyObject *session) -/*[clinic end generated code: output=5c5d6d9b41f99332 input=a9205d097fd45a82]*/ +/*[clinic end generated code: output=5c5d6d9b41f99332 input=58277fc962a60182]*/ { char *hostname = NULL; PyObject *res; @@ -4313,12 +4511,13 @@ _ssl__SSLContext__wrap_bio_impl(PySSLContext *self, PySSLMemoryBIO *incoming, } /*[clinic input] +@critical_section _ssl._SSLContext.session_stats [clinic start generated code]*/ static PyObject * _ssl__SSLContext_session_stats_impl(PySSLContext *self) -/*[clinic end generated code: output=0d96411c42893bfb input=7e0a81fb11102c8b]*/ +/*[clinic end generated code: output=0d96411c42893bfb input=db62af53004127a4]*/ { int r; PyObject *value, *stats = PyDict_New(); @@ -4357,12 +4556,13 @@ _ssl__SSLContext_session_stats_impl(PySSLContext *self) } /*[clinic input] +@critical_section _ssl._SSLContext.set_default_verify_paths [clinic start generated code]*/ static PyObject * _ssl__SSLContext_set_default_verify_paths_impl(PySSLContext *self) -/*[clinic end generated code: output=0bee74e6e09deaaa input=35f3408021463d74]*/ +/*[clinic end generated code: output=0bee74e6e09deaaa input=939a88e78f634119]*/ { int rc; Py_BEGIN_ALLOW_THREADS @@ -4376,6 +4576,7 @@ _ssl__SSLContext_set_default_verify_paths_impl(PySSLContext *self) } /*[clinic input] +@critical_section _ssl._SSLContext.set_ecdh_curve name: object / @@ -4383,8 +4584,8 @@ _ssl._SSLContext.set_ecdh_curve [clinic start generated code]*/ static PyObject * -_ssl__SSLContext_set_ecdh_curve(PySSLContext *self, PyObject *name) -/*[clinic end generated code: output=23022c196e40d7d2 input=c2bafb6f6e34726b]*/ +_ssl__SSLContext_set_ecdh_curve_impl(PySSLContext *self, PyObject *name) +/*[clinic end generated code: output=01081151ce0ecc45 input=039df032e666870e]*/ { PyObject *name_bytes; int nid; @@ -4446,9 +4647,9 @@ _servername_callback(SSL *s, int *al, void *args) * will be passed. If both do not exist only then the C-level object is * passed. */ if (ssl->owner) - ssl_socket = _PyWeakref_GET_REF(ssl->owner); + PyWeakref_GetRef(ssl->owner, &ssl_socket); else if (ssl->Socket) - ssl_socket = _PyWeakref_GET_REF(ssl->Socket); + PyWeakref_GetRef(ssl->Socket, &ssl_socket); else ssl_socket = Py_NewRef(ssl); @@ -4518,8 +4719,22 @@ _servername_callback(SSL *s, int *al, void *args) return ret; } +/*[clinic input] +@critical_section +@getter +_ssl._SSLContext.sni_callback + +Set a callback that will be called when a server name is provided by the SSL/TLS client in the SNI extension. + +If the argument is None then the callback is disabled. The method is called +with the SSLSocket, the server name as a string, and the SSLContext object. + +See RFC 6066 for details of the SNI extension. +[clinic start generated code]*/ + static PyObject * -get_sni_callback(PySSLContext *self, void *c) +_ssl__SSLContext_sni_callback_get_impl(PySSLContext *self) +/*[clinic end generated code: output=961e6575cdfaf036 input=9b2473c5e984cfe6]*/ { PyObject *cb = self->set_sni_cb; if (cb == NULL) { @@ -4528,8 +4743,15 @@ get_sni_callback(PySSLContext *self, void *c) return Py_NewRef(cb); } +/*[clinic input] +@critical_section +@setter +_ssl._SSLContext.sni_callback +[clinic start generated code]*/ + static int -set_sni_callback(PySSLContext *self, PyObject *arg, void *c) +_ssl__SSLContext_sni_callback_set_impl(PySSLContext *self, PyObject *value) +/*[clinic end generated code: output=b32736c6b891f61a input=c3c4ff33540b3c85]*/ { if (self->protocol == PY_SSL_VERSION_TLS_CLIENT) { PyErr_SetString(PyExc_ValueError, @@ -4537,17 +4759,17 @@ set_sni_callback(PySSLContext *self, PyObject *arg, void *c) return -1; } Py_CLEAR(self->set_sni_cb); - if (arg == Py_None) { + if (value == Py_None) { SSL_CTX_set_tlsext_servername_callback(self->ctx, NULL); } else { - if (!PyCallable_Check(arg)) { + if (!PyCallable_Check(value)) { SSL_CTX_set_tlsext_servername_callback(self->ctx, NULL); PyErr_SetString(PyExc_TypeError, "not a callable object"); return -1; } - self->set_sni_cb = Py_NewRef(arg); + self->set_sni_cb = Py_NewRef(value); SSL_CTX_set_tlsext_servername_callback(self->ctx, _servername_callback); SSL_CTX_set_tlsext_servername_arg(self->ctx, self); } @@ -4598,14 +4820,8 @@ X509_STORE_get1_objects(X509_STORE *store) } #endif -PyDoc_STRVAR(PySSLContext_sni_callback_doc, -"Set a callback that will be called when a server name is provided by the SSL/TLS client in the SNI extension.\n\ -\n\ -If the argument is None then the callback is disabled. The method is called\n\ -with the SSLSocket, the server name as a string, and the SSLContext object.\n\ -See RFC 6066 for details of the SNI extension."); - /*[clinic input] +@critical_section _ssl._SSLContext.cert_store_stats Returns quantities of loaded X.509 certificates. @@ -4619,7 +4835,7 @@ been used at least once. static PyObject * _ssl__SSLContext_cert_store_stats_impl(PySSLContext *self) -/*[clinic end generated code: output=5f356f4d9cca874d input=eb40dd0f6d0e40cf]*/ +/*[clinic end generated code: output=5f356f4d9cca874d input=d13c6e3f2b48539b]*/ { X509_STORE *store; STACK_OF(X509_OBJECT) *objs; @@ -4656,6 +4872,7 @@ _ssl__SSLContext_cert_store_stats_impl(PySSLContext *self) } /*[clinic input] +@critical_section _ssl._SSLContext.get_ca_certs binary_form: bool = False @@ -4670,7 +4887,7 @@ been used at least once. static PyObject * _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) -/*[clinic end generated code: output=0d58f148f37e2938 input=6887b5a09b7f9076]*/ +/*[clinic end generated code: output=0d58f148f37e2938 input=eb0592909c9ad6e7]*/ { X509_STORE *store; STACK_OF(X509_OBJECT) *objs; @@ -4771,7 +4988,9 @@ static unsigned int psk_client_callback(SSL *s, goto error; } - if (identity_len_ + 1 > max_identity_len || psk_len_ > max_psk_len) { + if ((size_t)identity_len_ + 1 > max_identity_len + || (size_t)psk_len_ > max_psk_len) + { Py_DECREF(result); goto error; } @@ -4796,6 +5015,7 @@ static unsigned int psk_client_callback(SSL *s, #endif /*[clinic input] +@critical_section _ssl._SSLContext.set_psk_client_callback callback: object @@ -4804,7 +5024,7 @@ _ssl._SSLContext.set_psk_client_callback static PyObject * _ssl__SSLContext_set_psk_client_callback_impl(PySSLContext *self, PyObject *callback) -/*[clinic end generated code: output=0aba86f6ed75119e input=7627bae0e5ee7635]*/ +/*[clinic end generated code: output=0aba86f6ed75119e input=1e436eea625cfc35]*/ { #ifndef OPENSSL_NO_PSK if (self->protocol == PY_SSL_VERSION_TLS_SERVER) { @@ -4883,7 +5103,7 @@ static unsigned int psk_server_callback(SSL *s, goto error; } - if (psk_len_ > max_psk_len) { + if ((size_t)psk_len_ > max_psk_len) { Py_DECREF(result); goto error; } @@ -4904,6 +5124,7 @@ static unsigned int psk_server_callback(SSL *s, #endif /*[clinic input] +@critical_section _ssl._SSLContext.set_psk_server_callback callback: object identity_hint: str(accept={str, NoneType}) = None @@ -4914,7 +5135,7 @@ static PyObject * _ssl__SSLContext_set_psk_server_callback_impl(PySSLContext *self, PyObject *callback, const char *identity_hint) -/*[clinic end generated code: output=1f4d6a4e09a92b03 input=65d4b6022aa85ea3]*/ +/*[clinic end generated code: output=1f4d6a4e09a92b03 input=5f79d932458284a7]*/ { #ifndef OPENSSL_NO_PSK if (self->protocol == PY_SSL_VERSION_TLS_CLIENT) { @@ -4959,41 +5180,30 @@ _ssl__SSLContext_set_psk_server_callback_impl(PySSLContext *self, static PyGetSetDef context_getsetlist[] = { - {"check_hostname", (getter) get_check_hostname, - (setter) set_check_hostname, NULL}, - {"_host_flags", (getter) get_host_flags, - (setter) set_host_flags, NULL}, - {"minimum_version", (getter) get_minimum_version, - (setter) set_minimum_version, NULL}, - {"maximum_version", (getter) get_maximum_version, - (setter) set_maximum_version, NULL}, + _SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF + _SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF + _SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF + _SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF {"keylog_filename", (getter) _PySSLContext_get_keylog_filename, (setter) _PySSLContext_set_keylog_filename, NULL}, {"_msg_callback", (getter) _PySSLContext_get_msg_callback, (setter) _PySSLContext_set_msg_callback, NULL}, - {"sni_callback", (getter) get_sni_callback, - (setter) set_sni_callback, PySSLContext_sni_callback_doc}, -#ifdef TLS1_3_VERSION - {"num_tickets", (getter) get_num_tickets, - (setter) set_num_tickets, PySSLContext_num_tickets_doc}, + _SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF +#if defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) + _SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF #endif - {"options", (getter) get_options, - (setter) set_options, NULL}, + _SSL__SSLCONTEXT_OPTIONS_GETSETDEF {"post_handshake_auth", (getter) get_post_handshake_auth, -#ifdef TLS1_3_VERSION +#if defined(PySSL_HAVE_POST_HS_AUTH) (setter) set_post_handshake_auth, #else NULL, #endif NULL}, - {"protocol", (getter) get_protocol, - NULL, NULL}, - {"verify_flags", (getter) get_verify_flags, - (setter) set_verify_flags, NULL}, - {"verify_mode", (getter) get_verify_mode, - (setter) set_verify_mode, NULL}, - {"security_level", (getter) get_security_level, - NULL, PySSLContext_security_level_doc}, + _SSL__SSLCONTEXT_PROTOCOL_GETSETDEF + _SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF + _SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF + _SSL__SSLCONTEXT_SECURITY_LEVEL_GETSETDEF {NULL}, /* sentinel */ }; @@ -5040,6 +5250,7 @@ static PyType_Spec PySSLContext_spec = { */ /*[clinic input] +@critical_section @classmethod _ssl.MemoryBIO.__new__ @@ -5047,7 +5258,7 @@ _ssl.MemoryBIO.__new__ static PyObject * _ssl_MemoryBIO_impl(PyTypeObject *type) -/*[clinic end generated code: output=8820a58db78330ac input=26d22e4909ecb1b5]*/ +/*[clinic end generated code: output=8820a58db78330ac input=87f146cf30af454e]*/ { BIO *bio; PySSLMemoryBIO *self; @@ -5092,26 +5303,40 @@ memory_bio_dealloc(PySSLMemoryBIO *self) Py_DECREF(tp); } +/*[clinic input] +@critical_section +@getter +_ssl.MemoryBIO.pending + +The number of bytes pending in the memory BIO. +[clinic start generated code]*/ + static PyObject * -memory_bio_get_pending(PySSLMemoryBIO *self, void *c) +_ssl_MemoryBIO_pending_get_impl(PySSLMemoryBIO *self) +/*[clinic end generated code: output=19236a32a51ac8ff input=02d9063d8ac31732]*/ { - return PyLong_FromSize_t(BIO_ctrl_pending(self->bio)); + size_t res = BIO_ctrl_pending(self->bio); + return PyLong_FromSize_t(res); } -PyDoc_STRVAR(PySSL_memory_bio_pending_doc, -"The number of bytes pending in the memory BIO."); +/*[clinic input] +@critical_section +@getter +_ssl.MemoryBIO.eof + +Whether the memory BIO is at EOF. +[clinic start generated code]*/ static PyObject * -memory_bio_get_eof(PySSLMemoryBIO *self, void *c) +_ssl_MemoryBIO_eof_get_impl(PySSLMemoryBIO *self) +/*[clinic end generated code: output=c255a9ea16e31b92 input=c6ecc12c4509de1f]*/ { - return PyBool_FromLong((BIO_ctrl_pending(self->bio) == 0) - && self->eof_written); + size_t pending = BIO_ctrl_pending(self->bio); + return PyBool_FromLong((pending == 0) && self->eof_written); } -PyDoc_STRVAR(PySSL_memory_bio_eof_doc, -"Whether the memory BIO is at EOF."); - /*[clinic input] +@critical_section _ssl.MemoryBIO.read size as len: int = -1 / @@ -5126,7 +5351,7 @@ distinguish between the two. static PyObject * _ssl_MemoryBIO_read_impl(PySSLMemoryBIO *self, int len) -/*[clinic end generated code: output=a657aa1e79cd01b3 input=574d7be06a902366]*/ +/*[clinic end generated code: output=a657aa1e79cd01b3 input=21046f2d7dac3a90]*/ { int avail, nbytes; PyObject *result; @@ -5156,6 +5381,7 @@ _ssl_MemoryBIO_read_impl(PySSLMemoryBIO *self, int len) } /*[clinic input] +@critical_section _ssl.MemoryBIO.write b: Py_buffer / @@ -5167,7 +5393,7 @@ Returns the number of bytes written. static PyObject * _ssl_MemoryBIO_write_impl(PySSLMemoryBIO *self, Py_buffer *b) -/*[clinic end generated code: output=156ec59110d75935 input=e45757b3e17c4808]*/ +/*[clinic end generated code: output=156ec59110d75935 input=107da3f5fba26b37]*/ { int nbytes; @@ -5197,6 +5423,7 @@ _ssl_MemoryBIO_write_impl(PySSLMemoryBIO *self, Py_buffer *b) } /*[clinic input] +@critical_section _ssl.MemoryBIO.write_eof Write an EOF marker to the memory BIO. @@ -5206,7 +5433,7 @@ When all data has been read, the "eof" property will be True. static PyObject * _ssl_MemoryBIO_write_eof_impl(PySSLMemoryBIO *self) -/*[clinic end generated code: output=d4106276ccd1ed34 input=56a945f1d29e8bd6]*/ +/*[clinic end generated code: output=d4106276ccd1ed34 input=1e914231b1c5900a]*/ { self->eof_written = 1; /* After an EOF is written, a zero return from read() should be a real EOF @@ -5218,10 +5445,8 @@ _ssl_MemoryBIO_write_eof_impl(PySSLMemoryBIO *self) } static PyGetSetDef memory_bio_getsetlist[] = { - {"pending", (getter) memory_bio_get_pending, NULL, - PySSL_memory_bio_pending_doc}, - {"eof", (getter) memory_bio_get_eof, NULL, - PySSL_memory_bio_eof_doc}, + _SSL_MEMORYBIO_PENDING_GETSETDEF + _SSL_MEMORYBIO_EOF_GETSETDEF {NULL}, /* sentinel */ }; @@ -5270,14 +5495,14 @@ PySSLSession_dealloc(PySSLSession *self) static PyObject * PySSLSession_richcompare(PyObject *left, PyObject *right, int op) { - int result; - PyTypeObject *sesstype = ((PySSLSession*)left)->ctx->state->PySSLSession_Type; - if (left == NULL || right == NULL) { PyErr_BadInternalCall(); return NULL; } + int result; + PyTypeObject *sesstype = ((PySSLSession*)left)->ctx->state->PySSLSession_Type; + if (!Py_IS_TYPE(left, sesstype) || !Py_IS_TYPE(right, sesstype)) { Py_RETURN_NOTIMPLEMENTED; } @@ -5341,70 +5566,99 @@ PySSLSession_clear(PySSLSession *self) } +/*[clinic input] +@critical_section +@getter +_ssl.SSLSession.time + +Session creation time (seconds since epoch). +[clinic start generated code]*/ + static PyObject * -PySSLSession_get_time(PySSLSession *self, void *closure) { +_ssl_SSLSession_time_get_impl(PySSLSession *self) +/*[clinic end generated code: output=4b887b9299de9be4 input=67f2325284450ae2]*/ +{ +#if OPENSSL_VERSION_NUMBER >= 0x30300000L + return _PyLong_FromTime_t(SSL_SESSION_get_time_ex(self->session)); +#else return PyLong_FromLong(SSL_SESSION_get_time(self->session)); +#endif } -PyDoc_STRVAR(PySSLSession_get_time_doc, -"Session creation time (seconds since epoch)."); +/*[clinic input] +@critical_section +@getter +_ssl.SSLSession.timeout +Session timeout (delta in seconds). +[clinic start generated code]*/ static PyObject * -PySSLSession_get_timeout(PySSLSession *self, void *closure) { - return PyLong_FromLong(SSL_SESSION_get_timeout(self->session)); +_ssl_SSLSession_timeout_get_impl(PySSLSession *self) +/*[clinic end generated code: output=82339c148ab2f7d1 input=cd17c2b087c442f2]*/ +{ + long timeout = SSL_SESSION_get_timeout(self->session); + PyObject *res = PyLong_FromLong(timeout); + return res; } -PyDoc_STRVAR(PySSLSession_get_timeout_doc, -"Session timeout (delta in seconds)."); +/*[clinic input] +@critical_section +@getter +_ssl.SSLSession.ticket_lifetime_hint +Ticket life time hint. +[clinic start generated code]*/ static PyObject * -PySSLSession_get_ticket_lifetime_hint(PySSLSession *self, void *closure) { +_ssl_SSLSession_ticket_lifetime_hint_get_impl(PySSLSession *self) +/*[clinic end generated code: output=c8b6db498136c275 input=f0e2df50961a7806]*/ +{ unsigned long hint = SSL_SESSION_get_ticket_lifetime_hint(self->session); return PyLong_FromUnsignedLong(hint); } -PyDoc_STRVAR(PySSLSession_get_ticket_lifetime_hint_doc, -"Ticket life time hint."); +/*[clinic input] +@critical_section +@getter +_ssl.SSLSession.id +Session ID. +[clinic start generated code]*/ static PyObject * -PySSLSession_get_session_id(PySSLSession *self, void *closure) { +_ssl_SSLSession_id_get_impl(PySSLSession *self) +/*[clinic end generated code: output=c532fb96b10c5adf input=0a379e64312b776d]*/ + +{ const unsigned char *id; unsigned int len; id = SSL_SESSION_get_id(self->session, &len); return PyBytes_FromStringAndSize((const char *)id, len); } -PyDoc_STRVAR(PySSLSession_get_session_id_doc, -"Session id"); +/*[clinic input] +@critical_section +@getter +_ssl.SSLSession.has_ticket +Does the session contain a ticket? +[clinic start generated code]*/ static PyObject * -PySSLSession_get_has_ticket(PySSLSession *self, void *closure) { - if (SSL_SESSION_has_ticket(self->session)) { - Py_RETURN_TRUE; - } else { - Py_RETURN_FALSE; - } +_ssl_SSLSession_has_ticket_get_impl(PySSLSession *self) +/*[clinic end generated code: output=aa3ccfc40b10b96d input=fa475555f53a5086]*/ +{ + int res = SSL_SESSION_has_ticket(self->session); + return res ? Py_True : Py_False; } -PyDoc_STRVAR(PySSLSession_get_has_ticket_doc, -"Does the session contain a ticket?"); - - static PyGetSetDef PySSLSession_getsetlist[] = { - {"has_ticket", (getter) PySSLSession_get_has_ticket, NULL, - PySSLSession_get_has_ticket_doc}, - {"id", (getter) PySSLSession_get_session_id, NULL, - PySSLSession_get_session_id_doc}, - {"ticket_lifetime_hint", (getter) PySSLSession_get_ticket_lifetime_hint, - NULL, PySSLSession_get_ticket_lifetime_hint_doc}, - {"time", (getter) PySSLSession_get_time, NULL, - PySSLSession_get_time_doc}, - {"timeout", (getter) PySSLSession_get_timeout, NULL, - PySSLSession_get_timeout_doc}, + _SSL_SSLSESSION_HAS_TICKET_GETSETDEF + _SSL_SSLSESSION_ID_GETSETDEF + _SSL_SSLSESSION_TICKET_LIFETIME_HINT_GETSETDEF + _SSL_SSLSESSION_TIME_GETSETDEF + _SSL_SSLSESSION_TIMEOUT_GETSETDEF {NULL}, /* sentinel */ }; @@ -5429,6 +5683,7 @@ static PyType_Spec PySSLSession_spec = { /* helper routines for seeding the SSL PRNG */ /*[clinic input] +@critical_section _ssl.RAND_add string as view: Py_buffer(accept={str, buffer}) entropy: double @@ -5442,7 +5697,7 @@ string. See RFC 4086. static PyObject * _ssl_RAND_add_impl(PyObject *module, Py_buffer *view, double entropy) -/*[clinic end generated code: output=e6dd48df9c9024e9 input=5c33017422828f5c]*/ +/*[clinic end generated code: output=e6dd48df9c9024e9 input=313cb73b34db31d5]*/ { const char *buf; Py_ssize_t len, written; @@ -5498,6 +5753,7 @@ PySSL_RAND(PyObject *module, int len, int pseudo) } /*[clinic input] +@critical_section _ssl.RAND_bytes n: int / @@ -5507,13 +5763,14 @@ Generate n cryptographically strong pseudo-random bytes. static PyObject * _ssl_RAND_bytes_impl(PyObject *module, int n) -/*[clinic end generated code: output=977da635e4838bc7 input=678ddf2872dfebfc]*/ +/*[clinic end generated code: output=977da635e4838bc7 input=2e78ce1e86336776]*/ { return PySSL_RAND(module, n, 0); } /*[clinic input] +@critical_section _ssl.RAND_status Returns True if the OpenSSL PRNG has been seeded with enough data and False if not. @@ -5524,12 +5781,13 @@ using the ssl() function. static PyObject * _ssl_RAND_status_impl(PyObject *module) -/*[clinic end generated code: output=7e0aaa2d39fdc1ad input=d5ae5aea52f36e01]*/ +/*[clinic end generated code: output=7e0aaa2d39fdc1ad input=636fb5659ea2e727]*/ { return PyBool_FromLong(RAND_status()); } /*[clinic input] +@critical_section _ssl.get_default_verify_paths Return search paths and environment vars that are used by SSLContext's set_default_verify_paths() to load default CAs. @@ -5539,7 +5797,7 @@ The values are 'cert_file_env', 'cert_file', 'cert_dir_env', 'cert_dir'. static PyObject * _ssl_get_default_verify_paths_impl(PyObject *module) -/*[clinic end generated code: output=e5b62a466271928b input=5210c953d98c3eb5]*/ +/*[clinic end generated code: output=e5b62a466271928b input=c6ae00bc04eb2b6e]*/ { PyObject *ofile_env = NULL; PyObject *ofile = NULL; @@ -5588,6 +5846,7 @@ asn1obj2py(_sslmodulestate *state, ASN1_OBJECT *obj) } /*[clinic input] +@critical_section _ssl.txt2obj txt: str name: bool = False @@ -5600,7 +5859,7 @@ long name are also matched. static PyObject * _ssl_txt2obj_impl(PyObject *module, const char *txt, int name) -/*[clinic end generated code: output=c38e3991347079c1 input=1c1e7d0aa7c48602]*/ +/*[clinic end generated code: output=c38e3991347079c1 input=c99b134d70173c5e]*/ { PyObject *result = NULL; ASN1_OBJECT *obj; @@ -5616,6 +5875,7 @@ _ssl_txt2obj_impl(PyObject *module, const char *txt, int name) } /*[clinic input] +@critical_section _ssl.nid2obj nid: int / @@ -5625,7 +5885,7 @@ Lookup NID, short name, long name and OID of an ASN1_OBJECT by NID. static PyObject * _ssl_nid2obj_impl(PyObject *module, int nid) -/*[clinic end generated code: output=4a98ab691cd4f84a input=51787a3bee7d8f98]*/ +/*[clinic end generated code: output=4a98ab691cd4f84a input=1b1170506fa83a53]*/ { PyObject *result = NULL; ASN1_OBJECT *obj; @@ -5777,6 +6037,7 @@ ssl_collect_certificates(const char *store_name) } /*[clinic input] +@critical_section _ssl.enum_certificates store_name: str @@ -5791,7 +6052,7 @@ a set of OIDs or the boolean True. static PyObject * _ssl_enum_certificates_impl(PyObject *module, const char *store_name) -/*[clinic end generated code: output=5134dc8bb3a3c893 input=915f60d70461ea4e]*/ +/*[clinic end generated code: output=5134dc8bb3a3c893 input=263c22e6c6988cf3]*/ { HCERTSTORE hCollectionStore = NULL; PCCERT_CONTEXT pCertCtx = NULL; @@ -5876,6 +6137,7 @@ _ssl_enum_certificates_impl(PyObject *module, const char *store_name) } /*[clinic input] +@critical_section _ssl.enum_crls store_name: str @@ -5889,7 +6151,7 @@ X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. static PyObject * _ssl_enum_crls_impl(PyObject *module, const char *store_name) -/*[clinic end generated code: output=bce467f60ccd03b6 input=a1f1d7629f1c5d3d]*/ +/*[clinic end generated code: output=bce467f60ccd03b6 input=51a1b1059e55ce43]*/ { HCERTSTORE hCollectionStore = NULL; PCCRL_CONTEXT pCrlCtx = NULL; @@ -6322,6 +6584,12 @@ sslmodule_init_constants(PyObject *m) addbool(m, "HAS_PSK", 1); #endif +#ifdef SSL_VERIFY_POST_HANDSHAKE + addbool(m, "HAS_PHA", 1); +#else + addbool(m, "HAS_PHA", 0); +#endif + #undef addbool #undef ADD_INT_CONST @@ -6525,6 +6793,7 @@ static PyModuleDef_Slot sslmodule_slots[] = { {Py_mod_exec, sslmodule_init_strings}, {Py_mod_exec, sslmodule_init_lock}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_ssl/cert.c b/Modules/_ssl/cert.c index bda66dc4d94ae6..c11ed8e3a282e6 100644 --- a/Modules/_ssl/cert.c +++ b/Modules/_ssl/cert.c @@ -153,10 +153,13 @@ _x509name_print(_sslmodulestate *state, X509_NAME *name, int indent, unsigned lo * PySSLCertificate_Type */ +#define _PySSLCertificate_CAST(op) ((PySSLCertificate *)(op)) + static PyObject * -certificate_repr(PySSLCertificate *self) +certificate_repr(PyObject *op) { PyObject *osubject, *result; + PySSLCertificate *self = _PySSLCertificate_CAST(op); /* subject string is ASCII encoded, UTF-8 chars are quoted */ osubject = _x509name_print( @@ -176,8 +179,9 @@ certificate_repr(PySSLCertificate *self) } static Py_hash_t -certificate_hash(PySSLCertificate *self) +certificate_hash(PyObject *op) { + PySSLCertificate *self = _PySSLCertificate_CAST(op); if (self->hash == (Py_hash_t)-1) { unsigned long hash; hash = X509_subject_name_hash(self->cert); @@ -191,19 +195,20 @@ certificate_hash(PySSLCertificate *self) } static PyObject * -certificate_richcompare(PySSLCertificate *self, PyObject *other, int op) +certificate_richcompare(PyObject *lhs, PyObject *rhs, int op) { int cmp; + PySSLCertificate *self = _PySSLCertificate_CAST(lhs); _sslmodulestate *state = get_state_cert(self); - if (Py_TYPE(other) != state->PySSLCertificate_Type) { + if (Py_TYPE(rhs) != state->PySSLCertificate_Type) { Py_RETURN_NOTIMPLEMENTED; } /* only support == and != */ if ((op != Py_EQ) && (op != Py_NE)) { Py_RETURN_NOTIMPLEMENTED; } - cmp = X509_cmp(self->cert, ((PySSLCertificate*)other)->cert); + cmp = X509_cmp(self->cert, ((PySSLCertificate*)rhs)->cert); if (((op == Py_EQ) && (cmp == 0)) || ((op == Py_NE) && (cmp != 0))) { Py_RETURN_TRUE; } else { @@ -212,11 +217,12 @@ certificate_richcompare(PySSLCertificate *self, PyObject *other, int op) } static void -certificate_dealloc(PySSLCertificate *self) +certificate_dealloc(PyObject *op) { + PySSLCertificate *self = _PySSLCertificate_CAST(op); PyTypeObject *tp = Py_TYPE(self); X509_free(self->cert); - Py_TYPE(self)->tp_free(self); + (void)Py_TYPE(self)->tp_free(self); Py_DECREF(tp); } diff --git a/Modules/_ssl/clinic/cert.c.h b/Modules/_ssl/clinic/cert.c.h index 0748d32291e3c5..3e0c5b405092db 100644 --- a/Modules/_ssl/clinic/cert.c.h +++ b/Modules/_ssl/clinic/cert.c.h @@ -20,7 +20,7 @@ static PyObject * _ssl_Certificate_public_bytes_impl(PySSLCertificate *self, int format); static PyObject * -_ssl_Certificate_public_bytes(PySSLCertificate *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_ssl_Certificate_public_bytes(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -52,7 +52,8 @@ _ssl_Certificate_public_bytes(PySSLCertificate *self, PyObject *const *args, Py_ Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; int format = PY_SSL_ENCODING_PEM; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -64,7 +65,7 @@ _ssl_Certificate_public_bytes(PySSLCertificate *self, PyObject *const *args, Py_ goto exit; } skip_optional_pos: - return_value = _ssl_Certificate_public_bytes_impl(self, format); + return_value = _ssl_Certificate_public_bytes_impl((PySSLCertificate *)self, format); exit: return return_value; @@ -82,8 +83,8 @@ static PyObject * _ssl_Certificate_get_info_impl(PySSLCertificate *self); static PyObject * -_ssl_Certificate_get_info(PySSLCertificate *self, PyObject *Py_UNUSED(ignored)) +_ssl_Certificate_get_info(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _ssl_Certificate_get_info_impl(self); + return _ssl_Certificate_get_info_impl((PySSLCertificate *)self); } -/*[clinic end generated code: output=5b668226f933a677 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=51365b498b975ee0 input=a9049054013a1b77]*/ diff --git a/Modules/_ssl/debughelpers.c b/Modules/_ssl/debughelpers.c index 07e9ce7a6fce2d..318c045a0eec3c 100644 --- a/Modules/_ssl/debughelpers.c +++ b/Modules/_ssl/debughelpers.c @@ -28,12 +28,12 @@ _PySSL_msg_callback(int write_p, int version, int content_type, PyObject *ssl_socket; /* ssl.SSLSocket or ssl.SSLObject */ if (ssl_obj->owner) - ssl_socket = _PyWeakref_GET_REF(ssl_obj->owner); + PyWeakref_GetRef(ssl_obj->owner, &ssl_socket); else if (ssl_obj->Socket) - ssl_socket = _PyWeakref_GET_REF(ssl_obj->Socket); + PyWeakref_GetRef(ssl_obj->Socket, &ssl_socket); else ssl_socket = (PyObject *)Py_NewRef(ssl_obj); - assert(ssl_socket != NULL); // _PyWeakref_GET_REF() can return NULL + assert(ssl_socket != NULL); // PyWeakref_GetRef() can return NULL /* assume that OpenSSL verifies all payload and buf len is of sufficient length */ @@ -180,8 +180,8 @@ _PySSLContext_set_keylog_filename(PySSLContext *self, PyObject *arg, void *c) { return 0; } - /* _Py_fopen_obj() also checks that arg is of proper type. */ - fp = _Py_fopen_obj(arg, "a" PY_STDIOTEXTMODE); + /* Py_fopen() also checks that arg is of proper type. */ + fp = Py_fopen(arg, "a" PY_STDIOTEXTMODE); if (fp == NULL) return -1; diff --git a/Modules/_ssl_data_111.h b/Modules/_ssl_data_111.h index 093c786e6a26f6..061fac2bd5822b 100644 --- a/Modules/_ssl_data_111.h +++ b/Modules/_ssl_data_111.h @@ -1,4 +1,6 @@ -/* File generated by Tools/ssl/make_ssl_data.py *//* Generated on 2023-06-01T02:58:04.081473 */ +/* File generated by Tools/ssl/make_ssl_data.py */ +/* Generated on 2024-11-27T12:48:46.194048+00:00 */ +/* Generated from Git commit OpenSSL_1_1_1w-0-ge04bd3433f */ static struct py_ssl_library_code library_codes[] = { #ifdef ERR_LIB_ASN1 {"ASN1", ERR_LIB_ASN1}, diff --git a/Modules/_ssl_data_300.h b/Modules/_ssl_data_300.h index dc66731f6b6093..b687ce43c77d66 100644 --- a/Modules/_ssl_data_300.h +++ b/Modules/_ssl_data_300.h @@ -1,4 +1,7 @@ -/* File generated by Tools/ssl/make_ssl_data.py *//* Generated on 2023-06-01T03:03:52.163218 */ +/* File generated by Tools/ssl/make_ssl_data.py */ +/* Generated on 2023-06-01T03:03:52.163218 */ +/* Manually edited to add definitions from 1.1.1 (GH-105174) */ + static struct py_ssl_library_code library_codes[] = { #ifdef ERR_LIB_ASN1 {"ASN1", ERR_LIB_ASN1}, diff --git a/Modules/_ssl_data_31.h b/Modules/_ssl_data_34.h similarity index 92% rename from Modules/_ssl_data_31.h rename to Modules/_ssl_data_34.h index c589c501f4e948..d4af3e1c1fa928 100644 --- a/Modules/_ssl_data_31.h +++ b/Modules/_ssl_data_34.h @@ -1,4 +1,6 @@ -/* File generated by Tools/ssl/make_ssl_data.py *//* Generated on 2023-06-01T03:04:00.275280 */ +/* File generated by Tools/ssl/make_ssl_data.py */ +/* Generated on 2024-11-27T12:35:52.276767+00:00 */ +/* Generated from Git commit openssl-3.4.0-0-g98acb6b028 */ static struct py_ssl_library_code library_codes[] = { #ifdef ERR_LIB_ASN1 {"ASN1", ERR_LIB_ASN1}, @@ -300,6 +302,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"FIRST_NUM_TOO_LARGE", 13, 122}, #endif + #ifdef ASN1_R_GENERALIZEDTIME_IS_TOO_SHORT + {"GENERALIZEDTIME_IS_TOO_SHORT", ERR_LIB_ASN1, ASN1_R_GENERALIZEDTIME_IS_TOO_SHORT}, + #else + {"GENERALIZEDTIME_IS_TOO_SHORT", 13, 232}, + #endif #ifdef ASN1_R_HEADER_TOO_LONG {"HEADER_TOO_LONG", ERR_LIB_ASN1, ASN1_R_HEADER_TOO_LONG}, #else @@ -730,6 +737,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNSUPPORTED_TYPE", 13, 196}, #endif + #ifdef ASN1_R_UTCTIME_IS_TOO_SHORT + {"UTCTIME_IS_TOO_SHORT", ERR_LIB_ASN1, ASN1_R_UTCTIME_IS_TOO_SHORT}, + #else + {"UTCTIME_IS_TOO_SHORT", 13, 233}, + #endif #ifdef ASN1_R_WRONG_INTEGER_TYPE {"WRONG_INTEGER_TYPE", ERR_LIB_ASN1, ASN1_R_WRONG_INTEGER_TYPE}, #else @@ -845,6 +857,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"LISTEN_V6_ONLY", 32, 136}, #endif + #ifdef BIO_R_LOCAL_ADDR_NOT_AVAILABLE + {"LOCAL_ADDR_NOT_AVAILABLE", ERR_LIB_BIO, BIO_R_LOCAL_ADDR_NOT_AVAILABLE}, + #else + {"LOCAL_ADDR_NOT_AVAILABLE", 32, 111}, + #endif #ifdef BIO_R_LOOKUP_RETURNED_NOTHING {"LOOKUP_RETURNED_NOTHING", ERR_LIB_BIO, BIO_R_LOOKUP_RETURNED_NOTHING}, #else @@ -860,6 +877,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"NBIO_CONNECT_ERROR", 32, 110}, #endif + #ifdef BIO_R_NON_FATAL + {"NON_FATAL", ERR_LIB_BIO, BIO_R_NON_FATAL}, + #else + {"NON_FATAL", 32, 112}, + #endif #ifdef BIO_R_NO_ACCEPT_ADDR_OR_SERVICE_SPECIFIED {"NO_ACCEPT_ADDR_OR_SERVICE_SPECIFIED", ERR_LIB_BIO, BIO_R_NO_ACCEPT_ADDR_OR_SERVICE_SPECIFIED}, #else @@ -880,6 +902,26 @@ static struct py_ssl_error_code error_codes[] = { #else {"NO_SUCH_FILE", 32, 128}, #endif + #ifdef BIO_R_PEER_ADDR_NOT_AVAILABLE + {"PEER_ADDR_NOT_AVAILABLE", ERR_LIB_BIO, BIO_R_PEER_ADDR_NOT_AVAILABLE}, + #else + {"PEER_ADDR_NOT_AVAILABLE", 32, 114}, + #endif + #ifdef BIO_R_PORT_MISMATCH + {"PORT_MISMATCH", ERR_LIB_BIO, BIO_R_PORT_MISMATCH}, + #else + {"PORT_MISMATCH", 32, 150}, + #endif + #ifdef BIO_R_TFO_DISABLED + {"TFO_DISABLED", ERR_LIB_BIO, BIO_R_TFO_DISABLED}, + #else + {"TFO_DISABLED", 32, 106}, + #endif + #ifdef BIO_R_TFO_NO_KERNEL_SUPPORT + {"TFO_NO_KERNEL_SUPPORT", ERR_LIB_BIO, BIO_R_TFO_NO_KERNEL_SUPPORT}, + #else + {"TFO_NO_KERNEL_SUPPORT", 32, 108}, + #endif #ifdef BIO_R_TRANSFER_ERROR {"TRANSFER_ERROR", ERR_LIB_BIO, BIO_R_TRANSFER_ERROR}, #else @@ -920,6 +962,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNABLE_TO_REUSEADDR", 32, 139}, #endif + #ifdef BIO_R_UNABLE_TO_TFO + {"UNABLE_TO_TFO", ERR_LIB_BIO, BIO_R_UNABLE_TO_TFO}, + #else + {"UNABLE_TO_TFO", 32, 109}, + #endif #ifdef BIO_R_UNAVAILABLE_IP_FAMILY {"UNAVAILABLE_IP_FAMILY", ERR_LIB_BIO, BIO_R_UNAVAILABLE_IP_FAMILY}, #else @@ -1230,6 +1277,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"ERROR_VALIDATING_SIGNATURE", 58, 171}, #endif + #ifdef CMP_R_EXPECTED_POLLREQ + {"EXPECTED_POLLREQ", ERR_LIB_CMP, CMP_R_EXPECTED_POLLREQ}, + #else + {"EXPECTED_POLLREQ", 58, 104}, + #endif #ifdef CMP_R_FAILED_BUILDING_OWN_CHAIN {"FAILED_BUILDING_OWN_CHAIN", ERR_LIB_CMP, CMP_R_FAILED_BUILDING_OWN_CHAIN}, #else @@ -1250,16 +1302,51 @@ static struct py_ssl_error_code error_codes[] = { #else {"FAIL_INFO_OUT_OF_RANGE", 58, 129}, #endif + #ifdef CMP_R_GENERATE_CERTREQTEMPLATE + {"GENERATE_CERTREQTEMPLATE", ERR_LIB_CMP, CMP_R_GENERATE_CERTREQTEMPLATE}, + #else + {"GENERATE_CERTREQTEMPLATE", 58, 197}, + #endif + #ifdef CMP_R_GENERATE_CRLSTATUS + {"GENERATE_CRLSTATUS", ERR_LIB_CMP, CMP_R_GENERATE_CRLSTATUS}, + #else + {"GENERATE_CRLSTATUS", 58, 198}, + #endif + #ifdef CMP_R_GETTING_GENP + {"GETTING_GENP", ERR_LIB_CMP, CMP_R_GETTING_GENP}, + #else + {"GETTING_GENP", 58, 192}, + #endif + #ifdef CMP_R_GET_ITAV + {"GET_ITAV", ERR_LIB_CMP, CMP_R_GET_ITAV}, + #else + {"GET_ITAV", 58, 199}, + #endif #ifdef CMP_R_INVALID_ARGS {"INVALID_ARGS", ERR_LIB_CMP, CMP_R_INVALID_ARGS}, #else {"INVALID_ARGS", 58, 100}, #endif + #ifdef CMP_R_INVALID_GENP + {"INVALID_GENP", ERR_LIB_CMP, CMP_R_INVALID_GENP}, + #else + {"INVALID_GENP", 58, 193}, + #endif + #ifdef CMP_R_INVALID_KEYSPEC + {"INVALID_KEYSPEC", ERR_LIB_CMP, CMP_R_INVALID_KEYSPEC}, + #else + {"INVALID_KEYSPEC", 58, 202}, + #endif #ifdef CMP_R_INVALID_OPTION {"INVALID_OPTION", ERR_LIB_CMP, CMP_R_INVALID_OPTION}, #else {"INVALID_OPTION", 58, 174}, #endif + #ifdef CMP_R_INVALID_ROOTCAKEYUPDATE + {"INVALID_ROOTCAKEYUPDATE", ERR_LIB_CMP, CMP_R_INVALID_ROOTCAKEYUPDATE}, + #else + {"INVALID_ROOTCAKEYUPDATE", 58, 195}, + #endif #ifdef CMP_R_MISSING_CERTID {"MISSING_CERTID", ERR_LIB_CMP, CMP_R_MISSING_CERTID}, #else @@ -1425,6 +1512,21 @@ static struct py_ssl_error_code error_codes[] = { #else {"TRANSFER_ERROR", 58, 159}, #endif + #ifdef CMP_R_UNCLEAN_CTX + {"UNCLEAN_CTX", ERR_LIB_CMP, CMP_R_UNCLEAN_CTX}, + #else + {"UNCLEAN_CTX", 58, 191}, + #endif + #ifdef CMP_R_UNEXPECTED_CERTPROFILE + {"UNEXPECTED_CERTPROFILE", ERR_LIB_CMP, CMP_R_UNEXPECTED_CERTPROFILE}, + #else + {"UNEXPECTED_CERTPROFILE", 58, 196}, + #endif + #ifdef CMP_R_UNEXPECTED_CRLSTATUSLIST + {"UNEXPECTED_CRLSTATUSLIST", ERR_LIB_CMP, CMP_R_UNEXPECTED_CRLSTATUSLIST}, + #else + {"UNEXPECTED_CRLSTATUSLIST", 58, 201}, + #endif #ifdef CMP_R_UNEXPECTED_PKIBODY {"UNEXPECTED_PKIBODY", ERR_LIB_CMP, CMP_R_UNEXPECTED_PKIBODY}, #else @@ -1435,11 +1537,21 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNEXPECTED_PKISTATUS", 58, 185}, #endif + #ifdef CMP_R_UNEXPECTED_POLLREQ + {"UNEXPECTED_POLLREQ", ERR_LIB_CMP, CMP_R_UNEXPECTED_POLLREQ}, + #else + {"UNEXPECTED_POLLREQ", 58, 105}, + #endif #ifdef CMP_R_UNEXPECTED_PVNO {"UNEXPECTED_PVNO", ERR_LIB_CMP, CMP_R_UNEXPECTED_PVNO}, #else {"UNEXPECTED_PVNO", 58, 153}, #endif + #ifdef CMP_R_UNEXPECTED_SENDER + {"UNEXPECTED_SENDER", ERR_LIB_CMP, CMP_R_UNEXPECTED_SENDER}, + #else + {"UNEXPECTED_SENDER", 58, 106}, + #endif #ifdef CMP_R_UNKNOWN_ALGORITHM_ID {"UNKNOWN_ALGORITHM_ID", ERR_LIB_CMP, CMP_R_UNKNOWN_ALGORITHM_ID}, #else @@ -1450,6 +1562,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNKNOWN_CERT_TYPE", 58, 135}, #endif + #ifdef CMP_R_UNKNOWN_CRL_ISSUER + {"UNKNOWN_CRL_ISSUER", ERR_LIB_CMP, CMP_R_UNKNOWN_CRL_ISSUER}, + #else + {"UNKNOWN_CRL_ISSUER", 58, 200}, + #endif #ifdef CMP_R_UNKNOWN_PKISTATUS {"UNKNOWN_PKISTATUS", ERR_LIB_CMP, CMP_R_UNKNOWN_PKISTATUS}, #else @@ -1465,6 +1582,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNSUPPORTED_KEY_TYPE", 58, 137}, #endif + #ifdef CMP_R_UNSUPPORTED_PKIBODY + {"UNSUPPORTED_PKIBODY", ERR_LIB_CMP, CMP_R_UNSUPPORTED_PKIBODY}, + #else + {"UNSUPPORTED_PKIBODY", 58, 101}, + #endif #ifdef CMP_R_UNSUPPORTED_PROTECTION_ALG_DHBASEDMAC {"UNSUPPORTED_PROTECTION_ALG_DHBASEDMAC", ERR_LIB_CMP, CMP_R_UNSUPPORTED_PROTECTION_ALG_DHBASEDMAC}, #else @@ -1825,6 +1947,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"NO_SIGNERS", 46, 135}, #endif + #ifdef CMS_R_OPERATION_UNSUPPORTED + {"OPERATION_UNSUPPORTED", ERR_LIB_CMS, CMS_R_OPERATION_UNSUPPORTED}, + #else + {"OPERATION_UNSUPPORTED", 46, 182}, + #endif #ifdef CMS_R_PEER_KEY_ERROR {"PEER_KEY_ERROR", ERR_LIB_CMS, CMS_R_PEER_KEY_ERROR}, #else @@ -1960,6 +2087,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNSUPPORTED_RECIPIENT_TYPE", 46, 154}, #endif + #ifdef CMS_R_UNSUPPORTED_SIGNATURE_ALGORITHM + {"UNSUPPORTED_SIGNATURE_ALGORITHM", ERR_LIB_CMS, CMS_R_UNSUPPORTED_SIGNATURE_ALGORITHM}, + #else + {"UNSUPPORTED_SIGNATURE_ALGORITHM", 46, 195}, + #endif #ifdef CMS_R_UNSUPPORTED_TYPE {"UNSUPPORTED_TYPE", ERR_LIB_CMS, CMS_R_UNSUPPORTED_TYPE}, #else @@ -1985,6 +2117,31 @@ static struct py_ssl_error_code error_codes[] = { #else {"WRAP_ERROR", 46, 159}, #endif + #ifdef COMP_R_BROTLI_DECODE_ERROR + {"BROTLI_DECODE_ERROR", ERR_LIB_COMP, COMP_R_BROTLI_DECODE_ERROR}, + #else + {"BROTLI_DECODE_ERROR", 41, 102}, + #endif + #ifdef COMP_R_BROTLI_DEFLATE_ERROR + {"BROTLI_DEFLATE_ERROR", ERR_LIB_COMP, COMP_R_BROTLI_DEFLATE_ERROR}, + #else + {"BROTLI_DEFLATE_ERROR", 41, 103}, + #endif + #ifdef COMP_R_BROTLI_ENCODE_ERROR + {"BROTLI_ENCODE_ERROR", ERR_LIB_COMP, COMP_R_BROTLI_ENCODE_ERROR}, + #else + {"BROTLI_ENCODE_ERROR", 41, 106}, + #endif + #ifdef COMP_R_BROTLI_INFLATE_ERROR + {"BROTLI_INFLATE_ERROR", ERR_LIB_COMP, COMP_R_BROTLI_INFLATE_ERROR}, + #else + {"BROTLI_INFLATE_ERROR", 41, 104}, + #endif + #ifdef COMP_R_BROTLI_NOT_SUPPORTED + {"BROTLI_NOT_SUPPORTED", ERR_LIB_COMP, COMP_R_BROTLI_NOT_SUPPORTED}, + #else + {"BROTLI_NOT_SUPPORTED", 41, 105}, + #endif #ifdef COMP_R_ZLIB_DEFLATE_ERROR {"ZLIB_DEFLATE_ERROR", ERR_LIB_COMP, COMP_R_ZLIB_DEFLATE_ERROR}, #else @@ -2000,6 +2157,26 @@ static struct py_ssl_error_code error_codes[] = { #else {"ZLIB_NOT_SUPPORTED", 41, 101}, #endif + #ifdef COMP_R_ZSTD_COMPRESS_ERROR + {"ZSTD_COMPRESS_ERROR", ERR_LIB_COMP, COMP_R_ZSTD_COMPRESS_ERROR}, + #else + {"ZSTD_COMPRESS_ERROR", 41, 107}, + #endif + #ifdef COMP_R_ZSTD_DECODE_ERROR + {"ZSTD_DECODE_ERROR", ERR_LIB_COMP, COMP_R_ZSTD_DECODE_ERROR}, + #else + {"ZSTD_DECODE_ERROR", 41, 108}, + #endif + #ifdef COMP_R_ZSTD_DECOMPRESS_ERROR + {"ZSTD_DECOMPRESS_ERROR", ERR_LIB_COMP, COMP_R_ZSTD_DECOMPRESS_ERROR}, + #else + {"ZSTD_DECOMPRESS_ERROR", 41, 109}, + #endif + #ifdef COMP_R_ZSTD_NOT_SUPPORTED + {"ZSTD_NOT_SUPPORTED", ERR_LIB_COMP, COMP_R_ZSTD_NOT_SUPPORTED}, + #else + {"ZSTD_NOT_SUPPORTED", 41, 110}, + #endif #ifdef CONF_R_ERROR_LOADING_DSO {"ERROR_LOADING_DSO", ERR_LIB_CONF, CONF_R_ERROR_LOADING_DSO}, #else @@ -2085,6 +2262,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"RECURSIVE_DIRECTORY_INCLUDE", 14, 111}, #endif + #ifdef CONF_R_RECURSIVE_SECTION_REFERENCE + {"RECURSIVE_SECTION_REFERENCE", ERR_LIB_CONF, CONF_R_RECURSIVE_SECTION_REFERENCE}, + #else + {"RECURSIVE_SECTION_REFERENCE", 14, 126}, + #endif #ifdef CONF_R_RELATIVE_PATH {"RELATIVE_PATH", ERR_LIB_CONF, CONF_R_RELATIVE_PATH}, #else @@ -2370,6 +2552,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"TOO_MANY_BYTES", 15, 113}, #endif + #ifdef CRYPTO_R_TOO_MANY_NAMES + {"TOO_MANY_NAMES", ERR_LIB_CRYPTO, CRYPTO_R_TOO_MANY_NAMES}, + #else + {"TOO_MANY_NAMES", 15, 132}, + #endif #ifdef CRYPTO_R_TOO_MANY_RECORDS {"TOO_MANY_RECORDS", ERR_LIB_CRYPTO, CRYPTO_R_TOO_MANY_RECORDS}, #else @@ -2560,6 +2747,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"INVALID_SECRET", 5, 128}, #endif + #ifdef DH_R_INVALID_SIZE + {"INVALID_SIZE", ERR_LIB_DH, DH_R_INVALID_SIZE}, + #else + {"INVALID_SIZE", 5, 129}, + #endif #ifdef DH_R_KDF_PARAMETER_ERROR {"KDF_PARAMETER_ERROR", ERR_LIB_DH, DH_R_KDF_PARAMETER_ERROR}, #else @@ -2610,6 +2802,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"PEER_KEY_ERROR", 5, 111}, #endif + #ifdef DH_R_Q_TOO_LARGE + {"Q_TOO_LARGE", ERR_LIB_DH, DH_R_Q_TOO_LARGE}, + #else + {"Q_TOO_LARGE", 5, 130}, + #endif #ifdef DH_R_SHARED_INFO_ERROR {"SHARED_INFO_ERROR", ERR_LIB_DH, DH_R_SHARED_INFO_ERROR}, #else @@ -3545,6 +3742,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"GENERATE_ERROR", 6, 214}, #endif + #ifdef EVP_R_GETTING_ALGORITHMIDENTIFIER_NOT_SUPPORTED + {"GETTING_ALGORITHMIDENTIFIER_NOT_SUPPORTED", ERR_LIB_EVP, EVP_R_GETTING_ALGORITHMIDENTIFIER_NOT_SUPPORTED}, + #else + {"GETTING_ALGORITHMIDENTIFIER_NOT_SUPPORTED", 6, 229}, + #endif #ifdef EVP_R_GET_RAW_KEY_FAILED {"GET_RAW_KEY_FAILED", ERR_LIB_EVP, EVP_R_GET_RAW_KEY_FAILED}, #else @@ -3745,6 +3947,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", 6, 150}, #endif + #ifdef EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_SIGNATURE_TYPE + {"OPERATION_NOT_SUPPORTED_FOR_THIS_SIGNATURE_TYPE", ERR_LIB_EVP, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_SIGNATURE_TYPE}, + #else + {"OPERATION_NOT_SUPPORTED_FOR_THIS_SIGNATURE_TYPE", 6, 226}, + #endif #ifdef EVP_R_OUTPUT_WOULD_OVERFLOW {"OUTPUT_WOULD_OVERFLOW", ERR_LIB_EVP, EVP_R_OUTPUT_WOULD_OVERFLOW}, #else @@ -3795,6 +4002,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"SET_DEFAULT_PROPERTY_FAILURE", 6, 209}, #endif + #ifdef EVP_R_SIGNATURE_TYPE_AND_KEY_TYPE_INCOMPATIBLE + {"SIGNATURE_TYPE_AND_KEY_TYPE_INCOMPATIBLE", ERR_LIB_EVP, EVP_R_SIGNATURE_TYPE_AND_KEY_TYPE_INCOMPATIBLE}, + #else + {"SIGNATURE_TYPE_AND_KEY_TYPE_INCOMPATIBLE", 6, 228}, + #endif #ifdef EVP_R_TOO_MANY_RECORDS {"TOO_MANY_RECORDS", ERR_LIB_EVP, EVP_R_TOO_MANY_RECORDS}, #else @@ -3825,6 +4037,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNABLE_TO_SET_CALLBACKS", 6, 217}, #endif + #ifdef EVP_R_UNKNOWN_BITS + {"UNKNOWN_BITS", ERR_LIB_EVP, EVP_R_UNKNOWN_BITS}, + #else + {"UNKNOWN_BITS", 6, 166}, + #endif #ifdef EVP_R_UNKNOWN_CIPHER {"UNKNOWN_CIPHER", ERR_LIB_EVP, EVP_R_UNKNOWN_CIPHER}, #else @@ -3840,6 +4057,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNKNOWN_KEY_TYPE", 6, 207}, #endif + #ifdef EVP_R_UNKNOWN_MAX_SIZE + {"UNKNOWN_MAX_SIZE", ERR_LIB_EVP, EVP_R_UNKNOWN_MAX_SIZE}, + #else + {"UNKNOWN_MAX_SIZE", 6, 167}, + #endif #ifdef EVP_R_UNKNOWN_OPTION {"UNKNOWN_OPTION", ERR_LIB_EVP, EVP_R_UNKNOWN_OPTION}, #else @@ -3850,6 +4072,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNKNOWN_PBE_ALGORITHM", 6, 121}, #endif + #ifdef EVP_R_UNKNOWN_SECURITY_BITS + {"UNKNOWN_SECURITY_BITS", ERR_LIB_EVP, EVP_R_UNKNOWN_SECURITY_BITS}, + #else + {"UNKNOWN_SECURITY_BITS", 6, 168}, + #endif #ifdef EVP_R_UNSUPPORTED_ALGORITHM {"UNSUPPORTED_ALGORITHM", ERR_LIB_EVP, EVP_R_UNSUPPORTED_ALGORITHM}, #else @@ -4040,6 +4267,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"RESPONSE_PARSE_ERROR", 61, 104}, #endif + #ifdef HTTP_R_RESPONSE_TOO_MANY_HDRLINES + {"RESPONSE_TOO_MANY_HDRLINES", ERR_LIB_HTTP, HTTP_R_RESPONSE_TOO_MANY_HDRLINES}, + #else + {"RESPONSE_TOO_MANY_HDRLINES", 61, 130}, + #endif #ifdef HTTP_R_RETRY_TIMEOUT {"RETRY_TIMEOUT", ERR_LIB_HTTP, HTTP_R_RETRY_TIMEOUT}, #else @@ -4530,6 +4762,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNSUPPORTED_PUBLIC_KEY_TYPE", 9, 110}, #endif + #ifdef PKCS12_R_CALLBACK_FAILED + {"CALLBACK_FAILED", ERR_LIB_PKCS12, PKCS12_R_CALLBACK_FAILED}, + #else + {"CALLBACK_FAILED", 35, 115}, + #endif #ifdef PKCS12_R_CANT_PACK_STRUCTURE {"CANT_PACK_STRUCTURE", ERR_LIB_PKCS12, PKCS12_R_CANT_PACK_STRUCTURE}, #else @@ -4920,6 +5157,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"CIPHER_OPERATION_FAILED", 57, 102}, #endif + #ifdef PROV_R_COFACTOR_REQUIRED + {"COFACTOR_REQUIRED", ERR_LIB_PROV, PROV_R_COFACTOR_REQUIRED}, + #else + {"COFACTOR_REQUIRED", 57, 236}, + #endif #ifdef PROV_R_DERIVATION_FUNCTION_INIT_FAILED {"DERIVATION_FUNCTION_INIT_FAILED", ERR_LIB_PROV, PROV_R_DERIVATION_FUNCTION_INIT_FAILED}, #else @@ -4935,6 +5177,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"EMS_NOT_ENABLED", 57, 233}, #endif + #ifdef PROV_R_ENTROPY_SOURCE_FAILED_CONTINUOUS_TESTS + {"ENTROPY_SOURCE_FAILED_CONTINUOUS_TESTS", ERR_LIB_PROV, PROV_R_ENTROPY_SOURCE_FAILED_CONTINUOUS_TESTS}, + #else + {"ENTROPY_SOURCE_FAILED_CONTINUOUS_TESTS", 57, 244}, + #endif #ifdef PROV_R_ENTROPY_SOURCE_STRENGTH_TOO_WEAK {"ENTROPY_SOURCE_STRENGTH_TOO_WEAK", ERR_LIB_PROV, PROV_R_ENTROPY_SOURCE_STRENGTH_TOO_WEAK}, #else @@ -4990,6 +5237,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"FAILED_TO_SIGN", 57, 175}, #endif + #ifdef PROV_R_FINAL_CALL_OUT_OF_ORDER + {"FINAL_CALL_OUT_OF_ORDER", ERR_LIB_PROV, PROV_R_FINAL_CALL_OUT_OF_ORDER}, + #else + {"FINAL_CALL_OUT_OF_ORDER", 57, 237}, + #endif #ifdef PROV_R_FIPS_MODULE_CONDITIONAL_ERROR {"FIPS_MODULE_CONDITIONAL_ERROR", ERR_LIB_PROV, PROV_R_FIPS_MODULE_CONDITIONAL_ERROR}, #else @@ -5020,6 +5272,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"INDICATOR_INTEGRITY_FAILURE", 57, 210}, #endif + #ifdef PROV_R_INIT_CALL_OUT_OF_ORDER + {"INIT_CALL_OUT_OF_ORDER", ERR_LIB_PROV, PROV_R_INIT_CALL_OUT_OF_ORDER}, + #else + {"INIT_CALL_OUT_OF_ORDER", 57, 238}, + #endif #ifdef PROV_R_INSUFFICIENT_DRBG_STRENGTH {"INSUFFICIENT_DRBG_STRENGTH", ERR_LIB_PROV, PROV_R_INSUFFICIENT_DRBG_STRENGTH}, #else @@ -5030,6 +5287,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"INVALID_AAD", 57, 108}, #endif + #ifdef PROV_R_INVALID_AEAD + {"INVALID_AEAD", ERR_LIB_PROV, PROV_R_INVALID_AEAD}, + #else + {"INVALID_AEAD", 57, 231}, + #endif #ifdef PROV_R_INVALID_CONFIG_DATA {"INVALID_CONFIG_DATA", ERR_LIB_PROV, PROV_R_INVALID_CONFIG_DATA}, #else @@ -5070,6 +5332,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"INVALID_DIGEST_SIZE", 57, 218}, #endif + #ifdef PROV_R_INVALID_EDDSA_INSTANCE_FOR_ATTEMPTED_OPERATION + {"INVALID_EDDSA_INSTANCE_FOR_ATTEMPTED_OPERATION", ERR_LIB_PROV, PROV_R_INVALID_EDDSA_INSTANCE_FOR_ATTEMPTED_OPERATION}, + #else + {"INVALID_EDDSA_INSTANCE_FOR_ATTEMPTED_OPERATION", 57, 243}, + #endif #ifdef PROV_R_INVALID_INPUT_LENGTH {"INVALID_INPUT_LENGTH", ERR_LIB_PROV, PROV_R_INVALID_INPUT_LENGTH}, #else @@ -5085,6 +5352,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"INVALID_IV_LENGTH", 57, 109}, #endif + #ifdef PROV_R_INVALID_KDF + {"INVALID_KDF", ERR_LIB_PROV, PROV_R_INVALID_KDF}, + #else + {"INVALID_KDF", 57, 232}, + #endif #ifdef PROV_R_INVALID_KEY {"INVALID_KEY", ERR_LIB_PROV, PROV_R_INVALID_KEY}, #else @@ -5100,6 +5372,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"INVALID_MAC", 57, 151}, #endif + #ifdef PROV_R_INVALID_MEMORY_SIZE + {"INVALID_MEMORY_SIZE", ERR_LIB_PROV, PROV_R_INVALID_MEMORY_SIZE}, + #else + {"INVALID_MEMORY_SIZE", 57, 235}, + #endif #ifdef PROV_R_INVALID_MGF1_MD {"INVALID_MGF1_MD", ERR_LIB_PROV, PROV_R_INVALID_MGF1_MD}, #else @@ -5120,6 +5397,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"INVALID_PADDING_MODE", 57, 168}, #endif + #ifdef PROV_R_INVALID_PREHASHED_DIGEST_LENGTH + {"INVALID_PREHASHED_DIGEST_LENGTH", ERR_LIB_PROV, PROV_R_INVALID_PREHASHED_DIGEST_LENGTH}, + #else + {"INVALID_PREHASHED_DIGEST_LENGTH", 57, 241}, + #endif #ifdef PROV_R_INVALID_PUBINFO {"INVALID_PUBINFO", ERR_LIB_PROV, PROV_R_INVALID_PUBINFO}, #else @@ -5155,6 +5437,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"INVALID_TAG_LENGTH", 57, 118}, #endif + #ifdef PROV_R_INVALID_THREAD_POOL_SIZE + {"INVALID_THREAD_POOL_SIZE", ERR_LIB_PROV, PROV_R_INVALID_THREAD_POOL_SIZE}, + #else + {"INVALID_THREAD_POOL_SIZE", 57, 234}, + #endif #ifdef PROV_R_INVALID_UKM_LENGTH {"INVALID_UKM_LENGTH", ERR_LIB_PROV, PROV_R_INVALID_UKM_LENGTH}, #else @@ -5300,6 +5587,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"NOT_XOF_OR_INVALID_LENGTH", 57, 113}, #endif + #ifdef PROV_R_NO_INSTANCE_ALLOWED + {"NO_INSTANCE_ALLOWED", ERR_LIB_PROV, PROV_R_NO_INSTANCE_ALLOWED}, + #else + {"NO_INSTANCE_ALLOWED", 57, 242}, + #endif #ifdef PROV_R_NO_KEY_SET {"NO_KEY_SET", ERR_LIB_PROV, PROV_R_NO_KEY_SET}, #else @@ -5310,6 +5602,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"NO_PARAMETERS_SET", 57, 177}, #endif + #ifdef PROV_R_ONESHOT_CALL_OUT_OF_ORDER + {"ONESHOT_CALL_OUT_OF_ORDER", ERR_LIB_PROV, PROV_R_ONESHOT_CALL_OUT_OF_ORDER}, + #else + {"ONESHOT_CALL_OUT_OF_ORDER", 57, 239}, + #endif #ifdef PROV_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE {"OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", ERR_LIB_PROV, PROV_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE}, #else @@ -5460,6 +5757,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNSUPPORTED_NUMBER_OF_ROUNDS", 57, 152}, #endif + #ifdef PROV_R_UPDATE_CALL_OUT_OF_ORDER + {"UPDATE_CALL_OUT_OF_ORDER", ERR_LIB_PROV, PROV_R_UPDATE_CALL_OUT_OF_ORDER}, + #else + {"UPDATE_CALL_OUT_OF_ORDER", 57, 240}, + #endif #ifdef PROV_R_URI_AUTHORITY_UNSUPPORTED {"URI_AUTHORITY_UNSUPPORTED", ERR_LIB_PROV, PROV_R_URI_AUTHORITY_UNSUPPORTED}, #else @@ -5595,6 +5897,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"INTERNAL_ERROR", 36, 113}, #endif + #ifdef RAND_R_INVALID_PROPERTY_QUERY + {"INVALID_PROPERTY_QUERY", ERR_LIB_RAND, RAND_R_INVALID_PROPERTY_QUERY}, + #else + {"INVALID_PROPERTY_QUERY", 36, 137}, + #endif #ifdef RAND_R_IN_ERROR_STATE {"IN_ERROR_STATE", ERR_LIB_RAND, RAND_R_IN_ERROR_STATE}, #else @@ -6210,6 +6517,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"AT_LEAST_TLS_1_2_NEEDED_IN_SUITEB_MODE", 20, 158}, #endif + #ifdef SSL_R_BAD_CERTIFICATE + {"BAD_CERTIFICATE", ERR_LIB_SSL, SSL_R_BAD_CERTIFICATE}, + #else + {"BAD_CERTIFICATE", 20, 348}, + #endif #ifdef SSL_R_BAD_CHANGE_CIPHER_SPEC {"BAD_CHANGE_CIPHER_SPEC", ERR_LIB_SSL, SSL_R_BAD_CHANGE_CIPHER_SPEC}, #else @@ -6220,6 +6532,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"BAD_CIPHER", 20, 186}, #endif + #ifdef SSL_R_BAD_COMPRESSION_ALGORITHM + {"BAD_COMPRESSION_ALGORITHM", ERR_LIB_SSL, SSL_R_BAD_COMPRESSION_ALGORITHM}, + #else + {"BAD_COMPRESSION_ALGORITHM", 20, 326}, + #endif #ifdef SSL_R_BAD_DATA {"BAD_DATA", ERR_LIB_SSL, SSL_R_BAD_DATA}, #else @@ -6495,6 +6812,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"CONNECTION_TYPE_NOT_SET", 20, 144}, #endif + #ifdef SSL_R_CONN_USE_ONLY + {"CONN_USE_ONLY", ERR_LIB_SSL, SSL_R_CONN_USE_ONLY}, + #else + {"CONN_USE_ONLY", 20, 356}, + #endif #ifdef SSL_R_CONTEXT_NOT_DANE_ENABLED {"CONTEXT_NOT_DANE_ENABLED", ERR_LIB_SSL, SSL_R_CONTEXT_NOT_DANE_ENABLED}, #else @@ -6635,6 +6957,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"EE_KEY_TOO_SMALL", 20, 399}, #endif + #ifdef SSL_R_EMPTY_RAW_PUBLIC_KEY + {"EMPTY_RAW_PUBLIC_KEY", ERR_LIB_SSL, SSL_R_EMPTY_RAW_PUBLIC_KEY}, + #else + {"EMPTY_RAW_PUBLIC_KEY", 20, 349}, + #endif #ifdef SSL_R_EMPTY_SRTP_PROTECTION_PROFILE_LIST {"EMPTY_SRTP_PROTECTION_PROFILE_LIST", ERR_LIB_SSL, SSL_R_EMPTY_SRTP_PROTECTION_PROFILE_LIST}, #else @@ -6650,6 +6977,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"ERROR_IN_RECEIVED_CIPHER_LIST", 20, 151}, #endif + #ifdef SSL_R_ERROR_IN_SYSTEM_DEFAULT_CONFIG + {"ERROR_IN_SYSTEM_DEFAULT_CONFIG", ERR_LIB_SSL, SSL_R_ERROR_IN_SYSTEM_DEFAULT_CONFIG}, + #else + {"ERROR_IN_SYSTEM_DEFAULT_CONFIG", 20, 419}, + #endif #ifdef SSL_R_ERROR_SETTING_TLSA_BASE_DOMAIN {"ERROR_SETTING_TLSA_BASE_DOMAIN", ERR_LIB_SSL, SSL_R_ERROR_SETTING_TLSA_BASE_DOMAIN}, #else @@ -6680,11 +7012,26 @@ static struct py_ssl_error_code error_codes[] = { #else {"EXT_LENGTH_MISMATCH", 20, 163}, #endif + #ifdef SSL_R_FAILED_TO_GET_PARAMETER + {"FAILED_TO_GET_PARAMETER", ERR_LIB_SSL, SSL_R_FAILED_TO_GET_PARAMETER}, + #else + {"FAILED_TO_GET_PARAMETER", 20, 316}, + #endif #ifdef SSL_R_FAILED_TO_INIT_ASYNC {"FAILED_TO_INIT_ASYNC", ERR_LIB_SSL, SSL_R_FAILED_TO_INIT_ASYNC}, #else {"FAILED_TO_INIT_ASYNC", 20, 405}, #endif + #ifdef SSL_R_FEATURE_NEGOTIATION_NOT_COMPLETE + {"FEATURE_NEGOTIATION_NOT_COMPLETE", ERR_LIB_SSL, SSL_R_FEATURE_NEGOTIATION_NOT_COMPLETE}, + #else + {"FEATURE_NEGOTIATION_NOT_COMPLETE", 20, 417}, + #endif + #ifdef SSL_R_FEATURE_NOT_RENEGOTIABLE + {"FEATURE_NOT_RENEGOTIABLE", ERR_LIB_SSL, SSL_R_FEATURE_NOT_RENEGOTIABLE}, + #else + {"FEATURE_NOT_RENEGOTIABLE", 20, 413}, + #endif #ifdef SSL_R_FRAGMENTED_CLIENT_HELLO {"FRAGMENTED_CLIENT_HELLO", ERR_LIB_SSL, SSL_R_FRAGMENTED_CLIENT_HELLO}, #else @@ -6805,6 +7152,16 @@ static struct py_ssl_error_code error_codes[] = { #else {"INVALID_NULL_CMD_NAME", 20, 385}, #endif + #ifdef SSL_R_INVALID_RAW_PUBLIC_KEY + {"INVALID_RAW_PUBLIC_KEY", ERR_LIB_SSL, SSL_R_INVALID_RAW_PUBLIC_KEY}, + #else + {"INVALID_RAW_PUBLIC_KEY", 20, 350}, + #endif + #ifdef SSL_R_INVALID_RECORD + {"INVALID_RECORD", ERR_LIB_SSL, SSL_R_INVALID_RECORD}, + #else + {"INVALID_RECORD", 20, 317}, + #endif #ifdef SSL_R_INVALID_SEQUENCE_NUMBER {"INVALID_SEQUENCE_NUMBER", ERR_LIB_SSL, SSL_R_INVALID_SEQUENCE_NUMBER}, #else @@ -6865,6 +7222,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"LIBRARY_HAS_NO_CIPHERS", 20, 161}, #endif + #ifdef SSL_R_MAXIMUM_ENCRYPTED_PKTS_REACHED + {"MAXIMUM_ENCRYPTED_PKTS_REACHED", ERR_LIB_SSL, SSL_R_MAXIMUM_ENCRYPTED_PKTS_REACHED}, + #else + {"MAXIMUM_ENCRYPTED_PKTS_REACHED", 20, 395}, + #endif #ifdef SSL_R_MISSING_DSA_SIGNING_CERT {"MISSING_DSA_SIGNING_CERT", ERR_LIB_SSL, SSL_R_MISSING_DSA_SIGNING_CERT}, #else @@ -6925,6 +7287,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"MISSING_SUPPORTED_GROUPS_EXTENSION", 20, 209}, #endif + #ifdef SSL_R_MISSING_SUPPORTED_VERSIONS_EXTENSION + {"MISSING_SUPPORTED_VERSIONS_EXTENSION", ERR_LIB_SSL, SSL_R_MISSING_SUPPORTED_VERSIONS_EXTENSION}, + #else + {"MISSING_SUPPORTED_VERSIONS_EXTENSION", 20, 420}, + #endif #ifdef SSL_R_MISSING_TMP_DH_KEY {"MISSING_TMP_DH_KEY", ERR_LIB_SSL, SSL_R_MISSING_TMP_DH_KEY}, #else @@ -7065,6 +7432,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"NO_SRTP_PROFILES", 20, 359}, #endif + #ifdef SSL_R_NO_STREAM + {"NO_STREAM", ERR_LIB_SSL, SSL_R_NO_STREAM}, + #else + {"NO_STREAM", 20, 355}, + #endif #ifdef SSL_R_NO_SUITABLE_DIGEST_ALGORITHM {"NO_SUITABLE_DIGEST_ALGORITHM", ERR_LIB_SSL, SSL_R_NO_SUITABLE_DIGEST_ALGORITHM}, #else @@ -7080,6 +7452,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"NO_SUITABLE_KEY_SHARE", 20, 101}, #endif + #ifdef SSL_R_NO_SUITABLE_RECORD_LAYER + {"NO_SUITABLE_RECORD_LAYER", ERR_LIB_SSL, SSL_R_NO_SUITABLE_RECORD_LAYER}, + #else + {"NO_SUITABLE_RECORD_LAYER", 20, 322}, + #endif #ifdef SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM {"NO_SUITABLE_SIGNATURE_ALGORITHM", ERR_LIB_SSL, SSL_R_NO_SUITABLE_SIGNATURE_ALGORITHM}, #else @@ -7160,6 +7537,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"PIPELINE_FAILURE", 20, 406}, #endif + #ifdef SSL_R_POLL_REQUEST_NOT_SUPPORTED + {"POLL_REQUEST_NOT_SUPPORTED", ERR_LIB_SSL, SSL_R_POLL_REQUEST_NOT_SUPPORTED}, + #else + {"POLL_REQUEST_NOT_SUPPORTED", 20, 418}, + #endif #ifdef SSL_R_POST_HANDSHAKE_AUTH_ENCODING_ERR {"POST_HANDSHAKE_AUTH_ENCODING_ERR", ERR_LIB_SSL, SSL_R_POST_HANDSHAKE_AUTH_ENCODING_ERR}, #else @@ -7190,6 +7572,21 @@ static struct py_ssl_error_code error_codes[] = { #else {"PSK_NO_SERVER_CB", 20, 225}, #endif + #ifdef SSL_R_QUIC_HANDSHAKE_LAYER_ERROR + {"QUIC_HANDSHAKE_LAYER_ERROR", ERR_LIB_SSL, SSL_R_QUIC_HANDSHAKE_LAYER_ERROR}, + #else + {"QUIC_HANDSHAKE_LAYER_ERROR", 20, 393}, + #endif + #ifdef SSL_R_QUIC_NETWORK_ERROR + {"QUIC_NETWORK_ERROR", ERR_LIB_SSL, SSL_R_QUIC_NETWORK_ERROR}, + #else + {"QUIC_NETWORK_ERROR", 20, 387}, + #endif + #ifdef SSL_R_QUIC_PROTOCOL_ERROR + {"QUIC_PROTOCOL_ERROR", ERR_LIB_SSL, SSL_R_QUIC_PROTOCOL_ERROR}, + #else + {"QUIC_PROTOCOL_ERROR", 20, 382}, + #endif #ifdef SSL_R_READ_BIO_NOT_SET {"READ_BIO_NOT_SET", ERR_LIB_SSL, SSL_R_READ_BIO_NOT_SET}, #else @@ -7200,6 +7597,16 @@ static struct py_ssl_error_code error_codes[] = { #else {"READ_TIMEOUT_EXPIRED", 20, 312}, #endif + #ifdef SSL_R_RECORDS_NOT_RELEASED + {"RECORDS_NOT_RELEASED", ERR_LIB_SSL, SSL_R_RECORDS_NOT_RELEASED}, + #else + {"RECORDS_NOT_RELEASED", 20, 321}, + #endif + #ifdef SSL_R_RECORD_LAYER_FAILURE + {"RECORD_LAYER_FAILURE", ERR_LIB_SSL, SSL_R_RECORD_LAYER_FAILURE}, + #else + {"RECORD_LAYER_FAILURE", 20, 313}, + #endif #ifdef SSL_R_RECORD_LENGTH_MISMATCH {"RECORD_LENGTH_MISMATCH", ERR_LIB_SSL, SSL_R_RECORD_LENGTH_MISMATCH}, #else @@ -7210,6 +7617,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"RECORD_TOO_SMALL", 20, 298}, #endif + #ifdef SSL_R_REMOTE_PEER_ADDRESS_NOT_SET + {"REMOTE_PEER_ADDRESS_NOT_SET", ERR_LIB_SSL, SSL_R_REMOTE_PEER_ADDRESS_NOT_SET}, + #else + {"REMOTE_PEER_ADDRESS_NOT_SET", 20, 346}, + #endif #ifdef SSL_R_RENEGOTIATE_EXT_TOO_LONG {"RENEGOTIATE_EXT_TOO_LONG", ERR_LIB_SSL, SSL_R_RENEGOTIATE_EXT_TOO_LONG}, #else @@ -7255,6 +7667,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"SCT_VERIFICATION_FAILED", 20, 208}, #endif + #ifdef SSL_R_SEQUENCE_CTR_WRAPPED + {"SEQUENCE_CTR_WRAPPED", ERR_LIB_SSL, SSL_R_SEQUENCE_CTR_WRAPPED}, + #else + {"SEQUENCE_CTR_WRAPPED", 20, 327}, + #endif #ifdef SSL_R_SERVERHELLO_TLSEXT {"SERVERHELLO_TLSEXT", ERR_LIB_SSL, SSL_R_SERVERHELLO_TLSEXT}, #else @@ -7325,6 +7742,16 @@ static struct py_ssl_error_code error_codes[] = { #else {"SSLV3_ALERT_BAD_CERTIFICATE", 20, 1042}, #endif + #ifdef SSL_R_SSLV3_ALERT_BAD_CERTIFICATE + {"SSLV3_ALERT_BAD_CERTIFICATE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_BAD_CERTIFICATE}, + #else + {"SSLV3_ALERT_BAD_CERTIFICATE", 20, 1042}, + #endif + #ifdef SSL_R_SSLV3_ALERT_BAD_RECORD_MAC + {"SSLV3_ALERT_BAD_RECORD_MAC", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_BAD_RECORD_MAC}, + #else + {"SSLV3_ALERT_BAD_RECORD_MAC", 20, 1020}, + #endif #ifdef SSL_R_SSLV3_ALERT_BAD_RECORD_MAC {"SSLV3_ALERT_BAD_RECORD_MAC", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_BAD_RECORD_MAC}, #else @@ -7335,11 +7762,26 @@ static struct py_ssl_error_code error_codes[] = { #else {"SSLV3_ALERT_CERTIFICATE_EXPIRED", 20, 1045}, #endif + #ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED + {"SSLV3_ALERT_CERTIFICATE_EXPIRED", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED}, + #else + {"SSLV3_ALERT_CERTIFICATE_EXPIRED", 20, 1045}, + #endif #ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED {"SSLV3_ALERT_CERTIFICATE_REVOKED", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED}, #else {"SSLV3_ALERT_CERTIFICATE_REVOKED", 20, 1044}, #endif + #ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED + {"SSLV3_ALERT_CERTIFICATE_REVOKED", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED}, + #else + {"SSLV3_ALERT_CERTIFICATE_REVOKED", 20, 1044}, + #endif + #ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN + {"SSLV3_ALERT_CERTIFICATE_UNKNOWN", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN}, + #else + {"SSLV3_ALERT_CERTIFICATE_UNKNOWN", 20, 1046}, + #endif #ifdef SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN {"SSLV3_ALERT_CERTIFICATE_UNKNOWN", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN}, #else @@ -7350,6 +7792,16 @@ static struct py_ssl_error_code error_codes[] = { #else {"SSLV3_ALERT_DECOMPRESSION_FAILURE", 20, 1030}, #endif + #ifdef SSL_R_SSLV3_ALERT_DECOMPRESSION_FAILURE + {"SSLV3_ALERT_DECOMPRESSION_FAILURE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_DECOMPRESSION_FAILURE}, + #else + {"SSLV3_ALERT_DECOMPRESSION_FAILURE", 20, 1030}, + #endif + #ifdef SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE + {"SSLV3_ALERT_HANDSHAKE_FAILURE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE}, + #else + {"SSLV3_ALERT_HANDSHAKE_FAILURE", 20, 1040}, + #endif #ifdef SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE {"SSLV3_ALERT_HANDSHAKE_FAILURE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE}, #else @@ -7360,11 +7812,26 @@ static struct py_ssl_error_code error_codes[] = { #else {"SSLV3_ALERT_ILLEGAL_PARAMETER", 20, 1047}, #endif + #ifdef SSL_R_SSLV3_ALERT_ILLEGAL_PARAMETER + {"SSLV3_ALERT_ILLEGAL_PARAMETER", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_ILLEGAL_PARAMETER}, + #else + {"SSLV3_ALERT_ILLEGAL_PARAMETER", 20, 1047}, + #endif #ifdef SSL_R_SSLV3_ALERT_NO_CERTIFICATE {"SSLV3_ALERT_NO_CERTIFICATE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_NO_CERTIFICATE}, #else {"SSLV3_ALERT_NO_CERTIFICATE", 20, 1041}, #endif + #ifdef SSL_R_SSLV3_ALERT_NO_CERTIFICATE + {"SSLV3_ALERT_NO_CERTIFICATE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_NO_CERTIFICATE}, + #else + {"SSLV3_ALERT_NO_CERTIFICATE", 20, 1041}, + #endif + #ifdef SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE + {"SSLV3_ALERT_UNEXPECTED_MESSAGE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE}, + #else + {"SSLV3_ALERT_UNEXPECTED_MESSAGE", 20, 1010}, + #endif #ifdef SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE {"SSLV3_ALERT_UNEXPECTED_MESSAGE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE}, #else @@ -7375,6 +7842,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"SSLV3_ALERT_UNSUPPORTED_CERTIFICATE", 20, 1043}, #endif + #ifdef SSL_R_SSLV3_ALERT_UNSUPPORTED_CERTIFICATE + {"SSLV3_ALERT_UNSUPPORTED_CERTIFICATE", ERR_LIB_SSL, SSL_R_SSLV3_ALERT_UNSUPPORTED_CERTIFICATE}, + #else + {"SSLV3_ALERT_UNSUPPORTED_CERTIFICATE", 20, 1043}, + #endif #ifdef SSL_R_SSL_COMMAND_SECTION_EMPTY {"SSL_COMMAND_SECTION_EMPTY", ERR_LIB_SSL, SSL_R_SSL_COMMAND_SECTION_EMPTY}, #else @@ -7450,6 +7922,36 @@ static struct py_ssl_error_code error_codes[] = { #else {"STILL_IN_INIT", 20, 121}, #endif + #ifdef SSL_R_STREAM_COUNT_LIMITED + {"STREAM_COUNT_LIMITED", ERR_LIB_SSL, SSL_R_STREAM_COUNT_LIMITED}, + #else + {"STREAM_COUNT_LIMITED", 20, 411}, + #endif + #ifdef SSL_R_STREAM_FINISHED + {"STREAM_FINISHED", ERR_LIB_SSL, SSL_R_STREAM_FINISHED}, + #else + {"STREAM_FINISHED", 20, 365}, + #endif + #ifdef SSL_R_STREAM_RECV_ONLY + {"STREAM_RECV_ONLY", ERR_LIB_SSL, SSL_R_STREAM_RECV_ONLY}, + #else + {"STREAM_RECV_ONLY", 20, 366}, + #endif + #ifdef SSL_R_STREAM_RESET + {"STREAM_RESET", ERR_LIB_SSL, SSL_R_STREAM_RESET}, + #else + {"STREAM_RESET", 20, 375}, + #endif + #ifdef SSL_R_STREAM_SEND_ONLY + {"STREAM_SEND_ONLY", ERR_LIB_SSL, SSL_R_STREAM_SEND_ONLY}, + #else + {"STREAM_SEND_ONLY", 20, 379}, + #endif + #ifdef SSL_R_TLSV13_ALERT_CERTIFICATE_REQUIRED + {"TLSV13_ALERT_CERTIFICATE_REQUIRED", ERR_LIB_SSL, SSL_R_TLSV13_ALERT_CERTIFICATE_REQUIRED}, + #else + {"TLSV13_ALERT_CERTIFICATE_REQUIRED", 20, 1116}, + #endif #ifdef SSL_R_TLSV13_ALERT_CERTIFICATE_REQUIRED {"TLSV13_ALERT_CERTIFICATE_REQUIRED", ERR_LIB_SSL, SSL_R_TLSV13_ALERT_CERTIFICATE_REQUIRED}, #else @@ -7460,6 +7962,16 @@ static struct py_ssl_error_code error_codes[] = { #else {"TLSV13_ALERT_MISSING_EXTENSION", 20, 1109}, #endif + #ifdef SSL_R_TLSV13_ALERT_MISSING_EXTENSION + {"TLSV13_ALERT_MISSING_EXTENSION", ERR_LIB_SSL, SSL_R_TLSV13_ALERT_MISSING_EXTENSION}, + #else + {"TLSV13_ALERT_MISSING_EXTENSION", 20, 1109}, + #endif + #ifdef SSL_R_TLSV1_ALERT_ACCESS_DENIED + {"TLSV1_ALERT_ACCESS_DENIED", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_ACCESS_DENIED}, + #else + {"TLSV1_ALERT_ACCESS_DENIED", 20, 1049}, + #endif #ifdef SSL_R_TLSV1_ALERT_ACCESS_DENIED {"TLSV1_ALERT_ACCESS_DENIED", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_ACCESS_DENIED}, #else @@ -7470,11 +7982,26 @@ static struct py_ssl_error_code error_codes[] = { #else {"TLSV1_ALERT_DECODE_ERROR", 20, 1050}, #endif + #ifdef SSL_R_TLSV1_ALERT_DECODE_ERROR + {"TLSV1_ALERT_DECODE_ERROR", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_DECODE_ERROR}, + #else + {"TLSV1_ALERT_DECODE_ERROR", 20, 1050}, + #endif #ifdef SSL_R_TLSV1_ALERT_DECRYPTION_FAILED {"TLSV1_ALERT_DECRYPTION_FAILED", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_DECRYPTION_FAILED}, #else {"TLSV1_ALERT_DECRYPTION_FAILED", 20, 1021}, #endif + #ifdef SSL_R_TLSV1_ALERT_DECRYPTION_FAILED + {"TLSV1_ALERT_DECRYPTION_FAILED", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_DECRYPTION_FAILED}, + #else + {"TLSV1_ALERT_DECRYPTION_FAILED", 20, 1021}, + #endif + #ifdef SSL_R_TLSV1_ALERT_DECRYPT_ERROR + {"TLSV1_ALERT_DECRYPT_ERROR", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_DECRYPT_ERROR}, + #else + {"TLSV1_ALERT_DECRYPT_ERROR", 20, 1051}, + #endif #ifdef SSL_R_TLSV1_ALERT_DECRYPT_ERROR {"TLSV1_ALERT_DECRYPT_ERROR", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_DECRYPT_ERROR}, #else @@ -7485,6 +8012,16 @@ static struct py_ssl_error_code error_codes[] = { #else {"TLSV1_ALERT_EXPORT_RESTRICTION", 20, 1060}, #endif + #ifdef SSL_R_TLSV1_ALERT_EXPORT_RESTRICTION + {"TLSV1_ALERT_EXPORT_RESTRICTION", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_EXPORT_RESTRICTION}, + #else + {"TLSV1_ALERT_EXPORT_RESTRICTION", 20, 1060}, + #endif + #ifdef SSL_R_TLSV1_ALERT_INAPPROPRIATE_FALLBACK + {"TLSV1_ALERT_INAPPROPRIATE_FALLBACK", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_INAPPROPRIATE_FALLBACK}, + #else + {"TLSV1_ALERT_INAPPROPRIATE_FALLBACK", 20, 1086}, + #endif #ifdef SSL_R_TLSV1_ALERT_INAPPROPRIATE_FALLBACK {"TLSV1_ALERT_INAPPROPRIATE_FALLBACK", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_INAPPROPRIATE_FALLBACK}, #else @@ -7495,11 +8032,36 @@ static struct py_ssl_error_code error_codes[] = { #else {"TLSV1_ALERT_INSUFFICIENT_SECURITY", 20, 1071}, #endif + #ifdef SSL_R_TLSV1_ALERT_INSUFFICIENT_SECURITY + {"TLSV1_ALERT_INSUFFICIENT_SECURITY", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_INSUFFICIENT_SECURITY}, + #else + {"TLSV1_ALERT_INSUFFICIENT_SECURITY", 20, 1071}, + #endif + #ifdef SSL_R_TLSV1_ALERT_INTERNAL_ERROR + {"TLSV1_ALERT_INTERNAL_ERROR", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_INTERNAL_ERROR}, + #else + {"TLSV1_ALERT_INTERNAL_ERROR", 20, 1080}, + #endif #ifdef SSL_R_TLSV1_ALERT_INTERNAL_ERROR {"TLSV1_ALERT_INTERNAL_ERROR", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_INTERNAL_ERROR}, #else {"TLSV1_ALERT_INTERNAL_ERROR", 20, 1080}, #endif + #ifdef SSL_R_TLSV1_ALERT_NO_APPLICATION_PROTOCOL + {"TLSV1_ALERT_NO_APPLICATION_PROTOCOL", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_NO_APPLICATION_PROTOCOL}, + #else + {"TLSV1_ALERT_NO_APPLICATION_PROTOCOL", 20, 1120}, + #endif + #ifdef SSL_R_TLSV1_ALERT_NO_APPLICATION_PROTOCOL + {"TLSV1_ALERT_NO_APPLICATION_PROTOCOL", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_NO_APPLICATION_PROTOCOL}, + #else + {"TLSV1_ALERT_NO_APPLICATION_PROTOCOL", 20, 1120}, + #endif + #ifdef SSL_R_TLSV1_ALERT_NO_RENEGOTIATION + {"TLSV1_ALERT_NO_RENEGOTIATION", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_NO_RENEGOTIATION}, + #else + {"TLSV1_ALERT_NO_RENEGOTIATION", 20, 1100}, + #endif #ifdef SSL_R_TLSV1_ALERT_NO_RENEGOTIATION {"TLSV1_ALERT_NO_RENEGOTIATION", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_NO_RENEGOTIATION}, #else @@ -7510,6 +8072,16 @@ static struct py_ssl_error_code error_codes[] = { #else {"TLSV1_ALERT_PROTOCOL_VERSION", 20, 1070}, #endif + #ifdef SSL_R_TLSV1_ALERT_PROTOCOL_VERSION + {"TLSV1_ALERT_PROTOCOL_VERSION", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_PROTOCOL_VERSION}, + #else + {"TLSV1_ALERT_PROTOCOL_VERSION", 20, 1070}, + #endif + #ifdef SSL_R_TLSV1_ALERT_RECORD_OVERFLOW + {"TLSV1_ALERT_RECORD_OVERFLOW", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_RECORD_OVERFLOW}, + #else + {"TLSV1_ALERT_RECORD_OVERFLOW", 20, 1022}, + #endif #ifdef SSL_R_TLSV1_ALERT_RECORD_OVERFLOW {"TLSV1_ALERT_RECORD_OVERFLOW", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_RECORD_OVERFLOW}, #else @@ -7520,11 +8092,36 @@ static struct py_ssl_error_code error_codes[] = { #else {"TLSV1_ALERT_UNKNOWN_CA", 20, 1048}, #endif + #ifdef SSL_R_TLSV1_ALERT_UNKNOWN_CA + {"TLSV1_ALERT_UNKNOWN_CA", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_UNKNOWN_CA}, + #else + {"TLSV1_ALERT_UNKNOWN_CA", 20, 1048}, + #endif + #ifdef SSL_R_TLSV1_ALERT_UNKNOWN_PSK_IDENTITY + {"TLSV1_ALERT_UNKNOWN_PSK_IDENTITY", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_UNKNOWN_PSK_IDENTITY}, + #else + {"TLSV1_ALERT_UNKNOWN_PSK_IDENTITY", 20, 1115}, + #endif + #ifdef SSL_R_TLSV1_ALERT_UNKNOWN_PSK_IDENTITY + {"TLSV1_ALERT_UNKNOWN_PSK_IDENTITY", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_UNKNOWN_PSK_IDENTITY}, + #else + {"TLSV1_ALERT_UNKNOWN_PSK_IDENTITY", 20, 1115}, + #endif #ifdef SSL_R_TLSV1_ALERT_USER_CANCELLED {"TLSV1_ALERT_USER_CANCELLED", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_USER_CANCELLED}, #else {"TLSV1_ALERT_USER_CANCELLED", 20, 1090}, #endif + #ifdef SSL_R_TLSV1_ALERT_USER_CANCELLED + {"TLSV1_ALERT_USER_CANCELLED", ERR_LIB_SSL, SSL_R_TLSV1_ALERT_USER_CANCELLED}, + #else + {"TLSV1_ALERT_USER_CANCELLED", 20, 1090}, + #endif + #ifdef SSL_R_TLSV1_BAD_CERTIFICATE_HASH_VALUE + {"TLSV1_BAD_CERTIFICATE_HASH_VALUE", ERR_LIB_SSL, SSL_R_TLSV1_BAD_CERTIFICATE_HASH_VALUE}, + #else + {"TLSV1_BAD_CERTIFICATE_HASH_VALUE", 20, 1114}, + #endif #ifdef SSL_R_TLSV1_BAD_CERTIFICATE_HASH_VALUE {"TLSV1_BAD_CERTIFICATE_HASH_VALUE", ERR_LIB_SSL, SSL_R_TLSV1_BAD_CERTIFICATE_HASH_VALUE}, #else @@ -7535,11 +8132,26 @@ static struct py_ssl_error_code error_codes[] = { #else {"TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE", 20, 1113}, #endif + #ifdef SSL_R_TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE + {"TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE", ERR_LIB_SSL, SSL_R_TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE}, + #else + {"TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE", 20, 1113}, + #endif #ifdef SSL_R_TLSV1_CERTIFICATE_UNOBTAINABLE {"TLSV1_CERTIFICATE_UNOBTAINABLE", ERR_LIB_SSL, SSL_R_TLSV1_CERTIFICATE_UNOBTAINABLE}, #else {"TLSV1_CERTIFICATE_UNOBTAINABLE", 20, 1111}, #endif + #ifdef SSL_R_TLSV1_CERTIFICATE_UNOBTAINABLE + {"TLSV1_CERTIFICATE_UNOBTAINABLE", ERR_LIB_SSL, SSL_R_TLSV1_CERTIFICATE_UNOBTAINABLE}, + #else + {"TLSV1_CERTIFICATE_UNOBTAINABLE", 20, 1111}, + #endif + #ifdef SSL_R_TLSV1_UNRECOGNIZED_NAME + {"TLSV1_UNRECOGNIZED_NAME", ERR_LIB_SSL, SSL_R_TLSV1_UNRECOGNIZED_NAME}, + #else + {"TLSV1_UNRECOGNIZED_NAME", 20, 1112}, + #endif #ifdef SSL_R_TLSV1_UNRECOGNIZED_NAME {"TLSV1_UNRECOGNIZED_NAME", ERR_LIB_SSL, SSL_R_TLSV1_UNRECOGNIZED_NAME}, #else @@ -7550,6 +8162,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"TLSV1_UNSUPPORTED_EXTENSION", 20, 1110}, #endif + #ifdef SSL_R_TLSV1_UNSUPPORTED_EXTENSION + {"TLSV1_UNSUPPORTED_EXTENSION", ERR_LIB_SSL, SSL_R_TLSV1_UNSUPPORTED_EXTENSION}, + #else + {"TLSV1_UNSUPPORTED_EXTENSION", 20, 1110}, + #endif #ifdef SSL_R_TLS_ILLEGAL_EXPORTER_LABEL {"TLS_ILLEGAL_EXPORTER_LABEL", ERR_LIB_SSL, SSL_R_TLS_ILLEGAL_EXPORTER_LABEL}, #else @@ -7665,6 +8282,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNKNOWN_KEY_EXCHANGE_TYPE", 20, 250}, #endif + #ifdef SSL_R_UNKNOWN_MANDATORY_PARAMETER + {"UNKNOWN_MANDATORY_PARAMETER", ERR_LIB_SSL, SSL_R_UNKNOWN_MANDATORY_PARAMETER}, + #else + {"UNKNOWN_MANDATORY_PARAMETER", 20, 323}, + #endif #ifdef SSL_R_UNKNOWN_PKEY_TYPE {"UNKNOWN_PKEY_TYPE", ERR_LIB_SSL, SSL_R_UNKNOWN_PKEY_TYPE}, #else @@ -7700,6 +8322,21 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNSUPPORTED_COMPRESSION_ALGORITHM", 20, 257}, #endif + #ifdef SSL_R_UNSUPPORTED_CONFIG_VALUE + {"UNSUPPORTED_CONFIG_VALUE", ERR_LIB_SSL, SSL_R_UNSUPPORTED_CONFIG_VALUE}, + #else + {"UNSUPPORTED_CONFIG_VALUE", 20, 414}, + #endif + #ifdef SSL_R_UNSUPPORTED_CONFIG_VALUE_CLASS + {"UNSUPPORTED_CONFIG_VALUE_CLASS", ERR_LIB_SSL, SSL_R_UNSUPPORTED_CONFIG_VALUE_CLASS}, + #else + {"UNSUPPORTED_CONFIG_VALUE_CLASS", 20, 415}, + #endif + #ifdef SSL_R_UNSUPPORTED_CONFIG_VALUE_OP + {"UNSUPPORTED_CONFIG_VALUE_OP", ERR_LIB_SSL, SSL_R_UNSUPPORTED_CONFIG_VALUE_OP}, + #else + {"UNSUPPORTED_CONFIG_VALUE_OP", 20, 416}, + #endif #ifdef SSL_R_UNSUPPORTED_ELLIPTIC_CURVE {"UNSUPPORTED_ELLIPTIC_CURVE", ERR_LIB_SSL, SSL_R_UNSUPPORTED_ELLIPTIC_CURVE}, #else @@ -7720,6 +8357,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNSUPPORTED_STATUS_TYPE", 20, 329}, #endif + #ifdef SSL_R_UNSUPPORTED_WRITE_FLAG + {"UNSUPPORTED_WRITE_FLAG", ERR_LIB_SSL, SSL_R_UNSUPPORTED_WRITE_FLAG}, + #else + {"UNSUPPORTED_WRITE_FLAG", 20, 412}, + #endif #ifdef SSL_R_USE_SRTP_NOT_NEGOTIATED {"USE_SRTP_NOT_NEGOTIATED", ERR_LIB_SSL, SSL_R_USE_SRTP_NOT_NEGOTIATED}, #else @@ -7750,6 +8392,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"WRONG_CURVE", 20, 378}, #endif + #ifdef SSL_R_WRONG_RPK_TYPE + {"WRONG_RPK_TYPE", ERR_LIB_SSL, SSL_R_WRONG_RPK_TYPE}, + #else + {"WRONG_RPK_TYPE", 20, 351}, + #endif #ifdef SSL_R_WRONG_SIGNATURE_LENGTH {"WRONG_SIGNATURE_LENGTH", ERR_LIB_SSL, SSL_R_WRONG_SIGNATURE_LENGTH}, #else @@ -8055,6 +8702,16 @@ static struct py_ssl_error_code error_codes[] = { #else {"BAD_OBJECT", 34, 119}, #endif + #ifdef X509V3_R_BAD_OPTION + {"BAD_OPTION", ERR_LIB_X509V3, X509V3_R_BAD_OPTION}, + #else + {"BAD_OPTION", 34, 170}, + #endif + #ifdef X509V3_R_BAD_VALUE + {"BAD_VALUE", ERR_LIB_X509V3, X509V3_R_BAD_VALUE}, + #else + {"BAD_VALUE", 34, 171}, + #endif #ifdef X509V3_R_BN_DEC2BN_ERROR {"BN_DEC2BN_ERROR", ERR_LIB_X509V3, X509V3_R_BN_DEC2BN_ERROR}, #else @@ -8370,6 +9027,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNKNOWN_OPTION", 34, 120}, #endif + #ifdef X509V3_R_UNKNOWN_VALUE + {"UNKNOWN_VALUE", ERR_LIB_X509V3, X509V3_R_UNKNOWN_VALUE}, + #else + {"UNKNOWN_VALUE", 34, 172}, + #endif #ifdef X509V3_R_UNSUPPORTED_OPTION {"UNSUPPORTED_OPTION", ERR_LIB_X509V3, X509V3_R_UNSUPPORTED_OPTION}, #else @@ -8430,6 +9092,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"CRL_VERIFY_FAILURE", 11, 131}, #endif + #ifdef X509_R_DUPLICATE_ATTRIBUTE + {"DUPLICATE_ATTRIBUTE", ERR_LIB_X509, X509_R_DUPLICATE_ATTRIBUTE}, + #else + {"DUPLICATE_ATTRIBUTE", 11, 140}, + #endif #ifdef X509_R_ERROR_GETTING_MD_BY_NID {"ERROR_GETTING_MD_BY_NID", ERR_LIB_X509, X509_R_ERROR_GETTING_MD_BY_NID}, #else @@ -8590,6 +9257,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNSUPPORTED_ALGORITHM", 11, 111}, #endif + #ifdef X509_R_UNSUPPORTED_VERSION + {"UNSUPPORTED_VERSION", ERR_LIB_X509, X509_R_UNSUPPORTED_VERSION}, + #else + {"UNSUPPORTED_VERSION", 11, 145}, + #endif #ifdef X509_R_WRONG_LOOKUP_TYPE {"WRONG_LOOKUP_TYPE", ERR_LIB_X509, X509_R_WRONG_LOOKUP_TYPE}, #else diff --git a/Modules/_stat.c b/Modules/_stat.c index b43e79453f5b2f..13a2bec252f448 100644 --- a/Modules/_stat.c +++ b/Modules/_stat.c @@ -11,11 +11,10 @@ * */ +// Need limited C API version 3.13 for PyModule_Add() on Windows #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.13 for PyModule_Add() on Windows -#define Py_LIMITED_API 0x030d0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -507,7 +506,7 @@ S_IWOTH: write by others\n\ S_IXOTH: execute by others\n\ \n" -"UF_SETTABLE: mask of owner changable flags\n\ +"UF_SETTABLE: mask of owner changeable flags\n\ UF_NODUMP: do not dump file\n\ UF_IMMUTABLE: file may not be changed\n\ UF_APPEND: file may only be appended to\n\ @@ -680,6 +679,7 @@ stat_exec(PyObject *module) static PyModuleDef_Slot stat_slots[] = { {Py_mod_exec, stat_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_statisticsmodule.c b/Modules/_statisticsmodule.c index a04a2a779a5d3d..b84f731ad6a1da 100644 --- a/Modules/_statisticsmodule.c +++ b/Modules/_statisticsmodule.c @@ -1,8 +1,9 @@ /* statistics accelerator C extension: _statistics module. */ -// clinic/_statisticsmodule.c.h uses internal pycore_modsupport.h API -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 +// Need limited C API version 3.13 for Py_mod_gil +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -135,6 +136,7 @@ PyDoc_STRVAR(statistics_doc, static struct PyModuleDef_Slot _statisticsmodule_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_struct.c b/Modules/_struct.c index fa2cd37e003e0a..21582b945be23d 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -12,6 +12,9 @@ #include "pycore_long.h" // _PyLong_AsByteArray() #include "pycore_moduleobject.h" // _PyModule_GetState() +#ifdef Py_HAVE_C_COMPLEX +# include "_complex.h" // complex +#endif #include // offsetof() /*[clinic input] @@ -80,6 +83,10 @@ typedef struct { char c; int x; } st_int; typedef struct { char c; long x; } st_long; typedef struct { char c; float x; } st_float; typedef struct { char c; double x; } st_double; +#ifdef Py_HAVE_C_COMPLEX +typedef struct { char c; float complex x; } st_float_complex; +typedef struct { char c; double complex x; } st_double_complex; +#endif typedef struct { char c; void *x; } st_void_p; typedef struct { char c; size_t x; } st_size_t; typedef struct { char c; _Bool x; } st_bool; @@ -89,6 +96,10 @@ typedef struct { char c; _Bool x; } st_bool; #define LONG_ALIGN (sizeof(st_long) - sizeof(long)) #define FLOAT_ALIGN (sizeof(st_float) - sizeof(float)) #define DOUBLE_ALIGN (sizeof(st_double) - sizeof(double)) +#ifdef Py_HAVE_C_COMPLEX +# define FLOAT_COMPLEX_ALIGN (sizeof(st_float_complex) - sizeof(float complex)) +# define DOUBLE_COMPLEX_ALIGN (sizeof(st_double_complex) - sizeof(double complex)) +#endif #define VOID_P_ALIGN (sizeof(st_void_p) - sizeof(void *)) #define SIZE_T_ALIGN (sizeof(st_size_t) - sizeof(size_t)) #define BOOL_ALIGN (sizeof(st_bool) - sizeof(_Bool)) @@ -281,7 +292,7 @@ get_size_t(_structmodulestate *state, PyObject *v, size_t *p) #define RANGE_ERROR(state, f, flag) return _range_error(state, f, flag) -/* Floating point helpers */ +/* Floating-point helpers */ static PyObject * unpack_halffloat(const char *p, /* start of 2-byte string */ @@ -407,7 +418,7 @@ static PyObject * nu_short(_structmodulestate *state, const char *p, const formatdef *f) { short x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromLong((long)x); } @@ -415,7 +426,7 @@ static PyObject * nu_ushort(_structmodulestate *state, const char *p, const formatdef *f) { unsigned short x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromLong((long)x); } @@ -423,7 +434,7 @@ static PyObject * nu_int(_structmodulestate *state, const char *p, const formatdef *f) { int x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromLong((long)x); } @@ -431,7 +442,7 @@ static PyObject * nu_uint(_structmodulestate *state, const char *p, const formatdef *f) { unsigned int x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromUnsignedLong((unsigned long)x); } @@ -439,7 +450,7 @@ static PyObject * nu_long(_structmodulestate *state, const char *p, const formatdef *f) { long x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromLong(x); } @@ -447,7 +458,7 @@ static PyObject * nu_ulong(_structmodulestate *state, const char *p, const formatdef *f) { unsigned long x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromUnsignedLong(x); } @@ -455,7 +466,7 @@ static PyObject * nu_ssize_t(_structmodulestate *state, const char *p, const formatdef *f) { Py_ssize_t x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromSsize_t(x); } @@ -463,7 +474,7 @@ static PyObject * nu_size_t(_structmodulestate *state, const char *p, const formatdef *f) { size_t x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromSize_t(x); } @@ -471,7 +482,7 @@ static PyObject * nu_longlong(_structmodulestate *state, const char *p, const formatdef *f) { long long x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromLongLong(x); } @@ -479,16 +490,15 @@ static PyObject * nu_ulonglong(_structmodulestate *state, const char *p, const formatdef *f) { unsigned long long x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromUnsignedLongLong(x); } static PyObject * nu_bool(_structmodulestate *state, const char *p, const formatdef *f) { - _Bool x; - memcpy((char *)&x, p, sizeof x); - return PyBool_FromLong(x != 0); + const _Bool bool_false = 0; + return PyBool_FromLong(memcmp(p, &bool_false, sizeof(_Bool))); } @@ -506,7 +516,7 @@ static PyObject * nu_float(_structmodulestate *state, const char *p, const formatdef *f) { float x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyFloat_FromDouble((double)x); } @@ -514,15 +524,35 @@ static PyObject * nu_double(_structmodulestate *state, const char *p, const formatdef *f) { double x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyFloat_FromDouble(x); } +#ifdef Py_HAVE_C_COMPLEX +static PyObject * +nu_float_complex(_structmodulestate *state, const char *p, const formatdef *f) +{ + float complex x; + + memcpy(&x, p, sizeof(x)); + return PyComplex_FromDoubles(creal(x), cimag(x)); +} + +static PyObject * +nu_double_complex(_structmodulestate *state, const char *p, const formatdef *f) +{ + double complex x; + + memcpy(&x, p, sizeof(x)); + return PyComplex_FromDoubles(creal(x), cimag(x)); +} +#endif + static PyObject * nu_void_p(_structmodulestate *state, const char *p, const formatdef *f) { void *x; - memcpy((char *)&x, p, sizeof x); + memcpy(&x, p, sizeof x); return PyLong_FromVoidPtr(x); } @@ -587,7 +617,7 @@ np_short(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) RANGE_ERROR(state, f, 0); } y = (short)x; - memcpy(p, (char *)&y, sizeof y); + memcpy(p, &y, sizeof y); return 0; } @@ -606,7 +636,7 @@ np_ushort(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) RANGE_ERROR(state, f, 1); } y = (unsigned short)x; - memcpy(p, (char *)&y, sizeof y); + memcpy(p, &y, sizeof y); return 0; } @@ -626,7 +656,7 @@ np_int(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) RANGE_ERROR(state, f, 0); #endif y = (int)x; - memcpy(p, (char *)&y, sizeof y); + memcpy(p, &y, sizeof y); return 0; } @@ -646,7 +676,7 @@ np_uint(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) if (x > ((unsigned long)UINT_MAX)) RANGE_ERROR(state, f, 1); #endif - memcpy(p, (char *)&y, sizeof y); + memcpy(p, &y, sizeof y); return 0; } @@ -660,7 +690,7 @@ np_long(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) } return -1; } - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -674,7 +704,7 @@ np_ulong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) } return -1; } - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -688,7 +718,7 @@ np_ssize_t(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) } return -1; } - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -702,7 +732,7 @@ np_size_t(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) } return -1; } - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -720,7 +750,7 @@ np_longlong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) } return -1; } - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -737,7 +767,7 @@ np_ulonglong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f } return -1; } - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -751,7 +781,7 @@ np_bool(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) if (y < 0) return -1; x = y; - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -774,7 +804,7 @@ np_float(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) "required argument is not a float"); return -1; } - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -787,10 +817,62 @@ np_double(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) "required argument is not a float"); return -1; } - memcpy(p, (char *)&x, sizeof(double)); + memcpy(p, &x, sizeof(double)); + return 0; +} + +#ifdef Py_HAVE_C_COMPLEX +static int +np_float_complex(_structmodulestate *state, char *p, PyObject *v, + const formatdef *f) +{ + Py_complex c = PyComplex_AsCComplex(v); + float complex x = CMPLXF((float)c.real, (float)c.imag); + + if (c.real == -1 && PyErr_Occurred()) { + PyErr_SetString(state->StructError, + "required argument is not a complex"); + return -1; + } + memcpy(p, &x, sizeof(x)); return 0; } +static int +np_double_complex(_structmodulestate *state, char *p, PyObject *v, + const formatdef *f) +{ + Py_complex c = PyComplex_AsCComplex(v); + double complex x = CMPLX(c.real, c.imag); + + if (c.real == -1 && PyErr_Occurred()) { + PyErr_SetString(state->StructError, + "required argument is not a complex"); + return -1; + } + memcpy(p, &x, sizeof(x)); + return 0; +} +#else +static int +np_complex_stub(_structmodulestate *state, char *p, PyObject *v, + const formatdef *f) +{ + PyErr_Format(state->StructError, + "'%c' format not supported on this system", + f->format); + return -1; +} +static PyObject * +nu_complex_stub(_structmodulestate *state, const char *p, const formatdef *f) +{ + PyErr_Format(state->StructError, + "'%c' format not supported on this system", + f->format); + return NULL; +} +#endif + static int np_void_p(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { @@ -804,7 +886,7 @@ np_void_p(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) Py_DECREF(v); if (x == NULL && PyErr_Occurred()) return -1; - memcpy(p, (char *)&x, sizeof x); + memcpy(p, &x, sizeof x); return 0; } @@ -829,6 +911,13 @@ static const formatdef native_table[] = { {'e', sizeof(short), SHORT_ALIGN, nu_halffloat, np_halffloat}, {'f', sizeof(float), FLOAT_ALIGN, nu_float, np_float}, {'d', sizeof(double), DOUBLE_ALIGN, nu_double, np_double}, +#ifdef Py_HAVE_C_COMPLEX + {'E', sizeof(float complex), FLOAT_COMPLEX_ALIGN, nu_float_complex, np_float_complex}, + {'C', sizeof(double complex), DOUBLE_COMPLEX_ALIGN, nu_double_complex, np_double_complex}, +#else + {'E', 1, 0, nu_complex_stub, np_complex_stub}, + {'C', 1, 0, nu_complex_stub, np_complex_stub}, +#endif {'P', sizeof(void *), VOID_P_ALIGN, nu_void_p, np_void_p}, {0} }; @@ -929,6 +1018,38 @@ bu_double(_structmodulestate *state, const char *p, const formatdef *f) return unpack_double(p, 0); } +#ifdef Py_HAVE_C_COMPLEX +static PyObject * +bu_float_complex(_structmodulestate *state, const char *p, const formatdef *f) +{ + double x = PyFloat_Unpack4(p, 0); + if (x == -1.0 && PyErr_Occurred()) { + return NULL; + } + double y = PyFloat_Unpack4(p + 4, 0); + if (y == -1.0 && PyErr_Occurred()) { + return NULL; + } + return PyComplex_FromDoubles(x, y); +} + +static PyObject * +bu_double_complex(_structmodulestate *state, const char *p, const formatdef *f) +{ + double x, y; + + x = PyFloat_Unpack8(p, 0); + if (x == -1.0 && PyErr_Occurred()) { + return NULL; + } + y = PyFloat_Unpack8(p + 8, 0); + if (y == -1.0 && PyErr_Occurred()) { + return NULL; + } + return PyComplex_FromDoubles(x, y); +} +#endif + static PyObject * bu_bool(_structmodulestate *state, const char *p, const formatdef *f) { @@ -1068,6 +1189,38 @@ bp_double(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) return PyFloat_Pack8(x, p, 0); } +#ifdef Py_HAVE_C_COMPLEX +static int +bp_float_complex(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) +{ + Py_complex x = PyComplex_AsCComplex(v); + if (x.real == -1 && PyErr_Occurred()) { + PyErr_SetString(state->StructError, + "required argument is not a complex"); + return -1; + } + if (PyFloat_Pack4(x.real, p, 0)) { + return -1; + } + return PyFloat_Pack4(x.imag, p + 4, 0); +} + +static int +bp_double_complex(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) +{ + Py_complex x = PyComplex_AsCComplex(v); + if (x.real == -1 && PyErr_Occurred()) { + PyErr_SetString(state->StructError, + "required argument is not a complex"); + return -1; + } + if (PyFloat_Pack8(x.real, p, 0)) { + return -1; + } + return PyFloat_Pack8(x.imag, p + 8, 0); +} +#endif + static int bp_bool(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { @@ -1098,6 +1251,13 @@ static formatdef bigendian_table[] = { {'e', 2, 0, bu_halffloat, bp_halffloat}, {'f', 4, 0, bu_float, bp_float}, {'d', 8, 0, bu_double, bp_double}, +#ifdef Py_HAVE_C_COMPLEX + {'E', 8, 0, bu_float_complex, bp_float_complex}, + {'C', 16, 0, bu_double_complex, bp_double_complex}, +#else + {'E', 1, 0, nu_complex_stub, np_complex_stub}, + {'C', 1, 0, nu_complex_stub, np_complex_stub}, +#endif {0} }; @@ -1197,6 +1357,38 @@ lu_double(_structmodulestate *state, const char *p, const formatdef *f) return unpack_double(p, 1); } +#ifdef Py_HAVE_C_COMPLEX +static PyObject * +lu_float_complex(_structmodulestate *state, const char *p, const formatdef *f) +{ + double x = PyFloat_Unpack4(p, 1); + if (x == -1.0 && PyErr_Occurred()) { + return NULL; + } + double y = PyFloat_Unpack4(p + 4, 1); + if (y == -1.0 && PyErr_Occurred()) { + return NULL; + } + return PyComplex_FromDoubles(x, y); +} + +static PyObject * +lu_double_complex(_structmodulestate *state, const char *p, const formatdef *f) +{ + double x, y; + + x = PyFloat_Unpack8(p, 1); + if (x == -1.0 && PyErr_Occurred()) { + return NULL; + } + y = PyFloat_Unpack8(p + 8, 1); + if (y == -1.0 && PyErr_Occurred()) { + return NULL; + } + return PyComplex_FromDoubles(x, y); +} +#endif + static int lp_int(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { @@ -1330,6 +1522,39 @@ lp_double(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) return PyFloat_Pack8(x, p, 1); } +#ifdef Py_HAVE_C_COMPLEX +static int +lp_float_complex(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) +{ + Py_complex x = PyComplex_AsCComplex(v); + if (x.real == -1 && PyErr_Occurred()) { + PyErr_SetString(state->StructError, + "required argument is not a complex"); + return -1; + } + if (PyFloat_Pack4(x.real, p, 1)) { + return -1; + } + return PyFloat_Pack4(x.imag, p + 4, 1); + +} + +static int +lp_double_complex(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) +{ + Py_complex x = PyComplex_AsCComplex(v); + if (x.real == -1 && PyErr_Occurred()) { + PyErr_SetString(state->StructError, + "required argument is not a complex"); + return -1; + } + if (PyFloat_Pack8(x.real, p, 1)) { + return -1; + } + return PyFloat_Pack8(x.imag, p + 8, 1); +} +#endif + static formatdef lilendian_table[] = { {'x', 1, 0, NULL}, {'b', 1, 0, nu_byte, np_byte}, @@ -1350,6 +1575,13 @@ static formatdef lilendian_table[] = { {'e', 2, 0, lu_halffloat, lp_halffloat}, {'f', 4, 0, lu_float, lp_float}, {'d', 8, 0, lu_double, lp_double}, +#ifdef Py_HAVE_C_COMPLEX + {'E', 8, 0, lu_float_complex, lp_float_complex}, + {'C', 16, 0, lu_double_complex, lp_double_complex}, +#else + {'E', 1, 0, nu_complex_stub, np_complex_stub}, + {'C', 1, 0, nu_complex_stub, np_complex_stub}, +#endif {0} }; @@ -1373,7 +1605,7 @@ whichtable(const char **pfmt) } default: --*pfmt; /* Back out of pointer increment */ - /* Fall through */ + _Py_FALLTHROUGH; case '@': return native_table; } @@ -1475,7 +1707,7 @@ prepare_s(PyStructObject *self) return -1; switch (c) { - case 's': /* fall through */ + case 's': _Py_FALLTHROUGH; case 'p': len++; ncodes++; break; case 'x': break; default: len += num; if (num) ncodes++; break; @@ -1669,9 +1901,16 @@ s_unpack_internal(PyStructObject *soself, const char *startfrom, if (e->format == 's') { v = PyBytes_FromStringAndSize(res, code->size); } else if (e->format == 'p') { - Py_ssize_t n = *(unsigned char*)res; - if (n >= code->size) - n = code->size - 1; + Py_ssize_t n; + if (code->size == 0) { + n = 0; + } + else { + n = *(unsigned char*)res; + if (n >= code->size) { + n = code->size - 1; + } + } v = PyBytes_FromStringAndSize(res + 1, n); } else { v = e->unpack(state, res, e); @@ -1982,8 +2221,12 @@ s_pack_internal(PyStructObject *soself, PyObject *const *args, int offset, n = PyByteArray_GET_SIZE(v); p = PyByteArray_AS_STRING(v); } - if (n > (code->size - 1)) + if (code->size == 0) { + n = 0; + } + else if (n > (code->size - 1)) { n = code->size - 1; + } if (n > 0) memcpy(res + 1, p, n); if (n > 255) @@ -2593,6 +2836,7 @@ _structmodule_exec(PyObject *m) static PyModuleDef_Slot _structmodule_slots[] = { {Py_mod_exec, _structmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_suggestions.c b/Modules/_suggestions.c index 30b524d70c1211..80c7179c4c251c 100644 --- a/Modules/_suggestions.c +++ b/Modules/_suggestions.c @@ -49,15 +49,21 @@ static PyMethodDef module_methods[] = { {NULL, NULL, 0, NULL} // Sentinel }; +static PyModuleDef_Slot module_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static struct PyModuleDef suggestions_module = { PyModuleDef_HEAD_INIT, "_suggestions", NULL, - -1, - module_methods + 0, + module_methods, + module_slots, }; PyMODINIT_FUNC PyInit__suggestions(void) { - return PyModule_Create(&suggestions_module); + return PyModuleDef_Init(&suggestions_module); } - diff --git a/Modules/_sysconfig.c b/Modules/_sysconfig.c index c76b9e6b3ebafa..c50c5cfabc2f1f 100644 --- a/Modules/_sysconfig.c +++ b/Modules/_sysconfig.c @@ -80,6 +80,7 @@ static struct PyMethodDef sysconfig_methods[] = { static PyModuleDef_Slot sysconfig_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index 5084bcadb10f85..b47bc8a830cb5e 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -24,11 +24,13 @@ static PyTypeObject NDArray_Type; #define NDArray_Check(v) Py_IS_TYPE(v, &NDArray_Type) #define CHECK_LIST_OR_TUPLE(v) \ - if (!PyList_Check(v) && !PyTuple_Check(v)) { \ - PyErr_SetString(PyExc_TypeError, \ - #v " must be a list or a tuple"); \ - return NULL; \ - } \ + do { \ + if (!PyList_Check(v) && !PyTuple_Check(v)) { \ + PyErr_SetString(PyExc_TypeError, \ + #v " must be a list or a tuple"); \ + return NULL; \ + } \ + } while (0) #define PyMem_XFree(v) \ do { if (v) PyMem_Free(v); } while (0) @@ -1180,7 +1182,7 @@ init_ndbuf(PyObject *items, PyObject *shape, PyObject *strides, Py_ssize_t itemsize; /* ndim = len(shape) */ - CHECK_LIST_OR_TUPLE(shape) + CHECK_LIST_OR_TUPLE(shape); ndim = PySequence_Fast_GET_SIZE(shape); if (ndim > ND_MAX_NDIM) { PyErr_Format(PyExc_ValueError, @@ -1190,7 +1192,7 @@ init_ndbuf(PyObject *items, PyObject *shape, PyObject *strides, /* len(strides) = len(shape) */ if (strides) { - CHECK_LIST_OR_TUPLE(strides) + CHECK_LIST_OR_TUPLE(strides); if (PySequence_Fast_GET_SIZE(strides) == 0) strides = NULL; else if (flags & ND_FORTRAN) { @@ -1217,12 +1219,12 @@ init_ndbuf(PyObject *items, PyObject *shape, PyObject *strides, /* convert scalar to list */ if (ndim == 0) { - items = Py_BuildValue("(O)", items); + items = PyTuple_Pack(1, items); if (items == NULL) return NULL; } else { - CHECK_LIST_OR_TUPLE(items) + CHECK_LIST_OR_TUPLE(items); Py_INCREF(items); } @@ -2820,6 +2822,9 @@ static int _testbuffer_exec(PyObject *mod) { Py_SET_TYPE(&NDArray_Type, &PyType_Type); + if (PyType_Ready(&NDArray_Type)) { + return -1; + } if (PyModule_AddType(mod, &NDArray_Type) < 0) { return -1; } @@ -2898,6 +2903,9 @@ PyInit__testbuffer(void) if (mod == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); +#endif if (_testbuffer_exec(mod) < 0) { Py_DECREF(mod); return NULL; diff --git a/Modules/_testcapi/abstract.c b/Modules/_testcapi/abstract.c index a8ba009eb6a54b..8c2c7137cdce40 100644 --- a/Modules/_testcapi/abstract.c +++ b/Modules/_testcapi/abstract.c @@ -2,59 +2,6 @@ #include "util.h" -static PyObject * -object_repr(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyObject_Repr(arg); -} - -static PyObject * -object_ascii(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyObject_ASCII(arg); -} - -static PyObject * -object_str(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyObject_Str(arg); -} - -static PyObject * -object_bytes(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyObject_Bytes(arg); -} - -static PyObject * -object_getattr(PyObject *self, PyObject *args) -{ - PyObject *obj, *attr_name; - if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(attr_name); - return PyObject_GetAttr(obj, attr_name); -} - -static PyObject * -object_getattrstring(PyObject *self, PyObject *args) -{ - PyObject *obj; - const char *attr_name; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { - return NULL; - } - NULLABLE(obj); - return PyObject_GetAttrString(obj, attr_name); -} - static PyObject * object_getoptionalattr(PyObject *self, PyObject *args) { @@ -106,31 +53,6 @@ object_getoptionalattrstring(PyObject *self, PyObject *args) } } -static PyObject * -object_hasattr(PyObject *self, PyObject *args) -{ - PyObject *obj, *attr_name; - if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(attr_name); - return PyLong_FromLong(PyObject_HasAttr(obj, attr_name)); -} - -static PyObject * -object_hasattrstring(PyObject *self, PyObject *args) -{ - PyObject *obj; - const char *attr_name; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { - return NULL; - } - NULLABLE(obj); - return PyLong_FromLong(PyObject_HasAttrString(obj, attr_name)); -} - static PyObject * object_hasattrwitherror(PyObject *self, PyObject *args) { @@ -157,121 +79,17 @@ object_hasattrstringwitherror(PyObject *self, PyObject *args) } static PyObject * -object_setattr(PyObject *self, PyObject *args) -{ - PyObject *obj, *attr_name, *value; - if (!PyArg_ParseTuple(args, "OOO", &obj, &attr_name, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(attr_name); - NULLABLE(value); - RETURN_INT(PyObject_SetAttr(obj, attr_name, value)); -} - -static PyObject * -object_setattrstring(PyObject *self, PyObject *args) -{ - PyObject *obj, *value; - const char *attr_name; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#O", &obj, &attr_name, &size, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(value); - RETURN_INT(PyObject_SetAttrString(obj, attr_name, value)); -} - -static PyObject * -object_delattr(PyObject *self, PyObject *args) -{ - PyObject *obj, *attr_name; -if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(attr_name); - RETURN_INT(PyObject_DelAttr(obj, attr_name)); -} - -static PyObject * -object_delattrstring(PyObject *self, PyObject *args) +mapping_getoptionalitemstring(PyObject *self, PyObject *args) { - PyObject *obj; + PyObject *obj, *value = UNINITIALIZED_PTR; const char *attr_name; Py_ssize_t size; if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { return NULL; } NULLABLE(obj); - RETURN_INT(PyObject_DelAttrString(obj, attr_name)); -} -static PyObject * -number_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyBool_FromLong(PyNumber_Check(obj)); -} - -static PyObject * -mapping_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyMapping_Check(obj)); -} - -static PyObject * -mapping_size(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PyMapping_Size(obj)); -} - -static PyObject * -mapping_length(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PyMapping_Length(obj)); -} - -static PyObject * -object_getitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - return PyObject_GetItem(mapping, key); -} - -static PyObject * -mapping_getitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - return PyMapping_GetItemString(mapping, key); -} - -static PyObject * -mapping_getoptionalitem(PyObject *self, PyObject *args) -{ - PyObject *obj, *attr_name, *value = UNINITIALIZED_PTR; - if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(attr_name); - - switch (PyMapping_GetOptionalItem(obj, attr_name, &value)) { + switch (PyMapping_GetOptionalItemString(obj, attr_name, &value)) { case -1: assert(value == NULL); return NULL; @@ -281,23 +99,22 @@ mapping_getoptionalitem(PyObject *self, PyObject *args) case 1: return value; default: - Py_FatalError("PyMapping_GetOptionalItem() returned invalid code"); + Py_FatalError("PyMapping_GetOptionalItemString() returned invalid code"); Py_UNREACHABLE(); } } static PyObject * -mapping_getoptionalitemstring(PyObject *self, PyObject *args) +mapping_getoptionalitem(PyObject *self, PyObject *args) { - PyObject *obj, *value = UNINITIALIZED_PTR; - const char *attr_name; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + PyObject *obj, *attr_name, *value = UNINITIALIZED_PTR; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { return NULL; } NULLABLE(obj); + NULLABLE(attr_name); - switch (PyMapping_GetOptionalItemString(obj, attr_name, &value)) { + switch (PyMapping_GetOptionalItem(obj, attr_name, &value)) { case -1: assert(value == NULL); return NULL; @@ -307,400 +124,49 @@ mapping_getoptionalitemstring(PyObject *self, PyObject *args) case 1: return value; default: - Py_FatalError("PyMapping_GetOptionalItemString() returned invalid code"); + Py_FatalError("PyMapping_GetOptionalItem() returned invalid code"); Py_UNREACHABLE(); } } static PyObject * -mapping_haskey(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - return PyLong_FromLong(PyMapping_HasKey(mapping, key)); -} - -static PyObject * -mapping_haskeystring(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - return PyLong_FromLong(PyMapping_HasKeyString(mapping, key)); -} - -static PyObject * -mapping_haskeywitherror(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - RETURN_INT(PyMapping_HasKeyWithError(mapping, key)); -} - -static PyObject * -mapping_haskeystringwitherror(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - RETURN_INT(PyMapping_HasKeyStringWithError(mapping, key)); -} - -static PyObject * -object_setitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key, *value; - if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - NULLABLE(value); - RETURN_INT(PyObject_SetItem(mapping, key, value)); -} - -static PyObject * -mapping_setitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping, *value; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(value); - RETURN_INT(PyMapping_SetItemString(mapping, key, value)); -} - -static PyObject * -object_delitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - RETURN_INT(PyObject_DelItem(mapping, key)); -} - -static PyObject * -mapping_delitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - RETURN_INT(PyMapping_DelItem(mapping, key)); -} - -static PyObject * -mapping_delitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - RETURN_INT(PyMapping_DelItemString(mapping, key)); -} - -static PyObject * -mapping_keys(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyMapping_Keys(obj); -} - -static PyObject * -mapping_values(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyMapping_Values(obj); -} - -static PyObject * -mapping_items(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyMapping_Items(obj); -} - - -static PyObject * -sequence_check(PyObject* self, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PySequence_Check(obj)); -} - -static PyObject * -sequence_size(PyObject* self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PySequence_Size(obj)); -} - -static PyObject * -sequence_length(PyObject* self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PySequence_Length(obj)); -} - -static PyObject * -sequence_concat(PyObject *self, PyObject *args) -{ - PyObject *seq1, *seq2; - if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { - return NULL; - } - NULLABLE(seq1); - NULLABLE(seq2); - - return PySequence_Concat(seq1, seq2); -} - -static PyObject * -sequence_repeat(PyObject *self, PyObject *args) +pyiter_next(PyObject *self, PyObject *iter) { - PyObject *seq; - Py_ssize_t count; - if (!PyArg_ParseTuple(args, "On", &seq, &count)) { - return NULL; + PyObject *item = PyIter_Next(iter); + if (item == NULL && !PyErr_Occurred()) { + Py_RETURN_NONE; } - NULLABLE(seq); - - return PySequence_Repeat(seq, count); + return item; } static PyObject * -sequence_inplaceconcat(PyObject *self, PyObject *args) +pyiter_nextitem(PyObject *self, PyObject *iter) { - PyObject *seq1, *seq2; - if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { + PyObject *item; + int rc = PyIter_NextItem(iter, &item); + if (rc < 0) { + assert(PyErr_Occurred()); + assert(item == NULL); return NULL; } - NULLABLE(seq1); - NULLABLE(seq2); - - return PySequence_InPlaceConcat(seq1, seq2); -} - -static PyObject * -sequence_inplacerepeat(PyObject *self, PyObject *args) -{ - PyObject *seq; - Py_ssize_t count; - if (!PyArg_ParseTuple(args, "On", &seq, &count)) { - return NULL; + assert(!PyErr_Occurred()); + if (item == NULL) { + Py_RETURN_NONE; } - NULLABLE(seq); - - return PySequence_InPlaceRepeat(seq, count); -} - -static PyObject * -sequence_getitem(PyObject *self, PyObject *args) -{ - PyObject *seq; - Py_ssize_t i; - if (!PyArg_ParseTuple(args, "On", &seq, &i)) { - return NULL; - } - NULLABLE(seq); - - return PySequence_GetItem(seq, i); -} - -static PyObject * -sequence_setitem(PyObject *self, PyObject *args) -{ - Py_ssize_t i; - PyObject *seq, *val; - if (!PyArg_ParseTuple(args, "OnO", &seq, &i, &val)) { - return NULL; - } - NULLABLE(seq); - NULLABLE(val); - - RETURN_INT(PySequence_SetItem(seq, i, val)); -} - - -static PyObject * -sequence_delitem(PyObject *self, PyObject *args) -{ - Py_ssize_t i; - PyObject *seq; - if (!PyArg_ParseTuple(args, "On", &seq, &i)) { - return NULL; - } - NULLABLE(seq); - - RETURN_INT(PySequence_DelItem(seq, i)); -} - -static PyObject * -sequence_setslice(PyObject* self, PyObject *args) -{ - PyObject *sequence, *obj; - Py_ssize_t i1, i2; - if (!PyArg_ParseTuple(args, "OnnO", &sequence, &i1, &i2, &obj)) { - return NULL; - } - NULLABLE(sequence); - NULLABLE(obj); - - RETURN_INT(PySequence_SetSlice(sequence, i1, i2, obj)); -} - -static PyObject * -sequence_delslice(PyObject *self, PyObject *args) -{ - PyObject *sequence; - Py_ssize_t i1, i2; - if (!PyArg_ParseTuple(args, "Onn", &sequence, &i1, &i2)) { - return NULL; - } - NULLABLE(sequence); - - RETURN_INT(PySequence_DelSlice(sequence, i1, i2)); -} - -static PyObject * -sequence_count(PyObject *self, PyObject *args) -{ - PyObject *seq, *value; - if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { - return NULL; - } - NULLABLE(seq); - NULLABLE(value); - - RETURN_SIZE(PySequence_Count(seq, value)); -} - -static PyObject * -sequence_contains(PyObject *self, PyObject *args) -{ - PyObject *seq, *value; - if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { - return NULL; - } - NULLABLE(seq); - NULLABLE(value); - - RETURN_INT(PySequence_Contains(seq, value)); -} - -static PyObject * -sequence_index(PyObject *self, PyObject *args) -{ - PyObject *seq, *value; - if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { - return NULL; - } - NULLABLE(seq); - NULLABLE(value); - - RETURN_SIZE(PySequence_Index(seq, value)); -} - -static PyObject * -sequence_list(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PySequence_List(obj); -} - -static PyObject * -sequence_tuple(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PySequence_Tuple(obj); + return item; } static PyMethodDef test_methods[] = { - {"object_repr", object_repr, METH_O}, - {"object_ascii", object_ascii, METH_O}, - {"object_str", object_str, METH_O}, - {"object_bytes", object_bytes, METH_O}, - - {"object_getattr", object_getattr, METH_VARARGS}, - {"object_getattrstring", object_getattrstring, METH_VARARGS}, {"object_getoptionalattr", object_getoptionalattr, METH_VARARGS}, {"object_getoptionalattrstring", object_getoptionalattrstring, METH_VARARGS}, - {"object_hasattr", object_hasattr, METH_VARARGS}, - {"object_hasattrstring", object_hasattrstring, METH_VARARGS}, {"object_hasattrwitherror", object_hasattrwitherror, METH_VARARGS}, {"object_hasattrstringwitherror", object_hasattrstringwitherror, METH_VARARGS}, - {"object_setattr", object_setattr, METH_VARARGS}, - {"object_setattrstring", object_setattrstring, METH_VARARGS}, - {"object_delattr", object_delattr, METH_VARARGS}, - {"object_delattrstring", object_delattrstring, METH_VARARGS}, - - {"number_check", number_check, METH_O}, - {"mapping_check", mapping_check, METH_O}, - {"mapping_size", mapping_size, METH_O}, - {"mapping_length", mapping_length, METH_O}, - {"object_getitem", object_getitem, METH_VARARGS}, - {"mapping_getitemstring", mapping_getitemstring, METH_VARARGS}, {"mapping_getoptionalitem", mapping_getoptionalitem, METH_VARARGS}, {"mapping_getoptionalitemstring", mapping_getoptionalitemstring, METH_VARARGS}, - {"mapping_haskey", mapping_haskey, METH_VARARGS}, - {"mapping_haskeystring", mapping_haskeystring, METH_VARARGS}, - {"mapping_haskeywitherror", mapping_haskeywitherror, METH_VARARGS}, - {"mapping_haskeystringwitherror", mapping_haskeystringwitherror, METH_VARARGS}, - {"object_setitem", object_setitem, METH_VARARGS}, - {"mapping_setitemstring", mapping_setitemstring, METH_VARARGS}, - {"object_delitem", object_delitem, METH_VARARGS}, - {"mapping_delitem", mapping_delitem, METH_VARARGS}, - {"mapping_delitemstring", mapping_delitemstring, METH_VARARGS}, - {"mapping_keys", mapping_keys, METH_O}, - {"mapping_values", mapping_values, METH_O}, - {"mapping_items", mapping_items, METH_O}, - - {"sequence_check", sequence_check, METH_O}, - {"sequence_size", sequence_size, METH_O}, - {"sequence_length", sequence_length, METH_O}, - {"sequence_concat", sequence_concat, METH_VARARGS}, - {"sequence_repeat", sequence_repeat, METH_VARARGS}, - {"sequence_inplaceconcat", sequence_inplaceconcat, METH_VARARGS}, - {"sequence_inplacerepeat", sequence_inplacerepeat, METH_VARARGS}, - {"sequence_getitem", sequence_getitem, METH_VARARGS}, - {"sequence_setitem", sequence_setitem, METH_VARARGS}, - {"sequence_delitem", sequence_delitem, METH_VARARGS}, - {"sequence_setslice", sequence_setslice, METH_VARARGS}, - {"sequence_delslice", sequence_delslice, METH_VARARGS}, - {"sequence_count", sequence_count, METH_VARARGS}, - {"sequence_contains", sequence_contains, METH_VARARGS}, - {"sequence_index", sequence_index, METH_VARARGS}, - {"sequence_list", sequence_list, METH_O}, - {"sequence_tuple", sequence_tuple, METH_O}, + {"PyIter_Next", pyiter_next, METH_O}, + {"PyIter_NextItem", pyiter_nextitem, METH_O}, {NULL}, }; diff --git a/Modules/_testcapi/bytes.c b/Modules/_testcapi/bytes.c index da10503f6f6856..33903de14ba68d 100644 --- a/Modules/_testcapi/bytes.c +++ b/Modules/_testcapi/bytes.c @@ -2,245 +2,58 @@ #include "util.h" -/* Test PyBytes_Check() */ +/* Test _PyBytes_Resize() */ static PyObject * -bytes_check(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyBytes_Check(obj)); -} - -/* Test PyBytes_CheckExact() */ -static PyObject * -bytes_checkexact(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyBytes_CheckExact(obj)); -} - -/* Test PyBytes_FromStringAndSize() */ -static PyObject * -bytes_fromstringandsize(PyObject *Py_UNUSED(module), PyObject *args) -{ - const char *s; - Py_ssize_t bsize; - Py_ssize_t size = -100; - - if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { - return NULL; - } - - if (size == -100) { - size = bsize; - } - return PyBytes_FromStringAndSize(s, size); -} - -/* Test PyBytes_FromString() */ -static PyObject * -bytes_fromstring(PyObject *Py_UNUSED(module), PyObject *arg) -{ - const char *s; - Py_ssize_t size; - - if (!PyArg_Parse(arg, "z#", &s, &size)) { - return NULL; - } - return PyBytes_FromString(s); -} - -/* Test PyBytes_FromObject() */ -static PyObject * -bytes_fromobject(PyObject *Py_UNUSED(module), PyObject *arg) -{ - NULLABLE(arg); - return PyBytes_FromObject(arg); -} - -/* Test PyBytes_Size() */ -static PyObject * -bytes_size(PyObject *Py_UNUSED(module), PyObject *arg) -{ - NULLABLE(arg); - RETURN_SIZE(PyBytes_Size(arg)); -} - -/* Test PyUnicode_AsString() */ -static PyObject * -bytes_asstring(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj; - Py_ssize_t buflen; - const char *s; - - if (!PyArg_ParseTuple(args, "On", &obj, &buflen)) - return NULL; - - NULLABLE(obj); - s = PyBytes_AsString(obj); - if (s == NULL) - return NULL; - - return PyBytes_FromStringAndSize(s, buflen); -} - -/* Test PyBytes_AsStringAndSize() */ -static PyObject * -bytes_asstringandsize(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj; - Py_ssize_t buflen; - char *s = UNINITIALIZED_PTR; - Py_ssize_t size = UNINITIALIZED_SIZE; - - if (!PyArg_ParseTuple(args, "On", &obj, &buflen)) - return NULL; - - NULLABLE(obj); - if (PyBytes_AsStringAndSize(obj, &s, &size) < 0) { - return NULL; - } - - if (s == NULL) { - return Py_BuildValue("(On)", Py_None, size); - } - else { - return Py_BuildValue("(y#n)", s, buflen, size); - } -} - -static PyObject * -bytes_asstringandsize_null(PyObject *Py_UNUSED(module), PyObject *args) +bytes_resize(PyObject *Py_UNUSED(module), PyObject *args) { PyObject *obj; - Py_ssize_t buflen; - char *s = UNINITIALIZED_PTR; + Py_ssize_t newsize; + int new; - if (!PyArg_ParseTuple(args, "On", &obj, &buflen)) - return NULL; - - NULLABLE(obj); - if (PyBytes_AsStringAndSize(obj, &s, NULL) < 0) { - return NULL; - } - - if (s == NULL) { - Py_RETURN_NONE; - } - else { - return PyBytes_FromStringAndSize(s, buflen); - } -} - -/* Test PyBytes_Repr() */ -static PyObject * -bytes_repr(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj; - int smartquotes; - if (!PyArg_ParseTuple(args, "Oi", &obj, &smartquotes)) + if (!PyArg_ParseTuple(args, "Onp", &obj, &newsize, &new)) return NULL; NULLABLE(obj); - return PyBytes_Repr(obj, smartquotes); -} - -/* Test PyBytes_Concat() */ -static PyObject * -bytes_concat(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *left, *right; - int new = 0; - - if (!PyArg_ParseTuple(args, "OO|p", &left, &right, &new)) - return NULL; - - NULLABLE(left); - NULLABLE(right); if (new) { - assert(left != NULL); - assert(PyBytes_CheckExact(left)); - left = PyBytes_FromStringAndSize(PyBytes_AS_STRING(left), - PyBytes_GET_SIZE(left)); - if (left == NULL) { + assert(obj != NULL); + assert(PyBytes_CheckExact(obj)); + PyObject *newobj = PyBytes_FromStringAndSize(NULL, PyBytes_Size(obj)); + if (newobj == NULL) { return NULL; } + memcpy(PyBytes_AsString(newobj), PyBytes_AsString(obj), PyBytes_Size(obj)); + obj = newobj; } else { - Py_XINCREF(left); + Py_XINCREF(obj); } - PyBytes_Concat(&left, right); - if (left == NULL && !PyErr_Occurred()) { - Py_RETURN_NONE; - } - return left; -} - -/* Test PyBytes_ConcatAndDel() */ -static PyObject * -bytes_concatanddel(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *left, *right; - int new = 0; - - if (!PyArg_ParseTuple(args, "OO|p", &left, &right, &new)) - return NULL; - - NULLABLE(left); - NULLABLE(right); - if (new) { - assert(left != NULL); - assert(PyBytes_CheckExact(left)); - left = PyBytes_FromStringAndSize(PyBytes_AS_STRING(left), - PyBytes_GET_SIZE(left)); - if (left == NULL) { - return NULL; - } + if (_PyBytes_Resize(&obj, newsize) < 0) { + assert(obj == NULL); } else { - Py_XINCREF(left); - } - Py_XINCREF(right); - PyBytes_ConcatAndDel(&left, right); - if (left == NULL && !PyErr_Occurred()) { - Py_RETURN_NONE; + assert(obj != NULL); } - return left; + return obj; } -/* Test PyBytes_DecodeEscape() */ + +/* Test PyBytes_Join() */ static PyObject * -bytes_decodeescape(PyObject *Py_UNUSED(module), PyObject *args) +bytes_join(PyObject *Py_UNUSED(module), PyObject *args) { - const char *s; - Py_ssize_t bsize; - Py_ssize_t size = -100; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "z#|zn", &s, &bsize, &errors, &size)) + PyObject *sep, *iterable; + if (!PyArg_ParseTuple(args, "OO", &sep, &iterable)) { return NULL; - - if (size == -100) { - size = bsize; } - return PyBytes_DecodeEscape(s, size, errors, 0, NULL); + NULLABLE(sep); + NULLABLE(iterable); + return PyBytes_Join(sep, iterable); } static PyMethodDef test_methods[] = { - {"bytes_check", bytes_check, METH_O}, - {"bytes_checkexact", bytes_checkexact, METH_O}, - {"bytes_fromstringandsize", bytes_fromstringandsize, METH_VARARGS}, - {"bytes_fromstring", bytes_fromstring, METH_O}, - {"bytes_fromobject", bytes_fromobject, METH_O}, - {"bytes_size", bytes_size, METH_O}, - {"bytes_asstring", bytes_asstring, METH_VARARGS}, - {"bytes_asstringandsize", bytes_asstringandsize, METH_VARARGS}, - {"bytes_asstringandsize_null", bytes_asstringandsize_null, METH_VARARGS}, - {"bytes_repr", bytes_repr, METH_VARARGS}, - {"bytes_concat", bytes_concat, METH_VARARGS}, - {"bytes_concatanddel", bytes_concatanddel, METH_VARARGS}, - {"bytes_decodeescape", bytes_decodeescape, METH_VARARGS}, + {"bytes_resize", bytes_resize, METH_VARARGS}, + {"bytes_join", bytes_join, METH_VARARGS}, {NULL}, }; diff --git a/Modules/_testcapi/clinic/exceptions.c.h b/Modules/_testcapi/clinic/exceptions.c.h index a797444c1a72b9..7c18bac0f686e2 100644 --- a/Modules/_testcapi/clinic/exceptions.c.h +++ b/Modules/_testcapi/clinic/exceptions.c.h @@ -104,7 +104,8 @@ _testcapi_make_exception_with_doc(PyObject *module, PyObject *const *args, Py_ss PyObject *base = NULL; PyObject *dict = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -456,4 +457,4 @@ _testcapi_unstable_exc_prep_reraise_star(PyObject *module, PyObject *const *args exit: return return_value; } -/*[clinic end generated code: output=0b11ef105030a48e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d917e9ec082e69ee input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/clinic/file.c.h b/Modules/_testcapi/clinic/file.c.h new file mode 100644 index 00000000000000..fddbf48071bd3b --- /dev/null +++ b/Modules/_testcapi/clinic/file.c.h @@ -0,0 +1,37 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + +PyDoc_STRVAR(_testcapi_py_fopen__doc__, +"py_fopen($module, path, mode, /)\n" +"--\n" +"\n" +"Call Py_fopen(), fread(256) and Py_fclose(). Return read bytes."); + +#define _TESTCAPI_PY_FOPEN_METHODDEF \ + {"py_fopen", _PyCFunction_CAST(_testcapi_py_fopen), METH_FASTCALL, _testcapi_py_fopen__doc__}, + +static PyObject * +_testcapi_py_fopen_impl(PyObject *module, PyObject *path, const char *mode, + Py_ssize_t mode_length); + +static PyObject * +_testcapi_py_fopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *path; + const char *mode; + Py_ssize_t mode_length; + + if (!_PyArg_ParseStack(args, nargs, "Oz#:py_fopen", + &path, &mode, &mode_length)) { + goto exit; + } + return_value = _testcapi_py_fopen_impl(module, path, mode, mode_length); + +exit: + return return_value; +} +/*[clinic end generated code: output=c4dc92400306c3eb input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/clinic/long.c.h b/Modules/_testcapi/clinic/long.c.h index e2f7042be12c48..767c671abb8eae 100644 --- a/Modules/_testcapi/clinic/long.c.h +++ b/Modules/_testcapi/clinic/long.c.h @@ -2,137 +2,6 @@ preserve [clinic start generated code]*/ -PyDoc_STRVAR(_testcapi_test_long_api__doc__, -"test_long_api($module, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_TEST_LONG_API_METHODDEF \ - {"test_long_api", (PyCFunction)_testcapi_test_long_api, METH_NOARGS, _testcapi_test_long_api__doc__}, - -static PyObject * -_testcapi_test_long_api_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_api(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_api_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_longlong_api__doc__, -"test_longlong_api($module, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_TEST_LONGLONG_API_METHODDEF \ - {"test_longlong_api", (PyCFunction)_testcapi_test_longlong_api, METH_NOARGS, _testcapi_test_longlong_api__doc__}, - -static PyObject * -_testcapi_test_longlong_api_impl(PyObject *module); - -static PyObject * -_testcapi_test_longlong_api(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_longlong_api_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_long_and_overflow__doc__, -"test_long_and_overflow($module, /)\n" -"--\n" -"\n" -"Test the PyLong_AsLongAndOverflow API.\n" -"\n" -"General conversion to PY_LONG is tested by test_long_api_inner.\n" -"This test will concentrate on proper handling of overflow."); - -#define _TESTCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF \ - {"test_long_and_overflow", (PyCFunction)_testcapi_test_long_and_overflow, METH_NOARGS, _testcapi_test_long_and_overflow__doc__}, - -static PyObject * -_testcapi_test_long_and_overflow_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_and_overflow(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_and_overflow_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_long_long_and_overflow__doc__, -"test_long_long_and_overflow($module, /)\n" -"--\n" -"\n" -"Test the PyLong_AsLongLongAndOverflow API.\n" -"\n" -"General conversion to long long is tested by test_long_api_inner.\n" -"This test will concentrate on proper handling of overflow."); - -#define _TESTCAPI_TEST_LONG_LONG_AND_OVERFLOW_METHODDEF \ - {"test_long_long_and_overflow", (PyCFunction)_testcapi_test_long_long_and_overflow, METH_NOARGS, _testcapi_test_long_long_and_overflow__doc__}, - -static PyObject * -_testcapi_test_long_long_and_overflow_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_long_and_overflow(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_long_and_overflow_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_long_as_size_t__doc__, -"test_long_as_size_t($module, /)\n" -"--\n" -"\n" -"Test the PyLong_As{Size,Ssize}_t API.\n" -"\n" -"At present this just tests that non-integer arguments are handled correctly.\n" -"It should be extended to test overflow handling."); - -#define _TESTCAPI_TEST_LONG_AS_SIZE_T_METHODDEF \ - {"test_long_as_size_t", (PyCFunction)_testcapi_test_long_as_size_t, METH_NOARGS, _testcapi_test_long_as_size_t__doc__}, - -static PyObject * -_testcapi_test_long_as_size_t_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_as_size_t(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_as_size_t_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_long_as_unsigned_long_long_mask__doc__, -"test_long_as_unsigned_long_long_mask($module, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_TEST_LONG_AS_UNSIGNED_LONG_LONG_MASK_METHODDEF \ - {"test_long_as_unsigned_long_long_mask", (PyCFunction)_testcapi_test_long_as_unsigned_long_long_mask, METH_NOARGS, _testcapi_test_long_as_unsigned_long_long_mask__doc__}, - -static PyObject * -_testcapi_test_long_as_unsigned_long_long_mask_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_as_unsigned_long_long_mask(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_as_unsigned_long_long_mask_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_long_as_double__doc__, -"test_long_as_double($module, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_TEST_LONG_AS_DOUBLE_METHODDEF \ - {"test_long_as_double", (PyCFunction)_testcapi_test_long_as_double, METH_NOARGS, _testcapi_test_long_as_double__doc__}, - -static PyObject * -_testcapi_test_long_as_double_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_as_double(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_as_double_impl(module); -} - PyDoc_STRVAR(_testcapi_call_long_compact_api__doc__, "call_long_compact_api($module, arg, /)\n" "--\n" @@ -140,12 +9,4 @@ PyDoc_STRVAR(_testcapi_call_long_compact_api__doc__, #define _TESTCAPI_CALL_LONG_COMPACT_API_METHODDEF \ {"call_long_compact_api", (PyCFunction)_testcapi_call_long_compact_api, METH_O, _testcapi_call_long_compact_api__doc__}, - -PyDoc_STRVAR(_testcapi_PyLong_AsInt__doc__, -"PyLong_AsInt($module, arg, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_PYLONG_ASINT_METHODDEF \ - {"PyLong_AsInt", (PyCFunction)_testcapi_PyLong_AsInt, METH_O, _testcapi_PyLong_AsInt__doc__}, -/*[clinic end generated code: output=de762870526e241d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0ddcbc6ea06e5e21 input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/clinic/vectorcall_limited.c.h b/Modules/_testcapi/clinic/vectorcall_limited.c.h deleted file mode 100644 index a233aefec79ecd..00000000000000 --- a/Modules/_testcapi/clinic/vectorcall_limited.c.h +++ /dev/null @@ -1,20 +0,0 @@ -/*[clinic input] -preserve -[clinic start generated code]*/ - -PyDoc_STRVAR(_testcapi_call_vectorcall__doc__, -"call_vectorcall($module, callable, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_CALL_VECTORCALL_METHODDEF \ - {"call_vectorcall", (PyCFunction)_testcapi_call_vectorcall, METH_O, _testcapi_call_vectorcall__doc__}, - -PyDoc_STRVAR(_testcapi_call_vectorcall_method__doc__, -"call_vectorcall_method($module, callable, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF \ - {"call_vectorcall_method", (PyCFunction)_testcapi_call_vectorcall_method, METH_O, _testcapi_call_vectorcall_method__doc__}, -/*[clinic end generated code: output=e980906a39602528 input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/codec.c b/Modules/_testcapi/codec.c index d13f51e20331a1..e27e64e066c458 100644 --- a/Modules/_testcapi/codec.c +++ b/Modules/_testcapi/codec.c @@ -1,15 +1,219 @@ #include "parts.h" #include "util.h" +// === Codecs registration and un-registration ================================ + +static PyObject * +codec_register(PyObject *Py_UNUSED(module), PyObject *search_function) +{ + if (PyCodec_Register(search_function) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +codec_unregister(PyObject *Py_UNUSED(module), PyObject *search_function) +{ + if (PyCodec_Unregister(search_function) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +codec_known_encoding(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + if (!PyArg_ParseTuple(args, "z", &encoding)) { + return NULL; + } + return PyCodec_KnownEncoding(encoding) ? Py_True : Py_False; +} + +// === Codecs encoding and decoding interfaces ================================ + +static PyObject * +codec_encode(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *input; + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "O|zz", &input, &encoding, &errors)) { + return NULL; + } + return PyCodec_Encode(input, encoding, errors); +} + +static PyObject * +codec_decode(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *input; + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "O|zz", &input, &encoding, &errors)) { + return NULL; + } + return PyCodec_Decode(input, encoding, errors); +} + +static PyObject * +codec_encoder(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + if (!PyArg_ParseTuple(args, "z", &encoding)) { + return NULL; + } + return PyCodec_Encoder(encoding); +} + +static PyObject * +codec_decoder(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + if (!PyArg_ParseTuple(args, "z", &encoding)) { + return NULL; + } + return PyCodec_Decoder(encoding); +} + +static PyObject * +codec_incremental_encoder(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "zz", &encoding, &errors)) { + return NULL; + } + return PyCodec_IncrementalEncoder(encoding, errors); +} + +static PyObject * +codec_incremental_decoder(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "zz", &encoding, &errors)) { + return NULL; + } + return PyCodec_IncrementalDecoder(encoding, errors); +} + +static PyObject * +codec_stream_reader(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + PyObject *stream; + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "zOz", &encoding, &stream, &errors)) { + return NULL; + } + return PyCodec_StreamReader(encoding, stream, errors); +} + +static PyObject * +codec_stream_writer(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + PyObject *stream; + const char *errors; // can be NULL + if (!PyArg_ParseTuple(args, "zOz", &encoding, &stream, &errors)) { + return NULL; + } + return PyCodec_StreamWriter(encoding, stream, errors); +} + +// === Codecs errors handlers ================================================= + +static PyObject * +codec_register_error(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *encoding; // must not be NULL + PyObject *error; + if (!PyArg_ParseTuple(args, "sO", &encoding, &error)) { + return NULL; + } + if (PyCodec_RegisterError(encoding, error) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +codec_lookup_error(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *NULL_WOULD_RAISE(encoding); // NULL case will be tested + if (!PyArg_ParseTuple(args, "z", &encoding)) { + return NULL; + } + return PyCodec_LookupError(encoding); +} + +static PyObject * +codec_strict_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_StrictErrors(exc); +} + +static PyObject * +codec_ignore_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_IgnoreErrors(exc); +} + +static PyObject * +codec_replace_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_ReplaceErrors(exc); +} + +static PyObject * +codec_xmlcharrefreplace_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_XMLCharRefReplaceErrors(exc); +} + +static PyObject * +codec_backslashreplace_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_BackslashReplaceErrors(exc); +} static PyMethodDef test_methods[] = { - {NULL}, + /* codecs registration */ + {"codec_register", codec_register, METH_O}, + {"codec_unregister", codec_unregister, METH_O}, + {"codec_known_encoding", codec_known_encoding, METH_VARARGS}, + /* encoding and decoding interface */ + {"codec_encode", codec_encode, METH_VARARGS}, + {"codec_decode", codec_decode, METH_VARARGS}, + {"codec_encoder", codec_encoder, METH_VARARGS}, + {"codec_decoder", codec_decoder, METH_VARARGS}, + {"codec_incremental_encoder", codec_incremental_encoder, METH_VARARGS}, + {"codec_incremental_decoder", codec_incremental_decoder, METH_VARARGS}, + {"codec_stream_reader", codec_stream_reader, METH_VARARGS}, + {"codec_stream_writer", codec_stream_writer, METH_VARARGS}, + /* error handling */ + {"codec_register_error", codec_register_error, METH_VARARGS}, + {"codec_lookup_error", codec_lookup_error, METH_VARARGS}, + {"codec_strict_errors", codec_strict_errors, METH_O}, + {"codec_ignore_errors", codec_ignore_errors, METH_O}, + {"codec_replace_errors", codec_replace_errors, METH_O}, + {"codec_xmlcharrefreplace_errors", codec_xmlcharrefreplace_errors, METH_O}, + {"codec_backslashreplace_errors", codec_backslashreplace_errors, METH_O}, + // PyCodec_NameReplaceErrors() is tested in _testlimitedcapi/codec.c + {NULL, NULL, 0, NULL}, }; int _PyTestCapi_Init_Codec(PyObject *m) { - if (PyModule_AddFunctions(m, test_methods) < 0){ + if (PyModule_AddFunctions(m, test_methods) < 0) { return -1; } diff --git a/Modules/_testcapi/complex.c b/Modules/_testcapi/complex.c index 4a70217eb90d62..b726cd3236f179 100644 --- a/Modules/_testcapi/complex.c +++ b/Modules/_testcapi/complex.c @@ -2,20 +2,6 @@ #include "util.h" -static PyObject * -complex_check(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyComplex_Check(obj)); -} - -static PyObject * -complex_checkexact(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyComplex_CheckExact(obj)); -} - static PyObject * complex_fromccomplex(PyObject *Py_UNUSED(module), PyObject *obj) { @@ -28,48 +14,6 @@ complex_fromccomplex(PyObject *Py_UNUSED(module), PyObject *obj) return PyComplex_FromCComplex(complex); } -static PyObject * -complex_fromdoubles(PyObject *Py_UNUSED(module), PyObject *args) -{ - double real, imag; - - if (!PyArg_ParseTuple(args, "dd", &real, &imag)) { - return NULL; - } - - return PyComplex_FromDoubles(real, imag); -} - -static PyObject * -complex_realasdouble(PyObject *Py_UNUSED(module), PyObject *obj) -{ - double real; - - NULLABLE(obj); - real = PyComplex_RealAsDouble(obj); - - if (real == -1. && PyErr_Occurred()) { - return NULL; - } - - return PyFloat_FromDouble(real); -} - -static PyObject * -complex_imagasdouble(PyObject *Py_UNUSED(module), PyObject *obj) -{ - double imag; - - NULLABLE(obj); - imag = PyComplex_ImagAsDouble(obj); - - if (imag == -1. && PyErr_Occurred()) { - return NULL; - } - - return PyFloat_FromDouble(imag); -} - static PyObject * complex_asccomplex(PyObject *Py_UNUSED(module), PyObject *obj) { @@ -102,21 +46,59 @@ _py_c_neg(PyObject *Py_UNUSED(module), PyObject *num) static PyObject * \ _py_c_##suffix(PyObject *Py_UNUSED(module), PyObject *args) \ { \ - Py_complex num, exp, res; \ + Py_complex a, b, res; \ + \ + if (!PyArg_ParseTuple(args, "DD", &a, &b)) { \ + return NULL; \ + } \ + \ + errno = 0; \ + res = _Py_c_##suffix(a, b); \ + return Py_BuildValue("Di", &res, errno); \ + }; + +#define _PY_CR_FUNC2(suffix) \ + static PyObject * \ + _py_cr_##suffix(PyObject *Py_UNUSED(module), PyObject *args) \ + { \ + Py_complex a, res; \ + double b; \ + \ + if (!PyArg_ParseTuple(args, "Dd", &a, &b)) { \ + return NULL; \ + } \ + \ + errno = 0; \ + res = _Py_cr_##suffix(a, b); \ + return Py_BuildValue("Di", &res, errno); \ + }; + +#define _PY_RC_FUNC2(suffix) \ + static PyObject * \ + _py_rc_##suffix(PyObject *Py_UNUSED(module), PyObject *args) \ + { \ + Py_complex b, res; \ + double a; \ \ - if (!PyArg_ParseTuple(args, "DD", &num, &exp)) { \ + if (!PyArg_ParseTuple(args, "dD", &a, &b)) { \ return NULL; \ } \ \ errno = 0; \ - res = _Py_c_##suffix(num, exp); \ + res = _Py_rc_##suffix(a, b); \ return Py_BuildValue("Di", &res, errno); \ }; _PY_C_FUNC2(sum) +_PY_CR_FUNC2(sum) _PY_C_FUNC2(diff) +_PY_CR_FUNC2(diff) +_PY_RC_FUNC2(diff) _PY_C_FUNC2(prod) +_PY_CR_FUNC2(prod) _PY_C_FUNC2(quot) +_PY_CR_FUNC2(quot) +_PY_RC_FUNC2(quot) _PY_C_FUNC2(pow) static PyObject* @@ -139,18 +121,19 @@ _py_c_abs(PyObject *Py_UNUSED(module), PyObject* obj) static PyMethodDef test_methods[] = { - {"complex_check", complex_check, METH_O}, - {"complex_checkexact", complex_checkexact, METH_O}, {"complex_fromccomplex", complex_fromccomplex, METH_O}, - {"complex_fromdoubles", complex_fromdoubles, METH_VARARGS}, - {"complex_realasdouble", complex_realasdouble, METH_O}, - {"complex_imagasdouble", complex_imagasdouble, METH_O}, {"complex_asccomplex", complex_asccomplex, METH_O}, {"_py_c_sum", _py_c_sum, METH_VARARGS}, + {"_py_cr_sum", _py_cr_sum, METH_VARARGS}, {"_py_c_diff", _py_c_diff, METH_VARARGS}, + {"_py_cr_diff", _py_cr_diff, METH_VARARGS}, + {"_py_rc_diff", _py_rc_diff, METH_VARARGS}, {"_py_c_neg", _py_c_neg, METH_O}, {"_py_c_prod", _py_c_prod, METH_VARARGS}, + {"_py_cr_prod", _py_cr_prod, METH_VARARGS}, {"_py_c_quot", _py_c_quot, METH_VARARGS}, + {"_py_cr_quot", _py_cr_quot, METH_VARARGS}, + {"_py_rc_quot", _py_rc_quot, METH_VARARGS}, {"_py_c_pow", _py_c_pow, METH_VARARGS}, {"_py_c_abs", _py_c_abs, METH_O}, {NULL}, diff --git a/Modules/_testcapi/config.c b/Modules/_testcapi/config.c new file mode 100644 index 00000000000000..bb3b7e88953a53 --- /dev/null +++ b/Modules/_testcapi/config.c @@ -0,0 +1,68 @@ +#include "parts.h" + + +static PyObject * +_testcapi_config_get(PyObject *module, PyObject *name_obj) +{ + const char *name; + if (PyArg_Parse(name_obj, "s", &name) < 0) { + return NULL; + } + + return PyConfig_Get(name); +} + + +static PyObject * +_testcapi_config_getint(PyObject *module, PyObject *name_obj) +{ + const char *name; + if (PyArg_Parse(name_obj, "s", &name) < 0) { + return NULL; + } + + int value; + if (PyConfig_GetInt(name, &value) < 0) { + return NULL; + } + return PyLong_FromLong(value); +} + + +static PyObject * +_testcapi_config_names(PyObject *module, PyObject* Py_UNUSED(args)) +{ + return PyConfig_Names(); +} + + +static PyObject * +_testcapi_config_set(PyObject *module, PyObject *args) +{ + const char *name; + PyObject *value; + if (PyArg_ParseTuple(args, "sO", &name, &value) < 0) { + return NULL; + } + + int res = PyConfig_Set(name, value); + if (res < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + +static PyMethodDef test_methods[] = { + {"config_get", _testcapi_config_get, METH_O}, + {"config_getint", _testcapi_config_getint, METH_O}, + {"config_names", _testcapi_config_names, METH_NOARGS}, + {"config_set", _testcapi_config_set, METH_VARARGS}, + {NULL} +}; + +int +_PyTestCapi_Init_Config(PyObject *mod) +{ + return PyModule_AddFunctions(mod, test_methods); +} diff --git a/Modules/_testcapi/datetime.c b/Modules/_testcapi/datetime.c index b1796039f0d83a..b800f9b8eb3473 100644 --- a/Modules/_testcapi/datetime.c +++ b/Modules/_testcapi/datetime.c @@ -22,10 +22,17 @@ test_datetime_capi(PyObject *self, PyObject *args) test_run_counter++; PyDateTime_IMPORT; - if (PyDateTimeAPI) { - Py_RETURN_NONE; + if (PyDateTimeAPI == NULL) { + return NULL; } - return NULL; + // The following C API types need to outlive interpreters, since the + // borrowed references to them can be held by users without being updated. + assert(!PyType_HasFeature(PyDateTimeAPI->DateType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->TimeType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->DateTimeType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->DeltaType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->TZInfoType, Py_TPFLAGS_HEAPTYPE)); + Py_RETURN_NONE; } /* Functions exposing the C API type checking for testing */ @@ -122,7 +129,7 @@ static PyObject * get_timezones_offset_zero(PyObject *self, PyObject *args) { PyObject *offset = PyDelta_FromDSU(0, 0, 0); - PyObject *name = PyUnicode_FromString(""); + PyObject *name = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (offset == NULL || name == NULL) { Py_XDECREF(offset); Py_XDECREF(name); @@ -479,3 +486,38 @@ _PyTestCapi_Init_DateTime(PyObject *mod) } return 0; } + + +/* --------------------------------------------------------------------------- + * Test module for subinterpreters. + */ + +static int +_testcapi_datetime_exec(PyObject *mod) +{ + if (test_datetime_capi(NULL, NULL) == NULL) { + return -1; + } + return 0; +} + +static PyModuleDef_Slot _testcapi_datetime_slots[] = { + {Py_mod_exec, _testcapi_datetime_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + +static struct PyModuleDef _testcapi_datetime_module = { + PyModuleDef_HEAD_INIT, + .m_name = "_testcapi_datetime", + .m_size = 0, + .m_methods = test_methods, + .m_slots = _testcapi_datetime_slots, +}; + +PyMODINIT_FUNC +PyInit__testcapi_datetime(void) +{ + return PyModuleDef_Init(&_testcapi_datetime_module); +} diff --git a/Modules/_testcapi/dict.c b/Modules/_testcapi/dict.c index fe03c24f75e196..307797f98f12ae 100644 --- a/Modules/_testcapi/dict.c +++ b/Modules/_testcapi/dict.c @@ -1,60 +1,6 @@ #include "parts.h" #include "util.h" - -static PyObject * -dict_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyDict_Check(obj)); -} - -static PyObject * -dict_checkexact(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyDict_CheckExact(obj)); -} - -static PyObject * -dict_new(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - return PyDict_New(); -} - -static PyObject * -dictproxy_new(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyDictProxy_New(obj); -} - -static PyObject * -dict_clear(PyObject *self, PyObject *obj) -{ - PyDict_Clear(obj); - Py_RETURN_NONE; -} - -static PyObject * -dict_copy(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyDict_Copy(obj); -} - -static PyObject * -dict_contains(PyObject *self, PyObject *args) -{ - PyObject *obj, *key; - if (!PyArg_ParseTuple(args, "OO", &obj, &key)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(key); - RETURN_INT(PyDict_Contains(obj, key)); -} - static PyObject * dict_containsstring(PyObject *self, PyObject *args) { @@ -68,72 +14,6 @@ dict_containsstring(PyObject *self, PyObject *args) RETURN_INT(PyDict_ContainsString(obj, key)); } -static PyObject * -dict_size(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PyDict_Size(obj)); -} - -static PyObject * -dict_getitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - PyObject *value = PyDict_GetItem(mapping, key); - if (value == NULL) { - if (PyErr_Occurred()) { - return NULL; - } - return Py_NewRef(PyExc_KeyError); - } - return Py_NewRef(value); -} - -static PyObject * -dict_getitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - PyObject *value = PyDict_GetItemString(mapping, key); - if (value == NULL) { - if (PyErr_Occurred()) { - return NULL; - } - return Py_NewRef(PyExc_KeyError); - } - return Py_NewRef(value); -} - -static PyObject * -dict_getitemwitherror(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - PyObject *value = PyDict_GetItemWithError(mapping, key); - if (value == NULL) { - if (PyErr_Occurred()) { - return NULL; - } - return Py_NewRef(PyExc_KeyError); - } - return Py_NewRef(value); -} - - static PyObject * dict_getitemref(PyObject *self, PyObject *args) { @@ -185,33 +65,6 @@ dict_getitemstringref(PyObject *self, PyObject *args) } } -static PyObject * -dict_setitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key, *value; - if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - NULLABLE(value); - RETURN_INT(PyDict_SetItem(mapping, key, value)); -} - -static PyObject * -dict_setitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping, *value; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(value); - RETURN_INT(PyDict_SetItemString(mapping, key, value)); -} - static PyObject * dict_setdefault(PyObject *self, PyObject *args) { @@ -250,112 +103,6 @@ dict_setdefaultref(PyObject *self, PyObject *args) } } -static PyObject * -dict_delitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - RETURN_INT(PyDict_DelItem(mapping, key)); -} - -static PyObject * -dict_delitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - RETURN_INT(PyDict_DelItemString(mapping, key)); -} - -static PyObject * -dict_keys(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyDict_Keys(obj); -} - -static PyObject * -dict_values(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyDict_Values(obj); -} - -static PyObject * -dict_items(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyDict_Items(obj); -} - -static PyObject * -dict_next(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key = UNINITIALIZED_PTR, *value = UNINITIALIZED_PTR; - Py_ssize_t pos; - if (!PyArg_ParseTuple(args, "On", &mapping, &pos)) { - return NULL; - } - NULLABLE(mapping); - int rc = PyDict_Next(mapping, &pos, &key, &value); - if (rc != 0) { - return Py_BuildValue("inOO", rc, pos, key, value); - } - assert(key == UNINITIALIZED_PTR); - assert(value == UNINITIALIZED_PTR); - if (PyErr_Occurred()) { - return NULL; - } - Py_RETURN_NONE; -} - -static PyObject * -dict_merge(PyObject *self, PyObject *args) -{ - PyObject *mapping, *mapping2; - int override; - if (!PyArg_ParseTuple(args, "OOi", &mapping, &mapping2, &override)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(mapping2); - RETURN_INT(PyDict_Merge(mapping, mapping2, override)); -} - -static PyObject * -dict_update(PyObject *self, PyObject *args) -{ - PyObject *mapping, *mapping2; - if (!PyArg_ParseTuple(args, "OO", &mapping, &mapping2)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(mapping2); - RETURN_INT(PyDict_Update(mapping, mapping2)); -} - -static PyObject * -dict_mergefromseq2(PyObject *self, PyObject *args) -{ - PyObject *mapping, *seq; - int override; - if (!PyArg_ParseTuple(args, "OOi", &mapping, &seq, &override)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(seq); - RETURN_INT(PyDict_MergeFromSeq2(mapping, seq, override)); -} - - static PyObject * dict_pop(PyObject *self, PyObject *args) { @@ -382,7 +129,6 @@ dict_pop(PyObject *self, PyObject *args) return Py_BuildValue("iN", res, result); } - static PyObject * dict_pop_null(PyObject *self, PyObject *args) { @@ -396,7 +142,6 @@ dict_pop_null(PyObject *self, PyObject *args) RETURN_INT(PyDict_Pop(dict, key, NULL)); } - static PyObject * dict_popstring(PyObject *self, PyObject *args) { @@ -423,7 +168,6 @@ dict_popstring(PyObject *self, PyObject *args) return Py_BuildValue("iN", res, result); } - static PyObject * dict_popstring_null(PyObject *self, PyObject *args) { @@ -437,35 +181,12 @@ dict_popstring_null(PyObject *self, PyObject *args) RETURN_INT(PyDict_PopString(dict, key, NULL)); } - static PyMethodDef test_methods[] = { - {"dict_check", dict_check, METH_O}, - {"dict_checkexact", dict_checkexact, METH_O}, - {"dict_new", dict_new, METH_NOARGS}, - {"dictproxy_new", dictproxy_new, METH_O}, - {"dict_clear", dict_clear, METH_O}, - {"dict_copy", dict_copy, METH_O}, - {"dict_size", dict_size, METH_O}, - {"dict_getitem", dict_getitem, METH_VARARGS}, - {"dict_getitemwitherror", dict_getitemwitherror, METH_VARARGS}, - {"dict_getitemstring", dict_getitemstring, METH_VARARGS}, + {"dict_containsstring", dict_containsstring, METH_VARARGS}, {"dict_getitemref", dict_getitemref, METH_VARARGS}, {"dict_getitemstringref", dict_getitemstringref, METH_VARARGS}, - {"dict_contains", dict_contains, METH_VARARGS}, - {"dict_containsstring", dict_containsstring, METH_VARARGS}, - {"dict_setitem", dict_setitem, METH_VARARGS}, - {"dict_setitemstring", dict_setitemstring, METH_VARARGS}, - {"dict_delitem", dict_delitem, METH_VARARGS}, - {"dict_delitemstring", dict_delitemstring, METH_VARARGS}, {"dict_setdefault", dict_setdefault, METH_VARARGS}, {"dict_setdefaultref", dict_setdefaultref, METH_VARARGS}, - {"dict_keys", dict_keys, METH_O}, - {"dict_values", dict_values, METH_O}, - {"dict_items", dict_items, METH_O}, - {"dict_next", dict_next, METH_VARARGS}, - {"dict_merge", dict_merge, METH_VARARGS}, - {"dict_update", dict_update, METH_VARARGS}, - {"dict_mergefromseq2", dict_mergefromseq2, METH_VARARGS}, {"dict_pop", dict_pop, METH_VARARGS}, {"dict_pop_null", dict_pop_null, METH_VARARGS}, {"dict_popstring", dict_popstring, METH_VARARGS}, diff --git a/Modules/_testcapi/docstring.c b/Modules/_testcapi/docstring.c index d99fbdd904b594..3f7acbae1b181b 100644 --- a/Modules/_testcapi/docstring.c +++ b/Modules/_testcapi/docstring.c @@ -169,6 +169,13 @@ static PyMethodDef DocStringUnrepresentableSignatureTest_methods[] = { "--\n\n" "This docstring has a signature with unrepresentable default." )}, + {"with_default", + (PyCFunction)test_with_docstring, METH_VARARGS, + PyDoc_STR( + "with_default($self, /, x=ONE)\n" + "--\n\n" + "This instance method has a default parameter value from the module scope." + )}, {NULL}, }; @@ -193,5 +200,8 @@ _PyTestCapi_Init_Docstring(PyObject *mod) if (PyModule_AddType(mod, &DocStringUnrepresentableSignatureTest) < 0) { return -1; } + if (PyModule_AddObject(mod, "ONE", PyLong_FromLong(1)) < 0) { + return -1; + } return 0; } diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index 42a9915143e6fa..b647bfc71eae24 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -3,6 +3,7 @@ #include "parts.h" #include "util.h" + #include "clinic/exceptions.c.h" @@ -34,11 +35,11 @@ err_restore(PyObject *self, PyObject *args) { case 3: traceback = PyTuple_GetItem(args, 2); Py_INCREF(traceback); - /* fall through */ + _Py_FALLTHROUGH; case 2: value = PyTuple_GetItem(args, 1); Py_INCREF(value); - /* fall through */ + _Py_FALLTHROUGH; case 1: type = PyTuple_GetItem(args, 0); Py_INCREF(type); @@ -359,6 +360,161 @@ _testcapi_unstable_exc_prep_reraise_star_impl(PyObject *module, return PyUnstable_Exc_PrepReraiseStar(orig, excs); } +/* Test PyUnicodeEncodeError_GetStart */ +static PyObject * +unicode_encode_get_start(PyObject *Py_UNUSED(module), PyObject *arg) +{ + Py_ssize_t start; + if (PyUnicodeEncodeError_GetStart(arg, &start) < 0) { + return NULL; + } + RETURN_SIZE(start); +} + +/* Test PyUnicodeDecodeError_GetStart */ +static PyObject * +unicode_decode_get_start(PyObject *Py_UNUSED(module), PyObject *arg) +{ + Py_ssize_t start; + if (PyUnicodeDecodeError_GetStart(arg, &start) < 0) { + return NULL; + } + RETURN_SIZE(start); +} + +/* Test PyUnicodeTranslateError_GetStart */ +static PyObject * +unicode_translate_get_start(PyObject *Py_UNUSED(module), PyObject *arg) +{ + Py_ssize_t start; + if (PyUnicodeTranslateError_GetStart(arg, &start) < 0) { + return NULL; + } + RETURN_SIZE(start); +} + +/* Test PyUnicodeEncodeError_SetStart */ +static PyObject * +unicode_encode_set_start(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *exc; + Py_ssize_t start; + if (PyArg_ParseTuple(args, "On", &exc, &start) < 0) { + return NULL; + } + if (PyUnicodeEncodeError_SetStart(exc, start) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/* Test PyUnicodeDecodeError_SetStart */ +static PyObject * +unicode_decode_set_start(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *exc; + Py_ssize_t start; + if (PyArg_ParseTuple(args, "On", &exc, &start) < 0) { + return NULL; + } + if (PyUnicodeDecodeError_SetStart(exc, start) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/* Test PyUnicodeTranslateError_SetStart */ +static PyObject * +unicode_translate_set_start(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *exc; + Py_ssize_t start; + if (PyArg_ParseTuple(args, "On", &exc, &start) < 0) { + return NULL; + } + if (PyUnicodeTranslateError_SetStart(exc, start) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/* Test PyUnicodeEncodeError_GetEnd */ +static PyObject * +unicode_encode_get_end(PyObject *Py_UNUSED(module), PyObject *arg) +{ + Py_ssize_t end; + if (PyUnicodeEncodeError_GetEnd(arg, &end) < 0) { + return NULL; + } + RETURN_SIZE(end); +} + +/* Test PyUnicodeDecodeError_GetEnd */ +static PyObject * +unicode_decode_get_end(PyObject *Py_UNUSED(module), PyObject *arg) +{ + Py_ssize_t end; + if (PyUnicodeDecodeError_GetEnd(arg, &end) < 0) { + return NULL; + } + RETURN_SIZE(end); +} + +/* Test PyUnicodeTranslateError_GetEnd */ +static PyObject * +unicode_translate_get_end(PyObject *Py_UNUSED(module), PyObject *arg) +{ + Py_ssize_t end; + if (PyUnicodeTranslateError_GetEnd(arg, &end) < 0) { + return NULL; + } + RETURN_SIZE(end); +} + +/* Test PyUnicodeEncodeError_SetEnd */ +static PyObject * +unicode_encode_set_end(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *exc; + Py_ssize_t end; + if (PyArg_ParseTuple(args, "On", &exc, &end) < 0) { + return NULL; + } + if (PyUnicodeEncodeError_SetEnd(exc, end) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/* Test PyUnicodeDecodeError_SetEnd */ +static PyObject * +unicode_decode_set_end(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *exc; + Py_ssize_t end; + if (PyArg_ParseTuple(args, "On", &exc, &end) < 0) { + return NULL; + } + if (PyUnicodeDecodeError_SetEnd(exc, end) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/* Test PyUnicodeTranslateError_SetEnd */ +static PyObject * +unicode_translate_set_end(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *exc; + Py_ssize_t end; + if (PyArg_ParseTuple(args, "On", &exc, &end) < 0) { + return NULL; + } + if (PyUnicodeTranslateError_SetEnd(exc, end) < 0) { + return NULL; + } + Py_RETURN_NONE; +} /* * Define the PyRecurdingInfinitelyError_Type @@ -403,6 +559,18 @@ static PyMethodDef test_methods[] = { _TESTCAPI_SET_EXCEPTION_METHODDEF _TESTCAPI_TRACEBACK_PRINT_METHODDEF _TESTCAPI_UNSTABLE_EXC_PREP_RERAISE_STAR_METHODDEF + {"unicode_encode_get_start", unicode_encode_get_start, METH_O}, + {"unicode_decode_get_start", unicode_decode_get_start, METH_O}, + {"unicode_translate_get_start", unicode_translate_get_start, METH_O}, + {"unicode_encode_set_start", unicode_encode_set_start, METH_VARARGS}, + {"unicode_decode_set_start", unicode_decode_set_start, METH_VARARGS}, + {"unicode_translate_set_start", unicode_translate_set_start, METH_VARARGS}, + {"unicode_encode_get_end", unicode_encode_get_end, METH_O}, + {"unicode_decode_get_end", unicode_decode_get_end, METH_O}, + {"unicode_translate_get_end", unicode_translate_get_end, METH_O}, + {"unicode_encode_set_end", unicode_encode_set_end, METH_VARARGS}, + {"unicode_decode_set_end", unicode_decode_set_end, METH_VARARGS}, + {"unicode_translate_set_end", unicode_translate_set_end, METH_VARARGS}, {NULL}, }; diff --git a/Modules/_testcapi/file.c b/Modules/_testcapi/file.c index 634563f6ea12cb..d15173fc7959e5 100644 --- a/Modules/_testcapi/file.c +++ b/Modules/_testcapi/file.c @@ -1,8 +1,45 @@ +// clinic/file.c.h uses internal pycore_modsupport.h API +#define PYTESTCAPI_NEED_INTERNAL_API + #include "parts.h" #include "util.h" +#include "clinic/file.c.h" + +/*[clinic input] +module _testcapi +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/ + +/*[clinic input] +_testcapi.py_fopen + + path: object + mode: str(zeroes=True, accept={robuffer, str, NoneType}) + / + +Call Py_fopen(), fread(256) and Py_fclose(). Return read bytes. +[clinic start generated code]*/ +static PyObject * +_testcapi_py_fopen_impl(PyObject *module, PyObject *path, const char *mode, + Py_ssize_t mode_length) +/*[clinic end generated code: output=69840d0cfd8b7fbb input=f3a579dd7eb60926]*/ +{ + NULLABLE(path); + FILE *fp = Py_fopen(path, mode); + if (fp == NULL) { + return NULL; + } + + char buffer[256]; + size_t size = fread(buffer, 1, Py_ARRAY_LENGTH(buffer), fp); + Py_fclose(fp); + + return PyBytes_FromStringAndSize(buffer, size); +} static PyMethodDef test_methods[] = { + _TESTCAPI_PY_FOPEN_METHODDEF {NULL}, }; diff --git a/Modules/_testcapi/float.c b/Modules/_testcapi/float.c index 4fcbaf3bb2aa1e..15ea97ec4520b7 100644 --- a/Modules/_testcapi/float.c +++ b/Modules/_testcapi/float.c @@ -6,71 +6,6 @@ #include "clinic/float.c.h" -static PyObject * -float_check(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyFloat_Check(obj)); -} - -static PyObject * -float_checkexact(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyFloat_CheckExact(obj)); -} - -static PyObject * -float_fromstring(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyFloat_FromString(obj); -} - -static PyObject * -float_fromdouble(PyObject *Py_UNUSED(module), PyObject *obj) -{ - double d; - - if (!PyArg_Parse(obj, "d", &d)) { - return NULL; - } - - return PyFloat_FromDouble(d); -} - -static PyObject * -float_asdouble(PyObject *Py_UNUSED(module), PyObject *obj) -{ - double d; - - NULLABLE(obj); - d = PyFloat_AsDouble(obj); - if (d == -1. && PyErr_Occurred()) { - return NULL; - } - - return PyFloat_FromDouble(d); -} - -static PyObject * -float_getinfo(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) -{ - return PyFloat_GetInfo(); -} - -static PyObject * -float_getmax(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) -{ - return PyFloat_FromDouble(PyFloat_GetMax()); -} - -static PyObject * -float_getmin(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) -{ - return PyFloat_FromDouble(PyFloat_GetMin()); -} - /*[clinic input] module _testcapi [clinic start generated code]*/ @@ -165,14 +100,6 @@ _testcapi_float_unpack_impl(PyObject *module, const char *data, } static PyMethodDef test_methods[] = { - {"float_check", float_check, METH_O}, - {"float_checkexact", float_checkexact, METH_O}, - {"float_fromstring", float_fromstring, METH_O}, - {"float_fromdouble", float_fromdouble, METH_O}, - {"float_asdouble", float_asdouble, METH_O}, - {"float_getinfo", float_getinfo, METH_NOARGS}, - {"float_getmax", float_getmax, METH_NOARGS}, - {"float_getmin", float_getmin, METH_NOARGS}, _TESTCAPI_FLOAT_PACK_METHODDEF _TESTCAPI_FLOAT_UNPACK_METHODDEF {NULL}, diff --git a/Modules/_testcapi/gc.c b/Modules/_testcapi/gc.c index f4feaaafbdc6cc..7e33e0d4861e84 100644 --- a/Modules/_testcapi/gc.c +++ b/Modules/_testcapi/gc.c @@ -72,7 +72,7 @@ without_gc(PyObject *Py_UNUSED(self), PyObject *obj) if (PyType_IS_GC(tp)) { // Don't try this at home, kids: tp->tp_flags -= Py_TPFLAGS_HAVE_GC; - tp->tp_free = PyObject_Del; + tp->tp_free = PyObject_Free; tp->tp_traverse = NULL; tp->tp_clear = NULL; } @@ -99,10 +99,11 @@ slot_tp_del(PyObject *self) return; } /* Execute __del__ method, if any. */ - del = _PyType_Lookup(Py_TYPE(self), tp_del); + del = _PyType_LookupRef(Py_TYPE(self), tp_del); Py_DECREF(tp_del); if (del != NULL) { res = PyObject_CallOneArg(del, self); + Py_DECREF(del); if (res == NULL) PyErr_WriteUnraisable(del); else diff --git a/Modules/_testcapi/getargs.c b/Modules/_testcapi/getargs.c index 0d61d8c8969f82..ee04c760d27213 100644 --- a/Modules/_testcapi/getargs.c +++ b/Modules/_testcapi/getargs.c @@ -141,6 +141,122 @@ getargs_w_star(PyObject *self, PyObject *args) return result; } +static PyObject * +getargs_w_star_opt(PyObject *self, PyObject *args) +{ + Py_buffer buffer; + Py_buffer buf2; + int number = 1; + + if (!PyArg_ParseTuple(args, "w*|w*i:getargs_w_star", + &buffer, &buf2, &number)) { + return NULL; + } + + if (2 <= buffer.len) { + char *str = buffer.buf; + str[0] = '['; + str[buffer.len-1] = ']'; + } + + PyObject *result = PyBytes_FromStringAndSize(buffer.buf, buffer.len); + PyBuffer_Release(&buffer); + return result; +} + +/* Test the old w and w# codes that no longer work */ +static PyObject * +test_w_code_invalid(PyObject *self, PyObject *arg) +{ + static const char * const keywords[] = {"a", "b", "c", "d", NULL}; + char *formats_3[] = {"O|w#$O", + "O|w$O", + "O|w#O", + "O|wO", + NULL}; + char *formats_4[] = {"O|w#O$O", + "O|wO$O", + "O|Ow#O", + "O|OwO", + "O|Ow#$O", + "O|Ow$O", + NULL}; + size_t n; + PyObject *args; + PyObject *kwargs; + PyObject *tmp; + + if (!(args = PyTuple_Pack(1, Py_None))) { + return NULL; + } + + kwargs = PyDict_New(); + if (!kwargs) { + Py_DECREF(args); + return NULL; + } + + if (PyDict_SetItemString(kwargs, "c", Py_None)) { + Py_DECREF(args); + Py_XDECREF(kwargs); + return NULL; + } + + for (n = 0; formats_3[n]; ++n) { + if (PyArg_ParseTupleAndKeywords(args, kwargs, formats_3[n], + (char**) keywords, + &tmp, &tmp, &tmp)) { + Py_DECREF(args); + Py_DECREF(kwargs); + PyErr_Format(PyExc_AssertionError, + "test_w_code_invalid_suffix: %s", + formats_3[n]); + return NULL; + } + else { + if (!PyErr_ExceptionMatches(PyExc_SystemError)) { + Py_DECREF(args); + Py_DECREF(kwargs); + return NULL; + } + PyErr_Clear(); + } + } + + if (PyDict_DelItemString(kwargs, "c") || + PyDict_SetItemString(kwargs, "d", Py_None)) { + + Py_DECREF(kwargs); + Py_DECREF(args); + return NULL; + } + + for (n = 0; formats_4[n]; ++n) { + if (PyArg_ParseTupleAndKeywords(args, kwargs, formats_4[n], + (char**) keywords, + &tmp, &tmp, &tmp, &tmp)) { + Py_DECREF(args); + Py_DECREF(kwargs); + PyErr_Format(PyExc_AssertionError, + "test_w_code_invalid_suffix: %s", + formats_4[n]); + return NULL; + } + else { + if (!PyErr_ExceptionMatches(PyExc_SystemError)) { + Py_DECREF(args); + Py_DECREF(kwargs); + return NULL; + } + PyErr_Clear(); + } + } + + Py_DECREF(args); + Py_DECREF(kwargs); + Py_RETURN_NONE; +} + static PyObject * getargs_empty(PyObject *self, PyObject *args, PyObject *kwargs) { @@ -684,6 +800,7 @@ static PyMethodDef test_methods[] = { {"getargs_s_star", getargs_s_star, METH_VARARGS}, {"getargs_tuple", getargs_tuple, METH_VARARGS}, {"getargs_w_star", getargs_w_star, METH_VARARGS}, + {"getargs_w_star_opt", getargs_w_star_opt, METH_VARARGS}, {"getargs_empty", _PyCFunction_CAST(getargs_empty), METH_VARARGS|METH_KEYWORDS}, {"getargs_y", getargs_y, METH_VARARGS}, {"getargs_y_hash", getargs_y_hash, METH_VARARGS}, @@ -693,6 +810,7 @@ static PyMethodDef test_methods[] = { {"getargs_z_star", getargs_z_star, METH_VARARGS}, {"parse_tuple_and_keywords", parse_tuple_and_keywords, METH_VARARGS}, {"gh_99240_clear_args", gh_99240_clear_args, METH_VARARGS}, + {"test_w_code_invalid", test_w_code_invalid, METH_NOARGS}, {NULL}, }; diff --git a/Modules/_testcapi/hash.c b/Modules/_testcapi/hash.c index aee76787dcddb3..1525344a93fbcf 100644 --- a/Modules/_testcapi/hash.c +++ b/Modules/_testcapi/hash.c @@ -45,6 +45,14 @@ hash_getfuncdef(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) } +static PyObject * +long_from_hash(Py_hash_t hash) +{ + Py_BUILD_ASSERT(sizeof(long long) >= sizeof(hash)); + return PyLong_FromLongLong(hash); +} + + static PyObject * hash_pointer(PyObject *Py_UNUSED(module), PyObject *arg) { @@ -54,14 +62,38 @@ hash_pointer(PyObject *Py_UNUSED(module), PyObject *arg) } Py_hash_t hash = Py_HashPointer(ptr); - Py_BUILD_ASSERT(sizeof(long long) >= sizeof(hash)); - return PyLong_FromLongLong(hash); + return long_from_hash(hash); +} + + +static PyObject * +hash_buffer(PyObject *Py_UNUSED(module), PyObject *args) +{ + char *ptr; + Py_ssize_t len; + if (!PyArg_ParseTuple(args, "y#", &ptr, &len)) { + return NULL; + } + + Py_hash_t hash = Py_HashBuffer(ptr, len); + return long_from_hash(hash); +} + + +static PyObject * +object_generichash(PyObject *Py_UNUSED(module), PyObject *arg) +{ + NULLABLE(arg); + Py_hash_t hash = PyObject_GenericHash(arg); + return long_from_hash(hash); } static PyMethodDef test_methods[] = { {"hash_getfuncdef", hash_getfuncdef, METH_NOARGS}, {"hash_pointer", hash_pointer, METH_O}, + {"hash_buffer", hash_buffer, METH_VARARGS}, + {"object_generichash", object_generichash, METH_O}, {NULL}, }; diff --git a/Modules/_testcapi/heaptype.c b/Modules/_testcapi/heaptype.c index 4526583a8059d9..cc88147dfcd7fb 100644 --- a/Modules/_testcapi/heaptype.c +++ b/Modules/_testcapi/heaptype.c @@ -269,16 +269,16 @@ test_type_from_ephemeral_spec(PyObject *self, PyObject *Py_UNUSED(ignored)) // (Explicitly overwrite memory before freeing, // so bugs show themselves even without the debug allocator's help.) memset(spec, 0xdd, sizeof(PyType_Spec)); - PyMem_Del(spec); + PyMem_Free(spec); spec = NULL; memset(name, 0xdd, sizeof(NAME)); - PyMem_Del(name); + PyMem_Free(name); name = NULL; memset(doc, 0xdd, sizeof(DOC)); - PyMem_Del(doc); + PyMem_Free(doc); doc = NULL; memset(slots, 0xdd, 3 * sizeof(PyType_Slot)); - PyMem_Del(slots); + PyMem_Free(slots); slots = NULL; /* check that everything works */ @@ -304,10 +304,10 @@ test_type_from_ephemeral_spec(PyObject *self, PyObject *Py_UNUSED(ignored)) result = Py_NewRef(Py_None); finally: - PyMem_Del(spec); - PyMem_Del(name); - PyMem_Del(doc); - PyMem_Del(slots); + PyMem_Free(spec); + PyMem_Free(name); + PyMem_Free(doc); + PyMem_Free(slots); Py_XDECREF(class); Py_XDECREF(instance); Py_XDECREF(obj); @@ -410,6 +410,118 @@ pyobject_getitemdata(PyObject *self, PyObject *o) } +static PyObject * +create_type_with_token(PyObject *module, PyObject *args) +{ + const char *name; + PyObject *py_token; + if (!PyArg_ParseTuple(args, "sO", &name, &py_token)) { + return NULL; + } + void *token = PyLong_AsVoidPtr(py_token); + if (token == Py_TP_USE_SPEC) { + // Py_TP_USE_SPEC requires the spec that at least outlives the class + static PyType_Slot slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, + {0}, + }; + static PyType_Spec spec = { + .name = "_testcapi.DefaultTokenTest", + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = slots, + }; + PyObject *type = PyType_FromMetaclass(NULL, NULL, &spec, NULL); + if (!type) { + return NULL; + } + token = PyType_GetSlot((PyTypeObject *)type, Py_tp_token); + assert(!PyErr_Occurred()); + Py_DECREF(type); + if (token != &spec) { + PyErr_SetString(PyExc_AssertionError, + "failed to convert token from Py_TP_USE_SPEC"); + return NULL; + } + } + // Test non-NULL token that must also outlive the class + PyType_Slot slots[] = { + {Py_tp_token, token}, + {0}, + }; + PyType_Spec spec = { + .name = name, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = slots, + }; + return PyType_FromMetaclass(NULL, module, &spec, NULL); +} + +static PyObject * +get_tp_token(PyObject *self, PyObject *type) +{ + void *token = PyType_GetSlot((PyTypeObject *)type, Py_tp_token); + if (PyErr_Occurred()) { + return NULL; + } + return PyLong_FromVoidPtr(token); +} + +static PyObject * +pytype_getbasebytoken(PyObject *self, PyObject *args) +{ + PyTypeObject *type; + PyObject *py_token, *use_mro, *need_result; + if (!PyArg_ParseTuple(args, "OOOO", + &type, &py_token, &use_mro, &need_result)) { + return NULL; + } + + PyObject *mro_save = NULL; + if (use_mro != Py_True) { + // Test internal detail: PyType_GetBaseByToken works even with + // types that are only partially initialized (or torn down): + // if tp_mro=NULL we fall back to tp_bases. + assert(PyType_Check(type)); + mro_save = type->tp_mro; + type->tp_mro = NULL; + } + + void *token = PyLong_AsVoidPtr(py_token); + PyObject *result; + int ret; + if (need_result == Py_True) { + ret = PyType_GetBaseByToken(type, token, (PyTypeObject **)&result); + } + else { + result = NULL; + ret = PyType_GetBaseByToken(type, token, NULL); + } + + if (use_mro != Py_True) { + type->tp_mro = mro_save; + } + if (ret < 0) { + assert(result == NULL); + return NULL; + } + PyObject *py_ret = PyLong_FromLong(ret); + if (py_ret == NULL) { + goto error; + } + PyObject *tuple = PyTuple_New(2); + if (tuple == NULL) { + goto error; + } + PyTuple_SET_ITEM(tuple, 0, py_ret); + PyTuple_SET_ITEM(tuple, 1, result ? result : Py_None); + return tuple; +error: + Py_XDECREF(py_ret); + Py_XDECREF(result); + return NULL; +} + + static PyMethodDef TestMethods[] = { {"pytype_fromspec_meta", pytype_fromspec_meta, METH_O}, {"test_type_from_ephemeral_spec", test_type_from_ephemeral_spec, METH_NOARGS}, @@ -423,6 +535,9 @@ static PyMethodDef TestMethods[] = { {"make_immutable_type_with_base", make_immutable_type_with_base, METH_O}, {"make_type_with_base", make_type_with_base, METH_O}, {"pyobject_getitemdata", pyobject_getitemdata, METH_O}, + {"create_type_with_token", create_type_with_token, METH_VARARGS}, + {"get_tp_token", get_tp_token, METH_O}, + {"pytype_getbasebytoken", pytype_getbasebytoken, METH_VARARGS}, {NULL}, }; @@ -1008,6 +1123,89 @@ static PyType_Spec HeapCTypeSetattr_spec = { HeapCTypeSetattr_slots }; +/* + * The code below is for a test that uses PyType_FromSpec API to create a heap + * type that simultaneously exposes + * + * - A regular __new__ / __init__ constructor pair + * - A vector call handler in the type object + * + * A general requirement of vector call implementations is that they should + * behave identically (except being potentially faster). The example below + * deviates from this rule by initializing the instance with a different value. + * This is only done here only so that we can see which path was taken and is + * strongly discouraged in other cases. + */ + +typedef struct { + PyObject_HEAD + long value; +} HeapCTypeVectorcallObject; + +static PyObject *heapctype_vectorcall_vectorcall(PyObject *self, + PyObject *const *args_in, + size_t nargsf, + PyObject *kwargs_in) +{ + if (kwargs_in || PyVectorcall_NARGS(nargsf)) { + return PyErr_Format(PyExc_IndexError, "HeapCTypeVectorcall() takes no arguments!"); + } + + HeapCTypeVectorcallObject *r = + PyObject_New(HeapCTypeVectorcallObject, (PyTypeObject *) self); + + if (!r) { + return NULL; + } + + r->value = 1; + + return (PyObject *) r; +} + +static PyObject * +heapctype_vectorcall_new(PyTypeObject* type, PyObject* args, PyObject *kwargs) +{ + if (PyTuple_GET_SIZE(args) || kwargs) { + return PyErr_Format(PyExc_IndexError, "HeapCTypeVectorcall() takes no arguments!"); + } + + return (PyObject *) PyObject_New(HeapCTypeVectorcallObject, type); +} + +static int +heapctype_vectorcall_init(PyObject *self, PyObject *args, PyObject *kwargs) { + if (PyTuple_GET_SIZE(args) || kwargs) { + PyErr_Format(PyExc_IndexError, "HeapCTypeVectorcall() takes no arguments!"); + return -1; + } + + HeapCTypeVectorcallObject *o = (HeapCTypeVectorcallObject *) self; + o->value = 2; + return 0; +} + +static struct PyMemberDef heapctype_vectorcall_members[] = { + {"value", Py_T_LONG, offsetof(HeapCTypeVectorcallObject, value), 0, NULL}, + {NULL} +}; + +static PyType_Slot HeapCTypeVectorcall_slots[] = { + {Py_tp_new, heapctype_vectorcall_new}, + {Py_tp_init, heapctype_vectorcall_init}, + {Py_tp_vectorcall, heapctype_vectorcall_vectorcall}, + {Py_tp_members, heapctype_vectorcall_members}, + {0, 0}, +}; + +static PyType_Spec HeapCTypeVectorcall_spec = { + "_testcapi.HeapCTypeVectorcall", + sizeof(HeapCTypeVectorcallObject), + 0, + Py_TPFLAGS_DEFAULT, + HeapCTypeVectorcall_slots +}; + PyDoc_STRVAR(HeapCCollection_doc, "Tuple-like heap type that uses PyObject_GetItemData for items."); @@ -1180,6 +1378,9 @@ _PyTestCapi_Init_Heaptype(PyObject *m) { PyObject *HeapCTypeSetattr = PyType_FromSpec(&HeapCTypeSetattr_spec); ADD("HeapCTypeSetattr", HeapCTypeSetattr); + PyObject *HeapCTypeVectorcall = PyType_FromSpec(&HeapCTypeVectorcall_spec); + ADD("HeapCTypeVectorcall", HeapCTypeVectorcall); + PyObject *subclass_with_finalizer_bases = PyTuple_Pack(1, HeapCTypeSubclass); if (subclass_with_finalizer_bases == NULL) { return -1; @@ -1201,6 +1402,8 @@ _PyTestCapi_Init_Heaptype(PyObject *m) { &PyType_Type, m, &HeapCTypeMetaclassNullNew_spec, (PyObject *) &PyType_Type); ADD("HeapCTypeMetaclassNullNew", HeapCTypeMetaclassNullNew); + ADD("Py_TP_USE_SPEC", PyLong_FromVoidPtr(Py_TP_USE_SPEC)); + PyObject *HeapCCollection = PyType_FromMetaclass( NULL, m, &HeapCCollection_spec, NULL); if (HeapCCollection == NULL) { diff --git a/Modules/_testcapi/immortal.c b/Modules/_testcapi/immortal.c index 9f81389811c645..0663c3781d426a 100644 --- a/Modules/_testcapi/immortal.c +++ b/Modules/_testcapi/immortal.c @@ -1,5 +1,8 @@ #include "parts.h" +#define Py_BUILD_CORE +#include "internal/pycore_long.h" // IMMORTALITY_BIT_MASK + int verify_immortality(PyObject *object) { assert(_Py_IsImmortal(object)); @@ -26,14 +29,31 @@ static PyObject * test_immortal_small_ints(PyObject *self, PyObject *Py_UNUSED(ignored)) { for (int i = -5; i <= 256; i++) { - assert(verify_immortality(PyLong_FromLong(i))); + PyObject *obj = PyLong_FromLong(i); + assert(verify_immortality(obj)); + int has_int_immortal_bit = ((PyLongObject *)obj)->long_value.lv_tag & IMMORTALITY_BIT_MASK; + assert(has_int_immortal_bit); + } + for (int i = 257; i <= 260; i++) { + PyObject *obj = PyLong_FromLong(i); + assert(obj); + int has_int_immortal_bit = ((PyLongObject *)obj)->long_value.lv_tag & IMMORTALITY_BIT_MASK; + assert(!has_int_immortal_bit); + Py_DECREF(obj); } Py_RETURN_NONE; } +static PyObject * +is_immortal(PyObject *self, PyObject *op) +{ + return PyBool_FromLong(PyUnstable_IsImmortal(op)); +} + static PyMethodDef test_methods[] = { {"test_immortal_builtins", test_immortal_builtins, METH_NOARGS}, {"test_immortal_small_ints", test_immortal_small_ints, METH_NOARGS}, + {"is_immortal", is_immortal, METH_O}, {NULL}, }; diff --git a/Modules/_testcapi/list.c b/Modules/_testcapi/list.c index 2cb6499e28336d..09cec4c30c8c36 100644 --- a/Modules/_testcapi/list.c +++ b/Modules/_testcapi/list.c @@ -1,32 +1,6 @@ #include "parts.h" #include "util.h" -static PyObject * -list_check(PyObject* Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyList_Check(obj)); -} - -static PyObject * -list_check_exact(PyObject* Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyList_CheckExact(obj)); -} - -static PyObject * -list_new(PyObject* Py_UNUSED(module), PyObject *obj) -{ - return PyList_New(PyLong_AsSsize_t(obj)); -} - -static PyObject * -list_size(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PyList_Size(obj)); -} static PyObject * list_get_size(PyObject *Py_UNUSED(module), PyObject *obj) @@ -35,17 +9,6 @@ list_get_size(PyObject *Py_UNUSED(module), PyObject *obj) RETURN_SIZE(PyList_GET_SIZE(obj)); } -static PyObject * -list_getitem(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj; - Py_ssize_t i; - if (!PyArg_ParseTuple(args, "On", &obj, &i)) { - return NULL; - } - NULLABLE(obj); - return Py_XNewRef(PyList_GetItem(obj, i)); -} static PyObject * list_get_item(PyObject *Py_UNUSED(module), PyObject *args) @@ -59,31 +22,6 @@ list_get_item(PyObject *Py_UNUSED(module), PyObject *args) return Py_XNewRef(PyList_GET_ITEM(obj, i)); } -static PyObject * -list_get_item_ref(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj; - Py_ssize_t i; - if (!PyArg_ParseTuple(args, "On", &obj, &i)) { - return NULL; - } - NULLABLE(obj); - return PyList_GetItemRef(obj, i); -} - -static PyObject * -list_setitem(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj, *value; - Py_ssize_t i; - if (!PyArg_ParseTuple(args, "OnO", &obj, &i, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(value); - RETURN_INT(PyList_SetItem(obj, i, Py_XNewRef(value))); - -} static PyObject * list_set_item(PyObject *Py_UNUSED(module), PyObject *args) @@ -100,79 +38,6 @@ list_set_item(PyObject *Py_UNUSED(module), PyObject *args) } -static PyObject * -list_insert(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj, *value; - Py_ssize_t where; - if (!PyArg_ParseTuple(args, "OnO", &obj, &where, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(value); - RETURN_INT(PyList_Insert(obj, where, Py_XNewRef(value))); - -} - -static PyObject * -list_append(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj, *value; - if (!PyArg_ParseTuple(args, "OO", &obj, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(value); - RETURN_INT(PyList_Append(obj, value)); -} - -static PyObject * -list_getslice(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj; - Py_ssize_t ilow, ihigh; - if (!PyArg_ParseTuple(args, "Onn", &obj, &ilow, &ihigh)) { - return NULL; - } - NULLABLE(obj); - return PyList_GetSlice(obj, ilow, ihigh); - -} - -static PyObject * -list_setslice(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj, *value; - Py_ssize_t ilow, ihigh; - if (!PyArg_ParseTuple(args, "OnnO", &obj, &ilow, &ihigh, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(value); - RETURN_INT(PyList_SetSlice(obj, ilow, ihigh, value)); -} - -static PyObject * -list_sort(PyObject* Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyList_Sort(obj)); -} - -static PyObject * -list_reverse(PyObject* Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyList_Reverse(obj)); -} - -static PyObject * -list_astuple(PyObject* Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyList_AsTuple(obj); -} - static PyObject * list_clear(PyObject* Py_UNUSED(module), PyObject *obj) @@ -196,25 +61,12 @@ list_extend(PyObject* Py_UNUSED(module), PyObject *args) static PyMethodDef test_methods[] = { - {"list_check", list_check, METH_O}, - {"list_check_exact", list_check_exact, METH_O}, - {"list_new", list_new, METH_O}, - {"list_size", list_size, METH_O}, {"list_get_size", list_get_size, METH_O}, - {"list_getitem", list_getitem, METH_VARARGS}, {"list_get_item", list_get_item, METH_VARARGS}, - {"list_get_item_ref", list_get_item_ref, METH_VARARGS}, - {"list_setitem", list_setitem, METH_VARARGS}, {"list_set_item", list_set_item, METH_VARARGS}, - {"list_insert", list_insert, METH_VARARGS}, - {"list_append", list_append, METH_VARARGS}, - {"list_getslice", list_getslice, METH_VARARGS}, - {"list_setslice", list_setslice, METH_VARARGS}, - {"list_sort", list_sort, METH_O}, - {"list_reverse", list_reverse, METH_O}, - {"list_astuple", list_astuple, METH_O}, {"list_clear", list_clear, METH_O}, {"list_extend", list_extend, METH_VARARGS}, + {NULL}, }; diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index dc21cf9f475228..42243023a45768 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -12,529 +12,6 @@ module _testcapi /*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/ -static PyObject * -raiseTestError(const char* test_name, const char* msg) -{ - PyErr_Format(PyExc_AssertionError, "%s: %s", test_name, msg); - return NULL; -} - -/* Tests of PyLong_{As, From}{Unsigned,}Long(), and - PyLong_{As, From}{Unsigned,}LongLong(). - - Note that the meat of the test is contained in testcapi_long.h. - This is revolting, but delicate code duplication is worse: "almost - exactly the same" code is needed to test long long, but the ubiquitous - dependence on type names makes it impossible to use a parameterized - function. A giant macro would be even worse than this. A C++ template - would be perfect. - - The "report an error" functions are deliberately not part of the #include - file: if the test fails, you can set a breakpoint in the appropriate - error function directly, and crawl back from there in the debugger. -*/ - -#define UNBIND(X) Py_DECREF(X); (X) = NULL - -static PyObject * -raise_test_long_error(const char* msg) -{ - return raiseTestError("test_long_api", msg); -} - -// Test PyLong_FromLong()/PyLong_AsLong() -// and PyLong_FromUnsignedLong()/PyLong_AsUnsignedLong(). - -#define TESTNAME test_long_api_inner -#define TYPENAME long -#define F_S_TO_PY PyLong_FromLong -#define F_PY_TO_S PyLong_AsLong -#define F_U_TO_PY PyLong_FromUnsignedLong -#define F_PY_TO_U PyLong_AsUnsignedLong - -#include "testcapi_long.h" - -/*[clinic input] -_testcapi.test_long_api -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_api_impl(PyObject *module) -/*[clinic end generated code: output=4405798ca1e9f444 input=e9b8880d7331c688]*/ -{ - return TESTNAME(raise_test_long_error); -} - -#undef TESTNAME -#undef TYPENAME -#undef F_S_TO_PY -#undef F_PY_TO_S -#undef F_U_TO_PY -#undef F_PY_TO_U - -// Test PyLong_FromLongLong()/PyLong_AsLongLong() -// and PyLong_FromUnsignedLongLong()/PyLong_AsUnsignedLongLong(). - -static PyObject * -raise_test_longlong_error(const char* msg) -{ - return raiseTestError("test_longlong_api", msg); -} - -#define TESTNAME test_longlong_api_inner -#define TYPENAME long long -#define F_S_TO_PY PyLong_FromLongLong -#define F_PY_TO_S PyLong_AsLongLong -#define F_U_TO_PY PyLong_FromUnsignedLongLong -#define F_PY_TO_U PyLong_AsUnsignedLongLong - -#include "testcapi_long.h" - -/*[clinic input] -_testcapi.test_longlong_api -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_longlong_api_impl(PyObject *module) -/*[clinic end generated code: output=2b3414ba8c31dfe6 input=ccbb2a48c2b3c4a5]*/ -{ - return TESTNAME(raise_test_longlong_error); -} - -#undef TESTNAME -#undef TYPENAME -#undef F_S_TO_PY -#undef F_PY_TO_S -#undef F_U_TO_PY -#undef F_PY_TO_U - - -/*[clinic input] -_testcapi.test_long_and_overflow - -Test the PyLong_AsLongAndOverflow API. - -General conversion to PY_LONG is tested by test_long_api_inner. -This test will concentrate on proper handling of overflow. -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_and_overflow_impl(PyObject *module) -/*[clinic end generated code: output=f8460ca115e31d8e input=762f6b62da0a3cdc]*/ -{ - PyObject *num, *one, *temp; - long value; - int overflow; - - /* Test that overflow is set properly for a large value. */ - /* num is a number larger than LONG_MAX even on 64-bit platforms */ - num = PyLong_FromString("FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_and_overflow", - "return value was not set to -1"); - if (overflow != 1) - return raiseTestError("test_long_and_overflow", - "overflow was not set to 1"); - - /* Same again, with num = LONG_MAX + 1 */ - num = PyLong_FromLong(LONG_MAX); - if (num == NULL) - return NULL; - one = PyLong_FromLong(1L); - if (one == NULL) { - Py_DECREF(num); - return NULL; - } - temp = PyNumber_Add(num, one); - Py_DECREF(one); - Py_SETREF(num, temp); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_and_overflow", - "return value was not set to -1"); - if (overflow != 1) - return raiseTestError("test_long_and_overflow", - "overflow was not set to 1"); - - /* Test that overflow is set properly for a large negative value. */ - /* num is a number smaller than LONG_MIN even on 64-bit platforms */ - num = PyLong_FromString("-FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_and_overflow", - "return value was not set to -1"); - if (overflow != -1) - return raiseTestError("test_long_and_overflow", - "overflow was not set to -1"); - - /* Same again, with num = LONG_MIN - 1 */ - num = PyLong_FromLong(LONG_MIN); - if (num == NULL) - return NULL; - one = PyLong_FromLong(1L); - if (one == NULL) { - Py_DECREF(num); - return NULL; - } - temp = PyNumber_Subtract(num, one); - Py_DECREF(one); - Py_SETREF(num, temp); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_and_overflow", - "return value was not set to -1"); - if (overflow != -1) - return raiseTestError("test_long_and_overflow", - "overflow was not set to -1"); - - /* Test that overflow is cleared properly for small values. */ - num = PyLong_FromString("FF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != 0xFF) - return raiseTestError("test_long_and_overflow", - "expected return value 0xFF"); - if (overflow != 0) - return raiseTestError("test_long_and_overflow", - "overflow was not cleared"); - - num = PyLong_FromString("-FF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -0xFF) - return raiseTestError("test_long_and_overflow", - "expected return value 0xFF"); - if (overflow != 0) - return raiseTestError("test_long_and_overflow", - "overflow was set incorrectly"); - - num = PyLong_FromLong(LONG_MAX); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != LONG_MAX) - return raiseTestError("test_long_and_overflow", - "expected return value LONG_MAX"); - if (overflow != 0) - return raiseTestError("test_long_and_overflow", - "overflow was not cleared"); - - num = PyLong_FromLong(LONG_MIN); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != LONG_MIN) - return raiseTestError("test_long_and_overflow", - "expected return value LONG_MIN"); - if (overflow != 0) - return raiseTestError("test_long_and_overflow", - "overflow was not cleared"); - - Py_RETURN_NONE; -} - -/*[clinic input] -_testcapi.test_long_long_and_overflow - -Test the PyLong_AsLongLongAndOverflow API. - -General conversion to long long is tested by test_long_api_inner. -This test will concentrate on proper handling of overflow. -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_long_and_overflow_impl(PyObject *module) -/*[clinic end generated code: output=0b92330786f45483 input=544bb0aefe5e8a9e]*/ -{ - PyObject *num, *one, *temp; - long long value; - int overflow; - - /* Test that overflow is set properly for a large value. */ - /* num is a number larger than LLONG_MAX on a typical machine. */ - num = PyLong_FromString("FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_long_and_overflow", - "return value was not set to -1"); - if (overflow != 1) - return raiseTestError("test_long_long_and_overflow", - "overflow was not set to 1"); - - /* Same again, with num = LLONG_MAX + 1 */ - num = PyLong_FromLongLong(LLONG_MAX); - if (num == NULL) - return NULL; - one = PyLong_FromLong(1L); - if (one == NULL) { - Py_DECREF(num); - return NULL; - } - temp = PyNumber_Add(num, one); - Py_DECREF(one); - Py_SETREF(num, temp); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_long_and_overflow", - "return value was not set to -1"); - if (overflow != 1) - return raiseTestError("test_long_long_and_overflow", - "overflow was not set to 1"); - - /* Test that overflow is set properly for a large negative value. */ - /* num is a number smaller than LLONG_MIN on a typical platform */ - num = PyLong_FromString("-FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_long_and_overflow", - "return value was not set to -1"); - if (overflow != -1) - return raiseTestError("test_long_long_and_overflow", - "overflow was not set to -1"); - - /* Same again, with num = LLONG_MIN - 1 */ - num = PyLong_FromLongLong(LLONG_MIN); - if (num == NULL) - return NULL; - one = PyLong_FromLong(1L); - if (one == NULL) { - Py_DECREF(num); - return NULL; - } - temp = PyNumber_Subtract(num, one); - Py_DECREF(one); - Py_SETREF(num, temp); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_long_and_overflow", - "return value was not set to -1"); - if (overflow != -1) - return raiseTestError("test_long_long_and_overflow", - "overflow was not set to -1"); - - /* Test that overflow is cleared properly for small values. */ - num = PyLong_FromString("FF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != 0xFF) - return raiseTestError("test_long_long_and_overflow", - "expected return value 0xFF"); - if (overflow != 0) - return raiseTestError("test_long_long_and_overflow", - "overflow was not cleared"); - - num = PyLong_FromString("-FF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -0xFF) - return raiseTestError("test_long_long_and_overflow", - "expected return value 0xFF"); - if (overflow != 0) - return raiseTestError("test_long_long_and_overflow", - "overflow was set incorrectly"); - - num = PyLong_FromLongLong(LLONG_MAX); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != LLONG_MAX) - return raiseTestError("test_long_long_and_overflow", - "expected return value LLONG_MAX"); - if (overflow != 0) - return raiseTestError("test_long_long_and_overflow", - "overflow was not cleared"); - - num = PyLong_FromLongLong(LLONG_MIN); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != LLONG_MIN) - return raiseTestError("test_long_long_and_overflow", - "expected return value LLONG_MIN"); - if (overflow != 0) - return raiseTestError("test_long_long_and_overflow", - "overflow was not cleared"); - - Py_RETURN_NONE; -} - -/*[clinic input] -_testcapi.test_long_as_size_t - -Test the PyLong_As{Size,Ssize}_t API. - -At present this just tests that non-integer arguments are handled correctly. -It should be extended to test overflow handling. -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_as_size_t_impl(PyObject *module) -/*[clinic end generated code: output=f6490ea2b41e6173 input=922990c4a3edfb0d]*/ -{ - size_t out_u; - Py_ssize_t out_s; - - Py_INCREF(Py_None); - - out_u = PyLong_AsSize_t(Py_None); - if (out_u != (size_t)-1 || !PyErr_Occurred()) - return raiseTestError("test_long_as_size_t", - "PyLong_AsSize_t(None) didn't complain"); - if (!PyErr_ExceptionMatches(PyExc_TypeError)) - return raiseTestError("test_long_as_size_t", - "PyLong_AsSize_t(None) raised " - "something other than TypeError"); - PyErr_Clear(); - - out_s = PyLong_AsSsize_t(Py_None); - if (out_s != (Py_ssize_t)-1 || !PyErr_Occurred()) - return raiseTestError("test_long_as_size_t", - "PyLong_AsSsize_t(None) didn't complain"); - if (!PyErr_ExceptionMatches(PyExc_TypeError)) - return raiseTestError("test_long_as_size_t", - "PyLong_AsSsize_t(None) raised " - "something other than TypeError"); - PyErr_Clear(); - - /* Py_INCREF(Py_None) omitted - we already have a reference to it. */ - return Py_None; -} - -/*[clinic input] -_testcapi.test_long_as_unsigned_long_long_mask -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_as_unsigned_long_long_mask_impl(PyObject *module) -/*[clinic end generated code: output=e3e16cd0189440cc input=eb2438493ae7b9af]*/ -{ - unsigned long long res = PyLong_AsUnsignedLongLongMask(NULL); - - if (res != (unsigned long long)-1 || !PyErr_Occurred()) { - return raiseTestError("test_long_as_unsigned_long_long_mask", - "PyLong_AsUnsignedLongLongMask(NULL) didn't " - "complain"); - } - if (!PyErr_ExceptionMatches(PyExc_SystemError)) { - return raiseTestError("test_long_as_unsigned_long_long_mask", - "PyLong_AsUnsignedLongLongMask(NULL) raised " - "something other than SystemError"); - } - PyErr_Clear(); - Py_RETURN_NONE; -} - -/*[clinic input] -_testcapi.test_long_as_double -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_as_double_impl(PyObject *module) -/*[clinic end generated code: output=deca0898e15adde5 input=c77bc88ef5a1de76]*/ -{ - double out; - - Py_INCREF(Py_None); - - out = PyLong_AsDouble(Py_None); - if (out != -1.0 || !PyErr_Occurred()) - return raiseTestError("test_long_as_double", - "PyLong_AsDouble(None) didn't complain"); - if (!PyErr_ExceptionMatches(PyExc_TypeError)) - return raiseTestError("test_long_as_double", - "PyLong_AsDouble(None) raised " - "something other than TypeError"); - PyErr_Clear(); - - /* Py_INCREF(Py_None) omitted - we already have a reference to it. */ - return Py_None; -} - /*[clinic input] _testcapi.call_long_compact_api arg: object @@ -555,48 +32,6 @@ _testcapi_call_long_compact_api(PyObject *module, PyObject *arg) return Py_BuildValue("in", is_compact, value); } -static PyObject * -pylong_check(PyObject *module, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyLong_Check(obj)); -} - -static PyObject * -pylong_checkexact(PyObject *module, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyLong_CheckExact(obj)); -} - -static PyObject * -pylong_fromdouble(PyObject *module, PyObject *arg) -{ - double value; - if (!PyArg_Parse(arg, "d", &value)) { - return NULL; - } - return PyLong_FromDouble(value); -} - -static PyObject * -pylong_fromstring(PyObject *module, PyObject *args) -{ - const char *str; - Py_ssize_t len; - int base; - char *end = UNINITIALIZED_PTR; - if (!PyArg_ParseTuple(args, "z#i", &str, &len, &base)) { - return NULL; - } - - PyObject *result = PyLong_FromString(str, &end, base); - if (result == NULL) { - // XXX 'end' is not always set. - return NULL; - } - return Py_BuildValue("Nn", result, (Py_ssize_t)(end - str)); -} static PyObject * pylong_fromunicodeobject(PyObject *module, PyObject *args) @@ -611,246 +46,235 @@ pylong_fromunicodeobject(PyObject *module, PyObject *args) return PyLong_FromUnicodeObject(unicode, base); } -static PyObject * -pylong_fromvoidptr(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - return PyLong_FromVoidPtr((void *)arg); -} - -/*[clinic input] -_testcapi.PyLong_AsInt - arg: object - / -[clinic start generated code]*/ static PyObject * -_testcapi_PyLong_AsInt(PyObject *module, PyObject *arg) -/*[clinic end generated code: output=0df9f19de5fa575b input=9561b97105493a67]*/ +pylong_asnativebytes(PyObject *module, PyObject *args) { - NULLABLE(arg); - assert(!PyErr_Occurred()); - int value = PyLong_AsInt(arg); - if (value == -1 && PyErr_Occurred()) { + PyObject *v; + Py_buffer buffer; + Py_ssize_t n, flags; + if (!PyArg_ParseTuple(args, "Ow*nn", &v, &buffer, &n, &flags)) { return NULL; } - return PyLong_FromLong(value); -} - -static PyObject * -pylong_aslong(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - long value = PyLong_AsLong(arg); - if (value == -1 && PyErr_Occurred()) { + if (buffer.readonly) { + PyErr_SetString(PyExc_TypeError, "buffer must be writable"); + PyBuffer_Release(&buffer); return NULL; } - return PyLong_FromLong(value); -} - -static PyObject * -pylong_aslongandoverflow(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - int overflow = UNINITIALIZED_INT; - long value = PyLong_AsLongAndOverflow(arg, &overflow); - if (value == -1 && PyErr_Occurred()) { - assert(overflow == -1); + if (buffer.len < n) { + PyErr_SetString(PyExc_ValueError, "buffer must be at least 'n' bytes"); + PyBuffer_Release(&buffer); return NULL; } - return Py_BuildValue("li", value, overflow); + Py_ssize_t res = PyLong_AsNativeBytes(v, buffer.buf, n, (int)flags); + PyBuffer_Release(&buffer); + return res >= 0 ? PyLong_FromSsize_t(res) : NULL; } + static PyObject * -pylong_asunsignedlong(PyObject *module, PyObject *arg) +pylong_fromnativebytes(PyObject *module, PyObject *args) { - NULLABLE(arg); - unsigned long value = PyLong_AsUnsignedLong(arg); - if (value == (unsigned long)-1 && PyErr_Occurred()) { + Py_buffer buffer; + Py_ssize_t n, flags, signed_; + if (!PyArg_ParseTuple(args, "y*nnn", &buffer, &n, &flags, &signed_)) { return NULL; } - return PyLong_FromUnsignedLong(value); -} - -static PyObject * -pylong_asunsignedlongmask(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - unsigned long value = PyLong_AsUnsignedLongMask(arg); - if (value == (unsigned long)-1 && PyErr_Occurred()) { + if (buffer.len < n) { + PyErr_SetString(PyExc_ValueError, "buffer must be at least 'n' bytes"); + PyBuffer_Release(&buffer); return NULL; } - return PyLong_FromUnsignedLong(value); + PyObject *res = signed_ + ? PyLong_FromNativeBytes(buffer.buf, n, (int)flags) + : PyLong_FromUnsignedNativeBytes(buffer.buf, n, (int)flags); + PyBuffer_Release(&buffer); + return res; } + static PyObject * -pylong_aslonglong(PyObject *module, PyObject *arg) +pylong_getsign(PyObject *module, PyObject *arg) { + int sign; NULLABLE(arg); - long long value = PyLong_AsLongLong(arg); - if (value == -1 && PyErr_Occurred()) { + if (PyLong_GetSign(arg, &sign) == -1) { return NULL; } - return PyLong_FromLongLong(value); + return PyLong_FromLong(sign); } + static PyObject * -pylong_aslonglongandoverflow(PyObject *module, PyObject *arg) +pylong_ispositive(PyObject *module, PyObject *arg) { NULLABLE(arg); - int overflow = UNINITIALIZED_INT; - long long value = PyLong_AsLongLongAndOverflow(arg, &overflow); - if (value == -1 && PyErr_Occurred()) { - assert(overflow == -1); - return NULL; - } - return Py_BuildValue("Li", value, overflow); + RETURN_INT(PyLong_IsPositive(arg)); } + static PyObject * -pylong_asunsignedlonglong(PyObject *module, PyObject *arg) +pylong_isnegative(PyObject *module, PyObject *arg) { NULLABLE(arg); - unsigned long long value = PyLong_AsUnsignedLongLong(arg); - if (value == (unsigned long long)-1 && PyErr_Occurred()) { - return NULL; - } - return PyLong_FromUnsignedLongLong(value); + RETURN_INT(PyLong_IsNegative(arg)); } + static PyObject * -pylong_asunsignedlonglongmask(PyObject *module, PyObject *arg) +pylong_iszero(PyObject *module, PyObject *arg) { NULLABLE(arg); - unsigned long long value = PyLong_AsUnsignedLongLongMask(arg); - if (value == (unsigned long long)-1 && PyErr_Occurred()) { - return NULL; - } - return PyLong_FromUnsignedLongLong(value); + RETURN_INT(PyLong_IsZero(arg)); } + static PyObject * -pylong_as_ssize_t(PyObject *module, PyObject *arg) +pylong_aspid(PyObject *module, PyObject *arg) { NULLABLE(arg); - Py_ssize_t value = PyLong_AsSsize_t(arg); + pid_t value = PyLong_AsPid(arg); if (value == -1 && PyErr_Occurred()) { return NULL; } - return PyLong_FromSsize_t(value); + return PyLong_FromPid(value); } + static PyObject * -pylong_as_size_t(PyObject *module, PyObject *arg) +layout_to_dict(const PyLongLayout *layout) { - NULLABLE(arg); - size_t value = PyLong_AsSize_t(arg); - if (value == (size_t)-1 && PyErr_Occurred()) { - return NULL; - } - return PyLong_FromSize_t(value); + return Py_BuildValue("{sisisisi}", + "bits_per_digit", (int)layout->bits_per_digit, + "digit_size", (int)layout->digit_size, + "digits_order", (int)layout->digits_order, + "digit_endianness", (int)layout->digit_endianness); } + static PyObject * -pylong_asdouble(PyObject *module, PyObject *arg) +pylong_export(PyObject *module, PyObject *obj) { - NULLABLE(arg); - double value = PyLong_AsDouble(arg); - if (value == -1.0 && PyErr_Occurred()) { + PyLongExport export_long; + if (PyLong_Export(obj, &export_long) < 0) { return NULL; } - return PyFloat_FromDouble(value); -} -static PyObject * -pylong_asvoidptr(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - void *value = PyLong_AsVoidPtr(arg); - if (value == NULL) { - if (PyErr_Occurred()) { - return NULL; + if (export_long.digits == NULL) { + assert(export_long.negative == 0); + assert(export_long.ndigits == 0); + assert(export_long.digits == NULL); + PyObject *res = PyLong_FromInt64(export_long.value); + PyLong_FreeExport(&export_long); + return res; + } + + assert(PyLong_GetNativeLayout()->digit_size == sizeof(digit)); + const digit *export_long_digits = export_long.digits; + + PyObject *digits = PyList_New(0); + if (digits == NULL) { + goto error; + } + for (Py_ssize_t i = 0; i < export_long.ndigits; i++) { + PyObject *item = PyLong_FromUnsignedLong(export_long_digits[i]); + if (item == NULL) { + goto error; } - Py_RETURN_NONE; + + if (PyList_Append(digits, item) < 0) { + Py_DECREF(item); + goto error; + } + Py_DECREF(item); } - return Py_NewRef((PyObject *)value); + + assert(export_long.value == 0); + PyObject *res = Py_BuildValue("(iN)", export_long.negative, digits); + + PyLong_FreeExport(&export_long); + assert(export_long._reserved == 0); + + return res; + +error: + Py_XDECREF(digits); + PyLong_FreeExport(&export_long); + return NULL; } + static PyObject * -pylong_asnativebytes(PyObject *module, PyObject *args) +pylongwriter_create(PyObject *module, PyObject *args) { - PyObject *v; - Py_buffer buffer; - Py_ssize_t n, endianness; - if (!PyArg_ParseTuple(args, "Ow*nn", &v, &buffer, &n, &endianness)) { + int negative; + PyObject *list; + // TODO(vstinner): write test for negative ndigits and digits==NULL + if (!PyArg_ParseTuple(args, "iO!", &negative, &PyList_Type, &list)) { return NULL; } - if (buffer.readonly) { - PyErr_SetString(PyExc_TypeError, "buffer must be writable"); - PyBuffer_Release(&buffer); - return NULL; + Py_ssize_t ndigits = PyList_GET_SIZE(list); + + digit *digits = PyMem_Malloc((size_t)ndigits * sizeof(digit)); + if (digits == NULL) { + return PyErr_NoMemory(); } - if (buffer.len < n) { - PyErr_SetString(PyExc_ValueError, "buffer must be at least 'n' bytes"); - PyBuffer_Release(&buffer); - return NULL; + + for (Py_ssize_t i = 0; i < ndigits; i++) { + PyObject *item = PyList_GET_ITEM(list, i); + + long num = PyLong_AsLong(item); + if (num == -1 && PyErr_Occurred()) { + goto error; + } + + if (num < 0 || num >= PyLong_BASE) { + PyErr_SetString(PyExc_ValueError, "digit doesn't fit into digit"); + goto error; + } + digits[i] = (digit)num; } - Py_ssize_t res = PyLong_AsNativeBytes(v, buffer.buf, n, (int)endianness); - PyBuffer_Release(&buffer); - return res >= 0 ? PyLong_FromSsize_t(res) : NULL; + + void *writer_digits; + PyLongWriter *writer = PyLongWriter_Create(negative, ndigits, + &writer_digits); + if (writer == NULL) { + goto error; + } + assert(PyLong_GetNativeLayout()->digit_size == sizeof(digit)); + memcpy(writer_digits, digits, (size_t)ndigits * sizeof(digit)); + PyObject *res = PyLongWriter_Finish(writer); + PyMem_Free(digits); + + return res; + +error: + PyMem_Free(digits); + return NULL; } + static PyObject * -pylong_fromnativebytes(PyObject *module, PyObject *args) +get_pylong_layout(PyObject *module, PyObject *Py_UNUSED(args)) { - Py_buffer buffer; - Py_ssize_t n, endianness, signed_; - if (!PyArg_ParseTuple(args, "y*nnn", &buffer, &n, &endianness, &signed_)) { - return NULL; - } - if (buffer.len < n) { - PyErr_SetString(PyExc_ValueError, "buffer must be at least 'n' bytes"); - PyBuffer_Release(&buffer); - return NULL; - } - PyObject *res = signed_ - ? PyLong_FromNativeBytes(buffer.buf, n, (int)endianness) - : PyLong_FromUnsignedNativeBytes(buffer.buf, n, (int)endianness); - PyBuffer_Release(&buffer); - return res; + const PyLongLayout *layout = PyLong_GetNativeLayout(); + return layout_to_dict(layout); } static PyMethodDef test_methods[] = { - _TESTCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF - _TESTCAPI_TEST_LONG_API_METHODDEF - _TESTCAPI_TEST_LONG_AS_DOUBLE_METHODDEF - _TESTCAPI_TEST_LONG_AS_SIZE_T_METHODDEF - _TESTCAPI_TEST_LONG_AS_UNSIGNED_LONG_LONG_MASK_METHODDEF - _TESTCAPI_TEST_LONG_LONG_AND_OVERFLOW_METHODDEF - _TESTCAPI_TEST_LONGLONG_API_METHODDEF _TESTCAPI_CALL_LONG_COMPACT_API_METHODDEF - {"pylong_check", pylong_check, METH_O}, - {"pylong_checkexact", pylong_checkexact, METH_O}, - {"pylong_fromdouble", pylong_fromdouble, METH_O}, - {"pylong_fromstring", pylong_fromstring, METH_VARARGS}, {"pylong_fromunicodeobject", pylong_fromunicodeobject, METH_VARARGS}, - {"pylong_fromvoidptr", pylong_fromvoidptr, METH_O}, - _TESTCAPI_PYLONG_ASINT_METHODDEF - {"pylong_aslong", pylong_aslong, METH_O}, - {"pylong_aslongandoverflow", pylong_aslongandoverflow, METH_O}, - {"pylong_asunsignedlong", pylong_asunsignedlong, METH_O}, - {"pylong_asunsignedlongmask", pylong_asunsignedlongmask, METH_O}, - {"pylong_aslonglong", pylong_aslonglong, METH_O}, - {"pylong_aslonglongandoverflow", pylong_aslonglongandoverflow, METH_O}, - {"pylong_asunsignedlonglong", pylong_asunsignedlonglong, METH_O}, - {"pylong_asunsignedlonglongmask", pylong_asunsignedlonglongmask, METH_O}, - {"pylong_as_ssize_t", pylong_as_ssize_t, METH_O}, - {"pylong_as_size_t", pylong_as_size_t, METH_O}, - {"pylong_asdouble", pylong_asdouble, METH_O}, - {"pylong_asvoidptr", pylong_asvoidptr, METH_O}, {"pylong_asnativebytes", pylong_asnativebytes, METH_VARARGS}, {"pylong_fromnativebytes", pylong_fromnativebytes, METH_VARARGS}, + {"pylong_getsign", pylong_getsign, METH_O}, + {"pylong_aspid", pylong_aspid, METH_O}, + {"pylong_export", pylong_export, METH_O}, + {"pylongwriter_create", pylongwriter_create, METH_VARARGS}, + {"get_pylong_layout", get_pylong_layout, METH_NOARGS}, + {"pylong_ispositive", pylong_ispositive, METH_O}, + {"pylong_isnegative", pylong_isnegative, METH_O}, + {"pylong_iszero", pylong_iszero, METH_O}, {NULL}, }; diff --git a/Modules/_testcapi/mem.c b/Modules/_testcapi/mem.c index ab4ad934644c38..ecae5ba26226a6 100644 --- a/Modules/_testcapi/mem.c +++ b/Modules/_testcapi/mem.c @@ -557,8 +557,9 @@ tracemalloc_untrack(PyObject *self, PyObject *args) { unsigned int domain; PyObject *ptr_obj; + int release_gil = 0; - if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) { + if (!PyArg_ParseTuple(args, "IO|i", &domain, &ptr_obj, &release_gil)) { return NULL; } void *ptr = PyLong_AsVoidPtr(ptr_obj); @@ -566,7 +567,15 @@ tracemalloc_untrack(PyObject *self, PyObject *args) return NULL; } - int res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr); + int res; + if (release_gil) { + Py_BEGIN_ALLOW_THREADS + res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr); + Py_END_ALLOW_THREADS + } + else { + res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr); + } if (res < 0) { PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Untrack error"); return NULL; diff --git a/Modules/_testcapi/monitoring.c b/Modules/_testcapi/monitoring.c new file mode 100644 index 00000000000000..e475e3b5937199 --- /dev/null +++ b/Modules/_testcapi/monitoring.c @@ -0,0 +1,527 @@ +#include "parts.h" +#include "util.h" + +#include "monitoring.h" + +#define Py_BUILD_CORE +#include "internal/pycore_instruments.h" + +typedef struct { + PyObject_HEAD + PyMonitoringState *monitoring_states; + uint64_t version; + int num_events; + /* Other fields */ +} PyCodeLikeObject; + + +static PyObject * +CodeLike_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + int num_events; + if (!PyArg_ParseTuple(args, "i", &num_events)) { + return NULL; + } + PyMonitoringState *states = (PyMonitoringState *)PyMem_Calloc( + num_events, sizeof(PyMonitoringState)); + if (states == NULL) { + return NULL; + } + PyCodeLikeObject *self = (PyCodeLikeObject *) type->tp_alloc(type, 0); + if (self != NULL) { + self->version = 0; + self->monitoring_states = states; + self->num_events = num_events; + } + else { + PyMem_Free(states); + } + return (PyObject *) self; +} + +static void +CodeLike_dealloc(PyCodeLikeObject *self) +{ + if (self->monitoring_states) { + PyMem_Free(self->monitoring_states); + } + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static PyObject * +CodeLike_str(PyCodeLikeObject *self) +{ + PyObject *res = NULL; + PyObject *sep = NULL; + PyObject *parts = NULL; + if (self->monitoring_states) { + parts = PyList_New(0); + if (parts == NULL) { + goto end; + } + + PyObject *heading = PyUnicode_FromString("PyCodeLikeObject"); + if (heading == NULL) { + goto end; + } + int err = PyList_Append(parts, heading); + Py_DECREF(heading); + if (err < 0) { + goto end; + } + + for (int i = 0; i < self->num_events; i++) { + PyObject *part = PyUnicode_FromFormat(" %d", self->monitoring_states[i].active); + if (part == NULL) { + goto end; + } + int err = PyList_Append(parts, part); + Py_XDECREF(part); + if (err < 0) { + goto end; + } + } + sep = PyUnicode_FromString(": "); + if (sep == NULL) { + goto end; + } + res = PyUnicode_Join(sep, parts); + } +end: + Py_XDECREF(sep); + Py_XDECREF(parts); + return res; +} + +static PyTypeObject PyCodeLike_Type = { + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "monitoring.CodeLike", + .tp_doc = PyDoc_STR("CodeLike objects"), + .tp_basicsize = sizeof(PyCodeLikeObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = CodeLike_new, + .tp_dealloc = (destructor) CodeLike_dealloc, + .tp_str = (reprfunc) CodeLike_str, +}; + +#define RAISE_UNLESS_CODELIKE(v) if (!Py_IS_TYPE((v), &PyCodeLike_Type)) { \ + PyErr_Format(PyExc_TypeError, "expected a code-like, got %s", Py_TYPE(v)->tp_name); \ + return NULL; \ + } + +/*******************************************************************/ + +static PyMonitoringState * +setup_fire(PyObject *codelike, int offset, PyObject *exc) +{ + RAISE_UNLESS_CODELIKE(codelike); + PyCodeLikeObject *cl = ((PyCodeLikeObject *)codelike); + assert(offset >= 0 && offset < cl->num_events); + PyMonitoringState *state = &cl->monitoring_states[offset]; + + if (exc != NULL) { + PyErr_SetRaisedException(Py_NewRef(exc)); + } + return state; +} + +static int +teardown_fire(int res, PyMonitoringState *state, PyObject *exception) +{ + if (res == -1) { + return -1; + } + if (exception) { + assert(PyErr_Occurred()); + assert(((PyObject*)Py_TYPE(exception)) == PyErr_Occurred()); + } + + else { + assert(!PyErr_Occurred()); + } + PyErr_Clear(); + return state->active; +} + +static PyObject * +fire_event_py_start(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + if (!PyArg_ParseTuple(args, "Oi", &codelike, &offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyStartEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_resume(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + if (!PyArg_ParseTuple(args, "Oi", &codelike, &offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyResumeEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_return(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *retval; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &retval)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyReturnEvent(state, codelike, offset, retval); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_c_return(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *retval; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &retval)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireCReturnEvent(state, codelike, offset, retval); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_yield(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *retval; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &retval)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyYieldEvent(state, codelike, offset, retval); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_call(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *callable, *arg0; + if (!PyArg_ParseTuple(args, "OiOO", &codelike, &offset, &callable, &arg0)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireCallEvent(state, codelike, offset, callable, arg0); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_line(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset, lineno; + if (!PyArg_ParseTuple(args, "Oii", &codelike, &offset, &lineno)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireLineEvent(state, codelike, offset, lineno); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_jump(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *target_offset; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &target_offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireJumpEvent(state, codelike, offset, target_offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_branch_right(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *target_offset; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &target_offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireBranchRightEvent(state, codelike, offset, target_offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_branch_left(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *target_offset; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &target_offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireBranchLeftEvent(state, codelike, offset, target_offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_throw(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyThrowEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_raise(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireRaiseEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_c_raise(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireCRaiseEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_reraise(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireReraiseEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_exception_handled(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireExceptionHandledEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_unwind(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyUnwindEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_stop_iteration(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *value; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &value)) { + return NULL; + } + NULLABLE(value); + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireStopIterationEvent(state, codelike, offset, value); + RETURN_INT(teardown_fire(res, state, exception)); +} + +/*******************************************************************/ + +static PyObject * +enter_scope(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int event1, event2=0; + Py_ssize_t num_events = PyTuple_Size(args) - 1; + if (num_events == 1) { + if (!PyArg_ParseTuple(args, "Oi", &codelike, &event1)) { + return NULL; + } + } + else { + assert(num_events == 2); + if (!PyArg_ParseTuple(args, "Oii", &codelike, &event1, &event2)) { + return NULL; + } + } + RAISE_UNLESS_CODELIKE(codelike); + PyCodeLikeObject *cl = (PyCodeLikeObject *) codelike; + + uint8_t events[] = { event1, event2 }; + + PyMonitoring_EnterScope(cl->monitoring_states, + &cl->version, + events, + num_events); + + Py_RETURN_NONE; +} + +static PyObject * +exit_scope(PyObject *self, PyObject *args) +{ + PyMonitoring_ExitScope(); + Py_RETURN_NONE; +} + +static PyMethodDef TestMethods[] = { + {"fire_event_py_start", fire_event_py_start, METH_VARARGS}, + {"fire_event_py_resume", fire_event_py_resume, METH_VARARGS}, + {"fire_event_py_return", fire_event_py_return, METH_VARARGS}, + {"fire_event_c_return", fire_event_c_return, METH_VARARGS}, + {"fire_event_py_yield", fire_event_py_yield, METH_VARARGS}, + {"fire_event_call", fire_event_call, METH_VARARGS}, + {"fire_event_line", fire_event_line, METH_VARARGS}, + {"fire_event_jump", fire_event_jump, METH_VARARGS}, + {"fire_event_branch_left", fire_event_branch_left, METH_VARARGS}, + {"fire_event_branch_right", fire_event_branch_right, METH_VARARGS}, + {"fire_event_py_throw", fire_event_py_throw, METH_VARARGS}, + {"fire_event_raise", fire_event_raise, METH_VARARGS}, + {"fire_event_c_raise", fire_event_c_raise, METH_VARARGS}, + {"fire_event_reraise", fire_event_reraise, METH_VARARGS}, + {"fire_event_exception_handled", fire_event_exception_handled, METH_VARARGS}, + {"fire_event_py_unwind", fire_event_py_unwind, METH_VARARGS}, + {"fire_event_stop_iteration", fire_event_stop_iteration, METH_VARARGS}, + {"monitoring_enter_scope", enter_scope, METH_VARARGS}, + {"monitoring_exit_scope", exit_scope, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Monitoring(PyObject *m) +{ + if (PyType_Ready(&PyCodeLike_Type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "CodeLike", (PyObject *) &PyCodeLike_Type) < 0) { + Py_DECREF(m); + return -1; + } + if (PyModule_AddFunctions(m, TestMethods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testcapi/numbers.c b/Modules/_testcapi/numbers.c index 6f7fa3fa7a4186..e16ff73744067a 100644 --- a/Modules/_testcapi/numbers.c +++ b/Modules/_testcapi/numbers.c @@ -1,7 +1,168 @@ #include "parts.h" #include "util.h" + +static PyObject * +number_check(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyNumber_Check(obj)); +} + +#define BINARYFUNC(funcsuffix, methsuffix) \ + static PyObject * \ + number_##methsuffix(PyObject *Py_UNUSED(module), PyObject *args) \ + { \ + PyObject *o1, *o2; \ + \ + if (!PyArg_ParseTuple(args, "OO", &o1, &o2)) { \ + return NULL; \ + } \ + \ + NULLABLE(o1); \ + NULLABLE(o2); \ + return PyNumber_##funcsuffix(o1, o2); \ + }; + +BINARYFUNC(Add, add) +BINARYFUNC(Subtract, subtract) +BINARYFUNC(Multiply, multiply) +BINARYFUNC(MatrixMultiply, matrixmultiply) +BINARYFUNC(FloorDivide, floordivide) +BINARYFUNC(TrueDivide, truedivide) +BINARYFUNC(Remainder, remainder) +BINARYFUNC(Divmod, divmod) + +#define TERNARYFUNC(funcsuffix, methsuffix) \ + static PyObject * \ + number_##methsuffix(PyObject *Py_UNUSED(module), PyObject *args) \ + { \ + PyObject *o1, *o2, *o3 = Py_None; \ + \ + if (!PyArg_ParseTuple(args, "OO|O", &o1, &o2, &o3)) { \ + return NULL; \ + } \ + \ + NULLABLE(o1); \ + NULLABLE(o2); \ + return PyNumber_##funcsuffix(o1, o2, o3); \ + }; + +TERNARYFUNC(Power, power) + +#define UNARYFUNC(funcsuffix, methsuffix) \ + static PyObject * \ + number_##methsuffix(PyObject *Py_UNUSED(module), PyObject *obj) \ + { \ + NULLABLE(obj); \ + return PyNumber_##funcsuffix(obj); \ + }; + +UNARYFUNC(Negative, negative) +UNARYFUNC(Positive, positive) +UNARYFUNC(Absolute, absolute) +UNARYFUNC(Invert, invert) + +BINARYFUNC(Lshift, lshift) +BINARYFUNC(Rshift, rshift) +BINARYFUNC(And, and) +BINARYFUNC(Xor, xor) +BINARYFUNC(Or, or) + +BINARYFUNC(InPlaceAdd, inplaceadd) +BINARYFUNC(InPlaceSubtract, inplacesubtract) +BINARYFUNC(InPlaceMultiply, inplacemultiply) +BINARYFUNC(InPlaceMatrixMultiply, inplacematrixmultiply) +BINARYFUNC(InPlaceFloorDivide, inplacefloordivide) +BINARYFUNC(InPlaceTrueDivide, inplacetruedivide) +BINARYFUNC(InPlaceRemainder, inplaceremainder) + +TERNARYFUNC(InPlacePower, inplacepower) + +BINARYFUNC(InPlaceLshift, inplacelshift) +BINARYFUNC(InPlaceRshift, inplacershift) +BINARYFUNC(InPlaceAnd, inplaceand) +BINARYFUNC(InPlaceXor, inplacexor) +BINARYFUNC(InPlaceOr, inplaceor) + +UNARYFUNC(Long, long) +UNARYFUNC(Float, float) +UNARYFUNC(Index, index) + +static PyObject * +number_tobase(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *n; + int base; + + if (!PyArg_ParseTuple(args, "Oi", &n, &base)) { + return NULL; + } + + NULLABLE(n); + return PyNumber_ToBase(n, base); +} + +static PyObject * +number_asssizet(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *o, *exc; + Py_ssize_t ret; + + if (!PyArg_ParseTuple(args, "OO", &o, &exc)) { + return NULL; + } + + NULLABLE(o); + NULLABLE(exc); + ret = PyNumber_AsSsize_t(o, exc); + + if (ret == (Py_ssize_t)(-1) && PyErr_Occurred()) { + return NULL; + } + + return PyLong_FromSsize_t(ret); +} + + static PyMethodDef test_methods[] = { + {"number_check", number_check, METH_O}, + {"number_add", number_add, METH_VARARGS}, + {"number_subtract", number_subtract, METH_VARARGS}, + {"number_multiply", number_multiply, METH_VARARGS}, + {"number_matrixmultiply", number_matrixmultiply, METH_VARARGS}, + {"number_floordivide", number_floordivide, METH_VARARGS}, + {"number_truedivide", number_truedivide, METH_VARARGS}, + {"number_remainder", number_remainder, METH_VARARGS}, + {"number_divmod", number_divmod, METH_VARARGS}, + {"number_power", number_power, METH_VARARGS}, + {"number_negative", number_negative, METH_O}, + {"number_positive", number_positive, METH_O}, + {"number_absolute", number_absolute, METH_O}, + {"number_invert", number_invert, METH_O}, + {"number_lshift", number_lshift, METH_VARARGS}, + {"number_rshift", number_rshift, METH_VARARGS}, + {"number_and", number_and, METH_VARARGS}, + {"number_xor", number_xor, METH_VARARGS}, + {"number_or", number_or, METH_VARARGS}, + {"number_inplaceadd", number_inplaceadd, METH_VARARGS}, + {"number_inplacesubtract", number_inplacesubtract, METH_VARARGS}, + {"number_inplacemultiply", number_inplacemultiply, METH_VARARGS}, + {"number_inplacematrixmultiply", number_inplacematrixmultiply, METH_VARARGS}, + {"number_inplacefloordivide", number_inplacefloordivide, METH_VARARGS}, + {"number_inplacetruedivide", number_inplacetruedivide, METH_VARARGS}, + {"number_inplaceremainder", number_inplaceremainder, METH_VARARGS}, + {"number_inplacepower", number_inplacepower, METH_VARARGS}, + {"number_inplacelshift", number_inplacelshift, METH_VARARGS}, + {"number_inplacershift", number_inplacershift, METH_VARARGS}, + {"number_inplaceand", number_inplaceand, METH_VARARGS}, + {"number_inplacexor", number_inplacexor, METH_VARARGS}, + {"number_inplaceor", number_inplaceor, METH_VARARGS}, + {"number_long", number_long, METH_O}, + {"number_float", number_float, METH_O}, + {"number_index", number_index, METH_O}, + {"number_tobase", number_tobase, METH_VARARGS}, + {"number_asssizet", number_asssizet, METH_VARARGS}, {NULL}, }; diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c new file mode 100644 index 00000000000000..409b0c83c18cfd --- /dev/null +++ b/Modules/_testcapi/object.c @@ -0,0 +1,207 @@ +#include "parts.h" +#include "util.h" + +static PyObject * +call_pyobject_print(PyObject *self, PyObject * args) +{ + PyObject *object; + PyObject *filename; + PyObject *print_raw; + FILE *fp; + int flags = 0; + + if (!PyArg_UnpackTuple(args, "call_pyobject_print", 3, 3, + &object, &filename, &print_raw)) { + return NULL; + } + + fp = Py_fopen(filename, "w+"); + + if (Py_IsTrue(print_raw)) { + flags = Py_PRINT_RAW; + } + + if (PyObject_Print(object, fp, flags) < 0) { + fclose(fp); + return NULL; + } + + fclose(fp); + + Py_RETURN_NONE; +} + +static PyObject * +pyobject_print_null(PyObject *self, PyObject *args) +{ + PyObject *filename; + FILE *fp; + + if (!PyArg_UnpackTuple(args, "call_pyobject_print", 1, 1, &filename)) { + return NULL; + } + + fp = Py_fopen(filename, "w+"); + + if (PyObject_Print(NULL, fp, 0) < 0) { + fclose(fp); + return NULL; + } + + fclose(fp); + + Py_RETURN_NONE; +} + +static PyObject * +pyobject_print_noref_object(PyObject *self, PyObject *args) +{ + PyObject *test_string; + PyObject *filename; + FILE *fp; + char correct_string[100]; + + test_string = PyUnicode_FromString("Spam spam spam"); + + Py_SET_REFCNT(test_string, 0); + + PyOS_snprintf(correct_string, 100, "", + Py_REFCNT(test_string), (void *)test_string); + + if (!PyArg_UnpackTuple(args, "call_pyobject_print", 1, 1, &filename)) { + return NULL; + } + + fp = Py_fopen(filename, "w+"); + + if (PyObject_Print(test_string, fp, 0) < 0){ + fclose(fp); + Py_SET_REFCNT(test_string, 1); + Py_DECREF(test_string); + return NULL; + } + + fclose(fp); + + Py_SET_REFCNT(test_string, 1); + Py_DECREF(test_string); + + return PyUnicode_FromString(correct_string); +} + +static PyObject * +pyobject_print_os_error(PyObject *self, PyObject *args) +{ + PyObject *test_string; + PyObject *filename; + FILE *fp; + + test_string = PyUnicode_FromString("Spam spam spam"); + + if (!PyArg_UnpackTuple(args, "call_pyobject_print", 1, 1, &filename)) { + return NULL; + } + + // open file in read mode to induce OSError + fp = Py_fopen(filename, "r"); + + if (PyObject_Print(test_string, fp, 0) < 0) { + fclose(fp); + Py_DECREF(test_string); + return NULL; + } + + fclose(fp); + Py_DECREF(test_string); + + Py_RETURN_NONE; +} + +static PyObject * +pyobject_clear_weakrefs_no_callbacks(PyObject *self, PyObject *obj) +{ + PyUnstable_Object_ClearWeakRefsNoCallbacks(obj); + Py_RETURN_NONE; +} + +static PyObject * +pyobject_enable_deferred_refcount(PyObject *self, PyObject *obj) +{ + int result = PyUnstable_Object_EnableDeferredRefcount(obj); + return PyLong_FromLong(result); +} + +static int MyObject_dealloc_called = 0; + +static void +MyObject_dealloc(PyObject *op) +{ + // PyUnstable_TryIncRef should return 0 if object is being deallocated + assert(Py_REFCNT(op) == 0); + assert(!PyUnstable_TryIncRef(op)); + assert(Py_REFCNT(op) == 0); + + MyObject_dealloc_called++; + Py_TYPE(op)->tp_free(op); +} + +static PyTypeObject MyType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "MyType", + .tp_basicsize = sizeof(PyObject), + .tp_dealloc = MyObject_dealloc, +}; + +static PyObject * +test_py_try_inc_ref(PyObject *self, PyObject *unused) +{ + if (PyType_Ready(&MyType) < 0) { + return NULL; + } + + MyObject_dealloc_called = 0; + + PyObject *op = PyObject_New(PyObject, &MyType); + if (op == NULL) { + return NULL; + } + + PyUnstable_EnableTryIncRef(op); +#ifdef Py_GIL_DISABLED + // PyUnstable_EnableTryIncRef sets the shared flags to + // `_Py_REF_MAYBE_WEAKREF` if the flags are currently zero to ensure that + // the shared reference count is merged on deallocation. + assert((op->ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) >= _Py_REF_MAYBE_WEAKREF); +#endif + + if (!PyUnstable_TryIncRef(op)) { + PyErr_SetString(PyExc_AssertionError, "PyUnstable_TryIncRef failed"); + Py_DECREF(op); + return NULL; + } + Py_DECREF(op); // undo try-incref + Py_DECREF(op); // dealloc + assert(MyObject_dealloc_called == 1); + Py_RETURN_NONE; +} + +static PyMethodDef test_methods[] = { + {"call_pyobject_print", call_pyobject_print, METH_VARARGS}, + {"pyobject_print_null", pyobject_print_null, METH_VARARGS}, + {"pyobject_print_noref_object", pyobject_print_noref_object, METH_VARARGS}, + {"pyobject_print_os_error", pyobject_print_os_error, METH_VARARGS}, + {"pyobject_clear_weakrefs_no_callbacks", pyobject_clear_weakrefs_no_callbacks, METH_O}, + {"pyobject_enable_deferred_refcount", pyobject_enable_deferred_refcount, METH_O}, + {"test_py_try_inc_ref", test_py_try_inc_ref, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Object(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index e8cfb2423500d4..65ba77596c760e 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -31,7 +31,6 @@ int _PyTestCapi_Init_Vectorcall(PyObject *module); int _PyTestCapi_Init_Heaptype(PyObject *module); int _PyTestCapi_Init_Abstract(PyObject *module); -int _PyTestCapi_Init_ByteArray(PyObject *module); int _PyTestCapi_Init_Bytes(PyObject *module); int _PyTestCapi_Init_Unicode(PyObject *module); int _PyTestCapi_Init_GetArgs(PyObject *module); @@ -52,16 +51,15 @@ int _PyTestCapi_Init_Exceptions(PyObject *module); int _PyTestCapi_Init_Code(PyObject *module); int _PyTestCapi_Init_Buffer(PyObject *module); int _PyTestCapi_Init_PyAtomic(PyObject *module); -int _PyTestCapi_Init_PyOS(PyObject *module); +int _PyTestCapi_Init_Run(PyObject *module); int _PyTestCapi_Init_File(PyObject *module); int _PyTestCapi_Init_Codec(PyObject *module); int _PyTestCapi_Init_Immortal(PyObject *module); int _PyTestCapi_Init_GC(PyObject *module); -int _PyTestCapi_Init_Sys(PyObject *module); int _PyTestCapi_Init_Hash(PyObject *module); int _PyTestCapi_Init_Time(PyObject *module); - -int _PyTestCapi_Init_VectorcallLimited(PyObject *module); -int _PyTestCapi_Init_HeaptypeRelative(PyObject *module); +int _PyTestCapi_Init_Monitoring(PyObject *module); +int _PyTestCapi_Init_Object(PyObject *module); +int _PyTestCapi_Init_Config(PyObject *mod); #endif // Py_TESTCAPI_PARTS_H diff --git a/Modules/_testcapi/pyatomic.c b/Modules/_testcapi/pyatomic.c index 4f72844535ebd6..850de6f9c3366b 100644 --- a/Modules/_testcapi/pyatomic.c +++ b/Modules/_testcapi/pyatomic.c @@ -125,6 +125,7 @@ test_atomic_fences(PyObject *self, PyObject *obj) { // Just make sure that the fences compile. We are not // testing any synchronizing ordering. _Py_atomic_fence_seq_cst(); + _Py_atomic_fence_acquire(); _Py_atomic_fence_release(); Py_RETURN_NONE; } diff --git a/Modules/_testcapi/run.c b/Modules/_testcapi/run.c new file mode 100644 index 00000000000000..c9db7ccdd965a2 --- /dev/null +++ b/Modules/_testcapi/run.c @@ -0,0 +1,113 @@ +#define PYTESTCAPI_NEED_INTERNAL_API +#include "parts.h" +#include "util.h" +#include "pycore_fileutils.h" // _Py_IsValidFD() + +#include +#include + + +static PyObject * +run_stringflags(PyObject *mod, PyObject *pos_args) +{ + const char *str; + Py_ssize_t size; + int start; + PyObject *globals = NULL; + PyObject *locals = NULL; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + int cf_feature_version = 0; + + if (!PyArg_ParseTuple(pos_args, "z#iO|Oii", + &str, &size, &start, &globals, &locals, + &cf_flags, &cf_feature_version)) { + return NULL; + } + + NULLABLE(globals); + NULLABLE(locals); + if (cf_flags || cf_feature_version) { + flags.cf_flags = cf_flags; + flags.cf_feature_version = cf_feature_version; + pflags = &flags; + } + + return PyRun_StringFlags(str, start, globals, locals, pflags); +} + +static PyObject * +run_fileexflags(PyObject *mod, PyObject *pos_args) +{ + PyObject *result = NULL; + const char *filename = NULL; + Py_ssize_t filename_size; + int start; + PyObject *globals = NULL; + PyObject *locals = NULL; + int closeit = 0; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + int cf_feature_version = 0; + + FILE *fp = NULL; + + if (!PyArg_ParseTuple(pos_args, "z#iO|Oiii", + &filename, &filename_size, &start, &globals, &locals, + &closeit, &cf_flags, &cf_feature_version)) { + return NULL; + } + + NULLABLE(globals); + NULLABLE(locals); + if (cf_flags || cf_feature_version) { + flags.cf_flags = cf_flags; + flags.cf_feature_version = cf_feature_version; + pflags = &flags; + } + + fp = fopen(filename, "r"); + if (fp == NULL) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename); + return NULL; + } + int fd = fileno(fp); + + result = PyRun_FileExFlags(fp, filename, start, globals, locals, closeit, pflags); + + if (closeit && result && _Py_IsValidFD(fd)) { + PyErr_SetString(PyExc_AssertionError, "File was not closed after execution"); + Py_DECREF(result); + fclose(fp); + return NULL; + } + + if (!closeit && !_Py_IsValidFD(fd)) { + PyErr_SetString(PyExc_AssertionError, "Bad file descriptor after execution"); + Py_XDECREF(result); + return NULL; + } + + if (!closeit) { + fclose(fp); /* don't need open file any more*/ + } + + return result; +} + +static PyMethodDef test_methods[] = { + {"run_stringflags", run_stringflags, METH_VARARGS}, + {"run_fileexflags", run_fileexflags, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Run(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testcapi/set.c b/Modules/_testcapi/set.c index 2fbd0aeffcd9f9..31b52cee5e9623 100644 --- a/Modules/_testcapi/set.c +++ b/Modules/_testcapi/set.c @@ -1,75 +1,6 @@ #include "parts.h" #include "util.h" -static PyObject * -set_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PySet_Check(obj)); -} - -static PyObject * -set_checkexact(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PySet_CheckExact(obj)); -} - -static PyObject * -frozenset_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyFrozenSet_Check(obj)); -} - -static PyObject * -frozenset_checkexact(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyFrozenSet_CheckExact(obj)); -} - -static PyObject * -anyset_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyAnySet_Check(obj)); -} - -static PyObject * -anyset_checkexact(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyAnySet_CheckExact(obj)); -} - -static PyObject * -set_new(PyObject *self, PyObject *args) -{ - PyObject *iterable = NULL; - if (!PyArg_ParseTuple(args, "|O", &iterable)) { - return NULL; - } - return PySet_New(iterable); -} - -static PyObject * -frozenset_new(PyObject *self, PyObject *args) -{ - PyObject *iterable = NULL; - if (!PyArg_ParseTuple(args, "|O", &iterable)) { - return NULL; - } - return PyFrozenSet_New(iterable); -} - -static PyObject * -set_size(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PySet_Size(obj)); -} - static PyObject * set_get_size(PyObject *self, PyObject *obj) { @@ -77,111 +8,8 @@ set_get_size(PyObject *self, PyObject *obj) RETURN_SIZE(PySet_GET_SIZE(obj)); } -static PyObject * -set_contains(PyObject *self, PyObject *args) -{ - PyObject *obj, *item; - if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(item); - RETURN_INT(PySet_Contains(obj, item)); -} - -static PyObject * -set_add(PyObject *self, PyObject *args) -{ - PyObject *obj, *item; - if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(item); - RETURN_INT(PySet_Add(obj, item)); -} - -static PyObject * -set_discard(PyObject *self, PyObject *args) -{ - PyObject *obj, *item; - if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(item); - RETURN_INT(PySet_Discard(obj, item)); -} - -static PyObject * -set_pop(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PySet_Pop(obj); -} - -static PyObject * -set_clear(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PySet_Clear(obj)); -} - -static PyObject * -test_frozenset_add_in_capi(PyObject *self, PyObject *Py_UNUSED(obj)) -{ - // Test that `frozenset` can be used with `PySet_Add`, - // when frozenset is just created in CAPI. - PyObject *fs = PyFrozenSet_New(NULL); - if (fs == NULL) { - return NULL; - } - PyObject *num = PyLong_FromLong(1); - if (num == NULL) { - goto error; - } - if (PySet_Add(fs, num) < 0) { - goto error; - } - int contains = PySet_Contains(fs, num); - if (contains < 0) { - goto error; - } - else if (contains == 0) { - goto unexpected; - } - Py_DECREF(fs); - Py_DECREF(num); - Py_RETURN_NONE; - -unexpected: - PyErr_SetString(PyExc_ValueError, "set does not contain expected value"); -error: - Py_DECREF(fs); - Py_XDECREF(num); - return NULL; -} - static PyMethodDef test_methods[] = { - {"set_check", set_check, METH_O}, - {"set_checkexact", set_checkexact, METH_O}, - {"frozenset_check", frozenset_check, METH_O}, - {"frozenset_checkexact", frozenset_checkexact, METH_O}, - {"anyset_check", anyset_check, METH_O}, - {"anyset_checkexact", anyset_checkexact, METH_O}, - - {"set_new", set_new, METH_VARARGS}, - {"frozenset_new", frozenset_new, METH_VARARGS}, - - {"set_size", set_size, METH_O}, {"set_get_size", set_get_size, METH_O}, - {"set_contains", set_contains, METH_VARARGS}, - {"set_add", set_add, METH_VARARGS}, - {"set_discard", set_discard, METH_VARARGS}, - {"set_pop", set_pop, METH_O}, - {"set_clear", set_clear, METH_O}, - - {"test_frozenset_add_in_capi", test_frozenset_add_in_capi, METH_NOARGS}, {NULL}, }; diff --git a/Modules/_testcapi/structmember.c b/Modules/_testcapi/structmember.c index 096eaecd40855f..ef30a5a9944e3c 100644 --- a/Modules/_testcapi/structmember.c +++ b/Modules/_testcapi/structmember.c @@ -22,6 +22,7 @@ typedef struct { char inplace_member[6]; long long longlong_member; unsigned long long ulonglong_member; + char char_member; } all_structmembers; typedef struct { @@ -46,6 +47,7 @@ static struct PyMemberDef test_members_newapi[] = { {"T_STRING_INPLACE", Py_T_STRING_INPLACE, offsetof(test_structmembers, structmembers.inplace_member), 0, NULL}, {"T_LONGLONG", Py_T_LONGLONG, offsetof(test_structmembers, structmembers.longlong_member), 0, NULL}, {"T_ULONGLONG", Py_T_ULONGLONG, offsetof(test_structmembers, structmembers.ulonglong_member), 0, NULL}, + {"T_CHAR", Py_T_CHAR, offsetof(test_structmembers, structmembers.char_member), 0, NULL}, {NULL} }; @@ -56,9 +58,9 @@ test_structmembers_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) "T_BOOL", "T_BYTE", "T_UBYTE", "T_SHORT", "T_USHORT", "T_INT", "T_UINT", "T_LONG", "T_ULONG", "T_PYSSIZET", "T_FLOAT", "T_DOUBLE", "T_STRING_INPLACE", - "T_LONGLONG", "T_ULONGLONG", + "T_LONGLONG", "T_ULONGLONG", "T_CHAR", NULL}; - static const char fmt[] = "|bbBhHiIlknfds#LK"; + static const char fmt[] = "|bbBhHiIlknfds#LKc"; test_structmembers *ob; const char *s = NULL; Py_ssize_t string_len = 0; @@ -82,7 +84,8 @@ test_structmembers_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) &ob->structmembers.double_member, &s, &string_len, &ob->structmembers.longlong_member, - &ob->structmembers.ulonglong_member)) + &ob->structmembers.ulonglong_member, + &ob->structmembers.char_member)) { Py_DECREF(ob); return NULL; @@ -132,6 +135,7 @@ static struct PyMemberDef test_members[] = { {"T_STRING_INPLACE", T_STRING_INPLACE, offsetof(test_structmembers, structmembers.inplace_member), 0, NULL}, {"T_LONGLONG", T_LONGLONG, offsetof(test_structmembers, structmembers.longlong_member), 0, NULL}, {"T_ULONGLONG", T_ULONGLONG, offsetof(test_structmembers, structmembers.ulonglong_member), 0, NULL}, + {"T_CHAR", T_CHAR, offsetof(test_structmembers, structmembers.char_member), 0, NULL}, {NULL} }; diff --git a/Modules/_testcapi/time.c b/Modules/_testcapi/time.c index 68f082bf3f3d88..464cf5c3125012 100644 --- a/Modules/_testcapi/time.c +++ b/Modules/_testcapi/time.c @@ -51,6 +51,25 @@ test_pytime_monotonic(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) PyTime_t t; int res = PyTime_Monotonic(&t); if (res < 0) { + assert(t == 0); + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyObject* +test_pytime_monotonic_raw(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res; + Py_BEGIN_ALLOW_THREADS + res = PyTime_MonotonicRaw(&t); + Py_END_ALLOW_THREADS + if (res < 0) { + assert(t == 0); + PyErr_SetString(PyExc_RuntimeError, "PyTime_MonotonicRaw() failed"); return NULL; } assert(res == 0); @@ -64,6 +83,25 @@ test_pytime_perf_counter(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) PyTime_t t; int res = PyTime_PerfCounter(&t); if (res < 0) { + assert(t == 0); + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyObject* +test_pytime_perf_counter_raw(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res; + Py_BEGIN_ALLOW_THREADS + res = PyTime_PerfCounterRaw(&t); + Py_END_ALLOW_THREADS + if (res < 0) { + assert(t == 0); + PyErr_SetString(PyExc_RuntimeError, "PyTime_PerfCounterRaw() failed"); return NULL; } assert(res == 0); @@ -77,6 +115,25 @@ test_pytime_time(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) PyTime_t t; int res = PyTime_Time(&t); if (res < 0) { + assert(t == 0); + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyObject* +test_pytime_time_raw(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res; + Py_BEGIN_ALLOW_THREADS + res = PyTime_TimeRaw(&t); + Py_END_ALLOW_THREADS + if (res < 0) { + assert(t == 0); + PyErr_SetString(PyExc_RuntimeError, "PyTime_TimeRaw() failed"); return NULL; } assert(res == 0); @@ -87,8 +144,11 @@ test_pytime_time(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) static PyMethodDef test_methods[] = { {"PyTime_AsSecondsDouble", test_pytime_assecondsdouble, METH_VARARGS}, {"PyTime_Monotonic", test_pytime_monotonic, METH_NOARGS}, + {"PyTime_MonotonicRaw", test_pytime_monotonic_raw, METH_NOARGS}, {"PyTime_PerfCounter", test_pytime_perf_counter, METH_NOARGS}, + {"PyTime_PerfCounterRaw", test_pytime_perf_counter_raw, METH_NOARGS}, {"PyTime_Time", test_pytime_time, METH_NOARGS}, + {"PyTime_TimeRaw", test_pytime_time_raw, METH_NOARGS}, {NULL}, }; diff --git a/Modules/_testcapi/tuple.c b/Modules/_testcapi/tuple.c index 95dde8c0edadbe..d9c02ba0ff04fe 100644 --- a/Modules/_testcapi/tuple.c +++ b/Modules/_testcapi/tuple.c @@ -2,14 +2,121 @@ #include "util.h" +static PyObject * +tuple_get_size(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyTuple_GET_SIZE(obj)); +} + +static PyObject * +tuple_get_item(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &obj, &i)) { + return NULL; + } + NULLABLE(obj); + return Py_XNewRef(PyTuple_GET_ITEM(obj, i)); +} + +static PyObject * +tuple_copy(PyObject *tuple) +{ + Py_ssize_t size = PyTuple_GET_SIZE(tuple); + PyObject *newtuple = PyTuple_New(size); + if (!newtuple) { + return NULL; + } + for (Py_ssize_t n = 0; n < size; n++) { + PyTuple_SET_ITEM(newtuple, n, Py_XNewRef(PyTuple_GET_ITEM(tuple, n))); + } + return newtuple; +} + +static PyObject * +tuple_set_item(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value, *newtuple; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "OnO", &obj, &i, &value)) { + return NULL; + } + NULLABLE(value); + if (PyTuple_CheckExact(obj)) { + newtuple = tuple_copy(obj); + if (!newtuple) { + return NULL; + } + + PyObject *val = PyTuple_GET_ITEM(newtuple, i); + PyTuple_SET_ITEM(newtuple, i, Py_XNewRef(value)); + Py_DECREF(val); + return newtuple; + } + else { + NULLABLE(obj); + + PyObject *val = PyTuple_GET_ITEM(obj, i); + PyTuple_SET_ITEM(obj, i, Py_XNewRef(value)); + Py_DECREF(val); + return Py_XNewRef(obj); + } +} + +static PyObject * +_tuple_resize(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *tup; + Py_ssize_t newsize; + int new = 1; + if (!PyArg_ParseTuple(args, "On|p", &tup, &newsize, &new)) { + return NULL; + } + if (new) { + tup = tuple_copy(tup); + if (!tup) { + return NULL; + } + } + else { + NULLABLE(tup); + Py_XINCREF(tup); + } + int r = _PyTuple_Resize(&tup, newsize); + if (r == -1) { + assert(tup == NULL); + return NULL; + } + return tup; +} + +static PyObject * +_check_tuple_item_is_NULL(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &obj, &i)) { + return NULL; + } + return PyLong_FromLong(PyTuple_GET_ITEM(obj, i) == NULL); +} + + static PyMethodDef test_methods[] = { + {"tuple_get_size", tuple_get_size, METH_O}, + {"tuple_get_item", tuple_get_item, METH_VARARGS}, + {"tuple_set_item", tuple_set_item, METH_VARARGS}, + {"_tuple_resize", _tuple_resize, METH_VARARGS}, + {"_check_tuple_item_is_NULL", _check_tuple_item_is_NULL, METH_VARARGS}, {NULL}, }; int _PyTestCapi_Init_Tuple(PyObject *m) { - if (PyModule_AddFunctions(m, test_methods) < 0){ + if (PyModule_AddFunctions(m, test_methods) < 0) { return -1; } diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index a10183dddeca98..b8ecf53f4f8b9c 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -3,103 +3,29 @@ #include "parts.h" #include "util.h" -static struct PyModuleDef *_testcapimodule = NULL; // set at initialization - -static PyObject * -codec_incrementalencoder(PyObject *self, PyObject *args) -{ - const char *encoding, *errors = NULL; - if (!PyArg_ParseTuple(args, "s|s:test_incrementalencoder", - &encoding, &errors)) - return NULL; - return PyCodec_IncrementalEncoder(encoding, errors); -} - -static PyObject * -codec_incrementaldecoder(PyObject *self, PyObject *args) -{ - const char *encoding, *errors = NULL; - if (!PyArg_ParseTuple(args, "s|s:test_incrementaldecoder", - &encoding, &errors)) - return NULL; - return PyCodec_IncrementalDecoder(encoding, errors); -} - -static PyObject * -test_unicode_compare_with_ascii(PyObject *self, PyObject *Py_UNUSED(ignored)) { - PyObject *py_s = PyUnicode_FromStringAndSize("str\0", 4); - int result; - if (py_s == NULL) - return NULL; - result = PyUnicode_CompareWithASCIIString(py_s, "str"); - Py_DECREF(py_s); - if (!result) { - PyErr_SetString(PyExc_AssertionError, "Python string ending in NULL " - "should not compare equal to c string."); - return NULL; - } - Py_RETURN_NONE; -} - +/* Test PyUnicode_New() */ static PyObject * -test_widechar(PyObject *self, PyObject *Py_UNUSED(ignored)) +unicode_new(PyObject *self, PyObject *args) { -#if defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4) - const wchar_t wtext[2] = {(wchar_t)0x10ABCDu}; - size_t wtextlen = 1; - const wchar_t invalid[1] = {(wchar_t)0x110000u}; -#else - const wchar_t wtext[3] = {(wchar_t)0xDBEAu, (wchar_t)0xDFCDu}; - size_t wtextlen = 2; -#endif - PyObject *wide, *utf8; - - wide = PyUnicode_FromWideChar(wtext, wtextlen); - if (wide == NULL) - return NULL; + Py_ssize_t size; + unsigned int maxchar; + PyObject *result; - utf8 = PyUnicode_FromString("\xf4\x8a\xaf\x8d"); - if (utf8 == NULL) { - Py_DECREF(wide); + if (!PyArg_ParseTuple(args, "nI", &size, &maxchar)) { return NULL; } - if (PyUnicode_GET_LENGTH(wide) != PyUnicode_GET_LENGTH(utf8)) { - Py_DECREF(wide); - Py_DECREF(utf8); - PyErr_SetString(PyExc_AssertionError, - "test_widechar: " - "wide string and utf8 string " - "have different length"); - return NULL; - } - if (PyUnicode_Compare(wide, utf8)) { - Py_DECREF(wide); - Py_DECREF(utf8); - if (PyErr_Occurred()) - return NULL; - PyErr_SetString(PyExc_AssertionError, - "test_widechar: " - "wide string and utf8 string " - "are different"); + result = PyUnicode_New(size, (Py_UCS4)maxchar); + if (!result) { return NULL; } - - Py_DECREF(wide); - Py_DECREF(utf8); - -#if defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4) - wide = PyUnicode_FromWideChar(invalid, 1); - if (wide == NULL) - PyErr_Clear(); - else { - PyErr_SetString(PyExc_AssertionError, - "test_widechar: " - "PyUnicode_FromWideChar(L\"\\U00110000\", 1) didn't fail"); + if (size > 0 && maxchar <= 0x10ffff && + PyUnicode_Fill(result, 0, size, (Py_UCS4)maxchar) < 0) + { + Py_DECREF(result); return NULL; } -#endif - Py_RETURN_NONE; + return result; } @@ -130,30 +56,6 @@ unicode_copy(PyObject *unicode) return copy; } -/* Test PyUnicode_New() */ -static PyObject * -unicode_new(PyObject *self, PyObject *args) -{ - Py_ssize_t size; - unsigned int maxchar; - PyObject *result; - - if (!PyArg_ParseTuple(args, "nI", &size, &maxchar)) { - return NULL; - } - - result = PyUnicode_New(size, (Py_UCS4)maxchar); - if (!result) { - return NULL; - } - if (size > 0 && maxchar <= 0x10ffff && - PyUnicode_Fill(result, 0, size, (Py_UCS4)maxchar) < 0) - { - Py_DECREF(result); - return NULL; - } - return result; -} /* Test PyUnicode_Fill() */ static PyObject * @@ -180,1929 +82,490 @@ unicode_fill(PyObject *self, PyObject *args) return Py_BuildValue("(Nn)", to_copy, filled); } -/* Test PyUnicode_WriteChar() */ + +/* Test PyUnicode_FromKindAndData() */ static PyObject * -unicode_writechar(PyObject *self, PyObject *args) +unicode_fromkindanddata(PyObject *self, PyObject *args) { - PyObject *to, *to_copy; - Py_ssize_t index; - unsigned int character; - int result; + int kind; + void *buffer; + Py_ssize_t bsize; + Py_ssize_t size = -100; - if (!PyArg_ParseTuple(args, "OnI", &to, &index, &character)) { + if (!PyArg_ParseTuple(args, "iz#|n", &kind, &buffer, &bsize, &size)) { return NULL; } - NULLABLE(to); - if (!(to_copy = unicode_copy(to)) && to) { - return NULL; + if (size == -100) { + size = bsize; } - - result = PyUnicode_WriteChar(to_copy, index, (Py_UCS4)character); - if (result == -1 && PyErr_Occurred()) { - Py_DECREF(to_copy); + if (kind && size % kind) { + PyErr_SetString(PyExc_AssertionError, + "invalid size in unicode_fromkindanddata()"); return NULL; } - return Py_BuildValue("(Ni)", to_copy, result); + return PyUnicode_FromKindAndData(kind, buffer, kind ? size / kind : 0); } -/* Test PyUnicode_Resize() */ + +// Test PyUnicode_AsUCS4(). +// Part of the limited C API, but the test needs PyUnicode_FromKindAndData(). static PyObject * -unicode_resize(PyObject *self, PyObject *args) +unicode_asucs4(PyObject *self, PyObject *args) { - PyObject *obj, *copy; - Py_ssize_t length; - int result; + PyObject *unicode, *result; + Py_UCS4 *buffer; + int copy_null; + Py_ssize_t str_len, buf_len; - if (!PyArg_ParseTuple(args, "On", &obj, &length)) { + if (!PyArg_ParseTuple(args, "Onp:unicode_asucs4", &unicode, &str_len, ©_null)) { return NULL; } - NULLABLE(obj); - if (!(copy = unicode_copy(obj)) && obj) { - return NULL; + NULLABLE(unicode); + buf_len = str_len + 1; + buffer = PyMem_NEW(Py_UCS4, buf_len); + if (buffer == NULL) { + return PyErr_NoMemory(); } - result = PyUnicode_Resize(©, length); - if (result == -1 && PyErr_Occurred()) { - Py_XDECREF(copy); + memset(buffer, 0, sizeof(Py_UCS4)*buf_len); + buffer[str_len] = 0xffffU; + + if (!PyUnicode_AsUCS4(unicode, buffer, buf_len, copy_null)) { + PyMem_Free(buffer); return NULL; } - if (obj && PyUnicode_Check(obj) && length > PyUnicode_GET_LENGTH(obj)) { - if (PyUnicode_Fill(copy, PyUnicode_GET_LENGTH(obj), length, 0U) < 0) { - Py_DECREF(copy); - return NULL; - } - } - return Py_BuildValue("(Ni)", copy, result); + + result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buffer, buf_len); + PyMem_Free(buffer); + return result; } -/* Test PyUnicode_Append() */ + +// Test PyUnicode_AsUCS4Copy(). +// Part of the limited C API, but the test needs PyUnicode_FromKindAndData(). static PyObject * -unicode_append(PyObject *self, PyObject *args) +unicode_asucs4copy(PyObject *self, PyObject *args) { - PyObject *left, *right, *left_copy; + PyObject *unicode; + Py_UCS4 *buffer; + PyObject *result; - if (!PyArg_ParseTuple(args, "OO", &left, &right)) + if (!PyArg_ParseTuple(args, "O", &unicode)) { return NULL; + } - NULLABLE(left); - NULLABLE(right); - if (!(left_copy = unicode_copy(left)) && left) { + NULLABLE(unicode); + buffer = PyUnicode_AsUCS4Copy(unicode); + if (buffer == NULL) { return NULL; } - PyUnicode_Append(&left_copy, right); - return left_copy; + result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buffer, + PyUnicode_GET_LENGTH(unicode) + 1); + PyMem_FREE(buffer); + return result; } -/* Test PyUnicode_AppendAndDel() */ + +/* Test PyUnicode_AsUTF8() */ static PyObject * -unicode_appendanddel(PyObject *self, PyObject *args) +unicode_asutf8(PyObject *self, PyObject *args) { - PyObject *left, *right, *left_copy; + PyObject *unicode; + Py_ssize_t buflen; + const char *s; - if (!PyArg_ParseTuple(args, "OO", &left, &right)) + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) return NULL; - NULLABLE(left); - NULLABLE(right); - if (!(left_copy = unicode_copy(left)) && left) { + NULLABLE(unicode); + s = PyUnicode_AsUTF8(unicode); + if (s == NULL) return NULL; - } - Py_XINCREF(right); - PyUnicode_AppendAndDel(&left_copy, right); - return left_copy; + + return PyBytes_FromStringAndSize(s, buflen); } -/* Test PyUnicode_FromStringAndSize() */ + +/* Test PyUnicode_CopyCharacters() */ static PyObject * -unicode_fromstringandsize(PyObject *self, PyObject *args) +unicode_copycharacters(PyObject *self, PyObject *args) { - const char *s; - Py_ssize_t bsize; - Py_ssize_t size = -100; + PyObject *from, *to, *to_copy; + Py_ssize_t from_start, to_start, how_many, copied; - if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { + if (!PyArg_ParseTuple(args, "UnOnn", &to, &to_start, + &from, &from_start, &how_many)) { return NULL; } - if (size == -100) { - size = bsize; + NULLABLE(from); + if (!(to_copy = PyUnicode_New(PyUnicode_GET_LENGTH(to), + PyUnicode_MAX_CHAR_VALUE(to)))) { + return NULL; + } + if (PyUnicode_Fill(to_copy, 0, PyUnicode_GET_LENGTH(to_copy), 0U) < 0) { + Py_DECREF(to_copy); + return NULL; + } + + copied = PyUnicode_CopyCharacters(to_copy, to_start, from, + from_start, how_many); + if (copied == -1 && PyErr_Occurred()) { + Py_DECREF(to_copy); + return NULL; } - return PyUnicode_FromStringAndSize(s, size); + + return Py_BuildValue("(Nn)", to_copy, copied); } -/* Test PyUnicode_FromString() */ + +// --- PyUnicodeWriter type ------------------------------------------------- + +typedef struct { + PyObject_HEAD + PyUnicodeWriter *writer; +} WriterObject; + + static PyObject * -unicode_fromstring(PyObject *self, PyObject *arg) +writer_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - const char *s; - Py_ssize_t size; - - if (!PyArg_Parse(arg, "z#", &s, &size)) { + WriterObject *self = (WriterObject *)type->tp_alloc(type, 0); + if (!self) { return NULL; } - return PyUnicode_FromString(s); + self->writer = NULL; + return (PyObject*)self; } -/* Test PyUnicode_FromKindAndData() */ -static PyObject * -unicode_fromkindanddata(PyObject *self, PyObject *args) + +static int +writer_init(PyObject *self_raw, PyObject *args, PyObject *kwargs) { - int kind; - void *buffer; - Py_ssize_t bsize; - Py_ssize_t size = -100; + WriterObject *self = (WriterObject *)self_raw; - if (!PyArg_ParseTuple(args, "iz#|n", &kind, &buffer, &bsize, &size)) { - return NULL; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "n", &size)) { + return -1; } - if (size == -100) { - size = bsize; + if (self->writer) { + PyUnicodeWriter_Discard(self->writer); } - if (kind && size % kind) { - PyErr_SetString(PyExc_AssertionError, - "invalid size in unicode_fromkindanddata()"); - return NULL; + + self->writer = PyUnicodeWriter_Create(size); + if (self->writer == NULL) { + return -1; } - return PyUnicode_FromKindAndData(kind, buffer, kind ? size / kind : 0); + return 0; } -/* Test PyUnicode_Substring() */ -static PyObject * -unicode_substring(PyObject *self, PyObject *args) -{ - PyObject *str; - Py_ssize_t start, end; - if (!PyArg_ParseTuple(args, "Onn", &str, &start, &end)) { - return NULL; +static void +writer_dealloc(PyObject *self_raw) +{ + WriterObject *self = (WriterObject *)self_raw; + PyTypeObject *tp = Py_TYPE(self); + if (self->writer) { + PyUnicodeWriter_Discard(self->writer); } - - NULLABLE(str); - return PyUnicode_Substring(str, start, end); + tp->tp_free(self); + Py_DECREF(tp); } -/* Test PyUnicode_GetLength() */ -static PyObject * -unicode_getlength(PyObject *self, PyObject *arg) + +static inline int +writer_check(WriterObject *self) { - NULLABLE(arg); - RETURN_SIZE(PyUnicode_GetLength(arg)); + if (self->writer == NULL) { + PyErr_SetString(PyExc_ValueError, "operation on finished writer"); + return -1; + } + return 0; } -/* Test PyUnicode_ReadChar() */ -static PyObject * -unicode_readchar(PyObject *self, PyObject *args) + +static PyObject* +writer_write_char(PyObject *self_raw, PyObject *args) { - PyObject *unicode; - Py_ssize_t index; - Py_UCS4 result; + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } - if (!PyArg_ParseTuple(args, "On", &unicode, &index)) { + PyObject *str; + if (!PyArg_ParseTuple(args, "U", &str)) { return NULL; } + if (PyUnicode_GET_LENGTH(str) != 1) { + PyErr_SetString(PyExc_ValueError, "expect a single character"); + } + Py_UCS4 ch = PyUnicode_READ_CHAR(str, 0); - NULLABLE(unicode); - result = PyUnicode_ReadChar(unicode, index); - if (result == (Py_UCS4)-1) + if (PyUnicodeWriter_WriteChar(self->writer, ch) < 0) { return NULL; - return PyLong_FromUnsignedLong(result); + } + Py_RETURN_NONE; } -/* Test PyUnicode_FromEncodedObject() */ -static PyObject * -unicode_fromencodedobject(PyObject *self, PyObject *args) + +static PyObject* +writer_write_utf8(PyObject *self_raw, PyObject *args) { - PyObject *obj; - const char *encoding; - const char *errors = NULL; + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } - if (!PyArg_ParseTuple(args, "Oz|z", &obj, &encoding, &errors)) { + char *str; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "yn", &str, &size)) { return NULL; } - NULLABLE(obj); - return PyUnicode_FromEncodedObject(obj, encoding, errors); + if (PyUnicodeWriter_WriteUTF8(self->writer, str, size) < 0) { + return NULL; + } + Py_RETURN_NONE; } -/* Test PyUnicode_FromObject() */ -static PyObject * -unicode_fromobject(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_FromObject(arg); -} -/* Test PyUnicode_InternInPlace() */ -static PyObject * -unicode_interninplace(PyObject *self, PyObject *arg) +static PyObject* +writer_write_widechar(PyObject *self_raw, PyObject *args) { - NULLABLE(arg); - Py_XINCREF(arg); - PyUnicode_InternInPlace(&arg); - return arg; -} + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + PyObject *str; + if (!PyArg_ParseTuple(args, "U", &str)) { + return NULL; + } -/* Test PyUnicode_InternFromString() */ -static PyObject * -unicode_internfromstring(PyObject *self, PyObject *arg) -{ - const char *s; Py_ssize_t size; + wchar_t *wstr = PyUnicode_AsWideCharString(str, &size); + if (wstr == NULL) { + return NULL; + } - if (!PyArg_Parse(arg, "z#", &s, &size)) { + int res = PyUnicodeWriter_WriteWideChar(self->writer, wstr, size); + PyMem_Free(wstr); + if (res < 0) { return NULL; } - return PyUnicode_InternFromString(s); + Py_RETURN_NONE; } -/* Test PyUnicode_FromWideChar() */ -static PyObject * -unicode_fromwidechar(PyObject *self, PyObject *args) -{ - const char *s; - Py_ssize_t bsize; - Py_ssize_t size = -100; - if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { +static PyObject* +writer_write_ucs4(PyObject *self_raw, PyObject *args) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; } - if (size == -100) { - if (bsize % SIZEOF_WCHAR_T) { - PyErr_SetString(PyExc_AssertionError, - "invalid size in unicode_fromwidechar()"); - return NULL; - } - size = bsize / SIZEOF_WCHAR_T; - } - return PyUnicode_FromWideChar((const wchar_t *)s, size); -} -/* Test PyUnicode_AsWideChar() */ -static PyObject * -unicode_aswidechar(PyObject *self, PyObject *args) -{ - PyObject *unicode, *result; - Py_ssize_t buflen, size; - wchar_t *buffer; - - if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) - return NULL; - NULLABLE(unicode); - buffer = PyMem_New(wchar_t, buflen); - if (buffer == NULL) - return PyErr_NoMemory(); - - size = PyUnicode_AsWideChar(unicode, buffer, buflen); - if (size == -1) { - PyMem_Free(buffer); - return NULL; - } - - if (size < buflen) - buflen = size + 1; - else - buflen = size; - result = PyUnicode_FromWideChar(buffer, buflen); - PyMem_Free(buffer); - if (result == NULL) - return NULL; - - return Py_BuildValue("(Nn)", result, size); -} - -/* Test PyUnicode_AsWideCharString() with NULL as buffer */ -static PyObject * -unicode_aswidechar_null(PyObject *self, PyObject *args) -{ - PyObject *unicode; - Py_ssize_t buflen; - - if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) - return NULL; - NULLABLE(unicode); - RETURN_SIZE(PyUnicode_AsWideChar(unicode, NULL, buflen)); -} - -/* Test PyUnicode_AsWideCharString() */ -static PyObject * -unicode_aswidecharstring(PyObject *self, PyObject *args) -{ - PyObject *unicode, *result; - Py_ssize_t size = UNINITIALIZED_SIZE; - wchar_t *buffer; - - if (!PyArg_ParseTuple(args, "O", &unicode)) - return NULL; - - NULLABLE(unicode); - buffer = PyUnicode_AsWideCharString(unicode, &size); - if (buffer == NULL) { - assert(size == UNINITIALIZED_SIZE); - return NULL; - } - - result = PyUnicode_FromWideChar(buffer, size + 1); - PyMem_Free(buffer); - if (result == NULL) - return NULL; - return Py_BuildValue("(Nn)", result, size); -} - -/* Test PyUnicode_AsWideCharString() with NULL as the size address */ -static PyObject * -unicode_aswidecharstring_null(PyObject *self, PyObject *args) -{ - PyObject *unicode, *result; - wchar_t *buffer; - - if (!PyArg_ParseTuple(args, "O", &unicode)) - return NULL; - - NULLABLE(unicode); - buffer = PyUnicode_AsWideCharString(unicode, NULL); - if (buffer == NULL) - return NULL; - - result = PyUnicode_FromWideChar(buffer, -1); - PyMem_Free(buffer); - if (result == NULL) - return NULL; - return result; -} - -/* Test PyUnicode_AsUCS4() */ -static PyObject * -unicode_asucs4(PyObject *self, PyObject *args) -{ - PyObject *unicode, *result; - Py_UCS4 *buffer; - int copy_null; - Py_ssize_t str_len, buf_len; - - if (!PyArg_ParseTuple(args, "Onp:unicode_asucs4", &unicode, &str_len, ©_null)) { - return NULL; - } - - NULLABLE(unicode); - buf_len = str_len + 1; - buffer = PyMem_NEW(Py_UCS4, buf_len); - if (buffer == NULL) { - return PyErr_NoMemory(); - } - memset(buffer, 0, sizeof(Py_UCS4)*buf_len); - buffer[str_len] = 0xffffU; - - if (!PyUnicode_AsUCS4(unicode, buffer, buf_len, copy_null)) { - PyMem_Free(buffer); - return NULL; - } - - result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buffer, buf_len); - PyMem_Free(buffer); - return result; -} - -/* Test PyUnicode_AsUCS4Copy() */ -static PyObject * -unicode_asucs4copy(PyObject *self, PyObject *args) -{ - PyObject *unicode; - Py_UCS4 *buffer; - PyObject *result; - - if (!PyArg_ParseTuple(args, "O", &unicode)) { - return NULL; - } - - NULLABLE(unicode); - buffer = PyUnicode_AsUCS4Copy(unicode); - if (buffer == NULL) { - return NULL; - } - result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - buffer, - PyUnicode_GET_LENGTH(unicode) + 1); - PyMem_FREE(buffer); - return result; -} - -/* Test PyUnicode_FromOrdinal() */ -static PyObject * -unicode_fromordinal(PyObject *self, PyObject *args) -{ - int ordinal; - - if (!PyArg_ParseTuple(args, "i", &ordinal)) - return NULL; - - return PyUnicode_FromOrdinal(ordinal); -} - -/* Test PyUnicode_AsUTF8() */ -static PyObject * -unicode_asutf8(PyObject *self, PyObject *args) -{ - PyObject *unicode; - Py_ssize_t buflen; - const char *s; - - if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) - return NULL; - - NULLABLE(unicode); - s = PyUnicode_AsUTF8(unicode); - if (s == NULL) - return NULL; - - return PyBytes_FromStringAndSize(s, buflen); -} - -/* Test PyUnicode_AsUTF8AndSize() */ -static PyObject * -unicode_asutf8andsize(PyObject *self, PyObject *args) -{ - PyObject *unicode; - Py_ssize_t buflen; - const char *s; - Py_ssize_t size = UNINITIALIZED_SIZE; - - if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) - return NULL; - - NULLABLE(unicode); - s = PyUnicode_AsUTF8AndSize(unicode, &size); - if (s == NULL) { - assert(size == -1); - return NULL; - } - - return Py_BuildValue("(y#n)", s, buflen, size); -} - -/* Test PyUnicode_AsUTF8AndSize() with NULL as the size address */ -static PyObject * -unicode_asutf8andsize_null(PyObject *self, PyObject *args) -{ - PyObject *unicode; - Py_ssize_t buflen; - const char *s; - - if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) - return NULL; - - NULLABLE(unicode); - s = PyUnicode_AsUTF8AndSize(unicode, NULL); - if (s == NULL) - return NULL; - - return PyBytes_FromStringAndSize(s, buflen); -} - -/* Test PyUnicode_GetDefaultEncoding() */ -static PyObject * -unicode_getdefaultencoding(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - const char *s = PyUnicode_GetDefaultEncoding(); - if (s == NULL) - return NULL; - - return PyBytes_FromString(s); -} - -/* Test PyUnicode_Decode() */ -static PyObject * -unicode_decode(PyObject *self, PyObject *args) -{ - const char *s; - Py_ssize_t size; - const char *encoding; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#z|z", &s, &size, &encoding, &errors)) - return NULL; - - return PyUnicode_Decode(s, size, encoding, errors); -} - -/* Test PyUnicode_AsEncodedString() */ -static PyObject * -unicode_asencodedstring(PyObject *self, PyObject *args) -{ - PyObject *unicode; - const char *encoding; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "Oz|z", &unicode, &encoding, &errors)) - return NULL; - - NULLABLE(unicode); - return PyUnicode_AsEncodedString(unicode, encoding, errors); -} - -/* Test PyUnicode_BuildEncodingMap() */ -static PyObject * -unicode_buildencodingmap(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_BuildEncodingMap(arg); -} - -/* Test PyUnicode_DecodeUTF7() */ -static PyObject * -unicode_decodeutf7(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeUTF7(data, size, errors); -} - -/* Test PyUnicode_DecodeUTF7Stateful() */ -static PyObject * -unicode_decodeutf7stateful(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF7Stateful(data, size, errors, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); - return NULL; - } - return Py_BuildValue("(Nn)", result, consumed); -} - -/* Test PyUnicode_DecodeUTF8() */ -static PyObject * -unicode_decodeutf8(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeUTF8(data, size, errors); -} - -/* Test PyUnicode_DecodeUTF8Stateful() */ -static PyObject * -unicode_decodeutf8stateful(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF8Stateful(data, size, errors, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); - return NULL; - } - return Py_BuildValue("(Nn)", result, consumed); -} - -/* Test PyUnicode_AsUTF8String() */ -static PyObject * -unicode_asutf8string(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsUTF8String(arg); -} - -/* Test PyUnicode_DecodeUTF32() */ -static PyObject * -unicode_decodeutf32(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - int byteorder = UNINITIALIZED_INT; - PyObject *result; - - if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF32(data, size, errors, &byteorder); - if (!result) { - return NULL; - } - return Py_BuildValue("(iN)", byteorder, result); -} - -/* Test PyUnicode_DecodeUTF32Stateful() */ -static PyObject * -unicode_decodeutf32stateful(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - int byteorder = UNINITIALIZED_INT; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF32Stateful(data, size, errors, &byteorder, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); - return NULL; - } - return Py_BuildValue("(iNn)", byteorder, result, consumed); -} - -/* Test PyUnicode_AsUTF32String() */ -static PyObject * -unicode_asutf32string(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsUTF32String(arg); -} - -/* Test PyUnicode_DecodeUTF16() */ -static PyObject * -unicode_decodeutf16(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - int byteorder = UNINITIALIZED_INT; - PyObject *result; - - if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF16(data, size, errors, &byteorder); - if (!result) { - return NULL; - } - return Py_BuildValue("(iN)", byteorder, result); -} - -/* Test PyUnicode_DecodeUTF16Stateful() */ -static PyObject * -unicode_decodeutf16stateful(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - int byteorder = UNINITIALIZED_INT; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF16Stateful(data, size, errors, &byteorder, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); - return NULL; - } - return Py_BuildValue("(iNn)", byteorder, result, consumed); -} - -/* Test PyUnicode_AsUTF16String() */ -static PyObject * -unicode_asutf16string(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsUTF16String(arg); -} - -/* Test PyUnicode_DecodeUnicodeEscape() */ -static PyObject * -unicode_decodeunicodeescape(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeUnicodeEscape(data, size, errors); -} - -/* Test PyUnicode_AsUnicodeEscapeString() */ -static PyObject * -unicode_asunicodeescapestring(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsUnicodeEscapeString(arg); -} - -static PyObject * -unicode_decoderawunicodeescape(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeRawUnicodeEscape(data, size, errors); -} - -/* Test PyUnicode_AsRawUnicodeEscapeString() */ -static PyObject * -unicode_asrawunicodeescapestring(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsRawUnicodeEscapeString(arg); -} - -static PyObject * -unicode_decodelatin1(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeLatin1(data, size, errors); -} - -/* Test PyUnicode_AsLatin1String() */ -static PyObject * -unicode_aslatin1string(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsLatin1String(arg); -} - -/* Test PyUnicode_DecodeASCII() */ -static PyObject * -unicode_decodeascii(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeASCII(data, size, errors); -} - -/* Test PyUnicode_AsASCIIString() */ -static PyObject * -unicode_asasciistring(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsASCIIString(arg); -} - -/* Test PyUnicode_DecodeCharmap() */ -static PyObject * -unicode_decodecharmap(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - PyObject *mapping; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#O|z", &data, &size, &mapping, &errors)) - return NULL; - - NULLABLE(mapping); - return PyUnicode_DecodeCharmap(data, size, mapping, errors); -} - -/* Test PyUnicode_AsCharmapString() */ -static PyObject * -unicode_ascharmapstring(PyObject *self, PyObject *args) -{ - PyObject *unicode; - PyObject *mapping; - - if (!PyArg_ParseTuple(args, "OO", &unicode, &mapping)) - return NULL; - - NULLABLE(unicode); - NULLABLE(mapping); - return PyUnicode_AsCharmapString(unicode, mapping); -} - -#ifdef MS_WINDOWS - -/* Test PyUnicode_DecodeMBCS() */ -static PyObject * -unicode_decodembcs(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeMBCS(data, size, errors); -} - -/* Test PyUnicode_DecodeMBCSStateful() */ -static PyObject * -unicode_decodembcsstateful(PyObject *self, PyObject *args) -{ - const char *data; + PyObject *str; Py_ssize_t size; - const char *errors = NULL; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeMBCSStateful(data, size, errors, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); + if (!PyArg_ParseTuple(args, "Un", &str, &size)) { return NULL; } - return Py_BuildValue("(Nn)", result, consumed); -} - -/* Test PyUnicode_DecodeCodePageStateful() */ -static PyObject * -unicode_decodecodepagestateful(PyObject *self, PyObject *args) -{ - int code_page; - const char *data; - Py_ssize_t size; - const char *errors = NULL; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "iy#|z", &code_page, &data, &size, &errors)) - return NULL; + Py_ssize_t len = PyUnicode_GET_LENGTH(str); + size = Py_MIN(size, len); - result = PyUnicode_DecodeCodePageStateful(code_page, data, size, errors, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); + Py_UCS4 *ucs4 = PyUnicode_AsUCS4Copy(str); + if (ucs4 == NULL) { return NULL; } - return Py_BuildValue("(Nn)", result, consumed); -} - -/* Test PyUnicode_AsMBCSString() */ -static PyObject * -unicode_asmbcsstring(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsMBCSString(arg); -} - -/* Test PyUnicode_EncodeCodePage() */ -static PyObject * -unicode_encodecodepage(PyObject *self, PyObject *args) -{ - int code_page; - PyObject *unicode; - const char *errors; - - if (!PyArg_ParseTuple(args, "iO|z", &code_page, &unicode, &errors)) - return NULL; - - NULLABLE(unicode); - return PyUnicode_EncodeCodePage(code_page, unicode, errors); -} - -#endif /* MS_WINDOWS */ - -/* Test PyUnicode_DecodeLocaleAndSize() */ -static PyObject * -unicode_decodelocaleandsize(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeLocaleAndSize(data, size, errors); -} - -/* Test PyUnicode_DecodeLocale() */ -static PyObject * -unicode_decodelocale(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeLocale(data, errors); -} - -/* Test PyUnicode_EncodeLocale() */ -static PyObject * -unicode_encodelocale(PyObject *self, PyObject *args) -{ - PyObject *unicode; - const char *errors; - - if (!PyArg_ParseTuple(args, "O|z", &unicode, &errors)) - return NULL; - - NULLABLE(unicode); - return PyUnicode_EncodeLocale(unicode, errors); -} -/* Test PyUnicode_DecodeFSDefault() */ -static PyObject * -unicode_decodefsdefault(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - - if (!PyArg_ParseTuple(args, "y#", &data, &size)) - return NULL; - - return PyUnicode_DecodeFSDefault(data); -} - -/* Test PyUnicode_DecodeFSDefaultAndSize() */ -static PyObject * -unicode_decodefsdefaultandsize(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - - if (!PyArg_ParseTuple(args, "y#|n", &data, &size, &size)) + int res = PyUnicodeWriter_WriteUCS4(self->writer, ucs4, size); + PyMem_Free(ucs4); + if (res < 0) { return NULL; - - return PyUnicode_DecodeFSDefaultAndSize(data, size); -} - -/* Test PyUnicode_EncodeFSDefault() */ -static PyObject * -unicode_encodefsdefault(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_EncodeFSDefault(arg); + } + Py_RETURN_NONE; } -/* Test PyUnicode_Concat() */ -static PyObject * -unicode_concat(PyObject *self, PyObject *args) -{ - PyObject *left; - PyObject *right; - - if (!PyArg_ParseTuple(args, "OO", &left, &right)) - return NULL; - - NULLABLE(left); - NULLABLE(right); - return PyUnicode_Concat(left, right); -} -/* Test PyUnicode_Split() */ -static PyObject * -unicode_split(PyObject *self, PyObject *args) +static PyObject* +writer_write_str(PyObject *self_raw, PyObject *args) { - PyObject *s; - PyObject *sep; - Py_ssize_t maxsplit = -1; - - if (!PyArg_ParseTuple(args, "OO|n", &s, &sep, &maxsplit)) + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; + } - NULLABLE(s); - NULLABLE(sep); - return PyUnicode_Split(s, sep, maxsplit); -} - -/* Test PyUnicode_RSplit() */ -static PyObject * -unicode_rsplit(PyObject *self, PyObject *args) -{ - PyObject *s; - PyObject *sep; - Py_ssize_t maxsplit = -1; - - if (!PyArg_ParseTuple(args, "OO|n", &s, &sep, &maxsplit)) + PyObject *obj; + if (!PyArg_ParseTuple(args, "O", &obj)) { return NULL; + } - NULLABLE(s); - NULLABLE(sep); - return PyUnicode_RSplit(s, sep, maxsplit); -} - -/* Test PyUnicode_Splitlines() */ -static PyObject * -unicode_splitlines(PyObject *self, PyObject *args) -{ - PyObject *s; - int keepends = 0; - - if (!PyArg_ParseTuple(args, "O|i", &s, &keepends)) + if (PyUnicodeWriter_WriteStr(self->writer, obj) < 0) { return NULL; - - NULLABLE(s); - return PyUnicode_Splitlines(s, keepends); + } + Py_RETURN_NONE; } -/* Test PyUnicode_Partition() */ -static PyObject * -unicode_partition(PyObject *self, PyObject *args) -{ - PyObject *s; - PyObject *sep; - - if (!PyArg_ParseTuple(args, "OO", &s, &sep)) - return NULL; - - NULLABLE(s); - NULLABLE(sep); - return PyUnicode_Partition(s, sep); -} -/* Test PyUnicode_RPartition() */ -static PyObject * -unicode_rpartition(PyObject *self, PyObject *args) +static PyObject* +writer_write_repr(PyObject *self_raw, PyObject *args) { - PyObject *s; - PyObject *sep; - - if (!PyArg_ParseTuple(args, "OO", &s, &sep)) + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; + } - NULLABLE(s); - NULLABLE(sep); - return PyUnicode_RPartition(s, sep); -} - -/* Test PyUnicode_Translate() */ -static PyObject * -unicode_translate(PyObject *self, PyObject *args) -{ PyObject *obj; - PyObject *table; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "OO|z", &obj, &table, &errors)) + if (!PyArg_ParseTuple(args, "O", &obj)) { return NULL; + } - NULLABLE(obj); - NULLABLE(table); - return PyUnicode_Translate(obj, table, errors); -} - -/* Test PyUnicode_Join() */ -static PyObject * -unicode_join(PyObject *self, PyObject *args) -{ - PyObject *sep; - PyObject *seq; - - if (!PyArg_ParseTuple(args, "OO", &sep, &seq)) + if (PyUnicodeWriter_WriteRepr(self->writer, obj) < 0) { return NULL; - - NULLABLE(sep); - NULLABLE(seq); - return PyUnicode_Join(sep, seq); + } + Py_RETURN_NONE; } -/* Test PyUnicode_Count() */ -static PyObject * -unicode_count(PyObject *self, PyObject *args) -{ - PyObject *str; - PyObject *substr; - Py_ssize_t start; - Py_ssize_t end; - - if (!PyArg_ParseTuple(args, "OOnn", &str, &substr, &start, &end)) - return NULL; - - NULLABLE(str); - NULLABLE(substr); - RETURN_SIZE(PyUnicode_Count(str, substr, start, end)); -} -/* Test PyUnicode_Find() */ -static PyObject * -unicode_find(PyObject *self, PyObject *args) +static PyObject* +writer_write_substring(PyObject *self_raw, PyObject *args) { - PyObject *str; - PyObject *substr; - Py_ssize_t start; - Py_ssize_t end; - int direction; - Py_ssize_t result; - - if (!PyArg_ParseTuple(args, "OOnni", &str, &substr, &start, &end, &direction)) - return NULL; - - NULLABLE(str); - NULLABLE(substr); - result = PyUnicode_Find(str, substr, start, end, direction); - if (result == -2) { - assert(PyErr_Occurred()); + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; } - assert(!PyErr_Occurred()); - return PyLong_FromSsize_t(result); -} - -/* Test PyUnicode_Tailmatch() */ -static PyObject * -unicode_tailmatch(PyObject *self, PyObject *args) -{ - PyObject *str; - PyObject *substr; - Py_ssize_t start; - Py_ssize_t end; - int direction; - - if (!PyArg_ParseTuple(args, "OOnni", &str, &substr, &start, &end, &direction)) - return NULL; - - NULLABLE(str); - NULLABLE(substr); - RETURN_SIZE(PyUnicode_Tailmatch(str, substr, start, end, direction)); -} -/* Test PyUnicode_FindChar() */ -static PyObject * -unicode_findchar(PyObject *self, PyObject *args) -{ PyObject *str; - int direction; - unsigned int ch; - Py_ssize_t result; Py_ssize_t start, end; - - if (!PyArg_ParseTuple(args, "OInni:unicode_findchar", &str, &ch, - &start, &end, &direction)) { - return NULL; - } - NULLABLE(str); - result = PyUnicode_FindChar(str, (Py_UCS4)ch, start, end, direction); - if (result == -2) { - assert(PyErr_Occurred()); + if (!PyArg_ParseTuple(args, "Unn", &str, &start, &end)) { return NULL; } - assert(!PyErr_Occurred()); - return PyLong_FromSsize_t(result); -} - -/* Test PyUnicode_Replace() */ -static PyObject * -unicode_replace(PyObject *self, PyObject *args) -{ - PyObject *str; - PyObject *substr; - PyObject *replstr; - Py_ssize_t maxcount = -1; - - if (!PyArg_ParseTuple(args, "OOO|n", &str, &substr, &replstr, &maxcount)) - return NULL; - - NULLABLE(str); - NULLABLE(substr); - NULLABLE(replstr); - return PyUnicode_Replace(str, substr, replstr, maxcount); -} - -/* Test PyUnicode_Compare() */ -static PyObject * -unicode_compare(PyObject *self, PyObject *args) -{ - PyObject *left; - PyObject *right; - int result; - if (!PyArg_ParseTuple(args, "OO", &left, &right)) - return NULL; - - NULLABLE(left); - NULLABLE(right); - result = PyUnicode_Compare(left, right); - if (result == -1 && PyErr_Occurred()) { + if (PyUnicodeWriter_WriteSubstring(self->writer, str, start, end) < 0) { return NULL; } - assert(!PyErr_Occurred()); - return PyLong_FromLong(result); + Py_RETURN_NONE; } -/* Test PyUnicode_CompareWithASCIIString() */ -static PyObject * -unicode_comparewithasciistring(PyObject *self, PyObject *args) -{ - PyObject *left; - const char *right = NULL; - Py_ssize_t right_len; - int result; - - if (!PyArg_ParseTuple(args, "O|y#", &left, &right, &right_len)) - return NULL; - NULLABLE(left); - result = PyUnicode_CompareWithASCIIString(left, right); - if (result == -1 && PyErr_Occurred()) { +static PyObject* +writer_decodeutf8stateful(PyObject *self_raw, PyObject *args) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; } - return PyLong_FromLong(result); -} - -/* Test PyUnicode_EqualToUTF8() */ -static PyObject * -unicode_equaltoutf8(PyObject *self, PyObject *args) -{ - PyObject *left; - const char *right = NULL; - Py_ssize_t right_len; - int result; - if (!PyArg_ParseTuple(args, "Oz#", &left, &right, &right_len)) { + const char *str; + Py_ssize_t len; + const char *errors; + int use_consumed = 0; + if (!PyArg_ParseTuple(args, "yny|i", &str, &len, &errors, &use_consumed)) { return NULL; } - NULLABLE(left); - result = PyUnicode_EqualToUTF8(left, right); - assert(!PyErr_Occurred()); - return PyLong_FromLong(result); -} - -/* Test PyUnicode_EqualToUTF8AndSize() */ -static PyObject * -unicode_equaltoutf8andsize(PyObject *self, PyObject *args) -{ - PyObject *left; - const char *right = NULL; - Py_ssize_t right_len; - Py_ssize_t size = -100; - int result; - - if (!PyArg_ParseTuple(args, "Oz#|n", &left, &right, &right_len, &size)) { + Py_ssize_t consumed = 12345; + Py_ssize_t *pconsumed = use_consumed ? &consumed : NULL; + if (PyUnicodeWriter_DecodeUTF8Stateful(self->writer, str, len, + errors, pconsumed) < 0) { + if (use_consumed) { + assert(consumed == 0); + } return NULL; } - NULLABLE(left); - if (size == -100) { - size = right_len; + if (use_consumed) { + return PyLong_FromSsize_t(consumed); } - result = PyUnicode_EqualToUTF8AndSize(left, right, size); - assert(!PyErr_Occurred()); - return PyLong_FromLong(result); -} - -/* Test PyUnicode_RichCompare() */ -static PyObject * -unicode_richcompare(PyObject *self, PyObject *args) -{ - PyObject *left; - PyObject *right; - int op; - - if (!PyArg_ParseTuple(args, "OOi", &left, &right, &op)) - return NULL; - - NULLABLE(left); - NULLABLE(right); - return PyUnicode_RichCompare(left, right, op); + Py_RETURN_NONE; } -/* Test PyUnicode_Format() */ -static PyObject * -unicode_format(PyObject *self, PyObject *args) -{ - PyObject *format; - PyObject *fargs; - - if (!PyArg_ParseTuple(args, "OO", &format, &fargs)) - return NULL; - - NULLABLE(format); - NULLABLE(fargs); - return PyUnicode_Format(format, fargs); -} -/* Test PyUnicode_Contains() */ -static PyObject * -unicode_contains(PyObject *self, PyObject *args) +static PyObject* +writer_get_pointer(PyObject *self_raw, PyObject *args) { - PyObject *container; - PyObject *element; - - if (!PyArg_ParseTuple(args, "OO", &container, &element)) + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; + } - NULLABLE(container); - NULLABLE(element); - RETURN_INT(PyUnicode_Contains(container, element)); + return PyLong_FromVoidPtr(self->writer); } -/* Test PyUnicode_IsIdentifier() */ -static PyObject * -unicode_isidentifier(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - RETURN_INT(PyUnicode_IsIdentifier(arg)); -} -/* Test PyUnicode_CopyCharacters() */ -static PyObject * -unicode_copycharacters(PyObject *self, PyObject *args) +static PyObject* +writer_finish(PyObject *self_raw, PyObject *Py_UNUSED(args)) { - PyObject *from, *to, *to_copy; - Py_ssize_t from_start, to_start, how_many, copied; - - if (!PyArg_ParseTuple(args, "UnOnn", &to, &to_start, - &from, &from_start, &how_many)) { - return NULL; - } - - NULLABLE(from); - if (!(to_copy = PyUnicode_New(PyUnicode_GET_LENGTH(to), - PyUnicode_MAX_CHAR_VALUE(to)))) { - return NULL; - } - if (PyUnicode_Fill(to_copy, 0, PyUnicode_GET_LENGTH(to_copy), 0U) < 0) { - Py_DECREF(to_copy); - return NULL; - } - - copied = PyUnicode_CopyCharacters(to_copy, to_start, from, - from_start, how_many); - if (copied == -1 && PyErr_Occurred()) { - Py_DECREF(to_copy); + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; } - return Py_BuildValue("(Nn)", to_copy, copied); + PyObject *str = PyUnicodeWriter_Finish(self->writer); + self->writer = NULL; + return str; } -static int -check_raised_systemerror(PyObject *result, char* msg) -{ - if (result) { - // no exception - PyErr_Format(PyExc_AssertionError, - "SystemError not raised: %s", - msg); - return 0; - } - if (PyErr_ExceptionMatches(PyExc_SystemError)) { - // expected exception - PyErr_Clear(); - return 1; - } - // unexpected exception - return 0; -} -static PyObject * -test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - PyObject *result; - PyObject *unicode = PyUnicode_FromString("None"); - -#define CHECK_FORMAT_2(FORMAT, EXPECTED, ARG1, ARG2) \ - result = PyUnicode_FromFormat(FORMAT, ARG1, ARG2); \ - if (EXPECTED == NULL) { \ - if (!check_raised_systemerror(result, FORMAT)) { \ - goto Fail; \ - } \ - } \ - else if (result == NULL) \ - return NULL; \ - else if (PyUnicode_CompareWithASCIIString(result, EXPECTED) != 0) { \ - PyErr_Format(PyExc_AssertionError, \ - "test_string_from_format: failed at \"%s\" " \ - "expected \"%s\" got \"%s\"", \ - FORMAT, EXPECTED, PyUnicode_AsUTF8(result)); \ - goto Fail; \ - } \ - Py_XDECREF(result) - -#define CHECK_FORMAT_1(FORMAT, EXPECTED, ARG) \ - CHECK_FORMAT_2(FORMAT, EXPECTED, ARG, 0) - -#define CHECK_FORMAT_0(FORMAT, EXPECTED) \ - CHECK_FORMAT_2(FORMAT, EXPECTED, 0, 0) - - // Unrecognized - CHECK_FORMAT_2("%u %? %u", NULL, 1, 2); - - // "%%" (options are rejected) - CHECK_FORMAT_0( "%%", "%"); - CHECK_FORMAT_0( "%0%", NULL); - CHECK_FORMAT_0("%00%", NULL); - CHECK_FORMAT_0( "%2%", NULL); - CHECK_FORMAT_0("%02%", NULL); - CHECK_FORMAT_0("%.0%", NULL); - CHECK_FORMAT_0("%.2%", NULL); - - // "%c" - CHECK_FORMAT_1( "%c", "c", 'c'); - CHECK_FORMAT_1( "%0c", "c", 'c'); - CHECK_FORMAT_1("%00c", "c", 'c'); - CHECK_FORMAT_1( "%2c", NULL, 'c'); - CHECK_FORMAT_1("%02c", NULL, 'c'); - CHECK_FORMAT_1("%.0c", NULL, 'c'); - CHECK_FORMAT_1("%.2c", NULL, 'c'); - - // Integers - CHECK_FORMAT_1("%d", "123", (int)123); - CHECK_FORMAT_1("%i", "123", (int)123); - CHECK_FORMAT_1("%u", "123", (unsigned int)123); - CHECK_FORMAT_1("%x", "7b", (unsigned int)123); - CHECK_FORMAT_1("%X", "7B", (unsigned int)123); - CHECK_FORMAT_1("%o", "173", (unsigned int)123); - CHECK_FORMAT_1("%ld", "123", (long)123); - CHECK_FORMAT_1("%li", "123", (long)123); - CHECK_FORMAT_1("%lu", "123", (unsigned long)123); - CHECK_FORMAT_1("%lx", "7b", (unsigned long)123); - CHECK_FORMAT_1("%lX", "7B", (unsigned long)123); - CHECK_FORMAT_1("%lo", "173", (unsigned long)123); - CHECK_FORMAT_1("%lld", "123", (long long)123); - CHECK_FORMAT_1("%lli", "123", (long long)123); - CHECK_FORMAT_1("%llu", "123", (unsigned long long)123); - CHECK_FORMAT_1("%llx", "7b", (unsigned long long)123); - CHECK_FORMAT_1("%llX", "7B", (unsigned long long)123); - CHECK_FORMAT_1("%llo", "173", (unsigned long long)123); - CHECK_FORMAT_1("%zd", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%zi", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%zu", "123", (size_t)123); - CHECK_FORMAT_1("%zx", "7b", (size_t)123); - CHECK_FORMAT_1("%zX", "7B", (size_t)123); - CHECK_FORMAT_1("%zo", "173", (size_t)123); - CHECK_FORMAT_1("%td", "123", (ptrdiff_t)123); - CHECK_FORMAT_1("%ti", "123", (ptrdiff_t)123); - CHECK_FORMAT_1("%tu", "123", (ptrdiff_t)123); - CHECK_FORMAT_1("%tx", "7b", (ptrdiff_t)123); - CHECK_FORMAT_1("%tX", "7B", (ptrdiff_t)123); - CHECK_FORMAT_1("%to", "173", (ptrdiff_t)123); - CHECK_FORMAT_1("%jd", "123", (intmax_t)123); - CHECK_FORMAT_1("%ji", "123", (intmax_t)123); - CHECK_FORMAT_1("%ju", "123", (uintmax_t)123); - CHECK_FORMAT_1("%jx", "7b", (uintmax_t)123); - CHECK_FORMAT_1("%jX", "7B", (uintmax_t)123); - CHECK_FORMAT_1("%jo", "173", (uintmax_t)123); - - CHECK_FORMAT_1("%d", "-123", (int)-123); - CHECK_FORMAT_1("%i", "-123", (int)-123); - CHECK_FORMAT_1("%ld", "-123", (long)-123); - CHECK_FORMAT_1("%li", "-123", (long)-123); - CHECK_FORMAT_1("%lld", "-123", (long long)-123); - CHECK_FORMAT_1("%lli", "-123", (long long)-123); - CHECK_FORMAT_1("%zd", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%zi", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%td", "-123", (ptrdiff_t)-123); - CHECK_FORMAT_1("%ti", "-123", (ptrdiff_t)-123); - CHECK_FORMAT_1("%jd", "-123", (intmax_t)-123); - CHECK_FORMAT_1("%ji", "-123", (intmax_t)-123); - - // Integers: width < length - CHECK_FORMAT_1("%1d", "123", (int)123); - CHECK_FORMAT_1("%1i", "123", (int)123); - CHECK_FORMAT_1("%1u", "123", (unsigned int)123); - CHECK_FORMAT_1("%1ld", "123", (long)123); - CHECK_FORMAT_1("%1li", "123", (long)123); - CHECK_FORMAT_1("%1lu", "123", (unsigned long)123); - CHECK_FORMAT_1("%1lld", "123", (long long)123); - CHECK_FORMAT_1("%1lli", "123", (long long)123); - CHECK_FORMAT_1("%1llu", "123", (unsigned long long)123); - CHECK_FORMAT_1("%1zd", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%1zi", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%1zu", "123", (size_t)123); - CHECK_FORMAT_1("%1x", "7b", (int)123); - - CHECK_FORMAT_1("%1d", "-123", (int)-123); - CHECK_FORMAT_1("%1i", "-123", (int)-123); - CHECK_FORMAT_1("%1ld", "-123", (long)-123); - CHECK_FORMAT_1("%1li", "-123", (long)-123); - CHECK_FORMAT_1("%1lld", "-123", (long long)-123); - CHECK_FORMAT_1("%1lli", "-123", (long long)-123); - CHECK_FORMAT_1("%1zd", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%1zi", "-123", (Py_ssize_t)-123); - - // Integers: width > length - CHECK_FORMAT_1("%5d", " 123", (int)123); - CHECK_FORMAT_1("%5i", " 123", (int)123); - CHECK_FORMAT_1("%5u", " 123", (unsigned int)123); - CHECK_FORMAT_1("%5ld", " 123", (long)123); - CHECK_FORMAT_1("%5li", " 123", (long)123); - CHECK_FORMAT_1("%5lu", " 123", (unsigned long)123); - CHECK_FORMAT_1("%5lld", " 123", (long long)123); - CHECK_FORMAT_1("%5lli", " 123", (long long)123); - CHECK_FORMAT_1("%5llu", " 123", (unsigned long long)123); - CHECK_FORMAT_1("%5zd", " 123", (Py_ssize_t)123); - CHECK_FORMAT_1("%5zi", " 123", (Py_ssize_t)123); - CHECK_FORMAT_1("%5zu", " 123", (size_t)123); - CHECK_FORMAT_1("%5x", " 7b", (int)123); - - CHECK_FORMAT_1("%5d", " -123", (int)-123); - CHECK_FORMAT_1("%5i", " -123", (int)-123); - CHECK_FORMAT_1("%5ld", " -123", (long)-123); - CHECK_FORMAT_1("%5li", " -123", (long)-123); - CHECK_FORMAT_1("%5lld", " -123", (long long)-123); - CHECK_FORMAT_1("%5lli", " -123", (long long)-123); - CHECK_FORMAT_1("%5zd", " -123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%5zi", " -123", (Py_ssize_t)-123); - - // Integers: width > length, 0-flag - CHECK_FORMAT_1("%05d", "00123", (int)123); - CHECK_FORMAT_1("%05i", "00123", (int)123); - CHECK_FORMAT_1("%05u", "00123", (unsigned int)123); - CHECK_FORMAT_1("%05ld", "00123", (long)123); - CHECK_FORMAT_1("%05li", "00123", (long)123); - CHECK_FORMAT_1("%05lu", "00123", (unsigned long)123); - CHECK_FORMAT_1("%05lld", "00123", (long long)123); - CHECK_FORMAT_1("%05lli", "00123", (long long)123); - CHECK_FORMAT_1("%05llu", "00123", (unsigned long long)123); - CHECK_FORMAT_1("%05zd", "00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%05zi", "00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%05zu", "00123", (size_t)123); - CHECK_FORMAT_1("%05x", "0007b", (int)123); - - CHECK_FORMAT_1("%05d", "-0123", (int)-123); - CHECK_FORMAT_1("%05i", "-0123", (int)-123); - CHECK_FORMAT_1("%05ld", "-0123", (long)-123); - CHECK_FORMAT_1("%05li", "-0123", (long)-123); - CHECK_FORMAT_1("%05lld", "-0123", (long long)-123); - CHECK_FORMAT_1("%05lli", "-0123", (long long)-123); - CHECK_FORMAT_1("%05zd", "-0123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%05zi", "-0123", (Py_ssize_t)-123); - - // Integers: precision < length - CHECK_FORMAT_1("%.1d", "123", (int)123); - CHECK_FORMAT_1("%.1i", "123", (int)123); - CHECK_FORMAT_1("%.1u", "123", (unsigned int)123); - CHECK_FORMAT_1("%.1ld", "123", (long)123); - CHECK_FORMAT_1("%.1li", "123", (long)123); - CHECK_FORMAT_1("%.1lu", "123", (unsigned long)123); - CHECK_FORMAT_1("%.1lld", "123", (long long)123); - CHECK_FORMAT_1("%.1lli", "123", (long long)123); - CHECK_FORMAT_1("%.1llu", "123", (unsigned long long)123); - CHECK_FORMAT_1("%.1zd", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%.1zi", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%.1zu", "123", (size_t)123); - CHECK_FORMAT_1("%.1x", "7b", (int)123); - - CHECK_FORMAT_1("%.1d", "-123", (int)-123); - CHECK_FORMAT_1("%.1i", "-123", (int)-123); - CHECK_FORMAT_1("%.1ld", "-123", (long)-123); - CHECK_FORMAT_1("%.1li", "-123", (long)-123); - CHECK_FORMAT_1("%.1lld", "-123", (long long)-123); - CHECK_FORMAT_1("%.1lli", "-123", (long long)-123); - CHECK_FORMAT_1("%.1zd", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%.1zi", "-123", (Py_ssize_t)-123); - - // Integers: precision > length - CHECK_FORMAT_1("%.5d", "00123", (int)123); - CHECK_FORMAT_1("%.5i", "00123", (int)123); - CHECK_FORMAT_1("%.5u", "00123", (unsigned int)123); - CHECK_FORMAT_1("%.5ld", "00123", (long)123); - CHECK_FORMAT_1("%.5li", "00123", (long)123); - CHECK_FORMAT_1("%.5lu", "00123", (unsigned long)123); - CHECK_FORMAT_1("%.5lld", "00123", (long long)123); - CHECK_FORMAT_1("%.5lli", "00123", (long long)123); - CHECK_FORMAT_1("%.5llu", "00123", (unsigned long long)123); - CHECK_FORMAT_1("%.5zd", "00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%.5zi", "00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%.5zu", "00123", (size_t)123); - CHECK_FORMAT_1("%.5x", "0007b", (int)123); - - CHECK_FORMAT_1("%.5d", "-00123", (int)-123); - CHECK_FORMAT_1("%.5i", "-00123", (int)-123); - CHECK_FORMAT_1("%.5ld", "-00123", (long)-123); - CHECK_FORMAT_1("%.5li", "-00123", (long)-123); - CHECK_FORMAT_1("%.5lld", "-00123", (long long)-123); - CHECK_FORMAT_1("%.5lli", "-00123", (long long)-123); - CHECK_FORMAT_1("%.5zd", "-00123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%.5zi", "-00123", (Py_ssize_t)-123); - - // Integers: width > precision > length - CHECK_FORMAT_1("%7.5d", " 00123", (int)123); - CHECK_FORMAT_1("%7.5i", " 00123", (int)123); - CHECK_FORMAT_1("%7.5u", " 00123", (unsigned int)123); - CHECK_FORMAT_1("%7.5ld", " 00123", (long)123); - CHECK_FORMAT_1("%7.5li", " 00123", (long)123); - CHECK_FORMAT_1("%7.5lu", " 00123", (unsigned long)123); - CHECK_FORMAT_1("%7.5lld", " 00123", (long long)123); - CHECK_FORMAT_1("%7.5lli", " 00123", (long long)123); - CHECK_FORMAT_1("%7.5llu", " 00123", (unsigned long long)123); - CHECK_FORMAT_1("%7.5zd", " 00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%7.5zi", " 00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%7.5zu", " 00123", (size_t)123); - CHECK_FORMAT_1("%7.5x", " 0007b", (int)123); - - CHECK_FORMAT_1("%7.5d", " -00123", (int)-123); - CHECK_FORMAT_1("%7.5i", " -00123", (int)-123); - CHECK_FORMAT_1("%7.5ld", " -00123", (long)-123); - CHECK_FORMAT_1("%7.5li", " -00123", (long)-123); - CHECK_FORMAT_1("%7.5lld", " -00123", (long long)-123); - CHECK_FORMAT_1("%7.5lli", " -00123", (long long)-123); - CHECK_FORMAT_1("%7.5zd", " -00123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%7.5zi", " -00123", (Py_ssize_t)-123); - - // Integers: width > precision > length, 0-flag - CHECK_FORMAT_1("%07.5d", "0000123", (int)123); - CHECK_FORMAT_1("%07.5i", "0000123", (int)123); - CHECK_FORMAT_1("%07.5u", "0000123", (unsigned int)123); - CHECK_FORMAT_1("%07.5ld", "0000123", (long)123); - CHECK_FORMAT_1("%07.5li", "0000123", (long)123); - CHECK_FORMAT_1("%07.5lu", "0000123", (unsigned long)123); - CHECK_FORMAT_1("%07.5lld", "0000123", (long long)123); - CHECK_FORMAT_1("%07.5lli", "0000123", (long long)123); - CHECK_FORMAT_1("%07.5llu", "0000123", (unsigned long long)123); - CHECK_FORMAT_1("%07.5zd", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%07.5zi", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%07.5zu", "0000123", (size_t)123); - CHECK_FORMAT_1("%07.5x", "000007b", (int)123); - - CHECK_FORMAT_1("%07.5d", "-000123", (int)-123); - CHECK_FORMAT_1("%07.5i", "-000123", (int)-123); - CHECK_FORMAT_1("%07.5ld", "-000123", (long)-123); - CHECK_FORMAT_1("%07.5li", "-000123", (long)-123); - CHECK_FORMAT_1("%07.5lld", "-000123", (long long)-123); - CHECK_FORMAT_1("%07.5lli", "-000123", (long long)-123); - CHECK_FORMAT_1("%07.5zd", "-000123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%07.5zi", "-000123", (Py_ssize_t)-123); - - // Integers: precision > width > length - CHECK_FORMAT_1("%5.7d", "0000123", (int)123); - CHECK_FORMAT_1("%5.7i", "0000123", (int)123); - CHECK_FORMAT_1("%5.7u", "0000123", (unsigned int)123); - CHECK_FORMAT_1("%5.7ld", "0000123", (long)123); - CHECK_FORMAT_1("%5.7li", "0000123", (long)123); - CHECK_FORMAT_1("%5.7lu", "0000123", (unsigned long)123); - CHECK_FORMAT_1("%5.7lld", "0000123", (long long)123); - CHECK_FORMAT_1("%5.7lli", "0000123", (long long)123); - CHECK_FORMAT_1("%5.7llu", "0000123", (unsigned long long)123); - CHECK_FORMAT_1("%5.7zd", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%5.7zi", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%5.7zu", "0000123", (size_t)123); - CHECK_FORMAT_1("%5.7x", "000007b", (int)123); - - CHECK_FORMAT_1("%5.7d", "-0000123", (int)-123); - CHECK_FORMAT_1("%5.7i", "-0000123", (int)-123); - CHECK_FORMAT_1("%5.7ld", "-0000123", (long)-123); - CHECK_FORMAT_1("%5.7li", "-0000123", (long)-123); - CHECK_FORMAT_1("%5.7lld", "-0000123", (long long)-123); - CHECK_FORMAT_1("%5.7lli", "-0000123", (long long)-123); - CHECK_FORMAT_1("%5.7zd", "-0000123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%5.7zi", "-0000123", (Py_ssize_t)-123); - - // Integers: precision > width > length, 0-flag - CHECK_FORMAT_1("%05.7d", "0000123", (int)123); - CHECK_FORMAT_1("%05.7i", "0000123", (int)123); - CHECK_FORMAT_1("%05.7u", "0000123", (unsigned int)123); - CHECK_FORMAT_1("%05.7ld", "0000123", (long)123); - CHECK_FORMAT_1("%05.7li", "0000123", (long)123); - CHECK_FORMAT_1("%05.7lu", "0000123", (unsigned long)123); - CHECK_FORMAT_1("%05.7lld", "0000123", (long long)123); - CHECK_FORMAT_1("%05.7lli", "0000123", (long long)123); - CHECK_FORMAT_1("%05.7llu", "0000123", (unsigned long long)123); - CHECK_FORMAT_1("%05.7zd", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%05.7zi", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%05.7zu", "0000123", (size_t)123); - CHECK_FORMAT_1("%05.7x", "000007b", (int)123); - - CHECK_FORMAT_1("%05.7d", "-0000123", (int)-123); - CHECK_FORMAT_1("%05.7i", "-0000123", (int)-123); - CHECK_FORMAT_1("%05.7ld", "-0000123", (long)-123); - CHECK_FORMAT_1("%05.7li", "-0000123", (long)-123); - CHECK_FORMAT_1("%05.7lld", "-0000123", (long long)-123); - CHECK_FORMAT_1("%05.7lli", "-0000123", (long long)-123); - CHECK_FORMAT_1("%05.7zd", "-0000123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%05.7zi", "-0000123", (Py_ssize_t)-123); - - // Integers: precision = 0, arg = 0 (empty string in C) - CHECK_FORMAT_1("%.0d", "0", (int)0); - CHECK_FORMAT_1("%.0i", "0", (int)0); - CHECK_FORMAT_1("%.0u", "0", (unsigned int)0); - CHECK_FORMAT_1("%.0ld", "0", (long)0); - CHECK_FORMAT_1("%.0li", "0", (long)0); - CHECK_FORMAT_1("%.0lu", "0", (unsigned long)0); - CHECK_FORMAT_1("%.0lld", "0", (long long)0); - CHECK_FORMAT_1("%.0lli", "0", (long long)0); - CHECK_FORMAT_1("%.0llu", "0", (unsigned long long)0); - CHECK_FORMAT_1("%.0zd", "0", (Py_ssize_t)0); - CHECK_FORMAT_1("%.0zi", "0", (Py_ssize_t)0); - CHECK_FORMAT_1("%.0zu", "0", (size_t)0); - CHECK_FORMAT_1("%.0x", "0", (int)0); - - // Strings - CHECK_FORMAT_1("%s", "None", "None"); - CHECK_FORMAT_1("%ls", "None", L"None"); - CHECK_FORMAT_1("%U", "None", unicode); - CHECK_FORMAT_1("%A", "None", Py_None); - CHECK_FORMAT_1("%S", "None", Py_None); - CHECK_FORMAT_1("%R", "None", Py_None); - CHECK_FORMAT_2("%V", "None", unicode, "ignored"); - CHECK_FORMAT_2("%V", "None", NULL, "None"); - CHECK_FORMAT_2("%lV", "None", NULL, L"None"); - - // Strings: width < length - CHECK_FORMAT_1("%1s", "None", "None"); - CHECK_FORMAT_1("%1ls", "None", L"None"); - CHECK_FORMAT_1("%1U", "None", unicode); - CHECK_FORMAT_1("%1A", "None", Py_None); - CHECK_FORMAT_1("%1S", "None", Py_None); - CHECK_FORMAT_1("%1R", "None", Py_None); - CHECK_FORMAT_2("%1V", "None", unicode, "ignored"); - CHECK_FORMAT_2("%1V", "None", NULL, "None"); - CHECK_FORMAT_2("%1lV", "None", NULL, L"None"); - - // Strings: width > length - CHECK_FORMAT_1("%5s", " None", "None"); - CHECK_FORMAT_1("%5ls", " None", L"None"); - CHECK_FORMAT_1("%5U", " None", unicode); - CHECK_FORMAT_1("%5A", " None", Py_None); - CHECK_FORMAT_1("%5S", " None", Py_None); - CHECK_FORMAT_1("%5R", " None", Py_None); - CHECK_FORMAT_2("%5V", " None", unicode, "ignored"); - CHECK_FORMAT_2("%5V", " None", NULL, "None"); - CHECK_FORMAT_2("%5lV", " None", NULL, L"None"); - - // Strings: precision < length - CHECK_FORMAT_1("%.1s", "N", "None"); - CHECK_FORMAT_1("%.1ls", "N", L"None"); - CHECK_FORMAT_1("%.1U", "N", unicode); - CHECK_FORMAT_1("%.1A", "N", Py_None); - CHECK_FORMAT_1("%.1S", "N", Py_None); - CHECK_FORMAT_1("%.1R", "N", Py_None); - CHECK_FORMAT_2("%.1V", "N", unicode, "ignored"); - CHECK_FORMAT_2("%.1V", "N", NULL, "None"); - CHECK_FORMAT_2("%.1lV", "N", NULL, L"None"); - - // Strings: precision > length - CHECK_FORMAT_1("%.5s", "None", "None"); - CHECK_FORMAT_1("%.5ls", "None", L"None"); - CHECK_FORMAT_1("%.5U", "None", unicode); - CHECK_FORMAT_1("%.5A", "None", Py_None); - CHECK_FORMAT_1("%.5S", "None", Py_None); - CHECK_FORMAT_1("%.5R", "None", Py_None); - CHECK_FORMAT_2("%.5V", "None", unicode, "ignored"); - CHECK_FORMAT_2("%.5V", "None", NULL, "None"); - CHECK_FORMAT_2("%.5lV", "None", NULL, L"None"); - - // Strings: precision < length, width > length - CHECK_FORMAT_1("%5.1s", " N", "None"); - CHECK_FORMAT_1("%5.1ls"," N", L"None"); - CHECK_FORMAT_1("%5.1U", " N", unicode); - CHECK_FORMAT_1("%5.1A", " N", Py_None); - CHECK_FORMAT_1("%5.1S", " N", Py_None); - CHECK_FORMAT_1("%5.1R", " N", Py_None); - CHECK_FORMAT_2("%5.1V", " N", unicode, "ignored"); - CHECK_FORMAT_2("%5.1V", " N", NULL, "None"); - CHECK_FORMAT_2("%5.1lV"," N", NULL, L"None"); - - // Strings: width < length, precision > length - CHECK_FORMAT_1("%1.5s", "None", "None"); - CHECK_FORMAT_1("%1.5ls", "None", L"None"); - CHECK_FORMAT_1("%1.5U", "None", unicode); - CHECK_FORMAT_1("%1.5A", "None", Py_None); - CHECK_FORMAT_1("%1.5S", "None", Py_None); - CHECK_FORMAT_1("%1.5R", "None", Py_None); - CHECK_FORMAT_2("%1.5V", "None", unicode, "ignored"); - CHECK_FORMAT_2("%1.5V", "None", NULL, "None"); - CHECK_FORMAT_2("%1.5lV", "None", NULL, L"None"); - - Py_XDECREF(unicode); - Py_RETURN_NONE; +static PyMethodDef writer_methods[] = { + {"write_char", _PyCFunction_CAST(writer_write_char), METH_VARARGS}, + {"write_utf8", _PyCFunction_CAST(writer_write_utf8), METH_VARARGS}, + {"write_widechar", _PyCFunction_CAST(writer_write_widechar), METH_VARARGS}, + {"write_ucs4", _PyCFunction_CAST(writer_write_ucs4), METH_VARARGS}, + {"write_str", _PyCFunction_CAST(writer_write_str), METH_VARARGS}, + {"write_repr", _PyCFunction_CAST(writer_write_repr), METH_VARARGS}, + {"write_substring", _PyCFunction_CAST(writer_write_substring), METH_VARARGS}, + {"decodeutf8stateful", _PyCFunction_CAST(writer_decodeutf8stateful), METH_VARARGS}, + {"get_pointer", _PyCFunction_CAST(writer_get_pointer), METH_VARARGS}, + {"finish", _PyCFunction_CAST(writer_finish), METH_NOARGS}, + {NULL, NULL} /* sentinel */ +}; - Fail: - Py_XDECREF(result); - Py_XDECREF(unicode); - return NULL; +static PyType_Slot Writer_Type_slots[] = { + {Py_tp_new, writer_new}, + {Py_tp_init, writer_init}, + {Py_tp_dealloc, writer_dealloc}, + {Py_tp_methods, writer_methods}, + {0, 0}, /* sentinel */ +}; + +static PyType_Spec Writer_spec = { + .name = "_testcapi.PyUnicodeWriter", + .basicsize = sizeof(WriterObject), + .flags = Py_TPFLAGS_DEFAULT, + .slots = Writer_Type_slots, +}; -#undef CHECK_FORMAT_2 -#undef CHECK_FORMAT_1 -#undef CHECK_FORMAT_0 -} static PyMethodDef TestMethods[] = { - {"codec_incrementalencoder", codec_incrementalencoder, METH_VARARGS}, - {"codec_incrementaldecoder", codec_incrementaldecoder, METH_VARARGS}, - {"test_unicode_compare_with_ascii", - test_unicode_compare_with_ascii, METH_NOARGS}, - {"test_string_from_format", test_string_from_format, METH_NOARGS}, - {"test_widechar", test_widechar, METH_NOARGS}, {"unicode_new", unicode_new, METH_VARARGS}, {"unicode_fill", unicode_fill, METH_VARARGS}, - {"unicode_writechar", unicode_writechar, METH_VARARGS}, - {"unicode_resize", unicode_resize, METH_VARARGS}, - {"unicode_append", unicode_append, METH_VARARGS}, - {"unicode_appendanddel", unicode_appendanddel, METH_VARARGS}, - {"unicode_fromstringandsize",unicode_fromstringandsize, METH_VARARGS}, - {"unicode_fromstring", unicode_fromstring, METH_O}, {"unicode_fromkindanddata", unicode_fromkindanddata, METH_VARARGS}, - {"unicode_substring", unicode_substring, METH_VARARGS}, - {"unicode_getlength", unicode_getlength, METH_O}, - {"unicode_readchar", unicode_readchar, METH_VARARGS}, - {"unicode_fromencodedobject",unicode_fromencodedobject, METH_VARARGS}, - {"unicode_fromobject", unicode_fromobject, METH_O}, - {"unicode_interninplace", unicode_interninplace, METH_O}, - {"unicode_internfromstring", unicode_internfromstring, METH_O}, - {"unicode_fromwidechar", unicode_fromwidechar, METH_VARARGS}, - {"unicode_aswidechar", unicode_aswidechar, METH_VARARGS}, - {"unicode_aswidechar_null", unicode_aswidechar_null, METH_VARARGS}, - {"unicode_aswidecharstring", unicode_aswidecharstring, METH_VARARGS}, - {"unicode_aswidecharstring_null",unicode_aswidecharstring_null,METH_VARARGS}, {"unicode_asucs4", unicode_asucs4, METH_VARARGS}, {"unicode_asucs4copy", unicode_asucs4copy, METH_VARARGS}, - {"unicode_fromordinal", unicode_fromordinal, METH_VARARGS}, {"unicode_asutf8", unicode_asutf8, METH_VARARGS}, - {"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS}, - {"unicode_asutf8andsize_null",unicode_asutf8andsize_null, METH_VARARGS}, - {"unicode_getdefaultencoding",unicode_getdefaultencoding, METH_NOARGS}, - {"unicode_decode", unicode_decode, METH_VARARGS}, - {"unicode_asencodedstring", unicode_asencodedstring, METH_VARARGS}, - {"unicode_buildencodingmap", unicode_buildencodingmap, METH_O}, - {"unicode_decodeutf7", unicode_decodeutf7, METH_VARARGS}, - {"unicode_decodeutf7stateful",unicode_decodeutf7stateful, METH_VARARGS}, - {"unicode_decodeutf8", unicode_decodeutf8, METH_VARARGS}, - {"unicode_decodeutf8stateful",unicode_decodeutf8stateful, METH_VARARGS}, - {"unicode_asutf8string", unicode_asutf8string, METH_O}, - {"unicode_decodeutf16", unicode_decodeutf16, METH_VARARGS}, - {"unicode_decodeutf16stateful",unicode_decodeutf16stateful, METH_VARARGS}, - {"unicode_asutf16string", unicode_asutf16string, METH_O}, - {"unicode_decodeutf32", unicode_decodeutf32, METH_VARARGS}, - {"unicode_decodeutf32stateful",unicode_decodeutf32stateful, METH_VARARGS}, - {"unicode_asutf32string", unicode_asutf32string, METH_O}, - {"unicode_decodeunicodeescape",unicode_decodeunicodeescape, METH_VARARGS}, - {"unicode_asunicodeescapestring",unicode_asunicodeescapestring,METH_O}, - {"unicode_decoderawunicodeescape",unicode_decoderawunicodeescape,METH_VARARGS}, - {"unicode_asrawunicodeescapestring",unicode_asrawunicodeescapestring,METH_O}, - {"unicode_decodelatin1", unicode_decodelatin1, METH_VARARGS}, - {"unicode_aslatin1string", unicode_aslatin1string, METH_O}, - {"unicode_decodeascii", unicode_decodeascii, METH_VARARGS}, - {"unicode_asasciistring", unicode_asasciistring, METH_O}, - {"unicode_decodecharmap", unicode_decodecharmap, METH_VARARGS}, - {"unicode_ascharmapstring", unicode_ascharmapstring, METH_VARARGS}, -#ifdef MS_WINDOWS - {"unicode_decodembcs", unicode_decodembcs, METH_VARARGS}, - {"unicode_decodembcsstateful",unicode_decodembcsstateful, METH_VARARGS}, - {"unicode_decodecodepagestateful",unicode_decodecodepagestateful,METH_VARARGS}, - {"unicode_asmbcsstring", unicode_asmbcsstring, METH_O}, - {"unicode_encodecodepage", unicode_encodecodepage, METH_VARARGS}, -#endif /* MS_WINDOWS */ - {"unicode_decodelocaleandsize",unicode_decodelocaleandsize, METH_VARARGS}, - {"unicode_decodelocale", unicode_decodelocale, METH_VARARGS}, - {"unicode_encodelocale", unicode_encodelocale, METH_VARARGS}, - {"unicode_decodefsdefault", unicode_decodefsdefault, METH_VARARGS}, - {"unicode_decodefsdefaultandsize",unicode_decodefsdefaultandsize,METH_VARARGS}, - {"unicode_encodefsdefault", unicode_encodefsdefault, METH_O}, - {"unicode_concat", unicode_concat, METH_VARARGS}, - {"unicode_splitlines", unicode_splitlines, METH_VARARGS}, - {"unicode_split", unicode_split, METH_VARARGS}, - {"unicode_rsplit", unicode_rsplit, METH_VARARGS}, - {"unicode_partition", unicode_partition, METH_VARARGS}, - {"unicode_rpartition", unicode_rpartition, METH_VARARGS}, - {"unicode_translate", unicode_translate, METH_VARARGS}, - {"unicode_join", unicode_join, METH_VARARGS}, - {"unicode_count", unicode_count, METH_VARARGS}, - {"unicode_tailmatch", unicode_tailmatch, METH_VARARGS}, - {"unicode_find", unicode_find, METH_VARARGS}, - {"unicode_findchar", unicode_findchar, METH_VARARGS}, - {"unicode_replace", unicode_replace, METH_VARARGS}, - {"unicode_compare", unicode_compare, METH_VARARGS}, - {"unicode_comparewithasciistring",unicode_comparewithasciistring,METH_VARARGS}, - {"unicode_equaltoutf8", unicode_equaltoutf8, METH_VARARGS}, - {"unicode_equaltoutf8andsize",unicode_equaltoutf8andsize, METH_VARARGS}, - {"unicode_richcompare", unicode_richcompare, METH_VARARGS}, - {"unicode_format", unicode_format, METH_VARARGS}, - {"unicode_contains", unicode_contains, METH_VARARGS}, - {"unicode_isidentifier", unicode_isidentifier, METH_O}, {"unicode_copycharacters", unicode_copycharacters, METH_VARARGS}, {NULL}, }; int _PyTestCapi_Init_Unicode(PyObject *m) { - _testcapimodule = PyModule_GetDef(m); - if (PyModule_AddFunctions(m, TestMethods) < 0) { return -1; } + PyTypeObject *writer_type = (PyTypeObject *)PyType_FromSpec(&Writer_spec); + if (writer_type == NULL) { + return -1; + } + if (PyModule_AddType(m, writer_type) < 0) { + Py_DECREF(writer_type); + return -1; + } + Py_DECREF(writer_type); + return 0; } diff --git a/Modules/_testcapi/util.h b/Modules/_testcapi/util.h index f26d7656a10138..042e522542eddb 100644 --- a/Modules/_testcapi/util.h +++ b/Modules/_testcapi/util.h @@ -31,3 +31,13 @@ static const char uninitialized[] = "uninitialized"; #define UNINITIALIZED_SIZE ((Py_ssize_t)236892191) /* Marker to check that integer value was set. */ #define UNINITIALIZED_INT (63256717) +/* + * Marker to indicate that a NULL parameter would not be allowed + * at runtime but that the test interface will check that it is + * indeed the case. + * + * Use this macro only if passing NULL to the C API would raise + * a catchable exception (and not a fatal exception that would + * crash the interpreter). + */ + #define NULL_WOULD_RAISE(NAME) NAME diff --git a/Modules/_testcapi/vectorcall.c b/Modules/_testcapi/vectorcall.c index b30c5e8704c8af..03aaacb328e0b6 100644 --- a/Modules/_testcapi/vectorcall.c +++ b/Modules/_testcapi/vectorcall.c @@ -348,6 +348,9 @@ static PyObject * MethodDescriptor2_new(PyTypeObject* type, PyObject* args, PyObject *kw) { MethodDescriptor2Object *op = PyObject_New(MethodDescriptor2Object, type); + if (op == NULL) { + return NULL; + } op->base.vectorcall = NULL; op->vectorcall = MethodDescriptor_vectorcall; return (PyObject *)op; diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index 1eb0db2c2e6576..321d3aeffb6ad1 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -8,6 +8,7 @@ #define Py_BUILD_CORE #include "pycore_function.h" // FUNC_MAX_WATCHERS #include "pycore_code.h" // CODE_MAX_WATCHERS +#include "pycore_context.h" // CONTEXT_MAX_WATCHERS /*[clinic input] module _testcapi @@ -622,6 +623,152 @@ allocate_too_many_func_watchers(PyObject *self, PyObject *args) Py_RETURN_NONE; } +// Test contexct object watchers +#define NUM_CONTEXT_WATCHERS 2 +static int context_watcher_ids[NUM_CONTEXT_WATCHERS] = {-1, -1}; +static PyObject *context_switches[NUM_CONTEXT_WATCHERS]; + +static int +handle_context_watcher_event(int which_watcher, PyContextEvent event, PyObject *ctx) { + if (event == Py_CONTEXT_SWITCHED) { + PyList_Append(context_switches[which_watcher], ctx); + } + else { + return -1; + } + return 0; +} + +static int +first_context_watcher_callback(PyContextEvent event, PyObject *ctx) { + return handle_context_watcher_event(0, event, ctx); +} + +static int +second_context_watcher_callback(PyContextEvent event, PyObject *ctx) { + return handle_context_watcher_event(1, event, ctx); +} + +static int +noop_context_event_handler(PyContextEvent event, PyObject *ctx) { + return 0; +} + +static int +error_context_event_handler(PyContextEvent event, PyObject *ctx) { + PyErr_SetString(PyExc_RuntimeError, "boom!"); + return -1; +} + +static PyObject * +add_context_watcher(PyObject *self, PyObject *which_watcher) +{ + static const PyContext_WatchCallback callbacks[] = { + &first_context_watcher_callback, + &second_context_watcher_callback, + &error_context_event_handler, + }; + assert(PyLong_Check(which_watcher)); + long which_l = PyLong_AsLong(which_watcher); + if (which_l < 0 || which_l >= (long)Py_ARRAY_LENGTH(callbacks)) { + PyErr_Format(PyExc_ValueError, "invalid watcher %d", which_l); + return NULL; + } + int watcher_id = PyContext_AddWatcher(callbacks[which_l]); + if (watcher_id < 0) { + return NULL; + } + if (which_l >= 0 && which_l < NUM_CONTEXT_WATCHERS) { + context_watcher_ids[which_l] = watcher_id; + Py_XSETREF(context_switches[which_l], PyList_New(0)); + if (context_switches[which_l] == NULL) { + return NULL; + } + } + return PyLong_FromLong(watcher_id); +} + +static PyObject * +clear_context_watcher(PyObject *self, PyObject *watcher_id) +{ + assert(PyLong_Check(watcher_id)); + long watcher_id_l = PyLong_AsLong(watcher_id); + if (PyContext_ClearWatcher(watcher_id_l) < 0) { + return NULL; + } + // reset static events counters + if (watcher_id_l >= 0) { + for (int i = 0; i < NUM_CONTEXT_WATCHERS; i++) { + if (watcher_id_l == context_watcher_ids[i]) { + context_watcher_ids[i] = -1; + Py_CLEAR(context_switches[i]); + } + } + } + Py_RETURN_NONE; +} + +static PyObject * +clear_context_stack(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyThreadState *tstate = PyThreadState_Get(); + if (tstate->context == NULL) { + Py_RETURN_NONE; + } + if (((PyContext *)tstate->context)->ctx_prev != NULL) { + PyErr_SetString(PyExc_RuntimeError, + "must first exit all non-base contexts"); + return NULL; + } + Py_CLEAR(tstate->context); + Py_RETURN_NONE; +} + +static PyObject * +get_context_switches(PyObject *Py_UNUSED(self), PyObject *watcher_id) +{ + assert(PyLong_Check(watcher_id)); + long watcher_id_l = PyLong_AsLong(watcher_id); + if (watcher_id_l < 0 || watcher_id_l >= NUM_CONTEXT_WATCHERS) { + PyErr_Format(PyExc_ValueError, "invalid watcher %ld", watcher_id_l); + return NULL; + } + if (context_switches[watcher_id_l] == NULL) { + return PyList_New(0); + } + return Py_NewRef(context_switches[watcher_id_l]); +} + +static PyObject * +allocate_too_many_context_watchers(PyObject *self, PyObject *args) +{ + int watcher_ids[CONTEXT_MAX_WATCHERS + 1]; + int num_watchers = 0; + for (unsigned long i = 0; i < sizeof(watcher_ids) / sizeof(int); i++) { + int watcher_id = PyContext_AddWatcher(noop_context_event_handler); + if (watcher_id == -1) { + break; + } + watcher_ids[i] = watcher_id; + num_watchers++; + } + PyObject *exc = PyErr_GetRaisedException(); + for (int i = 0; i < num_watchers; i++) { + if (PyContext_ClearWatcher(watcher_ids[i]) < 0) { + PyErr_WriteUnraisable(Py_None); + break; + } + } + if (exc) { + PyErr_SetRaisedException(exc); + return NULL; + } + else if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NONE; +} + /*[clinic input] _testcapi.set_func_defaults_via_capi func: object @@ -689,6 +836,14 @@ static PyMethodDef test_methods[] = { _TESTCAPI_SET_FUNC_KWDEFAULTS_VIA_CAPI_METHODDEF {"allocate_too_many_func_watchers", allocate_too_many_func_watchers, METH_NOARGS, NULL}, + + // Code object watchers. + {"add_context_watcher", add_context_watcher, METH_O, NULL}, + {"clear_context_watcher", clear_context_watcher, METH_O, NULL}, + {"clear_context_stack", clear_context_stack, METH_NOARGS, NULL}, + {"get_context_switches", get_context_switches, METH_O, NULL}, + {"allocate_too_many_context_watchers", + (PyCFunction) allocate_too_many_context_watchers, METH_NOARGS, NULL}, {NULL}, }; diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index b03f871b089c8a..c405a352ed74a1 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -13,7 +13,6 @@ #include "_testcapi/parts.h" #include "frameobject.h" // PyFrame_New() -#include "interpreteridobject.h" // PyInterpreterID_Type #include "marshal.h" // PyMarshal_WriteLongToFile() #include // FLT_MAX @@ -82,8 +81,11 @@ static PyObject* test_config(PyObject *self, PyObject *Py_UNUSED(ignored)) { #define CHECK_SIZEOF(FATNAME, TYPE) \ - if (FATNAME != sizeof(TYPE)) \ - return sizeof_error(self, #FATNAME, #TYPE, FATNAME, sizeof(TYPE)) + do { \ + if (FATNAME != sizeof(TYPE)) { \ + return sizeof_error(self, #FATNAME, #TYPE, FATNAME, sizeof(TYPE)); \ + } \ + } while (0) CHECK_SIZEOF(SIZEOF_SHORT, short); CHECK_SIZEOF(SIZEOF_INT, int); @@ -104,21 +106,25 @@ test_sizeof_c_types(PyObject *self, PyObject *Py_UNUSED(ignored)) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wtype-limits" #endif -#define CHECK_SIZEOF(TYPE, EXPECTED) \ - if (EXPECTED != sizeof(TYPE)) { \ - PyErr_Format(get_testerror(self), \ - "sizeof(%s) = %u instead of %u", \ - #TYPE, sizeof(TYPE), EXPECTED); \ - return (PyObject*)NULL; \ - } +#define CHECK_SIZEOF(TYPE, EXPECTED) \ + do { \ + if (EXPECTED != sizeof(TYPE)) { \ + PyErr_Format(get_testerror(self), \ + "sizeof(%s) = %u instead of %u", \ + #TYPE, sizeof(TYPE), EXPECTED); \ + return (PyObject*)NULL; \ + } \ + } while (0) #define IS_SIGNED(TYPE) (((TYPE)-1) < (TYPE)0) -#define CHECK_SIGNNESS(TYPE, SIGNED) \ - if (IS_SIGNED(TYPE) != SIGNED) { \ - PyErr_Format(get_testerror(self), \ - "%s signness is %i, instead of %i", \ - #TYPE, IS_SIGNED(TYPE), SIGNED); \ - return (PyObject*)NULL; \ - } +#define CHECK_SIGNNESS(TYPE, SIGNED) \ + do { \ + if (IS_SIGNED(TYPE) != SIGNED) { \ + PyErr_Format(get_testerror(self), \ + "%s signness is %i, instead of %i", \ + #TYPE, IS_SIGNED(TYPE), SIGNED); \ + return (PyObject*)NULL; \ + } \ + } while (0) /* integer types */ CHECK_SIZEOF(Py_UCS1, 1); @@ -283,7 +289,7 @@ static PyTypeObject _HashInheritanceTester_Type = { "hashinheritancetester", /* Name of this type */ sizeof(PyObject), /* Basic object size */ 0, /* Item size for varobject */ - (destructor)PyObject_Del, /* tp_dealloc */ + (destructor)PyObject_Free, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -597,83 +603,39 @@ get_heaptype_for_name(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyType_FromSpec(&HeapTypeNameType_Spec); } + static PyObject * -test_get_type_name(PyObject *self, PyObject *Py_UNUSED(ignored)) +get_type_name(PyObject *self, PyObject *type) { - PyObject *tp_name = PyType_GetName(&PyLong_Type); - assert(strcmp(PyUnicode_AsUTF8(tp_name), "int") == 0); - Py_DECREF(tp_name); - - tp_name = PyType_GetName(&PyModule_Type); - assert(strcmp(PyUnicode_AsUTF8(tp_name), "module") == 0); - Py_DECREF(tp_name); - - PyObject *HeapTypeNameType = PyType_FromSpec(&HeapTypeNameType_Spec); - if (HeapTypeNameType == NULL) { - Py_RETURN_NONE; - } - tp_name = PyType_GetName((PyTypeObject *)HeapTypeNameType); - assert(strcmp(PyUnicode_AsUTF8(tp_name), "HeapTypeNameType") == 0); - Py_DECREF(tp_name); - - PyObject *name = PyUnicode_FromString("test_name"); - if (name == NULL) { - goto done; - } - if (PyObject_SetAttrString(HeapTypeNameType, "__name__", name) < 0) { - Py_DECREF(name); - goto done; - } - tp_name = PyType_GetName((PyTypeObject *)HeapTypeNameType); - assert(strcmp(PyUnicode_AsUTF8(tp_name), "test_name") == 0); - Py_DECREF(name); - Py_DECREF(tp_name); - - done: - Py_DECREF(HeapTypeNameType); - Py_RETURN_NONE; + assert(PyType_Check(type)); + return PyType_GetName((PyTypeObject *)type); } static PyObject * -test_get_type_qualname(PyObject *self, PyObject *Py_UNUSED(ignored)) +get_type_qualname(PyObject *self, PyObject *type) { - PyObject *tp_qualname = PyType_GetQualName(&PyLong_Type); - assert(strcmp(PyUnicode_AsUTF8(tp_qualname), "int") == 0); - Py_DECREF(tp_qualname); + assert(PyType_Check(type)); + return PyType_GetQualName((PyTypeObject *)type); +} - tp_qualname = PyType_GetQualName(&PyODict_Type); - assert(strcmp(PyUnicode_AsUTF8(tp_qualname), "OrderedDict") == 0); - Py_DECREF(tp_qualname); - PyObject *HeapTypeNameType = PyType_FromSpec(&HeapTypeNameType_Spec); - if (HeapTypeNameType == NULL) { - Py_RETURN_NONE; - } - tp_qualname = PyType_GetQualName((PyTypeObject *)HeapTypeNameType); - assert(strcmp(PyUnicode_AsUTF8(tp_qualname), "HeapTypeNameType") == 0); - Py_DECREF(tp_qualname); +static PyObject * +get_type_fullyqualname(PyObject *self, PyObject *type) +{ + assert(PyType_Check(type)); + return PyType_GetFullyQualifiedName((PyTypeObject *)type); +} - PyObject *spec_name = PyUnicode_FromString(HeapTypeNameType_Spec.name); - if (spec_name == NULL) { - goto done; - } - if (PyObject_SetAttrString(HeapTypeNameType, - "__qualname__", spec_name) < 0) { - Py_DECREF(spec_name); - goto done; - } - tp_qualname = PyType_GetQualName((PyTypeObject *)HeapTypeNameType); - assert(strcmp(PyUnicode_AsUTF8(tp_qualname), - "_testcapi.HeapTypeNameType") == 0); - Py_DECREF(spec_name); - Py_DECREF(tp_qualname); - done: - Py_DECREF(HeapTypeNameType); - Py_RETURN_NONE; +static PyObject * +get_type_module_name(PyObject *self, PyObject *type) +{ + assert(PyType_Check(type)); + return PyType_GetModuleName((PyTypeObject *)type); } + static PyObject * test_get_type_dict(PyObject *self, PyObject *Py_UNUSED(ignored)) { @@ -809,6 +771,14 @@ test_thread_state(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject * +gilstate_ensure_release(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyGILState_STATE state = PyGILState_Ensure(); + PyGILState_Release(state); + Py_RETURN_NONE; +} + #ifndef MS_WINDOWS static PyThread_type_lock wait_done = NULL; @@ -864,25 +834,55 @@ static int _pending_callback(void *arg) * run from any python thread. */ static PyObject * -pending_threadfunc(PyObject *self, PyObject *arg) +pending_threadfunc(PyObject *self, PyObject *arg, PyObject *kwargs) { + static char *kwlist[] = {"callback", "num", + "blocking", "ensure_added", NULL}; PyObject *callable; - int r; - if (PyArg_ParseTuple(arg, "O", &callable) == 0) + unsigned int num = 1; + int blocking = 0; + int ensure_added = 0; + if (!PyArg_ParseTupleAndKeywords(arg, kwargs, + "O|I$pp:_pending_threadfunc", kwlist, + &callable, &num, &blocking, &ensure_added)) + { return NULL; + } /* create the reference for the callbackwhile we hold the lock */ - Py_INCREF(callable); + for (unsigned int i = 0; i < num; i++) { + Py_INCREF(callable); + } - Py_BEGIN_ALLOW_THREADS - r = Py_AddPendingCall(&_pending_callback, callable); - Py_END_ALLOW_THREADS + PyThreadState *save_tstate = NULL; + if (!blocking) { + save_tstate = PyEval_SaveThread(); + } + + unsigned int num_added = 0; + for (; num_added < num; num_added++) { + if (ensure_added) { + int r; + do { + r = Py_AddPendingCall(&_pending_callback, callable); + } while (r < 0); + } + else { + if (Py_AddPendingCall(&_pending_callback, callable) < 0) { + break; + } + } + } - if (r<0) { + if (!blocking) { + PyEval_RestoreThread(save_tstate); + } + + for (unsigned int i = num_added; i < num; i++) { Py_DECREF(callable); /* unsuccessful add, destroy the extra reference */ - Py_RETURN_FALSE; } - Py_RETURN_TRUE; + /* The callable is decref'ed above in each added _pending_callback(). */ + return PyLong_FromUnsignedLong((unsigned long)num_added); } /* Test PyOS_string_to_double. */ @@ -891,27 +891,34 @@ test_string_to_double(PyObject *self, PyObject *Py_UNUSED(ignored)) { double result; const char *msg; -#define CHECK_STRING(STR, expected) \ - result = PyOS_string_to_double(STR, NULL, NULL); \ - if (result == -1.0 && PyErr_Occurred()) \ - return NULL; \ - if (result != (double)expected) { \ - msg = "conversion of " STR " to float failed"; \ - goto fail; \ - } +#define CHECK_STRING(STR, expected) \ + do { \ + result = PyOS_string_to_double(STR, NULL, NULL); \ + if (result == -1.0 && PyErr_Occurred()) { \ + return NULL; \ + } \ + if (result != (double)expected) { \ + msg = "conversion of " STR " to float failed"; \ + goto fail; \ + } \ + } while (0) -#define CHECK_INVALID(STR) \ - result = PyOS_string_to_double(STR, NULL, NULL); \ - if (result == -1.0 && PyErr_Occurred()) { \ - if (PyErr_ExceptionMatches(PyExc_ValueError)) \ - PyErr_Clear(); \ - else \ - return NULL; \ - } \ - else { \ - msg = "conversion of " STR " didn't raise ValueError"; \ - goto fail; \ - } +#define CHECK_INVALID(STR) \ + do { \ + result = PyOS_string_to_double(STR, NULL, NULL); \ + if (result == -1.0 && PyErr_Occurred()) { \ + if (PyErr_ExceptionMatches(PyExc_ValueError)) { \ + PyErr_Clear(); \ + } \ + else { \ + return NULL; \ + } \ + } \ + else { \ + msg = "conversion of " STR " didn't raise ValueError"; \ + goto fail; \ + } \ + } while (0) CHECK_STRING("0.1", 0.1); CHECK_STRING("1.234", 1.234); @@ -978,16 +985,22 @@ test_capsule(PyObject *self, PyObject *Py_UNUSED(ignored)) }; known_capsule *known = &known_capsules[0]; -#define FAIL(x) { error = (x); goto exit; } +#define FAIL(x) \ + do { \ + error = (x); \ + goto exit; \ + } while (0) #define CHECK_DESTRUCTOR \ - if (capsule_error) { \ - FAIL(capsule_error); \ - } \ - else if (!capsule_destructor_call_count) { \ - FAIL("destructor not called!"); \ - } \ - capsule_destructor_call_count = 0; \ + do { \ + if (capsule_error) { \ + FAIL(capsule_error); \ + } \ + else if (!capsule_destructor_call_count) { \ + FAIL("destructor not called!"); \ + } \ + capsule_destructor_call_count = 0; \ + } while (0) object = PyCapsule_New(capsule_pointer, capsule_name, capsule_destructor); PyCapsule_SetContext(object, capsule_context); @@ -1031,12 +1044,12 @@ test_capsule(PyObject *self, PyObject *Py_UNUSED(ignored)) static char buffer[256]; #undef FAIL #define FAIL(x) \ - { \ - sprintf(buffer, "%s module: \"%s\" attribute: \"%s\"", \ - x, known->module, known->attribute); \ - error = buffer; \ - goto exit; \ - } \ + do { \ + sprintf(buffer, "%s module: \"%s\" attribute: \"%s\"", \ + x, known->module, known->attribute); \ + error = buffer; \ + goto exit; \ + } while (0) PyObject *module = PyImport_ImportModule(known->module); if (module) { @@ -1493,36 +1506,6 @@ run_in_subinterp(PyObject *self, PyObject *args) return PyLong_FromLong(r); } -static PyObject * -get_interpreterid_type(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - return Py_NewRef(&PyInterpreterID_Type); -} - -static PyObject * -link_interpreter_refcount(PyObject *self, PyObject *idobj) -{ - PyInterpreterState *interp = PyInterpreterID_LookUp(idobj); - if (interp == NULL) { - assert(PyErr_Occurred()); - return NULL; - } - _PyInterpreterState_RequireIDRef(interp, 1); - Py_RETURN_NONE; -} - -static PyObject * -unlink_interpreter_refcount(PyObject *self, PyObject *idobj) -{ - PyInterpreterState *interp = PyInterpreterID_LookUp(idobj); - if (interp == NULL) { - assert(PyErr_Occurred()); - return NULL; - } - _PyInterpreterState_RequireIDRef(interp, 0); - Py_RETURN_NONE; -} - static PyMethodDef ml; static PyObject * @@ -1761,7 +1744,7 @@ pymarshal_write_long_to_file(PyObject* self, PyObject *args) &value, &filename, &version)) return NULL; - fp = _Py_fopen_obj(filename, "wb"); + fp = Py_fopen(filename, "wb"); if (fp == NULL) { PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -1786,7 +1769,7 @@ pymarshal_write_object_to_file(PyObject* self, PyObject *args) &obj, &filename, &version)) return NULL; - fp = _Py_fopen_obj(filename, "wb"); + fp = Py_fopen(filename, "wb"); if (fp == NULL) { PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -1810,7 +1793,7 @@ pymarshal_read_short_from_file(PyObject* self, PyObject *args) if (!PyArg_ParseTuple(args, "O:pymarshal_read_short_from_file", &filename)) return NULL; - fp = _Py_fopen_obj(filename, "rb"); + fp = Py_fopen(filename, "rb"); if (fp == NULL) { PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -1835,7 +1818,7 @@ pymarshal_read_long_from_file(PyObject* self, PyObject *args) if (!PyArg_ParseTuple(args, "O:pymarshal_read_long_from_file", &filename)) return NULL; - fp = _Py_fopen_obj(filename, "rb"); + fp = Py_fopen(filename, "rb"); if (fp == NULL) { PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -1857,7 +1840,7 @@ pymarshal_read_last_object_from_file(PyObject* self, PyObject *args) if (!PyArg_ParseTuple(args, "O:pymarshal_read_last_object_from_file", &filename)) return NULL; - FILE *fp = _Py_fopen_obj(filename, "rb"); + FILE *fp = Py_fopen(filename, "rb"); if (fp == NULL) { PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -1880,7 +1863,7 @@ pymarshal_read_object_from_file(PyObject* self, PyObject *args) if (!PyArg_ParseTuple(args, "O:pymarshal_read_object_from_file", &filename)) return NULL; - FILE *fp = _Py_fopen_obj(filename, "rb"); + FILE *fp = Py_fopen(filename, "rb"); if (fp == NULL) { PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -1926,25 +1909,6 @@ getitem_with_error(PyObject *self, PyObject *args) return PyObject_GetItem(map, key); } -static PyObject * -dict_get_version(PyObject *self, PyObject *args) -{ - PyDictObject *dict; - uint64_t version; - - if (!PyArg_ParseTuple(args, "O!", &PyDict_Type, &dict)) - return NULL; - - _Py_COMP_DIAG_PUSH - _Py_COMP_DIAG_IGNORE_DEPR_DECLS - version = dict->ma_version_tag; - _Py_COMP_DIAG_POP - - static_assert(sizeof(unsigned long long) >= sizeof(version), - "version is larger than unsigned long long"); - return PyLong_FromUnsignedLongLong((unsigned long long)version); -} - static PyObject * raise_SIGINT_then_send_None(PyObject *self, PyObject *args) @@ -2015,11 +1979,15 @@ test_pythread_tss_key_state(PyObject *self, PyObject *args) "an already initialized key"); } #define CHECK_TSS_API(expr) \ + do { \ (void)(expr); \ if (!PyThread_tss_is_created(&tss_key)) { \ return raiseTestError(self, "test_pythread_tss_key_state", \ "TSS key initialization state was not " \ - "preserved after calling " #expr); } + "preserved after calling " #expr); \ + } \ + } while (0) + CHECK_TSS_API(PyThread_tss_set(&tss_key, NULL)); CHECK_TSS_API(PyThread_tss_get(&tss_key)); #undef CHECK_TSS_API @@ -2341,7 +2309,7 @@ test_py_setref(PyObject *self, PyObject *Py_UNUSED(ignored)) \ Py_DECREF(obj); \ Py_RETURN_NONE; \ - } while (0) \ + } while (0) // Test Py_NewRef() and Py_XNewRef() macros @@ -2440,21 +2408,6 @@ type_modified(PyObject *self, PyObject *type) Py_RETURN_NONE; } -// Circumvents standard version assignment machinery - use with caution and only on -// short-lived heap types -static PyObject * -type_assign_specific_version_unsafe(PyObject *self, PyObject *args) -{ - PyTypeObject *type; - unsigned int version; - if (!PyArg_ParseTuple(args, "Oi:type_assign_specific_version_unsafe", &type, &version)) { - return NULL; - } - assert(!PyType_HasFeature(type, Py_TPFLAGS_IMMUTABLETYPE)); - type->tp_version_tag = version; - type->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; - Py_RETURN_NONE; -} static PyObject * type_assign_version(PyObject *self, PyObject *type) @@ -2520,7 +2473,7 @@ get_basic_static_type(PyObject *self, PyObject *args) PyTypeObject *cls = &BasicStaticTypes[num_basic_static_types_used++]; if (base != NULL) { - cls->tp_bases = Py_BuildValue("(O)", base); + cls->tp_bases = PyTuple_Pack(1, base); if (cls->tp_bases == NULL) { return NULL; } @@ -2683,18 +2636,6 @@ test_frame_getvarstring(PyObject *self, PyObject *args) } -static PyObject * -eval_get_func_name(PyObject *self, PyObject *func) -{ - return PyUnicode_FromString(PyEval_GetFuncName(func)); -} - -static PyObject * -eval_get_func_desc(PyObject *self, PyObject *func) -{ - return PyUnicode_FromString(PyEval_GetFuncDesc(func)); -} - static PyObject * gen_get_code(PyObject *self, PyObject *gen) { @@ -2720,107 +2661,60 @@ eval_eval_code_ex(PyObject *mod, PyObject *pos_args) PyObject **c_kwargs = NULL; - if (!PyArg_UnpackTuple(pos_args, - "eval_code_ex", - 2, - 8, - &code, - &globals, - &locals, - &args, - &kwargs, - &defaults, - &kw_defaults, - &closure)) + if (!PyArg_ParseTuple(pos_args, + "OO|OO!O!O!OO:eval_code_ex", + &code, + &globals, + &locals, + &PyTuple_Type, &args, + &PyDict_Type, &kwargs, + &PyTuple_Type, &defaults, + &kw_defaults, + &closure)) { goto exit; } - if (!PyCode_Check(code)) { - PyErr_SetString(PyExc_TypeError, - "code must be a Python code object"); - goto exit; - } - - if (!PyDict_Check(globals)) { - PyErr_SetString(PyExc_TypeError, "globals must be a dict"); - goto exit; - } - - if (locals && !PyMapping_Check(locals)) { - PyErr_SetString(PyExc_TypeError, "locals must be a mapping"); - goto exit; - } - if (locals == Py_None) { - locals = NULL; - } + NULLABLE(code); + NULLABLE(globals); + NULLABLE(locals); + NULLABLE(kw_defaults); + NULLABLE(closure); PyObject **c_args = NULL; Py_ssize_t c_args_len = 0; - - if (args) - { - if (!PyTuple_Check(args)) { - PyErr_SetString(PyExc_TypeError, "args must be a tuple"); - goto exit; - } else { - c_args = &PyTuple_GET_ITEM(args, 0); - c_args_len = PyTuple_Size(args); - } + if (args) { + c_args = &PyTuple_GET_ITEM(args, 0); + c_args_len = PyTuple_Size(args); } Py_ssize_t c_kwargs_len = 0; + if (kwargs) { + c_kwargs_len = PyDict_Size(kwargs); + if (c_kwargs_len > 0) { + c_kwargs = PyMem_NEW(PyObject*, 2 * c_kwargs_len); + if (!c_kwargs) { + PyErr_NoMemory(); + goto exit; + } - if (kwargs) - { - if (!PyDict_Check(kwargs)) { - PyErr_SetString(PyExc_TypeError, "keywords must be a dict"); - goto exit; - } else { - c_kwargs_len = PyDict_Size(kwargs); - if (c_kwargs_len > 0) { - c_kwargs = PyMem_NEW(PyObject*, 2 * c_kwargs_len); - if (!c_kwargs) { - PyErr_NoMemory(); - goto exit; - } - - Py_ssize_t i = 0; - Py_ssize_t pos = 0; - - while (PyDict_Next(kwargs, - &pos, - &c_kwargs[i], - &c_kwargs[i + 1])) - { - i += 2; - } - c_kwargs_len = i / 2; - /* XXX This is broken if the caller deletes dict items! */ + Py_ssize_t i = 0; + Py_ssize_t pos = 0; + while (PyDict_Next(kwargs, &pos, &c_kwargs[i], &c_kwargs[i + 1])) { + i += 2; } + c_kwargs_len = i / 2; + /* XXX This is broken if the caller deletes dict items! */ } } - PyObject **c_defaults = NULL; Py_ssize_t c_defaults_len = 0; - - if (defaults && PyTuple_Check(defaults)) { + if (defaults) { c_defaults = &PyTuple_GET_ITEM(defaults, 0); c_defaults_len = PyTuple_Size(defaults); } - if (kw_defaults && !PyDict_Check(kw_defaults)) { - PyErr_SetString(PyExc_TypeError, "kw_defaults must be a dict"); - goto exit; - } - - if (closure && !PyTuple_Check(closure)) { - PyErr_SetString(PyExc_TypeError, "closure must be a tuple of cells"); - goto exit; - } - - result = PyEval_EvalCodeEx( code, globals, @@ -3139,49 +3033,30 @@ function_set_kw_defaults(PyObject *self, PyObject *args) } static PyObject * -check_pyimport_addmodule(PyObject *self, PyObject *args) +function_get_closure(PyObject *self, PyObject *func) { - const char *name; - if (!PyArg_ParseTuple(args, "s", &name)) { + PyObject *closure = PyFunction_GetClosure(func); + if (closure != NULL) { + return Py_NewRef(closure); + } else if (PyErr_Occurred()) { return NULL; + } else { + Py_RETURN_NONE; // This can happen when `closure` is set to `None` } +} - // test PyImport_AddModuleRef() - PyObject *module = PyImport_AddModuleRef(name); - if (module == NULL) { +static PyObject * +function_set_closure(PyObject *self, PyObject *args) +{ + PyObject *func = NULL, *closure = NULL; + if (!PyArg_ParseTuple(args, "OO", &func, &closure)) { return NULL; } - assert(PyModule_Check(module)); - // module is a strong reference - - // test PyImport_AddModule() - PyObject *module2 = PyImport_AddModule(name); - if (module2 == NULL) { - goto error; - } - assert(PyModule_Check(module2)); - assert(module2 == module); - // module2 is a borrowed ref - - // test PyImport_AddModuleObject() - PyObject *name_obj = PyUnicode_FromString(name); - if (name_obj == NULL) { - goto error; - } - PyObject *module3 = PyImport_AddModuleObject(name_obj); - Py_DECREF(name_obj); - if (module3 == NULL) { - goto error; + int result = PyFunction_SetClosure(func, closure); + if (result == -1) { + return NULL; } - assert(PyModule_Check(module3)); - assert(module3 == module); - // module3 is a borrowed ref - - return module; - -error: - Py_DECREF(module); - return NULL; + Py_RETURN_NONE; } @@ -3223,6 +3098,7 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) PyObject *ref = UNINITIALIZED_PTR; assert(PyWeakref_GetRef(weakref, &ref) == 1); assert(ref == obj); + assert(!PyWeakref_IsDead(weakref)); assert(Py_REFCNT(obj) == (refcnt + 1)); Py_DECREF(ref); @@ -3238,6 +3114,8 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) assert(Py_REFCNT(obj) == 1); Py_DECREF(obj); + assert(PyWeakref_IsDead(weakref)); + // test PyWeakref_GET_OBJECT(), reference is dead assert(PyWeakref_GET_OBJECT(weakref) == Py_None); @@ -3260,6 +3138,12 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) PyErr_Clear(); assert(ref == NULL); + // test PyWeakRef_IsDead(), invalid type + assert(!PyErr_Occurred()); + assert(PyWeakref_IsDead(invalid_weakref) == -1); + assert(PyErr_ExceptionMatches(PyExc_TypeError)); + PyErr_Clear(); + // test PyWeakref_GetObject(), invalid type assert(PyWeakref_GetObject(invalid_weakref) == NULL); assert(PyErr_ExceptionMatches(PyExc_SystemError)); @@ -3272,6 +3156,11 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) assert(ref == NULL); PyErr_Clear(); + // test PyWeakref_IsDead(NULL) + assert(PyWeakref_IsDead(NULL) == -1); + assert(PyErr_ExceptionMatches(PyExc_SystemError)); + PyErr_Clear(); + // test PyWeakref_GetObject(NULL) assert(PyWeakref_GetObject(NULL) == NULL); assert(PyErr_ExceptionMatches(PyExc_SystemError)); @@ -3284,6 +3173,320 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) _Py_COMP_DIAG_POP } +struct simpletracer_data { + int create_count; + int destroy_count; + void* addresses[10]; +}; + +static int _simpletracer(PyObject *obj, PyRefTracerEvent event, void* data) { + struct simpletracer_data* the_data = (struct simpletracer_data*)data; + assert(the_data->create_count + the_data->destroy_count < (int)Py_ARRAY_LENGTH(the_data->addresses)); + the_data->addresses[the_data->create_count + the_data->destroy_count] = obj; + if (event == PyRefTracer_CREATE) { + the_data->create_count++; + } else { + the_data->destroy_count++; + } + return 0; +} + +static PyObject * +test_reftracer(PyObject *ob, PyObject *Py_UNUSED(ignored)) +{ + // Save the current tracer and data to restore it later + void* current_data; + PyRefTracer current_tracer = PyRefTracer_GetTracer(¤t_data); + + struct simpletracer_data tracer_data = {0}; + void* the_data = &tracer_data; + // Install a simple tracer function + if (PyRefTracer_SetTracer(_simpletracer, the_data) != 0) { + goto failed; + } + + // Check that the tracer was correctly installed + void* data; + if (PyRefTracer_GetTracer(&data) != _simpletracer || data != the_data) { + PyErr_SetString(PyExc_AssertionError, "The reftracer not correctly installed"); + (void)PyRefTracer_SetTracer(NULL, NULL); + goto failed; + } + + // Create a bunch of objects + PyObject* obj = PyList_New(0); + if (obj == NULL) { + goto failed; + } + PyObject* obj2 = PyDict_New(); + if (obj2 == NULL) { + Py_DECREF(obj); + goto failed; + } + + // Kill all objects + Py_DECREF(obj); + Py_DECREF(obj2); + + // Remove the tracer + (void)PyRefTracer_SetTracer(NULL, NULL); + + // Check that the tracer was removed + if (PyRefTracer_GetTracer(&data) != NULL || data != NULL) { + PyErr_SetString(PyExc_ValueError, "The reftracer was not correctly removed"); + goto failed; + } + + if (tracer_data.create_count != 2 || + tracer_data.addresses[0] != obj || + tracer_data.addresses[1] != obj2) { + PyErr_SetString(PyExc_ValueError, "The object creation was not correctly traced"); + goto failed; + } + + if (tracer_data.destroy_count != 2 || + tracer_data.addresses[2] != obj || + tracer_data.addresses[3] != obj2) { + PyErr_SetString(PyExc_ValueError, "The object destruction was not correctly traced"); + goto failed; + } + PyRefTracer_SetTracer(current_tracer, current_data); + Py_RETURN_NONE; +failed: + PyRefTracer_SetTracer(current_tracer, current_data); + return NULL; +} + +static PyObject * +function_set_warning(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + if (PyErr_WarnEx(PyExc_RuntimeWarning, "Testing PyErr_WarnEx", 2)) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +test_critical_sections(PyObject *module, PyObject *Py_UNUSED(args)) +{ + Py_BEGIN_CRITICAL_SECTION(module); + Py_END_CRITICAL_SECTION(); + + Py_BEGIN_CRITICAL_SECTION2(module, module); + Py_END_CRITICAL_SECTION2(); + + Py_RETURN_NONE; +} + + +// Used by `finalize_thread_hang`. +#ifdef _POSIX_THREADS +static void finalize_thread_hang_cleanup_callback(void *Py_UNUSED(arg)) { + // Should not reach here. + Py_FatalError("pthread thread termination was triggered unexpectedly"); +} +#endif + +// Tests that finalization does not trigger pthread cleanup. +// +// Must be called with a single nullary callable function that should block +// (with GIL released) until finalization is in progress. +static PyObject * +finalize_thread_hang(PyObject *self, PyObject *callback) +{ + // WASI builds some pthread stuff but doesn't have these APIs today? +#if defined(_POSIX_THREADS) && !defined(__wasi__) + pthread_cleanup_push(finalize_thread_hang_cleanup_callback, NULL); +#endif + PyObject_CallNoArgs(callback); + // Should not reach here. + Py_FatalError("thread unexpectedly did not hang"); +#if defined(_POSIX_THREADS) && !defined(__wasi__) + pthread_cleanup_pop(0); +#endif + Py_RETURN_NONE; +} + + +static PyObject * +type_freeze(PyObject *module, PyObject *args) +{ + PyTypeObject *type; + if (!PyArg_ParseTuple(args, "O!", &PyType_Type, &type)) { + return NULL; + } + if (PyType_Freeze(type) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +struct atexit_data { + int called; + PyThreadState *tstate; + PyInterpreterState *interp; +}; + +static void +atexit_callback(void *data) +{ + struct atexit_data *at_data = (struct atexit_data *)data; + // Ensure that the callback is from the same interpreter + assert(PyThreadState_Get() == at_data->tstate); + assert(PyInterpreterState_Get() == at_data->interp); + ++at_data->called; +} + +static PyObject * +test_atexit(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyThreadState *oldts = PyThreadState_Swap(NULL); + PyThreadState *tstate = Py_NewInterpreter(); + + struct atexit_data data = {0}; + data.tstate = PyThreadState_Get(); + data.interp = PyInterpreterState_Get(); + + int amount = 10; + for (int i = 0; i < amount; ++i) + { + int res = PyUnstable_AtExit(tstate->interp, atexit_callback, (void *)&data); + if (res < 0) { + Py_EndInterpreter(tstate); + PyThreadState_Swap(oldts); + PyErr_SetString(PyExc_RuntimeError, "atexit callback failed"); + return NULL; + } + } + + Py_EndInterpreter(tstate); + PyThreadState_Swap(oldts); + + if (data.called != amount) { + PyErr_SetString(PyExc_RuntimeError, "atexit callback not called"); + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject* +code_offset_to_line(PyObject* self, PyObject* const* args, Py_ssize_t nargsf) +{ + Py_ssize_t nargs = _PyVectorcall_NARGS(nargsf); + if (nargs != 2) { + PyErr_SetString(PyExc_TypeError, "code_offset_to_line takes 2 arguments"); + return NULL; + } + int offset; + if (PyLong_AsInt32(args[1], &offset) < 0) { + return NULL; + } + PyCodeObject *code = (PyCodeObject *)args[0]; + if (!PyCode_Check(code)) { + PyErr_SetString(PyExc_TypeError, "first arg must be a code object"); + return NULL; + } + return PyLong_FromInt32(PyCode_Addr2Line(code, offset)); +} + + +static void +tracemalloc_track_race_thread(void *data) +{ + PyTraceMalloc_Track(123, 10, 1); + PyTraceMalloc_Untrack(123, 10); + + PyThread_type_lock lock = (PyThread_type_lock)data; + PyThread_release_lock(lock); +} + +// gh-128679: Test fix for tracemalloc.stop() race condition +static PyObject * +tracemalloc_track_race(PyObject *self, PyObject *args) +{ +#define NTHREAD 50 + PyObject *tracemalloc = NULL; + PyObject *stop = NULL; + PyThread_type_lock locks[NTHREAD]; + memset(locks, 0, sizeof(locks)); + + // Call tracemalloc.start() + tracemalloc = PyImport_ImportModule("tracemalloc"); + if (tracemalloc == NULL) { + goto error; + } + PyObject *start = PyObject_GetAttrString(tracemalloc, "start"); + if (start == NULL) { + goto error; + } + PyObject *res = PyObject_CallNoArgs(start); + Py_DECREF(start); + if (res == NULL) { + goto error; + } + Py_DECREF(res); + + stop = PyObject_GetAttrString(tracemalloc, "stop"); + Py_CLEAR(tracemalloc); + if (stop == NULL) { + goto error; + } + + // Start threads + for (size_t i = 0; i < NTHREAD; i++) { + PyThread_type_lock lock = PyThread_allocate_lock(); + if (!lock) { + PyErr_NoMemory(); + goto error; + } + locks[i] = lock; + PyThread_acquire_lock(lock, 1); + + unsigned long thread; + thread = PyThread_start_new_thread(tracemalloc_track_race_thread, + (void*)lock); + if (thread == (unsigned long)-1) { + PyErr_SetString(PyExc_RuntimeError, "can't start new thread"); + goto error; + } + } + + // Call tracemalloc.stop() while threads are running + res = PyObject_CallNoArgs(stop); + Py_CLEAR(stop); + if (res == NULL) { + goto error; + } + Py_DECREF(res); + + // Wait until threads complete with the GIL released + Py_BEGIN_ALLOW_THREADS + for (size_t i = 0; i < NTHREAD; i++) { + PyThread_type_lock lock = locks[i]; + PyThread_acquire_lock(lock, 1); + PyThread_release_lock(lock); + } + Py_END_ALLOW_THREADS + + // Free threads locks + for (size_t i=0; i < NTHREAD; i++) { + PyThread_type_lock lock = locks[i]; + PyThread_free_lock(lock); + } + Py_RETURN_NONE; + +error: + Py_CLEAR(tracemalloc); + Py_CLEAR(stop); + for (size_t i=0; i < NTHREAD; i++) { + PyThread_type_lock lock = locks[i]; + if (lock) { + PyThread_free_lock(lock); + } + } + return NULL; +#undef NTHREAD +} static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, @@ -3317,15 +3520,20 @@ static PyMethodDef TestMethods[] = { {"test_buildvalue_N", test_buildvalue_N, METH_NOARGS}, {"test_get_statictype_slots", test_get_statictype_slots, METH_NOARGS}, {"get_heaptype_for_name", get_heaptype_for_name, METH_NOARGS}, - {"test_get_type_name", test_get_type_name, METH_NOARGS}, - {"test_get_type_qualname", test_get_type_qualname, METH_NOARGS}, + {"get_type_name", get_type_name, METH_O}, + {"get_type_qualname", get_type_qualname, METH_O}, + {"get_type_fullyqualname", get_type_fullyqualname, METH_O}, + {"get_type_module_name", get_type_module_name, METH_O}, {"test_get_type_dict", test_get_type_dict, METH_NOARGS}, + {"test_reftracer", test_reftracer, METH_NOARGS}, {"_test_thread_state", test_thread_state, METH_VARARGS}, + {"gilstate_ensure_release", gilstate_ensure_release, METH_NOARGS}, #ifndef MS_WINDOWS {"_spawn_pthread_waiter", spawn_pthread_waiter, METH_NOARGS}, {"_end_spawned_pthread", end_spawned_pthread, METH_NOARGS}, #endif - {"_pending_threadfunc", pending_threadfunc, METH_VARARGS}, + {"_pending_threadfunc", _PyCFunction_CAST(pending_threadfunc), + METH_VARARGS|METH_KEYWORDS}, #ifdef HAVE_GETTIMEOFDAY {"profile_int", profile_int, METH_NOARGS}, #endif @@ -3338,9 +3546,6 @@ static PyMethodDef TestMethods[] = { {"crash_no_current_thread", crash_no_current_thread, METH_NOARGS}, {"test_current_tstate_matches", test_current_tstate_matches, METH_NOARGS}, {"run_in_subinterp", run_in_subinterp, METH_VARARGS}, - {"get_interpreterid_type", get_interpreterid_type, METH_NOARGS}, - {"link_interpreter_refcount", link_interpreter_refcount, METH_O}, - {"unlink_interpreter_refcount", unlink_interpreter_refcount, METH_O}, {"create_cfunction", create_cfunction, METH_NOARGS}, {"call_in_temporary_c_thread", call_in_temporary_c_thread, METH_VARARGS, PyDoc_STR("set_error_class(error_class) -> None")}, @@ -3361,7 +3566,6 @@ static PyMethodDef TestMethods[] = { {"return_result_with_error", return_result_with_error, METH_NOARGS}, {"getitem_with_error", getitem_with_error, METH_VARARGS}, {"Py_CompileString", pycompilestring, METH_O}, - {"dict_get_version", dict_get_version, METH_VARARGS}, {"raise_SIGINT_then_send_None", raise_SIGINT_then_send_None, METH_VARARGS}, {"stack_pointer", stack_pointer, METH_NOARGS}, #ifdef W_STOPCODE @@ -3390,8 +3594,6 @@ static PyMethodDef TestMethods[] = { {"test_py_is_funcs", test_py_is_funcs, METH_NOARGS}, {"type_get_version", type_get_version, METH_O, PyDoc_STR("type->tp_version_tag")}, {"type_modified", type_modified, METH_O, PyDoc_STR("PyType_Modified")}, - {"type_assign_specific_version_unsafe", type_assign_specific_version_unsafe, METH_VARARGS, - PyDoc_STR("forcefully assign type->tp_version_tag")}, {"type_assign_version", type_assign_version, METH_O, PyDoc_STR("PyUnstable_Type_AssignVersionTag")}, {"type_get_tp_bases", type_get_tp_bases, METH_O}, {"type_get_tp_mro", type_get_tp_mro, METH_O}, @@ -3405,8 +3607,6 @@ static PyMethodDef TestMethods[] = { {"frame_new", frame_new, METH_VARARGS, NULL}, {"frame_getvar", test_frame_getvar, METH_VARARGS, NULL}, {"frame_getvarstring", test_frame_getvarstring, METH_VARARGS, NULL}, - {"eval_get_func_name", eval_get_func_name, METH_O, NULL}, - {"eval_get_func_desc", eval_get_func_desc, METH_O, NULL}, {"gen_get_code", gen_get_code, METH_O, NULL}, {"get_feature_macros", get_feature_macros, METH_NOARGS, NULL}, {"test_code_api", test_code_api, METH_NOARGS, NULL}, @@ -3421,8 +3621,16 @@ static PyMethodDef TestMethods[] = { {"function_set_defaults", function_set_defaults, METH_VARARGS, NULL}, {"function_get_kw_defaults", function_get_kw_defaults, METH_O, NULL}, {"function_set_kw_defaults", function_set_kw_defaults, METH_VARARGS, NULL}, - {"check_pyimport_addmodule", check_pyimport_addmodule, METH_VARARGS}, + {"function_get_closure", function_get_closure, METH_O, NULL}, + {"function_set_closure", function_set_closure, METH_VARARGS, NULL}, {"test_weakref_capi", test_weakref_capi, METH_NOARGS}, + {"function_set_warning", function_set_warning, METH_NOARGS}, + {"test_critical_sections", test_critical_sections, METH_NOARGS}, + {"finalize_thread_hang", finalize_thread_hang, METH_O, NULL}, + {"type_freeze", type_freeze, METH_VARARGS}, + {"test_atexit", test_atexit, METH_NOARGS}, + {"code_offset_to_line", _PyCFunction_CAST(code_offset_to_line), METH_FASTCALL}, + {"tracemalloc_track_race", tracemalloc_track_race, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; @@ -3527,7 +3735,7 @@ static PyTypeObject matmulType = { 0, 0, PyType_GenericNew, /* tp_new */ - PyObject_Del, /* tp_free */ + PyObject_Free, /* tp_free */ }; typedef struct { @@ -3537,7 +3745,7 @@ typedef struct { static PyObject * ipowType_ipow(PyObject *self, PyObject *other, PyObject *mod) { - return Py_BuildValue("OO", other, mod); + return PyTuple_Pack(2, other, mod); } static PyNumberMethods ipowType_as_number = { @@ -3639,7 +3847,7 @@ static PyTypeObject awaitType = { 0, 0, awaitObject_new, /* tp_new */ - PyObject_Del, /* tp_free */ + PyObject_Free, /* tp_free */ }; @@ -3895,6 +4103,61 @@ static PyTypeObject ContainerNoGC_type = { .tp_new = ContainerNoGC_new, }; +/* Manually allocated heap type */ + +typedef struct { + PyObject_HEAD + PyObject *dict; +} ManualHeapType; + +static int +ManualHeapType_traverse(PyObject *self, visitproc visit, void *arg) +{ + ManualHeapType *mht = (ManualHeapType *)self; + Py_VISIT(mht->dict); + return 0; +} + +static void +ManualHeapType_dealloc(PyObject *self) +{ + ManualHeapType *mht = (ManualHeapType *)self; + PyObject_GC_UnTrack(self); + Py_XDECREF(mht->dict); + PyTypeObject *type = Py_TYPE(self); + Py_TYPE(self)->tp_free(self); + Py_DECREF(type); +} + +static PyObject * +create_manual_heap_type(void) +{ + // gh-128923: Ensure that a heap type allocated through PyType_Type.tp_alloc + // with minimal initialization works correctly. + PyHeapTypeObject *heap_type = (PyHeapTypeObject *)PyType_Type.tp_alloc(&PyType_Type, 0); + if (heap_type == NULL) { + return NULL; + } + PyTypeObject* type = &heap_type->ht_type; + type->tp_basicsize = sizeof(ManualHeapType); + type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE | Py_TPFLAGS_HAVE_GC; + type->tp_new = PyType_GenericNew; + type->tp_name = "ManualHeapType"; + type->tp_dictoffset = offsetof(ManualHeapType, dict); + type->tp_traverse = ManualHeapType_traverse; + type->tp_dealloc = ManualHeapType_dealloc; + heap_type->ht_name = PyUnicode_FromString(type->tp_name); + if (!heap_type->ht_name) { + Py_DECREF(type); + return NULL; + } + heap_type->ht_qualname = Py_NewRef(heap_type->ht_name); + if (PyType_Ready(type) < 0) { + Py_DECREF(type); + return NULL; + } + return (PyObject *)type; +} static struct PyModuleDef _testcapimodule = { PyModuleDef_HEAD_INIT, @@ -3914,9 +4177,14 @@ PyInit__testcapi(void) m = PyModule_Create(&_testcapimodule); if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif Py_SET_TYPE(&_HashInheritanceTester_Type, &PyType_Type); - + if (PyType_Ready(&_HashInheritanceTester_Type) < 0) { + return NULL; + } if (PyType_Ready(&matmulType) < 0) return NULL; Py_INCREF(&matmulType); @@ -3988,12 +4256,29 @@ PyInit__testcapi(void) PyModule_AddObject(m, "SIZEOF_WCHAR_T", PyLong_FromSsize_t(sizeof(wchar_t))); PyModule_AddObject(m, "SIZEOF_VOID_P", PyLong_FromSsize_t(sizeof(void*))); PyModule_AddObject(m, "SIZEOF_TIME_T", PyLong_FromSsize_t(sizeof(time_t))); + PyModule_AddObject(m, "SIZEOF_PID_T", PyLong_FromSsize_t(sizeof(pid_t))); PyModule_AddObject(m, "Py_Version", PyLong_FromUnsignedLong(Py_Version)); Py_INCREF(&PyInstanceMethod_Type); PyModule_AddObject(m, "instancemethod", (PyObject *)&PyInstanceMethod_Type); PyModule_AddIntConstant(m, "the_number_three", 3); PyModule_AddIntMacro(m, Py_C_RECURSION_LIMIT); + PyModule_AddObject(m, "INT32_MIN", PyLong_FromInt32(INT32_MIN)); + PyModule_AddObject(m, "INT32_MAX", PyLong_FromInt32(INT32_MAX)); + PyModule_AddObject(m, "UINT32_MAX", PyLong_FromUInt32(UINT32_MAX)); + PyModule_AddObject(m, "INT64_MIN", PyLong_FromInt64(INT64_MIN)); + PyModule_AddObject(m, "INT64_MAX", PyLong_FromInt64(INT64_MAX)); + PyModule_AddObject(m, "UINT64_MAX", PyLong_FromUInt64(UINT64_MAX)); + + if (PyModule_AddIntMacro(m, Py_single_input)) { + return NULL; + } + if (PyModule_AddIntMacro(m, Py_file_input)) { + return NULL; + } + if (PyModule_AddIntMacro(m, Py_eval_input)) { + return NULL; + } testcapistate_t *state = get_testcapi_state(m); state->error = PyErr_NewException("_testcapi.error", NULL, NULL); @@ -4007,6 +4292,15 @@ PyInit__testcapi(void) (PyObject *) &ContainerNoGC_type) < 0) return NULL; + PyObject *manual_heap_type = create_manual_heap_type(); + if (manual_heap_type == NULL) { + return NULL; + } + if (PyModule_Add(m, "ManualHeapType", manual_heap_type) < 0) { + return NULL; + } + + /* Include tests from the _testcapi/ directory */ if (_PyTestCapi_Init_Vectorcall(m) < 0) { return NULL; @@ -4017,9 +4311,6 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Abstract(m) < 0) { return NULL; } - if (_PyTestCapi_Init_ByteArray(m) < 0) { - return NULL; - } if (_PyTestCapi_Init_Bytes(m) < 0) { return NULL; } @@ -4077,18 +4368,12 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Buffer(m) < 0) { return NULL; } - if (_PyTestCapi_Init_PyOS(m) < 0) { - return NULL; - } if (_PyTestCapi_Init_File(m) < 0) { return NULL; } if (_PyTestCapi_Init_Codec(m) < 0) { return NULL; } - if (_PyTestCapi_Init_Sys(m) < 0) { - return NULL; - } if (_PyTestCapi_Init_Immortal(m) < 0) { return NULL; } @@ -4098,10 +4383,7 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_PyAtomic(m) < 0) { return NULL; } - if (_PyTestCapi_Init_VectorcallLimited(m) < 0) { - return NULL; - } - if (_PyTestCapi_Init_HeaptypeRelative(m) < 0) { + if (_PyTestCapi_Init_Run(m) < 0) { return NULL; } if (_PyTestCapi_Init_Hash(m) < 0) { @@ -4110,6 +4392,15 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Time(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Monitoring(m) < 0) { + return NULL; + } + if (_PyTestCapi_Init_Object(m) < 0) { + return NULL; + } + if (_PyTestCapi_Init_Config(m) < 0) { + return NULL; + } PyState_AddModule(m, &_testcapimodule); return m; diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index fb0936bbccd318..a5c4a4c40b3949 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -58,6 +58,21 @@ pack_arguments_newref(int argc, ...) return tuple; } +static PyObject * +pack_arguments_2pos_varpos(PyObject *a, PyObject *b, + PyObject * const *args, Py_ssize_t args_length) +/*[clinic end generated code: output=267032f41bd039cc input=86ee3064b7853e86]*/ +{ + PyObject *tuple = _PyTuple_FromArray(args, args_length); + if (tuple == NULL) { + return NULL; + } + PyObject *result = pack_arguments_newref(3, a, b, tuple); + Py_DECREF(tuple); + return result; +} + + /* Pack arguments to a tuple. * `wrapper` is function which converts primitive type to PyObject. * `arg_type` is type that arguments should be converted to before wrapped. */ @@ -963,70 +978,106 @@ keyword_only_parameter_impl(PyObject *module, PyObject *a) /*[clinic input] -posonly_vararg +varpos + + *args: tuple + +[clinic start generated code]*/ + +static PyObject * +varpos_impl(PyObject *module, PyObject *args) +/*[clinic end generated code: output=7b0b9545872bdca4 input=ae7ccecd997aaff4]*/ +{ + return Py_NewRef(args); +} + + +/*[clinic input] +posonly_varpos a: object - / b: object - *args: object + / + *args: tuple [clinic start generated code]*/ static PyObject * -posonly_vararg_impl(PyObject *module, PyObject *a, PyObject *b, +posonly_varpos_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *args) -/*[clinic end generated code: output=ee6713acda6b954e input=783427fe7ec2b67a]*/ +/*[clinic end generated code: output=5dae5eb2a0d623cd input=6dd74417b62cbe67]*/ { return pack_arguments_newref(3, a, b, args); } /*[clinic input] -vararg_and_posonly +posonly_req_opt_varpos a: object - *args: object + b: object = False / + *args: tuple [clinic start generated code]*/ static PyObject * -vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=42792f799465a14d input=defe017b19ba52e8]*/ +posonly_req_opt_varpos_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *args) +/*[clinic end generated code: output=67f82f90838e166a input=e08ed48926a5b760]*/ { - return pack_arguments_newref(2, a, args); + return pack_arguments_newref(3, a, b, args); +} + + +/*[clinic input] +posonly_poskw_varpos + + a: object + / + b: object + *args: tuple + +[clinic start generated code]*/ + +static PyObject * +posonly_poskw_varpos_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *args) +/*[clinic end generated code: output=bffdb7649941c939 input=e5a2c4cab6745ca1]*/ +{ + return pack_arguments_newref(3, a, b, args); } /*[clinic input] -vararg +poskw_varpos a: object - *args: object + *args: tuple [clinic start generated code]*/ static PyObject * -vararg_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=91ab7a0efc52dd5e input=02c0f772d05f591e]*/ +poskw_varpos_impl(PyObject *module, PyObject *a, PyObject *args) +/*[clinic end generated code: output=2413ddfb5ef22794 input=e78114436a07aefe]*/ { return pack_arguments_newref(2, a, args); } /*[clinic input] -vararg_with_default +poskw_varpos_kwonly_opt a: object - *args: object + *args: tuple b: bool = False [clinic start generated code]*/ static PyObject * -vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, - int b) -/*[clinic end generated code: output=182c01035958ce92 input=68cafa6a79f89e36]*/ +poskw_varpos_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *args, + int b) +/*[clinic end generated code: output=f36d35ba6133463b input=ebecfb189f4a3380]*/ { PyObject *obj_b = b ? Py_True : Py_False; return pack_arguments_newref(3, a, args, obj_b); @@ -1034,28 +1085,141 @@ vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, /*[clinic input] -vararg_with_only_defaults +poskw_varpos_kwonly_opt2 - *args: object - b: object = None + a: object + *args: tuple + b: object = False + c: object = False + +[clinic start generated code]*/ + +static PyObject * +poskw_varpos_kwonly_opt2_impl(PyObject *module, PyObject *a, PyObject *args, + PyObject *b, PyObject *c) +/*[clinic end generated code: output=846cef62c6c40463 input=1aff29829431f711]*/ +{ + return pack_arguments_newref(4, a, args, b, c); +} + + +/*[clinic input] +varpos_kwonly_opt + + *args: tuple + b: object = False [clinic start generated code]*/ static PyObject * -vararg_with_only_defaults_impl(PyObject *module, PyObject *args, PyObject *b) -/*[clinic end generated code: output=c06b1826d91f2f7b input=678c069bc67550e1]*/ +varpos_kwonly_opt_impl(PyObject *module, PyObject *args, PyObject *b) +/*[clinic end generated code: output=3b7bf98b091f5731 input=1bec50dc49fca2eb]*/ { return pack_arguments_newref(2, args, b); } +/*[clinic input] +varpos_kwonly_req_opt + + *args: tuple + a: object + b: object = False + c: object = False + +[clinic start generated code]*/ + +static PyObject * +varpos_kwonly_req_opt_impl(PyObject *module, PyObject *args, PyObject *a, + PyObject *b, PyObject *c) +/*[clinic end generated code: output=165274e1fd037ae9 input=355271f6119bb6c8]*/ +{ + return pack_arguments_newref(4, args, a, b, c); +} + + +/*[clinic input] +varpos_array + + *args: array + +[clinic start generated code]*/ + +static PyObject * +varpos_array_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=a25f42f39c9b13ad input=97b8bdcf87e019c7]*/ +{ + return _PyTuple_FromArray(args, args_length); +} + + +/*[clinic input] +posonly_varpos_array + + a: object + b: object + / + *args: array + +[clinic start generated code]*/ + +static PyObject * +posonly_varpos_array_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject * const *args, Py_ssize_t args_length) +/*[clinic end generated code: output=267032f41bd039cc input=86ee3064b7853e86]*/ +{ + return pack_arguments_2pos_varpos(a, b, args, args_length); +} + + +/*[clinic input] +posonly_req_opt_varpos_array + + a: object + b: object = False + / + *args: array + +[clinic start generated code]*/ + +static PyObject * +posonly_req_opt_varpos_array_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=f2f93c77ead93699 input=b01d7728164fd93e]*/ +{ + return pack_arguments_2pos_varpos(a, b, args, args_length); +} + + +/*[clinic input] +posonly_poskw_varpos_array + + a: object + / + b: object + *args: array + +[clinic start generated code]*/ + +static PyObject * +posonly_poskw_varpos_array_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=155811a8b2d65a12 input=5fb08cdc6afb9d7c]*/ +{ + return pack_arguments_2pos_varpos(a, b, args, args_length); +} + + /*[clinic input] gh_32092_oob pos1: object pos2: object - *varargs: object + *varargs: tuple kw1: object = None kw2: object = None @@ -1066,7 +1230,7 @@ Proof-of-concept of GH-32092 OOB bug. static PyObject * gh_32092_oob_impl(PyObject *module, PyObject *pos1, PyObject *pos2, PyObject *varargs, PyObject *kw1, PyObject *kw2) -/*[clinic end generated code: output=ee259c130054653f input=46d15c881608f8ff]*/ +/*[clinic end generated code: output=ee259c130054653f input=63aeeca881979b91]*/ { Py_RETURN_NONE; } @@ -1076,7 +1240,7 @@ gh_32092_oob_impl(PyObject *module, PyObject *pos1, PyObject *pos2, gh_32092_kw_pass pos: object - *args: object + *args: tuple kw: object = None Proof-of-concept of GH-32092 keyword args passing bug. @@ -1086,7 +1250,7 @@ Proof-of-concept of GH-32092 keyword args passing bug. static PyObject * gh_32092_kw_pass_impl(PyObject *module, PyObject *pos, PyObject *args, PyObject *kw) -/*[clinic end generated code: output=4a2bbe4f7c8604e9 input=5c0bd5b9079a0cce]*/ +/*[clinic end generated code: output=4a2bbe4f7c8604e9 input=258987971f3ee97a]*/ { Py_RETURN_NONE; } @@ -1095,8 +1259,7 @@ gh_32092_kw_pass_impl(PyObject *module, PyObject *pos, PyObject *args, /*[clinic input] gh_99233_refcount - *args: object - / + *args: tuple Proof-of-concept of GH-99233 refcount error bug. @@ -1104,7 +1267,7 @@ Proof-of-concept of GH-99233 refcount error bug. static PyObject * gh_99233_refcount_impl(PyObject *module, PyObject *args) -/*[clinic end generated code: output=585855abfbca9a7f input=85f5fb47ac91a626]*/ +/*[clinic end generated code: output=585855abfbca9a7f input=f5204359fd852613]*/ { Py_RETURN_NONE; } @@ -1132,7 +1295,7 @@ gh_99240_double_free_impl(PyObject *module, char *a, char *b) null_or_tuple_for_varargs name: object - *constraints: object + *constraints: tuple covariant: bool = False See https://github.com/python/cpython/issues/110864 @@ -1141,7 +1304,7 @@ See https://github.com/python/cpython/issues/110864 static PyObject * null_or_tuple_for_varargs_impl(PyObject *module, PyObject *name, PyObject *constraints, int covariant) -/*[clinic end generated code: output=a785b35421358983 input=c9bce186637956b3]*/ +/*[clinic end generated code: output=a785b35421358983 input=4df930e019f32878]*/ { assert(name != NULL); assert(constraints != NULL); @@ -1214,26 +1377,257 @@ clone_with_conv_f2_impl(PyObject *module, custom_t path) /*[clinic input] -class _testclinic.TestClass "PyObject *" "PyObject" +class _testclinic.TestClass "PyObject *" "&PyBaseObject_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=668a591c65bec947]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c991635bb3c91f1a]*/ /*[clinic input] -_testclinic.TestClass.meth_method_no_params +_testclinic.TestClass.get_defining_class cls: defining_class +[clinic start generated code]*/ + +static PyObject * +_testclinic_TestClass_get_defining_class_impl(PyObject *self, + PyTypeObject *cls) +/*[clinic end generated code: output=94f9b0b5f7add930 input=537c59417471dee3]*/ +{ + return Py_NewRef(cls); +} + +/*[clinic input] +_testclinic.TestClass.get_defining_class_arg + cls: defining_class + arg: object +[clinic start generated code]*/ + +static PyObject * +_testclinic_TestClass_get_defining_class_arg_impl(PyObject *self, + PyTypeObject *cls, + PyObject *arg) +/*[clinic end generated code: output=fe7e49d96cbb7718 input=d1b83d3b853af6d9]*/ +{ + return PyTuple_Pack(2, cls, arg); +} + +/*[clinic input] +_testclinic.TestClass.defclass_varpos + cls: defining_class + *args: tuple +[clinic start generated code]*/ + +static PyObject * +_testclinic_TestClass_defclass_varpos_impl(PyObject *self, PyTypeObject *cls, + PyObject *args) +/*[clinic end generated code: output=fad33f2d3a8d778d input=332043286e393d38]*/ +{ + return PyTuple_Pack(2, cls, args); +} + +/*[clinic input] +_testclinic.TestClass.defclass_posonly_varpos + cls: defining_class + a: object + b: object / + *args: tuple [clinic start generated code]*/ static PyObject * -_testclinic_TestClass_meth_method_no_params_impl(PyObject *self, - PyTypeObject *cls) -/*[clinic end generated code: output=c140f100080c2fc8 input=6bd34503d11c63c1]*/ +_testclinic_TestClass_defclass_posonly_varpos_impl(PyObject *self, + PyTypeObject *cls, + PyObject *a, PyObject *b, + PyObject *args) +/*[clinic end generated code: output=1740fcf48d230b07 input=191da4557203c413]*/ { - Py_RETURN_NONE; + return pack_arguments_newref(4, cls, a, b, args); +} + + +/*[clinic input] +@classmethod +_testclinic.TestClass.__new__ as varpos_no_fastcall + + *args: tuple + +[clinic start generated code]*/ + +static PyObject * +varpos_no_fastcall_impl(PyTypeObject *type, PyObject *args) +/*[clinic end generated code: output=04e94f2898bb2dde input=c5d3d30a6589f97f]*/ +{ + return Py_NewRef(args); +} + + +/*[clinic input] +@classmethod +_testclinic.TestClass.__new__ as posonly_varpos_no_fastcall + + a: object + b: object + / + *args: tuple + +[clinic start generated code]*/ + +static PyObject * +posonly_varpos_no_fastcall_impl(PyTypeObject *type, PyObject *a, PyObject *b, + PyObject *args) +/*[clinic end generated code: output=b0a0425719f69f5a input=10f29f2c2c6bfdc4]*/ +{ + return pack_arguments_newref(3, a, b, args); +} + + +/*[clinic input] +@classmethod +_testclinic.TestClass.__new__ as posonly_req_opt_varpos_no_fastcall + + a: object + b: object = False + / + *args: tuple + +[clinic start generated code]*/ + +static PyObject * +posonly_req_opt_varpos_no_fastcall_impl(PyTypeObject *type, PyObject *a, + PyObject *b, PyObject *args) +/*[clinic end generated code: output=3c44915b1a554e2d input=d319302a8748147c]*/ +{ + return pack_arguments_newref(3, a, b, args); +} + + +/*[clinic input] +@classmethod +_testclinic.TestClass.__new__ as posonly_poskw_varpos_no_fastcall + + a: object + / + b: object + *args: tuple + +[clinic start generated code]*/ + +static PyObject * +posonly_poskw_varpos_no_fastcall_impl(PyTypeObject *type, PyObject *a, + PyObject *b, PyObject *args) +/*[clinic end generated code: output=6ad74bed4bdc7f96 input=1f8c113e749414a3]*/ +{ + return pack_arguments_newref(3, a, b, args); +} + + +/*[clinic input] +@classmethod +_testclinic.TestClass.__new__ as varpos_array_no_fastcall + + *args: array + +[clinic start generated code]*/ + +static PyObject * +varpos_array_no_fastcall_impl(PyTypeObject *type, PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=f99d984346c60d42 input=368d8eea6de48c12]*/ +{ + return _PyTuple_FromArray(args, args_length); +} + + +/*[clinic input] +@classmethod +_testclinic.TestClass.__new__ as posonly_varpos_array_no_fastcall + + a: object + b: object + / + *args: array + +[clinic start generated code]*/ + +static PyObject * +posonly_varpos_array_no_fastcall_impl(PyTypeObject *type, PyObject *a, + PyObject *b, PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=1eec4da1fb5b5978 input=7330c8d819a23548]*/ +{ + return pack_arguments_2pos_varpos(a, b, args, args_length); +} + + +/*[clinic input] +@classmethod +_testclinic.TestClass.__new__ as posonly_req_opt_varpos_array_no_fastcall + + a: object + b: object = False + / + *args: array + +[clinic start generated code]*/ + +static PyObject * +posonly_req_opt_varpos_array_no_fastcall_impl(PyTypeObject *type, + PyObject *a, PyObject *b, + PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=88041c2176135218 input=7f5fd34ee5f9e0bf]*/ +{ + return pack_arguments_2pos_varpos(a, b, args, args_length); +} + + +/*[clinic input] +@classmethod +_testclinic.TestClass.__new__ as posonly_poskw_varpos_array_no_fastcall + + a: object + / + b: object + *args: array + +[clinic start generated code]*/ + +static PyObject * +posonly_poskw_varpos_array_no_fastcall_impl(PyTypeObject *type, PyObject *a, + PyObject *b, + PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=70eda18c3667681e input=2b0fcd7bd9bb865c]*/ +{ + return pack_arguments_2pos_varpos(a, b, args, args_length); } static struct PyMethodDef test_class_methods[] = { - _TESTCLINIC_TESTCLASS_METH_METHOD_NO_PARAMS_METHODDEF + _TESTCLINIC_TESTCLASS_GET_DEFINING_CLASS_METHODDEF + _TESTCLINIC_TESTCLASS_GET_DEFINING_CLASS_ARG_METHODDEF + _TESTCLINIC_TESTCLASS_DEFCLASS_VARPOS_METHODDEF + _TESTCLINIC_TESTCLASS_DEFCLASS_POSONLY_VARPOS_METHODDEF + + {"varpos_no_fastcall", _PyCFunction_CAST(varpos_no_fastcall), + METH_VARARGS|METH_KEYWORDS|METH_CLASS, ""}, + {"posonly_varpos_no_fastcall", _PyCFunction_CAST(posonly_varpos_no_fastcall), + METH_VARARGS|METH_KEYWORDS|METH_CLASS, ""}, + {"posonly_req_opt_varpos_no_fastcall", _PyCFunction_CAST(posonly_req_opt_varpos_no_fastcall), + METH_VARARGS|METH_KEYWORDS|METH_CLASS, ""}, + {"posonly_poskw_varpos_no_fastcall", _PyCFunction_CAST(posonly_poskw_varpos_no_fastcall), + METH_VARARGS|METH_KEYWORDS|METH_CLASS, ""}, + + {"varpos_array_no_fastcall", + _PyCFunction_CAST(varpos_array_no_fastcall), + METH_VARARGS|METH_KEYWORDS|METH_CLASS, ""}, + {"posonly_varpos_array_no_fastcall", + _PyCFunction_CAST(posonly_varpos_array_no_fastcall), + METH_VARARGS|METH_KEYWORDS|METH_CLASS, ""}, + {"posonly_req_opt_varpos_array_no_fastcall", + _PyCFunction_CAST(posonly_req_opt_varpos_array_no_fastcall), + METH_VARARGS|METH_KEYWORDS|METH_CLASS, ""}, + {"posonly_poskw_varpos_array_no_fastcall", + _PyCFunction_CAST(posonly_poskw_varpos_array_no_fastcall), + METH_VARARGS|METH_KEYWORDS|METH_CLASS, ""}, + {NULL, NULL} }; @@ -1872,6 +2266,7 @@ static PyMethodDef tester_methods[] = { STR_CONVERTER_METHODDEF STR_CONVERTER_ENCODING_METHODDEF PY_BUFFER_CONVERTER_METHODDEF + KEYWORDS_METHODDEF KEYWORDS_KWONLY_METHODDEF KEYWORDS_OPT_METHODDEF @@ -1888,16 +2283,28 @@ static PyMethodDef tester_methods[] = { POSONLY_KEYWORDS_OPT_KWONLY_OPT_METHODDEF POSONLY_OPT_KEYWORDS_OPT_KWONLY_OPT_METHODDEF KEYWORD_ONLY_PARAMETER_METHODDEF - POSONLY_VARARG_METHODDEF - VARARG_AND_POSONLY_METHODDEF - VARARG_METHODDEF - VARARG_WITH_DEFAULT_METHODDEF - VARARG_WITH_ONLY_DEFAULTS_METHODDEF + + VARPOS_METHODDEF + POSONLY_VARPOS_METHODDEF + POSONLY_REQ_OPT_VARPOS_METHODDEF + POSONLY_POSKW_VARPOS_METHODDEF + POSKW_VARPOS_METHODDEF + POSKW_VARPOS_KWONLY_OPT_METHODDEF + POSKW_VARPOS_KWONLY_OPT2_METHODDEF + VARPOS_KWONLY_OPT_METHODDEF + VARPOS_KWONLY_REQ_OPT_METHODDEF + + VARPOS_ARRAY_METHODDEF + POSONLY_VARPOS_ARRAY_METHODDEF + POSONLY_REQ_OPT_VARPOS_ARRAY_METHODDEF + POSONLY_POSKW_VARPOS_ARRAY_METHODDEF + GH_32092_OOB_METHODDEF GH_32092_KW_PASS_METHODDEF GH_99233_REFCOUNT_METHODDEF GH_99240_DOUBLE_FREE_METHODDEF NULL_OR_TUPLE_FOR_VARARGS_METHODDEF + CLONE_F1_METHODDEF CLONE_F2_METHODDEF CLONE_WITH_CONV_F1_METHODDEF @@ -1940,6 +2347,9 @@ PyInit__testclinic(void) if (m == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif if (PyModule_AddType(m, &TestClass) < 0) { goto error; } diff --git a/Modules/_testclinic_limited.c b/Modules/_testclinic_limited.c index ef595be0b626db..370433b3e2a0d9 100644 --- a/Modules/_testclinic_limited.c +++ b/Modules/_testclinic_limited.c @@ -4,11 +4,10 @@ #undef Py_BUILD_CORE_MODULE #undef Py_BUILD_CORE_BUILTIN +// For now, AC only supports the limited C API version 3.13 #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// For now, only limited C API 3.13 is supported -#define Py_LIMITED_API 0x030d0000 +# define Py_LIMITED_API 0x030d0000 #endif /* Always enable assertions */ @@ -72,10 +71,64 @@ my_int_sum_impl(PyObject *module, int x, int y) } +/*[clinic input] +my_float_sum -> float + + x: float + y: float + / + +[clinic start generated code]*/ + +static float +my_float_sum_impl(PyObject *module, float x, float y) +/*[clinic end generated code: output=634f59a5a419cad7 input=d4b5313bdf4dc377]*/ +{ + return x + y; +} + + +/*[clinic input] +my_double_sum -> double + + x: double + y: double + / + +[clinic start generated code]*/ + +static double +my_double_sum_impl(PyObject *module, double x, double y) +/*[clinic end generated code: output=a75576d9e4d8557f input=16b11c8aba172801]*/ +{ + return x + y; +} + + +/*[clinic input] +get_file_descriptor -> int + + file as fd: fildes + / + +Get a file descriptor. +[clinic start generated code]*/ + +static int +get_file_descriptor_impl(PyObject *module, int fd) +/*[clinic end generated code: output=80051ebad54db8a8 input=82e2a1418848cd5b]*/ +{ + return fd; +} + + static PyMethodDef tester_methods[] = { TEST_EMPTY_FUNCTION_METHODDEF MY_INT_FUNC_METHODDEF MY_INT_SUM_METHODDEF + MY_FLOAT_SUM_METHODDEF + MY_DOUBLE_SUM_METHODDEF + GET_FILE_DESCRIPTOR_METHODDEF {NULL, NULL} }; @@ -93,5 +146,8 @@ PyInit__testclinic_limited(void) if (m == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif return m; } diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index 4929a7bf5a984e..5a28c0e266226b 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -15,7 +15,13 @@ # include #endif -#ifdef __APPLE__ +#if defined(__APPLE__) +# include +// Older macOS SDKs do not define TARGET_OS_OSX +# if !defined(TARGET_OS_OSX) +# define TARGET_OS_OSX 1 +# endif +# if TARGET_OS_OSX # include # include # include @@ -26,6 +32,7 @@ # include # include # include +# endif #endif #include @@ -44,16 +51,38 @@ # define Py_BUILD_CORE_MODULE 1 #endif #include "Python.h" -#include +#include // _Py_DebugOffsets +#include // FRAME_OWNED_BY_CSTACK +#include // Py_TAG_BITS #ifndef HAVE_PROCESS_VM_READV # define HAVE_PROCESS_VM_READV 0 #endif -#ifdef __APPLE__ -static void* -analyze_macho64(mach_port_t proc_ref, void* base, void* map) -{ +struct _Py_AsyncioModuleDebugOffsets { + struct _asyncio_task_object { + uint64_t size; + uint64_t task_name; + uint64_t task_awaited_by; + uint64_t task_is_task; + uint64_t task_awaited_by_is_set; + uint64_t task_coro; + } asyncio_task_object; + struct _asyncio_thread_state { + uint64_t size; + uint64_t asyncio_running_loop; + uint64_t asyncio_running_task; + } asyncio_thread_state; +}; + +#if defined(__APPLE__) && TARGET_OS_OSX +static uintptr_t +return_section_address( + const char* section, + mach_port_t proc_ref, + uintptr_t base, + void* map +) { struct mach_header_64* hdr = (struct mach_header_64*)map; int ncmds = hdr->ncmds; @@ -63,35 +92,40 @@ analyze_macho64(mach_port_t proc_ref, void* base, void* map) mach_vm_size_t size = 0; mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t); mach_vm_address_t address = (mach_vm_address_t)base; - vm_region_basic_info_data_64_t region_info; + vm_region_basic_info_data_64_t r_info; mach_port_t object_name; + uintptr_t vmaddr = 0; for (int i = 0; cmd_cnt < 2 && i < ncmds; i++) { + if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__TEXT") == 0) { + vmaddr = cmd->vmaddr; + } if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__DATA") == 0) { while (cmd->filesize != size) { address += size; - if (mach_vm_region( - proc_ref, - &address, - &size, - VM_REGION_BASIC_INFO_64, - (vm_region_info_t)®ion_info, // cppcheck-suppress [uninitvar] - &count, - &object_name) - != KERN_SUCCESS) - { - PyErr_SetString(PyExc_RuntimeError, "Cannot get any more VM maps.\n"); - return NULL; + kern_return_t ret = mach_vm_region( + proc_ref, + &address, + &size, + VM_REGION_BASIC_INFO_64, + (vm_region_info_t)&r_info, // cppcheck-suppress [uninitvar] + &count, + &object_name + ); + if (ret != KERN_SUCCESS) { + PyErr_SetString( + PyExc_RuntimeError, "Cannot get any more VM maps.\n"); + return 0; } } - base = (void*)address - cmd->vmaddr; int nsects = cmd->nsects; - struct section_64* sec = - (struct section_64*)((void*)cmd + sizeof(struct segment_command_64)); + struct section_64* sec = (struct section_64*)( + (void*)cmd + sizeof(struct segment_command_64) + ); for (int j = 0; j < nsects; j++) { - if (strcmp(sec[j].sectname, "PyRuntime") == 0) { - return base + sec[j].addr; + if (strcmp(sec[j].sectname, section) == 0) { + return base + sec[j].addr - vmaddr; } } cmd_cnt++; @@ -99,33 +133,39 @@ analyze_macho64(mach_port_t proc_ref, void* base, void* map) cmd = (struct segment_command_64*)((void*)cmd + cmd->cmdsize); } - return NULL; + return 0; } -static void* -analyze_macho(char* path, void* base, mach_vm_size_t size, mach_port_t proc_ref) -{ +static uintptr_t +search_section_in_file( + const char* secname, + char* path, + uintptr_t base, + mach_vm_size_t size, + mach_port_t proc_ref +) { int fd = open(path, O_RDONLY); if (fd == -1) { PyErr_Format(PyExc_RuntimeError, "Cannot open binary %s\n", path); - return NULL; + return 0; } struct stat fs; if (fstat(fd, &fs) == -1) { - PyErr_Format(PyExc_RuntimeError, "Cannot get size of binary %s\n", path); + PyErr_Format( + PyExc_RuntimeError, "Cannot get size of binary %s\n", path); close(fd); - return NULL; + return 0; } void* map = mmap(0, fs.st_size, PROT_READ, MAP_SHARED, fd, 0); if (map == MAP_FAILED) { PyErr_Format(PyExc_RuntimeError, "Cannot map binary %s\n", path); close(fd); - return NULL; + return 0; } - void* result = NULL; + uintptr_t result = 0; struct mach_header_64* hdr = (struct mach_header_64*)map; switch (hdr->magic) { @@ -133,11 +173,13 @@ analyze_macho(char* path, void* base, mach_vm_size_t size, mach_port_t proc_ref) case MH_CIGAM: case FAT_MAGIC: case FAT_CIGAM: - PyErr_SetString(PyExc_RuntimeError, "32-bit Mach-O binaries are not supported"); + PyErr_SetString( + PyExc_RuntimeError, + "32-bit Mach-O binaries are not supported"); break; case MH_MAGIC_64: case MH_CIGAM_64: - result = analyze_macho64(proc_ref, base, map); + result = return_section_address(secname, proc_ref, base, map); break; default: PyErr_SetString(PyExc_RuntimeError, "Unknown Mach-O magic"); @@ -165,9 +207,8 @@ pid_to_task(pid_t pid) return task; } -static void* -get_py_runtime_macos(pid_t pid) -{ +static uintptr_t +search_map_for_section(pid_t pid, const char* secname, const char* substr) { mach_vm_address_t address = 0; mach_vm_size_t size = 0; mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t); @@ -177,12 +218,11 @@ get_py_runtime_macos(pid_t pid) mach_port_t proc_ref = pid_to_task(pid); if (proc_ref == 0) { PyErr_SetString(PyExc_PermissionError, "Cannot get task for PID"); - return NULL; + return 0; } int match_found = 0; char map_filename[MAXPATHLEN + 1]; - void* result_address = NULL; while (mach_vm_region( proc_ref, &address, @@ -190,15 +230,21 @@ get_py_runtime_macos(pid_t pid) VM_REGION_BASIC_INFO_64, (vm_region_info_t)®ion_info, &count, - &object_name) - == KERN_SUCCESS) + &object_name) == KERN_SUCCESS) { - int path_len = proc_regionfilename(pid, address, map_filename, MAXPATHLEN); + int path_len = proc_regionfilename( + pid, address, map_filename, MAXPATHLEN); if (path_len == 0) { address += size; continue; } + if ((region_info.protection & VM_PROT_READ) == 0 + || (region_info.protection & VM_PROT_EXECUTE) == 0) { + address += size; + continue; + } + char* filename = strrchr(map_filename, '/'); if (filename != NULL) { filename++; // Move past the '/' @@ -206,26 +252,20 @@ get_py_runtime_macos(pid_t pid) filename = map_filename; // No path, use the whole string } - // Check if the filename starts with "python" or "libpython" - if (!match_found && strncmp(filename, "python", 6) == 0) { - match_found = 1; - result_address = analyze_macho(map_filename, (void*)address, size, proc_ref); - } - if (strncmp(filename, "libpython", 9) == 0) { + if (!match_found && strncmp(filename, substr, strlen(substr)) == 0) { match_found = 1; - result_address = analyze_macho(map_filename, (void*)address, size, proc_ref); - break; + return search_section_in_file( + secname, map_filename, address, size, proc_ref); } address += size; } - return result_address; + return 0; } -#endif -#ifdef __linux__ -void* -find_python_map_start_address(pid_t pid, char* result_filename) +#elif defined(__linux__) +static uintptr_t +find_map_start_address(pid_t pid, char* result_filename, const char* map) { char maps_file_path[64]; sprintf(maps_file_path, "/proc/%d/maps", pid); @@ -233,17 +273,20 @@ find_python_map_start_address(pid_t pid, char* result_filename) FILE* maps_file = fopen(maps_file_path, "r"); if (maps_file == NULL) { PyErr_SetFromErrno(PyExc_OSError); - return NULL; + return 0; } int match_found = 0; char line[256]; char map_filename[PATH_MAX]; - void* result_address = 0; + uintptr_t result_address = 0; while (fgets(line, sizeof(line), maps_file) != NULL) { unsigned long start_address = 0; - sscanf(line, "%lx-%*x %*s %*s %*s %*s %s", &start_address, map_filename); + sscanf( + line, "%lx-%*x %*s %*s %*s %*s %s", + &start_address, map_filename + ); char* filename = strrchr(map_filename, '/'); if (filename != NULL) { filename++; // Move past the '/' @@ -251,15 +294,9 @@ find_python_map_start_address(pid_t pid, char* result_filename) filename = map_filename; // No path, use the whole string } - // Check if the filename starts with "python" or "libpython" - if (!match_found && strncmp(filename, "python", 6) == 0) { - match_found = 1; - result_address = (void*)start_address; - strcpy(result_filename, map_filename); - } - if (strncmp(filename, "libpython", 9) == 0) { + if (!match_found && strncmp(filename, map, strlen(map)) == 0) { match_found = 1; - result_address = (void*)start_address; + result_address = start_address; strcpy(result_filename, map_filename); break; } @@ -274,18 +311,17 @@ find_python_map_start_address(pid_t pid, char* result_filename) return result_address; } -void* -get_py_runtime_linux(pid_t pid) +static uintptr_t +search_map_for_section(pid_t pid, const char* secname, const char* map) { char elf_file[256]; - void* start_address = (void*)find_python_map_start_address(pid, elf_file); + uintptr_t start_address = find_map_start_address(pid, elf_file, map); if (start_address == 0) { - PyErr_SetString(PyExc_RuntimeError, "No memory map associated with python or libpython found"); - return NULL; + return 0; } - void* result = NULL; + uintptr_t result = 0; void* file_memory = NULL; int fd = open(elf_file, O_RDONLY); @@ -308,20 +344,29 @@ get_py_runtime_linux(pid_t pid) Elf_Ehdr* elf_header = (Elf_Ehdr*)file_memory; - Elf_Shdr* section_header_table = (Elf_Shdr*)(file_memory + elf_header->e_shoff); + Elf_Shdr* section_header_table = + (Elf_Shdr*)(file_memory + elf_header->e_shoff); Elf_Shdr* shstrtab_section = §ion_header_table[elf_header->e_shstrndx]; char* shstrtab = (char*)(file_memory + shstrtab_section->sh_offset); - Elf_Shdr* py_runtime_section = NULL; + Elf_Shdr* section = NULL; for (int i = 0; i < elf_header->e_shnum; i++) { - if (strcmp(".PyRuntime", shstrtab + section_header_table[i].sh_name) == 0) { - py_runtime_section = §ion_header_table[i]; + const char* this_sec_name = ( + shstrtab + + section_header_table[i].sh_name + + 1 // "+1" accounts for the leading "." + ); + + if (strcmp(secname, this_sec_name) == 0) { + section = §ion_header_table[i]; break; } } - Elf_Phdr* program_header_table = (Elf_Phdr*)(file_memory + elf_header->e_phoff); + Elf_Phdr* program_header_table = + (Elf_Phdr*)(file_memory + elf_header->e_phoff); + // Find the first PT_LOAD segment Elf_Phdr* first_load_segment = NULL; for (int i = 0; i < elf_header->e_phnum; i++) { @@ -331,10 +376,16 @@ get_py_runtime_linux(pid_t pid) } } - if (py_runtime_section != NULL && first_load_segment != NULL) { - uintptr_t elf_load_addr = first_load_segment->p_vaddr - - (first_load_segment->p_vaddr % first_load_segment->p_align); - result = start_address + py_runtime_section->sh_addr - elf_load_addr; + if (section != NULL && first_load_segment != NULL) { + uintptr_t elf_load_addr = + first_load_segment->p_vaddr - ( + first_load_segment->p_vaddr % first_load_segment->p_align + ); + result = start_address + (uintptr_t)section->sh_addr - elf_load_addr; + } + else { + PyErr_Format(PyExc_KeyError, + "cannot find map for section %s", secname); } exit: @@ -346,10 +397,37 @@ get_py_runtime_linux(pid_t pid) } return result; } +#else +static uintptr_t +search_map_for_section(pid_t pid, const char* secname, const char* map) +{ + return 0; +} #endif -ssize_t -read_memory(pid_t pid, void* remote_address, size_t len, void* dst) +static uintptr_t +get_py_runtime(pid_t pid) +{ + uintptr_t address = search_map_for_section(pid, "PyRuntime", "libpython"); + if (address == 0) { + address = search_map_for_section(pid, "PyRuntime", "python"); + } + return address; +} + +static uintptr_t +get_async_debug(pid_t pid) +{ + uintptr_t result = search_map_for_section(pid, "AsyncioDebug", "_asyncio.cpython"); + if (result == 0 && !PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, "Cannot find AsyncioDebug section"); + } + return result; +} + + +static ssize_t +read_memory(pid_t pid, uintptr_t remote_address, size_t len, void* dst) { ssize_t total_bytes_read = 0; #if defined(__linux__) && HAVE_PROCESS_VM_READV @@ -373,7 +451,7 @@ read_memory(pid_t pid, void* remote_address, size_t len, void* dst) result += read; } while ((size_t)read != local[0].iov_len); total_bytes_read = result; -#elif defined(__APPLE__) +#elif defined(__APPLE__) && TARGET_OS_OSX ssize_t result = -1; kern_return_t kr = mach_vm_read_overwrite( pid_to_task(pid), @@ -385,13 +463,19 @@ read_memory(pid_t pid, void* remote_address, size_t len, void* dst) if (kr != KERN_SUCCESS) { switch (kr) { case KERN_PROTECTION_FAILURE: - PyErr_SetString(PyExc_PermissionError, "Not enough permissions to read memory"); + PyErr_SetString( + PyExc_PermissionError, + "Not enough permissions to read memory"); break; case KERN_INVALID_ARGUMENT: - PyErr_SetString(PyExc_PermissionError, "Invalid argument to mach_vm_read_overwrite"); + PyErr_SetString( + PyExc_PermissionError, + "Invalid argument to mach_vm_read_overwrite"); break; default: - PyErr_SetString(PyExc_RuntimeError, "Unknown error reading memory"); + PyErr_SetString( + PyExc_RuntimeError, + "Unknown error reading memory"); } return -1; } @@ -402,13 +486,22 @@ read_memory(pid_t pid, void* remote_address, size_t len, void* dst) return total_bytes_read; } -int -read_string(pid_t pid, _Py_DebugOffsets* debug_offsets, void* address, char* buffer, Py_ssize_t size) -{ +static int +read_string( + pid_t pid, + _Py_DebugOffsets* debug_offsets, + uintptr_t address, + char* buffer, + Py_ssize_t size +) { Py_ssize_t len; - ssize_t bytes_read = - read_memory(pid, address + debug_offsets->unicode_object.length, sizeof(Py_ssize_t), &len); - if (bytes_read == -1) { + ssize_t bytes_read = read_memory( + pid, + address + debug_offsets->unicode_object.length, + sizeof(Py_ssize_t), + &len + ); + if (bytes_read < 0) { return -1; } if (len >= size) { @@ -417,51 +510,652 @@ read_string(pid_t pid, _Py_DebugOffsets* debug_offsets, void* address, char* buf } size_t offset = debug_offsets->unicode_object.asciiobject_size; bytes_read = read_memory(pid, address + offset, len, buffer); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } buffer[len] = '\0'; return 0; } -void* -get_py_runtime(pid_t pid) + +static inline int +read_ptr(pid_t pid, uintptr_t address, uintptr_t *ptr_addr) { -#if defined(__linux__) - return get_py_runtime_linux(pid); -#elif defined(__APPLE__) - return get_py_runtime_macos(pid); -#else - return NULL; -#endif + int bytes_read = read_memory(pid, address, sizeof(void*), ptr_addr); + if (bytes_read < 0) { + return -1; + } + return 0; +} + +static inline int +read_ssize_t(pid_t pid, uintptr_t address, Py_ssize_t *size) +{ + int bytes_read = read_memory(pid, address, sizeof(Py_ssize_t), size); + if (bytes_read < 0) { + return -1; + } + return 0; } static int -parse_code_object( - int pid, - PyObject* result, - struct _Py_DebugOffsets* offsets, - void* address, - void** previous_frame) +read_py_ptr(pid_t pid, uintptr_t address, uintptr_t *ptr_addr) +{ + if (read_ptr(pid, address, ptr_addr)) { + return -1; + } + *ptr_addr &= ~Py_TAG_BITS; + return 0; +} + +static int +read_char(pid_t pid, uintptr_t address, char *result) +{ + int bytes_read = read_memory(pid, address, sizeof(char), result); + if (bytes_read < 0) { + return -1; + } + return 0; +} + +static int +read_int(pid_t pid, uintptr_t address, int *result) +{ + int bytes_read = read_memory(pid, address, sizeof(int), result); + if (bytes_read < 0) { + return -1; + } + return 0; +} + +static int +read_unsigned_long(pid_t pid, uintptr_t address, unsigned long *result) +{ + int bytes_read = read_memory(pid, address, sizeof(unsigned long), result); + if (bytes_read < 0) { + return -1; + } + return 0; +} + +static int +read_pyobj(pid_t pid, uintptr_t address, PyObject *ptr_addr) +{ + int bytes_read = read_memory(pid, address, sizeof(PyObject), ptr_addr); + if (bytes_read < 0) { + return -1; + } + return 0; +} + +static PyObject * +read_py_str( + pid_t pid, + _Py_DebugOffsets* debug_offsets, + uintptr_t address, + ssize_t max_len +) { + assert(max_len > 0); + + PyObject *result = NULL; + + char *buf = (char *)PyMem_RawMalloc(max_len); + if (buf == NULL) { + PyErr_NoMemory(); + return NULL; + } + if (read_string(pid, debug_offsets, address, buf, max_len)) { + goto err; + } + + result = PyUnicode_FromString(buf); + if (result == NULL) { + goto err; + } + + PyMem_RawFree(buf); + assert(result != NULL); + return result; + +err: + PyMem_RawFree(buf); + return NULL; +} + +static long +read_py_long(pid_t pid, _Py_DebugOffsets* offsets, uintptr_t address) { - void* address_of_function_name; - read_memory( + unsigned int shift = PYLONG_BITS_IN_DIGIT; + + ssize_t size; + uintptr_t lv_tag; + + int bytes_read = read_memory( + pid, address + offsets->long_object.lv_tag, + sizeof(uintptr_t), + &lv_tag); + if (bytes_read < 0) { + return -1; + } + + int negative = (lv_tag & 3) == 2; + size = lv_tag >> 3; + + if (size == 0) { + return 0; + } + + digit *digits = (digit *)PyMem_RawMalloc(size * sizeof(digit)); + if (!digits) { + PyErr_NoMemory(); + return -1; + } + + bytes_read = read_memory( + pid, + address + offsets->long_object.ob_digit, + sizeof(digit) * size, + digits + ); + if (bytes_read < 0) { + goto error; + } + + long value = 0; + + // In theory this can overflow, but because of llvm/llvm-project#16778 + // we can't use __builtin_mul_overflow because it fails to link with + // __muloti4 on aarch64. In practice this is fine because all we're + // testing here are task numbers that would fit in a single byte. + for (ssize_t i = 0; i < size; ++i) { + long long factor = digits[i] * (1UL << (ssize_t)(shift * i)); + value += factor; + } + PyMem_RawFree(digits); + if (negative) { + value *= -1; + } + return value; +error: + PyMem_RawFree(digits); + return -1; +} + +static PyObject * +parse_task_name( + int pid, + _Py_DebugOffsets* offsets, + struct _Py_AsyncioModuleDebugOffsets* async_offsets, + uintptr_t task_address +) { + uintptr_t task_name_addr; + int err = read_py_ptr( + pid, + task_address + async_offsets->asyncio_task_object.task_name, + &task_name_addr); + if (err) { + return NULL; + } + + // The task name can be a long or a string so we need to check the type + + PyObject task_name_obj; + err = read_pyobj( + pid, + task_name_addr, + &task_name_obj); + if (err) { + return NULL; + } + + unsigned long flags; + err = read_unsigned_long( + pid, + (uintptr_t)task_name_obj.ob_type + offsets->type_object.tp_flags, + &flags); + if (err) { + return NULL; + } + + if ((flags & Py_TPFLAGS_LONG_SUBCLASS)) { + long res = read_py_long(pid, offsets, task_name_addr); + if (res == -1) { + PyErr_SetString(PyExc_RuntimeError, "Failed to get task name"); + return NULL; + } + return PyUnicode_FromFormat("Task-%d", res); + } + + if(!(flags & Py_TPFLAGS_UNICODE_SUBCLASS)) { + PyErr_SetString(PyExc_RuntimeError, "Invalid task name object"); + return NULL; + } + + return read_py_str( + pid, + offsets, + task_name_addr, + 255 + ); +} + +static int +parse_coro_chain( + int pid, + struct _Py_DebugOffsets* offsets, + struct _Py_AsyncioModuleDebugOffsets* async_offsets, + uintptr_t coro_address, + PyObject *render_to +) { + assert((void*)coro_address != NULL); + + uintptr_t gen_type_addr; + int err = read_ptr( + pid, + coro_address + sizeof(void*), + &gen_type_addr); + if (err) { + return -1; + } + + uintptr_t gen_name_addr; + err = read_py_ptr( + pid, + coro_address + offsets->gen_object.gi_name, + &gen_name_addr); + if (err) { + return -1; + } + + PyObject *name = read_py_str( + pid, + offsets, + gen_name_addr, + 255 + ); + if (name == NULL) { + return -1; + } + + if (PyList_Append(render_to, name)) { + return -1; + } + Py_DECREF(name); + + int gi_frame_state; + err = read_int( + pid, + coro_address + offsets->gen_object.gi_frame_state, + &gi_frame_state); + + if (gi_frame_state == FRAME_SUSPENDED_YIELD_FROM) { + char owner; + err = read_char( pid, - (void*)(address + offsets->code_object.name), - sizeof(void*), - &address_of_function_name); + coro_address + offsets->gen_object.gi_iframe + + offsets->interpreter_frame.owner, + &owner + ); + if (err) { + return -1; + } + if (owner != FRAME_OWNED_BY_GENERATOR) { + PyErr_SetString( + PyExc_RuntimeError, + "generator doesn't own its frame \\_o_/"); + return -1; + } - if (address_of_function_name == NULL) { - PyErr_SetString(PyExc_RuntimeError, "No function name found"); + uintptr_t stackpointer_addr; + err = read_py_ptr( + pid, + coro_address + offsets->gen_object.gi_iframe + + offsets->interpreter_frame.stackpointer, + &stackpointer_addr); + if (err) { + return -1; + } + + if ((void*)stackpointer_addr != NULL) { + uintptr_t gi_await_addr; + err = read_py_ptr( + pid, + stackpointer_addr - sizeof(void*), + &gi_await_addr); + if (err) { + return -1; + } + + if ((void*)gi_await_addr != NULL) { + uintptr_t gi_await_addr_type_addr; + int err = read_ptr( + pid, + gi_await_addr + sizeof(void*), + &gi_await_addr_type_addr); + if (err) { + return -1; + } + + if (gen_type_addr == gi_await_addr_type_addr) { + /* This needs an explanation. We always start with parsing + native coroutine / generator frames. Ultimately they + are awaiting on something. That something can be + a native coroutine frame or... an iterator. + If it's the latter -- we can't continue building + our chain. So the condition to bail out of this is + to do that when the type of the current coroutine + doesn't match the type of whatever it points to + in its cr_await. + */ + err = parse_coro_chain( + pid, + offsets, + async_offsets, + gi_await_addr, + render_to + ); + if (err) { + return -1; + } + } + } + } + + } + + return 0; +} + + +static int +parse_task_awaited_by( + int pid, + struct _Py_DebugOffsets* offsets, + struct _Py_AsyncioModuleDebugOffsets* async_offsets, + uintptr_t task_address, + PyObject *awaited_by +); + + +static int +parse_task( + int pid, + struct _Py_DebugOffsets* offsets, + struct _Py_AsyncioModuleDebugOffsets* async_offsets, + uintptr_t task_address, + PyObject *render_to +) { + char is_task; + int err = read_char( + pid, + task_address + async_offsets->asyncio_task_object.task_is_task, + &is_task); + if (err) { return -1; } - char function_name[256]; - if (read_string(pid, offsets, address_of_function_name, function_name, sizeof(function_name)) != 0) { + uintptr_t refcnt; + read_ptr(pid, task_address + sizeof(Py_ssize_t), &refcnt); + + PyObject* result = PyList_New(0); + if (result == NULL) { return -1; } - PyObject* py_function_name = PyUnicode_FromString(function_name); + PyObject *call_stack = PyList_New(0); + if (call_stack == NULL) { + goto err; + } + if (PyList_Append(result, call_stack)) { + Py_DECREF(call_stack); + goto err; + } + /* we can operate on a borrowed one to simplify cleanup */ + Py_DECREF(call_stack); + + if (is_task) { + PyObject *tn = parse_task_name( + pid, offsets, async_offsets, task_address); + if (tn == NULL) { + goto err; + } + if (PyList_Append(result, tn)) { + Py_DECREF(tn); + goto err; + } + Py_DECREF(tn); + + uintptr_t coro_addr; + err = read_py_ptr( + pid, + task_address + async_offsets->asyncio_task_object.task_coro, + &coro_addr); + if (err) { + goto err; + } + + if ((void*)coro_addr != NULL) { + err = parse_coro_chain( + pid, + offsets, + async_offsets, + coro_addr, + call_stack + ); + if (err) { + goto err; + } + + if (PyList_Reverse(call_stack)) { + goto err; + } + } + } + + if (PyList_Append(render_to, result)) { + goto err; + } + Py_DECREF(result); + + PyObject *awaited_by = PyList_New(0); + if (awaited_by == NULL) { + goto err; + } + if (PyList_Append(result, awaited_by)) { + Py_DECREF(awaited_by); + goto err; + } + /* we can operate on a borrowed one to simplify cleanup */ + Py_DECREF(awaited_by); + + if (parse_task_awaited_by(pid, offsets, async_offsets, + task_address, awaited_by) + ) { + goto err; + } + + return 0; + +err: + Py_DECREF(result); + return -1; +} + +static int +parse_tasks_in_set( + int pid, + struct _Py_DebugOffsets* offsets, + struct _Py_AsyncioModuleDebugOffsets* async_offsets, + uintptr_t set_addr, + PyObject *awaited_by +) { + uintptr_t set_obj; + if (read_py_ptr( + pid, + set_addr, + &set_obj) + ) { + return -1; + } + + Py_ssize_t num_els; + if (read_ssize_t( + pid, + set_obj + offsets->set_object.used, + &num_els) + ) { + return -1; + } + + Py_ssize_t set_len; + if (read_ssize_t( + pid, + set_obj + offsets->set_object.mask, + &set_len) + ) { + return -1; + } + set_len++; // The set contains the `mask+1` element slots. + + uintptr_t table_ptr; + if (read_ptr( + pid, + set_obj + offsets->set_object.table, + &table_ptr) + ) { + return -1; + } + + Py_ssize_t i = 0; + Py_ssize_t els = 0; + while (i < set_len) { + uintptr_t key_addr; + if (read_py_ptr(pid, table_ptr, &key_addr)) { + return -1; + } + + if ((void*)key_addr != NULL) { + Py_ssize_t ref_cnt; + if (read_ssize_t(pid, table_ptr, &ref_cnt)) { + return -1; + } + + if (ref_cnt) { + // if 'ref_cnt=0' it's a set dummy marker + + if (parse_task( + pid, + offsets, + async_offsets, + key_addr, + awaited_by) + ) { + return -1; + } + + if (++els == num_els) { + break; + } + } + } + + table_ptr += sizeof(void*) * 2; + i++; + } + return 0; +} + + +static int +parse_task_awaited_by( + int pid, + struct _Py_DebugOffsets* offsets, + struct _Py_AsyncioModuleDebugOffsets* async_offsets, + uintptr_t task_address, + PyObject *awaited_by +) { + uintptr_t task_ab_addr; + int err = read_py_ptr( + pid, + task_address + async_offsets->asyncio_task_object.task_awaited_by, + &task_ab_addr); + if (err) { + return -1; + } + + if ((void*)task_ab_addr == NULL) { + return 0; + } + + char awaited_by_is_a_set; + err = read_char( + pid, + task_address + async_offsets->asyncio_task_object.task_awaited_by_is_set, + &awaited_by_is_a_set); + if (err) { + return -1; + } + + if (awaited_by_is_a_set) { + if (parse_tasks_in_set( + pid, + offsets, + async_offsets, + task_address + async_offsets->asyncio_task_object.task_awaited_by, + awaited_by) + ) { + return -1; + } + } else { + uintptr_t sub_task; + if (read_py_ptr( + pid, + task_address + async_offsets->asyncio_task_object.task_awaited_by, + &sub_task) + ) { + return -1; + } + + if (parse_task( + pid, + offsets, + async_offsets, + sub_task, + awaited_by) + ) { + return -1; + } + } + + return 0; +} + +static int +parse_code_object( + int pid, + PyObject* result, + struct _Py_DebugOffsets* offsets, + uintptr_t address, + uintptr_t* previous_frame +) { + uintptr_t address_of_function_name; + int bytes_read = read_memory( + pid, + address + offsets->code_object.name, + sizeof(void*), + &address_of_function_name + ); + if (bytes_read < 0) { + return -1; + } + + if ((void*)address_of_function_name == NULL) { + PyErr_SetString(PyExc_RuntimeError, "No function name found"); + return -1; + } + + PyObject* py_function_name = read_py_str( + pid, offsets, address_of_function_name, 256); if (py_function_name == NULL) { return -1; } @@ -477,53 +1171,283 @@ parse_code_object( static int parse_frame_object( - int pid, - PyObject* result, - struct _Py_DebugOffsets* offsets, - void* address, - void** previous_frame) -{ + int pid, + PyObject* result, + struct _Py_DebugOffsets* offsets, + uintptr_t address, + uintptr_t* previous_frame +) { + int err; + ssize_t bytes_read = read_memory( + pid, + address + offsets->interpreter_frame.previous, + sizeof(void*), + previous_frame + ); + if (bytes_read < 0) { + return -1; + } + + char owner; + if (read_char(pid, address + offsets->interpreter_frame.owner, &owner)) { + return -1; + } + + if (owner >= FRAME_OWNED_BY_INTERPRETER) { + return 0; + } + + uintptr_t address_of_code_object; + err = read_py_ptr( + pid, + address + offsets->interpreter_frame.executable, + &address_of_code_object + ); + if (err) { + return -1; + } + + if ((void*)address_of_code_object == NULL) { + return 0; + } + + return parse_code_object( + pid, result, offsets, address_of_code_object, previous_frame); +} + +static int +parse_async_frame_object( + int pid, + PyObject* result, + struct _Py_DebugOffsets* offsets, + uintptr_t address, + uintptr_t* previous_frame, + uintptr_t* code_object +) { + int err; + + ssize_t bytes_read = read_memory( + pid, + address + offsets->interpreter_frame.previous, + sizeof(void*), + previous_frame + ); + if (bytes_read < 0) { + return -1; + } + + char owner; + bytes_read = read_memory( + pid, address + offsets->interpreter_frame.owner, sizeof(char), &owner); + if (bytes_read < 0) { + return -1; + } + + if (owner == FRAME_OWNED_BY_CSTACK || owner == FRAME_OWNED_BY_INTERPRETER) { + return 0; // C frame + } + + if (owner != FRAME_OWNED_BY_GENERATOR + && owner != FRAME_OWNED_BY_THREAD) { + PyErr_Format(PyExc_RuntimeError, "Unhandled frame owner %d.\n", owner); + return -1; + } + + err = read_py_ptr( + pid, + address + offsets->interpreter_frame.executable, + code_object + ); + if (err) { + return -1; + } + + assert(code_object != NULL); + if ((void*)*code_object == NULL) { + return 0; + } + + if (parse_code_object( + pid, result, offsets, *code_object, previous_frame)) { + return -1; + } + + return 1; +} + +static int +read_offsets( + int pid, + uintptr_t *runtime_start_address, + _Py_DebugOffsets* debug_offsets +) { + *runtime_start_address = get_py_runtime(pid); + if ((void*)*runtime_start_address == NULL) { + if (!PyErr_Occurred()) { + PyErr_SetString( + PyExc_RuntimeError, "Failed to get .PyRuntime address"); + } + return -1; + } + size_t size = sizeof(struct _Py_DebugOffsets); + ssize_t bytes_read = read_memory( + pid, *runtime_start_address, size, debug_offsets); + if (bytes_read < 0) { + return -1; + } + return 0; +} + +static int +read_async_debug( + int pid, + struct _Py_AsyncioModuleDebugOffsets* async_debug +) { + uintptr_t async_debug_addr = get_async_debug(pid); + if (!async_debug_addr) { + return -1; + } + size_t size = sizeof(struct _Py_AsyncioModuleDebugOffsets); + ssize_t bytes_read = read_memory( + pid, async_debug_addr, size, async_debug); + if (bytes_read < 0) { + return -1; + } + return 0; +} + +static int +find_running_frame( + int pid, + uintptr_t runtime_start_address, + _Py_DebugOffsets* local_debug_offsets, + uintptr_t *frame +) { + off_t interpreter_state_list_head = + local_debug_offsets->runtime_state.interpreters_head; + + uintptr_t address_of_interpreter_state; + int bytes_read = read_memory( pid, - (void*)(address + offsets->interpreter_frame.previous), + runtime_start_address + interpreter_state_list_head, sizeof(void*), - previous_frame); - if (bytes_read == -1) { + &address_of_interpreter_state); + if (bytes_read < 0) { return -1; } - char owner; - bytes_read = - read_memory(pid, (void*)(address + offsets->interpreter_frame.owner), sizeof(char), &owner); + if (address_of_interpreter_state == 0) { + PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); + return -1; + } + + uintptr_t address_of_thread; + bytes_read = read_memory( + pid, + address_of_interpreter_state + + local_debug_offsets->interpreter_state.threads_head, + sizeof(void*), + &address_of_thread); if (bytes_read < 0) { return -1; } - if (owner == FRAME_OWNED_BY_CSTACK) { + // No Python frames are available for us (can happen at tear-down). + if ((void*)address_of_thread != NULL) { + int err = read_ptr( + pid, + address_of_thread + local_debug_offsets->thread_state.current_frame, + frame); + if (err) { + return -1; + } return 0; } - void* address_of_code_object; + *frame = (uintptr_t)NULL; + return 0; +} + +static int +find_running_task( + int pid, + uintptr_t runtime_start_address, + _Py_DebugOffsets *local_debug_offsets, + struct _Py_AsyncioModuleDebugOffsets *async_offsets, + uintptr_t *running_task_addr +) { + *running_task_addr = (uintptr_t)NULL; + + off_t interpreter_state_list_head = + local_debug_offsets->runtime_state.interpreters_head; + + uintptr_t address_of_interpreter_state; + int bytes_read = read_memory( + pid, + runtime_start_address + interpreter_state_list_head, + sizeof(void*), + &address_of_interpreter_state); + if (bytes_read < 0) { + return -1; + } + + if (address_of_interpreter_state == 0) { + PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); + return -1; + } + + uintptr_t address_of_thread; bytes_read = read_memory( pid, - (void*)(address + offsets->interpreter_frame.executable), + address_of_interpreter_state + + local_debug_offsets->interpreter_state.threads_head, sizeof(void*), - &address_of_code_object); + &address_of_thread); + if (bytes_read < 0) { + return -1; + } + + uintptr_t address_of_running_loop; + // No Python frames are available for us (can happen at tear-down). + if ((void*)address_of_thread == NULL) { + return 0; + } + + bytes_read = read_py_ptr( + pid, + address_of_thread + + async_offsets->asyncio_thread_state.asyncio_running_loop, + &address_of_running_loop); if (bytes_read == -1) { return -1; } - if (address_of_code_object == NULL) { + // no asyncio loop is now running + if ((void*)address_of_running_loop == NULL) { return 0; } - return parse_code_object(pid, result, offsets, address_of_code_object, previous_frame); + + int err = read_ptr( + pid, + address_of_thread + + async_offsets->asyncio_thread_state.asyncio_running_task, + running_task_addr); + if (err) { + return -1; + } + + return 0; } static PyObject* get_stack_trace(PyObject* self, PyObject* args) { -#if (!defined(__linux__) && !defined(__APPLE__)) || (defined(__linux__) && !HAVE_PROCESS_VM_READV) - PyErr_SetString(PyExc_RuntimeError, "get_stack_trace is not supported on this platform"); +#if (!defined(__linux__) && !defined(__APPLE__)) || \ + (defined(__linux__) && !HAVE_PROCESS_VM_READV) + PyErr_SetString( + PyExc_RuntimeError, + "get_stack_trace is not supported on this platform"); return NULL; #endif int pid; @@ -532,95 +1456,219 @@ get_stack_trace(PyObject* self, PyObject* args) return NULL; } - void* runtime_start_address = get_py_runtime(pid); - if (runtime_start_address == NULL) { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_RuntimeError, "Failed to get .PyRuntime address"); - } + uintptr_t runtime_start_address = get_py_runtime(pid); + struct _Py_DebugOffsets local_debug_offsets; + + if (read_offsets(pid, &runtime_start_address, &local_debug_offsets)) { return NULL; } - size_t size = sizeof(struct _Py_DebugOffsets); - struct _Py_DebugOffsets local_debug_offsets; - ssize_t bytes_read = read_memory(pid, runtime_start_address, size, &local_debug_offsets); - if (bytes_read == -1) { + uintptr_t address_of_current_frame; + if (find_running_frame( + pid, runtime_start_address, &local_debug_offsets, + &address_of_current_frame) + ) { return NULL; } - off_t thread_state_list_head = local_debug_offsets.runtime_state.interpreters_head; - void* address_of_interpreter_state; - bytes_read = read_memory( - pid, - (void*)(runtime_start_address + thread_state_list_head), - sizeof(void*), - &address_of_interpreter_state); - if (bytes_read == -1) { + PyObject* result = PyList_New(0); + if (result == NULL) { return NULL; } - if (address_of_interpreter_state == NULL) { - PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); + while ((void*)address_of_current_frame != NULL) { + if (parse_frame_object( + pid, + result, + &local_debug_offsets, + address_of_current_frame, + &address_of_current_frame) + < 0) + { + Py_DECREF(result); + return NULL; + } + } + + return result; +} + +static PyObject* +get_async_stack_trace(PyObject* self, PyObject* args) +{ +#if (!defined(__linux__) && !defined(__APPLE__)) || \ + (defined(__linux__) && !HAVE_PROCESS_VM_READV) + PyErr_SetString( + PyExc_RuntimeError, + "get_stack_trace is not supported on this platform"); + return NULL; +#endif + int pid; + + if (!PyArg_ParseTuple(args, "i", &pid)) { return NULL; } - void* address_of_thread; - bytes_read = read_memory( - pid, - (void*)(address_of_interpreter_state + local_debug_offsets.interpreter_state.threads_head), - sizeof(void*), - &address_of_thread); - if (bytes_read == -1) { + uintptr_t runtime_start_address = get_py_runtime(pid); + struct _Py_DebugOffsets local_debug_offsets; + + if (read_offsets(pid, &runtime_start_address, &local_debug_offsets)) { return NULL; } - PyObject* result = PyList_New(0); + struct _Py_AsyncioModuleDebugOffsets local_async_debug; + if (read_async_debug(pid, &local_async_debug)) { + return NULL; + } + + PyObject* result = PyList_New(1); if (result == NULL) { return NULL; } + PyObject* calls = PyList_New(0); + if (calls == NULL) { + return NULL; + } + if (PyList_SetItem(result, 0, calls)) { /* steals ref to 'calls' */ + Py_DECREF(result); + Py_DECREF(calls); + return NULL; + } - // No Python frames are available for us (can happen at tear-down). - if (address_of_thread != NULL) { - void* address_of_current_frame; - (void)read_memory( - pid, - (void*)(address_of_thread + local_debug_offsets.thread_state.current_frame), - sizeof(void*), - &address_of_current_frame); - while (address_of_current_frame != NULL) { - if (parse_frame_object( - pid, - result, - &local_debug_offsets, - address_of_current_frame, - &address_of_current_frame) - < 0) - { - Py_DECREF(result); - return NULL; - } + uintptr_t running_task_addr = (uintptr_t)NULL; + if (find_running_task( + pid, runtime_start_address, &local_debug_offsets, &local_async_debug, + &running_task_addr) + ) { + goto result_err; + } + + if ((void*)running_task_addr == NULL) { + PyErr_SetString(PyExc_RuntimeError, "No running task found"); + goto result_err; + } + + uintptr_t running_coro_addr; + if (read_py_ptr( + pid, + running_task_addr + local_async_debug.asyncio_task_object.task_coro, + &running_coro_addr + )) { + goto result_err; + } + + if ((void*)running_coro_addr == NULL) { + PyErr_SetString(PyExc_RuntimeError, "Running task coro is NULL"); + goto result_err; + } + + // note: genobject's gi_iframe is an embedded struct so the address to + // the offset leads directly to its first field: f_executable + uintptr_t address_of_running_task_code_obj; + if (read_py_ptr( + pid, + running_coro_addr + local_debug_offsets.gen_object.gi_iframe, + &address_of_running_task_code_obj + )) { + goto result_err; + } + + if ((void*)address_of_running_task_code_obj == NULL) { + PyErr_SetString(PyExc_RuntimeError, "Running task code object is NULL"); + goto result_err; + } + + uintptr_t address_of_current_frame; + if (find_running_frame( + pid, runtime_start_address, &local_debug_offsets, + &address_of_current_frame) + ) { + goto result_err; + } + + uintptr_t address_of_code_object; + while ((void*)address_of_current_frame != NULL) { + int res = parse_async_frame_object( + pid, + calls, + &local_debug_offsets, + address_of_current_frame, + &address_of_current_frame, + &address_of_code_object + ); + + if (res < 0) { + goto result_err; } + + if (address_of_code_object == address_of_running_task_code_obj) { + break; + } + } + + PyObject *tn = parse_task_name( + pid, &local_debug_offsets, &local_async_debug, running_task_addr); + if (tn == NULL) { + goto result_err; + } + if (PyList_Append(result, tn)) { + Py_DECREF(tn); + goto result_err; + } + Py_DECREF(tn); + + PyObject* awaited_by = PyList_New(0); + if (awaited_by == NULL) { + goto result_err; + } + if (PyList_Append(result, awaited_by)) { + Py_DECREF(awaited_by); + goto result_err; + } + Py_DECREF(awaited_by); + + if (parse_task_awaited_by( + pid, &local_debug_offsets, &local_async_debug, + running_task_addr, awaited_by) + ) { + goto result_err; } return result; + +result_err: + Py_DECREF(result); + return NULL; } + static PyMethodDef methods[] = { - {"get_stack_trace", get_stack_trace, METH_VARARGS, "Get the Python stack from a given PID"}, - {NULL, NULL, 0, NULL}, + {"get_stack_trace", get_stack_trace, METH_VARARGS, + "Get the Python stack from a given PID"}, + {"get_async_stack_trace", get_async_stack_trace, METH_VARARGS, + "Get the asyncio stack from a given PID"}, + {NULL, NULL, 0, NULL}, }; static struct PyModuleDef module = { - .m_base = PyModuleDef_HEAD_INIT, - .m_name = "_testexternalinspection", - .m_size = -1, - .m_methods = methods, + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_testexternalinspection", + .m_size = -1, + .m_methods = methods, }; PyMODINIT_FUNC PyInit__testexternalinspection(void) { PyObject* mod = PyModule_Create(&module); - int rc = PyModule_AddIntConstant(mod, "PROCESS_VM_READV_SUPPORTED", HAVE_PROCESS_VM_READV); + if (mod == NULL) { + return NULL; + } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); +#endif + int rc = PyModule_AddIntConstant( + mod, "PROCESS_VM_READV_SUPPORTED", HAVE_PROCESS_VM_READV); if (rc < 0) { Py_DECREF(mod); return NULL; diff --git a/Modules/_testimportmultiple.c b/Modules/_testimportmultiple.c index 7e6556ad400cde..c147596f88a3a8 100644 --- a/Modules/_testimportmultiple.c +++ b/Modules/_testimportmultiple.c @@ -5,20 +5,25 @@ */ #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -#define Py_LIMITED_API 0x03020000 +# define Py_LIMITED_API 0x030d0000 #endif #include +static PyModuleDef_Slot shared_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static struct PyModuleDef _testimportmultiple = { PyModuleDef_HEAD_INIT, "_testimportmultiple", "_testimportmultiple doc", - -1, - NULL, + 0, NULL, + shared_slots, NULL, NULL, NULL @@ -26,16 +31,16 @@ static struct PyModuleDef _testimportmultiple = { PyMODINIT_FUNC PyInit__testimportmultiple(void) { - return PyModule_Create(&_testimportmultiple); + return PyModuleDef_Init(&_testimportmultiple); } static struct PyModuleDef _foomodule = { PyModuleDef_HEAD_INIT, "_testimportmultiple_foo", "_testimportmultiple_foo doc", - -1, - NULL, + 0, NULL, + shared_slots, NULL, NULL, NULL @@ -43,21 +48,21 @@ static struct PyModuleDef _foomodule = { PyMODINIT_FUNC PyInit__testimportmultiple_foo(void) { - return PyModule_Create(&_foomodule); + return PyModuleDef_Init(&_foomodule); } static struct PyModuleDef _barmodule = { PyModuleDef_HEAD_INIT, "_testimportmultiple_bar", "_testimportmultiple_bar doc", - -1, - NULL, + 0, NULL, + shared_slots, NULL, NULL, NULL }; PyMODINIT_FUNC PyInit__testimportmultiple_bar(void){ - return PyModule_Create(&_barmodule); + return PyModuleDef_Init(&_barmodule); } diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index db8817418950b9..e44b629897c58a 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -10,27 +10,27 @@ #undef NDEBUG #include "Python.h" +#include "pycore_backoff.h" // JUMP_BACKWARD_INITIAL_VALUE #include "pycore_bitutils.h" // _Py_bswap32() #include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_ceval.h" // _PyEval_AddPendingCall() +#include "pycore_code.h" // _PyCode_GetTLBCFast() #include "pycore_compile.h" // _PyCompile_CodeGen() #include "pycore_context.h" // _PyContext_NewHamtForTests() -#include "pycore_dict.h" // _PyDictOrValues_GetValues() +#include "pycore_dict.h" // _PyManagedDictPointer_GetValues() #include "pycore_fileutils.h" // _Py_normpath() +#include "pycore_flowgraph.h" // _PyCompile_OptimizeCfg() #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() -#include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() -#include "pycore_long.h" // _PyLong_Sign() +#include "pycore_instruction_sequence.h" // _PyInstructionSequence_New() #include "pycore_object.h" // _PyObject_IsFreed() -#include "pycore_optimizer.h" // _Py_UopsSymbol, etc. +#include "pycore_optimizer.h" // JitOptSymbol, etc. #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_pylifecycle.h" // _PyInterpreterConfig_AsDict() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_typeobject.h" // _PyType_GetModuleName() - -#include "interpreteridobject.h" // PyInterpreterID_LookUp() #include "clinic/_testinternalcapi.c.h" @@ -316,41 +316,6 @@ test_hashtable(PyObject *self, PyObject *Py_UNUSED(args)) } -static PyObject * -test_get_config(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) -{ - PyConfig config; - PyConfig_InitIsolatedConfig(&config); - if (_PyInterpreterState_GetConfigCopy(&config) < 0) { - PyConfig_Clear(&config); - return NULL; - } - PyObject *dict = _PyConfig_AsDict(&config); - PyConfig_Clear(&config); - return dict; -} - - -static PyObject * -test_set_config(PyObject *Py_UNUSED(self), PyObject *dict) -{ - PyConfig config; - PyConfig_InitIsolatedConfig(&config); - if (_PyConfig_FromDict(&config, dict) < 0) { - goto error; - } - if (_PyInterpreterState_SetConfig(&config) < 0) { - goto error; - } - PyConfig_Clear(&config); - Py_RETURN_NONE; - -error: - PyConfig_Clear(&config); - return NULL; -} - - static PyObject * test_reset_path_config(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(arg)) { @@ -680,13 +645,13 @@ set_eval_frame_default(PyObject *self, PyObject *Py_UNUSED(args)) static PyObject * record_eval(PyThreadState *tstate, struct _PyInterpreterFrame *f, int exc) { - if (PyFunction_Check(f->f_funcobj)) { + if (PyStackRef_FunctionCheck(f->f_funcobj)) { + PyFunctionObject *func = _PyFrame_GetFunction(f); PyObject *module = _get_current_module(); assert(module != NULL); module_state *state = get_module_state(module); Py_DECREF(module); - int res = PyList_Append(state->record_list, - ((PyFunctionObject *)f->f_funcobj)->func_name); + int res = PyList_Append(state->record_list, func->func_name); if (res < 0) { return NULL; } @@ -724,6 +689,19 @@ _testinternalcapi_compiler_cleandoc_impl(PyObject *module, PyObject *doc) return _PyCompile_CleanDoc(doc); } +/*[clinic input] + +_testinternalcapi.new_instruction_sequence -> object + +Return a new, empty InstructionSequence. +[clinic start generated code]*/ + +static PyObject * +_testinternalcapi_new_instruction_sequence_impl(PyObject *module) +/*[clinic end generated code: output=ea4243fddb9057fd input=1dec2591b173be83]*/ +{ + return _PyInstructionSequence_New(); +} /*[clinic input] @@ -832,6 +810,7 @@ _testinternalcapi_assemble_code_object_impl(PyObject *module, } +// Maybe this could be replaced by get_interpreter_config()? static PyObject * get_interp_settings(PyObject *self, PyObject *args) { @@ -961,58 +940,23 @@ iframe_getlasti(PyObject *self, PyObject *frame) } static PyObject * -new_counter_optimizer(PyObject *self, PyObject *arg) +get_co_framesize(PyObject *self, PyObject *arg) { - return PyUnstable_Optimizer_NewCounter(); -} - -static PyObject * -new_uop_optimizer(PyObject *self, PyObject *arg) -{ - return PyUnstable_Optimizer_NewUOpOptimizer(); -} - -static PyObject * -set_optimizer(PyObject *self, PyObject *opt) -{ - if (opt == Py_None) { - opt = NULL; - } - if (PyUnstable_SetOptimizer((_PyOptimizerObject*)opt) < 0) { + if (!PyCode_Check(arg)) { + PyErr_SetString(PyExc_TypeError, "argument must be a code object"); return NULL; } - Py_RETURN_NONE; + PyCodeObject *code = (PyCodeObject *)arg; + return PyLong_FromLong(code->co_framesize); } static PyObject * -get_optimizer(PyObject *self, PyObject *Py_UNUSED(ignored)) +jit_enabled(PyObject *self, PyObject *arg) { - PyObject *opt = (PyObject *)PyUnstable_GetOptimizer(); - if (opt == NULL) { - Py_RETURN_NONE; - } - return opt; + return PyBool_FromLong(_PyInterpreterState_GET()->jit); } -static PyObject * -get_executor(PyObject *self, PyObject *const *args, Py_ssize_t nargs) -{ - - if (!_PyArg_CheckPositional("get_executor", nargs, 2, 2)) { - return NULL; - } - PyObject *code = args[0]; - PyObject *offset = args[1]; - long ioffset = PyLong_AsLong(offset); - if (ioffset == -1 && PyErr_Occurred()) { - return NULL; - } - if (!PyCode_Check(code)) { - PyErr_SetString(PyExc_TypeError, "first argument must be a code object"); - return NULL; - } - return (PyObject *)PyUnstable_GetExecutor((PyCodeObject *)code, ioffset); -} +#ifdef _Py_TIER2 static PyObject * add_executor_dependency(PyObject *self, PyObject *args) @@ -1022,12 +966,6 @@ add_executor_dependency(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "OO", &exec, &obj)) { return NULL; } - /* No way to tell in general if exec is an executor, so we only accept - * counting_executor */ - if (strcmp(Py_TYPE(exec)->tp_name, "counting_executor")) { - PyErr_SetString(PyExc_TypeError, "argument must be a counting_executor"); - return NULL; - } _Py_Executor_DependsOn((_PyExecutorObject *)exec, obj); Py_RETURN_NONE; } @@ -1040,6 +978,8 @@ invalidate_executors(PyObject *self, PyObject *obj) Py_RETURN_NONE; } +#endif + static int _pending_callback(void *arg) { /* we assume the argument is callable object to which we own a reference */ @@ -1057,37 +997,56 @@ static PyObject * pending_threadfunc(PyObject *self, PyObject *args, PyObject *kwargs) { PyObject *callable; + unsigned int num = 1; + int blocking = 0; int ensure_added = 0; - static char *kwlist[] = {"", "ensure_added", NULL}; + static char *kwlist[] = {"callback", "num", + "blocking", "ensure_added", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "O|$p:pending_threadfunc", kwlist, - &callable, &ensure_added)) + "O|I$pp:pending_threadfunc", kwlist, + &callable, &num, &blocking, &ensure_added)) { return NULL; } PyInterpreterState *interp = _PyInterpreterState_GET(); /* create the reference for the callbackwhile we hold the lock */ - Py_INCREF(callable); + for (unsigned int i = 0; i < num; i++) { + Py_INCREF(callable); + } - int r; - Py_BEGIN_ALLOW_THREADS - r = _PyEval_AddPendingCall(interp, &_pending_callback, callable, 0); - Py_END_ALLOW_THREADS - if (r < 0) { - /* unsuccessful add */ - if (!ensure_added) { - Py_DECREF(callable); - Py_RETURN_FALSE; + PyThreadState *save_tstate = NULL; + if (!blocking) { + save_tstate = PyEval_SaveThread(); + } + + unsigned int num_added = 0; + for (; num_added < num; num_added++) { + if (ensure_added) { + _Py_add_pending_call_result r; + do { + r = _PyEval_AddPendingCall(interp, &_pending_callback, callable, 0); + assert(r == _Py_ADD_PENDING_SUCCESS + || r == _Py_ADD_PENDING_FULL); + } while (r == _Py_ADD_PENDING_FULL); + } + else { + if (_PyEval_AddPendingCall(interp, &_pending_callback, callable, 0) < 0) { + break; + } } - do { - Py_BEGIN_ALLOW_THREADS - r = _PyEval_AddPendingCall(interp, &_pending_callback, callable, 0); - Py_END_ALLOW_THREADS - } while (r < 0); } - Py_RETURN_TRUE; + if (!blocking) { + PyEval_RestoreThread(save_tstate); + } + + for (unsigned int i = num_added; i < num; i++) { + Py_DECREF(callable); /* unsuccessful add, destroy the extra reference */ + } + + /* The callable is decref'ed in _pending_callback() above. */ + return PyLong_FromUnsignedLong((unsigned long)num_added); } @@ -1113,7 +1072,7 @@ pending_identify(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "O:pending_identify", &interpid)) { return NULL; } - PyInterpreterState *interp = PyInterpreterID_LookUp(interpid); + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(interpid); if (interp == NULL) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_ValueError, "interpreter not found"); @@ -1130,14 +1089,16 @@ pending_identify(PyObject *self, PyObject *args) PyThread_acquire_lock(mutex, WAIT_LOCK); /* It gets released in _pending_identify_callback(). */ - int r; + _Py_add_pending_call_result r; do { Py_BEGIN_ALLOW_THREADS r = _PyEval_AddPendingCall(interp, &_pending_identify_callback, (void *)mutex, 0); Py_END_ALLOW_THREADS - } while (r < 0); + assert(r == _Py_ADD_PENDING_SUCCESS + || r == _Py_ADD_PENDING_FULL); + } while (r == _Py_ADD_PENDING_FULL); /* Wait for the pending call to complete. */ PyThread_acquire_lock(mutex, WAIT_LOCK); @@ -1201,39 +1162,6 @@ unicode_transformdecimalandspacetoascii(PyObject *self, PyObject *arg) return _PyUnicode_TransformDecimalAndSpaceToASCII(arg); } - -struct atexit_data { - int called; -}; - -static void -callback(void *data) -{ - ((struct atexit_data *)data)->called += 1; -} - -static PyObject * -test_atexit(PyObject *self, PyObject *Py_UNUSED(args)) -{ - PyThreadState *oldts = PyThreadState_Swap(NULL); - PyThreadState *tstate = Py_NewInterpreter(); - - struct atexit_data data = {0}; - int res = PyUnstable_AtExit(tstate->interp, callback, (void *)&data); - Py_EndInterpreter(tstate); - PyThreadState_Swap(oldts); - if (res < 0) { - return NULL; - } - - if (data.called == 0) { - PyErr_SetString(PyExc_RuntimeError, "atexit callback not called"); - return NULL; - } - Py_RETURN_NONE; -} - - static PyObject * test_pyobject_is_freed(const char *test_name, PyObject *op) { @@ -1288,8 +1216,8 @@ check_pyobject_forbidden_bytes_is_freed(PyObject *self, static PyObject * check_pyobject_freed_is_freed(PyObject *self, PyObject *Py_UNUSED(args)) { - /* This test would fail if run with the address sanitizer */ -#ifdef _Py_ADDRESS_SANITIZER + /* ASan or TSan would report an use-after-free error */ +#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER) Py_RETURN_NONE; #else PyObject *op = PyObject_CallNoArgs((PyObject *)&PyBaseObject_Type); @@ -1320,14 +1248,13 @@ static PyObject * get_object_dict_values(PyObject *self, PyObject *obj) { PyTypeObject *type = Py_TYPE(obj); - if (!_PyType_HasFeature(type, Py_TPFLAGS_MANAGED_DICT)) { + if (!_PyType_HasFeature(type, Py_TPFLAGS_INLINE_VALUES)) { Py_RETURN_NONE; } - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj); - if (!_PyDictOrValues_IsValues(dorv)) { + PyDictValues *values = _PyObject_InlineValues(obj); + if (!values->valid) { Py_RETURN_NONE; } - PyDictValues *values = _PyDictOrValues_GetValues(dorv); PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; assert(keys != NULL); int size = (int)keys->dk_nentries; @@ -1379,147 +1306,413 @@ dict_getitem_knownhash(PyObject *self, PyObject *args) } -/* To run some code in a sub-interpreter. */ -static PyObject * -run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) +static int +_init_interp_config_from_object(PyInterpreterConfig *config, PyObject *obj) { - const char *code; - int use_main_obmalloc = -1; - int allow_fork = -1; - int allow_exec = -1; - int allow_threads = -1; - int allow_daemon_threads = -1; - int check_multi_interp_extensions = -1; - int gil = -1; - int r; - PyThreadState *substate, *mainstate; - /* only initialise 'cflags.cf_flags' to test backwards compatibility */ - PyCompilerFlags cflags = {0}; + if (obj == NULL) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; + return 0; + } + + PyObject *dict = PyObject_GetAttrString(obj, "__dict__"); + if (dict == NULL) { + PyErr_Format(PyExc_TypeError, "bad config %R", obj); + return -1; + } + int res = _PyInterpreterConfig_InitFromDict(config, dict); + Py_DECREF(dict); + if (res < 0) { + return -1; + } + return 0; +} - static char *kwlist[] = {"code", - "use_main_obmalloc", - "allow_fork", - "allow_exec", - "allow_threads", - "allow_daemon_threads", - "check_multi_interp_extensions", - "gil", - NULL}; +static PyInterpreterState * +_new_interpreter(PyInterpreterConfig *config, long whence) +{ + if (whence == _PyInterpreterState_WHENCE_XI) { + return _PyXI_NewInterpreter(config, &whence, NULL, NULL); + } + PyObject *exc = NULL; + PyInterpreterState *interp = NULL; + if (whence == _PyInterpreterState_WHENCE_UNKNOWN) { + assert(config == NULL); + interp = PyInterpreterState_New(); + } + else if (whence == _PyInterpreterState_WHENCE_CAPI + || whence == _PyInterpreterState_WHENCE_LEGACY_CAPI) + { + PyThreadState *tstate = NULL; + PyThreadState *save_tstate = PyThreadState_Swap(NULL); + if (whence == _PyInterpreterState_WHENCE_LEGACY_CAPI) { + assert(config == NULL); + tstate = Py_NewInterpreter(); + PyThreadState_Swap(save_tstate); + } + else { + PyStatus status = Py_NewInterpreterFromConfig(&tstate, config); + PyThreadState_Swap(save_tstate); + if (PyStatus_Exception(status)) { + assert(tstate == NULL); + _PyErr_SetFromPyStatus(status); + exc = PyErr_GetRaisedException(); + } + } + if (tstate != NULL) { + interp = PyThreadState_GetInterpreter(tstate); + // Throw away the initial tstate. + PyThreadState_Swap(tstate); + PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); + } + } + else { + PyErr_Format(PyExc_ValueError, + "unsupported whence %ld", whence); + return NULL; + } + + if (interp == NULL) { + PyErr_SetString(PyExc_InterpreterError, + "sub-interpreter creation failed"); + if (exc != NULL) { + _PyErr_ChainExceptions1(exc); + } + } + return interp; +} + +// This exists mostly for testing the _interpreters module, as an +// alternative to _interpreters.create() +static PyObject * +create_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"config", "whence", NULL}; + PyObject *configobj = NULL; + long whence = _PyInterpreterState_WHENCE_XI; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "s$ppppppi:run_in_subinterp_with_config", kwlist, - &code, &use_main_obmalloc, - &allow_fork, &allow_exec, - &allow_threads, &allow_daemon_threads, - &check_multi_interp_extensions, - &gil)) { + "|O$l:create_interpreter", kwlist, + &configobj, &whence)) + { + return NULL; + } + if (configobj == Py_None) { + configobj = NULL; + } + + // Resolve the config. + PyInterpreterConfig *config = NULL; + PyInterpreterConfig _config; + if (whence == _PyInterpreterState_WHENCE_UNKNOWN + || whence == _PyInterpreterState_WHENCE_LEGACY_CAPI) + { + if (configobj != NULL) { + PyErr_SetString(PyExc_ValueError, "got unexpected config"); + return NULL; + } + } + else { + config = &_config; + if (_init_interp_config_from_object(config, configobj) < 0) { + return NULL; + } + } + + // Create the interpreter. + PyInterpreterState *interp = _new_interpreter(config, whence); + if (interp == NULL) { return NULL; } - if (use_main_obmalloc < 0) { - PyErr_SetString(PyExc_ValueError, "missing use_main_obmalloc"); + + // Return the ID. + PyObject *idobj = _PyInterpreterState_GetIDObject(interp); + if (idobj == NULL) { + _PyXI_EndInterpreter(interp, NULL, NULL); return NULL; } - if (allow_fork < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_fork"); + + return idobj; +} + +// This exists mostly for testing the _interpreters module, as an +// alternative to _interpreters.destroy() +static PyObject * +destroy_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"id", NULL}; + PyObject *idobj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "O:destroy_interpreter", kwlist, + &idobj)) + { return NULL; } - if (allow_exec < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_exec"); + + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { return NULL; } - if (allow_threads < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_threads"); + + _PyXI_EndInterpreter(interp, NULL, NULL); + Py_RETURN_NONE; +} + +// This exists mostly for testing the _interpreters module, as an +// alternative to _interpreters.destroy() +static PyObject * +exec_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"id", "code", "main", NULL}; + PyObject *idobj; + const char *code; + int runningmain = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "Os|$p:exec_interpreter", kwlist, + &idobj, &code, &runningmain)) + { return NULL; } - if (gil < 0) { - PyErr_SetString(PyExc_ValueError, "missing gil"); + + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { return NULL; } - if (allow_daemon_threads < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_daemon_threads"); + + PyObject *res = NULL; + PyThreadState *tstate = + _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_EXEC); + + PyThreadState *save_tstate = PyThreadState_Swap(tstate); + + if (runningmain) { + if (_PyInterpreterState_SetRunningMain(interp) < 0) { + goto finally; + } + } + + /* only initialise 'cflags.cf_flags' to test backwards compatibility */ + PyCompilerFlags cflags = {0}; + int r = PyRun_SimpleStringFlags(code, &cflags); + if (PyErr_Occurred()) { + PyErr_PrintEx(0); + } + + if (runningmain) { + _PyInterpreterState_SetNotRunningMain(interp); + } + + res = PyLong_FromLong(r); + +finally: + PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); + return res; +} + + +/* To run some code in a sub-interpreter. + +Generally you can use test.support.interpreters, +but we keep this helper as a distinct implementation. +That's especially important for testing test.support.interpreters. +*/ +static PyObject * +run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) +{ + const char *code; + PyObject *configobj; + int xi = 0; + static char *kwlist[] = {"code", "config", "xi", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "sO|$p:run_in_subinterp_with_config", kwlist, + &code, &configobj, &xi)) + { return NULL; } - if (check_multi_interp_extensions < 0) { - PyErr_SetString(PyExc_ValueError, "missing check_multi_interp_extensions"); + + PyInterpreterConfig config; + if (_init_interp_config_from_object(&config, configobj) < 0) { return NULL; } - mainstate = PyThreadState_Get(); + /* only initialise 'cflags.cf_flags' to test backwards compatibility */ + PyCompilerFlags cflags = {0}; - PyThreadState_Swap(NULL); + int r; + if (xi) { + PyThreadState *save_tstate; + PyThreadState *tstate; + + /* Create an interpreter, staying switched to it. */ + PyInterpreterState *interp = \ + _PyXI_NewInterpreter(&config, NULL, &tstate, &save_tstate); + if (interp == NULL) { + return NULL; + } + + /* Exec the code in the new interpreter. */ + r = PyRun_SimpleStringFlags(code, &cflags); - const PyInterpreterConfig config = { - .use_main_obmalloc = use_main_obmalloc, - .allow_fork = allow_fork, - .allow_exec = allow_exec, - .allow_threads = allow_threads, - .allow_daemon_threads = allow_daemon_threads, - .check_multi_interp_extensions = check_multi_interp_extensions, - .gil = gil, - }; - PyStatus status = Py_NewInterpreterFromConfig(&substate, &config); - if (PyStatus_Exception(status)) { - /* Since no new thread state was created, there is no exception to - propagate; raise a fresh one after swapping in the old thread - state. */ + /* clean up post-exec. */ + _PyXI_EndInterpreter(interp, tstate, &save_tstate); + } + else { + PyThreadState *substate; + PyThreadState *mainstate = PyThreadState_Swap(NULL); + + /* Create an interpreter, staying switched to it. */ + PyStatus status = Py_NewInterpreterFromConfig(&substate, &config); + if (PyStatus_Exception(status)) { + /* Since no new thread state was created, there is no exception to + propagate; raise a fresh one after swapping in the old thread + state. */ + PyThreadState_Swap(mainstate); + _PyErr_SetFromPyStatus(status); + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_InterpreterError, + "sub-interpreter creation failed"); + _PyErr_ChainExceptions1(exc); + return NULL; + } + + /* Exec the code in the new interpreter. */ + r = PyRun_SimpleStringFlags(code, &cflags); + + /* clean up post-exec. */ + Py_EndInterpreter(substate); PyThreadState_Swap(mainstate); - _PyErr_SetFromPyStatus(status); - PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(PyExc_RuntimeError, "sub-interpreter creation failed"); - _PyErr_ChainExceptions1(exc); + } + + return PyLong_FromLong(r); +} + + +static PyObject * +normalize_interp_id(PyObject *self, PyObject *idobj) +{ + int64_t interpid = _PyInterpreterState_ObjectToID(idobj); + if (interpid < 0) { return NULL; } - assert(substate != NULL); - r = PyRun_SimpleStringFlags(code, &cflags); - Py_EndInterpreter(substate); + return PyLong_FromLongLong(interpid); +} - PyThreadState_Swap(mainstate); +static PyObject * +next_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + int64_t interpid = _PyRuntime.interpreters.next_id; + return PyLong_FromLongLong(interpid); +} - return PyLong_FromLong(r); +static PyObject * +unused_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + int64_t interpid = INT64_MAX; + assert(interpid > _PyRuntime.interpreters.next_id); + return PyLong_FromLongLong(interpid); } +static PyObject * +interpreter_exists(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + if (PyErr_ExceptionMatches(PyExc_InterpreterNotFoundError)) { + PyErr_Clear(); + Py_RETURN_FALSE; + } + assert(PyErr_Occurred()); + return NULL; + } + Py_RETURN_TRUE; +} static PyObject * get_interpreter_refcount(PyObject *self, PyObject *idobj) { - PyInterpreterState *interp = PyInterpreterID_LookUp(idobj); + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); if (interp == NULL) { return NULL; } return PyLong_FromLongLong(interp->id_refcount); } +static PyObject * +link_interpreter_refcount(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + _PyInterpreterState_RequireIDRef(interp, 1); + Py_RETURN_NONE; +} + +static PyObject * +unlink_interpreter_refcount(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + _PyInterpreterState_RequireIDRef(interp, 0); + Py_RETURN_NONE; +} + +static PyObject * +interpreter_refcount_linked(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + if (_PyInterpreterState_RequiresIDRef(interp)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static void _xid_capsule_destructor(PyObject *capsule) { - _PyCrossInterpreterData *data = \ - (_PyCrossInterpreterData *)PyCapsule_GetPointer(capsule, NULL); + _PyXIData_t *data = (_PyXIData_t *)PyCapsule_GetPointer(capsule, NULL); if (data != NULL) { - assert(_PyCrossInterpreterData_Release(data) == 0); - _PyCrossInterpreterData_Free(data); + assert(_PyXIData_Release(data) == 0); + _PyXIData_Free(data); } } static PyObject * get_crossinterp_data(PyObject *self, PyObject *args) { + PyInterpreterState *interp = PyInterpreterState_Get(); + _PyXIData_lookup_context_t ctx; + if (_PyXIData_GetLookupContext(interp, &ctx) < 0) { + return NULL; + } + PyObject *obj = NULL; if (!PyArg_ParseTuple(args, "O:get_crossinterp_data", &obj)) { return NULL; } - _PyCrossInterpreterData *data = _PyCrossInterpreterData_New(); + _PyXIData_t *data = _PyXIData_New(); if (data == NULL) { return NULL; } - if (_PyObject_GetCrossInterpreterData(obj, data) != 0) { - _PyCrossInterpreterData_Free(data); + if (_PyObject_GetXIData(&ctx, obj, data) != 0) { + _PyXIData_Free(data); return NULL; } PyObject *capsule = PyCapsule_New(data, NULL, _xid_capsule_destructor); if (capsule == NULL) { - assert(_PyCrossInterpreterData_Release(data) == 0); - _PyCrossInterpreterData_Free(data); + assert(_PyXIData_Release(data) == 0); + _PyXIData_Free(data); } return capsule; } @@ -1532,12 +1725,11 @@ restore_crossinterp_data(PyObject *self, PyObject *args) return NULL; } - _PyCrossInterpreterData *data = \ - (_PyCrossInterpreterData *)PyCapsule_GetPointer(capsule, NULL); + _PyXIData_t *data = (_PyXIData_t *)PyCapsule_GetPointer(capsule, NULL); if (data == NULL) { return NULL; } - return _PyCrossInterpreterData_NewObject(data); + return _PyXIData_NewObject(data); } @@ -1559,7 +1751,7 @@ _testinternalcapi_test_long_numbits_impl(PyObject *module) { struct triple { long input; - size_t nbits; + uint64_t nbits; int sign; } testcases[] = {{0, 0, 0}, {1L, 1, 1}, @@ -1579,15 +1771,15 @@ _testinternalcapi_test_long_numbits_impl(PyObject *module) size_t i; for (i = 0; i < Py_ARRAY_LENGTH(testcases); ++i) { - size_t nbits; - int sign; + uint64_t nbits; + int sign = -7; PyObject *plong; plong = PyLong_FromLong(testcases[i].input); if (plong == NULL) return NULL; nbits = _PyLong_NumBits(plong); - sign = _PyLong_Sign(plong); + (void)PyLong_GetSign(plong, &sign); Py_DECREF(plong); if (nbits != testcases[i].nbits) @@ -1595,7 +1787,7 @@ _testinternalcapi_test_long_numbits_impl(PyObject *module) "wrong result for _PyLong_NumBits"); if (sign != testcases[i].sign) return raiseTestError("test_long_numbits", - "wrong result for _PyLong_Sign"); + "wrong result for PyLong_GetSign()"); } Py_RETURN_NONE; } @@ -1631,13 +1823,6 @@ perf_trampoline_set_persist_after_fork(PyObject *self, PyObject *args) } -static PyObject * -get_type_module_name(PyObject *self, PyObject *type) -{ - assert(PyType_Check(type)); - return _PyType_GetModuleName((PyTypeObject *)type); -} - static PyObject * get_rare_event_counters(PyObject *self, PyObject *type) { @@ -1676,8 +1861,137 @@ get_py_thread_id(PyObject *self, PyObject *Py_UNUSED(ignored)) Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(tid)); return PyLong_FromUnsignedLongLong(tid); } + +static PyCodeObject * +get_code(PyObject *obj) +{ + if (PyCode_Check(obj)) { + return (PyCodeObject *)obj; + } + else if (PyFunction_Check(obj)) { + return (PyCodeObject *)PyFunction_GetCode(obj); + } + return (PyCodeObject *)PyErr_Format( + PyExc_TypeError, "expected function or code object, got %s", + Py_TYPE(obj)->tp_name); +} + +static PyObject * +get_tlbc(PyObject *Py_UNUSED(module), PyObject *obj) +{ + PyCodeObject *code = get_code(obj); + if (code == NULL) { + return NULL; + } + _Py_CODEUNIT *bc = _PyCode_GetTLBCFast(PyThreadState_GET(), code); + if (bc == NULL) { + Py_RETURN_NONE; + } + return PyBytes_FromStringAndSize((const char *)bc, _PyCode_NBYTES(code)); +} + +static PyObject * +get_tlbc_id(PyObject *Py_UNUSED(module), PyObject *obj) +{ + PyCodeObject *code = get_code(obj); + if (code == NULL) { + return NULL; + } + _Py_CODEUNIT *bc = _PyCode_GetTLBCFast(PyThreadState_GET(), code); + if (bc == NULL) { + Py_RETURN_NONE; + } + return PyLong_FromVoidPtr(bc); +} #endif +static PyObject * +has_inline_values(PyObject *self, PyObject *obj) +{ + if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + _PyObject_InlineValues(obj)->valid) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +static PyObject * +has_split_table(PyObject *self, PyObject *obj) +{ + if (PyDict_Check(obj) && _PyDict_HasSplitTable((PyDictObject *)obj)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +// Circumvents standard version assignment machinery - use with caution and only on +// short-lived heap types +static PyObject * +type_assign_specific_version_unsafe(PyObject *self, PyObject *args) +{ + PyTypeObject *type; + unsigned int version; + if (!PyArg_ParseTuple(args, "Oi:type_assign_specific_version_unsafe", &type, &version)) { + return NULL; + } + assert(!PyType_HasFeature(type, Py_TPFLAGS_IMMUTABLETYPE)); + _PyType_SetVersion(type, version); + Py_RETURN_NONE; +} + +/*[clinic input] +gh_119213_getargs + + spam: object = None + +Test _PyArg_Parser.kwtuple +[clinic start generated code]*/ + +static PyObject * +gh_119213_getargs_impl(PyObject *module, PyObject *spam) +/*[clinic end generated code: output=d8d9c95d5b446802 input=65ef47511da80fc2]*/ +{ + // It must never have been called in the main interprer + assert(!_Py_IsMainInterpreter(PyInterpreterState_Get())); + return Py_NewRef(spam); +} + + +static PyObject * +get_static_builtin_types(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _PyStaticType_GetBuiltins(); +} + + +static PyObject * +identify_type_slot_wrappers(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _PyType_GetSlotWrapperNames(); +} + + +static PyObject * +has_deferred_refcount(PyObject *self, PyObject *op) +{ + return PyBool_FromLong(_PyObject_HasDeferredRefcount(op)); +} + +static PyObject * +get_tracked_heap_size(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyLong_FromInt64(PyInterpreterState_Get()->gc.heap_size); +} + +static PyObject * +is_static_immortal(PyObject *self, PyObject *op) +{ + if (_Py_IsStaticImmortal(op)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -1686,8 +2000,6 @@ static PyMethodDef module_functions[] = { {"test_popcount", test_popcount, METH_NOARGS}, {"test_bit_length", test_bit_length, METH_NOARGS}, {"test_hashtable", test_hashtable, METH_NOARGS}, - {"get_config", test_get_config, METH_NOARGS}, - {"set_config", test_set_config, METH_O}, {"reset_path_config", test_reset_path_config, METH_NOARGS}, {"test_edit_cost", test_edit_cost, METH_NOARGS}, {"test_bytes_find", test_bytes_find, METH_NOARGS}, @@ -1698,6 +2010,7 @@ static PyMethodDef module_functions[] = { {"set_eval_frame_default", set_eval_frame_default, METH_NOARGS, NULL}, {"set_eval_frame_record", set_eval_frame_record, METH_O, NULL}, _TESTINTERNALCAPI_COMPILER_CLEANDOC_METHODDEF + _TESTINTERNALCAPI_NEW_INSTRUCTION_SEQUENCE_METHODDEF _TESTINTERNALCAPI_COMPILER_CODEGEN_METHODDEF _TESTINTERNALCAPI_OPTIMIZE_CFG_METHODDEF _TESTINTERNALCAPI_ASSEMBLE_CODE_OBJECT_METHODDEF @@ -1708,20 +2021,18 @@ static PyMethodDef module_functions[] = { {"iframe_getcode", iframe_getcode, METH_O, NULL}, {"iframe_getline", iframe_getline, METH_O, NULL}, {"iframe_getlasti", iframe_getlasti, METH_O, NULL}, - {"get_optimizer", get_optimizer, METH_NOARGS, NULL}, - {"set_optimizer", set_optimizer, METH_O, NULL}, - {"get_executor", _PyCFunction_CAST(get_executor), METH_FASTCALL, NULL}, - {"new_counter_optimizer", new_counter_optimizer, METH_NOARGS, NULL}, - {"new_uop_optimizer", new_uop_optimizer, METH_NOARGS, NULL}, + {"get_co_framesize", get_co_framesize, METH_O, NULL}, + {"jit_enabled", jit_enabled, METH_NOARGS, NULL}, +#ifdef _Py_TIER2 {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL}, {"invalidate_executors", invalidate_executors, METH_O, NULL}, +#endif {"pending_threadfunc", _PyCFunction_CAST(pending_threadfunc), METH_VARARGS | METH_KEYWORDS}, {"pending_identify", pending_identify, METH_VARARGS, NULL}, {"_PyTraceMalloc_GetTraceback", tracemalloc_get_traceback, METH_VARARGS}, {"test_tstate_capi", test_tstate_capi, METH_NOARGS, NULL}, {"_PyUnicode_TransformDecimalAndSpaceToASCII", unicode_transformdecimalandspacetoascii, METH_O}, - {"test_atexit", test_atexit, METH_NOARGS}, {"check_pyobject_forbidden_bytes_is_freed", check_pyobject_forbidden_bytes_is_freed, METH_NOARGS}, {"check_pyobject_freed_is_freed", check_pyobject_freed_is_freed, METH_NOARGS}, @@ -1732,22 +2043,49 @@ static PyMethodDef module_functions[] = { {"get_object_dict_values", get_object_dict_values, METH_O}, {"hamt", new_hamt, METH_NOARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, + {"create_interpreter", _PyCFunction_CAST(create_interpreter), + METH_VARARGS | METH_KEYWORDS}, + {"destroy_interpreter", _PyCFunction_CAST(destroy_interpreter), + METH_VARARGS | METH_KEYWORDS}, + {"exec_interpreter", _PyCFunction_CAST(exec_interpreter), + METH_VARARGS | METH_KEYWORDS}, {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, + {"normalize_interp_id", normalize_interp_id, METH_O}, + {"next_interpreter_id", next_interpreter_id, METH_NOARGS}, + {"unused_interpreter_id", unused_interpreter_id, METH_NOARGS}, + {"interpreter_exists", interpreter_exists, METH_O}, {"get_interpreter_refcount", get_interpreter_refcount, METH_O}, + {"link_interpreter_refcount", link_interpreter_refcount, METH_O}, + {"unlink_interpreter_refcount", unlink_interpreter_refcount, METH_O}, + {"interpreter_refcount_linked", interpreter_refcount_linked, METH_O}, {"compile_perf_trampoline_entry", compile_perf_trampoline_entry, METH_VARARGS}, {"perf_trampoline_set_persist_after_fork", perf_trampoline_set_persist_after_fork, METH_VARARGS}, {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, {"restore_crossinterp_data", restore_crossinterp_data, METH_VARARGS}, _TESTINTERNALCAPI_TEST_LONG_NUMBITS_METHODDEF - {"get_type_module_name", get_type_module_name, METH_O}, {"get_rare_event_counters", get_rare_event_counters, METH_NOARGS}, {"reset_rare_event_counters", reset_rare_event_counters, METH_NOARGS}, + {"has_inline_values", has_inline_values, METH_O}, + {"has_split_table", has_split_table, METH_O}, + {"type_assign_specific_version_unsafe", type_assign_specific_version_unsafe, METH_VARARGS, + PyDoc_STR("forcefully assign type->tp_version_tag")}, + #ifdef Py_GIL_DISABLED {"py_thread_id", get_py_thread_id, METH_NOARGS}, + {"get_tlbc", get_tlbc, METH_O, NULL}, + {"get_tlbc_id", get_tlbc_id, METH_O, NULL}, #endif +#ifdef _Py_TIER2 {"uop_symbols_test", _Py_uop_symbols_test, METH_NOARGS}, +#endif + GH_119213_GETARGS_METHODDEF + {"get_static_builtin_types", get_static_builtin_types, METH_NOARGS}, + {"identify_type_slot_wrappers", identify_type_slot_wrappers, METH_NOARGS}, + {"has_deferred_refcount", has_deferred_refcount, METH_O}, + {"get_tracked_heap_size", get_tracked_heap_size, METH_NOARGS}, + {"is_static_immortal", is_static_immortal, METH_O}, {NULL, NULL} /* sentinel */ }; @@ -1795,12 +2133,33 @@ module_exec(PyObject *module) return 1; } + if (PyModule_Add(module, "TIER2_THRESHOLD", + PyLong_FromLong(JUMP_BACKWARD_INITIAL_VALUE + 1)) < 0) { + return 1; + } + + if (PyModule_Add(module, "SPECIALIZATION_THRESHOLD", + PyLong_FromLong(ADAPTIVE_WARMUP_VALUE + 1)) < 0) { + return 1; + } + + if (PyModule_Add(module, "SPECIALIZATION_COOLDOWN", + PyLong_FromLong(ADAPTIVE_COOLDOWN_VALUE + 1)) < 0) { + return 1; + } + + if (PyModule_Add(module, "SHARED_KEYS_MAX_SIZE", + PyLong_FromLong(SHARED_KEYS_MAX_SIZE)) < 0) { + return 1; + } + return 0; } static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_testinternalcapi/set.c b/Modules/_testinternalcapi/set.c index 0305a7885d217c..01aab03cc109ed 100644 --- a/Modules/_testinternalcapi/set.c +++ b/Modules/_testinternalcapi/set.c @@ -1,6 +1,7 @@ #include "parts.h" #include "../_testcapi/util.h" // NULLABLE, RETURN_INT +#include "pycore_critical_section.h" #include "pycore_setobject.h" @@ -27,10 +28,13 @@ set_next_entry(PyObject *self, PyObject *args) return NULL; } NULLABLE(set); - - rc = _PySet_NextEntry(set, &pos, &item, &hash); + Py_BEGIN_CRITICAL_SECTION(set); + rc = _PySet_NextEntryRef(set, &pos, &item, &hash); + Py_END_CRITICAL_SECTION(); if (rc == 1) { - return Py_BuildValue("innO", rc, pos, hash, item); + PyObject *ret = Py_BuildValue("innO", rc, pos, hash, item); + Py_DECREF(item); + return ret; } assert(item == UNINITIALIZED_PTR); assert(hash == (Py_hash_t)UNINITIALIZED_SIZE); diff --git a/Modules/_testinternalcapi/test_critical_sections.c b/Modules/_testinternalcapi/test_critical_sections.c index 94da0468fcf149..1df960f9881f70 100644 --- a/Modules/_testinternalcapi/test_critical_sections.c +++ b/Modules/_testinternalcapi/test_critical_sections.c @@ -49,15 +49,6 @@ test_critical_sections(PyObject *self, PyObject *Py_UNUSED(args)) Py_END_CRITICAL_SECTION2(); assert_nogil(!PyMutex_IsLocked(&d2->ob_mutex)); - // Optional variant behaves the same if the object is non-NULL - Py_XBEGIN_CRITICAL_SECTION(d1); - assert_nogil(PyMutex_IsLocked(&d1->ob_mutex)); - Py_XEND_CRITICAL_SECTION(); - - // No-op - Py_XBEGIN_CRITICAL_SECTION(NULL); - Py_XEND_CRITICAL_SECTION(); - Py_DECREF(d2); Py_DECREF(d1); Py_RETURN_NONE; @@ -139,6 +130,7 @@ test_critical_sections_suspend(PyObject *self, PyObject *Py_UNUSED(args)) Py_RETURN_NONE; } +#ifdef Py_CAN_START_THREADS struct test_data { PyObject *obj1; PyObject *obj2; @@ -179,7 +171,6 @@ thread_critical_sections(void *arg) } } -#ifdef Py_CAN_START_THREADS static PyObject * test_critical_sections_threads(PyObject *self, PyObject *Py_UNUSED(args)) { @@ -194,7 +185,7 @@ test_critical_sections_threads(PyObject *self, PyObject *Py_UNUSED(args)) assert(test_data.obj2 != NULL); assert(test_data.obj3 != NULL); - for (int i = 0; i < NUM_THREADS; i++) { + for (Py_ssize_t i = 0; i < NUM_THREADS; i++) { PyThread_start_new_thread(&thread_critical_sections, &test_data); } PyEvent_Wait(&test_data.done_event); @@ -204,6 +195,90 @@ test_critical_sections_threads(PyObject *self, PyObject *Py_UNUSED(args)) Py_DECREF(test_data.obj1); Py_RETURN_NONE; } + +static void +pysleep(int ms) +{ +#ifdef MS_WINDOWS + Sleep(ms); +#else + usleep(ms * 1000); +#endif +} + +struct test_data_gc { + PyObject *obj; + Py_ssize_t num_threads; + Py_ssize_t id; + Py_ssize_t countdown; + PyEvent done_event; + PyEvent ready; +}; + +static void +thread_gc(void *arg) +{ + struct test_data_gc *test_data = arg; + PyGILState_STATE gil = PyGILState_Ensure(); + + Py_ssize_t id = _Py_atomic_add_ssize(&test_data->id, 1); + if (id == test_data->num_threads - 1) { + _PyEvent_Notify(&test_data->ready); + } + else { + // wait for all test threads to more reliably reproduce the issue. + PyEvent_Wait(&test_data->ready); + } + + if (id == 0) { + Py_BEGIN_CRITICAL_SECTION(test_data->obj); + // pause long enough that the lock would be handed off directly to + // a waiting thread. + pysleep(5); + PyGC_Collect(); + Py_END_CRITICAL_SECTION(); + } + else if (id == 1) { + pysleep(1); + Py_BEGIN_CRITICAL_SECTION(test_data->obj); + pysleep(1); + Py_END_CRITICAL_SECTION(); + } + else if (id == 2) { + // sleep long enough so that thread 0 is waiting to stop the world + pysleep(6); + Py_BEGIN_CRITICAL_SECTION(test_data->obj); + pysleep(1); + Py_END_CRITICAL_SECTION(); + } + + PyGILState_Release(gil); + if (_Py_atomic_add_ssize(&test_data->countdown, -1) == 1) { + // last thread to finish sets done_event + _PyEvent_Notify(&test_data->done_event); + } +} + +static PyObject * +test_critical_sections_gc(PyObject *self, PyObject *Py_UNUSED(args)) +{ + // gh-118332: Contended critical sections should not deadlock with GC + const Py_ssize_t NUM_THREADS = 3; + struct test_data_gc test_data = { + .obj = PyDict_New(), + .countdown = NUM_THREADS, + .num_threads = NUM_THREADS, + }; + assert(test_data.obj != NULL); + + for (Py_ssize_t i = 0; i < NUM_THREADS; i++) { + PyThread_start_new_thread(&thread_gc, &test_data); + } + PyEvent_Wait(&test_data.done_event); + Py_DECREF(test_data.obj); + Py_RETURN_NONE; +} + #endif static PyMethodDef test_methods[] = { @@ -212,6 +287,7 @@ static PyMethodDef test_methods[] = { {"test_critical_sections_suspend", test_critical_sections_suspend, METH_NOARGS}, #ifdef Py_CAN_START_THREADS {"test_critical_sections_threads", test_critical_sections_threads, METH_NOARGS}, + {"test_critical_sections_gc", test_critical_sections_gc, METH_NOARGS}, #endif {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testinternalcapi/test_lock.c b/Modules/_testinternalcapi/test_lock.c index 1c5048170e9f2e..8d678412fe7179 100644 --- a/Modules/_testinternalcapi/test_lock.c +++ b/Modules/_testinternalcapi/test_lock.c @@ -2,7 +2,7 @@ #include "parts.h" #include "pycore_lock.h" -#include "pycore_time.h" // _PyTime_MonotonicUnchecked() +#include "pycore_pythread.h" // PyThread_get_thread_ident_ex() #include "clinic/test_lock.c.h" @@ -36,9 +36,9 @@ test_lock_basic(PyObject *self, PyObject *obj) // uncontended lock and unlock PyMutex_Lock(&m); - assert(m.v == 1); + assert(m._bits == 1); PyMutex_Unlock(&m); - assert(m.v == 0); + assert(m._bits == 0); Py_RETURN_NONE; } @@ -57,10 +57,10 @@ lock_thread(void *arg) _Py_atomic_store_int(&test_data->started, 1); PyMutex_Lock(m); - assert(m->v == 1); + assert(m->_bits == 1); PyMutex_Unlock(m); - assert(m->v == 0); + assert(m->_bits == 0); _PyEvent_Notify(&test_data->done); } @@ -73,7 +73,7 @@ test_lock_two_threads(PyObject *self, PyObject *obj) memset(&test_data, 0, sizeof(test_data)); PyMutex_Lock(&test_data.m); - assert(test_data.m.v == 1); + assert(test_data.m._bits == 1); PyThread_start_new_thread(lock_thread, &test_data); @@ -82,17 +82,17 @@ test_lock_two_threads(PyObject *self, PyObject *obj) uint8_t v; do { pysleep(10); // allow some time for the other thread to try to lock - v = _Py_atomic_load_uint8_relaxed(&test_data.m.v); + v = _Py_atomic_load_uint8_relaxed(&test_data.m._bits); assert(v == 1 || v == 3); iters++; } while (v != 3 && iters < 200); // both the "locked" and the "has parked" bits should be set - assert(test_data.m.v == 3); + assert(test_data.m._bits == 3); PyMutex_Unlock(&test_data.m); PyEvent_Wait(&test_data.done); - assert(test_data.m.v == 0); + assert(test_data.m._bits == 0); Py_RETURN_NONE; } @@ -290,7 +290,10 @@ _testinternalcapi_benchmark_locks_impl(PyObject *module, goto exit; } - PyTime_t start = _PyTime_MonotonicUnchecked(); + PyTime_t start, end; + if (PyTime_PerfCounter(&start) < 0) { + goto exit; + } for (Py_ssize_t i = 0; i < num_threads; i++) { thread_data[i].bench_data = &bench_data; @@ -307,7 +310,9 @@ _testinternalcapi_benchmark_locks_impl(PyObject *module, } Py_ssize_t total_iters = bench_data.total_iters; - PyTime_t end = _PyTime_MonotonicUnchecked(); + if (PyTime_PerfCounter(&end) < 0) { + goto exit; + } // Return the total number of acquisitions and the number of acquisitions // for each thread. @@ -319,7 +324,8 @@ _testinternalcapi_benchmark_locks_impl(PyObject *module, PyList_SET_ITEM(thread_iters, i, iter); } - double rate = total_iters * 1000000000.0 / (end - start); + assert(end != start); + double rate = total_iters * 1e9 / (end - start); res = Py_BuildValue("(dO)", rate, thread_iters); exit: @@ -471,6 +477,29 @@ test_lock_rwlock(PyObject *self, PyObject *obj) Py_RETURN_NONE; } +static PyObject * +test_lock_recursive(PyObject *self, PyObject *obj) +{ + _PyRecursiveMutex m = (_PyRecursiveMutex){0}; + assert(!_PyRecursiveMutex_IsLockedByCurrentThread(&m)); + + _PyRecursiveMutex_Lock(&m); + assert(m.thread == PyThread_get_thread_ident_ex()); + assert(PyMutex_IsLocked(&m.mutex)); + assert(m.level == 0); + + _PyRecursiveMutex_Lock(&m); + assert(m.level == 1); + _PyRecursiveMutex_Unlock(&m); + + _PyRecursiveMutex_Unlock(&m); + assert(m.thread == 0); + assert(!PyMutex_IsLocked(&m.mutex)); + assert(m.level == 0); + + Py_RETURN_NONE; +} + static PyMethodDef test_methods[] = { {"test_lock_basic", test_lock_basic, METH_NOARGS}, {"test_lock_two_threads", test_lock_two_threads, METH_NOARGS}, @@ -480,6 +509,7 @@ static PyMethodDef test_methods[] = { {"test_lock_benchmark", test_lock_benchmark, METH_NOARGS}, {"test_lock_once", test_lock_once, METH_NOARGS}, {"test_lock_rwlock", test_lock_rwlock, METH_NOARGS}, + {"test_lock_recursive", test_lock_recursive, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c new file mode 100644 index 00000000000000..82dac1c999470f --- /dev/null +++ b/Modules/_testlimitedcapi.c @@ -0,0 +1,93 @@ +/* + * Test the limited C API. + * + * The 'test_*' functions exported by this module are run as part of the + * standard Python regression test, via Lib/test/test_capi.py. + */ + +#include "_testlimitedcapi/parts.h" + +static PyMethodDef TestMethods[] = { + {NULL, NULL} /* sentinel */ +}; + +static struct PyModuleDef _testlimitedcapimodule = { + PyModuleDef_HEAD_INIT, + .m_name = "_testlimitedcapi", + .m_size = 0, + .m_methods = TestMethods, +}; + +PyMODINIT_FUNC +PyInit__testlimitedcapi(void) +{ + PyObject *mod = PyModule_Create(&_testlimitedcapimodule); + if (mod == NULL) { + return NULL; + } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); +#endif + + if (_PyTestLimitedCAPI_Init_Abstract(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_ByteArray(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Bytes(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Codec(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Complex(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Dict(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Eval(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Float(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_HeaptypeRelative(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Import(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_List(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Long(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Object(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_PyOS(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Set(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Sys(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Tuple(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Unicode(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_VectorcallLimited(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Version(mod) < 0) { + return NULL; + } + return mod; +} diff --git a/Modules/_testlimitedcapi/abstract.c b/Modules/_testlimitedcapi/abstract.c new file mode 100644 index 00000000000000..6056dd100d6069 --- /dev/null +++ b/Modules/_testlimitedcapi/abstract.c @@ -0,0 +1,582 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +object_repr(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_Repr(arg); +} + +static PyObject * +object_ascii(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_ASCII(arg); +} + +static PyObject * +object_str(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_Str(arg); +} + +static PyObject * +object_bytes(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_Bytes(arg); +} + +static PyObject * +object_getattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + return PyObject_GetAttr(obj, attr_name); +} + +static PyObject * +object_getattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + return PyObject_GetAttrString(obj, attr_name); +} + +static PyObject * +object_hasattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + return PyLong_FromLong(PyObject_HasAttr(obj, attr_name)); +} + +static PyObject * +object_hasattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + return PyLong_FromLong(PyObject_HasAttrString(obj, attr_name)); +} + +static PyObject * +object_setattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name, *value; + if (!PyArg_ParseTuple(args, "OOO", &obj, &attr_name, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + NULLABLE(value); + RETURN_INT(PyObject_SetAttr(obj, attr_name, value)); +} + +static PyObject * +object_setattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj, *value; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &obj, &attr_name, &size, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyObject_SetAttrString(obj, attr_name, value)); +} + +static PyObject * +object_delattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name; +if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + RETURN_INT(PyObject_DelAttr(obj, attr_name)); +} + +static PyObject * +object_delattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + RETURN_INT(PyObject_DelAttrString(obj, attr_name)); +} + +static PyObject * +number_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyBool_FromLong(PyNumber_Check(obj)); +} + +static PyObject * +mapping_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyMapping_Check(obj)); +} + +static PyObject * +mapping_size(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyMapping_Size(obj)); +} + +static PyObject * +mapping_length(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyMapping_Length(obj)); +} + +static PyObject * +object_getitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + return PyObject_GetItem(mapping, key); +} + +static PyObject * +mapping_getitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + return PyMapping_GetItemString(mapping, key); +} + +static PyObject * +mapping_haskey(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + return PyLong_FromLong(PyMapping_HasKey(mapping, key)); +} + +static PyObject * +mapping_haskeystring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + return PyLong_FromLong(PyMapping_HasKeyString(mapping, key)); +} + +static PyObject * +mapping_haskeywitherror(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyMapping_HasKeyWithError(mapping, key)); +} + +static PyObject * +mapping_haskeystringwitherror(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + RETURN_INT(PyMapping_HasKeyStringWithError(mapping, key)); +} + +static PyObject * +object_setitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key, *value; + if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + NULLABLE(value); + RETURN_INT(PyObject_SetItem(mapping, key, value)); +} + +static PyObject * +mapping_setitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping, *value; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(value); + RETURN_INT(PyMapping_SetItemString(mapping, key, value)); +} + +static PyObject * +object_delitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyObject_DelItem(mapping, key)); +} + +static PyObject * +mapping_delitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyMapping_DelItem(mapping, key)); +} + +static PyObject * +mapping_delitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + RETURN_INT(PyMapping_DelItemString(mapping, key)); +} + +static PyObject * +mapping_keys(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyMapping_Keys(obj); +} + +static PyObject * +mapping_values(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyMapping_Values(obj); +} + +static PyObject * +mapping_items(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyMapping_Items(obj); +} + + +static PyObject * +sequence_check(PyObject* self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PySequence_Check(obj)); +} + +static PyObject * +sequence_size(PyObject* self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PySequence_Size(obj)); +} + +static PyObject * +sequence_length(PyObject* self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PySequence_Length(obj)); +} + +static PyObject * +sequence_concat(PyObject *self, PyObject *args) +{ + PyObject *seq1, *seq2; + if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { + return NULL; + } + NULLABLE(seq1); + NULLABLE(seq2); + + return PySequence_Concat(seq1, seq2); +} + +static PyObject * +sequence_repeat(PyObject *self, PyObject *args) +{ + PyObject *seq; + Py_ssize_t count; + if (!PyArg_ParseTuple(args, "On", &seq, &count)) { + return NULL; + } + NULLABLE(seq); + + return PySequence_Repeat(seq, count); +} + +static PyObject * +sequence_inplaceconcat(PyObject *self, PyObject *args) +{ + PyObject *seq1, *seq2; + if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { + return NULL; + } + NULLABLE(seq1); + NULLABLE(seq2); + + return PySequence_InPlaceConcat(seq1, seq2); +} + +static PyObject * +sequence_inplacerepeat(PyObject *self, PyObject *args) +{ + PyObject *seq; + Py_ssize_t count; + if (!PyArg_ParseTuple(args, "On", &seq, &count)) { + return NULL; + } + NULLABLE(seq); + + return PySequence_InPlaceRepeat(seq, count); +} + +static PyObject * +sequence_getitem(PyObject *self, PyObject *args) +{ + PyObject *seq; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &seq, &i)) { + return NULL; + } + NULLABLE(seq); + + return PySequence_GetItem(seq, i); +} + +static PyObject * +sequence_setitem(PyObject *self, PyObject *args) +{ + Py_ssize_t i; + PyObject *seq, *val; + if (!PyArg_ParseTuple(args, "OnO", &seq, &i, &val)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(val); + + RETURN_INT(PySequence_SetItem(seq, i, val)); +} + + +static PyObject * +sequence_delitem(PyObject *self, PyObject *args) +{ + Py_ssize_t i; + PyObject *seq; + if (!PyArg_ParseTuple(args, "On", &seq, &i)) { + return NULL; + } + NULLABLE(seq); + + RETURN_INT(PySequence_DelItem(seq, i)); +} + +static PyObject * +sequence_setslice(PyObject* self, PyObject *args) +{ + PyObject *sequence, *obj; + Py_ssize_t i1, i2; + if (!PyArg_ParseTuple(args, "OnnO", &sequence, &i1, &i2, &obj)) { + return NULL; + } + NULLABLE(sequence); + NULLABLE(obj); + + RETURN_INT(PySequence_SetSlice(sequence, i1, i2, obj)); +} + +static PyObject * +sequence_delslice(PyObject *self, PyObject *args) +{ + PyObject *sequence; + Py_ssize_t i1, i2; + if (!PyArg_ParseTuple(args, "Onn", &sequence, &i1, &i2)) { + return NULL; + } + NULLABLE(sequence); + + RETURN_INT(PySequence_DelSlice(sequence, i1, i2)); +} + +static PyObject * +sequence_count(PyObject *self, PyObject *args) +{ + PyObject *seq, *value; + if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(value); + + RETURN_SIZE(PySequence_Count(seq, value)); +} + +static PyObject * +sequence_contains(PyObject *self, PyObject *args) +{ + PyObject *seq, *value; + if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(value); + + RETURN_INT(PySequence_Contains(seq, value)); +} + +static PyObject * +sequence_index(PyObject *self, PyObject *args) +{ + PyObject *seq, *value; + if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(value); + + RETURN_SIZE(PySequence_Index(seq, value)); +} + +static PyObject * +sequence_list(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PySequence_List(obj); +} + +static PyObject * +sequence_tuple(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PySequence_Tuple(obj); +} + + +static PyMethodDef test_methods[] = { + {"object_repr", object_repr, METH_O}, + {"object_ascii", object_ascii, METH_O}, + {"object_str", object_str, METH_O}, + {"object_bytes", object_bytes, METH_O}, + + {"object_getattr", object_getattr, METH_VARARGS}, + {"object_getattrstring", object_getattrstring, METH_VARARGS}, + {"object_hasattr", object_hasattr, METH_VARARGS}, + {"object_hasattrstring", object_hasattrstring, METH_VARARGS}, + {"object_setattr", object_setattr, METH_VARARGS}, + {"object_setattrstring", object_setattrstring, METH_VARARGS}, + {"object_delattr", object_delattr, METH_VARARGS}, + {"object_delattrstring", object_delattrstring, METH_VARARGS}, + + {"number_check", number_check, METH_O}, + {"mapping_check", mapping_check, METH_O}, + {"mapping_size", mapping_size, METH_O}, + {"mapping_length", mapping_length, METH_O}, + {"object_getitem", object_getitem, METH_VARARGS}, + {"mapping_getitemstring", mapping_getitemstring, METH_VARARGS}, + {"mapping_haskey", mapping_haskey, METH_VARARGS}, + {"mapping_haskeystring", mapping_haskeystring, METH_VARARGS}, + {"mapping_haskeywitherror", mapping_haskeywitherror, METH_VARARGS}, + {"mapping_haskeystringwitherror", mapping_haskeystringwitherror, METH_VARARGS}, + {"object_setitem", object_setitem, METH_VARARGS}, + {"mapping_setitemstring", mapping_setitemstring, METH_VARARGS}, + {"object_delitem", object_delitem, METH_VARARGS}, + {"mapping_delitem", mapping_delitem, METH_VARARGS}, + {"mapping_delitemstring", mapping_delitemstring, METH_VARARGS}, + {"mapping_keys", mapping_keys, METH_O}, + {"mapping_values", mapping_values, METH_O}, + {"mapping_items", mapping_items, METH_O}, + + {"sequence_check", sequence_check, METH_O}, + {"sequence_size", sequence_size, METH_O}, + {"sequence_length", sequence_length, METH_O}, + {"sequence_concat", sequence_concat, METH_VARARGS}, + {"sequence_repeat", sequence_repeat, METH_VARARGS}, + {"sequence_inplaceconcat", sequence_inplaceconcat, METH_VARARGS}, + {"sequence_inplacerepeat", sequence_inplacerepeat, METH_VARARGS}, + {"sequence_getitem", sequence_getitem, METH_VARARGS}, + {"sequence_setitem", sequence_setitem, METH_VARARGS}, + {"sequence_delitem", sequence_delitem, METH_VARARGS}, + {"sequence_setslice", sequence_setslice, METH_VARARGS}, + {"sequence_delslice", sequence_delslice, METH_VARARGS}, + {"sequence_count", sequence_count, METH_VARARGS}, + {"sequence_contains", sequence_contains, METH_VARARGS}, + {"sequence_index", sequence_index, METH_VARARGS}, + {"sequence_list", sequence_list, METH_O}, + {"sequence_tuple", sequence_tuple, METH_O}, + + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Abstract(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/bytearray.c b/Modules/_testlimitedcapi/bytearray.c similarity index 98% rename from Modules/_testcapi/bytearray.c rename to Modules/_testlimitedcapi/bytearray.c index dc47ed2c306f40..68b029e25af677 100644 --- a/Modules/_testcapi/bytearray.c +++ b/Modules/_testlimitedcapi/bytearray.c @@ -113,7 +113,7 @@ static PyMethodDef test_methods[] = { }; int -_PyTestCapi_Init_ByteArray(PyObject *m) +_PyTestLimitedCAPI_Init_ByteArray(PyObject *m) { if (PyModule_AddFunctions(m, test_methods) < 0) { return -1; diff --git a/Modules/_testlimitedcapi/bytes.c b/Modules/_testlimitedcapi/bytes.c new file mode 100644 index 00000000000000..157d4089954af5 --- /dev/null +++ b/Modules/_testlimitedcapi/bytes.c @@ -0,0 +1,255 @@ +#include "parts.h" +#include "util.h" + + +/* Test PyBytes_Check() */ +static PyObject * +bytes_check(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyBytes_Check(obj)); +} + +/* Test PyBytes_CheckExact() */ +static PyObject * +bytes_checkexact(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyBytes_CheckExact(obj)); +} + +/* Test PyBytes_FromStringAndSize() */ +static PyObject * +bytes_fromstringandsize(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *s; + Py_ssize_t bsize; + Py_ssize_t size = -100; + + if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { + return NULL; + } + + if (size == -100) { + size = bsize; + } + return PyBytes_FromStringAndSize(s, size); +} + +/* Test PyBytes_FromString() */ +static PyObject * +bytes_fromstring(PyObject *Py_UNUSED(module), PyObject *arg) +{ + const char *s; + Py_ssize_t size; + + if (!PyArg_Parse(arg, "z#", &s, &size)) { + return NULL; + } + return PyBytes_FromString(s); +} + +/* Test PyBytes_FromObject() */ +static PyObject * +bytes_fromobject(PyObject *Py_UNUSED(module), PyObject *arg) +{ + NULLABLE(arg); + return PyBytes_FromObject(arg); +} + +/* Test PyBytes_Size() */ +static PyObject * +bytes_size(PyObject *Py_UNUSED(module), PyObject *arg) +{ + NULLABLE(arg); + RETURN_SIZE(PyBytes_Size(arg)); +} + +/* Test PyUnicode_AsString() */ +static PyObject * +bytes_asstring(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t buflen; + const char *s; + + if (!PyArg_ParseTuple(args, "On", &obj, &buflen)) + return NULL; + + NULLABLE(obj); + s = PyBytes_AsString(obj); + if (s == NULL) + return NULL; + + return PyBytes_FromStringAndSize(s, buflen); +} + +/* Test PyBytes_AsStringAndSize() */ +static PyObject * +bytes_asstringandsize(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t buflen; + char *s = UNINITIALIZED_PTR; + Py_ssize_t size = UNINITIALIZED_SIZE; + + if (!PyArg_ParseTuple(args, "On", &obj, &buflen)) + return NULL; + + NULLABLE(obj); + if (PyBytes_AsStringAndSize(obj, &s, &size) < 0) { + return NULL; + } + + if (s == NULL) { + return Py_BuildValue("(On)", Py_None, size); + } + else { + return Py_BuildValue("(y#n)", s, buflen, size); + } +} + +static PyObject * +bytes_asstringandsize_null(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t buflen; + char *s = UNINITIALIZED_PTR; + + if (!PyArg_ParseTuple(args, "On", &obj, &buflen)) + return NULL; + + NULLABLE(obj); + if (PyBytes_AsStringAndSize(obj, &s, NULL) < 0) { + return NULL; + } + + if (s == NULL) { + Py_RETURN_NONE; + } + else { + return PyBytes_FromStringAndSize(s, buflen); + } +} + +/* Test PyBytes_Repr() */ +static PyObject * +bytes_repr(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + int smartquotes; + if (!PyArg_ParseTuple(args, "Oi", &obj, &smartquotes)) + return NULL; + + NULLABLE(obj); + return PyBytes_Repr(obj, smartquotes); +} + +/* Test PyBytes_Concat() */ +static PyObject * +bytes_concat(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *left, *right; + int new = 0; + + if (!PyArg_ParseTuple(args, "OO|p", &left, &right, &new)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + if (new) { + assert(left != NULL); + assert(PyBytes_CheckExact(left)); + left = PyBytes_FromStringAndSize(PyBytes_AsString(left), + PyBytes_Size(left)); + if (left == NULL) { + return NULL; + } + } + else { + Py_XINCREF(left); + } + PyBytes_Concat(&left, right); + if (left == NULL && !PyErr_Occurred()) { + Py_RETURN_NONE; + } + return left; +} + +/* Test PyBytes_ConcatAndDel() */ +static PyObject * +bytes_concatanddel(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *left, *right; + int new = 0; + + if (!PyArg_ParseTuple(args, "OO|p", &left, &right, &new)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + if (new) { + assert(left != NULL); + assert(PyBytes_CheckExact(left)); + left = PyBytes_FromStringAndSize(PyBytes_AsString(left), + PyBytes_Size(left)); + if (left == NULL) { + return NULL; + } + } + else { + Py_XINCREF(left); + } + Py_XINCREF(right); + PyBytes_ConcatAndDel(&left, right); + if (left == NULL && !PyErr_Occurred()) { + Py_RETURN_NONE; + } + return left; +} + +/* Test PyBytes_DecodeEscape() */ +static PyObject * +bytes_decodeescape(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *s; + Py_ssize_t bsize; + Py_ssize_t size = -100; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "z#|zn", &s, &bsize, &errors, &size)) + return NULL; + + if (size == -100) { + size = bsize; + } + return PyBytes_DecodeEscape(s, size, errors, 0, NULL); +} + + +static PyMethodDef test_methods[] = { + {"bytes_check", bytes_check, METH_O}, + {"bytes_checkexact", bytes_checkexact, METH_O}, + {"bytes_fromstringandsize", bytes_fromstringandsize, METH_VARARGS}, + {"bytes_fromstring", bytes_fromstring, METH_O}, + {"bytes_fromobject", bytes_fromobject, METH_O}, + {"bytes_size", bytes_size, METH_O}, + {"bytes_asstring", bytes_asstring, METH_VARARGS}, + {"bytes_asstringandsize", bytes_asstringandsize, METH_VARARGS}, + {"bytes_asstringandsize_null", bytes_asstringandsize_null, METH_VARARGS}, + {"bytes_repr", bytes_repr, METH_VARARGS}, + {"bytes_concat", bytes_concat, METH_VARARGS}, + {"bytes_concatanddel", bytes_concatanddel, METH_VARARGS}, + {"bytes_decodeescape", bytes_decodeescape, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Bytes(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/clinic/heaptype_relative.c.h b/Modules/_testlimitedcapi/clinic/heaptype_relative.c.h new file mode 100644 index 00000000000000..994f83102adfb0 --- /dev/null +++ b/Modules/_testlimitedcapi/clinic/heaptype_relative.c.h @@ -0,0 +1,44 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(make_heaptype_with_member__doc__, +"make_heaptype_with_member($module, /, extra_base_size=0, basicsize=0,\n" +" member_offset=0, add_relative_flag=False, *,\n" +" member_name=\'memb\', member_flags=0,\n" +" member_type=-1)\n" +"--\n" +"\n"); + +#define MAKE_HEAPTYPE_WITH_MEMBER_METHODDEF \ + {"make_heaptype_with_member", (PyCFunction)(void(*)(void))make_heaptype_with_member, METH_VARARGS|METH_KEYWORDS, make_heaptype_with_member__doc__}, + +static PyObject * +make_heaptype_with_member_impl(PyObject *module, int extra_base_size, + int basicsize, int member_offset, + int add_relative_flag, + const char *member_name, int member_flags, + int member_type); + +static PyObject * +make_heaptype_with_member(PyObject *module, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + static char *_keywords[] = {"extra_base_size", "basicsize", "member_offset", "add_relative_flag", "member_name", "member_flags", "member_type", NULL}; + int extra_base_size = 0; + int basicsize = 0; + int member_offset = 0; + int add_relative_flag = 0; + const char *member_name = "memb"; + int member_flags = 0; + int member_type = Py_T_BYTE; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|iiip$sii:make_heaptype_with_member", _keywords, + &extra_base_size, &basicsize, &member_offset, &add_relative_flag, &member_name, &member_flags, &member_type)) + goto exit; + return_value = make_heaptype_with_member_impl(module, extra_base_size, basicsize, member_offset, add_relative_flag, member_name, member_flags, member_type); + +exit: + return return_value; +} +/*[clinic end generated code: output=01933185947faecc input=a9049054013a1b77]*/ diff --git a/Modules/_testlimitedcapi/clinic/long.c.h b/Modules/_testlimitedcapi/clinic/long.c.h new file mode 100644 index 00000000000000..ebaeb53921a82f --- /dev/null +++ b/Modules/_testlimitedcapi/clinic/long.c.h @@ -0,0 +1,143 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(_testlimitedcapi_test_long_api__doc__, +"test_long_api($module, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_TEST_LONG_API_METHODDEF \ + {"test_long_api", (PyCFunction)_testlimitedcapi_test_long_api, METH_NOARGS, _testlimitedcapi_test_long_api__doc__}, + +static PyObject * +_testlimitedcapi_test_long_api_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_api(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_api_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_longlong_api__doc__, +"test_longlong_api($module, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_TEST_LONGLONG_API_METHODDEF \ + {"test_longlong_api", (PyCFunction)_testlimitedcapi_test_longlong_api, METH_NOARGS, _testlimitedcapi_test_longlong_api__doc__}, + +static PyObject * +_testlimitedcapi_test_longlong_api_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_longlong_api(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_longlong_api_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_long_and_overflow__doc__, +"test_long_and_overflow($module, /)\n" +"--\n" +"\n" +"Test the PyLong_AsLongAndOverflow API.\n" +"\n" +"General conversion to PY_LONG is tested by test_long_api_inner.\n" +"This test will concentrate on proper handling of overflow."); + +#define _TESTLIMITEDCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF \ + {"test_long_and_overflow", (PyCFunction)_testlimitedcapi_test_long_and_overflow, METH_NOARGS, _testlimitedcapi_test_long_and_overflow__doc__}, + +static PyObject * +_testlimitedcapi_test_long_and_overflow_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_and_overflow(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_and_overflow_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_long_long_and_overflow__doc__, +"test_long_long_and_overflow($module, /)\n" +"--\n" +"\n" +"Test the PyLong_AsLongLongAndOverflow API.\n" +"\n" +"General conversion to long long is tested by test_long_api_inner.\n" +"This test will concentrate on proper handling of overflow."); + +#define _TESTLIMITEDCAPI_TEST_LONG_LONG_AND_OVERFLOW_METHODDEF \ + {"test_long_long_and_overflow", (PyCFunction)_testlimitedcapi_test_long_long_and_overflow, METH_NOARGS, _testlimitedcapi_test_long_long_and_overflow__doc__}, + +static PyObject * +_testlimitedcapi_test_long_long_and_overflow_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_long_and_overflow(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_long_and_overflow_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_long_as_size_t__doc__, +"test_long_as_size_t($module, /)\n" +"--\n" +"\n" +"Test the PyLong_As{Size,Ssize}_t API.\n" +"\n" +"At present this just tests that non-integer arguments are handled correctly.\n" +"It should be extended to test overflow handling."); + +#define _TESTLIMITEDCAPI_TEST_LONG_AS_SIZE_T_METHODDEF \ + {"test_long_as_size_t", (PyCFunction)_testlimitedcapi_test_long_as_size_t, METH_NOARGS, _testlimitedcapi_test_long_as_size_t__doc__}, + +static PyObject * +_testlimitedcapi_test_long_as_size_t_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_as_size_t(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_as_size_t_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_long_as_unsigned_long_long_mask__doc__, +"test_long_as_unsigned_long_long_mask($module, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_TEST_LONG_AS_UNSIGNED_LONG_LONG_MASK_METHODDEF \ + {"test_long_as_unsigned_long_long_mask", (PyCFunction)_testlimitedcapi_test_long_as_unsigned_long_long_mask, METH_NOARGS, _testlimitedcapi_test_long_as_unsigned_long_long_mask__doc__}, + +static PyObject * +_testlimitedcapi_test_long_as_unsigned_long_long_mask_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_as_unsigned_long_long_mask(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_as_unsigned_long_long_mask_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_long_as_double__doc__, +"test_long_as_double($module, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_TEST_LONG_AS_DOUBLE_METHODDEF \ + {"test_long_as_double", (PyCFunction)_testlimitedcapi_test_long_as_double, METH_NOARGS, _testlimitedcapi_test_long_as_double__doc__}, + +static PyObject * +_testlimitedcapi_test_long_as_double_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_as_double(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_as_double_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_PyLong_AsInt__doc__, +"PyLong_AsInt($module, arg, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_PYLONG_ASINT_METHODDEF \ + {"PyLong_AsInt", (PyCFunction)_testlimitedcapi_PyLong_AsInt, METH_O, _testlimitedcapi_PyLong_AsInt__doc__}, +/*[clinic end generated code: output=bc52b73c599f96c2 input=a9049054013a1b77]*/ diff --git a/Modules/_testlimitedcapi/clinic/vectorcall_limited.c.h b/Modules/_testlimitedcapi/clinic/vectorcall_limited.c.h new file mode 100644 index 00000000000000..6631a9c42deab0 --- /dev/null +++ b/Modules/_testlimitedcapi/clinic/vectorcall_limited.c.h @@ -0,0 +1,20 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(_testlimitedcapi_call_vectorcall__doc__, +"call_vectorcall($module, callable, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_CALL_VECTORCALL_METHODDEF \ + {"call_vectorcall", (PyCFunction)_testlimitedcapi_call_vectorcall, METH_O, _testlimitedcapi_call_vectorcall__doc__}, + +PyDoc_STRVAR(_testlimitedcapi_call_vectorcall_method__doc__, +"call_vectorcall_method($module, callable, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_CALL_VECTORCALL_METHOD_METHODDEF \ + {"call_vectorcall_method", (PyCFunction)_testlimitedcapi_call_vectorcall_method, METH_O, _testlimitedcapi_call_vectorcall_method__doc__}, +/*[clinic end generated code: output=5976b9b360e1ff30 input=a9049054013a1b77]*/ diff --git a/Modules/_testlimitedcapi/clinic/version.c.h b/Modules/_testlimitedcapi/clinic/version.c.h new file mode 100644 index 00000000000000..096c7dd528b332 --- /dev/null +++ b/Modules/_testlimitedcapi/clinic/version.c.h @@ -0,0 +1,93 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(_testlimitedcapi_pack_full_version__doc__, +"pack_full_version($module, major, minor, micro, level, serial, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_PACK_FULL_VERSION_METHODDEF \ + {"pack_full_version", (PyCFunction)(void(*)(void))_testlimitedcapi_pack_full_version, METH_FASTCALL, _testlimitedcapi_pack_full_version__doc__}, + +static PyObject * +_testlimitedcapi_pack_full_version_impl(PyObject *module, int major, + int minor, int micro, int level, + int serial); + +static PyObject * +_testlimitedcapi_pack_full_version(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int major; + int minor; + int micro; + int level; + int serial; + + if (nargs != 5) { + PyErr_Format(PyExc_TypeError, "pack_full_version expected 5 arguments, got %zd", nargs); + goto exit; + } + major = PyLong_AsInt(args[0]); + if (major == -1 && PyErr_Occurred()) { + goto exit; + } + minor = PyLong_AsInt(args[1]); + if (minor == -1 && PyErr_Occurred()) { + goto exit; + } + micro = PyLong_AsInt(args[2]); + if (micro == -1 && PyErr_Occurred()) { + goto exit; + } + level = PyLong_AsInt(args[3]); + if (level == -1 && PyErr_Occurred()) { + goto exit; + } + serial = PyLong_AsInt(args[4]); + if (serial == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _testlimitedcapi_pack_full_version_impl(module, major, minor, micro, level, serial); + +exit: + return return_value; +} + +PyDoc_STRVAR(_testlimitedcapi_pack_version__doc__, +"pack_version($module, major, minor, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_PACK_VERSION_METHODDEF \ + {"pack_version", (PyCFunction)(void(*)(void))_testlimitedcapi_pack_version, METH_FASTCALL, _testlimitedcapi_pack_version__doc__}, + +static PyObject * +_testlimitedcapi_pack_version_impl(PyObject *module, int major, int minor); + +static PyObject * +_testlimitedcapi_pack_version(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int major; + int minor; + + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "pack_version expected 2 arguments, got %zd", nargs); + goto exit; + } + major = PyLong_AsInt(args[0]); + if (major == -1 && PyErr_Occurred()) { + goto exit; + } + minor = PyLong_AsInt(args[1]); + if (minor == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _testlimitedcapi_pack_version_impl(module, major, minor); + +exit: + return return_value; +} +/*[clinic end generated code: output=aed3e226da77f2d2 input=a9049054013a1b77]*/ diff --git a/Modules/_testlimitedcapi/codec.c b/Modules/_testlimitedcapi/codec.c new file mode 100644 index 00000000000000..fdc18eedc2d288 --- /dev/null +++ b/Modules/_testlimitedcapi/codec.c @@ -0,0 +1,29 @@ +#include "pyconfig.h" // Py_GIL_DISABLED + +// Need limited C API version 3.5 for PyCodec_NameReplaceErrors() +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +# define Py_LIMITED_API 0x03050000 +#endif + +#include "parts.h" + +static PyObject * +codec_namereplace_errors(PyObject *Py_UNUSED(module), PyObject *exc) +{ + assert(exc != NULL); + return PyCodec_NameReplaceErrors(exc); +} + +static PyMethodDef test_methods[] = { + {"codec_namereplace_errors", codec_namereplace_errors, METH_O}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Codec(PyObject *module) +{ + if (PyModule_AddFunctions(module, test_methods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testlimitedcapi/complex.c b/Modules/_testlimitedcapi/complex.c new file mode 100644 index 00000000000000..e4c244e5c88d06 --- /dev/null +++ b/Modules/_testlimitedcapi/complex.c @@ -0,0 +1,79 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +complex_check(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyComplex_Check(obj)); +} + +static PyObject * +complex_checkexact(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyComplex_CheckExact(obj)); +} + +static PyObject * +complex_fromdoubles(PyObject *Py_UNUSED(module), PyObject *args) +{ + double real, imag; + + if (!PyArg_ParseTuple(args, "dd", &real, &imag)) { + return NULL; + } + + return PyComplex_FromDoubles(real, imag); +} + +static PyObject * +complex_realasdouble(PyObject *Py_UNUSED(module), PyObject *obj) +{ + double real; + + NULLABLE(obj); + real = PyComplex_RealAsDouble(obj); + + if (real == -1. && PyErr_Occurred()) { + return NULL; + } + + return PyFloat_FromDouble(real); +} + +static PyObject * +complex_imagasdouble(PyObject *Py_UNUSED(module), PyObject *obj) +{ + double imag; + + NULLABLE(obj); + imag = PyComplex_ImagAsDouble(obj); + + if (imag == -1. && PyErr_Occurred()) { + return NULL; + } + + return PyFloat_FromDouble(imag); +} + + +static PyMethodDef test_methods[] = { + {"complex_check", complex_check, METH_O}, + {"complex_checkexact", complex_checkexact, METH_O}, + {"complex_fromdoubles", complex_fromdoubles, METH_VARARGS}, + {"complex_realasdouble", complex_realasdouble, METH_O}, + {"complex_imagasdouble", complex_imagasdouble, METH_O}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Complex(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/dict.c b/Modules/_testlimitedcapi/dict.c new file mode 100644 index 00000000000000..ec32712eef6434 --- /dev/null +++ b/Modules/_testlimitedcapi/dict.c @@ -0,0 +1,291 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +dict_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyDict_Check(obj)); +} + +static PyObject * +dict_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyDict_CheckExact(obj)); +} + +static PyObject * +dict_new(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyDict_New(); +} + +static PyObject * +dictproxy_new(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDictProxy_New(obj); +} + +static PyObject * +dict_clear(PyObject *self, PyObject *obj) +{ + PyDict_Clear(obj); + Py_RETURN_NONE; +} + +static PyObject * +dict_copy(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Copy(obj); +} + +static PyObject * +dict_contains(PyObject *self, PyObject *args) +{ + PyObject *obj, *key; + if (!PyArg_ParseTuple(args, "OO", &obj, &key)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(key); + RETURN_INT(PyDict_Contains(obj, key)); +} + +static PyObject * +dict_size(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyDict_Size(obj)); +} + +static PyObject * +dict_getitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + PyObject *value = PyDict_GetItem(mapping, key); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return Py_NewRef(PyExc_KeyError); + } + return Py_NewRef(value); +} + +static PyObject * +dict_getitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + PyObject *value = PyDict_GetItemString(mapping, key); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return Py_NewRef(PyExc_KeyError); + } + return Py_NewRef(value); +} + +static PyObject * +dict_getitemwitherror(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + PyObject *value = PyDict_GetItemWithError(mapping, key); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return Py_NewRef(PyExc_KeyError); + } + return Py_NewRef(value); +} + + +static PyObject * +dict_setitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key, *value; + if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + NULLABLE(value); + RETURN_INT(PyDict_SetItem(mapping, key, value)); +} + +static PyObject * +dict_setitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping, *value; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(value); + RETURN_INT(PyDict_SetItemString(mapping, key, value)); +} + +static PyObject * +dict_delitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyDict_DelItem(mapping, key)); +} + +static PyObject * +dict_delitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + RETURN_INT(PyDict_DelItemString(mapping, key)); +} + +static PyObject * +dict_keys(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Keys(obj); +} + +static PyObject * +dict_values(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Values(obj); +} + +static PyObject * +dict_items(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Items(obj); +} + +static PyObject * +dict_next(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key = UNINITIALIZED_PTR, *value = UNINITIALIZED_PTR; + Py_ssize_t pos; + if (!PyArg_ParseTuple(args, "On", &mapping, &pos)) { + return NULL; + } + NULLABLE(mapping); + int rc = PyDict_Next(mapping, &pos, &key, &value); + if (rc != 0) { + return Py_BuildValue("inOO", rc, pos, key, value); + } + assert(key == UNINITIALIZED_PTR); + assert(value == UNINITIALIZED_PTR); + if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +dict_merge(PyObject *self, PyObject *args) +{ + PyObject *mapping, *mapping2; + int override; + if (!PyArg_ParseTuple(args, "OOi", &mapping, &mapping2, &override)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(mapping2); + RETURN_INT(PyDict_Merge(mapping, mapping2, override)); +} + +static PyObject * +dict_update(PyObject *self, PyObject *args) +{ + PyObject *mapping, *mapping2; + if (!PyArg_ParseTuple(args, "OO", &mapping, &mapping2)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(mapping2); + RETURN_INT(PyDict_Update(mapping, mapping2)); +} + +static PyObject * +dict_mergefromseq2(PyObject *self, PyObject *args) +{ + PyObject *mapping, *seq; + int override; + if (!PyArg_ParseTuple(args, "OOi", &mapping, &seq, &override)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(seq); + RETURN_INT(PyDict_MergeFromSeq2(mapping, seq, override)); +} + + +static PyMethodDef test_methods[] = { + {"dict_check", dict_check, METH_O}, + {"dict_checkexact", dict_checkexact, METH_O}, + {"dict_new", dict_new, METH_NOARGS}, + {"dictproxy_new", dictproxy_new, METH_O}, + {"dict_clear", dict_clear, METH_O}, + {"dict_copy", dict_copy, METH_O}, + {"dict_size", dict_size, METH_O}, + {"dict_getitem", dict_getitem, METH_VARARGS}, + {"dict_getitemwitherror", dict_getitemwitherror, METH_VARARGS}, + {"dict_getitemstring", dict_getitemstring, METH_VARARGS}, + {"dict_contains", dict_contains, METH_VARARGS}, + {"dict_setitem", dict_setitem, METH_VARARGS}, + {"dict_setitemstring", dict_setitemstring, METH_VARARGS}, + {"dict_delitem", dict_delitem, METH_VARARGS}, + {"dict_delitemstring", dict_delitemstring, METH_VARARGS}, + {"dict_keys", dict_keys, METH_O}, + {"dict_values", dict_values, METH_O}, + {"dict_items", dict_items, METH_O}, + {"dict_next", dict_next, METH_VARARGS}, + {"dict_merge", dict_merge, METH_VARARGS}, + {"dict_update", dict_update, METH_VARARGS}, + {"dict_mergefromseq2", dict_mergefromseq2, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Dict(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/eval.c b/Modules/_testlimitedcapi/eval.c new file mode 100644 index 00000000000000..28f5746dfb1783 --- /dev/null +++ b/Modules/_testlimitedcapi/eval.c @@ -0,0 +1,95 @@ +#include "parts.h" +#include "util.h" + +static PyObject * +eval_get_func_name(PyObject *self, PyObject *func) +{ + return PyUnicode_FromString(PyEval_GetFuncName(func)); +} + +static PyObject * +eval_get_func_desc(PyObject *self, PyObject *func) +{ + return PyUnicode_FromString(PyEval_GetFuncDesc(func)); +} + +static PyObject * +eval_getlocals(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return Py_XNewRef(PyEval_GetLocals()); +} + +static PyObject * +eval_getglobals(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return Py_XNewRef(PyEval_GetGlobals()); +} + +static PyObject * +eval_getbuiltins(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return Py_XNewRef(PyEval_GetBuiltins()); +} + +static PyObject * +eval_getframe(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return Py_XNewRef(PyEval_GetFrame()); +} + +static PyObject * +eval_getframe_builtins(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return PyEval_GetFrameBuiltins(); +} + +static PyObject * +eval_getframe_globals(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return PyEval_GetFrameGlobals(); +} + +static PyObject * +eval_getframe_locals(PyObject *module, PyObject *Py_UNUSED(args)) +{ + return PyEval_GetFrameLocals(); +} + +static PyObject * +eval_get_recursion_limit(PyObject *module, PyObject *Py_UNUSED(args)) +{ + int limit = Py_GetRecursionLimit(); + return PyLong_FromLong(limit); +} + +static PyObject * +eval_set_recursion_limit(PyObject *module, PyObject *args) +{ + int limit; + if (!PyArg_ParseTuple(args, "i", &limit)) { + return NULL; + } + Py_SetRecursionLimit(limit); + Py_RETURN_NONE; +} + +static PyMethodDef test_methods[] = { + {"eval_get_func_name", eval_get_func_name, METH_O, NULL}, + {"eval_get_func_desc", eval_get_func_desc, METH_O, NULL}, + {"eval_getlocals", eval_getlocals, METH_NOARGS}, + {"eval_getglobals", eval_getglobals, METH_NOARGS}, + {"eval_getbuiltins", eval_getbuiltins, METH_NOARGS}, + {"eval_getframe", eval_getframe, METH_NOARGS}, + {"eval_getframe_builtins", eval_getframe_builtins, METH_NOARGS}, + {"eval_getframe_globals", eval_getframe_globals, METH_NOARGS}, + {"eval_getframe_locals", eval_getframe_locals, METH_NOARGS}, + {"eval_get_recursion_limit", eval_get_recursion_limit, METH_NOARGS}, + {"eval_set_recursion_limit", eval_set_recursion_limit, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Eval(PyObject *m) +{ + return PyModule_AddFunctions(m, test_methods); +} diff --git a/Modules/_testlimitedcapi/float.c b/Modules/_testlimitedcapi/float.c new file mode 100644 index 00000000000000..88dc91f682ff39 --- /dev/null +++ b/Modules/_testlimitedcapi/float.c @@ -0,0 +1,91 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +float_check(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyFloat_Check(obj)); +} + +static PyObject * +float_checkexact(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyFloat_CheckExact(obj)); +} + +static PyObject * +float_fromstring(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyFloat_FromString(obj); +} + +static PyObject * +float_fromdouble(PyObject *Py_UNUSED(module), PyObject *obj) +{ + double d; + + if (!PyArg_Parse(obj, "d", &d)) { + return NULL; + } + + return PyFloat_FromDouble(d); +} + +static PyObject * +float_asdouble(PyObject *Py_UNUSED(module), PyObject *obj) +{ + double d; + + NULLABLE(obj); + d = PyFloat_AsDouble(obj); + if (d == -1. && PyErr_Occurred()) { + return NULL; + } + + return PyFloat_FromDouble(d); +} + +static PyObject * +float_getinfo(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) +{ + return PyFloat_GetInfo(); +} + +static PyObject * +float_getmax(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) +{ + return PyFloat_FromDouble(PyFloat_GetMax()); +} + +static PyObject * +float_getmin(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) +{ + return PyFloat_FromDouble(PyFloat_GetMin()); +} + + +static PyMethodDef test_methods[] = { + {"float_check", float_check, METH_O}, + {"float_checkexact", float_checkexact, METH_O}, + {"float_fromstring", float_fromstring, METH_O}, + {"float_fromdouble", float_fromdouble, METH_O}, + {"float_asdouble", float_asdouble, METH_O}, + {"float_getinfo", float_getinfo, METH_NOARGS}, + {"float_getmax", float_getmax, METH_NOARGS}, + {"float_getmin", float_getmin, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Float(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/heaptype_relative.c b/Modules/_testlimitedcapi/heaptype_relative.c similarity index 65% rename from Modules/_testcapi/heaptype_relative.c rename to Modules/_testlimitedcapi/heaptype_relative.c index 52bda75736b316..45d65ee47349f9 100644 --- a/Modules/_testcapi/heaptype_relative.c +++ b/Modules/_testlimitedcapi/heaptype_relative.c @@ -1,13 +1,15 @@ +// Need limited C API version 3.12 for PyType_FromMetaclass() #include "pyconfig.h" // Py_GIL_DISABLED - -#ifndef Py_GIL_DISABLED -#define Py_LIMITED_API 0x030c0000 // 3.12 +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +# define Py_LIMITED_API 0x030c0000 #endif #include "parts.h" #include // max_align_t #include // memset +#include "clinic/heaptype_relative.c.h" + static PyType_Slot empty_slots[] = { {0, NULL}, }; @@ -247,6 +249,81 @@ heaptype_with_member_set_memb_relative(PyObject *self, PyObject *value) Py_RETURN_NONE; } +typedef struct { + int padding; // just so the offset isn't 0 + PyObject *dict; +} HeapCTypeWithDictStruct; + +static void +heapctypewithrelativedict_dealloc(PyObject* self) +{ + PyTypeObject *tp = Py_TYPE(self); + HeapCTypeWithDictStruct *data = PyObject_GetTypeData(self, tp); + Py_XDECREF(data->dict); + PyObject_Free(self); + Py_DECREF(tp); +} + +static PyType_Spec HeapCTypeWithRelativeDict_spec = { + .name = "_testcapi.HeapCTypeWithRelativeDict", + .basicsize = -(int)sizeof(HeapCTypeWithDictStruct), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = (PyType_Slot[]) { + {Py_tp_dealloc, heapctypewithrelativedict_dealloc}, + {Py_tp_getset, (PyGetSetDef[]) { + {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict}, + {NULL} /* Sentinel */ + }}, + {Py_tp_members, (PyMemberDef[]) { + {"dictobj", _Py_T_OBJECT, + offsetof(HeapCTypeWithDictStruct, dict), + Py_RELATIVE_OFFSET}, + {"__dictoffset__", Py_T_PYSSIZET, + offsetof(HeapCTypeWithDictStruct, dict), + Py_READONLY | Py_RELATIVE_OFFSET}, + {NULL} /* Sentinel */ + }}, + {0, 0}, + } +}; + +typedef struct { + char padding; // just so the offset isn't 0 + PyObject *weakreflist; +} HeapCTypeWithWeakrefStruct; + +static void +heapctypewithrelativeweakref_dealloc(PyObject* self) +{ + PyTypeObject *tp = Py_TYPE(self); + HeapCTypeWithWeakrefStruct *data = PyObject_GetTypeData(self, tp); + if (data->weakreflist != NULL) { + PyObject_ClearWeakRefs(self); + } + Py_XDECREF(data->weakreflist); + PyObject_Free(self); + Py_DECREF(tp); +} + +static PyType_Spec HeapCTypeWithRelativeWeakref_spec = { + .name = "_testcapi.HeapCTypeWithRelativeWeakref", + .basicsize = -(int)sizeof(HeapCTypeWithWeakrefStruct), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = (PyType_Slot[]) { + {Py_tp_dealloc, heapctypewithrelativeweakref_dealloc}, + {Py_tp_members, (PyMemberDef[]) { + {"weakreflist", _Py_T_OBJECT, + offsetof(HeapCTypeWithWeakrefStruct, weakreflist), + Py_RELATIVE_OFFSET}, + {"__weaklistoffset__", Py_T_PYSSIZET, + offsetof(HeapCTypeWithWeakrefStruct, weakreflist), + Py_READONLY | Py_RELATIVE_OFFSET}, + {NULL} /* Sentinel */ + }}, + {0, 0}, + } +}; + static PyMethodDef heaptype_with_member_methods[] = { {"get_memb", heaptype_with_member_get_memb, METH_NOARGS}, {"set_memb", heaptype_with_member_set_memb, METH_O}, @@ -256,19 +333,31 @@ static PyMethodDef heaptype_with_member_methods[] = { {NULL}, }; +/*[clinic input] +make_heaptype_with_member + + extra_base_size: int = 0 + basicsize: int = 0 + member_offset: int = 0 + add_relative_flag: bool = False + * + member_name: str = "memb" + member_flags: int = 0 + member_type: int(c_default="Py_T_BYTE") = -1 + +[clinic start generated code]*/ + static PyObject * -make_heaptype_with_member(PyObject *module, PyObject *args) +make_heaptype_with_member_impl(PyObject *module, int extra_base_size, + int basicsize, int member_offset, + int add_relative_flag, + const char *member_name, int member_flags, + int member_type) +/*[clinic end generated code: output=7005db9a07396997 input=007e29cdbe1d3390]*/ { PyObject *base = NULL; PyObject *result = NULL; - int extra_base_size, basicsize, offset, add_flag; - - int r = PyArg_ParseTuple(args, "iiip", &extra_base_size, &basicsize, &offset, &add_flag); - if (!r) { - goto finally; - } - PyType_Spec base_spec = { .name = "_testcapi.Base", .basicsize = sizeof(PyObject) + extra_base_size, @@ -281,7 +370,8 @@ make_heaptype_with_member(PyObject *module, PyObject *args) } PyMemberDef members[] = { - {"memb", Py_T_BYTE, offset, add_flag ? Py_RELATIVE_OFFSET : 0}, + {member_name, member_type, member_offset, + member_flags | (add_relative_flag ? Py_RELATIVE_OFFSET : 0)}, {0}, }; PyType_Slot slots[] = { @@ -325,13 +415,14 @@ static PyMethodDef TestMethods[] = { {"make_sized_heaptypes", make_sized_heaptypes, METH_VARARGS}, {"subclass_var_heaptype", subclass_var_heaptype, METH_VARARGS}, {"subclass_heaptype", subclass_heaptype, METH_VARARGS}, - {"make_heaptype_with_member", make_heaptype_with_member, METH_VARARGS}, + MAKE_HEAPTYPE_WITH_MEMBER_METHODDEF {"test_alignof_max_align_t", test_alignof_max_align_t, METH_NOARGS}, {NULL}, }; int -_PyTestCapi_Init_HeaptypeRelative(PyObject *m) { +_PyTestLimitedCAPI_Init_HeaptypeRelative(PyObject *m) +{ if (PyModule_AddFunctions(m, TestMethods) < 0) { return -1; } @@ -340,5 +431,42 @@ _PyTestCapi_Init_HeaptypeRelative(PyObject *m) { return -1; } +#define ADD_FROM_SPEC(SPEC) do { \ + PyObject *tp = PyType_FromSpec(SPEC); \ + if (!tp) { \ + return -1; \ + } \ + if (PyModule_AddType(m, (PyTypeObject *)tp) < 0) { \ + return -1; \ + } \ + } while (0) + + PyObject *tp; + + tp = PyType_FromSpec(&HeapCTypeWithRelativeDict_spec); + if (!tp) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject *)tp) < 0) { + return -1; + } + Py_DECREF(tp); + + tp = PyType_FromSpec(&HeapCTypeWithRelativeWeakref_spec); + if (!tp) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject *)tp) < 0) { + return -1; + } + Py_DECREF(tp); + + if (PyModule_AddIntMacro(m, Py_T_PYSSIZET) < 0) { + return -1; + } + if (PyModule_AddIntMacro(m, Py_READONLY) < 0) { + return -1; + } + return 0; } diff --git a/Modules/_testlimitedcapi/import.c b/Modules/_testlimitedcapi/import.c new file mode 100644 index 00000000000000..3707dbedeea0d9 --- /dev/null +++ b/Modules/_testlimitedcapi/import.c @@ -0,0 +1,306 @@ +// Need limited C API version 3.13 for PyImport_AddModuleRef() +#include "pyconfig.h" // Py_GIL_DISABLED +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +# define Py_LIMITED_API 0x030d0000 +#endif + +#include "parts.h" +#include "util.h" + + +/* Test PyImport_GetMagicNumber() */ +static PyObject * +pyimport_getmagicnumber(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + long magic = PyImport_GetMagicNumber(); + return PyLong_FromLong(magic); +} + + +/* Test PyImport_GetMagicTag() */ +static PyObject * +pyimport_getmagictag(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + const char *tag = PyImport_GetMagicTag(); + return PyUnicode_FromString(tag); +} + + +/* Test PyImport_GetModuleDict() */ +static PyObject * +pyimport_getmoduledict(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + return Py_XNewRef(PyImport_GetModuleDict()); +} + + +/* Test PyImport_GetModule() */ +static PyObject * +pyimport_getmodule(PyObject *Py_UNUSED(module), PyObject *name) +{ + assert(!PyErr_Occurred()); + NULLABLE(name); + PyObject *module = PyImport_GetModule(name); + if (module == NULL && !PyErr_Occurred()) { + return Py_NewRef(PyExc_KeyError); + } + return module; +} + + +/* Test PyImport_AddModuleObject() */ +static PyObject * +pyimport_addmoduleobject(PyObject *Py_UNUSED(module), PyObject *name) +{ + NULLABLE(name); + return Py_XNewRef(PyImport_AddModuleObject(name)); +} + + +/* Test PyImport_AddModule() */ +static PyObject * +pyimport_addmodule(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "z#", &name, &size)) { + return NULL; + } + + return Py_XNewRef(PyImport_AddModule(name)); +} + + +/* Test PyImport_AddModuleRef() */ +static PyObject * +pyimport_addmoduleref(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "z#", &name, &size)) { + return NULL; + } + + return PyImport_AddModuleRef(name); +} + + +/* Test PyImport_Import() */ +static PyObject * +pyimport_import(PyObject *Py_UNUSED(module), PyObject *name) +{ + NULLABLE(name); + return PyImport_Import(name); +} + + +/* Test PyImport_ImportModule() */ +static PyObject * +pyimport_importmodule(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "z#", &name, &size)) { + return NULL; + } + + return PyImport_ImportModule(name); +} + + +/* Test PyImport_ImportModuleNoBlock() */ +static PyObject * +pyimport_importmodulenoblock(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "z#", &name, &size)) { + return NULL; + } + + _Py_COMP_DIAG_PUSH + _Py_COMP_DIAG_IGNORE_DEPR_DECLS + return PyImport_ImportModuleNoBlock(name); + _Py_COMP_DIAG_POP +} + + +/* Test PyImport_ImportModuleEx() */ +static PyObject * +pyimport_importmoduleex(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + PyObject *globals, *locals, *fromlist; + if (!PyArg_ParseTuple(args, "z#OOO", + &name, &size, &globals, &locals, &fromlist)) { + return NULL; + } + NULLABLE(globals); + NULLABLE(locals); + NULLABLE(fromlist); + + return PyImport_ImportModuleEx(name, globals, locals, fromlist); +} + + +/* Test PyImport_ImportModuleLevel() */ +static PyObject * +pyimport_importmodulelevel(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + PyObject *globals, *locals, *fromlist; + int level; + if (!PyArg_ParseTuple(args, "z#OOOi", + &name, &size, &globals, &locals, &fromlist, &level)) { + return NULL; + } + NULLABLE(globals); + NULLABLE(locals); + NULLABLE(fromlist); + + return PyImport_ImportModuleLevel(name, globals, locals, fromlist, level); +} + + +/* Test PyImport_ImportModuleLevelObject() */ +static PyObject * +pyimport_importmodulelevelobject(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *name, *globals, *locals, *fromlist; + int level; + if (!PyArg_ParseTuple(args, "OOOOi", + &name, &globals, &locals, &fromlist, &level)) { + return NULL; + } + NULLABLE(name); + NULLABLE(globals); + NULLABLE(locals); + NULLABLE(fromlist); + + return PyImport_ImportModuleLevelObject(name, globals, locals, fromlist, level); +} + + +/* Test PyImport_ImportFrozenModule() */ +static PyObject * +pyimport_importfrozenmodule(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "z#", &name, &size)) { + return NULL; + } + + RETURN_INT(PyImport_ImportFrozenModule(name)); +} + + +/* Test PyImport_ImportFrozenModuleObject() */ +static PyObject * +pyimport_importfrozenmoduleobject(PyObject *Py_UNUSED(module), PyObject *name) +{ + NULLABLE(name); + RETURN_INT(PyImport_ImportFrozenModuleObject(name)); +} + + +/* Test PyImport_ExecCodeModule() */ +static PyObject * +pyimport_executecodemodule(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + PyObject *code; + if (!PyArg_ParseTuple(args, "z#O", &name, &size, &code)) { + return NULL; + } + NULLABLE(code); + + return PyImport_ExecCodeModule(name, code); +} + + +/* Test PyImport_ExecCodeModuleEx() */ +static PyObject * +pyimport_executecodemoduleex(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + PyObject *code; + const char *pathname; + if (!PyArg_ParseTuple(args, "z#Oz#", &name, &size, &code, &pathname, &size)) { + return NULL; + } + NULLABLE(code); + + return PyImport_ExecCodeModuleEx(name, code, pathname); +} + + +/* Test PyImport_ExecCodeModuleWithPathnames() */ +static PyObject * +pyimport_executecodemodulewithpathnames(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + PyObject *code; + const char *pathname; + const char *cpathname; + if (!PyArg_ParseTuple(args, "z#Oz#z#", &name, &size, &code, &pathname, &size, &cpathname, &size)) { + return NULL; + } + NULLABLE(code); + + return PyImport_ExecCodeModuleWithPathnames(name, code, + pathname, cpathname); +} + + +/* Test PyImport_ExecCodeModuleObject() */ +static PyObject * +pyimport_executecodemoduleobject(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *name, *code, *pathname, *cpathname; + if (!PyArg_ParseTuple(args, "OOOO", &name, &code, &pathname, &cpathname)) { + return NULL; + } + NULLABLE(name); + NULLABLE(code); + NULLABLE(pathname); + NULLABLE(cpathname); + + return PyImport_ExecCodeModuleObject(name, code, pathname, cpathname); +} + + +static PyMethodDef test_methods[] = { + {"PyImport_GetMagicNumber", pyimport_getmagicnumber, METH_NOARGS}, + {"PyImport_GetMagicTag", pyimport_getmagictag, METH_NOARGS}, + {"PyImport_GetModuleDict", pyimport_getmoduledict, METH_NOARGS}, + {"PyImport_GetModule", pyimport_getmodule, METH_O}, + {"PyImport_AddModuleObject", pyimport_addmoduleobject, METH_O}, + {"PyImport_AddModule", pyimport_addmodule, METH_VARARGS}, + {"PyImport_AddModuleRef", pyimport_addmoduleref, METH_VARARGS}, + {"PyImport_Import", pyimport_import, METH_O}, + {"PyImport_ImportModule", pyimport_importmodule, METH_VARARGS}, + {"PyImport_ImportModuleNoBlock", pyimport_importmodulenoblock, METH_VARARGS}, + {"PyImport_ImportModuleEx", pyimport_importmoduleex, METH_VARARGS}, + {"PyImport_ImportModuleLevel", pyimport_importmodulelevel, METH_VARARGS}, + {"PyImport_ImportModuleLevelObject", pyimport_importmodulelevelobject, METH_VARARGS}, + {"PyImport_ImportFrozenModule", pyimport_importfrozenmodule, METH_VARARGS}, + {"PyImport_ImportFrozenModuleObject", pyimport_importfrozenmoduleobject, METH_O}, + {"PyImport_ExecCodeModule", pyimport_executecodemodule, METH_VARARGS}, + {"PyImport_ExecCodeModuleEx", pyimport_executecodemoduleex, METH_VARARGS}, + {"PyImport_ExecCodeModuleWithPathnames", pyimport_executecodemodulewithpathnames, METH_VARARGS}, + {"PyImport_ExecCodeModuleObject", pyimport_executecodemoduleobject, METH_VARARGS}, + {NULL}, +}; + + +int +_PyTestLimitedCAPI_Init_Import(PyObject *module) +{ + return PyModule_AddFunctions(module, test_methods); +} diff --git a/Modules/_testlimitedcapi/list.c b/Modules/_testlimitedcapi/list.c new file mode 100644 index 00000000000000..ed492c3e719727 --- /dev/null +++ b/Modules/_testlimitedcapi/list.c @@ -0,0 +1,175 @@ +// Need limited C API version 3.13 for PyList_GetItemRef() +#include "pyconfig.h" // Py_GIL_DISABLED +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +# define Py_LIMITED_API 0x030d0000 +#endif + +#include "parts.h" +#include "util.h" + +static PyObject * +list_check(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyList_Check(obj)); +} + +static PyObject * +list_check_exact(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyList_CheckExact(obj)); +} + +static PyObject * +list_new(PyObject* Py_UNUSED(module), PyObject *obj) +{ + return PyList_New(PyLong_AsSsize_t(obj)); +} + +static PyObject * +list_size(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyList_Size(obj)); +} + +static PyObject * +list_getitem(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &obj, &i)) { + return NULL; + } + NULLABLE(obj); + return Py_XNewRef(PyList_GetItem(obj, i)); +} + +static PyObject * +list_get_item_ref(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &obj, &i)) { + return NULL; + } + NULLABLE(obj); + return PyList_GetItemRef(obj, i); +} + +static PyObject * +list_setitem(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "OnO", &obj, &i, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyList_SetItem(obj, i, Py_XNewRef(value))); + +} + +static PyObject * +list_insert(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value; + Py_ssize_t where; + if (!PyArg_ParseTuple(args, "OnO", &obj, &where, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyList_Insert(obj, where, Py_XNewRef(value))); + +} + +static PyObject * +list_append(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value; + if (!PyArg_ParseTuple(args, "OO", &obj, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyList_Append(obj, value)); +} + +static PyObject * +list_getslice(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t ilow, ihigh; + if (!PyArg_ParseTuple(args, "Onn", &obj, &ilow, &ihigh)) { + return NULL; + } + NULLABLE(obj); + return PyList_GetSlice(obj, ilow, ihigh); + +} + +static PyObject * +list_setslice(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value; + Py_ssize_t ilow, ihigh; + if (!PyArg_ParseTuple(args, "OnnO", &obj, &ilow, &ihigh, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyList_SetSlice(obj, ilow, ihigh, value)); +} + +static PyObject * +list_sort(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyList_Sort(obj)); +} + +static PyObject * +list_reverse(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyList_Reverse(obj)); +} + +static PyObject * +list_astuple(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyList_AsTuple(obj); +} + + +static PyMethodDef test_methods[] = { + {"list_check", list_check, METH_O}, + {"list_check_exact", list_check_exact, METH_O}, + {"list_new", list_new, METH_O}, + {"list_size", list_size, METH_O}, + {"list_getitem", list_getitem, METH_VARARGS}, + {"list_get_item_ref", list_get_item_ref, METH_VARARGS}, + {"list_setitem", list_setitem, METH_VARARGS}, + {"list_insert", list_insert, METH_VARARGS}, + {"list_append", list_append, METH_VARARGS}, + {"list_getslice", list_getslice, METH_VARARGS}, + {"list_setslice", list_setslice, METH_VARARGS}, + {"list_sort", list_sort, METH_O}, + {"list_reverse", list_reverse, METH_O}, + {"list_astuple", list_astuple, METH_O}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_List(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/long.c b/Modules/_testlimitedcapi/long.c new file mode 100644 index 00000000000000..b9c35803b423c2 --- /dev/null +++ b/Modules/_testlimitedcapi/long.c @@ -0,0 +1,848 @@ +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED + // Need limited C API 3.14 to test PyLong_AsInt64() +# define Py_LIMITED_API 0x030e0000 +#endif + +#include "parts.h" +#include "util.h" +#include "clinic/long.c.h" + +/*[clinic input] +module _testlimitedcapi +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=2700057f9c1135ba]*/ + + +static PyObject * +raiseTestError(const char* test_name, const char* msg) +{ + PyErr_Format(PyExc_AssertionError, "%s: %s", test_name, msg); + return NULL; +} + +/* Tests of PyLong_{As, From}{Unsigned,}Long(), and + PyLong_{As, From}{Unsigned,}LongLong(). + + Note that the meat of the test is contained in testcapi_long.h. + This is revolting, but delicate code duplication is worse: "almost + exactly the same" code is needed to test long long, but the ubiquitous + dependence on type names makes it impossible to use a parameterized + function. A giant macro would be even worse than this. A C++ template + would be perfect. + + The "report an error" functions are deliberately not part of the #include + file: if the test fails, you can set a breakpoint in the appropriate + error function directly, and crawl back from there in the debugger. +*/ + +#define UNBIND(X) Py_DECREF(X); (X) = NULL + +static PyObject * +raise_test_long_error(const char* msg) +{ + return raiseTestError("test_long_api", msg); +} + +// Test PyLong_FromLong()/PyLong_AsLong() +// and PyLong_FromUnsignedLong()/PyLong_AsUnsignedLong(). + +#define TESTNAME test_long_api_inner +#define TYPENAME long +#define F_S_TO_PY PyLong_FromLong +#define F_PY_TO_S PyLong_AsLong +#define F_U_TO_PY PyLong_FromUnsignedLong +#define F_PY_TO_U PyLong_AsUnsignedLong + +#include "testcapi_long.h" + +/*[clinic input] +_testlimitedcapi.test_long_api +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_api_impl(PyObject *module) +/*[clinic end generated code: output=06a2c02366d1853a input=9012b3d6a483df63]*/ +{ + return TESTNAME(raise_test_long_error); +} + +#undef TESTNAME +#undef TYPENAME +#undef F_S_TO_PY +#undef F_PY_TO_S +#undef F_U_TO_PY +#undef F_PY_TO_U + +// Test PyLong_FromLongLong()/PyLong_AsLongLong() +// and PyLong_FromUnsignedLongLong()/PyLong_AsUnsignedLongLong(). + +static PyObject * +raise_test_longlong_error(const char* msg) +{ + return raiseTestError("test_longlong_api", msg); +} + +#define TESTNAME test_longlong_api_inner +#define TYPENAME long long +#define F_S_TO_PY PyLong_FromLongLong +#define F_PY_TO_S PyLong_AsLongLong +#define F_U_TO_PY PyLong_FromUnsignedLongLong +#define F_PY_TO_U PyLong_AsUnsignedLongLong + +#include "testcapi_long.h" + +/*[clinic input] +_testlimitedcapi.test_longlong_api +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_longlong_api_impl(PyObject *module) +/*[clinic end generated code: output=8faa10e1c35214bf input=2b582a9d25bd68e7]*/ +{ + return TESTNAME(raise_test_longlong_error); +} + +#undef TESTNAME +#undef TYPENAME +#undef F_S_TO_PY +#undef F_PY_TO_S +#undef F_U_TO_PY +#undef F_PY_TO_U + + +/*[clinic input] +_testlimitedcapi.test_long_and_overflow + +Test the PyLong_AsLongAndOverflow API. + +General conversion to PY_LONG is tested by test_long_api_inner. +This test will concentrate on proper handling of overflow. +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_and_overflow_impl(PyObject *module) +/*[clinic end generated code: output=fdfd3c1eeabb6d14 input=e3a18791de6519fe]*/ +{ + PyObject *num, *one, *temp; + long value; + int overflow; + + /* Test that overflow is set properly for a large value. */ + /* num is a number larger than LONG_MAX even on 64-bit platforms */ + num = PyLong_FromString("FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_and_overflow", + "return value was not set to -1"); + if (overflow != 1) + return raiseTestError("test_long_and_overflow", + "overflow was not set to 1"); + + /* Same again, with num = LONG_MAX + 1 */ + num = PyLong_FromLong(LONG_MAX); + if (num == NULL) + return NULL; + one = PyLong_FromLong(1L); + if (one == NULL) { + Py_DECREF(num); + return NULL; + } + temp = PyNumber_Add(num, one); + Py_DECREF(one); + Py_DECREF(num); + num = temp; + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_and_overflow", + "return value was not set to -1"); + if (overflow != 1) + return raiseTestError("test_long_and_overflow", + "overflow was not set to 1"); + + /* Test that overflow is set properly for a large negative value. */ + /* num is a number smaller than LONG_MIN even on 64-bit platforms */ + num = PyLong_FromString("-FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_and_overflow", + "return value was not set to -1"); + if (overflow != -1) + return raiseTestError("test_long_and_overflow", + "overflow was not set to -1"); + + /* Same again, with num = LONG_MIN - 1 */ + num = PyLong_FromLong(LONG_MIN); + if (num == NULL) + return NULL; + one = PyLong_FromLong(1L); + if (one == NULL) { + Py_DECREF(num); + return NULL; + } + temp = PyNumber_Subtract(num, one); + Py_DECREF(one); + Py_DECREF(num); num = temp; + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_and_overflow", + "return value was not set to -1"); + if (overflow != -1) + return raiseTestError("test_long_and_overflow", + "overflow was not set to -1"); + + /* Test that overflow is cleared properly for small values. */ + num = PyLong_FromString("FF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != 0xFF) + return raiseTestError("test_long_and_overflow", + "expected return value 0xFF"); + if (overflow != 0) + return raiseTestError("test_long_and_overflow", + "overflow was not cleared"); + + num = PyLong_FromString("-FF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -0xFF) + return raiseTestError("test_long_and_overflow", + "expected return value 0xFF"); + if (overflow != 0) + return raiseTestError("test_long_and_overflow", + "overflow was set incorrectly"); + + num = PyLong_FromLong(LONG_MAX); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != LONG_MAX) + return raiseTestError("test_long_and_overflow", + "expected return value LONG_MAX"); + if (overflow != 0) + return raiseTestError("test_long_and_overflow", + "overflow was not cleared"); + + num = PyLong_FromLong(LONG_MIN); + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != LONG_MIN) + return raiseTestError("test_long_and_overflow", + "expected return value LONG_MIN"); + if (overflow != 0) + return raiseTestError("test_long_and_overflow", + "overflow was not cleared"); + + Py_RETURN_NONE; +} + +/*[clinic input] +_testlimitedcapi.test_long_long_and_overflow + +Test the PyLong_AsLongLongAndOverflow API. + +General conversion to long long is tested by test_long_api_inner. +This test will concentrate on proper handling of overflow. +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_long_and_overflow_impl(PyObject *module) +/*[clinic end generated code: output=3d2721a49c09a307 input=741c593b606cc6b3]*/ +{ + PyObject *num, *one, *temp; + long long value; + int overflow; + + /* Test that overflow is set properly for a large value. */ + /* num is a number larger than LLONG_MAX on a typical machine. */ + num = PyLong_FromString("FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_long_and_overflow", + "return value was not set to -1"); + if (overflow != 1) + return raiseTestError("test_long_long_and_overflow", + "overflow was not set to 1"); + + /* Same again, with num = LLONG_MAX + 1 */ + num = PyLong_FromLongLong(LLONG_MAX); + if (num == NULL) + return NULL; + one = PyLong_FromLong(1L); + if (one == NULL) { + Py_DECREF(num); + return NULL; + } + temp = PyNumber_Add(num, one); + Py_DECREF(one); + Py_DECREF(num); num = temp; + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_long_and_overflow", + "return value was not set to -1"); + if (overflow != 1) + return raiseTestError("test_long_long_and_overflow", + "overflow was not set to 1"); + + /* Test that overflow is set properly for a large negative value. */ + /* num is a number smaller than LLONG_MIN on a typical platform */ + num = PyLong_FromString("-FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_long_and_overflow", + "return value was not set to -1"); + if (overflow != -1) + return raiseTestError("test_long_long_and_overflow", + "overflow was not set to -1"); + + /* Same again, with num = LLONG_MIN - 1 */ + num = PyLong_FromLongLong(LLONG_MIN); + if (num == NULL) + return NULL; + one = PyLong_FromLong(1L); + if (one == NULL) { + Py_DECREF(num); + return NULL; + } + temp = PyNumber_Subtract(num, one); + Py_DECREF(one); + Py_DECREF(num); num = temp; + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_long_and_overflow", + "return value was not set to -1"); + if (overflow != -1) + return raiseTestError("test_long_long_and_overflow", + "overflow was not set to -1"); + + /* Test that overflow is cleared properly for small values. */ + num = PyLong_FromString("FF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != 0xFF) + return raiseTestError("test_long_long_and_overflow", + "expected return value 0xFF"); + if (overflow != 0) + return raiseTestError("test_long_long_and_overflow", + "overflow was not cleared"); + + num = PyLong_FromString("-FF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -0xFF) + return raiseTestError("test_long_long_and_overflow", + "expected return value 0xFF"); + if (overflow != 0) + return raiseTestError("test_long_long_and_overflow", + "overflow was set incorrectly"); + + num = PyLong_FromLongLong(LLONG_MAX); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != LLONG_MAX) + return raiseTestError("test_long_long_and_overflow", + "expected return value LLONG_MAX"); + if (overflow != 0) + return raiseTestError("test_long_long_and_overflow", + "overflow was not cleared"); + + num = PyLong_FromLongLong(LLONG_MIN); + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != LLONG_MIN) + return raiseTestError("test_long_long_and_overflow", + "expected return value LLONG_MIN"); + if (overflow != 0) + return raiseTestError("test_long_long_and_overflow", + "overflow was not cleared"); + + Py_RETURN_NONE; +} + +/*[clinic input] +_testlimitedcapi.test_long_as_size_t + +Test the PyLong_As{Size,Ssize}_t API. + +At present this just tests that non-integer arguments are handled correctly. +It should be extended to test overflow handling. +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_as_size_t_impl(PyObject *module) +/*[clinic end generated code: output=297a9f14a42f55af input=8923d8f2038c46f4]*/ +{ + size_t out_u; + Py_ssize_t out_s; + + Py_INCREF(Py_None); + + out_u = PyLong_AsSize_t(Py_None); + if (out_u != (size_t)-1 || !PyErr_Occurred()) + return raiseTestError("test_long_as_size_t", + "PyLong_AsSize_t(None) didn't complain"); + if (!PyErr_ExceptionMatches(PyExc_TypeError)) + return raiseTestError("test_long_as_size_t", + "PyLong_AsSize_t(None) raised " + "something other than TypeError"); + PyErr_Clear(); + + out_s = PyLong_AsSsize_t(Py_None); + if (out_s != (Py_ssize_t)-1 || !PyErr_Occurred()) + return raiseTestError("test_long_as_size_t", + "PyLong_AsSsize_t(None) didn't complain"); + if (!PyErr_ExceptionMatches(PyExc_TypeError)) + return raiseTestError("test_long_as_size_t", + "PyLong_AsSsize_t(None) raised " + "something other than TypeError"); + PyErr_Clear(); + + /* Py_INCREF(Py_None) omitted - we already have a reference to it. */ + return Py_None; +} + +/*[clinic input] +_testlimitedcapi.test_long_as_unsigned_long_long_mask +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_as_unsigned_long_long_mask_impl(PyObject *module) +/*[clinic end generated code: output=90be09ffeec8ecab input=17c660bd58becad5]*/ +{ + unsigned long long res = PyLong_AsUnsignedLongLongMask(NULL); + + if (res != (unsigned long long)-1 || !PyErr_Occurred()) { + return raiseTestError("test_long_as_unsigned_long_long_mask", + "PyLong_AsUnsignedLongLongMask(NULL) didn't " + "complain"); + } + if (!PyErr_ExceptionMatches(PyExc_SystemError)) { + return raiseTestError("test_long_as_unsigned_long_long_mask", + "PyLong_AsUnsignedLongLongMask(NULL) raised " + "something other than SystemError"); + } + PyErr_Clear(); + Py_RETURN_NONE; +} + +/*[clinic input] +_testlimitedcapi.test_long_as_double +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_as_double_impl(PyObject *module) +/*[clinic end generated code: output=0e688c2acf224f88 input=e7b5712385064a48]*/ +{ + double out; + + Py_INCREF(Py_None); + + out = PyLong_AsDouble(Py_None); + if (out != -1.0 || !PyErr_Occurred()) + return raiseTestError("test_long_as_double", + "PyLong_AsDouble(None) didn't complain"); + if (!PyErr_ExceptionMatches(PyExc_TypeError)) + return raiseTestError("test_long_as_double", + "PyLong_AsDouble(None) raised " + "something other than TypeError"); + PyErr_Clear(); + + /* Py_INCREF(Py_None) omitted - we already have a reference to it. */ + return Py_None; +} + +static PyObject * +pylong_check(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyLong_Check(obj)); +} + +static PyObject * +pylong_checkexact(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyLong_CheckExact(obj)); +} + +static PyObject * +pylong_fromdouble(PyObject *module, PyObject *arg) +{ + double value; + if (!PyArg_Parse(arg, "d", &value)) { + return NULL; + } + return PyLong_FromDouble(value); +} + +static PyObject * +pylong_fromstring(PyObject *module, PyObject *args) +{ + const char *str; + Py_ssize_t len; + int base; + char *end = UNINITIALIZED_PTR; + if (!PyArg_ParseTuple(args, "z#i", &str, &len, &base)) { + return NULL; + } + + PyObject *result = PyLong_FromString(str, &end, base); + if (result == NULL) { + // XXX 'end' is not always set. + return NULL; + } + return Py_BuildValue("Nn", result, (Py_ssize_t)(end - str)); +} + +static PyObject * +pylong_fromvoidptr(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + return PyLong_FromVoidPtr((void *)arg); +} + +/*[clinic input] +_testlimitedcapi.PyLong_AsInt + arg: object + / +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_PyLong_AsInt(PyObject *module, PyObject *arg) +/*[clinic end generated code: output=d91db4c1287f85fa input=32c66be86f3265a1]*/ +{ + NULLABLE(arg); + assert(!PyErr_Occurred()); + int value = PyLong_AsInt(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLong(value); +} + +static PyObject * +pylong_aslong(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + long value = PyLong_AsLong(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLong(value); +} + +static PyObject * +pylong_aslongandoverflow(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + int overflow = UNINITIALIZED_INT; + long value = PyLong_AsLongAndOverflow(arg, &overflow); + if (value == -1 && PyErr_Occurred()) { + assert(overflow == -1); + return NULL; + } + return Py_BuildValue("li", value, overflow); +} + +static PyObject * +pylong_asunsignedlong(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + unsigned long value = PyLong_AsUnsignedLong(arg); + if (value == (unsigned long)-1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromUnsignedLong(value); +} + +static PyObject * +pylong_asunsignedlongmask(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + unsigned long value = PyLong_AsUnsignedLongMask(arg); + if (value == (unsigned long)-1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromUnsignedLong(value); +} + +static PyObject * +pylong_aslonglong(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + long long value = PyLong_AsLongLong(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLongLong(value); +} + +static PyObject * +pylong_aslonglongandoverflow(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + int overflow = UNINITIALIZED_INT; + long long value = PyLong_AsLongLongAndOverflow(arg, &overflow); + if (value == -1 && PyErr_Occurred()) { + assert(overflow == -1); + return NULL; + } + return Py_BuildValue("Li", value, overflow); +} + +static PyObject * +pylong_asunsignedlonglong(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + unsigned long long value = PyLong_AsUnsignedLongLong(arg); + if (value == (unsigned long long)-1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromUnsignedLongLong(value); +} + +static PyObject * +pylong_asunsignedlonglongmask(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + unsigned long long value = PyLong_AsUnsignedLongLongMask(arg); + if (value == (unsigned long long)-1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromUnsignedLongLong(value); +} + +static PyObject * +pylong_as_ssize_t(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + Py_ssize_t value = PyLong_AsSsize_t(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromSsize_t(value); +} + +static PyObject * +pylong_as_size_t(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + size_t value = PyLong_AsSize_t(arg); + if (value == (size_t)-1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromSize_t(value); +} + +static PyObject * +pylong_asdouble(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + double value = PyLong_AsDouble(arg); + if (value == -1.0 && PyErr_Occurred()) { + return NULL; + } + return PyFloat_FromDouble(value); +} + +static PyObject * +pylong_asvoidptr(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + void *value = PyLong_AsVoidPtr(arg); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NONE; + } + return Py_NewRef((PyObject *)value); +} + +static PyObject * +pylong_aspid(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + pid_t value = PyLong_AsPid(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromPid(value); +} + + +static PyObject * +pylong_asint32(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + int32_t value; + if (PyLong_AsInt32(arg, &value) < 0) { + return NULL; + } + return PyLong_FromInt32(value); +} + +static PyObject * +pylong_asuint32(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + uint32_t value; + if (PyLong_AsUInt32(arg, &value) < 0) { + return NULL; + } + return PyLong_FromUInt32(value); +} + + +static PyObject * +pylong_asint64(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + int64_t value; + if (PyLong_AsInt64(arg, &value) < 0) { + return NULL; + } + return PyLong_FromInt64(value); +} + +static PyObject * +pylong_asuint64(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + uint64_t value; + if (PyLong_AsUInt64(arg, &value) < 0) { + return NULL; + } + return PyLong_FromUInt64(value); +} + + +static PyMethodDef test_methods[] = { + _TESTLIMITEDCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF + _TESTLIMITEDCAPI_TEST_LONG_API_METHODDEF + _TESTLIMITEDCAPI_TEST_LONG_AS_DOUBLE_METHODDEF + _TESTLIMITEDCAPI_TEST_LONG_AS_SIZE_T_METHODDEF + _TESTLIMITEDCAPI_TEST_LONG_AS_UNSIGNED_LONG_LONG_MASK_METHODDEF + _TESTLIMITEDCAPI_TEST_LONG_LONG_AND_OVERFLOW_METHODDEF + _TESTLIMITEDCAPI_TEST_LONGLONG_API_METHODDEF + {"pylong_check", pylong_check, METH_O}, + {"pylong_checkexact", pylong_checkexact, METH_O}, + {"pylong_fromdouble", pylong_fromdouble, METH_O}, + {"pylong_fromstring", pylong_fromstring, METH_VARARGS}, + {"pylong_fromvoidptr", pylong_fromvoidptr, METH_O}, + _TESTLIMITEDCAPI_PYLONG_ASINT_METHODDEF + {"pylong_aslong", pylong_aslong, METH_O}, + {"pylong_aslongandoverflow", pylong_aslongandoverflow, METH_O}, + {"pylong_asunsignedlong", pylong_asunsignedlong, METH_O}, + {"pylong_asunsignedlongmask", pylong_asunsignedlongmask, METH_O}, + {"pylong_aslonglong", pylong_aslonglong, METH_O}, + {"pylong_aslonglongandoverflow", pylong_aslonglongandoverflow, METH_O}, + {"pylong_asunsignedlonglong", pylong_asunsignedlonglong, METH_O}, + {"pylong_asunsignedlonglongmask", pylong_asunsignedlonglongmask, METH_O}, + {"pylong_as_ssize_t", pylong_as_ssize_t, METH_O}, + {"pylong_as_size_t", pylong_as_size_t, METH_O}, + {"pylong_asdouble", pylong_asdouble, METH_O}, + {"pylong_asvoidptr", pylong_asvoidptr, METH_O}, + {"pylong_aspid", pylong_aspid, METH_O}, + {"pylong_asint32", pylong_asint32, METH_O}, + {"pylong_asuint32", pylong_asuint32, METH_O}, + {"pylong_asint64", pylong_asint64, METH_O}, + {"pylong_asuint64", pylong_asuint64, METH_O}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Long(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testlimitedcapi/object.c b/Modules/_testlimitedcapi/object.c new file mode 100644 index 00000000000000..da6fe3e4efa34c --- /dev/null +++ b/Modules/_testlimitedcapi/object.c @@ -0,0 +1,80 @@ +// Need limited C API version 3.13 for Py_GetConstant() +#include "pyconfig.h" // Py_GIL_DISABLED +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +# define Py_LIMITED_API 0x030d0000 +#endif + +#include "parts.h" +#include "util.h" + + +/* Test Py_GetConstant() */ +static PyObject * +get_constant(PyObject *Py_UNUSED(module), PyObject *args) +{ + int constant_id; + if (!PyArg_ParseTuple(args, "i", &constant_id)) { + return NULL; + } + + PyObject *obj = Py_GetConstant(constant_id); + if (obj == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + return obj; +} + + +/* Test Py_GetConstantBorrowed() */ +static PyObject * +get_constant_borrowed(PyObject *Py_UNUSED(module), PyObject *args) +{ + int constant_id; + if (!PyArg_ParseTuple(args, "i", &constant_id)) { + return NULL; + } + + PyObject *obj = Py_GetConstantBorrowed(constant_id); + if (obj == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + return Py_NewRef(obj); +} + + +/* Test constants */ +static PyObject * +test_constants(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + // Test that implementation of constants in the limited C API: + // check that the C code compiles. + // + // Test also that constants and Py_GetConstant() return the same + // objects. + assert(Py_None == Py_GetConstant(Py_CONSTANT_NONE)); + assert(Py_False == Py_GetConstant(Py_CONSTANT_FALSE)); + assert(Py_True == Py_GetConstant(Py_CONSTANT_TRUE)); + assert(Py_Ellipsis == Py_GetConstant(Py_CONSTANT_ELLIPSIS)); + assert(Py_NotImplemented == Py_GetConstant(Py_CONSTANT_NOT_IMPLEMENTED)); + // Other constants are tested in test_capi.test_object + Py_RETURN_NONE; +} + +static PyMethodDef test_methods[] = { + {"get_constant", get_constant, METH_VARARGS}, + {"get_constant_borrowed", get_constant_borrowed, METH_VARARGS}, + {"test_constants", test_constants, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Object(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h new file mode 100644 index 00000000000000..9efcd8dcb71e5b --- /dev/null +++ b/Modules/_testlimitedcapi/parts.h @@ -0,0 +1,46 @@ +#ifndef Py_TESTLIMITEDCAPI_PARTS_H +#define Py_TESTLIMITEDCAPI_PARTS_H + +// Always enable assertions +#undef NDEBUG + +#include "pyconfig.h" // Py_GIL_DISABLED + +// Use the limited C API +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) + // need limited C API version 3.5 for PyModule_AddFunctions() +# define Py_LIMITED_API 0x03050000 +#endif + +// Make sure that the internal C API cannot be used. +#undef Py_BUILD_CORE_MODULE +#undef Py_BUILD_CORE_BUILTIN + +#include "Python.h" + +#ifdef Py_BUILD_CORE +# error "Py_BUILD_CORE macro must not be defined" +#endif + +int _PyTestLimitedCAPI_Init_Abstract(PyObject *module); +int _PyTestLimitedCAPI_Init_ByteArray(PyObject *module); +int _PyTestLimitedCAPI_Init_Bytes(PyObject *module); +int _PyTestLimitedCAPI_Init_Codec(PyObject *module); +int _PyTestLimitedCAPI_Init_Complex(PyObject *module); +int _PyTestLimitedCAPI_Init_Dict(PyObject *module); +int _PyTestLimitedCAPI_Init_Eval(PyObject *module); +int _PyTestLimitedCAPI_Init_Float(PyObject *module); +int _PyTestLimitedCAPI_Init_HeaptypeRelative(PyObject *module); +int _PyTestLimitedCAPI_Init_Import(PyObject *module); +int _PyTestLimitedCAPI_Init_Object(PyObject *module); +int _PyTestLimitedCAPI_Init_List(PyObject *module); +int _PyTestLimitedCAPI_Init_Long(PyObject *module); +int _PyTestLimitedCAPI_Init_PyOS(PyObject *module); +int _PyTestLimitedCAPI_Init_Set(PyObject *module); +int _PyTestLimitedCAPI_Init_Sys(PyObject *module); +int _PyTestLimitedCAPI_Init_Tuple(PyObject *module); +int _PyTestLimitedCAPI_Init_Unicode(PyObject *module); +int _PyTestLimitedCAPI_Init_VectorcallLimited(PyObject *module); +int _PyTestLimitedCAPI_Init_Version(PyObject *module); + +#endif // Py_TESTLIMITEDCAPI_PARTS_H diff --git a/Modules/_testcapi/pyos.c b/Modules/_testlimitedcapi/pyos.c similarity index 97% rename from Modules/_testcapi/pyos.c rename to Modules/_testlimitedcapi/pyos.c index 63140e914875db..0f61801eae322a 100644 --- a/Modules/_testcapi/pyos.c +++ b/Modules/_testlimitedcapi/pyos.c @@ -50,7 +50,7 @@ static PyMethodDef test_methods[] = { }; int -_PyTestCapi_Init_PyOS(PyObject *mod) +_PyTestLimitedCAPI_Init_PyOS(PyObject *mod) { if (PyModule_AddFunctions(mod, test_methods) < 0) { return -1; diff --git a/Modules/_testlimitedcapi/set.c b/Modules/_testlimitedcapi/set.c new file mode 100644 index 00000000000000..35da5fa5f008e1 --- /dev/null +++ b/Modules/_testlimitedcapi/set.c @@ -0,0 +1,189 @@ +#include "parts.h" +#include "util.h" + +static PyObject * +set_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PySet_Check(obj)); +} + +static PyObject * +set_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PySet_CheckExact(obj)); +} + +static PyObject * +frozenset_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyFrozenSet_Check(obj)); +} + +static PyObject * +frozenset_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyFrozenSet_CheckExact(obj)); +} + +static PyObject * +anyset_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyAnySet_Check(obj)); +} + +static PyObject * +anyset_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyAnySet_CheckExact(obj)); +} + +static PyObject * +set_new(PyObject *self, PyObject *args) +{ + PyObject *iterable = NULL; + if (!PyArg_ParseTuple(args, "|O", &iterable)) { + return NULL; + } + return PySet_New(iterable); +} + +static PyObject * +frozenset_new(PyObject *self, PyObject *args) +{ + PyObject *iterable = NULL; + if (!PyArg_ParseTuple(args, "|O", &iterable)) { + return NULL; + } + return PyFrozenSet_New(iterable); +} + +static PyObject * +set_size(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PySet_Size(obj)); +} + +static PyObject * +set_contains(PyObject *self, PyObject *args) +{ + PyObject *obj, *item; + if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(item); + RETURN_INT(PySet_Contains(obj, item)); +} + +static PyObject * +set_add(PyObject *self, PyObject *args) +{ + PyObject *obj, *item; + if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(item); + RETURN_INT(PySet_Add(obj, item)); +} + +static PyObject * +set_discard(PyObject *self, PyObject *args) +{ + PyObject *obj, *item; + if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(item); + RETURN_INT(PySet_Discard(obj, item)); +} + +static PyObject * +set_pop(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PySet_Pop(obj); +} + +static PyObject * +set_clear(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PySet_Clear(obj)); +} + +static PyObject * +test_frozenset_add_in_capi(PyObject *self, PyObject *Py_UNUSED(obj)) +{ + // Test that `frozenset` can be used with `PySet_Add`, + // when frozenset is just created in CAPI. + PyObject *fs = PyFrozenSet_New(NULL); + if (fs == NULL) { + return NULL; + } + PyObject *num = PyLong_FromLong(1); + if (num == NULL) { + goto error; + } + if (PySet_Add(fs, num) < 0) { + goto error; + } + int contains = PySet_Contains(fs, num); + if (contains < 0) { + goto error; + } + else if (contains == 0) { + goto unexpected; + } + Py_DECREF(fs); + Py_DECREF(num); + Py_RETURN_NONE; + +unexpected: + PyErr_SetString(PyExc_ValueError, "set does not contain expected value"); +error: + Py_DECREF(fs); + Py_XDECREF(num); + return NULL; +} + +static PyMethodDef test_methods[] = { + {"set_check", set_check, METH_O}, + {"set_checkexact", set_checkexact, METH_O}, + {"frozenset_check", frozenset_check, METH_O}, + {"frozenset_checkexact", frozenset_checkexact, METH_O}, + {"anyset_check", anyset_check, METH_O}, + {"anyset_checkexact", anyset_checkexact, METH_O}, + + {"set_new", set_new, METH_VARARGS}, + {"frozenset_new", frozenset_new, METH_VARARGS}, + + {"set_size", set_size, METH_O}, + {"set_contains", set_contains, METH_VARARGS}, + {"set_add", set_add, METH_VARARGS}, + {"set_discard", set_discard, METH_VARARGS}, + {"set_pop", set_pop, METH_O}, + {"set_clear", set_clear, METH_O}, + + {"test_frozenset_add_in_capi", test_frozenset_add_in_capi, METH_NOARGS}, + + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Set(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/sys.c b/Modules/_testlimitedcapi/sys.c similarity index 96% rename from Modules/_testcapi/sys.c rename to Modules/_testlimitedcapi/sys.c index aa40e3cd5b9b29..7d8b7a8569e515 100644 --- a/Modules/_testcapi/sys.c +++ b/Modules/_testlimitedcapi/sys.c @@ -46,7 +46,7 @@ static PyMethodDef test_methods[] = { }; int -_PyTestCapi_Init_Sys(PyObject *m) +_PyTestLimitedCAPI_Init_Sys(PyObject *m) { if (PyModule_AddFunctions(m, test_methods) < 0) { return -1; diff --git a/Modules/_testcapi/testcapi_long.h b/Modules/_testlimitedcapi/testcapi_long.h similarity index 100% rename from Modules/_testcapi/testcapi_long.h rename to Modules/_testlimitedcapi/testcapi_long.h diff --git a/Modules/_testlimitedcapi/tuple.c b/Modules/_testlimitedcapi/tuple.c new file mode 100644 index 00000000000000..231ec12d517046 --- /dev/null +++ b/Modules/_testlimitedcapi/tuple.c @@ -0,0 +1,136 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +tuple_check(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyTuple_Check(obj)); +} + +static PyObject * +tuple_checkexact(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyTuple_CheckExact(obj)); +} + +static PyObject * +tuple_new(PyObject* Py_UNUSED(module), PyObject *len) +{ + return PyTuple_New(PyLong_AsSsize_t(len)); +} + +static PyObject * +tuple_pack(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *arg1 = NULL, *arg2 = NULL; + Py_ssize_t size; + + if (!PyArg_ParseTuple(args, "n|OO", &size, &arg1, &arg2)) { + return NULL; + } + if (arg1) { + NULLABLE(arg1); + if (arg2) { + NULLABLE(arg2); + return PyTuple_Pack(size, arg1, arg2); + } + return PyTuple_Pack(size, arg1); + } + return PyTuple_Pack(size); +} + +static PyObject * +tuple_size(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyTuple_Size(obj)); +} + +static PyObject * +tuple_getitem(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &obj, &i)) { + return NULL; + } + NULLABLE(obj); + return Py_XNewRef(PyTuple_GetItem(obj, i)); +} + +static PyObject * +tuple_getslice(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t ilow, ihigh; + if (!PyArg_ParseTuple(args, "Onn", &obj, &ilow, &ihigh)) { + return NULL; + } + NULLABLE(obj); + return PyTuple_GetSlice(obj, ilow, ihigh); +} + +static PyObject * +tuple_setitem(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value, *newtuple = NULL; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "OnO", &obj, &i, &value)) { + return NULL; + } + NULLABLE(value); + if (PyTuple_CheckExact(obj)) { + Py_ssize_t size = PyTuple_Size(obj); + newtuple = PyTuple_New(size); + if (!newtuple) { + return NULL; + } + for (Py_ssize_t n = 0; n < size; n++) { + if (PyTuple_SetItem(newtuple, n, + Py_XNewRef(PyTuple_GetItem(obj, n))) == -1) { + Py_DECREF(newtuple); + return NULL; + } + } + + if (PyTuple_SetItem(newtuple, i, Py_XNewRef(value)) == -1) { + Py_DECREF(newtuple); + return NULL; + } + return newtuple; + } + else { + NULLABLE(obj); + + if (PyTuple_SetItem(obj, i, Py_XNewRef(value)) == -1) { + return NULL; + } + return Py_XNewRef(obj); + } +} + + +static PyMethodDef test_methods[] = { + {"tuple_check", tuple_check, METH_O}, + {"tuple_checkexact", tuple_checkexact, METH_O}, + {"tuple_new", tuple_new, METH_O}, + {"tuple_pack", tuple_pack, METH_VARARGS}, + {"tuple_size", tuple_size, METH_O}, + {"tuple_getitem", tuple_getitem, METH_VARARGS}, + {"tuple_getslice", tuple_getslice, METH_VARARGS}, + {"tuple_setitem", tuple_setitem, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Tuple(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/unicode.c b/Modules/_testlimitedcapi/unicode.c new file mode 100644 index 00000000000000..c7a23d5d1cbd71 --- /dev/null +++ b/Modules/_testlimitedcapi/unicode.c @@ -0,0 +1,1956 @@ +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED + // Need limited C API 3.14 to test PyUnicode_Equal() +# define Py_LIMITED_API 0x030e0000 +#endif + +#include "parts.h" +#include "util.h" + +#include // ptrdiff_t +#include // memset() + + +static PyObject * +codec_incrementalencoder(PyObject *self, PyObject *args) +{ + const char *encoding, *errors = NULL; + if (!PyArg_ParseTuple(args, "s|s:test_incrementalencoder", + &encoding, &errors)) + return NULL; + return PyCodec_IncrementalEncoder(encoding, errors); +} + +static PyObject * +codec_incrementaldecoder(PyObject *self, PyObject *args) +{ + const char *encoding, *errors = NULL; + if (!PyArg_ParseTuple(args, "s|s:test_incrementaldecoder", + &encoding, &errors)) + return NULL; + return PyCodec_IncrementalDecoder(encoding, errors); +} + +static PyObject * +test_unicode_compare_with_ascii(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *py_s = PyUnicode_FromStringAndSize("str\0", 4); + int result; + if (py_s == NULL) + return NULL; + result = PyUnicode_CompareWithASCIIString(py_s, "str"); + Py_DECREF(py_s); + if (!result) { + PyErr_SetString(PyExc_AssertionError, "Python string ending in NULL " + "should not compare equal to c string."); + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +test_widechar(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ +#if defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4) + const wchar_t wtext[2] = {(wchar_t)0x10ABCDu}; + size_t wtextlen = 1; + const wchar_t invalid[1] = {(wchar_t)0x110000u}; +#else + const wchar_t wtext[3] = {(wchar_t)0xDBEAu, (wchar_t)0xDFCDu}; + size_t wtextlen = 2; +#endif + PyObject *wide, *utf8; + + wide = PyUnicode_FromWideChar(wtext, wtextlen); + if (wide == NULL) + return NULL; + + utf8 = PyUnicode_FromString("\xf4\x8a\xaf\x8d"); + if (utf8 == NULL) { + Py_DECREF(wide); + return NULL; + } + + if (PyUnicode_GetLength(wide) != PyUnicode_GetLength(utf8)) { + Py_DECREF(wide); + Py_DECREF(utf8); + PyErr_SetString(PyExc_AssertionError, + "test_widechar: " + "wide string and utf8 string " + "have different length"); + return NULL; + } + if (PyUnicode_Compare(wide, utf8)) { + Py_DECREF(wide); + Py_DECREF(utf8); + if (PyErr_Occurred()) + return NULL; + PyErr_SetString(PyExc_AssertionError, + "test_widechar: " + "wide string and utf8 string " + "are different"); + return NULL; + } + + Py_DECREF(wide); + Py_DECREF(utf8); + +#if defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4) + wide = PyUnicode_FromWideChar(invalid, 1); + if (wide == NULL) + PyErr_Clear(); + else { + PyErr_SetString(PyExc_AssertionError, + "test_widechar: " + "PyUnicode_FromWideChar(L\"\\U00110000\", 1) didn't fail"); + return NULL; + } +#endif + Py_RETURN_NONE; +} + + +static PyObject * +unicode_copy(PyObject *unicode) +{ + if (!unicode) { + return NULL; + } + if (!PyUnicode_Check(unicode)) { + Py_INCREF(unicode); + return unicode; + } + + // Create a new string by encoding to UTF-8 and then decode from UTF-8 + PyObject *utf8 = PyUnicode_AsUTF8String(unicode); + if (!utf8) { + return NULL; + } + + PyObject *copy = PyUnicode_DecodeUTF8( + PyBytes_AsString(utf8), + PyBytes_Size(utf8), + NULL); + Py_DECREF(utf8); + + return copy; +} + +/* Test PyUnicode_WriteChar() */ +static PyObject * +unicode_writechar(PyObject *self, PyObject *args) +{ + PyObject *to, *to_copy; + Py_ssize_t index; + unsigned int character; + int result; + + if (!PyArg_ParseTuple(args, "OnI", &to, &index, &character)) { + return NULL; + } + + NULLABLE(to); + if (!(to_copy = unicode_copy(to)) && to) { + return NULL; + } + + result = PyUnicode_WriteChar(to_copy, index, (Py_UCS4)character); + if (result == -1 && PyErr_Occurred()) { + Py_DECREF(to_copy); + return NULL; + } + return Py_BuildValue("(Ni)", to_copy, result); +} + +static void +unicode_fill(PyObject *str, Py_ssize_t start, Py_ssize_t end, Py_UCS4 ch) +{ + assert(0 <= start); + assert(end <= PyUnicode_GetLength(str)); + for (Py_ssize_t i = start; i < end; i++) { + int res = PyUnicode_WriteChar(str, i, ch); + assert(res == 0); + } +} + + +/* Test PyUnicode_Resize() */ +static PyObject * +unicode_resize(PyObject *self, PyObject *args) +{ + PyObject *obj, *copy; + Py_ssize_t length; + int result; + + if (!PyArg_ParseTuple(args, "On", &obj, &length)) { + return NULL; + } + + NULLABLE(obj); + if (!(copy = unicode_copy(obj)) && obj) { + return NULL; + } + result = PyUnicode_Resize(©, length); + if (result == -1 && PyErr_Occurred()) { + Py_XDECREF(copy); + return NULL; + } + if (obj && PyUnicode_Check(obj) && length > PyUnicode_GetLength(obj)) { + unicode_fill(copy, PyUnicode_GetLength(obj), length, 0U); + } + return Py_BuildValue("(Ni)", copy, result); +} + +/* Test PyUnicode_Append() */ +static PyObject * +unicode_append(PyObject *self, PyObject *args) +{ + PyObject *left, *right, *left_copy; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + if (!(left_copy = unicode_copy(left)) && left) { + return NULL; + } + PyUnicode_Append(&left_copy, right); + return left_copy; +} + +/* Test PyUnicode_AppendAndDel() */ +static PyObject * +unicode_appendanddel(PyObject *self, PyObject *args) +{ + PyObject *left, *right, *left_copy; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + if (!(left_copy = unicode_copy(left)) && left) { + return NULL; + } + Py_XINCREF(right); + PyUnicode_AppendAndDel(&left_copy, right); + return left_copy; +} + +/* Test PyUnicode_FromStringAndSize() */ +static PyObject * +unicode_fromstringandsize(PyObject *self, PyObject *args) +{ + const char *s; + Py_ssize_t bsize; + Py_ssize_t size = -100; + + if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { + return NULL; + } + + if (size == -100) { + size = bsize; + } + return PyUnicode_FromStringAndSize(s, size); +} + +/* Test PyUnicode_FromString() */ +static PyObject * +unicode_fromstring(PyObject *self, PyObject *arg) +{ + const char *s; + Py_ssize_t size; + + if (!PyArg_Parse(arg, "z#", &s, &size)) { + return NULL; + } + return PyUnicode_FromString(s); +} + +/* Test PyUnicode_Substring() */ +static PyObject * +unicode_substring(PyObject *self, PyObject *args) +{ + PyObject *str; + Py_ssize_t start, end; + + if (!PyArg_ParseTuple(args, "Onn", &str, &start, &end)) { + return NULL; + } + + NULLABLE(str); + return PyUnicode_Substring(str, start, end); +} + +/* Test PyUnicode_GetLength() */ +static PyObject * +unicode_getlength(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + RETURN_SIZE(PyUnicode_GetLength(arg)); +} + +/* Test PyUnicode_ReadChar() */ +static PyObject * +unicode_readchar(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t index; + Py_UCS4 result; + + if (!PyArg_ParseTuple(args, "On", &unicode, &index)) { + return NULL; + } + + NULLABLE(unicode); + result = PyUnicode_ReadChar(unicode, index); + if (result == (Py_UCS4)-1) + return NULL; + return PyLong_FromUnsignedLong(result); +} + +/* Test PyUnicode_FromEncodedObject() */ +static PyObject * +unicode_fromencodedobject(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *encoding; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "Oz|z", &obj, &encoding, &errors)) { + return NULL; + } + + NULLABLE(obj); + return PyUnicode_FromEncodedObject(obj, encoding, errors); +} + +/* Test PyUnicode_FromObject() */ +static PyObject * +unicode_fromobject(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_FromObject(arg); +} + +/* Test PyUnicode_InternInPlace() */ +static PyObject * +unicode_interninplace(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + Py_XINCREF(arg); + PyUnicode_InternInPlace(&arg); + return arg; +} + +/* Test PyUnicode_InternFromString() */ +static PyObject * +unicode_internfromstring(PyObject *self, PyObject *arg) +{ + const char *s; + Py_ssize_t size; + + if (!PyArg_Parse(arg, "z#", &s, &size)) { + return NULL; + } + return PyUnicode_InternFromString(s); +} + +/* Test PyUnicode_FromWideChar() */ +static PyObject * +unicode_fromwidechar(PyObject *self, PyObject *args) +{ + const char *s; + Py_ssize_t bsize; + Py_ssize_t size = -100; + + if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { + return NULL; + } + if (size == -100) { + if (bsize % SIZEOF_WCHAR_T) { + PyErr_SetString(PyExc_AssertionError, + "invalid size in unicode_fromwidechar()"); + return NULL; + } + size = bsize / SIZEOF_WCHAR_T; + } + return PyUnicode_FromWideChar((const wchar_t *)s, size); +} + +/* Test PyUnicode_AsWideChar() */ +static PyObject * +unicode_aswidechar(PyObject *self, PyObject *args) +{ + PyObject *unicode, *result; + Py_ssize_t buflen, size; + wchar_t *buffer; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) + return NULL; + NULLABLE(unicode); + buffer = PyMem_New(wchar_t, buflen); + if (buffer == NULL) + return PyErr_NoMemory(); + + size = PyUnicode_AsWideChar(unicode, buffer, buflen); + if (size == -1) { + PyMem_Free(buffer); + return NULL; + } + + if (size < buflen) + buflen = size + 1; + else + buflen = size; + result = PyUnicode_FromWideChar(buffer, buflen); + PyMem_Free(buffer); + if (result == NULL) + return NULL; + + return Py_BuildValue("(Nn)", result, size); +} + +/* Test PyUnicode_AsWideCharString() with NULL as buffer */ +static PyObject * +unicode_aswidechar_null(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t buflen; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) + return NULL; + NULLABLE(unicode); + RETURN_SIZE(PyUnicode_AsWideChar(unicode, NULL, buflen)); +} + +/* Test PyUnicode_AsWideCharString() */ +static PyObject * +unicode_aswidecharstring(PyObject *self, PyObject *args) +{ + PyObject *unicode, *result; + Py_ssize_t size = UNINITIALIZED_SIZE; + wchar_t *buffer; + + if (!PyArg_ParseTuple(args, "O", &unicode)) + return NULL; + + NULLABLE(unicode); + buffer = PyUnicode_AsWideCharString(unicode, &size); + if (buffer == NULL) { + assert(size == UNINITIALIZED_SIZE); + return NULL; + } + + result = PyUnicode_FromWideChar(buffer, size + 1); + PyMem_Free(buffer); + if (result == NULL) + return NULL; + return Py_BuildValue("(Nn)", result, size); +} + +/* Test PyUnicode_AsWideCharString() with NULL as the size address */ +static PyObject * +unicode_aswidecharstring_null(PyObject *self, PyObject *args) +{ + PyObject *unicode, *result; + wchar_t *buffer; + + if (!PyArg_ParseTuple(args, "O", &unicode)) + return NULL; + + NULLABLE(unicode); + buffer = PyUnicode_AsWideCharString(unicode, NULL); + if (buffer == NULL) + return NULL; + + result = PyUnicode_FromWideChar(buffer, -1); + PyMem_Free(buffer); + if (result == NULL) + return NULL; + return result; +} + + +/* Test PyUnicode_FromOrdinal() */ +static PyObject * +unicode_fromordinal(PyObject *self, PyObject *args) +{ + int ordinal; + + if (!PyArg_ParseTuple(args, "i", &ordinal)) + return NULL; + + return PyUnicode_FromOrdinal(ordinal); +} + +/* Test PyUnicode_AsUTF8AndSize() */ +static PyObject * +unicode_asutf8andsize(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t buflen; + const char *s; + Py_ssize_t size = UNINITIALIZED_SIZE; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) + return NULL; + + NULLABLE(unicode); + s = PyUnicode_AsUTF8AndSize(unicode, &size); + if (s == NULL) { + assert(size == -1); + return NULL; + } + + return Py_BuildValue("(y#n)", s, buflen, size); +} + +/* Test PyUnicode_AsUTF8AndSize() with NULL as the size address */ +static PyObject * +unicode_asutf8andsize_null(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t buflen; + const char *s; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) + return NULL; + + NULLABLE(unicode); + s = PyUnicode_AsUTF8AndSize(unicode, NULL); + if (s == NULL) + return NULL; + + return PyBytes_FromStringAndSize(s, buflen); +} + +/* Test PyUnicode_GetDefaultEncoding() */ +static PyObject * +unicode_getdefaultencoding(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + const char *s = PyUnicode_GetDefaultEncoding(); + if (s == NULL) + return NULL; + + return PyBytes_FromString(s); +} + +/* Test PyUnicode_Decode() */ +static PyObject * +unicode_decode(PyObject *self, PyObject *args) +{ + const char *s; + Py_ssize_t size; + const char *encoding; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#z|z", &s, &size, &encoding, &errors)) + return NULL; + + return PyUnicode_Decode(s, size, encoding, errors); +} + +/* Test PyUnicode_AsEncodedString() */ +static PyObject * +unicode_asencodedstring(PyObject *self, PyObject *args) +{ + PyObject *unicode; + const char *encoding; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "Oz|z", &unicode, &encoding, &errors)) + return NULL; + + NULLABLE(unicode); + return PyUnicode_AsEncodedString(unicode, encoding, errors); +} + +/* Test PyUnicode_BuildEncodingMap() */ +static PyObject * +unicode_buildencodingmap(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_BuildEncodingMap(arg); +} + +/* Test PyUnicode_DecodeUTF7() */ +static PyObject * +unicode_decodeutf7(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeUTF7(data, size, errors); +} + +/* Test PyUnicode_DecodeUTF7Stateful() */ +static PyObject * +unicode_decodeutf7stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF7Stateful(data, size, errors, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + +/* Test PyUnicode_DecodeUTF8() */ +static PyObject * +unicode_decodeutf8(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeUTF8(data, size, errors); +} + +/* Test PyUnicode_DecodeUTF8Stateful() */ +static PyObject * +unicode_decodeutf8stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF8Stateful(data, size, errors, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + +/* Test PyUnicode_AsUTF8String() */ +static PyObject * +unicode_asutf8string(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsUTF8String(arg); +} + +/* Test PyUnicode_DecodeUTF32() */ +static PyObject * +unicode_decodeutf32(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + int byteorder = UNINITIALIZED_INT; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF32(data, size, errors, &byteorder); + if (!result) { + return NULL; + } + return Py_BuildValue("(iN)", byteorder, result); +} + +/* Test PyUnicode_DecodeUTF32Stateful() */ +static PyObject * +unicode_decodeutf32stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + int byteorder = UNINITIALIZED_INT; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF32Stateful(data, size, errors, &byteorder, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(iNn)", byteorder, result, consumed); +} + +/* Test PyUnicode_AsUTF32String() */ +static PyObject * +unicode_asutf32string(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsUTF32String(arg); +} + +/* Test PyUnicode_DecodeUTF16() */ +static PyObject * +unicode_decodeutf16(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + int byteorder = UNINITIALIZED_INT; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF16(data, size, errors, &byteorder); + if (!result) { + return NULL; + } + return Py_BuildValue("(iN)", byteorder, result); +} + +/* Test PyUnicode_DecodeUTF16Stateful() */ +static PyObject * +unicode_decodeutf16stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + int byteorder = UNINITIALIZED_INT; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF16Stateful(data, size, errors, &byteorder, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(iNn)", byteorder, result, consumed); +} + +/* Test PyUnicode_AsUTF16String() */ +static PyObject * +unicode_asutf16string(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsUTF16String(arg); +} + +/* Test PyUnicode_DecodeUnicodeEscape() */ +static PyObject * +unicode_decodeunicodeescape(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeUnicodeEscape(data, size, errors); +} + +/* Test PyUnicode_AsUnicodeEscapeString() */ +static PyObject * +unicode_asunicodeescapestring(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsUnicodeEscapeString(arg); +} + +static PyObject * +unicode_decoderawunicodeescape(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeRawUnicodeEscape(data, size, errors); +} + +/* Test PyUnicode_AsRawUnicodeEscapeString() */ +static PyObject * +unicode_asrawunicodeescapestring(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsRawUnicodeEscapeString(arg); +} + +static PyObject * +unicode_decodelatin1(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeLatin1(data, size, errors); +} + +/* Test PyUnicode_AsLatin1String() */ +static PyObject * +unicode_aslatin1string(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsLatin1String(arg); +} + +/* Test PyUnicode_DecodeASCII() */ +static PyObject * +unicode_decodeascii(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeASCII(data, size, errors); +} + +/* Test PyUnicode_AsASCIIString() */ +static PyObject * +unicode_asasciistring(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsASCIIString(arg); +} + +/* Test PyUnicode_DecodeCharmap() */ +static PyObject * +unicode_decodecharmap(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + PyObject *mapping; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#O|z", &data, &size, &mapping, &errors)) + return NULL; + + NULLABLE(mapping); + return PyUnicode_DecodeCharmap(data, size, mapping, errors); +} + +/* Test PyUnicode_AsCharmapString() */ +static PyObject * +unicode_ascharmapstring(PyObject *self, PyObject *args) +{ + PyObject *unicode; + PyObject *mapping; + + if (!PyArg_ParseTuple(args, "OO", &unicode, &mapping)) + return NULL; + + NULLABLE(unicode); + NULLABLE(mapping); + return PyUnicode_AsCharmapString(unicode, mapping); +} + +#ifdef MS_WINDOWS + +/* Test PyUnicode_DecodeMBCS() */ +static PyObject * +unicode_decodembcs(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeMBCS(data, size, errors); +} + +/* Test PyUnicode_DecodeMBCSStateful() */ +static PyObject * +unicode_decodembcsstateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeMBCSStateful(data, size, errors, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + +/* Test PyUnicode_DecodeCodePageStateful() */ +static PyObject * +unicode_decodecodepagestateful(PyObject *self, PyObject *args) +{ + int code_page; + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &code_page, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeCodePageStateful(code_page, data, size, errors, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + +/* Test PyUnicode_AsMBCSString() */ +static PyObject * +unicode_asmbcsstring(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsMBCSString(arg); +} + +/* Test PyUnicode_EncodeCodePage() */ +static PyObject * +unicode_encodecodepage(PyObject *self, PyObject *args) +{ + int code_page; + PyObject *unicode; + const char *errors; + + if (!PyArg_ParseTuple(args, "iO|z", &code_page, &unicode, &errors)) + return NULL; + + NULLABLE(unicode); + return PyUnicode_EncodeCodePage(code_page, unicode, errors); +} + +#endif /* MS_WINDOWS */ + +/* Test PyUnicode_DecodeLocaleAndSize() */ +static PyObject * +unicode_decodelocaleandsize(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeLocaleAndSize(data, size, errors); +} + +/* Test PyUnicode_DecodeLocale() */ +static PyObject * +unicode_decodelocale(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeLocale(data, errors); +} + +/* Test PyUnicode_EncodeLocale() */ +static PyObject * +unicode_encodelocale(PyObject *self, PyObject *args) +{ + PyObject *unicode; + const char *errors; + + if (!PyArg_ParseTuple(args, "O|z", &unicode, &errors)) + return NULL; + + NULLABLE(unicode); + return PyUnicode_EncodeLocale(unicode, errors); +} + +/* Test PyUnicode_DecodeFSDefault() */ +static PyObject * +unicode_decodefsdefault(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + + if (!PyArg_ParseTuple(args, "y#", &data, &size)) + return NULL; + + return PyUnicode_DecodeFSDefault(data); +} + +/* Test PyUnicode_DecodeFSDefaultAndSize() */ +static PyObject * +unicode_decodefsdefaultandsize(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + + if (!PyArg_ParseTuple(args, "y#|n", &data, &size, &size)) + return NULL; + + return PyUnicode_DecodeFSDefaultAndSize(data, size); +} + +/* Test PyUnicode_EncodeFSDefault() */ +static PyObject * +unicode_encodefsdefault(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_EncodeFSDefault(arg); +} + +/* Test PyUnicode_Concat() */ +static PyObject * +unicode_concat(PyObject *self, PyObject *args) +{ + PyObject *left; + PyObject *right; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + return PyUnicode_Concat(left, right); +} + +/* Test PyUnicode_Split() */ +static PyObject * +unicode_split(PyObject *self, PyObject *args) +{ + PyObject *s; + PyObject *sep; + Py_ssize_t maxsplit = -1; + + if (!PyArg_ParseTuple(args, "OO|n", &s, &sep, &maxsplit)) + return NULL; + + NULLABLE(s); + NULLABLE(sep); + return PyUnicode_Split(s, sep, maxsplit); +} + +/* Test PyUnicode_RSplit() */ +static PyObject * +unicode_rsplit(PyObject *self, PyObject *args) +{ + PyObject *s; + PyObject *sep; + Py_ssize_t maxsplit = -1; + + if (!PyArg_ParseTuple(args, "OO|n", &s, &sep, &maxsplit)) + return NULL; + + NULLABLE(s); + NULLABLE(sep); + return PyUnicode_RSplit(s, sep, maxsplit); +} + +/* Test PyUnicode_Splitlines() */ +static PyObject * +unicode_splitlines(PyObject *self, PyObject *args) +{ + PyObject *s; + int keepends = 0; + + if (!PyArg_ParseTuple(args, "O|i", &s, &keepends)) + return NULL; + + NULLABLE(s); + return PyUnicode_Splitlines(s, keepends); +} + +/* Test PyUnicode_Partition() */ +static PyObject * +unicode_partition(PyObject *self, PyObject *args) +{ + PyObject *s; + PyObject *sep; + + if (!PyArg_ParseTuple(args, "OO", &s, &sep)) + return NULL; + + NULLABLE(s); + NULLABLE(sep); + return PyUnicode_Partition(s, sep); +} + +/* Test PyUnicode_RPartition() */ +static PyObject * +unicode_rpartition(PyObject *self, PyObject *args) +{ + PyObject *s; + PyObject *sep; + + if (!PyArg_ParseTuple(args, "OO", &s, &sep)) + return NULL; + + NULLABLE(s); + NULLABLE(sep); + return PyUnicode_RPartition(s, sep); +} + +/* Test PyUnicode_Translate() */ +static PyObject * +unicode_translate(PyObject *self, PyObject *args) +{ + PyObject *obj; + PyObject *table; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "OO|z", &obj, &table, &errors)) + return NULL; + + NULLABLE(obj); + NULLABLE(table); + return PyUnicode_Translate(obj, table, errors); +} + +/* Test PyUnicode_Join() */ +static PyObject * +unicode_join(PyObject *self, PyObject *args) +{ + PyObject *sep; + PyObject *seq; + + if (!PyArg_ParseTuple(args, "OO", &sep, &seq)) + return NULL; + + NULLABLE(sep); + NULLABLE(seq); + return PyUnicode_Join(sep, seq); +} + +/* Test PyUnicode_Count() */ +static PyObject * +unicode_count(PyObject *self, PyObject *args) +{ + PyObject *str; + PyObject *substr; + Py_ssize_t start; + Py_ssize_t end; + + if (!PyArg_ParseTuple(args, "OOnn", &str, &substr, &start, &end)) + return NULL; + + NULLABLE(str); + NULLABLE(substr); + RETURN_SIZE(PyUnicode_Count(str, substr, start, end)); +} + +/* Test PyUnicode_Find() */ +static PyObject * +unicode_find(PyObject *self, PyObject *args) +{ + PyObject *str; + PyObject *substr; + Py_ssize_t start; + Py_ssize_t end; + int direction; + Py_ssize_t result; + + if (!PyArg_ParseTuple(args, "OOnni", &str, &substr, &start, &end, &direction)) + return NULL; + + NULLABLE(str); + NULLABLE(substr); + result = PyUnicode_Find(str, substr, start, end, direction); + if (result == -2) { + assert(PyErr_Occurred()); + return NULL; + } + assert(!PyErr_Occurred()); + return PyLong_FromSsize_t(result); +} + +/* Test PyUnicode_Tailmatch() */ +static PyObject * +unicode_tailmatch(PyObject *self, PyObject *args) +{ + PyObject *str; + PyObject *substr; + Py_ssize_t start; + Py_ssize_t end; + int direction; + + if (!PyArg_ParseTuple(args, "OOnni", &str, &substr, &start, &end, &direction)) + return NULL; + + NULLABLE(str); + NULLABLE(substr); + RETURN_SIZE(PyUnicode_Tailmatch(str, substr, start, end, direction)); +} + +/* Test PyUnicode_FindChar() */ +static PyObject * +unicode_findchar(PyObject *self, PyObject *args) +{ + PyObject *str; + int direction; + unsigned int ch; + Py_ssize_t result; + Py_ssize_t start, end; + + if (!PyArg_ParseTuple(args, "OInni:unicode_findchar", &str, &ch, + &start, &end, &direction)) { + return NULL; + } + NULLABLE(str); + result = PyUnicode_FindChar(str, (Py_UCS4)ch, start, end, direction); + if (result == -2) { + assert(PyErr_Occurred()); + return NULL; + } + assert(!PyErr_Occurred()); + return PyLong_FromSsize_t(result); +} + +/* Test PyUnicode_Replace() */ +static PyObject * +unicode_replace(PyObject *self, PyObject *args) +{ + PyObject *str; + PyObject *substr; + PyObject *replstr; + Py_ssize_t maxcount = -1; + + if (!PyArg_ParseTuple(args, "OOO|n", &str, &substr, &replstr, &maxcount)) + return NULL; + + NULLABLE(str); + NULLABLE(substr); + NULLABLE(replstr); + return PyUnicode_Replace(str, substr, replstr, maxcount); +} + +/* Test PyUnicode_Compare() */ +static PyObject * +unicode_compare(PyObject *self, PyObject *args) +{ + PyObject *left; + PyObject *right; + int result; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + result = PyUnicode_Compare(left, right); + if (result == -1 && PyErr_Occurred()) { + return NULL; + } + assert(!PyErr_Occurred()); + return PyLong_FromLong(result); +} + +/* Test PyUnicode_CompareWithASCIIString() */ +static PyObject * +unicode_comparewithasciistring(PyObject *self, PyObject *args) +{ + PyObject *left; + const char *right = NULL; + Py_ssize_t right_len; + int result; + + if (!PyArg_ParseTuple(args, "O|y#", &left, &right, &right_len)) + return NULL; + + NULLABLE(left); + result = PyUnicode_CompareWithASCIIString(left, right); + if (result == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLong(result); +} + +/* Test PyUnicode_EqualToUTF8() */ +static PyObject * +unicode_equaltoutf8(PyObject *self, PyObject *args) +{ + PyObject *left; + const char *right = NULL; + Py_ssize_t right_len; + int result; + + if (!PyArg_ParseTuple(args, "Oz#", &left, &right, &right_len)) { + return NULL; + } + + NULLABLE(left); + result = PyUnicode_EqualToUTF8(left, right); + assert(!PyErr_Occurred()); + return PyLong_FromLong(result); +} + +/* Test PyUnicode_EqualToUTF8AndSize() */ +static PyObject * +unicode_equaltoutf8andsize(PyObject *self, PyObject *args) +{ + PyObject *left; + const char *right = NULL; + Py_ssize_t right_len; + Py_ssize_t size = -100; + int result; + + if (!PyArg_ParseTuple(args, "Oz#|n", &left, &right, &right_len, &size)) { + return NULL; + } + + NULLABLE(left); + if (size == -100) { + size = right_len; + } + result = PyUnicode_EqualToUTF8AndSize(left, right, size); + assert(!PyErr_Occurred()); + return PyLong_FromLong(result); +} + +/* Test PyUnicode_RichCompare() */ +static PyObject * +unicode_richcompare(PyObject *self, PyObject *args) +{ + PyObject *left; + PyObject *right; + int op; + + if (!PyArg_ParseTuple(args, "OOi", &left, &right, &op)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + return PyUnicode_RichCompare(left, right, op); +} + +/* Test PyUnicode_Format() */ +static PyObject * +unicode_format(PyObject *self, PyObject *args) +{ + PyObject *format; + PyObject *fargs; + + if (!PyArg_ParseTuple(args, "OO", &format, &fargs)) + return NULL; + + NULLABLE(format); + NULLABLE(fargs); + return PyUnicode_Format(format, fargs); +} + +/* Test PyUnicode_Contains() */ +static PyObject * +unicode_contains(PyObject *self, PyObject *args) +{ + PyObject *container; + PyObject *element; + + if (!PyArg_ParseTuple(args, "OO", &container, &element)) + return NULL; + + NULLABLE(container); + NULLABLE(element); + RETURN_INT(PyUnicode_Contains(container, element)); +} + +/* Test PyUnicode_IsIdentifier() */ +static PyObject * +unicode_isidentifier(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + RETURN_INT(PyUnicode_IsIdentifier(arg)); +} + + +static int +check_raised_systemerror(PyObject *result, char* msg) +{ + if (result) { + // no exception + PyErr_Format(PyExc_AssertionError, + "SystemError not raised: %s", + msg); + return 0; + } + if (PyErr_ExceptionMatches(PyExc_SystemError)) { + // expected exception + PyErr_Clear(); + return 1; + } + // unexpected exception + return 0; +} + +static PyObject * +test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *result; + PyObject *unicode = PyUnicode_FromString("None"); + +#define CHECK_FORMAT_2(FORMAT, EXPECTED, ARG1, ARG2) \ + result = PyUnicode_FromFormat(FORMAT, ARG1, ARG2); \ + if (EXPECTED == NULL) { \ + if (!check_raised_systemerror(result, FORMAT)) { \ + goto Fail; \ + } \ + } \ + else if (result == NULL) \ + return NULL; \ + else if (PyUnicode_CompareWithASCIIString(result, EXPECTED) != 0) { \ + PyObject *utf8 = PyUnicode_AsUTF8String(result); \ + PyErr_Format(PyExc_AssertionError, \ + "test_string_from_format: failed at \"%s\" " \ + "expected \"%s\" got \"%s\"", \ + FORMAT, EXPECTED, utf8); \ + Py_XDECREF(utf8); \ + goto Fail; \ + } \ + Py_XDECREF(result) + +#define CHECK_FORMAT_1(FORMAT, EXPECTED, ARG) \ + CHECK_FORMAT_2(FORMAT, EXPECTED, ARG, 0) + +#define CHECK_FORMAT_0(FORMAT, EXPECTED) \ + CHECK_FORMAT_2(FORMAT, EXPECTED, 0, 0) + + // Unrecognized + CHECK_FORMAT_2("%u %? %u", NULL, 1, 2); + + // "%%" (options are rejected) + CHECK_FORMAT_0( "%%", "%"); + CHECK_FORMAT_0( "%0%", NULL); + CHECK_FORMAT_0("%00%", NULL); + CHECK_FORMAT_0( "%2%", NULL); + CHECK_FORMAT_0("%02%", NULL); + CHECK_FORMAT_0("%.0%", NULL); + CHECK_FORMAT_0("%.2%", NULL); + + // "%c" + CHECK_FORMAT_1( "%c", "c", 'c'); + CHECK_FORMAT_1( "%0c", "c", 'c'); + CHECK_FORMAT_1("%00c", "c", 'c'); + CHECK_FORMAT_1( "%2c", NULL, 'c'); + CHECK_FORMAT_1("%02c", NULL, 'c'); + CHECK_FORMAT_1("%.0c", NULL, 'c'); + CHECK_FORMAT_1("%.2c", NULL, 'c'); + + // Integers + CHECK_FORMAT_1("%d", "123", (int)123); + CHECK_FORMAT_1("%i", "123", (int)123); + CHECK_FORMAT_1("%u", "123", (unsigned int)123); + CHECK_FORMAT_1("%x", "7b", (unsigned int)123); + CHECK_FORMAT_1("%X", "7B", (unsigned int)123); + CHECK_FORMAT_1("%o", "173", (unsigned int)123); + CHECK_FORMAT_1("%ld", "123", (long)123); + CHECK_FORMAT_1("%li", "123", (long)123); + CHECK_FORMAT_1("%lu", "123", (unsigned long)123); + CHECK_FORMAT_1("%lx", "7b", (unsigned long)123); + CHECK_FORMAT_1("%lX", "7B", (unsigned long)123); + CHECK_FORMAT_1("%lo", "173", (unsigned long)123); + CHECK_FORMAT_1("%lld", "123", (long long)123); + CHECK_FORMAT_1("%lli", "123", (long long)123); + CHECK_FORMAT_1("%llu", "123", (unsigned long long)123); + CHECK_FORMAT_1("%llx", "7b", (unsigned long long)123); + CHECK_FORMAT_1("%llX", "7B", (unsigned long long)123); + CHECK_FORMAT_1("%llo", "173", (unsigned long long)123); + CHECK_FORMAT_1("%zd", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%zi", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%zu", "123", (size_t)123); + CHECK_FORMAT_1("%zx", "7b", (size_t)123); + CHECK_FORMAT_1("%zX", "7B", (size_t)123); + CHECK_FORMAT_1("%zo", "173", (size_t)123); + CHECK_FORMAT_1("%td", "123", (ptrdiff_t)123); + CHECK_FORMAT_1("%ti", "123", (ptrdiff_t)123); + CHECK_FORMAT_1("%tu", "123", (ptrdiff_t)123); + CHECK_FORMAT_1("%tx", "7b", (ptrdiff_t)123); + CHECK_FORMAT_1("%tX", "7B", (ptrdiff_t)123); + CHECK_FORMAT_1("%to", "173", (ptrdiff_t)123); + CHECK_FORMAT_1("%jd", "123", (intmax_t)123); + CHECK_FORMAT_1("%ji", "123", (intmax_t)123); + CHECK_FORMAT_1("%ju", "123", (uintmax_t)123); + CHECK_FORMAT_1("%jx", "7b", (uintmax_t)123); + CHECK_FORMAT_1("%jX", "7B", (uintmax_t)123); + CHECK_FORMAT_1("%jo", "173", (uintmax_t)123); + + CHECK_FORMAT_1("%d", "-123", (int)-123); + CHECK_FORMAT_1("%i", "-123", (int)-123); + CHECK_FORMAT_1("%ld", "-123", (long)-123); + CHECK_FORMAT_1("%li", "-123", (long)-123); + CHECK_FORMAT_1("%lld", "-123", (long long)-123); + CHECK_FORMAT_1("%lli", "-123", (long long)-123); + CHECK_FORMAT_1("%zd", "-123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%zi", "-123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%td", "-123", (ptrdiff_t)-123); + CHECK_FORMAT_1("%ti", "-123", (ptrdiff_t)-123); + CHECK_FORMAT_1("%jd", "-123", (intmax_t)-123); + CHECK_FORMAT_1("%ji", "-123", (intmax_t)-123); + + // Integers: width < length + CHECK_FORMAT_1("%1d", "123", (int)123); + CHECK_FORMAT_1("%1i", "123", (int)123); + CHECK_FORMAT_1("%1u", "123", (unsigned int)123); + CHECK_FORMAT_1("%1ld", "123", (long)123); + CHECK_FORMAT_1("%1li", "123", (long)123); + CHECK_FORMAT_1("%1lu", "123", (unsigned long)123); + CHECK_FORMAT_1("%1lld", "123", (long long)123); + CHECK_FORMAT_1("%1lli", "123", (long long)123); + CHECK_FORMAT_1("%1llu", "123", (unsigned long long)123); + CHECK_FORMAT_1("%1zd", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%1zi", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%1zu", "123", (size_t)123); + CHECK_FORMAT_1("%1x", "7b", (int)123); + + CHECK_FORMAT_1("%1d", "-123", (int)-123); + CHECK_FORMAT_1("%1i", "-123", (int)-123); + CHECK_FORMAT_1("%1ld", "-123", (long)-123); + CHECK_FORMAT_1("%1li", "-123", (long)-123); + CHECK_FORMAT_1("%1lld", "-123", (long long)-123); + CHECK_FORMAT_1("%1lli", "-123", (long long)-123); + CHECK_FORMAT_1("%1zd", "-123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%1zi", "-123", (Py_ssize_t)-123); + + // Integers: width > length + CHECK_FORMAT_1("%5d", " 123", (int)123); + CHECK_FORMAT_1("%5i", " 123", (int)123); + CHECK_FORMAT_1("%5u", " 123", (unsigned int)123); + CHECK_FORMAT_1("%5ld", " 123", (long)123); + CHECK_FORMAT_1("%5li", " 123", (long)123); + CHECK_FORMAT_1("%5lu", " 123", (unsigned long)123); + CHECK_FORMAT_1("%5lld", " 123", (long long)123); + CHECK_FORMAT_1("%5lli", " 123", (long long)123); + CHECK_FORMAT_1("%5llu", " 123", (unsigned long long)123); + CHECK_FORMAT_1("%5zd", " 123", (Py_ssize_t)123); + CHECK_FORMAT_1("%5zi", " 123", (Py_ssize_t)123); + CHECK_FORMAT_1("%5zu", " 123", (size_t)123); + CHECK_FORMAT_1("%5x", " 7b", (int)123); + + CHECK_FORMAT_1("%5d", " -123", (int)-123); + CHECK_FORMAT_1("%5i", " -123", (int)-123); + CHECK_FORMAT_1("%5ld", " -123", (long)-123); + CHECK_FORMAT_1("%5li", " -123", (long)-123); + CHECK_FORMAT_1("%5lld", " -123", (long long)-123); + CHECK_FORMAT_1("%5lli", " -123", (long long)-123); + CHECK_FORMAT_1("%5zd", " -123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%5zi", " -123", (Py_ssize_t)-123); + + // Integers: width > length, 0-flag + CHECK_FORMAT_1("%05d", "00123", (int)123); + CHECK_FORMAT_1("%05i", "00123", (int)123); + CHECK_FORMAT_1("%05u", "00123", (unsigned int)123); + CHECK_FORMAT_1("%05ld", "00123", (long)123); + CHECK_FORMAT_1("%05li", "00123", (long)123); + CHECK_FORMAT_1("%05lu", "00123", (unsigned long)123); + CHECK_FORMAT_1("%05lld", "00123", (long long)123); + CHECK_FORMAT_1("%05lli", "00123", (long long)123); + CHECK_FORMAT_1("%05llu", "00123", (unsigned long long)123); + CHECK_FORMAT_1("%05zd", "00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%05zi", "00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%05zu", "00123", (size_t)123); + CHECK_FORMAT_1("%05x", "0007b", (int)123); + + CHECK_FORMAT_1("%05d", "-0123", (int)-123); + CHECK_FORMAT_1("%05i", "-0123", (int)-123); + CHECK_FORMAT_1("%05ld", "-0123", (long)-123); + CHECK_FORMAT_1("%05li", "-0123", (long)-123); + CHECK_FORMAT_1("%05lld", "-0123", (long long)-123); + CHECK_FORMAT_1("%05lli", "-0123", (long long)-123); + CHECK_FORMAT_1("%05zd", "-0123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%05zi", "-0123", (Py_ssize_t)-123); + + // Integers: precision < length + CHECK_FORMAT_1("%.1d", "123", (int)123); + CHECK_FORMAT_1("%.1i", "123", (int)123); + CHECK_FORMAT_1("%.1u", "123", (unsigned int)123); + CHECK_FORMAT_1("%.1ld", "123", (long)123); + CHECK_FORMAT_1("%.1li", "123", (long)123); + CHECK_FORMAT_1("%.1lu", "123", (unsigned long)123); + CHECK_FORMAT_1("%.1lld", "123", (long long)123); + CHECK_FORMAT_1("%.1lli", "123", (long long)123); + CHECK_FORMAT_1("%.1llu", "123", (unsigned long long)123); + CHECK_FORMAT_1("%.1zd", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%.1zi", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%.1zu", "123", (size_t)123); + CHECK_FORMAT_1("%.1x", "7b", (int)123); + + CHECK_FORMAT_1("%.1d", "-123", (int)-123); + CHECK_FORMAT_1("%.1i", "-123", (int)-123); + CHECK_FORMAT_1("%.1ld", "-123", (long)-123); + CHECK_FORMAT_1("%.1li", "-123", (long)-123); + CHECK_FORMAT_1("%.1lld", "-123", (long long)-123); + CHECK_FORMAT_1("%.1lli", "-123", (long long)-123); + CHECK_FORMAT_1("%.1zd", "-123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%.1zi", "-123", (Py_ssize_t)-123); + + // Integers: precision > length + CHECK_FORMAT_1("%.5d", "00123", (int)123); + CHECK_FORMAT_1("%.5i", "00123", (int)123); + CHECK_FORMAT_1("%.5u", "00123", (unsigned int)123); + CHECK_FORMAT_1("%.5ld", "00123", (long)123); + CHECK_FORMAT_1("%.5li", "00123", (long)123); + CHECK_FORMAT_1("%.5lu", "00123", (unsigned long)123); + CHECK_FORMAT_1("%.5lld", "00123", (long long)123); + CHECK_FORMAT_1("%.5lli", "00123", (long long)123); + CHECK_FORMAT_1("%.5llu", "00123", (unsigned long long)123); + CHECK_FORMAT_1("%.5zd", "00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%.5zi", "00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%.5zu", "00123", (size_t)123); + CHECK_FORMAT_1("%.5x", "0007b", (int)123); + + CHECK_FORMAT_1("%.5d", "-00123", (int)-123); + CHECK_FORMAT_1("%.5i", "-00123", (int)-123); + CHECK_FORMAT_1("%.5ld", "-00123", (long)-123); + CHECK_FORMAT_1("%.5li", "-00123", (long)-123); + CHECK_FORMAT_1("%.5lld", "-00123", (long long)-123); + CHECK_FORMAT_1("%.5lli", "-00123", (long long)-123); + CHECK_FORMAT_1("%.5zd", "-00123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%.5zi", "-00123", (Py_ssize_t)-123); + + // Integers: width > precision > length + CHECK_FORMAT_1("%7.5d", " 00123", (int)123); + CHECK_FORMAT_1("%7.5i", " 00123", (int)123); + CHECK_FORMAT_1("%7.5u", " 00123", (unsigned int)123); + CHECK_FORMAT_1("%7.5ld", " 00123", (long)123); + CHECK_FORMAT_1("%7.5li", " 00123", (long)123); + CHECK_FORMAT_1("%7.5lu", " 00123", (unsigned long)123); + CHECK_FORMAT_1("%7.5lld", " 00123", (long long)123); + CHECK_FORMAT_1("%7.5lli", " 00123", (long long)123); + CHECK_FORMAT_1("%7.5llu", " 00123", (unsigned long long)123); + CHECK_FORMAT_1("%7.5zd", " 00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%7.5zi", " 00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%7.5zu", " 00123", (size_t)123); + CHECK_FORMAT_1("%7.5x", " 0007b", (int)123); + + CHECK_FORMAT_1("%7.5d", " -00123", (int)-123); + CHECK_FORMAT_1("%7.5i", " -00123", (int)-123); + CHECK_FORMAT_1("%7.5ld", " -00123", (long)-123); + CHECK_FORMAT_1("%7.5li", " -00123", (long)-123); + CHECK_FORMAT_1("%7.5lld", " -00123", (long long)-123); + CHECK_FORMAT_1("%7.5lli", " -00123", (long long)-123); + CHECK_FORMAT_1("%7.5zd", " -00123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%7.5zi", " -00123", (Py_ssize_t)-123); + + // Integers: width > precision > length, 0-flag + CHECK_FORMAT_1("%07.5d", "0000123", (int)123); + CHECK_FORMAT_1("%07.5i", "0000123", (int)123); + CHECK_FORMAT_1("%07.5u", "0000123", (unsigned int)123); + CHECK_FORMAT_1("%07.5ld", "0000123", (long)123); + CHECK_FORMAT_1("%07.5li", "0000123", (long)123); + CHECK_FORMAT_1("%07.5lu", "0000123", (unsigned long)123); + CHECK_FORMAT_1("%07.5lld", "0000123", (long long)123); + CHECK_FORMAT_1("%07.5lli", "0000123", (long long)123); + CHECK_FORMAT_1("%07.5llu", "0000123", (unsigned long long)123); + CHECK_FORMAT_1("%07.5zd", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%07.5zi", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%07.5zu", "0000123", (size_t)123); + CHECK_FORMAT_1("%07.5x", "000007b", (int)123); + + CHECK_FORMAT_1("%07.5d", "-000123", (int)-123); + CHECK_FORMAT_1("%07.5i", "-000123", (int)-123); + CHECK_FORMAT_1("%07.5ld", "-000123", (long)-123); + CHECK_FORMAT_1("%07.5li", "-000123", (long)-123); + CHECK_FORMAT_1("%07.5lld", "-000123", (long long)-123); + CHECK_FORMAT_1("%07.5lli", "-000123", (long long)-123); + CHECK_FORMAT_1("%07.5zd", "-000123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%07.5zi", "-000123", (Py_ssize_t)-123); + + // Integers: precision > width > length + CHECK_FORMAT_1("%5.7d", "0000123", (int)123); + CHECK_FORMAT_1("%5.7i", "0000123", (int)123); + CHECK_FORMAT_1("%5.7u", "0000123", (unsigned int)123); + CHECK_FORMAT_1("%5.7ld", "0000123", (long)123); + CHECK_FORMAT_1("%5.7li", "0000123", (long)123); + CHECK_FORMAT_1("%5.7lu", "0000123", (unsigned long)123); + CHECK_FORMAT_1("%5.7lld", "0000123", (long long)123); + CHECK_FORMAT_1("%5.7lli", "0000123", (long long)123); + CHECK_FORMAT_1("%5.7llu", "0000123", (unsigned long long)123); + CHECK_FORMAT_1("%5.7zd", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%5.7zi", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%5.7zu", "0000123", (size_t)123); + CHECK_FORMAT_1("%5.7x", "000007b", (int)123); + + CHECK_FORMAT_1("%5.7d", "-0000123", (int)-123); + CHECK_FORMAT_1("%5.7i", "-0000123", (int)-123); + CHECK_FORMAT_1("%5.7ld", "-0000123", (long)-123); + CHECK_FORMAT_1("%5.7li", "-0000123", (long)-123); + CHECK_FORMAT_1("%5.7lld", "-0000123", (long long)-123); + CHECK_FORMAT_1("%5.7lli", "-0000123", (long long)-123); + CHECK_FORMAT_1("%5.7zd", "-0000123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%5.7zi", "-0000123", (Py_ssize_t)-123); + + // Integers: precision > width > length, 0-flag + CHECK_FORMAT_1("%05.7d", "0000123", (int)123); + CHECK_FORMAT_1("%05.7i", "0000123", (int)123); + CHECK_FORMAT_1("%05.7u", "0000123", (unsigned int)123); + CHECK_FORMAT_1("%05.7ld", "0000123", (long)123); + CHECK_FORMAT_1("%05.7li", "0000123", (long)123); + CHECK_FORMAT_1("%05.7lu", "0000123", (unsigned long)123); + CHECK_FORMAT_1("%05.7lld", "0000123", (long long)123); + CHECK_FORMAT_1("%05.7lli", "0000123", (long long)123); + CHECK_FORMAT_1("%05.7llu", "0000123", (unsigned long long)123); + CHECK_FORMAT_1("%05.7zd", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%05.7zi", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%05.7zu", "0000123", (size_t)123); + CHECK_FORMAT_1("%05.7x", "000007b", (int)123); + + CHECK_FORMAT_1("%05.7d", "-0000123", (int)-123); + CHECK_FORMAT_1("%05.7i", "-0000123", (int)-123); + CHECK_FORMAT_1("%05.7ld", "-0000123", (long)-123); + CHECK_FORMAT_1("%05.7li", "-0000123", (long)-123); + CHECK_FORMAT_1("%05.7lld", "-0000123", (long long)-123); + CHECK_FORMAT_1("%05.7lli", "-0000123", (long long)-123); + CHECK_FORMAT_1("%05.7zd", "-0000123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%05.7zi", "-0000123", (Py_ssize_t)-123); + + // Integers: precision = 0, arg = 0 (empty string in C) + CHECK_FORMAT_1("%.0d", "0", (int)0); + CHECK_FORMAT_1("%.0i", "0", (int)0); + CHECK_FORMAT_1("%.0u", "0", (unsigned int)0); + CHECK_FORMAT_1("%.0ld", "0", (long)0); + CHECK_FORMAT_1("%.0li", "0", (long)0); + CHECK_FORMAT_1("%.0lu", "0", (unsigned long)0); + CHECK_FORMAT_1("%.0lld", "0", (long long)0); + CHECK_FORMAT_1("%.0lli", "0", (long long)0); + CHECK_FORMAT_1("%.0llu", "0", (unsigned long long)0); + CHECK_FORMAT_1("%.0zd", "0", (Py_ssize_t)0); + CHECK_FORMAT_1("%.0zi", "0", (Py_ssize_t)0); + CHECK_FORMAT_1("%.0zu", "0", (size_t)0); + CHECK_FORMAT_1("%.0x", "0", (int)0); + + // Strings + CHECK_FORMAT_1("%s", "None", "None"); + CHECK_FORMAT_1("%ls", "None", L"None"); + CHECK_FORMAT_1("%U", "None", unicode); + CHECK_FORMAT_1("%A", "None", Py_None); + CHECK_FORMAT_1("%S", "None", Py_None); + CHECK_FORMAT_1("%R", "None", Py_None); + CHECK_FORMAT_2("%V", "None", unicode, "ignored"); + CHECK_FORMAT_2("%V", "None", NULL, "None"); + CHECK_FORMAT_2("%lV", "None", NULL, L"None"); + + // Strings: width < length + CHECK_FORMAT_1("%1s", "None", "None"); + CHECK_FORMAT_1("%1ls", "None", L"None"); + CHECK_FORMAT_1("%1U", "None", unicode); + CHECK_FORMAT_1("%1A", "None", Py_None); + CHECK_FORMAT_1("%1S", "None", Py_None); + CHECK_FORMAT_1("%1R", "None", Py_None); + CHECK_FORMAT_2("%1V", "None", unicode, "ignored"); + CHECK_FORMAT_2("%1V", "None", NULL, "None"); + CHECK_FORMAT_2("%1lV", "None", NULL, L"None"); + + // Strings: width > length + CHECK_FORMAT_1("%5s", " None", "None"); + CHECK_FORMAT_1("%5ls", " None", L"None"); + CHECK_FORMAT_1("%5U", " None", unicode); + CHECK_FORMAT_1("%5A", " None", Py_None); + CHECK_FORMAT_1("%5S", " None", Py_None); + CHECK_FORMAT_1("%5R", " None", Py_None); + CHECK_FORMAT_2("%5V", " None", unicode, "ignored"); + CHECK_FORMAT_2("%5V", " None", NULL, "None"); + CHECK_FORMAT_2("%5lV", " None", NULL, L"None"); + + // Strings: precision < length + CHECK_FORMAT_1("%.1s", "N", "None"); + CHECK_FORMAT_1("%.1ls", "N", L"None"); + CHECK_FORMAT_1("%.1U", "N", unicode); + CHECK_FORMAT_1("%.1A", "N", Py_None); + CHECK_FORMAT_1("%.1S", "N", Py_None); + CHECK_FORMAT_1("%.1R", "N", Py_None); + CHECK_FORMAT_2("%.1V", "N", unicode, "ignored"); + CHECK_FORMAT_2("%.1V", "N", NULL, "None"); + CHECK_FORMAT_2("%.1lV", "N", NULL, L"None"); + + // Strings: precision > length + CHECK_FORMAT_1("%.5s", "None", "None"); + CHECK_FORMAT_1("%.5ls", "None", L"None"); + CHECK_FORMAT_1("%.5U", "None", unicode); + CHECK_FORMAT_1("%.5A", "None", Py_None); + CHECK_FORMAT_1("%.5S", "None", Py_None); + CHECK_FORMAT_1("%.5R", "None", Py_None); + CHECK_FORMAT_2("%.5V", "None", unicode, "ignored"); + CHECK_FORMAT_2("%.5V", "None", NULL, "None"); + CHECK_FORMAT_2("%.5lV", "None", NULL, L"None"); + + // Strings: precision < length, width > length + CHECK_FORMAT_1("%5.1s", " N", "None"); + CHECK_FORMAT_1("%5.1ls"," N", L"None"); + CHECK_FORMAT_1("%5.1U", " N", unicode); + CHECK_FORMAT_1("%5.1A", " N", Py_None); + CHECK_FORMAT_1("%5.1S", " N", Py_None); + CHECK_FORMAT_1("%5.1R", " N", Py_None); + CHECK_FORMAT_2("%5.1V", " N", unicode, "ignored"); + CHECK_FORMAT_2("%5.1V", " N", NULL, "None"); + CHECK_FORMAT_2("%5.1lV"," N", NULL, L"None"); + + // Strings: width < length, precision > length + CHECK_FORMAT_1("%1.5s", "None", "None"); + CHECK_FORMAT_1("%1.5ls", "None", L"None"); + CHECK_FORMAT_1("%1.5U", "None", unicode); + CHECK_FORMAT_1("%1.5A", "None", Py_None); + CHECK_FORMAT_1("%1.5S", "None", Py_None); + CHECK_FORMAT_1("%1.5R", "None", Py_None); + CHECK_FORMAT_2("%1.5V", "None", unicode, "ignored"); + CHECK_FORMAT_2("%1.5V", "None", NULL, "None"); + CHECK_FORMAT_2("%1.5lV", "None", NULL, L"None"); + + Py_XDECREF(unicode); + Py_RETURN_NONE; + + Fail: + Py_XDECREF(result); + Py_XDECREF(unicode); + return NULL; + +#undef CHECK_FORMAT_2 +#undef CHECK_FORMAT_1 +#undef CHECK_FORMAT_0 +} + + +/* Test PyUnicode_Equal() */ +static PyObject * +unicode_equal(PyObject *module, PyObject *args) +{ + PyObject *str1, *str2; + if (!PyArg_ParseTuple(args, "OO", &str1, &str2)) { + return NULL; + } + + NULLABLE(str1); + NULLABLE(str2); + RETURN_INT(PyUnicode_Equal(str1, str2)); +} + + + +static PyMethodDef TestMethods[] = { + {"codec_incrementalencoder", codec_incrementalencoder, METH_VARARGS}, + {"codec_incrementaldecoder", codec_incrementaldecoder, METH_VARARGS}, + {"test_unicode_compare_with_ascii", + test_unicode_compare_with_ascii, METH_NOARGS}, + {"test_string_from_format", test_string_from_format, METH_NOARGS}, + {"test_widechar", test_widechar, METH_NOARGS}, + {"unicode_writechar", unicode_writechar, METH_VARARGS}, + {"unicode_resize", unicode_resize, METH_VARARGS}, + {"unicode_append", unicode_append, METH_VARARGS}, + {"unicode_appendanddel", unicode_appendanddel, METH_VARARGS}, + {"unicode_fromstringandsize",unicode_fromstringandsize, METH_VARARGS}, + {"unicode_fromstring", unicode_fromstring, METH_O}, + {"unicode_substring", unicode_substring, METH_VARARGS}, + {"unicode_getlength", unicode_getlength, METH_O}, + {"unicode_readchar", unicode_readchar, METH_VARARGS}, + {"unicode_fromencodedobject",unicode_fromencodedobject, METH_VARARGS}, + {"unicode_fromobject", unicode_fromobject, METH_O}, + {"unicode_interninplace", unicode_interninplace, METH_O}, + {"unicode_internfromstring", unicode_internfromstring, METH_O}, + {"unicode_fromwidechar", unicode_fromwidechar, METH_VARARGS}, + {"unicode_aswidechar", unicode_aswidechar, METH_VARARGS}, + {"unicode_aswidechar_null", unicode_aswidechar_null, METH_VARARGS}, + {"unicode_aswidecharstring", unicode_aswidecharstring, METH_VARARGS}, + {"unicode_aswidecharstring_null",unicode_aswidecharstring_null,METH_VARARGS}, + {"unicode_fromordinal", unicode_fromordinal, METH_VARARGS}, + {"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS}, + {"unicode_asutf8andsize_null",unicode_asutf8andsize_null, METH_VARARGS}, + {"unicode_getdefaultencoding",unicode_getdefaultencoding, METH_NOARGS}, + {"unicode_decode", unicode_decode, METH_VARARGS}, + {"unicode_asencodedstring", unicode_asencodedstring, METH_VARARGS}, + {"unicode_buildencodingmap", unicode_buildencodingmap, METH_O}, + {"unicode_decodeutf7", unicode_decodeutf7, METH_VARARGS}, + {"unicode_decodeutf7stateful",unicode_decodeutf7stateful, METH_VARARGS}, + {"unicode_decodeutf8", unicode_decodeutf8, METH_VARARGS}, + {"unicode_decodeutf8stateful",unicode_decodeutf8stateful, METH_VARARGS}, + {"unicode_asutf8string", unicode_asutf8string, METH_O}, + {"unicode_decodeutf16", unicode_decodeutf16, METH_VARARGS}, + {"unicode_decodeutf16stateful",unicode_decodeutf16stateful, METH_VARARGS}, + {"unicode_asutf16string", unicode_asutf16string, METH_O}, + {"unicode_decodeutf32", unicode_decodeutf32, METH_VARARGS}, + {"unicode_decodeutf32stateful",unicode_decodeutf32stateful, METH_VARARGS}, + {"unicode_asutf32string", unicode_asutf32string, METH_O}, + {"unicode_decodeunicodeescape",unicode_decodeunicodeescape, METH_VARARGS}, + {"unicode_asunicodeescapestring",unicode_asunicodeescapestring,METH_O}, + {"unicode_decoderawunicodeescape",unicode_decoderawunicodeescape,METH_VARARGS}, + {"unicode_asrawunicodeescapestring",unicode_asrawunicodeescapestring,METH_O}, + {"unicode_decodelatin1", unicode_decodelatin1, METH_VARARGS}, + {"unicode_aslatin1string", unicode_aslatin1string, METH_O}, + {"unicode_decodeascii", unicode_decodeascii, METH_VARARGS}, + {"unicode_asasciistring", unicode_asasciistring, METH_O}, + {"unicode_decodecharmap", unicode_decodecharmap, METH_VARARGS}, + {"unicode_ascharmapstring", unicode_ascharmapstring, METH_VARARGS}, +#ifdef MS_WINDOWS + {"unicode_decodembcs", unicode_decodembcs, METH_VARARGS}, + {"unicode_decodembcsstateful",unicode_decodembcsstateful, METH_VARARGS}, + {"unicode_decodecodepagestateful",unicode_decodecodepagestateful,METH_VARARGS}, + {"unicode_asmbcsstring", unicode_asmbcsstring, METH_O}, + {"unicode_encodecodepage", unicode_encodecodepage, METH_VARARGS}, +#endif /* MS_WINDOWS */ + {"unicode_decodelocaleandsize",unicode_decodelocaleandsize, METH_VARARGS}, + {"unicode_decodelocale", unicode_decodelocale, METH_VARARGS}, + {"unicode_encodelocale", unicode_encodelocale, METH_VARARGS}, + {"unicode_decodefsdefault", unicode_decodefsdefault, METH_VARARGS}, + {"unicode_decodefsdefaultandsize",unicode_decodefsdefaultandsize,METH_VARARGS}, + {"unicode_encodefsdefault", unicode_encodefsdefault, METH_O}, + {"unicode_concat", unicode_concat, METH_VARARGS}, + {"unicode_splitlines", unicode_splitlines, METH_VARARGS}, + {"unicode_split", unicode_split, METH_VARARGS}, + {"unicode_rsplit", unicode_rsplit, METH_VARARGS}, + {"unicode_partition", unicode_partition, METH_VARARGS}, + {"unicode_rpartition", unicode_rpartition, METH_VARARGS}, + {"unicode_translate", unicode_translate, METH_VARARGS}, + {"unicode_join", unicode_join, METH_VARARGS}, + {"unicode_count", unicode_count, METH_VARARGS}, + {"unicode_tailmatch", unicode_tailmatch, METH_VARARGS}, + {"unicode_find", unicode_find, METH_VARARGS}, + {"unicode_findchar", unicode_findchar, METH_VARARGS}, + {"unicode_replace", unicode_replace, METH_VARARGS}, + {"unicode_compare", unicode_compare, METH_VARARGS}, + {"unicode_comparewithasciistring",unicode_comparewithasciistring,METH_VARARGS}, + {"unicode_equaltoutf8", unicode_equaltoutf8, METH_VARARGS}, + {"unicode_equaltoutf8andsize",unicode_equaltoutf8andsize, METH_VARARGS}, + {"unicode_richcompare", unicode_richcompare, METH_VARARGS}, + {"unicode_format", unicode_format, METH_VARARGS}, + {"unicode_contains", unicode_contains, METH_VARARGS}, + {"unicode_isidentifier", unicode_isidentifier, METH_O}, + {"unicode_equal", unicode_equal, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Unicode(PyObject *m) +{ + if (PyModule_AddFunctions(m, TestMethods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/util.h b/Modules/_testlimitedcapi/util.h new file mode 100644 index 00000000000000..f26d7656a10138 --- /dev/null +++ b/Modules/_testlimitedcapi/util.h @@ -0,0 +1,33 @@ +#define NULLABLE(x) do { \ + if (x == Py_None) { \ + x = NULL; \ + } \ + } while (0); + +#define RETURN_INT(value) do { \ + int _ret = (value); \ + if (_ret == -1) { \ + assert(PyErr_Occurred()); \ + return NULL; \ + } \ + assert(!PyErr_Occurred()); \ + return PyLong_FromLong(_ret); \ + } while (0) + +#define RETURN_SIZE(value) do { \ + Py_ssize_t _ret = (value); \ + if (_ret == -1) { \ + assert(PyErr_Occurred()); \ + return NULL; \ + } \ + assert(!PyErr_Occurred()); \ + return PyLong_FromSsize_t(_ret); \ + } while (0) + +/* Marker to check that pointer value was set. */ +static const char uninitialized[] = "uninitialized"; +#define UNINITIALIZED_PTR ((void *)uninitialized) +/* Marker to check that Py_ssize_t value was set. */ +#define UNINITIALIZED_SIZE ((Py_ssize_t)236892191) +/* Marker to check that integer value was set. */ +#define UNINITIALIZED_INT (63256717) diff --git a/Modules/_testcapi/vectorcall_limited.c b/Modules/_testlimitedcapi/vectorcall_limited.c similarity index 62% rename from Modules/_testcapi/vectorcall_limited.c rename to Modules/_testlimitedcapi/vectorcall_limited.c index d7070d37bb9e9b..4a7af965776470 100644 --- a/Modules/_testcapi/vectorcall_limited.c +++ b/Modules/_testlimitedcapi/vectorcall_limited.c @@ -1,18 +1,20 @@ /* Test Vectorcall in the limited API */ +// Need limited C API version 3.12 for PyObject_Vectorcall() #include "pyconfig.h" // Py_GIL_DISABLED - -#ifndef Py_GIL_DISABLED -#define Py_LIMITED_API 0x030c0000 // 3.12 +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +# define Py_LIMITED_API 0x030c0000 #endif +#include // offsetof + #include "parts.h" #include "clinic/vectorcall_limited.c.h" /*[clinic input] -module _testcapi +module _testlimitedcapi [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=2700057f9c1135ba]*/ static PyObject * LimitedVectorCallClass_tpcall(PyObject *self, PyObject *args, PyObject *kwargs) { @@ -40,15 +42,15 @@ LimitedVectorCallClass_new(PyTypeObject *tp, PyTypeObject *a, PyTypeObject *kw) } /*[clinic input] -_testcapi.call_vectorcall +_testlimitedcapi.call_vectorcall callable: object / [clinic start generated code]*/ static PyObject * -_testcapi_call_vectorcall(PyObject *module, PyObject *callable) -/*[clinic end generated code: output=bae81eec97fcaad7 input=55d88f92240957ee]*/ +_testlimitedcapi_call_vectorcall(PyObject *module, PyObject *callable) +/*[clinic end generated code: output=9cbb7832263a8eef input=0743636c12dccb28]*/ { PyObject *args[3] = { NULL, NULL, NULL }; PyObject *kwname = NULL, *kwnames = NULL, *result = NULL; @@ -93,15 +95,15 @@ _testcapi_call_vectorcall(PyObject *module, PyObject *callable) } /*[clinic input] -_testcapi.call_vectorcall_method +_testlimitedcapi.call_vectorcall_method callable: object / [clinic start generated code]*/ static PyObject * -_testcapi_call_vectorcall_method(PyObject *module, PyObject *callable) -/*[clinic end generated code: output=e661f48dda08b6fb input=5ba81c27511395b6]*/ +_testlimitedcapi_call_vectorcall_method(PyObject *module, PyObject *callable) +/*[clinic end generated code: output=4558323a46cc09eb input=a736f7dbf15f1be5]*/ { PyObject *args[3] = { NULL, NULL, NULL }; PyObject *name = NULL, *kwname = NULL, @@ -167,7 +169,7 @@ static PyType_Slot LimitedVectorallClass_slots[] = { }; static PyType_Spec LimitedVectorCallClass_spec = { - .name = "_testcapi.LimitedVectorCallClass", + .name = "_testlimitedcapi.LimitedVectorCallClass", .basicsize = (int)(sizeof(PyObject) + sizeof(vectorcallfunc)), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_VECTORCALL @@ -175,14 +177,50 @@ static PyType_Spec LimitedVectorCallClass_spec = { .slots = LimitedVectorallClass_slots, }; +typedef struct { + vectorcallfunc vfunc; +} LimitedRelativeVectorCallStruct; + +static PyObject * +LimitedRelativeVectorCallClass_new(PyTypeObject *tp, PyTypeObject *a, PyTypeObject *kw) +{ + PyObject *self = ((allocfunc)PyType_GetSlot(tp, Py_tp_alloc))(tp, 0); + if (!self) { + return NULL; + } + LimitedRelativeVectorCallStruct *data = PyObject_GetTypeData(self, tp); + data->vfunc = LimitedVectorCallClass_vectorcall; + return self; +} + + +static PyType_Spec LimitedRelativeVectorCallClass_spec = { + .name = "_testlimitedcapi.LimitedRelativeVectorCallClass", + .basicsize = -(int)sizeof(LimitedRelativeVectorCallStruct), + .flags = Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_HAVE_VECTORCALL, + .slots = (PyType_Slot[]) { + {Py_tp_new, LimitedRelativeVectorCallClass_new}, + {Py_tp_call, LimitedVectorCallClass_tpcall}, + {Py_tp_members, (PyMemberDef[]){ + {"__vectorcalloffset__", Py_T_PYSSIZET, + offsetof(LimitedRelativeVectorCallStruct, vfunc), + Py_READONLY | Py_RELATIVE_OFFSET}, + {NULL} + }}, + {0} + }, +}; + static PyMethodDef TestMethods[] = { - _TESTCAPI_CALL_VECTORCALL_METHODDEF - _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF + _TESTLIMITEDCAPI_CALL_VECTORCALL_METHODDEF + _TESTLIMITEDCAPI_CALL_VECTORCALL_METHOD_METHODDEF {NULL}, }; int -_PyTestCapi_Init_VectorcallLimited(PyObject *m) { +_PyTestLimitedCAPI_Init_VectorcallLimited(PyObject *m) +{ if (PyModule_AddFunctions(m, TestMethods) < 0) { return -1; } @@ -196,5 +234,16 @@ _PyTestCapi_Init_VectorcallLimited(PyObject *m) { return -1; } Py_DECREF(LimitedVectorCallClass); + + PyObject *LimitedRelativeVectorCallClass = PyType_FromModuleAndSpec( + m, &LimitedRelativeVectorCallClass_spec, NULL); + if (!LimitedRelativeVectorCallClass) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject *)LimitedRelativeVectorCallClass) < 0) { + return -1; + } + Py_DECREF(LimitedRelativeVectorCallClass); + return 0; } diff --git a/Modules/_testlimitedcapi/version.c b/Modules/_testlimitedcapi/version.c new file mode 100644 index 00000000000000..57cd6e4e928ea3 --- /dev/null +++ b/Modules/_testlimitedcapi/version.c @@ -0,0 +1,77 @@ +/* Test version macros in the limited API */ + +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030e0000 // Added in 3.14 +#endif + +#include "parts.h" +#include "clinic/version.c.h" +#include + +/*[clinic input] +module _testlimitedcapi +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=2700057f9c1135ba]*/ + +/*[clinic input] +_testlimitedcapi.pack_full_version + + major: int + minor: int + micro: int + level: int + serial: int + / +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_pack_full_version_impl(PyObject *module, int major, + int minor, int micro, int level, + int serial) +/*[clinic end generated code: output=b87a1e9805648861 input=2a304423be61d2ac]*/ +{ + uint32_t macro_result = Py_PACK_FULL_VERSION( + major, minor, micro, level, serial); +#undef Py_PACK_FULL_VERSION + uint32_t func_result = Py_PACK_FULL_VERSION( + major, minor, micro, level, serial); + + assert(macro_result == func_result); + return PyLong_FromUnsignedLong((unsigned long)func_result); +} + +/*[clinic input] +_testlimitedcapi.pack_version + + major: int + minor: int + / +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_pack_version_impl(PyObject *module, int major, int minor) +/*[clinic end generated code: output=771247bbd06e7883 input=3e39e9dcbc09e86a]*/ +{ + uint32_t macro_result = Py_PACK_VERSION(major, minor); +#undef Py_PACK_VERSION + uint32_t func_result = Py_PACK_VERSION(major, minor); + + assert(macro_result == func_result); + return PyLong_FromUnsignedLong((unsigned long)func_result); +} + +static PyMethodDef TestMethods[] = { + _TESTLIMITEDCAPI_PACK_FULL_VERSION_METHODDEF + _TESTLIMITEDCAPI_PACK_VERSION_METHODDEF + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Version(PyObject *m) +{ + if (PyModule_AddFunctions(m, TestMethods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index 21c5f696a4f2ec..886b260aceb20d 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -431,6 +431,7 @@ static int execfunc(PyObject *m) static PyModuleDef_Slot main_slots[] = { {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -519,13 +520,18 @@ PyInit__testmultiphase_nonmodule_with_methods(void) /**** Non-ASCII-named modules ****/ +static PyModuleDef_Slot nonascii_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static PyModuleDef def_nonascii_latin = { \ PyModuleDef_HEAD_INIT, /* m_base */ "_testmultiphase_nonascii_latin", /* m_name */ PyDoc_STR("Module named in Czech"), /* m_doc */ 0, /* m_size */ NULL, /* m_methods */ - NULL, /* m_slots */ + nonascii_slots, /* m_slots */ NULL, /* m_traverse */ NULL, /* m_clear */ NULL, /* m_free */ @@ -543,7 +549,7 @@ static PyModuleDef def_nonascii_kana = { \ PyDoc_STR("Module named in Japanese"), /* m_doc */ 0, /* m_size */ NULL, /* m_methods */ - NULL, /* m_slots */ + nonascii_slots, /* m_slots */ NULL, /* m_traverse */ NULL, /* m_clear */ NULL, /* m_free */ @@ -757,6 +763,7 @@ static PyModuleDef_Slot slots_nonmodule_with_exec_slots[] = { {Py_mod_create, createfunc_nonmodule}, {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -778,6 +785,7 @@ execfunc_err(PyObject *mod) static PyModuleDef_Slot slots_exec_err[] = { {Py_mod_exec, execfunc_err}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -800,6 +808,7 @@ execfunc_raise(PyObject *spec) static PyModuleDef_Slot slots_exec_raise[] = { {Py_mod_exec, execfunc_raise}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -822,6 +831,7 @@ execfunc_unreported_exception(PyObject *mod) static PyModuleDef_Slot slots_exec_unreported_exception[] = { {Py_mod_exec, execfunc_unreported_exception}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -857,6 +867,7 @@ meth_state_access_exec(PyObject *m) static PyModuleDef_Slot meth_state_access_slots[] = { {Py_mod_exec, meth_state_access_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; @@ -889,6 +900,9 @@ PyInit__test_module_state_shared(void) if (module == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); +#endif if (PyModule_AddObjectRef(module, "Error", PyExc_Exception) < 0) { Py_DECREF(module); @@ -903,6 +917,7 @@ PyInit__test_module_state_shared(void) static PyModuleDef_Slot slots_multiple_multiple_interpreters_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -920,6 +935,7 @@ PyInit__testmultiphase_multiple_multiple_interpreters_slots(void) static PyModuleDef_Slot non_isolated_slots[] = { {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -940,6 +956,7 @@ static PyModuleDef_Slot shared_gil_only_slots[] = { We put it here explicitly to draw attention to the contrast with Py_MOD_PER_INTERPRETER_GIL_SUPPORTED. */ {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index 092673a9ea43e1..2c59085d15b5be 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -1,6 +1,207 @@ /* Testing module for single-phase initialization of extension modules - */ + +This file contains several distinct modules, meaning each as its own name +and its own init function (PyInit_...). The default import system will +only find the one matching the filename: _testsinglephase. To load the +others you must do so manually. For example: + +```python +name = '_testsinglephase_base_wrapper' +filename = _testsinglephase.__file__ +loader = importlib.machinery.ExtensionFileLoader(name, filename) +spec = importlib.util.spec_from_file_location(name, filename, loader=loader) +mod = importlib._bootstrap._load(spec) +loader.exec_module(module) +sys.modules[modname] = module +``` + +(The last two lines are just for completeness.) + +Here are the modules: + +* _testsinglephase + * def: _testsinglephase_basic, + * m_name: "_testsinglephase" + * m_size: -1 + * state + * process-global + * initialized_count (default to -1; will never be 0) + * module (see module state below) + * module state: no + * initial __dict__: see common initial __dict__ below + * init function + 1. create module + 2. clear .module + 3. initialize .module: see module state below + 4. initialize module: set initial __dict__ + 5. increment .initialized_count + * functions + * (3 common, see below) + * initialized_count() - return .module.initialized_count + * import system + * caches + * global extensions cache: yes + * def.m_base.m_copy: yes + * def.m_base.m_init: no + * per-interpreter cache: yes (all single-phase init modules) + * load in main interpreter + * initial (not already in global cache) + 1. get init function from shared object file + 2. run init function + 3. copy __dict__ into def.m_base.m_copy + 4. set entry in global cache + 5. set entry in per-interpreter cache + 6. set entry in sys.modules + * reload (already in sys.modules) + 1. get def from global cache + 2. get module from sys.modules + 3. update module with contents of def.m_base.m_copy + * already loaded in other interpreter (already in global cache) + * same as reload, but create new module and update *it* + * not in any sys.modules, still in global cache + * same as already loaded + * load in legacy (non-isolated) interpreter + * same as main interpreter + * unload: never (all single-phase init modules) +* _testsinglephase_basic_wrapper + * identical to _testsinglephase except module name +* _testsinglephase_basic_copy + * def: static local variable in init function + * m_name: "_testsinglephase_basic_copy" + * m_size: -1 + * state: same as _testsinglephase + * init function: same as _testsinglephase + * functions: same as _testsinglephase + * import system: same as _testsinglephase +* _testsinglephase_with_reinit + * def: _testsinglephase_with_reinit, + * m_name: "_testsinglephase_with_reinit" + * m_size: 0 + * state + * process-global state: no + * module state: no + * initial __dict__: see common initial __dict__ below + * init function + 1. create module + 2. initialize temporary module state (local var): see module state below + 3. initialize module: set initial __dict__ + * functions: see common functions below + * import system + * caches + * global extensions cache: only if loaded in main interpreter + * def.m_base.m_copy: no + * def.m_base.m_init: only if loaded in the main interpreter + * per-interpreter cache: yes (all single-phase init modules) + * load in main interpreter + * initial (not already in global cache) + * (same as _testsinglephase except step 3) + 1. get init function from shared object file + 2. run init function + 3. set def.m_base.m_init to the init function + 4. set entry in global cache + 5. set entry in per-interpreter cache + 6. set entry in sys.modules + * reload (already in sys.modules) + 1. get def from global cache + 2. call def->m_base.m_init to get a new module object + 3. replace the existing module in sys.modules + * already loaded in other interpreter (already in global cache) + * same as reload (since will only be in cache for main interp) + * not in any sys.modules, still in global cache + * same as already loaded + * load in legacy (non-isolated) interpreter + * initial (not already in global cache) + * (same as main interpreter except skip steps 3 & 4 there) + 1. get init function from shared object file + 2. run init function + ... + 5. set entry in per-interpreter cache + 6. set entry in sys.modules + * reload (already in sys.modules) + * same as initial (load from scratch) + * already loaded in other interpreter (already in global cache) + * same as initial (load from scratch) + * not in any sys.modules, still in global cache + * same as initial (load from scratch) + * unload: never (all single-phase init modules) +* _testsinglephase_with_state + * def: _testsinglephase_with_state, + * m_name: "_testsinglephase_with_state" + * m_size: sizeof(module_state) + * state + * process-global: no + * module state: see module state below + * initial __dict__: see common initial __dict__ below + * init function + 1. create module + 3. initialize module state: see module state below + 4. initialize module: set initial __dict__ + 5. increment .initialized_count + * functions: see common functions below + * import system: same as _testsinglephase_basic_copy +* _testsinglephase_check_cache_first + * def: _testsinglepahse_check_cache_first + * m_name: "_testsinglephase_check_cache_first" + * m_size: -1 + * state: none + * init function: + * tries PyState_FindModule() first + * otherwise creates empty module + * functions: none + * import system: same as _testsinglephase +* _testsinglephase_with_reinit_check_cache_first + * def: _testsinglepahse_with_reinit_check_cache_first + * m_name: "_testsinglephase_with_reinit_check_cache_first" + * m_size: 0 + * state: none + * init function: same as _testsinglephase_check_cache_first + * functions: none + * import system: same as _testsinglephase_with_reinit +* _testsinglephase_with_state_check_cache_first + * def: _testsinglepahse_with_state_check_cache_first + * m_name: "_testsinglephase_with_state_check_cache_first" + * m_size: 42 + * state: none + * init function: same as _testsinglephase_check_cache_first + * functions: none + * import system: same as _testsinglephase_with_state + +* _testsinglephase_circular + Regression test for gh-123880. + Does not have the common attributes & methods. + See test_singlephase_circular test.test_import.SinglephaseInitTests. + +Module state: + +* fields + * initialized - when the module was first initialized + * *error + * *int_const + * *str_const +* initialization + 1. set state.initialized to the current time + 2. set state.error to a new exception class + 3. set state->int_const to int(1969) + 4. set state->str_const to "something different" + +Common initial __dict__: + +* error: state.error +* int_const: state.int_const +* str_const: state.str_const +* _module_initialized: state.initialized + +Common functions: + +* look_up_self() - return the module from the per-interpreter "by-index" cache +* sum() - return a + b +* state_initialized() - return state->initialized (or None if m_size == 0) + +See Python/import.c, especially the long comments, for more about +single-phase init modules. +*/ + #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif @@ -305,6 +506,9 @@ init__testsinglephase_basic(PyModuleDef *def) if (module == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); +#endif module_state *state = &global_state.module; // It may have been set by a previous run or under a different name. @@ -396,6 +600,9 @@ PyInit__testsinglephase_with_reinit(void) if (module == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); +#endif assert(get_module_state(module) == NULL); @@ -458,6 +665,9 @@ PyInit__testsinglephase_with_state(void) if (module == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); +#endif module_state *state = get_module_state(module); assert(state != NULL); @@ -475,3 +685,117 @@ PyInit__testsinglephase_with_state(void) finally: return module; } + + +/****************************************************/ +/* the _testsinglephase_*_check_cache_first modules */ +/****************************************************/ + +/* Each of these modules should only be freshly loaded. That means + clearing the caches and each module def's m_base after each load. */ + +static struct PyModuleDef _testsinglephase_check_cache_first = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_check_cache_first", + .m_doc = PyDoc_STR("Test module _testsinglephase_check_cache_first"), + .m_size = -1, // no module state +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_check_cache_first(void) +{ + assert(_testsinglephase_check_cache_first.m_base.m_index == 0); + PyObject *mod = PyState_FindModule(&_testsinglephase_check_cache_first); + if (mod != NULL) { + return Py_NewRef(mod); + } + return PyModule_Create(&_testsinglephase_check_cache_first); +} + + +static struct PyModuleDef _testsinglephase_with_reinit_check_cache_first = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_with_reinit_check_cache_first", + .m_doc = PyDoc_STR("Test module _testsinglephase_with_reinit_check_cache_first"), + .m_size = 0, // no module state +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_with_reinit_check_cache_first(void) +{ + assert(_testsinglephase_with_reinit_check_cache_first.m_base.m_index == 0); + PyObject *mod = PyState_FindModule(&_testsinglephase_with_reinit_check_cache_first); + if (mod != NULL) { + return Py_NewRef(mod); + } + return PyModule_Create(&_testsinglephase_with_reinit_check_cache_first); +} + + +static struct PyModuleDef _testsinglephase_with_state_check_cache_first = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_with_state_check_cache_first", + .m_doc = PyDoc_STR("Test module _testsinglephase_with_state_check_cache_first"), + .m_size = 42, // not used +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_with_state_check_cache_first(void) +{ + assert(_testsinglephase_with_state_check_cache_first.m_base.m_index == 0); + PyObject *mod = PyState_FindModule(&_testsinglephase_with_state_check_cache_first); + if (mod != NULL) { + return Py_NewRef(mod); + } + return PyModule_Create(&_testsinglephase_with_state_check_cache_first); +} + + +/****************************************/ +/* the _testsinglephase_circular module */ +/****************************************/ + +static PyObject *static_module_circular; + +static PyObject * +circularmod_clear_static_var(PyObject *self, PyObject *arg) +{ + PyObject *result = static_module_circular; + static_module_circular = NULL; + return result; +} + +static struct PyModuleDef _testsinglephase_circular = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_circular", + .m_doc = PyDoc_STR("Test module _testsinglephase_circular"), + .m_methods = (PyMethodDef[]) { + {"clear_static_var", circularmod_clear_static_var, METH_NOARGS, + "Clear the static variable and return its previous value."}, + {NULL, NULL} /* sentinel */ + } +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_circular(void) +{ + if (!static_module_circular) { + static_module_circular = PyModule_Create(&_testsinglephase_circular); + if (!static_module_circular) { + return NULL; + } + } + static const char helper_mod_name[] = ( + "test.test_import.data.circular_imports.singlephase"); + PyObject *helper_mod = PyImport_ImportModule(helper_mod_name); + Py_XDECREF(helper_mod); + if (!helper_mod) { + return NULL; + } + if(PyModule_AddStringConstant(static_module_circular, + "helper_mod_name", + helper_mod_name) < 0) { + return NULL; + } + return Py_NewRef(static_module_circular); +} diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 3a8f77d6dfbbc6..dbc574f7816b85 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -12,16 +12,16 @@ #include "pycore_time.h" // _PyTime_FromSeconds() #include "pycore_weakref.h" // _PyWeakref_GET_REF() -#include #include // offsetof() #ifdef HAVE_SIGNAL_H # include // SIGINT #endif +#include "clinic/_threadmodule.c.h" + // ThreadError is just an alias to PyExc_RuntimeError #define ThreadError PyExc_RuntimeError - // Forward declarations static struct PyModuleDef thread_module; @@ -32,6 +32,10 @@ typedef struct { PyTypeObject *local_type; PyTypeObject *local_dummy_type; PyTypeObject *thread_handle_type; + + // Linked list of handles to all non-daemon threads created by the + // threading module. We wait for these to finish at shutdown. + struct llist_node shutdown_handles; } thread_module_state; static inline thread_module_state* @@ -42,78 +46,165 @@ get_thread_state(PyObject *module) return (thread_module_state *)state; } + +#ifdef MS_WINDOWS +typedef HRESULT (WINAPI *PF_GET_THREAD_DESCRIPTION)(HANDLE, PCWSTR*); +typedef HRESULT (WINAPI *PF_SET_THREAD_DESCRIPTION)(HANDLE, PCWSTR); +static PF_GET_THREAD_DESCRIPTION pGetThreadDescription = NULL; +static PF_SET_THREAD_DESCRIPTION pSetThreadDescription = NULL; +#endif + + +/*[clinic input] +module _thread +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=be8dbe5cc4b16df7]*/ + + // _ThreadHandle type -// Handles transition from RUNNING to one of JOINED, DETACHED, or INVALID (post -// fork). +// Handles state transitions according to the following diagram: +// +// NOT_STARTED -> STARTING -> RUNNING -> DONE +// | ^ +// | | +// +----- error --------+ typedef enum { - THREAD_HANDLE_RUNNING = 1, - THREAD_HANDLE_JOINED = 2, - THREAD_HANDLE_DETACHED = 3, - THREAD_HANDLE_INVALID = 4, + THREAD_HANDLE_NOT_STARTED = 1, + THREAD_HANDLE_STARTING = 2, + THREAD_HANDLE_RUNNING = 3, + THREAD_HANDLE_DONE = 4, } ThreadHandleState; -// A handle around an OS thread. +// A handle to wait for thread completion. // -// The OS thread is either joined or detached after the handle is destroyed. +// This may be used to wait for threads that were spawned by the threading +// module as well as for the "main" thread of the threading module. In the +// former case an OS thread, identified by the `os_handle` field, will be +// associated with the handle. The handle "owns" this thread and ensures that +// the thread is either joined or detached after the handle is destroyed. // -// Joining the handle is idempotent; the underlying OS thread is joined or -// detached only once. Concurrent join operations are serialized until it is -// their turn to execute or an earlier operation completes successfully. Once a -// join has completed successfully all future joins complete immediately. +// Joining the handle is idempotent; the underlying OS thread, if any, is +// joined or detached only once. Concurrent join operations are serialized +// until it is their turn to execute or an earlier operation completes +// successfully. Once a join has completed successfully all future joins +// complete immediately. +// +// This must be separately reference counted because it may be destroyed +// in `thread_run()` after the PyThreadState has been destroyed. typedef struct { - PyObject_HEAD struct llist_node node; // linked list node (see _pythread_runtime_state) - // The `ident` and `handle` fields are immutable once the object is visible - // to threads other than its creator, thus they do not need to be accessed - // atomically. + // linked list node (see thread_module_state) + struct llist_node shutdown_node; + + // The `ident`, `os_handle`, `has_os_handle`, and `state` fields are + // protected by `mutex`. PyThread_ident_t ident; - PyThread_handle_t handle; + PyThread_handle_t os_handle; + int has_os_handle; // Holds a value from the `ThreadHandleState` enum. int state; + PyMutex mutex; + // Set immediately before `thread_run` returns to indicate that the OS // thread is about to exit. This is used to avoid false positives when // detecting self-join attempts. See the comment in `ThreadHandle_join()` // for a more detailed explanation. - _PyEventRc *thread_is_exiting; + PyEvent thread_is_exiting; - // Serializes calls to `join`. + // Serializes calls to `join` and `set_done`. _PyOnceFlag once; -} ThreadHandleObject; + + Py_ssize_t refcount; +} ThreadHandle; static inline int -get_thread_handle_state(ThreadHandleObject *handle) +get_thread_handle_state(ThreadHandle *handle) { - return _Py_atomic_load_int(&handle->state); + PyMutex_Lock(&handle->mutex); + int state = handle->state; + PyMutex_Unlock(&handle->mutex); + return state; } static inline void -set_thread_handle_state(ThreadHandleObject *handle, ThreadHandleState state) +set_thread_handle_state(ThreadHandle *handle, ThreadHandleState state) { - _Py_atomic_store_int(&handle->state, state); + PyMutex_Lock(&handle->mutex); + handle->state = state; + PyMutex_Unlock(&handle->mutex); } -static ThreadHandleObject* -new_thread_handle(thread_module_state* state) +static PyThread_ident_t +ThreadHandle_ident(ThreadHandle *handle) { - _PyEventRc *event = _PyEventRc_New(); - if (event == NULL) { - PyErr_NoMemory(); - return NULL; + PyMutex_Lock(&handle->mutex); + PyThread_ident_t ident = handle->ident; + PyMutex_Unlock(&handle->mutex); + return ident; +} + +static int +ThreadHandle_get_os_handle(ThreadHandle *handle, PyThread_handle_t *os_handle) +{ + PyMutex_Lock(&handle->mutex); + int has_os_handle = handle->has_os_handle; + if (has_os_handle) { + *os_handle = handle->os_handle; } - ThreadHandleObject* self = PyObject_New(ThreadHandleObject, state->thread_handle_type); + PyMutex_Unlock(&handle->mutex); + return has_os_handle; +} + +static void +add_to_shutdown_handles(thread_module_state *state, ThreadHandle *handle) +{ + HEAD_LOCK(&_PyRuntime); + llist_insert_tail(&state->shutdown_handles, &handle->shutdown_node); + HEAD_UNLOCK(&_PyRuntime); +} + +static void +clear_shutdown_handles(thread_module_state *state) +{ + HEAD_LOCK(&_PyRuntime); + struct llist_node *node; + llist_for_each_safe(node, &state->shutdown_handles) { + llist_remove(node); + } + HEAD_UNLOCK(&_PyRuntime); +} + +static void +remove_from_shutdown_handles(ThreadHandle *handle) +{ + HEAD_LOCK(&_PyRuntime); + if (handle->shutdown_node.next != NULL) { + llist_remove(&handle->shutdown_node); + } + HEAD_UNLOCK(&_PyRuntime); +} + +static ThreadHandle * +ThreadHandle_new(void) +{ + ThreadHandle *self = + (ThreadHandle *)PyMem_RawCalloc(1, sizeof(ThreadHandle)); if (self == NULL) { - _PyEventRc_Decref(event); + PyErr_NoMemory(); return NULL; } self->ident = 0; - self->handle = 0; - self->thread_is_exiting = event; + self->os_handle = 0; + self->has_os_handle = 0; + self->thread_is_exiting = (PyEvent){0}; + self->mutex = (PyMutex){_Py_UNLOCKED}; self->once = (_PyOnceFlag){0}; - self->state = THREAD_HANDLE_INVALID; + self->state = THREAD_HANDLE_NOT_STARTED; + self->refcount = 1; HEAD_LOCK(&_PyRuntime); llist_insert_tail(&_PyRuntime.threads.handles, &self->node); @@ -123,9 +214,34 @@ new_thread_handle(thread_module_state* state) } static void -ThreadHandle_dealloc(ThreadHandleObject *self) +ThreadHandle_incref(ThreadHandle *self) { - PyObject *tp = (PyObject *) Py_TYPE(self); + _Py_atomic_add_ssize(&self->refcount, 1); +} + +static int +detach_thread(ThreadHandle *self) +{ + if (!self->has_os_handle) { + return 0; + } + // This is typically short so no need to release the GIL + if (PyThread_detach_thread(self->os_handle)) { + fprintf(stderr, "detach_thread: failed detaching thread\n"); + return -1; + } + return 0; +} + +// NB: This may be called after the PyThreadState in `thread_run` has been +// deleted; it cannot call anything that relies on a valid PyThreadState +// existing. +static void +ThreadHandle_decref(ThreadHandle *self) +{ + if (_Py_atomic_add_ssize(&self->refcount, -1) > 1) { + return; + } // Remove ourself from the global list of handles HEAD_LOCK(&_PyRuntime); @@ -134,23 +250,17 @@ ThreadHandle_dealloc(ThreadHandleObject *self) } HEAD_UNLOCK(&_PyRuntime); + assert(self->shutdown_node.next == NULL); + // It's safe to access state non-atomically: // 1. This is the destructor; nothing else holds a reference. - // 2. The refcount going to zero is a "synchronizes-with" event; - // all changes from other threads are visible. - if (self->state == THREAD_HANDLE_RUNNING) { - // This is typically short so no need to release the GIL - if (PyThread_detach_thread(self->handle)) { - PyErr_SetString(ThreadError, "Failed detaching thread"); - PyErr_WriteUnraisable(tp); - } - else { - self->state = THREAD_HANDLE_DETACHED; - } + // 2. The refcount going to zero is a "synchronizes-with" event; all + // changes from other threads are visible. + if (self->state == THREAD_HANDLE_RUNNING && !detach_thread(self)) { + self->state = THREAD_HANDLE_DONE; } - _PyEventRc_Decref(self->thread_is_exiting); - PyObject_Free(self); - Py_DECREF(tp); + + PyMem_RawFree(self); } void @@ -164,55 +274,230 @@ _PyThread_AfterFork(struct _pythread_runtime_state *state) struct llist_node *node; llist_for_each_safe(node, &state->handles) { - ThreadHandleObject *hobj = llist_data(node, ThreadHandleObject, node); - if (hobj->ident == current) { + ThreadHandle *handle = llist_data(node, ThreadHandle, node); + if (handle->ident == current) { continue; } - // Disallow calls to join() as they could crash. We are the only - // thread; it's safe to set this without an atomic. - hobj->state = THREAD_HANDLE_INVALID; + // Mark all threads as done. Any attempts to join or detach the + // underlying OS thread (if any) could crash. We are the only thread; + // it's safe to set this non-atomically. + handle->state = THREAD_HANDLE_DONE; + handle->once = (_PyOnceFlag){_Py_ONCE_INITIALIZED}; + handle->mutex = (PyMutex){_Py_UNLOCKED}; + _PyEvent_Notify(&handle->thread_is_exiting); llist_remove(node); + remove_from_shutdown_handles(handle); } } -static PyObject * -ThreadHandle_repr(ThreadHandleObject *self) +// bootstate is used to "bootstrap" new threads. Any arguments needed by +// `thread_run()`, which can only take a single argument due to platform +// limitations, are contained in bootstate. +struct bootstate { + PyThreadState *tstate; + PyObject *func; + PyObject *args; + PyObject *kwargs; + ThreadHandle *handle; + PyEvent handle_ready; +}; + +static void +thread_bootstate_free(struct bootstate *boot, int decref) { - return PyUnicode_FromFormat("<%s object: ident=%" PY_FORMAT_THREAD_IDENT_T ">", - Py_TYPE(self)->tp_name, self->ident); + if (decref) { + Py_DECREF(boot->func); + Py_DECREF(boot->args); + Py_XDECREF(boot->kwargs); + } + ThreadHandle_decref(boot->handle); + PyMem_RawFree(boot); } -static PyObject * -ThreadHandle_get_ident(ThreadHandleObject *self, void *ignored) +static void +thread_run(void *boot_raw) { - return PyLong_FromUnsignedLongLong(self->ident); + struct bootstate *boot = (struct bootstate *) boot_raw; + PyThreadState *tstate = boot->tstate; + + // Wait until the handle is marked as running + PyEvent_Wait(&boot->handle_ready); + + // `handle` needs to be manipulated after bootstate has been freed + ThreadHandle *handle = boot->handle; + ThreadHandle_incref(handle); + + // gh-108987: If _thread.start_new_thread() is called before or while + // Python is being finalized, thread_run() can called *after*. + // _PyRuntimeState_SetFinalizing() is called. At this point, all Python + // threads must exit, except of the thread calling Py_Finalize() which + // holds the GIL and must not exit. + // + // At this stage, tstate can be a dangling pointer (point to freed memory), + // it's ok to call _PyThreadState_MustExit() with a dangling pointer. + if (_PyThreadState_MustExit(tstate)) { + // Don't call PyThreadState_Clear() nor _PyThreadState_DeleteCurrent(). + // These functions are called on tstate indirectly by Py_Finalize() + // which calls _PyInterpreterState_Clear(). + // + // Py_DECREF() cannot be called because the GIL is not held: leak + // references on purpose. Python is being finalized anyway. + thread_bootstate_free(boot, 0); + goto exit; + } + + _PyThreadState_Bind(tstate); + PyEval_AcquireThread(tstate); + _Py_atomic_add_ssize(&tstate->interp->threads.count, 1); + + PyObject *res = PyObject_Call(boot->func, boot->args, boot->kwargs); + if (res == NULL) { + if (PyErr_ExceptionMatches(PyExc_SystemExit)) + /* SystemExit is ignored silently */ + PyErr_Clear(); + else { + PyErr_FormatUnraisable( + "Exception ignored in thread started by %R", boot->func); + } + } + else { + Py_DECREF(res); + } + + thread_bootstate_free(boot, 1); + + _Py_atomic_add_ssize(&tstate->interp->threads.count, -1); + PyThreadState_Clear(tstate); + _PyThreadState_DeleteCurrent(tstate); + +exit: + // Don't need to wait for this thread anymore + remove_from_shutdown_handles(handle); + + _PyEvent_Notify(&handle->thread_is_exiting); + ThreadHandle_decref(handle); + + // bpo-44434: Don't call explicitly PyThread_exit_thread(). On Linux with + // the glibc, pthread_exit() can abort the whole process if dlopen() fails + // to open the libgcc_s.so library (ex: EMFILE error). + return; } static int -join_thread(ThreadHandleObject *handle) +force_done(ThreadHandle *handle) +{ + assert(get_thread_handle_state(handle) == THREAD_HANDLE_STARTING); + _PyEvent_Notify(&handle->thread_is_exiting); + set_thread_handle_state(handle, THREAD_HANDLE_DONE); + return 0; +} + +static int +ThreadHandle_start(ThreadHandle *self, PyObject *func, PyObject *args, + PyObject *kwargs) +{ + // Mark the handle as starting to prevent any other threads from doing so + PyMutex_Lock(&self->mutex); + if (self->state != THREAD_HANDLE_NOT_STARTED) { + PyMutex_Unlock(&self->mutex); + PyErr_SetString(ThreadError, "thread already started"); + return -1; + } + self->state = THREAD_HANDLE_STARTING; + PyMutex_Unlock(&self->mutex); + + // Do all the heavy lifting outside of the mutex. All other operations on + // the handle should fail since the handle is in the starting state. + + // gh-109795: Use PyMem_RawMalloc() instead of PyMem_Malloc(), + // because it should be possible to call thread_bootstate_free() + // without holding the GIL. + struct bootstate *boot = PyMem_RawMalloc(sizeof(struct bootstate)); + if (boot == NULL) { + PyErr_NoMemory(); + goto start_failed; + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + boot->tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_THREADING); + if (boot->tstate == NULL) { + PyMem_RawFree(boot); + if (!PyErr_Occurred()) { + PyErr_NoMemory(); + } + goto start_failed; + } + boot->func = Py_NewRef(func); + boot->args = Py_NewRef(args); + boot->kwargs = Py_XNewRef(kwargs); + boot->handle = self; + ThreadHandle_incref(self); + boot->handle_ready = (PyEvent){0}; + + PyThread_ident_t ident; + PyThread_handle_t os_handle; + if (PyThread_start_joinable_thread(thread_run, boot, &ident, &os_handle)) { + PyThreadState_Clear(boot->tstate); + PyThreadState_Delete(boot->tstate); + thread_bootstate_free(boot, 1); + PyErr_SetString(ThreadError, "can't start new thread"); + goto start_failed; + } + + // Mark the handle running + PyMutex_Lock(&self->mutex); + assert(self->state == THREAD_HANDLE_STARTING); + self->ident = ident; + self->has_os_handle = 1; + self->os_handle = os_handle; + self->state = THREAD_HANDLE_RUNNING; + PyMutex_Unlock(&self->mutex); + + // Unblock the thread + _PyEvent_Notify(&boot->handle_ready); + + return 0; + +start_failed: + _PyOnceFlag_CallOnce(&self->once, (_Py_once_fn_t *)force_done, self); + return -1; +} + +static int +join_thread(ThreadHandle *handle) { assert(get_thread_handle_state(handle) == THREAD_HANDLE_RUNNING); + PyThread_handle_t os_handle; + if (ThreadHandle_get_os_handle(handle, &os_handle)) { + int err = 0; + Py_BEGIN_ALLOW_THREADS + err = PyThread_join_thread(os_handle); + Py_END_ALLOW_THREADS + if (err) { + PyErr_SetString(ThreadError, "Failed joining thread"); + return -1; + } + } + set_thread_handle_state(handle, THREAD_HANDLE_DONE); + return 0; +} - int err; - Py_BEGIN_ALLOW_THREADS - err = PyThread_join_thread(handle->handle); - Py_END_ALLOW_THREADS - if (err) { - PyErr_SetString(ThreadError, "Failed joining thread"); +static int +check_started(ThreadHandle *self) +{ + ThreadHandleState state = get_thread_handle_state(self); + if (state < THREAD_HANDLE_RUNNING) { + PyErr_SetString(ThreadError, "thread not started"); return -1; } - set_thread_handle_state(handle, THREAD_HANDLE_JOINED); return 0; } -static PyObject * -ThreadHandle_join(ThreadHandleObject *self, void* ignored) +static int +ThreadHandle_join(ThreadHandle *self, PyTime_t timeout_ns) { - if (get_thread_handle_state(self) == THREAD_HANDLE_INVALID) { - PyErr_SetString(PyExc_ValueError, - "the handle is invalid and thus cannot be joined"); - return NULL; + if (check_started(self) < 0) { + return -1; } // We want to perform this check outside of the `_PyOnceFlag` to prevent @@ -225,45 +510,212 @@ ThreadHandle_join(ThreadHandleObject *self, void* ignored) // To work around this, we set `thread_is_exiting` immediately before // `thread_run` returns. We can be sure that we are not attempting to join // ourselves if the handle's thread is about to exit. - if (!_PyEvent_IsSet(&self->thread_is_exiting->event) && - self->ident == PyThread_get_thread_ident_ex()) { + if (!_PyEvent_IsSet(&self->thread_is_exiting) && + ThreadHandle_ident(self) == PyThread_get_thread_ident_ex()) { // PyThread_join_thread() would deadlock or error out. PyErr_SetString(ThreadError, "Cannot join current thread"); - return NULL; + return -1; + } + + // Wait until the deadline for the thread to exit. + PyTime_t deadline = timeout_ns != -1 ? _PyDeadline_Init(timeout_ns) : 0; + int detach = 1; + while (!PyEvent_WaitTimed(&self->thread_is_exiting, timeout_ns, detach)) { + if (deadline) { + // _PyDeadline_Get will return a negative value if the deadline has + // been exceeded. + timeout_ns = Py_MAX(_PyDeadline_Get(deadline), 0); + } + + if (timeout_ns) { + // Interrupted + if (Py_MakePendingCalls() < 0) { + return -1; + } + } + else { + // Timed out + return 0; + } } if (_PyOnceFlag_CallOnce(&self->once, (_Py_once_fn_t *)join_thread, self) == -1) { + return -1; + } + assert(get_thread_handle_state(self) == THREAD_HANDLE_DONE); + return 0; +} + +static int +set_done(ThreadHandle *handle) +{ + assert(get_thread_handle_state(handle) == THREAD_HANDLE_RUNNING); + if (detach_thread(handle) < 0) { + PyErr_SetString(ThreadError, "failed detaching handle"); + return -1; + } + _PyEvent_Notify(&handle->thread_is_exiting); + set_thread_handle_state(handle, THREAD_HANDLE_DONE); + return 0; +} + +static int +ThreadHandle_set_done(ThreadHandle *self) +{ + if (check_started(self) < 0) { + return -1; + } + + if (_PyOnceFlag_CallOnce(&self->once, (_Py_once_fn_t *)set_done, self) == + -1) { + return -1; + } + assert(get_thread_handle_state(self) == THREAD_HANDLE_DONE); + return 0; +} + +// A wrapper around a ThreadHandle. +typedef struct { + PyObject_HEAD + + ThreadHandle *handle; +} PyThreadHandleObject; + +static PyThreadHandleObject * +PyThreadHandleObject_new(PyTypeObject *type) +{ + ThreadHandle *handle = ThreadHandle_new(); + if (handle == NULL) { + return NULL; + } + + PyThreadHandleObject *self = + (PyThreadHandleObject *)type->tp_alloc(type, 0); + if (self == NULL) { + ThreadHandle_decref(handle); + return NULL; + } + + self->handle = handle; + + return self; +} + +static PyObject * +PyThreadHandleObject_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + return (PyObject *)PyThreadHandleObject_new(type); +} + +static int +PyThreadHandleObject_traverse(PyThreadHandleObject *self, visitproc visit, + void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + +static void +PyThreadHandleObject_dealloc(PyObject *op) +{ + PyThreadHandleObject *self = (PyThreadHandleObject*)op; + PyObject_GC_UnTrack(self); + PyTypeObject *tp = Py_TYPE(self); + ThreadHandle_decref(self->handle); + tp->tp_free(self); + Py_DECREF(tp); +} + +static PyObject * +PyThreadHandleObject_repr(PyObject *op) +{ + PyThreadHandleObject *self = (PyThreadHandleObject*)op; + PyThread_ident_t ident = ThreadHandle_ident(self->handle); + return PyUnicode_FromFormat("<%s object: ident=%" PY_FORMAT_THREAD_IDENT_T ">", + Py_TYPE(self)->tp_name, ident); +} + +static PyObject * +PyThreadHandleObject_get_ident(PyObject *op, void *Py_UNUSED(ignored)) +{ + PyThreadHandleObject *self = (PyThreadHandleObject*)op; + return PyLong_FromUnsignedLongLong(ThreadHandle_ident(self->handle)); +} + +static PyObject * +PyThreadHandleObject_join(PyObject *op, PyObject *args) +{ + PyThreadHandleObject *self = (PyThreadHandleObject*)op; + + PyObject *timeout_obj = NULL; + if (!PyArg_ParseTuple(args, "|O:join", &timeout_obj)) { + return NULL; + } + + PyTime_t timeout_ns = -1; + if (timeout_obj != NULL && timeout_obj != Py_None) { + if (_PyTime_FromSecondsObject(&timeout_ns, timeout_obj, + _PyTime_ROUND_TIMEOUT) < 0) { + return NULL; + } + } + + if (ThreadHandle_join(self->handle, timeout_ns) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +PyThreadHandleObject_is_done(PyObject *op, PyObject *Py_UNUSED(ignored)) +{ + PyThreadHandleObject *self = (PyThreadHandleObject*)op; + if (_PyEvent_IsSet(&self->handle->thread_is_exiting)) { + Py_RETURN_TRUE; + } + else { + Py_RETURN_FALSE; + } +} + +static PyObject * +PyThreadHandleObject_set_done(PyObject *op, PyObject *Py_UNUSED(ignored)) +{ + PyThreadHandleObject *self = (PyThreadHandleObject*)op; + if (ThreadHandle_set_done(self->handle) < 0) { return NULL; } - assert(get_thread_handle_state(self) == THREAD_HANDLE_JOINED); Py_RETURN_NONE; } static PyGetSetDef ThreadHandle_getsetlist[] = { - {"ident", (getter)ThreadHandle_get_ident, NULL, NULL}, + {"ident", PyThreadHandleObject_get_ident, NULL, NULL}, {0}, }; -static PyMethodDef ThreadHandle_methods[] = -{ - {"join", (PyCFunction)ThreadHandle_join, METH_NOARGS}, +static PyMethodDef ThreadHandle_methods[] = { + {"join", PyThreadHandleObject_join, METH_VARARGS, NULL}, + {"_set_done", PyThreadHandleObject_set_done, METH_NOARGS, NULL}, + {"is_done", PyThreadHandleObject_is_done, METH_NOARGS, NULL}, {0, 0} }; static PyType_Slot ThreadHandle_Type_slots[] = { - {Py_tp_dealloc, (destructor)ThreadHandle_dealloc}, - {Py_tp_repr, (reprfunc)ThreadHandle_repr}, + {Py_tp_dealloc, PyThreadHandleObject_dealloc}, + {Py_tp_repr, PyThreadHandleObject_repr}, {Py_tp_getset, ThreadHandle_getsetlist}, + {Py_tp_traverse, PyThreadHandleObject_traverse}, {Py_tp_methods, ThreadHandle_methods}, + {Py_tp_new, PyThreadHandleObject_tp_new}, {0, 0} }; static PyType_Spec ThreadHandle_Type_spec = { "_thread._ThreadHandle", - sizeof(ThreadHandleObject), + sizeof(PyThreadHandleObject), 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC, ThreadHandle_Type_slots, }; @@ -271,41 +723,27 @@ static PyType_Spec ThreadHandle_Type_spec = { typedef struct { PyObject_HEAD - PyThread_type_lock lock_lock; - PyObject *in_weakreflist; - char locked; /* for sanity checking */ + PyMutex lock; } lockobject; static int -lock_traverse(lockobject *self, visitproc visit, void *arg) +lock_traverse(PyObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); return 0; } static void -lock_dealloc(lockobject *self) +lock_dealloc(PyObject *op) { + lockobject *self = (lockobject*)op; PyObject_GC_UnTrack(self); - if (self->in_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *) self); - } - if (self->lock_lock != NULL) { - /* Unlock the lock so it's safe to free it */ - if (self->locked) - PyThread_release_lock(self->lock_lock); - PyThread_free_lock(self->lock_lock); - } + PyObject_ClearWeakRefs((PyObject *) self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free((PyObject*)self); Py_DECREF(tp); } -static inline PyLockStatus -acquire_timed(PyThread_type_lock lock, PyTime_t timeout) -{ - return PyThread_acquire_lock_timed_with_retries(lock, timeout); -} static int lock_acquire_parse_args(PyObject *args, PyObject *kwds, @@ -354,25 +792,27 @@ lock_acquire_parse_args(PyObject *args, PyObject *kwds, } static PyObject * -lock_PyThread_acquire_lock(lockobject *self, PyObject *args, PyObject *kwds) +lock_PyThread_acquire_lock(PyObject *op, PyObject *args, PyObject *kwds) { + lockobject *self = (lockobject*)op; + PyTime_t timeout; - if (lock_acquire_parse_args(args, kwds, &timeout) < 0) + if (lock_acquire_parse_args(args, kwds, &timeout) < 0) { return NULL; + } - PyLockStatus r = acquire_timed(self->lock_lock, timeout); + PyLockStatus r = _PyMutex_LockTimed(&self->lock, timeout, + _PY_LOCK_HANDLE_SIGNALS | _PY_LOCK_DETACH); if (r == PY_LOCK_INTR) { return NULL; } - if (r == PY_LOCK_ACQUIRED) - self->locked = 1; return PyBool_FromLong(r == PY_LOCK_ACQUIRED); } PyDoc_STRVAR(acquire_doc, -"acquire(blocking=True, timeout=-1) -> bool\n\ -(acquire_lock() is an obsolete synonym)\n\ +"acquire($self, /, blocking=True, timeout=-1)\n\ +--\n\ \n\ Lock the lock. Without argument, this blocks if the lock is already\n\ locked (even by the same thread), waiting for another thread to release\n\ @@ -381,58 +821,84 @@ With an argument, this will only block if the argument is true,\n\ and the return value reflects whether the lock is acquired.\n\ The blocking operation is interruptible."); +PyDoc_STRVAR(acquire_lock_doc, +"acquire_lock($self, /, blocking=True, timeout=-1)\n\ +--\n\ +\n\ +An obsolete synonym of acquire()."); + +PyDoc_STRVAR(enter_doc, +"__enter__($self, /)\n\ +--\n\ +\n\ +Lock the lock."); + static PyObject * -lock_PyThread_release_lock(lockobject *self, PyObject *Py_UNUSED(ignored)) +lock_PyThread_release_lock(PyObject *op, PyObject *Py_UNUSED(ignored)) { + lockobject *self = (lockobject*)op; /* Sanity check: the lock must be locked */ - if (!self->locked) { + if (_PyMutex_TryUnlock(&self->lock) < 0) { PyErr_SetString(ThreadError, "release unlocked lock"); return NULL; } - PyThread_release_lock(self->lock_lock); - self->locked = 0; Py_RETURN_NONE; } PyDoc_STRVAR(release_doc, -"release()\n\ -(release_lock() is an obsolete synonym)\n\ +"release($self, /)\n\ +--\n\ \n\ Release the lock, allowing another thread that is blocked waiting for\n\ the lock to acquire the lock. The lock must be in the locked state,\n\ but it needn't be locked by the same thread that unlocks it."); +PyDoc_STRVAR(release_lock_doc, +"release_lock($self, /)\n\ +--\n\ +\n\ +An obsolete synonym of release()."); + +PyDoc_STRVAR(lock_exit_doc, +"__exit__($self, /, *exc_info)\n\ +--\n\ +\n\ +Release the lock."); + static PyObject * -lock_locked_lock(lockobject *self, PyObject *Py_UNUSED(ignored)) +lock_locked_lock(PyObject *op, PyObject *Py_UNUSED(ignored)) { - return PyBool_FromLong((long)self->locked); + lockobject *self = (lockobject*)op; + return PyBool_FromLong(PyMutex_IsLocked(&self->lock)); } PyDoc_STRVAR(locked_doc, -"locked() -> bool\n\ -(locked_lock() is an obsolete synonym)\n\ +"locked($self, /)\n\ +--\n\ \n\ Return whether the lock is in the locked state."); +PyDoc_STRVAR(locked_lock_doc, +"locked_lock($self, /)\n\ +--\n\ +\n\ +An obsolete synonym of locked()."); + static PyObject * -lock_repr(lockobject *self) +lock_repr(PyObject *op) { + lockobject *self = (lockobject*)op; return PyUnicode_FromFormat("<%s %s object at %p>", - self->locked ? "locked" : "unlocked", Py_TYPE(self)->tp_name, self); + PyMutex_IsLocked(&self->lock) ? "locked" : "unlocked", Py_TYPE(self)->tp_name, self); } #ifdef HAVE_FORK static PyObject * -lock__at_fork_reinit(lockobject *self, PyObject *Py_UNUSED(args)) +lock__at_fork_reinit(PyObject *op, PyObject *Py_UNUSED(args)) { - if (_PyThread_at_fork_reinit(&self->lock_lock) < 0) { - PyErr_SetString(ThreadError, "failed to reinitialize lock at fork"); - return NULL; - } - - self->locked = 0; - + lockobject *self = (lockobject *)op; + _PyMutex_at_fork_reinit(&self->lock); Py_RETURN_NONE; } #endif /* HAVE_FORK */ @@ -461,30 +927,33 @@ lock_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) static PyMethodDef lock_methods[] = { {"acquire_lock", _PyCFunction_CAST(lock_PyThread_acquire_lock), - METH_VARARGS | METH_KEYWORDS, acquire_doc}, + METH_VARARGS | METH_KEYWORDS, acquire_lock_doc}, {"acquire", _PyCFunction_CAST(lock_PyThread_acquire_lock), METH_VARARGS | METH_KEYWORDS, acquire_doc}, - {"release_lock", (PyCFunction)lock_PyThread_release_lock, - METH_NOARGS, release_doc}, - {"release", (PyCFunction)lock_PyThread_release_lock, + {"release_lock", lock_PyThread_release_lock, + METH_NOARGS, release_lock_doc}, + {"release", lock_PyThread_release_lock, METH_NOARGS, release_doc}, - {"locked_lock", (PyCFunction)lock_locked_lock, - METH_NOARGS, locked_doc}, - {"locked", (PyCFunction)lock_locked_lock, + {"locked_lock", lock_locked_lock, + METH_NOARGS, locked_lock_doc}, + {"locked", lock_locked_lock, METH_NOARGS, locked_doc}, {"__enter__", _PyCFunction_CAST(lock_PyThread_acquire_lock), - METH_VARARGS | METH_KEYWORDS, acquire_doc}, - {"__exit__", (PyCFunction)lock_PyThread_release_lock, - METH_VARARGS, release_doc}, + METH_VARARGS | METH_KEYWORDS, enter_doc}, + {"__exit__", lock_PyThread_release_lock, + METH_VARARGS, lock_exit_doc}, #ifdef HAVE_FORK - {"_at_fork_reinit", (PyCFunction)lock__at_fork_reinit, + {"_at_fork_reinit", lock__at_fork_reinit, METH_NOARGS, NULL}, #endif {NULL, NULL} /* sentinel */ }; PyDoc_STRVAR(lock_doc, -"A lock object is a synchronization primitive. To create a lock,\n\ +"lock()\n\ +--\n\ +\n\ +A lock object is a synchronization primitive. To create a lock,\n\ call threading.Lock(). Methods are:\n\ \n\ acquire() -- lock the lock, possibly blocking until it can be obtained\n\ @@ -495,18 +964,12 @@ A lock is not owned by the thread that locked it; another thread may\n\ unlock it. A thread attempting to lock a lock that it has already locked\n\ will block until another thread unlocks it. Deadlocks may ensue."); -static PyMemberDef lock_type_members[] = { - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(lockobject, in_weakreflist), Py_READONLY}, - {NULL}, -}; - static PyType_Slot lock_type_slots[] = { - {Py_tp_dealloc, (destructor)lock_dealloc}, - {Py_tp_repr, (reprfunc)lock_repr}, + {Py_tp_dealloc, lock_dealloc}, + {Py_tp_repr, lock_repr}, {Py_tp_doc, (void *)lock_doc}, {Py_tp_methods, lock_methods}, {Py_tp_traverse, lock_traverse}, - {Py_tp_members, lock_type_members}, {Py_tp_new, lock_new}, {0, 0} }; @@ -515,7 +978,7 @@ static PyType_Spec lock_type_spec = { .name = "_thread.lock", .basicsize = sizeof(lockobject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), .slots = lock_type_slots, }; @@ -523,10 +986,7 @@ static PyType_Spec lock_type_spec = { typedef struct { PyObject_HEAD - PyThread_type_lock rlock_lock; - PyThread_ident_t rlock_owner; - unsigned long rlock_count; - PyObject *in_weakreflist; + _PyRecursiveMutex lock; } rlockobject; static int @@ -538,61 +998,30 @@ rlock_traverse(rlockobject *self, visitproc visit, void *arg) static void -rlock_dealloc(rlockobject *self) +rlock_dealloc(PyObject *op) { + rlockobject *self = (rlockobject*)op; PyObject_GC_UnTrack(self); - if (self->in_weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) self); - /* self->rlock_lock can be NULL if PyThread_allocate_lock() failed - in rlock_new() */ - if (self->rlock_lock != NULL) { - /* Unlock the lock so it's safe to free it */ - if (self->rlock_count > 0) - PyThread_release_lock(self->rlock_lock); - - PyThread_free_lock(self->rlock_lock); - } + PyObject_ClearWeakRefs((PyObject *) self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); Py_DECREF(tp); } -static bool -rlock_is_owned_by(rlockobject *self, PyThread_ident_t tid) -{ - PyThread_ident_t owner_tid = - _Py_atomic_load_ullong_relaxed(&self->rlock_owner); - return owner_tid == tid && self->rlock_count > 0; -} static PyObject * -rlock_acquire(rlockobject *self, PyObject *args, PyObject *kwds) +rlock_acquire(PyObject *op, PyObject *args, PyObject *kwds) { + rlockobject *self = (rlockobject*)op; PyTime_t timeout; - PyThread_ident_t tid; - PyLockStatus r = PY_LOCK_ACQUIRED; - if (lock_acquire_parse_args(args, kwds, &timeout) < 0) + if (lock_acquire_parse_args(args, kwds, &timeout) < 0) { return NULL; - - tid = PyThread_get_thread_ident_ex(); - if (rlock_is_owned_by(self, tid)) { - unsigned long count = self->rlock_count + 1; - if (count <= self->rlock_count) { - PyErr_SetString(PyExc_OverflowError, - "Internal lock count overflowed"); - return NULL; - } - self->rlock_count = count; - Py_RETURN_TRUE; - } - r = acquire_timed(self->rlock_lock, timeout); - if (r == PY_LOCK_ACQUIRED) { - assert(self->rlock_count == 0); - _Py_atomic_store_ullong_relaxed(&self->rlock_owner, tid); - self->rlock_count = 1; } - else if (r == PY_LOCK_INTR) { + + PyLockStatus r = _PyRecursiveMutex_LockTimed(&self->lock, timeout, + _PY_LOCK_HANDLE_SIGNALS | _PY_LOCK_DETACH); + if (r == PY_LOCK_INTR) { return NULL; } @@ -600,7 +1029,8 @@ rlock_acquire(rlockobject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(rlock_acquire_doc, -"acquire(blocking=True) -> bool\n\ +"acquire($self, /, blocking=True, timeout=-1)\n\ +--\n\ \n\ Lock the lock. `blocking` indicates whether we should wait\n\ for the lock to be available or not. If `blocking` is False\n\ @@ -615,25 +1045,28 @@ Precisely, if the current thread already holds the lock, its\n\ internal counter is simply incremented. If nobody holds the lock,\n\ the lock is taken and its internal counter initialized to 1."); +PyDoc_STRVAR(rlock_enter_doc, +"__enter__($self, /)\n\ +--\n\ +\n\ +Lock the lock."); + static PyObject * -rlock_release(rlockobject *self, PyObject *Py_UNUSED(ignored)) +rlock_release(PyObject *op, PyObject *Py_UNUSED(ignored)) { - PyThread_ident_t tid = PyThread_get_thread_ident_ex(); + rlockobject *self = (rlockobject*)op; - if (!rlock_is_owned_by(self, tid)) { + if (_PyRecursiveMutex_TryUnlock(&self->lock) < 0) { PyErr_SetString(PyExc_RuntimeError, "cannot release un-acquired lock"); return NULL; } - if (--self->rlock_count == 0) { - _Py_atomic_store_ullong_relaxed(&self->rlock_owner, 0); - PyThread_release_lock(self->rlock_lock); - } Py_RETURN_NONE; } PyDoc_STRVAR(rlock_release_doc, -"release()\n\ +"release($self, /)\n\ +--\n\ \n\ Release the lock, allowing another thread that is blocked waiting for\n\ the lock to acquire the lock. The lock must be in the locked state,\n\ @@ -644,89 +1077,86 @@ Do note that if the lock was acquire()d several times in a row by the\n\ current thread, release() needs to be called as many times for the lock\n\ to be available for other threads."); +PyDoc_STRVAR(rlock_exit_doc, +"__exit__($self, /, *exc_info)\n\ +--\n\ +\n\ +Release the lock."); + static PyObject * -rlock_acquire_restore(rlockobject *self, PyObject *args) +rlock_acquire_restore(PyObject *op, PyObject *args) { + rlockobject *self = (rlockobject*)op; PyThread_ident_t owner; - unsigned long count; - int r = 1; + Py_ssize_t count; - if (!PyArg_ParseTuple(args, "(k" Py_PARSE_THREAD_IDENT_T "):_acquire_restore", + if (!PyArg_ParseTuple(args, "(n" Py_PARSE_THREAD_IDENT_T "):_acquire_restore", &count, &owner)) return NULL; - if (!PyThread_acquire_lock(self->rlock_lock, 0)) { - Py_BEGIN_ALLOW_THREADS - r = PyThread_acquire_lock(self->rlock_lock, 1); - Py_END_ALLOW_THREADS - } - if (!r) { - PyErr_SetString(ThreadError, "couldn't acquire lock"); - return NULL; - } - assert(self->rlock_count == 0); - _Py_atomic_store_ullong_relaxed(&self->rlock_owner, owner); - self->rlock_count = count; + _PyRecursiveMutex_Lock(&self->lock); + _Py_atomic_store_ullong_relaxed(&self->lock.thread, owner); + self->lock.level = (size_t)count - 1; Py_RETURN_NONE; } PyDoc_STRVAR(rlock_acquire_restore_doc, -"_acquire_restore(state) -> None\n\ +"_acquire_restore($self, state, /)\n\ +--\n\ \n\ For internal use by `threading.Condition`."); static PyObject * -rlock_release_save(rlockobject *self, PyObject *Py_UNUSED(ignored)) +rlock_release_save(PyObject *op, PyObject *Py_UNUSED(ignored)) { - PyThread_ident_t owner; - unsigned long count; + rlockobject *self = (rlockobject*)op; - if (self->rlock_count == 0) { + if (!_PyRecursiveMutex_IsLockedByCurrentThread(&self->lock)) { PyErr_SetString(PyExc_RuntimeError, "cannot release un-acquired lock"); return NULL; } - owner = self->rlock_owner; - count = self->rlock_count; - self->rlock_count = 0; - _Py_atomic_store_ullong_relaxed(&self->rlock_owner, 0); - PyThread_release_lock(self->rlock_lock); - return Py_BuildValue("k" Py_PARSE_THREAD_IDENT_T, count, owner); + PyThread_ident_t owner = self->lock.thread; + Py_ssize_t count = self->lock.level + 1; + self->lock.level = 0; // ensure the unlock releases the lock + _PyRecursiveMutex_Unlock(&self->lock); + return Py_BuildValue("n" Py_PARSE_THREAD_IDENT_T, count, owner); } PyDoc_STRVAR(rlock_release_save_doc, -"_release_save() -> tuple\n\ +"_release_save($self, /)\n\ +--\n\ \n\ For internal use by `threading.Condition`."); static PyObject * -rlock_recursion_count(rlockobject *self, PyObject *Py_UNUSED(ignored)) +rlock_recursion_count(PyObject *op, PyObject *Py_UNUSED(ignored)) { - PyThread_ident_t tid = PyThread_get_thread_ident_ex(); - PyThread_ident_t owner = - _Py_atomic_load_ullong_relaxed(&self->rlock_owner); - return PyLong_FromUnsignedLong(owner == tid ? self->rlock_count : 0UL); + rlockobject *self = (rlockobject*)op; + if (_PyRecursiveMutex_IsLockedByCurrentThread(&self->lock)) { + return PyLong_FromSize_t(self->lock.level + 1); + } + return PyLong_FromLong(0); } PyDoc_STRVAR(rlock_recursion_count_doc, -"_recursion_count() -> int\n\ +"_recursion_count($self, /)\n\ +--\n\ \n\ For internal use by reentrancy checks."); static PyObject * -rlock_is_owned(rlockobject *self, PyObject *Py_UNUSED(ignored)) +rlock_is_owned(PyObject *op, PyObject *Py_UNUSED(ignored)) { - PyThread_ident_t tid = PyThread_get_thread_ident_ex(); - - if (rlock_is_owned_by(self, tid)) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + rlockobject *self = (rlockobject*)op; + long owned = _PyRecursiveMutex_IsLockedByCurrentThread(&self->lock); + return PyBool_FromLong(owned); } PyDoc_STRVAR(rlock_is_owned_doc, -"_is_owned() -> bool\n\ +"_is_owned($self, /)\n\ +--\n\ \n\ For internal use by `threading.Condition`."); @@ -737,29 +1167,21 @@ rlock_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (self == NULL) { return NULL; } - self->in_weakreflist = NULL; - self->rlock_owner = 0; - self->rlock_count = 0; - - self->rlock_lock = PyThread_allocate_lock(); - if (self->rlock_lock == NULL) { - Py_DECREF(self); - PyErr_SetString(ThreadError, "can't allocate lock"); - return NULL; - } + self->lock = (_PyRecursiveMutex){0}; return (PyObject *) self; } static PyObject * -rlock_repr(rlockobject *self) +rlock_repr(PyObject *op) { - PyThread_ident_t owner = - _Py_atomic_load_ullong_relaxed(&self->rlock_owner); + rlockobject *self = (rlockobject*)op; + PyThread_ident_t owner = self->lock.thread; + size_t count = self->lock.level + 1; return PyUnicode_FromFormat( - "<%s %s object owner=%" PY_FORMAT_THREAD_IDENT_T " count=%lu at %p>", - self->rlock_count ? "locked" : "unlocked", + "<%s %s object owner=%" PY_FORMAT_THREAD_IDENT_T " count=%zu at %p>", + owner ? "locked" : "unlocked", Py_TYPE(self)->tp_name, owner, - self->rlock_count, self); + count, self); } @@ -767,14 +1189,7 @@ rlock_repr(rlockobject *self) static PyObject * rlock__at_fork_reinit(rlockobject *self, PyObject *Py_UNUSED(args)) { - if (_PyThread_at_fork_reinit(&self->rlock_lock) < 0) { - PyErr_SetString(ThreadError, "failed to reinitialize lock at fork"); - return NULL; - } - - self->rlock_owner = 0; - self->rlock_count = 0; - + self->lock = (_PyRecursiveMutex){0}; Py_RETURN_NONE; } #endif /* HAVE_FORK */ @@ -783,20 +1198,20 @@ rlock__at_fork_reinit(rlockobject *self, PyObject *Py_UNUSED(args)) static PyMethodDef rlock_methods[] = { {"acquire", _PyCFunction_CAST(rlock_acquire), METH_VARARGS | METH_KEYWORDS, rlock_acquire_doc}, - {"release", (PyCFunction)rlock_release, + {"release", rlock_release, METH_NOARGS, rlock_release_doc}, - {"_is_owned", (PyCFunction)rlock_is_owned, + {"_is_owned", rlock_is_owned, METH_NOARGS, rlock_is_owned_doc}, - {"_acquire_restore", (PyCFunction)rlock_acquire_restore, + {"_acquire_restore", rlock_acquire_restore, METH_VARARGS, rlock_acquire_restore_doc}, - {"_release_save", (PyCFunction)rlock_release_save, + {"_release_save", rlock_release_save, METH_NOARGS, rlock_release_save_doc}, - {"_recursion_count", (PyCFunction)rlock_recursion_count, + {"_recursion_count", rlock_recursion_count, METH_NOARGS, rlock_recursion_count_doc}, {"__enter__", _PyCFunction_CAST(rlock_acquire), - METH_VARARGS | METH_KEYWORDS, rlock_acquire_doc}, - {"__exit__", (PyCFunction)rlock_release, - METH_VARARGS, rlock_release_doc}, + METH_VARARGS | METH_KEYWORDS, rlock_enter_doc}, + {"__exit__", rlock_release, + METH_VARARGS, rlock_exit_doc}, #ifdef HAVE_FORK {"_at_fork_reinit", (PyCFunction)rlock__at_fork_reinit, METH_NOARGS, NULL}, @@ -805,18 +1220,12 @@ static PyMethodDef rlock_methods[] = { }; -static PyMemberDef rlock_type_members[] = { - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(rlockobject, in_weakreflist), Py_READONLY}, - {NULL}, -}; - static PyType_Slot rlock_type_slots[] = { - {Py_tp_dealloc, (destructor)rlock_dealloc}, - {Py_tp_repr, (reprfunc)rlock_repr}, + {Py_tp_dealloc, rlock_dealloc}, + {Py_tp_repr, rlock_repr}, {Py_tp_methods, rlock_methods}, {Py_tp_alloc, PyType_GenericAlloc}, {Py_tp_new, rlock_new}, - {Py_tp_members, rlock_type_members}, {Py_tp_traverse, rlock_traverse}, {0, 0}, }; @@ -825,7 +1234,7 @@ static PyType_Spec rlock_type_spec = { .name = "_thread.RLock", .basicsize = sizeof(rlockobject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), .slots = rlock_type_slots, }; @@ -839,16 +1248,7 @@ newlockobject(PyObject *module) if (self == NULL) { return NULL; } - - self->lock_lock = PyThread_allocate_lock(); - self->locked = 0; - self->in_weakreflist = NULL; - - if (self->lock_lock == NULL) { - Py_DECREF(self); - PyErr_SetString(ThreadError, "can't allocate lock"); - return NULL; - } + self->lock = (PyMutex){0}; return self; } @@ -869,39 +1269,51 @@ newlockobject(PyObject *module) Our implementation uses small "localdummy" objects in order to break the reference chain. These trivial objects are hashable (using the default scheme of identity hashing) and weakrefable. - Each thread-state holds a separate localdummy for each local object - (as a /strong reference/), - and each thread-local object holds a dict mapping /weak references/ - of localdummies to local dicts. + + Each thread-state holds two separate localdummy objects: + + - `threading_local_key` is used as a key to retrieve the locals dictionary + for the thread in any `threading.local` object. + - `threading_local_sentinel` is used to signal when a thread is being + destroyed. Consequently, the associated thread-state must hold the only + reference. + + Each `threading.local` object contains a dict mapping localdummy keys to + locals dicts and a set containing weak references to localdummy + sentinels. Each sentinel weak reference has a callback that removes itself + and the locals dict for the key from the `threading.local` object when + called. Therefore: - - only the thread-state dict holds a strong reference to the dummies - - only the thread-local object holds a strong reference to the local dicts - - only outside objects (application- or library-level) hold strong - references to the thread-local objects - - as soon as a thread-state dict is destroyed, the weakref callbacks of all - dummies attached to that thread are called, and destroy the corresponding - local dicts from thread-local objects - - as soon as a thread-local object is destroyed, its local dicts are - destroyed and its dummies are manually removed from all thread states - - the GC can do its work correctly when a thread-local object is dangling, - without any interference from the thread-state dicts - - As an additional optimization, each localdummy holds a borrowed reference - to the corresponding localdict. This borrowed reference is only used - by the thread-local object which has created the localdummy, which should - guarantee that the localdict still exists when accessed. + - The thread-state only holds strong references to localdummy objects, which + cannot participate in cycles. + - Only outside objects (application- or library-level) hold strong + references to the thread-local objects. + - As soon as thread-state's sentinel dummy is destroyed the callbacks for + all weakrefs attached to the sentinel are called, and destroy the + corresponding local dicts from thread-local objects. + - As soon as a thread-local object is destroyed, its local dicts are + destroyed. + - The GC can do its work correctly when a thread-local object is dangling, + without any interference from the thread-state dicts. + + This dual key arrangement is necessary to ensure that `threading.local` + values can be retrieved from finalizers. If we were to only keep a mapping + of localdummy weakrefs to locals dicts it's possible that the weakrefs would + be cleared before finalizers were called (GC currently clears weakrefs that + are garbage before invoking finalizers), causing lookups in finalizers to + fail. */ typedef struct { PyObject_HEAD - PyObject *localdict; /* Borrowed reference! */ PyObject *weakreflist; /* List of weak references to self */ } localdummyobject; static void -localdummy_dealloc(localdummyobject *self) +localdummy_dealloc(PyObject *op) { + localdummyobject *self = (localdummyobject*)op; if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) self); PyTypeObject *tp = Py_TYPE(self); @@ -915,7 +1327,7 @@ static PyMemberDef local_dummy_type_members[] = { }; static PyType_Slot local_dummy_type_slots[] = { - {Py_tp_dealloc, (destructor)localdummy_dealloc}, + {Py_tp_dealloc, localdummy_dealloc}, {Py_tp_doc, "Thread-local dummy"}, {Py_tp_members, local_dummy_type_members}, {0, 0} @@ -932,80 +1344,60 @@ static PyType_Spec local_dummy_type_spec = { typedef struct { PyObject_HEAD - PyObject *key; PyObject *args; PyObject *kw; PyObject *weakreflist; /* List of weak references to self */ - /* A {localdummy weakref -> localdict} dict */ - PyObject *dummies; - /* The callback for weakrefs to localdummies */ - PyObject *wr_callback; + /* A {localdummy -> localdict} dict */ + PyObject *localdicts; + /* A set of weakrefs to thread sentinels localdummies*/ + PyObject *thread_watchdogs; } localobject; /* Forward declaration */ -static PyObject *_ldict(localobject *self, thread_module_state *state); -static PyObject *_localdummy_destroyed(PyObject *meth_self, PyObject *dummyweakref); +static int create_localsdict(localobject *self, thread_module_state *state, + PyObject **localsdict, PyObject **sentinel_wr); +static PyObject *clear_locals(PyObject *meth_self, PyObject *dummyweakref); -/* Create and register the dummy for the current thread. - Returns a borrowed reference of the corresponding local dict */ +/* Create a weakref to the sentinel localdummy for the current thread */ static PyObject * -_local_create_dummy(localobject *self, thread_module_state *state) +create_sentinel_wr(localobject *self) { - PyObject *ldict = NULL, *wr = NULL; - localdummyobject *dummy = NULL; - PyTypeObject *type = state->local_dummy_type; + static PyMethodDef wr_callback_def = { + "clear_locals", (PyCFunction) clear_locals, METH_O + }; - PyObject *tdict = PyThreadState_GetDict(); - if (tdict == NULL) { - PyErr_SetString(PyExc_SystemError, - "Couldn't get thread-state dictionary"); - goto err; - } + PyThreadState *tstate = PyThreadState_Get(); - ldict = PyDict_New(); - if (ldict == NULL) { - goto err; - } - dummy = (localdummyobject *) type->tp_alloc(type, 0); - if (dummy == NULL) { - goto err; - } - dummy->localdict = ldict; - wr = PyWeakref_NewRef((PyObject *) dummy, self->wr_callback); - if (wr == NULL) { - goto err; + /* We use a weak reference to self in the callback closure + in order to avoid spurious reference cycles */ + PyObject *self_wr = PyWeakref_NewRef((PyObject *) self, NULL); + if (self_wr == NULL) { + return NULL; } - /* As a side-effect, this will cache the weakref's hash before the - dummy gets deleted */ - int r = PyDict_SetItem(self->dummies, wr, ldict); - if (r < 0) { - goto err; + PyObject *args = PyTuple_New(2); + if (args == NULL) { + Py_DECREF(self_wr); + return NULL; } - Py_CLEAR(wr); - r = PyDict_SetItem(tdict, self->key, (PyObject *) dummy); - if (r < 0) { - goto err; + PyTuple_SET_ITEM(args, 0, self_wr); + PyTuple_SET_ITEM(args, 1, Py_NewRef(tstate->threading_local_key)); + + PyObject *cb = PyCFunction_New(&wr_callback_def, args); + Py_DECREF(args); + if (cb == NULL) { + return NULL; } - Py_CLEAR(dummy); - Py_DECREF(ldict); - return ldict; + PyObject *wr = PyWeakref_NewRef(tstate->threading_local_sentinel, cb); + Py_DECREF(cb); -err: - Py_XDECREF(ldict); - Py_XDECREF(wr); - Py_XDECREF(dummy); - return NULL; + return wr; } static PyObject * local_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - static PyMethodDef wr_callback_def = { - "_localdummy_destroyed", (PyCFunction) _localdummy_destroyed, METH_O - }; - if (type->tp_init == PyBaseObject_Type.tp_init) { int rc = 0; if (args != NULL) @@ -1030,32 +1422,31 @@ local_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; } + // gh-128691: Use deferred reference counting for thread-locals to avoid + // contention on the shared object. + _PyObject_SetDeferredRefcount((PyObject *)self); + self->args = Py_XNewRef(args); self->kw = Py_XNewRef(kw); - self->key = PyUnicode_FromFormat("thread.local.%p", self); - if (self->key == NULL) { - goto err; - } - self->dummies = PyDict_New(); - if (self->dummies == NULL) { + self->localdicts = PyDict_New(); + if (self->localdicts == NULL) { goto err; } - /* We use a weak reference to self in the callback closure - in order to avoid spurious reference cycles */ - PyObject *wr = PyWeakref_NewRef((PyObject *) self, NULL); - if (wr == NULL) { + self->thread_watchdogs = PySet_New(NULL); + if (self->thread_watchdogs == NULL) { goto err; } - self->wr_callback = PyCFunction_NewEx(&wr_callback_def, wr, NULL); - Py_DECREF(wr); - if (self->wr_callback == NULL) { - goto err; - } - if (_local_create_dummy(self, state) == NULL) { + + PyObject *localsdict = NULL; + PyObject *sentinel_wr = NULL; + if (create_localsdict(self, state, &localsdict, &sentinel_wr) < 0) { goto err; } + Py_DECREF(localsdict); + Py_DECREF(sentinel_wr); + return (PyObject *)self; err: @@ -1069,7 +1460,8 @@ local_traverse(localobject *self, visitproc visit, void *arg) Py_VISIT(Py_TYPE(self)); Py_VISIT(self->args); Py_VISIT(self->kw); - Py_VISIT(self->dummies); + Py_VISIT(self->localdicts); + Py_VISIT(self->thread_watchdogs); return 0; } @@ -1078,27 +1470,8 @@ local_clear(localobject *self) { Py_CLEAR(self->args); Py_CLEAR(self->kw); - Py_CLEAR(self->dummies); - Py_CLEAR(self->wr_callback); - /* Remove all strong references to dummies from the thread states */ - if (self->key) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyRuntimeState *runtime = &_PyRuntime; - HEAD_LOCK(runtime); - PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); - HEAD_UNLOCK(runtime); - while (tstate) { - if (tstate->dict) { - if (PyDict_Pop(tstate->dict, self->key, NULL) < 0) { - // Silently ignore error - PyErr_Clear(); - } - } - HEAD_LOCK(runtime); - tstate = PyThreadState_Next(tstate); - HEAD_UNLOCK(runtime); - } - } + Py_CLEAR(self->localdicts); + Py_CLEAR(self->thread_watchdogs); return 0; } @@ -1114,48 +1487,142 @@ local_dealloc(localobject *self) PyObject_GC_UnTrack(self); local_clear(self); - Py_XDECREF(self->key); PyTypeObject *tp = Py_TYPE(self); tp->tp_free((PyObject*)self); Py_DECREF(tp); } -/* Returns a borrowed reference to the local dict, creating it if necessary */ +/* Create the TLS key and sentinel if they don't exist */ +static int +create_localdummies(thread_module_state *state) +{ + PyThreadState *tstate = _PyThreadState_GET(); + + if (tstate->threading_local_key != NULL) { + return 0; + } + + PyTypeObject *ld_type = state->local_dummy_type; + tstate->threading_local_key = ld_type->tp_alloc(ld_type, 0); + if (tstate->threading_local_key == NULL) { + return -1; + } + + tstate->threading_local_sentinel = ld_type->tp_alloc(ld_type, 0); + if (tstate->threading_local_sentinel == NULL) { + Py_CLEAR(tstate->threading_local_key); + return -1; + } + + return 0; +} + +/* Insert a localsdict and sentinel weakref for the current thread, placing + strong references in localsdict and sentinel_wr, respectively. +*/ +static int +create_localsdict(localobject *self, thread_module_state *state, + PyObject **localsdict, PyObject **sentinel_wr) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *ldict = NULL; + PyObject *wr = NULL; + + if (create_localdummies(state) < 0) { + goto err; + } + + /* Create and insert the locals dict and sentinel weakref */ + ldict = PyDict_New(); + if (ldict == NULL) { + goto err; + } + + if (PyDict_SetItem(self->localdicts, tstate->threading_local_key, ldict) < + 0) { + goto err; + } + + wr = create_sentinel_wr(self); + if (wr == NULL) { + PyObject *exc = PyErr_GetRaisedException(); + if (PyDict_DelItem(self->localdicts, tstate->threading_local_key) < + 0) { + PyErr_WriteUnraisable((PyObject *)self); + } + PyErr_SetRaisedException(exc); + goto err; + } + + if (PySet_Add(self->thread_watchdogs, wr) < 0) { + PyObject *exc = PyErr_GetRaisedException(); + if (PyDict_DelItem(self->localdicts, tstate->threading_local_key) < + 0) { + PyErr_WriteUnraisable((PyObject *)self); + } + PyErr_SetRaisedException(exc); + goto err; + } + + *localsdict = ldict; + *sentinel_wr = wr; + return 0; + +err: + Py_XDECREF(ldict); + Py_XDECREF(wr); + return -1; +} + +/* Return a strong reference to the locals dict for the current thread, + creating it if necessary. +*/ static PyObject * _ldict(localobject *self, thread_module_state *state) { - PyObject *tdict = PyThreadState_GetDict(); - if (tdict == NULL) { - PyErr_SetString(PyExc_SystemError, - "Couldn't get thread-state dictionary"); + if (create_localdummies(state) < 0) { return NULL; } + /* Check if a localsdict already exists */ PyObject *ldict; - PyObject *dummy = PyDict_GetItemWithError(tdict, self->key); - if (dummy == NULL) { - if (PyErr_Occurred()) { - return NULL; - } - ldict = _local_create_dummy(self, state); - if (ldict == NULL) - return NULL; + PyThreadState *tstate = _PyThreadState_GET(); + if (PyDict_GetItemRef(self->localdicts, tstate->threading_local_key, + &ldict) < 0) { + return NULL; + } + if (ldict != NULL) { + return ldict; + } - if (Py_TYPE(self)->tp_init != PyBaseObject_Type.tp_init && - Py_TYPE(self)->tp_init((PyObject*)self, - self->args, self->kw) < 0) { - /* we need to get rid of ldict from thread so - we create a new one the next time we do an attr - access */ - PyDict_DelItem(tdict, self->key); - return NULL; - } + /* threading.local hasn't been instantiated for this thread */ + PyObject *wr; + if (create_localsdict(self, state, &ldict, &wr) < 0) { + return NULL; } - else { - assert(Py_IS_TYPE(dummy, state->local_dummy_type)); - ldict = ((localdummyobject *) dummy)->localdict; + + /* run __init__ if we're a subtype of `threading.local` */ + if (Py_TYPE(self)->tp_init != PyBaseObject_Type.tp_init && + Py_TYPE(self)->tp_init((PyObject *)self, self->args, self->kw) < 0) { + /* we need to get rid of ldict from thread so + we create a new one the next time we do an attr + access */ + PyObject *exc = PyErr_GetRaisedException(); + if (PyDict_DelItem(self->localdicts, tstate->threading_local_key) < + 0) { + PyErr_WriteUnraisable((PyObject *)self); + PyErr_Clear(); + } + if (PySet_Discard(self->thread_watchdogs, wr) < 0) { + PyErr_WriteUnraisable((PyObject *)self); + } + PyErr_SetRaisedException(exc); + Py_DECREF(ldict); + Py_DECREF(wr); + return NULL; } + Py_DECREF(wr); return ldict; } @@ -1169,21 +1636,28 @@ local_setattro(localobject *self, PyObject *name, PyObject *v) PyObject *ldict = _ldict(self, state); if (ldict == NULL) { - return -1; + goto err; } int r = PyObject_RichCompareBool(name, &_Py_ID(__dict__), Py_EQ); if (r == -1) { - return -1; + goto err; } if (r == 1) { PyErr_Format(PyExc_AttributeError, - "'%.100s' object attribute '%U' is read-only", + "'%.100s' object attribute %R is read-only", Py_TYPE(self)->tp_name, name); - return -1; + goto err; } - return _PyObject_GenericSetAttrWithDict((PyObject *)self, name, v, ldict); + int st = + _PyObject_GenericSetAttrWithDict((PyObject *)self, name, v, ldict); + Py_DECREF(ldict); + return st; + +err: + Py_XDECREF(ldict); + return -1; } static PyObject *local_getattro(localobject *, PyObject *); @@ -1197,7 +1671,7 @@ static PyType_Slot local_type_slots[] = { {Py_tp_dealloc, (destructor)local_dealloc}, {Py_tp_getattro, (getattrofunc)local_getattro}, {Py_tp_setattro, (setattrofunc)local_setattro}, - {Py_tp_doc, "Thread-local data"}, + {Py_tp_doc, "_local()\n--\n\nThread-local data"}, {Py_tp_traverse, (traverseproc)local_traverse}, {Py_tp_clear, (inquiry)local_clear}, {Py_tp_new, local_new}, @@ -1226,148 +1700,67 @@ local_getattro(localobject *self, PyObject *name) int r = PyObject_RichCompareBool(name, &_Py_ID(__dict__), Py_EQ); if (r == 1) { - return Py_NewRef(ldict); - } - if (r == -1) { - return NULL; - } - - if (!Py_IS_TYPE(self, state->local_type)) { - /* use generic lookup for subtypes */ - return _PyObject_GenericGetAttrWithDict((PyObject *)self, name, - ldict, 0); - } - - /* Optimization: just look in dict ourselves */ - PyObject *value; - if (PyDict_GetItemRef(ldict, name, &value) != 0) { - // found or error - return value; - } - - /* Fall back on generic to get __class__ and __dict__ */ - return _PyObject_GenericGetAttrWithDict( - (PyObject *)self, name, ldict, 0); -} - -/* Called when a dummy is destroyed. */ -static PyObject * -_localdummy_destroyed(PyObject *localweakref, PyObject *dummyweakref) -{ - localobject *self = (localobject *)_PyWeakref_GET_REF(localweakref); - if (self == NULL) { - Py_RETURN_NONE; - } - - /* If the thread-local object is still alive and not being cleared, - remove the corresponding local dict */ - if (self->dummies != NULL) { - PyObject *ldict; - ldict = PyDict_GetItemWithError(self->dummies, dummyweakref); - if (ldict != NULL) { - PyDict_DelItem(self->dummies, dummyweakref); - } - if (PyErr_Occurred()) - PyErr_WriteUnraisable((PyObject*)self); - } - Py_DECREF(self); - Py_RETURN_NONE; -} - -/* Module functions */ - -// bootstate is used to "bootstrap" new threads. Any arguments needed by -// `thread_run()`, which can only take a single argument due to platform -// limitations, are contained in bootstate. -struct bootstate { - PyThreadState *tstate; - PyObject *func; - PyObject *args; - PyObject *kwargs; - _PyEventRc *thread_is_exiting; -}; - - -static void -thread_bootstate_free(struct bootstate *boot, int decref) -{ - if (decref) { - Py_DECREF(boot->func); - Py_DECREF(boot->args); - Py_XDECREF(boot->kwargs); + return ldict; } - if (boot->thread_is_exiting != NULL) { - _PyEventRc_Decref(boot->thread_is_exiting); + if (r == -1) { + Py_DECREF(ldict); + return NULL; } - PyMem_RawFree(boot); -} + if (!Py_IS_TYPE(self, state->local_type)) { + /* use generic lookup for subtypes */ + PyObject *res = + _PyObject_GenericGetAttrWithDict((PyObject *)self, name, ldict, 0); + Py_DECREF(ldict); + return res; + } -static void -thread_run(void *boot_raw) -{ - struct bootstate *boot = (struct bootstate *) boot_raw; - PyThreadState *tstate = boot->tstate; + /* Optimization: just look in dict ourselves */ + PyObject *value; + if (PyDict_GetItemRef(ldict, name, &value) != 0) { + // found or error + Py_DECREF(ldict); + return value; + } - // `thread_is_exiting` needs to be set after bootstate has been freed - _PyEventRc *thread_is_exiting = boot->thread_is_exiting; - boot->thread_is_exiting = NULL; + /* Fall back on generic to get __class__ and __dict__ */ + PyObject *res = + _PyObject_GenericGetAttrWithDict((PyObject *)self, name, ldict, 0); + Py_DECREF(ldict); + return res; +} - // gh-108987: If _thread.start_new_thread() is called before or while - // Python is being finalized, thread_run() can called *after*. - // _PyRuntimeState_SetFinalizing() is called. At this point, all Python - // threads must exit, except of the thread calling Py_Finalize() whch holds - // the GIL and must not exit. - // - // At this stage, tstate can be a dangling pointer (point to freed memory), - // it's ok to call _PyThreadState_MustExit() with a dangling pointer. - if (_PyThreadState_MustExit(tstate)) { - // Don't call PyThreadState_Clear() nor _PyThreadState_DeleteCurrent(). - // These functions are called on tstate indirectly by Py_Finalize() - // which calls _PyInterpreterState_Clear(). - // - // Py_DECREF() cannot be called because the GIL is not held: leak - // references on purpose. Python is being finalized anyway. - thread_bootstate_free(boot, 0); - goto exit; +/* Called when a dummy is destroyed, indicating that the owning thread is being + * cleared. */ +static PyObject * +clear_locals(PyObject *locals_and_key, PyObject *dummyweakref) +{ + PyObject *localweakref = PyTuple_GetItem(locals_and_key, 0); + localobject *self = (localobject *)_PyWeakref_GET_REF(localweakref); + if (self == NULL) { + Py_RETURN_NONE; } - _PyThreadState_Bind(tstate); - PyEval_AcquireThread(tstate); - _Py_atomic_add_ssize(&tstate->interp->threads.count, 1); - - PyObject *res = PyObject_Call(boot->func, boot->args, boot->kwargs); - if (res == NULL) { - if (PyErr_ExceptionMatches(PyExc_SystemExit)) - /* SystemExit is ignored silently */ - PyErr_Clear(); - else { - PyErr_FormatUnraisable( - "Exception ignored in thread started by %R", boot->func); + /* If the thread-local object is still alive and not being cleared, + remove the corresponding local dict */ + if (self->localdicts != NULL) { + PyObject *key = PyTuple_GetItem(locals_and_key, 1); + if (PyDict_Pop(self->localdicts, key, NULL) < 0) { + PyErr_WriteUnraisable((PyObject*)self); } } - else { - Py_DECREF(res); - } - - thread_bootstate_free(boot, 1); - - _Py_atomic_add_ssize(&tstate->interp->threads.count, -1); - PyThreadState_Clear(tstate); - _PyThreadState_DeleteCurrent(tstate); - -exit: - if (thread_is_exiting != NULL) { - _PyEvent_Notify(&thread_is_exiting->event); - _PyEventRc_Decref(thread_is_exiting); + if (self->thread_watchdogs != NULL) { + if (PySet_Discard(self->thread_watchdogs, dummyweakref) < 0) { + PyErr_WriteUnraisable((PyObject *)self); + } } - // bpo-44434: Don't call explicitly PyThread_exit_thread(). On Linux with - // the glibc, pthread_exit() can abort the whole process if dlopen() fails - // to open the libgcc_s.so library (ex: EMFILE error). - return; + Py_DECREF(self); + Py_RETURN_NONE; } +/* Module functions */ + static PyObject * thread_daemon_threads_allowed(PyObject *module, PyObject *Py_UNUSED(ignored)) { @@ -1381,17 +1774,15 @@ thread_daemon_threads_allowed(PyObject *module, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(daemon_threads_allowed_doc, -"daemon_threads_allowed()\n\ +"daemon_threads_allowed($module, /)\n\ +--\n\ \n\ Return True if daemon threads are allowed in the current interpreter,\n\ and False otherwise.\n"); static int -do_start_new_thread(thread_module_state* state, - PyObject *func, PyObject* args, PyObject* kwargs, - int joinable, - PyThread_ident_t* ident, PyThread_handle_t* handle, - _PyEventRc *thread_is_exiting) +do_start_new_thread(thread_module_state *state, PyObject *func, PyObject *args, + PyObject *kwargs, ThreadHandle *handle, int daemon) { PyInterpreterState *interp = _PyInterpreterState_GET(); if (!_PyInterpreterState_HasFeature(interp, Py_RTFLAGS_THREADS)) { @@ -1399,50 +1790,26 @@ do_start_new_thread(thread_module_state* state, "thread is not supported for isolated subinterpreters"); return -1; } - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { PyErr_SetString(PyExc_PythonFinalizationError, "can't create new thread at interpreter shutdown"); return -1; } - // gh-109795: Use PyMem_RawMalloc() instead of PyMem_Malloc(), - // because it should be possible to call thread_bootstate_free() - // without holding the GIL. - struct bootstate *boot = PyMem_RawMalloc(sizeof(struct bootstate)); - if (boot == NULL) { - PyErr_NoMemory(); - return -1; + if (!daemon) { + // Add the handle before starting the thread to avoid adding a handle + // to a thread that has already finished (i.e. if the thread finishes + // before the call to `ThreadHandle_start()` below returns). + add_to_shutdown_handles(state, handle); } - boot->tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_THREADING); - if (boot->tstate == NULL) { - PyMem_RawFree(boot); - if (!PyErr_Occurred()) { - PyErr_NoMemory(); + + if (ThreadHandle_start(handle, func, args, kwargs) < 0) { + if (!daemon) { + remove_from_shutdown_handles(handle); } return -1; } - boot->func = Py_NewRef(func); - boot->args = Py_NewRef(args); - boot->kwargs = Py_XNewRef(kwargs); - boot->thread_is_exiting = thread_is_exiting; - if (thread_is_exiting != NULL) { - _PyEventRc_Incref(thread_is_exiting); - } - int err; - if (joinable) { - err = PyThread_start_joinable_thread(thread_run, (void*) boot, ident, handle); - } else { - *handle = 0; - *ident = PyThread_start_new_thread(thread_run, (void*) boot); - err = (*ident == PYTHREAD_INVALID_THREAD_ID); - } - if (err) { - PyErr_SetString(ThreadError, "can't start new thread"); - PyThreadState_Clear(boot->tstate); - thread_bootstate_free(boot, 1); - return -1; - } return 0; } @@ -1476,18 +1843,25 @@ thread_PyThread_start_new_thread(PyObject *module, PyObject *fargs) return NULL; } - PyThread_ident_t ident = 0; - PyThread_handle_t handle; - if (do_start_new_thread(state, func, args, kwargs, /*joinable=*/ 0, - &ident, &handle, NULL)) { + ThreadHandle *handle = ThreadHandle_new(); + if (handle == NULL) { + return NULL; + } + + int st = + do_start_new_thread(state, func, args, kwargs, handle, /*daemon=*/1); + if (st < 0) { + ThreadHandle_decref(handle); return NULL; } + PyThread_ident_t ident = ThreadHandle_ident(handle); + ThreadHandle_decref(handle); return PyLong_FromUnsignedLongLong(ident); } -PyDoc_STRVAR(start_new_doc, -"start_new_thread(function, args[, kwargs])\n\ -(start_new() is an obsolete synonym)\n\ +PyDoc_STRVAR(start_new_thread_doc, +"start_new_thread($module, function, args, kwargs={}, /)\n\ +--\n\ \n\ Start a new thread and return its identifier.\n\ \n\ @@ -1496,12 +1870,28 @@ tuple args and keyword arguments taken from the optional dictionary\n\ kwargs. The thread exits when the function returns; the return value\n\ is ignored. The thread will also exit when the function raises an\n\ unhandled exception; a stack trace will be printed unless the exception\n\ -is SystemExit.\n"); +is SystemExit."); + +PyDoc_STRVAR(start_new_doc, +"start_new($module, function, args, kwargs={}, /)\n\ +--\n\ +\n\ +An obsolete synonym of start_new_thread()."); static PyObject * -thread_PyThread_start_joinable_thread(PyObject *module, PyObject *func) +thread_PyThread_start_joinable_thread(PyObject *module, PyObject *fargs, + PyObject *fkwargs) { + static char *keywords[] = {"function", "handle", "daemon", NULL}; + PyObject *func = NULL; + int daemon = 1; thread_module_state *state = get_thread_state(module); + PyObject *hobj = NULL; + if (!PyArg_ParseTupleAndKeywords(fargs, fkwargs, + "O|Op:start_joinable_thread", keywords, + &func, &hobj, &daemon)) { + return NULL; + } if (!PyCallable_Check(func)) { PyErr_SetString(PyExc_TypeError, @@ -1509,32 +1899,46 @@ thread_PyThread_start_joinable_thread(PyObject *module, PyObject *func) return NULL; } - if (PySys_Audit("_thread.start_joinable_thread", "O", func) < 0) { + if (hobj == NULL) { + hobj = Py_None; + } + else if (hobj != Py_None && !Py_IS_TYPE(hobj, state->thread_handle_type)) { + PyErr_SetString(PyExc_TypeError, "'handle' must be a _ThreadHandle"); return NULL; } - PyObject* args = PyTuple_New(0); - if (args == NULL) { + if (PySys_Audit("_thread.start_joinable_thread", "OiO", func, daemon, + hobj) < 0) { return NULL; } - ThreadHandleObject* hobj = new_thread_handle(state); - if (hobj == NULL) { - Py_DECREF(args); + + if (hobj == Py_None) { + hobj = (PyObject *)PyThreadHandleObject_new(state->thread_handle_type); + if (hobj == NULL) { + return NULL; + } + } + else { + Py_INCREF(hobj); + } + + PyObject* args = PyTuple_New(0); + if (args == NULL) { return NULL; } - if (do_start_new_thread(state, func, args, /*kwargs=*/ NULL, /*joinable=*/ 1, - &hobj->ident, &hobj->handle, hobj->thread_is_exiting)) { - Py_DECREF(args); + int st = do_start_new_thread(state, func, args, + /*kwargs=*/ NULL, ((PyThreadHandleObject*)hobj)->handle, daemon); + Py_DECREF(args); + if (st < 0) { Py_DECREF(hobj); return NULL; } - set_thread_handle_state(hobj, THREAD_HANDLE_RUNNING); - Py_DECREF(args); - return (PyObject*) hobj; + return (PyObject *) hobj; } PyDoc_STRVAR(start_joinable_doc, -"start_joinable_thread(function)\n\ +"start_joinable_thread($module, /, function, handle=None, daemon=True)\n\ +--\n\ \n\ *For internal use only*: start a new thread.\n\ \n\ @@ -1542,7 +1946,9 @@ Like start_new_thread(), this starts a new thread calling the given function.\n\ Unlike start_new_thread(), this returns a handle object with methods to join\n\ or detach the given thread.\n\ This function is not for third-party code, please use the\n\ -`threading` module instead.\n"); +`threading` module instead. During finalization the runtime will not wait for\n\ +the thread to exit if daemon is True. If handle is provided it must be a\n\ +newly created thread._ThreadHandle instance."); static PyObject * thread_PyThread_exit_thread(PyObject *self, PyObject *Py_UNUSED(ignored)) @@ -1552,12 +1958,18 @@ thread_PyThread_exit_thread(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(exit_doc, -"exit()\n\ -(exit_thread() is an obsolete synonym)\n\ +"exit($module, /)\n\ +--\n\ \n\ This is synonymous to ``raise SystemExit''. It will cause the current\n\ thread to exit silently unless the exception is caught."); +PyDoc_STRVAR(exit_thread_doc, +"exit_thread($module, /)\n\ +--\n\ +\n\ +An obsolete synonym of exit()."); + static PyObject * thread_PyThread_interrupt_main(PyObject *self, PyObject *args) { @@ -1574,7 +1986,8 @@ thread_PyThread_interrupt_main(PyObject *self, PyObject *args) } PyDoc_STRVAR(interrupt_doc, -"interrupt_main(signum=signal.SIGINT, /)\n\ +"interrupt_main($module, signum=signal.SIGINT, /)\n\ +--\n\ \n\ Simulate the arrival of the given signal in the main thread,\n\ where the corresponding signal handler will be executed.\n\ @@ -1590,13 +2003,19 @@ thread_PyThread_allocate_lock(PyObject *module, PyObject *Py_UNUSED(ignored)) return (PyObject *) newlockobject(module); } -PyDoc_STRVAR(allocate_doc, -"allocate_lock() -> lock object\n\ -(allocate() is an obsolete synonym)\n\ +PyDoc_STRVAR(allocate_lock_doc, +"allocate_lock($module, /)\n\ +--\n\ \n\ Create a new lock object. See help(type(threading.Lock())) for\n\ information about locks."); +PyDoc_STRVAR(allocate_doc, +"allocate($module, /)\n\ +--\n\ +\n\ +An obsolete synonym of allocate_lock()."); + static PyObject * thread_get_ident(PyObject *self, PyObject *Py_UNUSED(ignored)) { @@ -1609,7 +2028,8 @@ thread_get_ident(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(get_ident_doc, -"get_ident() -> integer\n\ +"get_ident($module, /)\n\ +--\n\ \n\ Return a non-zero integer that uniquely identifies the current thread\n\ amongst other threads that exist simultaneously.\n\ @@ -1628,7 +2048,8 @@ thread_get_native_id(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(get_native_id_doc, -"get_native_id() -> integer\n\ +"get_native_id($module, /)\n\ +--\n\ \n\ Return a non-negative integer identifying the thread as reported\n\ by the OS (kernel). This may be used to uniquely identify a\n\ @@ -1643,9 +2064,9 @@ thread__count(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(_count_doc, -"_count() -> integer\n\ +"_count($module, /)\n\ +--\n\ \n\ -\ Return the number of currently running Python threads, excluding\n\ the main thread. The returned number comprises all threads created\n\ through `start_new_thread()` as well as `threading.Thread`, and not\n\ @@ -1654,67 +2075,6 @@ yet finished.\n\ This function is meant for internal and specialized purposes only.\n\ In most applications `threading.enumerate()` should be used instead."); -static void -release_sentinel(void *weakref_raw) -{ - PyObject *weakref = _PyObject_CAST(weakref_raw); - - /* Tricky: this function is called when the current thread state - is being deleted. Therefore, only simple C code can safely - execute here. */ - lockobject *lock = (lockobject *)_PyWeakref_GET_REF(weakref); - if (lock != NULL) { - if (lock->locked) { - PyThread_release_lock(lock->lock_lock); - lock->locked = 0; - } - Py_DECREF(lock); - } - - /* Deallocating a weakref with a NULL callback only calls - PyObject_GC_Del(), which can't call any Python code. */ - Py_DECREF(weakref); -} - -static PyObject * -thread__set_sentinel(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - PyObject *wr; - PyThreadState *tstate = _PyThreadState_GET(); - lockobject *lock; - - if (tstate->on_delete_data != NULL) { - /* We must support the re-creation of the lock from a - fork()ed child. */ - assert(tstate->on_delete == &release_sentinel); - wr = (PyObject *) tstate->on_delete_data; - tstate->on_delete = NULL; - tstate->on_delete_data = NULL; - Py_DECREF(wr); - } - lock = newlockobject(module); - if (lock == NULL) - return NULL; - /* The lock is owned by whoever called _set_sentinel(), but the weakref - hangs to the thread state. */ - wr = PyWeakref_NewRef((PyObject *) lock, NULL); - if (wr == NULL) { - Py_DECREF(lock); - return NULL; - } - tstate->on_delete_data = (void *) wr; - tstate->on_delete = &release_sentinel; - return (PyObject *) lock; -} - -PyDoc_STRVAR(_set_sentinel_doc, -"_set_sentinel() -> lock\n\ -\n\ -Set a sentinel lock that will be released when the current thread\n\ -state is finalized (after it is untied from the interpreter).\n\ -\n\ -This is a private API for the threading module."); - static PyObject * thread_stack_size(PyObject *self, PyObject *args) { @@ -1750,7 +2110,8 @@ thread_stack_size(PyObject *self, PyObject *args) } PyDoc_STRVAR(stack_size_doc, -"stack_size([size]) -> size\n\ +"stack_size($module, size=0, /)\n\ +--\n\ \n\ Return the thread stack size used when creating new threads. The\n\ optional size argument specifies the stack size (in bytes) to be used\n\ @@ -1905,7 +2266,8 @@ thread_excepthook(PyObject *module, PyObject *args) } PyDoc_STRVAR(excepthook_doc, -"excepthook(exc_type, exc_value, exc_traceback, thread)\n\ +"_excepthook($module, (exc_type, exc_value, exc_traceback, thread), /)\n\ +--\n\ \n\ Handle uncaught Thread.run() exception."); @@ -1917,25 +2279,255 @@ thread__is_main_interpreter(PyObject *module, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(thread__is_main_interpreter_doc, -"_is_main_interpreter()\n\ +"_is_main_interpreter($module, /)\n\ +--\n\ \n\ Return True if the current interpreter is the main Python interpreter."); +static PyObject * +thread_shutdown(PyObject *self, PyObject *args) +{ + PyThread_ident_t ident = PyThread_get_thread_ident_ex(); + thread_module_state *state = get_thread_state(self); + + for (;;) { + ThreadHandle *handle = NULL; + + // Find a thread that's not yet finished. + HEAD_LOCK(&_PyRuntime); + struct llist_node *node; + llist_for_each_safe(node, &state->shutdown_handles) { + ThreadHandle *cur = llist_data(node, ThreadHandle, shutdown_node); + if (cur->ident != ident) { + ThreadHandle_incref(cur); + handle = cur; + break; + } + } + HEAD_UNLOCK(&_PyRuntime); + + if (!handle) { + // No more threads to wait on! + break; + } + + // Wait for the thread to finish. If we're interrupted, such + // as by a ctrl-c we print the error and exit early. + if (ThreadHandle_join(handle, -1) < 0) { + PyErr_WriteUnraisable(NULL); + ThreadHandle_decref(handle); + Py_RETURN_NONE; + } + + ThreadHandle_decref(handle); + } + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(shutdown_doc, +"_shutdown($module, /)\n\ +--\n\ +\n\ +Wait for all non-daemon threads (other than the calling thread) to stop."); + +static PyObject * +thread__make_thread_handle(PyObject *module, PyObject *identobj) +{ + thread_module_state *state = get_thread_state(module); + if (!PyLong_Check(identobj)) { + PyErr_SetString(PyExc_TypeError, "ident must be an integer"); + return NULL; + } + PyThread_ident_t ident = PyLong_AsUnsignedLongLong(identobj); + if (PyErr_Occurred()) { + return NULL; + } + PyThreadHandleObject *hobj = + PyThreadHandleObject_new(state->thread_handle_type); + if (hobj == NULL) { + return NULL; + } + PyMutex_Lock(&hobj->handle->mutex); + hobj->handle->ident = ident; + hobj->handle->state = THREAD_HANDLE_RUNNING; + PyMutex_Unlock(&hobj->handle->mutex); + return (PyObject*) hobj; +} + +PyDoc_STRVAR(thread__make_thread_handle_doc, +"_make_thread_handle($module, ident, /)\n\ +--\n\ +\n\ +Internal only. Make a thread handle for threads not spawned\n\ +by the _thread or threading module."); + +static PyObject * +thread__get_main_thread_ident(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return PyLong_FromUnsignedLongLong(_PyRuntime.main_thread); +} + +PyDoc_STRVAR(thread__get_main_thread_ident_doc, +"_get_main_thread_ident($module, /)\n\ +--\n\ +\n\ +Internal only. Return a non-zero integer that uniquely identifies the main thread\n\ +of the main interpreter."); + + +#if defined(HAVE_PTHREAD_GETNAME_NP) || defined(MS_WINDOWS) +/*[clinic input] +_thread._get_name + +Get the name of the current thread. +[clinic start generated code]*/ + +static PyObject * +_thread__get_name_impl(PyObject *module) +/*[clinic end generated code: output=20026e7ee3da3dd7 input=35cec676833d04c8]*/ +{ +#ifndef MS_WINDOWS + // Linux and macOS are limited to respectively 16 and 64 bytes + char name[100]; + pthread_t thread = pthread_self(); + int rc = pthread_getname_np(thread, name, Py_ARRAY_LENGTH(name)); + if (rc) { + errno = rc; + return PyErr_SetFromErrno(PyExc_OSError); + } + +#ifdef __sun + return PyUnicode_DecodeUTF8(name, strlen(name), "surrogateescape"); +#else + return PyUnicode_DecodeFSDefault(name); +#endif +#else + // Windows implementation + assert(pGetThreadDescription != NULL); + + wchar_t *name; + HRESULT hr = pGetThreadDescription(GetCurrentThread(), &name); + if (FAILED(hr)) { + PyErr_SetFromWindowsErr(0); + return NULL; + } + + PyObject *name_obj = PyUnicode_FromWideChar(name, -1); + LocalFree(name); + return name_obj; +#endif +} +#endif // HAVE_PTHREAD_GETNAME_NP + + +#if defined(HAVE_PTHREAD_SETNAME_NP) || defined(MS_WINDOWS) +/*[clinic input] +_thread.set_name + + name as name_obj: unicode + +Set the name of the current thread. +[clinic start generated code]*/ + +static PyObject * +_thread_set_name_impl(PyObject *module, PyObject *name_obj) +/*[clinic end generated code: output=402b0c68e0c0daed input=7e7acd98261be82f]*/ +{ +#ifndef MS_WINDOWS +#ifdef __sun + // Solaris always uses UTF-8 + const char *encoding = "utf-8"; +#else + // Encode the thread name to the filesystem encoding using the "replace" + // error handler + PyInterpreterState *interp = _PyInterpreterState_GET(); + const char *encoding = interp->unicode.fs_codec.encoding; +#endif + PyObject *name_encoded; + name_encoded = PyUnicode_AsEncodedString(name_obj, encoding, "replace"); + if (name_encoded == NULL) { + return NULL; + } + +#ifdef _PYTHREAD_NAME_MAXLEN + // Truncate to _PYTHREAD_NAME_MAXLEN bytes + the NUL byte if needed + if (PyBytes_GET_SIZE(name_encoded) > _PYTHREAD_NAME_MAXLEN) { + PyObject *truncated; + truncated = PyBytes_FromStringAndSize(PyBytes_AS_STRING(name_encoded), + _PYTHREAD_NAME_MAXLEN); + if (truncated == NULL) { + Py_DECREF(name_encoded); + return NULL; + } + Py_SETREF(name_encoded, truncated); + } +#endif + + const char *name = PyBytes_AS_STRING(name_encoded); +#ifdef __APPLE__ + int rc = pthread_setname_np(name); +#elif defined(__NetBSD__) + pthread_t thread = pthread_self(); + int rc = pthread_setname_np(thread, "%s", (void *)name); +#else + pthread_t thread = pthread_self(); + int rc = pthread_setname_np(thread, name); +#endif + Py_DECREF(name_encoded); + if (rc) { + errno = rc; + return PyErr_SetFromErrno(PyExc_OSError); + } + Py_RETURN_NONE; +#else + // Windows implementation + assert(pSetThreadDescription != NULL); + + Py_ssize_t len; + wchar_t *name = PyUnicode_AsWideCharString(name_obj, &len); + if (name == NULL) { + return NULL; + } + + if (len > _PYTHREAD_NAME_MAXLEN) { + // Truncate the name + Py_UCS4 ch = name[_PYTHREAD_NAME_MAXLEN-1]; + if (Py_UNICODE_IS_HIGH_SURROGATE(ch)) { + name[_PYTHREAD_NAME_MAXLEN-1] = 0; + } + else { + name[_PYTHREAD_NAME_MAXLEN] = 0; + } + } + + HRESULT hr = pSetThreadDescription(GetCurrentThread(), name); + PyMem_Free(name); + if (FAILED(hr)) { + PyErr_SetFromWindowsErr((int)hr); + return NULL; + } + Py_RETURN_NONE; +#endif +} +#endif // HAVE_PTHREAD_SETNAME_NP + + static PyMethodDef thread_methods[] = { {"start_new_thread", (PyCFunction)thread_PyThread_start_new_thread, - METH_VARARGS, start_new_doc}, + METH_VARARGS, start_new_thread_doc}, {"start_new", (PyCFunction)thread_PyThread_start_new_thread, METH_VARARGS, start_new_doc}, - {"start_joinable_thread", (PyCFunction)thread_PyThread_start_joinable_thread, - METH_O, start_joinable_doc}, + {"start_joinable_thread", _PyCFunction_CAST(thread_PyThread_start_joinable_thread), + METH_VARARGS | METH_KEYWORDS, start_joinable_doc}, {"daemon_threads_allowed", (PyCFunction)thread_daemon_threads_allowed, METH_NOARGS, daemon_threads_allowed_doc}, {"allocate_lock", thread_PyThread_allocate_lock, - METH_NOARGS, allocate_doc}, + METH_NOARGS, allocate_lock_doc}, {"allocate", thread_PyThread_allocate_lock, METH_NOARGS, allocate_doc}, {"exit_thread", thread_PyThread_exit_thread, - METH_NOARGS, exit_doc}, + METH_NOARGS, exit_thread_doc}, {"exit", thread_PyThread_exit_thread, METH_NOARGS, exit_doc}, {"interrupt_main", (PyCFunction)thread_PyThread_interrupt_main, @@ -1950,12 +2542,18 @@ static PyMethodDef thread_methods[] = { METH_NOARGS, _count_doc}, {"stack_size", (PyCFunction)thread_stack_size, METH_VARARGS, stack_size_doc}, - {"_set_sentinel", thread__set_sentinel, - METH_NOARGS, _set_sentinel_doc}, {"_excepthook", thread_excepthook, METH_O, excepthook_doc}, {"_is_main_interpreter", thread__is_main_interpreter, METH_NOARGS, thread__is_main_interpreter_doc}, + {"_shutdown", thread_shutdown, + METH_NOARGS, shutdown_doc}, + {"_make_thread_handle", thread__make_thread_handle, + METH_O, thread__make_thread_handle_doc}, + {"_get_main_thread_ident", thread__get_main_thread_ident, + METH_NOARGS, thread__get_main_thread_ident_doc}, + _THREAD_SET_NAME_METHODDEF + _THREAD__GET_NAME_METHODDEF {NULL, NULL} /* sentinel */ }; @@ -2045,6 +2643,40 @@ thread_module_exec(PyObject *module) return -1; } + llist_init(&state->shutdown_handles); + +#ifdef _PYTHREAD_NAME_MAXLEN + if (PyModule_AddIntConstant(module, "_NAME_MAXLEN", + _PYTHREAD_NAME_MAXLEN) < 0) { + return -1; + } +#endif + +#ifdef MS_WINDOWS + HMODULE kernelbase = GetModuleHandleW(L"kernelbase.dll"); + if (kernelbase != NULL) { + if (pGetThreadDescription == NULL) { + pGetThreadDescription = (PF_GET_THREAD_DESCRIPTION)GetProcAddress( + kernelbase, "GetThreadDescription"); + } + if (pSetThreadDescription == NULL) { + pSetThreadDescription = (PF_SET_THREAD_DESCRIPTION)GetProcAddress( + kernelbase, "SetThreadDescription"); + } + } + + if (pGetThreadDescription == NULL) { + if (PyObject_DelAttrString(module, "_get_name") < 0) { + return -1; + } + } + if (pSetThreadDescription == NULL) { + if (PyObject_DelAttrString(module, "set_name") < 0) { + return -1; + } + } +#endif + return 0; } @@ -2070,6 +2702,10 @@ thread_module_clear(PyObject *module) Py_CLEAR(state->local_type); Py_CLEAR(state->local_dummy_type); Py_CLEAR(state->thread_handle_type); + // Remove any remaining handles (e.g. if shutdown exited early due to + // interrupt) so that attempts to unlink the handle after our module state + // is destroyed do not crash. + clear_shutdown_handles(state); return 0; } @@ -2088,6 +2724,7 @@ The 'threading' module provides a more convenient interface."); static PyModuleDef_Slot thread_module_slots[] = { {Py_mod_exec, thread_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index e3789867dc085f..45897817a56051 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -69,6 +69,12 @@ Copyright (C) 1994 Steen Lumholt. #define USE_DEPRECATED_TOMMATH_API 1 #endif +// As suggested by https://core.tcl-lang.org/tcl/wiki?name=Migrating+C+extensions+to+Tcl+9 +#ifndef TCL_SIZE_MAX +typedef int Tcl_Size; +#define TCL_SIZE_MAX INT_MAX +#endif + #if !(defined(MS_WINDOWS) || defined(__CYGWIN__)) #define HAVE_CREATEFILEHANDLER #endif @@ -137,7 +143,7 @@ _get_tcl_lib_path(void) struct stat stat_buf; int stat_return_value; - PyObject *prefix = PySys_GetObject("prefix"); // borrowed reference + PyObject *prefix = PySys_GetObject("base_prefix"); // borrowed reference if (prefix == NULL) { return NULL; } @@ -306,6 +312,7 @@ typedef struct { int threaded; /* True if tcl_platform[threaded] */ Tcl_ThreadId thread_id; int dispatching; + PyObject *trace; /* We cannot include tclInt.h, as this is internal. So we cache interesting types here. */ const Tcl_ObjType *OldBooleanType; @@ -317,6 +324,8 @@ typedef struct { const Tcl_ObjType *BignumType; const Tcl_ObjType *ListType; const Tcl_ObjType *StringType; + const Tcl_ObjType *UTF32StringType; + const Tcl_ObjType *PixelType; } TkappObject; #define Tkapp_Interp(v) (((TkappObject *) (v))->interp) @@ -485,24 +494,28 @@ unicodeFromTclString(const char *s) } static PyObject * -unicodeFromTclObj(Tcl_Obj *value) +unicodeFromTclObj(TkappObject *tkapp, Tcl_Obj *value) { - int len; + Tcl_Size len; #if USE_TCL_UNICODE - int byteorder = NATIVE_BYTEORDER; - const Tcl_UniChar *u = Tcl_GetUnicodeFromObj(value, &len); - if (sizeof(Tcl_UniChar) == 2) - return PyUnicode_DecodeUTF16((const char *)u, len * 2, - "surrogatepass", &byteorder); - else if (sizeof(Tcl_UniChar) == 4) - return PyUnicode_DecodeUTF32((const char *)u, len * 4, - "surrogatepass", &byteorder); - else - Py_UNREACHABLE(); -#else + if (value->typePtr != NULL && tkapp != NULL && + (value->typePtr == tkapp->StringType || + value->typePtr == tkapp->UTF32StringType)) + { + int byteorder = NATIVE_BYTEORDER; + const Tcl_UniChar *u = Tcl_GetUnicodeFromObj(value, &len); + if (sizeof(Tcl_UniChar) == 2) + return PyUnicode_DecodeUTF16((const char *)u, len * 2, + "surrogatepass", &byteorder); + else if (sizeof(Tcl_UniChar) == 4) + return PyUnicode_DecodeUTF32((const char *)u, len * 4, + "surrogatepass", &byteorder); + else + Py_UNREACHABLE(); + } +#endif /* USE_TCL_UNICODE */ const char *s = Tcl_GetStringFromObj(value, &len); return unicodeFromTclStringAndSize(s, len); -#endif } /*[clinic input] @@ -515,6 +528,10 @@ class _tkinter.tktimertoken "TkttObject *" "&Tktt_Type_spec" /**** Tkapp Object ****/ +#if TK_MAJOR_VERSION >= 9 +int Tcl_AppInit(Tcl_Interp *); +#endif + #ifndef WITH_APPINIT int Tcl_AppInit(Tcl_Interp *interp) @@ -570,6 +587,7 @@ Tkapp_New(const char *screenName, const char *className, TCL_GLOBAL_ONLY) != NULL; v->thread_id = Tcl_GetCurrentThread(); v->dispatching = 0; + v->trace = NULL; #ifndef TCL_THREADS if (v->threaded) { @@ -586,14 +604,41 @@ Tkapp_New(const char *screenName, const char *className, } v->OldBooleanType = Tcl_GetObjType("boolean"); - v->BooleanType = Tcl_GetObjType("booleanString"); - v->ByteArrayType = Tcl_GetObjType("bytearray"); + { + Tcl_Obj *value; + int boolValue; + + /* Tcl 8.5 "booleanString" type is not registered + and is renamed to "boolean" in Tcl 9.0. + Based on approach suggested at + https://core.tcl-lang.org/tcl/info/3bb3bcf2da5b */ + value = Tcl_NewStringObj("true", -1); + Tcl_GetBooleanFromObj(NULL, value, &boolValue); + v->BooleanType = value->typePtr; + Tcl_DecrRefCount(value); + + // "bytearray" type is not registered in Tcl 9.0 + value = Tcl_NewByteArrayObj(NULL, 0); + v->ByteArrayType = value->typePtr; + Tcl_DecrRefCount(value); + } v->DoubleType = Tcl_GetObjType("double"); + /* TIP 484 suggests retrieving the "int" type without Tcl_GetObjType("int") + since it is no longer registered in Tcl 9.0. But even though Tcl 8.7 + only uses the "wideInt" type on platforms with 32-bit long, it still has + a registered "int" type, which FromObj() should recognize just in case. */ v->IntType = Tcl_GetObjType("int"); + if (v->IntType == NULL) { + Tcl_Obj *value = Tcl_NewIntObj(0); + v->IntType = value->typePtr; + Tcl_DecrRefCount(value); + } v->WideIntType = Tcl_GetObjType("wideInt"); v->BignumType = Tcl_GetObjType("bignum"); v->ListType = Tcl_GetObjType("list"); v->StringType = Tcl_GetObjType("string"); + v->UTF32StringType = Tcl_GetObjType("utf32string"); + v->PixelType = Tcl_GetObjType("pixel"); /* Delete the 'exit' command, which can screw things up */ Tcl_DeleteCommand(v->interp, "exit"); @@ -754,7 +799,7 @@ PyTclObject_string(PyObject *_self, void *ignored) { PyTclObject *self = (PyTclObject *)_self; if (!self->string) { - self->string = unicodeFromTclObj(self->value); + self->string = unicodeFromTclObj(NULL, self->value); if (!self->string) return NULL; } @@ -769,7 +814,7 @@ PyTclObject_str(PyObject *_self) return Py_NewRef(self->string); } /* XXX Could cache result if it is non-ASCII. */ - return unicodeFromTclObj(self->value); + return unicodeFromTclObj(NULL, self->value); } static PyObject * @@ -975,7 +1020,9 @@ AsObj(PyObject *value) PyErr_SetString(PyExc_OverflowError, "string is too long"); return NULL; } - if (PyUnicode_IS_ASCII(value)) { + if (PyUnicode_IS_ASCII(value) && + strlen(PyUnicode_DATA(value)) == (size_t)PyUnicode_GET_LENGTH(value)) + { return Tcl_NewStringObj((const char *)PyUnicode_DATA(value), (int)size); } @@ -990,9 +1037,6 @@ AsObj(PyObject *value) "surrogatepass", NATIVE_BYTEORDER); else Py_UNREACHABLE(); -#else - encoded = _PyUnicode_AsUTF8String(value, "surrogateescape"); -#endif if (!encoded) { return NULL; } @@ -1002,12 +1046,39 @@ AsObj(PyObject *value) PyErr_SetString(PyExc_OverflowError, "string is too long"); return NULL; } -#if USE_TCL_UNICODE result = Tcl_NewUnicodeObj((const Tcl_UniChar *)PyBytes_AS_STRING(encoded), (int)(size / sizeof(Tcl_UniChar))); #else + encoded = _PyUnicode_AsUTF8String(value, "surrogateescape"); + if (!encoded) { + return NULL; + } + size = PyBytes_GET_SIZE(encoded); + if (strlen(PyBytes_AS_STRING(encoded)) != (size_t)size) { + /* The string contains embedded null characters. + * Tcl needs a null character to be represented as \xc0\x80 in + * the Modified UTF-8 encoding. Otherwise the string can be + * truncated in some internal operations. + * + * NOTE: stringlib_replace() could be used here, but optimizing + * this obscure case isn't worth it unless stringlib_replace() + * was already exposed in the C API for other reasons. */ + Py_SETREF(encoded, + PyObject_CallMethod(encoded, "replace", "y#y#", + "\0", (Py_ssize_t)1, + "\xc0\x80", (Py_ssize_t)2)); + if (!encoded) { + return NULL; + } + size = PyBytes_GET_SIZE(encoded); + } + if (size > INT_MAX) { + Py_DECREF(encoded); + PyErr_SetString(PyExc_OverflowError, "string is too long"); + return NULL; + } result = Tcl_NewStringObj(PyBytes_AS_STRING(encoded), (int)size); -#endif +#endif /* USE_TCL_UNICODE */ Py_DECREF(encoded); return result; } @@ -1104,7 +1175,7 @@ FromObj(TkappObject *tkapp, Tcl_Obj *value) Tcl_Interp *interp = Tkapp_Interp(tkapp); if (value->typePtr == NULL) { - return unicodeFromTclObj(value); + return unicodeFromTclObj(tkapp, value); } if (value->typePtr == tkapp->BooleanType || @@ -1113,7 +1184,7 @@ FromObj(TkappObject *tkapp, Tcl_Obj *value) } if (value->typePtr == tkapp->ByteArrayType) { - int size; + Tcl_Size size; char *data = (char*)Tcl_GetByteArrayFromObj(value, &size); return PyBytes_FromStringAndSize(data, size); } @@ -1122,14 +1193,6 @@ FromObj(TkappObject *tkapp, Tcl_Obj *value) return PyFloat_FromDouble(value->internalRep.doubleValue); } - if (value->typePtr == tkapp->IntType) { - long longValue; - if (Tcl_GetLongFromObj(interp, value, &longValue) == TCL_OK) - return PyLong_FromLong(longValue); - /* If there is an error in the long conversion, - fall through to wideInt handling. */ - } - if (value->typePtr == tkapp->IntType || value->typePtr == tkapp->WideIntType) { result = fromWideIntObj(tkapp, value); @@ -1147,8 +1210,8 @@ FromObj(TkappObject *tkapp, Tcl_Obj *value) } if (value->typePtr == tkapp->ListType) { - int size; - int i, status; + Tcl_Size i, size; + int status; PyObject *elem; Tcl_Obj *tcl_elem; @@ -1174,15 +1237,11 @@ FromObj(TkappObject *tkapp, Tcl_Obj *value) return result; } - if (value->typePtr == tkapp->StringType) { - return unicodeFromTclObj(value); - } - - if (tkapp->BooleanType == NULL && - strcmp(value->typePtr->name, "booleanString") == 0) { - /* booleanString type is not registered in Tcl */ - tkapp->BooleanType = value->typePtr; - return fromBoolean(tkapp, value); + if (value->typePtr == tkapp->StringType || + value->typePtr == tkapp->UTF32StringType || + value->typePtr == tkapp->PixelType) + { + return unicodeFromTclObj(tkapp, value); } if (tkapp->BignumType == NULL && @@ -1209,9 +1268,9 @@ typedef struct Tkapp_CallEvent { } Tkapp_CallEvent; static void -Tkapp_CallDeallocArgs(Tcl_Obj** objv, Tcl_Obj** objStore, int objc) +Tkapp_CallDeallocArgs(Tcl_Obj** objv, Tcl_Obj** objStore, Tcl_Size objc) { - int i; + Tcl_Size i; for (i = 0; i < objc; i++) Tcl_DecrRefCount(objv[i]); if (objv != objStore) @@ -1222,7 +1281,7 @@ Tkapp_CallDeallocArgs(Tcl_Obj** objv, Tcl_Obj** objStore, int objc) interpreter thread, which may or may not be the calling thread. */ static Tcl_Obj** -Tkapp_CallArgs(PyObject *args, Tcl_Obj** objStore, int *pobjc) +Tkapp_CallArgs(PyObject *args, Tcl_Obj** objStore, Tcl_Size *pobjc) { Tcl_Obj **objv = objStore; Py_ssize_t objc = 0, i; @@ -1270,10 +1329,10 @@ Tkapp_CallArgs(PyObject *args, Tcl_Obj** objStore, int *pobjc) Tcl_IncrRefCount(objv[i]); } } - *pobjc = (int)objc; + *pobjc = (Tcl_Size)objc; return objv; finally: - Tkapp_CallDeallocArgs(objv, objStore, (int)objc); + Tkapp_CallDeallocArgs(objv, objStore, (Tcl_Size)objc); return NULL; } @@ -1282,7 +1341,7 @@ Tkapp_CallArgs(PyObject *args, Tcl_Obj** objStore, int *pobjc) static PyObject * Tkapp_UnicodeResult(TkappObject *self) { - return unicodeFromTclObj(Tcl_GetObjResult(self->interp)); + return unicodeFromTclObj(self, Tcl_GetObjResult(self->interp)); } @@ -1301,11 +1360,34 @@ Tkapp_ObjectResult(TkappObject *self) res = FromObj(self, value); Tcl_DecrRefCount(value); } else { - res = unicodeFromTclObj(value); + res = unicodeFromTclObj(self, value); } return res; } +static int +Tkapp_Trace(TkappObject *self, PyObject *args) +{ + if (args == NULL) { + return 0; + } + if (self->trace) { + PyObject *res = PyObject_CallObject(self->trace, args); + if (res == NULL) { + Py_DECREF(args); + return 0; + } + Py_DECREF(res); + } + Py_DECREF(args); + return 1; +} + +#define TRACE(_self, ARGS) do { \ + if ((_self)->trace && !Tkapp_Trace((_self), Py_BuildValue ARGS)) { \ + return NULL; \ + } \ + } while (0) /* Tkapp_CallProc is the event procedure that is executed in the context of the Tcl interpreter thread. Initially, it holds the Tcl lock, and doesn't @@ -1317,10 +1399,15 @@ Tkapp_CallProc(Tcl_Event *evPtr, int flags) Tkapp_CallEvent *e = (Tkapp_CallEvent *)evPtr; Tcl_Obj *objStore[ARGSZ]; Tcl_Obj **objv; - int objc; + Tcl_Size objc; int i; ENTER_PYTHON - objv = Tkapp_CallArgs(e->args, objStore, &objc); + if (e->self->trace && !Tkapp_Trace(e->self, PyTuple_Pack(1, e->args))) { + objv = NULL; + } + else { + objv = Tkapp_CallArgs(e->args, objStore, &objc); + } if (!objv) { *(e->exc) = PyErr_GetRaisedException(); *(e->res) = NULL; @@ -1368,7 +1455,7 @@ Tkapp_Call(PyObject *selfptr, PyObject *args) { Tcl_Obj *objStore[ARGSZ]; Tcl_Obj **objv = NULL; - int objc, i; + Tcl_Size objc; PyObject *res = NULL; TkappObject *self = (TkappObject*)selfptr; int flags = TCL_EVAL_DIRECT | TCL_EVAL_GLOBAL; @@ -1384,7 +1471,7 @@ Tkapp_Call(PyObject *selfptr, PyObject *args) marshal the parameters to the interpreter thread. */ Tkapp_CallEvent *ev; Tcl_Condition cond = NULL; - PyObject *exc; + PyObject *exc = NULL; // init to make static analyzers happy if (!WaitForMainloop(self)) return NULL; ev = (Tkapp_CallEvent*)attemptckalloc(sizeof(Tkapp_CallEvent)); @@ -1413,7 +1500,9 @@ Tkapp_Call(PyObject *selfptr, PyObject *args) } else { + TRACE(self, ("(O)", args)); + int i; objv = Tkapp_CallArgs(args, objStore, &objc); if (!objv) return NULL; @@ -1455,6 +1544,8 @@ _tkinter_tkapp_eval_impl(TkappObject *self, const char *script) CHECK_STRING_LENGTH(script); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ss))", "eval", script)); + ENTER_TCL err = Tcl_Eval(Tkapp_Interp(self), script); ENTER_OVERLAP @@ -1484,6 +1575,8 @@ _tkinter_tkapp_evalfile_impl(TkappObject *self, const char *fileName) CHECK_STRING_LENGTH(fileName); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ss))", "source", fileName)); + ENTER_TCL err = Tcl_EvalFile(Tkapp_Interp(self), fileName); ENTER_OVERLAP @@ -1513,6 +1606,8 @@ _tkinter_tkapp_record_impl(TkappObject *self, const char *script) CHECK_STRING_LENGTH(script); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ssss))", "history", "add", script, "exec")); + ENTER_TCL err = Tcl_RecordAndEval(Tkapp_Interp(self), script, TCL_NO_EVAL); ENTER_OVERLAP @@ -1650,7 +1745,8 @@ var_invoke(EventFunc func, PyObject *selfptr, PyObject *args, int flags) TkappObject *self = (TkappObject*)selfptr; if (self->threaded && self->thread_id != Tcl_GetCurrentThread()) { VarEvent *ev; - PyObject *res, *exc; + // init 'res' and 'exc' to make static analyzers happy + PyObject *res = NULL, *exc = NULL; Tcl_Condition cond = NULL; /* The current thread is not the interpreter thread. Marshal @@ -1702,6 +1798,15 @@ SetVar(TkappObject *self, PyObject *args, int flags) newval = AsObj(newValue); if (newval == NULL) return NULL; + + if (flags & TCL_GLOBAL_ONLY) { + TRACE((TkappObject *)self, ("((ssssO))", "uplevel", "#0", "set", + name1, newValue)); + } + else { + TRACE((TkappObject *)self, ("((ssO))", "set", name1, newValue)); + } + ENTER_TCL ok = Tcl_SetVar2Ex(Tkapp_Interp(self), name1, NULL, newval, flags); @@ -1719,8 +1824,22 @@ SetVar(TkappObject *self, PyObject *args, int flags) return NULL; CHECK_STRING_LENGTH(name1); CHECK_STRING_LENGTH(name2); + /* XXX must hold tcl lock already??? */ newval = AsObj(newValue); + if (((TkappObject *)self)->trace) { + if (flags & TCL_GLOBAL_ONLY) { + TRACE((TkappObject *)self, ("((sssNO))", "uplevel", "#0", "set", + PyUnicode_FromFormat("%s(%s)", name1, name2), + newValue)); + } + else { + TRACE((TkappObject *)self, ("((sNO))", "set", + PyUnicode_FromFormat("%s(%s)", name1, name2), + newValue)); + } + } + ENTER_TCL ok = Tcl_SetVar2Ex(Tkapp_Interp(self), name1, name2, newval, flags); ENTER_OVERLAP @@ -1774,7 +1893,7 @@ GetVar(TkappObject *self, PyObject *args, int flags) res = FromObj(self, tres); } else { - res = unicodeFromTclObj(tres); + res = unicodeFromTclObj(self, tres); } } LEAVE_OVERLAP_TCL @@ -1807,6 +1926,28 @@ UnsetVar(TkappObject *self, PyObject *args, int flags) CHECK_STRING_LENGTH(name1); CHECK_STRING_LENGTH(name2); + + if (((TkappObject *)self)->trace) { + if (flags & TCL_GLOBAL_ONLY) { + if (name2) { + TRACE((TkappObject *)self, ("((sssN))", "uplevel", "#0", "unset", + PyUnicode_FromFormat("%s(%s)", name1, name2))); + } + else { + TRACE((TkappObject *)self, ("((ssss))", "uplevel", "#0", "unset", name1)); + } + } + else { + if (name2) { + TRACE((TkappObject *)self, ("((sN))", "unset", + PyUnicode_FromFormat("%s(%s)", name1, name2))); + } + else { + TRACE((TkappObject *)self, ("((ss))", "unset", name1)); + } + } + } + ENTER_TCL code = Tcl_UnsetVar2(Tkapp_Interp(self), name1, name2, flags); ENTER_OVERLAP @@ -1973,6 +2114,8 @@ _tkinter_tkapp_exprstring_impl(TkappObject *self, const char *s) CHECK_STRING_LENGTH(s); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ss))", "expr", s)); + ENTER_TCL retval = Tcl_ExprString(Tkapp_Interp(self), s); ENTER_OVERLAP @@ -2003,6 +2146,8 @@ _tkinter_tkapp_exprlong_impl(TkappObject *self, const char *s) CHECK_STRING_LENGTH(s); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ss))", "expr", s)); + ENTER_TCL retval = Tcl_ExprLong(Tkapp_Interp(self), s, &v); ENTER_OVERLAP @@ -2032,6 +2177,9 @@ _tkinter_tkapp_exprdouble_impl(TkappObject *self, const char *s) CHECK_STRING_LENGTH(s); CHECK_TCL_APPARTMENT; + + TRACE(self, ("((ss))", "expr", s)); + ENTER_TCL retval = Tcl_ExprDouble(Tkapp_Interp(self), s, &v); ENTER_OVERLAP @@ -2061,6 +2209,9 @@ _tkinter_tkapp_exprboolean_impl(TkappObject *self, const char *s) CHECK_STRING_LENGTH(s); CHECK_TCL_APPARTMENT; + + TRACE(self, ("((ss))", "expr", s)); + ENTER_TCL retval = Tcl_ExprBoolean(Tkapp_Interp(self), s, &v); ENTER_OVERLAP @@ -2087,13 +2238,12 @@ _tkinter_tkapp_splitlist(TkappObject *self, PyObject *arg) /*[clinic end generated code: output=13b51d34386d36fb input=2b2e13351e3c0b53]*/ { char *list; - int argc; + Tcl_Size argc, i; const char **argv; PyObject *v; - int i; if (PyTclObject_Check(arg)) { - int objc; + Tcl_Size objc; Tcl_Obj **objv; if (Tcl_ListObjGetElements(Tkapp_Interp(self), ((PyTclObject*)arg)->value, @@ -2156,7 +2306,7 @@ _tkinter_tkapp_splitlist(TkappObject *self, PyObject *arg) /* Client data struct */ typedef struct { - PyObject *self; + TkappObject *self; PyObject *func; } PythonCmd_ClientData; @@ -2180,6 +2330,7 @@ PythonCmd(ClientData clientData, Tcl_Interp *interp, PyObject *args, *res; int i; Tcl_Obj *obj_res; + int objargs = data->self->wantobjects >= 2; ENTER_PYTHON @@ -2188,7 +2339,8 @@ PythonCmd(ClientData clientData, Tcl_Interp *interp, return PythonCmd_Error(interp); for (i = 0; i < (objc - 1); i++) { - PyObject *s = unicodeFromTclObj(objv[i + 1]); + PyObject *s = objargs ? FromObj(data->self, objv[i + 1]) + : unicodeFromTclObj(data->self, objv[i + 1]); if (!s) { Py_DECREF(args); return PythonCmd_Error(interp); @@ -2286,12 +2438,17 @@ _tkinter_tkapp_createcommand_impl(TkappObject *self, const char *name, !WaitForMainloop(self)) return NULL; + TRACE(self, ("((ss()O))", "proc", name, func)); + data = PyMem_NEW(PythonCmd_ClientData, 1); if (!data) return PyErr_NoMemory(); - data->self = Py_NewRef(self); + Py_INCREF(self); + data->self = self; data->func = Py_NewRef(func); if (self->threaded && self->thread_id != Tcl_GetCurrentThread()) { + err = 0; // init to make static analyzers happy + Tcl_Condition cond = NULL; CommandEvent *ev = (CommandEvent*)attemptckalloc(sizeof(CommandEvent)); if (ev == NULL) { @@ -2344,7 +2501,11 @@ _tkinter_tkapp_deletecommand_impl(TkappObject *self, const char *name) CHECK_STRING_LENGTH(name); + TRACE(self, ("((sss))", "rename", name, "")); + if (self->threaded && self->thread_id != Tcl_GetCurrentThread()) { + err = 0; // init to make static analyzers happy + Tcl_Condition cond = NULL; CommandEvent *ev; ev = (CommandEvent*)attemptckalloc(sizeof(CommandEvent)); @@ -2469,6 +2630,8 @@ _tkinter_tkapp_createfilehandler_impl(TkappObject *self, PyObject *file, return NULL; } + TRACE(self, ("((ssiiO))", "#", "createfilehandler", tfile, mask, func)); + data = NewFHCD(func, file, tfile); if (data == NULL) return NULL; @@ -2500,6 +2663,8 @@ _tkinter_tkapp_deletefilehandler(TkappObject *self, PyObject *file) if (tfile < 0) return NULL; + TRACE(self, ("((ssi))", "#", "deletefilehandler", tfile)); + DeleteFHCD(tfile); /* Ought to check for null Tcl_File object... */ @@ -2534,6 +2699,7 @@ _tkinter_tktimertoken_deletetimerhandler_impl(TkttObject *self) PyObject *func = v->func; if (v->token != NULL) { + /* TRACE(...) */ Tcl_DeleteTimerHandler(v->token); v->token = NULL; } @@ -2636,6 +2802,8 @@ _tkinter_tkapp_createtimerhandler_impl(TkappObject *self, int milliseconds, CHECK_TCL_APPARTMENT; + TRACE(self, ("((siO))", "after", milliseconds, func)); + v = Tktt_New(func); if (v) { v->token = Tcl_CreateTimerHandler(milliseconds, TimerHandler, @@ -2794,15 +2962,56 @@ Tkapp_WantObjects(PyObject *self, PyObject *args) { int wantobjects = -1; - if (!PyArg_ParseTuple(args, "|p:wantobjects", &wantobjects)) + if (!PyArg_ParseTuple(args, "|i:wantobjects", &wantobjects)) return NULL; if (wantobjects == -1) - return PyBool_FromLong(((TkappObject*)self)->wantobjects); + return PyLong_FromLong(((TkappObject*)self)->wantobjects); ((TkappObject*)self)->wantobjects = wantobjects; Py_RETURN_NONE; } +/*[clinic input] +_tkinter.tkapp.settrace + + func: object + / + +Set the tracing function. +[clinic start generated code]*/ + +static PyObject * +_tkinter_tkapp_settrace(TkappObject *self, PyObject *func) +/*[clinic end generated code: output=847f6ebdf46e84fa input=31b260d46d3d018a]*/ +{ + if (func == Py_None) { + func = NULL; + } + else { + Py_INCREF(func); + } + Py_XSETREF(self->trace, func); + Py_RETURN_NONE; +} + +/*[clinic input] +_tkinter.tkapp.gettrace + +Get the tracing function. +[clinic start generated code]*/ + +static PyObject * +_tkinter_tkapp_gettrace_impl(TkappObject *self) +/*[clinic end generated code: output=d4e2ba7d63e77bb5 input=ac2aea5be74e8c4c]*/ +{ + PyObject *func = self->trace; + if (!func) { + func = Py_None; + } + Py_INCREF(func); + return func; +} + /*[clinic input] _tkinter.tkapp.willdispatch @@ -2828,6 +3037,7 @@ Tkapp_Dealloc(PyObject *self) ENTER_TCL Tcl_DeleteInterp(Tkapp_Interp(self)); LEAVE_TCL + Py_XDECREF(((TkappObject *)self)->trace); PyObject_Free(self); Py_DECREF(tp); DisableEventHook(); @@ -2941,7 +3151,7 @@ _tkinter.create baseName: str = "" className: str = "Tk" interactive: bool = False - wantobjects: bool = False + wantobjects: int = 0 wantTk: bool = True if false, then Tk_Init() doesn't get called sync: bool = False @@ -2957,7 +3167,7 @@ _tkinter_create_impl(PyObject *module, const char *screenName, const char *baseName, const char *className, int interactive, int wantobjects, int wantTk, int sync, const char *use) -/*[clinic end generated code: output=e3315607648e6bb4 input=09afef9adea70a19]*/ +/*[clinic end generated code: output=e3315607648e6bb4 input=7e382ba431bed537]*/ { /* XXX baseName is not used anymore; * try getting rid of it. */ @@ -3038,6 +3248,8 @@ static PyMethodDef Tkapp_methods[] = { _TKINTER_TKAPP_WILLDISPATCH_METHODDEF {"wantobjects", Tkapp_WantObjects, METH_VARARGS}, + _TKINTER_TKAPP_SETTRACE_METHODDEF + _TKINTER_TKAPP_GETTRACE_METHODDEF {"call", Tkapp_Call, METH_VARARGS}, _TKINTER_TKAPP_EVAL_METHODDEF _TKINTER_TKAPP_EVALFILE_METHODDEF @@ -3180,17 +3392,40 @@ DisableEventHook(void) #endif } +static int +module_clear(PyObject *Py_UNUSED(mod)) +{ + Py_CLEAR(Tkinter_TclError); + Py_CLEAR(Tkapp_Type); + Py_CLEAR(Tktt_Type); + Py_CLEAR(PyTclObject_Type); + return 0; +} + +static int +module_traverse(PyObject *Py_UNUSED(module), visitproc visit, void *arg) +{ + Py_VISIT(Tkinter_TclError); + Py_VISIT(Tkapp_Type); + Py_VISIT(Tktt_Type); + Py_VISIT(PyTclObject_Type); + return 0; +} + +static void +module_free(void *mod) +{ + (void)module_clear((PyObject *)mod); +} static struct PyModuleDef _tkintermodule = { PyModuleDef_HEAD_INIT, - "_tkinter", - NULL, - -1, - moduleMethods, - NULL, - NULL, - NULL, - NULL + .m_name = "_tkinter", + .m_size = -1, + .m_methods = moduleMethods, + .m_traverse = module_traverse, + .m_clear = module_clear, + .m_free = module_free }; PyMODINIT_FUNC @@ -3205,6 +3440,9 @@ PyInit__tkinter(void) m = PyModule_Create(&_tkintermodule); if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif Tkinter_TclError = PyErr_NewException("_tkinter.TclError", NULL, NULL); if (PyModule_AddObjectRef(m, "TclError", Tkinter_TclError)) { diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 6dba3cac01c1c8..be71fc9fc9c341 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -215,15 +215,14 @@ static struct PyModuleDef module_def = { PyMODINIT_FUNC PyInit__tracemalloc(void) { - PyObject *m; - m = PyModule_Create(&module_def); - if (m == NULL) - return NULL; - - if (_PyTraceMalloc_Init() < 0) { - Py_DECREF(m); + PyObject *mod = PyModule_Create(&module_def); + if (mod == NULL) { return NULL; } - return m; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); +#endif + + return mod; } diff --git a/Modules/_typingmodule.c b/Modules/_typingmodule.c index 9ea72bf89ce0b2..09fbb3c5e8b91d 100644 --- a/Modules/_typingmodule.c +++ b/Modules/_typingmodule.c @@ -63,12 +63,16 @@ _typing_exec(PyObject *m) if (PyModule_AddObjectRef(m, "TypeAliasType", (PyObject *)&_PyTypeAlias_Type) < 0) { return -1; } + if (PyModule_AddObjectRef(m, "NoDefault", (PyObject *)&_Py_NoDefaultStruct) < 0) { + return -1; + } return 0; } static struct PyModuleDef_Slot _typingmodule_slots[] = { {Py_mod_exec, _typing_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_uuidmodule.c b/Modules/_uuidmodule.c index 4b6852c0d0ec73..c5e78b1510b5e3 100644 --- a/Modules/_uuidmodule.c +++ b/Modules/_uuidmodule.c @@ -3,11 +3,10 @@ * DCE compatible Universally Unique Identifier library. */ +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -112,6 +111,7 @@ static PyMethodDef uuid_methods[] = { static PyModuleDef_Slot uuid_slots[] = { {Py_mod_exec, uuid_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_weakref.c b/Modules/_weakref.c index 7225dbc9ce4a1b..ecaa08ff60f203 100644 --- a/Modules/_weakref.c +++ b/Modules/_weakref.c @@ -14,7 +14,6 @@ module _weakref #include "clinic/_weakref.c.h" /*[clinic input] -@critical_section object _weakref.getweakrefcount -> Py_ssize_t object: object @@ -25,19 +24,14 @@ Return the number of weak references to 'object'. static Py_ssize_t _weakref_getweakrefcount_impl(PyObject *module, PyObject *object) -/*[clinic end generated code: output=301806d59558ff3e input=6535a580f1d0ebdc]*/ +/*[clinic end generated code: output=301806d59558ff3e input=7d4d04fcaccf64d5]*/ { - if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(object))) { - return 0; - } - PyWeakReference **list = GET_WEAKREFS_LISTPTR(object); - Py_ssize_t count = _PyWeakref_GetWeakrefCount(*list); - return count; + return _PyWeakref_GetWeakrefCount(object); } static int -is_dead_weakref(PyObject *value) +is_dead_weakref(PyObject *value, void *unused) { if (!PyWeakref_Check(value)) { PyErr_SetString(PyExc_TypeError, "not a weakref"); @@ -62,22 +56,14 @@ _weakref__remove_dead_weakref_impl(PyObject *module, PyObject *dct, PyObject *key) /*[clinic end generated code: output=d9ff53061fcb875c input=19fc91f257f96a1d]*/ { - if (_PyDict_DelItemIf(dct, key, is_dead_weakref) < 0) { - if (PyErr_ExceptionMatches(PyExc_KeyError)) - /* This function is meant to allow safe weak-value dicts - with GC in another thread (see issue #28427), so it's - ok if the key doesn't exist anymore. - */ - PyErr_Clear(); - else - return NULL; + if (_PyDict_DelItemIf(dct, key, is_dead_weakref, NULL) < 0) { + return NULL; } Py_RETURN_NONE; } /*[clinic input] -@critical_section object _weakref.getweakrefs object: object / @@ -86,26 +72,39 @@ Return a list of all weak reference objects pointing to 'object'. [clinic start generated code]*/ static PyObject * -_weakref_getweakrefs_impl(PyObject *module, PyObject *object) -/*[clinic end generated code: output=5ec268989fb8f035 input=3dea95b8f5b31bbb]*/ +_weakref_getweakrefs(PyObject *module, PyObject *object) +/*[clinic end generated code: output=25c7731d8e011824 input=00c6d0e5d3206693]*/ { if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(object))) { return PyList_New(0); } - PyWeakReference **list = GET_WEAKREFS_LISTPTR(object); - Py_ssize_t count = _PyWeakref_GetWeakrefCount(*list); - - PyObject *result = PyList_New(count); + PyObject *result = PyList_New(0); if (result == NULL) { return NULL; } - PyWeakReference *current = *list; - for (Py_ssize_t i = 0; i < count; ++i) { - PyList_SET_ITEM(result, i, Py_NewRef(current)); + LOCK_WEAKREFS(object); + PyWeakReference *current = *GET_WEAKREFS_LISTPTR(object); + while (current != NULL) { + PyObject *curobj = (PyObject *) current; + if (_Py_TryIncref(curobj)) { + if (PyList_Append(result, curobj)) { + UNLOCK_WEAKREFS(object); + Py_DECREF(curobj); + Py_DECREF(result); + return NULL; + } + else { + // Undo our _Py_TryIncref. This is safe to do with the lock + // held in free-threaded builds; the list holds a reference to + // curobj so we're guaranteed not to invoke the destructor. + Py_DECREF(curobj); + } + } current = current->wr_next; } + UNLOCK_WEAKREFS(object); return result; } @@ -165,6 +164,7 @@ weakref_exec(PyObject *module) static struct PyModuleDef_Slot weakref_slots[] = { {Py_mod_exec, weakref_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 8f9b8520bb3f34..260cab48091c16 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -72,9 +72,45 @@ #ifndef STARTF_USESHOWWINDOW #define STARTF_USESHOWWINDOW 0x00000001 #endif +#ifndef STARTF_USESIZE +#define STARTF_USESIZE 0x00000002 +#endif +#ifndef STARTF_USEPOSITION +#define STARTF_USEPOSITION 0x00000004 +#endif +#ifndef STARTF_USECOUNTCHARS +#define STARTF_USECOUNTCHARS 0x00000008 +#endif +#ifndef STARTF_USEFILLATTRIBUTE +#define STARTF_USEFILLATTRIBUTE 0x00000010 +#endif +#ifndef STARTF_RUNFULLSCREEN +#define STARTF_RUNFULLSCREEN 0x00000020 +#endif +#ifndef STARTF_FORCEONFEEDBACK +#define STARTF_FORCEONFEEDBACK 0x00000040 +#endif +#ifndef STARTF_FORCEOFFFEEDBACK +#define STARTF_FORCEOFFFEEDBACK 0x00000080 +#endif #ifndef STARTF_USESTDHANDLES #define STARTF_USESTDHANDLES 0x00000100 #endif +#ifndef STARTF_USEHOTKEY +#define STARTF_USEHOTKEY 0x00000200 +#endif +#ifndef STARTF_TITLEISLINKNAME +#define STARTF_TITLEISLINKNAME 0x00000800 +#endif +#ifndef STARTF_TITLEISAPPID +#define STARTF_TITLEISAPPID 0x00001000 +#endif +#ifndef STARTF_PREVENTPINNING +#define STARTF_PREVENTPINNING 0x00002000 +#endif +#ifndef STARTF_UNTRUSTEDSOURCE +#define STARTF_UNTRUSTEDSOURCE 0x00008000 +#endif typedef struct { PyTypeObject *overlapped_type; @@ -188,7 +224,6 @@ create_converter('LPCVOID', '" F_POINTER "') create_converter('BOOL', 'i') # F_BOOL used previously (always 'i') create_converter('DWORD', 'k') # F_DWORD is always "k" (which is much shorter) -create_converter('LPCTSTR', 's') create_converter('UINT', 'I') # F_UINT used previously (always 'I') class LPCWSTR_converter(Py_UNICODE_converter): @@ -223,7 +258,7 @@ class LPVOID_return_converter(CReturnConverter): data.return_conversion.append( 'return_value = HANDLE_TO_PYNUM(_return_value);\n') [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=ef52a757a1830d92]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=da0a4db751936ee7]*/ #include "clinic/_winapi.c.h" @@ -494,7 +529,7 @@ _winapi_CreateFile_impl(PyObject *module, LPCWSTR file_name, { HANDLE handle; - if (PySys_Audit("_winapi.CreateFile", "uIIII", + if (PySys_Audit("_winapi.CreateFile", "ukkkk", file_name, desired_access, share_mode, creation_disposition, flags_and_attributes) < 0) { return INVALID_HANDLE_VALUE; @@ -741,7 +776,7 @@ _winapi_CreateMutexW_impl(PyObject *module, /*[clinic input] _winapi.CreateNamedPipe -> HANDLE - name: LPCTSTR + name: LPCWSTR open_mode: DWORD pipe_mode: DWORD max_instances: DWORD @@ -753,25 +788,25 @@ _winapi.CreateNamedPipe -> HANDLE [clinic start generated code]*/ static HANDLE -_winapi_CreateNamedPipe_impl(PyObject *module, LPCTSTR name, DWORD open_mode, +_winapi_CreateNamedPipe_impl(PyObject *module, LPCWSTR name, DWORD open_mode, DWORD pipe_mode, DWORD max_instances, DWORD out_buffer_size, DWORD in_buffer_size, DWORD default_timeout, LPSECURITY_ATTRIBUTES security_attributes) -/*[clinic end generated code: output=80f8c07346a94fbc input=5a73530b84d8bc37]*/ +/*[clinic end generated code: output=7d6fde93227680ba input=5bd4e4a55639ee02]*/ { HANDLE handle; - if (PySys_Audit("_winapi.CreateNamedPipe", "uII", + if (PySys_Audit("_winapi.CreateNamedPipe", "ukk", name, open_mode, pipe_mode) < 0) { return INVALID_HANDLE_VALUE; } Py_BEGIN_ALLOW_THREADS - handle = CreateNamedPipe(name, open_mode, pipe_mode, - max_instances, out_buffer_size, - in_buffer_size, default_timeout, - security_attributes); + handle = CreateNamedPipeW(name, open_mode, pipe_mode, + max_instances, out_buffer_size, + in_buffer_size, default_timeout, + security_attributes); Py_END_ALLOW_THREADS if (handle == INVALID_HANDLE_VALUE) @@ -1013,7 +1048,7 @@ getenvironment(PyObject* environment) } normalized_environment = normalize_environment(environment); - if (normalize_environment == NULL) { + if (normalized_environment == NULL) { return NULL; } @@ -1517,6 +1552,49 @@ _winapi_GetLastError_impl(PyObject *module) return GetLastError(); } + +/*[clinic input] +_winapi.GetLongPathName + + path: LPCWSTR + +Return the long version of the provided path. + +If the path is already in its long form, returns the same value. + +The path must already be a 'str'. If the type is not known, use +os.fsdecode before calling this function. +[clinic start generated code]*/ + +static PyObject * +_winapi_GetLongPathName_impl(PyObject *module, LPCWSTR path) +/*[clinic end generated code: output=c4774b080275a2d0 input=9872e211e3a4a88f]*/ +{ + DWORD cchBuffer; + PyObject *result = NULL; + + Py_BEGIN_ALLOW_THREADS + cchBuffer = GetLongPathNameW(path, NULL, 0); + Py_END_ALLOW_THREADS + if (cchBuffer) { + WCHAR *buffer = (WCHAR *)PyMem_Malloc(cchBuffer * sizeof(WCHAR)); + if (buffer) { + Py_BEGIN_ALLOW_THREADS + cchBuffer = GetLongPathNameW(path, buffer, cchBuffer); + Py_END_ALLOW_THREADS + if (cchBuffer) { + result = PyUnicode_FromWideChar(buffer, cchBuffer); + } else { + PyErr_SetFromWindowsErr(0); + } + PyMem_Free((void *)buffer); + } + } else { + PyErr_SetFromWindowsErr(0); + } + return result; +} + /*[clinic input] _winapi.GetModuleFileName @@ -1551,6 +1629,48 @@ _winapi_GetModuleFileName_impl(PyObject *module, HMODULE module_handle) return PyUnicode_FromWideChar(filename, wcslen(filename)); } +/*[clinic input] +_winapi.GetShortPathName + + path: LPCWSTR + +Return the short version of the provided path. + +If the path is already in its short form, returns the same value. + +The path must already be a 'str'. If the type is not known, use +os.fsdecode before calling this function. +[clinic start generated code]*/ + +static PyObject * +_winapi_GetShortPathName_impl(PyObject *module, LPCWSTR path) +/*[clinic end generated code: output=dab6ae494c621e81 input=43fa349aaf2ac718]*/ +{ + DWORD cchBuffer; + PyObject *result = NULL; + + Py_BEGIN_ALLOW_THREADS + cchBuffer = GetShortPathNameW(path, NULL, 0); + Py_END_ALLOW_THREADS + if (cchBuffer) { + WCHAR *buffer = (WCHAR *)PyMem_Malloc(cchBuffer * sizeof(WCHAR)); + if (buffer) { + Py_BEGIN_ALLOW_THREADS + cchBuffer = GetShortPathNameW(path, buffer, cchBuffer); + Py_END_ALLOW_THREADS + if (cchBuffer) { + result = PyUnicode_FromWideChar(buffer, cchBuffer); + } else { + PyErr_SetFromWindowsErr(0); + } + PyMem_Free((void *)buffer); + } + } else { + PyErr_SetFromWindowsErr(0); + } + return result; +} + /*[clinic input] _winapi.GetStdHandle -> HANDLE @@ -1669,7 +1789,7 @@ _winapi_OpenEventW_impl(PyObject *module, DWORD desired_access, { HANDLE handle; - if (PySys_Audit("_winapi.OpenEventW", "Iu", desired_access, name) < 0) { + if (PySys_Audit("_winapi.OpenEventW", "ku", desired_access, name) < 0) { return INVALID_HANDLE_VALUE; } @@ -1700,7 +1820,7 @@ _winapi_OpenMutexW_impl(PyObject *module, DWORD desired_access, { HANDLE handle; - if (PySys_Audit("_winapi.OpenMutexW", "Iu", desired_access, name) < 0) { + if (PySys_Audit("_winapi.OpenMutexW", "ku", desired_access, name) < 0) { return INVALID_HANDLE_VALUE; } @@ -1761,7 +1881,7 @@ _winapi_OpenProcess_impl(PyObject *module, DWORD desired_access, { HANDLE handle; - if (PySys_Audit("_winapi.OpenProcess", "II", + if (PySys_Audit("_winapi.OpenProcess", "kk", process_id, desired_access) < 0) { return INVALID_HANDLE_VALUE; } @@ -2115,19 +2235,19 @@ _winapi_VirtualQuerySize_impl(PyObject *module, LPCVOID address) /*[clinic input] _winapi.WaitNamedPipe - name: LPCTSTR + name: LPCWSTR timeout: DWORD / [clinic start generated code]*/ static PyObject * -_winapi_WaitNamedPipe_impl(PyObject *module, LPCTSTR name, DWORD timeout) -/*[clinic end generated code: output=c2866f4439b1fe38 input=36fc781291b1862c]*/ +_winapi_WaitNamedPipe_impl(PyObject *module, LPCWSTR name, DWORD timeout) +/*[clinic end generated code: output=e161e2e630b3e9c2 input=099a4746544488fa]*/ { BOOL success; Py_BEGIN_ALLOW_THREADS - success = WaitNamedPipe(name, timeout); + success = WaitNamedPipeW(name, timeout); Py_END_ALLOW_THREADS if (!success) @@ -2197,7 +2317,7 @@ _winapi_BatchedWaitForMultipleObjects_impl(PyObject *module, BOOL wait_all, DWORD milliseconds) /*[clinic end generated code: output=d21c1a4ad0a252fd input=7e196f29005dc77b]*/ { - Py_ssize_t thread_count = 0, handle_count = 0, i, j; + Py_ssize_t thread_count = 0, handle_count = 0, i; Py_ssize_t nhandles; BatchedWaitData *thread_data[MAXIMUM_WAIT_OBJECTS]; HANDLE handles[MAXIMUM_WAIT_OBJECTS]; @@ -2258,7 +2378,7 @@ _winapi_BatchedWaitForMultipleObjects_impl(PyObject *module, if (data->handle_count > MAXIMUM_WAIT_OBJECTS - 1) { data->handle_count = MAXIMUM_WAIT_OBJECTS - 1; } - for (j = 0; j < data->handle_count; ++i, ++j) { + for (DWORD j = 0; j < data->handle_count; ++i, ++j) { PyObject *v = PySequence_GetItem(handle_seq, i); if (!v || !PyArg_Parse(v, F_HANDLE, &data->handles[j])) { Py_XDECREF(v); @@ -2406,7 +2526,7 @@ _winapi_BatchedWaitForMultipleObjects_impl(PyObject *module, if (triggered_indices) { for (i = 0; i < thread_count; ++i) { Py_ssize_t triggered = (Py_ssize_t)thread_data[i]->result - WAIT_OBJECT_0; - if (triggered >= 0 && triggered < thread_data[i]->handle_count - 1) { + if (triggered >= 0 && (size_t)triggered < thread_data[i]->handle_count - 1) { PyObject *v = PyLong_FromSsize_t(thread_data[i]->handle_base + triggered); if (!v || PyList_Append(triggered_indices, v) < 0) { Py_XDECREF(v); @@ -2683,7 +2803,7 @@ _winapi__mimetypes_read_windows_registry_impl(PyObject *module, } err = RegOpenKeyExW(hkcr, ext, 0, KEY_READ, &subkey); - if (err == ERROR_FILE_NOT_FOUND) { + if (err == ERROR_FILE_NOT_FOUND || err == ERROR_ACCESS_DENIED) { err = ERROR_SUCCESS; continue; } else if (err != ERROR_SUCCESS) { @@ -2796,7 +2916,7 @@ _winapi_CopyFile2_impl(PyObject *module, LPCWSTR existing_file_name, HRESULT hr; COPYFILE2_EXTENDED_PARAMETERS params = { sizeof(COPYFILE2_EXTENDED_PARAMETERS) }; - if (PySys_Audit("_winapi.CopyFile2", "uuI", + if (PySys_Audit("_winapi.CopyFile2", "uuk", existing_file_name, new_file_name, flags) < 0) { return NULL; } @@ -2846,7 +2966,9 @@ static PyMethodDef winapi_functions[] = { _WINAPI_GETCURRENTPROCESS_METHODDEF _WINAPI_GETEXITCODEPROCESS_METHODDEF _WINAPI_GETLASTERROR_METHODDEF + _WINAPI_GETLONGPATHNAME_METHODDEF _WINAPI_GETMODULEFILENAME_METHODDEF + _WINAPI_GETSHORTPATHNAME_METHODDEF _WINAPI_GETSTDHANDLE_METHODDEF _WINAPI_GETVERSION_METHODDEF _WINAPI_MAPVIEWOFFILE_METHODDEF @@ -2974,7 +3096,19 @@ static int winapi_exec(PyObject *m) WINAPI_CONSTANT(F_DWORD, SEC_RESERVE); WINAPI_CONSTANT(F_DWORD, SEC_WRITECOMBINE); WINAPI_CONSTANT(F_DWORD, STARTF_USESHOWWINDOW); + WINAPI_CONSTANT(F_DWORD, STARTF_USESIZE); + WINAPI_CONSTANT(F_DWORD, STARTF_USEPOSITION); + WINAPI_CONSTANT(F_DWORD, STARTF_USECOUNTCHARS); + WINAPI_CONSTANT(F_DWORD, STARTF_USEFILLATTRIBUTE); + WINAPI_CONSTANT(F_DWORD, STARTF_RUNFULLSCREEN); + WINAPI_CONSTANT(F_DWORD, STARTF_FORCEONFEEDBACK); + WINAPI_CONSTANT(F_DWORD, STARTF_FORCEOFFFEEDBACK); WINAPI_CONSTANT(F_DWORD, STARTF_USESTDHANDLES); + WINAPI_CONSTANT(F_DWORD, STARTF_USEHOTKEY); + WINAPI_CONSTANT(F_DWORD, STARTF_TITLEISLINKNAME); + WINAPI_CONSTANT(F_DWORD, STARTF_TITLEISAPPID); + WINAPI_CONSTANT(F_DWORD, STARTF_PREVENTPINNING); + WINAPI_CONSTANT(F_DWORD, STARTF_UNTRUSTEDSOURCE); WINAPI_CONSTANT(F_DWORD, STD_INPUT_HANDLE); WINAPI_CONSTANT(F_DWORD, STD_OUTPUT_HANDLE); WINAPI_CONSTANT(F_DWORD, STD_ERROR_HANDLE); @@ -3032,6 +3166,11 @@ static int winapi_exec(PyObject *m) #define COPY_FILE_REQUEST_COMPRESSED_TRAFFIC 0x10000000 #endif WINAPI_CONSTANT(F_DWORD, COPY_FILE_REQUEST_COMPRESSED_TRAFFIC); +#ifndef COPY_FILE_DIRECTORY + // Only defined in newer WinSDKs + #define COPY_FILE_DIRECTORY 0x00000080 +#endif + WINAPI_CONSTANT(F_DWORD, COPY_FILE_DIRECTORY); WINAPI_CONSTANT(F_DWORD, COPYFILE2_CALLBACK_CHUNK_STARTED); WINAPI_CONSTANT(F_DWORD, COPYFILE2_CALLBACK_CHUNK_FINISHED); @@ -3054,6 +3193,7 @@ static int winapi_exec(PyObject *m) static PyModuleDef_Slot winapi_slots[] = { {Py_mod_exec, winapi_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_xxtestfuzz/README.rst b/Modules/_xxtestfuzz/README.rst index b951858458c82f..68d5d589d2a551 100644 --- a/Modules/_xxtestfuzz/README.rst +++ b/Modules/_xxtestfuzz/README.rst @@ -23,7 +23,7 @@ Add the test name on a new line in ``fuzz_tests.txt``. In ``fuzzer.c``, add a function to be run:: - int $test_name (const char* data, size_t size) { + static int $fuzz_test_name(const char* data, size_t size) { ... return 0; } @@ -31,10 +31,12 @@ In ``fuzzer.c``, add a function to be run:: And invoke it from ``LLVMFuzzerTestOneInput``:: - #if _Py_FUZZ_YES(fuzz_builtin_float) - rv |= _run_fuzz(data, size, fuzz_builtin_float); + #if !defined(_Py_FUZZ_ONE) || defined(_Py_FUZZ_$fuzz_test_name) + rv |= _run_fuzz(data, size, $fuzz_test_name); #endif +Don't forget to replace ``$fuzz_test_name`` with your actual test name. + ``LLVMFuzzerTestOneInput`` will run in oss-fuzz, with each test in ``fuzz_tests.txt`` run separately. diff --git a/Modules/_xxtestfuzz/_xxtestfuzz.c b/Modules/_xxtestfuzz/_xxtestfuzz.c index a2dbabce71ed67..2952d7043e01fe 100644 --- a/Modules/_xxtestfuzz/_xxtestfuzz.c +++ b/Modules/_xxtestfuzz/_xxtestfuzz.c @@ -28,13 +28,18 @@ static PyMethodDef module_methods[] = { {NULL}, }; +static PyModuleDef_Slot module_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static struct PyModuleDef _fuzzmodule = { PyModuleDef_HEAD_INIT, "_fuzz", NULL, 0, module_methods, - NULL, + module_slots, NULL, NULL, NULL @@ -43,5 +48,5 @@ static struct PyModuleDef _fuzzmodule = { PyMODINIT_FUNC PyInit__xxtestfuzz(void) { - return PyModule_Create(&_fuzzmodule); + return PyModuleDef_Init(&_fuzzmodule); } diff --git a/Modules/_xxtestfuzz/fuzzer.c b/Modules/_xxtestfuzz/fuzzer.c index 6ea9f64d628530..a04f1412eefda1 100644 --- a/Modules/_xxtestfuzz/fuzzer.c +++ b/Modules/_xxtestfuzz/fuzzer.c @@ -15,7 +15,6 @@ #endif #include -#include "pycore_pyhash.h" // _Py_HashBytes() #include #include @@ -45,7 +44,7 @@ static int fuzz_builtin_int(const char* data, size_t size) { /* Pick a random valid base. (When the fuzzed function takes extra parameters, it's somewhat normal to hash the input to generate those parameters. We want to exercise all code paths, so we do so here.) */ - int base = _Py_HashBytes(data, size) % 37; + int base = Py_HashBuffer(data, size) % 37; if (base == 1) { // 1 is the only number between 0 and 36 that is not a valid base. base = 0; diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index fcd4af64df0be9..c5292575c22f23 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -3,6 +3,7 @@ #endif #include "Python.h" +#include "pycore_critical_section.h" // _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED() #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() @@ -298,15 +299,20 @@ get_weak_cache(zoneinfo_state *state, PyTypeObject *type) } } +/*[clinic input] +@critical_section +@classmethod +zoneinfo.ZoneInfo.__new__ + + key: object + +Create a new ZoneInfo instance. +[clinic start generated code]*/ + static PyObject * -zoneinfo_new(PyTypeObject *type, PyObject *args, PyObject *kw) +zoneinfo_ZoneInfo_impl(PyTypeObject *type, PyObject *key) +/*[clinic end generated code: output=95e61dab86bb95c3 input=ef73d7a83bf8790e]*/ { - PyObject *key = NULL; - static char *kwlist[] = {"key", NULL}; - if (PyArg_ParseTupleAndKeywords(args, kw, "O", kwlist, &key) == 0) { - return NULL; - } - zoneinfo_state *state = zoneinfo_get_state_by_self(type); PyObject *instance = zone_from_strong_cache(state, type, key); if (instance != NULL || PyErr_Occurred()) { @@ -467,6 +473,7 @@ zoneinfo_ZoneInfo_no_cache_impl(PyTypeObject *type, PyTypeObject *cls, } /*[clinic input] +@critical_section @classmethod zoneinfo.ZoneInfo.clear_cache @@ -481,7 +488,7 @@ Clear the ZoneInfo cache. static PyObject * zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyTypeObject *cls, PyObject *only_keys) -/*[clinic end generated code: output=114d9b7c8a22e660 input=e32ca3bb396788ba]*/ +/*[clinic end generated code: output=114d9b7c8a22e660 input=35944715df26d24e]*/ { zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); PyObject *weak_cache = get_weak_cache(state, type); @@ -816,14 +823,10 @@ zoneinfo_ZoneInfo__unpickle_impl(PyTypeObject *type, PyTypeObject *cls, /*[clinic end generated code: output=556712fc709deecb input=6ac8c73eed3de316]*/ { if (from_cache) { - PyObject *val_args = PyTuple_Pack(1, key); - if (val_args == NULL) { - return NULL; - } - - PyObject *rv = zoneinfo_new(type, val_args, NULL); - - Py_DECREF(val_args); + PyObject *rv; + Py_BEGIN_CRITICAL_SECTION(type); + rv = zoneinfo_ZoneInfo_impl(type, key); + Py_END_CRITICAL_SECTION(); return rv; } else { @@ -858,8 +861,7 @@ load_timedelta(zoneinfo_state *state, long seconds) 0, seconds, 0, 1, PyDateTimeAPI->DeltaType); if (tmp != NULL) { - rv = PyDict_SetDefault(state->TIMEDELTA_CACHE, pyoffset, tmp); - Py_XINCREF(rv); + PyDict_SetDefaultRef(state->TIMEDELTA_CACHE, pyoffset, tmp, &rv); Py_DECREF(tmp); } } @@ -944,6 +946,7 @@ ttinfo_eq(const _ttinfo *const tti0, const _ttinfo *const tti1) static int load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj) { + int rv = 0; PyObject *data_tuple = NULL; long *utcoff = NULL; @@ -1220,7 +1223,6 @@ load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj) } } - int rv = 0; goto cleanup; error: // These resources only need to be freed if we have failed, if we succeed @@ -2368,6 +2370,7 @@ strong_cache_free(StrongCacheNode *root) static void remove_from_strong_cache(zoneinfo_state *state, StrongCacheNode *node) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(state->ZoneInfoType); if (state->ZONEINFO_STRONG_CACHE == node) { state->ZONEINFO_STRONG_CACHE = node->next; } @@ -2422,6 +2425,7 @@ eject_from_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, return 0; } + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(state->ZoneInfoType); StrongCacheNode *cache = state->ZONEINFO_STRONG_CACHE; StrongCacheNode *node = find_in_strong_cache(cache, key); if (node != NULL) { @@ -2478,6 +2482,7 @@ zone_from_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, return NULL; // Strong cache currently only implemented for base class } + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(state->ZoneInfoType); StrongCacheNode *cache = state->ZONEINFO_STRONG_CACHE; StrongCacheNode *node = find_in_strong_cache(cache, key); @@ -2504,6 +2509,7 @@ update_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, return; } + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(state->ZoneInfoType); StrongCacheNode *new_node = strong_cache_node_new(key, zone); if (new_node == NULL) { return; @@ -2631,7 +2637,7 @@ static PyType_Slot zoneinfo_slots[] = { {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_methods, zoneinfo_methods}, {Py_tp_members, zoneinfo_members}, - {Py_tp_new, zoneinfo_new}, + {Py_tp_new, zoneinfo_ZoneInfo}, {Py_tp_dealloc, zoneinfo_dealloc}, {Py_tp_traverse, zoneinfo_traverse}, {Py_tp_clear, zoneinfo_clear}, @@ -2760,6 +2766,7 @@ zoneinfomodule_exec(PyObject *m) static PyModuleDef_Slot zoneinfomodule_slots[] = { {Py_mod_exec, zoneinfomodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index df09d9d84789f7..28c6a0ae05c598 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -247,7 +247,7 @@ BB_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) if (!PyArg_Parse(v, "b;array item must be integer", &x)) return -1; if (i >= 0) - ((char *)ap->ob_item)[i] = x; + ((unsigned char *)ap->ob_item)[i] = x; return 0; } @@ -260,20 +260,32 @@ u_getitem(arrayobject *ap, Py_ssize_t i) static int u_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { - PyObject *u; - if (!PyArg_Parse(v, "U;array item must be unicode character", &u)) { + if (!PyUnicode_Check(v)) { + PyErr_Format(PyExc_TypeError, + "array item must be a unicode character, not %T", + v); return -1; } - Py_ssize_t len = PyUnicode_AsWideChar(u, NULL, 0); + Py_ssize_t len = PyUnicode_AsWideChar(v, NULL, 0); if (len != 2) { - PyErr_SetString(PyExc_TypeError, - "array item must be unicode character"); + if (PyUnicode_GET_LENGTH(v) != 1) { + PyErr_Format(PyExc_TypeError, + "array item must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(v)); + } + else { + PyErr_Format(PyExc_TypeError, + "string %A cannot be converted to " + "a single wchar_t character", + v); + } return -1; } wchar_t w; - len = PyUnicode_AsWideChar(u, &w, 1); + len = PyUnicode_AsWideChar(v, &w, 1); assert(len == 1); if (i >= 0) { @@ -291,19 +303,23 @@ w_getitem(arrayobject *ap, Py_ssize_t i) static int w_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { - PyObject *u; - if (!PyArg_Parse(v, "U;array item must be unicode character", &u)) { + if (!PyUnicode_Check(v)) { + PyErr_Format(PyExc_TypeError, + "array item must be a unicode character, not %T", + v); return -1; } - if (PyUnicode_GetLength(u) != 1) { - PyErr_SetString(PyExc_TypeError, - "array item must be unicode character"); + if (PyUnicode_GET_LENGTH(v) != 1) { + PyErr_Format(PyExc_TypeError, + "array item must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(v)); return -1; } if (i >= 0) { - ((Py_UCS4 *)ap->ob_item)[i] = PyUnicode_READ_CHAR(u, 0); + ((Py_UCS4 *)ap->ob_item)[i] = PyUnicode_READ_CHAR(v, 0); } return 0; } @@ -1541,7 +1557,7 @@ array_array_fromfile_impl(arrayobject *self, PyTypeObject *cls, PyObject *f, not_enough_bytes = (PyBytes_GET_SIZE(b) != nbytes); - res = array_array_frombytes(self, b); + res = array_array_frombytes((PyObject *)self, b); Py_DECREF(b); if (res == NULL) return NULL; @@ -2781,8 +2797,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) else if (initial != NULL && (PyByteArray_Check(initial) || PyBytes_Check(initial))) { PyObject *v; - v = array_array_frombytes((arrayobject *)a, - initial); + v = array_array_frombytes((PyObject *)a, initial); if (v == NULL) { Py_DECREF(a); return NULL; @@ -2847,7 +2862,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyDoc_STRVAR(module_doc, "This module defines an object type which can efficiently represent\n\ -an array of basic values: characters, integers, floating point\n\ +an array of basic values: characters, integers, floating-point\n\ numbers. Arrays are sequence types and behave very much like lists,\n\ except that the type of objects stored in them is constrained.\n"); @@ -2875,8 +2890,8 @@ The following type codes are defined:\n\ 'L' unsigned integer 4\n\ 'q' signed integer 8 (see note)\n\ 'Q' unsigned integer 8 (see note)\n\ - 'f' floating point 4\n\ - 'd' floating point 8\n\ + 'f' floating-point 4\n\ + 'd' floating-point 8\n\ \n\ NOTE: The 'u' typecode corresponds to Python's unicode character. On\n\ narrow builds this is 2-bytes on wide builds this is 4-bytes.\n\ @@ -3074,11 +3089,16 @@ array_arrayiterator___setstate__(arrayiterobject *self, PyObject *state) Py_ssize_t index = PyLong_AsSsize_t(state); if (index == -1 && PyErr_Occurred()) return NULL; - if (index < 0) - index = 0; - else if (index > Py_SIZE(self->ao)) - index = Py_SIZE(self->ao); /* iterator exhausted */ - self->index = index; + arrayobject *ao = self->ao; + if (ao != NULL) { + if (index < 0) { + index = 0; + } + else if (index > Py_SIZE(ao)) { + index = Py_SIZE(ao); /* iterator exhausted */ + } + self->index = index; + } Py_RETURN_NONE; } @@ -3220,6 +3240,7 @@ array_modexec(PyObject *m) static PyModuleDef_Slot arrayslots[] = { {Py_mod_exec, array_modexec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/atexitmodule.c b/Modules/atexitmodule.c index b6f1bcbca67916..1b89b32ba907d7 100644 --- a/Modules/atexitmodule.c +++ b/Modules/atexitmodule.c @@ -27,7 +27,10 @@ int PyUnstable_AtExit(PyInterpreterState *interp, atexit_datacallbackfunc func, void *data) { - assert(interp == _PyInterpreterState_GET()); + PyThreadState *tstate = _PyThreadState_GET(); + _Py_EnsureTstateNotNULL(tstate); + assert(tstate->interp == interp); + atexit_callback *callback = PyMem_Malloc(sizeof(atexit_callback)); if (callback == NULL) { PyErr_NoMemory(); @@ -38,43 +41,25 @@ PyUnstable_AtExit(PyInterpreterState *interp, callback->next = NULL; struct atexit_state *state = &interp->atexit; - if (state->ll_callbacks == NULL) { + _PyAtExit_LockCallbacks(state); + atexit_callback *top = state->ll_callbacks; + if (top == NULL) { state->ll_callbacks = callback; - state->last_ll_callback = callback; } else { - state->last_ll_callback->next = callback; + callback->next = top; + state->ll_callbacks = callback; } + _PyAtExit_UnlockCallbacks(state); return 0; } -static void -atexit_delete_cb(struct atexit_state *state, int i) -{ - atexit_py_callback *cb = state->callbacks[i]; - state->callbacks[i] = NULL; - - Py_DECREF(cb->func); - Py_DECREF(cb->args); - Py_XDECREF(cb->kwargs); - PyMem_Free(cb); -} - - /* Clear all callbacks without calling them */ static void atexit_cleanup(struct atexit_state *state) { - atexit_py_callback *cb; - for (int i = 0; i < state->ncallbacks; i++) { - cb = state->callbacks[i]; - if (cb == NULL) - continue; - - atexit_delete_cb(state, i); - } - state->ncallbacks = 0; + PyList_Clear(state->callbacks); } @@ -85,23 +70,21 @@ _PyAtExit_Init(PyInterpreterState *interp) // _PyAtExit_Init() must only be called once assert(state->callbacks == NULL); - state->callback_len = 32; - state->ncallbacks = 0; - state->callbacks = PyMem_New(atexit_py_callback*, state->callback_len); + state->callbacks = PyList_New(0); if (state->callbacks == NULL) { return _PyStatus_NO_MEMORY(); } return _PyStatus_OK(); } - void _PyAtExit_Fini(PyInterpreterState *interp) { + // In theory, there shouldn't be any threads left by now, so we + // won't lock this. struct atexit_state *state = &interp->atexit; atexit_cleanup(state); - PyMem_Free(state->callbacks); - state->callbacks = NULL; + Py_CLEAR(state->callbacks); atexit_callback *next = state->ll_callbacks; state->ll_callbacks = NULL; @@ -116,35 +99,44 @@ _PyAtExit_Fini(PyInterpreterState *interp) } } - static void atexit_callfuncs(struct atexit_state *state) { assert(!PyErr_Occurred()); + assert(state->callbacks != NULL); + assert(PyList_CheckExact(state->callbacks)); - if (state->ncallbacks == 0) { + // Create a copy of the list for thread safety + PyObject *copy = PyList_GetSlice(state->callbacks, 0, PyList_GET_SIZE(state->callbacks)); + if (copy == NULL) + { + PyErr_WriteUnraisable(NULL); return; } - for (int i = state->ncallbacks - 1; i >= 0; i--) { - atexit_py_callback *cb = state->callbacks[i]; - if (cb == NULL) { - continue; - } + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(copy); ++i) { + // We don't have to worry about evil borrowed references, because + // no other threads can access this list. + PyObject *tuple = PyList_GET_ITEM(copy, i); + assert(PyTuple_CheckExact(tuple)); - // bpo-46025: Increment the refcount of cb->func as the call itself may unregister it - PyObject* the_func = Py_NewRef(cb->func); - PyObject *res = PyObject_Call(cb->func, cb->args, cb->kwargs); + PyObject *func = PyTuple_GET_ITEM(tuple, 0); + PyObject *args = PyTuple_GET_ITEM(tuple, 1); + PyObject *kwargs = PyTuple_GET_ITEM(tuple, 2); + + PyObject *res = PyObject_Call(func, + args, + kwargs == Py_None ? NULL : kwargs); if (res == NULL) { PyErr_FormatUnraisable( - "Exception ignored in atexit callback %R", the_func); + "Exception ignored in atexit callback %R", func); } else { Py_DECREF(res); } - Py_DECREF(the_func); } + Py_DECREF(copy); atexit_cleanup(state); assert(!PyErr_Occurred()); @@ -164,7 +156,8 @@ _PyAtExit_Call(PyInterpreterState *interp) PyDoc_STRVAR(atexit_register__doc__, -"register(func, *args, **kwargs) -> func\n\ +"register($module, func, /, *args, **kwargs)\n\ +--\n\ \n\ Register a function to be executed upon normal program termination\n\ \n\ @@ -189,39 +182,34 @@ atexit_register(PyObject *module, PyObject *args, PyObject *kwargs) "the first argument must be callable"); return NULL; } + PyObject *func_args = PyTuple_GetSlice(args, 1, PyTuple_GET_SIZE(args)); + PyObject *func_kwargs = kwargs; - struct atexit_state *state = get_atexit_state(); - if (state->ncallbacks >= state->callback_len) { - atexit_py_callback **r; - state->callback_len += 16; - size_t size = sizeof(atexit_py_callback*) * (size_t)state->callback_len; - r = (atexit_py_callback**)PyMem_Realloc(state->callbacks, size); - if (r == NULL) { - return PyErr_NoMemory(); - } - state->callbacks = r; + if (func_kwargs == NULL) + { + func_kwargs = Py_None; } - - atexit_py_callback *callback = PyMem_Malloc(sizeof(atexit_py_callback)); - if (callback == NULL) { - return PyErr_NoMemory(); + PyObject *callback = PyTuple_Pack(3, func, func_args, func_kwargs); + if (callback == NULL) + { + return NULL; } - callback->args = PyTuple_GetSlice(args, 1, PyTuple_GET_SIZE(args)); - if (callback->args == NULL) { - PyMem_Free(callback); + struct atexit_state *state = get_atexit_state(); + // atexit callbacks go in a LIFO order + if (PyList_Insert(state->callbacks, 0, callback) < 0) + { + Py_DECREF(callback); return NULL; } - callback->func = Py_NewRef(func); - callback->kwargs = Py_XNewRef(kwargs); - - state->callbacks[state->ncallbacks++] = callback; + Py_DECREF(callback); return Py_NewRef(func); } PyDoc_STRVAR(atexit_run_exitfuncs__doc__, -"_run_exitfuncs() -> None\n\ +"_run_exitfuncs($module, /)\n\ +--\n\ \n\ Run all registered exit functions.\n\ \n\ @@ -236,7 +224,8 @@ atexit_run_exitfuncs(PyObject *module, PyObject *unused) } PyDoc_STRVAR(atexit_clear__doc__, -"_clear() -> None\n\ +"_clear($module, /)\n\ +--\n\ \n\ Clear the list of previously registered exit functions."); @@ -248,7 +237,8 @@ atexit_clear(PyObject *module, PyObject *unused) } PyDoc_STRVAR(atexit_ncallbacks__doc__, -"_ncallbacks() -> int\n\ +"_ncallbacks($module, /)\n\ +--\n\ \n\ Return the number of registered exit functions."); @@ -256,11 +246,38 @@ static PyObject * atexit_ncallbacks(PyObject *module, PyObject *unused) { struct atexit_state *state = get_atexit_state(); - return PyLong_FromSsize_t(state->ncallbacks); + assert(state->callbacks != NULL); + assert(PyList_CheckExact(state->callbacks)); + return PyLong_FromSsize_t(PyList_GET_SIZE(state->callbacks)); +} + +static int +atexit_unregister_locked(PyObject *callbacks, PyObject *func) +{ + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(callbacks); ++i) { + PyObject *tuple = PyList_GET_ITEM(callbacks, i); + assert(PyTuple_CheckExact(tuple)); + PyObject *to_compare = PyTuple_GET_ITEM(tuple, 0); + int cmp = PyObject_RichCompareBool(func, to_compare, Py_EQ); + if (cmp < 0) + { + return -1; + } + if (cmp == 1) { + // We found a callback! + if (PyList_SetSlice(callbacks, i, i + 1, NULL) < 0) { + return -1; + } + --i; + } + } + + return 0; } PyDoc_STRVAR(atexit_unregister__doc__, -"unregister(func) -> None\n\ +"unregister($module, func, /)\n\ +--\n\ \n\ Unregister an exit function which was previously registered using\n\ atexit.register\n\ @@ -271,22 +288,11 @@ static PyObject * atexit_unregister(PyObject *module, PyObject *func) { struct atexit_state *state = get_atexit_state(); - for (int i = 0; i < state->ncallbacks; i++) - { - atexit_py_callback *cb = state->callbacks[i]; - if (cb == NULL) { - continue; - } - - int eq = PyObject_RichCompareBool(cb->func, func, Py_EQ); - if (eq < 0) { - return NULL; - } - if (eq) { - atexit_delete_cb(state, i); - } - } - Py_RETURN_NONE; + int result; + Py_BEGIN_CRITICAL_SECTION(state->callbacks); + result = atexit_unregister_locked(state->callbacks, func); + Py_END_CRITICAL_SECTION(); + return result < 0 ? NULL : Py_None; } @@ -317,6 +323,7 @@ Two public functions, register and unregister, are defined.\n\ static PyModuleDef_Slot atexitmodule_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/binascii.c b/Modules/binascii.c index 86493241a1fb7e..6bb01d148b6faa 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -414,6 +414,13 @@ binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode) if (this_ch == BASE64_PAD) { padding_started = 1; + if (strict_mode && quad_pos == 0) { + state = get_binascii_state(module); + if (state) { + PyErr_SetString(state->Error, "Excess padding not allowed"); + } + goto error_end; + } if (quad_pos >= 2 && quad_pos + ++pads >= 4) { /* A pad sequence means we should not parse more input. ** We've already interpreted the data from the quad at this point. @@ -1278,6 +1285,7 @@ binascii_exec(PyObject *module) static PyModuleDef_Slot binascii_slots[] = { {Py_mod_exec, binascii_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/blake2module.c b/Modules/blake2module.c new file mode 100644 index 00000000000000..016c834c01bbe2 --- /dev/null +++ b/Modules/blake2module.c @@ -0,0 +1,1000 @@ +/* + * Written in 2013 by Dmitry Chestnykh + * Modified for CPython by Christian Heimes + * Updated to use HACL* by Jonathan Protzenko + * + * To the extent possible under law, the author have dedicated all + * copyright and related and neighboring rights to this software to + * the public domain worldwide. This software is distributed without + * any warranty. http://creativecommons.org/publicdomain/zero/1.0/ + */ + +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +#include "pyconfig.h" +#include "Python.h" +#include "hashlib.h" +#include "pycore_strhex.h" // _Py_strhex() +#include "pycore_typeobject.h" +#include "pycore_moduleobject.h" + +// QUICK CPU AUTODETECTION +// +// See https://github.com/python/cpython/pull/119316 -- we only enable +// vectorized versions for Intel CPUs, even though HACL*'s "vec128" modules also +// run on ARM NEON. (We could enable them on POWER -- but I don't have access to +// a test machine to see if that speeds anything up.) +// +// Note that configure.ac and the rest of the build are written in such a way +// that if the configure script finds suitable flags to compile HACL's SIMD128 +// (resp. SIMD256) files, then Hacl_Hash_Blake2b_Simd128.c (resp. ...) will be +// pulled into the build automatically, and then only the CPU autodetection will +// need to be updated here. + +#if defined(__x86_64__) && defined(__GNUC__) +#include +#elif defined(_M_X64) +#include +#endif + +#include + +// SIMD256 can't be compiled on macOS ARM64, and performance of SIMD128 isn't +// great; but when compiling a universal2 binary, autoconf will set +// HACL_CAN_COMPILE_SIMD128 and HACL_CAN_COMPILE_SIMD256 because they *can* be +// compiled on x86_64. If we're on macOS ARM64, disable these preprocessor +// symbols. +#if defined(__APPLE__) && defined(__arm64__) +# undef HACL_CAN_COMPILE_SIMD128 +# undef HACL_CAN_COMPILE_SIMD256 +#endif + +// ECX +#define ECX_SSE3 (1 << 0) +#define ECX_SSSE3 (1 << 9) +#define ECX_SSE4_1 (1 << 19) +#define ECX_SSE4_2 (1 << 20) +#define ECX_AVX (1 << 28) + +// EBX +#define EBX_AVX2 (1 << 5) + +// EDX +#define EDX_SSE (1 << 25) +#define EDX_SSE2 (1 << 26) +#define EDX_CMOV (1 << 15) + +// zero-initialized by default +typedef struct { + bool sse, sse2, sse3, sse41, sse42, cmov, avx, avx2; + bool done; +} cpu_flags; + +void detect_cpu_features(cpu_flags *flags) { + if (!flags->done) { + int eax1 = 0, ebx1 = 0, ecx1 = 0, edx1 = 0; + int eax7 = 0, ebx7 = 0, ecx7 = 0, edx7 = 0; +#if defined(__x86_64__) && defined(__GNUC__) + __cpuid_count(1, 0, eax1, ebx1, ecx1, edx1); + __cpuid_count(7, 0, eax7, ebx7, ecx7, edx7); +#elif defined(_M_X64) + int info1[4] = { 0 }; + int info7[4] = { 0 }; + __cpuidex(info1, 1, 0); + __cpuidex(info7, 7, 0); + eax1 = info1[0]; + ebx1 = info1[1]; + ecx1 = info1[2]; + edx1 = info1[3]; + eax7 = info7[0]; + ebx7 = info7[1]; + ecx7 = info7[2]; + edx7 = info7[3]; +#else + (void) eax1; (void) ebx1; (void) ecx1; (void) edx1; + (void) eax7; (void) ebx7; (void) ecx7; (void) edx7; +#endif + + flags->avx = (ecx1 & ECX_AVX) != 0; + + flags->avx2 = (ebx7 & EBX_AVX2) != 0; + + flags->sse = (edx1 & EDX_SSE) != 0; + flags->sse2 = (edx1 & EDX_SSE2) != 0; + flags->cmov = (edx1 & EDX_CMOV) != 0; + + flags->sse3 = (ecx1 & ECX_SSE3) != 0; + /* ssse3 = (ecx1 & ECX_SSSE3) != 0; */ + flags->sse41 = (ecx1 & ECX_SSE4_1) != 0; + flags->sse42 = (ecx1 & ECX_SSE4_2) != 0; + + flags->done = true; + } +} + +#ifdef HACL_CAN_COMPILE_SIMD128 +static inline bool has_simd128(cpu_flags *flags) { + // For now this is Intel-only, could conceivably be #ifdef'd to something + // else. + return flags->sse && flags->sse2 && flags->sse3 && flags->sse41 && flags->sse42 && flags->cmov; +} +#endif + +#ifdef HACL_CAN_COMPILE_SIMD256 +static inline bool has_simd256(cpu_flags *flags) { + return flags->avx && flags->avx2; +} +#endif + +// Small mismatch between the variable names Python defines as part of configure +// at the ones HACL* expects to be set in order to enable those headers. +#define HACL_CAN_COMPILE_VEC128 HACL_CAN_COMPILE_SIMD128 +#define HACL_CAN_COMPILE_VEC256 HACL_CAN_COMPILE_SIMD256 + +#include "_hacl/Hacl_Hash_Blake2b.h" +#include "_hacl/Hacl_Hash_Blake2s.h" +#if HACL_CAN_COMPILE_SIMD256 +#include "_hacl/Hacl_Hash_Blake2b_Simd256.h" +#endif +#if HACL_CAN_COMPILE_SIMD128 +#include "_hacl/Hacl_Hash_Blake2s_Simd128.h" +#endif + +// MODULE TYPE SLOTS + +static PyType_Spec blake2b_type_spec; +static PyType_Spec blake2s_type_spec; + +PyDoc_STRVAR(blake2mod__doc__, +"_blake2b provides BLAKE2b for hashlib\n" +); + +typedef struct { + PyTypeObject* blake2b_type; + PyTypeObject* blake2s_type; + cpu_flags flags; +} Blake2State; + +static inline Blake2State* +blake2_get_state(PyObject *module) +{ + void *state = _PyModule_GetState(module); + assert(state != NULL); + return (Blake2State *)state; +} + +#if defined(HACL_CAN_COMPILE_SIMD128) || defined(HACL_CAN_COMPILE_SIMD256) +static inline Blake2State* +blake2_get_state_from_type(PyTypeObject *module) +{ + void *state = _PyType_GetModuleState(module); + assert(state != NULL); + return (Blake2State *)state; +} +#endif + +static struct PyMethodDef blake2mod_functions[] = { + {NULL, NULL} +}; + +static int +_blake2_traverse(PyObject *module, visitproc visit, void *arg) +{ + Blake2State *state = blake2_get_state(module); + Py_VISIT(state->blake2b_type); + Py_VISIT(state->blake2s_type); + return 0; +} + +static int +_blake2_clear(PyObject *module) +{ + Blake2State *state = blake2_get_state(module); + Py_CLEAR(state->blake2b_type); + Py_CLEAR(state->blake2s_type); + return 0; +} + +static void +_blake2_free(void *module) +{ + (void)_blake2_clear((PyObject *)module); +} + +#define ADD_INT(d, name, value) do { \ + PyObject *x = PyLong_FromLong(value); \ + if (!x) \ + return -1; \ + if (PyDict_SetItemString(d, name, x) < 0) { \ + Py_DECREF(x); \ + return -1; \ + } \ + Py_DECREF(x); \ +} while(0) + +#define ADD_INT_CONST(NAME, VALUE) do { \ + if (PyModule_AddIntConstant(m, NAME, VALUE) < 0) { \ + return -1; \ + } \ +} while (0) + +static int +blake2_exec(PyObject *m) +{ + Blake2State* st = blake2_get_state(m); + + // This is called at module initialization-time, and so appears to be as + // good a place as any to probe the CPU flags. + detect_cpu_features(&st->flags); + + st->blake2b_type = (PyTypeObject *)PyType_FromModuleAndSpec( + m, &blake2b_type_spec, NULL); + + if (st->blake2b_type == NULL) { + return -1; + } + /* BLAKE2b */ + if (PyModule_AddType(m, st->blake2b_type) < 0) { + return -1; + } + + PyObject *d = st->blake2b_type->tp_dict; + ADD_INT(d, "SALT_SIZE", HACL_HASH_BLAKE2B_SALT_BYTES); + ADD_INT(d, "PERSON_SIZE", HACL_HASH_BLAKE2B_PERSONAL_BYTES); + ADD_INT(d, "MAX_KEY_SIZE", HACL_HASH_BLAKE2B_KEY_BYTES); + ADD_INT(d, "MAX_DIGEST_SIZE", HACL_HASH_BLAKE2B_OUT_BYTES); + + ADD_INT_CONST("BLAKE2B_SALT_SIZE", HACL_HASH_BLAKE2B_SALT_BYTES); + ADD_INT_CONST("BLAKE2B_PERSON_SIZE", HACL_HASH_BLAKE2B_PERSONAL_BYTES); + ADD_INT_CONST("BLAKE2B_MAX_KEY_SIZE", HACL_HASH_BLAKE2B_KEY_BYTES); + ADD_INT_CONST("BLAKE2B_MAX_DIGEST_SIZE", HACL_HASH_BLAKE2B_OUT_BYTES); + + /* BLAKE2s */ + st->blake2s_type = (PyTypeObject *)PyType_FromModuleAndSpec( + m, &blake2s_type_spec, NULL); + + if (NULL == st->blake2s_type) + return -1; + + if (PyModule_AddType(m, st->blake2s_type) < 0) { + return -1; + } + + d = st->blake2s_type->tp_dict; + ADD_INT(d, "SALT_SIZE", HACL_HASH_BLAKE2S_SALT_BYTES); + ADD_INT(d, "PERSON_SIZE", HACL_HASH_BLAKE2S_PERSONAL_BYTES); + ADD_INT(d, "MAX_KEY_SIZE", HACL_HASH_BLAKE2S_KEY_BYTES); + ADD_INT(d, "MAX_DIGEST_SIZE", HACL_HASH_BLAKE2S_OUT_BYTES); + + ADD_INT_CONST("BLAKE2S_SALT_SIZE", HACL_HASH_BLAKE2S_SALT_BYTES); + ADD_INT_CONST("BLAKE2S_PERSON_SIZE", HACL_HASH_BLAKE2S_PERSONAL_BYTES); + ADD_INT_CONST("BLAKE2S_MAX_KEY_SIZE", HACL_HASH_BLAKE2S_KEY_BYTES); + ADD_INT_CONST("BLAKE2S_MAX_DIGEST_SIZE", HACL_HASH_BLAKE2S_OUT_BYTES); + + return 0; +} + +#undef ADD_INT +#undef ADD_INT_CONST + +static PyModuleDef_Slot _blake2_slots[] = { + {Py_mod_exec, blake2_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL} +}; + +static struct PyModuleDef blake2_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_blake2", + .m_doc = blake2mod__doc__, + .m_size = sizeof(Blake2State), + .m_methods = blake2mod_functions, + .m_slots = _blake2_slots, + .m_traverse = _blake2_traverse, + .m_clear = _blake2_clear, + .m_free = _blake2_free, +}; + +PyMODINIT_FUNC +PyInit__blake2(void) +{ + return PyModuleDef_Init(&blake2_module); +} + +// IMPLEMENTATION OF METHODS + +// The HACL* API does not offer an agile API that can deal with either Blake2S +// or Blake2B -- the reason is that the underlying states are optimized (uint32s +// for S, uint64s for B). Therefore, we use a tagged union in this module to +// correctly dispatch. Note that the previous incarnation of this code +// transformed the Blake2b implementation into the Blake2s one using a script, +// so this is an improvement. +// +// The 128 and 256 versions are only available if i) we were able to compile +// them, and ii) if the CPU we run on also happens to have the right instruction +// set. +typedef enum { Blake2s, Blake2b, Blake2s_128, Blake2b_256 } blake2_impl; + +static inline bool is_blake2b(blake2_impl impl) { + return impl == Blake2b || impl == Blake2b_256; +} + +static inline bool is_blake2s(blake2_impl impl) { + return !is_blake2b(impl); +} + +static inline blake2_impl type_to_impl(PyTypeObject *type) { +#if defined(HACL_CAN_COMPILE_SIMD128) || defined(HACL_CAN_COMPILE_SIMD256) + Blake2State* st = blake2_get_state_from_type(type); +#endif + if (!strcmp(type->tp_name, blake2b_type_spec.name)) { +#ifdef HACL_CAN_COMPILE_SIMD256 + if (has_simd256(&st->flags)) + return Blake2b_256; + else +#endif + return Blake2b; + } else if (!strcmp(type->tp_name, blake2s_type_spec.name)) { +#ifdef HACL_CAN_COMPILE_SIMD128 + if (has_simd128(&st->flags)) + return Blake2s_128; + else +#endif + return Blake2s; + } else { + Py_UNREACHABLE(); + } +} + +typedef struct { + PyObject_HEAD + union { + Hacl_Hash_Blake2s_state_t *blake2s_state; + Hacl_Hash_Blake2b_state_t *blake2b_state; +#ifdef HACL_CAN_COMPILE_SIMD128 + Hacl_Hash_Blake2s_Simd128_state_t *blake2s_128_state; +#endif +#ifdef HACL_CAN_COMPILE_SIMD256 + Hacl_Hash_Blake2b_Simd256_state_t *blake2b_256_state; +#endif + }; + blake2_impl impl; + bool use_mutex; + PyMutex mutex; +} Blake2Object; + +#define _Blake2Object_CAST(op) ((Blake2Object *)(op)) + +#include "clinic/blake2module.c.h" + +/*[clinic input] +module _blake2 +class _blake2.blake2b "Blake2Object *" "&PyBlake2_BLAKE2bType" +class _blake2.blake2s "Blake2Object *" "&PyBlake2_BLAKE2sType" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b7526666bd18af83]*/ + + +static Blake2Object * +new_Blake2Object(PyTypeObject *type) +{ + Blake2Object *self = PyObject_GC_New(Blake2Object, type); + if (self == NULL) { + return NULL; + } + HASHLIB_INIT_MUTEX(self); + + PyObject_GC_Track(self); + return self; +} + +/* HACL* takes a uint32_t for the length of its parameter, but Py_ssize_t can be + * 64 bits so we loop in <4gig chunks when needed. */ + +#if PY_SSIZE_T_MAX > UINT32_MAX +#define HACL_UPDATE_LOOP(update,state,buf,len) \ + while (len > UINT32_MAX) { \ + update(state, buf, UINT32_MAX); \ + len -= UINT32_MAX; \ + buf += UINT32_MAX; \ + } +#else +#define HACL_UPDATE_LOOP(update,state,buf,len) +#endif + +#define HACL_UPDATE(update,state,buf,len) do { \ + /* Note: we explicitly ignore the error code on the basis that it would take > + * 1 billion years to overflow the maximum admissible length for SHA2-256 + * (namely, 2^61-1 bytes). */ \ + HACL_UPDATE_LOOP(update,state,buf,len) \ + /* Cast to uint32_t is safe: len <= UINT32_MAX at this point. */ \ + update(state, buf, (uint32_t) len); \ +} while (0) + +static void update(Blake2Object *self, uint8_t *buf, Py_ssize_t len) { + switch (self->impl) { + // These need to be ifdef'd out otherwise it's an unresolved symbol at + // link-time. +#ifdef HACL_CAN_COMPILE_SIMD256 + case Blake2b_256: + HACL_UPDATE(Hacl_Hash_Blake2b_Simd256_update,self->blake2b_256_state, buf, len); + return; +#endif +#ifdef HACL_CAN_COMPILE_SIMD128 + case Blake2s_128: + HACL_UPDATE(Hacl_Hash_Blake2s_Simd128_update,self->blake2s_128_state, buf, len); + return; +#endif + case Blake2b: + HACL_UPDATE(Hacl_Hash_Blake2b_update,self->blake2b_state, buf, len); + return; + case Blake2s: + HACL_UPDATE(Hacl_Hash_Blake2s_update,self->blake2s_state, buf, len); + return; + default: + Py_UNREACHABLE(); + } +} + +static PyObject * +py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, + Py_buffer *key, Py_buffer *salt, Py_buffer *person, + int fanout, int depth, unsigned long leaf_size, + unsigned long long node_offset, int node_depth, + int inner_size, int last_node, int usedforsecurity) + +{ + Blake2Object *self = NULL; + Py_buffer buf; + + self = new_Blake2Object(type); + if (self == NULL) { + goto error; + } + + self->impl = type_to_impl(type); + // Ensure that the states are NULL-initialized in case of an error. + // See: py_blake2_clear() for more details. + switch (self->impl) { +#if HACL_CAN_COMPILE_SIMD256 + case Blake2b_256: + self->blake2b_256_state = NULL; + break; +#endif +#if HACL_CAN_COMPILE_SIMD128 + case Blake2s_128: + self->blake2s_128_state = NULL; + break; +#endif + case Blake2b: + self->blake2b_state = NULL; + break; + case Blake2s: + self->blake2s_state = NULL; + break; + default: + Py_UNREACHABLE(); + } + // Using Blake2b because we statically know that these are greater than the + // Blake2s sizes -- this avoids a VLA. + uint8_t salt_[HACL_HASH_BLAKE2B_SALT_BYTES] = { 0 }; + uint8_t personal_[HACL_HASH_BLAKE2B_PERSONAL_BYTES] = { 0 }; + + /* Validate digest size. */ + if (digest_size <= 0 || + (unsigned) digest_size > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_OUT_BYTES : HACL_HASH_BLAKE2S_OUT_BYTES)) + { + PyErr_Format(PyExc_ValueError, + "digest_size for %s must be between 1 and %d bytes, here it is %d", + is_blake2b(self->impl) ? "Blake2b" : "Blake2s", + is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_OUT_BYTES : HACL_HASH_BLAKE2S_OUT_BYTES, + digest_size); + goto error; + } + + /* Validate salt parameter. */ + if ((salt->obj != NULL) && salt->len) { + if ((size_t)salt->len > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_SALT_BYTES : HACL_HASH_BLAKE2S_SALT_BYTES)) { + PyErr_Format(PyExc_ValueError, + "maximum salt length is %d bytes", + (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_SALT_BYTES : HACL_HASH_BLAKE2S_SALT_BYTES)); + goto error; + } + memcpy(salt_, salt->buf, salt->len); + } + + /* Validate personalization parameter. */ + if ((person->obj != NULL) && person->len) { + if ((size_t)person->len > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_PERSONAL_BYTES : HACL_HASH_BLAKE2S_PERSONAL_BYTES)) { + PyErr_Format(PyExc_ValueError, + "maximum person length is %d bytes", + (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_PERSONAL_BYTES : HACL_HASH_BLAKE2S_PERSONAL_BYTES)); + goto error; + } + memcpy(personal_, person->buf, person->len); + } + + /* Validate tree parameters. */ + if (fanout < 0 || fanout > 255) { + PyErr_SetString(PyExc_ValueError, + "fanout must be between 0 and 255"); + goto error; + } + + if (depth <= 0 || depth > 255) { + PyErr_SetString(PyExc_ValueError, + "depth must be between 1 and 255"); + goto error; + } + + if (leaf_size > 0xFFFFFFFFU) { + PyErr_SetString(PyExc_OverflowError, "leaf_size is too large"); + goto error; + } + + if (is_blake2s(self->impl) && node_offset > 0xFFFFFFFFFFFFULL) { + /* maximum 2**48 - 1 */ + PyErr_SetString(PyExc_OverflowError, "node_offset is too large"); + goto error; + } + + if (node_depth < 0 || node_depth > 255) { + PyErr_SetString(PyExc_ValueError, + "node_depth must be between 0 and 255"); + goto error; + } + + if (inner_size < 0 || + (unsigned) inner_size > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_OUT_BYTES : HACL_HASH_BLAKE2S_OUT_BYTES)) { + PyErr_Format(PyExc_ValueError, + "inner_size must be between 0 and is %d", + (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_OUT_BYTES : HACL_HASH_BLAKE2S_OUT_BYTES)); + goto error; + } + + /* Set key length. */ + if ((key->obj != NULL) && key->len) { + if ((size_t)key->len > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_KEY_BYTES : HACL_HASH_BLAKE2S_KEY_BYTES)) { + PyErr_Format(PyExc_ValueError, + "maximum key length is %d bytes", + (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_KEY_BYTES : HACL_HASH_BLAKE2S_KEY_BYTES)); + goto error; + } + } + + // Unlike the state types, the parameters share a single (client-friendly) + // structure. + + Hacl_Hash_Blake2b_blake2_params params = { + .digest_length = digest_size, + .key_length = (uint8_t)key->len, + .fanout = fanout, + .depth = depth, + .leaf_length = leaf_size, + .node_offset = node_offset, + .node_depth = node_depth, + .inner_length = inner_size, + .salt = salt_, + .personal = personal_ + }; + + switch (self->impl) { +#if HACL_CAN_COMPILE_SIMD256 + case Blake2b_256: + self->blake2b_256_state = Hacl_Hash_Blake2b_Simd256_malloc_with_params_and_key(¶ms, last_node, key->buf); + break; +#endif +#if HACL_CAN_COMPILE_SIMD128 + case Blake2s_128: + self->blake2s_128_state = Hacl_Hash_Blake2s_Simd128_malloc_with_params_and_key(¶ms, last_node, key->buf); + break; +#endif + case Blake2b: + self->blake2b_state = Hacl_Hash_Blake2b_malloc_with_params_and_key(¶ms, last_node, key->buf); + break; + case Blake2s: + self->blake2s_state = Hacl_Hash_Blake2s_malloc_with_params_and_key(¶ms, last_node, key->buf); + break; + default: + Py_UNREACHABLE(); + } + + /* Process initial data if any. */ + if (data != NULL) { + GET_BUFFER_VIEW_OR_ERROR(data, &buf, goto error); + + if (buf.len >= HASHLIB_GIL_MINSIZE) { + Py_BEGIN_ALLOW_THREADS + update(self, buf.buf, buf.len); + Py_END_ALLOW_THREADS + } else { + update(self, buf.buf, buf.len); + } + PyBuffer_Release(&buf); + } + + return (PyObject *)self; +error: + Py_XDECREF(self); + return NULL; +} + +/*[clinic input] +@classmethod +_blake2.blake2b.__new__ as py_blake2b_new + data: object(c_default="NULL") = b'' + / + * + digest_size: int(c_default="HACL_HASH_BLAKE2B_OUT_BYTES") = _blake2.blake2b.MAX_DIGEST_SIZE + key: Py_buffer(c_default="NULL", py_default="b''") = None + salt: Py_buffer(c_default="NULL", py_default="b''") = None + person: Py_buffer(c_default="NULL", py_default="b''") = None + fanout: int = 1 + depth: int = 1 + leaf_size: unsigned_long = 0 + node_offset: unsigned_long_long = 0 + node_depth: int = 0 + inner_size: int = 0 + last_node: bool = False + usedforsecurity: bool = True + +Return a new BLAKE2b hash object. +[clinic start generated code]*/ + +static PyObject * +py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size, + Py_buffer *key, Py_buffer *salt, Py_buffer *person, + int fanout, int depth, unsigned long leaf_size, + unsigned long long node_offset, int node_depth, + int inner_size, int last_node, int usedforsecurity) +/*[clinic end generated code: output=32bfd8f043c6896f input=8fee2b7b11428b2d]*/ +{ + return py_blake2b_or_s_new(type, data, digest_size, key, salt, person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); +} + +/*[clinic input] +@classmethod +_blake2.blake2s.__new__ as py_blake2s_new + data: object(c_default="NULL") = b'' + / + * + digest_size: int(c_default="HACL_HASH_BLAKE2S_OUT_BYTES") = _blake2.blake2s.MAX_DIGEST_SIZE + key: Py_buffer(c_default="NULL", py_default="b''") = None + salt: Py_buffer(c_default="NULL", py_default="b''") = None + person: Py_buffer(c_default="NULL", py_default="b''") = None + fanout: int = 1 + depth: int = 1 + leaf_size: unsigned_long = 0 + node_offset: unsigned_long_long = 0 + node_depth: int = 0 + inner_size: int = 0 + last_node: bool = False + usedforsecurity: bool = True + +Return a new BLAKE2s hash object. +[clinic start generated code]*/ + +static PyObject * +py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, + Py_buffer *key, Py_buffer *salt, Py_buffer *person, + int fanout, int depth, unsigned long leaf_size, + unsigned long long node_offset, int node_depth, + int inner_size, int last_node, int usedforsecurity) +/*[clinic end generated code: output=556181f73905c686 input=8165a11980eac7f3]*/ +{ + return py_blake2b_or_s_new(type, data, digest_size, key, salt, person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); +} + +/*[clinic input] +_blake2.blake2b.copy + +Return a copy of the hash object. +[clinic start generated code]*/ + +static PyObject * +_blake2_blake2b_copy_impl(Blake2Object *self) +/*[clinic end generated code: output=622d1c56b91c50d8 input=e383c2d199fd8a2e]*/ +{ + Blake2Object *cpy; + + if ((cpy = new_Blake2Object(Py_TYPE(self))) == NULL) + return NULL; + + ENTER_HASHLIB(self); + switch (self->impl) { +#if HACL_CAN_COMPILE_SIMD256 + case Blake2b_256: + cpy->blake2b_256_state = Hacl_Hash_Blake2b_Simd256_copy(self->blake2b_256_state); + break; +#endif +#if HACL_CAN_COMPILE_SIMD128 + case Blake2s_128: + cpy->blake2s_128_state = Hacl_Hash_Blake2s_Simd128_copy(self->blake2s_128_state); + break; +#endif + case Blake2b: + cpy->blake2b_state = Hacl_Hash_Blake2b_copy(self->blake2b_state); + break; + case Blake2s: + cpy->blake2s_state = Hacl_Hash_Blake2s_copy(self->blake2s_state); + break; + default: + Py_UNREACHABLE(); + } + cpy->impl = self->impl; + LEAVE_HASHLIB(self); + return (PyObject *)cpy; +} + +/*[clinic input] +_blake2.blake2b.update + + data: object + / + +Update this hash object's state with the provided bytes-like object. +[clinic start generated code]*/ + +static PyObject * +_blake2_blake2b_update(Blake2Object *self, PyObject *data) +/*[clinic end generated code: output=e6d1ac88471df308 input=ffc4aa6a6a225d31]*/ +{ + Py_buffer buf; + + GET_BUFFER_VIEW_OR_ERROUT(data, &buf); + + if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { + self->use_mutex = true; + } + if (self->use_mutex) { + Py_BEGIN_ALLOW_THREADS + PyMutex_Lock(&self->mutex); + update(self, buf.buf, buf.len); + PyMutex_Unlock(&self->mutex); + Py_END_ALLOW_THREADS + } else { + update(self, buf.buf, buf.len); + } + + PyBuffer_Release(&buf); + + Py_RETURN_NONE; +} + +/*[clinic input] +_blake2.blake2b.digest + +Return the digest value as a bytes object. +[clinic start generated code]*/ + +static PyObject * +_blake2_blake2b_digest_impl(Blake2Object *self) +/*[clinic end generated code: output=31ab8ad477f4a2f7 input=7d21659e9c5fff02]*/ +{ + uint8_t digest[HACL_HASH_BLAKE2B_OUT_BYTES]; + + ENTER_HASHLIB(self); + uint8_t digest_length = 0; + switch (self->impl) { +#if HACL_CAN_COMPILE_SIMD256 + case Blake2b_256: + digest_length = Hacl_Hash_Blake2b_Simd256_digest(self->blake2b_256_state, digest); + break; +#endif +#if HACL_CAN_COMPILE_SIMD128 + case Blake2s_128: + digest_length = Hacl_Hash_Blake2s_Simd128_digest(self->blake2s_128_state, digest); + break; +#endif + case Blake2b: + digest_length = Hacl_Hash_Blake2b_digest(self->blake2b_state, digest); + break; + case Blake2s: + digest_length = Hacl_Hash_Blake2s_digest(self->blake2s_state, digest); + break; + default: + Py_UNREACHABLE(); + } + LEAVE_HASHLIB(self); + return PyBytes_FromStringAndSize((const char *)digest, digest_length); +} + +/*[clinic input] +_blake2.blake2b.hexdigest + +Return the digest value as a string of hexadecimal digits. +[clinic start generated code]*/ + +static PyObject * +_blake2_blake2b_hexdigest_impl(Blake2Object *self) +/*[clinic end generated code: output=5ef54b138db6610a input=76930f6946351f56]*/ +{ + uint8_t digest[HACL_HASH_BLAKE2B_OUT_BYTES]; + + ENTER_HASHLIB(self); + uint8_t digest_length = 0; + switch (self->impl) { +#if HACL_CAN_COMPILE_SIMD256 + case Blake2b_256: + digest_length = Hacl_Hash_Blake2b_Simd256_digest(self->blake2b_256_state, digest); + break; +#endif +#if HACL_CAN_COMPILE_SIMD128 + case Blake2s_128: + digest_length = Hacl_Hash_Blake2s_Simd128_digest(self->blake2s_128_state, digest); + break; +#endif + case Blake2b: + digest_length = Hacl_Hash_Blake2b_digest(self->blake2b_state, digest); + break; + case Blake2s: + digest_length = Hacl_Hash_Blake2s_digest(self->blake2s_state, digest); + break; + default: + Py_UNREACHABLE(); + } + LEAVE_HASHLIB(self); + return _Py_strhex((const char *)digest, digest_length); +} + + +static PyMethodDef py_blake2b_methods[] = { + _BLAKE2_BLAKE2B_COPY_METHODDEF + _BLAKE2_BLAKE2B_DIGEST_METHODDEF + _BLAKE2_BLAKE2B_HEXDIGEST_METHODDEF + _BLAKE2_BLAKE2B_UPDATE_METHODDEF + {NULL, NULL} +}; + + +static PyObject * +py_blake2b_get_name(PyObject *op, void *Py_UNUSED(closure)) +{ + Blake2Object *self = _Blake2Object_CAST(op); + return PyUnicode_FromString(is_blake2b(self->impl) ? "blake2b" : "blake2s"); +} + + + +static PyObject * +py_blake2b_get_block_size(PyObject *op, void *Py_UNUSED(closure)) +{ + Blake2Object *self = _Blake2Object_CAST(op); + return PyLong_FromLong(is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_BLOCK_BYTES : HACL_HASH_BLAKE2S_BLOCK_BYTES); +} + + + +static PyObject * +py_blake2b_get_digest_size(PyObject *op, void *Py_UNUSED(closure)) +{ + Blake2Object *self = _Blake2Object_CAST(op); + switch (self->impl) { +#if HACL_CAN_COMPILE_SIMD256 + case Blake2b_256: + return PyLong_FromLong(Hacl_Hash_Blake2b_Simd256_info(self->blake2b_256_state).digest_length); +#endif +#if HACL_CAN_COMPILE_SIMD128 + case Blake2s_128: + return PyLong_FromLong(Hacl_Hash_Blake2s_Simd128_info(self->blake2s_128_state).digest_length); +#endif + case Blake2b: + return PyLong_FromLong(Hacl_Hash_Blake2b_info(self->blake2b_state).digest_length); + case Blake2s: + return PyLong_FromLong(Hacl_Hash_Blake2s_info(self->blake2s_state).digest_length); + default: + Py_UNREACHABLE(); + } +} + + +static PyGetSetDef py_blake2b_getsetters[] = { + {"name", py_blake2b_get_name, NULL, NULL, NULL}, + {"block_size", py_blake2b_get_block_size, NULL, NULL, NULL}, + {"digest_size", py_blake2b_get_digest_size, NULL, NULL, NULL}, + {NULL} /* Sentinel */ +}; + + +static int +py_blake2_clear(PyObject *op) +{ + Blake2Object *self = (Blake2Object *)op; + // The initialization function uses PyObject_GC_New() but explicitly + // initializes the HACL* internal state to NULL before allocating + // it. If an error occurs in the constructor, we should only free + // states that were allocated (i.e. that are not NULL). + switch (self->impl) { +#if HACL_CAN_COMPILE_SIMD256 + case Blake2b_256: + if (self->blake2b_256_state != NULL) { + Hacl_Hash_Blake2b_Simd256_free(self->blake2b_256_state); + self->blake2b_256_state = NULL; + } + break; +#endif +#if HACL_CAN_COMPILE_SIMD128 + case Blake2s_128: + if (self->blake2s_128_state != NULL) { + Hacl_Hash_Blake2s_Simd128_free(self->blake2s_128_state); + self->blake2s_128_state = NULL; + } + break; +#endif + case Blake2b: + if (self->blake2b_state != NULL) { + Hacl_Hash_Blake2b_free(self->blake2b_state); + self->blake2b_state = NULL; + } + break; + case Blake2s: + if (self->blake2s_state != NULL) { + Hacl_Hash_Blake2s_free(self->blake2s_state); + self->blake2s_state = NULL; + } + break; + default: + Py_UNREACHABLE(); + } + return 0; +} + +static void +py_blake2_dealloc(PyObject *self) +{ + PyTypeObject *type = Py_TYPE(self); + PyObject_GC_UnTrack(self); + (void)py_blake2_clear(self); + type->tp_free(self); + Py_DECREF(type); +} + +static int +py_blake2_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + +static PyType_Slot blake2b_type_slots[] = { + {Py_tp_clear, py_blake2_clear}, + {Py_tp_dealloc, py_blake2_dealloc}, + {Py_tp_traverse, py_blake2_traverse}, + {Py_tp_doc, (char *)py_blake2b_new__doc__}, + {Py_tp_methods, py_blake2b_methods}, + {Py_tp_getset, py_blake2b_getsetters}, + {Py_tp_new, py_blake2b_new}, + {0,0} +}; + +static PyType_Slot blake2s_type_slots[] = { + {Py_tp_clear, py_blake2_clear}, + {Py_tp_dealloc, py_blake2_dealloc}, + {Py_tp_traverse, py_blake2_traverse}, + {Py_tp_doc, (char *)py_blake2s_new__doc__}, + {Py_tp_methods, py_blake2b_methods}, + {Py_tp_getset, py_blake2b_getsetters}, + // only the constructor differs, so that it can receive a clinic-generated + // default digest length suitable for blake2s + {Py_tp_new, py_blake2s_new}, + {0,0} +}; + +static PyType_Spec blake2b_type_spec = { + .name = "_blake2.blake2b", + .basicsize = sizeof(Blake2Object), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HEAPTYPE, + .slots = blake2b_type_slots +}; + +static PyType_Spec blake2s_type_spec = { + .name = "_blake2.blake2s", + .basicsize = sizeof(Blake2Object), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HEAPTYPE, + .slots = blake2s_type_slots +}; diff --git a/Modules/cjkcodecs/_codecs_iso2022.c b/Modules/cjkcodecs/_codecs_iso2022.c index e8835ad0909633..ef6faeb71274e1 100644 --- a/Modules/cjkcodecs/_codecs_iso2022.c +++ b/Modules/cjkcodecs/_codecs_iso2022.c @@ -806,7 +806,7 @@ jisx0213_encoder(const MultibyteCodec *codec, const Py_UCS4 *data, jisx0213_pair_encmap, JISX0213_ENCPAIRS); if (coded != DBCINV) return coded; - /* fall through */ + _Py_FALLTHROUGH; case -1: /* flush unterminated */ *length = 1; diff --git a/Modules/cjkcodecs/cjkcodecs.h b/Modules/cjkcodecs/cjkcodecs.h index 766f82983025e4..2b446ba5226ac0 100644 --- a/Modules/cjkcodecs/cjkcodecs.h +++ b/Modules/cjkcodecs/cjkcodecs.h @@ -503,6 +503,7 @@ static struct PyMethodDef _cjk_methods[] = { static PyModuleDef_Slot _cjk_slots[] = { {Py_mod_exec, _cjk_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/cjkcodecs/clinic/multibytecodec.c.h b/Modules/cjkcodecs/clinic/multibytecodec.c.h index b5639d5cf10a22..d77bbd48066354 100644 --- a/Modules/cjkcodecs/clinic/multibytecodec.c.h +++ b/Modules/cjkcodecs/clinic/multibytecodec.c.h @@ -12,7 +12,7 @@ PyDoc_STRVAR(_multibytecodec_MultibyteCodec_encode__doc__, "encode($self, /, input, errors=None)\n" "--\n" "\n" -"Return an encoded string version of `input\'.\n" +"Return an encoded string version of \'input\'.\n" "\n" "\'errors\' may be given to set a different error handling scheme. Default is\n" "\'strict\' meaning that encoding errors raise a UnicodeEncodeError. Other possible\n" @@ -28,7 +28,7 @@ _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self, const char *errors); static PyObject * -_multibytecodec_MultibyteCodec_encode(MultibyteCodecObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_multibytecodec_MultibyteCodec_encode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -61,7 +61,8 @@ _multibytecodec_MultibyteCodec_encode(MultibyteCodecObject *self, PyObject *cons PyObject *input; const char *errors = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -88,7 +89,7 @@ _multibytecodec_MultibyteCodec_encode(MultibyteCodecObject *self, PyObject *cons goto exit; } skip_optional_pos: - return_value = _multibytecodec_MultibyteCodec_encode_impl(self, input, errors); + return_value = _multibytecodec_MultibyteCodec_encode_impl((MultibyteCodecObject *)self, input, errors); exit: return return_value; @@ -114,7 +115,7 @@ _multibytecodec_MultibyteCodec_decode_impl(MultibyteCodecObject *self, const char *errors); static PyObject * -_multibytecodec_MultibyteCodec_decode(MultibyteCodecObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_multibytecodec_MultibyteCodec_decode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -147,7 +148,8 @@ _multibytecodec_MultibyteCodec_decode(MultibyteCodecObject *self, PyObject *cons Py_buffer input = {NULL, NULL}; const char *errors = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -176,7 +178,7 @@ _multibytecodec_MultibyteCodec_decode(MultibyteCodecObject *self, PyObject *cons goto exit; } skip_optional_pos: - return_value = _multibytecodec_MultibyteCodec_decode_impl(self, &input, errors); + return_value = _multibytecodec_MultibyteCodec_decode_impl((MultibyteCodecObject *)self, &input, errors); exit: /* Cleanup for input */ @@ -201,7 +203,7 @@ _multibytecodec_MultibyteIncrementalEncoder_encode_impl(MultibyteIncrementalEnco int final); static PyObject * -_multibytecodec_MultibyteIncrementalEncoder_encode(MultibyteIncrementalEncoderObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_multibytecodec_MultibyteIncrementalEncoder_encode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -234,7 +236,8 @@ _multibytecodec_MultibyteIncrementalEncoder_encode(MultibyteIncrementalEncoderOb PyObject *input; int final = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -247,7 +250,7 @@ _multibytecodec_MultibyteIncrementalEncoder_encode(MultibyteIncrementalEncoderOb goto exit; } skip_optional_pos: - return_value = _multibytecodec_MultibyteIncrementalEncoder_encode_impl(self, input, final); + return_value = _multibytecodec_MultibyteIncrementalEncoder_encode_impl((MultibyteIncrementalEncoderObject *)self, input, final); exit: return return_value; @@ -265,9 +268,9 @@ static PyObject * _multibytecodec_MultibyteIncrementalEncoder_getstate_impl(MultibyteIncrementalEncoderObject *self); static PyObject * -_multibytecodec_MultibyteIncrementalEncoder_getstate(MultibyteIncrementalEncoderObject *self, PyObject *Py_UNUSED(ignored)) +_multibytecodec_MultibyteIncrementalEncoder_getstate(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _multibytecodec_MultibyteIncrementalEncoder_getstate_impl(self); + return _multibytecodec_MultibyteIncrementalEncoder_getstate_impl((MultibyteIncrementalEncoderObject *)self); } PyDoc_STRVAR(_multibytecodec_MultibyteIncrementalEncoder_setstate__doc__, @@ -283,7 +286,7 @@ _multibytecodec_MultibyteIncrementalEncoder_setstate_impl(MultibyteIncrementalEn PyLongObject *statelong); static PyObject * -_multibytecodec_MultibyteIncrementalEncoder_setstate(MultibyteIncrementalEncoderObject *self, PyObject *arg) +_multibytecodec_MultibyteIncrementalEncoder_setstate(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; PyLongObject *statelong; @@ -293,7 +296,7 @@ _multibytecodec_MultibyteIncrementalEncoder_setstate(MultibyteIncrementalEncoder goto exit; } statelong = (PyLongObject *)arg; - return_value = _multibytecodec_MultibyteIncrementalEncoder_setstate_impl(self, statelong); + return_value = _multibytecodec_MultibyteIncrementalEncoder_setstate_impl((MultibyteIncrementalEncoderObject *)self, statelong); exit: return return_value; @@ -311,9 +314,9 @@ static PyObject * _multibytecodec_MultibyteIncrementalEncoder_reset_impl(MultibyteIncrementalEncoderObject *self); static PyObject * -_multibytecodec_MultibyteIncrementalEncoder_reset(MultibyteIncrementalEncoderObject *self, PyObject *Py_UNUSED(ignored)) +_multibytecodec_MultibyteIncrementalEncoder_reset(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _multibytecodec_MultibyteIncrementalEncoder_reset_impl(self); + return _multibytecodec_MultibyteIncrementalEncoder_reset_impl((MultibyteIncrementalEncoderObject *)self); } PyDoc_STRVAR(_multibytecodec_MultibyteIncrementalDecoder_decode__doc__, @@ -330,7 +333,7 @@ _multibytecodec_MultibyteIncrementalDecoder_decode_impl(MultibyteIncrementalDeco int final); static PyObject * -_multibytecodec_MultibyteIncrementalDecoder_decode(MultibyteIncrementalDecoderObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_multibytecodec_MultibyteIncrementalDecoder_decode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -363,7 +366,8 @@ _multibytecodec_MultibyteIncrementalDecoder_decode(MultibyteIncrementalDecoderOb Py_buffer input = {NULL, NULL}; int final = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -378,7 +382,7 @@ _multibytecodec_MultibyteIncrementalDecoder_decode(MultibyteIncrementalDecoderOb goto exit; } skip_optional_pos: - return_value = _multibytecodec_MultibyteIncrementalDecoder_decode_impl(self, &input, final); + return_value = _multibytecodec_MultibyteIncrementalDecoder_decode_impl((MultibyteIncrementalDecoderObject *)self, &input, final); exit: /* Cleanup for input */ @@ -401,9 +405,9 @@ static PyObject * _multibytecodec_MultibyteIncrementalDecoder_getstate_impl(MultibyteIncrementalDecoderObject *self); static PyObject * -_multibytecodec_MultibyteIncrementalDecoder_getstate(MultibyteIncrementalDecoderObject *self, PyObject *Py_UNUSED(ignored)) +_multibytecodec_MultibyteIncrementalDecoder_getstate(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _multibytecodec_MultibyteIncrementalDecoder_getstate_impl(self); + return _multibytecodec_MultibyteIncrementalDecoder_getstate_impl((MultibyteIncrementalDecoderObject *)self); } PyDoc_STRVAR(_multibytecodec_MultibyteIncrementalDecoder_setstate__doc__, @@ -419,7 +423,7 @@ _multibytecodec_MultibyteIncrementalDecoder_setstate_impl(MultibyteIncrementalDe PyObject *state); static PyObject * -_multibytecodec_MultibyteIncrementalDecoder_setstate(MultibyteIncrementalDecoderObject *self, PyObject *arg) +_multibytecodec_MultibyteIncrementalDecoder_setstate(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; PyObject *state; @@ -429,7 +433,7 @@ _multibytecodec_MultibyteIncrementalDecoder_setstate(MultibyteIncrementalDecoder goto exit; } state = arg; - return_value = _multibytecodec_MultibyteIncrementalDecoder_setstate_impl(self, state); + return_value = _multibytecodec_MultibyteIncrementalDecoder_setstate_impl((MultibyteIncrementalDecoderObject *)self, state); exit: return return_value; @@ -447,9 +451,9 @@ static PyObject * _multibytecodec_MultibyteIncrementalDecoder_reset_impl(MultibyteIncrementalDecoderObject *self); static PyObject * -_multibytecodec_MultibyteIncrementalDecoder_reset(MultibyteIncrementalDecoderObject *self, PyObject *Py_UNUSED(ignored)) +_multibytecodec_MultibyteIncrementalDecoder_reset(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _multibytecodec_MultibyteIncrementalDecoder_reset_impl(self); + return _multibytecodec_MultibyteIncrementalDecoder_reset_impl((MultibyteIncrementalDecoderObject *)self); } PyDoc_STRVAR(_multibytecodec_MultibyteStreamReader_read__doc__, @@ -465,7 +469,7 @@ _multibytecodec_MultibyteStreamReader_read_impl(MultibyteStreamReaderObject *sel PyObject *sizeobj); static PyObject * -_multibytecodec_MultibyteStreamReader_read(MultibyteStreamReaderObject *self, PyObject *const *args, Py_ssize_t nargs) +_multibytecodec_MultibyteStreamReader_read(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *sizeobj = Py_None; @@ -478,7 +482,7 @@ _multibytecodec_MultibyteStreamReader_read(MultibyteStreamReaderObject *self, Py } sizeobj = args[0]; skip_optional: - return_value = _multibytecodec_MultibyteStreamReader_read_impl(self, sizeobj); + return_value = _multibytecodec_MultibyteStreamReader_read_impl((MultibyteStreamReaderObject *)self, sizeobj); exit: return return_value; @@ -497,7 +501,7 @@ _multibytecodec_MultibyteStreamReader_readline_impl(MultibyteStreamReaderObject PyObject *sizeobj); static PyObject * -_multibytecodec_MultibyteStreamReader_readline(MultibyteStreamReaderObject *self, PyObject *const *args, Py_ssize_t nargs) +_multibytecodec_MultibyteStreamReader_readline(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *sizeobj = Py_None; @@ -510,7 +514,7 @@ _multibytecodec_MultibyteStreamReader_readline(MultibyteStreamReaderObject *self } sizeobj = args[0]; skip_optional: - return_value = _multibytecodec_MultibyteStreamReader_readline_impl(self, sizeobj); + return_value = _multibytecodec_MultibyteStreamReader_readline_impl((MultibyteStreamReaderObject *)self, sizeobj); exit: return return_value; @@ -529,7 +533,7 @@ _multibytecodec_MultibyteStreamReader_readlines_impl(MultibyteStreamReaderObject PyObject *sizehintobj); static PyObject * -_multibytecodec_MultibyteStreamReader_readlines(MultibyteStreamReaderObject *self, PyObject *const *args, Py_ssize_t nargs) +_multibytecodec_MultibyteStreamReader_readlines(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *sizehintobj = Py_None; @@ -542,7 +546,7 @@ _multibytecodec_MultibyteStreamReader_readlines(MultibyteStreamReaderObject *sel } sizehintobj = args[0]; skip_optional: - return_value = _multibytecodec_MultibyteStreamReader_readlines_impl(self, sizehintobj); + return_value = _multibytecodec_MultibyteStreamReader_readlines_impl((MultibyteStreamReaderObject *)self, sizehintobj); exit: return return_value; @@ -560,9 +564,9 @@ static PyObject * _multibytecodec_MultibyteStreamReader_reset_impl(MultibyteStreamReaderObject *self); static PyObject * -_multibytecodec_MultibyteStreamReader_reset(MultibyteStreamReaderObject *self, PyObject *Py_UNUSED(ignored)) +_multibytecodec_MultibyteStreamReader_reset(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _multibytecodec_MultibyteStreamReader_reset_impl(self); + return _multibytecodec_MultibyteStreamReader_reset_impl((MultibyteStreamReaderObject *)self); } PyDoc_STRVAR(_multibytecodec_MultibyteStreamWriter_write__doc__, @@ -579,7 +583,7 @@ _multibytecodec_MultibyteStreamWriter_write_impl(MultibyteStreamWriterObject *se PyObject *strobj); static PyObject * -_multibytecodec_MultibyteStreamWriter_write(MultibyteStreamWriterObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_multibytecodec_MultibyteStreamWriter_write(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -598,12 +602,13 @@ _multibytecodec_MultibyteStreamWriter_write(MultibyteStreamWriterObject *self, P PyObject *argsbuf[1]; PyObject *strobj; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } strobj = args[0]; - return_value = _multibytecodec_MultibyteStreamWriter_write_impl(self, cls, strobj); + return_value = _multibytecodec_MultibyteStreamWriter_write_impl((MultibyteStreamWriterObject *)self, cls, strobj); exit: return return_value; @@ -623,7 +628,7 @@ _multibytecodec_MultibyteStreamWriter_writelines_impl(MultibyteStreamWriterObjec PyObject *lines); static PyObject * -_multibytecodec_MultibyteStreamWriter_writelines(MultibyteStreamWriterObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_multibytecodec_MultibyteStreamWriter_writelines(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -642,12 +647,13 @@ _multibytecodec_MultibyteStreamWriter_writelines(MultibyteStreamWriterObject *se PyObject *argsbuf[1]; PyObject *lines; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } lines = args[0]; - return_value = _multibytecodec_MultibyteStreamWriter_writelines_impl(self, cls, lines); + return_value = _multibytecodec_MultibyteStreamWriter_writelines_impl((MultibyteStreamWriterObject *)self, cls, lines); exit: return return_value; @@ -666,13 +672,13 @@ _multibytecodec_MultibyteStreamWriter_reset_impl(MultibyteStreamWriterObject *se PyTypeObject *cls); static PyObject * -_multibytecodec_MultibyteStreamWriter_reset(MultibyteStreamWriterObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_multibytecodec_MultibyteStreamWriter_reset(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "reset() takes no arguments"); return NULL; } - return _multibytecodec_MultibyteStreamWriter_reset_impl(self, cls); + return _multibytecodec_MultibyteStreamWriter_reset_impl((MultibyteStreamWriterObject *)self, cls); } PyDoc_STRVAR(_multibytecodec___create_codec__doc__, @@ -682,4 +688,4 @@ PyDoc_STRVAR(_multibytecodec___create_codec__doc__, #define _MULTIBYTECODEC___CREATE_CODEC_METHODDEF \ {"__create_codec", (PyCFunction)_multibytecodec___create_codec, METH_O, _multibytecodec___create_codec__doc__}, -/*[clinic end generated code: output=ee767a6d93c7108a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6571941b8e45b013 input=a9049054013a1b77]*/ diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 2125da437963d2..08b74740bda4bf 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -56,6 +56,27 @@ class _multibytecodec.MultibyteStreamWriter "MultibyteStreamWriterObject *" "cli /*[clinic end generated code: output=da39a3ee5e6b4b0d input=305a76dfdd24b99c]*/ #undef clinic_get_state +#define _MultibyteCodec_CAST(op) ((MultibyteCodec *)(op)) +#define _MultibyteCodecObject_CAST(op) ((MultibyteCodecObject *)(op)) + +#define _MultibyteStatefulCodecContext_CAST(op) \ + ((MultibyteStatefulCodecContext *)(op)) + +#define _MultibyteStatefulEncoderContext_CAST(op) \ + ((MultibyteStatefulEncoderContext *)(op)) +#define _MultibyteStatefulDecoderContext_CAST(op) \ + ((MultibyteStatefulDecoderContext *)(op)) + +#define _MultibyteIncrementalEncoderObject_CAST(op) \ + ((MultibyteIncrementalEncoderObject *)(op)) +#define _MultibyteIncrementalDecoderObject_CAST(op) \ + ((MultibyteIncrementalDecoderObject *)(op)) + +#define _MultibyteStreamReaderObject_CAST(op) \ + ((MultibyteStreamReaderObject *)(op)) +#define _MultibyteStreamWriterObject_CAST(op) \ + ((MultibyteStreamWriterObject *)(op)) + typedef struct { PyObject *inobj; Py_ssize_t inpos, inlen; @@ -136,9 +157,10 @@ call_error_callback(PyObject *errors, PyObject *exc) } static PyObject * -codecctx_errors_get(MultibyteStatefulCodecContext *self, void *Py_UNUSED(ignored)) +codecctx_errors_get(PyObject *op, void *Py_UNUSED(closure)) { const char *errors; + MultibyteStatefulCodecContext *self = _MultibyteStatefulCodecContext_CAST(op); if (self->errors == ERROR_STRICT) errors = "strict"; @@ -154,11 +176,11 @@ codecctx_errors_get(MultibyteStatefulCodecContext *self, void *Py_UNUSED(ignored } static int -codecctx_errors_set(MultibyteStatefulCodecContext *self, PyObject *value, - void *closure) +codecctx_errors_set(PyObject *op, PyObject *value, void *Py_UNUSED(closure)) { PyObject *cb; const char *str; + MultibyteStatefulCodecContext *self = _MultibyteStatefulCodecContext_CAST(op); if (value == NULL) { PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); @@ -184,9 +206,8 @@ codecctx_errors_set(MultibyteStatefulCodecContext *self, PyObject *value, /* This getset handlers list is used by all the stateful codec objects */ static PyGetSetDef codecctx_getsets[] = { - {"errors", (getter)codecctx_errors_get, - (setter)codecctx_errors_set, - PyDoc_STR("how to treat errors")}, + {"errors", codecctx_errors_get, codecctx_errors_set, + PyDoc_STR("how to treat errors")}, {NULL,} }; @@ -574,7 +595,7 @@ _multibytecodec.MultibyteCodec.encode input: object errors: str(accept={str, NoneType}) = None -Return an encoded string version of `input'. +Return an encoded string version of 'input'. 'errors' may be given to set a different error handling scheme. Default is 'strict' meaning that encoding errors raise a UnicodeEncodeError. Other possible @@ -586,7 +607,7 @@ static PyObject * _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self, PyObject *input, const char *errors) -/*[clinic end generated code: output=7b26652045ba56a9 input=606d0e128a577bae]*/ +/*[clinic end generated code: output=7b26652045ba56a9 input=2841745b95ed338f]*/ { MultibyteCodec_State state; PyObject *errorcb, *r, *ucvt; @@ -669,7 +690,7 @@ _multibytecodec_MultibyteCodec_decode_impl(MultibyteCodecObject *self, if (datalen == 0) { ERROR_DECREF(errorcb); - return make_tuple(PyUnicode_New(0, 0), 0); + return make_tuple(Py_GetConstant(Py_CONSTANT_EMPTY_STR), 0); } _PyUnicodeWriter_Init(&buf.writer); @@ -719,22 +740,24 @@ static struct PyMethodDef multibytecodec_methods[] = { }; static int -multibytecodec_clear(MultibyteCodecObject *self) +multibytecodec_clear(PyObject *op) { + MultibyteCodecObject *self = _MultibyteCodecObject_CAST(op); Py_CLEAR(self->cjk_module); return 0; } static int -multibytecodec_traverse(MultibyteCodecObject *self, visitproc visit, void *arg) +multibytecodec_traverse(PyObject *op, visitproc visit, void *arg) { + MultibyteCodecObject *self = _MultibyteCodecObject_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->cjk_module); return 0; } static void -multibytecodec_dealloc(MultibyteCodecObject *self) +multibytecodec_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); PyTypeObject *tp = Py_TYPE(self); @@ -825,8 +848,15 @@ encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx, if (inpos < datalen) { if (datalen - inpos > MAXENCPENDING) { /* normal codecs can't reach here */ - PyErr_SetString(PyExc_UnicodeError, - "pending buffer overflow"); + PyObject *excobj = PyObject_CallFunction(PyExc_UnicodeEncodeError, + "sOnns", + ctx->codec->encoding, + inbuf, + inpos, datalen, + "pending buffer overflow"); + if (excobj == NULL) goto errorexit; + PyErr_SetObject(PyExc_UnicodeEncodeError, excobj); + Py_DECREF(excobj); goto errorexit; } ctx->pending = PyUnicode_Substring(inbuf, inpos, datalen); @@ -857,7 +887,16 @@ decoder_append_pending(MultibyteStatefulDecoderContext *ctx, npendings = (Py_ssize_t)(buf->inbuf_end - buf->inbuf); if (npendings + ctx->pendingsize > MAXDECPENDING || npendings > PY_SSIZE_T_MAX - ctx->pendingsize) { - PyErr_SetString(PyExc_UnicodeError, "pending buffer overflow"); + Py_ssize_t bufsize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top); + PyObject *excobj = PyUnicodeDecodeError_Create(ctx->codec->encoding, + (const char *)buf->inbuf_top, + bufsize, + 0, + bufsize, + "pending buffer overflow"); + if (excobj == NULL) return -1; + PyErr_SetObject(PyExc_UnicodeDecodeError, excobj); + Py_DECREF(excobj); return -1; } memcpy(ctx->pending + ctx->pendingsize, buf->inbuf, npendings); @@ -938,7 +977,17 @@ _multibytecodec_MultibyteIncrementalEncoder_getstate_impl(MultibyteIncrementalEn return NULL; } if (pendingsize > MAXENCPENDING*4) { - PyErr_SetString(PyExc_UnicodeError, "pending buffer too large"); + PyObject *excobj = PyObject_CallFunction(PyExc_UnicodeEncodeError, + "sOnns", + self->codec->encoding, + self->pending, + 0, PyUnicode_GET_LENGTH(self->pending), + "pending buffer too large"); + if (excobj == NULL) { + return NULL; + } + PyErr_SetObject(PyExc_UnicodeEncodeError, excobj); + Py_DECREF(excobj); return NULL; } statebytes[0] = (unsigned char)pendingsize; @@ -1080,17 +1129,18 @@ mbiencoder_init(PyObject *self, PyObject *args, PyObject *kwds) } static int -mbiencoder_traverse(MultibyteIncrementalEncoderObject *self, - visitproc visit, void *arg) +mbiencoder_traverse(PyObject *op, visitproc visit, void *arg) { + MultibyteIncrementalEncoderObject *self = _MultibyteIncrementalEncoderObject_CAST(op); if (ERROR_ISCUSTOM(self->errors)) Py_VISIT(self->errors); return 0; } static void -mbiencoder_dealloc(MultibyteIncrementalEncoderObject *self) +mbiencoder_dealloc(PyObject *op) { + MultibyteIncrementalEncoderObject *self = _MultibyteIncrementalEncoderObject_CAST(op); PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); ERROR_DECREF(self->errors); @@ -1267,7 +1317,13 @@ _multibytecodec_MultibyteIncrementalDecoder_setstate_impl(MultibyteIncrementalDe } if (buffersize > MAXDECPENDING) { - PyErr_SetString(PyExc_UnicodeError, "pending buffer too large"); + PyObject *excobj = PyUnicodeDecodeError_Create(self->codec->encoding, + PyBytes_AS_STRING(buffer), buffersize, + 0, buffersize, + "pending buffer too large"); + if (excobj == NULL) return NULL; + PyErr_SetObject(PyExc_UnicodeDecodeError, excobj); + Py_DECREF(excobj); return NULL; } @@ -1356,17 +1412,18 @@ mbidecoder_init(PyObject *self, PyObject *args, PyObject *kwds) } static int -mbidecoder_traverse(MultibyteIncrementalDecoderObject *self, - visitproc visit, void *arg) +mbidecoder_traverse(PyObject *op, visitproc visit, void *arg) { + MultibyteIncrementalDecoderObject *self = _MultibyteIncrementalDecoderObject_CAST(op); if (ERROR_ISCUSTOM(self->errors)) Py_VISIT(self->errors); return 0; } static void -mbidecoder_dealloc(MultibyteIncrementalDecoderObject *self) +mbidecoder_dealloc(PyObject *op) { + MultibyteIncrementalDecoderObject *self = _MultibyteIncrementalDecoderObject_CAST(op); PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); ERROR_DECREF(self->errors); @@ -1402,7 +1459,7 @@ mbstreamreader_iread(MultibyteStreamReaderObject *self, Py_ssize_t rsize; if (sizehint == 0) - return PyUnicode_New(0, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); _PyUnicodeWriter_Init(&buf.writer); buf.excobj = NULL; @@ -1672,9 +1729,9 @@ mbstreamreader_init(PyObject *self, PyObject *args, PyObject *kwds) } static int -mbstreamreader_traverse(MultibyteStreamReaderObject *self, - visitproc visit, void *arg) +mbstreamreader_traverse(PyObject *op, visitproc visit, void *arg) { + MultibyteStreamReaderObject *self = _MultibyteStreamReaderObject_CAST(op); if (ERROR_ISCUSTOM(self->errors)) Py_VISIT(self->errors); Py_VISIT(self->stream); @@ -1682,8 +1739,9 @@ mbstreamreader_traverse(MultibyteStreamReaderObject *self, } static void -mbstreamreader_dealloc(MultibyteStreamReaderObject *self) +mbstreamreader_dealloc(PyObject *op) { + MultibyteStreamReaderObject *self = _MultibyteStreamReaderObject_CAST(op); PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); ERROR_DECREF(self->errors); @@ -1895,9 +1953,9 @@ mbstreamwriter_init(PyObject *self, PyObject *args, PyObject *kwds) } static int -mbstreamwriter_traverse(MultibyteStreamWriterObject *self, - visitproc visit, void *arg) +mbstreamwriter_traverse(PyObject *op, visitproc visit, void *arg) { + MultibyteStreamWriterObject *self = _MultibyteStreamWriterObject_CAST(op); if (ERROR_ISCUSTOM(self->errors)) Py_VISIT(self->errors); Py_VISIT(self->stream); @@ -1905,8 +1963,9 @@ mbstreamwriter_traverse(MultibyteStreamWriterObject *self, } static void -mbstreamwriter_dealloc(MultibyteStreamWriterObject *self) +mbstreamwriter_dealloc(PyObject *op) { + MultibyteStreamWriterObject *self = _MultibyteStreamWriterObject_CAST(op); PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); ERROR_DECREF(self->errors); @@ -2012,7 +2071,7 @@ _multibytecodec_clear(PyObject *mod) static void _multibytecodec_free(void *mod) { - _multibytecodec_clear((PyObject *)mod); + (void)_multibytecodec_clear((PyObject *)mod); } #define CREATE_TYPE(module, type, spec) \ @@ -2062,6 +2121,7 @@ static struct PyMethodDef _multibytecodec_methods[] = { static PyModuleDef_Slot _multibytecodec_slots[] = { {Py_mod_exec, _multibytecodec_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index 6a9c8ff6d8fdd9..c6b7e39788be71 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -6,8 +6,34 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#if !defined(_asyncio_Future__asyncio_awaited_by_DOCSTR) +# define _asyncio_Future__asyncio_awaited_by_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__ASYNCIO_AWAITED_BY_GETSETDEF) +# undef _ASYNCIO_FUTURE__ASYNCIO_AWAITED_BY_GETSETDEF +# define _ASYNCIO_FUTURE__ASYNCIO_AWAITED_BY_GETSETDEF {"_asyncio_awaited_by", (getter)_asyncio_Future__asyncio_awaited_by_get, (setter)_asyncio_Future__asyncio_awaited_by_set, _asyncio_Future__asyncio_awaited_by_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__ASYNCIO_AWAITED_BY_GETSETDEF {"_asyncio_awaited_by", (getter)_asyncio_Future__asyncio_awaited_by_get, NULL, _asyncio_Future__asyncio_awaited_by_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__asyncio_awaited_by_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__asyncio_awaited_by_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__asyncio_awaited_by_get_impl((FutureObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_asyncio_Future___init____doc__, "Future(*, loop=None)\n" "--\n" @@ -63,7 +89,8 @@ _asyncio_Future___init__(PyObject *self, PyObject *args, PyObject *kwargs) Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; PyObject *loop = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 0, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -95,9 +122,15 @@ static PyObject * _asyncio_Future_result_impl(FutureObj *self); static PyObject * -_asyncio_Future_result(FutureObj *self, PyObject *Py_UNUSED(ignored)) +_asyncio_Future_result(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Future_result_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future_result_impl((FutureObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Future_exception__doc__, @@ -118,13 +151,20 @@ static PyObject * _asyncio_Future_exception_impl(FutureObj *self, PyTypeObject *cls); static PyObject * -_asyncio_Future_exception(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_asyncio_Future_exception(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "exception() takes no arguments"); - return NULL; + goto exit; } - return _asyncio_Future_exception_impl(self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future_exception_impl((FutureObj *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_asyncio_Future_set_result__doc__, @@ -144,7 +184,7 @@ _asyncio_Future_set_result_impl(FutureObj *self, PyTypeObject *cls, PyObject *result); static PyObject * -_asyncio_Future_set_result(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_asyncio_Future_set_result(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -163,12 +203,15 @@ _asyncio_Future_set_result(FutureObj *self, PyTypeObject *cls, PyObject *const * PyObject *argsbuf[1]; PyObject *result; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } result = args[0]; - return_value = _asyncio_Future_set_result_impl(self, cls, result); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future_set_result_impl((FutureObj *)self, cls, result); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -191,7 +234,7 @@ _asyncio_Future_set_exception_impl(FutureObj *self, PyTypeObject *cls, PyObject *exception); static PyObject * -_asyncio_Future_set_exception(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_asyncio_Future_set_exception(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -210,12 +253,15 @@ _asyncio_Future_set_exception(FutureObj *self, PyTypeObject *cls, PyObject *cons PyObject *argsbuf[1]; PyObject *exception; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } exception = args[0]; - return_value = _asyncio_Future_set_exception_impl(self, cls, exception); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future_set_exception_impl((FutureObj *)self, cls, exception); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -239,7 +285,7 @@ _asyncio_Future_add_done_callback_impl(FutureObj *self, PyTypeObject *cls, PyObject *fn, PyObject *context); static PyObject * -_asyncio_Future_add_done_callback(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_asyncio_Future_add_done_callback(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -272,7 +318,8 @@ _asyncio_Future_add_done_callback(FutureObj *self, PyTypeObject *cls, PyObject * PyObject *fn; PyObject *context = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -282,7 +329,9 @@ _asyncio_Future_add_done_callback(FutureObj *self, PyTypeObject *cls, PyObject * } context = args[1]; skip_optional_kwonly: - return_value = _asyncio_Future_add_done_callback_impl(self, cls, fn, context); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future_add_done_callback_impl((FutureObj *)self, cls, fn, context); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -304,7 +353,7 @@ _asyncio_Future_remove_done_callback_impl(FutureObj *self, PyTypeObject *cls, PyObject *fn); static PyObject * -_asyncio_Future_remove_done_callback(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_asyncio_Future_remove_done_callback(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -323,12 +372,15 @@ _asyncio_Future_remove_done_callback(FutureObj *self, PyTypeObject *cls, PyObjec PyObject *argsbuf[1]; PyObject *fn; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } fn = args[0]; - return_value = _asyncio_Future_remove_done_callback_impl(self, cls, fn); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future_remove_done_callback_impl((FutureObj *)self, cls, fn); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -352,7 +404,7 @@ _asyncio_Future_cancel_impl(FutureObj *self, PyTypeObject *cls, PyObject *msg); static PyObject * -_asyncio_Future_cancel(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_asyncio_Future_cancel(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -384,7 +436,8 @@ _asyncio_Future_cancel(FutureObj *self, PyTypeObject *cls, PyObject *const *args Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *msg = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -393,7 +446,9 @@ _asyncio_Future_cancel(FutureObj *self, PyTypeObject *cls, PyObject *const *args } msg = args[0]; skip_optional_pos: - return_value = _asyncio_Future_cancel_impl(self, cls, msg); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future_cancel_impl((FutureObj *)self, cls, msg); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -412,9 +467,15 @@ static PyObject * _asyncio_Future_cancelled_impl(FutureObj *self); static PyObject * -_asyncio_Future_cancelled(FutureObj *self, PyObject *Py_UNUSED(ignored)) +_asyncio_Future_cancelled(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Future_cancelled_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future_cancelled_impl((FutureObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Future_done__doc__, @@ -433,9 +494,15 @@ static PyObject * _asyncio_Future_done_impl(FutureObj *self); static PyObject * -_asyncio_Future_done(FutureObj *self, PyObject *Py_UNUSED(ignored)) +_asyncio_Future_done(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Future_done_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future_done_impl((FutureObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Future_get_loop__doc__, @@ -451,13 +518,321 @@ static PyObject * _asyncio_Future_get_loop_impl(FutureObj *self, PyTypeObject *cls); static PyObject * -_asyncio_Future_get_loop(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_asyncio_Future_get_loop(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "get_loop() takes no arguments"); - return NULL; + goto exit; } - return _asyncio_Future_get_loop_impl(self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future_get_loop_impl((FutureObj *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +#if !defined(_asyncio_Future__asyncio_future_blocking_DOCSTR) +# define _asyncio_Future__asyncio_future_blocking_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF) +# undef _ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF +# define _ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF {"_asyncio_future_blocking", (getter)_asyncio_Future__asyncio_future_blocking_get, (setter)_asyncio_Future__asyncio_future_blocking_set, _asyncio_Future__asyncio_future_blocking_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF {"_asyncio_future_blocking", (getter)_asyncio_Future__asyncio_future_blocking_get, NULL, _asyncio_Future__asyncio_future_blocking_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__asyncio_future_blocking_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__asyncio_future_blocking_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__asyncio_future_blocking_get_impl((FutureObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__asyncio_future_blocking_DOCSTR) +# define _asyncio_Future__asyncio_future_blocking_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF) +# undef _ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF +# define _ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF {"_asyncio_future_blocking", (getter)_asyncio_Future__asyncio_future_blocking_get, (setter)_asyncio_Future__asyncio_future_blocking_set, _asyncio_Future__asyncio_future_blocking_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__ASYNCIO_FUTURE_BLOCKING_GETSETDEF {"_asyncio_future_blocking", NULL, (setter)_asyncio_Future__asyncio_future_blocking_set, NULL}, +#endif + +static int +_asyncio_Future__asyncio_future_blocking_set_impl(FutureObj *self, + PyObject *value); + +static int +_asyncio_Future__asyncio_future_blocking_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__asyncio_future_blocking_set_impl((FutureObj *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__log_traceback_DOCSTR) +# define _asyncio_Future__log_traceback_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF) +# undef _ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF +# define _ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF {"_log_traceback", (getter)_asyncio_Future__log_traceback_get, (setter)_asyncio_Future__log_traceback_set, _asyncio_Future__log_traceback_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF {"_log_traceback", (getter)_asyncio_Future__log_traceback_get, NULL, _asyncio_Future__log_traceback_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__log_traceback_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__log_traceback_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__log_traceback_get_impl((FutureObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__log_traceback_DOCSTR) +# define _asyncio_Future__log_traceback_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF) +# undef _ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF +# define _ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF {"_log_traceback", (getter)_asyncio_Future__log_traceback_get, (setter)_asyncio_Future__log_traceback_set, _asyncio_Future__log_traceback_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__LOG_TRACEBACK_GETSETDEF {"_log_traceback", NULL, (setter)_asyncio_Future__log_traceback_set, NULL}, +#endif + +static int +_asyncio_Future__log_traceback_set_impl(FutureObj *self, PyObject *value); + +static int +_asyncio_Future__log_traceback_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__log_traceback_set_impl((FutureObj *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__loop_DOCSTR) +# define _asyncio_Future__loop_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__LOOP_GETSETDEF) +# undef _ASYNCIO_FUTURE__LOOP_GETSETDEF +# define _ASYNCIO_FUTURE__LOOP_GETSETDEF {"_loop", (getter)_asyncio_Future__loop_get, (setter)_asyncio_Future__loop_set, _asyncio_Future__loop_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__LOOP_GETSETDEF {"_loop", (getter)_asyncio_Future__loop_get, NULL, _asyncio_Future__loop_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__loop_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__loop_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__loop_get_impl((FutureObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__callbacks_DOCSTR) +# define _asyncio_Future__callbacks_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__CALLBACKS_GETSETDEF) +# undef _ASYNCIO_FUTURE__CALLBACKS_GETSETDEF +# define _ASYNCIO_FUTURE__CALLBACKS_GETSETDEF {"_callbacks", (getter)_asyncio_Future__callbacks_get, (setter)_asyncio_Future__callbacks_set, _asyncio_Future__callbacks_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__CALLBACKS_GETSETDEF {"_callbacks", (getter)_asyncio_Future__callbacks_get, NULL, _asyncio_Future__callbacks_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__callbacks_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__callbacks_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__callbacks_get_impl((FutureObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__result_DOCSTR) +# define _asyncio_Future__result_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__RESULT_GETSETDEF) +# undef _ASYNCIO_FUTURE__RESULT_GETSETDEF +# define _ASYNCIO_FUTURE__RESULT_GETSETDEF {"_result", (getter)_asyncio_Future__result_get, (setter)_asyncio_Future__result_set, _asyncio_Future__result_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__RESULT_GETSETDEF {"_result", (getter)_asyncio_Future__result_get, NULL, _asyncio_Future__result_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__result_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__result_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__result_get_impl((FutureObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__exception_DOCSTR) +# define _asyncio_Future__exception_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__EXCEPTION_GETSETDEF) +# undef _ASYNCIO_FUTURE__EXCEPTION_GETSETDEF +# define _ASYNCIO_FUTURE__EXCEPTION_GETSETDEF {"_exception", (getter)_asyncio_Future__exception_get, (setter)_asyncio_Future__exception_set, _asyncio_Future__exception_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__EXCEPTION_GETSETDEF {"_exception", (getter)_asyncio_Future__exception_get, NULL, _asyncio_Future__exception_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__exception_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__exception_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__exception_get_impl((FutureObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__source_traceback_DOCSTR) +# define _asyncio_Future__source_traceback_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__SOURCE_TRACEBACK_GETSETDEF) +# undef _ASYNCIO_FUTURE__SOURCE_TRACEBACK_GETSETDEF +# define _ASYNCIO_FUTURE__SOURCE_TRACEBACK_GETSETDEF {"_source_traceback", (getter)_asyncio_Future__source_traceback_get, (setter)_asyncio_Future__source_traceback_set, _asyncio_Future__source_traceback_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__SOURCE_TRACEBACK_GETSETDEF {"_source_traceback", (getter)_asyncio_Future__source_traceback_get, NULL, _asyncio_Future__source_traceback_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__source_traceback_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__source_traceback_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__source_traceback_get_impl((FutureObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__cancel_message_DOCSTR) +# define _asyncio_Future__cancel_message_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF) +# undef _ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF +# define _ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF {"_cancel_message", (getter)_asyncio_Future__cancel_message_get, (setter)_asyncio_Future__cancel_message_set, _asyncio_Future__cancel_message_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF {"_cancel_message", (getter)_asyncio_Future__cancel_message_get, NULL, _asyncio_Future__cancel_message_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__cancel_message_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__cancel_message_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__cancel_message_get_impl((FutureObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__cancel_message_DOCSTR) +# define _asyncio_Future__cancel_message_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF) +# undef _ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF +# define _ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF {"_cancel_message", (getter)_asyncio_Future__cancel_message_get, (setter)_asyncio_Future__cancel_message_set, _asyncio_Future__cancel_message_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__CANCEL_MESSAGE_GETSETDEF {"_cancel_message", NULL, (setter)_asyncio_Future__cancel_message_set, NULL}, +#endif + +static int +_asyncio_Future__cancel_message_set_impl(FutureObj *self, PyObject *value); + +static int +_asyncio_Future__cancel_message_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__cancel_message_set_impl((FutureObj *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Future__state_DOCSTR) +# define _asyncio_Future__state_DOCSTR NULL +#endif +#if defined(_ASYNCIO_FUTURE__STATE_GETSETDEF) +# undef _ASYNCIO_FUTURE__STATE_GETSETDEF +# define _ASYNCIO_FUTURE__STATE_GETSETDEF {"_state", (getter)_asyncio_Future__state_get, (setter)_asyncio_Future__state_set, _asyncio_Future__state_DOCSTR}, +#else +# define _ASYNCIO_FUTURE__STATE_GETSETDEF {"_state", (getter)_asyncio_Future__state_get, NULL, _asyncio_Future__state_DOCSTR}, +#endif + +static PyObject * +_asyncio_Future__state_get_impl(FutureObj *self); + +static PyObject * +_asyncio_Future__state_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__state_get_impl((FutureObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Future__make_cancelled_error__doc__, @@ -476,9 +851,15 @@ static PyObject * _asyncio_Future__make_cancelled_error_impl(FutureObj *self); static PyObject * -_asyncio_Future__make_cancelled_error(FutureObj *self, PyObject *Py_UNUSED(ignored)) +_asyncio_Future__make_cancelled_error(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Future__make_cancelled_error_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Future__make_cancelled_error_impl((FutureObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Task___init____doc__, @@ -531,7 +912,8 @@ _asyncio_Task___init__(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *context = Py_None; int eager_start = 0; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -568,6 +950,131 @@ _asyncio_Task___init__(PyObject *self, PyObject *args, PyObject *kwargs) return return_value; } +#if !defined(_asyncio_Task__log_destroy_pending_DOCSTR) +# define _asyncio_Task__log_destroy_pending_DOCSTR NULL +#endif +#if defined(_ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF) +# undef _ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF +# define _ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF {"_log_destroy_pending", (getter)_asyncio_Task__log_destroy_pending_get, (setter)_asyncio_Task__log_destroy_pending_set, _asyncio_Task__log_destroy_pending_DOCSTR}, +#else +# define _ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF {"_log_destroy_pending", (getter)_asyncio_Task__log_destroy_pending_get, NULL, _asyncio_Task__log_destroy_pending_DOCSTR}, +#endif + +static PyObject * +_asyncio_Task__log_destroy_pending_get_impl(TaskObj *self); + +static PyObject * +_asyncio_Task__log_destroy_pending_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task__log_destroy_pending_get_impl((TaskObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Task__log_destroy_pending_DOCSTR) +# define _asyncio_Task__log_destroy_pending_DOCSTR NULL +#endif +#if defined(_ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF) +# undef _ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF +# define _ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF {"_log_destroy_pending", (getter)_asyncio_Task__log_destroy_pending_get, (setter)_asyncio_Task__log_destroy_pending_set, _asyncio_Task__log_destroy_pending_DOCSTR}, +#else +# define _ASYNCIO_TASK__LOG_DESTROY_PENDING_GETSETDEF {"_log_destroy_pending", NULL, (setter)_asyncio_Task__log_destroy_pending_set, NULL}, +#endif + +static int +_asyncio_Task__log_destroy_pending_set_impl(TaskObj *self, PyObject *value); + +static int +_asyncio_Task__log_destroy_pending_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task__log_destroy_pending_set_impl((TaskObj *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Task__must_cancel_DOCSTR) +# define _asyncio_Task__must_cancel_DOCSTR NULL +#endif +#if defined(_ASYNCIO_TASK__MUST_CANCEL_GETSETDEF) +# undef _ASYNCIO_TASK__MUST_CANCEL_GETSETDEF +# define _ASYNCIO_TASK__MUST_CANCEL_GETSETDEF {"_must_cancel", (getter)_asyncio_Task__must_cancel_get, (setter)_asyncio_Task__must_cancel_set, _asyncio_Task__must_cancel_DOCSTR}, +#else +# define _ASYNCIO_TASK__MUST_CANCEL_GETSETDEF {"_must_cancel", (getter)_asyncio_Task__must_cancel_get, NULL, _asyncio_Task__must_cancel_DOCSTR}, +#endif + +static PyObject * +_asyncio_Task__must_cancel_get_impl(TaskObj *self); + +static PyObject * +_asyncio_Task__must_cancel_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task__must_cancel_get_impl((TaskObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Task__coro_DOCSTR) +# define _asyncio_Task__coro_DOCSTR NULL +#endif +#if defined(_ASYNCIO_TASK__CORO_GETSETDEF) +# undef _ASYNCIO_TASK__CORO_GETSETDEF +# define _ASYNCIO_TASK__CORO_GETSETDEF {"_coro", (getter)_asyncio_Task__coro_get, (setter)_asyncio_Task__coro_set, _asyncio_Task__coro_DOCSTR}, +#else +# define _ASYNCIO_TASK__CORO_GETSETDEF {"_coro", (getter)_asyncio_Task__coro_get, NULL, _asyncio_Task__coro_DOCSTR}, +#endif + +static PyObject * +_asyncio_Task__coro_get_impl(TaskObj *self); + +static PyObject * +_asyncio_Task__coro_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task__coro_get_impl((TaskObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_asyncio_Task__fut_waiter_DOCSTR) +# define _asyncio_Task__fut_waiter_DOCSTR NULL +#endif +#if defined(_ASYNCIO_TASK__FUT_WAITER_GETSETDEF) +# undef _ASYNCIO_TASK__FUT_WAITER_GETSETDEF +# define _ASYNCIO_TASK__FUT_WAITER_GETSETDEF {"_fut_waiter", (getter)_asyncio_Task__fut_waiter_get, (setter)_asyncio_Task__fut_waiter_set, _asyncio_Task__fut_waiter_DOCSTR}, +#else +# define _ASYNCIO_TASK__FUT_WAITER_GETSETDEF {"_fut_waiter", (getter)_asyncio_Task__fut_waiter_get, NULL, _asyncio_Task__fut_waiter_DOCSTR}, +#endif + +static PyObject * +_asyncio_Task__fut_waiter_get_impl(TaskObj *self); + +static PyObject * +_asyncio_Task__fut_waiter_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task__fut_waiter_get_impl((TaskObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_asyncio_Task__make_cancelled_error__doc__, "_make_cancelled_error($self, /)\n" "--\n" @@ -584,9 +1091,15 @@ static PyObject * _asyncio_Task__make_cancelled_error_impl(TaskObj *self); static PyObject * -_asyncio_Task__make_cancelled_error(TaskObj *self, PyObject *Py_UNUSED(ignored)) +_asyncio_Task__make_cancelled_error(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Task__make_cancelled_error_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task__make_cancelled_error_impl((TaskObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Task_cancel__doc__, @@ -621,7 +1134,7 @@ static PyObject * _asyncio_Task_cancel_impl(TaskObj *self, PyObject *msg); static PyObject * -_asyncio_Task_cancel(TaskObj *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_asyncio_Task_cancel(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -653,7 +1166,8 @@ _asyncio_Task_cancel(TaskObj *self, PyObject *const *args, Py_ssize_t nargs, PyO Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *msg = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -662,7 +1176,9 @@ _asyncio_Task_cancel(TaskObj *self, PyObject *const *args, Py_ssize_t nargs, PyO } msg = args[0]; skip_optional_pos: - return_value = _asyncio_Task_cancel_impl(self, msg); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task_cancel_impl((TaskObj *)self, msg); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -684,9 +1200,15 @@ static PyObject * _asyncio_Task_cancelling_impl(TaskObj *self); static PyObject * -_asyncio_Task_cancelling(TaskObj *self, PyObject *Py_UNUSED(ignored)) +_asyncio_Task_cancelling(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Task_cancelling_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task_cancelling_impl((TaskObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Task_uncancel__doc__, @@ -707,9 +1229,15 @@ static PyObject * _asyncio_Task_uncancel_impl(TaskObj *self); static PyObject * -_asyncio_Task_uncancel(TaskObj *self, PyObject *Py_UNUSED(ignored)) +_asyncio_Task_uncancel(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Task_uncancel_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task_uncancel_impl((TaskObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Task_get_stack__doc__, @@ -744,7 +1272,7 @@ _asyncio_Task_get_stack_impl(TaskObj *self, PyTypeObject *cls, PyObject *limit); static PyObject * -_asyncio_Task_get_stack(TaskObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_asyncio_Task_get_stack(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -776,7 +1304,8 @@ _asyncio_Task_get_stack(TaskObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *limit = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -785,7 +1314,7 @@ _asyncio_Task_get_stack(TaskObj *self, PyTypeObject *cls, PyObject *const *args, } limit = args[0]; skip_optional_kwonly: - return_value = _asyncio_Task_get_stack_impl(self, cls, limit); + return_value = _asyncio_Task_get_stack_impl((TaskObj *)self, cls, limit); exit: return return_value; @@ -811,7 +1340,7 @@ _asyncio_Task_print_stack_impl(TaskObj *self, PyTypeObject *cls, PyObject *limit, PyObject *file); static PyObject * -_asyncio_Task_print_stack(TaskObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_asyncio_Task_print_stack(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -844,7 +1373,8 @@ _asyncio_Task_print_stack(TaskObj *self, PyTypeObject *cls, PyObject *const *arg PyObject *limit = Py_None; PyObject *file = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -859,7 +1389,7 @@ _asyncio_Task_print_stack(TaskObj *self, PyTypeObject *cls, PyObject *const *arg } file = args[1]; skip_optional_kwonly: - return_value = _asyncio_Task_print_stack_impl(self, cls, limit, file); + return_value = _asyncio_Task_print_stack_impl((TaskObj *)self, cls, limit, file); exit: return return_value; @@ -893,9 +1423,15 @@ static PyObject * _asyncio_Task_get_coro_impl(TaskObj *self); static PyObject * -_asyncio_Task_get_coro(TaskObj *self, PyObject *Py_UNUSED(ignored)) +_asyncio_Task_get_coro(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Task_get_coro_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task_get_coro_impl((TaskObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Task_get_context__doc__, @@ -910,9 +1446,9 @@ static PyObject * _asyncio_Task_get_context_impl(TaskObj *self); static PyObject * -_asyncio_Task_get_context(TaskObj *self, PyObject *Py_UNUSED(ignored)) +_asyncio_Task_get_context(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Task_get_context_impl(self); + return _asyncio_Task_get_context_impl((TaskObj *)self); } PyDoc_STRVAR(_asyncio_Task_get_name__doc__, @@ -927,9 +1463,15 @@ static PyObject * _asyncio_Task_get_name_impl(TaskObj *self); static PyObject * -_asyncio_Task_get_name(TaskObj *self, PyObject *Py_UNUSED(ignored)) +_asyncio_Task_get_name(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _asyncio_Task_get_name_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task_get_name_impl((TaskObj *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_asyncio_Task_set_name__doc__, @@ -940,6 +1482,21 @@ PyDoc_STRVAR(_asyncio_Task_set_name__doc__, #define _ASYNCIO_TASK_SET_NAME_METHODDEF \ {"set_name", (PyCFunction)_asyncio_Task_set_name, METH_O, _asyncio_Task_set_name__doc__}, +static PyObject * +_asyncio_Task_set_name_impl(TaskObj *self, PyObject *value); + +static PyObject * +_asyncio_Task_set_name(TaskObj *self, PyObject *value) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _asyncio_Task_set_name_impl((TaskObj *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_asyncio__get_running_loop__doc__, "_get_running_loop($module, /)\n" "--\n" @@ -1064,7 +1621,8 @@ _asyncio__register_task(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *argsbuf[1]; PyObject *task; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1121,7 +1679,8 @@ _asyncio__register_eager_task(PyObject *module, PyObject *const *args, Py_ssize_ PyObject *argsbuf[1]; PyObject *task; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1178,7 +1737,8 @@ _asyncio__unregister_task(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *argsbuf[1]; PyObject *task; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1235,7 +1795,8 @@ _asyncio__unregister_eager_task(PyObject *module, PyObject *const *args, Py_ssiz PyObject *argsbuf[1]; PyObject *task; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1295,7 +1856,8 @@ _asyncio__enter_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *loop; PyObject *task; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1356,7 +1918,8 @@ _asyncio__leave_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *loop; PyObject *task; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1416,7 +1979,8 @@ _asyncio__swap_current_task(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *loop; PyObject *task; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1473,7 +2037,8 @@ _asyncio_current_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *loop = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1487,4 +2052,126 @@ _asyncio_current_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=b26155080c82c472 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_asyncio_all_tasks__doc__, +"all_tasks($module, /, loop=None)\n" +"--\n" +"\n" +"Return a set of all tasks for the loop."); + +#define _ASYNCIO_ALL_TASKS_METHODDEF \ + {"all_tasks", _PyCFunction_CAST(_asyncio_all_tasks), METH_FASTCALL|METH_KEYWORDS, _asyncio_all_tasks__doc__}, + +static PyObject * +_asyncio_all_tasks_impl(PyObject *module, PyObject *loop); + +static PyObject * +_asyncio_all_tasks(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(loop), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"loop", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "all_tasks", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *loop = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + loop = args[0]; +skip_optional_pos: + return_value = _asyncio_all_tasks_impl(module, loop); + +exit: + return return_value; +} + +PyDoc_STRVAR(_asyncio_future_add_to_awaited_by__doc__, +"future_add_to_awaited_by($module, fut, waiter, /)\n" +"--\n" +"\n" +"Record that `fut` is awaited on by `waiter`."); + +#define _ASYNCIO_FUTURE_ADD_TO_AWAITED_BY_METHODDEF \ + {"future_add_to_awaited_by", _PyCFunction_CAST(_asyncio_future_add_to_awaited_by), METH_FASTCALL, _asyncio_future_add_to_awaited_by__doc__}, + +static PyObject * +_asyncio_future_add_to_awaited_by_impl(PyObject *module, PyObject *fut, + PyObject *waiter); + +static PyObject * +_asyncio_future_add_to_awaited_by(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *fut; + PyObject *waiter; + + if (!_PyArg_CheckPositional("future_add_to_awaited_by", nargs, 2, 2)) { + goto exit; + } + fut = args[0]; + waiter = args[1]; + return_value = _asyncio_future_add_to_awaited_by_impl(module, fut, waiter); + +exit: + return return_value; +} + +PyDoc_STRVAR(_asyncio_future_discard_from_awaited_by__doc__, +"future_discard_from_awaited_by($module, fut, waiter, /)\n" +"--\n" +"\n"); + +#define _ASYNCIO_FUTURE_DISCARD_FROM_AWAITED_BY_METHODDEF \ + {"future_discard_from_awaited_by", _PyCFunction_CAST(_asyncio_future_discard_from_awaited_by), METH_FASTCALL, _asyncio_future_discard_from_awaited_by__doc__}, + +static PyObject * +_asyncio_future_discard_from_awaited_by_impl(PyObject *module, PyObject *fut, + PyObject *waiter); + +static PyObject * +_asyncio_future_discard_from_awaited_by(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *fut; + PyObject *waiter; + + if (!_PyArg_CheckPositional("future_discard_from_awaited_by", nargs, 2, 2)) { + goto exit; + } + fut = args[0]; + waiter = args[1]; + return_value = _asyncio_future_discard_from_awaited_by_impl(module, fut, waiter); + +exit: + return return_value; +} +/*[clinic end generated code: output=fe4ffe08404ad566 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_bisectmodule.c.h b/Modules/clinic/_bisectmodule.c.h index 9955d0edb2699f..910fd52c07dd30 100644 --- a/Modules/clinic/_bisectmodule.c.h +++ b/Modules/clinic/_bisectmodule.c.h @@ -44,7 +44,7 @@ _bisect_bisect_right(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(x), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('x'), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -69,7 +69,8 @@ _bisect_bisect_right(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *key = Py_None; Py_ssize_t _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -152,7 +153,7 @@ _bisect_insort_right(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(x), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('x'), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -176,7 +177,8 @@ _bisect_insort_right(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py_ssize_t hi = -1; PyObject *key = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -257,7 +259,7 @@ _bisect_bisect_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(x), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('x'), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -282,7 +284,8 @@ _bisect_bisect_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *key = Py_None; Py_ssize_t _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -365,7 +368,7 @@ _bisect_insort_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(x), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('x'), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -389,7 +392,8 @@ _bisect_insort_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P Py_ssize_t hi = -1; PyObject *key = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -434,4 +438,4 @@ _bisect_insort_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P exit: return return_value; } -/*[clinic end generated code: output=4af5bd405149bf5f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=972e8938ba5a0eac input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_bz2module.c.h b/Modules/clinic/_bz2module.c.h index de7b3993596446..a599bd1a8be96a 100644 --- a/Modules/clinic/_bz2module.c.h +++ b/Modules/clinic/_bz2module.c.h @@ -27,7 +27,7 @@ static PyObject * _bz2_BZ2Compressor_compress_impl(BZ2Compressor *self, Py_buffer *data); static PyObject * -_bz2_BZ2Compressor_compress(BZ2Compressor *self, PyObject *arg) +_bz2_BZ2Compressor_compress(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; Py_buffer data = {NULL, NULL}; @@ -35,7 +35,7 @@ _bz2_BZ2Compressor_compress(BZ2Compressor *self, PyObject *arg) if (PyObject_GetBuffer(arg, &data, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = _bz2_BZ2Compressor_compress_impl(self, &data); + return_value = _bz2_BZ2Compressor_compress_impl((BZ2Compressor *)self, &data); exit: /* Cleanup for data */ @@ -63,9 +63,9 @@ static PyObject * _bz2_BZ2Compressor_flush_impl(BZ2Compressor *self); static PyObject * -_bz2_BZ2Compressor_flush(BZ2Compressor *self, PyObject *Py_UNUSED(ignored)) +_bz2_BZ2Compressor_flush(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _bz2_BZ2Compressor_flush_impl(self); + return _bz2_BZ2Compressor_flush_impl((BZ2Compressor *)self); } PyDoc_STRVAR(_bz2_BZ2Compressor__doc__, @@ -137,7 +137,7 @@ _bz2_BZ2Decompressor_decompress_impl(BZ2Decompressor *self, Py_buffer *data, Py_ssize_t max_length); static PyObject * -_bz2_BZ2Decompressor_decompress(BZ2Decompressor *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_bz2_BZ2Decompressor_decompress(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -170,7 +170,8 @@ _bz2_BZ2Decompressor_decompress(BZ2Decompressor *self, PyObject *const *args, Py Py_buffer data = {NULL, NULL}; Py_ssize_t max_length = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -193,7 +194,7 @@ _bz2_BZ2Decompressor_decompress(BZ2Decompressor *self, PyObject *const *args, Py max_length = ival; } skip_optional_pos: - return_value = _bz2_BZ2Decompressor_decompress_impl(self, &data, max_length); + return_value = _bz2_BZ2Decompressor_decompress_impl((BZ2Decompressor *)self, &data, max_length); exit: /* Cleanup for data */ @@ -234,4 +235,4 @@ _bz2_BZ2Decompressor(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=8daa62f47cc4853d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0fc5a6292c5fd2c5 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index 1c0f37442ab350..46476a948b66d9 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -123,7 +123,8 @@ _codecs_encode(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje const char *encoding = NULL; const char *errors = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -223,7 +224,8 @@ _codecs_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje const char *encoding = NULL; const char *errors = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2683,6 +2685,56 @@ _codecs_register_error(PyObject *module, PyObject *const *args, Py_ssize_t nargs return return_value; } +PyDoc_STRVAR(_codecs__unregister_error__doc__, +"_unregister_error($module, errors, /)\n" +"--\n" +"\n" +"Un-register the specified error handler for the error handling `errors\'.\n" +"\n" +"Only custom error handlers can be un-registered. An exception is raised\n" +"if the error handling is a built-in one (e.g., \'strict\'), or if an error\n" +"occurs.\n" +"\n" +"Otherwise, this returns True if a custom handler has been successfully\n" +"un-registered, and False if no custom handler for the specified error\n" +"handling exists."); + +#define _CODECS__UNREGISTER_ERROR_METHODDEF \ + {"_unregister_error", (PyCFunction)_codecs__unregister_error, METH_O, _codecs__unregister_error__doc__}, + +static int +_codecs__unregister_error_impl(PyObject *module, const char *errors); + +static PyObject * +_codecs__unregister_error(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + const char *errors; + int _return_value; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("_unregister_error", "argument", "str", arg); + goto exit; + } + Py_ssize_t errors_length; + errors = PyUnicode_AsUTF8AndSize(arg, &errors_length); + if (errors == NULL) { + goto exit; + } + if (strlen(errors) != (size_t)errors_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + _return_value = _codecs__unregister_error_impl(module, errors); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(_codecs_lookup_error__doc__, "lookup_error($module, name, /)\n" "--\n" @@ -2746,4 +2798,4 @@ _codecs_lookup_error(PyObject *module, PyObject *arg) #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=e50d5fdf65bd45fa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f8a7fdd0b7edc0c4 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_collectionsmodule.c.h b/Modules/clinic/_collectionsmodule.c.h index fa0ff2133a05d2..ddf18c2c77a8cd 100644 --- a/Modules/clinic/_collectionsmodule.c.h +++ b/Modules/clinic/_collectionsmodule.c.h @@ -23,12 +23,12 @@ static PyObject * deque_pop_impl(dequeobject *deque); static PyObject * -deque_pop(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +deque_pop(PyObject *deque, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(deque); - return_value = deque_pop_impl(deque); + return_value = deque_pop_impl((dequeobject *)deque); Py_END_CRITICAL_SECTION(); return return_value; @@ -47,12 +47,12 @@ static PyObject * deque_popleft_impl(dequeobject *deque); static PyObject * -deque_popleft(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +deque_popleft(PyObject *deque, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(deque); - return_value = deque_popleft_impl(deque); + return_value = deque_popleft_impl((dequeobject *)deque); Py_END_CRITICAL_SECTION(); return return_value; @@ -76,7 +76,7 @@ deque_append(dequeobject *deque, PyObject *item) PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(deque); - return_value = deque_append_impl(deque, item); + return_value = deque_append_impl((dequeobject *)deque, item); Py_END_CRITICAL_SECTION(); return return_value; @@ -100,7 +100,7 @@ deque_appendleft(dequeobject *deque, PyObject *item) PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(deque); - return_value = deque_appendleft_impl(deque, item); + return_value = deque_appendleft_impl((dequeobject *)deque, item); Py_END_CRITICAL_SECTION(); return return_value; @@ -124,7 +124,7 @@ deque_extend(dequeobject *deque, PyObject *iterable) PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(deque); - return_value = deque_extend_impl(deque, iterable); + return_value = deque_extend_impl((dequeobject *)deque, iterable); Py_END_CRITICAL_SECTION(); return return_value; @@ -148,7 +148,7 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(deque); - return_value = deque_extendleft_impl(deque, iterable); + return_value = deque_extendleft_impl((dequeobject *)deque, iterable); Py_END_CRITICAL_SECTION(); return return_value; @@ -167,12 +167,12 @@ static PyObject * deque_copy_impl(dequeobject *deque); static PyObject * -deque_copy(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +deque_copy(PyObject *deque, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(deque); - return_value = deque_copy_impl(deque); + return_value = deque_copy_impl((dequeobject *)deque); Py_END_CRITICAL_SECTION(); return return_value; @@ -191,12 +191,12 @@ static PyObject * deque___copy___impl(dequeobject *deque); static PyObject * -deque___copy__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +deque___copy__(PyObject *deque, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(deque); - return_value = deque___copy___impl(deque); + return_value = deque___copy___impl((dequeobject *)deque); Py_END_CRITICAL_SECTION(); return return_value; @@ -215,12 +215,12 @@ static PyObject * deque_clearmethod_impl(dequeobject *deque); static PyObject * -deque_clearmethod(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +deque_clearmethod(PyObject *deque, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(deque); - return_value = deque_clearmethod_impl(deque); + return_value = deque_clearmethod_impl((dequeobject *)deque); Py_END_CRITICAL_SECTION(); return return_value; @@ -239,7 +239,7 @@ static PyObject * deque_rotate_impl(dequeobject *deque, Py_ssize_t n); static PyObject * -deque_rotate(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +deque_rotate(PyObject *deque, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t n = 1; @@ -264,7 +264,7 @@ deque_rotate(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) } skip_optional: Py_BEGIN_CRITICAL_SECTION(deque); - return_value = deque_rotate_impl(deque, n); + return_value = deque_rotate_impl((dequeobject *)deque, n); Py_END_CRITICAL_SECTION(); exit: @@ -284,12 +284,12 @@ static PyObject * deque_reverse_impl(dequeobject *deque); static PyObject * -deque_reverse(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +deque_reverse(PyObject *deque, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(deque); - return_value = deque_reverse_impl(deque); + return_value = deque_reverse_impl((dequeobject *)deque); Py_END_CRITICAL_SECTION(); return return_value; @@ -313,7 +313,7 @@ deque_count(dequeobject *deque, PyObject *v) PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(deque); - return_value = deque_count_impl(deque, v); + return_value = deque_count_impl((dequeobject *)deque, v); Py_END_CRITICAL_SECTION(); return return_value; @@ -335,7 +335,7 @@ deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start, Py_ssize_t stop); static PyObject * -deque_index(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +deque_index(PyObject *deque, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *v; @@ -360,7 +360,7 @@ deque_index(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) } skip_optional: Py_BEGIN_CRITICAL_SECTION(deque); - return_value = deque_index_impl(deque, v, start, stop); + return_value = deque_index_impl((dequeobject *)deque, v, start, stop); Py_END_CRITICAL_SECTION(); exit: @@ -380,7 +380,7 @@ static PyObject * deque_insert_impl(dequeobject *deque, Py_ssize_t index, PyObject *value); static PyObject * -deque_insert(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) +deque_insert(PyObject *deque, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t index; @@ -403,7 +403,7 @@ deque_insert(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) } value = args[1]; Py_BEGIN_CRITICAL_SECTION(deque); - return_value = deque_insert_impl(deque, index, value); + return_value = deque_insert_impl((dequeobject *)deque, index, value); Py_END_CRITICAL_SECTION(); exit: @@ -428,7 +428,7 @@ deque_remove(dequeobject *deque, PyObject *value) PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(deque); - return_value = deque_remove_impl(deque, value); + return_value = deque_remove_impl((dequeobject *)deque, value); Py_END_CRITICAL_SECTION(); return return_value; @@ -447,9 +447,9 @@ static PyObject * deque___reduce___impl(dequeobject *deque); static PyObject * -deque___reduce__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +deque___reduce__(PyObject *deque, PyObject *Py_UNUSED(ignored)) { - return deque___reduce___impl(deque); + return deque___reduce___impl((dequeobject *)deque); } PyDoc_STRVAR(deque_init__doc__, @@ -497,7 +497,8 @@ deque_init(PyObject *deque, PyObject *args, PyObject *kwargs) PyObject *iterable = NULL; PyObject *maxlenobj = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -533,12 +534,12 @@ static PyObject * deque___sizeof___impl(dequeobject *deque); static PyObject * -deque___sizeof__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +deque___sizeof__(PyObject *deque, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(deque); - return_value = deque___sizeof___impl(deque); + return_value = deque___sizeof___impl((dequeobject *)deque); Py_END_CRITICAL_SECTION(); return return_value; @@ -557,9 +558,9 @@ static PyObject * deque___reversed___impl(dequeobject *deque); static PyObject * -deque___reversed__(dequeobject *deque, PyObject *Py_UNUSED(ignored)) +deque___reversed__(PyObject *deque, PyObject *Py_UNUSED(ignored)) { - return deque___reversed___impl(deque); + return deque___reversed___impl((dequeobject *)deque); } PyDoc_STRVAR(_collections__count_elements__doc__, @@ -629,4 +630,4 @@ tuplegetter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=64c32b16df7be07a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2d89c39288fc7389 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_csv.c.h b/Modules/clinic/_csv.c.h index 2442bdcde5679f..b5608c850b6a97 100644 --- a/Modules/clinic/_csv.c.h +++ b/Modules/clinic/_csv.c.h @@ -74,7 +74,8 @@ _csv_unregister_dialect(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *argsbuf[1]; PyObject *name; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -131,7 +132,8 @@ _csv_get_dialect(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb PyObject *argsbuf[1]; PyObject *name; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -192,7 +194,8 @@ _csv_field_size_limit(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *new_limit = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -206,4 +209,4 @@ _csv_field_size_limit(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=9ec59717f5414d8b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9c7314c2434156b3 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_curses_panel.c.h b/Modules/clinic/_curses_panel.c.h index 457f71370afda9..6f4966825ec4bf 100644 --- a/Modules/clinic/_curses_panel.c.h +++ b/Modules/clinic/_curses_panel.c.h @@ -2,6 +2,9 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_curses_panel_panel_bottom__doc__, @@ -17,13 +20,13 @@ static PyObject * _curses_panel_panel_bottom_impl(PyCursesPanelObject *self, PyTypeObject *cls); static PyObject * -_curses_panel_panel_bottom(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_bottom(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "bottom() takes no arguments"); return NULL; } - return _curses_panel_panel_bottom_impl(self, cls); + return _curses_panel_panel_bottom_impl((PyCursesPanelObject *)self, cls); } PyDoc_STRVAR(_curses_panel_panel_hide__doc__, @@ -41,13 +44,13 @@ static PyObject * _curses_panel_panel_hide_impl(PyCursesPanelObject *self, PyTypeObject *cls); static PyObject * -_curses_panel_panel_hide(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_hide(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "hide() takes no arguments"); return NULL; } - return _curses_panel_panel_hide_impl(self, cls); + return _curses_panel_panel_hide_impl((PyCursesPanelObject *)self, cls); } PyDoc_STRVAR(_curses_panel_panel_show__doc__, @@ -63,13 +66,13 @@ static PyObject * _curses_panel_panel_show_impl(PyCursesPanelObject *self, PyTypeObject *cls); static PyObject * -_curses_panel_panel_show(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_show(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "show() takes no arguments"); return NULL; } - return _curses_panel_panel_show_impl(self, cls); + return _curses_panel_panel_show_impl((PyCursesPanelObject *)self, cls); } PyDoc_STRVAR(_curses_panel_panel_top__doc__, @@ -85,13 +88,13 @@ static PyObject * _curses_panel_panel_top_impl(PyCursesPanelObject *self, PyTypeObject *cls); static PyObject * -_curses_panel_panel_top(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_top(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "top() takes no arguments"); return NULL; } - return _curses_panel_panel_top_impl(self, cls); + return _curses_panel_panel_top_impl((PyCursesPanelObject *)self, cls); } PyDoc_STRVAR(_curses_panel_panel_above__doc__, @@ -107,9 +110,9 @@ static PyObject * _curses_panel_panel_above_impl(PyCursesPanelObject *self); static PyObject * -_curses_panel_panel_above(PyCursesPanelObject *self, PyObject *Py_UNUSED(ignored)) +_curses_panel_panel_above(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _curses_panel_panel_above_impl(self); + return _curses_panel_panel_above_impl((PyCursesPanelObject *)self); } PyDoc_STRVAR(_curses_panel_panel_below__doc__, @@ -125,9 +128,9 @@ static PyObject * _curses_panel_panel_below_impl(PyCursesPanelObject *self); static PyObject * -_curses_panel_panel_below(PyCursesPanelObject *self, PyObject *Py_UNUSED(ignored)) +_curses_panel_panel_below(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _curses_panel_panel_below_impl(self); + return _curses_panel_panel_below_impl((PyCursesPanelObject *)self); } PyDoc_STRVAR(_curses_panel_panel_hidden__doc__, @@ -143,9 +146,9 @@ static PyObject * _curses_panel_panel_hidden_impl(PyCursesPanelObject *self); static PyObject * -_curses_panel_panel_hidden(PyCursesPanelObject *self, PyObject *Py_UNUSED(ignored)) +_curses_panel_panel_hidden(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _curses_panel_panel_hidden_impl(self); + return _curses_panel_panel_hidden_impl((PyCursesPanelObject *)self); } PyDoc_STRVAR(_curses_panel_panel_move__doc__, @@ -162,7 +165,7 @@ _curses_panel_panel_move_impl(PyCursesPanelObject *self, PyTypeObject *cls, int y, int x); static PyObject * -_curses_panel_panel_move(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_move(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -182,7 +185,8 @@ _curses_panel_panel_move(PyCursesPanelObject *self, PyTypeObject *cls, PyObject int y; int x; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -194,7 +198,7 @@ _curses_panel_panel_move(PyCursesPanelObject *self, PyTypeObject *cls, PyObject if (x == -1 && PyErr_Occurred()) { goto exit; } - return_value = _curses_panel_panel_move_impl(self, cls, y, x); + return_value = _curses_panel_panel_move_impl((PyCursesPanelObject *)self, cls, y, x); exit: return return_value; @@ -213,9 +217,9 @@ static PyObject * _curses_panel_panel_window_impl(PyCursesPanelObject *self); static PyObject * -_curses_panel_panel_window(PyCursesPanelObject *self, PyObject *Py_UNUSED(ignored)) +_curses_panel_panel_window(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _curses_panel_panel_window_impl(self); + return _curses_panel_panel_window_impl((PyCursesPanelObject *)self); } PyDoc_STRVAR(_curses_panel_panel_replace__doc__, @@ -233,7 +237,7 @@ _curses_panel_panel_replace_impl(PyCursesPanelObject *self, PyCursesWindowObject *win); static PyObject * -_curses_panel_panel_replace(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_replace(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -252,7 +256,8 @@ _curses_panel_panel_replace(PyCursesPanelObject *self, PyTypeObject *cls, PyObje PyObject *argsbuf[1]; PyCursesWindowObject *win; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -261,7 +266,7 @@ _curses_panel_panel_replace(PyCursesPanelObject *self, PyTypeObject *cls, PyObje goto exit; } win = (PyCursesWindowObject *)args[0]; - return_value = _curses_panel_panel_replace_impl(self, cls, win); + return_value = _curses_panel_panel_replace_impl((PyCursesPanelObject *)self, cls, win); exit: return return_value; @@ -281,7 +286,7 @@ _curses_panel_panel_set_userptr_impl(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *obj); static PyObject * -_curses_panel_panel_set_userptr(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_set_userptr(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -300,12 +305,13 @@ _curses_panel_panel_set_userptr(PyCursesPanelObject *self, PyTypeObject *cls, Py PyObject *argsbuf[1]; PyObject *obj; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } obj = args[0]; - return_value = _curses_panel_panel_set_userptr_impl(self, cls, obj); + return_value = _curses_panel_panel_set_userptr_impl((PyCursesPanelObject *)self, cls, obj); exit: return return_value; @@ -325,13 +331,13 @@ _curses_panel_panel_userptr_impl(PyCursesPanelObject *self, PyTypeObject *cls); static PyObject * -_curses_panel_panel_userptr(PyCursesPanelObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_userptr(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "userptr() takes no arguments"); return NULL; } - return _curses_panel_panel_userptr_impl(self, cls); + return _curses_panel_panel_userptr_impl((PyCursesPanelObject *)self, cls); } PyDoc_STRVAR(_curses_panel_bottom_panel__doc__, @@ -418,4 +424,4 @@ _curses_panel_update_panels(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _curses_panel_update_panels_impl(module); } -/*[clinic end generated code: output=7bac14e9a1194c87 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=36853ecb4a979814 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h index f7e0aaf7b23649..8291d5d635c79d 100644 --- a/Modules/clinic/_cursesmodule.c.h +++ b/Modules/clinic/_cursesmodule.c.h @@ -35,7 +35,7 @@ _curses_window_addch_impl(PyCursesWindowObject *self, int group_left_1, long attr); static PyObject * -_curses_window_addch(PyCursesWindowObject *self, PyObject *args) +_curses_window_addch(PyObject *self, PyObject *args) { PyObject *return_value = NULL; int group_left_1 = 0; @@ -74,7 +74,7 @@ _curses_window_addch(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.addch requires 1 to 4 arguments"); goto exit; } - return_value = _curses_window_addch_impl(self, group_left_1, y, x, ch, group_right_1, attr); + return_value = _curses_window_addch_impl((PyCursesWindowObject *)self, group_left_1, y, x, ch, group_right_1, attr); exit: return return_value; @@ -107,7 +107,7 @@ _curses_window_addstr_impl(PyCursesWindowObject *self, int group_left_1, long attr); static PyObject * -_curses_window_addstr(PyCursesWindowObject *self, PyObject *args) +_curses_window_addstr(PyObject *self, PyObject *args) { PyObject *return_value = NULL; int group_left_1 = 0; @@ -146,7 +146,7 @@ _curses_window_addstr(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.addstr requires 1 to 4 arguments"); goto exit; } - return_value = _curses_window_addstr_impl(self, group_left_1, y, x, str, group_right_1, attr); + return_value = _curses_window_addstr_impl((PyCursesWindowObject *)self, group_left_1, y, x, str, group_right_1, attr); exit: return return_value; @@ -181,7 +181,7 @@ _curses_window_addnstr_impl(PyCursesWindowObject *self, int group_left_1, int group_right_1, long attr); static PyObject * -_curses_window_addnstr(PyCursesWindowObject *self, PyObject *args) +_curses_window_addnstr(PyObject *self, PyObject *args) { PyObject *return_value = NULL; int group_left_1 = 0; @@ -221,7 +221,7 @@ _curses_window_addnstr(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.addnstr requires 2 to 5 arguments"); goto exit; } - return_value = _curses_window_addnstr_impl(self, group_left_1, y, x, str, n, group_right_1, attr); + return_value = _curses_window_addnstr_impl((PyCursesWindowObject *)self, group_left_1, y, x, str, n, group_right_1, attr); exit: return return_value; @@ -245,7 +245,7 @@ static PyObject * _curses_window_bkgd_impl(PyCursesWindowObject *self, PyObject *ch, long attr); static PyObject * -_curses_window_bkgd(PyCursesWindowObject *self, PyObject *const *args, Py_ssize_t nargs) +_curses_window_bkgd(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *ch; @@ -263,7 +263,7 @@ _curses_window_bkgd(PyCursesWindowObject *self, PyObject *const *args, Py_ssize_ goto exit; } skip_optional: - return_value = _curses_window_bkgd_impl(self, ch, attr); + return_value = _curses_window_bkgd_impl((PyCursesWindowObject *)self, ch, attr); exit: return return_value; @@ -282,7 +282,7 @@ static PyObject * _curses_window_attroff_impl(PyCursesWindowObject *self, long attr); static PyObject * -_curses_window_attroff(PyCursesWindowObject *self, PyObject *arg) +_curses_window_attroff(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; long attr; @@ -291,7 +291,7 @@ _curses_window_attroff(PyCursesWindowObject *self, PyObject *arg) if (attr == -1 && PyErr_Occurred()) { goto exit; } - return_value = _curses_window_attroff_impl(self, attr); + return_value = _curses_window_attroff_impl((PyCursesWindowObject *)self, attr); exit: return return_value; @@ -310,7 +310,7 @@ static PyObject * _curses_window_attron_impl(PyCursesWindowObject *self, long attr); static PyObject * -_curses_window_attron(PyCursesWindowObject *self, PyObject *arg) +_curses_window_attron(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; long attr; @@ -319,7 +319,7 @@ _curses_window_attron(PyCursesWindowObject *self, PyObject *arg) if (attr == -1 && PyErr_Occurred()) { goto exit; } - return_value = _curses_window_attron_impl(self, attr); + return_value = _curses_window_attron_impl((PyCursesWindowObject *)self, attr); exit: return return_value; @@ -338,7 +338,7 @@ static PyObject * _curses_window_attrset_impl(PyCursesWindowObject *self, long attr); static PyObject * -_curses_window_attrset(PyCursesWindowObject *self, PyObject *arg) +_curses_window_attrset(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; long attr; @@ -347,7 +347,7 @@ _curses_window_attrset(PyCursesWindowObject *self, PyObject *arg) if (attr == -1 && PyErr_Occurred()) { goto exit; } - return_value = _curses_window_attrset_impl(self, attr); + return_value = _curses_window_attrset_impl((PyCursesWindowObject *)self, attr); exit: return return_value; @@ -372,7 +372,7 @@ _curses_window_bkgdset_impl(PyCursesWindowObject *self, PyObject *ch, long attr); static PyObject * -_curses_window_bkgdset(PyCursesWindowObject *self, PyObject *const *args, Py_ssize_t nargs) +_curses_window_bkgdset(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *ch; @@ -390,7 +390,7 @@ _curses_window_bkgdset(PyCursesWindowObject *self, PyObject *const *args, Py_ssi goto exit; } skip_optional: - return_value = _curses_window_bkgdset_impl(self, ch, attr); + return_value = _curses_window_bkgdset_impl((PyCursesWindowObject *)self, ch, attr); exit: return return_value; @@ -437,7 +437,7 @@ _curses_window_border_impl(PyCursesWindowObject *self, PyObject *ls, PyObject *br); static PyObject * -_curses_window_border(PyCursesWindowObject *self, PyObject *const *args, Py_ssize_t nargs) +_curses_window_border(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *ls = NULL; @@ -485,7 +485,7 @@ _curses_window_border(PyCursesWindowObject *self, PyObject *const *args, Py_ssiz } br = args[7]; skip_optional: - return_value = _curses_window_border_impl(self, ls, rs, ts, bs, tl, tr, bl, br); + return_value = _curses_window_border_impl((PyCursesWindowObject *)self, ls, rs, ts, bs, tl, tr, bl, br); exit: return return_value; @@ -511,7 +511,7 @@ _curses_window_box_impl(PyCursesWindowObject *self, int group_right_1, PyObject *verch, PyObject *horch); static PyObject * -_curses_window_box(PyCursesWindowObject *self, PyObject *args) +_curses_window_box(PyObject *self, PyObject *args) { PyObject *return_value = NULL; int group_right_1 = 0; @@ -531,7 +531,7 @@ _curses_window_box(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.box requires 0 to 2 arguments"); goto exit; } - return_value = _curses_window_box_impl(self, group_right_1, verch, horch); + return_value = _curses_window_box_impl((PyCursesWindowObject *)self, group_right_1, verch, horch); exit: return return_value; @@ -554,7 +554,7 @@ _curses_window_delch_impl(PyCursesWindowObject *self, int group_right_1, int y, int x); static PyObject * -_curses_window_delch(PyCursesWindowObject *self, PyObject *args) +_curses_window_delch(PyObject *self, PyObject *args) { PyObject *return_value = NULL; int group_right_1 = 0; @@ -574,7 +574,7 @@ _curses_window_delch(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.delch requires 0 to 2 arguments"); goto exit; } - return_value = _curses_window_delch_impl(self, group_right_1, y, x); + return_value = _curses_window_delch_impl((PyCursesWindowObject *)self, group_right_1, y, x); exit: return return_value; @@ -605,7 +605,7 @@ _curses_window_derwin_impl(PyCursesWindowObject *self, int group_left_1, int nlines, int ncols, int begin_y, int begin_x); static PyObject * -_curses_window_derwin(PyCursesWindowObject *self, PyObject *args) +_curses_window_derwin(PyObject *self, PyObject *args) { PyObject *return_value = NULL; int group_left_1 = 0; @@ -630,7 +630,7 @@ _curses_window_derwin(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.derwin requires 2 to 4 arguments"); goto exit; } - return_value = _curses_window_derwin_impl(self, group_left_1, nlines, ncols, begin_y, begin_x); + return_value = _curses_window_derwin_impl((PyCursesWindowObject *)self, group_left_1, nlines, ncols, begin_y, begin_x); exit: return return_value; @@ -655,7 +655,7 @@ _curses_window_echochar_impl(PyCursesWindowObject *self, PyObject *ch, long attr); static PyObject * -_curses_window_echochar(PyCursesWindowObject *self, PyObject *const *args, Py_ssize_t nargs) +_curses_window_echochar(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *ch; @@ -673,7 +673,7 @@ _curses_window_echochar(PyCursesWindowObject *self, PyObject *const *args, Py_ss goto exit; } skip_optional: - return_value = _curses_window_echochar_impl(self, ch, attr); + return_value = _curses_window_echochar_impl((PyCursesWindowObject *)self, ch, attr); exit: return return_value; @@ -699,7 +699,7 @@ static PyObject * _curses_window_enclose_impl(PyCursesWindowObject *self, int y, int x); static PyObject * -_curses_window_enclose(PyCursesWindowObject *self, PyObject *const *args, Py_ssize_t nargs) +_curses_window_enclose(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int y; @@ -716,7 +716,7 @@ _curses_window_enclose(PyCursesWindowObject *self, PyObject *const *args, Py_ssi if (x == -1 && PyErr_Occurred()) { goto exit; } - return_value = _curses_window_enclose_impl(self, y, x); + return_value = _curses_window_enclose_impl((PyCursesWindowObject *)self, y, x); exit: return return_value; @@ -737,12 +737,12 @@ static long _curses_window_getbkgd_impl(PyCursesWindowObject *self); static PyObject * -_curses_window_getbkgd(PyCursesWindowObject *self, PyObject *Py_UNUSED(ignored)) +_curses_window_getbkgd(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; long _return_value; - _return_value = _curses_window_getbkgd_impl(self); + _return_value = _curses_window_getbkgd_impl((PyCursesWindowObject *)self); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -773,7 +773,7 @@ _curses_window_getch_impl(PyCursesWindowObject *self, int group_right_1, int y, int x); static PyObject * -_curses_window_getch(PyCursesWindowObject *self, PyObject *args) +_curses_window_getch(PyObject *self, PyObject *args) { PyObject *return_value = NULL; int group_right_1 = 0; @@ -794,7 +794,7 @@ _curses_window_getch(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.getch requires 0 to 2 arguments"); goto exit; } - _return_value = _curses_window_getch_impl(self, group_right_1, y, x); + _return_value = _curses_window_getch_impl((PyCursesWindowObject *)self, group_right_1, y, x); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -825,7 +825,7 @@ _curses_window_getkey_impl(PyCursesWindowObject *self, int group_right_1, int y, int x); static PyObject * -_curses_window_getkey(PyCursesWindowObject *self, PyObject *args) +_curses_window_getkey(PyObject *self, PyObject *args) { PyObject *return_value = NULL; int group_right_1 = 0; @@ -845,7 +845,7 @@ _curses_window_getkey(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.getkey requires 0 to 2 arguments"); goto exit; } - return_value = _curses_window_getkey_impl(self, group_right_1, y, x); + return_value = _curses_window_getkey_impl((PyCursesWindowObject *)self, group_right_1, y, x); exit: return return_value; @@ -873,7 +873,7 @@ _curses_window_get_wch_impl(PyCursesWindowObject *self, int group_right_1, int y, int x); static PyObject * -_curses_window_get_wch(PyCursesWindowObject *self, PyObject *args) +_curses_window_get_wch(PyObject *self, PyObject *args) { PyObject *return_value = NULL; int group_right_1 = 0; @@ -893,7 +893,7 @@ _curses_window_get_wch(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.get_wch requires 0 to 2 arguments"); goto exit; } - return_value = _curses_window_get_wch_impl(self, group_right_1, y, x); + return_value = _curses_window_get_wch_impl((PyCursesWindowObject *)self, group_right_1, y, x); exit: return return_value; @@ -925,7 +925,7 @@ _curses_window_hline_impl(PyCursesWindowObject *self, int group_left_1, int group_right_1, long attr); static PyObject * -_curses_window_hline(PyCursesWindowObject *self, PyObject *args) +_curses_window_hline(PyObject *self, PyObject *args) { PyObject *return_value = NULL; int group_left_1 = 0; @@ -965,7 +965,7 @@ _curses_window_hline(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.hline requires 2 to 5 arguments"); goto exit; } - return_value = _curses_window_hline_impl(self, group_left_1, y, x, ch, n, group_right_1, attr); + return_value = _curses_window_hline_impl((PyCursesWindowObject *)self, group_left_1, y, x, ch, n, group_right_1, attr); exit: return return_value; @@ -996,7 +996,7 @@ _curses_window_insch_impl(PyCursesWindowObject *self, int group_left_1, long attr); static PyObject * -_curses_window_insch(PyCursesWindowObject *self, PyObject *args) +_curses_window_insch(PyObject *self, PyObject *args) { PyObject *return_value = NULL; int group_left_1 = 0; @@ -1035,7 +1035,7 @@ _curses_window_insch(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.insch requires 1 to 4 arguments"); goto exit; } - return_value = _curses_window_insch_impl(self, group_left_1, y, x, ch, group_right_1, attr); + return_value = _curses_window_insch_impl((PyCursesWindowObject *)self, group_left_1, y, x, ch, group_right_1, attr); exit: return return_value; @@ -1060,7 +1060,7 @@ _curses_window_inch_impl(PyCursesWindowObject *self, int group_right_1, int y, int x); static PyObject * -_curses_window_inch(PyCursesWindowObject *self, PyObject *args) +_curses_window_inch(PyObject *self, PyObject *args) { PyObject *return_value = NULL; int group_right_1 = 0; @@ -1081,7 +1081,7 @@ _curses_window_inch(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.inch requires 0 to 2 arguments"); goto exit; } - _return_value = _curses_window_inch_impl(self, group_right_1, y, x); + _return_value = _curses_window_inch_impl((PyCursesWindowObject *)self, group_right_1, y, x); if ((_return_value == (unsigned long)-1) && PyErr_Occurred()) { goto exit; } @@ -1119,7 +1119,7 @@ _curses_window_insstr_impl(PyCursesWindowObject *self, int group_left_1, long attr); static PyObject * -_curses_window_insstr(PyCursesWindowObject *self, PyObject *args) +_curses_window_insstr(PyObject *self, PyObject *args) { PyObject *return_value = NULL; int group_left_1 = 0; @@ -1158,7 +1158,7 @@ _curses_window_insstr(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.insstr requires 1 to 4 arguments"); goto exit; } - return_value = _curses_window_insstr_impl(self, group_left_1, y, x, str, group_right_1, attr); + return_value = _curses_window_insstr_impl((PyCursesWindowObject *)self, group_left_1, y, x, str, group_right_1, attr); exit: return return_value; @@ -1195,7 +1195,7 @@ _curses_window_insnstr_impl(PyCursesWindowObject *self, int group_left_1, int group_right_1, long attr); static PyObject * -_curses_window_insnstr(PyCursesWindowObject *self, PyObject *args) +_curses_window_insnstr(PyObject *self, PyObject *args) { PyObject *return_value = NULL; int group_left_1 = 0; @@ -1235,7 +1235,7 @@ _curses_window_insnstr(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.insnstr requires 2 to 5 arguments"); goto exit; } - return_value = _curses_window_insnstr_impl(self, group_left_1, y, x, str, n, group_right_1, attr); + return_value = _curses_window_insnstr_impl((PyCursesWindowObject *)self, group_left_1, y, x, str, n, group_right_1, attr); exit: return return_value; @@ -1259,7 +1259,7 @@ static PyObject * _curses_window_is_linetouched_impl(PyCursesWindowObject *self, int line); static PyObject * -_curses_window_is_linetouched(PyCursesWindowObject *self, PyObject *arg) +_curses_window_is_linetouched(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; int line; @@ -1268,7 +1268,7 @@ _curses_window_is_linetouched(PyCursesWindowObject *self, PyObject *arg) if (line == -1 && PyErr_Occurred()) { goto exit; } - return_value = _curses_window_is_linetouched_impl(self, line); + return_value = _curses_window_is_linetouched_impl((PyCursesWindowObject *)self, line); exit: return return_value; @@ -1294,7 +1294,7 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self, int smaxcol); static PyObject * -_curses_window_noutrefresh(PyCursesWindowObject *self, PyObject *args) +_curses_window_noutrefresh(PyObject *self, PyObject *args) { PyObject *return_value = NULL; int group_right_1 = 0; @@ -1318,7 +1318,7 @@ _curses_window_noutrefresh(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.noutrefresh requires 0 to 6 arguments"); goto exit; } - return_value = _curses_window_noutrefresh_impl(self, group_right_1, pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol); + return_value = _curses_window_noutrefresh_impl((PyCursesWindowObject *)self, group_right_1, pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol); exit: return return_value; @@ -1345,9 +1345,9 @@ static PyObject * _curses_window_noutrefresh_impl(PyCursesWindowObject *self); static PyObject * -_curses_window_noutrefresh(PyCursesWindowObject *self, PyObject *Py_UNUSED(ignored)) +_curses_window_noutrefresh(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _curses_window_noutrefresh_impl(self); + return _curses_window_noutrefresh_impl((PyCursesWindowObject *)self); } #endif /* !defined(py_is_pad) */ @@ -1375,7 +1375,7 @@ _curses_window_overlay_impl(PyCursesWindowObject *self, int dmincol, int dmaxrow, int dmaxcol); static PyObject * -_curses_window_overlay(PyCursesWindowObject *self, PyObject *args) +_curses_window_overlay(PyObject *self, PyObject *args) { PyObject *return_value = NULL; PyCursesWindowObject *destwin; @@ -1389,12 +1389,12 @@ _curses_window_overlay(PyCursesWindowObject *self, PyObject *args) switch (PyTuple_GET_SIZE(args)) { case 1: - if (!PyArg_ParseTuple(args, "O!:overlay", &PyCursesWindow_Type, &destwin)) { + if (!PyArg_ParseTuple(args, "O!:overlay", clinic_state()->window_type, &destwin)) { goto exit; } break; case 7: - if (!PyArg_ParseTuple(args, "O!iiiiii:overlay", &PyCursesWindow_Type, &destwin, &sminrow, &smincol, &dminrow, &dmincol, &dmaxrow, &dmaxcol)) { + if (!PyArg_ParseTuple(args, "O!iiiiii:overlay", clinic_state()->window_type, &destwin, &sminrow, &smincol, &dminrow, &dmincol, &dmaxrow, &dmaxcol)) { goto exit; } group_right_1 = 1; @@ -1403,7 +1403,7 @@ _curses_window_overlay(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.overlay requires 1 to 7 arguments"); goto exit; } - return_value = _curses_window_overlay_impl(self, destwin, group_right_1, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol); + return_value = _curses_window_overlay_impl((PyCursesWindowObject *)self, destwin, group_right_1, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol); exit: return return_value; @@ -1434,7 +1434,7 @@ _curses_window_overwrite_impl(PyCursesWindowObject *self, int dmaxcol); static PyObject * -_curses_window_overwrite(PyCursesWindowObject *self, PyObject *args) +_curses_window_overwrite(PyObject *self, PyObject *args) { PyObject *return_value = NULL; PyCursesWindowObject *destwin; @@ -1448,12 +1448,12 @@ _curses_window_overwrite(PyCursesWindowObject *self, PyObject *args) switch (PyTuple_GET_SIZE(args)) { case 1: - if (!PyArg_ParseTuple(args, "O!:overwrite", &PyCursesWindow_Type, &destwin)) { + if (!PyArg_ParseTuple(args, "O!:overwrite", clinic_state()->window_type, &destwin)) { goto exit; } break; case 7: - if (!PyArg_ParseTuple(args, "O!iiiiii:overwrite", &PyCursesWindow_Type, &destwin, &sminrow, &smincol, &dminrow, &dmincol, &dmaxrow, &dmaxcol)) { + if (!PyArg_ParseTuple(args, "O!iiiiii:overwrite", clinic_state()->window_type, &destwin, &sminrow, &smincol, &dminrow, &dmincol, &dmaxrow, &dmaxcol)) { goto exit; } group_right_1 = 1; @@ -1462,7 +1462,7 @@ _curses_window_overwrite(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.overwrite requires 1 to 7 arguments"); goto exit; } - return_value = _curses_window_overwrite_impl(self, destwin, group_right_1, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol); + return_value = _curses_window_overwrite_impl((PyCursesWindowObject *)self, destwin, group_right_1, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol); exit: return return_value; @@ -1499,7 +1499,7 @@ static PyObject * _curses_window_redrawln_impl(PyCursesWindowObject *self, int beg, int num); static PyObject * -_curses_window_redrawln(PyCursesWindowObject *self, PyObject *const *args, Py_ssize_t nargs) +_curses_window_redrawln(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int beg; @@ -1516,7 +1516,7 @@ _curses_window_redrawln(PyCursesWindowObject *self, PyObject *const *args, Py_ss if (num == -1 && PyErr_Occurred()) { goto exit; } - return_value = _curses_window_redrawln_impl(self, beg, num); + return_value = _curses_window_redrawln_impl((PyCursesWindowObject *)self, beg, num); exit: return return_value; @@ -1547,7 +1547,7 @@ _curses_window_refresh_impl(PyCursesWindowObject *self, int group_right_1, int smincol, int smaxrow, int smaxcol); static PyObject * -_curses_window_refresh(PyCursesWindowObject *self, PyObject *args) +_curses_window_refresh(PyObject *self, PyObject *args) { PyObject *return_value = NULL; int group_right_1 = 0; @@ -1571,7 +1571,7 @@ _curses_window_refresh(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.refresh requires 0 to 6 arguments"); goto exit; } - return_value = _curses_window_refresh_impl(self, group_right_1, pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol); + return_value = _curses_window_refresh_impl((PyCursesWindowObject *)self, group_right_1, pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol); exit: return return_value; @@ -1598,7 +1598,7 @@ _curses_window_setscrreg_impl(PyCursesWindowObject *self, int top, int bottom); static PyObject * -_curses_window_setscrreg(PyCursesWindowObject *self, PyObject *const *args, Py_ssize_t nargs) +_curses_window_setscrreg(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int top; @@ -1615,7 +1615,7 @@ _curses_window_setscrreg(PyCursesWindowObject *self, PyObject *const *args, Py_s if (bottom == -1 && PyErr_Occurred()) { goto exit; } - return_value = _curses_window_setscrreg_impl(self, top, bottom); + return_value = _curses_window_setscrreg_impl((PyCursesWindowObject *)self, top, bottom); exit: return return_value; @@ -1645,7 +1645,7 @@ _curses_window_subwin_impl(PyCursesWindowObject *self, int group_left_1, int nlines, int ncols, int begin_y, int begin_x); static PyObject * -_curses_window_subwin(PyCursesWindowObject *self, PyObject *args) +_curses_window_subwin(PyObject *self, PyObject *args) { PyObject *return_value = NULL; int group_left_1 = 0; @@ -1670,7 +1670,7 @@ _curses_window_subwin(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.subwin requires 2 to 4 arguments"); goto exit; } - return_value = _curses_window_subwin_impl(self, group_left_1, nlines, ncols, begin_y, begin_x); + return_value = _curses_window_subwin_impl((PyCursesWindowObject *)self, group_left_1, nlines, ncols, begin_y, begin_x); exit: return return_value; @@ -1693,7 +1693,7 @@ _curses_window_scroll_impl(PyCursesWindowObject *self, int group_right_1, int lines); static PyObject * -_curses_window_scroll(PyCursesWindowObject *self, PyObject *args) +_curses_window_scroll(PyObject *self, PyObject *args) { PyObject *return_value = NULL; int group_right_1 = 0; @@ -1712,7 +1712,7 @@ _curses_window_scroll(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.scroll requires 0 to 1 arguments"); goto exit; } - return_value = _curses_window_scroll_impl(self, group_right_1, lines); + return_value = _curses_window_scroll_impl((PyCursesWindowObject *)self, group_right_1, lines); exit: return return_value; @@ -1733,7 +1733,7 @@ _curses_window_touchline_impl(PyCursesWindowObject *self, int start, int count, int group_right_1, int changed); static PyObject * -_curses_window_touchline(PyCursesWindowObject *self, PyObject *args) +_curses_window_touchline(PyObject *self, PyObject *args) { PyObject *return_value = NULL; int start; @@ -1757,7 +1757,7 @@ _curses_window_touchline(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.touchline requires 2 to 3 arguments"); goto exit; } - return_value = _curses_window_touchline_impl(self, start, count, group_right_1, changed); + return_value = _curses_window_touchline_impl((PyCursesWindowObject *)self, start, count, group_right_1, changed); exit: return return_value; @@ -1787,7 +1787,7 @@ _curses_window_vline_impl(PyCursesWindowObject *self, int group_left_1, int group_right_1, long attr); static PyObject * -_curses_window_vline(PyCursesWindowObject *self, PyObject *args) +_curses_window_vline(PyObject *self, PyObject *args) { PyObject *return_value = NULL; int group_left_1 = 0; @@ -1827,7 +1827,7 @@ _curses_window_vline(PyCursesWindowObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.vline requires 2 to 5 arguments"); goto exit; } - return_value = _curses_window_vline_impl(self, group_left_1, y, x, ch, n, group_right_1, attr); + return_value = _curses_window_vline_impl((PyCursesWindowObject *)self, group_left_1, y, x, ch, n, group_right_1, attr); exit: return return_value; @@ -2714,7 +2714,8 @@ _curses_setupterm(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO const char *term = NULL; int fd = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3693,25 +3694,55 @@ PyDoc_STRVAR(_curses_resizeterm__doc__, {"resizeterm", _PyCFunction_CAST(_curses_resizeterm), METH_FASTCALL, _curses_resizeterm__doc__}, static PyObject * -_curses_resizeterm_impl(PyObject *module, int nlines, int ncols); +_curses_resizeterm_impl(PyObject *module, short nlines, short ncols); static PyObject * _curses_resizeterm(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - int nlines; - int ncols; + short nlines; + short ncols; if (!_PyArg_CheckPositional("resizeterm", nargs, 2, 2)) { goto exit; } - nlines = PyLong_AsInt(args[0]); - if (nlines == -1 && PyErr_Occurred()) { - goto exit; + { + long ival = PyLong_AsLong(args[0]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < SHRT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is less than minimum"); + goto exit; + } + else if (ival > SHRT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is greater than maximum"); + goto exit; + } + else { + nlines = (short) ival; + } } - ncols = PyLong_AsInt(args[1]); - if (ncols == -1 && PyErr_Occurred()) { - goto exit; + { + long ival = PyLong_AsLong(args[1]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < SHRT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is less than minimum"); + goto exit; + } + else if (ival > SHRT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is greater than maximum"); + goto exit; + } + else { + ncols = (short) ival; + } } return_value = _curses_resizeterm_impl(module, nlines, ncols); @@ -3744,25 +3775,55 @@ PyDoc_STRVAR(_curses_resize_term__doc__, {"resize_term", _PyCFunction_CAST(_curses_resize_term), METH_FASTCALL, _curses_resize_term__doc__}, static PyObject * -_curses_resize_term_impl(PyObject *module, int nlines, int ncols); +_curses_resize_term_impl(PyObject *module, short nlines, short ncols); static PyObject * _curses_resize_term(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - int nlines; - int ncols; + short nlines; + short ncols; if (!_PyArg_CheckPositional("resize_term", nargs, 2, 2)) { goto exit; } - nlines = PyLong_AsInt(args[0]); - if (nlines == -1 && PyErr_Occurred()) { - goto exit; + { + long ival = PyLong_AsLong(args[0]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < SHRT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is less than minimum"); + goto exit; + } + else if (ival > SHRT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is greater than maximum"); + goto exit; + } + else { + nlines = (short) ival; + } } - ncols = PyLong_AsInt(args[1]); - if (ncols == -1 && PyErr_Occurred()) { - goto exit; + { + long ival = PyLong_AsLong(args[1]); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < SHRT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is less than minimum"); + goto exit; + } + else if (ival > SHRT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "signed short integer is greater than maximum"); + goto exit; + } + else { + ncols = (short) ival; + } } return_value = _curses_resize_term_impl(module, nlines, ncols); @@ -4318,4 +4379,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #ifndef _CURSES_USE_DEFAULT_COLORS_METHODDEF #define _CURSES_USE_DEFAULT_COLORS_METHODDEF #endif /* !defined(_CURSES_USE_DEFAULT_COLORS_METHODDEF) */ -/*[clinic end generated code: output=96887782374f070a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c4211865ed96c2af input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_datetimemodule.c.h b/Modules/clinic/_datetimemodule.c.h index 48499e0aaf7783..8f33c9e7d4eae5 100644 --- a/Modules/clinic/_datetimemodule.c.h +++ b/Modules/clinic/_datetimemodule.c.h @@ -60,7 +60,8 @@ iso_calendar_date_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int week; int weekday; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 3, 3, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -96,7 +97,7 @@ datetime_date_replace_impl(PyDateTime_Date *self, int year, int month, int day); static PyObject * -datetime_date_replace(PyDateTime_Date *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +datetime_date_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -130,7 +131,8 @@ datetime_date_replace(PyDateTime_Date *self, PyObject *const *args, Py_ssize_t n int month = GET_MONTH(self); int day = GET_DAY(self); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -160,7 +162,7 @@ datetime_date_replace(PyDateTime_Date *self, PyObject *const *args, Py_ssize_t n goto exit; } skip_optional_pos: - return_value = datetime_date_replace_impl(self, year, month, day); + return_value = datetime_date_replace_impl((PyDateTime_Date *)self, year, month, day); exit: return return_value; @@ -182,7 +184,7 @@ datetime_time_replace_impl(PyDateTime_Time *self, int hour, int minute, int fold); static PyObject * -datetime_time_replace(PyDateTime_Time *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +datetime_time_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -216,10 +218,11 @@ datetime_time_replace(PyDateTime_Time *self, PyObject *const *args, Py_ssize_t n int minute = TIME_GET_MINUTE(self); int second = TIME_GET_SECOND(self); int microsecond = TIME_GET_MICROSECOND(self); - PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None; + PyObject *tzinfo = HASTZINFO(self) ? ((PyDateTime_Time *)self)->tzinfo : Py_None; int fold = TIME_GET_FOLD(self); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 5, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 5, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -277,7 +280,7 @@ datetime_time_replace(PyDateTime_Time *self, PyObject *const *args, Py_ssize_t n goto exit; } skip_optional_kwonly: - return_value = datetime_time_replace_impl(self, hour, minute, second, microsecond, tzinfo, fold); + return_value = datetime_time_replace_impl((PyDateTime_Time *)self, hour, minute, second, microsecond, tzinfo, fold); exit: return return_value; @@ -333,7 +336,8 @@ datetime_datetime_now(PyTypeObject *type, PyObject *const *args, Py_ssize_t narg Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *tz = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -366,7 +370,7 @@ datetime_datetime_replace_impl(PyDateTime_DateTime *self, int year, int fold); static PyObject * -datetime_datetime_replace(PyDateTime_DateTime *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +datetime_datetime_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -403,10 +407,11 @@ datetime_datetime_replace(PyDateTime_DateTime *self, PyObject *const *args, Py_s int minute = DATE_GET_MINUTE(self); int second = DATE_GET_SECOND(self); int microsecond = DATE_GET_MICROSECOND(self); - PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None; + PyObject *tzinfo = HASTZINFO(self) ? ((PyDateTime_DateTime *)self)->tzinfo : Py_None; int fold = DATE_GET_FOLD(self); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 8, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 8, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -491,9 +496,9 @@ datetime_datetime_replace(PyDateTime_DateTime *self, PyObject *const *args, Py_s goto exit; } skip_optional_kwonly: - return_value = datetime_datetime_replace_impl(self, year, month, day, hour, minute, second, microsecond, tzinfo, fold); + return_value = datetime_datetime_replace_impl((PyDateTime_DateTime *)self, year, month, day, hour, minute, second, microsecond, tzinfo, fold); exit: return return_value; } -/*[clinic end generated code: output=c7a04b865b1e0890 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8acf62fbc7328f79 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_dbmmodule.c.h b/Modules/clinic/_dbmmodule.c.h index d06271e18a49b2..5e503194408776 100644 --- a/Modules/clinic/_dbmmodule.c.h +++ b/Modules/clinic/_dbmmodule.c.h @@ -2,6 +2,9 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_dbm_dbm_close__doc__, @@ -17,9 +20,9 @@ static PyObject * _dbm_dbm_close_impl(dbmobject *self); static PyObject * -_dbm_dbm_close(dbmobject *self, PyObject *Py_UNUSED(ignored)) +_dbm_dbm_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _dbm_dbm_close_impl(self); + return _dbm_dbm_close_impl((dbmobject *)self); } PyDoc_STRVAR(_dbm_dbm_keys__doc__, @@ -35,13 +38,13 @@ static PyObject * _dbm_dbm_keys_impl(dbmobject *self, PyTypeObject *cls); static PyObject * -_dbm_dbm_keys(dbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_dbm_dbm_keys(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "keys() takes no arguments"); return NULL; } - return _dbm_dbm_keys_impl(self, cls); + return _dbm_dbm_keys_impl((dbmobject *)self, cls); } PyDoc_STRVAR(_dbm_dbm_get__doc__, @@ -58,7 +61,7 @@ _dbm_dbm_get_impl(dbmobject *self, PyTypeObject *cls, const char *key, Py_ssize_t key_length, PyObject *default_value); static PyObject * -_dbm_dbm_get(dbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_dbm_dbm_get(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -82,7 +85,7 @@ _dbm_dbm_get(dbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize &key, &key_length, &default_value)) { goto exit; } - return_value = _dbm_dbm_get_impl(self, cls, key, key_length, default_value); + return_value = _dbm_dbm_get_impl((dbmobject *)self, cls, key, key_length, default_value); exit: return return_value; @@ -104,7 +107,7 @@ _dbm_dbm_setdefault_impl(dbmobject *self, PyTypeObject *cls, const char *key, Py_ssize_t key_length, PyObject *default_value); static PyObject * -_dbm_dbm_setdefault(dbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_dbm_dbm_setdefault(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -128,7 +131,7 @@ _dbm_dbm_setdefault(dbmobject *self, PyTypeObject *cls, PyObject *const *args, P &key, &key_length, &default_value)) { goto exit; } - return_value = _dbm_dbm_setdefault_impl(self, cls, key, key_length, default_value); + return_value = _dbm_dbm_setdefault_impl((dbmobject *)self, cls, key, key_length, default_value); exit: return return_value; @@ -147,13 +150,13 @@ static PyObject * _dbm_dbm_clear_impl(dbmobject *self, PyTypeObject *cls); static PyObject * -_dbm_dbm_clear(dbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_dbm_dbm_clear(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "clear() takes no arguments"); return NULL; } - return _dbm_dbm_clear_impl(self, cls); + return _dbm_dbm_clear_impl((dbmobject *)self, cls); } PyDoc_STRVAR(dbmopen__doc__, @@ -218,4 +221,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=743ce0cea116747e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3b456118f231b160 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h index 10b2dd1c15f7fd..78391887b615cf 100644 --- a/Modules/clinic/_elementtree.c.h +++ b/Modules/clinic/_elementtree.c.h @@ -4,7 +4,7 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() @@ -22,7 +22,7 @@ _elementtree_Element_append_impl(ElementObject *self, PyTypeObject *cls, PyObject *subelement); static PyObject * -_elementtree_Element_append(ElementObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_elementtree_Element_append(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -41,7 +41,8 @@ _elementtree_Element_append(ElementObject *self, PyTypeObject *cls, PyObject *co PyObject *argsbuf[1]; PyObject *subelement; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -50,7 +51,7 @@ _elementtree_Element_append(ElementObject *self, PyTypeObject *cls, PyObject *co goto exit; } subelement = args[0]; - return_value = _elementtree_Element_append_impl(self, cls, subelement); + return_value = _elementtree_Element_append_impl((ElementObject *)self, cls, subelement); exit: return return_value; @@ -68,9 +69,9 @@ static PyObject * _elementtree_Element_clear_impl(ElementObject *self); static PyObject * -_elementtree_Element_clear(ElementObject *self, PyObject *Py_UNUSED(ignored)) +_elementtree_Element_clear(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _elementtree_Element_clear_impl(self); + return _elementtree_Element_clear_impl((ElementObject *)self); } PyDoc_STRVAR(_elementtree_Element___copy____doc__, @@ -85,13 +86,13 @@ static PyObject * _elementtree_Element___copy___impl(ElementObject *self, PyTypeObject *cls); static PyObject * -_elementtree_Element___copy__(ElementObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_elementtree_Element___copy__(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "__copy__() takes no arguments"); return NULL; } - return _elementtree_Element___copy___impl(self, cls); + return _elementtree_Element___copy___impl((ElementObject *)self, cls); } PyDoc_STRVAR(_elementtree_Element___deepcopy____doc__, @@ -106,7 +107,7 @@ static PyObject * _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo); static PyObject * -_elementtree_Element___deepcopy__(ElementObject *self, PyObject *arg) +_elementtree_Element___deepcopy__(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; PyObject *memo; @@ -116,7 +117,7 @@ _elementtree_Element___deepcopy__(ElementObject *self, PyObject *arg) goto exit; } memo = arg; - return_value = _elementtree_Element___deepcopy___impl(self, memo); + return_value = _elementtree_Element___deepcopy___impl((ElementObject *)self, memo); exit: return return_value; @@ -134,12 +135,12 @@ static size_t _elementtree_Element___sizeof___impl(ElementObject *self); static PyObject * -_elementtree_Element___sizeof__(ElementObject *self, PyObject *Py_UNUSED(ignored)) +_elementtree_Element___sizeof__(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; size_t _return_value; - _return_value = _elementtree_Element___sizeof___impl(self); + _return_value = _elementtree_Element___sizeof___impl((ElementObject *)self); if ((_return_value == (size_t)-1) && PyErr_Occurred()) { goto exit; } @@ -161,9 +162,9 @@ static PyObject * _elementtree_Element___getstate___impl(ElementObject *self); static PyObject * -_elementtree_Element___getstate__(ElementObject *self, PyObject *Py_UNUSED(ignored)) +_elementtree_Element___getstate__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _elementtree_Element___getstate___impl(self); + return _elementtree_Element___getstate___impl((ElementObject *)self); } PyDoc_STRVAR(_elementtree_Element___setstate____doc__, @@ -179,7 +180,7 @@ _elementtree_Element___setstate___impl(ElementObject *self, PyTypeObject *cls, PyObject *state); static PyObject * -_elementtree_Element___setstate__(ElementObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_elementtree_Element___setstate__(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -198,12 +199,13 @@ _elementtree_Element___setstate__(ElementObject *self, PyTypeObject *cls, PyObje PyObject *argsbuf[1]; PyObject *state; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } state = args[0]; - return_value = _elementtree_Element___setstate___impl(self, cls, state); + return_value = _elementtree_Element___setstate___impl((ElementObject *)self, cls, state); exit: return return_value; @@ -222,7 +224,7 @@ _elementtree_Element_extend_impl(ElementObject *self, PyTypeObject *cls, PyObject *elements); static PyObject * -_elementtree_Element_extend(ElementObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_elementtree_Element_extend(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -241,12 +243,13 @@ _elementtree_Element_extend(ElementObject *self, PyTypeObject *cls, PyObject *co PyObject *argsbuf[1]; PyObject *elements; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } elements = args[0]; - return_value = _elementtree_Element_extend_impl(self, cls, elements); + return_value = _elementtree_Element_extend_impl((ElementObject *)self, cls, elements); exit: return return_value; @@ -265,7 +268,7 @@ _elementtree_Element_find_impl(ElementObject *self, PyTypeObject *cls, PyObject *path, PyObject *namespaces); static PyObject * -_elementtree_Element_find(ElementObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_elementtree_Element_find(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -298,7 +301,8 @@ _elementtree_Element_find(ElementObject *self, PyTypeObject *cls, PyObject *cons PyObject *path; PyObject *namespaces = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -308,7 +312,7 @@ _elementtree_Element_find(ElementObject *self, PyTypeObject *cls, PyObject *cons } namespaces = args[1]; skip_optional_pos: - return_value = _elementtree_Element_find_impl(self, cls, path, namespaces); + return_value = _elementtree_Element_find_impl((ElementObject *)self, cls, path, namespaces); exit: return return_value; @@ -328,7 +332,7 @@ _elementtree_Element_findtext_impl(ElementObject *self, PyTypeObject *cls, PyObject *namespaces); static PyObject * -_elementtree_Element_findtext(ElementObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_elementtree_Element_findtext(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -362,7 +366,8 @@ _elementtree_Element_findtext(ElementObject *self, PyTypeObject *cls, PyObject * PyObject *default_value = Py_None; PyObject *namespaces = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -378,7 +383,7 @@ _elementtree_Element_findtext(ElementObject *self, PyTypeObject *cls, PyObject * } namespaces = args[2]; skip_optional_pos: - return_value = _elementtree_Element_findtext_impl(self, cls, path, default_value, namespaces); + return_value = _elementtree_Element_findtext_impl((ElementObject *)self, cls, path, default_value, namespaces); exit: return return_value; @@ -397,7 +402,7 @@ _elementtree_Element_findall_impl(ElementObject *self, PyTypeObject *cls, PyObject *path, PyObject *namespaces); static PyObject * -_elementtree_Element_findall(ElementObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_elementtree_Element_findall(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -430,7 +435,8 @@ _elementtree_Element_findall(ElementObject *self, PyTypeObject *cls, PyObject *c PyObject *path; PyObject *namespaces = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -440,7 +446,7 @@ _elementtree_Element_findall(ElementObject *self, PyTypeObject *cls, PyObject *c } namespaces = args[1]; skip_optional_pos: - return_value = _elementtree_Element_findall_impl(self, cls, path, namespaces); + return_value = _elementtree_Element_findall_impl((ElementObject *)self, cls, path, namespaces); exit: return return_value; @@ -459,7 +465,7 @@ _elementtree_Element_iterfind_impl(ElementObject *self, PyTypeObject *cls, PyObject *path, PyObject *namespaces); static PyObject * -_elementtree_Element_iterfind(ElementObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_elementtree_Element_iterfind(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -492,7 +498,8 @@ _elementtree_Element_iterfind(ElementObject *self, PyTypeObject *cls, PyObject * PyObject *path; PyObject *namespaces = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -502,7 +509,7 @@ _elementtree_Element_iterfind(ElementObject *self, PyTypeObject *cls, PyObject * } namespaces = args[1]; skip_optional_pos: - return_value = _elementtree_Element_iterfind_impl(self, cls, path, namespaces); + return_value = _elementtree_Element_iterfind_impl((ElementObject *)self, cls, path, namespaces); exit: return return_value; @@ -521,7 +528,7 @@ _elementtree_Element_get_impl(ElementObject *self, PyObject *key, PyObject *default_value); static PyObject * -_elementtree_Element_get(ElementObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_elementtree_Element_get(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -554,7 +561,8 @@ _elementtree_Element_get(ElementObject *self, PyObject *const *args, Py_ssize_t PyObject *key; PyObject *default_value = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -564,7 +572,7 @@ _elementtree_Element_get(ElementObject *self, PyObject *const *args, Py_ssize_t } default_value = args[1]; skip_optional_pos: - return_value = _elementtree_Element_get_impl(self, key, default_value); + return_value = _elementtree_Element_get_impl((ElementObject *)self, key, default_value); exit: return return_value; @@ -583,7 +591,7 @@ _elementtree_Element_iter_impl(ElementObject *self, PyTypeObject *cls, PyObject *tag); static PyObject * -_elementtree_Element_iter(ElementObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_elementtree_Element_iter(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -615,7 +623,8 @@ _elementtree_Element_iter(ElementObject *self, PyTypeObject *cls, PyObject *cons Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *tag = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -624,7 +633,7 @@ _elementtree_Element_iter(ElementObject *self, PyTypeObject *cls, PyObject *cons } tag = args[0]; skip_optional_pos: - return_value = _elementtree_Element_iter_impl(self, cls, tag); + return_value = _elementtree_Element_iter_impl((ElementObject *)self, cls, tag); exit: return return_value; @@ -642,13 +651,13 @@ static PyObject * _elementtree_Element_itertext_impl(ElementObject *self, PyTypeObject *cls); static PyObject * -_elementtree_Element_itertext(ElementObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_elementtree_Element_itertext(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "itertext() takes no arguments"); return NULL; } - return _elementtree_Element_itertext_impl(self, cls); + return _elementtree_Element_itertext_impl((ElementObject *)self, cls); } PyDoc_STRVAR(_elementtree_Element_insert__doc__, @@ -664,7 +673,7 @@ _elementtree_Element_insert_impl(ElementObject *self, Py_ssize_t index, PyObject *subelement); static PyObject * -_elementtree_Element_insert(ElementObject *self, PyObject *const *args, Py_ssize_t nargs) +_elementtree_Element_insert(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t index; @@ -690,7 +699,7 @@ _elementtree_Element_insert(ElementObject *self, PyObject *const *args, Py_ssize goto exit; } subelement = args[1]; - return_value = _elementtree_Element_insert_impl(self, index, subelement); + return_value = _elementtree_Element_insert_impl((ElementObject *)self, index, subelement); exit: return return_value; @@ -708,9 +717,9 @@ static PyObject * _elementtree_Element_items_impl(ElementObject *self); static PyObject * -_elementtree_Element_items(ElementObject *self, PyObject *Py_UNUSED(ignored)) +_elementtree_Element_items(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _elementtree_Element_items_impl(self); + return _elementtree_Element_items_impl((ElementObject *)self); } PyDoc_STRVAR(_elementtree_Element_keys__doc__, @@ -725,9 +734,9 @@ static PyObject * _elementtree_Element_keys_impl(ElementObject *self); static PyObject * -_elementtree_Element_keys(ElementObject *self, PyObject *Py_UNUSED(ignored)) +_elementtree_Element_keys(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _elementtree_Element_keys_impl(self); + return _elementtree_Element_keys_impl((ElementObject *)self); } PyDoc_STRVAR(_elementtree_Element_makeelement__doc__, @@ -743,7 +752,7 @@ _elementtree_Element_makeelement_impl(ElementObject *self, PyTypeObject *cls, PyObject *tag, PyObject *attrib); static PyObject * -_elementtree_Element_makeelement(ElementObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_elementtree_Element_makeelement(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -763,7 +772,8 @@ _elementtree_Element_makeelement(ElementObject *self, PyTypeObject *cls, PyObjec PyObject *tag; PyObject *attrib; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -773,7 +783,7 @@ _elementtree_Element_makeelement(ElementObject *self, PyTypeObject *cls, PyObjec goto exit; } attrib = args[1]; - return_value = _elementtree_Element_makeelement_impl(self, cls, tag, attrib); + return_value = _elementtree_Element_makeelement_impl((ElementObject *)self, cls, tag, attrib); exit: return return_value; @@ -791,7 +801,7 @@ static PyObject * _elementtree_Element_remove_impl(ElementObject *self, PyObject *subelement); static PyObject * -_elementtree_Element_remove(ElementObject *self, PyObject *arg) +_elementtree_Element_remove(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; PyObject *subelement; @@ -801,7 +811,7 @@ _elementtree_Element_remove(ElementObject *self, PyObject *arg) goto exit; } subelement = arg; - return_value = _elementtree_Element_remove_impl(self, subelement); + return_value = _elementtree_Element_remove_impl((ElementObject *)self, subelement); exit: return return_value; @@ -820,7 +830,7 @@ _elementtree_Element_set_impl(ElementObject *self, PyObject *key, PyObject *value); static PyObject * -_elementtree_Element_set(ElementObject *self, PyObject *const *args, Py_ssize_t nargs) +_elementtree_Element_set(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *key; @@ -831,7 +841,7 @@ _elementtree_Element_set(ElementObject *self, PyObject *const *args, Py_ssize_t } key = args[0]; value = args[1]; - return_value = _elementtree_Element_set_impl(self, key, value); + return_value = _elementtree_Element_set_impl((ElementObject *)self, key, value); exit: return return_value; @@ -883,7 +893,8 @@ _elementtree_TreeBuilder___init__(PyObject *self, PyObject *args, PyObject *kwar int insert_comments = 0; int insert_pis = 0; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -1002,7 +1013,7 @@ _elementtree_TreeBuilder_pi_impl(TreeBuilderObject *self, PyObject *target, PyObject *text); static PyObject * -_elementtree_TreeBuilder_pi(TreeBuilderObject *self, PyObject *const *args, Py_ssize_t nargs) +_elementtree_TreeBuilder_pi(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *target; @@ -1017,7 +1028,7 @@ _elementtree_TreeBuilder_pi(TreeBuilderObject *self, PyObject *const *args, Py_s } text = args[1]; skip_optional: - return_value = _elementtree_TreeBuilder_pi_impl(self, target, text); + return_value = _elementtree_TreeBuilder_pi_impl((TreeBuilderObject *)self, target, text); exit: return return_value; @@ -1035,9 +1046,9 @@ static PyObject * _elementtree_TreeBuilder_close_impl(TreeBuilderObject *self); static PyObject * -_elementtree_TreeBuilder_close(TreeBuilderObject *self, PyObject *Py_UNUSED(ignored)) +_elementtree_TreeBuilder_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _elementtree_TreeBuilder_close_impl(self); + return _elementtree_TreeBuilder_close_impl((TreeBuilderObject *)self); } PyDoc_STRVAR(_elementtree_TreeBuilder_start__doc__, @@ -1053,7 +1064,7 @@ _elementtree_TreeBuilder_start_impl(TreeBuilderObject *self, PyObject *tag, PyObject *attrs); static PyObject * -_elementtree_TreeBuilder_start(TreeBuilderObject *self, PyObject *const *args, Py_ssize_t nargs) +_elementtree_TreeBuilder_start(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *tag; @@ -1068,7 +1079,7 @@ _elementtree_TreeBuilder_start(TreeBuilderObject *self, PyObject *const *args, P goto exit; } attrs = args[1]; - return_value = _elementtree_TreeBuilder_start_impl(self, tag, attrs); + return_value = _elementtree_TreeBuilder_start_impl((TreeBuilderObject *)self, tag, attrs); exit: return return_value; @@ -1114,7 +1125,8 @@ _elementtree_XMLParser___init__(PyObject *self, PyObject *args, PyObject *kwargs PyObject *target = Py_None; const char *encoding = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 0, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -1164,9 +1176,9 @@ static PyObject * _elementtree_XMLParser_close_impl(XMLParserObject *self); static PyObject * -_elementtree_XMLParser_close(XMLParserObject *self, PyObject *Py_UNUSED(ignored)) +_elementtree_XMLParser_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _elementtree_XMLParser_close_impl(self); + return _elementtree_XMLParser_close_impl((XMLParserObject *)self); } PyDoc_STRVAR(_elementtree_XMLParser_flush__doc__, @@ -1181,9 +1193,9 @@ static PyObject * _elementtree_XMLParser_flush_impl(XMLParserObject *self); static PyObject * -_elementtree_XMLParser_flush(XMLParserObject *self, PyObject *Py_UNUSED(ignored)) +_elementtree_XMLParser_flush(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _elementtree_XMLParser_flush_impl(self); + return _elementtree_XMLParser_flush_impl((XMLParserObject *)self); } PyDoc_STRVAR(_elementtree_XMLParser_feed__doc__, @@ -1216,7 +1228,7 @@ _elementtree_XMLParser__setevents_impl(XMLParserObject *self, PyObject *events_to_report); static PyObject * -_elementtree_XMLParser__setevents(XMLParserObject *self, PyObject *const *args, Py_ssize_t nargs) +_elementtree_XMLParser__setevents(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *events_queue; @@ -1231,9 +1243,9 @@ _elementtree_XMLParser__setevents(XMLParserObject *self, PyObject *const *args, } events_to_report = args[1]; skip_optional: - return_value = _elementtree_XMLParser__setevents_impl(self, events_queue, events_to_report); + return_value = _elementtree_XMLParser__setevents_impl((XMLParserObject *)self, events_queue, events_to_report); exit: return return_value; } -/*[clinic end generated code: output=aed9f53eeb0404e0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e5c758958f14f102 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_functoolsmodule.c.h b/Modules/clinic/_functoolsmodule.c.h index e98984dc4d3a09..afd5eb4eb12b78 100644 --- a/Modules/clinic/_functoolsmodule.c.h +++ b/Modules/clinic/_functoolsmodule.c.h @@ -56,7 +56,8 @@ _functools_cmp_to_key(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *argsbuf[1]; PyObject *mycmp; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -67,6 +68,79 @@ _functools_cmp_to_key(PyObject *module, PyObject *const *args, Py_ssize_t nargs, return return_value; } +PyDoc_STRVAR(_functools_reduce__doc__, +"reduce($module, function, iterable, /, initial=)\n" +"--\n" +"\n" +"Apply a function of two arguments cumulatively to the items of an iterable, from left to right.\n" +"\n" +"This effectively reduces the iterable to a single value. If initial is present,\n" +"it is placed before the items of the iterable in the calculation, and serves as\n" +"a default when the iterable is empty.\n" +"\n" +"For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5])\n" +"calculates ((((1 + 2) + 3) + 4) + 5)."); + +#define _FUNCTOOLS_REDUCE_METHODDEF \ + {"reduce", _PyCFunction_CAST(_functools_reduce), METH_FASTCALL|METH_KEYWORDS, _functools_reduce__doc__}, + +static PyObject * +_functools_reduce_impl(PyObject *module, PyObject *func, PyObject *seq, + PyObject *result); + +static PyObject * +_functools_reduce(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(initial), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "", "initial", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "reduce", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *func; + PyObject *seq; + PyObject *result = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + func = args[0]; + seq = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + result = args[2]; +skip_optional_pos: + return_value = _functools_reduce_impl(module, func, seq, result); + +exit: + return return_value; +} + PyDoc_STRVAR(_functools__lru_cache_wrapper_cache_info__doc__, "cache_info($self, /)\n" "--\n" @@ -114,4 +188,4 @@ _functools__lru_cache_wrapper_cache_clear(PyObject *self, PyObject *Py_UNUSED(ig return return_value; } -/*[clinic end generated code: output=755265bb6d5ea751 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e6edcc01f0720daf input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_gdbmmodule.c.h b/Modules/clinic/_gdbmmodule.c.h index 626e4678809d4f..00950f18e53541 100644 --- a/Modules/clinic/_gdbmmodule.c.h +++ b/Modules/clinic/_gdbmmodule.c.h @@ -2,6 +2,9 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_gdbm_gdbm_get__doc__, @@ -17,7 +20,7 @@ static PyObject * _gdbm_gdbm_get_impl(gdbmobject *self, PyObject *key, PyObject *default_value); static PyObject * -_gdbm_gdbm_get(gdbmobject *self, PyObject *const *args, Py_ssize_t nargs) +_gdbm_gdbm_get(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *key; @@ -32,7 +35,7 @@ _gdbm_gdbm_get(gdbmobject *self, PyObject *const *args, Py_ssize_t nargs) } default_value = args[1]; skip_optional: - return_value = _gdbm_gdbm_get_impl(self, key, default_value); + return_value = _gdbm_gdbm_get_impl((gdbmobject *)self, key, default_value); exit: return return_value; @@ -52,7 +55,7 @@ _gdbm_gdbm_setdefault_impl(gdbmobject *self, PyObject *key, PyObject *default_value); static PyObject * -_gdbm_gdbm_setdefault(gdbmobject *self, PyObject *const *args, Py_ssize_t nargs) +_gdbm_gdbm_setdefault(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *key; @@ -67,7 +70,7 @@ _gdbm_gdbm_setdefault(gdbmobject *self, PyObject *const *args, Py_ssize_t nargs) } default_value = args[1]; skip_optional: - return_value = _gdbm_gdbm_setdefault_impl(self, key, default_value); + return_value = _gdbm_gdbm_setdefault_impl((gdbmobject *)self, key, default_value); exit: return return_value; @@ -86,9 +89,9 @@ static PyObject * _gdbm_gdbm_close_impl(gdbmobject *self); static PyObject * -_gdbm_gdbm_close(gdbmobject *self, PyObject *Py_UNUSED(ignored)) +_gdbm_gdbm_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _gdbm_gdbm_close_impl(self); + return _gdbm_gdbm_close_impl((gdbmobject *)self); } PyDoc_STRVAR(_gdbm_gdbm_keys__doc__, @@ -104,13 +107,13 @@ static PyObject * _gdbm_gdbm_keys_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * -_gdbm_gdbm_keys(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_gdbm_gdbm_keys(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "keys() takes no arguments"); return NULL; } - return _gdbm_gdbm_keys_impl(self, cls); + return _gdbm_gdbm_keys_impl((gdbmobject *)self, cls); } PyDoc_STRVAR(_gdbm_gdbm_firstkey__doc__, @@ -130,13 +133,13 @@ static PyObject * _gdbm_gdbm_firstkey_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * -_gdbm_gdbm_firstkey(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_gdbm_gdbm_firstkey(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "firstkey() takes no arguments"); return NULL; } - return _gdbm_gdbm_firstkey_impl(self, cls); + return _gdbm_gdbm_firstkey_impl((gdbmobject *)self, cls); } PyDoc_STRVAR(_gdbm_gdbm_nextkey__doc__, @@ -161,7 +164,7 @@ _gdbm_gdbm_nextkey_impl(gdbmobject *self, PyTypeObject *cls, const char *key, Py_ssize_t key_length); static PyObject * -_gdbm_gdbm_nextkey(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_gdbm_gdbm_nextkey(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -184,7 +187,7 @@ _gdbm_gdbm_nextkey(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, P &key, &key_length)) { goto exit; } - return_value = _gdbm_gdbm_nextkey_impl(self, cls, key, key_length); + return_value = _gdbm_gdbm_nextkey_impl((gdbmobject *)self, cls, key, key_length); exit: return return_value; @@ -209,13 +212,13 @@ static PyObject * _gdbm_gdbm_reorganize_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * -_gdbm_gdbm_reorganize(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_gdbm_gdbm_reorganize(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "reorganize() takes no arguments"); return NULL; } - return _gdbm_gdbm_reorganize_impl(self, cls); + return _gdbm_gdbm_reorganize_impl((gdbmobject *)self, cls); } PyDoc_STRVAR(_gdbm_gdbm_sync__doc__, @@ -234,13 +237,13 @@ static PyObject * _gdbm_gdbm_sync_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * -_gdbm_gdbm_sync(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_gdbm_gdbm_sync(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "sync() takes no arguments"); return NULL; } - return _gdbm_gdbm_sync_impl(self, cls); + return _gdbm_gdbm_sync_impl((gdbmobject *)self, cls); } PyDoc_STRVAR(_gdbm_gdbm_clear__doc__, @@ -256,13 +259,13 @@ static PyObject * _gdbm_gdbm_clear_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * -_gdbm_gdbm_clear(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_gdbm_gdbm_clear(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "clear() takes no arguments"); return NULL; } - return _gdbm_gdbm_clear_impl(self, cls); + return _gdbm_gdbm_clear_impl((gdbmobject *)self, cls); } PyDoc_STRVAR(dbmopen__doc__, @@ -340,4 +343,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=6b4c19905ac9967d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d974cb39e4ee5d67 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h index 58650dff288444..d219b80b791a66 100644 --- a/Modules/clinic/_hashopenssl.c.h +++ b/Modules/clinic/_hashopenssl.c.h @@ -22,9 +22,9 @@ static PyObject * EVP_copy_impl(EVPobject *self); static PyObject * -EVP_copy(EVPobject *self, PyObject *Py_UNUSED(ignored)) +EVP_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return EVP_copy_impl(self); + return EVP_copy_impl((EVPobject *)self); } PyDoc_STRVAR(EVP_digest__doc__, @@ -40,9 +40,9 @@ static PyObject * EVP_digest_impl(EVPobject *self); static PyObject * -EVP_digest(EVPobject *self, PyObject *Py_UNUSED(ignored)) +EVP_digest(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return EVP_digest_impl(self); + return EVP_digest_impl((EVPobject *)self); } PyDoc_STRVAR(EVP_hexdigest__doc__, @@ -58,9 +58,9 @@ static PyObject * EVP_hexdigest_impl(EVPobject *self); static PyObject * -EVP_hexdigest(EVPobject *self, PyObject *Py_UNUSED(ignored)) +EVP_hexdigest(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return EVP_hexdigest_impl(self); + return EVP_hexdigest_impl((EVPobject *)self); } PyDoc_STRVAR(EVP_update__doc__, @@ -87,7 +87,7 @@ static PyObject * EVPXOF_digest_impl(EVPobject *self, Py_ssize_t length); static PyObject * -EVPXOF_digest(EVPobject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +EVPXOF_digest(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -118,7 +118,8 @@ EVPXOF_digest(EVPobject *self, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *argsbuf[1]; Py_ssize_t length; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -134,7 +135,7 @@ EVPXOF_digest(EVPobject *self, PyObject *const *args, Py_ssize_t nargs, PyObject } length = ival; } - return_value = EVPXOF_digest_impl(self, length); + return_value = EVPXOF_digest_impl((EVPobject *)self, length); exit: return return_value; @@ -157,7 +158,7 @@ static PyObject * EVPXOF_hexdigest_impl(EVPobject *self, Py_ssize_t length); static PyObject * -EVPXOF_hexdigest(EVPobject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +EVPXOF_hexdigest(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -188,7 +189,8 @@ EVPXOF_hexdigest(EVPobject *self, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *argsbuf[1]; Py_ssize_t length; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -204,7 +206,7 @@ EVPXOF_hexdigest(EVPobject *self, PyObject *const *args, Py_ssize_t nargs, PyObj } length = ival; } - return_value = EVPXOF_hexdigest_impl(self, length); + return_value = EVPXOF_hexdigest_impl((EVPobject *)self, length); exit: return return_value; @@ -265,7 +267,8 @@ EVP_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -341,7 +344,8 @@ _hashlib_openssl_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -416,7 +420,8 @@ _hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -491,7 +496,8 @@ _hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -566,7 +572,8 @@ _hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -641,7 +648,8 @@ _hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -716,7 +724,8 @@ _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -793,7 +802,8 @@ _hashlib_openssl_sha3_224(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -872,7 +882,8 @@ _hashlib_openssl_sha3_256(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -951,7 +962,8 @@ _hashlib_openssl_sha3_384(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1030,7 +1042,8 @@ _hashlib_openssl_sha3_512(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1109,7 +1122,8 @@ _hashlib_openssl_shake_128(PyObject *module, PyObject *const *args, Py_ssize_t n PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1188,7 +1202,8 @@ _hashlib_openssl_shake_256(PyObject *module, PyObject *const *args, Py_ssize_t n PyObject *data_obj = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1270,7 +1285,8 @@ pbkdf2_hmac(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject long iterations; PyObject *dklen_obj = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 5, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 4, /*maxpos*/ 5, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1347,7 +1363,7 @@ _hashlib_scrypt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(password), &_Py_ID(salt), &_Py_ID(n), &_Py_ID(r), &_Py_ID(p), &_Py_ID(maxmem), &_Py_ID(dklen), }, + .ob_item = { &_Py_ID(password), &_Py_ID(salt), _Py_LATIN1_CHR('n'), _Py_LATIN1_CHR('r'), _Py_LATIN1_CHR('p'), &_Py_ID(maxmem), &_Py_ID(dklen), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1373,7 +1389,8 @@ _hashlib_scrypt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj long maxmem = 0; long dklen = 64; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1499,7 +1516,8 @@ _hashlib_hmac_singleshot(PyObject *module, PyObject *const *args, Py_ssize_t nar Py_buffer msg = {NULL, NULL}; PyObject *digest; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1573,7 +1591,8 @@ _hashlib_hmac_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO PyObject *msg_obj = NULL; PyObject *digestmod = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1615,9 +1634,9 @@ static PyObject * _hashlib_HMAC_copy_impl(HMACobject *self); static PyObject * -_hashlib_HMAC_copy(HMACobject *self, PyObject *Py_UNUSED(ignored)) +_hashlib_HMAC_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _hashlib_HMAC_copy_impl(self); + return _hashlib_HMAC_copy_impl((HMACobject *)self); } PyDoc_STRVAR(_hashlib_HMAC_update__doc__, @@ -1633,7 +1652,7 @@ static PyObject * _hashlib_HMAC_update_impl(HMACobject *self, PyObject *msg); static PyObject * -_hashlib_HMAC_update(HMACobject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_hashlib_HMAC_update(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -1664,12 +1683,13 @@ _hashlib_HMAC_update(HMACobject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *argsbuf[1]; PyObject *msg; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } msg = args[0]; - return_value = _hashlib_HMAC_update_impl(self, msg); + return_value = _hashlib_HMAC_update_impl((HMACobject *)self, msg); exit: return return_value; @@ -1688,9 +1708,9 @@ static PyObject * _hashlib_HMAC_digest_impl(HMACobject *self); static PyObject * -_hashlib_HMAC_digest(HMACobject *self, PyObject *Py_UNUSED(ignored)) +_hashlib_HMAC_digest(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _hashlib_HMAC_digest_impl(self); + return _hashlib_HMAC_digest_impl((HMACobject *)self); } PyDoc_STRVAR(_hashlib_HMAC_hexdigest__doc__, @@ -1709,9 +1729,9 @@ static PyObject * _hashlib_HMAC_hexdigest_impl(HMACobject *self); static PyObject * -_hashlib_HMAC_hexdigest(HMACobject *self, PyObject *Py_UNUSED(ignored)) +_hashlib_HMAC_hexdigest(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _hashlib_HMAC_hexdigest_impl(self); + return _hashlib_HMAC_hexdigest_impl((HMACobject *)self); } PyDoc_STRVAR(_hashlib_get_fips_mode__doc__, @@ -1824,4 +1844,4 @@ _hashlib_compare_digest(PyObject *module, PyObject *const *args, Py_ssize_t narg #ifndef _HASHLIB_SCRYPT_METHODDEF #define _HASHLIB_SCRYPT_METHODDEF #endif /* !defined(_HASHLIB_SCRYPT_METHODDEF) */ -/*[clinic end generated code: output=b7eddeb3d6ccdeec input=a9049054013a1b77]*/ +/*[clinic end generated code: output=811a8b50beae1018 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lsprof.c.h b/Modules/clinic/_lsprof.c.h index b3b7fda5660bfd..6a75a8f9833d1e 100644 --- a/Modules/clinic/_lsprof.c.h +++ b/Modules/clinic/_lsprof.c.h @@ -2,6 +2,12 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(_lsprof_Profiler_getstats__doc__, "getstats($self, /)\n" "--\n" @@ -37,12 +43,368 @@ static PyObject * _lsprof_Profiler_getstats_impl(ProfilerObject *self, PyTypeObject *cls); static PyObject * -_lsprof_Profiler_getstats(ProfilerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_lsprof_Profiler_getstats(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "getstats() takes no arguments"); return NULL; } - return _lsprof_Profiler_getstats_impl(self, cls); + return _lsprof_Profiler_getstats_impl((ProfilerObject *)self, cls); +} + +PyDoc_STRVAR(_lsprof_Profiler__pystart_callback__doc__, +"_pystart_callback($self, code, instruction_offset, /)\n" +"--\n" +"\n"); + +#define _LSPROF_PROFILER__PYSTART_CALLBACK_METHODDEF \ + {"_pystart_callback", _PyCFunction_CAST(_lsprof_Profiler__pystart_callback), METH_FASTCALL, _lsprof_Profiler__pystart_callback__doc__}, + +static PyObject * +_lsprof_Profiler__pystart_callback_impl(ProfilerObject *self, PyObject *code, + PyObject *instruction_offset); + +static PyObject * +_lsprof_Profiler__pystart_callback(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *code; + PyObject *instruction_offset; + + if (!_PyArg_CheckPositional("_pystart_callback", nargs, 2, 2)) { + goto exit; + } + code = args[0]; + instruction_offset = args[1]; + return_value = _lsprof_Profiler__pystart_callback_impl((ProfilerObject *)self, code, instruction_offset); + +exit: + return return_value; +} + +PyDoc_STRVAR(_lsprof_Profiler__pyreturn_callback__doc__, +"_pyreturn_callback($self, code, instruction_offset, retval, /)\n" +"--\n" +"\n"); + +#define _LSPROF_PROFILER__PYRETURN_CALLBACK_METHODDEF \ + {"_pyreturn_callback", _PyCFunction_CAST(_lsprof_Profiler__pyreturn_callback), METH_FASTCALL, _lsprof_Profiler__pyreturn_callback__doc__}, + +static PyObject * +_lsprof_Profiler__pyreturn_callback_impl(ProfilerObject *self, + PyObject *code, + PyObject *instruction_offset, + PyObject *retval); + +static PyObject * +_lsprof_Profiler__pyreturn_callback(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *code; + PyObject *instruction_offset; + PyObject *retval; + + if (!_PyArg_CheckPositional("_pyreturn_callback", nargs, 3, 3)) { + goto exit; + } + code = args[0]; + instruction_offset = args[1]; + retval = args[2]; + return_value = _lsprof_Profiler__pyreturn_callback_impl((ProfilerObject *)self, code, instruction_offset, retval); + +exit: + return return_value; +} + +PyDoc_STRVAR(_lsprof_Profiler__ccall_callback__doc__, +"_ccall_callback($self, code, instruction_offset, callable, self_arg, /)\n" +"--\n" +"\n"); + +#define _LSPROF_PROFILER__CCALL_CALLBACK_METHODDEF \ + {"_ccall_callback", _PyCFunction_CAST(_lsprof_Profiler__ccall_callback), METH_FASTCALL, _lsprof_Profiler__ccall_callback__doc__}, + +static PyObject * +_lsprof_Profiler__ccall_callback_impl(ProfilerObject *self, PyObject *code, + PyObject *instruction_offset, + PyObject *callable, PyObject *self_arg); + +static PyObject * +_lsprof_Profiler__ccall_callback(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *code; + PyObject *instruction_offset; + PyObject *callable; + PyObject *self_arg; + + if (!_PyArg_CheckPositional("_ccall_callback", nargs, 4, 4)) { + goto exit; + } + code = args[0]; + instruction_offset = args[1]; + callable = args[2]; + self_arg = args[3]; + return_value = _lsprof_Profiler__ccall_callback_impl((ProfilerObject *)self, code, instruction_offset, callable, self_arg); + +exit: + return return_value; +} + +PyDoc_STRVAR(_lsprof_Profiler__creturn_callback__doc__, +"_creturn_callback($self, code, instruction_offset, callable, self_arg,\n" +" /)\n" +"--\n" +"\n"); + +#define _LSPROF_PROFILER__CRETURN_CALLBACK_METHODDEF \ + {"_creturn_callback", _PyCFunction_CAST(_lsprof_Profiler__creturn_callback), METH_FASTCALL, _lsprof_Profiler__creturn_callback__doc__}, + +static PyObject * +_lsprof_Profiler__creturn_callback_impl(ProfilerObject *self, PyObject *code, + PyObject *instruction_offset, + PyObject *callable, + PyObject *self_arg); + +static PyObject * +_lsprof_Profiler__creturn_callback(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *code; + PyObject *instruction_offset; + PyObject *callable; + PyObject *self_arg; + + if (!_PyArg_CheckPositional("_creturn_callback", nargs, 4, 4)) { + goto exit; + } + code = args[0]; + instruction_offset = args[1]; + callable = args[2]; + self_arg = args[3]; + return_value = _lsprof_Profiler__creturn_callback_impl((ProfilerObject *)self, code, instruction_offset, callable, self_arg); + +exit: + return return_value; +} + +PyDoc_STRVAR(_lsprof_Profiler_enable__doc__, +"enable($self, /, subcalls=True, builtins=True)\n" +"--\n" +"\n" +"Start collecting profiling information.\n" +"\n" +" subcalls\n" +" If True, also records for each function\n" +" statistics separated according to its current caller.\n" +" builtins\n" +" If True, records the time spent in\n" +" built-in functions separately from their caller."); + +#define _LSPROF_PROFILER_ENABLE_METHODDEF \ + {"enable", _PyCFunction_CAST(_lsprof_Profiler_enable), METH_FASTCALL|METH_KEYWORDS, _lsprof_Profiler_enable__doc__}, + +static PyObject * +_lsprof_Profiler_enable_impl(ProfilerObject *self, int subcalls, + int builtins); + +static PyObject * +_lsprof_Profiler_enable(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(subcalls), &_Py_ID(builtins), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"subcalls", "builtins", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "enable", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int subcalls = 1; + int builtins = 1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + subcalls = PyObject_IsTrue(args[0]); + if (subcalls < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + builtins = PyObject_IsTrue(args[1]); + if (builtins < 0) { + goto exit; + } +skip_optional_pos: + return_value = _lsprof_Profiler_enable_impl((ProfilerObject *)self, subcalls, builtins); + +exit: + return return_value; +} + +PyDoc_STRVAR(_lsprof_Profiler_disable__doc__, +"disable($self, /)\n" +"--\n" +"\n" +"Stop collecting profiling information."); + +#define _LSPROF_PROFILER_DISABLE_METHODDEF \ + {"disable", (PyCFunction)_lsprof_Profiler_disable, METH_NOARGS, _lsprof_Profiler_disable__doc__}, + +static PyObject * +_lsprof_Profiler_disable_impl(ProfilerObject *self); + +static PyObject * +_lsprof_Profiler_disable(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _lsprof_Profiler_disable_impl((ProfilerObject *)self); +} + +PyDoc_STRVAR(_lsprof_Profiler_clear__doc__, +"clear($self, /)\n" +"--\n" +"\n" +"Clear all profiling information collected so far."); + +#define _LSPROF_PROFILER_CLEAR_METHODDEF \ + {"clear", (PyCFunction)_lsprof_Profiler_clear, METH_NOARGS, _lsprof_Profiler_clear__doc__}, + +static PyObject * +_lsprof_Profiler_clear_impl(ProfilerObject *self); + +static PyObject * +_lsprof_Profiler_clear(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _lsprof_Profiler_clear_impl((ProfilerObject *)self); +} + +PyDoc_STRVAR(profiler_init__doc__, +"Profiler(timer=None, timeunit=0.0, subcalls=True, builtins=True)\n" +"--\n" +"\n" +"Build a profiler object using the specified timer function.\n" +"\n" +"The default timer is a fast built-in one based on real time.\n" +"For custom timer functions returning integers, \'timeunit\' can\n" +"be a float specifying a scale (that is, how long each integer unit\n" +"is, in seconds)."); + +static int +profiler_init_impl(ProfilerObject *self, PyObject *timer, double timeunit, + int subcalls, int builtins); + +static int +profiler_init(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(timer), &_Py_ID(timeunit), &_Py_ID(subcalls), &_Py_ID(builtins), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"timer", "timeunit", "subcalls", "builtins", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "Profiler", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + PyObject *timer = NULL; + double timeunit = 0.0; + int subcalls = 1; + int builtins = 1; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (fastargs[0]) { + timer = fastargs[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[1]) { + if (PyFloat_CheckExact(fastargs[1])) { + timeunit = PyFloat_AS_DOUBLE(fastargs[1]); + } + else + { + timeunit = PyFloat_AsDouble(fastargs[1]); + if (timeunit == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[2]) { + subcalls = PyObject_IsTrue(fastargs[2]); + if (subcalls < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + builtins = PyObject_IsTrue(fastargs[3]); + if (builtins < 0) { + goto exit; + } +skip_optional_pos: + return_value = profiler_init_impl((ProfilerObject *)self, timer, timeunit, subcalls, builtins); + +exit: + return return_value; } -/*[clinic end generated code: output=5c9d87d89863dc83 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d983dbf23fd8ac3b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lzmamodule.c.h b/Modules/clinic/_lzmamodule.c.h index 51fab5eab3f7dc..c7c81d8d1f1b9d 100644 --- a/Modules/clinic/_lzmamodule.c.h +++ b/Modules/clinic/_lzmamodule.c.h @@ -27,7 +27,7 @@ static PyObject * _lzma_LZMACompressor_compress_impl(Compressor *self, Py_buffer *data); static PyObject * -_lzma_LZMACompressor_compress(Compressor *self, PyObject *arg) +_lzma_LZMACompressor_compress(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; Py_buffer data = {NULL, NULL}; @@ -35,7 +35,7 @@ _lzma_LZMACompressor_compress(Compressor *self, PyObject *arg) if (PyObject_GetBuffer(arg, &data, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = _lzma_LZMACompressor_compress_impl(self, &data); + return_value = _lzma_LZMACompressor_compress_impl((Compressor *)self, &data); exit: /* Cleanup for data */ @@ -63,9 +63,9 @@ static PyObject * _lzma_LZMACompressor_flush_impl(Compressor *self); static PyObject * -_lzma_LZMACompressor_flush(Compressor *self, PyObject *Py_UNUSED(ignored)) +_lzma_LZMACompressor_flush(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _lzma_LZMACompressor_flush_impl(self); + return _lzma_LZMACompressor_flush_impl((Compressor *)self); } PyDoc_STRVAR(_lzma_LZMADecompressor_decompress__doc__, @@ -95,7 +95,7 @@ _lzma_LZMADecompressor_decompress_impl(Decompressor *self, Py_buffer *data, Py_ssize_t max_length); static PyObject * -_lzma_LZMADecompressor_decompress(Decompressor *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_lzma_LZMADecompressor_decompress(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -128,7 +128,8 @@ _lzma_LZMADecompressor_decompress(Decompressor *self, PyObject *const *args, Py_ Py_buffer data = {NULL, NULL}; Py_ssize_t max_length = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -151,7 +152,7 @@ _lzma_LZMADecompressor_decompress(Decompressor *self, PyObject *const *args, Py_ max_length = ival; } skip_optional_pos: - return_value = _lzma_LZMADecompressor_decompress_impl(self, &data, max_length); + return_value = _lzma_LZMADecompressor_decompress_impl((Decompressor *)self, &data, max_length); exit: /* Cleanup for data */ @@ -226,7 +227,8 @@ _lzma_LZMADecompressor(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *memlimit = Py_None; PyObject *filters = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 3, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -327,4 +329,4 @@ _lzma__decode_filter_properties(PyObject *module, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=5e79c05ace76dc96 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=19ed9b1182f5ddf9 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_opcode.c.h b/Modules/clinic/_opcode.c.h index c7fd0f9f8a7420..e61762e124ad72 100644 --- a/Modules/clinic/_opcode.c.h +++ b/Modules/clinic/_opcode.c.h @@ -57,7 +57,8 @@ _opcode_stack_effect(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *jump = Py_None; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -131,7 +132,8 @@ _opcode_is_valid(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb int opcode; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -194,7 +196,8 @@ _opcode_has_arg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj int opcode; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -257,7 +260,8 @@ _opcode_has_const(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO int opcode; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -320,7 +324,8 @@ _opcode_has_name(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb int opcode; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -383,7 +388,8 @@ _opcode_has_jump(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb int opcode; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -451,7 +457,8 @@ _opcode_has_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb int opcode; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -514,7 +521,8 @@ _opcode_has_local(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO int opcode; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -577,7 +585,8 @@ _opcode_has_exc(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj int opcode; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -668,4 +677,83 @@ _opcode_get_intrinsic2_descs(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _opcode_get_intrinsic2_descs_impl(module); } -/*[clinic end generated code: output=a1052bb1deffb7f2 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_opcode_get_special_method_names__doc__, +"get_special_method_names($module, /)\n" +"--\n" +"\n" +"Return a list of special method names."); + +#define _OPCODE_GET_SPECIAL_METHOD_NAMES_METHODDEF \ + {"get_special_method_names", (PyCFunction)_opcode_get_special_method_names, METH_NOARGS, _opcode_get_special_method_names__doc__}, + +static PyObject * +_opcode_get_special_method_names_impl(PyObject *module); + +static PyObject * +_opcode_get_special_method_names(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _opcode_get_special_method_names_impl(module); +} + +PyDoc_STRVAR(_opcode_get_executor__doc__, +"get_executor($module, /, code, offset)\n" +"--\n" +"\n" +"Return the executor object at offset in code if exists, None otherwise."); + +#define _OPCODE_GET_EXECUTOR_METHODDEF \ + {"get_executor", _PyCFunction_CAST(_opcode_get_executor), METH_FASTCALL|METH_KEYWORDS, _opcode_get_executor__doc__}, + +static PyObject * +_opcode_get_executor_impl(PyObject *module, PyObject *code, int offset); + +static PyObject * +_opcode_get_executor(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(code), &_Py_ID(offset), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"code", "offset", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "get_executor", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *code; + int offset; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + code = args[0]; + offset = PyLong_AsInt(args[1]); + if (offset == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _opcode_get_executor_impl(module, code, offset); + +exit: + return return_value; +} +/*[clinic end generated code: output=2e0c39286d0253e7 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_operator.c.h b/Modules/clinic/_operator.c.h index 08615d690922a1..48a8ea8c3379ab 100644 --- a/Modules/clinic/_operator.c.h +++ b/Modules/clinic/_operator.c.h @@ -1393,6 +1393,24 @@ _operator_is_not(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(_operator_is_none__doc__, +"is_none($module, a, /)\n" +"--\n" +"\n" +"Same as a is None."); + +#define _OPERATOR_IS_NONE_METHODDEF \ + {"is_none", (PyCFunction)_operator_is_none, METH_O, _operator_is_none__doc__}, + +PyDoc_STRVAR(_operator_is_not_none__doc__, +"is_not_none($module, a, /)\n" +"--\n" +"\n" +"Same as a is not None."); + +#define _OPERATOR_IS_NOT_NONE_METHODDEF \ + {"is_not_none", (PyCFunction)_operator_is_not_none, METH_O, _operator_is_not_none__doc__}, + PyDoc_STRVAR(_operator_length_hint__doc__, "length_hint($module, obj, default=0, /)\n" "--\n" @@ -1489,4 +1507,4 @@ _operator__compare_digest(PyObject *module, PyObject *const *args, Py_ssize_t na exit: return return_value; } -/*[clinic end generated code: output=ddbba2cd943571eb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=972e2543c4fcf1ba input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_pickle.c.h b/Modules/clinic/_pickle.c.h index 5a6ae7be6b6ea7..91d355c5afb353 100644 --- a/Modules/clinic/_pickle.c.h +++ b/Modules/clinic/_pickle.c.h @@ -4,7 +4,7 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() @@ -26,9 +26,9 @@ static PyObject * _pickle_Pickler_clear_memo_impl(PicklerObject *self); static PyObject * -_pickle_Pickler_clear_memo(PicklerObject *self, PyObject *Py_UNUSED(ignored)) +_pickle_Pickler_clear_memo(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _pickle_Pickler_clear_memo_impl(self); + return _pickle_Pickler_clear_memo_impl((PicklerObject *)self); } PyDoc_STRVAR(_pickle_Pickler_dump__doc__, @@ -45,7 +45,7 @@ _pickle_Pickler_dump_impl(PicklerObject *self, PyTypeObject *cls, PyObject *obj); static PyObject * -_pickle_Pickler_dump(PicklerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_pickle_Pickler_dump(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -64,12 +64,13 @@ _pickle_Pickler_dump(PicklerObject *self, PyTypeObject *cls, PyObject *const *ar PyObject *argsbuf[1]; PyObject *obj; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } obj = args[0]; - return_value = _pickle_Pickler_dump_impl(self, cls, obj); + return_value = _pickle_Pickler_dump_impl((PicklerObject *)self, cls, obj); exit: return return_value; @@ -88,12 +89,12 @@ static size_t _pickle_Pickler___sizeof___impl(PicklerObject *self); static PyObject * -_pickle_Pickler___sizeof__(PicklerObject *self, PyObject *Py_UNUSED(ignored)) +_pickle_Pickler___sizeof__(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; size_t _return_value; - _return_value = _pickle_Pickler___sizeof___impl(self); + _return_value = _pickle_Pickler___sizeof___impl((PicklerObject *)self); if ((_return_value == (size_t)-1) && PyErr_Occurred()) { goto exit; } @@ -111,7 +112,7 @@ PyDoc_STRVAR(_pickle_Pickler___init____doc__, "\n" "The optional *protocol* argument tells the pickler to use the given\n" "protocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default\n" -"protocol is 4. It was introduced in Python 3.4, and is incompatible\n" +"protocol is 5. It was introduced in Python 3.8, and is incompatible\n" "with previous versions.\n" "\n" "Specifying a negative protocol version selects the highest protocol\n" @@ -181,7 +182,8 @@ _pickle_Pickler___init__(PyObject *self, PyObject *args, PyObject *kwargs) int fix_imports = 1; PyObject *buffer_callback = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 4, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -225,9 +227,9 @@ static PyObject * _pickle_PicklerMemoProxy_clear_impl(PicklerMemoProxyObject *self); static PyObject * -_pickle_PicklerMemoProxy_clear(PicklerMemoProxyObject *self, PyObject *Py_UNUSED(ignored)) +_pickle_PicklerMemoProxy_clear(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _pickle_PicklerMemoProxy_clear_impl(self); + return _pickle_PicklerMemoProxy_clear_impl((PicklerMemoProxyObject *)self); } PyDoc_STRVAR(_pickle_PicklerMemoProxy_copy__doc__, @@ -243,9 +245,9 @@ static PyObject * _pickle_PicklerMemoProxy_copy_impl(PicklerMemoProxyObject *self); static PyObject * -_pickle_PicklerMemoProxy_copy(PicklerMemoProxyObject *self, PyObject *Py_UNUSED(ignored)) +_pickle_PicklerMemoProxy_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _pickle_PicklerMemoProxy_copy_impl(self); + return _pickle_PicklerMemoProxy_copy_impl((PicklerMemoProxyObject *)self); } PyDoc_STRVAR(_pickle_PicklerMemoProxy___reduce____doc__, @@ -261,9 +263,9 @@ static PyObject * _pickle_PicklerMemoProxy___reduce___impl(PicklerMemoProxyObject *self); static PyObject * -_pickle_PicklerMemoProxy___reduce__(PicklerMemoProxyObject *self, PyObject *Py_UNUSED(ignored)) +_pickle_PicklerMemoProxy___reduce__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _pickle_PicklerMemoProxy___reduce___impl(self); + return _pickle_PicklerMemoProxy___reduce___impl((PicklerMemoProxyObject *)self); } PyDoc_STRVAR(_pickle_Unpickler_persistent_load__doc__, @@ -279,7 +281,7 @@ _pickle_Unpickler_persistent_load_impl(UnpicklerObject *self, PyTypeObject *cls, PyObject *pid); static PyObject * -_pickle_Unpickler_persistent_load(UnpicklerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_pickle_Unpickler_persistent_load(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -298,12 +300,13 @@ _pickle_Unpickler_persistent_load(UnpicklerObject *self, PyTypeObject *cls, PyOb PyObject *argsbuf[1]; PyObject *pid; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } pid = args[0]; - return_value = _pickle_Unpickler_persistent_load_impl(self, cls, pid); + return_value = _pickle_Unpickler_persistent_load_impl((UnpicklerObject *)self, cls, pid); exit: return return_value; @@ -326,13 +329,13 @@ static PyObject * _pickle_Unpickler_load_impl(UnpicklerObject *self, PyTypeObject *cls); static PyObject * -_pickle_Unpickler_load(UnpicklerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_pickle_Unpickler_load(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "load() takes no arguments"); return NULL; } - return _pickle_Unpickler_load_impl(self, cls); + return _pickle_Unpickler_load_impl((UnpicklerObject *)self, cls); } PyDoc_STRVAR(_pickle_Unpickler_find_class__doc__, @@ -357,7 +360,7 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self, PyTypeObject *cls, PyObject *global_name); static PyObject * -_pickle_Unpickler_find_class(UnpicklerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_pickle_Unpickler_find_class(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -377,13 +380,14 @@ _pickle_Unpickler_find_class(UnpicklerObject *self, PyTypeObject *cls, PyObject PyObject *module_name; PyObject *global_name; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } module_name = args[0]; global_name = args[1]; - return_value = _pickle_Unpickler_find_class_impl(self, cls, module_name, global_name); + return_value = _pickle_Unpickler_find_class_impl((UnpicklerObject *)self, cls, module_name, global_name); exit: return return_value; @@ -402,12 +406,12 @@ static size_t _pickle_Unpickler___sizeof___impl(UnpicklerObject *self); static PyObject * -_pickle_Unpickler___sizeof__(UnpicklerObject *self, PyObject *Py_UNUSED(ignored)) +_pickle_Unpickler___sizeof__(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; size_t _return_value; - _return_value = _pickle_Unpickler___sizeof___impl(self); + _return_value = _pickle_Unpickler___sizeof___impl((UnpicklerObject *)self); if ((_return_value == (size_t)-1) && PyErr_Occurred()) { goto exit; } @@ -487,7 +491,8 @@ _pickle_Unpickler___init__(PyObject *self, PyObject *args, PyObject *kwargs) const char *errors = "strict"; PyObject *buffers = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -561,9 +566,9 @@ static PyObject * _pickle_UnpicklerMemoProxy_clear_impl(UnpicklerMemoProxyObject *self); static PyObject * -_pickle_UnpicklerMemoProxy_clear(UnpicklerMemoProxyObject *self, PyObject *Py_UNUSED(ignored)) +_pickle_UnpicklerMemoProxy_clear(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _pickle_UnpicklerMemoProxy_clear_impl(self); + return _pickle_UnpicklerMemoProxy_clear_impl((UnpicklerMemoProxyObject *)self); } PyDoc_STRVAR(_pickle_UnpicklerMemoProxy_copy__doc__, @@ -579,9 +584,9 @@ static PyObject * _pickle_UnpicklerMemoProxy_copy_impl(UnpicklerMemoProxyObject *self); static PyObject * -_pickle_UnpicklerMemoProxy_copy(UnpicklerMemoProxyObject *self, PyObject *Py_UNUSED(ignored)) +_pickle_UnpicklerMemoProxy_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _pickle_UnpicklerMemoProxy_copy_impl(self); + return _pickle_UnpicklerMemoProxy_copy_impl((UnpicklerMemoProxyObject *)self); } PyDoc_STRVAR(_pickle_UnpicklerMemoProxy___reduce____doc__, @@ -597,9 +602,9 @@ static PyObject * _pickle_UnpicklerMemoProxy___reduce___impl(UnpicklerMemoProxyObject *self); static PyObject * -_pickle_UnpicklerMemoProxy___reduce__(UnpicklerMemoProxyObject *self, PyObject *Py_UNUSED(ignored)) +_pickle_UnpicklerMemoProxy___reduce__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _pickle_UnpicklerMemoProxy___reduce___impl(self); + return _pickle_UnpicklerMemoProxy___reduce___impl((UnpicklerMemoProxyObject *)self); } PyDoc_STRVAR(_pickle_dump__doc__, @@ -614,7 +619,7 @@ PyDoc_STRVAR(_pickle_dump__doc__, "\n" "The optional *protocol* argument tells the pickler to use the given\n" "protocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default\n" -"protocol is 4. It was introduced in Python 3.4, and is incompatible\n" +"protocol is 5. It was introduced in Python 3.8, and is incompatible\n" "with previous versions.\n" "\n" "Specifying a negative protocol version selects the highest protocol\n" @@ -679,7 +684,8 @@ _pickle_dump(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject int fix_imports = 1; PyObject *buffer_callback = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -724,7 +730,7 @@ PyDoc_STRVAR(_pickle_dumps__doc__, "\n" "The optional *protocol* argument tells the pickler to use the given\n" "protocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default\n" -"protocol is 4. It was introduced in Python 3.4, and is incompatible\n" +"protocol is 5. It was introduced in Python 3.8, and is incompatible\n" "with previous versions.\n" "\n" "Specifying a negative protocol version selects the highest protocol\n" @@ -782,7 +788,8 @@ _pickle_dumps(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec int fix_imports = 1; PyObject *buffer_callback = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -891,7 +898,8 @@ _pickle_load(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject const char *errors = "strict"; PyObject *buffers = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1017,7 +1025,8 @@ _pickle_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec const char *errors = "strict"; PyObject *buffers = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1077,4 +1086,4 @@ _pickle_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=bd63c85a8737b0aa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d71dc73af298ebe8 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_posixsubprocess.c.h b/Modules/clinic/_posixsubprocess.c.h index dd7644de6b7534..d52629cf6eaa5b 100644 --- a/Modules/clinic/_posixsubprocess.c.h +++ b/Modules/clinic/_posixsubprocess.c.h @@ -9,7 +9,7 @@ PyDoc_STRVAR(subprocess_fork_exec__doc__, " env, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite,\n" " errpipe_read, errpipe_write, restore_signals, call_setsid,\n" " pgid_to_set, gid, extra_groups, uid, child_umask, preexec_fn,\n" -" allow_vfork, /)\n" +" /)\n" "--\n" "\n" "Spawn a fresh new child process.\n" @@ -48,7 +48,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, pid_t pgid_to_set, PyObject *gid_object, PyObject *extra_groups_packed, PyObject *uid_object, int child_umask, - PyObject *preexec_fn, int allow_vfork); + PyObject *preexec_fn); static PyObject * subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) @@ -76,9 +76,8 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *uid_object; int child_umask; PyObject *preexec_fn; - int allow_vfork; - if (!_PyArg_CheckPositional("fork_exec", nargs, 23, 23)) { + if (!_PyArg_CheckPositional("fork_exec", nargs, 22, 22)) { goto exit; } process_args = args[0]; @@ -146,13 +145,9 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } preexec_fn = args[21]; - allow_vfork = PyObject_IsTrue(args[22]); - if (allow_vfork < 0) { - goto exit; - } - return_value = subprocess_fork_exec_impl(module, process_args, executable_list, close_fds, py_fds_to_keep, cwd_obj, env_list, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, errpipe_read, errpipe_write, restore_signals, call_setsid, pgid_to_set, gid_object, extra_groups_packed, uid_object, child_umask, preexec_fn, allow_vfork); + return_value = subprocess_fork_exec_impl(module, process_args, executable_list, close_fds, py_fds_to_keep, cwd_obj, env_list, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, errpipe_read, errpipe_write, restore_signals, call_setsid, pgid_to_set, gid_object, extra_groups_packed, uid_object, child_umask, preexec_fn); exit: return return_value; } -/*[clinic end generated code: output=48555f5965a871be input=a9049054013a1b77]*/ +/*[clinic end generated code: output=942bc2748a9c2785 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h index 6f4c715c722965..2dfc3e6be1984e 100644 --- a/Modules/clinic/_queuemodule.c.h +++ b/Modules/clinic/_queuemodule.c.h @@ -55,7 +55,7 @@ _queue_SimpleQueue_put_impl(simplequeueobject *self, PyObject *item, int block, PyObject *timeout); static PyObject * -_queue_SimpleQueue_put(simplequeueobject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_queue_SimpleQueue_put(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -89,7 +89,8 @@ _queue_SimpleQueue_put(simplequeueobject *self, PyObject *const *args, Py_ssize_ int block = 1; PyObject *timeout = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -109,7 +110,7 @@ _queue_SimpleQueue_put(simplequeueobject *self, PyObject *const *args, Py_ssize_ timeout = args[2]; skip_optional_pos: Py_BEGIN_CRITICAL_SECTION(self); - return_value = _queue_SimpleQueue_put_impl(self, item, block, timeout); + return_value = _queue_SimpleQueue_put_impl((simplequeueobject *)self, item, block, timeout); Py_END_CRITICAL_SECTION(); exit: @@ -132,7 +133,7 @@ static PyObject * _queue_SimpleQueue_put_nowait_impl(simplequeueobject *self, PyObject *item); static PyObject * -_queue_SimpleQueue_put_nowait(simplequeueobject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_queue_SimpleQueue_put_nowait(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -163,13 +164,14 @@ _queue_SimpleQueue_put_nowait(simplequeueobject *self, PyObject *const *args, Py PyObject *argsbuf[1]; PyObject *item; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } item = args[0]; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _queue_SimpleQueue_put_nowait_impl(self, item); + return_value = _queue_SimpleQueue_put_nowait_impl((simplequeueobject *)self, item); Py_END_CRITICAL_SECTION(); exit: @@ -198,7 +200,7 @@ _queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, int block, PyObject *timeout_obj); static PyObject * -_queue_SimpleQueue_get(simplequeueobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_queue_SimpleQueue_get(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -231,7 +233,8 @@ _queue_SimpleQueue_get(simplequeueobject *self, PyTypeObject *cls, PyObject *con int block = 1; PyObject *timeout_obj = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -250,7 +253,7 @@ _queue_SimpleQueue_get(simplequeueobject *self, PyTypeObject *cls, PyObject *con timeout_obj = args[1]; skip_optional_pos: Py_BEGIN_CRITICAL_SECTION(self); - return_value = _queue_SimpleQueue_get_impl(self, cls, block, timeout_obj); + return_value = _queue_SimpleQueue_get_impl((simplequeueobject *)self, cls, block, timeout_obj); Py_END_CRITICAL_SECTION(); exit: @@ -274,7 +277,7 @@ _queue_SimpleQueue_get_nowait_impl(simplequeueobject *self, PyTypeObject *cls); static PyObject * -_queue_SimpleQueue_get_nowait(simplequeueobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_queue_SimpleQueue_get_nowait(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; @@ -283,7 +286,7 @@ _queue_SimpleQueue_get_nowait(simplequeueobject *self, PyTypeObject *cls, PyObje goto exit; } Py_BEGIN_CRITICAL_SECTION(self); - return_value = _queue_SimpleQueue_get_nowait_impl(self, cls); + return_value = _queue_SimpleQueue_get_nowait_impl((simplequeueobject *)self, cls); Py_END_CRITICAL_SECTION(); exit: @@ -303,13 +306,13 @@ static int _queue_SimpleQueue_empty_impl(simplequeueobject *self); static PyObject * -_queue_SimpleQueue_empty(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) +_queue_SimpleQueue_empty(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; int _return_value; Py_BEGIN_CRITICAL_SECTION(self); - _return_value = _queue_SimpleQueue_empty_impl(self); + _return_value = _queue_SimpleQueue_empty_impl((simplequeueobject *)self); Py_END_CRITICAL_SECTION(); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; @@ -333,13 +336,13 @@ static Py_ssize_t _queue_SimpleQueue_qsize_impl(simplequeueobject *self); static PyObject * -_queue_SimpleQueue_qsize(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) +_queue_SimpleQueue_qsize(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_ssize_t _return_value; Py_BEGIN_CRITICAL_SECTION(self); - _return_value = _queue_SimpleQueue_qsize_impl(self); + _return_value = _queue_SimpleQueue_qsize_impl((simplequeueobject *)self); Py_END_CRITICAL_SECTION(); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; @@ -349,4 +352,4 @@ _queue_SimpleQueue_qsize(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=44a718f40072018a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e04e15a1b959c700 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_randommodule.c.h b/Modules/clinic/_randommodule.c.h index 6193acac67e7ac..b2d67e11c63595 100644 --- a/Modules/clinic/_randommodule.c.h +++ b/Modules/clinic/_randommodule.c.h @@ -18,12 +18,12 @@ static PyObject * _random_Random_random_impl(RandomObject *self); static PyObject * -_random_Random_random(RandomObject *self, PyObject *Py_UNUSED(ignored)) +_random_Random_random(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _random_Random_random_impl(self); + return_value = _random_Random_random_impl((RandomObject *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -45,7 +45,7 @@ static PyObject * _random_Random_seed_impl(RandomObject *self, PyObject *n); static PyObject * -_random_Random_seed(RandomObject *self, PyObject *const *args, Py_ssize_t nargs) +_random_Random_seed(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *n = Py_None; @@ -59,7 +59,7 @@ _random_Random_seed(RandomObject *self, PyObject *const *args, Py_ssize_t nargs) n = args[0]; skip_optional: Py_BEGIN_CRITICAL_SECTION(self); - return_value = _random_Random_seed_impl(self, n); + return_value = _random_Random_seed_impl((RandomObject *)self, n); Py_END_CRITICAL_SECTION(); exit: @@ -79,12 +79,12 @@ static PyObject * _random_Random_getstate_impl(RandomObject *self); static PyObject * -_random_Random_getstate(RandomObject *self, PyObject *Py_UNUSED(ignored)) +_random_Random_getstate(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _random_Random_getstate_impl(self); + return_value = _random_Random_getstate_impl((RandomObject *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -108,7 +108,7 @@ _random_Random_setstate(RandomObject *self, PyObject *state) PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = _random_Random_setstate_impl(self, state); + return_value = _random_Random_setstate_impl((RandomObject *)self, state); Py_END_CRITICAL_SECTION(); return return_value; @@ -127,7 +127,7 @@ static PyObject * _random_Random_getrandbits_impl(RandomObject *self, int k); static PyObject * -_random_Random_getrandbits(RandomObject *self, PyObject *arg) +_random_Random_getrandbits(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; int k; @@ -137,10 +137,10 @@ _random_Random_getrandbits(RandomObject *self, PyObject *arg) goto exit; } Py_BEGIN_CRITICAL_SECTION(self); - return_value = _random_Random_getrandbits_impl(self, k); + return_value = _random_Random_getrandbits_impl((RandomObject *)self, k); Py_END_CRITICAL_SECTION(); exit: return return_value; } -/*[clinic end generated code: output=bf49ece1d341b1b6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=859cfbf59c133a4e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 2940f16a2cb7f6..73c5d304f1a141 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -6,7 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif -#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_ssl__SSLSocket_do_handshake__doc__, @@ -21,9 +21,15 @@ static PyObject * _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self); static PyObject * -_ssl__SSLSocket_do_handshake(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) +_ssl__SSLSocket_do_handshake(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLSocket_do_handshake_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_do_handshake_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__test_decode_cert__doc__, @@ -73,7 +79,7 @@ static PyObject * _ssl__SSLSocket_getpeercert_impl(PySSLSocket *self, int binary_mode); static PyObject * -_ssl__SSLSocket_getpeercert(PySSLSocket *self, PyObject *const *args, Py_ssize_t nargs) +_ssl__SSLSocket_getpeercert(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int binary_mode = 0; @@ -89,7 +95,9 @@ _ssl__SSLSocket_getpeercert(PySSLSocket *self, PyObject *const *args, Py_ssize_t goto exit; } skip_optional: - return_value = _ssl__SSLSocket_getpeercert_impl(self, binary_mode); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_getpeercert_impl((PySSLSocket *)self, binary_mode); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -107,9 +115,15 @@ static PyObject * _ssl__SSLSocket_get_verified_chain_impl(PySSLSocket *self); static PyObject * -_ssl__SSLSocket_get_verified_chain(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) +_ssl__SSLSocket_get_verified_chain(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLSocket_get_verified_chain_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_get_verified_chain_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLSocket_get_unverified_chain__doc__, @@ -124,9 +138,15 @@ static PyObject * _ssl__SSLSocket_get_unverified_chain_impl(PySSLSocket *self); static PyObject * -_ssl__SSLSocket_get_unverified_chain(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) +_ssl__SSLSocket_get_unverified_chain(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLSocket_get_unverified_chain_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_get_unverified_chain_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLSocket_shared_ciphers__doc__, @@ -141,9 +161,15 @@ static PyObject * _ssl__SSLSocket_shared_ciphers_impl(PySSLSocket *self); static PyObject * -_ssl__SSLSocket_shared_ciphers(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) +_ssl__SSLSocket_shared_ciphers(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLSocket_shared_ciphers_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_shared_ciphers_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLSocket_cipher__doc__, @@ -158,9 +184,15 @@ static PyObject * _ssl__SSLSocket_cipher_impl(PySSLSocket *self); static PyObject * -_ssl__SSLSocket_cipher(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) +_ssl__SSLSocket_cipher(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLSocket_cipher_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_cipher_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLSocket_version__doc__, @@ -175,9 +207,15 @@ static PyObject * _ssl__SSLSocket_version_impl(PySSLSocket *self); static PyObject * -_ssl__SSLSocket_version(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) +_ssl__SSLSocket_version(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLSocket_version_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_version_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLSocket_selected_alpn_protocol__doc__, @@ -192,9 +230,15 @@ static PyObject * _ssl__SSLSocket_selected_alpn_protocol_impl(PySSLSocket *self); static PyObject * -_ssl__SSLSocket_selected_alpn_protocol(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) +_ssl__SSLSocket_selected_alpn_protocol(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLSocket_selected_alpn_protocol_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_selected_alpn_protocol_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLSocket_compression__doc__, @@ -209,9 +253,193 @@ static PyObject * _ssl__SSLSocket_compression_impl(PySSLSocket *self); static PyObject * -_ssl__SSLSocket_compression(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) +_ssl__SSLSocket_compression(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _ssl__SSLSocket_compression_impl((PySSLSocket *)self); +} + +PyDoc_STRVAR(_ssl__SSLSocket_context__doc__, +"This changes the context associated with the SSLSocket.\n" +"\n" +"This is typically used from within a callback function set by the sni_callback\n" +"on the SSLContext to change the certificate information associated with the\n" +"SSLSocket before the cryptographic exchange handshake messages."); +#if defined(_ssl__SSLSocket_context_DOCSTR) +# undef _ssl__SSLSocket_context_DOCSTR +#endif +#define _ssl__SSLSocket_context_DOCSTR _ssl__SSLSocket_context__doc__ + +#if !defined(_ssl__SSLSocket_context_DOCSTR) +# define _ssl__SSLSocket_context_DOCSTR NULL +#endif +#if defined(_SSL__SSLSOCKET_CONTEXT_GETSETDEF) +# undef _SSL__SSLSOCKET_CONTEXT_GETSETDEF +# define _SSL__SSLSOCKET_CONTEXT_GETSETDEF {"context", (getter)_ssl__SSLSocket_context_get, (setter)_ssl__SSLSocket_context_set, _ssl__SSLSocket_context_DOCSTR}, +#else +# define _SSL__SSLSOCKET_CONTEXT_GETSETDEF {"context", (getter)_ssl__SSLSocket_context_get, NULL, _ssl__SSLSocket_context_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLSocket_context_get_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_context_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_context_get_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ssl__SSLSocket_context_DOCSTR) +# define _ssl__SSLSocket_context_DOCSTR NULL +#endif +#if defined(_SSL__SSLSOCKET_CONTEXT_GETSETDEF) +# undef _SSL__SSLSOCKET_CONTEXT_GETSETDEF +# define _SSL__SSLSOCKET_CONTEXT_GETSETDEF {"context", (getter)_ssl__SSLSocket_context_get, (setter)_ssl__SSLSocket_context_set, _ssl__SSLSocket_context_DOCSTR}, +#else +# define _SSL__SSLSOCKET_CONTEXT_GETSETDEF {"context", NULL, (setter)_ssl__SSLSocket_context_set, NULL}, +#endif + +static int +_ssl__SSLSocket_context_set_impl(PySSLSocket *self, PyObject *value); + +static int +_ssl__SSLSocket_context_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_context_set_impl((PySSLSocket *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLSocket_server_side__doc__, +"Whether this is a server-side socket."); +#if defined(_ssl__SSLSocket_server_side_DOCSTR) +# undef _ssl__SSLSocket_server_side_DOCSTR +#endif +#define _ssl__SSLSocket_server_side_DOCSTR _ssl__SSLSocket_server_side__doc__ + +#if !defined(_ssl__SSLSocket_server_side_DOCSTR) +# define _ssl__SSLSocket_server_side_DOCSTR NULL +#endif +#if defined(_SSL__SSLSOCKET_SERVER_SIDE_GETSETDEF) +# undef _SSL__SSLSOCKET_SERVER_SIDE_GETSETDEF +# define _SSL__SSLSOCKET_SERVER_SIDE_GETSETDEF {"server_side", (getter)_ssl__SSLSocket_server_side_get, (setter)_ssl__SSLSocket_server_side_set, _ssl__SSLSocket_server_side_DOCSTR}, +#else +# define _SSL__SSLSOCKET_SERVER_SIDE_GETSETDEF {"server_side", (getter)_ssl__SSLSocket_server_side_get, NULL, _ssl__SSLSocket_server_side_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLSocket_server_side_get_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_server_side_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_server_side_get_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLSocket_server_hostname__doc__, +"The currently set server hostname (for SNI)."); +#if defined(_ssl__SSLSocket_server_hostname_DOCSTR) +# undef _ssl__SSLSocket_server_hostname_DOCSTR +#endif +#define _ssl__SSLSocket_server_hostname_DOCSTR _ssl__SSLSocket_server_hostname__doc__ + +#if !defined(_ssl__SSLSocket_server_hostname_DOCSTR) +# define _ssl__SSLSocket_server_hostname_DOCSTR NULL +#endif +#if defined(_SSL__SSLSOCKET_SERVER_HOSTNAME_GETSETDEF) +# undef _SSL__SSLSOCKET_SERVER_HOSTNAME_GETSETDEF +# define _SSL__SSLSOCKET_SERVER_HOSTNAME_GETSETDEF {"server_hostname", (getter)_ssl__SSLSocket_server_hostname_get, (setter)_ssl__SSLSocket_server_hostname_set, _ssl__SSLSocket_server_hostname_DOCSTR}, +#else +# define _SSL__SSLSOCKET_SERVER_HOSTNAME_GETSETDEF {"server_hostname", (getter)_ssl__SSLSocket_server_hostname_get, NULL, _ssl__SSLSocket_server_hostname_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLSocket_server_hostname_get_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_server_hostname_get(PyObject *self, void *Py_UNUSED(context)) { - return _ssl__SSLSocket_compression_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_server_hostname_get_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLSocket_owner__doc__, +"The Python-level owner of this object.\n" +"\n" +"Passed as \"self\" in servername callback."); +#if defined(_ssl__SSLSocket_owner_DOCSTR) +# undef _ssl__SSLSocket_owner_DOCSTR +#endif +#define _ssl__SSLSocket_owner_DOCSTR _ssl__SSLSocket_owner__doc__ + +#if !defined(_ssl__SSLSocket_owner_DOCSTR) +# define _ssl__SSLSocket_owner_DOCSTR NULL +#endif +#if defined(_SSL__SSLSOCKET_OWNER_GETSETDEF) +# undef _SSL__SSLSOCKET_OWNER_GETSETDEF +# define _SSL__SSLSOCKET_OWNER_GETSETDEF {"owner", (getter)_ssl__SSLSocket_owner_get, (setter)_ssl__SSLSocket_owner_set, _ssl__SSLSocket_owner_DOCSTR}, +#else +# define _SSL__SSLSOCKET_OWNER_GETSETDEF {"owner", (getter)_ssl__SSLSocket_owner_get, NULL, _ssl__SSLSocket_owner_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLSocket_owner_get_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_owner_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_owner_get_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ssl__SSLSocket_owner_DOCSTR) +# define _ssl__SSLSocket_owner_DOCSTR NULL +#endif +#if defined(_SSL__SSLSOCKET_OWNER_GETSETDEF) +# undef _SSL__SSLSOCKET_OWNER_GETSETDEF +# define _SSL__SSLSOCKET_OWNER_GETSETDEF {"owner", (getter)_ssl__SSLSocket_owner_get, (setter)_ssl__SSLSocket_owner_set, _ssl__SSLSocket_owner_DOCSTR}, +#else +# define _SSL__SSLSOCKET_OWNER_GETSETDEF {"owner", NULL, (setter)_ssl__SSLSocket_owner_set, NULL}, +#endif + +static int +_ssl__SSLSocket_owner_set_impl(PySSLSocket *self, PyObject *value); + +static int +_ssl__SSLSocket_owner_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_owner_set_impl((PySSLSocket *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLSocket_write__doc__, @@ -229,7 +457,7 @@ static PyObject * _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b); static PyObject * -_ssl__SSLSocket_write(PySSLSocket *self, PyObject *arg) +_ssl__SSLSocket_write(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; Py_buffer b = {NULL, NULL}; @@ -237,7 +465,9 @@ _ssl__SSLSocket_write(PySSLSocket *self, PyObject *arg) if (PyObject_GetBuffer(arg, &b, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = _ssl__SSLSocket_write_impl(self, &b); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_write_impl((PySSLSocket *)self, &b); + Py_END_CRITICAL_SECTION(); exit: /* Cleanup for b */ @@ -261,9 +491,15 @@ static PyObject * _ssl__SSLSocket_pending_impl(PySSLSocket *self); static PyObject * -_ssl__SSLSocket_pending(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) +_ssl__SSLSocket_pending(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLSocket_pending_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_pending_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLSocket_read__doc__, @@ -278,7 +514,7 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, int group_right_1, Py_buffer *buffer); static PyObject * -_ssl__SSLSocket_read(PySSLSocket *self, PyObject *args) +_ssl__SSLSocket_read(PyObject *self, PyObject *args) { PyObject *return_value = NULL; Py_ssize_t len; @@ -301,7 +537,9 @@ _ssl__SSLSocket_read(PySSLSocket *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_ssl._SSLSocket.read requires 1 to 2 arguments"); goto exit; } - return_value = _ssl__SSLSocket_read_impl(self, len, group_right_1, &buffer); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_read_impl((PySSLSocket *)self, len, group_right_1, &buffer); + Py_END_CRITICAL_SECTION(); exit: /* Cleanup for buffer */ @@ -325,9 +563,15 @@ static PyObject * _ssl__SSLSocket_shutdown_impl(PySSLSocket *self); static PyObject * -_ssl__SSLSocket_shutdown(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) +_ssl__SSLSocket_shutdown(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLSocket_shutdown_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_shutdown_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLSocket_get_channel_binding__doc__, @@ -348,7 +592,7 @@ _ssl__SSLSocket_get_channel_binding_impl(PySSLSocket *self, const char *cb_type); static PyObject * -_ssl__SSLSocket_get_channel_binding(PySSLSocket *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_ssl__SSLSocket_get_channel_binding(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -380,7 +624,8 @@ _ssl__SSLSocket_get_channel_binding(PySSLSocket *self, PyObject *const *args, Py Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; const char *cb_type = "tls-unique"; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -401,7 +646,9 @@ _ssl__SSLSocket_get_channel_binding(PySSLSocket *self, PyObject *const *args, Py goto exit; } skip_optional_pos: - return_value = _ssl__SSLSocket_get_channel_binding_impl(self, cb_type); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_get_channel_binding_impl((PySSLSocket *)self, cb_type); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -420,9 +667,104 @@ static PyObject * _ssl__SSLSocket_verify_client_post_handshake_impl(PySSLSocket *self); static PyObject * -_ssl__SSLSocket_verify_client_post_handshake(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) +_ssl__SSLSocket_verify_client_post_handshake(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLSocket_verify_client_post_handshake_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_verify_client_post_handshake_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLSocket_session__doc__, +"The underlying SSLSession object."); +#if defined(_ssl__SSLSocket_session_DOCSTR) +# undef _ssl__SSLSocket_session_DOCSTR +#endif +#define _ssl__SSLSocket_session_DOCSTR _ssl__SSLSocket_session__doc__ + +#if !defined(_ssl__SSLSocket_session_DOCSTR) +# define _ssl__SSLSocket_session_DOCSTR NULL +#endif +#if defined(_SSL__SSLSOCKET_SESSION_GETSETDEF) +# undef _SSL__SSLSOCKET_SESSION_GETSETDEF +# define _SSL__SSLSOCKET_SESSION_GETSETDEF {"session", (getter)_ssl__SSLSocket_session_get, (setter)_ssl__SSLSocket_session_set, _ssl__SSLSocket_session_DOCSTR}, +#else +# define _SSL__SSLSOCKET_SESSION_GETSETDEF {"session", (getter)_ssl__SSLSocket_session_get, NULL, _ssl__SSLSocket_session_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLSocket_session_get_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_session_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_session_get_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ssl__SSLSocket_session_DOCSTR) +# define _ssl__SSLSocket_session_DOCSTR NULL +#endif +#if defined(_SSL__SSLSOCKET_SESSION_GETSETDEF) +# undef _SSL__SSLSOCKET_SESSION_GETSETDEF +# define _SSL__SSLSOCKET_SESSION_GETSETDEF {"session", (getter)_ssl__SSLSocket_session_get, (setter)_ssl__SSLSocket_session_set, _ssl__SSLSocket_session_DOCSTR}, +#else +# define _SSL__SSLSOCKET_SESSION_GETSETDEF {"session", NULL, (setter)_ssl__SSLSocket_session_set, NULL}, +#endif + +static int +_ssl__SSLSocket_session_set_impl(PySSLSocket *self, PyObject *value); + +static int +_ssl__SSLSocket_session_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_session_set_impl((PySSLSocket *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLSocket_session_reused__doc__, +"Was the client session reused during handshake?"); +#if defined(_ssl__SSLSocket_session_reused_DOCSTR) +# undef _ssl__SSLSocket_session_reused_DOCSTR +#endif +#define _ssl__SSLSocket_session_reused_DOCSTR _ssl__SSLSocket_session_reused__doc__ + +#if !defined(_ssl__SSLSocket_session_reused_DOCSTR) +# define _ssl__SSLSocket_session_reused_DOCSTR NULL +#endif +#if defined(_SSL__SSLSOCKET_SESSION_REUSED_GETSETDEF) +# undef _SSL__SSLSOCKET_SESSION_REUSED_GETSETDEF +# define _SSL__SSLSOCKET_SESSION_REUSED_GETSETDEF {"session_reused", (getter)_ssl__SSLSocket_session_reused_get, (setter)_ssl__SSLSocket_session_reused_set, _ssl__SSLSocket_session_reused_DOCSTR}, +#else +# define _SSL__SSLSOCKET_SESSION_REUSED_GETSETDEF {"session_reused", (getter)_ssl__SSLSocket_session_reused_get, NULL, _ssl__SSLSocket_session_reused_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLSocket_session_reused_get_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_session_reused_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_session_reused_get_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } static PyObject * @@ -446,93 +788,571 @@ _ssl__SSLContext(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (proto_version == -1 && PyErr_Occurred()) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(type); return_value = _ssl__SSLContext_impl(type, proto_version); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLContext_set_ciphers__doc__, +"set_ciphers($self, cipherlist, /)\n" +"--\n" +"\n"); + +#define _SSL__SSLCONTEXT_SET_CIPHERS_METHODDEF \ + {"set_ciphers", (PyCFunction)_ssl__SSLContext_set_ciphers, METH_O, _ssl__SSLContext_set_ciphers__doc__}, + +static PyObject * +_ssl__SSLContext_set_ciphers_impl(PySSLContext *self, const char *cipherlist); + +static PyObject * +_ssl__SSLContext_set_ciphers(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + const char *cipherlist; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("set_ciphers", "argument", "str", arg); + goto exit; + } + Py_ssize_t cipherlist_length; + cipherlist = PyUnicode_AsUTF8AndSize(arg, &cipherlist_length); + if (cipherlist == NULL) { + goto exit; + } + if (strlen(cipherlist) != (size_t)cipherlist_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_set_ciphers_impl((PySSLContext *)self, cipherlist); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLContext_get_ciphers__doc__, +"get_ciphers($self, /)\n" +"--\n" +"\n"); + +#define _SSL__SSLCONTEXT_GET_CIPHERS_METHODDEF \ + {"get_ciphers", (PyCFunction)_ssl__SSLContext_get_ciphers, METH_NOARGS, _ssl__SSLContext_get_ciphers__doc__}, + +static PyObject * +_ssl__SSLContext_get_ciphers_impl(PySSLContext *self); + +static PyObject * +_ssl__SSLContext_get_ciphers(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_get_ciphers_impl((PySSLContext *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLContext__set_alpn_protocols__doc__, +"_set_alpn_protocols($self, protos, /)\n" +"--\n" +"\n"); + +#define _SSL__SSLCONTEXT__SET_ALPN_PROTOCOLS_METHODDEF \ + {"_set_alpn_protocols", (PyCFunction)_ssl__SSLContext__set_alpn_protocols, METH_O, _ssl__SSLContext__set_alpn_protocols__doc__}, + +static PyObject * +_ssl__SSLContext__set_alpn_protocols_impl(PySSLContext *self, + Py_buffer *protos); + +static PyObject * +_ssl__SSLContext__set_alpn_protocols(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_buffer protos = {NULL, NULL}; + + if (PyObject_GetBuffer(arg, &protos, PyBUF_SIMPLE) != 0) { + goto exit; + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext__set_alpn_protocols_impl((PySSLContext *)self, &protos); + Py_END_CRITICAL_SECTION(); + +exit: + /* Cleanup for protos */ + if (protos.obj) { + PyBuffer_Release(&protos); + } + + return return_value; +} + +#if !defined(_ssl__SSLContext_verify_mode_DOCSTR) +# define _ssl__SSLContext_verify_mode_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF) +# undef _SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF +# define _SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF {"verify_mode", (getter)_ssl__SSLContext_verify_mode_get, (setter)_ssl__SSLContext_verify_mode_set, _ssl__SSLContext_verify_mode_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF {"verify_mode", (getter)_ssl__SSLContext_verify_mode_get, NULL, _ssl__SSLContext_verify_mode_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLContext_verify_mode_get_impl(PySSLContext *self); + +static PyObject * +_ssl__SSLContext_verify_mode_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_verify_mode_get_impl((PySSLContext *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ssl__SSLContext_verify_mode_DOCSTR) +# define _ssl__SSLContext_verify_mode_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF) +# undef _SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF +# define _SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF {"verify_mode", (getter)_ssl__SSLContext_verify_mode_get, (setter)_ssl__SSLContext_verify_mode_set, _ssl__SSLContext_verify_mode_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF {"verify_mode", NULL, (setter)_ssl__SSLContext_verify_mode_set, NULL}, +#endif + +static int +_ssl__SSLContext_verify_mode_set_impl(PySSLContext *self, PyObject *value); + +static int +_ssl__SSLContext_verify_mode_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_verify_mode_set_impl((PySSLContext *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ssl__SSLContext_verify_flags_DOCSTR) +# define _ssl__SSLContext_verify_flags_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF) +# undef _SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF +# define _SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF {"verify_flags", (getter)_ssl__SSLContext_verify_flags_get, (setter)_ssl__SSLContext_verify_flags_set, _ssl__SSLContext_verify_flags_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF {"verify_flags", (getter)_ssl__SSLContext_verify_flags_get, NULL, _ssl__SSLContext_verify_flags_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLContext_verify_flags_get_impl(PySSLContext *self); + +static PyObject * +_ssl__SSLContext_verify_flags_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_verify_flags_get_impl((PySSLContext *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ssl__SSLContext_verify_flags_DOCSTR) +# define _ssl__SSLContext_verify_flags_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF) +# undef _SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF +# define _SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF {"verify_flags", (getter)_ssl__SSLContext_verify_flags_get, (setter)_ssl__SSLContext_verify_flags_set, _ssl__SSLContext_verify_flags_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF {"verify_flags", NULL, (setter)_ssl__SSLContext_verify_flags_set, NULL}, +#endif + +static int +_ssl__SSLContext_verify_flags_set_impl(PySSLContext *self, PyObject *value); + +static int +_ssl__SSLContext_verify_flags_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_verify_flags_set_impl((PySSLContext *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ssl__SSLContext_minimum_version_DOCSTR) +# define _ssl__SSLContext_minimum_version_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF) +# undef _SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF +# define _SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF {"minimum_version", (getter)_ssl__SSLContext_minimum_version_get, (setter)_ssl__SSLContext_minimum_version_set, _ssl__SSLContext_minimum_version_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF {"minimum_version", (getter)_ssl__SSLContext_minimum_version_get, NULL, _ssl__SSLContext_minimum_version_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLContext_minimum_version_get_impl(PySSLContext *self); + +static PyObject * +_ssl__SSLContext_minimum_version_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_minimum_version_get_impl((PySSLContext *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ssl__SSLContext_minimum_version_DOCSTR) +# define _ssl__SSLContext_minimum_version_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF) +# undef _SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF +# define _SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF {"minimum_version", (getter)_ssl__SSLContext_minimum_version_get, (setter)_ssl__SSLContext_minimum_version_set, _ssl__SSLContext_minimum_version_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF {"minimum_version", NULL, (setter)_ssl__SSLContext_minimum_version_set, NULL}, +#endif + +static int +_ssl__SSLContext_minimum_version_set_impl(PySSLContext *self, + PyObject *value); + +static int +_ssl__SSLContext_minimum_version_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_minimum_version_set_impl((PySSLContext *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ssl__SSLContext_maximum_version_DOCSTR) +# define _ssl__SSLContext_maximum_version_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF) +# undef _SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF +# define _SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF {"maximum_version", (getter)_ssl__SSLContext_maximum_version_get, (setter)_ssl__SSLContext_maximum_version_set, _ssl__SSLContext_maximum_version_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF {"maximum_version", (getter)_ssl__SSLContext_maximum_version_get, NULL, _ssl__SSLContext_maximum_version_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLContext_maximum_version_get_impl(PySSLContext *self); + +static PyObject * +_ssl__SSLContext_maximum_version_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_maximum_version_get_impl((PySSLContext *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ssl__SSLContext_maximum_version_DOCSTR) +# define _ssl__SSLContext_maximum_version_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF) +# undef _SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF +# define _SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF {"maximum_version", (getter)_ssl__SSLContext_maximum_version_get, (setter)_ssl__SSLContext_maximum_version_set, _ssl__SSLContext_maximum_version_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF {"maximum_version", NULL, (setter)_ssl__SSLContext_maximum_version_set, NULL}, +#endif + +static int +_ssl__SSLContext_maximum_version_set_impl(PySSLContext *self, + PyObject *value); + +static int +_ssl__SSLContext_maximum_version_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_maximum_version_set_impl((PySSLContext *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLContext_num_tickets__doc__, +"Control the number of TLSv1.3 session tickets."); +#if defined(_ssl__SSLContext_num_tickets_DOCSTR) +# undef _ssl__SSLContext_num_tickets_DOCSTR +#endif +#define _ssl__SSLContext_num_tickets_DOCSTR _ssl__SSLContext_num_tickets__doc__ + +#if !defined(_ssl__SSLContext_num_tickets_DOCSTR) +# define _ssl__SSLContext_num_tickets_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF) +# undef _SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF +# define _SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF {"num_tickets", (getter)_ssl__SSLContext_num_tickets_get, (setter)_ssl__SSLContext_num_tickets_set, _ssl__SSLContext_num_tickets_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF {"num_tickets", (getter)_ssl__SSLContext_num_tickets_get, NULL, _ssl__SSLContext_num_tickets_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLContext_num_tickets_get_impl(PySSLContext *self); + +static PyObject * +_ssl__SSLContext_num_tickets_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_num_tickets_get_impl((PySSLContext *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ssl__SSLContext_num_tickets_DOCSTR) +# define _ssl__SSLContext_num_tickets_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF) +# undef _SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF +# define _SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF {"num_tickets", (getter)_ssl__SSLContext_num_tickets_get, (setter)_ssl__SSLContext_num_tickets_set, _ssl__SSLContext_num_tickets_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF {"num_tickets", NULL, (setter)_ssl__SSLContext_num_tickets_set, NULL}, +#endif + +static int +_ssl__SSLContext_num_tickets_set_impl(PySSLContext *self, PyObject *value); + +static int +_ssl__SSLContext_num_tickets_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_num_tickets_set_impl((PySSLContext *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLContext_security_level__doc__, +"The current security level."); +#if defined(_ssl__SSLContext_security_level_DOCSTR) +# undef _ssl__SSLContext_security_level_DOCSTR +#endif +#define _ssl__SSLContext_security_level_DOCSTR _ssl__SSLContext_security_level__doc__ + +#if !defined(_ssl__SSLContext_security_level_DOCSTR) +# define _ssl__SSLContext_security_level_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_SECURITY_LEVEL_GETSETDEF) +# undef _SSL__SSLCONTEXT_SECURITY_LEVEL_GETSETDEF +# define _SSL__SSLCONTEXT_SECURITY_LEVEL_GETSETDEF {"security_level", (getter)_ssl__SSLContext_security_level_get, (setter)_ssl__SSLContext_security_level_set, _ssl__SSLContext_security_level_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_SECURITY_LEVEL_GETSETDEF {"security_level", (getter)_ssl__SSLContext_security_level_get, NULL, _ssl__SSLContext_security_level_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLContext_security_level_get_impl(PySSLContext *self); + +static PyObject * +_ssl__SSLContext_security_level_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_security_level_get_impl((PySSLContext *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ssl__SSLContext_options_DOCSTR) +# define _ssl__SSLContext_options_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_OPTIONS_GETSETDEF) +# undef _SSL__SSLCONTEXT_OPTIONS_GETSETDEF +# define _SSL__SSLCONTEXT_OPTIONS_GETSETDEF {"options", (getter)_ssl__SSLContext_options_get, (setter)_ssl__SSLContext_options_set, _ssl__SSLContext_options_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_OPTIONS_GETSETDEF {"options", (getter)_ssl__SSLContext_options_get, NULL, _ssl__SSLContext_options_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLContext_options_get_impl(PySSLContext *self); + +static PyObject * +_ssl__SSLContext_options_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_options_get_impl((PySSLContext *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ssl__SSLContext_options_DOCSTR) +# define _ssl__SSLContext_options_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_OPTIONS_GETSETDEF) +# undef _SSL__SSLCONTEXT_OPTIONS_GETSETDEF +# define _SSL__SSLCONTEXT_OPTIONS_GETSETDEF {"options", (getter)_ssl__SSLContext_options_get, (setter)_ssl__SSLContext_options_set, _ssl__SSLContext_options_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_OPTIONS_GETSETDEF {"options", NULL, (setter)_ssl__SSLContext_options_set, NULL}, +#endif + +static int +_ssl__SSLContext_options_set_impl(PySSLContext *self, PyObject *value); + +static int +_ssl__SSLContext_options_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_options_set_impl((PySSLContext *)self, value); + Py_END_CRITICAL_SECTION(); -exit: return return_value; } -PyDoc_STRVAR(_ssl__SSLContext_set_ciphers__doc__, -"set_ciphers($self, cipherlist, /)\n" -"--\n" -"\n"); - -#define _SSL__SSLCONTEXT_SET_CIPHERS_METHODDEF \ - {"set_ciphers", (PyCFunction)_ssl__SSLContext_set_ciphers, METH_O, _ssl__SSLContext_set_ciphers__doc__}, +#if !defined(_ssl__SSLContext__host_flags_DOCSTR) +# define _ssl__SSLContext__host_flags_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF) +# undef _SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF +# define _SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF {"_host_flags", (getter)_ssl__SSLContext__host_flags_get, (setter)_ssl__SSLContext__host_flags_set, _ssl__SSLContext__host_flags_DOCSTR}, +#else +# define _SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF {"_host_flags", (getter)_ssl__SSLContext__host_flags_get, NULL, _ssl__SSLContext__host_flags_DOCSTR}, +#endif static PyObject * -_ssl__SSLContext_set_ciphers_impl(PySSLContext *self, const char *cipherlist); +_ssl__SSLContext__host_flags_get_impl(PySSLContext *self); static PyObject * -_ssl__SSLContext_set_ciphers(PySSLContext *self, PyObject *arg) +_ssl__SSLContext__host_flags_get(PyObject *self, void *Py_UNUSED(context)) { PyObject *return_value = NULL; - const char *cipherlist; - if (!PyUnicode_Check(arg)) { - _PyArg_BadArgument("set_ciphers", "argument", "str", arg); - goto exit; - } - Py_ssize_t cipherlist_length; - cipherlist = PyUnicode_AsUTF8AndSize(arg, &cipherlist_length); - if (cipherlist == NULL) { - goto exit; - } - if (strlen(cipherlist) != (size_t)cipherlist_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } - return_value = _ssl__SSLContext_set_ciphers_impl(self, cipherlist); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext__host_flags_get_impl((PySSLContext *)self); + Py_END_CRITICAL_SECTION(); -exit: return return_value; } -PyDoc_STRVAR(_ssl__SSLContext_get_ciphers__doc__, -"get_ciphers($self, /)\n" -"--\n" -"\n"); +#if !defined(_ssl__SSLContext__host_flags_DOCSTR) +# define _ssl__SSLContext__host_flags_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF) +# undef _SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF +# define _SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF {"_host_flags", (getter)_ssl__SSLContext__host_flags_get, (setter)_ssl__SSLContext__host_flags_set, _ssl__SSLContext__host_flags_DOCSTR}, +#else +# define _SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF {"_host_flags", NULL, (setter)_ssl__SSLContext__host_flags_set, NULL}, +#endif -#define _SSL__SSLCONTEXT_GET_CIPHERS_METHODDEF \ - {"get_ciphers", (PyCFunction)_ssl__SSLContext_get_ciphers, METH_NOARGS, _ssl__SSLContext_get_ciphers__doc__}, +static int +_ssl__SSLContext__host_flags_set_impl(PySSLContext *self, PyObject *value); + +static int +_ssl__SSLContext__host_flags_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext__host_flags_set_impl((PySSLContext *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ssl__SSLContext_check_hostname_DOCSTR) +# define _ssl__SSLContext_check_hostname_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF) +# undef _SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF +# define _SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF {"check_hostname", (getter)_ssl__SSLContext_check_hostname_get, (setter)_ssl__SSLContext_check_hostname_set, _ssl__SSLContext_check_hostname_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF {"check_hostname", (getter)_ssl__SSLContext_check_hostname_get, NULL, _ssl__SSLContext_check_hostname_DOCSTR}, +#endif static PyObject * -_ssl__SSLContext_get_ciphers_impl(PySSLContext *self); +_ssl__SSLContext_check_hostname_get_impl(PySSLContext *self); static PyObject * -_ssl__SSLContext_get_ciphers(PySSLContext *self, PyObject *Py_UNUSED(ignored)) +_ssl__SSLContext_check_hostname_get(PyObject *self, void *Py_UNUSED(context)) { - return _ssl__SSLContext_get_ciphers_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_check_hostname_get_impl((PySSLContext *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } -PyDoc_STRVAR(_ssl__SSLContext__set_alpn_protocols__doc__, -"_set_alpn_protocols($self, protos, /)\n" -"--\n" -"\n"); +#if !defined(_ssl__SSLContext_check_hostname_DOCSTR) +# define _ssl__SSLContext_check_hostname_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF) +# undef _SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF +# define _SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF {"check_hostname", (getter)_ssl__SSLContext_check_hostname_get, (setter)_ssl__SSLContext_check_hostname_set, _ssl__SSLContext_check_hostname_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF {"check_hostname", NULL, (setter)_ssl__SSLContext_check_hostname_set, NULL}, +#endif -#define _SSL__SSLCONTEXT__SET_ALPN_PROTOCOLS_METHODDEF \ - {"_set_alpn_protocols", (PyCFunction)_ssl__SSLContext__set_alpn_protocols, METH_O, _ssl__SSLContext__set_alpn_protocols__doc__}, +static int +_ssl__SSLContext_check_hostname_set_impl(PySSLContext *self, PyObject *value); + +static int +_ssl__SSLContext_check_hostname_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_check_hostname_set_impl((PySSLContext *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ssl__SSLContext_protocol_DOCSTR) +# define _ssl__SSLContext_protocol_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_PROTOCOL_GETSETDEF) +# undef _SSL__SSLCONTEXT_PROTOCOL_GETSETDEF +# define _SSL__SSLCONTEXT_PROTOCOL_GETSETDEF {"protocol", (getter)_ssl__SSLContext_protocol_get, (setter)_ssl__SSLContext_protocol_set, _ssl__SSLContext_protocol_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_PROTOCOL_GETSETDEF {"protocol", (getter)_ssl__SSLContext_protocol_get, NULL, _ssl__SSLContext_protocol_DOCSTR}, +#endif static PyObject * -_ssl__SSLContext__set_alpn_protocols_impl(PySSLContext *self, - Py_buffer *protos); +_ssl__SSLContext_protocol_get_impl(PySSLContext *self); static PyObject * -_ssl__SSLContext__set_alpn_protocols(PySSLContext *self, PyObject *arg) +_ssl__SSLContext_protocol_get(PyObject *self, void *Py_UNUSED(context)) { PyObject *return_value = NULL; - Py_buffer protos = {NULL, NULL}; - - if (PyObject_GetBuffer(arg, &protos, PyBUF_SIMPLE) != 0) { - goto exit; - } - return_value = _ssl__SSLContext__set_alpn_protocols_impl(self, &protos); -exit: - /* Cleanup for protos */ - if (protos.obj) { - PyBuffer_Release(&protos); - } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_protocol_get_impl((PySSLContext *)self); + Py_END_CRITICAL_SECTION(); return return_value; } @@ -550,7 +1370,7 @@ _ssl__SSLContext_load_cert_chain_impl(PySSLContext *self, PyObject *certfile, PyObject *keyfile, PyObject *password); static PyObject * -_ssl__SSLContext_load_cert_chain(PySSLContext *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_ssl__SSLContext_load_cert_chain(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -584,7 +1404,8 @@ _ssl__SSLContext_load_cert_chain(PySSLContext *self, PyObject *const *args, Py_s PyObject *keyfile = Py_None; PyObject *password = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -600,7 +1421,9 @@ _ssl__SSLContext_load_cert_chain(PySSLContext *self, PyObject *const *args, Py_s } password = args[2]; skip_optional_pos: - return_value = _ssl__SSLContext_load_cert_chain_impl(self, certfile, keyfile, password); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_load_cert_chain_impl((PySSLContext *)self, certfile, keyfile, password); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -621,7 +1444,7 @@ _ssl__SSLContext_load_verify_locations_impl(PySSLContext *self, PyObject *cadata); static PyObject * -_ssl__SSLContext_load_verify_locations(PySSLContext *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_ssl__SSLContext_load_verify_locations(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -655,7 +1478,8 @@ _ssl__SSLContext_load_verify_locations(PySSLContext *self, PyObject *const *args PyObject *capath = Py_None; PyObject *cadata = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -676,7 +1500,9 @@ _ssl__SSLContext_load_verify_locations(PySSLContext *self, PyObject *const *args } cadata = args[2]; skip_optional_pos: - return_value = _ssl__SSLContext_load_verify_locations_impl(self, cafile, capath, cadata); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_load_verify_locations_impl((PySSLContext *)self, cafile, capath, cadata); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -690,6 +1516,21 @@ PyDoc_STRVAR(_ssl__SSLContext_load_dh_params__doc__, #define _SSL__SSLCONTEXT_LOAD_DH_PARAMS_METHODDEF \ {"load_dh_params", (PyCFunction)_ssl__SSLContext_load_dh_params, METH_O, _ssl__SSLContext_load_dh_params__doc__}, +static PyObject * +_ssl__SSLContext_load_dh_params_impl(PySSLContext *self, PyObject *filepath); + +static PyObject * +_ssl__SSLContext_load_dh_params(PySSLContext *self, PyObject *filepath) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_load_dh_params_impl((PySSLContext *)self, filepath); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_ssl__SSLContext__wrap_socket__doc__, "_wrap_socket($self, /, sock, server_side, server_hostname=None, *,\n" " owner=None, session=None)\n" @@ -705,7 +1546,7 @@ _ssl__SSLContext__wrap_socket_impl(PySSLContext *self, PyObject *sock, PyObject *owner, PyObject *session); static PyObject * -_ssl__SSLContext__wrap_socket(PySSLContext *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_ssl__SSLContext__wrap_socket(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -741,7 +1582,8 @@ _ssl__SSLContext__wrap_socket(PySSLContext *self, PyObject *const *args, Py_ssiz PyObject *owner = Py_None; PyObject *session = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -775,7 +1617,9 @@ _ssl__SSLContext__wrap_socket(PySSLContext *self, PyObject *const *args, Py_ssiz } session = args[4]; skip_optional_kwonly: - return_value = _ssl__SSLContext__wrap_socket_impl(self, sock, server_side, hostname_obj, owner, session); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext__wrap_socket_impl((PySSLContext *)self, sock, server_side, hostname_obj, owner, session); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -797,7 +1641,7 @@ _ssl__SSLContext__wrap_bio_impl(PySSLContext *self, PySSLMemoryBIO *incoming, PyObject *session); static PyObject * -_ssl__SSLContext__wrap_bio(PySSLContext *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_ssl__SSLContext__wrap_bio(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -834,7 +1678,8 @@ _ssl__SSLContext__wrap_bio(PySSLContext *self, PyObject *const *args, Py_ssize_t PyObject *owner = Py_None; PyObject *session = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -873,7 +1718,9 @@ _ssl__SSLContext__wrap_bio(PySSLContext *self, PyObject *const *args, Py_ssize_t } session = args[5]; skip_optional_kwonly: - return_value = _ssl__SSLContext__wrap_bio_impl(self, incoming, outgoing, server_side, hostname_obj, owner, session); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext__wrap_bio_impl((PySSLContext *)self, incoming, outgoing, server_side, hostname_obj, owner, session); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -891,9 +1738,15 @@ static PyObject * _ssl__SSLContext_session_stats_impl(PySSLContext *self); static PyObject * -_ssl__SSLContext_session_stats(PySSLContext *self, PyObject *Py_UNUSED(ignored)) +_ssl__SSLContext_session_stats(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLContext_session_stats_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_session_stats_impl((PySSLContext *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLContext_set_default_verify_paths__doc__, @@ -908,9 +1761,15 @@ static PyObject * _ssl__SSLContext_set_default_verify_paths_impl(PySSLContext *self); static PyObject * -_ssl__SSLContext_set_default_verify_paths(PySSLContext *self, PyObject *Py_UNUSED(ignored)) +_ssl__SSLContext_set_default_verify_paths(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLContext_set_default_verify_paths_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_set_default_verify_paths_impl((PySSLContext *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLContext_set_ecdh_curve__doc__, @@ -921,6 +1780,83 @@ PyDoc_STRVAR(_ssl__SSLContext_set_ecdh_curve__doc__, #define _SSL__SSLCONTEXT_SET_ECDH_CURVE_METHODDEF \ {"set_ecdh_curve", (PyCFunction)_ssl__SSLContext_set_ecdh_curve, METH_O, _ssl__SSLContext_set_ecdh_curve__doc__}, +static PyObject * +_ssl__SSLContext_set_ecdh_curve_impl(PySSLContext *self, PyObject *name); + +static PyObject * +_ssl__SSLContext_set_ecdh_curve(PySSLContext *self, PyObject *name) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_set_ecdh_curve_impl((PySSLContext *)self, name); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLContext_sni_callback__doc__, +"Set a callback that will be called when a server name is provided by the SSL/TLS client in the SNI extension.\n" +"\n" +"If the argument is None then the callback is disabled. The method is called\n" +"with the SSLSocket, the server name as a string, and the SSLContext object.\n" +"\n" +"See RFC 6066 for details of the SNI extension."); +#if defined(_ssl__SSLContext_sni_callback_DOCSTR) +# undef _ssl__SSLContext_sni_callback_DOCSTR +#endif +#define _ssl__SSLContext_sni_callback_DOCSTR _ssl__SSLContext_sni_callback__doc__ + +#if !defined(_ssl__SSLContext_sni_callback_DOCSTR) +# define _ssl__SSLContext_sni_callback_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF) +# undef _SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF +# define _SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF {"sni_callback", (getter)_ssl__SSLContext_sni_callback_get, (setter)_ssl__SSLContext_sni_callback_set, _ssl__SSLContext_sni_callback_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF {"sni_callback", (getter)_ssl__SSLContext_sni_callback_get, NULL, _ssl__SSLContext_sni_callback_DOCSTR}, +#endif + +static PyObject * +_ssl__SSLContext_sni_callback_get_impl(PySSLContext *self); + +static PyObject * +_ssl__SSLContext_sni_callback_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_sni_callback_get_impl((PySSLContext *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(_ssl__SSLContext_sni_callback_DOCSTR) +# define _ssl__SSLContext_sni_callback_DOCSTR NULL +#endif +#if defined(_SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF) +# undef _SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF +# define _SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF {"sni_callback", (getter)_ssl__SSLContext_sni_callback_get, (setter)_ssl__SSLContext_sni_callback_set, _ssl__SSLContext_sni_callback_DOCSTR}, +#else +# define _SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF {"sni_callback", NULL, (setter)_ssl__SSLContext_sni_callback_set, NULL}, +#endif + +static int +_ssl__SSLContext_sni_callback_set_impl(PySSLContext *self, PyObject *value); + +static int +_ssl__SSLContext_sni_callback_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_sni_callback_set_impl((PySSLContext *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_ssl__SSLContext_cert_store_stats__doc__, "cert_store_stats($self, /)\n" "--\n" @@ -940,9 +1876,15 @@ static PyObject * _ssl__SSLContext_cert_store_stats_impl(PySSLContext *self); static PyObject * -_ssl__SSLContext_cert_store_stats(PySSLContext *self, PyObject *Py_UNUSED(ignored)) +_ssl__SSLContext_cert_store_stats(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _ssl__SSLContext_cert_store_stats_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_cert_store_stats_impl((PySSLContext *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl__SSLContext_get_ca_certs__doc__, @@ -964,7 +1906,7 @@ static PyObject * _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form); static PyObject * -_ssl__SSLContext_get_ca_certs(PySSLContext *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_ssl__SSLContext_get_ca_certs(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -996,7 +1938,8 @@ _ssl__SSLContext_get_ca_certs(PySSLContext *self, PyObject *const *args, Py_ssiz Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; int binary_form = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1008,7 +1951,9 @@ _ssl__SSLContext_get_ca_certs(PySSLContext *self, PyObject *const *args, Py_ssiz goto exit; } skip_optional_pos: - return_value = _ssl__SSLContext_get_ca_certs_impl(self, binary_form); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_get_ca_certs_impl((PySSLContext *)self, binary_form); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -1027,7 +1972,7 @@ _ssl__SSLContext_set_psk_client_callback_impl(PySSLContext *self, PyObject *callback); static PyObject * -_ssl__SSLContext_set_psk_client_callback(PySSLContext *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_ssl__SSLContext_set_psk_client_callback(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -1058,12 +2003,15 @@ _ssl__SSLContext_set_psk_client_callback(PySSLContext *self, PyObject *const *ar PyObject *argsbuf[1]; PyObject *callback; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } callback = args[0]; - return_value = _ssl__SSLContext_set_psk_client_callback_impl(self, callback); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_set_psk_client_callback_impl((PySSLContext *)self, callback); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -1083,7 +2031,7 @@ _ssl__SSLContext_set_psk_server_callback_impl(PySSLContext *self, const char *identity_hint); static PyObject * -_ssl__SSLContext_set_psk_server_callback(PySSLContext *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_ssl__SSLContext_set_psk_server_callback(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -1116,7 +2064,8 @@ _ssl__SSLContext_set_psk_server_callback(PySSLContext *self, PyObject *const *ar PyObject *callback; const char *identity_hint = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1143,7 +2092,9 @@ _ssl__SSLContext_set_psk_server_callback(PySSLContext *self, PyObject *const *ar goto exit; } skip_optional_pos: - return_value = _ssl__SSLContext_set_psk_server_callback_impl(self, callback, identity_hint); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_set_psk_server_callback_impl((PySSLContext *)self, callback, identity_hint); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -1166,12 +2117,78 @@ _ssl_MemoryBIO(PyTypeObject *type, PyObject *args, PyObject *kwargs) !_PyArg_NoKeywords("MemoryBIO", kwargs)) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(type); return_value = _ssl_MemoryBIO_impl(type); + Py_END_CRITICAL_SECTION(); exit: return return_value; } +PyDoc_STRVAR(_ssl_MemoryBIO_pending__doc__, +"The number of bytes pending in the memory BIO."); +#if defined(_ssl_MemoryBIO_pending_DOCSTR) +# undef _ssl_MemoryBIO_pending_DOCSTR +#endif +#define _ssl_MemoryBIO_pending_DOCSTR _ssl_MemoryBIO_pending__doc__ + +#if !defined(_ssl_MemoryBIO_pending_DOCSTR) +# define _ssl_MemoryBIO_pending_DOCSTR NULL +#endif +#if defined(_SSL_MEMORYBIO_PENDING_GETSETDEF) +# undef _SSL_MEMORYBIO_PENDING_GETSETDEF +# define _SSL_MEMORYBIO_PENDING_GETSETDEF {"pending", (getter)_ssl_MemoryBIO_pending_get, (setter)_ssl_MemoryBIO_pending_set, _ssl_MemoryBIO_pending_DOCSTR}, +#else +# define _SSL_MEMORYBIO_PENDING_GETSETDEF {"pending", (getter)_ssl_MemoryBIO_pending_get, NULL, _ssl_MemoryBIO_pending_DOCSTR}, +#endif + +static PyObject * +_ssl_MemoryBIO_pending_get_impl(PySSLMemoryBIO *self); + +static PyObject * +_ssl_MemoryBIO_pending_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl_MemoryBIO_pending_get_impl((PySSLMemoryBIO *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl_MemoryBIO_eof__doc__, +"Whether the memory BIO is at EOF."); +#if defined(_ssl_MemoryBIO_eof_DOCSTR) +# undef _ssl_MemoryBIO_eof_DOCSTR +#endif +#define _ssl_MemoryBIO_eof_DOCSTR _ssl_MemoryBIO_eof__doc__ + +#if !defined(_ssl_MemoryBIO_eof_DOCSTR) +# define _ssl_MemoryBIO_eof_DOCSTR NULL +#endif +#if defined(_SSL_MEMORYBIO_EOF_GETSETDEF) +# undef _SSL_MEMORYBIO_EOF_GETSETDEF +# define _SSL_MEMORYBIO_EOF_GETSETDEF {"eof", (getter)_ssl_MemoryBIO_eof_get, (setter)_ssl_MemoryBIO_eof_set, _ssl_MemoryBIO_eof_DOCSTR}, +#else +# define _SSL_MEMORYBIO_EOF_GETSETDEF {"eof", (getter)_ssl_MemoryBIO_eof_get, NULL, _ssl_MemoryBIO_eof_DOCSTR}, +#endif + +static PyObject * +_ssl_MemoryBIO_eof_get_impl(PySSLMemoryBIO *self); + +static PyObject * +_ssl_MemoryBIO_eof_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl_MemoryBIO_eof_get_impl((PySSLMemoryBIO *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_ssl_MemoryBIO_read__doc__, "read($self, size=-1, /)\n" "--\n" @@ -1190,7 +2207,7 @@ static PyObject * _ssl_MemoryBIO_read_impl(PySSLMemoryBIO *self, int len); static PyObject * -_ssl_MemoryBIO_read(PySSLMemoryBIO *self, PyObject *const *args, Py_ssize_t nargs) +_ssl_MemoryBIO_read(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int len = -1; @@ -1206,7 +2223,9 @@ _ssl_MemoryBIO_read(PySSLMemoryBIO *self, PyObject *const *args, Py_ssize_t narg goto exit; } skip_optional: - return_value = _ssl_MemoryBIO_read_impl(self, len); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl_MemoryBIO_read_impl((PySSLMemoryBIO *)self, len); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -1227,7 +2246,7 @@ static PyObject * _ssl_MemoryBIO_write_impl(PySSLMemoryBIO *self, Py_buffer *b); static PyObject * -_ssl_MemoryBIO_write(PySSLMemoryBIO *self, PyObject *arg) +_ssl_MemoryBIO_write(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; Py_buffer b = {NULL, NULL}; @@ -1235,7 +2254,9 @@ _ssl_MemoryBIO_write(PySSLMemoryBIO *self, PyObject *arg) if (PyObject_GetBuffer(arg, &b, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = _ssl_MemoryBIO_write_impl(self, &b); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl_MemoryBIO_write_impl((PySSLMemoryBIO *)self, &b); + Py_END_CRITICAL_SECTION(); exit: /* Cleanup for b */ @@ -1261,9 +2282,175 @@ static PyObject * _ssl_MemoryBIO_write_eof_impl(PySSLMemoryBIO *self); static PyObject * -_ssl_MemoryBIO_write_eof(PySSLMemoryBIO *self, PyObject *Py_UNUSED(ignored)) +_ssl_MemoryBIO_write_eof(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl_MemoryBIO_write_eof_impl((PySSLMemoryBIO *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl_SSLSession_time__doc__, +"Session creation time (seconds since epoch)."); +#if defined(_ssl_SSLSession_time_DOCSTR) +# undef _ssl_SSLSession_time_DOCSTR +#endif +#define _ssl_SSLSession_time_DOCSTR _ssl_SSLSession_time__doc__ + +#if !defined(_ssl_SSLSession_time_DOCSTR) +# define _ssl_SSLSession_time_DOCSTR NULL +#endif +#if defined(_SSL_SSLSESSION_TIME_GETSETDEF) +# undef _SSL_SSLSESSION_TIME_GETSETDEF +# define _SSL_SSLSESSION_TIME_GETSETDEF {"time", (getter)_ssl_SSLSession_time_get, (setter)_ssl_SSLSession_time_set, _ssl_SSLSession_time_DOCSTR}, +#else +# define _SSL_SSLSESSION_TIME_GETSETDEF {"time", (getter)_ssl_SSLSession_time_get, NULL, _ssl_SSLSession_time_DOCSTR}, +#endif + +static PyObject * +_ssl_SSLSession_time_get_impl(PySSLSession *self); + +static PyObject * +_ssl_SSLSession_time_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl_SSLSession_time_get_impl((PySSLSession *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl_SSLSession_timeout__doc__, +"Session timeout (delta in seconds)."); +#if defined(_ssl_SSLSession_timeout_DOCSTR) +# undef _ssl_SSLSession_timeout_DOCSTR +#endif +#define _ssl_SSLSession_timeout_DOCSTR _ssl_SSLSession_timeout__doc__ + +#if !defined(_ssl_SSLSession_timeout_DOCSTR) +# define _ssl_SSLSession_timeout_DOCSTR NULL +#endif +#if defined(_SSL_SSLSESSION_TIMEOUT_GETSETDEF) +# undef _SSL_SSLSESSION_TIMEOUT_GETSETDEF +# define _SSL_SSLSESSION_TIMEOUT_GETSETDEF {"timeout", (getter)_ssl_SSLSession_timeout_get, (setter)_ssl_SSLSession_timeout_set, _ssl_SSLSession_timeout_DOCSTR}, +#else +# define _SSL_SSLSESSION_TIMEOUT_GETSETDEF {"timeout", (getter)_ssl_SSLSession_timeout_get, NULL, _ssl_SSLSession_timeout_DOCSTR}, +#endif + +static PyObject * +_ssl_SSLSession_timeout_get_impl(PySSLSession *self); + +static PyObject * +_ssl_SSLSession_timeout_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl_SSLSession_timeout_get_impl((PySSLSession *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl_SSLSession_ticket_lifetime_hint__doc__, +"Ticket life time hint."); +#if defined(_ssl_SSLSession_ticket_lifetime_hint_DOCSTR) +# undef _ssl_SSLSession_ticket_lifetime_hint_DOCSTR +#endif +#define _ssl_SSLSession_ticket_lifetime_hint_DOCSTR _ssl_SSLSession_ticket_lifetime_hint__doc__ + +#if !defined(_ssl_SSLSession_ticket_lifetime_hint_DOCSTR) +# define _ssl_SSLSession_ticket_lifetime_hint_DOCSTR NULL +#endif +#if defined(_SSL_SSLSESSION_TICKET_LIFETIME_HINT_GETSETDEF) +# undef _SSL_SSLSESSION_TICKET_LIFETIME_HINT_GETSETDEF +# define _SSL_SSLSESSION_TICKET_LIFETIME_HINT_GETSETDEF {"ticket_lifetime_hint", (getter)_ssl_SSLSession_ticket_lifetime_hint_get, (setter)_ssl_SSLSession_ticket_lifetime_hint_set, _ssl_SSLSession_ticket_lifetime_hint_DOCSTR}, +#else +# define _SSL_SSLSESSION_TICKET_LIFETIME_HINT_GETSETDEF {"ticket_lifetime_hint", (getter)_ssl_SSLSession_ticket_lifetime_hint_get, NULL, _ssl_SSLSession_ticket_lifetime_hint_DOCSTR}, +#endif + +static PyObject * +_ssl_SSLSession_ticket_lifetime_hint_get_impl(PySSLSession *self); + +static PyObject * +_ssl_SSLSession_ticket_lifetime_hint_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl_SSLSession_ticket_lifetime_hint_get_impl((PySSLSession *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl_SSLSession_id__doc__, +"Session ID."); +#if defined(_ssl_SSLSession_id_DOCSTR) +# undef _ssl_SSLSession_id_DOCSTR +#endif +#define _ssl_SSLSession_id_DOCSTR _ssl_SSLSession_id__doc__ + +#if !defined(_ssl_SSLSession_id_DOCSTR) +# define _ssl_SSLSession_id_DOCSTR NULL +#endif +#if defined(_SSL_SSLSESSION_ID_GETSETDEF) +# undef _SSL_SSLSESSION_ID_GETSETDEF +# define _SSL_SSLSESSION_ID_GETSETDEF {"id", (getter)_ssl_SSLSession_id_get, (setter)_ssl_SSLSession_id_set, _ssl_SSLSession_id_DOCSTR}, +#else +# define _SSL_SSLSESSION_ID_GETSETDEF {"id", (getter)_ssl_SSLSession_id_get, NULL, _ssl_SSLSession_id_DOCSTR}, +#endif + +static PyObject * +_ssl_SSLSession_id_get_impl(PySSLSession *self); + +static PyObject * +_ssl_SSLSession_id_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl_SSLSession_id_get_impl((PySSLSession *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl_SSLSession_has_ticket__doc__, +"Does the session contain a ticket?"); +#if defined(_ssl_SSLSession_has_ticket_DOCSTR) +# undef _ssl_SSLSession_has_ticket_DOCSTR +#endif +#define _ssl_SSLSession_has_ticket_DOCSTR _ssl_SSLSession_has_ticket__doc__ + +#if !defined(_ssl_SSLSession_has_ticket_DOCSTR) +# define _ssl_SSLSession_has_ticket_DOCSTR NULL +#endif +#if defined(_SSL_SSLSESSION_HAS_TICKET_GETSETDEF) +# undef _SSL_SSLSESSION_HAS_TICKET_GETSETDEF +# define _SSL_SSLSESSION_HAS_TICKET_GETSETDEF {"has_ticket", (getter)_ssl_SSLSession_has_ticket_get, (setter)_ssl_SSLSession_has_ticket_set, _ssl_SSLSession_has_ticket_DOCSTR}, +#else +# define _SSL_SSLSESSION_HAS_TICKET_GETSETDEF {"has_ticket", (getter)_ssl_SSLSession_has_ticket_get, NULL, _ssl_SSLSession_has_ticket_DOCSTR}, +#endif + +static PyObject * +_ssl_SSLSession_has_ticket_get_impl(PySSLSession *self); + +static PyObject * +_ssl_SSLSession_has_ticket_get(PyObject *self, void *Py_UNUSED(context)) { - return _ssl_MemoryBIO_write_eof_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl_SSLSession_has_ticket_get_impl((PySSLSession *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl_RAND_add__doc__, @@ -1316,7 +2503,9 @@ _ssl_RAND_add(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } } + Py_BEGIN_CRITICAL_SECTION(module); return_value = _ssl_RAND_add_impl(module, &view, entropy); + Py_END_CRITICAL_SECTION(); exit: /* Cleanup for view */ @@ -1349,7 +2538,9 @@ _ssl_RAND_bytes(PyObject *module, PyObject *arg) if (n == -1 && PyErr_Occurred()) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(module); return_value = _ssl_RAND_bytes_impl(module, n); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -1373,7 +2564,13 @@ _ssl_RAND_status_impl(PyObject *module); static PyObject * _ssl_RAND_status(PyObject *module, PyObject *Py_UNUSED(ignored)) { - return _ssl_RAND_status_impl(module); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(module); + return_value = _ssl_RAND_status_impl(module); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl_get_default_verify_paths__doc__, @@ -1393,7 +2590,13 @@ _ssl_get_default_verify_paths_impl(PyObject *module); static PyObject * _ssl_get_default_verify_paths(PyObject *module, PyObject *Py_UNUSED(ignored)) { - return _ssl_get_default_verify_paths_impl(module); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(module); + return_value = _ssl_get_default_verify_paths_impl(module); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_ssl_txt2obj__doc__, @@ -1445,7 +2648,8 @@ _ssl_txt2obj(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject const char *txt; int name = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1470,7 +2674,9 @@ _ssl_txt2obj(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(module); return_value = _ssl_txt2obj_impl(module, txt, name); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -1498,7 +2704,9 @@ _ssl_nid2obj(PyObject *module, PyObject *arg) if (nid == -1 && PyErr_Occurred()) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(module); return_value = _ssl_nid2obj_impl(module, nid); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -1556,7 +2764,8 @@ _ssl_enum_certificates(PyObject *module, PyObject *const *args, Py_ssize_t nargs PyObject *argsbuf[1]; const char *store_name; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1573,7 +2782,9 @@ _ssl_enum_certificates(PyObject *module, PyObject *const *args, Py_ssize_t nargs PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } + Py_BEGIN_CRITICAL_SECTION(module); return_value = _ssl_enum_certificates_impl(module, store_name); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -1632,7 +2843,8 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *argsbuf[1]; const char *store_name; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1649,7 +2861,9 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } + Py_BEGIN_CRITICAL_SECTION(module); return_value = _ssl_enum_crls_impl(module, store_name); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -1664,4 +2878,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=fd1c3378fbba5240 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bededfb2b927bd41 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_statisticsmodule.c.h b/Modules/clinic/_statisticsmodule.c.h index 653a2138aaad70..cd5b9bea8db2f2 100644 --- a/Modules/clinic/_statisticsmodule.c.h +++ b/Modules/clinic/_statisticsmodule.c.h @@ -2,15 +2,13 @@ preserve [clinic start generated code]*/ -#include "pycore_modsupport.h" // _PyArg_CheckPositional() - PyDoc_STRVAR(_statistics__normal_dist_inv_cdf__doc__, "_normal_dist_inv_cdf($module, p, mu, sigma, /)\n" "--\n" "\n"); #define _STATISTICS__NORMAL_DIST_INV_CDF_METHODDEF \ - {"_normal_dist_inv_cdf", _PyCFunction_CAST(_statistics__normal_dist_inv_cdf), METH_FASTCALL, _statistics__normal_dist_inv_cdf__doc__}, + {"_normal_dist_inv_cdf", (PyCFunction)(void(*)(void))_statistics__normal_dist_inv_cdf, METH_FASTCALL, _statistics__normal_dist_inv_cdf__doc__}, static double _statistics__normal_dist_inv_cdf_impl(PyObject *module, double p, double mu, @@ -25,38 +23,21 @@ _statistics__normal_dist_inv_cdf(PyObject *module, PyObject *const *args, Py_ssi double sigma; double _return_value; - if (!_PyArg_CheckPositional("_normal_dist_inv_cdf", nargs, 3, 3)) { + if (nargs != 3) { + PyErr_Format(PyExc_TypeError, "_normal_dist_inv_cdf expected 3 arguments, got %zd", nargs); goto exit; } - if (PyFloat_CheckExact(args[0])) { - p = PyFloat_AS_DOUBLE(args[0]); - } - else - { - p = PyFloat_AsDouble(args[0]); - if (p == -1.0 && PyErr_Occurred()) { - goto exit; - } - } - if (PyFloat_CheckExact(args[1])) { - mu = PyFloat_AS_DOUBLE(args[1]); - } - else - { - mu = PyFloat_AsDouble(args[1]); - if (mu == -1.0 && PyErr_Occurred()) { - goto exit; - } + p = PyFloat_AsDouble(args[0]); + if (p == -1.0 && PyErr_Occurred()) { + goto exit; } - if (PyFloat_CheckExact(args[2])) { - sigma = PyFloat_AS_DOUBLE(args[2]); + mu = PyFloat_AsDouble(args[1]); + if (mu == -1.0 && PyErr_Occurred()) { + goto exit; } - else - { - sigma = PyFloat_AsDouble(args[2]); - if (sigma == -1.0 && PyErr_Occurred()) { - goto exit; - } + sigma = PyFloat_AsDouble(args[2]); + if (sigma == -1.0 && PyErr_Occurred()) { + goto exit; } _return_value = _statistics__normal_dist_inv_cdf_impl(module, p, mu, sigma); if ((_return_value == -1.0) && PyErr_Occurred()) { @@ -67,4 +48,4 @@ _statistics__normal_dist_inv_cdf(PyObject *module, PyObject *const *args, Py_ssi exit: return return_value; } -/*[clinic end generated code: output=e7cead17f9f3e19f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0f0c849d51f16f1b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_struct.c.h b/Modules/clinic/_struct.c.h index 1a07532bdd75ad..7cf179f7a69d55 100644 --- a/Modules/clinic/_struct.c.h +++ b/Modules/clinic/_struct.c.h @@ -57,7 +57,8 @@ Struct___init__(PyObject *self, PyObject *args, PyObject *kwargs) Py_ssize_t nargs = PyTuple_GET_SIZE(args); PyObject *format; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -86,7 +87,7 @@ static PyObject * Struct_unpack_impl(PyStructObject *self, Py_buffer *buffer); static PyObject * -Struct_unpack(PyStructObject *self, PyObject *arg) +Struct_unpack(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; Py_buffer buffer = {NULL, NULL}; @@ -94,7 +95,7 @@ Struct_unpack(PyStructObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &buffer, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = Struct_unpack_impl(self, &buffer); + return_value = Struct_unpack_impl((PyStructObject *)self, &buffer); exit: /* Cleanup for buffer */ @@ -126,7 +127,7 @@ Struct_unpack_from_impl(PyStructObject *self, Py_buffer *buffer, Py_ssize_t offset); static PyObject * -Struct_unpack_from(PyStructObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +Struct_unpack_from(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -159,7 +160,8 @@ Struct_unpack_from(PyStructObject *self, PyObject *const *args, Py_ssize_t nargs Py_buffer buffer = {NULL, NULL}; Py_ssize_t offset = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -182,7 +184,7 @@ Struct_unpack_from(PyStructObject *self, PyObject *const *args, Py_ssize_t nargs offset = ival; } skip_optional_pos: - return_value = Struct_unpack_from_impl(self, &buffer, offset); + return_value = Struct_unpack_from_impl((PyStructObject *)self, &buffer, offset); exit: /* Cleanup for buffer */ @@ -357,7 +359,8 @@ unpack_from(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject Py_buffer buffer = {NULL, NULL}; Py_ssize_t offset = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -436,4 +439,4 @@ iter_unpack(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -/*[clinic end generated code: output=67bd299e5d72fee0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ec540c21be08e1d0 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index bb516be37ec3f0..434488adbbb77c 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -9,6 +9,7 @@ preserve #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_runtime.h" // _Py_ID() +#include "pycore_tuple.h" // _PyTuple_FromArray() PyDoc_STRVAR(test_empty_function__doc__, "test_empty_function($module, /)\n" @@ -234,10 +235,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - if (PyBytes_Check(args[0]) && PyBytes_GET_SIZE(args[0]) == 1) { + if (PyBytes_Check(args[0])) { + if (PyBytes_GET_SIZE(args[0]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 1 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[0])); + goto exit; + } a = PyBytes_AS_STRING(args[0])[0]; } - else if (PyByteArray_Check(args[0]) && PyByteArray_GET_SIZE(args[0]) == 1) { + else if (PyByteArray_Check(args[0])) { + if (PyByteArray_GET_SIZE(args[0]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 1 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[0])); + goto exit; + } a = PyByteArray_AS_STRING(args[0])[0]; } else { @@ -247,10 +262,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + if (PyBytes_Check(args[1])) { + if (PyBytes_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 2 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[1])); + goto exit; + } b = PyBytes_AS_STRING(args[1])[0]; } - else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + else if (PyByteArray_Check(args[1])) { + if (PyByteArray_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 2 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[1])); + goto exit; + } b = PyByteArray_AS_STRING(args[1])[0]; } else { @@ -260,10 +289,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - if (PyBytes_Check(args[2]) && PyBytes_GET_SIZE(args[2]) == 1) { + if (PyBytes_Check(args[2])) { + if (PyBytes_GET_SIZE(args[2]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 3 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[2])); + goto exit; + } c = PyBytes_AS_STRING(args[2])[0]; } - else if (PyByteArray_Check(args[2]) && PyByteArray_GET_SIZE(args[2]) == 1) { + else if (PyByteArray_Check(args[2])) { + if (PyByteArray_GET_SIZE(args[2]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 3 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[2])); + goto exit; + } c = PyByteArray_AS_STRING(args[2])[0]; } else { @@ -273,10 +316,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 4) { goto skip_optional; } - if (PyBytes_Check(args[3]) && PyBytes_GET_SIZE(args[3]) == 1) { + if (PyBytes_Check(args[3])) { + if (PyBytes_GET_SIZE(args[3]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 4 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[3])); + goto exit; + } d = PyBytes_AS_STRING(args[3])[0]; } - else if (PyByteArray_Check(args[3]) && PyByteArray_GET_SIZE(args[3]) == 1) { + else if (PyByteArray_Check(args[3])) { + if (PyByteArray_GET_SIZE(args[3]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 4 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[3])); + goto exit; + } d = PyByteArray_AS_STRING(args[3])[0]; } else { @@ -286,10 +343,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 5) { goto skip_optional; } - if (PyBytes_Check(args[4]) && PyBytes_GET_SIZE(args[4]) == 1) { + if (PyBytes_Check(args[4])) { + if (PyBytes_GET_SIZE(args[4]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 5 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[4])); + goto exit; + } e = PyBytes_AS_STRING(args[4])[0]; } - else if (PyByteArray_Check(args[4]) && PyByteArray_GET_SIZE(args[4]) == 1) { + else if (PyByteArray_Check(args[4])) { + if (PyByteArray_GET_SIZE(args[4]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 5 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[4])); + goto exit; + } e = PyByteArray_AS_STRING(args[4])[0]; } else { @@ -299,10 +370,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 6) { goto skip_optional; } - if (PyBytes_Check(args[5]) && PyBytes_GET_SIZE(args[5]) == 1) { + if (PyBytes_Check(args[5])) { + if (PyBytes_GET_SIZE(args[5]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 6 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[5])); + goto exit; + } f = PyBytes_AS_STRING(args[5])[0]; } - else if (PyByteArray_Check(args[5]) && PyByteArray_GET_SIZE(args[5]) == 1) { + else if (PyByteArray_Check(args[5])) { + if (PyByteArray_GET_SIZE(args[5]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 6 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[5])); + goto exit; + } f = PyByteArray_AS_STRING(args[5])[0]; } else { @@ -312,10 +397,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 7) { goto skip_optional; } - if (PyBytes_Check(args[6]) && PyBytes_GET_SIZE(args[6]) == 1) { + if (PyBytes_Check(args[6])) { + if (PyBytes_GET_SIZE(args[6]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 7 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[6])); + goto exit; + } g = PyBytes_AS_STRING(args[6])[0]; } - else if (PyByteArray_Check(args[6]) && PyByteArray_GET_SIZE(args[6]) == 1) { + else if (PyByteArray_Check(args[6])) { + if (PyByteArray_GET_SIZE(args[6]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 7 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[6])); + goto exit; + } g = PyByteArray_AS_STRING(args[6])[0]; } else { @@ -325,10 +424,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 8) { goto skip_optional; } - if (PyBytes_Check(args[7]) && PyBytes_GET_SIZE(args[7]) == 1) { + if (PyBytes_Check(args[7])) { + if (PyBytes_GET_SIZE(args[7]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 8 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[7])); + goto exit; + } h = PyBytes_AS_STRING(args[7])[0]; } - else if (PyByteArray_Check(args[7]) && PyByteArray_GET_SIZE(args[7]) == 1) { + else if (PyByteArray_Check(args[7])) { + if (PyByteArray_GET_SIZE(args[7]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 8 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[7])); + goto exit; + } h = PyByteArray_AS_STRING(args[7])[0]; } else { @@ -338,10 +451,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 9) { goto skip_optional; } - if (PyBytes_Check(args[8]) && PyBytes_GET_SIZE(args[8]) == 1) { + if (PyBytes_Check(args[8])) { + if (PyBytes_GET_SIZE(args[8]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 9 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[8])); + goto exit; + } i = PyBytes_AS_STRING(args[8])[0]; } - else if (PyByteArray_Check(args[8]) && PyByteArray_GET_SIZE(args[8]) == 1) { + else if (PyByteArray_Check(args[8])) { + if (PyByteArray_GET_SIZE(args[8]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 9 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[8])); + goto exit; + } i = PyByteArray_AS_STRING(args[8])[0]; } else { @@ -351,10 +478,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 10) { goto skip_optional; } - if (PyBytes_Check(args[9]) && PyBytes_GET_SIZE(args[9]) == 1) { + if (PyBytes_Check(args[9])) { + if (PyBytes_GET_SIZE(args[9]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 10 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[9])); + goto exit; + } j = PyBytes_AS_STRING(args[9])[0]; } - else if (PyByteArray_Check(args[9]) && PyByteArray_GET_SIZE(args[9]) == 1) { + else if (PyByteArray_Check(args[9])) { + if (PyByteArray_GET_SIZE(args[9]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 10 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[9])); + goto exit; + } j = PyByteArray_AS_STRING(args[9])[0]; } else { @@ -364,10 +505,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 11) { goto skip_optional; } - if (PyBytes_Check(args[10]) && PyBytes_GET_SIZE(args[10]) == 1) { + if (PyBytes_Check(args[10])) { + if (PyBytes_GET_SIZE(args[10]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 11 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[10])); + goto exit; + } k = PyBytes_AS_STRING(args[10])[0]; } - else if (PyByteArray_Check(args[10]) && PyByteArray_GET_SIZE(args[10]) == 1) { + else if (PyByteArray_Check(args[10])) { + if (PyByteArray_GET_SIZE(args[10]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 11 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[10])); + goto exit; + } k = PyByteArray_AS_STRING(args[10])[0]; } else { @@ -377,10 +532,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 12) { goto skip_optional; } - if (PyBytes_Check(args[11]) && PyBytes_GET_SIZE(args[11]) == 1) { + if (PyBytes_Check(args[11])) { + if (PyBytes_GET_SIZE(args[11]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 12 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[11])); + goto exit; + } l = PyBytes_AS_STRING(args[11])[0]; } - else if (PyByteArray_Check(args[11]) && PyByteArray_GET_SIZE(args[11]) == 1) { + else if (PyByteArray_Check(args[11])) { + if (PyByteArray_GET_SIZE(args[11]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 12 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[11])); + goto exit; + } l = PyByteArray_AS_STRING(args[11])[0]; } else { @@ -390,10 +559,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 13) { goto skip_optional; } - if (PyBytes_Check(args[12]) && PyBytes_GET_SIZE(args[12]) == 1) { + if (PyBytes_Check(args[12])) { + if (PyBytes_GET_SIZE(args[12]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 13 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[12])); + goto exit; + } m = PyBytes_AS_STRING(args[12])[0]; } - else if (PyByteArray_Check(args[12]) && PyByteArray_GET_SIZE(args[12]) == 1) { + else if (PyByteArray_Check(args[12])) { + if (PyByteArray_GET_SIZE(args[12]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 13 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[12])); + goto exit; + } m = PyByteArray_AS_STRING(args[12])[0]; } else { @@ -403,10 +586,24 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 14) { goto skip_optional; } - if (PyBytes_Check(args[13]) && PyBytes_GET_SIZE(args[13]) == 1) { + if (PyBytes_Check(args[13])) { + if (PyBytes_GET_SIZE(args[13]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 14 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[13])); + goto exit; + } n = PyBytes_AS_STRING(args[13])[0]; } - else if (PyByteArray_Check(args[13]) && PyByteArray_GET_SIZE(args[13]) == 1) { + else if (PyByteArray_Check(args[13])) { + if (PyByteArray_GET_SIZE(args[13]) != 1) { + PyErr_Format(PyExc_TypeError, + "char_converter(): argument 14 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[13])); + goto exit; + } n = PyByteArray_AS_STRING(args[13])[0]; } else { @@ -648,7 +845,10 @@ int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[2]) != 1) { - _PyArg_BadArgument("int_converter", "argument 3", "a unicode character", args[2]); + PyErr_Format(PyExc_TypeError, + "int_converter(): argument 3 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[2])); goto exit; } c = PyUnicode_READ_CHAR(args[2], 0); @@ -1259,7 +1459,7 @@ keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1279,7 +1479,8 @@ keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *a; PyObject *b; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1315,7 +1516,7 @@ keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1335,7 +1536,8 @@ keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *a; PyObject *b; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1371,7 +1573,7 @@ keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1393,7 +1595,8 @@ keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *b = Py_None; PyObject *c = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1440,7 +1643,7 @@ keywords_opt_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1463,7 +1666,8 @@ keywords_opt_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1520,7 +1724,7 @@ keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1542,7 +1746,8 @@ keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *b = Py_None; PyObject *c = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1588,7 +1793,7 @@ posonly_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1608,7 +1813,8 @@ posonly_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb PyObject *a; PyObject *b; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1644,7 +1850,7 @@ posonly_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1664,7 +1870,8 @@ posonly_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *a; PyObject *b; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1701,7 +1908,7 @@ posonly_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1722,7 +1929,8 @@ posonly_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *b; PyObject *c; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1760,7 +1968,7 @@ posonly_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1783,7 +1991,8 @@ posonly_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1831,7 +2040,7 @@ posonly_opt_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1854,7 +2063,8 @@ posonly_opt_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1907,7 +2117,7 @@ posonly_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1930,7 +2140,8 @@ posonly_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1978,7 +2189,7 @@ posonly_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2001,7 +2212,8 @@ posonly_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2054,7 +2266,7 @@ posonly_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2078,7 +2290,8 @@ posonly_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *d = Py_None; PyObject *e = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2129,7 +2342,7 @@ posonly_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssiz PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2153,7 +2366,8 @@ posonly_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssiz PyObject *d = Py_None; PyObject *e = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2213,7 +2427,7 @@ posonly_opt_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2236,7 +2450,8 @@ posonly_opt_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ PyObject *c = Py_None; PyObject *d = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2292,7 +2507,7 @@ keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t nargs PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2311,7 +2526,8 @@ keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t nargs PyObject *argsbuf[1]; PyObject *a; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2322,115 +2538,132 @@ keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t nargs return return_value; } -PyDoc_STRVAR(posonly_vararg__doc__, -"posonly_vararg($module, a, /, b, *args)\n" +PyDoc_STRVAR(varpos__doc__, +"varpos($module, /, *args)\n" "--\n" "\n"); -#define POSONLY_VARARG_METHODDEF \ - {"posonly_vararg", _PyCFunction_CAST(posonly_vararg), METH_FASTCALL|METH_KEYWORDS, posonly_vararg__doc__}, +#define VARPOS_METHODDEF \ + {"varpos", _PyCFunction_CAST(varpos), METH_FASTCALL, varpos__doc__}, static PyObject * -posonly_vararg_impl(PyObject *module, PyObject *a, PyObject *b, - PyObject *args); +varpos_impl(PyObject *module, PyObject *args); static PyObject * -posonly_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + PyObject *__clinic_args = NULL; - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) + __clinic_args = _PyTuple_FromArray(args, nargs); + if (__clinic_args == NULL) { + goto exit; + } + return_value = varpos_impl(module, __clinic_args); - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); - static const char * const _keywords[] = {"", "b", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "posonly_vararg", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[3]; + return return_value; +} + +PyDoc_STRVAR(posonly_varpos__doc__, +"posonly_varpos($module, a, b, /, *args)\n" +"--\n" +"\n"); + +#define POSONLY_VARPOS_METHODDEF \ + {"posonly_varpos", _PyCFunction_CAST(posonly_varpos), METH_FASTCALL, posonly_varpos__doc__}, + +static PyObject * +posonly_varpos_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *args); + +static PyObject * +posonly_varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; PyObject *a; PyObject *b; PyObject *__clinic_args = NULL; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, 2, argsbuf); - if (!args) { + if (!_PyArg_CheckPositional("posonly_varpos", nargs, 2, PY_SSIZE_T_MAX)) { goto exit; } a = args[0]; b = args[1]; - __clinic_args = args[2]; - return_value = posonly_vararg_impl(module, a, b, __clinic_args); + __clinic_args = _PyTuple_FromArray(args + 2, nargs - 2); + if (__clinic_args == NULL) { + goto exit; + } + return_value = posonly_varpos_impl(module, a, b, __clinic_args); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } -PyDoc_STRVAR(vararg_and_posonly__doc__, -"vararg_and_posonly($module, a, /, *args)\n" +PyDoc_STRVAR(posonly_req_opt_varpos__doc__, +"posonly_req_opt_varpos($module, a, b=False, /, *args)\n" "--\n" "\n"); -#define VARARG_AND_POSONLY_METHODDEF \ - {"vararg_and_posonly", _PyCFunction_CAST(vararg_and_posonly), METH_FASTCALL, vararg_and_posonly__doc__}, +#define POSONLY_REQ_OPT_VARPOS_METHODDEF \ + {"posonly_req_opt_varpos", _PyCFunction_CAST(posonly_req_opt_varpos), METH_FASTCALL, posonly_req_opt_varpos__doc__}, static PyObject * -vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args); +posonly_req_opt_varpos_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *args); static PyObject * -vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +posonly_req_opt_varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *a; + PyObject *b = Py_False; PyObject *__clinic_args = NULL; - if (!_PyArg_CheckPositional("vararg_and_posonly", nargs, 1, PY_SSIZE_T_MAX)) { + if (!_PyArg_CheckPositional("posonly_req_opt_varpos", nargs, 1, PY_SSIZE_T_MAX)) { goto exit; } a = args[0]; - __clinic_args = PyTuple_New(nargs - 1); - if (!__clinic_args) { - goto exit; + if (nargs < 2) { + goto skip_optional; } - for (Py_ssize_t i = 0; i < nargs - 1; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[1 + i])); + b = args[1]; +skip_optional: + __clinic_args = nargs > 2 + ? _PyTuple_FromArray(args + 2, nargs - 2) + : PyTuple_New(0); + if (__clinic_args == NULL) { + goto exit; } - return_value = vararg_and_posonly_impl(module, a, __clinic_args); + return_value = posonly_req_opt_varpos_impl(module, a, b, __clinic_args); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } -PyDoc_STRVAR(vararg__doc__, -"vararg($module, /, a, *args)\n" +PyDoc_STRVAR(posonly_poskw_varpos__doc__, +"posonly_poskw_varpos($module, a, /, b, *args)\n" "--\n" "\n"); -#define VARARG_METHODDEF \ - {"vararg", _PyCFunction_CAST(vararg), METH_FASTCALL|METH_KEYWORDS, vararg__doc__}, +#define POSONLY_POSKW_VARPOS_METHODDEF \ + {"posonly_poskw_varpos", _PyCFunction_CAST(posonly_poskw_varpos), METH_FASTCALL|METH_KEYWORDS, posonly_poskw_varpos__doc__}, static PyObject * -vararg_impl(PyObject *module, PyObject *a, PyObject *args); +posonly_poskw_varpos_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *args); static PyObject * -vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +posonly_poskw_varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -2442,7 +2675,7 @@ vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwna PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2451,56 +2684,66 @@ vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwna # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"a", NULL}; + static const char * const _keywords[] = {"", "b", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "vararg", + .fname = "posonly_poskw_varpos", .kwtuple = KWTUPLE, }; #undef KWTUPLE PyObject *argsbuf[2]; + PyObject * const *fastargs; PyObject *a; + PyObject *b; PyObject *__clinic_args = NULL; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { goto exit; } - a = args[0]; - __clinic_args = args[1]; - return_value = vararg_impl(module, a, __clinic_args); + a = fastargs[0]; + b = fastargs[1]; + __clinic_args = nargs > 2 + ? _PyTuple_FromArray(args + 2, nargs - 2) + : PyTuple_New(0); + if (__clinic_args == NULL) { + goto exit; + } + return_value = posonly_poskw_varpos_impl(module, a, b, __clinic_args); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } -PyDoc_STRVAR(vararg_with_default__doc__, -"vararg_with_default($module, /, a, *args, b=False)\n" +PyDoc_STRVAR(poskw_varpos__doc__, +"poskw_varpos($module, /, a, *args)\n" "--\n" "\n"); -#define VARARG_WITH_DEFAULT_METHODDEF \ - {"vararg_with_default", _PyCFunction_CAST(vararg_with_default), METH_FASTCALL|METH_KEYWORDS, vararg_with_default__doc__}, +#define POSKW_VARPOS_METHODDEF \ + {"poskw_varpos", _PyCFunction_CAST(poskw_varpos), METH_FASTCALL|METH_KEYWORDS, poskw_varpos__doc__}, static PyObject * -vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, - int b); +poskw_varpos_impl(PyObject *module, PyObject *a, PyObject *args); static PyObject * -vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +poskw_varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 1 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2509,65 +2752,65 @@ vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"a", "b", NULL}; + static const char * const _keywords[] = {"a", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "vararg_with_default", + .fname = "poskw_varpos", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; - Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *argsbuf[1]; + PyObject * const *fastargs; PyObject *a; PyObject *__clinic_args = NULL; - int b = 0; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { goto exit; } - a = args[0]; - __clinic_args = args[1]; - if (!noptargs) { - goto skip_optional_kwonly; - } - b = PyObject_IsTrue(args[2]); - if (b < 0) { + a = fastargs[0]; + __clinic_args = nargs > 1 + ? _PyTuple_FromArray(args + 1, nargs - 1) + : PyTuple_New(0); + if (__clinic_args == NULL) { goto exit; } -skip_optional_kwonly: - return_value = vararg_with_default_impl(module, a, __clinic_args, b); + return_value = poskw_varpos_impl(module, a, __clinic_args); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } -PyDoc_STRVAR(vararg_with_only_defaults__doc__, -"vararg_with_only_defaults($module, /, *args, b=None)\n" +PyDoc_STRVAR(poskw_varpos_kwonly_opt__doc__, +"poskw_varpos_kwonly_opt($module, /, a, *args, b=False)\n" "--\n" "\n"); -#define VARARG_WITH_ONLY_DEFAULTS_METHODDEF \ - {"vararg_with_only_defaults", _PyCFunction_CAST(vararg_with_only_defaults), METH_FASTCALL|METH_KEYWORDS, vararg_with_only_defaults__doc__}, +#define POSKW_VARPOS_KWONLY_OPT_METHODDEF \ + {"poskw_varpos_kwonly_opt", _PyCFunction_CAST(poskw_varpos_kwonly_opt), METH_FASTCALL|METH_KEYWORDS, poskw_varpos_kwonly_opt__doc__}, static PyObject * -vararg_with_only_defaults_impl(PyObject *module, PyObject *args, PyObject *b); +poskw_varpos_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *args, + int b); static PyObject * -vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +poskw_varpos_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 2 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2576,62 +2819,75 @@ vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize_t na # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"b", NULL}; + static const char * const _keywords[] = {"a", "b", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "vararg_with_only_defaults", + .fname = "poskw_varpos_kwonly_opt", .kwtuple = KWTUPLE, }; #undef KWTUPLE PyObject *argsbuf[2]; - Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject * const *fastargs; + Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; PyObject *__clinic_args = NULL; - PyObject *b = Py_None; + int b = 0; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { goto exit; } - __clinic_args = args[0]; + a = fastargs[0]; if (!noptargs) { goto skip_optional_kwonly; } - b = args[1]; + b = PyObject_IsTrue(fastargs[1]); + if (b < 0) { + goto exit; + } skip_optional_kwonly: - return_value = vararg_with_only_defaults_impl(module, __clinic_args, b); + __clinic_args = nargs > 1 + ? _PyTuple_FromArray(args + 1, nargs - 1) + : PyTuple_New(0); + if (__clinic_args == NULL) { + goto exit; + } + return_value = poskw_varpos_kwonly_opt_impl(module, a, __clinic_args, b); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } -PyDoc_STRVAR(gh_32092_oob__doc__, -"gh_32092_oob($module, /, pos1, pos2, *varargs, kw1=None, kw2=None)\n" +PyDoc_STRVAR(poskw_varpos_kwonly_opt2__doc__, +"poskw_varpos_kwonly_opt2($module, /, a, *args, b=False, c=False)\n" "--\n" -"\n" -"Proof-of-concept of GH-32092 OOB bug."); +"\n"); -#define GH_32092_OOB_METHODDEF \ - {"gh_32092_oob", _PyCFunction_CAST(gh_32092_oob), METH_FASTCALL|METH_KEYWORDS, gh_32092_oob__doc__}, +#define POSKW_VARPOS_KWONLY_OPT2_METHODDEF \ + {"poskw_varpos_kwonly_opt2", _PyCFunction_CAST(poskw_varpos_kwonly_opt2), METH_FASTCALL|METH_KEYWORDS, poskw_varpos_kwonly_opt2__doc__}, static PyObject * -gh_32092_oob_impl(PyObject *module, PyObject *pos1, PyObject *pos2, - PyObject *varargs, PyObject *kw1, PyObject *kw2); +poskw_varpos_kwonly_opt2_impl(PyObject *module, PyObject *a, PyObject *args, + PyObject *b, PyObject *c); static PyObject * -gh_32092_oob(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +poskw_varpos_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 4 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(pos1), &_Py_ID(pos2), &_Py_ID(kw1), &_Py_ID(kw2), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2640,73 +2896,78 @@ gh_32092_oob(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"pos1", "pos2", "kw1", "kw2", NULL}; + static const char * const _keywords[] = {"a", "b", "c", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "gh_32092_oob", + .fname = "poskw_varpos_kwonly_opt2", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[5]; - Py_ssize_t noptargs = Py_MIN(nargs, 2) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - PyObject *pos1; - PyObject *pos2; - PyObject *varargs = NULL; - PyObject *kw1 = Py_None; - PyObject *kw2 = Py_None; + PyObject *argsbuf[3]; + PyObject * const *fastargs; + Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *__clinic_args = NULL; + PyObject *b = Py_False; + PyObject *c = Py_False; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, 2, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { goto exit; } - pos1 = args[0]; - pos2 = args[1]; - varargs = args[2]; + a = fastargs[0]; if (!noptargs) { goto skip_optional_kwonly; } - if (args[3]) { - kw1 = args[3]; + if (fastargs[1]) { + b = fastargs[1]; if (!--noptargs) { goto skip_optional_kwonly; } } - kw2 = args[4]; + c = fastargs[2]; skip_optional_kwonly: - return_value = gh_32092_oob_impl(module, pos1, pos2, varargs, kw1, kw2); + __clinic_args = nargs > 1 + ? _PyTuple_FromArray(args + 1, nargs - 1) + : PyTuple_New(0); + if (__clinic_args == NULL) { + goto exit; + } + return_value = poskw_varpos_kwonly_opt2_impl(module, a, __clinic_args, b, c); exit: - Py_XDECREF(varargs); + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + return return_value; } -PyDoc_STRVAR(gh_32092_kw_pass__doc__, -"gh_32092_kw_pass($module, /, pos, *args, kw=None)\n" +PyDoc_STRVAR(varpos_kwonly_opt__doc__, +"varpos_kwonly_opt($module, /, *args, b=False)\n" "--\n" -"\n" -"Proof-of-concept of GH-32092 keyword args passing bug."); +"\n"); -#define GH_32092_KW_PASS_METHODDEF \ - {"gh_32092_kw_pass", _PyCFunction_CAST(gh_32092_kw_pass), METH_FASTCALL|METH_KEYWORDS, gh_32092_kw_pass__doc__}, +#define VARPOS_KWONLY_OPT_METHODDEF \ + {"varpos_kwonly_opt", _PyCFunction_CAST(varpos_kwonly_opt), METH_FASTCALL|METH_KEYWORDS, varpos_kwonly_opt__doc__}, static PyObject * -gh_32092_kw_pass_impl(PyObject *module, PyObject *pos, PyObject *args, - PyObject *kw); +varpos_kwonly_opt_impl(PyObject *module, PyObject *args, PyObject *b); static PyObject * -gh_32092_kw_pass(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +varpos_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 1 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(pos), &_Py_ID(kw), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2715,188 +2976,235 @@ gh_32092_kw_pass(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"pos", "kw", NULL}; + static const char * const _keywords[] = {"b", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "gh_32092_kw_pass", + .fname = "varpos_kwonly_opt", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; - Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - PyObject *pos; + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *__clinic_args = NULL; - PyObject *kw = Py_None; + PyObject *b = Py_False; - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { goto exit; } - pos = args[0]; - __clinic_args = args[1]; if (!noptargs) { goto skip_optional_kwonly; } - kw = args[2]; + b = fastargs[0]; skip_optional_kwonly: - return_value = gh_32092_kw_pass_impl(module, pos, __clinic_args, kw); + __clinic_args = _PyTuple_FromArray(args, nargs); + if (__clinic_args == NULL) { + goto exit; + } + return_value = varpos_kwonly_opt_impl(module, __clinic_args, b); exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } -PyDoc_STRVAR(gh_99233_refcount__doc__, -"gh_99233_refcount($module, /, *args)\n" +PyDoc_STRVAR(varpos_kwonly_req_opt__doc__, +"varpos_kwonly_req_opt($module, /, *args, a, b=False, c=False)\n" "--\n" -"\n" -"Proof-of-concept of GH-99233 refcount error bug."); +"\n"); -#define GH_99233_REFCOUNT_METHODDEF \ - {"gh_99233_refcount", _PyCFunction_CAST(gh_99233_refcount), METH_FASTCALL, gh_99233_refcount__doc__}, +#define VARPOS_KWONLY_REQ_OPT_METHODDEF \ + {"varpos_kwonly_req_opt", _PyCFunction_CAST(varpos_kwonly_req_opt), METH_FASTCALL|METH_KEYWORDS, varpos_kwonly_req_opt__doc__}, static PyObject * -gh_99233_refcount_impl(PyObject *module, PyObject *args); +varpos_kwonly_req_opt_impl(PyObject *module, PyObject *args, PyObject *a, + PyObject *b, PyObject *c); static PyObject * -gh_99233_refcount(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +varpos_kwonly_req_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "varpos_kwonly_req_opt", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject * const *fastargs; + Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *__clinic_args = NULL; + PyObject *a; + PyObject *b = Py_False; + PyObject *c = Py_False; - if (!_PyArg_CheckPositional("gh_99233_refcount", nargs, 0, PY_SSIZE_T_MAX)) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 1, /*varpos*/ 1, argsbuf); + if (!fastargs) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; + a = fastargs[0]; + if (!noptargs) { + goto skip_optional_kwonly; } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); + if (fastargs[1]) { + b = fastargs[1]; + if (!--noptargs) { + goto skip_optional_kwonly; + } } - return_value = gh_99233_refcount_impl(module, __clinic_args); - -exit: + c = fastargs[2]; +skip_optional_kwonly: + __clinic_args = _PyTuple_FromArray(args, nargs); + if (__clinic_args == NULL) { + goto exit; + } + return_value = varpos_kwonly_req_opt_impl(module, __clinic_args, a, b, c); + +exit: + /* Cleanup for args */ Py_XDECREF(__clinic_args); + return return_value; } -PyDoc_STRVAR(gh_99240_double_free__doc__, -"gh_99240_double_free($module, a, b, /)\n" +PyDoc_STRVAR(varpos_array__doc__, +"varpos_array($module, /, *args)\n" "--\n" -"\n" -"Proof-of-concept of GH-99240 double-free bug."); +"\n"); -#define GH_99240_DOUBLE_FREE_METHODDEF \ - {"gh_99240_double_free", _PyCFunction_CAST(gh_99240_double_free), METH_FASTCALL, gh_99240_double_free__doc__}, +#define VARPOS_ARRAY_METHODDEF \ + {"varpos_array", _PyCFunction_CAST(varpos_array), METH_FASTCALL, varpos_array__doc__}, static PyObject * -gh_99240_double_free_impl(PyObject *module, char *a, char *b); +varpos_array_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length); static PyObject * -gh_99240_double_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +varpos_array(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - char *a = NULL; - char *b = NULL; + PyObject * const *__clinic_args; + Py_ssize_t args_length; - if (!_PyArg_ParseStack(args, nargs, "eses:gh_99240_double_free", - "idna", &a, "idna", &b)) { - goto exit; - } - return_value = gh_99240_double_free_impl(module, a, b); - /* Post parse cleanup for a */ - PyMem_FREE(a); - /* Post parse cleanup for b */ - PyMem_FREE(b); + __clinic_args = args; + args_length = nargs; + return_value = varpos_array_impl(module, __clinic_args, args_length); -exit: return return_value; } -PyDoc_STRVAR(null_or_tuple_for_varargs__doc__, -"null_or_tuple_for_varargs($module, /, name, *constraints,\n" -" covariant=False)\n" +PyDoc_STRVAR(posonly_varpos_array__doc__, +"posonly_varpos_array($module, a, b, /, *args)\n" "--\n" -"\n" -"See https://github.com/python/cpython/issues/110864"); +"\n"); -#define NULL_OR_TUPLE_FOR_VARARGS_METHODDEF \ - {"null_or_tuple_for_varargs", _PyCFunction_CAST(null_or_tuple_for_varargs), METH_FASTCALL|METH_KEYWORDS, null_or_tuple_for_varargs__doc__}, +#define POSONLY_VARPOS_ARRAY_METHODDEF \ + {"posonly_varpos_array", _PyCFunction_CAST(posonly_varpos_array), METH_FASTCALL, posonly_varpos_array__doc__}, static PyObject * -null_or_tuple_for_varargs_impl(PyObject *module, PyObject *name, - PyObject *constraints, int covariant); +posonly_varpos_array_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject * const *args, Py_ssize_t args_length); static PyObject * -null_or_tuple_for_varargs(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +posonly_varpos_array(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + PyObject *a; + PyObject *b; + PyObject * const *__clinic_args; + Py_ssize_t args_length; - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(covariant), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) + if (!_PyArg_CheckPositional("posonly_varpos_array", nargs, 2, PY_SSIZE_T_MAX)) { + goto exit; + } + a = args[0]; + b = args[1]; + __clinic_args = args + 2; + args_length = nargs - 2; + return_value = posonly_varpos_array_impl(module, a, b, __clinic_args, args_length); - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE +exit: + return return_value; +} - static const char * const _keywords[] = {"name", "covariant", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "null_or_tuple_for_varargs", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[3]; - Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - PyObject *name; - PyObject *constraints = NULL; - int covariant = 0; +PyDoc_STRVAR(posonly_req_opt_varpos_array__doc__, +"posonly_req_opt_varpos_array($module, a, b=False, /, *args)\n" +"--\n" +"\n"); - args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); - if (!args) { +#define POSONLY_REQ_OPT_VARPOS_ARRAY_METHODDEF \ + {"posonly_req_opt_varpos_array", _PyCFunction_CAST(posonly_req_opt_varpos_array), METH_FASTCALL, posonly_req_opt_varpos_array__doc__}, + +static PyObject * +posonly_req_opt_varpos_array_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject * const *args, + Py_ssize_t args_length); + +static PyObject * +posonly_req_opt_varpos_array(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *a; + PyObject *b = Py_False; + PyObject * const *__clinic_args; + Py_ssize_t args_length; + + if (!_PyArg_CheckPositional("posonly_req_opt_varpos_array", nargs, 1, PY_SSIZE_T_MAX)) { goto exit; } - name = args[0]; - constraints = args[1]; - if (!noptargs) { - goto skip_optional_kwonly; - } - covariant = PyObject_IsTrue(args[2]); - if (covariant < 0) { - goto exit; + a = args[0]; + if (nargs < 2) { + goto skip_optional; } -skip_optional_kwonly: - return_value = null_or_tuple_for_varargs_impl(module, name, constraints, covariant); + b = args[1]; +skip_optional: + __clinic_args = nargs > 2 ? args + 2 : args; + args_length = Py_MAX(0, nargs - 2); + return_value = posonly_req_opt_varpos_array_impl(module, a, b, __clinic_args, args_length); exit: - Py_XDECREF(constraints); return return_value; } -PyDoc_STRVAR(clone_f1__doc__, -"clone_f1($module, /, path)\n" +PyDoc_STRVAR(posonly_poskw_varpos_array__doc__, +"posonly_poskw_varpos_array($module, a, /, b, *args)\n" "--\n" "\n"); -#define CLONE_F1_METHODDEF \ - {"clone_f1", _PyCFunction_CAST(clone_f1), METH_FASTCALL|METH_KEYWORDS, clone_f1__doc__}, +#define POSONLY_POSKW_VARPOS_ARRAY_METHODDEF \ + {"posonly_poskw_varpos_array", _PyCFunction_CAST(posonly_poskw_varpos_array), METH_FASTCALL|METH_KEYWORDS, posonly_poskw_varpos_array__doc__}, static PyObject * -clone_f1_impl(PyObject *module, const char *path); +posonly_poskw_varpos_array_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject * const *args, + Py_ssize_t args_length); static PyObject * -clone_f1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +posonly_poskw_varpos_array(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -2908,7 +3216,7 @@ clone_f1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(path), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2917,64 +3225,62 @@ clone_f1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"path", NULL}; + static const char * const _keywords[] = {"", "b", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "clone_f1", + .fname = "posonly_poskw_varpos_array", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[1]; - const char *path; + PyObject *argsbuf[2]; + PyObject * const *fastargs; + PyObject *a; + PyObject *b; + PyObject * const *__clinic_args; + Py_ssize_t args_length; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { - goto exit; - } - if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("clone_f1", "argument 'path'", "str", args[0]); - goto exit; - } - Py_ssize_t path_length; - path = PyUnicode_AsUTF8AndSize(args[0], &path_length); - if (path == NULL) { - goto exit; - } - if (strlen(path) != (size_t)path_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { goto exit; } - return_value = clone_f1_impl(module, path); + a = fastargs[0]; + b = fastargs[1]; + __clinic_args = nargs > 2 ? args + 2 : args; + args_length = Py_MAX(0, nargs - 2); + return_value = posonly_poskw_varpos_array_impl(module, a, b, __clinic_args, args_length); exit: return return_value; } -PyDoc_STRVAR(clone_f2__doc__, -"clone_f2($module, /, path)\n" +PyDoc_STRVAR(gh_32092_oob__doc__, +"gh_32092_oob($module, /, pos1, pos2, *varargs, kw1=None, kw2=None)\n" "--\n" -"\n"); +"\n" +"Proof-of-concept of GH-32092 OOB bug."); -#define CLONE_F2_METHODDEF \ - {"clone_f2", _PyCFunction_CAST(clone_f2), METH_FASTCALL|METH_KEYWORDS, clone_f2__doc__}, +#define GH_32092_OOB_METHODDEF \ + {"gh_32092_oob", _PyCFunction_CAST(gh_32092_oob), METH_FASTCALL|METH_KEYWORDS, gh_32092_oob__doc__}, static PyObject * -clone_f2_impl(PyObject *module, const char *path); +gh_32092_oob_impl(PyObject *module, PyObject *pos1, PyObject *pos2, + PyObject *varargs, PyObject *kw1, PyObject *kw2); static PyObject * -clone_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +gh_32092_oob(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 4 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(path), }, + .ob_item = { &_Py_ID(pos1), &_Py_ID(pos2), &_Py_ID(kw1), &_Py_ID(kw2), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2983,64 +3289,82 @@ clone_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"path", NULL}; + static const char * const _keywords[] = {"pos1", "pos2", "kw1", "kw2", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "clone_f2", + .fname = "gh_32092_oob", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[1]; - const char *path; + PyObject *argsbuf[4]; + PyObject * const *fastargs; + Py_ssize_t noptargs = Py_MIN(nargs, 2) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *pos1; + PyObject *pos2; + PyObject *varargs = NULL; + PyObject *kw1 = Py_None; + PyObject *kw2 = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { goto exit; } - if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("clone_f2", "argument 'path'", "str", args[0]); - goto exit; + pos1 = fastargs[0]; + pos2 = fastargs[1]; + if (!noptargs) { + goto skip_optional_kwonly; } - Py_ssize_t path_length; - path = PyUnicode_AsUTF8AndSize(args[0], &path_length); - if (path == NULL) { - goto exit; + if (fastargs[2]) { + kw1 = fastargs[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } } - if (strlen(path) != (size_t)path_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); + kw2 = fastargs[3]; +skip_optional_kwonly: + varargs = nargs > 2 + ? _PyTuple_FromArray(args + 2, nargs - 2) + : PyTuple_New(0); + if (varargs == NULL) { goto exit; } - return_value = clone_f2_impl(module, path); + return_value = gh_32092_oob_impl(module, pos1, pos2, varargs, kw1, kw2); exit: + /* Cleanup for varargs */ + Py_XDECREF(varargs); + return return_value; } -PyDoc_STRVAR(clone_with_conv_f1__doc__, -"clone_with_conv_f1($module, /, path=None)\n" +PyDoc_STRVAR(gh_32092_kw_pass__doc__, +"gh_32092_kw_pass($module, /, pos, *args, kw=None)\n" "--\n" -"\n"); +"\n" +"Proof-of-concept of GH-32092 keyword args passing bug."); -#define CLONE_WITH_CONV_F1_METHODDEF \ - {"clone_with_conv_f1", _PyCFunction_CAST(clone_with_conv_f1), METH_FASTCALL|METH_KEYWORDS, clone_with_conv_f1__doc__}, +#define GH_32092_KW_PASS_METHODDEF \ + {"gh_32092_kw_pass", _PyCFunction_CAST(gh_32092_kw_pass), METH_FASTCALL|METH_KEYWORDS, gh_32092_kw_pass__doc__}, static PyObject * -clone_with_conv_f1_impl(PyObject *module, custom_t path); +gh_32092_kw_pass_impl(PyObject *module, PyObject *pos, PyObject *args, + PyObject *kw); static PyObject * -clone_with_conv_f1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +gh_32092_kw_pass(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 2 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(path), }, + .ob_item = { &_Py_ID(pos), &_Py_ID(kw), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -3049,64 +3373,217 @@ clone_with_conv_f1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"path", NULL}; + static const char * const _keywords[] = {"pos", "kw", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "clone_with_conv_f1", + .fname = "gh_32092_kw_pass", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[1]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - custom_t path = { - .name = "clone_with_conv_f1", - }; + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *pos; + PyObject *__clinic_args = NULL; + PyObject *kw = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); - if (!args) { + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { goto exit; } + pos = fastargs[0]; if (!noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; } - if (!custom_converter(args[0], &path)) { + kw = fastargs[1]; +skip_optional_kwonly: + __clinic_args = nargs > 1 + ? _PyTuple_FromArray(args + 1, nargs - 1) + : PyTuple_New(0); + if (__clinic_args == NULL) { goto exit; } -skip_optional_pos: - return_value = clone_with_conv_f1_impl(module, path); + return_value = gh_32092_kw_pass_impl(module, pos, __clinic_args, kw); exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + return return_value; } -PyDoc_STRVAR(clone_with_conv_f2__doc__, -"clone_with_conv_f2($module, /, path=None)\n" +PyDoc_STRVAR(gh_99233_refcount__doc__, +"gh_99233_refcount($module, /, *args)\n" "--\n" -"\n"); +"\n" +"Proof-of-concept of GH-99233 refcount error bug."); -#define CLONE_WITH_CONV_F2_METHODDEF \ - {"clone_with_conv_f2", _PyCFunction_CAST(clone_with_conv_f2), METH_FASTCALL|METH_KEYWORDS, clone_with_conv_f2__doc__}, +#define GH_99233_REFCOUNT_METHODDEF \ + {"gh_99233_refcount", _PyCFunction_CAST(gh_99233_refcount), METH_FASTCALL, gh_99233_refcount__doc__}, static PyObject * -clone_with_conv_f2_impl(PyObject *module, custom_t path); +gh_99233_refcount_impl(PyObject *module, PyObject *args); static PyObject * -clone_with_conv_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +gh_99233_refcount(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + PyObject *__clinic_args = NULL; - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(path), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) + __clinic_args = _PyTuple_FromArray(args, nargs); + if (__clinic_args == NULL) { + goto exit; + } + return_value = gh_99233_refcount_impl(module, __clinic_args); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + +PyDoc_STRVAR(gh_99240_double_free__doc__, +"gh_99240_double_free($module, a, b, /)\n" +"--\n" +"\n" +"Proof-of-concept of GH-99240 double-free bug."); + +#define GH_99240_DOUBLE_FREE_METHODDEF \ + {"gh_99240_double_free", _PyCFunction_CAST(gh_99240_double_free), METH_FASTCALL, gh_99240_double_free__doc__}, + +static PyObject * +gh_99240_double_free_impl(PyObject *module, char *a, char *b); + +static PyObject * +gh_99240_double_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + char *a = NULL; + char *b = NULL; + + if (!_PyArg_ParseStack(args, nargs, "eses:gh_99240_double_free", + "idna", &a, "idna", &b)) { + goto exit; + } + return_value = gh_99240_double_free_impl(module, a, b); + /* Post parse cleanup for a */ + PyMem_FREE(a); + /* Post parse cleanup for b */ + PyMem_FREE(b); + +exit: + return return_value; +} + +PyDoc_STRVAR(null_or_tuple_for_varargs__doc__, +"null_or_tuple_for_varargs($module, /, name, *constraints,\n" +" covariant=False)\n" +"--\n" +"\n" +"See https://github.com/python/cpython/issues/110864"); + +#define NULL_OR_TUPLE_FOR_VARARGS_METHODDEF \ + {"null_or_tuple_for_varargs", _PyCFunction_CAST(null_or_tuple_for_varargs), METH_FASTCALL|METH_KEYWORDS, null_or_tuple_for_varargs__doc__}, + +static PyObject * +null_or_tuple_for_varargs_impl(PyObject *module, PyObject *name, + PyObject *constraints, int covariant); + +static PyObject * +null_or_tuple_for_varargs(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(name), &_Py_ID(covariant), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"name", "covariant", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "null_or_tuple_for_varargs", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *name; + PyObject *constraints = NULL; + int covariant = 0; + + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { + goto exit; + } + name = fastargs[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + covariant = PyObject_IsTrue(fastargs[1]); + if (covariant < 0) { + goto exit; + } +skip_optional_kwonly: + constraints = nargs > 1 + ? _PyTuple_FromArray(args + 1, nargs - 1) + : PyTuple_New(0); + if (constraints == NULL) { + goto exit; + } + return_value = null_or_tuple_for_varargs_impl(module, name, constraints, covariant); + +exit: + /* Cleanup for constraints */ + Py_XDECREF(constraints); + + return return_value; +} + +PyDoc_STRVAR(clone_f1__doc__, +"clone_f1($module, /, path)\n" +"--\n" +"\n"); + +#define CLONE_F1_METHODDEF \ + {"clone_f1", _PyCFunction_CAST(clone_f1), METH_FASTCALL|METH_KEYWORDS, clone_f1__doc__}, + +static PyObject * +clone_f1_impl(PyObject *module, const char *path); + +static PyObject * +clone_f1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) #else // !Py_BUILD_CORE # define KWTUPLE NULL @@ -3115,52 +3592,725 @@ clone_with_conv_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py static const char * const _keywords[] = {"path", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "clone_with_conv_f2", + .fname = "clone_f1", .kwtuple = KWTUPLE, }; #undef KWTUPLE PyObject *argsbuf[1]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - custom_t path = { - .name = "clone_with_conv_f2", - }; + const char *path; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } - if (!noptargs) { - goto skip_optional_pos; + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("clone_f1", "argument 'path'", "str", args[0]); + goto exit; } - if (!custom_converter(args[0], &path)) { + Py_ssize_t path_length; + path = PyUnicode_AsUTF8AndSize(args[0], &path_length); + if (path == NULL) { goto exit; } -skip_optional_pos: - return_value = clone_with_conv_f2_impl(module, path); + if (strlen(path) != (size_t)path_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + return_value = clone_f1_impl(module, path); exit: return return_value; } -PyDoc_STRVAR(_testclinic_TestClass_meth_method_no_params__doc__, -"meth_method_no_params($self, /)\n" +PyDoc_STRVAR(clone_f2__doc__, +"clone_f2($module, /, path)\n" "--\n" "\n"); -#define _TESTCLINIC_TESTCLASS_METH_METHOD_NO_PARAMS_METHODDEF \ - {"meth_method_no_params", _PyCFunction_CAST(_testclinic_TestClass_meth_method_no_params), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _testclinic_TestClass_meth_method_no_params__doc__}, +#define CLONE_F2_METHODDEF \ + {"clone_f2", _PyCFunction_CAST(clone_f2), METH_FASTCALL|METH_KEYWORDS, clone_f2__doc__}, static PyObject * -_testclinic_TestClass_meth_method_no_params_impl(PyObject *self, - PyTypeObject *cls); +clone_f2_impl(PyObject *module, const char *path); static PyObject * -_testclinic_TestClass_meth_method_no_params(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +clone_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { - PyErr_SetString(PyExc_TypeError, "meth_method_no_params() takes no arguments"); - return NULL; + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "clone_f2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + const char *path; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("clone_f2", "argument 'path'", "str", args[0]); + goto exit; + } + Py_ssize_t path_length; + path = PyUnicode_AsUTF8AndSize(args[0], &path_length); + if (path == NULL) { + goto exit; + } + if (strlen(path) != (size_t)path_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; } - return _testclinic_TestClass_meth_method_no_params_impl(self, cls); + return_value = clone_f2_impl(module, path); + +exit: + return return_value; +} + +PyDoc_STRVAR(clone_with_conv_f1__doc__, +"clone_with_conv_f1($module, /, path=None)\n" +"--\n" +"\n"); + +#define CLONE_WITH_CONV_F1_METHODDEF \ + {"clone_with_conv_f1", _PyCFunction_CAST(clone_with_conv_f1), METH_FASTCALL|METH_KEYWORDS, clone_with_conv_f1__doc__}, + +static PyObject * +clone_with_conv_f1_impl(PyObject *module, custom_t path); + +static PyObject * +clone_with_conv_f1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "clone_with_conv_f1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + custom_t path = { + .name = "clone_with_conv_f1", + }; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (!custom_converter(args[0], &path)) { + goto exit; + } +skip_optional_pos: + return_value = clone_with_conv_f1_impl(module, path); + +exit: + return return_value; +} + +PyDoc_STRVAR(clone_with_conv_f2__doc__, +"clone_with_conv_f2($module, /, path=None)\n" +"--\n" +"\n"); + +#define CLONE_WITH_CONV_F2_METHODDEF \ + {"clone_with_conv_f2", _PyCFunction_CAST(clone_with_conv_f2), METH_FASTCALL|METH_KEYWORDS, clone_with_conv_f2__doc__}, + +static PyObject * +clone_with_conv_f2_impl(PyObject *module, custom_t path); + +static PyObject * +clone_with_conv_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "clone_with_conv_f2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + custom_t path = { + .name = "clone_with_conv_f2", + }; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (!custom_converter(args[0], &path)) { + goto exit; + } +skip_optional_pos: + return_value = clone_with_conv_f2_impl(module, path); + +exit: + return return_value; +} + +PyDoc_STRVAR(_testclinic_TestClass_get_defining_class__doc__, +"get_defining_class($self, /)\n" +"--\n" +"\n"); + +#define _TESTCLINIC_TESTCLASS_GET_DEFINING_CLASS_METHODDEF \ + {"get_defining_class", _PyCFunction_CAST(_testclinic_TestClass_get_defining_class), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _testclinic_TestClass_get_defining_class__doc__}, + +static PyObject * +_testclinic_TestClass_get_defining_class_impl(PyObject *self, + PyTypeObject *cls); + +static PyObject * +_testclinic_TestClass_get_defining_class(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "get_defining_class() takes no arguments"); + return NULL; + } + return _testclinic_TestClass_get_defining_class_impl(self, cls); +} + +PyDoc_STRVAR(_testclinic_TestClass_get_defining_class_arg__doc__, +"get_defining_class_arg($self, /, arg)\n" +"--\n" +"\n"); + +#define _TESTCLINIC_TESTCLASS_GET_DEFINING_CLASS_ARG_METHODDEF \ + {"get_defining_class_arg", _PyCFunction_CAST(_testclinic_TestClass_get_defining_class_arg), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _testclinic_TestClass_get_defining_class_arg__doc__}, + +static PyObject * +_testclinic_TestClass_get_defining_class_arg_impl(PyObject *self, + PyTypeObject *cls, + PyObject *arg); + +static PyObject * +_testclinic_TestClass_get_defining_class_arg(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(arg), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"arg", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "get_defining_class_arg", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *arg; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + arg = args[0]; + return_value = _testclinic_TestClass_get_defining_class_arg_impl(self, cls, arg); + +exit: + return return_value; +} + +PyDoc_STRVAR(_testclinic_TestClass_defclass_varpos__doc__, +"defclass_varpos($self, /, *args)\n" +"--\n" +"\n"); + +#define _TESTCLINIC_TESTCLASS_DEFCLASS_VARPOS_METHODDEF \ + {"defclass_varpos", _PyCFunction_CAST(_testclinic_TestClass_defclass_varpos), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _testclinic_TestClass_defclass_varpos__doc__}, + +static PyObject * +_testclinic_TestClass_defclass_varpos_impl(PyObject *self, PyTypeObject *cls, + PyObject *args); + +static PyObject * +_testclinic_TestClass_defclass_varpos(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = { NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "defclass_varpos", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + PyObject *__clinic_args = NULL; + + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { + goto exit; + } + __clinic_args = _PyTuple_FromArray(args, nargs); + if (__clinic_args == NULL) { + goto exit; + } + return_value = _testclinic_TestClass_defclass_varpos_impl(self, cls, __clinic_args); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + +PyDoc_STRVAR(_testclinic_TestClass_defclass_posonly_varpos__doc__, +"defclass_posonly_varpos($self, a, b, /, *args)\n" +"--\n" +"\n"); + +#define _TESTCLINIC_TESTCLASS_DEFCLASS_POSONLY_VARPOS_METHODDEF \ + {"defclass_posonly_varpos", _PyCFunction_CAST(_testclinic_TestClass_defclass_posonly_varpos), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _testclinic_TestClass_defclass_posonly_varpos__doc__}, + +static PyObject * +_testclinic_TestClass_defclass_posonly_varpos_impl(PyObject *self, + PyTypeObject *cls, + PyObject *a, PyObject *b, + PyObject *args); + +static PyObject * +_testclinic_TestClass_defclass_posonly_varpos(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "defclass_posonly_varpos", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + PyObject *a; + PyObject *b; + PyObject *__clinic_args = NULL; + + fastargs = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + b = fastargs[1]; + __clinic_args = _PyTuple_FromArray(args + 2, nargs - 2); + if (__clinic_args == NULL) { + goto exit; + } + return_value = _testclinic_TestClass_defclass_posonly_varpos_impl(self, cls, a, b, __clinic_args); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + +static PyObject * +varpos_no_fastcall_impl(PyTypeObject *type, PyObject *args); + +static PyObject * +varpos_no_fastcall(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = &PyBaseObject_Type; + PyObject *__clinic_args = NULL; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("TestClass", kwargs)) { + goto exit; + } + __clinic_args = Py_NewRef(args); + return_value = varpos_no_fastcall_impl(type, __clinic_args); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + +static PyObject * +posonly_varpos_no_fastcall_impl(PyTypeObject *type, PyObject *a, PyObject *b, + PyObject *args); + +static PyObject * +posonly_varpos_no_fastcall(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = &PyBaseObject_Type; + PyObject *a; + PyObject *b; + PyObject *__clinic_args = NULL; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("TestClass", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("TestClass", PyTuple_GET_SIZE(args), 2, PY_SSIZE_T_MAX)) { + goto exit; + } + a = PyTuple_GET_ITEM(args, 0); + b = PyTuple_GET_ITEM(args, 1); + __clinic_args = PyTuple_GetSlice(args, 2, PY_SSIZE_T_MAX); + if (!__clinic_args) { + goto exit; + } + return_value = posonly_varpos_no_fastcall_impl(type, a, b, __clinic_args); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + +static PyObject * +posonly_req_opt_varpos_no_fastcall_impl(PyTypeObject *type, PyObject *a, + PyObject *b, PyObject *args); + +static PyObject * +posonly_req_opt_varpos_no_fastcall(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = &PyBaseObject_Type; + PyObject *a; + PyObject *b = Py_False; + PyObject *__clinic_args = NULL; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("TestClass", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("TestClass", PyTuple_GET_SIZE(args), 1, PY_SSIZE_T_MAX)) { + goto exit; + } + a = PyTuple_GET_ITEM(args, 0); + if (PyTuple_GET_SIZE(args) < 2) { + goto skip_optional; + } + b = PyTuple_GET_ITEM(args, 1); +skip_optional: + __clinic_args = PyTuple_GetSlice(args, 2, PY_SSIZE_T_MAX); + if (!__clinic_args) { + goto exit; + } + return_value = posonly_req_opt_varpos_no_fastcall_impl(type, a, b, __clinic_args); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + +static PyObject * +posonly_poskw_varpos_no_fastcall_impl(PyTypeObject *type, PyObject *a, + PyObject *b, PyObject *args); + +static PyObject * +posonly_poskw_varpos_no_fastcall(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { _Py_LATIN1_CHR('b'), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "TestClass", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *a; + PyObject *b; + PyObject *__clinic_args = NULL; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + b = fastargs[1]; + __clinic_args = PyTuple_GetSlice(args, 2, PY_SSIZE_T_MAX); + if (!__clinic_args) { + goto exit; + } + return_value = posonly_poskw_varpos_no_fastcall_impl(type, a, b, __clinic_args); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + +static PyObject * +varpos_array_no_fastcall_impl(PyTypeObject *type, PyObject * const *args, + Py_ssize_t args_length); + +static PyObject * +varpos_array_no_fastcall(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = &PyBaseObject_Type; + PyObject * const *__clinic_args; + Py_ssize_t args_length; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("TestClass", kwargs)) { + goto exit; + } + __clinic_args = _PyTuple_ITEMS(args); + args_length = PyTuple_GET_SIZE(args); + return_value = varpos_array_no_fastcall_impl(type, __clinic_args, args_length); + +exit: + return return_value; +} + +static PyObject * +posonly_varpos_array_no_fastcall_impl(PyTypeObject *type, PyObject *a, + PyObject *b, PyObject * const *args, + Py_ssize_t args_length); + +static PyObject * +posonly_varpos_array_no_fastcall(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = &PyBaseObject_Type; + PyObject *a; + PyObject *b; + PyObject * const *__clinic_args; + Py_ssize_t args_length; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("TestClass", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("TestClass", PyTuple_GET_SIZE(args), 2, PY_SSIZE_T_MAX)) { + goto exit; + } + a = PyTuple_GET_ITEM(args, 0); + b = PyTuple_GET_ITEM(args, 1); + __clinic_args = _PyTuple_ITEMS(args) + 2; + args_length = PyTuple_GET_SIZE(args) - 2; + return_value = posonly_varpos_array_no_fastcall_impl(type, a, b, __clinic_args, args_length); + +exit: + return return_value; +} + +static PyObject * +posonly_req_opt_varpos_array_no_fastcall_impl(PyTypeObject *type, + PyObject *a, PyObject *b, + PyObject * const *args, + Py_ssize_t args_length); + +static PyObject * +posonly_req_opt_varpos_array_no_fastcall(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = &PyBaseObject_Type; + PyObject *a; + PyObject *b = Py_False; + PyObject * const *__clinic_args; + Py_ssize_t args_length; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("TestClass", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("TestClass", PyTuple_GET_SIZE(args), 1, PY_SSIZE_T_MAX)) { + goto exit; + } + a = PyTuple_GET_ITEM(args, 0); + if (PyTuple_GET_SIZE(args) < 2) { + goto skip_optional; + } + b = PyTuple_GET_ITEM(args, 1); +skip_optional: + __clinic_args = PyTuple_GET_SIZE(args) > 2 ? _PyTuple_ITEMS(args) + 2 : _PyTuple_ITEMS(args); + args_length = Py_MAX(0, PyTuple_GET_SIZE(args) - 2); + return_value = posonly_req_opt_varpos_array_no_fastcall_impl(type, a, b, __clinic_args, args_length); + +exit: + return return_value; +} + +static PyObject * +posonly_poskw_varpos_array_no_fastcall_impl(PyTypeObject *type, PyObject *a, + PyObject *b, + PyObject * const *args, + Py_ssize_t args_length); + +static PyObject * +posonly_poskw_varpos_array_no_fastcall(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { _Py_LATIN1_CHR('b'), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "TestClass", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *a; + PyObject *b; + PyObject * const *__clinic_args; + Py_ssize_t args_length; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + b = fastargs[1]; + __clinic_args = PyTuple_GET_SIZE(args) > 2 ? _PyTuple_ITEMS(args) + 2 : _PyTuple_ITEMS(args); + args_length = Py_MAX(0, PyTuple_GET_SIZE(args) - 2); + return_value = posonly_poskw_varpos_array_no_fastcall_impl(type, a, b, __clinic_args, args_length); + +exit: + return return_value; } -/*[clinic end generated code: output=6520c1ca5392a3f0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=00085e5637450fe6 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_depr.c.h b/Modules/clinic/_testclinic_depr.c.h index 732c3810408399..9378038e6b9549 100644 --- a/Modules/clinic/_testclinic_depr.c.h +++ b/Modules/clinic/_testclinic_depr.c.h @@ -9,6 +9,7 @@ preserve #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_runtime.h" // _Py_ID() +#include "pycore_tuple.h" // _PyTuple_FromArray() PyDoc_STRVAR(depr_star_new__doc__, "DeprStarNew(a=None)\n" @@ -48,7 +49,7 @@ depr_star_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -79,7 +80,8 @@ depr_star_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto exit; } } - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -133,7 +135,7 @@ depr_star_new_clone(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyO PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -162,7 +164,8 @@ depr_star_new_clone(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyO goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -215,7 +218,7 @@ depr_star_init(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -246,7 +249,8 @@ depr_star_init(PyObject *self, PyObject *args, PyObject *kwargs) goto exit; } } - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -300,7 +304,7 @@ depr_star_init_clone(PyObject *self, PyObject *const *args, Py_ssize_t nargs, Py PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -329,7 +333,8 @@ depr_star_init_clone(PyObject *self, PyObject *const *args, Py_ssize_t nargs, Py goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -372,7 +377,7 @@ depr_star_init_noinline(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -451,7 +456,7 @@ depr_kwd_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -473,7 +478,8 @@ depr_kwd_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; PyObject *a = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -534,7 +540,7 @@ depr_kwd_init(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -556,7 +562,8 @@ depr_kwd_init(PyObject *self, PyObject *args, PyObject *kwargs) Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; PyObject *a = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -608,7 +615,7 @@ depr_kwd_init_noinline(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -635,7 +642,7 @@ depr_kwd_init_noinline(PyObject *self, PyObject *args, PyObject *kwargs) &a, &b, &c, &d, &d_length)) { goto exit; } - if (kwargs && PyDict_GET_SIZE(kwargs) && ((nargs < 2) || (nargs < 3 && PyDict_Contains(kwargs, &_Py_ID(c))))) { + if (kwargs && PyDict_GET_SIZE(kwargs) && ((nargs < 2) || (nargs < 3 && PyDict_Contains(kwargs, _Py_LATIN1_CHR('c'))))) { if (PyErr_Occurred()) { // PyDict_Contains() above can fail goto exit; } @@ -692,7 +699,7 @@ depr_star_pos0_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, + .ob_item = { _Py_LATIN1_CHR('a'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -720,7 +727,8 @@ depr_star_pos0_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -770,7 +778,7 @@ depr_star_pos0_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -799,7 +807,8 @@ depr_star_pos0_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -851,7 +860,7 @@ depr_star_pos0_len3_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -882,7 +891,8 @@ depr_star_pos0_len3_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -935,7 +945,7 @@ depr_star_pos1_len1_opt(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -965,7 +975,8 @@ depr_star_pos1_len1_opt(PyObject *module, PyObject *const *args, Py_ssize_t narg goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1020,7 +1031,7 @@ depr_star_pos1_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1049,7 +1060,8 @@ depr_star_pos1_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1101,7 +1113,7 @@ depr_star_pos1_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1132,7 +1144,8 @@ depr_star_pos1_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1186,7 +1199,7 @@ depr_star_pos2_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1216,7 +1229,8 @@ depr_star_pos2_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1269,7 +1283,7 @@ depr_star_pos2_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1300,7 +1314,8 @@ depr_star_pos2_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 4, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1354,7 +1369,7 @@ depr_star_pos2_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1386,7 +1401,8 @@ depr_star_pos2_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 4, /*maxpos*/ 4, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1441,7 +1457,7 @@ depr_star_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1525,7 +1541,7 @@ depr_star_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), &_Py_ID(g), &_Py_ID(h), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), _Py_LATIN1_CHR('f'), _Py_LATIN1_CHR('g'), _Py_LATIN1_CHR('h'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1562,7 +1578,8 @@ depr_star_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 7, 7, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 7, /*maxpos*/ 7, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1618,7 +1635,7 @@ depr_kwd_required_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1638,7 +1655,8 @@ depr_kwd_required_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *a; PyObject *b; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1699,7 +1717,7 @@ depr_kwd_required_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1720,7 +1738,8 @@ depr_kwd_required_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *b; PyObject *c; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1780,7 +1799,7 @@ depr_kwd_optional_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), }, + .ob_item = { _Py_LATIN1_CHR('b'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1801,7 +1820,8 @@ depr_kwd_optional_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *a; PyObject *b = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1866,7 +1886,7 @@ depr_kwd_optional_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1888,7 +1908,8 @@ depr_kwd_optional_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *b = Py_None; PyObject *c = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1959,7 +1980,7 @@ depr_kwd_optional_3(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1981,7 +2002,8 @@ depr_kwd_optional_3(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *b = Py_None; PyObject *c = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2057,7 +2079,7 @@ depr_kwd_required_optional(PyObject *module, PyObject *const *args, Py_ssize_t n PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2079,7 +2101,8 @@ depr_kwd_required_optional(PyObject *module, PyObject *const *args, Py_ssize_t n PyObject *b; PyObject *c = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2145,7 +2168,7 @@ depr_kwd_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2171,7 +2194,7 @@ depr_kwd_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO &a, &b, &c, &d, &d_length)) { goto exit; } - if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 2) || (nargs < 3 && PySequence_Contains(kwnames, &_Py_ID(c))))) { + if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 2) || (nargs < 3 && PySequence_Contains(kwnames, _Py_LATIN1_CHR('c'))))) { if (PyErr_Occurred()) { // PySequence_Contains() above can fail goto exit; } @@ -2232,7 +2255,7 @@ depr_kwd_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), &_Py_ID(g), &_Py_ID(h), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), _Py_LATIN1_CHR('f'), _Py_LATIN1_CHR('g'), _Py_LATIN1_CHR('h'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2258,7 +2281,8 @@ depr_kwd_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *g; PyObject *h; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 8, 8, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 8, /*maxpos*/ 8, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2333,7 +2357,7 @@ depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), &_Py_ID(g), }, + .ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), _Py_LATIN1_CHR('f'), _Py_LATIN1_CHR('g'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2368,7 +2392,8 @@ depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * goto exit; } } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 6, 6, 1, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 6, /*maxpos*/ 6, /*minkw*/ 1, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2393,4 +2418,4 @@ depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * exit: return return_value; } -/*[clinic end generated code: output=2c19d1804ba6e53b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=836affc1a13d67a1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_limited.c.h b/Modules/clinic/_testclinic_limited.c.h index eff72cee24975c..94897f4c6dc427 100644 --- a/Modules/clinic/_testclinic_limited.c.h +++ b/Modules/clinic/_testclinic_limited.c.h @@ -91,4 +91,119 @@ my_int_sum(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=5cf64baf978d2288 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(my_float_sum__doc__, +"my_float_sum($module, x, y, /)\n" +"--\n" +"\n"); + +#define MY_FLOAT_SUM_METHODDEF \ + {"my_float_sum", (PyCFunction)(void(*)(void))my_float_sum, METH_FASTCALL, my_float_sum__doc__}, + +static float +my_float_sum_impl(PyObject *module, float x, float y); + +static PyObject * +my_float_sum(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + float x; + float y; + float _return_value; + + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "my_float_sum expected 2 arguments, got %zd", nargs); + goto exit; + } + x = (float) PyFloat_AsDouble(args[0]); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + y = (float) PyFloat_AsDouble(args[1]); + if (y == -1.0 && PyErr_Occurred()) { + goto exit; + } + _return_value = my_float_sum_impl(module, x, y); + if ((_return_value == -1.0) && PyErr_Occurred()) { + goto exit; + } + return_value = PyFloat_FromDouble((double)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(my_double_sum__doc__, +"my_double_sum($module, x, y, /)\n" +"--\n" +"\n"); + +#define MY_DOUBLE_SUM_METHODDEF \ + {"my_double_sum", (PyCFunction)(void(*)(void))my_double_sum, METH_FASTCALL, my_double_sum__doc__}, + +static double +my_double_sum_impl(PyObject *module, double x, double y); + +static PyObject * +my_double_sum(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + double x; + double y; + double _return_value; + + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "my_double_sum expected 2 arguments, got %zd", nargs); + goto exit; + } + x = PyFloat_AsDouble(args[0]); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + y = PyFloat_AsDouble(args[1]); + if (y == -1.0 && PyErr_Occurred()) { + goto exit; + } + _return_value = my_double_sum_impl(module, x, y); + if ((_return_value == -1.0) && PyErr_Occurred()) { + goto exit; + } + return_value = PyFloat_FromDouble(_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(get_file_descriptor__doc__, +"get_file_descriptor($module, file, /)\n" +"--\n" +"\n" +"Get a file descriptor."); + +#define GET_FILE_DESCRIPTOR_METHODDEF \ + {"get_file_descriptor", (PyCFunction)get_file_descriptor, METH_O, get_file_descriptor__doc__}, + +static int +get_file_descriptor_impl(PyObject *module, int fd); + +static PyObject * +get_file_descriptor(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int fd; + int _return_value; + + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { + goto exit; + } + _return_value = get_file_descriptor_impl(module, fd); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); + +exit: + return return_value; +} +/*[clinic end generated code: output=03fd7811c056dc74 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index cba2a943d03456..d98d69df22f982 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -52,7 +52,8 @@ _testinternalcapi_compiler_cleandoc(PyObject *module, PyObject *const *args, Py_ PyObject *argsbuf[1]; PyObject *doc; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -67,6 +68,24 @@ _testinternalcapi_compiler_cleandoc(PyObject *module, PyObject *const *args, Py_ return return_value; } +PyDoc_STRVAR(_testinternalcapi_new_instruction_sequence__doc__, +"new_instruction_sequence($module, /)\n" +"--\n" +"\n" +"Return a new, empty InstructionSequence."); + +#define _TESTINTERNALCAPI_NEW_INSTRUCTION_SEQUENCE_METHODDEF \ + {"new_instruction_sequence", (PyCFunction)_testinternalcapi_new_instruction_sequence, METH_NOARGS, _testinternalcapi_new_instruction_sequence__doc__}, + +static PyObject * +_testinternalcapi_new_instruction_sequence_impl(PyObject *module); + +static PyObject * +_testinternalcapi_new_instruction_sequence(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testinternalcapi_new_instruction_sequence_impl(module); +} + PyDoc_STRVAR(_testinternalcapi_compiler_codegen__doc__, "compiler_codegen($module, /, ast, filename, optimize, compile_mode=0)\n" "--\n" @@ -117,7 +136,8 @@ _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_s int optimize; int compile_mode = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -188,7 +208,8 @@ _testinternalcapi_optimize_cfg(PyObject *module, PyObject *const *args, Py_ssize PyObject *consts; int nlocals; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -253,7 +274,8 @@ _testinternalcapi_assemble_code_object(PyObject *module, PyObject *const *args, PyObject *instructions; PyObject *metadata; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -282,4 +304,65 @@ _testinternalcapi_test_long_numbits(PyObject *module, PyObject *Py_UNUSED(ignore { return _testinternalcapi_test_long_numbits_impl(module); } -/*[clinic end generated code: output=679bf53bbae20085 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(gh_119213_getargs__doc__, +"gh_119213_getargs($module, /, spam=None)\n" +"--\n" +"\n" +"Test _PyArg_Parser.kwtuple"); + +#define GH_119213_GETARGS_METHODDEF \ + {"gh_119213_getargs", _PyCFunction_CAST(gh_119213_getargs), METH_FASTCALL|METH_KEYWORDS, gh_119213_getargs__doc__}, + +static PyObject * +gh_119213_getargs_impl(PyObject *module, PyObject *spam); + +static PyObject * +gh_119213_getargs(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(spam), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"spam", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "gh_119213_getargs", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *spam = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + spam = args[0]; +skip_optional_pos: + return_value = gh_119213_getargs_impl(module, spam); + +exit: + return return_value; +} +/*[clinic end generated code: output=ec77971c6c2663da input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testmultiphase.c.h b/Modules/clinic/_testmultiphase.c.h index 7ac361ece1acc3..01c29c0753ae13 100644 --- a/Modules/clinic/_testmultiphase.c.h +++ b/Modules/clinic/_testmultiphase.c.h @@ -25,13 +25,13 @@ _testmultiphase_StateAccessType_get_defining_module_impl(StateAccessTypeObject * PyTypeObject *cls); static PyObject * -_testmultiphase_StateAccessType_get_defining_module(StateAccessTypeObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_testmultiphase_StateAccessType_get_defining_module(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "get_defining_module() takes no arguments"); return NULL; } - return _testmultiphase_StateAccessType_get_defining_module_impl(self, cls); + return _testmultiphase_StateAccessType_get_defining_module_impl((StateAccessTypeObject *)self, cls); } PyDoc_STRVAR(_testmultiphase_StateAccessType_getmodulebydef_bad_def__doc__, @@ -48,13 +48,13 @@ _testmultiphase_StateAccessType_getmodulebydef_bad_def_impl(StateAccessTypeObjec PyTypeObject *cls); static PyObject * -_testmultiphase_StateAccessType_getmodulebydef_bad_def(StateAccessTypeObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_testmultiphase_StateAccessType_getmodulebydef_bad_def(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "getmodulebydef_bad_def() takes no arguments"); return NULL; } - return _testmultiphase_StateAccessType_getmodulebydef_bad_def_impl(self, cls); + return _testmultiphase_StateAccessType_getmodulebydef_bad_def_impl((StateAccessTypeObject *)self, cls); } PyDoc_STRVAR(_testmultiphase_StateAccessType_increment_count_clinic__doc__, @@ -76,7 +76,7 @@ _testmultiphase_StateAccessType_increment_count_clinic_impl(StateAccessTypeObjec int n, int twice); static PyObject * -_testmultiphase_StateAccessType_increment_count_clinic(StateAccessTypeObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_testmultiphase_StateAccessType_increment_count_clinic(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -88,7 +88,7 @@ _testmultiphase_StateAccessType_increment_count_clinic(StateAccessTypeObject *se PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(n), &_Py_ID(twice), }, + .ob_item = { _Py_LATIN1_CHR('n'), &_Py_ID(twice), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -109,7 +109,8 @@ _testmultiphase_StateAccessType_increment_count_clinic(StateAccessTypeObject *se int n = 1; int twice = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -134,7 +135,7 @@ _testmultiphase_StateAccessType_increment_count_clinic(StateAccessTypeObject *se goto exit; } skip_optional_kwonly: - return_value = _testmultiphase_StateAccessType_increment_count_clinic_impl(self, cls, n, twice); + return_value = _testmultiphase_StateAccessType_increment_count_clinic_impl((StateAccessTypeObject *)self, cls, n, twice); exit: return return_value; @@ -154,12 +155,12 @@ _testmultiphase_StateAccessType_get_count_impl(StateAccessTypeObject *self, PyTypeObject *cls); static PyObject * -_testmultiphase_StateAccessType_get_count(StateAccessTypeObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_testmultiphase_StateAccessType_get_count(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "get_count() takes no arguments"); return NULL; } - return _testmultiphase_StateAccessType_get_count_impl(self, cls); + return _testmultiphase_StateAccessType_get_count_impl((StateAccessTypeObject *)self, cls); } -/*[clinic end generated code: output=2c199bad52e9cda7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ea0ca98e467e53c2 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_threadmodule.c.h b/Modules/clinic/_threadmodule.c.h new file mode 100644 index 00000000000000..09b7afebd6d8d9 --- /dev/null +++ b/Modules/clinic/_threadmodule.c.h @@ -0,0 +1,104 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +#if (defined(HAVE_PTHREAD_GETNAME_NP) || defined(MS_WINDOWS)) + +PyDoc_STRVAR(_thread__get_name__doc__, +"_get_name($module, /)\n" +"--\n" +"\n" +"Get the name of the current thread."); + +#define _THREAD__GET_NAME_METHODDEF \ + {"_get_name", (PyCFunction)_thread__get_name, METH_NOARGS, _thread__get_name__doc__}, + +static PyObject * +_thread__get_name_impl(PyObject *module); + +static PyObject * +_thread__get_name(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _thread__get_name_impl(module); +} + +#endif /* (defined(HAVE_PTHREAD_GETNAME_NP) || defined(MS_WINDOWS)) */ + +#if (defined(HAVE_PTHREAD_SETNAME_NP) || defined(MS_WINDOWS)) + +PyDoc_STRVAR(_thread_set_name__doc__, +"set_name($module, /, name)\n" +"--\n" +"\n" +"Set the name of the current thread."); + +#define _THREAD_SET_NAME_METHODDEF \ + {"set_name", _PyCFunction_CAST(_thread_set_name), METH_FASTCALL|METH_KEYWORDS, _thread_set_name__doc__}, + +static PyObject * +_thread_set_name_impl(PyObject *module, PyObject *name_obj); + +static PyObject * +_thread_set_name(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(name), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"name", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "set_name", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *name_obj; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("set_name", "argument 'name'", "str", args[0]); + goto exit; + } + name_obj = args[0]; + return_value = _thread_set_name_impl(module, name_obj); + +exit: + return return_value; +} + +#endif /* (defined(HAVE_PTHREAD_SETNAME_NP) || defined(MS_WINDOWS)) */ + +#ifndef _THREAD__GET_NAME_METHODDEF + #define _THREAD__GET_NAME_METHODDEF +#endif /* !defined(_THREAD__GET_NAME_METHODDEF) */ + +#ifndef _THREAD_SET_NAME_METHODDEF + #define _THREAD_SET_NAME_METHODDEF +#endif /* !defined(_THREAD_SET_NAME_METHODDEF) */ +/*[clinic end generated code: output=6e88ef6b126cece8 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_tkinter.c.h b/Modules/clinic/_tkinter.c.h index 188bcc773cfc41..d6e783b04fe968 100644 --- a/Modules/clinic/_tkinter.c.h +++ b/Modules/clinic/_tkinter.c.h @@ -16,7 +16,7 @@ static PyObject * _tkinter_tkapp_eval_impl(TkappObject *self, const char *script); static PyObject * -_tkinter_tkapp_eval(TkappObject *self, PyObject *arg) +_tkinter_tkapp_eval(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; const char *script; @@ -34,7 +34,7 @@ _tkinter_tkapp_eval(TkappObject *self, PyObject *arg) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - return_value = _tkinter_tkapp_eval_impl(self, script); + return_value = _tkinter_tkapp_eval_impl((TkappObject *)self, script); exit: return return_value; @@ -52,7 +52,7 @@ static PyObject * _tkinter_tkapp_evalfile_impl(TkappObject *self, const char *fileName); static PyObject * -_tkinter_tkapp_evalfile(TkappObject *self, PyObject *arg) +_tkinter_tkapp_evalfile(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; const char *fileName; @@ -70,7 +70,7 @@ _tkinter_tkapp_evalfile(TkappObject *self, PyObject *arg) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - return_value = _tkinter_tkapp_evalfile_impl(self, fileName); + return_value = _tkinter_tkapp_evalfile_impl((TkappObject *)self, fileName); exit: return return_value; @@ -88,7 +88,7 @@ static PyObject * _tkinter_tkapp_record_impl(TkappObject *self, const char *script); static PyObject * -_tkinter_tkapp_record(TkappObject *self, PyObject *arg) +_tkinter_tkapp_record(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; const char *script; @@ -106,7 +106,7 @@ _tkinter_tkapp_record(TkappObject *self, PyObject *arg) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - return_value = _tkinter_tkapp_record_impl(self, script); + return_value = _tkinter_tkapp_record_impl((TkappObject *)self, script); exit: return return_value; @@ -124,7 +124,7 @@ static PyObject * _tkinter_tkapp_adderrorinfo_impl(TkappObject *self, const char *msg); static PyObject * -_tkinter_tkapp_adderrorinfo(TkappObject *self, PyObject *arg) +_tkinter_tkapp_adderrorinfo(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; const char *msg; @@ -142,7 +142,7 @@ _tkinter_tkapp_adderrorinfo(TkappObject *self, PyObject *arg) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - return_value = _tkinter_tkapp_adderrorinfo_impl(self, msg); + return_value = _tkinter_tkapp_adderrorinfo_impl((TkappObject *)self, msg); exit: return return_value; @@ -184,7 +184,7 @@ static PyObject * _tkinter_tkapp_exprstring_impl(TkappObject *self, const char *s); static PyObject * -_tkinter_tkapp_exprstring(TkappObject *self, PyObject *arg) +_tkinter_tkapp_exprstring(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; const char *s; @@ -202,7 +202,7 @@ _tkinter_tkapp_exprstring(TkappObject *self, PyObject *arg) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - return_value = _tkinter_tkapp_exprstring_impl(self, s); + return_value = _tkinter_tkapp_exprstring_impl((TkappObject *)self, s); exit: return return_value; @@ -220,7 +220,7 @@ static PyObject * _tkinter_tkapp_exprlong_impl(TkappObject *self, const char *s); static PyObject * -_tkinter_tkapp_exprlong(TkappObject *self, PyObject *arg) +_tkinter_tkapp_exprlong(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; const char *s; @@ -238,7 +238,7 @@ _tkinter_tkapp_exprlong(TkappObject *self, PyObject *arg) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - return_value = _tkinter_tkapp_exprlong_impl(self, s); + return_value = _tkinter_tkapp_exprlong_impl((TkappObject *)self, s); exit: return return_value; @@ -256,7 +256,7 @@ static PyObject * _tkinter_tkapp_exprdouble_impl(TkappObject *self, const char *s); static PyObject * -_tkinter_tkapp_exprdouble(TkappObject *self, PyObject *arg) +_tkinter_tkapp_exprdouble(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; const char *s; @@ -274,7 +274,7 @@ _tkinter_tkapp_exprdouble(TkappObject *self, PyObject *arg) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - return_value = _tkinter_tkapp_exprdouble_impl(self, s); + return_value = _tkinter_tkapp_exprdouble_impl((TkappObject *)self, s); exit: return return_value; @@ -292,7 +292,7 @@ static PyObject * _tkinter_tkapp_exprboolean_impl(TkappObject *self, const char *s); static PyObject * -_tkinter_tkapp_exprboolean(TkappObject *self, PyObject *arg) +_tkinter_tkapp_exprboolean(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; const char *s; @@ -310,7 +310,7 @@ _tkinter_tkapp_exprboolean(TkappObject *self, PyObject *arg) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - return_value = _tkinter_tkapp_exprboolean_impl(self, s); + return_value = _tkinter_tkapp_exprboolean_impl((TkappObject *)self, s); exit: return return_value; @@ -337,7 +337,7 @@ _tkinter_tkapp_createcommand_impl(TkappObject *self, const char *name, PyObject *func); static PyObject * -_tkinter_tkapp_createcommand(TkappObject *self, PyObject *const *args, Py_ssize_t nargs) +_tkinter_tkapp_createcommand(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; const char *name; @@ -360,7 +360,7 @@ _tkinter_tkapp_createcommand(TkappObject *self, PyObject *const *args, Py_ssize_ goto exit; } func = args[1]; - return_value = _tkinter_tkapp_createcommand_impl(self, name, func); + return_value = _tkinter_tkapp_createcommand_impl((TkappObject *)self, name, func); exit: return return_value; @@ -378,7 +378,7 @@ static PyObject * _tkinter_tkapp_deletecommand_impl(TkappObject *self, const char *name); static PyObject * -_tkinter_tkapp_deletecommand(TkappObject *self, PyObject *arg) +_tkinter_tkapp_deletecommand(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; const char *name; @@ -396,7 +396,7 @@ _tkinter_tkapp_deletecommand(TkappObject *self, PyObject *arg) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - return_value = _tkinter_tkapp_deletecommand_impl(self, name); + return_value = _tkinter_tkapp_deletecommand_impl((TkappObject *)self, name); exit: return return_value; @@ -417,7 +417,7 @@ _tkinter_tkapp_createfilehandler_impl(TkappObject *self, PyObject *file, int mask, PyObject *func); static PyObject * -_tkinter_tkapp_createfilehandler(TkappObject *self, PyObject *const *args, Py_ssize_t nargs) +_tkinter_tkapp_createfilehandler(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *file; @@ -433,7 +433,7 @@ _tkinter_tkapp_createfilehandler(TkappObject *self, PyObject *const *args, Py_ss goto exit; } func = args[2]; - return_value = _tkinter_tkapp_createfilehandler_impl(self, file, mask, func); + return_value = _tkinter_tkapp_createfilehandler_impl((TkappObject *)self, file, mask, func); exit: return return_value; @@ -465,9 +465,9 @@ static PyObject * _tkinter_tktimertoken_deletetimerhandler_impl(TkttObject *self); static PyObject * -_tkinter_tktimertoken_deletetimerhandler(TkttObject *self, PyObject *Py_UNUSED(ignored)) +_tkinter_tktimertoken_deletetimerhandler(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _tkinter_tktimertoken_deletetimerhandler_impl(self); + return _tkinter_tktimertoken_deletetimerhandler_impl((TkttObject *)self); } PyDoc_STRVAR(_tkinter_tkapp_createtimerhandler__doc__, @@ -483,7 +483,7 @@ _tkinter_tkapp_createtimerhandler_impl(TkappObject *self, int milliseconds, PyObject *func); static PyObject * -_tkinter_tkapp_createtimerhandler(TkappObject *self, PyObject *const *args, Py_ssize_t nargs) +_tkinter_tkapp_createtimerhandler(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int milliseconds; @@ -497,7 +497,7 @@ _tkinter_tkapp_createtimerhandler(TkappObject *self, PyObject *const *args, Py_s goto exit; } func = args[1]; - return_value = _tkinter_tkapp_createtimerhandler_impl(self, milliseconds, func); + return_value = _tkinter_tkapp_createtimerhandler_impl((TkappObject *)self, milliseconds, func); exit: return return_value; @@ -515,7 +515,7 @@ static PyObject * _tkinter_tkapp_mainloop_impl(TkappObject *self, int threshold); static PyObject * -_tkinter_tkapp_mainloop(TkappObject *self, PyObject *const *args, Py_ssize_t nargs) +_tkinter_tkapp_mainloop(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int threshold = 0; @@ -531,7 +531,7 @@ _tkinter_tkapp_mainloop(TkappObject *self, PyObject *const *args, Py_ssize_t nar goto exit; } skip_optional: - return_value = _tkinter_tkapp_mainloop_impl(self, threshold); + return_value = _tkinter_tkapp_mainloop_impl((TkappObject *)self, threshold); exit: return return_value; @@ -549,7 +549,7 @@ static PyObject * _tkinter_tkapp_dooneevent_impl(TkappObject *self, int flags); static PyObject * -_tkinter_tkapp_dooneevent(TkappObject *self, PyObject *const *args, Py_ssize_t nargs) +_tkinter_tkapp_dooneevent(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int flags = 0; @@ -565,7 +565,7 @@ _tkinter_tkapp_dooneevent(TkappObject *self, PyObject *const *args, Py_ssize_t n goto exit; } skip_optional: - return_value = _tkinter_tkapp_dooneevent_impl(self, flags); + return_value = _tkinter_tkapp_dooneevent_impl((TkappObject *)self, flags); exit: return return_value; @@ -583,9 +583,9 @@ static PyObject * _tkinter_tkapp_quit_impl(TkappObject *self); static PyObject * -_tkinter_tkapp_quit(TkappObject *self, PyObject *Py_UNUSED(ignored)) +_tkinter_tkapp_quit(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _tkinter_tkapp_quit_impl(self); + return _tkinter_tkapp_quit_impl((TkappObject *)self); } PyDoc_STRVAR(_tkinter_tkapp_interpaddr__doc__, @@ -600,9 +600,9 @@ static PyObject * _tkinter_tkapp_interpaddr_impl(TkappObject *self); static PyObject * -_tkinter_tkapp_interpaddr(TkappObject *self, PyObject *Py_UNUSED(ignored)) +_tkinter_tkapp_interpaddr(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _tkinter_tkapp_interpaddr_impl(self); + return _tkinter_tkapp_interpaddr_impl((TkappObject *)self); } PyDoc_STRVAR(_tkinter_tkapp_loadtk__doc__, @@ -617,9 +617,36 @@ static PyObject * _tkinter_tkapp_loadtk_impl(TkappObject *self); static PyObject * -_tkinter_tkapp_loadtk(TkappObject *self, PyObject *Py_UNUSED(ignored)) +_tkinter_tkapp_loadtk(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _tkinter_tkapp_loadtk_impl(self); + return _tkinter_tkapp_loadtk_impl((TkappObject *)self); +} + +PyDoc_STRVAR(_tkinter_tkapp_settrace__doc__, +"settrace($self, func, /)\n" +"--\n" +"\n" +"Set the tracing function."); + +#define _TKINTER_TKAPP_SETTRACE_METHODDEF \ + {"settrace", (PyCFunction)_tkinter_tkapp_settrace, METH_O, _tkinter_tkapp_settrace__doc__}, + +PyDoc_STRVAR(_tkinter_tkapp_gettrace__doc__, +"gettrace($self, /)\n" +"--\n" +"\n" +"Get the tracing function."); + +#define _TKINTER_TKAPP_GETTRACE_METHODDEF \ + {"gettrace", (PyCFunction)_tkinter_tkapp_gettrace, METH_NOARGS, _tkinter_tkapp_gettrace__doc__}, + +static PyObject * +_tkinter_tkapp_gettrace_impl(TkappObject *self); + +static PyObject * +_tkinter_tkapp_gettrace(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _tkinter_tkapp_gettrace_impl((TkappObject *)self); } PyDoc_STRVAR(_tkinter_tkapp_willdispatch__doc__, @@ -634,9 +661,9 @@ static PyObject * _tkinter_tkapp_willdispatch_impl(TkappObject *self); static PyObject * -_tkinter_tkapp_willdispatch(TkappObject *self, PyObject *Py_UNUSED(ignored)) +_tkinter_tkapp_willdispatch(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _tkinter_tkapp_willdispatch_impl(self); + return _tkinter_tkapp_willdispatch_impl((TkappObject *)self); } PyDoc_STRVAR(_tkinter__flatten__doc__, @@ -649,7 +676,7 @@ PyDoc_STRVAR(_tkinter__flatten__doc__, PyDoc_STRVAR(_tkinter_create__doc__, "create($module, screenName=None, baseName=\'\', className=\'Tk\',\n" -" interactive=False, wantobjects=False, wantTk=True, sync=False,\n" +" interactive=False, wantobjects=0, wantTk=True, sync=False,\n" " use=None, /)\n" "--\n" "\n" @@ -750,8 +777,8 @@ _tkinter_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 5) { goto skip_optional; } - wantobjects = PyObject_IsTrue(args[4]); - if (wantobjects < 0) { + wantobjects = PyLong_AsInt(args[4]); + if (wantobjects == -1 && PyErr_Occurred()) { goto exit; } if (nargs < 6) { @@ -861,4 +888,4 @@ _tkinter_getbusywaitinterval(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #define _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #endif /* !defined(_TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF) */ -/*[clinic end generated code: output=d447501ec5aa9447 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=172a98df5f209a84 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_weakref.c.h b/Modules/clinic/_weakref.c.h index 550b6c4d71a015..8d7bc5dc936610 100644 --- a/Modules/clinic/_weakref.c.h +++ b/Modules/clinic/_weakref.c.h @@ -2,7 +2,6 @@ preserve [clinic start generated code]*/ -#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_weakref_getweakrefcount__doc__, @@ -23,9 +22,7 @@ _weakref_getweakrefcount(PyObject *module, PyObject *object) PyObject *return_value = NULL; Py_ssize_t _return_value; - Py_BEGIN_CRITICAL_SECTION(object); _return_value = _weakref_getweakrefcount_impl(module, object); - Py_END_CRITICAL_SECTION(); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -79,21 +76,6 @@ PyDoc_STRVAR(_weakref_getweakrefs__doc__, #define _WEAKREF_GETWEAKREFS_METHODDEF \ {"getweakrefs", (PyCFunction)_weakref_getweakrefs, METH_O, _weakref_getweakrefs__doc__}, -static PyObject * -_weakref_getweakrefs_impl(PyObject *module, PyObject *object); - -static PyObject * -_weakref_getweakrefs(PyObject *module, PyObject *object) -{ - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(object); - return_value = _weakref_getweakrefs_impl(module, object); - Py_END_CRITICAL_SECTION(); - - return return_value; -} - PyDoc_STRVAR(_weakref_proxy__doc__, "proxy($module, object, callback=None, /)\n" "--\n" @@ -130,4 +112,4 @@ _weakref_proxy(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=d5d30707212a9870 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=60f59adc1dc9eab8 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index 468457e624c691..6a2f8d45cd4e0c 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -21,7 +21,7 @@ static PyObject * _winapi_Overlapped_GetOverlappedResult_impl(OverlappedObject *self, int wait); static PyObject * -_winapi_Overlapped_GetOverlappedResult(OverlappedObject *self, PyObject *arg) +_winapi_Overlapped_GetOverlappedResult(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; int wait; @@ -30,7 +30,7 @@ _winapi_Overlapped_GetOverlappedResult(OverlappedObject *self, PyObject *arg) if (wait < 0) { goto exit; } - return_value = _winapi_Overlapped_GetOverlappedResult_impl(self, wait); + return_value = _winapi_Overlapped_GetOverlappedResult_impl((OverlappedObject *)self, wait); exit: return return_value; @@ -48,9 +48,9 @@ static PyObject * _winapi_Overlapped_getbuffer_impl(OverlappedObject *self); static PyObject * -_winapi_Overlapped_getbuffer(OverlappedObject *self, PyObject *Py_UNUSED(ignored)) +_winapi_Overlapped_getbuffer(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _winapi_Overlapped_getbuffer_impl(self); + return _winapi_Overlapped_getbuffer_impl((OverlappedObject *)self); } PyDoc_STRVAR(_winapi_Overlapped_cancel__doc__, @@ -65,9 +65,9 @@ static PyObject * _winapi_Overlapped_cancel_impl(OverlappedObject *self); static PyObject * -_winapi_Overlapped_cancel(OverlappedObject *self, PyObject *Py_UNUSED(ignored)) +_winapi_Overlapped_cancel(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _winapi_Overlapped_cancel_impl(self); + return _winapi_Overlapped_cancel_impl((OverlappedObject *)self); } PyDoc_STRVAR(_winapi_CloseHandle__doc__, @@ -445,7 +445,7 @@ PyDoc_STRVAR(_winapi_CreateNamedPipe__doc__, {"CreateNamedPipe", _PyCFunction_CAST(_winapi_CreateNamedPipe), METH_FASTCALL, _winapi_CreateNamedPipe__doc__}, static HANDLE -_winapi_CreateNamedPipe_impl(PyObject *module, LPCTSTR name, DWORD open_mode, +_winapi_CreateNamedPipe_impl(PyObject *module, LPCWSTR name, DWORD open_mode, DWORD pipe_mode, DWORD max_instances, DWORD out_buffer_size, DWORD in_buffer_size, DWORD default_timeout, @@ -455,7 +455,7 @@ static PyObject * _winapi_CreateNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - LPCTSTR name; + LPCWSTR name = NULL; DWORD open_mode; DWORD pipe_mode; DWORD max_instances; @@ -465,8 +465,8 @@ _winapi_CreateNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t narg LPSECURITY_ATTRIBUTES security_attributes; HANDLE _return_value; - if (!_PyArg_ParseStack(args, nargs, "skkkkkk" F_POINTER ":CreateNamedPipe", - &name, &open_mode, &pipe_mode, &max_instances, &out_buffer_size, &in_buffer_size, &default_timeout, &security_attributes)) { + if (!_PyArg_ParseStack(args, nargs, "O&kkkkkk" F_POINTER ":CreateNamedPipe", + _PyUnicode_WideCharString_Converter, &name, &open_mode, &pipe_mode, &max_instances, &out_buffer_size, &in_buffer_size, &default_timeout, &security_attributes)) { goto exit; } _return_value = _winapi_CreateNamedPipe_impl(module, name, open_mode, pipe_mode, max_instances, out_buffer_size, in_buffer_size, default_timeout, security_attributes); @@ -479,6 +479,9 @@ _winapi_CreateNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t narg return_value = HANDLE_TO_PYNUM(_return_value); exit: + /* Cleanup for name */ + PyMem_Free((void *)name); + return return_value; } @@ -741,6 +744,77 @@ _winapi_GetLastError(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +PyDoc_STRVAR(_winapi_GetLongPathName__doc__, +"GetLongPathName($module, /, path)\n" +"--\n" +"\n" +"Return the long version of the provided path.\n" +"\n" +"If the path is already in its long form, returns the same value.\n" +"\n" +"The path must already be a \'str\'. If the type is not known, use\n" +"os.fsdecode before calling this function."); + +#define _WINAPI_GETLONGPATHNAME_METHODDEF \ + {"GetLongPathName", _PyCFunction_CAST(_winapi_GetLongPathName), METH_FASTCALL|METH_KEYWORDS, _winapi_GetLongPathName__doc__}, + +static PyObject * +_winapi_GetLongPathName_impl(PyObject *module, LPCWSTR path); + +static PyObject * +_winapi_GetLongPathName(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "GetLongPathName", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + LPCWSTR path = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("GetLongPathName", "argument 'path'", "str", args[0]); + goto exit; + } + path = PyUnicode_AsWideCharString(args[0], NULL); + if (path == NULL) { + goto exit; + } + return_value = _winapi_GetLongPathName_impl(module, path); + +exit: + /* Cleanup for path */ + PyMem_Free((void *)path); + + return return_value; +} + PyDoc_STRVAR(_winapi_GetModuleFileName__doc__, "GetModuleFileName($module, module_handle, /)\n" "--\n" @@ -775,6 +849,77 @@ _winapi_GetModuleFileName(PyObject *module, PyObject *arg) return return_value; } +PyDoc_STRVAR(_winapi_GetShortPathName__doc__, +"GetShortPathName($module, /, path)\n" +"--\n" +"\n" +"Return the short version of the provided path.\n" +"\n" +"If the path is already in its short form, returns the same value.\n" +"\n" +"The path must already be a \'str\'. If the type is not known, use\n" +"os.fsdecode before calling this function."); + +#define _WINAPI_GETSHORTPATHNAME_METHODDEF \ + {"GetShortPathName", _PyCFunction_CAST(_winapi_GetShortPathName), METH_FASTCALL|METH_KEYWORDS, _winapi_GetShortPathName__doc__}, + +static PyObject * +_winapi_GetShortPathName_impl(PyObject *module, LPCWSTR path); + +static PyObject * +_winapi_GetShortPathName(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "GetShortPathName", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + LPCWSTR path = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("GetShortPathName", "argument 'path'", "str", args[0]); + goto exit; + } + path = PyUnicode_AsWideCharString(args[0], NULL); + if (path == NULL) { + goto exit; + } + return_value = _winapi_GetShortPathName_impl(module, path); + +exit: + /* Cleanup for path */ + PyMem_Free((void *)path); + + return return_value; +} + PyDoc_STRVAR(_winapi_GetStdHandle__doc__, "GetStdHandle($module, std_handle, /)\n" "--\n" @@ -1520,22 +1665,25 @@ PyDoc_STRVAR(_winapi_WaitNamedPipe__doc__, {"WaitNamedPipe", _PyCFunction_CAST(_winapi_WaitNamedPipe), METH_FASTCALL, _winapi_WaitNamedPipe__doc__}, static PyObject * -_winapi_WaitNamedPipe_impl(PyObject *module, LPCTSTR name, DWORD timeout); +_winapi_WaitNamedPipe_impl(PyObject *module, LPCWSTR name, DWORD timeout); static PyObject * _winapi_WaitNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - LPCTSTR name; + LPCWSTR name = NULL; DWORD timeout; - if (!_PyArg_ParseStack(args, nargs, "sk:WaitNamedPipe", - &name, &timeout)) { + if (!_PyArg_ParseStack(args, nargs, "O&k:WaitNamedPipe", + _PyUnicode_WideCharString_Converter, &name, &timeout)) { goto exit; } return_value = _winapi_WaitNamedPipe_impl(module, name, timeout); exit: + /* Cleanup for name */ + PyMem_Free((void *)name); + return return_value; } @@ -1858,7 +2006,8 @@ _winapi__mimetypes_read_windows_registry(PyObject *module, PyObject *const *args PyObject *argsbuf[1]; PyObject *on_type_read; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1978,4 +2127,4 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } -/*[clinic end generated code: output=1f5bbcfa8d1847c5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=06b56212b2186250 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_zoneinfo.c.h b/Modules/clinic/_zoneinfo.c.h index 9905b6425e2f79..dd72b5e663aea6 100644 --- a/Modules/clinic/_zoneinfo.c.h +++ b/Modules/clinic/_zoneinfo.c.h @@ -6,8 +6,66 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +PyDoc_STRVAR(zoneinfo_ZoneInfo__doc__, +"ZoneInfo(key)\n" +"--\n" +"\n" +"Create a new ZoneInfo instance."); + +static PyObject * +zoneinfo_ZoneInfo_impl(PyTypeObject *type, PyObject *key); + +static PyObject * +zoneinfo_ZoneInfo(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(key), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"key", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "ZoneInfo", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *key; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + key = fastargs[0]; + Py_BEGIN_CRITICAL_SECTION(type); + return_value = zoneinfo_ZoneInfo_impl(type, key); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + PyDoc_STRVAR(zoneinfo_ZoneInfo_from_file__doc__, "from_file($type, file_obj, /, key=None)\n" "--\n" @@ -55,7 +113,8 @@ zoneinfo_ZoneInfo_from_file(PyTypeObject *type, PyTypeObject *cls, PyObject *con PyObject *file_obj; PyObject *key = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -116,7 +175,8 @@ zoneinfo_ZoneInfo_no_cache(PyTypeObject *type, PyTypeObject *cls, PyObject *cons PyObject *argsbuf[1]; PyObject *key; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -173,7 +233,8 @@ zoneinfo_ZoneInfo_clear_cache(PyTypeObject *type, PyTypeObject *cls, PyObject *c Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; PyObject *only_keys = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -182,7 +243,9 @@ zoneinfo_ZoneInfo_clear_cache(PyTypeObject *type, PyTypeObject *cls, PyObject *c } only_keys = args[0]; skip_optional_kwonly: + Py_BEGIN_CRITICAL_SECTION(type); return_value = zoneinfo_ZoneInfo_clear_cache_impl(type, cls, only_keys); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -221,7 +284,8 @@ zoneinfo_ZoneInfo_utcoffset(PyObject *self, PyTypeObject *cls, PyObject *const * PyObject *argsbuf[1]; PyObject *dt; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -264,7 +328,8 @@ zoneinfo_ZoneInfo_dst(PyObject *self, PyTypeObject *cls, PyObject *const *args, PyObject *argsbuf[1]; PyObject *dt; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -308,7 +373,8 @@ zoneinfo_ZoneInfo_tzname(PyObject *self, PyTypeObject *cls, PyObject *const *arg PyObject *argsbuf[1]; PyObject *dt; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -353,7 +419,8 @@ zoneinfo_ZoneInfo__unpickle(PyTypeObject *type, PyTypeObject *cls, PyObject *con PyObject *key; unsigned char from_cache; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -372,4 +439,4 @@ zoneinfo_ZoneInfo__unpickle(PyTypeObject *type, PyTypeObject *cls, PyObject *con exit: return return_value; } -/*[clinic end generated code: output=2a15f32fdd2ab6cd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f8a4fb4ff634d6c9 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index 60a03fe012550e..c5b62b16699d06 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -2,6 +2,9 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_runtime.h" // _Py_SINGLETON() +#endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_CheckPositional() @@ -18,9 +21,9 @@ static PyObject * array_array_clear_impl(arrayobject *self); static PyObject * -array_array_clear(arrayobject *self, PyObject *Py_UNUSED(ignored)) +array_array_clear(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_clear_impl(self); + return array_array_clear_impl((arrayobject *)self); } PyDoc_STRVAR(array_array___copy____doc__, @@ -36,9 +39,9 @@ static PyObject * array_array___copy___impl(arrayobject *self); static PyObject * -array_array___copy__(arrayobject *self, PyObject *Py_UNUSED(ignored)) +array_array___copy__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array___copy___impl(self); + return array_array___copy___impl((arrayobject *)self); } PyDoc_STRVAR(array_array___deepcopy____doc__, @@ -75,7 +78,7 @@ array_array_index_impl(arrayobject *self, PyObject *v, Py_ssize_t start, Py_ssize_t stop); static PyObject * -array_array_index(arrayobject *self, PyObject *const *args, Py_ssize_t nargs) +array_array_index(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *v; @@ -99,7 +102,7 @@ array_array_index(arrayobject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: - return_value = array_array_index_impl(self, v, start, stop); + return_value = array_array_index_impl((arrayobject *)self, v, start, stop); exit: return return_value; @@ -129,7 +132,7 @@ static PyObject * array_array_pop_impl(arrayobject *self, Py_ssize_t i); static PyObject * -array_array_pop(arrayobject *self, PyObject *const *args, Py_ssize_t nargs) +array_array_pop(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t i = -1; @@ -153,7 +156,7 @@ array_array_pop(arrayobject *self, PyObject *const *args, Py_ssize_t nargs) i = ival; } skip_optional: - return_value = array_array_pop_impl(self, i); + return_value = array_array_pop_impl((arrayobject *)self, i); exit: return return_value; @@ -172,7 +175,7 @@ static PyObject * array_array_extend_impl(arrayobject *self, PyTypeObject *cls, PyObject *bb); static PyObject * -array_array_extend(arrayobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +array_array_extend(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -191,12 +194,13 @@ array_array_extend(arrayobject *self, PyTypeObject *cls, PyObject *const *args, PyObject *argsbuf[1]; PyObject *bb; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } bb = args[0]; - return_value = array_array_extend_impl(self, cls, bb); + return_value = array_array_extend_impl((arrayobject *)self, cls, bb); exit: return return_value; @@ -215,7 +219,7 @@ static PyObject * array_array_insert_impl(arrayobject *self, Py_ssize_t i, PyObject *v); static PyObject * -array_array_insert(arrayobject *self, PyObject *const *args, Py_ssize_t nargs) +array_array_insert(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t i; @@ -237,7 +241,7 @@ array_array_insert(arrayobject *self, PyObject *const *args, Py_ssize_t nargs) i = ival; } v = args[1]; - return_value = array_array_insert_impl(self, i, v); + return_value = array_array_insert_impl((arrayobject *)self, i, v); exit: return return_value; @@ -259,9 +263,9 @@ static PyObject * array_array_buffer_info_impl(arrayobject *self); static PyObject * -array_array_buffer_info(arrayobject *self, PyObject *Py_UNUSED(ignored)) +array_array_buffer_info(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_buffer_info_impl(self); + return array_array_buffer_info_impl((arrayobject *)self); } PyDoc_STRVAR(array_array_append__doc__, @@ -289,9 +293,9 @@ static PyObject * array_array_byteswap_impl(arrayobject *self); static PyObject * -array_array_byteswap(arrayobject *self, PyObject *Py_UNUSED(ignored)) +array_array_byteswap(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_byteswap_impl(self); + return array_array_byteswap_impl((arrayobject *)self); } PyDoc_STRVAR(array_array_reverse__doc__, @@ -307,9 +311,9 @@ static PyObject * array_array_reverse_impl(arrayobject *self); static PyObject * -array_array_reverse(arrayobject *self, PyObject *Py_UNUSED(ignored)) +array_array_reverse(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_reverse_impl(self); + return array_array_reverse_impl((arrayobject *)self); } PyDoc_STRVAR(array_array_fromfile__doc__, @@ -326,7 +330,7 @@ array_array_fromfile_impl(arrayobject *self, PyTypeObject *cls, PyObject *f, Py_ssize_t n); static PyObject * -array_array_fromfile(arrayobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +array_array_fromfile(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -346,7 +350,8 @@ array_array_fromfile(arrayobject *self, PyTypeObject *cls, PyObject *const *args PyObject *f; Py_ssize_t n; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -363,7 +368,7 @@ array_array_fromfile(arrayobject *self, PyTypeObject *cls, PyObject *const *args } n = ival; } - return_value = array_array_fromfile_impl(self, cls, f, n); + return_value = array_array_fromfile_impl((arrayobject *)self, cls, f, n); exit: return return_value; @@ -382,7 +387,7 @@ static PyObject * array_array_tofile_impl(arrayobject *self, PyTypeObject *cls, PyObject *f); static PyObject * -array_array_tofile(arrayobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +array_array_tofile(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -401,12 +406,13 @@ array_array_tofile(arrayobject *self, PyTypeObject *cls, PyObject *const *args, PyObject *argsbuf[1]; PyObject *f; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } f = args[0]; - return_value = array_array_tofile_impl(self, cls, f); + return_value = array_array_tofile_impl((arrayobject *)self, cls, f); exit: return return_value; @@ -434,9 +440,9 @@ static PyObject * array_array_tolist_impl(arrayobject *self); static PyObject * -array_array_tolist(arrayobject *self, PyObject *Py_UNUSED(ignored)) +array_array_tolist(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_tolist_impl(self); + return array_array_tolist_impl((arrayobject *)self); } PyDoc_STRVAR(array_array_frombytes__doc__, @@ -452,7 +458,7 @@ static PyObject * array_array_frombytes_impl(arrayobject *self, Py_buffer *buffer); static PyObject * -array_array_frombytes(arrayobject *self, PyObject *arg) +array_array_frombytes(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; Py_buffer buffer = {NULL, NULL}; @@ -460,7 +466,7 @@ array_array_frombytes(arrayobject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &buffer, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = array_array_frombytes_impl(self, &buffer); + return_value = array_array_frombytes_impl((arrayobject *)self, &buffer); exit: /* Cleanup for buffer */ @@ -484,9 +490,9 @@ static PyObject * array_array_tobytes_impl(arrayobject *self); static PyObject * -array_array_tobytes(arrayobject *self, PyObject *Py_UNUSED(ignored)) +array_array_tobytes(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_tobytes_impl(self); + return array_array_tobytes_impl((arrayobject *)self); } PyDoc_STRVAR(array_array_fromunicode__doc__, @@ -506,7 +512,7 @@ static PyObject * array_array_fromunicode_impl(arrayobject *self, PyObject *ustr); static PyObject * -array_array_fromunicode(arrayobject *self, PyObject *arg) +array_array_fromunicode(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; PyObject *ustr; @@ -516,7 +522,7 @@ array_array_fromunicode(arrayobject *self, PyObject *arg) goto exit; } ustr = arg; - return_value = array_array_fromunicode_impl(self, ustr); + return_value = array_array_fromunicode_impl((arrayobject *)self, ustr); exit: return return_value; @@ -539,9 +545,9 @@ static PyObject * array_array_tounicode_impl(arrayobject *self); static PyObject * -array_array_tounicode(arrayobject *self, PyObject *Py_UNUSED(ignored)) +array_array_tounicode(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_tounicode_impl(self); + return array_array_tounicode_impl((arrayobject *)self); } PyDoc_STRVAR(array_array___sizeof____doc__, @@ -557,9 +563,9 @@ static PyObject * array_array___sizeof___impl(arrayobject *self); static PyObject * -array_array___sizeof__(arrayobject *self, PyObject *Py_UNUSED(ignored)) +array_array___sizeof__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array___sizeof___impl(self); + return array_array___sizeof___impl((arrayobject *)self); } PyDoc_STRVAR(array__array_reconstructor__doc__, @@ -596,7 +602,10 @@ array__array_reconstructor(PyObject *module, PyObject *const *args, Py_ssize_t n goto exit; } if (PyUnicode_GET_LENGTH(args[1]) != 1) { - _PyArg_BadArgument("_array_reconstructor", "argument 2", "a unicode character", args[1]); + PyErr_Format(PyExc_TypeError, + "_array_reconstructor(): argument 2 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[1])); goto exit; } typecode = PyUnicode_READ_CHAR(args[1], 0); @@ -625,7 +634,7 @@ array_array___reduce_ex___impl(arrayobject *self, PyTypeObject *cls, PyObject *value); static PyObject * -array_array___reduce_ex__(arrayobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +array_array___reduce_ex__(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -644,12 +653,13 @@ array_array___reduce_ex__(arrayobject *self, PyTypeObject *cls, PyObject *const PyObject *argsbuf[1]; PyObject *value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } value = args[0]; - return_value = array_array___reduce_ex___impl(self, cls, value); + return_value = array_array___reduce_ex___impl((arrayobject *)self, cls, value); exit: return return_value; @@ -668,13 +678,13 @@ static PyObject * array_arrayiterator___reduce___impl(arrayiterobject *self, PyTypeObject *cls); static PyObject * -array_arrayiterator___reduce__(arrayiterobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +array_arrayiterator___reduce__(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "__reduce__() takes no arguments"); return NULL; } - return array_arrayiterator___reduce___impl(self, cls); + return array_arrayiterator___reduce___impl((arrayiterobject *)self, cls); } PyDoc_STRVAR(array_arrayiterator___setstate____doc__, @@ -685,4 +695,4 @@ PyDoc_STRVAR(array_arrayiterator___setstate____doc__, #define ARRAY_ARRAYITERATOR___SETSTATE___METHODDEF \ {"__setstate__", (PyCFunction)array_arrayiterator___setstate__, METH_O, array_arrayiterator___setstate____doc__}, -/*[clinic end generated code: output=52c55d9b1d026c1c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8120dc5c4fa414b9 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/binascii.c.h b/Modules/clinic/binascii.c.h index 1adca415dfee12..f81f12c388f373 100644 --- a/Modules/clinic/binascii.c.h +++ b/Modules/clinic/binascii.c.h @@ -85,7 +85,8 @@ binascii_b2a_uu(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj Py_buffer data = {NULL, NULL}; int backtick = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -161,7 +162,8 @@ binascii_a2b_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P Py_buffer data = {NULL, NULL}; int strict_mode = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -232,7 +234,8 @@ binascii_b2a_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P Py_buffer data = {NULL, NULL}; int newline = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -412,7 +415,8 @@ binascii_b2a_hex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb PyObject *sep = NULL; int bytes_per_sep = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -501,7 +505,8 @@ binascii_hexlify(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb PyObject *sep = NULL; int bytes_per_sep = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -646,7 +651,8 @@ binascii_a2b_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj Py_buffer data = {NULL, NULL}; int header = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -724,7 +730,8 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj int istext = 1; int header = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -767,4 +774,4 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return return_value; } -/*[clinic end generated code: output=968767b663ed889d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9ed7fbeec13c6606 input=a9049054013a1b77]*/ diff --git a/Modules/_blake2/clinic/blake2b_impl.c.h b/Modules/clinic/blake2module.c.h similarity index 52% rename from Modules/_blake2/clinic/blake2b_impl.c.h rename to Modules/clinic/blake2module.c.h index 47d62717eb76e7..b5ac90143a1740 100644 --- a/Modules/_blake2/clinic/blake2b_impl.c.h +++ b/Modules/clinic/blake2module.c.h @@ -59,7 +59,7 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; PyObject *data = NULL; - int digest_size = BLAKE2B_OUTBYTES; + int digest_size = HACL_HASH_BLAKE2B_OUT_BYTES; Py_buffer key = {NULL, NULL}; Py_buffer salt = {NULL, NULL}; Py_buffer person = {NULL, NULL}; @@ -72,7 +72,8 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int last_node = 0; int usedforsecurity = 1; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -203,6 +204,201 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return return_value; } +PyDoc_STRVAR(py_blake2s_new__doc__, +"blake2s(data=b\'\', /, *, digest_size=_blake2.blake2s.MAX_DIGEST_SIZE,\n" +" key=b\'\', salt=b\'\', person=b\'\', fanout=1, depth=1, leaf_size=0,\n" +" node_offset=0, node_depth=0, inner_size=0, last_node=False,\n" +" usedforsecurity=True)\n" +"--\n" +"\n" +"Return a new BLAKE2s hash object."); + +static PyObject * +py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, + Py_buffer *key, Py_buffer *salt, Py_buffer *person, + int fanout, int depth, unsigned long leaf_size, + unsigned long long node_offset, int node_depth, + int inner_size, int last_node, int usedforsecurity); + +static PyObject * +py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 12 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(digest_size), &_Py_ID(key), &_Py_ID(salt), &_Py_ID(person), &_Py_ID(fanout), &_Py_ID(depth), &_Py_ID(leaf_size), &_Py_ID(node_offset), &_Py_ID(node_depth), &_Py_ID(inner_size), &_Py_ID(last_node), &_Py_ID(usedforsecurity), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "blake2s", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[13]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + PyObject *data = NULL; + int digest_size = HACL_HASH_BLAKE2S_OUT_BYTES; + Py_buffer key = {NULL, NULL}; + Py_buffer salt = {NULL, NULL}; + Py_buffer person = {NULL, NULL}; + int fanout = 1; + int depth = 1; + unsigned long leaf_size = 0; + unsigned long long node_offset = 0; + int node_depth = 0; + int inner_size = 0; + int last_node = 0; + int usedforsecurity = 1; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (nargs < 1) { + goto skip_optional_posonly; + } + noptargs--; + data = fastargs[0]; +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_kwonly; + } + if (fastargs[1]) { + digest_size = PyLong_AsInt(fastargs[1]); + if (digest_size == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[2]) { + if (PyObject_GetBuffer(fastargs[2], &key, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[3]) { + if (PyObject_GetBuffer(fastargs[3], &salt, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[4]) { + if (PyObject_GetBuffer(fastargs[4], &person, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[5]) { + fanout = PyLong_AsInt(fastargs[5]); + if (fanout == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[6]) { + depth = PyLong_AsInt(fastargs[6]); + if (depth == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[7]) { + if (!_PyLong_UnsignedLong_Converter(fastargs[7], &leaf_size)) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[8]) { + if (!_PyLong_UnsignedLongLong_Converter(fastargs[8], &node_offset)) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[9]) { + node_depth = PyLong_AsInt(fastargs[9]); + if (node_depth == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[10]) { + inner_size = PyLong_AsInt(fastargs[10]); + if (inner_size == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[11]) { + last_node = PyObject_IsTrue(fastargs[11]); + if (last_node < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + usedforsecurity = PyObject_IsTrue(fastargs[12]); + if (usedforsecurity < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = py_blake2s_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); + +exit: + /* Cleanup for key */ + if (key.obj) { + PyBuffer_Release(&key); + } + /* Cleanup for salt */ + if (salt.obj) { + PyBuffer_Release(&salt); + } + /* Cleanup for person */ + if (person.obj) { + PyBuffer_Release(&person); + } + + return return_value; +} + PyDoc_STRVAR(_blake2_blake2b_copy__doc__, "copy($self, /)\n" "--\n" @@ -213,12 +409,12 @@ PyDoc_STRVAR(_blake2_blake2b_copy__doc__, {"copy", (PyCFunction)_blake2_blake2b_copy, METH_NOARGS, _blake2_blake2b_copy__doc__}, static PyObject * -_blake2_blake2b_copy_impl(BLAKE2bObject *self); +_blake2_blake2b_copy_impl(Blake2Object *self); static PyObject * -_blake2_blake2b_copy(BLAKE2bObject *self, PyObject *Py_UNUSED(ignored)) +_blake2_blake2b_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _blake2_blake2b_copy_impl(self); + return _blake2_blake2b_copy_impl((Blake2Object *)self); } PyDoc_STRVAR(_blake2_blake2b_update__doc__, @@ -240,12 +436,12 @@ PyDoc_STRVAR(_blake2_blake2b_digest__doc__, {"digest", (PyCFunction)_blake2_blake2b_digest, METH_NOARGS, _blake2_blake2b_digest__doc__}, static PyObject * -_blake2_blake2b_digest_impl(BLAKE2bObject *self); +_blake2_blake2b_digest_impl(Blake2Object *self); static PyObject * -_blake2_blake2b_digest(BLAKE2bObject *self, PyObject *Py_UNUSED(ignored)) +_blake2_blake2b_digest(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _blake2_blake2b_digest_impl(self); + return _blake2_blake2b_digest_impl((Blake2Object *)self); } PyDoc_STRVAR(_blake2_blake2b_hexdigest__doc__, @@ -258,11 +454,11 @@ PyDoc_STRVAR(_blake2_blake2b_hexdigest__doc__, {"hexdigest", (PyCFunction)_blake2_blake2b_hexdigest, METH_NOARGS, _blake2_blake2b_hexdigest__doc__}, static PyObject * -_blake2_blake2b_hexdigest_impl(BLAKE2bObject *self); +_blake2_blake2b_hexdigest_impl(Blake2Object *self); static PyObject * -_blake2_blake2b_hexdigest(BLAKE2bObject *self, PyObject *Py_UNUSED(ignored)) +_blake2_blake2b_hexdigest(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _blake2_blake2b_hexdigest_impl(self); + return _blake2_blake2b_hexdigest_impl((Blake2Object *)self); } -/*[clinic end generated code: output=e18eeaee40623bfc input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6e03c947b7e0d973 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/cmathmodule.c.h b/Modules/clinic/cmathmodule.c.h index b709204af1d959..16eaba7aa7ee39 100644 --- a/Modules/clinic/cmathmodule.c.h +++ b/Modules/clinic/cmathmodule.c.h @@ -908,7 +908,7 @@ cmath_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(rel_tol), &_Py_ID(abs_tol), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), &_Py_ID(rel_tol), &_Py_ID(abs_tol), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -932,7 +932,8 @@ cmath_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec double abs_tol = 0.0; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -982,4 +983,4 @@ cmath_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=364093af55bfe53f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5fda69f15dc9dfc9 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/fcntlmodule.c.h b/Modules/clinic/fcntlmodule.c.h index 5dc2fc068d0f7e..53b139e09afdf1 100644 --- a/Modules/clinic/fcntlmodule.c.h +++ b/Modules/clinic/fcntlmodule.c.h @@ -2,9 +2,6 @@ preserve [clinic start generated code]*/ -#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() -#include "pycore_modsupport.h" // _PyArg_CheckPositional() - PyDoc_STRVAR(fcntl_fcntl__doc__, "fcntl($module, fd, cmd, arg=0, /)\n" "--\n" @@ -22,7 +19,7 @@ PyDoc_STRVAR(fcntl_fcntl__doc__, "corresponding to the return value of the fcntl call in the C code."); #define FCNTL_FCNTL_METHODDEF \ - {"fcntl", _PyCFunction_CAST(fcntl_fcntl), METH_FASTCALL, fcntl_fcntl__doc__}, + {"fcntl", (PyCFunction)(void(*)(void))fcntl_fcntl, METH_FASTCALL, fcntl_fcntl__doc__}, static PyObject * fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg); @@ -35,10 +32,16 @@ fcntl_fcntl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int code; PyObject *arg = NULL; - if (!_PyArg_CheckPositional("fcntl", nargs, 2, 3)) { + if (nargs < 2) { + PyErr_Format(PyExc_TypeError, "fcntl expected at least 2 arguments, got %zd", nargs); + goto exit; + } + if (nargs > 3) { + PyErr_Format(PyExc_TypeError, "fcntl expected at most 3 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } code = PyLong_AsInt(args[1]); @@ -90,10 +93,10 @@ PyDoc_STRVAR(fcntl_ioctl__doc__, "code."); #define FCNTL_IOCTL_METHODDEF \ - {"ioctl", _PyCFunction_CAST(fcntl_ioctl), METH_FASTCALL, fcntl_ioctl__doc__}, + {"ioctl", (PyCFunction)(void(*)(void))fcntl_ioctl, METH_FASTCALL, fcntl_ioctl__doc__}, static PyObject * -fcntl_ioctl_impl(PyObject *module, int fd, unsigned int code, +fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *ob_arg, int mutate_arg); static PyObject * @@ -101,20 +104,27 @@ fcntl_ioctl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int fd; - unsigned int code; + unsigned long code; PyObject *ob_arg = NULL; int mutate_arg = 1; - if (!_PyArg_CheckPositional("ioctl", nargs, 2, 4)) { + if (nargs < 2) { + PyErr_Format(PyExc_TypeError, "ioctl expected at least 2 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + if (nargs > 4) { + PyErr_Format(PyExc_TypeError, "ioctl expected at most 4 arguments, got %zd", nargs); goto exit; } - code = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); - if (code == (unsigned int)-1 && PyErr_Occurred()) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } + if (!PyLong_Check(args[1])) { + PyErr_Format(PyExc_TypeError, "ioctl() argument 2 must be int, not %T", args[1]); + goto exit; + } + code = PyLong_AsUnsignedLongMask(args[1]); if (nargs < 3) { goto skip_optional; } @@ -143,7 +153,7 @@ PyDoc_STRVAR(fcntl_flock__doc__, "function is emulated using fcntl())."); #define FCNTL_FLOCK_METHODDEF \ - {"flock", _PyCFunction_CAST(fcntl_flock), METH_FASTCALL, fcntl_flock__doc__}, + {"flock", (PyCFunction)(void(*)(void))fcntl_flock, METH_FASTCALL, fcntl_flock__doc__}, static PyObject * fcntl_flock_impl(PyObject *module, int fd, int code); @@ -155,10 +165,12 @@ fcntl_flock(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; int code; - if (!_PyArg_CheckPositional("flock", nargs, 2, 2)) { + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "flock expected 2 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } code = PyLong_AsInt(args[1]); @@ -199,7 +211,7 @@ PyDoc_STRVAR(fcntl_lockf__doc__, " 2 - relative to the end of the file (SEEK_END)"); #define FCNTL_LOCKF_METHODDEF \ - {"lockf", _PyCFunction_CAST(fcntl_lockf), METH_FASTCALL, fcntl_lockf__doc__}, + {"lockf", (PyCFunction)(void(*)(void))fcntl_lockf, METH_FASTCALL, fcntl_lockf__doc__}, static PyObject * fcntl_lockf_impl(PyObject *module, int fd, int code, PyObject *lenobj, @@ -215,10 +227,16 @@ fcntl_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *startobj = NULL; int whence = 0; - if (!_PyArg_CheckPositional("lockf", nargs, 2, 5)) { + if (nargs < 2) { + PyErr_Format(PyExc_TypeError, "lockf expected at least 2 arguments, got %zd", nargs); + goto exit; + } + if (nargs > 5) { + PyErr_Format(PyExc_TypeError, "lockf expected at most 5 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } code = PyLong_AsInt(args[1]); @@ -246,4 +264,4 @@ fcntl_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=732e33ba92042031 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=45a56f53fd17ff3c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index 9fff4da616ba00..0147020bd483f9 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -8,6 +8,7 @@ preserve #endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_tuple.h" // _PyTuple_FromArray() PyDoc_STRVAR(gc_enable__doc__, "enable($module, /)\n" @@ -125,7 +126,8 @@ gc_collect(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * int generation = NUM_GENERATIONS - 1; Py_ssize_t _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -312,28 +314,24 @@ PyDoc_STRVAR(gc_get_referrers__doc__, {"get_referrers", _PyCFunction_CAST(gc_get_referrers), METH_FASTCALL, gc_get_referrers__doc__}, static PyObject * -gc_get_referrers_impl(PyObject *module, PyObject *args); +gc_get_referrers_impl(PyObject *module, PyObject *objs); static PyObject * gc_get_referrers(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; + PyObject *objs = NULL; - if (!_PyArg_CheckPositional("get_referrers", nargs, 0, PY_SSIZE_T_MAX)) { + objs = _PyTuple_FromArray(args, nargs); + if (objs == NULL) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = gc_get_referrers_impl(module, __clinic_args); + return_value = gc_get_referrers_impl(module, objs); exit: - Py_XDECREF(__clinic_args); + /* Cleanup for objs */ + Py_XDECREF(objs); + return return_value; } @@ -347,28 +345,24 @@ PyDoc_STRVAR(gc_get_referents__doc__, {"get_referents", _PyCFunction_CAST(gc_get_referents), METH_FASTCALL, gc_get_referents__doc__}, static PyObject * -gc_get_referents_impl(PyObject *module, PyObject *args); +gc_get_referents_impl(PyObject *module, PyObject *objs); static PyObject * gc_get_referents(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; + PyObject *objs = NULL; - if (!_PyArg_CheckPositional("get_referents", nargs, 0, PY_SSIZE_T_MAX)) { + objs = _PyTuple_FromArray(args, nargs); + if (objs == NULL) { goto exit; } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = gc_get_referents_impl(module, __clinic_args); + return_value = gc_get_referents_impl(module, objs); exit: - Py_XDECREF(__clinic_args); + /* Cleanup for objs */ + Py_XDECREF(objs); + return return_value; } @@ -423,7 +417,8 @@ gc_get_objects(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; Py_ssize_t generation = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -585,4 +580,4 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=0a7e91917adcb937 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3e33248997e06c34 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/grpmodule.c.h b/Modules/clinic/grpmodule.c.h index 1da061f3fd8eb1..facfa3a43e490e 100644 --- a/Modules/clinic/grpmodule.c.h +++ b/Modules/clinic/grpmodule.c.h @@ -54,7 +54,8 @@ grp_getgrgid(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *argsbuf[1]; PyObject *id; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -111,7 +112,8 @@ grp_getgrnam(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *argsbuf[1]; PyObject *name; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -146,4 +148,4 @@ grp_getgrall(PyObject *module, PyObject *Py_UNUSED(ignored)) { return grp_getgrall_impl(module); } -/*[clinic end generated code: output=2f7011384604d38d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2154194308dab038 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/itertoolsmodule.c.h b/Modules/clinic/itertoolsmodule.c.h index 3ec479943a83d4..ea0cc495ffb969 100644 --- a/Modules/clinic/itertoolsmodule.c.h +++ b/Modules/clinic/itertoolsmodule.c.h @@ -47,7 +47,7 @@ batched_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(iterable), &_Py_ID(n), &_Py_ID(strict), }, + .ob_item = { &_Py_ID(iterable), _Py_LATIN1_CHR('n'), &_Py_ID(strict), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -71,7 +71,8 @@ batched_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t n; int strict = 0; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 2, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -186,7 +187,8 @@ itertools_groupby(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *it; PyObject *keyfunc = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -509,7 +511,7 @@ itertools_combinations(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(iterable), &_Py_ID(r), }, + .ob_item = { &_Py_ID(iterable), _Py_LATIN1_CHR('r'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -531,7 +533,8 @@ itertools_combinations(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *iterable; Py_ssize_t r; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 2, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -580,7 +583,7 @@ itertools_combinations_with_replacement(PyTypeObject *type, PyObject *args, PyOb PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(iterable), &_Py_ID(r), }, + .ob_item = { &_Py_ID(iterable), _Py_LATIN1_CHR('r'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -602,7 +605,8 @@ itertools_combinations_with_replacement(PyTypeObject *type, PyObject *args, PyOb PyObject *iterable; Py_ssize_t r; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 2, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -650,7 +654,7 @@ itertools_permutations(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(iterable), &_Py_ID(r), }, + .ob_item = { &_Py_ID(iterable), _Py_LATIN1_CHR('r'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -673,7 +677,8 @@ itertools_permutations(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *iterable; PyObject *robj = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -736,7 +741,8 @@ itertools_accumulate(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *binop = Py_None; PyObject *initial = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -809,7 +815,8 @@ itertools_compress(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *seq1; PyObject *seq2; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 2, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -908,7 +915,8 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *long_cnt = NULL; PyObject *long_step = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -928,4 +936,4 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=c6a515f765da86b5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=199eeac2d17e8f23 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/mathmodule.c.h b/Modules/clinic/mathmodule.c.h index ca14c03f16f706..a76dde1eb4350e 100644 --- a/Modules/clinic/mathmodule.c.h +++ b/Modules/clinic/mathmodule.c.h @@ -8,6 +8,60 @@ preserve #endif #include "pycore_modsupport.h" // _PyArg_CheckPositional() +PyDoc_STRVAR(math_gcd__doc__, +"gcd($module, /, *integers)\n" +"--\n" +"\n" +"Greatest Common Divisor."); + +#define MATH_GCD_METHODDEF \ + {"gcd", _PyCFunction_CAST(math_gcd), METH_FASTCALL, math_gcd__doc__}, + +static PyObject * +math_gcd_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length); + +static PyObject * +math_gcd(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject * const *__clinic_args; + Py_ssize_t args_length; + + __clinic_args = args; + args_length = nargs; + return_value = math_gcd_impl(module, __clinic_args, args_length); + + return return_value; +} + +PyDoc_STRVAR(math_lcm__doc__, +"lcm($module, /, *integers)\n" +"--\n" +"\n" +"Least Common Multiple."); + +#define MATH_LCM_METHODDEF \ + {"lcm", _PyCFunction_CAST(math_lcm), METH_FASTCALL, math_lcm__doc__}, + +static PyObject * +math_lcm_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length); + +static PyObject * +math_lcm(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject * const *__clinic_args; + Py_ssize_t args_length; + + __clinic_args = args; + args_length = nargs; + return_value = math_lcm_impl(module, __clinic_args, args_length); + + return return_value; +} + PyDoc_STRVAR(math_ceil__doc__, "ceil($module, x, /)\n" "--\n" @@ -34,9 +88,9 @@ PyDoc_STRVAR(math_fsum__doc__, "fsum($module, seq, /)\n" "--\n" "\n" -"Return an accurate floating point sum of values in the iterable seq.\n" +"Return an accurate floating-point sum of values in the iterable seq.\n" "\n" -"Assumes IEEE-754 floating point arithmetic."); +"Assumes IEEE-754 floating-point arithmetic."); #define MATH_FSUM_METHODDEF \ {"fsum", (PyCFunction)math_fsum, METH_O, math_fsum__doc__}, @@ -204,6 +258,67 @@ PyDoc_STRVAR(math_log10__doc__, #define MATH_LOG10_METHODDEF \ {"log10", (PyCFunction)math_log10, METH_O, math_log10__doc__}, +PyDoc_STRVAR(math_fma__doc__, +"fma($module, x, y, z, /)\n" +"--\n" +"\n" +"Fused multiply-add operation.\n" +"\n" +"Compute (x * y) + z with a single round."); + +#define MATH_FMA_METHODDEF \ + {"fma", _PyCFunction_CAST(math_fma), METH_FASTCALL, math_fma__doc__}, + +static PyObject * +math_fma_impl(PyObject *module, double x, double y, double z); + +static PyObject * +math_fma(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + double x; + double y; + double z; + + if (!_PyArg_CheckPositional("fma", nargs, 3, 3)) { + goto exit; + } + if (PyFloat_CheckExact(args[0])) { + x = PyFloat_AS_DOUBLE(args[0]); + } + else + { + x = PyFloat_AsDouble(args[0]); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + if (PyFloat_CheckExact(args[1])) { + y = PyFloat_AS_DOUBLE(args[1]); + } + else + { + y = PyFloat_AsDouble(args[1]); + if (y == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + if (PyFloat_CheckExact(args[2])) { + z = PyFloat_AS_DOUBLE(args[2]); + } + else + { + z = PyFloat_AsDouble(args[2]); + if (z == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + return_value = math_fma_impl(module, x, y, z); + +exit: + return return_value; +} + PyDoc_STRVAR(math_fmod__doc__, "fmod($module, x, y, /)\n" "--\n" @@ -290,6 +405,44 @@ math_dist(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(math_hypot__doc__, +"hypot($module, /, *coordinates)\n" +"--\n" +"\n" +"Multidimensional Euclidean distance from the origin to a point.\n" +"\n" +"Roughly equivalent to:\n" +" sqrt(sum(x**2 for x in coordinates))\n" +"\n" +"For a two dimensional point (x, y), gives the hypotenuse\n" +"using the Pythagorean theorem: sqrt(x*x + y*y).\n" +"\n" +"For example, the hypotenuse of a 3/4/5 right triangle is:\n" +"\n" +" >>> hypot(3.0, 4.0)\n" +" 5.0"); + +#define MATH_HYPOT_METHODDEF \ + {"hypot", _PyCFunction_CAST(math_hypot), METH_FASTCALL, math_hypot__doc__}, + +static PyObject * +math_hypot_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length); + +static PyObject * +math_hypot(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject * const *__clinic_args; + Py_ssize_t args_length; + + __clinic_args = args; + args_length = nargs; + return_value = math_hypot_impl(module, __clinic_args, args_length); + + return return_value; +} + PyDoc_STRVAR(math_sumprod__doc__, "sumprod($module, p, q, /)\n" "--\n" @@ -298,7 +451,7 @@ PyDoc_STRVAR(math_sumprod__doc__, "\n" "Roughly equivalent to:\n" "\n" -" sum(itertools.starmap(operator.mul, zip(p, q, strict=True)))\n" +" sum(map(operator.mul, p, q, strict=True))\n" "\n" "For float and mixed int/float inputs, the intermediate products\n" "and sums are computed with extended precision."); @@ -549,7 +702,7 @@ PyDoc_STRVAR(math_isclose__doc__, "isclose($module, /, a, b, *, rel_tol=1e-09, abs_tol=0.0)\n" "--\n" "\n" -"Determine whether two floating point numbers are close in value.\n" +"Determine whether two floating-point numbers are close in value.\n" "\n" " rel_tol\n" " maximum difference for being considered \"close\", relative to the\n" @@ -587,7 +740,7 @@ math_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(rel_tol), &_Py_ID(abs_tol), }, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), &_Py_ID(rel_tol), &_Py_ID(abs_tol), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -611,7 +764,8 @@ math_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject double abs_tol = 0.0; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -726,7 +880,8 @@ math_prod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k PyObject *iterable; PyObject *start = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -877,7 +1032,8 @@ math_nextafter(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje double y; PyObject *steps = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -950,4 +1106,4 @@ math_ulp(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=6b2eeaed8d8a76d5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1ccb4b9f570d6dad input=a9049054013a1b77]*/ diff --git a/Modules/clinic/md5module.c.h b/Modules/clinic/md5module.c.h index ee7fb3d7c613f2..1f0acebf47b6ff 100644 --- a/Modules/clinic/md5module.c.h +++ b/Modules/clinic/md5module.c.h @@ -21,13 +21,13 @@ static PyObject * MD5Type_copy_impl(MD5object *self, PyTypeObject *cls); static PyObject * -MD5Type_copy(MD5object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +MD5Type_copy(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } - return MD5Type_copy_impl(self, cls); + return MD5Type_copy_impl((MD5object *)self, cls); } PyDoc_STRVAR(MD5Type_digest__doc__, @@ -43,9 +43,9 @@ static PyObject * MD5Type_digest_impl(MD5object *self); static PyObject * -MD5Type_digest(MD5object *self, PyObject *Py_UNUSED(ignored)) +MD5Type_digest(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return MD5Type_digest_impl(self); + return MD5Type_digest_impl((MD5object *)self); } PyDoc_STRVAR(MD5Type_hexdigest__doc__, @@ -61,9 +61,9 @@ static PyObject * MD5Type_hexdigest_impl(MD5object *self); static PyObject * -MD5Type_hexdigest(MD5object *self, PyObject *Py_UNUSED(ignored)) +MD5Type_hexdigest(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return MD5Type_hexdigest_impl(self); + return MD5Type_hexdigest_impl((MD5object *)self); } PyDoc_STRVAR(MD5Type_update__doc__, @@ -121,7 +121,8 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *string = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -148,4 +149,4 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw exit: return return_value; } -/*[clinic end generated code: output=4dbca39332d3f52f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a4292eab710dcb60 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/overlapped.c.h b/Modules/clinic/overlapped.c.h index 8b285e4a8f0a72..7e5715660022c1 100644 --- a/Modules/clinic/overlapped.c.h +++ b/Modules/clinic/overlapped.c.h @@ -484,7 +484,8 @@ _overlapped_Overlapped(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; HANDLE event = INVALID_HANDLE_VALUE; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -515,9 +516,9 @@ static PyObject * _overlapped_Overlapped_cancel_impl(OverlappedObject *self); static PyObject * -_overlapped_Overlapped_cancel(OverlappedObject *self, PyObject *Py_UNUSED(ignored)) +_overlapped_Overlapped_cancel(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _overlapped_Overlapped_cancel_impl(self); + return _overlapped_Overlapped_cancel_impl((OverlappedObject *)self); } PyDoc_STRVAR(_overlapped_Overlapped_getresult__doc__, @@ -536,7 +537,7 @@ static PyObject * _overlapped_Overlapped_getresult_impl(OverlappedObject *self, BOOL wait); static PyObject * -_overlapped_Overlapped_getresult(OverlappedObject *self, PyObject *const *args, Py_ssize_t nargs) +_overlapped_Overlapped_getresult(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; BOOL wait = FALSE; @@ -552,7 +553,7 @@ _overlapped_Overlapped_getresult(OverlappedObject *self, PyObject *const *args, goto exit; } skip_optional: - return_value = _overlapped_Overlapped_getresult_impl(self, wait); + return_value = _overlapped_Overlapped_getresult_impl((OverlappedObject *)self, wait); exit: return return_value; @@ -572,7 +573,7 @@ _overlapped_Overlapped_ReadFile_impl(OverlappedObject *self, HANDLE handle, DWORD size); static PyObject * -_overlapped_Overlapped_ReadFile(OverlappedObject *self, PyObject *const *args, Py_ssize_t nargs) +_overlapped_Overlapped_ReadFile(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HANDLE handle; @@ -588,7 +589,7 @@ _overlapped_Overlapped_ReadFile(OverlappedObject *self, PyObject *const *args, P if (!_PyLong_UnsignedLong_Converter(args[1], &size)) { goto exit; } - return_value = _overlapped_Overlapped_ReadFile_impl(self, handle, size); + return_value = _overlapped_Overlapped_ReadFile_impl((OverlappedObject *)self, handle, size); exit: return return_value; @@ -608,7 +609,7 @@ _overlapped_Overlapped_ReadFileInto_impl(OverlappedObject *self, HANDLE handle, Py_buffer *bufobj); static PyObject * -_overlapped_Overlapped_ReadFileInto(OverlappedObject *self, PyObject *const *args, Py_ssize_t nargs) +_overlapped_Overlapped_ReadFileInto(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HANDLE handle; @@ -624,7 +625,7 @@ _overlapped_Overlapped_ReadFileInto(OverlappedObject *self, PyObject *const *arg if (PyObject_GetBuffer(args[1], &bufobj, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = _overlapped_Overlapped_ReadFileInto_impl(self, handle, &bufobj); + return_value = _overlapped_Overlapped_ReadFileInto_impl((OverlappedObject *)self, handle, &bufobj); exit: /* Cleanup for bufobj */ @@ -649,7 +650,7 @@ _overlapped_Overlapped_WSARecv_impl(OverlappedObject *self, HANDLE handle, DWORD size, DWORD flags); static PyObject * -_overlapped_Overlapped_WSARecv(OverlappedObject *self, PyObject *const *args, Py_ssize_t nargs) +_overlapped_Overlapped_WSARecv(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HANDLE handle; @@ -673,7 +674,7 @@ _overlapped_Overlapped_WSARecv(OverlappedObject *self, PyObject *const *args, Py goto exit; } skip_optional: - return_value = _overlapped_Overlapped_WSARecv_impl(self, handle, size, flags); + return_value = _overlapped_Overlapped_WSARecv_impl((OverlappedObject *)self, handle, size, flags); exit: return return_value; @@ -694,7 +695,7 @@ _overlapped_Overlapped_WSARecvInto_impl(OverlappedObject *self, DWORD flags); static PyObject * -_overlapped_Overlapped_WSARecvInto(OverlappedObject *self, PyObject *const *args, Py_ssize_t nargs) +_overlapped_Overlapped_WSARecvInto(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HANDLE handle; @@ -714,7 +715,7 @@ _overlapped_Overlapped_WSARecvInto(OverlappedObject *self, PyObject *const *args if (!_PyLong_UnsignedLong_Converter(args[2], &flags)) { goto exit; } - return_value = _overlapped_Overlapped_WSARecvInto_impl(self, handle, &bufobj, flags); + return_value = _overlapped_Overlapped_WSARecvInto_impl((OverlappedObject *)self, handle, &bufobj, flags); exit: /* Cleanup for bufobj */ @@ -739,7 +740,7 @@ _overlapped_Overlapped_WriteFile_impl(OverlappedObject *self, HANDLE handle, Py_buffer *bufobj); static PyObject * -_overlapped_Overlapped_WriteFile(OverlappedObject *self, PyObject *const *args, Py_ssize_t nargs) +_overlapped_Overlapped_WriteFile(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HANDLE handle; @@ -755,7 +756,7 @@ _overlapped_Overlapped_WriteFile(OverlappedObject *self, PyObject *const *args, if (PyObject_GetBuffer(args[1], &bufobj, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = _overlapped_Overlapped_WriteFile_impl(self, handle, &bufobj); + return_value = _overlapped_Overlapped_WriteFile_impl((OverlappedObject *)self, handle, &bufobj); exit: /* Cleanup for bufobj */ @@ -780,7 +781,7 @@ _overlapped_Overlapped_WSASend_impl(OverlappedObject *self, HANDLE handle, Py_buffer *bufobj, DWORD flags); static PyObject * -_overlapped_Overlapped_WSASend(OverlappedObject *self, PyObject *const *args, Py_ssize_t nargs) +_overlapped_Overlapped_WSASend(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HANDLE handle; @@ -800,7 +801,7 @@ _overlapped_Overlapped_WSASend(OverlappedObject *self, PyObject *const *args, Py if (!_PyLong_UnsignedLong_Converter(args[2], &flags)) { goto exit; } - return_value = _overlapped_Overlapped_WSASend_impl(self, handle, &bufobj, flags); + return_value = _overlapped_Overlapped_WSASend_impl((OverlappedObject *)self, handle, &bufobj, flags); exit: /* Cleanup for bufobj */ @@ -826,7 +827,7 @@ _overlapped_Overlapped_AcceptEx_impl(OverlappedObject *self, HANDLE AcceptSocket); static PyObject * -_overlapped_Overlapped_AcceptEx(OverlappedObject *self, PyObject *const *args, Py_ssize_t nargs) +_overlapped_Overlapped_AcceptEx(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HANDLE ListenSocket; @@ -843,7 +844,7 @@ _overlapped_Overlapped_AcceptEx(OverlappedObject *self, PyObject *const *args, P if (!AcceptSocket && PyErr_Occurred()) { goto exit; } - return_value = _overlapped_Overlapped_AcceptEx_impl(self, ListenSocket, AcceptSocket); + return_value = _overlapped_Overlapped_AcceptEx_impl((OverlappedObject *)self, ListenSocket, AcceptSocket); exit: return return_value; @@ -866,7 +867,7 @@ _overlapped_Overlapped_ConnectEx_impl(OverlappedObject *self, PyObject *AddressObj); static PyObject * -_overlapped_Overlapped_ConnectEx(OverlappedObject *self, PyObject *const *args, Py_ssize_t nargs) +_overlapped_Overlapped_ConnectEx(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HANDLE ConnectSocket; @@ -884,7 +885,7 @@ _overlapped_Overlapped_ConnectEx(OverlappedObject *self, PyObject *const *args, goto exit; } AddressObj = args[1]; - return_value = _overlapped_Overlapped_ConnectEx_impl(self, ConnectSocket, AddressObj); + return_value = _overlapped_Overlapped_ConnectEx_impl((OverlappedObject *)self, ConnectSocket, AddressObj); exit: return return_value; @@ -903,7 +904,7 @@ _overlapped_Overlapped_DisconnectEx_impl(OverlappedObject *self, HANDLE Socket, DWORD flags); static PyObject * -_overlapped_Overlapped_DisconnectEx(OverlappedObject *self, PyObject *const *args, Py_ssize_t nargs) +_overlapped_Overlapped_DisconnectEx(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HANDLE Socket; @@ -919,7 +920,7 @@ _overlapped_Overlapped_DisconnectEx(OverlappedObject *self, PyObject *const *arg if (!_PyLong_UnsignedLong_Converter(args[1], &flags)) { goto exit; } - return_value = _overlapped_Overlapped_DisconnectEx_impl(self, Socket, flags); + return_value = _overlapped_Overlapped_DisconnectEx_impl((OverlappedObject *)self, Socket, flags); exit: return return_value; @@ -943,7 +944,7 @@ _overlapped_Overlapped_TransmitFile_impl(OverlappedObject *self, DWORD count_per_send, DWORD flags); static PyObject * -_overlapped_Overlapped_TransmitFile(OverlappedObject *self, PyObject *const *args, Py_ssize_t nargs) +_overlapped_Overlapped_TransmitFile(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HANDLE Socket; @@ -980,7 +981,7 @@ _overlapped_Overlapped_TransmitFile(OverlappedObject *self, PyObject *const *arg if (!_PyLong_UnsignedLong_Converter(args[6], &flags)) { goto exit; } - return_value = _overlapped_Overlapped_TransmitFile_impl(self, Socket, File, offset, offset_high, count_to_write, count_per_send, flags); + return_value = _overlapped_Overlapped_TransmitFile_impl((OverlappedObject *)self, Socket, File, offset, offset_high, count_to_write, count_per_send, flags); exit: return return_value; @@ -1000,7 +1001,7 @@ _overlapped_Overlapped_ConnectNamedPipe_impl(OverlappedObject *self, HANDLE Pipe); static PyObject * -_overlapped_Overlapped_ConnectNamedPipe(OverlappedObject *self, PyObject *arg) +_overlapped_Overlapped_ConnectNamedPipe(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; HANDLE Pipe; @@ -1009,7 +1010,7 @@ _overlapped_Overlapped_ConnectNamedPipe(OverlappedObject *self, PyObject *arg) if (!Pipe && PyErr_Occurred()) { goto exit; } - return_value = _overlapped_Overlapped_ConnectNamedPipe_impl(self, Pipe); + return_value = _overlapped_Overlapped_ConnectNamedPipe_impl((OverlappedObject *)self, Pipe); exit: return return_value; @@ -1029,7 +1030,7 @@ _overlapped_Overlapped_ConnectPipe_impl(OverlappedObject *self, const wchar_t *Address); static PyObject * -_overlapped_Overlapped_ConnectPipe(OverlappedObject *self, PyObject *arg) +_overlapped_Overlapped_ConnectPipe(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; const wchar_t *Address = NULL; @@ -1042,7 +1043,7 @@ _overlapped_Overlapped_ConnectPipe(OverlappedObject *self, PyObject *arg) if (Address == NULL) { goto exit; } - return_value = _overlapped_Overlapped_ConnectPipe_impl(self, Address); + return_value = _overlapped_Overlapped_ConnectPipe_impl((OverlappedObject *)self, Address); exit: /* Cleanup for Address */ @@ -1104,7 +1105,7 @@ _overlapped_Overlapped_WSASendTo_impl(OverlappedObject *self, HANDLE handle, PyObject *AddressObj); static PyObject * -_overlapped_Overlapped_WSASendTo(OverlappedObject *self, PyObject *const *args, Py_ssize_t nargs) +_overlapped_Overlapped_WSASendTo(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HANDLE handle; @@ -1130,7 +1131,7 @@ _overlapped_Overlapped_WSASendTo(OverlappedObject *self, PyObject *const *args, goto exit; } AddressObj = args[3]; - return_value = _overlapped_Overlapped_WSASendTo_impl(self, handle, &bufobj, flags, AddressObj); + return_value = _overlapped_Overlapped_WSASendTo_impl((OverlappedObject *)self, handle, &bufobj, flags, AddressObj); exit: /* Cleanup for bufobj */ @@ -1156,7 +1157,7 @@ _overlapped_Overlapped_WSARecvFrom_impl(OverlappedObject *self, DWORD flags); static PyObject * -_overlapped_Overlapped_WSARecvFrom(OverlappedObject *self, PyObject *const *args, Py_ssize_t nargs) +_overlapped_Overlapped_WSARecvFrom(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HANDLE handle; @@ -1180,7 +1181,7 @@ _overlapped_Overlapped_WSARecvFrom(OverlappedObject *self, PyObject *const *args goto exit; } skip_optional: - return_value = _overlapped_Overlapped_WSARecvFrom_impl(self, handle, size, flags); + return_value = _overlapped_Overlapped_WSARecvFrom_impl((OverlappedObject *)self, handle, size, flags); exit: return return_value; @@ -1201,7 +1202,7 @@ _overlapped_Overlapped_WSARecvFromInto_impl(OverlappedObject *self, DWORD size, DWORD flags); static PyObject * -_overlapped_Overlapped_WSARecvFromInto(OverlappedObject *self, PyObject *const *args, Py_ssize_t nargs) +_overlapped_Overlapped_WSARecvFromInto(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HANDLE handle; @@ -1229,7 +1230,7 @@ _overlapped_Overlapped_WSARecvFromInto(OverlappedObject *self, PyObject *const * goto exit; } skip_optional: - return_value = _overlapped_Overlapped_WSARecvFromInto_impl(self, handle, &bufobj, size, flags); + return_value = _overlapped_Overlapped_WSARecvFromInto_impl((OverlappedObject *)self, handle, &bufobj, size, flags); exit: /* Cleanup for bufobj */ @@ -1239,4 +1240,4 @@ _overlapped_Overlapped_WSARecvFromInto(OverlappedObject *self, PyObject *const * return return_value; } -/*[clinic end generated code: output=958cbddbcc355f47 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d009cc9e53d9732a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index b49d64d4281889..abeb9c3e3e12b1 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -7,7 +7,6 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() -#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() #include "pycore_long.h" // _PyLong_UnsignedInt_Converter() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() @@ -73,11 +72,12 @@ os_stat(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("stat", "path", 0, 1); + path_t path = PATH_T_INITIALIZE_P("stat", "path", 0, 0, 0, 1); int dir_fd = DEFAULT_DIR_FD; int follow_symlinks = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -155,10 +155,11 @@ os_lstat(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("lstat", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("lstat", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -251,14 +252,15 @@ os_access(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("access", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("access", "path", 0, 0, 0, 0); int mode; int dir_fd = DEFAULT_DIR_FD; int effective_ids = 0; int follow_symlinks = 1; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -307,7 +309,7 @@ os_access(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k return return_value; } -#if defined(HAVE_TTYNAME) +#if defined(HAVE_TTYNAME_R) PyDoc_STRVAR(os_ttyname__doc__, "ttyname($module, fd, /)\n" @@ -340,7 +342,7 @@ os_ttyname(PyObject *module, PyObject *arg) return return_value; } -#endif /* defined(HAVE_TTYNAME) */ +#endif /* defined(HAVE_TTYNAME_R) */ #if defined(HAVE_CTERMID) @@ -410,9 +412,10 @@ os_chdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("chdir", "path", 0, PATH_HAVE_FCHDIR); + path_t path = PATH_T_INITIALIZE_P("chdir", "path", 0, 0, 0, PATH_HAVE_FCHDIR); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -477,11 +480,13 @@ os_fchdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k PyObject *argsbuf[1]; int fd; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } return_value = os_fchdir_impl(module, fd); @@ -560,12 +565,13 @@ os_chmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("chmod", "path", 0, PATH_HAVE_FCHMOD); + path_t path = PATH_T_INITIALIZE_P("chmod", "path", 0, 0, 0, PATH_HAVE_FCHMOD); int mode; int dir_fd = DEFAULT_DIR_FD; int follow_symlinks = CHMOD_DEFAULT_FOLLOW_SYMLINKS; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -658,7 +664,8 @@ os_fchmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k int fd; int mode; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -725,10 +732,11 @@ os_lchmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k }; #undef KWTUPLE PyObject *argsbuf[2]; - path_t path = PATH_T_INITIALIZE("lchmod", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("lchmod", "path", 0, 0, 0, 0); int mode; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -802,11 +810,12 @@ os_chflags(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("chflags", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("chflags", "path", 0, 0, 0, 0); unsigned long flags; int follow_symlinks = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -884,10 +893,11 @@ os_lchflags(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject }; #undef KWTUPLE PyObject *argsbuf[2]; - path_t path = PATH_T_INITIALIZE("lchflags", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("lchflags", "path", 0, 0, 0, 0); unsigned long flags; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -954,9 +964,10 @@ os_chroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("chroot", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("chroot", "path", 0, 0, 0, 0); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1020,11 +1031,13 @@ os_fsync(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *argsbuf[1]; int fd; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } return_value = os_fsync_impl(module, fd); @@ -1103,11 +1116,13 @@ os_fdatasync(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *argsbuf[1]; int fd; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } return_value = os_fdatasync_impl(module, fd); @@ -1188,13 +1203,14 @@ os_chown(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; - path_t path = PATH_T_INITIALIZE("chown", "path", 0, PATH_HAVE_FCHOWN); + path_t path = PATH_T_INITIALIZE_P("chown", "path", 0, 0, 0, PATH_HAVE_FCHOWN); uid_t uid; gid_t gid; int dir_fd = DEFAULT_DIR_FD; int follow_symlinks = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1284,7 +1300,8 @@ os_fchown(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k uid_t uid; gid_t gid; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1353,11 +1370,12 @@ os_lchown(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k }; #undef KWTUPLE PyObject *argsbuf[3]; - path_t path = PATH_T_INITIALIZE("lchown", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("lchown", "path", 0, 0, 0, 0); uid_t uid; gid_t gid; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1474,13 +1492,14 @@ os_link(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t src = PATH_T_INITIALIZE("link", "src", 0, 0); - path_t dst = PATH_T_INITIALIZE("link", "dst", 0, 0); + path_t src = PATH_T_INITIALIZE_P("link", "src", 0, 0, 0, 0); + path_t dst = PATH_T_INITIALIZE_P("link", "dst", 0, 0, 0, 0); int src_dir_fd = DEFAULT_DIR_FD; int dst_dir_fd = DEFAULT_DIR_FD; int follow_symlinks = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1581,9 +1600,10 @@ os_listdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[1]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - path_t path = PATH_T_INITIALIZE("listdir", "path", 1, PATH_HAVE_FDOPENDIR); + path_t path = PATH_T_INITIALIZE_P("listdir", "path", 1, 0, 0, PATH_HAVE_FDOPENDIR); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1697,9 +1717,10 @@ os_listmounts(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t volume = PATH_T_INITIALIZE("listmounts", "volume", 0, 0); + path_t volume = PATH_T_INITIALIZE_P("listmounts", "volume", 0, 0, 0, 0); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1761,9 +1782,10 @@ os__path_isdevdrive(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_path_isdevdrive", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_path_isdevdrive", "path", 0, 0, 0, 0); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1798,7 +1820,7 @@ static PyObject * os__getfullpathname(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("_getfullpathname", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_getfullpathname", "path", 0, 0, 0, 0); if (!path_converter(arg, &path)) { goto exit; @@ -1832,7 +1854,7 @@ static PyObject * os__getfinalpathname(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("_getfinalpathname", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_getfinalpathname", "path", 0, 0, 0, 0); if (!path_converter(arg, &path)) { goto exit; @@ -1866,7 +1888,7 @@ static PyObject * os__findfirstfile(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("_findfirstfile", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_findfirstfile", "path", 0, 0, 0, 0); if (!path_converter(arg, &path)) { goto exit; @@ -1926,9 +1948,10 @@ os__getvolumepathname(PyObject *module, PyObject *const *args, Py_ssize_t nargs, }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_getvolumepathname", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_getvolumepathname", "path", 0, 0, 0, 0); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1990,9 +2013,10 @@ os__path_splitroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_path_splitroot", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_path_splitroot", "path", 0, 0, 0, 0); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2012,6 +2036,146 @@ os__path_splitroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py #if defined(MS_WINDOWS) +PyDoc_STRVAR(os__path_exists__doc__, +"_path_exists($module, /, path)\n" +"--\n" +"\n" +"Test whether a path exists. Returns False for broken symbolic links."); + +#define OS__PATH_EXISTS_METHODDEF \ + {"_path_exists", _PyCFunction_CAST(os__path_exists), METH_FASTCALL|METH_KEYWORDS, os__path_exists__doc__}, + +static int +os__path_exists_impl(PyObject *module, path_t *path); + +static PyObject * +os__path_exists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_path_exists", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + path_t path = PATH_T_INITIALIZE_P("_path_exists", "path", 0, 0, 1, 1); + int _return_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_exists_impl(module, &path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + /* Cleanup for path */ + path_cleanup(&path); + + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(os__path_lexists__doc__, +"_path_lexists($module, /, path)\n" +"--\n" +"\n" +"Test whether a path exists. Returns True for broken symbolic links."); + +#define OS__PATH_LEXISTS_METHODDEF \ + {"_path_lexists", _PyCFunction_CAST(os__path_lexists), METH_FASTCALL|METH_KEYWORDS, os__path_lexists__doc__}, + +static int +os__path_lexists_impl(PyObject *module, path_t *path); + +static PyObject * +os__path_lexists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_path_lexists", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + path_t path = PATH_T_INITIALIZE_P("_path_lexists", "path", 0, 0, 1, 1); + int _return_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_lexists_impl(module, &path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + /* Cleanup for path */ + path_cleanup(&path); + + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + PyDoc_STRVAR(os__path_isdir__doc__, "_path_isdir($module, /, s)\n" "--\n" @@ -2021,8 +2185,8 @@ PyDoc_STRVAR(os__path_isdir__doc__, #define OS__PATH_ISDIR_METHODDEF \ {"_path_isdir", _PyCFunction_CAST(os__path_isdir), METH_FASTCALL|METH_KEYWORDS, os__path_isdir__doc__}, -static PyObject * -os__path_isdir_impl(PyObject *module, PyObject *s); +static int +os__path_isdir_impl(PyObject *module, path_t *path); static PyObject * os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2037,7 +2201,7 @@ os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(s), }, + .ob_item = { _Py_LATIN1_CHR('s'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -2054,16 +2218,27 @@ os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *s; + path_t path = PATH_T_INITIALIZE_P("_path_isdir", "path", 0, 0, 1, 1); + int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } - s = args[0]; - return_value = os__path_isdir_impl(module, s); + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_isdir_impl(module, &path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2080,8 +2255,8 @@ PyDoc_STRVAR(os__path_isfile__doc__, #define OS__PATH_ISFILE_METHODDEF \ {"_path_isfile", _PyCFunction_CAST(os__path_isfile), METH_FASTCALL|METH_KEYWORDS, os__path_isfile__doc__}, -static PyObject * -os__path_isfile_impl(PyObject *module, PyObject *path); +static int +os__path_isfile_impl(PyObject *module, path_t *path); static PyObject * os__path_isfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2113,16 +2288,27 @@ os__path_isfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE_P("_path_isfile", "path", 0, 0, 1, 1); + int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - return_value = os__path_isfile_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_isfile_impl(module, &path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2130,20 +2316,20 @@ os__path_isfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj #if defined(MS_WINDOWS) -PyDoc_STRVAR(os__path_exists__doc__, -"_path_exists($module, /, path)\n" +PyDoc_STRVAR(os__path_islink__doc__, +"_path_islink($module, /, path)\n" "--\n" "\n" -"Test whether a path exists. Returns False for broken symbolic links"); +"Test whether a path is a symbolic link"); -#define OS__PATH_EXISTS_METHODDEF \ - {"_path_exists", _PyCFunction_CAST(os__path_exists), METH_FASTCALL|METH_KEYWORDS, os__path_exists__doc__}, +#define OS__PATH_ISLINK_METHODDEF \ + {"_path_islink", _PyCFunction_CAST(os__path_islink), METH_FASTCALL|METH_KEYWORDS, os__path_islink__doc__}, -static PyObject * -os__path_exists_impl(PyObject *module, PyObject *path); +static int +os__path_islink_impl(PyObject *module, path_t *path); static PyObject * -os__path_exists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +os__path_islink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -2167,21 +2353,32 @@ os__path_exists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj static const char * const _keywords[] = {"path", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "_path_exists", + .fname = "_path_islink", .kwtuple = KWTUPLE, }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE_P("_path_islink", "path", 0, 0, 1, 1); + int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - return_value = os__path_exists_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_islink_impl(module, &path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2189,20 +2386,20 @@ os__path_exists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj #if defined(MS_WINDOWS) -PyDoc_STRVAR(os__path_islink__doc__, -"_path_islink($module, /, path)\n" +PyDoc_STRVAR(os__path_isjunction__doc__, +"_path_isjunction($module, /, path)\n" "--\n" "\n" -"Test whether a path is a symbolic link"); +"Test whether a path is a junction"); -#define OS__PATH_ISLINK_METHODDEF \ - {"_path_islink", _PyCFunction_CAST(os__path_islink), METH_FASTCALL|METH_KEYWORDS, os__path_islink__doc__}, +#define OS__PATH_ISJUNCTION_METHODDEF \ + {"_path_isjunction", _PyCFunction_CAST(os__path_isjunction), METH_FASTCALL|METH_KEYWORDS, os__path_isjunction__doc__}, -static PyObject * -os__path_islink_impl(PyObject *module, PyObject *path); +static int +os__path_isjunction_impl(PyObject *module, path_t *path); static PyObject * -os__path_islink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +os__path_isjunction(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -2226,37 +2423,111 @@ os__path_islink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj static const char * const _keywords[] = {"path", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "_path_islink", + .fname = "_path_isjunction", .kwtuple = KWTUPLE, }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE_P("_path_isjunction", "path", 0, 0, 1, 1); + int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - return_value = os__path_islink_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_isjunction_impl(module, &path); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } #endif /* defined(MS_WINDOWS) */ +PyDoc_STRVAR(os__path_splitroot_ex__doc__, +"_path_splitroot_ex($module, /, p)\n" +"--\n" +"\n" +"Split a pathname into drive, root and tail.\n" +"\n" +"The tail contains anything after the root."); + +#define OS__PATH_SPLITROOT_EX_METHODDEF \ + {"_path_splitroot_ex", _PyCFunction_CAST(os__path_splitroot_ex), METH_FASTCALL|METH_KEYWORDS, os__path_splitroot_ex__doc__}, + +static PyObject * +os__path_splitroot_ex_impl(PyObject *module, path_t *path); + +static PyObject * +os__path_splitroot_ex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { _Py_LATIN1_CHR('p'), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"p", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_path_splitroot_ex", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + path_t path = PATH_T_INITIALIZE("_path_splitroot_ex", "path", 0, 1, 1, 0, 0); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!path_converter(args[0], &path)) { + goto exit; + } + return_value = os__path_splitroot_ex_impl(module, &path); + +exit: + /* Cleanup for path */ + path_cleanup(&path); + + return return_value; +} + PyDoc_STRVAR(os__path_normpath__doc__, "_path_normpath($module, /, path)\n" "--\n" "\n" -"Basic path normalization."); +"Normalize path, eliminating double slashes, etc."); #define OS__PATH_NORMPATH_METHODDEF \ {"_path_normpath", _PyCFunction_CAST(os__path_normpath), METH_FASTCALL|METH_KEYWORDS, os__path_normpath__doc__}, static PyObject * -os__path_normpath_impl(PyObject *module, PyObject *path); +os__path_normpath_impl(PyObject *module, path_t *path); static PyObject * os__path_normpath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2288,16 +2559,22 @@ os__path_normpath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE("_path_normpath", "path", 0, 1, 1, 0, 0); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - return_value = os__path_normpath_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + return_value = os__path_normpath_impl(module, &path); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2352,11 +2629,12 @@ os_mkdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("mkdir", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("mkdir", "path", 0, 0, 0, 0); int mode = 511; int dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2471,7 +2749,8 @@ os_getpriority(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje int which; int who; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2539,7 +2818,8 @@ os_setpriority(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje int who; int priority; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2613,12 +2893,13 @@ os_rename(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t src = PATH_T_INITIALIZE("rename", "src", 0, 0); - path_t dst = PATH_T_INITIALIZE("rename", "dst", 0, 0); + path_t src = PATH_T_INITIALIZE_P("rename", "src", 0, 0, 0, 0); + path_t dst = PATH_T_INITIALIZE_P("rename", "dst", 0, 0, 0, 0); int src_dir_fd = DEFAULT_DIR_FD; int dst_dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2704,12 +2985,13 @@ os_replace(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t src = PATH_T_INITIALIZE("replace", "src", 0, 0); - path_t dst = PATH_T_INITIALIZE("replace", "dst", 0, 0); + path_t src = PATH_T_INITIALIZE_P("replace", "src", 0, 0, 0, 0); + path_t dst = PATH_T_INITIALIZE_P("replace", "dst", 0, 0, 0, 0); int src_dir_fd = DEFAULT_DIR_FD; int dst_dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2793,10 +3075,11 @@ os_rmdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("rmdir", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("rmdir", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2866,7 +3149,8 @@ os_system(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k const wchar_t *command = NULL; long _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -2940,7 +3224,8 @@ os_system(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k PyObject *command = NULL; long _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3042,10 +3327,11 @@ os_unlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("unlink", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("unlink", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3116,10 +3402,11 @@ os_remove(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("remove", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("remove", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3234,13 +3521,14 @@ os_utime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("utime", "path", 0, PATH_UTIME_HAVE_FD); + path_t path = PATH_T_INITIALIZE_P("utime", "path", 0, 0, 0, PATH_UTIME_HAVE_FD); PyObject *times = Py_None; PyObject *ns = NULL; int dir_fd = DEFAULT_DIR_FD; int follow_symlinks = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3332,7 +3620,8 @@ os__exit(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *argsbuf[1]; int status; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3369,7 +3658,7 @@ static PyObject * os_execv(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("execv", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("execv", "path", 0, 0, 0, 0); PyObject *argv; if (!_PyArg_CheckPositional("execv", nargs, 2, 2)) { @@ -3441,11 +3730,12 @@ os_execve(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k }; #undef KWTUPLE PyObject *argsbuf[3]; - path_t path = PATH_T_INITIALIZE("execve", "path", 0, PATH_HAVE_FEXECVE); + path_t path = PATH_T_INITIALIZE_P("execve", "path", 0, 0, 0, PATH_HAVE_FEXECVE); PyObject *argv; PyObject *env; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3537,7 +3827,7 @@ os_posix_spawn(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #undef KWTUPLE PyObject *argsbuf[10]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; - path_t path = PATH_T_INITIALIZE("posix_spawn", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("posix_spawn", "path", 0, 0, 0, 0); PyObject *argv; PyObject *env; PyObject *file_actions = NULL; @@ -3548,7 +3838,8 @@ os_posix_spawn(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *setsigdef = NULL; PyObject *scheduler = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3687,7 +3978,7 @@ os_posix_spawnp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj #undef KWTUPLE PyObject *argsbuf[10]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; - path_t path = PATH_T_INITIALIZE("posix_spawnp", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("posix_spawnp", "path", 0, 0, 0, 0); PyObject *argv; PyObject *env; PyObject *file_actions = NULL; @@ -3698,7 +3989,8 @@ os_posix_spawnp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *setsigdef = NULL; PyObject *scheduler = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -3791,7 +4083,7 @@ os_spawnv(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int mode; - path_t path = PATH_T_INITIALIZE("spawnv", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("spawnv", "path", 0, 0, 0, 0); PyObject *argv; if (!_PyArg_CheckPositional("spawnv", nargs, 3, 3)) { @@ -3845,7 +4137,7 @@ os_spawnve(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int mode; - path_t path = PATH_T_INITIALIZE("spawnve", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("spawnve", "path", 0, 0, 0, 0); PyObject *argv; PyObject *env; @@ -3934,7 +4226,8 @@ os_register_at_fork(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyObject *after_in_child = NULL; PyObject *after_in_parent = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -4057,7 +4350,8 @@ os_sched_get_priority_max(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *argsbuf[1]; int policy; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -4119,7 +4413,8 @@ os_sched_get_priority_min(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *argsbuf[1]; int policy; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -4157,7 +4452,8 @@ os_sched_getscheduler(PyObject *module, PyObject *arg) PyObject *return_value = NULL; pid_t pid; - if (!PyArg_Parse(arg, "" _Py_PARSE_PID ":sched_getscheduler", &pid)) { + pid = PyLong_AsPid(arg); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { goto exit; } return_value = os_sched_getscheduler_impl(module, pid); @@ -4216,7 +4512,8 @@ os_sched_param(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t nargs = PyTuple_GET_SIZE(args); PyObject *sched_priority; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -4255,10 +4552,18 @@ os_sched_setscheduler(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int policy; PyObject *param_obj; - if (!_PyArg_ParseStack(args, nargs, "" _Py_PARSE_PID "iO:sched_setscheduler", - &pid, &policy, ¶m_obj)) { + if (!_PyArg_CheckPositional("sched_setscheduler", nargs, 3, 3)) { goto exit; } + pid = PyLong_AsPid(args[0]); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { + goto exit; + } + policy = PyLong_AsInt(args[1]); + if (policy == -1 && PyErr_Occurred()) { + goto exit; + } + param_obj = args[2]; return_value = os_sched_setscheduler_impl(module, pid, policy, param_obj); exit: @@ -4290,7 +4595,8 @@ os_sched_getparam(PyObject *module, PyObject *arg) PyObject *return_value = NULL; pid_t pid; - if (!PyArg_Parse(arg, "" _Py_PARSE_PID ":sched_getparam", &pid)) { + pid = PyLong_AsPid(arg); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { goto exit; } return_value = os_sched_getparam_impl(module, pid); @@ -4325,10 +4631,14 @@ os_sched_setparam(PyObject *module, PyObject *const *args, Py_ssize_t nargs) pid_t pid; PyObject *param_obj; - if (!_PyArg_ParseStack(args, nargs, "" _Py_PARSE_PID "O:sched_setparam", - &pid, ¶m_obj)) { + if (!_PyArg_CheckPositional("sched_setparam", nargs, 2, 2)) { goto exit; } + pid = PyLong_AsPid(args[0]); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { + goto exit; + } + param_obj = args[1]; return_value = os_sched_setparam_impl(module, pid, param_obj); exit: @@ -4360,7 +4670,8 @@ os_sched_rr_get_interval(PyObject *module, PyObject *arg) pid_t pid; double _return_value; - if (!PyArg_Parse(arg, "" _Py_PARSE_PID ":sched_rr_get_interval", &pid)) { + pid = PyLong_AsPid(arg); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { goto exit; } _return_value = os_sched_rr_get_interval_impl(module, pid); @@ -4420,10 +4731,14 @@ os_sched_setaffinity(PyObject *module, PyObject *const *args, Py_ssize_t nargs) pid_t pid; PyObject *mask; - if (!_PyArg_ParseStack(args, nargs, "" _Py_PARSE_PID "O:sched_setaffinity", - &pid, &mask)) { + if (!_PyArg_CheckPositional("sched_setaffinity", nargs, 2, 2)) { goto exit; } + pid = PyLong_AsPid(args[0]); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { + goto exit; + } + mask = args[1]; return_value = os_sched_setaffinity_impl(module, pid, mask); exit: @@ -4454,7 +4769,8 @@ os_sched_getaffinity(PyObject *module, PyObject *arg) PyObject *return_value = NULL; pid_t pid; - if (!PyArg_Parse(arg, "" _Py_PARSE_PID ":sched_getaffinity", &pid)) { + pid = PyLong_AsPid(arg); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { goto exit; } return_value = os_sched_getaffinity_impl(module, pid); @@ -4531,7 +4847,8 @@ os_grantpt(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = os_grantpt_impl(module, fd); @@ -4567,7 +4884,8 @@ os_unlockpt(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = os_unlockpt_impl(module, fd); @@ -4604,7 +4922,8 @@ os_ptsname(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = os_ptsname_impl(module, fd); @@ -4664,7 +4983,8 @@ os_login_tty(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = os_login_tty_impl(module, fd); @@ -5049,14 +5369,20 @@ os_getpgid(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * static const char * const _keywords[] = {"pid", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .format = "" _Py_PARSE_PID ":getpgid", + .fname = "getpgid", .kwtuple = KWTUPLE, }; #undef KWTUPLE + PyObject *argsbuf[1]; pid_t pid; - if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, - &pid)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + pid = PyLong_AsPid(args[0]); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { goto exit; } return_value = os_getpgid_impl(module, pid); @@ -5201,10 +5527,25 @@ os_kill(PyObject *module, PyObject *const *args, Py_ssize_t nargs) pid_t pid; Py_ssize_t signal; - if (!_PyArg_ParseStack(args, nargs, "" _Py_PARSE_PID "n:kill", - &pid, &signal)) { + if (!_PyArg_CheckPositional("kill", nargs, 2, 2)) { + goto exit; + } + pid = PyLong_AsPid(args[0]); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { goto exit; } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + signal = ival; + } return_value = os_kill_impl(module, pid, signal); exit: @@ -5234,8 +5575,15 @@ os_killpg(PyObject *module, PyObject *const *args, Py_ssize_t nargs) pid_t pgid; int signal; - if (!_PyArg_ParseStack(args, nargs, "" _Py_PARSE_PID "i:killpg", - &pgid, &signal)) { + if (!_PyArg_CheckPositional("killpg", nargs, 2, 2)) { + goto exit; + } + pgid = PyLong_AsPid(args[0]); + if (pgid == (pid_t)(-1) && PyErr_Occurred()) { + goto exit; + } + signal = PyLong_AsInt(args[1]); + if (signal == -1 && PyErr_Occurred()) { goto exit; } return_value = os_killpg_impl(module, pgid, signal); @@ -5540,7 +5888,8 @@ os_wait3(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *argsbuf[1]; int options; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -5598,15 +5947,25 @@ os_wait4(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw static const char * const _keywords[] = {"pid", "options", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .format = "" _Py_PARSE_PID "i:wait4", + .fname = "wait4", .kwtuple = KWTUPLE, }; #undef KWTUPLE + PyObject *argsbuf[2]; pid_t pid; int options; - if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, - &pid, &options)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + pid = PyLong_AsPid(args[0]); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { + goto exit; + } + options = PyLong_AsInt(args[1]); + if (options == -1 && PyErr_Occurred()) { goto exit; } return_value = os_wait4_impl(module, pid, options); @@ -5650,8 +6009,18 @@ os_waitid(PyObject *module, PyObject *const *args, Py_ssize_t nargs) id_t id; int options; - if (!_PyArg_ParseStack(args, nargs, "i" _Py_PARSE_PID "i:waitid", - &idtype, &id, &options)) { + if (!_PyArg_CheckPositional("waitid", nargs, 3, 3)) { + goto exit; + } + if (!idtype_t_converter(args[0], &idtype)) { + goto exit; + } + id = (id_t)PyLong_AsPid(args[1]); + if (id == (id_t)(-1) && PyErr_Occurred()) { + goto exit; + } + options = PyLong_AsInt(args[2]); + if (options == -1 && PyErr_Occurred()) { goto exit; } return_value = os_waitid_impl(module, idtype, id, options); @@ -5688,8 +6057,15 @@ os_waitpid(PyObject *module, PyObject *const *args, Py_ssize_t nargs) pid_t pid; int options; - if (!_PyArg_ParseStack(args, nargs, "" _Py_PARSE_PID "i:waitpid", - &pid, &options)) { + if (!_PyArg_CheckPositional("waitpid", nargs, 2, 2)) { + goto exit; + } + pid = PyLong_AsPid(args[0]); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { + goto exit; + } + options = PyLong_AsInt(args[1]); + if (options == -1 && PyErr_Occurred()) { goto exit; } return_value = os_waitpid_impl(module, pid, options); @@ -5726,8 +6102,15 @@ os_waitpid(PyObject *module, PyObject *const *args, Py_ssize_t nargs) intptr_t pid; int options; - if (!_PyArg_ParseStack(args, nargs, "" _Py_PARSE_INTPTR "i:waitpid", - &pid, &options)) { + if (!_PyArg_CheckPositional("waitpid", nargs, 2, 2)) { + goto exit; + } + pid = (intptr_t)PyLong_AsVoidPtr(args[0]); + if (!pid && PyErr_Occurred()) { + goto exit; + } + options = PyLong_AsInt(args[1]); + if (options == -1 && PyErr_Occurred()) { goto exit; } return_value = os_waitpid_impl(module, pid, options); @@ -5763,7 +6146,7 @@ os_wait(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* defined(HAVE_WAIT) */ -#if (defined(__linux__) && defined(__NR_pidfd_open)) +#if (defined(__linux__) && defined(__NR_pidfd_open) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) PyDoc_STRVAR(os_pidfd_open__doc__, "pidfd_open($module, /, pid, flags=0)\n" @@ -5805,24 +6188,38 @@ os_pidfd_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec static const char * const _keywords[] = {"pid", "flags", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .format = "" _Py_PARSE_PID "|O&:pidfd_open", + .fname = "pidfd_open", .kwtuple = KWTUPLE, }; #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; pid_t pid; unsigned int flags = 0; - if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, - &pid, _PyLong_UnsignedInt_Converter, &flags)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + pid = PyLong_AsPid(args[0]); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { goto exit; } + if (!noptargs) { + goto skip_optional_pos; + } + if (!_PyLong_UnsignedInt_Converter(args[1], &flags)) { + goto exit; + } +skip_optional_pos: return_value = os_pidfd_open_impl(module, pid, flags); exit: return return_value; } -#endif /* (defined(__linux__) && defined(__NR_pidfd_open)) */ +#endif /* (defined(__linux__) && defined(__NR_pidfd_open) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) */ #if defined(HAVE_SETNS) @@ -5877,11 +6274,13 @@ os_setns(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw int fd; int nstype = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!noptargs) { @@ -5949,7 +6348,8 @@ os_unshare(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * PyObject *argsbuf[1]; int flags; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -6016,10 +6416,11 @@ os_readlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("readlink", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("readlink", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -6100,12 +6501,13 @@ os_symlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t src = PATH_T_INITIALIZE("symlink", "src", 0, 0); - path_t dst = PATH_T_INITIALIZE("symlink", "dst", 0, 0); + path_t src = PATH_T_INITIALIZE_P("symlink", "src", 0, 0, 0, 0); + path_t dst = PATH_T_INITIALIZE_P("symlink", "dst", 0, 0, 0, 0); int target_is_directory = 0; int dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -6156,7 +6558,7 @@ PyDoc_STRVAR(os_times__doc__, "\n" "The object returned behaves like a named tuple with these fields:\n" " (utime, stime, cutime, cstime, elapsed_time)\n" -"All fields are floating point numbers."); +"All fields are floating-point numbers."); #define OS_TIMES_METHODDEF \ {"times", (PyCFunction)os_times, METH_NOARGS, os_times__doc__}, @@ -6234,7 +6636,8 @@ os_timerfd_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO int clockid; int flags = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -6279,8 +6682,8 @@ PyDoc_STRVAR(os_timerfd_settime__doc__, {"timerfd_settime", _PyCFunction_CAST(os_timerfd_settime), METH_FASTCALL|METH_KEYWORDS, os_timerfd_settime__doc__}, static PyObject * -os_timerfd_settime_impl(PyObject *module, int fd, int flags, double initial, - double interval); +os_timerfd_settime_impl(PyObject *module, int fd, int flags, + double initial_double, double interval_double); static PyObject * os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -6315,14 +6718,16 @@ os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; int fd; int flags = 0; - double initial = 0.0; - double interval = 0.0; + double initial_double = 0.0; + double interval_double = 0.0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!noptargs) { @@ -6339,12 +6744,12 @@ os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py } if (args[2]) { if (PyFloat_CheckExact(args[2])) { - initial = PyFloat_AS_DOUBLE(args[2]); + initial_double = PyFloat_AS_DOUBLE(args[2]); } else { - initial = PyFloat_AsDouble(args[2]); - if (initial == -1.0 && PyErr_Occurred()) { + initial_double = PyFloat_AsDouble(args[2]); + if (initial_double == -1.0 && PyErr_Occurred()) { goto exit; } } @@ -6353,17 +6758,17 @@ os_timerfd_settime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py } } if (PyFloat_CheckExact(args[3])) { - interval = PyFloat_AS_DOUBLE(args[3]); + interval_double = PyFloat_AS_DOUBLE(args[3]); } else { - interval = PyFloat_AsDouble(args[3]); - if (interval == -1.0 && PyErr_Occurred()) { + interval_double = PyFloat_AsDouble(args[3]); + if (interval_double == -1.0 && PyErr_Occurred()) { goto exit; } } skip_optional_kwonly: - return_value = os_timerfd_settime_impl(module, fd, flags, initial, interval); + return_value = os_timerfd_settime_impl(module, fd, flags, initial_double, interval_double); exit: return return_value; @@ -6431,11 +6836,13 @@ os_timerfd_settime_ns(PyObject *module, PyObject *const *args, Py_ssize_t nargs, long long initial = 0; long long interval = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!noptargs) { @@ -6495,7 +6902,8 @@ os_timerfd_gettime(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = os_timerfd_gettime_impl(module, fd); @@ -6529,7 +6937,8 @@ os_timerfd_gettime_ns(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = os_timerfd_gettime_ns_impl(module, fd); @@ -6560,7 +6969,8 @@ os_getsid(PyObject *module, PyObject *arg) PyObject *return_value = NULL; pid_t pid; - if (!PyArg_Parse(arg, "" _Py_PARSE_PID ":getsid", &pid)) { + pid = PyLong_AsPid(arg); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { goto exit; } return_value = os_getsid_impl(module, pid); @@ -6614,8 +7024,15 @@ os_setpgid(PyObject *module, PyObject *const *args, Py_ssize_t nargs) pid_t pid; pid_t pgrp; - if (!_PyArg_ParseStack(args, nargs, "" _Py_PARSE_PID "" _Py_PARSE_PID ":setpgid", - &pid, &pgrp)) { + if (!_PyArg_CheckPositional("setpgid", nargs, 2, 2)) { + goto exit; + } + pid = PyLong_AsPid(args[0]); + if (pid == (pid_t)(-1) && PyErr_Occurred()) { + goto exit; + } + pgrp = PyLong_AsPid(args[1]); + if (pgrp == (pid_t)(-1) && PyErr_Occurred()) { goto exit; } return_value = os_setpgid_impl(module, pid, pgrp); @@ -6679,8 +7096,15 @@ os_tcsetpgrp(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; pid_t pgid; - if (!_PyArg_ParseStack(args, nargs, "i" _Py_PARSE_PID ":tcsetpgrp", - &fd, &pgid)) { + if (!_PyArg_CheckPositional("tcsetpgrp", nargs, 2, 2)) { + goto exit; + } + fd = PyLong_AsInt(args[0]); + if (fd == -1 && PyErr_Occurred()) { + goto exit; + } + pgid = PyLong_AsPid(args[1]); + if (pgid == (pid_t)(-1) && PyErr_Occurred()) { goto exit; } return_value = os_tcsetpgrp_impl(module, fd, pgid); @@ -6739,13 +7163,14 @@ os_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("open", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("open", "path", 0, 0, 0, 0); int flags; int mode = 511; int dir_fd = DEFAULT_DIR_FD; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -6833,7 +7258,8 @@ os_close(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *argsbuf[1]; int fd; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -6966,7 +7392,8 @@ os_dup2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn int inheritable = 1; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -7150,6 +7577,62 @@ os_read(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(os_readinto__doc__, +"readinto($module, fd, buffer, /)\n" +"--\n" +"\n" +"Read into a buffer object from a file descriptor.\n" +"\n" +"The buffer should be mutable and bytes-like. On success, returns the number of\n" +"bytes read. Less bytes may be read than the size of the buffer. The underlying\n" +"system call will be retried when interrupted by a signal, unless the signal\n" +"handler raises an exception. Other errors will not be retried and an error will\n" +"be raised.\n" +"\n" +"Returns 0 if *fd* is at end of file or if the provided *buffer* has length 0\n" +"(which can be used to check for errors without reading data). Never returns\n" +"negative."); + +#define OS_READINTO_METHODDEF \ + {"readinto", _PyCFunction_CAST(os_readinto), METH_FASTCALL, os_readinto__doc__}, + +static Py_ssize_t +os_readinto_impl(PyObject *module, int fd, Py_buffer *buffer); + +static PyObject * +os_readinto(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int fd; + Py_buffer buffer = {NULL, NULL}; + Py_ssize_t _return_value; + + if (!_PyArg_CheckPositional("readinto", nargs, 2, 2)) { + goto exit; + } + fd = PyLong_AsInt(args[0]); + if (fd == -1 && PyErr_Occurred()) { + goto exit; + } + if (PyObject_GetBuffer(args[1], &buffer, PyBUF_WRITABLE) < 0) { + _PyArg_BadArgument("readinto", "argument 2", "read-write bytes-like object", args[1]); + goto exit; + } + _return_value = os_readinto_impl(module, fd, &buffer); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + /* Cleanup for buffer */ + if (buffer.obj) { + PyBuffer_Release(&buffer); + } + + return return_value; +} + #if defined(HAVE_READV) PyDoc_STRVAR(os_readv__doc__, @@ -7426,7 +7909,8 @@ os_sendfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *trailers = NULL; int flags = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 7, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 4, /*maxpos*/ 7, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -7528,7 +8012,8 @@ os_sendfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *trailers = NULL; int flags = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 7, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 4, /*maxpos*/ 7, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -7633,7 +8118,8 @@ os_sendfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *offobj; Py_ssize_t count; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 4, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -7758,7 +8244,8 @@ os_fstat(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *argsbuf[1]; int fd; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -8111,7 +8598,8 @@ os_copy_file_range(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py PyObject *offset_src = Py_None; PyObject *offset_dst = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 5, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 5, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -8226,7 +8714,8 @@ os_splice(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k PyObject *offset_dst = Py_None; unsigned int flags = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 6, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 6, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -8327,11 +8816,12 @@ os_mkfifo(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("mkfifo", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("mkfifo", "path", 0, 0, 0, 0); int mode = 438; int dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -8427,12 +8917,13 @@ os_mknod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("mknod", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("mknod", "path", 0, 0, 0, 0); int mode = 384; dev_t device = 0; int dir_fd = DEFAULT_DIR_FD; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -8489,7 +8980,7 @@ PyDoc_STRVAR(os_major__doc__, #define OS_MAJOR_METHODDEF \ {"major", (PyCFunction)os_major, METH_O, os_major__doc__}, -static unsigned int +static PyObject * os_major_impl(PyObject *module, dev_t device); static PyObject * @@ -8497,16 +8988,11 @@ os_major(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; dev_t device; - unsigned int _return_value; if (!_Py_Dev_Converter(arg, &device)) { goto exit; } - _return_value = os_major_impl(module, device); - if ((_return_value == (unsigned int)-1) && PyErr_Occurred()) { - goto exit; - } - return_value = PyLong_FromUnsignedLong((unsigned long)_return_value); + return_value = os_major_impl(module, device); exit: return return_value; @@ -8525,7 +9011,7 @@ PyDoc_STRVAR(os_minor__doc__, #define OS_MINOR_METHODDEF \ {"minor", (PyCFunction)os_minor, METH_O, os_minor__doc__}, -static unsigned int +static PyObject * os_minor_impl(PyObject *module, dev_t device); static PyObject * @@ -8533,16 +9019,11 @@ os_minor(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; dev_t device; - unsigned int _return_value; if (!_Py_Dev_Converter(arg, &device)) { goto exit; } - _return_value = os_minor_impl(module, device); - if ((_return_value == (unsigned int)-1) && PyErr_Occurred()) { - goto exit; - } - return_value = PyLong_FromUnsignedLong((unsigned long)_return_value); + return_value = os_minor_impl(module, device); exit: return return_value; @@ -8562,25 +9043,23 @@ PyDoc_STRVAR(os_makedev__doc__, {"makedev", _PyCFunction_CAST(os_makedev), METH_FASTCALL, os_makedev__doc__}, static dev_t -os_makedev_impl(PyObject *module, int major, int minor); +os_makedev_impl(PyObject *module, dev_t major, dev_t minor); static PyObject * os_makedev(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - int major; - int minor; + dev_t major; + dev_t minor; dev_t _return_value; if (!_PyArg_CheckPositional("makedev", nargs, 2, 2)) { goto exit; } - major = PyLong_AsInt(args[0]); - if (major == -1 && PyErr_Occurred()) { + if (!_Py_Dev_Converter(args[0], &major)) { goto exit; } - minor = PyLong_AsInt(args[1]); - if (minor == -1 && PyErr_Occurred()) { + if (!_Py_Dev_Converter(args[1], &minor)) { goto exit; } _return_value = os_makedev_impl(module, major, minor); @@ -8681,10 +9160,11 @@ os_truncate(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject }; #undef KWTUPLE PyObject *argsbuf[2]; - path_t path = PATH_T_INITIALIZE("truncate", "path", 0, PATH_HAVE_FTRUNCATE); + path_t path = PATH_T_INITIALIZE_P("truncate", "path", 0, 0, 0, PATH_HAVE_FTRUNCATE); Py_off_t length; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -9076,7 +9556,8 @@ os_WIFCONTINUED(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj int status; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -9143,7 +9624,8 @@ os_WIFSTOPPED(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec int status; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -9210,7 +9692,8 @@ os_WIFSIGNALED(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje int status; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -9277,7 +9760,8 @@ os_WIFEXITED(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject int status; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -9344,7 +9828,8 @@ os_WEXITSTATUS(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje int status; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -9411,7 +9896,8 @@ os_WTERMSIG(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject int status; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -9478,7 +9964,8 @@ os_WSTOPSIG(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject int status; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -9580,9 +10067,10 @@ os_statvfs(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("statvfs", "path", 0, PATH_HAVE_FSTATVFS); + path_t path = PATH_T_INITIALIZE_P("statvfs", "path", 0, 0, 0, PATH_HAVE_FSTATVFS); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -9644,9 +10132,10 @@ os__getdiskusage(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_getdiskusage", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_getdiskusage", "path", 0, 0, 0, 0); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -9691,10 +10180,11 @@ os_fpathconf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("fpathconf", nargs, 2, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } - if (!conv_path_confname(args[1], &name)) { + if (!conv_confname(module, args[1], &name, "pathconf_names")) { goto exit; } _return_value = os_fpathconf_impl(module, fd, name); @@ -9757,18 +10247,19 @@ os_pathconf(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject }; #undef KWTUPLE PyObject *argsbuf[2]; - path_t path = PATH_T_INITIALIZE("pathconf", "path", 0, PATH_HAVE_FPATHCONF); + path_t path = PATH_T_INITIALIZE_P("pathconf", "path", 0, 0, 0, PATH_HAVE_FPATHCONF); int name; long _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } if (!path_converter(args[0], &path)) { goto exit; } - if (!conv_path_confname(args[1], &name)) { + if (!conv_confname(module, args[1], &name, "pathconf_names")) { goto exit; } _return_value = os_pathconf_impl(module, &path, name); @@ -9806,7 +10297,7 @@ os_confstr(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int name; - if (!conv_confstr_confname(arg, &name)) { + if (!conv_confname(module, arg, &name, "confstr_names")) { goto exit; } return_value = os_confstr_impl(module, name); @@ -9838,7 +10329,7 @@ os_sysconf(PyObject *module, PyObject *arg) int name; long _return_value; - if (!conv_sysconf_confname(arg, &name)) { + if (!conv_confname(module, arg, &name, "sysconf_names")) { goto exit; } _return_value = os_sysconf_impl(module, name); @@ -9947,13 +10438,14 @@ os_startfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t filepath = PATH_T_INITIALIZE("startfile", "filepath", 0, 0); + path_t filepath = PATH_T_INITIALIZE_P("startfile", "filepath", 0, 0, 0, 0); const wchar_t *operation = NULL; const wchar_t *arguments = NULL; - path_t cwd = PATH_T_INITIALIZE("startfile", "cwd", 1, 0); + path_t cwd = PATH_T_INITIALIZE_P("startfile", "cwd", 1, 0, 0, 0); int show_cmd = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 5, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 5, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -10092,7 +10584,8 @@ os_device_encoding(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py PyObject *argsbuf[1]; int fd; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -10285,11 +10778,12 @@ os_getxattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("getxattr", "path", 0, 1); - path_t attribute = PATH_T_INITIALIZE("getxattr", "attribute", 0, 0); + path_t path = PATH_T_INITIALIZE_P("getxattr", "path", 0, 0, 0, 1); + path_t attribute = PATH_T_INITIALIZE_P("getxattr", "attribute", 0, 0, 0, 0); int follow_symlinks = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -10372,13 +10866,14 @@ os_setxattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; - path_t path = PATH_T_INITIALIZE("setxattr", "path", 0, 1); - path_t attribute = PATH_T_INITIALIZE("setxattr", "attribute", 0, 0); + path_t path = PATH_T_INITIALIZE_P("setxattr", "path", 0, 0, 0, 1); + path_t attribute = PATH_T_INITIALIZE_P("setxattr", "attribute", 0, 0, 0, 0); Py_buffer value = {NULL, NULL}; int flags = 0; int follow_symlinks = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -10480,11 +10975,12 @@ os_removexattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("removexattr", "path", 0, 1); - path_t attribute = PATH_T_INITIALIZE("removexattr", "attribute", 0, 0); + path_t path = PATH_T_INITIALIZE_P("removexattr", "path", 0, 0, 0, 1); + path_t attribute = PATH_T_INITIALIZE_P("removexattr", "attribute", 0, 0, 0, 0); int follow_symlinks = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -10566,10 +11062,11 @@ os_listxattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - path_t path = PATH_T_INITIALIZE("listxattr", "path", 1, 1); + path_t path = PATH_T_INITIALIZE_P("listxattr", "path", 1, 0, 0, 1); int follow_symlinks = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -10687,7 +11184,8 @@ os_memfd_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *name = NULL; unsigned int flags = MFD_CLOEXEC; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -10761,7 +11259,8 @@ os_eventfd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * unsigned int initval; int flags = EFD_CLOEXEC; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -10830,11 +11329,13 @@ os_eventfd_read(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *argsbuf[1]; int fd; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } return_value = os_eventfd_read_impl(module, fd); @@ -10892,11 +11393,13 @@ os_eventfd_write(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb int fd; unsigned long long value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!_PyLong_UnsignedLongLong_Converter(args[1], &value)) { @@ -11071,7 +11574,8 @@ os_get_handle_inheritable(PyObject *module, PyObject *arg) intptr_t handle; int _return_value; - if (!PyArg_Parse(arg, "" _Py_PARSE_INTPTR ":get_handle_inheritable", &handle)) { + handle = (intptr_t)PyLong_AsVoidPtr(arg); + if (!handle && PyErr_Occurred()) { goto exit; } _return_value = os_get_handle_inheritable_impl(module, handle); @@ -11108,8 +11612,15 @@ os_set_handle_inheritable(PyObject *module, PyObject *const *args, Py_ssize_t na intptr_t handle; int inheritable; - if (!_PyArg_ParseStack(args, nargs, "" _Py_PARSE_INTPTR "p:set_handle_inheritable", - &handle, &inheritable)) { + if (!_PyArg_CheckPositional("set_handle_inheritable", nargs, 2, 2)) { + goto exit; + } + handle = (intptr_t)PyLong_AsVoidPtr(args[0]); + if (!handle && PyErr_Occurred()) { + goto exit; + } + inheritable = PyObject_IsTrue(args[1]); + if (inheritable < 0) { goto exit; } return_value = os_set_handle_inheritable_impl(module, handle, inheritable); @@ -11207,7 +11718,7 @@ static int os_DirEntry_is_symlink_impl(DirEntry *self, PyTypeObject *defining_class); static PyObject * -os_DirEntry_is_symlink(DirEntry *self, PyTypeObject *defining_class, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +os_DirEntry_is_symlink(PyObject *self, PyTypeObject *defining_class, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; int _return_value; @@ -11216,7 +11727,7 @@ os_DirEntry_is_symlink(DirEntry *self, PyTypeObject *defining_class, PyObject *c PyErr_SetString(PyExc_TypeError, "is_symlink() takes no arguments"); goto exit; } - _return_value = os_DirEntry_is_symlink_impl(self, defining_class); + _return_value = os_DirEntry_is_symlink_impl((DirEntry *)self, defining_class); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -11239,12 +11750,12 @@ static int os_DirEntry_is_junction_impl(DirEntry *self); static PyObject * -os_DirEntry_is_junction(DirEntry *self, PyObject *Py_UNUSED(ignored)) +os_DirEntry_is_junction(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; int _return_value; - _return_value = os_DirEntry_is_junction_impl(self); + _return_value = os_DirEntry_is_junction_impl((DirEntry *)self); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -11268,7 +11779,7 @@ os_DirEntry_stat_impl(DirEntry *self, PyTypeObject *defining_class, int follow_symlinks); static PyObject * -os_DirEntry_stat(DirEntry *self, PyTypeObject *defining_class, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +os_DirEntry_stat(PyObject *self, PyTypeObject *defining_class, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -11300,7 +11811,8 @@ os_DirEntry_stat(DirEntry *self, PyTypeObject *defining_class, PyObject *const * Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; int follow_symlinks = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -11312,7 +11824,7 @@ os_DirEntry_stat(DirEntry *self, PyTypeObject *defining_class, PyObject *const * goto exit; } skip_optional_kwonly: - return_value = os_DirEntry_stat_impl(self, defining_class, follow_symlinks); + return_value = os_DirEntry_stat_impl((DirEntry *)self, defining_class, follow_symlinks); exit: return return_value; @@ -11332,7 +11844,7 @@ os_DirEntry_is_dir_impl(DirEntry *self, PyTypeObject *defining_class, int follow_symlinks); static PyObject * -os_DirEntry_is_dir(DirEntry *self, PyTypeObject *defining_class, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +os_DirEntry_is_dir(PyObject *self, PyTypeObject *defining_class, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -11365,7 +11877,8 @@ os_DirEntry_is_dir(DirEntry *self, PyTypeObject *defining_class, PyObject *const int follow_symlinks = 1; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -11377,7 +11890,7 @@ os_DirEntry_is_dir(DirEntry *self, PyTypeObject *defining_class, PyObject *const goto exit; } skip_optional_kwonly: - _return_value = os_DirEntry_is_dir_impl(self, defining_class, follow_symlinks); + _return_value = os_DirEntry_is_dir_impl((DirEntry *)self, defining_class, follow_symlinks); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -11401,7 +11914,7 @@ os_DirEntry_is_file_impl(DirEntry *self, PyTypeObject *defining_class, int follow_symlinks); static PyObject * -os_DirEntry_is_file(DirEntry *self, PyTypeObject *defining_class, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +os_DirEntry_is_file(PyObject *self, PyTypeObject *defining_class, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -11434,7 +11947,8 @@ os_DirEntry_is_file(DirEntry *self, PyTypeObject *defining_class, PyObject *cons int follow_symlinks = 1; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -11446,7 +11960,7 @@ os_DirEntry_is_file(DirEntry *self, PyTypeObject *defining_class, PyObject *cons goto exit; } skip_optional_kwonly: - _return_value = os_DirEntry_is_file_impl(self, defining_class, follow_symlinks); + _return_value = os_DirEntry_is_file_impl((DirEntry *)self, defining_class, follow_symlinks); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -11469,9 +11983,9 @@ static PyObject * os_DirEntry_inode_impl(DirEntry *self); static PyObject * -os_DirEntry_inode(DirEntry *self, PyObject *Py_UNUSED(ignored)) +os_DirEntry_inode(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return os_DirEntry_inode_impl(self); + return os_DirEntry_inode_impl((DirEntry *)self); } PyDoc_STRVAR(os_DirEntry___fspath____doc__, @@ -11487,9 +12001,9 @@ static PyObject * os_DirEntry___fspath___impl(DirEntry *self); static PyObject * -os_DirEntry___fspath__(DirEntry *self, PyObject *Py_UNUSED(ignored)) +os_DirEntry___fspath__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return os_DirEntry___fspath___impl(self); + return os_DirEntry___fspath___impl((DirEntry *)self); } PyDoc_STRVAR(os_scandir__doc__, @@ -11541,9 +12055,10 @@ os_scandir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[1]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - path_t path = PATH_T_INITIALIZE("scandir", "path", 1, PATH_HAVE_FDOPENDIR); + path_t path = PATH_T_INITIALIZE_P("scandir", "path", 1, 0, 0, PATH_HAVE_FDOPENDIR); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -11611,7 +12126,8 @@ os_fspath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k PyObject *argsbuf[1]; PyObject *path; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -11670,7 +12186,8 @@ os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject Py_ssize_t size; int flags = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -11753,9 +12270,10 @@ os__add_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nargs, }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_add_dll_directory", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_add_dll_directory", "path", 0, 0, 0, 0); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -11823,7 +12341,8 @@ os__remove_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *argsbuf[1]; PyObject *cookie; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -11894,7 +12413,8 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *argsbuf[1]; PyObject *status_obj; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -11929,6 +12449,82 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* defined(MS_WINDOWS) */ +PyDoc_STRVAR(os__inputhook__doc__, +"_inputhook($module, /)\n" +"--\n" +"\n" +"Calls PyOS_CallInputHook droppong the GIL first"); + +#define OS__INPUTHOOK_METHODDEF \ + {"_inputhook", (PyCFunction)os__inputhook, METH_NOARGS, os__inputhook__doc__}, + +static PyObject * +os__inputhook_impl(PyObject *module); + +static PyObject * +os__inputhook(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os__inputhook_impl(module); +} + +PyDoc_STRVAR(os__is_inputhook_installed__doc__, +"_is_inputhook_installed($module, /)\n" +"--\n" +"\n" +"Checks if PyOS_CallInputHook is set"); + +#define OS__IS_INPUTHOOK_INSTALLED_METHODDEF \ + {"_is_inputhook_installed", (PyCFunction)os__is_inputhook_installed, METH_NOARGS, os__is_inputhook_installed__doc__}, + +static PyObject * +os__is_inputhook_installed_impl(PyObject *module); + +static PyObject * +os__is_inputhook_installed(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os__is_inputhook_installed_impl(module); +} + +PyDoc_STRVAR(os__create_environ__doc__, +"_create_environ($module, /)\n" +"--\n" +"\n" +"Create the environment dictionary."); + +#define OS__CREATE_ENVIRON_METHODDEF \ + {"_create_environ", (PyCFunction)os__create_environ, METH_NOARGS, os__create_environ__doc__}, + +static PyObject * +os__create_environ_impl(PyObject *module); + +static PyObject * +os__create_environ(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os__create_environ_impl(module); +} + +#if defined(__EMSCRIPTEN__) + +PyDoc_STRVAR(os__emscripten_debugger__doc__, +"_emscripten_debugger($module, /)\n" +"--\n" +"\n" +"Create a breakpoint for the JavaScript debugger. Emscripten only."); + +#define OS__EMSCRIPTEN_DEBUGGER_METHODDEF \ + {"_emscripten_debugger", (PyCFunction)os__emscripten_debugger, METH_NOARGS, os__emscripten_debugger__doc__}, + +static PyObject * +os__emscripten_debugger_impl(PyObject *module); + +static PyObject * +os__emscripten_debugger(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os__emscripten_debugger_impl(module); +} + +#endif /* defined(__EMSCRIPTEN__) */ + #ifndef OS_TTYNAME_METHODDEF #define OS_TTYNAME_METHODDEF #endif /* !defined(OS_TTYNAME_METHODDEF) */ @@ -12025,6 +12621,14 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #define OS__PATH_SPLITROOT_METHODDEF #endif /* !defined(OS__PATH_SPLITROOT_METHODDEF) */ +#ifndef OS__PATH_EXISTS_METHODDEF + #define OS__PATH_EXISTS_METHODDEF +#endif /* !defined(OS__PATH_EXISTS_METHODDEF) */ + +#ifndef OS__PATH_LEXISTS_METHODDEF + #define OS__PATH_LEXISTS_METHODDEF +#endif /* !defined(OS__PATH_LEXISTS_METHODDEF) */ + #ifndef OS__PATH_ISDIR_METHODDEF #define OS__PATH_ISDIR_METHODDEF #endif /* !defined(OS__PATH_ISDIR_METHODDEF) */ @@ -12033,14 +12637,14 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #define OS__PATH_ISFILE_METHODDEF #endif /* !defined(OS__PATH_ISFILE_METHODDEF) */ -#ifndef OS__PATH_EXISTS_METHODDEF - #define OS__PATH_EXISTS_METHODDEF -#endif /* !defined(OS__PATH_EXISTS_METHODDEF) */ - #ifndef OS__PATH_ISLINK_METHODDEF #define OS__PATH_ISLINK_METHODDEF #endif /* !defined(OS__PATH_ISLINK_METHODDEF) */ +#ifndef OS__PATH_ISJUNCTION_METHODDEF + #define OS__PATH_ISJUNCTION_METHODDEF +#endif /* !defined(OS__PATH_ISJUNCTION_METHODDEF) */ + #ifndef OS_NICE_METHODDEF #define OS_NICE_METHODDEF #endif /* !defined(OS_NICE_METHODDEF) */ @@ -12588,4 +13192,8 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=268af5cbc8baa9d4 input=a9049054013a1b77]*/ + +#ifndef OS__EMSCRIPTEN_DEBUGGER_METHODDEF + #define OS__EMSCRIPTEN_DEBUGGER_METHODDEF +#endif /* !defined(OS__EMSCRIPTEN_DEBUGGER_METHODDEF) */ +/*[clinic end generated code: output=8318c26fc2cd236c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pwdmodule.c.h b/Modules/clinic/pwdmodule.c.h index 43d4825031c7e6..365d99aab1dd22 100644 --- a/Modules/clinic/pwdmodule.c.h +++ b/Modules/clinic/pwdmodule.c.h @@ -2,8 +2,6 @@ preserve [clinic start generated code]*/ -#include "pycore_modsupport.h" // _PyArg_BadArgument() - PyDoc_STRVAR(pwd_getpwuid__doc__, "getpwuid($module, uidobj, /)\n" "--\n" @@ -36,7 +34,7 @@ pwd_getpwnam(PyObject *module, PyObject *arg) PyObject *name; if (!PyUnicode_Check(arg)) { - _PyArg_BadArgument("getpwnam", "argument", "str", arg); + PyErr_Format(PyExc_TypeError, "getpwnam() argument must be str, not %T", arg); goto exit; } name = arg; @@ -73,4 +71,4 @@ pwd_getpwall(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef PWD_GETPWALL_METHODDEF #define PWD_GETPWALL_METHODDEF #endif /* !defined(PWD_GETPWALL_METHODDEF) */ -/*[clinic end generated code: output=5a8fb12939ff4ea3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dac88d500f6d6f49 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h index 343cb91b975038..9eba59731c3fba 100644 --- a/Modules/clinic/pyexpat.c.h +++ b/Modules/clinic/pyexpat.c.h @@ -4,7 +4,7 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() +# include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() @@ -22,7 +22,7 @@ pyexpat_xmlparser_SetReparseDeferralEnabled_impl(xmlparseobject *self, int enabled); static PyObject * -pyexpat_xmlparser_SetReparseDeferralEnabled(xmlparseobject *self, PyObject *arg) +pyexpat_xmlparser_SetReparseDeferralEnabled(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; int enabled; @@ -31,7 +31,7 @@ pyexpat_xmlparser_SetReparseDeferralEnabled(xmlparseobject *self, PyObject *arg) if (enabled < 0) { goto exit; } - return_value = pyexpat_xmlparser_SetReparseDeferralEnabled_impl(self, enabled); + return_value = pyexpat_xmlparser_SetReparseDeferralEnabled_impl((xmlparseobject *)self, enabled); exit: return return_value; @@ -50,9 +50,9 @@ static PyObject * pyexpat_xmlparser_GetReparseDeferralEnabled_impl(xmlparseobject *self); static PyObject * -pyexpat_xmlparser_GetReparseDeferralEnabled(xmlparseobject *self, PyObject *Py_UNUSED(ignored)) +pyexpat_xmlparser_GetReparseDeferralEnabled(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return pyexpat_xmlparser_GetReparseDeferralEnabled_impl(self); + return pyexpat_xmlparser_GetReparseDeferralEnabled_impl((xmlparseobject *)self); } PyDoc_STRVAR(pyexpat_xmlparser_Parse__doc__, @@ -61,7 +61,7 @@ PyDoc_STRVAR(pyexpat_xmlparser_Parse__doc__, "\n" "Parse XML data.\n" "\n" -"`isfinal\' should be true at end of input."); +"\'isfinal\' should be true at end of input."); #define PYEXPAT_XMLPARSER_PARSE_METHODDEF \ {"Parse", _PyCFunction_CAST(pyexpat_xmlparser_Parse), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_Parse__doc__}, @@ -71,7 +71,7 @@ pyexpat_xmlparser_Parse_impl(xmlparseobject *self, PyTypeObject *cls, PyObject *data, int isfinal); static PyObject * -pyexpat_xmlparser_Parse(xmlparseobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +pyexpat_xmlparser_Parse(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -91,7 +91,8 @@ pyexpat_xmlparser_Parse(xmlparseobject *self, PyTypeObject *cls, PyObject *const PyObject *data; int isfinal = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -104,7 +105,7 @@ pyexpat_xmlparser_Parse(xmlparseobject *self, PyTypeObject *cls, PyObject *const goto exit; } skip_optional_posonly: - return_value = pyexpat_xmlparser_Parse_impl(self, cls, data, isfinal); + return_value = pyexpat_xmlparser_Parse_impl((xmlparseobject *)self, cls, data, isfinal); exit: return return_value; @@ -124,7 +125,7 @@ pyexpat_xmlparser_ParseFile_impl(xmlparseobject *self, PyTypeObject *cls, PyObject *file); static PyObject * -pyexpat_xmlparser_ParseFile(xmlparseobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +pyexpat_xmlparser_ParseFile(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -143,12 +144,13 @@ pyexpat_xmlparser_ParseFile(xmlparseobject *self, PyTypeObject *cls, PyObject *c PyObject *argsbuf[1]; PyObject *file; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } file = args[0]; - return_value = pyexpat_xmlparser_ParseFile_impl(self, cls, file); + return_value = pyexpat_xmlparser_ParseFile_impl((xmlparseobject *)self, cls, file); exit: return return_value; @@ -167,7 +169,7 @@ static PyObject * pyexpat_xmlparser_SetBase_impl(xmlparseobject *self, const char *base); static PyObject * -pyexpat_xmlparser_SetBase(xmlparseobject *self, PyObject *arg) +pyexpat_xmlparser_SetBase(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; const char *base; @@ -185,7 +187,7 @@ pyexpat_xmlparser_SetBase(xmlparseobject *self, PyObject *arg) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - return_value = pyexpat_xmlparser_SetBase_impl(self, base); + return_value = pyexpat_xmlparser_SetBase_impl((xmlparseobject *)self, base); exit: return return_value; @@ -204,9 +206,9 @@ static PyObject * pyexpat_xmlparser_GetBase_impl(xmlparseobject *self); static PyObject * -pyexpat_xmlparser_GetBase(xmlparseobject *self, PyObject *Py_UNUSED(ignored)) +pyexpat_xmlparser_GetBase(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return pyexpat_xmlparser_GetBase_impl(self); + return pyexpat_xmlparser_GetBase_impl((xmlparseobject *)self); } PyDoc_STRVAR(pyexpat_xmlparser_GetInputContext__doc__, @@ -225,9 +227,9 @@ static PyObject * pyexpat_xmlparser_GetInputContext_impl(xmlparseobject *self); static PyObject * -pyexpat_xmlparser_GetInputContext(xmlparseobject *self, PyObject *Py_UNUSED(ignored)) +pyexpat_xmlparser_GetInputContext(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return pyexpat_xmlparser_GetInputContext_impl(self); + return pyexpat_xmlparser_GetInputContext_impl((xmlparseobject *)self); } PyDoc_STRVAR(pyexpat_xmlparser_ExternalEntityParserCreate__doc__, @@ -247,7 +249,7 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self, const char *encoding); static PyObject * -pyexpat_xmlparser_ExternalEntityParserCreate(xmlparseobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +pyexpat_xmlparser_ExternalEntityParserCreate(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -267,7 +269,8 @@ pyexpat_xmlparser_ExternalEntityParserCreate(xmlparseobject *self, PyTypeObject const char *context; const char *encoding = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -306,7 +309,7 @@ pyexpat_xmlparser_ExternalEntityParserCreate(xmlparseobject *self, PyTypeObject goto exit; } skip_optional_posonly: - return_value = pyexpat_xmlparser_ExternalEntityParserCreate_impl(self, cls, context, encoding); + return_value = pyexpat_xmlparser_ExternalEntityParserCreate_impl((xmlparseobject *)self, cls, context, encoding); exit: return return_value; @@ -330,7 +333,7 @@ static PyObject * pyexpat_xmlparser_SetParamEntityParsing_impl(xmlparseobject *self, int flag); static PyObject * -pyexpat_xmlparser_SetParamEntityParsing(xmlparseobject *self, PyObject *arg) +pyexpat_xmlparser_SetParamEntityParsing(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; int flag; @@ -339,7 +342,7 @@ pyexpat_xmlparser_SetParamEntityParsing(xmlparseobject *self, PyObject *arg) if (flag == -1 && PyErr_Occurred()) { goto exit; } - return_value = pyexpat_xmlparser_SetParamEntityParsing_impl(self, flag); + return_value = pyexpat_xmlparser_SetParamEntityParsing_impl((xmlparseobject *)self, flag); exit: return return_value; @@ -365,7 +368,7 @@ pyexpat_xmlparser_UseForeignDTD_impl(xmlparseobject *self, PyTypeObject *cls, int flag); static PyObject * -pyexpat_xmlparser_UseForeignDTD(xmlparseobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +pyexpat_xmlparser_UseForeignDTD(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -384,7 +387,8 @@ pyexpat_xmlparser_UseForeignDTD(xmlparseobject *self, PyTypeObject *cls, PyObjec PyObject *argsbuf[1]; int flag = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -396,7 +400,7 @@ pyexpat_xmlparser_UseForeignDTD(xmlparseobject *self, PyTypeObject *cls, PyObjec goto exit; } skip_optional_posonly: - return_value = pyexpat_xmlparser_UseForeignDTD_impl(self, cls, flag); + return_value = pyexpat_xmlparser_UseForeignDTD_impl((xmlparseobject *)self, cls, flag); exit: return return_value; @@ -453,7 +457,8 @@ pyexpat_ParserCreate(PyObject *module, PyObject *const *args, Py_ssize_t nargs, const char *namespace_separator = NULL; PyObject *intern = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -545,4 +550,4 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg) #ifndef PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF) */ -/*[clinic end generated code: output=892e48e41f9b6e4b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7ee30ae5b666d0a8 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/selectmodule.c.h b/Modules/clinic/selectmodule.c.h index 67e76d4c89f59a..d8bdd6f95f3d29 100644 --- a/Modules/clinic/selectmodule.c.h +++ b/Modules/clinic/selectmodule.c.h @@ -6,7 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif -#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() @@ -26,7 +26,7 @@ PyDoc_STRVAR(select_select__doc__, "gotten from a fileno() method call on one of those.\n" "\n" "The optional 4th argument specifies a timeout in seconds; it may be\n" -"a floating point number to specify fractions of seconds. If it is absent\n" +"a floating-point number to specify fractions of seconds. If it is absent\n" "or None, the call will never time out.\n" "\n" "The return value is a tuple of three lists corresponding to the first three\n" @@ -91,7 +91,7 @@ static PyObject * select_poll_register_impl(pollObject *self, int fd, unsigned short eventmask); static PyObject * -select_poll_register(pollObject *self, PyObject *const *args, Py_ssize_t nargs) +select_poll_register(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int fd; @@ -100,7 +100,8 @@ select_poll_register(pollObject *self, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("register", nargs, 1, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (nargs < 2) { @@ -110,7 +111,9 @@ select_poll_register(pollObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: - return_value = select_poll_register_impl(self, fd, eventmask); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_poll_register_impl((pollObject *)self, fd, eventmask); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -139,7 +142,7 @@ static PyObject * select_poll_modify_impl(pollObject *self, int fd, unsigned short eventmask); static PyObject * -select_poll_modify(pollObject *self, PyObject *const *args, Py_ssize_t nargs) +select_poll_modify(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int fd; @@ -148,13 +151,16 @@ select_poll_modify(pollObject *self, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("modify", nargs, 2, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!_PyLong_UnsignedShort_Converter(args[1], &eventmask)) { goto exit; } - return_value = select_poll_modify_impl(self, fd, eventmask); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_poll_modify_impl((pollObject *)self, fd, eventmask); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -177,15 +183,18 @@ static PyObject * select_poll_unregister_impl(pollObject *self, int fd); static PyObject * -select_poll_unregister(pollObject *self, PyObject *arg) +select_poll_unregister(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } - return_value = select_poll_unregister_impl(self, fd); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_poll_unregister_impl((pollObject *)self, fd); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -215,7 +224,7 @@ static PyObject * select_poll_poll_impl(pollObject *self, PyObject *timeout_obj); static PyObject * -select_poll_poll(pollObject *self, PyObject *const *args, Py_ssize_t nargs) +select_poll_poll(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *timeout_obj = Py_None; @@ -228,7 +237,9 @@ select_poll_poll(pollObject *self, PyObject *const *args, Py_ssize_t nargs) } timeout_obj = args[0]; skip_optional: - return_value = select_poll_poll_impl(self, timeout_obj); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_poll_poll_impl((pollObject *)self, timeout_obj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -259,7 +270,7 @@ select_devpoll_register_impl(devpollObject *self, int fd, unsigned short eventmask); static PyObject * -select_devpoll_register(devpollObject *self, PyObject *const *args, Py_ssize_t nargs) +select_devpoll_register(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int fd; @@ -268,7 +279,8 @@ select_devpoll_register(devpollObject *self, PyObject *const *args, Py_ssize_t n if (!_PyArg_CheckPositional("register", nargs, 1, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (nargs < 2) { @@ -278,7 +290,9 @@ select_devpoll_register(devpollObject *self, PyObject *const *args, Py_ssize_t n goto exit; } skip_optional: - return_value = select_devpoll_register_impl(self, fd, eventmask); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_devpoll_register_impl((devpollObject *)self, fd, eventmask); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -309,7 +323,7 @@ select_devpoll_modify_impl(devpollObject *self, int fd, unsigned short eventmask); static PyObject * -select_devpoll_modify(devpollObject *self, PyObject *const *args, Py_ssize_t nargs) +select_devpoll_modify(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int fd; @@ -318,7 +332,8 @@ select_devpoll_modify(devpollObject *self, PyObject *const *args, Py_ssize_t nar if (!_PyArg_CheckPositional("modify", nargs, 1, 2)) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (nargs < 2) { @@ -328,7 +343,9 @@ select_devpoll_modify(devpollObject *self, PyObject *const *args, Py_ssize_t nar goto exit; } skip_optional: - return_value = select_devpoll_modify_impl(self, fd, eventmask); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_devpoll_modify_impl((devpollObject *)self, fd, eventmask); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -351,15 +368,18 @@ static PyObject * select_devpoll_unregister_impl(devpollObject *self, int fd); static PyObject * -select_devpoll_unregister(devpollObject *self, PyObject *arg) +select_devpoll_unregister(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } - return_value = select_devpoll_unregister_impl(self, fd); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_devpoll_unregister_impl((devpollObject *)self, fd); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -389,7 +409,7 @@ static PyObject * select_devpoll_poll_impl(devpollObject *self, PyObject *timeout_obj); static PyObject * -select_devpoll_poll(devpollObject *self, PyObject *const *args, Py_ssize_t nargs) +select_devpoll_poll(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *timeout_obj = Py_None; @@ -402,7 +422,9 @@ select_devpoll_poll(devpollObject *self, PyObject *const *args, Py_ssize_t nargs } timeout_obj = args[0]; skip_optional: - return_value = select_devpoll_poll_impl(self, timeout_obj); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_devpoll_poll_impl((devpollObject *)self, timeout_obj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -427,9 +449,15 @@ static PyObject * select_devpoll_close_impl(devpollObject *self); static PyObject * -select_devpoll_close(devpollObject *self, PyObject *Py_UNUSED(ignored)) +select_devpoll_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return select_devpoll_close_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_devpoll_close_impl((devpollObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* (defined(HAVE_POLL) && !defined(HAVE_BROKEN_POLL)) && defined(HAVE_SYS_DEVPOLL_H) */ @@ -449,9 +477,15 @@ static PyObject * select_devpoll_fileno_impl(devpollObject *self); static PyObject * -select_devpoll_fileno(devpollObject *self, PyObject *Py_UNUSED(ignored)) +select_devpoll_fileno(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return select_devpoll_fileno_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_devpoll_fileno_impl((devpollObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* (defined(HAVE_POLL) && !defined(HAVE_BROKEN_POLL)) && defined(HAVE_SYS_DEVPOLL_H) */ @@ -562,7 +596,8 @@ select_epoll(PyTypeObject *type, PyObject *args, PyObject *kwargs) int sizehint = -1; int flags = 0; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -608,9 +643,15 @@ static PyObject * select_epoll_close_impl(pyEpoll_Object *self); static PyObject * -select_epoll_close(pyEpoll_Object *self, PyObject *Py_UNUSED(ignored)) +select_epoll_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return select_epoll_close_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_epoll_close_impl((pyEpoll_Object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_EPOLL) */ @@ -630,9 +671,9 @@ static PyObject * select_epoll_fileno_impl(pyEpoll_Object *self); static PyObject * -select_epoll_fileno(pyEpoll_Object *self, PyObject *Py_UNUSED(ignored)) +select_epoll_fileno(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return select_epoll_fileno_impl(self); + return select_epoll_fileno_impl((pyEpoll_Object *)self); } #endif /* defined(HAVE_EPOLL) */ @@ -693,7 +734,7 @@ select_epoll_register_impl(pyEpoll_Object *self, int fd, unsigned int eventmask); static PyObject * -select_epoll_register(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +select_epoll_register(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -726,11 +767,13 @@ select_epoll_register(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t na int fd; unsigned int eventmask = EPOLLIN | EPOLLPRI | EPOLLOUT; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } if (!noptargs) { @@ -741,7 +784,7 @@ select_epoll_register(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t na goto exit; } skip_optional_pos: - return_value = select_epoll_register_impl(self, fd, eventmask); + return_value = select_epoll_register_impl((pyEpoll_Object *)self, fd, eventmask); exit: return return_value; @@ -770,7 +813,7 @@ select_epoll_modify_impl(pyEpoll_Object *self, int fd, unsigned int eventmask); static PyObject * -select_epoll_modify(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +select_epoll_modify(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -802,18 +845,20 @@ select_epoll_modify(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t narg int fd; unsigned int eventmask; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } eventmask = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); if (eventmask == (unsigned int)-1 && PyErr_Occurred()) { goto exit; } - return_value = select_epoll_modify_impl(self, fd, eventmask); + return_value = select_epoll_modify_impl((pyEpoll_Object *)self, fd, eventmask); exit: return return_value; @@ -839,7 +884,7 @@ static PyObject * select_epoll_unregister_impl(pyEpoll_Object *self, int fd); static PyObject * -select_epoll_unregister(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +select_epoll_unregister(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -870,14 +915,16 @@ select_epoll_unregister(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t PyObject *argsbuf[1]; int fd; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } - return_value = select_epoll_unregister_impl(self, fd); + return_value = select_epoll_unregister_impl((pyEpoll_Object *)self, fd); exit: return return_value; @@ -910,7 +957,7 @@ select_epoll_poll_impl(pyEpoll_Object *self, PyObject *timeout_obj, int maxevents); static PyObject * -select_epoll_poll(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +select_epoll_poll(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -943,7 +990,8 @@ select_epoll_poll(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t nargs, PyObject *timeout_obj = Py_None; int maxevents = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -961,7 +1009,7 @@ select_epoll_poll(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t nargs, goto exit; } skip_optional_pos: - return_value = select_epoll_poll_impl(self, timeout_obj, maxevents); + return_value = select_epoll_poll_impl((pyEpoll_Object *)self, timeout_obj, maxevents); exit: return return_value; @@ -983,9 +1031,9 @@ static PyObject * select_epoll___enter___impl(pyEpoll_Object *self); static PyObject * -select_epoll___enter__(pyEpoll_Object *self, PyObject *Py_UNUSED(ignored)) +select_epoll___enter__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return select_epoll___enter___impl(self); + return select_epoll___enter___impl((pyEpoll_Object *)self); } #endif /* defined(HAVE_EPOLL) */ @@ -1005,7 +1053,7 @@ select_epoll___exit___impl(pyEpoll_Object *self, PyObject *exc_type, PyObject *exc_value, PyObject *exc_tb); static PyObject * -select_epoll___exit__(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t nargs) +select_epoll___exit__(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *exc_type = Py_None; @@ -1028,7 +1076,7 @@ select_epoll___exit__(pyEpoll_Object *self, PyObject *const *args, Py_ssize_t na } exc_tb = args[2]; skip_optional: - return_value = select_epoll___exit___impl(self, exc_type, exc_value, exc_tb); + return_value = select_epoll___exit___impl((pyEpoll_Object *)self, exc_type, exc_value, exc_tb); exit: return return_value; @@ -1098,9 +1146,15 @@ static PyObject * select_kqueue_close_impl(kqueue_queue_Object *self); static PyObject * -select_kqueue_close(kqueue_queue_Object *self, PyObject *Py_UNUSED(ignored)) +select_kqueue_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return select_kqueue_close_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_kqueue_close_impl((kqueue_queue_Object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_KQUEUE) */ @@ -1120,9 +1174,9 @@ static PyObject * select_kqueue_fileno_impl(kqueue_queue_Object *self); static PyObject * -select_kqueue_fileno(kqueue_queue_Object *self, PyObject *Py_UNUSED(ignored)) +select_kqueue_fileno(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return select_kqueue_fileno_impl(self); + return select_kqueue_fileno_impl((kqueue_queue_Object *)self); } #endif /* defined(HAVE_KQUEUE) */ @@ -1184,7 +1238,7 @@ select_kqueue_control_impl(kqueue_queue_Object *self, PyObject *changelist, int maxevents, PyObject *otimeout); static PyObject * -select_kqueue_control(kqueue_queue_Object *self, PyObject *const *args, Py_ssize_t nargs) +select_kqueue_control(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *changelist; @@ -1204,7 +1258,7 @@ select_kqueue_control(kqueue_queue_Object *self, PyObject *const *args, Py_ssize } otimeout = args[2]; skip_optional: - return_value = select_kqueue_control_impl(self, changelist, maxevents, otimeout); + return_value = select_kqueue_control_impl((kqueue_queue_Object *)self, changelist, maxevents, otimeout); exit: return return_value; @@ -1311,4 +1365,4 @@ select_kqueue_control(kqueue_queue_Object *self, PyObject *const *args, Py_ssize #ifndef SELECT_KQUEUE_CONTROL_METHODDEF #define SELECT_KQUEUE_CONTROL_METHODDEF #endif /* !defined(SELECT_KQUEUE_CONTROL_METHODDEF) */ -/*[clinic end generated code: output=4c2dcb31cb17c2c6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c18fd93efc5f4dce input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha1module.c.h b/Modules/clinic/sha1module.c.h index b89c7e505c788e..ddd8e66a41d7ff 100644 --- a/Modules/clinic/sha1module.c.h +++ b/Modules/clinic/sha1module.c.h @@ -21,13 +21,13 @@ static PyObject * SHA1Type_copy_impl(SHA1object *self, PyTypeObject *cls); static PyObject * -SHA1Type_copy(SHA1object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +SHA1Type_copy(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } - return SHA1Type_copy_impl(self, cls); + return SHA1Type_copy_impl((SHA1object *)self, cls); } PyDoc_STRVAR(SHA1Type_digest__doc__, @@ -43,9 +43,9 @@ static PyObject * SHA1Type_digest_impl(SHA1object *self); static PyObject * -SHA1Type_digest(SHA1object *self, PyObject *Py_UNUSED(ignored)) +SHA1Type_digest(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return SHA1Type_digest_impl(self); + return SHA1Type_digest_impl((SHA1object *)self); } PyDoc_STRVAR(SHA1Type_hexdigest__doc__, @@ -61,9 +61,9 @@ static PyObject * SHA1Type_hexdigest_impl(SHA1object *self); static PyObject * -SHA1Type_hexdigest(SHA1object *self, PyObject *Py_UNUSED(ignored)) +SHA1Type_hexdigest(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return SHA1Type_hexdigest_impl(self); + return SHA1Type_hexdigest_impl((SHA1object *)self); } PyDoc_STRVAR(SHA1Type_update__doc__, @@ -121,7 +121,8 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * PyObject *string = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -148,4 +149,4 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * exit: return return_value; } -/*[clinic end generated code: output=af5a640df662066f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ad6f3788a6e7ff6f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha2module.c.h b/Modules/clinic/sha2module.c.h index cf4b88d52856b8..d86f5510d752e8 100644 --- a/Modules/clinic/sha2module.c.h +++ b/Modules/clinic/sha2module.c.h @@ -21,13 +21,13 @@ static PyObject * SHA256Type_copy_impl(SHA256object *self, PyTypeObject *cls); static PyObject * -SHA256Type_copy(SHA256object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +SHA256Type_copy(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } - return SHA256Type_copy_impl(self, cls); + return SHA256Type_copy_impl((SHA256object *)self, cls); } PyDoc_STRVAR(SHA512Type_copy__doc__, @@ -43,13 +43,13 @@ static PyObject * SHA512Type_copy_impl(SHA512object *self, PyTypeObject *cls); static PyObject * -SHA512Type_copy(SHA512object *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +SHA512Type_copy(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } - return SHA512Type_copy_impl(self, cls); + return SHA512Type_copy_impl((SHA512object *)self, cls); } PyDoc_STRVAR(SHA256Type_digest__doc__, @@ -65,9 +65,9 @@ static PyObject * SHA256Type_digest_impl(SHA256object *self); static PyObject * -SHA256Type_digest(SHA256object *self, PyObject *Py_UNUSED(ignored)) +SHA256Type_digest(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return SHA256Type_digest_impl(self); + return SHA256Type_digest_impl((SHA256object *)self); } PyDoc_STRVAR(SHA512Type_digest__doc__, @@ -83,9 +83,9 @@ static PyObject * SHA512Type_digest_impl(SHA512object *self); static PyObject * -SHA512Type_digest(SHA512object *self, PyObject *Py_UNUSED(ignored)) +SHA512Type_digest(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return SHA512Type_digest_impl(self); + return SHA512Type_digest_impl((SHA512object *)self); } PyDoc_STRVAR(SHA256Type_hexdigest__doc__, @@ -101,9 +101,9 @@ static PyObject * SHA256Type_hexdigest_impl(SHA256object *self); static PyObject * -SHA256Type_hexdigest(SHA256object *self, PyObject *Py_UNUSED(ignored)) +SHA256Type_hexdigest(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return SHA256Type_hexdigest_impl(self); + return SHA256Type_hexdigest_impl((SHA256object *)self); } PyDoc_STRVAR(SHA512Type_hexdigest__doc__, @@ -119,9 +119,9 @@ static PyObject * SHA512Type_hexdigest_impl(SHA512object *self); static PyObject * -SHA512Type_hexdigest(SHA512object *self, PyObject *Py_UNUSED(ignored)) +SHA512Type_hexdigest(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return SHA512Type_hexdigest_impl(self); + return SHA512Type_hexdigest_impl((SHA512object *)self); } PyDoc_STRVAR(SHA256Type_update__doc__, @@ -188,7 +188,8 @@ _sha2_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *string = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -262,7 +263,8 @@ _sha2_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *string = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -336,7 +338,8 @@ _sha2_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *string = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -410,7 +413,8 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *string = NULL; int usedforsecurity = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -437,4 +441,4 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject exit: return return_value; } -/*[clinic end generated code: output=b46da764024b1764 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1d7fec114eb6b6e3 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha3module.c.h b/Modules/clinic/sha3module.c.h index 738e9589503900..729e216ce023cf 100644 --- a/Modules/clinic/sha3module.c.h +++ b/Modules/clinic/sha3module.c.h @@ -54,7 +54,8 @@ py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *data = NULL; int usedforsecurity = 1; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -91,9 +92,9 @@ static PyObject * _sha3_sha3_224_copy_impl(SHA3object *self); static PyObject * -_sha3_sha3_224_copy(SHA3object *self, PyObject *Py_UNUSED(ignored)) +_sha3_sha3_224_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _sha3_sha3_224_copy_impl(self); + return _sha3_sha3_224_copy_impl((SHA3object *)self); } PyDoc_STRVAR(_sha3_sha3_224_digest__doc__, @@ -109,9 +110,9 @@ static PyObject * _sha3_sha3_224_digest_impl(SHA3object *self); static PyObject * -_sha3_sha3_224_digest(SHA3object *self, PyObject *Py_UNUSED(ignored)) +_sha3_sha3_224_digest(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _sha3_sha3_224_digest_impl(self); + return _sha3_sha3_224_digest_impl((SHA3object *)self); } PyDoc_STRVAR(_sha3_sha3_224_hexdigest__doc__, @@ -127,9 +128,9 @@ static PyObject * _sha3_sha3_224_hexdigest_impl(SHA3object *self); static PyObject * -_sha3_sha3_224_hexdigest(SHA3object *self, PyObject *Py_UNUSED(ignored)) +_sha3_sha3_224_hexdigest(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _sha3_sha3_224_hexdigest_impl(self); + return _sha3_sha3_224_hexdigest_impl((SHA3object *)self); } PyDoc_STRVAR(_sha3_sha3_224_update__doc__, @@ -154,7 +155,7 @@ static PyObject * _sha3_shake_128_digest_impl(SHA3object *self, unsigned long length); static PyObject * -_sha3_shake_128_digest(SHA3object *self, PyObject *arg) +_sha3_shake_128_digest(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; unsigned long length; @@ -162,7 +163,7 @@ _sha3_shake_128_digest(SHA3object *self, PyObject *arg) if (!_PyLong_UnsignedLong_Converter(arg, &length)) { goto exit; } - return_value = _sha3_shake_128_digest_impl(self, length); + return_value = _sha3_shake_128_digest_impl((SHA3object *)self, length); exit: return return_value; @@ -181,7 +182,7 @@ static PyObject * _sha3_shake_128_hexdigest_impl(SHA3object *self, unsigned long length); static PyObject * -_sha3_shake_128_hexdigest(SHA3object *self, PyObject *arg) +_sha3_shake_128_hexdigest(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; unsigned long length; @@ -189,9 +190,9 @@ _sha3_shake_128_hexdigest(SHA3object *self, PyObject *arg) if (!_PyLong_UnsignedLong_Converter(arg, &length)) { goto exit; } - return_value = _sha3_shake_128_hexdigest_impl(self, length); + return_value = _sha3_shake_128_hexdigest_impl((SHA3object *)self, length); exit: return return_value; } -/*[clinic end generated code: output=a8e76a880d1421ec input=a9049054013a1b77]*/ +/*[clinic end generated code: output=21da06d9570969d8 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h index bc33e066654364..dd8da456ce08ce 100644 --- a/Modules/clinic/signalmodule.c.h +++ b/Modules/clinic/signalmodule.c.h @@ -2,6 +2,10 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(signal_default_int_handler__doc__, @@ -276,6 +280,78 @@ signal_siginterrupt(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #endif /* defined(HAVE_SIGINTERRUPT) */ +PyDoc_STRVAR(signal_set_wakeup_fd__doc__, +"set_wakeup_fd($module, fd, /, *, warn_on_full_buffer=True)\n" +"--\n" +"\n" +"Sets the fd to be written to (with the signal number) when a signal comes in.\n" +"\n" +"A library can use this to wakeup select or poll.\n" +"The previous fd or -1 is returned.\n" +"\n" +"The fd must be non-blocking."); + +#define SIGNAL_SET_WAKEUP_FD_METHODDEF \ + {"set_wakeup_fd", _PyCFunction_CAST(signal_set_wakeup_fd), METH_FASTCALL|METH_KEYWORDS, signal_set_wakeup_fd__doc__}, + +static PyObject * +signal_set_wakeup_fd_impl(PyObject *module, PyObject *fdobj, + int warn_on_full_buffer); + +static PyObject * +signal_set_wakeup_fd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(warn_on_full_buffer), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "warn_on_full_buffer", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "set_wakeup_fd", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *fdobj; + int warn_on_full_buffer = 1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + fdobj = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + warn_on_full_buffer = PyObject_IsTrue(args[1]); + if (warn_on_full_buffer < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = signal_set_wakeup_fd_impl(module, fdobj, warn_on_full_buffer); + +exit: + return return_value; +} + #if defined(HAVE_SETITIMER) PyDoc_STRVAR(signal_setitimer__doc__, @@ -522,7 +598,7 @@ PyDoc_STRVAR(signal_sigtimedwait__doc__, "\n" "Like sigwaitinfo(), but with a timeout.\n" "\n" -"The timeout is specified in seconds, with floating point numbers allowed."); +"The timeout is specified in seconds, with floating-point numbers allowed."); #define SIGNAL_SIGTIMEDWAIT_METHODDEF \ {"sigtimedwait", _PyCFunction_CAST(signal_sigtimedwait), METH_FASTCALL, signal_sigtimedwait__doc__}, @@ -595,7 +671,7 @@ signal_pthread_kill(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #endif /* defined(HAVE_PTHREAD_KILL) */ -#if (defined(__linux__) && defined(__NR_pidfd_send_signal)) +#if (defined(__linux__) && defined(__NR_pidfd_send_signal) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) PyDoc_STRVAR(signal_pidfd_send_signal__doc__, "pidfd_send_signal($module, pidfd, signalnum, siginfo=None, flags=0, /)\n" @@ -648,7 +724,7 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar return return_value; } -#endif /* (defined(__linux__) && defined(__NR_pidfd_send_signal)) */ +#endif /* (defined(__linux__) && defined(__NR_pidfd_send_signal) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) */ #ifndef SIGNAL_ALARM_METHODDEF #define SIGNAL_ALARM_METHODDEF @@ -701,4 +777,4 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #define SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #endif /* !defined(SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF) */ -/*[clinic end generated code: output=5a9928cb2dc75b5f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=356e1acc44a4f377 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/socketmodule.c.h b/Modules/clinic/socketmodule.c.h index 3f4056efff2fec..dc62c4290d3e3b 100644 --- a/Modules/clinic/socketmodule.c.h +++ b/Modules/clinic/socketmodule.c.h @@ -8,6 +8,26 @@ preserve #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +PyDoc_STRVAR(_socket_socket_close__doc__, +"close($self, /)\n" +"--\n" +"\n" +"close()\n" +"\n" +"Close the socket. It cannot be used after this call."); + +#define _SOCKET_SOCKET_CLOSE_METHODDEF \ + {"close", (PyCFunction)_socket_socket_close, METH_NOARGS, _socket_socket_close__doc__}, + +static PyObject * +_socket_socket_close_impl(PySocketSockObject *s); + +static PyObject * +_socket_socket_close(PyObject *s, PyObject *Py_UNUSED(ignored)) +{ + return _socket_socket_close_impl((PySocketSockObject *)s); +} + static int sock_initobj_impl(PySocketSockObject *self, int family, int type, int proto, PyObject *fdobj); @@ -50,7 +70,8 @@ sock_initobj(PyObject *self, PyObject *args, PyObject *kwargs) int proto = -1; PyObject *fdobj = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 4, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -105,7 +126,7 @@ static PyObject * _socket_socket_ntohs_impl(PySocketSockObject *self, int x); static PyObject * -_socket_socket_ntohs(PySocketSockObject *self, PyObject *arg) +_socket_socket_ntohs(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; int x; @@ -114,7 +135,7 @@ _socket_socket_ntohs(PySocketSockObject *self, PyObject *arg) if (x == -1 && PyErr_Occurred()) { goto exit; } - return_value = _socket_socket_ntohs_impl(self, x); + return_value = _socket_socket_ntohs_impl((PySocketSockObject *)self, x); exit: return return_value; @@ -133,7 +154,7 @@ static PyObject * _socket_socket_htons_impl(PySocketSockObject *self, int x); static PyObject * -_socket_socket_htons(PySocketSockObject *self, PyObject *arg) +_socket_socket_htons(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; int x; @@ -142,7 +163,7 @@ _socket_socket_htons(PySocketSockObject *self, PyObject *arg) if (x == -1 && PyErr_Occurred()) { goto exit; } - return_value = _socket_socket_htons_impl(self, x); + return_value = _socket_socket_htons_impl((PySocketSockObject *)self, x); exit: return return_value; @@ -161,7 +182,7 @@ static PyObject * _socket_socket_inet_aton_impl(PySocketSockObject *self, const char *ip_addr); static PyObject * -_socket_socket_inet_aton(PySocketSockObject *self, PyObject *arg) +_socket_socket_inet_aton(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; const char *ip_addr; @@ -179,7 +200,7 @@ _socket_socket_inet_aton(PySocketSockObject *self, PyObject *arg) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - return_value = _socket_socket_inet_aton_impl(self, ip_addr); + return_value = _socket_socket_inet_aton_impl((PySocketSockObject *)self, ip_addr); exit: return return_value; @@ -200,7 +221,7 @@ static PyObject * _socket_socket_inet_ntoa_impl(PySocketSockObject *self, Py_buffer *packed_ip); static PyObject * -_socket_socket_inet_ntoa(PySocketSockObject *self, PyObject *arg) +_socket_socket_inet_ntoa(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; Py_buffer packed_ip = {NULL, NULL}; @@ -208,7 +229,7 @@ _socket_socket_inet_ntoa(PySocketSockObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &packed_ip, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = _socket_socket_inet_ntoa_impl(self, &packed_ip); + return_value = _socket_socket_inet_ntoa_impl((PySocketSockObject *)self, &packed_ip); exit: /* Cleanup for packed_ip */ @@ -236,7 +257,7 @@ static PyObject * _socket_socket_if_nametoindex_impl(PySocketSockObject *self, PyObject *oname); static PyObject * -_socket_socket_if_nametoindex(PySocketSockObject *self, PyObject *arg) +_socket_socket_if_nametoindex(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; PyObject *oname; @@ -244,7 +265,7 @@ _socket_socket_if_nametoindex(PySocketSockObject *self, PyObject *arg) if (!PyUnicode_FSConverter(arg, &oname)) { goto exit; } - return_value = _socket_socket_if_nametoindex_impl(self, oname); + return_value = _socket_socket_if_nametoindex_impl((PySocketSockObject *)self, oname); exit: return return_value; @@ -259,4 +280,4 @@ _socket_socket_if_nametoindex(PySocketSockObject *self, PyObject *arg) #ifndef _SOCKET_SOCKET_IF_NAMETOINDEX_METHODDEF #define _SOCKET_SOCKET_IF_NAMETOINDEX_METHODDEF #endif /* !defined(_SOCKET_SOCKET_IF_NAMETOINDEX_METHODDEF) */ -/*[clinic end generated code: output=eb37b5d88a1e4661 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d39efc30d811e74b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/syslogmodule.c.h b/Modules/clinic/syslogmodule.c.h index 58b0ea57b065b3..7c3fcd64191500 100644 --- a/Modules/clinic/syslogmodule.c.h +++ b/Modules/clinic/syslogmodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(syslog_openlog__doc__, @@ -57,7 +58,8 @@ syslog_openlog(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje long logopt = 0; long facility = LOG_USER; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -88,7 +90,9 @@ syslog_openlog(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(module); return_value = syslog_openlog_impl(module, ident, logopt, facility); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -129,7 +133,9 @@ syslog_syslog(PyObject *module, PyObject *args) PyErr_SetString(PyExc_TypeError, "syslog.syslog requires 1 to 2 arguments"); goto exit; } + Py_BEGIN_CRITICAL_SECTION(module); return_value = syslog_syslog_impl(module, group_left_1, priority, message); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -150,7 +156,13 @@ syslog_closelog_impl(PyObject *module); static PyObject * syslog_closelog(PyObject *module, PyObject *Py_UNUSED(ignored)) { - return syslog_closelog_impl(module); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(module); + return_value = syslog_closelog_impl(module); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(syslog_setlogmask__doc__, @@ -251,4 +263,4 @@ syslog_LOG_UPTO(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=86ca2fd84b2da98e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f867a4f7a9267de1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/termios.c.h b/Modules/clinic/termios.c.h index 1c340681e5862f..83f5a4f6e9f882 100644 --- a/Modules/clinic/termios.c.h +++ b/Modules/clinic/termios.c.h @@ -2,9 +2,6 @@ preserve [clinic start generated code]*/ -#include "pycore_fileutils.h" // _PyLong_FileDescriptor_Converter() -#include "pycore_modsupport.h" // _PyArg_CheckPositional() - PyDoc_STRVAR(termios_tcgetattr__doc__, "tcgetattr($module, fd, /)\n" "--\n" @@ -30,7 +27,8 @@ termios_tcgetattr(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = termios_tcgetattr_impl(module, fd); @@ -53,7 +51,7 @@ PyDoc_STRVAR(termios_tcsetattr__doc__, "queued output and discarding all queued input."); #define TERMIOS_TCSETATTR_METHODDEF \ - {"tcsetattr", _PyCFunction_CAST(termios_tcsetattr), METH_FASTCALL, termios_tcsetattr__doc__}, + {"tcsetattr", (PyCFunction)(void(*)(void))termios_tcsetattr, METH_FASTCALL, termios_tcsetattr__doc__}, static PyObject * termios_tcsetattr_impl(PyObject *module, int fd, int when, PyObject *term); @@ -66,10 +64,12 @@ termios_tcsetattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int when; PyObject *term; - if (!_PyArg_CheckPositional("tcsetattr", nargs, 3, 3)) { + if (nargs != 3) { + PyErr_Format(PyExc_TypeError, "tcsetattr expected 3 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } when = PyLong_AsInt(args[1]); @@ -93,7 +93,7 @@ PyDoc_STRVAR(termios_tcsendbreak__doc__, "has a system dependent meaning."); #define TERMIOS_TCSENDBREAK_METHODDEF \ - {"tcsendbreak", _PyCFunction_CAST(termios_tcsendbreak), METH_FASTCALL, termios_tcsendbreak__doc__}, + {"tcsendbreak", (PyCFunction)(void(*)(void))termios_tcsendbreak, METH_FASTCALL, termios_tcsendbreak__doc__}, static PyObject * termios_tcsendbreak_impl(PyObject *module, int fd, int duration); @@ -105,10 +105,12 @@ termios_tcsendbreak(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; int duration; - if (!_PyArg_CheckPositional("tcsendbreak", nargs, 2, 2)) { + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "tcsendbreak expected 2 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } duration = PyLong_AsInt(args[1]); @@ -139,7 +141,8 @@ termios_tcdrain(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = termios_tcdrain_impl(module, fd); @@ -159,7 +162,7 @@ PyDoc_STRVAR(termios_tcflush__doc__, "both queues."); #define TERMIOS_TCFLUSH_METHODDEF \ - {"tcflush", _PyCFunction_CAST(termios_tcflush), METH_FASTCALL, termios_tcflush__doc__}, + {"tcflush", (PyCFunction)(void(*)(void))termios_tcflush, METH_FASTCALL, termios_tcflush__doc__}, static PyObject * termios_tcflush_impl(PyObject *module, int fd, int queue); @@ -171,10 +174,12 @@ termios_tcflush(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; int queue; - if (!_PyArg_CheckPositional("tcflush", nargs, 2, 2)) { + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "tcflush expected 2 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } queue = PyLong_AsInt(args[1]); @@ -198,7 +203,7 @@ PyDoc_STRVAR(termios_tcflow__doc__, "or termios.TCION to restart input."); #define TERMIOS_TCFLOW_METHODDEF \ - {"tcflow", _PyCFunction_CAST(termios_tcflow), METH_FASTCALL, termios_tcflow__doc__}, + {"tcflow", (PyCFunction)(void(*)(void))termios_tcflow, METH_FASTCALL, termios_tcflow__doc__}, static PyObject * termios_tcflow_impl(PyObject *module, int fd, int action); @@ -210,10 +215,12 @@ termios_tcflow(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; int action; - if (!_PyArg_CheckPositional("tcflow", nargs, 2, 2)) { + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "tcflow expected 2 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } action = PyLong_AsInt(args[1]); @@ -246,7 +253,8 @@ termios_tcgetwinsize(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int fd; - if (!_PyLong_FileDescriptor_Converter(arg, &fd)) { + fd = PyObject_AsFileDescriptor(arg); + if (fd < 0) { goto exit; } return_value = termios_tcgetwinsize_impl(module, fd); @@ -265,7 +273,7 @@ PyDoc_STRVAR(termios_tcsetwinsize__doc__, "is a two-item tuple (ws_row, ws_col) like the one returned by tcgetwinsize()."); #define TERMIOS_TCSETWINSIZE_METHODDEF \ - {"tcsetwinsize", _PyCFunction_CAST(termios_tcsetwinsize), METH_FASTCALL, termios_tcsetwinsize__doc__}, + {"tcsetwinsize", (PyCFunction)(void(*)(void))termios_tcsetwinsize, METH_FASTCALL, termios_tcsetwinsize__doc__}, static PyObject * termios_tcsetwinsize_impl(PyObject *module, int fd, PyObject *winsz); @@ -277,10 +285,12 @@ termios_tcsetwinsize(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; PyObject *winsz; - if (!_PyArg_CheckPositional("tcsetwinsize", nargs, 2, 2)) { + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, "tcsetwinsize expected 2 arguments, got %zd", nargs); goto exit; } - if (!_PyLong_FileDescriptor_Converter(args[0], &fd)) { + fd = PyObject_AsFileDescriptor(args[0]); + if (fd < 0) { goto exit; } winsz = args[1]; @@ -289,4 +299,4 @@ termios_tcsetwinsize(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=f31382658135c774 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c6c6192583b0da36 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/unicodedata.c.h b/Modules/clinic/unicodedata.c.h index 739f498f1d2672..345440eeee89a6 100644 --- a/Modules/clinic/unicodedata.c.h +++ b/Modules/clinic/unicodedata.c.h @@ -36,7 +36,10 @@ unicodedata_UCD_decimal(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[0]) != 1) { - _PyArg_BadArgument("decimal", "argument 1", "a unicode character", args[0]); + PyErr_Format(PyExc_TypeError, + "decimal(): argument 1 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[0])); goto exit; } chr = PyUnicode_READ_CHAR(args[0], 0); @@ -82,7 +85,10 @@ unicodedata_UCD_digit(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[0]) != 1) { - _PyArg_BadArgument("digit", "argument 1", "a unicode character", args[0]); + PyErr_Format(PyExc_TypeError, + "digit(): argument 1 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[0])); goto exit; } chr = PyUnicode_READ_CHAR(args[0], 0); @@ -129,7 +135,10 @@ unicodedata_UCD_numeric(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[0]) != 1) { - _PyArg_BadArgument("numeric", "argument 1", "a unicode character", args[0]); + PyErr_Format(PyExc_TypeError, + "numeric(): argument 1 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[0])); goto exit; } chr = PyUnicode_READ_CHAR(args[0], 0); @@ -167,7 +176,10 @@ unicodedata_UCD_category(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("category", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "category(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -202,7 +214,10 @@ unicodedata_UCD_bidirectional(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("bidirectional", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "bidirectional(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -238,7 +253,10 @@ unicodedata_UCD_combining(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("combining", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "combining(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -279,7 +297,10 @@ unicodedata_UCD_mirrored(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("mirrored", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "mirrored(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -316,7 +337,10 @@ unicodedata_UCD_east_asian_width(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("east_asian_width", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "east_asian_width(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -351,7 +375,10 @@ unicodedata_UCD_decomposition(PyObject *self, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("decomposition", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "decomposition(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } chr = PyUnicode_READ_CHAR(arg, 0); @@ -473,7 +500,10 @@ unicodedata_UCD_name(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (PyUnicode_GET_LENGTH(args[0]) != 1) { - _PyArg_BadArgument("name", "argument 1", "a unicode character", args[0]); + PyErr_Format(PyExc_TypeError, + "name(): argument 1 must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[0])); goto exit; } chr = PyUnicode_READ_CHAR(args[0], 0); @@ -519,4 +549,4 @@ unicodedata_UCD_lookup(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=ea30f89007b2bfff input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8a59d430cee41058 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index 7ff3edf5a557f8..91a3ac76bcf0cc 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -63,7 +63,8 @@ zlib_compress(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec int level = Z_DEFAULT_COMPRESSION; int wbits = MAX_WBITS; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -153,7 +154,8 @@ zlib_decompress(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj int wbits = MAX_WBITS; Py_ssize_t bufsize = DEF_BUF_SIZE; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -271,7 +273,8 @@ zlib_compressobj(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb int strategy = Z_DEFAULT_STRATEGY; Py_buffer zdict = {NULL, NULL}; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 6, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 6, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -390,7 +393,8 @@ zlib_decompressobj(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py int wbits = MAX_WBITS; PyObject *zdict = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -435,7 +439,7 @@ zlib_Compress_compress_impl(compobject *self, PyTypeObject *cls, Py_buffer *data); static PyObject * -zlib_Compress_compress(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +zlib_Compress_compress(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -454,14 +458,15 @@ zlib_Compress_compress(compobject *self, PyTypeObject *cls, PyObject *const *arg PyObject *argsbuf[1]; Py_buffer data = {NULL, NULL}; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = zlib_Compress_compress_impl(self, cls, &data); + return_value = zlib_Compress_compress_impl((compobject *)self, cls, &data); exit: /* Cleanup for data */ @@ -497,7 +502,7 @@ zlib_Decompress_decompress_impl(compobject *self, PyTypeObject *cls, Py_buffer *data, Py_ssize_t max_length); static PyObject * -zlib_Decompress_decompress(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +zlib_Decompress_decompress(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -530,7 +535,8 @@ zlib_Decompress_decompress(compobject *self, PyTypeObject *cls, PyObject *const Py_buffer data = {NULL, NULL}; Py_ssize_t max_length = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -553,7 +559,7 @@ zlib_Decompress_decompress(compobject *self, PyTypeObject *cls, PyObject *const max_length = ival; } skip_optional_pos: - return_value = zlib_Decompress_decompress_impl(self, cls, &data, max_length); + return_value = zlib_Decompress_decompress_impl((compobject *)self, cls, &data, max_length); exit: /* Cleanup for data */ @@ -583,7 +589,7 @@ static PyObject * zlib_Compress_flush_impl(compobject *self, PyTypeObject *cls, int mode); static PyObject * -zlib_Compress_flush(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +zlib_Compress_flush(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -602,7 +608,8 @@ zlib_Compress_flush(compobject *self, PyTypeObject *cls, PyObject *const *args, PyObject *argsbuf[1]; int mode = Z_FINISH; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -614,7 +621,7 @@ zlib_Compress_flush(compobject *self, PyTypeObject *cls, PyObject *const *args, goto exit; } skip_optional_posonly: - return_value = zlib_Compress_flush_impl(self, cls, mode); + return_value = zlib_Compress_flush_impl((compobject *)self, cls, mode); exit: return return_value; @@ -635,13 +642,13 @@ static PyObject * zlib_Compress_copy_impl(compobject *self, PyTypeObject *cls); static PyObject * -zlib_Compress_copy(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +zlib_Compress_copy(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } - return zlib_Compress_copy_impl(self, cls); + return zlib_Compress_copy_impl((compobject *)self, cls); } #endif /* defined(HAVE_ZLIB_COPY) */ @@ -660,13 +667,13 @@ static PyObject * zlib_Compress___copy___impl(compobject *self, PyTypeObject *cls); static PyObject * -zlib_Compress___copy__(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +zlib_Compress___copy__(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "__copy__() takes no arguments"); return NULL; } - return zlib_Compress___copy___impl(self, cls); + return zlib_Compress___copy___impl((compobject *)self, cls); } #endif /* defined(HAVE_ZLIB_COPY) */ @@ -686,7 +693,7 @@ zlib_Compress___deepcopy___impl(compobject *self, PyTypeObject *cls, PyObject *memo); static PyObject * -zlib_Compress___deepcopy__(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +zlib_Compress___deepcopy__(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -705,12 +712,13 @@ zlib_Compress___deepcopy__(compobject *self, PyTypeObject *cls, PyObject *const PyObject *argsbuf[1]; PyObject *memo; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } memo = args[0]; - return_value = zlib_Compress___deepcopy___impl(self, cls, memo); + return_value = zlib_Compress___deepcopy___impl((compobject *)self, cls, memo); exit: return return_value; @@ -733,13 +741,13 @@ static PyObject * zlib_Decompress_copy_impl(compobject *self, PyTypeObject *cls); static PyObject * -zlib_Decompress_copy(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +zlib_Decompress_copy(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); return NULL; } - return zlib_Decompress_copy_impl(self, cls); + return zlib_Decompress_copy_impl((compobject *)self, cls); } #endif /* defined(HAVE_ZLIB_COPY) */ @@ -758,13 +766,13 @@ static PyObject * zlib_Decompress___copy___impl(compobject *self, PyTypeObject *cls); static PyObject * -zlib_Decompress___copy__(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +zlib_Decompress___copy__(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "__copy__() takes no arguments"); return NULL; } - return zlib_Decompress___copy___impl(self, cls); + return zlib_Decompress___copy___impl((compobject *)self, cls); } #endif /* defined(HAVE_ZLIB_COPY) */ @@ -784,7 +792,7 @@ zlib_Decompress___deepcopy___impl(compobject *self, PyTypeObject *cls, PyObject *memo); static PyObject * -zlib_Decompress___deepcopy__(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +zlib_Decompress___deepcopy__(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -803,12 +811,13 @@ zlib_Decompress___deepcopy__(compobject *self, PyTypeObject *cls, PyObject *cons PyObject *argsbuf[1]; PyObject *memo; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } memo = args[0]; - return_value = zlib_Decompress___deepcopy___impl(self, cls, memo); + return_value = zlib_Decompress___deepcopy___impl((compobject *)self, cls, memo); exit: return return_value; @@ -833,7 +842,7 @@ zlib_Decompress_flush_impl(compobject *self, PyTypeObject *cls, Py_ssize_t length); static PyObject * -zlib_Decompress_flush(compobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +zlib_Decompress_flush(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -852,7 +861,8 @@ zlib_Decompress_flush(compobject *self, PyTypeObject *cls, PyObject *const *args PyObject *argsbuf[1]; Py_ssize_t length = DEF_BUF_SIZE; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -872,7 +882,7 @@ zlib_Decompress_flush(compobject *self, PyTypeObject *cls, PyObject *const *args length = ival; } skip_optional_posonly: - return_value = zlib_Decompress_flush_impl(self, cls, length); + return_value = zlib_Decompress_flush_impl((compobject *)self, cls, length); exit: return return_value; @@ -905,7 +915,7 @@ zlib_ZlibDecompressor_decompress_impl(ZlibDecompressor *self, Py_buffer *data, Py_ssize_t max_length); static PyObject * -zlib_ZlibDecompressor_decompress(ZlibDecompressor *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +zlib_ZlibDecompressor_decompress(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -938,7 +948,8 @@ zlib_ZlibDecompressor_decompress(ZlibDecompressor *self, PyObject *const *args, Py_buffer data = {NULL, NULL}; Py_ssize_t max_length = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -961,7 +972,7 @@ zlib_ZlibDecompressor_decompress(ZlibDecompressor *self, PyObject *const *args, max_length = ival; } skip_optional_pos: - return_value = zlib_ZlibDecompressor_decompress_impl(self, &data, max_length); + return_value = zlib_ZlibDecompressor_decompress_impl((ZlibDecompressor *)self, &data, max_length); exit: /* Cleanup for data */ @@ -1098,4 +1109,4 @@ zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #ifndef ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #define ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #endif /* !defined(ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF) */ -/*[clinic end generated code: output=8bb840fb6af43dd4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=969872868c303e8a input=a9049054013a1b77]*/ diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 57bc55632be485..81cbf0d554de3c 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -117,7 +117,7 @@ enum special_types { static enum special_types special_type(double d) { - if (Py_IS_FINITE(d)) { + if (isfinite(d)) { if (d != 0) { if (copysign(1., d) == 1.) return ST_POS; @@ -131,7 +131,7 @@ special_type(double d) return ST_NZERO; } } - if (Py_IS_NAN(d)) + if (isnan(d)) return ST_NAN; if (copysign(1., d) == 1.) return ST_PINF; @@ -139,18 +139,18 @@ special_type(double d) return ST_NINF; } -#define SPECIAL_VALUE(z, table) \ - if (!Py_IS_FINITE((z).real) || !Py_IS_FINITE((z).imag)) { \ - errno = 0; \ - return table[special_type((z).real)] \ - [special_type((z).imag)]; \ +#define SPECIAL_VALUE(z, table) \ + if (!isfinite((z).real) || !isfinite((z).imag)) { \ + errno = 0; \ + return table[special_type((z).real)] \ + [special_type((z).imag)]; \ } #define P Py_MATH_PI #define P14 0.25*Py_MATH_PI #define P12 0.5*Py_MATH_PI #define P34 0.75*Py_MATH_PI -#define INF Py_HUGE_VAL +#define INF Py_INFINITY #define N Py_NAN #define U -9.5426319407711027e33 /* unlikely value, used as placeholder */ @@ -185,15 +185,8 @@ cmath_acos_impl(PyObject *module, Py_complex z) if (fabs(z.real) > CM_LARGE_DOUBLE || fabs(z.imag) > CM_LARGE_DOUBLE) { /* avoid unnecessary overflow for large arguments */ r.real = atan2(fabs(z.imag), z.real); - /* split into cases to make sure that the branch cut has the - correct continuity on systems with unsigned zeros */ - if (z.real < 0.) { - r.imag = -copysign(log(hypot(z.real/2., z.imag/2.)) + - M_LN2*2., z.imag); - } else { - r.imag = copysign(log(hypot(z.real/2., z.imag/2.)) + - M_LN2*2., -z.imag); - } + r.imag = -copysign(log(hypot(z.real/2., z.imag/2.)) + + M_LN2*2., z.imag); } else { s1.real = 1.-z.real; s1.imag = -z.imag; @@ -324,36 +317,6 @@ cmath_atan_impl(PyObject *module, Py_complex z) return r; } -/* Windows screws up atan2 for inf and nan, and alpha Tru64 5.1 doesn't follow - C99 for atan2(0., 0.). */ -static double -c_atan2(Py_complex z) -{ - if (Py_IS_NAN(z.real) || Py_IS_NAN(z.imag)) - return Py_NAN; - if (Py_IS_INFINITY(z.imag)) { - if (Py_IS_INFINITY(z.real)) { - if (copysign(1., z.real) == 1.) - /* atan2(+-inf, +inf) == +-pi/4 */ - return copysign(0.25*Py_MATH_PI, z.imag); - else - /* atan2(+-inf, -inf) == +-pi*3/4 */ - return copysign(0.75*Py_MATH_PI, z.imag); - } - /* atan2(+-inf, x) == +-pi/2 for finite x */ - return copysign(0.5*Py_MATH_PI, z.imag); - } - if (Py_IS_INFINITY(z.real) || z.imag == 0.) { - if (copysign(1., z.real) == 1.) - /* atan2(+-y, +inf) = atan2(+-0, +x) = +-0. */ - return copysign(0., z.imag); - else - /* atan2(+-y, -inf) = atan2(+-0., -x) = +-pi. */ - return copysign(Py_MATH_PI, z.imag); - } - return atan2(z.imag, z.real); -} - static Py_complex atanh_special_values[7][7]; @@ -386,11 +349,7 @@ cmath_atanh_impl(PyObject *module, Py_complex z) */ h = hypot(z.real/2., z.imag/2.); /* safe from overflow */ r.real = z.real/4./h/h; - /* the two negations in the next line cancel each other out - except when working with unsigned zeros: they're there to - ensure that the branch cut has the correct continuity on - systems that don't support signed zeros */ - r.imag = -copysign(Py_MATH_PI/2., -z.imag); + r.imag = copysign(Py_MATH_PI/2., z.imag); errno = 0; } else if (z.real == 1. && ay < CM_SQRT_DBL_MIN) { /* C99 standard says: atanh(1+/-0.) should be inf +/- 0i */ @@ -448,8 +407,8 @@ cmath_cosh_impl(PyObject *module, Py_complex z) double x_minus_one; /* special treatment for cosh(+/-inf + iy) if y is not a NaN */ - if (!Py_IS_FINITE(z.real) || !Py_IS_FINITE(z.imag)) { - if (Py_IS_INFINITY(z.real) && Py_IS_FINITE(z.imag) && + if (!isfinite(z.real) || !isfinite(z.imag)) { + if (isinf(z.real) && isfinite(z.imag) && (z.imag != 0.)) { if (z.real > 0) { r.real = copysign(INF, cos(z.imag)); @@ -466,7 +425,7 @@ cmath_cosh_impl(PyObject *module, Py_complex z) } /* need to set errno = EDOM if y is +/- infinity and x is not a NaN */ - if (Py_IS_INFINITY(z.imag) && !Py_IS_NAN(z.real)) + if (isinf(z.imag) && !isnan(z.real)) errno = EDOM; else errno = 0; @@ -484,7 +443,7 @@ cmath_cosh_impl(PyObject *module, Py_complex z) r.imag = sin(z.imag) * sinh(z.real); } /* detect overflow, and set errno accordingly */ - if (Py_IS_INFINITY(r.real) || Py_IS_INFINITY(r.imag)) + if (isinf(r.real) || isinf(r.imag)) errno = ERANGE; else errno = 0; @@ -509,8 +468,8 @@ cmath_exp_impl(PyObject *module, Py_complex z) Py_complex r; double l; - if (!Py_IS_FINITE(z.real) || !Py_IS_FINITE(z.imag)) { - if (Py_IS_INFINITY(z.real) && Py_IS_FINITE(z.imag) + if (!isfinite(z.real) || !isfinite(z.imag)) { + if (isinf(z.real) && isfinite(z.imag) && (z.imag != 0.)) { if (z.real > 0) { r.real = copysign(INF, cos(z.imag)); @@ -527,9 +486,9 @@ cmath_exp_impl(PyObject *module, Py_complex z) } /* need to set errno = EDOM if y is +/- infinity and x is not a NaN and not -infinity */ - if (Py_IS_INFINITY(z.imag) && - (Py_IS_FINITE(z.real) || - (Py_IS_INFINITY(z.real) && z.real > 0))) + if (isinf(z.imag) && + (isfinite(z.real) || + (isinf(z.real) && z.real > 0))) errno = EDOM; else errno = 0; @@ -546,7 +505,7 @@ cmath_exp_impl(PyObject *module, Py_complex z) r.imag = l*sin(z.imag); } /* detect overflow, and set errno accordingly */ - if (Py_IS_INFINITY(r.real) || Py_IS_INFINITY(r.imag)) + if (isinf(r.real) || isinf(r.imag)) errno = ERANGE; else errno = 0; @@ -686,8 +645,8 @@ cmath_sinh_impl(PyObject *module, Py_complex z) /* special treatment for sinh(+/-inf + iy) if y is finite and nonzero */ - if (!Py_IS_FINITE(z.real) || !Py_IS_FINITE(z.imag)) { - if (Py_IS_INFINITY(z.real) && Py_IS_FINITE(z.imag) + if (!isfinite(z.real) || !isfinite(z.imag)) { + if (isinf(z.real) && isfinite(z.imag) && (z.imag != 0.)) { if (z.real > 0) { r.real = copysign(INF, cos(z.imag)); @@ -704,7 +663,7 @@ cmath_sinh_impl(PyObject *module, Py_complex z) } /* need to set errno = EDOM if y is +/- infinity and x is not a NaN */ - if (Py_IS_INFINITY(z.imag) && !Py_IS_NAN(z.real)) + if (isinf(z.imag) && !isnan(z.real)) errno = EDOM; else errno = 0; @@ -720,7 +679,7 @@ cmath_sinh_impl(PyObject *module, Py_complex z) r.imag = sin(z.imag) * cosh(z.real); } /* detect overflow, and set errno accordingly */ - if (Py_IS_INFINITY(r.real) || Py_IS_INFINITY(r.imag)) + if (isinf(r.real) || isinf(r.imag)) errno = ERANGE; else errno = 0; @@ -856,8 +815,8 @@ cmath_tanh_impl(PyObject *module, Py_complex z) /* special treatment for tanh(+/-inf + iy) if y is finite and nonzero */ - if (!Py_IS_FINITE(z.real) || !Py_IS_FINITE(z.imag)) { - if (Py_IS_INFINITY(z.real) && Py_IS_FINITE(z.imag) + if (!isfinite(z.real) || !isfinite(z.imag)) { + if (isinf(z.real) && isfinite(z.imag) && (z.imag != 0.)) { if (z.real > 0) { r.real = 1.0; @@ -876,7 +835,7 @@ cmath_tanh_impl(PyObject *module, Py_complex z) } /* need to set errno = EDOM if z.imag is +/-infinity and z.real is finite */ - if (Py_IS_INFINITY(z.imag) && Py_IS_FINITE(z.real)) + if (isinf(z.imag) && isfinite(z.real)) errno = EDOM; else errno = 0; @@ -966,7 +925,7 @@ cmath_phase_impl(PyObject *module, Py_complex z) double phi; errno = 0; - phi = c_atan2(z); /* should not cause any exception */ + phi = atan2(z.imag, z.real); /* should not cause any exception */ if (errno != 0) return math_error(); else @@ -991,7 +950,7 @@ cmath_polar_impl(PyObject *module, Py_complex z) double r, phi; errno = 0; - phi = c_atan2(z); /* should not cause any exception */ + phi = atan2(z.imag, z.real); /* should not cause any exception */ r = _Py_c_abs(z); /* sets errno to ERANGE on overflow */ if (errno != 0) return math_error(); @@ -1030,11 +989,11 @@ cmath_rect_impl(PyObject *module, double r, double phi) errno = 0; /* deal with special values */ - if (!Py_IS_FINITE(r) || !Py_IS_FINITE(phi)) { + if (!isfinite(r) || !isfinite(phi)) { /* if r is +/-infinity and phi is finite but nonzero then result is (+-INF +-INF i), but we need to compute cos(phi) and sin(phi) to figure out the signs. */ - if (Py_IS_INFINITY(r) && (Py_IS_FINITE(phi) + if (isinf(r) && (isfinite(phi) && (phi != 0.))) { if (r > 0) { z.real = copysign(INF, cos(phi)); @@ -1051,7 +1010,7 @@ cmath_rect_impl(PyObject *module, double r, double phi) } /* need to set errno = EDOM if r is a nonzero number and phi is infinite */ - if (r != 0. && !Py_IS_NAN(r) && Py_IS_INFINITY(phi)) + if (r != 0. && !isnan(r) && isinf(phi)) errno = EDOM; else errno = 0; @@ -1085,7 +1044,7 @@ static PyObject * cmath_isfinite_impl(PyObject *module, Py_complex z) /*[clinic end generated code: output=ac76611e2c774a36 input=848e7ee701895815]*/ { - return PyBool_FromLong(Py_IS_FINITE(z.real) && Py_IS_FINITE(z.imag)); + return PyBool_FromLong(isfinite(z.real) && isfinite(z.imag)); } /*[clinic input] @@ -1098,7 +1057,7 @@ static PyObject * cmath_isnan_impl(PyObject *module, Py_complex z) /*[clinic end generated code: output=e7abf6e0b28beab7 input=71799f5d284c9baf]*/ { - return PyBool_FromLong(Py_IS_NAN(z.real) || Py_IS_NAN(z.imag)); + return PyBool_FromLong(isnan(z.real) || isnan(z.imag)); } /*[clinic input] @@ -1111,8 +1070,7 @@ static PyObject * cmath_isinf_impl(PyObject *module, Py_complex z) /*[clinic end generated code: output=502a75a79c773469 input=363df155c7181329]*/ { - return PyBool_FromLong(Py_IS_INFINITY(z.real) || - Py_IS_INFINITY(z.imag)); + return PyBool_FromLong(isinf(z.real) || isinf(z.imag)); } /*[clinic input] @@ -1167,8 +1125,7 @@ cmath_isclose_impl(PyObject *module, Py_complex a, Py_complex b, above. */ - if (Py_IS_INFINITY(a.real) || Py_IS_INFINITY(a.imag) || - Py_IS_INFINITY(b.real) || Py_IS_INFINITY(b.imag)) { + if (isinf(a.real) || isinf(a.imag) || isinf(b.real) || isinf(b.imag)) { return 0; } @@ -1261,8 +1218,8 @@ cmath_exec(PyObject *mod) INIT_SPECIAL_VALUES(acosh_special_values, { C(INF,-P34) C(INF,-P) C(INF,-P) C(INF,P) C(INF,P) C(INF,P34) C(INF,N) C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(0.,-P12) C(0.,P12) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(0.,-P12) C(0.,P12) C(U,U) C(INF,P12) C(N,N) + C(INF,-P12) C(U,U) C(0.,-P12) C(0.,P12) C(U,U) C(INF,P12) C(N,P12) + C(INF,-P12) C(U,U) C(0.,-P12) C(0.,P12) C(U,U) C(INF,P12) C(N,P12) C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) C(INF,-P14) C(INF,-0.) C(INF,-0.) C(INF,0.) C(INF,0.) C(INF,P14) C(INF,N) C(INF,N) C(N,N) C(N,N) C(N,N) C(N,N) C(INF,N) C(N,N) @@ -1341,8 +1298,8 @@ cmath_exec(PyObject *mod) INIT_SPECIAL_VALUES(tanh_special_values, { C(-1.,0.) C(U,U) C(-1.,-0.) C(-1.,0.) C(U,U) C(-1.,0.) C(-1.,0.) C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(N,N) C(U,U) C(-0.,-0.) C(-0.,0.) C(U,U) C(N,N) C(N,N) - C(N,N) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(N,N) C(N,N) + C(-0.0,N) C(U,U) C(-0.,-0.) C(-0.,0.) C(U,U) C(-0.0,N) C(-0.,N) + C(0.0,N) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.0,N) C(0.,N) C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) C(1.,0.) C(U,U) C(1.,-0.) C(1.,0.) C(U,U) C(1.,0.) C(1.,0.) C(N,N) C(N,N) C(N,-0.) C(N,0.) C(N,N) C(N,N) C(N,N) @@ -1363,6 +1320,7 @@ cmath_exec(PyObject *mod) static PyModuleDef_Slot cmath_slots[] = { {Py_mod_exec, cmath_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/config.c.in b/Modules/config.c.in index 53b4fb285498d0..c578cd103dc629 100644 --- a/Modules/config.c.in +++ b/Modules/config.c.in @@ -1,13 +1,3 @@ -/* -*- C -*- *********************************************** -Copyright (c) 2000, BeOpen.com. -Copyright (c) 1995-2000, Corporation for National Research Initiatives. -Copyright (c) 1990-1995, Stichting Mathematisch Centrum. -All rights reserved. - -See the file "Misc/COPYRIGHT" for information on usage and -redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES. -******************************************************************/ - /* Module configuration */ /* !!! !!! !!! This file is edited by the makesetup script !!! !!! !!! */ diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index 1100e9f6094352..9557d68e759497 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -1,10 +1,9 @@ /* Errno module */ +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -846,6 +845,9 @@ errno_exec(PyObject *module) #ifdef ENOKEY add_errcode("ENOKEY", ENOKEY, "Required key not available"); #endif +#ifdef EHWPOISON + add_errcode("EHWPOISON", EHWPOISON, "Memory page has hardware error"); +#endif #ifdef EKEYEXPIRED add_errcode("EKEYEXPIRED", EKEYEXPIRED, "Key has expired"); #endif @@ -952,6 +954,7 @@ errno_exec(PyObject *module) static PyModuleDef_Slot errno_slots[] = { {Py_mod_exec, errno_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 95464b0dd17735..523b37d8d5787d 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -18,6 +18,7 @@ Copyright (c) 2022 Thijs Schreijer Copyright (c) 2023 Hanno Böck Copyright (c) 2023 Sony Corporation / Snild Dolkow + Copyright (c) 2024 Taichi Haradaguchi <20001722@ymail.ne.jp> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -129,7 +130,9 @@ enum XML_Error { /* Added in 2.3.0. */ XML_ERROR_NO_BUFFER, /* Added in 2.4.0. */ - XML_ERROR_AMPLIFICATION_LIMIT_BREACH + XML_ERROR_AMPLIFICATION_LIMIT_BREACH, + /* Added in 2.6.4. */ + XML_ERROR_NOT_STARTED, }; enum XML_Content_Type { @@ -1042,7 +1045,7 @@ typedef struct { XMLPARSEAPI(const XML_Feature *) XML_GetFeatureList(void); -#if XML_GE == 1 +#if defined(XML_DTD) || (defined(XML_GE) && XML_GE == 1) /* Added in Expat 2.4.0 for XML_DTD defined and * added in Expat 2.6.0 for XML_GE == 1. */ XMLPARSEAPI(XML_Bool) @@ -1065,7 +1068,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); */ #define XML_MAJOR_VERSION 2 #define XML_MINOR_VERSION 6 -#define XML_MICRO_VERSION 0 +#define XML_MICRO_VERSION 4 #ifdef __cplusplus } diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index 12c560e14716ff..567872b09836e1 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -40,6 +40,10 @@ #ifndef Expat_External_INCLUDED #define Expat_External_INCLUDED 1 +/* Namespace external symbols to allow multiple libexpat version to + co-exist. */ +#include "pyexpatns.h" + /* External API definitions */ /* Expat tries very hard to make the API boundary very specifically @@ -64,11 +68,6 @@ compiled with the cdecl calling convention as the default since system headers may assume the cdecl convention. */ - -/* Namespace external symbols to allow multiple libexpat version to - co-exist. */ -#include "pyexpatns.h" - #ifndef XMLCALL # if defined(_MSC_VER) # define XMLCALL __cdecl diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index cce71e4c5164b5..167ec36804a43b 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -28,10 +28,11 @@ Copyright (c) 2002-2003 Fred L. Drake, Jr. Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2003 Greg Stein - Copyright (c) 2016-2023 Sebastian Pipping + Copyright (c) 2016-2024 Sebastian Pipping Copyright (c) 2018 Yury Gribov Copyright (c) 2019 David Loffredo - Copyright (c) 2023 Sony Corporation / Snild Dolkow + Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow + Copyright (c) 2024 Taichi Haradaguchi <20001722@ymail.ne.jp> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -155,14 +156,20 @@ extern "C" { void _INTERNAL_trim_to_complete_utf8_characters(const char *from, const char **fromLimRef); -#if XML_GE == 1 +#if defined(XML_GE) && XML_GE == 1 unsigned long long testingAccountingGetCountBytesDirect(XML_Parser parser); unsigned long long testingAccountingGetCountBytesIndirect(XML_Parser parser); const char *unsignedCharToPrintable(unsigned char c); #endif -extern XML_Bool g_reparseDeferralEnabledDefault; // written ONLY in runtests.c -extern unsigned int g_parseAttempts; // used for testing only +extern +#if ! defined(XML_TESTING) + const +#endif + XML_Bool g_reparseDeferralEnabledDefault; // written ONLY in runtests.c +#if defined(XML_TESTING) +extern unsigned int g_bytesScanned; // used for testing only +#endif #ifdef __cplusplus } diff --git a/Modules/expat/refresh.sh b/Modules/expat/refresh.sh new file mode 100755 index 00000000000000..82a9dbc23ad26b --- /dev/null +++ b/Modules/expat/refresh.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# +# Use this script to update libexpat + +set -e +set -o pipefail + +if [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then + echo "A bash version >= 4 required. Got: $BASH_VERSION" >&2 + exit 1 +fi + +# Update this when updating to a new version after verifying that the changes +# the update brings in are good. These values are used for verifying the SBOM, too. +expected_libexpat_tag="R_2_6_4" +expected_libexpat_version="2.6.4" +expected_libexpat_sha256="fd03b7172b3bd7427a3e7a812063f74754f24542429b634e0db6511b53fb2278" + +expat_dir="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")" +cd ${expat_dir} + +# Step 1: download and copy files +curl --location "https://github.com/libexpat/libexpat/releases/download/${expected_libexpat_tag}/expat-${expected_libexpat_version}.tar.gz" > libexpat.tar.gz +echo "${expected_libexpat_sha256} libexpat.tar.gz" | sha256sum --check + +# Step 2: Pull files from the libexpat distribution +declare -a lib_files +lib_files=( + ascii.h + asciitab.h + expat.h + expat_external.h + iasciitab.h + internal.h + latin1tab.h + nametab.h + siphash.h + utf8tab.h + winconfig.h + xmlparse.c + xmlrole.c + xmlrole.h + xmltok.c + xmltok.h + xmltok_impl.c + xmltok_impl.h + xmltok_ns.c +) +for f in "${lib_files[@]}"; do + tar xzvf libexpat.tar.gz "expat-${expected_libexpat_version}/lib/${f}" --strip-components 2 +done +rm libexpat.tar.gz + +# Step 3: Add the namespacing include to expat_external.h +sed -i 's/#define Expat_External_INCLUDED 1/&\n\n\/* Namespace external symbols to allow multiple libexpat version to\n co-exist. \*\/\n#include "pyexpatns.h"/' expat_external.h + +echo "Updated; verify all is okay using git diff and git status." diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h index a1ed99e687bd6e..04f6f74585b5a2 100644 --- a/Modules/expat/siphash.h +++ b/Modules/expat/siphash.h @@ -126,8 +126,7 @@ | ((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40) \ | ((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56)) -#define SIPHASH_INITIALIZER \ - { 0, 0, 0, 0, {0}, 0, 0 } +#define SIPHASH_INITIALIZER {0, 0, 0, 0, {0}, 0, 0} struct siphash { uint64_t v0, v1, v2, v3; diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index aaf0fa9c8f96d1..a4e091e7c33c0a 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* 628e24d4966bedbd4800f6ed128d06d29703765b4bce12d3b7f099f90f842fc9 (2.6.0+) +/* c5625880f4bf417c1463deee4eb92d86ff413f802048621c57e25fe483eb59e4 (2.6.4+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -38,7 +38,9 @@ Copyright (c) 2022 Jann Horn Copyright (c) 2022 Sean McBride Copyright (c) 2023 Owain Davies - Copyright (c) 2023 Sony Corporation / Snild Dolkow + Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow + Copyright (c) 2024 Berkay Eren Ürün + Copyright (c) 2024 Hanno Böck Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -210,7 +212,7 @@ typedef char ICHAR; #endif /* Round up n to be a multiple of sz, where sz is a power of 2. */ -#define ROUND_UP(n, sz) (((n) + ((sz)-1)) & ~((sz)-1)) +#define ROUND_UP(n, sz) (((n) + ((sz) - 1)) & ~((sz) - 1)) /* Do safe (NULL-aware) pointer arithmetic */ #define EXPAT_SAFE_PTR_DIFF(p, q) (((p) && (q)) ? ((p) - (q)) : 0) @@ -248,7 +250,7 @@ static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key); it odd, since odd numbers are always relative prime to a power of 2. */ #define SECOND_HASH(hash, mask, power) \ - ((((hash) & ~(mask)) >> ((power)-1)) & ((mask) >> 2)) + ((((hash) & ~(mask)) >> ((power) - 1)) & ((mask) >> 2)) #define PROBE_STEP(hash, mask, power) \ ((unsigned char)((SECOND_HASH(hash, mask, power)) | 1)) @@ -294,7 +296,7 @@ typedef struct { The name of the element is stored in both the document and API encodings. The memory buffer 'buf' is a separately-allocated memory area which stores the name. During the XML_Parse()/ - XMLParseBuffer() when the element is open, the memory for the 'raw' + XML_ParseBuffer() when the element is open, the memory for the 'raw' version of the name (in the document encoding) is shared with the document buffer. If the element is open across calls to XML_Parse()/XML_ParseBuffer(), the buffer is re-allocated to @@ -629,8 +631,14 @@ static unsigned long getDebugLevel(const char *variableName, ? 0 \ : ((*((pool)->ptr)++ = c), 1)) -XML_Bool g_reparseDeferralEnabledDefault = XML_TRUE; // write ONLY in runtests.c -unsigned int g_parseAttempts = 0; // used for testing only +#if ! defined(XML_TESTING) +const +#endif + XML_Bool g_reparseDeferralEnabledDefault + = XML_TRUE; // write ONLY in runtests.c +#if defined(XML_TESTING) +unsigned int g_bytesScanned = 0; // used for testing only +#endif struct XML_ParserStruct { /* The first member must be m_userData so that the XML_GetUserData @@ -1017,7 +1025,9 @@ callProcessor(XML_Parser parser, const char *start, const char *end, return XML_ERROR_NONE; } } - g_parseAttempts += 1; +#if defined(XML_TESTING) + g_bytesScanned += (unsigned)have_now; +#endif const enum XML_Error ret = parser->m_processor(parser, start, end, endPtr); if (ret == XML_ERROR_NONE) { // if we consumed nothing, remember what we had on this parse attempt. @@ -2030,6 +2040,12 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { if (parser == NULL) return XML_STATUS_ERROR; + + if (len < 0) { + parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; + return XML_STATUS_ERROR; + } + switch (parser->m_parsingStatus.parsing) { case XML_SUSPENDED: parser->m_errorCode = XML_ERROR_SUSPENDED; @@ -2219,6 +2235,9 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable) { if (parser == NULL) return XML_STATUS_ERROR; switch (parser->m_parsingStatus.parsing) { + case XML_INITIALIZED: + parser->m_errorCode = XML_ERROR_NOT_STARTED; + return XML_STATUS_ERROR; case XML_SUSPENDED: if (resumable) { parser->m_errorCode = XML_ERROR_SUSPENDED; @@ -2229,7 +2248,7 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable) { case XML_FINISHED: parser->m_errorCode = XML_ERROR_FINISHED; return XML_STATUS_ERROR; - default: + case XML_PARSING: if (resumable) { #ifdef XML_DTD if (parser->m_isParamEntity) { @@ -2240,6 +2259,9 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable) { parser->m_parsingStatus.parsing = XML_SUSPENDED; } else parser->m_parsingStatus.parsing = XML_FINISHED; + break; + default: + assert(0); } return XML_STATUS_OK; } @@ -2504,6 +2526,9 @@ XML_ErrorString(enum XML_Error code) { case XML_ERROR_AMPLIFICATION_LIMIT_BREACH: return XML_L( "limit on input amplification factor (from DTD and entities) breached"); + /* Added in 2.6.4. */ + case XML_ERROR_NOT_STARTED: + return XML_L("parser not started"); } return NULL; } @@ -5838,18 +5863,17 @@ processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) { /* Set a safe default value in case 'next' does not get set */ next = textStart; -#ifdef XML_DTD if (entity->is_param) { int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next); result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, tok, next, &next, XML_FALSE, XML_FALSE, XML_ACCOUNT_ENTITY_EXPANSION); - } else -#endif /* XML_DTD */ + } else { result = doContent(parser, parser->m_tagLevel, parser->m_internalEncoding, textStart, textEnd, &next, XML_FALSE, XML_ACCOUNT_ENTITY_EXPANSION); + } if (result == XML_ERROR_NONE) { if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) { @@ -5886,18 +5910,17 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, /* Set a safe default value in case 'next' does not get set */ next = textStart; -#ifdef XML_DTD if (entity->is_param) { int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next); result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, tok, next, &next, XML_FALSE, XML_TRUE, XML_ACCOUNT_ENTITY_EXPANSION); - } else -#endif /* XML_DTD */ + } else { result = doContent(parser, openEntity->startTagLevel, parser->m_internalEncoding, textStart, textEnd, &next, XML_FALSE, XML_ACCOUNT_ENTITY_EXPANSION); + } if (result != XML_ERROR_NONE) return result; @@ -5924,7 +5947,6 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, return XML_ERROR_NONE; } -#ifdef XML_DTD if (entity->is_param) { int tok; parser->m_processor = prologProcessor; @@ -5932,9 +5954,7 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr, (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE, XML_ACCOUNT_DIRECT); - } else -#endif /* XML_DTD */ - { + } else { parser->m_processor = contentProcessor; /* see externalEntityContentProcessor vs contentProcessor */ result = doContent(parser, parser->m_parentParser ? 1 : 0, @@ -6232,7 +6252,7 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, dtd->keepProcessing = dtd->standalone; goto endEntityValue; } - if (entity->open) { + if (entity->open || (entity == parser->m_declEntity)) { if (enc == parser->m_encoding) parser->m_eventPtr = entityTextPtr; result = XML_ERROR_RECURSIVE_ENTITY_REF; @@ -7008,6 +7028,16 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, if (! newE) return 0; if (oldE->nDefaultAtts) { + /* Detect and prevent integer overflow. + * The preprocessor guard addresses the "always false" warning + * from -Wtype-limits on platforms where + * sizeof(int) < sizeof(size_t), e.g. on x86_64. */ +#if UINT_MAX >= SIZE_MAX + if ((size_t)oldE->nDefaultAtts + > ((size_t)(-1) / sizeof(DEFAULT_ATTRIBUTE))) { + return 0; + } +#endif newE->defaultAtts = ms->malloc_fcn(oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); if (! newE->defaultAtts) { @@ -7550,6 +7580,15 @@ nextScaffoldPart(XML_Parser parser) { int next; if (! dtd->scaffIndex) { + /* Detect and prevent integer overflow. + * The preprocessor guard addresses the "always false" warning + * from -Wtype-limits on platforms where + * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ +#if UINT_MAX >= SIZE_MAX + if (parser->m_groupSize > ((size_t)(-1) / sizeof(int))) { + return -1; + } +#endif dtd->scaffIndex = (int *)MALLOC(parser, parser->m_groupSize * sizeof(int)); if (! dtd->scaffIndex) return -1; @@ -7779,6 +7818,8 @@ copyString(const XML_Char *s, const XML_Memory_Handling_Suite *memsuite) { static float accountingGetCurrentAmplification(XML_Parser rootParser) { + // 1.........1.........12 => 22 + const size_t lenOfShortestInclude = sizeof("") - 1; const XmlBigCount countBytesOutput = rootParser->m_accounting.countBytesDirect + rootParser->m_accounting.countBytesIndirect; @@ -7786,7 +7827,9 @@ accountingGetCurrentAmplification(XML_Parser rootParser) { = rootParser->m_accounting.countBytesDirect ? (countBytesOutput / (float)(rootParser->m_accounting.countBytesDirect)) - : 1.0f; + : ((lenOfShortestInclude + + rootParser->m_accounting.countBytesIndirect) + / (float)lenOfShortestInclude); assert(! rootParser->m_parentParser); return amplificationFactor; } @@ -7823,7 +7866,7 @@ accountingReportDiff(XML_Parser rootParser, assert(! rootParser->m_parentParser); fprintf(stderr, - " (+" EXPAT_FMT_PTRDIFF_T("6") " bytes %s|%d, xmlparse.c:%d) %*s\"", + " (+" EXPAT_FMT_PTRDIFF_T("6") " bytes %s|%u, xmlparse.c:%d) %*s\"", bytesMore, (account == XML_ACCOUNT_DIRECT) ? "DIR" : "EXP", levelsAwayFromRootParser, source_line, 10, ""); @@ -7936,7 +7979,7 @@ entityTrackingReportStats(XML_Parser rootParser, ENTITY *entity, fprintf( stderr, - "expat: Entities(%p): Count %9d, depth %2d/%2d %*s%s%s; %s length %d (xmlparse.c:%d)\n", + "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s; %s length %d (xmlparse.c:%d)\n", (void *)rootParser, rootParser->m_entity_stats.countEverOpened, rootParser->m_entity_stats.currentDepth, rootParser->m_entity_stats.maximumDepthSeen, diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 02e94a21191483..b44b964b29484b 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "pycore_ceval.h" // _PyEval_IsGILEnabled #include "pycore_initconfig.h" // _PyStatus_ERR #include "pycore_pyerrors.h" // _Py_DumpExtensionModules #include "pycore_pystate.h" // _PyThreadState_GET() @@ -27,6 +28,8 @@ # include // getauxval() #endif +/* Sentinel to ignore all_threads on free-threading */ +#define FT_IGNORE_ALL_THREADS 2 /* Allocate at maximum 100 MiB of the stack to raise the stack overflow */ #define STACK_OVERFLOW_MAX_SIZE (100 * 1024 * 1024) @@ -75,7 +78,7 @@ static fault_handler_t faulthandler_handlers[] = { #ifdef SIGILL {SIGILL, 0, "Illegal instruction", }, #endif - {SIGFPE, 0, "Floating point exception", }, + {SIGFPE, 0, "Floating-point exception", }, {SIGABRT, 0, "Aborted", }, /* define SIGSEGV at the end to make it the default choice if searching the handler fails in faulthandler_fatal_error() */ @@ -201,10 +204,13 @@ faulthandler_dump_traceback(int fd, int all_threads, PyGILState_GetThisThreadState(). */ PyThreadState *tstate = PyGILState_GetThisThreadState(); - if (all_threads) { + if (all_threads == 1) { (void)_Py_DumpTracebackThreads(fd, NULL, tstate); } else { + if (all_threads == FT_IGNORE_ALL_THREADS) { + PUTS(fd, "\n"); + } if (tstate != NULL) _Py_DumpTraceback(fd, tstate); } @@ -237,7 +243,12 @@ faulthandler_dump_traceback_py(PyObject *self, return NULL; if (all_threads) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + /* gh-128400: Accessing other thread states while they're running + * isn't safe if those threads are running. */ + _PyEval_StopTheWorld(interp); errmsg = _Py_DumpTracebackThreads(fd, NULL, tstate); + _PyEval_StartTheWorld(interp); if (errmsg != NULL) { PyErr_SetString(PyExc_RuntimeError, errmsg); return NULL; @@ -266,6 +277,27 @@ faulthandler_disable_fatal_handler(fault_handler_t *handler) #endif } +static int +deduce_all_threads(void) +{ +#ifndef Py_GIL_DISABLED + return fatal_error.all_threads; +#else + if (fatal_error.all_threads == 0) { + return 0; + } + // We can't use _PyThreadState_GET, so use the stored GILstate one + PyThreadState *tstate = PyGILState_GetThisThreadState(); + if (tstate == NULL) { + return 0; + } + + /* In theory, it's safe to dump all threads if the GIL is enabled */ + return _PyEval_IsGILEnabled(tstate) + ? fatal_error.all_threads + : FT_IGNORE_ALL_THREADS; +#endif +} /* Handler for SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL signals. @@ -320,7 +352,7 @@ faulthandler_fatal_error(int signum) PUTS(fd, "\n\n"); } - faulthandler_dump_traceback(fd, fatal_error.all_threads, + faulthandler_dump_traceback(fd, deduce_all_threads(), fatal_error.interp); _Py_DumpExtensionModules(fd, fatal_error.interp); @@ -396,7 +428,7 @@ faulthandler_exc_handler(struct _EXCEPTION_POINTERS *exc_info) } } - faulthandler_dump_traceback(fd, fatal_error.all_threads, + faulthandler_dump_traceback(fd, deduce_all_threads(), fatal_error.interp); /* call the next exception handler */ @@ -1192,58 +1224,67 @@ PyDoc_STRVAR(module_doc, static PyMethodDef module_methods[] = { {"enable", _PyCFunction_CAST(faulthandler_py_enable), METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("enable(file=sys.stderr, all_threads=True): " - "enable the fault handler")}, + PyDoc_STR("enable($module, /, file=sys.stderr, all_threads=True)\n--\n\n" + "Enable the fault handler.")}, {"disable", faulthandler_disable_py, METH_NOARGS, - PyDoc_STR("disable(): disable the fault handler")}, + PyDoc_STR("disable($module, /)\n--\n\n" + "Disable the fault handler.")}, {"is_enabled", faulthandler_is_enabled, METH_NOARGS, - PyDoc_STR("is_enabled()->bool: check if the handler is enabled")}, + PyDoc_STR("is_enabled($module, /)\n--\n\n" + "Check if the handler is enabled.")}, {"dump_traceback", _PyCFunction_CAST(faulthandler_dump_traceback_py), METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("dump_traceback(file=sys.stderr, all_threads=True): " - "dump the traceback of the current thread, or of all threads " - "if all_threads is True, into file")}, + PyDoc_STR("dump_traceback($module, /, file=sys.stderr, all_threads=True)\n--\n\n" + "Dump the traceback of the current thread, or of all threads " + "if all_threads is True, into file.")}, {"dump_traceback_later", _PyCFunction_CAST(faulthandler_dump_traceback_later), METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("dump_traceback_later(timeout, repeat=False, file=sys.stderr, exit=False):\n" - "dump the traceback of all threads in timeout seconds,\n" + PyDoc_STR("dump_traceback_later($module, /, timeout, repeat=False, file=sys.stderr, exit=False)\n--\n\n" + "Dump the traceback of all threads in timeout seconds,\n" "or each timeout seconds if repeat is True. If exit is True, " "call _exit(1) which is not safe.")}, {"cancel_dump_traceback_later", faulthandler_cancel_dump_traceback_later_py, METH_NOARGS, - PyDoc_STR("cancel_dump_traceback_later():\ncancel the previous call " - "to dump_traceback_later().")}, + PyDoc_STR("cancel_dump_traceback_later($module, /)\n--\n\n" + "Cancel the previous call to dump_traceback_later().")}, #ifdef FAULTHANDLER_USER {"register", _PyCFunction_CAST(faulthandler_register_py), METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("register(signum, file=sys.stderr, all_threads=True, chain=False): " - "register a handler for the signal 'signum': dump the " + PyDoc_STR("register($module, /, signum, file=sys.stderr, all_threads=True, chain=False)\n--\n\n" + "Register a handler for the signal 'signum': dump the " "traceback of the current thread, or of all threads if " - "all_threads is True, into file")}, + "all_threads is True, into file.")}, {"unregister", - _PyCFunction_CAST(faulthandler_unregister_py), METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("unregister(signum): unregister the handler of the signal " - "'signum' registered by register()")}, + _PyCFunction_CAST(faulthandler_unregister_py), METH_VARARGS, + PyDoc_STR("unregister($module, signum, /)\n--\n\n" + "Unregister the handler of the signal " + "'signum' registered by register().")}, #endif {"_read_null", faulthandler_read_null, METH_NOARGS, - PyDoc_STR("_read_null(): read from NULL, raise " - "a SIGSEGV or SIGBUS signal depending on the platform")}, + PyDoc_STR("_read_null($module, /)\n--\n\n" + "Read from NULL, raise " + "a SIGSEGV or SIGBUS signal depending on the platform.")}, {"_sigsegv", faulthandler_sigsegv, METH_VARARGS, - PyDoc_STR("_sigsegv(release_gil=False): raise a SIGSEGV signal")}, + PyDoc_STR("_sigsegv($module, release_gil=False, /)\n--\n\n" + "Raise a SIGSEGV signal.")}, {"_fatal_error_c_thread", faulthandler_fatal_error_c_thread, METH_NOARGS, - PyDoc_STR("fatal_error_c_thread(): " - "call Py_FatalError() in a new C thread.")}, + PyDoc_STR("_fatal_error_c_thread($module, /)\n--\n\n" + "Call Py_FatalError() in a new C thread.")}, {"_sigabrt", faulthandler_sigabrt, METH_NOARGS, - PyDoc_STR("_sigabrt(): raise a SIGABRT signal")}, + PyDoc_STR("_sigabrt($module, /)\n--\n\n" + "Raise a SIGABRT signal.")}, {"_sigfpe", (PyCFunction)faulthandler_sigfpe, METH_NOARGS, - PyDoc_STR("_sigfpe(): raise a SIGFPE signal")}, + PyDoc_STR("_sigfpe($module, /)\n--\n\n" + "Raise a SIGFPE signal.")}, #ifdef FAULTHANDLER_STACK_OVERFLOW {"_stack_overflow", faulthandler_stack_overflow, METH_NOARGS, - PyDoc_STR("_stack_overflow(): recursive call to raise a stack overflow")}, + PyDoc_STR("_stack_overflow($module, /)\n--\n\n" + "Recursive call to raise a stack overflow.")}, #endif #ifdef MS_WINDOWS {"_raise_exception", faulthandler_raise_exception, METH_VARARGS, - PyDoc_STR("raise_exception(code, flags=0): Call RaiseException(code, flags).")}, + PyDoc_STR("_raise_exception($module, code, flags=0, /)\n--\n\n" + "Call RaiseException(code, flags).")}, #endif {NULL, NULL} /* sentinel */ }; @@ -1283,6 +1324,7 @@ static PyModuleDef_Slot faulthandler_slots[] = { {Py_mod_exec, PyExec_faulthandler}, // XXX gh-103092: fix isolation. //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index 0d16602692b62d..90ebfd7e99a777 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -1,22 +1,25 @@ /* fcntl module */ -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 +// Need limited C API version 3.13 for PyLong_AsInt() +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" +#include // EINTR +#include // fcntl() +#include // memcpy() +#include // ioctl() #ifdef HAVE_SYS_FILE_H -#include +# include // flock() #endif #ifdef HAVE_LINUX_FS_H -#include +# include #endif - -#include -#include #ifdef HAVE_STROPTS_H -#include +# include // I_FLUSHBAND #endif /*[clinic input] @@ -109,7 +112,7 @@ fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) fcntl.ioctl fd: fildes - request as code: unsigned_int(bitwise=True) + request as code: unsigned_long(bitwise=True) arg as ob_arg: object(c_default='NULL') = 0 mutate_flag as mutate_arg: bool = True / @@ -145,9 +148,9 @@ code. [clinic start generated code]*/ static PyObject * -fcntl_ioctl_impl(PyObject *module, int fd, unsigned int code, +fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *ob_arg, int mutate_arg) -/*[clinic end generated code: output=7f7f5840c65991be input=967b4a4cbeceb0a8]*/ +/*[clinic end generated code: output=3d8eb6828666cea1 input=cee70f6a27311e58]*/ { #define IOCTL_BUFSZ 1024 /* We use the unsigned non-checked 'I' format for the 'code' parameter @@ -167,7 +170,7 @@ fcntl_ioctl_impl(PyObject *module, int fd, unsigned int code, Py_ssize_t len; char buf[IOCTL_BUFSZ+1]; /* argument plus NUL byte */ - if (PySys_Audit("fcntl.ioctl", "iIO", fd, code, + if (PySys_Audit("fcntl.ioctl", "ikO", fd, code, ob_arg ? ob_arg : Py_None) < 0) { return NULL; } @@ -577,12 +580,17 @@ all_ins(PyObject* m) #ifdef F_GETPIPE_SZ if (PyModule_AddIntMacro(m, F_GETPIPE_SZ)) return -1; #endif + +/* On Android, FICLONE is blocked by SELinux. */ +#ifndef __ANDROID__ #ifdef FICLONE if (PyModule_AddIntMacro(m, FICLONE)) return -1; #endif #ifdef FICLONERANGE if (PyModule_AddIntMacro(m, FICLONERANGE)) return -1; #endif +#endif + #ifdef F_GETOWN_EX // since Linux 2.6.32 if (PyModule_AddIntMacro(m, F_GETOWN_EX)) return -1; @@ -742,6 +750,7 @@ fcntl_exec(PyObject *module) static PyModuleDef_Slot fcntl_slots[] = { {Py_mod_exec, fcntl_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 9807d2e7d48a36..ad13496b06deaf 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -8,6 +8,7 @@ #include "pycore_gc.h" #include "pycore_object.h" // _PyObject_IS_GC() #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_tuple.h" // _PyTuple_FromArray() typedef struct _gc_runtime_state GCState; @@ -158,17 +159,12 @@ gc_set_threshold_impl(PyObject *module, int threshold0, int group_right_1, { GCState *gcstate = get_gc_state(); - gcstate->generations[0].threshold = threshold0; + gcstate->young.threshold = threshold0; if (group_right_1) { - gcstate->generations[1].threshold = threshold1; + gcstate->old[0].threshold = threshold1; } if (group_right_2) { - gcstate->generations[2].threshold = threshold2; - - /* generations higher than 2 get the same threshold */ - for (int i = 3; i < NUM_GENERATIONS; i++) { - gcstate->generations[i].threshold = gcstate->generations[2].threshold; - } + gcstate->old[1].threshold = threshold2; } Py_RETURN_NONE; } @@ -185,9 +181,9 @@ gc_get_threshold_impl(PyObject *module) { GCState *gcstate = get_gc_state(); return Py_BuildValue("(iii)", - gcstate->generations[0].threshold, - gcstate->generations[1].threshold, - gcstate->generations[2].threshold); + gcstate->young.threshold, + gcstate->old[0].threshold, + 0); } /*[clinic input] @@ -207,34 +203,34 @@ gc_get_count_impl(PyObject *module) struct _gc_thread_state *gc = &tstate->gc; // Flush the local allocation count to the global count - _Py_atomic_add_int(&gcstate->generations[0].count, (int)gc->alloc_count); + _Py_atomic_add_int(&gcstate->young.count, (int)gc->alloc_count); gc->alloc_count = 0; #endif return Py_BuildValue("(iii)", - gcstate->generations[0].count, - gcstate->generations[1].count, - gcstate->generations[2].count); + gcstate->young.count, + gcstate->old[gcstate->visited_space].count, + gcstate->old[gcstate->visited_space^1].count); } /*[clinic input] gc.get_referrers - *objs as args: object + *objs: tuple Return the list of objects that directly refer to any of 'objs'. [clinic start generated code]*/ static PyObject * -gc_get_referrers_impl(PyObject *module, PyObject *args) -/*[clinic end generated code: output=296a09587f6a86b5 input=bae96961b14a0922]*/ +gc_get_referrers_impl(PyObject *module, PyObject *objs) +/*[clinic end generated code: output=929d6dff26f609b9 input=9102be7ebee69ee3]*/ { - if (PySys_Audit("gc.get_referrers", "(O)", args) < 0) { + if (PySys_Audit("gc.get_referrers", "(O)", objs) < 0) { return NULL; } PyInterpreterState *interp = _PyInterpreterState_GET(); - return _PyGC_GetReferrers(interp, args); + return _PyGC_GetReferrers(interp, objs); } /* Append obj to list; return true if error (out of memory), false if OK. */ @@ -268,27 +264,28 @@ append_referrents(PyObject *result, PyObject *args) /*[clinic input] gc.get_referents - *objs as args: object + *objs: tuple Return the list of objects that are directly referred to by 'objs'. [clinic start generated code]*/ static PyObject * -gc_get_referents_impl(PyObject *module, PyObject *args) -/*[clinic end generated code: output=d47dc02cefd06fe8 input=b3ceab0c34038cbf]*/ +gc_get_referents_impl(PyObject *module, PyObject *objs) +/*[clinic end generated code: output=6dfde40cd1588e1d input=55c078a6d0248fe0]*/ { - if (PySys_Audit("gc.get_referents", "(O)", args) < 0) { + if (PySys_Audit("gc.get_referents", "(O)", objs) < 0) { return NULL; } PyInterpreterState *interp = _PyInterpreterState_GET(); PyObject *result = PyList_New(0); - if (result == NULL) + if (result == NULL) { return NULL; + } // NOTE: stop the world is a no-op in default build _PyEval_StopTheWorld(interp); - int err = append_referrents(result, args); + int err = append_referrents(result, objs); _PyEval_StartTheWorld(interp); if (err < 0) { @@ -331,7 +328,7 @@ gc_get_objects_impl(PyObject *module, Py_ssize_t generation) } PyInterpreterState *interp = _PyInterpreterState_GET(); - return _PyGC_GetObjects(interp, generation); + return _PyGC_GetObjects(interp, (int)generation); } /*[clinic input] @@ -540,6 +537,7 @@ gcmodule_exec(PyObject *module) static PyModuleDef_Slot gcmodule_slots[] = { {Py_mod_exec, gcmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/getpath.c b/Modules/getpath.c index abed139028244a..2d3c9757298d16 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -17,10 +17,13 @@ #endif #ifdef __APPLE__ -# include # include #endif +#ifdef HAVE_DLFCN_H +# include +#endif + /* Reference the precompiled getpath.py */ #include "Python/frozen_modules/getpath.h" @@ -108,7 +111,7 @@ getpath_dirname(PyObject *Py_UNUSED(self), PyObject *args) Py_ssize_t end = PyUnicode_GET_LENGTH(path); Py_ssize_t pos = PyUnicode_FindChar(path, SEP, 0, end, -1); if (pos < 0) { - return PyUnicode_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } return PyUnicode_Substring(path, 0, pos); } @@ -258,7 +261,7 @@ getpath_joinpath(PyObject *Py_UNUSED(self), PyObject *args) } Py_ssize_t n = PyTuple_GET_SIZE(args); if (n == 0) { - return PyUnicode_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } /* Convert all parts to wchar and accumulate max final length */ wchar_t **parts = (wchar_t **)PyMem_Malloc(n * sizeof(wchar_t *)); @@ -302,7 +305,7 @@ getpath_joinpath(PyObject *Py_UNUSED(self), PyObject *args) PyErr_NoMemory(); return NULL; } - return PyUnicode_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } final[0] = '\0'; @@ -803,36 +806,25 @@ progname_to_dict(PyObject *dict, const char *key) static int library_to_dict(PyObject *dict, const char *key) { +/* macOS framework builds do not link against a libpython dynamic library, but + instead link against a macOS Framework. */ +#if defined(Py_ENABLE_SHARED) || defined(WITH_NEXT_FRAMEWORK) + #ifdef MS_WINDOWS -#ifdef Py_ENABLE_SHARED extern HMODULE PyWin_DLLhModule; if (PyWin_DLLhModule) { return winmodule_to_dict(dict, key, PyWin_DLLhModule); } #endif -#elif defined(WITH_NEXT_FRAMEWORK) - static char modPath[MAXPATHLEN + 1]; - static int modPathInitialized = -1; - if (modPathInitialized < 0) { - modPathInitialized = 0; - - /* On Mac OS X we have a special case if we're running from a framework. - This is because the python home should be set relative to the library, - which is in the framework, not relative to the executable, which may - be outside of the framework. Except when we're in the build - directory... */ - Dl_info pythonInfo; - if (dladdr(&Py_Initialize, &pythonInfo)) { - if (pythonInfo.dli_fname) { - strncpy(modPath, pythonInfo.dli_fname, MAXPATHLEN); - modPathInitialized = 1; - } - } - } - if (modPathInitialized > 0) { - return decode_to_dict(dict, key, modPath); + +#if HAVE_DLADDR + Dl_info libpython_info; + if (dladdr(&Py_Initialize, &libpython_info) && libpython_info.dli_fname) { + return decode_to_dict(dict, key, libpython_info.dli_fname); } #endif +#endif + return PyDict_SetItemString(dict, key, Py_None) == 0; } @@ -951,6 +943,11 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) !wchar_to_dict(dict, "executable_dir", NULL) || !wchar_to_dict(dict, "py_setpath", _PyPathConfig_GetGlobalModuleSearchPath()) || !funcs_to_dict(dict, config->pathconfig_warnings) || +#ifdef Py_GIL_DISABLED + !decode_to_dict(dict, "ABI_THREAD", "t") || +#else + !decode_to_dict(dict, "ABI_THREAD", "") || +#endif #ifndef MS_WINDOWS PyDict_SetItemString(dict, "winreg", Py_None) < 0 || #endif diff --git a/Modules/getpath.py b/Modules/getpath.py index 1410ffdbed8c70..9d531e29becbc8 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -40,6 +40,7 @@ # EXE_SUFFIX -- [in, opt] '.exe' on Windows/Cygwin/similar # VERSION_MAJOR -- [in] sys.version_info.major # VERSION_MINOR -- [in] sys.version_info.minor +# ABI_THREAD -- [in] either 't' for free-threaded builds or '' # PYWINVER -- [in] the Windows platform-specific version (e.g. 3.8-32) # ** Values read from the environment ** @@ -172,17 +173,18 @@ # ****************************************************************************** platlibdir = config.get('platlibdir') or PLATLIBDIR +ABI_THREAD = ABI_THREAD or '' if os_name == 'posix' or os_name == 'darwin': BUILDDIR_TXT = 'pybuilddir.txt' BUILD_LANDMARK = 'Modules/Setup.local' DEFAULT_PROGRAM_NAME = f'python{VERSION_MAJOR}' - STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}' + STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}{ABI_THREAD}' STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}/os.py', f'{STDLIB_SUBDIR}/os.pyc'] - PLATSTDLIB_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}/lib-dynload' + PLATSTDLIB_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}{ABI_THREAD}/lib-dynload' BUILDSTDLIB_LANDMARKS = ['Lib/os.py'] VENV_LANDMARK = 'pyvenv.cfg' - ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}.zip' + ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}{ABI_THREAD}.zip' DELIM = ':' SEP = '/' @@ -310,7 +312,10 @@ def search_up(prefix, *landmarks, test=isfile): # and should not affect base_executable. base_executable = f"{dirname(library)}/bin/python{VERSION_MAJOR}.{VERSION_MINOR}" else: - base_executable = executable + # Use the real executable as our base, or argv[0] otherwise + # (on Windows, argv[0] is likely to be ENV___PYVENV_LAUNCHER__; on + # other platforms, real_executable is likely to be empty) + base_executable = real_executable or executable if not real_executable: real_executable = base_executable @@ -339,9 +344,10 @@ def search_up(prefix, *landmarks, test=isfile): venv_prefix = None -# Calling Py_SetPythonHome(), Py_SetPath() or -# setting $PYTHONHOME will override venv detection. -if not home and not py_setpath: +# Calling Py_SetPath() will override venv detection. +# Calling Py_SetPythonHome() or setting $PYTHONHOME will override the 'home' key +# specified in pyvenv.cfg. +if not py_setpath: try: # prefix2 is just to avoid calculating dirname again later, # as the path in venv_prefix is the more common case. @@ -358,10 +364,23 @@ def search_up(prefix, *landmarks, test=isfile): venv_prefix = None pyvenvcfg = [] + # Search for the 'home' key in pyvenv.cfg. Currently, we don't consider the + # presence of a pyvenv.cfg file without a 'home' key to signify the + # existence of a virtual environment — we quietly ignore them. + # XXX: If we don't find a 'home' key, we don't look for another pyvenv.cfg! for line in pyvenvcfg: key, had_equ, value = line.partition('=') if had_equ and key.strip().lower() == 'home': + # If PYTHONHOME was set, ignore 'home' from pyvenv.cfg. + if home: + break + # Override executable_dir/real_executable_dir with the value from 'home'. + # These values may be later used to calculate prefix/base_prefix, if a more + # reliable source — like the runtime library (libpython) path — isn't available. executable_dir = real_executable_dir = value.strip() + # If base_executable — which points to the Python interpreted from + # the base installation — isn't set (eg. when embedded), try to find + # it in 'home'. if not base_executable: # First try to resolve symlinked executables, since that may be # more accurate than assuming the executable in 'home'. @@ -395,6 +414,7 @@ def search_up(prefix, *landmarks, test=isfile): break break else: + # We didn't find a 'home' key in pyvenv.cfg (no break), reset venv_prefix. venv_prefix = None @@ -408,13 +428,14 @@ def search_up(prefix, *landmarks, test=isfile): if not real_executable: real_executable = base_executable -try: - real_executable = realpath(real_executable) -except OSError as ex: - # Only warn if the file actually exists and was unresolvable - # Otherwise users who specify a fake executable may get spurious warnings. - if isfile(real_executable): - warn(f'Failed to find real location of {base_executable}') +if real_executable: + try: + real_executable = realpath(real_executable) + except OSError as ex: + # Only warn if the file actually exists and was unresolvable + # Otherwise users who specify a fake executable may get spurious warnings. + if isfile(real_executable): + warn(f'Failed to find real location of {real_executable}') if not executable_dir and os_name == 'darwin' and library: # QUIRK: macOS checks adjacent to its library early @@ -427,12 +448,12 @@ def search_up(prefix, *landmarks, test=isfile): # If we do not have the executable's directory, we can calculate it. # This is the directory used to find prefix/exec_prefix if necessary. -if not executable_dir: +if not executable_dir and real_executable: executable_dir = real_executable_dir = dirname(real_executable) # If we do not have the real executable's directory, we calculate it. # This is the directory used to detect build layouts. -if not real_executable_dir: +if not real_executable_dir and real_executable: real_executable_dir = dirname(real_executable) # ****************************************************************************** @@ -604,6 +625,8 @@ def search_up(prefix, *landmarks, test=isfile): # gh-100320: Our PYDs are assumed to be relative to the Lib directory # (that is, prefix) rather than the executable (that is, executable_dir) exec_prefix = prefix + if not exec_prefix and prefix and isdir(joinpath(prefix, PLATSTDLIB_LANDMARK)): + exec_prefix = prefix if not exec_prefix and executable_dir: exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir) if not exec_prefix and EXEC_PREFIX: @@ -634,11 +657,20 @@ def search_up(prefix, *landmarks, test=isfile): # For a venv, update the main prefix/exec_prefix but leave the base ones unchanged -# XXX: We currently do not update prefix here, but it happens in site.py -#if venv_prefix: -# base_prefix = prefix -# base_exec_prefix = exec_prefix -# prefix = exec_prefix = venv_prefix +if venv_prefix: + if not base_prefix: + base_prefix = prefix + if not base_exec_prefix: + base_exec_prefix = exec_prefix + prefix = exec_prefix = venv_prefix + + +# After calculating prefix and exec_prefix, use their values for base_prefix and +# base_exec_prefix if they haven't been set. +if not base_prefix: + base_prefix = prefix +if not base_exec_prefix: + base_exec_prefix = exec_prefix # ****************************************************************************** @@ -673,7 +705,7 @@ def search_up(prefix, *landmarks, test=isfile): # QUIRK: POSIX uses the default prefix when in the build directory pythonpath.append(joinpath(PREFIX, ZIP_LANDMARK)) else: - pythonpath.append(joinpath(prefix, ZIP_LANDMARK)) + pythonpath.append(joinpath(base_prefix, ZIP_LANDMARK)) if os_name == 'nt' and use_environment and winreg: # QUIRK: Windows also lists paths in the registry. Paths are stored @@ -708,13 +740,13 @@ def search_up(prefix, *landmarks, test=isfile): # Then add any entries compiled into the PYTHONPATH macro. if PYTHONPATH: for p in PYTHONPATH.split(DELIM): - pythonpath.append(joinpath(prefix, p)) + pythonpath.append(joinpath(base_prefix, p)) # Then add stdlib_dir and platstdlib_dir - if not stdlib_dir and prefix: - stdlib_dir = joinpath(prefix, STDLIB_SUBDIR) - if not platstdlib_dir and exec_prefix: - platstdlib_dir = joinpath(exec_prefix, PLATSTDLIB_LANDMARK) + if not stdlib_dir and base_prefix: + stdlib_dir = joinpath(base_prefix, STDLIB_SUBDIR) + if not platstdlib_dir and base_exec_prefix: + platstdlib_dir = joinpath(base_exec_prefix, PLATSTDLIB_LANDMARK) if os_name == 'nt': # QUIRK: Windows generates paths differently @@ -744,9 +776,26 @@ def search_up(prefix, *landmarks, test=isfile): # QUIRK: Non-Windows replaces prefix/exec_prefix with defaults when running # in build directory. This happens after pythonpath calculation. +# Virtual environments using the build directory Python still keep their prefix. if os_name != 'nt' and build_prefix: - prefix = config.get('prefix') or PREFIX - exec_prefix = config.get('exec_prefix') or EXEC_PREFIX or prefix + if not venv_prefix: + prefix = config.get('prefix') or PREFIX + exec_prefix = config.get('exec_prefix') or EXEC_PREFIX or prefix + base_prefix = config.get('base_prefix') or PREFIX + base_exec_prefix = config.get('base_exec_prefix') or EXEC_PREFIX or base_prefix + + +# ****************************************************************************** +# MISC. RUNTIME WARNINGS +# ****************************************************************************** + +# When running Python from the build directory, if libpython is dynamically +# linked, the wrong library might be loaded. +if build_prefix and library and not dirname(abspath(library)).startswith(build_prefix): + msg = f'The runtime library has been loaded from outside the build directory ({library})!' + if os_name == 'posix': + msg += ' Consider setting LD_LIBRARY_PATH=. to force it to be loaded from the build directory.' + warn(msg) # ****************************************************************************** @@ -782,8 +831,8 @@ def search_up(prefix, *landmarks, test=isfile): config['base_executable'] = base_executable config['prefix'] = prefix config['exec_prefix'] = exec_prefix -config['base_prefix'] = base_prefix or prefix -config['base_exec_prefix'] = base_exec_prefix or exec_prefix +config['base_prefix'] = base_prefix +config['base_exec_prefix'] = base_exec_prefix config['platlibdir'] = platlibdir # test_embed expects empty strings, not None diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c index 9756f1c2fb15b1..29da9936b65504 100644 --- a/Modules/grpmodule.c +++ b/Modules/grpmodule.c @@ -1,7 +1,6 @@ - /* UNIX group file access module */ -// clinic/grpmodule.c.h uses internal pycore_modsupport.h API +// Argument Clinic uses the internal C API #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif @@ -9,7 +8,9 @@ #include "Python.h" #include "posixmodule.h" +#include // ERANGE #include // getgrgid_r() +#include // memcpy() #include // sysconf() #include "clinic/grpmodule.c.h" @@ -88,7 +89,7 @@ mkgrent(PyObject *module, struct group *p) Py_DECREF(x); } -#define SET(i,val) PyStructSequence_SET_ITEM(v, i, val) +#define SET(i,val) PyStructSequence_SetItem(v, i, val) SET(setIndex++, PyUnicode_DecodeFSDefault(p->gr_name)); if (p->gr_passwd) SET(setIndex++, PyUnicode_DecodeFSDefault(p->gr_passwd)); @@ -279,23 +280,33 @@ static PyObject * grp_getgrall_impl(PyObject *module) /*[clinic end generated code: output=585dad35e2e763d7 input=d7df76c825c367df]*/ { - PyObject *d; - struct group *p; - - if ((d = PyList_New(0)) == NULL) + PyObject *d = PyList_New(0); + if (d == NULL) { return NULL; + } + + static PyMutex getgrall_mutex = {0}; + PyMutex_Lock(&getgrall_mutex); setgrent(); + + struct group *p; while ((p = getgrent()) != NULL) { + // gh-126316: Don't release the mutex around mkgrent() since + // setgrent()/endgrent() are not reentrant / thread-safe. A deadlock + // is unlikely since mkgrent() should not be able to call arbitrary + // Python code. PyObject *v = mkgrent(module, p); if (v == NULL || PyList_Append(d, v) != 0) { Py_XDECREF(v); - Py_DECREF(d); - endgrent(); - return NULL; + Py_CLEAR(d); + goto done; } Py_DECREF(v); } + +done: endgrent(); + PyMutex_Unlock(&getgrall_mutex); return d; } @@ -340,6 +351,7 @@ grpmodule_exec(PyObject *module) static PyModuleDef_Slot grpmodule_slots[] = { {Py_mod_exec, grpmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index d377c2c57006aa..3f736f0cf19968 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1,13 +1,14 @@ #include "Python.h" -#include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_ceval.h" // _PyEval_GetBuiltin() -#include "pycore_long.h" // _PyLong_GetZero() -#include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_typeobject.h" // _PyType_GetModuleState() -#include "pycore_object.h" // _PyObject_GC_TRACK() -#include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() +#include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_tuple.h" // _PyTuple_ITEMS() -#include // offsetof() +#include // offsetof() /* Itertools module written and maintained by Raymond D. Hettinger @@ -93,15 +94,6 @@ class itertools.pairwise "pairwiseobject *" "clinic_state()->pairwise_type" #undef clinic_state_by_cls #undef clinic_state -/* Deprecation of pickle support: GH-101588 *********************************/ - -#define ITERTOOL_PICKLE_DEPRECATION \ - if (PyErr_WarnEx( \ - PyExc_DeprecationWarning, \ - "Pickle, copy, and deepcopy support will be " \ - "removed from itertools in Python 3.14.", 1) < 0) { \ - return NULL; \ - } /* batched object ************************************************************/ @@ -270,6 +262,7 @@ typedef struct { PyObject_HEAD PyObject *it; PyObject *old; + PyObject *result; } pairwiseobject; /*[clinic input] @@ -301,6 +294,11 @@ pairwise_new_impl(PyTypeObject *type, PyObject *iterable) } po->it = it; po->old = NULL; + po->result = PyTuple_Pack(2, Py_None, Py_None); + if (po->result == NULL) { + Py_DECREF(po); + return NULL; + } return (PyObject *)po; } @@ -311,6 +309,7 @@ pairwise_dealloc(pairwiseobject *po) PyObject_GC_UnTrack(po); Py_XDECREF(po->it); Py_XDECREF(po->old); + Py_XDECREF(po->result); tp->tp_free(po); Py_DECREF(tp); } @@ -321,6 +320,7 @@ pairwise_traverse(pairwiseobject *po, visitproc visit, void *arg) Py_VISIT(Py_TYPE(po)); Py_VISIT(po->it); Py_VISIT(po->old); + Py_VISIT(po->result); return 0; } @@ -355,8 +355,30 @@ pairwise_next(pairwiseobject *po) Py_DECREF(old); return NULL; } - /* Future optimization: Reuse the result tuple as we do in enumerate() */ - result = PyTuple_Pack(2, old, new); + + result = po->result; + if (Py_REFCNT(result) == 1) { + Py_INCREF(result); + PyObject *last_old = PyTuple_GET_ITEM(result, 0); + PyObject *last_new = PyTuple_GET_ITEM(result, 1); + PyTuple_SET_ITEM(result, 0, Py_NewRef(old)); + PyTuple_SET_ITEM(result, 1, Py_NewRef(new)); + Py_DECREF(last_old); + Py_DECREF(last_new); + // bpo-42536: The GC may have untracked this result tuple. Since we're + // recycling it, make sure it's tracked again: + if (!_PyObject_GC_IS_TRACKED(result)) { + _PyObject_GC_TRACK(result); + } + } + else { + result = PyTuple_New(2); + if (result != NULL) { + PyTuple_SET_ITEM(result, 0, Py_NewRef(old)); + PyTuple_SET_ITEM(result, 1, Py_NewRef(new)); + } + } + Py_XSETREF(po->old, new); Py_DECREF(old); return result; @@ -524,57 +546,6 @@ groupby_next(groupbyobject *gbo) return r; } -static PyObject * -groupby_reduce(groupbyobject *lz, PyObject *Py_UNUSED(ignored)) -{ - /* reduce as a 'new' call with an optional 'setstate' if groupby - * has started - */ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *value; - if (lz->tgtkey && lz->currkey && lz->currvalue) - value = Py_BuildValue("O(OO)(OOO)", Py_TYPE(lz), - lz->it, lz->keyfunc, lz->currkey, lz->currvalue, lz->tgtkey); - else - value = Py_BuildValue("O(OO)", Py_TYPE(lz), - lz->it, lz->keyfunc); - - return value; -} - -PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); - -static PyObject * -groupby_setstate(groupbyobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *currkey, *currvalue, *tgtkey; - if (!PyTuple_Check(state)) { - PyErr_SetString(PyExc_TypeError, "state is not a tuple"); - return NULL; - } - if (!PyArg_ParseTuple(state, "OOO", &currkey, &currvalue, &tgtkey)) { - return NULL; - } - Py_INCREF(currkey); - Py_XSETREF(lz->currkey, currkey); - Py_INCREF(currvalue); - Py_XSETREF(lz->currvalue, currvalue); - Py_INCREF(tgtkey); - Py_XSETREF(lz->tgtkey, tgtkey); - Py_RETURN_NONE; -} - -PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); - -static PyMethodDef groupby_methods[] = { - {"__reduce__", (PyCFunction)groupby_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)groupby_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot groupby_slots[] = { {Py_tp_dealloc, groupby_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -582,7 +553,6 @@ static PyType_Slot groupby_slots[] = { {Py_tp_traverse, groupby_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, groupby_next}, - {Py_tp_methods, groupby_methods}, {Py_tp_new, itertools_groupby}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -683,29 +653,12 @@ _grouper_next(_grouperobject *igo) return r; } -static PyObject * -_grouper_reduce(_grouperobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (((groupbyobject *)lz->parent)->currgrouper != lz) { - return Py_BuildValue("N(())", _PyEval_GetBuiltin(&_Py_ID(iter))); - } - return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->parent, lz->tgtkey); -} - -static PyMethodDef _grouper_methods[] = { - {"__reduce__", (PyCFunction)_grouper_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot _grouper_slots[] = { {Py_tp_dealloc, _grouper_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_traverse, _grouper_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, _grouper_next}, - {Py_tp_methods, _grouper_methods}, {Py_tp_new, itertools__grouper}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -815,10 +768,9 @@ teedataobject_traverse(teedataobject *tdo, visitproc visit, void * arg) } static void -teedataobject_safe_decref(PyObject *obj, PyTypeObject *tdo_type) +teedataobject_safe_decref(PyObject *obj) { - while (obj && Py_IS_TYPE(obj, tdo_type) && - Py_REFCNT(obj) == 1) { + while (obj && Py_REFCNT(obj) == 1) { PyObject *nextlink = ((teedataobject *)obj)->nextlink; ((teedataobject *)obj)->nextlink = NULL; Py_SETREF(obj, nextlink); @@ -837,8 +789,7 @@ teedataobject_clear(teedataobject *tdo) Py_CLEAR(tdo->values[i]); tmp = tdo->nextlink; tdo->nextlink = NULL; - itertools_state *state = get_module_state_by_cls(Py_TYPE(tdo)); - teedataobject_safe_decref(tmp, state->teedataobject_type); + teedataobject_safe_decref(tmp); return 0; } @@ -852,25 +803,6 @@ teedataobject_dealloc(teedataobject *tdo) Py_DECREF(tp); } -static PyObject * -teedataobject_reduce(teedataobject *tdo, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - int i; - /* create a temporary list of already iterated values */ - PyObject *values = PyList_New(tdo->numread); - - if (!values) - return NULL; - for (i=0 ; inumread ; i++) { - Py_INCREF(tdo->values[i]); - PyList_SET_ITEM(values, i, tdo->values[i]); - } - return Py_BuildValue("O(ONO)", Py_TYPE(tdo), tdo->it, - values, - tdo->nextlink ? tdo->nextlink : Py_None); -} - /*[clinic input] @classmethod itertools.teedataobject.__new__ @@ -925,19 +857,12 @@ itertools_teedataobject_impl(PyTypeObject *type, PyObject *it, return NULL; } -static PyMethodDef teedataobject_methods[] = { - {"__reduce__", (PyCFunction)teedataobject_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot teedataobject_slots[] = { {Py_tp_dealloc, teedataobject_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_doc, (void *)itertools_teedataobject__doc__}, {Py_tp_traverse, teedataobject_traverse}, {Py_tp_clear, teedataobject_clear}, - {Py_tp_methods, teedataobject_methods}, {Py_tp_new, itertools_teedataobject}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -1066,41 +991,8 @@ tee_dealloc(teeobject *to) Py_DECREF(tp); } -static PyObject * -tee_reduce(teeobject *to, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - return Py_BuildValue("O(())(Oi)", Py_TYPE(to), to->dataobj, to->index); -} - -static PyObject * -tee_setstate(teeobject *to, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - teedataobject *tdo; - int index; - if (!PyTuple_Check(state)) { - PyErr_SetString(PyExc_TypeError, "state is not a tuple"); - return NULL; - } - PyTypeObject *tdo_type = to->state->teedataobject_type; - if (!PyArg_ParseTuple(state, "O!i", tdo_type, &tdo, &index)) { - return NULL; - } - if (index < 0 || index > LINKCELLS) { - PyErr_SetString(PyExc_ValueError, "Index out of range"); - return NULL; - } - Py_INCREF(tdo); - Py_XSETREF(to->dataobj, tdo); - to->index = index; - Py_RETURN_NONE; -} - static PyMethodDef tee_methods[] = { {"__copy__", (PyCFunction)tee_copy, METH_NOARGS, teecopy_doc}, - {"__reduce__", (PyCFunction)tee_reduce, METH_NOARGS, reduce_doc}, - {"__setstate__", (PyCFunction)tee_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -1144,7 +1036,7 @@ itertools_tee_impl(PyObject *module, PyObject *iterable, Py_ssize_t n) /*[clinic end generated code: output=1c64519cd859c2f0 input=c99a1472c425d66d]*/ { Py_ssize_t i; - PyObject *it, *copyable, *copyfunc, *result; + PyObject *it, *to, *result; if (n < 0) { PyErr_SetString(PyExc_ValueError, "n must be >= 0"); @@ -1161,41 +1053,23 @@ itertools_tee_impl(PyObject *module, PyObject *iterable, Py_ssize_t n) return NULL; } - if (PyObject_GetOptionalAttr(it, &_Py_ID(__copy__), ©func) < 0) { - Py_DECREF(it); + itertools_state *state = get_module_state(module); + to = tee_fromiterable(state, it); + Py_DECREF(it); + if (to == NULL) { Py_DECREF(result); return NULL; } - if (copyfunc != NULL) { - copyable = it; - } - else { - itertools_state *state = get_module_state(module); - copyable = tee_fromiterable(state, it); - Py_DECREF(it); - if (copyable == NULL) { - Py_DECREF(result); - return NULL; - } - copyfunc = PyObject_GetAttr(copyable, &_Py_ID(__copy__)); - if (copyfunc == NULL) { - Py_DECREF(copyable); - Py_DECREF(result); - return NULL; - } - } - PyTuple_SET_ITEM(result, 0, copyable); + PyTuple_SET_ITEM(result, 0, to); for (i = 1; i < n; i++) { - copyable = _PyObject_CallNoArgs(copyfunc); - if (copyable == NULL) { - Py_DECREF(copyfunc); + to = tee_copy((teeobject *)to, NULL); + if (to == NULL) { Py_DECREF(result); return NULL; } - PyTuple_SET_ITEM(result, i, copyable); + PyTuple_SET_ITEM(result, i, to); } - Py_DECREF(copyfunc); return result; } @@ -1302,59 +1176,6 @@ cycle_next(cycleobject *lz) return Py_NewRef(item); } -static PyObject * -cycle_reduce(cycleobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - /* Create a new cycle with the iterator tuple, then set the saved state */ - if (lz->it == NULL) { - PyObject *it = PyObject_GetIter(lz->saved); - if (it == NULL) - return NULL; - if (lz->index != 0) { - PyObject *res = _PyObject_CallMethod(it, &_Py_ID(__setstate__), - "n", lz->index); - if (res == NULL) { - Py_DECREF(it); - return NULL; - } - Py_DECREF(res); - } - return Py_BuildValue("O(N)(OO)", Py_TYPE(lz), it, lz->saved, Py_True); - } - return Py_BuildValue("O(O)(OO)", Py_TYPE(lz), lz->it, lz->saved, - lz->firstpass ? Py_True : Py_False); -} - -static PyObject * -cycle_setstate(cycleobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *saved=NULL; - int firstpass; - if (!PyTuple_Check(state)) { - PyErr_SetString(PyExc_TypeError, "state is not a tuple"); - return NULL; - } - // The second item can be 1/0 in old pickles and True/False in new pickles - if (!PyArg_ParseTuple(state, "O!i", &PyList_Type, &saved, &firstpass)) { - return NULL; - } - Py_INCREF(saved); - Py_XSETREF(lz->saved, saved); - lz->firstpass = firstpass != 0; - lz->index = 0; - Py_RETURN_NONE; -} - -static PyMethodDef cycle_methods[] = { - {"__reduce__", (PyCFunction)cycle_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)cycle_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot cycle_slots[] = { {Py_tp_dealloc, cycle_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -1362,7 +1183,6 @@ static PyType_Slot cycle_slots[] = { {Py_tp_traverse, cycle_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, cycle_next}, - {Py_tp_methods, cycle_methods}, {Py_tp_new, itertools_cycle}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -1475,32 +1295,6 @@ dropwhile_next(dropwhileobject *lz) } } -static PyObject * -dropwhile_reduce(dropwhileobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - return Py_BuildValue("O(OO)l", Py_TYPE(lz), lz->func, lz->it, lz->start); -} - -static PyObject * -dropwhile_setstate(dropwhileobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - int start = PyObject_IsTrue(state); - if (start < 0) - return NULL; - lz->start = start; - Py_RETURN_NONE; -} - -static PyMethodDef dropwhile_methods[] = { - {"__reduce__", (PyCFunction)dropwhile_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)dropwhile_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot dropwhile_slots[] = { {Py_tp_dealloc, dropwhile_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -1508,7 +1302,6 @@ static PyType_Slot dropwhile_slots[] = { {Py_tp_traverse, dropwhile_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, dropwhile_next}, - {Py_tp_methods, dropwhile_methods}, {Py_tp_new, itertools_dropwhile}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -1615,33 +1408,6 @@ takewhile_next(takewhileobject *lz) return NULL; } -static PyObject * -takewhile_reduce(takewhileobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - return Py_BuildValue("O(OO)l", Py_TYPE(lz), lz->func, lz->it, lz->stop); -} - -static PyObject * -takewhile_reduce_setstate(takewhileobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - int stop = PyObject_IsTrue(state); - - if (stop < 0) - return NULL; - lz->stop = stop; - Py_RETURN_NONE; -} - -static PyMethodDef takewhile_reduce_methods[] = { - {"__reduce__", (PyCFunction)takewhile_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)takewhile_reduce_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot takewhile_slots[] = { {Py_tp_dealloc, takewhile_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -1649,7 +1415,6 @@ static PyType_Slot takewhile_slots[] = { {Py_tp_traverse, takewhile_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, takewhile_next}, - {Py_tp_methods, takewhile_reduce_methods}, {Py_tp_new, itertools_takewhile}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -1819,59 +1584,6 @@ islice_next(isliceobject *lz) return NULL; } -static PyObject * -islice_reduce(isliceobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - /* When unpickled, generate a new object with the same bounds, - * then 'setstate' with the next and count - */ - PyObject *stop; - - if (lz->it == NULL) { - PyObject *empty_list; - PyObject *empty_it; - empty_list = PyList_New(0); - if (empty_list == NULL) - return NULL; - empty_it = PyObject_GetIter(empty_list); - Py_DECREF(empty_list); - if (empty_it == NULL) - return NULL; - return Py_BuildValue("O(Nn)n", Py_TYPE(lz), empty_it, 0, 0); - } - if (lz->stop == -1) { - stop = Py_NewRef(Py_None); - } else { - stop = PyLong_FromSsize_t(lz->stop); - if (stop == NULL) - return NULL; - } - return Py_BuildValue("O(OnNn)n", Py_TYPE(lz), - lz->it, lz->next, stop, lz->step, - lz->cnt); -} - -static PyObject * -islice_setstate(isliceobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - Py_ssize_t cnt = PyLong_AsSsize_t(state); - - if (cnt == -1 && PyErr_Occurred()) - return NULL; - lz->cnt = cnt; - Py_RETURN_NONE; -} - -static PyMethodDef islice_methods[] = { - {"__reduce__", (PyCFunction)islice_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)islice_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - PyDoc_STRVAR(islice_doc, "islice(iterable, stop) --> islice object\n\ islice(iterable, start, stop[, step]) --> islice object\n\ @@ -1890,7 +1602,6 @@ static PyType_Slot islice_slots[] = { {Py_tp_traverse, islice_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, islice_next}, - {Py_tp_methods, islice_methods}, {Py_tp_new, islice_new}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -1988,20 +1699,6 @@ starmap_next(starmapobject *lz) return result; } -static PyObject * -starmap_reduce(starmapobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - /* Just pickle the iterator */ - return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->func, lz->it); -} - -static PyMethodDef starmap_methods[] = { - {"__reduce__", (PyCFunction)starmap_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot starmap_slots[] = { {Py_tp_dealloc, starmap_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -2009,7 +1706,6 @@ static PyType_Slot starmap_slots[] = { {Py_tp_traverse, starmap_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, starmap_next}, - {Py_tp_methods, starmap_methods}, {Py_tp_new, itertools_starmap}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -2145,53 +1841,9 @@ chain_next(chainobject *lz) return NULL; } -static PyObject * -chain_reduce(chainobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (lz->source) { - /* we can't pickle function objects (itertools.from_iterable) so - * we must use setstate to replace the iterable. One day we - * will fix pickling of functions - */ - if (lz->active) { - return Py_BuildValue("O()(OO)", Py_TYPE(lz), lz->source, lz->active); - } else { - return Py_BuildValue("O()(O)", Py_TYPE(lz), lz->source); - } - } else { - return Py_BuildValue("O()", Py_TYPE(lz)); /* exhausted */ - } - return NULL; -} - -static PyObject * -chain_setstate(chainobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *source, *active=NULL; - - if (!PyTuple_Check(state)) { - PyErr_SetString(PyExc_TypeError, "state is not a tuple"); - return NULL; - } - if (!PyArg_ParseTuple(state, "O|O", &source, &active)) { - return NULL; - } - if (!PyIter_Check(source) || (active != NULL && !PyIter_Check(active))) { - PyErr_SetString(PyExc_TypeError, "Arguments must be iterators."); - return NULL; - } - - Py_INCREF(source); - Py_XSETREF(lz->source, source); - Py_XINCREF(active); - Py_XSETREF(lz->active, active); - Py_RETURN_NONE; -} - PyDoc_STRVAR(chain_doc, -"chain(*iterables) --> chain object\n\ +"chain(*iterables)\n\ +--\n\ \n\ Return a chain object whose .__next__() method returns elements from the\n\ first iterable until it is exhausted, then elements from the next\n\ @@ -2199,10 +1851,6 @@ iterable, until all of the iterables are exhausted."); static PyMethodDef chain_methods[] = { ITERTOOLS_CHAIN_FROM_ITERABLE_METHODDEF - {"__reduce__", (PyCFunction)chain_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)chain_setstate, METH_O, - setstate_doc}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ @@ -2441,96 +2089,15 @@ product_next(productobject *lz) return NULL; } -static PyObject * -product_reduce(productobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (lz->stopped) { - return Py_BuildValue("O(())", Py_TYPE(lz)); - } else if (lz->result == NULL) { - return Py_BuildValue("OO", Py_TYPE(lz), lz->pools); - } else { - PyObject *indices; - Py_ssize_t n, i; - - /* we must pickle the indices use them for setstate, and - * additionally indicate that the iterator has started - */ - n = PyTuple_GET_SIZE(lz->pools); - indices = PyTuple_New(n); - if (indices == NULL) - return NULL; - for (i=0; iindices[i]); - if (!index) { - Py_DECREF(indices); - return NULL; - } - PyTuple_SET_ITEM(indices, i, index); - } - return Py_BuildValue("OON", Py_TYPE(lz), lz->pools, indices); - } -} - -static PyObject * -product_setstate(productobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *result; - Py_ssize_t n, i; - - n = PyTuple_GET_SIZE(lz->pools); - if (!PyTuple_Check(state) || PyTuple_GET_SIZE(state) != n) { - PyErr_SetString(PyExc_ValueError, "invalid arguments"); - return NULL; - } - for (i=0; ipools, i); - poolsize = PyTuple_GET_SIZE(pool); - if (poolsize == 0) { - lz->stopped = 1; - Py_RETURN_NONE; - } - /* clamp the index */ - if (index < 0) - index = 0; - else if (index > poolsize-1) - index = poolsize-1; - lz->indices[i] = index; - } - - result = PyTuple_New(n); - if (!result) - return NULL; - for (i=0; ipools, i); - PyObject *element = PyTuple_GET_ITEM(pool, lz->indices[i]); - Py_INCREF(element); - PyTuple_SET_ITEM(result, i, element); - } - Py_XSETREF(lz->result, result); - Py_RETURN_NONE; -} - static PyMethodDef product_methods[] = { - {"__reduce__", (PyCFunction)product_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)product_setstate, METH_O, - setstate_doc}, {"__sizeof__", (PyCFunction)product_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ }; PyDoc_STRVAR(product_doc, -"product(*iterables, repeat=1) --> product object\n\ +"product(*iterables, repeat=1)\n\ +--\n\ \n\ Cartesian product of input iterables. Equivalent to nested for-loops.\n\n\ For example, product(A, B) returns the same as: ((x,y) for x in A for y in B).\n\ @@ -2751,83 +2318,7 @@ combinations_next(combinationsobject *co) return NULL; } -static PyObject * -combinations_reduce(combinationsobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (lz->result == NULL) { - return Py_BuildValue("O(On)", Py_TYPE(lz), lz->pool, lz->r); - } else if (lz->stopped) { - return Py_BuildValue("O(()n)", Py_TYPE(lz), lz->r); - } else { - PyObject *indices; - Py_ssize_t i; - - /* we must pickle the indices and use them for setstate */ - indices = PyTuple_New(lz->r); - if (!indices) - return NULL; - for (i=0; ir; i++) - { - PyObject* index = PyLong_FromSsize_t(lz->indices[i]); - if (!index) { - Py_DECREF(indices); - return NULL; - } - PyTuple_SET_ITEM(indices, i, index); - } - - return Py_BuildValue("O(On)N", Py_TYPE(lz), lz->pool, lz->r, indices); - } -} - -static PyObject * -combinations_setstate(combinationsobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *result; - Py_ssize_t i; - Py_ssize_t n = PyTuple_GET_SIZE(lz->pool); - - if (!PyTuple_Check(state) || PyTuple_GET_SIZE(state) != lz->r) { - PyErr_SetString(PyExc_ValueError, "invalid arguments"); - return NULL; - } - - for (i=0; ir; i++) { - Py_ssize_t max; - PyObject* indexObject = PyTuple_GET_ITEM(state, i); - Py_ssize_t index = PyLong_AsSsize_t(indexObject); - - if (index == -1 && PyErr_Occurred()) - return NULL; /* not an integer */ - max = i + n - lz->r; - /* clamp the index (beware of negative max) */ - if (index > max) - index = max; - if (index < 0) - index = 0; - lz->indices[i] = index; - } - - result = PyTuple_New(lz->r); - if (result == NULL) - return NULL; - for (i=0; ir; i++) { - PyObject *element = PyTuple_GET_ITEM(lz->pool, lz->indices[i]); - Py_INCREF(element); - PyTuple_SET_ITEM(result, i, element); - } - - Py_XSETREF(lz->result, result); - Py_RETURN_NONE; -} - static PyMethodDef combinations_methods[] = { - {"__reduce__", (PyCFunction)combinations_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)combinations_setstate, METH_O, - setstate_doc}, {"__sizeof__", (PyCFunction)combinations_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ @@ -3061,79 +2552,7 @@ cwr_next(cwrobject *co) return NULL; } -static PyObject * -cwr_reduce(cwrobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (lz->result == NULL) { - return Py_BuildValue("O(On)", Py_TYPE(lz), lz->pool, lz->r); - } else if (lz->stopped) { - return Py_BuildValue("O(()n)", Py_TYPE(lz), lz->r); - } else { - PyObject *indices; - Py_ssize_t i; - - /* we must pickle the indices and use them for setstate */ - indices = PyTuple_New(lz->r); - if (!indices) - return NULL; - for (i=0; ir; i++) { - PyObject* index = PyLong_FromSsize_t(lz->indices[i]); - if (!index) { - Py_DECREF(indices); - return NULL; - } - PyTuple_SET_ITEM(indices, i, index); - } - - return Py_BuildValue("O(On)N", Py_TYPE(lz), lz->pool, lz->r, indices); - } -} - -static PyObject * -cwr_setstate(cwrobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *result; - Py_ssize_t n, i; - - if (!PyTuple_Check(state) || PyTuple_GET_SIZE(state) != lz->r) - { - PyErr_SetString(PyExc_ValueError, "invalid arguments"); - return NULL; - } - - n = PyTuple_GET_SIZE(lz->pool); - for (i=0; ir; i++) { - PyObject* indexObject = PyTuple_GET_ITEM(state, i); - Py_ssize_t index = PyLong_AsSsize_t(indexObject); - - if (index < 0 && PyErr_Occurred()) - return NULL; /* not an integer */ - /* clamp the index */ - if (index < 0) - index = 0; - else if (index > n-1) - index = n-1; - lz->indices[i] = index; - } - result = PyTuple_New(lz->r); - if (result == NULL) - return NULL; - for (i=0; ir; i++) { - PyObject *element = PyTuple_GET_ITEM(lz->pool, lz->indices[i]); - Py_INCREF(element); - PyTuple_SET_ITEM(result, i, element); - } - Py_XSETREF(lz->result, result); - Py_RETURN_NONE; -} - static PyMethodDef cwr_methods[] = { - {"__reduce__", (PyCFunction)cwr_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)cwr_setstate, METH_O, - setstate_doc}, {"__sizeof__", (PyCFunction)cwr_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ @@ -3398,113 +2817,7 @@ permutations_next(permutationsobject *po) return NULL; } -static PyObject * -permutations_reduce(permutationsobject *po, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (po->result == NULL) { - return Py_BuildValue("O(On)", Py_TYPE(po), po->pool, po->r); - } else if (po->stopped) { - return Py_BuildValue("O(()n)", Py_TYPE(po), po->r); - } else { - PyObject *indices=NULL, *cycles=NULL; - Py_ssize_t n, i; - - /* we must pickle the indices and cycles and use them for setstate */ - n = PyTuple_GET_SIZE(po->pool); - indices = PyTuple_New(n); - if (indices == NULL) - goto err; - for (i=0; iindices[i]); - if (!index) - goto err; - PyTuple_SET_ITEM(indices, i, index); - } - - cycles = PyTuple_New(po->r); - if (cycles == NULL) - goto err; - for (i=0 ; ir ; i++) { - PyObject* index = PyLong_FromSsize_t(po->cycles[i]); - if (!index) - goto err; - PyTuple_SET_ITEM(cycles, i, index); - } - return Py_BuildValue("O(On)(NN)", Py_TYPE(po), - po->pool, po->r, - indices, cycles); - err: - Py_XDECREF(indices); - Py_XDECREF(cycles); - return NULL; - } -} - -static PyObject * -permutations_setstate(permutationsobject *po, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - PyObject *indices, *cycles, *result; - Py_ssize_t n, i; - - if (!PyTuple_Check(state)) { - PyErr_SetString(PyExc_TypeError, "state is not a tuple"); - return NULL; - } - if (!PyArg_ParseTuple(state, "O!O!", - &PyTuple_Type, &indices, - &PyTuple_Type, &cycles)) { - return NULL; - } - - n = PyTuple_GET_SIZE(po->pool); - if (PyTuple_GET_SIZE(indices) != n || PyTuple_GET_SIZE(cycles) != po->r) { - PyErr_SetString(PyExc_ValueError, "invalid arguments"); - return NULL; - } - - for (i=0; i n-1) - index = n-1; - po->indices[i] = index; - } - - for (i=0; ir; i++) { - PyObject* indexObject = PyTuple_GET_ITEM(cycles, i); - Py_ssize_t index = PyLong_AsSsize_t(indexObject); - if (index < 0 && PyErr_Occurred()) - return NULL; /* not an integer */ - if (index < 1) - index = 1; - else if (index > n-i) - index = n-i; - po->cycles[i] = index; - } - result = PyTuple_New(po->r); - if (result == NULL) - return NULL; - for (i=0; ir; i++) { - PyObject *element = PyTuple_GET_ITEM(po->pool, po->indices[i]); - Py_INCREF(element); - PyTuple_SET_ITEM(result, i, element); - } - Py_XSETREF(po->result, result); - Py_RETURN_NONE; -} - static PyMethodDef permuations_methods[] = { - {"__reduce__", (PyCFunction)permutations_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)permutations_setstate, METH_O, - setstate_doc}, {"__sizeof__", (PyCFunction)permutations_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ @@ -3639,59 +2952,6 @@ accumulate_next(accumulateobject *lz) return newtotal; } -static PyObject * -accumulate_reduce(accumulateobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - itertools_state *state = lz->state; - - if (lz->initial != Py_None) { - PyObject *it; - - assert(lz->total == NULL); - it = PyObject_CallFunction((PyObject *)(state->chain_type), "(O)O", - lz->initial, lz->it); - if (it == NULL) - return NULL; - return Py_BuildValue("O(NO)O", Py_TYPE(lz), - it, lz->binop?lz->binop:Py_None, Py_None); - } - if (lz->total == Py_None) { - PyObject *it; - - it = PyObject_CallFunction((PyObject *)(state->chain_type), "(O)O", - lz->total, lz->it); - if (it == NULL) - return NULL; - it = PyObject_CallFunction((PyObject *)Py_TYPE(lz), "NO", - it, lz->binop ? lz->binop : Py_None); - if (it == NULL) - return NULL; - - return Py_BuildValue("O(NiO)", state->islice_type, it, 1, Py_None); - } - return Py_BuildValue("O(OO)O", Py_TYPE(lz), - lz->it, lz->binop?lz->binop:Py_None, - lz->total?lz->total:Py_None); -} - -static PyObject * -accumulate_setstate(accumulateobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - Py_INCREF(state); - Py_XSETREF(lz->total, state); - Py_RETURN_NONE; -} - -static PyMethodDef accumulate_methods[] = { - {"__reduce__", (PyCFunction)accumulate_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)accumulate_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot accumulate_slots[] = { {Py_tp_dealloc, accumulate_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -3699,7 +2959,6 @@ static PyType_Slot accumulate_slots[] = { {Py_tp_traverse, accumulate_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, accumulate_next}, - {Py_tp_methods, accumulate_methods}, {Py_tp_new, itertools_accumulate}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -3824,20 +3083,6 @@ compress_next(compressobject *lz) } } -static PyObject * -compress_reduce(compressobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - return Py_BuildValue("O(OO)", Py_TYPE(lz), - lz->data, lz->selectors); -} - -static PyMethodDef compress_methods[] = { - {"__reduce__", (PyCFunction)compress_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot compress_slots[] = { {Py_tp_dealloc, compress_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -3845,7 +3090,6 @@ static PyType_Slot compress_slots[] = { {Py_tp_traverse, compress_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, compress_next}, - {Py_tp_methods, compress_methods}, {Py_tp_new, itertools_compress}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -3957,19 +3201,6 @@ filterfalse_next(filterfalseobject *lz) } } -static PyObject * -filterfalse_reduce(filterfalseobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->func, lz->it); -} - -static PyMethodDef filterfalse_methods[] = { - {"__reduce__", (PyCFunction)filterfalse_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot filterfalse_slots[] = { {Py_tp_dealloc, filterfalse_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -3977,7 +3208,6 @@ static PyType_Slot filterfalse_slots[] = { {Py_tp_traverse, filterfalse_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, filterfalse_next}, - {Py_tp_methods, filterfalse_methods}, {Py_tp_new, itertools_filterfalse}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -4005,9 +3235,9 @@ typedef struct { fast_mode: when cnt an integer < PY_SSIZE_T_MAX and no step is specified. - assert(cnt != PY_SSIZE_T_MAX && long_cnt == NULL && long_step==PyLong(1)); + assert(long_cnt == NULL && long_step==PyLong(1)); Advances with: cnt += 1 - When count hits Y_SSIZE_T_MAX, switch to slow_mode. + When count hits PY_SSIZE_T_MAX, switch to slow_mode. slow_mode: when cnt == PY_SSIZE_T_MAX, step is not int(1), or cnt is a float. @@ -4092,7 +3322,7 @@ itertools_count_impl(PyTypeObject *type, PyObject *long_cnt, else cnt = PY_SSIZE_T_MAX; - assert((cnt != PY_SSIZE_T_MAX && long_cnt == NULL && fast_mode) || + assert((long_cnt == NULL && fast_mode) || (cnt == PY_SSIZE_T_MAX && long_cnt != NULL && !fast_mode)); assert(!fast_mode || (PyLong_Check(long_step) && PyLong_AS_LONG(long_step) == 1)); @@ -4156,15 +3386,36 @@ count_nextlong(countobject *lz) static PyObject * count_next(countobject *lz) { +#ifndef Py_GIL_DISABLED if (lz->cnt == PY_SSIZE_T_MAX) return count_nextlong(lz); return PyLong_FromSsize_t(lz->cnt++); +#else + // free-threading version + // fast mode uses compare-exchange loop + // slow mode uses a critical section + PyObject *returned; + Py_ssize_t cnt; + + cnt = _Py_atomic_load_ssize_relaxed(&lz->cnt); + for (;;) { + if (cnt == PY_SSIZE_T_MAX) { + Py_BEGIN_CRITICAL_SECTION(lz); + returned = count_nextlong(lz); + Py_END_CRITICAL_SECTION(); + return returned; + } + if (_Py_atomic_compare_exchange_ssize(&lz->cnt, &cnt, cnt + 1)) { + return PyLong_FromSsize_t(cnt); + } + } +#endif } static PyObject * count_repr(countobject *lz) { - if (lz->cnt != PY_SSIZE_T_MAX) + if (lz->long_cnt == NULL) return PyUnicode_FromFormat("%s(%zd)", _PyType_Name(Py_TYPE(lz)), lz->cnt); @@ -4185,21 +3436,6 @@ count_repr(countobject *lz) lz->long_cnt, lz->long_step); } -static PyObject * -count_reduce(countobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - if (lz->cnt == PY_SSIZE_T_MAX) - return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->long_cnt, lz->long_step); - return Py_BuildValue("O(n)", Py_TYPE(lz), lz->cnt); -} - -static PyMethodDef count_methods[] = { - {"__reduce__", (PyCFunction)count_reduce, METH_NOARGS, - reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - static PyType_Slot count_slots[] = { {Py_tp_dealloc, count_dealloc}, {Py_tp_repr, count_repr}, @@ -4208,7 +3444,6 @@ static PyType_Slot count_slots[] = { {Py_tp_traverse, count_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, count_next}, - {Py_tp_methods, count_methods}, {Py_tp_new, itertools_count}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -4309,22 +3544,8 @@ repeat_len(repeatobject *ro, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); -static PyObject * -repeat_reduce(repeatobject *ro, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - /* unpickle this so that a new repeat iterator is constructed with an - * object, then call __setstate__ on it to set cnt - */ - if (ro->cnt >= 0) - return Py_BuildValue("O(On)", Py_TYPE(ro), ro->element, ro->cnt); - else - return Py_BuildValue("O(O)", Py_TYPE(ro), ro->element); -} - static PyMethodDef repeat_methods[] = { {"__length_hint__", (PyCFunction)repeat_len, METH_NOARGS, length_hint_doc}, - {"__reduce__", (PyCFunction)repeat_reduce, METH_NOARGS, reduce_doc}, {NULL, NULL} /* sentinel */ }; @@ -4530,52 +3751,9 @@ zip_longest_next(ziplongestobject *lz) return result; } -static PyObject * -zip_longest_reduce(ziplongestobject *lz, PyObject *Py_UNUSED(ignored)) -{ - ITERTOOL_PICKLE_DEPRECATION; - /* Create a new tuple with empty sequences where appropriate to pickle. - * Then use setstate to set the fillvalue - */ - int i; - PyObject *args = PyTuple_New(PyTuple_GET_SIZE(lz->ittuple)); - - if (args == NULL) - return NULL; - for (i=0; iittuple); i++) { - PyObject *elem = PyTuple_GET_ITEM(lz->ittuple, i); - if (elem == NULL) { - elem = PyTuple_New(0); - if (elem == NULL) { - Py_DECREF(args); - return NULL; - } - } else - Py_INCREF(elem); - PyTuple_SET_ITEM(args, i, elem); - } - return Py_BuildValue("ONO", Py_TYPE(lz), args, lz->fillvalue); -} - -static PyObject * -zip_longest_setstate(ziplongestobject *lz, PyObject *state) -{ - ITERTOOL_PICKLE_DEPRECATION; - Py_INCREF(state); - Py_XSETREF(lz->fillvalue, state); - Py_RETURN_NONE; -} - -static PyMethodDef zip_longest_methods[] = { - {"__reduce__", (PyCFunction)zip_longest_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)zip_longest_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - PyDoc_STRVAR(zip_longest_doc, -"zip_longest(iter1 [,iter2 [...]], [fillvalue=None]) --> zip_longest object\n\ +"zip_longest(*iterables, fillvalue=None)\n\ +--\n\ \n\ Return a zip_longest object whose .__next__() method returns a tuple where\n\ the i-th element comes from the i-th iterable argument. The .__next__()\n\ @@ -4592,7 +3770,6 @@ static PyType_Slot ziplongest_slots[] = { {Py_tp_traverse, zip_longest_traverse}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, zip_longest_next}, - {Py_tp_methods, zip_longest_methods}, {Py_tp_new, zip_longest_new}, {Py_tp_free, PyObject_GC_Del}, {0, NULL}, @@ -4750,6 +3927,7 @@ itertoolsmodule_exec(PyObject *mod) static struct PyModuleDef_Slot itertoolsmodule_slots[] = { {Py_mod_exec, itertoolsmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/main.c b/Modules/main.c index df2ce550245088..5bb1de2d04d30c 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -4,6 +4,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_initconfig.h" // _PyArgv #include "pycore_interp.h" // _PyInterpreterState.sysdict +#include "pycore_long.h" // _PyLong_GetOne() #include "pycore_pathconfig.h" // _PyPathConfig_ComputeSysPath0() #include "pycore_pylifecycle.h" // _Py_PreInitializeFromPyArgv() #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -259,6 +260,57 @@ pymain_run_command(wchar_t *command) } +static int +pymain_start_pyrepl_no_main(void) +{ + int res = 0; + PyObject *console = NULL; + PyObject *empty_tuple = NULL; + PyObject *kwargs = NULL; + PyObject *console_result = NULL; + + PyObject *pyrepl = PyImport_ImportModule("_pyrepl.main"); + if (pyrepl == NULL) { + fprintf(stderr, "Could not import _pyrepl.main\n"); + res = pymain_exit_err_print(); + goto done; + } + console = PyObject_GetAttrString(pyrepl, "interactive_console"); + if (console == NULL) { + fprintf(stderr, "Could not access _pyrepl.main.interactive_console\n"); + res = pymain_exit_err_print(); + goto done; + } + empty_tuple = PyTuple_New(0); + if (empty_tuple == NULL) { + res = pymain_exit_err_print(); + goto done; + } + kwargs = PyDict_New(); + if (kwargs == NULL) { + res = pymain_exit_err_print(); + goto done; + } + if (!PyDict_SetItemString(kwargs, "pythonstartup", _PyLong_GetOne())) { + _PyRuntime.signals.unhandled_keyboard_interrupt = 0; + console_result = PyObject_Call(console, empty_tuple, kwargs); + if (!console_result && PyErr_Occurred() == PyExc_KeyboardInterrupt) { + _PyRuntime.signals.unhandled_keyboard_interrupt = 1; + } + if (console_result == NULL) { + res = pymain_exit_err_print(); + } + } +done: + Py_XDECREF(console_result); + Py_XDECREF(kwargs); + Py_XDECREF(empty_tuple); + Py_XDECREF(console); + Py_XDECREF(pyrepl); + return res; +} + + static int pymain_run_module(const wchar_t *modname, int set_argv0) { @@ -318,10 +370,11 @@ pymain_run_file_obj(PyObject *program_name, PyObject *filename, return pymain_exit_err_print(); } - FILE *fp = _Py_fopen_obj(filename, "rb"); + FILE *fp = Py_fopen(filename, "rb"); if (fp == NULL) { // Ignore the OSError PyErr_Clear(); + // TODO(picnixz): strerror() is locale dependent but not PySys_FormatStderr(). PySys_FormatStderr("%S: can't open file %R: [Errno %d] %s\n", program_name, filename, errno, strerror(errno)); return 2; @@ -412,7 +465,7 @@ pymain_run_startup(PyConfig *config, int *exitcode) goto error; } - FILE *fp = _Py_fopen_obj(startup, "r"); + FILE *fp = Py_fopen(startup, "r"); if (fp == NULL) { int save_errno = errno; PyErr_Clear(); @@ -513,8 +566,13 @@ pymain_run_stdin(PyConfig *config) return pymain_exit_err_print(); } - PyCompilerFlags cf = _PyCompilerFlags_INIT; - int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); + if (!isatty(fileno(stdin)) + || _Py_GetEnv(config->use_environment, "PYTHON_BASIC_REPL")) { + PyCompilerFlags cf = _PyCompilerFlags_INIT; + int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); + return (run != 0); + } + int run = pymain_run_module(L"_pyrepl", 0); return (run != 0); } @@ -537,9 +595,20 @@ pymain_repl(PyConfig *config, int *exitcode) return; } - PyCompilerFlags cf = _PyCompilerFlags_INIT; - int res = PyRun_AnyFileFlags(stdin, "", &cf); - *exitcode = (res != 0); + if (PySys_Audit("cpython.run_stdin", NULL) < 0) { + return; + } + + if (!isatty(fileno(stdin)) + || _Py_GetEnv(config->use_environment, "PYTHON_BASIC_REPL")) { + PyCompilerFlags cf = _PyCompilerFlags_INIT; + int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); + *exitcode = (run != 0); + return; + } + int run = pymain_start_pyrepl_no_main(); + *exitcode = (run != 0); + return; } diff --git a/Modules/makesetup b/Modules/makesetup index d41b6640bb5186..8bb971b152a522 100755 --- a/Modules/makesetup +++ b/Modules/makesetup @@ -274,7 +274,7 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | ;; esac rule="$file: $objs" - rule="$rule; \$(BLDSHARED) $objs $libs \$(MODULE_LDFLAGS) -o $file" + rule="$rule; \$(BLDSHARED) $objs $libs \$(LIBPYTHON) -o $file" echo "$rule" >>$rulesf done done diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index a877bfcd6afb68..b4c15a143f9838 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -107,7 +107,7 @@ typedef struct{ double hi; double lo; } DoubleLength; static DoubleLength dl_fast_sum(double a, double b) { - /* Algorithm 1.1. Compensated summation of two floating point numbers. */ + /* Algorithm 1.1. Compensated summation of two floating-point numbers. */ assert(fabs(a) >= fabs(b)); double x = a + b; double y = (a - x) + b; @@ -237,7 +237,7 @@ m_sinpi(double x) double y, r; int n; /* this function should only ever be called for finite arguments */ - assert(Py_IS_FINITE(x)); + assert(isfinite(x)); y = fmod(fabs(x), 2.0); n = (int)round(2.0*y); assert(0 <= n && n <= 4); @@ -396,8 +396,8 @@ m_tgamma(double x) double absx, r, y, z, sqrtpow; /* special cases */ - if (!Py_IS_FINITE(x)) { - if (Py_IS_NAN(x) || x > 0.0) + if (!isfinite(x)) { + if (isnan(x) || x > 0.0) return x; /* tgamma(nan) = nan, tgamma(inf) = inf */ else { errno = EDOM; @@ -424,7 +424,7 @@ m_tgamma(double x) /* tiny arguments: tgamma(x) ~ 1/x for x near 0 */ if (absx < 1e-20) { r = 1.0/x; - if (Py_IS_INFINITY(r)) + if (isinf(r)) errno = ERANGE; return r; } @@ -438,7 +438,7 @@ m_tgamma(double x) } else { errno = ERANGE; - return Py_HUGE_VAL; + return Py_INFINITY; } } @@ -481,7 +481,7 @@ m_tgamma(double x) r *= sqrtpow; } } - if (Py_IS_INFINITY(r)) + if (isinf(r)) errno = ERANGE; return r; } @@ -498,18 +498,18 @@ m_lgamma(double x) double absx; /* special cases */ - if (!Py_IS_FINITE(x)) { - if (Py_IS_NAN(x)) + if (!isfinite(x)) { + if (isnan(x)) return x; /* lgamma(nan) = nan */ else - return Py_HUGE_VAL; /* lgamma(+-inf) = +inf */ + return Py_INFINITY; /* lgamma(+-inf) = +inf */ } /* integer arguments */ if (x == floor(x) && x <= 2.0) { if (x <= 0.0) { errno = EDOM; /* lgamma(n) = inf, divide-by-zero for */ - return Py_HUGE_VAL; /* integers n <= 0 */ + return Py_INFINITY; /* integers n <= 0 */ } else { return 0.0; /* lgamma(1) = lgamma(2) = 0.0 */ @@ -530,48 +530,11 @@ m_lgamma(double x) if (x < 0.0) /* Use reflection formula to get value for negative x. */ r = logpi - log(fabs(m_sinpi(absx))) - log(absx) - r; - if (Py_IS_INFINITY(r)) + if (isinf(r)) errno = ERANGE; return r; } -/* - wrapper for atan2 that deals directly with special cases before - delegating to the platform libm for the remaining cases. This - is necessary to get consistent behaviour across platforms. - Windows, FreeBSD and alpha Tru64 are amongst platforms that don't - always follow C99. -*/ - -static double -m_atan2(double y, double x) -{ - if (Py_IS_NAN(x) || Py_IS_NAN(y)) - return Py_NAN; - if (Py_IS_INFINITY(y)) { - if (Py_IS_INFINITY(x)) { - if (copysign(1., x) == 1.) - /* atan2(+-inf, +inf) == +-pi/4 */ - return copysign(0.25*Py_MATH_PI, y); - else - /* atan2(+-inf, -inf) == +-pi*3/4 */ - return copysign(0.75*Py_MATH_PI, y); - } - /* atan2(+-inf, x) == +-pi/2 for finite x */ - return copysign(0.5*Py_MATH_PI, y); - } - if (Py_IS_INFINITY(x) || y == 0.) { - if (copysign(1., x) == 1.) - /* atan2(+-y, +inf) = atan2(+-0, +x) = +-0. */ - return copysign(0., y); - else - /* atan2(+-y, -inf) = atan2(+-0., -x) = +-pi. */ - return copysign(Py_MATH_PI, y); - } - return atan2(y, x); -} - - /* IEEE 754-style remainder operation: x - n*y where n*y is the nearest multiple of y to x, taking n even in the case of a tie. Assuming an IEEE 754 binary floating-point format, the result is always exact. */ @@ -580,7 +543,7 @@ static double m_remainder(double x, double y) { /* Deal with most common case first. */ - if (Py_IS_FINITE(x) && Py_IS_FINITE(y)) { + if (isfinite(x) && isfinite(y)) { double absx, absy, c, m, r; if (y == 0.0) { @@ -653,16 +616,16 @@ m_remainder(double x, double y) } /* Special values. */ - if (Py_IS_NAN(x)) { + if (isnan(x)) { return x; } - if (Py_IS_NAN(y)) { + if (isnan(y)) { return y; } - if (Py_IS_INFINITY(x)) { + if (isinf(x)) { return Py_NAN; } - assert(Py_IS_INFINITY(y)); + assert(isinf(y)); return x; } @@ -677,16 +640,16 @@ m_remainder(double x, double y) static double m_log(double x) { - if (Py_IS_FINITE(x)) { + if (isfinite(x)) { if (x > 0.0) return log(x); errno = EDOM; if (x == 0.0) - return -Py_HUGE_VAL; /* log(0) = -inf */ + return -Py_INFINITY; /* log(0) = -inf */ else return Py_NAN; /* log(-ve) = nan */ } - else if (Py_IS_NAN(x)) + else if (isnan(x)) return x; /* log(nan) = nan */ else if (x > 0.0) return x; /* log(inf) = inf */ @@ -709,8 +672,8 @@ m_log(double x) static double m_log2(double x) { - if (!Py_IS_FINITE(x)) { - if (Py_IS_NAN(x)) + if (!isfinite(x)) { + if (isnan(x)) return x; /* log2(nan) = nan */ else if (x > 0.0) return x; /* log2(+inf) = +inf */ @@ -725,7 +688,7 @@ m_log2(double x) } else if (x == 0.0) { errno = EDOM; - return -Py_HUGE_VAL; /* log2(0) = -inf, divide-by-zero */ + return -Py_INFINITY; /* log2(0) = -inf, divide-by-zero */ } else { errno = EDOM; @@ -736,16 +699,16 @@ m_log2(double x) static double m_log10(double x) { - if (Py_IS_FINITE(x)) { + if (isfinite(x)) { if (x > 0.0) return log10(x); errno = EDOM; if (x == 0.0) - return -Py_HUGE_VAL; /* log10(0) = -inf */ + return -Py_INFINITY; /* log10(0) = -inf */ else return Py_NAN; /* log10(-ve) = nan */ } - else if (Py_IS_NAN(x)) + else if (isnan(x)) return x; /* log10(nan) = nan */ else if (x > 0.0) return x; /* log10(inf) = inf */ @@ -756,16 +719,26 @@ m_log10(double x) } +/*[clinic input] +math.gcd + + *integers as args: array + +Greatest Common Divisor. +[clinic start generated code]*/ + static PyObject * -math_gcd(PyObject *module, PyObject * const *args, Py_ssize_t nargs) +math_gcd_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=a26c95907374ffb4 input=ded7f0ea3850c05c]*/ { // Fast-path for the common case: gcd(int, int) - if (nargs == 2 && PyLong_CheckExact(args[0]) && PyLong_CheckExact(args[1])) + if (args_length == 2 && PyLong_CheckExact(args[0]) && PyLong_CheckExact(args[1])) { return _PyLong_GCD(args[0], args[1]); } - if (nargs == 0) { + if (args_length == 0) { return PyLong_FromLong(0); } @@ -773,13 +746,13 @@ math_gcd(PyObject *module, PyObject * const *args, Py_ssize_t nargs) if (res == NULL) { return NULL; } - if (nargs == 1) { + if (args_length == 1) { Py_SETREF(res, PyNumber_Absolute(res)); return res; } PyObject *one = _PyLong_GetOne(); // borrowed ref - for (Py_ssize_t i = 1; i < nargs; i++) { + for (Py_ssize_t i = 1; i < args_length; i++) { PyObject *x = _PyNumber_Index(args[i]); if (x == NULL) { Py_DECREF(res); @@ -800,12 +773,6 @@ math_gcd(PyObject *module, PyObject * const *args, Py_ssize_t nargs) return res; } -PyDoc_STRVAR(math_gcd_doc, -"gcd($module, *integers)\n" -"--\n" -"\n" -"Greatest Common Divisor."); - static PyObject * long_lcm(PyObject *a, PyObject *b) @@ -835,26 +802,36 @@ long_lcm(PyObject *a, PyObject *b) } +/*[clinic input] +math.lcm + + *integers as args: array + +Least Common Multiple. +[clinic start generated code]*/ + static PyObject * -math_lcm(PyObject *module, PyObject * const *args, Py_ssize_t nargs) +math_lcm_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=c8a59a5c2e55c816 input=3e4f4b7cdf948a98]*/ { PyObject *res, *x; Py_ssize_t i; - if (nargs == 0) { + if (args_length == 0) { return PyLong_FromLong(1); } res = PyNumber_Index(args[0]); if (res == NULL) { return NULL; } - if (nargs == 1) { + if (args_length == 1) { Py_SETREF(res, PyNumber_Absolute(res)); return res; } PyObject *zero = _PyLong_GetZero(); // borrowed ref - for (i = 1; i < nargs; i++) { + for (i = 1; i < args_length; i++) { x = PyNumber_Index(args[i]); if (x == NULL) { Py_DECREF(res); @@ -876,24 +853,20 @@ math_lcm(PyObject *module, PyObject * const *args, Py_ssize_t nargs) } -PyDoc_STRVAR(math_lcm_doc, -"lcm($module, *integers)\n" -"--\n" -"\n" -"Least Common Multiple."); - - /* Call is_error when errno != 0, and where x is the result libm * returned. is_error will usually set up an exception and return * true (1), but may return false (0) without setting up an exception. */ static int -is_error(double x) +is_error(double x, int raise_edom) { int result = 1; /* presumption of guilt */ assert(errno); /* non-zero errno is a precondition for calling */ - if (errno == EDOM) - PyErr_SetString(PyExc_ValueError, "math domain error"); + if (errno == EDOM) { + if (raise_edom) { + PyErr_SetString(PyExc_ValueError, "math domain error"); + } + } else if (errno == ERANGE) { /* ANSI C generally requires libm functions to set ERANGE @@ -958,7 +931,8 @@ is_error(double x) */ static PyObject * -math_1(PyObject *arg, double (*func) (double), int can_overflow) +math_1(PyObject *arg, double (*func) (double), int can_overflow, + const char *err_msg) { double x, r; x = PyFloat_AsDouble(arg); @@ -966,25 +940,34 @@ math_1(PyObject *arg, double (*func) (double), int can_overflow) return NULL; errno = 0; r = (*func)(x); - if (Py_IS_NAN(r) && !Py_IS_NAN(x)) { - PyErr_SetString(PyExc_ValueError, - "math domain error"); /* invalid arg */ - return NULL; - } - if (Py_IS_INFINITY(r) && Py_IS_FINITE(x)) { + if (isnan(r) && !isnan(x)) + goto domain_err; /* domain error */ + if (isinf(r) && isfinite(x)) { if (can_overflow) PyErr_SetString(PyExc_OverflowError, "math range error"); /* overflow */ else - PyErr_SetString(PyExc_ValueError, - "math domain error"); /* singularity */ + goto domain_err; /* singularity */ return NULL; } - if (Py_IS_FINITE(r) && errno && is_error(r)) + if (isfinite(r) && errno && is_error(r, 1)) /* this branch unnecessary on most platforms */ return NULL; return PyFloat_FromDouble(r); + +domain_err: + if (err_msg) { + char *buf = PyOS_double_to_string(x, 'r', 0, Py_DTSF_ADD_DOT_0, NULL); + if (buf) { + PyErr_Format(PyExc_ValueError, err_msg, buf); + PyMem_Free(buf); + } + } + else { + PyErr_SetString(PyExc_ValueError, "math domain error"); + } + return NULL; } /* variant of math_1, to be used when the function being wrapped is known to @@ -992,7 +975,7 @@ math_1(PyObject *arg, double (*func) (double), int can_overflow) errno = ERANGE for overflow). */ static PyObject * -math_1a(PyObject *arg, double (*func) (double)) +math_1a(PyObject *arg, double (*func) (double), const char *err_msg) { double x, r; x = PyFloat_AsDouble(arg); @@ -1000,8 +983,17 @@ math_1a(PyObject *arg, double (*func) (double)) return NULL; errno = 0; r = (*func)(x); - if (errno && is_error(r)) + if (errno && is_error(r, err_msg ? 0 : 1)) { + if (err_msg && errno == EDOM) { + assert(!PyErr_Occurred()); /* exception is not set by is_error() */ + char *buf = PyOS_double_to_string(x, 'r', 0, Py_DTSF_ADD_DOT_0, NULL); + if (buf) { + PyErr_Format(PyExc_ValueError, err_msg, buf); + PyMem_Free(buf); + } + } return NULL; + } return PyFloat_FromDouble(r); } @@ -1049,19 +1041,19 @@ math_2(PyObject *const *args, Py_ssize_t nargs, } errno = 0; r = (*func)(x, y); - if (Py_IS_NAN(r)) { - if (!Py_IS_NAN(x) && !Py_IS_NAN(y)) + if (isnan(r)) { + if (!isnan(x) && !isnan(y)) errno = EDOM; else errno = 0; } - else if (Py_IS_INFINITY(r)) { - if (Py_IS_FINITE(x) && Py_IS_FINITE(y)) + else if (isinf(r)) { + if (isfinite(x) && isfinite(y)) errno = ERANGE; else errno = 0; } - if (errno && is_error(r)) + if (errno && is_error(r, 1)) return NULL; else return PyFloat_FromDouble(r); @@ -1069,13 +1061,25 @@ math_2(PyObject *const *args, Py_ssize_t nargs, #define FUNC1(funcname, func, can_overflow, docstring) \ static PyObject * math_##funcname(PyObject *self, PyObject *args) { \ - return math_1(args, func, can_overflow); \ + return math_1(args, func, can_overflow, NULL); \ + }\ + PyDoc_STRVAR(math_##funcname##_doc, docstring); + +#define FUNC1D(funcname, func, can_overflow, docstring, err_msg) \ + static PyObject * math_##funcname(PyObject *self, PyObject *args) { \ + return math_1(args, func, can_overflow, err_msg); \ }\ PyDoc_STRVAR(math_##funcname##_doc, docstring); #define FUNC1A(funcname, func, docstring) \ static PyObject * math_##funcname(PyObject *self, PyObject *args) { \ - return math_1a(args, func); \ + return math_1a(args, func, NULL); \ + }\ + PyDoc_STRVAR(math_##funcname##_doc, docstring); + +#define FUNC1AD(funcname, func, docstring, err_msg) \ + static PyObject * math_##funcname(PyObject *self, PyObject *args) { \ + return math_1a(args, func, err_msg); \ }\ PyDoc_STRVAR(math_##funcname##_doc, docstring); @@ -1103,13 +1107,14 @@ FUNC1(atan, atan, 0, "atan($module, x, /)\n--\n\n" "Return the arc tangent (measured in radians) of x.\n\n" "The result is between -pi/2 and pi/2.") -FUNC2(atan2, m_atan2, +FUNC2(atan2, atan2, "atan2($module, y, x, /)\n--\n\n" "Return the arc tangent (measured in radians) of y/x.\n\n" "Unlike atan(y/x), the signs of both x and y are considered.") -FUNC1(atanh, atanh, 0, +FUNC1D(atanh, atanh, 0, "atanh($module, x, /)\n--\n\n" - "Return the inverse hyperbolic tangent of x.") + "Return the inverse hyperbolic tangent of x.", + "expected a number between -1 and 1, got %s") FUNC1(cbrt, cbrt, 0, "cbrt($module, x, /)\n--\n\n" "Return the cube root of x.") @@ -1220,9 +1225,10 @@ math_floor(PyObject *module, PyObject *number) return PyLong_FromDouble(floor(x)); } -FUNC1A(gamma, m_tgamma, +FUNC1AD(gamma, m_tgamma, "gamma($module, x, /)\n--\n\n" - "Gamma function at x.") + "Gamma function at x.", + "expected a float or nonnegative integer, got %s") FUNC1A(lgamma, m_lgamma, "lgamma($module, x, /)\n--\n\n" "Natural logarithm of absolute value of Gamma function at x.") @@ -1242,9 +1248,10 @@ FUNC1(sin, sin, 0, FUNC1(sinh, sinh, 1, "sinh($module, x, /)\n--\n\n" "Return the hyperbolic sine of x.") -FUNC1(sqrt, sqrt, 0, +FUNC1D(sqrt, sqrt, 0, "sqrt($module, x, /)\n--\n\n" - "Return the square root of x.") + "Return the square root of x.", + "expected a nonnegative input, got %s") FUNC1(tan, tan, 0, "tan($module, x, /)\n--\n\n" "Return the tangent of x (measured in radians).") @@ -1253,7 +1260,7 @@ FUNC1(tanh, tanh, 0, "Return the hyperbolic tangent of x.") /* Precision summation function as msum() by Raymond Hettinger in - , + , enhanced with the exact partials sum and roundoff from Mark Dickinson's post at . See those links for more details, proofs and other references. @@ -1354,14 +1361,14 @@ math.fsum seq: object / -Return an accurate floating point sum of values in the iterable seq. +Return an accurate floating-point sum of values in the iterable seq. -Assumes IEEE-754 floating point arithmetic. +Assumes IEEE-754 floating-point arithmetic. [clinic start generated code]*/ static PyObject * math_fsum(PyObject *module, PyObject *seq) -/*[clinic end generated code: output=ba5c672b87fe34fc input=c51b7d8caf6f6e82]*/ +/*[clinic end generated code: output=ba5c672b87fe34fc input=4506244ded6057dc]*/ { PyObject *item, *iter, *sum = NULL; Py_ssize_t i, j, n = 0, m = NUM_PARTIALS; @@ -1403,17 +1410,17 @@ math_fsum(PyObject *module, PyObject *seq) n = i; /* ps[i:] = [x] */ if (x != 0.0) { - if (! Py_IS_FINITE(x)) { + if (! isfinite(x)) { /* a nonfinite x could arise either as a result of intermediate overflow, or as a result of a nan or inf in the summands */ - if (Py_IS_FINITE(xsave)) { + if (isfinite(xsave)) { PyErr_SetString(PyExc_OverflowError, "intermediate overflow in fsum"); goto _fsum_error; } - if (Py_IS_INFINITY(xsave)) + if (isinf(xsave)) inf_sum += xsave; special_sum += xsave; /* reset partials */ @@ -1427,7 +1434,7 @@ math_fsum(PyObject *module, PyObject *seq) } if (special_sum != 0.0) { - if (Py_IS_NAN(inf_sum)) + if (isnan(inf_sum)) PyErr_SetString(PyExc_ValueError, "-inf + inf in fsum"); else @@ -1694,7 +1701,7 @@ math_isqrt(PyObject *module, PyObject *n) /*[clinic end generated code: output=35a6f7f980beab26 input=5b6e7ae4fa6c43d6]*/ { int a_too_large, c_bit_length; - size_t c, d; + int64_t c, d; uint64_t m; uint32_t u; PyObject *a = NULL, *b; @@ -1717,14 +1724,13 @@ math_isqrt(PyObject *module, PyObject *n) /* c = (n.bit_length() - 1) // 2 */ c = _PyLong_NumBits(n); - if (c == (size_t)(-1)) { - goto error; - } - c = (c - 1U) / 2U; + assert(c > 0); + assert(!PyErr_Occurred()); + c = (c - 1) / 2; /* Fast path: if c <= 31 then n < 2**64 and we can compute directly with a fast, almost branch-free algorithm. */ - if (c <= 31U) { + if (c <= 31) { int shift = 31 - (int)c; m = (uint64_t)PyLong_AsUnsignedLongLong(n); Py_DECREF(n); @@ -1741,13 +1747,13 @@ math_isqrt(PyObject *module, PyObject *n) /* From n >= 2**64 it follows that c.bit_length() >= 6. */ c_bit_length = 6; - while ((c >> c_bit_length) > 0U) { + while ((c >> c_bit_length) > 0) { ++c_bit_length; } /* Initialise d and a. */ d = c >> (c_bit_length - 5); - b = _PyLong_Rshift(n, 2U*c - 62U); + b = _PyLong_Rshift(n, 2*c - 62); if (b == NULL) { goto error; } @@ -1764,12 +1770,12 @@ math_isqrt(PyObject *module, PyObject *n) for (int s = c_bit_length - 6; s >= 0; --s) { PyObject *q; - size_t e = d; + int64_t e = d; d = c >> s; /* q = (n >> 2*c - e - d + 1) // a */ - q = _PyLong_Rshift(n, 2U*c - d - e + 1U); + q = _PyLong_Rshift(n, 2*c - d - e + 1); if (q == NULL) { goto error; } @@ -1779,7 +1785,7 @@ math_isqrt(PyObject *module, PyObject *n) } /* a = (a << d - 1 - e) + q */ - Py_SETREF(a, _PyLong_Lshift(a, d - 1U - e)); + Py_SETREF(a, _PyLong_Lshift(a, d - 1 - e)); if (a == NULL) { Py_DECREF(q); goto error; @@ -2108,7 +2114,7 @@ math_frexp_impl(PyObject *module, double x) int i; /* deal with special cases directly, to sidestep platform differences */ - if (Py_IS_NAN(x) || Py_IS_INFINITY(x) || !x) { + if (isnan(x) || isinf(x) || !x) { i = 0; } else { @@ -2153,13 +2159,13 @@ math_ldexp_impl(PyObject *module, double x, PyObject *i) return NULL; } - if (x == 0. || !Py_IS_FINITE(x)) { + if (x == 0. || !isfinite(x)) { /* NaNs, zeros and infinities are returned unchanged */ r = x; errno = 0; } else if (exp > INT_MAX) { /* overflow */ - r = copysign(Py_HUGE_VAL, x); + r = copysign(Py_INFINITY, x); errno = ERANGE; } else if (exp < INT_MIN) { /* underflow to +-0 */ @@ -2168,11 +2174,11 @@ math_ldexp_impl(PyObject *module, double x, PyObject *i) } else { errno = 0; r = ldexp(x, (int)exp); - if (Py_IS_INFINITY(r)) + if (isinf(r)) errno = ERANGE; } - if (errno && is_error(r)) + if (errno && is_error(r, 1)) return NULL; return PyFloat_FromDouble(r); } @@ -2196,9 +2202,9 @@ math_modf_impl(PyObject *module, double x) double y; /* some platforms don't do the right thing for NaNs and infinities, so we take care of special cases directly. */ - if (Py_IS_INFINITY(x)) + if (isinf(x)) return Py_BuildValue("(dd)", copysign(0., x), x); - else if (Py_IS_NAN(x)) + else if (isnan(x)) return Py_BuildValue("(dd)", x, x); errno = 0; @@ -2222,12 +2228,12 @@ loghelper(PyObject* arg, double (*func)(double)) /* If it is int, do it ourselves. */ if (PyLong_Check(arg)) { double x, result; - Py_ssize_t e; + int64_t e; /* Negative or zero inputs give a ValueError. */ if (!_PyLong_IsPositive((PyLongObject *)arg)) { - PyErr_SetString(PyExc_ValueError, - "math domain error"); + PyErr_Format(PyExc_ValueError, + "expected a positive input, got %S", arg); return NULL; } @@ -2239,8 +2245,8 @@ loghelper(PyObject* arg, double (*func)(double)) to compute the log anyway. Clear the exception and continue. */ PyErr_Clear(); x = _PyLong_Frexp((PyLongObject *)arg, &e); - if (x == -1.0 && PyErr_Occurred()) - return NULL; + assert(e >= 0); + assert(!PyErr_Occurred()); /* Value is ~= x * 2**e, so the log ~= log(x) + log(2) * e. */ result = func(x) + func(2.0) * e; } @@ -2251,7 +2257,7 @@ loghelper(PyObject* arg, double (*func)(double)) } /* Else let libm handle it by itself. */ - return math_1(arg, func, 0); + return math_1(arg, func, 0, "expected a positive input, got %s"); } @@ -2321,6 +2327,48 @@ math_log10(PyObject *module, PyObject *x) } +/*[clinic input] +math.fma + + x: double + y: double + z: double + / + +Fused multiply-add operation. + +Compute (x * y) + z with a single round. +[clinic start generated code]*/ + +static PyObject * +math_fma_impl(PyObject *module, double x, double y, double z) +/*[clinic end generated code: output=4fc8626dbc278d17 input=e3ad1f4a4c89626e]*/ +{ + double r = fma(x, y, z); + + /* Fast path: if we got a finite result, we're done. */ + if (isfinite(r)) { + return PyFloat_FromDouble(r); + } + + /* Non-finite result. Raise an exception if appropriate, else return r. */ + if (isnan(r)) { + if (!isnan(x) && !isnan(y) && !isnan(z)) { + /* NaN result from non-NaN inputs. */ + PyErr_SetString(PyExc_ValueError, "invalid operation in fma"); + return NULL; + } + } + else if (isfinite(x) && isfinite(y) && isfinite(z)) { + /* Infinite result from finite inputs. */ + PyErr_SetString(PyExc_OverflowError, "overflow in fma"); + return NULL; + } + + return PyFloat_FromDouble(r); +} + + /*[clinic input] math.fmod @@ -2339,17 +2387,26 @@ math_fmod_impl(PyObject *module, double x, double y) { double r; /* fmod(x, +/-Inf) returns x for finite x. */ - if (Py_IS_INFINITY(y) && Py_IS_FINITE(x)) + if (isinf(y) && isfinite(x)) return PyFloat_FromDouble(x); errno = 0; r = fmod(x, y); - if (Py_IS_NAN(r)) { - if (!Py_IS_NAN(x) && !Py_IS_NAN(y)) +#ifdef _MSC_VER + /* Windows (e.g. Windows 10 with MSC v.1916) loose sign + for zero result. But C99+ says: "if y is nonzero, the result + has the same sign as x". + */ + if (r == 0.0 && y != 0.0) { + r = copysign(r, x); + } +#endif + if (isnan(r)) { + if (!isnan(x) && !isnan(y)) errno = EDOM; else errno = 0; } - if (errno && is_error(r)) + if (errno && is_error(r, 1)) return NULL; else return PyFloat_FromDouble(r); @@ -2411,7 +2468,7 @@ Since lo**2 is less than 1/2 ulp(csum), we have csum+lo*lo == csum. To minimize loss of information during the accumulation of fractional values, each term has a separate accumulator. This also breaks up sequential dependencies in the inner loop so the CPU can maximize -floating point throughput. [4] On an Apple M1 Max, hypot(*vec) +floating-point throughput. [4] On an Apple M1 Max, hypot(*vec) takes only 3.33 µsec when len(vec) == 1000. The square root differential correction is needed because a @@ -2466,7 +2523,7 @@ vector_norm(Py_ssize_t n, double *vec, double max, int found_nan) int max_e; Py_ssize_t i; - if (Py_IS_INFINITY(max)) { + if (isinf(max)) { return max; } if (found_nan) { @@ -2488,7 +2545,7 @@ vector_norm(Py_ssize_t n, double *vec, double max, int found_nan) assert(max * scale < 1.0); for (i=0 ; i < n ; i++) { x = vec[i]; - assert(Py_IS_FINITE(x) && fabs(x) <= max); + assert(isfinite(x) && fabs(x) <= max); x *= scale; // lossless scaling assert(fabs(x) < 1.0); pr = dl_mul(x, x); // lossless squaring @@ -2578,7 +2635,7 @@ math_dist_impl(PyObject *module, PyObject *p, PyObject *q) ASSIGN_DOUBLE(qx, item, error_exit); x = fabs(px - qx); diffs[i] = x; - found_nan |= Py_IS_NAN(x); + found_nan |= isnan(x); if (x > max) { max = x; } @@ -2608,9 +2665,29 @@ math_dist_impl(PyObject *module, PyObject *p, PyObject *q) return NULL; } -/* AC: cannot convert yet, waiting for *args support */ +/*[clinic input] +math.hypot + + *coordinates as args: array + +Multidimensional Euclidean distance from the origin to a point. + +Roughly equivalent to: + sqrt(sum(x**2 for x in coordinates)) + +For a two dimensional point (x, y), gives the hypotenuse +using the Pythagorean theorem: sqrt(x*x + y*y). + +For example, the hypotenuse of a 3/4/5 right triangle is: + + >>> hypot(3.0, 4.0) + 5.0 +[clinic start generated code]*/ + static PyObject * -math_hypot(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +math_hypot_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=c9de404e24370068 input=1bceaf7d4fdcd9c2]*/ { Py_ssize_t i; PyObject *item; @@ -2620,23 +2697,23 @@ math_hypot(PyObject *self, PyObject *const *args, Py_ssize_t nargs) double coord_on_stack[NUM_STACK_ELEMS]; double *coordinates = coord_on_stack; - if (nargs > NUM_STACK_ELEMS) { - coordinates = (double *) PyMem_Malloc(nargs * sizeof(double)); + if (args_length > NUM_STACK_ELEMS) { + coordinates = (double *) PyMem_Malloc(args_length * sizeof(double)); if (coordinates == NULL) { return PyErr_NoMemory(); } } - for (i = 0; i < nargs; i++) { + for (i = 0; i < args_length; i++) { item = args[i]; ASSIGN_DOUBLE(x, item, error_exit); x = fabs(x); coordinates[i] = x; - found_nan |= Py_IS_NAN(x); + found_nan |= isnan(x); if (x > max) { max = x; } } - result = vector_norm(nargs, coordinates, max, found_nan); + result = vector_norm(args_length, coordinates, max, found_nan); if (coordinates != coord_on_stack) { PyMem_Free(coordinates); } @@ -2651,22 +2728,6 @@ math_hypot(PyObject *self, PyObject *const *args, Py_ssize_t nargs) #undef NUM_STACK_ELEMS -PyDoc_STRVAR(math_hypot_doc, - "hypot(*coordinates) -> value\n\n\ -Multidimensional Euclidean distance from the origin to a point.\n\ -\n\ -Roughly equivalent to:\n\ - sqrt(sum(x**2 for x in coordinates))\n\ -\n\ -For a two dimensional point (x, y), gives the hypotenuse\n\ -using the Pythagorean theorem: sqrt(x*x + y*y).\n\ -\n\ -For example, the hypotenuse of a 3/4/5 right triangle is:\n\ -\n\ - >>> hypot(3.0, 4.0)\n\ - 5.0\n\ -"); - /** sumprod() ***************************************************************/ /* Forward declaration */ @@ -2689,7 +2750,7 @@ Return the sum of products of values from two iterables p and q. Roughly equivalent to: - sum(itertools.starmap(operator.mul, zip(p, q, strict=True))) + sum(map(operator.mul, p, q, strict=True)) For float and mixed int/float inputs, the intermediate products and sums are computed with extended precision. @@ -2697,7 +2758,7 @@ and sums are computed with extended precision. static PyObject * math_sumprod_impl(PyObject *module, PyObject *p, PyObject *q) -/*[clinic end generated code: output=6722dbfe60664554 input=82be54fe26f87e30]*/ +/*[clinic end generated code: output=6722dbfe60664554 input=a2880317828c61d2]*/ { PyObject *p_i = NULL, *q_i = NULL, *term_i = NULL, *new_total = NULL; PyObject *p_it, *q_it, *total; @@ -2934,14 +2995,14 @@ math_pow_impl(PyObject *module, double x, double y) /* deal directly with IEEE specials, to cope with problems on various platforms whose semantics don't exactly match C99 */ r = 0.; /* silence compiler warning */ - if (!Py_IS_FINITE(x) || !Py_IS_FINITE(y)) { + if (!isfinite(x) || !isfinite(y)) { errno = 0; - if (Py_IS_NAN(x)) + if (isnan(x)) r = y == 0. ? 1. : x; /* NaN**0 = 1 */ - else if (Py_IS_NAN(y)) + else if (isnan(y)) r = x == 1. ? 1. : y; /* 1**NaN = 1 */ - else if (Py_IS_INFINITY(x)) { - odd_y = Py_IS_FINITE(y) && fmod(fabs(y), 2.0) == 1.0; + else if (isinf(x)) { + odd_y = isfinite(y) && fmod(fabs(y), 2.0) == 1.0; if (y > 0.) r = odd_y ? x : fabs(x); else if (y == 0.) @@ -2950,7 +3011,7 @@ math_pow_impl(PyObject *module, double x, double y) r = odd_y ? copysign(0., x) : 0.; } else { - assert(Py_IS_INFINITY(y)); + assert(isinf(y)); if (fabs(x) == 1.0) r = 1.; else if (y > 0. && fabs(x) > 1.0) @@ -2968,8 +3029,8 @@ math_pow_impl(PyObject *module, double x, double y) r = pow(x, y); /* a NaN result should arise only from (-ve)**(finite non-integer); in this case we want to raise ValueError. */ - if (!Py_IS_FINITE(r)) { - if (Py_IS_NAN(r)) { + if (!isfinite(r)) { + if (isnan(r)) { errno = EDOM; } /* @@ -2977,7 +3038,7 @@ math_pow_impl(PyObject *module, double x, double y) (A) (+/-0.)**negative (-> divide-by-zero) (B) overflow of x**y with x and y finite */ - else if (Py_IS_INFINITY(r)) { + else if (isinf(r)) { if (x == 0.) errno = EDOM; else @@ -2986,7 +3047,7 @@ math_pow_impl(PyObject *module, double x, double y) } } - if (errno && is_error(r)) + if (errno && is_error(r, 1)) return NULL; else return PyFloat_FromDouble(r); @@ -3043,7 +3104,7 @@ static PyObject * math_isfinite_impl(PyObject *module, double x) /*[clinic end generated code: output=8ba1f396440c9901 input=46967d254812e54a]*/ { - return PyBool_FromLong((long)Py_IS_FINITE(x)); + return PyBool_FromLong((long)isfinite(x)); } @@ -3060,7 +3121,7 @@ static PyObject * math_isnan_impl(PyObject *module, double x) /*[clinic end generated code: output=f537b4d6df878c3e input=935891e66083f46a]*/ { - return PyBool_FromLong((long)Py_IS_NAN(x)); + return PyBool_FromLong((long)isnan(x)); } @@ -3077,7 +3138,7 @@ static PyObject * math_isinf_impl(PyObject *module, double x) /*[clinic end generated code: output=9f00cbec4de7b06b input=32630e4212cf961f]*/ { - return PyBool_FromLong((long)Py_IS_INFINITY(x)); + return PyBool_FromLong((long)isinf(x)); } @@ -3094,7 +3155,7 @@ math.isclose -> bool maximum difference for being considered "close", regardless of the magnitude of the input values -Determine whether two floating point numbers are close in value. +Determine whether two floating-point numbers are close in value. Return True if a is close in value to b, and False otherwise. @@ -3109,7 +3170,7 @@ only close to themselves. static int math_isclose_impl(PyObject *module, double a, double b, double rel_tol, double abs_tol) -/*[clinic end generated code: output=b73070207511952d input=f28671871ea5bfba]*/ +/*[clinic end generated code: output=b73070207511952d input=12d41764468bfdb8]*/ { double diff = 0.0; @@ -3134,7 +3195,7 @@ math_isclose_impl(PyObject *module, double a, double b, double rel_tol, above. */ - if (Py_IS_INFINITY(a) || Py_IS_INFINITY(b)) { + if (isinf(a) || isinf(b)) { return 0; } @@ -3884,10 +3945,10 @@ math_nextafter_impl(PyObject *module, double x, double y, PyObject *steps) Bug fixed in bos.adt.libm 7.2.2.0 by APAR IV95512. */ return PyFloat_FromDouble(y); } - if (Py_IS_NAN(x)) { + if (isnan(x)) { return PyFloat_FromDouble(x); } - if (Py_IS_NAN(y)) { + if (isnan(y)) { return PyFloat_FromDouble(y); } #endif @@ -3933,10 +3994,10 @@ math_nextafter_impl(PyObject *module, double x, double y, PyObject *steps) if (usteps == 0) { return PyFloat_FromDouble(x); } - if (Py_IS_NAN(x)) { + if (isnan(x)) { return PyFloat_FromDouble(x); } - if (Py_IS_NAN(y)) { + if (isnan(y)) { return PyFloat_FromDouble(y); } @@ -4002,16 +4063,16 @@ static double math_ulp_impl(PyObject *module, double x) /*[clinic end generated code: output=f5207867a9384dd4 input=31f9bfbbe373fcaa]*/ { - if (Py_IS_NAN(x)) { + if (isnan(x)) { return x; } x = fabs(x); - if (Py_IS_INFINITY(x)) { + if (isinf(x)) { return x; } double inf = Py_INFINITY; double x2 = nextafter(x, inf); - if (Py_IS_INFINITY(x2)) { + if (isinf(x2)) { /* special case: x is the largest positive representable float */ x2 = nextafter(x, -inf); return x - x2; @@ -4094,18 +4155,19 @@ static PyMethodDef math_methods[] = { {"fabs", math_fabs, METH_O, math_fabs_doc}, MATH_FACTORIAL_METHODDEF MATH_FLOOR_METHODDEF + MATH_FMA_METHODDEF MATH_FMOD_METHODDEF MATH_FREXP_METHODDEF MATH_FSUM_METHODDEF {"gamma", math_gamma, METH_O, math_gamma_doc}, - {"gcd", _PyCFunction_CAST(math_gcd), METH_FASTCALL, math_gcd_doc}, - {"hypot", _PyCFunction_CAST(math_hypot), METH_FASTCALL, math_hypot_doc}, + MATH_GCD_METHODDEF + MATH_HYPOT_METHODDEF MATH_ISCLOSE_METHODDEF MATH_ISFINITE_METHODDEF MATH_ISINF_METHODDEF MATH_ISNAN_METHODDEF MATH_ISQRT_METHODDEF - {"lcm", _PyCFunction_CAST(math_lcm), METH_FASTCALL, math_lcm_doc}, + MATH_LCM_METHODDEF MATH_LDEXP_METHODDEF {"lgamma", math_lgamma, METH_O, math_lgamma_doc}, {"log", _PyCFunction_CAST(math_log), METH_FASTCALL, math_log_doc}, @@ -4134,6 +4196,7 @@ static PyMethodDef math_methods[] = { static PyModuleDef_Slot math_slots[] = { {Py_mod_exec, math_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/md5module.c b/Modules/md5module.c index 7d2b3275f213fd..d86c8e555012d7 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -51,9 +51,11 @@ typedef struct { // Prevents undefined behavior via multiple threads entering the C API. bool use_mutex; PyMutex mutex; - Hacl_Streaming_MD5_state *hash_state; + Hacl_Hash_MD5_state_t *hash_state; } MD5object; +#define _MD5object_CAST(op) ((MD5object *)(op)) + #include "clinic/md5module.c.h" @@ -72,7 +74,7 @@ md5_get_state(PyObject *module) static MD5object * newMD5object(MD5State * st) { - MD5object *md5 = (MD5object *)PyObject_GC_New(MD5object, st->md5_type); + MD5object *md5 = PyObject_GC_New(MD5object, st->md5_type); if (!md5) { return NULL; } @@ -91,10 +93,11 @@ MD5_traverse(PyObject *ptr, visitproc visit, void *arg) } static void -MD5_dealloc(MD5object *ptr) +MD5_dealloc(PyObject *op) { - Hacl_Streaming_MD5_legacy_free(ptr->hash_state); - PyTypeObject *tp = Py_TYPE((PyObject*)ptr); + MD5object *ptr = _MD5object_CAST(op); + Hacl_Hash_MD5_free(ptr->hash_state); + PyTypeObject *tp = Py_TYPE(op); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); Py_DECREF(tp); @@ -122,7 +125,7 @@ MD5Type_copy_impl(MD5object *self, PyTypeObject *cls) return NULL; ENTER_HASHLIB(self); - newobj->hash_state = Hacl_Streaming_MD5_legacy_copy(self->hash_state); + newobj->hash_state = Hacl_Hash_MD5_copy(self->hash_state); LEAVE_HASHLIB(self); return (PyObject *)newobj; } @@ -139,7 +142,7 @@ MD5Type_digest_impl(MD5object *self) { unsigned char digest[MD5_DIGESTSIZE]; ENTER_HASHLIB(self); - Hacl_Streaming_MD5_legacy_finish(self->hash_state, digest); + Hacl_Hash_MD5_digest(self->hash_state, digest); LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, MD5_DIGESTSIZE); } @@ -156,7 +159,7 @@ MD5Type_hexdigest_impl(MD5object *self) { unsigned char digest[MD5_DIGESTSIZE]; ENTER_HASHLIB(self); - Hacl_Streaming_MD5_legacy_finish(self->hash_state, digest); + Hacl_Hash_MD5_digest(self->hash_state, digest); LEAVE_HASHLIB(self); const char *hexdigits = "0123456789abcdef"; @@ -170,15 +173,15 @@ MD5Type_hexdigest_impl(MD5object *self) return PyUnicode_FromStringAndSize(digest_hex, sizeof(digest_hex)); } -static void update(Hacl_Streaming_MD5_state *state, uint8_t *buf, Py_ssize_t len) { +static void update(Hacl_Hash_MD5_state_t *state, uint8_t *buf, Py_ssize_t len) { #if PY_SSIZE_T_MAX > UINT32_MAX while (len > UINT32_MAX) { - Hacl_Streaming_MD5_legacy_update(state, buf, UINT32_MAX); + Hacl_Hash_MD5_update(state, buf, UINT32_MAX); len -= UINT32_MAX; buf += UINT32_MAX; } #endif - Hacl_Streaming_MD5_legacy_update(state, buf, (uint32_t) len); + Hacl_Hash_MD5_update(state, buf, (uint32_t) len); } /*[clinic input] @@ -224,36 +227,27 @@ static PyMethodDef MD5_methods[] = { }; static PyObject * -MD5_get_block_size(PyObject *self, void *closure) +MD5_get_block_size(PyObject *Py_UNUSED(self), void *Py_UNUSED(closure)) { return PyLong_FromLong(MD5_BLOCKSIZE); } static PyObject * -MD5_get_name(PyObject *self, void *closure) +MD5_get_name(PyObject *Py_UNUSED(self), void *Py_UNUSED(closure)) { return PyUnicode_FromStringAndSize("md5", 3); } static PyObject * -md5_get_digest_size(PyObject *self, void *closure) +md5_get_digest_size(PyObject *Py_UNUSED(self), void *Py_UNUSED(closure)) { return PyLong_FromLong(MD5_DIGESTSIZE); } static PyGetSetDef MD5_getseters[] = { - {"block_size", - (getter)MD5_get_block_size, NULL, - NULL, - NULL}, - {"name", - (getter)MD5_get_name, NULL, - NULL, - NULL}, - {"digest_size", - (getter)md5_get_digest_size, NULL, - NULL, - NULL}, + {"block_size", MD5_get_block_size, NULL, NULL, NULL}, + {"name", MD5_get_name, NULL, NULL, NULL}, + {"digest_size", md5_get_digest_size, NULL, NULL, NULL}, {NULL} /* Sentinel */ }; @@ -302,7 +296,7 @@ _md5_md5_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - new->hash_state = Hacl_Streaming_MD5_legacy_create_in(); + new->hash_state = Hacl_Hash_MD5_malloc(); if (PyErr_Occurred()) { Py_DECREF(new); @@ -375,6 +369,7 @@ md5_exec(PyObject *m) static PyModuleDef_Slot _md5_slots[] = { {Py_mod_exec, md5_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 0cce7c27f9b16a..e1c26e19932664 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -41,6 +41,7 @@ #ifdef MS_WINDOWS #include +#include // LsaNtStatusToWinError static int my_getpagesize(void) { @@ -255,6 +256,208 @@ do { \ } while (0) #endif /* UNIX */ +#if defined(MS_WINDOWS) && !defined(DONT_USE_SEH) +static DWORD +filter_page_exception(EXCEPTION_POINTERS *ptrs, EXCEPTION_RECORD *record) +{ + *record = *ptrs->ExceptionRecord; + if (record->ExceptionCode == EXCEPTION_IN_PAGE_ERROR || + record->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) + { + return EXCEPTION_EXECUTE_HANDLER; + } + return EXCEPTION_CONTINUE_SEARCH; +} + +static DWORD +filter_page_exception_method(mmap_object *self, EXCEPTION_POINTERS *ptrs, + EXCEPTION_RECORD *record) +{ + *record = *ptrs->ExceptionRecord; + if (record->ExceptionCode == EXCEPTION_IN_PAGE_ERROR || + record->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) + { + + ULONG_PTR address = record->ExceptionInformation[1]; + if (address >= (ULONG_PTR) self->data && + address < (ULONG_PTR) self->data + (ULONG_PTR) self->size) + { + return EXCEPTION_EXECUTE_HANDLER; + } + } + return EXCEPTION_CONTINUE_SEARCH; +} +#endif + +#if defined(MS_WINDOWS) && !defined(DONT_USE_SEH) +#define HANDLE_INVALID_MEM(sourcecode) \ +do { \ + EXCEPTION_RECORD record; \ + __try { \ + sourcecode \ + } \ + __except (filter_page_exception(GetExceptionInformation(), &record)) { \ + assert(record.ExceptionCode == EXCEPTION_IN_PAGE_ERROR || \ + record.ExceptionCode == EXCEPTION_ACCESS_VIOLATION); \ + if (record.ExceptionCode == EXCEPTION_IN_PAGE_ERROR) { \ + NTSTATUS status = (NTSTATUS) record.ExceptionInformation[2]; \ + ULONG code = LsaNtStatusToWinError(status); \ + PyErr_SetFromWindowsErr(code); \ + } \ + else if (record.ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { \ + PyErr_SetFromWindowsErr(ERROR_NOACCESS); \ + } \ + return -1; \ + } \ +} while (0) +#else +#define HANDLE_INVALID_MEM(sourcecode) \ +do { \ + sourcecode \ +} while (0) +#endif + +#if defined(MS_WINDOWS) && !defined(DONT_USE_SEH) +#define HANDLE_INVALID_MEM_METHOD(self, sourcecode) \ +do { \ + EXCEPTION_RECORD record; \ + __try { \ + sourcecode \ + } \ + __except (filter_page_exception_method(self, GetExceptionInformation(), \ + &record)) { \ + assert(record.ExceptionCode == EXCEPTION_IN_PAGE_ERROR || \ + record.ExceptionCode == EXCEPTION_ACCESS_VIOLATION); \ + if (record.ExceptionCode == EXCEPTION_IN_PAGE_ERROR) { \ + NTSTATUS status = (NTSTATUS) record.ExceptionInformation[2]; \ + ULONG code = LsaNtStatusToWinError(status); \ + PyErr_SetFromWindowsErr(code); \ + } \ + else if (record.ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { \ + PyErr_SetFromWindowsErr(ERROR_NOACCESS); \ + } \ + return -1; \ + } \ +} while (0) +#else +#define HANDLE_INVALID_MEM_METHOD(self, sourcecode) \ +do { \ + sourcecode \ +} while (0) +#endif + +int +safe_memcpy(void *dest, const void *src, size_t count) +{ + HANDLE_INVALID_MEM( + memcpy(dest, src, count); + ); + return 0; +} + +int +safe_byte_copy(char *dest, const char *src) +{ + HANDLE_INVALID_MEM( + *dest = *src; + ); + return 0; +} + +int +safe_memchr(char **out, const void *ptr, int ch, size_t count) +{ + HANDLE_INVALID_MEM( + *out = (char *) memchr(ptr, ch, count); + ); + return 0; +} + +int +safe_memmove(void *dest, const void *src, size_t count) +{ + HANDLE_INVALID_MEM( + memmove(dest, src, count); + ); + return 0; +} + +int +safe_copy_from_slice(char *dest, const char *src, Py_ssize_t start, + Py_ssize_t step, Py_ssize_t slicelen) +{ + HANDLE_INVALID_MEM( + size_t cur; + Py_ssize_t i; + for (cur = start, i = 0; i < slicelen; cur += step, i++) { + dest[cur] = src[i]; + } + ); + return 0; +} + +int +safe_copy_to_slice(char *dest, const char *src, Py_ssize_t start, + Py_ssize_t step, Py_ssize_t slicelen) +{ + HANDLE_INVALID_MEM( + size_t cur; + Py_ssize_t i; + for (cur = start, i = 0; i < slicelen; cur += step, i++) { + dest[i] = src[cur]; + } + ); + return 0; +} + + +int +_safe_PyBytes_Find(Py_ssize_t *out, mmap_object *self, const char *haystack, + Py_ssize_t len_haystack, const char *needle, + Py_ssize_t len_needle, Py_ssize_t offset) +{ + HANDLE_INVALID_MEM_METHOD(self, + *out = _PyBytes_Find(haystack, len_haystack, needle, len_needle, offset); + ); + return 0; +} + +int +_safe_PyBytes_ReverseFind(Py_ssize_t *out, mmap_object *self, + const char *haystack, Py_ssize_t len_haystack, + const char *needle, Py_ssize_t len_needle, + Py_ssize_t offset) +{ + HANDLE_INVALID_MEM_METHOD(self, + *out = _PyBytes_ReverseFind(haystack, len_haystack, needle, len_needle, + offset); + ); + return 0; +} + +PyObject * +_safe_PyBytes_FromStringAndSize(char *start, size_t num_bytes) { + if (num_bytes == 1) { + char dest; + if (safe_byte_copy(&dest, start) < 0) { + return NULL; + } + else { + return PyBytes_FromStringAndSize(&dest, 1); + } + } + else { + PyObject *result = PyBytes_FromStringAndSize(NULL, num_bytes); + if (result == NULL) { + return NULL; + } + if (safe_memcpy(PyBytes_AS_STRING(result), start, num_bytes) < 0) { + Py_CLEAR(result); + } + return result; + } +} + static PyObject * mmap_read_byte_method(mmap_object *self, PyObject *Py_UNUSED(ignored)) @@ -264,7 +467,12 @@ mmap_read_byte_method(mmap_object *self, PyErr_SetString(PyExc_ValueError, "read byte out of range"); return NULL; } - return PyLong_FromLong((unsigned char)self->data[self->pos++]); + char dest; + if (safe_byte_copy(&dest, self->data + self->pos) < 0) { + return NULL; + } + self->pos++; + return PyLong_FromLong((unsigned char) dest); } static PyObject * @@ -273,21 +481,27 @@ mmap_read_line_method(mmap_object *self, { Py_ssize_t remaining; char *start, *eol; - PyObject *result; CHECK_VALID(NULL); remaining = (self->pos < self->size) ? self->size - self->pos : 0; if (!remaining) - return PyBytes_FromString(""); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); start = self->data + self->pos; - eol = memchr(start, '\n', remaining); + + if (safe_memchr(&eol, start, '\n', remaining) < 0) { + return NULL; + } + if (!eol) eol = self->data + self->size; else ++eol; /* advance past newline */ - result = PyBytes_FromStringAndSize(start, (eol - start)); - self->pos += (eol - start); + + PyObject *result = _safe_PyBytes_FromStringAndSize(start, eol - start); + if (result != NULL) { + self->pos += (eol - start); + } return result; } @@ -296,7 +510,6 @@ mmap_read_method(mmap_object *self, PyObject *args) { Py_ssize_t num_bytes = PY_SSIZE_T_MAX, remaining; - PyObject *result; CHECK_VALID(NULL); if (!PyArg_ParseTuple(args, "|O&:read", _Py_convert_optional_to_ssize_t, &num_bytes)) @@ -307,8 +520,12 @@ mmap_read_method(mmap_object *self, remaining = (self->pos < self->size) ? self->size - self->pos : 0; if (num_bytes < 0 || num_bytes > remaining) num_bytes = remaining; - result = PyBytes_FromStringAndSize(&self->data[self->pos], num_bytes); - self->pos += num_bytes; + + PyObject *result = _safe_PyBytes_FromStringAndSize(self->data + self->pos, + num_bytes); + if (result != NULL) { + self->pos += num_bytes; + } return result; } @@ -341,25 +558,38 @@ mmap_gfind(mmap_object *self, else if (end > self->size) end = self->size; - Py_ssize_t res; + Py_ssize_t index; + PyObject *result; CHECK_VALID_OR_RELEASE(NULL, view); if (end < start) { - res = -1; + result = PyLong_FromSsize_t(-1); } else if (reverse) { assert(0 <= start && start <= end && end <= self->size); - res = _PyBytes_ReverseFind( + if (_safe_PyBytes_ReverseFind(&index, self, self->data + start, end - start, - view.buf, view.len, start); + view.buf, view.len, start) < 0) + { + result = NULL; + } + else { + result = PyLong_FromSsize_t(index); + } } else { assert(0 <= start && start <= end && end <= self->size); - res = _PyBytes_Find( + if (_safe_PyBytes_Find(&index, self, self->data + start, end - start, - view.buf, view.len, start); + view.buf, view.len, start) < 0) + { + result = NULL; + } + else { + result = PyLong_FromSsize_t(index); + } } PyBuffer_Release(&view); - return PyLong_FromSsize_t(res); + return result; } } @@ -432,10 +662,16 @@ mmap_write_method(mmap_object *self, } CHECK_VALID_OR_RELEASE(NULL, data); - memcpy(&self->data[self->pos], data.buf, data.len); - self->pos += data.len; + PyObject *result; + if (safe_memcpy(self->data + self->pos, data.buf, data.len) < 0) { + result = NULL; + } + else { + self->pos += data.len; + result = PyLong_FromSsize_t(data.len); + } PyBuffer_Release(&data); - return PyLong_FromSsize_t(data.len); + return result; } static PyObject * @@ -452,14 +688,16 @@ mmap_write_byte_method(mmap_object *self, return NULL; CHECK_VALID(NULL); - if (self->pos < self->size) { - self->data[self->pos++] = value; - Py_RETURN_NONE; - } - else { + if (self->pos >= self->size) { PyErr_SetString(PyExc_ValueError, "write byte out of range"); return NULL; } + + if (safe_byte_copy(self->data + self->pos, &value) < 0) { + return NULL; + } + self->pos++; + Py_RETURN_NONE; } static PyObject * @@ -763,8 +1001,9 @@ mmap_move_method(mmap_object *self, PyObject *args) goto bounds; CHECK_VALID(NULL); - memmove(&self->data[dest], &self->data[src], cnt); - + if (safe_memmove(self->data + dest, self->data + src, cnt) < 0) { + return NULL; + }; Py_RETURN_NONE; bounds: @@ -855,6 +1094,29 @@ mmap__sizeof__method(mmap_object *self, void *Py_UNUSED(ignored)) } #endif +#if defined(MS_WINDOWS) && defined(Py_DEBUG) +static PyObject * +mmap_protect_method(mmap_object *self, PyObject *args) { + DWORD flNewProtect, flOldProtect; + Py_ssize_t start, length; + + CHECK_VALID(NULL); + + if (!PyArg_ParseTuple(args, "Inn:protect", &flNewProtect, &start, &length)) { + return NULL; + } + + if (!VirtualProtect((void *) (self->data + start), length, flNewProtect, + &flOldProtect)) + { + PyErr_SetFromWindowsErr(GetLastError()); + return NULL; + } + + Py_RETURN_NONE; +} +#endif + #ifdef HAVE_MADVISE static PyObject * mmap_madvise_method(mmap_object *self, PyObject *args) @@ -924,7 +1186,10 @@ static struct PyMethodDef mmap_object_methods[] = { {"__exit__", (PyCFunction) mmap__exit__method, METH_VARARGS}, #ifdef MS_WINDOWS {"__sizeof__", (PyCFunction) mmap__sizeof__method, METH_NOARGS}, -#endif +#ifdef Py_DEBUG + {"_protect", (PyCFunction) mmap_protect_method, METH_VARARGS}, +#endif // Py_DEBUG +#endif // MS_WINDOWS {NULL, NULL} /* sentinel */ }; @@ -968,7 +1233,12 @@ mmap_item(mmap_object *self, Py_ssize_t i) PyErr_SetString(PyExc_IndexError, "mmap index out of range"); return NULL; } - return PyBytes_FromStringAndSize(self->data + i, 1); + + char dest; + if (safe_byte_copy(&dest, self->data + i) < 0) { + return NULL; + } + return PyBytes_FromStringAndSize(&dest, 1); } static PyObject * @@ -987,7 +1257,12 @@ mmap_subscript(mmap_object *self, PyObject *item) return NULL; } CHECK_VALID(NULL); - return PyLong_FromLong(Py_CHARMASK(self->data[i])); + + char dest; + if (safe_byte_copy(&dest, self->data + i) < 0) { + return NULL; + } + return PyLong_FromLong(Py_CHARMASK(dest)); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; @@ -999,25 +1274,24 @@ mmap_subscript(mmap_object *self, PyObject *item) CHECK_VALID(NULL); if (slicelen <= 0) - return PyBytes_FromStringAndSize("", 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); else if (step == 1) - return PyBytes_FromStringAndSize(self->data + start, - slicelen); + return _safe_PyBytes_FromStringAndSize(self->data + start, slicelen); else { char *result_buf = (char *)PyMem_Malloc(slicelen); - size_t cur; - Py_ssize_t i; PyObject *result; if (result_buf == NULL) return PyErr_NoMemory(); - for (cur = start, i = 0; i < slicelen; - cur += step, i++) { - result_buf[i] = self->data[cur]; + if (safe_copy_to_slice(result_buf, self->data, start, step, + slicelen) < 0) + { + result = NULL; + } + else { + result = PyBytes_FromStringAndSize(result_buf, slicelen); } - result = PyBytes_FromStringAndSize(result_buf, - slicelen); PyMem_Free(result_buf); return result; } @@ -1052,7 +1326,10 @@ mmap_ass_item(mmap_object *self, Py_ssize_t i, PyObject *v) if (!is_writable(self)) return -1; buf = PyBytes_AsString(v); - self->data[i] = buf[0]; + + if (safe_byte_copy(self->data + i, buf) < 0) { + return -1; + } return 0; } @@ -1097,7 +1374,11 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value) return -1; } CHECK_VALID(-1); - self->data[i] = (char) v; + + char v_char = (char) v; + if (safe_byte_copy(self->data + i, &v_char) < 0) { + return -1; + } return 0; } else if (PySlice_Check(item)) { @@ -1123,24 +1404,23 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value) } CHECK_VALID_OR_RELEASE(-1, vbuf); + int result = 0; if (slicelen == 0) { } else if (step == 1) { - memcpy(self->data + start, vbuf.buf, slicelen); + if (safe_memcpy(self->data + start, vbuf.buf, slicelen) < 0) { + result = -1; + } } else { - size_t cur; - Py_ssize_t i; - - for (cur = start, i = 0; - i < slicelen; - cur += step, i++) + if (safe_copy_from_slice(self->data, (char *)vbuf.buf, start, step, + slicelen) < 0) { - self->data[cur] = ((char *)vbuf.buf)[i]; + result = -1; } } PyBuffer_Release(&vbuf); - return 0; + return result; } else { PyErr_SetString(PyExc_TypeError, @@ -1801,6 +2081,7 @@ mmap_exec(PyObject *module) static PyModuleDef_Slot mmap_slots[] = { {Py_mod_exec, mmap_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/overlapped.c b/Modules/overlapped.c index fd40e91d0f50c4..308a0dab7fab1a 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -723,6 +723,24 @@ Overlapped_dealloc(OverlappedObject *self) if (!HasOverlappedIoCompleted(&self->overlapped) && self->type != TYPE_NOT_STARTED) { + // NOTE: We should not get here, if we do then something is wrong in + // the IocpProactor or ProactorEventLoop. Since everything uses IOCP if + // the overlapped IO hasn't completed yet then we should not be + // deallocating! + // + // The problem is likely that this OverlappedObject was removed from + // the IocpProactor._cache before it was complete. The _cache holds a + // reference while IO is pending so that it does not get deallocated + // while the kernel has retained the OVERLAPPED structure. + // + // CancelIoEx (likely called from self.cancel()) may have successfully + // completed, but the OVERLAPPED is still in use until either + // HasOverlappedIoCompleted() is true or GetQueuedCompletionStatus has + // returned this OVERLAPPED object. + // + // NOTE: Waiting when IOCP is in use can hang indefinitely, but this + // CancelIoEx is superfluous in that self.cancel() was already called, + // so I've only ever seen this return FALSE with GLE=ERROR_NOT_FOUND Py_BEGIN_ALLOW_THREADS if (CancelIoEx(self->handle, &self->overlapped)) wait = TRUE; @@ -905,7 +923,7 @@ _overlapped_Overlapped_getresult_impl(OverlappedObject *self, BOOL wait) { break; } - /* fall through */ + _Py_FALLTHROUGH; default: return SetFromWindowsErr(err); } @@ -2038,6 +2056,7 @@ overlapped_exec(PyObject *module) WINAPI_CONSTANT(F_DWORD, ERROR_OPERATION_ABORTED); WINAPI_CONSTANT(F_DWORD, ERROR_SEM_TIMEOUT); WINAPI_CONSTANT(F_DWORD, ERROR_PIPE_BUSY); + WINAPI_CONSTANT(F_DWORD, ERROR_PORT_UNREACHABLE); WINAPI_CONSTANT(F_DWORD, INFINITE); WINAPI_CONSTANT(F_HANDLE, INVALID_HANDLE_VALUE); WINAPI_CONSTANT(F_HANDLE, NULL); @@ -2051,6 +2070,7 @@ overlapped_exec(PyObject *module) static PyModuleDef_Slot overlapped_slots[] = { {Py_mod_exec, overlapped_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index fd70b38bddec79..a35a848a7ca4b8 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -16,14 +16,15 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_ReInitThreads() #include "pycore_fileutils.h" // _Py_closerange() -#include "pycore_import.h" // _PyImport_ReInitLock() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() +#include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_object.h" // _PyObject_LookupSpecial() #include "pycore_pylifecycle.h" // _PyOS_URandom() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_signal.h" // Py_NSIG #include "pycore_time.h" // _PyLong_FromTime_t() +#include "pycore_typeobject.h" // _PyType_AddMethod() #ifdef HAVE_UNISTD_H # include // symlink() @@ -37,6 +38,8 @@ # include # include // UNLEN # include "osdefs.h" // SEP +# include // SetEntriesInAcl +# include // SDDL_REVISION_1 # if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) # define HAVE_SYMLINK # endif /* MS_WINDOWS_DESKTOP | MS_WINDOWS_SYSTEM */ @@ -70,6 +73,9 @@ #ifdef HAVE_SYS_TIME_H # include // futimes() #endif +#ifdef HAVE_SYS_PIDFD_H +# include // PIDFD_NONBLOCK +#endif // SGI apparently needs this forward declaration @@ -78,6 +84,9 @@ extern char * _getpty(int *, int, mode_t, int); #endif +#ifdef __EMSCRIPTEN__ +#include "emscripten.h" // emscripten_debugger() +#endif /* * A number of APIs are available on macOS from a certain macOS version. @@ -123,6 +132,7 @@ # define HAVE_PWRITEV_RUNTIME __builtin_available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) # define HAVE_MKFIFOAT_RUNTIME __builtin_available(macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) # define HAVE_MKNODAT_RUNTIME __builtin_available(macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) +# define HAVE_PTSNAME_R_RUNTIME __builtin_available(macOS 10.13.4, iOS 11.3, tvOS 11.3, watchOS 4.3, *) # define HAVE_POSIX_SPAWN_SETSID_RUNTIME __builtin_available(macOS 10.15, *) @@ -204,6 +214,10 @@ # define HAVE_MKNODAT_RUNTIME (mknodat != NULL) # endif +# ifdef HAVE_PTSNAME_R +# define HAVE_PTSNAME_R_RUNTIME (ptsname_r != NULL) +# endif + #endif #ifdef HAVE_FUTIMESAT @@ -229,6 +243,7 @@ # define HAVE_PWRITEV_RUNTIME 1 # define HAVE_MKFIFOAT_RUNTIME 1 # define HAVE_MKNODAT_RUNTIME 1 +# define HAVE_PTSNAME_R_RUNTIME 1 #endif @@ -296,6 +311,10 @@ corresponding Unix manual entries for more information on calls."); # include #endif +#ifdef HAVE_LINUX_SCHED_H +# include +#endif + #if !defined(CPU_ALLOC) && defined(HAVE_SCHED_SETAFFINITY) # undef HAVE_SCHED_SETAFFINITY #endif @@ -613,16 +632,18 @@ PyOS_BeforeFork(void) run_at_forkers(interp->before_forkers, 1); _PyImport_AcquireLock(interp); + _PyEval_StopTheWorldAll(&_PyRuntime); + HEAD_LOCK(&_PyRuntime); } void PyOS_AfterFork_Parent(void) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - if (_PyImport_ReleaseLock(interp) <= 0) { - Py_FatalError("failed releasing import lock after fork"); - } + HEAD_UNLOCK(&_PyRuntime); + _PyEval_StartTheWorldAll(&_PyRuntime); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyImport_ReleaseLock(interp); run_at_forkers(interp->after_forkers_parent, 0); } @@ -632,6 +653,7 @@ PyOS_AfterFork_Child(void) PyStatus status; _PyRuntimeState *runtime = &_PyRuntime; + // re-creates runtime->interpreters.mutex (HEAD_UNLOCK) status = _PyRuntimeState_ReInitThreads(runtime); if (_PyStatus_EXCEPTION(status)) { goto fatal_error; @@ -640,6 +662,7 @@ PyOS_AfterFork_Child(void) PyThreadState *tstate = _PyThreadState_GET(); _Py_EnsureTstateNotNULL(tstate); + assert(tstate->thread_id == PyThread_get_thread_ident()); #ifdef PY_HAVE_THREAD_NATIVE_ID tstate->native_thread_id = PyThread_get_thread_native_id(); #endif @@ -649,15 +672,24 @@ PyOS_AfterFork_Child(void) _Py_qsbr_after_fork((_PyThreadStateImpl *)tstate); #endif + // Ideally we could guarantee tstate is running main. + _PyInterpreterState_ReinitRunningMain(tstate); + status = _PyEval_ReInitThreads(tstate); if (_PyStatus_EXCEPTION(status)) { goto fatal_error; } - status = _PyImport_ReInitLock(tstate->interp); - if (_PyStatus_EXCEPTION(status)) { - goto fatal_error; - } + // Remove the dead thread states. We "start the world" once we are the only + // thread state left to undo the stop the world call in `PyOS_BeforeFork`. + // That needs to happen before `_PyThreadState_DeleteList`, because that + // may call destructors. + PyThreadState *list = _PyThreadState_RemoveExcept(tstate); + _PyEval_StartTheWorldAll(&_PyRuntime); + _PyThreadState_DeleteList(list); + + _PyImport_ReInitLock(tstate->interp); + _PyImport_ReleaseLock(tstate->interp); _PySignal_AfterFork(); @@ -947,16 +979,46 @@ _Py_Gid_Converter(PyObject *obj, gid_t *p) #endif /* MS_WINDOWS */ -#define _PyLong_FromDev PyLong_FromLongLong +static PyObject * +_PyLong_FromDev(dev_t dev) +{ +#ifdef NODEV + if (dev == NODEV) { + return PyLong_FromLongLong((long long)dev); + } +#endif + return PyLong_FromUnsignedLongLong((unsigned long long)dev); +} #if (defined(HAVE_MKNOD) && defined(HAVE_MAKEDEV)) || defined(HAVE_DEVICE_MACROS) static int _Py_Dev_Converter(PyObject *obj, void *p) { - *((dev_t *)p) = PyLong_AsUnsignedLongLong(obj); - if (PyErr_Occurred()) +#ifdef NODEV + if (PyLong_Check(obj) && _PyLong_IsNegative((PyLongObject *)obj)) { + int overflow; + long long result = PyLong_AsLongLongAndOverflow(obj, &overflow); + if (result == -1 && PyErr_Occurred()) { + return 0; + } + if (!overflow && result == (long long)NODEV) { + *((dev_t *)p) = NODEV; + return 1; + } + } +#endif + + unsigned long long result = PyLong_AsUnsignedLongLong(obj); + if (result == (unsigned long long)-1 && PyErr_Occurred()) { + return 0; + } + if ((unsigned long long)(dev_t)result != result) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C dev_t"); return 0; + } + *((dev_t *)p) = (dev_t)result; return 1; } #endif /* (HAVE_MKNOD && HAVE_MAKEDEV) || HAVE_DEVICE_MACROS */ @@ -1072,16 +1134,15 @@ get_posix_state(PyObject *module) * * path_converter accepts (Unicode) strings and their * subclasses, and bytes and their subclasses. What - * it does with the argument depends on the platform: + * it does with the argument depends on path.make_wide: * - * * On Windows, if we get a (Unicode) string we - * extract the wchar_t * and return it; if we get - * bytes we decode to wchar_t * and return that. + * * If path.make_wide is nonzero, if we get a (Unicode) + * string we extract the wchar_t * and return it; if we + * get bytes we decode to wchar_t * and return that. * - * * On all other platforms, strings are encoded - * to bytes using PyUnicode_FSConverter, then we - * extract the char * from the bytes object and - * return that. + * * If path.make_wide is zero, if we get bytes we extract + * the char_t * and return it; if we get a (Unicode) + * string we encode to char_t * and return that. * * path_converter also optionally accepts signed * integers (representing open file descriptors) instead @@ -1090,6 +1151,15 @@ get_posix_state(PyObject *module) * Input fields: * path.nullable * If nonzero, the path is permitted to be None. + * path.nonstrict + * If nonzero, the path is permitted to contain + * embedded null characters and have any length. + * path.make_wide + * If nonzero, the converter always uses wide, decoding if necessary, else + * it always uses narrow, encoding if necessary. The default value is + * nonzero on Windows, else zero. + * path.suppress_value_error + * If nonzero, raising ValueError is suppressed. * path.allow_fd * If nonzero, the path is permitted to be a file handle * (a signed int) instead of a string. @@ -1105,12 +1175,10 @@ get_posix_state(PyObject *module) * Output fields: * path.wide * Points to the path if it was expressed as Unicode - * and was not encoded. (Only used on Windows.) + * or if it was bytes and decoded to Unicode. * path.narrow * Points to the path if it was expressed as bytes, - * or it was Unicode and was encoded to bytes. (On Windows, - * is a non-zero integer if the path was expressed as bytes. - * The type is deliberately incompatible to prevent misuse.) + * or if it was Unicode and encoded to bytes. * path.fd * Contains a file descriptor if path.accept_fd was true * and the caller provided a signed integer instead of any @@ -1120,6 +1188,9 @@ get_posix_state(PyObject *module) * unspecified, path_converter will never get called. * So if you set allow_fd, you *MUST* initialize path.fd = -1 * yourself! + * path.value_error + * If nonzero, then suppress_value_error was specified and a ValueError + * occurred. * path.length * The length of the path in characters, if specified as * a string. @@ -1152,28 +1223,38 @@ get_posix_state(PyObject *module) * path_cleanup(). However it is safe to do so.) */ typedef struct { + // Input fields const char *function_name; const char *argument_name; int nullable; + int nonstrict; + int make_wide; + int suppress_value_error; int allow_fd; + // Output fields const wchar_t *wide; -#ifdef MS_WINDOWS - BOOL narrow; -#else const char *narrow; -#endif int fd; + int value_error; Py_ssize_t length; PyObject *object; PyObject *cleanup; } path_t; +#define PATH_T_INITIALIZE(function_name, argument_name, nullable, nonstrict, \ + make_wide, suppress_value_error, allow_fd) \ + {function_name, argument_name, nullable, nonstrict, make_wide, \ + suppress_value_error, allow_fd, NULL, NULL, -1, 0, 0, NULL, NULL} #ifdef MS_WINDOWS -#define PATH_T_INITIALIZE(function_name, argument_name, nullable, allow_fd) \ - {function_name, argument_name, nullable, allow_fd, NULL, FALSE, -1, 0, NULL, NULL} +#define PATH_T_INITIALIZE_P(function_name, argument_name, nullable, \ + nonstrict, suppress_value_error, allow_fd) \ + PATH_T_INITIALIZE(function_name, argument_name, nullable, nonstrict, 1, \ + suppress_value_error, allow_fd) #else -#define PATH_T_INITIALIZE(function_name, argument_name, nullable, allow_fd) \ - {function_name, argument_name, nullable, allow_fd, NULL, NULL, -1, 0, NULL, NULL} +#define PATH_T_INITIALIZE_P(function_name, argument_name, nullable, \ + nonstrict, suppress_value_error, allow_fd) \ + PATH_T_INITIALIZE(function_name, argument_name, nullable, nonstrict, 0, \ + suppress_value_error, allow_fd) #endif static void @@ -1194,10 +1275,8 @@ path_converter(PyObject *o, void *p) Py_ssize_t length = 0; int is_index, is_bytes, is_unicode; const char *narrow; -#ifdef MS_WINDOWS PyObject *wo = NULL; wchar_t *wide = NULL; -#endif #define FORMAT_EXCEPTION(exc, fmt) \ PyErr_Format(exc, "%s%s" fmt, \ @@ -1218,11 +1297,7 @@ path_converter(PyObject *o, void *p) if ((o == Py_None) && path->nullable) { path->wide = NULL; -#ifdef MS_WINDOWS - path->narrow = FALSE; -#else path->narrow = NULL; -#endif path->fd = -1; goto success_exit; } @@ -1266,30 +1341,33 @@ path_converter(PyObject *o, void *p) } if (is_unicode) { + if (path->make_wide) { + wide = PyUnicode_AsWideCharString(o, &length); + if (!wide) { + goto error_exit; + } #ifdef MS_WINDOWS - wide = PyUnicode_AsWideCharString(o, &length); - if (!wide) { - goto error_exit; - } - if (length > 32767) { - FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); - goto error_exit; - } - if (wcslen(wide) != length) { - FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character in %s"); - goto error_exit; - } + if (!path->nonstrict && length > 32767) { + FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); + goto error_exit; + } +#endif + if (!path->nonstrict && wcslen(wide) != (size_t)length) { + FORMAT_EXCEPTION(PyExc_ValueError, + "embedded null character in %s"); + goto error_exit; + } - path->wide = wide; - path->narrow = FALSE; - path->fd = -1; - wide = NULL; - goto success_exit; -#else - if (!PyUnicode_FSConverter(o, &bytes)) { + path->wide = wide; + path->narrow = NULL; + path->fd = -1; + wide = NULL; + goto success_exit; + } + bytes = PyUnicode_EncodeFSDefault(o); + if (!bytes) { goto error_exit; } -#endif } else if (is_bytes) { bytes = Py_NewRef(o); @@ -1299,11 +1377,7 @@ path_converter(PyObject *o, void *p) goto error_exit; } path->wide = NULL; -#ifdef MS_WINDOWS - path->narrow = FALSE; -#else path->narrow = NULL; -#endif goto success_exit; } else { @@ -1323,52 +1397,54 @@ path_converter(PyObject *o, void *p) length = PyBytes_GET_SIZE(bytes); narrow = PyBytes_AS_STRING(bytes); - if ((size_t)length != strlen(narrow)) { + if (!path->nonstrict && strlen(narrow) != (size_t)length) { FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character in %s"); goto error_exit; } -#ifdef MS_WINDOWS - wo = PyUnicode_DecodeFSDefaultAndSize( - narrow, - length - ); - if (!wo) { - goto error_exit; - } + if (path->make_wide) { + wo = PyUnicode_DecodeFSDefaultAndSize(narrow, length); + if (!wo) { + goto error_exit; + } - wide = PyUnicode_AsWideCharString(wo, &length); - Py_DECREF(wo); - if (!wide) { - goto error_exit; - } - if (length > 32767) { - FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); - goto error_exit; - } - if (wcslen(wide) != length) { - FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character in %s"); - goto error_exit; - } - path->wide = wide; - path->narrow = TRUE; - Py_DECREF(bytes); - wide = NULL; -#else - path->wide = NULL; - path->narrow = narrow; - if (bytes == o) { - /* Still a reference owned by path->object, don't have to - worry about path->narrow is used after free. */ + wide = PyUnicode_AsWideCharString(wo, &length); + Py_DECREF(wo); + if (!wide) { + goto error_exit; + } +#ifdef MS_WINDOWS + if (!path->nonstrict && length > 32767) { + FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); + goto error_exit; + } +#endif + if (!path->nonstrict && wcslen(wide) != (size_t)length) { + FORMAT_EXCEPTION(PyExc_ValueError, + "embedded null character in %s"); + goto error_exit; + } + path->wide = wide; + path->narrow = NULL; Py_DECREF(bytes); + wide = NULL; } else { - path->cleanup = bytes; + path->wide = NULL; + path->narrow = narrow; + if (bytes == o) { + /* Still a reference owned by path->object, don't have to + worry about path->narrow is used after free. */ + Py_DECREF(bytes); + } + else { + path->cleanup = bytes; + } } -#endif path->fd = -1; success_exit: + path->value_error = 0; path->length = length; path->object = o; return Py_CLEANUP_SUPPORTED; @@ -1376,10 +1452,20 @@ path_converter(PyObject *o, void *p) error_exit: Py_XDECREF(o); Py_XDECREF(bytes); -#ifdef MS_WINDOWS PyMem_Free(wide); -#endif - return 0; + if (!path->suppress_value_error || + !PyErr_ExceptionMatches(PyExc_ValueError)) + { + return 0; + } + PyErr_Clear(); + path->wide = NULL; + path->narrow = NULL; + path->fd = -1; + path->value_error = 1; + path->length = 0; + path->object = NULL; + return Py_CLEANUP_SUPPORTED; } static void @@ -1429,11 +1515,7 @@ follow_symlinks_specified(const char *function_name, int follow_symlinks) static int path_and_dir_fd_invalid(const char *function_name, path_t *path, int dir_fd) { - if (!path->wide && (dir_fd != DEFAULT_DIR_FD) -#ifndef MS_WINDOWS - && !path->narrow -#endif - ) { + if (!path->wide && (dir_fd != DEFAULT_DIR_FD) && !path->narrow) { PyErr_Format(PyExc_ValueError, "%s: can't specify dir_fd without matching path", function_name); @@ -1480,6 +1562,19 @@ dir_fd_and_follow_symlinks_invalid(const char *function_name, int dir_fd, return 0; } +#if defined(HAVE_WAITID) +static int +idtype_t_converter(PyObject *arg, void *addr) +{ + int value = PyLong_AsInt(arg); + if (value == -1 && PyErr_Occurred()) { + return 0; + } + *((idtype_t *)addr) = (idtype_t)(value); + return 1; +} +#endif + #ifdef MS_WINDOWS typedef long long Py_off_t; #else @@ -2893,7 +2988,9 @@ class path_t_converter(CConverter): converter = 'path_converter' - def converter_init(self, *, allow_fd=False, nullable=False): + def converter_init(self, *, allow_fd=False, make_wide=None, + nonstrict=False, nullable=False, + suppress_value_error=False): # right now path_t doesn't support default values. # to support a default value, you'll need to override initialize(). if self.default not in (unspecified, None): @@ -2903,6 +3000,9 @@ class path_t_converter(CConverter): raise RuntimeError("Can't specify a c_default to the path_t converter!") self.nullable = nullable + self.nonstrict = nonstrict + self.make_wide = make_wide + self.suppress_value_error = suppress_value_error self.allow_fd = allow_fd def pre_render(self): @@ -2912,11 +3012,24 @@ class path_t_converter(CConverter): return str(int(bool(value))) # add self.py_name here when merging with posixmodule conversion - self.c_default = 'PATH_T_INITIALIZE("{}", "{}", {}, {})'.format( - self.function.name, - self.name, - strify(self.nullable), - strify(self.allow_fd), + if self.make_wide is None: + self.c_default = 'PATH_T_INITIALIZE_P("{}", "{}", {}, {}, {}, {})'.format( + self.function.name, + self.name, + strify(self.nullable), + strify(self.nonstrict), + strify(self.suppress_value_error), + strify(self.allow_fd), + ) + else: + self.c_default = 'PATH_T_INITIALIZE("{}", "{}", {}, {}, {}, {}, {})'.format( + self.function.name, + self.name, + strify(self.nullable), + strify(self.nonstrict), + strify(self.make_wide), + strify(self.suppress_value_error), + strify(self.allow_fd), ) def cleanup(self): @@ -2966,17 +3079,42 @@ class pid_t_converter(CConverter): type = 'pid_t' format_unit = '" _Py_PARSE_PID "' -class idtype_t_converter(int_converter): + def parse_arg(self, argname, displayname, *, limited_capi): + return self.format_code(""" + {paramname} = PyLong_AsPid({argname}); + if ({paramname} == (pid_t)(-1) && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, argname=argname) + +class idtype_t_converter(CConverter): type = 'idtype_t' + converter = 'idtype_t_converter' class id_t_converter(CConverter): type = 'id_t' format_unit = '" _Py_PARSE_PID "' + def parse_arg(self, argname, displayname, *, limited_capi): + return self.format_code(""" + {paramname} = (id_t)PyLong_AsPid({argname}); + if ({paramname} == (id_t)(-1) && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, argname=argname) + class intptr_t_converter(CConverter): type = 'intptr_t' format_unit = '" _Py_PARSE_INTPTR "' + def parse_arg(self, argname, displayname, *, limited_capi): + return self.format_code(""" + {paramname} = (intptr_t)PyLong_AsVoidPtr({argname}); + if (!{paramname} && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """, argname=argname) + class Py_off_t_converter(CConverter): type = 'Py_off_t' converter = 'Py_off_t_converter' @@ -2985,18 +3123,22 @@ class Py_off_t_return_converter(long_return_converter): type = 'Py_off_t' conversion_fn = 'PyLong_FromPy_off_t' -class path_confname_converter(CConverter): +class confname_converter(CConverter): type="int" - converter="conv_path_confname" + converter="conv_confname" -class confstr_confname_converter(path_confname_converter): - converter='conv_confstr_confname' + def converter_init(self, *, table): + self.table = table -class sysconf_confname_converter(path_confname_converter): - converter="conv_sysconf_confname" + def parse_arg(self, argname, displayname, *, limited_capi): + return self.format_code(""" + if (!{converter}(module, {argname}, &{paramname}, "{table}")) {{{{ + goto exit; + }}}} + """, argname=argname, converter=self.converter, table=self.table) [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=3338733161aa7879]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=8189d5ae78244626]*/ /*[clinic input] @@ -3205,7 +3347,7 @@ os_access_impl(PyObject *module, path_t *path, int mode, int dir_fd, #endif -#ifdef HAVE_TTYNAME +#ifdef HAVE_TTYNAME_R /*[clinic input] os.ttyname @@ -4088,6 +4230,20 @@ posix_getcwd(int use_bytes) else { obj = PyUnicode_DecodeFSDefault(buf); } +#ifdef __linux__ + if (buf[0] != '/') { + /* + * On Linux >= 2.6.36 with glibc < 2.27, getcwd() can return a + * relative pathname starting with '(unreachable)'. We detect this + * and fail with ENOENT, matching newer glibc behaviour. + */ + errno = ENOENT; + path_object_error(obj); + PyMem_RawFree(buf); + return NULL; + } +#endif + assert(buf[0] == '/'); PyMem_RawFree(buf); return obj; @@ -4251,7 +4407,7 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) { PyObject *v; HANDLE hFindFile = INVALID_HANDLE_VALUE; - BOOL result; + BOOL result, return_bytes; wchar_t namebuf[MAX_PATH+4]; /* Overallocate for "\*.*" */ /* only claim to have space for MAX_PATH */ Py_ssize_t len = Py_ARRAY_LENGTH(namebuf)-4; @@ -4263,9 +4419,11 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) if (!path->wide) { /* Default arg: "." */ po_wchars = L"."; len = 1; + return_bytes = 0; } else { po_wchars = path->wide; len = wcslen(path->wide); + return_bytes = PyBytes_Check(path->object); } /* The +5 is so we can append "\\*.*\0" */ wnamebuf = PyMem_New(wchar_t, len + 5); @@ -4300,7 +4458,7 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) wcscmp(wFileData.cFileName, L"..") != 0) { v = PyUnicode_FromWideChar(wFileData.cFileName, wcslen(wFileData.cFileName)); - if (path->narrow && v) { + if (return_bytes && v) { Py_SETREF(v, PyUnicode_EncodeFSDefault(v)); } if (v == NULL) { @@ -4843,7 +5001,7 @@ os__getfullpathname_impl(PyObject *module, path_t *path) if (str == NULL) { return NULL; } - if (path->narrow) { + if (PyBytes_Check(path->object)) { Py_SETREF(str, PyUnicode_EncodeFSDefault(str)); } return str; @@ -4916,7 +5074,7 @@ os__getfinalpathname_impl(PyObject *module, path_t *path) } result = PyUnicode_FromWideChar(target_path, result_length); - if (result && path->narrow) { + if (result && PyBytes_Check(path->object)) { Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); } @@ -4999,7 +5157,7 @@ os__getvolumepathname_impl(PyObject *module, path_t *path) goto exit; } result = PyUnicode_FromWideChar(mountpath, wcslen(mountpath)); - if (path->narrow) + if (PyBytes_Check(path->object)) Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); exit: @@ -5054,413 +5212,426 @@ os__path_splitroot_impl(PyObject *module, path_t *path) } -/*[clinic input] -os._path_isdir - - s: 'O' +#define PY_IFREG 1 // Regular file +#define PY_IFDIR 2 // Directory +#define PY_IFLNK 4 // Symlink +#define PY_IFMNT 8 // Mount Point (junction) +#define PY_IFLRP 16 // Link Reparse Point (name-surrogate, symlink, junction) +#define PY_IFRRP 32 // Regular Reparse Point -Return true if the pathname refers to an existing directory. +static inline BOOL +_testInfo(DWORD attributes, DWORD reparseTag, BOOL diskDevice, int testedType) +{ + switch (testedType) { + case PY_IFREG: + return diskDevice && attributes && + !(attributes & FILE_ATTRIBUTE_DIRECTORY); + case PY_IFDIR: + return attributes & FILE_ATTRIBUTE_DIRECTORY; + case PY_IFLNK: + return (attributes & FILE_ATTRIBUTE_REPARSE_POINT) && + reparseTag == IO_REPARSE_TAG_SYMLINK; + case PY_IFMNT: + return (attributes & FILE_ATTRIBUTE_REPARSE_POINT) && + reparseTag == IO_REPARSE_TAG_MOUNT_POINT; + case PY_IFLRP: + return (attributes & FILE_ATTRIBUTE_REPARSE_POINT) && + IsReparseTagNameSurrogate(reparseTag); + case PY_IFRRP: + return (attributes & FILE_ATTRIBUTE_REPARSE_POINT) && + reparseTag && !IsReparseTagNameSurrogate(reparseTag); + } -[clinic start generated code]*/ + return FALSE; +} -static PyObject * -os__path_isdir_impl(PyObject *module, PyObject *s) -/*[clinic end generated code: output=9d87ab3c8b8a4e61 input=c17f7ef21d22d64e]*/ +static BOOL +_testFileTypeByHandle(HANDLE hfile, int testedType, BOOL diskOnly) { - HANDLE hfile; - BOOL close_file = TRUE; + assert(testedType == PY_IFREG || testedType == PY_IFDIR || + testedType == PY_IFLNK || testedType == PY_IFMNT || + testedType == PY_IFLRP || testedType == PY_IFRRP); + + BOOL diskDevice = GetFileType(hfile) == FILE_TYPE_DISK; + if (diskOnly && !diskDevice) { + return FALSE; + } + if (testedType != PY_IFREG && testedType != PY_IFDIR) { + FILE_ATTRIBUTE_TAG_INFO info; + return GetFileInformationByHandleEx(hfile, FileAttributeTagInfo, &info, + sizeof(info)) && + _testInfo(info.FileAttributes, info.ReparseTag, diskDevice, + testedType); + } FILE_BASIC_INFO info; - path_t _path = PATH_T_INITIALIZE("isdir", "s", 0, 1); - int result; - BOOL slow_path = TRUE; - FILE_STAT_BASIC_INFORMATION statInfo; + return GetFileInformationByHandleEx(hfile, FileBasicInfo, &info, + sizeof(info)) && + _testInfo(info.FileAttributes, 0, diskDevice, testedType); +} - if (!path_converter(s, &_path)) { - path_cleanup(&_path); - if (PyErr_ExceptionMatches(PyExc_ValueError)) { - PyErr_Clear(); - Py_RETURN_FALSE; +static BOOL +_testFileTypeByName(LPCWSTR path, int testedType) +{ + assert(testedType == PY_IFREG || testedType == PY_IFDIR || + testedType == PY_IFLNK || testedType == PY_IFMNT || + testedType == PY_IFLRP || testedType == PY_IFRRP); + + FILE_STAT_BASIC_INFORMATION info; + if (_Py_GetFileInformationByName(path, FileStatBasicByNameInfo, &info, + sizeof(info))) + { + BOOL diskDevice = info.DeviceType == FILE_DEVICE_DISK || + info.DeviceType == FILE_DEVICE_VIRTUAL_DISK || + info.DeviceType == FILE_DEVICE_CD_ROM; + BOOL result = _testInfo(info.FileAttributes, info.ReparseTag, + diskDevice, testedType); + if (!result || (testedType != PY_IFREG && testedType != PY_IFDIR) || + !(info.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) + { + return result; } - return NULL; + } + else if (_Py_GetFileInformationByName_ErrorIsTrustworthy( + GetLastError())) + { + return FALSE; } - Py_BEGIN_ALLOW_THREADS - if (_path.wide) { - if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, - &statInfo, sizeof(statInfo))) { - if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { - slow_path = FALSE; - result = statInfo.FileAttributes & FILE_ATTRIBUTE_DIRECTORY; - } else if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_DIRECTORY)) { - slow_path = FALSE; - result = 0; - } - } else if (_Py_GetFileInformationByName_ErrorIsTrustworthy(GetLastError())) { - slow_path = FALSE; - result = 0; - } + DWORD flags = FILE_FLAG_BACKUP_SEMANTICS; + if (testedType != PY_IFREG && testedType != PY_IFDIR) { + flags |= FILE_FLAG_OPEN_REPARSE_POINT; } - if (slow_path) { - if (_path.fd != -1) { - hfile = _Py_get_osfhandle_noraise(_path.fd); - close_file = FALSE; + HANDLE hfile = CreateFileW(path, FILE_READ_ATTRIBUTES, 0, NULL, + OPEN_EXISTING, flags, NULL); + if (hfile != INVALID_HANDLE_VALUE) { + BOOL result = _testFileTypeByHandle(hfile, testedType, FALSE); + CloseHandle(hfile); + return result; + } + + switch (GetLastError()) { + case ERROR_ACCESS_DENIED: + case ERROR_SHARING_VIOLATION: + case ERROR_CANT_ACCESS_FILE: + case ERROR_INVALID_PARAMETER: + int rc; + STRUCT_STAT st; + if (testedType == PY_IFREG || testedType == PY_IFDIR) { + rc = STAT(path, &st); } else { - hfile = CreateFileW(_path.wide, FILE_READ_ATTRIBUTES, 0, NULL, - OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + // PY_IFRRP is not generally supported in this case, except for + // unhandled reparse points such as IO_REPARSE_TAG_APPEXECLINK. + rc = LSTAT(path, &st); } - if (hfile != INVALID_HANDLE_VALUE) { - if (GetFileInformationByHandleEx(hfile, FileBasicInfo, &info, - sizeof(info))) - { - result = info.FileAttributes & FILE_ATTRIBUTE_DIRECTORY; - } - else { - result = 0; - } - if (close_file) { - CloseHandle(hfile); - } - } - else { - STRUCT_STAT st; - switch (GetLastError()) { - case ERROR_ACCESS_DENIED: - case ERROR_SHARING_VIOLATION: - case ERROR_CANT_ACCESS_FILE: - case ERROR_INVALID_PARAMETER: - if (STAT(_path.wide, &st)) { - result = 0; - } - else { - result = S_ISDIR(st.st_mode); - } - break; - default: - result = 0; - } + if (!rc) { + return _testInfo(st.st_file_attributes, st.st_reparse_tag, + st.st_mode & S_IFREG, testedType); } } - Py_END_ALLOW_THREADS - path_cleanup(&_path); - if (result) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + return FALSE; } -/*[clinic input] -os._path_isfile +static BOOL +_testFileExistsByName(LPCWSTR path, BOOL followLinks) +{ + FILE_STAT_BASIC_INFORMATION info; + if (_Py_GetFileInformationByName(path, FileStatBasicByNameInfo, &info, + sizeof(info))) + { + if (!(info.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) || + !followLinks && IsReparseTagNameSurrogate(info.ReparseTag)) + { + return TRUE; + } + } + else if (_Py_GetFileInformationByName_ErrorIsTrustworthy( + GetLastError())) + { + return FALSE; + } + + DWORD flags = FILE_FLAG_BACKUP_SEMANTICS; + if (!followLinks) { + flags |= FILE_FLAG_OPEN_REPARSE_POINT; + } + HANDLE hfile = CreateFileW(path, FILE_READ_ATTRIBUTES, 0, NULL, + OPEN_EXISTING, flags, NULL); + if (hfile != INVALID_HANDLE_VALUE) { + if (followLinks) { + CloseHandle(hfile); + return TRUE; + } + // Regular Reparse Points (PY_IFRRP) have to be traversed. + BOOL result = _testFileTypeByHandle(hfile, PY_IFRRP, FALSE); + CloseHandle(hfile); + if (!result) { + return TRUE; + } + hfile = CreateFileW(path, FILE_READ_ATTRIBUTES, 0, NULL, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hfile != INVALID_HANDLE_VALUE) { + CloseHandle(hfile); + return TRUE; + } + } - path: 'O' + switch (GetLastError()) { + case ERROR_ACCESS_DENIED: + case ERROR_SHARING_VIOLATION: + case ERROR_CANT_ACCESS_FILE: + case ERROR_INVALID_PARAMETER: + STRUCT_STAT _st; + return followLinks ? !STAT(path, &_st): !LSTAT(path, &_st); + } -Test whether a path is a regular file + return FALSE; +} -[clinic start generated code]*/ -static PyObject * -os__path_isfile_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=2394ed7c4b5cfd85 input=de22d74960ade365]*/ +static BOOL +_testFileExists(path_t *path, BOOL followLinks) { - HANDLE hfile; - BOOL close_file = TRUE; - FILE_BASIC_INFO info; - path_t _path = PATH_T_INITIALIZE("isfile", "path", 0, 1); - int result; - BOOL slow_path = TRUE; - FILE_STAT_BASIC_INFORMATION statInfo; - - if (!path_converter(path, &_path)) { - path_cleanup(&_path); - if (PyErr_ExceptionMatches(PyExc_ValueError)) { - PyErr_Clear(); - Py_RETURN_FALSE; - } - return NULL; + BOOL result = FALSE; + if (path->value_error) { + return FALSE; } Py_BEGIN_ALLOW_THREADS - if (_path.wide) { - if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, - &statInfo, sizeof(statInfo))) { - if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { - slow_path = FALSE; - result = !(statInfo.FileAttributes & FILE_ATTRIBUTE_DIRECTORY); - } else if (statInfo.FileAttributes & FILE_ATTRIBUTE_DIRECTORY) { - slow_path = FALSE; - result = 0; + if (path->fd != -1) { + HANDLE hfile = _Py_get_osfhandle_noraise(path->fd); + if (hfile != INVALID_HANDLE_VALUE) { + if (GetFileType(hfile) != FILE_TYPE_UNKNOWN || !GetLastError()) { + result = TRUE; } - } else if (_Py_GetFileInformationByName_ErrorIsTrustworthy(GetLastError())) { - slow_path = FALSE; - result = 0; } } - if (slow_path) { - if (_path.fd != -1) { - hfile = _Py_get_osfhandle_noraise(_path.fd); - close_file = FALSE; - } - else { - hfile = CreateFileW(_path.wide, FILE_READ_ATTRIBUTES, 0, NULL, - OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); - } + else if (path->wide) { + result = _testFileExistsByName(path->wide, followLinks); + } + Py_END_ALLOW_THREADS + + return result; +} + + +static BOOL +_testFileType(path_t *path, int testedType) +{ + BOOL result = FALSE; + if (path->value_error) { + return FALSE; + } + + Py_BEGIN_ALLOW_THREADS + if (path->fd != -1) { + HANDLE hfile = _Py_get_osfhandle_noraise(path->fd); if (hfile != INVALID_HANDLE_VALUE) { - if (GetFileInformationByHandleEx(hfile, FileBasicInfo, &info, - sizeof(info))) - { - result = !(info.FileAttributes & FILE_ATTRIBUTE_DIRECTORY); - } - else { - result = 0; - } - if (close_file) { - CloseHandle(hfile); - } - } - else { - STRUCT_STAT st; - switch (GetLastError()) { - case ERROR_ACCESS_DENIED: - case ERROR_SHARING_VIOLATION: - case ERROR_CANT_ACCESS_FILE: - case ERROR_INVALID_PARAMETER: - if (STAT(_path.wide, &st)) { - result = 0; - } - else { - result = S_ISREG(st.st_mode); - } - break; - default: - result = 0; - } + result = _testFileTypeByHandle(hfile, testedType, TRUE); } } + else if (path->wide) { + result = _testFileTypeByName(path->wide, testedType); + } Py_END_ALLOW_THREADS - path_cleanup(&_path); - if (result) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + return result; } /*[clinic input] -os._path_exists +os._path_exists -> bool - path: 'O' + path: path_t(allow_fd=True, suppress_value_error=True) -Test whether a path exists. Returns False for broken symbolic links +Test whether a path exists. Returns False for broken symbolic links. [clinic start generated code]*/ -static PyObject * -os__path_exists_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=f508c3b35e13a249 input=380f77cdfa0f7ae8]*/ +static int +os__path_exists_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=8da13acf666e16ba input=142beabfc66783eb]*/ { - HANDLE hfile; - BOOL close_file = TRUE; - path_t _path = PATH_T_INITIALIZE("exists", "path", 0, 1); - int result; - BOOL slow_path = TRUE; - FILE_STAT_BASIC_INFORMATION statInfo; + return _testFileExists(path, TRUE); +} - if (!path_converter(path, &_path)) { - path_cleanup(&_path); - if (PyErr_ExceptionMatches(PyExc_ValueError)) { - PyErr_Clear(); - Py_RETURN_FALSE; - } - return NULL; - } - Py_BEGIN_ALLOW_THREADS - if (_path.wide) { - if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, - &statInfo, sizeof(statInfo))) { - if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { - slow_path = FALSE; - result = 1; - } - } else if (_Py_GetFileInformationByName_ErrorIsTrustworthy(GetLastError())) { - slow_path = FALSE; - result = 0; - } - } - if (slow_path) { - if (_path.fd != -1) { - hfile = _Py_get_osfhandle_noraise(_path.fd); - close_file = FALSE; - } - else { - hfile = CreateFileW(_path.wide, FILE_READ_ATTRIBUTES, 0, NULL, - OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); - } - if (hfile != INVALID_HANDLE_VALUE) { - result = 1; - if (close_file) { - CloseHandle(hfile); - } - } - else { - STRUCT_STAT st; - switch (GetLastError()) { - case ERROR_ACCESS_DENIED: - case ERROR_SHARING_VIOLATION: - case ERROR_CANT_ACCESS_FILE: - case ERROR_INVALID_PARAMETER: - if (STAT(_path.wide, &st)) { - result = 0; - } - else { - result = 1; - } - break; - default: - result = 0; - } - } - } - Py_END_ALLOW_THREADS +/*[clinic input] +os._path_lexists -> bool - path_cleanup(&_path); - if (result) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + path: path_t(allow_fd=True, suppress_value_error=True) + +Test whether a path exists. Returns True for broken symbolic links. + +[clinic start generated code]*/ + +static int +os__path_lexists_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=e7240ed5fc45bff3 input=208205112a3cc1ed]*/ +{ + return _testFileExists(path, FALSE); +} + + +/*[clinic input] +os._path_isdir -> bool + + s as path: path_t(allow_fd=True, suppress_value_error=True) + +Return true if the pathname refers to an existing directory. + +[clinic start generated code]*/ + +static int +os__path_isdir_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=d5786196f9e2fa7a input=132a3b5301aecf79]*/ +{ + return _testFileType(path, PY_IFDIR); +} + + +/*[clinic input] +os._path_isfile -> bool + + path: path_t(allow_fd=True, suppress_value_error=True) + +Test whether a path is a regular file + +[clinic start generated code]*/ + +static int +os__path_isfile_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=5c3073bc212b9863 input=4ac1fd350b30a39e]*/ +{ + return _testFileType(path, PY_IFREG); } /*[clinic input] -os._path_islink +os._path_islink -> bool - path: 'O' + path: path_t(allow_fd=True, suppress_value_error=True) Test whether a path is a symbolic link [clinic start generated code]*/ +static int +os__path_islink_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=30da7bda8296adcc input=7510ce05b547debb]*/ +{ + return _testFileType(path, PY_IFLNK); +} + + +/*[clinic input] +os._path_isjunction -> bool + + path: path_t(allow_fd=True, suppress_value_error=True) + +Test whether a path is a junction + +[clinic start generated code]*/ + +static int +os__path_isjunction_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=e1d17a9dd18a9945 input=7dcb8bc4e972fcaf]*/ +{ + return _testFileType(path, PY_IFMNT); +} + +#undef PY_IFREG +#undef PY_IFDIR +#undef PY_IFLNK +#undef PY_IFMNT +#undef PY_IFLRP +#undef PY_IFRRP + +#endif /* MS_WINDOWS */ + + +/*[clinic input] +os._path_splitroot_ex + + p as path: path_t(make_wide=True, nonstrict=True) + +Split a pathname into drive, root and tail. + +The tail contains anything after the root. +[clinic start generated code]*/ + static PyObject * -os__path_islink_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=6d8640b1a390c054 input=38a3cb937ccf59bf]*/ +os__path_splitroot_ex_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=4b0072b6cdf4b611 input=4556b615c7cc13f2]*/ { - HANDLE hfile; - BOOL close_file = TRUE; - FILE_ATTRIBUTE_TAG_INFO info; - path_t _path = PATH_T_INITIALIZE("islink", "path", 0, 1); - int result; - BOOL slow_path = TRUE; - FILE_STAT_BASIC_INFORMATION statInfo; + Py_ssize_t drvsize, rootsize; + PyObject *drv = NULL, *root = NULL, *tail = NULL, *result = NULL; - if (!path_converter(path, &_path)) { - path_cleanup(&_path); - if (PyErr_ExceptionMatches(PyExc_ValueError)) { - PyErr_Clear(); - Py_RETURN_FALSE; - } - return NULL; + const wchar_t *buffer = path->wide; + _Py_skiproot(buffer, path->length, &drvsize, &rootsize); + drv = PyUnicode_FromWideChar(buffer, drvsize); + if (drv == NULL) { + goto exit; } - - Py_BEGIN_ALLOW_THREADS - if (_path.wide) { - if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, - &statInfo, sizeof(statInfo))) { - slow_path = FALSE; - if (statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { - result = (statInfo.ReparseTag == IO_REPARSE_TAG_SYMLINK); - } - else { - result = 0; - } - } else if (_Py_GetFileInformationByName_ErrorIsTrustworthy(GetLastError())) { - slow_path = FALSE; - result = 0; - } + root = PyUnicode_FromWideChar(&buffer[drvsize], rootsize); + if (root == NULL) { + goto exit; } - if (slow_path) { - if (_path.fd != -1) { - hfile = _Py_get_osfhandle_noraise(_path.fd); - close_file = FALSE; - } - else { - hfile = CreateFileW(_path.wide, FILE_READ_ATTRIBUTES, 0, NULL, - OPEN_EXISTING, - FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS, - NULL); + tail = PyUnicode_FromWideChar(&buffer[drvsize + rootsize], + path->length - drvsize - rootsize); + if (tail == NULL) { + goto exit; + } + if (PyBytes_Check(path->object)) { + Py_SETREF(drv, PyUnicode_EncodeFSDefault(drv)); + if (drv == NULL) { + goto exit; } - if (hfile != INVALID_HANDLE_VALUE) { - if (GetFileInformationByHandleEx(hfile, FileAttributeTagInfo, &info, - sizeof(info))) - { - result = (info.ReparseTag == IO_REPARSE_TAG_SYMLINK); - } - else { - result = 0; - } - if (close_file) { - CloseHandle(hfile); - } + Py_SETREF(root, PyUnicode_EncodeFSDefault(root)); + if (root == NULL) { + goto exit; } - else { - STRUCT_STAT st; - switch (GetLastError()) { - case ERROR_ACCESS_DENIED: - case ERROR_SHARING_VIOLATION: - case ERROR_CANT_ACCESS_FILE: - case ERROR_INVALID_PARAMETER: - if (LSTAT(_path.wide, &st)) { - result = 0; - } - else { - result = S_ISLNK(st.st_mode); - } - break; - default: - result = 0; - } + Py_SETREF(tail, PyUnicode_EncodeFSDefault(tail)); + if (tail == NULL) { + goto exit; } } - Py_END_ALLOW_THREADS - - path_cleanup(&_path); - if (result) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; -} - -#endif /* MS_WINDOWS */ + result = PyTuple_Pack(3, drv, root, tail); +exit: + Py_XDECREF(drv); + Py_XDECREF(root); + Py_XDECREF(tail); + return result; +} /*[clinic input] os._path_normpath - path: object + path: path_t(make_wide=True, nonstrict=True) -Basic path normalization. +Normalize path, eliminating double slashes, etc. [clinic start generated code]*/ static PyObject * -os__path_normpath_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=b94d696d828019da input=5e90c39e12549dc0]*/ +os__path_normpath_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=d353e7ed9410c044 input=3d4ac23b06332dcb]*/ { - if (!PyUnicode_Check(path)) { - PyErr_Format(PyExc_TypeError, "expected 'str', not '%.200s'", - Py_TYPE(path)->tp_name); - return NULL; + PyObject *result; + Py_ssize_t norm_len; + wchar_t *norm_path = _Py_normpath_and_size((wchar_t *)path->wide, + path->length, &norm_len); + if (!norm_len) { + result = PyUnicode_FromOrdinal('.'); } - Py_ssize_t len; - wchar_t *buffer = PyUnicode_AsWideCharString(path, &len); - if (!buffer) { - return NULL; + else { + result = PyUnicode_FromWideChar(norm_path, norm_len); + } + if (PyBytes_Check(path->object)) { + Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); } - Py_ssize_t norm_len; - wchar_t *norm_path = _Py_normpath_and_size(buffer, len, &norm_len); - PyObject *result = PyUnicode_FromWideChar(norm_path, norm_len); - PyMem_Free(buffer); return result; } @@ -5493,6 +5664,12 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) /*[clinic end generated code: output=a70446903abe821f input=a61722e1576fab03]*/ { int result; +#ifdef MS_WINDOWS + int error = 0; + int pathError = 0; + SECURITY_ATTRIBUTES secAttr = { sizeof(secAttr) }; + SECURITY_ATTRIBUTES *pSecAttr = NULL; +#endif #ifdef HAVE_MKDIRAT int mkdirat_unavailable = 0; #endif @@ -5504,11 +5681,38 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS - result = CreateDirectoryW(path->wide, NULL); + if (mode == 0700 /* 0o700 */) { + ULONG sdSize; + pSecAttr = &secAttr; + // Set a discretionary ACL (D) that is protected (P) and includes + // inheritable (OICI) entries that allow (A) full control (FA) to + // SYSTEM (SY), Administrators (BA), and the owner (OW). + if (!ConvertStringSecurityDescriptorToSecurityDescriptorW( + L"D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)", + SDDL_REVISION_1, + &secAttr.lpSecurityDescriptor, + &sdSize + )) { + error = GetLastError(); + } + } + if (!error) { + result = CreateDirectoryW(path->wide, pSecAttr); + if (secAttr.lpSecurityDescriptor && + // uncommonly, LocalFree returns non-zero on error, but still uses + // GetLastError() to see what the error code is + LocalFree(secAttr.lpSecurityDescriptor)) { + error = GetLastError(); + } + } Py_END_ALLOW_THREADS - if (!result) + if (error) { + return PyErr_SetFromWindowsErr(error); + } + if (!result) { return path_error(path); + } #else Py_BEGIN_ALLOW_THREADS #if HAVE_MKDIRAT @@ -7727,14 +7931,20 @@ os_register_at_fork_impl(PyObject *module, PyObject *before, } #endif /* HAVE_FORK */ +#if defined(HAVE_FORK1) || defined(HAVE_FORKPTY) || defined(HAVE_FORK) // Common code to raise a warning if we detect there is more than one thread // running in the process. Best effort, silent if unable to count threads. // Constraint: Quick. Never overcounts. Never leaves an error set. // -// This code might do an import, thus acquiring the import lock, which -// PyOS_BeforeFork() also does. As this should only be called from -// the parent process, it is in the same thread so that works. -static void warn_about_fork_with_threads(const char* name) { +// This should only be called from the parent process after +// PyOS_AfterFork_Parent(). +static void +warn_about_fork_with_threads(const char* name) +{ + // It's not safe to issue the warning while the world is stopped, because + // other threads might be holding locks that we need, which would deadlock. + assert(!_PyRuntime.stoptheworld.world_stopped); + // TODO: Consider making an `os` module API to return the current number // of threads in the process. That'd presumably use this platform code but // raise an error rather than using the inaccurate fallback. @@ -7825,6 +8035,7 @@ static void warn_about_fork_with_threads(const char* name) { PyErr_Clear(); } } +#endif // HAVE_FORK1 || HAVE_FORKPTY || HAVE_FORK #ifdef HAVE_FORK1 /*[clinic input] @@ -7842,7 +8053,7 @@ os_fork1_impl(PyObject *module) pid_t pid; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { PyErr_SetString(PyExc_PythonFinalizationError, "can't fork at interpreter shutdown"); return NULL; @@ -7858,9 +8069,10 @@ os_fork1_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { - warn_about_fork_with_threads("fork1"); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); + // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. + warn_about_fork_with_threads("fork1"); } if (pid == -1) { errno = saved_errno; @@ -7886,7 +8098,7 @@ os_fork_impl(PyObject *module) { pid_t pid; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { PyErr_SetString(PyExc_PythonFinalizationError, "can't fork at interpreter shutdown"); return NULL; @@ -7906,9 +8118,10 @@ os_fork_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { - warn_about_fork_with_threads("fork"); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); + // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. + warn_about_fork_with_threads("fork"); } if (pid == -1) { errno = saved_errno; @@ -8013,6 +8226,16 @@ os_sched_param_impl(PyTypeObject *type, PyObject *sched_priority) return res; } +static PyObject * +os_sched_param_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return Py_BuildValue("(O(N))", Py_TYPE(self), PyStructSequence_GetItem(self, 0)); +} + +static PyMethodDef os_sched_param_reduce_method = { + "__reduce__", (PyCFunction)os_sched_param_reduce, METH_NOARGS|METH_COEXIST, NULL, +}; + PyDoc_VAR(os_sched_param__doc__); static PyStructSequence_Field sched_param_fields[] = { @@ -8480,6 +8703,19 @@ os_unlockpt_impl(PyObject *module, int fd) #endif /* HAVE_UNLOCKPT */ #if defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R) +static PyObject * +py_ptsname(int fd) +{ + // POSIX manpage: Upon failure, ptsname() shall return a null pointer + // and may set errno. Always initialize errno to avoid undefined behavior. + errno = 0; + char *name = ptsname(fd); + if (name == NULL) { + return posix_error(); + } + return PyUnicode_DecodeFSDefault(name); +} + /*[clinic input] os.ptsname @@ -8501,22 +8737,22 @@ os_ptsname_impl(PyObject *module, int fd) int ret; char name[MAXPATHLEN+1]; - ret = ptsname_r(fd, name, sizeof(name)); + if (HAVE_PTSNAME_R_RUNTIME) { + ret = ptsname_r(fd, name, sizeof(name)); + } + else { + // fallback to ptsname() if ptsname_r() is not available in runtime. + return py_ptsname(fd); + } if (ret != 0) { errno = ret; return posix_error(); } -#else - char *name; - - name = ptsname(fd); - /* POSIX manpage: Upon failure, ptsname() shall return a null pointer and may set errno. - *MAY* set errno? Hmm... */ - if (name == NULL) - return posix_error(); -#endif /* HAVE_PTSNAME_R */ return PyUnicode_DecodeFSDefault(name); +#else + return py_ptsname(fd); +#endif /* HAVE_PTSNAME_R */ } #endif /* defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R) */ @@ -8719,7 +8955,7 @@ os_forkpty_impl(PyObject *module) pid_t pid; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { PyErr_SetString(PyExc_PythonFinalizationError, "can't fork at interpreter shutdown"); return NULL; @@ -8737,9 +8973,10 @@ os_forkpty_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { - warn_about_fork_with_threads("forkpty"); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); + // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. + warn_about_fork_with_threads("forkpty"); } if (pid == -1) { return posix_error(); @@ -8869,7 +9106,7 @@ os_getgrouplist_impl(PyObject *module, const char *user, gid_t basegid) /* * NGROUPS_MAX is defined by POSIX.1 as the maximum - * number of supplimental groups a users can belong to. + * number of supplemental groups a users can belong to. * We have to increment it by one because * getgrouplist() returns both the supplemental groups * and the primary group, i.e. all of the groups the @@ -9115,7 +9352,81 @@ os_setpgrp_impl(PyObject *module) #ifdef HAVE_GETPPID #ifdef MS_WINDOWS -#include +#include +#include + +// The structure definition in winternl.h may be incomplete. +// This structure is the full version from the MSDN documentation. +typedef struct _PROCESS_BASIC_INFORMATION_FULL { + NTSTATUS ExitStatus; + PVOID PebBaseAddress; + ULONG_PTR AffinityMask; + LONG BasePriority; + ULONG_PTR UniqueProcessId; + ULONG_PTR InheritedFromUniqueProcessId; +} PROCESS_BASIC_INFORMATION_FULL; + +typedef NTSTATUS (NTAPI *PNT_QUERY_INFORMATION_PROCESS) ( + IN HANDLE ProcessHandle, + IN PROCESSINFOCLASS ProcessInformationClass, + OUT PVOID ProcessInformation, + IN ULONG ProcessInformationLength, + OUT PULONG ReturnLength OPTIONAL); + +// This function returns the process ID of the parent process. +// Returns 0 on failure. +static ULONG +win32_getppid_fast(void) +{ + NTSTATUS status; + HMODULE ntdll; + PNT_QUERY_INFORMATION_PROCESS pNtQueryInformationProcess; + PROCESS_BASIC_INFORMATION_FULL basic_information; + static ULONG cached_ppid = 0; + + if (cached_ppid) { + // No need to query the kernel again. + return cached_ppid; + } + + ntdll = GetModuleHandleW(L"ntdll.dll"); + if (!ntdll) { + return 0; + } + + pNtQueryInformationProcess = (PNT_QUERY_INFORMATION_PROCESS) GetProcAddress(ntdll, "NtQueryInformationProcess"); + if (!pNtQueryInformationProcess) { + return 0; + } + + status = pNtQueryInformationProcess(GetCurrentProcess(), + ProcessBasicInformation, + &basic_information, + sizeof(basic_information), + NULL); + + if (!NT_SUCCESS(status)) { + return 0; + } + + // Perform sanity check on the parent process ID we received from NtQueryInformationProcess. + // The check covers values which exceed the 32-bit range (if running on x64) as well as + // zero and (ULONG) -1. + + if (basic_information.InheritedFromUniqueProcessId == 0 || + basic_information.InheritedFromUniqueProcessId >= ULONG_MAX) + { + return 0; + } + + // Now that we have reached this point, the BasicInformation.InheritedFromUniqueProcessId + // structure member contains a ULONG_PTR which represents the process ID of our parent + // process. This process ID will be correctly returned even if the parent process has + // exited or been terminated. + + cached_ppid = (ULONG) basic_information.InheritedFromUniqueProcessId; + return cached_ppid; +} static PyObject* win32_getppid(void) @@ -9123,8 +9434,16 @@ win32_getppid(void) DWORD error; PyObject* result = NULL; HANDLE process = GetCurrentProcess(); - HPSS snapshot = NULL; + ULONG pid; + + pid = win32_getppid_fast(); + if (pid != 0) { + return PyLong_FromUnsignedLong(pid); + } + + // If failure occurs in win32_getppid_fast(), fall back to using the PSS API. + error = PssCaptureSnapshot(process, PSS_CAPTURE_NONE, 0, &snapshot); if (error != ERROR_SUCCESS) { return PyErr_SetFromWindowsErr(error); @@ -9263,42 +9582,33 @@ os_kill_impl(PyObject *module, pid_t pid, Py_ssize_t signal) Py_RETURN_NONE; #else /* !MS_WINDOWS */ - PyObject *result; DWORD sig = (DWORD)signal; - DWORD err; - HANDLE handle; #ifdef HAVE_WINDOWS_CONSOLE_IO /* Console processes which share a common console can be sent CTRL+C or CTRL+BREAK events, provided they handle said events. */ if (sig == CTRL_C_EVENT || sig == CTRL_BREAK_EVENT) { if (GenerateConsoleCtrlEvent(sig, (DWORD)pid) == 0) { - err = GetLastError(); - PyErr_SetFromWindowsErr(err); - } - else { - Py_RETURN_NONE; + return PyErr_SetFromWindowsErr(0); } + Py_RETURN_NONE; } #endif /* HAVE_WINDOWS_CONSOLE_IO */ /* If the signal is outside of what GenerateConsoleCtrlEvent can use, attempt to open and terminate the process. */ - handle = OpenProcess(PROCESS_ALL_ACCESS, FALSE, (DWORD)pid); + HANDLE handle = OpenProcess(PROCESS_ALL_ACCESS, FALSE, (DWORD)pid); if (handle == NULL) { - err = GetLastError(); - return PyErr_SetFromWindowsErr(err); + return PyErr_SetFromWindowsErr(0); } - if (TerminateProcess(handle, sig) == 0) { - err = GetLastError(); - result = PyErr_SetFromWindowsErr(err); - } else { - result = Py_NewRef(Py_None); + BOOL res = TerminateProcess(handle, sig); + CloseHandle(handle); + if (res == 0) { + return PyErr_SetFromWindowsErr(0); } - CloseHandle(handle); - return result; + Py_RETURN_NONE; #endif /* !MS_WINDOWS */ } #endif /* HAVE_KILL */ @@ -9577,36 +9887,39 @@ wait_helper(PyObject *module, pid_t pid, int status, struct rusage *ru) if (!result) return NULL; + int pos = 0; + #ifndef doubletime #define doubletime(TV) ((double)(TV).tv_sec + (TV).tv_usec * 0.000001) #endif - PyStructSequence_SET_ITEM(result, 0, - PyFloat_FromDouble(doubletime(ru->ru_utime))); - PyStructSequence_SET_ITEM(result, 1, - PyFloat_FromDouble(doubletime(ru->ru_stime))); -#define SET_INT(result, index, value)\ - PyStructSequence_SET_ITEM(result, index, PyLong_FromLong(value)) - SET_INT(result, 2, ru->ru_maxrss); - SET_INT(result, 3, ru->ru_ixrss); - SET_INT(result, 4, ru->ru_idrss); - SET_INT(result, 5, ru->ru_isrss); - SET_INT(result, 6, ru->ru_minflt); - SET_INT(result, 7, ru->ru_majflt); - SET_INT(result, 8, ru->ru_nswap); - SET_INT(result, 9, ru->ru_inblock); - SET_INT(result, 10, ru->ru_oublock); - SET_INT(result, 11, ru->ru_msgsnd); - SET_INT(result, 12, ru->ru_msgrcv); - SET_INT(result, 13, ru->ru_nsignals); - SET_INT(result, 14, ru->ru_nvcsw); - SET_INT(result, 15, ru->ru_nivcsw); -#undef SET_INT - - if (PyErr_Occurred()) { - Py_DECREF(result); - return NULL; - } +#define SET_RESULT(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + Py_DECREF(result); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(result, pos++, item); \ + } while(0) + + SET_RESULT(PyFloat_FromDouble(doubletime(ru->ru_utime))); + SET_RESULT(PyFloat_FromDouble(doubletime(ru->ru_stime))); + SET_RESULT(PyLong_FromLong(ru->ru_maxrss)); + SET_RESULT(PyLong_FromLong(ru->ru_ixrss)); + SET_RESULT(PyLong_FromLong(ru->ru_idrss)); + SET_RESULT(PyLong_FromLong(ru->ru_isrss)); + SET_RESULT(PyLong_FromLong(ru->ru_minflt)); + SET_RESULT(PyLong_FromLong(ru->ru_majflt)); + SET_RESULT(PyLong_FromLong(ru->ru_nswap)); + SET_RESULT(PyLong_FromLong(ru->ru_inblock)); + SET_RESULT(PyLong_FromLong(ru->ru_oublock)); + SET_RESULT(PyLong_FromLong(ru->ru_msgsnd)); + SET_RESULT(PyLong_FromLong(ru->ru_msgrcv)); + SET_RESULT(PyLong_FromLong(ru->ru_nsignals)); + SET_RESULT(PyLong_FromLong(ru->ru_nvcsw)); + SET_RESULT(PyLong_FromLong(ru->ru_nivcsw)); +#undef SET_RESULT return Py_BuildValue("NiN", PyLong_FromPid(pid), status, result); } @@ -9729,15 +10042,25 @@ os_waitid_impl(PyObject *module, idtype_t idtype, id_t id, int options) if (!result) return NULL; - PyStructSequence_SET_ITEM(result, 0, PyLong_FromPid(si.si_pid)); - PyStructSequence_SET_ITEM(result, 1, _PyLong_FromUid(si.si_uid)); - PyStructSequence_SET_ITEM(result, 2, PyLong_FromLong((long)(si.si_signo))); - PyStructSequence_SET_ITEM(result, 3, PyLong_FromLong((long)(si.si_status))); - PyStructSequence_SET_ITEM(result, 4, PyLong_FromLong((long)(si.si_code))); - if (PyErr_Occurred()) { - Py_DECREF(result); - return NULL; - } + int pos = 0; + +#define SET_RESULT(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + Py_DECREF(result); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(result, pos++, item); \ + } while(0) + + SET_RESULT(PyLong_FromPid(si.si_pid)); + SET_RESULT(_PyLong_FromUid(si.si_uid)); + SET_RESULT(PyLong_FromLong((long)(si.si_signo))); + SET_RESULT(PyLong_FromLong((long)(si.si_status))); + SET_RESULT(PyLong_FromLong((long)(si.si_code))); + +#undef SET_RESULT return result; } @@ -9851,7 +10174,10 @@ os_wait_impl(PyObject *module) } #endif /* HAVE_WAIT */ -#if defined(__linux__) && defined(__NR_pidfd_open) + +// This system call always crashes on older Android versions. +#if defined(__linux__) && defined(__NR_pidfd_open) && \ + !(defined(__ANDROID__) && __ANDROID_API__ < 31) /*[clinic input] os.pidfd_open pid: pid_t @@ -10051,7 +10377,7 @@ os_readlink_impl(PyObject *module, path_t *path, int dir_fd) name[1] = L'\\'; } result = PyUnicode_FromWideChar(name, nameLen); - if (result && path->narrow) { + if (result && PyBytes_Check(path->object)) { Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); } } @@ -10334,12 +10660,12 @@ Return a collection containing process timing information. The object returned behaves like a named tuple with these fields: (utime, stime, cutime, cstime, elapsed_time) -All fields are floating point numbers. +All fields are floating-point numbers. [clinic start generated code]*/ static PyObject * os_times_impl(PyObject *module) -/*[clinic end generated code: output=35f640503557d32a input=2bf9df3d6ab2e48b]*/ +/*[clinic end generated code: output=35f640503557d32a input=8dbfe33a2dcc3df3]*/ { #ifdef MS_WINDOWS FILETIME create, exit, kernel, user; @@ -10470,30 +10796,40 @@ os.timerfd_settime * flags: int = 0 0 or a bit mask of TFD_TIMER_ABSTIME or TFD_TIMER_CANCEL_ON_SET. - initial: double = 0.0 + initial as initial_double: double = 0.0 The initial expiration time, in seconds. - interval: double = 0.0 + interval as interval_double: double = 0.0 The timer's interval, in seconds. Alter a timer file descriptor's internal timer in seconds. [clinic start generated code]*/ static PyObject * -os_timerfd_settime_impl(PyObject *module, int fd, int flags, double initial, - double interval) -/*[clinic end generated code: output=0dda31115317adb9 input=6c24e47e7a4d799e]*/ +os_timerfd_settime_impl(PyObject *module, int fd, int flags, + double initial_double, double interval_double) +/*[clinic end generated code: output=df4c1bce6859224e input=81d2c0d7e936e8a7]*/ { - struct itimerspec new_value; - struct itimerspec old_value; - int result; - if (_PyTime_AsTimespec(_PyTime_FromSecondsDouble(initial, _PyTime_ROUND_FLOOR), &new_value.it_value) < 0) { + PyTime_t initial, interval; + if (_PyTime_FromSecondsDouble(initial_double, _PyTime_ROUND_FLOOR, + &initial) < 0) { + return NULL; + } + if (_PyTime_FromSecondsDouble(interval_double, _PyTime_ROUND_FLOOR, + &interval) < 0) { + return NULL; + } + + struct itimerspec new_value, old_value; + if (_PyTime_AsTimespec(initial, &new_value.it_value) < 0) { PyErr_SetString(PyExc_ValueError, "invalid initial value"); return NULL; } - if (_PyTime_AsTimespec(_PyTime_FromSecondsDouble(interval, _PyTime_ROUND_FLOOR), &new_value.it_interval) < 0) { + if (_PyTime_AsTimespec(interval, &new_value.it_interval) < 0) { PyErr_SetString(PyExc_ValueError, "invalid interval value"); return NULL; } + + int result; Py_BEGIN_ALLOW_THREADS result = timerfd_settime(fd, flags, &new_value, &old_value); Py_END_ALLOW_THREADS @@ -11097,6 +11433,38 @@ os_read_impl(PyObject *module, int fd, Py_ssize_t length) return buffer; } +/*[clinic input] +os.readinto -> Py_ssize_t + fd: int + buffer: Py_buffer(accept={rwbuffer}) + / + +Read into a buffer object from a file descriptor. + +The buffer should be mutable and bytes-like. On success, returns the number of +bytes read. Less bytes may be read than the size of the buffer. The underlying +system call will be retried when interrupted by a signal, unless the signal +handler raises an exception. Other errors will not be retried and an error will +be raised. + +Returns 0 if *fd* is at end of file or if the provided *buffer* has length 0 +(which can be used to check for errors without reading data). Never returns +negative. +[clinic start generated code]*/ + +static Py_ssize_t +os_readinto_impl(PyObject *module, int fd, Py_buffer *buffer) +/*[clinic end generated code: output=8091a3513c683a80 input=d40074d0a68de575]*/ +{ + assert(buffer->len >= 0); + Py_ssize_t result = _Py_read(fd, buffer->buf, buffer->len); + /* Ensure negative is never returned without an error. Simplifies calling + code. _Py_read should succeed, possibly reading 0 bytes, _or_ set an + error. */ + assert(result >= 0 || (result == -1 && PyErr_Occurred())); + return result; +} + #if (defined(HAVE_SENDFILE) && (defined(__FreeBSD__) || defined(__DragonFly__) \ || defined(__APPLE__))) \ || defined(HAVE_READV) || defined(HAVE_PREADV) || defined (HAVE_PREADV2) \ @@ -12283,8 +12651,30 @@ os_mknod_impl(PyObject *module, path_t *path, int mode, dev_t device, #ifdef HAVE_DEVICE_MACROS +static PyObject * +major_minor_conv(unsigned int value) +{ +#ifdef NODEV + if (value == (unsigned int)NODEV) { + return PyLong_FromLong((int)NODEV); + } +#endif + return PyLong_FromUnsignedLong(value); +} + +static int +major_minor_check(dev_t value) +{ +#ifdef NODEV + if (value == NODEV) { + return 1; + } +#endif + return (dev_t)(unsigned int)value == value; +} + /*[clinic input] -os.major -> unsigned_int +os.major device: dev_t / @@ -12292,16 +12682,16 @@ os.major -> unsigned_int Extracts a device major number from a raw device number. [clinic start generated code]*/ -static unsigned int +static PyObject * os_major_impl(PyObject *module, dev_t device) -/*[clinic end generated code: output=5b3b2589bafb498e input=1e16a4d30c4d4462]*/ +/*[clinic end generated code: output=4071ffee17647891 input=b1a0a14ec9448229]*/ { - return major(device); + return major_minor_conv(major(device)); } /*[clinic input] -os.minor -> unsigned_int +os.minor device: dev_t / @@ -12309,28 +12699,33 @@ os.minor -> unsigned_int Extracts a device minor number from a raw device number. [clinic start generated code]*/ -static unsigned int +static PyObject * os_minor_impl(PyObject *module, dev_t device) -/*[clinic end generated code: output=5e1a25e630b0157d input=0842c6d23f24c65e]*/ +/*[clinic end generated code: output=306cb78e3bc5004f input=2f686e463682a9da]*/ { - return minor(device); + return major_minor_conv(minor(device)); } /*[clinic input] os.makedev -> dev_t - major: int - minor: int + major: dev_t + minor: dev_t / Composes a raw device number from the major and minor device numbers. [clinic start generated code]*/ static dev_t -os_makedev_impl(PyObject *module, int major, int minor) -/*[clinic end generated code: output=881aaa4aba6f6a52 input=4b9fd8fc73cbe48f]*/ +os_makedev_impl(PyObject *module, dev_t major, dev_t minor) +/*[clinic end generated code: output=cad6125c51f5af80 input=2146126ec02e55c1]*/ { + if (!major_minor_check(major) || !major_minor_check(minor)) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C unsigned int"); + return (dev_t)-1; + } return makedev(major, minor); } #endif /* HAVE_DEVICE_MACROS */ @@ -12957,46 +13352,50 @@ _pystatvfs_fromstructstatvfs(PyObject *module, struct statvfs st) { if (v == NULL) return NULL; + int pos = 0; + +#define SET_RESULT(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + Py_DECREF(v); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(v, pos++, item); \ + } while(0) + #if !defined(HAVE_LARGEFILE_SUPPORT) - PyStructSequence_SET_ITEM(v, 0, PyLong_FromLong((long) st.f_bsize)); - PyStructSequence_SET_ITEM(v, 1, PyLong_FromLong((long) st.f_frsize)); - PyStructSequence_SET_ITEM(v, 2, PyLong_FromLong((long) st.f_blocks)); - PyStructSequence_SET_ITEM(v, 3, PyLong_FromLong((long) st.f_bfree)); - PyStructSequence_SET_ITEM(v, 4, PyLong_FromLong((long) st.f_bavail)); - PyStructSequence_SET_ITEM(v, 5, PyLong_FromLong((long) st.f_files)); - PyStructSequence_SET_ITEM(v, 6, PyLong_FromLong((long) st.f_ffree)); - PyStructSequence_SET_ITEM(v, 7, PyLong_FromLong((long) st.f_favail)); - PyStructSequence_SET_ITEM(v, 8, PyLong_FromLong((long) st.f_flag)); - PyStructSequence_SET_ITEM(v, 9, PyLong_FromLong((long) st.f_namemax)); + SET_RESULT(PyLong_FromLong((long) st.f_bsize)); + SET_RESULT(PyLong_FromLong((long) st.f_frsize)); + SET_RESULT(PyLong_FromLong((long) st.f_blocks)); + SET_RESULT(PyLong_FromLong((long) st.f_bfree)); + SET_RESULT(PyLong_FromLong((long) st.f_bavail)); + SET_RESULT(PyLong_FromLong((long) st.f_files)); + SET_RESULT(PyLong_FromLong((long) st.f_ffree)); + SET_RESULT(PyLong_FromLong((long) st.f_favail)); + SET_RESULT(PyLong_FromLong((long) st.f_flag)); + SET_RESULT(PyLong_FromLong((long) st.f_namemax)); #else - PyStructSequence_SET_ITEM(v, 0, PyLong_FromLong((long) st.f_bsize)); - PyStructSequence_SET_ITEM(v, 1, PyLong_FromLong((long) st.f_frsize)); - PyStructSequence_SET_ITEM(v, 2, - PyLong_FromLongLong((long long) st.f_blocks)); - PyStructSequence_SET_ITEM(v, 3, - PyLong_FromLongLong((long long) st.f_bfree)); - PyStructSequence_SET_ITEM(v, 4, - PyLong_FromLongLong((long long) st.f_bavail)); - PyStructSequence_SET_ITEM(v, 5, - PyLong_FromLongLong((long long) st.f_files)); - PyStructSequence_SET_ITEM(v, 6, - PyLong_FromLongLong((long long) st.f_ffree)); - PyStructSequence_SET_ITEM(v, 7, - PyLong_FromLongLong((long long) st.f_favail)); - PyStructSequence_SET_ITEM(v, 8, PyLong_FromLong((long) st.f_flag)); - PyStructSequence_SET_ITEM(v, 9, PyLong_FromLong((long) st.f_namemax)); + SET_RESULT(PyLong_FromLong((long) st.f_bsize)); + SET_RESULT(PyLong_FromLong((long) st.f_frsize)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_blocks)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_bfree)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_bavail)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_files)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_ffree)); + SET_RESULT(PyLong_FromLongLong((long long) st.f_favail)); + SET_RESULT(PyLong_FromLong((long) st.f_flag)); + SET_RESULT(PyLong_FromLong((long) st.f_namemax)); #endif /* The _ALL_SOURCE feature test macro defines f_fsid as a structure * (issue #32390). */ #if defined(_AIX) && defined(_ALL_SOURCE) - PyStructSequence_SET_ITEM(v, 10, PyLong_FromUnsignedLong(st.f_fsid.val[0])); + SET_RESULT(PyLong_FromUnsignedLong(st.f_fsid.val[0])); #else - PyStructSequence_SET_ITEM(v, 10, PyLong_FromUnsignedLong(st.f_fsid)); + SET_RESULT(PyLong_FromUnsignedLong(st.f_fsid)); #endif - if (PyErr_Occurred()) { - Py_DECREF(v); - return NULL; - } + +#undef SET_RESULT return v; } @@ -13185,46 +13584,38 @@ struct constdef { }; static int -conv_confname(PyObject *arg, int *valuep, struct constdef *table, - size_t tablesize) +conv_confname(PyObject *module, PyObject *arg, int *valuep, const char *tablename) { - if (PyLong_Check(arg)) { - int value = PyLong_AsInt(arg); - if (value == -1 && PyErr_Occurred()) - return 0; - *valuep = value; - return 1; - } - else { - /* look up the value in the table using a binary search */ - size_t lo = 0; - size_t mid; - size_t hi = tablesize; - int cmp; - const char *confname; - if (!PyUnicode_Check(arg)) { - PyErr_SetString(PyExc_TypeError, - "configuration names must be strings or integers"); + if (PyUnicode_Check(arg)) { + PyObject *table = PyObject_GetAttrString(module, tablename); + if (table == NULL) { return 0; } - confname = PyUnicode_AsUTF8(arg); - if (confname == NULL) + + arg = PyObject_GetItem(table, arg); + Py_DECREF(table); + if (arg == NULL) { + PyErr_SetString( + PyExc_ValueError, "unrecognized configuration name"); return 0; - while (lo < hi) { - mid = (lo + hi) / 2; - cmp = strcmp(confname, table[mid].name); - if (cmp < 0) - hi = mid; - else if (cmp > 0) - lo = mid + 1; - else { - *valuep = table[mid].value; - return 1; - } } - PyErr_SetString(PyExc_ValueError, "unrecognized configuration name"); - return 0; + } else { + Py_INCREF(arg); // Match the Py_DECREF below. + } + + int success = 0; + if (!PyLong_Check(arg)) { + PyErr_SetString(PyExc_TypeError, + "configuration names must be strings or integers"); + } else { + int value = PyLong_AsInt(arg); + if (!(value == -1 && PyErr_Occurred())) { + *valuep = value; + success = 1; + } } + Py_DECREF(arg); + return success; } @@ -13315,14 +13706,6 @@ static struct constdef posix_constants_pathconf[] = { {"PC_TIMESTAMP_RESOLUTION", _PC_TIMESTAMP_RESOLUTION}, #endif }; - -static int -conv_path_confname(PyObject *arg, int *valuep) -{ - return conv_confname(arg, valuep, posix_constants_pathconf, - sizeof(posix_constants_pathconf) - / sizeof(struct constdef)); -} #endif @@ -13331,7 +13714,7 @@ conv_path_confname(PyObject *arg, int *valuep) os.fpathconf -> long fd: fildes - name: path_confname + name: confname(table="pathconf_names") / Return the configuration limit name for the file descriptor fd. @@ -13341,7 +13724,7 @@ If there is no limit, return -1. static long os_fpathconf_impl(PyObject *module, int fd, int name) -/*[clinic end generated code: output=d5b7042425fc3e21 input=5b8d2471cfaae186]*/ +/*[clinic end generated code: output=d5b7042425fc3e21 input=023d44589c9ed6aa]*/ { long limit; @@ -13359,7 +13742,7 @@ os_fpathconf_impl(PyObject *module, int fd, int name) /*[clinic input] os.pathconf -> long path: path_t(allow_fd='PATH_HAVE_FPATHCONF') - name: path_confname + name: confname(table="pathconf_names") Return the configuration limit name for the file or directory path. @@ -13370,7 +13753,7 @@ On some platforms, path may also be specified as an open file descriptor. static long os_pathconf_impl(PyObject *module, path_t *path, int name) -/*[clinic end generated code: output=5bedee35b293a089 input=bc3e2a985af27e5e]*/ +/*[clinic end generated code: output=5bedee35b293a089 input=6f6072f57b10c787]*/ { long limit; @@ -13547,19 +13930,11 @@ static struct constdef posix_constants_confstr[] = { #endif }; -static int -conv_confstr_confname(PyObject *arg, int *valuep) -{ - return conv_confname(arg, valuep, posix_constants_confstr, - sizeof(posix_constants_confstr) - / sizeof(struct constdef)); -} - /*[clinic input] os.confstr - name: confstr_confname + name: confname(table="confstr_names") / Return a string-valued system configuration variable. @@ -13567,7 +13942,7 @@ Return a string-valued system configuration variable. static PyObject * os_confstr_impl(PyObject *module, int name) -/*[clinic end generated code: output=bfb0b1b1e49b9383 input=18fb4d0567242e65]*/ +/*[clinic end generated code: output=bfb0b1b1e49b9383 input=4c6ffca2837ec959]*/ { PyObject *result = NULL; char buffer[255]; @@ -14104,18 +14479,10 @@ static struct constdef posix_constants_sysconf[] = { #endif }; -static int -conv_sysconf_confname(PyObject *arg, int *valuep) -{ - return conv_confname(arg, valuep, posix_constants_sysconf, - sizeof(posix_constants_sysconf) - / sizeof(struct constdef)); -} - /*[clinic input] os.sysconf -> long - name: sysconf_confname + name: confname(table="sysconf_names") / Return an integer-valued system configuration variable. @@ -14123,7 +14490,7 @@ Return an integer-valued system configuration variable. static long os_sysconf_impl(PyObject *module, int name) -/*[clinic end generated code: output=3662f945fc0cc756 input=279e3430a33f29e4]*/ +/*[clinic end generated code: output=3662f945fc0cc756 input=930b8f23b5d15086]*/ { long value; @@ -14136,40 +14503,15 @@ os_sysconf_impl(PyObject *module, int name) #endif /* HAVE_SYSCONF */ -/* This code is used to ensure that the tables of configuration value names - * are in sorted order as required by conv_confname(), and also to build - * the exported dictionaries that are used to publish information about the - * names available on the host platform. - * - * Sorting the table at runtime ensures that the table is properly ordered - * when used, even for platforms we're not able to test on. It also makes - * it easier to add additional entries to the tables. - */ - -static int -cmp_constdefs(const void *v1, const void *v2) -{ - const struct constdef *c1 = - (const struct constdef *) v1; - const struct constdef *c2 = - (const struct constdef *) v2; - - return strcmp(c1->name, c2->name); -} - static int setup_confname_table(struct constdef *table, size_t tablesize, const char *tablename, PyObject *module) { - PyObject *d = NULL; - size_t i; - - qsort(table, tablesize, sizeof(struct constdef), cmp_constdefs); - d = PyDict_New(); + PyObject *d = PyDict_New(); if (d == NULL) return -1; - for (i=0; i < tablesize; ++i) { + for (size_t i=0; i < tablesize; ++i) { PyObject *o = PyLong_FromLong(table[i].value); if (o == NULL || PyDict_SetItemString(d, table[i].name, o) == -1) { Py_XDECREF(o); @@ -14968,12 +15310,23 @@ os_get_terminal_size_impl(PyObject *module, int fd) termsize = PyStructSequence_New((PyTypeObject *)TerminalSizeType); if (termsize == NULL) return NULL; - PyStructSequence_SET_ITEM(termsize, 0, PyLong_FromLong(columns)); - PyStructSequence_SET_ITEM(termsize, 1, PyLong_FromLong(lines)); - if (PyErr_Occurred()) { - Py_DECREF(termsize); - return NULL; - } + + int pos = 0; + +#define SET_TERMSIZE(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + Py_DECREF(termsize); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(termsize, pos++, item); \ + } while(0) + + SET_TERMSIZE(PyLong_FromLong(columns)); + SET_TERMSIZE(PyLong_FromLong(lines)); +#undef SET_TERMSIZE + return termsize; } #endif /* defined(TERMSIZE_USE_CONIO) || defined(TERMSIZE_USE_IOCTL) */ @@ -15647,7 +16000,8 @@ DirEntry_from_find_data(PyObject *module, path_t *path, WIN32_FIND_DATAW *dataW) entry->name = PyUnicode_FromWideChar(dataW->cFileName, -1); if (!entry->name) goto error; - if (path->narrow) { + int return_bytes = path->wide && PyBytes_Check(path->object); + if (return_bytes) { Py_SETREF(entry->name, PyUnicode_EncodeFSDefault(entry->name)); if (!entry->name) goto error; @@ -15661,7 +16015,7 @@ DirEntry_from_find_data(PyObject *module, path_t *path, WIN32_FIND_DATAW *dataW) PyMem_Free(joined_path); if (!entry->path) goto error; - if (path->narrow) { + if (return_bytes) { Py_SETREF(entry->path, PyUnicode_EncodeFSDefault(entry->path)); if (!entry->path) goto error; @@ -15670,6 +16024,10 @@ DirEntry_from_find_data(PyObject *module, path_t *path, WIN32_FIND_DATAW *dataW) find_data_to_file_info(dataW, &file_info, &reparse_tag); _Py_attribute_data_to_stat(&file_info, reparse_tag, NULL, NULL, &entry->win32_lstat); + /* ctime is only deprecated from 3.12, so we copy birthtime across */ + entry->win32_lstat.st_ctime = entry->win32_lstat.st_birthtime; + entry->win32_lstat.st_ctime_nsec = entry->win32_lstat.st_birthtime_nsec; + return (PyObject *)entry; error: @@ -16471,9 +16829,70 @@ os__supports_virtual_terminal_impl(PyObject *module) } #endif +/*[clinic input] +os._inputhook + +Calls PyOS_CallInputHook droppong the GIL first +[clinic start generated code]*/ -static PyMethodDef posix_methods[] = { +static PyObject * +os__inputhook_impl(PyObject *module) +/*[clinic end generated code: output=525aca4ef3c6149f input=fc531701930d064f]*/ +{ + int result = 0; + if (PyOS_InputHook) { + Py_BEGIN_ALLOW_THREADS; + result = PyOS_InputHook(); + Py_END_ALLOW_THREADS; + } + return PyLong_FromLong(result); +} + +/*[clinic input] +os._is_inputhook_installed + +Checks if PyOS_CallInputHook is set +[clinic start generated code]*/ + +static PyObject * +os__is_inputhook_installed_impl(PyObject *module) +/*[clinic end generated code: output=3b3eab4f672c689a input=ff177c9938dd76d8]*/ +{ + return PyBool_FromLong(PyOS_InputHook != NULL); +} + +/*[clinic input] +os._create_environ +Create the environment dictionary. +[clinic start generated code]*/ + +static PyObject * +os__create_environ_impl(PyObject *module) +/*[clinic end generated code: output=19d9039ab14f8ad4 input=a4c05686b34635e8]*/ +{ + return convertenviron(); +} + + +#ifdef __EMSCRIPTEN__ +/*[clinic input] +os._emscripten_debugger + +Create a breakpoint for the JavaScript debugger. Emscripten only. +[clinic start generated code]*/ + +static PyObject * +os__emscripten_debugger_impl(PyObject *module) +/*[clinic end generated code: output=ad47dc3bf0661343 input=d814b1877fb6083a]*/ +{ + emscripten_debugger(); + Py_RETURN_NONE; +} +#endif /* __EMSCRIPTEN__ */ + + +static PyMethodDef posix_methods[] = { OS_STAT_METHODDEF OS_ACCESS_METHODDEF OS_TTYNAME_METHODDEF @@ -16586,6 +17005,7 @@ static PyMethodDef posix_methods[] = { OS_LOCKF_METHODDEF OS_LSEEK_METHODDEF OS_READ_METHODDEF + OS_READINTO_METHODDEF OS_READV_METHODDEF OS_PREAD_METHODDEF OS_PREADV_METHODDEF @@ -16635,6 +17055,7 @@ static PyMethodDef posix_methods[] = { OS__FINDFIRSTFILE_METHODDEF OS__GETVOLUMEPATHNAME_METHODDEF OS__PATH_SPLITROOT_METHODDEF + OS__PATH_SPLITROOT_EX_METHODDEF OS__PATH_NORMPATH_METHODDEF OS_GETLOADAVG_METHODDEF OS_URANDOM_METHODDEF @@ -16678,9 +17099,15 @@ static PyMethodDef posix_methods[] = { OS__PATH_ISDIR_METHODDEF OS__PATH_ISFILE_METHODDEF OS__PATH_ISLINK_METHODDEF + OS__PATH_ISJUNCTION_METHODDEF OS__PATH_EXISTS_METHODDEF + OS__PATH_LEXISTS_METHODDEF OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF + OS__INPUTHOOK_METHODDEF + OS__IS_INPUTHOOK_INSTALLED_METHODDEF + OS__CREATE_ENVIRON_METHODDEF + OS__EMSCRIPTEN_DEBUGGER_METHODDEF {NULL, NULL} /* Sentinel */ }; @@ -17124,9 +17551,15 @@ all_ins(PyObject *m) #ifdef SCHED_OTHER if (PyModule_AddIntMacro(m, SCHED_OTHER)) return -1; #endif +#ifdef SCHED_DEADLINE + if (PyModule_AddIntMacro(m, SCHED_DEADLINE)) return -1; +#endif #ifdef SCHED_FIFO if (PyModule_AddIntMacro(m, SCHED_FIFO)) return -1; #endif +#ifdef SCHED_NORMAL + if (PyModule_AddIntMacro(m, SCHED_NORMAL)) return -1; +#endif #ifdef SCHED_RR if (PyModule_AddIntMacro(m, SCHED_RR)) return -1; #endif @@ -17392,6 +17825,9 @@ PROBE(probe_futimens, HAVE_FUTIMENS_RUNTIME) PROBE(probe_utimensat, HAVE_UTIMENSAT_RUNTIME) #endif +#ifdef HAVE_PTSNAME_R +PROBE(probe_ptsname_r, HAVE_PTSNAME_R_RUNTIME) +#endif @@ -17532,6 +17968,10 @@ static const struct have_function { { "HAVE_UTIMENSAT", probe_utimensat }, #endif +#ifdef HAVE_PTSNAME_R + { "HAVE_PTSNAME_R", probe_ptsname_r }, +#endif + #ifdef MS_WINDOWS { "MS_WINDOWS", NULL }, #endif @@ -17553,11 +17993,11 @@ posixmodule_exec(PyObject *m) return -1; } - if (PyDict_DelItemString(dct, "pwritev") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "pwritev", NULL) < 0) { + return -1; } - if (PyDict_DelItemString(dct, "preadv") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "preadv", NULL) < 0) { + return -1; } } #endif @@ -17609,6 +18049,12 @@ posixmodule_exec(PyObject *m) return -1; } ((PyTypeObject *)state->SchedParamType)->tp_new = os_sched_param; + if (_PyType_AddMethod((PyTypeObject *)state->SchedParamType, + &os_sched_param_reduce_method) < 0) + { + return -1; + } + PyType_Modified((PyTypeObject *)state->SchedParamType); #endif /* initialize TerminalSize_info */ @@ -17696,6 +18142,7 @@ posixmodule_exec(PyObject *m) static PyModuleDef_Slot posixmodile_slots[] = { {Py_mod_exec, posixmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/posixmodule.h b/Modules/posixmodule.h index 8827ce153fed8c..be732db04faf94 100644 --- a/Modules/posixmodule.h +++ b/Modules/posixmodule.h @@ -2,7 +2,6 @@ #ifndef Py_POSIXMODULE_H #define Py_POSIXMODULE_H -#ifndef Py_LIMITED_API #ifdef __cplusplus extern "C" { #endif @@ -34,5 +33,4 @@ extern int _Py_Sigset_Converter(PyObject *, void *); #ifdef __cplusplus } #endif -#endif // !Py_LIMITED_API #endif // !Py_POSIXMODULE_H diff --git a/Modules/pwdmodule.c b/Modules/pwdmodule.c index b7034369c4731e..2240e2078b2d98 100644 --- a/Modules/pwdmodule.c +++ b/Modules/pwdmodule.c @@ -1,9 +1,16 @@ /* UNIX password file access module */ +// Need limited C API version 3.13 for PyMem_RawRealloc() +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 +#endif + #include "Python.h" #include "posixmodule.h" +#include // ERANGE #include // getpwuid() #include // sysconf() @@ -64,53 +71,52 @@ static struct PyModuleDef pwdmodule; #define DEFAULT_BUFFER_SIZE 1024 -static void -sets(PyObject *v, int i, const char* val) -{ - if (val) { - PyObject *o = PyUnicode_DecodeFSDefault(val); - PyStructSequence_SET_ITEM(v, i, o); - } - else { - PyStructSequence_SET_ITEM(v, i, Py_None); - Py_INCREF(Py_None); - } -} - static PyObject * mkpwent(PyObject *module, struct passwd *p) { - int setIndex = 0; PyObject *v = PyStructSequence_New(get_pwd_state(module)->StructPwdType); - if (v == NULL) + if (v == NULL) { return NULL; + } -#define SETS(i,val) sets(v, i, val) + int setIndex = 0; + +#define SET_STRING(VAL) \ + SET_RESULT((VAL) ? PyUnicode_DecodeFSDefault((VAL)) : Py_NewRef(Py_None)) - SETS(setIndex++, p->pw_name); +#define SET_RESULT(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + goto error; \ + } \ + PyStructSequence_SetItem(v, setIndex++, item); \ + } while(0) + + SET_STRING(p->pw_name); #if defined(HAVE_STRUCT_PASSWD_PW_PASSWD) && !defined(__ANDROID__) - SETS(setIndex++, p->pw_passwd); + SET_STRING(p->pw_passwd); #else - SETS(setIndex++, ""); + SET_STRING(""); #endif - PyStructSequence_SET_ITEM(v, setIndex++, _PyLong_FromUid(p->pw_uid)); - PyStructSequence_SET_ITEM(v, setIndex++, _PyLong_FromGid(p->pw_gid)); + SET_RESULT(_PyLong_FromUid(p->pw_uid)); + SET_RESULT(_PyLong_FromGid(p->pw_gid)); #if defined(HAVE_STRUCT_PASSWD_PW_GECOS) - SETS(setIndex++, p->pw_gecos); + SET_STRING(p->pw_gecos); #else - SETS(setIndex++, ""); + SET_STRING(""); #endif - SETS(setIndex++, p->pw_dir); - SETS(setIndex++, p->pw_shell); + SET_STRING(p->pw_dir); + SET_STRING(p->pw_shell); -#undef SETS - - if (PyErr_Occurred()) { - Py_XDECREF(v); - return NULL; - } +#undef SET_STRING +#undef SET_RESULT return v; + +error: + Py_DECREF(v); + return NULL; } /*[clinic input] @@ -338,6 +344,7 @@ pwdmodule_exec(PyObject *module) static PyModuleDef_Slot pwdmodule_slots[] = { {Py_mod_exec, pwdmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index f04f96bc2f7601..9931ca2a8d4749 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -754,13 +754,13 @@ pyexpat.xmlparser.Parse Parse XML data. -`isfinal' should be true at end of input. +'isfinal' should be true at end of input. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_Parse_impl(xmlparseobject *self, PyTypeObject *cls, PyObject *data, int isfinal) -/*[clinic end generated code: output=8faffe07fe1f862a input=d0eb2a69fab3b9f1]*/ +/*[clinic end generated code: output=8faffe07fe1f862a input=053e0f047e55c05a]*/ { const char *s; Py_ssize_t slen; @@ -1651,7 +1651,7 @@ PyDoc_STRVAR(pyexpat_module_documentation, static int init_handler_descrs(pyexpat_state *state) { int i; - assert(!PyType_HasFeature(state->xml_parse_type, Py_TPFLAGS_VALID_VERSION_TAG)); + assert(state->xml_parse_type->tp_version_tag == 0); for (i = 0; handler_info[i].name != NULL; i++) { struct HandlerInfo *hi = &handler_info[i]; hi->getset.name = hi->name; @@ -1767,7 +1767,10 @@ struct ErrorInfo error_info_of[] = { {"XML_ERROR_NO_BUFFER", "a successful prior call to function XML_GetBuffer is required"}, /* Added in 2.4.0. */ - {"XML_ERROR_AMPLIFICATION_LIMIT_BREACH", "limit on input amplification factor (from DTD and entities) breached"} + {"XML_ERROR_AMPLIFICATION_LIMIT_BREACH", "limit on input amplification factor (from DTD and entities) breached"}, + + /* Added in 2.6.4. */ + {"XML_ERROR_NOT_STARTED", "parser not started"}, }; static int @@ -1782,7 +1785,12 @@ add_error(PyObject *errors_module, PyObject *codes_dict, * with the other uses of the XML_ErrorString function * elsewhere within this file. pyexpat's copy of the messages * only acts as a fallback in case of outdated runtime libexpat, - * where it returns NULL. */ + * where it returns NULL. + * + * In addition, XML_ErrorString is assumed to return UTF-8 encoded + * strings (in conv_string_to_unicode, we decode them using 'strict' + * error handling). + */ const char *error_string = XML_ErrorString(error_code); if (error_string == NULL) { error_string = error_info_of[error_index].description; @@ -2117,6 +2125,7 @@ pyexpat_free(void *module) static PyModuleDef_Slot pyexpat_slots[] = { {Py_mod_exec, pyexpat_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/readline.c b/Modules/readline.c index e29051c37f8827..7d1f703f7dbdde 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -351,6 +351,12 @@ readline_append_history_file_impl(PyObject *module, int nelements, PyObject *filename_obj) /*[clinic end generated code: output=5df06fc9da56e4e4 input=784b774db3a4b7c5]*/ { + if (nelements < 0) + { + PyErr_SetString(PyExc_ValueError, "nelements must be positive"); + return NULL; + } + PyObject *filename_bytes; const char *filename; int err; @@ -1041,7 +1047,7 @@ on_hook(PyObject *func) } static int -#if defined(_RL_FUNCTION_TYPEDEF) +#if defined(_RL_FUNCTION_TYPEDEF) || !defined(Py_RL_STARTUP_HOOK_TAKES_ARGS) on_startup_hook(void) #else on_startup_hook(const char *Py_UNUSED(text), int Py_UNUSED(state)) @@ -1061,7 +1067,7 @@ on_startup_hook(const char *Py_UNUSED(text), int Py_UNUSED(state)) #ifdef HAVE_RL_PRE_INPUT_HOOK static int -#if defined(_RL_FUNCTION_TYPEDEF) +#if defined(_RL_FUNCTION_TYPEDEF) || !defined(Py_RL_STARTUP_HOOK_TAKES_ARGS) on_pre_input_hook(void) #else on_pre_input_hook(const char *Py_UNUSED(text), int Py_UNUSED(state)) @@ -1552,6 +1558,9 @@ PyInit_readline(void) if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif if (PyModule_AddIntConstant(m, "_READLINE_VERSION", RL_READLINE_VERSION) < 0) { diff --git a/Modules/resource.c b/Modules/resource.c index 19020b8cc1b6db..3fe18e7c98e3d8 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -1,8 +1,7 @@ +// Need limited C API version 3.13 for PySys_Audit() #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.13 for PySys_Audit() -#define Py_LIMITED_API 0x030d0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -514,6 +513,7 @@ resource_exec(PyObject *module) static struct PyModuleDef_Slot resource_slots[] = { {Py_mod_exec, resource_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/rotatingtree.c b/Modules/rotatingtree.c index 217e495b3d2a9d..5910e25bed6389 100644 --- a/Modules/rotatingtree.c +++ b/Modules/rotatingtree.c @@ -1,9 +1,4 @@ -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 -#endif - #include "Python.h" -#include "pycore_lock.h" #include "rotatingtree.h" #define KEY_LOWER_THAN(key1, key2) ((char*)(key1) < (char*)(key2)) diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index f16173aafa7d3c..e14e114a6dafd0 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -262,7 +262,7 @@ A file descriptor is either a socket or file object, or a small integer gotten from a fileno() method call on one of those. The optional 4th argument specifies a timeout in seconds; it may be -a floating point number to specify fractions of seconds. If it is absent +a floating-point number to specify fractions of seconds. If it is absent or None, the call will never time out. The return value is a tuple of three lists corresponding to the first three @@ -277,7 +277,7 @@ descriptors can be used. static PyObject * select_select_impl(PyObject *module, PyObject *rlist, PyObject *wlist, PyObject *xlist, PyObject *timeout_obj) -/*[clinic end generated code: output=2b3cfa824f7ae4cf input=e467f5d68033de00]*/ +/*[clinic end generated code: output=2b3cfa824f7ae4cf input=1199d5e101abca4a]*/ { #ifdef SELECT_USES_HEAP pylist *rfd2obj, *wfd2obj, *efd2obj; @@ -473,6 +473,7 @@ update_ufd_array(pollObject *self) } /*[clinic input] +@critical_section select.poll.register fd: fildes @@ -486,7 +487,7 @@ Register a file descriptor with the polling object. static PyObject * select_poll_register_impl(pollObject *self, int fd, unsigned short eventmask) -/*[clinic end generated code: output=0dc7173c800a4a65 input=34e16cfb28d3c900]*/ +/*[clinic end generated code: output=0dc7173c800a4a65 input=c475e029ce6c2830]*/ { PyObject *key, *value; int err; @@ -514,6 +515,7 @@ select_poll_register_impl(pollObject *self, int fd, unsigned short eventmask) /*[clinic input] +@critical_section select.poll.modify fd: fildes @@ -528,7 +530,7 @@ Modify an already registered file descriptor. static PyObject * select_poll_modify_impl(pollObject *self, int fd, unsigned short eventmask) -/*[clinic end generated code: output=1a7b88bf079eff17 input=a8e383df075c32cf]*/ +/*[clinic end generated code: output=1a7b88bf079eff17 input=38c9db5346711872]*/ { PyObject *key, *value; int err; @@ -566,6 +568,7 @@ select_poll_modify_impl(pollObject *self, int fd, unsigned short eventmask) /*[clinic input] +@critical_section select.poll.unregister fd: fildes @@ -576,7 +579,7 @@ Remove a file descriptor being tracked by the polling object. static PyObject * select_poll_unregister_impl(pollObject *self, int fd) -/*[clinic end generated code: output=8c9f42e75e7d291b input=4b4fccc1040e79cb]*/ +/*[clinic end generated code: output=8c9f42e75e7d291b input=ae6315d7f5243704]*/ { PyObject *key; @@ -599,6 +602,7 @@ select_poll_unregister_impl(pollObject *self, int fd) } /*[clinic input] +@critical_section select.poll.poll timeout as timeout_obj: object = None @@ -614,7 +618,7 @@ report, as a list of (fd, event) 2-tuples. static PyObject * select_poll_poll_impl(pollObject *self, PyObject *timeout_obj) -/*[clinic end generated code: output=876e837d193ed7e4 input=c2f6953ec45e5622]*/ +/*[clinic end generated code: output=876e837d193ed7e4 input=54310631457efdec]*/ { PyObject *result_list = NULL; int poll_result, i, j; @@ -813,14 +817,14 @@ static int devpoll_flush(devpollObject *self) if (n < size) { /* - ** Data writed to /dev/poll is a binary data structure. It is not + ** Data written to /dev/poll is a binary data structure. It is not ** clear what to do if a partial write occurred. For now, raise ** an exception and see if we actually found this problem in ** the wild. - ** See http://bugs.python.org/issue6397. + ** See https://github.com/python/cpython/issues/50646. */ PyErr_Format(PyExc_OSError, "failed to write all pollfds. " - "Please, report at http://bugs.python.org/. " + "Please, report at https://github.com/python/cpython/issues/. " "Data to report: Size tried: %d, actual size written: %d.", size, n); return -1; @@ -857,6 +861,7 @@ internal_devpoll_register(devpollObject *self, int fd, } /*[clinic input] +@critical_section select.devpoll.register fd: fildes @@ -872,12 +877,13 @@ Register a file descriptor with the polling object. static PyObject * select_devpoll_register_impl(devpollObject *self, int fd, unsigned short eventmask) -/*[clinic end generated code: output=6e07fe8b74abba0c input=22006fabe9567522]*/ +/*[clinic end generated code: output=6e07fe8b74abba0c input=8d48bd2653a61c42]*/ { return internal_devpoll_register(self, fd, eventmask, 0); } /*[clinic input] +@critical_section select.devpoll.modify fd: fildes @@ -893,12 +899,13 @@ Modify a possible already registered file descriptor. static PyObject * select_devpoll_modify_impl(devpollObject *self, int fd, unsigned short eventmask) -/*[clinic end generated code: output=bc2e6d23aaff98b4 input=09fa335db7cdc09e]*/ +/*[clinic end generated code: output=bc2e6d23aaff98b4 input=773b37e9abca2460]*/ { return internal_devpoll_register(self, fd, eventmask, 1); } /*[clinic input] +@critical_section select.devpoll.unregister fd: fildes @@ -909,7 +916,7 @@ Remove a file descriptor being tracked by the polling object. static PyObject * select_devpoll_unregister_impl(devpollObject *self, int fd) -/*[clinic end generated code: output=95519ffa0c7d43fe input=b4ea42a4442fd467]*/ +/*[clinic end generated code: output=95519ffa0c7d43fe input=6052d368368d4d05]*/ { if (self->fd_devpoll < 0) return devpoll_err_closed(); @@ -926,6 +933,7 @@ select_devpoll_unregister_impl(devpollObject *self, int fd) } /*[clinic input] +@critical_section select.devpoll.poll timeout as timeout_obj: object = None The maximum time to wait in milliseconds, or else None (or a negative @@ -940,7 +948,7 @@ report, as a list of (fd, event) 2-tuples. static PyObject * select_devpoll_poll_impl(devpollObject *self, PyObject *timeout_obj) -/*[clinic end generated code: output=2654e5457cca0b3c input=3c3f0a355ec2bedb]*/ +/*[clinic end generated code: output=2654e5457cca0b3c input=fe7a3f6dcbc118c5]*/ { struct dvpoll dvp; PyObject *result_list = NULL; @@ -1059,6 +1067,7 @@ devpoll_internal_close(devpollObject *self) } /*[clinic input] +@critical_section select.devpoll.close Close the devpoll file descriptor. @@ -1068,7 +1077,7 @@ Further operations on the devpoll object will raise an exception. static PyObject * select_devpoll_close_impl(devpollObject *self) -/*[clinic end generated code: output=26b355bd6429f21b input=6273c30f5560a99b]*/ +/*[clinic end generated code: output=26b355bd6429f21b input=408fde21a377ccfb]*/ { errno = devpoll_internal_close(self); if (errno < 0) { @@ -1088,6 +1097,7 @@ devpoll_get_closed(devpollObject *self, void *Py_UNUSED(ignored)) } /*[clinic input] +@critical_section select.devpoll.fileno Return the file descriptor. @@ -1095,7 +1105,7 @@ Return the file descriptor. static PyObject * select_devpoll_fileno_impl(devpollObject *self) -/*[clinic end generated code: output=26920929f8d292f4 input=ef15331ebde6c368]*/ +/*[clinic end generated code: output=26920929f8d292f4 input=8c9db2efa1ade538]*/ { if (self->fd_devpoll < 0) return devpoll_err_closed(); @@ -1378,6 +1388,7 @@ pyepoll_dealloc(pyEpoll_Object *self) } /*[clinic input] +@critical_section select.epoll.close Close the epoll control file descriptor. @@ -1387,7 +1398,7 @@ Further operations on the epoll object will raise an exception. static PyObject * select_epoll_close_impl(pyEpoll_Object *self) -/*[clinic end generated code: output=ee2144c446a1a435 input=ca6c66ba5a736bfd]*/ +/*[clinic end generated code: output=ee2144c446a1a435 input=f626a769192e1dbe]*/ { errno = pyepoll_internal_close(self); if (errno < 0) { @@ -2023,10 +2034,8 @@ kqueue_tracking_init(PyObject *module) { } static int -kqueue_tracking_add(_selectstate *state, kqueue_queue_Object *self) { - if (!state->kqueue_tracking_initialized) { - kqueue_tracking_init(PyType_GetModule(Py_TYPE(self))); - } +kqueue_tracking_add_lock_held(_selectstate *state, kqueue_queue_Object *self) +{ assert(self->kqfd >= 0); _kqueue_list_item *item = PyMem_New(_kqueue_list_item, 1); if (item == NULL) { @@ -2039,8 +2048,23 @@ kqueue_tracking_add(_selectstate *state, kqueue_queue_Object *self) { return 0; } +static int +kqueue_tracking_add(_selectstate *state, kqueue_queue_Object *self) +{ + int ret; + PyObject *module = PyType_GetModule(Py_TYPE(self)); + Py_BEGIN_CRITICAL_SECTION(module); + if (!state->kqueue_tracking_initialized) { + kqueue_tracking_init(module); + } + ret = kqueue_tracking_add_lock_held(state, self); + Py_END_CRITICAL_SECTION(); + return ret; +} + static void -kqueue_tracking_remove(_selectstate *state, kqueue_queue_Object *self) { +kqueue_tracking_remove_lock_held(_selectstate *state, kqueue_queue_Object *self) +{ _kqueue_list *listptr = &state->kqueue_open_list; while (*listptr != NULL) { _kqueue_list_item *item = *listptr; @@ -2056,6 +2080,14 @@ kqueue_tracking_remove(_selectstate *state, kqueue_queue_Object *self) { assert(0); } +static void +kqueue_tracking_remove(_selectstate *state, kqueue_queue_Object *self) +{ + Py_BEGIN_CRITICAL_SECTION(PyType_GetModule(Py_TYPE(self))); + kqueue_tracking_remove_lock_held(state, self); + Py_END_CRITICAL_SECTION(); +} + static int kqueue_queue_internal_close(kqueue_queue_Object *self) { @@ -2150,6 +2182,7 @@ kqueue_queue_finalize(kqueue_queue_Object *self) } /*[clinic input] +@critical_section select.kqueue.close Close the kqueue control file descriptor. @@ -2159,7 +2192,7 @@ Further operations on the kqueue object will raise an exception. static PyObject * select_kqueue_close_impl(kqueue_queue_Object *self) -/*[clinic end generated code: output=d1c7df0b407a4bc1 input=0b12d95430e0634c]*/ +/*[clinic end generated code: output=d1c7df0b407a4bc1 input=6d763c858b17b690]*/ { errno = kqueue_queue_internal_close(self); if (errno < 0) { @@ -2682,6 +2715,10 @@ _select_exec(PyObject *m) #ifdef EPOLLMSG ADD_INT(EPOLLMSG); #endif +#ifdef EPOLLWAKEUP + /* Kernel 3.5+ */ + ADD_INT(EPOLLWAKEUP); +#endif #ifdef EPOLL_CLOEXEC ADD_INT(EPOLL_CLOEXEC); @@ -2802,6 +2839,7 @@ _select_exec(PyObject *m) static PyModuleDef_Slot _select_slots[] = { {Py_mod_exec, _select_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/sha1module.c b/Modules/sha1module.c index eda6b5608d52f7..d0b1e8250770d0 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -52,9 +52,11 @@ typedef struct { bool use_mutex; PyMutex mutex; PyThread_type_lock lock; - Hacl_Streaming_SHA1_state *hash_state; + Hacl_Hash_SHA1_state_t *hash_state; } SHA1object; +#define _SHA1object_CAST(op) ((SHA1object *)(op)) + #include "clinic/sha1module.c.h" @@ -73,7 +75,7 @@ sha1_get_state(PyObject *module) static SHA1object * newSHA1object(SHA1State *st) { - SHA1object *sha = (SHA1object *)PyObject_GC_New(SHA1object, st->sha1_type); + SHA1object *sha = PyObject_GC_New(SHA1object, st->sha1_type); if (sha == NULL) { return NULL; } @@ -93,9 +95,10 @@ SHA1_traverse(PyObject *ptr, visitproc visit, void *arg) } static void -SHA1_dealloc(SHA1object *ptr) +SHA1_dealloc(PyObject *op) { - Hacl_Streaming_SHA1_legacy_free(ptr->hash_state); + SHA1object *ptr = _SHA1object_CAST(op); + Hacl_Hash_SHA1_free(ptr->hash_state); PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -124,7 +127,7 @@ SHA1Type_copy_impl(SHA1object *self, PyTypeObject *cls) return NULL; ENTER_HASHLIB(self); - newobj->hash_state = Hacl_Streaming_SHA1_legacy_copy(self->hash_state); + newobj->hash_state = Hacl_Hash_SHA1_copy(self->hash_state); LEAVE_HASHLIB(self); return (PyObject *)newobj; } @@ -141,7 +144,7 @@ SHA1Type_digest_impl(SHA1object *self) { unsigned char digest[SHA1_DIGESTSIZE]; ENTER_HASHLIB(self); - Hacl_Streaming_SHA1_legacy_finish(self->hash_state, digest); + Hacl_Hash_SHA1_digest(self->hash_state, digest); LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, SHA1_DIGESTSIZE); } @@ -158,20 +161,20 @@ SHA1Type_hexdigest_impl(SHA1object *self) { unsigned char digest[SHA1_DIGESTSIZE]; ENTER_HASHLIB(self); - Hacl_Streaming_SHA1_legacy_finish(self->hash_state, digest); + Hacl_Hash_SHA1_digest(self->hash_state, digest); LEAVE_HASHLIB(self); return _Py_strhex((const char *)digest, SHA1_DIGESTSIZE); } -static void update(Hacl_Streaming_SHA1_state *state, uint8_t *buf, Py_ssize_t len) { +static void update(Hacl_Hash_SHA1_state_t *state, uint8_t *buf, Py_ssize_t len) { #if PY_SSIZE_T_MAX > UINT32_MAX while (len > UINT32_MAX) { - Hacl_Streaming_SHA1_legacy_update(state, buf, UINT32_MAX); + Hacl_Hash_SHA1_update(state, buf, UINT32_MAX); len -= UINT32_MAX; buf += UINT32_MAX; } #endif - Hacl_Streaming_SHA1_legacy_update(state, buf, (uint32_t) len); + Hacl_Hash_SHA1_update(state, buf, (uint32_t) len); } /*[clinic input] @@ -217,36 +220,27 @@ static PyMethodDef SHA1_methods[] = { }; static PyObject * -SHA1_get_block_size(PyObject *self, void *closure) +SHA1_get_block_size(PyObject *Py_UNUSED(self), void *Py_UNUSED(closure)) { return PyLong_FromLong(SHA1_BLOCKSIZE); } static PyObject * -SHA1_get_name(PyObject *self, void *closure) +SHA1_get_name(PyObject *Py_UNUSED(self), void *Py_UNUSED(closure)) { return PyUnicode_FromStringAndSize("sha1", 4); } static PyObject * -sha1_get_digest_size(PyObject *self, void *closure) +sha1_get_digest_size(PyObject *Py_UNUSED(self), void *Py_UNUSED(closure)) { return PyLong_FromLong(SHA1_DIGESTSIZE); } static PyGetSetDef SHA1_getseters[] = { - {"block_size", - (getter)SHA1_get_block_size, NULL, - NULL, - NULL}, - {"name", - (getter)SHA1_get_name, NULL, - NULL, - NULL}, - {"digest_size", - (getter)sha1_get_digest_size, NULL, - NULL, - NULL}, + {"block_size", SHA1_get_block_size, NULL, NULL, NULL}, + {"name", SHA1_get_name, NULL, NULL, NULL}, + {"digest_size", sha1_get_digest_size, NULL, NULL, NULL}, {NULL} /* Sentinel */ }; @@ -295,7 +289,7 @@ _sha1_sha1_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - new->hash_state = Hacl_Streaming_SHA1_legacy_create_in(); + new->hash_state = Hacl_Hash_SHA1_malloc(); if (PyErr_Occurred()) { Py_DECREF(new); @@ -346,7 +340,7 @@ _sha1_clear(PyObject *module) static void _sha1_free(void *module) { - _sha1_clear((PyObject *)module); + (void)_sha1_clear((PyObject *)module); } static int @@ -371,6 +365,7 @@ _sha1_exec(PyObject *module) static PyModuleDef_Slot _sha1_slots[] = { {Py_mod_exec, _sha1_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/sha2module.c b/Modules/sha2module.c index 968493ba51b50d..45fa120cf76758 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -55,7 +55,7 @@ typedef struct { // Prevents undefined behavior via multiple threads entering the C API. bool use_mutex; PyMutex mutex; - Hacl_Streaming_SHA2_state_sha2_256 *state; + Hacl_Hash_SHA2_state_t_256 *state; } SHA256object; typedef struct { @@ -64,9 +64,12 @@ typedef struct { // Prevents undefined behavior via multiple threads entering the C API. bool use_mutex; PyMutex mutex; - Hacl_Streaming_SHA2_state_sha2_512 *state; + Hacl_Hash_SHA2_state_t_512 *state; } SHA512object; +#define _SHA256object_CAST(op) ((SHA256object *)(op)) +#define _SHA512object_CAST(op) ((SHA512object *)(op)) + #include "clinic/sha2module.c.h" /* We shall use run-time type information in the remainder of this module to @@ -89,20 +92,19 @@ sha2_get_state(PyObject *module) static void SHA256copy(SHA256object *src, SHA256object *dest) { dest->digestsize = src->digestsize; - dest->state = Hacl_Streaming_SHA2_copy_256(src->state); + dest->state = Hacl_Hash_SHA2_copy_256(src->state); } static void SHA512copy(SHA512object *src, SHA512object *dest) { dest->digestsize = src->digestsize; - dest->state = Hacl_Streaming_SHA2_copy_512(src->state); + dest->state = Hacl_Hash_SHA2_copy_512(src->state); } static SHA256object * newSHA224object(sha2_state *state) { - SHA256object *sha = (SHA256object *)PyObject_GC_New( - SHA256object, state->sha224_type); + SHA256object *sha = PyObject_GC_New(SHA256object, state->sha224_type); if (!sha) { return NULL; } @@ -115,8 +117,7 @@ newSHA224object(sha2_state *state) static SHA256object * newSHA256object(sha2_state *state) { - SHA256object *sha = (SHA256object *)PyObject_GC_New( - SHA256object, state->sha256_type); + SHA256object *sha = PyObject_GC_New(SHA256object, state->sha256_type); if (!sha) { return NULL; } @@ -129,8 +130,7 @@ newSHA256object(sha2_state *state) static SHA512object * newSHA384object(sha2_state *state) { - SHA512object *sha = (SHA512object *)PyObject_GC_New( - SHA512object, state->sha384_type); + SHA512object *sha = PyObject_GC_New(SHA512object, state->sha384_type); if (!sha) { return NULL; } @@ -143,8 +143,7 @@ newSHA384object(sha2_state *state) static SHA512object * newSHA512object(sha2_state *state) { - SHA512object *sha = (SHA512object *)PyObject_GC_New( - SHA512object, state->sha512_type); + SHA512object *sha = PyObject_GC_New(SHA512object, state->sha512_type); if (!sha) { return NULL; } @@ -164,9 +163,10 @@ SHA2_traverse(PyObject *ptr, visitproc visit, void *arg) } static void -SHA256_dealloc(SHA256object *ptr) +SHA256_dealloc(PyObject *op) { - Hacl_Streaming_SHA2_free_256(ptr->state); + SHA256object *ptr = _SHA256object_CAST(op); + Hacl_Hash_SHA2_free_256(ptr->state); PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -174,9 +174,10 @@ SHA256_dealloc(SHA256object *ptr) } static void -SHA512_dealloc(SHA512object *ptr) +SHA512_dealloc(PyObject *op) { - Hacl_Streaming_SHA2_free_512(ptr->state); + SHA512object *ptr = _SHA512object_CAST(op); + Hacl_Hash_SHA2_free_512(ptr->state); PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -186,34 +187,34 @@ SHA512_dealloc(SHA512object *ptr) /* HACL* takes a uint32_t for the length of its parameter, but Py_ssize_t can be * 64 bits so we loop in <4gig chunks when needed. */ -static void update_256(Hacl_Streaming_SHA2_state_sha2_256 *state, uint8_t *buf, Py_ssize_t len) { +static void update_256(Hacl_Hash_SHA2_state_t_256 *state, uint8_t *buf, Py_ssize_t len) { /* Note: we explicitly ignore the error code on the basis that it would take > * 1 billion years to overflow the maximum admissible length for SHA2-256 * (namely, 2^61-1 bytes). */ #if PY_SSIZE_T_MAX > UINT32_MAX while (len > UINT32_MAX) { - Hacl_Streaming_SHA2_update_256(state, buf, UINT32_MAX); + Hacl_Hash_SHA2_update_256(state, buf, UINT32_MAX); len -= UINT32_MAX; buf += UINT32_MAX; } #endif /* Cast to uint32_t is safe: len <= UINT32_MAX at this point. */ - Hacl_Streaming_SHA2_update_256(state, buf, (uint32_t) len); + Hacl_Hash_SHA2_update_256(state, buf, (uint32_t) len); } -static void update_512(Hacl_Streaming_SHA2_state_sha2_512 *state, uint8_t *buf, Py_ssize_t len) { +static void update_512(Hacl_Hash_SHA2_state_t_512 *state, uint8_t *buf, Py_ssize_t len) { /* Note: we explicitly ignore the error code on the basis that it would take > * 1 billion years to overflow the maximum admissible length for this API * (namely, 2^64-1 bytes). */ #if PY_SSIZE_T_MAX > UINT32_MAX while (len > UINT32_MAX) { - Hacl_Streaming_SHA2_update_512(state, buf, UINT32_MAX); + Hacl_Hash_SHA2_update_512(state, buf, UINT32_MAX); len -= UINT32_MAX; buf += UINT32_MAX; } #endif /* Cast to uint32_t is safe: len <= UINT32_MAX at this point. */ - Hacl_Streaming_SHA2_update_512(state, buf, (uint32_t) len); + Hacl_Hash_SHA2_update_512(state, buf, (uint32_t) len); } @@ -296,7 +297,7 @@ SHA256Type_digest_impl(SHA256object *self) ENTER_HASHLIB(self); // HACL* performs copies under the hood so that self->state remains valid // after this call. - Hacl_Streaming_SHA2_finish_256(self->state, digest); + Hacl_Hash_SHA2_digest_256(self->state, digest); LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); } @@ -316,7 +317,7 @@ SHA512Type_digest_impl(SHA512object *self) ENTER_HASHLIB(self); // HACL* performs copies under the hood so that self->state remains valid // after this call. - Hacl_Streaming_SHA2_finish_512(self->state, digest); + Hacl_Hash_SHA2_digest_512(self->state, digest); LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); } @@ -334,7 +335,7 @@ SHA256Type_hexdigest_impl(SHA256object *self) uint8_t digest[SHA256_DIGESTSIZE]; assert(self->digestsize <= SHA256_DIGESTSIZE); ENTER_HASHLIB(self); - Hacl_Streaming_SHA2_finish_256(self->state, digest); + Hacl_Hash_SHA2_digest_256(self->state, digest); LEAVE_HASHLIB(self); return _Py_strhex((const char *)digest, self->digestsize); } @@ -352,7 +353,7 @@ SHA512Type_hexdigest_impl(SHA512object *self) uint8_t digest[SHA512_DIGESTSIZE]; assert(self->digestsize <= SHA512_DIGESTSIZE); ENTER_HASHLIB(self); - Hacl_Streaming_SHA2_finish_512(self->state, digest); + Hacl_Hash_SHA2_digest_512(self->state, digest); LEAVE_HASHLIB(self); return _Py_strhex((const char *)digest, self->digestsize); } @@ -442,32 +443,35 @@ static PyMethodDef SHA512_methods[] = { }; static PyObject * -SHA256_get_block_size(PyObject *self, void *closure) +SHA256_get_block_size(PyObject *Py_UNUSED(self), void *Py_UNUSED(closure)) { return PyLong_FromLong(SHA256_BLOCKSIZE); } static PyObject * -SHA512_get_block_size(PyObject *self, void *closure) +SHA512_get_block_size(PyObject *Py_UNUSED(self), void *Py_UNUSED(closure)) { return PyLong_FromLong(SHA512_BLOCKSIZE); } static PyObject * -SHA256_get_digest_size(SHA256object *self, void *closure) +SHA256_get_digest_size(PyObject *op, void *Py_UNUSED(closure)) { + SHA256object *self = _SHA256object_CAST(op); return PyLong_FromLong(self->digestsize); } static PyObject * -SHA512_get_digest_size(SHA512object *self, void *closure) +SHA512_get_digest_size(PyObject *op, void *Py_UNUSED(closure)) { + SHA512object *self = _SHA512object_CAST(op); return PyLong_FromLong(self->digestsize); } static PyObject * -SHA256_get_name(SHA256object *self, void *closure) +SHA256_get_name(PyObject *op, void *Py_UNUSED(closure)) { + SHA256object *self = _SHA256object_CAST(op); if (self->digestsize == 28) { return PyUnicode_FromStringAndSize("sha224", 6); } @@ -475,8 +479,9 @@ SHA256_get_name(SHA256object *self, void *closure) } static PyObject * -SHA512_get_name(SHA512object *self, void *closure) +SHA512_get_name(PyObject *op, void *Py_UNUSED(closure)) { + SHA512object *self = _SHA512object_CAST(op); if (self->digestsize == 64) { return PyUnicode_FromStringAndSize("sha512", 6); } @@ -484,34 +489,16 @@ SHA512_get_name(SHA512object *self, void *closure) } static PyGetSetDef SHA256_getseters[] = { - {"block_size", - (getter)SHA256_get_block_size, NULL, - NULL, - NULL}, - {"name", - (getter)SHA256_get_name, NULL, - NULL, - NULL}, - {"digest_size", - (getter)SHA256_get_digest_size, NULL, - NULL, - NULL}, + {"block_size", SHA256_get_block_size, NULL, NULL, NULL}, + {"name", SHA256_get_name, NULL, NULL, NULL}, + {"digest_size", SHA256_get_digest_size, NULL, NULL, NULL}, {NULL} /* Sentinel */ }; static PyGetSetDef SHA512_getseters[] = { - {"block_size", - (getter)SHA512_get_block_size, NULL, - NULL, - NULL}, - {"name", - (getter)SHA512_get_name, NULL, - NULL, - NULL}, - {"digest_size", - (getter)SHA512_get_digest_size, NULL, - NULL, - NULL}, + {"block_size", SHA512_get_block_size, NULL, NULL, NULL}, + {"name", SHA512_get_name, NULL, NULL, NULL}, + {"digest_size", SHA512_get_digest_size, NULL, NULL, NULL}, {NULL} /* Sentinel */ }; @@ -597,7 +584,7 @@ _sha2_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - new->state = Hacl_Streaming_SHA2_create_in_256(); + new->state = Hacl_Hash_SHA2_malloc_256(); new->digestsize = 32; if (PyErr_Occurred()) { @@ -651,7 +638,7 @@ _sha2_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - new->state = Hacl_Streaming_SHA2_create_in_224(); + new->state = Hacl_Hash_SHA2_malloc_224(); new->digestsize = 28; if (PyErr_Occurred()) { @@ -705,7 +692,7 @@ _sha2_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - new->state = Hacl_Streaming_SHA2_create_in_512(); + new->state = Hacl_Hash_SHA2_malloc_512(); new->digestsize = 64; if (PyErr_Occurred()) { @@ -758,7 +745,7 @@ _sha2_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - new->state = Hacl_Streaming_SHA2_create_in_384(); + new->state = Hacl_Hash_SHA2_malloc_384(); new->digestsize = 48; if (PyErr_Occurred()) { @@ -818,7 +805,7 @@ _sha2_clear(PyObject *module) static void _sha2_free(void *module) { - _sha2_clear((PyObject *)module); + (void)_sha2_clear((PyObject *)module); } /* Initialize this module. */ @@ -866,6 +853,7 @@ static int sha2_exec(PyObject *module) static PyModuleDef_Slot _sha2_slots[] = { {Py_mod_exec, sha2_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/sha3module.c b/Modules/sha3module.c index d9d2f6c385a68b..72a11602b0e1fd 100644 --- a/Modules/sha3module.c +++ b/Modules/sha3module.c @@ -63,36 +63,38 @@ typedef struct { // Prevents undefined behavior via multiple threads entering the C API. bool use_mutex; PyMutex mutex; - Hacl_Streaming_Keccak_state *hash_state; + Hacl_Hash_SHA3_state_t *hash_state; } SHA3object; +#define _SHA3object_CAST(op) ((SHA3object *)(op)) + #include "clinic/sha3module.c.h" static SHA3object * newSHA3object(PyTypeObject *type) { - SHA3object *newobj; - newobj = (SHA3object *)PyObject_New(SHA3object, type); + SHA3object *newobj = PyObject_GC_New(SHA3object, type); if (newobj == NULL) { return NULL; } HASHLIB_INIT_MUTEX(newobj); + PyObject_GC_Track(newobj); return newobj; } -static void sha3_update(Hacl_Streaming_Keccak_state *state, uint8_t *buf, Py_ssize_t len) { +static void sha3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *buf, Py_ssize_t len) { /* Note: we explicitly ignore the error code on the basis that it would take > * 1 billion years to hash more than 2^64 bytes. */ #if PY_SSIZE_T_MAX > UINT32_MAX while (len > UINT32_MAX) { - Hacl_Streaming_Keccak_update(state, buf, UINT32_MAX); + Hacl_Hash_SHA3_update(state, buf, UINT32_MAX); len -= UINT32_MAX; buf += UINT32_MAX; } #endif /* Cast to uint32_t is safe: len <= UINT32_MAX at this point. */ - Hacl_Streaming_Keccak_update(state, buf, (uint32_t) len); + Hacl_Hash_SHA3_update(state, buf, (uint32_t) len); } /*[clinic input] @@ -120,17 +122,17 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) assert(state != NULL); if (type == state->sha3_224_type) { - self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_224); + self->hash_state = Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_SHA3_224); } else if (type == state->sha3_256_type) { - self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_256); + self->hash_state = Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_SHA3_256); } else if (type == state->sha3_384_type) { - self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_384); + self->hash_state = Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_SHA3_384); } else if (type == state->sha3_512_type) { - self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_SHA3_512); + self->hash_state = Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_SHA3_512); } else if (type == state->shake_128_type) { - self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_Shake128); + self->hash_state = Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_Shake128); } else if (type == state->shake_256_type) { - self->hash_state = Hacl_Streaming_Keccak_malloc(Spec_Hash_Definitions_Shake256); + self->hash_state = Hacl_Hash_SHA3_malloc(Spec_Hash_Definitions_Shake256); } else { PyErr_BadInternalCall(); goto error; @@ -166,15 +168,33 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) /* Internal methods for a hash object */ +static int +SHA3_clear(PyObject *op) +{ + SHA3object *self = _SHA3object_CAST(op); + if (self->hash_state != NULL) { + Hacl_Hash_SHA3_free(self->hash_state); + self->hash_state = NULL; + } + return 0; +} + static void -SHA3_dealloc(SHA3object *self) +SHA3_dealloc(PyObject *self) { - Hacl_Streaming_Keccak_free(self->hash_state); PyTypeObject *tp = Py_TYPE(self); - PyObject_Free(self); + PyObject_GC_UnTrack(self); + (void)SHA3_clear(self); + tp->tp_free(self); Py_DECREF(tp); } +static int +SHA3_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} /* External methods for a hash object */ @@ -195,7 +215,7 @@ _sha3_sha3_224_copy_impl(SHA3object *self) return NULL; } ENTER_HASHLIB(self); - newobj->hash_state = Hacl_Streaming_Keccak_copy(self->hash_state); + newobj->hash_state = Hacl_Hash_SHA3_copy(self->hash_state); LEAVE_HASHLIB(self); return (PyObject *)newobj; } @@ -215,10 +235,10 @@ _sha3_sha3_224_digest_impl(SHA3object *self) // This function errors out if the algorithm is Shake. Here, we know this // not to be the case, and therefore do not perform error checking. ENTER_HASHLIB(self); - Hacl_Streaming_Keccak_finish(self->hash_state, digest); + Hacl_Hash_SHA3_digest(self->hash_state, digest); LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, - Hacl_Streaming_Keccak_hash_len(self->hash_state)); + Hacl_Hash_SHA3_hash_len(self->hash_state)); } @@ -234,10 +254,10 @@ _sha3_sha3_224_hexdigest_impl(SHA3object *self) { unsigned char digest[SHA3_MAX_DIGESTSIZE]; ENTER_HASHLIB(self); - Hacl_Streaming_Keccak_finish(self->hash_state, digest); + Hacl_Hash_SHA3_digest(self->hash_state, digest); LEAVE_HASHLIB(self); return _Py_strhex((const char *)digest, - Hacl_Streaming_Keccak_hash_len(self->hash_state)); + Hacl_Hash_SHA3_hash_len(self->hash_state)); } @@ -286,15 +306,16 @@ static PyMethodDef SHA3_methods[] = { static PyObject * -SHA3_get_block_size(SHA3object *self, void *closure) +SHA3_get_block_size(PyObject *op, void *Py_UNUSED(closure)) { - uint32_t rate = Hacl_Streaming_Keccak_block_len(self->hash_state); + SHA3object *self = _SHA3object_CAST(op); + uint32_t rate = Hacl_Hash_SHA3_block_len(self->hash_state); return PyLong_FromLong(rate); } static PyObject * -SHA3_get_name(SHA3object *self, void *closure) +SHA3_get_name(PyObject *self, void *Py_UNUSED(closure)) { PyTypeObject *type = Py_TYPE(self); @@ -321,57 +342,63 @@ SHA3_get_name(SHA3object *self, void *closure) static PyObject * -SHA3_get_digest_size(SHA3object *self, void *closure) +SHA3_get_digest_size(PyObject *op, void *Py_UNUSED(closure)) { // Preserving previous behavior: variable-length algorithms return 0 - if (Hacl_Streaming_Keccak_is_shake(self->hash_state)) + SHA3object *self = _SHA3object_CAST(op); + if (Hacl_Hash_SHA3_is_shake(self->hash_state)) return PyLong_FromLong(0); else - return PyLong_FromLong(Hacl_Streaming_Keccak_hash_len(self->hash_state)); + return PyLong_FromLong(Hacl_Hash_SHA3_hash_len(self->hash_state)); } static PyObject * -SHA3_get_capacity_bits(SHA3object *self, void *closure) +SHA3_get_capacity_bits(PyObject *op, void *Py_UNUSED(closure)) { - uint32_t rate = Hacl_Streaming_Keccak_block_len(self->hash_state) * 8; + SHA3object *self = _SHA3object_CAST(op); + uint32_t rate = Hacl_Hash_SHA3_block_len(self->hash_state) * 8; + assert(rate <= 1600); int capacity = 1600 - rate; return PyLong_FromLong(capacity); } static PyObject * -SHA3_get_rate_bits(SHA3object *self, void *closure) +SHA3_get_rate_bits(PyObject *op, void *Py_UNUSED(closure)) { - uint32_t rate = Hacl_Streaming_Keccak_block_len(self->hash_state) * 8; + SHA3object *self = _SHA3object_CAST(op); + uint32_t rate = Hacl_Hash_SHA3_block_len(self->hash_state) * 8; return PyLong_FromLong(rate); } static PyObject * -SHA3_get_suffix(SHA3object *self, void *closure) +SHA3_get_suffix(PyObject *Py_UNUSED(self), void *Py_UNUSED(closure)) { unsigned char suffix[2] = {0x06, 0}; return PyBytes_FromStringAndSize((const char *)suffix, 1); } static PyGetSetDef SHA3_getseters[] = { - {"block_size", (getter)SHA3_get_block_size, NULL, NULL, NULL}, - {"name", (getter)SHA3_get_name, NULL, NULL, NULL}, - {"digest_size", (getter)SHA3_get_digest_size, NULL, NULL, NULL}, - {"_capacity_bits", (getter)SHA3_get_capacity_bits, NULL, NULL, NULL}, - {"_rate_bits", (getter)SHA3_get_rate_bits, NULL, NULL, NULL}, - {"_suffix", (getter)SHA3_get_suffix, NULL, NULL, NULL}, + {"block_size", SHA3_get_block_size, NULL, NULL, NULL}, + {"name", SHA3_get_name, NULL, NULL, NULL}, + {"digest_size", SHA3_get_digest_size, NULL, NULL, NULL}, + {"_capacity_bits", SHA3_get_capacity_bits, NULL, NULL, NULL}, + {"_rate_bits", SHA3_get_rate_bits, NULL, NULL, NULL}, + {"_suffix", SHA3_get_suffix, NULL, NULL, NULL}, {NULL} /* Sentinel */ }; #define SHA3_TYPE_SLOTS(type_slots_obj, type_doc, type_methods, type_getseters) \ static PyType_Slot type_slots_obj[] = { \ + {Py_tp_clear, SHA3_clear}, \ {Py_tp_dealloc, SHA3_dealloc}, \ + {Py_tp_traverse, SHA3_traverse}, \ {Py_tp_doc, (char*)type_doc}, \ {Py_tp_methods, type_methods}, \ {Py_tp_getset, type_getseters}, \ {Py_tp_new, py_sha3_new}, \ - {0,0} \ + {0, NULL} \ } // Using _PyType_GetModuleState() on these types is safe since they @@ -380,7 +407,8 @@ static PyGetSetDef SHA3_getseters[] = { static PyType_Spec type_spec_obj = { \ .name = "_sha3." type_name, \ .basicsize = sizeof(SHA3object), \ - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, \ + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE \ + | Py_TPFLAGS_HAVE_GC, \ .slots = type_slots \ } @@ -417,10 +445,11 @@ SHA3_TYPE_SLOTS(sha3_512_slots, sha3_512__doc__, SHA3_methods, SHA3_getseters); SHA3_TYPE_SPEC(sha3_512_spec, "sha3_512", sha3_512_slots); static PyObject * -_SHAKE_digest(SHA3object *self, unsigned long digestlen, int hex) +_SHAKE_digest(PyObject *op, unsigned long digestlen, int hex) { unsigned char *digest = NULL; PyObject *result = NULL; + SHA3object *self = _SHA3object_CAST(op); if (digestlen >= (1 << 29)) { PyErr_SetString(PyExc_ValueError, "length is too large"); @@ -432,11 +461,11 @@ _SHAKE_digest(SHA3object *self, unsigned long digestlen, int hex) } /* Get the raw (binary) digest value. The HACL functions errors out if: - * - the algorith is not shake -- not the case here + * - the algorithm is not shake -- not the case here * - the output length is zero -- we follow the existing behavior and return * an empty digest, without raising an error */ if (digestlen > 0) { - Hacl_Streaming_Keccak_squeeze(self->hash_state, digest, digestlen); + Hacl_Hash_SHA3_squeeze(self->hash_state, digest, digestlen); } if (hex) { result = _Py_strhex((const char *)digest, digestlen); @@ -444,9 +473,7 @@ _SHAKE_digest(SHA3object *self, unsigned long digestlen, int hex) result = PyBytes_FromStringAndSize((const char *)digest, digestlen); } - if (digest != NULL) { - PyMem_Free(digest); - } + PyMem_Free(digest); return result; } @@ -464,7 +491,7 @@ static PyObject * _sha3_shake_128_digest_impl(SHA3object *self, unsigned long length) /*[clinic end generated code: output=2313605e2f87bb8f input=418ef6a36d2e6082]*/ { - return _SHAKE_digest(self, length, 0); + return _SHAKE_digest((PyObject *)self, length, 0); } @@ -481,17 +508,17 @@ static PyObject * _sha3_shake_128_hexdigest_impl(SHA3object *self, unsigned long length) /*[clinic end generated code: output=bf8e2f1e490944a8 input=69fb29b0926ae321]*/ { - return _SHAKE_digest(self, length, 1); + return _SHAKE_digest((PyObject *)self, length, 1); } static PyObject * -SHAKE_get_digest_size(SHA3object *self, void *closure) +SHAKE_get_digest_size(PyObject *Py_UNUSED(self), void *Py_UNUSED(closure)) { return PyLong_FromLong(0); } static PyObject * -SHAKE_get_suffix(SHA3object *self, void *closure) +SHAKE_get_suffix(PyObject *Py_UNUSED(self), void *Py_UNUSED(closure)) { unsigned char suffix[2] = {0x1f, 0}; return PyBytes_FromStringAndSize((const char *)suffix, 1); @@ -499,12 +526,12 @@ SHAKE_get_suffix(SHA3object *self, void *closure) static PyGetSetDef SHAKE_getseters[] = { - {"block_size", (getter)SHA3_get_block_size, NULL, NULL, NULL}, - {"name", (getter)SHA3_get_name, NULL, NULL, NULL}, - {"digest_size", (getter)SHAKE_get_digest_size, NULL, NULL, NULL}, - {"_capacity_bits", (getter)SHA3_get_capacity_bits, NULL, NULL, NULL}, - {"_rate_bits", (getter)SHA3_get_rate_bits, NULL, NULL, NULL}, - {"_suffix", (getter)SHAKE_get_suffix, NULL, NULL, NULL}, + {"block_size", SHA3_get_block_size, NULL, NULL, NULL}, + {"name", SHA3_get_name, NULL, NULL, NULL}, + {"digest_size", SHAKE_get_digest_size, NULL, NULL, NULL}, + {"_capacity_bits", SHA3_get_capacity_bits, NULL, NULL, NULL}, + {"_rate_bits", SHA3_get_rate_bits, NULL, NULL, NULL}, + {"_suffix", SHAKE_get_suffix, NULL, NULL, NULL}, {NULL} /* Sentinel */ }; @@ -563,7 +590,7 @@ _sha3_clear(PyObject *module) static void _sha3_free(void *module) { - _sha3_clear((PyObject *)module); + (void)_sha3_clear((PyObject *)module); } static int @@ -602,6 +629,7 @@ _sha3_exec(PyObject *m) static PyModuleDef_Slot _sha3_slots[] = { {Py_mod_exec, _sha3_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 5804e30af1b426..0e53a36bca55f0 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -638,7 +638,7 @@ signal_strsignal_impl(PyObject *module, int signalnum) res = "Aborted"; break; case SIGFPE: - res = "Floating point exception"; + res = "Floating-point exception"; break; case SIGSEGV: res = "Segmentation fault"; @@ -706,35 +706,43 @@ signal_siginterrupt_impl(PyObject *module, int signalnum, int flag) #endif -static PyObject* -signal_set_wakeup_fd(PyObject *self, PyObject *args, PyObject *kwds) +/*[clinic input] +signal.set_wakeup_fd + + fd as fdobj: object + / + * + warn_on_full_buffer: bool = True + +Sets the fd to be written to (with the signal number) when a signal comes in. + +A library can use this to wakeup select or poll. +The previous fd or -1 is returned. + +The fd must be non-blocking. +[clinic start generated code]*/ + +static PyObject * +signal_set_wakeup_fd_impl(PyObject *module, PyObject *fdobj, + int warn_on_full_buffer) +/*[clinic end generated code: output=2280d72dd2a54c4f input=5b545946a28b8339]*/ { struct _Py_stat_struct status; - static char *kwlist[] = { - "", "warn_on_full_buffer", NULL, - }; - int warn_on_full_buffer = 1; #ifdef MS_WINDOWS - PyObject *fdobj; SOCKET_T sockfd, old_sockfd; int res; int res_size = sizeof res; PyObject *mod; int is_socket; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|$p:set_wakeup_fd", kwlist, - &fdobj, &warn_on_full_buffer)) - return NULL; - sockfd = PyLong_AsSocket_t(fdobj); if (sockfd == (SOCKET_T)(-1) && PyErr_Occurred()) return NULL; #else - int fd; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "i|$p:set_wakeup_fd", kwlist, - &fd, &warn_on_full_buffer)) + int fd = PyLong_AsInt(fdobj); + if (fd == -1 && PyErr_Occurred()) { return NULL; + } #endif PyThreadState *tstate = _PyThreadState_GET(); @@ -820,15 +828,6 @@ signal_set_wakeup_fd(PyObject *self, PyObject *args, PyObject *kwds) #endif } -PyDoc_STRVAR(set_wakeup_fd_doc, -"set_wakeup_fd(fd, *, warn_on_full_buffer=True) -> fd\n\ -\n\ -Sets the fd to be written to (with the signal number) when a signal\n\ -comes in. A library can use this to wakeup select or poll.\n\ -The previous fd or -1 is returned.\n\ -\n\ -The fd must be non-blocking."); - /* C API for the same, without all the error checking */ int PySignal_SetWakeupFd(int fd) @@ -1200,13 +1199,13 @@ signal.sigtimedwait Like sigwaitinfo(), but with a timeout. -The timeout is specified in seconds, with floating point numbers allowed. +The timeout is specified in seconds, with floating-point numbers allowed. [clinic start generated code]*/ static PyObject * signal_sigtimedwait_impl(PyObject *module, sigset_t sigset, PyObject *timeout_obj) -/*[clinic end generated code: output=59c8971e8ae18a64 input=87fd39237cf0b7ba]*/ +/*[clinic end generated code: output=59c8971e8ae18a64 input=955773219c1596cd]*/ { PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, @@ -1300,7 +1299,9 @@ signal_pthread_kill_impl(PyObject *module, unsigned long thread_id, #endif /* #if defined(HAVE_PTHREAD_KILL) */ -#if defined(__linux__) && defined(__NR_pidfd_send_signal) +// This system call always crashes on older Android versions. +#if defined(__linux__) && defined(__NR_pidfd_send_signal) && \ + !(defined(__ANDROID__) && __ANDROID_API__ < 31) /*[clinic input] signal.pidfd_send_signal @@ -1344,7 +1345,7 @@ static PyMethodDef signal_methods[] = { SIGNAL_RAISE_SIGNAL_METHODDEF SIGNAL_STRSIGNAL_METHODDEF SIGNAL_GETSIGNAL_METHODDEF - {"set_wakeup_fd", _PyCFunction_CAST(signal_set_wakeup_fd), METH_VARARGS | METH_KEYWORDS, set_wakeup_fd_doc}, + SIGNAL_SET_WAKEUP_FD_METHODDEF SIGNAL_SIGINTERRUPT_METHODDEF SIGNAL_PAUSE_METHODDEF SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF @@ -1699,6 +1700,7 @@ _signal_module_free(void *module) static PyModuleDef_Slot signal_slots[] = { {Py_mod_exec, signal_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index cd9a803648be71..01811afa578119 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -110,6 +110,8 @@ Local naming conventions: #include "pycore_fileutils.h" // _Py_set_inheritable() #include "pycore_moduleobject.h" // _PyModule_GetState #include "pycore_time.h" // _PyTime_AsMilliseconds() +#include "pycore_pystate.h" // _Py_AssertHoldsTstate() +#include "pycore_pyatomic_ft_wrappers.h" #ifdef _Py_MEMORY_SANITIZER # include @@ -549,20 +551,58 @@ typedef struct _socket_state { /* Default timeout for new sockets */ PyTime_t defaulttimeout; +} socket_state; #if defined(HAVE_ACCEPT) || defined(HAVE_ACCEPT4) #if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) - /* accept4() is available on Linux 2.6.28+ and glibc 2.10 */ - int accept4_works; +/* accept4() is available on Linux 2.6.28+ and glibc 2.10 */ +static int accept4_works = -1; #endif #endif #ifdef SOCK_CLOEXEC - /* socket() and socketpair() fail with EINVAL on Linux kernel older - * than 2.6.27 if SOCK_CLOEXEC flag is set in the socket type. */ - int sock_cloexec_works; +/* socket() and socketpair() fail with EINVAL on Linux kernel older + * than 2.6.27 if SOCK_CLOEXEC flag is set in the socket type. */ +static int sock_cloexec_works = -1; #endif -} socket_state; + +static inline void +set_sock_fd(PySocketSockObject *s, SOCKET_T fd) +{ +#ifdef Py_GIL_DISABLED +#if SIZEOF_SOCKET_T == SIZEOF_INT + _Py_atomic_store_int_relaxed((int *)&s->sock_fd, (int)fd); +#elif SIZEOF_SOCKET_T == SIZEOF_LONG + _Py_atomic_store_long_relaxed((long *)&s->sock_fd, (long)fd); +#elif SIZEOF_SOCKET_T == SIZEOF_LONG_LONG + _Py_atomic_store_llong_relaxed((long long *)&s->sock_fd, (long long)fd); +#else + #error "Unsupported SIZEOF_SOCKET_T" +#endif +#else + s->sock_fd = fd; +#endif +} + +static inline SOCKET_T +get_sock_fd(PySocketSockObject *s) +{ +#ifdef Py_GIL_DISABLED +#if SIZEOF_SOCKET_T == SIZEOF_INT + return (SOCKET_T)_Py_atomic_load_int_relaxed((int *)&s->sock_fd); +#elif SIZEOF_SOCKET_T == SIZEOF_LONG + return (SOCKET_T)_Py_atomic_load_long_relaxed((long *)&s->sock_fd); +#elif SIZEOF_SOCKET_T == SIZEOF_LONG_LONG + return (SOCKET_T)_Py_atomic_load_llong_relaxed((long long *)&s->sock_fd); +#else + #error "Unsupported SIZEOF_SOCKET_T" +#endif +#else + return s->sock_fd; +#endif +} + +#define _PySocketSockObject_CAST(op) ((PySocketSockObject *)(op)) static inline socket_state * get_module_state(PyObject *mod) @@ -736,10 +776,10 @@ internal_setblocking(PySocketSockObject *s, int block) #ifndef MS_WINDOWS #if (defined(HAVE_SYS_IOCTL_H) && defined(FIONBIO)) block = !block; - if (ioctl(s->sock_fd, FIONBIO, (unsigned int *)&block) == -1) + if (ioctl(get_sock_fd(s), FIONBIO, (unsigned int *)&block) == -1) goto done; #else - delay_flag = fcntl(s->sock_fd, F_GETFL, 0); + delay_flag = fcntl(get_sock_fd(s), F_GETFL, 0); if (delay_flag == -1) goto done; if (block) @@ -747,12 +787,12 @@ internal_setblocking(PySocketSockObject *s, int block) else new_delay_flag = delay_flag | O_NONBLOCK; if (new_delay_flag != delay_flag) - if (fcntl(s->sock_fd, F_SETFL, new_delay_flag) == -1) + if (fcntl(get_sock_fd(s), F_SETFL, new_delay_flag) == -1) goto done; #endif #else /* MS_WINDOWS */ arg = !block; - if (ioctlsocket(s->sock_fd, FIONBIO, &arg) != 0) + if (ioctlsocket(get_sock_fd(s), FIONBIO, &arg) != 0) goto done; #endif /* MS_WINDOWS */ @@ -785,20 +825,20 @@ internal_select(PySocketSockObject *s, int writing, PyTime_t interval, struct timeval tv, *tvp; #endif - /* must be called with the GIL held */ - assert(PyGILState_Check()); + /* must be called with a thread state */ + _Py_AssertHoldsTstate(); /* Error condition is for output only */ assert(!(connect && !writing)); /* Guard against closed socket */ - if (s->sock_fd == INVALID_SOCKET) + if (get_sock_fd(s) == INVALID_SOCKET) return 0; /* Prefer poll, if available, since you can poll() any fd * which can't be done with select(). */ #ifdef HAVE_POLL - pollfd.fd = s->sock_fd; + pollfd.fd = get_sock_fd(s); pollfd.events = writing ? POLLOUT : POLLIN; if (connect) { /* On Windows, the socket becomes writable on connection success, @@ -810,7 +850,9 @@ internal_select(PySocketSockObject *s, int writing, PyTime_t interval, /* s->sock_timeout is in seconds, timeout in ms */ ms = _PyTime_AsMilliseconds(interval, _PyTime_ROUND_CEILING); - assert(ms <= INT_MAX); + if (ms > INT_MAX) { + ms = INT_MAX; + } /* On some OSes, typically BSD-based ones, the timeout parameter of the poll() syscall, when negative, must be exactly INFTIM, where defined, @@ -822,6 +864,7 @@ internal_select(PySocketSockObject *s, int writing, PyTime_t interval, ms = -1; #endif } + assert(INT_MIN <= ms && ms <= INT_MAX); Py_BEGIN_ALLOW_THREADS; n = poll(&pollfd, 1, (int)ms); @@ -835,23 +878,23 @@ internal_select(PySocketSockObject *s, int writing, PyTime_t interval, tvp = NULL; FD_ZERO(&fds); - FD_SET(s->sock_fd, &fds); + FD_SET(get_sock_fd(s), &fds); FD_ZERO(&efds); if (connect) { /* On Windows, the socket becomes writable on connection success, but a connection failure is notified as an error. On POSIX, the socket becomes writable on connection success or on connection failure. */ - FD_SET(s->sock_fd, &efds); + FD_SET(get_sock_fd(s), &efds); } /* See if the socket is ready */ Py_BEGIN_ALLOW_THREADS; if (writing) - n = select(Py_SAFE_DOWNCAST(s->sock_fd+1, SOCKET_T, int), + n = select(Py_SAFE_DOWNCAST(get_sock_fd(s)+1, SOCKET_T, int), NULL, &fds, &efds, tvp); else - n = select(Py_SAFE_DOWNCAST(s->sock_fd+1, SOCKET_T, int), + n = select(Py_SAFE_DOWNCAST(get_sock_fd(s)+1, SOCKET_T, int), &fds, NULL, &efds, tvp); Py_END_ALLOW_THREADS; #endif @@ -896,8 +939,8 @@ sock_call_ex(PySocketSockObject *s, int deadline_initialized = 0; int res; - /* sock_call() must be called with the GIL held. */ - assert(PyGILState_Check()); + /* sock_call() must be called with a thread state. */ + _Py_AssertHoldsTstate(); /* outer loop to retry select() when select() is interrupted by a signal or to retry select()+sock_func() on false positive (see above) */ @@ -1027,7 +1070,7 @@ static int init_sockobject(socket_state *state, PySocketSockObject *s, SOCKET_T fd, int family, int type, int proto) { - s->sock_fd = fd; + set_sock_fd(s, fd); s->sock_family = family; s->sock_type = type; @@ -1054,8 +1097,8 @@ init_sockobject(socket_state *state, PySocketSockObject *s, else #endif { - s->sock_timeout = state->defaulttimeout; - if (state->defaulttimeout >= 0) { + s->sock_timeout = _Py_atomic_load_int64_relaxed(&state->defaulttimeout); + if (s->sock_timeout >= 0) { if (internal_setblocking(s, 0) == -1) { return -1; } @@ -1887,12 +1930,14 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, #ifdef AF_RDS case AF_RDS: - /* RDS sockets use sockaddr_in: fall-through */ + /* RDS sockets use sockaddr_in */ + _Py_FALLTHROUGH; #endif /* AF_RDS */ #ifdef AF_DIVERT case AF_DIVERT: - /* FreeBSD divert(4) sockets use sockaddr_in: fall-through */ + /* FreeBSD divert(4) sockets use sockaddr_in */ + _Py_FALLTHROUGH; #endif /* AF_DIVERT */ case AF_INET: @@ -2122,7 +2167,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, } strncpy(ifr.ifr_name, interfaceName, sizeof(ifr.ifr_name)); ifr.ifr_name[(sizeof(ifr.ifr_name))-1] = '\0'; - if (ioctl(s->sock_fd, SIOCGIFINDEX, &ifr) < 0) { + if (ioctl(get_sock_fd(s), SIOCGIFINDEX, &ifr) < 0) { s->errorhandler(); PyBuffer_Release(&haddr); return 0; @@ -2214,7 +2259,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, switch (s->sock_proto) { #ifdef CAN_RAW case CAN_RAW: - /* fall-through */ + _Py_FALLTHROUGH; #endif #ifdef CAN_BCM case CAN_BCM: @@ -2247,7 +2292,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, } else if ((size_t)len < sizeof(ifr.ifr_name)) { strncpy(ifr.ifr_name, PyBytes_AS_STRING(interfaceName), sizeof(ifr.ifr_name)); ifr.ifr_name[(sizeof(ifr.ifr_name))-1] = '\0'; - if (ioctl(s->sock_fd, SIOCGIFINDEX, &ifr) < 0) { + if (ioctl(get_sock_fd(s), SIOCGIFINDEX, &ifr) < 0) { s->errorhandler(); Py_DECREF(interfaceName); return 0; @@ -2291,7 +2336,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, } else if ((size_t)len < sizeof(ifr.ifr_name)) { strncpy(ifr.ifr_name, PyBytes_AS_STRING(interfaceName), sizeof(ifr.ifr_name)); ifr.ifr_name[(sizeof(ifr.ifr_name))-1] = '\0'; - if (ioctl(s->sock_fd, SIOCGIFINDEX, &ifr) < 0) { + if (ioctl(get_sock_fd(s), SIOCGIFINDEX, &ifr) < 0) { s->errorhandler(); Py_DECREF(interfaceName); return 0; @@ -2339,7 +2384,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, } else if ((size_t)len < sizeof(ifr.ifr_name)) { strncpy(ifr.ifr_name, PyBytes_AS_STRING(interfaceName), sizeof(ifr.ifr_name)); ifr.ifr_name[(sizeof(ifr.ifr_name))-1] = '\0'; - if (ioctl(s->sock_fd, SIOCGIFINDEX, &ifr) < 0) { + if (ioctl(get_sock_fd(s), SIOCGIFINDEX, &ifr) < 0) { s->errorhandler(); Py_DECREF(interfaceName); return 0; @@ -2398,7 +2443,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, sizeof(info.ctl_name)); Py_DECREF(ctl_name); - if (ioctl(s->sock_fd, CTLIOCGINFO, &info)) { + if (ioctl(get_sock_fd(s), CTLIOCGINFO, &info)) { PyErr_SetString(PyExc_OSError, "cannot find kernel control with provided name"); return 0; @@ -2590,7 +2635,8 @@ getsockaddrlen(PySocketSockObject *s, socklen_t *len_ret) #ifdef AF_RDS case AF_RDS: - /* RDS sockets use sockaddr_in: fall-through */ + /* RDS sockets use sockaddr_in */ + _Py_FALLTHROUGH; #endif /* AF_RDS */ case AF_INET: @@ -2861,19 +2907,18 @@ sock_accept_impl(PySocketSockObject *s, void *data) #endif #if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) - socket_state *state = s->state; - if (state->accept4_works != 0) { - ctx->result = accept4(s->sock_fd, addr, paddrlen, + if (_Py_atomic_load_int_relaxed(&accept4_works) != 0) { + ctx->result = accept4(get_sock_fd(s), addr, paddrlen, SOCK_CLOEXEC); - if (ctx->result == INVALID_SOCKET && state->accept4_works == -1) { + if (ctx->result == INVALID_SOCKET && _Py_atomic_load_int_relaxed(&accept4_works) == -1) { /* On Linux older than 2.6.28, accept4() fails with ENOSYS */ - state->accept4_works = (errno != ENOSYS); + _Py_atomic_store_int_relaxed(&accept4_works, errno != ENOSYS); } } - if (state->accept4_works == 0) - ctx->result = accept(s->sock_fd, addr, paddrlen); + if (_Py_atomic_load_int_relaxed(&accept4_works) == 0) + ctx->result = accept(get_sock_fd(s), addr, paddrlen); #else - ctx->result = accept(s->sock_fd, addr, paddrlen); + ctx->result = accept(get_sock_fd(s), addr, paddrlen); #endif #ifdef MS_WINDOWS @@ -2886,8 +2931,10 @@ sock_accept_impl(PySocketSockObject *s, void *data) /* s._accept() -> (fd, address) */ static PyObject * -sock_accept(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) +sock_accept(PyObject *self, PyObject *Py_UNUSED(ignored)) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); + sock_addr_t addrbuf; SOCKET_T newfd; socklen_t addrlen; @@ -2923,8 +2970,7 @@ sock_accept(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) #else #if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) - socket_state *state = s->state; - if (!state->accept4_works) + if (!_Py_atomic_load_int_relaxed(&accept4_works)) #endif { if (_Py_set_inheritable(newfd, 0, NULL) < 0) { @@ -2940,7 +2986,7 @@ sock_accept(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) goto finally; } - addr = makesockaddr(s->sock_fd, SAS2SA(&addrbuf), + addr = makesockaddr(get_sock_fd(s), SAS2SA(&addrbuf), addrlen, s->sock_proto); if (addr == NULL) goto finally; @@ -2968,7 +3014,7 @@ For IP sockets, the address info is a pair (hostaddr, port)."); */ static PyObject * -sock_setblocking(PySocketSockObject *s, PyObject *arg) +sock_setblocking(PyObject *self, PyObject *arg) { long block; @@ -2976,6 +3022,7 @@ sock_setblocking(PySocketSockObject *s, PyObject *arg) if (block < 0) return NULL; + PySocketSockObject *s = _PySocketSockObject_CAST(self); s->sock_timeout = _PyTime_FromSeconds(block ? -1 : 0); if (internal_setblocking(s, block) == -1) { return NULL; @@ -2995,8 +3042,9 @@ setblocking(False) is equivalent to settimeout(0.0)."); False if it is in non-blocking mode. */ static PyObject * -sock_getblocking(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) +sock_getblocking(PyObject *self, PyObject *Py_UNUSED(ignored)) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); if (s->sock_timeout) { Py_RETURN_TRUE; } @@ -3059,13 +3107,14 @@ socket_parse_timeout(PyTime_t *timeout, PyObject *timeout_obj) < 0 -- illegal; raises an exception */ static PyObject * -sock_settimeout(PySocketSockObject *s, PyObject *arg) +sock_settimeout(PyObject *self, PyObject *arg) { PyTime_t timeout; if (socket_parse_timeout(&timeout, arg) < 0) return NULL; + PySocketSockObject *s = _PySocketSockObject_CAST(self); s->sock_timeout = timeout; int block = timeout < 0; @@ -3107,8 +3156,9 @@ Setting a timeout of zero is the same as setblocking(0)."); /* s.gettimeout() method. Returns the timeout associated with a socket. */ static PyObject * -sock_gettimeout(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) +sock_gettimeout_impl(PyObject *self, void *Py_UNUSED(ignored)) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); if (s->sock_timeout < 0) { Py_RETURN_NONE; } @@ -3118,6 +3168,18 @@ sock_gettimeout(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) } } +static inline PyObject * +sock_gettimeout_method(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return sock_gettimeout_impl(self, NULL); +} + +static inline PyObject * +sock_gettimeout_getter(PyObject *self, void *Py_UNUSED(closure)) +{ + return sock_gettimeout_impl(self, NULL); +} + PyDoc_STRVAR(gettimeout_doc, "gettimeout() -> timeout\n\ \n\ @@ -3135,8 +3197,10 @@ operations are disabled."); */ static PyObject * -sock_setsockopt(PySocketSockObject *s, PyObject *args) +sock_setsockopt(PyObject *self, PyObject *args) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); + int level; int optname; int res; @@ -3152,7 +3216,7 @@ sock_setsockopt(PySocketSockObject *s, PyObject *args) if (PyArg_ParseTuple(args, "iiK:setsockopt", &level, &optname, &vflag)) { // level should always be set to AF_VSOCK - res = setsockopt(s->sock_fd, level, optname, + res = setsockopt(get_sock_fd(s), level, optname, (void*)&vflag, sizeof vflag); goto done; } @@ -3163,7 +3227,18 @@ sock_setsockopt(PySocketSockObject *s, PyObject *args) /* setsockopt(level, opt, flag) */ if (PyArg_ParseTuple(args, "iii:setsockopt", &level, &optname, &flag)) { - res = setsockopt(s->sock_fd, level, optname, +#ifdef MS_WINDOWS + if (optname == SIO_TCP_SET_ACK_FREQUENCY) { + int dummy; + res = WSAIoctl(get_sock_fd(s), SIO_TCP_SET_ACK_FREQUENCY, &flag, + sizeof(flag), NULL, 0, &dummy, NULL, NULL); + if (res >= 0) { + s->quickack = flag; + } + goto done; + } +#endif + res = setsockopt(get_sock_fd(s), level, optname, (char*)&flag, sizeof flag); goto done; } @@ -3173,7 +3248,7 @@ sock_setsockopt(PySocketSockObject *s, PyObject *args) if (PyArg_ParseTuple(args, "iiO!I:setsockopt", &level, &optname, Py_TYPE(Py_None), &none, &optlen)) { assert(sizeof(socklen_t) >= sizeof(unsigned int)); - res = setsockopt(s->sock_fd, level, optname, + res = setsockopt(get_sock_fd(s), level, optname, NULL, (socklen_t)optlen); goto done; } @@ -3192,10 +3267,10 @@ sock_setsockopt(PySocketSockObject *s, PyObject *args) INT_MAX); return NULL; } - res = setsockopt(s->sock_fd, level, optname, + res = setsockopt(get_sock_fd(s), level, optname, optval.buf, (int)optval.len); #else - res = setsockopt(s->sock_fd, level, optname, optval.buf, optval.len); + res = setsockopt(get_sock_fd(s), level, optname, optval.buf, optval.len); #endif PyBuffer_Release(&optval); @@ -3223,8 +3298,10 @@ None, optlen."); use optional built-in module 'struct' to decode the string. */ static PyObject * -sock_getsockopt(PySocketSockObject *s, PyObject *args) +sock_getsockopt(PyObject *self, PyObject *args) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); + int level; int optname; int res; @@ -3242,15 +3319,20 @@ sock_getsockopt(PySocketSockObject *s, PyObject *args) if (s->sock_family == AF_VSOCK) { uint64_t vflag = 0; // Must be set width of 64 bits flagsize = sizeof vflag; - res = getsockopt(s->sock_fd, level, optname, + res = getsockopt(get_sock_fd(s), level, optname, (void *)&vflag, &flagsize); if (res < 0) return s->errorhandler(); return PyLong_FromUnsignedLong(vflag); } +#endif +#ifdef MS_WINDOWS + if (optname == SIO_TCP_SET_ACK_FREQUENCY) { + return PyLong_FromLong(s->quickack); + } #endif flagsize = sizeof flag; - res = getsockopt(s->sock_fd, level, optname, + res = getsockopt(get_sock_fd(s), level, optname, (void *)&flag, &flagsize); if (res < 0) return s->errorhandler(); @@ -3271,7 +3353,7 @@ sock_getsockopt(PySocketSockObject *s, PyObject *args) buf = PyBytes_FromStringAndSize((char *)NULL, buflen); if (buf == NULL) return NULL; - res = getsockopt(s->sock_fd, level, optname, + res = getsockopt(get_sock_fd(s), level, optname, (void *)PyBytes_AS_STRING(buf), &buflen); if (res < 0) { Py_DECREF(buf); @@ -3293,8 +3375,10 @@ string of that length; otherwise it is an integer."); /* s.bind(sockaddr) method */ static PyObject * -sock_bind(PySocketSockObject *s, PyObject *addro) +sock_bind(PyObject *self, PyObject *addro) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); + sock_addr_t addrbuf; int addrlen; int res; @@ -3308,7 +3392,7 @@ sock_bind(PySocketSockObject *s, PyObject *addro) } Py_BEGIN_ALLOW_THREADS - res = bind(s->sock_fd, SAS2SA(&addrbuf), addrlen); + res = bind(get_sock_fd(s), SAS2SA(&addrbuf), addrlen); Py_END_ALLOW_THREADS if (res < 0) return s->errorhandler(); @@ -3328,15 +3412,25 @@ sockets the address is a tuple (ifname, proto [,pkttype [,hatype [,addr]]])"); Set the file descriptor to -1 so operations tried subsequently will surely fail. */ +/*[clinic input] +_socket.socket.close + self as s: self(type="PySocketSockObject *") + +close() + +Close the socket. It cannot be used after this call. +[clinic start generated code]*/ + static PyObject * -sock_close(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) +_socket_socket_close_impl(PySocketSockObject *s) +/*[clinic end generated code: output=038b2418e07f6f6c input=dc487e470e55a83c]*/ { SOCKET_T fd; int res; - fd = s->sock_fd; + fd = get_sock_fd(s); if (fd != INVALID_SOCKET) { - s->sock_fd = INVALID_SOCKET; + set_sock_fd(s, INVALID_SOCKET); /* We do not want to retry upon EINTR: see http://lwn.net/Articles/576478/ and @@ -3354,16 +3448,12 @@ sock_close(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } -PyDoc_STRVAR(sock_close_doc, -"close()\n\ -\n\ -Close the socket. It cannot be used after this call."); - static PyObject * -sock_detach(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) +sock_detach(PyObject *self, PyObject *Py_UNUSED(ignored)) { - SOCKET_T fd = s->sock_fd; - s->sock_fd = INVALID_SOCKET; + PySocketSockObject *s = _PySocketSockObject_CAST(self); + SOCKET_T fd = get_sock_fd(s); + set_sock_fd(s, INVALID_SOCKET); return PyLong_FromSocket_t(fd); } @@ -3381,7 +3471,7 @@ sock_connect_impl(PySocketSockObject *s, void* Py_UNUSED(data)) int err; socklen_t size = sizeof err; - if (getsockopt(s->sock_fd, SOL_SOCKET, SO_ERROR, (void *)&err, &size)) { + if (getsockopt(get_sock_fd(s), SOL_SOCKET, SO_ERROR, (void *)&err, &size)) { /* getsockopt() failed */ return 0; } @@ -3396,6 +3486,18 @@ sock_connect_impl(PySocketSockObject *s, void* Py_UNUSED(data)) return 1; } +/* Common functionality for socket.connect and socket.connect_ex. + * + * If *raise* is set: + * - On success, return 0. + * - On any failure, return -1 with an exception set. + * If *raise* is zero: + * - On success, return 0. + * - On connect() failure, return errno (without an exception set) + * - On other error, return -1 with an exception set. + * + * Note that -1 is a valid errno value on some systems. + */ static int internal_connect(PySocketSockObject *s, struct sockaddr *addr, int addrlen, int raise) @@ -3403,7 +3505,7 @@ internal_connect(PySocketSockObject *s, struct sockaddr *addr, int addrlen, int res, err, wait_connect; Py_BEGIN_ALLOW_THREADS - res = connect(s->sock_fd, addr, addrlen); + res = connect(get_sock_fd(s), addr, addrlen); Py_END_ALLOW_THREADS if (!res) { @@ -3465,8 +3567,10 @@ internal_connect(PySocketSockObject *s, struct sockaddr *addr, int addrlen, /* s.connect(sockaddr) method */ static PyObject * -sock_connect(PySocketSockObject *s, PyObject *addro) +sock_connect(PyObject *self, PyObject *addro) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); + sock_addr_t addrbuf; int addrlen; int res; @@ -3480,8 +3584,10 @@ sock_connect(PySocketSockObject *s, PyObject *addro) } res = internal_connect(s, SAS2SA(&addrbuf), addrlen, 1); - if (res < 0) + if (res < 0) { + assert(PyErr_Occurred()); return NULL; + } Py_RETURN_NONE; } @@ -3496,8 +3602,10 @@ is a pair (host, port)."); /* s.connect_ex(sockaddr) method */ static PyObject * -sock_connect_ex(PySocketSockObject *s, PyObject *addro) +sock_connect_ex(PyObject *self, PyObject *addro) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); + sock_addr_t addrbuf; int addrlen; int res; @@ -3511,8 +3619,9 @@ sock_connect_ex(PySocketSockObject *s, PyObject *addro) } res = internal_connect(s, SAS2SA(&addrbuf), addrlen, 0); - if (res < 0) + if (res == -1 && PyErr_Occurred()) { return NULL; + } return PyLong_FromLong((long) res); } @@ -3528,9 +3637,10 @@ instead of raising an exception when an error occurs."); /* s.fileno() method */ static PyObject * -sock_fileno(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) +sock_fileno(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return PyLong_FromSocket_t(s->sock_fd); + PySocketSockObject *s = _PySocketSockObject_CAST(self); + return PyLong_FromSocket_t(get_sock_fd(s)); } PyDoc_STRVAR(fileno_doc, @@ -3543,8 +3653,10 @@ Return the integer file descriptor of the socket."); /* s.getsockname() method */ static PyObject * -sock_getsockname(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) +sock_getsockname(PyObject *self, PyObject *Py_UNUSED(ignored)) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); + sock_addr_t addrbuf; int res; socklen_t addrlen; @@ -3553,11 +3665,11 @@ sock_getsockname(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) return NULL; memset(&addrbuf, 0, addrlen); Py_BEGIN_ALLOW_THREADS - res = getsockname(s->sock_fd, SAS2SA(&addrbuf), &addrlen); + res = getsockname(get_sock_fd(s), SAS2SA(&addrbuf), &addrlen); Py_END_ALLOW_THREADS if (res < 0) return s->errorhandler(); - return makesockaddr(s->sock_fd, SAS2SA(&addrbuf), addrlen, + return makesockaddr(get_sock_fd(s), SAS2SA(&addrbuf), addrlen, s->sock_proto); } @@ -3575,8 +3687,10 @@ address family. For IPv4 sockets, the address info is a pair\n\ /* s.getpeername() method */ static PyObject * -sock_getpeername(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) +sock_getpeername(PyObject *self, PyObject *Py_UNUSED(ignored)) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); + sock_addr_t addrbuf; int res; socklen_t addrlen; @@ -3585,11 +3699,11 @@ sock_getpeername(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) return NULL; memset(&addrbuf, 0, addrlen); Py_BEGIN_ALLOW_THREADS - res = getpeername(s->sock_fd, SAS2SA(&addrbuf), &addrlen); + res = getpeername(get_sock_fd(s), SAS2SA(&addrbuf), &addrlen); Py_END_ALLOW_THREADS if (res < 0) return s->errorhandler(); - return makesockaddr(s->sock_fd, SAS2SA(&addrbuf), addrlen, + return makesockaddr(get_sock_fd(s), SAS2SA(&addrbuf), addrlen, s->sock_proto); } @@ -3606,8 +3720,9 @@ info is a pair (hostaddr, port)."); /* s.listen(n) method */ static PyObject * -sock_listen(PySocketSockObject *s, PyObject *args) +sock_listen(PyObject *self, PyObject *args) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); /* We try to choose a default backlog high enough to avoid connection drops * for common workloads, yet not too high to limit resource usage. */ int backlog = Py_MIN(SOMAXCONN, 128); @@ -3621,7 +3736,7 @@ sock_listen(PySocketSockObject *s, PyObject *args) * (which doesn't make sense anyway) we force a minimum value of 0. */ if (backlog < 0) backlog = 0; - res = listen(s->sock_fd, backlog); + res = listen(get_sock_fd(s), backlog); Py_END_ALLOW_THREADS if (res < 0) return s->errorhandler(); @@ -3652,9 +3767,9 @@ sock_recv_impl(PySocketSockObject *s, void *data) #ifdef MS_WINDOWS if (ctx->len > INT_MAX) ctx->len = INT_MAX; - ctx->result = recv(s->sock_fd, ctx->cbuf, (int)ctx->len, ctx->flags); + ctx->result = recv(get_sock_fd(s), ctx->cbuf, (int)ctx->len, ctx->flags); #else - ctx->result = recv(s->sock_fd, ctx->cbuf, ctx->len, ctx->flags); + ctx->result = recv(get_sock_fd(s), ctx->cbuf, ctx->len, ctx->flags); #endif return (ctx->result >= 0); } @@ -3696,8 +3811,10 @@ sock_recv_guts(PySocketSockObject *s, char* cbuf, Py_ssize_t len, int flags) /* s.recv(nbytes [,flags]) method */ static PyObject * -sock_recv(PySocketSockObject *s, PyObject *args) +sock_recv(PyObject *self, PyObject *args) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); + Py_ssize_t recvlen, outlen; int flags = 0; PyObject *buf; @@ -3745,9 +3862,10 @@ the remote end is closed and all data is read, return the empty string."); /* s.recv_into(buffer, [nbytes [,flags]]) method */ static PyObject* -sock_recv_into(PySocketSockObject *s, PyObject *args, PyObject *kwds) +sock_recv_into(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"buffer", "nbytes", "flags", 0}; + PySocketSockObject *s = _PySocketSockObject_CAST(self); int flags = 0; Py_buffer pbuf; @@ -3823,10 +3941,10 @@ sock_recvfrom_impl(PySocketSockObject *s, void *data) #ifdef MS_WINDOWS if (ctx->len > INT_MAX) ctx->len = INT_MAX; - ctx->result = recvfrom(s->sock_fd, ctx->cbuf, (int)ctx->len, ctx->flags, + ctx->result = recvfrom(get_sock_fd(s), ctx->cbuf, (int)ctx->len, ctx->flags, SAS2SA(ctx->addrbuf), ctx->addrlen); #else - ctx->result = recvfrom(s->sock_fd, ctx->cbuf, ctx->len, ctx->flags, + ctx->result = recvfrom(get_sock_fd(s), ctx->cbuf, ctx->len, ctx->flags, SAS2SA(ctx->addrbuf), ctx->addrlen); #endif return (ctx->result >= 0); @@ -3870,7 +3988,7 @@ sock_recvfrom_guts(PySocketSockObject *s, char* cbuf, Py_ssize_t len, int flags, if (sock_call(s, 0, sock_recvfrom_impl, &ctx) < 0) return -1; - *addr = makesockaddr(s->sock_fd, SAS2SA(&addrbuf), addrlen, + *addr = makesockaddr(get_sock_fd(s), SAS2SA(&addrbuf), addrlen, s->sock_proto); if (*addr == NULL) return -1; @@ -3881,8 +3999,10 @@ sock_recvfrom_guts(PySocketSockObject *s, char* cbuf, Py_ssize_t len, int flags, /* s.recvfrom(nbytes [,flags]) method */ static PyObject * -sock_recvfrom(PySocketSockObject *s, PyObject *args) +sock_recvfrom(PyObject *self, PyObject *args) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); + PyObject *buf = NULL; PyObject *addr = NULL; PyObject *ret = NULL; @@ -3933,9 +4053,10 @@ Like recv(buffersize, flags) but also return the sender's address info."); /* s.recvfrom_into(buffer[, nbytes [,flags]]) method */ static PyObject * -sock_recvfrom_into(PySocketSockObject *s, PyObject *args, PyObject* kwds) +sock_recvfrom_into(PyObject *self, PyObject *args, PyObject* kwds) { static char *kwlist[] = {"buffer", "nbytes", "flags", 0}; + PySocketSockObject *s = _PySocketSockObject_CAST(self); int flags = 0; Py_buffer pbuf; @@ -4001,7 +4122,7 @@ sock_recvmsg_impl(PySocketSockObject *s, void *data) { struct sock_recvmsg *ctx = data; - ctx->result = recvmsg(s->sock_fd, ctx->msg, ctx->flags); + ctx->result = recvmsg(get_sock_fd(s), ctx->msg, ctx->flags); return (ctx->result >= 0); } @@ -4110,7 +4231,7 @@ sock_recvmsg_guts(PySocketSockObject *s, struct iovec *iov, int iovlen, (*makeval)(ctx.result, makeval_data), cmsg_list, (int)msg.msg_flags, - makesockaddr(s->sock_fd, SAS2SA(&addrbuf), + makesockaddr(get_sock_fd(s), SAS2SA(&addrbuf), ((msg.msg_namelen > addrbuflen) ? addrbuflen : msg.msg_namelen), s->sock_proto)); @@ -4161,8 +4282,10 @@ makeval_recvmsg(ssize_t received, void *data) /* s.recvmsg(bufsize[, ancbufsize[, flags]]) method */ static PyObject * -sock_recvmsg(PySocketSockObject *s, PyObject *args) +sock_recvmsg(PyObject *self, PyObject *args) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); + Py_ssize_t bufsize, ancbufsize = 0; int flags = 0; struct iovec iov; @@ -4228,8 +4351,10 @@ makeval_recvmsg_into(ssize_t received, void *data) /* s.recvmsg_into(buffers[, ancbufsize[, flags]]) method */ static PyObject * -sock_recvmsg_into(PySocketSockObject *s, PyObject *args) +sock_recvmsg_into(PyObject *self, PyObject *args) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); + Py_ssize_t ancbufsize = 0; int flags = 0; struct iovec *iovs = NULL; @@ -4329,9 +4454,9 @@ sock_send_impl(PySocketSockObject *s, void *data) #ifdef MS_WINDOWS if (ctx->len > INT_MAX) ctx->len = INT_MAX; - ctx->result = send(s->sock_fd, ctx->buf, (int)ctx->len, ctx->flags); + ctx->result = send(get_sock_fd(s), ctx->buf, (int)ctx->len, ctx->flags); #else - ctx->result = send(s->sock_fd, ctx->buf, ctx->len, ctx->flags); + ctx->result = send(get_sock_fd(s), ctx->buf, ctx->len, ctx->flags); #endif return (ctx->result >= 0); } @@ -4339,8 +4464,10 @@ sock_send_impl(PySocketSockObject *s, void *data) /* s.send(data [,flags]) method */ static PyObject * -sock_send(PySocketSockObject *s, PyObject *args) +sock_send(PyObject *self, PyObject *args) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); + int flags = 0; Py_buffer pbuf; struct sock_send ctx; @@ -4375,8 +4502,10 @@ sent; this may be less than len(data) if the network is busy."); /* s.sendall(data [,flags]) method */ static PyObject * -sock_sendall(PySocketSockObject *s, PyObject *args) +sock_sendall(PyObject *self, PyObject *args) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); + char *buf; Py_ssize_t len, n; int flags = 0; @@ -4468,10 +4597,10 @@ sock_sendto_impl(PySocketSockObject *s, void *data) #ifdef MS_WINDOWS if (ctx->len > INT_MAX) ctx->len = INT_MAX; - ctx->result = sendto(s->sock_fd, ctx->buf, (int)ctx->len, ctx->flags, + ctx->result = sendto(get_sock_fd(s), ctx->buf, (int)ctx->len, ctx->flags, SAS2SA(ctx->addrbuf), ctx->addrlen); #else - ctx->result = sendto(s->sock_fd, ctx->buf, ctx->len, ctx->flags, + ctx->result = sendto(get_sock_fd(s), ctx->buf, ctx->len, ctx->flags, SAS2SA(ctx->addrbuf), ctx->addrlen); #endif return (ctx->result >= 0); @@ -4480,8 +4609,10 @@ sock_sendto_impl(PySocketSockObject *s, void *data) /* s.sendto(data, [flags,] sockaddr) method */ static PyObject * -sock_sendto(PySocketSockObject *s, PyObject *args) +sock_sendto(PyObject *self, PyObject *args) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); + Py_buffer pbuf; PyObject *addro; Py_ssize_t arglen; @@ -4617,15 +4748,17 @@ sock_sendmsg_impl(PySocketSockObject *s, void *data) { struct sock_sendmsg *ctx = data; - ctx->result = sendmsg(s->sock_fd, ctx->msg, ctx->flags); + ctx->result = sendmsg(get_sock_fd(s), ctx->msg, ctx->flags); return (ctx->result >= 0); } /* s.sendmsg(buffers[, ancdata[, flags[, address]]]) method */ static PyObject * -sock_sendmsg(PySocketSockObject *s, PyObject *args) +sock_sendmsg(PyObject *self, PyObject *args) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); + Py_ssize_t i, ndatabufs = 0, ncmsgs, ncmsgbufs = 0; Py_buffer *databufs = NULL; sock_addr_t addrbuf; @@ -4828,8 +4961,10 @@ data sent."); #ifdef HAVE_SOCKADDR_ALG static PyObject* -sock_sendmsg_afalg(PySocketSockObject *self, PyObject *args, PyObject *kwds) +sock_sendmsg_afalg(PyObject *s, PyObject *args, PyObject *kwds) { + PySocketSockObject *self = _PySocketSockObject_CAST(s); + PyObject *retval = NULL; Py_ssize_t i, ndatabufs = 0; @@ -4996,8 +5131,10 @@ operation socket."); /* s.shutdown(how) method */ static PyObject * -sock_shutdown(PySocketSockObject *s, PyObject *arg) +sock_shutdown(PyObject *self, PyObject *arg) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); + int how; int res; @@ -5005,7 +5142,7 @@ sock_shutdown(PySocketSockObject *s, PyObject *arg) if (how == -1 && PyErr_Occurred()) return NULL; Py_BEGIN_ALLOW_THREADS - res = shutdown(s->sock_fd, how); + res = shutdown(get_sock_fd(s), how); Py_END_ALLOW_THREADS if (res < 0) return s->errorhandler(); @@ -5021,8 +5158,10 @@ of the socket (flag == SHUT_WR), or both ends (flag == SHUT_RDWR)."); #if defined(MS_WINDOWS) && defined(SIO_RCVALL) static PyObject* -sock_ioctl(PySocketSockObject *s, PyObject *arg) +sock_ioctl(PyObject *self, PyObject *arg) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); + unsigned long cmd = SIO_RCVALL; PyObject *argO; DWORD recv; @@ -5035,7 +5174,7 @@ sock_ioctl(PySocketSockObject *s, PyObject *arg) unsigned int option = RCVALL_ON; if (!PyArg_ParseTuple(arg, "kI:ioctl", &cmd, &option)) return NULL; - if (WSAIoctl(s->sock_fd, cmd, &option, sizeof(option), + if (WSAIoctl(get_sock_fd(s), cmd, &option, sizeof(option), NULL, 0, &recv, NULL, NULL) == SOCKET_ERROR) { return set_error(); } @@ -5045,7 +5184,7 @@ sock_ioctl(PySocketSockObject *s, PyObject *arg) if (!PyArg_ParseTuple(arg, "k(kkk):ioctl", &cmd, &ka.onoff, &ka.keepalivetime, &ka.keepaliveinterval)) return NULL; - if (WSAIoctl(s->sock_fd, cmd, &ka, sizeof(ka), + if (WSAIoctl(get_sock_fd(s), cmd, &ka, sizeof(ka), NULL, 0, &recv, NULL, NULL) == SOCKET_ERROR) { return set_error(); } @@ -5055,7 +5194,7 @@ sock_ioctl(PySocketSockObject *s, PyObject *arg) unsigned int option; if (!PyArg_ParseTuple(arg, "kI:ioctl", &cmd, &option)) return NULL; - if (WSAIoctl(s->sock_fd, cmd, &option, sizeof(option), + if (WSAIoctl(get_sock_fd(s), cmd, &option, sizeof(option), NULL, 0, &recv, NULL, NULL) == SOCKET_ERROR) { return set_error(); } @@ -5077,8 +5216,10 @@ SIO_LOOPBACK_FAST_PATH: 'option' is a boolean value, and is disabled by default" #if defined(MS_WINDOWS) static PyObject* -sock_share(PySocketSockObject *s, PyObject *arg) +sock_share(PyObject *self, PyObject *arg) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); + WSAPROTOCOL_INFOW info; DWORD processId; int result; @@ -5087,7 +5228,7 @@ sock_share(PySocketSockObject *s, PyObject *arg) return NULL; Py_BEGIN_ALLOW_THREADS - result = WSADuplicateSocketW(s->sock_fd, processId, &info); + result = WSADuplicateSocketW(get_sock_fd(s), processId, &info); Py_END_ALLOW_THREADS if (result == SOCKET_ERROR) return set_error(); @@ -5108,94 +5249,82 @@ socket.fromshare()."); static PyMethodDef sock_methods[] = { #if defined(HAVE_ACCEPT) || defined(HAVE_ACCEPT4) - {"_accept", (PyCFunction)sock_accept, METH_NOARGS, - accept_doc}, + {"_accept", sock_accept, METH_NOARGS, accept_doc}, #endif #ifdef HAVE_BIND - {"bind", (PyCFunction)sock_bind, METH_O, - bind_doc}, + {"bind", sock_bind, METH_O, bind_doc}, #endif - {"close", (PyCFunction)sock_close, METH_NOARGS, - sock_close_doc}, + _SOCKET_SOCKET_CLOSE_METHODDEF #ifdef HAVE_CONNECT - {"connect", (PyCFunction)sock_connect, METH_O, - connect_doc}, - {"connect_ex", (PyCFunction)sock_connect_ex, METH_O, - connect_ex_doc}, -#endif - {"detach", (PyCFunction)sock_detach, METH_NOARGS, - detach_doc}, - {"fileno", (PyCFunction)sock_fileno, METH_NOARGS, - fileno_doc}, + {"connect", sock_connect, METH_O, connect_doc}, + {"connect_ex", sock_connect_ex, METH_O, connect_ex_doc}, +#endif + {"detach", sock_detach, METH_NOARGS, detach_doc}, + {"fileno", sock_fileno, METH_NOARGS, fileno_doc}, #ifdef HAVE_GETPEERNAME - {"getpeername", (PyCFunction)sock_getpeername, - METH_NOARGS, getpeername_doc}, + {"getpeername", sock_getpeername, METH_NOARGS, getpeername_doc}, #endif #ifdef HAVE_GETSOCKNAME - {"getsockname", (PyCFunction)sock_getsockname, - METH_NOARGS, getsockname_doc}, + {"getsockname", sock_getsockname, METH_NOARGS, getsockname_doc}, #endif - {"getsockopt", (PyCFunction)sock_getsockopt, METH_VARARGS, - getsockopt_doc}, + {"getsockopt", sock_getsockopt, METH_VARARGS, getsockopt_doc}, #if defined(MS_WINDOWS) && defined(SIO_RCVALL) - {"ioctl", (PyCFunction)sock_ioctl, METH_VARARGS, - sock_ioctl_doc}, + {"ioctl", sock_ioctl, METH_VARARGS, sock_ioctl_doc}, #endif #if defined(MS_WINDOWS) - {"share", (PyCFunction)sock_share, METH_VARARGS, - sock_share_doc}, + {"share", sock_share, METH_VARARGS, sock_share_doc}, #endif #ifdef HAVE_LISTEN - {"listen", (PyCFunction)sock_listen, METH_VARARGS, - listen_doc}, + {"listen", sock_listen, METH_VARARGS, listen_doc}, #endif - {"recv", (PyCFunction)sock_recv, METH_VARARGS, - recv_doc}, - {"recv_into", _PyCFunction_CAST(sock_recv_into), METH_VARARGS | METH_KEYWORDS, - recv_into_doc}, + {"recv", sock_recv, METH_VARARGS, recv_doc}, + { + "recv_into", + _PyCFunction_CAST(sock_recv_into), + METH_VARARGS | METH_KEYWORDS, + recv_into_doc + }, #ifdef HAVE_RECVFROM - {"recvfrom", (PyCFunction)sock_recvfrom, METH_VARARGS, - recvfrom_doc}, - {"recvfrom_into", _PyCFunction_CAST(sock_recvfrom_into), METH_VARARGS | METH_KEYWORDS, - recvfrom_into_doc}, -#endif - {"send", (PyCFunction)sock_send, METH_VARARGS, - send_doc}, - {"sendall", (PyCFunction)sock_sendall, METH_VARARGS, - sendall_doc}, + {"recvfrom", sock_recvfrom, METH_VARARGS, recvfrom_doc}, + { + "recvfrom_into", + _PyCFunction_CAST(sock_recvfrom_into), + METH_VARARGS | METH_KEYWORDS, + recvfrom_into_doc + }, +#endif + {"send", sock_send, METH_VARARGS, send_doc}, + {"sendall", sock_sendall, METH_VARARGS, sendall_doc}, #ifdef HAVE_SENDTO - {"sendto", (PyCFunction)sock_sendto, METH_VARARGS, - sendto_doc}, -#endif - {"setblocking", (PyCFunction)sock_setblocking, METH_O, - setblocking_doc}, - {"getblocking", (PyCFunction)sock_getblocking, METH_NOARGS, - getblocking_doc}, - {"settimeout", (PyCFunction)sock_settimeout, METH_O, - settimeout_doc}, - {"gettimeout", (PyCFunction)sock_gettimeout, METH_NOARGS, - gettimeout_doc}, + {"sendto", sock_sendto, METH_VARARGS, sendto_doc}, +#endif + {"setblocking", sock_setblocking, METH_O, setblocking_doc}, + {"getblocking", sock_getblocking, METH_NOARGS, getblocking_doc}, + {"settimeout", sock_settimeout, METH_O, settimeout_doc}, + { + "gettimeout", sock_gettimeout_method, METH_NOARGS, + gettimeout_doc + }, #ifdef HAVE_SETSOCKOPT - {"setsockopt", (PyCFunction)sock_setsockopt, METH_VARARGS, - setsockopt_doc}, + {"setsockopt", sock_setsockopt, METH_VARARGS, setsockopt_doc}, #endif #ifdef HAVE_SHUTDOWN - {"shutdown", (PyCFunction)sock_shutdown, METH_O, - shutdown_doc}, + {"shutdown", sock_shutdown, METH_O, shutdown_doc}, #endif #ifdef CMSG_LEN - {"recvmsg", (PyCFunction)sock_recvmsg, METH_VARARGS, - recvmsg_doc}, - {"recvmsg_into", (PyCFunction)sock_recvmsg_into, METH_VARARGS, - recvmsg_into_doc,}, - {"sendmsg", (PyCFunction)sock_sendmsg, METH_VARARGS, - sendmsg_doc}, + {"recvmsg", sock_recvmsg, METH_VARARGS, recvmsg_doc}, + {"recvmsg_into", sock_recvmsg_into, METH_VARARGS, recvmsg_into_doc}, + {"sendmsg", sock_sendmsg, METH_VARARGS, sendmsg_doc}, #endif #ifdef HAVE_SOCKADDR_ALG - {"sendmsg_afalg", _PyCFunction_CAST(sock_sendmsg_afalg), METH_VARARGS | METH_KEYWORDS, - sendmsg_afalg_doc}, + { + "sendmsg_afalg", + _PyCFunction_CAST(sock_sendmsg_afalg), + METH_VARARGS | METH_KEYWORDS, + sendmsg_afalg_doc + }, #endif - {NULL, NULL} /* sentinel */ + {NULL, NULL, 0, NULL} /* sentinel */ }; /* SockObject members */ @@ -5207,7 +5336,7 @@ static PyMemberDef sock_memberlist[] = { }; static PyGetSetDef sock_getsetlist[] = { - {"timeout", (getter)sock_gettimeout, NULL, PyDoc_STR("the socket timeout")}, + {"timeout", sock_gettimeout_getter, NULL, PyDoc_STR("the socket timeout")}, {NULL} /* sentinel */ }; @@ -5215,14 +5344,16 @@ static PyGetSetDef sock_getsetlist[] = { First close the file description. */ static void -sock_finalize(PySocketSockObject *s) +sock_finalize(PyObject *self) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); + SOCKET_T fd; /* Save the current exception, if any. */ PyObject *exc = PyErr_GetRaisedException(); - if (s->sock_fd != INVALID_SOCKET) { + if (get_sock_fd(s) != INVALID_SOCKET) { if (PyErr_ResourceWarning((PyObject *)s, 1, "unclosed %R", s)) { /* Spurious errors can appear at shutdown */ if (PyErr_ExceptionMatches(PyExc_Warning)) { @@ -5234,8 +5365,8 @@ sock_finalize(PySocketSockObject *s) to allow the logger to call socket methods like socket.getsockname(). If the socket is closed before, socket methods fails with the EBADF error. */ - fd = s->sock_fd; - s->sock_fd = INVALID_SOCKET; + fd = get_sock_fd(s); + set_sock_fd(s, INVALID_SOCKET); /* We do not want to retry upon EINTR: see sock_close() */ Py_BEGIN_ALLOW_THREADS @@ -5248,35 +5379,37 @@ sock_finalize(PySocketSockObject *s) } static int -sock_traverse(PySocketSockObject *s, visitproc visit, void *arg) +sock_traverse(PyObject *s, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(s)); return 0; } static void -sock_dealloc(PySocketSockObject *s) +sock_dealloc(PyObject *s) { - if (PyObject_CallFinalizerFromDealloc((PyObject *)s) < 0) { + if (PyObject_CallFinalizerFromDealloc(s) < 0) { return; } PyTypeObject *tp = Py_TYPE(s); PyObject_GC_UnTrack(s); - tp->tp_free((PyObject *)s); + tp->tp_free(s); Py_DECREF(tp); } static PyObject * -sock_repr(PySocketSockObject *s) +sock_repr(PyObject *self) { + PySocketSockObject *s = _PySocketSockObject_CAST(self); + long sock_fd; /* On Windows, this test is needed because SOCKET_T is unsigned */ - if (s->sock_fd == INVALID_SOCKET) { + if (get_sock_fd(s) == INVALID_SOCKET) { sock_fd = -1; } #if SIZEOF_SOCKET_T > SIZEOF_LONG - else if (s->sock_fd > LONG_MAX) { + else if (get_sock_fd(s) > LONG_MAX) { /* this can occur on Win64, and actually there is a special ugly printf formatter for decimal pointer length integer printing, only bother if necessary*/ @@ -5287,7 +5420,7 @@ sock_repr(PySocketSockObject *s) } #endif else - sock_fd = (long)s->sock_fd; + sock_fd = (long)get_sock_fd(s); return PyUnicode_FromFormat( "", sock_fd, s->sock_family, @@ -5308,6 +5441,9 @@ sock_new(PyTypeObject *type, PyObject *args, PyObject *kwds) ((PySocketSockObject *)new)->sock_fd = INVALID_SOCKET; ((PySocketSockObject *)new)->sock_timeout = _PyTime_FromSeconds(-1); ((PySocketSockObject *)new)->errorhandler = &set_error; +#ifdef MS_WINDOWS + ((PySocketSockObject *)new)->quickack = 0; +#endif } return new; } @@ -5346,7 +5482,7 @@ sock_initobj_impl(PySocketSockObject *self, int family, int type, int proto, #ifndef MS_WINDOWS #ifdef SOCK_CLOEXEC - int *atomic_flag_works = &state->sock_cloexec_works; + int *atomic_flag_works = &sock_cloexec_works; #else int *atomic_flag_works = NULL; #endif @@ -5501,15 +5637,16 @@ sock_initobj_impl(PySocketSockObject *self, int family, int type, int proto, /* UNIX */ Py_BEGIN_ALLOW_THREADS #ifdef SOCK_CLOEXEC - if (state->sock_cloexec_works != 0) { + if (_Py_atomic_load_int_relaxed(&sock_cloexec_works) != 0) { fd = socket(family, type | SOCK_CLOEXEC, proto); - if (state->sock_cloexec_works == -1) { + if (_Py_atomic_load_int_relaxed(&sock_cloexec_works) == -1) { if (fd >= 0) { - state->sock_cloexec_works = 1; + _Py_atomic_store_int_relaxed(&sock_cloexec_works, 1); } + else if (errno == EINVAL) { /* Linux older than 2.6.27 does not support SOCK_CLOEXEC */ - state->sock_cloexec_works = 0; + _Py_atomic_store_int_relaxed(&sock_cloexec_works, 0); fd = socket(family, type, proto); } } @@ -5594,7 +5731,7 @@ socket_gethostname(PyObject *self, PyObject *unused) return PyErr_SetFromWindowsErr(0); if (size == 0) - return PyUnicode_New(0, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); /* MSDN says ERROR_MORE_DATA may occur because DNS allows longer names */ @@ -6250,7 +6387,7 @@ socket_socketpair(PyObject *self, PyObject *args) PyObject *res = NULL; socket_state *state = get_module_state(self); #ifdef SOCK_CLOEXEC - int *atomic_flag_works = &state->sock_cloexec_works; + int *atomic_flag_works = &sock_cloexec_works; #else int *atomic_flag_works = NULL; #endif @@ -6268,15 +6405,15 @@ socket_socketpair(PyObject *self, PyObject *args) /* Create a pair of socket fds */ Py_BEGIN_ALLOW_THREADS #ifdef SOCK_CLOEXEC - if (state->sock_cloexec_works != 0) { + if (_Py_atomic_load_int_relaxed(&sock_cloexec_works) != 0) { ret = socketpair(family, type | SOCK_CLOEXEC, proto, sv); - if (state->sock_cloexec_works == -1) { + if (_Py_atomic_load_int_relaxed(&sock_cloexec_works) == -1) { if (ret >= 0) { - state->sock_cloexec_works = 1; + _Py_atomic_store_int_relaxed(&sock_cloexec_works, 1); } else if (errno == EINVAL) { /* Linux older than 2.6.27 does not support SOCK_CLOEXEC */ - state->sock_cloexec_works = 0; + _Py_atomic_store_int_relaxed(&sock_cloexec_works, 0); ret = socketpair(family, type, proto, sv); } } @@ -6913,11 +7050,12 @@ static PyObject * socket_getdefaulttimeout(PyObject *self, PyObject *Py_UNUSED(ignored)) { socket_state *state = get_module_state(self); - if (state->defaulttimeout < 0) { + PyTime_t timeout = _Py_atomic_load_int64_relaxed(&state->defaulttimeout); + if (timeout < 0) { Py_RETURN_NONE; } else { - double seconds = PyTime_AsSecondsDouble(state->defaulttimeout); + double seconds = PyTime_AsSecondsDouble(timeout); return PyFloat_FromDouble(seconds); } } @@ -6938,7 +7076,7 @@ socket_setdefaulttimeout(PyObject *self, PyObject *arg) return NULL; socket_state *state = get_module_state(self); - state->defaulttimeout = timeout; + _Py_atomic_store_int64_relaxed(&state->defaulttimeout, timeout); Py_RETURN_NONE; } @@ -7383,17 +7521,8 @@ socket_exec(PyObject *m) } socket_state *state = get_module_state(m); - state->defaulttimeout = _PYTIME_FROMSECONDS(-1); -#if defined(HAVE_ACCEPT) || defined(HAVE_ACCEPT4) -#if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) - state->accept4_works = -1; -#endif -#endif - -#ifdef SOCK_CLOEXEC - state->sock_cloexec_works = -1; -#endif + _Py_atomic_store_int64_relaxed(&state->defaulttimeout, _PYTIME_FROMSECONDS(-1)); #define ADD_EXC(MOD, NAME, VAR, BASE) do { \ VAR = PyErr_NewException("socket." NAME, BASE, NULL); \ @@ -7548,36 +7677,20 @@ socket_exec(PyObject *m) /* */ ADD_INT_MACRO(m, AF_NETLINK); ADD_INT_MACRO(m, NETLINK_ROUTE); -#ifdef NETLINK_SKIP - ADD_INT_MACRO(m, NETLINK_SKIP); -#endif -#ifdef NETLINK_W1 - ADD_INT_MACRO(m, NETLINK_W1); -#endif ADD_INT_MACRO(m, NETLINK_USERSOCK); ADD_INT_MACRO(m, NETLINK_FIREWALL); -#ifdef NETLINK_TCPDIAG - ADD_INT_MACRO(m, NETLINK_TCPDIAG); -#endif #ifdef NETLINK_NFLOG ADD_INT_MACRO(m, NETLINK_NFLOG); #endif #ifdef NETLINK_XFRM ADD_INT_MACRO(m, NETLINK_XFRM); #endif -#ifdef NETLINK_ARPD - ADD_INT_MACRO(m, NETLINK_ARPD); -#endif -#ifdef NETLINK_ROUTE6 - ADD_INT_MACRO(m, NETLINK_ROUTE6); -#endif +#ifdef NETLINK_IP6_FW ADD_INT_MACRO(m, NETLINK_IP6_FW); +#endif #ifdef NETLINK_DNRTMSG ADD_INT_MACRO(m, NETLINK_DNRTMSG); #endif -#ifdef NETLINK_TAPBASE - ADD_INT_MACRO(m, NETLINK_TAPBASE); -#endif #ifdef NETLINK_CRYPTO ADD_INT_MACRO(m, NETLINK_CRYPTO); #endif @@ -7595,6 +7708,7 @@ socket_exec(PyObject *m) ADD_INT_CONST(m, "SO_VM_SOCKETS_BUFFER_MAX_SIZE", 2); ADD_INT_CONST(m, "VMADDR_CID_ANY", 0xffffffff); ADD_INT_CONST(m, "VMADDR_PORT_ANY", 0xffffffff); + ADD_INT_CONST(m, "VMADDR_CID_LOCAL", 1); ADD_INT_CONST(m, "VMADDR_CID_HOST", 2); ADD_INT_CONST(m, "VM_SOCKETS_INVALID_VERSION", 0xffffffff); ADD_INT_CONST(m, "IOCTL_VM_SOCKETS_GET_LOCAL_CID", _IO(7, 0xb9)); @@ -7877,11 +7991,17 @@ socket_exec(PyObject *m) #ifdef SO_OOBINLINE ADD_INT_MACRO(m, SO_OOBINLINE); #endif +#ifdef SO_ORIGINAL_DST + ADD_INT_MACRO(m, SO_ORIGINAL_DST); +#endif #ifndef __GNU__ #ifdef SO_REUSEPORT ADD_INT_MACRO(m, SO_REUSEPORT); #endif #endif +#ifdef SO_REUSEPORT_LB + ADD_INT_MACRO(m, SO_REUSEPORT_LB); +#endif #ifdef SO_SNDBUF ADD_INT_MACRO(m, SO_SNDBUF); #endif @@ -8410,15 +8530,24 @@ socket_exec(PyObject *m) #ifdef IP_TTL ADD_INT_MACRO(m, IP_TTL); #endif +#ifdef IP_RECVERR + ADD_INT_MACRO(m, IP_RECVERR); +#endif #ifdef IP_RECVOPTS ADD_INT_MACRO(m, IP_RECVOPTS); #endif +#ifdef IP_RECVORIGDSTADDR + ADD_INT_MACRO(m, IP_RECVORIGDSTADDR); +#endif #ifdef IP_RECVRETOPTS ADD_INT_MACRO(m, IP_RECVRETOPTS); #endif #ifdef IP_RECVTOS ADD_INT_MACRO(m, IP_RECVTOS); #endif +#ifdef IP_RECVTTL + ADD_INT_MACRO(m, IP_RECVTTL); +#endif #ifdef IP_RECVDSTADDR ADD_INT_MACRO(m, IP_RECVDSTADDR); #endif @@ -8522,6 +8651,9 @@ socket_exec(PyObject *m) #ifdef IPV6_RECVDSTOPTS ADD_INT_MACRO(m, IPV6_RECVDSTOPTS); #endif +#ifdef IPV6_RECVERR + ADD_INT_MACRO(m, IPV6_RECVERR); +#endif #ifdef IPV6_RECVHOPLIMIT ADD_INT_MACRO(m, IPV6_RECVHOPLIMIT); #endif @@ -8597,6 +8729,9 @@ socket_exec(PyObject *m) #ifdef TCP_CONNECTION_INFO ADD_INT_MACRO(m, TCP_CONNECTION_INFO); #endif +#ifdef SIO_TCP_SET_ACK_FREQUENCY +#define TCP_QUICKACK SIO_TCP_SET_ACK_FREQUENCY +#endif #ifdef TCP_QUICKACK ADD_INT_MACRO(m, TCP_QUICKACK); #endif @@ -8895,6 +9030,7 @@ socket_exec(PyObject *m) static struct PyModuleDef_Slot socket_slots[] = { {Py_mod_exec, socket_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index 09fd70f351f1d8..e1d96377728eb9 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -172,6 +172,10 @@ typedef int socklen_t; # undef AF_VSOCK #endif +#ifdef HAVE_LINUX_NETFILTER_IPV4_H +# include +#endif + #ifdef HAVE_SOCKADDR_ALG # include @@ -325,6 +329,9 @@ typedef struct { PyTime_t sock_timeout; /* Operation timeout in seconds; 0.0 means non-blocking */ struct _socket_state *state; +#ifdef MS_WINDOWS + int quickack; +#endif } PySocketSockObject; /* --- C API ----------------------------------------------------*/ diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index ddc9ac3324356d..d0d5223e5acea8 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -75,24 +75,27 @@ symtable_init_constants(PyObject *m) if (PyModule_AddIntMacro(m, DEF_NONLOCAL) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_LOCAL) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_PARAM) < 0) return -1; - if (PyModule_AddIntMacro(m, DEF_FREE) < 0) return -1; + if (PyModule_AddIntMacro(m, DEF_TYPE_PARAM) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_FREE_CLASS) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_IMPORT) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_BOUND) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_ANNOT) < 0) return -1; + if (PyModule_AddIntMacro(m, DEF_COMP_ITER) < 0) return -1; + if (PyModule_AddIntMacro(m, DEF_COMP_CELL) < 0) return -1; if (PyModule_AddIntConstant(m, "TYPE_FUNCTION", FunctionBlock) < 0) return -1; - if (PyModule_AddIntConstant(m, "TYPE_CLASS", ClassBlock) < 0) return -1; + if (PyModule_AddIntConstant(m, "TYPE_CLASS", ClassBlock) < 0) + return -1; if (PyModule_AddIntConstant(m, "TYPE_MODULE", ModuleBlock) < 0) return -1; if (PyModule_AddIntConstant(m, "TYPE_ANNOTATION", AnnotationBlock) < 0) return -1; - if (PyModule_AddIntConstant(m, "TYPE_TYPE_VAR_BOUND", TypeVarBoundBlock) < 0) - return -1; if (PyModule_AddIntConstant(m, "TYPE_TYPE_ALIAS", TypeAliasBlock) < 0) return -1; - if (PyModule_AddIntConstant(m, "TYPE_TYPE_PARAM", TypeParamBlock) < 0) + if (PyModule_AddIntConstant(m, "TYPE_TYPE_PARAMETERS", TypeParametersBlock) < 0) + return -1; + if (PyModule_AddIntConstant(m, "TYPE_TYPE_VARIABLE", TypeVariableBlock) < 0) return -1; if (PyModule_AddIntMacro(m, LOCAL) < 0) return -1; @@ -110,6 +113,7 @@ symtable_init_constants(PyObject *m) static PyModuleDef_Slot symtable_slots[] = { {Py_mod_exec, symtable_init_constants}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index 62c7816f891ee2..aa1bc9da91dfb9 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -132,6 +132,7 @@ syslog_get_argv(void) /*[clinic input] +@critical_section syslog.openlog ident: unicode = NULL @@ -144,7 +145,7 @@ Set logging options of subsequent syslog() calls. static PyObject * syslog_openlog_impl(PyObject *module, PyObject *ident, long logopt, long facility) -/*[clinic end generated code: output=5476c12829b6eb75 input=8a987a96a586eee7]*/ +/*[clinic end generated code: output=5476c12829b6eb75 input=ee700b8786f81c23]*/ { // Since the sys.openlog changes the process level state of syslog library, // this operation is only allowed for the main interpreter. @@ -175,7 +176,7 @@ syslog_openlog_impl(PyObject *module, PyObject *ident, long logopt, } } if (PySys_Audit("syslog.openlog", "Oll", ident ? ident : Py_None, logopt, facility) < 0) { - Py_DECREF(ident); + Py_XDECREF(ident); return NULL; } @@ -189,6 +190,7 @@ syslog_openlog_impl(PyObject *module, PyObject *ident, long logopt, /*[clinic input] +@critical_section syslog.syslog [ @@ -205,7 +207,7 @@ Send the string message to the system logger. static PyObject * syslog_syslog_impl(PyObject *module, int group_left_1, int priority, const char *message) -/*[clinic end generated code: output=c3dbc73445a0e078 input=ac83d92b12ea3d4e]*/ +/*[clinic end generated code: output=c3dbc73445a0e078 input=6588ddb0b113af8e]*/ { if (PySys_Audit("syslog.syslog", "is", priority, message) < 0) { return NULL; @@ -243,6 +245,7 @@ syslog_syslog_impl(PyObject *module, int group_left_1, int priority, /*[clinic input] +@critical_section syslog.closelog Reset the syslog module values and call the system library closelog(). @@ -250,12 +253,12 @@ Reset the syslog module values and call the system library closelog(). static PyObject * syslog_closelog_impl(PyObject *module) -/*[clinic end generated code: output=97890a80a24b1b84 input=fb77a54d447acf07]*/ +/*[clinic end generated code: output=97890a80a24b1b84 input=167f489868bd5a72]*/ { // Since the sys.closelog changes the process level state of syslog library, // this operation is only allowed for the main interpreter. if (!is_main_interpreter()) { - PyErr_SetString(PyExc_RuntimeError, "sunbinterpreter can't use syslog.closelog()"); + PyErr_SetString(PyExc_RuntimeError, "subinterpreter can't use syslog.closelog()"); return NULL; } @@ -436,6 +439,7 @@ syslog_exec(PyObject *module) static PyModuleDef_Slot syslog_slots[] = { {Py_mod_exec, syslog_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/termios.c b/Modules/termios.c index 4635fefb8f3f5a..efb5fcc17fa5ef 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -1,11 +1,19 @@ /* termios.c -- POSIX terminal I/O module implementation. */ -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 +// Need limited C API version 3.13 for PyLong_AsInt() +// in code generated by Argument Clinic. +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" +#include // memset() +#include +#include +#include // _POSIX_VDISABLE + // On QNX 6, struct termio must be declared by including sys/termio.h // if TCGETA, TCSETA, TCSETAW, or TCSETAF are used. sys/termio.h must // be included before termios.h or it will generate an error. @@ -19,28 +27,26 @@ # define CTRL(c) ((c)&037) #endif +// We could do better. Check bpo-32660 #if defined(__sun) -/* We could do better. Check issue-32660 */ -#include -#include +# include +# include #endif -#include -#include -#include // _POSIX_VDISABLE - /* HP-UX requires that this be included to pick up MDCD, MCTS, MDSR, * MDTR, MRI, and MRTS (apparently used internally by some things * defined as macros; these are not used here directly). */ #ifdef HAVE_SYS_MODEM_H -#include +# include #endif + /* HP-UX requires that this be included to pick up TIOCGPGRP and friends */ #ifdef HAVE_SYS_BSDTTY_H -#include +# include #endif + /*[clinic input] module termios [clinic start generated code]*/ @@ -120,7 +126,7 @@ termios_tcgetattr_impl(PyObject *module, int fd) v = PyBytes_FromStringAndSize(&ch, 1); if (v == NULL) goto err; - PyList_SET_ITEM(cc, i, v); + PyList_SetItem(cc, i, v); } /* Convert the MIN and TIME slots to integer. On some systems, the @@ -154,7 +160,7 @@ termios_tcgetattr_impl(PyObject *module, int fd) Py_DECREF(v); \ goto err; \ } \ - PyList_SET_ITEM(v, index, l); \ + PyList_SetItem(v, index, l); \ } while (0) ADD_LONG_ITEM(0, mode.c_iflag); @@ -165,7 +171,7 @@ termios_tcgetattr_impl(PyObject *module, int fd) ADD_LONG_ITEM(5, ospeed); #undef ADD_LONG_ITEM - PyList_SET_ITEM(v, 6, cc); + PyList_SetItem(v, 6, cc); return v; err: Py_DECREF(cc); @@ -214,7 +220,7 @@ termios_tcsetattr_impl(PyObject *module, int fd, int when, PyObject *term) speed_t ispeed, ospeed; #define SET_FROM_LIST(TYPE, VAR, LIST, N) do { \ - PyObject *item = PyList_GET_ITEM(LIST, N); \ + PyObject *item = PyList_GetItem(LIST, N); \ long num = PyLong_AsLong(item); \ if (num == -1 && PyErr_Occurred()) { \ return NULL; \ @@ -230,7 +236,7 @@ termios_tcsetattr_impl(PyObject *module, int fd, int when, PyObject *term) SET_FROM_LIST(speed_t, ospeed, term, 5); #undef SET_FROM_LIST - PyObject *cc = PyList_GET_ITEM(term, 6); + PyObject *cc = PyList_GetItem(term, 6); if (!PyList_Check(cc) || PyList_Size(cc) != NCCS) { PyErr_Format(PyExc_TypeError, "tcsetattr: attributes[6] must be %d element list", @@ -254,7 +260,7 @@ termios_tcsetattr_impl(PyObject *module, int fd, int when, PyObject *term) } else { PyErr_SetString(PyExc_TypeError, - "tcsetattr: elements of attributes must be characters or integers"); + "tcsetattr: elements of attributes must be bytes objects of length 1 or integers"); return NULL; } } @@ -1346,9 +1352,21 @@ termios_exec(PyObject *mod) } while (constant->name != NULL) { - if (PyModule_AddIntConstant( - mod, constant->name, constant->value) < 0) { - return -1; + if (strncmp(constant->name, "TIO", 3) == 0) { + // gh-119770: Convert value to unsigned int for ioctl() constants, + // constants can be negative on macOS whereas ioctl() expects an + // unsigned long 'request'. + unsigned int value = constant->value & UINT_MAX; + if (PyModule_Add(mod, constant->name, + PyLong_FromUnsignedLong(value)) < 0) { + return -1; + } + } + else { + if (PyModule_AddIntConstant( + mod, constant->name, constant->value) < 0) { + return -1; + } } ++constant; } @@ -1358,6 +1376,7 @@ termios_exec(PyObject *mod) static PyModuleDef_Slot termios_slots[] = { {Py_mod_exec, termios_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/timemodule.c b/Modules/timemodule.c index ed41ffd3662aa8..5d0cd52a93a2d3 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -116,7 +116,7 @@ time_time(PyObject *self, PyObject *unused) PyDoc_STRVAR(time_doc, -"time() -> floating point number\n\ +"time() -> floating-point number\n\ \n\ Return the current time in seconds since the Epoch.\n\ Fractions of a second may be present if the system clock provides them."); @@ -350,7 +350,7 @@ time_clock_getres(PyObject *self, PyObject *args) } PyDoc_STRVAR(clock_getres_doc, -"clock_getres(clk_id) -> floating point number\n\ +"clock_getres(clk_id) -> floating-point number\n\ \n\ Return the resolution (precision) of the specified clock clk_id."); @@ -413,7 +413,7 @@ PyDoc_STRVAR(sleep_doc, "sleep(seconds)\n\ \n\ Delay execution for a given number of seconds. The argument may be\n\ -a floating point number for subsecond precision."); +a floating-point number for subsecond precision."); static PyStructSequence_Field struct_time_type_fields[] = { {"tm_year", "year, for example, 1993"}, @@ -462,7 +462,18 @@ tmtotuple(time_module_state *state, struct tm *p if (v == NULL) return NULL; -#define SET(i,val) PyStructSequence_SET_ITEM(v, i, PyLong_FromLong((long) val)) +#define SET_ITEM(INDEX, CALL) \ + do { \ + PyObject *obj = (CALL); \ + if (obj == NULL) { \ + Py_DECREF(v); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(v, (INDEX), obj); \ + } while (0) + +#define SET(INDEX, VAL) \ + SET_ITEM((INDEX), PyLong_FromLong((long) (VAL))) SET(0, p->tm_year + 1900); SET(1, p->tm_mon + 1); /* Want January == 1 */ @@ -474,19 +485,15 @@ tmtotuple(time_module_state *state, struct tm *p SET(7, p->tm_yday + 1); /* Want January, 1 == 1 */ SET(8, p->tm_isdst); #ifdef HAVE_STRUCT_TM_TM_ZONE - PyStructSequence_SET_ITEM(v, 9, - PyUnicode_DecodeLocale(p->tm_zone, "surrogateescape")); + SET_ITEM(9, PyUnicode_DecodeLocale(p->tm_zone, "surrogateescape")); SET(10, p->tm_gmtoff); #else - PyStructSequence_SET_ITEM(v, 9, - PyUnicode_DecodeLocale(zone, "surrogateescape")); - PyStructSequence_SET_ITEM(v, 10, _PyLong_FromTime_t(gmtoff)); + SET_ITEM(9, PyUnicode_DecodeLocale(zone, "surrogateescape")); + SET_ITEM(10, _PyLong_FromTime_t(gmtoff)); #endif /* HAVE_STRUCT_TM_TM_ZONE */ + #undef SET - if (PyErr_Occurred()) { - Py_XDECREF(v); - return NULL; - } +#undef SET_ITEM return v; } @@ -732,10 +739,6 @@ checktm(struct tm* buf) return 1; } -#ifdef MS_WINDOWS - /* wcsftime() doesn't format correctly time zones, see issue #10653 */ -# undef HAVE_WCSFTIME -#endif #define STRFTIME_FORMAT_CODES \ "Commonly used format codes:\n\ \n\ @@ -769,27 +772,100 @@ the C library strftime function.\n" #endif static PyObject * -time_strftime(PyObject *module, PyObject *args) +time_strftime1(time_char **outbuf, size_t *bufsize, + time_char *format, size_t fmtlen, + struct tm *tm) { - PyObject *tup = NULL; - struct tm buf; - const time_char *fmt; + size_t buflen; +#if defined(MS_WINDOWS) && !defined(HAVE_WCSFTIME) + /* check that the format string contains only valid directives */ + for (const time_char *f = strchr(format, '%'); + f != NULL; + f = strchr(f + 2, '%')) + { + if (f[1] == '#') + ++f; /* not documented by python, */ + if (f[1] == '\0') + break; + if ((f[1] == 'y') && tm->tm_year < 0) { + PyErr_SetString(PyExc_ValueError, + "format %y requires year >= 1900 on Windows"); + return NULL; + } + } +#elif (defined(_AIX) || (defined(__sun) && defined(__SVR4))) && defined(HAVE_WCSFTIME) + for (const time_char *f = wcschr(format, '%'); + f != NULL; + f = wcschr(f + 2, '%')) + { + if (f[1] == L'\0') + break; + /* Issue #19634: On AIX, wcsftime("y", (1899, 1, 1, 0, 0, 0, 0, 0, 0)) + returns "0/" instead of "99" */ + if (f[1] == L'y' && tm->tm_year < 0) { + PyErr_SetString(PyExc_ValueError, + "format %y requires year >= 1900 on AIX"); + return NULL; + } + } +#endif + + /* I hate these functions that presume you know how big the output + * will be ahead of time... + */ + while (1) { + if (*bufsize > PY_SSIZE_T_MAX/sizeof(time_char)) { + PyErr_NoMemory(); + return NULL; + } + *outbuf = (time_char *)PyMem_Realloc(*outbuf, + *bufsize*sizeof(time_char)); + if (*outbuf == NULL) { + PyErr_NoMemory(); + return NULL; + } +#if defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__) + errno = 0; +#endif + _Py_BEGIN_SUPPRESS_IPH + buflen = format_time(*outbuf, *bufsize, format, tm); + _Py_END_SUPPRESS_IPH +#if defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__) + /* VisualStudio .NET 2005 does this properly */ + if (buflen == 0 && errno == EINVAL) { + PyErr_SetString(PyExc_ValueError, "Invalid format string"); + return NULL; + } +#endif + if (buflen == 0 && *bufsize < 256 * fmtlen) { + *bufsize += *bufsize; + continue; + } + /* If the buffer is 256 times as long as the format, + it's probably not failing for lack of room! + More likely, the format yields an empty result, + e.g. an empty format, or %Z when the timezone + is unknown. */ #ifdef HAVE_WCSFTIME - wchar_t *format; + return PyUnicode_FromWideChar(*outbuf, buflen); #else - PyObject *format; + return PyUnicode_DecodeLocaleAndSize(*outbuf, buflen, "surrogateescape"); #endif + } +} + +static PyObject * +time_strftime(PyObject *module, PyObject *args) +{ + PyObject *tup = NULL; + struct tm buf; PyObject *format_arg; - size_t fmtlen, buflen; - time_char *outbuf = NULL; - size_t i; - PyObject *ret = NULL; + Py_ssize_t format_size; + time_char *format, *outbuf = NULL; + size_t fmtlen, bufsize = 1024; memset((void *) &buf, '\0', sizeof(buf)); - /* Will always expect a unicode string to be passed as format. - Given that there's no str type anymore in py3k this seems safe. - */ if (!PyArg_ParseTuple(args, "U|O:strftime", &format_arg, &tup)) return NULL; @@ -806,7 +882,12 @@ time_strftime(PyObject *module, PyObject *args) return NULL; } -#if defined(_MSC_VER) || (defined(__sun) && defined(__SVR4)) || defined(_AIX) || defined(__VXWORKS__) +// Some platforms only support a limited range of years. +// +// Android works with negative years on the emulator, but fails on some +// physical devices (#123017). +#if defined(_MSC_VER) || (defined(__sun) && defined(__SVR4)) || defined(_AIX) \ + || defined(__VXWORKS__) || defined(__ANDROID__) if (buf.tm_year + 1900 < 1 || 9999 < buf.tm_year + 1900) { PyErr_SetString(PyExc_ValueError, "strftime() requires year in [1; 9999]"); @@ -822,101 +903,64 @@ time_strftime(PyObject *module, PyObject *args) else if (buf.tm_isdst > 1) buf.tm_isdst = 1; -#ifdef HAVE_WCSFTIME - format = PyUnicode_AsWideCharString(format_arg, NULL); - if (format == NULL) + format_size = PyUnicode_GET_LENGTH(format_arg); + if ((size_t)format_size > PY_SSIZE_T_MAX/sizeof(time_char) - 1) { + PyErr_NoMemory(); return NULL; - fmt = format; -#else - /* Convert the unicode string to an ascii one */ - format = PyUnicode_EncodeLocale(format_arg, "surrogateescape"); - if (format == NULL) + } + format = PyMem_Malloc((format_size + 1)*sizeof(time_char)); + if (format == NULL) { + PyErr_NoMemory(); return NULL; - fmt = PyBytes_AS_STRING(format); -#endif - -#if defined(MS_WINDOWS) && !defined(HAVE_WCSFTIME) - /* check that the format string contains only valid directives */ - for (outbuf = strchr(fmt, '%'); - outbuf != NULL; - outbuf = strchr(outbuf+2, '%')) - { - if (outbuf[1] == '#') - ++outbuf; /* not documented by python, */ - if (outbuf[1] == '\0') - break; - if ((outbuf[1] == 'y') && buf.tm_year < 0) { - PyErr_SetString(PyExc_ValueError, - "format %y requires year >= 1900 on Windows"); - Py_DECREF(format); - return NULL; - } } -#elif (defined(_AIX) || (defined(__sun) && defined(__SVR4))) && defined(HAVE_WCSFTIME) - for (outbuf = wcschr(fmt, '%'); - outbuf != NULL; - outbuf = wcschr(outbuf+2, '%')) - { - if (outbuf[1] == L'\0') - break; - /* Issue #19634: On AIX, wcsftime("y", (1899, 1, 1, 0, 0, 0, 0, 0, 0)) - returns "0/" instead of "99" */ - if (outbuf[1] == L'y' && buf.tm_year < 0) { - PyErr_SetString(PyExc_ValueError, - "format %y requires year >= 1900 on AIX"); - PyMem_Free(format); - return NULL; - } + PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); + if (writer == NULL) { + goto error; } -#endif - - fmtlen = time_strlen(fmt); - - /* I hate these functions that presume you know how big the output - * will be ahead of time... - */ - for (i = 1024; ; i += i) { - outbuf = (time_char *)PyMem_Malloc(i*sizeof(time_char)); - if (outbuf == NULL) { - PyErr_NoMemory(); - break; + Py_ssize_t i = 0; + while (i < format_size) { + fmtlen = 0; + for (; i < format_size; i++) { + Py_UCS4 c = PyUnicode_READ_CHAR(format_arg, i); + if (!c || c > 127) { + break; + } + format[fmtlen++] = (char)c; } -#if defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__) - errno = 0; -#endif - _Py_BEGIN_SUPPRESS_IPH - buflen = format_time(outbuf, i, fmt, &buf); - _Py_END_SUPPRESS_IPH -#if defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__) - /* VisualStudio .NET 2005 does this properly */ - if (buflen == 0 && errno == EINVAL) { - PyErr_SetString(PyExc_ValueError, "Invalid format string"); - PyMem_Free(outbuf); - break; + if (fmtlen) { + format[fmtlen] = 0; + PyObject *unicode = time_strftime1(&outbuf, &bufsize, + format, fmtlen, &buf); + if (unicode == NULL) { + goto error; + } + if (PyUnicodeWriter_WriteStr(writer, unicode) < 0) { + Py_DECREF(unicode); + goto error; + } + Py_DECREF(unicode); } -#endif - if (buflen > 0 || i >= 256 * fmtlen) { - /* If the buffer is 256 times as long as the format, - it's probably not failing for lack of room! - More likely, the format yields an empty result, - e.g. an empty format, or %Z when the timezone - is unknown. */ -#ifdef HAVE_WCSFTIME - ret = PyUnicode_FromWideChar(outbuf, buflen); -#else - ret = PyUnicode_DecodeLocaleAndSize(outbuf, buflen, "surrogateescape"); -#endif - PyMem_Free(outbuf); - break; + + Py_ssize_t start = i; + for (; i < format_size; i++) { + Py_UCS4 c = PyUnicode_READ_CHAR(format_arg, i); + if (c == '%') { + break; + } + } + if (PyUnicodeWriter_WriteSubstring(writer, format_arg, start, i) < 0) { + goto error; } - PyMem_Free(outbuf); } -#ifdef HAVE_WCSFTIME + + PyMem_Free(outbuf); PyMem_Free(format); -#else - Py_DECREF(format); -#endif - return ret; + return PyUnicodeWriter_Finish(writer); +error: + PyMem_Free(outbuf); + PyMem_Free(format); + PyUnicodeWriter_Discard(writer); + return NULL; } #undef time_char @@ -1097,7 +1141,7 @@ time_mktime(PyObject *module, PyObject *tm_tuple) } PyDoc_STRVAR(mktime_doc, -"mktime(tuple) -> floating point number\n\ +"mktime(tuple) -> floating-point number\n\ \n\ Convert a time tuple in local time to seconds since the Epoch.\n\ Note that mktime(gmtime(0)) will not generally return zero for most\n\ @@ -1281,9 +1325,14 @@ py_process_time(time_module_state *state, PyTime_t *tp, /* clock_gettime */ // gh-115714: Don't use CLOCK_PROCESS_CPUTIME_ID on WASI. +/* CLOCK_PROF is defined on NetBSD, but not supported. + * CLOCK_PROCESS_CPUTIME_ID is broken on NetBSD for the same reason as + * CLOCK_THREAD_CPUTIME_ID (see comment below). + */ #if defined(HAVE_CLOCK_GETTIME) \ && (defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_PROF)) \ - && !defined(__wasi__) + && !defined(__wasi__) \ + && !defined(__NetBSD__) struct timespec ts; if (HAVE_CLOCK_GETTIME_RUNTIME) { @@ -1476,12 +1525,19 @@ _PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) return 0; } +/* CLOCK_THREAD_CPUTIME_ID is broken on NetBSD: the result of clock_gettime() + * includes the sleeping time, that defeats the purpose of the clock. + * Also, clock_getres() does not support it. + * https://github.com/python/cpython/issues/123978 + * https://gnats.netbsd.org/57512 + */ #elif defined(HAVE_CLOCK_GETTIME) && \ - defined(CLOCK_PROCESS_CPUTIME_ID) && \ - !defined(__EMSCRIPTEN__) && !defined(__wasi__) + defined(CLOCK_THREAD_CPUTIME_ID) && \ + !defined(__EMSCRIPTEN__) && !defined(__wasi__) && \ + !defined(__NetBSD__) #define HAVE_THREAD_TIME -#if defined(__APPLE__) && defined(__has_attribute) && __has_attribute(availability) +#if defined(__APPLE__) && _Py__has_attribute(availability) static int _PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) __attribute__((availability(macos, introduced=10.12))) @@ -1895,9 +1951,9 @@ PyDoc_STRVAR(module_doc, \n\ There are two standard representations of time. One is the number\n\ of seconds since the Epoch, in UTC (a.k.a. GMT). It may be an integer\n\ -or a floating point number (to represent fractions of seconds).\n\ -The Epoch is system-defined; on Unix, it is generally January 1st, 1970.\n\ -The actual value can be retrieved by calling gmtime(0).\n\ +or a floating-point number (to represent fractions of seconds).\n\ +The epoch is the point where the time starts, the return value of time.gmtime(0).\n\ +It is January 1, 1970, 00:00:00 (UTC) on all platforms.\n\ \n\ The other representation is a tuple of 9 integers giving local time.\n\ The tuple items are:\n\ @@ -1928,20 +1984,20 @@ time_exec(PyObject *module) return -1; } - if (PyDict_DelItemString(dct, "clock_gettime") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "clock_gettime", NULL) < 0) { + return -1; } - if (PyDict_DelItemString(dct, "clock_gettime_ns") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "clock_gettime_ns", NULL) < 0) { + return -1; } - if (PyDict_DelItemString(dct, "clock_settime") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "clock_settime", NULL) < 0) { + return -1; } - if (PyDict_DelItemString(dct, "clock_settime_ns") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "clock_settime_ns", NULL) < 0) { + return -1; } - if (PyDict_DelItemString(dct, "clock_getres") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "clock_getres", NULL) < 0) { + return -1; } } #endif @@ -1951,11 +2007,11 @@ time_exec(PyObject *module) } else { PyObject* dct = PyModule_GetDict(module); - if (PyDict_DelItemString(dct, "thread_time") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "thread_time", NULL) < 0) { + return -1; } - if (PyDict_DelItemString(dct, "thread_time_ns") == -1) { - PyErr_Clear(); + if (PyDict_PopString(dct, "thread_time_ns", NULL) < 0) { + return -1; } } #endif @@ -2128,6 +2184,7 @@ time_module_free(void *module) static struct PyModuleDef_Slot time_slots[] = { {Py_mod_exec, time_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index 6ae35b9372b830..60bde755d24574 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -413,7 +413,7 @@ unicodedata_UCD_decomposition_impl(PyObject *self, int chr) if (UCD_Check(self)) { const change_record *old = get_old_record(self, c); if (old->category_changed == 0) - return PyUnicode_FromString(""); /* unassigned */ + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); /* unassigned */ } if (code < 0 || code >= 0x110000) @@ -1668,6 +1668,7 @@ unicodedata_exec(PyObject *module) static PyModuleDef_Slot unicodedata_slots[] = { {Py_mod_exec, unicodedata_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/unicodedata_db.h b/Modules/unicodedata_db.h index 3e210863448b78..610ee6b749cb0c 100644 --- a/Modules/unicodedata_db.h +++ b/Modules/unicodedata_db.h @@ -1,6 +1,6 @@ -/* this file was generated by ./Tools/unicode/makeunicodedata.py 3.3 */ +/* this file was generated by Tools/unicode/makeunicodedata.py 3.3 */ -#define UNIDATA_VERSION "15.1.0" +#define UNIDATA_VERSION "16.0.0" /* a list of unique database records */ const _PyUnicode_DatabaseRecord _PyUnicode_Database_Records[] = { {0, 0, 0, 0, 0, 0}, @@ -334,24 +334,28 @@ const _PyUnicode_DatabaseRecord _PyUnicode_Database_Records[] = { {30, 0, 4, 0, 0, 0}, {1, 0, 4, 0, 0, 0}, {2, 0, 4, 0, 0, 0}, + {27, 0, 4, 0, 0, 0}, {9, 0, 12, 0, 0, 0}, {9, 0, 5, 0, 0, 0}, + {5, 0, 1, 0, 0, 90}, {4, 9, 1, 0, 0, 0}, + {4, 0, 14, 0, 0, 90}, + {19, 0, 1, 0, 0, 90}, {4, 0, 14, 0, 2, 0}, {5, 6, 1, 0, 2, 0}, + {30, 0, 1, 0, 0, 136}, + {7, 0, 9, 0, 0, 136}, {30, 0, 1, 0, 0, 170}, {5, 216, 1, 0, 0, 0}, {5, 226, 1, 0, 0, 0}, - {27, 0, 1, 0, 0, 136}, - {7, 0, 9, 0, 0, 136}, - {30, 0, 1, 0, 0, 136}, + {9, 0, 1, 0, 2, 0}, {30, 0, 1, 0, 4, 0}, {29, 0, 19, 0, 2, 0}, }; /* Reindexing of NFC first characters. */ -#define TOTAL_FIRST 377 -#define TOTAL_LAST 63 +#define TOTAL_FIRST 391 +#define TOTAL_LAST 72 struct reindex{int start;short count,index;}; static struct reindex nfc_first[] = { { 60, 2, 0}, @@ -557,14 +561,27 @@ static struct reindex nfc_first[] = { { 12507, 0, 361}, { 12527, 3, 362}, { 12541, 0, 366}, - { 69785, 0, 367}, - { 69787, 0, 368}, - { 69797, 0, 369}, - { 69937, 1, 370}, - { 70471, 0, 372}, - { 70841, 0, 373}, - { 71096, 1, 374}, - { 71989, 0, 376}, + { 67026, 0, 367}, + { 67034, 0, 368}, + { 69785, 0, 369}, + { 69787, 0, 370}, + { 69797, 0, 371}, + { 69937, 1, 372}, + { 70471, 0, 374}, + { 70530, 0, 375}, + { 70532, 0, 376}, + { 70539, 0, 377}, + { 70544, 0, 378}, + { 70594, 0, 379}, + { 70841, 0, 380}, + { 71096, 1, 381}, + { 71989, 0, 383}, + { 90398, 0, 384}, + { 90401, 1, 385}, + { 90409, 0, 387}, + { 93539, 0, 388}, + { 93543, 0, 389}, + { 93545, 0, 390}, {0,0,0} }; @@ -604,11 +621,18 @@ static struct reindex nfc_last[] = { { 69927, 0, 55}, { 70462, 0, 56}, { 70487, 0, 57}, - { 70832, 0, 58}, - { 70842, 0, 59}, - { 70845, 0, 60}, - { 71087, 0, 61}, - { 71984, 0, 62}, + { 70584, 0, 58}, + { 70587, 0, 59}, + { 70594, 0, 60}, + { 70601, 0, 61}, + { 70832, 0, 62}, + { 70842, 0, 63}, + { 70845, 0, 64}, + { 71087, 0, 65}, + { 71984, 0, 66}, + { 90398, 2, 67}, + { 90409, 0, 70}, + { 93543, 0, 71}, {0,0,0} }; @@ -742,42 +766,41 @@ static const unsigned short index1[] = { 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 121, 121, 122, 123, 124, 125, 126, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 41, 41, 145, 146, 147, 148, 149, - 150, 151, 152, 153, 154, 155, 156, 157, 137, 158, 159, 160, 161, 162, - 163, 164, 165, 166, 167, 168, 137, 169, 170, 137, 171, 172, 173, 174, - 137, 175, 176, 177, 178, 179, 180, 181, 137, 182, 183, 184, 185, 137, - 186, 187, 188, 41, 41, 41, 41, 41, 41, 41, 189, 190, 41, 191, 137, 137, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, + 164, 165, 166, 167, 168, 169, 170, 171, 172, 137, 173, 174, 175, 176, + 137, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 137, + 189, 190, 191, 41, 41, 41, 41, 41, 41, 41, 192, 193, 41, 194, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 137, 137, 137, 192, 41, 41, 41, 41, 41, 41, 41, 41, 193, 137, 137, + 137, 137, 137, 137, 195, 41, 41, 41, 41, 41, 41, 41, 41, 196, 41, 41, 41, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 134, 41, 41, 41, 41, 197, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 41, 41, 41, 41, 194, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 137, 41, 41, 41, 41, 195, 196, 197, 198, 137, 137, 137, 137, 199, - 200, 201, 202, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, + 137, 137, 137, 137, 137, 137, 137, 137, 137, 198, 137, 137, 137, 137, + 137, 137, 137, 137, 137, 137, 137, 137, 137, 41, 41, 41, 41, 199, 200, + 201, 202, 137, 137, 203, 137, 204, 205, 206, 207, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 203, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 204, 205, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, + 101, 208, 101, 101, 101, 101, 101, 101, 101, 101, 101, 209, 210, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 137, 137, 137, 206, 101, 101, 207, 101, 101, 208, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 209, 210, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 211, 101, 101, + 212, 101, 101, 213, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 137, 137, 137, 137, 137, 137, 137, 137, 214, 215, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 211, 212, 78, 213, - 214, 215, 216, 217, 218, 137, 219, 220, 221, 222, 223, 224, 225, 226, 78, - 78, 78, 78, 227, 228, 137, 137, 137, 137, 137, 137, 137, 137, 229, 137, - 230, 231, 232, 137, 137, 233, 137, 137, 137, 234, 137, 137, 137, 137, - 137, 235, 236, 237, 238, 137, 137, 137, 137, 137, 239, 240, 241, 137, - 242, 243, 137, 137, 244, 245, 246, 247, 248, 137, 249, 250, 251, 252, - 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, - 137, 137, 137, 137, 137, 137, 137, 137, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, + 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 216, 217, + 216, 216, 216, 218, 219, 220, 78, 221, 222, 223, 224, 225, 226, 137, 227, + 228, 229, 230, 231, 232, 233, 234, 78, 78, 78, 78, 235, 236, 137, 137, + 137, 137, 137, 137, 137, 137, 237, 137, 238, 239, 240, 137, 137, 241, + 137, 137, 137, 242, 137, 243, 137, 137, 137, 244, 245, 246, 247, 137, + 137, 137, 137, 137, 248, 249, 250, 137, 251, 252, 137, 137, 253, 254, + 255, 256, 257, 137, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, + 268, 269, 270, 271, 272, 273, 216, 274, 137, 137, 137, 137, 137, 137, + 137, 137, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, @@ -800,57 +823,57 @@ static const unsigned short index1[] = { 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 267, 101, 101, 101, 101, 101, 101, 101, 101, + 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 275, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 268, 101, 269, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, + 101, 101, 101, 101, 276, 101, 277, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 270, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, + 101, 101, 101, 101, 101, 101, 101, 101, 101, 278, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 271, 101, 101, 101, 101, 272, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 121, 121, 121, 121, 274, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 275, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, + 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 279, 101, 101, + 101, 101, 280, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 121, 121, 121, 121, 282, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 283, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 276, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, + 101, 101, 101, 101, 101, 101, 284, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 277, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 273, 275, 137, 137, 137, 137, - 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 285, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 283, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, @@ -1215,8 +1238,8 @@ static const unsigned short index1[] = { 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 137, 137, 137, 137, 137, 278, 137, 279, 280, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 286, 137, 287, 288, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, @@ -1252,6 +1275,7 @@ static const unsigned short index1[] = { 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 137, 137, 137, 137, 137, 137, 137, 137, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, @@ -1288,7 +1312,7 @@ static const unsigned short index1[] = { 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 281, 120, 120, 120, 120, 120, 120, + 120, 289, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, @@ -1324,8 +1348,7 @@ static const unsigned short index1[] = { 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 281, + 120, 120, 120, 120, 120, 120, 120, 120, 120, 289, }; static const unsigned short index2[] = { @@ -1457,7 +1480,7 @@ static const unsigned short index2[] = { 118, 118, 118, 118, 118, 118, 118, 118, 0, 0, 0, 0, 0, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 140, 118, 118, 118, 118, 118, 118, 0, - 108, 108, 0, 0, 0, 0, 0, 0, 81, 86, 86, 86, 81, 81, 81, 81, 118, 118, + 108, 108, 0, 0, 0, 0, 0, 81, 81, 86, 86, 86, 81, 81, 81, 81, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 120, 81, 81, 81, @@ -1713,46 +1736,46 @@ static const unsigned short index2[] = { 48, 142, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 147, 148, 135, 135, 135, 135, 135, 149, 135, 149, 141, 141, 149, 149, 135, - 149, 176, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 146, 146, 146, 146, + 149, 176, 48, 48, 48, 48, 48, 48, 48, 48, 0, 83, 83, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 83, 83, 83, 83, 83, 83, 83, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 81, 86, 81, 81, 81, 81, 81, 81, 81, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 83, 83, 0, 135, 135, 141, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 141, 135, 135, 135, 135, 141, 141, 135, 135, 176, - 144, 135, 135, 48, 48, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, + 80, 80, 80, 80, 80, 80, 83, 83, 83, 135, 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 141, 135, 135, 135, 135, 141, 141, 135, 135, + 176, 144, 135, 135, 48, 48, 146, 146, 146, 146, 146, 146, 146, 146, 146, + 146, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 147, 141, 135, 135, 141, 141, 141, 135, - 141, 135, 135, 135, 176, 176, 0, 0, 0, 0, 0, 0, 0, 0, 83, 83, 83, 83, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 147, 141, 135, 135, 141, 141, 141, + 135, 141, 135, 135, 135, 176, 176, 0, 0, 0, 0, 0, 0, 0, 0, 83, 83, 83, + 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, - 141, 141, 141, 141, 141, 141, 141, 135, 135, 135, 135, 135, 135, 135, - 135, 141, 141, 135, 147, 0, 0, 0, 83, 83, 83, 83, 83, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 0, 0, 0, 48, 48, 48, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 141, 141, 141, 141, 141, 141, 141, 141, 135, 135, 135, 135, 135, 135, + 135, 135, 141, 141, 135, 147, 0, 0, 0, 83, 83, 83, 83, 83, 146, 146, 146, + 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 48, 48, 48, 146, 146, 146, + 146, 146, 146, 146, 146, 146, 146, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 53, 53, 53, 53, 53, 53, 83, 83, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 0, 0, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, + 48, 48, 48, 53, 53, 53, 53, 53, 53, 83, 83, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 44, 47, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 0, 0, 44, 44, 44, 83, 83, - 83, 83, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 81, 81, 81, 83, 178, 86, - 86, 86, 86, 86, 81, 81, 86, 86, 86, 86, 81, 141, 178, 178, 178, 178, 178, - 178, 178, 48, 48, 48, 48, 86, 48, 48, 48, 48, 48, 48, 81, 48, 48, 141, - 81, 81, 48, 0, 0, 0, 0, 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 0, 0, 44, 44, + 44, 83, 83, 83, 83, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 81, 81, 81, + 83, 178, 86, 86, 86, 86, 86, 81, 81, 86, 86, 86, 86, 81, 141, 178, 178, + 178, 178, 178, 178, 178, 48, 48, 48, 48, 86, 48, 48, 48, 48, 48, 48, 81, + 48, 48, 141, 81, 81, 48, 0, 0, 0, 0, 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 51, 51, 51, - 53, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 53, 51, 51, 51, 51, 51, - 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 53, 51, 51, 51, 51, - 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, - 51, 51, 51, 51, 51, 51, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 51, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 51, 51, 51, 53, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 53, 51, 51, + 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 53, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, + 51, 51, 51, 51, 51, 51, 51, 51, 51, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 51, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, - 51, 81, 81, 86, 81, 81, 81, 81, 81, 81, 81, 86, 81, 81, 179, 180, 86, - 181, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, + 51, 51, 51, 51, 81, 81, 86, 81, 81, 81, 81, 81, 81, 81, 86, 81, 81, 179, + 180, 86, 181, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, - 81, 81, 182, 88, 88, 86, 183, 81, 184, 86, 81, 86, 38, 43, 38, 43, 38, + 81, 81, 81, 81, 182, 88, 88, 86, 183, 81, 184, 86, 81, 86, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, @@ -1761,245 +1784,245 @@ static const unsigned short index2[] = { 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, - 43, 43, 43, 43, 43, 35, 185, 47, 47, 44, 47, 38, 43, 38, 43, 38, 43, 38, + 43, 38, 43, 43, 43, 43, 43, 35, 185, 47, 47, 44, 47, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, - 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 44, 47, 44, 47, 44, 47, 43, - 43, 43, 43, 43, 43, 43, 43, 38, 38, 38, 38, 38, 38, 38, 38, 43, 43, 43, - 43, 43, 43, 0, 0, 38, 38, 38, 38, 38, 38, 0, 0, 43, 43, 43, 43, 43, 43, - 43, 43, 38, 38, 38, 38, 38, 38, 38, 38, 43, 43, 43, 43, 43, 43, 43, 43, - 38, 38, 38, 38, 38, 38, 38, 38, 43, 43, 43, 43, 43, 43, 0, 0, 38, 38, 38, - 38, 38, 38, 0, 0, 43, 43, 43, 43, 43, 43, 43, 43, 0, 38, 0, 38, 0, 38, 0, - 38, 43, 43, 43, 43, 43, 43, 43, 43, 38, 38, 38, 38, 38, 38, 38, 38, 43, - 186, 43, 186, 43, 186, 43, 186, 43, 186, 43, 186, 43, 186, 0, 0, 43, 43, - 43, 43, 43, 43, 43, 43, 187, 187, 187, 187, 187, 187, 187, 187, 43, 43, - 43, 43, 43, 43, 43, 43, 187, 187, 187, 187, 187, 187, 187, 187, 43, 43, - 43, 43, 43, 43, 43, 43, 187, 187, 187, 187, 187, 187, 187, 187, 43, 43, - 43, 43, 43, 0, 43, 43, 38, 38, 38, 188, 187, 58, 186, 58, 58, 76, 43, 43, - 43, 0, 43, 43, 38, 188, 38, 188, 187, 76, 76, 76, 43, 43, 43, 186, 0, 0, - 43, 43, 38, 38, 38, 188, 0, 76, 76, 76, 43, 43, 43, 186, 43, 43, 43, 43, - 38, 38, 38, 188, 38, 76, 189, 189, 0, 0, 43, 43, 43, 0, 43, 43, 38, 188, - 38, 188, 187, 189, 58, 0, 190, 190, 191, 191, 191, 191, 191, 191, 191, - 191, 191, 177, 177, 177, 192, 193, 194, 195, 84, 194, 194, 194, 22, 196, - 197, 198, 199, 200, 197, 198, 199, 200, 22, 22, 22, 138, 201, 201, 201, - 22, 202, 203, 204, 205, 206, 207, 208, 21, 209, 110, 209, 210, 211, 22, - 196, 196, 138, 28, 36, 22, 196, 138, 201, 212, 212, 138, 138, 138, 213, - 165, 166, 196, 196, 196, 138, 138, 138, 138, 138, 138, 138, 138, 78, 138, - 212, 138, 138, 196, 138, 138, 138, 138, 138, 138, 138, 191, 177, 177, - 177, 177, 177, 0, 214, 215, 216, 217, 177, 177, 177, 177, 177, 177, 218, - 51, 0, 0, 34, 218, 218, 218, 218, 218, 219, 219, 220, 221, 222, 223, 218, - 34, 34, 34, 34, 218, 218, 218, 218, 218, 219, 219, 220, 221, 222, 0, 51, - 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 0, 0, 0, 85, 85, 85, 85, - 85, 85, 85, 85, 224, 225, 85, 85, 23, 85, 85, 85, 85, 85, 85, 85, 85, 85, - 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 81, 81, 178, 178, 81, 81, 81, 81, 178, 178, 178, 81, 81, - 82, 82, 82, 82, 81, 82, 82, 82, 178, 178, 81, 86, 81, 178, 178, 86, 86, - 86, 86, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 226, 49, - 227, 26, 227, 226, 49, 26, 227, 35, 49, 49, 49, 35, 35, 49, 49, 49, 46, - 26, 49, 227, 26, 78, 49, 49, 49, 49, 49, 26, 26, 226, 227, 227, 26, 49, - 26, 228, 26, 49, 26, 188, 228, 49, 49, 229, 35, 49, 49, 44, 49, 35, 158, - 158, 158, 158, 35, 26, 226, 35, 35, 49, 49, 230, 78, 78, 78, 78, 49, 35, - 35, 35, 35, 26, 78, 26, 26, 47, 80, 231, 231, 231, 37, 37, 231, 231, 231, - 231, 231, 231, 37, 37, 37, 37, 231, 232, 232, 232, 232, 232, 232, 232, - 232, 232, 232, 232, 232, 233, 233, 233, 233, 232, 232, 232, 232, 232, - 232, 232, 232, 232, 232, 233, 233, 233, 233, 233, 233, 175, 175, 175, 44, - 47, 175, 175, 175, 175, 37, 26, 26, 0, 0, 0, 0, 40, 40, 40, 40, 40, 30, - 30, 30, 30, 30, 234, 234, 26, 26, 26, 26, 78, 26, 26, 78, 26, 26, 78, 26, - 26, 26, 26, 26, 26, 26, 234, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 30, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 235, 234, 234, 26, 26, 40, 26, 40, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 40, - 236, 237, 237, 238, 78, 78, 40, 237, 238, 236, 237, 238, 236, 78, 40, 78, - 237, 239, 240, 78, 237, 236, 78, 78, 78, 237, 236, 236, 237, 40, 237, - 237, 236, 236, 40, 238, 40, 238, 40, 40, 40, 40, 237, 241, 230, 237, 230, - 230, 236, 236, 236, 40, 40, 40, 40, 78, 236, 78, 236, 237, 237, 236, 236, - 236, 238, 236, 236, 238, 236, 236, 238, 237, 238, 236, 236, 237, 78, 78, - 78, 78, 78, 237, 236, 236, 236, 78, 78, 78, 78, 78, 78, 78, 78, 78, 236, - 242, 40, 238, 78, 237, 237, 237, 237, 236, 236, 237, 237, 78, 234, 242, - 242, 238, 238, 236, 236, 238, 238, 236, 236, 238, 238, 236, 236, 236, - 236, 236, 236, 238, 238, 237, 237, 238, 238, 237, 237, 238, 238, 236, - 236, 236, 78, 78, 236, 236, 236, 236, 78, 78, 40, 78, 78, 236, 40, 78, - 78, 78, 78, 78, 78, 78, 78, 236, 236, 78, 40, 236, 236, 236, 236, 236, - 236, 238, 238, 238, 238, 236, 236, 236, 236, 236, 236, 236, 236, 236, 78, - 78, 78, 78, 78, 236, 237, 78, 78, 78, 78, 78, 78, 78, 78, 78, 236, 236, - 236, 236, 236, 78, 78, 236, 236, 78, 78, 78, 78, 236, 236, 236, 236, 236, - 236, 236, 236, 236, 236, 238, 238, 238, 238, 236, 236, 236, 236, 236, - 236, 238, 238, 238, 238, 78, 78, 236, 236, 236, 236, 236, 236, 236, 236, - 236, 236, 236, 236, 236, 236, 236, 236, 26, 26, 26, 26, 26, 26, 26, 26, - 165, 166, 165, 166, 26, 26, 26, 26, 26, 26, 30, 26, 26, 26, 26, 26, 26, - 26, 243, 243, 26, 26, 26, 26, 236, 236, 26, 26, 26, 26, 26, 26, 26, 244, - 245, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 80, 80, 80, 80, 80, 80, + 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 44, 47, 44, 47, 44, + 47, 43, 43, 43, 43, 43, 43, 43, 43, 38, 38, 38, 38, 38, 38, 38, 38, 43, + 43, 43, 43, 43, 43, 0, 0, 38, 38, 38, 38, 38, 38, 0, 0, 43, 43, 43, 43, + 43, 43, 43, 43, 38, 38, 38, 38, 38, 38, 38, 38, 43, 43, 43, 43, 43, 43, + 43, 43, 38, 38, 38, 38, 38, 38, 38, 38, 43, 43, 43, 43, 43, 43, 0, 0, 38, + 38, 38, 38, 38, 38, 0, 0, 43, 43, 43, 43, 43, 43, 43, 43, 0, 38, 0, 38, + 0, 38, 0, 38, 43, 43, 43, 43, 43, 43, 43, 43, 38, 38, 38, 38, 38, 38, 38, + 38, 43, 186, 43, 186, 43, 186, 43, 186, 43, 186, 43, 186, 43, 186, 0, 0, + 43, 43, 43, 43, 43, 43, 43, 43, 187, 187, 187, 187, 187, 187, 187, 187, + 43, 43, 43, 43, 43, 43, 43, 43, 187, 187, 187, 187, 187, 187, 187, 187, + 43, 43, 43, 43, 43, 43, 43, 43, 187, 187, 187, 187, 187, 187, 187, 187, + 43, 43, 43, 43, 43, 0, 43, 43, 38, 38, 38, 188, 187, 58, 186, 58, 58, 76, + 43, 43, 43, 0, 43, 43, 38, 188, 38, 188, 187, 76, 76, 76, 43, 43, 43, + 186, 0, 0, 43, 43, 38, 38, 38, 188, 0, 76, 76, 76, 43, 43, 43, 186, 43, + 43, 43, 43, 38, 38, 38, 188, 38, 76, 189, 189, 0, 0, 43, 43, 43, 0, 43, + 43, 38, 188, 38, 188, 187, 189, 58, 0, 190, 190, 191, 191, 191, 191, 191, + 191, 191, 191, 191, 177, 177, 177, 192, 193, 194, 195, 84, 194, 194, 194, + 22, 196, 197, 198, 199, 200, 197, 198, 199, 200, 22, 22, 22, 138, 201, + 201, 201, 22, 202, 203, 204, 205, 206, 207, 208, 21, 209, 110, 209, 210, + 211, 22, 196, 196, 138, 28, 36, 22, 196, 138, 201, 212, 212, 138, 138, + 138, 213, 165, 166, 196, 196, 196, 138, 138, 138, 138, 138, 138, 138, + 138, 78, 138, 212, 138, 138, 196, 138, 138, 138, 138, 138, 138, 138, 191, + 177, 177, 177, 177, 177, 0, 214, 215, 216, 217, 177, 177, 177, 177, 177, + 177, 218, 51, 0, 0, 34, 218, 218, 218, 218, 218, 219, 219, 220, 221, 222, + 223, 218, 34, 34, 34, 34, 218, 218, 218, 218, 218, 219, 219, 220, 221, + 222, 0, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 0, 0, 0, 85, + 85, 85, 85, 85, 85, 85, 85, 224, 225, 85, 85, 23, 85, 85, 85, 85, 85, 85, + 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 81, 178, 178, 81, 81, 81, 81, 178, 178, + 178, 81, 81, 82, 82, 82, 82, 81, 82, 82, 82, 178, 178, 81, 86, 81, 178, + 178, 86, 86, 86, 86, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 226, 226, 49, 227, 26, 227, 226, 49, 26, 227, 35, 49, 49, 49, 35, 35, 49, + 49, 49, 46, 26, 49, 227, 26, 78, 49, 49, 49, 49, 49, 26, 26, 226, 227, + 227, 26, 49, 26, 228, 26, 49, 26, 188, 228, 49, 49, 229, 35, 49, 49, 44, + 49, 35, 158, 158, 158, 158, 35, 26, 226, 35, 35, 49, 49, 230, 78, 78, 78, + 78, 49, 35, 35, 35, 35, 26, 78, 26, 26, 47, 80, 231, 231, 231, 37, 37, + 231, 231, 231, 231, 231, 231, 37, 37, 37, 37, 231, 232, 232, 232, 232, + 232, 232, 232, 232, 232, 232, 232, 232, 233, 233, 233, 233, 232, 232, + 232, 232, 232, 232, 232, 232, 232, 232, 233, 233, 233, 233, 233, 233, + 175, 175, 175, 44, 47, 175, 175, 175, 175, 37, 26, 26, 0, 0, 0, 0, 40, + 40, 40, 40, 40, 30, 30, 30, 30, 30, 234, 234, 26, 26, 26, 26, 78, 26, 26, + 78, 26, 26, 78, 26, 26, 26, 26, 26, 26, 26, 234, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 30, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 235, 234, 234, 26, 26, 40, 26, 40, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 78, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 40, 236, 237, 237, 238, 78, 78, 40, 237, 238, 236, 237, + 238, 236, 78, 40, 78, 237, 239, 240, 78, 237, 236, 78, 78, 78, 237, 236, + 236, 237, 40, 237, 237, 236, 236, 40, 238, 40, 238, 40, 40, 40, 40, 237, + 241, 230, 237, 230, 230, 236, 236, 236, 40, 40, 40, 40, 78, 236, 78, 236, + 237, 237, 236, 236, 236, 238, 236, 236, 238, 236, 236, 238, 237, 238, + 236, 236, 237, 78, 78, 78, 78, 78, 237, 236, 236, 236, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 236, 242, 40, 238, 78, 237, 237, 237, 237, 236, 236, + 237, 237, 78, 238, 242, 242, 238, 238, 236, 236, 238, 238, 236, 236, 238, + 238, 236, 236, 236, 236, 236, 236, 238, 238, 237, 237, 238, 238, 237, + 237, 238, 238, 236, 236, 236, 78, 78, 236, 236, 236, 236, 78, 78, 40, 78, + 78, 236, 40, 78, 78, 78, 78, 78, 78, 78, 78, 236, 236, 78, 40, 236, 236, + 236, 236, 236, 236, 238, 238, 238, 238, 236, 236, 236, 236, 236, 236, + 236, 236, 236, 78, 78, 78, 78, 78, 236, 237, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 236, 236, 236, 236, 236, 78, 78, 236, 236, 78, 78, 78, 78, 236, + 236, 236, 236, 236, 236, 236, 236, 236, 236, 238, 238, 238, 238, 236, + 236, 236, 236, 236, 236, 238, 238, 238, 238, 78, 78, 236, 236, 236, 236, + 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 26, 26, 26, + 26, 26, 26, 26, 26, 165, 166, 165, 166, 26, 26, 26, 26, 26, 26, 30, 26, + 26, 26, 26, 26, 26, 26, 243, 243, 26, 26, 26, 26, 236, 236, 26, 26, 26, + 26, 26, 26, 26, 244, 245, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 26, 78, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 80, - 26, 26, 26, 26, 26, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 26, 26, 26, 26, 26, 26, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 26, 78, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 80, 26, 26, 26, 26, 26, 78, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 78, 78, - 78, 78, 78, 78, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 26, 26, - 26, 243, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 78, 78, 78, 78, 78, 78, 26, 26, 26, 26, 26, 26, 26, 243, 243, + 243, 243, 26, 26, 26, 243, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 37, 37, 37, 37, 37, 37, 37, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, - 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 246, 246, - 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, + 37, 37, 37, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, - 246, 246, 246, 246, 246, 246, 231, 247, 247, 247, 247, 247, 247, 247, - 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 30, - 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 231, 247, + 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, + 247, 247, 247, 247, 247, 247, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, - 30, 30, 30, 26, 26, 26, 26, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 26, 26, 26, 26, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, - 30, 30, 30, 30, 30, 30, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 26, - 26, 30, 30, 30, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 30, 26, - 30, 30, 30, 30, 30, 30, 30, 26, 26, 26, 26, 26, 26, 26, 26, 30, 30, 26, - 26, 30, 40, 26, 26, 26, 26, 30, 30, 26, 26, 30, 40, 26, 26, 26, 26, 30, - 30, 30, 26, 26, 30, 26, 26, 30, 30, 30, 30, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 30, 30, 30, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 30, 26, 26, 26, 26, 26, 26, 26, 26, 78, 78, 78, 78, 78, - 248, 248, 78, 26, 26, 26, 26, 26, 30, 30, 26, 26, 30, 26, 26, 26, 26, 30, - 30, 26, 26, 26, 26, 243, 243, 26, 26, 26, 26, 26, 26, 30, 26, 30, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 26, 30, 26, 26, - 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 30, 26, 30, 30, 30, - 26, 30, 30, 30, 30, 26, 30, 30, 26, 40, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 30, 30, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 80, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, - 30, 26, 26, 26, 26, 243, 243, 30, 30, 30, 30, 30, 30, 30, 30, 243, 30, - 30, 30, 30, 30, 243, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, - 26, 30, 26, 26, 26, 26, 30, 30, 243, 30, 30, 30, 30, 30, 30, 30, 243, - 243, 30, 243, 30, 30, 30, 30, 243, 30, 30, 243, 30, 30, 26, 26, 26, 26, - 26, 243, 26, 26, 26, 26, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 243, 26, 243, 26, 26, 26, 26, 243, 243, 243, 26, 243, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 165, 166, 165, 166, 165, - 166, 165, 166, 165, 166, 165, 166, 165, 166, 247, 247, 247, 247, 247, - 247, 247, 247, 247, 247, 155, 155, 155, 155, 155, 155, 155, 155, 155, - 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 26, 243, 243, 243, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 26, 26, 30, 30, 30, 30, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 30, 30, 26, 30, 30, 30, 30, 30, 30, 30, 26, 26, 26, + 26, 26, 26, 26, 26, 30, 30, 26, 26, 30, 40, 26, 26, 26, 26, 30, 30, 26, + 26, 30, 40, 26, 26, 26, 26, 30, 30, 30, 26, 26, 30, 26, 26, 30, 30, 30, + 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, + 30, 30, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 26, 26, 26, 26, 26, + 26, 26, 26, 78, 78, 78, 78, 78, 248, 248, 78, 26, 26, 26, 26, 26, 30, 30, + 26, 26, 30, 26, 26, 26, 26, 30, 30, 26, 26, 26, 26, 243, 243, 26, 26, 26, + 26, 26, 26, 30, 26, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, + 26, 26, 26, 26, 26, 30, 26, 30, 26, 26, 26, 26, 26, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 30, 30, 26, 30, 30, 30, 26, 30, 30, 30, 30, 26, 30, 30, + 26, 40, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 26, + 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 30, 26, 243, 26, + 26, 26, 26, 26, 26, 26, 26, 243, 243, 80, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 30, 26, 26, 26, 26, 243, 243, + 30, 30, 30, 30, 30, 30, 30, 30, 243, 30, 30, 30, 30, 30, 243, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 26, 30, 26, 26, 26, 26, 30, 30, + 243, 30, 30, 30, 30, 30, 30, 30, 243, 243, 30, 243, 30, 30, 30, 30, 243, + 30, 30, 243, 30, 30, 26, 26, 26, 26, 26, 243, 26, 26, 26, 26, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 243, 236, 78, 78, 236, 236, 165, 166, 78, 236, 236, 78, 236, - 236, 236, 78, 78, 78, 78, 78, 236, 236, 236, 236, 78, 78, 78, 78, 78, - 236, 236, 236, 78, 78, 78, 236, 236, 236, 236, 9, 10, 9, 10, 9, 10, 9, - 10, 165, 166, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 26, 243, 26, 26, 26, 26, + 243, 243, 243, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 165, 166, 165, 166, 165, 166, 165, 166, 165, 166, 165, + 166, 165, 166, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 155, + 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, + 155, 155, 155, 155, 155, 26, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 236, 78, 78, + 236, 236, 165, 166, 78, 236, 236, 78, 236, 236, 236, 78, 78, 78, 78, 78, + 236, 236, 236, 236, 78, 78, 78, 78, 78, 236, 236, 236, 78, 78, 78, 236, + 236, 236, 236, 9, 10, 9, 10, 9, 10, 9, 10, 165, 166, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 78, 78, 78, 165, 166, 9, 10, 165, 166, 165, 166, 165, - 166, 165, 166, 165, 166, 165, 166, 165, 166, 165, 166, 165, 166, 78, 78, - 236, 236, 236, 236, 236, 236, 78, 236, 236, 236, 236, 236, 236, 236, 236, - 236, 236, 236, 236, 236, 236, 78, 78, 78, 78, 78, 78, 78, 78, 236, 78, - 78, 78, 78, 78, 78, 78, 236, 236, 236, 236, 236, 236, 78, 78, 78, 236, - 78, 78, 78, 78, 236, 236, 236, 236, 236, 78, 236, 236, 78, 78, 165, 166, - 165, 166, 236, 78, 78, 78, 78, 236, 78, 236, 236, 236, 78, 78, 236, 236, - 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 236, 236, 236, 236, 236, 236, 78, - 78, 165, 166, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 236, 236, - 230, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, - 236, 236, 236, 78, 236, 236, 236, 236, 78, 78, 236, 78, 236, 78, 78, 236, - 78, 236, 236, 236, 236, 78, 78, 78, 78, 78, 236, 236, 78, 78, 78, 78, 78, - 78, 236, 236, 236, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 236, 236, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 78, 236, 236, 78, 78, 78, 78, 236, 236, 236, 236, 78, - 236, 236, 78, 78, 236, 230, 220, 220, 78, 78, 236, 236, 236, 236, 236, + 165, 166, 9, 10, 165, 166, 165, 166, 165, 166, 165, 166, 165, 166, 165, + 166, 165, 166, 165, 166, 165, 166, 78, 78, 236, 236, 236, 236, 236, 236, + 78, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, + 78, 78, 78, 78, 78, 78, 78, 78, 236, 78, 78, 78, 78, 78, 78, 78, 236, + 236, 236, 236, 236, 236, 78, 78, 78, 236, 78, 78, 78, 78, 236, 236, 236, + 236, 236, 78, 236, 236, 78, 78, 165, 166, 165, 166, 236, 78, 78, 78, 78, + 236, 78, 236, 236, 236, 78, 78, 236, 236, 78, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 236, 236, 236, 236, 236, 236, 78, 78, 165, 166, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 236, 236, 230, 236, 236, 236, 236, 236, + 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 78, 236, 236, 236, + 236, 78, 78, 236, 78, 236, 78, 78, 236, 78, 236, 236, 236, 236, 78, 78, + 78, 78, 78, 236, 236, 78, 78, 78, 78, 78, 78, 236, 236, 236, 78, 78, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 78, 236, 236, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 236, + 236, 78, 78, 78, 78, 236, 236, 236, 236, 78, 236, 236, 78, 78, 236, 230, + 220, 220, 78, 78, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, - 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 78, 78, 236, 236, 236, - 236, 236, 236, 236, 236, 78, 236, 236, 236, 236, 236, 236, 236, 236, 236, + 236, 236, 236, 236, 78, 78, 236, 236, 236, 236, 236, 236, 236, 236, 78, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, - 236, 236, 236, 78, 78, 78, 78, 78, 249, 78, 236, 78, 78, 78, 236, 236, - 236, 236, 236, 78, 78, 78, 78, 78, 236, 236, 236, 78, 78, 78, 78, 236, - 78, 78, 78, 236, 236, 236, 236, 236, 78, 236, 78, 78, 26, 26, 26, 26, 26, + 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 78, 78, 78, + 78, 78, 249, 78, 236, 78, 78, 78, 236, 236, 236, 236, 236, 78, 78, 78, + 78, 78, 236, 236, 236, 78, 78, 78, 78, 236, 78, 78, 78, 236, 236, 236, + 236, 236, 78, 236, 78, 78, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 26, 26, 78, 78, 78, 78, 78, 78, - 26, 26, 26, 243, 26, 26, 26, 26, 243, 30, 30, 30, 30, 26, 26, 26, 26, 26, + 26, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 26, 26, 78, 78, 78, 78, 78, 78, 26, 26, 26, 243, 26, 26, + 26, 26, 243, 30, 30, 30, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 250, 26, 44, 44, + 26, 26, 26, 26, 26, 26, 26, 26, 250, 26, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 47, 47, 47, 47, 47, 47, 47, 47, + 44, 44, 44, 44, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 44, 47, 44, 44, 44, 47, 47, 44, 47, 44, 47, 44, 47, 44, - 44, 44, 44, 47, 44, 47, 47, 44, 47, 47, 47, 47, 47, 47, 51, 51, 44, 44, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 44, 47, + 44, 44, 44, 47, 47, 44, 47, 44, 47, 44, 47, 44, 44, 44, 44, 47, 44, 47, + 47, 44, 47, 47, 47, 47, 47, 47, 51, 51, 44, 44, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, - 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 47, 26, 26, 26, 26, 26, 26, 44, - 47, 44, 47, 81, 81, 81, 44, 47, 0, 0, 0, 0, 0, 138, 138, 138, 138, 155, - 138, 138, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 44, 47, 44, 47, 47, 26, 26, 26, 26, 26, 26, 44, 47, 44, 47, 81, 81, 81, + 44, 47, 0, 0, 0, 0, 0, 138, 138, 138, 138, 155, 138, 138, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 0, 47, 0, 0, 0, 0, 0, 47, 0, 0, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 0, 47, 0, + 0, 0, 0, 0, 47, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, - 0, 51, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, - 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, - 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, - 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 81, 81, 81, 81, + 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 51, 83, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, + 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, + 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, + 0, 48, 48, 48, 48, 48, 48, 48, 0, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, - 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 138, 138, 28, 36, 28, 36, 138, - 138, 138, 28, 36, 138, 28, 36, 138, 138, 138, 138, 138, 138, 138, 138, - 138, 84, 138, 138, 84, 138, 28, 36, 138, 138, 28, 36, 165, 166, 165, 166, - 165, 166, 165, 166, 138, 138, 138, 138, 138, 52, 138, 138, 138, 138, 138, - 138, 138, 138, 138, 138, 84, 84, 138, 138, 138, 138, 84, 138, 199, 138, - 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 26, 26, 138, - 138, 138, 165, 166, 165, 166, 165, 166, 165, 166, 84, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, - 243, 243, 243, 243, 251, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 81, 81, 81, 81, 138, 138, 28, 36, 28, 36, 138, 138, 138, 28, 36, 138, 28, + 36, 138, 138, 138, 138, 138, 138, 138, 138, 138, 84, 138, 138, 84, 138, + 28, 36, 138, 138, 28, 36, 165, 166, 165, 166, 165, 166, 165, 166, 138, + 138, 138, 138, 138, 52, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, + 84, 84, 138, 138, 138, 138, 84, 138, 199, 138, 138, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 138, 138, 26, 26, 138, 138, 138, 165, 166, 165, + 166, 165, 166, 165, 166, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 243, 243, 243, 243, 251, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 251, - 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 251, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, @@ -2014,59 +2037,60 @@ static const unsigned short index2[] = { 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, - 251, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 252, 253, 253, 253, 243, 254, 172, 255, 256, - 257, 256, 257, 256, 257, 256, 257, 256, 257, 243, 243, 256, 257, 256, - 257, 256, 257, 256, 257, 258, 259, 260, 260, 243, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 261, 262, 263, 264, 265, 265, 258, 254, 254, - 254, 254, 254, 251, 243, 266, 266, 266, 254, 172, 253, 243, 26, 0, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 267, 172, 267, 172, + 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 252, 253, + 253, 253, 243, 254, 172, 255, 256, 257, 256, 257, 256, 257, 256, 257, + 256, 257, 243, 243, 256, 257, 256, 257, 256, 257, 256, 257, 258, 259, + 260, 260, 243, 255, 255, 255, 255, 255, 255, 255, 255, 255, 261, 262, + 263, 264, 265, 265, 258, 254, 254, 254, 254, 254, 251, 243, 266, 266, + 266, 254, 172, 253, 243, 26, 0, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, + 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 172, + 267, 172, 267, 172, 267, 172, 172, 172, 172, 172, 172, 267, 267, 172, + 267, 267, 172, 267, 267, 172, 267, 267, 172, 267, 267, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 267, 172, 172, 0, 0, 268, 268, 269, 269, 254, + 270, 271, 258, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, - 267, 172, 267, 172, 267, 172, 172, 267, 172, 267, 172, 267, 172, 172, - 172, 172, 172, 172, 267, 267, 172, 267, 267, 172, 267, 267, 172, 267, - 267, 172, 267, 267, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 267, 172, - 172, 0, 0, 268, 268, 269, 269, 254, 270, 271, 258, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 267, 172, 267, 172, 267, 172, 267, - 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, - 172, 267, 172, 172, 267, 172, 267, 172, 267, 172, 172, 172, 172, 172, - 172, 267, 267, 172, 267, 267, 172, 267, 267, 172, 267, 267, 172, 267, - 267, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 267, 172, 172, 267, 267, - 267, 267, 253, 254, 254, 270, 271, 0, 0, 0, 0, 0, 172, 172, 172, 172, + 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 172, 267, 172, 267, + 172, 267, 172, 172, 172, 172, 172, 172, 267, 267, 172, 267, 267, 172, + 267, 267, 172, 267, 267, 172, 267, 267, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 267, 172, 172, 267, 267, 267, 267, 253, 254, 254, 270, 271, 0, + 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 0, 271, 271, 271, - 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 0, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - 271, 271, 271, 271, 271, 271, 271, 0, 272, 272, 273, 273, 273, 273, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 172, 172, 172, 172, 172, + 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 0, 272, + 272, 273, 273, 273, 273, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 243, + 172, 172, 172, 172, 172, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 251, 251, 0, 273, 273, 273, 273, 273, 273, 273, 273, 273, - 273, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 251, 251, 0, 273, 273, + 273, 273, 273, 273, 273, 273, 273, 273, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 275, 275, 275, 275, 275, 275, 275, 275, 251, 276, 276, - 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 275, 275, 275, 275, + 275, 275, 275, 275, 251, 276, 276, 276, 276, 276, 276, 276, 276, 276, + 276, 276, 276, 276, 276, 276, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 251, - 251, 251, 272, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, 274, + 274, 274, 274, 274, 274, 274, 251, 251, 251, 272, 273, 273, 273, 273, + 273, 273, 273, 273, 273, 273, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 276, 276, 276, 276, - 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 251, 251, 251, 251, 274, + 274, 274, 274, 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, + 276, 276, 276, 276, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 251, 251, 251, 251, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, @@ -2078,18 +2102,17 @@ static const unsigned short index2[] = { 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 251, 251, - 251, 251, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 251, 251, 251, 251, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 251, 251, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 251, 251, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 251, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 251, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, @@ -2102,12 +2125,14 @@ static const unsigned short index2[] = { 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 254, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 254, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, @@ -2115,100 +2140,100 @@ static const unsigned short index2[] = { 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 0, 0, 0, + 172, 172, 172, 172, 172, 172, 172, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 53, 53, 53, 53, 53, 53, 83, 83, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 53, 138, 138, 138, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, - 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, - 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 48, 81, - 82, 82, 82, 138, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 138, 52, 44, 47, - 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, - 44, 47, 44, 47, 44, 47, 44, 47, 51, 51, 81, 81, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 53, 53, 53, 53, 53, 53, 83, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 53, 138, 138, 138, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, + 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, + 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, + 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, + 47, 44, 47, 44, 47, 44, 47, 44, 47, 48, 81, 82, 82, 82, 138, 81, 81, 81, + 81, 81, 81, 81, 81, 81, 81, 138, 52, 44, 47, 44, 47, 44, 47, 44, 47, 44, + 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, + 47, 51, 51, 81, 81, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 175, 175, 175, 175, 175, 175, - 175, 175, 175, 175, 81, 81, 83, 83, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, - 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 52, 52, 52, 52, 52, 52, 52, 52, 52, 54, 54, 44, - 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 47, 47, 44, 47, 44, + 48, 48, 48, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 81, 81, 83, + 83, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, + 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 54, 54, 44, 47, 44, 47, 44, 47, 44, 47, 44, + 47, 44, 47, 44, 47, 47, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, + 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 51, 47, 47, + 47, 47, 47, 47, 47, 47, 44, 47, 44, 47, 44, 44, 47, 44, 47, 44, 47, 44, + 47, 44, 47, 52, 277, 277, 44, 47, 44, 47, 48, 44, 47, 44, 47, 47, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, - 47, 44, 47, 44, 47, 51, 47, 47, 47, 47, 47, 47, 47, 47, 44, 47, 44, 47, - 44, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 52, 277, 277, 44, 47, 44, 47, - 48, 44, 47, 44, 47, 47, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, - 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 44, 44, 44, 44, 47, 44, 44, 44, - 44, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, - 44, 44, 44, 47, 44, 47, 0, 0, 0, 0, 0, 44, 47, 0, 47, 0, 47, 44, 47, 44, - 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 51, 51, 51, 44, 47, 48, 51, 51, 47, 48, 48, 48, 48, 48, 48, 48, 135, - 48, 48, 48, 144, 48, 48, 48, 48, 135, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 135, - 135, 141, 26, 26, 26, 26, 144, 0, 0, 0, 150, 150, 150, 150, 150, 150, 80, - 80, 85, 229, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 138, 138, 138, 138, 0, 0, 0, 0, 0, 0, 0, 0, 141, - 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 47, 44, 44, 44, 44, 44, 47, 44, 44, 44, 44, 44, 47, 44, 47, 44, 47, 44, + 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 44, 44, 44, 47, 44, 47, 44, 44, + 47, 0, 0, 44, 47, 0, 47, 0, 47, 44, 47, 44, 47, 44, 47, 44, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 51, 51, 44, 47, + 48, 51, 51, 47, 48, 48, 48, 48, 48, 48, 48, 135, 48, 48, 48, 144, 48, 48, + 48, 48, 135, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 135, 135, 141, 26, 26, 26, 26, + 144, 0, 0, 0, 150, 150, 150, 150, 150, 150, 80, 80, 85, 229, 0, 0, 0, 0, + 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, - 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, - 144, 135, 0, 0, 0, 0, 0, 0, 0, 0, 83, 83, 146, 146, 146, 146, 146, 146, - 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 81, 81, 81, 81, 81, 81, 81, 81, 81, - 81, 81, 81, 81, 81, 81, 81, 81, 81, 48, 48, 48, 48, 48, 48, 83, 83, 83, - 48, 83, 48, 48, 135, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 135, 135, 135, 135, 135, 86, 86, - 86, 83, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 141, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 172, 172, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 138, + 138, 138, 138, 0, 0, 0, 0, 0, 0, 0, 0, 141, 141, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 144, 135, 0, 0, 0, 0, 0, 0, 0, 0, + 83, 83, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, + 0, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, + 81, 48, 48, 48, 48, 48, 48, 83, 83, 83, 48, 83, 48, 48, 135, 146, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 135, 135, 135, 135, 135, 86, 86, 86, 83, 83, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 141, 176, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 83, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 0, 0, 0, - 135, 135, 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 147, 141, - 141, 135, 135, 135, 135, 141, 141, 135, 135, 141, 141, 176, 83, 83, 83, - 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 0, 53, 146, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 0, 0, 0, 0, 83, 83, 48, 48, 48, 48, 48, 135, 53, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 146, 146, 146, 146, 146, 146, 146, - 146, 146, 146, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 135, 135, 135, - 135, 135, 135, 141, 141, 135, 135, 141, 141, 135, 135, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 48, 48, 48, 135, 48, 48, 48, 48, 48, 48, 48, 48, 135, 141, 0, 0, - 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 83, 83, 83, 83, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 53, 48, - 48, 48, 48, 48, 48, 80, 80, 80, 48, 141, 135, 141, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 81, 48, 81, 81, 86, 48, 48, 81, 81, - 48, 48, 48, 48, 48, 81, 81, 48, 81, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 53, 83, 83, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 141, 135, 135, 141, 141, 83, 83, 48, 53, - 53, 141, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 0, 0, - 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 172, 172, 172, 172, 172, 172, 0, 0, 0, 135, 135, 135, 141, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 147, 141, 141, 135, 135, 135, 135, 141, + 141, 135, 135, 141, 141, 176, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, + 83, 83, 0, 53, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, + 0, 83, 83, 48, 48, 48, 48, 48, 135, 53, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 48, 48, 48, 48, 48, + 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 135, 135, 135, 135, 135, 135, 141, 141, 135, 135, + 141, 141, 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 135, 48, 48, + 48, 48, 48, 48, 48, 48, 135, 141, 0, 0, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 0, 0, 83, 83, 83, 83, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 53, 48, 48, 48, 48, 48, 48, 80, 80, 80, + 48, 141, 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 81, 48, 81, 81, 86, 48, 48, 81, 81, 48, 48, 48, 48, 48, 81, 81, 48, + 81, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 48, 48, 53, 83, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 141, 135, 135, 141, 141, 83, 83, 48, 53, 53, 141, 144, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 48, 48, 48, 48, 0, 0, + 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, + 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 277, 51, 51, 51, 51, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 51, 54, 54, 0, 0, 0, 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 277, + 51, 51, 51, 51, 47, 47, 47, 47, 47, 47, 47, 47, 47, 51, 54, 54, 0, 0, 0, + 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, - 135, 141, 141, 135, 141, 141, 83, 141, 144, 0, 0, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 267, 267, 267, 267, 267, + 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 135, 141, 141, 135, 141, 141, + 83, 141, 144, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, + 0, 0, 0, 0, 0, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, @@ -2219,13 +2244,12 @@ static const unsigned short index2[] = { 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, - 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, - 267, 267, 267, 267, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 0, 0, 0, 0, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, @@ -2234,7 +2258,8 @@ static const unsigned short index2[] = { 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - 278, 278, 278, 278, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, + 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 279, 279, + 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, @@ -2243,7 +2268,6 @@ static const unsigned short index2[] = { 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, - 279, 279, 279, 279, 279, 279, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, @@ -2253,38 +2277,39 @@ static const unsigned short index2[] = { 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, - 280, 280, 280, 280, 280, 280, 280, 280, 172, 172, 280, 172, 280, 172, - 172, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 172, 280, 172, - 280, 172, 172, 280, 280, 172, 172, 172, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 172, 172, 280, 172, 280, 172, 172, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 172, 280, 172, 280, 172, 172, 280, 280, 172, + 172, 172, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, - 280, 280, 280, 280, 280, 280, 281, 281, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 281, 281, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, - 280, 280, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 35, 35, 35, - 35, 35, 35, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 35, 35, 35, 35, - 0, 0, 0, 0, 0, 282, 283, 282, 284, 284, 284, 284, 284, 284, 284, 284, - 284, 219, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, - 282, 0, 282, 282, 282, 282, 282, 0, 282, 0, 282, 282, 0, 282, 282, 0, - 282, 282, 282, 282, 282, 282, 282, 282, 282, 284, 131, 131, 131, 131, + 281, 281, 281, 281, 281, 281, 35, 35, 35, 35, 35, 35, 35, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 35, 35, 35, 35, 35, 0, 0, 0, 0, 0, 282, 283, 282, + 284, 284, 284, 284, 284, 284, 284, 284, 284, 219, 282, 282, 282, 282, + 282, 282, 282, 282, 282, 282, 282, 282, 282, 0, 282, 282, 282, 282, 282, + 0, 282, 0, 282, 282, 0, 282, 282, 0, 282, 282, 282, 282, 282, 282, 282, + 282, 282, 284, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 140, 140, 140, 140, - 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, + 140, 140, 140, 140, 140, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, @@ -2300,27 +2325,27 @@ static const unsigned short index2[] = { 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 285, 199, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 285, 199, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 0, 0, 131, + 131, 131, 131, 131, 131, 131, 0, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 0, 0, 0, 0, 0, 0, - 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 286, 26, 26, 26, 71, 71, 71, 71, 71, 71, 71, 71, 71, - 71, 71, 71, 71, 71, 71, 71, 287, 287, 287, 287, 287, 287, 287, 288, 289, - 287, 0, 0, 0, 0, 0, 0, 81, 81, 81, 81, 81, 81, 81, 86, 86, 86, 86, 86, - 86, 86, 81, 81, 287, 290, 290, 291, 291, 288, 289, 288, 289, 288, 289, - 288, 289, 288, 289, 288, 289, 288, 289, 288, 289, 253, 253, 288, 289, - 287, 287, 287, 287, 291, 291, 291, 292, 287, 292, 0, 287, 292, 287, 287, - 290, 293, 294, 293, 294, 293, 294, 295, 287, 287, 296, 297, 298, 298, - 299, 0, 287, 300, 295, 287, 0, 0, 0, 0, 131, 131, 131, 118, 131, 0, 131, + 131, 131, 131, 131, 131, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 286, 26, 26, + 26, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 287, + 287, 287, 287, 287, 287, 287, 288, 289, 287, 0, 0, 0, 0, 0, 0, 81, 81, + 81, 81, 81, 81, 81, 86, 86, 86, 86, 86, 86, 86, 81, 81, 287, 290, 290, + 291, 291, 288, 289, 288, 289, 288, 289, 288, 289, 288, 289, 288, 289, + 288, 289, 288, 289, 253, 253, 288, 289, 287, 287, 287, 287, 291, 291, + 291, 292, 287, 292, 0, 287, 292, 287, 287, 290, 293, 294, 293, 294, 293, + 294, 295, 287, 287, 296, 297, 298, 298, 299, 0, 287, 300, 295, 287, 0, 0, + 0, 0, 131, 131, 131, 118, 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, @@ -2330,133 +2355,133 @@ static const unsigned short index2[] = { 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 131, 131, 131, 131, 0, 0, 177, 0, 301, 301, 302, 303, - 302, 301, 301, 304, 305, 301, 306, 307, 308, 307, 307, 309, 309, 309, - 309, 309, 309, 309, 309, 309, 309, 307, 301, 310, 311, 310, 301, 301, + 131, 0, 0, 177, 0, 301, 301, 302, 303, 302, 301, 301, 304, 305, 301, 306, + 307, 308, 307, 307, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, + 307, 301, 310, 311, 310, 301, 301, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, - 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 304, 301, - 305, 313, 314, 313, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, + 312, 312, 312, 312, 312, 304, 301, 305, 313, 314, 313, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, - 315, 315, 304, 311, 305, 311, 304, 305, 316, 317, 318, 316, 316, 319, - 319, 319, 319, 319, 319, 319, 319, 319, 319, 320, 319, 319, 319, 319, + 315, 315, 315, 315, 315, 315, 315, 315, 315, 304, 311, 305, 311, 304, + 305, 316, 317, 318, 316, 316, 319, 319, 319, 319, 319, 319, 319, 319, + 319, 319, 320, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 320, - 320, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, + 319, 319, 319, 319, 319, 319, 320, 320, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - 319, 319, 319, 319, 0, 0, 0, 319, 319, 319, 319, 319, 319, 0, 0, 319, - 319, 319, 319, 319, 319, 0, 0, 319, 319, 319, 319, 319, 319, 0, 0, 319, - 319, 319, 0, 0, 0, 303, 303, 311, 313, 321, 303, 303, 0, 322, 323, 323, - 323, 323, 322, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 324, 324, 26, 30, - 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 0, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, + 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 0, 0, 0, 319, 319, + 319, 319, 319, 319, 0, 0, 319, 319, 319, 319, 319, 319, 0, 0, 319, 319, + 319, 319, 319, 319, 0, 0, 319, 319, 319, 0, 0, 0, 303, 303, 311, 313, + 321, 303, 303, 0, 322, 323, 323, 323, 323, 322, 322, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 324, 324, 324, 26, 30, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 0, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 83, 138, 83, 0, 0, 0, 0, 150, - 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, + 0, 83, 138, 83, 0, 0, 0, 0, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, - 150, 150, 0, 0, 0, 80, 80, 80, 80, 80, 80, 80, 80, 80, 325, 325, 325, + 150, 150, 150, 150, 150, 150, 150, 150, 0, 0, 0, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - 325, 325, 325, 325, 325, 325, 325, 325, 155, 155, 155, 155, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 155, 155, 26, 80, - 80, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 26, + 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 155, + 155, 155, 155, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 155, 155, 26, 80, 80, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 86, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, - 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 0, 0, 0, - 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 150, 150, - 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 175, 48, 48, 48, 48, 48, 48, - 48, 48, 175, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 326, 326, 326, 326, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 81, 81, 81, 81, 81, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 83, 48, 48, 48, 48, + 175, 48, 48, 48, 48, 48, 48, 48, 48, 175, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 48, - 48, 48, 48, 48, 48, 48, 48, 83, 175, 175, 175, 175, 175, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 81, 81, + 81, 81, 81, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 0, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 83, 175, 175, + 175, 175, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, + 44, 44, 44, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, - 146, 146, 0, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 146, 146, 146, + 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 0, 0, 0, 0, 47, 47, 47, 47, 47, 47, 47, 47, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 0, 0, 0, 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 0, 0, 0, 0, 48, 48, 48, 48, 48, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 0, 0, 0, + 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, - 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 0, 44, 44, 44, 44, 44, 44, 44, 0, 44, 44, 0, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 0, 47, 47, 47, 47, 47, 47, 47, 0, 47, 47, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 0, 44, 44, 44, 44, 44, 44, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 0, 44, 44, 44, 44, 44, 44, 44, 0, 44, + 44, 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 0, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 0, 47, 47, 47, 47, 47, 47, 47, 0, + 47, 47, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 142, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 142, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, - 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 53, 51, 51, 51, 51, 51, 0, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, + 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 53, 51, 51, 51, 51, 51, 0, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, - 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 0, 51, 51, 51, 51, - 51, 51, 51, 51, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 0, 51, 51, 51, + 51, 51, 51, 51, 51, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 107, 107, 107, 107, 107, 107, 0, 0, 107, 0, 107, 107, 107, 107, + 0, 0, 0, 0, 0, 107, 107, 107, 107, 107, 107, 0, 0, 107, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 0, 107, 107, - 0, 0, 0, 107, 0, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 0, 104, - 327, 327, 327, 327, 327, 327, 327, 327, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 0, 107, + 107, 0, 0, 0, 107, 0, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 0, + 104, 327, 327, 327, 327, 327, 327, 327, 327, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 328, 328, 327, 327, 327, 327, 327, 327, 327, 107, 107, + 107, 107, 107, 107, 328, 328, 327, 327, 327, 327, 327, 327, 327, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 0, 0, 0, 0, 0, 0, 0, 0, 327, 327, 327, 327, 327, 327, 327, 327, 327, + 107, 107, 0, 0, 0, 0, 0, 0, 0, 0, 327, 327, 327, 327, 327, 327, 327, 327, + 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 0, 107, 107, 0, 0, 0, 0, 0, 327, 327, 327, 327, 327, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 327, 327, 327, 327, 327, @@ -2524,150 +2549,165 @@ static const unsigned short index2[] = { 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 81, 81, 81, 81, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 107, 107, 107, 107, 137, 107, 329, 329, 329, + 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, + 329, 329, 329, 329, 329, 0, 0, 0, 81, 81, 81, 81, 81, 84, 137, 330, 330, + 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + 330, 330, 330, 330, 330, 330, 0, 0, 0, 0, 0, 0, 0, 0, 331, 331, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 331, 331, 331, 331, 331, 331, 331, 331, 331, 331, - 331, 331, 331, 331, 331, 331, 331, 331, 331, 331, 331, 331, 331, 331, - 331, 331, 331, 331, 331, 331, 331, 0, 107, 107, 107, 107, 107, 107, 107, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, 332, 332, 332, 332, 332, 332, + 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, + 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 0, 81, 81, 102, 0, 0, 107, 107, 0, 0, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 0, 81, 81, 102, 0, 0, + 107, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 118, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 86, 86, 86, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 0, 0, 0, 0, 0, 0, 0, 135, 86, 86, 86, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 327, 327, 327, 327, 327, 327, 327, 327, 327, 327, - 107, 0, 0, 0, 0, 0, 0, 0, 0, 118, 118, 118, 118, 118, 118, 118, 118, 118, - 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 86, 86, - 81, 81, 81, 86, 81, 86, 86, 86, 86, 332, 332, 332, 332, 113, 113, 113, - 113, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 81, 86, 81, 86, 104, 104, 104, 104, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 107, 107, 107, 107, 107, 107, 107, 107, 327, 327, 327, 327, 327, 327, + 327, 327, 327, 327, 107, 0, 0, 0, 0, 0, 0, 0, 0, 118, 118, 118, 118, 118, + 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, + 118, 118, 118, 86, 86, 81, 81, 81, 86, 81, 86, 86, 86, 86, 333, 333, 333, + 333, 113, 113, 113, 113, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 327, 327, 327, - 327, 327, 327, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 141, 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 144, 83, 83, 83, 83, 83, 83, 83, 0, 0, 0, 0, 155, 155, 155, - 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, - 155, 155, 155, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 144, 48, - 48, 135, 135, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 135, 135, 141, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 142, 48, 142, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 142, 48, 48, 48, 48, 141, 141, 141, 135, 135, 135, 135, 141, 141, - 144, 143, 83, 83, 192, 83, 83, 83, 83, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 192, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 146, 146, - 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 81, 81, 81, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 151, - 135, 135, 135, 135, 141, 135, 152, 152, 135, 135, 135, 144, 144, 0, 146, - 146, 146, 146, 146, 146, 146, 146, 146, 146, 83, 83, 83, 83, 48, 141, - 141, 48, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 147, 83, 83, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 135, 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, - 141, 135, 135, 135, 135, 135, 135, 135, 135, 135, 141, 176, 48, 48, 48, - 48, 83, 83, 83, 83, 135, 147, 135, 135, 83, 141, 135, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 48, 83, 48, 83, 83, 83, 0, 150, 150, 150, - 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, - 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 141, 141, 141, 135, 135, 135, 141, 141, 135, 176, 147, 135, 83, - 83, 83, 83, 83, 83, 135, 48, 48, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 0, 48, 48, 48, 48, 0, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 83, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, + 107, 107, 107, 107, 107, 107, 107, 107, 81, 86, 81, 86, 104, 104, 104, + 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 327, 327, 327, 327, 327, 327, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 135, 141, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 135, 135, 135, 135, 135, 135, + 135, 135, 135, 135, 135, 135, 135, 135, 144, 83, 83, 83, 83, 83, 83, 83, + 0, 0, 0, 0, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, + 155, 155, 155, 155, 155, 155, 155, 155, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 144, 48, 48, 135, 135, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 144, 135, 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 142, 48, 142, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 142, 48, 48, 48, 48, 141, 141, 141, + 135, 135, 135, 135, 141, 141, 144, 143, 83, 83, 192, 83, 83, 83, 83, 135, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, + 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, + 0, 0, 0, 81, 81, 81, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 151, 135, 135, 135, 135, 141, 135, 152, 152, 135, + 135, 135, 144, 144, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, + 83, 83, 83, 83, 48, 141, 141, 48, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 135, 141, 141, 141, 135, 135, 135, 135, 135, 135, 147, - 144, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, - 0, 0, 0, 0, 0, 135, 135, 141, 141, 0, 48, 48, 48, 48, 48, 48, 48, 48, 0, - 0, 48, 48, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, - 0, 48, 48, 48, 48, 48, 0, 147, 147, 48, 148, 141, 135, 141, 141, 141, - 141, 0, 0, 141, 141, 0, 0, 149, 149, 176, 0, 0, 48, 0, 0, 0, 0, 0, 0, - 148, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 141, 141, 0, 0, 81, 81, 81, 81, - 81, 81, 81, 0, 0, 0, 81, 81, 81, 81, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 147, 83, 83, 48, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 135, 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, - 141, 141, 135, 135, 135, 135, 135, 135, 135, 135, 141, 141, 144, 135, - 135, 141, 147, 48, 48, 48, 48, 83, 83, 83, 83, 83, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 83, 83, 0, 83, 81, 48, 48, 48, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 141, 141, 141, 135, 135, 135, 135, 135, 135, 135, 135, + 135, 141, 176, 48, 48, 48, 48, 83, 83, 83, 83, 135, 147, 135, 135, 83, + 141, 135, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 48, 83, 48, + 83, 83, 83, 0, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, + 150, 150, 150, 150, 150, 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 148, 141, 141, 135, - 135, 135, 135, 135, 135, 141, 151, 149, 149, 148, 149, 135, 135, 141, - 144, 147, 48, 48, 83, 48, 0, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 141, 135, 135, 135, 141, + 141, 135, 176, 147, 135, 83, 83, 83, 83, 83, 83, 135, 48, 48, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 0, + 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 83, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 148, 141, 141, 135, 135, 135, - 135, 0, 0, 141, 141, 149, 149, 135, 135, 141, 144, 147, 83, 83, 83, 83, - 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, - 83, 48, 48, 48, 48, 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 135, 141, 141, 141, 135, 135, + 135, 135, 135, 135, 147, 144, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 135, 135, 141, 141, 0, 48, 48, + 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 0, 0, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, + 48, 48, 48, 48, 48, 0, 48, 48, 0, 48, 48, 48, 48, 48, 0, 147, 147, 48, + 148, 141, 135, 141, 141, 141, 141, 0, 0, 141, 141, 0, 0, 149, 149, 176, + 0, 0, 48, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 141, + 141, 0, 0, 81, 81, 81, 81, 81, 81, 81, 0, 0, 0, 81, 81, 81, 81, 81, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 142, 48, 142, 48, 48, 48, 48, 0, + 48, 0, 0, 142, 0, 48, 142, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 0, 48, 148, 141, 141, 151, 135, 135, 135, + 135, 135, 0, 148, 0, 0, 334, 0, 334, 334, 148, 141, 0, 141, 141, 144, + 176, 144, 48, 135, 48, 83, 83, 0, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 135, + 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 141, 135, 135, 135, 135, - 135, 135, 135, 135, 141, 141, 135, 141, 144, 135, 83, 83, 83, 48, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, - 146, 0, 0, 0, 0, 0, 0, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, - 138, 138, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 141, 141, 141, 135, 135, 135, 135, 135, 135, 135, 135, 141, + 141, 144, 135, 135, 141, 147, 48, 48, 48, 48, 83, 83, 83, 83, 83, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 146, 83, 83, 0, 83, 81, 48, 48, + 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 148, + 141, 141, 135, 135, 135, 135, 135, 135, 141, 151, 149, 149, 148, 149, + 135, 135, 141, 144, 147, 48, 48, 83, 48, 0, 0, 0, 0, 0, 0, 0, 0, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 135, 141, 135, 141, 141, 135, 135, 135, 135, - 135, 135, 176, 147, 48, 83, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 148, 141, + 141, 135, 135, 135, 135, 0, 0, 141, 141, 149, 149, 135, 135, 141, 144, + 147, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, + 83, 83, 83, 83, 83, 83, 48, 48, 48, 48, 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, + 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 141, 135, + 135, 135, 135, 135, 135, 135, 135, 141, 141, 135, 141, 144, 135, 83, 83, + 83, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 138, 138, 138, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 0, 0, 135, 135, 135, 141, 141, 135, 135, 135, 135, 141, 135, 135, - 135, 135, 144, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, - 146, 150, 150, 83, 83, 83, 80, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 135, 141, 135, 141, 141, 135, + 135, 135, 135, 135, 135, 176, 147, 48, 83, 0, 0, 0, 0, 0, 0, 146, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 146, 146, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, + 135, 141, 135, 141, 141, 135, 135, 135, 135, 141, 135, 135, 135, 135, + 144, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 150, + 150, 83, 83, 83, 80, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 141, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 141, 144, 147, 83, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 141, 135, 135, 135, 135, + 135, 135, 135, 135, 135, 141, 144, 147, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 44, 44, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 47, 47, 47, 47, 47, 47, 47, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 146, 146, 146, 146, 146, 146, 146, 146, 146, - 146, 150, 150, 150, 150, 150, 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 48, 0, 0, 48, 48, 48, - 48, 48, 48, 48, 48, 0, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 148, 141, 141, - 141, 141, 141, 0, 141, 149, 0, 0, 135, 135, 176, 144, 48, 141, 48, 141, - 147, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, - 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 47, 47, 47, 47, 47, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, + 150, 150, 150, 150, 150, 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 48, 0, 0, 48, 48, 48, 48, + 48, 48, 48, 48, 0, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 148, 141, 141, 141, + 141, 141, 0, 141, 149, 0, 0, 135, 135, 176, 144, 48, 141, 48, 141, 147, + 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 48, 48, 48, 48, + 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 141, 135, 135, 135, 135, 0, 0, 135, 135, 141, 141, 141, 141, 144, 48, 83, @@ -2692,10 +2732,16 @@ static const unsigned short index2[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 83, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 135, - 135, 135, 135, 135, 135, 135, 0, 135, 135, 135, 135, 135, 135, 141, 333, + 135, 135, 135, 135, 135, 135, 0, 135, 135, 135, 135, 135, 135, 141, 335, 48, 83, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 0, 0, 0, 83, 83, @@ -2730,15 +2776,15 @@ static const unsigned short index2[] = { 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 135, 135, 135, 135, 135, 0, 0, 0, 141, 141, 135, 176, 144, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 146, 146, 146, 146, 146, 146, 146, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 150, 150, 150, + 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, - 150, 150, 150, 26, 26, 26, 26, 26, 26, 26, 26, 85, 85, 85, 85, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 150, 150, 150, 150, 26, 26, 26, 26, 26, 26, 26, 26, 85, 85, 85, 85, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2771,14 +2817,21 @@ static const unsigned short index2[] = { 48, 48, 48, 48, 48, 48, 48, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 135, 48, 48, 48, 48, 48, 48, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, + 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 151, 151, 151, 336, 336, 336, 336, 336, 336, 336, 336, 151, 141, 141, + 141, 135, 135, 144, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, @@ -2808,30 +2861,36 @@ static const unsigned short index2[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, + 0, 0, 0, 0, 0, 0, 53, 53, 53, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 173, 337, 142, 142, 53, 53, 83, 83, 83, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, + 44, 44, 44, 44, 44, 44, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 150, 150, 150, - 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, - 150, 150, 150, 150, 150, 150, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 47, 47, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, + 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 0, 0, 0, 0, 135, 48, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 48, 48, 48, 48, 48, 0, 0, 0, 0, 135, 48, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, - 141, 141, 141, 0, 0, 0, 0, 0, 0, 0, 135, 135, 135, 135, 53, 53, 53, 53, - 53, 53, 53, 53, 53, 53, 53, 53, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 141, 141, 141, 141, 141, 141, 141, 0, 0, 0, 0, 0, 0, 0, 135, 135, 135, + 135, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 254, 254, 253, 254, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 335, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 254, 253, 254, 338, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 339, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, @@ -2840,16 +2899,16 @@ static const unsigned short index2[] = { 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2858,14 +2917,14 @@ static const unsigned short index2[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 254, 254, 254, 0, - 254, 254, 254, 254, 254, 254, 254, 0, 254, 254, 0, 172, 172, 172, 172, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, + 254, 254, 254, 0, 254, 254, 254, 254, 254, 254, 254, 0, 254, 254, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 172, 172, 172, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, @@ -2875,34 +2934,55 @@ static const unsigned short index2[] = { 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 0, 0, 0, 0, 48, 48, 48, 48, 48, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 0, 0, 80, 135, 178, 83, 177, 177, 177, 177, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, + 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 0, 0, 80, 135, 178, 83, 177, 177, 177, 177, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340, 341, 341, 341, 341, 341, 341, 341, 341, + 341, 341, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 135, 135, 135, 135, 135, 135, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 0, 0, 135, 135, + 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 0, 0, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 80, + 135, 135, 135, 135, 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 80, 80, 80, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 80, 80, 80, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, @@ -2914,13 +2994,13 @@ static const unsigned short index2[] = { 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 0, 0, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 336, 336, - 336, 336, 336, 336, 336, 337, 337, 178, 178, 178, 80, 80, 80, 338, 337, - 337, 337, 337, 337, 177, 177, 177, 177, 177, 177, 177, 177, 86, 86, 86, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 342, 342, + 342, 342, 342, 342, 342, 343, 343, 178, 178, 178, 80, 80, 80, 344, 343, + 343, 343, 343, 343, 177, 177, 177, 177, 177, 177, 177, 177, 86, 86, 86, 86, 86, 86, 86, 86, 80, 80, 81, 81, 81, 81, 81, 86, 86, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 81, 81, 81, 81, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 336, 336, 336, 336, 336, 336, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 342, 342, 342, 342, 342, 342, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2936,129 +3016,137 @@ static const unsigned short index2[] = { 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, - 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, - 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, - 0, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, + 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 150, + 150, 0, 0, 0, 0, 0, 0, 0, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, + 35, 0, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 49, 0, 49, 49, 0, 0, 49, 0, 0, 49, 49, 0, 0, 49, 49, 49, 49, 0, 49, 49, + 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 0, 35, 0, 35, 35, 35, 35, 35, 35, + 35, 0, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 0, 49, 49, 49, 49, 0, + 0, 49, 49, 49, 49, 49, 49, 49, 49, 0, 49, 49, 49, 49, 49, 49, 49, 0, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 49, 49, 0, 49, 49, 49, 49, 0, 49, 49, 49, 49, + 49, 0, 49, 0, 0, 0, 49, 49, 49, 49, 49, 49, 49, 0, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 0, 35, 35, 35, + 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 0, 49, 49, 0, 0, - 49, 0, 0, 49, 49, 0, 0, 49, 49, 49, 49, 0, 49, 49, 49, 49, 49, 49, 49, - 49, 35, 35, 35, 35, 0, 35, 0, 35, 35, 35, 35, 35, 35, 35, 0, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 49, 49, 0, 49, 49, 49, 49, 0, 0, 49, 49, 49, 49, - 49, 49, 49, 49, 0, 49, 49, 49, 49, 49, 49, 49, 0, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 49, 49, 0, 49, 49, 49, 49, 0, 49, 49, 49, 49, 49, 0, 49, 0, 0, 0, - 49, 49, 49, 49, 49, 49, 49, 0, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, + 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, + 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 0, 0, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 220, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 230, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 49, 49, 49, 49, 49, 220, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 230, 35, 35, 35, 35, + 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 220, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 230, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 0, 0, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 339, 35, 35, 35, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 220, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 230, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 339, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 230, 35, 35, 35, 35, 35, 35, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 339, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 230, 35, 35, 35, 35, 35, - 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 339, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 230, 35, - 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 339, 35, 35, 35, 35, 35, + 35, 35, 35, 230, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 220, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 230, 35, 35, 35, 35, 35, 35, 49, 35, 0, 0, 340, 340, 340, 340, - 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, - 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, - 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, - 340, 340, 340, 340, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, + 35, 35, 35, 35, 35, 35, 35, 230, 35, 35, 35, 35, 35, 35, 49, 35, 0, 0, + 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, + 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, + 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, + 341, 341, 341, 341, 341, 341, 341, 341, 135, 135, 135, 135, 135, 135, + 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, + 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, + 135, 135, 135, 135, 135, 135, 135, 80, 80, 80, 80, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 80, 80, 80, 80, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, + 135, 135, 135, 135, 80, 80, 80, 80, 80, 80, 80, 80, 135, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 135, 80, 80, 83, 83, 83, 83, 83, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 135, 135, 135, 135, 0, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 80, - 80, 80, 80, 80, 80, 80, 80, 135, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 135, 80, 80, 83, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 135, 135, 135, 135, 135, 0, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, + 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 0, 0, 0, 0, 0, 0, 47, 47, 47, 47, 47, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 0, 0, 0, 0, 0, 0, 47, - 47, 47, 47, 47, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 81, 81, 81, - 81, 81, 81, 0, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, - 81, 81, 81, 0, 0, 81, 81, 81, 81, 81, 81, 81, 0, 81, 81, 0, 81, 81, 81, - 81, 81, 0, 0, 0, 0, 0, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, + 0, 0, 0, 0, 0, 81, 81, 81, 81, 81, 81, 81, 0, 81, 81, 81, 81, 81, 81, 81, + 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 0, 0, 81, 81, 81, 81, 81, 81, 81, + 0, 81, 81, 0, 81, 81, 81, 81, 81, 0, 0, 0, 0, 0, 51, 51, 51, 51, 51, 51, + 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, - 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 0, 0, 0, 0, 0, 0, + 51, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 0, 0, 0, 81, 81, 81, 81, 81, 81, 81, 53, 53, 53, - 53, 53, 53, 53, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, - 0, 0, 0, 0, 48, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 81, 81, 81, 81, + 81, 81, 81, 53, 53, 53, 53, 53, 53, 53, 0, 0, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 146, 0, 0, 0, 0, 48, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 81, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 81, 81, 81, 81, 146, - 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 85, 0, 0, 0, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 81, 81, 81, 81, 146, 146, 146, 146, 146, 146, 146, 146, 146, + 146, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 53, 182, 182, 86, 81, 146, 146, 146, 146, 146, 146, 146, 146, + 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 53, 182, 182, 86, 81, - 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 81, 86, 48, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, - 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 0, 48, 48, 0, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 107, 107, 107, 107, 107, 107, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, + 48, 48, 48, 0, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, @@ -3072,112 +3160,110 @@ static const unsigned short index2[] = { 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 0, 0, 327, 327, 327, 327, - 327, 327, 327, 327, 327, 86, 86, 86, 86, 86, 86, 86, 0, 0, 0, 0, 0, 0, 0, + 107, 107, 107, 107, 0, 0, 327, 327, 327, 327, 327, 327, 327, 327, 327, + 86, 86, 86, 86, 86, 86, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 329, 329, 329, 329, 329, 329, 329, + 0, 0, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, - 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 330, 330, - 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + 329, 329, 329, 329, 329, 329, 329, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, - 330, 330, 330, 330, 81, 81, 81, 81, 81, 81, 147, 137, 0, 0, 0, 0, 136, - 136, 136, 136, 136, 136, 136, 136, 136, 136, 0, 0, 0, 0, 104, 104, 0, 0, + 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 81, 81, + 81, 81, 81, 81, 147, 137, 0, 0, 0, 0, 136, 136, 136, 136, 136, 136, 136, + 136, 136, 136, 0, 0, 0, 0, 104, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, - 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, - 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, - 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, - 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, - 332, 332, 133, 332, 332, 332, 111, 332, 332, 332, 332, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 333, 333, 333, 333, 333, 333, + 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, + 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, + 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, + 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 133, 333, 333, 333, + 111, 333, 333, 333, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, - 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, - 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, - 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, - 332, 332, 133, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, 332, - 332, 332, 332, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 333, 333, 333, 333, 333, 333, + 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, + 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, + 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 133, 333, 333, 333, + 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 131, 131, 131, 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 131, 131, 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 0, 131, 131, 0, 131, 0, 0, 131, 0, 131, 131, 131, 131, - 131, 131, 131, 131, 131, 131, 0, 131, 131, 131, 131, 0, 131, 0, 131, 0, - 0, 0, 0, 0, 0, 131, 0, 0, 0, 0, 131, 0, 131, 0, 131, 0, 131, 131, 131, 0, - 131, 131, 0, 131, 0, 0, 131, 0, 131, 0, 131, 0, 131, 0, 131, 0, 131, 131, - 0, 131, 0, 0, 131, 131, 131, 131, 0, 131, 131, 131, 131, 131, 131, 131, - 0, 131, 131, 131, 131, 0, 131, 131, 131, 131, 0, 131, 0, 131, 131, 131, - 131, 131, 131, 131, 131, 131, 131, 0, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 0, 0, 0, 0, 0, 131, - 131, 131, 0, 131, 131, 131, 131, 131, 0, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 78, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 243, 26, 26, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 0, 131, 131, + 0, 131, 0, 0, 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 0, 131, 131, 131, 131, 0, 131, 0, 131, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 0, + 131, 0, 131, 0, 131, 0, 131, 131, 131, 0, 131, 131, 0, 131, 0, 0, 131, 0, + 131, 0, 131, 0, 131, 0, 131, 0, 131, 131, 0, 131, 0, 0, 131, 131, 131, + 131, 0, 131, 131, 131, 131, 131, 131, 131, 0, 131, 131, 131, 131, 0, 131, + 131, 131, 131, 0, 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 0, 0, 0, 0, 0, 131, 131, 131, 0, 131, 131, 131, 131, + 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 0, 26, 26, + 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 243, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 155, - 155, 26, 26, 26, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 155, 155, 26, 26, 26, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, - 246, 246, 246, 246, 246, 341, 26, 246, 246, 246, 246, 246, 246, 246, 246, + 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 340, 26, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, - 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 342, 342, 342, 342, - 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, - 342, 342, 342, 342, 342, 342, 342, 342, 226, 226, 226, 26, 26, 26, 342, - 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, - 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, - 342, 272, 342, 246, 272, 272, 272, 272, 272, 272, 272, 272, 272, 272, - 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, - 342, 342, 342, 342, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 274, 274, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 274, + 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, + 246, 246, 246, 246, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, + 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, + 346, 346, 226, 226, 226, 26, 26, 26, 346, 346, 346, 346, 346, 346, 346, + 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, + 346, 346, 346, 346, 346, 346, 346, 346, 346, 272, 346, 246, 272, 272, + 272, 272, 272, 272, 272, 272, 272, 272, 346, 346, 346, 346, 346, 346, + 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 26, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 274, 274, 274, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 0, - 0, 0, 0, 274, 274, 274, 274, 274, 274, 274, 274, 274, 0, 0, 0, 0, 0, 0, - 0, 274, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, - 243, 243, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 274, 274, 274, 274, 274, 274, 274, 274, 0, 0, 0, 0, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 0, 0, 0, 0, 0, 0, 0, 274, 274, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 26, 26, 26, 26, 243, 243, 243, 243, 243, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 243, 26, 26, 26, - 243, 243, 243, 343, 343, 343, 343, 343, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, + 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 26, 26, 26, 243, 26, 26, 26, 243, 243, 243, 347, 347, 347, + 347, 347, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 26, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, @@ -3189,71 +3275,73 @@ static const unsigned short index2[] = { 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, - 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 26, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, - 243, 26, 26, 26, 243, 243, 243, 26, 26, 243, 243, 243, 0, 0, 0, 0, 243, - 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 0, - 0, 0, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, - 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 243, 26, 26, 26, 243, 243, + 243, 26, 26, 243, 243, 243, 0, 0, 0, 0, 243, 243, 243, 243, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 0, 0, 0, 26, 26, 26, 26, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, + 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, - 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 0, 0, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, + 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, @@ -3266,58 +3354,50 @@ static const unsigned short index2[] = { 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, + 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 243, 243, - 243, 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 340, 340, 340, 340, 340, 340, 340, - 340, 340, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 341, 341, + 341, 341, 341, 341, 341, 341, 341, 341, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 281, 281, 281, 281, 281, - 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 281, 281, 281, 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, @@ -3325,9 +3405,9 @@ static const unsigned short index2[] = { 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, @@ -3338,17 +3418,18 @@ static const unsigned short index2[] = { 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, @@ -3359,9 +3440,9 @@ static const unsigned short index2[] = { 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, - 280, 280, 280, 280, 280, 280, 280, 281, 281, 281, 281, 281, 281, 281, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, @@ -3377,34 +3458,34 @@ static const unsigned short index2[] = { 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 0, 0, 172, 172, 172, 172, 172, 172, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 281, - 281, 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 281, 281, 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 0, 177, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, - 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 71, - 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, + 177, 177, 177, 177, 177, 177, 177, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, @@ -3417,7 +3498,8 @@ static const unsigned short index2[] = { 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, - 71, 71, 71, 71, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, @@ -3426,7 +3508,7 @@ static const unsigned short index2[] = { 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, - 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 0, 0, + 279, 279, 279, 279, 279, 279, 279, 0, 0, }; /* decomposition data */ @@ -4203,156 +4285,161 @@ static const unsigned int decomp_data[] = { 12638, 272, 12639, 272, 12640, 272, 12641, 272, 12642, 272, 12643, 264, 162, 264, 163, 264, 172, 264, 175, 264, 166, 264, 165, 264, 8361, 272, 9474, 272, 8592, 272, 8593, 272, 8594, 272, 8595, 272, 9632, 272, 9675, - 259, 720, 259, 721, 259, 230, 259, 665, 259, 595, 259, 675, 259, 43878, - 259, 677, 259, 676, 259, 598, 259, 599, 259, 7569, 259, 600, 259, 606, - 259, 681, 259, 612, 259, 610, 259, 608, 259, 667, 259, 295, 259, 668, - 259, 615, 259, 644, 259, 682, 259, 683, 259, 620, 259, 122628, 259, - 42894, 259, 622, 259, 122629, 259, 654, 259, 122630, 259, 248, 259, 630, - 259, 631, 259, 113, 259, 634, 259, 122632, 259, 637, 259, 638, 259, 640, - 259, 680, 259, 678, 259, 43879, 259, 679, 259, 648, 259, 11377, 259, 655, - 259, 673, 259, 674, 259, 664, 259, 448, 259, 449, 259, 450, 259, 122634, - 259, 122654, 512, 69785, 69818, 512, 69787, 69818, 512, 69797, 69818, - 512, 69937, 69927, 512, 69938, 69927, 512, 70471, 70462, 512, 70471, - 70487, 512, 70841, 70842, 512, 70841, 70832, 512, 70841, 70845, 512, - 71096, 71087, 512, 71097, 71087, 512, 71989, 71984, 512, 119127, 119141, - 512, 119128, 119141, 512, 119135, 119150, 512, 119135, 119151, 512, - 119135, 119152, 512, 119135, 119153, 512, 119135, 119154, 512, 119225, - 119141, 512, 119226, 119141, 512, 119227, 119150, 512, 119228, 119150, - 512, 119227, 119151, 512, 119228, 119151, 262, 65, 262, 71, 262, 74, 262, - 75, 262, 79, 262, 83, 262, 84, 262, 85, 262, 86, 262, 87, 262, 88, 262, - 89, 262, 97, 262, 98, 262, 99, 262, 102, 262, 107, 262, 109, 262, 110, - 262, 112, 262, 113, 262, 114, 262, 115, 262, 116, 262, 117, 262, 118, - 262, 119, 262, 120, 262, 121, 262, 122, 262, 305, 262, 567, 262, 913, - 262, 914, 262, 916, 262, 917, 262, 918, 262, 919, 262, 920, 262, 921, - 262, 922, 262, 923, 262, 924, 262, 925, 262, 926, 262, 927, 262, 929, - 262, 1012, 262, 931, 262, 932, 262, 933, 262, 934, 262, 935, 262, 936, - 262, 937, 262, 8711, 262, 945, 262, 946, 262, 948, 262, 949, 262, 950, - 262, 951, 262, 952, 262, 953, 262, 954, 262, 955, 262, 956, 262, 957, - 262, 958, 262, 959, 262, 961, 262, 962, 262, 963, 262, 964, 262, 965, - 262, 966, 262, 967, 262, 968, 262, 969, 262, 8706, 262, 1013, 262, 977, - 262, 1008, 262, 981, 262, 1009, 262, 982, 262, 988, 262, 989, 262, 48, - 262, 49, 262, 50, 262, 51, 262, 52, 262, 53, 262, 54, 262, 55, 262, 56, - 262, 57, 259, 1072, 259, 1073, 259, 1074, 259, 1075, 259, 1076, 259, - 1077, 259, 1078, 259, 1079, 259, 1080, 259, 1082, 259, 1083, 259, 1084, - 259, 1086, 259, 1087, 259, 1088, 259, 1089, 259, 1090, 259, 1091, 259, - 1092, 259, 1093, 259, 1094, 259, 1095, 259, 1096, 259, 1099, 259, 1101, - 259, 1102, 259, 42633, 259, 1241, 259, 1110, 259, 1112, 259, 1257, 259, - 1199, 259, 1231, 261, 1072, 261, 1073, 261, 1074, 261, 1075, 261, 1076, - 261, 1077, 261, 1078, 261, 1079, 261, 1080, 261, 1082, 261, 1083, 261, - 1086, 261, 1087, 261, 1089, 261, 1091, 261, 1092, 261, 1093, 261, 1094, - 261, 1095, 261, 1096, 261, 1098, 261, 1099, 261, 1169, 261, 1110, 261, - 1109, 261, 1119, 259, 1195, 259, 42577, 259, 1201, 262, 1575, 262, 1576, - 262, 1580, 262, 1583, 262, 1608, 262, 1586, 262, 1581, 262, 1591, 262, - 1610, 262, 1603, 262, 1604, 262, 1605, 262, 1606, 262, 1587, 262, 1593, - 262, 1601, 262, 1589, 262, 1602, 262, 1585, 262, 1588, 262, 1578, 262, - 1579, 262, 1582, 262, 1584, 262, 1590, 262, 1592, 262, 1594, 262, 1646, - 262, 1722, 262, 1697, 262, 1647, 262, 1607, 514, 48, 46, 514, 48, 44, - 514, 49, 44, 514, 50, 44, 514, 51, 44, 514, 52, 44, 514, 53, 44, 514, 54, - 44, 514, 55, 44, 514, 56, 44, 514, 57, 44, 770, 40, 65, 41, 770, 40, 66, - 41, 770, 40, 67, 41, 770, 40, 68, 41, 770, 40, 69, 41, 770, 40, 70, 41, - 770, 40, 71, 41, 770, 40, 72, 41, 770, 40, 73, 41, 770, 40, 74, 41, 770, - 40, 75, 41, 770, 40, 76, 41, 770, 40, 77, 41, 770, 40, 78, 41, 770, 40, - 79, 41, 770, 40, 80, 41, 770, 40, 81, 41, 770, 40, 82, 41, 770, 40, 83, - 41, 770, 40, 84, 41, 770, 40, 85, 41, 770, 40, 86, 41, 770, 40, 87, 41, - 770, 40, 88, 41, 770, 40, 89, 41, 770, 40, 90, 41, 770, 12308, 83, 12309, - 519, 67, 68, 519, 87, 90, 266, 65, 266, 66, 266, 67, 266, 68, 266, 69, - 266, 70, 266, 71, 266, 72, 266, 73, 266, 74, 266, 75, 266, 76, 266, 77, - 266, 78, 266, 79, 266, 80, 266, 81, 266, 82, 266, 83, 266, 84, 266, 85, - 266, 86, 266, 87, 266, 88, 266, 89, 266, 90, 522, 72, 86, 522, 83, 68, - 522, 83, 83, 778, 80, 80, 86, 522, 87, 67, 515, 77, 67, 515, 77, 68, 515, - 77, 82, 522, 68, 74, 522, 12411, 12363, 522, 12467, 12467, 266, 12469, - 266, 25163, 266, 23383, 266, 21452, 266, 12487, 266, 20108, 266, 22810, - 266, 35299, 266, 22825, 266, 20132, 266, 26144, 266, 28961, 266, 26009, - 266, 21069, 266, 24460, 266, 20877, 266, 26032, 266, 21021, 266, 32066, - 266, 29983, 266, 36009, 266, 22768, 266, 21561, 266, 28436, 266, 25237, - 266, 25429, 266, 19968, 266, 19977, 266, 36938, 266, 24038, 266, 20013, - 266, 21491, 266, 25351, 266, 36208, 266, 25171, 266, 31105, 266, 31354, - 266, 21512, 266, 28288, 266, 26377, 266, 26376, 266, 30003, 266, 21106, - 266, 21942, 266, 37197, 770, 12308, 26412, 12309, 770, 12308, 19977, - 12309, 770, 12308, 20108, 12309, 770, 12308, 23433, 12309, 770, 12308, - 28857, 12309, 770, 12308, 25171, 12309, 770, 12308, 30423, 12309, 770, - 12308, 21213, 12309, 770, 12308, 25943, 12309, 263, 24471, 263, 21487, - 256, 20029, 256, 20024, 256, 20033, 256, 131362, 256, 20320, 256, 20411, - 256, 20482, 256, 20602, 256, 20633, 256, 20687, 256, 13470, 256, 132666, - 256, 20820, 256, 20836, 256, 20855, 256, 132380, 256, 13497, 256, 20839, - 256, 20877, 256, 132427, 256, 20887, 256, 20900, 256, 20172, 256, 20908, - 256, 168415, 256, 20981, 256, 20995, 256, 13535, 256, 21051, 256, 21062, - 256, 21106, 256, 21111, 256, 13589, 256, 21253, 256, 21254, 256, 21321, - 256, 21338, 256, 21363, 256, 21373, 256, 21375, 256, 133676, 256, 28784, - 256, 21450, 256, 21471, 256, 133987, 256, 21483, 256, 21489, 256, 21510, - 256, 21662, 256, 21560, 256, 21576, 256, 21608, 256, 21666, 256, 21750, - 256, 21776, 256, 21843, 256, 21859, 256, 21892, 256, 21931, 256, 21939, - 256, 21954, 256, 22294, 256, 22295, 256, 22097, 256, 22132, 256, 22766, - 256, 22478, 256, 22516, 256, 22541, 256, 22411, 256, 22578, 256, 22577, - 256, 22700, 256, 136420, 256, 22770, 256, 22775, 256, 22790, 256, 22810, - 256, 22818, 256, 22882, 256, 136872, 256, 136938, 256, 23020, 256, 23067, - 256, 23079, 256, 23000, 256, 23142, 256, 14062, 256, 14076, 256, 23304, - 256, 23358, 256, 137672, 256, 23491, 256, 23512, 256, 23539, 256, 138008, - 256, 23551, 256, 23558, 256, 24403, 256, 23586, 256, 14209, 256, 23648, - 256, 23744, 256, 23693, 256, 138724, 256, 23875, 256, 138726, 256, 23918, - 256, 23915, 256, 23932, 256, 24033, 256, 24034, 256, 14383, 256, 24061, - 256, 24104, 256, 24125, 256, 24169, 256, 14434, 256, 139651, 256, 14460, - 256, 24240, 256, 24243, 256, 24246, 256, 172946, 256, 24318, 256, 140081, - 256, 33281, 256, 24354, 256, 14535, 256, 144056, 256, 156122, 256, 24418, - 256, 24427, 256, 14563, 256, 24474, 256, 24525, 256, 24535, 256, 24569, - 256, 24705, 256, 14650, 256, 14620, 256, 141012, 256, 24775, 256, 24904, - 256, 24908, 256, 24954, 256, 25010, 256, 24996, 256, 25007, 256, 25054, - 256, 25104, 256, 25115, 256, 25181, 256, 25265, 256, 25300, 256, 25424, - 256, 142092, 256, 25405, 256, 25340, 256, 25448, 256, 25475, 256, 25572, - 256, 142321, 256, 25634, 256, 25541, 256, 25513, 256, 14894, 256, 25705, - 256, 25726, 256, 25757, 256, 25719, 256, 14956, 256, 25964, 256, 143370, - 256, 26083, 256, 26360, 256, 26185, 256, 15129, 256, 15112, 256, 15076, - 256, 20882, 256, 20885, 256, 26368, 256, 26268, 256, 32941, 256, 17369, - 256, 26401, 256, 26462, 256, 26451, 256, 144323, 256, 15177, 256, 26618, - 256, 26501, 256, 26706, 256, 144493, 256, 26766, 256, 26655, 256, 26900, - 256, 26946, 256, 27043, 256, 27114, 256, 27304, 256, 145059, 256, 27355, - 256, 15384, 256, 27425, 256, 145575, 256, 27476, 256, 15438, 256, 27506, - 256, 27551, 256, 27579, 256, 146061, 256, 138507, 256, 146170, 256, - 27726, 256, 146620, 256, 27839, 256, 27853, 256, 27751, 256, 27926, 256, - 27966, 256, 28009, 256, 28024, 256, 28037, 256, 146718, 256, 27956, 256, - 28207, 256, 28270, 256, 15667, 256, 28359, 256, 147153, 256, 28153, 256, - 28526, 256, 147294, 256, 147342, 256, 28614, 256, 28729, 256, 28699, 256, - 15766, 256, 28746, 256, 28797, 256, 28791, 256, 28845, 256, 132389, 256, - 28997, 256, 148067, 256, 29084, 256, 148395, 256, 29224, 256, 29264, 256, - 149000, 256, 29312, 256, 29333, 256, 149301, 256, 149524, 256, 29562, - 256, 29579, 256, 16044, 256, 29605, 256, 16056, 256, 29767, 256, 29788, - 256, 29829, 256, 29898, 256, 16155, 256, 29988, 256, 150582, 256, 30014, - 256, 150674, 256, 139679, 256, 30224, 256, 151457, 256, 151480, 256, - 151620, 256, 16380, 256, 16392, 256, 151795, 256, 151794, 256, 151833, - 256, 151859, 256, 30494, 256, 30495, 256, 30603, 256, 16454, 256, 16534, - 256, 152605, 256, 30798, 256, 16611, 256, 153126, 256, 153242, 256, - 153285, 256, 31211, 256, 16687, 256, 31306, 256, 31311, 256, 153980, 256, - 154279, 256, 31470, 256, 16898, 256, 154539, 256, 31686, 256, 31689, 256, - 16935, 256, 154752, 256, 31954, 256, 17056, 256, 31976, 256, 31971, 256, - 32000, 256, 155526, 256, 32099, 256, 17153, 256, 32199, 256, 32258, 256, - 32325, 256, 17204, 256, 156200, 256, 156231, 256, 17241, 256, 156377, - 256, 32634, 256, 156478, 256, 32661, 256, 32762, 256, 156890, 256, - 156963, 256, 32864, 256, 157096, 256, 32880, 256, 144223, 256, 17365, - 256, 32946, 256, 33027, 256, 17419, 256, 33086, 256, 23221, 256, 157607, - 256, 157621, 256, 144275, 256, 144284, 256, 33284, 256, 36766, 256, - 17515, 256, 33425, 256, 33419, 256, 33437, 256, 21171, 256, 33457, 256, - 33459, 256, 33469, 256, 33510, 256, 158524, 256, 33565, 256, 33635, 256, - 33709, 256, 33571, 256, 33725, 256, 33767, 256, 33619, 256, 33738, 256, - 33740, 256, 33756, 256, 158774, 256, 159083, 256, 158933, 256, 17707, - 256, 34033, 256, 34035, 256, 34070, 256, 160714, 256, 34148, 256, 159532, - 256, 17757, 256, 17761, 256, 159665, 256, 159954, 256, 17771, 256, 34384, - 256, 34407, 256, 34409, 256, 34473, 256, 34440, 256, 34574, 256, 34530, - 256, 34600, 256, 34667, 256, 34694, 256, 17879, 256, 34785, 256, 34817, - 256, 17913, 256, 34912, 256, 34915, 256, 161383, 256, 35031, 256, 35038, - 256, 17973, 256, 35066, 256, 13499, 256, 161966, 256, 162150, 256, 18110, - 256, 18119, 256, 35488, 256, 35925, 256, 162984, 256, 36011, 256, 36033, - 256, 36123, 256, 36215, 256, 163631, 256, 133124, 256, 36299, 256, 36284, - 256, 36336, 256, 133342, 256, 36564, 256, 165330, 256, 165357, 256, - 37012, 256, 37105, 256, 37137, 256, 165678, 256, 37147, 256, 37432, 256, - 37591, 256, 37592, 256, 37500, 256, 37881, 256, 37909, 256, 166906, 256, - 38283, 256, 18837, 256, 38327, 256, 167287, 256, 18918, 256, 38595, 256, - 23986, 256, 38691, 256, 168261, 256, 168474, 256, 19054, 256, 19062, 256, - 38880, 256, 168970, 256, 19122, 256, 169110, 256, 38953, 256, 169398, - 256, 39138, 256, 19251, 256, 39209, 256, 39335, 256, 39362, 256, 39422, - 256, 19406, 256, 170800, 256, 40000, 256, 40189, 256, 19662, 256, 19693, - 256, 40295, 256, 172238, 256, 19704, 256, 172293, 256, 172558, 256, - 172689, 256, 40635, 256, 19798, 256, 40697, 256, 40702, 256, 40709, 256, - 40719, 256, 40726, 256, 40763, 256, 173568, + 512, 67026, 775, 512, 67034, 775, 259, 720, 259, 721, 259, 230, 259, 665, + 259, 595, 259, 675, 259, 43878, 259, 677, 259, 676, 259, 598, 259, 599, + 259, 7569, 259, 600, 259, 606, 259, 681, 259, 612, 259, 610, 259, 608, + 259, 667, 259, 295, 259, 668, 259, 615, 259, 644, 259, 682, 259, 683, + 259, 620, 259, 122628, 259, 42894, 259, 622, 259, 122629, 259, 654, 259, + 122630, 259, 248, 259, 630, 259, 631, 259, 113, 259, 634, 259, 122632, + 259, 637, 259, 638, 259, 640, 259, 680, 259, 678, 259, 43879, 259, 679, + 259, 648, 259, 11377, 259, 655, 259, 673, 259, 674, 259, 664, 259, 448, + 259, 449, 259, 450, 259, 122634, 259, 122654, 512, 69785, 69818, 512, + 69787, 69818, 512, 69797, 69818, 512, 69937, 69927, 512, 69938, 69927, + 512, 70471, 70462, 512, 70471, 70487, 512, 70530, 70601, 512, 70532, + 70587, 512, 70539, 70594, 512, 70544, 70601, 512, 70594, 70594, 512, + 70594, 70584, 512, 70594, 70601, 512, 70841, 70842, 512, 70841, 70832, + 512, 70841, 70845, 512, 71096, 71087, 512, 71097, 71087, 512, 71989, + 71984, 512, 90398, 90398, 512, 90398, 90409, 512, 90398, 90399, 512, + 90409, 90399, 512, 90398, 90400, 512, 90401, 90399, 512, 90402, 90399, + 512, 90401, 90400, 512, 93543, 93543, 512, 93539, 93543, 512, 93545, + 93543, 262, 65, 262, 71, 262, 74, 262, 75, 262, 79, 262, 83, 262, 84, + 262, 85, 262, 86, 262, 87, 262, 88, 262, 89, 262, 48, 262, 49, 262, 50, + 262, 51, 262, 52, 262, 53, 262, 54, 262, 55, 262, 56, 262, 57, 512, + 119127, 119141, 512, 119128, 119141, 512, 119135, 119150, 512, 119135, + 119151, 512, 119135, 119152, 512, 119135, 119153, 512, 119135, 119154, + 512, 119225, 119141, 512, 119226, 119141, 512, 119227, 119150, 512, + 119228, 119150, 512, 119227, 119151, 512, 119228, 119151, 262, 97, 262, + 98, 262, 99, 262, 102, 262, 107, 262, 109, 262, 110, 262, 112, 262, 113, + 262, 114, 262, 115, 262, 116, 262, 117, 262, 118, 262, 119, 262, 120, + 262, 121, 262, 122, 262, 305, 262, 567, 262, 913, 262, 914, 262, 916, + 262, 917, 262, 918, 262, 919, 262, 920, 262, 921, 262, 922, 262, 923, + 262, 924, 262, 925, 262, 926, 262, 927, 262, 929, 262, 1012, 262, 931, + 262, 932, 262, 933, 262, 934, 262, 935, 262, 936, 262, 937, 262, 8711, + 262, 945, 262, 946, 262, 948, 262, 949, 262, 950, 262, 951, 262, 952, + 262, 953, 262, 954, 262, 955, 262, 956, 262, 957, 262, 958, 262, 959, + 262, 961, 262, 962, 262, 963, 262, 964, 262, 965, 262, 966, 262, 967, + 262, 968, 262, 969, 262, 8706, 262, 1013, 262, 977, 262, 1008, 262, 981, + 262, 1009, 262, 982, 262, 988, 262, 989, 259, 1072, 259, 1073, 259, 1074, + 259, 1075, 259, 1076, 259, 1077, 259, 1078, 259, 1079, 259, 1080, 259, + 1082, 259, 1083, 259, 1084, 259, 1086, 259, 1087, 259, 1088, 259, 1089, + 259, 1090, 259, 1091, 259, 1092, 259, 1093, 259, 1094, 259, 1095, 259, + 1096, 259, 1099, 259, 1101, 259, 1102, 259, 42633, 259, 1241, 259, 1110, + 259, 1112, 259, 1257, 259, 1199, 259, 1231, 261, 1072, 261, 1073, 261, + 1074, 261, 1075, 261, 1076, 261, 1077, 261, 1078, 261, 1079, 261, 1080, + 261, 1082, 261, 1083, 261, 1086, 261, 1087, 261, 1089, 261, 1091, 261, + 1092, 261, 1093, 261, 1094, 261, 1095, 261, 1096, 261, 1098, 261, 1099, + 261, 1169, 261, 1110, 261, 1109, 261, 1119, 259, 1195, 259, 42577, 259, + 1201, 262, 1575, 262, 1576, 262, 1580, 262, 1583, 262, 1608, 262, 1586, + 262, 1581, 262, 1591, 262, 1610, 262, 1603, 262, 1604, 262, 1605, 262, + 1606, 262, 1587, 262, 1593, 262, 1601, 262, 1589, 262, 1602, 262, 1585, + 262, 1588, 262, 1578, 262, 1579, 262, 1582, 262, 1584, 262, 1590, 262, + 1592, 262, 1594, 262, 1646, 262, 1722, 262, 1697, 262, 1647, 262, 1607, + 514, 48, 46, 514, 48, 44, 514, 49, 44, 514, 50, 44, 514, 51, 44, 514, 52, + 44, 514, 53, 44, 514, 54, 44, 514, 55, 44, 514, 56, 44, 514, 57, 44, 770, + 40, 65, 41, 770, 40, 66, 41, 770, 40, 67, 41, 770, 40, 68, 41, 770, 40, + 69, 41, 770, 40, 70, 41, 770, 40, 71, 41, 770, 40, 72, 41, 770, 40, 73, + 41, 770, 40, 74, 41, 770, 40, 75, 41, 770, 40, 76, 41, 770, 40, 77, 41, + 770, 40, 78, 41, 770, 40, 79, 41, 770, 40, 80, 41, 770, 40, 81, 41, 770, + 40, 82, 41, 770, 40, 83, 41, 770, 40, 84, 41, 770, 40, 85, 41, 770, 40, + 86, 41, 770, 40, 87, 41, 770, 40, 88, 41, 770, 40, 89, 41, 770, 40, 90, + 41, 770, 12308, 83, 12309, 519, 67, 68, 519, 87, 90, 266, 65, 266, 66, + 266, 67, 266, 68, 266, 69, 266, 70, 266, 71, 266, 72, 266, 73, 266, 74, + 266, 75, 266, 76, 266, 77, 266, 78, 266, 79, 266, 80, 266, 81, 266, 82, + 266, 83, 266, 84, 266, 85, 266, 86, 266, 87, 266, 88, 266, 89, 266, 90, + 522, 72, 86, 522, 83, 68, 522, 83, 83, 778, 80, 80, 86, 522, 87, 67, 515, + 77, 67, 515, 77, 68, 515, 77, 82, 522, 68, 74, 522, 12411, 12363, 522, + 12467, 12467, 266, 12469, 266, 25163, 266, 23383, 266, 21452, 266, 12487, + 266, 20108, 266, 22810, 266, 35299, 266, 22825, 266, 20132, 266, 26144, + 266, 28961, 266, 26009, 266, 21069, 266, 24460, 266, 20877, 266, 26032, + 266, 21021, 266, 32066, 266, 29983, 266, 36009, 266, 22768, 266, 21561, + 266, 28436, 266, 25237, 266, 25429, 266, 19968, 266, 19977, 266, 36938, + 266, 24038, 266, 20013, 266, 21491, 266, 25351, 266, 36208, 266, 25171, + 266, 31105, 266, 31354, 266, 21512, 266, 28288, 266, 26377, 266, 26376, + 266, 30003, 266, 21106, 266, 21942, 266, 37197, 770, 12308, 26412, 12309, + 770, 12308, 19977, 12309, 770, 12308, 20108, 12309, 770, 12308, 23433, + 12309, 770, 12308, 28857, 12309, 770, 12308, 25171, 12309, 770, 12308, + 30423, 12309, 770, 12308, 21213, 12309, 770, 12308, 25943, 12309, 263, + 24471, 263, 21487, 256, 20029, 256, 20024, 256, 20033, 256, 131362, 256, + 20320, 256, 20411, 256, 20482, 256, 20602, 256, 20633, 256, 20687, 256, + 13470, 256, 132666, 256, 20820, 256, 20836, 256, 20855, 256, 132380, 256, + 13497, 256, 20839, 256, 20877, 256, 132427, 256, 20887, 256, 20900, 256, + 20172, 256, 20908, 256, 168415, 256, 20981, 256, 20995, 256, 13535, 256, + 21051, 256, 21062, 256, 21106, 256, 21111, 256, 13589, 256, 21253, 256, + 21254, 256, 21321, 256, 21338, 256, 21363, 256, 21373, 256, 21375, 256, + 133676, 256, 28784, 256, 21450, 256, 21471, 256, 133987, 256, 21483, 256, + 21489, 256, 21510, 256, 21662, 256, 21560, 256, 21576, 256, 21608, 256, + 21666, 256, 21750, 256, 21776, 256, 21843, 256, 21859, 256, 21892, 256, + 21931, 256, 21939, 256, 21954, 256, 22294, 256, 22295, 256, 22097, 256, + 22132, 256, 22766, 256, 22478, 256, 22516, 256, 22541, 256, 22411, 256, + 22578, 256, 22577, 256, 22700, 256, 136420, 256, 22770, 256, 22775, 256, + 22790, 256, 22810, 256, 22818, 256, 22882, 256, 136872, 256, 136938, 256, + 23020, 256, 23067, 256, 23079, 256, 23000, 256, 23142, 256, 14062, 256, + 14076, 256, 23304, 256, 23358, 256, 137672, 256, 23491, 256, 23512, 256, + 23539, 256, 138008, 256, 23551, 256, 23558, 256, 24403, 256, 23586, 256, + 14209, 256, 23648, 256, 23744, 256, 23693, 256, 138724, 256, 23875, 256, + 138726, 256, 23918, 256, 23915, 256, 23932, 256, 24033, 256, 24034, 256, + 14383, 256, 24061, 256, 24104, 256, 24125, 256, 24169, 256, 14434, 256, + 139651, 256, 14460, 256, 24240, 256, 24243, 256, 24246, 256, 172946, 256, + 24318, 256, 140081, 256, 33281, 256, 24354, 256, 14535, 256, 144056, 256, + 156122, 256, 24418, 256, 24427, 256, 14563, 256, 24474, 256, 24525, 256, + 24535, 256, 24569, 256, 24705, 256, 14650, 256, 14620, 256, 141012, 256, + 24775, 256, 24904, 256, 24908, 256, 24954, 256, 25010, 256, 24996, 256, + 25007, 256, 25054, 256, 25104, 256, 25115, 256, 25181, 256, 25265, 256, + 25300, 256, 25424, 256, 142092, 256, 25405, 256, 25340, 256, 25448, 256, + 25475, 256, 25572, 256, 142321, 256, 25634, 256, 25541, 256, 25513, 256, + 14894, 256, 25705, 256, 25726, 256, 25757, 256, 25719, 256, 14956, 256, + 25964, 256, 143370, 256, 26083, 256, 26360, 256, 26185, 256, 15129, 256, + 15112, 256, 15076, 256, 20882, 256, 20885, 256, 26368, 256, 26268, 256, + 32941, 256, 17369, 256, 26401, 256, 26462, 256, 26451, 256, 144323, 256, + 15177, 256, 26618, 256, 26501, 256, 26706, 256, 144493, 256, 26766, 256, + 26655, 256, 26900, 256, 26946, 256, 27043, 256, 27114, 256, 27304, 256, + 145059, 256, 27355, 256, 15384, 256, 27425, 256, 145575, 256, 27476, 256, + 15438, 256, 27506, 256, 27551, 256, 27579, 256, 146061, 256, 138507, 256, + 146170, 256, 27726, 256, 146620, 256, 27839, 256, 27853, 256, 27751, 256, + 27926, 256, 27966, 256, 28009, 256, 28024, 256, 28037, 256, 146718, 256, + 27956, 256, 28207, 256, 28270, 256, 15667, 256, 28359, 256, 147153, 256, + 28153, 256, 28526, 256, 147294, 256, 147342, 256, 28614, 256, 28729, 256, + 28699, 256, 15766, 256, 28746, 256, 28797, 256, 28791, 256, 28845, 256, + 132389, 256, 28997, 256, 148067, 256, 29084, 256, 148395, 256, 29224, + 256, 29264, 256, 149000, 256, 29312, 256, 29333, 256, 149301, 256, + 149524, 256, 29562, 256, 29579, 256, 16044, 256, 29605, 256, 16056, 256, + 29767, 256, 29788, 256, 29829, 256, 29898, 256, 16155, 256, 29988, 256, + 150582, 256, 30014, 256, 150674, 256, 139679, 256, 30224, 256, 151457, + 256, 151480, 256, 151620, 256, 16380, 256, 16392, 256, 151795, 256, + 151794, 256, 151833, 256, 151859, 256, 30494, 256, 30495, 256, 30603, + 256, 16454, 256, 16534, 256, 152605, 256, 30798, 256, 16611, 256, 153126, + 256, 153242, 256, 153285, 256, 31211, 256, 16687, 256, 31306, 256, 31311, + 256, 153980, 256, 154279, 256, 31470, 256, 16898, 256, 154539, 256, + 31686, 256, 31689, 256, 16935, 256, 154752, 256, 31954, 256, 17056, 256, + 31976, 256, 31971, 256, 32000, 256, 155526, 256, 32099, 256, 17153, 256, + 32199, 256, 32258, 256, 32325, 256, 17204, 256, 156200, 256, 156231, 256, + 17241, 256, 156377, 256, 32634, 256, 156478, 256, 32661, 256, 32762, 256, + 156890, 256, 156963, 256, 32864, 256, 157096, 256, 32880, 256, 144223, + 256, 17365, 256, 32946, 256, 33027, 256, 17419, 256, 33086, 256, 23221, + 256, 157607, 256, 157621, 256, 144275, 256, 144284, 256, 33284, 256, + 36766, 256, 17515, 256, 33425, 256, 33419, 256, 33437, 256, 21171, 256, + 33457, 256, 33459, 256, 33469, 256, 33510, 256, 158524, 256, 33565, 256, + 33635, 256, 33709, 256, 33571, 256, 33725, 256, 33767, 256, 33619, 256, + 33738, 256, 33740, 256, 33756, 256, 158774, 256, 159083, 256, 158933, + 256, 17707, 256, 34033, 256, 34035, 256, 34070, 256, 160714, 256, 34148, + 256, 159532, 256, 17757, 256, 17761, 256, 159665, 256, 159954, 256, + 17771, 256, 34384, 256, 34407, 256, 34409, 256, 34473, 256, 34440, 256, + 34574, 256, 34530, 256, 34600, 256, 34667, 256, 34694, 256, 17879, 256, + 34785, 256, 34817, 256, 17913, 256, 34912, 256, 34915, 256, 161383, 256, + 35031, 256, 35038, 256, 17973, 256, 35066, 256, 13499, 256, 161966, 256, + 162150, 256, 18110, 256, 18119, 256, 35488, 256, 35925, 256, 162984, 256, + 36011, 256, 36033, 256, 36123, 256, 36215, 256, 163631, 256, 133124, 256, + 36299, 256, 36284, 256, 36336, 256, 133342, 256, 36564, 256, 165330, 256, + 165357, 256, 37012, 256, 37105, 256, 37137, 256, 165678, 256, 37147, 256, + 37432, 256, 37591, 256, 37592, 256, 37500, 256, 37881, 256, 37909, 256, + 166906, 256, 38283, 256, 18837, 256, 38327, 256, 167287, 256, 18918, 256, + 38595, 256, 23986, 256, 38691, 256, 168261, 256, 168474, 256, 19054, 256, + 19062, 256, 38880, 256, 168970, 256, 19122, 256, 169110, 256, 38953, 256, + 169398, 256, 39138, 256, 19251, 256, 39209, 256, 39335, 256, 39362, 256, + 39422, 256, 19406, 256, 170800, 256, 40000, 256, 40189, 256, 19662, 256, + 19693, 256, 40295, 256, 172238, 256, 19704, 256, 172293, 256, 172558, + 256, 172689, 256, 40635, 256, 19798, 256, 40697, 256, 40702, 256, 40709, + 256, 40719, 256, 40726, 256, 40763, 256, 173568, }; /* index tables for the decomposition data */ @@ -4380,15 +4467,16 @@ static const unsigned char decomp_index1[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, - 74, 75, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 79, 0, 0, 0, 80, 0, 0, 81, 0, - 82, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 74, 75, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 78, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 80, 0, 0, 0, 81, 82, 0, 83, 0, + 84, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -4396,12 +4484,12 @@ static const unsigned char decomp_index1[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 89, 90, 0, 0, 0, 0, 91, 92, + 93, 94, 95, 96, 97, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 100, 101, 0, 0, 0, 0, 102, 103, 104, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 85, 0, 0, 0, 0, 86, 87, - 88, 89, 90, 91, 92, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 95, 96, 0, 0, 0, 0, 97, 98, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -4421,8 +4509,8 @@ static const unsigned char decomp_index1[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 107, 108, 109, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 102, 103, 104, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -4720,8 +4808,7 @@ static const unsigned char decomp_index1[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; static const unsigned short decomp_index2[] = { @@ -5295,690 +5382,763 @@ static const unsigned short decomp_index2[] = { 10263, 10265, 10267, 10269, 10271, 0, 0, 10273, 10275, 10277, 10279, 10281, 10283, 0, 0, 10285, 10287, 10289, 0, 0, 0, 10291, 10293, 10295, 10297, 10299, 10301, 10303, 0, 10305, 10307, 10309, 10311, 10313, 10315, - 10317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10319, - 10321, 10323, 10325, 10327, 0, 10329, 10331, 10333, 10335, 10337, 10339, - 10341, 10343, 10345, 10347, 10349, 10351, 10353, 10355, 10357, 10359, - 10361, 10363, 10365, 10367, 10369, 10371, 10373, 10375, 10377, 10379, - 10381, 10383, 10385, 10387, 10389, 10391, 10393, 10395, 10397, 10399, - 10401, 10403, 10405, 10407, 10409, 10411, 0, 10413, 10415, 10417, 10419, - 10421, 10423, 10425, 10427, 10429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 10317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10431, 0, 10434, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 10437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 10322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 10325, 10327, 10329, 10331, 10333, 0, 10335, 10337, + 10339, 10341, 10343, 10345, 10347, 10349, 10351, 10353, 10355, 10357, + 10359, 10361, 10363, 10365, 10367, 10369, 10371, 10373, 10375, 10377, + 10379, 10381, 10383, 10385, 10387, 10389, 10391, 10393, 10395, 10397, + 10399, 10401, 10403, 10405, 10407, 10409, 10411, 10413, 10415, 10417, 0, + 10419, 10421, 10423, 10425, 10427, 10429, 10431, 10433, 10435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10437, 0, + 10440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10440, 10443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 10446, 10449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 10446, 10449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10452, 10455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 10452, 10455, 0, 10458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 10458, 0, 10461, 0, 0, 0, 0, 0, 0, 0, 0, 10464, 0, 0, + 10467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 10470, 0, 10473, 10476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10461, 10464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 10479, 10482, 0, 10485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10488, 10491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10470, 10473, - 10476, 10479, 10482, 10485, 10488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 10494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 10491, 10494, 10497, 10500, 10503, 10506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10497, 10500, 10503, 10506, 10509, + 10512, 10515, 10518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 10509, 3253, 3192, 3289, 3257, 3259, 10511, 3212, 3218, - 10513, 10515, 3220, 3261, 3224, 10517, 3229, 3231, 3233, 10519, 10521, - 10523, 10525, 10527, 10529, 10531, 3245, 10533, 10535, 10537, 3291, 3255, - 10539, 3210, 3214, 3273, 3293, 10541, 3222, 10543, 10545, 3263, 10547, - 10549, 10551, 10553, 10555, 10557, 10559, 10561, 10563, 10565, 10567, - 10509, 3253, 3192, 3289, 3257, 3259, 10511, 3212, 3218, 10513, 10515, - 3220, 3261, 3224, 10517, 3229, 3231, 3233, 10519, 10521, 10523, 10525, - 10527, 10529, 10531, 3245, 10533, 10535, 10537, 3291, 3255, 10539, 3210, - 0, 3273, 3293, 10541, 3222, 10543, 10545, 3263, 10547, 10549, 10551, - 10553, 10555, 10557, 10559, 10561, 10563, 10565, 10567, 10509, 3253, - 3192, 3289, 3257, 3259, 10511, 3212, 3218, 10513, 10515, 3220, 3261, - 3224, 10517, 3229, 3231, 3233, 10519, 10521, 10523, 10525, 10527, 10529, - 10531, 3245, 10533, 10535, 10537, 3291, 3255, 10539, 3210, 3214, 3273, - 3293, 10541, 3222, 10543, 10545, 3263, 10547, 10549, 10551, 10553, 10555, - 10557, 10559, 10561, 10563, 10565, 10567, 10509, 0, 3192, 3289, 0, 0, - 10511, 0, 0, 10513, 10515, 0, 0, 3224, 10517, 3229, 3231, 0, 10519, - 10521, 10523, 10525, 10527, 10529, 10531, 3245, 10533, 10535, 10537, - 3291, 0, 10539, 0, 3214, 3273, 3293, 10541, 3222, 10543, 10545, 0, 10547, - 10549, 10551, 10553, 10555, 10557, 10559, 10561, 10563, 10565, 10567, - 10509, 3253, 3192, 3289, 3257, 3259, 10511, 3212, 3218, 10513, 10515, - 3220, 3261, 3224, 10517, 3229, 3231, 3233, 10519, 10521, 10523, 10525, - 10527, 10529, 10531, 3245, 10533, 10535, 10537, 3291, 3255, 10539, 3210, - 3214, 3273, 3293, 10541, 3222, 10543, 10545, 3263, 10547, 10549, 10551, - 10553, 10555, 10557, 10559, 10561, 10563, 10565, 10567, 10509, 3253, 0, - 3289, 3257, 3259, 10511, 0, 0, 10513, 10515, 3220, 3261, 3224, 10517, - 3229, 3231, 0, 10519, 10521, 10523, 10525, 10527, 10529, 10531, 0, 10533, - 10535, 10537, 3291, 3255, 10539, 3210, 3214, 3273, 3293, 10541, 3222, - 10543, 10545, 3263, 10547, 10549, 10551, 10553, 10555, 10557, 10559, - 10561, 10563, 10565, 10567, 10509, 3253, 0, 3289, 3257, 3259, 10511, 0, - 3218, 10513, 10515, 3220, 3261, 0, 10517, 0, 0, 0, 10519, 10521, 10523, - 10525, 10527, 10529, 10531, 0, 10533, 10535, 10537, 3291, 3255, 10539, - 3210, 3214, 3273, 3293, 10541, 3222, 10543, 10545, 3263, 10547, 10549, - 10551, 10553, 10555, 10557, 10559, 10561, 10563, 10565, 10567, 10509, - 3253, 3192, 3289, 3257, 3259, 10511, 3212, 3218, 10513, 10515, 3220, - 3261, 3224, 10517, 3229, 3231, 3233, 10519, 10521, 10523, 10525, 10527, - 10529, 10531, 3245, 10533, 10535, 10537, 3291, 3255, 10539, 3210, 3214, - 3273, 3293, 10541, 3222, 10543, 10545, 3263, 10547, 10549, 10551, 10553, - 10555, 10557, 10559, 10561, 10563, 10565, 10567, 10509, 3253, 3192, 3289, - 3257, 3259, 10511, 3212, 3218, 10513, 10515, 3220, 3261, 3224, 10517, - 3229, 3231, 3233, 10519, 10521, 10523, 10525, 10527, 10529, 10531, 3245, - 10533, 10535, 10537, 3291, 3255, 10539, 3210, 3214, 3273, 3293, 10541, - 3222, 10543, 10545, 3263, 10547, 10549, 10551, 10553, 10555, 10557, - 10559, 10561, 10563, 10565, 10567, 10509, 3253, 3192, 3289, 3257, 3259, - 10511, 3212, 3218, 10513, 10515, 3220, 3261, 3224, 10517, 3229, 3231, - 3233, 10519, 10521, 10523, 10525, 10527, 10529, 10531, 3245, 10533, - 10535, 10537, 3291, 3255, 10539, 3210, 3214, 3273, 3293, 10541, 3222, - 10543, 10545, 3263, 10547, 10549, 10551, 10553, 10555, 10557, 10559, - 10561, 10563, 10565, 10567, 10509, 3253, 3192, 3289, 3257, 3259, 10511, - 3212, 3218, 10513, 10515, 3220, 3261, 3224, 10517, 3229, 3231, 3233, - 10519, 10521, 10523, 10525, 10527, 10529, 10531, 3245, 10533, 10535, - 10537, 3291, 3255, 10539, 3210, 3214, 3273, 3293, 10541, 3222, 10543, - 10545, 3263, 10547, 10549, 10551, 10553, 10555, 10557, 10559, 10561, - 10563, 10565, 10567, 10509, 3253, 3192, 3289, 3257, 3259, 10511, 3212, - 3218, 10513, 10515, 3220, 3261, 3224, 10517, 3229, 3231, 3233, 10519, - 10521, 10523, 10525, 10527, 10529, 10531, 3245, 10533, 10535, 10537, - 3291, 3255, 10539, 3210, 3214, 3273, 3293, 10541, 3222, 10543, 10545, - 3263, 10547, 10549, 10551, 10553, 10555, 10557, 10559, 10561, 10563, - 10565, 10567, 10509, 3253, 3192, 3289, 3257, 3259, 10511, 3212, 3218, - 10513, 10515, 3220, 3261, 3224, 10517, 3229, 3231, 3233, 10519, 10521, - 10523, 10525, 10527, 10529, 10531, 3245, 10533, 10535, 10537, 3291, 3255, - 10539, 3210, 3214, 3273, 3293, 10541, 3222, 10543, 10545, 3263, 10547, - 10549, 10551, 10553, 10555, 10557, 10559, 10561, 10563, 10565, 10567, - 10569, 10571, 0, 0, 10573, 10575, 3283, 10577, 10579, 10581, 10583, - 10585, 10587, 10589, 10591, 10593, 10595, 10597, 10599, 3285, 10601, - 10603, 10605, 10607, 10609, 10611, 10613, 10615, 10617, 10619, 10621, - 10623, 3281, 10625, 10627, 10629, 10631, 10633, 10635, 10637, 10639, - 10641, 10643, 10645, 10647, 3279, 10649, 10651, 10653, 10655, 10657, - 10659, 10661, 10663, 10665, 10667, 10669, 10671, 10673, 10675, 10677, - 10679, 10573, 10575, 3283, 10577, 10579, 10581, 10583, 10585, 10587, - 10589, 10591, 10593, 10595, 10597, 10599, 3285, 10601, 10603, 10605, - 10607, 10609, 10611, 10613, 10615, 10617, 10619, 10621, 10623, 3281, - 10625, 10627, 10629, 10631, 10633, 10635, 10637, 10639, 10641, 10643, - 10645, 10647, 3279, 10649, 10651, 10653, 10655, 10657, 10659, 10661, - 10663, 10665, 10667, 10669, 10671, 10673, 10675, 10677, 10679, 10573, - 10575, 3283, 10577, 10579, 10581, 10583, 10585, 10587, 10589, 10591, - 10593, 10595, 10597, 10599, 3285, 10601, 10603, 10605, 10607, 10609, - 10611, 10613, 10615, 10617, 10619, 10621, 10623, 3281, 10625, 10627, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 10521, 10524, 10527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 10530, 3253, 3192, 3289, 3257, 3259, 10532, 3212, 3218, 10534, 10536, + 3220, 3261, 3224, 10538, 3229, 3231, 3233, 10540, 10542, 10544, 10546, + 10548, 10550, 10552, 3245, 10554, 10556, 10558, 10560, 10562, 10564, + 10566, 10568, 10570, 10572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10574, 10577, 10580, 10583, 10586, + 10589, 10592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10595, 10598, + 10601, 10604, 10607, 10610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 10530, 3253, 3192, 3289, 3257, 3259, 10532, 3212, 3218, 10534, 10536, + 3220, 3261, 3224, 10538, 3229, 3231, 3233, 10540, 10542, 10544, 10546, + 10548, 10550, 10552, 3245, 10613, 10615, 10617, 3291, 3255, 10619, 3210, + 3214, 3273, 3293, 10621, 3222, 10623, 10625, 3263, 10627, 10629, 10631, + 10633, 10635, 10637, 10639, 10641, 10643, 10645, 10647, 10530, 3253, + 3192, 3289, 3257, 3259, 10532, 3212, 3218, 10534, 10536, 3220, 3261, + 3224, 10538, 3229, 3231, 3233, 10540, 10542, 10544, 10546, 10548, 10550, + 10552, 3245, 10613, 10615, 10617, 3291, 3255, 10619, 3210, 0, 3273, 3293, + 10621, 3222, 10623, 10625, 3263, 10627, 10629, 10631, 10633, 10635, + 10637, 10639, 10641, 10643, 10645, 10647, 10530, 3253, 3192, 3289, 3257, + 3259, 10532, 3212, 3218, 10534, 10536, 3220, 3261, 3224, 10538, 3229, + 3231, 3233, 10540, 10542, 10544, 10546, 10548, 10550, 10552, 3245, 10613, + 10615, 10617, 3291, 3255, 10619, 3210, 3214, 3273, 3293, 10621, 3222, + 10623, 10625, 3263, 10627, 10629, 10631, 10633, 10635, 10637, 10639, + 10641, 10643, 10645, 10647, 10530, 0, 3192, 3289, 0, 0, 10532, 0, 0, + 10534, 10536, 0, 0, 3224, 10538, 3229, 3231, 0, 10540, 10542, 10544, + 10546, 10548, 10550, 10552, 3245, 10613, 10615, 10617, 3291, 0, 10619, 0, + 3214, 3273, 3293, 10621, 3222, 10623, 10625, 0, 10627, 10629, 10631, + 10633, 10635, 10637, 10639, 10641, 10643, 10645, 10647, 10530, 3253, + 3192, 3289, 3257, 3259, 10532, 3212, 3218, 10534, 10536, 3220, 3261, + 3224, 10538, 3229, 3231, 3233, 10540, 10542, 10544, 10546, 10548, 10550, + 10552, 3245, 10613, 10615, 10617, 3291, 3255, 10619, 3210, 3214, 3273, + 3293, 10621, 3222, 10623, 10625, 3263, 10627, 10629, 10631, 10633, 10635, + 10637, 10639, 10641, 10643, 10645, 10647, 10530, 3253, 0, 3289, 3257, + 3259, 10532, 0, 0, 10534, 10536, 3220, 3261, 3224, 10538, 3229, 3231, 0, + 10540, 10542, 10544, 10546, 10548, 10550, 10552, 0, 10613, 10615, 10617, + 3291, 3255, 10619, 3210, 3214, 3273, 3293, 10621, 3222, 10623, 10625, + 3263, 10627, 10629, 10631, 10633, 10635, 10637, 10639, 10641, 10643, + 10645, 10647, 10530, 3253, 0, 3289, 3257, 3259, 10532, 0, 3218, 10534, + 10536, 3220, 3261, 0, 10538, 0, 0, 0, 10540, 10542, 10544, 10546, 10548, + 10550, 10552, 0, 10613, 10615, 10617, 3291, 3255, 10619, 3210, 3214, + 3273, 3293, 10621, 3222, 10623, 10625, 3263, 10627, 10629, 10631, 10633, + 10635, 10637, 10639, 10641, 10643, 10645, 10647, 10530, 3253, 3192, 3289, + 3257, 3259, 10532, 3212, 3218, 10534, 10536, 3220, 3261, 3224, 10538, + 3229, 3231, 3233, 10540, 10542, 10544, 10546, 10548, 10550, 10552, 3245, + 10613, 10615, 10617, 3291, 3255, 10619, 3210, 3214, 3273, 3293, 10621, + 3222, 10623, 10625, 3263, 10627, 10629, 10631, 10633, 10635, 10637, + 10639, 10641, 10643, 10645, 10647, 10530, 3253, 3192, 3289, 3257, 3259, + 10532, 3212, 3218, 10534, 10536, 3220, 3261, 3224, 10538, 3229, 3231, + 3233, 10540, 10542, 10544, 10546, 10548, 10550, 10552, 3245, 10613, + 10615, 10617, 3291, 3255, 10619, 3210, 3214, 3273, 3293, 10621, 3222, + 10623, 10625, 3263, 10627, 10629, 10631, 10633, 10635, 10637, 10639, + 10641, 10643, 10645, 10647, 10530, 3253, 3192, 3289, 3257, 3259, 10532, + 3212, 3218, 10534, 10536, 3220, 3261, 3224, 10538, 3229, 3231, 3233, + 10540, 10542, 10544, 10546, 10548, 10550, 10552, 3245, 10613, 10615, + 10617, 3291, 3255, 10619, 3210, 3214, 3273, 3293, 10621, 3222, 10623, + 10625, 3263, 10627, 10629, 10631, 10633, 10635, 10637, 10639, 10641, + 10643, 10645, 10647, 10530, 3253, 3192, 3289, 3257, 3259, 10532, 3212, + 3218, 10534, 10536, 3220, 3261, 3224, 10538, 3229, 3231, 3233, 10540, + 10542, 10544, 10546, 10548, 10550, 10552, 3245, 10613, 10615, 10617, + 3291, 3255, 10619, 3210, 3214, 3273, 3293, 10621, 3222, 10623, 10625, + 3263, 10627, 10629, 10631, 10633, 10635, 10637, 10639, 10641, 10643, + 10645, 10647, 10530, 3253, 3192, 3289, 3257, 3259, 10532, 3212, 3218, + 10534, 10536, 3220, 3261, 3224, 10538, 3229, 3231, 3233, 10540, 10542, + 10544, 10546, 10548, 10550, 10552, 3245, 10613, 10615, 10617, 3291, 3255, + 10619, 3210, 3214, 3273, 3293, 10621, 3222, 10623, 10625, 3263, 10627, 10629, 10631, 10633, 10635, 10637, 10639, 10641, 10643, 10645, 10647, - 3279, 10649, 10651, 10653, 10655, 10657, 10659, 10661, 10663, 10665, - 10667, 10669, 10671, 10673, 10675, 10677, 10679, 10573, 10575, 3283, - 10577, 10579, 10581, 10583, 10585, 10587, 10589, 10591, 10593, 10595, - 10597, 10599, 3285, 10601, 10603, 10605, 10607, 10609, 10611, 10613, - 10615, 10617, 10619, 10621, 10623, 3281, 10625, 10627, 10629, 10631, - 10633, 10635, 10637, 10639, 10641, 10643, 10645, 10647, 3279, 10649, - 10651, 10653, 10655, 10657, 10659, 10661, 10663, 10665, 10667, 10669, - 10671, 10673, 10675, 10677, 10679, 10573, 10575, 3283, 10577, 10579, - 10581, 10583, 10585, 10587, 10589, 10591, 10593, 10595, 10597, 10599, - 3285, 10601, 10603, 10605, 10607, 10609, 10611, 10613, 10615, 10617, - 10619, 10621, 10623, 3281, 10625, 10627, 10629, 10631, 10633, 10635, - 10637, 10639, 10641, 10643, 10645, 10647, 3279, 10649, 10651, 10653, - 10655, 10657, 10659, 10661, 10663, 10665, 10667, 10669, 10671, 10673, - 10675, 10677, 10679, 10681, 10683, 0, 0, 10685, 10687, 10689, 10691, - 10693, 10695, 10697, 10699, 10701, 10703, 10685, 10687, 10689, 10691, - 10693, 10695, 10697, 10699, 10701, 10703, 10685, 10687, 10689, 10691, - 10693, 10695, 10697, 10699, 10701, 10703, 10685, 10687, 10689, 10691, - 10693, 10695, 10697, 10699, 10701, 10703, 10685, 10687, 10689, 10691, - 10693, 10695, 10697, 10699, 10701, 10703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10705, 10707, 10709, 10711, - 10713, 10715, 10717, 10719, 10721, 10723, 10725, 10727, 10729, 10731, + 10530, 3253, 3192, 3289, 3257, 3259, 10532, 3212, 3218, 10534, 10536, + 3220, 3261, 3224, 10538, 3229, 3231, 3233, 10540, 10542, 10544, 10546, + 10548, 10550, 10552, 3245, 10613, 10615, 10617, 3291, 3255, 10619, 3210, + 3214, 3273, 3293, 10621, 3222, 10623, 10625, 3263, 10627, 10629, 10631, + 10633, 10635, 10637, 10639, 10641, 10643, 10645, 10647, 10649, 10651, 0, + 0, 10653, 10655, 3283, 10657, 10659, 10661, 10663, 10665, 10667, 10669, + 10671, 10673, 10675, 10677, 10679, 3285, 10681, 10683, 10685, 10687, + 10689, 10691, 10693, 10695, 10697, 10699, 10701, 10703, 3281, 10705, + 10707, 10709, 10711, 10713, 10715, 10717, 10719, 10721, 10723, 10725, + 10727, 3279, 10729, 10731, 10733, 10735, 10737, 10739, 10741, 10743, + 10745, 10747, 10749, 10751, 10753, 10755, 10757, 10759, 10653, 10655, + 3283, 10657, 10659, 10661, 10663, 10665, 10667, 10669, 10671, 10673, + 10675, 10677, 10679, 3285, 10681, 10683, 10685, 10687, 10689, 10691, + 10693, 10695, 10697, 10699, 10701, 10703, 3281, 10705, 10707, 10709, + 10711, 10713, 10715, 10717, 10719, 10721, 10723, 10725, 10727, 3279, + 10729, 10731, 10733, 10735, 10737, 10739, 10741, 10743, 10745, 10747, + 10749, 10751, 10753, 10755, 10757, 10759, 10653, 10655, 3283, 10657, + 10659, 10661, 10663, 10665, 10667, 10669, 10671, 10673, 10675, 10677, + 10679, 3285, 10681, 10683, 10685, 10687, 10689, 10691, 10693, 10695, + 10697, 10699, 10701, 10703, 3281, 10705, 10707, 10709, 10711, 10713, + 10715, 10717, 10719, 10721, 10723, 10725, 10727, 3279, 10729, 10731, 10733, 10735, 10737, 10739, 10741, 10743, 10745, 10747, 10749, 10751, - 10753, 10755, 10757, 10759, 10761, 10763, 10765, 10767, 10769, 10771, - 10773, 10775, 10777, 10779, 10781, 10783, 10785, 10787, 10789, 10791, - 10793, 10795, 10797, 10799, 10801, 10803, 10805, 10807, 10809, 10811, - 10813, 10815, 10817, 10819, 10821, 10823, 10825, 10827, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10829, 10831, 10833, 10835, 0, 10837, + 10753, 10755, 10757, 10759, 10653, 10655, 3283, 10657, 10659, 10661, + 10663, 10665, 10667, 10669, 10671, 10673, 10675, 10677, 10679, 3285, + 10681, 10683, 10685, 10687, 10689, 10691, 10693, 10695, 10697, 10699, + 10701, 10703, 3281, 10705, 10707, 10709, 10711, 10713, 10715, 10717, + 10719, 10721, 10723, 10725, 10727, 3279, 10729, 10731, 10733, 10735, + 10737, 10739, 10741, 10743, 10745, 10747, 10749, 10751, 10753, 10755, + 10757, 10759, 10653, 10655, 3283, 10657, 10659, 10661, 10663, 10665, + 10667, 10669, 10671, 10673, 10675, 10677, 10679, 3285, 10681, 10683, + 10685, 10687, 10689, 10691, 10693, 10695, 10697, 10699, 10701, 10703, + 3281, 10705, 10707, 10709, 10711, 10713, 10715, 10717, 10719, 10721, + 10723, 10725, 10727, 3279, 10729, 10731, 10733, 10735, 10737, 10739, + 10741, 10743, 10745, 10747, 10749, 10751, 10753, 10755, 10757, 10759, + 10761, 10763, 0, 0, 10554, 10556, 10558, 10560, 10562, 10564, 10566, + 10568, 10570, 10572, 10554, 10556, 10558, 10560, 10562, 10564, 10566, + 10568, 10570, 10572, 10554, 10556, 10558, 10560, 10562, 10564, 10566, + 10568, 10570, 10572, 10554, 10556, 10558, 10560, 10562, 10564, 10566, + 10568, 10570, 10572, 10554, 10556, 10558, 10560, 10562, 10564, 10566, + 10568, 10570, 10572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 10765, 10767, 10769, 10771, 10773, 10775, 10777, + 10779, 10781, 10783, 10785, 10787, 10789, 10791, 10793, 10795, 10797, + 10799, 10801, 10803, 10805, 10807, 10809, 10811, 10813, 10815, 10817, + 10819, 10821, 10823, 10825, 10827, 10829, 10831, 10833, 10835, 10837, 10839, 10841, 10843, 10845, 10847, 10849, 10851, 10853, 10855, 10857, 10859, 10861, 10863, 10865, 10867, 10869, 10871, 10873, 10875, 10877, - 10879, 10881, 10883, 10885, 10887, 10889, 0, 10831, 10833, 0, 10891, 0, - 0, 10841, 0, 10845, 10847, 10849, 10851, 10853, 10855, 10857, 10859, - 10861, 10863, 0, 10867, 10869, 10871, 10873, 0, 10877, 0, 10881, 0, 0, 0, - 0, 0, 0, 10833, 0, 0, 0, 0, 10841, 0, 10845, 0, 10849, 0, 10853, 10855, - 10857, 0, 10861, 10863, 0, 10867, 0, 0, 10873, 0, 10877, 0, 10881, 0, - 10885, 0, 10889, 0, 10831, 10833, 0, 10891, 0, 0, 10841, 10843, 10845, - 10847, 0, 10851, 10853, 10855, 10857, 10859, 10861, 10863, 0, 10867, - 10869, 10871, 10873, 0, 10877, 10879, 10881, 10883, 0, 10887, 0, 10829, - 10831, 10833, 10835, 10891, 10837, 10839, 10841, 10843, 10845, 0, 10849, - 10851, 10853, 10855, 10857, 10859, 10861, 10863, 10865, 10867, 10869, - 10871, 10873, 10875, 10877, 10879, 10881, 0, 0, 0, 0, 0, 10831, 10833, - 10835, 0, 10837, 10839, 10841, 10843, 10845, 0, 10849, 10851, 10853, - 10855, 10857, 10859, 10861, 10863, 10865, 10867, 10869, 10871, 10873, - 10875, 10877, 10879, 10881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 10893, 10896, 10899, 10902, 10905, 10908, 10911, 10914, - 10917, 10920, 10923, 0, 0, 0, 0, 0, 10926, 10930, 10934, 10938, 10942, - 10946, 10950, 10954, 10958, 10962, 10966, 10970, 10974, 10978, 10982, - 10986, 10990, 10994, 10998, 11002, 11006, 11010, 11014, 11018, 11022, - 11026, 11030, 3926, 3956, 11034, 11037, 0, 11040, 11042, 11044, 11046, - 11048, 11050, 11052, 11054, 11056, 11058, 11060, 11062, 11064, 11066, - 11068, 11070, 11072, 11074, 11076, 11078, 11080, 11082, 11084, 11086, - 11088, 11090, 11092, 6348, 11095, 11098, 11101, 11105, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11108, 11111, - 11114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11117, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 11120, 11123, 11126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 11128, 11130, 11132, 11134, 11136, 11138, 11140, 11142, 11144, - 11146, 11148, 11150, 11152, 11154, 11156, 11158, 11160, 11162, 11164, - 11166, 11168, 11170, 11172, 11174, 11176, 11178, 11180, 11182, 11184, - 11186, 11188, 11190, 11192, 11194, 11196, 11198, 11200, 11202, 11204, - 11206, 11208, 11210, 11212, 11214, 0, 0, 0, 0, 11216, 11220, 11224, - 11228, 11232, 11236, 11240, 11244, 11248, 0, 0, 0, 0, 0, 0, 0, 11252, - 11254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10685, 10687, 10689, - 10691, 10693, 10695, 10697, 10699, 10701, 10703, 0, 0, 0, 0, 0, 0, 11256, - 11258, 11260, 11262, 11264, 7195, 11266, 11268, 11270, 11272, 7197, - 11274, 11276, 11278, 7199, 11280, 11282, 11284, 11286, 11288, 11290, - 11292, 11294, 11296, 11298, 11300, 11302, 7315, 11304, 11306, 11308, - 11310, 11312, 11314, 11316, 11318, 11320, 7325, 7201, 7203, 7327, 11322, - 11324, 6817, 11326, 7205, 11328, 11330, 11332, 11334, 11334, 11334, - 11336, 11338, 11340, 11342, 11344, 11346, 11348, 11350, 11352, 11354, - 11356, 11358, 11360, 11362, 11364, 11366, 11368, 11370, 11370, 7331, - 11372, 11374, 11376, 11378, 7209, 11380, 11382, 11384, 7123, 11386, - 11388, 11390, 11392, 11394, 11396, 11398, 11400, 11402, 11404, 11406, - 11408, 11410, 11412, 11414, 11416, 11418, 11420, 11422, 11424, 11426, - 11428, 11430, 11432, 11434, 11436, 11436, 11438, 11440, 11442, 6809, - 11444, 11446, 11448, 11450, 11452, 11454, 11456, 11458, 7219, 11460, - 11462, 11464, 11466, 11468, 11470, 11472, 11474, 11476, 11478, 11480, - 11482, 11484, 11486, 11488, 11490, 11492, 11494, 11496, 11498, 11500, - 6701, 11502, 11504, 11506, 11506, 11508, 11510, 11510, 11512, 11514, - 11516, 11518, 11520, 11522, 11524, 11526, 11528, 11530, 11532, 11534, - 11536, 7221, 11538, 11540, 11542, 11544, 7355, 11544, 11546, 7225, 11548, - 11550, 11552, 11554, 7227, 6647, 11556, 11558, 11560, 11562, 11564, - 11566, 11568, 11570, 11572, 11574, 11576, 11578, 11580, 11582, 11584, - 11586, 11588, 11590, 11592, 11594, 11596, 11598, 7229, 11600, 11602, - 11604, 11606, 11608, 11610, 7233, 11612, 11614, 11616, 11618, 11620, - 11622, 11624, 11626, 6703, 7371, 11628, 11630, 11632, 11634, 11636, - 11638, 11640, 11642, 7235, 11644, 11646, 11648, 11650, 7457, 11652, - 11654, 11656, 11658, 11660, 11662, 11664, 11666, 11668, 11670, 11672, - 11674, 11676, 6843, 11678, 11680, 11682, 11684, 11686, 11688, 11690, - 11692, 11694, 11696, 11698, 7237, 7017, 11700, 11702, 11704, 11706, - 11708, 11710, 11712, 11714, 7379, 11716, 11718, 11720, 11722, 11724, - 11726, 11728, 11730, 7381, 11732, 11734, 11736, 11738, 11740, 11742, - 11744, 11746, 11748, 11750, 11752, 11754, 7385, 11756, 11758, 11760, - 11762, 11764, 11766, 11768, 11770, 11772, 11774, 11776, 11776, 11778, - 11780, 7389, 11782, 11784, 11786, 11788, 11790, 11792, 11794, 6815, - 11796, 11798, 11800, 11802, 11804, 11806, 11808, 7401, 11810, 11812, - 11814, 11816, 11818, 11820, 11820, 7403, 7461, 11822, 11824, 11826, - 11828, 11830, 6739, 7407, 11832, 11834, 7259, 11836, 11838, 7167, 11840, - 11842, 7267, 11844, 11846, 11848, 11850, 11850, 11852, 11854, 11856, - 11858, 11860, 11862, 11864, 11866, 11868, 11870, 11872, 11874, 11876, - 11878, 11880, 11882, 11884, 11886, 11888, 11890, 11892, 11894, 11896, - 11898, 11900, 11902, 11904, 7279, 11906, 11908, 11910, 11912, 11914, - 11916, 11918, 11920, 11922, 11924, 11926, 11928, 11930, 11932, 11934, - 11936, 11508, 11938, 11940, 11942, 11944, 11946, 11948, 11950, 11952, - 11954, 11956, 11958, 11960, 6851, 11962, 11964, 11966, 11968, 11970, - 11972, 7285, 11974, 11976, 11978, 11980, 11982, 11984, 11986, 11988, - 11990, 11992, 11994, 11996, 11998, 12000, 12002, 12004, 12006, 12008, - 12010, 12012, 6729, 12014, 12016, 12018, 12020, 12022, 12024, 7421, - 12026, 12028, 12030, 12032, 12034, 12036, 12038, 12040, 12042, 12044, - 12046, 12048, 12050, 12052, 12054, 12056, 12058, 12060, 12062, 12064, - 7431, 7433, 12066, 12068, 12070, 12072, 12074, 12076, 12078, 12080, - 12082, 12084, 12086, 12088, 12090, 7435, 12092, 12094, 12096, 12098, - 12100, 12102, 12104, 12106, 12108, 12110, 12112, 12114, 12116, 12118, - 12120, 12122, 12124, 12126, 12128, 12130, 12132, 12134, 12136, 12138, - 12140, 12142, 12144, 12146, 12148, 12150, 7447, 7447, 12152, 12154, - 12156, 12158, 12160, 12162, 12164, 12166, 12168, 12170, 7449, 12172, - 12174, 12176, 12178, 12180, 12182, 12184, 12186, 12188, 12190, 12192, - 12194, 12196, 12198, 12200, 12202, 12204, 12206, 12208, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 10879, 10881, 10883, 10885, 10887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 10889, 10891, 10893, 10895, 0, 10897, 10899, 10901, 10903, + 10905, 10907, 10909, 10911, 10913, 10915, 10917, 10919, 10921, 10923, + 10925, 10927, 10929, 10931, 10933, 10935, 10937, 10939, 10941, 10943, + 10945, 10947, 10949, 0, 10891, 10893, 0, 10951, 0, 0, 10901, 0, 10905, + 10907, 10909, 10911, 10913, 10915, 10917, 10919, 10921, 10923, 0, 10927, + 10929, 10931, 10933, 0, 10937, 0, 10941, 0, 0, 0, 0, 0, 0, 10893, 0, 0, + 0, 0, 10901, 0, 10905, 0, 10909, 0, 10913, 10915, 10917, 0, 10921, 10923, + 0, 10927, 0, 0, 10933, 0, 10937, 0, 10941, 0, 10945, 0, 10949, 0, 10891, + 10893, 0, 10951, 0, 0, 10901, 10903, 10905, 10907, 0, 10911, 10913, + 10915, 10917, 10919, 10921, 10923, 0, 10927, 10929, 10931, 10933, 0, + 10937, 10939, 10941, 10943, 0, 10947, 0, 10889, 10891, 10893, 10895, + 10951, 10897, 10899, 10901, 10903, 10905, 0, 10909, 10911, 10913, 10915, + 10917, 10919, 10921, 10923, 10925, 10927, 10929, 10931, 10933, 10935, + 10937, 10939, 10941, 0, 0, 0, 0, 0, 10891, 10893, 10895, 0, 10897, 10899, + 10901, 10903, 10905, 0, 10909, 10911, 10913, 10915, 10917, 10919, 10921, + 10923, 10925, 10927, 10929, 10931, 10933, 10935, 10937, 10939, 10941, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10953, 10956, + 10959, 10962, 10965, 10968, 10971, 10974, 10977, 10980, 10983, 0, 0, 0, + 0, 0, 10986, 10990, 10994, 10998, 11002, 11006, 11010, 11014, 11018, + 11022, 11026, 11030, 11034, 11038, 11042, 11046, 11050, 11054, 11058, + 11062, 11066, 11070, 11074, 11078, 11082, 11086, 11090, 3926, 3956, + 11094, 11097, 0, 11100, 11102, 11104, 11106, 11108, 11110, 11112, 11114, + 11116, 11118, 11120, 11122, 11124, 11126, 11128, 11130, 11132, 11134, + 11136, 11138, 11140, 11142, 11144, 11146, 11148, 11150, 11152, 6348, + 11155, 11158, 11161, 11165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11168, 11171, 11174, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 11177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11180, + 11183, 11186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11188, 11190, 11192, + 11194, 11196, 11198, 11200, 11202, 11204, 11206, 11208, 11210, 11212, + 11214, 11216, 11218, 11220, 11222, 11224, 11226, 11228, 11230, 11232, + 11234, 11236, 11238, 11240, 11242, 11244, 11246, 11248, 11250, 11252, + 11254, 11256, 11258, 11260, 11262, 11264, 11266, 11268, 11270, 11272, + 11274, 0, 0, 0, 0, 11276, 11280, 11284, 11288, 11292, 11296, 11300, + 11304, 11308, 0, 0, 0, 0, 0, 0, 0, 11312, 11314, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 10554, 10556, 10558, 10560, 10562, 10564, 10566, 10568, + 10570, 10572, 0, 0, 0, 0, 0, 0, 11316, 11318, 11320, 11322, 11324, 7195, + 11326, 11328, 11330, 11332, 7197, 11334, 11336, 11338, 7199, 11340, + 11342, 11344, 11346, 11348, 11350, 11352, 11354, 11356, 11358, 11360, + 11362, 7315, 11364, 11366, 11368, 11370, 11372, 11374, 11376, 11378, + 11380, 7325, 7201, 7203, 7327, 11382, 11384, 6817, 11386, 7205, 11388, + 11390, 11392, 11394, 11394, 11394, 11396, 11398, 11400, 11402, 11404, + 11406, 11408, 11410, 11412, 11414, 11416, 11418, 11420, 11422, 11424, + 11426, 11428, 11430, 11430, 7331, 11432, 11434, 11436, 11438, 7209, + 11440, 11442, 11444, 7123, 11446, 11448, 11450, 11452, 11454, 11456, + 11458, 11460, 11462, 11464, 11466, 11468, 11470, 11472, 11474, 11476, + 11478, 11480, 11482, 11484, 11486, 11488, 11490, 11492, 11494, 11496, + 11496, 11498, 11500, 11502, 6809, 11504, 11506, 11508, 11510, 11512, + 11514, 11516, 11518, 7219, 11520, 11522, 11524, 11526, 11528, 11530, + 11532, 11534, 11536, 11538, 11540, 11542, 11544, 11546, 11548, 11550, + 11552, 11554, 11556, 11558, 11560, 6701, 11562, 11564, 11566, 11566, + 11568, 11570, 11570, 11572, 11574, 11576, 11578, 11580, 11582, 11584, + 11586, 11588, 11590, 11592, 11594, 11596, 7221, 11598, 11600, 11602, + 11604, 7355, 11604, 11606, 7225, 11608, 11610, 11612, 11614, 7227, 6647, + 11616, 11618, 11620, 11622, 11624, 11626, 11628, 11630, 11632, 11634, + 11636, 11638, 11640, 11642, 11644, 11646, 11648, 11650, 11652, 11654, + 11656, 11658, 7229, 11660, 11662, 11664, 11666, 11668, 11670, 7233, + 11672, 11674, 11676, 11678, 11680, 11682, 11684, 11686, 6703, 7371, + 11688, 11690, 11692, 11694, 11696, 11698, 11700, 11702, 7235, 11704, + 11706, 11708, 11710, 7457, 11712, 11714, 11716, 11718, 11720, 11722, + 11724, 11726, 11728, 11730, 11732, 11734, 11736, 6843, 11738, 11740, + 11742, 11744, 11746, 11748, 11750, 11752, 11754, 11756, 11758, 7237, + 7017, 11760, 11762, 11764, 11766, 11768, 11770, 11772, 11774, 7379, + 11776, 11778, 11780, 11782, 11784, 11786, 11788, 11790, 7381, 11792, + 11794, 11796, 11798, 11800, 11802, 11804, 11806, 11808, 11810, 11812, + 11814, 7385, 11816, 11818, 11820, 11822, 11824, 11826, 11828, 11830, + 11832, 11834, 11836, 11836, 11838, 11840, 7389, 11842, 11844, 11846, + 11848, 11850, 11852, 11854, 6815, 11856, 11858, 11860, 11862, 11864, + 11866, 11868, 7401, 11870, 11872, 11874, 11876, 11878, 11880, 11880, + 7403, 7461, 11882, 11884, 11886, 11888, 11890, 6739, 7407, 11892, 11894, + 7259, 11896, 11898, 7167, 11900, 11902, 7267, 11904, 11906, 11908, 11910, + 11910, 11912, 11914, 11916, 11918, 11920, 11922, 11924, 11926, 11928, + 11930, 11932, 11934, 11936, 11938, 11940, 11942, 11944, 11946, 11948, + 11950, 11952, 11954, 11956, 11958, 11960, 11962, 11964, 7279, 11966, + 11968, 11970, 11972, 11974, 11976, 11978, 11980, 11982, 11984, 11986, + 11988, 11990, 11992, 11994, 11996, 11568, 11998, 12000, 12002, 12004, + 12006, 12008, 12010, 12012, 12014, 12016, 12018, 12020, 6851, 12022, + 12024, 12026, 12028, 12030, 12032, 7285, 12034, 12036, 12038, 12040, + 12042, 12044, 12046, 12048, 12050, 12052, 12054, 12056, 12058, 12060, + 12062, 12064, 12066, 12068, 12070, 12072, 6729, 12074, 12076, 12078, + 12080, 12082, 12084, 7421, 12086, 12088, 12090, 12092, 12094, 12096, + 12098, 12100, 12102, 12104, 12106, 12108, 12110, 12112, 12114, 12116, + 12118, 12120, 12122, 12124, 7431, 7433, 12126, 12128, 12130, 12132, + 12134, 12136, 12138, 12140, 12142, 12144, 12146, 12148, 12150, 7435, + 12152, 12154, 12156, 12158, 12160, 12162, 12164, 12166, 12168, 12170, + 12172, 12174, 12176, 12178, 12180, 12182, 12184, 12186, 12188, 12190, + 12192, 12194, 12196, 12198, 12200, 12202, 12204, 12206, 12208, 12210, + 7447, 7447, 12212, 12214, 12216, 12218, 12220, 12222, 12224, 12226, + 12228, 12230, 7449, 12232, 12234, 12236, 12238, 12240, 12242, 12244, + 12246, 12248, 12250, 12252, 12254, 12256, 12258, 12260, 12262, 12264, + 12266, 12268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, }; /* NFC pairs */ #define COMP_SHIFT 2 static const unsigned short comp_index[] = { - 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, - 5, 6, 7, 8, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 11, 0, 12, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 15, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 17, 18, 19, 20, 21, 22, 0, 0, 0, 0, 0, 0, 0, 0, 23, 24, 25, 26, 27, - 28, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 31, 32, 33, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 36, - 37, 38, 39, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 42, 43, 44, 45, 46, 47, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 49, 0, 50, 0, 51, 52, 53, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 55, 56, 57, - 58, 59, 0, 0, 0, 0, 0, 0, 0, 0, 60, 61, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 63, 64, 65, 0, 66, 67, 68, 0, 0, 0, 0, 0, 0, 0, 0, 69, 70, 71, - 72, 73, 74, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 77, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 78, 79, 0, 80, 81, 82, 83, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 84, 85, 86, 0, 87, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 90, 0, 91, - 92, 93, 0, 0, 0, 0, 0, 0, 0, 0, 94, 95, 96, 97, 98, 99, 100, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 101, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, - 104, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 107, 108, 109, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 111, 112, 0, 113, 114, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 117, - 118, 119, 120, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 123, 0, - 124, 0, 0, 0, 0, 0, 0, 0, 0, 125, 126, 127, 128, 0, 129, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 130, 0, 131, 132, 133, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 135, 136, 137, 138, 139, 140, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 144, 145, 146, 0, 147, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 148, 149, 150, 151, 152, 153, 154, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 155, 156, 157, 158, 159, 160, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 162, 0, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 165, 166, - 167, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 170, 171, 172, 173, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 174, 175, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 177, 178, 179, 180, 0, 181, 182, 183, 0, 0, 0, 0, 0, 0, 0, 0, 184, 185, - 186, 187, 188, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 191, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 193, 194, 195, 196, 197, 198, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 199, 200, 201, 0, 202, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 204, 205, 206, 207, 208, 209, 0, 0, 0, 0, 0, 0, 0, 0, 210, 211, - 212, 213, 214, 215, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 0, 0, 218, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 220, 221, 222, 0, 223, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 226, - 227, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 230, 231, 0, 232, 0, - 233, 0, 0, 0, 0, 0, 0, 0, 0, 234, 235, 0, 0, 0, 0, 0, 236, 0, 0, 0, 0, 0, - 0, 0, 0, 237, 238, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 245, 246, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, - 249, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 252, 253, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 257, 0, 258, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 260, 261, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, - 267, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 270, 271, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 273, 274, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 278, 279, 0, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 282, - 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 284, 285, 286, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 288, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 301, 302, 303, 0, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 306, - 307, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 310, 311, 0, 312, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 0, 314, 0, 315, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 327, - 0, 328, 329, 0, 0, 330, 0, 0, 0, 0, 0, 0, 0, 0, 331, 0, 0, 332, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 333, 334, 0, 0, 335, 0, 0, 0, 336, 0, 0, 0, 0, 0, - 0, 0, 337, 338, 339, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 0, 0, - 342, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 345, 346, 347, 0, 348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 349, 0, 0, 0, 350, 0, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 0, 0, 0, 0, - 0, 0, 354, 355, 356, 0, 357, 0, 0, 358, 359, 0, 0, 0, 0, 0, 0, 0, 360, 0, - 0, 0, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 0, 0, 363, 364, 0, 0, - 365, 0, 0, 0, 0, 0, 0, 0, 0, 366, 367, 0, 368, 0, 0, 0, 369, 0, 0, 0, 0, - 0, 0, 0, 370, 371, 0, 0, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 375, 376, 377, 378, 0, 0, - 379, 0, 0, 0, 0, 0, 0, 0, 0, 380, 0, 0, 381, 0, 0, 0, 382, 0, 0, 0, 0, 0, - 0, 0, 383, 384, 0, 0, 0, 0, 0, 385, 0, 0, 0, 0, 0, 0, 0, 0, 386, 0, 0, 0, - 0, 0, 0, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 0, 0, 0, - 0, 0, 0, 0, 0, 389, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 393, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 395, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 399, 400, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 4, 5, 6, 7, 8, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 10, 0, 0, 11, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 15, 0, + 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 18, 0, 19, 20, 21, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 23, 24, 25, 26, 27, 28, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 30, 31, 32, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 35, 36, 0, + 37, 38, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 41, 42, 43, 44, 45, 46, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 48, 0, 49, 0, 50, 51, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 53, 0, 54, 0, 55, 56, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 59, 0, 0, + 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 62, 63, 0, 64, 65, 66, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 68, 69, 70, 71, 72, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 73, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 75, 76, 77, 78, 79, 80, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 83, 84, + 0, 85, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 88, 0, 89, 90, 91, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 93, 94, 95, 96, 97, 98, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 101, 102, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 106, 107, 0, 108, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 110, 111, 0, 112, 0, 113, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 115, 116, 117, 118, 119, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 121, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 123, 124, 125, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 127, 128, 0, 129, 130, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 133, + 134, 135, 136, 137, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 141, 142, 0, 0, 143, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 145, 146, 0, 147, 148, 149, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 150, 151, 152, 153, 154, 155, 156, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 157, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 159, 0, 160, 0, 161, 162, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, + 0, 165, 0, 166, 167, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 170, 0, + 0, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 173, 174, 0, 175, + 176, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 179, 180, 181, 182, 183, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 186, 187, 188, 189, 190, 191, 192, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 193, 194, 195, 0, 196, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 198, 199, 0, 200, 201, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 203, 204, 205, 206, 207, 208, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, + 0, 0, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 213, 214, 0, + 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 218, 219, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 221, 222, 223, 0, 224, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 226, 0, 0, 0, 0, 0, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 0, + 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 232, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 236, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 240, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 245, 246, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 252, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 254, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 258, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 260, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, + 264, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 267, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 268, 0, 269, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 283, 0, 284, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 286, 0, 287, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 0, + 290, 0, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 0, 293, 0, 294, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 302, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 305, 306, 0, 307, 0, 0, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 309, 0, 0, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 0, 0, 312, + 0, 0, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 315, 0, 316, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 320, 321, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 0, + 324, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 328, 329, 0, 330, 0, 0, 0, 331, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 332, 0, 0, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, + 0, 335, 0, 0, 0, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 338, 0, 339, 0, + 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 0, 0, 342, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 344, 345, 0, 346, 0, 0, 0, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 348, 0, 0, 349, 0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, + 0, 0, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 0, 0, 0, 0, 0, 354, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 356, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 361, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 365, 366, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, + 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 381, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 382, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 384, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 391, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 397, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 404, 405, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 411, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 413, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 416, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 417, 418, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 423, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 429, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 430, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 436, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 442, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 446, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 456, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 0, 0, 0, 470, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 477, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 0, 0, 0, 0, - 0, 0, 484, 0, 0, 0, 0, 0, 0, 0, 0, 485, 0, 0, 0, 0, 0, 0, 486, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 0, 0, 0, - 0, 0, 0, 0, 0, 493, 0, 0, 0, 0, 0, 0, 494, 0, 0, 0, 0, 0, 0, 0, 0, 495, - 0, 0, 0, 0, 0, 0, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 503, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 505, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 0, 0, 0, 0, 0, 0, 509, 0, 0, 0, 0, 0, - 0, 0, 0, 510, 0, 0, 0, 0, 0, 0, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, - 0, 0, 0, 0, 519, 0, 0, 0, 0, 0, 0, 0, 0, 520, 0, 0, 0, 0, 0, 0, 521, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 0, - 0, 0, 0, 0, 0, 0, 0, 528, 0, 0, 0, 0, 0, 0, 529, 0, 0, 0, 0, 0, 0, 0, 0, - 530, 0, 0, 0, 0, 0, 0, 531, 0, 0, 0, 0, 0, 0, 0, 532, 533, 0, 0, 0, 0, 0, - 534, 0, 0, 0, 0, 0, 0, 0, 0, 535, 0, 0, 0, 0, 0, 0, 536, 0, 0, 0, 0, 0, - 0, 0, 0, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 0, 0, - 0, 0, 0, 0, 543, 0, 0, 0, 0, 0, 0, 0, 0, 544, 0, 0, 0, 0, 0, 0, 545, 0, - 0, 0, 0, 0, 0, 0, 546, 547, 0, 0, 0, 0, 0, 548, 0, 0, 0, 0, 0, 0, 0, 0, - 549, 0, 0, 0, 0, 0, 0, 550, 0, 0, 0, 0, 0, 0, 0, 0, 551, 0, 0, 0, 0, 0, - 0, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 558, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 0, - 0, 0, 0, 0, 561, 0, 0, 0, 0, 0, 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 572, 0, 0, 0, 0, 0, 0, 0, 573, 0, 0, 0, 0, 0, 0, 574, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 576, 0, 0, 0, 0, 0, 0, 0, 577, 578, 0, 0, 0, 0, 0, 579, + 0, 0, 0, 0, 0, 0, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 406, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 409, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 411, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 412, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 414, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 419, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 431, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 433, 0, 0, 0, 0, 0, 0, 434, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 440, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 442, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 0, 0, + 0, 0, 0, 0, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 0, 0, 0, 0, 0, 0, + 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 458, 0, 0, 0, 0, 0, 0, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 0, 0, + 0, 0, 0, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 472, 0, 0, 0, 0, 0, 0, 473, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 474, 0, 0, 0, 0, 0, 0, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 0, 0, 0, 0, 0, 0, 483, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 484, 0, 0, 0, 0, 0, 0, 485, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 0, 0, 0, 0, 0, 0, + 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 0, 0, 0, 0, 0, 0, 495, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 496, 0, 0, 0, 0, 0, 0, 497, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 498, 0, 0, 0, 0, 0, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, + 0, 0, 0, 0, 0, 0, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 0, 0, 0, 0, 0, + 0, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 0, 0, 0, 0, 0, 0, 509, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 510, 0, 0, 0, 0, 0, 0, 511, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 0, 0, 0, 0, 0, 0, 521, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 522, 0, 0, 0, 0, 0, 0, 523, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 0, 0, 0, 0, + 0, 0, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 538, 0, 0, 0, 0, 0, 0, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 580, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 582, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 595, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 584, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 585, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 593, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 597, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 601, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 604, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 613, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 619, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 623, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 626, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 629, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 632, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 635, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 638, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 639, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 642, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 645, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 648, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 654, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 660, 661, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 664, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 665, 666, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 669, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 675, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 678, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 681, + 0, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 601, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 605, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 606, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 614, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 618, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 619, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 622, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 623, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 626, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 627, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 628, 0, 0, 0, 0, 0, 629, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 631, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 635, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 639, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 643, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 646, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 647, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 648, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 652, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 655, }; static const unsigned int comp_data[] = { - 0, 0, 0, 0, 0, 0, 0, 8814, 0, 0, 8800, 0, 0, 8815, 0, 0, 0, 192, 193, - 194, 195, 256, 258, 550, 196, 7842, 197, 0, 461, 512, 514, 0, 0, 0, 7840, - 0, 7680, 0, 0, 260, 0, 0, 7682, 0, 0, 7684, 0, 0, 0, 0, 7686, 0, 262, - 264, 0, 0, 0, 266, 0, 0, 0, 0, 268, 0, 199, 0, 0, 0, 7690, 0, 0, 0, 0, - 270, 0, 0, 0, 0, 0, 7692, 0, 0, 0, 7696, 0, 7698, 0, 0, 7694, 0, 0, 0, 0, - 200, 201, 202, 7868, 274, 276, 278, 203, 7866, 0, 0, 282, 516, 518, 0, 0, - 0, 7864, 0, 0, 0, 552, 280, 7704, 0, 7706, 0, 0, 0, 7710, 0, 500, 284, 0, - 7712, 286, 288, 0, 0, 0, 0, 486, 0, 290, 0, 0, 0, 292, 0, 0, 0, 7714, - 7718, 0, 0, 0, 542, 0, 0, 0, 0, 0, 7716, 0, 0, 0, 7720, 0, 0, 7722, 0, 0, - 204, 205, 206, 296, 298, 300, 304, 207, 7880, 0, 0, 463, 520, 522, 0, 0, - 0, 7882, 0, 0, 0, 0, 302, 0, 0, 7724, 0, 0, 0, 308, 0, 7728, 0, 0, 0, 0, - 0, 488, 0, 7730, 0, 0, 0, 310, 0, 0, 0, 0, 7732, 0, 0, 0, 0, 0, 313, 0, - 317, 0, 0, 0, 0, 0, 7734, 0, 0, 0, 315, 0, 7740, 0, 0, 7738, 0, 0, 0, 0, - 0, 7742, 0, 0, 0, 0, 7744, 0, 0, 7746, 0, 504, 323, 0, 209, 0, 0, 7748, - 0, 0, 0, 0, 327, 0, 7750, 0, 0, 0, 325, 0, 7754, 0, 0, 7752, 0, 0, 0, 0, - 210, 211, 212, 213, 332, 334, 558, 214, 7886, 0, 336, 465, 524, 526, 0, - 0, 416, 7884, 0, 0, 0, 0, 490, 0, 0, 0, 0, 0, 7764, 7766, 0, 0, 0, 0, 0, - 340, 0, 0, 0, 0, 7768, 344, 528, 530, 0, 0, 0, 7770, 0, 0, 0, 342, 0, 0, - 0, 0, 7774, 0, 346, 348, 0, 0, 0, 7776, 0, 0, 0, 0, 352, 0, 7778, 0, 0, - 536, 350, 0, 0, 0, 7786, 0, 0, 0, 0, 356, 0, 7788, 0, 0, 538, 354, 0, - 7792, 0, 0, 7790, 0, 0, 0, 0, 217, 218, 219, 360, 362, 364, 0, 220, 7910, - 366, 368, 467, 532, 534, 0, 0, 431, 7908, 7794, 0, 0, 0, 370, 7798, 0, - 7796, 7804, 0, 0, 0, 0, 0, 7806, 0, 7808, 7810, 372, 0, 0, 0, 7814, 7812, - 0, 7816, 0, 0, 0, 7818, 7820, 0, 0, 0, 7922, 221, 374, 7928, 562, 0, - 7822, 376, 7926, 0, 0, 0, 0, 7924, 0, 0, 377, 7824, 0, 0, 0, 379, 381, 0, - 0, 0, 0, 0, 7826, 0, 0, 0, 0, 7828, 224, 225, 226, 227, 257, 259, 551, - 228, 7843, 229, 0, 462, 513, 515, 0, 0, 0, 7841, 0, 7681, 0, 0, 261, 0, - 0, 7683, 0, 0, 7685, 0, 0, 0, 0, 7687, 0, 0, 0, 0, 0, 263, 265, 0, 0, 0, - 267, 0, 0, 0, 0, 269, 0, 0, 0, 0, 0, 231, 0, 0, 0, 7691, 271, 0, 0, 0, 0, - 0, 7693, 0, 0, 0, 7697, 0, 7699, 0, 0, 7695, 232, 233, 234, 7869, 275, - 277, 279, 235, 7867, 0, 0, 283, 517, 519, 0, 0, 0, 7865, 0, 0, 0, 553, - 281, 7705, 0, 7707, 0, 0, 0, 7711, 0, 0, 0, 0, 0, 501, 285, 0, 7713, 287, - 289, 0, 0, 0, 0, 487, 0, 0, 0, 0, 0, 291, 0, 0, 0, 293, 0, 0, 0, 7715, - 7719, 0, 0, 0, 543, 0, 0, 0, 0, 0, 7717, 0, 0, 0, 7721, 0, 0, 7723, 0, - 7830, 236, 237, 238, 297, 299, 301, 0, 239, 7881, 0, 0, 464, 521, 523, 0, - 0, 0, 7883, 0, 0, 0, 0, 303, 0, 0, 7725, 0, 0, 0, 309, 0, 0, 0, 0, 496, - 0, 0, 0, 0, 7729, 0, 489, 0, 0, 0, 0, 0, 7731, 0, 0, 0, 311, 7733, 0, 0, - 0, 0, 0, 314, 0, 318, 0, 0, 0, 0, 0, 7735, 0, 0, 0, 316, 0, 7741, 0, 0, - 7739, 0, 7743, 0, 0, 0, 0, 7745, 0, 0, 7747, 0, 0, 0, 0, 0, 505, 324, 0, - 241, 0, 0, 7749, 0, 0, 0, 0, 328, 0, 7751, 0, 0, 0, 326, 0, 7755, 0, 0, - 7753, 0, 0, 0, 0, 242, 243, 244, 245, 333, 335, 559, 246, 7887, 0, 337, - 466, 525, 527, 0, 0, 417, 7885, 491, 0, 0, 0, 0, 0, 7765, 0, 0, 0, 0, - 7767, 0, 341, 0, 0, 0, 0, 7769, 0, 0, 0, 0, 345, 529, 531, 0, 0, 0, 7771, - 0, 0, 0, 343, 0, 0, 0, 0, 7775, 0, 347, 349, 0, 0, 0, 7777, 0, 0, 0, 0, - 353, 0, 7779, 0, 0, 537, 351, 0, 0, 0, 7787, 7831, 0, 0, 0, 357, 0, 0, 0, - 0, 0, 7789, 0, 0, 539, 355, 0, 7793, 0, 0, 7791, 0, 0, 0, 0, 249, 250, - 251, 361, 363, 365, 0, 252, 7911, 367, 369, 468, 533, 535, 0, 0, 432, - 7909, 7795, 0, 0, 0, 371, 7799, 0, 7797, 0, 0, 0, 0, 7805, 0, 7807, 0, 0, - 0, 0, 0, 7809, 7811, 373, 0, 0, 0, 7815, 7813, 0, 7832, 0, 0, 0, 7817, 0, - 0, 0, 7819, 7821, 0, 0, 0, 7923, 253, 375, 7929, 563, 0, 7823, 255, 7927, - 7833, 0, 0, 0, 7925, 0, 0, 378, 7825, 0, 0, 0, 380, 0, 0, 0, 0, 382, 0, - 7827, 0, 0, 0, 0, 7829, 0, 0, 0, 0, 8173, 901, 0, 0, 0, 0, 0, 0, 8129, 0, - 0, 7846, 7844, 0, 7850, 0, 0, 0, 0, 7848, 0, 0, 478, 0, 0, 0, 506, 0, 0, - 508, 0, 0, 482, 0, 0, 0, 7688, 0, 7872, 7870, 0, 7876, 0, 0, 0, 0, 7874, - 0, 0, 0, 7726, 0, 0, 0, 0, 0, 7890, 7888, 0, 7894, 0, 0, 0, 0, 7892, 0, - 0, 0, 7756, 0, 0, 556, 0, 0, 7758, 0, 0, 0, 554, 0, 0, 0, 510, 0, 0, 0, - 0, 0, 475, 471, 0, 0, 469, 0, 0, 473, 0, 0, 0, 7847, 7845, 0, 7851, 0, 0, - 0, 0, 7849, 0, 0, 479, 0, 0, 0, 507, 0, 0, 509, 0, 0, 483, 0, 0, 0, 7689, - 0, 7873, 7871, 0, 7877, 0, 0, 0, 0, 7875, 0, 0, 0, 7727, 0, 0, 0, 0, 0, - 7891, 7889, 0, 7895, 0, 0, 0, 0, 7893, 0, 0, 0, 7757, 0, 0, 557, 0, 0, - 7759, 0, 0, 0, 555, 0, 0, 0, 511, 0, 0, 0, 0, 0, 476, 472, 0, 0, 470, 0, - 0, 474, 0, 0, 0, 7856, 7854, 0, 7860, 0, 0, 0, 0, 7858, 0, 0, 7857, 7855, - 0, 7861, 0, 0, 0, 0, 7859, 0, 0, 7700, 7702, 0, 0, 0, 0, 0, 7701, 7703, - 0, 0, 0, 0, 0, 7760, 7762, 0, 7761, 7763, 0, 0, 0, 7780, 0, 0, 7781, 0, - 0, 7782, 0, 0, 0, 0, 0, 0, 7783, 0, 7800, 0, 0, 7801, 0, 0, 0, 0, 7802, - 0, 0, 7803, 0, 0, 0, 0, 0, 7835, 0, 0, 0, 0, 7900, 7898, 0, 7904, 0, 0, - 0, 0, 7902, 7906, 0, 0, 0, 0, 0, 7901, 7899, 0, 7905, 0, 0, 0, 0, 7903, - 0, 0, 0, 0, 7907, 0, 7914, 7912, 0, 7918, 0, 0, 0, 0, 7916, 0, 0, 0, 0, - 7920, 0, 7915, 7913, 0, 7919, 7917, 0, 0, 0, 0, 7921, 0, 0, 0, 0, 494, 0, - 0, 0, 492, 0, 0, 493, 0, 0, 480, 0, 0, 0, 0, 0, 0, 481, 0, 0, 0, 7708, 0, - 0, 7709, 0, 560, 0, 0, 0, 0, 0, 0, 561, 0, 495, 0, 0, 0, 8122, 902, 0, 0, - 8121, 8120, 0, 0, 0, 0, 7944, 7945, 0, 0, 0, 0, 0, 8124, 0, 8136, 904, 0, - 0, 0, 0, 7960, 7961, 0, 0, 0, 8138, 905, 0, 0, 0, 0, 7976, 7977, 0, 8140, - 0, 0, 0, 0, 0, 8154, 906, 0, 0, 8153, 8152, 0, 938, 0, 0, 7992, 7993, 0, - 0, 0, 8184, 908, 0, 0, 0, 0, 8008, 8009, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, - 8170, 910, 0, 0, 8169, 8168, 0, 939, 0, 0, 0, 8025, 0, 0, 0, 8186, 911, - 8040, 8041, 0, 0, 0, 0, 0, 8188, 0, 0, 8116, 0, 0, 8132, 0, 0, 0, 0, 0, - 8048, 940, 0, 0, 8113, 8112, 0, 0, 0, 0, 7936, 7937, 0, 0, 0, 0, 8118, - 8115, 0, 0, 0, 0, 0, 8050, 941, 7952, 7953, 0, 0, 0, 8052, 942, 0, 0, 0, - 0, 7968, 7969, 0, 0, 0, 0, 8134, 8131, 0, 8054, 943, 0, 0, 8145, 8144, 0, - 970, 0, 0, 7984, 7985, 8150, 0, 0, 0, 0, 0, 0, 8056, 972, 0, 0, 0, 0, - 8000, 8001, 0, 8164, 8165, 0, 0, 0, 8058, 973, 0, 0, 8161, 8160, 0, 971, - 0, 0, 0, 0, 0, 0, 8016, 8017, 0, 0, 0, 0, 8166, 0, 0, 8060, 974, 0, 0, 0, - 0, 8032, 8033, 8182, 8179, 0, 0, 0, 0, 0, 8146, 912, 0, 0, 0, 0, 0, 0, - 8151, 0, 0, 8162, 944, 0, 0, 8167, 0, 0, 0, 8180, 0, 0, 979, 0, 0, 0, 0, - 0, 980, 0, 0, 1031, 0, 0, 0, 0, 1232, 0, 1234, 0, 0, 0, 0, 1027, 0, 1024, - 0, 0, 0, 0, 1238, 0, 1025, 1217, 0, 1244, 0, 0, 1246, 0, 0, 0, 1037, 0, - 0, 0, 1250, 1049, 0, 1252, 0, 0, 0, 0, 1036, 0, 0, 0, 0, 1254, 0, 0, 0, - 1262, 1038, 0, 1264, 0, 0, 1266, 0, 0, 0, 1268, 0, 0, 0, 0, 0, 0, 1272, - 0, 0, 1260, 0, 0, 0, 0, 1233, 0, 1235, 0, 0, 0, 0, 1107, 0, 1104, 0, 0, - 0, 0, 1239, 0, 1105, 1218, 0, 1245, 0, 0, 1247, 0, 0, 0, 1117, 0, 0, 0, - 1251, 1081, 0, 1253, 0, 0, 0, 0, 1116, 0, 0, 0, 0, 1255, 0, 0, 0, 1263, - 1118, 0, 1265, 0, 0, 1267, 0, 0, 0, 1269, 0, 0, 0, 0, 0, 0, 1273, 0, 0, - 1261, 0, 0, 1111, 0, 0, 0, 1142, 0, 0, 1143, 0, 0, 0, 0, 0, 1242, 0, 0, - 1243, 0, 0, 1258, 0, 0, 0, 0, 0, 0, 1259, 0, 1570, 1571, 1573, 0, 1572, - 0, 0, 1574, 0, 0, 0, 0, 0, 0, 1730, 0, 0, 1747, 0, 0, 1728, 0, 0, 0, 0, - 2345, 0, 0, 2353, 0, 0, 2356, 0, 0, 0, 2507, 2508, 0, 0, 0, 2891, 2888, - 2892, 2964, 0, 0, 0, 0, 0, 3018, 3020, 0, 3019, 0, 0, 0, 0, 3144, 0, 0, - 0, 0, 3264, 0, 3274, 3271, 3272, 0, 3275, 0, 0, 0, 0, 3402, 3404, 0, - 3403, 0, 0, 0, 0, 3546, 3548, 3550, 0, 0, 0, 0, 3549, 0, 0, 0, 0, 0, - 4134, 0, 0, 0, 6918, 0, 0, 6920, 0, 0, 6922, 0, 0, 6924, 0, 0, 0, 0, 0, - 0, 6926, 0, 0, 6930, 0, 0, 6971, 0, 0, 6973, 0, 0, 0, 0, 0, 0, 6976, 0, - 0, 6977, 0, 0, 6979, 0, 0, 0, 7736, 0, 0, 7737, 0, 0, 0, 0, 0, 0, 7772, - 0, 0, 7773, 0, 0, 0, 0, 7784, 0, 0, 7785, 0, 0, 7852, 0, 0, 7862, 0, 0, - 0, 7853, 0, 0, 7863, 0, 0, 0, 7878, 0, 0, 7879, 0, 0, 7896, 0, 0, 7897, - 0, 0, 0, 0, 7938, 7940, 0, 0, 7942, 8064, 0, 7939, 7941, 0, 0, 7943, - 8065, 0, 0, 8066, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 8068, 0, 0, 8069, 0, 0, - 8070, 0, 0, 0, 0, 0, 0, 8071, 0, 7946, 7948, 0, 0, 7950, 8072, 0, 7947, - 7949, 0, 0, 7951, 8073, 0, 0, 8074, 0, 0, 0, 0, 0, 0, 8075, 0, 0, 8076, - 0, 0, 8077, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 8079, 0, 7954, 7956, 0, 7955, - 7957, 0, 0, 0, 0, 0, 7962, 7964, 0, 0, 0, 0, 0, 7963, 7965, 0, 7970, - 7972, 0, 0, 7974, 8080, 0, 7971, 7973, 0, 0, 7975, 8081, 0, 0, 8082, 0, - 0, 0, 0, 0, 0, 8083, 0, 0, 8084, 0, 0, 8085, 0, 0, 8086, 0, 0, 0, 0, 0, - 0, 8087, 0, 7978, 7980, 0, 0, 7982, 8088, 0, 7979, 7981, 0, 0, 7983, - 8089, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 8092, 0, 0, 8093, 0, 0, - 8094, 0, 0, 0, 0, 0, 0, 8095, 0, 7986, 7988, 0, 0, 7990, 0, 0, 7987, - 7989, 0, 0, 7991, 0, 0, 0, 0, 0, 0, 7994, 7996, 0, 0, 0, 0, 0, 0, 7998, - 0, 0, 7995, 7997, 0, 0, 7999, 0, 0, 8002, 8004, 0, 8003, 8005, 0, 0, 0, - 0, 0, 8010, 8012, 0, 0, 0, 0, 0, 8011, 8013, 0, 8018, 8020, 0, 0, 8022, - 0, 0, 8019, 8021, 0, 0, 8023, 0, 0, 0, 0, 0, 0, 8027, 8029, 0, 0, 0, 0, - 0, 0, 8031, 0, 0, 8034, 8036, 0, 0, 8038, 8096, 0, 8035, 8037, 0, 0, - 8039, 8097, 0, 0, 8098, 0, 0, 8099, 0, 0, 0, 0, 0, 0, 8100, 0, 0, 8101, - 0, 0, 8102, 0, 0, 8103, 0, 0, 0, 0, 0, 8042, 8044, 0, 0, 8046, 8104, 0, - 8043, 8045, 0, 0, 8047, 8105, 0, 0, 8106, 0, 0, 8107, 0, 0, 0, 0, 0, 0, - 8108, 0, 0, 8109, 0, 0, 8110, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 8114, 0, 0, - 8130, 0, 0, 8178, 0, 0, 8119, 0, 0, 0, 0, 0, 8141, 8142, 0, 0, 8143, 0, - 0, 0, 8135, 0, 0, 8183, 0, 0, 0, 0, 0, 8157, 8158, 0, 0, 0, 0, 0, 0, - 8159, 0, 8602, 0, 0, 8603, 0, 0, 0, 0, 0, 0, 8622, 0, 0, 8653, 0, 0, - 8655, 0, 0, 8654, 0, 0, 0, 0, 0, 0, 8708, 0, 0, 8713, 0, 0, 8716, 0, 0, - 8740, 0, 0, 0, 0, 0, 0, 8742, 0, 0, 8769, 0, 0, 8772, 0, 0, 8775, 0, 0, - 0, 0, 0, 0, 8777, 0, 0, 8813, 0, 0, 8802, 0, 0, 8816, 0, 0, 0, 0, 0, 0, - 8817, 0, 0, 8820, 0, 0, 8821, 0, 0, 8824, 0, 0, 0, 0, 0, 0, 8825, 0, 0, - 8832, 0, 0, 8833, 0, 0, 8928, 0, 0, 0, 0, 0, 0, 8929, 0, 0, 8836, 0, 0, - 8837, 0, 0, 8840, 0, 0, 0, 0, 0, 0, 8841, 0, 0, 8930, 0, 0, 8931, 0, 0, - 8876, 0, 0, 0, 0, 0, 0, 8877, 0, 0, 8878, 0, 0, 8879, 0, 0, 8938, 0, 0, - 0, 0, 0, 0, 8939, 0, 0, 8940, 0, 0, 8941, 0, 0, 0, 12436, 0, 0, 12364, 0, - 0, 0, 0, 0, 0, 12366, 0, 0, 12368, 0, 0, 12370, 0, 0, 12372, 0, 0, 0, 0, - 0, 0, 12374, 0, 0, 12376, 0, 0, 12378, 0, 0, 12380, 0, 0, 0, 0, 0, 0, - 12382, 0, 0, 12384, 0, 0, 12386, 0, 0, 12389, 0, 0, 0, 0, 0, 0, 12391, 0, - 0, 12393, 0, 0, 12400, 12401, 0, 12403, 12404, 0, 0, 0, 0, 0, 12406, - 12407, 0, 0, 0, 0, 0, 12409, 12410, 0, 12412, 12413, 0, 12446, 0, 0, 0, - 0, 0, 0, 12532, 0, 0, 12460, 0, 0, 12462, 0, 0, 12464, 0, 0, 0, 0, 0, 0, - 12466, 0, 0, 12468, 0, 0, 12470, 0, 0, 12472, 0, 0, 0, 0, 0, 0, 12474, 0, - 0, 12476, 0, 0, 12478, 0, 0, 12480, 0, 0, 0, 0, 0, 0, 12482, 0, 0, 12485, - 0, 0, 12487, 0, 0, 12489, 0, 0, 0, 0, 0, 0, 12496, 12497, 0, 0, 0, 0, 0, - 12499, 12500, 0, 12502, 12503, 0, 12505, 12506, 0, 0, 0, 0, 0, 12508, - 12509, 0, 0, 0, 0, 0, 12535, 0, 0, 12536, 0, 0, 12537, 0, 0, 0, 0, 0, 0, - 12538, 0, 0, 12542, 0, 0, 0, 0, 69786, 0, 0, 69788, 0, 0, 69803, 0, 0, 0, - 69934, 0, 0, 69935, 0, 0, 0, 70475, 70476, 0, 0, 0, 70844, 70843, 70846, - 0, 0, 0, 71098, 0, 0, 71099, 0, 0, 0, 71992, + 0, 0, 0, 0, 0, 0, 0, 8814, 0, 0, 0, 8800, 0, 0, 0, 8815, 192, 193, 194, + 195, 256, 258, 550, 196, 7842, 197, 0, 461, 512, 514, 0, 0, 0, 7840, 0, + 7680, 0, 0, 260, 0, 0, 0, 7682, 0, 0, 7684, 0, 0, 0, 0, 7686, 0, 0, 262, + 264, 0, 0, 0, 266, 0, 0, 0, 0, 268, 0, 199, 0, 0, 0, 0, 7690, 0, 0, 0, 0, + 270, 0, 7692, 0, 0, 0, 7696, 0, 7698, 0, 0, 7694, 0, 200, 201, 202, 7868, + 274, 276, 278, 203, 7866, 0, 0, 282, 516, 518, 0, 0, 0, 7864, 0, 0, 0, + 552, 280, 7704, 0, 7706, 0, 0, 0, 0, 7710, 0, 0, 500, 284, 0, 7712, 286, + 288, 0, 0, 0, 0, 486, 0, 290, 0, 0, 0, 0, 292, 0, 0, 0, 7714, 7718, 0, 0, + 0, 542, 0, 7716, 0, 0, 0, 7720, 0, 0, 7722, 0, 0, 0, 204, 205, 206, 296, + 298, 300, 304, 207, 7880, 0, 0, 463, 520, 522, 0, 0, 0, 7882, 0, 0, 0, 0, + 302, 0, 0, 7724, 0, 0, 0, 0, 308, 0, 0, 7728, 0, 0, 0, 0, 0, 488, 0, + 7730, 0, 0, 0, 310, 0, 0, 0, 0, 7732, 0, 0, 313, 0, 0, 0, 0, 0, 317, 0, + 7734, 0, 0, 0, 315, 0, 7740, 0, 0, 7738, 0, 0, 7742, 0, 0, 0, 0, 7744, 0, + 0, 7746, 0, 0, 504, 323, 0, 209, 0, 0, 7748, 0, 0, 0, 0, 327, 0, 7750, 0, + 0, 0, 325, 0, 7754, 0, 0, 7752, 0, 210, 211, 212, 213, 332, 334, 558, + 214, 7886, 0, 336, 465, 524, 526, 0, 0, 416, 7884, 0, 0, 0, 0, 490, 0, 0, + 7764, 0, 0, 0, 0, 7766, 0, 0, 340, 0, 0, 0, 0, 7768, 0, 0, 0, 0, 344, + 528, 530, 0, 0, 0, 7770, 0, 0, 0, 342, 0, 0, 0, 0, 7774, 0, 0, 346, 348, + 0, 0, 0, 7776, 0, 0, 0, 0, 352, 0, 7778, 0, 0, 536, 350, 0, 0, 0, 0, + 7786, 0, 0, 0, 0, 356, 0, 7788, 0, 0, 538, 354, 0, 7792, 0, 0, 7790, 0, + 217, 218, 219, 360, 362, 364, 0, 220, 7910, 366, 368, 467, 532, 534, 0, + 0, 431, 7908, 7794, 0, 0, 0, 370, 7798, 0, 7796, 0, 0, 0, 0, 0, 7804, 0, + 7806, 0, 0, 7808, 7810, 372, 0, 0, 0, 7814, 7812, 0, 7816, 0, 0, 0, 0, + 7818, 7820, 7922, 221, 374, 7928, 562, 0, 7822, 376, 7926, 0, 0, 0, 0, + 7924, 0, 0, 0, 377, 7824, 0, 0, 0, 379, 0, 0, 0, 0, 381, 0, 7826, 0, 0, + 0, 0, 7828, 0, 224, 225, 226, 227, 257, 259, 551, 228, 7843, 229, 0, 462, + 513, 515, 0, 0, 0, 7841, 0, 7681, 0, 0, 261, 0, 0, 0, 7683, 0, 0, 7685, + 0, 0, 0, 0, 7687, 0, 0, 263, 265, 0, 0, 0, 267, 0, 0, 0, 0, 269, 0, 231, + 0, 0, 0, 0, 7691, 0, 0, 0, 0, 271, 0, 7693, 0, 0, 0, 7697, 0, 7699, 0, 0, + 7695, 0, 232, 233, 234, 7869, 275, 277, 279, 235, 7867, 0, 0, 283, 517, + 519, 0, 0, 0, 7865, 0, 0, 0, 553, 281, 7705, 0, 7707, 0, 0, 0, 0, 7711, + 0, 0, 501, 285, 0, 7713, 287, 289, 0, 0, 0, 0, 487, 0, 291, 0, 0, 0, 0, + 293, 0, 0, 0, 7715, 7719, 0, 0, 0, 543, 0, 7717, 0, 0, 0, 7721, 0, 0, + 7723, 0, 7830, 0, 236, 237, 238, 297, 299, 301, 0, 239, 7881, 0, 0, 464, + 521, 523, 0, 0, 0, 7883, 0, 0, 0, 0, 303, 0, 0, 7725, 0, 0, 0, 0, 309, 0, + 0, 0, 0, 496, 0, 7729, 0, 0, 0, 0, 0, 489, 0, 7731, 0, 0, 0, 311, 0, 0, + 0, 0, 7733, 0, 0, 314, 0, 0, 0, 0, 0, 318, 0, 7735, 0, 0, 0, 316, 0, + 7741, 0, 0, 7739, 0, 0, 7743, 0, 0, 0, 0, 7745, 0, 0, 7747, 0, 0, 505, + 324, 0, 241, 0, 0, 7749, 0, 0, 0, 0, 328, 0, 7751, 0, 0, 0, 326, 0, 7755, + 0, 0, 7753, 0, 242, 243, 244, 245, 333, 335, 559, 246, 7887, 0, 337, 466, + 525, 527, 0, 0, 417, 7885, 0, 0, 0, 0, 491, 0, 0, 7765, 0, 0, 0, 0, 7767, + 0, 0, 341, 0, 0, 0, 0, 7769, 0, 0, 0, 0, 345, 529, 531, 0, 0, 0, 7771, 0, + 0, 0, 343, 0, 0, 0, 0, 7775, 0, 0, 347, 349, 0, 0, 0, 7777, 0, 0, 0, 0, + 353, 0, 7779, 0, 0, 537, 351, 0, 0, 0, 0, 7787, 7831, 0, 0, 0, 357, 0, + 7789, 0, 0, 539, 355, 0, 7793, 0, 0, 7791, 0, 249, 250, 251, 361, 363, + 365, 0, 252, 7911, 367, 369, 468, 533, 535, 0, 0, 432, 7909, 7795, 0, 0, + 0, 371, 7799, 0, 7797, 0, 0, 0, 0, 0, 7805, 0, 7807, 0, 0, 7809, 7811, + 373, 0, 0, 0, 7815, 7813, 0, 7832, 0, 0, 0, 7817, 0, 0, 0, 0, 7819, 7821, + 7923, 253, 375, 7929, 563, 0, 7823, 255, 7927, 7833, 0, 0, 0, 7925, 0, 0, + 0, 378, 7825, 0, 0, 0, 380, 0, 0, 0, 0, 382, 0, 7827, 0, 0, 0, 0, 7829, + 0, 8173, 901, 0, 0, 8129, 0, 0, 0, 7846, 7844, 0, 7850, 7848, 0, 0, 0, + 478, 0, 0, 0, 0, 506, 0, 0, 0, 508, 0, 0, 482, 0, 0, 0, 0, 7688, 0, 0, + 7872, 7870, 0, 7876, 7874, 0, 0, 0, 0, 7726, 0, 0, 7890, 7888, 0, 7894, + 7892, 0, 0, 0, 0, 7756, 0, 0, 556, 0, 0, 7758, 554, 0, 0, 0, 0, 510, 0, + 0, 475, 471, 0, 0, 469, 0, 0, 0, 0, 0, 0, 473, 7847, 7845, 0, 7851, 7849, + 0, 0, 0, 479, 0, 0, 0, 0, 507, 0, 0, 0, 509, 0, 0, 483, 0, 0, 0, 0, 7689, + 0, 0, 7873, 7871, 0, 7877, 7875, 0, 0, 0, 0, 7727, 0, 0, 7891, 7889, 0, + 7895, 7893, 0, 0, 0, 0, 7757, 0, 0, 557, 0, 0, 7759, 555, 0, 0, 0, 0, + 511, 0, 0, 476, 472, 0, 0, 470, 0, 0, 0, 0, 0, 0, 474, 7856, 7854, 0, + 7860, 7858, 0, 0, 0, 7857, 7855, 0, 7861, 7859, 0, 0, 0, 7700, 7702, 0, + 0, 7701, 7703, 0, 0, 7760, 7762, 0, 0, 7761, 7763, 0, 0, 0, 0, 7780, 0, + 0, 0, 7781, 0, 0, 0, 7782, 0, 0, 0, 7783, 0, 0, 7800, 0, 0, 0, 7801, 0, + 0, 0, 0, 0, 7802, 0, 0, 0, 7803, 0, 0, 7835, 0, 7900, 7898, 0, 7904, + 7902, 0, 0, 0, 0, 7906, 0, 0, 7901, 7899, 0, 7905, 7903, 0, 0, 0, 0, + 7907, 0, 0, 7914, 7912, 0, 7918, 7916, 0, 0, 0, 0, 7920, 0, 0, 7915, + 7913, 0, 7919, 7917, 0, 0, 0, 0, 7921, 0, 0, 0, 0, 0, 494, 492, 0, 0, 0, + 493, 0, 0, 0, 480, 0, 0, 0, 481, 0, 0, 0, 0, 7708, 0, 0, 0, 7709, 0, 0, + 560, 0, 0, 0, 561, 0, 0, 0, 0, 0, 0, 495, 8122, 902, 0, 0, 8121, 8120, 0, + 0, 0, 0, 7944, 7945, 0, 8124, 0, 0, 8136, 904, 0, 0, 0, 0, 7960, 7961, + 8138, 905, 0, 0, 0, 0, 7976, 7977, 0, 8140, 0, 0, 8154, 906, 0, 0, 8153, + 8152, 0, 938, 0, 0, 7992, 7993, 8184, 908, 0, 0, 0, 0, 8008, 8009, 0, 0, + 0, 8172, 8170, 910, 0, 0, 8169, 8168, 0, 939, 0, 0, 0, 8025, 8186, 911, + 0, 0, 0, 0, 8040, 8041, 0, 8188, 0, 0, 0, 8116, 0, 0, 0, 8132, 0, 0, + 8048, 940, 0, 0, 8113, 8112, 0, 0, 0, 0, 7936, 7937, 8118, 8115, 0, 0, + 8050, 941, 0, 0, 0, 0, 7952, 7953, 8052, 942, 0, 0, 0, 0, 7968, 7969, + 8134, 8131, 0, 0, 8054, 943, 0, 0, 8145, 8144, 0, 970, 0, 0, 7984, 7985, + 8150, 0, 0, 0, 8056, 972, 0, 0, 0, 0, 8000, 8001, 0, 0, 8164, 8165, 8058, + 973, 0, 0, 8161, 8160, 0, 971, 0, 0, 8016, 8017, 8166, 0, 0, 0, 8060, + 974, 0, 0, 0, 0, 8032, 8033, 8182, 8179, 0, 0, 8146, 912, 0, 0, 8151, 0, + 0, 0, 8162, 944, 0, 0, 8167, 0, 0, 0, 0, 8180, 0, 0, 0, 979, 0, 0, 0, 0, + 0, 980, 0, 0, 0, 1031, 0, 1232, 0, 1234, 0, 1027, 0, 0, 1024, 0, 0, 0, 0, + 1238, 0, 1025, 0, 1217, 0, 1244, 0, 0, 0, 1246, 1037, 0, 0, 0, 1250, + 1049, 0, 1252, 0, 1036, 0, 0, 0, 0, 0, 1254, 1262, 1038, 0, 1264, 0, 0, + 1266, 0, 0, 0, 0, 1268, 0, 0, 0, 1272, 0, 0, 0, 1260, 0, 1233, 0, 1235, + 0, 1107, 0, 0, 1104, 0, 0, 0, 0, 1239, 0, 1105, 0, 1218, 0, 1245, 0, 0, + 0, 1247, 1117, 0, 0, 0, 1251, 1081, 0, 1253, 0, 1116, 0, 0, 0, 0, 0, + 1255, 1263, 1118, 0, 1265, 0, 0, 1267, 0, 0, 0, 0, 1269, 0, 0, 0, 1273, + 0, 0, 0, 1261, 0, 0, 0, 1111, 1142, 0, 0, 0, 1143, 0, 0, 0, 0, 0, 0, + 1242, 0, 0, 0, 1243, 0, 0, 0, 1258, 0, 0, 0, 1259, 0, 0, 1570, 1571, + 1573, 0, 0, 0, 0, 0, 0, 1572, 0, 0, 0, 1574, 0, 0, 0, 1730, 0, 0, 0, + 1747, 0, 0, 0, 1728, 0, 2345, 0, 0, 0, 2353, 0, 0, 0, 2356, 0, 0, 0, 0, + 2507, 2508, 2891, 2888, 2892, 0, 2964, 0, 0, 0, 0, 0, 0, 3018, 3020, 0, + 0, 0, 0, 0, 0, 3019, 0, 3144, 0, 0, 0, 0, 0, 3264, 0, 0, 3274, 3271, + 3272, 0, 0, 0, 0, 0, 0, 3275, 0, 3402, 3404, 0, 0, 3403, 0, 0, 0, 0, 0, + 3546, 3548, 3550, 0, 0, 0, 0, 0, 3549, 0, 0, 4134, 0, 0, 0, 0, 6918, 0, + 0, 0, 6920, 0, 0, 0, 6922, 0, 0, 0, 6924, 0, 0, 0, 6926, 0, 0, 0, 6930, + 0, 0, 0, 6971, 0, 0, 0, 6973, 0, 0, 0, 6976, 0, 0, 0, 6977, 0, 0, 0, + 6979, 7736, 0, 0, 0, 7737, 0, 0, 0, 7772, 0, 0, 0, 7773, 0, 0, 0, 0, 0, + 7784, 0, 0, 0, 7785, 0, 0, 0, 7852, 0, 0, 7862, 0, 0, 0, 0, 7853, 0, 0, + 7863, 0, 0, 0, 0, 7878, 0, 0, 0, 7879, 0, 0, 0, 7896, 0, 0, 0, 7897, 0, + 7938, 7940, 0, 0, 7942, 8064, 0, 0, 7939, 7941, 0, 0, 7943, 8065, 0, 0, + 0, 8066, 0, 0, 0, 8067, 0, 0, 0, 8068, 0, 0, 0, 8069, 0, 0, 0, 8070, 0, + 0, 0, 8071, 0, 0, 7946, 7948, 0, 0, 7950, 8072, 0, 0, 7947, 7949, 0, 0, + 7951, 8073, 0, 0, 0, 8074, 0, 0, 0, 8075, 0, 0, 0, 8076, 0, 0, 0, 8077, + 0, 0, 0, 8078, 0, 0, 0, 8079, 0, 0, 7954, 7956, 0, 0, 7955, 7957, 0, 0, + 7962, 7964, 0, 0, 7963, 7965, 0, 0, 7970, 7972, 0, 0, 7974, 8080, 0, 0, + 7971, 7973, 0, 0, 7975, 8081, 0, 0, 0, 8082, 0, 0, 0, 8083, 0, 0, 0, + 8084, 0, 0, 0, 8085, 0, 0, 0, 8086, 0, 0, 0, 8087, 0, 0, 7978, 7980, 0, + 0, 7982, 8088, 0, 0, 7979, 7981, 0, 0, 7983, 8089, 0, 0, 0, 8090, 0, 0, + 0, 8091, 0, 0, 0, 8092, 0, 0, 0, 8093, 0, 0, 0, 8094, 0, 0, 0, 8095, 0, + 0, 7986, 7988, 0, 0, 7990, 0, 0, 0, 7987, 7989, 0, 0, 7991, 0, 0, 0, + 7994, 7996, 0, 0, 7998, 0, 0, 0, 7995, 7997, 0, 0, 7999, 0, 0, 0, 8002, + 8004, 0, 0, 8003, 8005, 0, 0, 8010, 8012, 0, 0, 8011, 8013, 0, 0, 8018, + 8020, 0, 0, 8022, 0, 0, 0, 8019, 8021, 0, 0, 8023, 0, 0, 0, 8027, 8029, + 0, 0, 8031, 0, 0, 0, 8034, 8036, 0, 0, 8038, 8096, 0, 0, 8035, 8037, 0, + 0, 8039, 8097, 0, 0, 0, 8098, 0, 0, 0, 8099, 0, 0, 0, 8100, 0, 0, 0, + 8101, 0, 0, 0, 8102, 0, 0, 0, 8103, 0, 0, 8042, 8044, 0, 0, 8046, 8104, + 0, 0, 8043, 8045, 0, 0, 8047, 8105, 0, 0, 0, 8106, 0, 0, 0, 8107, 0, 0, + 0, 8108, 0, 0, 0, 8109, 0, 0, 0, 8110, 0, 0, 0, 8111, 0, 0, 0, 8114, 0, + 0, 0, 8130, 0, 0, 0, 8178, 0, 0, 0, 8119, 0, 0, 8141, 8142, 0, 0, 8143, + 0, 0, 0, 0, 8135, 0, 0, 0, 8183, 0, 0, 8157, 8158, 0, 0, 8159, 0, 0, 0, + 0, 0, 0, 8602, 0, 0, 0, 8603, 0, 0, 0, 8622, 0, 0, 0, 8653, 0, 0, 0, + 8655, 0, 0, 0, 8654, 0, 0, 0, 8708, 0, 0, 0, 8713, 0, 0, 0, 8716, 0, 0, + 0, 8740, 0, 0, 0, 8742, 0, 0, 0, 8769, 0, 0, 0, 8772, 0, 0, 0, 8775, 0, + 0, 0, 8777, 0, 0, 0, 8813, 0, 0, 0, 8802, 0, 0, 0, 8816, 0, 0, 0, 8817, + 0, 0, 0, 8820, 0, 0, 0, 8821, 0, 0, 0, 8824, 0, 0, 0, 8825, 0, 0, 0, + 8832, 0, 0, 0, 8833, 0, 0, 0, 8928, 0, 0, 0, 8929, 0, 0, 0, 8836, 0, 0, + 0, 8837, 0, 0, 0, 8840, 0, 0, 0, 8841, 0, 0, 0, 8930, 0, 0, 0, 8931, 0, + 0, 0, 8876, 0, 0, 0, 8877, 0, 0, 0, 8878, 0, 0, 0, 8879, 0, 0, 0, 8938, + 0, 0, 0, 8939, 0, 0, 0, 8940, 0, 0, 0, 8941, 12436, 0, 0, 0, 12364, 0, 0, + 0, 12366, 0, 0, 0, 12368, 0, 0, 0, 12370, 0, 0, 0, 12372, 0, 0, 0, 12374, + 0, 0, 0, 12376, 0, 0, 0, 12378, 0, 0, 0, 12380, 0, 0, 0, 12382, 0, 0, 0, + 12384, 0, 0, 0, 12386, 0, 0, 0, 12389, 0, 0, 0, 12391, 0, 0, 0, 12393, 0, + 0, 0, 12400, 12401, 0, 0, 12403, 12404, 0, 0, 12406, 12407, 0, 0, 12409, + 12410, 0, 0, 12412, 12413, 0, 0, 12446, 0, 0, 0, 12532, 0, 0, 0, 12460, + 0, 0, 0, 12462, 0, 0, 0, 12464, 0, 0, 0, 12466, 0, 0, 0, 12468, 0, 0, 0, + 12470, 0, 0, 0, 12472, 0, 0, 0, 12474, 0, 0, 0, 12476, 0, 0, 0, 12478, 0, + 0, 0, 12480, 0, 0, 0, 12482, 0, 0, 0, 12485, 0, 0, 0, 12487, 0, 0, 0, + 12489, 0, 0, 0, 12496, 12497, 0, 0, 12499, 12500, 0, 0, 12502, 12503, 0, + 0, 12505, 12506, 0, 0, 12508, 12509, 0, 0, 12535, 0, 0, 0, 12536, 0, 0, + 0, 12537, 0, 0, 0, 12538, 0, 0, 0, 12542, 0, 0, 0, 0, 0, 67017, 0, 0, 0, + 67044, 0, 0, 0, 69786, 0, 0, 0, 69788, 0, 0, 0, 69803, 0, 0, 0, 0, 69934, + 0, 0, 0, 69935, 70475, 70476, 0, 0, 0, 70531, 0, 0, 0, 0, 0, 70533, + 70542, 0, 0, 0, 0, 70545, 0, 0, 0, 0, 70599, 0, 70597, 70600, 0, 0, 0, 0, + 70844, 70843, 70846, 0, 0, 0, 0, 71098, 0, 0, 0, 71099, 0, 0, 0, 0, + 71992, 0, 0, 0, 0, 90401, 90403, 90405, 90402, 0, 90406, 90408, 0, 0, + 90407, 0, 0, 0, 90404, 0, 0, 0, 0, 0, 0, 93545, 0, 0, 0, 93544, 0, 0, 0, + 93546, }; static const change_record change_records_3_2_0[] = { @@ -6039,6 +6199,7 @@ static const change_record change_records_3_2_0[] = { { 255, 255, 255, 255, 255, 1000000000000.0 }, { 255, 255, 255, 255, 255, 1e+20 }, { 255, 19, 255, 255, 255, -1 }, + { 1, 255, 255, 255, 255, 0 }, { 1, 255, 255, 0, 255, 0 }, }; static const unsigned char changes_3_2_0_index[] = { @@ -6056,31 +6217,31 @@ static const unsigned char changes_3_2_0_index[] = { 1, 1, 1, 1, 78, 79, 80, 1, 81, 82, 83, 84, 85, 86, 87, 88, 89, 28, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 28, 28, 28, 115, 116, 117, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 118, 28, 28, 28, 28, 119, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 28, 28, 120, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 28, 28, 121, - 122, 1, 1, 123, 124, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 125, 28, 28, 28, 28, 126, 127, 1, + 1, 1, 1, 1, 1, 1, 1, 118, 28, 28, 28, 28, 119, 28, 28, 28, 28, 28, 28, + 28, 28, 28, 28, 28, 28, 28, 28, 120, 28, 28, 121, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 122, 1, 1, 1, 1, 1, + 1, 28, 28, 123, 124, 1, 125, 126, 127, 28, 28, 28, 28, 28, 28, 28, 28, + 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 128, 28, 28, + 28, 28, 129, 130, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 131, 28, 132, 133, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 134, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 135, + 28, 136, 137, 1, 138, 139, 140, 141, 1, 142, 143, 28, 28, 144, 1, 1, 1, + 1, 145, 146, 147, 148, 1, 149, 150, 1, 151, 152, 153, 1, 1, 154, 155, + 156, 1, 157, 158, 159, 28, 28, 28, 160, 161, 162, 28, 163, 164, 1, 1, 1, + 1, 165, 67, 1, 1, 1, 1, 1, 1, 1, 166, 167, 168, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 169, 1, 1, 1, 1, 1, 170, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 171, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 128, 28, 129, 130, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 131, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 132, 1, 133, - 134, 135, 136, 1, 137, 138, 28, 28, 139, 1, 1, 1, 1, 140, 141, 142, 143, - 1, 144, 1, 1, 145, 146, 147, 1, 1, 148, 149, 150, 1, 151, 152, 153, 28, - 28, 28, 154, 155, 156, 28, 157, 158, 1, 1, 1, 1, 159, 67, 1, 1, 1, 1, 1, - 1, 1, 160, 161, 162, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 163, 1, 1, 1, 1, 1, 164, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 165, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 166, 1, + 1, 1, 1, 1, 1, 1, 1, 172, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 167, 28, 28, 28, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 168, 169, 28, 28, 28, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, - 170, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 171, 28, 28, 43, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 163, 1, 1, 1, 1, 1, 1, 1, 28, 28, 28, 28, 28, 28, 28, 28, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 172, 28, 28, 28, 28, 28, 28, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 173, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 173, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, + 28, 174, 175, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, + 28, 28, 28, 28, 28, 28, 176, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, + 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 177, + 28, 28, 43, 1, 1, 1, 1, 1, 1, 1, 1, 1, 169, 1, 1, 1, 1, 1, 1, 1, 28, 28, + 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 178, + 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 179, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -6196,7 +6357,7 @@ static const unsigned char changes_3_2_0_index[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 174, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 180, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -6228,7 +6389,7 @@ static const unsigned char changes_3_2_0_index[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, }; static const unsigned char changes_3_2_0_data[] = { @@ -6323,7 +6484,7 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -6514,9 +6675,9 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -6527,7 +6688,7 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, @@ -6580,7 +6741,7 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6598,7 +6759,7 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, @@ -6620,27 +6781,27 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 0, 0, - 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 9, 0, - 0, 0, 0, 9, 9, 9, 0, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, + 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, + 9, 0, 0, 0, 0, 9, 9, 9, 0, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, @@ -6654,24 +6815,24 @@ static const unsigned char changes_3_2_0_data[] = { 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, - 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -6686,22 +6847,22 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 9, 0, 0, 0, 0, 0, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 9, 0, 0, 0, 0, 0, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, - 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, - 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, + 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, + 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6718,9 +6879,9 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6729,51 +6890,51 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6785,7 +6946,7 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6800,20 +6961,20 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, + 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6824,8 +6985,8 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6838,7 +6999,7 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6849,12 +7010,12 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6863,7 +7024,7 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6872,13 +7033,13 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6887,10 +7048,10 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6906,7 +7067,7 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6920,7 +7081,7 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6928,11 +7089,11 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6949,12 +7110,12 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6970,12 +7131,12 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6993,12 +7154,12 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -7006,7 +7167,7 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -7015,51 +7176,51 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 0, 0, 0, 9, 9, 0, 9, 0, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 9, 9, 0, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, + 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, - 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, + 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7067,208 +7228,208 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 19, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 19, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 19, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 49, 50, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 0, 0, 0, 49, 50, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, - 21, 21, 21, 21, 21, 0, 0, 0, 1, 1, 21, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 21, 21, 21, 21, 21, 21, 0, 0, 0, 1, 1, 21, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 14, 14, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 14, 14, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, - 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, + 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, - 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, + 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 9, 0, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, 0, 9, 0, 0, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, 0, 9, 0, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, - 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, + 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7277,102 +7438,102 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 0, 0, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 0, 0, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 0, 9, 9, 9, 9, 0, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, - 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 0, 0, 9, 9, 9, 0, - 0, 9, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 0, 9, 9, 9, 9, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, + 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 0, 0, 9, 9, 9, + 0, 0, 9, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 0, 0, 9, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 0, 0, 9, + 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7380,41 +7541,39 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 0, 0, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7422,25 +7581,27 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 0, 9, 9, 0, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 0, 9, 9, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7453,35 +7614,35 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -7489,7 +7650,7 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7503,25 +7664,46 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7529,50 +7711,61 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, - 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -7582,7 +7775,7 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -7591,9 +7784,9 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7614,12 +7807,12 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -7635,60 +7828,77 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7697,43 +7907,48 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, - 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, - 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7742,21 +7957,21 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, - 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7769,12 +7984,22 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7783,9 +8008,9 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7794,9 +8019,9 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 0, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 0, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -7804,13 +8029,13 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7822,15 +8047,15 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7838,50 +8063,50 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 9, 9, 0, 9, 0, 0, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, - 9, 0, 9, 0, 9, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 9, 0, 9, 0, 9, 0, 9, 9, - 9, 0, 9, 9, 0, 9, 0, 0, 9, 0, 9, 0, 9, 0, 9, 0, 9, 0, 9, 9, 0, 9, 0, 0, - 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 0, 9, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 0, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 0, 9, 9, 0, 9, 0, 0, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, + 9, 9, 0, 9, 0, 9, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 9, 0, 9, 0, 9, 0, 9, + 9, 9, 0, 9, 9, 0, 9, 0, 0, 9, 0, 9, 0, 9, 0, 9, 0, 9, 0, 9, 9, 0, 9, 0, + 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 0, + 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 0, 9, 9, 9, 9, 9, 0, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -7890,68 +8115,68 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, - 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, + 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7965,9 +8190,9 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7983,7 +8208,7 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7994,12 +8219,12 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -8013,7 +8238,7 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -8029,11 +8254,11 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -8042,7 +8267,7 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -8058,8 +8283,8 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -8072,11 +8297,11 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -8091,10 +8316,10 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -8104,8 +8329,8 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; static const change_record* get_change_3_2_0(Py_UCS4 n) diff --git a/Modules/unicodename_db.h b/Modules/unicodename_db.h index 4fe60ad5ed4452..0697e259b39019 100644 --- a/Modules/unicodename_db.h +++ b/Modules/unicodename_db.h @@ -1,8936 +1,9227 @@ -/* this file was generated by ./Tools/unicode/makeunicodedata.py 3.3 */ +/* this file was generated by Tools/unicode/makeunicodedata.py 3.3 */ #define NAME_MAXLEN 256 /* name->code dictionary */ static const unsigned char packed_name_dawg[] = { - 228, 174, 4, 250, 2, 65, 206, 254, 2, 66, 222, 254, 2, 67, 198, 245, 4, - 68, 222, 134, 1, 69, 250, 151, 1, 70, 206, 52, 71, 186, 154, 1, 72, 242, - 223, 1, 73, 146, 65, 74, 138, 23, 75, 230, 134, 1, 76, 146, 130, 3, 77, - 162, 200, 3, 78, 162, 107, 79, 226, 124, 80, 128, 159, 1, 2, 81, 85, 194, - 7, 82, 182, 169, 1, 83, 166, 175, 4, 84, 170, 203, 2, 85, 222, 90, 86, - 182, 80, 87, 182, 111, 88, 254, 4, 89, 179, 48, 90, 182, 40, 222, 2, 67, - 238, 1, 68, 182, 10, 69, 172, 5, 4, 72, 79, 77, 32, 244, 6, 7, 73, 82, - 80, 76, 65, 78, 69, 82, 76, 210, 32, 77, 198, 1, 78, 142, 26, 80, 142, - 20, 82, 158, 136, 2, 83, 210, 2, 84, 70, 85, 214, 1, 86, 172, 213, 11, 4, - 70, 71, 72, 65, 206, 192, 9, 66, 220, 219, 2, 9, 75, 84, 73, 69, 83, 69, - 76, 83, 75, 164, 187, 2, 2, 81, 85, 199, 135, 10, 88, 18, 132, 1, 2, 67, - 79, 46, 75, 20, 5, 85, 84, 69, 32, 65, 140, 230, 16, 6, 84, 73, 86, 65, - 84, 69, 225, 150, 22, 5, 32, 67, 85, 82, 82, 4, 162, 185, 25, 85, 237, - 182, 13, 2, 82, 68, 5, 147, 175, 33, 78, 4, 178, 207, 34, 67, 167, 207, - 3, 78, 188, 1, 232, 1, 4, 76, 65, 77, 32, 242, 7, 77, 216, 141, 13, 7, - 72, 69, 83, 73, 86, 69, 32, 168, 151, 4, 19, 68, 82, 69, 83, 83, 69, 68, - 32, 84, 79, 32, 84, 72, 69, 32, 83, 85, 66, 74, 234, 178, 13, 85, 161, - 186, 1, 6, 73, 32, 83, 72, 65, 75, 176, 1, 218, 1, 67, 44, 4, 83, 77, 65, - 76, 168, 4, 7, 71, 69, 77, 73, 78, 65, 84, 116, 8, 73, 78, 73, 84, 73, - 65, 76, 32, 38, 78, 178, 209, 2, 72, 220, 244, 22, 4, 65, 76, 73, 70, 0, - 5, 86, 79, 87, 69, 76, 131, 130, 12, 68, 70, 40, 5, 65, 80, 73, 84, 65, - 215, 4, 79, 68, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 68, 142, 2, - 68, 38, 71, 34, 74, 2, 77, 22, 75, 46, 78, 42, 83, 210, 158, 29, 81, 190, - 205, 4, 76, 130, 40, 67, 202, 154, 4, 90, 212, 7, 2, 65, 76, 170, 2, 66, - 2, 89, 146, 1, 87, 182, 89, 84, 150, 14, 80, 158, 20, 70, 2, 72, 2, 82, - 2, 86, 186, 2, 69, 2, 73, 2, 79, 3, 85, 4, 250, 186, 5, 65, 247, 251, 33, - 72, 4, 174, 162, 39, 66, 215, 22, 65, 2, 171, 182, 38, 73, 6, 194, 183, - 38, 65, 154, 98, 80, 191, 28, 72, 6, 206, 232, 38, 85, 158, 77, 72, 3, - 89, 4, 188, 181, 2, 6, 73, 78, 78, 89, 73, 73, 135, 128, 37, 72, 4, 40, - 4, 69, 32, 67, 79, 151, 206, 38, 73, 2, 173, 174, 38, 13, 78, 83, 79, 78, - 65, 78, 84, 32, 77, 79, 68, 73, 70, 4, 218, 222, 32, 69, 191, 141, 4, 81, - 4, 166, 151, 25, 65, 199, 185, 10, 85, 4, 182, 215, 26, 69, 185, 202, 11, - 12, 73, 83, 83, 73, 79, 78, 32, 84, 73, 67, 75, 69, 116, 80, 5, 71, 69, - 65, 78, 32, 249, 144, 33, 9, 82, 73, 65, 76, 32, 84, 82, 65, 77, 114, - 136, 1, 3, 68, 82, 89, 0, 6, 76, 73, 81, 85, 73, 68, 60, 8, 77, 69, 65, - 83, 85, 82, 69, 32, 26, 87, 130, 185, 26, 78, 147, 185, 11, 67, 2, 153, - 2, 11, 32, 77, 69, 65, 83, 85, 82, 69, 32, 70, 73, 4, 246, 1, 83, 31, 84, - 14, 100, 6, 69, 73, 71, 72, 84, 32, 241, 1, 14, 79, 82, 68, 32, 83, 69, - 80, 65, 82, 65, 84, 79, 82, 32, 10, 54, 70, 66, 83, 30, 84, 65, 5, 66, - 65, 83, 69, 32, 4, 38, 73, 89, 5, 79, 85, 82, 84, 72, 2, 85, 3, 82, 83, - 84, 2, 49, 4, 69, 67, 79, 78, 2, 21, 3, 72, 73, 82, 2, 11, 68, 2, 25, 4, - 32, 83, 85, 66, 2, 133, 148, 39, 2, 85, 78, 4, 130, 151, 38, 76, 247, 72, - 68, 130, 1, 140, 2, 22, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, - 71, 78, 32, 77, 69, 68, 73, 65, 76, 32, 88, 7, 76, 69, 84, 84, 69, 82, - 32, 242, 1, 83, 168, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, - 140, 234, 31, 8, 78, 85, 77, 66, 69, 82, 32, 84, 219, 209, 5, 68, 6, 26, - 76, 159, 171, 39, 82, 4, 132, 227, 38, 7, 73, 71, 65, 84, 73, 78, 71, - 207, 74, 65, 68, 154, 1, 65, 210, 195, 35, 68, 46, 84, 222, 225, 1, 76, - 246, 189, 1, 78, 126, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, - 2, 77, 2, 82, 3, 83, 9, 45, 9, 76, 84, 69, 82, 78, 65, 84, 69, 32, 6, - 146, 169, 39, 66, 2, 71, 3, 84, 10, 60, 4, 73, 71, 78, 32, 241, 193, 22, - 5, 89, 77, 66, 79, 76, 8, 50, 83, 182, 130, 25, 75, 145, 233, 9, 2, 82, - 85, 4, 148, 183, 36, 4, 77, 65, 76, 76, 207, 162, 2, 69, 22, 66, 65, 162, - 218, 35, 85, 210, 200, 1, 73, 206, 134, 2, 69, 3, 79, 11, 186, 169, 39, - 65, 2, 73, 2, 77, 3, 87, 7, 11, 32, 4, 252, 190, 34, 3, 68, 69, 80, 141, - 153, 4, 5, 65, 82, 82, 73, 86, 130, 2, 232, 1, 20, 67, 72, 69, 77, 73, - 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 166, 28, 69, - 60, 2, 76, 32, 82, 77, 108, 6, 84, 69, 82, 78, 65, 84, 208, 179, 28, 7, - 73, 69, 78, 32, 77, 79, 78, 225, 230, 6, 3, 65, 82, 77, 232, 1, 158, 2, - 65, 246, 2, 66, 210, 1, 67, 138, 3, 68, 98, 71, 38, 72, 132, 1, 4, 73, - 82, 79, 78, 82, 76, 74, 77, 144, 1, 2, 78, 73, 34, 80, 176, 2, 3, 81, 85, - 73, 74, 82, 146, 2, 83, 210, 5, 84, 166, 1, 86, 200, 1, 2, 87, 65, 214, - 245, 33, 85, 190, 206, 1, 69, 202, 153, 1, 79, 195, 27, 70, 30, 194, 1, - 76, 72, 3, 81, 85, 65, 204, 8, 7, 78, 84, 73, 77, 79, 78, 89, 148, 137, - 3, 3, 77, 65, 76, 176, 222, 8, 5, 85, 82, 73, 80, 73, 188, 129, 12, 3, - 82, 83, 69, 234, 156, 14, 83, 159, 91, 73, 8, 224, 24, 2, 69, 77, 252, - 209, 25, 4, 75, 65, 76, 73, 179, 167, 13, 85, 10, 38, 32, 149, 139, 24, - 3, 70, 79, 82, 8, 216, 7, 4, 86, 73, 84, 65, 225, 194, 22, 4, 82, 69, 71, - 73, 16, 148, 1, 7, 65, 84, 72, 32, 79, 70, 32, 204, 6, 6, 73, 83, 77, 85, - 84, 72, 144, 1, 4, 79, 82, 65, 88, 164, 1, 4, 76, 65, 67, 75, 203, 232, - 37, 82, 4, 230, 182, 16, 77, 253, 218, 21, 5, 86, 65, 80, 79, 85, 28, 82, - 65, 92, 6, 79, 80, 80, 69, 82, 32, 34, 82, 217, 206, 36, 4, 73, 78, 78, - 65, 6, 216, 218, 5, 2, 68, 85, 196, 156, 32, 9, 80, 85, 84, 32, 77, 79, - 82, 84, 85, 235, 27, 76, 4, 230, 13, 65, 179, 240, 37, 79, 16, 72, 8, 79, - 67, 85, 83, 32, 79, 70, 32, 53, 6, 85, 67, 73, 66, 76, 69, 6, 228, 16, 5, - 67, 79, 80, 80, 69, 219, 232, 37, 73, 11, 11, 45, 8, 138, 157, 39, 50, 2, - 51, 2, 52, 3, 53, 8, 44, 2, 73, 83, 185, 133, 5, 3, 65, 89, 45, 6, 144, - 2, 4, 83, 79, 76, 86, 203, 169, 36, 84, 4, 234, 182, 29, 79, 191, 212, 9, - 85, 8, 32, 4, 65, 76, 70, 32, 47, 79, 4, 240, 191, 34, 2, 79, 85, 255, - 161, 3, 68, 4, 132, 245, 2, 4, 82, 83, 69, 32, 183, 237, 35, 85, 6, 56, - 3, 32, 79, 82, 73, 7, 45, 67, 79, 80, 80, 69, 82, 4, 147, 226, 25, 69, 4, - 48, 3, 69, 65, 68, 221, 158, 13, 3, 79, 68, 69, 2, 135, 225, 34, 32, 10, - 120, 16, 69, 82, 67, 85, 82, 89, 32, 83, 85, 66, 76, 73, 77, 65, 84, 69, - 176, 133, 16, 4, 65, 82, 67, 65, 135, 148, 21, 79, 7, 207, 151, 39, 45, - 4, 142, 248, 37, 84, 235, 117, 71, 14, 108, 11, 72, 73, 76, 79, 83, 79, - 80, 72, 69, 82, 83, 34, 79, 98, 85, 181, 155, 29, 6, 82, 69, 67, 73, 80, - 73, 2, 209, 9, 4, 32, 83, 85, 76, 6, 52, 4, 87, 68, 69, 82, 161, 145, 35, - 3, 84, 32, 65, 5, 237, 231, 37, 5, 69, 68, 32, 66, 82, 4, 180, 143, 33, - 4, 84, 82, 69, 70, 161, 245, 5, 3, 82, 73, 70, 4, 132, 185, 34, 5, 78, - 84, 69, 83, 83, 249, 196, 3, 4, 67, 75, 32, 76, 24, 58, 69, 213, 220, 25, - 8, 79, 67, 75, 32, 83, 65, 76, 84, 20, 68, 5, 71, 85, 76, 85, 83, 164, 7, - 3, 65, 76, 71, 179, 172, 25, 84, 15, 32, 4, 32, 79, 70, 32, 71, 45, 6, - 228, 219, 25, 8, 65, 78, 84, 73, 77, 79, 78, 89, 255, 147, 12, 73, 6, - 170, 147, 39, 50, 2, 51, 3, 52, 34, 164, 1, 2, 65, 76, 194, 1, 84, 142, - 1, 85, 184, 139, 17, 2, 80, 73, 200, 255, 11, 2, 73, 76, 194, 243, 8, 79, - 133, 20, 11, 67, 69, 80, 84, 69, 82, 32, 79, 70, 32, 74, 8, 58, 84, 197, - 234, 37, 8, 45, 65, 77, 77, 79, 78, 73, 65, 7, 25, 4, 32, 79, 70, 32, 4, - 52, 8, 67, 79, 80, 80, 69, 82, 32, 65, 231, 5, 65, 2, 237, 149, 29, 7, - 78, 84, 73, 77, 79, 78, 73, 6, 236, 3, 8, 65, 82, 82, 69, 68, 32, 84, 82, - 169, 212, 25, 19, 82, 65, 84, 85, 77, 32, 83, 85, 80, 69, 82, 32, 83, 84, + 238, 255, 4, 254, 2, 65, 218, 130, 3, 66, 130, 154, 3, 67, 174, 245, 4, + 68, 206, 135, 1, 69, 186, 157, 1, 70, 190, 54, 71, 210, 164, 1, 72, 182, + 225, 1, 73, 222, 65, 74, 138, 23, 75, 238, 138, 1, 76, 130, 158, 3, 77, + 250, 200, 3, 78, 242, 107, 79, 138, 131, 1, 80, 128, 159, 1, 2, 81, 85, + 226, 7, 82, 194, 180, 1, 83, 214, 193, 4, 84, 134, 225, 2, 85, 242, 104, + 86, 138, 84, 87, 170, 114, 88, 254, 4, 89, 187, 48, 90, 214, 40, 222, 2, + 67, 238, 1, 68, 182, 10, 69, 172, 5, 4, 72, 79, 77, 32, 244, 6, 7, 73, + 82, 80, 76, 65, 78, 69, 82, 76, 146, 36, 77, 198, 1, 78, 154, 26, 80, + 138, 20, 82, 226, 136, 2, 83, 210, 2, 84, 70, 85, 214, 1, 86, 144, 254, + 11, 4, 70, 71, 72, 65, 186, 231, 9, 66, 180, 227, 2, 9, 75, 84, 73, 69, + 83, 69, 76, 83, 75, 196, 198, 2, 2, 81, 85, 199, 196, 10, 88, 18, 132, 1, + 2, 67, 79, 46, 75, 20, 5, 85, 84, 69, 32, 65, 164, 153, 17, 6, 84, 73, + 86, 65, 84, 69, 145, 135, 23, 5, 32, 67, 85, 82, 82, 4, 170, 142, 26, 85, + 173, 133, 14, 2, 82, 68, 5, 151, 168, 34, 78, 4, 154, 199, 35, 67, 143, + 248, 3, 78, 188, 1, 232, 1, 4, 76, 65, 77, 32, 242, 7, 77, 212, 179, 13, + 7, 72, 69, 83, 73, 86, 69, 32, 144, 165, 4, 19, 68, 82, 69, 83, 83, 69, + 68, 32, 84, 79, 32, 84, 72, 69, 32, 83, 85, 66, 74, 146, 233, 13, 85, + 237, 185, 1, 6, 73, 32, 83, 72, 65, 75, 176, 1, 218, 1, 67, 44, 4, 83, + 77, 65, 76, 168, 4, 7, 71, 69, 77, 73, 78, 65, 84, 116, 8, 73, 78, 73, + 84, 73, 65, 76, 32, 38, 78, 198, 213, 2, 72, 200, 197, 23, 4, 65, 76, 73, + 70, 0, 5, 86, 79, 87, 69, 76, 211, 205, 12, 68, 70, 40, 5, 65, 80, 73, + 84, 65, 215, 4, 79, 68, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 68, + 142, 2, 68, 38, 71, 34, 74, 2, 77, 22, 75, 46, 78, 42, 83, 166, 133, 30, + 81, 238, 223, 4, 76, 194, 39, 67, 194, 197, 4, 90, 212, 7, 2, 65, 76, + 170, 2, 66, 2, 89, 154, 1, 87, 198, 89, 84, 150, 14, 80, 158, 20, 70, 2, + 72, 2, 82, 2, 86, 186, 2, 69, 2, 73, 2, 79, 3, 85, 4, 170, 218, 5, 65, + 155, 128, 35, 72, 4, 130, 198, 40, 66, 215, 22, 65, 2, 231, 217, 39, 73, + 6, 254, 218, 39, 65, 178, 98, 80, 191, 28, 72, 6, 150, 140, 40, 85, 170, + 77, 72, 3, 89, 4, 212, 185, 2, 6, 73, 78, 78, 89, 73, 73, 195, 159, 38, + 72, 4, 40, 4, 69, 32, 67, 79, 219, 241, 39, 73, 2, 233, 209, 39, 13, 78, + 83, 79, 78, 65, 78, 84, 32, 77, 79, 68, 73, 70, 4, 214, 204, 33, 69, 183, + 188, 4, 81, 4, 174, 236, 25, 65, 251, 217, 10, 85, 4, 238, 178, 27, 69, + 189, 146, 12, 12, 73, 83, 83, 73, 79, 78, 32, 84, 73, 67, 75, 69, 116, + 80, 5, 71, 69, 65, 78, 32, 241, 135, 34, 9, 82, 73, 65, 76, 32, 84, 82, + 65, 77, 114, 136, 1, 3, 68, 82, 89, 0, 6, 76, 73, 81, 85, 73, 68, 60, 8, + 77, 69, 65, 83, 85, 82, 69, 32, 26, 87, 230, 147, 27, 78, 183, 255, 11, + 67, 2, 153, 2, 11, 32, 77, 69, 65, 83, 85, 82, 69, 32, 70, 73, 4, 246, 1, + 83, 31, 84, 14, 100, 6, 69, 73, 71, 72, 84, 32, 241, 1, 14, 79, 82, 68, + 32, 83, 69, 80, 65, 82, 65, 84, 79, 82, 32, 10, 54, 70, 66, 83, 30, 84, + 65, 5, 66, 65, 83, 69, 32, 4, 38, 73, 89, 5, 79, 85, 82, 84, 72, 2, 85, + 3, 82, 83, 84, 2, 49, 4, 69, 67, 79, 78, 2, 21, 3, 72, 73, 82, 2, 11, 68, + 2, 25, 4, 32, 83, 85, 66, 2, 217, 183, 40, 2, 85, 78, 4, 190, 186, 39, + 76, 131, 73, 68, 130, 1, 140, 2, 22, 67, 79, 78, 83, 79, 78, 65, 78, 84, + 32, 83, 73, 71, 78, 32, 77, 69, 68, 73, 65, 76, 32, 88, 7, 76, 69, 84, + 84, 69, 82, 32, 242, 1, 83, 168, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, + 71, 78, 32, 140, 212, 32, 8, 78, 85, 77, 66, 69, 82, 32, 84, 171, 136, 6, + 68, 6, 26, 76, 243, 206, 40, 82, 4, 204, 134, 40, 7, 73, 71, 65, 84, 73, + 78, 71, 219, 74, 65, 68, 154, 1, 65, 214, 245, 36, 68, 114, 84, 218, 207, + 1, 76, 250, 192, 1, 78, 126, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, + 69, 72, 2, 77, 2, 82, 3, 83, 9, 45, 9, 76, 84, 69, 82, 78, 65, 84, 69, + 32, 6, 230, 204, 40, 66, 2, 71, 3, 84, 10, 60, 4, 73, 71, 78, 32, 153, + 149, 23, 5, 89, 77, 66, 79, 76, 8, 50, 83, 170, 215, 25, 75, 229, 139, + 10, 2, 82, 85, 4, 180, 207, 37, 4, 77, 65, 76, 76, 247, 173, 2, 69, 22, + 66, 65, 234, 249, 36, 85, 206, 201, 1, 73, 222, 137, 2, 69, 3, 79, 11, + 142, 205, 40, 65, 2, 73, 2, 77, 3, 87, 7, 11, 32, 4, 192, 182, 35, 3, 68, + 69, 80, 145, 197, 4, 5, 65, 82, 82, 73, 86, 150, 2, 216, 1, 20, 67, 72, + 69, 77, 73, 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, + 166, 28, 69, 60, 4, 73, 69, 78, 32, 208, 3, 2, 76, 32, 82, 77, 108, 6, + 84, 69, 82, 78, 65, 84, 141, 142, 36, 3, 65, 82, 77, 232, 1, 158, 2, 65, + 246, 2, 66, 210, 1, 67, 138, 3, 68, 98, 71, 38, 72, 132, 1, 4, 73, 82, + 79, 78, 82, 76, 74, 77, 144, 1, 2, 78, 73, 34, 80, 176, 2, 3, 81, 85, 73, + 74, 82, 146, 2, 83, 210, 5, 84, 166, 1, 86, 200, 1, 2, 87, 65, 170, 238, + 34, 85, 154, 215, 1, 69, 158, 181, 1, 79, 139, 31, 70, 30, 194, 1, 76, + 72, 3, 81, 85, 65, 204, 8, 7, 78, 84, 73, 77, 79, 78, 89, 236, 142, 3, 3, + 77, 65, 76, 128, 174, 21, 3, 82, 83, 69, 248, 229, 13, 5, 85, 82, 73, 80, + 73, 210, 134, 1, 83, 171, 91, 73, 8, 224, 24, 2, 69, 77, 212, 171, 26, 4, + 75, 65, 76, 73, 191, 241, 13, 85, 10, 38, 32, 129, 223, 24, 3, 70, 79, + 82, 8, 216, 7, 4, 86, 73, 84, 65, 145, 150, 23, 4, 82, 69, 71, 73, 16, + 148, 1, 7, 65, 84, 72, 32, 79, 70, 32, 204, 6, 6, 73, 83, 77, 85, 84, 72, + 144, 1, 4, 79, 82, 65, 88, 164, 1, 4, 76, 65, 67, 75, 139, 139, 39, 82, + 4, 138, 234, 16, 77, 165, 203, 22, 5, 86, 65, 80, 79, 85, 28, 82, 65, 92, + 6, 79, 80, 80, 69, 82, 32, 34, 82, 185, 160, 39, 4, 73, 78, 78, 65, 6, + 152, 250, 5, 2, 68, 85, 204, 159, 33, 9, 80, 85, 84, 32, 77, 79, 82, 84, + 85, 239, 28, 76, 4, 230, 13, 65, 255, 147, 39, 79, 16, 72, 8, 79, 67, 85, + 83, 32, 79, 70, 32, 53, 6, 85, 67, 73, 66, 76, 69, 6, 228, 16, 5, 67, 79, + 80, 80, 69, 171, 139, 39, 73, 11, 11, 45, 8, 238, 192, 40, 50, 2, 51, 2, + 52, 3, 53, 8, 44, 2, 73, 83, 129, 198, 37, 3, 65, 89, 45, 6, 144, 2, 4, + 83, 79, 76, 86, 255, 197, 37, 84, 4, 218, 161, 30, 79, 179, 141, 10, 85, + 8, 32, 4, 65, 76, 70, 32, 47, 79, 4, 200, 183, 35, 2, 79, 85, 203, 204, + 3, 68, 4, 212, 250, 2, 4, 82, 83, 69, 32, 191, 139, 37, 85, 6, 56, 3, 32, + 79, 82, 73, 7, 45, 67, 79, 80, 80, 69, 82, 4, 235, 187, 26, 69, 4, 48, 3, + 69, 65, 68, 229, 204, 13, 3, 79, 68, 69, 2, 223, 216, 35, 32, 10, 120, + 16, 69, 82, 67, 85, 82, 89, 32, 83, 85, 66, 76, 73, 77, 65, 84, 69, 196, + 183, 16, 4, 65, 82, 67, 65, 203, 130, 22, 79, 7, 179, 187, 40, 45, 4, + 218, 155, 39, 84, 131, 118, 71, 14, 108, 11, 72, 73, 76, 79, 83, 79, 80, + 72, 69, 82, 83, 34, 79, 98, 85, 173, 134, 30, 6, 82, 69, 67, 73, 80, 73, + 2, 209, 9, 4, 32, 83, 85, 76, 6, 52, 4, 87, 68, 69, 82, 205, 136, 36, 3, + 84, 32, 65, 5, 173, 138, 39, 5, 69, 68, 32, 66, 82, 4, 200, 136, 34, 4, + 84, 82, 69, 70, 241, 159, 6, 3, 82, 73, 70, 4, 220, 176, 35, 5, 78, 84, + 69, 83, 83, 237, 240, 3, 4, 67, 75, 32, 76, 24, 58, 69, 173, 182, 26, 8, + 79, 67, 75, 32, 83, 65, 76, 84, 20, 68, 5, 71, 85, 76, 85, 83, 164, 7, 3, + 65, 76, 71, 195, 129, 26, 84, 15, 32, 4, 32, 79, 70, 32, 71, 45, 6, 188, + 181, 26, 8, 65, 78, 84, 73, 77, 79, 78, 89, 247, 220, 12, 73, 6, 142, + 183, 40, 50, 2, 51, 3, 52, 34, 164, 1, 2, 65, 76, 194, 1, 84, 142, 1, 85, + 192, 191, 17, 2, 80, 73, 164, 178, 12, 2, 73, 76, 170, 176, 9, 79, 141, + 20, 11, 67, 69, 80, 84, 69, 82, 32, 79, 70, 32, 74, 8, 58, 84, 141, 141, + 39, 8, 45, 65, 77, 77, 79, 78, 73, 65, 7, 25, 4, 32, 79, 70, 32, 4, 52, + 8, 67, 79, 80, 80, 69, 82, 32, 65, 231, 5, 65, 2, 229, 128, 30, 7, 78, + 84, 73, 77, 79, 78, 73, 6, 236, 3, 8, 65, 82, 82, 69, 68, 32, 84, 82, + 129, 174, 26, 19, 82, 65, 84, 85, 77, 32, 83, 85, 80, 69, 82, 32, 83, 84, 82, 65, 84, 85, 77, 12, 44, 6, 66, 76, 73, 77, 65, 84, 155, 1, 76, 10, - 44, 5, 69, 32, 79, 70, 32, 215, 190, 38, 73, 8, 68, 8, 83, 65, 76, 84, - 32, 79, 70, 32, 130, 3, 65, 175, 209, 26, 67, 4, 254, 2, 65, 175, 209, - 26, 67, 2, 135, 149, 38, 70, 12, 68, 3, 65, 82, 84, 32, 2, 73, 78, 34, - 82, 189, 162, 37, 2, 85, 84, 4, 11, 65, 4, 219, 212, 25, 82, 4, 222, 162, - 34, 67, 167, 49, 32, 2, 145, 201, 38, 2, 73, 68, 14, 50, 73, 245, 238, - 36, 6, 69, 82, 68, 73, 71, 82, 12, 64, 5, 78, 69, 71, 65, 82, 149, 211, - 25, 5, 84, 82, 73, 79, 76, 9, 44, 5, 32, 79, 70, 32, 65, 251, 137, 39, - 45, 2, 221, 188, 23, 3, 78, 84, 73, 4, 234, 204, 38, 84, 227, 61, 88, 6, - 38, 77, 246, 246, 37, 70, 231, 118, 82, 2, 151, 227, 37, 66, 4, 134, 227, - 32, 69, 233, 136, 4, 11, 65, 82, 79, 85, 78, 68, 45, 80, 82, 79, 70, 9, - 49, 10, 79, 83, 84, 32, 69, 81, 85, 65, 76, 32, 6, 32, 2, 84, 79, 235, - 176, 31, 79, 5, 147, 205, 6, 32, 4, 240, 192, 10, 3, 73, 86, 69, 253, - 159, 27, 5, 69, 32, 79, 78, 69, 10, 162, 1, 80, 180, 194, 28, 6, 69, 82, - 73, 67, 65, 78, 200, 232, 5, 3, 66, 85, 76, 209, 204, 3, 16, 65, 76, 71, - 65, 77, 65, 84, 73, 79, 78, 32, 79, 82, 32, 67, 79, 4, 158, 205, 35, 69, - 171, 78, 72, 192, 9, 92, 3, 65, 84, 79, 174, 20, 71, 174, 1, 84, 150, - 167, 24, 67, 182, 203, 11, 68, 155, 252, 2, 75, 144, 9, 104, 17, 76, 73, - 65, 78, 32, 72, 73, 69, 82, 79, 71, 76, 89, 80, 72, 32, 65, 133, 241, 12, - 3, 77, 73, 67, 142, 9, 70, 48, 138, 4, 49, 198, 2, 50, 162, 3, 51, 174, - 5, 52, 163, 3, 53, 222, 1, 106, 50, 102, 52, 110, 54, 102, 57, 182, 7, - 51, 234, 190, 11, 49, 202, 174, 22, 48, 242, 1, 53, 2, 55, 3, 56, 22, - 238, 193, 33, 54, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, - 55, 2, 56, 3, 57, 28, 250, 135, 12, 54, 146, 185, 21, 49, 2, 53, 174, - 193, 5, 48, 2, 50, 2, 51, 2, 52, 2, 55, 2, 56, 3, 57, 26, 210, 238, 11, - 54, 250, 146, 27, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, - 57, 24, 186, 191, 33, 55, 2, 56, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, - 52, 2, 53, 2, 54, 3, 57, 232, 1, 98, 48, 114, 49, 254, 242, 11, 50, 2, - 51, 158, 129, 22, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 42, 162, 236, - 11, 52, 2, 55, 190, 24, 53, 146, 185, 21, 48, 2, 49, 2, 50, 174, 193, 5, - 51, 2, 54, 2, 56, 3, 57, 26, 238, 131, 12, 48, 146, 185, 21, 53, 174, - 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 222, 1, 102, - 48, 110, 49, 102, 57, 210, 1, 56, 170, 225, 11, 50, 2, 54, 250, 141, 22, - 51, 2, 52, 2, 53, 3, 55, 28, 150, 130, 12, 50, 146, 185, 21, 55, 2, 57, - 174, 193, 5, 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 54, 3, 56, 24, 186, 186, - 33, 53, 2, 54, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 55, 2, 56, - 3, 57, 24, 214, 185, 33, 52, 2, 57, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, - 2, 53, 2, 54, 2, 55, 3, 56, 228, 1, 102, 48, 2, 50, 2, 53, 102, 51, 110, - 54, 102, 56, 162, 1, 57, 174, 201, 11, 55, 162, 162, 22, 49, 3, 52, 22, - 134, 184, 33, 57, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, - 54, 2, 55, 3, 56, 30, 214, 229, 11, 54, 242, 150, 8, 50, 138, 252, 18, - 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 24, 182, 182, 33, - 52, 2, 56, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 53, 2, 54, 2, 55, 3, - 57, 26, 98, 51, 242, 180, 33, 49, 2, 54, 174, 193, 5, 48, 2, 50, 2, 52, - 2, 53, 2, 55, 2, 56, 3, 57, 4, 144, 173, 20, 6, 32, 82, 65, 32, 79, 82, - 139, 201, 18, 65, 20, 140, 203, 38, 3, 51, 32, 69, 210, 42, 48, 2, 49, 2, - 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 202, 1, 102, 49, 210, 1, - 53, 130, 132, 20, 57, 166, 228, 13, 48, 2, 50, 2, 51, 2, 52, 2, 54, 2, - 55, 3, 56, 22, 90, 48, 174, 243, 38, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, - 54, 2, 55, 2, 56, 3, 57, 4, 60, 6, 32, 66, 69, 71, 73, 78, 1, 5, 65, 32, - 69, 78, 68, 2, 129, 248, 15, 8, 32, 76, 79, 71, 79, 71, 82, 65, 24, 138, - 177, 33, 48, 2, 55, 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, - 2, 56, 3, 57, 60, 218, 142, 32, 51, 246, 214, 1, 48, 242, 1, 49, 3, 50, - 14, 96, 2, 76, 69, 250, 161, 2, 85, 208, 173, 31, 2, 83, 84, 230, 192, 3, - 69, 249, 162, 1, 2, 82, 89, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 254, - 133, 31, 83, 191, 161, 4, 85, 29, 104, 11, 73, 67, 76, 79, 67, 75, 87, - 73, 83, 69, 32, 233, 219, 35, 9, 69, 78, 78, 65, 32, 87, 73, 84, 72, 24, - 90, 84, 254, 220, 6, 67, 58, 68, 122, 71, 138, 4, 79, 253, 173, 27, 5, - 73, 78, 84, 69, 71, 12, 84, 15, 82, 73, 65, 78, 71, 76, 69, 45, 72, 69, - 65, 68, 69, 68, 32, 187, 225, 6, 79, 10, 112, 3, 76, 69, 70, 0, 4, 82, - 73, 71, 72, 12, 6, 66, 79, 84, 84, 79, 77, 0, 3, 84, 79, 80, 223, 224, 6, - 79, 2, 11, 84, 2, 169, 152, 36, 7, 32, 85, 45, 83, 72, 65, 80, 158, 1, - 128, 1, 20, 76, 32, 70, 85, 78, 67, 84, 73, 79, 78, 65, 76, 32, 83, 89, - 77, 66, 79, 76, 32, 210, 15, 79, 38, 80, 167, 219, 38, 67, 140, 1, 222, - 2, 67, 186, 1, 68, 190, 2, 73, 44, 4, 65, 76, 80, 72, 0, 4, 79, 77, 69, - 71, 32, 4, 74, 79, 84, 32, 36, 4, 76, 69, 70, 84, 68, 2, 81, 85, 222, 3, - 82, 54, 83, 92, 4, 69, 80, 83, 73, 32, 6, 66, 65, 67, 75, 83, 76, 76, 2, - 85, 80, 208, 216, 15, 12, 71, 82, 69, 65, 84, 69, 82, 45, 84, 72, 65, 78, - 0, 5, 84, 73, 76, 68, 69, 191, 195, 19, 90, 14, 64, 6, 73, 82, 67, 76, - 69, 32, 165, 152, 36, 4, 79, 77, 77, 65, 12, 80, 2, 83, 84, 202, 138, 19, - 68, 170, 246, 5, 66, 214, 157, 10, 85, 175, 248, 2, 74, 4, 246, 197, 37, - 73, 215, 104, 65, 22, 76, 2, 69, 76, 120, 3, 79, 87, 78, 253, 140, 31, 6, - 73, 65, 77, 79, 78, 68, 10, 30, 32, 53, 3, 84, 65, 32, 6, 182, 137, 19, - 68, 174, 155, 16, 84, 159, 163, 1, 83, 4, 254, 156, 35, 85, 207, 170, 1, - 83, 10, 22, 32, 171, 9, 87, 8, 52, 5, 84, 65, 67, 75, 32, 182, 1, 83, - 231, 6, 67, 4, 138, 156, 35, 85, 175, 248, 2, 74, 6, 40, 2, 79, 84, 173, - 155, 31, 2, 45, 66, 4, 11, 65, 5, 219, 139, 31, 32, 4, 158, 135, 19, 68, - 255, 147, 16, 85, 4, 28, 2, 32, 83, 191, 7, 87, 2, 149, 197, 36, 4, 72, - 79, 69, 32, 46, 44, 2, 65, 68, 137, 3, 4, 79, 84, 69, 32, 43, 11, 32, 40, - 178, 1, 67, 34, 68, 92, 2, 85, 80, 44, 2, 76, 69, 186, 158, 6, 81, 200, - 170, 13, 3, 78, 79, 84, 22, 69, 182, 176, 5, 66, 198, 133, 7, 83, 234, - 209, 4, 71, 162, 111, 82, 183, 79, 74, 4, 158, 146, 37, 73, 147, 127, 79, - 12, 54, 73, 36, 3, 79, 87, 78, 197, 188, 35, 2, 69, 76, 4, 218, 246, 30, - 86, 219, 174, 6, 65, 4, 152, 167, 10, 2, 32, 67, 215, 153, 27, 87, 4, - 174, 209, 36, 83, 255, 110, 70, 4, 242, 248, 11, 81, 139, 158, 23, 85, 4, - 184, 3, 5, 73, 71, 72, 84, 87, 131, 189, 38, 72, 10, 88, 5, 69, 77, 73, - 67, 79, 34, 76, 34, 84, 129, 247, 11, 7, 81, 85, 73, 83, 72, 32, 81, 2, - 229, 133, 31, 3, 76, 79, 78, 2, 177, 142, 36, 3, 65, 83, 72, 4, 248, 216, - 15, 2, 65, 82, 239, 133, 7, 73, 12, 22, 32, 171, 1, 87, 10, 78, 67, 36, - 5, 84, 65, 67, 75, 32, 253, 139, 38, 6, 83, 72, 79, 69, 32, 74, 2, 229, - 221, 22, 4, 65, 82, 69, 84, 6, 210, 255, 18, 68, 162, 195, 16, 79, 139, - 201, 2, 74, 2, 169, 207, 35, 6, 65, 82, 68, 83, 32, 86, 4, 242, 237, 32, - 83, 235, 157, 5, 76, 12, 92, 2, 82, 79, 153, 200, 25, 15, 76, 73, 67, 65, - 84, 73, 79, 78, 32, 80, 82, 79, 71, 82, 65, 10, 112, 9, 88, 73, 77, 65, - 84, 69, 76, 89, 32, 153, 189, 38, 13, 65, 67, 72, 69, 83, 32, 84, 72, 69, - 32, 76, 73, 77, 8, 76, 6, 69, 81, 85, 65, 76, 32, 201, 161, 24, 7, 66, - 85, 84, 32, 78, 79, 84, 6, 32, 2, 84, 79, 255, 128, 31, 79, 5, 181, 201, - 25, 13, 32, 79, 82, 32, 84, 72, 69, 32, 73, 77, 65, 71, 69, 230, 22, 148, - 1, 4, 65, 66, 73, 67, 208, 245, 1, 7, 77, 69, 78, 73, 65, 78, 32, 224, - 12, 3, 82, 79, 87, 236, 3, 2, 84, 73, 254, 188, 35, 73, 227, 147, 1, 67, - 148, 21, 54, 32, 141, 244, 1, 7, 45, 73, 78, 68, 73, 67, 32, 248, 20, - 178, 3, 67, 158, 1, 68, 242, 2, 69, 198, 1, 70, 212, 1, 2, 72, 65, 104, - 5, 75, 65, 83, 82, 65, 86, 76, 248, 167, 1, 2, 77, 65, 220, 18, 7, 78, - 85, 77, 66, 69, 82, 32, 36, 5, 79, 80, 69, 78, 32, 82, 80, 214, 1, 82, - 156, 2, 8, 66, 65, 83, 69, 76, 73, 78, 69, 52, 6, 73, 78, 86, 69, 82, 84, - 98, 83, 154, 33, 84, 202, 4, 86, 188, 245, 16, 9, 87, 65, 86, 89, 32, 72, - 65, 77, 90, 138, 164, 17, 81, 201, 190, 1, 6, 90, 87, 65, 82, 65, 75, 14, - 44, 5, 85, 82, 76, 89, 32, 191, 135, 36, 79, 12, 72, 4, 68, 65, 77, 77, - 0, 4, 70, 65, 84, 72, 1, 4, 75, 65, 83, 82, 4, 11, 65, 5, 199, 138, 37, - 84, 24, 154, 1, 65, 108, 5, 79, 85, 66, 76, 69, 152, 2, 16, 73, 83, 80, - 85, 84, 69, 68, 32, 69, 78, 68, 32, 79, 70, 32, 65, 137, 253, 22, 5, 69, - 67, 73, 77, 65, 14, 36, 3, 77, 77, 65, 171, 189, 32, 84, 13, 22, 32, 211, - 5, 84, 6, 158, 226, 1, 73, 58, 77, 167, 202, 35, 87, 6, 202, 4, 68, 157, - 193, 1, 18, 32, 82, 73, 71, 72, 84, 32, 65, 82, 82, 79, 87, 72, 69, 65, - 68, 32, 65, 8, 88, 12, 77, 80, 84, 89, 32, 67, 69, 78, 84, 82, 69, 32, - 57, 6, 78, 68, 32, 79, 70, 32, 4, 184, 131, 36, 4, 72, 73, 71, 72, 1, 3, - 76, 79, 87, 4, 26, 65, 139, 249, 28, 84, 2, 255, 148, 31, 89, 22, 84, 4, - 65, 84, 72, 65, 230, 209, 1, 79, 164, 196, 4, 3, 73, 86, 69, 223, 235, - 29, 85, 17, 22, 32, 139, 2, 84, 10, 52, 5, 87, 73, 84, 72, 32, 162, 222, - 1, 73, 59, 77, 6, 170, 254, 36, 84, 174, 77, 68, 191, 47, 82, 6, 72, 13, - 76, 70, 32, 77, 65, 68, 68, 65, 32, 79, 86, 69, 82, 231, 16, 77, 2, 181, - 157, 17, 2, 32, 77, 13, 18, 32, 43, 84, 6, 230, 72, 87, 146, 148, 1, 73, - 59, 77, 4, 161, 82, 2, 65, 78, 224, 15, 84, 5, 65, 82, 71, 69, 32, 146, - 1, 69, 149, 90, 8, 73, 71, 65, 84, 85, 82, 69, 32, 8, 64, 10, 82, 79, 85, - 78, 68, 32, 68, 79, 84, 32, 251, 218, 10, 67, 6, 220, 218, 10, 6, 73, 78, - 83, 73, 68, 69, 238, 213, 25, 66, 223, 155, 1, 65, 252, 7, 80, 5, 84, 84, - 69, 82, 32, 157, 251, 6, 9, 70, 84, 32, 65, 82, 82, 79, 87, 72, 248, 7, - 206, 2, 65, 240, 10, 2, 66, 69, 150, 4, 68, 214, 5, 70, 246, 4, 71, 226, - 2, 72, 156, 9, 2, 74, 69, 122, 75, 246, 5, 76, 222, 2, 77, 154, 1, 78, - 140, 3, 3, 80, 69, 72, 176, 1, 3, 81, 65, 70, 214, 1, 82, 230, 3, 83, - 158, 9, 84, 186, 7, 85, 148, 2, 2, 86, 69, 28, 3, 87, 65, 87, 202, 1, 89, - 246, 3, 79, 248, 2, 2, 90, 65, 31, 69, 112, 92, 7, 70, 82, 73, 67, 65, - 78, 32, 92, 2, 73, 78, 144, 2, 3, 76, 69, 70, 195, 193, 38, 69, 8, 52, 3, - 81, 65, 70, 214, 144, 32, 70, 243, 161, 3, 78, 5, 221, 64, 5, 32, 87, 73, - 84, 72, 21, 11, 32, 18, 72, 6, 87, 73, 84, 72, 32, 84, 154, 160, 1, 70, - 230, 43, 73, 203, 9, 77, 10, 60, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, - 32, 131, 55, 87, 6, 216, 204, 14, 17, 80, 79, 73, 78, 84, 73, 78, 71, 32, - 68, 79, 87, 78, 87, 65, 82, 68, 166, 221, 21, 66, 223, 155, 1, 65, 83, - 11, 32, 80, 70, 87, 196, 45, 6, 77, 65, 75, 83, 85, 82, 202, 112, 70, - 247, 52, 73, 70, 48, 4, 73, 84, 72, 32, 145, 45, 3, 65, 83, 76, 64, 192, - 2, 9, 65, 84, 84, 65, 67, 72, 69, 68, 32, 220, 1, 19, 82, 73, 71, 72, 84, - 32, 77, 73, 68, 68, 76, 69, 32, 83, 84, 82, 79, 75, 69, 188, 1, 6, 72, - 65, 77, 90, 65, 32, 44, 8, 87, 65, 86, 89, 32, 72, 65, 77, 254, 69, 69, - 204, 1, 4, 77, 65, 68, 68, 196, 139, 35, 9, 76, 69, 70, 84, 32, 77, 73, - 68, 68, 243, 230, 1, 68, 28, 204, 1, 17, 66, 79, 84, 84, 79, 77, 32, 82, - 73, 71, 72, 84, 32, 75, 65, 83, 82, 0, 14, 84, 79, 80, 32, 82, 73, 71, - 72, 84, 32, 70, 65, 84, 72, 94, 82, 56, 3, 76, 69, 70, 186, 201, 1, 75, - 223, 160, 31, 70, 6, 11, 65, 7, 29, 5, 32, 65, 78, 68, 32, 4, 196, 231, - 18, 3, 76, 69, 70, 255, 211, 18, 68, 8, 52, 3, 73, 71, 72, 253, 214, 1, - 4, 79, 85, 78, 68, 4, 17, 2, 84, 32, 4, 162, 176, 1, 82, 227, 37, 72, 12, - 146, 72, 65, 37, 5, 66, 69, 76, 79, 87, 4, 211, 143, 36, 90, 50, 22, 72, - 219, 63, 69, 41, 22, 32, 195, 63, 69, 28, 68, 5, 87, 73, 84, 72, 32, 198, - 150, 1, 70, 230, 43, 73, 203, 9, 77, 20, 116, 6, 83, 77, 65, 76, 76, 32, - 34, 84, 154, 19, 73, 200, 33, 10, 68, 79, 84, 32, 66, 69, 76, 79, 87, 32, - 215, 19, 72, 6, 226, 39, 77, 175, 232, 22, 86, 8, 88, 10, 72, 82, 69, 69, - 32, 68, 79, 84, 83, 32, 177, 49, 7, 87, 79, 32, 68, 79, 84, 83, 6, 144, - 1, 22, 80, 79, 73, 78, 84, 73, 78, 71, 32, 85, 80, 87, 65, 82, 68, 83, - 32, 66, 69, 76, 79, 87, 133, 29, 8, 72, 79, 82, 73, 90, 79, 78, 84, 5, - 227, 191, 14, 32, 76, 90, 65, 164, 4, 2, 68, 65, 40, 7, 79, 84, 76, 69, - 83, 83, 32, 190, 28, 89, 255, 24, 85, 42, 34, 76, 154, 4, 72, 187, 46, - 68, 25, 11, 32, 22, 56, 5, 87, 73, 84, 72, 32, 226, 145, 1, 70, 247, 52, - 73, 18, 132, 1, 9, 68, 79, 84, 32, 66, 69, 76, 79, 87, 16, 9, 73, 78, 86, - 69, 82, 84, 69, 68, 32, 34, 84, 202, 215, 11, 70, 203, 138, 26, 82, 5, - 243, 1, 32, 4, 226, 14, 83, 199, 164, 38, 86, 6, 160, 1, 10, 72, 82, 69, - 69, 32, 68, 79, 84, 83, 32, 33, 25, 87, 79, 32, 68, 79, 84, 83, 32, 86, - 69, 82, 84, 73, 67, 65, 76, 76, 89, 32, 66, 69, 76, 79, 87, 32, 4, 234, - 53, 65, 195, 227, 35, 66, 2, 253, 235, 14, 10, 65, 78, 68, 32, 83, 77, - 65, 76, 76, 32, 12, 22, 72, 207, 62, 76, 6, 199, 53, 65, 6, 190, 253, 31, - 66, 2, 70, 139, 178, 5, 81, 44, 60, 8, 65, 82, 83, 73, 32, 89, 69, 72, - 169, 2, 2, 69, 72, 23, 11, 32, 20, 68, 5, 87, 73, 84, 72, 32, 194, 140, - 1, 70, 230, 43, 73, 203, 9, 77, 12, 144, 1, 28, 69, 88, 84, 69, 78, 68, - 69, 68, 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, 32, 68, 73, - 71, 73, 84, 32, 30, 84, 163, 40, 73, 6, 250, 8, 70, 219, 49, 84, 4, 226, - 245, 6, 72, 131, 194, 7, 87, 23, 11, 32, 20, 68, 5, 87, 73, 84, 72, 32, - 154, 138, 1, 70, 230, 43, 73, 203, 9, 77, 12, 32, 4, 68, 79, 84, 32, 51, - 84, 6, 254, 40, 66, 249, 142, 15, 4, 77, 79, 86, 69, 6, 64, 10, 72, 82, - 69, 69, 32, 68, 79, 84, 83, 32, 203, 132, 36, 87, 4, 198, 17, 80, 227, - 129, 36, 66, 44, 72, 2, 65, 70, 160, 1, 4, 72, 65, 73, 78, 178, 21, 85, - 143, 147, 37, 82, 19, 11, 32, 16, 68, 5, 87, 73, 84, 72, 32, 194, 135, 1, - 70, 230, 43, 73, 203, 9, 77, 8, 162, 24, 84, 134, 193, 37, 82, 229, 32, - 8, 73, 78, 86, 69, 82, 84, 69, 68, 15, 11, 32, 12, 68, 5, 87, 73, 84, 72, - 32, 162, 134, 1, 70, 230, 43, 73, 203, 9, 77, 4, 254, 37, 84, 171, 140, - 26, 68, 82, 78, 65, 236, 5, 2, 69, 72, 161, 2, 9, 73, 71, 72, 32, 72, 65, - 77, 90, 65, 34, 34, 72, 213, 47, 3, 77, 90, 65, 31, 11, 32, 28, 68, 5, - 87, 73, 84, 72, 32, 186, 132, 1, 70, 230, 43, 73, 203, 9, 77, 20, 132, 2, - 29, 69, 88, 84, 69, 78, 68, 69, 68, 32, 65, 82, 65, 66, 73, 67, 45, 73, - 78, 68, 73, 67, 32, 68, 73, 71, 73, 84, 32, 70, 30, 73, 92, 24, 83, 77, - 65, 76, 76, 32, 65, 82, 65, 66, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, - 84, 65, 72, 32, 62, 84, 135, 52, 72, 2, 145, 186, 1, 2, 79, 85, 2, 45, 9, - 78, 86, 69, 82, 84, 69, 68, 32, 83, 2, 221, 253, 35, 6, 77, 65, 76, 76, - 32, 86, 6, 26, 65, 199, 139, 36, 66, 4, 174, 31, 78, 255, 135, 37, 66, 8, - 88, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 33, 8, 87, 79, 32, 68, - 79, 84, 83, 32, 4, 230, 8, 80, 191, 157, 37, 65, 4, 232, 165, 37, 8, 86, - 69, 82, 84, 73, 67, 65, 76, 27, 65, 41, 11, 32, 38, 144, 1, 4, 71, 79, - 65, 76, 88, 5, 87, 73, 84, 72, 32, 220, 46, 10, 68, 79, 65, 67, 72, 65, - 83, 72, 77, 69, 242, 78, 70, 230, 43, 73, 203, 9, 77, 13, 11, 32, 10, - 156, 49, 6, 87, 73, 84, 72, 32, 72, 250, 76, 70, 230, 43, 73, 203, 9, 77, - 8, 218, 26, 73, 245, 18, 3, 89, 69, 72, 9, 11, 32, 6, 166, 254, 22, 65, - 178, 238, 8, 89, 135, 181, 5, 87, 22, 28, 2, 69, 77, 131, 45, 72, 17, 11, - 32, 14, 140, 31, 6, 87, 73, 84, 72, 32, 84, 170, 93, 70, 230, 43, 73, - 203, 9, 77, 68, 98, 65, 208, 1, 4, 69, 72, 69, 72, 160, 3, 7, 73, 82, 71, - 72, 73, 90, 32, 205, 30, 2, 72, 65, 22, 46, 70, 177, 162, 1, 5, 83, 72, - 77, 73, 82, 21, 11, 32, 18, 64, 5, 87, 73, 84, 72, 32, 138, 122, 70, 230, - 43, 73, 203, 9, 77, 10, 42, 84, 130, 174, 34, 68, 199, 157, 3, 82, 4, - 174, 166, 14, 87, 207, 206, 21, 72, 25, 11, 32, 22, 64, 5, 87, 73, 84, - 72, 32, 234, 120, 70, 230, 43, 73, 203, 9, 77, 14, 38, 84, 226, 41, 83, - 143, 241, 36, 68, 10, 88, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, - 121, 8, 87, 79, 32, 68, 79, 84, 83, 32, 6, 42, 80, 226, 129, 36, 66, 223, - 155, 1, 65, 2, 165, 243, 35, 14, 79, 73, 78, 84, 73, 78, 71, 32, 85, 80, - 87, 65, 82, 68, 4, 48, 6, 86, 69, 82, 84, 73, 67, 191, 156, 37, 65, 2, - 145, 188, 11, 3, 65, 76, 76, 12, 198, 38, 79, 13, 2, 89, 85, 26, 40, 2, - 65, 77, 245, 169, 1, 2, 79, 87, 25, 11, 32, 22, 64, 5, 87, 73, 84, 72, - 32, 254, 116, 70, 230, 43, 73, 203, 9, 77, 14, 88, 2, 68, 79, 36, 6, 83, - 77, 65, 76, 76, 32, 216, 162, 32, 2, 84, 72, 223, 209, 4, 66, 4, 166, - 161, 17, 85, 231, 248, 19, 84, 4, 204, 26, 16, 65, 82, 65, 66, 73, 67, - 32, 76, 69, 84, 84, 69, 82, 32, 84, 65, 143, 252, 37, 86, 18, 36, 3, 69, - 69, 77, 195, 212, 37, 65, 17, 11, 32, 14, 64, 5, 87, 73, 84, 72, 32, 166, - 114, 70, 230, 43, 73, 203, 9, 77, 6, 130, 18, 84, 199, 148, 34, 68, 60, - 38, 71, 26, 89, 17, 3, 79, 79, 78, 21, 22, 79, 255, 108, 32, 10, 247, 25, - 69, 31, 11, 32, 28, 92, 5, 71, 72, 85, 78, 78, 16, 5, 87, 73, 84, 72, 32, - 186, 112, 70, 230, 43, 73, 203, 9, 77, 6, 131, 33, 65, 14, 116, 6, 83, - 77, 65, 76, 76, 32, 38, 84, 184, 23, 8, 73, 78, 86, 69, 82, 84, 69, 68, - 238, 131, 26, 68, 227, 165, 11, 82, 4, 214, 217, 30, 84, 203, 184, 7, 86, - 4, 238, 217, 6, 72, 247, 144, 29, 87, 25, 22, 32, 151, 23, 69, 12, 88, - 11, 87, 73, 84, 72, 32, 83, 77, 65, 76, 76, 32, 134, 110, 70, 230, 43, - 73, 203, 9, 77, 4, 26, 77, 203, 144, 38, 86, 2, 209, 147, 37, 3, 69, 69, - 77, 19, 11, 32, 16, 64, 5, 87, 73, 84, 72, 32, 250, 108, 70, 230, 43, 73, - 203, 9, 77, 8, 36, 4, 68, 79, 84, 32, 179, 12, 84, 6, 44, 5, 66, 69, 76, - 79, 87, 167, 146, 37, 65, 5, 241, 152, 14, 6, 32, 65, 78, 68, 32, 78, 52, - 108, 2, 69, 72, 192, 27, 3, 82, 69, 72, 136, 3, 4, 78, 79, 79, 78, 197, - 118, 7, 79, 72, 73, 78, 71, 89, 65, 35, 11, 32, 32, 52, 5, 87, 73, 84, - 72, 32, 194, 106, 70, 247, 52, 73, 28, 134, 1, 83, 96, 2, 84, 87, 234, 5, - 73, 250, 21, 72, 154, 148, 11, 70, 148, 226, 6, 5, 68, 79, 84, 32, 66, - 234, 39, 76, 207, 128, 19, 82, 10, 44, 5, 77, 65, 76, 76, 32, 251, 219, - 37, 84, 8, 194, 6, 65, 182, 198, 6, 78, 215, 150, 16, 86, 4, 37, 7, 79, - 32, 68, 79, 84, 83, 32, 4, 162, 8, 86, 147, 134, 37, 65, 60, 184, 1, 2, - 65, 68, 120, 3, 69, 69, 78, 132, 6, 4, 72, 69, 69, 78, 220, 147, 1, 4, - 85, 80, 69, 82, 156, 145, 4, 7, 84, 82, 65, 73, 71, 72, 84, 169, 220, 31, - 6, 87, 65, 83, 72, 32, 75, 17, 11, 32, 14, 68, 6, 87, 73, 84, 72, 32, 84, - 130, 102, 70, 230, 43, 73, 203, 9, 77, 6, 218, 148, 32, 72, 239, 204, 3, - 87, 27, 11, 32, 24, 64, 5, 87, 73, 84, 72, 32, 142, 101, 70, 230, 43, 73, - 203, 9, 77, 16, 232, 1, 3, 68, 79, 84, 50, 73, 44, 7, 83, 77, 65, 76, 76, - 32, 65, 110, 84, 250, 168, 11, 70, 177, 137, 5, 31, 69, 88, 84, 69, 78, - 68, 69, 68, 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, 32, 68, - 73, 71, 73, 84, 32, 70, 79, 85, 2, 249, 140, 18, 7, 32, 66, 69, 76, 79, - 87, 32, 2, 169, 20, 7, 78, 86, 69, 82, 84, 69, 68, 2, 85, 19, 82, 65, 66, - 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 84, 65, 72, 32, 65, 78, 2, 159, - 167, 22, 68, 6, 96, 11, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 66, 105, - 9, 87, 79, 32, 68, 79, 84, 83, 32, 86, 4, 25, 4, 69, 76, 79, 87, 5, 11, - 32, 2, 21, 3, 65, 78, 68, 2, 17, 2, 32, 84, 2, 243, 202, 6, 72, 2, 209, - 165, 11, 8, 69, 82, 84, 73, 67, 65, 76, 76, 13, 11, 32, 10, 46, 87, 158, - 95, 70, 230, 43, 73, 203, 9, 77, 2, 161, 139, 26, 5, 73, 84, 72, 32, 68, - 118, 92, 2, 65, 72, 144, 1, 4, 67, 72, 69, 72, 124, 2, 69, 72, 150, 3, - 72, 61, 3, 84, 69, 72, 19, 11, 32, 16, 64, 5, 87, 73, 84, 72, 32, 198, - 93, 70, 230, 43, 73, 203, 9, 77, 8, 26, 84, 179, 137, 26, 68, 6, 250, - 137, 14, 87, 139, 130, 18, 72, 25, 22, 32, 163, 5, 69, 12, 64, 5, 87, 73, - 84, 72, 32, 170, 92, 70, 230, 43, 73, 203, 9, 77, 4, 198, 13, 83, 143, - 241, 36, 68, 37, 22, 32, 167, 4, 69, 24, 62, 77, 116, 5, 87, 73, 84, 72, - 32, 190, 90, 70, 231, 43, 73, 10, 48, 6, 65, 82, 66, 85, 84, 65, 183, - 144, 1, 69, 9, 11, 32, 6, 238, 90, 70, 246, 52, 73, 197, 228, 35, 2, 71, - 79, 8, 104, 6, 83, 77, 65, 76, 76, 32, 56, 12, 84, 72, 82, 69, 69, 32, - 68, 79, 84, 83, 32, 65, 131, 171, 37, 82, 4, 32, 2, 84, 69, 143, 252, 37, - 86, 2, 151, 255, 36, 72, 2, 237, 135, 27, 4, 66, 79, 86, 69, 18, 42, 65, - 126, 69, 213, 129, 1, 2, 73, 78, 6, 131, 9, 76, 23, 18, 32, 91, 69, 10, - 60, 4, 87, 73, 84, 72, 230, 87, 70, 230, 43, 73, 203, 9, 77, 2, 129, 9, - 2, 32, 83, 10, 143, 11, 72, 15, 158, 1, 32, 181, 80, 33, 73, 71, 72, 85, - 82, 32, 75, 65, 90, 65, 75, 72, 32, 75, 73, 82, 71, 72, 73, 90, 32, 65, - 76, 69, 70, 32, 77, 65, 75, 83, 85, 82, 65, 8, 96, 16, 87, 73, 84, 72, - 32, 72, 65, 77, 90, 65, 32, 65, 66, 79, 86, 69, 186, 85, 70, 247, 52, 73, - 5, 155, 75, 32, 17, 234, 8, 72, 167, 76, 32, 25, 11, 32, 22, 52, 5, 87, - 73, 84, 72, 32, 202, 84, 70, 247, 52, 73, 18, 80, 4, 68, 79, 84, 32, 162, - 2, 69, 182, 1, 72, 234, 252, 13, 84, 163, 165, 23, 82, 4, 192, 178, 30, - 3, 87, 73, 84, 183, 199, 6, 65, 54, 28, 2, 69, 72, 227, 3, 85, 49, 11, - 32, 46, 100, 6, 66, 65, 82, 82, 69, 69, 252, 2, 5, 87, 73, 84, 72, 32, - 182, 79, 70, 230, 43, 73, 203, 9, 77, 17, 11, 32, 14, 52, 5, 87, 73, 84, - 72, 32, 238, 81, 70, 247, 52, 73, 10, 22, 69, 183, 1, 72, 4, 121, 28, 88, - 84, 69, 78, 68, 69, 68, 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, - 67, 32, 68, 73, 71, 73, 84, 32, 84, 4, 160, 229, 34, 3, 72, 82, 69, 161, - 145, 2, 2, 87, 79, 6, 21, 3, 65, 77, 90, 6, 11, 65, 6, 17, 2, 32, 65, 6, - 21, 3, 66, 79, 86, 6, 11, 69, 7, 171, 79, 32, 22, 64, 10, 72, 65, 77, 90, - 65, 32, 65, 66, 79, 86, 18, 83, 51, 84, 10, 179, 2, 69, 2, 25, 4, 77, 65, - 76, 76, 2, 255, 150, 31, 32, 10, 108, 18, 87, 79, 32, 68, 79, 84, 83, 32, - 66, 69, 76, 79, 87, 32, 65, 78, 68, 32, 174, 195, 35, 65, 183, 5, 72, 6, - 70, 72, 232, 176, 6, 7, 83, 77, 65, 76, 76, 32, 78, 191, 190, 30, 68, 2, - 205, 248, 32, 3, 65, 77, 90, 18, 26, 72, 17, 2, 73, 78, 11, 243, 71, 32, - 9, 11, 32, 6, 158, 76, 70, 246, 52, 73, 161, 11, 13, 87, 73, 84, 72, 32, - 73, 78, 86, 69, 82, 84, 69, 68, 220, 7, 230, 5, 65, 158, 5, 66, 176, 2, - 9, 68, 65, 68, 32, 87, 73, 84, 72, 32, 80, 9, 70, 69, 72, 32, 87, 73, 84, - 72, 32, 72, 11, 71, 72, 65, 73, 78, 32, 87, 73, 84, 72, 32, 154, 1, 72, - 134, 3, 74, 190, 2, 75, 252, 1, 9, 76, 65, 77, 32, 87, 73, 84, 72, 32, - 226, 4, 77, 152, 4, 10, 78, 79, 79, 78, 32, 87, 73, 84, 72, 32, 182, 2, - 81, 190, 2, 82, 178, 2, 83, 146, 16, 84, 196, 9, 7, 87, 65, 83, 65, 76, - 76, 65, 40, 9, 89, 69, 72, 32, 87, 73, 84, 72, 32, 216, 3, 53, 85, 73, - 71, 72, 85, 82, 32, 75, 73, 82, 71, 72, 73, 90, 32, 89, 69, 72, 32, 87, - 73, 84, 72, 32, 72, 65, 77, 90, 65, 32, 65, 66, 79, 86, 69, 32, 87, 73, - 84, 72, 32, 65, 76, 69, 70, 32, 77, 65, 75, 83, 85, 82, 65, 205, 6, 14, - 90, 65, 72, 32, 87, 73, 84, 72, 32, 77, 69, 69, 77, 32, 54, 128, 1, 8, - 73, 78, 32, 87, 73, 84, 72, 32, 70, 76, 232, 57, 4, 75, 66, 65, 82, 205, - 168, 35, 8, 90, 90, 65, 32, 87, 65, 32, 74, 28, 152, 19, 4, 77, 69, 69, - 77, 150, 16, 74, 238, 23, 65, 255, 8, 89, 22, 56, 3, 65, 89, 72, 216, 1, - 3, 69, 70, 32, 207, 13, 76, 12, 30, 73, 122, 65, 151, 56, 69, 8, 52, 9, - 32, 65, 83, 45, 83, 65, 76, 65, 65, 47, 77, 4, 80, 4, 84, 85, 32, 87, - 139, 229, 37, 77, 4, 18, 65, 23, 32, 2, 17, 2, 65, 32, 2, 149, 153, 32, - 6, 65, 83, 45, 83, 65, 76, 8, 212, 65, 29, 77, 65, 75, 83, 85, 82, 65, - 32, 87, 73, 84, 72, 32, 83, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, - 65, 76, 69, 70, 1, 13, 87, 73, 84, 72, 32, 70, 65, 84, 72, 65, 84, 65, - 78, 44, 160, 1, 8, 69, 72, 32, 87, 73, 84, 72, 32, 141, 135, 25, 25, 73, - 83, 77, 73, 76, 76, 65, 72, 32, 65, 82, 45, 82, 65, 72, 77, 65, 78, 32, - 65, 82, 45, 82, 65, 72, 42, 98, 72, 24, 3, 75, 72, 65, 230, 52, 65, 250, - 3, 74, 78, 77, 86, 78, 78, 90, 242, 2, 82, 43, 89, 10, 22, 65, 171, 56, - 69, 6, 131, 59, 72, 36, 222, 6, 72, 158, 7, 75, 218, 38, 65, 250, 3, 74, - 2, 77, 134, 5, 82, 3, 89, 30, 170, 13, 75, 218, 38, 65, 250, 3, 74, 146, - 2, 77, 110, 72, 139, 2, 89, 22, 64, 5, 77, 69, 69, 77, 32, 250, 50, 65, - 250, 3, 74, 135, 5, 89, 10, 40, 5, 87, 73, 84, 72, 32, 235, 103, 73, 6, - 174, 21, 77, 170, 25, 65, 203, 12, 89, 40, 80, 8, 65, 72, 32, 87, 73, 84, - 72, 32, 69, 8, 69, 72, 32, 87, 73, 84, 72, 32, 22, 236, 3, 4, 77, 69, 69, - 77, 226, 45, 65, 138, 6, 74, 247, 2, 89, 18, 164, 1, 5, 77, 69, 69, 77, - 32, 196, 11, 6, 65, 76, 69, 70, 32, 77, 150, 2, 89, 134, 38, 74, 53, 16, - 83, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, 76, 69, 70, 8, 40, 5, - 87, 73, 84, 72, 32, 215, 100, 73, 4, 254, 43, 74, 3, 77, 30, 104, 9, 69, - 69, 77, 32, 87, 73, 84, 72, 32, 173, 230, 3, 11, 65, 76, 76, 65, 74, 65, - 76, 65, 76, 79, 85, 28, 62, 72, 60, 5, 77, 69, 69, 77, 32, 186, 45, 65, - 255, 8, 89, 8, 17, 2, 65, 72, 8, 11, 32, 8, 178, 29, 87, 191, 69, 73, 12, - 40, 5, 87, 73, 84, 72, 32, 171, 98, 73, 8, 166, 37, 72, 242, 3, 65, 203, - 12, 89, 64, 84, 8, 65, 70, 32, 87, 73, 84, 72, 32, 93, 9, 72, 65, 72, 32, - 87, 73, 84, 72, 32, 46, 168, 46, 2, 65, 76, 218, 1, 74, 96, 2, 76, 65, - 146, 2, 75, 14, 72, 30, 77, 239, 1, 89, 18, 54, 72, 250, 42, 65, 250, 3, - 74, 2, 77, 135, 5, 89, 2, 247, 8, 65, 74, 116, 5, 65, 76, 69, 70, 32, - 210, 1, 72, 124, 5, 74, 69, 69, 77, 32, 78, 75, 28, 5, 77, 69, 69, 77, - 32, 187, 47, 89, 20, 64, 5, 87, 73, 84, 72, 32, 138, 44, 77, 222, 6, 70, - 247, 52, 73, 12, 68, 6, 72, 65, 77, 90, 65, 32, 41, 7, 77, 65, 68, 68, - 65, 32, 65, 8, 38, 65, 249, 45, 4, 66, 69, 76, 79, 4, 229, 45, 3, 66, 79, - 86, 14, 28, 2, 65, 72, 163, 44, 69, 12, 11, 32, 12, 40, 5, 87, 73, 84, - 72, 32, 231, 92, 73, 8, 210, 35, 65, 130, 12, 77, 75, 89, 14, 40, 5, 87, - 73, 84, 72, 32, 151, 92, 73, 10, 130, 47, 74, 2, 77, 75, 89, 8, 129, 15, - 3, 72, 65, 72, 14, 150, 14, 87, 222, 33, 70, 230, 43, 73, 203, 9, 77, 48, - 84, 9, 69, 69, 77, 32, 87, 73, 84, 72, 32, 193, 36, 7, 79, 72, 65, 77, - 77, 65, 68, 46, 116, 5, 65, 76, 69, 70, 32, 66, 72, 0, 2, 75, 72, 104, 5, - 74, 69, 69, 77, 32, 92, 5, 77, 69, 69, 77, 32, 43, 89, 4, 22, 77, 219, - 45, 70, 2, 173, 35, 6, 65, 75, 83, 85, 82, 65, 10, 21, 3, 65, 72, 32, 10, - 40, 5, 87, 73, 84, 72, 32, 211, 88, 73, 6, 250, 31, 74, 2, 77, 143, 12, - 89, 12, 40, 5, 87, 73, 84, 72, 32, 131, 88, 73, 8, 170, 31, 77, 194, 7, - 75, 14, 72, 195, 4, 89, 8, 246, 40, 87, 246, 2, 70, 231, 43, 73, 2, 11, - 69, 2, 143, 33, 72, 60, 134, 1, 72, 28, 5, 74, 69, 69, 77, 32, 92, 5, 77, - 69, 69, 77, 32, 242, 31, 65, 154, 5, 78, 78, 90, 134, 1, 75, 238, 1, 82, - 43, 89, 14, 146, 27, 65, 155, 9, 69, 16, 40, 5, 87, 73, 84, 72, 32, 191, - 85, 73, 12, 186, 24, 72, 242, 3, 65, 130, 12, 77, 75, 89, 12, 206, 15, - 87, 218, 25, 70, 230, 43, 73, 203, 9, 77, 28, 70, 65, 197, 146, 30, 11, - 85, 68, 68, 73, 83, 65, 32, 83, 73, 82, 82, 26, 64, 7, 70, 32, 87, 73, - 84, 72, 32, 193, 8, 4, 76, 65, 32, 85, 24, 64, 5, 77, 69, 69, 77, 32, - 238, 29, 65, 246, 6, 72, 139, 2, 89, 12, 40, 5, 87, 73, 84, 72, 32, 223, - 82, 73, 8, 34, 77, 186, 21, 72, 187, 16, 89, 2, 129, 38, 3, 69, 69, 77, - 18, 30, 65, 129, 26, 2, 69, 72, 16, 128, 1, 14, 68, 73, 32, 65, 76, 76, - 65, 65, 72, 85, 32, 65, 78, 72, 76, 6, 72, 73, 77, 65, 72, 85, 161, 26, - 4, 83, 79, 85, 76, 11, 26, 85, 163, 197, 37, 65, 6, 26, 77, 251, 140, 36, - 78, 5, 131, 197, 37, 65, 4, 26, 32, 1, 2, 77, 32, 2, 193, 142, 30, 4, 65, - 76, 76, 65, 190, 1, 138, 1, 65, 192, 6, 9, 69, 69, 78, 32, 87, 73, 84, - 72, 32, 250, 3, 72, 233, 5, 13, 85, 66, 72, 65, 65, 78, 65, 72, 85, 32, - 87, 65, 32, 46, 48, 7, 68, 32, 87, 73, 84, 72, 32, 251, 1, 76, 32, 76, 4, - 72, 65, 72, 32, 82, 77, 238, 23, 65, 138, 4, 75, 246, 4, 82, 3, 89, 10, - 22, 87, 195, 77, 73, 6, 25, 4, 73, 84, 72, 32, 6, 162, 16, 72, 187, 16, - 89, 8, 21, 3, 69, 69, 77, 8, 11, 32, 8, 208, 31, 6, 87, 73, 84, 72, 32, - 77, 147, 45, 73, 14, 26, 65, 77, 2, 76, 65, 4, 134, 22, 77, 193, 189, 35, - 11, 65, 77, 85, 72, 85, 32, 65, 76, 65, 89, 78, 10, 34, 32, 133, 1, 3, - 76, 76, 65, 4, 22, 85, 159, 84, 73, 2, 217, 8, 23, 83, 69, 68, 32, 65, - 83, 32, 75, 79, 82, 65, 78, 73, 67, 32, 83, 84, 79, 80, 32, 83, 73, 71, - 6, 98, 72, 169, 191, 37, 18, 65, 72, 85, 32, 65, 76, 65, 89, 72, 73, 32, - 87, 65, 45, 65, 65, 76, 73, 4, 220, 63, 12, 79, 85, 32, 65, 76, 65, 89, - 72, 69, 32, 87, 65, 1, 22, 85, 32, 65, 76, 65, 89, 72, 73, 32, 87, 65, - 65, 65, 76, 73, 72, 69, 69, 32, 87, 65, 45, 60, 130, 1, 72, 100, 5, 74, - 69, 69, 77, 32, 84, 5, 75, 72, 65, 72, 32, 92, 5, 77, 69, 69, 77, 32, - 234, 15, 65, 254, 8, 82, 3, 89, 12, 32, 3, 65, 72, 32, 159, 21, 69, 8, - 156, 14, 6, 87, 73, 84, 72, 32, 74, 214, 56, 73, 203, 9, 77, 10, 52, 5, - 87, 73, 84, 72, 32, 254, 69, 73, 203, 9, 77, 4, 234, 12, 65, 139, 8, 72, - 10, 34, 87, 190, 69, 73, 203, 9, 77, 4, 25, 4, 73, 84, 72, 32, 4, 142, - 12, 65, 203, 12, 89, 16, 52, 5, 87, 73, 84, 72, 32, 206, 68, 73, 203, 9, - 77, 10, 202, 7, 72, 174, 4, 74, 199, 11, 77, 82, 96, 10, 65, 68, 68, 65, - 32, 87, 73, 84, 72, 32, 161, 1, 9, 69, 69, 78, 32, 87, 73, 84, 72, 32, - 18, 96, 4, 68, 65, 77, 77, 0, 4, 75, 65, 83, 82, 138, 11, 83, 237, 41, 6, - 70, 65, 84, 72, 65, 32, 6, 11, 65, 6, 28, 2, 84, 65, 203, 52, 32, 2, 163, - 12, 78, 64, 130, 1, 72, 36, 5, 74, 69, 69, 77, 32, 52, 5, 77, 69, 69, 77, - 32, 170, 11, 65, 228, 4, 4, 75, 72, 65, 72, 154, 4, 82, 3, 89, 18, 194, - 15, 69, 229, 3, 2, 65, 72, 10, 158, 18, 87, 246, 2, 70, 230, 43, 73, 203, - 9, 77, 16, 64, 5, 87, 73, 84, 72, 32, 158, 20, 70, 230, 43, 73, 203, 9, - 77, 8, 252, 2, 2, 75, 72, 243, 15, 77, 122, 66, 65, 176, 2, 8, 69, 72, - 32, 87, 73, 84, 72, 32, 179, 4, 72, 28, 88, 11, 66, 65, 65, 82, 65, 75, - 65, 32, 87, 65, 45, 29, 7, 72, 32, 87, 73, 84, 72, 32, 2, 177, 163, 27, - 2, 84, 65, 26, 64, 5, 77, 69, 69, 77, 32, 194, 8, 65, 246, 6, 72, 139, 2, - 89, 14, 52, 5, 87, 73, 84, 72, 32, 166, 61, 73, 203, 9, 77, 8, 34, 72, - 174, 4, 77, 143, 12, 89, 4, 133, 16, 2, 65, 72, 66, 138, 1, 72, 108, 3, - 75, 72, 65, 12, 4, 74, 69, 69, 77, 92, 5, 77, 69, 69, 77, 32, 238, 4, 65, - 154, 5, 78, 78, 90, 242, 2, 82, 43, 89, 14, 32, 3, 65, 72, 32, 227, 9, - 69, 10, 40, 5, 87, 73, 84, 72, 32, 143, 59, 73, 6, 182, 2, 77, 199, 11, - 74, 10, 11, 72, 10, 11, 32, 10, 40, 5, 87, 73, 84, 72, 32, 175, 58, 73, - 6, 154, 1, 65, 62, 77, 143, 12, 89, 18, 64, 5, 87, 73, 84, 72, 32, 226, - 13, 70, 230, 43, 73, 203, 9, 77, 10, 50, 65, 62, 74, 194, 7, 75, 14, 72, - 195, 4, 89, 2, 217, 12, 11, 76, 69, 70, 32, 77, 65, 75, 83, 85, 82, 65, - 2, 225, 7, 3, 69, 69, 77, 28, 56, 2, 65, 76, 117, 8, 69, 72, 32, 87, 73, - 84, 72, 32, 2, 37, 7, 32, 87, 73, 84, 72, 32, 83, 2, 197, 1, 15, 85, 80, - 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, 76, 69, 70, 26, 108, 3, 74, 69, - 69, 126, 65, 198, 4, 77, 86, 78, 78, 90, 242, 2, 82, 42, 89, 233, 53, 5, - 72, 69, 72, 32, 77, 2, 11, 77, 2, 11, 32, 2, 143, 63, 73, 114, 82, 65, - 38, 72, 246, 4, 78, 78, 90, 38, 74, 98, 75, 42, 77, 198, 1, 82, 43, 89, - 4, 217, 2, 5, 76, 69, 70, 32, 77, 76, 22, 65, 139, 3, 69, 72, 80, 15, 77, - 90, 65, 32, 65, 66, 79, 86, 69, 32, 87, 73, 84, 72, 32, 147, 5, 72, 66, - 118, 65, 126, 69, 42, 72, 78, 74, 18, 75, 62, 77, 86, 78, 22, 79, 16, 2, - 87, 65, 18, 89, 26, 90, 242, 2, 82, 67, 85, 12, 22, 76, 247, 6, 69, 8, - 21, 3, 69, 70, 32, 8, 34, 77, 222, 6, 70, 247, 52, 73, 4, 181, 6, 6, 65, - 75, 83, 85, 82, 65, 6, 11, 32, 6, 166, 6, 70, 231, 43, 73, 8, 22, 69, - 191, 3, 65, 4, 11, 72, 4, 11, 32, 4, 230, 4, 73, 163, 54, 77, 4, 251, 48, - 69, 2, 11, 72, 2, 11, 65, 2, 11, 72, 2, 149, 4, 2, 32, 73, 8, 17, 2, 69, - 69, 8, 11, 77, 8, 11, 32, 8, 198, 4, 70, 230, 43, 73, 203, 9, 77, 2, 93, - 2, 79, 79, 4, 231, 3, 69, 4, 215, 3, 87, 8, 186, 3, 69, 15, 85, 2, 17, 2, - 65, 73, 2, 239, 2, 78, 6, 21, 3, 69, 69, 77, 6, 11, 32, 6, 22, 87, 219, - 46, 73, 2, 141, 2, 5, 73, 84, 72, 32, 89, 4, 11, 72, 4, 11, 65, 4, 135, - 46, 72, 14, 21, 3, 69, 69, 77, 14, 11, 32, 14, 64, 5, 87, 73, 84, 72, 32, - 194, 1, 70, 230, 43, 73, 203, 9, 77, 6, 18, 77, 75, 89, 4, 21, 3, 69, 69, - 77, 4, 11, 32, 4, 18, 73, 119, 70, 2, 239, 44, 78, 2, 17, 2, 69, 72, 2, - 77, 2, 32, 70, 4, 11, 69, 4, 11, 72, 4, 11, 32, 4, 22, 70, 247, 52, 73, - 2, 205, 53, 2, 73, 78, 6, 202, 43, 73, 203, 9, 77, 166, 2, 92, 3, 68, 68, - 65, 52, 3, 82, 75, 32, 109, 11, 84, 72, 69, 77, 65, 84, 73, 67, 65, 76, - 32, 4, 148, 231, 35, 5, 32, 87, 65, 65, 74, 191, 61, 72, 4, 58, 78, 1, - 10, 83, 73, 68, 69, 87, 65, 89, 83, 32, 78, 2, 221, 254, 33, 7, 79, 79, - 78, 32, 71, 72, 85, 158, 2, 178, 2, 68, 144, 3, 8, 73, 78, 73, 84, 73, - 65, 76, 32, 222, 1, 76, 132, 2, 9, 79, 80, 69, 82, 65, 84, 79, 82, 32, - 166, 1, 83, 146, 3, 84, 158, 205, 23, 65, 102, 75, 110, 90, 246, 104, 74, - 2, 77, 166, 162, 5, 72, 170, 133, 1, 66, 2, 70, 2, 82, 2, 89, 174, 57, - 71, 198, 232, 2, 78, 154, 144, 2, 81, 255, 2, 87, 62, 26, 79, 187, 155, - 36, 65, 58, 88, 6, 84, 76, 69, 83, 83, 32, 61, 12, 85, 66, 76, 69, 45, - 83, 84, 82, 85, 67, 75, 32, 8, 214, 233, 30, 66, 2, 70, 242, 161, 3, 78, - 155, 144, 2, 81, 50, 190, 12, 83, 186, 20, 75, 158, 183, 23, 84, 74, 90, - 246, 104, 74, 2, 77, 166, 162, 5, 72, 170, 133, 1, 66, 2, 70, 2, 82, 2, - 89, 174, 57, 71, 198, 232, 2, 78, 246, 215, 1, 76, 150, 55, 68, 146, 1, - 81, 222, 1, 65, 163, 1, 87, 40, 182, 1, 84, 174, 9, 83, 150, 203, 23, 72, - 30, 75, 226, 105, 74, 2, 77, 206, 167, 6, 66, 2, 70, 2, 89, 174, 57, 71, - 198, 232, 2, 78, 246, 215, 1, 76, 166, 56, 81, 222, 1, 65, 159, 57, 68, - 4, 134, 230, 30, 72, 147, 178, 6, 69, 56, 48, 6, 79, 79, 80, 69, 68, 32, - 251, 135, 37, 65, 54, 210, 8, 83, 186, 20, 75, 150, 182, 23, 65, 74, 72, - 66, 84, 74, 90, 246, 104, 74, 2, 77, 206, 167, 6, 66, 2, 70, 2, 82, 2, - 89, 174, 57, 71, 198, 232, 2, 78, 246, 215, 1, 76, 150, 55, 68, 146, 1, - 81, 255, 2, 87, 4, 160, 154, 28, 23, 77, 69, 69, 77, 32, 87, 73, 84, 72, - 32, 72, 65, 72, 32, 87, 73, 84, 72, 32, 84, 65, 84, 87, 177, 233, 7, 9, - 72, 65, 72, 32, 87, 73, 84, 72, 32, 52, 84, 9, 84, 82, 69, 84, 67, 72, - 69, 68, 32, 134, 196, 35, 72, 14, 69, 227, 138, 1, 65, 46, 182, 1, 68, - 86, 84, 254, 2, 83, 150, 203, 23, 72, 30, 75, 226, 105, 74, 2, 77, 166, - 162, 5, 90, 170, 133, 1, 66, 2, 70, 2, 89, 174, 57, 71, 198, 232, 2, 78, - 154, 144, 2, 81, 223, 1, 65, 6, 52, 7, 79, 84, 76, 69, 83, 83, 32, 131, - 205, 36, 65, 4, 242, 223, 30, 66, 3, 70, 6, 214, 223, 30, 72, 146, 178, - 6, 65, 3, 69, 38, 42, 65, 158, 206, 23, 72, 247, 194, 13, 69, 32, 44, 5, - 73, 76, 69, 68, 32, 243, 145, 37, 72, 30, 150, 1, 68, 94, 83, 186, 20, - 75, 218, 160, 24, 74, 166, 162, 5, 72, 170, 133, 1, 89, 174, 57, 71, 198, - 232, 2, 78, 246, 215, 1, 76, 166, 56, 81, 223, 1, 65, 6, 52, 7, 79, 84, - 76, 69, 83, 83, 32, 143, 202, 36, 65, 4, 238, 254, 33, 78, 155, 144, 2, - 81, 6, 250, 190, 35, 72, 14, 69, 227, 138, 1, 65, 4, 238, 252, 17, 77, - 147, 140, 18, 83, 6, 128, 149, 24, 3, 75, 65, 83, 12, 4, 68, 65, 77, 77, - 1, 4, 70, 65, 84, 72, 10, 130, 1, 79, 212, 156, 12, 11, 76, 65, 67, 69, - 32, 79, 70, 32, 83, 65, 74, 144, 222, 5, 6, 73, 65, 83, 84, 82, 69, 175, - 173, 13, 69, 4, 148, 141, 12, 8, 69, 84, 73, 67, 32, 86, 69, 82, 205, - 237, 5, 3, 85, 78, 68, 14, 238, 1, 65, 96, 5, 69, 86, 69, 82, 83, 36, 15, - 73, 71, 72, 84, 32, 65, 82, 82, 79, 87, 72, 69, 65, 68, 32, 201, 206, 22, - 28, 79, 85, 78, 68, 69, 68, 32, 72, 73, 71, 72, 32, 83, 84, 79, 80, 32, - 87, 73, 84, 72, 32, 70, 73, 76, 76, 69, 68, 4, 40, 4, 73, 83, 69, 68, - 239, 138, 37, 89, 2, 17, 2, 32, 82, 2, 165, 193, 35, 3, 79, 85, 78, 2, - 133, 135, 21, 4, 69, 68, 32, 68, 6, 26, 65, 195, 241, 34, 66, 4, 193, - 230, 35, 3, 66, 79, 86, 202, 1, 238, 1, 69, 244, 2, 5, 72, 65, 68, 68, - 65, 36, 4, 73, 71, 78, 32, 240, 4, 5, 77, 65, 76, 76, 32, 246, 15, 85, - 244, 2, 6, 89, 77, 66, 79, 76, 32, 209, 178, 35, 18, 84, 65, 82, 84, 32, - 79, 70, 32, 82, 85, 66, 32, 69, 76, 32, 72, 73, 90, 20, 52, 7, 81, 85, - 69, 78, 67, 69, 32, 223, 209, 33, 77, 18, 184, 1, 26, 89, 69, 72, 32, 87, - 73, 84, 72, 32, 72, 65, 77, 90, 65, 32, 65, 66, 79, 86, 69, 32, 87, 73, - 84, 72, 32, 209, 210, 30, 13, 78, 79, 79, 78, 32, 87, 73, 84, 72, 32, 75, - 69, 72, 16, 70, 65, 142, 135, 36, 87, 182, 89, 89, 150, 14, 79, 214, 22, - 69, 3, 85, 6, 36, 3, 76, 69, 70, 131, 133, 37, 69, 5, 159, 13, 32, 7, 11, - 32, 4, 246, 22, 73, 59, 77, 22, 132, 1, 2, 82, 65, 158, 1, 83, 188, 1, 7, - 65, 76, 65, 89, 72, 69, 32, 160, 15, 2, 77, 73, 209, 148, 33, 6, 84, 65, - 75, 72, 65, 76, 4, 132, 1, 13, 72, 77, 65, 84, 85, 76, 76, 65, 72, 32, - 65, 76, 65, 213, 192, 32, 13, 68, 73, 32, 65, 76, 76, 65, 72, 79, 85, 32, - 65, 78, 2, 175, 213, 36, 89, 12, 46, 65, 193, 1, 6, 73, 78, 68, 72, 73, - 32, 8, 136, 1, 18, 76, 76, 65, 76, 76, 65, 72, 79, 85, 32, 65, 76, 65, - 89, 72, 69, 32, 87, 166, 200, 29, 78, 142, 193, 5, 70, 237, 99, 2, 77, - 86, 2, 17, 2, 65, 83, 2, 229, 173, 15, 3, 83, 65, 76, 4, 140, 249, 19, - 12, 80, 79, 83, 84, 80, 79, 83, 73, 84, 73, 79, 78, 233, 205, 13, 2, 65, - 77, 106, 132, 1, 2, 70, 65, 32, 5, 72, 73, 71, 72, 32, 240, 11, 4, 76, - 79, 87, 32, 110, 75, 170, 238, 20, 68, 146, 208, 9, 89, 135, 181, 5, 87, - 4, 190, 3, 82, 219, 131, 35, 84, 74, 236, 2, 17, 68, 79, 84, 76, 69, 83, - 83, 32, 72, 69, 65, 68, 32, 79, 70, 32, 75, 22, 70, 102, 76, 142, 3, 77, - 116, 4, 78, 79, 79, 78, 18, 83, 78, 84, 38, 87, 196, 2, 3, 89, 69, 72, - 228, 128, 14, 18, 85, 80, 82, 73, 71, 72, 84, 32, 82, 69, 67, 84, 65, 78, - 71, 85, 76, 65, 196, 180, 8, 7, 82, 79, 85, 78, 68, 69, 68, 130, 122, 90, - 246, 104, 74, 214, 217, 11, 81, 223, 1, 65, 2, 251, 194, 29, 72, 4, 24, - 2, 65, 82, 31, 79, 2, 11, 83, 2, 175, 2, 73, 2, 181, 220, 25, 6, 79, 84, - 78, 79, 84, 69, 10, 60, 8, 73, 71, 65, 84, 85, 82, 69, 32, 213, 11, 2, - 65, 77, 8, 88, 10, 65, 76, 69, 70, 32, 87, 73, 84, 72, 32, 116, 3, 81, - 65, 70, 1, 3, 83, 65, 68, 4, 84, 8, 76, 65, 77, 32, 87, 73, 84, 72, 153, - 140, 35, 7, 89, 69, 72, 32, 66, 65, 82, 2, 181, 197, 30, 2, 32, 89, 2, + 44, 5, 69, 32, 79, 70, 32, 175, 226, 39, 73, 8, 68, 8, 83, 65, 76, 84, + 32, 79, 70, 32, 130, 3, 65, 247, 172, 27, 67, 4, 254, 2, 65, 247, 172, + 27, 67, 2, 219, 184, 39, 70, 12, 68, 3, 65, 82, 84, 32, 2, 73, 78, 34, + 82, 157, 195, 38, 2, 85, 84, 4, 11, 65, 4, 179, 174, 26, 82, 4, 182, 154, + 35, 67, 167, 49, 32, 2, 233, 236, 39, 2, 73, 68, 14, 50, 73, 145, 140, + 38, 6, 69, 82, 68, 73, 71, 82, 12, 64, 5, 78, 69, 71, 65, 82, 237, 172, + 26, 5, 84, 82, 73, 79, 76, 9, 44, 5, 32, 79, 70, 32, 65, 223, 173, 40, + 45, 2, 181, 144, 24, 3, 78, 84, 73, 4, 194, 240, 39, 84, 239, 61, 88, 6, + 38, 77, 194, 154, 39, 70, 255, 118, 82, 2, 223, 133, 39, 66, 22, 104, 7, + 77, 79, 78, 83, 84, 69, 82, 138, 1, 83, 241, 204, 35, 10, 67, 82, 65, 66, + 32, 83, 84, 69, 80, 80, 11, 11, 32, 8, 88, 6, 67, 76, 79, 83, 69, 68, 0, + 4, 79, 80, 69, 78, 237, 170, 36, 4, 83, 84, 69, 80, 2, 205, 208, 37, 3, + 32, 74, 65, 8, 60, 6, 80, 73, 68, 69, 82, 32, 57, 5, 81, 85, 73, 68, 32, + 4, 140, 153, 29, 5, 67, 82, 79, 85, 67, 215, 245, 1, 83, 4, 56, 6, 67, + 76, 79, 83, 69, 68, 1, 4, 79, 80, 69, 78, 2, 217, 207, 27, 5, 32, 84, 69, + 78, 84, 4, 190, 214, 33, 69, 209, 174, 4, 11, 65, 82, 79, 85, 78, 68, 45, + 80, 82, 79, 70, 9, 49, 10, 79, 83, 84, 32, 69, 81, 85, 65, 76, 32, 6, 32, + 2, 84, 79, 251, 151, 32, 79, 5, 227, 232, 6, 32, 4, 232, 219, 10, 3, 73, + 86, 69, 253, 163, 28, 5, 69, 32, 79, 78, 69, 10, 162, 1, 80, 200, 165, + 29, 6, 69, 82, 73, 67, 65, 78, 188, 249, 5, 3, 66, 85, 76, 197, 248, 3, + 16, 65, 76, 71, 65, 77, 65, 84, 73, 79, 78, 32, 79, 82, 32, 67, 79, 4, + 210, 215, 36, 69, 139, 94, 72, 194, 9, 92, 3, 65, 84, 79, 174, 20, 71, + 174, 1, 84, 146, 248, 24, 67, 222, 139, 12, 68, 139, 139, 3, 75, 144, 9, + 104, 17, 76, 73, 65, 78, 32, 72, 73, 69, 82, 79, 71, 76, 89, 80, 72, 32, + 65, 141, 148, 13, 3, 77, 73, 67, 142, 9, 70, 48, 138, 4, 49, 198, 2, 50, + 162, 3, 51, 174, 5, 52, 163, 3, 53, 222, 1, 106, 50, 102, 52, 110, 54, + 102, 57, 182, 7, 51, 210, 219, 11, 49, 206, 134, 23, 48, 242, 1, 53, 2, + 55, 3, 56, 22, 170, 183, 34, 54, 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, 2, + 52, 2, 53, 2, 55, 2, 56, 3, 57, 28, 158, 164, 12, 54, 170, 146, 22, 49, + 2, 53, 134, 236, 5, 48, 2, 50, 2, 51, 2, 52, 2, 55, 2, 56, 3, 57, 26, + 246, 138, 12, 54, 234, 150, 28, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, + 55, 2, 56, 3, 57, 24, 246, 180, 34, 55, 2, 56, 134, 236, 5, 48, 2, 49, 2, + 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 57, 232, 1, 98, 48, 114, 49, 162, 143, + 12, 50, 2, 51, 230, 217, 22, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 42, + 198, 136, 12, 52, 2, 55, 190, 24, 53, 170, 146, 22, 48, 2, 49, 2, 50, + 134, 236, 5, 51, 2, 54, 2, 56, 3, 57, 26, 146, 160, 12, 48, 170, 146, 22, + 53, 134, 236, 5, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, + 222, 1, 102, 48, 110, 49, 102, 57, 210, 1, 56, 206, 253, 11, 50, 2, 54, + 194, 230, 22, 51, 2, 52, 2, 53, 3, 55, 28, 186, 158, 12, 50, 170, 146, + 22, 55, 2, 57, 134, 236, 5, 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 54, 3, 56, + 24, 246, 175, 34, 53, 2, 54, 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, + 2, 55, 2, 56, 3, 57, 24, 146, 175, 34, 52, 2, 57, 134, 236, 5, 48, 2, 49, + 2, 50, 2, 51, 2, 53, 2, 54, 2, 55, 3, 56, 228, 1, 102, 48, 2, 50, 2, 53, + 102, 51, 110, 54, 102, 56, 162, 1, 57, 130, 230, 11, 55, 186, 250, 22, + 49, 3, 52, 22, 194, 173, 34, 57, 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, 2, + 52, 2, 53, 2, 54, 2, 55, 3, 56, 30, 250, 129, 12, 54, 150, 196, 8, 50, + 214, 210, 19, 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 24, + 242, 171, 34, 52, 2, 56, 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, 2, 53, 2, + 54, 2, 55, 3, 57, 26, 98, 51, 174, 170, 34, 49, 2, 54, 134, 236, 5, 48, + 2, 50, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 4, 216, 246, 20, 6, 32, 82, 65, + 32, 79, 82, 215, 159, 19, 65, 20, 160, 235, 39, 3, 51, 32, 69, 210, 42, + 48, 2, 49, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 202, 1, 102, + 49, 210, 1, 53, 202, 205, 20, 57, 202, 143, 14, 48, 2, 50, 2, 51, 2, 52, + 2, 54, 2, 55, 3, 56, 22, 90, 48, 194, 147, 40, 49, 2, 50, 2, 51, 2, 52, + 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 4, 60, 6, 32, 66, 69, 71, 73, 78, 1, + 5, 65, 32, 69, 78, 68, 2, 213, 167, 16, 8, 32, 76, 79, 71, 79, 71, 82, + 65, 24, 198, 166, 34, 48, 2, 55, 134, 236, 5, 49, 2, 50, 2, 51, 2, 52, 2, + 53, 2, 54, 2, 56, 3, 57, 60, 150, 249, 32, 51, 166, 225, 1, 48, 242, 1, + 49, 3, 50, 14, 96, 2, 76, 69, 198, 162, 2, 85, 136, 162, 32, 2, 83, 84, + 242, 232, 3, 69, 241, 165, 1, 2, 82, 89, 7, 33, 6, 32, 87, 73, 84, 72, + 32, 4, 182, 236, 31, 83, 231, 184, 4, 85, 31, 76, 4, 69, 78, 78, 65, 41, + 11, 73, 67, 76, 79, 67, 75, 87, 73, 83, 69, 32, 5, 241, 230, 36, 5, 32, + 87, 73, 84, 72, 24, 90, 84, 222, 248, 6, 67, 58, 68, 122, 71, 138, 4, 79, + 153, 134, 28, 5, 73, 78, 84, 69, 71, 12, 84, 15, 82, 73, 65, 78, 71, 76, + 69, 45, 72, 69, 65, 68, 69, 68, 32, 155, 253, 6, 79, 10, 112, 3, 76, 69, + 70, 0, 4, 82, 73, 71, 72, 12, 6, 66, 79, 84, 84, 79, 77, 0, 3, 84, 79, + 80, 191, 252, 6, 79, 2, 11, 84, 2, 193, 178, 37, 7, 32, 85, 45, 83, 72, + 65, 80, 158, 1, 128, 1, 20, 76, 32, 70, 85, 78, 67, 84, 73, 79, 78, 65, + 76, 32, 83, 89, 77, 66, 79, 76, 32, 206, 15, 79, 38, 80, 179, 251, 39, + 67, 140, 1, 222, 2, 67, 186, 1, 68, 190, 2, 73, 44, 4, 65, 76, 80, 72, 0, + 4, 79, 77, 69, 71, 32, 4, 74, 79, 84, 32, 36, 4, 76, 69, 70, 84, 68, 2, + 81, 85, 218, 3, 82, 54, 83, 92, 4, 69, 80, 83, 73, 32, 6, 66, 65, 67, 75, + 83, 76, 76, 2, 85, 80, 156, 136, 16, 12, 71, 82, 69, 65, 84, 69, 82, 45, + 84, 72, 65, 78, 0, 5, 84, 73, 76, 68, 69, 203, 145, 20, 90, 14, 64, 6, + 73, 82, 67, 76, 69, 32, 169, 230, 38, 4, 79, 77, 77, 65, 12, 80, 2, 83, + 84, 190, 206, 19, 68, 234, 131, 6, 66, 246, 201, 10, 85, 215, 154, 3, 74, + 4, 250, 226, 38, 73, 207, 107, 65, 22, 76, 2, 69, 76, 120, 3, 79, 87, 78, + 129, 244, 31, 6, 73, 65, 77, 79, 78, 68, 10, 30, 32, 53, 3, 84, 65, 32, + 6, 170, 205, 19, 68, 142, 213, 16, 84, 223, 190, 1, 83, 4, 210, 154, 36, + 85, 143, 198, 1, 83, 10, 22, 32, 167, 9, 87, 8, 52, 5, 84, 65, 67, 75, + 32, 182, 1, 83, 227, 6, 67, 4, 222, 153, 36, 85, 215, 154, 3, 74, 6, 40, + 2, 79, 84, 225, 129, 32, 2, 45, 66, 4, 11, 65, 5, 223, 242, 31, 32, 4, + 146, 203, 19, 68, 223, 205, 16, 85, 4, 28, 2, 32, 83, 187, 7, 87, 2, 169, + 222, 37, 4, 72, 79, 69, 32, 46, 44, 2, 65, 68, 133, 3, 4, 79, 84, 69, 32, + 43, 11, 32, 40, 178, 1, 67, 38, 68, 92, 2, 85, 80, 36, 2, 76, 69, 142, + 186, 6, 81, 176, 216, 13, 3, 78, 79, 84, 22, 69, 178, 184, 5, 66, 194, + 158, 7, 83, 170, 132, 5, 71, 162, 114, 82, 195, 79, 74, 4, 178, 175, 38, + 73, 251, 129, 1, 79, 12, 54, 73, 36, 3, 79, 87, 78, 213, 199, 36, 2, 69, + 76, 4, 130, 221, 31, 86, 251, 230, 6, 65, 4, 222, 245, 3, 32, 255, 234, + 34, 87, 4, 162, 238, 37, 83, 255, 113, 70, 4, 238, 153, 12, 81, 231, 250, + 23, 85, 4, 184, 3, 5, 73, 71, 72, 84, 87, 143, 221, 39, 72, 10, 88, 5, + 69, 77, 73, 67, 79, 34, 76, 34, 84, 253, 151, 12, 7, 81, 85, 73, 83, 72, + 32, 81, 2, 237, 236, 31, 3, 76, 79, 78, 2, 185, 220, 38, 3, 65, 83, 72, + 4, 196, 136, 16, 2, 65, 82, 163, 166, 7, 73, 12, 22, 32, 171, 1, 87, 10, + 78, 67, 36, 5, 84, 65, 67, 75, 32, 253, 171, 39, 6, 83, 72, 79, 69, 32, + 74, 2, 229, 173, 23, 4, 65, 82, 69, 84, 6, 202, 195, 19, 68, 198, 138, + 17, 79, 239, 221, 2, 74, 2, 209, 222, 36, 6, 65, 82, 68, 83, 32, 86, 4, + 178, 227, 33, 83, 171, 200, 5, 76, 12, 92, 2, 82, 79, 177, 159, 26, 15, + 76, 73, 67, 65, 84, 73, 79, 78, 32, 80, 82, 79, 71, 82, 65, 10, 112, 9, + 88, 73, 77, 65, 84, 69, 76, 89, 32, 165, 221, 39, 13, 65, 67, 72, 69, 83, + 32, 84, 72, 69, 32, 76, 73, 77, 8, 76, 6, 69, 81, 85, 65, 76, 32, 245, + 242, 24, 7, 66, 85, 84, 32, 78, 79, 84, 6, 32, 2, 84, 79, 135, 232, 31, + 79, 5, 185, 160, 26, 13, 32, 79, 82, 32, 84, 72, 69, 32, 73, 77, 65, 71, + 69, 240, 22, 148, 1, 4, 65, 66, 73, 67, 156, 246, 1, 7, 77, 69, 78, 73, + 65, 78, 32, 224, 12, 3, 82, 79, 87, 228, 3, 2, 84, 73, 174, 220, 36, 73, + 251, 147, 1, 67, 158, 21, 54, 32, 217, 244, 1, 7, 45, 73, 78, 68, 73, 67, + 32, 130, 21, 178, 3, 67, 238, 1, 68, 246, 2, 69, 186, 1, 70, 212, 1, 2, + 72, 65, 104, 5, 75, 65, 83, 82, 65, 86, 76, 248, 167, 1, 2, 77, 65, 204, + 18, 7, 78, 85, 77, 66, 69, 82, 32, 36, 5, 79, 80, 69, 78, 32, 82, 80, + 238, 1, 82, 156, 2, 8, 66, 65, 83, 69, 76, 73, 78, 69, 52, 6, 73, 78, 86, + 69, 82, 84, 98, 83, 150, 33, 84, 202, 4, 86, 232, 184, 17, 9, 87, 65, 86, + 89, 32, 72, 65, 77, 90, 190, 249, 17, 81, 145, 197, 1, 6, 90, 87, 65, 82, + 65, 75, 16, 44, 2, 79, 77, 81, 5, 85, 82, 76, 89, 32, 4, 216, 158, 19, + 11, 66, 73, 78, 73, 78, 71, 32, 65, 76, 69, 70, 207, 209, 20, 77, 12, 72, + 4, 68, 65, 77, 77, 0, 4, 70, 65, 84, 72, 1, 4, 75, 65, 83, 82, 4, 11, 65, + 5, 183, 167, 38, 84, 24, 158, 1, 65, 108, 5, 79, 85, 66, 76, 69, 188, + 206, 23, 5, 69, 67, 73, 77, 65, 209, 192, 9, 16, 73, 83, 80, 85, 84, 69, + 68, 32, 69, 78, 68, 32, 79, 70, 32, 65, 14, 36, 3, 77, 77, 65, 147, 178, + 33, 84, 13, 22, 32, 199, 5, 84, 6, 154, 226, 1, 73, 54, 77, 135, 233, 36, + 87, 6, 190, 4, 68, 165, 193, 1, 18, 32, 82, 73, 71, 72, 84, 32, 65, 82, + 82, 79, 87, 72, 69, 65, 68, 32, 65, 8, 88, 12, 77, 80, 84, 89, 32, 67, + 69, 78, 84, 82, 69, 32, 57, 6, 78, 68, 32, 79, 70, 32, 4, 144, 156, 37, + 4, 72, 73, 71, 72, 1, 3, 76, 79, 87, 4, 146, 223, 29, 84, 255, 173, 3, + 65, 22, 84, 4, 65, 84, 72, 65, 238, 209, 1, 79, 156, 223, 4, 3, 73, 86, + 69, 195, 233, 30, 85, 17, 22, 32, 139, 2, 84, 10, 52, 5, 87, 73, 84, 72, + 32, 170, 222, 1, 73, 55, 77, 6, 250, 154, 38, 84, 138, 80, 68, 203, 47, + 82, 6, 72, 13, 76, 70, 32, 77, 65, 68, 68, 65, 32, 79, 86, 69, 82, 235, + 16, 77, 2, 165, 205, 17, 2, 32, 77, 13, 18, 32, 43, 84, 6, 254, 71, 87, + 130, 149, 1, 73, 55, 77, 4, 173, 82, 2, 65, 78, 230, 15, 84, 5, 65, 82, + 71, 69, 32, 146, 1, 69, 149, 90, 8, 73, 71, 65, 84, 85, 82, 69, 32, 8, + 64, 10, 82, 79, 85, 78, 68, 32, 68, 79, 84, 32, 179, 245, 10, 67, 6, 148, + 245, 10, 6, 73, 78, 83, 73, 68, 69, 162, 213, 26, 66, 167, 161, 1, 65, + 130, 8, 80, 5, 84, 84, 69, 82, 32, 185, 150, 7, 9, 70, 84, 32, 65, 82, + 82, 79, 87, 72, 254, 7, 210, 2, 65, 240, 10, 2, 66, 69, 150, 4, 68, 186, + 5, 70, 246, 4, 71, 226, 2, 72, 156, 9, 2, 74, 69, 154, 1, 75, 130, 5, 76, + 222, 2, 77, 154, 1, 78, 140, 3, 3, 80, 69, 72, 176, 1, 3, 81, 65, 70, + 214, 1, 82, 230, 3, 83, 162, 9, 84, 174, 8, 85, 148, 2, 2, 86, 69, 28, 3, + 87, 65, 87, 202, 1, 89, 246, 3, 79, 236, 2, 2, 90, 65, 31, 69, 112, 92, + 7, 70, 82, 73, 67, 65, 78, 32, 92, 2, 73, 78, 144, 2, 3, 76, 69, 70, 131, + 225, 39, 69, 8, 52, 3, 81, 65, 70, 218, 251, 32, 70, 139, 193, 3, 78, 5, + 241, 63, 5, 32, 87, 73, 84, 72, 21, 11, 32, 18, 72, 6, 87, 73, 84, 72, + 32, 84, 150, 160, 1, 70, 238, 43, 73, 199, 9, 77, 10, 60, 10, 72, 82, 69, + 69, 32, 68, 79, 84, 83, 32, 147, 54, 87, 6, 160, 250, 14, 17, 80, 79, 73, + 78, 84, 73, 78, 71, 32, 68, 79, 87, 78, 87, 65, 82, 68, 198, 201, 22, 66, + 167, 161, 1, 65, 83, 11, 32, 80, 70, 87, 212, 44, 6, 77, 65, 75, 83, 85, + 82, 182, 113, 70, 255, 52, 73, 70, 48, 4, 73, 84, 72, 32, 161, 44, 3, 65, + 83, 76, 64, 192, 2, 9, 65, 84, 84, 65, 67, 72, 69, 68, 32, 220, 1, 19, + 82, 73, 71, 72, 84, 32, 77, 73, 68, 68, 76, 69, 32, 83, 84, 82, 79, 75, + 69, 188, 1, 6, 72, 65, 77, 90, 65, 32, 44, 8, 87, 65, 86, 89, 32, 72, 65, + 77, 134, 70, 69, 204, 1, 4, 77, 65, 68, 68, 136, 165, 36, 9, 76, 69, 70, + 84, 32, 77, 73, 68, 68, 207, 236, 1, 68, 28, 204, 1, 17, 66, 79, 84, 84, + 79, 77, 32, 82, 73, 71, 72, 84, 32, 75, 65, 83, 82, 0, 14, 84, 79, 80, + 32, 82, 73, 71, 72, 84, 32, 70, 65, 84, 72, 94, 82, 56, 3, 76, 69, 70, + 190, 201, 1, 75, 207, 149, 32, 70, 6, 11, 65, 7, 29, 5, 32, 65, 78, 68, + 32, 4, 240, 170, 19, 3, 76, 69, 70, 251, 175, 19, 68, 8, 52, 3, 73, 71, + 72, 253, 214, 1, 4, 79, 85, 78, 68, 4, 17, 2, 84, 32, 4, 166, 176, 1, 82, + 223, 37, 72, 12, 154, 72, 65, 37, 5, 66, 69, 76, 79, 87, 4, 179, 168, 37, + 90, 50, 22, 72, 227, 63, 69, 41, 22, 32, 203, 63, 69, 28, 68, 5, 87, 73, + 84, 72, 32, 194, 150, 1, 70, 238, 43, 73, 199, 9, 77, 20, 116, 6, 83, 77, + 65, 76, 76, 32, 34, 84, 254, 18, 73, 248, 32, 10, 68, 79, 84, 32, 66, 69, + 76, 79, 87, 32, 191, 20, 72, 6, 242, 38, 77, 231, 184, 23, 86, 8, 88, 10, + 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 193, 48, 7, 87, 79, 32, 68, 79, + 84, 83, 6, 144, 1, 22, 80, 79, 73, 78, 84, 73, 78, 71, 32, 85, 80, 87, + 65, 82, 68, 83, 32, 66, 69, 76, 79, 87, 161, 54, 8, 72, 79, 82, 73, 90, + 79, 78, 84, 5, 171, 237, 14, 32, 78, 90, 65, 136, 4, 2, 68, 65, 40, 7, + 79, 84, 76, 69, 83, 83, 32, 234, 27, 89, 247, 25, 85, 44, 34, 76, 254, 3, + 72, 235, 45, 68, 27, 11, 32, 24, 56, 5, 87, 73, 84, 72, 32, 222, 145, 1, + 70, 255, 52, 73, 20, 108, 9, 73, 78, 86, 69, 82, 84, 69, 68, 32, 34, 84, + 168, 1, 3, 68, 79, 84, 250, 246, 11, 70, 207, 137, 27, 82, 4, 238, 14, + 83, 163, 196, 39, 86, 8, 132, 1, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, + 32, 33, 18, 87, 79, 32, 68, 79, 84, 83, 32, 86, 69, 82, 84, 73, 67, 65, + 76, 76, 89, 4, 182, 54, 65, 163, 253, 36, 66, 4, 33, 6, 32, 66, 69, 76, + 79, 87, 5, 157, 154, 15, 11, 32, 65, 78, 68, 32, 83, 77, 65, 76, 76, 32, + 12, 22, 72, 243, 62, 76, 6, 235, 53, 65, 6, 222, 232, 32, 66, 2, 70, 175, + 230, 5, 81, 44, 60, 8, 65, 82, 83, 73, 32, 89, 69, 72, 169, 2, 2, 69, 72, + 23, 11, 32, 20, 68, 5, 87, 73, 84, 72, 32, 218, 140, 1, 70, 238, 43, 73, + 199, 9, 77, 12, 144, 1, 28, 69, 88, 84, 69, 78, 68, 69, 68, 32, 65, 82, + 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, 32, 68, 73, 71, 73, 84, 32, 30, + 84, 207, 39, 73, 6, 250, 8, 70, 255, 49, 84, 4, 150, 145, 7, 72, 179, + 212, 7, 87, 23, 11, 32, 20, 68, 5, 87, 73, 84, 72, 32, 178, 138, 1, 70, + 238, 43, 73, 199, 9, 77, 12, 32, 4, 68, 79, 84, 32, 51, 84, 6, 174, 40, + 66, 229, 190, 15, 4, 77, 79, 86, 69, 6, 64, 10, 72, 82, 69, 69, 32, 68, + 79, 84, 83, 32, 199, 157, 37, 87, 4, 198, 17, 80, 231, 155, 37, 66, 44, + 72, 2, 65, 70, 160, 1, 4, 72, 65, 73, 78, 222, 20, 85, 167, 179, 38, 82, + 19, 11, 32, 16, 68, 5, 87, 73, 84, 72, 32, 218, 135, 1, 70, 238, 43, 73, + 199, 9, 77, 8, 206, 23, 84, 170, 225, 38, 82, 241, 32, 8, 73, 78, 86, 69, + 82, 84, 69, 68, 15, 11, 32, 12, 68, 5, 87, 73, 84, 72, 32, 186, 134, 1, + 70, 238, 43, 73, 199, 9, 77, 4, 174, 37, 84, 131, 228, 26, 68, 82, 78, + 65, 236, 5, 2, 69, 72, 161, 2, 9, 73, 71, 72, 32, 72, 65, 77, 90, 65, 34, + 34, 72, 249, 47, 3, 77, 90, 65, 31, 11, 32, 28, 68, 5, 87, 73, 84, 72, + 32, 210, 132, 1, 70, 238, 43, 73, 199, 9, 77, 20, 132, 2, 29, 69, 88, 84, + 69, 78, 68, 69, 68, 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, + 32, 68, 73, 71, 73, 84, 32, 70, 30, 73, 92, 24, 83, 77, 65, 76, 76, 32, + 65, 82, 65, 66, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 84, 65, 72, 32, + 62, 84, 159, 52, 72, 2, 173, 186, 1, 2, 79, 85, 2, 45, 9, 78, 86, 69, 82, + 84, 69, 68, 32, 83, 2, 217, 150, 37, 6, 77, 65, 76, 76, 32, 86, 6, 26, + 65, 203, 165, 37, 66, 4, 222, 30, 78, 155, 168, 38, 66, 8, 88, 10, 72, + 82, 69, 69, 32, 68, 79, 84, 83, 32, 33, 8, 87, 79, 32, 68, 79, 84, 83, + 32, 4, 230, 8, 80, 139, 189, 38, 65, 4, 180, 197, 38, 8, 86, 69, 82, 84, + 73, 67, 65, 76, 27, 65, 41, 11, 32, 38, 144, 1, 4, 71, 79, 65, 76, 88, 5, + 87, 73, 84, 72, 32, 128, 47, 10, 68, 79, 65, 67, 72, 65, 83, 72, 77, 69, + 230, 78, 70, 238, 43, 73, 199, 9, 77, 13, 11, 32, 10, 180, 49, 6, 87, 73, + 84, 72, 32, 72, 250, 76, 70, 238, 43, 73, 199, 9, 77, 8, 134, 26, 73, + 237, 19, 3, 89, 69, 72, 9, 11, 32, 6, 138, 206, 23, 65, 238, 137, 9, 89, + 179, 233, 5, 87, 22, 28, 2, 69, 77, 167, 45, 72, 17, 11, 32, 14, 68, 6, + 87, 73, 84, 72, 32, 84, 138, 124, 70, 238, 43, 73, 199, 9, 77, 6, 166, + 214, 14, 87, 179, 201, 18, 72, 70, 98, 65, 204, 1, 4, 69, 72, 69, 72, + 176, 2, 7, 73, 82, 71, 72, 73, 90, 32, 197, 31, 2, 72, 65, 24, 46, 70, + 177, 162, 1, 5, 83, 72, 77, 73, 82, 23, 11, 32, 20, 64, 5, 87, 73, 84, + 72, 32, 130, 122, 70, 238, 43, 73, 199, 9, 77, 12, 42, 84, 134, 161, 35, + 68, 243, 201, 3, 82, 6, 222, 27, 87, 251, 241, 36, 72, 25, 11, 32, 22, + 64, 5, 87, 73, 84, 72, 32, 230, 120, 70, 238, 43, 73, 199, 9, 77, 14, 38, + 84, 234, 41, 83, 175, 144, 38, 68, 10, 60, 10, 72, 82, 69, 69, 32, 68, + 79, 84, 83, 32, 139, 26, 87, 6, 42, 80, 230, 155, 37, 66, 167, 161, 1, + 65, 2, 161, 140, 37, 14, 79, 73, 78, 84, 73, 78, 71, 32, 85, 80, 87, 65, + 82, 68, 12, 190, 39, 79, 13, 2, 89, 85, 26, 40, 2, 65, 77, 233, 170, 1, + 2, 79, 87, 25, 11, 32, 22, 64, 5, 87, 73, 84, 72, 32, 234, 117, 70, 238, + 43, 73, 199, 9, 77, 14, 88, 2, 68, 79, 36, 6, 83, 77, 65, 76, 76, 32, + 184, 152, 33, 2, 84, 72, 159, 253, 4, 66, 4, 146, 229, 17, 85, 155, 213, + 20, 84, 4, 196, 27, 16, 65, 82, 65, 66, 73, 67, 32, 76, 69, 84, 84, 69, + 82, 32, 84, 65, 199, 155, 39, 86, 18, 36, 3, 69, 69, 77, 231, 244, 38, + 65, 17, 11, 32, 14, 64, 5, 87, 73, 84, 72, 32, 146, 115, 70, 238, 43, 73, + 199, 9, 77, 6, 134, 18, 84, 187, 136, 35, 68, 60, 38, 71, 26, 89, 17, 3, + 79, 79, 78, 21, 22, 79, 235, 109, 32, 10, 239, 26, 69, 31, 11, 32, 28, + 92, 5, 71, 72, 85, 78, 78, 16, 5, 87, 73, 84, 72, 32, 166, 113, 70, 238, + 43, 73, 199, 9, 77, 6, 251, 33, 65, 14, 116, 6, 83, 77, 65, 76, 76, 32, + 38, 84, 176, 24, 8, 73, 78, 86, 69, 82, 84, 69, 68, 210, 218, 26, 68, + 171, 238, 11, 82, 4, 206, 210, 32, 84, 131, 224, 6, 86, 4, 246, 245, 6, + 72, 191, 142, 30, 87, 25, 22, 32, 143, 24, 69, 12, 88, 11, 87, 73, 84, + 72, 32, 83, 77, 65, 76, 76, 32, 242, 110, 70, 238, 43, 73, 199, 9, 77, 4, + 26, 77, 251, 176, 39, 86, 2, 241, 179, 38, 3, 69, 69, 77, 19, 11, 32, 16, + 64, 5, 87, 73, 84, 72, 32, 230, 109, 70, 238, 43, 73, 199, 9, 77, 8, 36, + 4, 68, 79, 84, 32, 183, 12, 84, 6, 44, 5, 66, 69, 76, 79, 87, 199, 178, + 38, 65, 5, 169, 199, 14, 6, 32, 65, 78, 68, 32, 78, 52, 108, 2, 69, 72, + 184, 28, 3, 82, 69, 72, 252, 2, 4, 78, 79, 79, 78, 205, 118, 7, 79, 72, + 73, 78, 71, 89, 65, 35, 11, 32, 32, 52, 5, 87, 73, 84, 72, 32, 174, 107, + 70, 255, 52, 73, 28, 134, 1, 83, 96, 2, 84, 87, 234, 5, 73, 230, 22, 72, + 206, 180, 11, 70, 144, 133, 7, 5, 68, 79, 84, 32, 66, 234, 39, 76, 215, + 220, 19, 82, 10, 44, 5, 77, 65, 76, 76, 32, 171, 252, 38, 84, 8, 198, 6, + 65, 186, 226, 6, 78, 135, 203, 16, 86, 4, 37, 7, 79, 32, 68, 79, 84, 83, + 32, 4, 166, 8, 86, 175, 166, 38, 65, 60, 184, 1, 2, 65, 68, 120, 3, 69, + 69, 78, 136, 6, 4, 72, 69, 69, 78, 204, 148, 1, 4, 85, 80, 69, 82, 180, + 172, 4, 7, 84, 82, 65, 73, 71, 72, 84, 181, 224, 32, 6, 87, 65, 83, 72, + 32, 75, 17, 11, 32, 14, 68, 6, 87, 73, 84, 72, 32, 84, 238, 102, 70, 238, + 43, 73, 199, 9, 77, 6, 186, 138, 33, 72, 223, 240, 3, 87, 27, 11, 32, 24, + 64, 5, 87, 73, 84, 72, 32, 250, 101, 70, 238, 43, 73, 199, 9, 77, 16, + 232, 1, 3, 68, 79, 84, 50, 73, 48, 7, 83, 77, 65, 76, 76, 32, 65, 110, + 84, 150, 202, 11, 70, 221, 152, 5, 31, 69, 88, 84, 69, 78, 68, 69, 68, + 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, 32, 68, 73, 71, 73, + 84, 32, 70, 79, 85, 2, 149, 209, 18, 7, 32, 66, 69, 76, 79, 87, 32, 2, + 173, 186, 17, 7, 78, 86, 69, 82, 84, 69, 68, 2, 85, 19, 82, 65, 66, 73, + 67, 32, 76, 69, 84, 84, 69, 82, 32, 84, 65, 72, 32, 65, 78, 2, 179, 247, + 22, 68, 6, 96, 11, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 66, 105, 9, + 87, 79, 32, 68, 79, 84, 83, 32, 86, 4, 25, 4, 69, 76, 79, 87, 5, 11, 32, + 2, 21, 3, 65, 78, 68, 2, 17, 2, 32, 84, 2, 247, 230, 6, 72, 2, 237, 198, + 11, 8, 69, 82, 84, 73, 67, 65, 76, 76, 13, 11, 32, 10, 46, 87, 134, 96, + 70, 238, 43, 73, 199, 9, 77, 2, 249, 226, 26, 5, 73, 84, 72, 32, 68, 120, + 92, 2, 65, 72, 132, 2, 4, 67, 72, 69, 72, 124, 2, 69, 72, 150, 3, 72, 61, + 3, 84, 69, 72, 21, 11, 32, 18, 64, 5, 87, 73, 84, 72, 32, 174, 94, 70, + 238, 43, 73, 199, 9, 77, 10, 26, 84, 139, 225, 26, 68, 8, 26, 87, 199, + 129, 33, 72, 4, 37, 7, 79, 32, 68, 79, 84, 83, 32, 4, 48, 6, 86, 69, 82, + 84, 73, 67, 211, 162, 38, 65, 2, 165, 195, 11, 3, 65, 76, 76, 25, 22, 32, + 163, 5, 69, 12, 64, 5, 87, 73, 84, 72, 32, 158, 92, 70, 238, 43, 73, 199, + 9, 77, 4, 198, 13, 83, 175, 144, 38, 68, 37, 22, 32, 167, 4, 69, 24, 62, + 77, 116, 5, 87, 73, 84, 72, 32, 178, 90, 70, 239, 43, 73, 10, 48, 6, 65, + 82, 66, 85, 84, 65, 175, 144, 1, 69, 9, 11, 32, 6, 226, 90, 70, 254, 52, + 73, 233, 131, 37, 2, 71, 79, 8, 104, 6, 83, 77, 65, 76, 76, 32, 56, 12, + 84, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 65, 175, 202, 38, 82, 4, 32, + 2, 84, 69, 199, 155, 39, 86, 2, 191, 158, 38, 72, 2, 161, 233, 27, 4, 66, + 79, 86, 69, 18, 42, 65, 126, 69, 209, 129, 1, 2, 73, 78, 6, 131, 9, 76, + 23, 18, 32, 91, 69, 10, 60, 4, 87, 73, 84, 72, 218, 87, 70, 238, 43, 73, + 199, 9, 77, 2, 129, 9, 2, 32, 83, 10, 131, 11, 72, 15, 158, 1, 32, 169, + 80, 33, 73, 71, 72, 85, 82, 32, 75, 65, 90, 65, 75, 72, 32, 75, 73, 82, + 71, 72, 73, 90, 32, 65, 76, 69, 70, 32, 77, 65, 75, 83, 85, 82, 65, 8, + 96, 16, 87, 73, 84, 72, 32, 72, 65, 77, 90, 65, 32, 65, 66, 79, 86, 69, + 174, 85, 70, 255, 52, 73, 5, 143, 75, 32, 17, 222, 8, 72, 167, 76, 32, + 25, 11, 32, 22, 52, 5, 87, 73, 84, 72, 32, 190, 84, 70, 255, 52, 73, 18, + 80, 4, 68, 79, 84, 32, 162, 2, 69, 182, 1, 72, 170, 170, 14, 84, 143, + 151, 24, 82, 4, 164, 152, 31, 3, 87, 73, 84, 251, 128, 7, 65, 54, 28, 2, + 69, 72, 227, 3, 85, 49, 11, 32, 46, 100, 6, 66, 65, 82, 82, 69, 69, 252, + 2, 5, 87, 73, 84, 72, 32, 170, 79, 70, 238, 43, 73, 199, 9, 77, 17, 11, + 32, 14, 52, 5, 87, 73, 84, 72, 32, 226, 81, 70, 255, 52, 73, 10, 22, 69, + 183, 1, 72, 4, 121, 28, 88, 84, 69, 78, 68, 69, 68, 32, 65, 82, 65, 66, + 73, 67, 45, 73, 78, 68, 73, 67, 32, 68, 73, 71, 73, 84, 32, 84, 4, 184, + 239, 35, 3, 72, 82, 69, 177, 166, 2, 2, 87, 79, 6, 21, 3, 65, 77, 90, 6, + 11, 65, 6, 17, 2, 32, 65, 6, 21, 3, 66, 79, 86, 6, 11, 69, 7, 159, 79, + 32, 22, 64, 10, 72, 65, 77, 90, 65, 32, 65, 66, 79, 86, 18, 83, 39, 84, + 10, 167, 2, 69, 2, 165, 165, 17, 4, 77, 65, 76, 76, 10, 108, 18, 87, 79, + 32, 68, 79, 84, 83, 32, 66, 69, 76, 79, 87, 32, 65, 78, 68, 32, 146, 220, + 36, 65, 183, 5, 72, 6, 70, 72, 132, 204, 6, 7, 83, 77, 65, 76, 76, 32, + 78, 207, 194, 31, 68, 2, 169, 236, 33, 3, 65, 77, 90, 18, 26, 72, 17, 2, + 73, 78, 11, 243, 71, 32, 9, 11, 32, 6, 158, 76, 70, 254, 52, 73, 157, 11, + 13, 87, 73, 84, 72, 32, 73, 78, 86, 69, 82, 84, 69, 68, 220, 7, 230, 5, + 65, 158, 5, 66, 176, 2, 9, 68, 65, 68, 32, 87, 73, 84, 72, 32, 80, 9, 70, + 69, 72, 32, 87, 73, 84, 72, 32, 72, 11, 71, 72, 65, 73, 78, 32, 87, 73, + 84, 72, 32, 154, 1, 72, 134, 3, 74, 190, 2, 75, 252, 1, 9, 76, 65, 77, + 32, 87, 73, 84, 72, 32, 226, 4, 77, 152, 4, 10, 78, 79, 79, 78, 32, 87, + 73, 84, 72, 32, 182, 2, 81, 190, 2, 82, 178, 2, 83, 146, 16, 84, 196, 9, + 7, 87, 65, 83, 65, 76, 76, 65, 40, 9, 89, 69, 72, 32, 87, 73, 84, 72, 32, + 216, 3, 53, 85, 73, 71, 72, 85, 82, 32, 75, 73, 82, 71, 72, 73, 90, 32, + 89, 69, 72, 32, 87, 73, 84, 72, 32, 72, 65, 77, 90, 65, 32, 65, 66, 79, + 86, 69, 32, 87, 73, 84, 72, 32, 65, 76, 69, 70, 32, 77, 65, 75, 83, 85, + 82, 65, 205, 6, 14, 90, 65, 72, 32, 87, 73, 84, 72, 32, 77, 69, 69, 77, + 32, 54, 128, 1, 8, 73, 78, 32, 87, 73, 84, 72, 32, 70, 76, 232, 57, 4, + 75, 66, 65, 82, 129, 197, 36, 8, 90, 90, 65, 32, 87, 65, 32, 74, 28, 152, + 19, 4, 77, 69, 69, 77, 150, 16, 74, 238, 23, 65, 255, 8, 89, 22, 56, 3, + 65, 89, 72, 216, 1, 3, 69, 70, 32, 207, 13, 76, 12, 30, 73, 122, 65, 151, + 56, 69, 8, 52, 9, 32, 65, 83, 45, 83, 65, 76, 65, 65, 47, 77, 4, 80, 4, + 84, 85, 32, 87, 207, 132, 39, 77, 4, 18, 65, 23, 32, 2, 17, 2, 65, 32, 2, + 137, 142, 33, 6, 65, 83, 45, 83, 65, 76, 8, 212, 65, 29, 77, 65, 75, 83, + 85, 82, 65, 32, 87, 73, 84, 72, 32, 83, 85, 80, 69, 82, 83, 67, 82, 73, + 80, 84, 32, 65, 76, 69, 70, 1, 13, 87, 73, 84, 72, 32, 70, 65, 84, 72, + 65, 84, 65, 78, 44, 160, 1, 8, 69, 72, 32, 87, 73, 84, 72, 32, 177, 222, + 25, 25, 73, 83, 77, 73, 76, 76, 65, 72, 32, 65, 82, 45, 82, 65, 72, 77, + 65, 78, 32, 65, 82, 45, 82, 65, 72, 42, 98, 72, 24, 3, 75, 72, 65, 230, + 52, 65, 250, 3, 74, 78, 77, 86, 78, 78, 90, 242, 2, 82, 43, 89, 10, 22, + 65, 171, 56, 69, 6, 131, 59, 72, 36, 222, 6, 72, 158, 7, 75, 218, 38, 65, + 250, 3, 74, 2, 77, 134, 5, 82, 3, 89, 30, 170, 13, 75, 218, 38, 65, 250, + 3, 74, 146, 2, 77, 110, 72, 139, 2, 89, 22, 64, 5, 77, 69, 69, 77, 32, + 250, 50, 65, 250, 3, 74, 135, 5, 89, 10, 40, 5, 87, 73, 84, 72, 32, 243, + 103, 73, 6, 174, 21, 77, 170, 25, 65, 203, 12, 89, 40, 80, 8, 65, 72, 32, + 87, 73, 84, 72, 32, 69, 8, 69, 72, 32, 87, 73, 84, 72, 32, 22, 236, 3, 4, + 77, 69, 69, 77, 226, 45, 65, 138, 6, 74, 247, 2, 89, 18, 164, 1, 5, 77, + 69, 69, 77, 32, 196, 11, 6, 65, 76, 69, 70, 32, 77, 150, 2, 89, 134, 38, + 74, 53, 16, 83, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, 76, 69, + 70, 8, 40, 5, 87, 73, 84, 72, 32, 223, 100, 73, 4, 254, 43, 74, 3, 77, + 30, 104, 9, 69, 69, 77, 32, 87, 73, 84, 72, 32, 205, 129, 4, 11, 65, 76, + 76, 65, 74, 65, 76, 65, 76, 79, 85, 28, 62, 72, 60, 5, 77, 69, 69, 77, + 32, 186, 45, 65, 255, 8, 89, 8, 17, 2, 65, 72, 8, 11, 32, 8, 178, 29, 87, + 199, 69, 73, 12, 40, 5, 87, 73, 84, 72, 32, 179, 98, 73, 8, 166, 37, 72, + 242, 3, 65, 203, 12, 89, 64, 84, 8, 65, 70, 32, 87, 73, 84, 72, 32, 93, + 9, 72, 65, 72, 32, 87, 73, 84, 72, 32, 46, 168, 46, 2, 65, 76, 218, 1, + 74, 96, 2, 76, 65, 146, 2, 75, 14, 72, 30, 77, 239, 1, 89, 18, 54, 72, + 250, 42, 65, 250, 3, 74, 2, 77, 135, 5, 89, 2, 247, 8, 65, 74, 116, 5, + 65, 76, 69, 70, 32, 210, 1, 72, 124, 5, 74, 69, 69, 77, 32, 78, 75, 28, + 5, 77, 69, 69, 77, 32, 187, 47, 89, 20, 64, 5, 87, 73, 84, 72, 32, 138, + 44, 77, 222, 6, 70, 255, 52, 73, 12, 68, 6, 72, 65, 77, 90, 65, 32, 41, + 7, 77, 65, 68, 68, 65, 32, 65, 8, 38, 65, 249, 45, 4, 66, 69, 76, 79, 4, + 229, 45, 3, 66, 79, 86, 14, 28, 2, 65, 72, 163, 44, 69, 12, 11, 32, 12, + 40, 5, 87, 73, 84, 72, 32, 239, 92, 73, 8, 210, 35, 65, 130, 12, 77, 75, + 89, 14, 40, 5, 87, 73, 84, 72, 32, 159, 92, 73, 10, 130, 47, 74, 2, 77, + 75, 89, 8, 129, 15, 3, 72, 65, 72, 14, 150, 14, 87, 222, 33, 70, 238, 43, + 73, 199, 9, 77, 48, 84, 9, 69, 69, 77, 32, 87, 73, 84, 72, 32, 193, 36, + 7, 79, 72, 65, 77, 77, 65, 68, 46, 116, 5, 65, 76, 69, 70, 32, 66, 72, 0, + 2, 75, 72, 104, 5, 74, 69, 69, 77, 32, 92, 5, 77, 69, 69, 77, 32, 43, 89, + 4, 22, 77, 219, 45, 70, 2, 173, 35, 6, 65, 75, 83, 85, 82, 65, 10, 21, 3, + 65, 72, 32, 10, 40, 5, 87, 73, 84, 72, 32, 219, 88, 73, 6, 250, 31, 74, + 2, 77, 143, 12, 89, 12, 40, 5, 87, 73, 84, 72, 32, 139, 88, 73, 8, 170, + 31, 77, 194, 7, 75, 14, 72, 195, 4, 89, 8, 246, 40, 87, 246, 2, 70, 239, + 43, 73, 2, 11, 69, 2, 143, 33, 72, 60, 134, 1, 72, 28, 5, 74, 69, 69, 77, + 32, 92, 5, 77, 69, 69, 77, 32, 242, 31, 65, 154, 5, 78, 78, 90, 134, 1, + 75, 238, 1, 82, 43, 89, 14, 146, 27, 65, 155, 9, 69, 16, 40, 5, 87, 73, + 84, 72, 32, 199, 85, 73, 12, 186, 24, 72, 242, 3, 65, 130, 12, 77, 75, + 89, 12, 206, 15, 87, 218, 25, 70, 238, 43, 73, 199, 9, 77, 28, 70, 65, + 209, 138, 32, 11, 85, 68, 68, 73, 83, 65, 32, 83, 73, 82, 82, 26, 64, 7, + 70, 32, 87, 73, 84, 72, 32, 193, 8, 4, 76, 65, 32, 85, 24, 64, 5, 77, 69, + 69, 77, 32, 238, 29, 65, 246, 6, 72, 139, 2, 89, 12, 40, 5, 87, 73, 84, + 72, 32, 231, 82, 73, 8, 34, 77, 186, 21, 72, 187, 16, 89, 2, 129, 38, 3, + 69, 69, 77, 18, 30, 65, 129, 26, 2, 69, 72, 16, 128, 1, 14, 68, 73, 32, + 65, 76, 76, 65, 65, 72, 85, 32, 65, 78, 72, 76, 6, 72, 73, 77, 65, 72, + 85, 161, 26, 4, 83, 79, 85, 76, 11, 26, 85, 231, 228, 38, 65, 6, 26, 77, + 255, 170, 37, 78, 5, 199, 228, 38, 65, 4, 26, 32, 1, 2, 77, 32, 2, 205, + 134, 32, 4, 65, 76, 76, 65, 190, 1, 138, 1, 65, 192, 6, 9, 69, 69, 78, + 32, 87, 73, 84, 72, 32, 250, 3, 72, 233, 5, 13, 85, 66, 72, 65, 65, 78, + 65, 72, 85, 32, 87, 65, 32, 46, 48, 7, 68, 32, 87, 73, 84, 72, 32, 251, + 1, 76, 32, 76, 4, 72, 65, 72, 32, 82, 77, 238, 23, 65, 138, 4, 75, 246, + 4, 82, 3, 89, 10, 22, 87, 203, 77, 73, 6, 25, 4, 73, 84, 72, 32, 6, 162, + 16, 72, 187, 16, 89, 8, 21, 3, 69, 69, 77, 8, 11, 32, 8, 208, 31, 6, 87, + 73, 84, 72, 32, 77, 155, 45, 73, 14, 26, 65, 77, 2, 76, 65, 4, 134, 22, + 77, 129, 218, 36, 11, 65, 77, 85, 72, 85, 32, 65, 76, 65, 89, 78, 10, 34, + 32, 133, 1, 3, 76, 76, 65, 4, 22, 85, 167, 84, 73, 2, 217, 8, 23, 83, 69, + 68, 32, 65, 83, 32, 75, 79, 82, 65, 78, 73, 67, 32, 83, 84, 79, 80, 32, + 83, 73, 71, 6, 98, 72, 237, 222, 38, 18, 65, 72, 85, 32, 65, 76, 65, 89, + 72, 73, 32, 87, 65, 45, 65, 65, 76, 73, 4, 228, 63, 12, 79, 85, 32, 65, + 76, 65, 89, 72, 69, 32, 87, 65, 1, 22, 85, 32, 65, 76, 65, 89, 72, 73, + 32, 87, 65, 65, 65, 76, 73, 72, 69, 69, 32, 87, 65, 45, 60, 130, 1, 72, + 100, 5, 74, 69, 69, 77, 32, 84, 5, 75, 72, 65, 72, 32, 92, 5, 77, 69, 69, + 77, 32, 234, 15, 65, 254, 8, 82, 3, 89, 12, 32, 3, 65, 72, 32, 159, 21, + 69, 8, 156, 14, 6, 87, 73, 84, 72, 32, 74, 222, 56, 73, 199, 9, 77, 10, + 52, 5, 87, 73, 84, 72, 32, 134, 70, 73, 199, 9, 77, 4, 234, 12, 65, 139, + 8, 72, 10, 34, 87, 198, 69, 73, 199, 9, 77, 4, 25, 4, 73, 84, 72, 32, 4, + 142, 12, 65, 203, 12, 89, 16, 52, 5, 87, 73, 84, 72, 32, 214, 68, 73, + 199, 9, 77, 10, 202, 7, 72, 174, 4, 74, 199, 11, 77, 82, 96, 10, 65, 68, + 68, 65, 32, 87, 73, 84, 72, 32, 161, 1, 9, 69, 69, 78, 32, 87, 73, 84, + 72, 32, 18, 96, 4, 68, 65, 77, 77, 0, 4, 75, 65, 83, 82, 138, 11, 83, + 245, 41, 6, 70, 65, 84, 72, 65, 32, 6, 11, 65, 6, 28, 2, 84, 65, 211, 52, + 32, 2, 163, 12, 78, 64, 130, 1, 72, 36, 5, 74, 69, 69, 77, 32, 52, 5, 77, + 69, 69, 77, 32, 170, 11, 65, 228, 4, 4, 75, 72, 65, 72, 154, 4, 82, 3, + 89, 18, 194, 15, 69, 229, 3, 2, 65, 72, 10, 158, 18, 87, 246, 2, 70, 238, + 43, 73, 199, 9, 77, 16, 64, 5, 87, 73, 84, 72, 32, 158, 20, 70, 238, 43, + 73, 199, 9, 77, 8, 252, 2, 2, 75, 72, 243, 15, 77, 122, 66, 65, 176, 2, + 8, 69, 72, 32, 87, 73, 84, 72, 32, 179, 4, 72, 28, 88, 11, 66, 65, 65, + 82, 65, 75, 65, 32, 87, 65, 45, 29, 7, 72, 32, 87, 73, 84, 72, 32, 2, + 245, 133, 28, 2, 84, 65, 26, 64, 5, 77, 69, 69, 77, 32, 194, 8, 65, 246, + 6, 72, 139, 2, 89, 14, 52, 5, 87, 73, 84, 72, 32, 174, 61, 73, 199, 9, + 77, 8, 34, 72, 174, 4, 77, 143, 12, 89, 4, 133, 16, 2, 65, 72, 66, 138, + 1, 72, 108, 3, 75, 72, 65, 12, 4, 74, 69, 69, 77, 92, 5, 77, 69, 69, 77, + 32, 238, 4, 65, 154, 5, 78, 78, 90, 242, 2, 82, 43, 89, 14, 32, 3, 65, + 72, 32, 227, 9, 69, 10, 40, 5, 87, 73, 84, 72, 32, 151, 59, 73, 6, 182, + 2, 77, 199, 11, 74, 10, 11, 72, 10, 11, 32, 10, 40, 5, 87, 73, 84, 72, + 32, 183, 58, 73, 6, 154, 1, 65, 62, 77, 143, 12, 89, 18, 64, 5, 87, 73, + 84, 72, 32, 226, 13, 70, 238, 43, 73, 199, 9, 77, 10, 50, 65, 62, 74, + 194, 7, 75, 14, 72, 195, 4, 89, 2, 217, 12, 11, 76, 69, 70, 32, 77, 65, + 75, 83, 85, 82, 65, 2, 225, 7, 3, 69, 69, 77, 28, 56, 2, 65, 76, 117, 8, + 69, 72, 32, 87, 73, 84, 72, 32, 2, 37, 7, 32, 87, 73, 84, 72, 32, 83, 2, + 197, 1, 15, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, 76, 69, 70, + 26, 108, 3, 74, 69, 69, 126, 65, 198, 4, 77, 86, 78, 78, 90, 242, 2, 82, + 42, 89, 237, 53, 5, 72, 69, 72, 32, 77, 2, 11, 77, 2, 11, 32, 2, 151, 63, + 73, 114, 82, 65, 38, 72, 246, 4, 78, 78, 90, 38, 74, 98, 75, 42, 77, 198, + 1, 82, 43, 89, 4, 217, 2, 5, 76, 69, 70, 32, 77, 76, 22, 65, 139, 3, 69, + 72, 80, 15, 77, 90, 65, 32, 65, 66, 79, 86, 69, 32, 87, 73, 84, 72, 32, + 147, 5, 72, 66, 118, 65, 126, 69, 42, 72, 78, 74, 18, 75, 62, 77, 86, 78, + 22, 79, 16, 2, 87, 65, 18, 89, 26, 90, 242, 2, 82, 67, 85, 12, 22, 76, + 247, 6, 69, 8, 21, 3, 69, 70, 32, 8, 34, 77, 222, 6, 70, 255, 52, 73, 4, + 181, 6, 6, 65, 75, 83, 85, 82, 65, 6, 11, 32, 6, 166, 6, 70, 239, 43, 73, + 8, 22, 69, 191, 3, 65, 4, 11, 72, 4, 11, 32, 4, 230, 4, 73, 167, 54, 77, + 4, 131, 49, 69, 2, 11, 72, 2, 11, 65, 2, 11, 72, 2, 149, 4, 2, 32, 73, 8, + 17, 2, 69, 69, 8, 11, 77, 8, 11, 32, 8, 198, 4, 70, 238, 43, 73, 199, 9, + 77, 2, 93, 2, 79, 79, 4, 231, 3, 69, 4, 215, 3, 87, 8, 186, 3, 69, 15, + 85, 2, 17, 2, 65, 73, 2, 239, 2, 78, 6, 21, 3, 69, 69, 77, 6, 11, 32, 6, + 22, 87, 227, 46, 73, 2, 141, 2, 5, 73, 84, 72, 32, 89, 4, 11, 72, 4, 11, + 65, 4, 143, 46, 72, 14, 21, 3, 69, 69, 77, 14, 11, 32, 14, 64, 5, 87, 73, + 84, 72, 32, 194, 1, 70, 238, 43, 73, 199, 9, 77, 6, 18, 77, 75, 89, 4, + 21, 3, 69, 69, 77, 4, 11, 32, 4, 18, 73, 119, 70, 2, 247, 44, 78, 2, 17, + 2, 69, 72, 2, 77, 2, 32, 70, 4, 11, 69, 4, 11, 72, 4, 11, 32, 4, 22, 70, + 255, 52, 73, 2, 209, 53, 2, 73, 78, 6, 210, 43, 73, 199, 9, 77, 166, 2, + 92, 3, 68, 68, 65, 52, 3, 82, 75, 32, 109, 11, 84, 72, 69, 77, 65, 84, + 73, 67, 65, 76, 32, 4, 152, 133, 37, 5, 32, 87, 65, 65, 74, 239, 62, 72, + 4, 58, 78, 1, 10, 83, 73, 68, 69, 87, 65, 89, 83, 32, 78, 2, 173, 137, + 35, 7, 79, 79, 78, 32, 71, 72, 85, 158, 2, 174, 2, 68, 140, 3, 8, 73, 78, + 73, 84, 73, 65, 76, 32, 222, 1, 76, 132, 2, 9, 79, 80, 69, 82, 65, 84, + 79, 82, 32, 166, 1, 83, 142, 3, 84, 190, 162, 24, 65, 102, 75, 110, 90, + 134, 107, 74, 2, 77, 178, 187, 6, 66, 2, 70, 2, 82, 2, 89, 222, 7, 72, + 190, 59, 71, 242, 253, 2, 78, 166, 165, 2, 81, 135, 3, 87, 62, 26, 79, + 235, 186, 37, 65, 58, 88, 6, 84, 76, 69, 83, 83, 32, 61, 12, 85, 66, 76, + 69, 45, 83, 84, 82, 85, 67, 75, 32, 8, 226, 212, 31, 66, 2, 70, 138, 193, + 3, 78, 167, 165, 2, 81, 50, 178, 12, 83, 210, 20, 75, 170, 140, 24, 84, + 74, 90, 134, 107, 74, 2, 77, 178, 187, 6, 66, 2, 70, 2, 82, 2, 89, 222, + 7, 72, 190, 59, 71, 242, 253, 2, 78, 218, 235, 1, 76, 190, 56, 68, 146, + 1, 81, 222, 1, 65, 171, 1, 87, 40, 182, 1, 84, 166, 9, 83, 186, 160, 24, + 72, 30, 75, 242, 107, 74, 2, 77, 178, 187, 6, 66, 2, 70, 2, 89, 154, 67, + 71, 242, 253, 2, 78, 218, 235, 1, 76, 206, 57, 81, 222, 1, 65, 171, 57, + 68, 4, 150, 209, 31, 72, 207, 230, 6, 69, 56, 48, 6, 79, 79, 80, 69, 68, + 32, 199, 167, 38, 65, 54, 202, 8, 83, 210, 20, 75, 162, 139, 24, 65, 74, + 72, 66, 84, 74, 90, 134, 107, 74, 2, 77, 178, 187, 6, 66, 2, 70, 2, 82, + 2, 89, 154, 67, 71, 242, 253, 2, 78, 218, 235, 1, 76, 190, 56, 68, 146, + 1, 81, 135, 3, 87, 4, 168, 128, 29, 23, 77, 69, 69, 77, 32, 87, 73, 84, + 72, 32, 72, 65, 72, 32, 87, 73, 84, 72, 32, 84, 65, 84, 87, 221, 162, 8, + 9, 72, 65, 72, 32, 87, 73, 84, 72, 32, 52, 84, 9, 84, 82, 69, 84, 67, 72, + 69, 68, 32, 222, 224, 36, 72, 14, 69, 203, 141, 1, 65, 46, 178, 1, 68, + 86, 84, 250, 2, 83, 186, 160, 24, 72, 30, 75, 242, 107, 74, 2, 77, 178, + 187, 6, 66, 2, 70, 2, 89, 222, 7, 90, 190, 59, 71, 242, 253, 2, 78, 166, + 165, 2, 81, 223, 1, 65, 6, 52, 7, 79, 84, 76, 69, 83, 83, 32, 199, 236, + 37, 65, 4, 134, 203, 31, 66, 3, 70, 6, 234, 202, 31, 72, 206, 230, 6, 65, + 3, 69, 38, 42, 65, 190, 163, 24, 72, 167, 141, 14, 69, 32, 44, 5, 73, 76, + 69, 68, 32, 195, 177, 38, 72, 30, 146, 1, 68, 94, 83, 210, 20, 75, 246, + 247, 24, 74, 178, 187, 6, 89, 222, 7, 72, 190, 59, 71, 242, 253, 2, 78, + 218, 235, 1, 76, 206, 57, 81, 223, 1, 65, 6, 52, 7, 79, 84, 76, 69, 83, + 83, 32, 215, 233, 37, 65, 4, 158, 137, 35, 78, 167, 165, 2, 81, 6, 218, + 219, 36, 72, 14, 69, 203, 141, 1, 65, 4, 242, 197, 18, 77, 203, 226, 18, + 83, 6, 228, 235, 24, 3, 75, 65, 83, 12, 4, 68, 65, 77, 77, 1, 4, 70, 65, + 84, 72, 12, 118, 69, 38, 79, 172, 197, 12, 11, 76, 65, 67, 69, 32, 79, + 70, 32, 83, 65, 74, 165, 254, 5, 6, 73, 65, 83, 84, 82, 69, 4, 182, 157, + 32, 82, 235, 218, 5, 80, 4, 252, 181, 12, 8, 69, 84, 73, 67, 32, 86, 69, + 82, 209, 141, 6, 3, 85, 78, 68, 14, 238, 1, 65, 96, 5, 69, 86, 69, 82, + 83, 36, 15, 73, 71, 72, 84, 32, 65, 82, 82, 79, 87, 72, 69, 65, 68, 32, + 225, 158, 23, 28, 79, 85, 78, 68, 69, 68, 32, 72, 73, 71, 72, 32, 83, 84, + 79, 80, 32, 87, 73, 84, 72, 32, 70, 73, 76, 76, 69, 68, 4, 40, 4, 73, 83, + 69, 68, 171, 170, 38, 89, 2, 17, 2, 32, 82, 2, 149, 222, 36, 3, 79, 85, + 78, 2, 181, 214, 21, 4, 69, 68, 32, 68, 6, 26, 65, 167, 139, 36, 66, 4, + 157, 133, 37, 3, 66, 79, 86, 202, 1, 238, 1, 69, 244, 2, 5, 72, 65, 68, + 68, 65, 36, 4, 73, 71, 78, 32, 240, 4, 5, 77, 65, 76, 76, 32, 246, 15, + 85, 240, 2, 6, 89, 77, 66, 79, 76, 32, 209, 208, 36, 18, 84, 65, 82, 84, + 32, 79, 70, 32, 82, 85, 66, 32, 69, 76, 32, 72, 73, 90, 20, 52, 7, 81, + 85, 69, 78, 67, 69, 32, 187, 219, 34, 77, 18, 184, 1, 26, 89, 69, 72, 32, + 87, 73, 84, 72, 32, 72, 65, 77, 90, 65, 32, 65, 66, 79, 86, 69, 32, 87, + 73, 84, 72, 32, 209, 189, 31, 13, 78, 79, 79, 78, 32, 87, 73, 84, 72, 32, + 75, 69, 72, 16, 70, 65, 186, 166, 37, 87, 198, 89, 89, 150, 14, 79, 214, + 22, 69, 3, 85, 6, 36, 3, 76, 69, 70, 191, 164, 38, 69, 5, 159, 13, 32, 7, + 11, 32, 4, 246, 22, 73, 55, 77, 22, 132, 1, 2, 82, 65, 158, 1, 83, 188, + 1, 7, 65, 76, 65, 89, 72, 69, 32, 160, 15, 2, 77, 73, 209, 138, 34, 6, + 84, 65, 75, 72, 65, 76, 4, 132, 1, 13, 72, 77, 65, 84, 85, 76, 76, 65, + 72, 32, 65, 76, 65, 133, 180, 33, 13, 68, 73, 32, 65, 76, 76, 65, 72, 79, + 85, 32, 65, 78, 2, 235, 244, 37, 89, 12, 46, 65, 193, 1, 6, 73, 78, 68, + 72, 73, 32, 8, 136, 1, 18, 76, 76, 65, 76, 76, 65, 72, 79, 85, 32, 65, + 76, 65, 89, 72, 69, 32, 87, 170, 192, 31, 78, 194, 229, 4, 70, 217, 102, + 2, 77, 86, 2, 17, 2, 65, 83, 2, 189, 221, 15, 3, 83, 65, 76, 4, 152, 200, + 20, 12, 80, 79, 83, 84, 80, 79, 83, 73, 84, 73, 79, 78, 185, 136, 14, 2, + 65, 77, 106, 132, 1, 2, 70, 65, 32, 5, 72, 73, 71, 72, 32, 240, 11, 4, + 76, 79, 87, 32, 110, 75, 218, 189, 21, 68, 226, 235, 9, 89, 179, 233, 5, + 87, 4, 190, 3, 82, 147, 160, 36, 84, 74, 236, 2, 17, 68, 79, 84, 76, 69, + 83, 83, 32, 72, 69, 65, 68, 32, 79, 70, 32, 75, 22, 70, 102, 76, 142, 3, + 77, 116, 4, 78, 79, 79, 78, 18, 83, 78, 84, 38, 87, 196, 2, 3, 89, 69, + 72, 224, 175, 14, 18, 85, 80, 82, 73, 71, 72, 84, 32, 82, 69, 67, 84, 65, + 78, 71, 85, 76, 65, 236, 213, 8, 7, 82, 79, 85, 78, 68, 69, 68, 234, 126, + 90, 134, 107, 74, 222, 161, 12, 81, 223, 1, 65, 2, 255, 186, 31, 72, 4, + 24, 2, 65, 82, 31, 79, 2, 11, 83, 2, 175, 2, 73, 2, 253, 180, 26, 6, 79, + 84, 78, 79, 84, 69, 10, 60, 8, 73, 71, 65, 84, 85, 82, 69, 32, 213, 11, + 2, 65, 77, 8, 88, 10, 65, 76, 69, 70, 32, 87, 73, 84, 72, 32, 116, 3, 81, + 65, 70, 1, 3, 83, 65, 68, 4, 84, 8, 76, 65, 77, 32, 87, 73, 84, 72, 209, + 168, 36, 7, 89, 69, 72, 32, 66, 65, 82, 2, 181, 176, 31, 2, 32, 89, 2, 89, 20, 32, 87, 73, 84, 72, 32, 76, 65, 77, 32, 87, 73, 84, 72, 32, 65, - 76, 69, 70, 32, 2, 225, 193, 21, 3, 77, 65, 75, 6, 26, 69, 215, 201, 15, - 65, 4, 17, 2, 69, 77, 4, 17, 2, 32, 73, 4, 22, 78, 135, 9, 83, 2, 197, 9, - 2, 73, 84, 5, 207, 4, 32, 6, 252, 254, 34, 7, 73, 71, 78, 32, 83, 65, 70, - 202, 38, 69, 227, 138, 1, 65, 4, 194, 243, 22, 72, 159, 129, 14, 65, 20, - 40, 4, 79, 82, 68, 32, 147, 247, 35, 65, 18, 74, 65, 172, 1, 2, 83, 65, - 200, 214, 24, 3, 87, 65, 81, 143, 155, 11, 81, 10, 228, 225, 6, 3, 76, - 45, 74, 236, 253, 17, 3, 82, 45, 82, 132, 196, 6, 7, 84, 72, 45, 84, 72, - 65, 76, 240, 207, 4, 5, 78, 45, 78, 73, 83, 137, 92, 5, 83, 45, 83, 65, - 74, 4, 218, 206, 36, 75, 207, 36, 72, 5, 149, 150, 10, 12, 32, 66, 65, + 76, 69, 70, 32, 2, 165, 145, 22, 3, 77, 65, 75, 6, 26, 69, 191, 249, 15, + 65, 4, 17, 2, 69, 77, 4, 17, 2, 32, 73, 4, 22, 78, 135, 9, 83, 2, 193, 9, + 2, 73, 84, 5, 207, 4, 32, 6, 180, 155, 36, 7, 73, 71, 78, 32, 83, 65, 70, + 218, 38, 69, 203, 141, 1, 65, 4, 178, 196, 23, 72, 235, 207, 14, 65, 20, + 40, 4, 79, 82, 68, 32, 191, 150, 37, 65, 18, 74, 65, 172, 1, 2, 83, 65, + 172, 173, 25, 3, 87, 65, 81, 207, 227, 11, 81, 10, 192, 252, 6, 3, 76, + 45, 74, 244, 185, 18, 3, 82, 45, 82, 144, 226, 6, 7, 84, 72, 45, 84, 72, + 65, 76, 164, 250, 4, 5, 78, 45, 78, 73, 83, 161, 92, 5, 83, 45, 83, 65, + 74, 4, 150, 238, 37, 75, 207, 36, 72, 5, 193, 182, 10, 12, 32, 66, 65, 82, 82, 69, 69, 32, 87, 73, 84, 72, 20, 68, 5, 78, 79, 79, 78, 32, 70, - 87, 206, 150, 24, 77, 239, 137, 11, 83, 2, 33, 6, 87, 73, 84, 72, 32, 75, - 2, 11, 65, 2, 151, 173, 36, 83, 14, 40, 4, 79, 82, 68, 32, 255, 242, 35, - 65, 12, 106, 73, 198, 194, 15, 77, 204, 135, 9, 3, 84, 65, 83, 212, 192, - 8, 2, 83, 65, 237, 172, 3, 3, 81, 65, 83, 4, 252, 230, 12, 2, 77, 65, - 177, 189, 18, 3, 83, 72, 77, 12, 138, 1, 66, 64, 3, 75, 85, 78, 177, 152, + 87, 234, 237, 24, 77, 155, 207, 11, 83, 2, 33, 6, 87, 73, 84, 72, 32, 75, + 2, 11, 65, 2, 199, 204, 37, 83, 14, 40, 4, 79, 82, 68, 32, 171, 146, 37, + 65, 12, 106, 73, 174, 242, 15, 77, 148, 175, 9, 3, 84, 65, 83, 200, 218, + 8, 2, 83, 65, 249, 218, 3, 3, 81, 65, 83, 4, 156, 148, 13, 2, 77, 65, + 253, 132, 19, 3, 83, 72, 77, 12, 138, 1, 66, 64, 3, 75, 85, 78, 205, 239, 24, 22, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, 76, 69, 70, 32, 77, - 79, 75, 72, 65, 83, 83, 2, 33, 6, 83, 67, 82, 73, 80, 84, 2, 229, 204, - 21, 2, 32, 65, 9, 11, 32, 6, 34, 73, 58, 77, 255, 212, 34, 66, 2, 11, 83, - 2, 141, 239, 35, 6, 79, 76, 65, 84, 69, 68, 2, 11, 69, 2, 11, 68, 2, 11, - 73, 2, 189, 238, 35, 2, 65, 76, 34, 152, 1, 2, 68, 79, 114, 84, 224, 145, - 5, 8, 83, 77, 65, 76, 76, 32, 84, 65, 156, 229, 25, 4, 70, 79, 85, 82, - 228, 124, 4, 87, 65, 83, 76, 147, 166, 4, 82, 6, 88, 16, 85, 66, 76, 69, - 32, 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 159, 210, 34, 84, 2, 151, - 196, 34, 82, 16, 88, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 121, 8, - 87, 79, 32, 68, 79, 84, 83, 32, 8, 184, 246, 30, 17, 80, 79, 73, 78, 84, - 73, 78, 71, 32, 68, 79, 87, 78, 87, 65, 82, 68, 154, 219, 3, 66, 223, - 155, 1, 65, 8, 176, 208, 34, 10, 86, 69, 82, 84, 73, 67, 65, 76, 76, 89, - 42, 66, 223, 155, 1, 65, 30, 202, 1, 65, 152, 2, 4, 79, 78, 69, 32, 244, - 209, 10, 12, 82, 73, 80, 76, 69, 32, 68, 79, 84, 32, 80, 85, 152, 164, 6, - 8, 85, 82, 78, 69, 68, 32, 68, 65, 149, 222, 13, 8, 72, 79, 85, 83, 65, - 78, 68, 83, 12, 68, 5, 84, 87, 69, 69, 76, 157, 180, 9, 6, 73, 76, 32, + 79, 75, 72, 65, 83, 83, 2, 33, 6, 83, 67, 82, 73, 80, 84, 2, 169, 156, + 22, 2, 32, 65, 9, 11, 32, 6, 34, 73, 54, 77, 231, 238, 35, 66, 2, 11, 83, + 2, 213, 204, 31, 5, 79, 76, 65, 84, 69, 2, 11, 69, 2, 11, 68, 2, 11, 73, + 2, 237, 141, 37, 2, 65, 76, 34, 152, 1, 2, 68, 79, 114, 84, 248, 172, 5, + 8, 83, 77, 65, 76, 76, 32, 84, 65, 244, 190, 26, 4, 70, 79, 85, 82, 204, + 123, 4, 87, 65, 83, 76, 239, 209, 4, 82, 6, 88, 16, 85, 66, 76, 69, 32, + 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 135, 236, 35, 84, 2, 247, + 220, 35, 82, 16, 88, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 121, 8, + 87, 79, 32, 68, 79, 84, 83, 32, 8, 168, 235, 31, 17, 80, 79, 73, 78, 84, + 73, 78, 71, 32, 68, 79, 87, 78, 87, 65, 82, 68, 146, 128, 4, 66, 167, + 161, 1, 65, 8, 152, 234, 35, 10, 86, 69, 82, 84, 73, 67, 65, 76, 76, 89, + 42, 66, 167, 161, 1, 65, 30, 202, 1, 65, 152, 2, 4, 79, 78, 69, 32, 168, + 244, 10, 12, 82, 73, 80, 76, 69, 32, 68, 79, 84, 32, 80, 85, 144, 197, 6, + 8, 85, 82, 78, 69, 68, 32, 68, 65, 217, 143, 14, 8, 72, 79, 85, 83, 65, + 78, 68, 83, 12, 68, 5, 84, 87, 69, 69, 76, 153, 235, 35, 6, 73, 76, 32, 70, 82, 65, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 112, 11, 79, 86, 69, - 82, 83, 84, 82, 85, 67, 75, 32, 184, 166, 5, 7, 70, 65, 84, 72, 65, 84, - 65, 131, 226, 4, 84, 4, 26, 72, 223, 230, 35, 87, 2, 197, 254, 28, 2, 65, - 77, 12, 68, 3, 79, 78, 69, 234, 159, 29, 84, 205, 171, 5, 4, 76, 79, 79, - 80, 4, 213, 245, 32, 2, 32, 68, 8, 64, 10, 79, 87, 69, 76, 32, 83, 73, - 71, 78, 32, 243, 178, 23, 69, 6, 72, 10, 73, 78, 86, 69, 82, 84, 69, 68, - 32, 83, 2, 83, 147, 236, 24, 68, 2, 25, 4, 77, 65, 76, 76, 2, 213, 229, - 35, 2, 32, 86, 28, 104, 4, 80, 69, 82, 32, 136, 214, 5, 3, 67, 85, 66, - 208, 128, 5, 5, 70, 79, 85, 82, 84, 211, 156, 24, 68, 4, 250, 182, 24, + 82, 83, 84, 82, 85, 67, 75, 32, 208, 193, 5, 7, 70, 65, 84, 72, 65, 84, + 65, 155, 231, 4, 84, 4, 26, 72, 143, 134, 37, 87, 2, 169, 228, 29, 2, 65, + 77, 12, 68, 3, 79, 78, 69, 214, 133, 30, 84, 201, 223, 5, 4, 76, 79, 79, + 80, 4, 221, 232, 33, 2, 32, 68, 8, 64, 10, 79, 87, 69, 76, 32, 83, 73, + 71, 78, 32, 183, 137, 24, 69, 6, 72, 10, 73, 78, 86, 69, 82, 84, 69, 68, + 32, 83, 2, 83, 255, 194, 25, 68, 2, 25, 4, 77, 65, 76, 76, 2, 133, 133, + 37, 2, 32, 86, 28, 104, 4, 80, 69, 82, 32, 232, 240, 5, 3, 67, 85, 66, + 156, 137, 5, 5, 70, 79, 85, 82, 84, 227, 149, 25, 68, 4, 158, 142, 25, 77, 31, 84, 188, 1, 222, 1, 65, 34, 67, 138, 3, 69, 60, 7, 83, 77, 65, - 76, 76, 32, 76, 240, 176, 20, 17, 77, 79, 68, 73, 70, 73, 69, 82, 32, 76, - 69, 84, 84, 69, 82, 32, 76, 222, 225, 8, 72, 184, 168, 2, 3, 68, 82, 65, - 218, 213, 2, 70, 83, 81, 4, 214, 215, 30, 66, 135, 26, 80, 78, 80, 14, - 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 187, 147, 34, 79, + 76, 76, 32, 76, 152, 128, 21, 17, 77, 79, 68, 73, 70, 73, 69, 82, 32, 76, + 69, 84, 84, 69, 82, 32, 76, 162, 248, 8, 72, 252, 182, 2, 3, 68, 82, 65, + 138, 250, 2, 70, 83, 81, 4, 198, 204, 31, 66, 139, 26, 80, 78, 80, 14, + 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 155, 172, 35, 79, 76, 250, 1, 84, 36, 2, 89, 73, 150, 3, 67, 38, 82, 34, 69, 42, 71, 34, - 74, 38, 75, 22, 80, 38, 83, 106, 65, 22, 86, 158, 1, 76, 254, 161, 30, - 70, 2, 88, 170, 227, 1, 73, 134, 255, 2, 66, 2, 77, 234, 54, 78, 226, 26, - 90, 186, 96, 72, 190, 28, 68, 171, 1, 79, 4, 178, 139, 35, 73, 163, 209, - 1, 79, 5, 219, 140, 36, 87, 4, 216, 135, 25, 6, 77, 80, 72, 65, 83, 73, - 191, 233, 5, 88, 92, 76, 6, 69, 84, 84, 69, 82, 32, 157, 5, 8, 73, 71, - 65, 84, 85, 82, 69, 32, 80, 242, 1, 67, 38, 82, 34, 69, 42, 71, 34, 74, - 38, 75, 22, 80, 38, 83, 34, 84, 74, 65, 22, 86, 32, 2, 89, 73, 126, 76, - 254, 161, 30, 70, 2, 88, 170, 227, 1, 73, 134, 255, 2, 66, 2, 77, 234, - 54, 78, 226, 26, 90, 186, 96, 72, 190, 28, 68, 171, 1, 79, 8, 34, 72, - 250, 216, 36, 65, 3, 79, 4, 230, 215, 36, 69, 147, 1, 65, 6, 198, 215, - 36, 67, 146, 1, 72, 3, 84, 4, 142, 146, 36, 72, 203, 53, 73, 4, 238, 164, - 30, 72, 163, 179, 6, 65, 4, 207, 225, 32, 69, 4, 166, 217, 3, 73, 163, - 253, 32, 69, 4, 250, 212, 36, 72, 171, 1, 69, 6, 68, 7, 85, 82, 78, 69, - 68, 32, 65, 174, 133, 35, 73, 163, 209, 1, 79, 2, 167, 156, 35, 89, 4, - 166, 216, 35, 69, 147, 126, 79, 7, 222, 216, 20, 32, 231, 173, 15, 87, - 12, 84, 5, 69, 67, 72, 32, 89, 20, 4, 77, 69, 78, 32, 133, 136, 29, 4, - 86, 69, 87, 32, 2, 251, 131, 35, 73, 8, 230, 161, 30, 88, 170, 227, 1, - 73, 238, 181, 3, 78, 231, 119, 69, 14, 116, 10, 32, 80, 79, 73, 78, 84, - 73, 78, 71, 32, 177, 151, 35, 13, 72, 69, 65, 68, 45, 83, 72, 65, 80, 69, - 68, 32, 80, 12, 156, 2, 24, 82, 73, 71, 72, 84, 87, 65, 82, 68, 83, 32, - 84, 72, 69, 78, 32, 67, 85, 82, 86, 73, 78, 71, 32, 48, 16, 85, 80, 87, - 65, 82, 68, 83, 32, 84, 72, 69, 78, 32, 78, 79, 82, 201, 248, 33, 22, 68, - 79, 87, 78, 87, 65, 82, 68, 83, 32, 84, 72, 69, 78, 32, 67, 85, 82, 86, - 73, 78, 71, 6, 44, 3, 83, 79, 85, 210, 239, 25, 68, 35, 85, 2, 173, 173, - 28, 4, 84, 72, 32, 87, 4, 144, 135, 30, 10, 67, 85, 76, 65, 84, 69, 68, - 32, 76, 79, 137, 133, 5, 6, 83, 84, 32, 80, 65, 76, 20, 58, 67, 38, 84, - 146, 200, 22, 89, 241, 156, 9, 2, 83, 69, 4, 222, 141, 8, 69, 175, 128, - 11, 73, 12, 48, 4, 69, 82, 73, 83, 36, 2, 79, 78, 31, 82, 6, 238, 192, 2, - 75, 247, 141, 34, 77, 2, 209, 199, 13, 2, 73, 83, 4, 226, 254, 29, 65, - 189, 204, 4, 22, 79, 78, 79, 77, 73, 67, 65, 76, 32, 83, 89, 77, 66, 79, - 76, 32, 70, 79, 82, 32, 85, 82, 4, 216, 219, 18, 6, 72, 76, 69, 84, 73, - 67, 189, 222, 16, 2, 79, 77, 10, 68, 2, 84, 79, 240, 142, 21, 2, 83, 84, - 185, 162, 2, 3, 66, 69, 82, 6, 54, 77, 201, 205, 35, 7, 32, 82, 73, 67, - 75, 83, 72, 4, 218, 176, 23, 79, 153, 182, 5, 12, 65, 84, 69, 68, 32, 84, - 69, 76, 76, 69, 82, 32, 112, 56, 6, 69, 83, 84, 65, 78, 32, 133, 196, 4, - 2, 79, 67, 110, 52, 7, 76, 69, 84, 84, 69, 82, 32, 167, 194, 30, 65, 108, - 234, 1, 65, 58, 71, 42, 72, 34, 78, 50, 88, 42, 83, 42, 89, 34, 84, 246, - 246, 32, 85, 158, 144, 1, 79, 182, 56, 73, 198, 134, 1, 66, 2, 68, 2, 90, - 246, 63, 69, 194, 41, 67, 2, 70, 2, 74, 2, 75, 2, 76, 2, 77, 2, 80, 2, - 82, 3, 86, 17, 254, 130, 34, 65, 158, 133, 2, 69, 150, 64, 78, 3, 79, 6, - 162, 177, 36, 71, 2, 72, 215, 22, 69, 4, 250, 176, 36, 77, 215, 22, 69, - 12, 46, 71, 174, 176, 36, 78, 2, 89, 215, 22, 69, 6, 170, 176, 36, 86, 2, - 89, 215, 22, 69, 8, 38, 72, 166, 153, 36, 83, 143, 45, 69, 4, 218, 175, - 36, 89, 215, 22, 69, 6, 186, 175, 36, 72, 2, 84, 215, 22, 69, 134, 37, - 178, 1, 65, 182, 169, 1, 69, 132, 16, 9, 72, 65, 73, 75, 83, 85, 75, 73, - 32, 198, 6, 73, 130, 3, 76, 186, 34, 79, 130, 55, 82, 138, 19, 85, 138, - 8, 89, 222, 139, 34, 80, 147, 1, 83, 182, 14, 132, 1, 2, 66, 89, 94, 67, - 206, 2, 68, 146, 1, 71, 106, 76, 140, 28, 4, 77, 85, 77, 32, 230, 114, - 78, 178, 1, 82, 98, 83, 223, 6, 84, 11, 11, 32, 8, 238, 188, 12, 67, 134, - 212, 5, 66, 204, 209, 13, 3, 65, 78, 71, 227, 205, 3, 83, 16, 62, 75, - 172, 223, 8, 5, 84, 82, 73, 65, 78, 139, 147, 27, 79, 12, 42, 32, 46, 83, - 229, 186, 10, 2, 45, 84, 4, 178, 220, 9, 87, 161, 145, 20, 2, 79, 70, 6, - 132, 1, 26, 76, 65, 78, 84, 69, 68, 32, 83, 79, 85, 84, 72, 32, 65, 82, - 82, 79, 87, 32, 87, 73, 84, 72, 32, 72, 79, 175, 131, 36, 80, 4, 150, - 215, 28, 82, 245, 130, 5, 2, 79, 75, 4, 188, 215, 33, 27, 77, 73, 78, 84, - 79, 78, 32, 82, 65, 67, 81, 85, 69, 84, 32, 65, 78, 68, 32, 83, 72, 85, - 84, 84, 76, 69, 67, 223, 170, 2, 71, 6, 228, 247, 29, 6, 85, 69, 84, 84, - 69, 32, 234, 189, 5, 69, 229, 6, 8, 71, 65, 71, 69, 32, 67, 76, 65, 150, - 2, 44, 6, 73, 78, 69, 83, 69, 32, 131, 24, 76, 248, 1, 188, 2, 6, 67, 65, - 82, 73, 75, 32, 104, 7, 76, 69, 84, 84, 69, 82, 32, 224, 8, 15, 77, 85, - 83, 73, 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, 196, 6, 2, 80, 65, - 120, 5, 83, 73, 71, 78, 32, 180, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, - 71, 78, 32, 192, 246, 13, 5, 65, 68, 69, 71, 32, 162, 204, 18, 87, 219, - 247, 1, 68, 6, 32, 2, 80, 65, 255, 230, 2, 83, 4, 156, 130, 29, 5, 77, - 85, 78, 71, 75, 213, 231, 5, 3, 82, 69, 82, 110, 166, 2, 65, 112, 2, 66, - 65, 32, 2, 67, 65, 32, 2, 68, 65, 110, 69, 32, 2, 71, 65, 30, 73, 2, 79, - 2, 85, 36, 2, 74, 65, 30, 75, 68, 2, 76, 65, 18, 78, 72, 2, 80, 65, 36, - 2, 82, 65, 16, 2, 83, 65, 54, 84, 128, 1, 2, 86, 69, 0, 3, 90, 65, 76, - 170, 175, 36, 72, 2, 77, 2, 87, 3, 89, 10, 226, 2, 75, 184, 3, 5, 83, 89, - 85, 82, 65, 172, 243, 14, 8, 82, 67, 72, 65, 73, 67, 32, 74, 243, 217, 3, - 73, 5, 213, 165, 18, 3, 32, 75, 69, 5, 221, 192, 22, 3, 32, 76, 65, 9, - 17, 2, 32, 77, 6, 44, 5, 85, 82, 68, 65, 32, 175, 208, 32, 65, 4, 166, - 129, 14, 77, 25, 3, 65, 76, 80, 4, 254, 3, 70, 163, 237, 35, 75, 5, 185, - 202, 33, 2, 32, 71, 4, 11, 75, 4, 229, 14, 2, 65, 82, 5, 133, 169, 34, 2, - 32, 74, 8, 34, 65, 225, 2, 3, 72, 79, 84, 7, 222, 2, 70, 247, 252, 13, - 32, 7, 199, 12, 32, 8, 34, 65, 182, 177, 36, 71, 3, 89, 5, 221, 154, 26, - 4, 32, 82, 65, 77, 5, 157, 170, 35, 4, 32, 75, 65, 80, 7, 235, 11, 32, 7, - 21, 3, 32, 83, 65, 4, 194, 176, 36, 71, 3, 80, 10, 30, 65, 97, 3, 90, 73, - 82, 9, 11, 32, 6, 196, 253, 13, 6, 77, 85, 82, 68, 65, 32, 208, 7, 3, 76, - 65, 84, 151, 232, 17, 84, 2, 229, 249, 13, 2, 32, 83, 56, 168, 1, 10, 67, - 79, 77, 66, 73, 78, 73, 78, 71, 32, 246, 1, 68, 172, 1, 10, 76, 69, 70, - 84, 45, 72, 65, 78, 68, 32, 117, 11, 82, 73, 71, 72, 84, 45, 72, 65, 78, - 68, 32, 18, 132, 1, 4, 75, 69, 77, 80, 78, 74, 180, 181, 19, 2, 66, 69, - 136, 159, 7, 3, 69, 78, 68, 232, 166, 3, 3, 84, 69, 71, 195, 246, 3, 71, - 8, 32, 2, 76, 73, 1, 2, 85, 76, 5, 37, 7, 32, 87, 73, 84, 72, 32, 74, 2, - 225, 243, 13, 3, 69, 71, 79, 20, 50, 65, 90, 69, 174, 220, 35, 73, 2, 79, - 3, 85, 10, 40, 2, 78, 71, 218, 220, 35, 69, 3, 73, 7, 11, 32, 4, 158, 4, - 83, 227, 203, 10, 71, 4, 170, 220, 35, 85, 155, 80, 78, 10, 76, 6, 79, - 80, 69, 78, 32, 80, 113, 9, 67, 76, 79, 83, 69, 68, 32, 80, 76, 6, 186, - 219, 35, 65, 2, 73, 3, 85, 8, 72, 8, 67, 76, 79, 83, 69, 68, 32, 84, 29, - 6, 79, 80, 69, 78, 32, 68, 4, 158, 168, 36, 65, 3, 85, 4, 198, 170, 36, - 65, 3, 85, 10, 30, 77, 49, 3, 78, 84, 73, 6, 44, 3, 65, 68, 65, 129, 238, - 33, 2, 69, 78, 5, 233, 209, 35, 5, 32, 76, 65, 78, 84, 12, 110, 83, 20, - 4, 85, 76, 85, 32, 188, 240, 28, 3, 66, 73, 83, 150, 134, 1, 67, 153, - 248, 1, 4, 82, 69, 82, 69, 2, 247, 146, 24, 85, 4, 130, 245, 19, 67, 213, - 161, 16, 3, 82, 73, 67, 30, 120, 3, 76, 65, 32, 32, 3, 82, 65, 32, 12, 4, + 74, 38, 75, 22, 80, 38, 83, 106, 65, 22, 86, 158, 1, 76, 130, 141, 31, + 70, 2, 88, 218, 235, 1, 73, 158, 168, 3, 66, 2, 77, 198, 57, 78, 226, 26, + 90, 210, 96, 72, 190, 28, 68, 171, 1, 79, 4, 254, 167, 36, 73, 151, 212, + 1, 79, 5, 143, 172, 37, 87, 4, 138, 230, 31, 88, 169, 198, 4, 6, 77, 80, + 72, 65, 83, 73, 92, 76, 6, 69, 84, 84, 69, 82, 32, 157, 5, 8, 73, 71, 65, + 84, 85, 82, 69, 32, 80, 242, 1, 67, 38, 82, 34, 69, 42, 71, 34, 74, 38, + 75, 22, 80, 38, 83, 34, 84, 74, 65, 22, 86, 32, 2, 89, 73, 126, 76, 130, + 141, 31, 70, 2, 88, 218, 235, 1, 73, 158, 168, 3, 66, 2, 77, 198, 57, 78, + 226, 26, 90, 210, 96, 72, 190, 28, 68, 171, 1, 79, 8, 34, 72, 186, 248, + 37, 65, 3, 79, 4, 166, 247, 37, 69, 147, 1, 65, 6, 134, 247, 37, 67, 146, + 1, 72, 3, 84, 4, 194, 177, 37, 72, 215, 53, 73, 4, 242, 143, 31, 72, 223, + 231, 6, 65, 4, 211, 212, 33, 69, 4, 194, 244, 3, 73, 199, 129, 34, 69, 4, + 186, 244, 37, 72, 171, 1, 69, 6, 68, 7, 85, 82, 78, 69, 68, 32, 65, 250, + 161, 36, 73, 151, 212, 1, 79, 2, 167, 186, 36, 89, 4, 214, 247, 36, 69, + 163, 126, 79, 7, 146, 168, 21, 32, 231, 253, 15, 87, 12, 84, 5, 69, 67, + 72, 32, 89, 20, 4, 77, 69, 78, 32, 241, 237, 29, 4, 86, 69, 87, 32, 2, + 199, 160, 36, 73, 8, 234, 140, 31, 88, 218, 235, 1, 73, 226, 225, 3, 78, + 255, 119, 69, 14, 108, 10, 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 169, + 181, 36, 11, 72, 69, 65, 68, 45, 83, 72, 65, 80, 69, 68, 12, 156, 2, 24, + 82, 73, 71, 72, 84, 87, 65, 82, 68, 83, 32, 84, 72, 69, 78, 32, 67, 85, + 82, 86, 73, 78, 71, 32, 48, 16, 85, 80, 87, 65, 82, 68, 83, 32, 84, 72, + 69, 78, 32, 78, 79, 82, 161, 146, 35, 22, 68, 79, 87, 78, 87, 65, 82, 68, + 83, 32, 84, 72, 69, 78, 32, 67, 85, 82, 86, 73, 78, 71, 6, 44, 3, 83, 79, + 85, 198, 209, 26, 68, 35, 85, 2, 149, 147, 29, 4, 84, 72, 32, 87, 4, 168, + 240, 30, 10, 67, 85, 76, 65, 84, 69, 68, 32, 76, 79, 237, 184, 5, 6, 83, + 84, 32, 80, 65, 76, 20, 58, 67, 38, 84, 142, 153, 23, 89, 173, 191, 9, 2, + 83, 69, 4, 206, 168, 8, 69, 231, 180, 11, 73, 12, 48, 4, 69, 82, 73, 83, + 36, 2, 79, 78, 31, 82, 6, 146, 220, 2, 75, 155, 146, 35, 77, 2, 217, 246, + 13, 2, 73, 83, 4, 250, 231, 30, 65, 221, 255, 4, 22, 79, 78, 79, 77, 73, + 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 85, 82, 4, + 132, 171, 19, 6, 72, 76, 69, 84, 73, 67, 193, 174, 17, 2, 79, 77, 10, 68, + 2, 84, 79, 200, 222, 21, 2, 83, 84, 181, 169, 2, 3, 66, 69, 82, 6, 54, + 77, 129, 237, 36, 7, 32, 82, 73, 67, 75, 83, 72, 4, 174, 135, 24, 79, + 189, 197, 5, 12, 65, 84, 69, 68, 32, 84, 69, 76, 76, 69, 82, 32, 112, 56, + 6, 69, 83, 84, 65, 78, 32, 165, 223, 4, 2, 79, 67, 110, 52, 7, 76, 69, + 84, 84, 69, 82, 32, 159, 183, 31, 65, 108, 234, 1, 65, 58, 71, 42, 72, + 34, 78, 50, 88, 42, 83, 42, 89, 34, 84, 178, 146, 34, 85, 202, 141, 1, + 79, 134, 60, 73, 190, 137, 1, 66, 2, 68, 2, 90, 130, 64, 69, 206, 41, 67, + 2, 70, 2, 74, 2, 75, 2, 76, 2, 77, 2, 80, 2, 82, 3, 86, 17, 230, 155, 35, + 65, 242, 139, 2, 69, 162, 64, 78, 3, 79, 6, 234, 208, 37, 71, 2, 72, 215, + 22, 69, 4, 194, 208, 37, 77, 215, 22, 69, 12, 46, 71, 246, 207, 37, 78, + 2, 89, 215, 22, 69, 6, 242, 207, 37, 86, 2, 89, 215, 22, 69, 8, 38, 72, + 238, 184, 37, 83, 143, 45, 69, 4, 162, 207, 37, 89, 215, 22, 69, 6, 130, + 207, 37, 72, 2, 84, 215, 22, 69, 192, 41, 178, 1, 65, 242, 170, 1, 69, + 252, 15, 9, 72, 65, 73, 75, 83, 85, 75, 73, 32, 202, 6, 73, 130, 3, 76, + 182, 45, 79, 138, 70, 82, 242, 18, 85, 138, 8, 89, 130, 144, 35, 80, 147, + 1, 83, 190, 14, 132, 1, 2, 66, 89, 94, 67, 206, 2, 68, 146, 1, 71, 106, + 76, 192, 29, 4, 77, 85, 77, 32, 238, 114, 78, 178, 1, 82, 98, 83, 223, 6, + 84, 11, 11, 32, 8, 190, 234, 12, 67, 230, 241, 5, 66, 216, 249, 13, 3, + 65, 78, 71, 215, 249, 3, 83, 16, 62, 75, 160, 251, 8, 5, 84, 82, 73, 65, + 78, 211, 150, 28, 79, 12, 42, 32, 46, 83, 153, 222, 10, 2, 45, 84, 4, + 234, 252, 9, 87, 217, 218, 20, 2, 79, 70, 6, 132, 1, 26, 76, 65, 78, 84, + 69, 68, 32, 83, 79, 85, 84, 72, 32, 65, 82, 82, 79, 87, 32, 87, 73, 84, + 72, 32, 72, 79, 235, 162, 37, 80, 4, 130, 189, 29, 82, 221, 182, 5, 2, + 79, 75, 4, 144, 241, 34, 27, 77, 73, 78, 84, 79, 78, 32, 82, 65, 67, 81, + 85, 69, 84, 32, 65, 78, 68, 32, 83, 72, 85, 84, 84, 76, 69, 67, 199, 176, + 2, 71, 6, 192, 226, 30, 6, 85, 69, 84, 84, 69, 32, 190, 242, 5, 69, 229, + 6, 8, 71, 65, 71, 69, 32, 67, 76, 65, 156, 2, 44, 6, 73, 78, 69, 83, 69, + 32, 183, 25, 76, 254, 1, 132, 3, 6, 67, 65, 82, 73, 75, 32, 84, 15, 73, + 78, 86, 69, 82, 84, 69, 68, 32, 67, 65, 82, 73, 75, 32, 68, 7, 76, 69, + 84, 84, 69, 82, 32, 224, 8, 15, 77, 85, 83, 73, 67, 65, 76, 32, 83, 89, + 77, 66, 79, 76, 32, 196, 6, 2, 80, 65, 184, 1, 5, 83, 73, 71, 78, 32, + 176, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 224, 164, 14, 5, + 65, 68, 69, 71, 32, 254, 141, 19, 87, 239, 162, 2, 68, 6, 32, 2, 80, 65, + 219, 129, 3, 83, 4, 86, 82, 145, 249, 30, 5, 77, 85, 78, 71, 75, 4, 36, + 3, 80, 65, 82, 131, 129, 3, 83, 2, 165, 133, 36, 2, 69, 82, 110, 166, 2, + 65, 112, 2, 66, 65, 32, 2, 67, 65, 32, 2, 68, 65, 110, 69, 32, 2, 71, 65, + 30, 73, 2, 79, 2, 85, 36, 2, 74, 65, 30, 75, 68, 2, 76, 65, 18, 78, 72, + 2, 80, 65, 36, 2, 82, 65, 16, 2, 83, 65, 54, 84, 128, 1, 2, 86, 69, 0, 3, + 90, 65, 76, 250, 205, 37, 72, 2, 77, 2, 87, 3, 89, 10, 226, 2, 75, 184, + 3, 5, 83, 89, 85, 82, 65, 172, 162, 15, 8, 82, 67, 72, 65, 73, 67, 32, + 74, 167, 249, 3, 73, 5, 165, 244, 18, 3, 32, 75, 69, 5, 129, 248, 30, 3, + 32, 76, 65, 9, 17, 2, 32, 77, 6, 44, 5, 85, 82, 68, 65, 32, 231, 192, 33, + 65, 4, 130, 176, 14, 77, 25, 3, 65, 76, 80, 4, 254, 3, 70, 231, 139, 37, + 75, 5, 137, 227, 34, 2, 32, 71, 4, 11, 75, 4, 161, 15, 2, 65, 82, 5, 193, + 196, 35, 2, 32, 74, 8, 34, 65, 225, 2, 3, 72, 79, 84, 7, 222, 2, 70, 211, + 171, 14, 32, 7, 131, 13, 32, 8, 34, 65, 134, 208, 37, 71, 3, 89, 5, 173, + 252, 26, 4, 32, 82, 65, 77, 5, 213, 200, 36, 4, 32, 75, 65, 80, 7, 167, + 12, 32, 7, 21, 3, 32, 83, 65, 4, 146, 207, 37, 71, 3, 80, 10, 30, 65, 97, + 3, 90, 73, 82, 9, 11, 32, 6, 160, 172, 14, 6, 77, 85, 82, 68, 65, 32, + 208, 7, 3, 76, 65, 84, 255, 171, 18, 84, 2, 193, 168, 14, 2, 32, 83, 56, + 168, 1, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 246, 1, 68, 172, 1, + 10, 76, 69, 70, 84, 45, 72, 65, 78, 68, 32, 117, 11, 82, 73, 71, 72, 84, + 45, 72, 65, 78, 68, 32, 18, 132, 1, 4, 75, 69, 77, 80, 78, 74, 240, 131, + 20, 2, 66, 69, 180, 182, 7, 3, 69, 78, 68, 148, 171, 3, 3, 84, 69, 71, + 159, 164, 4, 71, 8, 32, 2, 76, 73, 1, 2, 85, 76, 5, 37, 7, 32, 87, 73, + 84, 72, 32, 74, 2, 189, 162, 14, 3, 69, 71, 79, 20, 50, 65, 90, 69, 242, + 250, 36, 73, 2, 79, 3, 85, 10, 40, 2, 78, 71, 158, 251, 36, 69, 3, 73, 7, + 11, 32, 4, 218, 4, 83, 155, 244, 10, 71, 4, 238, 250, 36, 85, 167, 80, + 78, 10, 76, 6, 79, 80, 69, 78, 32, 80, 113, 9, 67, 76, 79, 83, 69, 68, + 32, 80, 76, 6, 254, 249, 36, 65, 2, 73, 3, 85, 8, 72, 8, 67, 76, 79, 83, + 69, 68, 32, 84, 29, 6, 79, 80, 69, 78, 32, 68, 4, 238, 198, 37, 65, 3, + 85, 4, 150, 201, 37, 65, 3, 85, 12, 30, 77, 69, 3, 78, 84, 73, 6, 44, 3, + 65, 68, 65, 241, 133, 35, 2, 69, 78, 5, 69, 2, 32, 76, 7, 11, 32, 4, 38, + 76, 245, 146, 35, 3, 66, 65, 87, 2, 225, 239, 36, 3, 65, 78, 84, 12, 106, + 83, 20, 4, 85, 76, 85, 32, 170, 224, 30, 67, 240, 6, 3, 66, 73, 83, 217, + 249, 1, 4, 82, 69, 82, 69, 2, 179, 232, 24, 85, 4, 254, 194, 20, 67, 237, + 241, 16, 3, 82, 73, 67, 30, 120, 3, 76, 65, 32, 32, 3, 82, 65, 32, 12, 4, 83, 85, 75, 85, 34, 84, 104, 5, 80, 69, 80, 69, 84, 53, 3, 85, 76, 85, 4, - 165, 1, 4, 76, 69, 78, 71, 4, 115, 82, 5, 233, 173, 35, 3, 32, 73, 76, + 165, 1, 4, 76, 69, 78, 71, 4, 115, 82, 5, 237, 203, 36, 3, 32, 73, 76, 10, 36, 5, 65, 76, 73, 78, 71, 99, 69, 9, 11, 32, 6, 18, 82, 55, 84, 4, - 17, 2, 69, 80, 4, 11, 65, 5, 17, 2, 32, 84, 2, 11, 69, 2, 131, 245, 29, - 68, 5, 133, 157, 29, 3, 32, 83, 65, 30, 86, 79, 164, 196, 22, 5, 32, 79, - 70, 32, 89, 245, 204, 12, 6, 69, 84, 32, 83, 72, 79, 26, 32, 2, 79, 78, - 21, 2, 84, 32, 5, 199, 147, 34, 45, 22, 44, 2, 66, 79, 246, 1, 83, 159, - 162, 36, 88, 18, 38, 88, 205, 1, 4, 76, 68, 32, 83, 17, 33, 6, 32, 87, - 73, 84, 72, 32, 14, 82, 66, 86, 83, 140, 167, 12, 4, 76, 73, 71, 72, 210, - 145, 10, 67, 195, 233, 13, 88, 6, 52, 4, 79, 76, 68, 32, 141, 210, 35, 3, - 65, 76, 76, 4, 26, 83, 223, 184, 22, 67, 2, 137, 167, 12, 4, 67, 82, 73, - 80, 162, 10, 120, 2, 67, 79, 180, 1, 7, 76, 69, 84, 84, 69, 82, 32, 248, - 91, 3, 78, 74, 65, 186, 234, 27, 83, 146, 142, 5, 70, 83, 81, 8, 26, 77, - 255, 208, 35, 76, 6, 72, 12, 66, 73, 78, 73, 78, 71, 32, 77, 65, 82, 75, - 32, 219, 157, 36, 77, 4, 168, 137, 21, 6, 84, 85, 75, 87, 69, 78, 241, - 165, 3, 4, 75, 79, 81, 78, 146, 10, 170, 1, 70, 58, 75, 170, 1, 76, 66, - 77, 102, 78, 234, 1, 80, 206, 103, 82, 102, 83, 134, 1, 84, 54, 89, 202, - 193, 2, 87, 166, 215, 32, 69, 214, 22, 65, 2, 73, 2, 79, 3, 85, 8, 226, - 79, 65, 222, 183, 35, 69, 254, 5, 79, 219, 16, 85, 22, 78, 69, 46, 79, - 174, 176, 34, 89, 254, 233, 1, 80, 186, 2, 65, 2, 73, 3, 85, 6, 254, 144, - 35, 85, 142, 140, 1, 78, 3, 84, 7, 218, 173, 34, 86, 161, 222, 1, 2, 71, - 72, 10, 194, 102, 69, 190, 162, 25, 79, 182, 147, 10, 65, 2, 73, 3, 85, - 19, 62, 69, 166, 101, 66, 146, 182, 35, 65, 2, 73, 2, 79, 3, 85, 4, 254, - 174, 34, 69, 183, 236, 1, 78, 30, 102, 71, 50, 74, 50, 84, 150, 101, 85, - 250, 188, 33, 83, 246, 7, 68, 242, 219, 1, 89, 218, 19, 65, 3, 73, 6, - 178, 223, 31, 75, 190, 184, 4, 71, 187, 2, 65, 6, 222, 236, 25, 85, 198, - 150, 10, 69, 171, 4, 65, 4, 222, 244, 35, 85, 151, 14, 69, 254, 8, 76, 5, - 72, 65, 83, 69, 45, 210, 100, 69, 138, 2, 85, 254, 177, 35, 65, 3, 73, - 242, 8, 116, 2, 65, 32, 160, 18, 2, 66, 32, 136, 13, 2, 67, 32, 160, 20, - 2, 68, 32, 172, 18, 2, 69, 32, 137, 25, 2, 70, 32, 174, 1, 158, 1, 71, - 106, 75, 130, 1, 76, 102, 77, 190, 3, 78, 210, 4, 80, 146, 2, 83, 190, 2, - 84, 154, 1, 85, 144, 253, 28, 2, 70, 73, 182, 156, 5, 86, 211, 219, 1, - 82, 6, 60, 5, 72, 69, 85, 65, 69, 173, 46, 5, 66, 73, 69, 69, 32, 4, 168, - 27, 2, 71, 72, 179, 205, 25, 82, 12, 38, 65, 34, 69, 142, 67, 80, 3, 85, - 4, 234, 146, 36, 70, 187, 2, 81, 4, 132, 192, 26, 5, 85, 75, 69, 85, 84, - 255, 212, 9, 84, 10, 78, 85, 222, 65, 79, 224, 190, 25, 2, 65, 80, 129, - 154, 9, 4, 69, 84, 32, 75, 5, 243, 185, 26, 65, 34, 142, 1, 65, 168, 1, + 17, 2, 69, 80, 4, 11, 65, 5, 17, 2, 32, 84, 2, 11, 69, 2, 219, 222, 30, + 68, 5, 181, 230, 30, 3, 32, 83, 65, 30, 86, 79, 228, 147, 23, 5, 32, 79, + 70, 32, 89, 177, 155, 13, 6, 69, 84, 32, 83, 72, 79, 26, 32, 2, 79, 78, + 21, 2, 84, 32, 5, 195, 174, 35, 45, 22, 44, 2, 66, 79, 246, 1, 83, 179, + 192, 37, 88, 18, 38, 88, 205, 1, 4, 76, 68, 32, 83, 17, 33, 6, 32, 87, + 73, 84, 72, 32, 14, 82, 66, 86, 83, 168, 211, 12, 4, 76, 73, 71, 72, 246, + 180, 10, 67, 151, 184, 14, 88, 6, 52, 4, 79, 76, 68, 32, 149, 240, 36, 3, + 65, 76, 76, 4, 26, 83, 159, 136, 23, 67, 2, 165, 211, 12, 4, 67, 82, 73, + 80, 164, 10, 120, 2, 67, 79, 180, 1, 7, 76, 69, 84, 84, 69, 82, 32, 128, + 92, 3, 78, 74, 65, 138, 207, 28, 83, 238, 192, 5, 70, 83, 81, 8, 26, 77, + 135, 239, 36, 76, 6, 72, 12, 66, 73, 78, 73, 78, 71, 32, 77, 65, 82, 75, + 32, 239, 187, 37, 77, 4, 196, 215, 21, 6, 84, 85, 75, 87, 69, 78, 201, + 173, 3, 4, 75, 79, 81, 78, 148, 10, 170, 1, 70, 58, 75, 170, 1, 76, 66, + 77, 102, 78, 234, 1, 80, 214, 103, 82, 102, 83, 134, 1, 84, 54, 89, 178, + 219, 2, 87, 202, 219, 33, 69, 214, 22, 65, 2, 73, 2, 79, 3, 85, 8, 234, + 79, 65, 234, 213, 36, 69, 254, 5, 79, 219, 16, 85, 22, 78, 69, 46, 79, + 190, 203, 35, 89, 130, 237, 1, 80, 186, 2, 65, 2, 73, 3, 85, 6, 250, 174, + 36, 85, 166, 140, 1, 78, 3, 84, 7, 234, 200, 35, 86, 165, 225, 1, 2, 71, + 72, 10, 202, 102, 69, 202, 131, 26, 79, 182, 208, 10, 65, 2, 73, 3, 85, + 19, 62, 69, 174, 101, 66, 158, 212, 36, 65, 2, 73, 2, 79, 3, 85, 4, 142, + 202, 35, 69, 187, 239, 1, 78, 30, 102, 71, 50, 74, 50, 84, 158, 101, 85, + 130, 216, 34, 83, 246, 7, 68, 246, 222, 1, 89, 218, 19, 65, 3, 73, 6, + 186, 209, 32, 75, 202, 228, 4, 71, 187, 2, 65, 6, 242, 205, 26, 85, 198, + 211, 10, 69, 171, 4, 65, 4, 242, 146, 37, 85, 151, 14, 69, 128, 9, 76, 5, + 72, 65, 83, 69, 45, 218, 100, 69, 138, 2, 85, 138, 208, 36, 65, 3, 73, + 244, 8, 116, 2, 65, 32, 168, 18, 2, 66, 32, 136, 13, 2, 67, 32, 160, 20, + 2, 68, 32, 172, 18, 2, 69, 32, 137, 25, 2, 70, 32, 176, 1, 158, 1, 71, + 106, 75, 130, 1, 76, 102, 77, 198, 3, 78, 210, 4, 80, 146, 2, 83, 190, 2, + 84, 154, 1, 85, 184, 198, 30, 2, 70, 73, 150, 238, 4, 86, 215, 222, 1, + 82, 6, 60, 5, 72, 69, 85, 65, 69, 181, 46, 5, 66, 73, 69, 69, 32, 4, 176, + 27, 2, 71, 72, 191, 174, 26, 82, 12, 38, 65, 34, 69, 150, 67, 80, 3, 85, + 4, 254, 176, 37, 70, 187, 2, 81, 4, 240, 214, 33, 5, 85, 75, 69, 85, 84, + 167, 220, 3, 84, 10, 78, 85, 230, 65, 79, 236, 159, 26, 2, 65, 80, 241, + 214, 9, 4, 69, 84, 32, 75, 5, 159, 159, 27, 65, 36, 142, 1, 65, 176, 1, 2, 66, 65, 38, 79, 76, 6, 86, 69, 85, 65, 69, 78, 244, 70, 8, 69, 85, 78, - 74, 79, 77, 78, 68, 157, 152, 33, 2, 71, 66, 18, 62, 69, 164, 49, 2, 78, - 83, 149, 253, 30, 4, 80, 32, 80, 73, 14, 58, 77, 242, 254, 11, 75, 246, - 134, 7, 78, 163, 248, 16, 83, 9, 178, 24, 66, 206, 44, 86, 183, 248, 26, - 75, 4, 182, 133, 19, 78, 251, 139, 17, 81, 6, 36, 2, 79, 77, 237, 1, 2, - 78, 32, 4, 250, 207, 12, 80, 171, 199, 22, 69, 2, 131, 215, 34, 71, 48, - 122, 65, 32, 2, 68, 65, 54, 71, 98, 75, 32, 2, 83, 72, 38, 84, 102, 89, - 74, 90, 210, 210, 34, 74, 234, 105, 69, 239, 70, 73, 4, 250, 8, 65, 183, - 134, 36, 81, 4, 22, 65, 159, 22, 32, 2, 153, 44, 3, 78, 71, 71, 8, 52, 3, - 75, 85, 69, 222, 202, 31, 65, 215, 242, 2, 71, 4, 250, 7, 32, 181, 239, - 11, 2, 78, 90, 4, 246, 25, 65, 147, 253, 18, 73, 4, 166, 161, 34, 73, - 183, 236, 1, 65, 8, 40, 2, 65, 80, 229, 184, 27, 2, 79, 81, 7, 11, 32, 4, - 180, 157, 34, 2, 77, 70, 1, 2, 78, 84, 6, 26, 73, 143, 240, 35, 69, 5, - 193, 13, 7, 84, 32, 77, 79, 78, 71, 75, 4, 214, 5, 65, 245, 243, 11, 4, - 85, 78, 32, 77, 22, 58, 65, 80, 3, 79, 78, 32, 166, 238, 35, 69, 163, 28, - 85, 10, 42, 65, 154, 18, 32, 250, 17, 77, 15, 83, 4, 218, 221, 25, 82, - 155, 173, 10, 77, 8, 56, 4, 77, 70, 79, 78, 1, 6, 80, 65, 32, 78, 74, 73, - 4, 37, 7, 32, 80, 73, 80, 65, 69, 77, 4, 206, 16, 71, 231, 246, 35, 66, - 22, 70, 72, 194, 1, 79, 192, 66, 2, 69, 85, 158, 160, 35, 85, 167, 34, - 73, 10, 74, 73, 70, 85, 129, 224, 34, 10, 79, 81, 32, 78, 83, 72, 85, 84, - 32, 89, 4, 158, 219, 25, 82, 233, 194, 6, 8, 78, 68, 65, 32, 80, 65, 32, - 78, 4, 188, 11, 4, 69, 78, 83, 72, 183, 252, 35, 77, 6, 132, 206, 34, 2, - 78, 74, 190, 185, 1, 81, 3, 84, 10, 44, 2, 69, 85, 44, 3, 73, 84, 65, 31, - 85, 4, 234, 181, 34, 65, 1, 4, 84, 69, 85, 87, 2, 11, 32, 2, 223, 30, 77, - 4, 186, 26, 32, 215, 145, 26, 65, 4, 228, 3, 5, 32, 89, 85, 81, 32, 145, - 130, 27, 3, 78, 75, 78, 112, 164, 1, 7, 71, 72, 69, 85, 71, 72, 69, 34, - 75, 126, 76, 122, 77, 154, 3, 78, 234, 2, 80, 90, 83, 178, 1, 84, 106, - 89, 190, 22, 87, 204, 46, 2, 70, 69, 147, 157, 11, 86, 4, 198, 56, 85, - 203, 203, 35, 78, 12, 40, 2, 69, 85, 50, 73, 191, 242, 35, 65, 6, 230, - 54, 89, 246, 139, 12, 80, 191, 174, 23, 65, 4, 230, 240, 35, 69, 175, 18, - 81, 8, 46, 65, 240, 69, 2, 79, 77, 171, 160, 35, 69, 4, 50, 65, 129, 61, - 7, 77, 32, 78, 83, 72, 85, 84, 2, 247, 212, 25, 78, 26, 70, 65, 74, 66, - 140, 1, 2, 69, 85, 58, 70, 145, 23, 3, 79, 78, 84, 7, 21, 3, 32, 78, 74, - 4, 194, 134, 18, 85, 137, 192, 16, 3, 69, 85, 65, 10, 90, 65, 154, 46, - 85, 144, 201, 28, 2, 69, 85, 173, 182, 6, 7, 73, 84, 32, 77, 66, 65, 65, - 4, 150, 12, 65, 245, 242, 18, 4, 32, 77, 65, 69, 4, 252, 155, 31, 5, 84, - 32, 78, 71, 71, 227, 227, 4, 81, 4, 48, 4, 79, 78, 32, 84, 181, 235, 25, - 2, 73, 89, 2, 163, 65, 69, 24, 110, 71, 166, 1, 83, 50, 89, 152, 23, 8, - 84, 73, 69, 69, 32, 83, 72, 69, 193, 245, 33, 5, 68, 85, 32, 78, 74, 12, - 70, 71, 168, 132, 35, 8, 75, 73, 78, 68, 73, 32, 77, 86, 175, 104, 79, 8, - 60, 3, 85, 79, 81, 216, 133, 19, 2, 69, 85, 219, 189, 15, 65, 5, 137, - 208, 27, 2, 32, 76, 4, 26, 72, 227, 172, 35, 69, 2, 167, 199, 35, 85, 4, - 248, 45, 2, 65, 69, 211, 17, 73, 8, 152, 155, 10, 2, 69, 69, 210, 130, - 12, 65, 160, 146, 8, 3, 85, 78, 71, 199, 175, 5, 73, 12, 88, 2, 65, 75, - 16, 2, 72, 69, 160, 236, 18, 2, 69, 84, 230, 161, 15, 73, 227, 213, 1, - 85, 2, 211, 25, 69, 4, 244, 230, 25, 4, 84, 32, 78, 74, 129, 239, 4, 4, - 85, 65, 69, 81, 6, 32, 2, 85, 32, 151, 192, 34, 65, 4, 36, 4, 77, 65, 69, - 77, 139, 7, 78, 2, 11, 71, 2, 135, 7, 66, 4, 44, 4, 65, 70, 85, 32, 229, - 4, 2, 69, 85, 2, 221, 179, 31, 6, 76, 69, 69, 82, 65, 69, 196, 1, 170, 1, - 71, 82, 75, 206, 2, 76, 50, 77, 250, 3, 78, 238, 6, 80, 78, 83, 118, 84, - 176, 1, 3, 86, 69, 85, 46, 87, 62, 89, 250, 179, 29, 66, 166, 208, 2, 70, - 247, 187, 3, 82, 6, 40, 2, 72, 65, 221, 234, 18, 2, 66, 65, 4, 186, 201, - 25, 82, 155, 173, 10, 80, 22, 66, 69, 182, 1, 85, 248, 224, 25, 3, 80, - 65, 82, 139, 145, 10, 65, 14, 40, 2, 78, 32, 54, 85, 143, 245, 35, 84, 4, - 212, 148, 32, 4, 70, 65, 84, 73, 207, 226, 2, 76, 8, 42, 83, 162, 225, - 25, 75, 195, 147, 10, 77, 4, 232, 14, 3, 72, 69, 85, 239, 53, 69, 4, 48, - 6, 79, 80, 32, 78, 75, 65, 135, 244, 35, 84, 2, 11, 65, 2, 223, 198, 25, - 82, 8, 202, 47, 65, 230, 176, 25, 73, 183, 147, 10, 85, 38, 82, 65, 130, - 1, 66, 202, 196, 25, 85, 216, 25, 4, 71, 66, 65, 83, 143, 167, 8, 73, 8, - 18, 32, 79, 69, 4, 42, 78, 209, 159, 17, 4, 75, 69, 85, 65, 2, 11, 83, 2, - 223, 133, 34, 73, 4, 214, 196, 35, 77, 211, 25, 83, 24, 34, 65, 114, 69, - 82, 73, 35, 85, 6, 32, 2, 65, 32, 163, 229, 18, 78, 4, 204, 219, 30, 8, - 67, 65, 66, 66, 65, 71, 69, 45, 129, 195, 4, 2, 80, 73, 8, 50, 85, 130, - 195, 25, 82, 185, 201, 5, 2, 69, 75, 4, 150, 240, 35, 77, 3, 88, 7, 226, - 7, 82, 155, 232, 35, 84, 4, 174, 221, 35, 65, 175, 18, 69, 72, 130, 1, - 65, 54, 68, 110, 71, 222, 1, 74, 102, 83, 130, 1, 84, 102, 90, 237, 7, - 12, 89, 73, 82, 32, 77, 75, 80, 65, 82, 65, 81, 32, 4, 244, 218, 25, 4, - 78, 83, 65, 78, 195, 147, 10, 81, 12, 60, 2, 69, 85, 174, 41, 65, 146, - 205, 18, 79, 135, 210, 16, 73, 4, 128, 129, 34, 2, 65, 69, 195, 236, 1, - 84, 20, 50, 71, 98, 75, 210, 216, 25, 65, 223, 130, 10, 79, 12, 26, 85, - 247, 156, 35, 69, 11, 180, 39, 3, 65, 69, 78, 190, 245, 34, 79, 202, 26, - 69, 143, 53, 77, 4, 36, 3, 85, 69, 32, 171, 216, 25, 65, 2, 145, 230, 18, - 3, 77, 65, 69, 10, 34, 65, 34, 69, 255, 169, 12, 85, 4, 190, 218, 35, 69, - 219, 16, 77, 4, 194, 254, 33, 69, 171, 96, 85, 12, 78, 85, 202, 214, 25, - 72, 216, 216, 5, 2, 69, 85, 146, 170, 4, 79, 219, 16, 65, 4, 212, 214, - 35, 4, 79, 84, 32, 78, 179, 19, 78, 8, 54, 69, 232, 204, 35, 4, 85, 32, - 77, 66, 131, 26, 65, 4, 204, 225, 18, 2, 85, 78, 211, 135, 17, 78, 4, - 254, 192, 34, 69, 247, 167, 1, 65, 6, 26, 73, 227, 152, 35, 69, 4, 26, - 82, 155, 232, 35, 78, 2, 151, 151, 34, 73, 10, 30, 69, 50, 72, 255, 6, - 85, 4, 26, 84, 191, 219, 34, 85, 2, 231, 151, 35, 70, 4, 182, 166, 12, - 85, 203, 173, 13, 73, 10, 40, 2, 65, 65, 34, 69, 41, 2, 73, 84, 2, 11, - 83, 2, 191, 185, 25, 72, 4, 228, 25, 2, 85, 84, 223, 204, 35, 84, 4, 38, - 85, 153, 133, 32, 3, 65, 32, 89, 2, 235, 139, 26, 65, 4, 180, 145, 27, 2, - 65, 69, 171, 212, 8, 88, 4, 40, 4, 65, 78, 71, 75, 255, 228, 35, 85, 2, - 143, 19, 85, 10, 42, 85, 230, 163, 12, 69, 179, 190, 23, 65, 6, 210, 18, - 87, 212, 3, 4, 32, 77, 85, 79, 167, 206, 35, 77, 234, 1, 134, 1, 70, 68, - 2, 71, 72, 34, 75, 254, 1, 76, 142, 1, 77, 130, 3, 78, 130, 5, 80, 122, - 82, 90, 83, 190, 1, 84, 158, 1, 87, 39, 89, 4, 36, 3, 69, 85, 70, 167, - 224, 35, 65, 2, 11, 69, 2, 151, 2, 85, 4, 202, 1, 69, 191, 222, 35, 65, - 22, 46, 69, 146, 1, 85, 42, 87, 135, 244, 33, 89, 10, 26, 85, 215, 225, - 35, 84, 8, 72, 3, 65, 69, 84, 20, 5, 79, 84, 32, 77, 66, 246, 224, 35, - 77, 3, 80, 2, 251, 205, 11, 77, 2, 219, 179, 25, 85, 9, 134, 208, 35, 79, - 218, 16, 78, 3, 81, 2, 171, 171, 35, 65, 14, 58, 69, 182, 204, 25, 79, - 130, 167, 8, 73, 223, 219, 1, 85, 8, 42, 85, 138, 243, 33, 69, 183, 236, - 1, 84, 4, 178, 133, 26, 65, 139, 218, 9, 77, 41, 94, 65, 58, 66, 66, 69, - 38, 70, 80, 2, 71, 66, 214, 140, 31, 79, 246, 214, 3, 86, 135, 121, 85, - 4, 228, 139, 17, 2, 76, 69, 197, 157, 18, 3, 69, 78, 74, 6, 32, 2, 65, - 65, 235, 190, 35, 85, 5, 237, 130, 28, 2, 32, 83, 6, 250, 226, 17, 85, - 147, 142, 16, 69, 10, 44, 2, 69, 85, 238, 228, 33, 79, 207, 11, 73, 4, - 150, 198, 35, 65, 215, 22, 84, 6, 150, 240, 33, 73, 192, 68, 2, 79, 70, - 223, 39, 69, 72, 78, 68, 46, 71, 226, 1, 74, 98, 83, 110, 84, 34, 89, - 210, 215, 35, 73, 3, 85, 8, 194, 39, 69, 178, 228, 34, 79, 255, 62, 65, - 24, 18, 71, 99, 75, 12, 54, 65, 202, 42, 69, 162, 245, 30, 87, 135, 170, - 4, 85, 6, 152, 38, 2, 65, 77, 183, 180, 35, 80, 12, 68, 2, 69, 85, 174, - 237, 33, 73, 2, 89, 226, 156, 1, 85, 203, 79, 65, 4, 226, 152, 12, 65, - 167, 173, 23, 82, 12, 60, 2, 69, 85, 230, 15, 73, 158, 136, 12, 85, 243, - 192, 23, 65, 4, 206, 198, 35, 65, 175, 18, 84, 10, 46, 72, 32, 3, 73, 69, - 69, 183, 199, 35, 85, 4, 254, 187, 35, 85, 219, 5, 69, 4, 138, 216, 35, - 80, 3, 84, 6, 130, 14, 69, 135, 165, 35, 85, 8, 162, 187, 35, 69, 218, 5, - 85, 254, 5, 65, 219, 16, 73, 12, 42, 69, 46, 85, 182, 214, 35, 65, 3, 73, - 4, 208, 169, 25, 2, 85, 84, 155, 173, 10, 69, 4, 146, 186, 35, 85, 175, - 28, 81, 8, 48, 3, 69, 78, 32, 150, 194, 35, 73, 175, 1, 65, 4, 214, 240, - 25, 79, 227, 194, 9, 77, 26, 58, 72, 90, 85, 186, 16, 65, 174, 6, 69, - 167, 161, 35, 79, 12, 54, 69, 162, 193, 25, 79, 222, 255, 9, 73, 219, 19, - 85, 6, 214, 7, 85, 255, 204, 35, 69, 6, 222, 189, 35, 65, 214, 22, 69, 3, - 85, 18, 62, 69, 74, 85, 210, 191, 25, 79, 226, 252, 9, 65, 215, 22, 73, - 8, 26, 85, 255, 230, 33, 69, 6, 186, 130, 34, 65, 246, 208, 1, 78, 3, 84, - 5, 215, 182, 35, 79, 4, 134, 152, 31, 85, 223, 186, 4, 65, 10, 40, 2, 65, - 69, 18, 85, 191, 130, 35, 69, 2, 251, 3, 77, 6, 22, 87, 227, 13, 79, 2, - 195, 190, 25, 79, 186, 2, 178, 1, 70, 154, 1, 71, 202, 1, 75, 170, 1, 76, - 158, 1, 77, 134, 2, 78, 190, 7, 80, 166, 2, 82, 78, 83, 166, 1, 84, 246, - 1, 86, 66, 87, 34, 89, 226, 186, 35, 65, 2, 73, 3, 79, 18, 54, 85, 218, - 172, 22, 65, 202, 140, 13, 69, 255, 5, 79, 10, 26, 32, 231, 154, 30, 69, - 6, 148, 143, 23, 4, 82, 69, 77, 69, 134, 188, 10, 67, 191, 132, 2, 73, - 16, 24, 2, 66, 69, 39, 72, 4, 234, 194, 34, 85, 143, 140, 1, 84, 12, 34, - 65, 34, 69, 187, 189, 35, 79, 2, 11, 65, 2, 139, 161, 25, 77, 8, 26, 85, - 247, 205, 35, 84, 6, 158, 183, 35, 65, 214, 22, 78, 3, 88, 18, 50, 69, - 62, 80, 18, 85, 206, 204, 35, 73, 3, 79, 6, 26, 85, 255, 204, 35, 84, 4, - 166, 182, 35, 65, 215, 22, 88, 2, 211, 28, 69, 6, 158, 176, 35, 69, 162, - 28, 79, 15, 84, 18, 50, 65, 40, 2, 69, 85, 22, 79, 183, 203, 35, 85, 6, - 150, 187, 35, 65, 218, 16, 80, 3, 81, 2, 155, 185, 35, 65, 8, 238, 208, - 17, 79, 198, 250, 17, 77, 3, 81, 32, 110, 65, 44, 2, 66, 69, 34, 70, 20, - 2, 71, 66, 34, 73, 130, 156, 25, 85, 198, 221, 9, 69, 2, 79, 255, 59, 86, - 11, 234, 164, 17, 69, 174, 165, 18, 80, 3, 81, 4, 146, 185, 35, 85, 219, - 16, 69, 2, 227, 136, 12, 69, 4, 226, 249, 34, 69, 215, 79, 65, 5, 195, - 178, 35, 69, 80, 114, 68, 170, 1, 71, 138, 3, 74, 112, 2, 83, 72, 58, 84, - 32, 3, 89, 73, 32, 54, 90, 210, 129, 35, 65, 179, 47, 75, 12, 34, 65, 98, - 73, 187, 247, 34, 85, 6, 32, 2, 65, 32, 203, 199, 35, 80, 4, 128, 215, - 17, 3, 77, 89, 32, 249, 140, 13, 3, 83, 79, 70, 4, 214, 179, 25, 65, 183, - 147, 10, 81, 36, 78, 71, 50, 85, 122, 75, 118, 79, 188, 177, 11, 3, 69, - 85, 82, 195, 146, 24, 65, 12, 18, 69, 51, 85, 6, 26, 85, 183, 217, 33, - 69, 4, 199, 187, 23, 65, 6, 64, 6, 65, 69, 83, 72, 65, 69, 250, 151, 25, - 82, 155, 173, 10, 80, 2, 11, 32, 2, 203, 231, 22, 78, 12, 34, 65, 20, 2, - 69, 85, 35, 85, 5, 231, 176, 35, 65, 4, 138, 178, 35, 65, 175, 18, 88, 4, - 150, 196, 35, 77, 3, 80, 4, 250, 195, 35, 80, 3, 81, 8, 18, 65, 31, 69, - 2, 201, 244, 30, 2, 69, 77, 6, 26, 69, 139, 183, 34, 85, 5, 233, 160, 35, - 4, 32, 69, 80, 79, 6, 26, 85, 163, 214, 33, 73, 4, 198, 194, 35, 79, 15, - 69, 4, 222, 177, 35, 85, 207, 16, 65, 4, 128, 190, 25, 4, 67, 76, 69, 65, - 247, 240, 1, 66, 4, 174, 174, 25, 65, 3, 85, 34, 42, 65, 90, 69, 58, 73, - 50, 79, 23, 85, 8, 32, 2, 32, 80, 207, 155, 17, 65, 4, 152, 176, 27, 2, - 69, 79, 169, 135, 7, 2, 76, 85, 6, 26, 85, 211, 175, 35, 69, 4, 166, 192, - 35, 84, 3, 88, 7, 11, 69, 4, 202, 172, 25, 69, 183, 147, 10, 84, 5, 135, - 240, 34, 79, 11, 82, 65, 246, 190, 35, 69, 3, 77, 8, 46, 65, 238, 14, 69, - 185, 168, 18, 2, 73, 77, 4, 242, 190, 35, 69, 3, 81, 18, 62, 69, 30, 72, - 150, 131, 31, 85, 146, 170, 4, 79, 163, 14, 65, 4, 150, 190, 35, 69, 3, - 84, 8, 42, 69, 182, 154, 22, 79, 235, 143, 3, 73, 2, 209, 252, 11, 2, 85, - 65, 28, 34, 65, 94, 69, 50, 79, 39, 85, 10, 56, 2, 69, 78, 186, 153, 22, - 65, 158, 163, 13, 77, 3, 81, 2, 221, 165, 11, 3, 32, 78, 84, 6, 26, 85, - 155, 188, 35, 78, 5, 155, 251, 11, 65, 6, 130, 196, 33, 79, 131, 248, 1, - 81, 6, 170, 7, 77, 227, 157, 35, 65, 6, 26, 69, 207, 170, 35, 79, 4, 138, - 142, 25, 85, 155, 173, 10, 69, 6, 246, 10, 69, 255, 203, 31, 85, 26, 68, - 2, 69, 85, 46, 73, 32, 3, 79, 81, 32, 54, 85, 143, 185, 35, 65, 8, 246, - 175, 23, 65, 162, 138, 12, 77, 3, 88, 4, 150, 163, 35, 69, 215, 22, 84, - 4, 248, 150, 12, 4, 83, 87, 73, 77, 223, 162, 9, 67, 8, 226, 165, 25, 69, - 194, 194, 8, 65, 246, 208, 1, 78, 3, 81, 108, 162, 1, 75, 82, 76, 46, 77, - 98, 78, 190, 1, 80, 66, 82, 50, 83, 110, 84, 38, 89, 170, 196, 2, 87, - 252, 172, 9, 2, 86, 85, 170, 170, 23, 69, 242, 5, 70, 231, 16, 85, 14, - 242, 188, 17, 69, 146, 142, 16, 89, 254, 233, 1, 80, 186, 2, 65, 2, 79, - 3, 85, 6, 178, 163, 25, 79, 182, 147, 10, 65, 3, 73, 13, 42, 66, 34, 69, - 242, 181, 35, 65, 3, 79, 4, 186, 230, 34, 69, 159, 77, 65, 2, 187, 201, - 33, 69, 22, 94, 71, 38, 74, 38, 85, 250, 188, 33, 83, 246, 7, 68, 150, 3, - 84, 222, 216, 1, 89, 219, 19, 73, 4, 134, 250, 30, 75, 191, 184, 4, 71, - 4, 190, 135, 25, 85, 239, 154, 10, 65, 5, 223, 157, 35, 65, 6, 26, 69, - 239, 134, 25, 85, 4, 246, 167, 34, 85, 143, 140, 1, 69, 12, 186, 2, 69, - 154, 231, 8, 73, 143, 202, 26, 85, 14, 66, 72, 230, 2, 69, 194, 170, 18, - 65, 226, 224, 16, 85, 235, 36, 73, 6, 146, 159, 35, 73, 218, 19, 79, 3, - 85, 6, 186, 190, 20, 65, 223, 215, 14, 69, 4, 234, 158, 25, 79, 183, 147, - 10, 65, 4, 166, 155, 35, 65, 215, 22, 69, 14, 54, 69, 154, 231, 8, 73, - 186, 179, 26, 65, 215, 22, 85, 6, 150, 165, 34, 85, 142, 140, 1, 69, 3, - 78, 16, 62, 72, 50, 69, 194, 170, 18, 65, 226, 224, 16, 85, 235, 36, 73, - 8, 46, 69, 178, 156, 35, 73, 218, 19, 79, 3, 85, 2, 251, 163, 34, 85, 10, - 174, 181, 17, 69, 190, 134, 3, 65, 139, 244, 14, 73, 6, 138, 156, 25, 79, - 2, 85, 183, 147, 10, 65, 14, 42, 75, 166, 244, 33, 65, 211, 155, 1, 74, - 11, 49, 10, 78, 79, 84, 69, 32, 87, 73, 84, 72, 32, 8, 224, 173, 9, 2, - 80, 79, 190, 1, 89, 188, 131, 12, 2, 69, 85, 147, 156, 6, 68, 6, 38, 32, - 237, 172, 16, 3, 66, 69, 82, 4, 170, 226, 28, 67, 189, 186, 5, 5, 79, 70, - 32, 83, 79, 78, 72, 3, 75, 69, 84, 60, 7, 83, 65, 32, 86, 65, 72, 32, - 183, 167, 33, 69, 5, 241, 218, 15, 10, 66, 65, 76, 76, 32, 65, 78, 68, - 32, 72, 72, 104, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 164, 1, 7, - 76, 69, 84, 84, 69, 82, 32, 235, 222, 32, 70, 10, 52, 4, 72, 73, 71, 72, - 44, 3, 76, 79, 87, 39, 77, 4, 172, 140, 21, 2, 45, 76, 251, 207, 10, 32, - 4, 32, 2, 45, 77, 219, 219, 31, 32, 2, 201, 219, 31, 2, 73, 68, 60, 238, - 1, 68, 38, 69, 38, 71, 34, 75, 38, 85, 20, 2, 87, 65, 22, 89, 128, 238, - 27, 2, 72, 87, 154, 196, 2, 77, 178, 181, 2, 79, 154, 154, 2, 86, 246, 5, - 74, 2, 84, 2, 90, 162, 8, 67, 2, 83, 158, 20, 66, 2, 70, 2, 80, 186, 2, - 65, 3, 73, 4, 222, 235, 32, 72, 159, 188, 2, 79, 7, 222, 215, 30, 78, - 251, 207, 4, 69, 4, 202, 130, 35, 66, 219, 35, 65, 4, 202, 238, 27, 80, - 203, 184, 7, 65, 5, 135, 130, 35, 87, 5, 215, 129, 35, 68, 4, 138, 167, - 34, 69, 235, 104, 73, 121, 48, 3, 65, 75, 32, 230, 10, 72, 179, 218, 21, - 84, 112, 196, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, - 78, 32, 28, 7, 76, 69, 84, 84, 69, 82, 32, 248, 4, 3, 80, 65, 78, 50, 83, - 141, 2, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 4, 214, 163, 35, - 78, 87, 72, 76, 194, 1, 77, 114, 78, 68, 2, 80, 65, 38, 83, 232, 144, 18, - 4, 75, 65, 82, 79, 214, 141, 17, 66, 2, 67, 2, 68, 2, 71, 2, 72, 2, 74, - 2, 76, 2, 82, 2, 87, 2, 89, 186, 2, 65, 2, 73, 3, 85, 10, 26, 65, 251, - 159, 35, 66, 9, 45, 9, 78, 68, 65, 73, 76, 73, 78, 71, 32, 6, 198, 159, - 35, 72, 2, 78, 3, 83, 10, 152, 2, 2, 79, 82, 138, 157, 35, 68, 2, 71, 2, - 89, 187, 2, 65, 5, 189, 165, 20, 4, 75, 80, 65, 75, 24, 80, 10, 73, 77, - 65, 76, 85, 78, 71, 85, 78, 32, 96, 2, 79, 85, 195, 159, 35, 65, 20, 230, - 157, 35, 71, 2, 72, 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 87, 2, 89, 187, - 2, 65, 2, 145, 145, 18, 5, 84, 72, 69, 82, 78, 4, 166, 2, 71, 161, 145, - 18, 4, 79, 78, 71, 79, 10, 96, 12, 89, 77, 66, 79, 76, 32, 66, 73, 78, - 68, 85, 32, 193, 140, 10, 6, 73, 71, 78, 32, 84, 79, 8, 78, 80, 152, 243, - 10, 3, 74, 85, 68, 249, 166, 24, 6, 78, 65, 32, 77, 69, 84, 4, 64, 3, 65, - 78, 71, 133, 199, 22, 7, 73, 78, 65, 82, 66, 79, 82, 2, 139, 131, 25, 79, - 18, 122, 85, 228, 173, 25, 6, 80, 65, 75, 80, 65, 75, 180, 248, 2, 5, 75, - 65, 82, 79, 32, 182, 181, 6, 69, 150, 64, 73, 3, 79, 5, 177, 200, 29, 15, - 32, 70, 79, 82, 32, 83, 73, 77, 65, 76, 85, 78, 71, 85, 78, 5, 143, 134, - 23, 84, 254, 1, 134, 1, 65, 230, 2, 69, 50, 76, 130, 1, 78, 198, 10, 84, - 144, 220, 31, 2, 67, 65, 204, 162, 2, 5, 86, 69, 82, 65, 71, 191, 140, 1, - 68, 20, 136, 1, 4, 77, 69, 68, 32, 170, 1, 82, 164, 143, 3, 10, 67, 72, - 32, 87, 73, 84, 72, 32, 85, 77, 210, 144, 7, 84, 234, 185, 24, 86, 19, - 78, 8, 86, 65, 0, 2, 68, 69, 52, 4, 69, 73, 71, 72, 1, 7, 83, 73, 88, 84, - 69, 69, 78, 2, 201, 214, 19, 8, 83, 67, 69, 78, 68, 73, 78, 71, 2, 193, - 214, 19, 2, 84, 72, 4, 128, 196, 28, 3, 68, 69, 68, 179, 150, 6, 32, 4, - 176, 166, 8, 3, 82, 32, 77, 183, 207, 25, 84, 13, 11, 76, 11, 38, 32, - 217, 187, 24, 3, 72, 79, 80, 6, 162, 140, 12, 80, 212, 232, 15, 6, 87, - 73, 84, 72, 32, 67, 187, 142, 6, 83, 208, 1, 84, 5, 71, 65, 76, 73, 32, - 162, 9, 84, 37, 9, 90, 69, 78, 69, 32, 82, 73, 78, 71, 200, 1, 210, 1, - 65, 44, 9, 67, 85, 82, 82, 69, 78, 67, 89, 32, 148, 2, 7, 76, 69, 84, 84, - 69, 82, 32, 204, 3, 6, 82, 85, 80, 69, 69, 32, 34, 83, 242, 128, 22, 73, - 130, 4, 86, 242, 153, 11, 68, 141, 105, 3, 71, 65, 78, 6, 190, 147, 30, - 85, 138, 149, 1, 66, 51, 78, 12, 120, 10, 78, 85, 77, 69, 82, 65, 84, 79, - 82, 32, 221, 212, 22, 14, 68, 69, 78, 79, 77, 73, 78, 65, 84, 79, 82, 32, - 83, 73, 10, 52, 3, 79, 78, 69, 214, 170, 30, 70, 155, 250, 2, 84, 5, 165, - 201, 28, 19, 32, 76, 69, 83, 83, 32, 84, 72, 65, 78, 32, 84, 72, 69, 32, - 68, 69, 78, 79, 108, 226, 1, 75, 90, 82, 226, 232, 20, 86, 234, 239, 7, - 89, 150, 205, 2, 65, 38, 68, 46, 84, 230, 24, 85, 210, 200, 1, 73, 158, - 190, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 80, 254, 68, 72, 2, - 76, 2, 77, 186, 2, 69, 3, 79, 8, 26, 72, 235, 142, 35, 65, 6, 26, 65, - 155, 208, 13, 73, 5, 157, 128, 18, 3, 78, 68, 65, 10, 34, 65, 210, 139, - 35, 72, 3, 82, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 244, 179, 24, 5, 76, - 79, 87, 69, 82, 1, 6, 77, 73, 68, 68, 76, 69, 4, 202, 134, 34, 83, 179, - 69, 77, 20, 116, 19, 69, 81, 85, 69, 78, 67, 69, 32, 70, 79, 82, 32, 76, - 69, 84, 84, 69, 82, 32, 154, 154, 25, 65, 187, 138, 6, 73, 6, 150, 131, - 22, 82, 147, 134, 13, 89, 4, 182, 173, 25, 32, 251, 209, 8, 79, 5, 137, - 247, 31, 4, 32, 87, 73, 84, 4, 150, 186, 33, 87, 255, 61, 32, 194, 1, - 180, 2, 7, 76, 69, 84, 84, 69, 82, 32, 244, 1, 7, 78, 85, 77, 66, 69, 82, - 32, 72, 5, 83, 73, 71, 78, 32, 44, 11, 86, 79, 87, 69, 76, 32, 83, 73, - 71, 78, 32, 250, 143, 25, 68, 130, 229, 3, 87, 136, 97, 10, 71, 65, 80, - 32, 70, 73, 76, 76, 69, 82, 153, 229, 4, 12, 72, 85, 78, 68, 82, 69, 68, - 83, 32, 85, 78, 73, 92, 210, 1, 86, 154, 158, 31, 65, 38, 68, 46, 84, - 230, 24, 85, 210, 200, 1, 73, 158, 190, 1, 78, 46, 83, 82, 66, 2, 67, 2, - 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, 76, 2, 77, 2, 82, 2, 89, 186, 2, - 69, 3, 79, 8, 234, 1, 79, 255, 132, 35, 65, 36, 138, 100, 69, 38, 70, 66, - 78, 26, 83, 250, 241, 20, 84, 183, 150, 12, 79, 10, 142, 159, 31, 65, 74, - 67, 171, 163, 3, 86, 24, 84, 2, 86, 79, 166, 160, 31, 65, 190, 21, 85, - 210, 200, 1, 73, 206, 134, 2, 69, 3, 79, 6, 33, 6, 67, 65, 76, 73, 67, - 32, 6, 178, 161, 31, 82, 167, 227, 3, 76, 26, 148, 1, 4, 67, 89, 67, 76, - 36, 2, 71, 32, 28, 2, 76, 76, 46, 82, 66, 84, 212, 216, 18, 4, 79, 72, - 65, 90, 164, 217, 11, 2, 75, 73, 155, 128, 4, 83, 4, 210, 187, 31, 73, - 203, 199, 3, 69, 4, 186, 240, 32, 82, 67, 83, 4, 164, 175, 9, 2, 69, 68, - 219, 251, 22, 73, 4, 176, 188, 31, 7, 84, 72, 68, 65, 89, 32, 67, 255, - 197, 3, 68, 4, 160, 165, 13, 2, 67, 79, 149, 219, 21, 4, 73, 78, 71, 32, - 230, 3, 42, 65, 222, 29, 79, 249, 3, 2, 85, 69, 228, 2, 32, 2, 67, 75, - 135, 159, 17, 78, 226, 2, 22, 32, 143, 28, 45, 210, 2, 214, 1, 67, 254, - 4, 68, 174, 2, 70, 102, 72, 82, 76, 170, 2, 77, 250, 2, 80, 46, 82, 202, - 3, 83, 210, 3, 84, 82, 85, 248, 2, 3, 86, 69, 82, 198, 215, 11, 79, 208, - 240, 19, 3, 66, 79, 87, 154, 227, 1, 78, 179, 1, 81, 98, 196, 1, 5, 73, - 82, 67, 76, 69, 200, 1, 6, 85, 82, 86, 69, 68, 32, 192, 150, 25, 12, 82, - 79, 83, 83, 32, 79, 78, 32, 83, 72, 73, 69, 240, 155, 3, 5, 69, 78, 84, - 82, 69, 178, 238, 4, 72, 203, 13, 76, 11, 11, 32, 8, 72, 5, 87, 73, 84, - 72, 32, 145, 164, 16, 7, 70, 79, 82, 32, 82, 69, 67, 6, 224, 51, 8, 87, - 72, 73, 84, 69, 32, 68, 79, 238, 181, 18, 68, 221, 144, 2, 8, 84, 87, 79, - 32, 87, 72, 73, 84, 16, 84, 4, 68, 79, 87, 78, 0, 2, 85, 80, 56, 3, 76, - 69, 70, 1, 4, 82, 73, 71, 72, 4, 141, 139, 31, 9, 87, 65, 82, 68, 83, 32, - 65, 78, 68, 4, 53, 11, 84, 87, 65, 82, 68, 83, 32, 65, 78, 68, 32, 4, - 178, 164, 10, 85, 179, 237, 21, 68, 30, 64, 6, 73, 65, 77, 79, 78, 68, - 152, 1, 3, 79, 87, 78, 43, 82, 13, 11, 32, 10, 210, 15, 67, 152, 208, 12, - 10, 77, 73, 78, 85, 83, 32, 87, 72, 73, 84, 184, 134, 6, 6, 87, 73, 84, - 72, 32, 68, 242, 212, 14, 79, 131, 11, 83, 12, 142, 16, 32, 62, 45, 175, - 185, 30, 87, 6, 196, 155, 33, 2, 79, 80, 175, 20, 65, 8, 18, 76, 39, 79, - 4, 150, 134, 27, 79, 191, 240, 7, 65, 4, 172, 192, 2, 2, 85, 82, 231, - 146, 30, 76, 12, 38, 69, 146, 179, 33, 65, 239, 2, 79, 6, 216, 180, 33, - 2, 65, 82, 215, 10, 88, 32, 60, 3, 69, 70, 84, 202, 1, 79, 161, 228, 32, - 3, 65, 82, 71, 22, 96, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 64, 6, - 87, 65, 82, 68, 83, 32, 155, 183, 33, 32, 12, 162, 6, 68, 246, 7, 73, - 174, 170, 33, 80, 166, 24, 83, 51, 84, 4, 226, 142, 32, 69, 143, 137, 1, - 66, 6, 230, 13, 87, 143, 178, 33, 90, 30, 76, 6, 69, 68, 73, 85, 77, 32, - 197, 145, 21, 7, 79, 79, 78, 32, 76, 73, 76, 28, 66, 68, 42, 76, 36, 4, - 82, 73, 71, 72, 12, 2, 85, 80, 111, 83, 6, 84, 3, 79, 87, 78, 219, 182, - 33, 73, 6, 32, 2, 69, 70, 235, 189, 33, 79, 4, 11, 84, 4, 81, 18, 45, 80, - 79, 73, 78, 84, 73, 78, 71, 32, 84, 82, 73, 65, 78, 71, 76, 69, 5, 217, - 7, 2, 32, 67, 8, 130, 12, 77, 207, 196, 33, 81, 8, 142, 169, 24, 85, 222, - 141, 9, 65, 87, 69, 32, 52, 4, 73, 71, 72, 84, 178, 228, 32, 79, 251, - 105, 69, 28, 100, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 244, 1, 6, - 87, 65, 82, 68, 83, 32, 159, 199, 33, 32, 16, 90, 68, 88, 8, 84, 82, 73, - 65, 78, 71, 76, 69, 158, 7, 73, 162, 175, 33, 80, 179, 19, 83, 4, 65, 14, - 79, 85, 66, 76, 69, 32, 84, 82, 73, 65, 78, 71, 76, 69, 5, 247, 146, 12, - 32, 5, 197, 153, 20, 11, 32, 87, 73, 84, 72, 32, 68, 79, 85, 66, 76, 8, - 226, 211, 18, 65, 134, 180, 13, 69, 143, 137, 1, 66, 38, 150, 2, 77, 84, - 5, 81, 85, 65, 82, 69, 172, 156, 25, 3, 85, 78, 32, 196, 223, 1, 5, 75, - 85, 76, 76, 32, 208, 164, 3, 3, 78, 79, 87, 208, 195, 1, 5, 65, 70, 69, - 84, 89, 164, 118, 13, 76, 73, 71, 72, 84, 76, 89, 32, 83, 77, 65, 76, 76, - 146, 90, 67, 50, 72, 226, 1, 80, 143, 17, 84, 10, 40, 4, 65, 76, 76, 32, - 239, 182, 33, 73, 8, 134, 175, 33, 68, 134, 7, 76, 71, 83, 9, 11, 32, 6, - 54, 67, 228, 158, 32, 3, 70, 79, 82, 243, 152, 1, 66, 2, 245, 137, 30, 3, - 69, 78, 84, 14, 180, 4, 3, 73, 78, 89, 134, 221, 22, 82, 226, 221, 10, - 79, 54, 69, 135, 2, 87, 18, 38, 80, 133, 208, 31, 3, 78, 73, 86, 16, 46, - 32, 62, 45, 170, 1, 80, 135, 184, 30, 87, 2, 185, 194, 33, 10, 80, 79, - 73, 78, 84, 73, 78, 71, 32, 66, 8, 45, 9, 80, 79, 73, 78, 84, 73, 78, 71, - 32, 8, 66, 73, 236, 190, 33, 5, 68, 79, 85, 66, 76, 230, 3, 83, 51, 84, - 2, 129, 199, 32, 8, 83, 79, 83, 67, 69, 76, 69, 83, 4, 21, 3, 69, 82, 32, - 4, 190, 222, 23, 76, 143, 232, 8, 82, 10, 56, 6, 84, 73, 67, 65, 76, 32, - 29, 4, 89, 32, 83, 77, 4, 134, 195, 33, 69, 51, 82, 6, 21, 3, 65, 76, 76, - 6, 11, 32, 6, 182, 169, 33, 68, 134, 7, 76, 227, 19, 83, 16, 84, 15, 76, - 69, 84, 84, 69, 82, 32, 67, 65, 80, 73, 84, 65, 76, 32, 155, 234, 10, 70, - 10, 218, 227, 34, 67, 2, 72, 2, 73, 2, 82, 3, 90, 124, 84, 11, 67, 75, - 32, 83, 69, 88, 84, 65, 78, 84, 45, 238, 128, 2, 83, 135, 181, 10, 87, - 120, 62, 49, 214, 1, 50, 46, 51, 38, 52, 42, 53, 199, 223, 34, 54, 61, - 50, 50, 118, 51, 126, 52, 42, 53, 199, 223, 34, 54, 31, 46, 51, 194, 1, - 52, 42, 53, 199, 223, 34, 54, 15, 38, 52, 194, 1, 53, 199, 223, 34, 54, - 7, 130, 225, 34, 53, 3, 54, 15, 122, 52, 190, 248, 27, 53, 179, 231, 6, - 54, 31, 42, 51, 66, 52, 14, 53, 199, 223, 34, 54, 17, 34, 52, 42, 53, - 199, 223, 34, 54, 9, 38, 53, 199, 223, 34, 54, 7, 11, 53, 5, 195, 223, - 34, 54, 6, 178, 207, 21, 32, 241, 166, 8, 4, 66, 69, 82, 82, 144, 4, 200, - 1, 3, 76, 68, 32, 78, 79, 104, 7, 80, 79, 77, 79, 70, 79, 32, 188, 6, 2, - 84, 84, 140, 3, 5, 85, 81, 85, 69, 84, 54, 87, 198, 1, 88, 210, 204, 5, - 77, 194, 153, 3, 89, 218, 177, 25, 65, 255, 33, 78, 14, 246, 169, 8, 83, - 210, 164, 7, 69, 134, 252, 16, 70, 234, 2, 87, 207, 10, 71, 10, 26, 75, - 155, 198, 22, 77, 9, 40, 4, 77, 65, 82, 75, 155, 220, 34, 83, 5, 197, - 224, 22, 3, 32, 84, 65, 150, 1, 96, 13, 70, 73, 78, 65, 76, 32, 76, 69, - 84, 84, 69, 82, 32, 53, 7, 76, 69, 84, 84, 69, 82, 32, 10, 142, 219, 34, - 71, 2, 72, 2, 75, 2, 80, 3, 84, 140, 1, 234, 1, 65, 54, 85, 22, 69, 82, - 71, 46, 73, 70, 78, 38, 79, 98, 90, 226, 138, 7, 75, 162, 215, 23, 67, 2, - 76, 2, 83, 198, 36, 66, 210, 200, 1, 74, 206, 134, 2, 68, 2, 70, 2, 72, - 2, 77, 2, 80, 2, 81, 2, 82, 2, 84, 2, 86, 3, 88, 21, 50, 73, 2, 85, 74, - 78, 242, 215, 34, 72, 3, 77, 5, 227, 136, 34, 78, 17, 50, 78, 242, 215, - 34, 69, 2, 72, 2, 73, 3, 82, 7, 238, 215, 34, 71, 3, 78, 11, 210, 215, - 34, 72, 2, 78, 2, 85, 3, 87, 15, 164, 223, 32, 2, 78, 78, 130, 248, 1, - 72, 2, 77, 2, 82, 3, 85, 9, 206, 222, 32, 71, 151, 248, 1, 78, 17, 66, - 78, 146, 213, 33, 32, 238, 128, 1, 69, 2, 77, 2, 79, 3, 85, 4, 250, 213, - 34, 71, 3, 78, 9, 222, 213, 34, 72, 2, 73, 3, 89, 34, 104, 3, 79, 77, 32, - 197, 238, 20, 17, 76, 69, 32, 87, 73, 84, 72, 32, 80, 79, 80, 80, 73, 78, - 71, 32, 67, 32, 168, 1, 5, 72, 65, 76, 70, 32, 52, 14, 83, 81, 85, 65, - 82, 69, 32, 66, 82, 65, 67, 75, 69, 84, 134, 157, 15, 65, 166, 217, 15, - 67, 134, 2, 80, 126, 76, 22, 82, 175, 1, 84, 8, 226, 247, 30, 66, 46, 76, - 26, 82, 255, 214, 1, 73, 5, 193, 250, 30, 11, 32, 79, 86, 69, 82, 32, 84, - 79, 80, 32, 83, 5, 229, 195, 33, 8, 32, 79, 70, 32, 70, 76, 79, 87, 14, - 58, 76, 116, 3, 84, 73, 69, 181, 252, 31, 3, 32, 65, 78, 6, 26, 32, 187, - 128, 34, 73, 4, 160, 158, 9, 7, 79, 70, 32, 72, 89, 71, 73, 197, 161, 22, - 6, 87, 73, 84, 72, 32, 83, 7, 235, 196, 31, 32, 162, 2, 88, 10, 32, 68, - 82, 65, 87, 73, 78, 71, 83, 32, 137, 211, 33, 6, 73, 78, 71, 32, 71, 76, - 160, 2, 176, 1, 2, 68, 79, 148, 4, 6, 72, 69, 65, 86, 89, 32, 194, 2, 76, - 156, 17, 6, 82, 73, 71, 72, 84, 32, 144, 4, 3, 85, 80, 32, 245, 3, 9, 86, - 69, 82, 84, 73, 67, 65, 76, 32, 58, 48, 5, 85, 66, 76, 69, 32, 89, 3, 87, - 78, 32, 22, 68, 4, 68, 79, 87, 78, 0, 2, 85, 80, 250, 21, 86, 155, 177, - 28, 72, 6, 171, 22, 32, 36, 128, 1, 10, 72, 69, 65, 86, 89, 32, 65, 78, - 68, 32, 132, 1, 10, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 230, 29, 68, - 131, 4, 83, 12, 76, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 248, 26, 2, 85, - 80, 151, 5, 72, 4, 17, 2, 84, 32, 4, 250, 23, 85, 227, 136, 34, 76, 12, - 76, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 176, 27, 2, 85, 80, 235, 4, 72, - 4, 17, 2, 84, 32, 4, 218, 23, 85, 135, 9, 72, 46, 120, 4, 76, 69, 70, 84, - 68, 2, 85, 80, 176, 14, 2, 68, 79, 106, 81, 34, 84, 234, 1, 86, 154, 177, - 28, 72, 199, 219, 5, 82, 5, 45, 9, 32, 65, 78, 68, 32, 76, 73, 71, 72, 2, - 207, 250, 32, 84, 11, 29, 5, 32, 65, 78, 68, 32, 8, 42, 76, 238, 193, 28, - 72, 199, 219, 5, 82, 4, 204, 223, 23, 4, 73, 71, 72, 84, 211, 189, 10, - 69, 108, 56, 4, 69, 70, 84, 32, 169, 2, 5, 73, 71, 72, 84, 32, 16, 244, - 18, 19, 68, 79, 87, 78, 32, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 82, - 73, 71, 72, 24, 14, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 82, 73, 71, - 72, 100, 14, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 82, 73, 71, 72, 97, - 17, 85, 80, 32, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 82, 73, 71, 72, - 92, 172, 1, 4, 65, 82, 67, 32, 30, 68, 172, 9, 4, 76, 69, 70, 84, 62, 81, - 34, 84, 112, 2, 85, 80, 122, 86, 176, 134, 34, 10, 72, 79, 82, 73, 90, - 79, 78, 84, 65, 76, 175, 6, 82, 8, 158, 143, 17, 68, 67, 85, 48, 52, 8, - 73, 65, 71, 79, 78, 65, 76, 32, 203, 8, 79, 36, 100, 7, 77, 73, 68, 68, - 76, 69, 32, 216, 2, 6, 85, 80, 80, 69, 82, 32, 194, 129, 33, 67, 191, 2, - 68, 12, 88, 8, 76, 69, 70, 84, 32, 84, 79, 32, 161, 1, 9, 82, 73, 71, 72, - 84, 32, 84, 79, 32, 8, 136, 1, 28, 85, 80, 80, 69, 82, 32, 67, 69, 78, - 84, 82, 69, 32, 84, 79, 32, 77, 73, 68, 68, 76, 69, 32, 82, 73, 71, 72, - 84, 195, 3, 76, 5, 143, 132, 20, 32, 4, 232, 4, 15, 85, 80, 80, 69, 82, - 32, 67, 69, 78, 84, 82, 69, 32, 84, 79, 175, 255, 19, 76, 20, 196, 1, 17, - 67, 69, 78, 84, 82, 69, 32, 84, 79, 32, 77, 73, 68, 68, 76, 69, 32, 228, - 239, 22, 7, 76, 69, 70, 84, 32, 84, 79, 193, 155, 11, 14, 82, 73, 71, 72, - 84, 32, 84, 79, 32, 76, 79, 87, 69, 82, 16, 56, 4, 76, 69, 70, 84, 165, - 1, 5, 82, 73, 71, 72, 84, 9, 11, 32, 6, 60, 4, 84, 79, 32, 76, 137, 128, - 20, 5, 65, 78, 68, 32, 77, 4, 53, 11, 79, 87, 69, 82, 32, 67, 69, 78, 84, - 82, 69, 5, 169, 198, 31, 3, 32, 84, 79, 9, 11, 32, 6, 88, 3, 65, 78, 68, - 65, 15, 84, 79, 32, 76, 79, 87, 69, 82, 32, 67, 69, 78, 84, 82, 69, 2, - 249, 254, 19, 11, 32, 77, 73, 68, 68, 76, 69, 32, 76, 69, 70, 5, 165, - 187, 10, 9, 32, 84, 79, 32, 77, 73, 68, 68, 76, 12, 164, 1, 2, 85, 66, - 241, 1, 2, 87, 78, 5, 221, 235, 32, 10, 32, 65, 78, 68, 32, 72, 69, 65, - 86, 89, 4, 49, 5, 85, 65, 68, 82, 85, 4, 17, 2, 82, 73, 4, 11, 80, 4, 41, - 8, 76, 69, 32, 68, 65, 83, 72, 32, 4, 230, 130, 16, 86, 207, 175, 12, 72, - 11, 29, 5, 32, 65, 78, 68, 32, 8, 34, 72, 246, 140, 34, 76, 31, 82, 4, - 172, 207, 23, 4, 69, 65, 86, 89, 171, 226, 4, 79, 8, 37, 7, 69, 82, 84, - 73, 67, 65, 76, 9, 11, 32, 6, 25, 4, 65, 78, 68, 32, 6, 198, 176, 28, 72, - 170, 219, 5, 76, 31, 82, 16, 148, 2, 18, 68, 79, 87, 78, 32, 72, 69, 65, - 86, 89, 32, 65, 78, 68, 32, 76, 69, 70, 24, 13, 72, 69, 65, 86, 89, 32, - 65, 78, 68, 32, 76, 69, 70, 100, 13, 76, 73, 71, 72, 84, 32, 65, 78, 68, - 32, 76, 69, 70, 97, 16, 85, 80, 32, 72, 69, 65, 86, 89, 32, 65, 78, 68, - 32, 76, 69, 70, 2, 101, 3, 84, 32, 85, 6, 17, 2, 84, 32, 6, 58, 85, 178, - 3, 68, 245, 4, 6, 86, 69, 82, 84, 73, 67, 2, 227, 157, 32, 80, 6, 17, 2, - 84, 32, 6, 58, 85, 134, 4, 68, 205, 4, 6, 86, 69, 82, 84, 73, 67, 2, 239, - 8, 80, 2, 185, 2, 3, 84, 32, 68, 36, 128, 1, 10, 72, 69, 65, 86, 89, 32, - 65, 78, 68, 32, 188, 1, 10, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 186, - 2, 68, 131, 4, 83, 12, 80, 4, 68, 79, 87, 78, 24, 3, 76, 69, 70, 0, 4, - 82, 73, 71, 72, 255, 4, 72, 2, 145, 5, 2, 32, 72, 4, 17, 2, 84, 32, 4, - 26, 68, 179, 133, 34, 76, 2, 177, 154, 32, 3, 79, 87, 78, 12, 80, 4, 68, - 79, 87, 78, 24, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 211, 4, 72, 2, 229, - 4, 2, 32, 72, 4, 17, 2, 84, 32, 4, 22, 68, 131, 5, 72, 2, 233, 4, 3, 79, - 87, 78, 24, 130, 1, 68, 188, 1, 10, 72, 69, 65, 86, 89, 32, 65, 78, 68, - 32, 144, 1, 10, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 183, 1, 83, 6, - 49, 10, 79, 85, 66, 76, 69, 32, 65, 78, 68, 32, 6, 92, 3, 76, 69, 70, 0, - 4, 82, 73, 71, 72, 13, 10, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 2, 11, - 84, 2, 189, 157, 26, 2, 32, 83, 6, 54, 72, 68, 3, 76, 69, 70, 1, 4, 82, - 73, 71, 72, 2, 37, 7, 79, 82, 73, 90, 79, 78, 84, 2, 189, 149, 32, 2, 65, - 76, 2, 163, 149, 32, 84, 6, 54, 72, 60, 3, 76, 69, 70, 1, 4, 82, 73, 71, - 72, 2, 37, 7, 79, 82, 73, 90, 79, 78, 84, 2, 29, 2, 65, 76, 2, 11, 84, 2, - 17, 2, 32, 72, 2, 253, 151, 34, 3, 69, 65, 86, 6, 49, 10, 73, 78, 71, 76, - 69, 32, 65, 78, 68, 32, 6, 96, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 165, - 225, 25, 9, 72, 79, 82, 73, 90, 79, 78, 84, 65, 2, 175, 225, 25, 84, 134, - 6, 46, 65, 186, 14, 69, 150, 1, 73, 191, 1, 79, 232, 5, 36, 4, 72, 77, - 73, 32, 255, 9, 73, 230, 1, 192, 1, 7, 76, 69, 84, 84, 69, 82, 32, 196, - 2, 7, 78, 85, 77, 66, 69, 82, 32, 144, 2, 12, 80, 85, 78, 67, 84, 85, 65, - 84, 73, 79, 78, 32, 84, 5, 83, 73, 71, 78, 32, 126, 86, 215, 169, 24, 68, - 108, 210, 1, 79, 178, 187, 30, 65, 38, 68, 46, 84, 46, 86, 186, 24, 85, - 210, 200, 1, 73, 42, 76, 246, 189, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, - 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, 77, 2, 82, 2, 89, 187, 2, 69, 15, - 45, 9, 76, 68, 32, 84, 65, 77, 73, 76, 32, 12, 226, 165, 28, 76, 186, - 153, 2, 83, 190, 66, 78, 199, 221, 2, 82, 42, 82, 69, 38, 70, 66, 78, 26, - 83, 250, 241, 20, 84, 174, 213, 5, 79, 155, 156, 7, 74, 4, 193, 175, 30, - 4, 73, 71, 72, 84, 8, 26, 79, 135, 218, 20, 73, 4, 134, 184, 32, 82, 167, - 177, 1, 85, 4, 65, 3, 73, 78, 69, 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, - 5, 203, 143, 34, 84, 10, 46, 76, 170, 188, 12, 68, 169, 16, 2, 67, 82, 4, - 178, 138, 19, 79, 139, 253, 13, 73, 12, 204, 155, 16, 9, 79, 76, 68, 32, - 84, 65, 77, 73, 76, 182, 158, 14, 67, 198, 180, 1, 74, 158, 2, 86, 122, - 85, 231, 233, 1, 65, 34, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, - 247, 236, 31, 73, 32, 142, 1, 79, 140, 213, 28, 11, 66, 72, 65, 84, 84, - 73, 80, 82, 79, 76, 85, 250, 227, 1, 65, 106, 86, 214, 20, 85, 210, 200, - 1, 73, 207, 134, 2, 69, 7, 165, 185, 30, 10, 76, 68, 32, 84, 65, 77, 73, - 76, 32, 83, 130, 4, 72, 12, 76, 76, 69, 32, 80, 65, 84, 84, 69, 82, 78, - 32, 211, 156, 34, 78, 128, 4, 44, 5, 68, 79, 84, 83, 45, 215, 243, 6, 66, - 254, 3, 74, 49, 74, 50, 66, 51, 54, 52, 46, 53, 38, 54, 30, 55, 167, 153, - 34, 56, 129, 2, 66, 50, 66, 51, 54, 52, 46, 53, 38, 54, 30, 55, 167, 153, - 34, 56, 129, 1, 58, 51, 54, 52, 46, 53, 38, 54, 30, 55, 167, 153, 34, 56, - 65, 50, 52, 46, 53, 38, 54, 30, 55, 167, 153, 34, 56, 33, 42, 53, 38, 54, - 30, 55, 167, 153, 34, 56, 17, 34, 54, 30, 55, 167, 153, 34, 56, 9, 26, - 55, 167, 153, 34, 56, 5, 163, 153, 34, 56, 8, 26, 65, 163, 130, 34, 86, - 6, 212, 131, 20, 11, 75, 32, 80, 69, 82, 77, 73, 84, 84, 69, 68, 196, - 135, 8, 6, 83, 84, 45, 70, 69, 69, 219, 141, 6, 68, 10, 42, 68, 108, 2, - 69, 70, 207, 147, 34, 67, 4, 84, 6, 71, 69, 32, 65, 84, 32, 133, 234, 31, - 9, 69, 32, 87, 73, 84, 72, 32, 86, 69, 2, 167, 236, 33, 78, 4, 190, 134, - 31, 67, 167, 144, 3, 83, 12, 84, 4, 75, 69, 78, 32, 232, 177, 8, 2, 67, - 67, 220, 220, 24, 2, 87, 78, 207, 118, 79, 6, 140, 227, 26, 17, 67, 73, - 82, 67, 76, 69, 32, 87, 73, 84, 72, 32, 78, 79, 82, 84, 72, 178, 144, 6, - 66, 155, 27, 72, 134, 1, 208, 1, 4, 66, 66, 76, 69, 46, 71, 156, 4, 4, - 72, 73, 68, 32, 36, 2, 76, 76, 122, 83, 56, 4, 84, 84, 69, 82, 220, 230, - 10, 10, 73, 76, 68, 73, 78, 71, 32, 67, 79, 78, 208, 154, 16, 2, 82, 82, - 155, 185, 6, 67, 4, 216, 195, 27, 2, 32, 84, 195, 207, 6, 83, 63, 33, 6, - 73, 78, 69, 83, 69, 32, 60, 144, 1, 7, 76, 69, 84, 84, 69, 82, 32, 172, - 2, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 154, 252, 15, 69, 237, - 205, 13, 4, 80, 65, 76, 76, 46, 154, 1, 77, 34, 78, 198, 141, 34, 66, 2, - 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, 80, 2, 82, 2, 83, 2, 84, - 2, 86, 2, 89, 187, 2, 65, 4, 226, 141, 34, 80, 187, 2, 65, 12, 46, 71, - 34, 89, 246, 140, 34, 82, 187, 2, 65, 4, 146, 141, 34, 75, 187, 2, 65, 4, - 242, 140, 34, 67, 187, 2, 65, 10, 182, 248, 33, 65, 214, 22, 69, 2, 73, - 2, 79, 3, 85, 40, 134, 134, 10, 76, 183, 169, 18, 86, 10, 56, 2, 69, 84, - 20, 4, 72, 79, 82, 78, 151, 149, 9, 83, 5, 211, 227, 30, 32, 5, 221, 237, - 26, 5, 32, 87, 73, 84, 72, 9, 26, 84, 203, 194, 31, 32, 4, 214, 237, 26, - 83, 15, 32, 5, 207, 233, 33, 70, 240, 3, 140, 1, 23, 90, 65, 78, 84, 73, - 78, 69, 32, 77, 85, 83, 73, 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, - 173, 254, 19, 5, 84, 69, 32, 79, 82, 238, 3, 154, 2, 65, 128, 7, 2, 67, - 72, 170, 1, 68, 158, 5, 69, 206, 2, 70, 166, 8, 71, 202, 3, 73, 162, 2, - 75, 142, 5, 76, 190, 2, 77, 250, 4, 79, 114, 80, 174, 4, 82, 42, 83, 194, - 5, 84, 196, 5, 2, 86, 65, 250, 1, 89, 238, 162, 31, 78, 149, 136, 2, 9, - 88, 73, 82, 79, 78, 32, 75, 76, 65, 62, 60, 5, 71, 79, 71, 73, 32, 222, - 1, 78, 118, 80, 199, 2, 82, 16, 70, 65, 0, 2, 71, 79, 64, 2, 77, 69, 37, - 5, 80, 79, 76, 73, 32, 4, 17, 2, 82, 71, 4, 200, 181, 15, 2, 79, 84, 211, - 210, 18, 73, 4, 134, 138, 9, 84, 147, 234, 24, 83, 4, 26, 65, 1, 2, 71, - 79, 2, 223, 150, 17, 82, 6, 72, 6, 84, 73, 75, 69, 78, 79, 169, 229, 33, - 6, 65, 84, 82, 73, 67, 72, 4, 168, 32, 2, 75, 89, 239, 227, 33, 77, 22, - 52, 5, 69, 83, 79, 32, 69, 38, 79, 247, 241, 33, 76, 4, 204, 1, 2, 88, - 79, 147, 55, 75, 16, 80, 6, 83, 84, 82, 79, 70, 79, 188, 40, 3, 68, 69, - 82, 181, 128, 1, 2, 84, 72, 10, 24, 2, 73, 32, 79, 83, 4, 56, 9, 83, 89, - 78, 68, 69, 83, 77, 79, 83, 195, 24, 84, 2, 143, 44, 32, 7, 11, 32, 4, - 214, 60, 68, 251, 213, 21, 78, 18, 48, 2, 71, 79, 33, 6, 75, 84, 73, 75, - 79, 32, 4, 170, 21, 83, 135, 238, 33, 78, 14, 194, 242, 26, 86, 218, 241, - 6, 90, 162, 8, 75, 254, 2, 68, 2, 78, 162, 17, 71, 3, 80, 14, 80, 4, 82, - 79, 65, 32, 168, 37, 4, 79, 82, 69, 86, 221, 2, 4, 65, 77, 73, 76, 6, - 204, 161, 19, 2, 83, 80, 152, 184, 3, 3, 90, 89, 71, 161, 246, 9, 3, 75, - 76, 73, 42, 50, 73, 220, 205, 8, 2, 65, 83, 143, 148, 25, 89, 38, 122, - 65, 200, 1, 5, 69, 83, 73, 83, 32, 70, 71, 208, 1, 3, 80, 76, 73, 185, - 235, 26, 8, 70, 84, 79, 71, 71, 79, 83, 32, 10, 48, 6, 83, 84, 79, 76, - 73, 32, 187, 200, 32, 82, 8, 80, 6, 65, 80, 76, 73, 32, 77, 174, 55, 68, - 177, 159, 22, 5, 84, 72, 69, 83, 69, 4, 40, 2, 69, 71, 197, 245, 26, 2, - 73, 75, 2, 171, 175, 29, 65, 12, 38, 84, 246, 50, 65, 42, 68, 63, 77, 6, - 194, 10, 69, 239, 41, 82, 10, 72, 5, 79, 82, 71, 79, 78, 181, 252, 33, 7, - 82, 65, 77, 77, 65, 32, 71, 9, 69, 15, 32, 80, 65, 82, 69, 83, 84, 73, - 71, 77, 69, 78, 79, 78, 32, 6, 222, 38, 68, 137, 10, 8, 65, 82, 73, 83, - 84, 69, 82, 65, 5, 175, 47, 32, 16, 166, 1, 78, 116, 6, 84, 69, 82, 79, - 78, 32, 188, 44, 4, 88, 79, 32, 69, 252, 236, 31, 4, 80, 69, 71, 69, 196, - 46, 6, 75, 83, 84, 82, 69, 80, 209, 13, 3, 76, 65, 70, 4, 224, 22, 3, 68, - 79, 70, 157, 211, 26, 18, 65, 82, 88, 73, 83, 32, 75, 65, 73, 32, 70, 84, - 72, 79, 82, 65, 32, 86, 4, 208, 11, 5, 65, 82, 71, 79, 83, 151, 20, 80, - 44, 180, 1, 9, 65, 78, 69, 82, 79, 83, 73, 83, 32, 52, 6, 84, 72, 79, 82, - 65, 32, 165, 6, 22, 72, 84, 79, 82, 65, 32, 83, 75, 76, 73, 82, 79, 78, - 32, 67, 72, 82, 79, 77, 65, 32, 86, 6, 246, 4, 68, 14, 77, 25, 5, 84, 69, - 84, 82, 65, 36, 220, 2, 8, 65, 82, 67, 72, 65, 73, 79, 78, 80, 10, 68, - 73, 65, 84, 79, 78, 73, 75, 73, 32, 96, 11, 73, 32, 89, 70, 69, 83, 73, - 83, 32, 84, 69, 32, 15, 77, 65, 76, 65, 75, 79, 78, 32, 67, 72, 82, 79, - 77, 65, 32, 74, 78, 40, 8, 83, 75, 76, 73, 82, 79, 78, 32, 133, 157, 18, - 18, 69, 78, 65, 82, 77, 79, 78, 73, 79, 83, 32, 65, 78, 84, 73, 70, 79, - 78, 5, 57, 12, 32, 68, 69, 89, 84, 69, 82, 79, 85, 32, 73, 67, 2, 211, - 227, 26, 72, 14, 62, 78, 218, 212, 33, 90, 162, 8, 75, 254, 2, 68, 163, - 17, 80, 6, 242, 39, 73, 151, 145, 32, 65, 2, 237, 42, 4, 84, 65, 82, 84, - 4, 18, 68, 15, 77, 2, 35, 73, 2, 21, 3, 79, 78, 79, 2, 151, 19, 70, 4, - 166, 19, 65, 193, 218, 31, 2, 69, 78, 6, 84, 7, 67, 72, 82, 79, 77, 65, - 32, 205, 225, 16, 8, 68, 73, 65, 84, 79, 78, 79, 78, 4, 42, 86, 181, 229, - 16, 4, 83, 89, 78, 65, 2, 159, 212, 31, 65, 22, 80, 6, 69, 78, 73, 75, - 73, 32, 44, 2, 79, 82, 185, 26, 5, 82, 79, 78, 84, 72, 4, 140, 150, 30, - 2, 68, 73, 1, 2, 89, 70, 16, 68, 2, 71, 79, 225, 1, 10, 84, 72, 77, 73, - 75, 79, 78, 32, 78, 32, 12, 28, 2, 78, 32, 155, 1, 83, 10, 96, 14, 80, - 65, 82, 69, 83, 84, 73, 71, 77, 69, 78, 79, 78, 32, 242, 33, 65, 113, 3, - 78, 69, 79, 4, 214, 24, 68, 233, 201, 31, 5, 65, 82, 73, 83, 84, 2, 165, - 188, 32, 5, 89, 78, 84, 72, 69, 4, 142, 28, 65, 1, 2, 68, 73, 16, 56, 2, - 77, 73, 114, 83, 213, 236, 32, 4, 67, 72, 65, 68, 8, 34, 70, 169, 28, 3, - 68, 73, 65, 6, 40, 4, 84, 72, 79, 82, 183, 226, 32, 79, 4, 218, 156, 33, - 79, 215, 79, 65, 6, 44, 6, 65, 75, 73, 65, 32, 84, 231, 13, 79, 2, 145, - 141, 21, 12, 69, 76, 79, 85, 83, 32, 73, 67, 72, 73, 77, 65, 54, 108, 2, - 65, 84, 96, 6, 69, 78, 84, 73, 77, 65, 140, 1, 5, 76, 65, 83, 77, 65, 18, - 79, 98, 82, 175, 1, 89, 6, 40, 3, 65, 86, 65, 201, 3, 2, 72, 73, 4, 140, - 29, 5, 32, 84, 82, 79, 77, 227, 171, 33, 83, 18, 24, 2, 84, 65, 15, 32, - 11, 11, 32, 8, 36, 4, 78, 69, 79, 32, 183, 28, 65, 6, 220, 144, 25, 2, - 77, 69, 166, 247, 4, 75, 235, 183, 3, 65, 7, 243, 28, 32, 8, 64, 6, 78, - 84, 69, 86, 77, 65, 250, 198, 8, 82, 195, 255, 24, 85, 5, 165, 254, 13, - 2, 32, 65, 14, 44, 4, 65, 84, 73, 77, 105, 3, 69, 77, 65, 12, 18, 65, 35, - 79, 8, 182, 23, 32, 159, 205, 33, 84, 4, 134, 12, 75, 181, 201, 30, 5, - 89, 80, 79, 82, 82, 2, 187, 194, 26, 83, 2, 251, 196, 33, 76, 14, 22, 69, - 147, 22, 89, 12, 44, 5, 73, 77, 77, 65, 32, 167, 163, 29, 77, 10, 72, 2, - 69, 78, 0, 5, 73, 77, 73, 83, 69, 54, 84, 69, 3, 68, 89, 79, 2, 173, 212, - 26, 8, 79, 83, 32, 67, 72, 82, 79, 78, 4, 44, 5, 69, 83, 83, 65, 82, 1, - 2, 82, 73, 2, 17, 2, 79, 78, 2, 25, 4, 32, 67, 72, 82, 2, 235, 217, 32, - 79, 28, 84, 8, 65, 82, 84, 89, 82, 73, 65, 32, 241, 143, 27, 7, 73, 75, - 82, 79, 78, 32, 73, 26, 72, 5, 65, 76, 76, 73, 32, 38, 68, 38, 80, 134, - 1, 86, 30, 84, 87, 76, 4, 34, 68, 177, 2, 3, 80, 82, 79, 2, 237, 2, 5, - 69, 89, 84, 69, 82, 8, 60, 7, 76, 65, 71, 73, 79, 83, 32, 45, 4, 82, 79, - 84, 79, 4, 200, 1, 5, 84, 69, 84, 65, 82, 111, 73, 4, 22, 86, 227, 1, 83, - 2, 209, 1, 3, 65, 82, 89, 8, 56, 8, 69, 84, 65, 82, 84, 79, 83, 32, 61, - 2, 82, 73, 4, 22, 76, 135, 1, 73, 2, 21, 3, 69, 71, 69, 2, 63, 84, 4, 18, - 70, 35, 84, 2, 205, 137, 21, 3, 79, 78, 73, 2, 11, 79, 2, 11, 83, 2, 17, - 2, 32, 73, 2, 195, 178, 22, 67, 18, 92, 4, 76, 73, 71, 79, 180, 1, 5, 89, - 82, 65, 78, 73, 210, 14, 88, 141, 254, 32, 2, 77, 65, 4, 211, 1, 78, 42, + 74, 79, 77, 78, 68, 201, 175, 34, 2, 71, 66, 20, 62, 69, 172, 49, 2, 78, + 83, 149, 239, 31, 4, 80, 32, 80, 73, 16, 58, 77, 234, 170, 12, 75, 226, + 168, 7, 78, 211, 200, 17, 83, 11, 186, 24, 66, 14, 71, 194, 44, 86, 239, + 220, 27, 75, 4, 146, 211, 19, 78, 171, 220, 17, 81, 6, 36, 2, 79, 77, + 237, 1, 2, 78, 32, 4, 178, 252, 12, 80, 239, 184, 23, 69, 2, 207, 243, + 35, 71, 48, 122, 65, 32, 2, 68, 65, 54, 71, 98, 75, 32, 2, 83, 72, 38, + 84, 102, 89, 74, 90, 158, 239, 35, 74, 158, 107, 69, 251, 70, 73, 4, 250, + 8, 65, 195, 164, 37, 81, 4, 22, 65, 159, 22, 32, 2, 153, 44, 3, 78, 71, + 71, 8, 52, 3, 75, 85, 69, 222, 188, 32, 65, 239, 155, 3, 71, 4, 250, 7, + 32, 165, 155, 12, 2, 78, 90, 4, 246, 25, 65, 135, 203, 19, 73, 4, 174, + 188, 35, 73, 187, 239, 1, 65, 8, 40, 2, 65, 80, 157, 157, 28, 2, 79, 81, + 7, 11, 32, 4, 188, 184, 35, 2, 77, 70, 1, 2, 78, 84, 6, 26, 73, 155, 142, + 37, 69, 5, 193, 13, 7, 84, 32, 77, 79, 78, 71, 75, 4, 214, 5, 65, 229, + 159, 12, 4, 85, 78, 32, 77, 22, 58, 65, 80, 3, 79, 78, 32, 178, 140, 37, + 69, 163, 28, 85, 10, 42, 65, 154, 18, 32, 250, 17, 77, 15, 83, 4, 230, + 190, 26, 82, 155, 234, 10, 77, 8, 56, 4, 77, 70, 79, 78, 1, 6, 80, 65, + 32, 78, 74, 73, 4, 37, 7, 32, 80, 73, 80, 65, 69, 77, 4, 206, 16, 71, + 243, 148, 37, 66, 22, 70, 72, 194, 1, 79, 192, 66, 2, 69, 85, 170, 190, + 36, 85, 167, 34, 73, 10, 74, 73, 70, 85, 241, 252, 35, 10, 79, 81, 32, + 78, 83, 72, 85, 84, 32, 89, 4, 170, 188, 26, 82, 177, 211, 6, 8, 78, 68, + 65, 32, 80, 65, 32, 78, 4, 188, 11, 4, 69, 78, 83, 72, 195, 154, 37, 77, + 6, 208, 234, 35, 2, 78, 74, 254, 186, 1, 81, 3, 84, 10, 44, 2, 69, 85, + 44, 3, 73, 84, 65, 31, 85, 4, 130, 209, 35, 65, 1, 4, 84, 69, 85, 87, 2, + 11, 32, 2, 223, 30, 77, 4, 186, 26, 32, 251, 246, 26, 65, 4, 228, 3, 5, + 32, 89, 85, 81, 32, 217, 230, 27, 3, 78, 75, 78, 112, 164, 1, 7, 71, 72, + 69, 85, 71, 72, 69, 34, 75, 126, 76, 122, 77, 154, 3, 78, 234, 2, 80, 90, + 83, 178, 1, 84, 106, 89, 190, 22, 87, 204, 46, 2, 70, 69, 131, 201, 11, + 86, 4, 198, 56, 85, 215, 233, 36, 78, 12, 40, 2, 69, 85, 50, 73, 203, + 144, 37, 65, 6, 230, 54, 89, 174, 184, 12, 80, 147, 160, 24, 65, 4, 242, + 142, 37, 69, 175, 18, 81, 8, 46, 65, 240, 69, 2, 79, 77, 183, 190, 36, + 69, 4, 50, 65, 129, 61, 7, 77, 32, 78, 83, 72, 85, 84, 2, 131, 182, 26, + 78, 26, 70, 65, 74, 66, 140, 1, 2, 69, 85, 58, 70, 145, 23, 3, 79, 78, + 84, 7, 21, 3, 32, 78, 74, 4, 178, 212, 18, 85, 229, 142, 17, 3, 69, 85, + 65, 10, 90, 65, 154, 46, 85, 184, 146, 30, 2, 69, 85, 133, 139, 6, 7, 73, + 84, 32, 77, 66, 65, 65, 4, 150, 12, 65, 205, 192, 19, 4, 32, 77, 65, 69, + 4, 252, 141, 32, 5, 84, 32, 78, 71, 71, 239, 143, 5, 81, 4, 48, 4, 79, + 78, 32, 84, 193, 204, 26, 2, 73, 89, 2, 163, 65, 69, 24, 110, 71, 166, 1, + 83, 50, 89, 152, 23, 8, 84, 73, 69, 69, 32, 83, 72, 69, 201, 144, 35, 5, + 68, 85, 32, 78, 74, 12, 70, 71, 164, 162, 36, 8, 75, 73, 78, 68, 73, 32, + 77, 86, 191, 104, 79, 8, 60, 3, 85, 79, 81, 204, 211, 19, 2, 69, 85, 179, + 140, 16, 65, 5, 193, 180, 28, 2, 32, 76, 4, 26, 72, 227, 202, 36, 69, 2, + 167, 229, 36, 85, 4, 248, 45, 2, 65, 69, 211, 17, 73, 8, 200, 195, 10, 2, + 69, 69, 218, 169, 12, 65, 164, 182, 8, 3, 85, 78, 71, 151, 218, 5, 73, + 12, 88, 2, 65, 75, 16, 2, 72, 69, 252, 185, 19, 2, 69, 84, 146, 239, 15, + 73, 231, 216, 1, 85, 2, 211, 25, 69, 4, 128, 200, 26, 4, 84, 32, 78, 74, + 241, 128, 5, 4, 85, 65, 69, 81, 6, 32, 2, 85, 32, 227, 220, 35, 65, 4, + 36, 4, 77, 65, 69, 77, 139, 7, 78, 2, 11, 71, 2, 135, 7, 66, 4, 44, 4, + 65, 70, 85, 32, 229, 4, 2, 69, 85, 2, 221, 165, 32, 6, 76, 69, 69, 82, + 65, 69, 196, 1, 170, 1, 71, 82, 75, 206, 2, 76, 50, 77, 250, 3, 78, 238, + 6, 80, 78, 83, 118, 84, 176, 1, 3, 86, 69, 85, 46, 87, 62, 89, 202, 157, + 30, 66, 182, 218, 2, 70, 163, 230, 3, 82, 6, 40, 2, 72, 65, 185, 184, 19, + 2, 66, 65, 4, 198, 170, 26, 82, 155, 234, 10, 80, 22, 66, 69, 182, 1, 85, + 132, 194, 26, 3, 80, 65, 82, 139, 206, 10, 65, 14, 40, 2, 78, 32, 54, 85, + 155, 147, 37, 84, 4, 180, 136, 33, 4, 70, 65, 84, 73, 235, 140, 3, 76, 8, + 42, 83, 174, 194, 26, 75, 195, 208, 10, 77, 4, 232, 14, 3, 72, 69, 85, + 239, 53, 69, 4, 48, 6, 79, 80, 32, 78, 75, 65, 147, 146, 37, 84, 2, 11, + 65, 2, 235, 167, 26, 82, 8, 202, 47, 65, 242, 145, 26, 73, 183, 208, 10, + 85, 38, 82, 65, 130, 1, 66, 214, 165, 26, 85, 216, 25, 4, 71, 66, 65, 83, + 139, 225, 8, 73, 8, 18, 32, 79, 69, 4, 42, 78, 145, 231, 17, 4, 75, 69, + 85, 65, 2, 11, 83, 2, 231, 160, 35, 73, 4, 226, 226, 36, 77, 211, 25, 83, + 24, 34, 65, 114, 69, 82, 73, 35, 85, 6, 32, 2, 65, 32, 255, 178, 19, 78, + 4, 200, 206, 31, 8, 67, 65, 66, 66, 65, 71, 69, 45, 133, 238, 4, 2, 80, + 73, 8, 50, 85, 142, 164, 26, 82, 173, 218, 5, 2, 69, 75, 4, 162, 142, 37, + 77, 3, 88, 7, 226, 7, 82, 167, 134, 37, 84, 4, 186, 251, 36, 65, 175, 18, + 69, 72, 130, 1, 65, 54, 68, 110, 71, 222, 1, 74, 102, 83, 130, 1, 84, + 102, 90, 237, 7, 12, 89, 73, 82, 32, 77, 75, 80, 65, 82, 65, 81, 32, 4, + 128, 188, 26, 4, 78, 83, 65, 78, 195, 208, 10, 81, 12, 60, 2, 69, 85, + 174, 41, 65, 134, 155, 19, 79, 159, 162, 17, 73, 4, 136, 156, 35, 2, 65, + 69, 199, 239, 1, 84, 20, 50, 71, 98, 75, 222, 185, 26, 65, 223, 191, 10, + 79, 12, 26, 85, 247, 186, 36, 69, 11, 180, 39, 3, 65, 69, 78, 190, 147, + 36, 79, 202, 26, 69, 155, 53, 77, 4, 36, 3, 85, 69, 32, 183, 185, 26, 65, + 2, 241, 179, 19, 3, 77, 65, 69, 10, 34, 65, 34, 69, 183, 214, 12, 85, 4, + 202, 248, 36, 69, 219, 16, 77, 4, 202, 153, 35, 69, 151, 99, 85, 12, 78, + 85, 214, 183, 26, 72, 204, 233, 5, 2, 69, 85, 158, 214, 4, 79, 219, 16, + 65, 4, 224, 244, 36, 4, 79, 84, 32, 78, 179, 19, 78, 8, 54, 69, 244, 234, + 36, 4, 85, 32, 77, 66, 131, 26, 65, 4, 168, 175, 19, 2, 85, 78, 131, 216, + 17, 78, 4, 238, 221, 35, 69, 147, 169, 1, 65, 6, 26, 73, 227, 182, 36, + 69, 4, 26, 82, 167, 134, 37, 78, 2, 175, 178, 35, 73, 10, 30, 69, 50, 72, + 255, 6, 85, 4, 26, 84, 179, 249, 35, 85, 2, 231, 181, 36, 70, 4, 238, + 210, 12, 85, 159, 226, 13, 73, 10, 40, 2, 65, 65, 34, 69, 41, 2, 73, 84, + 2, 11, 83, 2, 203, 154, 26, 72, 4, 228, 25, 2, 85, 84, 235, 234, 36, 84, + 4, 38, 85, 249, 248, 32, 3, 65, 32, 89, 2, 143, 241, 26, 65, 4, 236, 245, + 27, 2, 65, 69, 255, 141, 9, 88, 4, 40, 4, 65, 78, 71, 75, 139, 131, 37, + 85, 2, 143, 19, 85, 10, 42, 85, 158, 208, 12, 69, 135, 176, 24, 65, 6, + 210, 18, 87, 212, 3, 4, 32, 77, 85, 79, 179, 236, 36, 77, 234, 1, 134, 1, + 70, 68, 2, 71, 72, 34, 75, 254, 1, 76, 142, 1, 77, 130, 3, 78, 130, 5, + 80, 122, 82, 90, 83, 190, 1, 84, 158, 1, 87, 39, 89, 4, 36, 3, 69, 85, + 70, 179, 254, 36, 65, 2, 11, 69, 2, 151, 2, 85, 4, 202, 1, 69, 203, 252, + 36, 65, 22, 46, 69, 146, 1, 85, 42, 87, 143, 143, 35, 89, 10, 26, 85, + 227, 255, 36, 84, 8, 72, 3, 65, 69, 84, 20, 5, 79, 84, 32, 77, 66, 130, + 255, 36, 77, 3, 80, 2, 235, 249, 11, 77, 2, 231, 148, 26, 85, 9, 146, + 238, 36, 79, 218, 16, 78, 3, 81, 2, 171, 201, 36, 65, 14, 58, 69, 194, + 173, 26, 79, 254, 224, 8, 73, 227, 222, 1, 85, 8, 42, 85, 146, 142, 35, + 69, 187, 239, 1, 84, 4, 214, 234, 26, 65, 243, 146, 10, 77, 41, 94, 65, + 58, 66, 66, 69, 38, 70, 80, 2, 71, 66, 214, 254, 31, 79, 242, 130, 4, 86, + 151, 121, 85, 4, 164, 211, 17, 2, 76, 69, 133, 244, 18, 3, 69, 78, 74, 6, + 32, 2, 65, 65, 247, 220, 36, 85, 5, 189, 231, 28, 2, 32, 83, 6, 234, 176, + 18, 85, 171, 219, 16, 69, 10, 44, 2, 69, 85, 246, 255, 34, 79, 207, 11, + 73, 4, 162, 228, 36, 65, 215, 22, 84, 6, 158, 139, 35, 73, 168, 70, 2, + 79, 70, 227, 40, 69, 72, 78, 68, 46, 71, 226, 1, 74, 98, 83, 110, 84, 34, + 89, 222, 245, 36, 73, 3, 85, 8, 194, 39, 69, 178, 130, 36, 79, 139, 63, + 65, 24, 18, 71, 99, 75, 12, 54, 65, 202, 42, 69, 162, 231, 31, 87, 147, + 214, 4, 85, 6, 152, 38, 2, 65, 77, 195, 210, 36, 80, 12, 68, 2, 69, 85, + 182, 136, 35, 73, 2, 89, 218, 159, 1, 85, 215, 79, 65, 4, 154, 197, 12, + 65, 251, 158, 24, 82, 12, 60, 2, 69, 85, 230, 15, 73, 214, 180, 12, 85, + 199, 178, 24, 65, 4, 218, 228, 36, 65, 175, 18, 84, 10, 46, 72, 32, 3, + 73, 69, 69, 195, 229, 36, 85, 4, 138, 218, 36, 85, 219, 5, 69, 4, 150, + 246, 36, 80, 3, 84, 6, 130, 14, 69, 147, 195, 36, 85, 8, 174, 217, 36, + 69, 218, 5, 85, 254, 5, 65, 219, 16, 73, 12, 42, 69, 46, 85, 194, 244, + 36, 65, 3, 73, 4, 220, 138, 26, 2, 85, 84, 155, 234, 10, 69, 4, 158, 216, + 36, 85, 175, 28, 81, 8, 48, 3, 69, 78, 32, 162, 224, 36, 73, 175, 1, 65, + 4, 238, 213, 26, 79, 215, 251, 9, 77, 26, 58, 72, 90, 85, 186, 16, 65, + 174, 6, 69, 179, 191, 36, 79, 12, 54, 69, 174, 162, 26, 79, 222, 188, 10, + 73, 219, 19, 85, 6, 214, 7, 85, 139, 235, 36, 69, 6, 234, 219, 36, 65, + 214, 22, 69, 3, 85, 18, 62, 69, 74, 85, 222, 160, 26, 79, 226, 185, 10, + 65, 215, 22, 73, 8, 26, 85, 135, 130, 35, 69, 6, 210, 157, 35, 65, 234, + 211, 1, 78, 3, 84, 5, 227, 212, 36, 79, 4, 134, 138, 32, 85, 235, 230, 4, + 65, 10, 40, 2, 65, 69, 18, 85, 191, 160, 36, 69, 2, 251, 3, 77, 6, 22, + 87, 227, 13, 79, 2, 207, 159, 26, 79, 186, 2, 178, 1, 70, 154, 1, 71, + 202, 1, 75, 170, 1, 76, 158, 1, 77, 134, 2, 78, 190, 7, 80, 166, 2, 82, + 78, 83, 166, 1, 84, 246, 1, 86, 66, 87, 34, 89, 238, 216, 36, 65, 2, 73, + 3, 79, 18, 54, 85, 234, 129, 23, 65, 198, 213, 13, 69, 255, 5, 79, 10, + 26, 32, 135, 142, 31, 69, 6, 132, 229, 23, 4, 82, 69, 77, 69, 146, 129, + 11, 67, 207, 135, 2, 73, 16, 24, 2, 66, 69, 39, 72, 4, 222, 224, 35, 85, + 167, 140, 1, 84, 12, 34, 65, 34, 69, 199, 219, 36, 79, 2, 11, 65, 2, 151, + 130, 26, 77, 8, 26, 85, 131, 236, 36, 84, 6, 170, 213, 36, 65, 214, 22, + 78, 3, 88, 18, 50, 69, 62, 80, 18, 85, 218, 234, 36, 73, 3, 79, 6, 26, + 85, 139, 235, 36, 84, 4, 178, 212, 36, 65, 215, 22, 88, 2, 211, 28, 69, + 6, 170, 206, 36, 69, 162, 28, 79, 15, 84, 18, 50, 65, 40, 2, 69, 85, 22, + 79, 195, 233, 36, 85, 6, 162, 217, 36, 65, 218, 16, 80, 3, 81, 2, 167, + 215, 36, 65, 8, 222, 158, 18, 79, 226, 202, 18, 77, 3, 81, 32, 110, 65, + 44, 2, 66, 69, 34, 70, 20, 2, 71, 66, 34, 73, 142, 253, 25, 85, 186, 154, + 10, 69, 2, 79, 139, 60, 86, 11, 250, 242, 17, 69, 170, 245, 18, 80, 3, + 81, 4, 158, 215, 36, 85, 219, 16, 69, 2, 155, 181, 12, 69, 4, 226, 151, + 36, 69, 227, 79, 65, 5, 207, 208, 36, 69, 80, 114, 68, 170, 1, 71, 138, + 3, 74, 112, 2, 83, 72, 58, 84, 32, 3, 89, 73, 32, 54, 90, 210, 159, 36, + 65, 191, 47, 75, 12, 34, 65, 98, 73, 187, 149, 36, 85, 6, 32, 2, 65, 32, + 215, 229, 36, 80, 4, 240, 164, 18, 3, 77, 89, 32, 137, 177, 13, 3, 83, + 79, 70, 4, 226, 148, 26, 65, 183, 208, 10, 81, 36, 78, 71, 50, 85, 122, + 75, 118, 79, 172, 221, 11, 3, 69, 85, 82, 223, 132, 25, 65, 12, 18, 69, + 51, 85, 6, 26, 85, 191, 244, 34, 69, 4, 255, 144, 24, 65, 6, 64, 6, 65, + 69, 83, 72, 65, 69, 134, 249, 25, 82, 155, 234, 10, 80, 2, 11, 32, 2, + 183, 189, 23, 78, 12, 34, 65, 20, 2, 69, 85, 35, 85, 5, 243, 206, 36, 65, + 4, 150, 208, 36, 65, 175, 18, 88, 4, 162, 226, 36, 77, 3, 80, 4, 134, + 226, 36, 80, 3, 81, 8, 18, 65, 31, 69, 2, 201, 230, 31, 2, 69, 77, 6, 26, + 69, 255, 212, 35, 85, 5, 245, 190, 36, 4, 32, 69, 80, 79, 6, 26, 85, 171, + 241, 34, 73, 4, 210, 224, 36, 79, 15, 69, 4, 234, 207, 36, 85, 207, 16, + 65, 4, 140, 159, 26, 4, 67, 76, 69, 65, 151, 244, 1, 66, 4, 186, 143, 26, + 65, 3, 85, 34, 42, 65, 90, 69, 58, 73, 50, 79, 23, 85, 8, 32, 2, 32, 80, + 223, 233, 17, 65, 4, 196, 148, 28, 2, 69, 79, 241, 192, 7, 2, 76, 85, 6, + 26, 85, 223, 205, 36, 69, 4, 178, 222, 36, 84, 3, 88, 7, 11, 69, 4, 214, + 141, 26, 69, 183, 208, 10, 84, 5, 135, 142, 36, 79, 11, 82, 65, 130, 221, + 36, 69, 3, 77, 8, 46, 65, 238, 14, 69, 149, 246, 18, 2, 73, 77, 4, 254, + 220, 36, 69, 3, 81, 18, 62, 69, 30, 72, 150, 245, 31, 85, 158, 214, 4, + 79, 163, 14, 65, 4, 162, 220, 36, 69, 3, 84, 8, 42, 69, 198, 239, 22, 79, + 231, 155, 3, 73, 2, 137, 169, 12, 2, 85, 65, 28, 34, 65, 94, 69, 50, 79, + 39, 85, 10, 56, 2, 69, 78, 202, 238, 22, 65, 154, 236, 13, 77, 3, 81, 2, + 205, 209, 11, 3, 32, 78, 84, 6, 26, 85, 167, 218, 36, 78, 5, 211, 167, + 12, 65, 6, 138, 223, 34, 79, 135, 251, 1, 81, 6, 170, 7, 77, 239, 187, + 36, 65, 6, 26, 69, 219, 200, 36, 79, 4, 150, 239, 25, 85, 155, 234, 10, + 69, 6, 246, 10, 69, 207, 187, 32, 85, 26, 68, 2, 69, 85, 46, 73, 32, 3, + 79, 81, 32, 54, 85, 155, 215, 36, 65, 8, 174, 133, 24, 65, 246, 210, 12, + 77, 3, 88, 4, 162, 193, 36, 69, 215, 22, 84, 4, 180, 195, 12, 4, 83, 87, + 73, 77, 227, 197, 9, 67, 8, 238, 134, 26, 69, 206, 252, 8, 65, 234, 211, + 1, 78, 3, 81, 108, 162, 1, 75, 82, 76, 46, 77, 98, 78, 190, 1, 80, 66, + 82, 50, 83, 110, 84, 38, 89, 146, 222, 2, 87, 204, 191, 9, 2, 86, 85, + 254, 155, 24, 69, 242, 5, 70, 231, 16, 85, 14, 226, 138, 18, 69, 170, + 219, 16, 89, 130, 237, 1, 80, 186, 2, 65, 2, 79, 3, 85, 6, 190, 132, 26, + 79, 182, 208, 10, 65, 3, 73, 13, 42, 66, 34, 69, 254, 211, 36, 65, 3, 79, + 4, 186, 132, 36, 69, 171, 77, 65, 2, 195, 228, 34, 69, 22, 94, 71, 38, + 74, 38, 85, 130, 216, 34, 83, 246, 7, 68, 150, 3, 84, 226, 219, 1, 89, + 219, 19, 73, 4, 134, 236, 31, 75, 203, 228, 4, 71, 4, 202, 232, 25, 85, + 239, 215, 10, 65, 5, 235, 187, 36, 65, 6, 26, 69, 251, 231, 25, 85, 4, + 234, 197, 35, 85, 167, 140, 1, 69, 12, 186, 2, 69, 150, 134, 9, 73, 159, + 201, 27, 85, 14, 66, 72, 230, 2, 69, 162, 248, 18, 65, 142, 177, 17, 85, + 235, 36, 73, 6, 158, 189, 36, 73, 218, 19, 79, 3, 85, 6, 182, 141, 21, + 65, 239, 166, 15, 69, 4, 246, 255, 25, 79, 183, 208, 10, 65, 4, 178, 185, + 36, 65, 215, 22, 69, 14, 54, 69, 150, 134, 9, 73, 202, 178, 27, 65, 215, + 22, 85, 6, 138, 195, 35, 85, 166, 140, 1, 69, 3, 78, 16, 62, 72, 50, 69, + 162, 248, 18, 65, 142, 177, 17, 85, 235, 36, 73, 8, 46, 69, 190, 186, 36, + 73, 218, 19, 79, 3, 85, 2, 239, 193, 35, 85, 10, 158, 131, 18, 69, 202, + 135, 3, 65, 155, 195, 15, 73, 6, 150, 253, 25, 79, 2, 85, 183, 208, 10, + 65, 14, 42, 75, 242, 144, 35, 65, 147, 157, 1, 74, 11, 49, 10, 78, 79, + 84, 69, 32, 87, 73, 84, 72, 32, 8, 184, 207, 9, 2, 80, 79, 198, 1, 89, + 156, 177, 12, 2, 69, 85, 163, 177, 6, 68, 6, 38, 32, 173, 244, 16, 3, 66, + 69, 82, 4, 134, 202, 29, 67, 213, 240, 5, 5, 79, 70, 32, 83, 79, 78, 72, + 3, 75, 69, 84, 60, 7, 83, 65, 32, 86, 65, 72, 32, 179, 194, 34, 69, 5, + 233, 156, 16, 10, 66, 65, 76, 76, 32, 65, 78, 68, 32, 72, 72, 104, 10, + 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 164, 1, 7, 76, 69, 84, 84, 69, + 82, 32, 151, 246, 33, 70, 10, 52, 4, 72, 73, 71, 72, 44, 3, 76, 79, 87, + 39, 77, 4, 236, 219, 21, 2, 45, 76, 159, 252, 10, 32, 4, 32, 2, 45, 77, + 191, 215, 32, 32, 2, 173, 215, 32, 2, 73, 68, 60, 238, 1, 68, 38, 69, 38, + 71, 34, 75, 38, 85, 20, 2, 87, 65, 22, 89, 212, 228, 29, 2, 72, 87, 230, + 191, 1, 77, 190, 218, 2, 79, 250, 160, 2, 86, 246, 5, 74, 2, 84, 2, 90, + 162, 8, 67, 2, 83, 158, 20, 66, 2, 70, 2, 80, 186, 2, 65, 3, 73, 4, 138, + 131, 34, 72, 255, 194, 2, 79, 7, 222, 201, 31, 78, 135, 252, 4, 69, 4, + 214, 160, 36, 66, 219, 35, 65, 4, 158, 229, 29, 80, 131, 224, 6, 65, 5, + 147, 160, 36, 87, 5, 227, 159, 36, 68, 4, 254, 196, 35, 69, 131, 105, 73, + 121, 48, 3, 65, 75, 32, 230, 10, 72, 143, 174, 22, 84, 112, 196, 1, 15, + 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 28, 7, 76, + 69, 84, 84, 69, 82, 32, 248, 4, 3, 80, 65, 78, 50, 83, 141, 2, 11, 86, + 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 4, 226, 193, 36, 78, 87, 72, 76, + 194, 1, 77, 114, 78, 68, 2, 80, 65, 38, 83, 196, 222, 18, 4, 75, 65, 82, + 79, 134, 222, 17, 66, 2, 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 76, 2, 82, 2, + 87, 2, 89, 186, 2, 65, 2, 73, 3, 85, 10, 26, 65, 135, 190, 36, 66, 9, 45, + 9, 78, 68, 65, 73, 76, 73, 78, 71, 32, 6, 210, 189, 36, 72, 2, 78, 3, 83, + 10, 152, 2, 2, 79, 82, 150, 187, 36, 68, 2, 71, 2, 89, 187, 2, 65, 5, + 189, 244, 20, 4, 75, 80, 65, 75, 24, 80, 10, 73, 77, 65, 76, 85, 78, 71, + 85, 78, 32, 96, 2, 79, 85, 207, 189, 36, 65, 20, 242, 187, 36, 71, 2, 72, + 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 87, 2, 89, 187, 2, 65, 2, 237, 222, + 18, 5, 84, 72, 69, 82, 78, 4, 166, 2, 71, 253, 222, 18, 4, 79, 78, 71, + 79, 10, 96, 12, 89, 77, 66, 79, 76, 32, 66, 73, 78, 68, 85, 32, 249, 179, + 10, 6, 73, 71, 78, 32, 84, 79, 8, 78, 80, 136, 159, 11, 3, 74, 85, 68, + 149, 153, 25, 6, 78, 65, 32, 77, 69, 84, 4, 64, 3, 65, 78, 71, 241, 156, + 23, 7, 73, 78, 65, 82, 66, 79, 82, 2, 151, 228, 25, 79, 18, 122, 85, 252, + 146, 26, 6, 80, 65, 75, 80, 65, 75, 232, 250, 2, 5, 75, 65, 82, 79, 32, + 234, 235, 6, 69, 162, 64, 73, 3, 79, 5, 241, 187, 30, 15, 32, 70, 79, 82, + 32, 83, 73, 77, 65, 76, 85, 78, 71, 85, 78, 5, 195, 219, 23, 84, 254, 1, + 134, 1, 65, 230, 2, 69, 50, 76, 130, 1, 78, 190, 10, 84, 224, 228, 32, 2, + 67, 65, 248, 183, 2, 5, 86, 69, 82, 65, 71, 215, 140, 1, 68, 20, 136, 1, + 4, 77, 69, 68, 32, 170, 1, 82, 136, 169, 3, 10, 67, 72, 32, 87, 73, 84, + 72, 32, 85, 77, 178, 158, 7, 84, 166, 176, 25, 86, 19, 78, 8, 86, 65, 0, + 2, 68, 69, 52, 4, 69, 73, 71, 72, 1, 7, 83, 73, 88, 84, 69, 69, 78, 2, + 197, 164, 20, 8, 83, 67, 69, 78, 68, 73, 78, 71, 2, 189, 164, 20, 2, 84, + 72, 4, 220, 181, 32, 3, 68, 69, 68, 215, 194, 3, 32, 4, 172, 197, 8, 3, + 82, 32, 77, 195, 203, 26, 84, 13, 11, 76, 11, 38, 32, 149, 156, 25, 3, + 72, 79, 80, 6, 238, 185, 12, 80, 152, 160, 16, 6, 87, 73, 84, 72, 32, 67, + 159, 199, 6, 83, 208, 1, 84, 5, 71, 65, 76, 73, 32, 158, 9, 84, 37, 9, + 90, 69, 78, 69, 32, 82, 73, 78, 71, 200, 1, 210, 1, 65, 40, 9, 67, 85, + 82, 82, 69, 78, 67, 89, 32, 148, 2, 7, 76, 69, 84, 84, 69, 82, 32, 204, + 3, 6, 82, 85, 80, 69, 69, 32, 34, 83, 250, 213, 22, 73, 134, 4, 86, 242, + 223, 11, 68, 249, 107, 3, 71, 65, 78, 6, 154, 154, 32, 66, 50, 78, 187, + 57, 85, 12, 120, 10, 78, 85, 77, 69, 82, 65, 84, 79, 82, 32, 209, 170, + 23, 14, 68, 69, 78, 79, 77, 73, 78, 65, 84, 79, 82, 32, 83, 73, 10, 52, + 3, 79, 78, 69, 218, 156, 31, 70, 163, 163, 3, 84, 5, 157, 177, 29, 19, + 32, 76, 69, 83, 83, 32, 84, 72, 65, 78, 32, 84, 72, 69, 32, 68, 69, 78, + 79, 108, 226, 1, 75, 90, 82, 146, 184, 21, 86, 138, 138, 8, 89, 134, 144, + 3, 65, 38, 68, 114, 84, 230, 5, 85, 206, 201, 1, 73, 162, 193, 1, 78, 46, + 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 80, 138, 69, 72, 2, 76, 2, 77, 186, + 2, 69, 3, 79, 8, 26, 72, 251, 172, 36, 65, 6, 26, 65, 219, 254, 13, 73, + 5, 253, 205, 18, 3, 78, 68, 65, 10, 34, 65, 226, 169, 36, 72, 3, 82, 7, + 33, 6, 32, 87, 73, 84, 72, 32, 4, 180, 148, 25, 5, 76, 79, 87, 69, 82, 1, + 6, 77, 73, 68, 68, 76, 69, 4, 194, 164, 35, 83, 191, 69, 77, 20, 116, 19, + 69, 81, 85, 69, 78, 67, 69, 32, 70, 79, 82, 32, 76, 69, 84, 84, 69, 82, + 32, 182, 255, 25, 65, 219, 149, 6, 73, 6, 158, 216, 22, 82, 155, 207, 13, + 89, 4, 226, 146, 26, 32, 199, 138, 9, 79, 5, 133, 179, 33, 3, 32, 87, 73, + 4, 182, 213, 34, 87, 219, 64, 32, 194, 1, 184, 2, 7, 76, 69, 84, 84, 69, + 82, 32, 244, 1, 7, 78, 85, 77, 66, 69, 82, 32, 72, 5, 83, 73, 71, 78, 32, + 48, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 142, 245, 25, 68, + 170, 243, 3, 87, 152, 186, 2, 10, 71, 65, 80, 32, 70, 73, 76, 76, 69, 82, + 205, 182, 3, 12, 72, 85, 78, 68, 82, 69, 68, 83, 32, 85, 78, 73, 92, 210, + 1, 86, 218, 202, 32, 65, 38, 68, 114, 84, 230, 5, 85, 206, 201, 1, 73, + 162, 193, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, + 69, 72, 2, 76, 2, 77, 2, 82, 2, 89, 186, 2, 69, 3, 79, 8, 234, 1, 79, + 143, 163, 36, 65, 36, 142, 126, 69, 38, 70, 66, 78, 26, 83, 138, 173, 21, + 84, 155, 223, 12, 79, 10, 206, 143, 32, 67, 210, 61, 65, 231, 147, 3, 86, + 24, 80, 2, 86, 79, 194, 207, 32, 65, 38, 85, 206, 201, 1, 73, 222, 137, + 2, 69, 3, 79, 6, 33, 6, 67, 65, 76, 73, 67, 32, 6, 158, 208, 32, 82, 203, + 210, 3, 76, 26, 148, 1, 4, 67, 89, 67, 76, 36, 2, 71, 32, 28, 2, 76, 76, + 46, 82, 66, 84, 204, 166, 19, 4, 79, 72, 65, 90, 176, 253, 11, 2, 75, 73, + 155, 172, 4, 83, 4, 174, 183, 32, 73, 255, 233, 3, 69, 4, 178, 139, 34, + 82, 67, 83, 4, 216, 215, 9, 2, 69, 68, 199, 235, 23, 73, 4, 140, 184, 32, + 7, 84, 72, 68, 65, 89, 32, 67, 179, 232, 3, 68, 4, 188, 211, 13, 2, 67, + 79, 137, 203, 22, 4, 73, 78, 71, 32, 192, 7, 42, 65, 174, 33, 79, 165, + 11, 2, 85, 69, 242, 2, 32, 2, 67, 75, 247, 236, 17, 78, 240, 2, 22, 32, + 223, 31, 45, 224, 2, 210, 1, 67, 254, 4, 68, 174, 2, 70, 102, 72, 82, 76, + 186, 4, 77, 250, 2, 78, 38, 80, 46, 82, 150, 4, 83, 154, 4, 84, 82, 85, + 248, 2, 3, 86, 69, 82, 214, 128, 12, 79, 156, 205, 20, 3, 66, 79, 87, + 251, 247, 1, 81, 98, 196, 1, 5, 73, 82, 67, 76, 69, 200, 1, 6, 85, 82, + 86, 69, 68, 32, 224, 251, 25, 12, 82, 79, 83, 83, 32, 79, 78, 32, 83, 72, + 73, 69, 180, 158, 3, 5, 69, 78, 84, 82, 69, 230, 161, 5, 72, 211, 13, 76, + 11, 11, 32, 8, 72, 5, 87, 73, 84, 72, 32, 217, 235, 16, 7, 70, 79, 82, + 32, 82, 69, 67, 6, 140, 69, 8, 87, 72, 73, 84, 69, 32, 68, 79, 198, 242, + 18, 68, 161, 146, 2, 8, 84, 87, 79, 32, 87, 72, 73, 84, 16, 84, 4, 68, + 79, 87, 78, 0, 2, 85, 80, 56, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 4, + 229, 252, 31, 9, 87, 65, 82, 68, 83, 32, 65, 78, 68, 4, 53, 11, 84, 87, + 65, 82, 68, 83, 32, 65, 78, 68, 32, 4, 158, 143, 21, 85, 231, 154, 12, + 68, 30, 64, 6, 73, 65, 77, 79, 78, 68, 152, 1, 3, 79, 87, 78, 43, 82, 13, + 11, 32, 10, 154, 19, 67, 244, 250, 12, 10, 77, 73, 78, 85, 83, 32, 87, + 72, 73, 84, 152, 166, 6, 6, 87, 73, 84, 72, 32, 68, 206, 161, 15, 79, + 135, 13, 83, 12, 214, 19, 32, 62, 45, 247, 167, 31, 87, 6, 212, 182, 34, + 2, 79, 80, 231, 20, 65, 8, 18, 76, 39, 79, 4, 206, 234, 27, 79, 155, 170, + 8, 65, 4, 252, 217, 2, 2, 85, 82, 151, 155, 31, 76, 12, 38, 69, 218, 206, + 34, 65, 251, 2, 79, 6, 160, 208, 34, 2, 65, 82, 247, 11, 88, 40, 64, 5, + 65, 82, 71, 69, 32, 140, 2, 3, 69, 70, 84, 203, 1, 79, 12, 48, 6, 67, 73, + 82, 67, 76, 69, 203, 242, 34, 83, 11, 37, 7, 32, 77, 73, 78, 85, 83, 32, + 8, 54, 76, 36, 4, 82, 73, 71, 72, 13, 3, 85, 80, 80, 4, 32, 2, 69, 70, + 13, 2, 79, 87, 2, 31, 84, 2, 17, 2, 69, 82, 2, 173, 147, 33, 8, 32, 81, + 85, 65, 82, 84, 69, 82, 22, 96, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, + 32, 64, 6, 87, 65, 82, 68, 83, 32, 235, 207, 34, 32, 12, 146, 7, 68, 190, + 8, 73, 190, 195, 34, 80, 206, 24, 83, 51, 84, 4, 242, 164, 33, 69, 255, + 139, 1, 66, 6, 158, 15, 87, 175, 203, 34, 90, 30, 76, 6, 69, 68, 73, 85, + 77, 32, 245, 222, 21, 7, 79, 79, 78, 32, 76, 73, 76, 28, 66, 68, 42, 76, + 36, 4, 82, 73, 71, 72, 12, 2, 85, 80, 111, 83, 6, 84, 3, 79, 87, 78, 163, + 209, 34, 73, 6, 32, 2, 69, 70, 195, 216, 34, 79, 4, 11, 84, 4, 81, 18, + 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 84, 82, 73, 65, 78, 71, 76, 69, + 5, 145, 9, 2, 32, 67, 8, 198, 13, 77, 247, 222, 34, 81, 4, 238, 145, 17, + 69, 155, 191, 17, 73, 8, 238, 135, 25, 85, 158, 201, 9, 65, 87, 69, 34, + 52, 4, 73, 71, 72, 84, 254, 252, 33, 79, 155, 85, 69, 30, 94, 32, 84, 10, + 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 245, 1, 6, 87, 65, 82, 68, 83, + 32, 6, 60, 9, 84, 82, 73, 65, 78, 71, 76, 69, 32, 171, 227, 34, 80, 2, + 231, 204, 6, 67, 16, 90, 68, 88, 8, 84, 82, 73, 65, 78, 71, 76, 69, 230, + 7, 73, 194, 200, 34, 80, 203, 19, 83, 4, 65, 14, 79, 85, 66, 76, 69, 32, + 84, 82, 73, 65, 78, 71, 76, 69, 5, 207, 189, 12, 32, 5, 205, 229, 20, 11, + 32, 87, 73, 84, 72, 32, 68, 79, 85, 66, 76, 8, 230, 158, 19, 65, 162, + 254, 13, 69, 255, 139, 1, 66, 40, 158, 2, 77, 148, 1, 5, 81, 85, 65, 82, + 69, 180, 253, 25, 3, 85, 78, 32, 172, 223, 1, 5, 75, 85, 76, 76, 32, 160, + 178, 3, 3, 78, 79, 87, 168, 223, 1, 5, 65, 70, 69, 84, 89, 192, 131, 1, + 13, 76, 73, 71, 72, 84, 76, 89, 32, 83, 77, 65, 76, 76, 254, 91, 67, 50, + 72, 222, 1, 80, 175, 19, 84, 12, 40, 4, 65, 76, 76, 32, 203, 208, 34, 73, + 10, 214, 200, 34, 68, 150, 7, 76, 70, 83, 213, 15, 13, 85, 80, 45, 80, + 79, 73, 78, 84, 73, 78, 71, 32, 67, 9, 11, 32, 6, 54, 67, 208, 178, 33, + 3, 70, 79, 82, 163, 158, 1, 66, 2, 181, 248, 30, 3, 69, 78, 84, 14, 192, + 4, 3, 73, 78, 89, 242, 174, 23, 82, 154, 165, 11, 79, 54, 69, 135, 2, 87, + 18, 38, 80, 229, 213, 32, 3, 78, 73, 86, 16, 46, 32, 62, 45, 170, 1, 80, + 207, 166, 31, 87, 2, 233, 219, 34, 10, 80, 79, 73, 78, 84, 73, 78, 71, + 32, 66, 8, 45, 9, 80, 79, 73, 78, 84, 73, 78, 71, 32, 8, 66, 73, 156, + 216, 34, 5, 68, 79, 85, 66, 76, 238, 3, 83, 51, 84, 2, 213, 218, 33, 8, + 83, 79, 83, 67, 69, 76, 69, 83, 4, 21, 3, 69, 82, 32, 4, 210, 182, 24, + 76, 207, 163, 9, 82, 10, 56, 6, 84, 73, 67, 65, 76, 32, 41, 4, 89, 32, + 83, 77, 4, 152, 198, 34, 2, 82, 69, 239, 22, 69, 6, 21, 3, 65, 76, 76, 6, + 11, 32, 6, 186, 194, 34, 68, 150, 7, 76, 247, 20, 83, 16, 84, 15, 76, 69, + 84, 84, 69, 82, 32, 67, 65, 80, 73, 84, 65, 76, 32, 227, 146, 11, 70, 10, + 154, 254, 35, 67, 2, 72, 2, 73, 2, 82, 3, 90, 200, 4, 52, 3, 67, 75, 32, + 166, 151, 2, 83, 183, 201, 10, 87, 196, 4, 80, 7, 79, 67, 84, 65, 78, 84, + 45, 149, 7, 8, 83, 69, 88, 84, 65, 78, 84, 45, 204, 3, 58, 49, 130, 3, + 50, 114, 53, 50, 54, 18, 51, 215, 1, 52, 234, 1, 74, 50, 246, 1, 51, 174, + 91, 52, 46, 53, 38, 54, 30, 55, 187, 157, 35, 56, 116, 62, 51, 226, 92, + 52, 46, 53, 38, 54, 30, 55, 187, 157, 35, 56, 55, 54, 52, 214, 92, 53, + 38, 54, 30, 55, 187, 157, 35, 56, 22, 50, 53, 166, 2, 54, 190, 90, 55, + 187, 157, 35, 56, 11, 42, 54, 210, 220, 28, 55, 159, 157, 7, 56, 4, 234, + 249, 35, 55, 3, 56, 56, 170, 1, 53, 50, 54, 210, 89, 52, 110, 55, 187, + 157, 35, 56, 118, 62, 52, 254, 89, 51, 98, 53, 38, 54, 30, 55, 187, 157, + 35, 56, 24, 46, 53, 50, 54, 190, 90, 55, 187, 157, 35, 56, 13, 206, 1, + 54, 186, 217, 28, 55, 159, 157, 7, 56, 7, 187, 90, 55, 61, 54, 52, 118, + 53, 230, 88, 54, 30, 55, 187, 157, 35, 56, 31, 46, 53, 170, 89, 54, 30, + 55, 187, 157, 35, 56, 15, 38, 54, 158, 89, 55, 187, 157, 35, 56, 7, 210, + 246, 35, 55, 3, 56, 14, 226, 88, 54, 30, 55, 187, 157, 35, 56, 31, 46, + 54, 234, 87, 53, 66, 55, 187, 157, 35, 56, 6, 166, 88, 55, 187, 157, 35, + 56, 120, 70, 49, 238, 1, 50, 190, 188, 25, 51, 38, 52, 30, 53, 147, 182, + 10, 54, 61, 58, 50, 126, 51, 150, 189, 25, 52, 30, 53, 147, 182, 10, 54, + 31, 50, 51, 222, 189, 25, 52, 30, 53, 147, 182, 10, 54, 15, 42, 52, 206, + 189, 25, 53, 147, 182, 10, 54, 7, 218, 243, 35, 53, 3, 54, 15, 146, 189, + 25, 52, 166, 153, 3, 53, 139, 157, 7, 54, 31, 50, 52, 138, 188, 25, 51, + 66, 53, 147, 182, 10, 54, 7, 199, 188, 25, 53, 6, 190, 153, 22, 32, 237, + 195, 8, 4, 66, 69, 82, 82, 232, 4, 200, 1, 3, 76, 68, 32, 78, 79, 104, 7, + 80, 79, 77, 79, 70, 79, 32, 188, 6, 2, 84, 84, 216, 6, 5, 85, 81, 85, 69, + 84, 54, 87, 198, 1, 88, 206, 215, 5, 77, 178, 168, 3, 89, 170, 167, 26, + 65, 139, 34, 78, 14, 254, 190, 8, 83, 142, 204, 7, 69, 190, 207, 17, 70, + 234, 2, 87, 215, 10, 71, 10, 26, 75, 215, 144, 23, 77, 9, 40, 4, 77, 65, + 82, 75, 175, 239, 35, 83, 5, 133, 171, 23, 3, 32, 84, 65, 150, 1, 96, 13, + 70, 73, 78, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 53, 7, 76, 69, 84, + 84, 69, 82, 32, 10, 162, 238, 35, 71, 2, 72, 2, 75, 2, 80, 3, 84, 140, 1, + 234, 1, 65, 54, 85, 22, 69, 82, 71, 46, 73, 70, 78, 38, 79, 98, 90, 170, + 154, 7, 75, 178, 174, 24, 67, 2, 76, 2, 83, 246, 76, 66, 206, 201, 1, 74, + 222, 137, 2, 68, 2, 70, 2, 72, 2, 77, 2, 80, 2, 81, 2, 82, 2, 84, 2, 86, + 3, 88, 21, 50, 73, 2, 85, 74, 78, 134, 235, 35, 72, 3, 77, 5, 235, 155, + 35, 78, 17, 50, 78, 134, 235, 35, 69, 2, 72, 2, 73, 3, 82, 7, 130, 235, + 35, 71, 3, 78, 11, 230, 234, 35, 72, 2, 78, 2, 85, 3, 87, 15, 180, 239, + 33, 2, 78, 78, 134, 251, 1, 72, 2, 77, 2, 82, 3, 85, 9, 222, 238, 33, 71, + 155, 251, 1, 78, 17, 66, 78, 142, 232, 34, 32, 134, 129, 1, 69, 2, 77, 2, + 79, 3, 85, 4, 142, 233, 35, 71, 3, 78, 9, 242, 232, 35, 72, 2, 73, 3, 89, + 66, 104, 3, 79, 77, 32, 133, 179, 21, 17, 76, 69, 32, 87, 73, 84, 72, 32, + 80, 79, 80, 80, 73, 78, 71, 32, 67, 64, 152, 2, 5, 72, 65, 76, 70, 32, + 232, 1, 5, 76, 69, 70, 84, 32, 88, 6, 82, 73, 71, 72, 84, 32, 88, 14, 83, + 81, 85, 65, 82, 69, 32, 66, 82, 65, 67, 75, 69, 84, 206, 213, 15, 65, + 226, 134, 16, 67, 210, 3, 80, 140, 3, 13, 74, 85, 83, 84, 73, 70, 73, 69, + 68, 32, 85, 80, 80, 135, 5, 84, 32, 148, 1, 16, 70, 79, 82, 87, 65, 82, + 68, 45, 70, 65, 67, 73, 78, 71, 32, 82, 130, 225, 31, 76, 22, 82, 252, 2, + 2, 83, 84, 174, 5, 66, 231, 243, 1, 73, 10, 252, 153, 29, 11, 85, 78, 78, + 69, 82, 32, 70, 82, 65, 77, 69, 223, 200, 2, 79, 8, 192, 230, 31, 13, 74, + 85, 83, 84, 73, 70, 73, 69, 68, 32, 85, 80, 80, 126, 67, 51, 72, 8, 230, + 230, 31, 67, 50, 72, 33, 13, 74, 85, 83, 84, 73, 70, 73, 69, 68, 32, 85, + 80, 80, 5, 225, 214, 23, 9, 32, 79, 86, 69, 82, 32, 84, 79, 80, 5, 149, + 211, 34, 8, 32, 79, 70, 32, 70, 76, 79, 87, 14, 58, 76, 116, 3, 84, 73, + 69, 141, 134, 33, 3, 32, 65, 78, 6, 26, 32, 247, 143, 35, 73, 4, 148, + 183, 9, 7, 79, 70, 32, 72, 89, 71, 73, 245, 130, 23, 6, 87, 73, 84, 72, + 32, 83, 7, 207, 195, 32, 32, 218, 2, 88, 10, 32, 68, 82, 65, 87, 73, 78, + 71, 83, 32, 193, 226, 34, 6, 73, 78, 71, 32, 71, 76, 216, 2, 176, 1, 2, + 68, 79, 228, 6, 6, 72, 69, 65, 86, 89, 32, 254, 2, 76, 204, 25, 6, 82, + 73, 71, 72, 84, 32, 144, 4, 3, 85, 80, 32, 245, 3, 9, 86, 69, 82, 84, 73, + 67, 65, 76, 32, 66, 52, 5, 85, 66, 76, 69, 32, 165, 3, 3, 87, 78, 32, 30, + 58, 68, 216, 2, 2, 85, 80, 234, 5, 86, 211, 163, 29, 72, 14, 64, 8, 73, + 65, 71, 79, 78, 65, 76, 32, 149, 2, 3, 79, 87, 78, 8, 104, 6, 85, 80, 80, + 69, 82, 32, 205, 14, 15, 76, 79, 87, 69, 82, 32, 76, 69, 70, 84, 32, 84, + 79, 32, 77, 6, 76, 8, 76, 69, 70, 84, 32, 84, 79, 32, 237, 24, 6, 82, 73, + 71, 72, 84, 32, 4, 204, 23, 14, 77, 73, 68, 68, 76, 69, 32, 67, 69, 78, + 84, 82, 69, 32, 159, 173, 23, 76, 6, 199, 26, 32, 36, 128, 1, 10, 72, 69, + 65, 86, 89, 32, 65, 78, 68, 32, 132, 1, 10, 76, 73, 71, 72, 84, 32, 65, + 78, 68, 32, 210, 38, 68, 131, 4, 83, 12, 76, 3, 76, 69, 70, 0, 4, 82, 73, + 71, 72, 228, 35, 2, 85, 80, 151, 5, 72, 4, 17, 2, 84, 32, 4, 230, 32, 85, + 239, 140, 35, 76, 12, 76, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 156, 36, + 2, 85, 80, 235, 4, 72, 4, 17, 2, 84, 32, 4, 198, 32, 85, 135, 9, 72, 46, + 136, 1, 4, 76, 69, 70, 84, 68, 2, 85, 80, 130, 1, 86, 172, 20, 2, 68, 79, + 170, 2, 81, 100, 2, 84, 82, 154, 140, 29, 72, 151, 134, 6, 82, 5, 45, 9, + 32, 65, 78, 68, 32, 76, 73, 71, 72, 2, 195, 132, 34, 84, 11, 29, 5, 32, + 65, 78, 68, 32, 8, 42, 76, 134, 164, 29, 72, 151, 134, 6, 82, 4, 224, + 174, 24, 4, 73, 71, 72, 84, 167, 251, 10, 69, 8, 209, 20, 7, 69, 82, 84, + 73, 67, 65, 76, 156, 1, 56, 4, 69, 70, 84, 32, 169, 2, 5, 73, 71, 72, 84, + 32, 16, 160, 27, 19, 68, 79, 87, 78, 32, 72, 69, 65, 86, 89, 32, 65, 78, + 68, 32, 82, 73, 71, 72, 24, 14, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, + 82, 73, 71, 72, 100, 14, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 82, 73, + 71, 72, 97, 17, 85, 80, 32, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 82, + 73, 71, 72, 140, 1, 248, 1, 4, 65, 82, 67, 32, 30, 68, 248, 15, 10, 72, + 79, 82, 73, 90, 79, 78, 84, 65, 76, 112, 4, 76, 69, 70, 84, 62, 81, 34, + 84, 172, 1, 2, 85, 80, 120, 8, 86, 69, 82, 84, 73, 67, 65, 76, 192, 197, + 13, 7, 66, 79, 84, 84, 79, 77, 32, 143, 203, 21, 82, 8, 230, 202, 17, 68, + 67, 85, 80, 52, 8, 73, 65, 71, 79, 78, 65, 76, 32, 199, 14, 79, 68, 168, + 1, 14, 76, 79, 87, 69, 82, 32, 76, 69, 70, 84, 32, 84, 79, 32, 96, 7, 77, + 73, 68, 68, 76, 69, 32, 136, 3, 6, 85, 80, 80, 69, 82, 32, 162, 136, 34, + 67, 183, 4, 68, 4, 38, 77, 33, 5, 85, 80, 80, 69, 82, 2, 29, 5, 73, 68, + 68, 76, 69, 2, 129, 12, 2, 32, 67, 16, 88, 8, 76, 69, 70, 84, 32, 84, 79, + 32, 213, 1, 9, 82, 73, 71, 72, 84, 32, 84, 79, 32, 10, 156, 1, 6, 76, 79, + 87, 69, 82, 32, 33, 28, 85, 80, 80, 69, 82, 32, 67, 69, 78, 84, 82, 69, + 32, 84, 79, 32, 77, 73, 68, 68, 76, 69, 32, 82, 73, 71, 72, 84, 6, 246, + 3, 67, 255, 155, 35, 82, 5, 135, 191, 20, 32, 6, 232, 4, 15, 85, 80, 80, + 69, 82, 32, 67, 69, 78, 84, 82, 69, 32, 84, 79, 235, 3, 76, 44, 144, 1, + 10, 67, 69, 78, 84, 82, 69, 32, 84, 79, 32, 248, 3, 8, 76, 69, 70, 84, + 32, 84, 79, 32, 193, 2, 9, 82, 73, 71, 72, 84, 32, 84, 79, 32, 20, 52, 7, + 77, 73, 68, 68, 76, 69, 32, 171, 176, 23, 76, 16, 56, 4, 76, 69, 70, 84, + 165, 1, 5, 82, 73, 71, 72, 84, 9, 11, 32, 6, 84, 10, 84, 79, 32, 76, 79, + 87, 69, 82, 32, 67, 233, 186, 20, 5, 65, 78, 68, 32, 77, 4, 29, 5, 69, + 78, 84, 82, 69, 5, 133, 196, 32, 3, 32, 84, 79, 9, 11, 32, 6, 88, 3, 65, + 78, 68, 65, 15, 84, 79, 32, 76, 79, 87, 69, 82, 32, 67, 69, 78, 84, 82, + 69, 2, 241, 185, 20, 11, 32, 77, 73, 68, 68, 76, 69, 32, 76, 69, 70, 5, + 189, 211, 10, 9, 32, 84, 79, 32, 77, 73, 68, 68, 76, 14, 68, 6, 76, 79, + 87, 69, 82, 32, 97, 7, 77, 73, 68, 68, 76, 69, 32, 6, 48, 6, 67, 69, 78, + 84, 82, 69, 227, 152, 35, 82, 5, 11, 32, 2, 233, 4, 4, 84, 79, 32, 85, 8, + 76, 10, 67, 69, 78, 84, 82, 69, 32, 84, 79, 32, 33, 5, 82, 73, 71, 72, + 84, 4, 250, 3, 85, 139, 201, 13, 76, 5, 11, 32, 2, 193, 204, 13, 2, 84, + 79, 10, 46, 76, 69, 7, 77, 73, 68, 68, 76, 69, 32, 4, 29, 5, 79, 87, 69, + 82, 32, 4, 198, 191, 32, 67, 235, 214, 2, 76, 6, 34, 67, 37, 4, 76, 69, + 70, 84, 2, 45, 6, 69, 78, 84, 82, 69, 32, 5, 11, 32, 2, 133, 170, 23, 2, + 84, 79, 12, 36, 2, 87, 78, 253, 2, 2, 85, 66, 9, 11, 32, 6, 25, 4, 65, + 78, 68, 32, 6, 210, 142, 29, 72, 250, 133, 6, 76, 31, 82, 9, 11, 32, 6, + 40, 4, 65, 78, 68, 32, 227, 141, 35, 87, 4, 26, 85, 179, 168, 23, 76, 2, + 193, 168, 23, 2, 80, 80, 5, 149, 237, 33, 10, 32, 65, 78, 68, 32, 72, 69, + 65, 86, 89, 4, 109, 5, 85, 65, 68, 82, 85, 6, 66, 82, 225, 199, 13, 10, + 79, 80, 32, 65, 78, 68, 32, 85, 80, 80, 4, 11, 73, 4, 11, 80, 4, 41, 8, + 76, 69, 32, 68, 65, 83, 72, 32, 4, 190, 178, 16, 86, 151, 217, 12, 72, + 11, 29, 5, 32, 65, 78, 68, 32, 8, 34, 72, 230, 144, 35, 76, 31, 82, 4, + 200, 149, 24, 4, 69, 65, 86, 89, 175, 245, 4, 79, 17, 29, 5, 32, 65, 78, + 68, 32, 14, 162, 143, 28, 66, 42, 84, 206, 122, 72, 250, 133, 6, 76, 31, + 82, 16, 148, 2, 18, 68, 79, 87, 78, 32, 72, 69, 65, 86, 89, 32, 65, 78, + 68, 32, 76, 69, 70, 24, 13, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 76, + 69, 70, 100, 13, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 76, 69, 70, 97, + 16, 85, 80, 32, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 76, 69, 70, 2, + 101, 3, 84, 32, 85, 6, 17, 2, 84, 32, 6, 58, 85, 178, 3, 68, 245, 4, 6, + 86, 69, 82, 84, 73, 67, 2, 183, 158, 33, 80, 6, 17, 2, 84, 32, 6, 58, 85, + 134, 4, 68, 205, 4, 6, 86, 69, 82, 84, 73, 67, 2, 239, 8, 80, 2, 185, 2, + 3, 84, 32, 68, 36, 128, 1, 10, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, + 188, 1, 10, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 186, 2, 68, 131, 4, + 83, 12, 80, 4, 68, 79, 87, 78, 24, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, + 255, 4, 72, 2, 145, 5, 2, 32, 72, 4, 17, 2, 84, 32, 4, 26, 68, 191, 137, + 35, 76, 2, 133, 155, 33, 3, 79, 87, 78, 12, 80, 4, 68, 79, 87, 78, 24, 3, + 76, 69, 70, 0, 4, 82, 73, 71, 72, 211, 4, 72, 2, 229, 4, 2, 32, 72, 4, + 17, 2, 84, 32, 4, 22, 68, 131, 5, 72, 2, 233, 4, 3, 79, 87, 78, 24, 130, + 1, 68, 188, 1, 10, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 144, 1, 10, + 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 183, 1, 83, 6, 49, 10, 79, 85, + 66, 76, 69, 32, 65, 78, 68, 32, 6, 92, 3, 76, 69, 70, 0, 4, 82, 73, 71, + 72, 13, 10, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 2, 11, 84, 2, 233, + 231, 26, 2, 32, 83, 6, 54, 72, 68, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, + 2, 37, 7, 79, 82, 73, 90, 79, 78, 84, 2, 145, 150, 33, 2, 65, 76, 2, 247, + 149, 33, 84, 6, 54, 72, 60, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 37, + 7, 79, 82, 73, 90, 79, 78, 84, 2, 29, 2, 65, 76, 2, 11, 84, 2, 17, 2, 32, + 72, 2, 137, 156, 35, 3, 69, 65, 86, 6, 49, 10, 73, 78, 71, 76, 69, 32, + 65, 78, 68, 32, 6, 96, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 221, 171, 26, + 9, 72, 79, 82, 73, 90, 79, 78, 84, 65, 2, 231, 171, 26, 84, 134, 6, 46, + 65, 178, 14, 69, 150, 1, 73, 175, 1, 79, 232, 5, 36, 4, 72, 77, 73, 32, + 247, 9, 73, 230, 1, 192, 1, 7, 76, 69, 84, 84, 69, 82, 32, 196, 2, 7, 78, + 85, 77, 66, 69, 82, 32, 144, 2, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, + 79, 78, 32, 84, 5, 83, 73, 71, 78, 32, 122, 86, 239, 244, 24, 68, 108, + 210, 1, 79, 238, 205, 31, 65, 38, 68, 114, 84, 46, 86, 186, 5, 85, 206, + 201, 1, 73, 42, 76, 250, 192, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, + 2, 75, 2, 80, 138, 69, 72, 2, 77, 2, 82, 2, 89, 187, 2, 69, 15, 45, 9, + 76, 68, 32, 84, 65, 77, 73, 76, 32, 12, 158, 255, 28, 76, 206, 149, 2, + 83, 134, 92, 78, 175, 242, 2, 82, 42, 82, 69, 38, 70, 66, 78, 26, 83, + 138, 173, 21, 84, 238, 228, 5, 79, 203, 213, 7, 74, 4, 145, 135, 31, 4, + 73, 71, 72, 84, 8, 26, 79, 227, 147, 21, 73, 4, 142, 185, 33, 82, 159, + 180, 1, 85, 4, 65, 3, 73, 78, 69, 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, + 5, 215, 147, 35, 84, 10, 46, 76, 202, 208, 12, 68, 177, 16, 2, 67, 82, 4, + 198, 190, 19, 79, 235, 204, 14, 73, 12, 246, 143, 31, 67, 152, 62, 9, 79, + 76, 68, 32, 84, 65, 77, 73, 76, 230, 157, 1, 74, 158, 2, 86, 122, 85, + 187, 240, 1, 65, 34, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 167, + 234, 32, 73, 32, 138, 1, 79, 208, 174, 29, 11, 66, 72, 65, 84, 84, 73, + 80, 82, 79, 76, 85, 214, 159, 2, 65, 38, 85, 22, 86, 186, 201, 1, 73, + 223, 137, 2, 69, 7, 253, 142, 31, 10, 76, 68, 32, 84, 65, 77, 73, 76, 32, + 83, 130, 4, 72, 12, 76, 76, 69, 32, 80, 65, 84, 84, 69, 82, 78, 32, 231, + 160, 35, 78, 128, 4, 44, 5, 68, 79, 84, 83, 45, 171, 244, 6, 66, 254, 3, + 74, 49, 74, 50, 66, 51, 54, 52, 46, 53, 38, 54, 30, 55, 187, 157, 35, 56, + 129, 2, 66, 50, 66, 51, 54, 52, 46, 53, 38, 54, 30, 55, 187, 157, 35, 56, + 129, 1, 58, 51, 54, 52, 46, 53, 38, 54, 30, 55, 187, 157, 35, 56, 65, 50, + 52, 46, 53, 38, 54, 30, 55, 187, 157, 35, 56, 33, 42, 53, 38, 54, 30, 55, + 187, 157, 35, 56, 17, 34, 54, 30, 55, 187, 157, 35, 56, 9, 26, 55, 187, + 157, 35, 56, 5, 183, 157, 35, 56, 8, 26, 65, 183, 134, 35, 86, 6, 156, + 185, 20, 11, 75, 32, 80, 69, 82, 77, 73, 84, 84, 69, 68, 196, 171, 8, 6, + 83, 84, 45, 70, 69, 69, 167, 184, 6, 68, 10, 42, 68, 92, 2, 69, 70, 243, + 151, 35, 67, 4, 204, 161, 32, 6, 71, 69, 32, 65, 84, 32, 193, 70, 9, 69, + 32, 87, 73, 84, 72, 32, 86, 69, 4, 190, 245, 31, 67, 203, 165, 3, 83, 12, + 84, 4, 75, 69, 78, 32, 176, 192, 8, 2, 67, 67, 160, 210, 25, 2, 87, 78, + 231, 118, 79, 6, 248, 173, 27, 17, 67, 73, 82, 67, 76, 69, 32, 87, 73, + 84, 72, 32, 78, 79, 82, 84, 72, 218, 202, 6, 66, 147, 26, 72, 134, 1, + 208, 1, 4, 66, 66, 76, 69, 46, 71, 156, 4, 4, 72, 73, 68, 32, 36, 2, 76, + 76, 122, 83, 56, 4, 84, 84, 69, 82, 176, 249, 10, 10, 73, 76, 68, 73, 78, + 71, 32, 67, 79, 78, 184, 213, 16, 2, 82, 82, 247, 239, 6, 67, 4, 204, + 145, 28, 2, 32, 84, 243, 133, 7, 83, 63, 33, 6, 73, 78, 69, 83, 69, 32, + 60, 144, 1, 7, 76, 69, 84, 84, 69, 82, 32, 172, 2, 11, 86, 79, 87, 69, + 76, 32, 83, 73, 71, 78, 32, 190, 176, 16, 69, 225, 241, 13, 4, 80, 65, + 76, 76, 46, 154, 1, 77, 34, 78, 234, 145, 35, 66, 2, 67, 2, 68, 2, 71, 2, + 72, 2, 74, 2, 75, 2, 76, 2, 80, 2, 82, 2, 83, 2, 84, 2, 86, 2, 89, 187, + 2, 65, 4, 134, 146, 35, 80, 187, 2, 65, 12, 46, 71, 34, 89, 154, 145, 35, + 82, 187, 2, 65, 4, 182, 145, 35, 75, 187, 2, 65, 4, 150, 145, 35, 67, + 187, 2, 65, 10, 218, 252, 34, 65, 214, 22, 69, 2, 73, 2, 79, 3, 85, 40, + 142, 152, 10, 76, 135, 241, 18, 86, 10, 56, 2, 69, 84, 20, 4, 72, 79, 82, + 78, 231, 162, 9, 83, 5, 179, 210, 31, 32, 5, 241, 184, 27, 5, 32, 87, 73, + 84, 72, 9, 26, 84, 143, 192, 32, 32, 4, 234, 184, 27, 83, 15, 32, 5, 243, + 237, 34, 70, 240, 3, 140, 1, 23, 90, 65, 78, 84, 73, 78, 69, 32, 77, 85, + 83, 73, 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, 245, 179, 20, 5, 84, + 69, 32, 79, 82, 238, 3, 154, 2, 65, 128, 7, 2, 67, 72, 170, 1, 68, 158, + 5, 69, 206, 2, 70, 166, 8, 71, 202, 3, 73, 162, 2, 75, 142, 5, 76, 190, + 2, 77, 250, 4, 79, 114, 80, 174, 4, 82, 42, 83, 194, 5, 84, 196, 5, 2, + 86, 65, 250, 1, 89, 178, 160, 32, 78, 245, 142, 2, 9, 88, 73, 82, 79, 78, + 32, 75, 76, 65, 62, 60, 5, 71, 79, 71, 73, 32, 222, 1, 78, 118, 80, 199, + 2, 82, 16, 70, 65, 0, 2, 71, 79, 64, 2, 77, 69, 37, 5, 80, 79, 76, 73, + 32, 4, 17, 2, 82, 71, 4, 160, 227, 15, 2, 79, 84, 159, 169, 19, 73, 4, + 214, 151, 9, 84, 231, 224, 25, 83, 4, 26, 65, 1, 2, 71, 79, 2, 131, 203, + 17, 82, 6, 72, 6, 84, 73, 75, 69, 78, 79, 205, 233, 34, 6, 65, 84, 82, + 73, 67, 72, 4, 168, 32, 2, 75, 89, 147, 232, 34, 77, 22, 52, 5, 69, 83, + 79, 32, 69, 38, 79, 155, 246, 34, 76, 4, 204, 1, 2, 88, 79, 147, 55, 75, + 16, 80, 6, 83, 84, 82, 79, 70, 79, 188, 40, 3, 68, 69, 82, 181, 128, 1, + 2, 84, 72, 10, 24, 2, 73, 32, 79, 83, 4, 56, 9, 83, 89, 78, 68, 69, 83, + 77, 79, 83, 195, 24, 84, 2, 143, 44, 32, 7, 11, 32, 4, 214, 60, 68, 255, + 145, 22, 78, 18, 48, 2, 71, 79, 33, 6, 75, 84, 73, 75, 79, 32, 4, 170, + 21, 83, 171, 242, 34, 78, 14, 182, 190, 27, 86, 138, 170, 7, 90, 162, 8, + 75, 254, 2, 68, 2, 78, 162, 17, 71, 3, 80, 14, 80, 4, 82, 79, 65, 32, + 168, 37, 4, 79, 82, 69, 86, 221, 2, 4, 65, 77, 73, 76, 6, 228, 214, 19, + 2, 83, 80, 220, 191, 3, 3, 90, 89, 71, 185, 188, 10, 3, 75, 76, 73, 42, + 50, 73, 172, 219, 8, 2, 65, 83, 227, 138, 26, 89, 38, 122, 65, 200, 1, 5, + 69, 83, 73, 83, 32, 70, 71, 208, 1, 3, 80, 76, 73, 173, 183, 27, 8, 70, + 84, 79, 71, 71, 79, 83, 32, 10, 48, 6, 83, 84, 79, 76, 73, 32, 179, 203, + 33, 82, 8, 80, 6, 65, 80, 76, 73, 32, 77, 174, 55, 68, 141, 220, 22, 5, + 84, 72, 69, 83, 69, 4, 40, 2, 69, 71, 133, 165, 28, 2, 73, 75, 2, 195, + 135, 30, 65, 12, 38, 84, 246, 50, 65, 42, 68, 63, 77, 6, 194, 10, 69, + 239, 41, 82, 10, 72, 5, 79, 82, 71, 79, 78, 217, 128, 35, 7, 82, 65, 77, + 77, 65, 32, 71, 9, 69, 15, 32, 80, 65, 82, 69, 83, 84, 73, 71, 77, 69, + 78, 79, 78, 32, 6, 222, 38, 68, 137, 10, 8, 65, 82, 73, 83, 84, 69, 82, + 65, 5, 175, 47, 32, 16, 166, 1, 78, 116, 6, 84, 69, 82, 79, 78, 32, 188, + 44, 4, 88, 79, 32, 69, 156, 238, 32, 4, 80, 69, 71, 69, 152, 48, 6, 75, + 83, 84, 82, 69, 80, 237, 13, 3, 76, 65, 70, 4, 224, 22, 3, 68, 79, 70, + 145, 159, 27, 18, 65, 82, 88, 73, 83, 32, 75, 65, 73, 32, 70, 84, 72, 79, + 82, 65, 32, 86, 4, 208, 11, 5, 65, 82, 71, 79, 83, 151, 20, 80, 44, 180, + 1, 9, 65, 78, 69, 82, 79, 83, 73, 83, 32, 52, 6, 84, 72, 79, 82, 65, 32, + 165, 6, 22, 72, 84, 79, 82, 65, 32, 83, 75, 76, 73, 82, 79, 78, 32, 67, + 72, 82, 79, 77, 65, 32, 86, 6, 246, 4, 68, 14, 77, 25, 5, 84, 69, 84, 82, + 65, 36, 220, 2, 8, 65, 82, 67, 72, 65, 73, 79, 78, 80, 10, 68, 73, 65, + 84, 79, 78, 73, 75, 73, 32, 96, 11, 73, 32, 89, 70, 69, 83, 73, 83, 32, + 84, 69, 32, 15, 77, 65, 76, 65, 75, 79, 78, 32, 67, 72, 82, 79, 77, 65, + 32, 74, 78, 40, 8, 83, 75, 76, 73, 82, 79, 78, 32, 141, 209, 18, 18, 69, + 78, 65, 82, 77, 79, 78, 73, 79, 83, 32, 65, 78, 84, 73, 70, 79, 78, 5, + 57, 12, 32, 68, 69, 89, 84, 69, 82, 79, 85, 32, 73, 67, 2, 199, 175, 27, + 72, 14, 62, 78, 254, 216, 34, 90, 162, 8, 75, 254, 2, 68, 163, 17, 80, 6, + 242, 39, 73, 251, 147, 33, 65, 2, 237, 42, 4, 84, 65, 82, 84, 4, 18, 68, + 15, 77, 2, 35, 73, 2, 21, 3, 79, 78, 79, 2, 151, 19, 70, 4, 166, 19, 65, + 213, 219, 32, 2, 69, 78, 6, 84, 7, 67, 72, 82, 79, 77, 65, 32, 193, 149, + 17, 8, 68, 73, 65, 84, 79, 78, 79, 78, 4, 42, 86, 169, 153, 17, 4, 83, + 89, 78, 65, 2, 251, 209, 32, 65, 22, 80, 6, 69, 78, 73, 75, 73, 32, 44, + 2, 79, 82, 185, 26, 5, 82, 79, 78, 84, 72, 4, 192, 242, 30, 2, 68, 73, 1, + 2, 89, 70, 16, 68, 2, 71, 79, 225, 1, 10, 84, 72, 77, 73, 75, 79, 78, 32, + 78, 32, 12, 28, 2, 78, 32, 155, 1, 83, 10, 96, 14, 80, 65, 82, 69, 83, + 84, 73, 71, 77, 69, 78, 79, 78, 32, 242, 33, 65, 113, 3, 78, 69, 79, 4, + 214, 24, 68, 249, 202, 32, 5, 65, 82, 73, 83, 84, 2, 153, 191, 33, 5, 89, + 78, 84, 72, 69, 4, 142, 28, 65, 1, 2, 68, 73, 16, 56, 2, 77, 73, 114, 83, + 225, 240, 33, 4, 67, 72, 65, 68, 8, 34, 70, 169, 28, 3, 68, 73, 65, 6, + 40, 4, 84, 72, 79, 82, 195, 230, 33, 79, 4, 242, 160, 34, 79, 227, 79, + 65, 6, 44, 6, 65, 75, 73, 65, 32, 84, 231, 13, 79, 2, 153, 201, 21, 12, + 69, 76, 79, 85, 83, 32, 73, 67, 72, 73, 77, 65, 54, 108, 2, 65, 84, 96, + 6, 69, 78, 84, 73, 77, 65, 140, 1, 5, 76, 65, 83, 77, 65, 18, 79, 98, 82, + 175, 1, 89, 6, 40, 3, 65, 86, 65, 201, 3, 2, 72, 73, 4, 140, 29, 5, 32, + 84, 82, 79, 77, 135, 176, 34, 83, 18, 24, 2, 84, 65, 15, 32, 11, 11, 32, + 8, 36, 4, 78, 69, 79, 32, 183, 28, 65, 6, 252, 140, 28, 2, 77, 69, 254, + 212, 2, 75, 151, 226, 3, 65, 7, 243, 28, 32, 8, 64, 6, 78, 84, 69, 86, + 77, 65, 202, 212, 8, 82, 151, 246, 25, 85, 5, 181, 166, 14, 2, 32, 65, + 14, 44, 4, 65, 84, 73, 77, 105, 3, 69, 77, 65, 12, 18, 65, 35, 79, 8, + 182, 23, 32, 195, 209, 34, 84, 4, 134, 12, 75, 181, 184, 31, 5, 89, 80, + 79, 82, 82, 2, 227, 140, 27, 83, 2, 159, 201, 34, 76, 14, 22, 69, 147, + 22, 89, 12, 44, 5, 73, 77, 77, 65, 32, 191, 251, 29, 77, 10, 72, 2, 69, + 78, 0, 5, 73, 77, 73, 83, 69, 54, 84, 69, 3, 68, 89, 79, 2, 161, 160, 27, + 8, 79, 83, 32, 67, 72, 82, 79, 78, 4, 44, 5, 69, 83, 83, 65, 82, 1, 2, + 82, 73, 2, 17, 2, 79, 78, 2, 25, 4, 32, 67, 72, 82, 2, 247, 221, 33, 79, + 28, 84, 8, 65, 82, 84, 89, 82, 73, 65, 32, 229, 231, 30, 7, 73, 75, 82, + 79, 78, 32, 73, 26, 72, 5, 65, 76, 76, 73, 32, 38, 68, 38, 80, 134, 1, + 86, 30, 84, 87, 76, 4, 34, 68, 177, 2, 3, 80, 82, 79, 2, 237, 2, 5, 69, + 89, 84, 69, 82, 8, 60, 7, 76, 65, 71, 73, 79, 83, 32, 45, 4, 82, 79, 84, + 79, 4, 200, 1, 5, 84, 69, 84, 65, 82, 111, 73, 4, 22, 86, 227, 1, 83, 2, + 209, 1, 3, 65, 82, 89, 8, 56, 8, 69, 84, 65, 82, 84, 79, 83, 32, 61, 2, + 82, 73, 4, 22, 76, 135, 1, 73, 2, 21, 3, 69, 71, 69, 2, 63, 84, 4, 18, + 70, 35, 84, 2, 209, 197, 21, 3, 79, 78, 73, 2, 11, 79, 2, 11, 83, 2, 17, + 2, 32, 73, 2, 155, 239, 22, 67, 18, 92, 4, 76, 73, 71, 79, 180, 1, 5, 89, + 82, 65, 78, 73, 210, 14, 88, 165, 130, 34, 2, 77, 65, 4, 211, 1, 78, 42, 60, 2, 65, 82, 100, 2, 73, 65, 90, 69, 169, 1, 2, 83, 73, 12, 40, 2, 65, - 75, 201, 246, 16, 2, 73, 67, 10, 52, 3, 65, 76, 69, 45, 6, 76, 73, 84, + 75, 181, 170, 17, 2, 73, 67, 10, 52, 3, 65, 76, 69, 45, 6, 76, 73, 84, 73, 75, 73, 4, 11, 83, 4, 17, 2, 77, 65, 4, 23, 32, 7, 11, 32, 4, 198, - 15, 65, 151, 219, 21, 78, 12, 72, 3, 84, 65, 83, 136, 3, 6, 76, 65, 83, - 84, 79, 78, 239, 202, 8, 82, 6, 26, 84, 211, 216, 33, 77, 4, 32, 2, 79, - 75, 231, 218, 33, 73, 2, 173, 185, 33, 2, 79, 85, 14, 36, 5, 70, 73, 83, + 15, 65, 155, 151, 22, 78, 12, 72, 3, 84, 65, 83, 136, 3, 6, 76, 65, 83, + 84, 79, 78, 191, 216, 8, 82, 6, 26, 84, 247, 220, 34, 77, 4, 32, 2, 79, + 75, 139, 223, 34, 73, 2, 209, 189, 34, 2, 79, 85, 14, 36, 5, 70, 73, 83, 84, 79, 67, 76, 10, 46, 80, 214, 1, 78, 170, 8, 76, 187, 1, 83, 2, 135, - 11, 65, 4, 138, 138, 33, 79, 215, 79, 73, 4, 178, 5, 69, 229, 178, 33, 2, + 11, 65, 4, 162, 142, 34, 79, 227, 79, 73, 4, 178, 5, 69, 137, 183, 34, 2, 65, 80, 42, 120, 5, 69, 73, 83, 77, 65, 32, 8, 73, 77, 65, 78, 83, 73, - 83, 32, 182, 1, 84, 154, 1, 89, 249, 240, 1, 3, 65, 88, 73, 5, 11, 32, 2, - 219, 230, 21, 78, 16, 36, 2, 65, 82, 1, 3, 84, 72, 69, 8, 25, 4, 83, 69, + 83, 32, 182, 1, 84, 154, 1, 89, 245, 240, 1, 3, 65, 88, 73, 5, 11, 32, 2, + 223, 162, 22, 78, 16, 36, 2, 65, 82, 1, 3, 84, 72, 69, 8, 25, 4, 83, 69, 79, 83, 9, 11, 32, 6, 18, 84, 39, 68, 4, 34, 82, 13, 4, 69, 84, 82, 65, - 2, 11, 73, 2, 153, 198, 26, 3, 83, 73, 77, 8, 68, 5, 65, 86, 82, 79, 83, - 52, 4, 82, 65, 71, 71, 175, 147, 16, 73, 5, 29, 5, 32, 65, 80, 79, 68, 2, - 175, 206, 8, 69, 2, 145, 241, 1, 2, 73, 83, 12, 34, 78, 149, 1, 3, 82, - 77, 65, 8, 36, 5, 65, 71, 77, 65, 32, 91, 69, 6, 154, 8, 65, 150, 219, - 21, 78, 233, 224, 4, 10, 77, 69, 84, 65, 32, 83, 84, 65, 86, 82, 2, 251, - 178, 33, 86, 5, 11, 84, 2, 255, 193, 16, 73, 40, 42, 69, 70, 72, 218, 1, - 82, 227, 2, 73, 6, 136, 12, 3, 84, 82, 65, 162, 148, 8, 76, 137, 167, 23, - 2, 83, 83, 12, 26, 69, 131, 174, 33, 73, 10, 64, 2, 77, 65, 133, 190, 32, + 2, 11, 73, 2, 141, 146, 27, 3, 83, 73, 77, 8, 68, 5, 65, 86, 82, 79, 83, + 52, 4, 82, 65, 71, 71, 179, 199, 16, 73, 5, 29, 5, 32, 65, 80, 79, 68, 2, + 255, 219, 8, 69, 2, 141, 241, 1, 2, 73, 83, 12, 34, 78, 149, 1, 3, 82, + 77, 65, 8, 36, 5, 65, 71, 77, 65, 32, 91, 69, 6, 154, 8, 65, 154, 151, + 22, 78, 217, 240, 4, 10, 77, 69, 84, 65, 32, 83, 84, 65, 86, 82, 2, 159, + 183, 34, 86, 5, 11, 84, 2, 243, 245, 16, 73, 40, 42, 69, 70, 72, 218, 1, + 82, 227, 2, 73, 6, 136, 12, 3, 84, 82, 65, 242, 161, 8, 76, 201, 154, 24, + 2, 83, 83, 12, 26, 69, 167, 178, 34, 73, 10, 64, 2, 77, 65, 145, 194, 33, 8, 83, 32, 75, 65, 73, 32, 65, 80, 9, 56, 2, 32, 65, 33, 8, 84, 73, 83, - 77, 79, 83, 32, 69, 2, 177, 209, 32, 3, 80, 76, 79, 4, 182, 178, 33, 83, - 3, 88, 20, 38, 73, 73, 5, 79, 77, 73, 75, 79, 6, 48, 2, 71, 79, 182, 129, - 29, 80, 131, 207, 4, 65, 2, 191, 153, 32, 82, 14, 42, 76, 32, 2, 78, 32, - 62, 80, 95, 83, 2, 11, 89, 2, 191, 174, 33, 71, 6, 26, 65, 135, 222, 21, - 78, 4, 250, 2, 82, 239, 169, 33, 76, 4, 46, 65, 141, 157, 32, 5, 83, 73, - 70, 73, 83, 2, 201, 173, 33, 6, 82, 65, 75, 65, 76, 69, 2, 11, 89, 2, - 141, 140, 16, 2, 78, 65, 10, 26, 82, 183, 172, 33, 84, 8, 21, 3, 69, 73, - 65, 8, 26, 32, 113, 2, 73, 32, 6, 38, 69, 242, 5, 68, 251, 213, 21, 78, - 2, 11, 75, 2, 29, 5, 70, 79, 78, 73, 84, 2, 141, 253, 32, 2, 73, 75, 2, - 11, 65, 2, 11, 82, 2, 129, 252, 32, 3, 67, 72, 65, 22, 28, 2, 70, 69, - 231, 3, 80, 14, 34, 78, 49, 4, 83, 73, 83, 32, 4, 11, 32, 4, 210, 234, - 29, 75, 235, 183, 3, 65, 10, 42, 65, 42, 68, 62, 77, 89, 2, 84, 82, 2, + 77, 79, 83, 32, 69, 2, 189, 213, 33, 3, 80, 76, 79, 4, 218, 182, 34, 83, + 3, 88, 20, 38, 73, 73, 5, 79, 77, 73, 75, 79, 6, 48, 2, 71, 79, 206, 217, + 29, 80, 143, 251, 4, 65, 2, 183, 156, 33, 82, 14, 42, 76, 32, 2, 78, 32, + 62, 80, 95, 83, 2, 11, 89, 2, 227, 178, 34, 71, 6, 26, 65, 139, 154, 22, + 78, 4, 250, 2, 82, 147, 174, 34, 76, 4, 46, 65, 129, 160, 33, 5, 83, 73, + 70, 73, 83, 2, 237, 177, 34, 6, 82, 65, 75, 65, 76, 69, 2, 11, 89, 2, + 145, 192, 16, 2, 78, 65, 10, 26, 82, 219, 176, 34, 84, 8, 21, 3, 69, 73, + 65, 8, 26, 32, 113, 2, 73, 32, 6, 38, 69, 242, 5, 68, 255, 145, 22, 78, + 2, 11, 75, 2, 29, 5, 70, 79, 78, 73, 84, 2, 165, 129, 34, 2, 73, 75, 2, + 11, 65, 2, 11, 82, 2, 153, 128, 34, 3, 67, 72, 65, 22, 28, 2, 70, 69, + 231, 3, 80, 14, 34, 78, 49, 4, 83, 73, 83, 32, 4, 11, 32, 4, 202, 196, + 30, 75, 151, 226, 3, 65, 10, 42, 65, 42, 68, 62, 77, 89, 2, 84, 82, 2, 133, 2, 6, 80, 76, 73, 32, 68, 89, 2, 233, 1, 11, 73, 71, 82, 65, 77, 77, 79, 83, 32, 69, 88, 2, 173, 1, 18, 79, 78, 79, 71, 82, 65, 77, 77, 79, 83, 32, 84, 69, 83, 83, 69, 82, 65, 4, 11, 73, 4, 60, 11, 71, 82, 65, 77, - 77, 79, 83, 32, 79, 75, 84, 59, 84, 2, 11, 79, 2, 189, 174, 2, 6, 32, 68, - 79, 68, 69, 75, 2, 129, 248, 32, 4, 73, 77, 79, 82, 8, 26, 79, 243, 248, - 28, 83, 6, 56, 6, 75, 82, 73, 83, 73, 83, 157, 133, 29, 2, 82, 82, 5, 17, - 2, 32, 68, 2, 11, 73, 2, 159, 248, 28, 80, 162, 82, 182, 1, 65, 134, 72, - 69, 230, 1, 72, 154, 31, 73, 240, 41, 3, 74, 75, 32, 238, 25, 76, 186, - 18, 79, 194, 113, 82, 162, 8, 85, 130, 128, 2, 89, 166, 221, 18, 71, 226, - 155, 10, 83, 203, 18, 67, 198, 13, 110, 68, 66, 76, 50, 77, 90, 78, 174, - 51, 80, 62, 82, 198, 7, 84, 138, 1, 85, 198, 240, 17, 67, 139, 181, 6, - 83, 4, 34, 85, 221, 189, 26, 2, 65, 32, 2, 201, 193, 31, 2, 67, 69, 4, - 144, 189, 9, 3, 76, 32, 77, 131, 130, 19, 69, 6, 36, 3, 69, 82, 65, 207, - 242, 32, 80, 5, 217, 226, 26, 7, 32, 87, 73, 84, 72, 32, 70, 193, 11, + 77, 79, 83, 32, 79, 75, 84, 59, 84, 2, 11, 79, 2, 181, 174, 2, 6, 32, 68, + 79, 68, 69, 75, 2, 153, 252, 33, 4, 73, 77, 79, 82, 8, 26, 79, 139, 209, + 29, 83, 6, 56, 6, 75, 82, 73, 83, 73, 83, 181, 221, 29, 2, 82, 82, 5, 17, + 2, 32, 68, 2, 11, 73, 2, 183, 208, 29, 80, 172, 82, 182, 1, 65, 134, 72, + 69, 230, 1, 72, 154, 31, 73, 208, 41, 3, 74, 75, 32, 138, 26, 76, 186, + 18, 79, 194, 113, 82, 234, 7, 85, 150, 128, 2, 89, 242, 164, 19, 71, 226, + 216, 10, 83, 203, 18, 67, 198, 13, 110, 68, 66, 76, 50, 77, 90, 78, 174, + 51, 80, 62, 82, 198, 7, 84, 138, 1, 85, 242, 164, 18, 67, 187, 203, 6, + 83, 4, 34, 85, 137, 139, 27, 2, 65, 32, 2, 221, 194, 32, 2, 67, 69, 4, + 152, 207, 9, 3, 76, 32, 77, 247, 200, 19, 69, 6, 36, 3, 69, 82, 65, 231, + 246, 33, 80, 5, 165, 177, 27, 7, 32, 87, 73, 84, 72, 32, 70, 193, 11, 148, 1, 16, 65, 68, 73, 65, 78, 32, 83, 89, 76, 76, 65, 66, 73, 67, 83, - 32, 148, 49, 2, 67, 69, 94, 68, 172, 152, 23, 3, 78, 69, 68, 139, 225, 9, - 79, 172, 11, 226, 1, 65, 190, 1, 66, 162, 2, 67, 242, 8, 69, 50, 70, 198, - 4, 72, 38, 73, 30, 75, 134, 1, 76, 82, 77, 174, 1, 78, 202, 4, 81, 178, - 1, 79, 194, 1, 80, 70, 82, 142, 1, 83, 194, 4, 84, 246, 3, 87, 254, 5, - 89, 223, 152, 17, 71, 21, 90, 65, 30, 73, 40, 10, 84, 72, 65, 80, 65, 83, - 67, 65, 78, 32, 242, 190, 33, 78, 3, 89, 7, 178, 191, 33, 73, 3, 89, 5, - 133, 150, 23, 5, 86, 73, 76, 73, 75, 4, 238, 190, 33, 77, 3, 83, 42, 156, - 1, 9, 76, 65, 67, 75, 70, 79, 79, 84, 32, 172, 226, 18, 9, 73, 66, 76, - 69, 45, 67, 82, 69, 69, 253, 180, 7, 10, 69, 65, 86, 69, 82, 32, 68, 69, - 78, 69, 36, 82, 87, 158, 208, 11, 75, 2, 78, 198, 236, 21, 65, 2, 69, 2, - 73, 2, 79, 3, 83, 11, 222, 188, 33, 65, 2, 69, 2, 73, 3, 79, 141, 3, 82, - 65, 186, 42, 87, 206, 155, 8, 72, 218, 182, 22, 79, 182, 56, 73, 207, - 134, 2, 69, 241, 2, 48, 6, 82, 82, 73, 69, 82, 32, 219, 180, 31, 65, 234, + 32, 148, 49, 2, 67, 69, 94, 68, 216, 227, 23, 3, 78, 69, 68, 131, 154, + 10, 79, 172, 11, 226, 1, 65, 190, 1, 66, 162, 2, 67, 242, 8, 69, 50, 70, + 198, 4, 72, 38, 73, 30, 75, 134, 1, 76, 82, 77, 174, 1, 78, 202, 4, 81, + 178, 1, 79, 194, 1, 80, 70, 82, 142, 1, 83, 194, 4, 84, 246, 3, 87, 254, + 5, 89, 247, 204, 17, 71, 21, 90, 65, 30, 73, 40, 10, 84, 72, 65, 80, 65, + 83, 67, 65, 78, 32, 150, 195, 34, 78, 3, 89, 7, 214, 195, 34, 73, 3, 89, + 5, 169, 221, 23, 5, 86, 73, 76, 73, 75, 4, 146, 195, 34, 77, 3, 83, 42, + 156, 1, 9, 76, 65, 67, 75, 70, 79, 79, 84, 32, 172, 151, 19, 9, 73, 66, + 76, 69, 45, 67, 82, 69, 69, 221, 202, 7, 10, 69, 65, 86, 69, 82, 32, 68, + 69, 78, 69, 36, 82, 87, 230, 228, 11, 75, 2, 78, 162, 220, 22, 65, 2, 69, + 2, 73, 2, 79, 3, 83, 11, 130, 193, 34, 65, 2, 69, 2, 73, 3, 79, 141, 3, + 82, 65, 186, 42, 87, 142, 169, 8, 72, 222, 166, 23, 79, 134, 60, 73, 223, + 137, 2, 69, 241, 2, 48, 6, 82, 82, 73, 69, 82, 32, 239, 181, 32, 65, 234, 2, 170, 1, 68, 122, 71, 94, 72, 46, 73, 46, 74, 82, 75, 30, 78, 66, 83, 66, 80, 2, 90, 58, 84, 38, 76, 132, 1, 2, 67, 72, 2, 77, 2, 82, 2, 87, 2, - 89, 171, 157, 33, 69, 32, 46, 69, 202, 5, 76, 2, 90, 255, 179, 33, 73, 6, - 26, 78, 171, 185, 33, 69, 4, 166, 210, 16, 69, 181, 191, 6, 2, 84, 65, - 30, 254, 4, 72, 174, 234, 20, 87, 166, 136, 7, 65, 154, 129, 5, 69, 150, - 64, 73, 2, 79, 3, 85, 19, 162, 4, 87, 170, 157, 33, 69, 215, 22, 73, 5, - 209, 223, 13, 6, 78, 73, 84, 73, 65, 76, 26, 202, 3, 74, 234, 243, 32, - 69, 222, 61, 87, 186, 2, 65, 2, 73, 2, 79, 3, 85, 26, 154, 1, 75, 227, 1, - 72, 14, 198, 246, 32, 69, 150, 64, 65, 2, 71, 2, 73, 2, 79, 3, 85, 26, - 62, 72, 202, 245, 32, 69, 150, 64, 65, 2, 73, 2, 79, 3, 85, 15, 198, 245, - 32, 69, 150, 64, 65, 2, 73, 2, 79, 3, 85, 72, 34, 76, 70, 84, 66, 72, 3, - 83, 24, 130, 1, 72, 234, 243, 32, 69, 150, 64, 65, 2, 73, 2, 79, 3, 85, - 24, 62, 83, 234, 243, 32, 69, 150, 64, 65, 2, 73, 2, 79, 3, 85, 12, 230, - 243, 32, 69, 150, 64, 65, 2, 73, 2, 79, 3, 85, 7, 236, 29, 4, 65, 83, 84, - 69, 215, 149, 33, 78, 51, 74, 65, 22, 73, 190, 231, 30, 85, 250, 11, 79, - 186, 79, 87, 203, 239, 1, 69, 7, 251, 171, 31, 65, 33, 40, 4, 78, 65, 76, - 32, 139, 178, 33, 73, 28, 160, 1, 2, 68, 79, 134, 1, 82, 78, 83, 150, - 221, 13, 71, 186, 2, 65, 132, 165, 3, 6, 66, 79, 84, 84, 79, 77, 0, 3, - 84, 79, 80, 202, 205, 12, 80, 211, 203, 2, 77, 6, 44, 5, 85, 66, 76, 69, - 32, 195, 212, 23, 87, 4, 232, 193, 6, 12, 83, 72, 79, 82, 84, 32, 86, 69, - 82, 84, 73, 67, 131, 159, 7, 65, 6, 38, 73, 253, 229, 31, 3, 65, 73, 83, - 4, 218, 132, 17, 71, 139, 170, 16, 78, 4, 184, 199, 24, 4, 77, 65, 76, - 76, 129, 202, 6, 4, 72, 79, 82, 84, 4, 198, 229, 25, 89, 139, 201, 7, 75, - 7, 170, 174, 33, 73, 3, 78, 39, 66, 87, 234, 27, 65, 230, 210, 30, 79, - 182, 56, 73, 207, 134, 2, 69, 19, 182, 250, 11, 65, 150, 244, 18, 79, - 182, 56, 73, 207, 134, 2, 69, 49, 142, 7, 72, 154, 20, 65, 66, 87, 166, - 210, 30, 79, 182, 56, 73, 207, 134, 2, 69, 43, 70, 69, 38, 79, 238, 25, - 65, 66, 87, 218, 138, 31, 73, 207, 134, 2, 72, 7, 229, 133, 26, 4, 68, - 73, 65, 76, 7, 11, 79, 5, 161, 233, 31, 8, 83, 69, 45, 67, 82, 69, 69, + 89, 207, 161, 34, 69, 32, 46, 69, 202, 5, 76, 2, 90, 163, 184, 34, 73, 6, + 26, 78, 207, 189, 34, 69, 4, 146, 134, 17, 69, 237, 210, 6, 2, 84, 65, + 30, 254, 4, 72, 182, 166, 21, 87, 234, 165, 7, 65, 230, 171, 5, 69, 162, + 64, 73, 2, 79, 3, 85, 19, 162, 4, 87, 206, 161, 34, 69, 215, 22, 73, 5, + 225, 135, 14, 6, 78, 73, 84, 73, 65, 76, 26, 202, 3, 74, 130, 248, 33, + 69, 234, 61, 87, 186, 2, 65, 2, 73, 2, 79, 3, 85, 26, 154, 1, 75, 227, 1, + 72, 14, 222, 250, 33, 69, 162, 64, 65, 2, 71, 2, 73, 2, 79, 3, 85, 26, + 62, 72, 226, 249, 33, 69, 162, 64, 65, 2, 73, 2, 79, 3, 85, 15, 222, 249, + 33, 69, 162, 64, 65, 2, 73, 2, 79, 3, 85, 72, 34, 76, 70, 84, 66, 72, 3, + 83, 24, 130, 1, 72, 130, 248, 33, 69, 162, 64, 65, 2, 73, 2, 79, 3, 85, + 24, 62, 83, 130, 248, 33, 69, 162, 64, 65, 2, 73, 2, 79, 3, 85, 12, 254, + 247, 33, 69, 162, 64, 65, 2, 73, 2, 79, 3, 85, 7, 236, 29, 4, 65, 83, 84, + 69, 251, 153, 34, 78, 51, 74, 65, 22, 73, 130, 229, 31, 85, 250, 11, 79, + 150, 83, 87, 207, 242, 1, 69, 7, 143, 173, 32, 65, 33, 40, 4, 78, 65, 76, + 32, 175, 182, 34, 73, 28, 160, 1, 2, 68, 79, 134, 1, 82, 78, 83, 166, + 133, 14, 71, 186, 2, 65, 128, 177, 3, 6, 66, 79, 84, 84, 79, 77, 0, 3, + 84, 79, 80, 166, 244, 12, 80, 247, 244, 2, 77, 6, 44, 5, 85, 66, 76, 69, + 32, 131, 160, 24, 87, 4, 252, 198, 6, 12, 83, 72, 79, 82, 84, 32, 86, 69, + 82, 84, 73, 67, 255, 193, 7, 65, 6, 38, 73, 213, 231, 32, 3, 65, 73, 83, + 4, 230, 184, 17, 71, 163, 250, 16, 78, 4, 152, 146, 25, 4, 77, 65, 76, + 76, 213, 252, 6, 4, 72, 79, 82, 84, 4, 150, 176, 26, 89, 223, 130, 8, 75, + 7, 206, 178, 34, 73, 3, 78, 39, 66, 87, 234, 27, 65, 170, 208, 31, 79, + 134, 60, 73, 223, 137, 2, 69, 19, 138, 143, 12, 65, 134, 221, 19, 79, + 134, 60, 73, 223, 137, 2, 69, 49, 142, 7, 72, 154, 20, 65, 66, 87, 234, + 207, 31, 79, 134, 60, 73, 223, 137, 2, 69, 43, 70, 69, 38, 79, 238, 25, + 65, 66, 87, 238, 139, 32, 73, 223, 137, 2, 72, 7, 197, 208, 26, 4, 68, + 73, 65, 76, 7, 11, 79, 5, 249, 234, 32, 8, 83, 69, 45, 67, 82, 69, 69, 32, 149, 1, 164, 1, 8, 45, 67, 82, 69, 69, 32, 84, 72, 38, 65, 250, 2, - 71, 76, 2, 78, 71, 48, 4, 85, 78, 65, 86, 142, 20, 79, 30, 87, 218, 138, - 31, 73, 206, 134, 2, 69, 3, 72, 6, 150, 163, 31, 73, 207, 134, 2, 69, 63, + 71, 76, 2, 78, 71, 48, 4, 85, 78, 65, 86, 142, 20, 79, 30, 87, 238, 139, + 32, 73, 222, 137, 2, 69, 3, 72, 6, 170, 164, 32, 73, 223, 137, 2, 69, 63, 104, 6, 83, 75, 65, 80, 73, 32, 188, 1, 7, 84, 84, 73, 76, 73, 75, 32, - 206, 160, 31, 65, 207, 134, 2, 89, 30, 70, 83, 86, 87, 210, 17, 67, 2, - 75, 2, 77, 2, 78, 2, 84, 3, 89, 14, 220, 230, 27, 2, 75, 87, 178, 124, - 67, 2, 80, 2, 84, 182, 213, 2, 87, 187, 113, 45, 4, 194, 136, 33, 79, - 191, 28, 65, 24, 30, 72, 1, 3, 83, 72, 82, 12, 202, 229, 27, 65, 174, - 130, 3, 79, 183, 56, 73, 19, 38, 65, 162, 231, 30, 79, 183, 56, 73, 9, - 210, 159, 31, 65, 207, 134, 2, 73, 15, 206, 228, 27, 65, 174, 130, 3, 79, - 183, 56, 73, 18, 224, 9, 3, 73, 75, 32, 149, 243, 22, 2, 85, 84, 33, 68, - 7, 74, 73, 66, 87, 65, 89, 32, 210, 164, 33, 78, 2, 79, 3, 89, 24, 74, - 78, 166, 176, 29, 83, 226, 243, 3, 67, 2, 75, 2, 77, 2, 80, 3, 84, 11, - 11, 87, 8, 246, 228, 30, 79, 183, 56, 73, 39, 206, 4, 87, 166, 13, 65, - 38, 79, 246, 138, 31, 73, 207, 134, 2, 69, 39, 104, 7, 45, 67, 82, 69, - 69, 32, 82, 146, 14, 87, 182, 2, 65, 230, 210, 30, 79, 182, 56, 73, 207, - 134, 2, 69, 4, 210, 139, 33, 87, 215, 22, 69, 125, 94, 65, 218, 1, 72, - 138, 1, 79, 238, 2, 87, 154, 175, 11, 80, 250, 229, 19, 73, 207, 134, 2, - 69, 43, 26, 89, 195, 154, 31, 65, 37, 25, 4, 73, 83, 73, 32, 34, 70, 72, - 54, 74, 218, 4, 83, 198, 135, 33, 89, 202, 18, 84, 147, 1, 77, 10, 166, - 225, 30, 79, 130, 191, 2, 65, 2, 69, 3, 73, 6, 214, 208, 29, 85, 159, - 207, 3, 73, 37, 70, 87, 202, 13, 79, 174, 222, 11, 65, 202, 172, 19, 73, - 207, 134, 2, 69, 16, 198, 13, 79, 150, 208, 27, 65, 226, 186, 3, 73, 207, - 134, 2, 69, 15, 80, 12, 85, 84, 72, 45, 83, 76, 65, 86, 69, 89, 32, 75, - 246, 157, 33, 79, 3, 89, 8, 226, 156, 33, 65, 2, 69, 2, 73, 3, 79, 117, - 110, 72, 190, 1, 76, 78, 84, 238, 8, 65, 66, 87, 226, 164, 11, 89, 198, - 173, 19, 79, 182, 56, 73, 207, 134, 2, 69, 39, 108, 7, 45, 67, 82, 69, - 69, 32, 84, 226, 4, 87, 222, 213, 27, 65, 174, 130, 3, 79, 182, 56, 73, - 207, 134, 2, 69, 16, 11, 72, 17, 174, 218, 27, 65, 174, 130, 3, 79, 182, - 56, 73, 207, 134, 2, 69, 12, 11, 72, 12, 142, 220, 30, 79, 174, 168, 2, - 87, 214, 22, 65, 2, 69, 3, 73, 24, 50, 72, 158, 154, 33, 65, 2, 69, 2, - 73, 3, 79, 17, 238, 216, 27, 65, 174, 130, 3, 79, 174, 168, 2, 87, 214, - 22, 69, 3, 73, 224, 1, 54, 69, 218, 3, 79, 174, 226, 11, 65, 203, 172, - 19, 73, 185, 1, 17, 2, 83, 84, 182, 1, 44, 6, 45, 67, 82, 69, 69, 32, + 226, 161, 32, 65, 223, 137, 2, 89, 30, 70, 83, 86, 87, 210, 17, 67, 2, + 75, 2, 77, 2, 78, 2, 84, 3, 89, 14, 168, 192, 28, 2, 75, 87, 254, 122, + 67, 2, 80, 2, 84, 190, 254, 2, 87, 175, 116, 45, 4, 230, 140, 34, 79, + 191, 28, 65, 24, 30, 72, 1, 3, 83, 72, 82, 12, 150, 191, 28, 65, 166, + 166, 3, 79, 135, 60, 73, 19, 38, 65, 230, 228, 31, 79, 135, 60, 73, 9, + 230, 160, 32, 65, 223, 137, 2, 73, 15, 154, 190, 28, 65, 166, 166, 3, 79, + 135, 60, 73, 18, 224, 9, 3, 73, 75, 32, 185, 186, 23, 2, 85, 84, 33, 68, + 7, 74, 73, 66, 87, 65, 89, 32, 246, 168, 34, 78, 2, 79, 3, 89, 24, 74, + 78, 142, 136, 30, 83, 158, 160, 4, 67, 2, 75, 2, 77, 2, 80, 3, 84, 11, + 11, 87, 8, 186, 226, 31, 79, 135, 60, 73, 39, 206, 4, 87, 166, 13, 65, + 38, 79, 138, 140, 32, 73, 223, 137, 2, 69, 39, 104, 7, 45, 67, 82, 69, + 69, 32, 82, 146, 14, 87, 182, 2, 65, 170, 208, 31, 79, 134, 60, 73, 223, + 137, 2, 69, 4, 246, 143, 34, 87, 215, 22, 69, 125, 94, 65, 218, 1, 72, + 138, 1, 79, 238, 2, 87, 226, 195, 11, 80, 198, 210, 20, 73, 223, 137, 2, + 69, 43, 26, 89, 215, 155, 32, 65, 37, 25, 4, 73, 83, 73, 32, 34, 70, 72, + 54, 74, 218, 4, 83, 234, 139, 34, 89, 202, 18, 84, 147, 1, 77, 10, 234, + 222, 31, 79, 226, 197, 2, 65, 2, 69, 3, 73, 6, 238, 208, 30, 85, 171, + 211, 3, 73, 37, 70, 87, 202, 13, 79, 130, 243, 11, 65, 138, 153, 20, 73, + 223, 137, 2, 69, 16, 198, 13, 79, 226, 169, 28, 65, 170, 226, 3, 73, 223, + 137, 2, 69, 15, 80, 12, 85, 84, 72, 45, 83, 76, 65, 86, 69, 89, 32, 75, + 154, 162, 34, 79, 3, 89, 8, 134, 161, 34, 65, 2, 69, 2, 73, 3, 79, 117, + 110, 72, 190, 1, 76, 78, 84, 238, 8, 65, 66, 87, 170, 185, 11, 89, 194, + 150, 20, 79, 134, 60, 73, 223, 137, 2, 69, 39, 108, 7, 45, 67, 82, 69, + 69, 32, 84, 226, 4, 87, 170, 175, 28, 65, 166, 166, 3, 79, 134, 60, 73, + 223, 137, 2, 69, 16, 11, 72, 17, 250, 179, 28, 65, 166, 166, 3, 79, 134, + 60, 73, 223, 137, 2, 69, 12, 11, 72, 12, 210, 217, 31, 79, 142, 175, 2, + 87, 214, 22, 65, 2, 69, 3, 73, 24, 50, 72, 194, 158, 34, 65, 2, 69, 2, + 73, 3, 79, 17, 186, 178, 28, 65, 166, 166, 3, 79, 142, 175, 2, 87, 214, + 22, 69, 3, 73, 224, 1, 54, 69, 218, 3, 79, 130, 247, 11, 65, 139, 153, + 20, 73, 185, 1, 17, 2, 83, 84, 182, 1, 44, 6, 45, 67, 82, 69, 69, 32, 251, 2, 69, 180, 1, 110, 76, 66, 77, 2, 80, 2, 89, 16, 2, 78, 87, 38, 82, - 62, 83, 26, 67, 2, 75, 18, 84, 26, 70, 199, 4, 87, 27, 178, 6, 87, 250, - 207, 27, 65, 174, 130, 3, 79, 131, 191, 2, 69, 17, 243, 5, 87, 6, 218, - 213, 27, 65, 175, 193, 5, 69, 13, 154, 167, 31, 87, 202, 239, 1, 65, 2, + 62, 83, 26, 67, 2, 75, 18, 84, 26, 70, 199, 4, 87, 27, 178, 6, 87, 198, + 169, 28, 65, 166, 166, 3, 79, 227, 197, 2, 69, 17, 243, 5, 87, 6, 166, + 175, 28, 65, 135, 236, 5, 69, 13, 186, 168, 32, 87, 206, 242, 1, 65, 2, 69, 2, 73, 3, 79, 28, 22, 72, 239, 4, 87, 14, 235, 4, 87, 16, 22, 72, - 199, 4, 87, 2, 159, 166, 31, 87, 2, 135, 240, 22, 82, 31, 11, 79, 29, 41, - 8, 68, 83, 45, 67, 82, 69, 69, 32, 26, 56, 2, 84, 72, 201, 149, 31, 6, - 70, 73, 78, 65, 76, 32, 25, 50, 87, 154, 148, 33, 65, 2, 69, 2, 73, 3, - 79, 14, 234, 210, 27, 65, 174, 130, 3, 79, 182, 56, 73, 251, 239, 1, 69, - 61, 92, 6, 45, 67, 82, 69, 69, 32, 150, 1, 65, 38, 79, 30, 87, 218, 138, - 31, 73, 207, 134, 2, 69, 24, 110, 80, 194, 209, 30, 67, 2, 75, 2, 76, 2, - 77, 2, 78, 2, 83, 2, 84, 2, 89, 214, 161, 2, 79, 247, 30, 87, 4, 190, - 162, 31, 87, 215, 208, 1, 79, 9, 150, 139, 31, 65, 207, 134, 2, 89, 7, - 190, 145, 33, 79, 3, 89, 14, 246, 207, 27, 65, 174, 130, 3, 79, 182, 56, - 73, 207, 134, 2, 69, 10, 26, 76, 203, 144, 33, 82, 9, 26, 32, 239, 162, - 9, 76, 4, 222, 217, 21, 67, 207, 208, 9, 84, 4, 178, 249, 32, 76, 215, - 22, 89, 4, 180, 247, 14, 3, 73, 84, 85, 253, 241, 7, 3, 82, 73, 67, 124, + 199, 4, 87, 2, 191, 167, 32, 87, 2, 171, 183, 23, 82, 31, 11, 79, 29, 41, + 8, 68, 83, 45, 67, 82, 69, 69, 32, 26, 56, 2, 84, 72, 225, 150, 32, 6, + 70, 73, 78, 65, 76, 32, 25, 50, 87, 190, 152, 34, 65, 2, 69, 2, 73, 3, + 79, 14, 182, 172, 28, 65, 166, 166, 3, 79, 134, 60, 73, 139, 243, 1, 69, + 61, 92, 6, 45, 67, 82, 69, 69, 32, 150, 1, 65, 38, 79, 30, 87, 238, 139, + 32, 73, 223, 137, 2, 69, 24, 110, 80, 134, 207, 31, 67, 2, 75, 2, 76, 2, + 77, 2, 78, 2, 83, 2, 84, 2, 89, 182, 168, 2, 79, 247, 30, 87, 4, 222, + 163, 32, 87, 219, 211, 1, 79, 9, 170, 140, 32, 65, 223, 137, 2, 89, 7, + 226, 149, 34, 79, 3, 89, 14, 194, 169, 28, 65, 166, 166, 3, 79, 134, 60, + 73, 223, 137, 2, 69, 10, 26, 76, 239, 148, 34, 82, 9, 26, 32, 155, 181, + 9, 76, 4, 182, 150, 22, 67, 151, 149, 10, 84, 4, 214, 253, 33, 76, 215, + 22, 89, 4, 216, 171, 15, 3, 73, 84, 85, 253, 132, 8, 3, 82, 73, 67, 124, 140, 1, 2, 68, 32, 102, 69, 72, 11, 73, 65, 78, 32, 76, 69, 84, 84, 69, - 82, 32, 210, 3, 79, 62, 80, 90, 82, 181, 250, 26, 4, 32, 83, 76, 73, 6, - 52, 5, 73, 78, 68, 69, 88, 157, 129, 32, 2, 70, 73, 5, 153, 255, 31, 6, - 32, 68, 73, 86, 73, 68, 6, 26, 84, 147, 131, 21, 32, 5, 209, 136, 10, 6, + 82, 32, 210, 3, 79, 62, 80, 90, 82, 141, 212, 27, 4, 32, 83, 76, 73, 6, + 52, 5, 73, 78, 68, 69, 88, 169, 133, 33, 2, 70, 73, 5, 165, 131, 33, 6, + 32, 68, 73, 86, 73, 68, 6, 26, 84, 227, 190, 21, 32, 5, 181, 156, 10, 6, 32, 73, 78, 83, 69, 82, 98, 196, 1, 2, 67, 45, 38, 76, 22, 77, 50, 78, - 38, 83, 46, 84, 22, 85, 230, 213, 3, 65, 2, 68, 2, 69, 2, 71, 2, 75, 2, - 80, 214, 208, 25, 82, 218, 220, 1, 73, 206, 134, 2, 66, 2, 79, 2, 81, 3, - 88, 4, 226, 163, 26, 49, 159, 148, 3, 51, 7, 139, 215, 3, 68, 11, 11, 66, - 9, 226, 138, 33, 50, 2, 51, 3, 52, 9, 190, 138, 33, 68, 2, 71, 3, 78, 13, - 162, 214, 3, 72, 2, 84, 251, 179, 29, 83, 7, 247, 213, 3, 84, 13, 11, 85, - 11, 11, 85, 9, 194, 137, 33, 50, 2, 51, 3, 85, 4, 148, 128, 32, 6, 85, - 83, 69, 76, 32, 72, 139, 137, 1, 78, 4, 152, 230, 26, 6, 32, 83, 84, 82, - 69, 65, 173, 164, 5, 7, 69, 78, 84, 82, 89, 32, 83, 4, 198, 240, 26, 73, - 155, 251, 5, 79, 9, 29, 5, 32, 70, 65, 67, 69, 7, 33, 6, 32, 87, 73, 84, - 72, 32, 4, 132, 229, 6, 2, 84, 69, 157, 132, 24, 6, 87, 82, 89, 32, 83, + 38, 83, 46, 84, 22, 85, 186, 213, 3, 65, 2, 68, 2, 69, 2, 71, 2, 75, 2, + 80, 130, 230, 26, 82, 238, 200, 1, 73, 222, 137, 2, 66, 2, 79, 2, 81, 3, + 88, 4, 174, 242, 26, 49, 219, 167, 3, 51, 7, 223, 214, 3, 68, 11, 11, 66, + 9, 134, 143, 34, 50, 2, 51, 3, 52, 9, 226, 142, 34, 68, 2, 71, 3, 78, 13, + 246, 213, 3, 72, 2, 84, 203, 184, 30, 83, 7, 203, 213, 3, 84, 13, 11, 85, + 11, 11, 85, 9, 230, 141, 34, 50, 2, 51, 3, 85, 4, 160, 132, 33, 6, 85, + 83, 69, 76, 32, 72, 163, 137, 1, 78, 4, 224, 189, 27, 6, 32, 83, 84, 82, + 69, 65, 249, 208, 5, 7, 69, 78, 84, 82, 89, 32, 83, 4, 142, 200, 27, 73, + 247, 167, 6, 79, 9, 29, 5, 32, 70, 65, 67, 69, 7, 33, 6, 32, 87, 73, 84, + 72, 32, 4, 208, 235, 6, 2, 84, 69, 129, 251, 24, 6, 87, 82, 89, 32, 83, 77, 108, 88, 16, 67, 65, 83, 73, 65, 78, 32, 65, 76, 66, 65, 78, 73, 65, - 78, 32, 159, 172, 29, 84, 106, 60, 7, 76, 69, 84, 84, 69, 82, 32, 189, - 160, 27, 2, 67, 73, 104, 174, 2, 65, 34, 67, 146, 1, 68, 78, 69, 34, 71, + 78, 32, 235, 137, 30, 84, 106, 60, 7, 76, 69, 84, 84, 69, 82, 32, 161, + 250, 27, 2, 67, 73, 104, 174, 2, 65, 34, 67, 146, 1, 68, 78, 69, 34, 71, 46, 73, 46, 74, 34, 75, 34, 76, 34, 80, 34, 83, 74, 84, 62, 89, 46, 90, - 140, 189, 16, 2, 81, 65, 166, 173, 6, 77, 208, 156, 6, 3, 86, 69, 89, - 182, 189, 2, 70, 220, 18, 3, 78, 79, 87, 146, 19, 82, 234, 21, 88, 150, - 46, 79, 202, 26, 66, 225, 24, 3, 72, 69, 89, 4, 194, 202, 32, 79, 167, - 28, 76, 16, 34, 65, 34, 72, 49, 2, 89, 65, 4, 250, 178, 32, 89, 215, 79, - 82, 8, 186, 205, 27, 65, 174, 163, 5, 79, 203, 17, 73, 4, 254, 129, 33, - 87, 3, 89, 8, 42, 90, 134, 186, 31, 89, 139, 171, 1, 65, 4, 202, 245, 31, - 89, 167, 122, 65, 4, 194, 177, 32, 89, 215, 79, 66, 4, 136, 214, 31, 2, - 72, 69, 151, 154, 1, 73, 6, 166, 198, 31, 82, 210, 106, 87, 251, 76, 78, - 4, 134, 244, 31, 72, 191, 14, 65, 4, 238, 129, 32, 73, 195, 69, 65, 4, - 138, 176, 32, 65, 159, 51, 89, 4, 142, 1, 73, 223, 174, 32, 69, 8, 34, - 72, 229, 251, 32, 2, 69, 89, 6, 142, 220, 19, 65, 167, 145, 13, 79, 6, - 38, 73, 198, 242, 31, 89, 159, 78, 65, 2, 223, 197, 32, 87, 4, 132, 184, - 32, 2, 65, 89, 1, 2, 79, 87, 6, 142, 223, 15, 72, 199, 177, 4, 65, 16, - 72, 2, 68, 73, 32, 2, 78, 84, 180, 155, 31, 3, 76, 84, 73, 207, 77, 82, - 4, 162, 246, 31, 32, 199, 100, 76, 8, 32, 2, 82, 69, 227, 245, 31, 32, 6, - 44, 5, 76, 73, 78, 69, 32, 207, 226, 4, 32, 4, 174, 154, 21, 76, 255, - 252, 9, 79, 252, 5, 102, 65, 170, 17, 69, 134, 8, 73, 150, 1, 79, 140, - 152, 4, 6, 82, 73, 83, 84, 77, 65, 195, 222, 18, 85, 198, 2, 66, 73, 32, - 4, 75, 77, 65, 32, 148, 6, 2, 77, 32, 131, 8, 82, 4, 226, 188, 32, 78, - 211, 61, 82, 142, 1, 156, 1, 7, 76, 69, 84, 84, 69, 82, 32, 142, 3, 83, - 98, 86, 170, 255, 22, 68, 170, 171, 7, 81, 236, 90, 5, 77, 65, 65, 89, - 89, 144, 174, 1, 2, 65, 85, 3, 79, 76, 202, 1, 68, 54, 78, 54, 84, 54, - 89, 142, 162, 28, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 76, 2, 80, 182, 228, - 2, 72, 2, 77, 2, 82, 2, 83, 2, 86, 2, 87, 146, 237, 1, 65, 186, 2, 69, 2, - 73, 3, 85, 8, 166, 163, 28, 68, 182, 228, 2, 72, 147, 237, 1, 65, 8, 166, - 135, 31, 71, 2, 78, 2, 89, 147, 237, 1, 65, 8, 190, 162, 28, 84, 182, - 228, 2, 72, 147, 237, 1, 65, 4, 190, 134, 31, 89, 147, 237, 1, 65, 8, 40, - 4, 73, 71, 78, 32, 195, 188, 21, 69, 6, 142, 143, 29, 67, 226, 182, 1, - 86, 223, 234, 1, 65, 26, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, - 203, 194, 30, 73, 24, 238, 139, 29, 65, 182, 25, 85, 210, 200, 1, 69, 2, + 128, 241, 16, 2, 81, 65, 214, 192, 6, 77, 148, 173, 6, 3, 86, 69, 89, + 178, 232, 2, 70, 128, 19, 3, 78, 79, 87, 150, 20, 82, 242, 21, 88, 154, + 46, 79, 202, 26, 66, 237, 24, 3, 72, 69, 89, 4, 218, 206, 33, 79, 179, + 28, 76, 16, 34, 65, 34, 72, 49, 2, 89, 65, 4, 146, 183, 33, 89, 227, 79, + 82, 8, 242, 166, 28, 65, 154, 206, 5, 79, 203, 17, 73, 4, 162, 134, 34, + 87, 3, 89, 8, 42, 90, 222, 187, 32, 89, 215, 173, 1, 65, 4, 214, 249, 32, + 89, 191, 122, 65, 4, 218, 181, 33, 89, 227, 79, 66, 4, 136, 217, 32, 2, + 72, 69, 187, 155, 1, 73, 6, 138, 201, 32, 82, 134, 108, 87, 135, 77, 78, + 4, 146, 248, 32, 72, 199, 14, 65, 4, 130, 134, 33, 73, 199, 69, 65, 4, + 162, 180, 33, 65, 171, 51, 89, 4, 142, 1, 73, 247, 178, 33, 69, 8, 34, + 72, 137, 128, 34, 2, 69, 89, 6, 182, 151, 20, 65, 163, 218, 13, 79, 6, + 38, 73, 210, 246, 32, 89, 171, 78, 65, 2, 247, 201, 33, 87, 4, 156, 188, + 33, 2, 65, 89, 1, 2, 79, 87, 6, 130, 147, 16, 72, 219, 185, 4, 65, 16, + 72, 2, 68, 73, 32, 2, 78, 84, 212, 156, 32, 3, 76, 84, 73, 187, 80, 82, + 4, 174, 250, 32, 32, 223, 100, 76, 8, 32, 2, 82, 69, 239, 249, 32, 32, 6, + 44, 5, 76, 73, 78, 69, 32, 147, 226, 4, 32, 4, 178, 214, 21, 76, 155, + 194, 10, 79, 252, 5, 102, 65, 170, 17, 69, 134, 8, 73, 150, 1, 79, 244, + 191, 13, 6, 82, 73, 83, 84, 77, 65, 139, 130, 10, 85, 198, 2, 66, 73, 32, + 4, 75, 77, 65, 32, 148, 6, 2, 77, 32, 131, 8, 82, 4, 250, 192, 33, 78, + 223, 61, 82, 142, 1, 156, 1, 7, 76, 69, 84, 84, 69, 82, 32, 142, 3, 83, + 98, 86, 214, 202, 23, 68, 194, 221, 7, 81, 200, 94, 5, 77, 65, 65, 89, + 89, 136, 177, 1, 2, 65, 85, 3, 79, 76, 202, 1, 68, 54, 78, 54, 84, 54, + 89, 166, 250, 28, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 76, 2, 80, 190, 141, + 3, 72, 2, 77, 2, 82, 2, 83, 2, 86, 2, 87, 150, 240, 1, 65, 186, 2, 69, 2, + 73, 3, 85, 8, 190, 251, 28, 68, 190, 141, 3, 72, 151, 240, 1, 65, 8, 198, + 136, 32, 71, 2, 78, 2, 89, 151, 240, 1, 65, 8, 214, 250, 28, 84, 190, + 141, 3, 72, 151, 240, 1, 65, 4, 222, 135, 32, 89, 151, 240, 1, 65, 8, 40, + 4, 73, 71, 78, 32, 155, 249, 21, 69, 6, 154, 229, 29, 67, 154, 222, 1, + 86, 179, 241, 1, 65, 26, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, + 143, 192, 31, 73, 24, 194, 158, 30, 65, 250, 6, 85, 206, 201, 1, 69, 2, 73, 3, 79, 166, 1, 236, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 112, 7, 76, 69, 84, 84, 69, 82, 32, 152, 4, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 60, 11, 86, 79, 87, 69, 76, - 32, 83, 73, 71, 78, 32, 175, 254, 30, 68, 14, 72, 6, 70, 73, 78, 65, 76, - 32, 138, 239, 32, 76, 2, 82, 2, 87, 3, 89, 6, 234, 240, 32, 78, 86, 72, + 32, 83, 73, 71, 78, 32, 207, 255, 31, 68, 14, 72, 6, 70, 73, 78, 65, 76, + 32, 174, 243, 33, 76, 2, 82, 2, 87, 3, 89, 6, 142, 245, 33, 78, 86, 72, 3, 77, 104, 132, 2, 6, 70, 73, 78, 65, 76, 32, 110, 78, 50, 77, 78, 80, - 146, 195, 11, 66, 198, 231, 5, 68, 234, 191, 2, 83, 166, 252, 10, 65, - 152, 191, 1, 2, 67, 72, 2, 71, 2, 74, 2, 75, 2, 84, 254, 68, 72, 2, 76, - 2, 82, 2, 86, 2, 89, 186, 2, 69, 2, 73, 2, 79, 3, 85, 22, 254, 246, 30, - 78, 198, 186, 1, 83, 194, 60, 67, 146, 1, 71, 2, 75, 2, 76, 2, 80, 2, 82, - 2, 84, 3, 89, 14, 46, 71, 34, 72, 134, 215, 32, 85, 215, 22, 65, 4, 162, - 215, 32, 85, 215, 22, 65, 6, 130, 215, 32, 85, 158, 20, 74, 187, 2, 65, - 6, 242, 234, 32, 72, 2, 80, 187, 2, 65, 8, 190, 136, 11, 83, 246, 142, - 12, 68, 45, 4, 84, 82, 73, 80, 20, 238, 135, 29, 65, 142, 222, 1, 73, - 186, 198, 1, 79, 2, 85, 191, 44, 69, 14, 72, 7, 65, 67, 84, 69, 82, 32, - 84, 49, 7, 84, 32, 87, 73, 84, 72, 32, 8, 140, 223, 9, 3, 65, 66, 85, - 211, 245, 22, 73, 6, 128, 1, 13, 85, 80, 87, 65, 82, 68, 83, 32, 84, 82, - 69, 78, 68, 149, 214, 24, 12, 68, 79, 87, 78, 87, 65, 82, 68, 83, 32, 84, - 82, 5, 233, 234, 6, 6, 32, 65, 78, 68, 32, 89, 234, 2, 80, 2, 67, 75, 82, - 69, 90, 82, 224, 247, 4, 2, 83, 84, 241, 219, 14, 2, 81, 85, 6, 56, 8, - 69, 82, 32, 66, 79, 65, 82, 68, 151, 167, 32, 32, 5, 223, 247, 29, 32, 4, - 184, 215, 26, 4, 83, 69, 32, 87, 177, 231, 4, 9, 82, 73, 78, 71, 32, 77, + 226, 215, 11, 66, 138, 135, 6, 68, 206, 134, 12, 83, 194, 130, 2, 65, + 156, 194, 1, 2, 67, 72, 2, 71, 2, 74, 2, 75, 2, 84, 138, 69, 72, 2, 76, + 2, 82, 2, 86, 2, 89, 186, 2, 69, 2, 73, 2, 79, 3, 85, 22, 158, 248, 31, + 78, 190, 189, 1, 83, 206, 60, 67, 146, 1, 71, 2, 75, 2, 76, 2, 80, 2, 82, + 2, 84, 3, 89, 14, 46, 71, 34, 72, 170, 219, 33, 85, 215, 22, 65, 4, 198, + 219, 33, 85, 215, 22, 65, 6, 166, 219, 33, 85, 158, 20, 74, 187, 2, 65, + 6, 150, 239, 33, 72, 2, 80, 187, 2, 65, 8, 246, 156, 11, 83, 186, 247, + 18, 68, 45, 4, 84, 82, 73, 80, 20, 158, 157, 30, 65, 242, 201, 1, 73, + 190, 201, 1, 79, 2, 85, 203, 44, 69, 14, 72, 7, 65, 67, 84, 69, 82, 32, + 84, 49, 7, 84, 32, 87, 73, 84, 72, 32, 8, 240, 242, 9, 3, 65, 66, 85, + 147, 230, 23, 73, 6, 128, 1, 13, 85, 80, 87, 65, 82, 68, 83, 32, 84, 82, + 69, 78, 68, 217, 160, 25, 12, 68, 79, 87, 78, 87, 65, 82, 68, 83, 32, 84, + 82, 5, 225, 242, 6, 6, 32, 65, 78, 68, 32, 89, 234, 2, 80, 2, 67, 75, 82, + 69, 90, 82, 156, 247, 4, 2, 83, 84, 229, 151, 15, 2, 81, 85, 6, 56, 8, + 69, 82, 32, 66, 79, 65, 82, 68, 175, 171, 33, 32, 5, 211, 244, 30, 32, 4, + 140, 177, 27, 4, 83, 69, 32, 87, 229, 144, 5, 9, 82, 73, 78, 71, 32, 77, 69, 71, 65, 220, 2, 40, 5, 79, 75, 69, 69, 32, 143, 5, 82, 216, 2, 46, 76, 1, 7, 83, 77, 65, 76, 76, 32, 76, 172, 1, 33, 6, 69, 84, 84, 69, 82, 32, 172, 1, 170, 1, 68, 74, 72, 74, 78, 70, 83, 62, 84, 54, 71, 2, 76, 2, - 77, 0, 2, 81, 85, 2, 87, 2, 89, 158, 224, 32, 75, 186, 2, 65, 2, 69, 2, - 73, 2, 79, 2, 85, 3, 86, 14, 222, 226, 32, 76, 186, 2, 65, 2, 69, 2, 73, - 2, 79, 2, 85, 3, 86, 14, 150, 226, 32, 78, 186, 2, 65, 2, 69, 2, 73, 2, - 79, 2, 85, 3, 86, 14, 166, 240, 28, 65, 226, 243, 3, 69, 2, 73, 2, 79, 2, - 85, 3, 86, 15, 194, 227, 32, 65, 2, 69, 2, 73, 2, 79, 2, 85, 3, 86, 30, - 50, 76, 2, 83, 214, 226, 32, 65, 2, 69, 3, 73, 12, 210, 226, 32, 65, 2, - 69, 2, 73, 2, 79, 2, 85, 3, 86, 4, 48, 6, 89, 32, 66, 76, 79, 83, 135, - 206, 31, 73, 2, 255, 208, 32, 83, 12, 96, 2, 76, 68, 174, 222, 7, 32, - 232, 239, 2, 2, 80, 77, 228, 193, 20, 2, 67, 75, 151, 129, 1, 82, 5, 213, - 128, 28, 7, 82, 69, 78, 32, 67, 82, 79, 60, 92, 8, 82, 65, 83, 77, 73, - 65, 78, 32, 222, 253, 4, 80, 249, 203, 11, 5, 67, 79, 76, 65, 84, 56, 52, - 7, 76, 69, 84, 84, 69, 82, 32, 243, 134, 21, 78, 42, 224, 1, 6, 67, 85, - 82, 76, 69, 68, 30, 83, 202, 131, 21, 68, 34, 76, 254, 194, 1, 82, 134, - 212, 2, 65, 50, 71, 90, 90, 98, 89, 154, 193, 1, 72, 234, 5, 75, 130, 76, - 66, 190, 173, 4, 78, 254, 1, 84, 2, 87, 202, 103, 80, 171, 4, 77, 2, 161, - 223, 31, 2, 32, 87, 6, 176, 131, 21, 6, 77, 65, 76, 76, 32, 65, 172, 210, - 5, 2, 65, 77, 139, 136, 5, 72, 232, 4, 66, 78, 20, 2, 82, 67, 233, 40, 7, - 84, 89, 83, 67, 65, 80, 69, 2, 247, 186, 32, 69, 226, 4, 28, 2, 76, 69, - 239, 39, 85, 220, 4, 30, 32, 181, 6, 2, 68, 32, 24, 244, 1, 5, 87, 73, - 84, 72, 32, 161, 133, 18, 49, 68, 73, 86, 73, 68, 69, 68, 32, 66, 89, 32, + 77, 0, 2, 81, 85, 2, 87, 2, 89, 194, 228, 33, 75, 186, 2, 65, 2, 69, 2, + 73, 2, 79, 2, 85, 3, 86, 14, 130, 231, 33, 76, 186, 2, 65, 2, 69, 2, 73, + 2, 79, 2, 85, 3, 86, 14, 186, 230, 33, 78, 186, 2, 65, 2, 69, 2, 73, 2, + 79, 2, 85, 3, 86, 14, 142, 200, 29, 65, 158, 160, 4, 69, 2, 73, 2, 79, 2, + 85, 3, 86, 15, 230, 231, 33, 65, 2, 69, 2, 73, 2, 79, 2, 85, 3, 86, 30, + 50, 76, 2, 83, 250, 230, 33, 65, 2, 69, 3, 73, 12, 246, 230, 33, 65, 2, + 69, 2, 73, 2, 79, 2, 85, 3, 86, 4, 48, 6, 89, 32, 66, 76, 79, 83, 147, + 210, 32, 73, 2, 163, 213, 33, 83, 12, 96, 2, 76, 68, 254, 235, 7, 32, + 204, 246, 2, 2, 80, 77, 224, 174, 21, 2, 67, 75, 255, 131, 1, 82, 5, 237, + 216, 28, 7, 82, 69, 78, 32, 67, 82, 79, 60, 92, 8, 82, 65, 83, 77, 73, + 65, 78, 32, 174, 254, 4, 80, 189, 255, 11, 5, 67, 79, 76, 65, 84, 56, 52, + 7, 76, 69, 84, 84, 69, 82, 32, 247, 194, 21, 78, 42, 224, 1, 6, 67, 85, + 82, 76, 69, 68, 30, 83, 206, 191, 21, 68, 34, 76, 158, 206, 1, 82, 178, + 215, 2, 65, 50, 71, 90, 90, 98, 89, 158, 208, 1, 72, 234, 5, 75, 198, 75, + 66, 178, 216, 4, 78, 134, 2, 84, 2, 87, 218, 103, 80, 171, 4, 77, 2, 181, + 227, 32, 2, 32, 87, 6, 180, 191, 21, 6, 77, 65, 76, 76, 32, 65, 252, 239, + 5, 2, 65, 77, 195, 178, 5, 72, 232, 4, 66, 78, 20, 2, 82, 67, 201, 40, 7, + 84, 89, 83, 67, 65, 80, 69, 2, 155, 191, 33, 69, 226, 4, 28, 2, 76, 69, + 207, 39, 85, 220, 4, 30, 32, 185, 6, 2, 68, 32, 24, 244, 1, 5, 87, 73, + 84, 72, 32, 185, 186, 18, 49, 68, 73, 86, 73, 68, 69, 68, 32, 66, 89, 32, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 66, 65, 82, 32, 65, 78, 68, 32, 84, 79, 80, 32, 72, 65, 76, 70, 32, 68, 73, 86, 73, 68, 69, 68, 32, - 66, 89, 22, 142, 2, 76, 46, 83, 108, 22, 84, 87, 79, 32, 72, 79, 82, 73, + 66, 89, 22, 146, 2, 76, 46, 83, 108, 22, 84, 87, 79, 32, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 83, 84, 82, 79, 75, 69, 83, 44, 6, 85, 80, 80, 69, 82, 32, 44, 17, 65, 76, 76, 32, 66, 85, 84, 32, 85, 80, 80, 69, - 82, 32, 76, 69, 70, 238, 230, 25, 86, 190, 227, 3, 82, 231, 58, 72, 4, - 138, 204, 29, 69, 49, 4, 79, 87, 69, 82, 4, 104, 11, 77, 65, 76, 76, 32, - 67, 73, 82, 67, 76, 69, 221, 5, 10, 85, 80, 69, 82, 73, 77, 80, 79, 83, - 69, 2, 173, 225, 29, 6, 32, 84, 79, 32, 84, 72, 4, 40, 4, 82, 73, 71, 72, - 227, 202, 29, 72, 2, 245, 202, 29, 10, 84, 32, 81, 85, 65, 68, 82, 65, - 78, 84, 196, 4, 230, 2, 65, 186, 1, 66, 58, 67, 230, 1, 68, 246, 1, 72, - 250, 2, 73, 210, 10, 75, 190, 2, 76, 38, 77, 136, 1, 7, 78, 85, 77, 66, - 69, 82, 32, 192, 4, 17, 79, 80, 69, 78, 32, 67, 69, 78, 84, 82, 69, 32, - 69, 73, 71, 72, 84, 54, 80, 114, 82, 50, 84, 78, 87, 224, 228, 9, 4, 90, - 69, 82, 79, 222, 253, 15, 69, 250, 129, 4, 86, 130, 65, 71, 190, 110, 83, - 199, 160, 1, 88, 6, 100, 12, 78, 84, 73, 67, 76, 79, 67, 75, 87, 73, 83, - 69, 249, 166, 29, 7, 83, 84, 69, 82, 73, 83, 75, 4, 160, 220, 8, 11, 45, - 82, 79, 84, 65, 84, 69, 68, 32, 68, 73, 211, 218, 22, 32, 4, 32, 2, 79, - 76, 139, 244, 30, 85, 2, 175, 225, 13, 68, 16, 60, 4, 82, 79, 83, 83, - 210, 2, 32, 142, 205, 32, 67, 3, 68, 10, 26, 32, 219, 193, 1, 73, 8, 64, - 6, 70, 79, 82, 77, 69, 69, 201, 226, 30, 4, 80, 79, 77, 77, 7, 33, 6, 32, - 87, 73, 84, 72, 32, 4, 254, 177, 30, 70, 251, 78, 84, 30, 34, 73, 70, 79, - 183, 232, 30, 65, 24, 152, 188, 4, 8, 86, 73, 83, 73, 79, 78, 32, 83, - 247, 163, 26, 71, 4, 64, 10, 76, 76, 65, 82, 32, 83, 73, 71, 78, 32, 227, - 162, 29, 84, 2, 197, 230, 18, 13, 87, 73, 84, 72, 32, 79, 86, 69, 82, 76, - 65, 73, 68, 64, 212, 1, 6, 65, 78, 71, 85, 76, 32, 184, 151, 12, 20, 79, + 82, 32, 76, 69, 70, 182, 181, 26, 86, 178, 136, 4, 82, 231, 249, 1, 72, + 4, 198, 191, 30, 69, 49, 4, 79, 87, 69, 82, 4, 104, 11, 77, 65, 76, 76, + 32, 67, 73, 82, 67, 76, 69, 221, 5, 10, 85, 80, 69, 82, 73, 77, 80, 79, + 83, 69, 2, 153, 217, 30, 6, 32, 84, 79, 32, 84, 72, 4, 40, 4, 82, 73, 71, + 72, 159, 190, 30, 72, 2, 177, 190, 30, 10, 84, 32, 81, 85, 65, 68, 82, + 65, 78, 84, 196, 4, 230, 2, 65, 186, 1, 66, 58, 67, 230, 1, 68, 246, 1, + 72, 230, 2, 73, 194, 10, 75, 190, 2, 76, 38, 77, 136, 1, 7, 78, 85, 77, + 66, 69, 82, 32, 192, 4, 17, 79, 80, 69, 78, 32, 67, 69, 78, 84, 82, 69, + 32, 69, 73, 71, 72, 84, 54, 80, 114, 82, 50, 84, 78, 87, 232, 248, 9, 4, + 90, 69, 82, 79, 234, 183, 16, 69, 246, 177, 4, 86, 158, 68, 71, 190, 113, + 83, 223, 160, 1, 88, 6, 100, 12, 78, 84, 73, 67, 76, 79, 67, 75, 87, 73, + 83, 69, 213, 149, 30, 7, 83, 84, 69, 82, 73, 83, 75, 4, 200, 238, 8, 11, + 45, 82, 79, 84, 65, 84, 69, 68, 32, 68, 73, 179, 204, 23, 32, 4, 32, 2, + 79, 76, 167, 245, 31, 85, 2, 131, 143, 14, 68, 16, 60, 4, 82, 79, 83, 83, + 210, 2, 32, 174, 209, 33, 67, 3, 68, 10, 26, 32, 155, 193, 1, 73, 8, 64, + 6, 70, 79, 82, 77, 69, 69, 229, 227, 31, 4, 80, 79, 77, 77, 7, 33, 6, 32, + 87, 73, 84, 72, 32, 4, 214, 175, 31, 70, 207, 82, 84, 30, 34, 73, 70, 79, + 211, 233, 31, 65, 24, 216, 187, 4, 8, 86, 73, 83, 73, 79, 78, 32, 83, + 211, 165, 27, 71, 4, 64, 10, 76, 76, 65, 82, 32, 83, 73, 71, 78, 32, 191, + 145, 30, 84, 2, 145, 156, 19, 13, 87, 73, 84, 72, 32, 79, 86, 69, 82, 76, + 65, 73, 68, 64, 192, 1, 6, 65, 78, 71, 85, 76, 32, 204, 191, 12, 20, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 66, 65, 82, 32, 87, 73, 84, 72, 32, - 78, 152, 181, 9, 4, 69, 65, 86, 89, 189, 158, 6, 8, 85, 77, 65, 78, 32, - 70, 73, 71, 58, 110, 67, 160, 128, 20, 5, 73, 69, 85, 78, 71, 42, 72, 30, - 75, 66, 77, 34, 78, 34, 80, 62, 82, 30, 83, 27, 84, 8, 254, 255, 19, 72, - 157, 3, 4, 73, 69, 85, 67, 116, 220, 1, 9, 68, 69, 79, 71, 82, 65, 80, - 72, 32, 212, 8, 27, 84, 65, 76, 73, 67, 32, 76, 65, 84, 73, 78, 32, 67, - 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 165, 176, 26, 9, - 78, 70, 79, 82, 77, 65, 84, 73, 79, 110, 178, 1, 65, 110, 67, 90, 69, 86, - 70, 62, 72, 38, 75, 70, 76, 50, 77, 86, 78, 62, 81, 30, 82, 74, 83, 246, - 128, 20, 84, 50, 87, 146, 204, 1, 73, 172, 24, 2, 80, 82, 231, 193, 9, - 79, 8, 130, 129, 20, 76, 180, 132, 1, 3, 67, 67, 69, 196, 212, 6, 3, 84, - 84, 69, 245, 17, 5, 68, 86, 65, 78, 84, 8, 26, 79, 159, 208, 29, 69, 6, - 168, 180, 10, 2, 82, 82, 146, 204, 9, 78, 175, 179, 12, 80, 8, 206, 128, - 20, 78, 246, 253, 8, 65, 148, 131, 3, 5, 88, 67, 69, 76, 76, 219, 24, 73, - 10, 218, 128, 20, 73, 128, 169, 5, 2, 69, 77, 131, 162, 6, 79, 4, 214, - 203, 28, 73, 203, 251, 2, 65, 4, 240, 141, 7, 8, 73, 78, 68, 69, 82, 71, - 65, 82, 199, 233, 21, 79, 6, 234, 255, 19, 65, 130, 197, 11, 79, 179, 83, - 69, 8, 38, 69, 130, 161, 31, 65, 199, 81, 79, 4, 128, 169, 31, 3, 68, 73, - 67, 251, 15, 84, 6, 26, 73, 139, 170, 31, 65, 4, 162, 151, 32, 71, 231, - 19, 78, 2, 237, 218, 27, 2, 85, 69, 8, 26, 69, 195, 150, 32, 73, 6, 242, - 254, 19, 83, 205, 225, 7, 2, 76, 73, 22, 90, 69, 38, 85, 164, 230, 13, 2, - 67, 72, 166, 152, 6, 79, 22, 80, 34, 84, 211, 180, 11, 73, 4, 178, 132, - 10, 67, 215, 234, 20, 86, 6, 198, 255, 19, 80, 180, 177, 4, 3, 73, 84, - 65, 223, 142, 8, 78, 4, 150, 191, 32, 67, 3, 82, 98, 116, 8, 65, 84, 65, - 75, 65, 78, 65, 32, 133, 1, 16, 79, 82, 69, 65, 78, 32, 67, 72, 65, 82, - 65, 67, 84, 69, 82, 32, 94, 194, 208, 10, 72, 2, 75, 2, 77, 2, 78, 2, 82, - 2, 83, 2, 84, 126, 87, 46, 89, 154, 236, 21, 65, 2, 69, 2, 73, 2, 79, 3, - 85, 4, 216, 214, 15, 3, 74, 85, 69, 165, 199, 16, 4, 67, 72, 65, 77, 106, - 166, 211, 26, 65, 203, 218, 3, 69, 4, 100, 19, 85, 76, 84, 73, 80, 76, - 73, 67, 65, 84, 73, 79, 78, 32, 83, 73, 71, 78, 32, 191, 184, 30, 73, 2, - 129, 84, 4, 87, 73, 84, 72, 98, 50, 69, 46, 70, 98, 83, 94, 84, 151, 251, - 19, 78, 6, 180, 1, 3, 73, 71, 72, 191, 222, 24, 76, 30, 28, 3, 73, 70, - 84, 35, 79, 6, 214, 1, 89, 203, 231, 30, 69, 24, 142, 2, 82, 255, 250, - 19, 85, 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 4, 11, 84, 4, 104, 2, 89, - 32, 191, 231, 30, 69, 52, 56, 2, 69, 78, 32, 4, 72, 73, 82, 84, 29, 2, - 87, 69, 5, 11, 32, 2, 135, 252, 24, 79, 24, 74, 89, 223, 230, 30, 69, 24, - 26, 78, 163, 187, 31, 76, 22, 17, 2, 84, 89, 23, 11, 32, 20, 72, 2, 79, - 78, 218, 201, 30, 70, 30, 83, 42, 84, 190, 83, 78, 211, 110, 69, 4, 226, - 250, 24, 32, 151, 188, 7, 69, 2, 213, 163, 30, 8, 32, 80, 79, 73, 78, 84, - 69, 68, 8, 184, 146, 17, 3, 79, 83, 84, 246, 223, 6, 65, 188, 189, 2, 8, - 69, 82, 80, 69, 78, 68, 73, 67, 203, 131, 4, 76, 4, 128, 139, 29, 3, 73, - 78, 71, 255, 151, 1, 69, 6, 52, 7, 82, 73, 65, 78, 71, 76, 69, 151, 224, - 25, 73, 5, 163, 204, 21, 32, 6, 44, 5, 72, 73, 84, 69, 32, 135, 180, 32, - 90, 4, 206, 215, 30, 66, 211, 24, 83, 6, 226, 198, 26, 77, 252, 138, 4, - 6, 76, 65, 84, 73, 79, 78, 233, 158, 1, 3, 83, 32, 84, 5, 233, 240, 30, - 6, 32, 65, 84, 32, 68, 85, 154, 18, 192, 1, 24, 67, 79, 77, 80, 65, 84, - 73, 66, 73, 76, 73, 84, 89, 32, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, - 144, 3, 8, 82, 65, 68, 73, 67, 65, 76, 32, 245, 17, 7, 83, 84, 82, 79, - 75, 69, 32, 236, 15, 24, 2, 50, 70, 75, 70, 188, 8, 34, 65, 194, 225, 10, - 56, 3, 57, 60, 202, 1, 49, 143, 182, 18, 48, 176, 7, 26, 65, 131, 225, - 10, 57, 176, 3, 134, 1, 54, 142, 182, 18, 48, 2, 49, 2, 50, 2, 51, 2, 52, - 2, 53, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 231, 238, 8, 68, 28, - 130, 175, 32, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, - 2, 57, 2, 65, 2, 66, 2, 67, 3, 68, 230, 1, 210, 1, 66, 126, 67, 226, 4, - 68, 62, 69, 50, 70, 34, 71, 50, 72, 86, 74, 154, 1, 76, 62, 77, 130, 1, - 80, 46, 82, 78, 83, 182, 3, 84, 82, 87, 200, 197, 10, 2, 78, 69, 252, 16, - 4, 75, 78, 73, 70, 159, 225, 11, 79, 14, 74, 79, 192, 229, 10, 4, 82, 85, - 83, 72, 154, 218, 4, 65, 179, 139, 13, 76, 6, 230, 140, 10, 76, 170, 136, - 22, 78, 215, 22, 88, 56, 104, 12, 45, 83, 73, 77, 80, 76, 73, 70, 73, 69, - 68, 32, 162, 3, 73, 50, 76, 234, 168, 31, 79, 175, 78, 72, 44, 114, 69, - 38, 70, 50, 71, 38, 76, 34, 83, 94, 84, 214, 146, 14, 87, 42, 68, 246, - 149, 16, 66, 234, 117, 72, 135, 2, 67, 4, 222, 216, 30, 86, 203, 180, 1, - 65, 6, 146, 196, 30, 73, 214, 211, 1, 76, 235, 16, 82, 4, 234, 195, 22, - 79, 235, 160, 8, 65, 4, 194, 167, 31, 69, 175, 48, 79, 10, 174, 204, 6, - 73, 242, 253, 16, 65, 142, 194, 1, 72, 232, 249, 6, 3, 80, 69, 69, 215, - 11, 69, 6, 214, 10, 85, 176, 155, 10, 2, 65, 78, 143, 255, 3, 79, 4, 156, - 150, 20, 3, 86, 73, 76, 195, 255, 11, 84, 4, 214, 161, 17, 73, 171, 241, - 13, 79, 6, 220, 201, 23, 2, 73, 86, 194, 226, 2, 69, 227, 249, 5, 79, 10, - 196, 6, 2, 65, 84, 254, 136, 32, 87, 3, 89, 4, 198, 133, 31, 73, 247, 79, - 79, 8, 244, 5, 4, 82, 65, 83, 83, 255, 215, 28, 72, 10, 48, 2, 69, 65, - 218, 198, 27, 79, 175, 208, 3, 65, 6, 186, 8, 82, 171, 156, 32, 68, 10, - 72, 12, 45, 83, 73, 77, 80, 76, 73, 70, 73, 69, 68, 32, 135, 151, 30, 65, - 8, 42, 84, 234, 142, 14, 68, 251, 167, 16, 69, 4, 194, 6, 85, 191, 154, - 14, 79, 12, 132, 221, 10, 3, 79, 78, 71, 197, 176, 15, 3, 65, 77, 69, 14, - 18, 69, 35, 79, 4, 174, 134, 32, 65, 159, 27, 83, 10, 148, 220, 10, 3, - 85, 78, 68, 158, 236, 12, 84, 166, 150, 7, 82, 147, 116, 79, 6, 200, 219, - 10, 2, 65, 87, 143, 243, 14, 69, 8, 34, 65, 233, 252, 27, 2, 69, 80, 6, - 202, 209, 31, 73, 214, 79, 77, 3, 80, 38, 122, 69, 90, 73, 186, 1, 78, - 196, 1, 4, 80, 73, 82, 73, 132, 214, 10, 4, 77, 65, 76, 76, 250, 222, 11, - 72, 219, 151, 9, 85, 8, 40, 4, 67, 79, 78, 68, 183, 150, 31, 65, 6, 11, - 32, 6, 230, 178, 30, 84, 203, 83, 79, 12, 60, 9, 77, 80, 76, 73, 70, 73, - 69, 68, 32, 207, 155, 32, 76, 10, 34, 72, 50, 87, 159, 161, 10, 89, 4, - 184, 139, 10, 3, 65, 76, 70, 251, 180, 17, 79, 4, 130, 194, 6, 65, 215, - 183, 21, 72, 6, 192, 1, 2, 79, 85, 211, 236, 31, 65, 8, 58, 85, 174, 142, - 24, 65, 230, 199, 1, 72, 135, 143, 3, 73, 2, 243, 187, 23, 82, 12, 26, - 65, 49, 2, 69, 83, 8, 176, 214, 10, 2, 76, 75, 1, 3, 84, 69, 82, 4, 131, - 214, 10, 84, 72, 110, 72, 162, 1, 80, 38, 83, 226, 161, 30, 84, 172, 247, - 1, 2, 66, 88, 2, 87, 2, 88, 86, 68, 2, 78, 3, 81, 29, 50, 80, 22, 90, - 136, 154, 32, 2, 88, 87, 87, 71, 5, 155, 154, 32, 87, 19, 50, 90, 150, - 162, 30, 87, 150, 248, 1, 71, 3, 84, 9, 146, 162, 30, 90, 151, 248, 1, - 80, 9, 130, 154, 32, 68, 2, 71, 3, 90, 21, 50, 87, 30, 90, 146, 153, 32, - 71, 2, 80, 3, 84, 7, 170, 153, 32, 71, 3, 90, 7, 186, 152, 32, 87, 87, - 90, 124, 62, 65, 194, 1, 73, 114, 79, 173, 181, 20, 4, 69, 65, 82, 32, 8, - 132, 1, 2, 80, 80, 252, 177, 18, 6, 83, 83, 73, 67, 65, 76, 241, 187, 12, - 14, 77, 83, 72, 69, 76, 76, 32, 77, 79, 66, 73, 76, 69, 32, 4, 144, 130, - 19, 5, 73, 78, 71, 32, 72, 199, 226, 1, 69, 6, 48, 6, 78, 75, 73, 78, 71, - 32, 207, 255, 30, 80, 4, 234, 198, 24, 71, 229, 169, 3, 6, 66, 69, 69, - 82, 32, 77, 108, 72, 2, 67, 75, 140, 10, 2, 83, 69, 236, 3, 2, 85, 68, - 227, 154, 29, 87, 70, 64, 6, 32, 70, 65, 67, 69, 32, 237, 2, 5, 87, 73, - 83, 69, 32, 48, 58, 69, 46, 70, 36, 2, 78, 73, 2, 79, 18, 83, 51, 84, 8, - 120, 2, 76, 69, 125, 4, 73, 71, 72, 84, 8, 178, 1, 73, 25, 3, 79, 85, 82, - 4, 155, 1, 78, 8, 26, 69, 125, 2, 73, 88, 4, 57, 2, 86, 69, 16, 38, 69, - 14, 87, 41, 3, 72, 82, 69, 4, 63, 78, 8, 24, 2, 69, 76, 27, 79, 4, 11, - 86, 4, 11, 69, 4, 196, 164, 28, 2, 32, 79, 213, 131, 2, 3, 45, 84, 72, - 22, 90, 67, 58, 68, 122, 71, 48, 5, 82, 73, 71, 72, 84, 226, 2, 84, 122, - 79, 247, 135, 30, 73, 4, 196, 1, 3, 76, 79, 83, 141, 99, 4, 79, 78, 84, - 79, 2, 141, 3, 26, 79, 87, 78, 87, 65, 82, 68, 83, 32, 65, 78, 68, 32, - 85, 80, 87, 65, 82, 68, 83, 32, 79, 80, 69, 78, 32, 2, 21, 3, 65, 80, 80, - 2, 133, 4, 2, 69, 68, 6, 228, 1, 14, 32, 65, 78, 68, 32, 76, 69, 70, 84, - 32, 83, 69, 77, 73, 45, 38, 87, 65, 82, 68, 83, 32, 65, 78, 68, 32, 76, - 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 80, 69, 78, 32, 67, 73, 82, 67, - 76, 69, 32, 65, 82, 82, 79, 87, 83, 2, 129, 184, 29, 6, 67, 73, 82, 67, - 76, 69, 5, 161, 83, 15, 32, 87, 73, 84, 72, 32, 67, 73, 82, 67, 76, 69, - 68, 32, 79, 4, 82, 79, 37, 16, 82, 73, 65, 78, 71, 76, 69, 45, 72, 69, - 65, 68, 69, 68, 32, 79, 2, 69, 6, 80, 32, 83, 69, 77, 73, 2, 21, 3, 80, - 69, 78, 2, 11, 32, 2, 161, 182, 29, 4, 67, 73, 82, 67, 26, 32, 2, 68, 32, - 167, 149, 31, 32, 24, 216, 1, 2, 83, 85, 66, 85, 254, 243, 18, 77, 180, - 3, 9, 76, 79, 67, 75, 32, 87, 73, 84, 72, 222, 192, 11, 66, 145, 147, 1, - 23, 73, 78, 84, 69, 82, 83, 69, 67, 84, 73, 79, 78, 32, 87, 73, 84, 72, - 32, 83, 69, 82, 73, 70, 8, 30, 66, 1, 3, 80, 69, 82, 4, 145, 177, 24, 3, - 83, 69, 84, 6, 82, 77, 33, 16, 78, 73, 79, 78, 32, 87, 73, 84, 72, 32, - 83, 69, 82, 73, 70, 83, 2, 197, 148, 24, 3, 66, 82, 69, 5, 221, 248, 30, - 9, 32, 65, 78, 68, 32, 83, 77, 65, 83, 11, 33, 6, 32, 87, 73, 84, 72, 32, - 8, 72, 4, 84, 79, 82, 78, 190, 162, 5, 76, 146, 151, 19, 83, 215, 161, 6, - 82, 2, 155, 140, 25, 65, 226, 9, 158, 1, 67, 82, 76, 98, 77, 162, 82, 78, - 220, 5, 2, 79, 75, 58, 80, 154, 19, 82, 138, 1, 85, 186, 165, 27, 87, - 160, 242, 2, 2, 70, 70, 234, 47, 73, 171, 51, 65, 6, 26, 75, 139, 149, 4, - 79, 4, 186, 214, 8, 82, 129, 151, 22, 4, 84, 65, 73, 76, 8, 44, 2, 79, - 78, 209, 187, 24, 3, 76, 73, 83, 7, 11, 32, 4, 146, 178, 25, 69, 171, - 203, 5, 83, 158, 6, 72, 7, 66, 73, 78, 73, 78, 71, 32, 166, 80, 77, 94, - 80, 239, 149, 31, 69, 142, 6, 214, 2, 65, 174, 2, 66, 134, 1, 67, 246, - 12, 68, 174, 9, 69, 146, 2, 70, 54, 71, 140, 9, 2, 72, 79, 82, 73, 216, - 1, 2, 75, 65, 162, 1, 76, 230, 13, 77, 250, 1, 78, 78, 79, 138, 2, 80, - 174, 1, 82, 244, 4, 5, 90, 73, 71, 90, 65, 166, 2, 83, 242, 3, 84, 238, - 2, 85, 148, 1, 9, 86, 69, 82, 84, 73, 67, 65, 76, 32, 36, 2, 87, 73, 183, - 1, 88, 24, 148, 1, 4, 67, 85, 84, 69, 86, 78, 168, 185, 24, 6, 83, 84, - 69, 82, 73, 83, 201, 172, 5, 14, 76, 77, 79, 83, 84, 32, 69, 81, 85, 65, - 76, 32, 84, 79, 10, 22, 45, 167, 33, 32, 4, 180, 164, 12, 6, 71, 82, 65, - 86, 69, 45, 195, 193, 3, 77, 6, 244, 2, 4, 84, 73, 67, 76, 177, 180, 3, - 4, 78, 85, 73, 84, 12, 42, 82, 129, 215, 29, 4, 73, 78, 68, 85, 10, 48, - 3, 69, 86, 69, 197, 228, 29, 3, 73, 68, 71, 7, 162, 252, 11, 45, 183, - 218, 17, 32, 142, 1, 142, 1, 65, 34, 76, 98, 79, 116, 8, 89, 82, 73, 76, + 78, 136, 210, 9, 4, 85, 77, 65, 78, 145, 1, 4, 69, 65, 86, 89, 58, 110, + 67, 184, 188, 20, 5, 73, 69, 85, 78, 71, 42, 72, 30, 75, 66, 77, 34, 78, + 34, 80, 62, 82, 30, 83, 27, 84, 8, 150, 188, 20, 72, 157, 3, 4, 73, 69, + 85, 67, 116, 220, 1, 9, 68, 69, 79, 71, 82, 65, 80, 72, 32, 196, 8, 27, + 84, 65, 76, 73, 67, 32, 76, 65, 84, 73, 78, 32, 67, 65, 80, 73, 84, 65, + 76, 32, 76, 69, 84, 84, 69, 82, 32, 157, 138, 27, 9, 78, 70, 79, 82, 77, + 65, 84, 73, 79, 110, 174, 1, 65, 110, 67, 90, 69, 86, 70, 62, 72, 38, 75, + 70, 76, 50, 77, 86, 78, 62, 81, 30, 82, 74, 83, 162, 173, 6, 80, 242, + 143, 14, 84, 50, 87, 170, 214, 1, 73, 255, 151, 10, 79, 8, 158, 189, 20, + 76, 184, 133, 1, 3, 67, 67, 69, 204, 239, 6, 3, 84, 84, 69, 249, 17, 5, + 68, 86, 65, 78, 84, 8, 26, 79, 163, 200, 30, 69, 6, 240, 200, 10, 2, 82, + 82, 230, 243, 9, 78, 203, 251, 12, 80, 8, 234, 188, 20, 78, 222, 163, 9, + 65, 188, 165, 3, 5, 88, 67, 69, 76, 76, 231, 24, 73, 10, 246, 188, 20, + 73, 208, 184, 5, 2, 69, 77, 191, 218, 6, 79, 4, 210, 163, 29, 73, 247, + 167, 3, 65, 4, 212, 155, 7, 8, 73, 78, 68, 69, 82, 71, 65, 82, 231, 189, + 22, 79, 6, 134, 188, 20, 65, 142, 141, 12, 79, 195, 83, 69, 8, 38, 69, + 182, 162, 32, 65, 191, 84, 79, 4, 160, 173, 32, 3, 68, 73, 67, 251, 15, + 84, 6, 26, 73, 171, 174, 32, 65, 4, 218, 155, 33, 71, 231, 19, 78, 2, + 153, 179, 28, 2, 85, 69, 8, 26, 69, 251, 154, 33, 73, 6, 142, 187, 20, + 83, 221, 253, 7, 2, 76, 73, 22, 90, 69, 38, 85, 144, 148, 14, 2, 67, 72, + 214, 166, 6, 79, 22, 80, 34, 84, 215, 252, 11, 73, 4, 246, 152, 10, 67, + 215, 215, 21, 86, 6, 226, 187, 20, 80, 166, 255, 1, 73, 139, 137, 11, 78, + 4, 218, 195, 33, 67, 3, 82, 98, 116, 8, 65, 84, 65, 75, 65, 78, 65, 32, + 133, 1, 16, 79, 82, 69, 65, 78, 32, 67, 72, 65, 82, 65, 67, 84, 69, 82, + 32, 94, 170, 229, 10, 72, 2, 75, 2, 77, 2, 78, 2, 82, 2, 83, 2, 84, 126, + 87, 46, 89, 246, 219, 22, 65, 2, 69, 2, 73, 2, 79, 3, 85, 4, 228, 138, + 16, 3, 74, 85, 69, 221, 151, 17, 4, 67, 72, 65, 77, 106, 158, 173, 27, + 65, 255, 129, 4, 69, 4, 100, 19, 85, 76, 84, 73, 80, 76, 73, 67, 65, 84, + 73, 79, 78, 32, 83, 73, 71, 78, 32, 243, 185, 31, 73, 2, 157, 84, 4, 87, + 73, 84, 72, 98, 50, 69, 46, 70, 98, 83, 94, 84, 191, 183, 20, 78, 6, 180, + 1, 3, 73, 71, 72, 199, 169, 25, 76, 30, 28, 3, 73, 70, 84, 35, 79, 6, + 214, 1, 89, 155, 233, 31, 69, 24, 142, 2, 82, 167, 183, 20, 85, 8, 40, 4, + 69, 86, 69, 78, 1, 2, 73, 88, 4, 11, 84, 4, 104, 2, 89, 32, 143, 233, 31, + 69, 52, 56, 2, 69, 78, 32, 4, 72, 73, 82, 84, 29, 2, 87, 69, 5, 11, 32, + 2, 247, 198, 25, 79, 24, 74, 89, 175, 232, 31, 69, 24, 26, 78, 215, 191, + 32, 76, 22, 17, 2, 84, 89, 23, 11, 32, 20, 72, 2, 79, 78, 154, 203, 31, + 70, 30, 83, 42, 84, 170, 86, 78, 235, 110, 69, 4, 210, 197, 25, 32, 235, + 245, 7, 69, 2, 129, 165, 31, 8, 32, 80, 79, 73, 78, 84, 69, 68, 8, 132, + 199, 17, 3, 79, 83, 84, 154, 246, 6, 65, 192, 204, 2, 8, 69, 82, 80, 69, + 78, 68, 73, 67, 139, 171, 4, 76, 4, 128, 250, 29, 3, 73, 78, 71, 171, + 170, 1, 69, 6, 52, 7, 82, 73, 65, 78, 71, 76, 69, 131, 175, 26, 73, 5, + 147, 147, 22, 32, 6, 44, 5, 72, 73, 84, 69, 32, 203, 184, 33, 90, 4, 142, + 217, 31, 66, 139, 25, 83, 6, 218, 160, 27, 77, 196, 178, 4, 6, 76, 65, + 84, 73, 79, 78, 225, 161, 1, 3, 83, 32, 84, 5, 225, 242, 31, 6, 32, 65, + 84, 32, 68, 85, 158, 18, 192, 1, 24, 67, 79, 77, 80, 65, 84, 73, 66, 73, + 76, 73, 84, 89, 32, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 144, 3, 8, + 82, 65, 68, 73, 67, 65, 76, 32, 245, 17, 7, 83, 84, 82, 79, 75, 69, 32, + 236, 15, 24, 2, 50, 70, 75, 70, 188, 8, 34, 65, 162, 246, 10, 56, 3, 57, + 60, 202, 1, 49, 135, 236, 18, 48, 176, 7, 26, 65, 227, 245, 10, 57, 176, + 3, 134, 1, 54, 134, 236, 18, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, + 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 139, 146, 9, 68, 28, 198, 179, 33, + 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, + 2, 66, 2, 67, 3, 68, 230, 1, 210, 1, 66, 126, 67, 226, 4, 68, 62, 69, 50, + 70, 34, 71, 50, 72, 86, 74, 154, 1, 76, 62, 77, 130, 1, 80, 46, 82, 78, + 83, 182, 3, 84, 82, 87, 168, 218, 10, 2, 78, 69, 144, 17, 4, 75, 78, 73, + 70, 251, 151, 12, 79, 14, 74, 79, 180, 250, 10, 4, 82, 85, 83, 72, 246, + 233, 4, 65, 251, 192, 13, 76, 6, 186, 161, 10, 76, 154, 248, 22, 78, 215, + 22, 88, 56, 104, 12, 45, 83, 73, 77, 80, 76, 73, 70, 73, 69, 68, 32, 162, + 3, 73, 50, 76, 158, 173, 32, 79, 191, 78, 72, 44, 114, 69, 38, 70, 50, + 71, 38, 76, 34, 83, 94, 84, 154, 199, 14, 87, 42, 68, 234, 226, 16, 66, + 222, 120, 72, 135, 2, 67, 4, 174, 218, 31, 86, 191, 183, 1, 65, 6, 210, + 197, 31, 73, 218, 214, 1, 76, 235, 16, 82, 4, 186, 143, 23, 79, 147, 215, + 8, 65, 4, 238, 171, 32, 69, 187, 48, 79, 10, 150, 219, 6, 73, 134, 186, + 17, 65, 198, 194, 1, 72, 248, 178, 7, 3, 80, 69, 69, 215, 11, 69, 6, 214, + 10, 85, 152, 176, 10, 2, 65, 78, 203, 158, 4, 79, 4, 140, 210, 20, 3, 86, + 73, 76, 151, 200, 12, 84, 4, 170, 214, 17, 73, 131, 193, 14, 79, 6, 216, + 148, 24, 2, 73, 86, 186, 241, 2, 69, 179, 164, 6, 79, 10, 196, 6, 2, 65, + 84, 194, 141, 33, 87, 3, 89, 4, 242, 137, 32, 73, 131, 80, 79, 8, 244, 5, + 4, 82, 65, 83, 83, 143, 186, 29, 72, 10, 48, 2, 69, 65, 146, 159, 28, 79, + 163, 252, 3, 65, 6, 186, 8, 82, 239, 160, 33, 68, 10, 72, 12, 45, 83, 73, + 77, 80, 76, 73, 70, 73, 69, 68, 32, 179, 152, 31, 65, 8, 42, 84, 174, + 195, 14, 68, 247, 244, 16, 69, 4, 194, 6, 85, 227, 206, 14, 79, 12, 248, + 241, 10, 3, 79, 78, 71, 197, 245, 15, 3, 65, 77, 69, 14, 18, 69, 35, 79, + 4, 242, 138, 33, 65, 159, 27, 83, 10, 136, 241, 10, 3, 85, 78, 68, 166, + 162, 13, 84, 162, 205, 7, 82, 211, 118, 79, 6, 188, 240, 10, 2, 65, 87, + 175, 182, 18, 69, 8, 34, 65, 169, 213, 28, 2, 69, 80, 6, 130, 214, 32, + 73, 226, 79, 77, 3, 80, 38, 122, 69, 90, 73, 186, 1, 78, 196, 1, 4, 80, + 73, 82, 73, 248, 234, 10, 4, 77, 65, 76, 76, 214, 149, 12, 72, 195, 208, + 9, 85, 8, 40, 4, 67, 79, 78, 68, 227, 154, 32, 65, 6, 11, 32, 6, 166, + 180, 31, 84, 183, 86, 79, 12, 60, 9, 77, 80, 76, 73, 70, 73, 69, 68, 32, + 147, 160, 33, 76, 10, 34, 72, 50, 87, 135, 182, 10, 89, 4, 140, 160, 10, + 3, 65, 76, 70, 223, 248, 17, 79, 4, 234, 208, 6, 65, 175, 129, 22, 72, 6, + 192, 1, 2, 79, 85, 151, 241, 32, 65, 8, 58, 85, 146, 217, 24, 65, 218, + 204, 1, 72, 147, 174, 3, 73, 2, 239, 134, 24, 82, 12, 26, 65, 49, 2, 69, + 83, 8, 164, 235, 10, 2, 76, 75, 1, 3, 84, 69, 82, 4, 247, 234, 10, 84, + 76, 110, 72, 182, 1, 80, 38, 83, 142, 163, 31, 84, 176, 250, 1, 2, 66, + 88, 2, 87, 2, 88, 86, 68, 2, 78, 3, 81, 31, 42, 80, 22, 88, 30, 90, 143, + 159, 33, 71, 5, 231, 158, 33, 87, 4, 210, 158, 33, 87, 87, 71, 19, 50, + 90, 194, 163, 31, 87, 154, 251, 1, 71, 3, 84, 9, 190, 163, 31, 90, 155, + 251, 1, 80, 9, 178, 158, 33, 68, 2, 71, 3, 90, 23, 50, 87, 30, 90, 194, + 157, 33, 71, 2, 80, 3, 84, 7, 218, 157, 33, 71, 3, 90, 9, 234, 156, 33, + 87, 86, 80, 3, 90, 124, 62, 65, 194, 1, 73, 114, 79, 181, 241, 20, 4, 69, + 65, 82, 32, 8, 132, 1, 2, 80, 80, 208, 231, 18, 6, 83, 83, 73, 67, 65, + 76, 169, 137, 13, 14, 77, 83, 72, 69, 76, 76, 32, 77, 79, 66, 73, 76, 69, + 32, 4, 196, 189, 19, 5, 73, 78, 71, 32, 72, 239, 227, 1, 69, 6, 48, 6, + 78, 75, 73, 78, 71, 32, 223, 131, 32, 80, 4, 190, 145, 25, 71, 181, 183, + 3, 6, 66, 69, 69, 82, 32, 77, 108, 72, 2, 67, 75, 140, 10, 2, 83, 69, + 236, 3, 2, 85, 68, 131, 153, 30, 87, 70, 64, 6, 32, 70, 65, 67, 69, 32, + 237, 2, 5, 87, 73, 83, 69, 32, 48, 58, 69, 46, 70, 36, 2, 78, 73, 2, 79, + 18, 83, 51, 84, 8, 120, 2, 76, 69, 125, 4, 73, 71, 72, 84, 8, 178, 1, 73, + 25, 3, 79, 85, 82, 4, 155, 1, 78, 8, 26, 69, 125, 2, 73, 88, 4, 57, 2, + 86, 69, 16, 38, 69, 14, 87, 41, 3, 72, 82, 69, 4, 63, 78, 8, 24, 2, 69, + 76, 27, 79, 4, 11, 86, 4, 11, 69, 4, 180, 252, 28, 2, 32, 79, 137, 173, + 2, 3, 45, 84, 72, 22, 90, 67, 58, 68, 122, 71, 48, 5, 82, 73, 71, 72, 84, + 226, 2, 84, 122, 79, 143, 137, 31, 73, 4, 196, 1, 3, 76, 79, 83, 141, 99, + 4, 79, 78, 84, 79, 2, 141, 3, 26, 79, 87, 78, 87, 65, 82, 68, 83, 32, 65, + 78, 68, 32, 85, 80, 87, 65, 82, 68, 83, 32, 79, 80, 69, 78, 32, 2, 21, 3, + 65, 80, 80, 2, 133, 4, 2, 69, 68, 6, 228, 1, 14, 32, 65, 78, 68, 32, 76, + 69, 70, 84, 32, 83, 69, 77, 73, 45, 38, 87, 65, 82, 68, 83, 32, 65, 78, + 68, 32, 76, 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 80, 69, 78, 32, 67, + 73, 82, 67, 76, 69, 32, 65, 82, 82, 79, 87, 83, 2, 185, 182, 30, 6, 67, + 73, 82, 67, 76, 69, 5, 161, 83, 15, 32, 87, 73, 84, 72, 32, 67, 73, 82, + 67, 76, 69, 68, 32, 79, 4, 82, 79, 37, 16, 82, 73, 65, 78, 71, 76, 69, + 45, 72, 69, 65, 68, 69, 68, 32, 79, 2, 69, 6, 80, 32, 83, 69, 77, 73, 2, + 21, 3, 80, 69, 78, 2, 11, 32, 2, 217, 180, 30, 4, 67, 73, 82, 67, 26, 32, + 2, 68, 32, 191, 153, 32, 32, 24, 216, 1, 2, 83, 85, 66, 85, 178, 175, 19, + 77, 180, 3, 9, 76, 79, 67, 75, 32, 87, 73, 84, 72, 222, 134, 12, 66, 249, + 149, 1, 23, 73, 78, 84, 69, 82, 83, 69, 67, 84, 73, 79, 78, 32, 87, 73, + 84, 72, 32, 83, 69, 82, 73, 70, 8, 30, 66, 1, 3, 80, 69, 82, 4, 181, 252, + 24, 3, 83, 69, 84, 6, 82, 77, 33, 16, 78, 73, 79, 78, 32, 87, 73, 84, 72, + 32, 83, 69, 82, 73, 70, 83, 2, 145, 223, 24, 3, 66, 82, 69, 5, 237, 252, + 31, 9, 32, 65, 78, 68, 32, 83, 77, 65, 83, 11, 33, 6, 32, 87, 73, 84, 72, + 32, 8, 72, 4, 84, 79, 82, 78, 214, 167, 5, 76, 206, 220, 19, 83, 135, + 218, 6, 82, 2, 203, 217, 25, 65, 226, 9, 158, 1, 67, 82, 76, 98, 77, 162, + 82, 78, 220, 5, 2, 79, 75, 58, 80, 150, 19, 82, 138, 1, 85, 170, 253, 27, + 87, 196, 158, 3, 2, 70, 70, 246, 47, 73, 183, 51, 65, 6, 26, 75, 203, + 148, 4, 79, 4, 142, 233, 8, 82, 189, 136, 23, 4, 84, 65, 73, 76, 8, 44, + 2, 79, 78, 165, 134, 25, 3, 76, 73, 83, 7, 11, 32, 4, 138, 128, 26, 69, + 195, 129, 6, 83, 158, 6, 72, 7, 66, 73, 78, 73, 78, 71, 32, 166, 80, 77, + 94, 80, 151, 154, 32, 69, 142, 6, 214, 2, 65, 174, 2, 66, 134, 1, 67, + 246, 12, 68, 174, 9, 69, 146, 2, 70, 54, 71, 140, 9, 2, 72, 79, 82, 73, + 216, 1, 2, 75, 65, 162, 1, 76, 230, 13, 77, 250, 1, 78, 78, 79, 138, 2, + 80, 174, 1, 82, 244, 4, 5, 90, 73, 71, 90, 65, 166, 2, 83, 242, 3, 84, + 238, 2, 85, 148, 1, 9, 86, 69, 82, 84, 73, 67, 65, 76, 32, 36, 2, 87, 73, + 183, 1, 88, 24, 148, 1, 4, 67, 85, 84, 69, 86, 78, 252, 131, 25, 6, 83, + 84, 69, 82, 73, 83, 197, 224, 5, 14, 76, 77, 79, 83, 84, 32, 69, 81, 85, + 65, 76, 32, 84, 79, 10, 22, 45, 167, 33, 32, 4, 200, 204, 12, 6, 71, 82, + 65, 86, 69, 45, 199, 205, 3, 77, 6, 244, 2, 4, 84, 73, 67, 76, 189, 179, + 3, 4, 78, 85, 73, 84, 12, 42, 82, 201, 212, 30, 4, 73, 78, 68, 85, 10, + 48, 3, 69, 86, 69, 149, 227, 30, 3, 73, 68, 71, 7, 166, 164, 12, 45, 251, + 175, 18, 32, 142, 1, 142, 1, 65, 34, 76, 98, 79, 116, 8, 89, 82, 73, 76, 76, 73, 67, 32, 184, 28, 11, 73, 82, 67, 85, 77, 70, 76, 69, 88, 32, 65, - 175, 238, 11, 69, 6, 162, 18, 82, 199, 131, 28, 78, 4, 41, 8, 79, 67, 75, - 87, 73, 83, 69, 32, 4, 184, 131, 12, 4, 82, 73, 78, 71, 239, 160, 16, 65, + 195, 150, 12, 69, 6, 162, 18, 82, 215, 217, 28, 78, 4, 41, 8, 79, 67, 75, + 87, 73, 83, 69, 32, 4, 204, 171, 12, 4, 82, 73, 78, 71, 231, 218, 16, 65, 10, 76, 4, 77, 77, 65, 32, 189, 16, 10, 78, 74, 79, 73, 78, 73, 78, 71, - 32, 77, 6, 250, 229, 12, 65, 247, 251, 16, 66, 116, 252, 1, 8, 72, 85, + 32, 77, 6, 210, 147, 13, 65, 239, 204, 17, 66, 116, 252, 1, 8, 72, 85, 78, 68, 82, 69, 68, 32, 32, 7, 76, 69, 84, 84, 69, 82, 32, 166, 5, 80, - 116, 5, 68, 65, 83, 73, 65, 38, 84, 106, 77, 206, 243, 2, 75, 132, 214, - 12, 15, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 66, 89, 217, - 241, 15, 2, 86, 90, 4, 194, 7, 77, 215, 158, 3, 84, 84, 238, 1, 66, 38, - 68, 50, 69, 82, 73, 106, 79, 22, 83, 66, 85, 30, 89, 214, 137, 3, 76, - 140, 5, 5, 77, 79, 78, 79, 71, 142, 18, 72, 202, 174, 12, 84, 190, 163, - 15, 90, 254, 82, 67, 2, 71, 182, 8, 70, 134, 14, 80, 2, 86, 158, 20, 75, - 187, 2, 65, 4, 174, 147, 6, 73, 199, 226, 25, 69, 4, 168, 155, 25, 3, 74, - 69, 82, 167, 218, 6, 69, 14, 58, 83, 230, 244, 31, 70, 2, 76, 2, 77, 2, - 78, 3, 82, 5, 183, 176, 30, 45, 11, 56, 8, 79, 84, 73, 70, 73, 69, 68, - 32, 151, 244, 31, 69, 6, 194, 145, 6, 66, 210, 226, 25, 65, 3, 69, 5, - 239, 243, 24, 77, 6, 26, 72, 155, 209, 15, 79, 4, 250, 251, 29, 67, 191, - 247, 1, 65, 5, 193, 153, 3, 2, 75, 82, 8, 142, 230, 27, 69, 190, 240, 3, - 65, 174, 28, 73, 3, 85, 8, 66, 65, 48, 4, 83, 73, 76, 73, 133, 212, 28, - 4, 79, 75, 82, 89, 4, 170, 246, 2, 89, 241, 175, 12, 3, 76, 65, 84, 2, + 116, 5, 68, 65, 83, 73, 65, 38, 84, 106, 77, 178, 243, 2, 75, 176, 138, + 13, 15, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 66, 89, 229, + 193, 16, 2, 86, 90, 4, 194, 7, 77, 195, 158, 3, 84, 84, 238, 1, 66, 38, + 68, 50, 69, 82, 73, 106, 79, 22, 83, 66, 85, 30, 89, 186, 137, 3, 76, + 140, 5, 5, 77, 79, 78, 79, 71, 150, 18, 72, 238, 226, 12, 84, 190, 243, + 15, 90, 150, 83, 67, 2, 71, 182, 8, 70, 134, 14, 80, 2, 86, 158, 20, 75, + 187, 2, 65, 4, 250, 161, 6, 73, 163, 216, 26, 69, 4, 248, 233, 25, 3, 74, + 69, 82, 255, 143, 7, 69, 14, 58, 83, 142, 249, 32, 70, 2, 76, 2, 77, 2, + 78, 3, 82, 5, 147, 178, 31, 45, 11, 56, 8, 79, 84, 73, 70, 73, 69, 68, + 32, 191, 248, 32, 69, 6, 142, 160, 6, 66, 174, 216, 26, 65, 3, 69, 5, + 159, 193, 25, 77, 6, 26, 72, 171, 133, 16, 79, 4, 158, 253, 30, 67, 195, + 250, 1, 65, 5, 173, 153, 3, 2, 75, 82, 8, 254, 189, 28, 69, 246, 156, 4, + 65, 174, 28, 73, 3, 85, 8, 66, 65, 48, 4, 83, 73, 76, 73, 189, 195, 29, + 4, 79, 75, 82, 89, 4, 142, 246, 2, 89, 133, 228, 12, 3, 76, 65, 84, 2, 145, 13, 5, 32, 80, 78, 69, 85, 10, 80, 2, 69, 78, 0, 7, 72, 79, 85, 83, - 65, 78, 68, 177, 8, 4, 73, 84, 76, 79, 2, 17, 2, 32, 77, 2, 169, 164, 21, + 65, 78, 68, 177, 8, 4, 73, 84, 76, 79, 2, 17, 2, 32, 77, 2, 129, 235, 21, 5, 73, 76, 76, 73, 79, 110, 62, 69, 244, 1, 8, 73, 65, 69, 82, 69, 83, - 73, 83, 39, 79, 38, 68, 9, 86, 65, 78, 65, 71, 65, 82, 73, 32, 249, 133, - 31, 2, 76, 69, 36, 92, 7, 76, 69, 84, 84, 69, 82, 32, 240, 179, 25, 6, - 83, 73, 71, 78, 32, 65, 167, 204, 4, 68, 14, 214, 218, 31, 86, 162, 17, - 75, 2, 78, 2, 80, 2, 82, 186, 2, 65, 3, 85, 7, 246, 140, 27, 45, 151, - 186, 2, 32, 66, 58, 84, 128, 1, 4, 85, 66, 76, 69, 241, 4, 2, 87, 78, 14, - 38, 32, 149, 215, 15, 3, 84, 69, 68, 10, 64, 5, 65, 66, 79, 86, 69, 241, - 165, 6, 5, 66, 69, 76, 79, 87, 7, 151, 193, 31, 32, 48, 22, 32, 191, 4, + 73, 83, 39, 79, 38, 68, 9, 86, 65, 78, 65, 71, 65, 82, 73, 32, 145, 138, + 32, 2, 76, 69, 36, 92, 7, 76, 69, 84, 84, 69, 82, 32, 216, 131, 26, 6, + 83, 73, 71, 78, 32, 65, 227, 253, 4, 68, 14, 254, 222, 32, 86, 162, 17, + 75, 2, 78, 2, 80, 2, 82, 186, 2, 65, 3, 85, 7, 146, 229, 27, 45, 195, + 223, 2, 32, 66, 58, 84, 128, 1, 4, 85, 66, 76, 69, 241, 4, 2, 87, 78, 14, + 38, 32, 173, 139, 16, 3, 84, 69, 68, 10, 64, 5, 65, 66, 79, 86, 69, 197, + 179, 6, 5, 66, 69, 76, 79, 87, 7, 191, 197, 32, 32, 48, 22, 32, 191, 4, 68, 46, 230, 1, 66, 0, 10, 73, 78, 86, 69, 82, 84, 69, 68, 32, 66, 26, 77, 54, 79, 34, 80, 68, 2, 82, 73, 48, 5, 84, 73, 76, 68, 69, 80, 9, 86, - 69, 82, 84, 73, 67, 65, 76, 32, 174, 210, 15, 65, 42, 71, 166, 179, 4, - 76, 187, 228, 10, 67, 4, 229, 16, 2, 82, 69, 4, 21, 3, 65, 67, 82, 4, - 181, 193, 16, 2, 79, 78, 4, 230, 35, 80, 255, 224, 29, 86, 6, 238, 36, - 76, 205, 206, 7, 9, 65, 82, 69, 78, 84, 72, 69, 83, 69, 4, 156, 51, 4, - 71, 72, 84, 87, 139, 228, 9, 78, 7, 11, 32, 4, 44, 3, 76, 69, 70, 1, 4, - 82, 73, 71, 72, 2, 199, 205, 29, 84, 6, 178, 239, 11, 83, 211, 255, 13, - 76, 2, 141, 205, 15, 2, 32, 67, 4, 128, 41, 2, 32, 84, 227, 158, 30, 87, - 16, 76, 9, 78, 67, 76, 79, 83, 73, 78, 71, 32, 129, 193, 25, 4, 81, 85, - 65, 76, 14, 132, 1, 6, 67, 73, 82, 67, 76, 69, 22, 83, 216, 145, 6, 3, - 75, 69, 89, 160, 241, 1, 7, 85, 80, 87, 65, 82, 68, 32, 227, 166, 22, 68, - 5, 219, 254, 17, 32, 4, 134, 138, 25, 67, 195, 186, 5, 81, 4, 32, 2, 69, - 82, 135, 137, 5, 79, 2, 143, 74, 77, 126, 88, 17, 76, 65, 71, 79, 76, 73, + 69, 82, 84, 73, 67, 65, 76, 32, 198, 134, 16, 65, 42, 71, 150, 187, 4, + 76, 203, 172, 11, 67, 4, 229, 16, 2, 82, 69, 4, 21, 3, 65, 67, 82, 4, + 229, 245, 16, 2, 79, 78, 4, 230, 35, 80, 163, 226, 30, 86, 6, 238, 36, + 76, 253, 224, 7, 9, 65, 82, 69, 78, 84, 72, 69, 83, 69, 4, 156, 51, 4, + 71, 72, 84, 87, 207, 248, 9, 78, 7, 11, 32, 4, 44, 3, 76, 69, 70, 1, 4, + 82, 73, 71, 72, 2, 199, 203, 30, 84, 6, 198, 151, 12, 83, 151, 177, 14, + 76, 2, 165, 129, 16, 2, 32, 67, 4, 128, 41, 2, 32, 84, 243, 162, 31, 87, + 16, 76, 9, 78, 67, 76, 79, 83, 73, 78, 71, 32, 205, 152, 26, 4, 81, 85, + 65, 76, 14, 132, 1, 6, 67, 73, 82, 67, 76, 69, 22, 83, 164, 160, 6, 3, + 75, 69, 89, 228, 203, 19, 7, 85, 80, 87, 65, 82, 68, 32, 191, 192, 5, 68, + 5, 175, 180, 18, 32, 4, 214, 216, 25, 67, 131, 240, 5, 81, 4, 32, 2, 69, + 82, 159, 142, 5, 79, 2, 139, 74, 77, 126, 88, 17, 76, 65, 71, 79, 76, 73, 84, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 143, 3, 82, 76, 238, 1, 68, - 30, 73, 46, 83, 50, 84, 182, 250, 5, 65, 22, 66, 94, 67, 134, 1, 70, 38, + 30, 73, 46, 83, 50, 84, 130, 137, 6, 65, 22, 66, 94, 67, 134, 1, 70, 38, 71, 238, 2, 77, 32, 2, 76, 74, 34, 78, 88, 2, 80, 79, 30, 82, 194, 2, 86, - 22, 89, 90, 90, 182, 199, 18, 72, 142, 243, 1, 85, 154, 253, 4, 79, 235, - 5, 75, 4, 238, 252, 5, 74, 31, 79, 11, 250, 253, 5, 78, 54, 79, 135, 182, - 25, 90, 8, 218, 254, 5, 77, 130, 4, 76, 243, 204, 20, 72, 4, 142, 131, 6, - 86, 255, 201, 25, 83, 50, 38, 65, 173, 3, 4, 69, 69, 75, 32, 36, 84, 5, - 78, 84, 72, 65, 32, 196, 1, 2, 86, 69, 153, 228, 27, 5, 80, 72, 69, 77, + 22, 89, 90, 90, 226, 132, 19, 72, 174, 128, 2, 85, 170, 168, 5, 79, 235, + 5, 75, 4, 186, 139, 6, 74, 31, 79, 11, 198, 140, 6, 78, 54, 79, 227, 171, + 26, 90, 8, 166, 141, 6, 77, 130, 4, 76, 211, 200, 22, 72, 4, 218, 145, 6, + 86, 219, 191, 26, 83, 50, 38, 65, 173, 3, 4, 69, 69, 75, 32, 36, 84, 5, + 78, 84, 72, 65, 32, 196, 1, 2, 86, 69, 133, 188, 28, 5, 80, 72, 69, 77, 69, 24, 68, 6, 68, 73, 71, 73, 84, 32, 65, 7, 76, 69, 84, 84, 69, 82, 32, - 14, 202, 177, 16, 83, 142, 192, 13, 70, 70, 84, 62, 90, 143, 83, 79, 10, - 222, 202, 31, 86, 162, 17, 75, 2, 78, 2, 80, 187, 2, 65, 10, 18, 32, 67, - 45, 6, 26, 65, 215, 242, 15, 84, 4, 245, 143, 4, 4, 67, 67, 69, 78, 4, - 208, 220, 11, 6, 65, 67, 85, 84, 69, 45, 195, 231, 3, 77, 14, 148, 1, 8, - 77, 85, 83, 73, 67, 65, 76, 32, 150, 151, 4, 75, 250, 182, 2, 80, 174, 4, - 89, 249, 227, 3, 11, 68, 73, 65, 76, 89, 84, 73, 75, 65, 32, 84, 6, 26, - 84, 131, 135, 15, 80, 4, 218, 135, 15, 69, 35, 82, 6, 206, 200, 12, 79, - 216, 149, 18, 8, 77, 79, 84, 72, 69, 84, 73, 67, 163, 45, 82, 14, 26, 78, - 207, 179, 29, 83, 12, 52, 7, 86, 69, 82, 84, 69, 68, 32, 143, 222, 27, + 14, 250, 229, 16, 83, 130, 141, 14, 70, 70, 84, 62, 90, 251, 85, 79, 10, + 134, 207, 32, 86, 162, 17, 75, 2, 78, 2, 80, 187, 2, 65, 10, 18, 32, 67, + 45, 6, 26, 65, 231, 166, 16, 84, 4, 209, 144, 4, 4, 67, 67, 69, 78, 4, + 212, 132, 12, 6, 65, 67, 85, 84, 69, 45, 215, 243, 3, 77, 14, 148, 1, 8, + 77, 85, 83, 73, 67, 65, 76, 32, 242, 151, 4, 75, 242, 195, 2, 80, 174, 4, + 89, 225, 238, 3, 11, 68, 73, 65, 76, 89, 84, 73, 75, 65, 32, 84, 6, 26, + 84, 247, 186, 15, 80, 4, 206, 187, 15, 69, 35, 82, 6, 166, 246, 12, 79, + 152, 236, 18, 8, 77, 79, 84, 72, 69, 84, 73, 67, 167, 45, 82, 14, 26, 78, + 151, 177, 30, 83, 12, 52, 7, 86, 69, 82, 84, 69, 68, 32, 251, 181, 28, 70, 10, 64, 2, 66, 82, 45, 10, 68, 79, 85, 66, 76, 69, 32, 65, 82, 67, 6, - 22, 69, 235, 37, 73, 4, 211, 181, 11, 86, 4, 167, 192, 29, 72, 8, 128, 1, - 16, 84, 65, 75, 65, 78, 65, 45, 72, 73, 82, 65, 71, 65, 78, 65, 32, 153, - 180, 17, 9, 86, 89, 75, 65, 32, 65, 66, 79, 86, 4, 146, 237, 9, 83, 35, + 22, 69, 235, 37, 73, 4, 219, 221, 11, 86, 4, 247, 190, 30, 72, 8, 128, 1, + 16, 84, 65, 75, 65, 78, 65, 45, 72, 73, 82, 65, 71, 65, 78, 65, 32, 229, + 233, 17, 9, 86, 89, 75, 65, 32, 65, 66, 79, 86, 4, 222, 129, 10, 83, 35, 86, 158, 1, 80, 5, 65, 84, 73, 78, 32, 156, 8, 3, 69, 70, 84, 184, 3, 2, 73, 71, 83, 79, 106, 156, 1, 21, 76, 69, 84, 84, 69, 82, 32, 83, 77, 65, 76, 76, 32, 67, 65, 80, 73, 84, 65, 76, 32, 53, 13, 83, 77, 65, 76, 76, - 32, 76, 69, 84, 84, 69, 82, 32, 10, 218, 213, 31, 71, 2, 76, 2, 77, 2, + 32, 76, 69, 84, 84, 69, 82, 32, 10, 130, 218, 32, 71, 2, 76, 2, 77, 2, 78, 3, 82, 96, 226, 1, 65, 70, 67, 34, 69, 30, 70, 78, 73, 86, 76, 106, - 79, 2, 85, 134, 1, 82, 50, 84, 206, 131, 12, 83, 194, 134, 2, 66, 226, - 156, 2, 87, 158, 168, 15, 68, 2, 71, 2, 72, 2, 75, 2, 77, 2, 78, 2, 80, - 2, 86, 2, 88, 3, 90, 13, 142, 244, 2, 32, 146, 235, 12, 76, 166, 244, 15, - 69, 2, 79, 3, 86, 5, 221, 227, 11, 3, 32, 67, 69, 7, 206, 209, 31, 83, 3, - 84, 5, 185, 219, 26, 14, 76, 65, 84, 84, 69, 78, 69, 68, 32, 79, 80, 69, - 78, 32, 11, 37, 7, 78, 83, 85, 76, 65, 82, 32, 8, 206, 209, 31, 68, 2, - 71, 2, 82, 3, 84, 7, 136, 210, 15, 14, 32, 87, 73, 84, 72, 32, 68, 79, - 85, 66, 76, 69, 32, 77, 165, 172, 12, 3, 79, 78, 71, 7, 33, 6, 32, 87, - 73, 84, 72, 32, 4, 194, 243, 11, 68, 201, 187, 12, 15, 76, 73, 71, 72, - 84, 32, 67, 69, 78, 84, 82, 65, 76, 73, 90, 7, 11, 32, 4, 146, 210, 11, - 82, 227, 228, 17, 66, 5, 129, 26, 6, 85, 82, 78, 69, 68, 32, 32, 46, 32, + 79, 2, 85, 134, 1, 82, 50, 84, 226, 171, 12, 83, 182, 146, 2, 66, 138, + 157, 2, 87, 150, 248, 15, 68, 2, 71, 2, 72, 2, 75, 2, 77, 2, 78, 2, 80, + 2, 86, 2, 88, 3, 90, 13, 242, 243, 2, 32, 198, 139, 26, 76, 182, 216, 3, + 69, 2, 79, 3, 86, 5, 241, 139, 12, 3, 32, 67, 69, 7, 246, 213, 32, 83, 3, + 84, 5, 249, 179, 27, 14, 76, 65, 84, 84, 69, 78, 69, 68, 32, 79, 80, 69, + 78, 32, 11, 37, 7, 78, 83, 85, 76, 65, 82, 32, 8, 246, 213, 32, 68, 2, + 71, 2, 82, 3, 84, 7, 164, 134, 16, 14, 32, 87, 73, 84, 72, 32, 68, 79, + 85, 66, 76, 69, 32, 77, 149, 218, 12, 3, 79, 78, 71, 7, 33, 6, 32, 87, + 73, 84, 72, 32, 4, 214, 155, 12, 68, 225, 222, 12, 15, 76, 73, 71, 72, + 84, 32, 67, 69, 78, 84, 82, 65, 76, 73, 90, 7, 11, 32, 4, 150, 250, 11, + 82, 175, 187, 18, 66, 5, 129, 26, 6, 85, 82, 78, 69, 68, 32, 32, 46, 32, 213, 2, 6, 87, 65, 82, 68, 83, 32, 28, 154, 1, 65, 112, 12, 80, 65, 82, - 69, 78, 84, 72, 69, 83, 73, 83, 32, 226, 12, 72, 182, 1, 84, 177, 165, - 29, 11, 82, 73, 71, 72, 84, 32, 65, 82, 82, 79, 87, 12, 48, 4, 82, 82, - 79, 87, 233, 179, 29, 2, 78, 71, 8, 26, 72, 243, 179, 29, 32, 4, 225, - 179, 29, 3, 69, 65, 68, 4, 244, 204, 7, 4, 65, 66, 79, 86, 245, 204, 23, - 5, 66, 69, 76, 79, 87, 4, 230, 13, 72, 133, 198, 11, 5, 65, 82, 82, 79, + 69, 78, 84, 72, 69, 83, 73, 83, 32, 226, 12, 72, 182, 1, 84, 129, 164, + 30, 11, 82, 73, 71, 72, 84, 32, 65, 82, 82, 79, 87, 12, 48, 4, 82, 82, + 79, 87, 185, 178, 30, 2, 78, 71, 8, 26, 72, 195, 178, 30, 32, 4, 177, + 178, 30, 3, 69, 65, 68, 4, 164, 223, 7, 4, 65, 66, 79, 86, 225, 190, 24, + 5, 66, 69, 76, 79, 87, 4, 230, 13, 72, 153, 238, 11, 5, 65, 82, 82, 79, 87, 10, 52, 6, 65, 84, 85, 82, 69, 32, 169, 17, 2, 72, 84, 8, 238, 1, 76, - 23, 82, 10, 36, 3, 78, 71, 32, 251, 176, 30, 87, 8, 172, 7, 7, 68, 79, + 23, 82, 10, 36, 3, 78, 71, 32, 139, 181, 31, 87, 8, 172, 7, 7, 68, 79, 85, 66, 76, 69, 32, 246, 7, 83, 71, 86, 20, 52, 5, 65, 67, 82, 79, 78, - 145, 164, 25, 2, 73, 78, 19, 18, 32, 127, 45, 10, 34, 76, 22, 82, 183, - 176, 29, 66, 4, 41, 2, 69, 70, 4, 21, 3, 73, 71, 72, 4, 181, 160, 16, 6, - 84, 32, 72, 65, 76, 70, 6, 166, 246, 11, 71, 186, 2, 65, 139, 157, 3, 66, - 4, 152, 9, 9, 85, 77, 66, 69, 82, 32, 83, 73, 71, 177, 173, 11, 2, 79, + 221, 251, 25, 2, 73, 78, 19, 18, 32, 127, 45, 10, 34, 76, 22, 82, 135, + 175, 30, 66, 4, 41, 2, 69, 70, 4, 21, 3, 73, 71, 72, 4, 229, 212, 16, 6, + 84, 32, 72, 65, 76, 70, 6, 186, 158, 12, 71, 186, 2, 65, 135, 169, 3, 66, + 4, 152, 9, 9, 85, 77, 66, 69, 82, 32, 83, 73, 71, 181, 213, 11, 2, 79, 84, 18, 136, 1, 17, 76, 68, 32, 80, 69, 82, 77, 73, 67, 32, 76, 69, 84, - 84, 69, 82, 32, 82, 80, 248, 243, 11, 4, 71, 79, 78, 69, 135, 237, 17, - 86, 10, 246, 43, 90, 206, 218, 17, 78, 186, 253, 8, 68, 138, 134, 2, 83, - 203, 236, 1, 65, 2, 213, 128, 24, 6, 69, 78, 32, 77, 65, 82, 12, 18, 65, - 107, 76, 8, 236, 7, 9, 82, 69, 78, 84, 72, 69, 83, 69, 83, 141, 248, 23, - 9, 76, 65, 84, 65, 76, 73, 90, 69, 68, 4, 197, 171, 29, 7, 85, 83, 32, - 83, 73, 71, 78, 38, 18, 69, 127, 73, 6, 72, 5, 86, 69, 82, 83, 69, 133, - 254, 23, 7, 84, 82, 79, 70, 76, 69, 88, 4, 22, 32, 227, 12, 68, 2, 141, + 84, 69, 82, 32, 82, 80, 140, 156, 12, 4, 71, 79, 78, 69, 151, 198, 18, + 86, 10, 242, 43, 90, 202, 148, 18, 78, 222, 155, 9, 68, 182, 171, 2, 83, + 159, 243, 1, 65, 2, 169, 203, 24, 6, 69, 78, 32, 77, 65, 82, 12, 18, 65, + 107, 76, 8, 236, 7, 9, 82, 69, 78, 84, 72, 69, 83, 69, 83, 225, 194, 24, + 9, 76, 65, 84, 65, 76, 73, 90, 69, 68, 4, 149, 170, 30, 7, 85, 83, 32, + 83, 73, 71, 78, 38, 18, 69, 127, 73, 6, 72, 5, 86, 69, 82, 83, 69, 217, + 200, 24, 7, 84, 82, 79, 70, 76, 69, 88, 4, 22, 32, 227, 12, 68, 2, 141, 8, 2, 83, 79, 32, 40, 3, 71, 72, 84, 157, 5, 2, 78, 71, 26, 50, 32, 149, 4, 7, 87, 65, 82, 68, 83, 32, 72, 24, 104, 5, 65, 82, 82, 79, 87, 218, 1, 72, 124, 12, 80, 65, 82, 69, 78, 84, 72, 69, 83, 73, 83, 32, 59, 84, 12, - 44, 5, 72, 69, 65, 68, 32, 155, 168, 29, 32, 8, 26, 65, 155, 168, 29, 66, - 6, 36, 3, 78, 68, 32, 219, 195, 30, 66, 4, 40, 4, 68, 79, 87, 78, 1, 2, - 85, 80, 2, 157, 203, 8, 9, 32, 65, 82, 82, 79, 87, 72, 69, 65, 6, 11, 65, - 6, 48, 6, 76, 70, 32, 82, 73, 78, 21, 2, 82, 80, 4, 163, 166, 29, 71, 2, - 17, 2, 79, 79, 2, 231, 193, 30, 78, 4, 174, 198, 9, 65, 137, 170, 20, 5, - 66, 69, 76, 79, 87, 2, 149, 249, 23, 2, 65, 67, 2, 233, 201, 20, 16, 65, + 44, 5, 72, 69, 65, 68, 32, 235, 166, 30, 32, 8, 26, 65, 235, 166, 30, 66, + 6, 36, 3, 78, 68, 32, 243, 199, 31, 66, 4, 40, 4, 68, 79, 87, 78, 1, 2, + 85, 80, 2, 133, 223, 8, 9, 32, 65, 82, 82, 79, 87, 72, 69, 65, 6, 11, 65, + 6, 48, 6, 76, 70, 32, 82, 73, 78, 21, 2, 82, 80, 4, 243, 164, 30, 71, 2, + 17, 2, 79, 79, 2, 255, 197, 31, 78, 4, 250, 218, 9, 65, 241, 150, 21, 5, + 66, 69, 76, 79, 87, 2, 233, 195, 24, 2, 65, 67, 2, 141, 144, 21, 16, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, 32, 66, 65, 82, 66, 6, 11, 32, 6, - 182, 197, 11, 79, 166, 223, 17, 66, 223, 155, 1, 65, 18, 188, 1, 5, 72, + 202, 237, 11, 79, 226, 181, 18, 66, 167, 161, 1, 65, 18, 188, 1, 5, 72, 79, 82, 84, 32, 160, 1, 7, 81, 85, 65, 82, 69, 32, 66, 56, 5, 84, 82, 79, - 78, 71, 232, 193, 3, 2, 85, 83, 148, 179, 20, 2, 78, 65, 233, 157, 5, 6, - 69, 65, 71, 85, 76, 76, 6, 18, 83, 71, 86, 4, 26, 79, 223, 194, 11, 84, - 2, 253, 194, 11, 5, 76, 73, 68, 85, 83, 2, 49, 10, 69, 82, 84, 73, 67, - 65, 76, 32, 76, 73, 2, 151, 194, 11, 78, 4, 148, 196, 7, 5, 82, 65, 67, - 75, 69, 191, 221, 21, 69, 2, 225, 244, 23, 17, 32, 67, 69, 78, 84, 82, + 78, 71, 168, 193, 3, 2, 85, 83, 168, 254, 20, 2, 78, 65, 221, 208, 5, 6, + 69, 65, 71, 85, 76, 76, 6, 18, 83, 71, 86, 4, 26, 79, 243, 234, 11, 84, + 2, 145, 235, 11, 5, 76, 73, 68, 85, 83, 2, 49, 10, 69, 82, 84, 73, 67, + 65, 76, 32, 76, 73, 2, 171, 234, 11, 78, 4, 196, 214, 7, 5, 82, 65, 67, + 75, 69, 223, 201, 22, 69, 2, 181, 191, 24, 17, 32, 67, 69, 78, 84, 82, 65, 76, 73, 90, 65, 84, 73, 79, 78, 32, 83, 20, 98, 72, 32, 4, 73, 76, 68, 69, 136, 1, 6, 82, 73, 80, 76, 69, 32, 69, 5, 85, 82, 78, 69, 68, 2, - 137, 194, 7, 3, 82, 69, 69, 11, 11, 32, 8, 76, 3, 76, 69, 70, 0, 4, 82, - 73, 71, 72, 182, 191, 11, 79, 167, 223, 17, 66, 2, 169, 144, 29, 6, 84, - 32, 72, 65, 76, 70, 6, 186, 161, 15, 65, 184, 132, 15, 5, 85, 78, 68, 69, - 82, 199, 64, 68, 2, 145, 252, 14, 2, 32, 67, 10, 34, 80, 238, 184, 30, - 82, 3, 83, 6, 42, 87, 141, 240, 23, 4, 32, 84, 65, 67, 2, 45, 9, 65, 82, - 68, 83, 32, 65, 82, 82, 79, 2, 159, 142, 29, 87, 6, 158, 188, 25, 76, - 171, 183, 2, 84, 6, 52, 3, 68, 69, 32, 133, 189, 11, 4, 71, 71, 76, 89, - 4, 92, 12, 73, 78, 86, 69, 82, 84, 69, 68, 32, 66, 82, 73, 161, 165, 28, - 5, 66, 82, 73, 68, 71, 2, 241, 238, 23, 2, 68, 71, 6, 178, 196, 11, 45, - 147, 214, 17, 32, 6, 52, 7, 69, 82, 67, 73, 65, 76, 32, 191, 178, 31, 65, - 4, 206, 152, 28, 77, 195, 253, 2, 65, 8, 178, 140, 8, 82, 144, 221, 15, - 3, 79, 83, 73, 246, 225, 2, 76, 171, 208, 3, 65, 38, 214, 1, 70, 84, 10, - 83, 84, 82, 85, 67, 84, 73, 79, 78, 32, 46, 84, 172, 236, 2, 8, 86, 69, - 78, 73, 69, 78, 67, 69, 172, 237, 17, 6, 73, 67, 65, 76, 32, 84, 170, - 243, 8, 74, 133, 97, 7, 71, 82, 85, 69, 78, 84, 32, 6, 168, 187, 26, 4, - 69, 84, 84, 73, 162, 214, 1, 85, 229, 141, 2, 4, 79, 85, 78, 68, 4, 240, - 144, 20, 2, 87, 79, 219, 151, 10, 83, 20, 80, 5, 65, 73, 78, 83, 32, 198, - 1, 79, 28, 4, 82, 79, 76, 32, 199, 146, 3, 73, 12, 48, 3, 65, 83, 32, 93, - 5, 87, 73, 84, 72, 32, 6, 242, 205, 23, 77, 253, 7, 15, 78, 79, 82, 77, - 65, 76, 32, 83, 85, 66, 71, 82, 79, 85, 80, 6, 162, 190, 4, 76, 218, 144, - 19, 86, 223, 196, 4, 79, 2, 193, 168, 29, 2, 85, 82, 4, 232, 176, 19, 3, - 75, 78, 79, 173, 242, 3, 8, 83, 69, 81, 85, 69, 78, 67, 69, 6, 26, 73, - 175, 158, 2, 69, 4, 250, 170, 31, 78, 87, 69, 206, 2, 36, 4, 84, 73, 67, - 32, 175, 18, 89, 202, 2, 186, 1, 67, 204, 1, 6, 69, 80, 65, 67, 84, 32, - 86, 70, 36, 11, 79, 76, 68, 32, 78, 85, 66, 73, 65, 78, 32, 110, 83, 129, - 133, 28, 13, 77, 79, 82, 80, 72, 79, 76, 79, 71, 73, 67, 65, 76, 126, 76, + 185, 212, 7, 3, 82, 69, 69, 11, 11, 32, 8, 76, 3, 76, 69, 70, 0, 4, 82, + 73, 71, 72, 202, 231, 11, 79, 227, 181, 18, 66, 2, 241, 141, 30, 6, 84, + 32, 72, 65, 76, 70, 6, 210, 213, 15, 65, 176, 212, 15, 5, 85, 78, 68, 69, + 82, 211, 64, 68, 2, 161, 176, 15, 2, 32, 67, 10, 34, 80, 134, 189, 31, + 82, 3, 83, 6, 42, 87, 225, 186, 24, 4, 32, 84, 65, 67, 2, 45, 9, 65, 82, + 68, 83, 32, 65, 82, 82, 79, 2, 231, 139, 30, 87, 6, 246, 149, 26, 76, + 199, 191, 2, 84, 6, 52, 3, 68, 69, 32, 153, 229, 11, 4, 71, 71, 76, 89, + 4, 92, 12, 73, 78, 86, 69, 82, 84, 69, 68, 32, 66, 82, 73, 169, 148, 29, + 5, 66, 82, 73, 68, 71, 2, 197, 185, 24, 2, 68, 71, 6, 198, 236, 11, 45, + 207, 172, 18, 32, 6, 52, 7, 69, 82, 67, 73, 65, 76, 32, 231, 182, 32, 65, + 4, 134, 136, 29, 77, 179, 146, 3, 65, 8, 138, 159, 8, 82, 140, 149, 16, + 3, 79, 83, 73, 190, 239, 2, 76, 159, 252, 3, 65, 38, 214, 1, 70, 84, 10, + 83, 84, 82, 85, 67, 84, 73, 79, 78, 32, 46, 84, 252, 235, 2, 8, 86, 69, + 78, 73, 69, 78, 67, 69, 180, 180, 18, 6, 73, 67, 65, 76, 32, 84, 246, + 173, 9, 74, 241, 99, 7, 71, 82, 85, 69, 78, 84, 32, 6, 228, 147, 27, 4, + 69, 84, 84, 73, 158, 237, 1, 85, 189, 162, 2, 4, 79, 85, 78, 68, 4, 164, + 206, 20, 2, 87, 79, 183, 222, 10, 83, 20, 80, 5, 65, 73, 78, 83, 32, 198, + 1, 79, 28, 4, 82, 79, 76, 32, 143, 146, 3, 73, 12, 48, 3, 65, 83, 32, 93, + 5, 87, 73, 84, 72, 32, 6, 222, 152, 24, 77, 181, 8, 15, 78, 79, 82, 77, + 65, 76, 32, 83, 85, 66, 71, 82, 79, 85, 80, 6, 186, 195, 4, 76, 174, 214, + 19, 86, 171, 233, 4, 79, 2, 217, 169, 30, 2, 85, 82, 4, 188, 236, 19, 3, + 75, 78, 79, 165, 129, 4, 8, 83, 69, 81, 85, 69, 78, 67, 69, 6, 26, 73, + 143, 158, 2, 69, 4, 162, 175, 32, 78, 87, 69, 206, 2, 36, 4, 84, 73, 67, + 32, 171, 18, 89, 202, 2, 186, 1, 67, 204, 1, 6, 69, 80, 65, 67, 84, 32, + 86, 70, 36, 11, 79, 76, 68, 32, 78, 85, 66, 73, 65, 78, 32, 110, 83, 181, + 244, 28, 13, 77, 79, 82, 80, 72, 79, 76, 79, 71, 73, 67, 65, 76, 126, 76, 9, 79, 77, 66, 73, 78, 73, 78, 71, 32, 161, 3, 5, 65, 80, 73, 84, 65, 6, - 68, 9, 83, 80, 73, 82, 73, 84, 85, 83, 32, 189, 171, 30, 2, 78, 73, 4, - 196, 135, 6, 2, 76, 69, 161, 198, 16, 2, 65, 83, 56, 184, 211, 19, 8, 84, - 72, 79, 85, 83, 65, 78, 68, 166, 146, 1, 78, 195, 190, 2, 68, 4, 230, - 241, 17, 82, 203, 234, 10, 85, 8, 58, 68, 0, 3, 73, 78, 68, 146, 15, 86, - 183, 204, 28, 70, 2, 145, 220, 28, 7, 73, 82, 69, 67, 84, 32, 81, 134, 1, - 56, 3, 77, 65, 76, 197, 11, 6, 89, 77, 66, 79, 76, 32, 120, 45, 9, 76, + 68, 9, 83, 80, 73, 82, 73, 84, 85, 83, 32, 213, 175, 31, 2, 78, 73, 4, + 152, 149, 6, 2, 76, 69, 173, 131, 17, 2, 65, 83, 56, 134, 173, 21, 78, + 230, 193, 2, 68, 145, 238, 6, 8, 84, 72, 79, 85, 83, 65, 78, 68, 4, 146, + 173, 18, 82, 231, 172, 11, 85, 8, 58, 68, 0, 3, 73, 78, 68, 142, 15, 86, + 131, 202, 29, 70, 2, 217, 217, 29, 7, 73, 82, 69, 67, 84, 32, 81, 134, 1, + 56, 3, 77, 65, 76, 193, 11, 6, 89, 77, 66, 79, 76, 32, 120, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 120, 138, 2, 65, 48, 6, 66, 79, 72, 65, 73, 82, 32, 2, 67, 82, 170, 1, 68, 160, 1, 2, 71, 65, 34, 72, 38, 75, 46, - 70, 34, 76, 66, 79, 138, 3, 83, 90, 84, 46, 90, 146, 197, 5, 86, 210, 55, - 80, 206, 203, 3, 73, 138, 179, 21, 82, 198, 3, 69, 218, 7, 77, 2, 78, - 163, 17, 85, 4, 44, 5, 75, 72, 77, 73, 77, 143, 134, 19, 76, 2, 177, 1, + 70, 34, 76, 66, 79, 134, 3, 83, 90, 84, 46, 90, 230, 211, 5, 86, 214, 54, + 80, 190, 210, 3, 73, 238, 162, 22, 82, 198, 3, 69, 218, 7, 77, 2, 78, + 163, 17, 85, 4, 44, 5, 75, 72, 77, 73, 77, 223, 193, 19, 76, 2, 177, 1, 4, 73, 67, 32, 75, 10, 92, 12, 89, 80, 84, 79, 71, 82, 65, 77, 77, 73, - 67, 32, 53, 7, 79, 83, 83, 69, 68, 32, 83, 8, 50, 83, 226, 4, 71, 170, - 129, 31, 69, 219, 7, 78, 2, 215, 144, 24, 72, 12, 80, 9, 73, 65, 76, 69, - 67, 84, 45, 80, 32, 204, 251, 30, 2, 65, 76, 175, 17, 69, 8, 222, 137, 8, - 72, 162, 245, 7, 65, 240, 252, 11, 2, 75, 65, 139, 145, 3, 78, 4, 190, 3, - 78, 183, 251, 30, 77, 4, 250, 150, 24, 79, 191, 196, 5, 65, 8, 42, 72, - 182, 250, 27, 65, 139, 145, 3, 83, 4, 186, 139, 31, 69, 219, 19, 73, 4, - 224, 131, 19, 7, 45, 83, 72, 65, 80, 69, 68, 207, 212, 7, 65, 35, 36, 3, - 76, 68, 32, 167, 249, 30, 79, 30, 76, 7, 67, 79, 80, 84, 73, 67, 32, 193, - 1, 7, 78, 85, 66, 73, 65, 78, 32, 22, 98, 71, 42, 72, 188, 1, 2, 83, 72, - 238, 169, 14, 68, 190, 224, 9, 79, 178, 171, 5, 69, 227, 101, 65, 2, 17, - 2, 65, 78, 2, 171, 199, 15, 71, 8, 198, 147, 24, 79, 246, 211, 1, 65, - 159, 161, 5, 69, 8, 50, 78, 176, 253, 15, 2, 83, 72, 163, 139, 5, 87, 4, - 238, 135, 31, 71, 3, 89, 10, 54, 72, 178, 137, 6, 65, 222, 236, 24, 79, - 219, 3, 73, 4, 226, 249, 30, 73, 187, 13, 69, 4, 196, 237, 30, 3, 72, 69, - 84, 167, 8, 65, 2, 219, 245, 30, 65, 14, 62, 75, 30, 77, 2, 80, 22, 83, - 253, 146, 27, 3, 84, 65, 85, 4, 26, 72, 227, 133, 31, 65, 2, 143, 147, - 27, 73, 4, 136, 251, 15, 6, 72, 73, 77, 65, 32, 83, 145, 246, 3, 3, 84, - 65, 85, 4, 220, 252, 23, 3, 76, 69, 70, 157, 133, 2, 4, 82, 73, 71, 72, - 6, 96, 6, 78, 73, 83, 72, 32, 86, 252, 240, 24, 8, 82, 69, 83, 80, 79, - 78, 68, 83, 163, 157, 5, 65, 2, 157, 246, 27, 4, 69, 82, 83, 69, 44, 104, - 2, 78, 84, 208, 132, 3, 6, 67, 72, 32, 65, 78, 68, 181, 138, 27, 8, 80, - 76, 69, 32, 87, 73, 84, 72, 40, 56, 2, 69, 82, 33, 8, 73, 78, 71, 32, 82, - 79, 68, 32, 4, 158, 221, 26, 66, 235, 121, 83, 36, 48, 4, 84, 69, 78, 83, - 1, 4, 85, 78, 73, 84, 18, 185, 145, 23, 2, 32, 68, 51, 82, 69, 76, 5, 73, - 67, 75, 69, 84, 34, 79, 254, 5, 85, 50, 89, 239, 210, 29, 65, 4, 220, - 148, 8, 3, 68, 73, 84, 153, 238, 19, 7, 83, 67, 69, 78, 84, 32, 77, 5, - 237, 158, 26, 3, 32, 66, 65, 28, 84, 2, 83, 83, 228, 244, 28, 3, 67, 79, - 68, 188, 148, 1, 3, 73, 83, 83, 243, 57, 87, 22, 46, 32, 240, 2, 3, 69, - 68, 32, 223, 1, 73, 14, 44, 3, 79, 70, 32, 82, 80, 151, 208, 30, 77, 4, - 236, 202, 27, 6, 74, 69, 82, 85, 83, 65, 245, 173, 2, 5, 76, 79, 82, 82, - 65, 8, 76, 10, 65, 84, 84, 89, 32, 87, 73, 84, 72, 32, 97, 5, 79, 77, 77, - 69, 69, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 241, 192, 28, 7, - 84, 32, 67, 82, 79, 83, 83, 5, 233, 161, 3, 11, 32, 87, 73, 84, 72, 32, - 72, 65, 76, 70, 45, 6, 196, 181, 24, 37, 78, 69, 71, 65, 84, 73, 86, 69, - 32, 83, 81, 85, 65, 82, 69, 68, 32, 76, 65, 84, 73, 78, 32, 67, 65, 80, - 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 160, 212, 2, 2, 70, 76, 137, - 169, 2, 3, 83, 87, 79, 2, 141, 160, 23, 5, 78, 71, 32, 76, 65, 4, 228, - 145, 17, 3, 90, 69, 73, 179, 217, 13, 84, 6, 194, 159, 19, 73, 137, 249, - 6, 4, 83, 84, 65, 76, 204, 19, 154, 1, 66, 20, 8, 78, 69, 73, 70, 79, 82, - 77, 32, 138, 250, 1, 80, 114, 82, 172, 3, 2, 83, 84, 254, 165, 15, 67, - 209, 194, 9, 6, 84, 32, 79, 70, 32, 77, 2, 207, 160, 5, 69, 168, 19, 176, - 1, 13, 78, 85, 77, 69, 82, 73, 67, 32, 83, 73, 71, 78, 32, 184, 20, 17, - 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 83, 73, 71, 78, 32, 233, - 1, 5, 83, 73, 71, 78, 32, 222, 1, 78, 69, 166, 2, 70, 152, 3, 2, 78, 73, - 154, 2, 79, 130, 3, 83, 151, 4, 84, 24, 68, 5, 73, 71, 72, 84, 32, 153, - 1, 7, 76, 65, 77, 73, 84, 69, 32, 16, 142, 14, 83, 214, 113, 85, 238, 50, - 71, 136, 56, 17, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 85, - 83, 83, 85, 186, 223, 25, 68, 171, 217, 1, 65, 8, 136, 197, 20, 5, 79, - 78, 69, 32, 84, 250, 214, 8, 70, 175, 14, 84, 58, 48, 4, 73, 86, 69, 32, - 125, 4, 79, 85, 82, 32, 26, 70, 83, 206, 1, 66, 250, 12, 65, 82, 71, 250, - 109, 85, 175, 202, 26, 68, 6, 182, 15, 72, 165, 212, 18, 5, 73, 88, 84, - 72, 83, 32, 150, 1, 66, 44, 18, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, - 82, 77, 32, 76, 73, 77, 77, 85, 206, 12, 65, 82, 71, 26, 83, 226, 109, - 85, 175, 202, 26, 68, 6, 188, 123, 3, 65, 78, 50, 139, 252, 25, 85, 9, - 194, 156, 12, 32, 195, 231, 18, 52, 24, 44, 4, 71, 73, 68, 65, 33, 3, 78, - 69, 32, 4, 146, 158, 29, 69, 227, 101, 77, 20, 156, 1, 19, 86, 65, 82, - 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 73, 76, 73, 77, 77, 85, 174, 7, - 83, 214, 113, 85, 238, 50, 71, 194, 151, 26, 68, 171, 217, 1, 65, 9, 182, - 255, 30, 32, 186, 2, 51, 3, 52, 28, 92, 16, 76, 68, 32, 65, 83, 83, 89, - 82, 73, 65, 78, 32, 79, 78, 69, 32, 37, 3, 78, 69, 32, 4, 170, 134, 18, - 83, 203, 250, 10, 81, 24, 166, 1, 69, 52, 8, 81, 85, 65, 82, 84, 69, 82, - 32, 206, 7, 66, 54, 71, 84, 5, 84, 72, 73, 82, 68, 224, 244, 23, 2, 83, - 72, 241, 136, 6, 6, 72, 65, 76, 70, 32, 71, 4, 158, 8, 83, 189, 164, 1, - 5, 73, 71, 72, 84, 72, 4, 202, 153, 29, 65, 219, 108, 71, 38, 144, 1, 5, - 69, 86, 69, 78, 32, 188, 1, 20, 72, 65, 82, 50, 32, 84, 73, 77, 69, 83, - 32, 71, 65, 76, 32, 80, 76, 85, 83, 32, 37, 3, 73, 88, 32, 18, 148, 1, - 17, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 73, 77, 73, 78, - 218, 1, 83, 214, 113, 85, 238, 50, 71, 194, 151, 26, 68, 171, 217, 1, 65, - 6, 218, 148, 12, 32, 195, 231, 18, 51, 4, 178, 189, 27, 68, 139, 191, 2, - 77, 16, 142, 1, 83, 142, 3, 65, 202, 110, 85, 238, 50, 71, 220, 130, 26, - 16, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 65, 83, 72, 231, - 20, 68, 2, 227, 61, 72, 50, 52, 5, 72, 82, 69, 69, 32, 241, 1, 3, 87, 79, - 32, 28, 142, 1, 66, 40, 4, 83, 72, 65, 82, 24, 16, 86, 65, 82, 73, 65, - 78, 84, 32, 70, 79, 82, 77, 32, 69, 83, 72, 118, 65, 82, 71, 167, 184, - 27, 68, 6, 248, 111, 3, 85, 82, 85, 199, 10, 65, 8, 210, 111, 50, 3, 85, - 4, 250, 144, 24, 49, 203, 3, 50, 22, 82, 65, 30, 66, 32, 2, 69, 83, 22, - 71, 26, 83, 61, 6, 84, 72, 73, 82, 68, 83, 4, 209, 184, 1, 2, 83, 72, 4, - 238, 120, 65, 199, 241, 25, 85, 2, 239, 147, 24, 72, 4, 53, 3, 69, 83, - 72, 4, 11, 72, 4, 17, 2, 65, 82, 4, 174, 246, 30, 50, 3, 85, 4, 11, 32, - 4, 128, 226, 26, 12, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 195, - 85, 68, 10, 160, 1, 9, 68, 73, 65, 71, 79, 78, 65, 76, 32, 188, 241, 24, - 6, 86, 69, 82, 84, 73, 67, 169, 225, 2, 14, 79, 76, 68, 32, 65, 83, 83, - 89, 82, 73, 65, 78, 32, 87, 6, 192, 190, 27, 2, 84, 82, 204, 229, 2, 4, - 81, 85, 65, 68, 15, 67, 192, 17, 202, 1, 65, 186, 15, 66, 170, 5, 68, - 190, 15, 69, 202, 10, 71, 182, 30, 72, 238, 3, 73, 238, 4, 75, 138, 20, + 67, 32, 53, 7, 79, 83, 83, 69, 68, 32, 83, 8, 50, 83, 226, 4, 71, 210, + 133, 32, 69, 219, 7, 78, 2, 207, 220, 24, 72, 12, 80, 9, 73, 65, 76, 69, + 67, 84, 45, 80, 32, 244, 255, 31, 2, 65, 76, 175, 17, 69, 8, 182, 156, 8, + 72, 250, 150, 8, 65, 244, 183, 12, 2, 75, 65, 255, 165, 3, 78, 4, 190, 3, + 78, 223, 255, 31, 77, 4, 190, 198, 25, 79, 215, 150, 5, 65, 8, 42, 72, + 234, 233, 28, 65, 255, 165, 3, 83, 4, 226, 143, 32, 69, 219, 19, 73, 4, + 176, 191, 19, 7, 45, 83, 72, 65, 80, 69, 68, 155, 241, 7, 65, 35, 36, 3, + 76, 68, 32, 207, 253, 31, 79, 30, 76, 7, 67, 79, 80, 84, 73, 67, 32, 189, + 1, 7, 78, 85, 66, 73, 65, 78, 32, 22, 98, 71, 42, 72, 184, 1, 2, 83, 72, + 154, 222, 14, 68, 142, 248, 9, 79, 222, 224, 5, 69, 207, 104, 65, 2, 17, + 2, 65, 78, 2, 183, 251, 15, 71, 8, 138, 195, 25, 79, 238, 125, 65, 139, + 204, 5, 69, 8, 50, 78, 228, 177, 16, 2, 83, 72, 155, 158, 5, 87, 4, 154, + 140, 32, 71, 3, 89, 10, 54, 72, 138, 151, 6, 65, 178, 227, 25, 79, 219, + 3, 73, 4, 142, 254, 31, 73, 187, 13, 69, 4, 240, 241, 31, 3, 72, 69, 84, + 167, 8, 65, 2, 135, 250, 31, 65, 14, 62, 75, 30, 77, 2, 80, 22, 83, 241, + 234, 27, 3, 84, 65, 85, 4, 26, 72, 143, 138, 32, 65, 2, 131, 235, 27, 73, + 4, 188, 175, 16, 6, 72, 73, 77, 65, 32, 83, 193, 254, 3, 3, 84, 65, 85, + 4, 188, 200, 24, 3, 76, 69, 70, 245, 145, 2, 4, 82, 73, 71, 72, 6, 96, 6, + 78, 73, 83, 72, 32, 86, 204, 200, 25, 8, 82, 69, 83, 80, 79, 78, 68, 83, + 231, 201, 5, 65, 2, 213, 229, 28, 4, 69, 82, 83, 69, 44, 104, 2, 78, 84, + 156, 132, 3, 6, 67, 72, 32, 65, 78, 68, 253, 142, 28, 8, 80, 76, 69, 32, + 87, 73, 84, 72, 40, 56, 2, 69, 82, 37, 8, 73, 78, 71, 32, 82, 79, 68, 32, + 4, 190, 181, 27, 66, 195, 131, 1, 83, 36, 48, 4, 84, 69, 78, 83, 1, 4, + 85, 78, 73, 84, 18, 133, 220, 23, 2, 32, 68, 51, 82, 69, 76, 5, 73, 67, + 75, 69, 84, 34, 79, 198, 5, 85, 50, 89, 143, 214, 30, 65, 4, 196, 168, 8, + 3, 68, 73, 84, 181, 201, 20, 7, 83, 67, 69, 78, 84, 32, 77, 5, 169, 247, + 26, 3, 32, 66, 65, 28, 84, 2, 83, 83, 152, 242, 29, 3, 67, 79, 68, 152, + 155, 1, 3, 73, 83, 83, 255, 57, 87, 22, 46, 32, 184, 2, 3, 69, 68, 32, + 223, 1, 73, 14, 44, 3, 79, 70, 32, 82, 80, 179, 212, 31, 77, 4, 224, 172, + 28, 6, 74, 69, 82, 85, 83, 65, 145, 208, 2, 5, 76, 79, 82, 82, 65, 8, 76, + 10, 65, 84, 84, 89, 32, 87, 73, 84, 72, 32, 41, 5, 79, 77, 77, 69, 69, 4, + 218, 165, 10, 82, 25, 3, 76, 69, 70, 5, 189, 161, 3, 11, 32, 87, 73, 84, + 72, 32, 72, 65, 76, 70, 45, 6, 204, 132, 25, 37, 78, 69, 71, 65, 84, 73, + 86, 69, 32, 83, 81, 85, 65, 82, 69, 68, 32, 76, 65, 84, 73, 78, 32, 67, + 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 192, 221, 2, 2, 70, + 76, 189, 210, 2, 3, 83, 87, 79, 2, 145, 235, 23, 5, 78, 71, 32, 76, 65, + 4, 248, 199, 17, 3, 90, 69, 73, 255, 167, 14, 84, 6, 130, 220, 19, 73, + 189, 149, 7, 4, 83, 84, 65, 76, 206, 19, 154, 1, 66, 20, 8, 78, 69, 73, + 70, 79, 82, 77, 32, 154, 250, 1, 80, 114, 82, 180, 3, 2, 83, 84, 242, + 219, 15, 67, 161, 229, 9, 6, 84, 32, 79, 70, 32, 77, 2, 211, 175, 5, 69, + 170, 19, 176, 1, 13, 78, 85, 77, 69, 82, 73, 67, 32, 83, 73, 71, 78, 32, + 184, 20, 17, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 83, 73, 71, + 78, 32, 233, 1, 5, 83, 73, 71, 78, 32, 222, 1, 78, 69, 166, 2, 70, 152, + 3, 2, 78, 73, 154, 2, 79, 130, 3, 83, 151, 4, 84, 24, 68, 5, 73, 71, 72, + 84, 32, 153, 1, 7, 76, 65, 77, 73, 84, 69, 32, 16, 142, 14, 83, 230, 113, + 85, 238, 50, 71, 136, 56, 17, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, + 77, 32, 85, 83, 83, 85, 214, 193, 26, 68, 219, 248, 1, 65, 8, 232, 140, + 21, 5, 79, 78, 69, 32, 84, 246, 144, 9, 70, 175, 14, 84, 58, 48, 4, 73, + 86, 69, 32, 125, 4, 79, 85, 82, 32, 26, 70, 83, 206, 1, 66, 250, 12, 65, + 82, 71, 138, 110, 85, 203, 172, 27, 68, 6, 182, 15, 72, 173, 144, 19, 5, + 73, 88, 84, 72, 83, 32, 150, 1, 66, 44, 18, 86, 65, 82, 73, 65, 78, 84, + 32, 70, 79, 82, 77, 32, 76, 73, 77, 77, 85, 206, 12, 65, 82, 71, 26, 83, + 242, 109, 85, 203, 172, 27, 68, 6, 204, 123, 3, 65, 78, 50, 163, 212, 26, + 85, 9, 214, 202, 12, 32, 143, 190, 19, 52, 24, 44, 4, 71, 73, 68, 65, 33, + 3, 78, 69, 32, 4, 238, 159, 30, 69, 207, 104, 77, 20, 156, 1, 19, 86, 65, + 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 73, 76, 73, 77, 77, 85, 174, + 7, 83, 230, 113, 85, 238, 50, 71, 222, 249, 26, 68, 219, 248, 1, 65, 9, + 150, 132, 32, 32, 186, 2, 51, 3, 52, 28, 92, 16, 76, 68, 32, 65, 83, 83, + 89, 82, 73, 65, 78, 32, 79, 78, 69, 32, 37, 3, 78, 69, 32, 4, 154, 194, + 18, 83, 175, 192, 11, 81, 24, 166, 1, 69, 52, 8, 81, 85, 65, 82, 84, 69, + 82, 32, 206, 7, 66, 54, 71, 84, 5, 84, 72, 73, 82, 68, 200, 194, 24, 2, + 83, 72, 217, 191, 6, 6, 72, 65, 76, 70, 32, 71, 4, 158, 8, 83, 205, 164, + 1, 5, 73, 71, 72, 84, 72, 4, 166, 155, 30, 65, 207, 111, 71, 38, 144, 1, + 5, 69, 86, 69, 78, 32, 188, 1, 20, 72, 65, 82, 50, 32, 84, 73, 77, 69, + 83, 32, 71, 65, 76, 32, 80, 76, 85, 83, 32, 37, 3, 73, 88, 32, 18, 148, + 1, 17, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 73, 77, 73, + 78, 218, 1, 83, 230, 113, 85, 238, 50, 71, 222, 249, 26, 68, 219, 248, 1, + 65, 6, 238, 194, 12, 32, 143, 190, 19, 51, 4, 222, 159, 28, 68, 167, 225, + 2, 77, 16, 142, 1, 83, 142, 3, 65, 218, 110, 85, 238, 50, 71, 144, 229, + 26, 16, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 65, 83, 72, + 207, 20, 68, 2, 227, 61, 72, 50, 52, 5, 72, 82, 69, 69, 32, 241, 1, 3, + 87, 79, 32, 28, 142, 1, 66, 40, 4, 83, 72, 65, 82, 24, 16, 86, 65, 82, + 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 69, 83, 72, 118, 65, 82, 71, 211, + 154, 28, 68, 6, 136, 112, 3, 85, 82, 85, 199, 10, 65, 8, 226, 111, 50, 3, + 85, 4, 130, 224, 24, 49, 203, 3, 50, 22, 82, 65, 30, 66, 32, 2, 69, 83, + 22, 71, 26, 83, 61, 6, 84, 72, 73, 82, 68, 83, 4, 225, 184, 1, 2, 83, 72, + 4, 254, 120, 65, 223, 201, 26, 85, 2, 247, 226, 24, 72, 4, 53, 3, 69, 83, + 72, 4, 11, 72, 4, 17, 2, 65, 82, 4, 142, 251, 31, 50, 3, 85, 4, 11, 32, + 4, 196, 186, 27, 12, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 171, + 95, 68, 10, 160, 1, 9, 68, 73, 65, 71, 79, 78, 65, 76, 32, 204, 203, 25, + 6, 86, 69, 82, 84, 73, 67, 133, 247, 2, 14, 79, 76, 68, 32, 65, 83, 83, + 89, 82, 73, 65, 78, 32, 87, 6, 192, 173, 28, 2, 84, 82, 160, 251, 2, 4, + 81, 85, 65, 68, 15, 67, 194, 17, 202, 1, 65, 186, 15, 66, 170, 5, 68, + 190, 15, 69, 202, 10, 71, 182, 30, 72, 238, 3, 73, 238, 4, 75, 154, 20, 76, 134, 37, 77, 134, 6, 78, 226, 17, 80, 210, 3, 82, 42, 83, 238, 19, 84, 186, 8, 85, 207, 20, 90, 141, 1, 160, 1, 7, 32, 84, 73, 77, 69, 83, 32, 142, 1, 66, 250, 3, 68, 38, 75, 90, 76, 212, 1, 3, 77, 65, 82, 78, - 78, 134, 2, 82, 42, 83, 210, 230, 29, 80, 191, 127, 50, 16, 128, 173, 1, - 5, 76, 65, 71, 65, 82, 226, 23, 71, 166, 187, 12, 73, 198, 144, 13, 77, - 142, 250, 1, 83, 226, 159, 1, 66, 234, 67, 72, 187, 2, 65, 45, 22, 32, - 219, 2, 50, 28, 48, 6, 84, 73, 77, 69, 83, 32, 251, 202, 1, 71, 26, 180, - 1, 2, 71, 65, 38, 73, 36, 2, 83, 72, 240, 112, 8, 85, 32, 80, 76, 85, 83, + 78, 134, 2, 82, 42, 83, 154, 235, 30, 80, 215, 127, 50, 16, 144, 173, 1, + 5, 76, 65, 71, 65, 82, 226, 23, 71, 246, 239, 12, 73, 154, 182, 13, 77, + 182, 161, 2, 83, 218, 162, 1, 66, 246, 67, 72, 187, 2, 65, 45, 22, 32, + 219, 2, 50, 28, 48, 6, 84, 73, 77, 69, 83, 32, 139, 203, 1, 71, 26, 180, + 1, 2, 71, 65, 38, 73, 36, 2, 83, 72, 128, 113, 8, 85, 32, 80, 76, 85, 83, 32, 85, 188, 56, 4, 68, 85, 78, 51, 166, 2, 65, 204, 2, 3, 78, 85, 78, - 242, 27, 76, 191, 160, 29, 72, 4, 178, 210, 1, 78, 151, 155, 29, 76, 4, - 194, 169, 1, 71, 167, 196, 28, 77, 4, 170, 137, 24, 85, 187, 254, 4, 69, - 15, 37, 7, 32, 84, 73, 77, 69, 83, 32, 12, 190, 133, 1, 77, 130, 59, 71, - 194, 9, 83, 234, 10, 84, 192, 177, 27, 2, 66, 65, 143, 230, 1, 65, 5, + 242, 27, 76, 143, 165, 30, 72, 4, 194, 210, 1, 78, 231, 159, 30, 76, 4, + 210, 169, 1, 71, 223, 200, 29, 77, 4, 178, 216, 24, 85, 143, 177, 5, 69, + 15, 37, 7, 32, 84, 73, 77, 69, 83, 32, 12, 206, 133, 1, 77, 130, 59, 71, + 194, 9, 83, 234, 10, 84, 140, 179, 28, 2, 66, 65, 147, 233, 1, 65, 5, 249, 51, 5, 32, 84, 73, 77, 69, 7, 37, 7, 32, 84, 73, 77, 69, 83, 32, 4, - 252, 27, 5, 83, 72, 73, 84, 65, 199, 80, 69, 23, 68, 7, 32, 84, 73, 77, - 69, 83, 32, 202, 240, 24, 69, 247, 169, 5, 65, 16, 102, 75, 140, 173, 1, - 2, 68, 73, 218, 253, 25, 71, 170, 217, 1, 85, 130, 92, 65, 146, 92, 83, - 215, 42, 72, 4, 242, 157, 1, 65, 183, 203, 29, 73, 7, 37, 7, 32, 84, 73, - 77, 69, 83, 32, 4, 130, 248, 3, 75, 207, 195, 26, 83, 13, 26, 32, 147, - 187, 30, 83, 8, 128, 1, 10, 80, 76, 85, 83, 32, 78, 65, 71, 65, 32, 148, - 164, 8, 7, 84, 72, 82, 69, 69, 32, 84, 245, 220, 5, 4, 79, 86, 69, 82, 4, - 128, 140, 1, 16, 79, 80, 80, 79, 83, 73, 78, 71, 32, 65, 78, 32, 80, 76, - 85, 83, 163, 226, 22, 83, 6, 200, 49, 2, 65, 68, 175, 219, 17, 75, 18, - 26, 72, 239, 245, 11, 65, 17, 42, 32, 138, 140, 18, 71, 191, 217, 12, 50, - 10, 68, 9, 79, 86, 69, 82, 32, 65, 83, 72, 32, 158, 97, 90, 135, 112, 75, - 6, 176, 1, 8, 79, 86, 69, 82, 32, 65, 83, 72, 141, 143, 1, 29, 84, 85, + 252, 27, 5, 83, 72, 73, 84, 65, 215, 80, 69, 23, 68, 7, 32, 84, 73, 77, + 69, 83, 32, 218, 202, 25, 69, 187, 212, 5, 65, 16, 102, 75, 156, 173, 1, + 2, 68, 73, 246, 223, 26, 71, 218, 248, 1, 85, 238, 94, 65, 170, 92, 83, + 215, 42, 72, 4, 130, 158, 1, 65, 135, 208, 30, 73, 7, 37, 7, 32, 84, 73, + 77, 69, 83, 32, 4, 210, 253, 3, 75, 223, 194, 27, 83, 13, 26, 32, 243, + 191, 31, 83, 8, 128, 1, 10, 80, 76, 85, 83, 32, 78, 65, 71, 65, 32, 128, + 185, 8, 7, 84, 72, 82, 69, 69, 32, 84, 177, 252, 5, 4, 79, 86, 69, 82, 4, + 144, 140, 1, 16, 79, 80, 80, 79, 83, 73, 78, 71, 32, 65, 78, 32, 80, 76, + 85, 83, 251, 175, 23, 83, 6, 200, 49, 2, 65, 68, 239, 151, 18, 75, 18, + 26, 72, 131, 164, 12, 65, 17, 42, 32, 202, 200, 18, 71, 223, 161, 13, 50, + 10, 68, 9, 79, 86, 69, 82, 32, 65, 83, 72, 32, 174, 97, 90, 135, 112, 75, + 6, 176, 1, 8, 79, 86, 69, 82, 32, 65, 83, 72, 157, 143, 1, 29, 84, 85, 71, 50, 32, 79, 86, 69, 82, 32, 84, 85, 71, 50, 32, 84, 85, 71, 50, 32, - 79, 86, 69, 82, 32, 84, 85, 71, 50, 5, 133, 145, 1, 27, 32, 67, 82, 79, + 79, 86, 69, 82, 32, 84, 85, 71, 50, 5, 149, 145, 1, 27, 32, 67, 82, 79, 83, 83, 73, 78, 71, 32, 65, 83, 72, 32, 79, 86, 69, 82, 32, 65, 83, 72, 32, 79, 86, 69, 82, 52, 30, 65, 158, 2, 73, 95, 85, 27, 66, 68, 44, 4, - 72, 65, 82, 50, 90, 76, 66, 82, 183, 140, 27, 71, 5, 129, 208, 1, 6, 32, - 84, 73, 77, 69, 83, 9, 37, 7, 32, 84, 73, 77, 69, 83, 32, 6, 222, 164, 1, - 65, 218, 168, 29, 78, 163, 17, 90, 7, 180, 215, 29, 7, 32, 79, 86, 69, - 82, 32, 66, 215, 136, 1, 65, 5, 203, 252, 23, 65, 9, 37, 7, 32, 84, 73, - 77, 69, 83, 32, 6, 246, 155, 1, 73, 198, 161, 28, 71, 171, 162, 1, 65, - 19, 50, 32, 168, 1, 3, 76, 85, 71, 223, 169, 1, 82, 8, 88, 8, 79, 86, 69, - 82, 32, 66, 85, 32, 217, 222, 26, 8, 67, 82, 79, 83, 83, 73, 78, 71, 6, - 144, 152, 12, 7, 84, 73, 77, 69, 83, 32, 78, 234, 139, 17, 65, 211, 106, - 85, 5, 237, 236, 3, 8, 32, 79, 86, 69, 82, 32, 66, 85, 180, 1, 34, 65, - 222, 5, 73, 195, 2, 85, 67, 50, 71, 150, 5, 82, 146, 49, 32, 163, 166, - 30, 77, 55, 26, 32, 171, 220, 30, 51, 50, 76, 13, 75, 73, 83, 73, 77, 53, - 32, 84, 73, 77, 69, 83, 32, 211, 198, 1, 84, 48, 146, 1, 65, 30, 66, 38, - 71, 112, 2, 73, 82, 42, 76, 94, 85, 166, 131, 1, 80, 162, 61, 84, 234, - 232, 26, 75, 146, 152, 2, 78, 254, 2, 83, 163, 17, 72, 4, 110, 32, 175, - 183, 29, 77, 4, 154, 244, 28, 65, 143, 230, 1, 73, 10, 34, 65, 58, 73, - 151, 147, 30, 85, 5, 11, 32, 2, 169, 244, 28, 6, 80, 76, 85, 83, 32, 77, - 5, 211, 245, 23, 82, 5, 181, 203, 13, 5, 32, 80, 76, 85, 83, 8, 26, 85, - 211, 216, 30, 65, 7, 144, 151, 1, 7, 32, 80, 76, 85, 83, 32, 77, 191, - 193, 29, 77, 6, 52, 7, 50, 32, 80, 76, 85, 83, 32, 203, 214, 30, 83, 4, - 128, 27, 2, 71, 73, 175, 215, 28, 77, 7, 171, 147, 12, 65, 25, 54, 77, - 150, 1, 78, 76, 2, 83, 72, 135, 213, 30, 66, 13, 44, 7, 32, 84, 73, 77, - 69, 83, 32, 59, 50, 6, 220, 69, 2, 85, 32, 134, 160, 13, 73, 199, 195, - 16, 83, 5, 153, 228, 12, 6, 32, 84, 73, 77, 69, 83, 5, 173, 179, 18, 14, - 32, 75, 65, 83, 75, 65, 76, 32, 85, 32, 71, 85, 78, 85, 5, 229, 157, 1, - 5, 32, 80, 76, 85, 83, 91, 70, 32, 66, 66, 94, 71, 234, 4, 78, 190, 234, - 23, 82, 215, 227, 6, 72, 6, 138, 176, 1, 71, 170, 3, 83, 249, 144, 12, 4, - 79, 86, 69, 82, 9, 52, 7, 32, 84, 73, 77, 69, 83, 32, 159, 211, 30, 50, - 4, 234, 145, 1, 69, 167, 148, 29, 83, 61, 52, 7, 32, 84, 73, 77, 69, 83, - 32, 175, 140, 30, 85, 56, 122, 65, 58, 68, 30, 71, 74, 75, 90, 76, 110, - 77, 50, 83, 242, 79, 69, 218, 58, 73, 250, 139, 16, 72, 158, 164, 13, 78, - 3, 80, 6, 32, 2, 83, 72, 139, 164, 30, 78, 5, 143, 154, 14, 32, 4, 182, - 129, 30, 73, 3, 85, 8, 26, 73, 215, 208, 30, 65, 7, 252, 140, 1, 2, 82, - 50, 199, 194, 29, 83, 8, 26, 85, 195, 185, 1, 65, 6, 40, 4, 83, 72, 85, - 50, 227, 207, 30, 82, 5, 175, 26, 32, 8, 26, 65, 45, 2, 85, 72, 6, 202, - 26, 77, 197, 207, 23, 3, 75, 45, 48, 2, 225, 57, 5, 32, 80, 76, 85, 83, - 6, 198, 233, 28, 65, 206, 167, 1, 69, 211, 61, 73, 4, 222, 138, 1, 73, - 131, 173, 29, 72, 11, 26, 51, 247, 205, 30, 52, 7, 143, 9, 32, 117, 130, + 72, 65, 82, 50, 90, 76, 66, 82, 251, 238, 27, 71, 5, 145, 208, 1, 6, 32, + 84, 73, 77, 69, 83, 9, 37, 7, 32, 84, 73, 77, 69, 83, 32, 6, 238, 164, 1, + 65, 170, 173, 30, 78, 163, 17, 90, 7, 252, 219, 30, 7, 32, 79, 86, 69, + 82, 32, 66, 239, 136, 1, 65, 5, 211, 203, 24, 65, 9, 37, 7, 32, 84, 73, + 77, 69, 83, 32, 6, 134, 156, 1, 73, 134, 167, 29, 71, 187, 161, 1, 65, + 19, 50, 32, 168, 1, 3, 76, 85, 71, 239, 169, 1, 82, 8, 88, 8, 79, 86, 69, + 82, 32, 66, 85, 32, 129, 183, 27, 8, 67, 82, 79, 83, 83, 73, 78, 71, 6, + 164, 198, 12, 7, 84, 73, 77, 69, 83, 32, 78, 246, 224, 17, 65, 135, 108, + 85, 5, 189, 242, 3, 8, 32, 79, 86, 69, 82, 32, 66, 85, 180, 1, 34, 65, + 222, 5, 73, 195, 2, 85, 67, 50, 71, 150, 5, 82, 146, 49, 32, 131, 171, + 31, 77, 55, 26, 32, 139, 225, 31, 51, 50, 76, 13, 75, 73, 83, 73, 77, 53, + 32, 84, 73, 77, 69, 83, 32, 227, 198, 1, 84, 48, 146, 1, 65, 30, 66, 38, + 71, 112, 2, 73, 82, 42, 76, 94, 85, 182, 131, 1, 80, 162, 61, 84, 218, + 230, 27, 75, 242, 158, 2, 78, 254, 2, 83, 163, 17, 72, 4, 110, 32, 255, + 188, 30, 77, 4, 246, 245, 29, 65, 147, 233, 1, 73, 10, 34, 65, 58, 73, + 235, 151, 31, 85, 5, 11, 32, 2, 133, 246, 29, 6, 80, 76, 85, 83, 32, 77, + 5, 219, 196, 24, 82, 5, 229, 255, 13, 5, 32, 80, 76, 85, 83, 8, 26, 85, + 179, 221, 31, 65, 7, 160, 151, 1, 7, 32, 80, 76, 85, 83, 32, 77, 143, + 198, 30, 77, 6, 52, 7, 50, 32, 80, 76, 85, 83, 32, 171, 219, 31, 83, 4, + 128, 27, 2, 71, 73, 139, 217, 29, 77, 7, 191, 193, 12, 65, 25, 54, 77, + 150, 1, 78, 76, 2, 83, 72, 231, 217, 31, 66, 13, 44, 7, 32, 84, 73, 77, + 69, 83, 32, 59, 50, 6, 220, 69, 2, 85, 32, 230, 212, 13, 73, 199, 147, + 17, 83, 5, 221, 152, 13, 6, 32, 84, 73, 77, 69, 83, 5, 181, 239, 18, 14, + 32, 75, 65, 83, 75, 65, 76, 32, 85, 32, 71, 85, 78, 85, 5, 245, 157, 1, + 5, 32, 80, 76, 85, 83, 91, 70, 32, 66, 66, 94, 71, 234, 4, 78, 198, 185, + 24, 82, 175, 153, 7, 72, 6, 154, 176, 1, 71, 170, 3, 83, 153, 197, 12, 4, + 79, 86, 69, 82, 9, 52, 7, 32, 84, 73, 77, 69, 83, 32, 255, 215, 31, 50, + 4, 250, 145, 1, 69, 247, 152, 30, 83, 61, 52, 7, 32, 84, 73, 77, 69, 83, + 32, 131, 145, 31, 85, 56, 122, 65, 58, 68, 30, 71, 74, 75, 90, 76, 110, + 77, 50, 83, 130, 80, 69, 218, 58, 73, 190, 198, 16, 72, 170, 238, 13, 78, + 3, 80, 6, 32, 2, 83, 72, 235, 168, 31, 78, 5, 215, 206, 14, 32, 4, 138, + 134, 31, 73, 3, 85, 8, 26, 73, 183, 213, 31, 65, 7, 140, 141, 1, 2, 82, + 50, 151, 199, 30, 83, 8, 26, 85, 211, 185, 1, 65, 6, 40, 4, 83, 72, 85, + 50, 195, 212, 31, 82, 5, 175, 26, 32, 8, 26, 65, 45, 2, 85, 72, 6, 202, + 26, 77, 205, 158, 24, 3, 75, 45, 48, 2, 225, 57, 5, 32, 80, 76, 85, 83, + 6, 162, 235, 29, 65, 198, 170, 1, 69, 223, 61, 73, 4, 238, 138, 1, 73, + 211, 177, 30, 72, 11, 26, 51, 215, 210, 31, 52, 7, 143, 9, 32, 117, 130, 1, 32, 90, 50, 210, 1, 78, 202, 1, 82, 60, 3, 83, 72, 50, 52, 3, 90, 69, - 78, 230, 164, 18, 71, 226, 163, 11, 68, 191, 127, 76, 4, 168, 25, 11, 79, - 86, 69, 82, 32, 69, 32, 78, 85, 78, 32, 237, 94, 4, 84, 73, 77, 69, 19, - 37, 7, 32, 84, 73, 77, 69, 83, 32, 16, 134, 1, 83, 128, 168, 1, 10, 65, - 32, 80, 76, 85, 83, 32, 72, 65, 32, 178, 128, 28, 71, 214, 17, 80, 146, - 24, 75, 238, 100, 77, 219, 19, 85, 4, 186, 193, 29, 65, 203, 114, 72, 15, + 78, 170, 225, 18, 71, 230, 235, 11, 68, 215, 127, 76, 4, 168, 25, 11, 79, + 86, 69, 82, 32, 69, 32, 78, 85, 78, 32, 253, 94, 4, 84, 73, 77, 69, 19, + 37, 7, 32, 84, 73, 77, 69, 83, 32, 16, 134, 1, 83, 144, 168, 1, 10, 65, + 32, 80, 76, 85, 83, 32, 72, 65, 32, 242, 133, 29, 71, 206, 16, 80, 154, + 24, 75, 254, 100, 77, 219, 19, 85, 4, 130, 198, 30, 65, 227, 114, 72, 15, 11, 32, 12, 96, 4, 67, 82, 79, 83, 0, 4, 79, 80, 80, 79, 36, 6, 84, 73, - 77, 69, 83, 32, 179, 208, 23, 83, 2, 189, 222, 13, 4, 83, 73, 78, 71, 6, - 188, 138, 1, 4, 71, 65, 78, 50, 147, 168, 29, 77, 6, 36, 3, 73, 78, 50, - 247, 248, 29, 69, 5, 187, 188, 29, 32, 5, 229, 16, 9, 32, 67, 82, 79, 83, + 77, 69, 83, 32, 155, 158, 24, 83, 2, 237, 146, 14, 4, 83, 73, 78, 71, 6, + 204, 138, 1, 4, 71, 65, 78, 50, 227, 172, 30, 77, 6, 36, 3, 73, 78, 50, + 203, 253, 30, 69, 5, 131, 193, 30, 32, 5, 229, 16, 9, 32, 67, 82, 79, 83, 83, 73, 78, 71, 63, 11, 32, 60, 96, 14, 83, 72, 69, 83, 72, 73, 71, 32, - 84, 73, 77, 69, 83, 32, 97, 6, 84, 73, 77, 69, 83, 32, 16, 162, 131, 1, - 73, 130, 219, 2, 77, 210, 131, 25, 65, 128, 105, 2, 76, 65, 182, 87, 83, + 84, 73, 77, 69, 83, 32, 97, 6, 84, 73, 77, 69, 83, 32, 16, 178, 131, 1, + 73, 194, 224, 2, 77, 222, 255, 25, 65, 244, 107, 2, 76, 65, 198, 87, 83, 147, 17, 72, 44, 134, 1, 65, 68, 5, 68, 85, 78, 51, 32, 26, 75, 58, 76, - 62, 83, 34, 85, 206, 127, 73, 204, 31, 2, 72, 65, 226, 249, 27, 71, 163, - 100, 66, 9, 228, 86, 9, 32, 80, 76, 85, 83, 32, 76, 65, 76, 195, 238, 29, - 78, 4, 217, 32, 2, 71, 85, 6, 192, 203, 23, 5, 65, 83, 75, 65, 76, 159, - 165, 3, 85, 8, 34, 65, 242, 195, 30, 73, 3, 85, 5, 185, 85, 2, 76, 32, 4, - 130, 173, 30, 72, 215, 22, 85, 4, 182, 195, 30, 50, 3, 68, 160, 2, 42, + 62, 83, 34, 85, 222, 127, 73, 204, 31, 2, 72, 65, 150, 253, 28, 71, 179, + 101, 66, 9, 244, 86, 9, 32, 80, 76, 85, 83, 32, 76, 65, 76, 147, 243, 30, + 78, 4, 217, 32, 2, 71, 85, 6, 168, 153, 24, 5, 65, 83, 75, 65, 76, 251, + 185, 3, 85, 8, 34, 65, 210, 200, 31, 73, 3, 85, 5, 201, 85, 2, 76, 32, 4, + 226, 177, 31, 72, 215, 22, 85, 4, 150, 200, 31, 50, 3, 68, 160, 2, 42, 65, 166, 19, 69, 122, 73, 135, 5, 85, 191, 1, 114, 50, 144, 16, 2, 66, - 65, 94, 68, 22, 76, 38, 78, 190, 140, 1, 32, 154, 6, 82, 218, 143, 27, - 83, 167, 142, 2, 77, 153, 1, 11, 32, 150, 1, 68, 6, 84, 73, 77, 69, 83, - 32, 137, 128, 1, 5, 79, 86, 69, 82, 32, 148, 1, 202, 1, 65, 162, 2, 66, + 65, 94, 68, 22, 76, 38, 78, 206, 140, 1, 32, 154, 6, 82, 146, 145, 28, + 83, 191, 145, 2, 77, 153, 1, 11, 32, 150, 1, 68, 6, 84, 73, 77, 69, 83, + 32, 153, 128, 1, 5, 79, 86, 69, 82, 32, 148, 1, 202, 1, 65, 162, 2, 66, 154, 1, 68, 178, 1, 69, 58, 71, 178, 1, 72, 230, 1, 73, 70, 75, 194, 1, - 76, 62, 77, 50, 78, 130, 1, 83, 142, 1, 85, 206, 154, 1, 84, 248, 179, - 22, 3, 90, 73, 90, 159, 225, 6, 80, 18, 132, 1, 6, 32, 80, 76, 85, 83, - 32, 50, 78, 52, 2, 83, 72, 161, 228, 17, 14, 66, 50, 32, 84, 69, 78, 85, - 32, 80, 76, 85, 83, 32, 84, 6, 186, 58, 68, 182, 147, 13, 73, 155, 238, - 16, 72, 5, 169, 61, 9, 32, 80, 76, 85, 83, 32, 75, 65, 75, 7, 11, 50, 5, - 245, 89, 6, 32, 80, 76, 85, 83, 32, 10, 26, 65, 77, 2, 85, 82, 6, 42, 72, - 44, 2, 82, 32, 183, 188, 30, 68, 2, 11, 65, 2, 255, 216, 23, 82, 5, 11, - 32, 2, 229, 241, 29, 4, 80, 76, 85, 83, 14, 34, 73, 54, 85, 187, 187, 30, - 65, 7, 17, 2, 77, 32, 4, 146, 22, 84, 191, 129, 1, 71, 6, 56, 8, 71, 32, - 84, 73, 77, 69, 83, 32, 255, 186, 30, 66, 4, 142, 119, 73, 151, 45, 75, - 10, 38, 78, 254, 2, 76, 203, 230, 28, 82, 5, 243, 28, 32, 18, 18, 65, 99, - 73, 11, 26, 82, 231, 158, 1, 78, 7, 33, 6, 32, 80, 76, 85, 83, 32, 4, - 254, 162, 30, 78, 255, 2, 68, 9, 158, 122, 52, 193, 176, 12, 7, 82, 50, - 32, 80, 76, 85, 83, 12, 62, 65, 138, 124, 85, 173, 174, 12, 6, 73, 32, + 76, 62, 77, 50, 78, 130, 1, 83, 142, 1, 85, 222, 154, 1, 84, 240, 130, + 23, 3, 90, 73, 90, 247, 150, 7, 80, 18, 132, 1, 6, 32, 80, 76, 85, 83, + 32, 50, 78, 52, 2, 83, 72, 225, 160, 18, 14, 66, 50, 32, 84, 69, 78, 85, + 32, 80, 76, 85, 83, 32, 84, 6, 202, 58, 68, 134, 200, 13, 73, 155, 190, + 17, 72, 5, 185, 61, 9, 32, 80, 76, 85, 83, 32, 75, 65, 75, 7, 11, 50, 5, + 133, 90, 6, 32, 80, 76, 85, 83, 32, 10, 26, 65, 77, 2, 85, 82, 6, 42, 72, + 44, 2, 82, 32, 151, 193, 31, 68, 2, 11, 65, 2, 135, 168, 24, 82, 5, 11, + 32, 2, 185, 246, 30, 4, 80, 76, 85, 83, 14, 34, 73, 54, 85, 155, 192, 31, + 65, 7, 17, 2, 77, 32, 4, 146, 22, 84, 207, 129, 1, 71, 6, 56, 8, 71, 32, + 84, 73, 77, 69, 83, 32, 223, 191, 31, 66, 4, 158, 119, 73, 151, 45, 75, + 10, 38, 78, 254, 2, 76, 183, 232, 29, 82, 5, 243, 28, 32, 18, 18, 65, 99, + 73, 11, 26, 82, 247, 158, 1, 78, 7, 33, 6, 32, 80, 76, 85, 83, 32, 4, + 222, 167, 31, 78, 255, 2, 68, 9, 174, 122, 52, 225, 228, 12, 7, 82, 50, + 32, 80, 76, 85, 83, 12, 62, 65, 154, 124, 85, 205, 226, 12, 6, 73, 32, 80, 76, 85, 83, 8, 40, 6, 32, 80, 76, 85, 83, 32, 83, 76, 4, 48, 6, 76, - 85, 32, 80, 76, 85, 219, 183, 30, 65, 2, 11, 83, 2, 199, 97, 32, 5, 201, - 169, 13, 5, 32, 80, 76, 85, 83, 4, 160, 100, 10, 83, 72, 32, 80, 76, 85, - 83, 32, 72, 85, 147, 15, 71, 12, 34, 65, 36, 2, 73, 68, 27, 85, 4, 218, - 252, 23, 83, 211, 185, 6, 75, 5, 209, 77, 2, 32, 80, 4, 60, 5, 83, 72, - 85, 50, 32, 181, 127, 5, 51, 32, 80, 76, 85, 2, 189, 158, 1, 3, 80, 76, - 85, 8, 26, 65, 231, 179, 30, 85, 7, 11, 77, 5, 211, 159, 1, 32, 6, 234, - 77, 69, 206, 129, 28, 85, 159, 229, 1, 73, 10, 26, 69, 73, 2, 85, 78, 7, - 33, 6, 32, 80, 76, 85, 83, 32, 4, 142, 208, 23, 69, 255, 207, 6, 71, 5, - 11, 32, 2, 171, 101, 79, 14, 42, 72, 250, 140, 23, 65, 171, 149, 7, 85, - 8, 18, 69, 51, 73, 5, 157, 186, 29, 7, 32, 80, 76, 85, 83, 32, 84, 4, - 178, 178, 30, 68, 3, 77, 7, 11, 68, 5, 161, 162, 13, 5, 32, 80, 76, 85, - 83, 7, 11, 32, 4, 170, 216, 28, 82, 133, 172, 1, 11, 67, 82, 79, 83, 83, - 73, 78, 71, 32, 71, 65, 5, 175, 131, 1, 32, 7, 246, 130, 1, 32, 167, 157, - 29, 65, 11, 11, 50, 9, 11, 32, 6, 80, 8, 67, 82, 79, 83, 83, 73, 78, 71, - 0, 4, 79, 86, 69, 82, 255, 221, 25, 84, 2, 181, 49, 3, 32, 71, 65, 8, 44, - 5, 83, 72, 84, 73, 78, 199, 203, 23, 50, 7, 37, 7, 32, 84, 73, 77, 69, - 83, 32, 4, 170, 182, 29, 75, 199, 120, 85, 49, 70, 32, 94, 52, 114, 82, - 190, 1, 83, 202, 168, 29, 68, 187, 130, 1, 71, 6, 172, 232, 8, 6, 84, 73, - 77, 69, 83, 32, 253, 212, 4, 8, 67, 82, 79, 83, 83, 73, 78, 71, 7, 11, + 85, 32, 80, 76, 85, 187, 188, 31, 65, 2, 11, 83, 2, 215, 97, 32, 5, 249, + 221, 13, 5, 32, 80, 76, 85, 83, 4, 176, 100, 10, 83, 72, 32, 80, 76, 85, + 83, 32, 72, 85, 147, 15, 71, 12, 34, 65, 36, 2, 73, 68, 27, 85, 4, 250, + 204, 24, 83, 147, 238, 6, 75, 5, 225, 77, 2, 32, 80, 4, 60, 5, 83, 72, + 85, 50, 32, 197, 127, 5, 51, 32, 80, 76, 85, 2, 205, 158, 1, 3, 80, 76, + 85, 8, 26, 65, 199, 184, 31, 85, 7, 11, 77, 5, 227, 159, 1, 32, 6, 250, + 77, 69, 154, 131, 29, 85, 163, 232, 1, 73, 10, 26, 69, 73, 2, 85, 78, 7, + 33, 6, 32, 80, 76, 85, 83, 32, 4, 150, 159, 24, 69, 215, 133, 7, 71, 5, + 11, 32, 2, 187, 101, 79, 14, 42, 72, 150, 216, 23, 65, 239, 206, 7, 85, + 8, 18, 69, 51, 73, 5, 237, 190, 30, 7, 32, 80, 76, 85, 83, 32, 84, 4, + 146, 183, 31, 68, 3, 77, 7, 11, 68, 5, 209, 214, 13, 5, 32, 80, 76, 85, + 83, 7, 11, 32, 4, 142, 218, 29, 82, 129, 175, 1, 11, 67, 82, 79, 83, 83, + 73, 78, 71, 32, 71, 65, 5, 191, 131, 1, 32, 7, 134, 131, 1, 32, 247, 161, + 30, 65, 11, 11, 50, 9, 11, 32, 6, 80, 8, 67, 82, 79, 83, 83, 73, 78, 71, + 0, 4, 79, 86, 69, 82, 211, 182, 26, 84, 2, 197, 49, 3, 32, 71, 65, 8, 44, + 5, 83, 72, 84, 73, 78, 207, 154, 24, 50, 7, 37, 7, 32, 84, 73, 77, 69, + 83, 32, 4, 250, 186, 30, 75, 215, 120, 85, 49, 70, 32, 94, 52, 114, 82, + 190, 1, 83, 146, 173, 30, 68, 211, 130, 1, 71, 6, 188, 253, 8, 6, 84, 73, + 77, 69, 83, 32, 205, 244, 4, 8, 67, 82, 79, 83, 83, 73, 78, 71, 7, 11, 32, 4, 64, 8, 67, 82, 79, 83, 83, 73, 78, 71, 1, 4, 79, 86, 69, 82, 2, - 225, 199, 23, 3, 32, 71, 73, 16, 26, 51, 131, 136, 1, 50, 13, 37, 7, 32, - 84, 73, 77, 69, 83, 32, 10, 70, 65, 0, 2, 76, 85, 190, 127, 71, 166, 187, - 12, 73, 155, 238, 16, 80, 2, 221, 186, 13, 7, 32, 80, 76, 85, 83, 32, 73, - 14, 26, 72, 203, 161, 29, 65, 13, 11, 32, 10, 22, 84, 247, 20, 67, 8, 44, - 5, 73, 77, 69, 83, 32, 159, 133, 30, 69, 6, 192, 20, 6, 71, 73, 83, 72, - 32, 67, 130, 126, 84, 175, 209, 28, 66, 43, 110, 50, 174, 1, 68, 218, 1, - 77, 54, 82, 176, 183, 13, 9, 32, 67, 82, 79, 83, 83, 73, 78, 71, 247, - 237, 16, 76, 15, 11, 32, 12, 48, 6, 84, 73, 77, 69, 83, 32, 151, 132, 1, - 71, 10, 252, 24, 3, 75, 65, 75, 178, 75, 73, 152, 20, 10, 83, 65, 76, 32, - 80, 76, 85, 83, 32, 84, 191, 175, 28, 78, 11, 11, 32, 8, 128, 1, 10, 80, - 76, 85, 83, 32, 71, 73, 83, 72, 32, 16, 6, 84, 73, 77, 69, 83, 32, 161, - 66, 8, 79, 86, 69, 82, 32, 71, 85, 68, 2, 247, 120, 84, 4, 156, 145, 1, - 5, 65, 32, 80, 76, 85, 175, 156, 28, 75, 5, 17, 2, 32, 84, 2, 225, 42, 4, - 73, 77, 69, 83, 9, 26, 85, 139, 165, 30, 55, 4, 246, 163, 30, 83, 147, 1, - 78, 50, 30, 65, 82, 73, 215, 1, 85, 11, 26, 32, 175, 164, 30, 76, 6, 32, - 2, 84, 69, 131, 128, 1, 71, 4, 219, 127, 78, 23, 37, 7, 32, 84, 73, 77, - 69, 83, 32, 20, 104, 3, 65, 83, 72, 162, 228, 26, 68, 174, 190, 2, 78, - 94, 75, 158, 57, 66, 2, 71, 150, 25, 83, 143, 45, 85, 7, 208, 55, 8, 32, - 79, 86, 69, 82, 32, 72, 73, 147, 235, 29, 50, 19, 48, 2, 66, 50, 158, - 190, 23, 76, 199, 226, 6, 83, 13, 37, 7, 32, 84, 73, 77, 69, 83, 32, 10, - 238, 138, 1, 75, 150, 216, 25, 76, 158, 181, 2, 72, 214, 57, 65, 195, 9, - 85, 49, 104, 3, 68, 73, 77, 94, 71, 214, 1, 76, 62, 77, 238, 154, 30, 32, + 233, 150, 24, 3, 32, 71, 73, 16, 26, 51, 147, 136, 1, 50, 13, 37, 7, 32, + 84, 73, 77, 69, 83, 32, 10, 70, 65, 0, 2, 76, 85, 206, 127, 71, 246, 239, + 12, 73, 155, 190, 17, 80, 2, 189, 239, 13, 7, 32, 80, 76, 85, 83, 32, 73, + 14, 26, 72, 147, 166, 30, 65, 13, 11, 32, 10, 22, 84, 247, 20, 67, 8, 44, + 5, 73, 77, 69, 83, 32, 255, 137, 31, 69, 6, 192, 20, 6, 71, 73, 83, 72, + 32, 67, 146, 126, 84, 243, 213, 29, 66, 43, 110, 50, 174, 1, 68, 218, 1, + 77, 54, 82, 144, 236, 13, 9, 32, 67, 82, 79, 83, 83, 73, 78, 71, 247, + 189, 17, 76, 15, 11, 32, 12, 48, 6, 84, 73, 77, 69, 83, 32, 167, 132, 1, + 71, 10, 252, 24, 3, 75, 65, 75, 194, 75, 73, 152, 20, 10, 83, 65, 76, 32, + 80, 76, 85, 83, 32, 84, 247, 179, 29, 78, 11, 11, 32, 8, 128, 1, 10, 80, + 76, 85, 83, 32, 71, 73, 83, 72, 32, 16, 6, 84, 73, 77, 69, 83, 32, 177, + 66, 8, 79, 86, 69, 82, 32, 71, 85, 68, 2, 135, 121, 84, 4, 172, 145, 1, + 5, 65, 32, 80, 76, 85, 239, 160, 29, 75, 5, 17, 2, 32, 84, 2, 241, 42, 4, + 73, 77, 69, 83, 9, 26, 85, 235, 169, 31, 55, 4, 214, 168, 31, 83, 147, 1, + 78, 50, 30, 65, 82, 73, 215, 1, 85, 11, 26, 32, 143, 169, 31, 76, 6, 32, + 2, 84, 69, 147, 128, 1, 71, 4, 235, 127, 78, 23, 37, 7, 32, 84, 73, 77, + 69, 83, 32, 20, 104, 3, 65, 83, 72, 206, 198, 27, 68, 202, 224, 2, 78, + 94, 75, 170, 57, 66, 2, 71, 162, 25, 83, 143, 45, 85, 7, 224, 55, 8, 32, + 79, 86, 69, 82, 32, 72, 73, 227, 239, 30, 50, 19, 48, 2, 66, 50, 166, + 141, 24, 76, 159, 152, 7, 83, 13, 37, 7, 32, 84, 73, 77, 69, 83, 32, 10, + 254, 138, 1, 75, 178, 186, 26, 76, 186, 215, 2, 72, 226, 57, 65, 195, 9, + 85, 49, 104, 3, 68, 73, 77, 94, 71, 214, 1, 76, 62, 77, 206, 159, 31, 32, 170, 1, 83, 146, 1, 66, 2, 78, 3, 82, 7, 53, 11, 32, 79, 86, 69, 82, 32, - 73, 68, 73, 77, 32, 4, 226, 166, 23, 83, 179, 128, 6, 66, 13, 11, 73, 11, - 11, 32, 8, 146, 123, 71, 168, 178, 11, 31, 79, 86, 69, 82, 32, 73, 71, + 73, 68, 73, 77, 32, 4, 202, 244, 23, 83, 155, 183, 6, 66, 13, 11, 73, 11, + 11, 32, 8, 162, 123, 71, 220, 230, 11, 31, 79, 86, 69, 82, 32, 73, 71, 73, 32, 83, 72, 73, 82, 32, 79, 86, 69, 82, 32, 83, 72, 73, 82, 32, 85, - 68, 32, 79, 86, 69, 82, 178, 183, 16, 68, 219, 166, 1, 82, 7, 26, 32, - 199, 157, 30, 50, 2, 153, 72, 4, 84, 73, 77, 69, 13, 26, 32, 183, 205, - 29, 73, 8, 76, 4, 67, 82, 79, 83, 0, 4, 79, 80, 80, 79, 146, 111, 84, - 183, 180, 22, 83, 2, 253, 153, 29, 5, 83, 73, 78, 71, 32, 222, 1, 78, 65, - 130, 15, 73, 242, 1, 85, 158, 73, 69, 237, 217, 22, 4, 87, 85, 51, 49, - 175, 1, 164, 1, 7, 32, 84, 73, 77, 69, 83, 32, 210, 9, 50, 66, 68, 102, - 75, 50, 76, 82, 77, 28, 4, 83, 75, 65, 76, 152, 140, 15, 6, 80, 32, 69, - 76, 65, 77, 235, 129, 15, 66, 134, 1, 170, 1, 65, 94, 66, 82, 69, 30, 71, - 182, 2, 73, 34, 75, 34, 76, 38, 77, 170, 1, 83, 118, 84, 34, 85, 198, 8, - 72, 246, 51, 78, 154, 157, 16, 80, 142, 147, 13, 82, 147, 17, 90, 13, 46, - 68, 154, 235, 29, 78, 165, 44, 2, 83, 72, 5, 213, 50, 7, 32, 80, 76, 85, - 83, 32, 75, 10, 34, 65, 226, 151, 30, 73, 3, 85, 6, 222, 177, 28, 76, - 130, 230, 1, 68, 3, 82, 4, 250, 24, 82, 151, 61, 83, 26, 30, 65, 98, 73, - 147, 1, 85, 11, 38, 82, 190, 123, 78, 151, 155, 29, 76, 5, 233, 21, 10, - 32, 80, 76, 85, 83, 32, 83, 72, 65, 51, 11, 32, 2, 83, 72, 163, 178, 23, - 82, 7, 11, 32, 4, 26, 67, 239, 130, 1, 80, 2, 37, 7, 82, 79, 83, 83, 73, - 78, 71, 2, 221, 214, 26, 2, 32, 71, 7, 242, 193, 26, 82, 151, 211, 3, 68, - 4, 138, 129, 30, 71, 219, 19, 77, 8, 234, 8, 73, 239, 232, 16, 65, 6, - 246, 238, 11, 85, 175, 165, 18, 73, 12, 18, 69, 83, 73, 9, 33, 6, 32, 80, - 76, 85, 83, 32, 6, 222, 238, 29, 68, 150, 14, 84, 255, 2, 71, 5, 45, 9, - 32, 80, 76, 85, 83, 32, 78, 85, 78, 2, 251, 155, 26, 85, 18, 62, 72, 246, - 174, 26, 65, 224, 234, 2, 2, 85, 72, 243, 119, 73, 10, 250, 149, 29, 85, - 138, 54, 73, 150, 70, 65, 3, 69, 4, 146, 217, 29, 65, 211, 56, 85, 17, - 110, 32, 222, 92, 82, 156, 182, 25, 9, 77, 85, 77, 32, 84, 73, 77, 69, - 83, 206, 252, 3, 83, 146, 1, 50, 3, 68, 2, 207, 190, 20, 85, 5, 177, 202, - 11, 11, 32, 67, 82, 79, 83, 83, 73, 78, 71, 32, 75, 10, 42, 53, 214, 143, - 30, 50, 2, 51, 3, 52, 5, 249, 169, 23, 9, 32, 79, 86, 69, 82, 32, 75, 65, - 68, 5, 173, 75, 8, 32, 84, 73, 77, 69, 83, 32, 73, 7, 11, 32, 4, 166, - 105, 84, 233, 235, 22, 9, 67, 82, 79, 83, 83, 73, 78, 71, 32, 4, 154, - 142, 30, 50, 3, 52, 7, 11, 32, 4, 70, 76, 1, 13, 79, 86, 69, 82, 32, 75, - 65, 83, 75, 65, 76, 32, 76, 2, 173, 116, 24, 65, 71, 65, 66, 32, 84, 73, - 77, 69, 83, 32, 85, 32, 79, 86, 69, 82, 32, 76, 65, 71, 65, 66, 32, 21, - 68, 7, 32, 84, 73, 77, 69, 83, 32, 50, 83, 198, 139, 30, 68, 3, 78, 6, - 26, 85, 187, 197, 29, 66, 5, 215, 139, 30, 68, 8, 52, 3, 73, 77, 53, 242, - 129, 29, 65, 159, 137, 1, 72, 5, 177, 165, 23, 11, 32, 79, 86, 69, 82, - 32, 75, 73, 83, 73, 77, 25, 200, 1, 29, 32, 79, 86, 69, 82, 32, 72, 73, - 32, 84, 73, 77, 69, 83, 32, 65, 83, 72, 50, 32, 75, 85, 32, 79, 86, 69, - 82, 32, 72, 18, 52, 54, 82, 186, 98, 83, 230, 1, 76, 162, 164, 29, 51, 2, - 55, 3, 78, 2, 155, 71, 73, 5, 157, 138, 29, 8, 32, 86, 65, 82, 73, 65, - 78, 84, 5, 213, 115, 9, 32, 79, 80, 80, 79, 83, 73, 78, 71, 240, 2, 30, - 65, 174, 26, 73, 59, 85, 141, 2, 68, 2, 71, 65, 252, 12, 2, 75, 45, 222, - 11, 76, 46, 77, 135, 55, 72, 118, 22, 66, 131, 11, 82, 109, 11, 32, 106, - 48, 6, 84, 73, 77, 69, 83, 32, 207, 141, 23, 83, 104, 190, 1, 65, 186, 1, - 66, 34, 71, 70, 72, 66, 73, 94, 75, 118, 76, 46, 77, 46, 83, 138, 2, 84, - 126, 85, 200, 150, 4, 8, 90, 85, 32, 79, 86, 69, 82, 32, 138, 196, 24, - 68, 214, 82, 69, 131, 57, 78, 15, 80, 6, 32, 80, 76, 85, 83, 32, 76, 4, - 83, 72, 32, 90, 226, 131, 30, 76, 3, 78, 6, 38, 68, 222, 225, 28, 71, - 131, 25, 76, 2, 201, 62, 5, 65, 32, 80, 76, 85, 2, 149, 112, 2, 73, 68, - 4, 178, 189, 29, 65, 151, 70, 73, 10, 48, 2, 85, 68, 210, 159, 26, 65, - 167, 227, 3, 73, 5, 191, 105, 32, 6, 244, 177, 17, 7, 73, 32, 84, 73, 77, - 69, 83, 211, 212, 11, 65, 8, 22, 77, 175, 62, 71, 7, 33, 6, 32, 80, 76, - 85, 83, 32, 4, 254, 220, 29, 76, 179, 34, 72, 10, 26, 85, 175, 137, 28, - 73, 6, 26, 76, 147, 129, 30, 51, 5, 41, 8, 32, 80, 76, 85, 83, 32, 72, - 73, 2, 219, 65, 32, 8, 198, 100, 65, 242, 182, 27, 73, 131, 105, 85, 6, - 26, 69, 243, 154, 28, 85, 5, 175, 25, 32, 12, 26, 72, 139, 239, 29, 85, - 10, 100, 14, 73, 84, 65, 32, 80, 76, 85, 83, 32, 71, 73, 83, 72, 32, 96, - 2, 85, 50, 217, 3, 2, 69, 32, 4, 48, 6, 80, 76, 85, 83, 32, 69, 223, 172, - 25, 84, 2, 11, 82, 2, 11, 73, 2, 215, 154, 23, 78, 5, 189, 73, 5, 32, 80, - 76, 85, 83, 6, 86, 65, 189, 30, 16, 69, 32, 80, 76, 85, 83, 32, 65, 32, - 80, 76, 85, 83, 32, 83, 85, 4, 162, 152, 23, 75, 251, 228, 6, 71, 13, 72, - 6, 32, 80, 76, 85, 83, 32, 190, 41, 50, 226, 209, 29, 83, 147, 1, 68, 4, - 26, 85, 147, 252, 29, 65, 2, 255, 40, 32, 11, 11, 32, 8, 68, 4, 71, 85, - 78, 85, 97, 9, 84, 73, 77, 69, 83, 32, 83, 72, 69, 5, 73, 16, 32, 79, 86, - 69, 82, 32, 76, 65, 71, 65, 82, 32, 71, 85, 78, 85, 2, 199, 205, 29, 32, - 5, 11, 32, 2, 229, 143, 14, 4, 80, 76, 85, 83, 136, 1, 82, 48, 146, 2, - 49, 30, 50, 114, 51, 82, 52, 222, 2, 54, 154, 4, 55, 243, 24, 53, 22, - 158, 1, 50, 30, 56, 180, 52, 15, 55, 57, 32, 79, 86, 69, 82, 32, 76, 65, - 75, 45, 48, 55, 57, 242, 199, 10, 53, 146, 152, 12, 54, 2, 57, 94, 51, - 143, 143, 3, 48, 4, 162, 248, 29, 49, 3, 53, 4, 144, 148, 23, 12, 49, 32, - 79, 86, 69, 82, 32, 76, 65, 75, 45, 48, 247, 227, 6, 48, 4, 222, 147, 23, - 52, 95, 51, 16, 46, 50, 38, 54, 230, 140, 23, 49, 159, 2, 51, 6, 230, - 246, 29, 48, 2, 53, 3, 56, 4, 194, 246, 29, 53, 3, 54, 12, 42, 52, 250, - 177, 11, 56, 143, 225, 11, 57, 6, 250, 245, 29, 51, 2, 55, 3, 56, 30, 62, - 52, 214, 1, 53, 30, 57, 178, 144, 23, 55, 143, 143, 3, 56, 14, 26, 57, - 255, 244, 29, 49, 13, 37, 7, 32, 84, 73, 77, 69, 83, 32, 10, 116, 9, 80, - 65, 80, 32, 80, 76, 85, 83, 32, 208, 227, 12, 7, 85, 50, 32, 80, 76, 85, - 83, 190, 31, 73, 235, 203, 16, 71, 4, 210, 13, 80, 51, 76, 4, 194, 243, - 29, 48, 3, 55, 8, 166, 243, 29, 48, 2, 50, 2, 51, 3, 53, 46, 60, 2, 49, - 55, 240, 1, 2, 52, 56, 138, 137, 23, 48, 23, 51, 23, 37, 7, 32, 84, 73, - 77, 69, 83, 32, 20, 122, 84, 34, 85, 162, 11, 75, 132, 34, 9, 68, 85, 78, - 51, 32, 71, 85, 78, 85, 186, 222, 27, 65, 254, 158, 1, 66, 235, 67, 76, - 4, 198, 184, 29, 65, 211, 56, 69, 6, 250, 146, 14, 82, 254, 221, 15, 50, - 3, 68, 21, 37, 7, 32, 84, 73, 77, 69, 83, 32, 18, 154, 1, 85, 220, 8, 4, - 80, 65, 80, 32, 146, 45, 73, 164, 167, 12, 10, 83, 72, 69, 83, 72, 32, - 80, 76, 85, 83, 250, 252, 4, 68, 170, 129, 12, 78, 163, 17, 71, 4, 138, - 145, 14, 82, 255, 221, 15, 68, 4, 234, 137, 23, 50, 211, 145, 3, 52, 5, - 11, 32, 2, 145, 6, 4, 84, 73, 77, 69, 7, 49, 10, 32, 84, 73, 77, 69, 83, - 32, 75, 85, 82, 5, 221, 160, 11, 5, 32, 80, 76, 85, 83, 9, 200, 160, 11, - 2, 77, 77, 222, 203, 18, 83, 147, 1, 76, 93, 90, 50, 212, 7, 3, 71, 65, - 76, 142, 1, 77, 130, 62, 32, 198, 165, 29, 51, 2, 72, 3, 76, 69, 11, 32, - 66, 88, 4, 67, 82, 79, 83, 0, 4, 79, 80, 80, 79, 44, 4, 71, 85, 78, 85, - 38, 83, 35, 84, 2, 205, 158, 11, 6, 83, 73, 78, 71, 32, 76, 2, 193, 24, - 5, 32, 84, 73, 77, 69, 6, 250, 68, 72, 139, 173, 22, 81, 54, 44, 5, 73, - 77, 69, 83, 32, 171, 197, 29, 69, 52, 188, 1, 4, 69, 83, 72, 50, 94, 72, - 42, 75, 68, 2, 76, 65, 34, 77, 60, 3, 80, 65, 80, 118, 83, 90, 84, 250, - 56, 71, 158, 230, 7, 78, 150, 135, 18, 68, 170, 181, 2, 65, 254, 66, 66, - 203, 53, 73, 7, 11, 32, 4, 26, 80, 207, 150, 25, 84, 2, 17, 2, 76, 85, 2, - 245, 222, 28, 3, 83, 32, 76, 4, 184, 66, 2, 73, 32, 171, 156, 28, 65, 8, - 32, 2, 65, 68, 183, 231, 29, 73, 6, 226, 19, 51, 211, 211, 29, 50, 4, - 174, 20, 32, 171, 249, 16, 71, 2, 11, 69, 2, 11, 32, 2, 209, 251, 12, 4, - 80, 76, 85, 83, 5, 11, 32, 2, 33, 6, 80, 76, 85, 83, 32, 80, 2, 45, 9, - 65, 80, 32, 80, 76, 85, 83, 32, 76, 2, 235, 145, 26, 85, 6, 26, 73, 195, - 192, 29, 72, 4, 194, 18, 32, 157, 211, 25, 7, 75, 50, 32, 80, 76, 85, 83, - 4, 162, 53, 85, 139, 24, 65, 9, 11, 32, 6, 22, 79, 207, 67, 83, 4, 56, 7, - 80, 80, 79, 83, 73, 78, 71, 1, 3, 86, 69, 82, 2, 21, 3, 32, 76, 85, 2, - 175, 218, 28, 71, 7, 45, 9, 32, 79, 86, 69, 82, 32, 76, 85, 77, 5, 187, - 59, 32, 70, 34, 65, 70, 69, 22, 73, 71, 85, 17, 170, 45, 32, 188, 1, 2, - 83, 72, 250, 179, 29, 50, 2, 72, 3, 82, 7, 187, 238, 25, 83, 7, 240, 142, - 26, 8, 32, 80, 76, 85, 83, 32, 90, 65, 151, 211, 3, 78, 43, 104, 2, 83, - 72, 186, 60, 71, 140, 215, 10, 5, 32, 79, 86, 69, 82, 40, 2, 82, 71, 237, - 183, 6, 2, 78, 83, 31, 22, 32, 183, 2, 51, 16, 136, 1, 9, 79, 86, 69, 82, - 32, 77, 85, 83, 72, 124, 6, 84, 73, 77, 69, 83, 32, 173, 254, 25, 10, 67, - 82, 79, 83, 83, 73, 78, 71, 32, 77, 9, 37, 7, 32, 84, 73, 77, 69, 83, 32, - 6, 42, 65, 254, 175, 27, 75, 175, 172, 2, 71, 2, 237, 142, 11, 5, 32, 80, - 76, 85, 83, 6, 242, 229, 28, 75, 142, 118, 90, 187, 2, 65, 13, 11, 32, - 10, 44, 6, 84, 73, 77, 69, 83, 32, 203, 57, 71, 8, 38, 65, 210, 201, 29, - 68, 163, 17, 90, 5, 173, 205, 12, 5, 32, 80, 76, 85, 83, 158, 1, 42, 65, - 134, 2, 69, 94, 73, 199, 7, 85, 23, 52, 2, 71, 65, 166, 1, 77, 246, 218, - 29, 50, 3, 52, 11, 26, 32, 255, 219, 29, 82, 6, 100, 8, 79, 80, 80, 79, - 83, 73, 78, 71, 146, 59, 73, 197, 14, 9, 84, 73, 77, 69, 83, 32, 83, 72, - 85, 2, 217, 152, 29, 3, 32, 78, 65, 7, 204, 21, 2, 32, 78, 167, 197, 29, - 50, 9, 11, 32, 6, 44, 6, 84, 73, 77, 69, 83, 32, 179, 57, 83, 4, 250, - 147, 29, 85, 151, 70, 65, 73, 90, 77, 78, 78, 204, 234, 19, 6, 32, 84, - 73, 77, 69, 83, 254, 135, 8, 83, 131, 230, 1, 50, 7, 45, 9, 32, 84, 73, - 77, 69, 83, 32, 71, 65, 4, 158, 3, 82, 179, 58, 78, 59, 36, 3, 68, 65, - 50, 163, 216, 29, 57, 55, 37, 7, 32, 84, 73, 77, 69, 83, 32, 52, 162, 1, - 65, 34, 71, 50, 75, 16, 5, 76, 65, 75, 45, 48, 22, 77, 86, 78, 34, 80, - 76, 3, 83, 72, 69, 94, 85, 240, 15, 3, 68, 73, 77, 174, 186, 28, 66, 211, - 117, 72, 6, 246, 2, 83, 223, 211, 29, 78, 8, 26, 73, 255, 211, 28, 85, 5, - 135, 213, 29, 83, 2, 211, 20, 69, 2, 251, 242, 22, 53, 4, 26, 69, 171, - 240, 27, 65, 2, 25, 4, 32, 80, 76, 85, 2, 177, 41, 3, 83, 32, 71, 4, 182, - 133, 29, 85, 215, 79, 69, 2, 33, 6, 65, 80, 32, 80, 76, 85, 2, 11, 83, 2, - 229, 195, 28, 2, 32, 80, 9, 37, 7, 32, 80, 76, 85, 83, 32, 65, 6, 26, 83, - 171, 140, 28, 32, 4, 11, 72, 5, 107, 32, 11, 50, 32, 34, 50, 246, 244, - 13, 82, 239, 220, 15, 83, 2, 233, 217, 13, 3, 80, 76, 85, 2, 11, 32, 2, - 21, 3, 80, 76, 85, 2, 11, 83, 2, 151, 237, 27, 32, 57, 24, 2, 49, 49, 99, - 78, 9, 11, 32, 6, 200, 25, 9, 79, 86, 69, 82, 32, 78, 85, 49, 49, 210, - 230, 24, 84, 191, 248, 2, 82, 47, 30, 32, 169, 3, 2, 85, 90, 18, 144, 1, - 12, 67, 82, 79, 83, 83, 73, 78, 71, 32, 78, 85, 78, 72, 12, 76, 65, 71, - 65, 82, 32, 84, 73, 77, 69, 83, 32, 174, 1, 79, 163, 252, 24, 84, 5, 209, - 40, 14, 32, 76, 65, 71, 65, 82, 32, 79, 86, 69, 82, 32, 76, 65, 10, 56, - 3, 83, 65, 76, 210, 233, 27, 77, 14, 85, 247, 66, 71, 5, 205, 197, 28, - 23, 32, 79, 86, 69, 82, 32, 78, 85, 78, 32, 76, 65, 71, 65, 82, 32, 84, - 73, 77, 69, 83, 32, 83, 2, 153, 253, 16, 3, 86, 69, 82, 27, 11, 32, 24, - 120, 10, 65, 66, 50, 32, 84, 73, 77, 69, 83, 32, 205, 37, 15, 75, 73, 83, - 73, 77, 53, 32, 84, 73, 77, 69, 83, 32, 66, 73, 20, 168, 1, 2, 75, 65, - 202, 7, 73, 212, 39, 2, 65, 83, 194, 171, 2, 68, 164, 166, 8, 3, 83, 73, - 76, 226, 230, 11, 85, 150, 185, 5, 71, 238, 147, 1, 78, 254, 2, 66, 163, - 17, 76, 2, 199, 247, 25, 68, 46, 42, 65, 36, 4, 69, 83, 72, 50, 23, 73, - 9, 242, 202, 29, 68, 2, 78, 3, 80, 5, 135, 186, 27, 32, 35, 22, 32, 151, - 1, 82, 20, 80, 6, 84, 73, 77, 69, 83, 32, 205, 199, 17, 8, 67, 82, 79, - 83, 83, 73, 78, 71, 18, 214, 21, 85, 146, 48, 65, 2, 73, 190, 195, 28, - 66, 175, 64, 69, 12, 32, 2, 73, 71, 239, 200, 29, 50, 11, 11, 32, 8, 96, - 6, 84, 73, 77, 69, 83, 32, 173, 193, 25, 12, 79, 80, 80, 79, 83, 73, 78, - 71, 32, 80, 73, 82, 6, 210, 190, 28, 75, 150, 67, 85, 223, 67, 90, 8, - 230, 67, 65, 234, 131, 29, 73, 3, 85, 202, 1, 46, 65, 250, 5, 72, 150, - 11, 73, 163, 1, 85, 63, 46, 71, 190, 4, 76, 122, 78, 147, 193, 29, 82, - 53, 11, 32, 50, 92, 4, 71, 85, 78, 85, 54, 78, 32, 6, 84, 73, 77, 69, 83, - 32, 241, 21, 4, 79, 86, 69, 82, 5, 29, 5, 32, 84, 73, 77, 69, 2, 167, - 170, 17, 83, 2, 177, 203, 24, 3, 85, 84, 73, 42, 150, 1, 73, 42, 75, 34, - 83, 64, 2, 84, 65, 38, 85, 216, 62, 2, 68, 85, 246, 219, 27, 76, 222, 39, - 78, 198, 48, 69, 254, 59, 77, 162, 17, 72, 187, 2, 65, 2, 11, 71, 2, 11, - 73, 2, 191, 31, 32, 4, 242, 138, 29, 85, 187, 53, 65, 6, 26, 72, 239, - 185, 28, 65, 4, 234, 139, 13, 69, 139, 241, 15, 73, 4, 234, 221, 22, 75, - 251, 228, 6, 66, 10, 174, 193, 29, 83, 146, 1, 50, 2, 66, 2, 77, 3, 82, - 5, 33, 6, 32, 76, 65, 71, 65, 66, 2, 37, 7, 32, 84, 73, 77, 69, 83, 32, - 2, 11, 65, 2, 11, 83, 2, 207, 221, 22, 72, 2, 131, 251, 10, 71, 106, 46, - 65, 250, 1, 69, 242, 2, 73, 239, 3, 85, 29, 50, 51, 182, 1, 54, 182, 215, - 22, 66, 223, 3, 82, 19, 37, 7, 32, 84, 73, 77, 69, 83, 32, 16, 90, 85, - 146, 25, 83, 194, 231, 25, 71, 130, 198, 2, 84, 166, 50, 66, 206, 47, 78, - 215, 22, 65, 5, 11, 32, 2, 129, 171, 25, 4, 80, 76, 85, 83, 5, 175, 45, - 32, 27, 62, 32, 140, 2, 2, 83, 72, 226, 232, 25, 71, 171, 211, 3, 78, 14, - 84, 8, 79, 86, 69, 82, 32, 83, 72, 69, 88, 5, 80, 76, 85, 83, 32, 255, - 151, 29, 72, 7, 11, 32, 4, 190, 15, 71, 141, 6, 12, 84, 65, 66, 32, 79, - 86, 69, 82, 32, 84, 65, 66, 6, 48, 2, 72, 85, 20, 2, 78, 65, 247, 153, - 28, 83, 2, 219, 216, 22, 66, 2, 199, 216, 22, 77, 7, 202, 130, 28, 76, - 191, 185, 1, 50, 40, 62, 68, 74, 77, 218, 1, 82, 182, 178, 25, 78, 155, - 132, 4, 84, 7, 37, 7, 32, 84, 73, 77, 69, 83, 32, 4, 162, 170, 29, 73, - 219, 16, 65, 25, 37, 7, 32, 84, 73, 77, 69, 83, 32, 22, 114, 66, 38, 73, - 130, 19, 75, 198, 181, 2, 77, 190, 198, 2, 76, 246, 135, 23, 71, 130, 25, - 83, 238, 9, 68, 191, 127, 65, 4, 218, 200, 2, 85, 203, 231, 25, 65, 4, - 249, 20, 2, 71, 73, 7, 11, 32, 4, 60, 9, 79, 86, 69, 82, 32, 83, 72, 73, - 82, 211, 230, 24, 84, 2, 241, 191, 28, 11, 32, 66, 85, 82, 32, 79, 86, - 69, 82, 32, 66, 13, 88, 14, 32, 79, 86, 69, 82, 32, 73, 78, 86, 69, 82, - 84, 69, 68, 34, 50, 187, 190, 28, 66, 2, 11, 32, 2, 167, 245, 24, 83, 7, - 33, 6, 32, 80, 76, 85, 83, 32, 4, 88, 7, 69, 50, 32, 84, 73, 77, 69, 173, - 234, 10, 9, 68, 85, 71, 32, 84, 73, 77, 69, 83, 2, 235, 207, 12, 83, 17, - 50, 32, 30, 71, 230, 234, 10, 76, 227, 230, 11, 75, 4, 138, 8, 84, 163, - 9, 71, 7, 11, 52, 5, 49, 10, 32, 79, 86, 69, 82, 32, 83, 73, 71, 52, 2, - 199, 14, 32, 19, 78, 68, 22, 77, 22, 82, 252, 164, 12, 5, 32, 79, 86, 69, - 82, 147, 150, 16, 72, 5, 247, 179, 29, 50, 5, 199, 206, 27, 65, 5, 207, - 179, 29, 57, 72, 46, 65, 150, 4, 73, 250, 1, 85, 227, 8, 69, 37, 66, 32, - 94, 66, 166, 1, 71, 148, 1, 2, 75, 52, 183, 175, 29, 82, 8, 60, 6, 84, - 73, 77, 69, 83, 32, 130, 14, 71, 207, 147, 27, 65, 4, 174, 158, 29, 72, - 3, 77, 7, 11, 32, 4, 188, 143, 17, 29, 79, 86, 69, 82, 32, 84, 65, 66, - 32, 78, 73, 32, 79, 86, 69, 82, 32, 78, 73, 32, 68, 73, 83, 72, 32, 79, - 86, 69, 82, 175, 169, 5, 83, 15, 37, 7, 32, 84, 73, 77, 69, 83, 32, 12, - 74, 84, 252, 233, 7, 2, 83, 72, 154, 155, 20, 71, 174, 100, 85, 191, 50, - 66, 2, 11, 85, 2, 243, 203, 22, 71, 5, 29, 5, 32, 80, 76, 85, 83, 2, 145, - 201, 27, 2, 32, 83, 17, 46, 82, 150, 29, 32, 182, 145, 29, 50, 3, 76, 9, - 11, 32, 6, 48, 8, 79, 86, 69, 82, 32, 84, 73, 82, 99, 84, 5, 11, 32, 2, - 11, 71, 2, 21, 3, 65, 68, 32, 2, 241, 5, 8, 79, 86, 69, 82, 32, 71, 65, - 68, 2, 217, 21, 6, 73, 77, 69, 83, 32, 84, 17, 50, 77, 114, 82, 138, 200, - 22, 71, 215, 227, 6, 75, 7, 37, 7, 32, 84, 73, 77, 69, 83, 32, 4, 46, 71, - 213, 137, 17, 5, 84, 72, 82, 69, 69, 2, 221, 16, 2, 65, 78, 5, 137, 221, - 10, 17, 32, 79, 86, 69, 82, 32, 84, 85, 82, 32, 90, 65, 32, 79, 86, 69, - 82, 179, 1, 138, 1, 32, 246, 2, 68, 226, 2, 78, 46, 77, 158, 2, 82, 128, - 9, 2, 83, 72, 174, 1, 90, 192, 133, 12, 2, 84, 85, 238, 145, 17, 50, 3, - 66, 12, 64, 7, 79, 86, 69, 82, 32, 85, 32, 158, 2, 85, 215, 252, 27, 71, - 6, 200, 1, 10, 80, 65, 32, 79, 86, 69, 82, 32, 80, 65, 168, 247, 8, 19, - 85, 32, 82, 69, 86, 69, 82, 83, 69, 68, 32, 79, 86, 69, 82, 32, 85, 32, - 82, 249, 183, 19, 10, 83, 85, 82, 32, 79, 86, 69, 82, 32, 83, 2, 11, 32, - 2, 45, 9, 71, 65, 82, 32, 79, 86, 69, 82, 32, 2, 255, 132, 28, 71, 5, - 171, 130, 29, 32, 21, 26, 32, 147, 166, 29, 85, 16, 70, 75, 44, 2, 83, - 72, 100, 6, 84, 73, 77, 69, 83, 32, 135, 1, 71, 2, 11, 85, 2, 11, 83, 2, - 151, 217, 10, 72, 4, 29, 5, 69, 83, 72, 73, 71, 5, 11, 32, 2, 11, 84, 2, - 149, 223, 28, 6, 73, 77, 69, 83, 32, 66, 8, 92, 14, 85, 32, 80, 76, 85, - 83, 32, 85, 32, 80, 76, 85, 83, 32, 142, 222, 28, 66, 203, 50, 77, 4, 11, - 85, 5, 11, 32, 2, 11, 71, 2, 147, 255, 28, 85, 21, 72, 7, 32, 84, 73, 77, - 69, 83, 32, 136, 1, 2, 85, 77, 231, 162, 28, 66, 10, 50, 76, 16, 2, 77, - 69, 50, 83, 183, 162, 29, 85, 2, 231, 6, 65, 5, 11, 32, 2, 177, 153, 25, - 4, 80, 76, 85, 83, 2, 255, 215, 10, 72, 7, 37, 7, 32, 84, 73, 77, 69, 83, - 32, 4, 158, 11, 75, 163, 148, 29, 80, 93, 54, 32, 102, 50, 194, 2, 73, - 22, 85, 235, 157, 29, 52, 4, 62, 83, 157, 168, 28, 9, 67, 82, 79, 83, 83, - 73, 78, 71, 32, 2, 141, 154, 25, 4, 72, 69, 83, 72, 23, 11, 32, 20, 42, - 73, 37, 6, 84, 73, 77, 69, 83, 32, 2, 165, 168, 23, 4, 78, 86, 69, 82, - 18, 46, 65, 82, 85, 202, 158, 28, 78, 227, 125, 72, 6, 48, 6, 32, 80, 76, - 85, 83, 32, 255, 158, 29, 76, 4, 194, 156, 29, 72, 3, 78, 8, 26, 50, 199, - 158, 29, 68, 7, 33, 6, 32, 80, 76, 85, 83, 32, 4, 130, 185, 27, 65, 199, - 209, 1, 66, 5, 251, 157, 29, 51, 59, 56, 7, 32, 84, 73, 77, 69, 83, 32, - 165, 4, 2, 68, 65, 52, 134, 1, 65, 46, 68, 38, 71, 82, 73, 46, 76, 78, - 83, 46, 85, 186, 247, 27, 66, 238, 34, 77, 214, 90, 84, 146, 17, 75, 162, - 17, 72, 3, 80, 5, 11, 83, 2, 11, 72, 2, 207, 194, 16, 71, 4, 186, 208, - 10, 65, 235, 251, 17, 85, 10, 26, 65, 187, 155, 29, 85, 9, 34, 78, 150, - 155, 29, 76, 3, 82, 2, 211, 9, 50, 6, 170, 135, 29, 71, 202, 18, 83, 147, - 1, 77, 6, 46, 85, 229, 178, 22, 5, 65, 75, 45, 54, 54, 4, 166, 154, 29, - 51, 3, 77, 4, 144, 181, 22, 2, 73, 71, 167, 206, 6, 72, 6, 42, 32, 186, - 187, 13, 82, 255, 221, 15, 68, 2, 241, 238, 27, 6, 80, 76, 85, 83, 32, - 71, 5, 11, 32, 2, 213, 159, 13, 4, 84, 73, 77, 69, 17, 84, 7, 32, 84, 73, - 77, 69, 83, 32, 244, 139, 28, 2, 85, 77, 142, 140, 1, 50, 3, 88, 8, 50, - 84, 168, 180, 25, 2, 75, 85, 167, 227, 3, 65, 2, 11, 65, 2, 199, 178, 22, - 75, 6, 26, 51, 147, 151, 29, 85, 5, 29, 5, 32, 84, 73, 77, 69, 2, 21, 3, - 83, 32, 75, 2, 11, 65, 2, 251, 220, 22, 83, 42, 50, 65, 190, 1, 73, 146, - 1, 85, 231, 175, 22, 69, 13, 50, 32, 198, 137, 28, 77, 142, 140, 1, 55, - 3, 71, 4, 56, 8, 83, 81, 85, 65, 82, 69, 68, 32, 175, 195, 24, 84, 2, 11, - 84, 2, 21, 3, 73, 77, 69, 2, 11, 83, 2, 157, 156, 28, 2, 32, 75, 15, 86, - 66, 132, 152, 22, 6, 32, 79, 86, 69, 82, 32, 154, 24, 90, 214, 227, 6, - 51, 3, 71, 5, 17, 2, 32, 75, 2, 17, 2, 65, 66, 2, 135, 2, 65, 15, 84, 10, - 32, 79, 86, 69, 82, 32, 90, 85, 32, 80, 42, 53, 246, 153, 28, 66, 199, - 120, 77, 2, 181, 240, 27, 5, 76, 85, 83, 32, 83, 7, 37, 7, 32, 84, 73, - 77, 69, 83, 32, 4, 44, 5, 84, 72, 82, 69, 69, 227, 145, 29, 65, 2, 29, 5, - 32, 68, 73, 83, 72, 2, 11, 32, 2, 203, 191, 24, 84, 8, 42, 32, 194, 150, - 22, 73, 187, 180, 3, 67, 4, 202, 212, 21, 79, 141, 190, 6, 8, 87, 73, 84, - 72, 32, 83, 84, 82, 18, 134, 1, 76, 154, 1, 82, 189, 202, 28, 23, 86, 69, - 68, 32, 83, 84, 69, 77, 32, 80, 65, 82, 65, 71, 82, 65, 80, 72, 32, 83, - 73, 71, 78, 10, 48, 2, 89, 32, 253, 147, 3, 4, 73, 78, 71, 32, 8, 60, 2, - 76, 79, 177, 128, 6, 7, 66, 82, 65, 67, 75, 69, 84, 6, 210, 226, 25, 71, - 215, 171, 3, 79, 6, 52, 5, 69, 78, 67, 89, 32, 53, 4, 89, 32, 65, 78, 4, - 248, 158, 24, 4, 69, 88, 67, 72, 143, 232, 3, 83, 2, 133, 160, 19, 3, 68, - 32, 82, 4, 174, 246, 27, 65, 133, 89, 2, 79, 77, 236, 8, 128, 1, 2, 80, - 82, 140, 9, 7, 82, 73, 76, 76, 73, 67, 32, 244, 134, 25, 7, 76, 73, 78, - 68, 82, 73, 67, 173, 210, 2, 2, 67, 76, 180, 2, 140, 1, 13, 73, 79, 84, - 32, 83, 89, 76, 76, 65, 66, 76, 69, 32, 169, 1, 16, 79, 45, 77, 73, 78, - 79, 65, 78, 32, 83, 73, 71, 78, 32, 67, 77, 110, 254, 156, 7, 75, 2, 76, - 2, 77, 2, 78, 2, 80, 2, 82, 2, 83, 2, 84, 126, 87, 254, 41, 74, 2, 90, - 194, 206, 6, 88, 138, 244, 14, 65, 2, 69, 2, 73, 2, 79, 3, 85, 198, 1, - 46, 48, 146, 5, 49, 141, 212, 23, 2, 51, 48, 170, 1, 102, 48, 78, 49, 74, - 50, 78, 51, 78, 52, 62, 53, 86, 55, 190, 140, 10, 57, 190, 3, 54, 179, - 234, 13, 56, 16, 246, 135, 29, 49, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, - 56, 3, 57, 16, 194, 3, 50, 234, 131, 29, 48, 2, 49, 2, 51, 2, 53, 2, 55, - 3, 57, 16, 226, 134, 29, 49, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, - 57, 16, 150, 134, 29, 48, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, - 57, 12, 202, 133, 29, 48, 2, 49, 2, 52, 2, 54, 2, 55, 3, 57, 18, 142, - 133, 29, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 20, - 82, 53, 234, 131, 29, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, 56, 3, - 57, 5, 231, 131, 29, 66, 24, 18, 48, 87, 49, 18, 190, 131, 29, 48, 2, 49, - 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 6, 234, 130, 29, 48, 2, - 50, 3, 52, 180, 6, 140, 1, 9, 67, 65, 80, 73, 84, 65, 76, 32, 76, 194, 4, - 75, 32, 7, 76, 69, 84, 84, 69, 82, 32, 132, 1, 3, 80, 65, 89, 30, 83, - 203, 41, 84, 240, 2, 44, 6, 69, 84, 84, 69, 82, 32, 159, 43, 73, 234, 2, - 150, 2, 76, 46, 78, 34, 80, 34, 82, 58, 84, 54, 85, 190, 5, 65, 214, 1, - 66, 242, 1, 67, 150, 1, 68, 254, 1, 69, 206, 2, 71, 130, 1, 72, 134, 1, - 73, 230, 3, 75, 226, 3, 77, 218, 1, 79, 226, 2, 83, 210, 7, 89, 166, 1, - 90, 230, 178, 28, 70, 134, 14, 74, 2, 86, 2, 87, 159, 20, 81, 6, 214, 25, - 73, 186, 206, 28, 74, 159, 20, 72, 4, 254, 26, 69, 231, 204, 28, 74, 8, - 246, 33, 69, 203, 200, 28, 83, 12, 212, 9, 3, 79, 85, 78, 226, 18, 69, - 139, 223, 28, 72, 18, 158, 33, 69, 106, 83, 174, 174, 28, 67, 187, 22, - 87, 13, 190, 34, 32, 115, 75, 2, 137, 244, 28, 3, 65, 86, 89, 6, 248, 10, - 5, 77, 85, 76, 84, 73, 156, 205, 12, 2, 80, 65, 229, 195, 11, 14, 83, 77, - 65, 76, 76, 32, 67, 65, 80, 73, 84, 65, 76, 32, 2, 237, 209, 28, 2, 69, - 82, 184, 3, 136, 1, 6, 77, 65, 76, 76, 32, 76, 193, 37, 22, 85, 66, 83, - 67, 82, 73, 80, 84, 32, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, - 32, 132, 3, 44, 6, 69, 84, 84, 69, 82, 32, 143, 36, 73, 254, 2, 154, 2, - 65, 214, 1, 66, 242, 1, 67, 150, 1, 68, 254, 1, 69, 206, 2, 71, 130, 1, - 72, 134, 1, 73, 230, 3, 75, 222, 2, 76, 134, 1, 77, 114, 78, 106, 79, - 110, 80, 50, 82, 198, 1, 83, 218, 2, 84, 210, 2, 85, 246, 1, 87, 54, 89, - 166, 1, 90, 230, 178, 28, 70, 134, 14, 74, 2, 86, 159, 20, 81, 17, 108, - 6, 32, 87, 73, 84, 72, 32, 36, 9, 66, 75, 72, 65, 83, 73, 65, 78, 32, - 229, 228, 11, 4, 76, 69, 85, 84, 4, 138, 154, 9, 68, 151, 170, 3, 66, 8, - 128, 209, 12, 3, 67, 72, 69, 250, 189, 6, 68, 139, 229, 9, 72, 18, 110, - 65, 78, 73, 32, 3, 82, 79, 65, 128, 146, 3, 6, 76, 69, 78, 68, 69, 68, - 178, 187, 9, 89, 243, 166, 16, 69, 6, 200, 21, 6, 82, 82, 69, 68, 32, 79, - 233, 205, 11, 5, 83, 72, 75, 73, 82, 4, 230, 2, 78, 187, 143, 3, 71, 2, - 179, 133, 10, 68, 14, 76, 2, 72, 69, 246, 9, 76, 164, 222, 12, 4, 82, 79, - 83, 83, 155, 222, 15, 67, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 142, 29, - 68, 195, 165, 28, 86, 26, 96, 2, 74, 69, 20, 6, 79, 85, 66, 76, 69, 32, - 66, 90, 182, 196, 28, 67, 186, 22, 87, 215, 22, 69, 5, 239, 151, 22, 82, - 4, 36, 3, 77, 79, 78, 219, 241, 28, 79, 2, 153, 13, 2, 79, 67, 12, 46, - 69, 138, 241, 27, 90, 182, 105, 72, 3, 87, 5, 155, 210, 28, 76, 41, 86, - 76, 98, 78, 110, 82, 166, 15, 32, 254, 185, 12, 83, 250, 247, 13, 77, - 195, 173, 2, 70, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 158, 20, 77, 222, - 182, 12, 68, 134, 248, 13, 84, 163, 92, 72, 13, 33, 6, 32, 87, 73, 84, - 72, 32, 10, 190, 19, 77, 222, 182, 12, 68, 202, 52, 76, 190, 195, 13, 84, - 163, 92, 72, 5, 229, 191, 27, 5, 32, 87, 73, 84, 72, 14, 32, 2, 72, 69, - 187, 215, 28, 74, 13, 33, 6, 32, 87, 73, 84, 72, 32, 10, 134, 18, 77, - 146, 15, 85, 206, 167, 12, 68, 143, 31, 83, 12, 26, 65, 195, 214, 28, 87, - 11, 48, 6, 32, 87, 73, 84, 72, 32, 143, 135, 27, 82, 6, 218, 199, 12, 68, - 166, 212, 14, 72, 231, 160, 1, 83, 35, 84, 6, 32, 87, 73, 84, 72, 32, 50, - 69, 74, 79, 201, 1, 6, 90, 72, 73, 84, 83, 65, 6, 254, 142, 9, 68, 214, - 10, 71, 239, 184, 3, 77, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 254, 152, - 9, 71, 195, 159, 3, 66, 17, 11, 84, 14, 48, 6, 73, 70, 73, 69, 68, 32, - 159, 234, 28, 65, 12, 80, 2, 67, 76, 38, 76, 214, 134, 3, 66, 130, 207, - 24, 89, 210, 147, 1, 65, 3, 69, 2, 33, 6, 79, 83, 69, 68, 32, 76, 2, 151, - 4, 73, 5, 145, 212, 12, 14, 32, 87, 73, 84, 72, 32, 68, 79, 85, 66, 76, - 69, 32, 71, 34, 102, 65, 98, 79, 244, 185, 28, 11, 72, 65, 75, 65, 83, - 83, 73, 65, 78, 32, 67, 186, 22, 74, 255, 2, 83, 11, 33, 6, 32, 87, 73, - 84, 72, 32, 8, 182, 194, 12, 68, 166, 212, 14, 72, 154, 160, 1, 86, 79, - 83, 18, 36, 3, 77, 73, 32, 251, 193, 25, 80, 16, 58, 68, 198, 236, 11, - 76, 2, 78, 2, 83, 2, 84, 3, 90, 6, 194, 236, 11, 90, 134, 227, 16, 74, - 215, 22, 69, 8, 94, 73, 236, 154, 10, 10, 79, 78, 71, 45, 76, 69, 71, 71, - 69, 68, 206, 179, 18, 74, 159, 20, 72, 2, 233, 130, 3, 4, 84, 84, 76, 69, - 4, 21, 3, 79, 78, 79, 4, 18, 67, 39, 71, 2, 205, 188, 18, 4, 85, 76, 65, - 82, 2, 237, 10, 4, 82, 65, 80, 72, 6, 62, 69, 204, 187, 18, 5, 65, 82, - 82, 79, 87, 155, 145, 10, 74, 2, 249, 192, 12, 5, 85, 84, 82, 65, 76, 11, - 52, 4, 77, 69, 71, 65, 166, 3, 32, 183, 223, 28, 84, 5, 233, 185, 28, 8, - 32, 87, 73, 84, 72, 32, 84, 73, 10, 130, 6, 69, 230, 183, 12, 65, 231, - 144, 16, 83, 14, 50, 69, 100, 4, 79, 85, 78, 68, 167, 222, 28, 72, 8, 37, - 7, 86, 69, 82, 83, 69, 68, 32, 8, 214, 249, 18, 68, 206, 222, 8, 84, 142, - 100, 89, 151, 14, 90, 4, 250, 241, 9, 32, 241, 237, 1, 2, 69, 68, 34, - 108, 4, 67, 72, 87, 65, 34, 72, 132, 1, 4, 79, 70, 84, 32, 170, 186, 12, - 84, 213, 1, 5, 69, 77, 73, 83, 79, 5, 11, 32, 2, 183, 218, 5, 87, 16, 92, - 4, 79, 82, 84, 32, 136, 185, 12, 2, 72, 65, 254, 173, 14, 67, 234, 224, - 1, 87, 215, 22, 65, 6, 254, 176, 26, 73, 195, 173, 2, 85, 8, 38, 69, 166, - 215, 27, 83, 255, 111, 68, 4, 242, 221, 28, 76, 3, 77, 26, 132, 1, 4, 65, - 76, 76, 32, 50, 69, 106, 83, 212, 211, 11, 11, 72, 82, 69, 69, 45, 76, - 69, 71, 71, 69, 68, 218, 218, 16, 67, 187, 22, 87, 6, 178, 179, 12, 72, - 206, 149, 15, 89, 255, 124, 84, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 26, - 77, 223, 182, 12, 68, 2, 185, 169, 21, 5, 73, 68, 68, 76, 69, 8, 226, - 196, 28, 72, 2, 83, 2, 87, 215, 22, 69, 15, 58, 32, 114, 75, 53, 8, 78, - 66, 76, 69, 78, 68, 69, 68, 6, 29, 5, 87, 73, 84, 72, 32, 6, 26, 68, 255, - 192, 12, 77, 4, 172, 255, 8, 5, 79, 85, 66, 76, 69, 187, 8, 73, 5, 11, - 82, 2, 213, 4, 6, 65, 73, 78, 73, 65, 78, 2, 159, 144, 24, 32, 4, 176, - 197, 27, 4, 73, 68, 69, 32, 227, 147, 1, 69, 18, 62, 65, 28, 3, 69, 82, - 85, 134, 216, 28, 73, 2, 78, 3, 85, 7, 158, 216, 28, 69, 3, 84, 7, 33, 6, - 32, 87, 73, 84, 72, 32, 4, 138, 251, 8, 68, 131, 186, 3, 66, 18, 18, 69, - 71, 72, 9, 156, 1, 7, 32, 87, 73, 84, 72, 32, 68, 173, 211, 28, 2, 77, - 76, 10, 26, 69, 247, 191, 28, 87, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, - 26, 68, 203, 163, 12, 66, 4, 222, 131, 9, 73, 179, 173, 3, 69, 6, 37, 7, - 71, 65, 84, 85, 82, 69, 32, 6, 66, 65, 176, 140, 15, 2, 84, 69, 189, 155, - 13, 4, 69, 78, 32, 71, 2, 199, 185, 28, 32, 52, 198, 1, 66, 38, 68, 38, - 69, 36, 3, 71, 72, 69, 38, 72, 246, 188, 21, 89, 222, 157, 5, 83, 202, - 110, 84, 238, 8, 90, 246, 63, 73, 138, 19, 67, 186, 22, 80, 2, 86, 158, - 20, 75, 186, 2, 65, 2, 79, 3, 85, 4, 170, 172, 12, 89, 243, 166, 16, 69, - 6, 238, 210, 27, 90, 139, 128, 1, 69, 6, 210, 210, 28, 70, 2, 76, 3, 83, - 5, 201, 5, 5, 32, 87, 73, 84, 72, 4, 11, 65, 5, 171, 236, 26, 82, 2, 209, - 188, 15, 4, 72, 79, 85, 83, 184, 15, 178, 1, 65, 138, 4, 67, 54, 69, 198, - 35, 73, 250, 22, 79, 142, 44, 82, 174, 3, 85, 180, 180, 12, 13, 78, 65, - 32, 68, 79, 85, 66, 76, 69, 32, 72, 69, 76, 154, 231, 14, 86, 195, 47, - 76, 30, 116, 4, 71, 71, 69, 82, 146, 1, 78, 32, 4, 82, 75, 32, 83, 36, 2, - 83, 72, 200, 177, 21, 2, 76, 69, 155, 133, 1, 84, 9, 11, 32, 6, 44, 5, - 87, 73, 84, 72, 32, 143, 190, 6, 75, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, - 71, 72, 2, 201, 183, 27, 4, 84, 32, 71, 85, 4, 166, 144, 28, 67, 239, 30, - 71, 4, 254, 253, 20, 85, 175, 224, 4, 72, 10, 30, 32, 105, 3, 69, 68, 32, - 4, 60, 9, 87, 73, 84, 72, 32, 76, 69, 70, 84, 239, 185, 27, 83, 2, 17, 2, - 32, 85, 2, 175, 181, 22, 80, 6, 170, 215, 4, 84, 182, 147, 12, 76, 255, - 252, 9, 79, 10, 134, 204, 28, 49, 2, 50, 2, 51, 2, 52, 3, 83, 176, 4, - 182, 2, 67, 132, 2, 5, 71, 82, 69, 69, 32, 98, 76, 80, 21, 78, 84, 73, + 68, 32, 79, 86, 69, 82, 142, 134, 17, 68, 155, 168, 1, 82, 7, 26, 32, + 167, 162, 31, 50, 2, 169, 72, 4, 84, 73, 77, 69, 13, 26, 32, 139, 210, + 30, 73, 8, 76, 4, 67, 82, 79, 83, 0, 4, 79, 80, 80, 79, 162, 111, 84, + 143, 130, 23, 83, 2, 197, 158, 30, 5, 83, 73, 78, 71, 32, 224, 1, 78, 65, + 146, 15, 73, 242, 1, 85, 158, 73, 69, 229, 168, 23, 4, 87, 85, 51, 49, + 177, 1, 164, 1, 7, 32, 84, 73, 77, 69, 83, 32, 210, 9, 50, 66, 68, 102, + 75, 50, 76, 98, 77, 28, 4, 83, 75, 65, 76, 248, 192, 15, 6, 80, 32, 69, + 76, 65, 77, 219, 209, 15, 66, 134, 1, 170, 1, 65, 94, 66, 82, 69, 30, 71, + 182, 2, 73, 34, 75, 34, 76, 38, 77, 170, 1, 83, 118, 84, 34, 85, 214, 8, + 72, 246, 51, 78, 222, 215, 16, 80, 154, 221, 13, 82, 147, 17, 90, 13, 46, + 68, 250, 239, 30, 78, 165, 44, 2, 83, 72, 5, 229, 50, 7, 32, 80, 76, 85, + 83, 32, 75, 10, 34, 65, 194, 156, 31, 73, 3, 85, 6, 186, 179, 29, 76, + 134, 233, 1, 68, 3, 82, 4, 138, 25, 82, 151, 61, 83, 26, 30, 65, 98, 73, + 147, 1, 85, 11, 38, 82, 206, 123, 78, 231, 159, 30, 76, 5, 249, 21, 10, + 32, 80, 76, 85, 83, 32, 83, 72, 65, 51, 11, 32, 2, 83, 72, 171, 129, 24, + 82, 7, 11, 32, 4, 26, 67, 255, 130, 1, 80, 2, 37, 7, 82, 79, 83, 83, 73, + 78, 71, 2, 137, 185, 27, 2, 32, 71, 7, 182, 164, 27, 82, 179, 245, 3, 68, + 4, 234, 133, 31, 71, 219, 19, 77, 8, 250, 8, 73, 195, 164, 17, 65, 6, + 218, 163, 12, 85, 171, 245, 18, 73, 12, 18, 69, 83, 73, 9, 33, 6, 32, 80, + 76, 85, 83, 32, 6, 190, 243, 30, 68, 150, 14, 84, 255, 2, 71, 5, 45, 9, + 32, 80, 76, 85, 83, 32, 78, 85, 78, 2, 159, 244, 26, 85, 18, 62, 72, 178, + 196, 27, 65, 244, 217, 2, 2, 85, 72, 131, 120, 73, 10, 202, 154, 30, 85, + 142, 54, 73, 162, 70, 65, 3, 69, 4, 230, 221, 30, 65, 223, 56, 85, 17, + 110, 32, 238, 92, 82, 180, 142, 26, 9, 77, 85, 77, 32, 84, 73, 77, 69, + 83, 134, 169, 4, 83, 146, 1, 50, 3, 68, 2, 235, 137, 21, 85, 5, 197, 248, + 11, 11, 32, 67, 82, 79, 83, 83, 73, 78, 71, 32, 75, 10, 42, 53, 182, 148, + 31, 50, 2, 51, 3, 52, 5, 129, 249, 23, 9, 32, 79, 86, 69, 82, 32, 75, 65, + 68, 5, 189, 75, 8, 32, 84, 73, 77, 69, 83, 32, 73, 9, 26, 32, 219, 130, + 31, 65, 4, 166, 105, 84, 249, 187, 23, 9, 67, 82, 79, 83, 83, 73, 78, 71, + 32, 4, 234, 146, 31, 50, 3, 52, 7, 11, 32, 4, 70, 76, 1, 13, 79, 86, 69, + 82, 32, 75, 65, 83, 75, 65, 76, 32, 76, 2, 173, 116, 24, 65, 71, 65, 66, + 32, 84, 73, 77, 69, 83, 32, 85, 32, 79, 86, 69, 82, 32, 76, 65, 71, 65, + 66, 32, 21, 68, 7, 32, 84, 73, 77, 69, 83, 32, 50, 83, 150, 144, 31, 68, + 3, 78, 6, 26, 85, 255, 201, 30, 66, 5, 167, 144, 31, 68, 8, 52, 3, 73, + 77, 53, 170, 134, 30, 65, 183, 137, 1, 72, 5, 169, 244, 23, 11, 32, 79, + 86, 69, 82, 32, 75, 73, 83, 73, 77, 25, 200, 1, 29, 32, 79, 86, 69, 82, + 32, 72, 73, 32, 84, 73, 77, 69, 83, 32, 65, 83, 72, 50, 32, 75, 85, 32, + 79, 86, 69, 82, 32, 72, 18, 52, 54, 82, 186, 98, 83, 230, 1, 76, 242, + 168, 30, 51, 2, 55, 3, 78, 2, 155, 71, 73, 5, 221, 142, 30, 8, 32, 86, + 65, 82, 73, 65, 78, 84, 5, 213, 115, 9, 32, 79, 80, 80, 79, 83, 73, 78, + 71, 240, 2, 30, 65, 174, 26, 73, 59, 85, 141, 2, 68, 2, 71, 65, 252, 12, + 2, 75, 45, 222, 11, 76, 46, 77, 135, 55, 72, 118, 22, 66, 131, 11, 82, + 109, 11, 32, 106, 48, 6, 84, 73, 77, 69, 83, 32, 167, 219, 23, 83, 104, + 190, 1, 65, 186, 1, 66, 34, 71, 70, 72, 66, 73, 94, 75, 118, 76, 46, 77, + 46, 83, 138, 2, 84, 126, 85, 188, 165, 4, 8, 90, 85, 32, 79, 86, 69, 82, + 32, 214, 186, 25, 68, 218, 81, 69, 143, 57, 78, 15, 80, 6, 32, 80, 76, + 85, 83, 32, 76, 4, 83, 72, 32, 90, 178, 136, 31, 76, 3, 78, 6, 38, 68, + 158, 231, 29, 71, 251, 23, 76, 2, 201, 62, 5, 65, 32, 80, 76, 85, 2, 149, + 112, 2, 73, 68, 4, 246, 193, 30, 65, 163, 70, 73, 10, 48, 2, 85, 68, 254, + 180, 27, 65, 203, 210, 3, 73, 5, 191, 105, 32, 6, 164, 238, 17, 7, 73, + 32, 84, 73, 77, 69, 83, 227, 156, 12, 65, 8, 22, 77, 175, 62, 71, 7, 33, + 6, 32, 80, 76, 85, 83, 32, 4, 206, 225, 30, 76, 179, 34, 72, 10, 26, 85, + 251, 138, 29, 73, 6, 26, 76, 227, 133, 31, 51, 5, 41, 8, 32, 80, 76, 85, + 83, 32, 72, 73, 2, 219, 65, 32, 8, 198, 100, 65, 190, 184, 28, 73, 247, + 107, 85, 6, 26, 69, 191, 156, 29, 85, 5, 175, 25, 32, 12, 26, 72, 219, + 243, 30, 85, 10, 100, 14, 73, 84, 65, 32, 80, 76, 85, 83, 32, 71, 73, 83, + 72, 32, 96, 2, 85, 50, 217, 3, 2, 69, 32, 4, 48, 6, 80, 76, 85, 83, 32, + 69, 163, 133, 26, 84, 2, 11, 82, 2, 11, 73, 2, 207, 233, 23, 78, 5, 189, + 73, 5, 32, 80, 76, 85, 83, 6, 86, 65, 189, 30, 16, 69, 32, 80, 76, 85, + 83, 32, 65, 32, 80, 76, 85, 83, 32, 83, 85, 4, 154, 231, 23, 75, 211, + 154, 7, 71, 13, 72, 6, 32, 80, 76, 85, 83, 32, 190, 41, 50, 178, 214, 30, + 83, 147, 1, 68, 4, 26, 85, 227, 128, 31, 65, 2, 255, 40, 32, 11, 11, 32, + 8, 68, 4, 71, 85, 78, 85, 97, 9, 84, 73, 77, 69, 83, 32, 83, 72, 69, 5, + 73, 16, 32, 79, 86, 69, 82, 32, 76, 65, 71, 65, 82, 32, 71, 85, 78, 85, + 2, 151, 210, 30, 32, 5, 11, 32, 2, 157, 196, 14, 4, 80, 76, 85, 83, 136, + 1, 82, 48, 146, 2, 49, 30, 50, 114, 51, 82, 52, 222, 2, 54, 154, 4, 55, + 243, 24, 53, 22, 158, 1, 50, 30, 56, 180, 52, 15, 55, 57, 32, 79, 86, 69, + 82, 32, 76, 65, 75, 45, 48, 55, 57, 246, 245, 10, 53, 134, 185, 12, 54, + 2, 57, 94, 51, 203, 162, 3, 48, 4, 242, 252, 30, 49, 3, 53, 4, 136, 227, + 23, 12, 49, 32, 79, 86, 69, 82, 32, 76, 65, 75, 45, 48, 207, 153, 7, 48, + 4, 214, 226, 23, 52, 95, 51, 16, 46, 50, 38, 54, 222, 219, 23, 49, 159, + 2, 51, 6, 182, 251, 30, 48, 2, 53, 3, 56, 4, 146, 251, 30, 53, 3, 54, 12, + 42, 52, 254, 223, 11, 56, 131, 130, 12, 57, 6, 202, 250, 30, 51, 2, 55, + 3, 56, 30, 62, 52, 214, 1, 53, 30, 57, 170, 223, 23, 55, 203, 162, 3, 56, + 14, 26, 57, 207, 249, 30, 49, 13, 37, 7, 32, 84, 73, 77, 69, 83, 32, 10, + 116, 9, 80, 65, 80, 32, 80, 76, 85, 83, 32, 240, 151, 13, 7, 85, 50, 32, + 80, 76, 85, 83, 238, 31, 73, 235, 155, 17, 71, 4, 210, 13, 80, 51, 76, 4, + 146, 248, 30, 48, 3, 55, 8, 246, 247, 30, 48, 2, 50, 2, 51, 3, 53, 46, + 60, 2, 49, 55, 240, 1, 2, 52, 56, 130, 216, 23, 48, 23, 51, 23, 37, 7, + 32, 84, 73, 77, 69, 83, 32, 20, 122, 84, 34, 85, 162, 11, 75, 132, 34, 9, + 68, 85, 78, 51, 32, 71, 85, 78, 85, 134, 224, 28, 65, 246, 161, 1, 66, + 247, 67, 76, 4, 138, 189, 30, 65, 223, 56, 69, 6, 178, 199, 14, 82, 150, + 174, 16, 50, 3, 68, 21, 37, 7, 32, 84, 73, 77, 69, 83, 32, 18, 154, 1, + 85, 220, 8, 4, 80, 65, 80, 32, 146, 45, 73, 196, 219, 12, 10, 83, 72, 69, + 83, 72, 32, 80, 76, 85, 83, 210, 132, 5, 68, 130, 202, 12, 78, 163, 17, + 71, 4, 194, 197, 14, 82, 151, 174, 16, 68, 4, 226, 216, 23, 50, 143, 165, + 3, 52, 5, 11, 32, 2, 145, 6, 4, 84, 73, 77, 69, 7, 49, 10, 32, 84, 73, + 77, 69, 83, 32, 75, 85, 82, 5, 225, 206, 11, 5, 32, 80, 76, 85, 83, 9, + 204, 206, 11, 2, 77, 77, 170, 162, 19, 83, 147, 1, 76, 93, 90, 50, 212, + 7, 3, 71, 65, 76, 142, 1, 77, 130, 62, 32, 150, 170, 30, 51, 2, 72, 3, + 76, 69, 11, 32, 66, 88, 4, 67, 82, 79, 83, 0, 4, 79, 80, 80, 79, 44, 4, + 71, 85, 78, 85, 38, 83, 35, 84, 2, 209, 204, 11, 6, 83, 73, 78, 71, 32, + 76, 2, 193, 24, 5, 32, 84, 73, 77, 69, 6, 250, 68, 72, 227, 250, 22, 81, + 54, 44, 5, 73, 77, 69, 83, 32, 251, 201, 30, 69, 52, 188, 1, 4, 69, 83, + 72, 50, 94, 72, 42, 75, 68, 2, 76, 65, 34, 77, 60, 3, 80, 65, 80, 118, + 83, 90, 84, 250, 56, 71, 158, 251, 7, 78, 178, 212, 18, 68, 198, 215, 2, + 65, 138, 67, 66, 215, 53, 73, 7, 11, 32, 4, 26, 80, 147, 239, 25, 84, 2, + 17, 2, 76, 85, 2, 173, 227, 29, 3, 83, 32, 76, 4, 184, 66, 2, 73, 32, + 227, 160, 29, 65, 8, 32, 2, 65, 68, 135, 236, 30, 73, 6, 226, 19, 51, + 163, 216, 30, 50, 4, 174, 20, 32, 219, 181, 17, 71, 2, 11, 69, 2, 11, 32, + 2, 241, 175, 13, 4, 80, 76, 85, 83, 5, 11, 32, 2, 33, 6, 80, 76, 85, 83, + 32, 80, 2, 45, 9, 65, 80, 32, 80, 76, 85, 83, 32, 76, 2, 159, 244, 26, + 85, 6, 26, 73, 147, 197, 30, 72, 4, 194, 18, 32, 181, 171, 26, 7, 75, 50, + 32, 80, 76, 85, 83, 4, 162, 53, 85, 139, 24, 65, 9, 11, 32, 6, 22, 79, + 207, 67, 83, 4, 56, 7, 80, 80, 79, 83, 73, 78, 71, 1, 3, 86, 69, 82, 2, + 21, 3, 32, 76, 85, 2, 231, 222, 29, 71, 7, 45, 9, 32, 79, 86, 69, 82, 32, + 76, 85, 77, 5, 187, 59, 32, 70, 34, 65, 70, 69, 22, 73, 71, 85, 17, 170, + 45, 32, 188, 1, 2, 83, 72, 202, 184, 30, 50, 2, 72, 3, 82, 7, 207, 198, + 26, 83, 7, 164, 241, 26, 8, 32, 80, 76, 85, 83, 32, 90, 65, 179, 245, 3, + 78, 43, 104, 2, 83, 72, 186, 60, 71, 144, 133, 11, 5, 32, 79, 86, 69, 82, + 40, 2, 82, 71, 225, 197, 6, 2, 78, 83, 31, 22, 32, 183, 2, 51, 16, 136, + 1, 9, 79, 86, 69, 82, 32, 77, 85, 83, 72, 124, 6, 84, 73, 77, 69, 83, 32, + 209, 216, 26, 10, 67, 82, 79, 83, 83, 73, 78, 71, 32, 77, 9, 37, 7, 32, + 84, 73, 77, 69, 83, 32, 6, 42, 65, 238, 173, 28, 75, 143, 179, 2, 71, 2, + 241, 188, 11, 5, 32, 80, 76, 85, 83, 6, 178, 234, 29, 75, 158, 118, 90, + 187, 2, 65, 13, 11, 32, 10, 44, 6, 84, 73, 77, 69, 83, 32, 203, 57, 71, + 8, 38, 65, 162, 206, 30, 68, 163, 17, 90, 5, 205, 129, 13, 5, 32, 80, 76, + 85, 83, 158, 1, 42, 65, 134, 2, 69, 94, 73, 199, 7, 85, 23, 52, 2, 71, + 65, 166, 1, 77, 198, 223, 30, 50, 3, 52, 11, 26, 32, 207, 224, 30, 82, 6, + 100, 8, 79, 80, 80, 79, 83, 73, 78, 71, 146, 59, 73, 197, 14, 9, 84, 73, + 77, 69, 83, 32, 83, 72, 85, 2, 157, 157, 30, 3, 32, 78, 65, 7, 204, 21, + 2, 32, 78, 247, 201, 30, 50, 9, 11, 32, 6, 44, 6, 84, 73, 77, 69, 83, 32, + 179, 57, 83, 4, 190, 152, 30, 85, 163, 70, 65, 73, 90, 77, 78, 78, 168, + 182, 20, 6, 32, 84, 73, 77, 69, 83, 238, 189, 8, 83, 135, 233, 1, 50, 7, + 45, 9, 32, 84, 73, 77, 69, 83, 32, 71, 65, 4, 158, 3, 82, 179, 58, 78, + 59, 36, 3, 68, 65, 50, 243, 220, 30, 57, 55, 37, 7, 32, 84, 73, 77, 69, + 83, 32, 52, 162, 1, 65, 34, 71, 50, 75, 16, 5, 76, 65, 75, 45, 48, 22, + 77, 86, 78, 34, 80, 76, 3, 83, 72, 69, 94, 85, 240, 15, 3, 68, 73, 77, + 230, 190, 29, 66, 235, 117, 72, 6, 246, 2, 83, 175, 216, 30, 78, 8, 26, + 73, 183, 216, 29, 85, 5, 215, 217, 30, 83, 2, 211, 20, 69, 2, 243, 193, + 23, 53, 4, 26, 69, 247, 241, 28, 65, 2, 25, 4, 32, 80, 76, 85, 2, 177, + 41, 3, 83, 32, 71, 4, 250, 137, 30, 85, 227, 79, 69, 2, 33, 6, 65, 80, + 32, 80, 76, 85, 2, 11, 83, 2, 157, 200, 29, 2, 32, 80, 9, 37, 7, 32, 80, + 76, 85, 83, 32, 65, 6, 26, 83, 175, 142, 29, 32, 4, 11, 72, 5, 107, 32, + 11, 50, 32, 34, 50, 174, 169, 14, 82, 135, 173, 16, 83, 2, 173, 142, 14, + 3, 80, 76, 85, 2, 11, 32, 2, 21, 3, 80, 76, 85, 2, 11, 83, 2, 227, 238, + 28, 32, 57, 24, 2, 49, 49, 99, 78, 9, 11, 32, 6, 200, 25, 9, 79, 86, 69, + 82, 32, 78, 85, 49, 49, 150, 191, 25, 84, 207, 161, 3, 82, 47, 30, 32, + 169, 3, 2, 85, 90, 18, 144, 1, 12, 67, 82, 79, 83, 83, 73, 78, 71, 32, + 78, 85, 78, 72, 12, 76, 65, 71, 65, 82, 32, 84, 73, 77, 69, 83, 32, 174, + 1, 79, 231, 212, 25, 84, 5, 209, 40, 14, 32, 76, 65, 71, 65, 82, 32, 79, + 86, 69, 82, 32, 76, 65, 10, 56, 3, 83, 65, 76, 158, 235, 28, 77, 14, 85, + 235, 70, 71, 5, 133, 202, 29, 23, 32, 79, 86, 69, 82, 32, 78, 85, 78, 32, + 76, 65, 71, 65, 82, 32, 84, 73, 77, 69, 83, 32, 83, 2, 201, 185, 17, 3, + 86, 69, 82, 27, 11, 32, 24, 120, 10, 65, 66, 50, 32, 84, 73, 77, 69, 83, + 32, 205, 37, 15, 75, 73, 83, 73, 77, 53, 32, 84, 73, 77, 69, 83, 32, 66, + 73, 20, 168, 1, 2, 75, 65, 202, 7, 73, 212, 39, 2, 65, 83, 130, 177, 2, + 68, 232, 206, 8, 3, 83, 73, 76, 214, 135, 12, 85, 210, 237, 5, 71, 138, + 149, 1, 78, 254, 2, 66, 163, 17, 76, 2, 251, 217, 26, 68, 46, 42, 65, 36, + 4, 69, 83, 72, 50, 23, 73, 9, 194, 207, 30, 68, 2, 78, 3, 80, 5, 191, + 187, 28, 32, 35, 22, 32, 151, 1, 82, 20, 80, 6, 84, 73, 77, 69, 83, 32, + 201, 131, 18, 8, 67, 82, 79, 83, 83, 73, 78, 71, 18, 214, 21, 85, 150, + 48, 65, 2, 73, 254, 199, 29, 66, 187, 64, 69, 12, 32, 2, 73, 71, 191, + 205, 30, 50, 11, 11, 32, 8, 96, 6, 84, 73, 77, 69, 83, 32, 197, 153, 26, + 12, 79, 80, 80, 79, 83, 73, 78, 71, 32, 80, 73, 82, 6, 138, 195, 29, 75, + 162, 67, 85, 235, 67, 90, 8, 234, 67, 65, 182, 136, 30, 73, 3, 85, 202, + 1, 46, 65, 250, 5, 72, 150, 11, 73, 163, 1, 85, 63, 46, 71, 190, 4, 76, + 122, 78, 227, 197, 30, 82, 53, 11, 32, 50, 92, 4, 71, 85, 78, 85, 54, 78, + 32, 6, 84, 73, 77, 69, 83, 32, 241, 21, 4, 79, 86, 69, 82, 5, 29, 5, 32, + 84, 73, 77, 69, 2, 159, 230, 17, 83, 2, 149, 164, 25, 3, 85, 84, 73, 42, + 150, 1, 73, 42, 75, 34, 83, 64, 2, 84, 65, 38, 85, 220, 62, 2, 68, 85, + 166, 223, 28, 76, 226, 40, 78, 210, 48, 69, 138, 60, 77, 162, 17, 72, + 187, 2, 65, 2, 11, 71, 2, 11, 73, 2, 191, 31, 32, 4, 182, 143, 30, 85, + 199, 53, 65, 6, 26, 72, 167, 190, 29, 65, 4, 162, 192, 13, 69, 151, 193, + 16, 73, 4, 226, 172, 23, 75, 211, 154, 7, 66, 10, 254, 197, 30, 83, 146, + 1, 50, 2, 66, 2, 77, 3, 82, 5, 33, 6, 32, 76, 65, 71, 65, 66, 2, 37, 7, + 32, 84, 73, 77, 69, 83, 32, 2, 11, 65, 2, 11, 83, 2, 199, 172, 23, 72, 2, + 135, 169, 11, 71, 106, 46, 65, 250, 1, 69, 242, 2, 73, 239, 3, 85, 29, + 50, 51, 182, 1, 54, 174, 166, 23, 66, 223, 3, 82, 19, 37, 7, 32, 84, 73, + 77, 69, 83, 32, 16, 90, 85, 146, 25, 83, 222, 201, 26, 71, 166, 232, 2, + 84, 170, 50, 66, 218, 47, 78, 215, 22, 65, 5, 11, 32, 2, 181, 131, 26, 4, + 80, 76, 85, 83, 5, 175, 45, 32, 27, 62, 32, 140, 2, 2, 83, 72, 150, 203, + 26, 71, 199, 245, 3, 78, 14, 84, 8, 79, 86, 69, 82, 32, 83, 72, 69, 88, + 5, 80, 76, 85, 83, 32, 207, 156, 30, 72, 7, 11, 32, 4, 190, 15, 71, 141, + 6, 12, 84, 65, 66, 32, 79, 86, 69, 82, 32, 84, 65, 66, 6, 48, 2, 72, 85, + 20, 2, 78, 65, 183, 159, 29, 83, 2, 211, 167, 23, 66, 2, 191, 167, 23, + 77, 7, 218, 133, 29, 76, 255, 186, 1, 50, 40, 62, 68, 74, 77, 218, 1, 82, + 206, 138, 26, 78, 211, 176, 4, 84, 7, 37, 7, 32, 84, 73, 77, 69, 83, 32, + 4, 242, 174, 30, 73, 219, 16, 65, 25, 37, 7, 32, 84, 73, 77, 69, 83, 32, + 22, 114, 66, 38, 73, 130, 19, 75, 134, 187, 2, 77, 178, 211, 2, 76, 130, + 251, 23, 71, 250, 23, 83, 238, 9, 68, 215, 127, 65, 4, 154, 206, 2, 85, + 195, 230, 26, 65, 4, 249, 20, 2, 71, 73, 7, 11, 32, 4, 60, 9, 79, 86, 69, + 82, 32, 83, 72, 73, 82, 151, 191, 25, 84, 2, 177, 196, 29, 11, 32, 66, + 85, 82, 32, 79, 86, 69, 82, 32, 66, 13, 88, 14, 32, 79, 86, 69, 82, 32, + 73, 78, 86, 69, 82, 84, 69, 68, 34, 50, 251, 194, 29, 66, 2, 11, 32, 2, + 235, 205, 25, 83, 7, 33, 6, 32, 80, 76, 85, 83, 32, 4, 88, 7, 69, 50, 32, + 84, 73, 77, 69, 177, 152, 11, 9, 68, 85, 71, 32, 84, 73, 77, 69, 83, 2, + 131, 132, 13, 83, 17, 50, 32, 30, 71, 234, 152, 11, 76, 215, 135, 12, 75, + 4, 138, 8, 84, 163, 9, 71, 7, 11, 52, 5, 49, 10, 32, 79, 86, 69, 82, 32, + 83, 73, 71, 52, 2, 199, 14, 32, 19, 78, 68, 22, 77, 22, 82, 156, 217, 12, + 5, 32, 79, 86, 69, 82, 179, 230, 16, 72, 5, 199, 184, 30, 50, 5, 147, + 208, 28, 65, 5, 159, 184, 30, 57, 72, 46, 65, 150, 4, 73, 250, 1, 85, + 227, 8, 69, 37, 66, 32, 94, 66, 166, 1, 71, 148, 1, 2, 75, 52, 135, 180, + 30, 82, 8, 60, 6, 84, 73, 77, 69, 83, 32, 130, 14, 71, 135, 149, 28, 65, + 4, 254, 162, 30, 72, 3, 77, 7, 11, 32, 4, 180, 203, 17, 29, 79, 86, 69, + 82, 32, 84, 65, 66, 32, 78, 73, 32, 79, 86, 69, 82, 32, 78, 73, 32, 68, + 73, 83, 72, 32, 79, 86, 69, 82, 143, 187, 5, 83, 15, 37, 7, 32, 84, 73, + 77, 69, 83, 32, 12, 74, 84, 252, 254, 7, 2, 83, 72, 206, 137, 21, 71, + 190, 101, 85, 203, 50, 66, 2, 11, 85, 2, 235, 154, 23, 71, 5, 29, 5, 32, + 80, 76, 85, 83, 2, 221, 202, 28, 2, 32, 83, 17, 46, 82, 150, 29, 32, 134, + 150, 30, 50, 3, 76, 9, 11, 32, 6, 48, 8, 79, 86, 69, 82, 32, 84, 73, 82, + 99, 84, 5, 11, 32, 2, 11, 71, 2, 21, 3, 65, 68, 32, 2, 241, 5, 8, 79, 86, + 69, 82, 32, 71, 65, 68, 2, 217, 21, 6, 73, 77, 69, 83, 32, 84, 17, 50, + 77, 114, 82, 130, 151, 23, 71, 175, 153, 7, 75, 7, 37, 7, 32, 84, 73, 77, + 69, 83, 32, 4, 46, 71, 205, 197, 17, 5, 84, 72, 82, 69, 69, 2, 221, 16, + 2, 65, 78, 5, 141, 139, 11, 17, 32, 79, 86, 69, 82, 32, 84, 85, 82, 32, + 90, 65, 32, 79, 86, 69, 82, 179, 1, 138, 1, 32, 246, 2, 68, 226, 2, 78, + 46, 77, 158, 2, 82, 128, 9, 2, 83, 72, 174, 1, 90, 224, 185, 12, 2, 84, + 85, 158, 226, 17, 50, 3, 66, 12, 64, 7, 79, 86, 69, 82, 32, 85, 32, 158, + 2, 85, 139, 128, 29, 71, 6, 200, 1, 10, 80, 65, 32, 79, 86, 69, 82, 32, + 80, 65, 216, 159, 9, 19, 85, 32, 82, 69, 86, 69, 82, 83, 69, 68, 32, 79, + 86, 69, 82, 32, 85, 32, 82, 137, 148, 20, 10, 83, 85, 82, 32, 79, 86, 69, + 82, 32, 83, 2, 11, 32, 2, 45, 9, 71, 65, 82, 32, 79, 86, 69, 82, 32, 2, + 191, 138, 29, 71, 5, 251, 134, 30, 32, 21, 26, 32, 227, 170, 30, 85, 16, + 70, 75, 44, 2, 83, 72, 100, 6, 84, 73, 77, 69, 83, 32, 135, 1, 71, 2, 11, + 85, 2, 11, 83, 2, 155, 135, 11, 72, 4, 29, 5, 69, 83, 72, 73, 71, 5, 11, + 32, 2, 11, 84, 2, 217, 227, 29, 6, 73, 77, 69, 83, 32, 66, 8, 92, 14, 85, + 32, 80, 76, 85, 83, 32, 85, 32, 80, 76, 85, 83, 32, 210, 226, 29, 66, + 215, 50, 77, 4, 11, 85, 5, 11, 32, 2, 11, 71, 2, 227, 131, 30, 85, 21, + 72, 7, 32, 84, 73, 77, 69, 83, 32, 136, 1, 2, 85, 77, 159, 167, 29, 66, + 10, 50, 76, 16, 2, 77, 69, 50, 83, 135, 167, 30, 85, 2, 231, 6, 65, 5, + 11, 32, 2, 201, 241, 25, 4, 80, 76, 85, 83, 2, 131, 134, 11, 72, 7, 37, + 7, 32, 84, 73, 77, 69, 83, 32, 4, 158, 11, 75, 243, 152, 30, 80, 93, 54, + 32, 102, 50, 194, 2, 73, 22, 85, 187, 162, 30, 52, 4, 62, 83, 221, 172, + 29, 9, 67, 82, 79, 83, 83, 73, 78, 71, 32, 2, 165, 242, 25, 4, 72, 69, + 83, 72, 23, 11, 32, 20, 42, 73, 37, 6, 84, 73, 77, 69, 83, 32, 2, 165, + 130, 24, 4, 78, 86, 69, 82, 18, 46, 65, 82, 85, 130, 163, 29, 78, 251, + 125, 72, 6, 48, 6, 32, 80, 76, 85, 83, 32, 207, 163, 30, 76, 4, 146, 161, + 30, 72, 3, 78, 8, 26, 50, 151, 163, 30, 68, 7, 33, 6, 32, 80, 76, 85, 83, + 32, 4, 206, 186, 28, 65, 203, 212, 1, 66, 5, 203, 162, 30, 51, 59, 56, 7, + 32, 84, 73, 77, 69, 83, 32, 165, 4, 2, 68, 65, 52, 134, 1, 65, 46, 68, + 38, 71, 82, 73, 46, 76, 78, 83, 46, 85, 250, 252, 28, 66, 230, 33, 77, + 238, 90, 84, 146, 17, 75, 162, 17, 72, 3, 80, 5, 11, 83, 2, 11, 72, 2, + 255, 254, 16, 71, 4, 190, 254, 10, 65, 171, 210, 18, 85, 10, 26, 65, 139, + 160, 30, 85, 9, 34, 78, 230, 159, 30, 76, 3, 82, 2, 211, 9, 50, 6, 250, + 139, 30, 71, 202, 18, 83, 147, 1, 77, 6, 46, 85, 221, 129, 23, 5, 65, 75, + 45, 54, 54, 4, 246, 158, 30, 51, 3, 77, 4, 136, 132, 23, 2, 73, 71, 255, + 131, 7, 72, 6, 42, 32, 242, 239, 13, 82, 151, 174, 16, 68, 2, 165, 242, + 28, 6, 80, 76, 85, 83, 32, 71, 5, 11, 32, 2, 153, 212, 13, 4, 84, 73, 77, + 69, 17, 84, 7, 32, 84, 73, 77, 69, 83, 32, 172, 144, 29, 2, 85, 77, 166, + 140, 1, 50, 3, 88, 8, 50, 84, 212, 201, 26, 2, 75, 85, 203, 210, 3, 65, + 2, 11, 65, 2, 191, 129, 23, 75, 6, 26, 51, 227, 155, 30, 85, 5, 29, 5, + 32, 84, 73, 77, 69, 2, 21, 3, 83, 32, 75, 2, 11, 65, 2, 139, 173, 23, 83, + 42, 50, 65, 190, 1, 73, 146, 1, 85, 223, 254, 22, 69, 13, 50, 32, 254, + 141, 29, 77, 166, 140, 1, 55, 3, 71, 4, 56, 8, 83, 81, 85, 65, 82, 69, + 68, 32, 243, 155, 25, 84, 2, 11, 84, 2, 21, 3, 73, 77, 69, 2, 11, 83, 2, + 221, 160, 29, 2, 32, 75, 15, 86, 66, 220, 229, 22, 6, 32, 79, 86, 69, 82, + 32, 186, 25, 90, 174, 153, 7, 51, 3, 71, 5, 17, 2, 32, 75, 2, 17, 2, 65, + 66, 2, 135, 2, 65, 15, 84, 10, 32, 79, 86, 69, 82, 32, 90, 85, 32, 80, + 42, 53, 182, 158, 29, 66, 215, 120, 77, 2, 245, 245, 28, 5, 76, 85, 83, + 32, 83, 7, 37, 7, 32, 84, 73, 77, 69, 83, 32, 4, 44, 5, 84, 72, 82, 69, + 69, 179, 150, 30, 65, 2, 29, 5, 32, 68, 73, 83, 72, 2, 11, 32, 2, 143, + 152, 25, 84, 8, 42, 32, 154, 228, 22, 73, 255, 200, 3, 67, 4, 198, 159, + 22, 79, 209, 247, 6, 8, 87, 73, 84, 72, 32, 83, 84, 82, 18, 134, 1, 76, + 162, 1, 82, 249, 206, 29, 23, 86, 69, 68, 32, 83, 84, 69, 77, 32, 80, 65, + 82, 65, 71, 82, 65, 80, 72, 32, 83, 73, 71, 78, 10, 48, 2, 89, 32, 241, + 162, 3, 4, 73, 78, 71, 32, 8, 68, 2, 76, 79, 193, 243, 27, 9, 66, 82, 65, + 67, 75, 69, 84, 32, 69, 6, 214, 209, 26, 71, 155, 193, 3, 79, 6, 52, 5, + 69, 78, 67, 89, 32, 53, 4, 89, 32, 65, 78, 4, 212, 247, 24, 4, 69, 88, + 67, 72, 227, 147, 4, 83, 2, 217, 235, 19, 3, 68, 32, 82, 4, 218, 131, 3, + 79, 135, 247, 25, 65, 240, 8, 128, 1, 2, 80, 82, 140, 9, 7, 82, 73, 76, + 76, 73, 67, 32, 132, 223, 25, 7, 76, 73, 78, 68, 82, 73, 67, 205, 253, 2, + 2, 67, 76, 180, 2, 140, 1, 13, 73, 79, 84, 32, 83, 89, 76, 76, 65, 66, + 76, 69, 32, 169, 1, 16, 79, 45, 77, 73, 78, 79, 65, 78, 32, 83, 73, 71, + 78, 32, 67, 77, 110, 238, 177, 7, 75, 2, 76, 2, 77, 2, 78, 2, 80, 2, 82, + 2, 83, 2, 84, 126, 87, 138, 42, 74, 2, 90, 130, 239, 6, 88, 154, 195, 15, + 65, 2, 69, 2, 73, 2, 79, 3, 85, 198, 1, 46, 48, 146, 5, 49, 153, 135, 26, + 2, 51, 48, 170, 1, 102, 48, 78, 49, 74, 50, 78, 51, 78, 52, 62, 53, 86, + 55, 190, 186, 10, 57, 190, 3, 54, 215, 149, 14, 56, 16, 194, 140, 30, 49, + 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 16, 194, 3, 50, 182, + 136, 30, 48, 2, 49, 2, 51, 2, 53, 2, 55, 3, 57, 16, 174, 139, 30, 49, 2, + 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 16, 226, 138, 30, 48, 2, + 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 12, 150, 138, 30, 48, 2, + 49, 2, 52, 2, 54, 2, 55, 3, 57, 18, 218, 137, 30, 48, 2, 49, 2, 50, 2, + 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 20, 82, 53, 182, 136, 30, 48, 2, + 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, 56, 3, 57, 5, 179, 136, 30, 66, 24, + 18, 48, 87, 49, 18, 138, 136, 30, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, + 2, 55, 2, 56, 3, 57, 6, 182, 135, 30, 48, 2, 50, 3, 52, 184, 6, 140, 1, + 9, 67, 65, 80, 73, 84, 65, 76, 32, 76, 202, 4, 75, 32, 7, 76, 69, 84, 84, + 69, 82, 32, 132, 1, 3, 80, 65, 89, 30, 83, 211, 41, 84, 242, 2, 44, 6, + 69, 84, 84, 69, 82, 32, 175, 43, 73, 236, 2, 150, 2, 76, 46, 78, 34, 80, + 34, 82, 58, 84, 62, 85, 190, 5, 65, 214, 1, 66, 242, 1, 67, 150, 1, 68, + 254, 1, 69, 206, 2, 71, 130, 1, 72, 134, 1, 73, 230, 3, 75, 226, 3, 77, + 218, 1, 79, 226, 2, 83, 218, 7, 89, 166, 1, 90, 162, 183, 29, 70, 134, + 14, 74, 2, 86, 2, 87, 159, 20, 81, 6, 222, 25, 73, 254, 210, 29, 74, 159, + 20, 72, 4, 134, 27, 69, 171, 209, 29, 74, 8, 134, 34, 69, 135, 205, 29, + 83, 12, 220, 9, 3, 79, 85, 78, 226, 18, 69, 207, 227, 29, 72, 20, 174, + 33, 69, 106, 83, 234, 178, 29, 67, 186, 22, 74, 3, 87, 13, 198, 34, 32, + 115, 75, 2, 205, 248, 29, 3, 65, 86, 89, 6, 248, 10, 5, 77, 85, 76, 84, + 73, 200, 129, 13, 2, 80, 65, 241, 231, 11, 14, 83, 77, 65, 76, 76, 32, + 67, 65, 80, 73, 84, 65, 76, 32, 2, 177, 214, 29, 2, 69, 82, 186, 3, 136, + 1, 6, 77, 65, 76, 76, 32, 76, 201, 37, 22, 85, 66, 83, 67, 82, 73, 80, + 84, 32, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 134, 3, 44, + 6, 69, 84, 84, 69, 82, 32, 151, 36, 73, 128, 3, 154, 2, 65, 214, 1, 66, + 242, 1, 67, 150, 1, 68, 254, 1, 69, 206, 2, 71, 130, 1, 72, 134, 1, 73, + 230, 3, 75, 222, 2, 76, 134, 1, 77, 114, 78, 106, 79, 110, 80, 50, 82, + 198, 1, 83, 218, 2, 84, 218, 2, 85, 246, 1, 87, 54, 89, 166, 1, 90, 162, + 183, 29, 70, 134, 14, 74, 2, 86, 159, 20, 81, 17, 108, 6, 32, 87, 73, 84, + 72, 32, 36, 9, 66, 75, 72, 65, 83, 73, 65, 78, 32, 249, 152, 12, 4, 76, + 69, 85, 84, 4, 186, 194, 9, 68, 147, 182, 3, 66, 8, 172, 133, 13, 3, 67, + 72, 69, 158, 213, 6, 68, 255, 157, 10, 72, 18, 110, 65, 78, 73, 32, 3, + 82, 79, 65, 232, 160, 3, 6, 76, 69, 78, 68, 69, 68, 246, 224, 9, 89, 139, + 247, 16, 69, 6, 200, 21, 6, 82, 82, 69, 68, 32, 79, 253, 129, 12, 5, 83, + 72, 75, 73, 82, 4, 230, 2, 78, 163, 158, 3, 71, 2, 171, 179, 10, 68, 14, + 76, 2, 72, 69, 246, 9, 76, 220, 146, 13, 4, 82, 79, 83, 83, 167, 174, 16, + 67, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 150, 29, 68, 255, 169, 29, 86, + 26, 96, 2, 74, 69, 20, 6, 79, 85, 66, 76, 69, 32, 66, 90, 250, 200, 29, + 67, 186, 22, 87, 215, 22, 69, 5, 219, 230, 22, 82, 4, 36, 3, 77, 79, 78, + 159, 246, 29, 79, 2, 153, 13, 2, 79, 67, 12, 46, 69, 182, 245, 28, 90, + 206, 105, 72, 3, 87, 5, 223, 214, 29, 76, 41, 86, 76, 98, 78, 110, 82, + 166, 15, 32, 170, 238, 12, 83, 178, 193, 14, 77, 163, 180, 2, 70, 11, 33, + 6, 32, 87, 73, 84, 72, 32, 8, 166, 20, 77, 130, 235, 12, 68, 190, 193, + 14, 84, 143, 96, 72, 13, 33, 6, 32, 87, 73, 84, 72, 32, 10, 198, 19, 77, + 130, 235, 12, 68, 202, 52, 76, 246, 140, 14, 84, 143, 96, 72, 5, 133, + 195, 28, 5, 32, 87, 73, 84, 72, 14, 32, 2, 72, 69, 255, 219, 29, 74, 13, + 33, 6, 32, 87, 73, 84, 72, 32, 10, 142, 18, 77, 146, 15, 85, 242, 219, + 12, 68, 155, 31, 83, 12, 26, 65, 135, 219, 29, 87, 11, 48, 6, 32, 87, 73, + 84, 72, 32, 207, 136, 28, 82, 6, 134, 252, 12, 68, 202, 161, 15, 72, 219, + 163, 1, 83, 35, 84, 6, 32, 87, 73, 84, 72, 32, 50, 69, 74, 79, 201, 1, 6, + 90, 72, 73, 84, 83, 65, 6, 174, 183, 9, 68, 214, 10, 71, 243, 196, 3, 77, + 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 174, 193, 9, 71, 191, 171, 3, 66, + 17, 11, 84, 14, 48, 6, 73, 70, 73, 69, 68, 32, 227, 238, 29, 65, 12, 80, + 2, 67, 76, 38, 76, 190, 149, 3, 66, 198, 196, 25, 89, 234, 147, 1, 65, 3, + 69, 2, 33, 6, 79, 83, 69, 68, 32, 76, 2, 151, 4, 73, 5, 197, 136, 13, 14, + 32, 87, 73, 84, 72, 32, 68, 79, 85, 66, 76, 69, 32, 71, 34, 102, 65, 98, + 79, 184, 190, 29, 11, 72, 65, 75, 65, 83, 83, 73, 65, 78, 32, 67, 186, + 22, 74, 255, 2, 83, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 226, 246, 12, + 68, 202, 161, 15, 72, 142, 163, 1, 86, 79, 83, 18, 36, 3, 77, 73, 32, + 203, 177, 26, 80, 16, 58, 68, 246, 160, 12, 76, 2, 78, 2, 83, 2, 84, 3, + 90, 6, 242, 160, 12, 90, 154, 179, 17, 74, 215, 22, 69, 8, 94, 73, 228, + 200, 10, 10, 79, 78, 71, 45, 76, 69, 71, 71, 69, 68, 154, 138, 19, 74, + 159, 20, 72, 2, 209, 145, 3, 4, 84, 84, 76, 69, 4, 21, 3, 79, 78, 79, 4, + 18, 67, 39, 71, 2, 145, 132, 19, 4, 85, 76, 65, 82, 2, 245, 10, 4, 82, + 65, 80, 72, 6, 62, 69, 144, 131, 19, 5, 65, 82, 82, 79, 87, 155, 206, 10, + 74, 2, 165, 245, 12, 5, 85, 84, 82, 65, 76, 11, 52, 4, 77, 69, 71, 65, + 166, 3, 32, 251, 227, 29, 84, 5, 173, 190, 29, 8, 32, 87, 73, 84, 72, 32, + 84, 73, 10, 138, 6, 69, 138, 236, 12, 65, 255, 224, 16, 83, 14, 50, 69, + 100, 4, 79, 85, 78, 68, 235, 226, 29, 72, 8, 37, 7, 86, 69, 82, 83, 69, + 68, 32, 8, 166, 197, 19, 68, 170, 151, 9, 84, 166, 100, 89, 151, 14, 90, + 4, 242, 159, 10, 32, 137, 244, 1, 2, 69, 68, 34, 108, 4, 67, 72, 87, 65, + 34, 72, 132, 1, 4, 79, 70, 84, 32, 214, 238, 12, 84, 213, 1, 5, 69, 77, + 73, 83, 79, 5, 11, 32, 2, 187, 238, 5, 87, 16, 92, 4, 79, 82, 84, 32, + 180, 237, 12, 2, 72, 65, 146, 251, 14, 67, 238, 227, 1, 87, 215, 22, 65, + 6, 226, 174, 27, 73, 163, 180, 2, 85, 8, 38, 69, 210, 219, 28, 83, 151, + 112, 68, 4, 182, 226, 29, 76, 3, 77, 28, 140, 1, 4, 65, 76, 76, 32, 50, + 69, 106, 83, 224, 135, 12, 11, 72, 82, 69, 69, 45, 76, 69, 71, 71, 69, + 68, 138, 171, 17, 67, 186, 22, 74, 3, 87, 6, 214, 231, 12, 72, 206, 229, + 15, 89, 151, 125, 84, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 26, 77, 131, + 235, 12, 68, 2, 189, 244, 21, 5, 73, 68, 68, 76, 69, 8, 158, 201, 29, 72, + 2, 83, 2, 87, 215, 22, 69, 15, 58, 32, 114, 75, 53, 8, 78, 66, 76, 69, + 78, 68, 69, 68, 6, 29, 5, 87, 73, 84, 72, 32, 6, 26, 68, 171, 245, 12, + 77, 4, 212, 167, 9, 5, 79, 85, 66, 76, 69, 187, 8, 73, 5, 11, 82, 2, 213, + 4, 6, 65, 73, 78, 73, 65, 78, 2, 207, 232, 24, 32, 4, 212, 201, 28, 4, + 73, 68, 69, 32, 251, 147, 1, 69, 18, 62, 65, 28, 3, 69, 82, 85, 194, 220, + 29, 73, 2, 78, 3, 85, 7, 218, 220, 29, 69, 3, 84, 7, 33, 6, 32, 87, 73, + 84, 72, 32, 4, 178, 163, 9, 68, 255, 197, 3, 66, 18, 18, 69, 71, 72, 9, + 156, 1, 7, 32, 87, 73, 84, 72, 32, 68, 233, 215, 29, 2, 77, 76, 10, 26, + 69, 179, 196, 29, 87, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 26, 68, 239, + 215, 12, 66, 4, 134, 172, 9, 73, 175, 185, 3, 69, 6, 37, 7, 71, 65, 84, + 85, 82, 69, 32, 6, 66, 65, 188, 198, 15, 2, 84, 69, 237, 229, 13, 4, 69, + 78, 32, 71, 2, 131, 190, 29, 32, 52, 198, 1, 66, 38, 68, 38, 69, 36, 3, + 71, 72, 69, 38, 72, 130, 137, 22, 89, 138, 211, 5, 83, 182, 113, 84, 238, + 8, 90, 130, 64, 73, 150, 19, 67, 186, 22, 80, 2, 86, 158, 20, 75, 186, 2, + 65, 2, 79, 3, 85, 4, 206, 224, 12, 89, 139, 247, 16, 69, 6, 146, 215, 28, + 90, 163, 128, 1, 69, 6, 142, 215, 29, 70, 2, 76, 3, 83, 5, 201, 5, 5, 32, + 87, 73, 84, 72, 4, 11, 65, 5, 227, 237, 27, 82, 2, 153, 248, 15, 4, 72, + 79, 85, 83, 212, 15, 178, 1, 65, 138, 4, 67, 54, 69, 146, 35, 73, 138, + 23, 79, 154, 45, 82, 174, 3, 85, 144, 232, 12, 13, 78, 65, 32, 68, 79, + 85, 66, 76, 69, 32, 72, 69, 76, 134, 183, 15, 86, 207, 47, 76, 30, 116, + 4, 71, 71, 69, 82, 146, 1, 78, 32, 4, 82, 75, 32, 83, 36, 2, 83, 72, 184, + 253, 21, 2, 76, 69, 139, 145, 1, 84, 9, 11, 32, 6, 44, 5, 87, 73, 84, 72, + 32, 219, 210, 6, 75, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 237, + 187, 28, 4, 84, 32, 71, 85, 4, 214, 148, 29, 67, 251, 30, 71, 4, 230, + 200, 21, 85, 211, 146, 5, 72, 10, 30, 32, 105, 3, 69, 68, 32, 4, 60, 9, + 87, 73, 84, 72, 32, 76, 69, 70, 84, 147, 190, 28, 83, 2, 17, 2, 32, 85, + 2, 143, 141, 23, 80, 6, 238, 233, 4, 84, 142, 189, 12, 76, 155, 194, 10, + 79, 10, 194, 208, 29, 49, 2, 50, 2, 51, 2, 52, 3, 83, 180, 4, 226, 1, 67, + 248, 1, 5, 71, 82, 69, 69, 32, 98, 76, 82, 78, 228, 3, 8, 80, 65, 82, 84, + 77, 69, 78, 84, 34, 83, 174, 6, 86, 192, 140, 15, 11, 82, 69, 76, 73, 67, + 84, 32, 72, 79, 85, 83, 168, 180, 10, 2, 65, 70, 143, 199, 3, 69, 8, 50, + 73, 177, 250, 5, 6, 82, 69, 65, 83, 69, 32, 6, 56, 4, 77, 65, 76, 32, + 205, 168, 9, 4, 68, 85, 79, 85, 4, 88, 9, 83, 69, 80, 65, 82, 65, 84, 79, + 82, 149, 248, 21, 7, 69, 88, 80, 79, 78, 69, 78, 2, 21, 3, 32, 75, 69, 2, + 155, 185, 28, 89, 6, 140, 194, 22, 4, 67, 69, 76, 83, 186, 131, 6, 83, + 177, 106, 8, 70, 65, 72, 82, 69, 78, 72, 69, 9, 248, 250, 16, 5, 73, 86, + 69, 82, 89, 248, 252, 5, 2, 84, 65, 231, 140, 5, 69, 34, 104, 20, 84, 73, 83, 84, 82, 89, 32, 83, 89, 77, 66, 79, 76, 32, 76, 73, 71, 72, 84, 32, - 160, 3, 8, 80, 65, 82, 84, 77, 69, 78, 84, 34, 83, 174, 6, 86, 188, 214, - 14, 11, 82, 69, 76, 73, 67, 84, 32, 72, 79, 85, 83, 132, 146, 7, 2, 65, - 70, 235, 154, 6, 69, 8, 50, 73, 217, 229, 5, 6, 82, 69, 65, 83, 69, 32, - 6, 48, 4, 68, 85, 79, 85, 21, 4, 77, 65, 76, 32, 2, 219, 178, 23, 83, 4, - 88, 9, 83, 69, 80, 65, 82, 65, 84, 79, 82, 197, 171, 21, 7, 69, 88, 80, - 79, 78, 69, 78, 2, 21, 3, 32, 75, 69, 2, 151, 180, 27, 89, 6, 200, 242, - 21, 4, 67, 69, 76, 83, 250, 205, 5, 83, 153, 106, 8, 70, 65, 72, 82, 69, - 78, 72, 69, 9, 176, 190, 16, 5, 73, 86, 69, 82, 89, 128, 225, 5, 2, 84, - 65, 215, 226, 4, 69, 30, 88, 4, 68, 79, 87, 78, 0, 2, 85, 80, 153, 1, 9, - 86, 69, 82, 84, 73, 67, 65, 76, 32, 8, 69, 15, 32, 65, 78, 68, 32, 72, - 79, 82, 73, 90, 79, 78, 84, 65, 76, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, - 234, 219, 26, 87, 170, 25, 67, 191, 43, 84, 14, 52, 4, 65, 78, 68, 32, - 85, 5, 87, 73, 84, 72, 32, 10, 200, 158, 17, 2, 84, 79, 190, 188, 9, 87, - 165, 189, 1, 6, 66, 79, 84, 84, 79, 77, 4, 218, 243, 26, 67, 191, 43, 84, - 2, 245, 137, 24, 3, 32, 83, 84, 170, 1, 64, 2, 67, 69, 48, 2, 69, 82, - 129, 5, 5, 75, 84, 79, 80, 32, 2, 217, 180, 26, 7, 78, 68, 73, 78, 71, - 32, 78, 164, 1, 32, 3, 69, 84, 32, 183, 4, 84, 160, 1, 56, 6, 67, 65, 80, - 73, 84, 65, 1, 4, 83, 77, 65, 76, 80, 45, 9, 76, 32, 76, 69, 84, 84, 69, - 82, 32, 80, 218, 1, 69, 96, 4, 76, 79, 78, 71, 0, 5, 83, 72, 79, 82, 84, - 74, 79, 30, 84, 2, 90, 246, 178, 11, 67, 206, 157, 15, 66, 2, 68, 2, 74, - 2, 80, 2, 86, 2, 89, 202, 96, 71, 2, 75, 134, 103, 87, 162, 19, 65, 203, - 17, 72, 20, 214, 202, 24, 83, 206, 251, 1, 78, 134, 247, 1, 84, 146, 1, - 70, 2, 76, 2, 77, 2, 82, 3, 87, 12, 11, 32, 12, 234, 201, 24, 65, 226, - 180, 1, 79, 130, 191, 2, 69, 3, 73, 4, 142, 189, 28, 73, 3, 87, 4, 190, - 208, 26, 72, 227, 213, 1, 69, 5, 225, 174, 27, 4, 32, 73, 83, 76, 4, 186, - 153, 16, 67, 229, 136, 11, 4, 87, 73, 78, 68, 202, 2, 100, 8, 65, 78, 65, - 71, 65, 82, 73, 32, 185, 18, 12, 73, 67, 69, 32, 67, 79, 78, 84, 82, 79, - 76, 32, 192, 2, 222, 1, 65, 38, 67, 22, 71, 36, 4, 72, 69, 65, 68, 96, 7, - 76, 69, 84, 84, 69, 82, 32, 198, 5, 83, 176, 7, 11, 86, 79, 87, 69, 76, - 32, 83, 73, 71, 78, 32, 158, 181, 18, 68, 148, 245, 5, 4, 74, 65, 73, 78, - 243, 239, 3, 79, 4, 138, 164, 12, 67, 191, 170, 12, 66, 2, 187, 253, 5, - 65, 4, 250, 163, 12, 82, 215, 236, 1, 65, 6, 44, 5, 32, 77, 65, 82, 75, - 163, 136, 28, 83, 5, 209, 195, 18, 7, 32, 87, 73, 84, 72, 32, 72, 158, 1, - 158, 2, 65, 54, 67, 62, 68, 50, 71, 62, 72, 52, 2, 77, 65, 46, 83, 154, - 10, 79, 218, 254, 6, 66, 82, 74, 218, 220, 4, 75, 238, 194, 3, 82, 166, - 212, 6, 89, 170, 109, 85, 238, 30, 78, 230, 46, 76, 238, 146, 1, 84, 46, - 86, 138, 225, 1, 73, 154, 191, 1, 80, 2, 90, 254, 68, 70, 2, 81, 187, 2, - 69, 13, 198, 181, 28, 65, 2, 73, 2, 85, 2, 87, 3, 89, 10, 26, 65, 195, - 178, 28, 72, 9, 185, 2, 4, 78, 68, 82, 65, 12, 170, 204, 24, 68, 246, - 229, 3, 72, 187, 2, 65, 10, 182, 185, 12, 76, 190, 179, 15, 72, 254, 68, - 71, 187, 2, 65, 4, 232, 150, 16, 4, 69, 65, 86, 89, 131, 157, 12, 65, 5, - 229, 133, 7, 6, 82, 87, 65, 82, 73, 32, 12, 38, 72, 174, 176, 28, 83, - 187, 2, 65, 8, 36, 3, 79, 82, 84, 191, 178, 28, 65, 6, 163, 212, 11, 32, - 68, 168, 1, 19, 69, 81, 85, 69, 78, 67, 69, 32, 70, 79, 82, 32, 76, 69, - 84, 84, 69, 82, 32, 104, 4, 73, 71, 78, 32, 141, 128, 26, 10, 84, 82, 69, - 83, 83, 32, 83, 73, 71, 78, 16, 226, 202, 3, 71, 2, 75, 220, 238, 22, 3, - 68, 68, 68, 2, 82, 226, 244, 1, 89, 38, 70, 2, 81, 3, 90, 48, 210, 3, 66, - 0, 10, 69, 88, 84, 69, 78, 68, 69, 68, 32, 66, 36, 11, 67, 65, 78, 68, - 82, 65, 66, 73, 78, 68, 85, 60, 8, 87, 69, 83, 84, 69, 82, 78, 32, 32, - 10, 82, 69, 86, 69, 82, 83, 69, 68, 32, 78, 148, 231, 4, 5, 80, 85, 83, - 72, 80, 182, 204, 1, 83, 156, 243, 3, 18, 68, 79, 85, 66, 76, 69, 32, 67, - 65, 78, 68, 82, 65, 66, 73, 78, 68, 85, 182, 148, 8, 73, 186, 137, 6, 65, - 126, 77, 46, 78, 204, 212, 2, 12, 72, 73, 71, 72, 32, 83, 80, 65, 67, 73, - 78, 71, 255, 77, 86, 4, 133, 170, 12, 4, 72, 65, 76, 69, 11, 11, 32, 8, - 206, 241, 21, 65, 210, 89, 86, 151, 244, 3, 84, 4, 30, 78, 21, 3, 70, 73, - 86, 2, 17, 2, 73, 78, 2, 217, 144, 21, 8, 69, 45, 76, 73, 75, 69, 32, 66, - 50, 146, 1, 65, 52, 7, 67, 65, 78, 68, 82, 65, 32, 62, 79, 190, 186, 18, - 80, 254, 165, 4, 85, 138, 228, 1, 83, 70, 86, 166, 221, 1, 73, 207, 134, - 2, 69, 10, 226, 169, 28, 65, 2, 73, 2, 85, 2, 87, 3, 89, 6, 176, 187, 18, - 4, 76, 79, 78, 71, 254, 237, 9, 69, 3, 79, 7, 158, 146, 28, 79, 215, 22, - 69, 10, 222, 193, 23, 70, 136, 6, 2, 83, 84, 146, 244, 2, 84, 203, 83, - 79, 228, 2, 182, 2, 65, 134, 2, 69, 74, 71, 224, 4, 6, 78, 71, 66, 65, - 84, 32, 248, 1, 5, 82, 69, 67, 84, 32, 98, 83, 178, 2, 86, 200, 7, 2, 89, - 65, 32, 4, 90, 90, 89, 32, 192, 152, 5, 2, 84, 84, 216, 229, 14, 10, 70, - 70, 69, 82, 69, 78, 67, 69, 32, 66, 229, 147, 7, 12, 77, 69, 78, 83, 73, - 79, 78, 32, 79, 82, 73, 71, 18, 26, 77, 167, 211, 8, 69, 16, 40, 4, 79, - 78, 68, 32, 195, 138, 3, 69, 14, 128, 1, 5, 87, 73, 84, 72, 32, 218, 220, - 17, 84, 128, 221, 2, 12, 83, 72, 65, 80, 69, 32, 87, 73, 84, 72, 32, 65, - 159, 192, 4, 79, 8, 202, 177, 21, 66, 162, 2, 84, 250, 228, 3, 76, 27, - 82, 14, 164, 240, 4, 5, 32, 70, 65, 67, 69, 225, 229, 16, 4, 83, 69, 76, - 32, 78, 64, 3, 73, 84, 32, 149, 2, 8, 82, 65, 77, 32, 70, 79, 82, 32, 60, - 98, 70, 44, 2, 78, 73, 2, 79, 14, 83, 46, 84, 44, 3, 90, 69, 82, 13, 5, - 69, 73, 71, 72, 84, 12, 128, 1, 2, 73, 86, 25, 3, 79, 85, 82, 6, 87, 78, - 12, 96, 4, 69, 86, 69, 78, 1, 2, 73, 88, 12, 28, 3, 72, 82, 69, 15, 87, - 6, 23, 69, 6, 11, 79, 7, 207, 186, 16, 32, 18, 88, 5, 69, 65, 82, 84, 72, - 64, 5, 71, 82, 69, 65, 84, 0, 4, 76, 69, 83, 83, 39, 72, 7, 25, 4, 76, - 89, 32, 72, 4, 238, 213, 23, 85, 187, 131, 1, 69, 4, 229, 207, 12, 4, 69, - 82, 32, 89, 4, 172, 129, 5, 7, 69, 65, 86, 69, 78, 76, 89, 1, 4, 85, 77, - 65, 78, 64, 120, 17, 78, 69, 71, 65, 84, 73, 86, 69, 32, 67, 73, 82, 67, - 76, 69, 68, 32, 41, 9, 67, 73, 82, 67, 76, 69, 68, 32, 83, 42, 38, 83, - 162, 32, 78, 139, 249, 19, 68, 22, 49, 10, 65, 78, 83, 45, 83, 69, 82, - 73, 70, 32, 22, 234, 31, 78, 203, 142, 26, 68, 4, 140, 143, 22, 15, 67, - 85, 82, 82, 69, 78, 84, 32, 83, 89, 77, 66, 79, 76, 32, 235, 240, 5, 72, - 10, 70, 65, 144, 1, 5, 67, 79, 78, 84, 73, 213, 251, 24, 3, 71, 85, 73, - 6, 76, 9, 80, 80, 79, 73, 78, 84, 69, 68, 32, 217, 255, 14, 4, 66, 76, - 69, 68, 4, 232, 226, 16, 7, 66, 85, 84, 32, 82, 69, 76, 211, 250, 10, 70, - 2, 53, 11, 78, 85, 79, 85, 83, 32, 85, 78, 68, 69, 82, 2, 213, 197, 16, - 3, 76, 73, 78, 156, 1, 80, 9, 69, 83, 32, 65, 75, 85, 82, 85, 32, 238, 5, - 73, 233, 190, 16, 2, 79, 82, 144, 1, 142, 2, 68, 36, 7, 76, 69, 84, 84, - 69, 82, 32, 236, 1, 5, 83, 73, 71, 78, 32, 58, 86, 252, 243, 8, 6, 77, - 69, 68, 73, 65, 76, 198, 247, 4, 71, 198, 211, 4, 69, 172, 189, 4, 12, - 80, 82, 69, 70, 73, 88, 69, 68, 32, 78, 65, 83, 241, 204, 4, 7, 73, 78, - 73, 84, 73, 65, 76, 22, 246, 192, 18, 79, 175, 231, 7, 73, 84, 186, 142, - 10, 84, 242, 209, 11, 89, 186, 116, 65, 130, 217, 1, 68, 146, 25, 85, - 210, 200, 1, 73, 42, 76, 246, 189, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, - 2, 75, 2, 80, 254, 68, 72, 2, 74, 2, 77, 2, 82, 2, 86, 2, 90, 186, 2, 69, - 3, 79, 8, 234, 128, 24, 72, 146, 45, 67, 98, 78, 223, 160, 3, 65, 18, 64, - 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 179, 225, 25, 73, 16, 178, - 219, 14, 65, 218, 232, 9, 85, 210, 200, 1, 73, 206, 134, 2, 69, 3, 79, - 10, 68, 5, 83, 73, 79, 78, 32, 204, 234, 1, 2, 78, 71, 235, 147, 25, 68, - 6, 26, 83, 247, 206, 5, 84, 4, 206, 172, 26, 76, 231, 94, 73, 2, 173, - 184, 18, 3, 32, 76, 65, 4, 190, 254, 26, 83, 255, 85, 70, 190, 5, 188, 2, - 6, 67, 85, 77, 69, 78, 84, 124, 7, 69, 83, 32, 78, 79, 84, 32, 194, 3, - 71, 210, 3, 76, 36, 10, 77, 73, 78, 79, 32, 84, 73, 76, 69, 32, 226, 1, - 78, 38, 84, 246, 2, 85, 204, 17, 2, 87, 78, 220, 128, 16, 8, 32, 78, 79, - 84, 32, 76, 73, 84, 136, 178, 11, 8, 86, 69, 32, 79, 70, 32, 80, 69, 174, - 4, 79, 223, 25, 68, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 40, 4, 84, 69, - 88, 84, 191, 133, 2, 80, 5, 185, 133, 2, 6, 32, 65, 78, 68, 32, 80, 22, - 164, 1, 11, 67, 79, 78, 84, 65, 73, 78, 32, 65, 83, 32, 92, 6, 68, 73, - 86, 73, 68, 69, 112, 2, 80, 82, 48, 7, 83, 85, 67, 67, 69, 69, 68, 233, - 251, 21, 2, 70, 79, 6, 248, 1, 15, 78, 79, 82, 77, 65, 76, 32, 83, 85, - 66, 71, 82, 79, 85, 80, 195, 170, 20, 77, 5, 209, 170, 21, 23, 32, 87, - 73, 84, 72, 32, 82, 69, 86, 69, 82, 83, 69, 68, 32, 78, 69, 71, 65, 84, - 73, 79, 78, 6, 44, 5, 69, 67, 69, 68, 69, 243, 141, 27, 79, 5, 217, 240, - 8, 2, 32, 79, 125, 36, 3, 82, 65, 32, 235, 204, 27, 32, 120, 120, 7, 76, - 69, 84, 84, 69, 82, 32, 212, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, - 78, 32, 158, 199, 22, 65, 191, 2, 83, 88, 214, 232, 23, 82, 206, 55, 65, - 38, 68, 46, 84, 230, 24, 85, 210, 200, 1, 73, 158, 190, 1, 78, 46, 83, - 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, 76, 2, 77, 2, - 86, 2, 89, 186, 2, 69, 3, 79, 22, 162, 193, 18, 86, 166, 225, 5, 65, 190, - 21, 85, 210, 200, 1, 73, 206, 134, 2, 69, 3, 79, 4, 182, 194, 20, 80, - 243, 173, 2, 76, 200, 1, 72, 8, 72, 79, 82, 73, 90, 79, 78, 84, 1, 6, 86, - 69, 82, 84, 73, 67, 100, 17, 2, 65, 76, 100, 32, 2, 45, 48, 191, 162, 23, - 32, 98, 58, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 14, 197, 219, - 19, 2, 45, 48, 4, 254, 237, 25, 75, 223, 143, 1, 71, 26, 34, 32, 57, 4, - 84, 69, 68, 32, 8, 254, 166, 24, 80, 34, 77, 202, 50, 79, 187, 173, 2, - 65, 18, 214, 1, 67, 34, 83, 144, 213, 15, 3, 76, 73, 78, 192, 132, 1, 4, - 79, 66, 69, 76, 160, 9, 9, 84, 82, 65, 78, 83, 80, 79, 83, 73, 134, 194, - 6, 70, 141, 41, 14, 82, 73, 71, 72, 84, 45, 80, 79, 73, 78, 84, 73, 78, - 71, 4, 238, 178, 26, 73, 167, 17, 82, 4, 182, 241, 25, 79, 223, 111, 81, - 162, 1, 40, 3, 66, 76, 69, 137, 17, 2, 71, 72, 160, 1, 42, 32, 190, 10, - 45, 241, 5, 2, 68, 32, 106, 226, 2, 67, 174, 1, 68, 38, 72, 32, 4, 73, - 78, 84, 69, 38, 76, 144, 1, 7, 78, 69, 83, 84, 69, 68, 32, 74, 80, 66, - 83, 130, 2, 85, 36, 9, 86, 69, 82, 84, 73, 67, 65, 76, 32, 132, 140, 6, - 6, 79, 66, 76, 73, 81, 85, 172, 167, 2, 9, 82, 73, 71, 72, 84, 32, 65, - 82, 67, 194, 174, 3, 65, 186, 189, 9, 69, 190, 141, 4, 81, 213, 101, 6, - 87, 65, 86, 89, 32, 79, 24, 100, 7, 73, 82, 67, 76, 69, 68, 32, 252, 166, - 7, 4, 85, 82, 76, 89, 145, 191, 1, 4, 79, 76, 79, 78, 20, 26, 78, 139, - 249, 19, 68, 2, 137, 199, 2, 5, 85, 77, 66, 69, 82, 4, 186, 160, 18, 79, - 131, 164, 6, 65, 4, 186, 243, 19, 73, 215, 63, 89, 4, 230, 136, 25, 82, - 223, 233, 1, 71, 12, 54, 79, 141, 222, 16, 7, 69, 70, 84, 32, 65, 82, 67, - 10, 26, 87, 163, 207, 24, 71, 6, 26, 45, 187, 225, 26, 32, 4, 162, 242, - 19, 82, 211, 1, 57, 6, 196, 204, 16, 9, 76, 69, 83, 83, 45, 84, 72, 65, - 78, 255, 158, 9, 71, 8, 26, 82, 247, 246, 25, 76, 6, 154, 152, 1, 69, - 139, 180, 15, 73, 18, 80, 6, 81, 85, 65, 82, 69, 32, 30, 84, 50, 85, 185, - 161, 13, 4, 79, 76, 73, 68, 4, 238, 221, 24, 73, 55, 85, 4, 252, 138, 12, - 3, 65, 67, 75, 187, 192, 4, 82, 8, 58, 83, 186, 150, 1, 67, 170, 182, 20, - 80, 191, 194, 4, 66, 2, 209, 142, 27, 4, 80, 69, 78, 83, 4, 210, 155, 18, - 80, 179, 139, 9, 78, 10, 36, 3, 66, 65, 82, 211, 221, 26, 76, 9, 11, 32, - 6, 52, 7, 68, 79, 85, 66, 76, 69, 32, 163, 215, 25, 76, 4, 158, 215, 25, - 76, 51, 82, 48, 112, 5, 76, 73, 78, 69, 32, 220, 1, 7, 83, 84, 82, 85, - 67, 75, 32, 189, 182, 8, 7, 69, 78, 68, 69, 68, 32, 77, 12, 48, 8, 83, - 76, 65, 78, 84, 69, 68, 32, 75, 69, 8, 70, 69, 56, 7, 71, 82, 69, 65, 84, - 69, 82, 1, 4, 76, 69, 83, 83, 4, 177, 232, 19, 9, 81, 85, 65, 76, 32, 84, - 79, 32, 79, 2, 193, 239, 13, 5, 45, 84, 72, 65, 78, 34, 160, 1, 8, 67, - 65, 80, 73, 84, 65, 76, 32, 92, 7, 73, 84, 65, 76, 73, 67, 32, 124, 6, - 83, 77, 65, 76, 76, 32, 153, 155, 13, 8, 78, 45, 65, 82, 89, 32, 83, 85, - 18, 234, 238, 11, 71, 230, 252, 13, 80, 206, 134, 2, 67, 2, 72, 2, 78, 2, - 81, 2, 82, 3, 90, 10, 76, 6, 83, 77, 65, 76, 76, 32, 185, 151, 21, 7, 67, - 65, 80, 73, 84, 65, 76, 8, 238, 240, 27, 68, 2, 69, 2, 73, 3, 74, 4, 146, - 237, 11, 71, 219, 239, 15, 80, 6, 146, 237, 9, 70, 26, 77, 255, 239, 16, - 83, 2, 179, 246, 26, 78, 144, 1, 86, 32, 156, 1, 10, 45, 80, 79, 73, 78, - 84, 73, 78, 71, 32, 65, 4, 87, 65, 82, 68, 10, 60, 4, 84, 65, 67, 75, - 250, 222, 24, 70, 30, 82, 251, 64, 65, 5, 29, 5, 32, 87, 73, 84, 72, 2, - 11, 32, 2, 11, 67, 2, 201, 169, 20, 4, 73, 82, 67, 76, 8, 178, 225, 24, - 82, 0, 7, 83, 77, 65, 76, 76, 32, 82, 35, 84, 126, 56, 8, 32, 70, 65, 67, - 73, 78, 71, 32, 89, 2, 83, 32, 8, 54, 72, 1, 9, 78, 79, 84, 67, 72, 69, - 68, 32, 72, 4, 137, 201, 26, 3, 79, 79, 75, 118, 158, 1, 65, 202, 2, 84, - 168, 192, 8, 2, 87, 72, 186, 138, 8, 90, 198, 184, 8, 66, 154, 1, 68, 50, - 70, 82, 72, 146, 4, 67, 46, 81, 42, 82, 22, 83, 243, 7, 80, 34, 40, 4, - 82, 82, 79, 87, 187, 255, 24, 78, 33, 11, 32, 30, 144, 1, 5, 87, 73, 84, - 72, 32, 208, 147, 3, 14, 76, 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 70, - 32, 85, 254, 235, 21, 65, 170, 10, 70, 175, 5, 84, 22, 180, 163, 5, 6, - 67, 79, 82, 78, 69, 82, 182, 222, 19, 68, 58, 76, 42, 77, 38, 78, 58, 83, - 66, 69, 246, 12, 84, 139, 59, 72, 36, 36, 2, 82, 73, 161, 2, 2, 87, 79, - 32, 44, 5, 65, 78, 71, 76, 69, 171, 146, 25, 80, 30, 56, 8, 45, 72, 69, - 65, 68, 69, 68, 32, 215, 152, 25, 32, 28, 68, 5, 65, 82, 82, 79, 87, 134, - 201, 16, 90, 162, 199, 8, 68, 39, 80, 23, 11, 32, 20, 216, 139, 25, 15, - 76, 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 70, 32, 85, 80, 98, 84, 19, - 87, 4, 162, 145, 25, 32, 105, 15, 45, 72, 69, 65, 68, 69, 68, 32, 65, 82, - 82, 79, 87, 32, 87, 22, 138, 1, 65, 106, 79, 152, 1, 12, 85, 77, 32, 87, - 73, 84, 72, 32, 68, 82, 85, 77, 152, 182, 16, 6, 73, 86, 69, 32, 83, 76, - 179, 149, 10, 69, 8, 238, 189, 2, 67, 236, 243, 5, 10, 70, 84, 73, 78, - 71, 32, 80, 79, 73, 78, 201, 192, 15, 3, 71, 79, 78, 8, 56, 6, 77, 69, - 68, 65, 82, 89, 34, 80, 251, 174, 26, 79, 2, 217, 160, 20, 3, 32, 67, 65, - 4, 240, 234, 17, 6, 32, 79, 70, 32, 66, 76, 191, 194, 9, 76, 2, 217, 165, - 12, 3, 83, 84, 73, 162, 2, 76, 7, 80, 76, 79, 89, 65, 78, 32, 128, 187, - 19, 2, 77, 80, 247, 162, 8, 67, 158, 2, 212, 2, 6, 65, 70, 70, 73, 88, - 32, 152, 7, 7, 76, 69, 84, 84, 69, 82, 32, 208, 188, 1, 16, 84, 72, 73, - 67, 75, 32, 76, 69, 84, 84, 69, 82, 32, 83, 69, 76, 236, 160, 3, 4, 68, - 79, 85, 66, 204, 172, 16, 19, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, - 32, 67, 72, 73, 78, 79, 79, 75, 153, 143, 5, 11, 83, 73, 71, 78, 32, 79, - 32, 87, 73, 84, 72, 64, 140, 1, 9, 65, 84, 84, 65, 67, 72, 69, 68, 32, - 184, 1, 5, 72, 73, 71, 72, 32, 102, 76, 40, 4, 82, 73, 71, 72, 169, 2, 4, - 77, 73, 68, 32, 14, 112, 2, 84, 65, 224, 4, 13, 76, 69, 70, 84, 45, 84, - 79, 45, 82, 73, 71, 72, 84, 26, 83, 146, 165, 20, 69, 3, 73, 6, 44, 5, - 78, 71, 69, 78, 84, 227, 210, 26, 73, 5, 227, 169, 20, 32, 20, 166, 2, - 76, 50, 84, 38, 86, 238, 134, 8, 71, 186, 2, 65, 154, 231, 17, 87, 170, - 25, 67, 223, 126, 68, 24, 36, 2, 69, 70, 29, 3, 79, 87, 32, 2, 209, 2, 3, - 84, 32, 72, 22, 90, 65, 38, 76, 50, 84, 38, 86, 238, 134, 8, 71, 210, - 233, 17, 87, 170, 25, 67, 223, 126, 68, 4, 202, 150, 21, 67, 131, 169, 5, - 82, 4, 220, 216, 7, 3, 79, 78, 71, 207, 231, 18, 73, 2, 173, 254, 7, 4, - 73, 71, 72, 84, 4, 37, 7, 69, 82, 84, 73, 67, 65, 76, 5, 131, 1, 32, 4, - 42, 72, 41, 6, 86, 69, 82, 84, 73, 67, 2, 37, 7, 79, 82, 73, 90, 79, 78, - 84, 2, 17, 2, 65, 76, 2, 11, 32, 2, 11, 83, 2, 237, 205, 26, 2, 69, 67, - 214, 1, 222, 1, 65, 22, 68, 34, 69, 30, 70, 22, 71, 22, 74, 162, 1, 75, - 66, 76, 50, 77, 62, 78, 134, 1, 79, 50, 80, 86, 82, 74, 83, 210, 2, 84, - 66, 85, 42, 86, 38, 87, 70, 88, 230, 206, 26, 72, 138, 60, 73, 194, 41, - 89, 215, 22, 66, 5, 203, 176, 27, 79, 7, 206, 151, 27, 32, 211, 61, 72, - 7, 254, 212, 27, 69, 3, 85, 5, 143, 133, 27, 32, 5, 207, 129, 24, 32, 19, - 11, 32, 16, 76, 8, 87, 73, 84, 72, 32, 68, 79, 84, 230, 5, 77, 2, 78, - 215, 170, 26, 83, 5, 233, 214, 26, 12, 83, 32, 73, 78, 83, 73, 68, 69, - 32, 65, 78, 68, 9, 26, 32, 131, 211, 27, 75, 4, 138, 128, 24, 82, 247, - 210, 3, 77, 9, 248, 243, 21, 3, 79, 78, 71, 227, 222, 5, 72, 11, 11, 32, - 8, 162, 4, 78, 238, 170, 26, 87, 147, 163, 1, 83, 19, 38, 32, 49, 5, 65, - 83, 65, 76, 32, 8, 202, 3, 77, 238, 170, 26, 87, 147, 163, 1, 83, 8, 150, - 209, 27, 65, 2, 73, 2, 79, 3, 85, 11, 218, 207, 27, 79, 146, 1, 65, 2, - 85, 3, 87, 9, 52, 7, 69, 82, 78, 73, 78, 32, 65, 179, 128, 27, 32, 4, - 130, 208, 27, 77, 3, 78, 11, 248, 240, 21, 6, 79, 77, 65, 78, 73, 65, - 158, 161, 5, 32, 211, 61, 72, 49, 58, 32, 164, 1, 5, 76, 79, 65, 78, 32, - 195, 158, 6, 72, 26, 102, 74, 22, 75, 2, 80, 2, 84, 20, 7, 87, 73, 84, - 72, 32, 68, 79, 214, 205, 27, 77, 2, 78, 3, 83, 5, 171, 144, 27, 32, 5, - 151, 149, 27, 32, 4, 183, 165, 12, 84, 18, 74, 69, 138, 176, 5, 79, 154, - 205, 21, 65, 198, 78, 68, 146, 1, 74, 3, 85, 6, 242, 204, 27, 69, 2, 72, - 3, 78, 9, 26, 32, 183, 204, 27, 72, 4, 190, 249, 23, 82, 247, 210, 3, 83, - 9, 186, 252, 26, 32, 214, 79, 72, 3, 73, 5, 217, 237, 6, 4, 79, 67, 65, - 76, 17, 66, 79, 178, 146, 27, 32, 250, 36, 69, 218, 19, 65, 2, 72, 3, 73, - 5, 255, 202, 27, 87, 250, 28, 234, 2, 65, 188, 3, 10, 68, 73, 84, 79, 82, - 73, 65, 76, 32, 67, 22, 71, 240, 75, 4, 73, 71, 72, 84, 198, 1, 76, 194, - 9, 77, 214, 5, 78, 246, 3, 79, 36, 2, 81, 85, 182, 8, 82, 222, 1, 83, - 118, 84, 242, 31, 85, 198, 1, 88, 252, 3, 2, 89, 69, 152, 72, 4, 45, 77, - 65, 73, 132, 207, 18, 3, 74, 69, 67, 168, 134, 2, 8, 86, 69, 82, 71, 82, - 69, 69, 78, 159, 147, 5, 80, 22, 38, 82, 130, 230, 25, 83, 179, 64, 71, - 19, 30, 32, 137, 1, 2, 84, 72, 6, 88, 3, 79, 70, 32, 221, 210, 4, 13, 87, - 73, 84, 72, 32, 72, 69, 65, 82, 73, 78, 71, 32, 4, 216, 138, 11, 2, 77, - 65, 227, 206, 6, 82, 11, 17, 2, 32, 71, 8, 44, 5, 76, 79, 66, 69, 32, - 239, 165, 24, 82, 6, 70, 65, 149, 207, 13, 11, 69, 85, 82, 79, 80, 69, - 45, 65, 70, 82, 73, 4, 128, 214, 9, 4, 77, 69, 82, 73, 133, 154, 2, 11, - 83, 73, 65, 45, 65, 85, 83, 84, 82, 65, 76, 2, 191, 163, 2, 79, 174, 17, - 96, 18, 89, 80, 84, 73, 65, 78, 32, 72, 73, 69, 82, 79, 71, 76, 89, 80, - 72, 32, 199, 195, 27, 71, 172, 17, 146, 3, 65, 178, 4, 66, 52, 2, 67, 48, - 224, 1, 2, 68, 48, 170, 4, 69, 198, 3, 70, 164, 4, 2, 71, 48, 210, 3, 72, - 222, 1, 73, 238, 2, 76, 122, 77, 134, 9, 78, 210, 5, 79, 140, 5, 2, 80, - 48, 120, 2, 82, 48, 232, 1, 2, 83, 48, 190, 3, 84, 220, 2, 2, 85, 48, - 250, 2, 86, 134, 5, 87, 236, 2, 3, 88, 48, 48, 88, 3, 89, 48, 48, 84, 2, - 90, 48, 224, 196, 3, 3, 75, 48, 48, 177, 218, 14, 3, 81, 48, 48, 228, 1, - 30, 48, 133, 3, 2, 65, 48, 160, 1, 86, 48, 98, 49, 102, 52, 234, 55, 51, - 138, 163, 20, 55, 230, 216, 1, 50, 2, 53, 3, 54, 24, 238, 68, 54, 146, - 185, 21, 53, 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, 55, 2, 56, 3, 57, - 24, 158, 253, 21, 52, 2, 55, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 53, - 2, 54, 2, 56, 3, 57, 28, 186, 252, 21, 48, 2, 50, 2, 51, 2, 53, 174, 193, - 5, 49, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 68, 46, 48, 186, 55, 51, 138, - 251, 21, 49, 3, 50, 22, 150, 66, 55, 190, 250, 26, 49, 2, 50, 2, 51, 2, - 52, 2, 53, 2, 54, 2, 56, 3, 57, 26, 148, 9, 4, 69, 71, 73, 78, 253, 25, - 2, 48, 48, 56, 34, 48, 90, 49, 207, 199, 8, 50, 24, 170, 40, 50, 250, - 146, 27, 49, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 22, 158, - 249, 21, 48, 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, - 2, 56, 3, 57, 184, 1, 70, 48, 94, 51, 102, 52, 102, 53, 106, 54, 134, 30, - 50, 251, 141, 22, 49, 20, 238, 247, 21, 56, 174, 193, 5, 49, 2, 50, 2, - 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 24, 146, 247, 21, 49, 2, 52, 174, - 193, 5, 48, 2, 50, 2, 51, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 24, 174, - 246, 21, 54, 2, 56, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, - 2, 55, 3, 57, 42, 154, 61, 48, 178, 184, 21, 50, 2, 52, 174, 193, 5, 49, - 2, 51, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 32, 134, 61, 55, 138, 249, 26, - 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 94, 30, 48, 189, 2, 2, 78, - 68, 88, 38, 48, 94, 50, 102, 51, 235, 7, 49, 22, 202, 243, 21, 56, 2, 57, - 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 24, 238, 242, - 21, 48, 2, 56, 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, - 3, 57, 18, 138, 242, 21, 52, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 54, - 2, 55, 3, 56, 6, 11, 32, 6, 68, 2, 83, 69, 136, 160, 5, 6, 87, 65, 76, - 76, 69, 68, 191, 19, 69, 2, 231, 238, 26, 71, 132, 1, 42, 48, 133, 9, 5, - 85, 76, 76, 32, 66, 130, 1, 54, 48, 94, 49, 102, 51, 102, 52, 102, 53, - 215, 1, 50, 20, 226, 239, 21, 49, 174, 193, 5, 50, 2, 51, 2, 52, 2, 53, - 2, 54, 2, 55, 2, 56, 3, 57, 22, 134, 239, 21, 51, 174, 193, 5, 48, 2, 49, - 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 26, 162, 238, 21, 49, 2, - 55, 2, 56, 174, 193, 5, 48, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 57, 26, - 190, 237, 21, 53, 2, 54, 2, 55, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, - 52, 2, 56, 3, 57, 14, 142, 27, 49, 250, 146, 27, 48, 2, 50, 3, 51, 128, - 1, 62, 48, 98, 49, 102, 51, 102, 52, 130, 28, 50, 239, 155, 8, 53, 24, - 214, 50, 55, 146, 185, 21, 54, 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, - 53, 2, 56, 3, 57, 22, 134, 235, 21, 49, 174, 193, 5, 48, 2, 50, 2, 51, 2, - 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 24, 162, 234, 21, 54, 2, 55, 174, - 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 56, 3, 57, 24, 190, - 233, 21, 51, 2, 53, 174, 193, 5, 48, 2, 49, 2, 50, 2, 52, 2, 54, 2, 55, - 2, 56, 3, 57, 24, 80, 2, 48, 48, 84, 4, 65, 76, 70, 32, 209, 40, 7, 79, - 82, 73, 90, 79, 78, 84, 18, 134, 232, 21, 54, 174, 193, 5, 49, 2, 50, 2, - 51, 2, 52, 2, 53, 2, 55, 3, 56, 4, 22, 66, 175, 43, 76, 2, 237, 233, 23, - 2, 76, 65, 52, 58, 48, 181, 1, 9, 78, 83, 69, 82, 84, 32, 65, 84, 32, 38, - 18, 48, 95, 49, 22, 174, 230, 21, 53, 2, 57, 174, 193, 5, 49, 2, 50, 2, - 51, 2, 52, 2, 54, 2, 55, 3, 56, 16, 210, 229, 21, 48, 2, 49, 174, 193, 5, + 135, 237, 21, 83, 30, 88, 4, 68, 79, 87, 78, 0, 2, 85, 80, 153, 1, 9, 86, + 69, 82, 84, 73, 67, 65, 76, 32, 8, 69, 15, 32, 65, 78, 68, 32, 72, 79, + 82, 73, 90, 79, 78, 84, 65, 76, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 150, + 221, 27, 87, 186, 25, 67, 159, 45, 84, 14, 52, 4, 65, 78, 68, 32, 45, 5, + 87, 73, 84, 72, 32, 10, 142, 156, 22, 66, 42, 84, 255, 191, 5, 87, 4, + 190, 245, 27, 67, 159, 45, 84, 2, 193, 226, 24, 3, 32, 83, 84, 170, 1, + 64, 2, 67, 69, 48, 2, 69, 82, 129, 5, 5, 75, 84, 79, 80, 32, 2, 153, 182, + 27, 7, 78, 68, 73, 78, 71, 32, 78, 164, 1, 32, 3, 69, 84, 32, 183, 4, 84, + 160, 1, 56, 6, 67, 65, 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 80, 45, 9, + 76, 32, 76, 69, 84, 84, 69, 82, 32, 80, 218, 1, 69, 96, 4, 76, 79, 78, + 71, 0, 5, 83, 72, 79, 82, 84, 74, 79, 30, 84, 2, 90, 158, 231, 11, 67, + 250, 234, 15, 66, 2, 68, 2, 74, 2, 80, 2, 86, 2, 89, 182, 99, 71, 2, 75, + 158, 103, 87, 162, 19, 65, 203, 17, 72, 20, 242, 162, 25, 83, 134, 165, + 2, 78, 138, 250, 1, 84, 146, 1, 70, 2, 76, 2, 77, 2, 82, 3, 87, 12, 11, + 32, 12, 134, 162, 25, 65, 190, 218, 1, 79, 226, 197, 2, 69, 3, 73, 4, + 230, 193, 29, 73, 3, 87, 4, 146, 210, 27, 72, 231, 216, 1, 69, 5, 161, + 179, 28, 4, 32, 73, 83, 76, 4, 158, 198, 26, 67, 193, 224, 1, 4, 87, 73, + 78, 68, 202, 2, 100, 8, 65, 78, 65, 71, 65, 82, 73, 32, 161, 18, 12, 73, + 67, 69, 32, 67, 79, 78, 84, 82, 79, 76, 32, 192, 2, 222, 1, 65, 38, 67, + 22, 71, 36, 4, 72, 69, 65, 68, 96, 7, 76, 69, 84, 84, 69, 82, 32, 198, 5, + 83, 148, 7, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 154, 129, 19, + 68, 212, 129, 6, 4, 74, 65, 73, 78, 171, 156, 4, 79, 4, 210, 216, 12, 67, + 151, 206, 12, 66, 2, 159, 146, 6, 65, 4, 194, 216, 12, 82, 135, 238, 1, + 65, 6, 44, 5, 32, 77, 65, 82, 75, 251, 140, 29, 83, 5, 181, 143, 19, 7, + 32, 87, 73, 84, 72, 32, 72, 158, 1, 158, 2, 65, 54, 67, 62, 68, 50, 71, + 62, 72, 52, 2, 77, 65, 46, 83, 130, 10, 79, 246, 147, 7, 66, 206, 252, 4, + 75, 150, 202, 3, 82, 238, 232, 6, 89, 154, 119, 85, 174, 30, 78, 254, + 198, 1, 74, 150, 52, 76, 70, 84, 46, 86, 134, 207, 1, 73, 158, 194, 1, + 80, 2, 90, 138, 69, 70, 2, 81, 187, 2, 69, 13, 158, 186, 29, 65, 2, 73, + 2, 85, 2, 87, 3, 89, 10, 26, 65, 155, 183, 29, 72, 9, 185, 2, 4, 78, 68, + 82, 65, 12, 178, 223, 25, 68, 198, 215, 3, 72, 187, 2, 65, 10, 130, 238, + 12, 76, 190, 131, 16, 72, 138, 69, 71, 187, 2, 65, 4, 232, 210, 16, 4, + 69, 65, 86, 89, 219, 229, 12, 65, 5, 233, 154, 7, 6, 82, 87, 65, 82, 73, + 32, 12, 38, 72, 134, 181, 29, 83, 187, 2, 65, 8, 36, 3, 79, 82, 84, 151, + 183, 29, 65, 6, 199, 136, 12, 32, 68, 168, 1, 19, 69, 81, 85, 69, 78, 67, + 69, 32, 70, 79, 82, 32, 76, 69, 84, 84, 69, 82, 32, 104, 4, 73, 71, 78, + 32, 133, 254, 26, 10, 84, 82, 69, 83, 83, 32, 83, 73, 71, 78, 16, 174, + 217, 3, 71, 2, 75, 228, 225, 23, 3, 68, 68, 68, 2, 82, 230, 247, 1, 89, + 38, 70, 2, 81, 3, 90, 48, 178, 3, 66, 0, 10, 69, 88, 84, 69, 78, 68, 69, + 68, 32, 66, 36, 11, 67, 65, 78, 68, 82, 65, 66, 73, 78, 68, 85, 64, 8, + 87, 69, 83, 84, 69, 82, 78, 32, 32, 10, 82, 69, 86, 69, 82, 83, 69, 68, + 32, 78, 226, 200, 6, 83, 190, 190, 12, 73, 142, 149, 6, 77, 46, 78, 242, + 60, 65, 72, 18, 68, 79, 85, 66, 76, 69, 32, 67, 65, 78, 68, 82, 65, 66, + 73, 78, 68, 85, 62, 80, 216, 196, 2, 12, 72, 73, 71, 72, 32, 83, 80, 65, + 67, 73, 78, 71, 139, 78, 86, 4, 241, 222, 12, 4, 72, 65, 76, 69, 11, 11, + 32, 8, 134, 194, 22, 65, 238, 152, 3, 86, 183, 230, 1, 84, 4, 30, 78, 21, + 3, 70, 73, 86, 2, 17, 2, 73, 78, 2, 129, 221, 21, 8, 69, 45, 76, 73, 75, + 69, 32, 66, 50, 150, 1, 65, 52, 7, 67, 65, 78, 68, 82, 65, 32, 62, 79, + 186, 134, 19, 80, 162, 180, 4, 85, 158, 224, 1, 83, 222, 63, 86, 186, + 201, 1, 73, 223, 137, 2, 69, 10, 210, 174, 29, 65, 2, 73, 2, 85, 2, 87, + 3, 89, 6, 172, 135, 19, 4, 76, 79, 78, 71, 242, 166, 10, 69, 3, 79, 7, + 142, 151, 29, 79, 215, 22, 69, 10, 194, 154, 24, 70, 136, 6, 2, 83, 84, + 154, 157, 3, 84, 183, 86, 79, 230, 2, 182, 2, 65, 150, 2, 69, 74, 71, + 224, 4, 6, 78, 71, 66, 65, 84, 32, 248, 1, 5, 82, 69, 67, 84, 32, 98, 83, + 178, 2, 86, 200, 7, 2, 89, 65, 32, 4, 90, 90, 89, 32, 224, 172, 5, 2, 84, + 84, 184, 156, 15, 10, 70, 70, 69, 82, 69, 78, 67, 69, 32, 66, 173, 205, + 7, 12, 77, 69, 78, 83, 73, 79, 78, 32, 79, 82, 73, 71, 20, 42, 77, 242, + 251, 8, 69, 183, 134, 13, 71, 16, 40, 4, 79, 78, 68, 32, 207, 152, 3, 69, + 14, 128, 1, 5, 87, 73, 84, 72, 32, 186, 164, 18, 84, 164, 224, 2, 12, 83, + 72, 65, 80, 69, 32, 87, 73, 84, 72, 32, 65, 183, 228, 4, 79, 8, 190, 128, + 22, 66, 182, 2, 84, 238, 137, 4, 76, 27, 82, 14, 176, 131, 5, 5, 32, 70, + 65, 67, 69, 133, 161, 17, 4, 83, 69, 76, 32, 78, 64, 3, 73, 84, 32, 149, + 2, 8, 82, 65, 77, 32, 70, 79, 82, 32, 60, 98, 70, 44, 2, 78, 73, 2, 79, + 14, 83, 46, 84, 44, 3, 90, 69, 82, 13, 5, 69, 73, 71, 72, 84, 12, 128, 1, + 2, 73, 86, 25, 3, 79, 85, 82, 6, 87, 78, 12, 96, 4, 69, 86, 69, 78, 1, 2, + 73, 88, 12, 28, 3, 72, 82, 69, 15, 87, 6, 23, 69, 6, 11, 79, 7, 143, 247, + 16, 32, 18, 88, 5, 69, 65, 82, 84, 72, 64, 5, 71, 82, 69, 65, 84, 0, 4, + 76, 69, 83, 83, 39, 72, 7, 25, 4, 76, 89, 32, 72, 4, 194, 174, 24, 85, + 147, 141, 1, 69, 4, 169, 132, 13, 4, 69, 82, 32, 89, 4, 188, 148, 5, 7, + 69, 65, 86, 69, 78, 76, 89, 1, 4, 85, 77, 65, 78, 64, 120, 17, 78, 69, + 71, 65, 84, 73, 86, 69, 32, 67, 73, 82, 67, 76, 69, 68, 32, 41, 9, 67, + 73, 82, 67, 76, 69, 68, 32, 83, 42, 38, 83, 154, 32, 78, 151, 196, 20, + 68, 22, 49, 10, 65, 78, 83, 45, 83, 69, 82, 73, 70, 32, 22, 226, 31, 78, + 175, 144, 27, 68, 4, 148, 233, 22, 15, 67, 85, 82, 82, 69, 78, 84, 32, + 83, 89, 77, 66, 79, 76, 32, 195, 155, 6, 72, 10, 70, 65, 144, 1, 5, 67, + 79, 78, 84, 73, 197, 235, 25, 3, 71, 85, 73, 6, 76, 9, 80, 80, 79, 73, + 78, 84, 69, 68, 32, 133, 132, 27, 4, 66, 76, 69, 68, 4, 252, 159, 17, 7, + 66, 85, 84, 32, 82, 69, 76, 147, 194, 11, 70, 2, 53, 11, 78, 85, 79, 85, + 83, 32, 85, 78, 68, 69, 82, 2, 137, 130, 17, 3, 76, 73, 78, 156, 1, 80, + 9, 69, 83, 32, 65, 75, 85, 82, 85, 32, 238, 5, 73, 157, 251, 16, 2, 79, + 82, 144, 1, 142, 2, 68, 36, 7, 76, 69, 84, 84, 69, 82, 32, 236, 1, 5, 83, + 73, 71, 78, 32, 58, 86, 140, 162, 9, 6, 77, 69, 68, 73, 65, 76, 182, 255, + 4, 71, 222, 232, 4, 69, 128, 203, 4, 12, 80, 82, 69, 70, 73, 88, 69, 68, + 32, 78, 65, 83, 217, 248, 4, 7, 73, 78, 73, 84, 73, 65, 76, 22, 174, 190, + 25, 79, 211, 235, 1, 73, 84, 142, 195, 10, 84, 190, 237, 11, 89, 162, + 126, 65, 138, 146, 2, 68, 214, 6, 85, 206, 201, 1, 73, 42, 76, 250, 192, + 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 75, 2, 80, 138, 69, 72, 2, 74, 2, + 77, 2, 82, 2, 86, 2, 90, 186, 2, 69, 3, 79, 8, 174, 217, 24, 72, 150, 43, + 67, 98, 78, 235, 206, 3, 65, 18, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, + 78, 32, 179, 223, 26, 73, 16, 134, 150, 15, 65, 218, 174, 10, 85, 206, + 201, 1, 73, 222, 137, 2, 69, 3, 79, 10, 68, 5, 83, 73, 79, 78, 32, 212, + 241, 1, 2, 78, 71, 171, 145, 26, 68, 6, 26, 83, 227, 227, 5, 84, 4, 170, + 174, 27, 76, 211, 97, 73, 2, 165, 132, 19, 3, 32, 76, 65, 4, 134, 131, + 28, 83, 139, 86, 70, 212, 5, 188, 2, 6, 67, 85, 77, 69, 78, 84, 124, 7, + 69, 83, 32, 78, 79, 84, 32, 194, 3, 71, 202, 3, 76, 36, 10, 77, 73, 78, + 79, 32, 84, 73, 76, 69, 32, 226, 1, 78, 38, 84, 246, 2, 85, 204, 17, 2, + 87, 78, 164, 189, 16, 8, 32, 78, 79, 84, 32, 76, 73, 84, 156, 250, 11, 8, + 86, 69, 32, 79, 70, 32, 80, 69, 174, 4, 79, 235, 25, 68, 9, 33, 6, 32, + 87, 73, 84, 72, 32, 6, 40, 4, 84, 69, 88, 84, 139, 142, 2, 80, 5, 133, + 142, 2, 6, 32, 65, 78, 68, 32, 80, 22, 164, 1, 11, 67, 79, 78, 84, 65, + 73, 78, 32, 65, 83, 32, 92, 6, 68, 73, 86, 73, 68, 69, 112, 2, 80, 82, + 48, 7, 83, 85, 67, 67, 69, 69, 68, 253, 213, 22, 2, 70, 79, 6, 248, 1, + 15, 78, 79, 82, 77, 65, 76, 32, 83, 85, 66, 71, 82, 79, 85, 80, 231, 245, + 20, 77, 5, 217, 249, 21, 23, 32, 87, 73, 84, 72, 32, 82, 69, 86, 69, 82, + 83, 69, 68, 32, 78, 69, 71, 65, 84, 73, 79, 78, 6, 44, 5, 69, 67, 69, 68, + 69, 195, 146, 28, 79, 5, 233, 158, 9, 2, 32, 79, 125, 36, 3, 82, 65, 32, + 191, 209, 28, 32, 120, 120, 7, 76, 69, 84, 84, 69, 82, 32, 208, 1, 11, + 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 170, 161, 23, 65, 187, 2, 83, + 88, 178, 179, 25, 65, 38, 68, 82, 82, 34, 84, 230, 5, 85, 206, 201, 1, + 73, 162, 193, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, + 138, 69, 72, 2, 76, 2, 77, 2, 86, 2, 89, 186, 2, 69, 3, 79, 22, 194, 140, + 19, 86, 246, 171, 6, 65, 38, 85, 206, 201, 1, 73, 222, 137, 2, 69, 3, 79, + 4, 202, 141, 21, 80, 183, 188, 2, 76, 200, 1, 72, 8, 72, 79, 82, 73, 90, + 79, 78, 84, 1, 6, 86, 69, 82, 84, 73, 67, 100, 17, 2, 65, 76, 100, 32, 2, + 45, 48, 155, 251, 23, 32, 98, 58, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, + 3, 54, 14, 205, 166, 20, 2, 45, 48, 4, 174, 239, 26, 75, 255, 146, 1, 71, + 26, 34, 32, 57, 4, 84, 69, 68, 32, 8, 170, 130, 25, 80, 34, 77, 194, 70, + 79, 239, 194, 2, 65, 18, 214, 1, 67, 34, 83, 220, 145, 16, 3, 76, 73, 78, + 148, 133, 1, 4, 79, 66, 69, 76, 244, 9, 9, 84, 82, 65, 78, 83, 80, 79, + 83, 73, 238, 220, 6, 70, 141, 41, 14, 82, 73, 71, 72, 84, 45, 80, 79, 73, + 78, 84, 73, 78, 71, 4, 226, 180, 27, 73, 231, 16, 82, 4, 134, 243, 26, + 79, 223, 114, 81, 162, 1, 40, 3, 66, 76, 69, 137, 17, 2, 71, 72, 160, 1, + 42, 32, 190, 10, 45, 241, 5, 2, 68, 32, 106, 226, 2, 67, 174, 1, 68, 38, + 72, 32, 4, 73, 78, 84, 69, 38, 76, 144, 1, 7, 78, 69, 83, 84, 69, 68, 32, + 74, 80, 66, 83, 130, 2, 85, 36, 9, 86, 69, 82, 84, 73, 67, 65, 76, 32, + 144, 161, 6, 6, 79, 66, 76, 73, 81, 85, 196, 189, 2, 9, 82, 73, 71, 72, + 84, 32, 65, 82, 67, 246, 183, 3, 65, 242, 215, 9, 69, 182, 188, 4, 81, + 177, 105, 6, 87, 65, 86, 89, 32, 79, 24, 100, 7, 73, 82, 67, 76, 69, 68, + 32, 180, 207, 7, 4, 85, 82, 76, 89, 241, 196, 1, 4, 79, 76, 79, 78, 20, + 26, 78, 151, 196, 20, 68, 2, 157, 213, 2, 5, 85, 77, 66, 69, 82, 4, 190, + 236, 18, 79, 135, 199, 6, 65, 4, 198, 190, 20, 73, 223, 63, 89, 4, 154, + 130, 26, 82, 251, 244, 1, 71, 12, 54, 79, 129, 156, 17, 7, 69, 70, 84, + 32, 65, 82, 67, 10, 26, 87, 199, 190, 25, 71, 6, 26, 45, 139, 230, 27, + 32, 4, 174, 189, 20, 82, 211, 1, 57, 6, 224, 137, 17, 9, 76, 69, 83, 83, + 45, 84, 72, 65, 78, 179, 227, 9, 71, 8, 26, 82, 207, 248, 26, 76, 6, 242, + 157, 1, 69, 207, 235, 15, 73, 18, 80, 6, 81, 85, 65, 82, 69, 32, 30, 84, + 50, 85, 149, 215, 13, 4, 79, 76, 73, 68, 4, 230, 205, 25, 73, 55, 85, 4, + 204, 191, 12, 3, 65, 67, 75, 135, 201, 4, 82, 8, 58, 83, 146, 156, 1, 67, + 222, 136, 21, 80, 151, 236, 4, 66, 2, 169, 147, 28, 4, 80, 69, 78, 83, 4, + 214, 231, 18, 80, 139, 196, 9, 78, 10, 36, 3, 66, 65, 82, 163, 226, 27, + 76, 9, 11, 32, 6, 52, 7, 68, 79, 85, 66, 76, 69, 32, 151, 213, 26, 76, 4, + 146, 213, 26, 76, 51, 82, 48, 112, 5, 76, 73, 78, 69, 32, 220, 1, 7, 83, + 84, 82, 85, 67, 75, 32, 141, 226, 8, 7, 69, 78, 68, 69, 68, 32, 77, 12, + 48, 8, 83, 76, 65, 78, 84, 69, 68, 32, 75, 69, 8, 70, 69, 56, 7, 71, 82, + 69, 65, 84, 69, 82, 1, 4, 76, 69, 83, 83, 4, 185, 179, 20, 9, 81, 85, 65, + 76, 32, 84, 79, 32, 79, 2, 221, 165, 14, 5, 45, 84, 72, 65, 78, 34, 160, + 1, 8, 67, 65, 80, 73, 84, 65, 76, 32, 92, 7, 73, 84, 65, 76, 73, 67, 32, + 124, 6, 83, 77, 65, 76, 76, 32, 245, 208, 13, 8, 78, 45, 65, 82, 89, 32, + 83, 85, 18, 198, 163, 12, 71, 226, 201, 14, 80, 222, 137, 2, 67, 2, 72, + 2, 78, 2, 81, 2, 82, 3, 90, 10, 76, 6, 83, 77, 65, 76, 76, 32, 201, 230, + 21, 7, 67, 65, 80, 73, 84, 65, 76, 8, 214, 245, 28, 68, 2, 69, 2, 73, 3, + 74, 4, 238, 161, 12, 71, 231, 191, 16, 80, 6, 218, 161, 10, 70, 26, 77, + 135, 192, 17, 83, 2, 139, 251, 27, 78, 166, 1, 50, 32, 158, 1, 45, 189, + 1, 4, 87, 65, 82, 68, 10, 60, 4, 84, 65, 67, 75, 226, 206, 25, 70, 30, + 82, 191, 79, 65, 5, 29, 5, 32, 87, 73, 84, 72, 2, 11, 32, 2, 11, 67, 2, + 129, 245, 20, 4, 73, 82, 67, 76, 28, 60, 9, 80, 79, 73, 78, 84, 73, 78, + 71, 32, 211, 207, 25, 70, 24, 62, 83, 246, 209, 25, 65, 86, 69, 62, 70, + 46, 82, 207, 1, 84, 4, 142, 180, 17, 84, 205, 160, 8, 6, 77, 65, 76, 76, + 32, 82, 128, 1, 56, 8, 32, 70, 65, 67, 73, 78, 71, 32, 89, 2, 83, 32, 8, + 54, 72, 1, 9, 78, 79, 84, 67, 72, 69, 68, 32, 72, 4, 181, 204, 27, 3, 79, + 79, 75, 120, 178, 1, 65, 200, 2, 6, 66, 76, 65, 67, 75, 32, 38, 84, 172, + 237, 8, 2, 87, 72, 150, 154, 8, 90, 218, 250, 8, 68, 50, 70, 82, 72, 150, + 4, 67, 46, 81, 42, 82, 22, 83, 247, 7, 80, 34, 40, 4, 82, 82, 79, 87, + 179, 253, 25, 78, 33, 11, 32, 30, 144, 1, 5, 87, 73, 84, 72, 32, 160, + 228, 13, 14, 76, 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 70, 32, 85, 166, + 153, 12, 65, 178, 10, 70, 179, 5, 84, 22, 184, 183, 5, 6, 67, 79, 82, 78, + 69, 82, 182, 200, 20, 68, 58, 76, 42, 77, 38, 78, 58, 83, 66, 69, 250, + 12, 84, 139, 58, 72, 6, 178, 130, 21, 65, 175, 255, 4, 67, 36, 36, 2, 82, + 73, 161, 2, 2, 87, 79, 32, 44, 5, 65, 78, 71, 76, 69, 143, 144, 26, 80, + 30, 56, 8, 45, 72, 69, 65, 68, 69, 68, 32, 203, 149, 26, 32, 28, 68, 5, + 65, 82, 82, 79, 87, 230, 133, 17, 90, 166, 136, 9, 68, 39, 80, 23, 11, + 32, 20, 184, 137, 26, 15, 76, 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 70, + 32, 85, 80, 98, 84, 23, 87, 4, 134, 143, 26, 32, 105, 15, 45, 72, 69, 65, + 68, 69, 68, 32, 65, 82, 82, 79, 87, 32, 87, 22, 138, 1, 65, 106, 79, 152, + 1, 12, 85, 77, 32, 87, 73, 84, 72, 32, 68, 82, 85, 77, 160, 242, 16, 6, + 73, 86, 69, 32, 83, 76, 231, 220, 10, 69, 8, 238, 202, 2, 67, 200, 147, + 6, 10, 70, 84, 73, 78, 71, 32, 80, 79, 73, 78, 133, 235, 15, 3, 71, 79, + 78, 8, 56, 6, 77, 69, 68, 65, 82, 89, 34, 80, 159, 177, 27, 79, 2, 217, + 234, 20, 3, 32, 67, 65, 4, 204, 181, 18, 6, 32, 79, 70, 32, 66, 76, 171, + 251, 9, 76, 2, 189, 217, 12, 3, 83, 84, 73, 162, 2, 76, 7, 80, 76, 79, + 89, 65, 78, 32, 244, 132, 20, 2, 77, 80, 215, 220, 8, 67, 158, 2, 212, 2, + 6, 65, 70, 70, 73, 88, 32, 160, 7, 7, 76, 69, 84, 84, 69, 82, 32, 148, + 194, 1, 16, 84, 72, 73, 67, 75, 32, 76, 69, 84, 84, 69, 82, 32, 83, 69, + 76, 180, 174, 3, 4, 68, 79, 85, 66, 220, 230, 16, 19, 80, 85, 78, 67, 84, + 85, 65, 84, 73, 79, 78, 32, 67, 72, 73, 78, 79, 79, 75, 149, 194, 5, 11, + 83, 73, 71, 78, 32, 79, 32, 87, 73, 84, 72, 64, 140, 1, 9, 65, 84, 84, + 65, 67, 72, 69, 68, 32, 184, 1, 5, 72, 73, 71, 72, 32, 106, 76, 40, 4, + 82, 73, 71, 72, 173, 2, 4, 77, 73, 68, 32, 14, 112, 2, 84, 65, 232, 4, + 13, 76, 69, 70, 84, 45, 84, 79, 45, 82, 73, 71, 72, 84, 26, 83, 166, 239, + 20, 69, 3, 73, 6, 44, 5, 78, 71, 69, 78, 84, 159, 214, 27, 73, 5, 255, + 243, 20, 32, 20, 174, 2, 76, 50, 84, 38, 86, 166, 174, 8, 71, 186, 2, 65, + 170, 192, 18, 87, 186, 25, 67, 199, 129, 1, 68, 24, 36, 2, 69, 70, 29, 3, + 79, 87, 32, 2, 213, 2, 3, 84, 32, 72, 22, 94, 65, 38, 76, 50, 84, 38, 86, + 166, 174, 8, 71, 226, 194, 18, 87, 186, 25, 67, 199, 129, 1, 68, 4, 186, + 229, 21, 67, 199, 221, 5, 82, 4, 132, 128, 8, 3, 79, 78, 71, 219, 195, + 19, 73, 2, 229, 165, 8, 4, 73, 71, 72, 84, 4, 37, 7, 69, 82, 84, 73, 67, + 65, 76, 5, 131, 1, 32, 4, 42, 72, 41, 6, 86, 69, 82, 84, 73, 67, 2, 37, + 7, 79, 82, 73, 90, 79, 78, 84, 2, 17, 2, 65, 76, 2, 11, 32, 2, 11, 83, 2, + 161, 209, 27, 2, 69, 67, 214, 1, 222, 1, 65, 22, 68, 34, 69, 30, 70, 22, + 71, 22, 74, 162, 1, 75, 66, 76, 50, 77, 62, 78, 134, 1, 79, 50, 80, 86, + 82, 74, 83, 210, 2, 84, 66, 85, 42, 86, 38, 87, 70, 88, 162, 210, 27, 72, + 142, 60, 73, 206, 41, 89, 215, 22, 66, 5, 151, 180, 28, 79, 7, 142, 155, + 28, 32, 223, 61, 72, 7, 202, 216, 28, 69, 3, 85, 5, 207, 136, 28, 32, 5, + 255, 226, 24, 32, 19, 11, 32, 16, 76, 8, 87, 73, 84, 72, 32, 68, 79, 84, + 230, 5, 77, 2, 78, 195, 173, 27, 83, 5, 165, 218, 27, 12, 83, 32, 73, 78, + 83, 73, 68, 69, 32, 65, 78, 68, 9, 26, 32, 207, 214, 28, 75, 4, 186, 225, + 24, 82, 147, 245, 3, 77, 9, 248, 204, 22, 3, 79, 78, 71, 175, 137, 6, 72, + 11, 11, 32, 8, 162, 4, 78, 218, 173, 27, 87, 243, 163, 1, 83, 19, 38, 32, + 49, 5, 65, 83, 65, 76, 32, 8, 202, 3, 77, 218, 173, 27, 87, 243, 163, 1, + 83, 8, 226, 212, 28, 65, 2, 73, 2, 79, 3, 85, 11, 166, 211, 28, 79, 146, + 1, 65, 2, 85, 3, 87, 9, 52, 7, 69, 82, 78, 73, 78, 32, 65, 243, 131, 28, + 32, 4, 206, 211, 28, 77, 3, 78, 11, 248, 201, 22, 6, 79, 77, 65, 78, 73, + 65, 222, 203, 5, 32, 223, 61, 72, 49, 58, 32, 164, 1, 5, 76, 79, 65, 78, + 32, 191, 178, 6, 72, 26, 102, 74, 22, 75, 2, 80, 2, 84, 20, 7, 87, 73, + 84, 72, 32, 68, 79, 162, 209, 28, 77, 2, 78, 3, 83, 5, 235, 147, 28, 32, + 5, 215, 152, 28, 32, 4, 139, 217, 12, 84, 18, 74, 69, 230, 195, 5, 79, + 254, 188, 22, 65, 210, 78, 68, 146, 1, 74, 3, 85, 6, 190, 208, 28, 69, 2, + 72, 3, 78, 9, 26, 32, 131, 208, 28, 72, 4, 238, 218, 24, 82, 147, 245, 3, + 83, 9, 250, 255, 27, 32, 226, 79, 72, 3, 73, 5, 245, 148, 7, 4, 79, 67, + 65, 76, 17, 66, 79, 242, 149, 28, 32, 134, 37, 69, 218, 19, 65, 2, 72, 3, + 73, 5, 203, 206, 28, 87, 188, 91, 234, 2, 65, 188, 3, 10, 68, 73, 84, 79, + 82, 73, 65, 76, 32, 67, 22, 71, 224, 79, 4, 73, 71, 72, 84, 146, 2, 76, + 194, 9, 77, 214, 5, 78, 246, 3, 79, 36, 2, 81, 85, 182, 8, 82, 222, 1, + 83, 118, 84, 242, 31, 85, 198, 1, 88, 128, 5, 2, 89, 69, 200, 80, 4, 45, + 77, 65, 73, 148, 140, 19, 3, 74, 69, 67, 228, 147, 2, 8, 86, 69, 82, 71, + 82, 69, 69, 78, 175, 190, 5, 80, 22, 38, 82, 202, 230, 26, 83, 179, 64, + 71, 19, 30, 32, 137, 1, 2, 84, 72, 6, 88, 3, 79, 70, 32, 233, 229, 4, 13, + 87, 73, 84, 72, 32, 72, 69, 65, 82, 73, 78, 71, 32, 4, 140, 190, 11, 2, + 77, 65, 135, 230, 6, 82, 11, 17, 2, 32, 71, 8, 44, 5, 76, 79, 66, 69, 32, + 203, 148, 25, 82, 6, 70, 65, 181, 235, 21, 11, 69, 85, 82, 79, 80, 69, + 45, 65, 70, 82, 73, 4, 176, 137, 10, 4, 77, 69, 82, 73, 133, 154, 2, 11, + 83, 73, 65, 45, 65, 85, 83, 84, 82, 65, 76, 2, 183, 176, 2, 79, 228, 79, + 92, 17, 89, 80, 84, 73, 65, 78, 32, 72, 73, 69, 82, 79, 71, 76, 89, 80, + 72, 151, 199, 28, 71, 226, 79, 30, 32, 197, 74, 2, 45, 49, 172, 17, 146, + 3, 65, 178, 4, 66, 52, 2, 67, 48, 224, 1, 2, 68, 48, 170, 4, 69, 178, 3, + 70, 164, 4, 2, 71, 48, 210, 3, 72, 214, 1, 73, 238, 2, 76, 122, 77, 222, + 8, 78, 210, 5, 79, 140, 5, 2, 80, 48, 120, 2, 82, 48, 232, 1, 2, 83, 48, + 190, 3, 84, 220, 2, 2, 85, 48, 250, 2, 86, 134, 5, 87, 236, 2, 3, 88, 48, + 48, 88, 3, 89, 48, 48, 84, 2, 90, 48, 252, 214, 3, 3, 75, 48, 48, 189, + 146, 15, 3, 81, 48, 48, 228, 1, 30, 48, 133, 3, 2, 65, 48, 160, 1, 86, + 48, 98, 49, 102, 52, 166, 55, 51, 162, 241, 20, 55, 150, 227, 1, 50, 2, + 53, 3, 54, 24, 170, 68, 54, 170, 146, 22, 53, 134, 236, 5, 49, 2, 50, 2, + 51, 2, 52, 2, 55, 2, 56, 3, 57, 24, 242, 213, 22, 52, 2, 55, 134, 236, 5, + 48, 2, 49, 2, 50, 2, 51, 2, 53, 2, 54, 2, 56, 3, 57, 28, 142, 213, 22, + 48, 2, 50, 2, 51, 2, 53, 134, 236, 5, 49, 2, 52, 2, 54, 2, 55, 2, 56, 3, + 57, 68, 46, 48, 246, 54, 51, 210, 211, 22, 49, 3, 50, 22, 210, 65, 55, + 174, 254, 27, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 26, + 148, 9, 4, 69, 71, 73, 78, 185, 25, 2, 48, 48, 56, 34, 48, 90, 49, 175, + 244, 8, 50, 24, 230, 39, 50, 234, 150, 28, 49, 2, 51, 2, 52, 2, 53, 2, + 54, 2, 55, 2, 56, 3, 57, 22, 242, 209, 22, 48, 134, 236, 5, 49, 2, 50, 2, + 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 184, 1, 70, 48, 94, 51, + 102, 52, 102, 53, 106, 54, 194, 29, 50, 195, 230, 22, 49, 20, 194, 208, + 22, 56, 134, 236, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, + 24, 230, 207, 22, 49, 2, 52, 134, 236, 5, 48, 2, 50, 2, 51, 2, 53, 2, 54, + 2, 55, 2, 56, 3, 57, 24, 130, 207, 22, 54, 2, 56, 134, 236, 5, 48, 2, 49, + 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 3, 57, 42, 214, 60, 48, 202, 145, 22, + 50, 2, 52, 134, 236, 5, 49, 2, 51, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 32, + 194, 60, 55, 250, 252, 27, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, + 94, 30, 48, 189, 2, 2, 78, 68, 88, 38, 48, 94, 50, 102, 51, 215, 7, 49, + 22, 158, 204, 22, 56, 2, 57, 134, 236, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, + 2, 54, 3, 55, 24, 194, 203, 22, 48, 2, 56, 134, 236, 5, 49, 2, 50, 2, 51, + 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 18, 222, 202, 22, 52, 134, 236, 5, 48, + 2, 49, 2, 50, 2, 51, 2, 54, 2, 55, 3, 56, 6, 11, 32, 6, 136, 180, 5, 6, + 87, 65, 76, 76, 69, 68, 210, 19, 69, 203, 211, 20, 83, 132, 1, 42, 48, + 133, 9, 5, 85, 76, 76, 32, 66, 130, 1, 54, 48, 94, 49, 102, 51, 102, 52, + 102, 53, 215, 1, 50, 20, 202, 200, 22, 49, 134, 236, 5, 50, 2, 51, 2, 52, + 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 22, 238, 199, 22, 51, 134, 236, 5, 48, + 2, 49, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 26, 138, 199, 22, + 49, 2, 55, 2, 56, 134, 236, 5, 48, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, + 57, 26, 166, 198, 22, 53, 2, 54, 2, 55, 134, 236, 5, 48, 2, 49, 2, 50, 2, + 51, 2, 52, 2, 56, 3, 57, 14, 222, 26, 49, 234, 150, 28, 48, 2, 50, 3, 51, + 128, 1, 62, 48, 98, 49, 102, 51, 102, 52, 210, 27, 50, 147, 201, 8, 53, + 24, 166, 50, 55, 170, 146, 22, 54, 134, 236, 5, 49, 2, 50, 2, 51, 2, 52, + 2, 53, 2, 56, 3, 57, 22, 238, 195, 22, 49, 134, 236, 5, 48, 2, 50, 2, 51, + 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 24, 138, 195, 22, 54, 2, 55, + 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 56, 3, 57, 24, + 166, 194, 22, 51, 2, 53, 134, 236, 5, 48, 2, 49, 2, 50, 2, 52, 2, 54, 2, + 55, 2, 56, 3, 57, 24, 80, 2, 48, 48, 84, 4, 65, 76, 70, 32, 161, 40, 7, + 79, 82, 73, 90, 79, 78, 84, 18, 238, 192, 22, 54, 134, 236, 5, 49, 2, 50, + 2, 51, 2, 52, 2, 53, 2, 55, 3, 56, 4, 22, 66, 255, 42, 76, 2, 223, 238, + 16, 76, 52, 58, 48, 181, 1, 9, 78, 83, 69, 82, 84, 32, 65, 84, 32, 38, + 18, 48, 95, 49, 22, 158, 191, 22, 53, 2, 57, 134, 236, 5, 49, 2, 50, 2, + 51, 2, 52, 2, 54, 2, 55, 3, 56, 16, 194, 190, 22, 48, 2, 49, 134, 236, 5, 50, 2, 51, 2, 52, 3, 53, 14, 68, 6, 66, 79, 84, 84, 79, 77, 0, 3, 84, 79, - 80, 239, 138, 19, 77, 7, 11, 32, 4, 134, 152, 26, 69, 221, 6, 2, 83, 84, - 22, 32, 2, 48, 48, 215, 162, 15, 79, 20, 238, 227, 21, 50, 2, 54, 174, - 193, 5, 49, 2, 51, 2, 52, 2, 53, 2, 55, 3, 56, 164, 1, 158, 1, 48, 128, - 4, 15, 79, 68, 73, 70, 73, 69, 82, 32, 68, 65, 77, 65, 71, 69, 68, 189, - 239, 23, 14, 73, 82, 82, 79, 82, 32, 72, 79, 82, 73, 90, 79, 78, 84, 132, - 1, 42, 48, 98, 49, 106, 50, 102, 51, 107, 52, 24, 182, 40, 49, 146, 185, - 21, 51, 174, 193, 5, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 44, - 138, 41, 50, 222, 183, 21, 48, 2, 53, 2, 54, 2, 55, 174, 193, 5, 49, 2, - 51, 2, 52, 2, 56, 3, 57, 26, 254, 223, 21, 50, 2, 52, 2, 56, 174, 193, 5, - 48, 2, 49, 2, 51, 2, 53, 2, 54, 2, 55, 3, 57, 26, 138, 38, 51, 146, 185, - 21, 49, 174, 193, 5, 48, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, - 12, 178, 222, 21, 48, 174, 193, 5, 49, 2, 50, 2, 51, 3, 52, 31, 25, 4, - 32, 65, 84, 32, 28, 96, 6, 66, 79, 84, 84, 79, 77, 120, 5, 83, 84, 65, - 82, 84, 68, 3, 84, 79, 80, 139, 143, 26, 69, 11, 11, 32, 8, 56, 5, 83, - 84, 65, 82, 84, 186, 1, 65, 199, 142, 26, 69, 5, 129, 2, 8, 32, 65, 78, - 68, 32, 84, 79, 80, 7, 29, 5, 32, 65, 78, 68, 32, 4, 146, 212, 23, 66, - 247, 207, 2, 84, 11, 11, 32, 8, 54, 65, 20, 5, 83, 84, 65, 82, 84, 179, - 142, 26, 69, 2, 73, 2, 78, 68, 5, 53, 11, 32, 65, 78, 68, 32, 66, 79, 84, - 84, 79, 77, 2, 235, 135, 19, 32, 194, 1, 50, 48, 128, 2, 2, 76, 48, 229, - 1, 2, 85, 48, 98, 58, 49, 98, 51, 194, 14, 50, 150, 6, 52, 155, 249, 21, - 48, 24, 146, 32, 56, 190, 250, 26, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, - 2, 54, 2, 55, 3, 57, 28, 194, 216, 21, 51, 2, 52, 2, 53, 2, 55, 174, 193, - 5, 48, 2, 49, 2, 50, 2, 54, 2, 56, 3, 57, 44, 34, 48, 94, 49, 151, 181, - 20, 50, 20, 186, 215, 21, 53, 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, - 54, 2, 55, 2, 56, 3, 57, 22, 222, 214, 21, 55, 174, 193, 5, 48, 2, 49, 2, - 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 52, 34, 49, 102, 50, 159, - 138, 22, 48, 26, 214, 213, 21, 48, 2, 49, 2, 56, 174, 193, 5, 50, 2, 51, - 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 8, 242, 212, 21, 50, 174, 193, 5, 48, - 3, 49, 152, 1, 50, 48, 229, 172, 18, 6, 86, 69, 82, 76, 65, 89, 150, 1, - 66, 48, 154, 1, 49, 138, 1, 50, 102, 51, 106, 53, 247, 134, 22, 52, 34, - 90, 54, 238, 210, 21, 49, 2, 53, 174, 193, 5, 50, 2, 51, 2, 52, 2, 55, 2, - 56, 3, 57, 15, 150, 148, 27, 65, 2, 66, 2, 67, 2, 68, 2, 69, 3, 70, 28, - 98, 48, 206, 209, 21, 57, 174, 193, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, - 54, 2, 55, 3, 56, 9, 246, 146, 27, 65, 2, 66, 3, 67, 28, 166, 209, 21, - 48, 2, 52, 2, 53, 2, 57, 174, 193, 5, 49, 2, 50, 2, 51, 2, 54, 2, 55, 3, - 56, 32, 134, 23, 54, 190, 185, 21, 48, 2, 51, 174, 193, 5, 49, 2, 50, 2, - 52, 2, 53, 2, 55, 2, 56, 3, 57, 8, 202, 22, 48, 191, 250, 26, 49, 26, 26, - 48, 235, 148, 8, 49, 22, 158, 207, 21, 49, 2, 51, 174, 193, 5, 50, 2, 52, - 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 68, 34, 48, 98, 49, 219, 132, 22, 50, - 24, 142, 21, 51, 146, 185, 21, 50, 174, 193, 5, 49, 2, 52, 2, 53, 2, 54, - 2, 55, 2, 56, 3, 57, 24, 190, 205, 21, 48, 2, 54, 174, 193, 5, 49, 2, 50, - 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 108, 50, 48, 94, 49, 106, 50, - 98, 51, 155, 226, 18, 52, 22, 166, 204, 21, 50, 2, 54, 174, 193, 5, 49, - 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 26, 186, 18, 52, 146, 185, 21, - 55, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 2, 53, 2, 54, 2, 56, 3, 57, 24, - 210, 17, 54, 190, 250, 26, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, - 2, 56, 3, 57, 22, 130, 202, 21, 53, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, - 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 90, 34, 48, 249, 12, 3, 65, 76, 76, - 88, 42, 48, 94, 49, 102, 51, 171, 254, 21, 50, 26, 206, 200, 21, 51, 2, - 55, 2, 56, 2, 57, 174, 193, 5, 49, 2, 50, 2, 52, 2, 53, 3, 54, 24, 242, - 199, 21, 49, 2, 54, 174, 193, 5, 48, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, - 2, 56, 3, 57, 18, 142, 199, 21, 50, 2, 51, 174, 193, 5, 48, 2, 49, 2, 52, - 2, 53, 3, 54, 94, 50, 48, 90, 50, 102, 51, 102, 52, 139, 251, 21, 49, 22, - 254, 12, 54, 190, 250, 26, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, - 3, 57, 24, 182, 197, 21, 51, 2, 57, 174, 193, 5, 48, 2, 49, 2, 50, 2, 52, - 2, 53, 2, 54, 2, 55, 3, 56, 22, 210, 196, 21, 50, 174, 193, 5, 48, 2, 49, - 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 6, 154, 133, 27, 48, 2, - 49, 3, 50, 158, 1, 42, 48, 185, 4, 5, 69, 82, 84, 73, 67, 156, 1, 62, 48, - 98, 49, 98, 50, 210, 1, 51, 201, 191, 21, 2, 52, 48, 42, 198, 9, 55, 98, - 49, 178, 184, 21, 50, 174, 193, 5, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, - 32, 186, 8, 49, 46, 50, 190, 250, 26, 48, 2, 51, 2, 52, 2, 53, 2, 54, 2, - 55, 2, 56, 3, 57, 50, 98, 48, 182, 192, 21, 51, 2, 56, 2, 57, 174, 193, - 5, 49, 2, 50, 2, 52, 2, 53, 2, 54, 3, 55, 27, 222, 129, 27, 65, 2, 66, 2, - 67, 2, 68, 2, 69, 2, 70, 2, 71, 2, 72, 2, 73, 2, 74, 2, 75, 3, 76, 28, - 198, 191, 21, 48, 2, 49, 2, 51, 2, 55, 174, 193, 5, 50, 2, 52, 2, 53, 2, - 54, 2, 56, 3, 57, 2, 169, 134, 23, 2, 65, 76, 66, 34, 48, 161, 2, 3, 73, - 68, 69, 64, 26, 48, 94, 49, 103, 50, 22, 134, 190, 21, 51, 2, 57, 174, - 193, 5, 49, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 28, 170, 189, 21, - 48, 2, 52, 2, 55, 2, 56, 174, 193, 5, 49, 2, 50, 2, 51, 2, 53, 2, 54, 3, - 57, 14, 198, 188, 21, 52, 174, 193, 5, 48, 2, 49, 2, 50, 2, 51, 3, 53, 2, - 17, 2, 32, 76, 2, 211, 250, 14, 79, 24, 202, 2, 52, 146, 185, 21, 54, 2, - 56, 174, 193, 5, 49, 2, 50, 2, 51, 2, 53, 3, 55, 18, 130, 187, 21, 49, - 174, 193, 5, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 82, 22, 48, - 167, 1, 49, 34, 90, 50, 46, 51, 146, 185, 21, 52, 2, 53, 174, 193, 5, 49, - 2, 54, 2, 55, 2, 56, 3, 57, 11, 230, 250, 26, 65, 2, 66, 2, 67, 3, 68, 7, - 186, 250, 26, 65, 3, 66, 48, 66, 53, 86, 54, 138, 249, 26, 48, 2, 49, 2, - 50, 2, 51, 3, 52, 21, 218, 249, 26, 65, 2, 66, 2, 67, 2, 68, 2, 69, 2, - 70, 2, 71, 2, 72, 3, 73, 19, 134, 249, 26, 65, 2, 66, 2, 67, 2, 68, 2, - 69, 2, 70, 2, 71, 3, 72, 14, 26, 32, 159, 171, 15, 72, 12, 42, 80, 242, - 133, 3, 84, 135, 225, 21, 83, 8, 242, 132, 3, 79, 213, 129, 16, 22, 69, - 84, 65, 76, 76, 69, 68, 32, 79, 85, 84, 76, 73, 78, 69, 68, 32, 66, 76, - 65, 67, 75, 160, 1, 132, 1, 13, 66, 65, 83, 65, 78, 32, 76, 69, 84, 84, - 69, 82, 32, 166, 3, 69, 176, 4, 7, 89, 77, 65, 73, 67, 32, 76, 155, 238, - 26, 70, 80, 230, 1, 71, 78, 76, 34, 78, 50, 82, 158, 236, 24, 69, 198, - 134, 1, 67, 2, 68, 2, 75, 2, 83, 2, 84, 2, 90, 182, 105, 66, 2, 70, 2, - 72, 2, 74, 2, 77, 2, 80, 2, 81, 2, 86, 2, 88, 214, 22, 65, 2, 73, 2, 79, - 2, 85, 3, 89, 8, 38, 72, 138, 221, 26, 74, 215, 22, 69, 4, 194, 168, 24, - 65, 155, 203, 2, 69, 4, 226, 220, 26, 76, 215, 22, 69, 8, 194, 220, 26, - 68, 2, 74, 214, 22, 65, 3, 69, 4, 146, 220, 26, 82, 215, 22, 69, 32, 96, - 5, 67, 84, 82, 73, 67, 160, 1, 7, 77, 69, 78, 84, 32, 79, 70, 246, 230, - 25, 80, 223, 80, 86, 10, 26, 32, 215, 165, 23, 65, 8, 98, 80, 244, 134, - 17, 2, 84, 79, 204, 175, 8, 9, 76, 73, 71, 72, 84, 32, 66, 85, 76, 131, - 32, 65, 2, 11, 76, 2, 131, 240, 26, 85, 19, 11, 32, 16, 72, 5, 87, 73, - 84, 72, 32, 157, 142, 16, 7, 79, 80, 69, 78, 73, 78, 71, 12, 130, 1, 76, - 32, 12, 84, 87, 79, 32, 72, 79, 82, 73, 90, 79, 78, 84, 186, 144, 19, 86, - 186, 149, 4, 85, 166, 47, 79, 131, 153, 2, 68, 2, 141, 209, 24, 3, 79, - 78, 71, 2, 233, 158, 14, 7, 65, 76, 32, 83, 84, 82, 79, 46, 170, 154, 4, - 69, 153, 147, 15, 15, 73, 71, 65, 84, 85, 82, 69, 32, 90, 65, 89, 73, 78, - 45, 89, 53, 48, 4, 79, 74, 73, 32, 218, 2, 80, 163, 3, 32, 18, 164, 1, - 10, 67, 79, 77, 80, 79, 78, 69, 78, 84, 32, 109, 26, 77, 79, 68, 73, 70, - 73, 69, 82, 32, 70, 73, 84, 90, 80, 65, 84, 82, 73, 67, 75, 32, 84, 89, - 80, 69, 45, 8, 140, 200, 14, 2, 82, 69, 12, 5, 67, 85, 82, 76, 89, 0, 5, - 87, 72, 73, 84, 69, 173, 190, 2, 2, 66, 65, 10, 152, 135, 20, 2, 49, 45, - 214, 227, 6, 51, 2, 52, 2, 53, 3, 54, 26, 44, 3, 84, 89, 32, 225, 229, 3, - 2, 72, 65, 24, 82, 78, 60, 3, 80, 65, 71, 20, 3, 83, 69, 84, 177, 165, - 26, 4, 68, 79, 67, 85, 8, 36, 3, 79, 84, 69, 187, 161, 23, 69, 7, 131, - 232, 12, 32, 4, 139, 229, 24, 69, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, - 234, 203, 13, 82, 24, 3, 76, 69, 70, 176, 166, 1, 2, 83, 77, 207, 220, 8, - 79, 38, 86, 32, 64, 2, 68, 32, 214, 1, 81, 20, 6, 86, 69, 76, 79, 80, 69, - 235, 164, 15, 84, 6, 42, 81, 186, 129, 25, 68, 139, 167, 1, 83, 2, 191, - 160, 26, 85, 20, 32, 3, 79, 70, 32, 131, 1, 87, 18, 88, 3, 80, 82, 79, - 146, 149, 20, 71, 56, 2, 83, 69, 194, 59, 77, 30, 84, 139, 251, 4, 76, 4, - 210, 149, 20, 84, 239, 206, 5, 79, 2, 241, 169, 19, 7, 73, 84, 72, 32, - 76, 69, 70, 5, 167, 209, 20, 85, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 42, - 76, 221, 140, 23, 4, 68, 79, 87, 78, 2, 209, 248, 21, 4, 73, 71, 72, 84, - 6, 238, 227, 26, 76, 2, 77, 3, 84, 44, 28, 2, 65, 76, 231, 6, 73, 38, 30, - 32, 133, 2, 2, 83, 32, 12, 56, 3, 84, 79, 32, 237, 220, 12, 5, 65, 78, - 68, 32, 80, 10, 68, 3, 79, 82, 32, 217, 144, 26, 8, 66, 89, 32, 68, 69, - 70, 73, 78, 8, 64, 3, 80, 82, 69, 28, 3, 83, 85, 67, 170, 210, 24, 71, - 39, 76, 2, 133, 230, 14, 2, 67, 69, 2, 221, 132, 25, 3, 67, 69, 69, 26, - 72, 4, 83, 73, 71, 78, 232, 207, 24, 4, 87, 73, 84, 72, 223, 192, 1, 67, - 23, 11, 32, 20, 42, 65, 201, 1, 5, 87, 73, 84, 72, 32, 12, 112, 5, 66, - 79, 86, 69, 32, 65, 19, 78, 68, 32, 83, 76, 65, 78, 84, 69, 68, 32, 80, - 65, 82, 65, 76, 76, 69, 76, 8, 158, 178, 20, 80, 154, 5, 84, 150, 184, 2, - 76, 231, 207, 2, 82, 5, 135, 205, 6, 32, 8, 160, 1, 4, 66, 85, 77, 80, - 20, 7, 73, 78, 70, 73, 78, 73, 84, 20, 18, 84, 87, 79, 32, 68, 79, 84, - 83, 32, 65, 66, 79, 86, 69, 32, 65, 78, 68, 155, 230, 14, 68, 2, 163, - 224, 25, 89, 2, 159, 182, 24, 89, 2, 17, 2, 32, 84, 2, 195, 181, 24, 87, - 6, 80, 7, 86, 65, 76, 69, 78, 84, 32, 185, 181, 20, 7, 65, 78, 71, 85, - 76, 65, 82, 4, 48, 6, 87, 73, 84, 72, 32, 70, 207, 188, 26, 84, 2, 11, - 79, 2, 177, 229, 2, 2, 85, 82, 20, 152, 1, 11, 82, 79, 82, 45, 66, 65, - 82, 82, 69, 68, 32, 216, 147, 5, 7, 73, 83, 32, 70, 79, 82, 77, 205, 162, - 7, 9, 65, 83, 69, 32, 84, 79, 32, 84, 72, 12, 240, 200, 5, 4, 87, 72, 73, - 84, 13, 5, 66, 76, 65, 67, 75, 10, 58, 67, 20, 6, 84, 73, 77, 65, 84, 69, - 239, 216, 26, 65, 5, 143, 148, 25, 65, 4, 218, 197, 25, 68, 147, 147, 1, - 83, 154, 8, 60, 7, 72, 73, 79, 80, 73, 67, 32, 134, 216, 26, 66, 3, 88, - 150, 8, 204, 1, 2, 67, 79, 232, 1, 7, 78, 85, 77, 66, 69, 82, 32, 114, - 80, 54, 83, 156, 24, 11, 84, 79, 78, 65, 76, 32, 77, 65, 82, 75, 32, 174, - 183, 18, 68, 218, 184, 5, 70, 82, 81, 137, 140, 2, 4, 87, 79, 82, 68, 10, - 26, 77, 183, 134, 26, 76, 8, 52, 7, 66, 73, 78, 73, 78, 71, 32, 167, 211, - 26, 77, 6, 60, 11, 71, 69, 77, 73, 78, 65, 84, 73, 79, 78, 32, 51, 86, 4, - 44, 5, 65, 78, 68, 32, 86, 207, 147, 26, 77, 2, 129, 213, 21, 4, 79, 87, - 69, 76, 22, 66, 84, 226, 171, 21, 72, 210, 188, 3, 69, 30, 70, 42, 78, - 39, 83, 8, 130, 134, 16, 69, 222, 227, 8, 72, 27, 87, 4, 242, 103, 65, - 201, 155, 25, 5, 82, 69, 70, 65, 67, 198, 7, 50, 69, 37, 8, 89, 76, 76, - 65, 66, 76, 69, 32, 4, 154, 157, 23, 77, 187, 204, 2, 67, 194, 7, 210, 1, - 66, 90, 67, 246, 1, 68, 186, 1, 70, 90, 71, 214, 2, 72, 162, 1, 75, 102, - 77, 90, 78, 90, 80, 138, 2, 81, 174, 1, 82, 86, 83, 210, 1, 84, 122, 74, - 2, 76, 138, 1, 87, 2, 89, 66, 88, 134, 1, 90, 95, 86, 38, 194, 12, 87, - 230, 8, 66, 158, 250, 20, 65, 2, 79, 154, 129, 5, 69, 150, 64, 73, 3, 85, - 78, 94, 67, 254, 15, 72, 146, 254, 20, 65, 2, 79, 154, 129, 5, 69, 222, - 61, 87, 186, 2, 73, 3, 85, 42, 70, 72, 198, 141, 21, 65, 154, 129, 5, 69, - 150, 64, 73, 2, 79, 3, 85, 28, 166, 19, 72, 158, 250, 20, 65, 154, 129, - 5, 69, 150, 64, 73, 2, 79, 3, 85, 60, 94, 68, 214, 14, 90, 198, 253, 20, - 65, 2, 79, 154, 129, 5, 69, 222, 61, 87, 186, 2, 73, 3, 85, 30, 210, 14, - 72, 198, 253, 20, 65, 2, 79, 154, 129, 5, 69, 222, 61, 87, 186, 2, 73, 3, - 85, 24, 190, 8, 87, 130, 131, 21, 65, 154, 129, 5, 69, 222, 61, 89, 186, - 2, 73, 2, 79, 3, 85, 118, 142, 1, 85, 226, 7, 71, 232, 3, 7, 76, 79, 84, - 84, 65, 76, 32, 158, 2, 87, 218, 1, 89, 158, 250, 20, 65, 2, 79, 154, - 129, 5, 69, 151, 64, 73, 39, 29, 5, 82, 65, 71, 69, 32, 36, 74, 66, 2, - 70, 2, 77, 2, 80, 46, 71, 2, 75, 2, 81, 191, 210, 11, 72, 4, 11, 87, 4, - 182, 179, 26, 69, 215, 22, 73, 6, 11, 87, 6, 202, 137, 26, 69, 151, 64, - 73, 52, 70, 72, 206, 135, 21, 65, 2, 79, 154, 129, 5, 69, 150, 64, 73, 3, - 85, 36, 202, 4, 87, 230, 8, 89, 158, 250, 20, 65, 154, 129, 5, 69, 150, - 64, 73, 2, 79, 3, 85, 64, 250, 4, 88, 134, 6, 87, 218, 1, 89, 158, 250, - 20, 65, 2, 79, 154, 129, 5, 69, 150, 64, 73, 3, 85, 26, 142, 3, 87, 130, - 131, 21, 65, 2, 79, 154, 129, 5, 69, 222, 61, 89, 186, 2, 73, 3, 85, 36, - 166, 7, 89, 146, 254, 20, 65, 2, 79, 154, 129, 5, 69, 222, 61, 87, 186, - 2, 73, 3, 85, 56, 82, 72, 142, 1, 87, 130, 131, 21, 65, 2, 79, 154, 129, - 5, 69, 150, 64, 73, 3, 85, 32, 74, 65, 194, 131, 21, 79, 154, 129, 5, 69, - 222, 61, 87, 186, 2, 73, 3, 85, 19, 160, 9, 8, 82, 89, 78, 71, 69, 65, - 76, 32, 203, 187, 26, 65, 8, 150, 132, 26, 69, 150, 64, 65, 3, 73, 64, - 94, 72, 134, 6, 87, 218, 1, 89, 158, 250, 20, 65, 2, 79, 154, 129, 5, 69, - 150, 64, 73, 3, 85, 24, 130, 6, 87, 246, 251, 20, 65, 154, 129, 5, 69, - 150, 64, 73, 2, 79, 3, 85, 20, 170, 129, 21, 65, 2, 79, 154, 129, 5, 69, - 222, 61, 87, 2, 89, 186, 2, 73, 3, 85, 74, 102, 69, 226, 1, 72, 170, 3, - 90, 78, 83, 158, 250, 20, 65, 2, 79, 246, 190, 5, 87, 186, 2, 73, 3, 85, - 13, 56, 8, 66, 65, 84, 66, 69, 73, 84, 32, 227, 192, 26, 69, 8, 226, 251, - 21, 66, 2, 70, 2, 77, 3, 80, 80, 118, 72, 76, 2, 84, 72, 62, 90, 162, 2, - 83, 234, 250, 20, 65, 2, 79, 154, 129, 5, 69, 222, 61, 87, 186, 2, 73, 3, - 85, 18, 142, 254, 20, 65, 2, 79, 154, 129, 5, 69, 222, 61, 87, 186, 2, - 73, 3, 85, 12, 218, 254, 25, 69, 222, 61, 65, 186, 2, 73, 2, 79, 3, 85, - 16, 134, 253, 20, 65, 2, 79, 154, 129, 5, 69, 150, 64, 73, 3, 85, 40, 82, - 87, 218, 1, 89, 158, 250, 20, 65, 2, 79, 154, 129, 5, 69, 150, 64, 73, 3, - 85, 10, 242, 251, 20, 65, 154, 129, 5, 69, 151, 64, 73, 48, 90, 72, 78, - 90, 158, 250, 20, 65, 2, 79, 154, 129, 5, 69, 222, 61, 87, 186, 2, 73, 3, - 85, 16, 230, 250, 20, 65, 154, 129, 5, 69, 222, 61, 87, 186, 2, 73, 2, - 79, 3, 85, 14, 154, 250, 20, 65, 154, 129, 5, 69, 150, 64, 73, 2, 79, 3, - 85, 20, 130, 1, 68, 74, 72, 30, 75, 42, 82, 0, 7, 83, 72, 79, 82, 84, 32, - 82, 176, 253, 3, 3, 67, 72, 73, 189, 134, 22, 3, 89, 73, 90, 6, 48, 4, - 69, 82, 69, 84, 129, 166, 25, 2, 73, 70, 5, 17, 2, 45, 72, 2, 173, 132, - 26, 2, 73, 68, 4, 204, 165, 25, 2, 69, 78, 219, 12, 85, 2, 217, 139, 4, - 3, 73, 75, 82, 10, 68, 2, 82, 79, 205, 174, 25, 9, 76, 69, 82, 32, 67, - 79, 78, 83, 84, 8, 92, 5, 80, 69, 65, 78, 32, 172, 153, 23, 8, 45, 67, - 85, 82, 82, 69, 78, 67, 171, 151, 2, 32, 4, 186, 248, 3, 67, 19, 80, 42, - 82, 67, 102, 84, 201, 248, 25, 12, 80, 82, 69, 83, 83, 73, 79, 78, 76, - 69, 83, 83, 6, 60, 9, 76, 65, 77, 65, 84, 73, 79, 78, 32, 203, 159, 25, - 69, 4, 202, 235, 23, 81, 151, 137, 2, 77, 34, 98, 82, 217, 223, 20, 18, - 69, 78, 68, 69, 68, 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, - 14, 140, 1, 12, 69, 77, 69, 76, 89, 32, 72, 69, 65, 86, 89, 32, 245, 226, - 24, 16, 65, 84, 69, 82, 82, 69, 83, 84, 82, 73, 65, 76, 32, 65, 76, 73, - 12, 50, 83, 214, 160, 24, 70, 234, 2, 87, 207, 10, 71, 4, 230, 161, 24, - 65, 43, 73, 7, 194, 227, 18, 71, 215, 207, 7, 83, 130, 4, 142, 1, 65, - 206, 18, 69, 194, 1, 73, 186, 7, 76, 214, 4, 79, 238, 6, 82, 142, 5, 85, - 136, 232, 19, 2, 86, 83, 254, 150, 4, 83, 207, 134, 2, 70, 90, 122, 67, - 254, 14, 76, 176, 2, 2, 88, 32, 230, 129, 3, 77, 252, 213, 5, 2, 82, 83, - 168, 250, 1, 2, 84, 72, 255, 192, 9, 73, 68, 72, 2, 69, 32, 236, 235, 15, - 4, 83, 73, 77, 73, 249, 183, 4, 2, 84, 79, 64, 226, 1, 83, 160, 1, 4, 87, - 73, 84, 72, 168, 203, 12, 2, 80, 65, 176, 130, 2, 2, 77, 65, 132, 201, - 10, 13, 84, 72, 82, 79, 87, 73, 78, 71, 32, 65, 32, 75, 73, 201, 1, 14, - 72, 79, 76, 68, 73, 78, 71, 32, 66, 65, 67, 75, 32, 84, 4, 212, 182, 16, - 18, 65, 86, 79, 85, 82, 73, 78, 71, 32, 68, 69, 76, 73, 67, 73, 79, 85, - 83, 161, 243, 4, 13, 67, 82, 69, 65, 77, 73, 78, 71, 32, 73, 78, 32, 70, - 52, 38, 32, 221, 190, 21, 3, 79, 85, 84, 50, 144, 4, 2, 67, 79, 44, 5, - 72, 69, 65, 68, 45, 38, 77, 98, 79, 92, 7, 78, 79, 32, 71, 79, 79, 68, - 238, 1, 80, 164, 1, 16, 83, 84, 85, 67, 75, 45, 79, 85, 84, 32, 84, 79, - 78, 71, 85, 69, 110, 84, 168, 140, 18, 22, 70, 73, 78, 71, 69, 82, 32, - 67, 79, 86, 69, 82, 73, 78, 71, 32, 67, 76, 79, 83, 69, 68, 172, 67, 3, - 82, 79, 76, 180, 217, 1, 13, 76, 79, 79, 75, 32, 79, 70, 32, 84, 82, 73, - 85, 77, 184, 139, 1, 8, 68, 73, 65, 71, 79, 78, 65, 76, 1, 20, 85, 78, - 69, 86, 69, 78, 32, 69, 89, 69, 83, 32, 65, 78, 68, 32, 87, 65, 86, 89, - 4, 252, 4, 3, 87, 66, 79, 163, 208, 18, 76, 2, 225, 205, 21, 4, 66, 65, - 78, 68, 4, 60, 6, 69, 68, 73, 67, 65, 76, 217, 216, 24, 3, 79, 78, 79, 2, - 165, 229, 24, 3, 32, 77, 65, 12, 90, 75, 36, 4, 80, 69, 78, 32, 165, 218, - 19, 10, 78, 69, 32, 69, 89, 69, 66, 82, 79, 87, 2, 217, 188, 21, 4, 32, - 71, 69, 83, 8, 116, 5, 77, 79, 85, 84, 72, 161, 183, 21, 18, 69, 89, 69, - 83, 32, 65, 78, 68, 32, 72, 65, 78, 68, 32, 79, 86, 69, 82, 7, 11, 32, 4, - 228, 206, 9, 3, 86, 79, 77, 241, 130, 9, 5, 65, 78, 68, 32, 67, 6, 132, - 1, 18, 65, 82, 84, 89, 32, 72, 79, 82, 78, 32, 65, 78, 68, 32, 80, 65, - 82, 84, 100, 2, 69, 69, 133, 208, 18, 4, 76, 69, 65, 68, 2, 213, 230, 21, - 2, 89, 32, 7, 29, 5, 32, 65, 78, 68, 32, 4, 36, 3, 87, 73, 78, 159, 208, - 18, 84, 2, 153, 170, 1, 4, 75, 73, 78, 71, 4, 50, 69, 149, 234, 21, 6, - 72, 69, 82, 77, 79, 77, 2, 197, 144, 26, 8, 65, 82, 83, 32, 79, 70, 32, - 74, 10, 34, 76, 237, 192, 21, 2, 65, 70, 8, 84, 13, 73, 78, 71, 32, 68, - 73, 65, 71, 79, 78, 65, 76, 32, 197, 236, 22, 2, 69, 78, 6, 128, 1, 9, - 67, 82, 79, 83, 83, 73, 78, 71, 32, 181, 227, 18, 16, 73, 78, 32, 87, 72, - 73, 84, 69, 32, 67, 73, 82, 67, 76, 69, 32, 4, 224, 197, 15, 3, 82, 73, - 83, 135, 165, 3, 78, 4, 166, 191, 14, 73, 175, 251, 3, 77, 14, 50, 65, - 54, 77, 44, 2, 82, 82, 147, 149, 18, 78, 4, 214, 196, 17, 84, 245, 156, - 8, 4, 82, 70, 85, 76, 4, 144, 188, 8, 2, 73, 78, 239, 157, 7, 65, 4, 168, - 199, 7, 2, 73, 83, 215, 214, 18, 89, 54, 200, 1, 5, 71, 85, 82, 69, 32, - 38, 76, 186, 1, 82, 182, 2, 83, 208, 12, 4, 86, 69, 32, 68, 172, 233, 4, - 10, 69, 76, 68, 32, 72, 79, 67, 75, 69, 89, 233, 157, 19, 9, 78, 73, 84, - 69, 32, 80, 65, 82, 84, 4, 222, 182, 24, 68, 139, 167, 1, 83, 10, 32, 2, - 69, 32, 65, 2, 77, 32, 6, 174, 133, 15, 70, 172, 59, 4, 67, 65, 66, 73, - 247, 193, 7, 83, 4, 52, 4, 80, 82, 79, 74, 149, 198, 19, 3, 70, 82, 65, - 2, 221, 225, 25, 2, 69, 67, 22, 34, 69, 189, 1, 3, 83, 84, 32, 13, 56, 2, - 32, 69, 68, 4, 87, 79, 82, 75, 207, 244, 14, 67, 4, 214, 254, 12, 78, - 237, 192, 4, 8, 88, 84, 73, 78, 71, 85, 73, 83, 4, 220, 209, 22, 6, 32, - 83, 80, 65, 82, 75, 171, 199, 3, 83, 10, 222, 137, 5, 81, 168, 146, 10, - 8, 83, 84, 82, 79, 78, 71, 32, 73, 219, 197, 6, 80, 10, 38, 72, 177, 224, - 22, 3, 84, 69, 68, 9, 236, 234, 3, 13, 73, 78, 71, 32, 80, 79, 76, 69, - 32, 65, 78, 68, 32, 182, 171, 5, 69, 209, 250, 15, 19, 32, 67, 65, 75, - 69, 32, 87, 73, 84, 72, 32, 83, 87, 73, 82, 76, 32, 68, 69, 36, 50, 65, - 134, 1, 69, 98, 79, 194, 1, 85, 39, 89, 10, 78, 84, 220, 148, 7, 6, 71, - 32, 73, 78, 32, 72, 229, 246, 16, 3, 77, 73, 78, 6, 214, 163, 8, 32, 202, - 170, 11, 66, 183, 176, 5, 78, 4, 132, 245, 22, 8, 88, 69, 68, 32, 66, 73, - 67, 69, 241, 130, 1, 7, 85, 82, 45, 68, 69, 45, 76, 10, 46, 82, 28, 3, - 87, 69, 82, 231, 208, 24, 80, 2, 173, 140, 25, 2, 65, 76, 7, 17, 2, 32, - 80, 4, 58, 85, 141, 187, 23, 8, 76, 65, 89, 73, 78, 71, 32, 67, 2, 193, - 169, 25, 4, 78, 67, 84, 85, 4, 214, 139, 3, 83, 155, 240, 22, 84, 9, 25, - 4, 73, 78, 71, 32, 6, 242, 213, 9, 68, 148, 179, 8, 2, 83, 65, 139, 181, - 1, 69, 54, 102, 71, 20, 2, 76, 68, 68, 2, 79, 84, 22, 82, 238, 1, 85, - 172, 173, 22, 2, 78, 68, 203, 163, 3, 88, 5, 151, 255, 25, 71, 4, 196, - 148, 9, 8, 73, 78, 71, 32, 72, 65, 78, 68, 183, 195, 16, 69, 5, 147, 224, - 13, 80, 16, 102, 75, 218, 252, 19, 77, 250, 141, 4, 32, 222, 112, 67, - 177, 120, 9, 84, 85, 78, 69, 32, 67, 79, 79, 75, 8, 80, 10, 32, 65, 78, - 68, 32, 75, 78, 73, 70, 69, 218, 225, 14, 69, 247, 219, 10, 73, 5, 133, - 146, 15, 7, 32, 87, 73, 84, 72, 32, 80, 22, 26, 82, 159, 200, 22, 78, 20, - 38, 32, 218, 2, 84, 255, 154, 18, 45, 16, 110, 67, 174, 1, 68, 234, 154, - 2, 66, 204, 240, 9, 7, 76, 69, 65, 70, 32, 67, 76, 242, 105, 84, 171, - 130, 11, 80, 4, 244, 246, 12, 3, 76, 85, 66, 229, 85, 32, 79, 82, 78, 69, - 82, 32, 65, 82, 82, 79, 87, 83, 32, 67, 73, 82, 67, 76, 73, 78, 71, 32, - 65, 78, 84, 73, 67, 76, 79, 67, 75, 87, 4, 21, 3, 79, 84, 32, 4, 186, - 222, 22, 80, 183, 235, 2, 77, 2, 255, 31, 72, 28, 78, 65, 226, 1, 69, 98, - 79, 165, 174, 16, 8, 73, 69, 68, 32, 83, 72, 82, 73, 10, 72, 6, 67, 84, - 73, 79, 78, 32, 69, 8, 77, 69, 32, 87, 73, 84, 72, 32, 4, 214, 168, 19, - 83, 177, 6, 9, 78, 85, 77, 69, 82, 65, 84, 79, 82, 6, 50, 80, 166, 155, - 2, 65, 213, 196, 20, 2, 84, 73, 2, 213, 158, 21, 2, 73, 67, 6, 48, 6, 78, - 67, 72, 32, 70, 82, 171, 185, 18, 69, 4, 150, 244, 24, 73, 249, 12, 3, - 65, 78, 67, 10, 52, 3, 78, 84, 45, 116, 2, 87, 78, 143, 201, 25, 71, 4, - 70, 84, 145, 128, 2, 11, 70, 65, 67, 73, 78, 71, 32, 66, 65, 66, 89, 2, - 253, 132, 12, 6, 73, 76, 84, 69, 68, 32, 5, 181, 176, 18, 15, 73, 78, 71, - 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 79, 224, 1, 80, 2, 76, 76, - 250, 5, 78, 216, 165, 16, 5, 69, 76, 32, 80, 85, 215, 194, 9, 83, 216, 1, - 42, 32, 73, 6, 87, 73, 68, 84, 72, 32, 10, 158, 210, 11, 77, 160, 159, 3, - 2, 79, 85, 194, 170, 8, 66, 131, 30, 83, 206, 1, 238, 1, 67, 42, 76, 78, - 78, 30, 80, 66, 82, 134, 1, 83, 38, 89, 130, 230, 9, 77, 130, 171, 10, - 65, 158, 2, 68, 58, 69, 98, 71, 118, 72, 190, 4, 81, 214, 139, 2, 87, - 182, 24, 84, 220, 113, 6, 66, 82, 79, 75, 69, 78, 202, 4, 70, 159, 177, - 1, 86, 10, 238, 148, 20, 73, 62, 79, 227, 75, 69, 116, 42, 69, 162, 152, - 20, 65, 171, 132, 4, 79, 10, 162, 1, 70, 139, 154, 20, 83, 4, 182, 234, - 20, 79, 35, 85, 6, 42, 79, 138, 155, 20, 69, 247, 203, 2, 76, 2, 239, - 154, 24, 85, 10, 36, 3, 73, 71, 72, 231, 237, 23, 69, 8, 17, 2, 84, 32, - 8, 182, 163, 22, 67, 240, 1, 5, 87, 72, 73, 84, 69, 22, 80, 155, 2, 83, - 4, 218, 201, 22, 69, 203, 165, 1, 79, 2, 247, 136, 24, 69, 4, 236, 146, - 11, 8, 67, 84, 73, 79, 78, 32, 65, 80, 253, 212, 8, 5, 69, 82, 65, 76, - 32, 228, 18, 110, 65, 62, 69, 198, 17, 73, 162, 1, 76, 134, 15, 79, 194, - 6, 82, 158, 93, 85, 226, 175, 21, 72, 215, 199, 3, 83, 4, 196, 214, 24, - 2, 82, 76, 189, 139, 1, 4, 77, 69, 32, 68, 242, 2, 112, 2, 65, 82, 110, - 77, 50, 79, 224, 220, 22, 9, 82, 77, 65, 78, 32, 80, 69, 78, 78, 130, - 153, 2, 84, 215, 105, 78, 7, 29, 5, 32, 87, 73, 84, 72, 4, 220, 230, 13, - 5, 79, 85, 84, 32, 72, 177, 236, 8, 5, 32, 72, 65, 78, 68, 4, 26, 32, - 131, 171, 21, 73, 2, 199, 172, 22, 83, 226, 2, 64, 6, 77, 69, 84, 82, 73, - 67, 73, 6, 82, 71, 73, 65, 78, 32, 6, 236, 185, 19, 4, 65, 76, 76, 89, - 137, 214, 1, 5, 32, 80, 82, 79, 80, 220, 2, 228, 1, 6, 67, 65, 80, 73, - 84, 65, 0, 4, 83, 77, 65, 76, 172, 3, 7, 76, 69, 84, 84, 69, 82, 32, 180, - 2, 24, 77, 84, 65, 86, 82, 85, 76, 73, 32, 67, 65, 80, 73, 84, 65, 76, - 32, 76, 69, 84, 84, 69, 82, 32, 165, 6, 2, 80, 65, 80, 45, 9, 76, 32, 76, - 69, 84, 84, 69, 82, 32, 80, 142, 2, 65, 34, 72, 166, 5, 67, 118, 71, 130, - 1, 74, 34, 75, 82, 80, 34, 83, 94, 90, 176, 155, 5, 2, 84, 65, 234, 250, - 7, 76, 214, 168, 2, 82, 230, 229, 8, 66, 2, 77, 2, 88, 142, 37, 78, 2, - 81, 238, 34, 86, 222, 47, 68, 14, 69, 2, 73, 2, 79, 2, 85, 2, 89, 131, - 57, 87, 4, 210, 165, 25, 69, 215, 79, 78, 10, 46, 65, 134, 222, 25, 73, - 2, 79, 215, 22, 69, 4, 214, 244, 25, 69, 3, 82, 94, 254, 1, 85, 178, 2, - 65, 42, 67, 74, 69, 46, 71, 34, 72, 98, 74, 34, 75, 34, 76, 50, 80, 34, - 83, 34, 84, 62, 90, 238, 190, 15, 82, 230, 229, 8, 66, 2, 77, 2, 88, 142, - 37, 78, 2, 81, 238, 34, 86, 222, 47, 68, 14, 73, 2, 79, 2, 89, 130, 57, - 87, 255, 2, 70, 4, 252, 174, 21, 4, 45, 66, 82, 74, 191, 195, 4, 78, 92, - 250, 1, 65, 42, 67, 74, 69, 46, 71, 34, 72, 98, 74, 34, 75, 34, 76, 50, - 80, 34, 83, 34, 84, 62, 90, 238, 190, 15, 82, 230, 229, 8, 66, 2, 77, 2, - 88, 142, 37, 78, 2, 81, 238, 34, 86, 222, 47, 68, 14, 73, 2, 79, 2, 85, - 2, 89, 130, 57, 87, 255, 2, 70, 6, 182, 160, 25, 69, 2, 73, 215, 79, 78, - 8, 38, 72, 162, 230, 24, 73, 203, 57, 65, 4, 230, 159, 25, 73, 135, 23, - 65, 4, 180, 227, 8, 2, 76, 73, 231, 139, 17, 78, 4, 186, 167, 24, 72, - 227, 119, 65, 12, 46, 65, 206, 215, 25, 73, 2, 79, 215, 22, 69, 6, 26, - 82, 135, 238, 25, 69, 5, 155, 231, 24, 68, 4, 186, 166, 24, 72, 155, 62, - 73, 4, 166, 203, 24, 72, 215, 82, 65, 4, 11, 65, 4, 134, 154, 15, 66, - 159, 211, 10, 83, 4, 214, 202, 24, 72, 219, 105, 65, 4, 162, 237, 24, 72, - 235, 47, 65, 6, 220, 177, 3, 6, 85, 82, 78, 69, 68, 32, 143, 234, 1, 65, - 4, 218, 201, 24, 72, 215, 82, 69, 2, 165, 219, 19, 7, 82, 65, 71, 82, 65, - 80, 72, 10, 48, 2, 77, 69, 20, 4, 78, 71, 69, 82, 31, 82, 2, 239, 215, - 24, 76, 2, 141, 252, 17, 2, 32, 82, 6, 38, 76, 237, 189, 13, 3, 65, 70, - 70, 5, 151, 215, 24, 83, 202, 1, 66, 65, 214, 13, 79, 217, 154, 25, 7, - 69, 73, 67, 72, 32, 83, 84, 194, 1, 84, 8, 71, 79, 76, 73, 84, 73, 67, - 32, 229, 12, 8, 83, 83, 32, 79, 70, 32, 77, 73, 192, 1, 56, 6, 67, 65, - 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 96, 45, 9, 76, 32, 76, 69, 84, 84, - 69, 82, 32, 96, 206, 1, 65, 22, 66, 42, 67, 94, 68, 94, 70, 38, 71, 46, - 73, 138, 2, 76, 58, 77, 66, 78, 34, 79, 30, 80, 58, 82, 30, 83, 186, 1, - 84, 110, 86, 22, 89, 90, 90, 182, 199, 18, 72, 142, 243, 1, 85, 131, 131, - 5, 75, 2, 199, 193, 25, 90, 4, 214, 3, 73, 253, 208, 25, 2, 85, 75, 4, - 48, 8, 65, 85, 68, 65, 84, 69, 32, 67, 15, 72, 2, 11, 72, 2, 197, 252, 8, - 2, 82, 73, 6, 42, 74, 30, 79, 129, 194, 25, 2, 90, 69, 2, 253, 251, 8, 2, - 69, 82, 2, 187, 248, 23, 66, 4, 238, 191, 20, 82, 131, 128, 5, 73, 2, 21, - 3, 76, 65, 71, 2, 255, 148, 21, 79, 13, 38, 78, 54, 79, 141, 1, 2, 90, - 72, 2, 181, 182, 25, 8, 73, 84, 73, 65, 76, 32, 73, 90, 4, 33, 6, 84, 65, - 84, 69, 68, 32, 4, 26, 66, 25, 2, 83, 77, 2, 11, 73, 2, 35, 71, 2, 21, 3, - 65, 76, 76, 2, 177, 223, 23, 2, 32, 89, 4, 178, 223, 25, 73, 211, 2, 69, - 4, 52, 9, 65, 84, 73, 78, 65, 84, 69, 32, 77, 35, 74, 2, 193, 223, 10, 3, - 89, 83, 76, 2, 177, 231, 8, 3, 85, 68, 73, 2, 11, 65, 2, 239, 188, 20, - 83, 4, 226, 187, 25, 78, 3, 84, 4, 26, 79, 151, 224, 25, 69, 2, 225, 245, - 21, 2, 75, 79, 2, 225, 161, 21, 2, 73, 84, 14, 106, 72, 58, 76, 168, 195, - 13, 6, 80, 73, 68, 69, 82, 89, 209, 237, 9, 8, 77, 65, 76, 76, 32, 89, - 85, 83, 6, 32, 2, 84, 65, 207, 222, 25, 65, 5, 223, 183, 24, 80, 2, 251, - 218, 8, 79, 6, 78, 86, 208, 201, 21, 9, 82, 79, 75, 85, 84, 65, 83, 84, - 73, 175, 128, 4, 83, 2, 157, 227, 18, 2, 82, 73, 2, 195, 208, 23, 69, 12, - 50, 69, 246, 184, 18, 65, 254, 163, 7, 79, 3, 85, 6, 174, 184, 20, 83, - 151, 228, 4, 82, 4, 192, 235, 8, 3, 69, 77, 76, 221, 172, 15, 4, 72, 73, - 86, 69, 2, 243, 216, 25, 76, 6, 212, 199, 11, 13, 66, 69, 32, 87, 73, 84, - 72, 32, 77, 69, 82, 73, 68, 146, 177, 4, 87, 179, 207, 8, 86, 68, 162, 1, - 65, 44, 12, 84, 72, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 178, 174, 17, - 71, 182, 55, 82, 252, 182, 5, 3, 78, 71, 71, 238, 179, 1, 79, 149, 75, 2, - 76, 70, 4, 132, 255, 14, 2, 76, 32, 231, 218, 10, 84, 54, 202, 2, 65, 50, - 72, 46, 73, 46, 78, 46, 80, 2, 81, 40, 2, 82, 65, 22, 84, 240, 227, 8, 2, - 87, 73, 142, 233, 11, 85, 160, 70, 3, 70, 65, 73, 204, 7, 4, 66, 65, 73, - 82, 248, 21, 2, 79, 84, 202, 30, 68, 218, 99, 77, 144, 117, 3, 83, 65, - 85, 138, 16, 69, 128, 26, 3, 76, 65, 71, 142, 197, 1, 74, 184, 16, 2, 71, - 73, 141, 12, 2, 75, 85, 4, 132, 212, 23, 3, 73, 72, 86, 171, 128, 2, 72, - 4, 154, 179, 13, 87, 141, 154, 11, 2, 65, 71, 4, 142, 229, 8, 85, 201, - 155, 14, 2, 71, 71, 6, 194, 149, 15, 73, 137, 195, 8, 2, 65, 85, 2, 137, - 145, 25, 5, 65, 73, 82, 84, 72, 2, 147, 176, 25, 73, 4, 136, 231, 20, 2, - 72, 73, 185, 152, 2, 2, 69, 73, 234, 9, 46, 65, 190, 6, 69, 206, 82, 73, - 139, 3, 79, 152, 1, 96, 7, 68, 85, 65, 84, 73, 79, 78, 32, 5, 78, 84, 72, - 65, 32, 226, 232, 19, 86, 215, 214, 4, 80, 2, 11, 32, 2, 211, 194, 24, - 67, 146, 1, 156, 1, 7, 76, 69, 84, 84, 69, 82, 32, 212, 2, 5, 83, 73, 71, - 78, 32, 104, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 234, 191, 7, - 65, 247, 253, 17, 79, 100, 214, 1, 86, 202, 231, 21, 65, 38, 68, 46, 84, - 230, 24, 85, 210, 200, 1, 73, 42, 76, 246, 189, 1, 78, 46, 83, 82, 66, 2, - 67, 2, 71, 2, 74, 2, 75, 2, 80, 194, 40, 79, 162, 8, 69, 158, 20, 72, 2, - 77, 2, 82, 3, 89, 14, 60, 5, 69, 68, 73, 67, 32, 246, 235, 21, 79, 231, - 227, 3, 65, 4, 212, 160, 23, 6, 68, 79, 85, 66, 76, 69, 151, 234, 1, 65, - 16, 66, 67, 162, 191, 21, 80, 202, 40, 65, 170, 1, 78, 203, 162, 3, 86, - 4, 238, 200, 7, 79, 207, 159, 14, 65, 26, 218, 233, 21, 65, 106, 86, 214, - 20, 85, 210, 200, 1, 73, 218, 231, 1, 79, 163, 8, 69, 192, 8, 76, 10, 65, - 84, 69, 82, 45, 84, 72, 65, 78, 32, 206, 7, 69, 159, 190, 24, 89, 56, - 134, 1, 65, 150, 3, 66, 62, 79, 216, 2, 11, 69, 81, 85, 65, 76, 32, 84, - 79, 32, 79, 82, 218, 173, 6, 67, 138, 4, 87, 175, 141, 18, 83, 16, 44, 5, - 66, 79, 86, 69, 32, 183, 178, 6, 78, 12, 150, 1, 83, 180, 1, 19, 68, 79, - 85, 66, 76, 69, 45, 76, 73, 78, 69, 32, 69, 81, 85, 65, 76, 32, 65, 220, - 172, 6, 4, 76, 69, 83, 83, 207, 252, 17, 82, 6, 148, 1, 7, 73, 77, 73, - 76, 65, 82, 32, 221, 175, 6, 23, 76, 65, 78, 84, 69, 68, 32, 69, 81, 85, - 65, 76, 32, 65, 66, 79, 86, 69, 32, 76, 69, 83, 83, 4, 26, 65, 179, 175, - 6, 79, 2, 65, 3, 66, 79, 86, 6, 40, 4, 69, 83, 73, 68, 155, 176, 6, 85, - 2, 231, 2, 69, 20, 40, 2, 82, 32, 245, 1, 3, 86, 69, 82, 16, 120, 16, 83, - 76, 65, 78, 84, 69, 68, 32, 69, 81, 85, 65, 76, 32, 84, 79, 226, 177, 6, - 65, 150, 213, 12, 69, 191, 177, 4, 76, 9, 49, 10, 32, 87, 73, 84, 72, 32, - 68, 79, 84, 32, 6, 44, 5, 65, 66, 79, 86, 69, 171, 220, 17, 73, 5, 251, - 147, 25, 32, 4, 52, 7, 76, 65, 80, 80, 73, 78, 71, 235, 158, 19, 32, 2, - 253, 182, 23, 2, 32, 76, 134, 8, 36, 2, 75, 32, 185, 73, 2, 78, 32, 254, - 7, 130, 3, 65, 216, 15, 8, 67, 65, 80, 73, 84, 65, 76, 32, 182, 11, 68, - 134, 1, 70, 68, 2, 73, 78, 222, 3, 75, 138, 1, 76, 174, 3, 78, 66, 77, - 84, 3, 88, 69, 83, 22, 79, 202, 1, 80, 90, 82, 182, 1, 83, 130, 22, 84, - 200, 2, 13, 85, 80, 83, 73, 76, 79, 78, 32, 87, 73, 84, 72, 32, 150, 1, - 86, 142, 2, 89, 254, 186, 7, 66, 164, 174, 3, 4, 71, 82, 65, 77, 204, 23, - 2, 90, 69, 167, 177, 11, 81, 112, 92, 10, 67, 82, 79, 80, 72, 79, 78, 73, - 67, 32, 172, 14, 6, 78, 79, 32, 84, 69, 76, 23, 82, 106, 188, 2, 6, 65, - 84, 84, 73, 67, 32, 222, 5, 67, 92, 3, 78, 65, 88, 32, 12, 68, 69, 76, - 80, 72, 73, 67, 32, 70, 73, 86, 69, 0, 14, 83, 84, 82, 65, 84, 73, 65, - 78, 32, 70, 73, 70, 84, 89, 40, 11, 69, 80, 73, 68, 65, 85, 82, 69, 65, - 78, 32, 112, 3, 72, 69, 82, 164, 1, 9, 77, 69, 83, 83, 69, 78, 73, 65, - 78, 35, 84, 48, 72, 2, 70, 73, 180, 2, 4, 79, 78, 69, 32, 205, 1, 4, 84, - 69, 78, 32, 26, 36, 3, 70, 84, 89, 105, 2, 86, 69, 11, 11, 32, 8, 22, 84, - 171, 4, 83, 6, 48, 7, 72, 79, 85, 83, 65, 78, 68, 215, 3, 65, 5, 231, 3, - 32, 17, 11, 32, 14, 56, 7, 72, 85, 78, 68, 82, 69, 68, 18, 84, 143, 3, - 83, 7, 131, 2, 32, 6, 48, 7, 72, 79, 85, 83, 65, 78, 68, 187, 2, 65, 5, - 213, 1, 2, 32, 84, 14, 98, 72, 48, 7, 84, 72, 79, 85, 83, 65, 78, 230, - 186, 23, 81, 229, 222, 1, 5, 68, 82, 65, 67, 72, 6, 44, 5, 85, 78, 68, - 82, 69, 203, 186, 23, 65, 4, 17, 2, 68, 32, 4, 22, 84, 131, 1, 83, 2, 95, - 65, 8, 30, 84, 86, 83, 175, 1, 77, 4, 50, 65, 21, 8, 72, 79, 85, 83, 65, - 78, 68, 32, 2, 171, 241, 15, 76, 2, 11, 83, 2, 201, 187, 23, 2, 84, 65, - 4, 88, 5, 65, 82, 89, 83, 84, 145, 1, 12, 89, 82, 69, 78, 65, 73, 67, 32, - 84, 87, 79, 32, 2, 101, 5, 73, 65, 78, 32, 70, 2, 17, 2, 32, 77, 2, 147, - 226, 12, 78, 6, 30, 70, 29, 3, 84, 87, 79, 2, 221, 248, 14, 2, 73, 86, 5, - 11, 32, 2, 161, 234, 9, 5, 68, 82, 65, 67, 72, 8, 112, 8, 77, 73, 79, 78, - 73, 65, 78, 32, 253, 145, 24, 14, 65, 69, 85, 77, 32, 79, 78, 69, 32, 80, - 76, 69, 84, 72, 6, 150, 140, 12, 70, 254, 216, 11, 84, 227, 55, 79, 2, - 11, 32, 2, 215, 228, 23, 84, 32, 92, 8, 72, 69, 83, 80, 73, 65, 78, 32, - 129, 1, 10, 82, 79, 69, 90, 69, 78, 73, 65, 78, 32, 20, 40, 2, 70, 73, - 38, 84, 239, 217, 17, 79, 6, 222, 139, 20, 86, 219, 190, 3, 70, 8, 246, - 244, 14, 72, 190, 239, 9, 69, 227, 48, 87, 12, 36, 2, 70, 73, 209, 24, 2, - 84, 69, 8, 142, 243, 17, 86, 145, 193, 6, 3, 70, 84, 89, 2, 231, 221, 9, - 69, 4, 252, 246, 21, 2, 79, 85, 149, 164, 1, 3, 84, 65, 66, 154, 2, 66, - 76, 174, 45, 82, 66, 68, 168, 188, 7, 2, 75, 65, 135, 6, 84, 144, 2, 44, - 6, 69, 84, 84, 69, 82, 32, 239, 45, 85, 142, 2, 198, 2, 65, 190, 1, 69, - 28, 4, 73, 79, 84, 65, 128, 1, 2, 79, 77, 156, 3, 3, 82, 72, 79, 46, 83, - 48, 7, 85, 80, 83, 73, 76, 79, 78, 146, 33, 80, 170, 2, 84, 138, 157, 5, - 68, 156, 164, 2, 2, 75, 65, 146, 192, 1, 71, 194, 223, 10, 67, 166, 255, - 1, 66, 2, 72, 2, 90, 166, 1, 76, 254, 210, 2, 89, 198, 43, 77, 2, 78, - 147, 17, 88, 48, 68, 4, 76, 80, 72, 65, 213, 28, 8, 82, 67, 72, 65, 73, - 67, 32, 83, 47, 33, 6, 32, 87, 73, 84, 72, 32, 44, 242, 2, 68, 30, 80, - 226, 29, 86, 226, 5, 79, 142, 226, 3, 84, 147, 140, 5, 77, 62, 186, 1, - 84, 131, 27, 80, 31, 33, 6, 32, 87, 73, 84, 72, 32, 28, 186, 5, 68, 136, - 25, 2, 80, 83, 158, 1, 86, 226, 5, 79, 142, 226, 3, 84, 147, 140, 5, 77, - 62, 28, 2, 69, 71, 235, 34, 73, 42, 11, 65, 43, 33, 6, 32, 87, 73, 84, - 72, 32, 40, 54, 68, 30, 80, 194, 35, 79, 22, 86, 251, 225, 3, 84, 16, 65, - 4, 65, 83, 73, 65, 18, 36, 4, 83, 73, 76, 73, 211, 16, 82, 17, 29, 5, 32, - 65, 78, 68, 32, 14, 44, 2, 79, 88, 0, 3, 86, 65, 82, 23, 80, 4, 81, 2, - 73, 65, 6, 60, 10, 69, 82, 73, 83, 80, 79, 77, 69, 78, 73, 175, 15, 82, - 5, 169, 15, 7, 32, 65, 78, 68, 32, 80, 82, 5, 161, 35, 7, 32, 87, 73, 84, - 72, 32, 68, 6, 170, 230, 7, 73, 206, 242, 16, 65, 227, 48, 72, 23, 33, 6, - 32, 87, 73, 84, 72, 32, 20, 66, 68, 166, 26, 86, 226, 5, 79, 142, 226, 3, - 84, 147, 140, 5, 77, 10, 130, 24, 65, 133, 203, 1, 5, 73, 65, 76, 89, 84, - 18, 76, 9, 73, 65, 76, 89, 84, 73, 75, 65, 32, 32, 3, 82, 65, 67, 227, - 22, 65, 8, 174, 24, 65, 223, 232, 3, 84, 2, 235, 146, 11, 72, 4, 40, 3, - 73, 86, 69, 1, 3, 79, 85, 82, 2, 205, 37, 2, 32, 79, 76, 144, 1, 27, 83, - 84, 82, 85, 77, 69, 78, 84, 65, 76, 32, 78, 79, 84, 65, 84, 73, 79, 78, - 32, 83, 89, 77, 66, 79, 76, 45, 165, 202, 21, 2, 68, 73, 74, 70, 49, 70, - 50, 62, 51, 62, 52, 170, 37, 53, 250, 252, 24, 55, 3, 56, 17, 218, 163, - 25, 49, 2, 50, 2, 51, 2, 52, 2, 55, 2, 56, 3, 57, 15, 150, 163, 25, 51, - 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 12, 218, 162, 25, 48, 2, 50, 2, 54, 2, - 55, 2, 56, 3, 57, 17, 158, 162, 25, 48, 2, 50, 2, 51, 2, 53, 2, 55, 2, - 56, 3, 57, 8, 54, 65, 38, 79, 169, 32, 6, 89, 65, 84, 72, 79, 83, 4, 210, - 216, 7, 80, 195, 181, 16, 73, 2, 11, 82, 2, 11, 79, 2, 251, 131, 23, 78, - 32, 128, 1, 6, 69, 84, 84, 69, 82, 32, 168, 2, 6, 79, 87, 69, 82, 32, 78, - 32, 6, 85, 78, 65, 84, 69, 32, 157, 225, 21, 2, 73, 84, 24, 94, 83, 140, - 13, 9, 65, 82, 67, 72, 65, 73, 67, 32, 75, 2, 75, 182, 11, 68, 135, 182, - 24, 89, 16, 88, 13, 77, 65, 76, 76, 32, 67, 65, 80, 73, 84, 65, 76, 32, - 210, 12, 65, 195, 205, 7, 84, 12, 74, 80, 154, 154, 9, 71, 174, 131, 9, - 79, 238, 195, 2, 82, 243, 152, 1, 76, 4, 238, 137, 25, 83, 219, 19, 73, - 2, 249, 223, 9, 3, 85, 77, 69, 4, 222, 25, 83, 163, 186, 7, 69, 4, 80, 4, - 69, 84, 82, 69, 253, 208, 22, 10, 85, 83, 73, 67, 65, 76, 32, 76, 69, 73, - 2, 139, 206, 2, 84, 12, 88, 3, 78, 69, 32, 142, 198, 9, 88, 200, 136, 5, - 3, 85, 78, 75, 161, 181, 5, 2, 66, 79, 6, 64, 8, 72, 65, 76, 70, 32, 83, - 73, 71, 21, 4, 81, 85, 65, 82, 4, 207, 155, 24, 78, 2, 159, 132, 20, 84, - 16, 62, 82, 206, 11, 83, 114, 69, 230, 197, 7, 72, 203, 180, 16, 73, 2, - 217, 29, 2, 79, 83, 6, 100, 3, 72, 79, 32, 241, 207, 7, 16, 69, 86, 69, - 82, 83, 69, 68, 32, 76, 85, 78, 65, 84, 69, 32, 69, 4, 140, 197, 13, 10, - 87, 73, 84, 72, 32, 83, 84, 82, 79, 75, 243, 192, 10, 83, 226, 2, 220, 1, - 5, 77, 65, 76, 76, 32, 192, 19, 22, 85, 66, 83, 67, 82, 73, 80, 84, 32, - 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 72, 10, 89, 77, 66, - 79, 76, 32, 84, 65, 85, 32, 237, 156, 23, 6, 73, 78, 85, 83, 79, 73, 212, - 2, 56, 7, 76, 69, 84, 84, 69, 82, 32, 202, 17, 82, 67, 68, 206, 2, 178, - 2, 65, 162, 2, 68, 38, 69, 52, 4, 73, 79, 84, 65, 0, 7, 85, 80, 83, 73, - 76, 79, 78, 254, 3, 75, 28, 2, 79, 77, 182, 5, 80, 112, 3, 82, 72, 79, - 94, 83, 94, 84, 192, 192, 7, 2, 70, 73, 246, 192, 1, 71, 194, 223, 10, - 67, 166, 255, 1, 66, 2, 72, 2, 90, 166, 1, 76, 194, 254, 2, 77, 2, 78, - 147, 17, 88, 58, 64, 4, 76, 80, 72, 65, 149, 1, 7, 82, 67, 72, 65, 73, - 67, 32, 55, 33, 6, 32, 87, 73, 84, 72, 32, 52, 82, 86, 226, 6, 68, 30, - 80, 114, 79, 142, 1, 89, 250, 227, 3, 84, 147, 140, 5, 77, 6, 154, 5, 82, - 155, 3, 65, 4, 18, 75, 23, 83, 2, 163, 206, 7, 79, 2, 11, 65, 2, 203, - 143, 13, 77, 4, 214, 184, 7, 73, 191, 180, 14, 69, 70, 22, 80, 215, 4, - 84, 20, 249, 7, 3, 83, 73, 76, 41, 33, 6, 32, 87, 73, 84, 72, 32, 38, 78, - 68, 166, 1, 80, 178, 1, 86, 226, 5, 79, 142, 226, 3, 84, 147, 140, 5, 77, - 18, 50, 65, 29, 8, 73, 65, 76, 89, 84, 73, 75, 65, 8, 153, 1, 3, 83, 73, - 65, 11, 29, 5, 32, 65, 78, 68, 32, 8, 170, 1, 80, 154, 6, 79, 22, 86, - 251, 225, 3, 84, 10, 18, 83, 115, 69, 8, 21, 3, 73, 76, 73, 9, 17, 2, 32, - 65, 6, 21, 3, 78, 68, 32, 6, 30, 80, 154, 6, 79, 23, 86, 2, 11, 69, 2, - 11, 82, 2, 181, 17, 4, 73, 83, 80, 79, 4, 22, 82, 147, 15, 65, 2, 177, - 235, 24, 2, 65, 67, 4, 154, 201, 7, 65, 3, 79, 70, 28, 2, 69, 71, 151, 3, - 73, 50, 11, 65, 51, 33, 6, 32, 87, 73, 84, 72, 32, 48, 58, 68, 30, 80, - 114, 79, 62, 86, 82, 89, 251, 227, 3, 84, 16, 61, 4, 65, 83, 73, 65, 20, - 32, 4, 83, 73, 76, 73, 91, 69, 17, 29, 5, 32, 65, 78, 68, 32, 14, 42, 79, - 12, 2, 80, 69, 50, 86, 83, 89, 4, 83, 88, 4, 89, 9, 82, 73, 83, 80, 79, - 77, 69, 78, 73, 4, 11, 65, 4, 11, 82, 4, 17, 2, 73, 65, 5, 33, 6, 32, 65, - 78, 68, 32, 89, 2, 243, 12, 80, 20, 17, 2, 67, 82, 20, 17, 2, 79, 78, 21, - 33, 6, 32, 87, 73, 84, 72, 32, 18, 88, 5, 68, 65, 83, 73, 65, 0, 5, 80, - 83, 73, 76, 73, 54, 79, 22, 86, 251, 225, 3, 84, 7, 29, 5, 32, 65, 78, - 68, 32, 4, 18, 79, 23, 86, 2, 151, 178, 9, 88, 2, 179, 9, 65, 8, 88, 11, - 65, 77, 80, 72, 89, 76, 73, 65, 78, 32, 68, 218, 242, 24, 72, 2, 83, 219, - 19, 73, 2, 215, 173, 7, 73, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 34, 68, - 201, 182, 20, 2, 80, 83, 2, 215, 159, 8, 65, 10, 54, 65, 134, 193, 7, 84, - 230, 1, 73, 175, 163, 17, 72, 4, 142, 131, 13, 77, 251, 129, 12, 78, 4, - 150, 224, 21, 72, 231, 255, 2, 65, 4, 41, 8, 69, 86, 69, 82, 83, 69, 68, - 32, 4, 18, 68, 43, 76, 2, 37, 7, 79, 84, 84, 69, 68, 32, 76, 2, 11, 85, - 2, 33, 6, 78, 65, 84, 69, 32, 83, 2, 245, 193, 7, 3, 73, 71, 77, 10, 214, - 255, 8, 71, 194, 223, 10, 67, 2, 80, 218, 103, 82, 207, 151, 1, 66, 2, - 167, 198, 20, 82, 16, 106, 72, 104, 7, 82, 89, 66, 76, 73, 79, 78, 44, 3, - 87, 79, 32, 150, 219, 3, 79, 141, 132, 16, 2, 65, 76, 6, 40, 4, 82, 69, - 69, 32, 219, 191, 7, 69, 4, 146, 1, 79, 185, 230, 21, 7, 81, 85, 65, 82, - 84, 69, 82, 2, 21, 3, 32, 66, 65, 2, 231, 232, 22, 83, 4, 42, 79, 253, - 234, 11, 4, 84, 72, 73, 82, 2, 229, 148, 19, 2, 66, 79, 6, 80, 5, 65, 67, - 85, 84, 69, 0, 9, 68, 73, 65, 69, 82, 69, 83, 73, 83, 39, 72, 2, 33, 6, - 32, 65, 78, 68, 32, 72, 2, 145, 157, 7, 2, 79, 79, 60, 102, 65, 21, 21, - 79, 67, 65, 76, 32, 78, 79, 84, 65, 84, 73, 79, 78, 32, 83, 89, 77, 66, - 79, 76, 45, 2, 207, 168, 9, 82, 58, 90, 50, 2, 53, 166, 195, 22, 49, 214, - 185, 2, 51, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 13, 246, 252, 24, 48, 2, - 49, 2, 50, 2, 51, 3, 52, 4, 26, 80, 139, 230, 19, 69, 2, 11, 79, 2, 33, - 6, 71, 69, 71, 82, 65, 77, 2, 253, 171, 20, 2, 77, 69, 8, 238, 189, 13, - 65, 142, 237, 9, 66, 210, 73, 72, 241, 64, 3, 83, 65, 76, 12, 60, 6, 78, - 78, 73, 78, 71, 32, 205, 242, 23, 3, 77, 65, 67, 10, 100, 4, 70, 65, 67, - 69, 217, 166, 17, 15, 67, 65, 84, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, - 32, 83, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 108, 23, 79, 78, 69, 32, 76, - 65, 82, 71, 69, 32, 65, 78, 68, 32, 79, 78, 69, 32, 83, 77, 65, 76, 76, - 35, 83, 2, 11, 32, 2, 159, 247, 7, 69, 4, 174, 165, 17, 77, 145, 1, 3, - 84, 65, 82, 6, 28, 3, 85, 80, 32, 39, 87, 4, 250, 222, 21, 83, 211, 215, - 2, 77, 2, 251, 211, 17, 73, 232, 3, 164, 1, 2, 65, 82, 70, 73, 52, 7, 74, - 65, 82, 65, 84, 73, 32, 212, 6, 12, 78, 74, 65, 76, 65, 32, 71, 79, 78, - 68, 73, 32, 137, 3, 7, 82, 77, 85, 75, 72, 73, 32, 4, 34, 65, 157, 172, - 20, 2, 68, 83, 2, 11, 78, 2, 239, 238, 23, 73, 4, 154, 211, 23, 84, 201, - 161, 1, 4, 68, 69, 32, 68, 182, 1, 168, 1, 7, 76, 69, 84, 84, 69, 82, 32, - 220, 1, 5, 83, 73, 71, 78, 32, 160, 2, 6, 86, 79, 87, 69, 76, 32, 226, - 175, 19, 65, 210, 24, 82, 154, 185, 3, 68, 199, 221, 1, 79, 98, 142, 139, - 21, 65, 38, 68, 46, 84, 46, 86, 186, 24, 85, 210, 200, 1, 73, 42, 76, - 234, 14, 90, 142, 175, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, - 2, 80, 254, 68, 72, 2, 77, 2, 82, 2, 89, 186, 2, 69, 3, 79, 24, 98, 67, - 28, 3, 77, 65, 68, 22, 84, 250, 194, 3, 83, 138, 199, 17, 65, 170, 1, 78, - 203, 162, 3, 86, 4, 118, 73, 143, 138, 21, 65, 2, 187, 184, 17, 68, 4, - 68, 5, 87, 79, 45, 67, 73, 29, 8, 72, 82, 69, 69, 45, 68, 79, 84, 2, 25, - 4, 82, 67, 76, 69, 2, 129, 249, 19, 5, 32, 78, 85, 75, 84, 34, 36, 5, 83, - 73, 71, 78, 32, 91, 67, 30, 86, 67, 142, 138, 21, 65, 106, 86, 214, 20, - 85, 210, 200, 1, 73, 206, 134, 2, 69, 3, 79, 4, 201, 138, 21, 5, 65, 78, - 68, 82, 65, 126, 108, 7, 76, 69, 84, 84, 69, 82, 32, 212, 1, 5, 83, 73, - 71, 78, 32, 38, 86, 186, 253, 22, 68, 199, 221, 1, 79, 80, 250, 132, 21, - 65, 38, 68, 46, 84, 230, 24, 85, 22, 78, 190, 200, 1, 73, 42, 76, 242, - 190, 1, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 194, 40, 79, 162, 8, 69, - 158, 20, 72, 2, 77, 2, 82, 2, 83, 2, 86, 3, 89, 4, 174, 188, 22, 86, 223, - 234, 1, 65, 20, 230, 6, 79, 247, 178, 22, 73, 172, 1, 194, 1, 65, 44, 7, - 76, 69, 84, 84, 69, 82, 32, 242, 1, 83, 228, 2, 2, 86, 79, 244, 226, 12, - 3, 84, 73, 80, 146, 249, 4, 73, 172, 130, 3, 5, 69, 75, 32, 79, 78, 162, - 152, 2, 68, 235, 169, 1, 85, 4, 138, 255, 20, 66, 253, 187, 1, 2, 68, 68, - 96, 202, 156, 8, 71, 2, 75, 218, 172, 12, 82, 206, 55, 65, 38, 68, 46, - 84, 230, 24, 85, 210, 200, 1, 73, 42, 76, 246, 189, 1, 78, 126, 66, 2, - 67, 2, 74, 2, 80, 2, 83, 194, 40, 79, 162, 8, 69, 158, 20, 70, 2, 72, 2, - 77, 2, 86, 2, 89, 3, 90, 26, 108, 19, 69, 81, 85, 69, 78, 67, 69, 32, 70, - 79, 82, 32, 76, 69, 84, 84, 69, 82, 32, 93, 4, 73, 71, 78, 32, 12, 70, - 71, 2, 75, 222, 238, 22, 83, 166, 213, 1, 76, 226, 31, 70, 3, 90, 2, 219, - 238, 22, 72, 14, 128, 1, 6, 65, 68, 65, 75, 32, 66, 2, 66, 164, 211, 14, - 2, 85, 68, 150, 172, 6, 78, 160, 128, 2, 3, 89, 65, 75, 171, 162, 1, 86, - 2, 139, 238, 7, 73, 18, 45, 9, 87, 69, 76, 32, 83, 73, 71, 78, 32, 18, - 226, 255, 20, 65, 190, 21, 85, 210, 200, 1, 73, 218, 231, 1, 79, 163, 8, - 69, 142, 23, 110, 65, 162, 95, 69, 206, 103, 73, 146, 9, 79, 142, 11, 84, - 30, 85, 130, 1, 89, 137, 186, 12, 4, 82, 89, 86, 78, 136, 11, 248, 1, 2, - 73, 82, 60, 8, 76, 70, 87, 73, 68, 84, 72, 32, 254, 10, 77, 210, 1, 78, - 236, 76, 21, 80, 80, 89, 32, 80, 69, 82, 83, 79, 78, 32, 82, 65, 73, 83, - 73, 78, 71, 32, 79, 78, 22, 84, 180, 183, 17, 2, 85, 77, 128, 141, 5, 2, - 82, 68, 227, 46, 68, 6, 26, 32, 163, 231, 23, 67, 4, 134, 178, 23, 80, - 187, 112, 83, 244, 1, 132, 2, 7, 72, 65, 78, 71, 85, 76, 32, 216, 4, 8, - 75, 65, 84, 65, 75, 65, 78, 65, 204, 3, 3, 76, 69, 70, 0, 4, 82, 73, 71, - 72, 58, 85, 228, 159, 6, 11, 70, 79, 82, 77, 83, 32, 76, 73, 71, 72, 84, - 194, 207, 6, 73, 174, 170, 4, 66, 190, 185, 4, 87, 167, 26, 68, 104, 52, - 7, 76, 69, 84, 84, 69, 82, 32, 255, 181, 10, 70, 102, 206, 1, 75, 28, 5, - 78, 73, 69, 85, 78, 42, 80, 24, 5, 82, 73, 69, 85, 76, 86, 83, 98, 89, - 222, 61, 67, 54, 69, 30, 73, 242, 4, 77, 138, 1, 84, 206, 3, 87, 198, 1, - 72, 214, 208, 23, 65, 2, 79, 151, 64, 85, 6, 242, 65, 72, 155, 3, 73, 7, - 11, 45, 4, 154, 72, 67, 131, 3, 72, 6, 166, 69, 72, 35, 73, 17, 11, 45, - 14, 226, 49, 84, 226, 14, 80, 130, 4, 77, 194, 3, 75, 218, 2, 72, 99, 83, - 12, 40, 4, 83, 65, 78, 71, 215, 178, 13, 73, 10, 230, 70, 67, 42, 75, 74, - 80, 34, 84, 211, 2, 83, 14, 154, 155, 22, 69, 238, 254, 1, 65, 150, 64, - 73, 2, 79, 3, 85, 118, 70, 32, 149, 238, 2, 11, 45, 72, 73, 82, 65, 71, - 65, 78, 65, 32, 80, 116, 76, 7, 76, 69, 84, 84, 69, 82, 32, 234, 237, 2, - 83, 34, 86, 147, 217, 20, 77, 110, 146, 1, 83, 222, 231, 2, 78, 150, 2, - 72, 2, 75, 2, 77, 2, 82, 2, 84, 170, 1, 89, 210, 41, 87, 202, 194, 21, - 65, 2, 69, 2, 73, 2, 79, 3, 85, 28, 76, 5, 77, 65, 76, 76, 32, 226, 214, - 24, 65, 2, 69, 2, 73, 2, 79, 3, 85, 18, 198, 234, 2, 89, 178, 199, 21, - 84, 234, 36, 65, 2, 69, 2, 73, 2, 79, 3, 85, 4, 11, 84, 4, 236, 243, 12, - 2, 32, 67, 231, 194, 10, 87, 2, 155, 182, 23, 80, 14, 56, 3, 77, 69, 82, - 106, 83, 173, 156, 21, 3, 66, 85, 82, 9, 29, 5, 32, 65, 78, 68, 32, 6, - 172, 128, 12, 3, 87, 82, 69, 180, 213, 3, 2, 83, 73, 175, 208, 7, 80, 4, - 136, 151, 24, 3, 84, 69, 82, 151, 61, 65, 194, 8, 118, 68, 202, 2, 71, - 128, 66, 13, 73, 70, 73, 32, 82, 79, 72, 73, 78, 71, 89, 65, 32, 165, 5, - 5, 85, 78, 79, 79, 32, 10, 100, 12, 32, 87, 73, 84, 72, 32, 73, 78, 68, - 69, 88, 32, 188, 1, 2, 66, 65, 217, 138, 21, 2, 83, 72, 4, 156, 1, 18, - 65, 78, 68, 32, 77, 73, 68, 68, 76, 69, 32, 70, 73, 78, 71, 69, 82, 83, - 1, 16, 70, 73, 78, 71, 69, 82, 32, 65, 78, 68, 32, 84, 72, 85, 77, 66, 2, - 145, 206, 15, 2, 32, 67, 4, 182, 199, 23, 76, 159, 137, 1, 71, 170, 7, - 84, 3, 85, 76, 32, 217, 64, 13, 90, 72, 79, 85, 32, 78, 85, 77, 69, 82, - 65, 76, 32, 146, 7, 164, 1, 9, 67, 72, 79, 83, 69, 79, 78, 71, 32, 244, - 15, 4, 68, 79, 85, 66, 0, 4, 83, 73, 78, 71, 46, 74, 180, 31, 7, 76, 69, - 84, 84, 69, 82, 32, 215, 246, 9, 70, 250, 1, 246, 1, 67, 172, 2, 5, 73, - 69, 85, 78, 71, 146, 1, 75, 132, 1, 5, 77, 73, 69, 85, 77, 56, 5, 78, 73, - 69, 85, 78, 74, 80, 172, 2, 5, 82, 73, 69, 85, 76, 210, 1, 83, 166, 3, - 84, 124, 2, 89, 69, 200, 40, 5, 72, 73, 69, 85, 72, 139, 238, 9, 70, 30, - 76, 2, 72, 73, 84, 7, 69, 79, 78, 71, 67, 72, 73, 121, 4, 73, 69, 85, 67, - 16, 40, 4, 69, 85, 67, 72, 41, 2, 84, 85, 7, 11, 45, 4, 202, 31, 75, 243, - 26, 72, 10, 21, 3, 69, 85, 77, 10, 22, 83, 155, 46, 67, 6, 40, 4, 83, 65, - 78, 71, 179, 162, 13, 73, 4, 194, 54, 67, 227, 3, 83, 5, 215, 30, 45, 27, - 11, 45, 24, 90, 80, 234, 30, 82, 242, 13, 67, 194, 5, 77, 138, 1, 84, - 186, 2, 75, 218, 2, 72, 99, 83, 6, 214, 50, 72, 214, 3, 73, 207, 2, 65, - 16, 80, 7, 65, 80, 89, 69, 79, 85, 78, 228, 20, 5, 73, 89, 69, 79, 75, - 131, 25, 72, 10, 234, 29, 82, 178, 15, 80, 30, 83, 231, 3, 77, 11, 11, - 45, 8, 158, 52, 75, 74, 80, 34, 84, 211, 2, 83, 15, 11, 45, 12, 190, 51, - 67, 42, 75, 74, 80, 34, 84, 242, 1, 72, 99, 83, 42, 68, 6, 72, 73, 69, - 85, 80, 72, 40, 4, 73, 69, 85, 80, 223, 53, 65, 7, 11, 45, 4, 158, 51, - 80, 147, 2, 72, 35, 11, 45, 32, 82, 83, 234, 20, 80, 214, 7, 75, 162, 12, - 67, 202, 6, 84, 226, 2, 78, 179, 2, 72, 14, 32, 3, 73, 79, 83, 251, 3, - 83, 13, 11, 45, 10, 242, 46, 84, 146, 2, 67, 42, 75, 75, 80, 29, 11, 45, - 26, 78, 75, 42, 83, 190, 44, 77, 154, 3, 67, 82, 78, 34, 80, 34, 84, 243, - 1, 72, 6, 170, 22, 65, 130, 19, 72, 135, 7, 73, 8, 40, 4, 83, 65, 78, 71, - 151, 155, 13, 73, 6, 206, 47, 75, 74, 80, 35, 84, 58, 48, 3, 73, 79, 83, - 217, 1, 4, 83, 65, 78, 71, 33, 11, 45, 30, 130, 1, 80, 44, 2, 83, 83, - 210, 22, 82, 210, 1, 75, 162, 12, 67, 194, 5, 77, 138, 1, 84, 226, 2, 78, - 178, 2, 72, 223, 223, 17, 73, 6, 184, 23, 4, 73, 69, 85, 80, 179, 19, 72, - 2, 233, 48, 3, 65, 78, 71, 26, 236, 17, 5, 67, 73, 69, 85, 67, 172, 3, 4, - 83, 73, 79, 83, 154, 1, 82, 186, 20, 84, 54, 89, 134, 2, 75, 42, 78, 34, - 80, 146, 2, 72, 223, 223, 17, 73, 16, 40, 5, 73, 75, 69, 85, 84, 195, 41, - 72, 15, 11, 45, 12, 226, 20, 82, 178, 19, 77, 154, 3, 67, 42, 75, 74, 80, - 243, 2, 83, 4, 150, 19, 83, 139, 22, 79, 2, 165, 211, 8, 6, 76, 69, 32, - 68, 79, 84, 216, 3, 92, 9, 79, 78, 71, 83, 69, 79, 78, 71, 32, 165, 21, - 9, 85, 78, 71, 83, 69, 79, 78, 71, 32, 154, 2, 226, 1, 67, 80, 5, 72, 73, - 69, 85, 72, 60, 5, 73, 69, 85, 78, 71, 46, 75, 220, 1, 5, 77, 73, 69, 85, - 77, 188, 1, 5, 78, 73, 69, 85, 78, 94, 80, 240, 2, 5, 82, 73, 69, 85, 76, - 190, 4, 83, 194, 3, 84, 213, 1, 2, 89, 69, 8, 36, 4, 73, 69, 85, 67, 239, - 30, 72, 7, 11, 45, 4, 162, 32, 83, 239, 7, 80, 11, 11, 45, 8, 174, 16, - 82, 178, 19, 77, 234, 3, 78, 35, 80, 9, 11, 45, 6, 194, 17, 75, 57, 2, - 83, 83, 28, 76, 7, 65, 80, 89, 69, 79, 85, 78, 40, 5, 73, 89, 69, 79, 75, - 215, 30, 72, 8, 130, 15, 82, 178, 15, 80, 131, 4, 77, 19, 11, 45, 16, - 166, 13, 75, 170, 1, 82, 42, 83, 224, 13, 2, 67, 72, 146, 9, 78, 34, 80, - 147, 2, 72, 27, 11, 45, 24, 74, 80, 30, 83, 134, 13, 82, 242, 13, 67, - 130, 9, 75, 42, 78, 179, 2, 72, 6, 174, 33, 73, 131, 6, 65, 6, 40, 4, 83, - 65, 78, 71, 143, 143, 13, 73, 4, 238, 35, 78, 147, 3, 83, 21, 11, 45, 18, - 174, 12, 82, 242, 13, 67, 202, 6, 84, 186, 2, 75, 218, 2, 72, 62, 80, 39, - 83, 36, 88, 6, 65, 78, 83, 73, 79, 83, 48, 6, 72, 73, 69, 85, 80, 72, 53, - 4, 73, 69, 85, 80, 7, 11, 45, 4, 236, 7, 2, 75, 65, 195, 26, 80, 9, 11, - 45, 6, 150, 11, 84, 234, 22, 80, 243, 2, 83, 23, 11, 45, 20, 112, 5, 82, - 73, 69, 85, 76, 24, 4, 83, 73, 79, 83, 134, 3, 80, 246, 19, 67, 194, 5, - 77, 170, 4, 84, 243, 1, 72, 5, 153, 3, 2, 45, 80, 5, 221, 32, 2, 45, 84, - 57, 11, 45, 54, 102, 75, 92, 5, 77, 73, 69, 85, 77, 50, 80, 126, 83, 74, - 84, 44, 2, 89, 69, 154, 28, 78, 179, 2, 72, 10, 52, 5, 73, 89, 69, 79, - 75, 190, 4, 65, 131, 19, 72, 7, 11, 45, 4, 254, 32, 72, 99, 83, 9, 11, - 45, 6, 130, 30, 75, 218, 2, 72, 99, 83, 14, 48, 4, 73, 69, 85, 80, 174, - 26, 72, 163, 6, 65, 11, 11, 45, 8, 42, 80, 222, 29, 84, 242, 1, 72, 99, - 83, 2, 243, 25, 72, 6, 40, 4, 83, 65, 78, 71, 255, 135, 13, 73, 4, 182, - 28, 75, 187, 3, 83, 6, 100, 5, 73, 75, 69, 85, 84, 151, 25, 72, 6, 56, 9, - 79, 82, 73, 78, 72, 73, 69, 85, 72, 191, 3, 83, 5, 255, 29, 45, 52, 48, - 3, 73, 79, 83, 161, 1, 4, 83, 65, 78, 71, 25, 11, 45, 22, 82, 75, 162, 3, - 82, 242, 13, 67, 198, 2, 80, 254, 2, 77, 138, 1, 84, 147, 5, 72, 4, 22, - 65, 135, 26, 73, 2, 237, 18, 6, 80, 89, 69, 79, 85, 78, 28, 160, 1, 5, - 82, 73, 69, 85, 76, 36, 6, 84, 73, 75, 69, 85, 84, 16, 3, 89, 69, 83, - 138, 19, 83, 178, 1, 77, 154, 3, 67, 42, 75, 42, 78, 34, 80, 239, 225, - 17, 73, 5, 17, 2, 45, 75, 2, 159, 17, 72, 5, 255, 16, 45, 2, 171, 250, - 17, 73, 20, 40, 5, 73, 75, 69, 85, 84, 155, 21, 72, 19, 11, 45, 16, 58, - 82, 42, 83, 42, 84, 162, 13, 67, 130, 9, 75, 75, 80, 2, 17, 2, 73, 69, 2, - 255, 160, 23, 85, 4, 21, 3, 73, 79, 83, 5, 223, 1, 45, 2, 255, 19, 72, - 18, 44, 6, 83, 73, 69, 85, 78, 71, 243, 19, 79, 17, 11, 45, 14, 50, 75, - 30, 83, 198, 17, 77, 154, 6, 72, 63, 80, 4, 166, 14, 72, 135, 7, 73, 4, - 26, 83, 175, 128, 13, 73, 2, 21, 3, 65, 78, 71, 2, 207, 20, 75, 190, 1, - 122, 65, 118, 69, 134, 1, 73, 92, 7, 83, 83, 65, 78, 71, 65, 82, 106, 79, - 138, 1, 85, 102, 89, 174, 15, 87, 175, 234, 9, 70, 23, 48, 4, 82, 65, 69, - 65, 98, 45, 239, 165, 24, 69, 13, 11, 45, 10, 190, 231, 21, 69, 130, 191, - 2, 65, 2, 73, 3, 85, 25, 18, 79, 55, 85, 9, 11, 45, 6, 130, 129, 24, 69, - 234, 36, 79, 3, 85, 15, 11, 45, 12, 170, 9, 69, 142, 156, 24, 65, 2, 79, - 3, 85, 31, 11, 45, 28, 66, 65, 34, 89, 166, 1, 79, 142, 254, 23, 69, 234, - 36, 73, 3, 85, 5, 11, 82, 2, 235, 212, 17, 65, 14, 50, 65, 230, 228, 21, - 69, 130, 191, 2, 79, 3, 85, 7, 238, 132, 24, 45, 247, 30, 69, 23, 26, 45, - 171, 163, 24, 69, 18, 50, 79, 22, 89, 226, 227, 21, 69, 131, 191, 2, 85, - 5, 155, 143, 24, 45, 8, 222, 227, 21, 69, 239, 254, 1, 65, 17, 11, 45, - 14, 158, 19, 89, 234, 136, 5, 73, 192, 170, 4, 3, 69, 79, 45, 214, 155, - 14, 65, 151, 64, 85, 62, 42, 65, 70, 69, 66, 73, 22, 79, 107, 85, 11, 26, - 45, 147, 161, 24, 69, 6, 154, 130, 24, 89, 246, 30, 79, 3, 85, 11, 11, - 79, 9, 11, 45, 6, 150, 158, 24, 89, 186, 2, 79, 3, 85, 5, 191, 251, 23, - 45, 19, 11, 45, 16, 58, 89, 186, 223, 23, 65, 162, 33, 69, 246, 30, 73, - 3, 79, 6, 182, 223, 23, 65, 163, 33, 69, 21, 11, 45, 18, 142, 16, 89, - 146, 208, 21, 69, 238, 254, 1, 65, 150, 64, 73, 2, 79, 3, 85, 186, 1, - 226, 1, 65, 46, 67, 54, 69, 30, 73, 22, 75, 188, 1, 5, 77, 73, 69, 85, - 77, 64, 5, 78, 73, 69, 85, 78, 70, 80, 168, 1, 5, 82, 73, 69, 85, 76, - 254, 1, 84, 90, 83, 246, 2, 87, 50, 89, 150, 1, 72, 214, 208, 23, 79, - 151, 64, 85, 9, 156, 13, 3, 82, 65, 69, 207, 143, 24, 69, 4, 22, 72, 207, - 8, 73, 2, 241, 249, 23, 2, 73, 69, 7, 138, 156, 24, 79, 3, 85, 5, 239, - 234, 17, 69, 14, 56, 7, 65, 80, 89, 69, 79, 85, 78, 106, 72, 155, 3, 73, - 8, 30, 80, 30, 83, 231, 3, 77, 4, 190, 4, 72, 215, 3, 73, 2, 25, 4, 83, - 65, 78, 71, 2, 207, 7, 80, 2, 253, 59, 2, 73, 69, 9, 11, 45, 6, 22, 80, - 247, 9, 83, 4, 142, 7, 73, 207, 2, 65, 13, 11, 45, 10, 234, 5, 67, 146, - 1, 84, 242, 1, 72, 62, 80, 39, 83, 20, 48, 4, 73, 69, 85, 80, 170, 2, 72, - 163, 6, 65, 17, 11, 45, 14, 42, 83, 186, 2, 84, 146, 2, 67, 43, 75, 6, - 21, 3, 73, 79, 83, 7, 11, 45, 4, 202, 4, 75, 107, 84, 27, 11, 45, 24, 68, - 2, 75, 73, 34, 77, 34, 80, 106, 84, 54, 89, 222, 4, 72, 99, 83, 4, 149, - 1, 4, 89, 69, 79, 75, 2, 11, 73, 2, 255, 238, 22, 69, 8, 30, 72, 34, 73, - 131, 6, 65, 2, 245, 156, 18, 3, 73, 69, 85, 4, 21, 3, 69, 85, 80, 5, 243, - 5, 45, 4, 22, 72, 151, 3, 73, 2, 229, 167, 19, 2, 73, 69, 2, 17, 2, 69, - 79, 2, 167, 4, 82, 28, 44, 3, 73, 79, 83, 57, 4, 83, 65, 78, 71, 13, 11, - 45, 10, 122, 67, 42, 75, 42, 78, 34, 80, 35, 84, 16, 78, 67, 42, 75, 42, - 78, 34, 80, 34, 84, 242, 1, 72, 98, 83, 255, 222, 17, 73, 2, 11, 73, 2, - 249, 236, 22, 2, 69, 85, 2, 11, 73, 2, 209, 233, 23, 2, 89, 69, 2, 11, - 73, 2, 243, 146, 23, 69, 2, 11, 73, 2, 135, 157, 23, 69, 2, 11, 73, 2, - 11, 75, 2, 255, 152, 23, 69, 10, 170, 211, 21, 69, 238, 254, 1, 65, 151, - 64, 73, 34, 58, 69, 206, 1, 79, 62, 85, 166, 207, 23, 65, 151, 64, 73, - 13, 42, 79, 73, 6, 83, 73, 69, 85, 78, 71, 5, 11, 82, 2, 17, 2, 73, 78, - 2, 11, 72, 2, 157, 200, 9, 2, 73, 69, 7, 11, 45, 4, 18, 80, 39, 83, 2, - 11, 65, 2, 11, 78, 2, 11, 83, 2, 139, 232, 12, 73, 9, 11, 45, 6, 26, 89, - 207, 143, 24, 73, 4, 183, 207, 23, 65, 9, 11, 45, 6, 26, 89, 147, 143, - 24, 73, 4, 143, 208, 21, 69, 24, 178, 223, 10, 84, 170, 194, 11, 70, 30, - 83, 230, 83, 78, 14, 79, 199, 110, 69, 100, 156, 1, 7, 76, 69, 84, 84, - 69, 82, 32, 196, 2, 5, 77, 65, 82, 75, 32, 72, 5, 83, 73, 71, 78, 32, - 196, 156, 2, 6, 86, 79, 87, 69, 76, 32, 167, 255, 19, 68, 58, 202, 1, 68, - 34, 75, 222, 249, 18, 84, 174, 113, 82, 246, 168, 2, 78, 246, 175, 1, 83, - 254, 68, 66, 2, 67, 2, 70, 2, 71, 2, 72, 2, 74, 2, 76, 2, 77, 2, 80, 2, - 86, 2, 87, 2, 89, 2, 90, 187, 2, 65, 4, 138, 137, 24, 68, 187, 2, 65, 8, - 56, 5, 73, 78, 78, 65, 32, 178, 136, 24, 72, 187, 2, 65, 4, 174, 136, 24, - 87, 3, 89, 4, 208, 232, 20, 6, 78, 65, 32, 75, 72, 79, 189, 162, 2, 3, - 83, 65, 75, 8, 52, 2, 84, 65, 209, 255, 21, 5, 72, 65, 82, 66, 65, 6, 42, - 72, 142, 203, 19, 83, 223, 187, 4, 78, 2, 135, 231, 23, 65, 42, 62, 76, - 172, 167, 18, 6, 83, 73, 71, 78, 32, 80, 139, 2, 86, 36, 33, 6, 69, 84, - 84, 69, 82, 32, 36, 158, 185, 20, 78, 210, 204, 3, 66, 2, 68, 2, 71, 2, - 72, 2, 75, 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, 89, 186, - 2, 65, 2, 73, 3, 85, 2, 135, 249, 22, 69, 54, 52, 5, 67, 72, 73, 78, 71, - 41, 4, 82, 65, 78, 32, 2, 17, 2, 32, 67, 2, 207, 215, 22, 72, 52, 52, 7, - 76, 69, 84, 84, 69, 82, 32, 211, 145, 6, 78, 42, 218, 1, 65, 226, 132, - 11, 90, 210, 46, 84, 150, 119, 76, 50, 81, 60, 6, 68, 65, 76, 69, 84, 72, - 198, 150, 4, 71, 122, 83, 66, 89, 154, 193, 1, 72, 234, 5, 75, 130, 76, - 66, 190, 173, 4, 78, 254, 1, 87, 202, 103, 80, 171, 4, 77, 4, 222, 193, - 16, 76, 207, 242, 6, 89, 170, 9, 230, 1, 65, 184, 32, 5, 66, 82, 69, 87, - 32, 230, 32, 76, 200, 1, 16, 78, 84, 65, 73, 71, 65, 78, 65, 32, 76, 69, - 84, 84, 69, 82, 32, 174, 12, 82, 120, 11, 88, 65, 71, 82, 65, 77, 32, 70, - 79, 82, 32, 137, 177, 23, 4, 68, 71, 69, 72, 212, 1, 42, 68, 98, 82, 201, - 1, 3, 86, 89, 32, 6, 44, 5, 83, 84, 79, 78, 69, 223, 215, 22, 80, 5, 205, - 219, 2, 7, 32, 71, 82, 65, 86, 69, 89, 12, 32, 2, 84, 32, 143, 222, 16, - 45, 10, 60, 5, 87, 73, 84, 72, 32, 210, 250, 11, 68, 139, 220, 8, 72, 6, - 76, 9, 84, 73, 80, 32, 79, 78, 32, 84, 72, 130, 200, 12, 82, 139, 157, - 10, 65, 2, 243, 204, 23, 69, 194, 1, 134, 2, 65, 202, 1, 66, 230, 2, 67, - 154, 3, 68, 162, 1, 69, 186, 3, 70, 94, 72, 62, 76, 222, 1, 77, 110, 79, - 162, 1, 82, 142, 2, 83, 228, 1, 3, 78, 79, 82, 198, 1, 84, 128, 2, 2, 85, - 80, 174, 1, 87, 146, 215, 13, 73, 146, 223, 3, 80, 242, 220, 3, 86, 215, - 74, 71, 12, 108, 17, 82, 82, 79, 87, 32, 83, 72, 65, 70, 84, 32, 87, 73, - 68, 84, 72, 32, 154, 217, 17, 77, 215, 146, 4, 83, 8, 36, 3, 79, 78, 69, - 235, 158, 22, 84, 7, 11, 32, 4, 182, 185, 13, 84, 151, 194, 8, 72, 14, - 48, 4, 65, 76, 76, 79, 21, 4, 76, 65, 67, 75, 2, 143, 140, 5, 84, 12, 30, - 32, 153, 1, 2, 45, 70, 6, 52, 7, 67, 85, 82, 86, 69, 68, 32, 151, 243, - 22, 72, 4, 40, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 245, 217, 22, 8, 87, - 65, 82, 68, 83, 32, 65, 78, 6, 45, 9, 69, 65, 84, 72, 69, 82, 69, 68, 32, - 6, 194, 159, 13, 83, 210, 164, 3, 78, 163, 149, 6, 82, 14, 116, 2, 72, - 69, 52, 5, 73, 82, 67, 76, 69, 245, 159, 17, 14, 79, 78, 67, 65, 86, 69, - 45, 80, 79, 73, 78, 84, 69, 68, 4, 156, 134, 20, 4, 86, 82, 79, 78, 247, - 176, 2, 67, 9, 64, 6, 32, 87, 73, 84, 72, 32, 165, 229, 21, 4, 68, 32, - 83, 65, 4, 52, 7, 83, 84, 82, 79, 75, 69, 32, 159, 227, 4, 67, 2, 29, 5, - 65, 78, 68, 32, 84, 2, 11, 87, 2, 11, 79, 2, 21, 3, 32, 68, 79, 2, 11, - 84, 2, 231, 248, 22, 83, 14, 52, 7, 65, 83, 72, 69, 68, 32, 84, 18, 73, - 31, 79, 2, 175, 16, 82, 2, 157, 201, 18, 2, 86, 73, 10, 192, 16, 3, 87, - 78, 87, 206, 214, 13, 85, 135, 162, 4, 76, 16, 120, 5, 73, 71, 72, 84, - 32, 156, 2, 16, 88, 67, 76, 65, 77, 65, 84, 73, 79, 78, 32, 77, 65, 82, - 75, 32, 231, 133, 18, 81, 10, 40, 2, 80, 79, 126, 84, 135, 225, 21, 83, - 6, 33, 6, 73, 78, 84, 69, 68, 32, 6, 174, 130, 16, 80, 128, 221, 5, 11, - 82, 69, 67, 84, 73, 76, 73, 78, 69, 65, 82, 23, 66, 2, 173, 225, 21, 24, - 69, 65, 82, 68, 82, 79, 80, 45, 83, 80, 79, 75, 69, 68, 32, 80, 82, 79, - 80, 69, 76, 76, 69, 82, 4, 150, 222, 22, 83, 187, 79, 79, 6, 44, 5, 79, - 85, 82, 32, 66, 227, 226, 4, 73, 2, 225, 218, 10, 6, 65, 76, 76, 79, 79, - 78, 4, 210, 254, 16, 79, 157, 170, 5, 6, 69, 65, 82, 84, 32, 69, 20, 74, - 65, 44, 3, 69, 70, 84, 28, 2, 79, 87, 237, 221, 4, 3, 73, 71, 65, 4, 212, - 236, 20, 2, 82, 71, 167, 196, 1, 84, 8, 254, 3, 45, 195, 6, 87, 6, 26, - 32, 179, 236, 9, 69, 4, 162, 225, 13, 68, 25, 4, 83, 73, 78, 71, 4, 56, - 8, 85, 76, 84, 73, 80, 76, 73, 67, 227, 211, 20, 73, 2, 25, 4, 65, 84, - 73, 79, 2, 163, 254, 4, 78, 6, 172, 234, 5, 9, 80, 69, 78, 32, 67, 69, - 78, 84, 82, 204, 152, 10, 13, 86, 65, 76, 32, 87, 73, 84, 72, 32, 79, 86, - 65, 76, 213, 214, 5, 5, 85, 84, 76, 73, 78, 12, 76, 4, 73, 71, 72, 84, - 149, 203, 22, 9, 79, 85, 78, 68, 45, 84, 73, 80, 80, 10, 62, 45, 109, 11, - 87, 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 4, 69, 15, 80, 79, 73, 78, - 84, 73, 78, 71, 32, 65, 78, 71, 76, 69, 32, 4, 130, 196, 6, 66, 247, 153, - 7, 81, 7, 139, 6, 32, 26, 106, 65, 78, 73, 44, 2, 79, 85, 168, 229, 13, - 7, 67, 82, 73, 80, 84, 32, 76, 225, 131, 1, 3, 80, 65, 82, 4, 244, 220, - 13, 10, 78, 83, 45, 83, 69, 82, 73, 70, 32, 73, 195, 250, 7, 76, 8, 224, - 218, 13, 2, 78, 71, 203, 249, 7, 88, 10, 21, 3, 84, 72, 32, 10, 60, 5, - 69, 65, 83, 84, 32, 29, 6, 87, 69, 83, 84, 32, 80, 6, 26, 80, 235, 204, - 22, 65, 4, 41, 8, 79, 73, 78, 84, 73, 78, 71, 32, 4, 178, 181, 16, 86, - 239, 134, 6, 66, 12, 88, 9, 69, 65, 82, 68, 82, 79, 80, 45, 83, 130, 1, - 82, 169, 192, 6, 4, 87, 69, 76, 86, 6, 64, 6, 80, 79, 75, 69, 68, 32, - 145, 197, 22, 4, 72, 65, 78, 75, 4, 196, 212, 21, 8, 80, 73, 78, 87, 72, - 69, 69, 76, 27, 65, 2, 137, 3, 5, 73, 65, 78, 71, 76, 6, 26, 87, 159, - 226, 9, 80, 4, 53, 11, 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 4, 29, - 5, 87, 73, 84, 72, 32, 4, 204, 181, 19, 5, 76, 65, 82, 71, 69, 187, 200, - 1, 69, 10, 72, 5, 72, 73, 84, 69, 32, 112, 2, 73, 68, 177, 254, 21, 3, - 69, 68, 71, 6, 56, 5, 68, 79, 87, 78, 45, 130, 147, 22, 67, 199, 46, 83, - 2, 165, 190, 22, 8, 80, 79, 73, 78, 84, 73, 78, 71, 2, 145, 153, 20, 2, - 69, 45, 140, 2, 112, 7, 65, 67, 67, 69, 78, 84, 32, 146, 8, 76, 164, 14, - 5, 77, 65, 82, 75, 32, 114, 80, 245, 188, 20, 2, 89, 79, 60, 132, 2, 9, - 65, 84, 78, 65, 72, 32, 72, 65, 70, 22, 68, 36, 3, 71, 69, 82, 74, 77, - 128, 1, 2, 80, 65, 28, 4, 69, 84, 78, 65, 20, 2, 81, 65, 58, 83, 58, 84, - 156, 1, 2, 89, 69, 78, 90, 152, 132, 8, 3, 82, 69, 86, 246, 179, 14, 79, - 221, 143, 1, 3, 73, 76, 85, 2, 255, 214, 17, 85, 4, 178, 186, 18, 69, - 175, 225, 4, 65, 6, 32, 3, 69, 83, 72, 195, 28, 83, 5, 237, 187, 6, 4, - 32, 77, 85, 81, 8, 88, 5, 69, 82, 75, 72, 65, 148, 164, 16, 2, 85, 78, - 145, 177, 1, 5, 65, 72, 65, 80, 65, 5, 149, 150, 19, 4, 32, 75, 69, 70, - 4, 26, 83, 191, 158, 23, 90, 2, 207, 183, 23, 72, 4, 196, 151, 23, 6, 82, - 78, 69, 89, 32, 80, 179, 35, 68, 4, 194, 24, 69, 241, 230, 21, 6, 72, 65, - 76, 83, 72, 69, 8, 38, 69, 181, 227, 21, 3, 73, 80, 69, 6, 48, 6, 76, 73, - 83, 72, 65, 32, 163, 183, 11, 86, 4, 136, 177, 21, 3, 81, 69, 84, 153, - 135, 2, 4, 71, 69, 68, 79, 4, 176, 171, 8, 10, 82, 65, 72, 32, 66, 69, - 78, 32, 89, 79, 167, 214, 2, 84, 8, 34, 65, 177, 160, 23, 2, 73, 78, 6, - 40, 4, 81, 69, 70, 32, 139, 222, 5, 82, 4, 194, 222, 10, 81, 197, 231, - 11, 3, 71, 65, 68, 150, 1, 76, 6, 69, 84, 84, 69, 82, 32, 201, 11, 8, 73, - 71, 65, 84, 85, 82, 69, 32, 140, 1, 134, 3, 65, 204, 1, 3, 66, 69, 84, 0, - 3, 75, 65, 70, 0, 2, 80, 69, 68, 6, 70, 73, 78, 65, 76, 32, 92, 2, 81, - 79, 16, 2, 72, 69, 52, 2, 78, 85, 0, 4, 90, 65, 89, 73, 18, 83, 48, 3, - 82, 69, 83, 170, 1, 84, 52, 4, 68, 65, 76, 69, 12, 5, 71, 73, 77, 69, 76, - 0, 5, 76, 65, 77, 69, 68, 0, 3, 77, 69, 77, 48, 3, 86, 65, 86, 80, 5, 87, - 73, 68, 69, 32, 153, 1, 3, 89, 79, 68, 14, 26, 76, 251, 212, 22, 89, 12, - 64, 2, 69, 70, 73, 10, 84, 69, 82, 78, 65, 84, 73, 86, 69, 32, 9, 33, 6, - 32, 87, 73, 84, 72, 32, 6, 250, 12, 77, 130, 1, 80, 35, 81, 4, 254, 145, - 16, 65, 187, 148, 1, 80, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 154, 15, - 82, 243, 172, 13, 68, 14, 88, 2, 75, 65, 236, 2, 2, 80, 69, 136, 152, 1, - 2, 84, 83, 218, 182, 21, 78, 239, 109, 77, 4, 235, 2, 70, 7, 236, 10, 5, - 32, 87, 73, 84, 72, 231, 198, 23, 84, 4, 167, 2, 78, 16, 44, 4, 65, 77, - 69, 75, 17, 3, 72, 73, 78, 4, 231, 1, 72, 13, 33, 6, 32, 87, 73, 84, 72, - 32, 10, 40, 6, 68, 65, 71, 69, 83, 72, 39, 83, 7, 33, 6, 32, 65, 78, 68, - 32, 83, 4, 210, 221, 5, 72, 167, 192, 2, 73, 12, 50, 69, 12, 2, 65, 86, - 1, 4, 83, 65, 68, 73, 4, 11, 84, 5, 161, 184, 13, 7, 32, 87, 73, 84, 72, - 32, 68, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 176, 251, 1, 2, 72, 79, 159, - 188, 11, 68, 16, 174, 2, 76, 250, 169, 8, 65, 246, 188, 2, 84, 158, 206, - 2, 82, 164, 186, 8, 2, 68, 65, 134, 91, 75, 198, 106, 72, 169, 4, 7, 70, - 73, 78, 65, 76, 32, 77, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 168, 7, 2, - 72, 73, 191, 174, 13, 68, 10, 72, 6, 65, 76, 69, 70, 32, 76, 29, 8, 89, - 73, 68, 68, 73, 83, 72, 32, 2, 133, 236, 18, 2, 65, 77, 8, 120, 7, 68, - 79, 85, 66, 76, 69, 32, 224, 4, 9, 89, 79, 68, 32, 89, 79, 68, 32, 80, - 237, 163, 20, 5, 86, 65, 86, 32, 89, 4, 210, 229, 10, 86, 251, 194, 9, - 89, 6, 80, 3, 76, 79, 87, 0, 3, 85, 80, 80, 153, 250, 21, 6, 77, 65, 83, - 79, 82, 65, 2, 185, 184, 22, 2, 69, 82, 50, 84, 5, 79, 73, 78, 84, 32, - 241, 5, 11, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 38, 220, 1, 9, - 68, 65, 71, 69, 83, 72, 32, 79, 82, 46, 72, 106, 80, 164, 1, 17, 74, 85, - 68, 69, 79, 45, 83, 80, 65, 78, 73, 83, 72, 32, 86, 65, 82, 22, 81, 86, - 82, 22, 83, 168, 174, 9, 2, 84, 83, 173, 207, 13, 3, 77, 69, 84, 2, 17, - 2, 32, 77, 2, 201, 1, 2, 65, 80, 12, 60, 5, 65, 84, 65, 70, 32, 106, 73, - 33, 4, 79, 76, 65, 77, 6, 38, 80, 34, 81, 165, 2, 2, 83, 69, 2, 11, 65, - 2, 251, 140, 16, 84, 2, 153, 177, 22, 3, 65, 77, 65, 2, 11, 82, 2, 207, - 177, 13, 73, 5, 145, 224, 10, 12, 32, 72, 65, 83, 69, 82, 32, 70, 79, 82, - 32, 86, 2, 195, 187, 23, 73, 6, 52, 5, 65, 77, 65, 84, 83, 133, 150, 11, - 2, 85, 66, 5, 193, 201, 10, 2, 32, 81, 2, 243, 191, 6, 65, 8, 34, 69, 22, - 72, 151, 145, 8, 73, 2, 179, 176, 22, 71, 4, 146, 145, 8, 73, 171, 187, - 7, 69, 12, 152, 1, 3, 71, 69, 82, 60, 3, 80, 65, 83, 20, 7, 83, 79, 70, - 32, 80, 65, 83, 224, 142, 21, 7, 78, 85, 78, 32, 72, 65, 70, 241, 176, 1, - 3, 77, 65, 81, 4, 26, 83, 131, 220, 21, 69, 2, 225, 190, 22, 3, 72, 65, - 89, 2, 199, 173, 13, 69, 2, 179, 173, 13, 85, 8, 114, 77, 248, 235, 11, - 15, 76, 83, 67, 72, 82, 69, 73, 66, 69, 82, 32, 80, 65, 85, 83, 221, 193, - 5, 3, 73, 67, 79, 4, 252, 188, 5, 12, 69, 84, 32, 87, 73, 84, 72, 32, 87, - 72, 73, 84, 223, 239, 16, 32, 188, 4, 164, 1, 2, 65, 45, 50, 72, 70, 75, - 230, 1, 77, 114, 78, 146, 2, 82, 66, 83, 154, 1, 84, 226, 1, 87, 62, 89, - 110, 69, 186, 206, 8, 85, 250, 148, 5, 79, 167, 178, 3, 73, 8, 234, 158, - 23, 87, 246, 30, 49, 2, 50, 3, 51, 72, 166, 6, 79, 182, 157, 8, 65, 222, - 201, 5, 85, 162, 115, 69, 3, 73, 74, 72, 2, 65, 45, 104, 2, 79, 45, 178, - 4, 73, 226, 3, 69, 211, 214, 14, 85, 24, 210, 193, 11, 49, 250, 227, 11, - 75, 214, 22, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 8, 222, - 167, 23, 75, 218, 19, 49, 2, 50, 3, 51, 54, 68, 2, 69, 45, 154, 7, 79, - 210, 214, 14, 65, 2, 73, 135, 191, 2, 85, 6, 134, 184, 23, 77, 186, 2, - 49, 3, 50, 68, 116, 2, 69, 45, 72, 2, 73, 45, 246, 2, 65, 246, 208, 8, - 79, 250, 148, 5, 85, 165, 160, 4, 6, 45, 77, 85, 45, 77, 79, 14, 170, - 154, 23, 75, 246, 30, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 16, 130, - 162, 23, 84, 214, 22, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 54, - 222, 4, 79, 2, 85, 210, 214, 14, 73, 134, 191, 2, 65, 3, 69, 68, 62, 65, - 2, 85, 226, 3, 73, 186, 206, 8, 69, 155, 136, 6, 79, 16, 11, 45, 16, 250, - 182, 23, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 64, 74, 69, - 20, 2, 79, 45, 72, 2, 85, 45, 178, 216, 14, 73, 135, 191, 2, 65, 18, 223, - 169, 18, 45, 14, 150, 179, 23, 82, 186, 2, 49, 2, 50, 2, 51, 2, 52, 2, - 53, 3, 54, 10, 146, 150, 23, 84, 246, 30, 49, 2, 50, 2, 51, 3, 52, 42, - 222, 207, 8, 65, 2, 73, 154, 136, 6, 79, 135, 191, 2, 69, 32, 40, 2, 65, - 45, 66, 79, 215, 149, 17, 85, 12, 242, 148, 23, 89, 246, 30, 49, 2, 50, - 2, 51, 2, 52, 3, 53, 12, 11, 45, 12, 154, 179, 23, 49, 2, 50, 2, 51, 2, - 52, 2, 53, 3, 54, 4, 176, 133, 8, 21, 77, 73, 84, 73, 65, 78, 32, 67, 79, - 78, 74, 85, 71, 65, 84, 69, 32, 77, 65, 84, 82, 175, 173, 15, 66, 128, 1, - 210, 2, 65, 110, 66, 144, 1, 2, 67, 79, 94, 68, 198, 2, 70, 66, 71, 40, - 4, 72, 79, 76, 68, 164, 1, 2, 73, 78, 112, 2, 77, 79, 58, 79, 122, 80, - 100, 2, 82, 69, 62, 83, 238, 1, 84, 210, 5, 87, 216, 227, 14, 4, 76, 73, - 77, 73, 184, 137, 5, 11, 89, 79, 85, 84, 72, 70, 85, 76, 32, 70, 79, 133, - 159, 3, 9, 69, 78, 84, 72, 85, 83, 73, 65, 83, 6, 76, 3, 80, 80, 82, 124, - 4, 70, 84, 69, 82, 245, 209, 18, 4, 66, 85, 78, 68, 2, 149, 140, 23, 2, - 79, 65, 6, 92, 5, 69, 70, 79, 82, 69, 192, 193, 3, 6, 73, 84, 73, 78, 71, - 32, 1, 4, 82, 69, 65, 75, 2, 253, 220, 22, 7, 32, 67, 79, 77, 80, 76, 69, - 6, 26, 78, 239, 200, 18, 77, 4, 196, 236, 19, 4, 84, 69, 77, 80, 217, - 177, 2, 3, 70, 76, 73, 14, 110, 69, 78, 73, 134, 205, 18, 85, 173, 200, - 2, 15, 65, 82, 75, 69, 78, 73, 78, 71, 32, 79, 70, 32, 84, 72, 69, 6, - 150, 201, 18, 67, 192, 6, 2, 76, 73, 201, 152, 4, 5, 86, 69, 76, 79, 80, - 4, 184, 191, 18, 21, 70, 70, 73, 67, 85, 76, 84, 89, 32, 65, 84, 32, 84, - 72, 69, 32, 66, 69, 71, 73, 78, 137, 210, 2, 4, 83, 80, 69, 82, 4, 148, - 196, 18, 2, 79, 76, 221, 144, 4, 5, 69, 76, 76, 79, 87, 12, 36, 5, 65, - 84, 72, 69, 82, 35, 82, 2, 213, 206, 14, 3, 73, 78, 71, 10, 40, 4, 69, - 65, 84, 32, 211, 235, 22, 65, 8, 22, 80, 207, 5, 84, 6, 22, 79, 143, 5, - 82, 4, 168, 2, 2, 83, 83, 135, 232, 22, 87, 8, 50, 78, 134, 197, 18, 67, - 217, 5, 3, 70, 76, 85, 4, 184, 185, 18, 5, 69, 82, 32, 84, 82, 161, 17, - 2, 79, 67, 4, 128, 189, 21, 3, 68, 69, 83, 221, 59, 3, 85, 84, 72, 6, 26, - 66, 37, 2, 80, 80, 2, 209, 213, 22, 4, 83, 84, 82, 85, 4, 26, 82, 171, - 212, 22, 79, 2, 169, 140, 21, 2, 69, 83, 6, 200, 211, 14, 9, 85, 83, 72, - 73, 78, 71, 32, 85, 80, 168, 134, 2, 3, 82, 79, 71, 219, 142, 6, 69, 6, - 26, 84, 235, 180, 18, 86, 4, 186, 198, 18, 85, 215, 57, 82, 8, 132, 1, 5, - 77, 65, 76, 76, 32, 204, 178, 20, 6, 84, 65, 78, 68, 83, 84, 209, 233, 1, - 11, 80, 76, 73, 84, 84, 73, 78, 71, 32, 65, 80, 4, 24, 2, 80, 82, 43, 84, - 2, 157, 199, 18, 5, 69, 80, 79, 78, 68, 2, 11, 65, 2, 215, 209, 22, 77, - 30, 36, 3, 72, 69, 32, 171, 148, 17, 82, 28, 186, 2, 65, 130, 1, 67, 132, - 1, 3, 70, 65, 77, 20, 9, 82, 69, 67, 69, 80, 84, 73, 86, 69, 30, 87, 156, - 21, 12, 77, 65, 82, 82, 89, 73, 78, 71, 32, 77, 65, 73, 232, 242, 4, 6, - 71, 69, 78, 84, 76, 69, 204, 199, 10, 13, 75, 69, 69, 80, 73, 78, 71, 32, - 83, 84, 73, 76, 76, 177, 220, 3, 7, 74, 79, 89, 79, 85, 83, 32, 6, 58, - 82, 141, 140, 11, 8, 66, 89, 83, 77, 65, 76, 32, 87, 4, 220, 217, 19, 8, - 79, 85, 83, 73, 78, 71, 32, 84, 247, 179, 3, 77, 6, 160, 129, 1, 2, 65, - 85, 152, 215, 18, 9, 82, 69, 65, 84, 73, 86, 69, 32, 72, 233, 180, 1, 9, - 76, 73, 78, 71, 73, 78, 71, 32, 70, 2, 163, 250, 22, 73, 2, 129, 215, 19, - 2, 32, 69, 4, 242, 152, 21, 69, 209, 198, 1, 5, 65, 78, 68, 69, 82, 4, - 174, 198, 6, 65, 217, 146, 6, 14, 79, 82, 75, 32, 79, 78, 32, 84, 72, 69, - 32, 68, 69, 67, 222, 1, 236, 1, 2, 71, 72, 180, 2, 7, 82, 65, 71, 65, 78, - 65, 32, 212, 4, 7, 83, 84, 79, 82, 73, 67, 32, 184, 130, 15, 7, 78, 68, - 85, 32, 84, 69, 77, 236, 33, 4, 75, 73, 78, 71, 150, 235, 1, 66, 241, - 128, 4, 8, 80, 80, 79, 80, 79, 84, 65, 77, 12, 18, 32, 119, 45, 6, 144, - 234, 4, 2, 66, 82, 156, 152, 16, 6, 86, 79, 76, 84, 65, 71, 149, 47, 9, - 79, 67, 84, 69, 84, 32, 80, 82, 69, 6, 92, 11, 83, 80, 69, 69, 68, 32, - 84, 82, 65, 73, 78, 233, 166, 5, 6, 72, 69, 69, 76, 69, 68, 5, 133, 145, - 11, 14, 32, 87, 73, 84, 72, 32, 66, 85, 76, 76, 69, 84, 32, 78, 200, 1, - 112, 9, 68, 73, 71, 82, 65, 80, 72, 32, 89, 20, 7, 76, 69, 84, 84, 69, - 82, 32, 234, 170, 1, 86, 215, 207, 19, 73, 2, 135, 142, 16, 79, 194, 1, - 194, 1, 65, 74, 83, 218, 161, 1, 66, 162, 3, 78, 150, 2, 68, 2, 71, 2, - 72, 2, 75, 2, 77, 2, 80, 2, 82, 2, 84, 2, 90, 126, 87, 46, 89, 178, 199, - 21, 86, 234, 36, 69, 2, 73, 2, 79, 3, 85, 7, 37, 7, 82, 67, 72, 65, 73, - 67, 32, 4, 130, 240, 22, 87, 151, 14, 89, 42, 76, 5, 77, 65, 76, 76, 32, - 254, 147, 23, 65, 2, 69, 2, 73, 2, 79, 3, 85, 32, 182, 167, 1, 87, 46, - 89, 130, 142, 5, 75, 178, 185, 16, 84, 234, 36, 65, 2, 69, 2, 73, 2, 79, - 3, 85, 2, 171, 145, 8, 83, 92, 166, 1, 76, 52, 3, 78, 69, 89, 46, 82, - 130, 6, 84, 138, 1, 85, 210, 130, 16, 83, 138, 203, 2, 67, 172, 149, 3, - 6, 77, 79, 84, 72, 69, 84, 214, 163, 1, 79, 155, 3, 80, 6, 180, 212, 15, - 4, 76, 79, 87, 32, 163, 189, 7, 69, 4, 238, 164, 21, 66, 137, 156, 1, 2, - 32, 80, 50, 60, 8, 73, 90, 79, 78, 84, 65, 76, 32, 137, 5, 2, 83, 69, 44, - 198, 1, 66, 108, 4, 76, 73, 78, 69, 28, 17, 79, 78, 69, 32, 69, 73, 71, + 80, 215, 212, 19, 77, 7, 11, 32, 4, 182, 155, 27, 69, 221, 6, 2, 83, 84, + 22, 32, 2, 48, 48, 203, 221, 15, 79, 20, 222, 188, 22, 50, 2, 54, 134, + 236, 5, 49, 2, 51, 2, 52, 2, 53, 2, 55, 3, 56, 164, 1, 118, 48, 128, 4, + 15, 79, 68, 73, 70, 73, 69, 82, 32, 68, 65, 77, 65, 71, 69, 68, 205, 228, + 24, 5, 73, 82, 82, 79, 82, 132, 1, 42, 48, 98, 49, 106, 50, 102, 51, 107, + 52, 24, 182, 40, 49, 170, 146, 22, 51, 134, 236, 5, 50, 2, 52, 2, 53, 2, + 54, 2, 55, 2, 56, 3, 57, 44, 138, 41, 50, 246, 144, 22, 48, 2, 53, 2, 54, + 2, 55, 134, 236, 5, 49, 2, 51, 2, 52, 2, 56, 3, 57, 26, 150, 185, 22, 50, + 2, 52, 2, 56, 134, 236, 5, 48, 2, 49, 2, 51, 2, 53, 2, 54, 2, 55, 3, 57, + 26, 138, 38, 51, 170, 146, 22, 49, 134, 236, 5, 48, 2, 50, 2, 52, 2, 53, + 2, 54, 2, 55, 2, 56, 3, 57, 12, 202, 183, 22, 48, 134, 236, 5, 49, 2, 50, + 2, 51, 3, 52, 31, 25, 4, 32, 65, 84, 32, 28, 96, 6, 66, 79, 84, 84, 79, + 77, 120, 5, 83, 84, 65, 82, 84, 68, 3, 84, 79, 80, 227, 146, 27, 69, 11, + 11, 32, 8, 56, 5, 83, 84, 65, 82, 84, 186, 1, 65, 159, 146, 27, 69, 5, + 129, 2, 8, 32, 65, 78, 68, 32, 84, 79, 80, 7, 29, 5, 32, 65, 78, 68, 32, + 4, 206, 181, 24, 66, 155, 242, 2, 84, 11, 11, 32, 8, 54, 65, 20, 5, 83, + 84, 65, 82, 84, 139, 146, 27, 69, 2, 73, 2, 78, 68, 5, 53, 11, 32, 65, + 78, 68, 32, 66, 79, 84, 84, 79, 77, 2, 251, 209, 19, 32, 194, 1, 50, 48, + 128, 2, 2, 76, 48, 229, 1, 2, 85, 48, 98, 58, 49, 98, 51, 194, 14, 50, + 150, 6, 52, 227, 209, 22, 48, 24, 146, 32, 56, 174, 254, 27, 48, 2, 49, + 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 28, 218, 177, 22, 51, 2, + 52, 2, 53, 2, 55, 134, 236, 5, 48, 2, 49, 2, 50, 2, 54, 2, 56, 3, 57, 44, + 34, 48, 94, 49, 175, 131, 21, 50, 20, 210, 176, 22, 53, 134, 236, 5, 49, + 2, 50, 2, 51, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 22, 246, 175, 22, 55, + 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, + 52, 34, 49, 102, 50, 231, 226, 22, 48, 26, 238, 174, 22, 48, 2, 49, 2, + 56, 134, 236, 5, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 8, 138, + 174, 22, 50, 134, 236, 5, 48, 3, 49, 152, 1, 50, 48, 145, 247, 18, 6, 86, + 69, 82, 76, 65, 89, 150, 1, 66, 48, 154, 1, 49, 138, 1, 50, 102, 51, 106, + 53, 191, 223, 22, 52, 34, 90, 54, 134, 172, 22, 49, 2, 53, 134, 236, 5, + 50, 2, 51, 2, 52, 2, 55, 2, 56, 3, 57, 15, 134, 152, 28, 65, 2, 66, 2, + 67, 2, 68, 2, 69, 3, 70, 28, 98, 48, 230, 170, 22, 57, 134, 236, 5, 49, + 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 9, 230, 150, 28, 65, 2, + 66, 3, 67, 28, 190, 170, 22, 48, 2, 52, 2, 53, 2, 57, 134, 236, 5, 49, 2, + 50, 2, 51, 2, 54, 2, 55, 3, 56, 32, 134, 23, 54, 214, 146, 22, 48, 2, 51, + 134, 236, 5, 49, 2, 50, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 8, 202, 22, + 48, 175, 254, 27, 49, 26, 26, 48, 143, 194, 8, 49, 22, 182, 168, 22, 49, + 2, 51, 134, 236, 5, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 68, 34, + 48, 98, 49, 163, 221, 22, 50, 24, 142, 21, 51, 170, 146, 22, 50, 134, + 236, 5, 49, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 24, 214, 166, 22, + 48, 2, 54, 134, 236, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, + 57, 108, 50, 48, 94, 49, 106, 50, 98, 51, 171, 172, 19, 52, 22, 190, 165, + 22, 50, 2, 54, 134, 236, 5, 49, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, + 26, 186, 18, 52, 170, 146, 22, 55, 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, + 2, 53, 2, 54, 2, 56, 3, 57, 24, 210, 17, 54, 174, 254, 27, 48, 2, 49, 2, + 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 22, 154, 163, 22, 53, 134, + 236, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 90, + 34, 48, 249, 12, 3, 65, 76, 76, 88, 42, 48, 94, 49, 102, 51, 243, 214, + 22, 50, 26, 230, 161, 22, 51, 2, 55, 2, 56, 2, 57, 134, 236, 5, 49, 2, + 50, 2, 52, 2, 53, 3, 54, 24, 138, 161, 22, 49, 2, 54, 134, 236, 5, 48, 2, + 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 18, 166, 160, 22, 50, 2, + 51, 134, 236, 5, 48, 2, 49, 2, 52, 2, 53, 3, 54, 94, 50, 48, 90, 50, 102, + 51, 102, 52, 211, 211, 22, 49, 22, 254, 12, 54, 174, 254, 27, 49, 2, 50, + 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 24, 206, 158, 22, 51, 2, 57, + 134, 236, 5, 48, 2, 49, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 22, + 234, 157, 22, 50, 134, 236, 5, 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 54, 2, + 55, 2, 56, 3, 57, 6, 138, 137, 28, 48, 2, 49, 3, 50, 158, 1, 42, 48, 185, + 4, 5, 69, 82, 84, 73, 67, 156, 1, 62, 48, 98, 49, 98, 50, 210, 1, 51, + 225, 152, 22, 2, 52, 48, 42, 198, 9, 55, 98, 49, 202, 145, 22, 50, 134, + 236, 5, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 32, 186, 8, 49, 46, 50, + 174, 254, 27, 48, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 50, + 98, 48, 206, 153, 22, 51, 2, 56, 2, 57, 134, 236, 5, 49, 2, 50, 2, 52, 2, + 53, 2, 54, 3, 55, 27, 206, 133, 28, 65, 2, 66, 2, 67, 2, 68, 2, 69, 2, + 70, 2, 71, 2, 72, 2, 73, 2, 74, 2, 75, 3, 76, 28, 222, 152, 22, 48, 2, + 49, 2, 51, 2, 55, 134, 236, 5, 50, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 2, + 221, 221, 23, 2, 65, 76, 66, 34, 48, 161, 2, 3, 73, 68, 69, 64, 26, 48, + 94, 49, 103, 50, 22, 158, 151, 22, 51, 2, 57, 134, 236, 5, 49, 2, 50, 2, + 52, 2, 53, 2, 54, 2, 55, 3, 56, 28, 194, 150, 22, 48, 2, 52, 2, 55, 2, + 56, 134, 236, 5, 49, 2, 50, 2, 51, 2, 53, 2, 54, 3, 57, 14, 222, 149, 22, + 52, 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, 3, 53, 2, 17, 2, 32, 76, 2, + 239, 181, 15, 79, 24, 202, 2, 52, 170, 146, 22, 54, 2, 56, 134, 236, 5, + 49, 2, 50, 2, 51, 2, 53, 3, 55, 18, 154, 148, 22, 49, 134, 236, 5, 50, 2, + 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 82, 22, 48, 167, 1, 49, 34, 90, + 50, 46, 51, 170, 146, 22, 52, 2, 53, 134, 236, 5, 49, 2, 54, 2, 55, 2, + 56, 3, 57, 11, 214, 254, 27, 65, 2, 66, 2, 67, 3, 68, 7, 170, 254, 27, + 65, 3, 66, 48, 66, 53, 86, 54, 250, 252, 27, 48, 2, 49, 2, 50, 2, 51, 3, + 52, 21, 202, 253, 27, 65, 2, 66, 2, 67, 2, 68, 2, 69, 2, 70, 2, 71, 2, + 72, 3, 73, 19, 246, 252, 27, 65, 2, 66, 2, 67, 2, 68, 2, 69, 2, 70, 2, + 71, 3, 72, 182, 62, 22, 51, 211, 1, 52, 192, 46, 106, 52, 174, 188, 5, + 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, 68, 2, 69, 3, 70, + 192, 2, 214, 179, 13, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, + 68, 2, 69, 3, 70, 246, 15, 42, 51, 158, 187, 5, 48, 2, 49, 3, 50, 246, 3, + 142, 1, 70, 186, 177, 13, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, + 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, 68, 3, 69, 22, 250, 248, 27, + 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 3, 65, + 18, 26, 32, 239, 226, 15, 72, 16, 70, 80, 124, 5, 82, 65, 89, 83, 32, + 190, 146, 3, 84, 231, 207, 22, 83, 8, 186, 146, 3, 79, 241, 185, 16, 22, + 69, 84, 65, 76, 76, 69, 68, 32, 79, 85, 84, 76, 73, 78, 69, 68, 32, 66, + 76, 65, 67, 75, 4, 228, 234, 18, 2, 73, 78, 1, 3, 79, 85, 84, 160, 1, + 132, 1, 13, 66, 65, 83, 65, 78, 32, 76, 69, 84, 84, 69, 82, 32, 166, 3, + 69, 176, 4, 7, 89, 77, 65, 73, 67, 32, 76, 171, 237, 27, 70, 80, 230, 1, + 71, 78, 76, 34, 78, 50, 82, 158, 232, 25, 69, 190, 137, 1, 67, 2, 68, 2, + 75, 2, 83, 2, 84, 2, 90, 206, 105, 66, 2, 70, 2, 72, 2, 74, 2, 77, 2, 80, + 2, 81, 2, 86, 2, 88, 214, 22, 65, 2, 73, 2, 79, 2, 85, 3, 89, 8, 38, 72, + 154, 220, 27, 74, 215, 22, 69, 4, 242, 160, 25, 65, 251, 209, 2, 69, 4, + 242, 219, 27, 76, 215, 22, 69, 8, 210, 219, 27, 68, 2, 74, 214, 22, 65, + 3, 69, 4, 162, 219, 27, 82, 215, 22, 69, 32, 96, 5, 67, 84, 82, 73, 67, + 160, 1, 7, 77, 69, 78, 84, 32, 79, 70, 238, 229, 26, 80, 235, 80, 86, 10, + 26, 32, 179, 130, 24, 65, 8, 98, 80, 144, 205, 17, 2, 84, 79, 128, 231, + 8, 9, 76, 73, 71, 72, 84, 32, 66, 85, 76, 171, 33, 65, 2, 11, 76, 2, 147, + 239, 27, 85, 19, 11, 32, 16, 72, 5, 87, 73, 84, 72, 32, 217, 207, 16, 7, + 79, 80, 69, 78, 73, 78, 71, 12, 130, 1, 76, 32, 12, 84, 87, 79, 32, 72, + 79, 82, 73, 90, 79, 78, 84, 142, 214, 19, 86, 194, 172, 4, 85, 234, 60, + 79, 219, 173, 2, 68, 2, 173, 201, 25, 3, 79, 78, 71, 2, 217, 213, 14, 7, + 65, 76, 32, 83, 84, 82, 79, 46, 254, 168, 4, 69, 129, 202, 15, 15, 73, + 71, 65, 84, 85, 82, 69, 32, 90, 65, 89, 73, 78, 45, 89, 53, 48, 4, 79, + 74, 73, 32, 218, 2, 80, 163, 3, 32, 18, 164, 1, 10, 67, 79, 77, 80, 79, + 78, 69, 78, 84, 32, 109, 26, 77, 79, 68, 73, 70, 73, 69, 82, 32, 70, 73, + 84, 90, 80, 65, 84, 82, 73, 67, 75, 32, 84, 89, 80, 69, 45, 8, 128, 255, + 14, 2, 82, 69, 12, 5, 67, 85, 82, 76, 89, 0, 5, 87, 72, 73, 84, 69, 213, + 205, 2, 2, 66, 65, 10, 208, 208, 20, 2, 49, 45, 174, 153, 7, 51, 2, 52, + 2, 53, 3, 54, 26, 44, 3, 84, 89, 32, 177, 244, 3, 2, 72, 65, 24, 82, 78, + 60, 3, 80, 65, 71, 20, 3, 83, 69, 84, 181, 164, 27, 4, 68, 79, 67, 85, 8, + 36, 3, 79, 84, 69, 151, 254, 23, 69, 7, 199, 152, 13, 32, 4, 139, 225, + 25, 69, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 130, 130, 14, 82, 24, 3, + 76, 69, 70, 212, 166, 1, 2, 83, 77, 179, 144, 9, 79, 38, 86, 32, 64, 2, + 68, 32, 214, 1, 81, 20, 6, 86, 69, 76, 79, 80, 69, 227, 220, 15, 84, 6, + 42, 81, 198, 253, 25, 68, 131, 170, 1, 83, 2, 195, 159, 27, 85, 20, 32, + 3, 79, 70, 32, 131, 1, 87, 18, 88, 3, 80, 82, 79, 242, 221, 20, 71, 56, + 2, 83, 69, 162, 71, 77, 30, 84, 195, 165, 5, 76, 4, 178, 222, 20, 84, + 135, 133, 6, 79, 2, 173, 239, 19, 7, 73, 84, 72, 32, 76, 69, 70, 5, 231, + 165, 21, 85, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 42, 76, 209, 233, 23, + 4, 68, 79, 87, 78, 2, 209, 203, 22, 4, 73, 71, 72, 84, 6, 254, 226, 27, + 76, 2, 77, 3, 84, 44, 28, 2, 65, 76, 231, 6, 73, 38, 30, 32, 133, 2, 2, + 83, 32, 12, 56, 3, 84, 79, 32, 177, 141, 13, 5, 65, 78, 68, 32, 80, 10, + 68, 3, 79, 82, 32, 221, 143, 27, 8, 66, 89, 32, 68, 69, 70, 73, 78, 8, + 64, 3, 80, 82, 69, 28, 3, 83, 85, 67, 162, 206, 25, 71, 39, 76, 2, 193, + 156, 15, 2, 67, 69, 2, 233, 128, 26, 3, 67, 69, 69, 26, 72, 4, 83, 73, + 71, 78, 224, 203, 25, 4, 87, 73, 84, 72, 235, 195, 1, 67, 23, 11, 32, 20, + 42, 65, 201, 1, 5, 87, 73, 84, 72, 32, 12, 112, 5, 66, 79, 86, 69, 32, + 65, 19, 78, 68, 32, 83, 76, 65, 78, 84, 69, 68, 32, 80, 65, 82, 65, 76, + 76, 69, 76, 8, 154, 132, 21, 80, 210, 5, 84, 182, 184, 2, 76, 139, 252, + 2, 82, 5, 243, 239, 6, 32, 8, 160, 1, 4, 66, 85, 77, 80, 20, 7, 73, 78, + 70, 73, 78, 73, 84, 20, 18, 84, 87, 79, 32, 68, 79, 84, 83, 32, 65, 66, + 79, 86, 69, 32, 65, 78, 68, 215, 156, 15, 68, 2, 163, 223, 26, 89, 2, + 207, 174, 25, 89, 2, 17, 2, 32, 84, 2, 243, 173, 25, 87, 6, 80, 7, 86, + 65, 76, 69, 78, 84, 32, 237, 135, 21, 7, 65, 78, 71, 85, 76, 65, 82, 4, + 48, 6, 87, 73, 84, 72, 32, 70, 223, 187, 27, 84, 2, 11, 79, 2, 201, 242, + 2, 2, 85, 82, 20, 152, 1, 11, 82, 79, 82, 45, 66, 65, 82, 82, 69, 68, 32, + 152, 163, 5, 7, 73, 83, 32, 70, 79, 82, 77, 193, 195, 7, 9, 65, 83, 69, + 32, 84, 79, 32, 84, 72, 12, 160, 220, 5, 4, 87, 72, 73, 84, 13, 5, 66, + 76, 65, 67, 75, 10, 58, 67, 20, 6, 84, 73, 77, 65, 84, 69, 255, 215, 27, + 65, 5, 211, 144, 26, 65, 4, 210, 196, 26, 68, 171, 147, 1, 83, 154, 8, + 60, 7, 72, 73, 79, 80, 73, 67, 32, 150, 215, 27, 66, 3, 88, 150, 8, 204, + 1, 2, 67, 79, 232, 1, 7, 78, 85, 77, 66, 69, 82, 32, 114, 80, 54, 83, + 156, 24, 11, 84, 79, 78, 65, 76, 32, 77, 65, 82, 75, 32, 226, 252, 18, + 68, 214, 235, 5, 70, 82, 81, 221, 146, 2, 4, 87, 79, 82, 68, 10, 26, 77, + 187, 133, 27, 76, 8, 52, 7, 66, 73, 78, 73, 78, 71, 32, 183, 210, 27, 77, + 6, 60, 11, 71, 69, 77, 73, 78, 65, 84, 73, 79, 78, 32, 51, 86, 4, 44, 5, + 65, 78, 68, 32, 86, 211, 146, 27, 77, 2, 201, 246, 23, 4, 79, 87, 69, 76, + 22, 66, 84, 226, 255, 21, 72, 222, 228, 3, 69, 30, 70, 42, 78, 39, 83, 8, + 194, 199, 16, 69, 170, 158, 9, 72, 27, 87, 4, 166, 113, 65, 153, 145, 26, + 5, 82, 69, 70, 65, 67, 198, 7, 50, 69, 37, 8, 89, 76, 76, 65, 66, 76, 69, + 32, 4, 202, 134, 24, 77, 139, 226, 2, 67, 194, 7, 210, 1, 66, 90, 67, + 246, 1, 68, 186, 1, 70, 90, 71, 214, 2, 72, 162, 1, 75, 102, 77, 90, 78, + 90, 80, 138, 2, 81, 174, 1, 82, 86, 83, 210, 1, 84, 122, 74, 2, 76, 138, + 1, 87, 2, 89, 66, 88, 134, 1, 90, 95, 86, 38, 194, 12, 87, 230, 8, 66, + 214, 206, 21, 65, 2, 79, 230, 171, 5, 69, 162, 64, 73, 3, 85, 78, 94, 67, + 254, 15, 72, 202, 210, 21, 65, 2, 79, 230, 171, 5, 69, 234, 61, 87, 186, + 2, 73, 3, 85, 42, 70, 72, 254, 225, 21, 65, 230, 171, 5, 69, 162, 64, 73, + 2, 79, 3, 85, 28, 166, 19, 72, 214, 206, 21, 65, 230, 171, 5, 69, 162, + 64, 73, 2, 79, 3, 85, 60, 94, 68, 214, 14, 90, 254, 209, 21, 65, 2, 79, + 230, 171, 5, 69, 234, 61, 87, 186, 2, 73, 3, 85, 30, 210, 14, 72, 254, + 209, 21, 65, 2, 79, 230, 171, 5, 69, 234, 61, 87, 186, 2, 73, 3, 85, 24, + 190, 8, 87, 186, 215, 21, 65, 230, 171, 5, 69, 234, 61, 89, 186, 2, 73, + 2, 79, 3, 85, 118, 142, 1, 85, 226, 7, 71, 232, 3, 7, 76, 79, 84, 84, 65, + 76, 32, 158, 2, 87, 218, 1, 89, 214, 206, 21, 65, 2, 79, 230, 171, 5, 69, + 163, 64, 73, 39, 29, 5, 82, 65, 71, 69, 32, 36, 74, 66, 2, 70, 2, 77, 2, + 80, 46, 71, 2, 75, 2, 81, 191, 130, 12, 72, 4, 11, 87, 4, 198, 178, 27, + 69, 215, 22, 73, 6, 11, 87, 6, 206, 136, 27, 69, 163, 64, 73, 52, 70, 72, + 134, 220, 21, 65, 2, 79, 230, 171, 5, 69, 162, 64, 73, 3, 85, 36, 202, 4, + 87, 230, 8, 89, 214, 206, 21, 65, 230, 171, 5, 69, 162, 64, 73, 2, 79, 3, + 85, 64, 250, 4, 88, 134, 6, 87, 218, 1, 89, 214, 206, 21, 65, 2, 79, 230, + 171, 5, 69, 162, 64, 73, 3, 85, 26, 142, 3, 87, 186, 215, 21, 65, 2, 79, + 230, 171, 5, 69, 234, 61, 89, 186, 2, 73, 3, 85, 36, 166, 7, 89, 202, + 210, 21, 65, 2, 79, 230, 171, 5, 69, 234, 61, 87, 186, 2, 73, 3, 85, 56, + 82, 72, 142, 1, 87, 186, 215, 21, 65, 2, 79, 230, 171, 5, 69, 162, 64, + 73, 3, 85, 32, 74, 65, 250, 215, 21, 79, 230, 171, 5, 69, 234, 61, 87, + 186, 2, 73, 3, 85, 19, 160, 9, 8, 82, 89, 78, 71, 69, 65, 76, 32, 219, + 186, 27, 65, 8, 154, 131, 27, 69, 162, 64, 65, 3, 73, 64, 94, 72, 134, 6, + 87, 218, 1, 89, 214, 206, 21, 65, 2, 79, 230, 171, 5, 69, 162, 64, 73, 3, + 85, 24, 130, 6, 87, 174, 208, 21, 65, 230, 171, 5, 69, 162, 64, 73, 2, + 79, 3, 85, 20, 226, 213, 21, 65, 2, 79, 230, 171, 5, 69, 234, 61, 87, 2, + 89, 186, 2, 73, 3, 85, 74, 102, 69, 226, 1, 72, 170, 3, 90, 78, 83, 214, + 206, 21, 65, 2, 79, 206, 233, 5, 87, 186, 2, 73, 3, 85, 13, 56, 8, 66, + 65, 84, 66, 69, 73, 84, 32, 243, 191, 27, 69, 8, 230, 206, 22, 66, 2, 70, + 2, 77, 3, 80, 80, 118, 72, 76, 2, 84, 72, 62, 90, 162, 2, 83, 162, 207, + 21, 65, 2, 79, 230, 171, 5, 69, 234, 61, 87, 186, 2, 73, 3, 85, 18, 198, + 210, 21, 65, 2, 79, 230, 171, 5, 69, 234, 61, 87, 186, 2, 73, 3, 85, 12, + 222, 253, 26, 69, 234, 61, 65, 186, 2, 73, 2, 79, 3, 85, 16, 190, 209, + 21, 65, 2, 79, 230, 171, 5, 69, 162, 64, 73, 3, 85, 40, 82, 87, 218, 1, + 89, 214, 206, 21, 65, 2, 79, 230, 171, 5, 69, 162, 64, 73, 3, 85, 10, + 170, 208, 21, 65, 230, 171, 5, 69, 163, 64, 73, 48, 90, 72, 78, 90, 214, + 206, 21, 65, 2, 79, 230, 171, 5, 69, 234, 61, 87, 186, 2, 73, 3, 85, 16, + 158, 207, 21, 65, 230, 171, 5, 69, 234, 61, 87, 186, 2, 73, 2, 79, 3, 85, + 14, 210, 206, 21, 65, 230, 171, 5, 69, 162, 64, 73, 2, 79, 3, 85, 20, + 130, 1, 68, 74, 72, 30, 75, 42, 82, 0, 7, 83, 72, 79, 82, 84, 32, 82, + 204, 140, 4, 3, 67, 72, 73, 165, 246, 22, 3, 89, 73, 90, 6, 48, 4, 69, + 82, 69, 84, 249, 164, 26, 2, 73, 70, 5, 17, 2, 45, 72, 2, 177, 131, 27, + 2, 73, 68, 4, 196, 164, 26, 2, 69, 78, 219, 12, 85, 2, 245, 154, 4, 3, + 73, 75, 82, 10, 68, 2, 82, 79, 197, 173, 26, 9, 76, 69, 82, 32, 67, 79, + 78, 83, 84, 8, 92, 5, 80, 69, 65, 78, 32, 204, 131, 24, 8, 45, 67, 85, + 82, 82, 69, 78, 67, 131, 172, 2, 32, 4, 214, 135, 4, 67, 19, 80, 50, 30, + 67, 102, 80, 187, 1, 84, 6, 60, 9, 76, 65, 77, 65, 84, 73, 79, 78, 32, + 247, 158, 26, 69, 4, 174, 228, 24, 81, 235, 143, 2, 77, 10, 96, 7, 76, + 79, 83, 73, 79, 78, 32, 157, 247, 26, 11, 82, 69, 83, 83, 73, 79, 78, 76, + 69, 83, 83, 8, 232, 209, 11, 4, 70, 82, 65, 77, 229, 146, 15, 8, 65, 84, + 32, 72, 79, 82, 73, 90, 34, 98, 82, 153, 179, 21, 18, 69, 78, 68, 69, 68, + 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, 14, 140, 1, 12, 69, + 77, 69, 76, 89, 32, 72, 69, 65, 86, 89, 32, 141, 222, 25, 16, 65, 84, 69, + 82, 82, 69, 83, 84, 82, 73, 65, 76, 32, 65, 76, 73, 12, 50, 83, 202, 155, + 25, 70, 234, 2, 87, 215, 10, 71, 4, 218, 156, 25, 65, 43, 73, 7, 250, + 167, 19, 71, 171, 137, 8, 83, 142, 4, 142, 1, 65, 130, 19, 69, 190, 1, + 73, 230, 7, 76, 242, 5, 79, 238, 6, 82, 234, 4, 85, 228, 183, 20, 2, 86, + 83, 202, 192, 4, 83, 223, 137, 2, 70, 92, 122, 67, 178, 15, 76, 176, 2, + 2, 88, 32, 238, 141, 3, 77, 172, 247, 5, 2, 82, 83, 188, 250, 1, 2, 84, + 72, 187, 230, 9, 73, 70, 72, 2, 69, 32, 248, 172, 16, 4, 83, 73, 77, 73, + 169, 202, 4, 2, 84, 79, 66, 226, 1, 83, 160, 1, 4, 87, 73, 84, 72, 224, + 250, 12, 2, 80, 65, 228, 136, 2, 2, 77, 65, 140, 145, 11, 13, 84, 72, 82, + 79, 87, 73, 78, 71, 32, 65, 32, 75, 73, 201, 1, 14, 72, 79, 76, 68, 73, + 78, 71, 32, 66, 65, 67, 75, 32, 84, 4, 232, 251, 16, 18, 65, 86, 79, 85, + 82, 73, 78, 71, 32, 68, 69, 76, 73, 67, 73, 79, 85, 83, 241, 128, 5, 13, + 67, 82, 69, 65, 77, 73, 78, 71, 32, 73, 78, 32, 70, 54, 38, 32, 201, 137, + 24, 3, 79, 85, 84, 52, 196, 4, 2, 67, 79, 44, 5, 72, 69, 65, 68, 45, 38, + 77, 98, 79, 92, 7, 78, 79, 32, 71, 79, 79, 68, 238, 1, 80, 164, 1, 16, + 83, 84, 85, 67, 75, 45, 79, 85, 84, 32, 84, 79, 78, 71, 85, 69, 110, 84, + 192, 177, 1, 9, 66, 65, 71, 83, 32, 85, 78, 68, 69, 224, 158, 17, 22, 70, + 73, 78, 71, 69, 82, 32, 67, 79, 86, 69, 82, 73, 78, 71, 32, 67, 76, 79, + 83, 69, 68, 244, 67, 3, 82, 79, 76, 252, 231, 1, 13, 76, 79, 79, 75, 32, + 79, 70, 32, 84, 82, 73, 85, 77, 232, 130, 3, 8, 68, 73, 65, 71, 79, 78, + 65, 76, 1, 20, 85, 78, 69, 86, 69, 78, 32, 69, 89, 69, 83, 32, 65, 78, + 68, 32, 87, 65, 86, 89, 4, 252, 4, 3, 87, 66, 79, 227, 148, 19, 76, 2, + 173, 159, 22, 4, 66, 65, 78, 68, 4, 60, 6, 69, 68, 73, 67, 65, 76, 189, + 211, 25, 3, 79, 78, 79, 2, 177, 224, 25, 3, 32, 77, 65, 12, 90, 75, 36, + 4, 80, 69, 78, 32, 205, 161, 20, 10, 78, 69, 32, 69, 89, 69, 66, 82, 79, + 87, 2, 165, 142, 22, 4, 32, 71, 69, 83, 8, 116, 5, 77, 79, 85, 84, 72, + 217, 129, 24, 18, 69, 89, 69, 83, 32, 65, 78, 68, 32, 72, 65, 78, 68, 32, + 79, 86, 69, 82, 7, 11, 32, 4, 136, 252, 9, 3, 86, 79, 77, 141, 154, 9, 5, + 65, 78, 68, 32, 67, 6, 132, 1, 18, 65, 82, 84, 89, 32, 72, 79, 82, 78, + 32, 65, 78, 68, 32, 80, 65, 82, 84, 100, 2, 69, 69, 197, 148, 19, 4, 76, + 69, 65, 68, 2, 161, 184, 22, 2, 89, 32, 7, 29, 5, 32, 65, 78, 68, 32, 4, + 36, 3, 87, 73, 78, 231, 148, 19, 84, 2, 157, 177, 1, 4, 75, 73, 78, 71, + 4, 50, 69, 225, 187, 22, 6, 72, 69, 82, 77, 79, 77, 2, 157, 142, 27, 8, + 65, 82, 83, 32, 79, 70, 32, 74, 10, 34, 76, 185, 146, 22, 2, 65, 70, 8, + 84, 13, 73, 78, 71, 32, 68, 73, 65, 71, 79, 78, 65, 76, 32, 189, 212, 23, + 2, 69, 78, 6, 128, 1, 9, 67, 82, 79, 83, 83, 73, 78, 71, 32, 185, 167, + 19, 16, 73, 78, 32, 87, 72, 73, 84, 69, 32, 67, 73, 82, 67, 76, 69, 32, + 4, 232, 133, 16, 3, 82, 73, 83, 143, 169, 3, 78, 4, 222, 244, 14, 73, + 255, 137, 4, 77, 14, 50, 65, 50, 77, 44, 2, 82, 82, 147, 217, 18, 78, 4, + 212, 161, 8, 3, 82, 70, 85, 147, 231, 9, 84, 4, 204, 233, 8, 2, 73, 78, + 143, 177, 7, 65, 4, 184, 238, 7, 2, 73, 83, 163, 173, 19, 89, 56, 156, 1, + 5, 71, 85, 82, 69, 32, 38, 76, 178, 1, 78, 98, 82, 182, 2, 83, 236, 13, + 4, 86, 69, 32, 68, 213, 249, 4, 10, 69, 76, 68, 32, 72, 79, 67, 75, 69, + 89, 4, 226, 177, 25, 68, 131, 170, 1, 83, 10, 32, 2, 69, 32, 65, 2, 77, + 32, 6, 166, 187, 13, 70, 136, 154, 2, 3, 67, 65, 66, 191, 150, 8, 83, 4, + 52, 4, 80, 82, 79, 74, 197, 142, 20, 3, 70, 82, 65, 2, 171, 203, 15, 69, + 4, 72, 4, 71, 69, 82, 80, 205, 144, 25, 8, 73, 84, 69, 32, 80, 65, 82, + 84, 2, 167, 255, 15, 82, 22, 34, 69, 189, 1, 3, 83, 84, 32, 13, 56, 2, + 32, 69, 68, 4, 87, 79, 82, 75, 171, 170, 15, 67, 4, 146, 179, 13, 78, + 153, 208, 4, 8, 88, 84, 73, 78, 71, 85, 73, 83, 4, 216, 172, 23, 6, 32, + 83, 80, 65, 82, 75, 223, 233, 3, 83, 10, 190, 170, 5, 81, 216, 176, 10, + 8, 83, 84, 82, 79, 78, 71, 32, 73, 239, 215, 6, 80, 10, 38, 72, 129, 200, + 23, 3, 84, 69, 68, 9, 168, 248, 3, 13, 73, 78, 71, 32, 80, 79, 76, 69, + 32, 65, 78, 68, 32, 246, 202, 5, 69, 237, 202, 16, 19, 32, 67, 65, 75, + 69, 32, 87, 73, 84, 72, 32, 83, 87, 73, 82, 76, 32, 68, 69, 42, 50, 65, + 170, 1, 69, 98, 79, 194, 1, 85, 39, 89, 12, 114, 84, 148, 157, 4, 5, 80, + 80, 73, 78, 71, 136, 158, 3, 6, 71, 32, 73, 78, 32, 72, 161, 202, 17, 3, + 77, 73, 78, 6, 198, 208, 8, 32, 250, 197, 11, 66, 139, 229, 5, 78, 4, + 160, 221, 23, 8, 88, 69, 68, 32, 66, 73, 67, 69, 153, 145, 1, 7, 85, 82, + 45, 68, 69, 45, 76, 10, 46, 82, 28, 3, 87, 69, 82, 167, 203, 25, 80, 2, + 161, 137, 26, 2, 65, 76, 7, 17, 2, 32, 80, 4, 58, 85, 169, 178, 24, 8, + 76, 65, 89, 73, 78, 71, 32, 67, 2, 189, 166, 26, 4, 78, 67, 84, 85, 4, + 162, 152, 3, 83, 219, 224, 23, 84, 13, 25, 4, 73, 78, 71, 32, 10, 64, 6, + 83, 65, 85, 67, 69, 82, 166, 130, 10, 68, 223, 130, 10, 69, 7, 29, 5, 32, + 87, 73, 84, 72, 4, 34, 32, 1, 4, 79, 85, 84, 32, 2, 21, 3, 66, 69, 65, 2, + 147, 208, 26, 77, 54, 102, 71, 20, 2, 76, 68, 68, 2, 79, 84, 22, 82, 238, + 1, 85, 148, 255, 22, 2, 78, 68, 235, 205, 3, 88, 5, 171, 251, 26, 71, 4, + 164, 192, 9, 8, 73, 78, 71, 32, 72, 65, 78, 68, 223, 147, 17, 69, 5, 139, + 148, 14, 80, 16, 102, 75, 158, 206, 20, 77, 186, 181, 4, 32, 214, 115, + 67, 201, 120, 9, 84, 85, 78, 69, 32, 67, 79, 79, 75, 8, 80, 10, 32, 65, + 78, 68, 32, 75, 78, 73, 70, 69, 162, 150, 15, 69, 183, 163, 11, 73, 5, + 249, 207, 15, 7, 32, 87, 73, 84, 72, 32, 80, 22, 26, 82, 255, 161, 23, + 78, 20, 38, 32, 218, 2, 84, 183, 221, 18, 45, 16, 110, 67, 174, 1, 68, + 134, 165, 2, 66, 248, 147, 10, 7, 76, 69, 65, 70, 32, 67, 76, 182, 111, + 84, 155, 200, 11, 80, 4, 128, 170, 13, 3, 76, 85, 66, 209, 86, 32, 79, + 82, 78, 69, 82, 32, 65, 82, 82, 79, 87, 83, 32, 67, 73, 82, 67, 76, 73, + 78, 71, 32, 65, 78, 84, 73, 67, 76, 79, 67, 75, 87, 4, 21, 3, 79, 84, 32, + 4, 138, 197, 23, 80, 239, 128, 3, 77, 2, 183, 38, 72, 28, 78, 65, 226, 1, + 69, 98, 79, 209, 241, 16, 8, 73, 69, 68, 32, 83, 72, 82, 73, 10, 72, 6, + 67, 84, 73, 79, 78, 32, 69, 8, 77, 69, 32, 87, 73, 84, 72, 32, 4, 146, + 239, 19, 83, 177, 6, 9, 78, 85, 77, 69, 82, 65, 84, 79, 82, 6, 50, 80, + 194, 165, 2, 65, 217, 161, 21, 2, 84, 73, 2, 221, 238, 21, 2, 73, 67, 6, + 48, 6, 78, 67, 72, 32, 70, 82, 235, 251, 18, 69, 4, 146, 240, 25, 73, + 249, 12, 3, 65, 78, 67, 10, 52, 3, 78, 84, 45, 116, 2, 87, 78, 151, 197, + 26, 71, 4, 70, 84, 173, 138, 2, 11, 70, 65, 67, 73, 78, 71, 32, 66, 65, + 66, 89, 2, 197, 178, 12, 6, 73, 76, 84, 69, 68, 32, 5, 149, 135, 8, 6, + 73, 78, 71, 32, 70, 65, 226, 1, 80, 2, 76, 76, 134, 6, 78, 156, 233, 16, + 5, 69, 76, 32, 80, 85, 191, 251, 9, 83, 216, 1, 42, 32, 73, 6, 87, 73, + 68, 84, 72, 32, 10, 246, 255, 11, 77, 252, 169, 3, 2, 79, 85, 210, 232, + 8, 66, 151, 29, 83, 206, 1, 242, 1, 67, 42, 76, 78, 78, 30, 80, 66, 82, + 142, 1, 83, 38, 89, 158, 146, 10, 77, 198, 208, 10, 65, 158, 2, 68, 58, + 69, 98, 71, 118, 72, 202, 4, 81, 190, 143, 2, 87, 218, 28, 84, 250, 145, + 1, 70, 148, 176, 1, 6, 66, 82, 79, 75, 69, 78, 211, 7, 86, 10, 214, 230, + 20, 73, 62, 79, 159, 75, 69, 116, 42, 69, 138, 234, 20, 65, 243, 171, 4, + 79, 10, 162, 1, 70, 251, 235, 20, 83, 4, 246, 186, 21, 79, 35, 85, 6, 42, + 79, 250, 236, 20, 69, 203, 225, 2, 76, 2, 159, 148, 25, 85, 10, 36, 3, + 73, 71, 72, 131, 231, 24, 69, 8, 17, 2, 84, 32, 8, 228, 162, 20, 5, 87, + 72, 73, 84, 69, 202, 211, 2, 67, 210, 3, 80, 239, 7, 83, 4, 166, 176, 23, + 69, 147, 184, 1, 79, 2, 159, 130, 25, 69, 6, 136, 192, 11, 8, 67, 84, 73, + 79, 78, 32, 65, 80, 176, 247, 8, 5, 69, 82, 65, 76, 32, 143, 183, 1, 78, + 226, 20, 114, 65, 250, 6, 69, 198, 17, 73, 162, 1, 76, 134, 15, 79, 194, + 6, 82, 178, 92, 85, 246, 131, 22, 72, 139, 234, 3, 83, 142, 1, 42, 82, + 237, 221, 26, 4, 77, 69, 32, 68, 140, 1, 36, 3, 65, 89, 32, 235, 208, 25, + 76, 138, 1, 166, 1, 67, 210, 1, 83, 192, 2, 6, 86, 79, 87, 69, 76, 32, + 132, 166, 8, 6, 82, 69, 68, 85, 80, 76, 202, 202, 10, 72, 230, 168, 1, + 80, 170, 171, 3, 77, 191, 189, 1, 68, 52, 42, 79, 205, 1, 5, 65, 80, 73, + 84, 65, 8, 88, 10, 77, 66, 73, 78, 73, 78, 71, 32, 68, 79, 37, 8, 78, 83, + 79, 78, 65, 78, 84, 32, 4, 246, 135, 12, 85, 195, 241, 13, 84, 4, 190, + 136, 12, 78, 203, 145, 11, 71, 46, 36, 3, 77, 65, 76, 215, 250, 21, 85, + 44, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 44, 200, 1, 4, 79, 76, 68, + 32, 174, 137, 20, 78, 238, 231, 6, 66, 2, 67, 2, 68, 2, 70, 2, 71, 2, 72, + 2, 74, 2, 75, 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, 88, 2, + 89, 187, 2, 65, 4, 150, 241, 26, 75, 3, 78, 12, 44, 5, 83, 73, 71, 78, + 32, 139, 177, 26, 76, 10, 226, 178, 26, 69, 162, 64, 65, 2, 73, 3, 79, + 242, 2, 112, 2, 65, 82, 110, 77, 50, 79, 204, 189, 23, 9, 82, 77, 65, 78, + 32, 80, 69, 78, 78, 218, 173, 2, 84, 239, 105, 78, 7, 29, 5, 32, 87, 73, + 84, 72, 4, 224, 147, 14, 5, 79, 85, 84, 32, 72, 149, 160, 9, 5, 32, 72, + 65, 78, 68, 4, 26, 32, 211, 244, 21, 73, 2, 251, 255, 22, 83, 226, 2, 64, + 6, 77, 69, 84, 82, 73, 67, 73, 6, 82, 71, 73, 65, 78, 32, 6, 236, 250, + 19, 4, 65, 76, 76, 89, 213, 222, 1, 5, 32, 80, 82, 79, 80, 220, 2, 228, + 1, 6, 67, 65, 80, 73, 84, 65, 0, 4, 83, 77, 65, 76, 172, 3, 7, 76, 69, + 84, 84, 69, 82, 32, 180, 2, 24, 77, 84, 65, 86, 82, 85, 76, 73, 32, 67, + 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 165, 6, 2, 80, + 65, 80, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 80, 142, 2, 65, 34, + 72, 166, 5, 67, 118, 71, 130, 1, 74, 34, 75, 82, 80, 34, 83, 94, 90, 220, + 180, 5, 2, 84, 65, 250, 142, 8, 76, 246, 179, 2, 82, 154, 160, 9, 66, 2, + 77, 2, 88, 202, 40, 78, 2, 81, 230, 33, 86, 234, 47, 68, 14, 69, 2, 73, + 2, 79, 2, 85, 2, 89, 143, 57, 87, 4, 162, 155, 26, 69, 227, 79, 78, 10, + 46, 65, 226, 211, 26, 73, 2, 79, 215, 22, 69, 4, 178, 234, 26, 69, 3, 82, + 94, 254, 1, 85, 178, 2, 65, 42, 67, 74, 69, 46, 71, 34, 72, 98, 74, 34, + 75, 34, 76, 50, 80, 34, 83, 34, 84, 62, 90, 202, 247, 15, 82, 154, 160, + 9, 66, 2, 77, 2, 88, 202, 40, 78, 2, 81, 230, 33, 86, 234, 47, 68, 14, + 73, 2, 79, 2, 89, 142, 57, 87, 255, 2, 70, 4, 204, 248, 21, 4, 45, 66, + 82, 74, 203, 239, 4, 78, 92, 250, 1, 65, 42, 67, 74, 69, 46, 71, 34, 72, + 98, 74, 34, 75, 34, 76, 50, 80, 34, 83, 34, 84, 62, 90, 202, 247, 15, 82, + 154, 160, 9, 66, 2, 77, 2, 88, 202, 40, 78, 2, 81, 230, 33, 86, 234, 47, + 68, 14, 73, 2, 79, 2, 85, 2, 89, 142, 57, 87, 255, 2, 70, 6, 134, 150, + 26, 69, 2, 73, 227, 79, 78, 8, 38, 72, 230, 219, 25, 73, 215, 57, 65, 4, + 182, 149, 26, 73, 135, 23, 65, 4, 224, 136, 9, 2, 76, 73, 151, 220, 17, + 78, 4, 202, 154, 25, 72, 163, 122, 65, 12, 46, 65, 170, 205, 26, 73, 2, + 79, 215, 22, 69, 6, 26, 82, 227, 227, 26, 69, 5, 223, 220, 25, 68, 4, + 202, 153, 25, 72, 207, 64, 73, 4, 242, 193, 25, 72, 219, 81, 65, 4, 11, + 65, 4, 146, 210, 15, 66, 239, 144, 11, 83, 4, 162, 193, 25, 72, 223, 104, + 65, 4, 230, 226, 25, 72, 247, 47, 65, 6, 196, 183, 3, 6, 85, 82, 78, 69, + 68, 32, 211, 253, 1, 65, 4, 166, 192, 25, 72, 219, 81, 69, 2, 177, 166, + 20, 7, 82, 65, 71, 82, 65, 80, 72, 10, 48, 2, 77, 69, 20, 4, 78, 71, 69, + 82, 31, 82, 2, 179, 205, 25, 76, 2, 141, 184, 18, 2, 32, 82, 6, 38, 76, + 173, 235, 13, 3, 65, 70, 70, 5, 219, 204, 25, 83, 202, 1, 66, 65, 214, + 13, 79, 169, 144, 26, 7, 69, 73, 67, 72, 32, 83, 84, 194, 1, 84, 8, 71, + 79, 76, 73, 84, 73, 67, 32, 229, 12, 8, 83, 83, 32, 79, 70, 32, 77, 73, + 192, 1, 56, 6, 67, 65, 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 96, 45, 9, + 76, 32, 76, 69, 84, 84, 69, 82, 32, 96, 206, 1, 65, 22, 66, 42, 67, 94, + 68, 94, 70, 38, 71, 46, 73, 138, 2, 76, 58, 77, 66, 78, 34, 79, 30, 80, + 58, 82, 30, 83, 186, 1, 84, 110, 86, 22, 89, 90, 90, 226, 132, 19, 72, + 174, 128, 2, 85, 147, 174, 5, 75, 2, 163, 183, 26, 90, 4, 214, 3, 73, + 217, 198, 26, 2, 85, 75, 4, 48, 8, 65, 85, 68, 65, 84, 69, 32, 67, 15, + 72, 2, 11, 72, 2, 217, 250, 19, 2, 82, 73, 6, 42, 74, 30, 79, 221, 183, + 26, 2, 90, 69, 2, 145, 250, 19, 2, 69, 82, 2, 147, 235, 24, 66, 4, 186, + 138, 21, 82, 147, 171, 5, 73, 2, 21, 3, 76, 65, 71, 2, 207, 222, 21, 79, + 13, 38, 78, 54, 79, 141, 1, 2, 90, 72, 2, 145, 172, 26, 8, 73, 84, 73, + 65, 76, 32, 73, 90, 4, 33, 6, 84, 65, 84, 69, 68, 32, 4, 26, 66, 25, 2, + 83, 77, 2, 11, 73, 2, 35, 71, 2, 21, 3, 65, 76, 76, 2, 253, 209, 24, 2, + 32, 89, 4, 142, 213, 26, 73, 211, 2, 69, 4, 52, 9, 65, 84, 73, 78, 65, + 84, 69, 32, 77, 35, 74, 2, 173, 133, 11, 3, 89, 83, 76, 2, 249, 140, 9, + 3, 85, 68, 73, 2, 11, 65, 2, 187, 135, 21, 83, 4, 190, 177, 26, 78, 3, + 84, 4, 26, 79, 243, 213, 26, 69, 2, 133, 191, 22, 2, 75, 79, 2, 177, 235, + 21, 2, 73, 84, 14, 106, 72, 58, 76, 172, 240, 13, 6, 80, 73, 68, 69, 82, + 89, 201, 175, 10, 8, 77, 65, 76, 76, 32, 89, 85, 83, 6, 32, 2, 84, 65, + 171, 212, 26, 65, 5, 159, 172, 25, 80, 2, 239, 246, 19, 79, 6, 78, 86, + 144, 147, 22, 9, 82, 79, 75, 85, 84, 65, 83, 84, 73, 203, 172, 4, 83, 2, + 129, 162, 19, 2, 82, 73, 2, 135, 195, 24, 69, 12, 50, 69, 214, 244, 18, + 65, 250, 221, 7, 79, 3, 85, 6, 250, 130, 21, 83, 155, 143, 5, 82, 4, 156, + 145, 9, 3, 69, 77, 76, 145, 250, 15, 4, 72, 73, 86, 69, 2, 207, 206, 26, + 76, 6, 228, 238, 11, 13, 66, 69, 32, 87, 73, 84, 72, 32, 77, 69, 82, 73, + 68, 250, 198, 4, 87, 255, 135, 9, 86, 68, 162, 1, 65, 44, 12, 84, 72, 73, + 67, 32, 76, 69, 84, 84, 69, 82, 32, 230, 161, 18, 82, 206, 102, 71, 172, + 131, 5, 3, 78, 71, 71, 182, 186, 1, 79, 161, 75, 2, 76, 70, 4, 144, 183, + 15, 2, 76, 32, 183, 152, 11, 84, 54, 206, 2, 65, 50, 72, 46, 73, 46, 78, + 46, 80, 2, 81, 40, 2, 82, 65, 22, 84, 200, 137, 9, 2, 87, 73, 230, 141, + 12, 85, 188, 69, 3, 70, 65, 73, 204, 7, 4, 66, 65, 73, 82, 128, 22, 2, + 79, 84, 150, 30, 68, 158, 123, 77, 164, 131, 1, 3, 83, 65, 85, 162, 16, + 69, 184, 29, 3, 76, 65, 71, 146, 200, 1, 74, 196, 16, 2, 71, 73, 141, 12, + 2, 75, 85, 4, 204, 198, 24, 3, 73, 72, 86, 187, 131, 2, 72, 4, 214, 224, + 13, 87, 145, 226, 11, 2, 65, 71, 4, 230, 138, 9, 85, 217, 229, 14, 2, 71, + 71, 6, 154, 206, 15, 73, 253, 252, 8, 2, 65, 85, 2, 213, 134, 26, 5, 65, + 73, 82, 84, 72, 2, 235, 165, 26, 73, 4, 192, 169, 23, 2, 72, 73, 233, 69, + 2, 69, 73, 234, 9, 46, 65, 198, 5, 69, 206, 82, 73, 151, 3, 79, 152, 1, + 96, 7, 68, 85, 65, 84, 73, 79, 78, 32, 5, 78, 84, 72, 65, 32, 242, 179, + 20, 86, 139, 129, 5, 80, 2, 11, 32, 2, 151, 184, 25, 67, 146, 1, 120, 7, + 76, 69, 84, 84, 69, 82, 32, 212, 2, 5, 83, 73, 71, 78, 32, 226, 231, 22, + 65, 248, 8, 2, 86, 79, 239, 195, 3, 79, 100, 214, 1, 86, 250, 235, 22, + 65, 38, 68, 114, 84, 230, 5, 85, 206, 201, 1, 73, 42, 76, 250, 192, 1, + 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 206, 40, 79, 162, + 8, 69, 158, 20, 72, 2, 77, 2, 82, 3, 89, 14, 60, 5, 69, 68, 73, 67, 32, + 210, 242, 22, 79, 139, 211, 3, 65, 4, 244, 143, 24, 6, 68, 79, 85, 66, + 76, 69, 235, 240, 1, 65, 16, 66, 67, 250, 176, 22, 78, 242, 60, 65, 182, + 1, 80, 179, 146, 3, 86, 4, 226, 238, 7, 79, 195, 193, 14, 65, 192, 8, 76, + 10, 65, 84, 69, 82, 45, 84, 72, 65, 78, 32, 206, 7, 69, 219, 180, 25, 89, + 56, 134, 1, 65, 150, 3, 66, 62, 79, 216, 2, 11, 69, 81, 85, 65, 76, 32, + 84, 79, 32, 79, 82, 222, 205, 6, 67, 138, 4, 87, 231, 227, 18, 83, 16, + 44, 5, 66, 79, 86, 69, 32, 187, 210, 6, 78, 12, 150, 1, 83, 180, 1, 19, + 68, 79, 85, 66, 76, 69, 45, 76, 73, 78, 69, 32, 69, 81, 85, 65, 76, 32, + 65, 224, 204, 6, 4, 76, 69, 83, 83, 135, 211, 18, 82, 6, 148, 1, 7, 73, + 77, 73, 76, 65, 82, 32, 225, 207, 6, 23, 76, 65, 78, 84, 69, 68, 32, 69, + 81, 85, 65, 76, 32, 65, 66, 79, 86, 69, 32, 76, 69, 83, 83, 4, 26, 65, + 183, 207, 6, 79, 2, 65, 3, 66, 79, 86, 6, 40, 4, 69, 83, 73, 68, 159, + 208, 6, 85, 2, 231, 2, 69, 20, 40, 2, 82, 32, 245, 1, 3, 86, 69, 82, 16, + 120, 16, 83, 76, 65, 78, 84, 69, 68, 32, 69, 81, 85, 65, 76, 32, 84, 79, + 230, 209, 6, 65, 138, 247, 12, 69, 131, 227, 4, 76, 9, 49, 10, 32, 87, + 73, 84, 72, 32, 68, 79, 84, 32, 6, 44, 5, 65, 66, 79, 86, 69, 163, 153, + 18, 73, 5, 195, 138, 26, 32, 4, 52, 7, 76, 65, 80, 80, 73, 78, 71, 227, + 232, 19, 32, 2, 185, 170, 24, 2, 32, 76, 134, 8, 36, 2, 75, 32, 185, 73, + 2, 78, 32, 254, 7, 130, 3, 65, 216, 15, 8, 67, 65, 80, 73, 84, 65, 76, + 32, 182, 11, 68, 134, 1, 70, 68, 2, 73, 78, 222, 3, 75, 138, 1, 76, 174, + 3, 78, 66, 77, 84, 3, 88, 69, 83, 22, 79, 202, 1, 80, 90, 82, 182, 1, 83, + 130, 22, 84, 200, 2, 13, 85, 80, 83, 73, 76, 79, 78, 32, 87, 73, 84, 72, + 32, 150, 1, 86, 142, 2, 89, 178, 225, 7, 66, 248, 175, 3, 4, 71, 82, 65, + 77, 204, 23, 2, 90, 69, 147, 249, 11, 81, 112, 92, 10, 67, 82, 79, 80, + 72, 79, 78, 73, 67, 32, 172, 14, 6, 78, 79, 32, 84, 69, 76, 23, 82, 106, + 188, 2, 6, 65, 84, 84, 73, 67, 32, 222, 5, 67, 92, 3, 78, 65, 88, 32, 12, + 68, 69, 76, 80, 72, 73, 67, 32, 70, 73, 86, 69, 0, 14, 83, 84, 82, 65, + 84, 73, 65, 78, 32, 70, 73, 70, 84, 89, 40, 11, 69, 80, 73, 68, 65, 85, + 82, 69, 65, 78, 32, 112, 3, 72, 69, 82, 164, 1, 9, 77, 69, 83, 83, 69, + 78, 73, 65, 78, 35, 84, 48, 72, 2, 70, 73, 180, 2, 4, 79, 78, 69, 32, + 205, 1, 4, 84, 69, 78, 32, 26, 36, 3, 70, 84, 89, 105, 2, 86, 69, 11, 11, + 32, 8, 22, 84, 171, 4, 83, 6, 48, 7, 72, 79, 85, 83, 65, 78, 68, 215, 3, + 65, 5, 231, 3, 32, 17, 11, 32, 14, 56, 7, 72, 85, 78, 68, 82, 69, 68, 18, + 84, 143, 3, 83, 7, 131, 2, 32, 6, 48, 7, 72, 79, 85, 83, 65, 78, 68, 187, + 2, 65, 5, 213, 1, 2, 32, 84, 14, 98, 72, 48, 7, 84, 72, 79, 85, 83, 65, + 78, 174, 174, 24, 81, 241, 225, 1, 5, 68, 82, 65, 67, 72, 6, 44, 5, 85, + 78, 68, 82, 69, 143, 174, 24, 65, 4, 17, 2, 68, 32, 4, 22, 84, 131, 1, + 83, 2, 95, 65, 8, 30, 84, 86, 83, 175, 1, 77, 4, 50, 65, 21, 8, 72, 79, + 85, 83, 65, 78, 68, 32, 2, 187, 174, 16, 76, 2, 11, 83, 2, 145, 175, 24, + 2, 84, 65, 4, 88, 5, 65, 82, 89, 83, 84, 145, 1, 12, 89, 82, 69, 78, 65, + 73, 67, 32, 84, 87, 79, 32, 2, 101, 5, 73, 65, 78, 32, 70, 2, 17, 2, 32, + 77, 2, 199, 144, 13, 78, 6, 30, 70, 29, 3, 84, 87, 79, 2, 177, 178, 15, + 2, 73, 86, 5, 11, 32, 2, 233, 144, 10, 5, 68, 82, 65, 67, 72, 8, 112, 8, + 77, 73, 79, 78, 73, 65, 78, 32, 189, 135, 25, 14, 65, 69, 85, 77, 32, 79, + 78, 69, 32, 80, 76, 69, 84, 72, 6, 238, 185, 12, 70, 134, 159, 12, 84, + 191, 58, 79, 2, 11, 32, 2, 183, 216, 24, 84, 32, 92, 8, 72, 69, 83, 80, + 73, 65, 78, 32, 129, 1, 10, 82, 79, 69, 90, 69, 78, 73, 65, 78, 32, 20, + 40, 2, 70, 73, 38, 84, 135, 151, 18, 79, 6, 162, 215, 20, 86, 231, 230, + 3, 70, 8, 202, 174, 15, 72, 178, 172, 10, 69, 239, 48, 87, 12, 36, 2, 70, + 73, 209, 24, 2, 84, 69, 8, 142, 176, 18, 86, 205, 250, 6, 3, 70, 84, 89, + 2, 159, 132, 10, 69, 4, 240, 215, 22, 2, 79, 85, 161, 181, 1, 3, 84, 65, + 66, 154, 2, 66, 76, 174, 45, 82, 66, 68, 220, 226, 7, 2, 75, 65, 135, 6, + 84, 144, 2, 44, 6, 69, 84, 84, 69, 82, 32, 239, 45, 85, 142, 2, 198, 2, + 65, 190, 1, 69, 28, 4, 73, 79, 84, 65, 128, 1, 2, 79, 77, 156, 3, 3, 82, + 72, 79, 46, 83, 48, 7, 85, 80, 83, 73, 76, 79, 78, 146, 33, 80, 170, 2, + 84, 202, 183, 5, 68, 144, 176, 2, 2, 75, 65, 166, 192, 1, 71, 190, 132, + 11, 67, 194, 149, 2, 66, 2, 72, 2, 90, 166, 1, 76, 230, 231, 2, 89, 210, + 43, 77, 2, 78, 147, 17, 88, 48, 68, 4, 76, 80, 72, 65, 213, 28, 8, 82, + 67, 72, 65, 73, 67, 32, 83, 47, 33, 6, 32, 87, 73, 84, 72, 32, 44, 242, + 2, 68, 30, 80, 226, 29, 86, 226, 5, 79, 246, 236, 3, 84, 239, 167, 5, 77, + 62, 186, 1, 84, 131, 27, 80, 31, 33, 6, 32, 87, 73, 84, 72, 32, 28, 186, + 5, 68, 136, 25, 2, 80, 83, 158, 1, 86, 226, 5, 79, 246, 236, 3, 84, 239, + 167, 5, 77, 62, 28, 2, 69, 71, 235, 34, 73, 42, 11, 65, 43, 33, 6, 32, + 87, 73, 84, 72, 32, 40, 54, 68, 30, 80, 194, 35, 79, 22, 86, 227, 236, 3, + 84, 16, 65, 4, 65, 83, 73, 65, 18, 36, 4, 83, 73, 76, 73, 211, 16, 82, + 17, 29, 5, 32, 65, 78, 68, 32, 14, 44, 2, 79, 88, 0, 3, 86, 65, 82, 23, + 80, 4, 81, 2, 73, 65, 6, 60, 10, 69, 82, 73, 83, 80, 79, 77, 69, 78, 73, + 175, 15, 82, 5, 169, 15, 7, 32, 65, 78, 68, 32, 80, 82, 5, 161, 35, 7, + 32, 87, 73, 84, 72, 32, 68, 6, 222, 140, 8, 73, 226, 194, 17, 65, 239, + 48, 72, 23, 33, 6, 32, 87, 73, 84, 72, 32, 20, 66, 68, 166, 26, 86, 226, + 5, 79, 246, 236, 3, 84, 239, 167, 5, 77, 10, 130, 24, 65, 181, 176, 22, + 5, 73, 65, 76, 89, 84, 18, 76, 9, 73, 65, 76, 89, 84, 73, 75, 65, 32, 32, + 3, 82, 65, 67, 227, 22, 65, 8, 174, 24, 65, 199, 243, 3, 84, 2, 243, 186, + 11, 72, 4, 40, 3, 73, 86, 69, 1, 3, 79, 85, 82, 2, 205, 37, 2, 32, 79, + 76, 144, 1, 27, 83, 84, 82, 85, 77, 69, 78, 84, 65, 76, 32, 78, 79, 84, + 65, 84, 73, 79, 78, 32, 83, 89, 77, 66, 79, 76, 45, 161, 154, 22, 2, 68, + 73, 74, 70, 49, 70, 50, 62, 51, 62, 52, 170, 37, 53, 206, 243, 25, 55, 3, + 56, 17, 174, 154, 26, 49, 2, 50, 2, 51, 2, 52, 2, 55, 2, 56, 3, 57, 15, + 234, 153, 26, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 12, 174, 153, 26, + 48, 2, 50, 2, 54, 2, 55, 2, 56, 3, 57, 17, 242, 152, 26, 48, 2, 50, 2, + 51, 2, 53, 2, 55, 2, 56, 3, 57, 8, 54, 65, 38, 79, 169, 32, 6, 89, 65, + 84, 72, 79, 83, 4, 134, 255, 7, 80, 203, 133, 17, 73, 2, 11, 82, 2, 11, + 79, 2, 135, 244, 23, 78, 32, 128, 1, 6, 69, 84, 84, 69, 82, 32, 168, 2, + 6, 79, 87, 69, 82, 32, 78, 32, 6, 85, 78, 65, 84, 69, 32, 145, 194, 22, + 2, 73, 84, 24, 94, 83, 140, 13, 9, 65, 82, 67, 72, 65, 73, 67, 32, 75, 2, + 75, 182, 11, 68, 207, 172, 25, 89, 16, 88, 13, 77, 65, 76, 76, 32, 67, + 65, 80, 73, 84, 65, 76, 32, 210, 12, 65, 247, 243, 7, 84, 12, 74, 80, + 226, 192, 9, 71, 194, 156, 9, 79, 218, 206, 2, 82, 139, 176, 1, 76, 4, + 194, 128, 26, 83, 219, 19, 73, 2, 221, 134, 10, 3, 85, 77, 69, 4, 222, + 25, 83, 215, 224, 7, 69, 4, 80, 4, 69, 84, 82, 69, 241, 192, 23, 10, 85, + 83, 73, 67, 65, 76, 32, 76, 69, 73, 2, 163, 212, 2, 84, 12, 88, 3, 78, + 69, 32, 198, 236, 9, 88, 148, 155, 5, 3, 85, 78, 75, 253, 198, 5, 2, 66, + 79, 6, 64, 8, 72, 65, 76, 70, 32, 83, 73, 71, 21, 4, 81, 85, 65, 82, 4, + 139, 146, 25, 78, 2, 255, 206, 20, 84, 16, 62, 82, 206, 11, 83, 114, 69, + 154, 236, 7, 72, 211, 132, 17, 73, 2, 217, 29, 2, 79, 83, 6, 100, 3, 72, + 79, 32, 165, 246, 7, 16, 69, 86, 69, 82, 83, 69, 68, 32, 76, 85, 78, 65, + 84, 69, 32, 69, 4, 180, 243, 13, 10, 87, 73, 84, 72, 32, 83, 84, 82, 79, + 75, 135, 137, 11, 83, 226, 2, 220, 1, 5, 77, 65, 76, 76, 32, 192, 19, 22, + 85, 66, 83, 67, 82, 73, 80, 84, 32, 83, 77, 65, 76, 76, 32, 76, 69, 84, + 84, 69, 82, 32, 72, 10, 89, 77, 66, 79, 76, 32, 84, 65, 85, 32, 189, 144, + 24, 6, 73, 78, 85, 83, 79, 73, 212, 2, 56, 7, 76, 69, 84, 84, 69, 82, 32, + 202, 17, 82, 67, 68, 206, 2, 178, 2, 65, 162, 2, 68, 38, 69, 52, 4, 73, + 79, 84, 65, 0, 7, 85, 80, 83, 73, 76, 79, 78, 254, 3, 75, 28, 2, 79, 77, + 182, 5, 80, 112, 3, 82, 72, 79, 94, 83, 94, 84, 244, 230, 7, 2, 70, 73, + 138, 193, 1, 71, 190, 132, 11, 67, 194, 149, 2, 66, 2, 72, 2, 90, 166, 1, + 76, 182, 147, 3, 77, 2, 78, 147, 17, 88, 58, 64, 4, 76, 80, 72, 65, 149, + 1, 7, 82, 67, 72, 65, 73, 67, 32, 55, 33, 6, 32, 87, 73, 84, 72, 32, 52, + 82, 86, 226, 6, 68, 30, 80, 114, 79, 142, 1, 89, 226, 238, 3, 84, 239, + 167, 5, 77, 6, 154, 5, 82, 155, 3, 65, 4, 18, 75, 23, 83, 2, 215, 244, 7, + 79, 2, 11, 65, 2, 203, 189, 13, 77, 4, 138, 223, 7, 73, 235, 239, 14, 69, + 70, 22, 80, 215, 4, 84, 20, 249, 7, 3, 83, 73, 76, 41, 33, 6, 32, 87, 73, + 84, 72, 32, 38, 78, 68, 166, 1, 80, 178, 1, 86, 226, 5, 79, 246, 236, 3, + 84, 239, 167, 5, 77, 18, 50, 65, 29, 8, 73, 65, 76, 89, 84, 73, 75, 65, + 8, 153, 1, 3, 83, 73, 65, 11, 29, 5, 32, 65, 78, 68, 32, 8, 170, 1, 80, + 154, 6, 79, 22, 86, 227, 236, 3, 84, 10, 18, 83, 115, 69, 8, 21, 3, 73, + 76, 73, 9, 17, 2, 32, 65, 6, 21, 3, 78, 68, 32, 6, 30, 80, 154, 6, 79, + 23, 86, 2, 11, 69, 2, 11, 82, 2, 181, 17, 4, 73, 83, 80, 79, 4, 22, 82, + 147, 15, 65, 2, 133, 226, 25, 2, 65, 67, 4, 206, 239, 7, 65, 3, 79, 70, + 28, 2, 69, 71, 151, 3, 73, 50, 11, 65, 51, 33, 6, 32, 87, 73, 84, 72, 32, + 48, 58, 68, 30, 80, 114, 79, 62, 86, 82, 89, 227, 238, 3, 84, 16, 61, 4, + 65, 83, 73, 65, 20, 32, 4, 83, 73, 76, 73, 91, 69, 17, 29, 5, 32, 65, 78, + 68, 32, 14, 42, 79, 12, 2, 80, 69, 50, 86, 83, 89, 4, 83, 88, 4, 89, 9, + 82, 73, 83, 80, 79, 77, 69, 78, 73, 4, 11, 65, 4, 11, 82, 4, 17, 2, 73, + 65, 5, 33, 6, 32, 65, 78, 68, 32, 89, 2, 243, 12, 80, 20, 17, 2, 67, 82, + 20, 17, 2, 79, 78, 21, 33, 6, 32, 87, 73, 84, 72, 32, 18, 88, 5, 68, 65, + 83, 73, 65, 0, 5, 80, 83, 73, 76, 73, 54, 79, 22, 86, 227, 236, 3, 84, 7, + 29, 5, 32, 65, 78, 68, 32, 4, 18, 79, 23, 86, 2, 207, 216, 9, 88, 2, 179, + 9, 65, 8, 88, 11, 65, 77, 80, 72, 89, 76, 73, 65, 78, 32, 68, 174, 233, + 25, 72, 2, 83, 219, 19, 73, 2, 139, 212, 7, 73, 7, 33, 6, 32, 87, 73, 84, + 72, 32, 4, 34, 68, 145, 129, 21, 2, 80, 83, 2, 243, 197, 8, 65, 10, 54, + 65, 186, 231, 7, 84, 230, 1, 73, 207, 243, 17, 72, 4, 142, 177, 13, 77, + 207, 202, 12, 78, 4, 246, 193, 22, 72, 219, 148, 3, 65, 4, 41, 8, 69, 86, + 69, 82, 83, 69, 68, 32, 4, 18, 68, 43, 76, 2, 37, 7, 79, 84, 84, 69, 68, + 32, 76, 2, 11, 85, 2, 33, 6, 78, 65, 84, 69, 32, 83, 2, 169, 232, 7, 3, + 73, 71, 77, 10, 158, 166, 9, 71, 190, 132, 11, 67, 2, 80, 222, 102, 82, + 231, 174, 1, 66, 2, 239, 144, 21, 82, 16, 106, 72, 104, 7, 82, 89, 66, + 76, 73, 79, 78, 44, 3, 87, 79, 32, 254, 229, 3, 79, 233, 196, 16, 2, 65, + 76, 6, 40, 4, 82, 69, 69, 32, 143, 230, 7, 69, 4, 146, 1, 79, 157, 200, + 22, 7, 81, 85, 65, 82, 84, 69, 82, 2, 21, 3, 32, 66, 65, 2, 231, 218, 23, + 83, 4, 42, 79, 221, 152, 12, 4, 84, 72, 73, 82, 2, 237, 224, 19, 2, 66, + 79, 6, 80, 5, 65, 67, 85, 84, 69, 0, 9, 68, 73, 65, 69, 82, 69, 83, 73, + 83, 39, 72, 2, 33, 6, 32, 65, 78, 68, 32, 72, 2, 197, 195, 7, 2, 79, 79, + 60, 102, 65, 21, 21, 79, 67, 65, 76, 32, 78, 79, 84, 65, 84, 73, 79, 78, + 32, 83, 89, 77, 66, 79, 76, 45, 2, 135, 207, 9, 82, 58, 90, 50, 2, 53, + 154, 179, 23, 49, 182, 192, 2, 51, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 13, + 202, 243, 25, 48, 2, 49, 2, 50, 2, 51, 3, 52, 4, 26, 80, 207, 177, 20, + 69, 2, 11, 79, 2, 33, 6, 71, 69, 71, 82, 65, 77, 2, 197, 246, 20, 2, 77, + 69, 8, 170, 237, 13, 65, 178, 177, 10, 66, 174, 76, 72, 253, 64, 3, 83, + 65, 76, 12, 60, 6, 78, 78, 73, 78, 71, 32, 137, 233, 24, 3, 77, 65, 67, + 10, 100, 4, 70, 65, 67, 69, 149, 228, 17, 15, 67, 65, 84, 32, 70, 65, 67, + 69, 32, 87, 73, 84, 72, 32, 83, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 108, + 23, 79, 78, 69, 32, 76, 65, 82, 71, 69, 32, 65, 78, 68, 32, 79, 78, 69, + 32, 83, 77, 65, 76, 76, 35, 83, 2, 11, 32, 2, 191, 157, 8, 69, 4, 32, 2, + 84, 65, 203, 226, 17, 77, 2, 187, 243, 22, 82, 6, 28, 3, 85, 80, 32, 39, + 87, 4, 214, 192, 22, 83, 179, 236, 2, 77, 2, 227, 144, 18, 73, 220, 4, + 136, 1, 2, 65, 82, 70, 73, 52, 7, 74, 65, 82, 65, 84, 73, 32, 208, 6, 12, + 78, 74, 65, 76, 65, 32, 71, 79, 78, 68, 73, 32, 143, 3, 82, 4, 34, 65, + 245, 246, 20, 2, 68, 83, 2, 11, 78, 2, 187, 229, 24, 73, 4, 238, 202, 24, + 84, 217, 160, 1, 4, 68, 69, 32, 68, 182, 1, 168, 1, 7, 76, 69, 84, 84, + 69, 82, 32, 220, 1, 5, 83, 73, 71, 78, 32, 160, 2, 6, 86, 79, 87, 69, 76, + 32, 238, 251, 19, 65, 154, 24, 82, 166, 225, 3, 68, 203, 224, 1, 79, 98, + 162, 144, 22, 65, 38, 68, 114, 84, 46, 86, 186, 5, 85, 206, 201, 1, 73, + 42, 76, 246, 14, 90, 134, 178, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, + 74, 2, 75, 2, 80, 138, 69, 72, 2, 77, 2, 82, 2, 89, 186, 2, 69, 3, 79, + 24, 98, 67, 28, 3, 77, 65, 68, 22, 84, 138, 202, 3, 83, 238, 137, 18, 78, + 242, 60, 65, 231, 147, 3, 86, 4, 118, 73, 219, 210, 21, 65, 2, 231, 135, + 19, 68, 4, 68, 5, 87, 79, 45, 67, 73, 29, 8, 72, 82, 69, 69, 45, 68, 79, + 84, 2, 25, 4, 82, 67, 76, 69, 2, 253, 195, 20, 5, 32, 78, 85, 75, 84, 34, + 36, 5, 83, 73, 71, 78, 32, 87, 67, 30, 82, 67, 130, 146, 22, 65, 38, 85, + 22, 86, 186, 201, 1, 73, 222, 137, 2, 69, 3, 79, 4, 245, 210, 21, 5, 65, + 78, 68, 82, 65, 126, 108, 7, 76, 69, 84, 84, 69, 82, 32, 216, 1, 5, 83, + 73, 71, 78, 32, 38, 86, 154, 241, 23, 68, 203, 224, 1, 79, 80, 130, 243, + 21, 78, 146, 23, 65, 38, 68, 114, 84, 230, 5, 85, 206, 201, 1, 73, 42, + 76, 246, 193, 1, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 206, 40, 79, 162, + 8, 69, 158, 20, 72, 2, 77, 2, 82, 2, 83, 2, 86, 3, 89, 4, 178, 172, 23, + 86, 179, 241, 1, 65, 20, 190, 7, 79, 163, 162, 23, 73, 160, 2, 84, 6, 77, + 85, 75, 72, 73, 32, 189, 7, 10, 85, 78, 71, 32, 75, 72, 69, 77, 65, 32, + 172, 1, 194, 1, 65, 44, 7, 76, 69, 84, 84, 69, 82, 32, 238, 1, 83, 228, + 2, 2, 86, 79, 172, 144, 13, 3, 84, 73, 80, 130, 237, 5, 73, 216, 170, 2, + 5, 69, 75, 32, 79, 78, 214, 193, 2, 68, 227, 172, 1, 85, 4, 218, 200, 21, + 66, 213, 225, 1, 2, 68, 68, 96, 162, 194, 8, 71, 2, 75, 134, 195, 13, 65, + 38, 68, 82, 82, 34, 84, 230, 5, 85, 206, 201, 1, 73, 42, 76, 250, 192, 1, + 78, 126, 66, 2, 67, 2, 74, 2, 80, 2, 83, 206, 40, 79, 162, 8, 69, 158, + 20, 70, 2, 72, 2, 77, 2, 86, 2, 89, 3, 90, 26, 108, 19, 69, 81, 85, 69, + 78, 67, 69, 32, 70, 79, 82, 32, 76, 69, 84, 84, 69, 82, 32, 93, 4, 73, + 71, 78, 32, 12, 70, 71, 2, 75, 230, 225, 23, 83, 170, 216, 1, 76, 226, + 31, 70, 3, 90, 2, 227, 225, 23, 72, 14, 128, 1, 6, 65, 68, 65, 75, 32, + 66, 2, 66, 176, 140, 15, 2, 85, 68, 254, 186, 6, 78, 180, 171, 2, 3, 89, + 65, 75, 163, 165, 1, 86, 2, 255, 147, 8, 73, 18, 45, 9, 87, 69, 76, 32, + 83, 73, 71, 78, 32, 18, 250, 134, 22, 65, 38, 85, 206, 201, 1, 73, 234, + 234, 1, 79, 163, 8, 69, 116, 216, 1, 22, 67, 79, 78, 83, 79, 78, 65, 78, + 84, 32, 83, 73, 71, 78, 32, 77, 69, 68, 73, 65, 76, 32, 44, 7, 76, 69, + 84, 84, 69, 82, 32, 168, 1, 5, 83, 73, 71, 78, 32, 56, 6, 86, 79, 87, 69, + 76, 32, 251, 228, 23, 68, 8, 234, 213, 25, 72, 2, 82, 2, 86, 3, 89, 60, + 198, 230, 21, 78, 182, 23, 68, 114, 84, 206, 145, 3, 66, 2, 67, 2, 71, 2, + 74, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, 2, 82, 2, 83, 2, 86, 2, 89, + 187, 2, 65, 4, 214, 145, 25, 65, 233, 35, 6, 84, 72, 79, 76, 72, 79, 24, + 130, 148, 20, 83, 155, 128, 5, 76, 162, 23, 110, 65, 174, 95, 69, 246, + 103, 73, 146, 9, 79, 158, 12, 84, 30, 85, 130, 1, 89, 249, 236, 12, 4, + 82, 89, 86, 78, 138, 11, 236, 1, 2, 73, 82, 60, 8, 76, 70, 87, 73, 68, + 84, 72, 32, 242, 10, 77, 210, 1, 78, 236, 76, 21, 80, 80, 89, 32, 80, 69, + 82, 83, 79, 78, 32, 82, 65, 73, 83, 73, 78, 71, 32, 79, 78, 22, 82, 38, + 84, 252, 242, 17, 2, 85, 77, 251, 241, 5, 68, 6, 26, 32, 163, 217, 24, + 67, 4, 242, 162, 24, 80, 211, 113, 83, 244, 1, 140, 2, 7, 72, 65, 78, 71, + 85, 76, 32, 216, 4, 8, 75, 65, 84, 65, 75, 65, 78, 65, 204, 3, 3, 76, 69, + 70, 0, 4, 82, 73, 71, 72, 140, 190, 6, 11, 70, 79, 82, 77, 83, 32, 76, + 73, 71, 72, 84, 142, 129, 4, 85, 174, 218, 2, 73, 250, 184, 4, 66, 242, + 227, 4, 87, 211, 35, 68, 104, 52, 7, 76, 69, 84, 84, 69, 82, 32, 167, + 217, 10, 70, 102, 206, 1, 75, 28, 5, 78, 73, 69, 85, 78, 42, 80, 24, 5, + 82, 73, 69, 85, 76, 86, 83, 98, 89, 202, 61, 67, 54, 69, 30, 73, 242, 4, + 77, 138, 1, 84, 206, 3, 87, 198, 1, 72, 230, 194, 24, 65, 2, 79, 163, 64, + 85, 6, 222, 65, 72, 155, 3, 73, 7, 11, 45, 4, 134, 72, 67, 131, 3, 72, 6, + 146, 69, 72, 35, 73, 17, 11, 45, 14, 206, 49, 84, 226, 14, 80, 130, 4, + 77, 194, 3, 75, 218, 2, 72, 99, 83, 12, 40, 4, 83, 65, 78, 71, 151, 221, + 13, 73, 10, 210, 70, 67, 42, 75, 74, 80, 34, 84, 211, 2, 83, 14, 194, + 134, 23, 69, 194, 133, 2, 65, 162, 64, 73, 2, 79, 3, 85, 118, 70, 32, + 193, 240, 2, 11, 45, 72, 73, 82, 65, 71, 65, 78, 65, 32, 80, 116, 76, 7, + 76, 69, 84, 84, 69, 82, 32, 150, 240, 2, 83, 34, 86, 215, 200, 21, 77, + 110, 146, 1, 83, 138, 234, 2, 78, 150, 2, 72, 2, 75, 2, 77, 2, 82, 2, 84, + 170, 1, 89, 222, 41, 87, 154, 178, 22, 65, 2, 69, 2, 73, 2, 79, 3, 85, + 28, 76, 5, 77, 65, 76, 76, 32, 234, 200, 25, 65, 2, 69, 2, 73, 2, 79, 3, + 85, 18, 242, 236, 2, 89, 142, 183, 22, 84, 234, 36, 65, 2, 69, 2, 73, 2, + 79, 3, 85, 4, 11, 84, 4, 212, 157, 13, 2, 32, 67, 239, 138, 11, 87, 14, + 56, 3, 77, 69, 82, 106, 83, 233, 248, 21, 3, 66, 85, 82, 9, 29, 5, 32, + 65, 78, 68, 32, 6, 168, 170, 12, 3, 87, 82, 69, 144, 228, 3, 2, 83, 73, + 207, 136, 8, 80, 4, 152, 137, 25, 3, 84, 69, 82, 163, 61, 65, 194, 8, + 118, 68, 202, 2, 71, 128, 66, 13, 73, 70, 73, 32, 82, 79, 72, 73, 78, 71, + 89, 65, 32, 165, 5, 5, 85, 78, 79, 79, 32, 10, 100, 12, 32, 87, 73, 84, + 72, 32, 73, 78, 68, 69, 88, 32, 188, 1, 2, 66, 65, 193, 218, 21, 2, 83, + 72, 4, 156, 1, 18, 65, 78, 68, 32, 77, 73, 68, 68, 76, 69, 32, 70, 73, + 78, 71, 69, 82, 83, 1, 16, 70, 73, 78, 71, 69, 82, 32, 65, 78, 68, 32, + 84, 72, 85, 77, 66, 2, 233, 134, 16, 2, 32, 67, 4, 186, 185, 24, 76, 183, + 137, 1, 71, 170, 7, 84, 3, 85, 76, 32, 217, 64, 13, 90, 72, 79, 85, 32, + 78, 85, 77, 69, 82, 65, 76, 32, 146, 7, 164, 1, 9, 67, 72, 79, 83, 69, + 79, 78, 71, 32, 244, 15, 4, 68, 79, 85, 66, 0, 4, 83, 73, 78, 71, 46, 74, + 180, 31, 7, 76, 69, 84, 84, 69, 82, 32, 147, 154, 10, 70, 250, 1, 246, 1, + 67, 172, 2, 5, 73, 69, 85, 78, 71, 146, 1, 75, 132, 1, 5, 77, 73, 69, 85, + 77, 56, 5, 78, 73, 69, 85, 78, 74, 80, 172, 2, 5, 82, 73, 69, 85, 76, + 210, 1, 83, 166, 3, 84, 124, 2, 89, 69, 200, 40, 5, 72, 73, 69, 85, 72, + 199, 145, 10, 70, 30, 76, 2, 72, 73, 84, 7, 69, 79, 78, 71, 67, 72, 73, + 121, 4, 73, 69, 85, 67, 16, 40, 4, 69, 85, 67, 72, 41, 2, 84, 85, 7, 11, + 45, 4, 202, 31, 75, 243, 26, 72, 10, 21, 3, 69, 85, 77, 10, 22, 83, 155, + 46, 67, 6, 40, 4, 83, 65, 78, 71, 135, 205, 13, 73, 4, 194, 54, 67, 227, + 3, 83, 5, 215, 30, 45, 27, 11, 45, 24, 90, 80, 234, 30, 82, 242, 13, 67, + 194, 5, 77, 138, 1, 84, 186, 2, 75, 218, 2, 72, 99, 83, 6, 214, 50, 72, + 214, 3, 73, 207, 2, 65, 16, 80, 7, 65, 80, 89, 69, 79, 85, 78, 228, 20, + 5, 73, 89, 69, 79, 75, 131, 25, 72, 10, 234, 29, 82, 178, 15, 80, 30, 83, + 231, 3, 77, 11, 11, 45, 8, 158, 52, 75, 74, 80, 34, 84, 211, 2, 83, 15, + 11, 45, 12, 190, 51, 67, 42, 75, 74, 80, 34, 84, 242, 1, 72, 99, 83, 42, + 68, 6, 72, 73, 69, 85, 80, 72, 40, 4, 73, 69, 85, 80, 223, 53, 65, 7, 11, + 45, 4, 158, 51, 80, 147, 2, 72, 35, 11, 45, 32, 82, 83, 234, 20, 80, 214, + 7, 75, 162, 12, 67, 202, 6, 84, 226, 2, 78, 179, 2, 72, 14, 32, 3, 73, + 79, 83, 251, 3, 83, 13, 11, 45, 10, 242, 46, 84, 146, 2, 67, 42, 75, 75, + 80, 29, 11, 45, 26, 78, 75, 42, 83, 190, 44, 77, 154, 3, 67, 82, 78, 34, + 80, 34, 84, 243, 1, 72, 6, 170, 22, 65, 130, 19, 72, 135, 7, 73, 8, 40, + 4, 83, 65, 78, 71, 235, 197, 13, 73, 6, 206, 47, 75, 74, 80, 35, 84, 58, + 48, 3, 73, 79, 83, 217, 1, 4, 83, 65, 78, 71, 33, 11, 45, 30, 130, 1, 80, + 44, 2, 83, 83, 210, 22, 82, 210, 1, 75, 162, 12, 67, 194, 5, 77, 138, 1, + 84, 226, 2, 78, 178, 2, 72, 191, 157, 18, 73, 6, 184, 23, 4, 73, 69, 85, + 80, 179, 19, 72, 2, 233, 48, 3, 65, 78, 71, 26, 236, 17, 5, 67, 73, 69, + 85, 67, 172, 3, 4, 83, 73, 79, 83, 154, 1, 82, 186, 20, 84, 54, 89, 134, + 2, 75, 42, 78, 34, 80, 146, 2, 72, 191, 157, 18, 73, 16, 40, 5, 73, 75, + 69, 85, 84, 195, 41, 72, 15, 11, 45, 12, 226, 20, 82, 178, 19, 77, 154, + 3, 67, 42, 75, 74, 80, 243, 2, 83, 4, 150, 19, 83, 139, 22, 79, 2, 169, + 245, 8, 6, 76, 69, 32, 68, 79, 84, 216, 3, 92, 9, 79, 78, 71, 83, 69, 79, + 78, 71, 32, 165, 21, 9, 85, 78, 71, 83, 69, 79, 78, 71, 32, 154, 2, 226, + 1, 67, 80, 5, 72, 73, 69, 85, 72, 60, 5, 73, 69, 85, 78, 71, 46, 75, 220, + 1, 5, 77, 73, 69, 85, 77, 188, 1, 5, 78, 73, 69, 85, 78, 94, 80, 240, 2, + 5, 82, 73, 69, 85, 76, 190, 4, 83, 194, 3, 84, 213, 1, 2, 89, 69, 8, 36, + 4, 73, 69, 85, 67, 239, 30, 72, 7, 11, 45, 4, 162, 32, 83, 239, 7, 80, + 11, 11, 45, 8, 174, 16, 82, 178, 19, 77, 234, 3, 78, 35, 80, 9, 11, 45, + 6, 194, 17, 75, 57, 2, 83, 83, 28, 76, 7, 65, 80, 89, 69, 79, 85, 78, 40, + 5, 73, 89, 69, 79, 75, 215, 30, 72, 8, 130, 15, 82, 178, 15, 80, 131, 4, + 77, 19, 11, 45, 16, 166, 13, 75, 170, 1, 82, 42, 83, 224, 13, 2, 67, 72, + 146, 9, 78, 34, 80, 147, 2, 72, 27, 11, 45, 24, 74, 80, 30, 83, 134, 13, + 82, 242, 13, 67, 130, 9, 75, 42, 78, 179, 2, 72, 6, 174, 33, 73, 131, 6, + 65, 6, 40, 4, 83, 65, 78, 71, 227, 185, 13, 73, 4, 238, 35, 78, 147, 3, + 83, 21, 11, 45, 18, 174, 12, 82, 242, 13, 67, 202, 6, 84, 186, 2, 75, + 218, 2, 72, 62, 80, 39, 83, 36, 88, 6, 65, 78, 83, 73, 79, 83, 48, 6, 72, + 73, 69, 85, 80, 72, 53, 4, 73, 69, 85, 80, 7, 11, 45, 4, 236, 7, 2, 75, + 65, 195, 26, 80, 9, 11, 45, 6, 150, 11, 84, 234, 22, 80, 243, 2, 83, 23, + 11, 45, 20, 112, 5, 82, 73, 69, 85, 76, 24, 4, 83, 73, 79, 83, 134, 3, + 80, 246, 19, 67, 194, 5, 77, 170, 4, 84, 243, 1, 72, 5, 153, 3, 2, 45, + 80, 5, 221, 32, 2, 45, 84, 57, 11, 45, 54, 102, 75, 92, 5, 77, 73, 69, + 85, 77, 50, 80, 126, 83, 74, 84, 44, 2, 89, 69, 154, 28, 78, 179, 2, 72, + 10, 52, 5, 73, 89, 69, 79, 75, 190, 4, 65, 131, 19, 72, 7, 11, 45, 4, + 254, 32, 72, 99, 83, 9, 11, 45, 6, 130, 30, 75, 218, 2, 72, 99, 83, 14, + 48, 4, 73, 69, 85, 80, 174, 26, 72, 163, 6, 65, 11, 11, 45, 8, 42, 80, + 222, 29, 84, 242, 1, 72, 99, 83, 2, 243, 25, 72, 6, 40, 4, 83, 65, 78, + 71, 211, 178, 13, 73, 4, 182, 28, 75, 187, 3, 83, 6, 100, 5, 73, 75, 69, + 85, 84, 151, 25, 72, 6, 56, 9, 79, 82, 73, 78, 72, 73, 69, 85, 72, 191, + 3, 83, 5, 255, 29, 45, 52, 48, 3, 73, 79, 83, 161, 1, 4, 83, 65, 78, 71, + 25, 11, 45, 22, 82, 75, 162, 3, 82, 242, 13, 67, 198, 2, 80, 254, 2, 77, + 138, 1, 84, 147, 5, 72, 4, 22, 65, 135, 26, 73, 2, 237, 18, 6, 80, 89, + 69, 79, 85, 78, 28, 160, 1, 5, 82, 73, 69, 85, 76, 36, 6, 84, 73, 75, 69, + 85, 84, 16, 3, 89, 69, 83, 138, 19, 83, 178, 1, 77, 154, 3, 67, 42, 75, + 42, 78, 34, 80, 207, 159, 18, 73, 5, 17, 2, 45, 75, 2, 159, 17, 72, 5, + 255, 16, 45, 2, 139, 184, 18, 73, 20, 40, 5, 73, 75, 69, 85, 84, 155, 21, + 72, 19, 11, 45, 16, 58, 82, 42, 83, 42, 84, 162, 13, 67, 130, 9, 75, 75, + 80, 2, 17, 2, 73, 69, 2, 131, 147, 24, 85, 4, 21, 3, 73, 79, 83, 5, 223, + 1, 45, 2, 255, 19, 72, 18, 44, 6, 83, 73, 69, 85, 78, 71, 243, 19, 79, + 17, 11, 45, 14, 50, 75, 30, 83, 198, 17, 77, 154, 6, 72, 63, 80, 4, 166, + 14, 72, 135, 7, 73, 4, 26, 83, 131, 171, 13, 73, 2, 21, 3, 65, 78, 71, 2, + 207, 20, 75, 190, 1, 122, 65, 118, 69, 134, 1, 73, 92, 7, 83, 83, 65, 78, + 71, 65, 82, 106, 79, 138, 1, 85, 102, 89, 174, 15, 87, 235, 141, 10, 70, + 23, 48, 4, 82, 65, 69, 65, 98, 45, 139, 152, 25, 69, 13, 11, 45, 10, 250, + 210, 22, 69, 226, 197, 2, 65, 2, 73, 3, 85, 25, 18, 79, 55, 85, 9, 11, + 45, 6, 158, 243, 24, 69, 234, 36, 79, 3, 85, 15, 11, 45, 12, 170, 9, 69, + 170, 142, 25, 65, 2, 79, 3, 85, 31, 11, 45, 28, 66, 65, 34, 89, 166, 1, + 79, 170, 240, 24, 69, 234, 36, 73, 3, 85, 5, 11, 82, 2, 215, 144, 18, 65, + 14, 50, 65, 162, 208, 22, 69, 226, 197, 2, 79, 3, 85, 7, 138, 247, 24, + 45, 247, 30, 69, 23, 26, 45, 199, 149, 25, 69, 18, 50, 79, 22, 89, 158, + 207, 22, 69, 227, 197, 2, 85, 5, 183, 129, 25, 45, 8, 154, 207, 22, 69, + 195, 133, 2, 65, 17, 11, 45, 14, 158, 19, 89, 186, 164, 5, 73, 152, 254, + 12, 3, 69, 79, 45, 190, 158, 6, 65, 163, 64, 85, 62, 42, 65, 70, 69, 66, + 73, 22, 79, 107, 85, 11, 26, 45, 175, 147, 25, 69, 6, 182, 244, 24, 89, + 246, 30, 79, 3, 85, 11, 11, 79, 9, 11, 45, 6, 178, 144, 25, 89, 186, 2, + 79, 3, 85, 5, 219, 237, 24, 45, 19, 11, 45, 16, 58, 89, 202, 209, 24, 65, + 174, 33, 69, 246, 30, 73, 3, 79, 6, 198, 209, 24, 65, 175, 33, 69, 21, + 11, 45, 18, 142, 16, 89, 206, 187, 22, 69, 194, 133, 2, 65, 162, 64, 73, + 2, 79, 3, 85, 186, 1, 226, 1, 65, 46, 67, 54, 69, 30, 73, 22, 75, 188, 1, + 5, 77, 73, 69, 85, 77, 64, 5, 78, 73, 69, 85, 78, 70, 80, 168, 1, 5, 82, + 73, 69, 85, 76, 254, 1, 84, 90, 83, 246, 2, 87, 50, 89, 150, 1, 72, 230, + 194, 24, 79, 163, 64, 85, 9, 156, 13, 3, 82, 65, 69, 235, 129, 25, 69, 4, + 22, 72, 207, 8, 73, 2, 141, 236, 24, 2, 73, 69, 7, 166, 142, 25, 79, 3, + 85, 5, 207, 168, 18, 69, 14, 56, 7, 65, 80, 89, 69, 79, 85, 78, 106, 72, + 155, 3, 73, 8, 30, 80, 30, 83, 231, 3, 77, 4, 190, 4, 72, 215, 3, 73, 2, + 25, 4, 83, 65, 78, 71, 2, 207, 7, 80, 2, 213, 60, 2, 73, 69, 9, 11, 45, + 6, 22, 80, 247, 9, 83, 4, 142, 7, 73, 207, 2, 65, 13, 11, 45, 10, 234, 5, + 67, 146, 1, 84, 242, 1, 72, 62, 80, 39, 83, 20, 48, 4, 73, 69, 85, 80, + 170, 2, 72, 163, 6, 65, 17, 11, 45, 14, 42, 83, 186, 2, 84, 146, 2, 67, + 43, 75, 6, 21, 3, 73, 79, 83, 7, 11, 45, 4, 202, 4, 75, 107, 84, 27, 11, + 45, 24, 68, 2, 75, 73, 34, 77, 34, 80, 106, 84, 54, 89, 222, 4, 72, 99, + 83, 4, 149, 1, 4, 89, 69, 79, 75, 2, 11, 73, 2, 255, 223, 23, 69, 8, 30, + 72, 34, 73, 131, 6, 65, 2, 193, 228, 18, 3, 73, 69, 85, 4, 21, 3, 69, 85, + 80, 5, 243, 5, 45, 4, 22, 72, 151, 3, 73, 2, 225, 230, 21, 2, 73, 69, 2, + 17, 2, 69, 79, 2, 167, 4, 82, 28, 44, 3, 73, 79, 83, 57, 4, 83, 65, 78, + 71, 13, 11, 45, 10, 122, 67, 42, 75, 42, 78, 34, 80, 35, 84, 16, 78, 67, + 42, 75, 42, 78, 34, 80, 34, 84, 242, 1, 72, 98, 83, 223, 156, 18, 73, 2, + 11, 73, 2, 249, 221, 23, 2, 69, 85, 2, 11, 73, 2, 237, 219, 24, 2, 89, + 69, 2, 11, 73, 2, 247, 132, 24, 69, 2, 11, 73, 2, 147, 143, 24, 69, 2, + 11, 73, 2, 11, 75, 2, 139, 139, 24, 69, 10, 230, 190, 22, 69, 194, 133, + 2, 65, 163, 64, 73, 34, 58, 69, 206, 1, 79, 62, 85, 182, 193, 24, 65, + 163, 64, 73, 13, 42, 79, 73, 6, 83, 73, 69, 85, 78, 71, 5, 11, 82, 2, 17, + 2, 73, 78, 2, 11, 72, 2, 161, 235, 9, 2, 73, 69, 7, 11, 45, 4, 18, 80, + 39, 83, 2, 11, 65, 2, 11, 78, 2, 11, 83, 2, 223, 146, 13, 73, 9, 11, 45, + 6, 26, 89, 235, 129, 25, 73, 4, 199, 193, 24, 65, 9, 11, 45, 6, 26, 89, + 175, 129, 25, 73, 4, 203, 187, 22, 69, 24, 210, 136, 11, 84, 162, 136, + 12, 70, 30, 83, 210, 86, 78, 14, 79, 223, 110, 69, 100, 156, 1, 7, 76, + 69, 84, 84, 69, 82, 32, 196, 2, 5, 77, 65, 82, 75, 32, 72, 5, 83, 73, 71, + 78, 32, 132, 159, 2, 6, 86, 79, 87, 69, 76, 32, 255, 235, 20, 68, 58, + 202, 1, 68, 34, 75, 254, 241, 20, 84, 154, 50, 82, 130, 223, 1, 78, 238, + 178, 1, 83, 138, 69, 66, 2, 67, 2, 70, 2, 71, 2, 72, 2, 74, 2, 76, 2, 77, + 2, 80, 2, 86, 2, 87, 2, 89, 2, 90, 187, 2, 65, 4, 166, 251, 24, 68, 187, + 2, 65, 8, 56, 5, 73, 78, 78, 65, 32, 206, 250, 24, 72, 187, 2, 65, 4, + 202, 250, 24, 87, 3, 89, 4, 248, 197, 21, 6, 78, 65, 32, 75, 72, 79, 153, + 183, 2, 3, 83, 65, 75, 8, 52, 2, 84, 65, 217, 238, 22, 5, 72, 65, 82, 66, + 65, 6, 42, 72, 158, 145, 20, 83, 235, 231, 4, 78, 2, 163, 217, 24, 65, + 42, 62, 76, 144, 239, 18, 6, 83, 73, 71, 78, 32, 80, 247, 1, 86, 36, 33, + 6, 69, 84, 84, 69, 82, 32, 36, 146, 137, 21, 78, 250, 238, 3, 66, 2, 68, + 2, 71, 2, 72, 2, 75, 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, + 89, 186, 2, 65, 2, 73, 3, 85, 2, 139, 235, 23, 69, 4, 130, 180, 23, 68, + 143, 197, 1, 80, 54, 52, 5, 67, 72, 73, 78, 71, 41, 4, 82, 65, 78, 32, 2, + 17, 2, 32, 67, 2, 163, 200, 23, 72, 52, 52, 7, 76, 69, 84, 84, 69, 82, + 32, 175, 179, 6, 78, 42, 218, 1, 65, 234, 173, 11, 90, 162, 47, 84, 150, + 119, 76, 50, 81, 60, 6, 68, 65, 76, 69, 84, 72, 146, 165, 4, 71, 122, 83, + 66, 89, 158, 208, 1, 72, 234, 5, 75, 198, 75, 66, 178, 216, 4, 78, 134, + 2, 87, 218, 103, 80, 171, 4, 77, 4, 130, 250, 16, 76, 151, 172, 7, 89, + 172, 9, 230, 1, 65, 240, 32, 5, 66, 82, 69, 87, 32, 210, 32, 76, 200, 1, + 16, 78, 84, 65, 73, 71, 65, 78, 65, 32, 76, 69, 84, 84, 69, 82, 32, 174, + 12, 82, 120, 11, 88, 65, 71, 82, 65, 77, 32, 70, 79, 82, 32, 221, 162, + 24, 4, 68, 71, 69, 72, 214, 1, 42, 68, 98, 82, 201, 1, 3, 86, 89, 32, 6, + 44, 5, 83, 84, 79, 78, 69, 187, 200, 23, 80, 5, 217, 225, 2, 7, 32, 71, + 82, 65, 86, 69, 89, 12, 32, 2, 84, 32, 139, 151, 17, 45, 10, 60, 5, 87, + 73, 84, 72, 32, 246, 163, 12, 68, 155, 143, 9, 72, 6, 76, 9, 84, 73, 80, + 32, 79, 78, 32, 84, 72, 174, 242, 12, 82, 191, 228, 10, 65, 2, 223, 190, + 24, 69, 196, 1, 134, 2, 65, 202, 1, 66, 230, 2, 67, 154, 3, 68, 162, 1, + 69, 186, 3, 70, 94, 72, 62, 76, 222, 1, 77, 110, 79, 162, 1, 82, 142, 2, + 83, 228, 1, 3, 78, 79, 82, 198, 1, 84, 128, 2, 2, 85, 80, 174, 1, 87, + 138, 140, 14, 73, 254, 238, 3, 80, 210, 131, 4, 86, 251, 77, 71, 12, 108, + 17, 82, 82, 79, 87, 32, 83, 72, 65, 70, 84, 32, 87, 73, 68, 84, 72, 32, + 182, 158, 18, 77, 155, 188, 4, 83, 8, 36, 3, 79, 78, 69, 223, 141, 23, + 84, 7, 11, 32, 4, 174, 238, 13, 84, 135, 252, 8, 72, 14, 48, 4, 65, 76, + 76, 79, 21, 4, 76, 65, 67, 75, 2, 187, 167, 5, 84, 12, 30, 32, 153, 1, 2, + 45, 70, 6, 52, 7, 67, 85, 82, 86, 69, 68, 32, 247, 228, 23, 72, 4, 40, 4, + 68, 79, 87, 78, 1, 2, 85, 80, 2, 213, 203, 23, 8, 87, 65, 82, 68, 83, 32, + 65, 78, 6, 45, 9, 69, 65, 84, 72, 69, 82, 69, 68, 32, 6, 234, 211, 13, + 83, 218, 168, 3, 78, 211, 206, 6, 82, 14, 116, 2, 72, 69, 52, 5, 73, 82, + 67, 76, 69, 149, 220, 17, 14, 79, 78, 67, 65, 86, 69, 45, 80, 79, 73, 78, + 84, 69, 68, 4, 216, 203, 20, 4, 86, 82, 79, 78, 239, 218, 2, 67, 9, 64, + 6, 32, 87, 73, 84, 72, 32, 133, 212, 22, 4, 68, 32, 83, 65, 4, 52, 7, 83, + 84, 82, 79, 75, 69, 32, 199, 254, 4, 67, 2, 29, 5, 65, 78, 68, 32, 84, 2, + 11, 87, 2, 11, 79, 2, 21, 3, 32, 68, 79, 2, 11, 84, 2, 207, 234, 23, 83, + 14, 52, 7, 65, 83, 72, 69, 68, 32, 84, 18, 73, 31, 79, 2, 175, 16, 82, 2, + 137, 144, 19, 2, 86, 73, 10, 192, 16, 3, 87, 78, 87, 198, 139, 14, 85, + 187, 180, 4, 76, 16, 120, 5, 73, 71, 72, 84, 32, 156, 2, 16, 88, 67, 76, + 65, 77, 65, 84, 73, 79, 78, 32, 77, 65, 82, 75, 32, 147, 205, 18, 81, 10, + 40, 2, 80, 79, 126, 84, 231, 207, 22, 83, 6, 33, 6, 73, 78, 84, 69, 68, + 32, 6, 202, 186, 16, 80, 196, 147, 6, 11, 82, 69, 67, 84, 73, 76, 73, 78, + 69, 65, 82, 23, 66, 2, 141, 208, 22, 24, 69, 65, 82, 68, 82, 79, 80, 45, + 83, 80, 79, 75, 69, 68, 32, 80, 82, 79, 80, 69, 76, 76, 69, 82, 4, 246, + 207, 23, 83, 199, 79, 79, 6, 44, 5, 79, 85, 82, 32, 66, 139, 254, 4, 73, + 2, 209, 131, 11, 6, 65, 76, 76, 79, 79, 78, 4, 242, 186, 17, 79, 169, + 221, 5, 6, 69, 65, 82, 84, 32, 69, 20, 74, 65, 44, 3, 69, 70, 84, 28, 2, + 79, 87, 149, 249, 4, 3, 73, 71, 65, 4, 172, 207, 21, 2, 82, 71, 147, 208, + 1, 84, 8, 254, 3, 45, 195, 6, 87, 6, 26, 32, 223, 143, 10, 69, 4, 154, + 150, 14, 68, 25, 4, 83, 73, 78, 71, 4, 56, 8, 85, 76, 84, 73, 80, 76, 73, + 67, 235, 176, 21, 73, 2, 25, 4, 65, 84, 73, 79, 2, 207, 153, 5, 78, 6, + 132, 140, 6, 9, 80, 69, 78, 32, 67, 69, 78, 84, 82, 144, 175, 10, 13, 86, + 65, 76, 32, 87, 73, 84, 72, 32, 79, 86, 65, 76, 153, 141, 6, 5, 85, 84, + 76, 73, 78, 12, 76, 4, 73, 71, 72, 84, 245, 188, 23, 9, 79, 85, 78, 68, + 45, 84, 73, 80, 80, 10, 62, 45, 109, 11, 87, 65, 82, 68, 83, 32, 65, 82, + 82, 79, 87, 4, 69, 15, 80, 79, 73, 78, 84, 73, 78, 71, 32, 65, 78, 71, + 76, 69, 32, 4, 222, 229, 6, 66, 147, 173, 7, 81, 7, 139, 6, 32, 26, 106, + 65, 78, 73, 44, 2, 79, 85, 160, 154, 14, 7, 67, 82, 73, 80, 84, 32, 76, + 157, 135, 1, 3, 80, 65, 82, 4, 236, 145, 14, 10, 78, 83, 45, 83, 69, 82, + 73, 70, 32, 73, 171, 180, 8, 76, 8, 216, 143, 14, 2, 78, 71, 179, 179, 8, + 88, 10, 21, 3, 84, 72, 32, 10, 60, 5, 69, 65, 83, 84, 32, 29, 6, 87, 69, + 83, 84, 32, 80, 6, 26, 80, 203, 190, 23, 65, 4, 41, 8, 79, 73, 78, 84, + 73, 78, 71, 32, 4, 242, 237, 16, 86, 139, 191, 6, 66, 12, 88, 9, 69, 65, + 82, 68, 82, 79, 80, 45, 83, 130, 1, 82, 241, 225, 6, 4, 87, 69, 76, 86, + 6, 64, 6, 80, 79, 75, 69, 68, 32, 241, 182, 23, 4, 72, 65, 78, 75, 4, + 164, 195, 22, 8, 80, 73, 78, 87, 72, 69, 69, 76, 27, 65, 2, 193, 3, 5, + 73, 65, 78, 71, 76, 6, 26, 87, 203, 133, 10, 80, 4, 53, 11, 65, 82, 68, + 83, 32, 65, 82, 82, 79, 87, 32, 4, 29, 5, 87, 73, 84, 72, 32, 4, 192, + 251, 19, 5, 76, 65, 82, 71, 69, 203, 238, 1, 69, 12, 76, 5, 72, 73, 84, + 69, 32, 164, 1, 2, 73, 68, 237, 236, 22, 3, 69, 68, 71, 8, 64, 6, 83, 81, + 85, 65, 82, 69, 166, 214, 17, 68, 215, 171, 5, 67, 5, 173, 161, 23, 19, + 32, 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, 65, 67, 75, 32, + 86, 2, 157, 232, 20, 2, 69, 45, 140, 2, 112, 7, 65, 67, 67, 69, 78, 84, + 32, 138, 8, 76, 164, 14, 5, 77, 65, 82, 75, 32, 114, 80, 217, 157, 21, 2, + 89, 79, 60, 128, 2, 9, 65, 84, 78, 65, 72, 32, 72, 65, 70, 22, 68, 36, 3, + 71, 69, 82, 74, 77, 124, 2, 80, 65, 28, 4, 69, 84, 78, 65, 20, 2, 81, 65, + 58, 83, 58, 84, 156, 1, 2, 89, 69, 78, 90, 196, 165, 8, 3, 82, 69, 86, + 142, 129, 15, 79, 225, 146, 1, 3, 73, 76, 85, 2, 243, 157, 18, 85, 4, + 230, 128, 19, 69, 179, 140, 5, 65, 6, 32, 3, 69, 83, 72, 179, 28, 83, 5, + 129, 221, 6, 4, 32, 77, 85, 81, 8, 84, 5, 69, 82, 75, 72, 65, 164, 238, + 17, 2, 85, 78, 249, 45, 5, 65, 72, 65, 80, 65, 5, 209, 219, 19, 4, 32, + 75, 69, 70, 4, 26, 83, 251, 143, 24, 90, 2, 151, 169, 24, 72, 4, 128, + 137, 24, 6, 82, 78, 69, 89, 32, 80, 191, 35, 68, 4, 182, 24, 69, 193, + 213, 22, 6, 72, 65, 76, 83, 72, 69, 8, 38, 69, 249, 209, 22, 3, 73, 80, + 69, 6, 48, 6, 76, 73, 83, 72, 65, 32, 207, 224, 11, 86, 4, 240, 155, 22, + 3, 81, 69, 84, 249, 141, 2, 4, 71, 69, 68, 79, 4, 128, 205, 8, 10, 82, + 65, 72, 32, 66, 69, 78, 32, 89, 79, 255, 221, 2, 84, 8, 34, 65, 237, 145, + 24, 2, 73, 78, 6, 40, 4, 81, 69, 70, 32, 183, 255, 5, 82, 4, 154, 135, + 11, 81, 157, 176, 12, 3, 71, 65, 68, 150, 1, 76, 6, 69, 84, 84, 69, 82, + 32, 201, 11, 8, 73, 71, 65, 84, 85, 82, 69, 32, 140, 1, 134, 3, 65, 204, + 1, 3, 66, 69, 84, 0, 3, 75, 65, 70, 0, 2, 80, 69, 68, 6, 70, 73, 78, 65, + 76, 32, 92, 2, 81, 79, 16, 2, 72, 69, 52, 2, 78, 85, 0, 4, 90, 65, 89, + 73, 18, 83, 48, 3, 82, 69, 83, 170, 1, 84, 52, 4, 68, 65, 76, 69, 12, 5, + 71, 73, 77, 69, 76, 0, 5, 76, 65, 77, 69, 68, 0, 3, 77, 69, 77, 48, 3, + 86, 65, 86, 80, 5, 87, 73, 68, 69, 32, 153, 1, 3, 89, 79, 68, 14, 26, 76, + 171, 198, 23, 89, 12, 64, 2, 69, 70, 73, 10, 84, 69, 82, 78, 65, 84, 73, + 86, 69, 32, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 130, 13, 77, 130, 1, 80, + 35, 81, 4, 242, 201, 16, 65, 251, 160, 1, 80, 7, 33, 6, 32, 87, 73, 84, + 72, 32, 4, 142, 15, 82, 199, 225, 13, 68, 14, 88, 2, 75, 65, 236, 2, 2, + 80, 69, 220, 153, 1, 2, 84, 83, 182, 166, 22, 78, 135, 110, 77, 4, 235, + 2, 70, 7, 244, 10, 5, 32, 87, 73, 84, 72, 167, 184, 24, 84, 4, 167, 2, + 78, 16, 44, 4, 65, 77, 69, 75, 17, 3, 72, 73, 78, 4, 231, 1, 72, 13, 33, + 6, 32, 87, 73, 84, 72, 32, 10, 40, 6, 68, 65, 71, 69, 83, 72, 39, 83, 7, + 33, 6, 32, 65, 78, 68, 32, 83, 4, 254, 254, 5, 72, 203, 192, 2, 73, 12, + 50, 69, 12, 2, 65, 86, 1, 4, 83, 65, 68, 73, 4, 11, 84, 5, 233, 236, 13, + 7, 32, 87, 73, 84, 72, 32, 68, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 148, + 253, 1, 2, 72, 79, 131, 239, 11, 68, 16, 174, 2, 76, 202, 203, 8, 65, + 210, 196, 2, 84, 186, 217, 2, 82, 160, 244, 8, 2, 68, 65, 242, 93, 75, + 222, 106, 72, 169, 4, 7, 70, 73, 78, 65, 76, 32, 77, 7, 33, 6, 32, 87, + 73, 84, 72, 32, 4, 176, 7, 2, 72, 73, 255, 226, 13, 68, 10, 72, 6, 65, + 76, 69, 70, 32, 76, 29, 8, 89, 73, 68, 68, 73, 83, 72, 32, 2, 193, 177, + 19, 2, 65, 77, 8, 120, 7, 68, 79, 85, 66, 76, 69, 32, 232, 4, 9, 89, 79, + 68, 32, 89, 79, 68, 32, 80, 185, 128, 21, 5, 86, 65, 86, 32, 89, 4, 254, + 142, 11, 86, 163, 246, 9, 89, 6, 80, 3, 76, 79, 87, 0, 3, 85, 80, 80, + 237, 232, 22, 6, 77, 65, 83, 79, 82, 65, 2, 233, 169, 23, 2, 69, 82, 50, + 84, 5, 79, 73, 78, 84, 32, 229, 5, 11, 85, 78, 67, 84, 85, 65, 84, 73, + 79, 78, 32, 38, 228, 1, 9, 68, 65, 71, 69, 83, 72, 32, 79, 82, 46, 72, + 106, 80, 166, 1, 81, 86, 82, 22, 83, 176, 209, 9, 2, 84, 83, 132, 142, + 11, 17, 74, 85, 68, 69, 79, 45, 83, 80, 65, 78, 73, 83, 72, 32, 86, 65, + 82, 233, 143, 3, 3, 77, 69, 84, 2, 17, 2, 32, 77, 2, 201, 1, 2, 65, 80, + 12, 60, 5, 65, 84, 65, 70, 32, 106, 73, 33, 4, 79, 76, 65, 77, 6, 38, 80, + 34, 81, 145, 2, 2, 83, 69, 2, 11, 65, 2, 131, 215, 17, 84, 2, 193, 162, + 23, 3, 65, 77, 65, 2, 11, 82, 2, 143, 230, 13, 73, 5, 181, 137, 11, 12, + 32, 72, 65, 83, 69, 82, 32, 70, 79, 82, 32, 86, 6, 52, 5, 65, 77, 65, 84, + 83, 189, 191, 11, 2, 85, 66, 5, 165, 242, 10, 2, 32, 81, 2, 167, 225, 6, + 65, 8, 34, 69, 22, 72, 243, 178, 8, 73, 2, 239, 161, 23, 71, 4, 238, 178, + 8, 73, 199, 209, 7, 69, 12, 152, 1, 3, 71, 69, 82, 60, 3, 80, 65, 83, 20, + 7, 83, 79, 70, 32, 80, 65, 83, 212, 249, 21, 7, 78, 85, 78, 32, 72, 65, + 70, 185, 183, 1, 3, 77, 65, 81, 4, 26, 83, 211, 202, 22, 69, 2, 157, 176, + 23, 3, 72, 65, 89, 2, 155, 226, 13, 69, 2, 135, 226, 13, 85, 8, 114, 77, + 160, 149, 12, 15, 76, 83, 67, 72, 82, 69, 73, 66, 69, 82, 32, 80, 65, 85, + 83, 185, 223, 5, 3, 73, 67, 79, 4, 176, 222, 5, 12, 69, 84, 32, 87, 73, + 84, 72, 32, 87, 72, 73, 84, 231, 191, 17, 32, 188, 4, 164, 1, 2, 65, 45, + 50, 72, 70, 75, 230, 1, 77, 114, 78, 146, 2, 82, 66, 83, 154, 1, 84, 226, + 1, 87, 62, 89, 110, 69, 134, 241, 8, 85, 190, 170, 5, 79, 143, 191, 3, + 73, 8, 190, 144, 24, 87, 246, 30, 49, 2, 50, 3, 51, 72, 166, 6, 79, 146, + 191, 8, 65, 146, 224, 5, 85, 158, 115, 69, 3, 73, 74, 72, 2, 65, 45, 104, + 2, 79, 45, 178, 4, 73, 226, 3, 69, 223, 142, 15, 85, 24, 210, 234, 11, + 49, 206, 172, 12, 75, 214, 22, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, + 56, 3, 57, 8, 178, 153, 24, 75, 218, 19, 49, 2, 50, 3, 51, 54, 68, 2, 69, + 45, 154, 7, 79, 222, 142, 15, 65, 2, 73, 243, 203, 2, 85, 6, 218, 169, + 24, 77, 186, 2, 49, 3, 50, 68, 116, 2, 69, 45, 72, 2, 73, 45, 246, 2, 65, + 194, 243, 8, 79, 190, 170, 5, 85, 169, 136, 6, 6, 45, 77, 85, 45, 77, 79, + 14, 254, 139, 24, 75, 246, 30, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 16, + 214, 147, 24, 84, 214, 22, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, + 54, 222, 4, 79, 2, 85, 222, 142, 15, 73, 242, 203, 2, 65, 3, 69, 68, 62, + 65, 2, 85, 226, 3, 73, 134, 241, 8, 69, 219, 157, 6, 79, 16, 11, 45, 16, + 206, 168, 24, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 64, + 74, 69, 20, 2, 79, 45, 72, 2, 85, 45, 190, 144, 15, 73, 243, 203, 2, 65, + 18, 139, 240, 18, 45, 14, 234, 164, 24, 82, 186, 2, 49, 2, 50, 2, 51, 2, + 52, 2, 53, 3, 54, 10, 230, 135, 24, 84, 246, 30, 49, 2, 50, 2, 51, 3, 52, + 42, 170, 242, 8, 65, 2, 73, 218, 157, 6, 79, 243, 203, 2, 69, 32, 40, 2, + 65, 45, 66, 79, 207, 218, 17, 85, 12, 198, 134, 24, 89, 246, 30, 49, 2, + 50, 2, 51, 2, 52, 3, 53, 12, 11, 45, 12, 238, 164, 24, 49, 2, 50, 2, 51, + 2, 52, 2, 53, 3, 54, 4, 140, 167, 8, 21, 77, 73, 84, 73, 65, 78, 32, 67, + 79, 78, 74, 85, 71, 65, 84, 69, 32, 77, 65, 84, 82, 167, 253, 15, 66, + 128, 1, 210, 2, 65, 110, 66, 144, 1, 2, 67, 79, 94, 68, 198, 2, 70, 66, + 71, 40, 4, 72, 79, 76, 68, 164, 1, 2, 73, 78, 116, 2, 77, 79, 58, 79, + 122, 80, 100, 2, 82, 69, 62, 83, 238, 1, 84, 210, 5, 87, 212, 155, 15, 4, + 76, 73, 77, 73, 220, 180, 5, 11, 89, 79, 85, 84, 72, 70, 85, 76, 32, 70, + 79, 181, 173, 3, 9, 69, 78, 84, 72, 85, 83, 73, 65, 83, 6, 76, 3, 80, 80, + 82, 124, 4, 70, 84, 69, 82, 189, 151, 19, 4, 66, 85, 78, 68, 2, 233, 253, + 23, 2, 79, 65, 6, 92, 5, 69, 70, 79, 82, 69, 128, 215, 3, 6, 73, 84, 73, + 78, 71, 32, 1, 4, 82, 69, 65, 75, 2, 197, 206, 23, 7, 32, 67, 79, 77, 80, + 76, 69, 6, 26, 78, 183, 142, 19, 77, 4, 228, 187, 20, 4, 84, 69, 77, 80, + 245, 211, 2, 3, 70, 76, 73, 14, 110, 69, 78, 73, 206, 146, 19, 85, 129, + 241, 2, 15, 65, 82, 75, 69, 78, 73, 78, 71, 32, 79, 70, 32, 84, 72, 69, + 6, 222, 142, 19, 67, 192, 6, 2, 76, 73, 201, 196, 4, 5, 86, 69, 76, 79, + 80, 4, 252, 132, 19, 21, 70, 70, 73, 67, 85, 76, 84, 89, 32, 65, 84, 32, + 84, 72, 69, 32, 66, 69, 71, 73, 78, 177, 248, 2, 4, 83, 80, 69, 82, 4, + 220, 137, 19, 2, 79, 76, 217, 188, 4, 5, 69, 76, 76, 79, 87, 12, 36, 5, + 65, 84, 72, 69, 82, 35, 82, 2, 225, 134, 15, 3, 73, 78, 71, 10, 40, 4, + 69, 65, 84, 32, 155, 221, 23, 65, 8, 22, 80, 211, 5, 84, 6, 22, 79, 147, + 5, 82, 4, 172, 2, 2, 83, 83, 203, 217, 23, 87, 8, 50, 78, 206, 138, 19, + 67, 217, 5, 3, 70, 76, 85, 4, 160, 144, 19, 2, 79, 67, 205, 231, 1, 5, + 69, 82, 32, 84, 82, 4, 204, 171, 22, 3, 68, 69, 83, 189, 61, 3, 85, 84, + 72, 6, 26, 66, 37, 2, 80, 80, 2, 149, 199, 23, 4, 83, 84, 82, 85, 4, 26, + 82, 239, 197, 23, 79, 2, 145, 248, 21, 2, 69, 83, 6, 196, 139, 15, 9, 85, + 83, 72, 73, 78, 71, 32, 85, 80, 204, 137, 2, 3, 82, 79, 71, 255, 196, 6, + 69, 6, 26, 84, 207, 250, 18, 86, 4, 254, 139, 19, 85, 223, 57, 82, 8, + 132, 1, 5, 77, 65, 76, 76, 32, 236, 156, 21, 6, 84, 65, 78, 68, 83, 84, + 233, 240, 1, 11, 80, 76, 73, 84, 84, 73, 78, 71, 32, 65, 80, 4, 24, 2, + 80, 82, 43, 84, 2, 225, 140, 19, 5, 69, 80, 79, 78, 68, 2, 11, 65, 2, + 155, 195, 23, 77, 30, 36, 3, 72, 69, 32, 175, 219, 17, 82, 28, 186, 2, + 65, 130, 1, 67, 132, 1, 3, 70, 65, 77, 20, 9, 82, 69, 67, 69, 80, 84, 73, + 86, 69, 30, 87, 172, 22, 12, 77, 65, 82, 82, 89, 73, 78, 71, 32, 77, 65, + 73, 168, 147, 5, 6, 71, 69, 78, 84, 76, 69, 248, 221, 10, 13, 75, 69, 69, + 80, 73, 78, 71, 32, 83, 84, 73, 76, 76, 201, 233, 3, 7, 74, 79, 89, 79, + 85, 83, 32, 6, 58, 82, 133, 181, 11, 8, 66, 89, 83, 77, 65, 76, 32, 87, + 4, 248, 168, 20, 8, 79, 85, 83, 73, 78, 71, 32, 84, 171, 214, 3, 77, 6, + 128, 131, 1, 2, 65, 85, 212, 164, 19, 9, 82, 69, 65, 84, 73, 86, 69, 32, + 72, 133, 212, 1, 9, 76, 73, 78, 71, 73, 78, 71, 32, 70, 2, 243, 235, 23, + 73, 2, 157, 166, 20, 2, 32, 69, 4, 178, 135, 22, 69, 213, 201, 1, 5, 65, + 78, 68, 69, 82, 4, 202, 231, 6, 65, 141, 166, 6, 14, 79, 82, 75, 32, 79, + 78, 32, 84, 72, 69, 32, 68, 69, 67, 222, 1, 236, 1, 2, 71, 72, 180, 2, 7, + 82, 65, 71, 65, 78, 65, 32, 212, 4, 7, 83, 84, 79, 82, 73, 67, 32, 168, + 186, 15, 7, 78, 68, 85, 32, 84, 69, 77, 240, 33, 4, 75, 73, 78, 71, 162, + 250, 1, 66, 177, 168, 4, 8, 80, 80, 79, 80, 79, 84, 65, 77, 12, 18, 32, + 119, 45, 6, 204, 135, 5, 2, 66, 82, 220, 231, 16, 6, 86, 79, 76, 84, 65, + 71, 229, 48, 9, 79, 67, 84, 69, 84, 32, 80, 82, 69, 6, 92, 11, 83, 80, + 69, 69, 68, 32, 84, 82, 65, 73, 78, 157, 200, 5, 6, 72, 69, 69, 76, 69, + 68, 5, 129, 186, 11, 14, 32, 87, 73, 84, 72, 32, 66, 85, 76, 76, 69, 84, + 32, 78, 200, 1, 112, 9, 68, 73, 71, 82, 65, 80, 72, 32, 89, 20, 7, 76, + 69, 84, 84, 69, 82, 32, 222, 172, 1, 86, 139, 185, 20, 73, 2, 243, 170, + 17, 79, 194, 1, 194, 1, 65, 74, 83, 206, 163, 1, 66, 162, 3, 78, 150, 2, + 68, 2, 71, 2, 72, 2, 75, 2, 77, 2, 80, 2, 82, 2, 84, 2, 90, 126, 87, 46, + 89, 142, 183, 22, 86, 234, 36, 69, 2, 73, 2, 79, 3, 85, 7, 37, 7, 82, 67, + 72, 65, 73, 67, 32, 4, 210, 225, 23, 87, 151, 14, 89, 42, 76, 5, 77, 65, + 76, 76, 32, 206, 133, 24, 65, 2, 69, 2, 73, 2, 79, 3, 85, 32, 170, 169, + 1, 87, 46, 89, 170, 173, 5, 75, 230, 137, 17, 84, 234, 36, 65, 2, 69, 2, + 73, 2, 79, 3, 85, 2, 139, 179, 8, 83, 108, 166, 1, 76, 52, 3, 78, 69, 89, + 46, 82, 146, 7, 84, 138, 1, 85, 154, 188, 16, 83, 246, 213, 2, 67, 156, + 192, 3, 6, 77, 79, 84, 72, 69, 84, 242, 164, 1, 79, 155, 3, 80, 6, 176, + 140, 16, 4, 76, 79, 87, 32, 247, 246, 7, 69, 4, 186, 147, 22, 66, 129, + 159, 1, 2, 32, 80, 66, 60, 8, 73, 90, 79, 78, 84, 65, 76, 32, 153, 6, 2, + 83, 69, 60, 218, 1, 66, 110, 76, 152, 1, 17, 79, 78, 69, 32, 69, 73, 71, 72, 84, 72, 32, 66, 76, 79, 67, 75, 45, 88, 10, 83, 67, 65, 78, 32, 76, - 73, 78, 69, 45, 46, 84, 210, 239, 20, 69, 251, 4, 77, 6, 44, 5, 76, 65, - 67, 75, 32, 243, 213, 22, 65, 4, 38, 79, 153, 215, 21, 3, 72, 69, 88, 2, - 139, 215, 21, 67, 2, 173, 244, 20, 2, 32, 69, 14, 160, 166, 16, 3, 49, - 51, 53, 198, 231, 6, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 8, 142, 141, - 23, 49, 2, 51, 2, 55, 3, 57, 10, 32, 2, 65, 66, 215, 246, 20, 82, 8, 26, - 85, 215, 245, 20, 32, 6, 33, 6, 76, 65, 84, 73, 79, 78, 7, 11, 32, 4, - 208, 163, 9, 8, 87, 73, 84, 72, 32, 74, 85, 83, 151, 179, 13, 83, 7, 11, - 32, 4, 140, 159, 7, 2, 82, 65, 155, 175, 15, 70, 10, 26, 32, 195, 129, - 22, 69, 8, 86, 80, 188, 175, 18, 5, 66, 69, 86, 69, 82, 204, 80, 3, 83, - 80, 82, 159, 137, 4, 68, 2, 175, 175, 14, 69, 12, 48, 6, 82, 71, 76, 65, - 83, 83, 77, 2, 83, 69, 5, 153, 208, 19, 14, 32, 87, 73, 84, 72, 32, 70, - 76, 79, 87, 73, 78, 71, 32, 9, 11, 32, 6, 88, 8, 87, 73, 84, 72, 32, 71, - 65, 82, 169, 132, 21, 8, 66, 85, 73, 76, 68, 73, 78, 71, 2, 147, 183, 21, - 68, 7, 242, 135, 23, 74, 3, 83, 8, 106, 83, 208, 243, 21, 11, 78, 68, 82, - 69, 68, 32, 80, 79, 73, 78, 84, 148, 11, 2, 71, 71, 139, 136, 1, 84, 2, - 171, 246, 21, 72, 18, 90, 80, 224, 1, 5, 83, 84, 69, 82, 69, 220, 180, - 16, 2, 71, 73, 193, 207, 4, 2, 65, 67, 12, 60, 3, 72, 69, 78, 129, 133, - 4, 6, 79, 68, 73, 65, 83, 84, 11, 34, 32, 82, 65, 227, 167, 19, 45, 4, - 26, 87, 203, 168, 21, 66, 2, 21, 3, 73, 84, 72, 2, 141, 168, 3, 2, 32, - 68, 2, 193, 169, 12, 6, 84, 73, 79, 78, 32, 80, 2, 253, 240, 21, 2, 83, - 73, 202, 5, 160, 1, 3, 67, 69, 32, 152, 1, 2, 68, 69, 222, 24, 77, 214, - 2, 78, 132, 37, 7, 90, 65, 75, 65, 89, 65, 32, 237, 137, 19, 9, 32, 76, - 79, 86, 69, 32, 89, 79, 85, 8, 114, 67, 252, 249, 10, 18, 72, 79, 67, 75, - 69, 89, 32, 83, 84, 73, 67, 75, 32, 65, 78, 68, 32, 80, 155, 179, 1, 83, - 4, 250, 184, 15, 82, 187, 212, 2, 85, 246, 1, 68, 3, 78, 84, 73, 201, 1, + 73, 78, 69, 45, 46, 84, 162, 217, 21, 67, 42, 69, 238, 6, 77, 150, 1, 82, + 247, 2, 90, 6, 44, 5, 76, 65, 67, 75, 32, 163, 199, 23, 65, 4, 38, 79, + 169, 199, 22, 3, 72, 69, 88, 2, 155, 199, 22, 67, 10, 40, 4, 73, 78, 69, + 32, 135, 223, 21, 65, 8, 44, 5, 87, 73, 84, 72, 32, 171, 223, 21, 69, 6, + 26, 84, 219, 224, 21, 70, 4, 250, 224, 21, 72, 171, 90, 73, 14, 136, 225, + 16, 3, 49, 51, 53, 158, 157, 7, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 8, + 206, 253, 23, 49, 2, 51, 2, 55, 3, 57, 10, 32, 2, 65, 66, 223, 227, 21, + 82, 8, 26, 85, 223, 226, 21, 32, 6, 33, 6, 76, 65, 84, 73, 79, 78, 7, 11, + 32, 4, 188, 197, 9, 8, 87, 73, 84, 72, 32, 74, 85, 83, 223, 129, 14, 83, + 7, 11, 32, 4, 180, 191, 7, 2, 82, 65, 167, 255, 15, 70, 10, 26, 32, 235, + 241, 22, 69, 8, 86, 80, 240, 243, 18, 5, 66, 69, 86, 69, 82, 160, 80, 3, + 83, 80, 82, 215, 181, 4, 68, 2, 167, 230, 14, 69, 12, 48, 6, 82, 71, 76, + 65, 83, 83, 77, 2, 83, 69, 5, 249, 170, 20, 14, 32, 87, 73, 84, 72, 32, + 70, 76, 79, 87, 73, 78, 71, 32, 9, 11, 32, 6, 88, 8, 87, 73, 84, 72, 32, + 71, 65, 82, 217, 241, 21, 8, 66, 85, 73, 76, 68, 73, 78, 71, 2, 223, 164, + 22, 68, 7, 178, 248, 23, 74, 3, 83, 8, 106, 83, 248, 227, 22, 11, 78, 68, + 82, 69, 68, 32, 80, 79, 73, 78, 84, 148, 11, 2, 71, 71, 163, 136, 1, 84, + 2, 211, 230, 22, 72, 18, 90, 80, 224, 1, 5, 83, 84, 69, 82, 69, 236, 238, + 16, 2, 71, 73, 229, 130, 5, 2, 65, 67, 12, 60, 3, 72, 69, 78, 245, 158, + 4, 6, 79, 68, 73, 65, 83, 84, 11, 34, 32, 82, 65, 231, 238, 19, 45, 4, + 26, 87, 135, 150, 22, 66, 2, 21, 3, 73, 84, 72, 2, 185, 188, 3, 2, 32, + 68, 2, 177, 220, 12, 6, 84, 73, 79, 78, 32, 80, 2, 165, 225, 22, 2, 83, + 73, 206, 5, 160, 1, 3, 67, 69, 32, 152, 1, 2, 68, 69, 222, 24, 77, 222, + 2, 78, 200, 37, 7, 90, 65, 75, 65, 89, 65, 32, 129, 228, 19, 9, 32, 76, + 79, 86, 69, 32, 89, 79, 85, 8, 114, 67, 232, 161, 11, 18, 72, 79, 67, 75, + 69, 89, 32, 83, 84, 73, 67, 75, 32, 65, 78, 68, 32, 80, 159, 190, 1, 83, + 4, 230, 239, 15, 82, 163, 226, 2, 85, 246, 1, 68, 3, 78, 84, 73, 201, 1, 9, 79, 71, 82, 65, 80, 72, 73, 67, 32, 8, 64, 4, 67, 65, 76, 32, 105, 8, - 70, 73, 67, 65, 84, 73, 79, 78, 6, 32, 2, 84, 79, 175, 255, 21, 87, 5, - 129, 188, 14, 12, 32, 65, 78, 68, 32, 83, 76, 65, 78, 84, 69, 68, 2, 229, - 232, 21, 2, 32, 67, 238, 1, 208, 2, 11, 65, 78, 78, 79, 84, 65, 84, 73, + 70, 73, 67, 65, 84, 73, 79, 78, 6, 32, 2, 84, 79, 215, 239, 22, 87, 5, + 237, 242, 14, 12, 32, 65, 78, 68, 32, 83, 76, 65, 78, 84, 69, 68, 2, 141, + 217, 22, 2, 32, 67, 238, 1, 208, 2, 11, 65, 78, 78, 79, 84, 65, 84, 73, 79, 78, 32, 242, 3, 67, 72, 2, 68, 69, 248, 5, 5, 78, 85, 77, 66, 69, 22, - 84, 220, 204, 5, 8, 72, 65, 76, 70, 32, 70, 73, 76, 144, 186, 1, 5, 69, - 78, 84, 69, 82, 0, 3, 82, 73, 83, 24, 5, 76, 69, 86, 69, 76, 236, 240, 8, - 5, 86, 65, 82, 73, 65, 158, 175, 4, 70, 230, 46, 73, 131, 222, 1, 83, 32, + 84, 128, 237, 5, 8, 72, 65, 76, 70, 32, 70, 73, 76, 148, 186, 1, 5, 69, + 78, 84, 69, 82, 0, 3, 82, 73, 83, 24, 5, 76, 69, 86, 69, 76, 140, 138, 9, + 5, 86, 65, 82, 73, 65, 182, 223, 4, 70, 158, 47, 73, 159, 228, 1, 83, 32, 232, 1, 5, 66, 79, 84, 84, 79, 22, 70, 82, 77, 62, 84, 140, 1, 4, 76, 73, - 78, 75, 188, 142, 1, 4, 83, 69, 67, 79, 150, 255, 5, 79, 228, 238, 5, 6, - 82, 69, 86, 69, 82, 83, 148, 147, 9, 5, 72, 69, 65, 86, 69, 165, 39, 3, - 69, 65, 82, 2, 175, 185, 22, 77, 6, 48, 3, 79, 85, 82, 229, 175, 22, 3, - 73, 82, 83, 4, 218, 184, 22, 84, 27, 32, 4, 36, 3, 73, 68, 68, 235, 144, - 22, 65, 2, 211, 253, 12, 76, 8, 34, 72, 46, 87, 131, 226, 7, 79, 4, 222, - 236, 8, 82, 181, 192, 13, 2, 73, 82, 2, 191, 183, 22, 79, 4, 36, 3, 76, - 79, 83, 167, 173, 20, 79, 2, 129, 183, 22, 3, 73, 78, 71, 36, 104, 20, + 78, 75, 160, 143, 1, 4, 83, 69, 67, 79, 218, 158, 6, 79, 144, 134, 6, 6, + 82, 69, 86, 69, 82, 83, 240, 203, 9, 5, 72, 69, 65, 86, 69, 169, 39, 3, + 69, 65, 82, 2, 227, 169, 23, 77, 6, 48, 3, 79, 85, 82, 153, 160, 23, 3, + 73, 82, 83, 4, 142, 169, 23, 84, 27, 32, 4, 36, 3, 73, 68, 68, 155, 129, + 23, 65, 2, 167, 181, 13, 76, 8, 34, 72, 46, 87, 203, 130, 8, 79, 4, 194, + 142, 9, 82, 133, 143, 14, 2, 73, 82, 2, 243, 167, 23, 79, 4, 36, 3, 76, + 79, 83, 135, 151, 21, 79, 2, 181, 167, 23, 3, 73, 78, 71, 36, 104, 20, 83, 67, 82, 73, 80, 84, 73, 79, 78, 32, 67, 72, 65, 82, 65, 67, 84, 69, - 82, 32, 247, 139, 7, 80, 34, 144, 2, 9, 65, 66, 79, 86, 69, 32, 84, 79, + 82, 32, 159, 172, 7, 80, 34, 144, 2, 9, 65, 66, 79, 86, 69, 32, 84, 79, 32, 84, 8, 76, 69, 70, 84, 32, 84, 79, 32, 76, 5, 79, 86, 69, 82, 76, 20, - 2, 83, 85, 202, 186, 14, 82, 152, 153, 5, 8, 70, 85, 76, 76, 32, 83, 85, - 82, 173, 207, 2, 15, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 82, 69, - 70, 76, 4, 60, 9, 77, 73, 68, 68, 76, 69, 32, 65, 78, 167, 220, 20, 66, - 2, 247, 205, 20, 68, 4, 180, 166, 21, 10, 77, 73, 68, 68, 76, 69, 32, 65, - 78, 68, 195, 163, 1, 82, 2, 231, 162, 16, 65, 18, 72, 12, 82, 82, 79, 85, - 78, 68, 32, 70, 82, 79, 77, 32, 163, 236, 16, 66, 16, 82, 76, 200, 163, - 11, 3, 85, 80, 80, 202, 182, 9, 66, 222, 155, 1, 65, 143, 82, 82, 6, 186, - 163, 11, 79, 175, 164, 11, 69, 2, 195, 180, 8, 82, 148, 1, 92, 10, 65, + 2, 83, 85, 182, 241, 14, 82, 252, 189, 5, 8, 70, 85, 76, 76, 32, 83, 85, + 82, 145, 228, 2, 15, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 82, 69, + 70, 76, 4, 60, 9, 77, 73, 68, 68, 76, 69, 32, 65, 78, 143, 199, 21, 66, + 2, 215, 183, 21, 68, 4, 128, 148, 22, 10, 77, 73, 68, 68, 76, 69, 32, 65, + 78, 68, 183, 166, 1, 82, 2, 247, 220, 16, 65, 18, 72, 12, 82, 82, 79, 85, + 78, 68, 32, 70, 82, 79, 77, 32, 147, 178, 17, 66, 16, 82, 76, 220, 203, + 11, 3, 85, 80, 80, 158, 249, 9, 66, 166, 161, 1, 65, 159, 82, 82, 6, 206, + 203, 11, 79, 219, 236, 11, 69, 2, 235, 213, 8, 82, 148, 1, 92, 10, 65, 76, 76, 89, 32, 77, 65, 82, 75, 32, 41, 9, 69, 76, 69, 71, 82, 65, 80, - 72, 32, 10, 138, 132, 21, 70, 70, 84, 203, 83, 79, 138, 1, 140, 1, 11, - 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 205, 179, 12, 17, 76, 73, 78, + 72, 32, 10, 198, 241, 21, 70, 70, 84, 183, 86, 79, 138, 1, 140, 1, 11, + 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 141, 231, 12, 17, 76, 73, 78, 69, 32, 70, 69, 69, 68, 32, 83, 69, 80, 65, 82, 65, 84, 136, 1, 182, 1, 65, 58, 68, 200, 2, 5, 72, 79, 85, 82, 32, 214, 1, 74, 28, 4, 70, 69, 66, - 82, 64, 2, 77, 65, 144, 137, 15, 3, 78, 79, 86, 0, 4, 83, 69, 80, 84, 25, - 4, 79, 67, 84, 79, 4, 232, 166, 19, 3, 85, 71, 85, 213, 154, 1, 2, 80, - 82, 64, 44, 3, 65, 89, 32, 217, 141, 15, 2, 69, 67, 62, 66, 84, 130, 177, - 5, 69, 66, 70, 70, 78, 26, 83, 219, 161, 16, 79, 34, 34, 72, 90, 87, 187, - 156, 22, 69, 8, 36, 3, 73, 82, 84, 139, 128, 21, 82, 6, 26, 89, 175, 155, - 21, 69, 5, 247, 194, 21, 45, 24, 26, 69, 243, 235, 22, 79, 22, 36, 3, 78, - 84, 89, 135, 239, 21, 76, 21, 147, 232, 14, 45, 50, 78, 84, 222, 174, 5, - 69, 66, 70, 70, 78, 26, 83, 206, 206, 15, 90, 143, 83, 79, 20, 42, 87, - 186, 176, 5, 72, 175, 234, 16, 69, 14, 26, 69, 159, 234, 22, 79, 12, 36, - 3, 78, 84, 89, 179, 237, 21, 76, 11, 159, 212, 16, 45, 6, 24, 2, 65, 78, - 35, 85, 2, 11, 85, 2, 131, 221, 16, 65, 4, 206, 210, 22, 78, 143, 5, 76, - 4, 218, 198, 22, 82, 171, 34, 89, 68, 40, 6, 65, 71, 69, 32, 79, 70, 83, - 80, 5, 233, 225, 8, 15, 32, 79, 82, 32, 65, 80, 80, 82, 79, 88, 73, 77, + 82, 64, 2, 77, 65, 148, 192, 15, 3, 78, 79, 86, 0, 4, 83, 69, 80, 84, 25, + 4, 79, 67, 84, 79, 4, 244, 244, 19, 3, 85, 71, 85, 169, 182, 1, 2, 80, + 82, 64, 44, 3, 65, 89, 32, 221, 196, 15, 2, 69, 67, 62, 66, 84, 162, 209, + 5, 69, 66, 70, 70, 78, 26, 83, 227, 241, 16, 79, 34, 34, 72, 90, 87, 239, + 140, 23, 69, 8, 36, 3, 73, 82, 84, 199, 237, 21, 82, 6, 26, 89, 251, 136, + 22, 69, 5, 155, 178, 22, 45, 24, 26, 69, 179, 220, 23, 79, 22, 36, 3, 78, + 84, 89, 183, 223, 22, 76, 21, 247, 158, 15, 45, 50, 78, 84, 254, 206, 5, + 69, 66, 70, 70, 78, 26, 83, 234, 155, 16, 90, 251, 85, 79, 20, 42, 87, + 218, 208, 5, 72, 195, 186, 17, 69, 14, 26, 69, 223, 218, 23, 79, 12, 36, + 3, 78, 84, 89, 227, 221, 22, 76, 11, 143, 154, 17, 45, 6, 24, 2, 65, 78, + 35, 85, 2, 11, 85, 2, 243, 162, 17, 65, 4, 142, 195, 23, 78, 143, 5, 76, + 4, 154, 183, 23, 82, 171, 34, 89, 68, 40, 6, 65, 71, 69, 32, 79, 70, 83, + 80, 5, 221, 131, 9, 15, 32, 79, 82, 32, 65, 80, 80, 82, 79, 88, 73, 77, 65, 84, 69, 65, 65, 14, 69, 82, 73, 65, 76, 32, 65, 82, 65, 77, 65, 73, - 67, 32, 62, 64, 7, 78, 85, 77, 66, 69, 82, 32, 234, 18, 76, 239, 249, 18, - 83, 16, 26, 84, 171, 140, 15, 79, 10, 138, 143, 11, 87, 186, 137, 1, 69, - 195, 225, 8, 72, 132, 3, 202, 1, 67, 234, 1, 68, 218, 6, 70, 132, 2, 5, - 72, 73, 66, 73, 84, 156, 1, 15, 80, 85, 84, 32, 83, 89, 77, 66, 79, 76, - 32, 70, 79, 82, 32, 206, 1, 83, 236, 5, 2, 84, 69, 206, 8, 86, 175, 211, - 9, 66, 10, 32, 2, 79, 77, 69, 2, 82, 69, 4, 144, 144, 16, 3, 73, 78, 71, - 129, 216, 2, 5, 80, 76, 69, 84, 69, 6, 36, 3, 65, 83, 69, 151, 160, 22, - 77, 4, 34, 32, 161, 141, 10, 2, 83, 32, 2, 161, 143, 11, 8, 70, 79, 78, - 84, 32, 83, 73, 90, 145, 1, 24, 2, 69, 88, 103, 73, 5, 205, 201, 21, 20, - 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 65, 84, 32, 84, 72, 69, 32, 86, - 73, 69, 138, 1, 72, 8, 67, 32, 83, 73, 89, 65, 81, 32, 209, 185, 17, 4, - 65, 78, 32, 82, 136, 1, 200, 1, 11, 65, 76, 84, 69, 82, 78, 65, 84, 69, - 32, 76, 2, 76, 28, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 100, 7, 78, 85, - 77, 66, 69, 82, 32, 222, 209, 8, 82, 201, 246, 2, 6, 80, 76, 65, 67, 69, - 72, 2, 237, 157, 22, 2, 65, 75, 6, 68, 4, 79, 78, 69, 32, 217, 224, 20, - 7, 84, 72, 82, 69, 69, 32, 81, 4, 178, 222, 20, 72, 43, 81, 122, 212, 1, + 67, 32, 62, 72, 7, 78, 85, 77, 66, 69, 82, 32, 230, 18, 76, 221, 195, 19, + 2, 83, 69, 16, 26, 84, 167, 195, 15, 79, 10, 162, 183, 11, 87, 138, 148, + 1, 69, 143, 156, 9, 72, 136, 3, 202, 1, 67, 234, 1, 68, 214, 6, 70, 132, + 2, 5, 72, 73, 66, 73, 84, 156, 1, 15, 80, 85, 84, 32, 83, 89, 77, 66, 79, + 76, 32, 70, 79, 82, 32, 206, 1, 83, 236, 5, 2, 84, 69, 206, 8, 86, 251, + 250, 9, 66, 10, 32, 2, 79, 77, 69, 2, 82, 69, 4, 240, 202, 16, 3, 73, 78, + 71, 157, 225, 2, 5, 80, 76, 69, 84, 69, 6, 36, 3, 65, 83, 69, 195, 144, + 23, 77, 4, 34, 32, 185, 181, 10, 2, 83, 32, 2, 173, 183, 11, 8, 70, 79, + 78, 84, 32, 83, 73, 90, 145, 1, 24, 2, 69, 88, 103, 73, 5, 237, 185, 22, + 20, 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 65, 84, 32, 84, 72, 69, 32, + 86, 73, 69, 138, 1, 72, 8, 67, 32, 83, 73, 89, 65, 81, 32, 249, 254, 17, + 4, 65, 78, 32, 82, 136, 1, 196, 1, 11, 65, 76, 84, 69, 82, 78, 65, 84, + 69, 32, 76, 2, 76, 28, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 100, 7, 78, + 85, 77, 66, 69, 82, 32, 190, 243, 8, 82, 149, 125, 6, 80, 76, 65, 67, 69, + 72, 2, 157, 142, 23, 2, 65, 75, 6, 68, 4, 79, 78, 69, 32, 137, 206, 21, + 7, 84, 72, 82, 69, 69, 32, 81, 4, 222, 203, 21, 72, 47, 81, 122, 212, 1, 10, 65, 76, 84, 69, 82, 78, 65, 84, 69, 32, 72, 5, 75, 65, 82, 79, 82, 0, - 4, 76, 65, 75, 72, 250, 226, 9, 69, 66, 70, 94, 78, 26, 83, 78, 84, 220, - 243, 4, 8, 80, 82, 69, 70, 73, 88, 69, 68, 167, 41, 79, 6, 26, 84, 175, - 195, 21, 79, 4, 236, 128, 6, 2, 69, 78, 223, 188, 16, 87, 5, 191, 140, + 4, 76, 65, 75, 72, 198, 138, 10, 69, 66, 70, 94, 78, 26, 83, 78, 84, 240, + 130, 5, 8, 80, 82, 69, 70, 73, 88, 69, 68, 199, 41, 79, 6, 26, 84, 211, + 179, 22, 79, 4, 244, 160, 6, 2, 69, 78, 147, 141, 17, 87, 5, 239, 252, 22, 65, 16, 72, 5, 73, 78, 73, 84, 89, 85, 9, 79, 82, 77, 65, 84, 73, 79, - 78, 32, 5, 45, 9, 32, 78, 69, 71, 65, 84, 69, 68, 32, 2, 237, 134, 8, 4, - 87, 73, 84, 72, 12, 42, 83, 253, 134, 16, 4, 68, 69, 83, 75, 10, 156, - 245, 6, 5, 69, 80, 65, 82, 65, 191, 214, 9, 79, 4, 11, 32, 4, 232, 136, + 78, 32, 5, 45, 9, 32, 78, 69, 71, 65, 84, 69, 68, 32, 2, 157, 168, 8, 4, + 87, 73, 84, 72, 12, 42, 83, 137, 203, 19, 4, 68, 69, 83, 75, 10, 192, + 149, 7, 5, 69, 80, 65, 82, 65, 139, 252, 9, 79, 4, 11, 32, 4, 152, 249, 22, 15, 65, 82, 65, 66, 73, 67, 32, 70, 79, 82, 77, 32, 83, 72, 65, 1, 14, 83, 89, 77, 77, 69, 84, 82, 73, 67, 32, 83, 87, 65, 80, 10, 80, 6, - 76, 65, 84, 73, 78, 32, 234, 185, 14, 83, 221, 143, 7, 4, 78, 85, 77, 66, + 76, 65, 84, 73, 78, 32, 198, 240, 14, 83, 165, 201, 7, 4, 78, 85, 77, 66, 6, 64, 6, 67, 65, 80, 73, 84, 65, 0, 4, 83, 77, 65, 76, 27, 76, 2, 21, 3, - 76, 32, 76, 2, 205, 217, 20, 2, 69, 84, 116, 84, 13, 67, 82, 73, 80, 84, - 73, 79, 78, 65, 76, 32, 80, 65, 233, 141, 15, 2, 69, 82, 114, 72, 6, 72, + 76, 32, 76, 2, 253, 198, 21, 2, 69, 84, 116, 84, 13, 67, 82, 73, 80, 84, + 73, 79, 78, 65, 76, 32, 80, 65, 209, 196, 15, 2, 69, 82, 114, 72, 6, 72, 76, 65, 86, 73, 32, 225, 1, 7, 82, 84, 72, 73, 65, 78, 32, 54, 48, 7, 76, - 69, 84, 84, 69, 82, 32, 191, 3, 78, 38, 234, 132, 10, 84, 226, 118, 65, - 22, 68, 34, 76, 22, 77, 50, 87, 238, 150, 4, 71, 90, 90, 34, 83, 66, 89, - 154, 193, 1, 72, 234, 5, 75, 130, 76, 66, 190, 173, 4, 78, 199, 105, 80, + 69, 84, 84, 69, 82, 32, 191, 3, 78, 38, 134, 173, 10, 84, 226, 118, 65, + 22, 68, 34, 76, 22, 77, 50, 87, 186, 165, 4, 71, 90, 90, 34, 83, 66, 89, + 158, 208, 1, 72, 234, 5, 75, 198, 75, 66, 178, 216, 4, 78, 223, 105, 80, 60, 22, 76, 251, 1, 78, 44, 11, 69, 44, 29, 5, 84, 84, 69, 82, 32, 44, - 250, 130, 10, 84, 246, 118, 68, 34, 76, 50, 81, 206, 194, 1, 82, 134, - 212, 2, 65, 50, 71, 90, 90, 34, 83, 66, 89, 154, 193, 1, 72, 234, 5, 75, - 130, 76, 66, 190, 173, 4, 78, 254, 1, 87, 202, 103, 80, 171, 4, 77, 16, - 33, 6, 85, 77, 66, 69, 82, 32, 16, 246, 249, 10, 84, 202, 253, 3, 79, - 171, 243, 2, 70, 50, 36, 4, 71, 82, 65, 76, 179, 3, 82, 23, 11, 32, 20, - 58, 65, 140, 1, 5, 87, 73, 84, 72, 32, 219, 181, 20, 69, 4, 96, 6, 86, - 69, 82, 65, 71, 69, 173, 237, 15, 12, 82, 79, 85, 78, 68, 32, 65, 32, 80, - 79, 73, 78, 2, 209, 238, 15, 5, 32, 87, 73, 84, 72, 14, 160, 1, 3, 84, - 73, 77, 20, 2, 85, 78, 200, 203, 6, 16, 76, 69, 70, 84, 87, 65, 82, 68, - 83, 32, 65, 82, 82, 79, 87, 32, 182, 231, 12, 73, 198, 1, 79, 251, 49, - 68, 2, 191, 180, 19, 69, 4, 202, 180, 19, 68, 203, 201, 2, 73, 28, 94, - 76, 232, 1, 7, 83, 69, 67, 84, 73, 79, 78, 202, 6, 82, 144, 200, 11, 2, + 150, 171, 10, 84, 246, 118, 68, 34, 76, 50, 81, 238, 205, 1, 82, 178, + 215, 2, 65, 50, 71, 90, 90, 34, 83, 66, 89, 158, 208, 1, 72, 234, 5, 75, + 198, 75, 66, 178, 216, 4, 78, 134, 2, 87, 218, 103, 80, 171, 4, 77, 16, + 33, 6, 85, 77, 66, 69, 82, 32, 16, 146, 162, 11, 84, 174, 140, 4, 79, + 219, 128, 3, 70, 50, 36, 4, 71, 82, 65, 76, 179, 3, 82, 23, 11, 32, 20, + 58, 65, 140, 1, 5, 87, 73, 84, 72, 32, 175, 160, 21, 69, 4, 96, 6, 86, + 69, 82, 65, 71, 69, 145, 168, 16, 12, 82, 79, 85, 78, 68, 32, 65, 32, 80, + 79, 73, 78, 2, 181, 169, 16, 5, 32, 87, 73, 84, 72, 14, 160, 1, 3, 84, + 73, 77, 20, 2, 85, 78, 248, 235, 6, 16, 76, 69, 70, 84, 87, 65, 82, 68, + 83, 32, 65, 82, 82, 79, 87, 32, 210, 162, 13, 73, 198, 1, 79, 247, 64, + 68, 2, 139, 144, 20, 69, 4, 150, 144, 20, 68, 175, 222, 2, 73, 28, 94, + 76, 232, 1, 7, 83, 69, 67, 84, 73, 79, 78, 146, 7, 82, 228, 249, 11, 2, 67, 65, 43, 73, 8, 164, 1, 17, 73, 78, 69, 65, 82, 32, 65, 78, 78, 79, - 84, 65, 84, 73, 79, 78, 32, 201, 200, 4, 17, 79, 67, 75, 69, 68, 32, 70, - 69, 77, 65, 76, 69, 32, 65, 78, 68, 32, 6, 232, 130, 8, 3, 65, 78, 67, - 170, 128, 8, 84, 199, 175, 3, 83, 15, 11, 32, 12, 168, 1, 6, 65, 66, 79, - 86, 69, 32, 64, 5, 87, 73, 84, 72, 32, 133, 174, 19, 22, 66, 69, 83, 73, + 84, 65, 84, 73, 79, 78, 32, 229, 232, 4, 17, 79, 67, 75, 69, 68, 32, 70, + 69, 77, 65, 76, 69, 32, 65, 78, 68, 32, 6, 140, 164, 8, 3, 65, 78, 67, + 170, 153, 8, 84, 243, 208, 3, 83, 15, 11, 32, 12, 168, 1, 6, 65, 66, 79, + 86, 69, 32, 64, 5, 87, 73, 84, 72, 32, 209, 137, 20, 22, 66, 69, 83, 73, 68, 69, 32, 65, 78, 68, 32, 74, 79, 73, 78, 69, 68, 32, 87, 73, 84, 72, - 4, 244, 174, 19, 9, 66, 65, 82, 32, 65, 66, 79, 86, 69, 23, 85, 6, 158, - 143, 4, 76, 190, 160, 15, 79, 139, 201, 2, 68, 36, 56, 2, 69, 82, 245, 4, - 7, 73, 83, 73, 66, 76, 69, 32, 30, 48, 3, 83, 69, 32, 153, 2, 4, 84, 69, - 68, 32, 12, 180, 1, 5, 67, 72, 69, 67, 75, 68, 24, 68, 79, 87, 78, 87, - 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 87, 73, 84, 72, 32, 84, 73, - 80, 166, 195, 19, 87, 230, 18, 77, 183, 147, 1, 66, 4, 196, 213, 19, 8, - 69, 82, 32, 66, 79, 65, 82, 68, 195, 175, 2, 32, 2, 229, 237, 19, 2, 32, - 76, 18, 144, 1, 6, 73, 78, 84, 69, 82, 82, 22, 76, 150, 195, 11, 80, 170, - 169, 4, 69, 148, 183, 1, 2, 79, 72, 136, 131, 2, 4, 85, 78, 68, 69, 163, - 83, 81, 2, 151, 179, 4, 79, 6, 60, 9, 79, 87, 32, 75, 65, 86, 89, 75, 65, - 187, 196, 6, 65, 5, 225, 204, 17, 11, 32, 87, 73, 84, 72, 32, 75, 65, 86, - 89, 75, 6, 38, 84, 138, 230, 18, 80, 143, 68, 83, 2, 207, 238, 15, 73, - 218, 1, 94, 65, 134, 21, 69, 62, 79, 42, 85, 165, 146, 11, 10, 73, 71, + 4, 192, 138, 20, 9, 66, 65, 82, 32, 65, 66, 79, 86, 69, 23, 85, 6, 202, + 171, 4, 76, 222, 223, 15, 79, 239, 221, 2, 68, 40, 56, 2, 69, 82, 189, 5, + 7, 73, 83, 73, 66, 76, 69, 32, 34, 48, 3, 83, 69, 32, 225, 2, 4, 84, 69, + 68, 32, 16, 174, 1, 66, 80, 5, 67, 72, 69, 67, 75, 68, 24, 68, 79, 87, + 78, 87, 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 87, 73, 84, 72, 32, + 84, 73, 80, 250, 163, 20, 87, 215, 26, 77, 6, 44, 5, 76, 65, 67, 75, 32, + 159, 215, 21, 85, 4, 226, 249, 21, 68, 211, 27, 83, 4, 136, 190, 20, 8, + 69, 82, 32, 66, 79, 65, 82, 68, 231, 182, 2, 32, 2, 233, 215, 20, 2, 32, + 76, 18, 144, 1, 6, 73, 78, 84, 69, 82, 82, 22, 76, 142, 242, 11, 80, 206, + 180, 4, 69, 220, 193, 1, 2, 79, 72, 168, 153, 2, 4, 85, 78, 68, 69, 179, + 97, 81, 2, 139, 211, 4, 79, 6, 60, 9, 79, 87, 32, 75, 65, 86, 89, 75, 65, + 163, 228, 6, 65, 5, 237, 144, 18, 11, 32, 87, 73, 84, 72, 32, 75, 65, 86, + 89, 75, 6, 38, 84, 194, 172, 19, 80, 223, 88, 83, 2, 235, 168, 16, 73, + 218, 1, 94, 65, 134, 21, 69, 62, 79, 42, 85, 165, 187, 11, 10, 73, 71, 83, 65, 87, 32, 80, 85, 90, 90, 202, 1, 120, 5, 67, 75, 45, 79, 45, 32, 7, 80, 65, 78, 69, 83, 69, 32, 184, 2, 7, 86, 65, 78, 69, 83, 69, 32, - 183, 190, 22, 82, 2, 233, 226, 17, 3, 76, 65, 78, 16, 246, 1, 67, 18, 80, - 128, 153, 1, 10, 73, 78, 68, 85, 83, 84, 82, 73, 65, 76, 140, 196, 3, 3, - 66, 65, 78, 250, 171, 3, 68, 244, 171, 12, 3, 71, 79, 66, 204, 105, 2, - 79, 71, 181, 98, 16, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 66, 69, - 71, 73, 78, 2, 199, 32, 65, 2, 197, 209, 12, 7, 79, 83, 84, 32, 79, 70, + 171, 174, 23, 82, 2, 209, 166, 18, 3, 76, 65, 78, 16, 246, 1, 67, 18, 80, + 136, 157, 1, 10, 73, 78, 68, 85, 83, 84, 82, 73, 65, 76, 216, 223, 3, 3, + 66, 65, 78, 186, 173, 3, 68, 252, 128, 11, 15, 83, 89, 77, 66, 79, 76, + 32, 70, 79, 82, 32, 66, 69, 71, 73, 200, 246, 1, 3, 71, 79, 66, 197, 108, + 2, 79, 71, 2, 203, 32, 65, 2, 197, 136, 13, 7, 79, 83, 84, 32, 79, 70, 70, 182, 1, 172, 2, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 76, 2, 76, 69, 40, 4, 82, 73, 71, 72, 224, 6, 2, 80, 65, 176, 3, 14, 84, 85, 82, 78, 69, 68, 32, 80, 65, 68, 65, 32, 80, 73, 88, 5, 83, 73, 71, 78, 32, 140, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, - 255, 192, 20, 68, 6, 52, 2, 75, 69, 228, 129, 16, 2, 80, 69, 175, 5, 67, - 2, 187, 134, 22, 82, 96, 38, 70, 65, 5, 84, 84, 69, 82, 32, 2, 41, 8, 84, - 32, 82, 69, 82, 69, 78, 71, 2, 171, 243, 20, 71, 94, 230, 1, 68, 26, 73, + 239, 173, 21, 68, 6, 52, 2, 75, 69, 152, 189, 16, 2, 80, 69, 175, 5, 67, + 2, 163, 246, 22, 82, 96, 38, 70, 65, 5, 84, 84, 69, 82, 32, 2, 41, 8, 84, + 32, 82, 69, 82, 69, 78, 71, 2, 211, 224, 21, 71, 94, 230, 1, 68, 26, 73, 48, 2, 75, 65, 66, 78, 130, 1, 66, 2, 67, 2, 71, 16, 2, 80, 65, 56, 2, - 82, 65, 32, 2, 83, 65, 42, 84, 74, 74, 186, 174, 20, 65, 150, 132, 2, 72, + 82, 65, 32, 2, 83, 65, 42, 84, 74, 74, 158, 155, 21, 65, 166, 135, 2, 72, 2, 76, 2, 77, 2, 87, 2, 89, 186, 2, 69, 2, 79, 3, 85, 8, 222, 3, 68, 15, - 65, 7, 164, 167, 5, 3, 32, 75, 65, 167, 145, 17, 73, 7, 11, 32, 4, 22, - 83, 215, 2, 77, 2, 145, 137, 20, 2, 65, 83, 14, 36, 2, 71, 65, 90, 89, + 65, 7, 232, 198, 5, 3, 32, 75, 65, 215, 225, 17, 73, 7, 11, 32, 4, 22, + 83, 215, 2, 77, 2, 165, 242, 20, 2, 65, 83, 14, 36, 2, 71, 65, 90, 89, 167, 1, 65, 7, 33, 6, 32, 76, 69, 76, 69, 84, 5, 29, 5, 32, 82, 65, 83, - 87, 2, 135, 192, 5, 65, 4, 163, 1, 65, 7, 11, 32, 4, 154, 1, 77, 129, - 178, 22, 3, 67, 69, 82, 5, 145, 133, 16, 3, 32, 65, 71, 7, 17, 2, 32, 77, + 87, 2, 227, 223, 5, 65, 4, 163, 1, 65, 7, 11, 32, 4, 154, 1, 77, 245, + 161, 23, 3, 67, 69, 82, 5, 201, 192, 16, 3, 32, 65, 71, 7, 17, 2, 32, 77, 4, 70, 85, 71, 65, 8, 18, 65, 55, 84, 5, 17, 2, 32, 77, 2, 11, 85, 2, - 147, 144, 22, 82, 4, 11, 65, 5, 11, 32, 2, 11, 77, 2, 11, 65, 2, 11, 72, - 2, 233, 232, 16, 2, 65, 80, 28, 40, 3, 68, 65, 32, 165, 3, 2, 78, 71, 24, - 174, 1, 65, 90, 76, 86, 80, 168, 240, 6, 10, 84, 73, 82, 84, 65, 32, 84, - 85, 77, 69, 138, 219, 11, 87, 128, 149, 2, 7, 73, 83, 69, 78, 45, 73, 83, - 153, 206, 1, 3, 77, 65, 68, 6, 44, 3, 68, 69, 71, 205, 161, 21, 2, 78, - 68, 5, 11, 32, 2, 181, 236, 21, 2, 65, 68, 6, 38, 85, 141, 175, 22, 3, - 73, 78, 71, 4, 184, 243, 17, 2, 78, 71, 211, 197, 3, 72, 4, 38, 73, 149, - 236, 17, 3, 65, 78, 71, 2, 217, 253, 15, 3, 83, 69, 76, 4, 208, 214, 12, - 5, 82, 65, 78, 71, 75, 171, 138, 9, 75, 10, 104, 5, 67, 69, 67, 65, 75, - 204, 165, 5, 5, 80, 65, 78, 89, 65, 176, 2, 3, 87, 73, 71, 239, 212, 10, - 76, 5, 233, 218, 17, 3, 32, 84, 69, 18, 116, 4, 83, 85, 75, 85, 42, 84, - 64, 4, 87, 85, 76, 85, 182, 251, 15, 80, 165, 210, 1, 7, 68, 73, 82, 71, - 65, 32, 77, 5, 221, 180, 21, 5, 32, 77, 69, 78, 68, 6, 26, 65, 243, 252, - 15, 79, 4, 218, 252, 15, 82, 139, 224, 5, 76, 5, 25, 4, 32, 77, 69, 76, - 2, 131, 170, 22, 73, 4, 36, 3, 76, 76, 89, 207, 254, 18, 65, 2, 155, 238, - 18, 70, 4, 228, 253, 20, 2, 89, 83, 147, 95, 73, 6, 148, 134, 14, 2, 71, - 71, 138, 230, 4, 80, 147, 161, 3, 78, 192, 24, 74, 65, 178, 78, 69, 202, - 1, 72, 178, 50, 73, 178, 2, 78, 50, 79, 111, 82, 198, 10, 200, 1, 5, 73, - 84, 72, 73, 32, 230, 4, 78, 168, 45, 6, 84, 65, 75, 65, 78, 65, 208, 13, - 3, 87, 73, 32, 240, 8, 7, 89, 65, 72, 32, 76, 73, 32, 252, 163, 4, 6, 75, - 84, 79, 86, 73, 75, 219, 143, 17, 65, 136, 1, 204, 1, 7, 76, 69, 84, 84, - 69, 82, 32, 174, 2, 83, 234, 115, 68, 156, 206, 3, 2, 86, 79, 168, 136, - 3, 11, 78, 85, 77, 66, 69, 82, 32, 83, 73, 71, 78, 154, 155, 9, 65, 197, - 213, 1, 6, 69, 78, 85, 77, 69, 82, 90, 210, 1, 68, 210, 189, 18, 65, 82, - 84, 230, 24, 85, 210, 200, 1, 73, 158, 190, 1, 78, 46, 83, 82, 66, 2, 67, - 2, 71, 2, 74, 2, 75, 2, 80, 2, 82, 254, 68, 72, 2, 76, 2, 77, 2, 86, 2, - 89, 186, 2, 69, 3, 79, 10, 38, 68, 194, 163, 22, 72, 187, 2, 65, 6, 186, - 174, 20, 68, 134, 245, 1, 72, 187, 2, 65, 12, 40, 4, 73, 71, 78, 32, 167, - 236, 10, 69, 10, 242, 190, 18, 67, 98, 78, 222, 160, 3, 65, 239, 1, 86, - 230, 4, 42, 71, 221, 39, 5, 78, 65, 68, 65, 32, 174, 3, 76, 11, 88, 73, - 32, 82, 65, 68, 73, 67, 65, 76, 32, 153, 227, 19, 2, 65, 82, 172, 3, 130, - 2, 65, 94, 66, 230, 2, 67, 150, 2, 68, 158, 3, 69, 198, 1, 70, 138, 2, - 71, 146, 1, 72, 210, 2, 73, 76, 2, 74, 65, 34, 75, 30, 76, 230, 1, 77, - 254, 1, 78, 62, 79, 102, 80, 110, 82, 166, 1, 83, 210, 6, 84, 226, 2, 86, - 50, 87, 210, 2, 89, 131, 244, 20, 85, 10, 56, 2, 82, 82, 238, 245, 20, - 71, 158, 101, 78, 195, 47, 88, 4, 138, 163, 21, 79, 207, 1, 73, 34, 58, - 65, 38, 73, 46, 76, 54, 79, 102, 82, 167, 215, 20, 69, 4, 190, 180, 5, - 77, 143, 180, 13, 68, 6, 206, 233, 20, 84, 186, 112, 82, 151, 70, 71, 6, - 138, 254, 18, 79, 154, 243, 1, 65, 251, 151, 1, 85, 10, 62, 76, 210, 130, - 22, 65, 218, 5, 78, 142, 5, 68, 203, 17, 87, 2, 173, 193, 3, 4, 84, 32, - 79, 70, 6, 42, 73, 246, 201, 9, 65, 159, 239, 10, 85, 2, 151, 189, 13, - 83, 28, 58, 65, 70, 76, 74, 79, 190, 135, 10, 72, 163, 171, 10, 73, 6, - 38, 85, 138, 129, 22, 82, 219, 5, 86, 2, 165, 249, 20, 2, 76, 68, 8, 42, - 65, 206, 151, 7, 73, 171, 241, 13, 79, 4, 234, 156, 22, 78, 3, 87, 10, - 214, 250, 20, 82, 204, 1, 2, 77, 80, 206, 98, 86, 134, 5, 76, 223, 56, - 87, 34, 38, 69, 38, 73, 98, 79, 195, 1, 82, 4, 214, 156, 20, 65, 183, - 198, 1, 69, 8, 38, 83, 158, 190, 13, 86, 175, 2, 80, 4, 200, 220, 18, 5, - 84, 73, 78, 71, 85, 199, 190, 3, 72, 16, 94, 84, 76, 3, 85, 66, 76, 214, - 191, 12, 87, 172, 137, 9, 2, 32, 78, 222, 23, 79, 211, 56, 71, 7, 25, 4, - 84, 69, 68, 32, 4, 184, 148, 7, 3, 67, 76, 73, 135, 194, 14, 84, 2, 171, - 170, 3, 69, 6, 162, 226, 20, 65, 158, 166, 1, 85, 219, 16, 89, 20, 106, - 65, 38, 78, 32, 3, 86, 69, 78, 216, 214, 8, 6, 77, 66, 82, 79, 73, 68, - 158, 150, 13, 73, 243, 19, 89, 6, 202, 152, 16, 82, 179, 255, 5, 84, 4, - 158, 25, 67, 219, 192, 21, 84, 5, 203, 198, 21, 73, 28, 74, 65, 50, 73, - 62, 76, 38, 82, 150, 20, 69, 138, 177, 21, 79, 223, 23, 85, 6, 206, 188, - 13, 84, 182, 195, 8, 67, 131, 22, 78, 8, 146, 177, 12, 69, 222, 186, 9, - 71, 230, 19, 82, 199, 21, 83, 4, 190, 209, 20, 85, 175, 196, 1, 89, 4, - 220, 232, 20, 2, 65, 71, 151, 172, 1, 79, 14, 58, 79, 52, 2, 82, 65, 218, - 204, 18, 72, 171, 131, 2, 65, 7, 202, 206, 21, 76, 229, 34, 5, 32, 83, - 76, 79, 87, 4, 214, 196, 21, 73, 135, 18, 83, 22, 58, 65, 142, 1, 69, 60, - 5, 73, 68, 73, 78, 71, 19, 79, 8, 38, 76, 150, 205, 21, 78, 199, 13, 73, - 4, 34, 70, 177, 252, 20, 2, 66, 69, 2, 41, 8, 32, 84, 82, 69, 69, 32, 84, - 82, 2, 255, 211, 18, 85, 6, 26, 65, 159, 146, 22, 77, 4, 250, 245, 21, - 82, 175, 28, 68, 2, 175, 19, 32, 6, 26, 82, 199, 142, 22, 79, 4, 134, - 251, 21, 83, 215, 22, 78, 6, 26, 78, 207, 250, 21, 67, 4, 26, 83, 247, - 143, 22, 67, 2, 171, 130, 21, 69, 4, 154, 250, 21, 68, 215, 22, 82, 2, - 253, 140, 5, 2, 78, 73, 22, 46, 65, 34, 69, 78, 73, 41, 3, 79, 78, 71, 4, - 174, 249, 21, 77, 191, 19, 67, 8, 38, 65, 238, 209, 21, 71, 187, 58, 69, - 4, 178, 181, 13, 84, 139, 218, 8, 70, 6, 194, 248, 21, 70, 2, 78, 215, - 22, 68, 5, 209, 146, 11, 3, 32, 83, 84, 22, 42, 69, 34, 73, 46, 79, 135, - 190, 21, 65, 4, 194, 190, 21, 76, 183, 51, 65, 4, 156, 185, 11, 2, 78, - 73, 203, 248, 8, 76, 12, 34, 82, 34, 85, 195, 189, 21, 79, 4, 138, 235, - 20, 84, 179, 81, 78, 6, 26, 78, 235, 139, 22, 84, 4, 198, 225, 20, 84, - 179, 171, 1, 68, 6, 26, 79, 143, 240, 21, 69, 4, 226, 245, 21, 83, 215, - 22, 84, 10, 40, 2, 78, 69, 22, 80, 199, 197, 21, 76, 5, 139, 247, 5, 83, - 4, 170, 132, 10, 80, 215, 173, 2, 69, 10, 54, 82, 242, 240, 20, 76, 166, - 1, 79, 151, 152, 1, 73, 4, 152, 144, 12, 2, 73, 86, 229, 218, 6, 2, 79, - 70, 18, 62, 65, 42, 73, 214, 255, 9, 79, 186, 186, 11, 85, 195, 9, 69, 6, - 178, 186, 21, 73, 214, 79, 80, 3, 84, 6, 232, 175, 12, 3, 71, 72, 84, - 150, 156, 9, 86, 143, 39, 67, 74, 170, 1, 65, 86, 67, 46, 69, 62, 72, - 166, 1, 73, 46, 76, 62, 80, 106, 84, 234, 159, 17, 87, 182, 145, 1, 78, - 190, 50, 79, 182, 155, 1, 77, 134, 100, 81, 254, 32, 75, 235, 47, 85, 6, - 192, 168, 3, 9, 67, 82, 73, 70, 73, 67, 73, 65, 76, 138, 195, 18, 76, - 175, 28, 89, 4, 150, 198, 10, 82, 229, 186, 5, 2, 72, 79, 8, 242, 203, - 20, 67, 234, 49, 65, 146, 8, 76, 143, 129, 1, 69, 10, 18, 69, 39, 79, 4, - 138, 253, 20, 76, 147, 137, 1, 69, 6, 40, 4, 82, 84, 32, 84, 175, 233, - 21, 79, 4, 222, 190, 15, 72, 137, 200, 4, 7, 65, 73, 76, 69, 68, 32, 66, - 4, 148, 162, 17, 2, 67, 75, 235, 223, 4, 76, 6, 26, 65, 227, 199, 21, 73, - 4, 250, 237, 21, 86, 199, 21, 83, 10, 50, 69, 34, 73, 178, 177, 18, 82, - 207, 130, 3, 79, 4, 170, 203, 21, 65, 171, 22, 69, 2, 163, 231, 21, 82, - 12, 34, 69, 34, 79, 167, 245, 20, 65, 4, 202, 242, 21, 65, 219, 16, 80, - 6, 26, 80, 151, 236, 21, 78, 5, 239, 177, 21, 80, 30, 82, 65, 98, 73, 34, - 79, 38, 82, 52, 2, 85, 82, 32, 2, 87, 79, 183, 176, 21, 69, 6, 38, 78, - 182, 224, 20, 66, 139, 24, 76, 2, 33, 6, 78, 69, 68, 32, 76, 69, 2, 159, - 167, 13, 65, 4, 190, 195, 21, 71, 143, 39, 76, 4, 150, 160, 18, 78, 231, - 225, 1, 79, 6, 214, 159, 16, 73, 186, 178, 4, 65, 251, 151, 1, 69, 4, - 242, 184, 20, 66, 147, 38, 84, 5, 215, 214, 18, 32, 4, 170, 157, 12, 65, - 129, 136, 5, 3, 73, 76, 76, 26, 58, 65, 110, 69, 42, 72, 32, 2, 73, 78, - 30, 79, 39, 82, 6, 32, 2, 76, 75, 135, 193, 21, 84, 5, 11, 32, 2, 11, 69, - 2, 17, 2, 78, 67, 2, 201, 158, 17, 2, 76, 79, 4, 184, 174, 21, 2, 65, 80, - 183, 51, 83, 4, 198, 185, 20, 73, 223, 48, 69, 4, 210, 253, 21, 68, 3, - 69, 4, 130, 182, 20, 77, 159, 196, 1, 82, 4, 166, 172, 21, 79, 227, 80, - 65, 2, 197, 228, 19, 2, 69, 76, 184, 1, 92, 2, 76, 69, 148, 2, 5, 83, 73, - 71, 78, 32, 238, 249, 16, 65, 174, 9, 86, 175, 136, 3, 68, 110, 44, 5, - 84, 84, 69, 82, 32, 235, 185, 21, 78, 108, 214, 128, 17, 76, 38, 78, 174, - 90, 82, 206, 55, 65, 38, 68, 46, 84, 46, 86, 186, 24, 85, 158, 144, 1, - 79, 182, 56, 73, 202, 190, 1, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, - 80, 162, 7, 69, 222, 61, 70, 2, 72, 2, 77, 3, 89, 22, 94, 67, 138, 1, 83, - 138, 145, 18, 65, 170, 1, 78, 230, 179, 1, 74, 150, 3, 85, 211, 235, 1, - 86, 4, 100, 19, 79, 77, 66, 73, 78, 73, 78, 71, 32, 65, 78, 85, 83, 86, - 65, 82, 65, 32, 65, 255, 145, 18, 65, 2, 185, 131, 19, 3, 66, 79, 86, 4, - 244, 135, 12, 6, 80, 65, 67, 73, 78, 71, 155, 249, 4, 73, 162, 2, 62, 32, - 173, 11, 10, 45, 72, 73, 82, 65, 71, 65, 78, 65, 32, 154, 2, 140, 1, 7, - 76, 69, 84, 84, 69, 82, 32, 242, 9, 86, 236, 159, 18, 10, 68, 73, 71, 82, - 65, 80, 72, 32, 75, 79, 234, 175, 1, 73, 247, 138, 1, 77, 146, 2, 186, 1, - 65, 178, 1, 66, 106, 77, 186, 2, 78, 54, 83, 226, 1, 68, 2, 71, 2, 72, 2, - 75, 2, 80, 2, 82, 2, 84, 2, 86, 2, 90, 126, 87, 46, 89, 154, 236, 21, 69, - 2, 73, 2, 79, 3, 85, 19, 60, 4, 73, 78, 85, 32, 45, 7, 82, 67, 72, 65, - 73, 67, 32, 8, 130, 7, 84, 142, 214, 21, 67, 215, 22, 80, 8, 38, 89, 170, - 206, 21, 87, 235, 36, 69, 4, 142, 243, 21, 69, 3, 73, 20, 50, 73, 194, - 242, 21, 65, 2, 69, 2, 79, 3, 85, 13, 253, 4, 9, 68, 65, 75, 85, 79, 78, - 32, 78, 71, 36, 50, 73, 218, 241, 21, 65, 2, 69, 2, 79, 3, 85, 29, 29, 5, - 78, 78, 65, 78, 32, 26, 96, 15, 78, 65, 83, 65, 76, 73, 90, 69, 68, 32, - 84, 79, 78, 69, 45, 69, 5, 84, 79, 78, 69, 45, 14, 210, 240, 21, 49, 2, - 50, 2, 51, 2, 52, 2, 53, 2, 55, 3, 56, 12, 142, 240, 21, 50, 2, 51, 2, - 52, 2, 53, 2, 55, 3, 56, 13, 210, 239, 21, 65, 2, 69, 2, 73, 2, 79, 3, - 85, 76, 76, 5, 77, 65, 76, 76, 32, 210, 238, 21, 65, 2, 69, 2, 73, 2, 79, - 3, 85, 66, 142, 1, 72, 2, 82, 54, 75, 46, 84, 30, 87, 46, 89, 254, 156, - 18, 78, 242, 142, 3, 83, 198, 27, 77, 234, 36, 65, 2, 69, 2, 73, 2, 79, - 3, 85, 10, 190, 237, 21, 65, 2, 69, 2, 73, 2, 79, 3, 85, 8, 138, 237, 21, - 65, 2, 69, 2, 79, 3, 85, 4, 222, 236, 21, 79, 3, 85, 8, 194, 236, 21, 65, - 2, 69, 2, 73, 3, 79, 6, 150, 236, 21, 65, 2, 79, 3, 85, 2, 189, 207, 19, - 5, 79, 73, 67, 69, 68, 8, 52, 5, 68, 79, 85, 66, 76, 22, 80, 38, 83, 35, - 86, 2, 251, 216, 7, 69, 2, 89, 6, 82, 79, 76, 79, 78, 71, 2, 29, 5, 69, - 77, 73, 45, 86, 2, 21, 3, 79, 73, 67, 2, 33, 6, 69, 68, 32, 83, 79, 85, - 2, 239, 157, 21, 78, 172, 1, 228, 1, 7, 76, 69, 84, 84, 69, 82, 32, 132, - 2, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 148, 3, 5, 83, 73, - 71, 78, 32, 92, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 178, 236, - 11, 68, 205, 183, 9, 3, 67, 79, 78, 94, 230, 1, 65, 214, 128, 4, 74, 154, - 253, 13, 68, 46, 84, 146, 4, 86, 214, 20, 85, 210, 200, 1, 73, 158, 190, - 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 75, 2, 80, 254, 68, 72, 2, 76, 2, - 77, 2, 82, 2, 87, 2, 89, 186, 2, 69, 3, 79, 7, 150, 230, 21, 65, 3, 73, - 22, 122, 67, 90, 68, 58, 70, 54, 83, 20, 12, 65, 76, 84, 69, 82, 78, 65, - 84, 69, 32, 83, 69, 149, 210, 20, 4, 84, 82, 73, 80, 4, 56, 8, 76, 79, - 83, 73, 78, 71, 32, 83, 203, 149, 20, 73, 2, 141, 219, 20, 2, 80, 73, 4, - 11, 79, 4, 132, 211, 20, 2, 85, 66, 151, 145, 1, 84, 4, 204, 148, 20, 5, - 73, 76, 76, 69, 68, 175, 54, 76, 6, 18, 69, 23, 80, 2, 203, 196, 10, 67, - 4, 156, 241, 3, 2, 65, 67, 195, 232, 16, 73, 10, 192, 238, 5, 2, 82, 69, - 166, 204, 1, 75, 218, 193, 10, 67, 226, 182, 1, 86, 223, 234, 1, 65, 20, - 66, 65, 134, 255, 3, 86, 170, 147, 14, 69, 2, 85, 211, 200, 1, 73, 6, - 244, 152, 16, 8, 76, 84, 69, 82, 78, 65, 84, 69, 210, 200, 5, 65, 3, 73, - 96, 148, 1, 7, 76, 69, 84, 84, 69, 82, 32, 204, 1, 5, 83, 73, 71, 78, 32, - 44, 5, 84, 79, 78, 69, 32, 92, 6, 86, 79, 87, 69, 76, 32, 247, 238, 19, - 68, 56, 154, 206, 16, 72, 190, 173, 1, 79, 230, 236, 1, 78, 246, 175, 1, - 75, 2, 80, 2, 83, 2, 84, 254, 68, 66, 2, 67, 2, 68, 2, 71, 2, 76, 2, 77, - 2, 82, 2, 86, 2, 87, 2, 89, 2, 90, 186, 2, 65, 3, 73, 4, 242, 204, 4, 67, - 201, 142, 17, 2, 83, 72, 6, 36, 5, 67, 65, 76, 89, 65, 23, 80, 5, 17, 2, - 32, 80, 2, 197, 155, 17, 3, 76, 79, 80, 10, 250, 156, 21, 69, 2, 85, 151, - 64, 79, 34, 24, 2, 76, 86, 23, 89, 2, 175, 230, 19, 73, 33, 52, 5, 66, - 79, 65, 82, 68, 37, 4, 67, 65, 80, 32, 5, 213, 172, 18, 4, 32, 65, 78, - 68, 26, 142, 197, 16, 78, 146, 134, 3, 65, 162, 34, 68, 171, 29, 84, 186, - 13, 202, 1, 65, 212, 9, 18, 73, 84, 65, 78, 32, 83, 77, 65, 76, 76, 32, - 83, 67, 82, 73, 80, 84, 32, 184, 3, 4, 77, 69, 82, 32, 204, 25, 5, 79, - 74, 75, 73, 32, 145, 6, 8, 85, 68, 65, 87, 65, 68, 73, 32, 138, 1, 56, 8, - 82, 79, 83, 72, 84, 72, 73, 32, 143, 180, 21, 78, 136, 1, 220, 1, 4, 68, - 73, 71, 73, 20, 7, 76, 69, 84, 84, 69, 82, 32, 144, 2, 7, 78, 85, 77, 66, - 69, 82, 32, 36, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 136, - 2, 5, 83, 73, 71, 78, 32, 198, 1, 86, 207, 155, 8, 70, 8, 203, 193, 15, - 84, 74, 182, 1, 75, 42, 84, 150, 229, 6, 78, 250, 135, 11, 68, 170, 160, - 3, 83, 82, 66, 2, 67, 2, 71, 2, 80, 2, 86, 254, 68, 72, 2, 74, 2, 76, 2, - 77, 2, 82, 2, 89, 2, 90, 187, 2, 65, 6, 166, 211, 21, 72, 2, 75, 187, 2, - 65, 12, 182, 237, 17, 84, 202, 229, 3, 72, 187, 2, 65, 8, 146, 149, 14, - 84, 159, 151, 2, 79, 18, 70, 67, 74, 68, 66, 76, 36, 5, 77, 65, 78, 71, - 65, 175, 131, 20, 83, 4, 26, 82, 135, 133, 20, 73, 2, 177, 132, 19, 6, - 69, 83, 67, 69, 78, 84, 6, 26, 79, 191, 254, 11, 65, 4, 134, 254, 11, 85, - 179, 213, 9, 84, 4, 246, 188, 6, 79, 199, 168, 7, 73, 2, 179, 153, 20, - 76, 12, 72, 2, 66, 65, 22, 67, 20, 2, 68, 79, 162, 162, 19, 86, 223, 234, - 1, 65, 2, 155, 213, 20, 82, 2, 255, 243, 5, 65, 4, 56, 8, 85, 66, 76, 69, - 32, 82, 73, 78, 203, 170, 19, 84, 2, 199, 170, 19, 71, 14, 44, 5, 79, 87, - 69, 76, 32, 243, 158, 19, 73, 12, 44, 5, 83, 73, 71, 78, 32, 203, 142, - 21, 76, 10, 242, 237, 3, 86, 198, 226, 17, 69, 2, 73, 2, 79, 3, 85, 174, - 7, 72, 12, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 175, 167, 7, - 70, 172, 7, 22, 66, 147, 1, 67, 128, 4, 146, 214, 7, 48, 2, 49, 2, 50, 2, - 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, 68, - 2, 69, 3, 70, 172, 3, 122, 68, 138, 212, 7, 48, 2, 49, 2, 50, 2, 51, 2, - 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 3, 67, 12, 254, 204, - 21, 48, 2, 49, 2, 50, 2, 51, 2, 52, 3, 53, 246, 2, 202, 1, 67, 240, 2, - 18, 73, 78, 68, 69, 80, 69, 78, 68, 69, 78, 84, 32, 86, 79, 87, 69, 76, - 32, 220, 2, 7, 76, 69, 84, 84, 69, 82, 32, 254, 2, 83, 208, 12, 6, 86, - 79, 87, 69, 76, 32, 195, 199, 19, 68, 70, 176, 1, 20, 79, 78, 83, 79, 78, - 65, 78, 84, 32, 83, 73, 71, 78, 32, 67, 79, 69, 78, 71, 32, 233, 232, 16, - 17, 85, 82, 82, 69, 78, 67, 89, 32, 83, 89, 77, 66, 79, 76, 32, 82, 73, - 68, 138, 1, 78, 154, 4, 67, 2, 75, 90, 80, 74, 84, 54, 68, 2, 76, 198, - 250, 20, 83, 146, 41, 77, 2, 82, 2, 86, 2, 89, 190, 28, 66, 3, 72, 8, - 190, 169, 21, 71, 2, 89, 246, 30, 65, 3, 79, 42, 82, 81, 196, 1, 11, 83, - 73, 71, 78, 32, 67, 79, 69, 78, 71, 32, 50, 76, 3, 82, 26, 82, 65, 44, 6, - 79, 79, 32, 84, 89, 80, 34, 85, 198, 191, 19, 73, 207, 134, 2, 69, 8, - 218, 198, 21, 65, 2, 73, 2, 81, 3, 85, 4, 11, 69, 4, 159, 231, 9, 32, 9, - 214, 152, 7, 85, 187, 173, 14, 75, 8, 18, 81, 31, 82, 4, 214, 197, 21, - 69, 3, 85, 4, 195, 210, 17, 89, 70, 138, 1, 67, 2, 75, 42, 78, 50, 80, - 30, 83, 46, 84, 54, 68, 2, 76, 214, 163, 21, 77, 2, 82, 2, 86, 2, 89, - 190, 28, 66, 2, 72, 3, 81, 8, 210, 1, 72, 202, 194, 21, 65, 3, 79, 8, - 254, 164, 21, 71, 2, 78, 2, 89, 247, 30, 79, 6, 122, 72, 203, 194, 21, - 79, 6, 178, 164, 21, 83, 190, 28, 72, 187, 2, 65, 12, 50, 72, 0, 2, 84, - 72, 202, 194, 21, 65, 3, 79, 4, 198, 194, 21, 65, 3, 79, 130, 1, 60, 4, + 135, 128, 23, 82, 4, 11, 65, 5, 11, 32, 2, 11, 77, 2, 11, 65, 2, 11, 72, + 2, 141, 174, 17, 2, 65, 80, 28, 40, 3, 68, 65, 32, 165, 3, 2, 78, 71, 24, + 174, 1, 65, 90, 76, 86, 80, 140, 144, 7, 10, 84, 73, 82, 84, 65, 32, 84, + 85, 77, 69, 130, 253, 11, 87, 164, 192, 2, 7, 73, 83, 69, 78, 45, 73, 83, + 141, 209, 1, 3, 77, 65, 68, 6, 44, 3, 68, 69, 71, 169, 145, 22, 2, 78, + 68, 5, 11, 32, 2, 157, 220, 22, 2, 65, 68, 6, 38, 85, 129, 159, 23, 3, + 73, 78, 71, 4, 160, 183, 18, 2, 78, 71, 207, 241, 3, 72, 4, 38, 73, 253, + 175, 18, 3, 65, 78, 71, 2, 145, 185, 16, 3, 83, 69, 76, 4, 220, 141, 13, + 5, 82, 65, 78, 71, 75, 135, 195, 9, 75, 10, 104, 5, 67, 69, 67, 65, 75, + 144, 197, 5, 5, 80, 65, 78, 89, 65, 176, 2, 3, 87, 73, 71, 227, 240, 10, + 76, 5, 209, 158, 18, 3, 32, 84, 69, 18, 116, 4, 83, 85, 75, 85, 42, 84, + 64, 4, 87, 85, 76, 85, 238, 182, 16, 80, 213, 218, 1, 7, 68, 73, 82, 71, + 65, 32, 77, 5, 193, 164, 22, 5, 32, 77, 69, 78, 68, 6, 26, 65, 171, 184, + 16, 79, 4, 146, 184, 16, 82, 187, 148, 6, 76, 5, 25, 4, 32, 77, 69, 76, + 2, 247, 153, 23, 73, 4, 36, 3, 76, 76, 89, 255, 216, 19, 65, 2, 219, 187, + 19, 70, 4, 180, 236, 21, 2, 89, 83, 171, 96, 73, 6, 168, 188, 14, 2, 71, + 71, 182, 253, 4, 80, 199, 195, 3, 78, 186, 25, 74, 65, 178, 78, 69, 218, + 1, 72, 150, 50, 73, 198, 6, 78, 50, 79, 111, 82, 200, 10, 200, 1, 5, 73, + 84, 72, 73, 32, 234, 4, 78, 188, 45, 6, 84, 65, 75, 65, 78, 65, 208, 13, + 3, 87, 73, 32, 220, 8, 7, 89, 65, 72, 32, 76, 73, 32, 204, 195, 4, 6, 75, + 84, 79, 86, 73, 75, 251, 223, 17, 65, 136, 1, 204, 1, 7, 76, 69, 84, 84, + 69, 82, 32, 178, 2, 83, 138, 116, 68, 208, 237, 3, 2, 86, 79, 160, 137, + 3, 11, 78, 85, 77, 66, 69, 82, 32, 83, 73, 71, 78, 230, 191, 9, 65, 229, + 211, 1, 6, 69, 78, 85, 77, 69, 82, 90, 214, 1, 68, 242, 187, 19, 65, 150, + 1, 84, 230, 5, 85, 206, 201, 1, 73, 162, 193, 1, 78, 46, 83, 82, 66, 2, + 67, 2, 71, 2, 74, 2, 75, 2, 80, 2, 82, 138, 69, 72, 2, 76, 2, 77, 2, 86, + 2, 89, 186, 2, 69, 3, 79, 10, 38, 68, 178, 147, 23, 72, 187, 2, 65, 6, + 166, 155, 21, 68, 138, 248, 1, 72, 187, 2, 65, 12, 40, 4, 73, 71, 78, 32, + 203, 148, 11, 69, 10, 202, 128, 19, 67, 98, 78, 234, 206, 3, 65, 239, 1, + 86, 230, 4, 42, 71, 241, 39, 5, 78, 65, 68, 65, 32, 174, 3, 76, 11, 88, + 73, 32, 82, 65, 68, 73, 67, 65, 76, 32, 169, 204, 20, 2, 65, 82, 172, 3, + 130, 2, 65, 94, 66, 230, 2, 67, 150, 2, 68, 158, 3, 69, 198, 1, 70, 138, + 2, 71, 146, 1, 72, 210, 2, 73, 76, 2, 74, 65, 34, 75, 30, 76, 230, 1, 77, + 254, 1, 78, 62, 79, 102, 80, 110, 82, 166, 1, 83, 230, 6, 84, 226, 2, 86, + 50, 87, 210, 2, 89, 199, 227, 21, 85, 10, 56, 2, 82, 82, 186, 228, 21, + 71, 182, 102, 78, 207, 47, 88, 4, 234, 146, 22, 79, 207, 1, 73, 34, 58, + 65, 38, 73, 46, 76, 54, 79, 102, 82, 203, 196, 21, 69, 4, 186, 196, 5, + 77, 163, 254, 13, 68, 6, 146, 216, 21, 84, 218, 113, 82, 163, 70, 71, 6, + 134, 217, 19, 79, 234, 134, 2, 65, 159, 153, 1, 85, 10, 62, 76, 194, 242, + 22, 65, 218, 5, 78, 142, 5, 68, 203, 17, 87, 2, 209, 218, 3, 4, 84, 32, + 79, 70, 6, 42, 73, 198, 241, 9, 65, 187, 180, 11, 85, 2, 191, 243, 13, + 83, 28, 58, 65, 70, 76, 74, 79, 214, 174, 10, 72, 247, 240, 10, 73, 6, + 38, 85, 250, 240, 22, 82, 219, 5, 86, 2, 129, 232, 21, 2, 76, 68, 8, 42, + 65, 206, 183, 7, 73, 131, 193, 14, 79, 4, 218, 140, 23, 78, 3, 87, 10, + 250, 233, 21, 82, 128, 2, 2, 77, 80, 218, 98, 86, 134, 5, 76, 235, 56, + 87, 34, 38, 69, 38, 73, 98, 79, 195, 1, 82, 4, 186, 137, 21, 65, 183, + 201, 1, 69, 8, 38, 83, 198, 244, 13, 86, 175, 2, 80, 4, 132, 170, 19, 5, + 84, 73, 78, 71, 85, 251, 224, 3, 72, 16, 94, 84, 76, 3, 85, 66, 76, 222, + 246, 12, 87, 136, 194, 9, 2, 32, 78, 222, 23, 79, 223, 56, 71, 7, 25, 4, + 84, 69, 68, 32, 4, 184, 180, 7, 3, 67, 76, 73, 235, 145, 15, 84, 2, 207, + 195, 3, 69, 6, 230, 208, 21, 65, 202, 167, 1, 85, 219, 16, 89, 20, 106, + 65, 38, 78, 32, 3, 86, 69, 78, 152, 252, 8, 6, 77, 66, 82, 79, 73, 68, + 206, 224, 13, 73, 243, 19, 89, 6, 234, 221, 16, 82, 131, 170, 6, 84, 4, + 178, 25, 67, 171, 176, 22, 84, 5, 175, 182, 22, 73, 28, 74, 65, 50, 73, + 62, 76, 38, 82, 170, 20, 69, 218, 160, 22, 79, 223, 23, 85, 6, 246, 242, + 13, 84, 254, 252, 8, 67, 131, 22, 78, 8, 142, 232, 12, 69, 210, 243, 9, + 71, 230, 19, 82, 199, 21, 83, 4, 226, 190, 21, 85, 251, 198, 1, 89, 4, + 168, 215, 21, 2, 65, 71, 187, 173, 1, 79, 14, 58, 79, 52, 2, 82, 65, 150, + 154, 19, 72, 147, 163, 2, 65, 7, 174, 190, 22, 76, 241, 34, 5, 32, 83, + 76, 79, 87, 4, 186, 180, 22, 73, 135, 18, 83, 22, 58, 65, 142, 1, 69, 60, + 5, 73, 68, 73, 78, 71, 19, 79, 8, 38, 76, 250, 188, 22, 78, 199, 13, 73, + 4, 34, 70, 137, 236, 21, 2, 66, 69, 2, 41, 8, 32, 84, 82, 69, 69, 32, 84, + 82, 2, 187, 161, 19, 85, 6, 26, 65, 143, 130, 23, 77, 4, 234, 229, 22, + 82, 175, 28, 68, 2, 195, 19, 32, 6, 26, 82, 183, 254, 22, 79, 4, 246, + 234, 22, 83, 215, 22, 78, 6, 26, 78, 191, 234, 22, 67, 4, 26, 83, 231, + 255, 22, 67, 2, 131, 242, 21, 69, 4, 138, 234, 22, 68, 215, 22, 82, 2, + 205, 172, 5, 2, 78, 73, 22, 46, 65, 34, 69, 78, 73, 41, 3, 79, 78, 71, 4, + 158, 233, 22, 77, 191, 19, 67, 8, 38, 65, 210, 193, 22, 71, 199, 58, 69, + 4, 218, 235, 13, 84, 211, 147, 9, 70, 6, 178, 232, 22, 70, 2, 78, 215, + 22, 68, 5, 161, 196, 11, 3, 32, 83, 84, 22, 42, 69, 34, 73, 46, 79, 235, + 173, 22, 65, 4, 166, 174, 22, 76, 195, 51, 65, 4, 188, 235, 11, 2, 78, + 73, 151, 179, 9, 76, 12, 34, 82, 34, 85, 167, 173, 22, 79, 4, 234, 219, + 21, 84, 183, 80, 78, 6, 26, 78, 219, 251, 22, 84, 4, 146, 208, 21, 84, + 215, 172, 1, 68, 6, 26, 79, 255, 223, 22, 69, 4, 210, 229, 22, 83, 215, + 22, 84, 10, 40, 2, 78, 69, 22, 80, 171, 181, 22, 76, 5, 235, 150, 6, 83, + 4, 198, 171, 10, 80, 195, 189, 2, 69, 10, 54, 82, 202, 224, 21, 76, 166, + 1, 79, 175, 152, 1, 73, 4, 156, 199, 12, 2, 73, 86, 225, 254, 6, 2, 79, + 70, 18, 62, 65, 42, 73, 242, 166, 10, 79, 130, 131, 12, 85, 195, 9, 69, + 6, 150, 170, 22, 73, 226, 79, 80, 3, 84, 6, 240, 230, 12, 3, 71, 72, 84, + 242, 212, 9, 86, 155, 39, 67, 74, 170, 1, 65, 86, 67, 54, 69, 62, 72, + 178, 1, 73, 46, 76, 62, 80, 106, 84, 186, 227, 17, 87, 166, 155, 1, 78, + 234, 63, 79, 150, 173, 1, 77, 254, 102, 81, 254, 32, 75, 247, 47, 85, 6, + 228, 193, 3, 9, 67, 82, 73, 70, 73, 67, 73, 65, 76, 214, 153, 19, 76, + 175, 28, 89, 4, 248, 238, 10, 2, 82, 73, 161, 215, 5, 2, 72, 79, 8, 158, + 186, 21, 67, 142, 51, 65, 146, 8, 76, 167, 129, 1, 69, 10, 18, 69, 39, + 79, 4, 218, 236, 21, 76, 171, 137, 1, 69, 6, 40, 4, 82, 84, 32, 84, 151, + 217, 22, 79, 4, 44, 5, 65, 73, 76, 69, 68, 175, 249, 15, 72, 2, 145, 243, + 20, 2, 32, 66, 4, 228, 229, 17, 2, 67, 75, 247, 139, 5, 76, 6, 26, 65, + 179, 183, 22, 73, 4, 214, 221, 22, 86, 199, 21, 83, 10, 50, 69, 34, 73, + 242, 254, 18, 82, 223, 164, 3, 79, 4, 250, 186, 22, 65, 183, 22, 69, 2, + 255, 214, 22, 82, 12, 34, 69, 34, 79, 235, 228, 21, 65, 4, 166, 226, 22, + 65, 219, 16, 80, 6, 26, 80, 243, 219, 22, 78, 5, 191, 161, 22, 80, 30, + 82, 65, 98, 73, 34, 79, 38, 82, 52, 2, 85, 82, 32, 2, 87, 79, 135, 160, + 22, 69, 6, 38, 78, 142, 205, 21, 66, 247, 26, 76, 2, 33, 6, 78, 69, 68, + 32, 76, 69, 2, 179, 221, 13, 65, 4, 142, 179, 22, 71, 155, 39, 76, 4, + 198, 229, 18, 78, 135, 137, 2, 79, 6, 250, 228, 16, 73, 206, 219, 4, 65, + 159, 153, 1, 69, 4, 130, 166, 21, 66, 219, 37, 84, 5, 239, 176, 19, 32, + 4, 162, 212, 12, 65, 217, 148, 5, 3, 73, 76, 76, 26, 58, 65, 110, 69, 42, + 72, 32, 2, 73, 78, 30, 79, 39, 82, 6, 32, 2, 76, 75, 215, 176, 22, 84, 5, + 11, 32, 2, 11, 69, 2, 17, 2, 78, 67, 2, 153, 226, 17, 2, 76, 79, 4, 136, + 158, 22, 2, 65, 80, 195, 51, 83, 4, 214, 166, 21, 73, 147, 51, 69, 4, + 174, 237, 22, 68, 3, 69, 4, 146, 163, 21, 77, 235, 198, 1, 82, 4, 246, + 155, 22, 79, 239, 80, 65, 2, 201, 206, 20, 2, 69, 76, 184, 1, 92, 2, 76, + 69, 148, 2, 5, 83, 73, 71, 78, 32, 150, 190, 17, 65, 250, 8, 86, 147, + 177, 3, 68, 110, 44, 5, 84, 84, 69, 82, 32, 187, 169, 22, 78, 108, 234, + 196, 17, 78, 150, 204, 1, 65, 38, 68, 46, 76, 38, 82, 34, 84, 46, 86, + 186, 5, 85, 202, 141, 1, 79, 134, 60, 73, 206, 193, 1, 83, 82, 66, 2, 67, + 2, 71, 2, 74, 2, 75, 2, 80, 162, 7, 69, 234, 61, 70, 2, 72, 2, 77, 3, 89, + 22, 94, 67, 138, 1, 83, 246, 211, 18, 78, 242, 60, 65, 174, 158, 1, 74, + 150, 3, 85, 167, 242, 1, 86, 4, 100, 19, 79, 77, 66, 73, 78, 73, 78, 71, + 32, 65, 78, 85, 83, 86, 65, 82, 65, 32, 65, 195, 211, 18, 65, 2, 225, + 230, 19, 3, 66, 79, 86, 4, 224, 190, 12, 6, 80, 65, 67, 73, 78, 71, 163, + 134, 5, 73, 162, 2, 62, 32, 173, 11, 10, 45, 72, 73, 82, 65, 71, 65, 78, + 65, 32, 154, 2, 140, 1, 7, 76, 69, 84, 84, 69, 82, 32, 242, 9, 86, 148, + 237, 18, 10, 68, 73, 71, 82, 65, 80, 72, 32, 75, 79, 246, 203, 1, 73, + 135, 145, 1, 77, 146, 2, 186, 1, 65, 178, 1, 66, 106, 77, 186, 2, 78, 54, + 83, 226, 1, 68, 2, 71, 2, 72, 2, 75, 2, 80, 2, 82, 2, 84, 2, 86, 2, 90, + 126, 87, 46, 89, 246, 219, 22, 69, 2, 73, 2, 79, 3, 85, 19, 60, 4, 73, + 78, 85, 32, 45, 7, 82, 67, 72, 65, 73, 67, 32, 8, 130, 7, 84, 234, 197, + 22, 67, 215, 22, 80, 8, 38, 89, 134, 190, 22, 87, 235, 36, 69, 4, 234, + 226, 22, 69, 3, 73, 20, 50, 73, 158, 226, 22, 65, 2, 69, 2, 79, 3, 85, + 13, 253, 4, 9, 68, 65, 75, 85, 79, 78, 32, 78, 71, 36, 50, 73, 182, 225, + 22, 65, 2, 69, 2, 79, 3, 85, 29, 29, 5, 78, 78, 65, 78, 32, 26, 96, 15, + 78, 65, 83, 65, 76, 73, 90, 69, 68, 32, 84, 79, 78, 69, 45, 69, 5, 84, + 79, 78, 69, 45, 14, 174, 224, 22, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, + 3, 56, 12, 234, 223, 22, 50, 2, 51, 2, 52, 2, 53, 2, 55, 3, 56, 13, 174, + 223, 22, 65, 2, 69, 2, 73, 2, 79, 3, 85, 76, 76, 5, 77, 65, 76, 76, 32, + 174, 222, 22, 65, 2, 69, 2, 73, 2, 79, 3, 85, 66, 142, 1, 72, 2, 82, 54, + 75, 46, 84, 30, 87, 46, 89, 206, 136, 19, 78, 242, 146, 3, 83, 210, 27, + 77, 234, 36, 65, 2, 69, 2, 73, 2, 79, 3, 85, 10, 154, 221, 22, 65, 2, 69, + 2, 73, 2, 79, 3, 85, 8, 230, 220, 22, 65, 2, 69, 2, 79, 3, 85, 4, 186, + 220, 22, 79, 3, 85, 8, 158, 220, 22, 65, 2, 69, 2, 73, 3, 79, 6, 242, + 219, 22, 65, 2, 79, 3, 85, 2, 241, 184, 20, 5, 79, 73, 67, 69, 68, 8, 52, + 5, 68, 79, 85, 66, 76, 22, 80, 38, 83, 35, 86, 2, 139, 250, 7, 69, 2, 89, + 6, 82, 79, 76, 79, 78, 71, 2, 29, 5, 69, 77, 73, 45, 86, 2, 21, 3, 79, + 73, 67, 2, 33, 6, 69, 68, 32, 83, 79, 85, 2, 191, 141, 22, 78, 174, 1, + 216, 1, 7, 76, 69, 84, 84, 69, 82, 32, 128, 2, 12, 80, 85, 78, 67, 84, + 85, 65, 84, 73, 79, 78, 32, 148, 3, 5, 83, 73, 71, 78, 32, 88, 11, 86, + 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 170, 163, 12, 68, 195, 209, 6, + 67, 94, 226, 1, 65, 166, 160, 4, 74, 230, 219, 14, 68, 114, 84, 230, 5, + 85, 22, 86, 186, 201, 1, 73, 162, 193, 1, 78, 46, 83, 82, 66, 2, 67, 2, + 71, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, 2, 82, 2, 87, 2, 89, 186, 2, + 69, 3, 79, 7, 130, 214, 22, 65, 3, 73, 22, 122, 67, 90, 68, 58, 70, 54, + 83, 20, 12, 65, 76, 84, 69, 82, 78, 65, 84, 69, 32, 83, 69, 233, 193, 21, + 4, 84, 82, 73, 80, 4, 56, 8, 76, 79, 83, 73, 78, 71, 32, 83, 195, 130, + 21, 73, 2, 225, 202, 21, 2, 80, 73, 4, 11, 79, 4, 216, 194, 21, 2, 85, + 66, 175, 145, 1, 84, 4, 196, 129, 21, 5, 73, 76, 76, 69, 68, 139, 57, 76, + 6, 18, 69, 23, 80, 2, 195, 237, 10, 67, 4, 236, 144, 4, 2, 65, 67, 199, + 184, 17, 73, 12, 242, 219, 7, 75, 162, 226, 10, 67, 98, 78, 162, 59, 82, + 154, 162, 1, 86, 179, 241, 1, 65, 20, 66, 65, 214, 158, 4, 86, 190, 223, + 14, 69, 2, 85, 207, 201, 1, 73, 6, 148, 222, 16, 8, 76, 84, 69, 82, 78, + 65, 84, 69, 162, 243, 5, 65, 3, 73, 96, 148, 1, 7, 76, 69, 84, 84, 69, + 82, 32, 200, 1, 5, 83, 73, 71, 78, 32, 44, 5, 84, 79, 78, 69, 32, 92, 6, + 86, 79, 87, 69, 76, 32, 231, 219, 20, 68, 56, 138, 189, 18, 79, 134, 7, + 72, 154, 145, 2, 78, 238, 178, 1, 75, 2, 80, 2, 83, 2, 84, 138, 69, 66, + 2, 67, 2, 68, 2, 71, 2, 76, 2, 77, 2, 82, 2, 86, 2, 87, 2, 89, 2, 90, + 186, 2, 65, 3, 73, 4, 182, 236, 4, 67, 249, 222, 17, 2, 83, 72, 6, 36, 5, + 67, 65, 76, 89, 65, 23, 80, 5, 17, 2, 32, 80, 2, 173, 223, 17, 3, 76, 79, + 80, 10, 226, 140, 22, 69, 2, 85, 163, 64, 79, 36, 24, 2, 76, 86, 23, 89, + 2, 159, 211, 20, 73, 35, 68, 5, 66, 79, 65, 82, 68, 36, 4, 67, 65, 80, + 32, 187, 244, 2, 72, 5, 245, 134, 19, 4, 32, 65, 78, 68, 26, 254, 136, + 17, 78, 238, 174, 3, 65, 182, 34, 68, 187, 29, 84, 188, 13, 202, 1, 65, + 212, 9, 18, 73, 84, 65, 78, 32, 83, 77, 65, 76, 76, 32, 83, 67, 82, 73, + 80, 84, 32, 204, 3, 4, 77, 69, 82, 32, 204, 25, 5, 79, 74, 75, 73, 32, + 141, 6, 8, 85, 68, 65, 87, 65, 68, 73, 32, 138, 1, 56, 8, 82, 79, 83, 72, + 84, 72, 73, 32, 243, 163, 22, 78, 136, 1, 220, 1, 4, 68, 73, 71, 73, 20, + 7, 76, 69, 84, 84, 69, 82, 32, 144, 2, 7, 78, 85, 77, 66, 69, 82, 32, 36, + 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 136, 2, 5, 83, 73, + 71, 78, 32, 198, 1, 86, 183, 194, 8, 70, 8, 223, 134, 16, 84, 74, 182, 1, + 75, 42, 84, 234, 133, 7, 78, 186, 229, 11, 68, 238, 145, 3, 83, 82, 66, + 2, 67, 2, 71, 2, 80, 2, 86, 138, 69, 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, + 89, 2, 90, 187, 2, 65, 6, 138, 195, 22, 72, 2, 75, 187, 2, 65, 12, 142, + 236, 18, 84, 214, 214, 3, 72, 187, 2, 65, 8, 162, 203, 14, 84, 227, 165, + 2, 79, 18, 70, 67, 74, 68, 66, 76, 36, 5, 77, 65, 78, 71, 65, 159, 240, + 20, 83, 4, 26, 82, 247, 241, 20, 73, 2, 145, 162, 21, 6, 69, 83, 67, 69, + 78, 84, 6, 26, 79, 251, 230, 18, 65, 4, 194, 230, 18, 85, 219, 220, 3, + 84, 4, 226, 220, 6, 79, 227, 190, 7, 73, 2, 215, 135, 21, 76, 12, 72, 2, + 66, 65, 22, 67, 20, 2, 68, 79, 166, 139, 20, 86, 179, 241, 1, 65, 2, 239, + 196, 21, 82, 2, 203, 147, 6, 65, 4, 56, 8, 85, 66, 76, 69, 32, 82, 73, + 78, 207, 147, 20, 84, 2, 203, 147, 20, 71, 14, 44, 5, 79, 87, 69, 76, 32, + 247, 135, 20, 73, 12, 44, 5, 83, 73, 71, 78, 32, 163, 254, 21, 76, 10, + 182, 141, 4, 86, 230, 178, 18, 69, 2, 73, 2, 79, 3, 85, 176, 7, 72, 12, + 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 179, 200, 7, 70, 174, 7, + 22, 66, 147, 1, 67, 128, 4, 170, 247, 7, 48, 2, 49, 2, 50, 2, 51, 2, 52, + 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, 68, 2, 69, 3, + 70, 174, 3, 142, 1, 68, 142, 245, 7, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, + 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 163, 198, 13, 70, + 12, 206, 188, 22, 48, 2, 49, 2, 50, 2, 51, 2, 52, 3, 53, 246, 2, 202, 1, + 67, 240, 2, 18, 73, 78, 68, 69, 80, 69, 78, 68, 69, 78, 84, 32, 86, 79, + 87, 69, 76, 32, 220, 2, 7, 76, 69, 84, 84, 69, 82, 32, 254, 2, 83, 208, + 12, 6, 86, 79, 87, 69, 76, 32, 143, 180, 20, 68, 70, 176, 1, 20, 79, 78, + 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 67, 79, 69, 78, 71, 32, + 173, 172, 17, 17, 85, 82, 82, 69, 78, 67, 89, 32, 83, 89, 77, 66, 79, 76, + 32, 82, 73, 68, 138, 1, 78, 154, 4, 67, 2, 75, 90, 80, 74, 84, 54, 68, 2, + 76, 138, 234, 21, 83, 158, 41, 77, 2, 82, 2, 86, 2, 89, 190, 28, 66, 3, + 72, 8, 142, 153, 22, 71, 2, 89, 246, 30, 65, 3, 79, 42, 82, 81, 196, 1, + 11, 83, 73, 71, 78, 32, 67, 79, 69, 78, 71, 32, 50, 76, 3, 82, 26, 82, + 65, 44, 6, 79, 79, 32, 84, 89, 80, 34, 85, 134, 172, 20, 73, 223, 137, 2, + 69, 8, 170, 182, 22, 65, 2, 73, 2, 81, 3, 85, 4, 11, 69, 4, 207, 142, 10, + 32, 9, 198, 185, 7, 85, 155, 252, 14, 75, 8, 18, 81, 31, 82, 4, 166, 181, + 22, 69, 3, 85, 4, 215, 149, 18, 89, 70, 138, 1, 67, 2, 75, 42, 78, 50, + 80, 30, 83, 46, 84, 54, 68, 2, 76, 166, 147, 22, 77, 2, 82, 2, 86, 2, 89, + 190, 28, 66, 2, 72, 3, 81, 8, 210, 1, 72, 154, 178, 22, 65, 3, 79, 8, + 206, 148, 22, 71, 2, 78, 2, 89, 247, 30, 79, 6, 122, 72, 155, 178, 22, + 79, 6, 130, 148, 22, 83, 190, 28, 72, 187, 2, 65, 12, 50, 72, 0, 2, 84, + 72, 154, 178, 22, 65, 3, 79, 4, 150, 178, 22, 65, 3, 79, 130, 1, 60, 4, 73, 71, 78, 32, 221, 6, 6, 89, 77, 66, 79, 76, 32, 46, 202, 2, 65, 104, 10, 83, 65, 77, 89, 79, 75, 32, 83, 65, 78, 22, 66, 118, 67, 86, 75, 74, - 82, 54, 84, 244, 191, 4, 3, 76, 69, 75, 244, 10, 6, 80, 72, 78, 65, 69, - 75, 180, 204, 11, 11, 89, 85, 85, 75, 65, 76, 69, 65, 80, 73, 78, 152, - 97, 3, 78, 73, 75, 244, 251, 2, 9, 77, 85, 85, 83, 73, 75, 65, 84, 79, - 249, 13, 4, 86, 73, 82, 73, 6, 100, 9, 86, 65, 75, 82, 65, 72, 65, 83, - 65, 152, 189, 17, 4, 84, 84, 72, 65, 153, 220, 3, 2, 72, 83, 2, 215, 187, - 21, 78, 8, 38, 65, 209, 180, 20, 3, 69, 89, 89, 6, 174, 6, 84, 216, 1, 2, - 78, 84, 189, 238, 19, 6, 82, 73, 89, 79, 79, 83, 4, 224, 244, 6, 12, 65, - 77, 78, 85, 67, 32, 80, 73, 73, 32, 75, 85, 155, 140, 12, 79, 6, 188, - 163, 11, 2, 65, 75, 230, 209, 8, 72, 245, 77, 4, 79, 79, 77, 85, 4, 130, - 163, 11, 79, 133, 208, 5, 4, 69, 65, 72, 77, 4, 136, 168, 20, 8, 79, 65, + 82, 54, 84, 144, 223, 4, 3, 76, 69, 75, 168, 11, 6, 80, 72, 78, 65, 69, + 75, 248, 226, 10, 10, 89, 85, 85, 75, 65, 76, 69, 65, 80, 73, 200, 238, + 1, 3, 78, 73, 75, 180, 165, 3, 9, 77, 85, 85, 83, 73, 75, 65, 84, 79, + 133, 15, 4, 86, 73, 82, 73, 6, 100, 9, 86, 65, 75, 82, 65, 72, 65, 83, + 65, 176, 128, 18, 4, 84, 84, 72, 65, 209, 136, 4, 2, 72, 83, 2, 167, 171, + 22, 78, 8, 38, 65, 137, 164, 21, 3, 69, 89, 89, 6, 174, 6, 84, 216, 1, 2, + 78, 84, 193, 219, 20, 6, 82, 73, 89, 79, 79, 83, 4, 152, 149, 7, 12, 65, + 77, 78, 85, 67, 32, 80, 73, 73, 32, 75, 85, 211, 212, 12, 79, 6, 140, + 214, 11, 2, 65, 75, 154, 140, 9, 72, 177, 80, 4, 79, 79, 77, 85, 4, 210, + 213, 11, 79, 249, 224, 5, 4, 69, 65, 72, 77, 4, 192, 151, 21, 8, 79, 65, 78, 68, 65, 75, 72, 73, 253, 2, 4, 82, 73, 73, 83, 84, 128, 1, 3, 68, 65, 80, 92, 10, 76, 69, 75, 32, 65, 84, 84, 65, 75, 32, 182, 1, 80, 72, 5, 84, 85, 84, 69, 89, 78, 66, 47, 77, 24, 22, 45, 223, 3, 32, 20, 30, 80, 238, 2, 66, 47, 77, 8, 138, 3, 73, 37, 3, 82, 65, 77, 20, 50, 80, 98, 66, - 130, 221, 10, 77, 187, 139, 10, 83, 12, 36, 3, 82, 65, 77, 251, 164, 21, - 73, 11, 11, 45, 8, 42, 66, 130, 221, 10, 77, 255, 158, 8, 80, 4, 182, - 232, 20, 85, 139, 60, 69, 26, 44, 2, 65, 84, 44, 3, 82, 65, 77, 91, 73, - 2, 21, 3, 72, 65, 77, 2, 223, 237, 15, 65, 20, 18, 45, 119, 32, 16, 34, + 130, 143, 11, 77, 255, 200, 10, 83, 12, 36, 3, 82, 65, 77, 203, 148, 22, + 73, 11, 11, 45, 8, 42, 66, 130, 143, 11, 77, 239, 213, 8, 80, 4, 250, + 215, 21, 85, 151, 60, 69, 26, 44, 2, 65, 84, 44, 3, 82, 65, 77, 91, 73, + 2, 21, 3, 72, 65, 77, 2, 223, 178, 16, 65, 20, 18, 45, 119, 32, 16, 34, 66, 32, 2, 80, 73, 15, 77, 8, 30, 69, 37, 3, 85, 79, 78, 4, 35, 73, 4, - 21, 3, 85, 79, 89, 4, 11, 32, 4, 34, 82, 229, 128, 21, 2, 75, 79, 2, 143, - 143, 20, 79, 42, 76, 10, 73, 78, 72, 69, 82, 69, 78, 84, 32, 65, 29, 5, - 83, 73, 71, 78, 32, 4, 138, 181, 21, 65, 3, 81, 38, 102, 65, 54, 73, 30, - 79, 38, 89, 180, 228, 2, 5, 67, 79, 69, 78, 71, 250, 150, 7, 85, 235, - 183, 11, 69, 10, 218, 142, 3, 65, 174, 165, 18, 69, 2, 73, 3, 85, 7, 210, - 179, 21, 69, 3, 73, 6, 182, 179, 21, 69, 2, 77, 3, 79, 7, 146, 179, 21, + 21, 3, 85, 79, 89, 4, 11, 32, 4, 34, 82, 169, 240, 21, 2, 75, 79, 2, 195, + 253, 20, 79, 42, 76, 10, 73, 78, 72, 69, 82, 69, 78, 84, 32, 65, 29, 5, + 83, 73, 71, 78, 32, 4, 218, 164, 22, 65, 3, 81, 38, 102, 65, 54, 73, 30, + 79, 38, 89, 184, 253, 2, 5, 67, 79, 69, 78, 71, 250, 165, 7, 85, 183, + 255, 11, 69, 10, 174, 174, 3, 65, 170, 245, 18, 69, 2, 73, 3, 85, 7, 162, + 163, 22, 69, 3, 73, 6, 134, 163, 22, 69, 2, 77, 3, 79, 7, 226, 162, 22, 65, 3, 89, 130, 1, 146, 1, 68, 88, 7, 76, 69, 84, 84, 69, 82, 32, 170, 2, - 83, 164, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 226, 156, 15, - 87, 219, 80, 65, 6, 48, 6, 79, 85, 66, 76, 69, 32, 179, 220, 11, 65, 4, - 166, 248, 9, 83, 255, 227, 1, 68, 90, 234, 1, 83, 130, 5, 66, 42, 71, - 246, 230, 5, 68, 222, 137, 4, 74, 214, 208, 7, 65, 82, 84, 222, 225, 1, - 76, 246, 189, 1, 78, 126, 67, 2, 75, 2, 80, 254, 68, 72, 2, 77, 2, 81, 2, - 82, 2, 86, 2, 89, 186, 2, 69, 2, 73, 2, 79, 3, 85, 4, 26, 72, 131, 175, - 21, 65, 2, 249, 238, 20, 3, 79, 82, 84, 12, 40, 4, 73, 71, 78, 32, 191, - 245, 9, 69, 10, 58, 83, 138, 205, 15, 86, 170, 251, 1, 78, 223, 160, 3, - 65, 4, 26, 72, 243, 222, 16, 85, 2, 11, 65, 2, 203, 136, 21, 68, 18, 246, - 202, 3, 86, 238, 253, 13, 65, 142, 222, 1, 73, 206, 134, 2, 69, 2, 79, 3, - 85, 138, 1, 100, 7, 76, 69, 84, 84, 69, 82, 32, 216, 2, 5, 83, 73, 71, - 78, 32, 170, 236, 15, 86, 227, 206, 3, 68, 94, 222, 1, 66, 42, 71, 42, - 74, 206, 230, 5, 68, 230, 162, 11, 82, 206, 55, 65, 82, 84, 230, 24, 85, - 210, 200, 1, 73, 158, 190, 1, 78, 126, 67, 2, 75, 2, 80, 2, 83, 254, 68, - 72, 2, 76, 2, 77, 2, 86, 2, 89, 186, 2, 69, 3, 79, 6, 226, 167, 21, 66, - 2, 72, 187, 2, 65, 6, 186, 167, 21, 71, 2, 72, 187, 2, 65, 6, 146, 167, - 21, 72, 2, 74, 187, 2, 65, 6, 178, 200, 15, 86, 170, 251, 1, 78, 223, - 160, 3, 65, 20, 92, 2, 83, 83, 140, 245, 19, 4, 87, 73, 70, 82, 158, 44, - 80, 216, 93, 2, 77, 79, 191, 18, 84, 13, 40, 4, 73, 78, 71, 32, 187, 230, - 20, 32, 8, 96, 4, 70, 65, 67, 69, 209, 212, 13, 14, 67, 65, 84, 32, 70, - 65, 67, 69, 32, 87, 73, 84, 72, 32, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, - 198, 211, 13, 83, 115, 67, 4, 236, 210, 14, 3, 69, 69, 76, 167, 183, 6, - 79, 4, 40, 4, 82, 69, 65, 78, 207, 131, 21, 65, 2, 33, 6, 32, 83, 84, 65, - 78, 68, 2, 129, 141, 18, 2, 65, 82, 2, 11, 79, 2, 155, 253, 9, 78, 128, - 41, 154, 1, 65, 190, 216, 1, 69, 242, 56, 73, 230, 83, 79, 146, 22, 85, - 94, 89, 182, 206, 7, 82, 168, 249, 8, 5, 32, 66, 32, 66, 65, 174, 157, 1, - 76, 227, 66, 70, 130, 22, 182, 1, 66, 44, 6, 67, 82, 79, 83, 83, 69, 46, - 68, 56, 2, 79, 32, 222, 11, 82, 208, 6, 4, 83, 84, 32, 81, 76, 4, 84, 73, - 78, 32, 161, 168, 19, 8, 78, 71, 85, 65, 71, 69, 32, 84, 4, 212, 131, 14, - 2, 32, 67, 183, 149, 6, 69, 2, 153, 173, 16, 6, 32, 83, 84, 73, 67, 75, - 4, 184, 192, 12, 5, 89, 32, 66, 69, 69, 183, 163, 8, 68, 174, 1, 200, 2, - 3, 72, 79, 32, 28, 7, 76, 69, 84, 84, 69, 82, 32, 214, 5, 83, 148, 1, 9, - 84, 79, 78, 69, 32, 77, 65, 73, 32, 84, 11, 86, 79, 87, 69, 76, 32, 83, - 73, 71, 78, 32, 128, 137, 4, 2, 75, 79, 188, 211, 12, 2, 89, 65, 162, - 157, 2, 69, 250, 46, 68, 248, 132, 1, 7, 67, 65, 78, 67, 69, 76, 76, 205, - 68, 6, 78, 73, 71, 71, 65, 72, 4, 210, 255, 20, 77, 3, 78, 94, 176, 1, 3, - 70, 79, 32, 78, 75, 96, 2, 76, 79, 50, 80, 154, 1, 83, 86, 84, 30, 72, - 206, 212, 15, 78, 214, 165, 5, 66, 2, 67, 2, 68, 2, 77, 2, 82, 2, 87, 2, - 89, 247, 30, 79, 8, 42, 70, 206, 235, 14, 83, 199, 247, 4, 84, 4, 246, - 204, 20, 79, 143, 62, 65, 10, 26, 72, 147, 156, 21, 79, 8, 32, 3, 77, 85, - 32, 231, 2, 79, 4, 182, 195, 20, 78, 195, 57, 71, 7, 17, 2, 32, 76, 4, - 202, 202, 20, 73, 67, 79, 30, 52, 4, 65, 76, 73, 32, 210, 1, 72, 151, - 153, 21, 79, 24, 190, 165, 6, 68, 34, 84, 206, 6, 78, 254, 246, 12, 66, - 2, 67, 2, 71, 2, 74, 167, 213, 1, 76, 8, 52, 9, 65, 78, 83, 75, 82, 73, - 84, 32, 83, 71, 79, 4, 146, 151, 21, 72, 3, 83, 6, 26, 72, 151, 153, 21, - 79, 4, 11, 79, 4, 11, 32, 4, 250, 231, 14, 83, 199, 247, 4, 84, 6, 112, - 14, 69, 77, 73, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 137, 147, 3, - 8, 73, 71, 78, 32, 80, 65, 76, 73, 4, 174, 191, 20, 78, 195, 57, 76, 8, - 50, 84, 252, 209, 16, 2, 67, 65, 255, 193, 4, 69, 4, 154, 248, 20, 72, - 247, 30, 73, 32, 106, 65, 44, 5, 77, 65, 73, 32, 75, 226, 162, 17, 89, - 222, 35, 85, 210, 200, 1, 69, 2, 73, 207, 134, 2, 79, 11, 130, 150, 21, - 65, 2, 73, 2, 77, 3, 89, 4, 130, 198, 20, 65, 3, 79, 56, 32, 2, 71, 69, - 179, 142, 20, 73, 54, 26, 32, 231, 188, 13, 82, 50, 254, 1, 66, 44, 4, - 71, 82, 69, 69, 22, 79, 250, 1, 84, 186, 180, 11, 68, 36, 2, 85, 80, 168, - 180, 3, 12, 76, 69, 70, 84, 32, 84, 82, 73, 65, 78, 71, 76, 224, 151, 4, + 83, 160, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 230, 225, 15, + 87, 211, 80, 65, 6, 48, 6, 79, 85, 66, 76, 69, 32, 219, 196, 18, 65, 4, + 170, 160, 10, 83, 163, 164, 8, 68, 90, 234, 1, 83, 254, 4, 66, 42, 71, + 186, 134, 6, 68, 214, 134, 12, 74, 158, 50, 65, 150, 1, 84, 218, 207, 1, + 76, 250, 192, 1, 78, 126, 67, 2, 75, 2, 80, 138, 69, 72, 2, 77, 2, 81, 2, + 82, 2, 86, 2, 89, 186, 2, 69, 2, 73, 2, 79, 3, 85, 4, 26, 72, 211, 158, + 22, 65, 2, 189, 222, 21, 3, 79, 82, 84, 12, 40, 4, 73, 71, 78, 32, 195, + 157, 10, 69, 10, 54, 83, 238, 137, 18, 78, 206, 61, 86, 159, 145, 3, 65, + 4, 26, 72, 187, 162, 17, 85, 2, 11, 65, 2, 159, 248, 21, 68, 18, 170, + 234, 3, 86, 154, 223, 14, 65, 242, 201, 1, 73, 222, 137, 2, 69, 2, 79, 3, + 85, 138, 1, 100, 7, 76, 69, 84, 84, 69, 82, 32, 176, 2, 5, 83, 73, 71, + 78, 32, 202, 177, 16, 86, 187, 246, 3, 68, 94, 222, 1, 66, 42, 71, 186, + 134, 6, 68, 174, 133, 12, 74, 198, 51, 65, 118, 82, 34, 84, 230, 5, 85, + 206, 201, 1, 73, 162, 193, 1, 78, 126, 67, 2, 75, 2, 80, 2, 83, 138, 69, + 72, 2, 76, 2, 77, 2, 86, 2, 89, 186, 2, 69, 3, 79, 6, 182, 151, 22, 66, + 2, 72, 187, 2, 65, 6, 142, 151, 22, 71, 2, 72, 187, 2, 65, 6, 190, 133, + 18, 78, 206, 61, 86, 159, 145, 3, 65, 136, 1, 140, 1, 8, 82, 65, 84, 32, + 82, 65, 73, 32, 216, 3, 2, 83, 83, 208, 223, 20, 4, 87, 73, 70, 82, 182, + 45, 80, 240, 93, 2, 77, 79, 191, 18, 84, 116, 140, 1, 7, 76, 69, 84, 84, + 69, 82, 32, 172, 1, 5, 83, 73, 71, 78, 32, 92, 11, 86, 79, 87, 69, 76, + 32, 83, 73, 71, 78, 32, 243, 229, 11, 68, 64, 206, 188, 18, 68, 114, 84, + 226, 222, 1, 78, 238, 178, 1, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 2, + 83, 138, 69, 72, 2, 76, 2, 77, 2, 82, 2, 86, 2, 89, 187, 2, 65, 12, 208, + 202, 9, 3, 84, 79, 78, 0, 2, 89, 85, 242, 251, 1, 83, 230, 137, 10, 65, + 239, 1, 86, 16, 246, 192, 18, 65, 174, 147, 3, 85, 162, 64, 69, 2, 73, 3, + 79, 13, 40, 4, 73, 78, 71, 32, 163, 210, 21, 32, 8, 96, 4, 70, 65, 67, + 69, 181, 135, 14, 14, 67, 65, 84, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, + 32, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 162, 134, 14, 83, 215, 144, 5, + 67, 4, 168, 146, 18, 3, 69, 69, 76, 215, 227, 3, 79, 4, 40, 4, 82, 69, + 65, 78, 187, 239, 21, 65, 2, 33, 6, 32, 83, 84, 65, 78, 68, 2, 129, 228, + 18, 2, 65, 82, 2, 11, 79, 2, 191, 161, 10, 78, 174, 43, 154, 1, 65, 150, + 232, 1, 69, 182, 62, 73, 238, 83, 79, 222, 28, 85, 94, 89, 226, 224, 7, + 82, 244, 179, 9, 5, 32, 66, 32, 66, 65, 166, 160, 1, 76, 239, 66, 70, + 252, 22, 182, 1, 66, 44, 6, 67, 82, 79, 83, 83, 69, 46, 68, 56, 2, 79, + 32, 222, 11, 82, 236, 21, 4, 83, 84, 32, 81, 76, 4, 84, 73, 78, 32, 237, + 129, 20, 8, 78, 71, 85, 65, 71, 69, 32, 84, 4, 176, 182, 14, 2, 32, 67, + 175, 206, 6, 69, 2, 153, 237, 16, 6, 32, 83, 84, 73, 67, 75, 4, 220, 242, + 12, 5, 89, 32, 66, 69, 69, 243, 220, 8, 68, 174, 1, 200, 2, 3, 72, 79, + 32, 28, 7, 76, 69, 84, 84, 69, 82, 32, 214, 5, 83, 148, 1, 9, 84, 79, 78, + 69, 32, 77, 65, 73, 32, 84, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, + 32, 188, 164, 4, 2, 75, 79, 224, 247, 12, 2, 89, 65, 230, 194, 2, 69, + 190, 50, 68, 236, 135, 1, 7, 67, 65, 78, 67, 69, 76, 76, 221, 68, 6, 78, + 73, 71, 71, 65, 72, 4, 190, 235, 21, 77, 3, 78, 94, 176, 1, 3, 70, 79, + 32, 78, 75, 96, 2, 76, 79, 50, 80, 154, 1, 83, 86, 84, 30, 72, 234, 149, + 16, 78, 166, 208, 5, 66, 2, 67, 2, 68, 2, 77, 2, 82, 2, 87, 2, 89, 247, + 30, 79, 8, 42, 70, 254, 162, 15, 83, 195, 170, 5, 84, 4, 214, 184, 21, + 79, 155, 62, 65, 10, 26, 72, 255, 135, 22, 79, 8, 32, 3, 77, 85, 32, 231, + 2, 79, 4, 146, 175, 21, 78, 211, 57, 71, 7, 17, 2, 32, 76, 4, 170, 182, + 21, 73, 67, 79, 30, 52, 4, 65, 76, 73, 32, 210, 1, 72, 131, 133, 22, 79, + 24, 154, 194, 6, 68, 34, 84, 206, 6, 78, 138, 195, 13, 66, 2, 67, 2, 71, + 2, 74, 171, 216, 1, 76, 8, 52, 9, 65, 78, 83, 75, 82, 73, 84, 32, 83, 71, + 79, 4, 254, 130, 22, 72, 3, 83, 6, 26, 72, 131, 133, 22, 79, 4, 11, 79, + 4, 11, 32, 4, 170, 159, 15, 83, 195, 170, 5, 84, 6, 112, 14, 69, 77, 73, + 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 169, 173, 18, 8, 73, 71, 78, + 32, 80, 65, 76, 73, 4, 138, 171, 21, 78, 211, 57, 76, 8, 50, 84, 220, + 145, 17, 2, 67, 65, 139, 238, 4, 69, 4, 134, 228, 21, 72, 247, 30, 73, + 32, 106, 65, 44, 5, 77, 65, 73, 32, 75, 146, 226, 17, 89, 142, 76, 85, + 206, 201, 1, 69, 2, 73, 223, 137, 2, 79, 11, 238, 129, 22, 65, 2, 73, 2, + 77, 3, 89, 4, 226, 177, 21, 65, 3, 79, 166, 1, 32, 2, 71, 69, 131, 250, + 20, 73, 164, 1, 26, 32, 199, 239, 13, 82, 160, 1, 254, 1, 66, 44, 4, 71, + 82, 69, 69, 22, 79, 250, 1, 84, 182, 231, 11, 68, 36, 2, 85, 80, 176, + 192, 3, 12, 76, 69, 70, 84, 32, 84, 82, 73, 65, 78, 71, 76, 164, 193, 4, 5, 80, 85, 82, 80, 76, 12, 3, 82, 69, 68, 0, 6, 89, 69, 76, 76, 79, 87, - 207, 64, 67, 10, 40, 3, 82, 79, 87, 201, 1, 2, 76, 85, 4, 251, 130, 19, - 78, 10, 48, 3, 78, 69, 32, 129, 1, 4, 82, 65, 78, 71, 4, 200, 248, 7, 5, - 68, 79, 84, 32, 79, 173, 221, 2, 18, 82, 73, 78, 71, 32, 79, 86, 69, 82, - 32, 84, 87, 79, 32, 82, 73, 78, 71, 6, 11, 69, 6, 11, 32, 6, 230, 193, - 19, 67, 226, 19, 68, 231, 26, 83, 6, 104, 3, 87, 79, 32, 161, 229, 17, - 17, 82, 73, 80, 76, 69, 32, 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, - 4, 162, 162, 17, 68, 205, 64, 19, 82, 73, 78, 71, 83, 32, 79, 86, 69, 82, - 32, 79, 78, 69, 32, 82, 73, 78, 71, 6, 53, 11, 85, 65, 82, 84, 69, 82, - 32, 77, 79, 79, 78, 7, 167, 220, 6, 32, 138, 20, 150, 1, 67, 244, 45, 18, + 243, 64, 67, 10, 40, 3, 82, 79, 87, 201, 1, 2, 76, 85, 4, 195, 235, 19, + 78, 10, 48, 3, 78, 69, 32, 129, 1, 4, 82, 65, 78, 71, 4, 160, 155, 8, 5, + 68, 79, 84, 32, 79, 181, 233, 2, 18, 82, 73, 78, 71, 32, 79, 86, 69, 82, + 32, 84, 87, 79, 32, 82, 73, 78, 71, 6, 11, 69, 6, 11, 32, 6, 210, 170, + 20, 67, 154, 21, 68, 139, 28, 83, 116, 156, 1, 3, 87, 79, 32, 108, 10, + 89, 80, 69, 32, 80, 73, 69, 67, 69, 32, 157, 186, 18, 17, 82, 73, 80, 76, + 69, 32, 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 4, 150, 225, 17, 68, + 193, 87, 19, 82, 73, 78, 71, 83, 32, 79, 86, 69, 82, 32, 79, 78, 69, 32, + 82, 73, 78, 71, 110, 182, 1, 67, 136, 2, 9, 68, 73, 65, 71, 79, 78, 65, + 76, 32, 218, 1, 76, 186, 2, 82, 158, 1, 83, 204, 3, 6, 85, 80, 80, 69, + 82, 32, 145, 2, 9, 86, 69, 82, 84, 69, 88, 32, 79, 70, 14, 80, 9, 69, 78, + 84, 82, 69, 32, 79, 70, 32, 73, 7, 82, 79, 83, 83, 66, 65, 82, 8, 252, 8, + 6, 90, 32, 87, 73, 84, 72, 142, 239, 21, 75, 2, 88, 3, 89, 7, 33, 6, 32, + 87, 73, 84, 72, 32, 4, 40, 3, 76, 79, 87, 1, 3, 85, 80, 80, 2, 181, 197, + 10, 2, 69, 82, 12, 48, 6, 85, 80, 80, 69, 82, 32, 227, 222, 9, 76, 8, 56, + 4, 76, 69, 70, 84, 157, 223, 9, 4, 82, 73, 71, 72, 5, 11, 32, 2, 21, 3, + 65, 78, 68, 2, 17, 2, 32, 76, 2, 17, 2, 79, 87, 2, 213, 194, 21, 2, 69, + 82, 26, 48, 5, 79, 87, 69, 82, 32, 129, 3, 2, 69, 70, 24, 84, 5, 76, 69, + 70, 84, 32, 56, 6, 82, 73, 71, 72, 84, 32, 158, 6, 72, 179, 1, 84, 8, 22, + 65, 207, 7, 67, 4, 194, 1, 78, 159, 202, 20, 82, 10, 22, 65, 151, 7, 67, + 6, 192, 1, 13, 78, 68, 32, 85, 80, 80, 69, 82, 32, 82, 73, 71, 72, 193, + 189, 19, 2, 82, 67, 4, 44, 4, 65, 73, 83, 69, 77, 3, 73, 71, 72, 2, 53, + 11, 68, 32, 85, 80, 80, 69, 82, 32, 76, 69, 70, 2, 187, 245, 18, 84, 2, + 141, 243, 20, 3, 84, 32, 65, 34, 48, 5, 72, 79, 82, 84, 32, 77, 3, 84, + 69, 77, 4, 40, 3, 76, 79, 87, 1, 3, 85, 80, 80, 2, 217, 4, 4, 69, 82, 32, + 84, 31, 44, 6, 32, 87, 73, 84, 72, 32, 171, 1, 45, 8, 44, 5, 76, 69, 70, + 84, 32, 30, 82, 59, 67, 4, 82, 67, 167, 214, 10, 74, 2, 21, 3, 73, 71, + 72, 2, 11, 84, 2, 17, 2, 32, 67, 2, 177, 205, 20, 4, 82, 79, 83, 83, 20, + 48, 2, 49, 50, 22, 50, 26, 52, 247, 181, 11, 51, 5, 207, 211, 14, 51, 9, + 11, 51, 7, 11, 52, 5, 243, 237, 21, 53, 18, 62, 72, 96, 3, 76, 69, 70, 0, + 4, 82, 73, 71, 72, 83, 84, 4, 65, 14, 65, 76, 70, 32, 86, 69, 82, 84, 69, + 88, 32, 79, 70, 32, 4, 218, 236, 21, 77, 3, 87, 6, 17, 2, 84, 32, 6, 26, + 67, 131, 240, 18, 65, 4, 186, 92, 82, 211, 147, 17, 79, 2, 201, 243, 8, + 3, 69, 82, 77, 2, 211, 219, 14, 32, 6, 53, 11, 85, 65, 82, 84, 69, 82, + 32, 77, 79, 79, 78, 7, 151, 234, 6, 32, 150, 20, 150, 1, 67, 148, 46, 18, 69, 80, 73, 71, 82, 65, 80, 72, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, - 252, 1, 7, 76, 69, 84, 84, 69, 82, 32, 183, 13, 83, 192, 7, 56, 8, 65, - 80, 73, 84, 65, 76, 32, 76, 239, 206, 19, 82, 190, 7, 76, 6, 69, 84, 84, - 69, 82, 32, 189, 44, 8, 73, 71, 65, 84, 85, 82, 69, 32, 186, 7, 154, 2, + 252, 1, 7, 76, 69, 84, 84, 69, 82, 32, 183, 13, 83, 200, 7, 56, 8, 65, + 80, 73, 84, 65, 76, 32, 76, 139, 168, 20, 82, 198, 7, 76, 6, 69, 84, 84, + 69, 82, 32, 221, 44, 8, 73, 71, 65, 84, 85, 82, 69, 32, 194, 7, 154, 2, 65, 158, 3, 66, 154, 1, 67, 254, 1, 68, 250, 1, 69, 174, 2, 70, 82, 71, - 194, 1, 72, 146, 1, 73, 154, 2, 74, 118, 75, 126, 76, 222, 2, 77, 114, - 78, 134, 2, 79, 198, 2, 80, 102, 81, 62, 82, 174, 2, 83, 254, 2, 84, 142, + 194, 1, 72, 146, 1, 73, 154, 2, 74, 118, 75, 126, 76, 234, 2, 77, 114, + 78, 134, 2, 79, 198, 2, 80, 102, 81, 62, 82, 190, 2, 83, 130, 3, 84, 142, 3, 85, 234, 1, 86, 146, 1, 87, 118, 88, 50, 89, 167, 1, 90, 93, 172, 1, - 6, 32, 87, 73, 84, 72, 32, 166, 1, 69, 182, 65, 78, 166, 209, 4, 76, 236, - 154, 9, 6, 70, 82, 73, 67, 65, 78, 178, 136, 4, 86, 138, 209, 2, 65, 2, - 79, 2, 85, 3, 89, 66, 182, 64, 66, 34, 68, 108, 3, 82, 73, 78, 242, 33, - 77, 254, 21, 67, 146, 49, 72, 158, 1, 79, 198, 10, 71, 186, 2, 65, 206, - 156, 3, 73, 214, 241, 12, 84, 175, 145, 3, 83, 7, 33, 6, 32, 87, 73, 84, - 72, 32, 4, 142, 183, 1, 65, 183, 182, 3, 77, 21, 60, 6, 32, 87, 73, 84, - 72, 32, 254, 67, 82, 175, 157, 20, 69, 14, 218, 66, 84, 146, 60, 70, 202, - 56, 76, 134, 224, 15, 68, 222, 157, 2, 72, 231, 160, 1, 83, 33, 108, 6, - 32, 87, 73, 84, 72, 32, 140, 70, 7, 76, 79, 83, 69, 68, 32, 73, 54, 85, - 154, 238, 19, 79, 255, 59, 72, 20, 94, 67, 134, 180, 1, 65, 222, 215, 3, - 80, 142, 167, 14, 72, 186, 46, 66, 246, 33, 68, 187, 80, 83, 8, 134, 68, - 69, 190, 112, 73, 255, 170, 18, 65, 31, 44, 6, 32, 87, 73, 84, 72, 32, - 179, 1, 90, 24, 78, 83, 182, 15, 67, 214, 47, 84, 218, 116, 76, 134, 224, - 15, 68, 223, 157, 2, 72, 8, 92, 13, 77, 65, 76, 76, 32, 76, 69, 84, 84, - 69, 82, 32, 90, 246, 136, 1, 72, 223, 200, 19, 84, 5, 177, 72, 2, 32, 87, - 91, 104, 6, 32, 87, 73, 84, 72, 32, 152, 1, 2, 90, 72, 134, 76, 71, 194, - 191, 16, 84, 210, 242, 3, 83, 63, 78, 70, 254, 73, 67, 158, 2, 68, 194, - 16, 84, 162, 23, 77, 250, 1, 86, 234, 44, 72, 158, 1, 79, 198, 10, 71, - 186, 2, 65, 206, 156, 3, 73, 62, 66, 199, 130, 16, 83, 7, 11, 32, 4, 138, - 70, 87, 211, 8, 82, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 246, 173, 19, - 72, 174, 80, 68, 187, 80, 83, 35, 76, 6, 32, 87, 73, 84, 72, 32, 190, 81, - 76, 138, 225, 17, 65, 227, 200, 2, 72, 20, 154, 84, 67, 250, 89, 65, 138, - 157, 3, 66, 174, 25, 77, 142, 133, 4, 79, 170, 195, 10, 72, 174, 80, 68, - 187, 80, 83, 29, 76, 6, 32, 87, 73, 84, 72, 32, 138, 126, 65, 238, 218, - 7, 87, 191, 210, 11, 69, 20, 186, 82, 66, 34, 67, 46, 68, 182, 216, 18, - 72, 231, 160, 1, 83, 59, 80, 6, 32, 87, 73, 84, 72, 32, 132, 88, 2, 78, - 83, 186, 254, 19, 79, 207, 36, 83, 40, 138, 1, 68, 162, 83, 67, 242, 1, - 77, 142, 1, 84, 130, 70, 72, 158, 1, 79, 198, 10, 71, 186, 2, 65, 206, - 156, 3, 73, 62, 66, 199, 130, 16, 83, 10, 22, 79, 191, 83, 73, 6, 234, - 120, 85, 155, 232, 17, 84, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 42, 67, - 206, 183, 17, 84, 175, 145, 3, 83, 4, 234, 169, 1, 73, 199, 205, 3, 82, - 25, 33, 6, 32, 87, 73, 84, 72, 32, 22, 182, 89, 67, 34, 68, 50, 83, 222, - 78, 65, 138, 1, 76, 186, 186, 7, 79, 171, 195, 10, 72, 37, 48, 6, 32, 87, - 73, 84, 72, 32, 163, 247, 20, 74, 32, 138, 1, 66, 36, 2, 68, 79, 52, 7, - 77, 73, 68, 68, 76, 69, 32, 38, 83, 174, 2, 67, 198, 91, 72, 226, 71, 65, - 138, 1, 76, 155, 141, 16, 84, 4, 246, 151, 12, 69, 207, 165, 8, 65, 6, - 22, 85, 135, 92, 84, 2, 205, 223, 4, 2, 66, 76, 4, 146, 180, 17, 84, 255, - 240, 2, 68, 4, 214, 2, 77, 211, 194, 20, 84, 19, 44, 6, 32, 87, 73, 84, - 72, 32, 223, 94, 73, 10, 254, 164, 1, 65, 142, 225, 15, 68, 150, 45, 84, - 203, 240, 1, 72, 33, 48, 6, 32, 87, 73, 84, 72, 32, 215, 243, 20, 74, 28, - 102, 67, 44, 2, 83, 77, 194, 96, 76, 130, 64, 71, 186, 2, 65, 102, 68, - 222, 186, 7, 79, 227, 210, 8, 84, 6, 202, 131, 1, 69, 22, 73, 131, 203, - 18, 65, 2, 249, 194, 10, 10, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 101, - 108, 6, 32, 87, 73, 84, 72, 32, 194, 103, 76, 202, 140, 4, 80, 150, 253, - 8, 77, 250, 255, 6, 73, 2, 79, 3, 85, 84, 144, 1, 2, 76, 79, 34, 77, 242, - 96, 67, 70, 68, 154, 2, 79, 38, 83, 66, 84, 106, 86, 218, 43, 72, 242, - 12, 71, 186, 2, 65, 206, 156, 3, 73, 63, 66, 4, 174, 99, 78, 199, 140, - 20, 79, 8, 170, 99, 65, 171, 141, 4, 73, 17, 33, 6, 32, 87, 73, 84, 72, - 32, 14, 146, 104, 70, 34, 83, 162, 55, 65, 234, 254, 17, 72, 175, 80, 68, - 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 230, 104, 68, 39, 83, 41, 78, 32, - 156, 1, 8, 69, 86, 69, 82, 83, 69, 68, 32, 145, 111, 3, 85, 77, 32, 28, - 40, 5, 87, 73, 84, 72, 32, 143, 112, 82, 26, 174, 78, 67, 234, 28, 68, - 170, 2, 84, 158, 48, 65, 138, 1, 76, 198, 155, 3, 73, 246, 158, 4, 79, - 143, 228, 11, 83, 8, 146, 110, 72, 218, 138, 4, 79, 162, 246, 12, 67, - 171, 253, 2, 69, 47, 136, 1, 6, 32, 87, 73, 84, 72, 32, 132, 1, 5, 77, - 65, 76, 76, 32, 156, 115, 2, 65, 76, 234, 1, 72, 152, 2, 2, 73, 71, 235, - 248, 3, 67, 30, 82, 67, 194, 111, 65, 118, 68, 126, 83, 174, 1, 86, 178, - 227, 7, 79, 171, 195, 10, 72, 10, 154, 112, 65, 218, 10, 69, 82, 79, 203, - 31, 73, 4, 200, 188, 18, 11, 81, 32, 87, 73, 84, 72, 32, 72, 79, 79, 75, - 149, 237, 1, 7, 67, 65, 80, 73, 84, 65, 76, 59, 136, 1, 6, 32, 87, 73, - 84, 72, 32, 168, 1, 6, 85, 82, 78, 69, 68, 32, 232, 122, 2, 72, 79, 176, - 1, 2, 79, 78, 74, 82, 231, 233, 19, 90, 22, 82, 67, 50, 68, 170, 152, 1, - 76, 186, 220, 3, 82, 170, 161, 14, 72, 231, 160, 1, 83, 8, 246, 119, 69, - 22, 73, 62, 79, 199, 202, 18, 65, 6, 234, 141, 1, 73, 207, 234, 15, 79, - 18, 162, 40, 73, 146, 249, 2, 65, 146, 197, 17, 72, 2, 75, 2, 76, 2, 77, - 2, 84, 3, 86, 79, 26, 32, 179, 245, 4, 80, 74, 44, 5, 87, 73, 84, 72, 32, - 251, 194, 19, 66, 72, 210, 131, 1, 67, 74, 68, 150, 2, 72, 170, 1, 77, - 134, 1, 79, 142, 1, 84, 186, 9, 71, 186, 2, 65, 206, 156, 3, 73, 62, 66, - 186, 136, 14, 82, 143, 250, 1, 83, 23, 88, 6, 32, 87, 73, 84, 72, 32, - 222, 138, 1, 73, 66, 79, 174, 202, 18, 69, 227, 141, 1, 89, 8, 142, 138, - 1, 68, 242, 151, 16, 84, 203, 240, 1, 72, 19, 48, 6, 32, 87, 73, 84, 72, - 32, 159, 132, 7, 89, 14, 234, 143, 1, 67, 18, 68, 70, 71, 186, 2, 65, - 235, 254, 17, 72, 7, 245, 139, 1, 7, 32, 87, 73, 84, 72, 32, 68, 29, 48, - 6, 32, 87, 73, 84, 72, 32, 147, 233, 16, 79, 24, 198, 142, 1, 67, 18, 68, - 70, 71, 22, 72, 42, 76, 254, 1, 65, 182, 182, 3, 77, 238, 215, 12, 84, - 175, 145, 3, 83, 25, 33, 6, 32, 87, 73, 84, 72, 32, 22, 170, 57, 67, 150, - 87, 65, 102, 68, 38, 76, 34, 83, 182, 214, 3, 80, 143, 167, 14, 72, 4, - 250, 175, 10, 73, 231, 152, 10, 79, 12, 128, 1, 5, 65, 82, 67, 72, 65, - 30, 73, 64, 9, 82, 69, 86, 69, 82, 83, 69, 68, 32, 197, 173, 10, 7, 83, - 73, 68, 69, 87, 65, 89, 2, 213, 199, 13, 2, 73, 67, 4, 136, 107, 5, 78, - 86, 69, 82, 84, 149, 156, 14, 3, 32, 76, 79, 4, 174, 221, 20, 70, 3, 80, - 138, 1, 198, 3, 65, 36, 2, 66, 73, 152, 1, 21, 73, 78, 86, 69, 82, 84, - 69, 68, 32, 71, 76, 79, 84, 84, 65, 76, 32, 83, 84, 79, 80, 72, 2, 82, - 69, 222, 1, 83, 162, 28, 87, 132, 33, 3, 84, 87, 79, 204, 246, 3, 2, 68, - 69, 140, 5, 2, 76, 65, 206, 11, 71, 248, 174, 1, 21, 80, 72, 65, 82, 89, - 78, 71, 69, 65, 76, 32, 86, 79, 73, 67, 69, 68, 32, 70, 82, 73, 192, 181, - 13, 20, 86, 79, 73, 67, 69, 68, 32, 76, 65, 82, 89, 78, 71, 69, 65, 76, - 32, 83, 80, 73, 155, 116, 89, 4, 150, 169, 4, 76, 223, 224, 15, 73, 6, - 76, 7, 76, 65, 66, 73, 65, 76, 32, 29, 8, 68, 69, 78, 84, 65, 76, 32, 80, - 4, 26, 80, 143, 188, 4, 67, 2, 153, 208, 15, 6, 69, 82, 67, 85, 83, 83, - 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 134, 225, 4, 67, 231, 198, 15, 83, - 8, 104, 7, 86, 69, 82, 83, 69, 68, 32, 149, 225, 4, 13, 84, 82, 79, 70, - 76, 69, 88, 32, 67, 76, 73, 67, 75, 4, 80, 3, 69, 83, 72, 193, 61, 12, - 71, 76, 79, 84, 84, 65, 76, 32, 83, 84, 79, 80, 2, 189, 132, 1, 2, 32, - 76, 96, 172, 1, 13, 77, 65, 76, 76, 32, 67, 65, 80, 73, 84, 65, 76, 32, - 220, 111, 10, 84, 82, 69, 84, 67, 72, 69, 68, 32, 67, 225, 211, 18, 10, - 73, 78, 79, 76, 79, 71, 73, 67, 65, 76, 90, 230, 1, 69, 30, 73, 22, 76, - 74, 79, 42, 82, 122, 84, 186, 164, 4, 66, 202, 41, 71, 166, 194, 15, 65, - 150, 64, 67, 2, 68, 2, 70, 2, 72, 2, 74, 2, 75, 2, 77, 2, 78, 2, 80, 2, - 81, 2, 83, 2, 85, 2, 86, 2, 87, 2, 89, 3, 90, 7, 190, 209, 20, 84, 3, 90, - 5, 191, 201, 4, 78, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 198, 199, 8, 66, - 187, 218, 11, 83, 9, 214, 94, 80, 130, 243, 19, 69, 3, 85, 11, 88, 8, 69, - 86, 69, 82, 83, 69, 68, 32, 216, 127, 5, 32, 87, 73, 84, 72, 167, 192, - 19, 85, 4, 210, 208, 20, 78, 3, 82, 13, 33, 6, 85, 82, 78, 69, 68, 32, - 10, 146, 208, 20, 69, 2, 71, 2, 75, 2, 77, 3, 82, 180, 11, 136, 1, 5, 77, - 65, 76, 76, 32, 253, 129, 1, 22, 85, 66, 83, 67, 82, 73, 80, 84, 32, 83, - 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 144, 11, 76, 15, 67, 65, - 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 27, 76, 4, 182, 53, - 73, 3, 85, 140, 11, 76, 6, 69, 84, 84, 69, 82, 32, 161, 127, 8, 73, 71, - 65, 84, 85, 82, 69, 32, 250, 10, 182, 2, 65, 190, 5, 66, 198, 3, 67, 186, - 4, 68, 182, 4, 69, 174, 9, 70, 214, 1, 71, 162, 2, 72, 230, 2, 73, 166, - 7, 74, 182, 1, 75, 178, 2, 76, 210, 6, 77, 178, 2, 78, 218, 3, 79, 242, - 8, 80, 230, 1, 81, 174, 1, 82, 142, 8, 83, 234, 10, 84, 246, 13, 85, 218, - 9, 86, 130, 3, 87, 110, 88, 226, 2, 89, 171, 3, 90, 101, 118, 32, 198, 3, - 69, 82, 78, 132, 208, 4, 4, 76, 80, 72, 65, 190, 164, 13, 86, 138, 209, - 2, 65, 2, 79, 2, 85, 3, 89, 72, 88, 5, 87, 73, 84, 72, 32, 209, 209, 5, - 11, 82, 69, 86, 69, 82, 83, 69, 68, 45, 83, 67, 70, 150, 1, 66, 34, 68, - 54, 82, 170, 34, 77, 254, 21, 67, 146, 49, 72, 158, 1, 79, 198, 10, 71, - 186, 2, 65, 206, 156, 3, 73, 214, 241, 12, 84, 175, 145, 3, 83, 12, 161, - 105, 4, 82, 69, 86, 69, 12, 22, 79, 155, 57, 73, 8, 218, 57, 84, 223, 12, - 85, 10, 26, 73, 255, 212, 4, 69, 8, 26, 78, 131, 156, 4, 71, 6, 17, 2, - 71, 32, 6, 240, 58, 4, 65, 66, 79, 86, 159, 243, 17, 66, 9, 33, 6, 32, - 87, 73, 84, 72, 32, 6, 242, 115, 71, 186, 2, 65, 183, 182, 3, 77, 2, 133, - 160, 10, 7, 71, 76, 73, 67, 65, 78, 65, 41, 140, 1, 6, 32, 87, 73, 84, - 72, 32, 130, 1, 65, 108, 11, 76, 65, 67, 75, 76, 69, 84, 84, 69, 82, 32, - 38, 82, 242, 182, 4, 79, 191, 230, 15, 69, 18, 110, 84, 146, 60, 70, 202, - 56, 76, 170, 207, 3, 77, 174, 7, 80, 178, 137, 12, 68, 222, 157, 2, 72, - 231, 160, 1, 83, 2, 255, 7, 79, 8, 64, 5, 82, 82, 69, 68, 32, 161, 80, 6, - 83, 69, 76, 73, 78, 69, 6, 182, 32, 65, 142, 162, 20, 69, 3, 79, 6, 186, - 177, 4, 79, 227, 144, 16, 69, 2, 133, 148, 10, 4, 79, 75, 69, 78, 47, - 108, 6, 32, 87, 73, 84, 72, 32, 196, 1, 2, 72, 73, 92, 6, 76, 79, 83, 69, - 68, 32, 90, 85, 155, 238, 19, 79, 24, 102, 67, 182, 112, 65, 222, 215, 3, - 80, 230, 5, 82, 170, 161, 14, 72, 186, 46, 66, 246, 33, 68, 187, 80, 83, - 10, 54, 69, 190, 112, 73, 238, 225, 6, 85, 147, 201, 11, 65, 4, 249, 51, - 5, 68, 73, 76, 76, 65, 7, 49, 10, 32, 87, 73, 84, 72, 32, 76, 79, 87, 32, - 4, 162, 106, 82, 45, 4, 76, 69, 70, 84, 8, 34, 73, 18, 79, 227, 181, 4, - 82, 2, 163, 87, 78, 4, 210, 202, 4, 80, 199, 243, 8, 77, 4, 37, 7, 65, - 84, 82, 73, 76, 76, 79, 5, 153, 224, 12, 5, 32, 87, 73, 84, 72, 73, 96, - 6, 32, 87, 73, 84, 72, 32, 182, 1, 69, 34, 79, 178, 1, 90, 246, 194, 4, - 66, 235, 229, 15, 85, 32, 98, 83, 34, 84, 130, 33, 67, 206, 44, 77, 170, - 31, 76, 206, 199, 3, 72, 138, 15, 80, 179, 137, 12, 68, 4, 134, 67, 72, - 223, 200, 19, 84, 4, 26, 79, 167, 142, 18, 65, 2, 215, 235, 17, 80, 8, - 246, 77, 90, 195, 200, 19, 76, 16, 60, 6, 84, 76, 69, 83, 83, 32, 49, 5, - 85, 66, 76, 69, 32, 8, 26, 74, 139, 186, 20, 73, 7, 235, 179, 4, 32, 8, - 42, 87, 210, 183, 4, 82, 187, 219, 5, 84, 2, 163, 219, 6, 89, 11, 11, 32, - 8, 26, 87, 219, 179, 4, 68, 2, 169, 89, 5, 73, 84, 72, 32, 67, 119, 112, - 6, 32, 87, 73, 84, 72, 32, 214, 4, 71, 72, 2, 78, 71, 68, 2, 83, 72, 164, - 1, 2, 90, 72, 147, 189, 16, 84, 76, 182, 1, 67, 158, 2, 68, 110, 78, 214, - 15, 84, 162, 23, 77, 250, 1, 86, 194, 3, 70, 170, 41, 72, 158, 1, 79, - 198, 10, 71, 186, 2, 65, 206, 156, 3, 73, 62, 66, 186, 64, 82, 143, 194, - 15, 83, 24, 92, 6, 69, 68, 73, 76, 76, 65, 32, 9, 73, 82, 67, 85, 77, 70, - 76, 69, 88, 179, 145, 19, 65, 5, 129, 131, 4, 3, 32, 65, 78, 19, 11, 32, - 16, 40, 4, 65, 78, 68, 32, 207, 156, 18, 66, 14, 162, 85, 67, 130, 2, 72, - 226, 11, 71, 186, 2, 65, 182, 182, 3, 77, 190, 162, 4, 68, 179, 181, 8, - 84, 12, 22, 79, 227, 97, 73, 10, 28, 2, 84, 32, 243, 50, 85, 8, 192, 87, - 5, 65, 66, 79, 86, 69, 239, 195, 17, 66, 2, 183, 232, 13, 79, 4, 237, - 237, 6, 13, 89, 80, 84, 79, 76, 79, 71, 73, 67, 65, 76, 32, 65, 7, 33, 6, - 32, 87, 73, 84, 72, 32, 4, 226, 176, 4, 67, 231, 9, 80, 13, 33, 6, 32, - 87, 73, 84, 72, 32, 10, 88, 10, 68, 79, 85, 66, 76, 69, 32, 66, 65, 82, - 170, 185, 4, 80, 142, 1, 67, 219, 4, 82, 5, 157, 186, 4, 4, 32, 65, 78, - 68, 15, 11, 32, 12, 38, 82, 37, 5, 87, 73, 84, 72, 32, 2, 209, 228, 13, - 4, 69, 86, 69, 82, 10, 54, 67, 246, 183, 4, 80, 230, 5, 82, 139, 197, 13, - 84, 4, 194, 194, 7, 85, 147, 201, 11, 65, 17, 84, 6, 32, 87, 73, 84, 72, - 32, 73, 11, 69, 78, 71, 32, 68, 73, 71, 82, 65, 80, 72, 10, 202, 175, 4, - 77, 174, 7, 80, 142, 167, 14, 72, 174, 80, 68, 187, 80, 83, 5, 197, 189, - 17, 8, 32, 87, 73, 84, 72, 32, 84, 82, 37, 72, 6, 32, 87, 73, 84, 72, 32, - 126, 76, 138, 225, 17, 65, 227, 200, 2, 72, 22, 218, 3, 67, 250, 89, 65, - 138, 157, 3, 66, 174, 25, 77, 170, 33, 80, 230, 227, 3, 79, 170, 195, 10, - 72, 174, 80, 68, 187, 80, 83, 8, 33, 6, 79, 84, 84, 65, 76, 32, 8, 178, - 225, 17, 83, 202, 202, 2, 65, 2, 73, 3, 85, 39, 140, 1, 6, 32, 87, 73, - 84, 72, 32, 166, 44, 65, 216, 25, 12, 79, 79, 75, 69, 68, 32, 83, 67, 72, - 87, 65, 32, 242, 225, 3, 69, 207, 130, 16, 86, 24, 86, 66, 34, 67, 46, - 68, 214, 90, 76, 214, 214, 3, 80, 142, 167, 14, 72, 231, 160, 1, 83, 2, - 157, 229, 12, 3, 82, 69, 86, 6, 158, 58, 69, 154, 32, 73, 255, 170, 18, - 65, 8, 234, 86, 73, 178, 173, 3, 69, 203, 182, 12, 79, 75, 80, 6, 32, 87, - 73, 84, 72, 32, 222, 4, 78, 188, 1, 2, 79, 84, 251, 161, 20, 83, 48, 178, - 1, 67, 34, 68, 210, 1, 77, 64, 6, 79, 71, 79, 78, 69, 75, 78, 84, 130, - 70, 72, 226, 11, 71, 186, 2, 65, 206, 156, 3, 73, 62, 66, 136, 64, 6, 83, - 84, 82, 79, 75, 69, 51, 82, 4, 210, 87, 73, 255, 170, 18, 65, 14, 18, 73, - 47, 79, 4, 221, 26, 7, 65, 69, 82, 69, 83, 73, 83, 10, 28, 2, 84, 32, - 231, 36, 85, 8, 64, 10, 65, 66, 79, 86, 69, 32, 65, 78, 68, 32, 227, 140, - 18, 66, 6, 150, 83, 71, 186, 2, 65, 163, 142, 16, 84, 4, 29, 5, 65, 67, - 82, 79, 78, 5, 233, 35, 4, 32, 65, 78, 68, 7, 145, 72, 15, 32, 65, 78, - 68, 32, 68, 79, 84, 32, 65, 66, 79, 86, 69, 32, 4, 21, 3, 73, 76, 68, 4, - 171, 251, 4, 69, 16, 46, 83, 93, 7, 86, 69, 82, 84, 69, 68, 32, 12, 29, - 5, 85, 76, 65, 82, 32, 12, 226, 162, 20, 68, 2, 70, 2, 71, 2, 82, 2, 83, - 3, 84, 4, 26, 65, 187, 139, 20, 79, 2, 231, 173, 4, 76, 6, 150, 145, 4, - 65, 229, 162, 6, 5, 73, 70, 73, 69, 68, 13, 33, 6, 32, 87, 73, 84, 72, - 32, 10, 94, 67, 216, 161, 4, 13, 68, 79, 84, 32, 65, 66, 79, 86, 69, 32, - 65, 78, 68, 235, 206, 15, 83, 6, 178, 81, 73, 198, 205, 3, 82, 187, 221, - 14, 65, 29, 48, 6, 32, 87, 73, 84, 72, 32, 163, 157, 20, 82, 24, 98, 67, - 34, 68, 50, 83, 222, 78, 65, 138, 1, 76, 214, 214, 3, 80, 230, 227, 3, - 79, 171, 195, 10, 72, 4, 210, 47, 69, 151, 203, 18, 65, 6, 214, 69, 73, - 134, 180, 3, 69, 175, 174, 4, 79, 4, 29, 5, 84, 82, 79, 75, 69, 5, 177, - 24, 6, 32, 65, 78, 68, 32, 68, 77, 156, 1, 6, 32, 87, 73, 84, 72, 32, - 248, 3, 5, 65, 77, 66, 68, 65, 22, 69, 48, 5, 79, 78, 71, 32, 83, 194, - 161, 4, 83, 2, 90, 234, 229, 15, 85, 219, 16, 74, 50, 178, 1, 66, 86, 67, - 56, 2, 68, 79, 100, 3, 77, 73, 68, 242, 1, 72, 226, 71, 65, 138, 1, 76, - 218, 206, 3, 73, 162, 1, 82, 222, 2, 70, 130, 4, 80, 198, 182, 12, 84, - 175, 145, 3, 83, 6, 36, 3, 69, 76, 84, 147, 226, 19, 65, 5, 241, 162, 4, - 6, 32, 65, 78, 68, 32, 80, 8, 146, 43, 69, 22, 73, 242, 129, 7, 85, 147, - 201, 11, 65, 8, 38, 84, 25, 5, 85, 66, 76, 69, 32, 4, 137, 24, 2, 32, 66, - 4, 162, 154, 4, 77, 243, 220, 14, 66, 8, 36, 4, 68, 76, 69, 32, 187, 43, - 45, 6, 198, 215, 16, 84, 178, 240, 2, 82, 79, 68, 2, 139, 155, 4, 32, 6, - 222, 162, 4, 90, 161, 192, 12, 3, 78, 73, 83, 9, 33, 6, 32, 87, 73, 84, - 72, 32, 6, 18, 68, 35, 72, 4, 202, 62, 73, 215, 216, 18, 79, 2, 133, 154, - 4, 2, 73, 71, 27, 56, 6, 32, 87, 73, 84, 72, 32, 102, 73, 151, 133, 20, - 85, 16, 134, 71, 65, 250, 205, 3, 67, 186, 2, 77, 174, 7, 80, 178, 137, - 12, 68, 150, 45, 84, 203, 240, 1, 72, 6, 25, 4, 68, 68, 76, 69, 6, 76, 7, - 45, 87, 69, 76, 83, 72, 32, 141, 194, 16, 6, 32, 83, 67, 79, 84, 83, 4, - 226, 139, 19, 76, 159, 137, 1, 86, 49, 58, 32, 216, 2, 2, 71, 32, 242, - 128, 20, 85, 219, 16, 74, 40, 88, 5, 87, 73, 84, 72, 32, 193, 250, 5, 11, - 80, 82, 69, 67, 69, 68, 69, 68, 32, 66, 89, 38, 122, 67, 74, 76, 154, 36, - 77, 234, 27, 71, 186, 2, 65, 102, 68, 250, 214, 3, 80, 230, 5, 82, 130, - 222, 3, 79, 227, 210, 8, 84, 10, 166, 35, 69, 22, 73, 202, 237, 3, 82, - 170, 148, 3, 85, 147, 201, 11, 65, 6, 128, 65, 3, 79, 78, 71, 202, 2, 73, - 135, 222, 3, 69, 2, 25, 4, 87, 73, 84, 72, 2, 149, 131, 17, 5, 32, 84, - 73, 76, 68, 115, 116, 6, 32, 87, 73, 84, 72, 32, 186, 6, 76, 52, 4, 80, - 69, 78, 32, 170, 137, 13, 77, 250, 255, 6, 73, 2, 79, 3, 85, 86, 154, 1, - 67, 70, 68, 152, 1, 2, 76, 79, 86, 77, 46, 79, 38, 83, 66, 84, 106, 86, - 218, 43, 72, 242, 12, 71, 186, 2, 65, 206, 156, 3, 73, 62, 66, 187, 64, - 82, 14, 168, 48, 9, 73, 82, 67, 85, 77, 70, 76, 69, 88, 187, 186, 18, 65, - 14, 18, 73, 47, 79, 4, 233, 12, 7, 65, 69, 82, 69, 83, 73, 83, 10, 22, - 84, 167, 46, 85, 6, 11, 32, 6, 152, 12, 5, 65, 66, 79, 86, 69, 247, 232, - 17, 66, 6, 66, 78, 188, 162, 12, 6, 87, 32, 82, 73, 78, 71, 139, 234, 7, - 79, 2, 155, 20, 71, 6, 11, 65, 6, 189, 2, 4, 67, 82, 79, 78, 4, 229, 10, - 5, 71, 79, 78, 69, 75, 4, 25, 4, 84, 82, 79, 75, 4, 11, 69, 5, 209, 48, - 2, 32, 65, 8, 25, 4, 73, 76, 68, 69, 9, 29, 5, 32, 65, 78, 68, 32, 6, - 158, 46, 68, 142, 13, 65, 183, 182, 3, 77, 6, 81, 18, 69, 82, 84, 73, 67, - 65, 76, 32, 76, 73, 78, 69, 32, 66, 69, 76, 79, 87, 7, 217, 42, 4, 32, - 65, 78, 68, 2, 229, 225, 9, 8, 68, 32, 80, 79, 76, 73, 83, 72, 16, 26, - 79, 195, 147, 4, 69, 13, 48, 6, 32, 87, 73, 84, 72, 32, 211, 136, 20, 69, - 8, 206, 54, 71, 186, 2, 65, 194, 221, 3, 82, 143, 194, 15, 83, 23, 48, 6, - 32, 87, 73, 84, 72, 32, 139, 244, 19, 72, 18, 86, 70, 34, 83, 162, 55, - 65, 178, 208, 3, 77, 174, 7, 80, 142, 167, 14, 72, 175, 80, 68, 2, 181, - 135, 14, 3, 76, 79, 85, 6, 202, 27, 84, 185, 130, 12, 6, 81, 85, 73, 82, - 82, 69, 13, 48, 6, 32, 87, 73, 84, 72, 32, 191, 143, 4, 80, 8, 42, 68, - 16, 4, 72, 79, 79, 75, 23, 83, 2, 211, 44, 73, 5, 179, 216, 17, 32, 2, - 181, 26, 6, 84, 82, 79, 75, 69, 32, 77, 90, 32, 228, 4, 8, 69, 86, 69, - 82, 83, 69, 68, 32, 148, 2, 2, 85, 77, 231, 129, 4, 65, 46, 36, 4, 87, - 73, 84, 72, 235, 6, 82, 44, 26, 32, 219, 253, 14, 79, 42, 166, 1, 67, 50, - 68, 200, 1, 8, 70, 73, 83, 72, 72, 79, 79, 75, 66, 76, 34, 84, 254, 17, - 77, 162, 30, 65, 206, 156, 3, 73, 146, 59, 80, 230, 227, 3, 79, 143, 228, - 11, 83, 6, 154, 19, 69, 222, 237, 3, 82, 187, 221, 14, 65, 8, 11, 79, 8, - 24, 2, 84, 32, 111, 85, 6, 26, 66, 239, 132, 19, 65, 4, 25, 4, 69, 76, - 79, 87, 5, 29, 5, 32, 65, 78, 68, 32, 2, 247, 231, 3, 77, 2, 21, 3, 66, - 76, 69, 2, 11, 32, 2, 211, 46, 71, 7, 29, 5, 32, 65, 78, 68, 32, 4, 138, - 129, 4, 77, 175, 7, 80, 4, 206, 49, 73, 211, 218, 3, 79, 4, 238, 210, 17, - 65, 251, 22, 73, 22, 162, 1, 72, 40, 6, 79, 80, 69, 78, 32, 69, 140, 130, - 4, 8, 82, 32, 87, 73, 84, 72, 32, 70, 168, 2, 3, 83, 67, 82, 218, 129, - 14, 69, 198, 84, 67, 171, 163, 1, 75, 2, 11, 65, 2, 189, 213, 9, 2, 76, - 70, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 202, 139, 4, 82, 171, 161, 14, - 72, 5, 11, 32, 2, 11, 82, 2, 153, 168, 10, 3, 79, 84, 85, 83, 180, 1, 6, - 32, 87, 73, 84, 72, 32, 202, 4, 65, 66, 67, 218, 1, 72, 34, 73, 156, 2, - 13, 81, 85, 65, 84, 32, 82, 69, 86, 69, 82, 83, 69, 68, 173, 198, 9, 6, - 84, 73, 82, 82, 85, 80, 38, 106, 65, 34, 67, 86, 68, 126, 83, 174, 1, 86, - 210, 9, 77, 254, 245, 3, 80, 230, 227, 3, 79, 171, 195, 10, 72, 4, 193, - 1, 4, 67, 85, 84, 69, 12, 58, 65, 218, 10, 69, 82, 79, 202, 31, 73, 239, - 225, 6, 85, 4, 101, 3, 82, 79, 78, 6, 21, 3, 79, 84, 32, 6, 26, 66, 211, - 252, 18, 65, 4, 25, 4, 69, 76, 79, 87, 5, 11, 32, 2, 185, 248, 18, 3, 65, + 6, 32, 87, 73, 84, 72, 32, 166, 1, 69, 230, 65, 78, 216, 146, 14, 6, 70, + 82, 73, 67, 65, 78, 222, 182, 3, 76, 130, 182, 2, 86, 182, 162, 1, 65, 2, + 79, 2, 85, 3, 89, 66, 230, 64, 66, 34, 68, 108, 3, 82, 73, 78, 242, 33, + 77, 250, 21, 67, 162, 49, 72, 158, 1, 79, 198, 10, 71, 186, 2, 65, 202, + 168, 3, 73, 186, 159, 13, 84, 227, 179, 3, 83, 7, 33, 6, 32, 87, 73, 84, + 72, 32, 4, 202, 183, 1, 65, 187, 194, 3, 77, 21, 60, 6, 32, 87, 73, 84, + 72, 32, 174, 68, 82, 207, 249, 20, 69, 14, 138, 67, 84, 142, 60, 70, 218, + 56, 76, 226, 143, 16, 68, 162, 199, 2, 72, 219, 163, 1, 83, 33, 108, 6, + 32, 87, 73, 84, 72, 32, 188, 70, 7, 76, 79, 83, 69, 68, 32, 73, 54, 85, + 174, 202, 20, 79, 139, 60, 72, 20, 94, 67, 194, 180, 1, 65, 230, 227, 3, + 80, 166, 244, 14, 72, 158, 50, 66, 238, 32, 68, 211, 80, 83, 8, 182, 68, + 69, 202, 112, 73, 255, 133, 19, 65, 31, 44, 6, 32, 87, 73, 84, 72, 32, + 179, 1, 90, 24, 78, 83, 194, 15, 67, 250, 47, 84, 230, 116, 76, 226, 143, + 16, 68, 163, 199, 2, 72, 8, 92, 13, 77, 65, 76, 76, 32, 76, 69, 84, 84, + 69, 82, 32, 90, 178, 137, 1, 72, 243, 164, 20, 84, 5, 225, 72, 2, 32, 87, + 91, 104, 6, 32, 87, 73, 84, 72, 32, 152, 1, 2, 90, 72, 182, 76, 71, 166, + 239, 16, 84, 142, 159, 4, 83, 63, 78, 70, 174, 74, 67, 158, 2, 68, 194, + 16, 84, 158, 23, 77, 250, 1, 86, 250, 44, 72, 158, 1, 79, 198, 10, 71, + 186, 2, 65, 202, 168, 3, 73, 62, 66, 223, 210, 16, 83, 7, 11, 32, 4, 186, + 70, 87, 211, 8, 82, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 210, 135, 20, + 72, 138, 83, 68, 211, 80, 83, 35, 76, 6, 32, 87, 73, 84, 72, 32, 238, 81, + 76, 202, 182, 18, 65, 195, 207, 2, 72, 20, 202, 84, 67, 134, 90, 65, 134, + 169, 3, 66, 182, 25, 77, 198, 140, 4, 79, 142, 137, 11, 72, 138, 83, 68, + 211, 80, 83, 29, 76, 6, 32, 87, 73, 84, 72, 32, 182, 126, 65, 246, 238, + 7, 87, 207, 154, 12, 69, 20, 234, 82, 66, 34, 67, 46, 68, 226, 177, 19, + 72, 219, 163, 1, 83, 59, 80, 6, 32, 87, 73, 84, 72, 32, 180, 88, 2, 78, + 83, 218, 218, 20, 79, 207, 36, 83, 40, 138, 1, 68, 210, 83, 67, 242, 1, + 77, 142, 1, 84, 142, 70, 72, 158, 1, 79, 198, 10, 71, 186, 2, 65, 202, + 168, 3, 73, 62, 66, 223, 210, 16, 83, 10, 22, 79, 239, 83, 73, 6, 150, + 121, 85, 231, 190, 18, 84, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 42, 67, + 234, 241, 17, 84, 227, 179, 3, 83, 4, 166, 170, 1, 73, 207, 217, 3, 82, + 25, 33, 6, 32, 87, 73, 84, 72, 32, 22, 230, 89, 67, 34, 68, 50, 83, 234, + 78, 65, 138, 1, 76, 246, 205, 7, 79, 143, 137, 11, 72, 41, 60, 6, 32, 87, + 73, 84, 72, 32, 238, 94, 65, 251, 244, 20, 74, 32, 138, 1, 66, 36, 2, 68, + 79, 52, 7, 77, 73, 68, 68, 76, 69, 32, 38, 83, 174, 2, 67, 230, 91, 72, + 242, 71, 65, 138, 1, 76, 251, 198, 16, 84, 4, 242, 186, 12, 69, 139, 223, + 8, 65, 6, 22, 85, 151, 92, 84, 2, 129, 236, 4, 2, 66, 76, 4, 162, 238, + 17, 84, 167, 147, 3, 68, 4, 214, 2, 77, 151, 159, 21, 84, 19, 44, 6, 32, + 87, 73, 84, 72, 32, 255, 94, 73, 10, 174, 165, 1, 65, 234, 144, 16, 68, + 154, 55, 84, 139, 144, 2, 72, 33, 48, 6, 32, 87, 73, 84, 72, 32, 155, + 208, 21, 74, 28, 102, 67, 44, 2, 83, 77, 226, 96, 76, 146, 64, 71, 186, + 2, 65, 102, 68, 154, 206, 7, 79, 135, 249, 8, 84, 6, 250, 131, 1, 69, 22, + 73, 131, 166, 19, 65, 2, 189, 226, 10, 10, 65, 76, 76, 32, 76, 69, 84, + 84, 69, 82, 101, 108, 6, 32, 87, 73, 84, 72, 32, 226, 103, 76, 226, 152, + 4, 80, 170, 150, 9, 77, 242, 182, 7, 73, 2, 79, 3, 85, 84, 144, 1, 2, 76, + 79, 34, 77, 146, 97, 67, 70, 68, 154, 2, 79, 38, 83, 66, 84, 106, 86, + 234, 43, 72, 242, 12, 71, 186, 2, 65, 202, 168, 3, 73, 63, 66, 4, 206, + 99, 78, 235, 232, 20, 79, 8, 202, 99, 65, 195, 153, 4, 73, 17, 33, 6, 32, + 87, 73, 84, 72, 32, 14, 178, 104, 70, 34, 83, 178, 55, 65, 138, 216, 18, + 72, 139, 83, 68, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 134, 105, 68, 39, + 83, 43, 94, 32, 156, 1, 8, 69, 86, 69, 82, 83, 69, 68, 32, 160, 111, 3, + 85, 77, 32, 243, 141, 4, 65, 28, 40, 5, 87, 73, 84, 72, 32, 159, 112, 82, + 26, 194, 78, 67, 230, 28, 68, 170, 2, 84, 174, 48, 65, 138, 1, 76, 194, + 167, 3, 73, 182, 166, 4, 79, 231, 172, 12, 83, 8, 162, 110, 72, 230, 150, + 4, 79, 202, 183, 13, 67, 155, 140, 3, 69, 49, 136, 1, 6, 32, 87, 73, 84, + 72, 32, 136, 1, 5, 77, 65, 76, 76, 32, 184, 115, 2, 65, 76, 234, 1, 72, + 152, 2, 2, 73, 71, 243, 132, 4, 67, 32, 86, 67, 210, 111, 65, 118, 68, + 138, 1, 83, 174, 1, 86, 238, 246, 7, 79, 143, 137, 11, 72, 10, 170, 112, + 65, 230, 10, 69, 82, 79, 203, 31, 73, 4, 152, 146, 19, 11, 81, 32, 87, + 73, 84, 72, 32, 72, 79, 79, 75, 233, 243, 1, 7, 67, 65, 80, 73, 84, 65, + 76, 59, 136, 1, 6, 32, 87, 73, 84, 72, 32, 168, 1, 6, 85, 82, 78, 69, 68, + 32, 132, 123, 2, 72, 79, 176, 1, 2, 79, 78, 74, 82, 251, 197, 20, 90, 22, + 82, 67, 50, 68, 198, 152, 1, 76, 182, 232, 3, 82, 206, 238, 14, 72, 219, + 163, 1, 83, 8, 146, 120, 69, 22, 73, 62, 79, 199, 165, 19, 65, 6, 134, + 142, 1, 73, 171, 154, 16, 79, 18, 178, 40, 73, 146, 133, 3, 65, 178, 149, + 18, 72, 2, 75, 2, 76, 2, 77, 2, 84, 3, 86, 79, 26, 32, 203, 129, 5, 80, + 74, 44, 5, 87, 73, 84, 72, 32, 155, 160, 20, 66, 72, 238, 131, 1, 67, 74, + 68, 150, 2, 72, 170, 1, 77, 134, 1, 79, 142, 1, 84, 186, 9, 71, 186, 2, + 65, 202, 168, 3, 73, 62, 66, 242, 209, 14, 82, 239, 128, 2, 83, 23, 88, + 6, 32, 87, 73, 84, 72, 32, 250, 138, 1, 73, 66, 79, 170, 166, 19, 69, + 251, 141, 1, 89, 8, 170, 138, 1, 68, 210, 209, 16, 84, 139, 144, 2, 72, + 19, 48, 6, 32, 87, 73, 84, 72, 32, 251, 145, 7, 89, 14, 134, 144, 1, 67, + 18, 68, 70, 71, 186, 2, 65, 139, 216, 18, 72, 7, 145, 140, 1, 7, 32, 87, + 73, 84, 72, 32, 68, 29, 48, 6, 32, 87, 73, 84, 72, 32, 135, 153, 17, 79, + 24, 226, 142, 1, 67, 18, 68, 70, 71, 22, 72, 42, 76, 254, 1, 65, 186, + 194, 3, 77, 202, 133, 13, 84, 227, 179, 3, 83, 25, 33, 6, 32, 87, 73, 84, + 72, 32, 22, 186, 57, 67, 162, 87, 65, 102, 68, 38, 76, 34, 83, 190, 226, + 3, 80, 167, 244, 14, 72, 4, 170, 207, 10, 73, 231, 213, 10, 79, 12, 128, + 1, 5, 65, 82, 67, 72, 65, 30, 73, 64, 9, 82, 69, 86, 69, 82, 83, 69, 68, + 32, 245, 204, 10, 7, 83, 73, 68, 69, 87, 65, 89, 2, 177, 235, 13, 2, 73, + 67, 4, 164, 107, 5, 78, 86, 69, 82, 84, 221, 205, 14, 3, 32, 76, 79, 4, + 222, 185, 21, 70, 3, 80, 138, 1, 198, 3, 65, 36, 2, 66, 73, 152, 1, 21, + 73, 78, 86, 69, 82, 84, 69, 68, 32, 71, 76, 79, 84, 84, 65, 76, 32, 83, + 84, 79, 80, 72, 2, 82, 69, 222, 1, 83, 128, 8, 3, 84, 87, 79, 178, 20, + 87, 216, 163, 4, 2, 68, 69, 148, 5, 2, 76, 65, 206, 11, 71, 128, 176, 1, + 21, 80, 72, 65, 82, 89, 78, 71, 69, 65, 76, 32, 86, 79, 73, 67, 69, 68, + 32, 70, 82, 73, 164, 131, 14, 20, 86, 79, 73, 67, 69, 68, 32, 76, 65, 82, + 89, 78, 71, 69, 65, 76, 32, 83, 80, 73, 179, 117, 89, 4, 174, 181, 4, 76, + 235, 176, 16, 73, 6, 76, 7, 76, 65, 66, 73, 65, 76, 32, 29, 8, 68, 69, + 78, 84, 65, 76, 32, 80, 4, 26, 80, 175, 200, 4, 67, 2, 161, 129, 16, 6, + 69, 82, 67, 85, 83, 83, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 170, 237, 4, + 67, 243, 150, 16, 83, 8, 104, 7, 86, 69, 82, 83, 69, 68, 32, 185, 237, 4, + 13, 84, 82, 79, 70, 76, 69, 88, 32, 67, 76, 73, 67, 75, 4, 80, 3, 69, 83, + 72, 157, 8, 12, 71, 76, 79, 84, 84, 65, 76, 32, 83, 84, 79, 80, 2, 217, + 132, 1, 2, 32, 76, 96, 172, 1, 13, 77, 65, 76, 76, 32, 67, 65, 80, 73, + 84, 65, 76, 32, 248, 111, 10, 84, 82, 69, 84, 67, 72, 69, 68, 32, 67, + 221, 175, 19, 10, 73, 78, 79, 76, 79, 71, 73, 67, 65, 76, 90, 230, 1, 69, + 30, 73, 22, 76, 74, 79, 42, 82, 122, 84, 210, 176, 4, 66, 214, 41, 71, + 166, 146, 16, 65, 162, 64, 67, 2, 68, 2, 70, 2, 72, 2, 74, 2, 75, 2, 77, + 2, 78, 2, 80, 2, 81, 2, 83, 2, 85, 2, 86, 2, 87, 2, 89, 3, 90, 7, 238, + 173, 21, 84, 3, 90, 5, 227, 213, 4, 78, 7, 33, 6, 32, 87, 73, 84, 72, 32, + 4, 162, 219, 8, 66, 143, 163, 12, 83, 9, 242, 94, 80, 150, 207, 20, 69, + 3, 85, 11, 88, 8, 69, 86, 69, 82, 83, 69, 68, 32, 244, 127, 5, 32, 87, + 73, 84, 72, 187, 156, 20, 85, 4, 130, 173, 21, 78, 3, 82, 13, 33, 6, 85, + 82, 78, 69, 68, 32, 10, 194, 172, 21, 69, 2, 71, 2, 75, 2, 77, 3, 82, + 184, 11, 136, 1, 5, 77, 65, 76, 76, 32, 153, 130, 1, 22, 85, 66, 83, 67, + 82, 73, 80, 84, 32, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, + 148, 11, 76, 15, 67, 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, + 32, 43, 76, 4, 18, 73, 3, 85, 2, 211, 220, 4, 32, 144, 11, 76, 6, 69, 84, + 84, 69, 82, 32, 173, 127, 8, 73, 71, 65, 84, 85, 82, 69, 32, 254, 10, + 182, 2, 65, 190, 5, 66, 198, 3, 67, 186, 4, 68, 182, 4, 69, 174, 9, 70, + 214, 1, 71, 162, 2, 72, 230, 2, 73, 166, 7, 74, 182, 1, 75, 178, 2, 76, + 206, 6, 77, 178, 2, 78, 218, 3, 79, 242, 8, 80, 230, 1, 81, 174, 1, 82, + 142, 8, 83, 250, 10, 84, 246, 13, 85, 218, 9, 86, 130, 3, 87, 110, 88, + 226, 2, 89, 171, 3, 90, 101, 118, 32, 198, 3, 69, 82, 78, 152, 220, 4, 4, + 76, 80, 72, 65, 158, 163, 15, 86, 182, 162, 1, 65, 2, 79, 2, 85, 3, 89, + 72, 88, 5, 87, 73, 84, 72, 32, 225, 222, 5, 11, 82, 69, 86, 69, 82, 83, + 69, 68, 45, 83, 67, 70, 150, 1, 66, 34, 68, 54, 82, 170, 34, 77, 250, 21, + 67, 162, 49, 72, 158, 1, 79, 198, 10, 71, 186, 2, 65, 202, 168, 3, 73, + 186, 159, 13, 84, 227, 179, 3, 83, 12, 173, 105, 4, 82, 69, 86, 69, 12, + 22, 79, 151, 57, 73, 8, 214, 57, 84, 223, 12, 85, 10, 26, 73, 135, 225, + 4, 69, 8, 26, 78, 139, 168, 4, 71, 6, 17, 2, 71, 32, 6, 236, 58, 4, 65, + 66, 79, 86, 235, 201, 18, 66, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 254, + 115, 71, 186, 2, 65, 187, 194, 3, 77, 2, 165, 191, 10, 7, 71, 76, 73, 67, + 65, 78, 65, 41, 140, 1, 6, 32, 87, 73, 84, 72, 32, 130, 1, 65, 108, 11, + 76, 65, 67, 75, 76, 69, 84, 84, 69, 82, 32, 38, 82, 134, 195, 4, 79, 203, + 182, 16, 69, 18, 110, 84, 142, 60, 70, 218, 56, 76, 178, 219, 3, 77, 174, + 7, 80, 134, 173, 12, 68, 162, 199, 2, 72, 219, 163, 1, 83, 2, 255, 7, 79, + 8, 64, 5, 82, 82, 69, 68, 32, 173, 80, 6, 83, 69, 76, 73, 78, 69, 6, 182, + 32, 65, 174, 254, 20, 69, 3, 79, 6, 202, 189, 4, 79, 243, 224, 16, 69, 2, + 165, 179, 10, 4, 79, 75, 69, 78, 47, 108, 6, 32, 87, 73, 84, 72, 32, 196, + 1, 2, 72, 73, 92, 6, 76, 79, 83, 69, 68, 32, 90, 85, 175, 202, 20, 79, + 24, 102, 67, 194, 112, 65, 230, 227, 3, 80, 218, 5, 82, 206, 238, 14, 72, + 158, 50, 66, 238, 32, 68, 211, 80, 83, 10, 54, 69, 202, 112, 73, 230, + 245, 6, 85, 155, 144, 12, 65, 4, 245, 51, 5, 68, 73, 76, 76, 65, 7, 49, + 10, 32, 87, 73, 84, 72, 32, 76, 79, 87, 32, 4, 174, 106, 82, 45, 4, 76, + 69, 70, 84, 8, 34, 73, 18, 79, 247, 193, 4, 82, 2, 175, 87, 78, 4, 218, + 214, 4, 80, 231, 140, 9, 77, 4, 37, 7, 65, 84, 82, 73, 76, 76, 79, 5, + 253, 130, 13, 5, 32, 87, 73, 84, 72, 73, 96, 6, 32, 87, 73, 84, 72, 32, + 182, 1, 69, 34, 79, 178, 1, 90, 138, 207, 4, 66, 247, 181, 16, 85, 32, + 98, 83, 34, 84, 238, 32, 67, 238, 44, 77, 170, 31, 76, 214, 211, 3, 72, + 138, 15, 80, 135, 173, 12, 68, 4, 146, 67, 72, 243, 164, 20, 84, 4, 26, + 79, 231, 227, 18, 65, 2, 243, 245, 19, 80, 8, 130, 78, 90, 215, 164, 20, + 76, 16, 60, 6, 84, 76, 69, 83, 83, 32, 49, 5, 85, 66, 76, 69, 32, 8, 26, + 74, 171, 150, 21, 73, 7, 255, 191, 4, 32, 8, 42, 87, 230, 195, 4, 82, + 199, 238, 5, 84, 2, 239, 232, 6, 89, 11, 11, 32, 8, 26, 87, 239, 191, 4, + 68, 2, 181, 89, 5, 73, 84, 72, 32, 67, 119, 112, 6, 32, 87, 73, 84, 72, + 32, 214, 4, 71, 72, 2, 78, 71, 68, 2, 83, 72, 164, 1, 2, 90, 72, 247, + 236, 16, 84, 76, 182, 1, 67, 158, 2, 68, 110, 78, 214, 15, 84, 158, 23, + 77, 250, 1, 86, 194, 3, 70, 186, 41, 72, 158, 1, 79, 198, 10, 71, 186, 2, + 65, 202, 168, 3, 73, 62, 66, 186, 64, 82, 167, 146, 16, 83, 24, 92, 6, + 69, 68, 73, 76, 76, 65, 32, 9, 73, 82, 67, 85, 77, 70, 76, 69, 88, 191, + 236, 19, 65, 5, 137, 143, 4, 3, 32, 65, 78, 19, 11, 32, 16, 40, 4, 65, + 78, 68, 32, 151, 243, 18, 66, 14, 174, 85, 67, 130, 2, 72, 226, 11, 71, + 186, 2, 65, 186, 194, 3, 77, 250, 169, 4, 68, 211, 219, 8, 84, 12, 22, + 79, 239, 97, 73, 10, 28, 2, 84, 32, 239, 50, 85, 8, 204, 87, 5, 65, 66, + 79, 86, 69, 171, 154, 18, 66, 2, 167, 142, 14, 79, 4, 221, 255, 6, 13, + 89, 80, 84, 79, 76, 79, 71, 73, 67, 65, 76, 32, 65, 7, 33, 6, 32, 87, 73, + 84, 72, 32, 4, 246, 188, 4, 67, 231, 9, 80, 13, 33, 6, 32, 87, 73, 84, + 72, 32, 10, 88, 10, 68, 79, 85, 66, 76, 69, 32, 66, 65, 82, 190, 197, 4, + 80, 142, 1, 67, 207, 4, 82, 5, 177, 198, 4, 4, 32, 65, 78, 68, 15, 11, + 32, 12, 38, 82, 37, 5, 87, 73, 84, 72, 32, 2, 193, 138, 14, 4, 69, 86, + 69, 82, 10, 54, 67, 138, 196, 4, 80, 218, 5, 82, 195, 142, 14, 84, 4, + 198, 214, 7, 85, 155, 144, 12, 65, 17, 84, 6, 32, 87, 73, 84, 72, 32, 73, + 11, 69, 78, 71, 32, 68, 73, 71, 82, 65, 80, 72, 10, 222, 187, 4, 77, 174, + 7, 80, 166, 244, 14, 72, 138, 83, 68, 211, 80, 83, 5, 181, 146, 18, 8, + 32, 87, 73, 84, 72, 32, 84, 82, 37, 72, 6, 32, 87, 73, 84, 72, 32, 126, + 76, 202, 182, 18, 65, 195, 207, 2, 72, 22, 218, 3, 67, 134, 90, 65, 134, + 169, 3, 66, 182, 25, 77, 174, 33, 80, 154, 235, 3, 79, 142, 137, 11, 72, + 138, 83, 68, 211, 80, 83, 8, 33, 6, 79, 84, 84, 65, 76, 32, 8, 242, 182, + 18, 83, 170, 209, 2, 65, 2, 73, 3, 85, 39, 140, 1, 6, 32, 87, 73, 84, 72, + 32, 162, 44, 65, 232, 25, 12, 79, 79, 75, 69, 68, 32, 83, 67, 72, 87, 65, + 32, 250, 237, 3, 69, 219, 210, 16, 86, 24, 86, 66, 34, 67, 46, 68, 226, + 90, 76, 222, 226, 3, 80, 166, 244, 14, 72, 219, 163, 1, 83, 2, 233, 135, + 13, 3, 82, 69, 86, 6, 170, 58, 69, 154, 32, 73, 255, 133, 19, 65, 8, 246, + 86, 73, 174, 185, 3, 69, 171, 218, 12, 79, 75, 80, 6, 32, 87, 73, 84, 72, + 32, 222, 4, 78, 188, 1, 2, 79, 84, 155, 254, 20, 83, 48, 178, 1, 67, 34, + 68, 210, 1, 77, 64, 6, 79, 71, 79, 78, 69, 75, 78, 84, 142, 70, 72, 226, + 11, 71, 186, 2, 65, 202, 168, 3, 73, 62, 66, 136, 64, 6, 83, 84, 82, 79, + 75, 69, 51, 82, 4, 222, 87, 73, 255, 133, 19, 65, 14, 18, 73, 47, 79, 4, + 217, 26, 7, 65, 69, 82, 69, 83, 73, 83, 10, 28, 2, 84, 32, 227, 36, 85, + 8, 64, 10, 65, 66, 79, 86, 69, 32, 65, 78, 68, 32, 171, 227, 18, 66, 6, + 162, 83, 71, 186, 2, 65, 131, 200, 16, 84, 4, 29, 5, 65, 67, 82, 79, 78, + 5, 229, 35, 4, 32, 65, 78, 68, 7, 157, 72, 15, 32, 65, 78, 68, 32, 68, + 79, 84, 32, 65, 66, 79, 86, 69, 32, 4, 21, 3, 73, 76, 68, 4, 211, 135, 5, + 69, 16, 46, 83, 93, 7, 86, 69, 82, 84, 69, 68, 32, 12, 29, 5, 85, 76, 65, + 82, 32, 12, 130, 255, 20, 68, 2, 70, 2, 71, 2, 82, 2, 83, 3, 84, 4, 26, + 65, 219, 231, 20, 79, 2, 247, 165, 17, 76, 6, 166, 157, 4, 65, 129, 186, + 6, 5, 73, 70, 73, 69, 68, 13, 33, 6, 32, 87, 73, 84, 72, 32, 10, 94, 67, + 236, 173, 4, 13, 68, 79, 84, 32, 65, 66, 79, 86, 69, 32, 65, 78, 68, 247, + 158, 16, 83, 6, 190, 81, 73, 206, 217, 3, 82, 179, 172, 15, 65, 29, 48, + 6, 32, 87, 73, 84, 72, 32, 195, 249, 20, 82, 24, 98, 67, 34, 68, 50, 83, + 234, 78, 65, 138, 1, 76, 222, 226, 3, 80, 154, 235, 3, 79, 143, 137, 11, + 72, 4, 222, 47, 69, 151, 166, 19, 65, 6, 226, 69, 73, 130, 192, 3, 69, + 243, 181, 4, 79, 4, 29, 5, 84, 82, 79, 75, 69, 5, 173, 24, 6, 32, 65, 78, + 68, 32, 68, 79, 136, 1, 6, 32, 87, 73, 84, 72, 32, 250, 3, 65, 38, 69, + 48, 5, 79, 78, 71, 32, 83, 218, 173, 4, 83, 2, 90, 246, 181, 16, 85, 219, + 16, 74, 50, 178, 1, 66, 86, 67, 56, 2, 68, 79, 100, 3, 77, 73, 68, 130, + 2, 72, 242, 71, 65, 138, 1, 76, 226, 218, 3, 73, 162, 1, 82, 222, 2, 70, + 130, 4, 80, 158, 228, 12, 84, 227, 179, 3, 83, 6, 36, 3, 69, 76, 84, 187, + 190, 20, 65, 5, 153, 175, 4, 6, 32, 65, 78, 68, 32, 80, 8, 178, 43, 69, + 22, 73, 234, 149, 7, 85, 155, 144, 12, 65, 8, 38, 84, 25, 5, 85, 66, 76, + 69, 32, 4, 153, 24, 2, 32, 66, 4, 202, 166, 4, 77, 239, 173, 15, 66, 8, + 36, 4, 68, 76, 69, 32, 219, 43, 45, 6, 198, 145, 17, 84, 218, 146, 3, 82, + 79, 68, 4, 133, 148, 4, 4, 77, 66, 68, 65, 6, 246, 174, 4, 90, 205, 250, + 12, 3, 78, 73, 83, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 18, 68, 35, 72, + 4, 218, 62, 73, 211, 180, 19, 79, 2, 157, 166, 4, 2, 73, 71, 27, 56, 6, + 32, 87, 73, 84, 72, 32, 102, 73, 187, 225, 20, 85, 16, 150, 71, 65, 130, + 218, 3, 67, 186, 2, 77, 174, 7, 80, 134, 173, 12, 68, 154, 55, 84, 139, + 144, 2, 72, 6, 25, 4, 68, 68, 76, 69, 6, 76, 7, 45, 87, 69, 76, 83, 72, + 32, 149, 252, 16, 6, 32, 83, 67, 79, 84, 83, 4, 238, 231, 19, 76, 183, + 137, 1, 86, 49, 58, 32, 216, 2, 2, 71, 32, 150, 221, 20, 85, 219, 16, 74, + 40, 88, 5, 87, 73, 84, 72, 32, 153, 136, 6, 11, 80, 82, 69, 67, 69, 68, + 69, 68, 32, 66, 89, 38, 122, 67, 74, 76, 170, 36, 77, 234, 27, 71, 186, + 2, 65, 102, 68, 130, 227, 3, 80, 218, 5, 82, 194, 229, 3, 79, 135, 249, + 8, 84, 10, 182, 35, 69, 22, 73, 210, 249, 3, 82, 154, 156, 3, 85, 155, + 144, 12, 65, 6, 144, 65, 3, 79, 78, 71, 202, 2, 73, 131, 234, 3, 69, 2, + 25, 4, 87, 73, 84, 72, 2, 153, 202, 17, 5, 32, 84, 73, 76, 68, 115, 116, + 6, 32, 87, 73, 84, 72, 32, 186, 6, 76, 52, 4, 80, 69, 78, 32, 214, 174, + 13, 77, 242, 182, 7, 73, 2, 79, 3, 85, 86, 154, 1, 67, 70, 68, 152, 1, 2, + 76, 79, 86, 77, 46, 79, 38, 83, 66, 84, 106, 86, 234, 43, 72, 242, 12, + 71, 186, 2, 65, 202, 168, 3, 73, 62, 66, 187, 64, 82, 14, 184, 48, 9, 73, + 82, 67, 85, 77, 70, 76, 69, 88, 187, 149, 19, 65, 14, 18, 73, 47, 79, 4, + 233, 12, 7, 65, 69, 82, 69, 83, 73, 83, 10, 22, 84, 183, 46, 85, 6, 11, + 32, 6, 152, 12, 5, 65, 66, 79, 86, 69, 195, 191, 18, 66, 6, 66, 78, 132, + 197, 12, 6, 87, 32, 82, 73, 78, 71, 231, 163, 8, 79, 2, 171, 20, 71, 6, + 11, 65, 6, 189, 2, 4, 67, 82, 79, 78, 4, 229, 10, 5, 71, 79, 78, 69, 75, + 4, 25, 4, 84, 82, 79, 75, 4, 11, 69, 5, 225, 48, 2, 32, 65, 8, 25, 4, 73, + 76, 68, 69, 9, 29, 5, 32, 65, 78, 68, 32, 6, 174, 46, 68, 142, 13, 65, + 187, 194, 3, 77, 6, 81, 18, 69, 82, 84, 73, 67, 65, 76, 32, 76, 73, 78, + 69, 32, 66, 69, 76, 79, 87, 7, 233, 42, 4, 32, 65, 78, 68, 2, 137, 129, + 10, 8, 68, 32, 80, 79, 76, 73, 83, 72, 16, 26, 79, 219, 159, 4, 69, 13, + 48, 6, 32, 87, 73, 84, 72, 32, 247, 228, 20, 69, 8, 222, 54, 71, 186, 2, + 65, 190, 233, 3, 82, 167, 146, 16, 83, 23, 48, 6, 32, 87, 73, 84, 72, 32, + 175, 208, 20, 72, 18, 86, 70, 34, 83, 178, 55, 65, 186, 220, 3, 77, 174, + 7, 80, 166, 244, 14, 72, 139, 83, 68, 2, 137, 185, 14, 3, 76, 79, 85, 6, + 218, 27, 84, 241, 164, 12, 6, 81, 85, 73, 82, 82, 69, 13, 48, 6, 32, 87, + 73, 84, 72, 32, 215, 155, 4, 80, 8, 42, 68, 16, 4, 72, 79, 79, 75, 23, + 83, 2, 227, 44, 73, 5, 247, 173, 18, 32, 2, 197, 26, 6, 84, 82, 79, 75, + 69, 32, 77, 90, 32, 228, 4, 8, 69, 86, 69, 82, 83, 69, 68, 32, 148, 2, 2, + 85, 77, 255, 141, 4, 65, 46, 36, 4, 87, 73, 84, 72, 235, 6, 82, 44, 26, + 32, 215, 174, 15, 79, 42, 166, 1, 67, 50, 68, 200, 1, 8, 70, 73, 83, 72, + 72, 79, 79, 75, 66, 76, 34, 84, 142, 18, 77, 162, 30, 65, 202, 168, 3, + 73, 158, 59, 80, 154, 235, 3, 79, 231, 172, 12, 83, 6, 170, 19, 69, 230, + 249, 3, 82, 179, 172, 15, 65, 8, 11, 79, 8, 24, 2, 84, 32, 111, 85, 6, + 26, 66, 131, 225, 19, 65, 4, 25, 4, 69, 76, 79, 87, 5, 29, 5, 32, 65, 78, + 68, 32, 2, 139, 244, 3, 77, 2, 21, 3, 66, 76, 69, 2, 11, 32, 2, 227, 46, + 71, 7, 29, 5, 32, 65, 78, 68, 32, 4, 162, 141, 4, 77, 175, 7, 80, 4, 222, + 49, 73, 207, 230, 3, 79, 4, 178, 168, 18, 65, 163, 26, 73, 22, 162, 1, + 72, 40, 6, 79, 80, 69, 78, 32, 69, 164, 142, 4, 8, 82, 32, 87, 73, 84, + 72, 32, 70, 168, 2, 3, 83, 67, 82, 226, 206, 14, 69, 234, 86, 67, 139, + 164, 1, 75, 2, 11, 65, 2, 225, 244, 9, 2, 76, 70, 7, 33, 6, 32, 87, 73, + 84, 72, 32, 4, 214, 151, 4, 82, 207, 238, 14, 72, 5, 11, 32, 2, 11, 82, + 2, 149, 253, 16, 3, 79, 84, 85, 85, 180, 1, 6, 32, 87, 73, 84, 72, 32, + 218, 4, 65, 66, 67, 218, 1, 72, 34, 73, 156, 2, 13, 81, 85, 65, 84, 32, + 82, 69, 86, 69, 82, 83, 69, 68, 193, 229, 9, 6, 84, 73, 82, 82, 85, 80, + 40, 110, 65, 34, 67, 86, 68, 138, 1, 83, 174, 1, 86, 210, 9, 77, 134, + 130, 4, 80, 154, 235, 3, 79, 143, 137, 11, 72, 4, 205, 1, 4, 67, 85, 84, + 69, 12, 58, 65, 230, 10, 69, 82, 79, 202, 31, 73, 231, 245, 6, 85, 4, + 113, 3, 82, 79, 78, 8, 32, 3, 79, 84, 32, 207, 32, 73, 6, 26, 66, 215, + 216, 19, 65, 4, 25, 4, 69, 76, 79, 87, 5, 11, 32, 2, 181, 212, 19, 3, 65, 78, 68, 4, 22, 72, 203, 42, 87, 2, 21, 3, 79, 82, 84, 2, 17, 2, 32, 83, - 2, 11, 84, 2, 21, 3, 82, 79, 75, 2, 11, 69, 2, 17, 2, 32, 79, 2, 237, - 235, 18, 4, 86, 69, 82, 76, 2, 37, 7, 69, 82, 84, 73, 67, 65, 76, 2, 205, - 40, 2, 32, 76, 4, 46, 76, 249, 226, 18, 5, 75, 72, 65, 32, 89, 2, 247, + 2, 11, 84, 2, 21, 3, 82, 79, 75, 2, 11, 69, 2, 17, 2, 32, 79, 2, 233, + 199, 19, 4, 86, 69, 82, 76, 2, 37, 7, 69, 82, 84, 73, 67, 65, 76, 2, 205, + 40, 2, 32, 76, 4, 46, 76, 245, 190, 19, 5, 75, 72, 65, 32, 89, 2, 247, 12, 84, 18, 48, 3, 72, 87, 65, 97, 5, 82, 73, 80, 84, 32, 11, 33, 6, 32, - 87, 73, 84, 72, 32, 8, 222, 35, 71, 186, 2, 65, 194, 221, 3, 82, 171, - 161, 14, 72, 8, 26, 82, 147, 243, 3, 71, 5, 173, 141, 11, 5, 32, 87, 73, - 84, 72, 2, 233, 161, 16, 3, 65, 82, 80, 14, 48, 7, 68, 69, 87, 65, 89, - 83, 32, 199, 1, 71, 12, 110, 79, 56, 4, 84, 85, 82, 78, 224, 160, 10, 11, - 68, 73, 65, 69, 82, 69, 83, 73, 90, 69, 68, 135, 210, 9, 85, 7, 26, 80, - 203, 245, 3, 32, 2, 145, 203, 9, 2, 69, 78, 2, 173, 220, 12, 2, 69, 68, - 2, 209, 159, 16, 4, 77, 79, 73, 68, 2, 227, 219, 9, 32, 147, 1, 188, 1, + 87, 73, 84, 72, 32, 8, 222, 35, 71, 186, 2, 65, 190, 233, 3, 82, 207, + 238, 14, 72, 8, 26, 82, 155, 255, 3, 71, 5, 253, 175, 11, 5, 32, 87, 73, + 84, 72, 2, 225, 219, 16, 3, 65, 82, 80, 14, 48, 7, 68, 69, 87, 65, 89, + 83, 32, 199, 1, 71, 12, 110, 79, 56, 4, 84, 85, 82, 78, 176, 195, 10, 11, + 68, 73, 65, 69, 82, 69, 83, 73, 90, 69, 68, 203, 139, 10, 85, 7, 26, 80, + 211, 129, 4, 32, 2, 165, 234, 9, 2, 69, 78, 2, 237, 255, 12, 2, 69, 68, + 2, 201, 217, 16, 4, 77, 79, 73, 68, 2, 247, 250, 9, 32, 147, 1, 188, 1, 6, 32, 87, 73, 84, 72, 32, 192, 3, 2, 69, 83, 70, 72, 130, 2, 79, 102, - 82, 54, 85, 242, 241, 3, 67, 202, 1, 83, 136, 210, 10, 9, 65, 73, 76, 76, - 69, 83, 83, 32, 80, 243, 163, 5, 90, 34, 110, 67, 182, 1, 68, 66, 77, - 170, 31, 76, 214, 214, 3, 80, 180, 5, 4, 72, 79, 79, 75, 50, 82, 143, - 194, 15, 83, 10, 58, 69, 22, 73, 62, 79, 182, 129, 7, 85, 147, 201, 11, - 65, 2, 195, 251, 11, 68, 2, 37, 7, 82, 67, 85, 77, 70, 76, 69, 2, 255, - 199, 17, 88, 2, 17, 2, 77, 77, 2, 215, 199, 17, 65, 8, 32, 2, 73, 65, - 215, 255, 15, 79, 4, 154, 21, 71, 215, 6, 69, 4, 17, 2, 73, 68, 4, 26, - 45, 195, 238, 3, 68, 2, 229, 252, 3, 6, 72, 69, 73, 71, 72, 84, 6, 45, 9, - 72, 32, 68, 73, 71, 82, 65, 80, 72, 7, 183, 244, 3, 32, 8, 64, 12, 32, - 87, 73, 84, 72, 32, 83, 84, 82, 73, 75, 69, 43, 79, 2, 245, 243, 15, 5, + 82, 54, 85, 250, 253, 3, 67, 202, 1, 83, 132, 247, 10, 9, 65, 73, 76, 76, + 69, 83, 83, 32, 80, 131, 207, 5, 90, 34, 110, 67, 182, 1, 68, 66, 77, + 170, 31, 76, 222, 226, 3, 80, 168, 5, 4, 72, 79, 79, 75, 50, 82, 167, + 146, 16, 83, 10, 58, 69, 22, 73, 62, 79, 174, 149, 7, 85, 155, 144, 12, + 65, 2, 251, 157, 12, 68, 2, 37, 7, 82, 67, 85, 77, 70, 76, 69, 2, 179, + 157, 18, 88, 2, 17, 2, 77, 77, 2, 139, 157, 18, 65, 8, 32, 2, 73, 65, + 179, 175, 16, 79, 4, 154, 21, 71, 215, 6, 69, 4, 17, 2, 73, 68, 4, 26, + 45, 203, 250, 3, 68, 2, 225, 136, 4, 6, 72, 69, 73, 71, 72, 84, 6, 45, 9, + 72, 32, 68, 73, 71, 82, 65, 80, 72, 7, 191, 128, 4, 32, 8, 64, 12, 32, + 87, 73, 84, 72, 32, 83, 84, 82, 73, 75, 69, 43, 79, 2, 205, 163, 16, 5, 84, 72, 82, 79, 85, 6, 17, 2, 82, 78, 7, 41, 8, 32, 87, 73, 84, 72, 32, - 83, 84, 4, 25, 4, 82, 79, 75, 69, 5, 11, 32, 2, 213, 197, 3, 6, 84, 72, - 82, 79, 85, 71, 8, 26, 78, 143, 244, 3, 80, 6, 17, 2, 69, 32, 6, 234, - 188, 4, 83, 190, 160, 9, 84, 215, 132, 1, 70, 2, 17, 2, 69, 83, 2, 11, - 73, 2, 243, 198, 19, 76, 76, 44, 5, 82, 78, 69, 68, 32, 131, 233, 19, 77, - 74, 162, 1, 68, 22, 72, 66, 73, 54, 79, 142, 1, 82, 110, 84, 22, 86, 138, - 240, 3, 65, 50, 77, 198, 2, 89, 246, 221, 15, 85, 218, 19, 69, 2, 71, 2, - 75, 2, 76, 3, 87, 2, 191, 195, 16, 69, 7, 209, 224, 3, 11, 32, 87, 73, - 84, 72, 32, 70, 73, 83, 72, 72, 5, 11, 78, 2, 149, 181, 9, 5, 83, 85, 76, - 65, 82, 12, 66, 69, 176, 213, 3, 7, 32, 79, 80, 69, 78, 45, 79, 167, 29, - 80, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 166, 200, 17, 72, 203, 237, 1, - 83, 15, 33, 6, 32, 87, 73, 84, 72, 32, 12, 138, 230, 3, 77, 174, 7, 80, - 142, 5, 76, 226, 197, 13, 84, 163, 92, 72, 5, 203, 237, 3, 32, 7, 11, 32, - 4, 161, 5, 4, 87, 73, 84, 72, 97, 90, 32, 224, 210, 3, 6, 80, 83, 73, 76, - 79, 78, 226, 144, 16, 69, 2, 73, 2, 77, 3, 79, 82, 48, 3, 66, 65, 82, 49, + 83, 84, 4, 25, 4, 82, 79, 75, 69, 5, 11, 32, 2, 209, 209, 3, 6, 84, 72, + 82, 79, 85, 71, 8, 26, 78, 151, 128, 4, 80, 6, 17, 2, 69, 32, 6, 134, + 201, 4, 83, 230, 197, 9, 84, 255, 131, 1, 70, 2, 17, 2, 69, 83, 2, 11, + 73, 2, 135, 163, 20, 76, 76, 44, 5, 82, 78, 69, 68, 32, 151, 197, 20, 77, + 74, 162, 1, 68, 22, 72, 66, 73, 54, 79, 142, 1, 82, 110, 84, 22, 86, 146, + 252, 3, 65, 38, 77, 198, 2, 89, 142, 174, 16, 85, 218, 19, 69, 2, 71, 2, + 75, 2, 76, 3, 87, 2, 223, 138, 17, 69, 7, 217, 236, 3, 11, 32, 87, 73, + 84, 72, 32, 70, 73, 83, 72, 72, 5, 11, 78, 2, 169, 212, 9, 5, 83, 85, 76, + 65, 82, 12, 66, 69, 180, 225, 3, 7, 32, 79, 80, 69, 78, 45, 79, 159, 29, + 80, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 202, 157, 18, 72, 187, 244, 1, + 83, 15, 33, 6, 32, 87, 73, 84, 72, 32, 12, 146, 242, 3, 77, 174, 7, 80, + 130, 5, 76, 154, 143, 14, 84, 143, 96, 72, 5, 211, 249, 3, 32, 7, 11, 32, + 4, 161, 5, 4, 87, 73, 84, 72, 97, 90, 32, 228, 222, 3, 6, 80, 83, 73, 76, + 79, 78, 242, 224, 16, 69, 2, 73, 2, 77, 3, 79, 82, 48, 3, 66, 65, 82, 49, 5, 87, 73, 84, 72, 32, 5, 245, 17, 8, 32, 87, 73, 84, 72, 32, 83, 72, 78, 142, 1, 67, 74, 68, 150, 2, 72, 170, 1, 77, 134, 1, 79, 106, 82, 38, 84, - 186, 9, 71, 82, 83, 234, 1, 65, 206, 156, 3, 73, 62, 66, 135, 66, 76, 6, - 172, 185, 4, 9, 73, 82, 67, 85, 77, 70, 76, 69, 88, 147, 132, 14, 65, 18, + 186, 9, 71, 82, 83, 234, 1, 65, 202, 168, 3, 73, 62, 66, 135, 66, 76, 6, + 200, 197, 4, 9, 73, 82, 67, 85, 77, 70, 76, 69, 88, 247, 210, 14, 65, 18, 52, 8, 73, 65, 69, 82, 69, 83, 73, 83, 131, 1, 79, 13, 11, 32, 10, 40, 4, - 65, 78, 68, 32, 223, 199, 17, 66, 8, 50, 67, 226, 13, 71, 186, 2, 65, - 183, 182, 3, 77, 2, 211, 187, 18, 65, 6, 26, 85, 207, 184, 17, 84, 4, 21, + 65, 78, 68, 32, 155, 158, 18, 66, 8, 50, 67, 226, 13, 71, 186, 2, 65, + 187, 194, 3, 77, 2, 211, 150, 19, 65, 6, 26, 85, 131, 142, 18, 84, 4, 21, 3, 66, 76, 69, 4, 11, 32, 4, 138, 13, 71, 187, 2, 65, 14, 11, 79, 14, 28, - 2, 82, 78, 251, 75, 79, 13, 29, 5, 32, 65, 78, 68, 32, 10, 66, 72, 226, - 11, 71, 186, 2, 65, 242, 216, 7, 68, 179, 181, 8, 84, 2, 145, 75, 2, 79, + 2, 82, 78, 191, 81, 79, 13, 29, 5, 32, 65, 78, 68, 32, 10, 66, 72, 226, + 11, 71, 186, 2, 65, 178, 236, 7, 68, 211, 219, 8, 84, 2, 213, 80, 2, 79, 79, 10, 29, 5, 65, 67, 82, 79, 78, 11, 29, 5, 32, 65, 78, 68, 32, 8, 50, - 68, 214, 10, 71, 186, 2, 65, 163, 142, 16, 84, 2, 171, 10, 73, 6, 29, 5, + 68, 214, 10, 71, 186, 2, 65, 131, 200, 16, 84, 2, 171, 10, 73, 6, 29, 5, 71, 79, 78, 69, 75, 7, 11, 32, 4, 25, 4, 65, 78, 68, 32, 4, 178, 12, 65, - 163, 142, 16, 84, 4, 222, 233, 3, 69, 131, 200, 13, 73, 6, 25, 4, 73, 76, - 68, 69, 7, 11, 32, 4, 26, 65, 191, 194, 17, 66, 2, 17, 2, 78, 68, 2, 11, + 131, 200, 16, 84, 4, 218, 245, 3, 69, 187, 145, 14, 73, 6, 25, 4, 73, 76, + 68, 69, 7, 11, 32, 4, 26, 65, 251, 152, 18, 66, 2, 17, 2, 78, 68, 2, 11, 32, 2, 139, 11, 65, 29, 84, 6, 32, 87, 73, 84, 72, 32, 162, 1, 73, 66, - 79, 174, 202, 18, 69, 227, 141, 1, 89, 14, 82, 68, 174, 225, 3, 80, 142, - 1, 67, 158, 7, 82, 158, 174, 12, 84, 203, 240, 1, 72, 4, 26, 73, 179, - 226, 7, 79, 2, 17, 2, 65, 71, 2, 189, 168, 19, 2, 79, 78, 2, 41, 8, 83, - 73, 71, 79, 84, 72, 73, 67, 2, 175, 225, 15, 32, 6, 33, 6, 76, 65, 80, - 85, 75, 32, 6, 146, 193, 19, 65, 2, 79, 3, 85, 19, 33, 6, 32, 87, 73, 84, - 72, 32, 16, 202, 4, 67, 18, 68, 70, 71, 186, 2, 65, 194, 165, 17, 82, - 171, 89, 72, 17, 33, 6, 32, 87, 73, 84, 72, 32, 14, 42, 68, 32, 2, 76, - 79, 255, 221, 3, 80, 4, 222, 3, 73, 131, 210, 18, 79, 8, 60, 11, 78, 71, + 79, 170, 166, 19, 69, 251, 141, 1, 89, 14, 82, 68, 182, 237, 3, 80, 142, + 1, 67, 146, 7, 82, 130, 220, 12, 84, 139, 144, 2, 72, 4, 26, 73, 243, + 245, 7, 79, 2, 17, 2, 65, 71, 2, 209, 132, 20, 2, 79, 78, 2, 41, 8, 83, + 73, 71, 79, 84, 72, 73, 67, 2, 135, 145, 16, 32, 6, 33, 6, 76, 65, 80, + 85, 75, 32, 6, 166, 157, 20, 65, 2, 79, 3, 85, 19, 33, 6, 32, 87, 73, 84, + 72, 32, 16, 202, 4, 67, 18, 68, 70, 71, 186, 2, 65, 246, 250, 17, 82, + 151, 93, 72, 17, 33, 6, 32, 87, 73, 84, 72, 32, 14, 42, 68, 32, 2, 76, + 79, 135, 234, 3, 80, 4, 222, 3, 73, 255, 173, 19, 79, 8, 60, 11, 78, 71, 32, 76, 69, 70, 84, 32, 76, 69, 71, 79, 87, 7, 11, 32, 4, 60, 7, 65, 78, 68, 32, 76, 79, 87, 65, 4, 87, 73, 84, 72, 2, 17, 2, 32, 82, 2, 21, 3, - 73, 71, 72, 2, 215, 236, 10, 84, 2, 197, 210, 18, 4, 32, 83, 69, 82, 33, - 48, 6, 32, 87, 73, 84, 72, 32, 187, 219, 15, 79, 28, 110, 67, 18, 68, 70, - 71, 22, 72, 42, 76, 22, 83, 234, 1, 65, 182, 182, 3, 77, 238, 215, 12, - 84, 163, 151, 1, 82, 2, 203, 3, 73, 6, 26, 73, 251, 227, 15, 79, 2, 17, - 2, 65, 69, 2, 243, 247, 15, 82, 2, 207, 233, 17, 82, 4, 17, 2, 79, 79, 4, - 187, 246, 4, 75, 2, 191, 216, 18, 79, 4, 26, 72, 167, 161, 19, 84, 2, 21, - 3, 79, 82, 84, 2, 245, 226, 7, 6, 32, 82, 73, 71, 72, 84, 31, 33, 6, 32, - 87, 73, 84, 72, 32, 28, 98, 65, 22, 67, 82, 68, 38, 76, 34, 83, 138, 207, - 3, 77, 174, 7, 80, 230, 5, 82, 171, 161, 14, 72, 2, 175, 140, 13, 67, 6, - 42, 73, 238, 225, 6, 85, 147, 201, 11, 65, 2, 177, 158, 11, 4, 82, 67, - 85, 77, 6, 234, 169, 3, 69, 203, 182, 12, 79, 2, 11, 73, 2, 243, 137, 12, - 78, 4, 26, 87, 151, 158, 19, 84, 2, 195, 190, 16, 65, 18, 90, 70, 222, - 157, 9, 73, 172, 9, 6, 76, 79, 78, 71, 32, 83, 226, 137, 10, 83, 219, 5, - 79, 10, 34, 70, 242, 204, 19, 73, 3, 76, 7, 238, 204, 19, 73, 3, 76, 36, - 150, 1, 83, 190, 203, 19, 65, 2, 69, 2, 72, 2, 73, 2, 74, 2, 75, 2, 76, - 2, 77, 2, 78, 2, 79, 2, 80, 2, 82, 2, 84, 2, 85, 2, 86, 3, 88, 5, 139, - 212, 4, 67, 160, 5, 200, 1, 2, 65, 70, 104, 2, 70, 84, 186, 33, 79, 20, - 5, 80, 67, 72, 65, 32, 164, 8, 8, 83, 83, 45, 84, 72, 65, 78, 32, 142, - 231, 15, 68, 248, 22, 6, 86, 69, 76, 32, 83, 76, 158, 209, 2, 77, 227, - 79, 71, 4, 180, 180, 1, 14, 32, 70, 76, 85, 84, 84, 69, 82, 73, 78, 71, - 32, 73, 78, 169, 186, 11, 3, 89, 32, 71, 200, 3, 58, 32, 186, 17, 45, - 169, 1, 6, 87, 65, 82, 68, 83, 32, 218, 1, 190, 1, 65, 174, 3, 66, 202, - 1, 67, 96, 2, 68, 79, 112, 2, 72, 65, 250, 2, 76, 46, 77, 38, 79, 38, 82, - 146, 3, 83, 82, 84, 174, 1, 87, 190, 159, 8, 70, 210, 3, 78, 202, 1, 80, - 187, 11, 86, 26, 22, 78, 211, 2, 82, 20, 28, 2, 68, 32, 131, 2, 71, 14, - 72, 3, 76, 79, 87, 0, 3, 85, 80, 80, 93, 6, 82, 73, 71, 72, 84, 32, 4, - 21, 3, 69, 82, 32, 4, 132, 190, 16, 9, 65, 78, 68, 32, 82, 73, 71, 72, - 84, 175, 30, 79, 6, 50, 84, 161, 166, 17, 6, 68, 79, 85, 66, 76, 69, 4, - 138, 190, 16, 82, 131, 216, 1, 65, 6, 202, 163, 8, 69, 185, 1, 4, 76, 69, - 32, 66, 6, 26, 67, 167, 167, 8, 82, 2, 129, 167, 8, 5, 32, 76, 69, 83, - 83, 12, 40, 4, 65, 82, 66, 32, 191, 167, 8, 76, 8, 40, 4, 68, 79, 87, 78, - 1, 2, 85, 80, 4, 57, 12, 32, 82, 73, 71, 72, 84, 32, 66, 65, 82, 66, 32, - 4, 208, 176, 16, 4, 68, 79, 87, 78, 1, 2, 85, 80, 14, 130, 167, 8, 85, - 230, 21, 79, 250, 222, 2, 69, 217, 153, 2, 8, 76, 79, 83, 69, 68, 32, 69, - 78, 10, 44, 5, 85, 66, 76, 69, 32, 175, 167, 8, 84, 8, 226, 168, 8, 87, - 234, 24, 65, 198, 248, 2, 81, 151, 172, 4, 80, 26, 36, 3, 76, 70, 32, - 203, 170, 8, 78, 24, 216, 1, 6, 67, 73, 82, 67, 76, 69, 190, 167, 8, 66, - 90, 70, 34, 82, 128, 147, 8, 30, 73, 78, 86, 69, 82, 83, 69, 32, 77, 69, - 68, 73, 85, 77, 32, 83, 72, 65, 68, 69, 32, 65, 78, 68, 32, 82, 73, 71, - 72, 84, 139, 19, 77, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 42, 84, 254, - 160, 17, 70, 175, 204, 1, 68, 4, 166, 187, 5, 72, 219, 180, 12, 87, 4, - 252, 148, 1, 2, 85, 71, 207, 147, 7, 79, 2, 161, 172, 18, 4, 85, 76, 84, - 73, 6, 214, 169, 8, 85, 215, 150, 8, 78, 30, 44, 5, 73, 71, 72, 84, 32, - 219, 170, 8, 65, 28, 152, 1, 5, 65, 82, 82, 79, 87, 132, 1, 12, 68, 79, - 85, 66, 76, 69, 32, 65, 82, 82, 79, 87, 30, 87, 218, 209, 8, 79, 234, - 215, 7, 66, 54, 83, 159, 53, 84, 11, 11, 32, 8, 72, 5, 87, 73, 84, 72, - 32, 193, 234, 17, 7, 84, 72, 82, 79, 85, 71, 72, 6, 146, 248, 15, 68, - 190, 145, 3, 86, 79, 83, 7, 197, 201, 8, 2, 32, 87, 4, 202, 220, 8, 65, - 255, 248, 7, 72, 34, 238, 168, 8, 45, 70, 69, 78, 73, 168, 1, 3, 80, 69, - 69, 26, 81, 227, 2, 85, 18, 58, 82, 246, 173, 8, 72, 178, 178, 7, 79, - 179, 169, 2, 65, 6, 40, 4, 73, 65, 78, 71, 143, 175, 8, 65, 4, 208, 227, - 4, 8, 76, 69, 32, 66, 69, 83, 73, 68, 235, 230, 11, 85, 16, 174, 177, 8, - 72, 202, 1, 73, 141, 246, 9, 2, 82, 73, 30, 90, 83, 230, 178, 8, 70, 206, - 1, 72, 130, 1, 80, 189, 3, 7, 84, 79, 45, 82, 73, 71, 72, 4, 44, 5, 73, - 68, 69, 32, 65, 171, 183, 8, 72, 2, 209, 138, 1, 2, 82, 67, 208, 1, 172, - 1, 5, 65, 82, 82, 79, 87, 134, 5, 66, 70, 72, 134, 4, 84, 178, 2, 87, - 230, 182, 8, 68, 210, 1, 70, 222, 7, 76, 26, 79, 34, 80, 50, 82, 70, 83, - 134, 136, 8, 67, 47, 81, 71, 26, 32, 247, 134, 17, 45, 66, 102, 65, 138, - 1, 84, 132, 2, 5, 87, 73, 84, 72, 32, 238, 184, 8, 70, 173, 215, 9, 4, - 79, 86, 69, 82, 12, 44, 5, 66, 79, 86, 69, 32, 219, 186, 8, 78, 10, 64, - 4, 83, 72, 79, 82, 210, 185, 8, 82, 202, 208, 4, 65, 55, 84, 2, 175, 146, - 18, 84, 12, 56, 7, 72, 82, 79, 85, 71, 72, 32, 53, 3, 79, 32, 66, 6, 242, - 135, 13, 83, 134, 155, 4, 76, 223, 142, 2, 88, 6, 32, 2, 65, 82, 199, - 187, 8, 76, 5, 221, 128, 12, 23, 32, 79, 86, 69, 82, 32, 82, 73, 71, 72, - 84, 87, 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 36, 154, 188, 8, 68, - 98, 76, 38, 80, 30, 83, 38, 84, 210, 139, 8, 77, 38, 78, 122, 69, 226, - 148, 1, 72, 155, 160, 1, 86, 8, 194, 189, 8, 65, 224, 10, 5, 79, 84, 84, - 79, 77, 251, 129, 8, 76, 30, 26, 65, 163, 207, 16, 69, 26, 48, 6, 82, 80, - 79, 79, 78, 32, 243, 231, 18, 78, 24, 100, 10, 87, 73, 84, 72, 32, 66, - 65, 82, 66, 32, 213, 191, 8, 9, 79, 86, 69, 82, 32, 82, 73, 71, 72, 22, - 40, 4, 68, 79, 87, 78, 117, 2, 85, 80, 10, 26, 32, 251, 212, 16, 87, 8, - 226, 192, 8, 66, 188, 2, 7, 65, 66, 79, 86, 69, 32, 82, 130, 138, 8, 70, - 175, 5, 84, 12, 26, 32, 135, 212, 16, 87, 10, 60, 6, 65, 66, 79, 86, 69, - 32, 238, 203, 16, 70, 175, 5, 84, 6, 42, 76, 205, 192, 8, 4, 82, 73, 71, - 72, 4, 242, 190, 8, 69, 207, 201, 4, 79, 54, 44, 2, 82, 73, 194, 196, 8, - 79, 159, 5, 87, 34, 44, 5, 65, 78, 71, 76, 69, 247, 200, 8, 80, 30, 56, - 8, 45, 72, 69, 65, 68, 69, 68, 32, 243, 218, 16, 32, 28, 52, 5, 65, 82, - 82, 79, 87, 210, 210, 16, 68, 39, 80, 25, 11, 32, 22, 184, 197, 8, 9, 79, - 86, 69, 82, 32, 82, 73, 71, 72, 22, 87, 155, 137, 8, 84, 6, 26, 72, 131, - 203, 8, 65, 4, 45, 9, 73, 84, 69, 32, 65, 82, 82, 79, 87, 5, 165, 216, - 16, 2, 32, 87, 5, 215, 144, 18, 80, 148, 1, 252, 1, 15, 67, 79, 78, 83, - 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 124, 7, 76, 69, 84, 84, 69, - 82, 32, 236, 1, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 198, - 1, 83, 172, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 175, 177, - 17, 68, 18, 66, 75, 22, 78, 202, 164, 19, 76, 2, 77, 2, 80, 2, 82, 3, 84, - 5, 239, 211, 18, 65, 5, 145, 170, 12, 4, 89, 73, 78, 45, 78, 194, 1, 75, - 2, 80, 198, 134, 7, 68, 250, 149, 10, 66, 2, 70, 2, 71, 2, 72, 2, 77, - 254, 14, 78, 202, 175, 1, 84, 46, 67, 2, 83, 254, 68, 74, 2, 76, 2, 82, - 2, 86, 2, 87, 2, 89, 187, 2, 65, 6, 166, 160, 19, 72, 2, 76, 187, 2, 65, - 10, 82, 84, 40, 14, 78, 89, 69, 84, 32, 84, 72, 89, 79, 79, 77, 32, 84, - 65, 43, 67, 6, 38, 65, 21, 5, 83, 72, 79, 79, 75, 2, 255, 233, 6, 45, 5, - 17, 2, 32, 67, 2, 149, 220, 14, 3, 69, 82, 45, 8, 92, 4, 73, 71, 78, 32, - 37, 15, 85, 66, 74, 79, 73, 78, 69, 68, 32, 76, 69, 84, 84, 69, 82, 4, - 202, 186, 15, 78, 151, 158, 2, 82, 4, 11, 32, 4, 170, 157, 19, 82, 3, 89, - 14, 170, 208, 15, 85, 158, 144, 1, 79, 202, 188, 2, 65, 186, 2, 69, 3, - 73, 52, 138, 1, 65, 128, 4, 11, 69, 81, 85, 65, 76, 32, 84, 79, 32, 79, - 82, 200, 1, 2, 66, 85, 42, 67, 186, 1, 79, 210, 2, 87, 175, 141, 18, 83, - 16, 40, 5, 66, 79, 86, 69, 32, 171, 4, 78, 12, 152, 1, 7, 71, 82, 69, 65, - 84, 69, 82, 110, 83, 176, 1, 19, 68, 79, 85, 66, 76, 69, 45, 76, 73, 78, - 69, 32, 69, 81, 85, 65, 76, 32, 65, 207, 170, 15, 76, 2, 181, 5, 23, 45, - 84, 72, 65, 78, 32, 65, 66, 79, 86, 69, 32, 68, 79, 85, 66, 76, 69, 45, - 76, 73, 78, 69, 6, 152, 1, 7, 73, 77, 73, 76, 65, 82, 32, 93, 26, 76, 65, - 78, 84, 69, 68, 32, 69, 81, 85, 65, 76, 32, 65, 66, 79, 86, 69, 32, 71, - 82, 69, 65, 84, 69, 82, 4, 18, 65, 59, 79, 2, 25, 4, 66, 79, 86, 69, 2, - 241, 138, 17, 2, 32, 71, 2, 227, 2, 82, 2, 145, 2, 6, 45, 84, 72, 65, 78, - 32, 4, 17, 2, 68, 32, 4, 220, 3, 5, 78, 79, 84, 32, 65, 129, 238, 12, 11, - 83, 73, 78, 71, 76, 69, 45, 76, 73, 78, 69, 4, 253, 215, 12, 5, 84, 32, - 78, 79, 84, 4, 65, 14, 76, 79, 83, 69, 68, 32, 66, 89, 32, 67, 85, 82, - 86, 69, 5, 11, 32, 2, 61, 13, 65, 66, 79, 86, 69, 32, 83, 76, 65, 78, 84, - 69, 68, 2, 17, 2, 32, 69, 2, 183, 186, 14, 81, 18, 40, 2, 82, 32, 193, - 190, 11, 2, 86, 69, 16, 114, 65, 48, 16, 83, 76, 65, 78, 84, 69, 68, 32, - 69, 81, 85, 65, 76, 32, 84, 79, 230, 212, 12, 69, 155, 177, 4, 71, 2, - 201, 154, 9, 7, 80, 80, 82, 79, 88, 73, 77, 9, 49, 10, 32, 87, 73, 84, - 72, 32, 68, 79, 84, 32, 6, 26, 65, 175, 170, 11, 73, 4, 25, 4, 66, 79, - 86, 69, 5, 139, 198, 17, 32, 6, 25, 4, 73, 84, 72, 32, 6, 66, 67, 40, 8, - 81, 85, 69, 83, 84, 73, 79, 78, 203, 194, 18, 68, 2, 133, 169, 11, 5, 73, - 82, 67, 76, 69, 2, 17, 2, 32, 77, 2, 17, 2, 65, 82, 2, 215, 149, 18, 75, - 136, 11, 190, 1, 71, 186, 5, 77, 162, 7, 78, 168, 65, 2, 80, 83, 20, 3, - 83, 85, 32, 130, 135, 15, 82, 152, 34, 11, 86, 82, 69, 32, 84, 79, 85, - 82, 78, 79, 73, 234, 44, 79, 130, 214, 1, 90, 175, 82, 66, 44, 26, 65, - 57, 2, 72, 84, 2, 165, 142, 9, 9, 84, 85, 82, 69, 32, 79, 80, 69, 78, 42, - 38, 32, 137, 4, 4, 78, 73, 78, 71, 36, 146, 1, 69, 38, 70, 126, 82, 40, - 3, 76, 69, 70, 86, 83, 42, 84, 152, 184, 7, 3, 66, 76, 85, 134, 224, 8, - 87, 134, 36, 86, 214, 74, 71, 195, 68, 67, 2, 201, 253, 16, 4, 73, 71, - 72, 84, 8, 94, 73, 201, 201, 17, 17, 79, 85, 82, 32, 80, 79, 73, 78, 84, - 69, 68, 32, 66, 76, 65, 67, 75, 4, 221, 249, 16, 2, 86, 69, 4, 36, 3, 73, - 71, 72, 171, 224, 16, 65, 2, 241, 230, 1, 16, 84, 32, 84, 79, 82, 84, 79, - 73, 83, 69, 32, 83, 72, 69, 76, 76, 6, 138, 157, 16, 72, 242, 93, 65, 43, - 73, 4, 216, 216, 15, 2, 87, 69, 21, 3, 72, 82, 69, 7, 29, 5, 32, 77, 79, - 79, 68, 5, 163, 130, 8, 32, 138, 1, 80, 3, 66, 85, 32, 229, 136, 8, 11, - 73, 84, 69, 68, 32, 76, 73, 65, 66, 73, 76, 136, 1, 128, 1, 7, 76, 69, - 84, 84, 69, 82, 32, 218, 1, 83, 200, 2, 5, 86, 79, 87, 69, 76, 206, 173, - 12, 69, 190, 141, 4, 81, 155, 92, 68, 60, 166, 1, 71, 174, 254, 5, 84, - 174, 192, 8, 89, 214, 122, 78, 134, 135, 3, 83, 82, 66, 2, 67, 2, 68, 2, - 74, 2, 75, 2, 80, 254, 68, 72, 2, 76, 2, 77, 2, 82, 3, 87, 6, 254, 192, - 17, 89, 254, 196, 1, 72, 187, 2, 65, 32, 108, 4, 73, 71, 78, 32, 128, 1, - 12, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 193, 236, 6, 2, 85, - 66, 8, 72, 3, 75, 69, 77, 0, 3, 77, 85, 75, 32, 2, 83, 65, 227, 197, 16, - 76, 2, 149, 202, 16, 3, 80, 72, 82, 2, 207, 242, 18, 45, 18, 138, 183, - 15, 78, 158, 138, 3, 65, 182, 66, 75, 2, 76, 2, 77, 2, 80, 2, 82, 3, 84, - 20, 84, 6, 32, 83, 73, 71, 78, 32, 253, 82, 10, 45, 67, 65, 82, 82, 73, - 69, 82, 32, 76, 18, 142, 199, 5, 65, 214, 254, 10, 79, 238, 254, 1, 69, - 150, 64, 73, 3, 85, 226, 8, 22, 69, 199, 64, 75, 222, 8, 34, 32, 177, 3, - 3, 65, 82, 32, 14, 120, 12, 73, 78, 84, 69, 71, 82, 65, 84, 73, 79, 78, - 32, 190, 240, 12, 70, 178, 249, 2, 83, 193, 130, 1, 4, 84, 65, 66, 85, 6, - 108, 5, 87, 73, 84, 72, 32, 157, 1, 17, 78, 79, 84, 32, 73, 78, 67, 76, - 85, 68, 73, 78, 71, 32, 84, 72, 69, 4, 76, 7, 82, 69, 67, 84, 65, 78, 71, - 1, 8, 83, 69, 77, 73, 67, 73, 82, 67, 2, 73, 16, 85, 76, 65, 82, 32, 80, - 65, 84, 72, 32, 65, 82, 79, 85, 78, 68, 2, 17, 2, 32, 80, 2, 151, 223, - 17, 79, 208, 8, 60, 8, 65, 32, 83, 73, 71, 78, 32, 65, 221, 24, 2, 66, - 32, 170, 5, 122, 49, 86, 51, 202, 2, 52, 214, 1, 53, 158, 4, 54, 142, 3, - 55, 148, 4, 2, 56, 48, 78, 66, 201, 179, 17, 3, 48, 50, 56, 6, 140, 155, - 12, 5, 48, 48, 45, 49, 48, 176, 169, 5, 2, 50, 48, 197, 19, 2, 51, 49, - 150, 1, 78, 48, 90, 49, 130, 1, 55, 210, 241, 13, 50, 2, 51, 2, 52, 2, - 53, 3, 54, 22, 178, 1, 57, 138, 252, 18, 49, 2, 50, 2, 51, 2, 52, 2, 53, - 2, 54, 2, 55, 3, 56, 24, 90, 51, 138, 252, 18, 48, 2, 49, 2, 50, 2, 52, - 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 6, 134, 252, 18, 65, 2, 66, 3, 67, 4, - 226, 251, 18, 48, 3, 49, 38, 18, 48, 91, 49, 20, 162, 1, 48, 2, 49, 2, - 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 74, 48, 2, 49, - 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 2, 161, 164, 6, 2, 45, - 86, 160, 1, 102, 48, 78, 49, 62, 51, 86, 52, 70, 53, 86, 54, 206, 11, 50, - 250, 159, 2, 57, 190, 192, 11, 55, 3, 56, 16, 134, 249, 18, 49, 2, 50, 2, - 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 12, 186, 248, 18, 48, 2, 49, 2, - 50, 2, 51, 2, 53, 3, 54, 18, 254, 247, 18, 48, 2, 49, 2, 50, 2, 52, 2, - 53, 2, 54, 2, 55, 2, 56, 3, 57, 14, 170, 247, 18, 48, 2, 49, 2, 50, 2, - 53, 2, 55, 2, 56, 3, 57, 18, 230, 246, 18, 48, 2, 49, 2, 50, 2, 51, 2, - 52, 2, 53, 2, 54, 2, 55, 3, 57, 12, 146, 246, 18, 51, 2, 52, 2, 53, 2, - 54, 2, 56, 3, 57, 104, 70, 48, 78, 50, 86, 51, 38, 52, 78, 54, 254, 230, - 13, 53, 243, 1, 49, 16, 142, 245, 18, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, - 54, 2, 56, 3, 57, 18, 194, 244, 18, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, - 54, 2, 55, 2, 56, 3, 57, 6, 238, 243, 18, 52, 2, 55, 3, 56, 16, 202, 243, - 18, 48, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 10, 254, 242, - 18, 48, 2, 49, 2, 50, 2, 51, 3, 52, 44, 86, 48, 134, 2, 49, 140, 246, 1, - 2, 51, 50, 157, 232, 16, 6, 50, 54, 32, 69, 89, 89, 26, 110, 57, 194, - 193, 8, 55, 182, 6, 50, 62, 54, 218, 58, 52, 198, 148, 3, 51, 110, 56, - 202, 196, 2, 49, 215, 94, 53, 10, 26, 45, 207, 231, 17, 32, 8, 96, 2, 50, - 32, 212, 136, 12, 3, 54, 32, 76, 184, 2, 3, 52, 32, 76, 141, 145, 3, 3, - 51, 32, 76, 2, 175, 140, 12, 76, 14, 110, 49, 22, 51, 32, 3, 52, 32, 65, - 0, 2, 53, 32, 198, 147, 4, 50, 202, 181, 4, 48, 141, 209, 6, 2, 55, 32, - 2, 243, 226, 17, 32, 2, 11, 32, 2, 219, 238, 11, 79, 2, 167, 180, 17, 66, - 16, 182, 238, 18, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, + 73, 71, 72, 2, 167, 143, 11, 84, 2, 193, 174, 19, 4, 32, 83, 69, 82, 33, + 48, 6, 32, 87, 73, 84, 72, 32, 147, 139, 16, 79, 28, 110, 67, 18, 68, 70, + 71, 22, 72, 42, 76, 22, 83, 234, 1, 65, 186, 194, 3, 77, 202, 133, 13, + 84, 247, 178, 1, 82, 2, 203, 3, 73, 6, 26, 73, 215, 147, 16, 79, 2, 17, + 2, 65, 69, 2, 151, 172, 16, 82, 2, 223, 194, 18, 82, 4, 17, 2, 79, 79, 4, + 171, 131, 5, 75, 2, 195, 180, 19, 79, 4, 26, 72, 187, 253, 19, 84, 2, 21, + 3, 79, 82, 84, 2, 233, 246, 7, 6, 32, 82, 73, 71, 72, 84, 31, 33, 6, 32, + 87, 73, 84, 72, 32, 28, 98, 65, 22, 67, 82, 68, 38, 76, 34, 83, 146, 219, + 3, 77, 174, 7, 80, 218, 5, 82, 207, 238, 14, 72, 2, 231, 179, 13, 67, 6, + 42, 73, 230, 245, 6, 85, 155, 144, 12, 65, 2, 241, 192, 11, 4, 82, 67, + 85, 77, 6, 230, 181, 3, 69, 171, 218, 12, 79, 2, 11, 73, 2, 179, 172, 12, + 78, 4, 26, 87, 171, 250, 19, 84, 2, 179, 133, 17, 65, 18, 90, 70, 242, + 188, 9, 73, 172, 9, 6, 76, 79, 78, 71, 32, 83, 226, 198, 10, 83, 219, 5, + 79, 10, 34, 70, 134, 169, 20, 73, 3, 76, 7, 130, 169, 20, 73, 3, 76, 36, + 150, 1, 83, 210, 167, 20, 65, 2, 69, 2, 72, 2, 73, 2, 74, 2, 75, 2, 76, + 2, 77, 2, 78, 2, 79, 2, 80, 2, 82, 2, 84, 2, 85, 2, 86, 3, 88, 5, 143, + 225, 4, 67, 224, 5, 204, 1, 2, 65, 70, 144, 1, 2, 70, 84, 210, 38, 79, + 20, 5, 80, 67, 72, 65, 32, 164, 8, 8, 83, 83, 45, 84, 72, 65, 78, 32, + 254, 167, 16, 68, 228, 23, 6, 86, 69, 76, 32, 83, 76, 134, 230, 2, 77, + 239, 79, 71, 6, 120, 3, 76, 69, 83, 204, 191, 1, 14, 32, 70, 76, 85, 84, + 84, 69, 82, 73, 78, 71, 32, 73, 78, 209, 212, 11, 3, 89, 32, 71, 2, 135, + 228, 14, 83, 134, 4, 58, 32, 250, 20, 45, 217, 2, 6, 87, 65, 82, 68, 83, + 32, 246, 1, 166, 2, 65, 234, 4, 66, 202, 1, 67, 96, 2, 68, 79, 112, 2, + 72, 65, 166, 3, 76, 46, 77, 38, 79, 102, 82, 146, 3, 83, 82, 84, 222, 1, + 87, 242, 176, 8, 70, 236, 5, 14, 74, 85, 83, 84, 73, 70, 73, 69, 68, 32, + 82, 73, 71, 72, 86, 78, 202, 1, 80, 153, 13, 10, 86, 69, 82, 84, 73, 67, + 65, 76, 32, 66, 28, 22, 78, 143, 4, 82, 22, 28, 2, 68, 32, 191, 3, 71, + 16, 96, 6, 76, 79, 87, 69, 82, 32, 32, 6, 82, 73, 71, 72, 84, 32, 89, 6, + 85, 80, 80, 69, 82, 32, 4, 202, 1, 65, 235, 175, 17, 79, 6, 50, 84, 209, + 250, 17, 6, 68, 79, 85, 66, 76, 69, 4, 250, 136, 17, 82, 147, 231, 1, 65, + 6, 82, 65, 53, 16, 79, 78, 69, 32, 69, 73, 71, 72, 84, 72, 32, 66, 76, + 79, 67, 75, 2, 237, 134, 17, 8, 78, 68, 32, 82, 73, 71, 72, 84, 5, 165, + 253, 18, 17, 32, 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, 65, + 67, 75, 6, 154, 182, 8, 69, 185, 1, 4, 76, 69, 32, 66, 6, 26, 67, 247, + 185, 8, 82, 2, 209, 185, 8, 5, 32, 76, 69, 83, 83, 12, 40, 4, 65, 82, 66, + 32, 143, 186, 8, 76, 8, 40, 4, 68, 79, 87, 78, 1, 2, 85, 80, 4, 57, 12, + 32, 82, 73, 71, 72, 84, 32, 66, 65, 82, 66, 32, 4, 240, 244, 16, 4, 68, + 79, 87, 78, 1, 2, 85, 80, 14, 210, 185, 8, 85, 166, 26, 79, 206, 231, 2, + 69, 233, 168, 2, 8, 76, 79, 83, 69, 68, 32, 69, 78, 10, 44, 5, 85, 66, + 76, 69, 32, 255, 185, 8, 84, 8, 178, 187, 8, 87, 190, 30, 65, 138, 128, + 3, 81, 131, 190, 4, 80, 38, 36, 3, 76, 70, 32, 247, 191, 8, 78, 36, 132, + 2, 6, 67, 73, 82, 67, 76, 69, 246, 186, 8, 66, 90, 70, 82, 72, 50, 82, + 58, 84, 70, 87, 246, 13, 76, 22, 85, 220, 185, 8, 30, 73, 78, 86, 69, 82, + 83, 69, 32, 77, 69, 68, 73, 85, 77, 32, 83, 72, 65, 68, 69, 32, 65, 78, + 68, 32, 82, 73, 71, 72, 84, 251, 26, 77, 11, 33, 6, 32, 87, 73, 84, 72, + 32, 8, 42, 84, 206, 243, 17, 70, 235, 210, 1, 68, 4, 242, 197, 5, 72, + 179, 128, 13, 87, 4, 152, 158, 1, 2, 85, 71, 199, 159, 7, 79, 2, 161, + 133, 19, 4, 85, 76, 84, 73, 8, 36, 3, 78, 69, 32, 199, 190, 8, 85, 6, + 190, 162, 17, 81, 146, 4, 69, 45, 5, 84, 72, 73, 82, 68, 30, 44, 5, 73, + 71, 72, 84, 32, 155, 191, 8, 65, 28, 152, 1, 5, 65, 82, 82, 79, 87, 132, + 1, 12, 68, 79, 85, 66, 76, 69, 32, 65, 82, 82, 79, 87, 30, 87, 222, 236, + 8, 79, 154, 128, 8, 66, 54, 83, 207, 68, 84, 11, 11, 32, 8, 72, 5, 87, + 73, 84, 72, 32, 165, 192, 18, 7, 84, 72, 82, 79, 85, 71, 72, 6, 182, 174, + 16, 68, 242, 179, 3, 86, 79, 83, 7, 201, 228, 8, 2, 32, 87, 4, 210, 247, + 8, 65, 215, 176, 8, 72, 34, 174, 189, 8, 45, 70, 69, 78, 73, 168, 1, 3, + 80, 69, 69, 26, 81, 227, 2, 85, 26, 106, 82, 134, 194, 8, 72, 174, 212, + 7, 79, 208, 133, 1, 8, 87, 79, 32, 84, 72, 73, 82, 68, 219, 196, 1, 65, + 6, 40, 4, 73, 65, 78, 71, 255, 196, 8, 65, 4, 236, 236, 4, 8, 76, 69, 32, + 66, 69, 83, 73, 68, 207, 175, 12, 85, 16, 162, 199, 8, 72, 202, 1, 73, + 169, 184, 10, 2, 82, 73, 62, 128, 1, 9, 80, 79, 73, 78, 84, 73, 78, 71, + 32, 138, 1, 83, 206, 199, 8, 70, 226, 2, 72, 153, 7, 7, 84, 79, 45, 82, + 73, 71, 72, 30, 86, 65, 214, 204, 8, 67, 94, 68, 58, 77, 58, 82, 182, 1, + 83, 74, 84, 255, 157, 8, 69, 6, 146, 205, 8, 78, 202, 160, 8, 84, 159, 2, + 73, 4, 44, 5, 73, 68, 69, 32, 65, 131, 208, 8, 72, 2, 253, 142, 1, 2, 82, + 67, 210, 1, 172, 1, 5, 65, 82, 82, 79, 87, 174, 5, 66, 70, 72, 134, 4, + 84, 178, 2, 87, 226, 207, 8, 68, 210, 1, 70, 222, 7, 76, 26, 79, 34, 80, + 50, 82, 70, 83, 230, 191, 8, 67, 47, 81, 73, 26, 32, 143, 215, 17, 45, + 68, 102, 65, 138, 1, 84, 132, 2, 5, 87, 73, 84, 72, 32, 198, 209, 8, 70, + 181, 149, 10, 4, 79, 86, 69, 82, 12, 44, 5, 66, 79, 86, 69, 32, 179, 211, + 8, 78, 10, 64, 4, 83, 72, 79, 82, 170, 210, 8, 82, 142, 226, 4, 65, 55, + 84, 2, 143, 233, 18, 84, 12, 56, 7, 72, 82, 79, 85, 71, 72, 32, 53, 3, + 79, 32, 66, 6, 142, 178, 13, 83, 202, 196, 4, 76, 247, 145, 2, 88, 6, 32, + 2, 65, 82, 187, 212, 8, 76, 5, 169, 158, 12, 23, 32, 79, 86, 69, 82, 32, + 82, 73, 71, 72, 84, 87, 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 36, + 118, 76, 154, 212, 8, 68, 182, 1, 80, 30, 83, 38, 84, 178, 195, 8, 77, + 38, 78, 122, 69, 226, 151, 1, 72, 143, 163, 1, 86, 4, 158, 153, 17, 65, + 219, 243, 1, 79, 8, 190, 214, 8, 65, 224, 10, 5, 79, 84, 84, 79, 77, 207, + 185, 8, 76, 30, 26, 65, 255, 159, 17, 69, 26, 48, 6, 82, 80, 79, 79, 78, + 32, 183, 190, 19, 78, 24, 100, 10, 87, 73, 84, 72, 32, 66, 65, 82, 66, + 32, 209, 216, 8, 9, 79, 86, 69, 82, 32, 82, 73, 71, 72, 22, 40, 4, 68, + 79, 87, 78, 117, 2, 85, 80, 10, 26, 32, 219, 165, 17, 87, 8, 222, 217, 8, + 66, 188, 2, 7, 65, 66, 79, 86, 69, 32, 82, 222, 193, 8, 70, 179, 5, 84, + 12, 26, 32, 231, 164, 17, 87, 10, 60, 6, 65, 66, 79, 86, 69, 32, 198, + 156, 17, 70, 179, 5, 84, 6, 42, 76, 201, 217, 8, 4, 82, 73, 71, 72, 4, + 238, 215, 8, 69, 199, 218, 4, 79, 54, 44, 2, 82, 73, 190, 221, 8, 79, + 159, 5, 87, 34, 44, 5, 65, 78, 71, 76, 69, 243, 225, 8, 80, 30, 56, 8, + 45, 72, 69, 65, 68, 69, 68, 32, 227, 170, 17, 32, 28, 52, 5, 65, 82, 82, + 79, 87, 178, 163, 17, 68, 39, 80, 25, 11, 32, 22, 180, 222, 8, 9, 79, 86, + 69, 82, 32, 82, 73, 71, 72, 22, 87, 251, 192, 8, 84, 6, 26, 72, 131, 228, + 8, 65, 4, 45, 9, 73, 84, 69, 32, 65, 82, 82, 79, 87, 5, 149, 168, 17, 2, + 32, 87, 5, 143, 231, 18, 80, 148, 1, 252, 1, 15, 67, 79, 78, 83, 79, 78, + 65, 78, 84, 32, 83, 73, 71, 78, 32, 124, 7, 76, 69, 84, 84, 69, 82, 32, + 236, 1, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 198, 1, 83, + 172, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 251, 132, 18, 68, + 18, 66, 75, 22, 78, 154, 251, 19, 76, 2, 77, 2, 80, 2, 82, 3, 84, 5, 179, + 170, 19, 65, 5, 233, 201, 12, 4, 89, 73, 78, 45, 78, 194, 1, 75, 2, 80, + 190, 148, 7, 68, 194, 219, 10, 66, 2, 70, 2, 71, 2, 72, 2, 77, 138, 15, + 78, 194, 178, 1, 84, 46, 67, 2, 83, 138, 69, 74, 2, 76, 2, 82, 2, 86, 2, + 87, 2, 89, 187, 2, 65, 6, 246, 246, 19, 72, 2, 76, 187, 2, 65, 10, 82, + 84, 40, 14, 78, 89, 69, 84, 32, 84, 72, 89, 79, 79, 77, 32, 84, 65, 43, + 67, 6, 38, 65, 21, 5, 83, 72, 79, 79, 75, 2, 179, 248, 6, 45, 5, 17, 2, + 32, 67, 2, 217, 134, 15, 3, 69, 82, 45, 8, 92, 4, 73, 71, 78, 32, 37, 15, + 85, 66, 74, 79, 73, 78, 69, 68, 32, 76, 69, 84, 84, 69, 82, 4, 130, 227, + 15, 78, 227, 201, 2, 82, 4, 11, 32, 4, 250, 243, 19, 82, 3, 89, 14, 238, + 162, 16, 85, 202, 141, 1, 79, 170, 195, 2, 65, 186, 2, 69, 3, 73, 52, + 138, 1, 65, 128, 4, 11, 69, 81, 85, 65, 76, 32, 84, 79, 32, 79, 82, 200, + 1, 2, 66, 85, 42, 67, 186, 1, 79, 210, 2, 87, 231, 227, 18, 83, 16, 40, + 5, 66, 79, 86, 69, 32, 171, 4, 78, 12, 152, 1, 7, 71, 82, 69, 65, 84, 69, + 82, 110, 83, 176, 1, 19, 68, 79, 85, 66, 76, 69, 45, 76, 73, 78, 69, 32, + 69, 81, 85, 65, 76, 32, 65, 227, 212, 15, 76, 2, 181, 5, 23, 45, 84, 72, + 65, 78, 32, 65, 66, 79, 86, 69, 32, 68, 79, 85, 66, 76, 69, 45, 76, 73, + 78, 69, 6, 152, 1, 7, 73, 77, 73, 76, 65, 82, 32, 93, 26, 76, 65, 78, 84, + 69, 68, 32, 69, 81, 85, 65, 76, 32, 65, 66, 79, 86, 69, 32, 71, 82, 69, + 65, 84, 69, 82, 4, 18, 65, 59, 79, 2, 25, 4, 66, 79, 86, 69, 2, 169, 222, + 17, 2, 32, 71, 2, 227, 2, 82, 2, 145, 2, 6, 45, 84, 72, 65, 78, 32, 4, + 17, 2, 68, 32, 4, 220, 3, 5, 78, 79, 84, 32, 65, 245, 151, 13, 11, 83, + 73, 78, 71, 76, 69, 45, 76, 73, 78, 69, 4, 241, 249, 12, 5, 84, 32, 78, + 79, 84, 4, 65, 14, 76, 79, 83, 69, 68, 32, 66, 89, 32, 67, 85, 82, 86, + 69, 5, 11, 32, 2, 61, 13, 65, 66, 79, 86, 69, 32, 83, 76, 65, 78, 84, 69, + 68, 2, 17, 2, 32, 69, 2, 251, 228, 14, 81, 18, 40, 2, 82, 32, 141, 220, + 11, 2, 86, 69, 16, 114, 65, 48, 16, 83, 76, 65, 78, 84, 69, 68, 32, 69, + 81, 85, 65, 76, 32, 84, 79, 218, 246, 12, 69, 223, 226, 4, 71, 2, 173, + 184, 9, 7, 80, 80, 82, 79, 88, 73, 77, 9, 49, 10, 32, 87, 73, 84, 72, 32, + 68, 79, 84, 32, 6, 26, 65, 163, 199, 11, 73, 4, 25, 4, 66, 79, 86, 69, 5, + 231, 153, 18, 32, 6, 25, 4, 73, 84, 72, 32, 6, 66, 67, 40, 8, 81, 85, 69, + 83, 84, 73, 79, 78, 143, 153, 19, 68, 2, 249, 197, 11, 5, 73, 82, 67, 76, + 69, 2, 17, 2, 32, 77, 2, 17, 2, 65, 82, 2, 151, 236, 18, 75, 136, 11, + 190, 1, 71, 186, 5, 77, 166, 7, 78, 168, 65, 2, 80, 83, 20, 3, 83, 85, + 32, 238, 199, 15, 82, 136, 35, 11, 86, 82, 69, 32, 84, 79, 85, 82, 78, + 79, 73, 210, 59, 79, 242, 219, 1, 90, 187, 82, 66, 44, 26, 65, 57, 2, 72, + 84, 2, 245, 167, 9, 9, 84, 85, 82, 69, 32, 79, 80, 69, 78, 42, 38, 32, + 137, 4, 4, 78, 73, 78, 71, 36, 146, 1, 69, 38, 70, 126, 82, 40, 3, 76, + 69, 70, 86, 83, 42, 84, 188, 198, 7, 3, 66, 76, 85, 138, 156, 9, 87, 250, + 41, 86, 250, 77, 71, 135, 69, 67, 2, 129, 209, 17, 4, 73, 71, 72, 84, 8, + 94, 73, 205, 157, 18, 17, 79, 85, 82, 32, 80, 79, 73, 78, 84, 69, 68, 32, + 66, 76, 65, 67, 75, 4, 149, 205, 17, 2, 86, 69, 4, 36, 3, 73, 71, 72, + 155, 176, 17, 65, 2, 165, 237, 1, 16, 84, 32, 84, 79, 82, 84, 79, 73, 83, + 69, 32, 83, 72, 69, 76, 76, 6, 170, 236, 16, 72, 138, 98, 65, 43, 73, 4, + 200, 153, 16, 2, 87, 69, 21, 3, 72, 82, 69, 7, 29, 5, 32, 77, 79, 79, 68, + 5, 155, 149, 8, 32, 138, 1, 80, 3, 66, 85, 32, 137, 157, 8, 11, 73, 84, + 69, 68, 32, 76, 73, 65, 66, 73, 76, 136, 1, 128, 1, 7, 76, 69, 84, 84, + 69, 82, 32, 222, 1, 83, 200, 2, 5, 86, 79, 87, 69, 76, 194, 206, 12, 69, + 182, 188, 4, 81, 247, 95, 68, 60, 170, 1, 71, 242, 139, 6, 84, 170, 221, + 8, 89, 186, 132, 1, 78, 162, 169, 3, 83, 82, 66, 2, 67, 2, 68, 2, 74, 2, + 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, 2, 82, 3, 87, 6, 254, 148, 18, 89, + 202, 199, 1, 72, 187, 2, 65, 32, 108, 4, 73, 71, 78, 32, 128, 1, 12, 77, + 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 181, 250, 6, 2, 85, 66, 8, + 72, 3, 75, 69, 77, 0, 3, 77, 85, 75, 32, 2, 83, 65, 207, 149, 17, 76, 2, + 129, 154, 17, 3, 80, 72, 82, 2, 155, 201, 19, 45, 18, 174, 235, 15, 78, + 186, 172, 3, 65, 194, 66, 75, 2, 76, 2, 77, 2, 80, 2, 82, 3, 84, 20, 84, + 6, 32, 83, 73, 71, 78, 32, 177, 85, 10, 45, 67, 65, 82, 82, 73, 69, 82, + 32, 76, 18, 170, 211, 5, 65, 166, 194, 11, 79, 194, 133, 2, 69, 162, 64, + 73, 3, 85, 226, 8, 22, 69, 199, 64, 75, 222, 8, 34, 32, 177, 3, 3, 65, + 82, 32, 14, 120, 12, 73, 78, 84, 69, 71, 82, 65, 84, 73, 79, 78, 32, 186, + 156, 13, 70, 150, 143, 3, 83, 245, 147, 1, 4, 84, 65, 66, 85, 6, 108, 5, + 87, 73, 84, 72, 32, 157, 1, 17, 78, 79, 84, 32, 73, 78, 67, 76, 85, 68, + 73, 78, 71, 32, 84, 72, 69, 4, 76, 7, 82, 69, 67, 84, 65, 78, 71, 1, 8, + 83, 69, 77, 73, 67, 73, 82, 67, 2, 73, 16, 85, 76, 65, 82, 32, 80, 65, + 84, 72, 32, 65, 82, 79, 85, 78, 68, 2, 17, 2, 32, 80, 2, 223, 178, 18, + 79, 208, 8, 60, 8, 65, 32, 83, 73, 71, 78, 32, 65, 221, 24, 2, 66, 32, + 170, 5, 122, 49, 86, 51, 202, 2, 52, 214, 1, 53, 158, 4, 54, 142, 3, 55, + 148, 4, 2, 56, 48, 78, 66, 213, 136, 18, 3, 48, 50, 56, 6, 128, 188, 12, + 5, 48, 48, 45, 49, 48, 200, 221, 5, 2, 50, 48, 233, 19, 2, 51, 49, 150, + 1, 78, 48, 90, 49, 130, 1, 55, 246, 156, 14, 50, 2, 51, 2, 52, 2, 53, 3, + 54, 22, 178, 1, 57, 214, 210, 19, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, + 2, 55, 3, 56, 24, 90, 51, 214, 210, 19, 48, 2, 49, 2, 50, 2, 52, 2, 53, + 2, 54, 2, 55, 2, 56, 3, 57, 6, 210, 210, 19, 65, 2, 66, 3, 67, 4, 174, + 210, 19, 48, 3, 49, 38, 18, 48, 91, 49, 20, 162, 1, 48, 2, 49, 2, 50, 2, + 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 74, 48, 2, 49, 2, 50, + 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 2, 205, 178, 6, 2, 45, 86, 160, + 1, 102, 48, 78, 49, 62, 51, 86, 52, 70, 53, 86, 54, 206, 11, 50, 150, + 166, 2, 57, 198, 229, 11, 55, 3, 56, 16, 210, 207, 19, 49, 2, 50, 2, 51, + 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 12, 134, 207, 19, 48, 2, 49, 2, 50, 2, + 51, 2, 53, 3, 54, 18, 202, 206, 19, 48, 2, 49, 2, 50, 2, 52, 2, 53, 2, + 54, 2, 55, 2, 56, 3, 57, 14, 246, 205, 19, 48, 2, 49, 2, 50, 2, 53, 2, + 55, 2, 56, 3, 57, 18, 178, 205, 19, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, + 53, 2, 54, 2, 55, 3, 57, 12, 222, 204, 19, 51, 2, 52, 2, 53, 2, 54, 2, + 56, 3, 57, 104, 70, 48, 78, 50, 86, 51, 38, 52, 78, 54, 162, 146, 14, 53, + 243, 1, 49, 16, 218, 203, 19, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, + 56, 3, 57, 18, 142, 203, 19, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, + 55, 2, 56, 3, 57, 6, 186, 202, 19, 52, 2, 55, 3, 56, 16, 150, 202, 19, + 48, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 10, 202, 201, 19, + 48, 2, 49, 2, 50, 2, 51, 3, 52, 44, 86, 48, 134, 2, 49, 196, 252, 1, 2, + 51, 50, 177, 184, 17, 6, 50, 54, 32, 69, 89, 89, 26, 110, 57, 142, 219, + 8, 55, 182, 6, 50, 62, 54, 230, 62, 52, 226, 151, 3, 51, 110, 56, 134, + 206, 2, 49, 147, 117, 53, 10, 26, 45, 131, 190, 18, 32, 8, 96, 2, 50, 32, + 200, 169, 12, 3, 54, 32, 76, 184, 2, 3, 52, 32, 76, 201, 164, 3, 3, 51, + 32, 76, 2, 163, 173, 12, 76, 14, 110, 49, 22, 51, 32, 3, 52, 32, 65, 0, + 2, 53, 32, 238, 154, 4, 50, 238, 199, 4, 48, 241, 235, 6, 2, 55, 32, 2, + 167, 185, 18, 32, 2, 11, 32, 2, 175, 142, 12, 79, 2, 179, 137, 18, 66, + 16, 130, 197, 19, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 162, 1, 22, 48, 155, 5, 49, 140, 1, 82, 49, 54, 50, 118, 51, 62, 52, 78, - 53, 86, 54, 62, 55, 70, 56, 243, 220, 13, 48, 10, 246, 236, 18, 48, 2, - 49, 2, 51, 2, 54, 3, 55, 28, 86, 49, 2, 50, 194, 70, 51, 174, 165, 18, - 48, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 7, 234, 235, 18, 70, 3, 77, 12, - 206, 235, 18, 48, 2, 49, 2, 52, 2, 55, 2, 56, 3, 57, 16, 146, 235, 18, - 48, 2, 49, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 198, 234, 18, - 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 12, 242, 233, - 18, 48, 2, 49, 2, 53, 2, 54, 2, 55, 3, 57, 14, 182, 233, 18, 48, 2, 51, - 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 12, 242, 232, 18, 48, 2, 49, 2, 50, 2, - 53, 2, 54, 3, 55, 22, 82, 50, 36, 2, 51, 49, 30, 56, 226, 255, 11, 49, - 206, 2, 54, 146, 1, 55, 3, 57, 6, 226, 231, 18, 48, 2, 50, 3, 51, 4, 190, - 231, 18, 65, 3, 66, 4, 162, 231, 18, 48, 3, 56, 166, 3, 116, 9, 73, 68, + 53, 86, 54, 62, 55, 70, 56, 151, 136, 14, 48, 10, 194, 195, 19, 48, 2, + 49, 2, 51, 2, 54, 3, 55, 28, 86, 49, 2, 50, 146, 77, 51, 170, 245, 18, + 48, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 7, 182, 194, 19, 70, 3, 77, 12, + 154, 194, 19, 48, 2, 49, 2, 52, 2, 55, 2, 56, 3, 57, 16, 222, 193, 19, + 48, 2, 49, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 146, 193, 19, + 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 12, 190, 192, + 19, 48, 2, 49, 2, 53, 2, 54, 2, 55, 3, 57, 14, 130, 192, 19, 48, 2, 51, + 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 12, 190, 191, 19, 48, 2, 49, 2, 50, 2, + 53, 2, 54, 3, 55, 22, 82, 50, 36, 2, 51, 49, 30, 56, 214, 160, 12, 49, + 206, 2, 54, 146, 1, 55, 3, 57, 6, 174, 190, 19, 48, 2, 50, 3, 51, 4, 138, + 190, 19, 65, 3, 66, 4, 238, 189, 19, 48, 3, 56, 166, 3, 116, 9, 73, 68, 69, 79, 71, 82, 65, 77, 32, 216, 17, 10, 77, 79, 78, 79, 71, 82, 65, 77, 32, 66, 249, 1, 2, 83, 89, 234, 1, 54, 66, 249, 15, 8, 86, 69, 83, 83, 69, 76, 32, 66, 176, 1, 22, 49, 255, 10, 50, 126, 86, 48, 210, 3, 50, 162, 1, 51, 86, 52, 122, 53, 114, 54, 146, 1, 55, 118, 56, 71, 57, 28, - 114, 53, 98, 54, 58, 55, 78, 56, 62, 57, 148, 247, 6, 3, 50, 32, 87, 234, - 84, 48, 237, 215, 10, 4, 52, 32, 68, 69, 6, 252, 245, 6, 2, 32, 69, 188, - 205, 10, 3, 70, 32, 77, 245, 79, 7, 77, 32, 83, 84, 65, 76, 76, 4, 224, - 233, 1, 3, 70, 32, 69, 221, 191, 15, 2, 77, 32, 4, 36, 3, 70, 32, 83, 1, - 2, 77, 32, 2, 213, 195, 11, 4, 72, 69, 45, 71, 4, 228, 250, 8, 3, 77, 32, - 66, 145, 205, 8, 3, 70, 32, 83, 4, 148, 221, 16, 4, 77, 32, 66, 85, 165, - 106, 3, 70, 32, 67, 10, 236, 243, 8, 4, 51, 32, 83, 80, 224, 10, 5, 49, - 32, 66, 65, 82, 160, 218, 4, 4, 50, 32, 79, 76, 20, 6, 53, 32, 67, 89, - 80, 69, 209, 99, 4, 48, 32, 87, 72, 6, 54, 49, 192, 178, 16, 3, 48, 32, - 79, 247, 172, 2, 50, 2, 145, 198, 17, 2, 32, 87, 10, 224, 5, 3, 53, 32, - 87, 244, 241, 8, 6, 48, 32, 66, 82, 79, 78, 172, 2, 4, 49, 32, 71, 79, - 150, 229, 9, 50, 3, 54, 16, 82, 57, 150, 226, 2, 49, 186, 251, 15, 48, 2, - 50, 2, 51, 2, 52, 2, 55, 3, 56, 2, 253, 90, 3, 32, 67, 76, 20, 184, 246, - 13, 5, 51, 32, 65, 82, 77, 204, 163, 4, 5, 50, 32, 71, 65, 82, 170, 67, - 48, 2, 49, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 202, 198, 13, - 54, 196, 150, 3, 4, 51, 32, 77, 79, 146, 255, 1, 48, 2, 49, 2, 50, 2, 52, - 2, 55, 2, 56, 3, 57, 14, 170, 219, 18, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, - 53, 3, 57, 4, 240, 157, 2, 3, 49, 32, 72, 247, 188, 16, 48, 50, 42, 50, - 110, 51, 130, 1, 52, 227, 1, 53, 4, 84, 8, 48, 32, 70, 79, 79, 84, 83, - 84, 181, 196, 6, 7, 53, 32, 66, 65, 84, 72, 84, 2, 219, 198, 17, 79, 12, - 104, 4, 51, 32, 83, 87, 248, 211, 13, 4, 48, 32, 83, 80, 130, 234, 3, 49, - 190, 154, 1, 50, 2, 52, 3, 54, 2, 223, 193, 17, 79, 16, 168, 1, 9, 48, - 32, 87, 72, 69, 69, 76, 69, 68, 2, 49, 34, 51, 240, 202, 16, 12, 50, 32, - 67, 72, 65, 82, 73, 79, 84, 32, 70, 82, 230, 139, 2, 53, 2, 54, 2, 56, 3, - 57, 2, 133, 205, 17, 3, 32, 67, 72, 2, 183, 176, 6, 32, 18, 184, 207, 17, - 3, 52, 32, 68, 134, 135, 1, 49, 2, 50, 2, 51, 2, 53, 2, 54, 2, 55, 2, 56, - 3, 57, 58, 50, 50, 204, 239, 11, 2, 49, 53, 1, 2, 51, 48, 54, 50, 50, - 250, 241, 11, 53, 230, 216, 1, 48, 3, 49, 12, 238, 212, 18, 49, 2, 50, 2, - 54, 2, 55, 2, 56, 3, 57, 12, 46, 49, 153, 12, 6, 50, 52, 55, 32, 68, 73, - 10, 50, 50, 70, 51, 181, 11, 5, 53, 54, 32, 84, 85, 4, 252, 167, 3, 3, - 55, 32, 75, 197, 140, 15, 5, 56, 32, 75, 65, 78, 4, 56, 3, 53, 32, 77, - 241, 173, 15, 5, 51, 32, 65, 82, 69, 2, 231, 201, 11, 69, 176, 1, 84, 9, - 76, 76, 65, 66, 76, 69, 32, 66, 48, 229, 12, 7, 77, 66, 79, 76, 32, 66, - 48, 148, 1, 114, 48, 142, 1, 49, 162, 1, 50, 230, 1, 51, 174, 1, 52, 162, - 1, 53, 154, 1, 54, 186, 1, 55, 198, 1, 56, 87, 57, 18, 118, 54, 250, 191, - 1, 55, 150, 8, 52, 162, 14, 57, 138, 211, 10, 53, 186, 146, 2, 56, 226, - 10, 49, 254, 2, 50, 207, 8, 51, 2, 171, 149, 17, 32, 16, 122, 54, 18, 55, - 158, 201, 1, 49, 142, 2, 50, 158, 26, 52, 196, 186, 1, 2, 53, 32, 198, - 220, 5, 48, 177, 186, 8, 2, 51, 32, 2, 135, 84, 32, 2, 159, 232, 10, 32, - 18, 166, 1, 51, 22, 54, 20, 3, 57, 32, 80, 224, 6, 2, 53, 32, 136, 168, - 3, 2, 48, 32, 228, 253, 13, 2, 55, 32, 140, 7, 2, 52, 32, 146, 89, 56, - 173, 44, 3, 49, 32, 81, 2, 151, 250, 13, 32, 2, 151, 192, 14, 32, 2, 151, - 233, 11, 85, 16, 134, 1, 48, 20, 2, 51, 32, 254, 188, 1, 55, 254, 34, 54, - 194, 233, 4, 57, 138, 148, 2, 56, 210, 154, 4, 49, 185, 180, 5, 3, 50, - 32, 81, 2, 215, 251, 13, 32, 2, 135, 1, 82, 16, 116, 2, 51, 32, 22, 53, - 242, 184, 1, 48, 222, 1, 49, 134, 6, 50, 222, 10, 52, 146, 5, 54, 233, - 180, 12, 3, 56, 32, 78, 2, 199, 246, 14, 65, 2, 171, 189, 16, 32, 18, - 130, 1, 51, 162, 185, 1, 49, 150, 1, 56, 42, 57, 102, 55, 214, 5, 48, - 158, 140, 2, 52, 200, 210, 14, 2, 50, 32, 157, 4, 2, 53, 32, 2, 159, 192, - 11, 32, 16, 132, 1, 2, 50, 32, 20, 2, 56, 32, 204, 1, 3, 54, 32, 84, 150, - 180, 1, 55, 226, 3, 57, 146, 2, 53, 158, 228, 6, 49, 163, 221, 9, 48, 2, - 191, 131, 17, 80, 2, 129, 228, 11, 2, 82, 79, 18, 172, 1, 3, 54, 32, 82, - 182, 180, 1, 55, 158, 8, 48, 142, 16, 53, 12, 3, 49, 32, 68, 228, 145, 7, - 2, 52, 32, 214, 235, 3, 50, 152, 229, 5, 3, 56, 32, 81, 241, 2, 2, 51, - 32, 2, 183, 226, 11, 65, 8, 178, 180, 1, 49, 160, 24, 3, 55, 32, 84, 172, - 230, 6, 2, 53, 32, 211, 129, 6, 48, 4, 158, 184, 12, 49, 21, 3, 48, 32, - 68, 28, 90, 52, 30, 54, 30, 56, 186, 220, 11, 53, 158, 2, 49, 30, 51, - 166, 1, 50, 175, 144, 3, 55, 4, 158, 196, 18, 55, 3, 57, 4, 130, 196, 18, - 51, 3, 52, 8, 230, 195, 18, 50, 2, 51, 2, 54, 3, 57, 4, 156, 173, 10, 9, - 69, 68, 32, 80, 65, 80, 69, 82, 67, 155, 131, 7, 32, 5, 155, 148, 17, 84, - 98, 96, 7, 76, 69, 84, 84, 69, 82, 32, 253, 219, 6, 11, 80, 85, 78, 67, - 84, 85, 65, 84, 73, 79, 78, 94, 234, 1, 84, 186, 234, 2, 68, 226, 249, 9, - 85, 222, 140, 2, 69, 22, 78, 214, 135, 3, 67, 2, 71, 2, 72, 2, 75, 2, 80, - 2, 83, 2, 89, 2, 90, 162, 7, 65, 2, 79, 222, 61, 66, 2, 70, 2, 74, 2, 76, - 2, 77, 2, 87, 2, 88, 187, 2, 73, 20, 64, 4, 79, 78, 69, 32, 154, 248, 17, - 83, 254, 68, 72, 187, 2, 65, 12, 48, 4, 77, 89, 65, 32, 229, 185, 1, 2, - 78, 65, 10, 174, 227, 3, 74, 246, 188, 14, 66, 158, 11, 84, 254, 16, 67, - 39, 78, 208, 1, 244, 1, 2, 67, 75, 132, 1, 6, 71, 73, 67, 65, 76, 32, - 236, 3, 3, 78, 71, 32, 230, 4, 84, 116, 3, 86, 69, 32, 70, 87, 144, 8, 5, - 90, 69, 78, 71, 69, 218, 212, 7, 66, 220, 204, 9, 8, 85, 68, 76, 89, 32, - 67, 82, 89, 133, 15, 4, 76, 76, 73, 80, 9, 96, 10, 73, 78, 71, 45, 83, - 72, 73, 70, 84, 32, 149, 16, 9, 32, 87, 73, 84, 72, 32, 73, 78, 75, 4, - 210, 207, 16, 90, 143, 83, 79, 28, 36, 3, 65, 78, 68, 85, 2, 79, 82, 15, - 33, 6, 32, 87, 73, 84, 72, 32, 12, 226, 1, 68, 94, 72, 58, 77, 187, 239, - 14, 85, 15, 11, 32, 12, 88, 13, 79, 86, 69, 82, 76, 65, 80, 80, 73, 78, - 71, 32, 76, 49, 5, 87, 73, 84, 72, 32, 2, 129, 172, 17, 7, 79, 71, 73, - 67, 65, 76, 32, 10, 26, 68, 94, 72, 59, 77, 6, 11, 79, 6, 44, 5, 85, 66, - 76, 69, 32, 247, 187, 17, 84, 4, 142, 240, 14, 85, 167, 47, 79, 2, 149, - 144, 15, 9, 79, 82, 73, 90, 79, 78, 84, 65, 76, 2, 213, 196, 7, 5, 73, - 68, 68, 76, 69, 34, 66, 68, 236, 1, 4, 76, 69, 70, 84, 109, 5, 82, 73, - 71, 72, 84, 6, 172, 1, 30, 65, 83, 72, 32, 70, 82, 79, 77, 32, 76, 69, - 70, 84, 32, 77, 69, 77, 66, 69, 82, 32, 79, 70, 32, 68, 79, 85, 66, 76, - 69, 196, 180, 13, 2, 73, 86, 191, 217, 3, 82, 2, 17, 2, 32, 86, 2, 181, - 172, 17, 5, 69, 82, 84, 73, 67, 16, 18, 32, 119, 87, 6, 48, 6, 82, 73, - 71, 72, 84, 32, 215, 170, 15, 84, 4, 130, 165, 15, 68, 203, 245, 1, 65, - 12, 26, 87, 215, 216, 8, 32, 10, 29, 5, 65, 82, 68, 83, 32, 10, 82, 65, - 0, 8, 68, 79, 85, 66, 76, 69, 32, 65, 153, 205, 7, 4, 83, 81, 85, 73, 4, - 25, 4, 82, 82, 79, 87, 5, 233, 211, 15, 2, 32, 70, 6, 92, 5, 73, 79, 78, - 32, 66, 144, 153, 17, 9, 32, 79, 70, 32, 70, 79, 82, 84, 85, 199, 91, 85, - 2, 131, 209, 9, 79, 4, 38, 76, 145, 209, 13, 3, 72, 79, 84, 2, 153, 251, - 16, 2, 69, 84, 118, 34, 32, 229, 1, 3, 69, 82, 32, 14, 138, 1, 66, 140, - 169, 10, 11, 68, 79, 85, 66, 76, 69, 32, 80, 82, 73, 77, 214, 246, 5, 65, - 148, 109, 6, 75, 65, 86, 89, 75, 65, 139, 10, 76, 4, 38, 82, 149, 239, 4, - 3, 65, 84, 84, 2, 213, 156, 17, 7, 73, 71, 72, 84, 78, 69, 83, 104, 152, - 1, 5, 72, 65, 76, 70, 32, 92, 5, 76, 69, 70, 84, 32, 220, 2, 6, 82, 73, - 71, 72, 84, 32, 210, 164, 15, 66, 74, 70, 154, 9, 79, 150, 13, 83, 63, - 84, 8, 208, 137, 1, 7, 73, 78, 86, 69, 82, 83, 69, 238, 180, 14, 77, 162, - 7, 66, 195, 153, 1, 67, 40, 174, 1, 66, 60, 8, 70, 79, 85, 78, 84, 65, - 73, 78, 22, 80, 52, 12, 83, 69, 77, 73, 67, 73, 82, 67, 85, 76, 65, 82, - 202, 1, 84, 232, 166, 11, 3, 67, 82, 65, 147, 144, 4, 81, 24, 56, 8, 65, - 76, 76, 80, 79, 73, 78, 84, 227, 176, 15, 76, 2, 183, 215, 9, 32, 4, 176, - 203, 14, 5, 65, 73, 78, 84, 66, 171, 109, 69, 2, 177, 173, 7, 5, 32, 65, - 78, 84, 73, 40, 78, 83, 82, 84, 158, 168, 15, 66, 238, 3, 67, 130, 9, 68, - 206, 1, 80, 39, 81, 6, 172, 172, 7, 11, 69, 77, 73, 67, 73, 82, 67, 85, - 76, 65, 82, 131, 140, 8, 72, 4, 139, 185, 15, 82, 5, 149, 136, 17, 25, - 32, 68, 73, 86, 73, 68, 69, 68, 32, 66, 89, 32, 72, 79, 82, 73, 90, 79, - 78, 84, 65, 76, 32, 82, 85, 6, 18, 71, 23, 78, 2, 231, 205, 13, 71, 4, - 140, 134, 17, 5, 65, 82, 32, 69, 67, 191, 100, 71, 114, 104, 12, 67, 73, - 65, 78, 32, 76, 69, 84, 84, 69, 82, 32, 228, 1, 5, 68, 73, 65, 78, 32, - 159, 157, 17, 73, 58, 206, 1, 77, 194, 130, 5, 75, 142, 175, 9, 66, 50, - 84, 178, 251, 1, 65, 2, 69, 2, 78, 130, 248, 1, 68, 2, 71, 2, 72, 2, 73, - 2, 74, 2, 76, 2, 80, 2, 81, 2, 82, 2, 83, 2, 85, 2, 87, 2, 88, 3, 90, 5, - 171, 165, 18, 77, 54, 88, 7, 76, 69, 84, 84, 69, 82, 32, 253, 177, 14, 9, - 84, 82, 73, 65, 78, 71, 85, 76, 65, 52, 198, 239, 12, 84, 254, 193, 1, - 76, 154, 239, 1, 83, 226, 11, 65, 2, 69, 2, 78, 130, 248, 1, 66, 2, 67, - 2, 68, 2, 70, 2, 71, 2, 73, 2, 75, 2, 77, 2, 79, 2, 81, 2, 82, 2, 85, 2, - 86, 3, 89, 194, 53, 162, 1, 65, 242, 103, 69, 138, 103, 73, 254, 24, 79, - 136, 112, 3, 82, 79, 32, 234, 3, 85, 164, 70, 7, 89, 65, 78, 77, 65, 82, - 32, 246, 193, 14, 86, 186, 61, 77, 27, 87, 204, 23, 186, 1, 71, 62, 72, - 130, 11, 75, 234, 3, 76, 156, 14, 2, 77, 77, 22, 78, 150, 17, 80, 118, - 82, 174, 7, 83, 194, 7, 84, 196, 36, 3, 89, 65, 78, 132, 129, 1, 3, 88, - 73, 77, 243, 151, 15, 67, 6, 156, 146, 17, 4, 73, 67, 32, 87, 226, 88, - 78, 143, 53, 69, 166, 1, 84, 6, 65, 74, 65, 78, 73, 32, 133, 3, 10, 74, - 79, 78, 71, 32, 84, 73, 76, 69, 32, 78, 38, 76, 166, 2, 83, 167, 220, 12, - 65, 72, 88, 6, 69, 84, 84, 69, 82, 32, 137, 149, 11, 10, 73, 71, 65, 84, - 85, 82, 69, 32, 83, 72, 70, 202, 173, 3, 78, 138, 208, 10, 82, 242, 55, - 68, 46, 84, 206, 160, 3, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, - 72, 2, 76, 2, 77, 2, 83, 2, 86, 186, 2, 65, 2, 69, 2, 73, 2, 79, 3, 85, - 4, 174, 227, 6, 69, 177, 211, 7, 5, 73, 71, 78, 32, 78, 88, 236, 1, 2, - 66, 65, 38, 69, 46, 70, 46, 78, 42, 79, 46, 80, 22, 83, 118, 84, 194, 1, - 87, 112, 5, 71, 82, 69, 69, 78, 0, 3, 82, 69, 68, 168, 128, 6, 3, 65, 85, - 84, 210, 21, 74, 141, 215, 10, 11, 67, 72, 82, 89, 83, 65, 78, 84, 72, - 69, 77, 4, 254, 173, 1, 77, 235, 232, 16, 67, 8, 228, 2, 4, 73, 71, 72, - 84, 195, 1, 65, 12, 172, 2, 2, 73, 86, 13, 3, 79, 85, 82, 8, 192, 1, 2, - 79, 82, 65, 2, 73, 78, 8, 218, 1, 78, 237, 197, 11, 3, 82, 67, 72, 2, - 187, 240, 16, 76, 18, 88, 2, 79, 85, 76, 4, 69, 86, 69, 78, 0, 2, 73, 88, - 162, 244, 11, 85, 235, 193, 1, 80, 2, 157, 2, 2, 84, 72, 12, 36, 3, 72, - 82, 69, 13, 2, 87, 79, 6, 11, 69, 6, 25, 4, 32, 79, 70, 32, 6, 46, 67, - 173, 238, 6, 5, 66, 65, 77, 66, 79, 4, 180, 132, 6, 2, 73, 82, 145, 148, - 10, 5, 72, 65, 82, 65, 67, 6, 50, 69, 60, 4, 72, 73, 84, 69, 163, 222, - 16, 73, 2, 17, 2, 83, 84, 2, 17, 2, 32, 87, 2, 171, 135, 17, 73, 2, 17, - 2, 32, 68, 2, 227, 221, 16, 82, 52, 52, 5, 65, 83, 65, 82, 32, 157, 206, - 14, 2, 69, 77, 50, 162, 1, 69, 40, 7, 76, 69, 84, 84, 69, 82, 32, 152, 1, - 5, 80, 65, 83, 83, 73, 32, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, - 32, 173, 136, 18, 3, 65, 78, 71, 2, 213, 159, 15, 5, 78, 68, 32, 79, 70, - 36, 162, 155, 16, 78, 242, 244, 1, 66, 2, 67, 2, 68, 2, 71, 2, 74, 2, 75, - 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 86, 2, 89, 187, 2, 65, 2, - 11, 77, 2, 227, 184, 17, 66, 8, 146, 145, 18, 69, 2, 73, 2, 79, 3, 85, - 246, 1, 80, 7, 65, 89, 65, 76, 65, 77, 32, 208, 11, 2, 69, 32, 225, 1, 3, - 84, 69, 83, 236, 1, 190, 1, 65, 22, 68, 44, 9, 70, 82, 65, 67, 84, 73, - 79, 78, 32, 240, 1, 7, 76, 69, 84, 84, 69, 82, 32, 164, 5, 7, 78, 85, 77, - 66, 69, 82, 32, 36, 5, 83, 73, 71, 78, 32, 143, 144, 13, 86, 2, 227, 142, - 13, 85, 22, 172, 146, 8, 2, 65, 84, 251, 141, 8, 73, 26, 56, 4, 79, 78, - 69, 32, 121, 6, 84, 72, 82, 69, 69, 32, 18, 82, 84, 142, 128, 5, 83, 202, - 219, 7, 70, 58, 79, 170, 176, 3, 69, 46, 72, 43, 81, 4, 254, 222, 12, 87, - 239, 174, 3, 69, 8, 166, 128, 5, 83, 174, 223, 7, 69, 110, 84, 163, 174, - 3, 81, 132, 1, 226, 1, 65, 82, 67, 158, 1, 68, 78, 84, 82, 86, 222, 221, - 12, 78, 230, 46, 76, 210, 90, 82, 130, 81, 85, 158, 144, 1, 79, 182, 56, - 73, 202, 190, 1, 83, 82, 66, 2, 71, 2, 74, 2, 75, 2, 80, 162, 7, 69, 222, - 61, 72, 2, 77, 3, 89, 11, 148, 206, 15, 7, 82, 67, 72, 65, 73, 67, 32, - 158, 188, 2, 65, 2, 73, 3, 85, 22, 26, 72, 203, 137, 18, 65, 20, 44, 5, - 73, 76, 76, 85, 32, 155, 137, 18, 65, 18, 226, 211, 12, 76, 182, 189, 3, - 78, 178, 191, 1, 82, 210, 56, 75, 2, 77, 3, 89, 10, 164, 198, 10, 4, 79, - 84, 32, 82, 238, 250, 6, 68, 254, 68, 72, 187, 2, 65, 10, 38, 84, 158, - 133, 18, 72, 187, 2, 65, 6, 154, 133, 18, 72, 2, 84, 187, 2, 65, 12, 250, - 225, 3, 69, 206, 193, 10, 79, 231, 227, 3, 65, 6, 158, 222, 12, 79, 231, - 215, 3, 84, 18, 50, 67, 114, 86, 194, 158, 14, 65, 179, 162, 3, 80, 6, - 54, 79, 120, 5, 73, 82, 67, 85, 76, 215, 158, 14, 65, 2, 185, 142, 13, 9, - 77, 66, 73, 78, 73, 78, 71, 32, 65, 6, 60, 9, 69, 82, 84, 73, 67, 65, 76, - 32, 66, 255, 193, 17, 73, 2, 17, 2, 65, 82, 2, 233, 163, 12, 2, 32, 86, - 8, 80, 12, 87, 73, 84, 72, 32, 83, 84, 82, 79, 75, 69, 32, 70, 65, 219, - 252, 16, 83, 4, 64, 10, 65, 78, 68, 32, 77, 65, 76, 69, 32, 65, 219, 252, - 16, 83, 2, 25, 4, 78, 68, 32, 70, 2, 11, 69, 2, 11, 77, 2, 199, 190, 7, - 65, 2, 231, 196, 16, 69, 2, 199, 131, 16, 79, 185, 1, 210, 1, 32, 220, 2, - 5, 68, 65, 73, 67, 32, 220, 3, 8, 73, 67, 72, 65, 69, 65, 78, 32, 222, 8, - 83, 200, 165, 2, 3, 85, 65, 76, 166, 181, 10, 65, 220, 168, 1, 8, 84, 69, - 76, 80, 73, 69, 67, 69, 147, 207, 3, 71, 12, 132, 1, 3, 73, 78, 32, 128, - 1, 5, 87, 73, 84, 72, 32, 196, 146, 2, 3, 68, 65, 78, 169, 194, 12, 8, - 65, 78, 68, 32, 87, 79, 77, 65, 4, 208, 240, 9, 19, 66, 85, 83, 73, 78, - 69, 83, 83, 32, 83, 85, 73, 84, 32, 76, 69, 86, 73, 84, 209, 148, 1, 4, - 84, 85, 88, 69, 4, 204, 195, 13, 8, 71, 85, 65, 32, 80, 73, 32, 77, 213, - 243, 2, 4, 84, 85, 82, 66, 58, 136, 1, 7, 76, 69, 84, 84, 69, 82, 32, - 230, 209, 10, 71, 176, 184, 3, 3, 86, 79, 67, 146, 70, 80, 177, 195, 2, - 6, 65, 70, 70, 82, 73, 67, 50, 82, 65, 176, 1, 2, 68, 85, 2, 85, 28, 3, - 72, 65, 76, 22, 73, 167, 180, 17, 75, 38, 154, 1, 75, 150, 198, 12, 84, - 226, 193, 1, 83, 194, 163, 3, 73, 214, 79, 66, 2, 68, 2, 71, 2, 72, 2, - 76, 2, 77, 2, 78, 2, 80, 2, 81, 2, 82, 3, 90, 5, 207, 248, 17, 83, 2, - 253, 239, 7, 2, 83, 72, 2, 159, 248, 17, 81, 4, 194, 250, 17, 78, 3, 84, - 102, 216, 1, 7, 76, 69, 84, 84, 69, 82, 32, 194, 4, 78, 80, 12, 80, 85, - 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 220, 1, 4, 83, 73, 71, 78, 185, - 172, 10, 16, 65, 66, 66, 82, 69, 86, 73, 65, 84, 73, 79, 78, 32, 77, 65, - 82, 72, 218, 1, 65, 46, 66, 38, 68, 30, 71, 30, 74, 2, 90, 30, 75, 30, - 81, 38, 83, 50, 84, 54, 88, 238, 154, 6, 76, 254, 194, 1, 82, 238, 213, - 2, 89, 154, 193, 1, 72, 166, 255, 4, 78, 254, 1, 87, 202, 103, 70, 2, 80, - 171, 4, 77, 6, 206, 180, 10, 76, 122, 65, 239, 193, 6, 89, 4, 238, 200, - 12, 72, 211, 174, 3, 69, 4, 206, 156, 6, 65, 23, 72, 4, 246, 179, 10, 72, - 15, 73, 4, 178, 180, 10, 72, 15, 65, 4, 246, 251, 11, 72, 15, 65, 4, 162, - 156, 6, 72, 199, 223, 5, 79, 8, 250, 176, 10, 83, 154, 3, 65, 191, 193, - 6, 72, 6, 158, 155, 6, 72, 186, 218, 9, 69, 247, 128, 1, 65, 4, 222, 250, - 11, 65, 3, 79, 10, 33, 6, 85, 77, 66, 69, 82, 32, 10, 186, 179, 10, 79, - 58, 84, 219, 183, 2, 70, 14, 80, 2, 68, 79, 116, 3, 76, 73, 78, 138, 159, - 5, 70, 142, 133, 11, 84, 255, 9, 83, 6, 54, 84, 13, 9, 85, 66, 76, 69, - 32, 68, 79, 84, 32, 5, 11, 32, 2, 25, 4, 87, 73, 84, 72, 2, 163, 192, 2, - 73, 2, 223, 201, 3, 69, 2, 159, 199, 16, 32, 2, 11, 32, 2, 221, 218, 17, - 2, 83, 72, 4, 92, 17, 32, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 76, - 73, 71, 72, 84, 175, 191, 10, 76, 2, 175, 193, 14, 72, 142, 1, 142, 1, - 65, 20, 5, 67, 72, 69, 78, 32, 160, 155, 6, 4, 82, 73, 65, 71, 165, 213, - 10, 13, 84, 73, 65, 76, 32, 65, 82, 84, 83, 32, 85, 78, 73, 2, 175, 153, - 5, 67, 136, 1, 152, 1, 7, 76, 69, 84, 84, 69, 82, 32, 194, 1, 83, 236, 2, - 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 154, 157, 17, 72, 225, 5, - 4, 77, 65, 82, 75, 60, 254, 3, 84, 146, 148, 2, 68, 178, 222, 13, 78, - 246, 175, 1, 67, 2, 75, 2, 80, 2, 83, 2, 90, 254, 68, 45, 2, 66, 2, 71, - 2, 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, 87, 2, 89, 187, 2, 65, 62, 96, 4, - 73, 71, 78, 32, 37, 16, 85, 66, 74, 79, 73, 78, 69, 68, 32, 76, 69, 84, - 84, 69, 82, 32, 4, 154, 133, 14, 67, 191, 161, 3, 65, 58, 182, 1, 84, - 146, 148, 2, 68, 178, 222, 13, 78, 246, 175, 1, 67, 2, 75, 2, 80, 2, 83, - 2, 90, 254, 68, 66, 2, 71, 2, 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, 87, 2, - 89, 187, 2, 65, 8, 178, 162, 17, 83, 254, 68, 72, 187, 2, 65, 10, 130, - 231, 17, 65, 186, 2, 69, 2, 73, 2, 79, 3, 85, 156, 1, 120, 11, 65, 82, - 65, 77, 32, 71, 79, 78, 68, 73, 32, 236, 5, 3, 67, 85, 76, 64, 5, 75, 32, - 87, 79, 82, 163, 160, 17, 85, 150, 1, 100, 7, 76, 69, 84, 84, 69, 82, 32, - 178, 2, 82, 56, 5, 83, 73, 71, 78, 32, 82, 86, 199, 245, 15, 68, 94, 210, - 1, 74, 42, 84, 206, 252, 13, 65, 38, 68, 146, 25, 85, 210, 200, 1, 73, - 42, 76, 210, 189, 1, 75, 38, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 80, - 254, 68, 72, 2, 77, 2, 82, 2, 86, 2, 89, 186, 2, 69, 3, 79, 6, 230, 226, - 17, 78, 38, 72, 187, 2, 65, 10, 230, 157, 17, 84, 254, 68, 72, 2, 82, - 187, 2, 65, 4, 32, 2, 65, 45, 163, 240, 1, 69, 2, 131, 160, 17, 75, 10, - 174, 176, 1, 67, 202, 160, 12, 72, 242, 45, 78, 130, 182, 1, 86, 223, - 234, 1, 65, 22, 26, 79, 207, 177, 15, 73, 20, 45, 9, 87, 69, 76, 32, 83, - 73, 71, 78, 32, 20, 78, 86, 238, 253, 13, 65, 190, 21, 85, 210, 200, 1, - 73, 206, 134, 2, 69, 3, 79, 2, 201, 182, 7, 6, 79, 67, 65, 76, 73, 67, 2, - 253, 168, 17, 11, 73, 78, 69, 32, 79, 82, 68, 73, 78, 65, 76, 2, 199, - 206, 16, 75, 226, 15, 76, 10, 72, 69, 77, 65, 84, 73, 67, 65, 76, 32, - 129, 162, 14, 3, 69, 32, 68, 224, 15, 252, 1, 5, 66, 79, 76, 68, 32, 168, - 6, 14, 68, 79, 85, 66, 76, 69, 45, 83, 84, 82, 85, 67, 75, 32, 238, 1, - 70, 164, 2, 7, 73, 84, 65, 76, 73, 67, 32, 180, 3, 10, 77, 79, 78, 79, - 83, 80, 65, 67, 69, 32, 40, 2, 82, 73, 36, 3, 76, 69, 70, 167, 1, 83, - 160, 5, 176, 1, 8, 67, 65, 80, 73, 84, 65, 76, 32, 130, 2, 83, 210, 14, - 73, 222, 3, 69, 38, 75, 38, 78, 30, 80, 98, 82, 242, 5, 84, 56, 7, 70, - 82, 65, 75, 84, 85, 82, 131, 211, 15, 68, 104, 190, 4, 68, 174, 15, 84, - 186, 4, 65, 22, 66, 2, 90, 42, 69, 98, 71, 22, 73, 22, 75, 34, 76, 22, - 79, 50, 80, 42, 82, 22, 83, 70, 85, 250, 196, 1, 67, 214, 173, 12, 77, 2, - 78, 210, 200, 1, 88, 206, 134, 2, 70, 2, 72, 2, 74, 2, 81, 2, 86, 2, 87, - 3, 89, 208, 1, 60, 5, 77, 65, 76, 76, 32, 213, 25, 5, 67, 82, 73, 80, 84, - 104, 250, 1, 68, 230, 19, 65, 22, 66, 2, 90, 42, 69, 38, 70, 62, 71, 22, - 73, 22, 75, 34, 76, 22, 79, 50, 80, 42, 82, 22, 83, 34, 84, 38, 85, 250, - 196, 1, 67, 214, 173, 12, 77, 2, 78, 210, 200, 1, 88, 206, 134, 2, 72, 2, - 74, 2, 81, 2, 86, 2, 87, 3, 89, 7, 26, 73, 191, 180, 14, 69, 2, 167, 213, - 1, 71, 110, 68, 8, 67, 65, 80, 73, 84, 65, 76, 32, 162, 23, 83, 195, 210, - 15, 68, 38, 250, 215, 17, 65, 2, 66, 2, 68, 2, 69, 2, 70, 2, 71, 2, 73, - 2, 74, 2, 75, 2, 76, 2, 77, 2, 79, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, - 88, 3, 89, 96, 52, 7, 82, 65, 75, 84, 85, 82, 32, 139, 252, 6, 65, 94, - 52, 8, 67, 65, 80, 73, 84, 65, 76, 32, 143, 21, 83, 42, 230, 213, 17, 65, - 2, 66, 2, 68, 2, 69, 2, 70, 2, 71, 2, 74, 2, 75, 2, 76, 2, 77, 2, 78, 2, - 79, 2, 80, 2, 81, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, 3, 89, 222, - 1, 100, 6, 83, 77, 65, 76, 76, 32, 222, 7, 67, 230, 2, 69, 38, 75, 38, - 78, 30, 80, 98, 82, 243, 5, 84, 104, 242, 1, 68, 198, 12, 65, 22, 66, 2, + 114, 53, 98, 54, 58, 55, 78, 56, 62, 57, 192, 133, 7, 3, 50, 32, 87, 138, + 88, 48, 225, 156, 11, 4, 52, 32, 68, 69, 6, 168, 132, 7, 2, 32, 69, 196, + 149, 11, 3, 70, 32, 77, 129, 80, 7, 77, 32, 83, 84, 65, 76, 76, 4, 152, + 240, 1, 3, 70, 32, 69, 177, 142, 16, 2, 77, 32, 4, 36, 3, 70, 32, 83, 1, + 2, 77, 32, 2, 145, 225, 11, 4, 72, 69, 45, 71, 4, 188, 152, 9, 3, 77, 32, + 66, 237, 133, 9, 3, 70, 32, 83, 4, 208, 176, 17, 4, 77, 32, 66, 85, 157, + 109, 3, 70, 32, 67, 10, 196, 145, 9, 4, 51, 32, 83, 80, 240, 10, 5, 49, + 32, 66, 65, 82, 220, 231, 4, 4, 50, 32, 79, 76, 20, 6, 53, 32, 67, 89, + 80, 69, 245, 98, 4, 48, 32, 87, 72, 6, 54, 49, 172, 130, 17, 3, 48, 32, + 79, 215, 179, 2, 50, 2, 197, 156, 18, 2, 32, 87, 10, 224, 5, 3, 53, 32, + 87, 204, 143, 9, 6, 48, 32, 66, 82, 79, 78, 172, 2, 4, 49, 32, 71, 79, + 138, 158, 10, 50, 3, 54, 16, 82, 57, 214, 232, 2, 49, 198, 203, 16, 48, + 2, 50, 2, 51, 2, 52, 2, 55, 3, 56, 2, 169, 97, 3, 32, 67, 76, 20, 248, + 160, 14, 5, 51, 32, 65, 82, 77, 204, 207, 4, 5, 50, 32, 71, 65, 82, 182, + 67, 48, 2, 49, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 134, 242, + 13, 54, 200, 190, 3, 4, 51, 32, 77, 79, 158, 130, 2, 48, 2, 49, 2, 50, 2, + 52, 2, 55, 2, 56, 3, 57, 14, 246, 177, 19, 48, 2, 49, 2, 50, 2, 51, 2, + 52, 2, 53, 3, 57, 4, 164, 164, 2, 3, 49, 32, 72, 143, 141, 17, 48, 50, + 42, 50, 110, 51, 130, 1, 52, 227, 1, 53, 4, 84, 8, 48, 32, 70, 79, 79, + 84, 83, 84, 169, 210, 6, 7, 53, 32, 66, 65, 84, 72, 84, 2, 143, 157, 18, + 79, 12, 104, 4, 51, 32, 83, 87, 156, 255, 13, 4, 48, 32, 83, 80, 146, + 149, 4, 49, 214, 154, 1, 50, 2, 52, 3, 54, 2, 147, 152, 18, 79, 16, 168, + 1, 9, 48, 32, 87, 72, 69, 69, 76, 69, 68, 2, 49, 34, 51, 168, 158, 17, + 12, 50, 32, 67, 72, 65, 82, 73, 79, 84, 32, 70, 82, 250, 142, 2, 53, 2, + 54, 2, 56, 3, 57, 2, 185, 163, 18, 3, 32, 67, 72, 2, 247, 190, 6, 32, 18, + 236, 165, 18, 3, 52, 32, 68, 158, 135, 1, 49, 2, 50, 2, 51, 2, 53, 2, 54, + 2, 55, 2, 56, 3, 57, 58, 50, 50, 192, 144, 12, 2, 49, 53, 1, 2, 51, 48, + 54, 50, 50, 238, 146, 12, 53, 150, 227, 1, 48, 3, 49, 12, 186, 171, 19, + 49, 2, 50, 2, 54, 2, 55, 2, 56, 3, 57, 12, 46, 49, 153, 12, 6, 50, 52, + 55, 32, 68, 73, 10, 50, 50, 70, 51, 181, 11, 5, 53, 54, 32, 84, 85, 4, + 208, 174, 3, 3, 55, 32, 75, 189, 220, 15, 5, 56, 32, 75, 65, 78, 4, 56, + 3, 53, 32, 77, 201, 239, 15, 5, 51, 32, 65, 82, 69, 2, 207, 203, 12, 69, + 176, 1, 84, 9, 76, 76, 65, 66, 76, 69, 32, 66, 48, 229, 12, 7, 77, 66, + 79, 76, 32, 66, 48, 148, 1, 114, 48, 142, 1, 49, 162, 1, 50, 230, 1, 51, + 174, 1, 52, 162, 1, 53, 154, 1, 54, 186, 1, 55, 198, 1, 56, 87, 57, 18, + 118, 54, 150, 198, 1, 55, 150, 8, 52, 190, 14, 57, 194, 246, 10, 53, 250, + 146, 2, 56, 198, 10, 49, 254, 2, 50, 207, 8, 51, 2, 183, 234, 17, 32, 16, + 122, 54, 18, 55, 190, 207, 1, 49, 134, 2, 50, 166, 26, 52, 248, 186, 1, + 2, 53, 32, 250, 242, 5, 48, 221, 243, 8, 2, 51, 32, 2, 183, 90, 32, 2, + 143, 133, 11, 32, 18, 166, 1, 51, 22, 54, 20, 3, 57, 32, 80, 224, 6, 2, + 53, 32, 220, 174, 3, 2, 48, 32, 196, 205, 14, 2, 55, 32, 140, 7, 2, 52, + 32, 158, 89, 56, 185, 44, 3, 49, 32, 81, 2, 215, 164, 14, 32, 2, 171, + 234, 14, 32, 2, 139, 138, 12, 85, 16, 134, 1, 48, 20, 2, 51, 32, 154, + 195, 1, 55, 254, 34, 54, 158, 241, 4, 57, 234, 163, 2, 56, 250, 168, 4, + 49, 133, 223, 5, 3, 50, 32, 81, 2, 151, 166, 14, 32, 2, 135, 1, 82, 16, + 116, 2, 51, 32, 22, 53, 142, 191, 1, 48, 222, 1, 49, 134, 6, 50, 218, 10, + 52, 178, 5, 54, 241, 216, 12, 3, 56, 32, 78, 2, 247, 170, 15, 65, 2, 223, + 144, 17, 32, 18, 130, 1, 51, 190, 191, 1, 49, 150, 1, 56, 42, 57, 102, + 55, 214, 5, 48, 194, 141, 2, 52, 212, 161, 15, 2, 50, 32, 157, 4, 2, 53, + 32, 2, 135, 194, 12, 32, 16, 132, 1, 2, 50, 32, 20, 2, 56, 32, 204, 1, 3, + 54, 32, 84, 178, 186, 1, 55, 226, 3, 57, 146, 2, 53, 206, 247, 6, 49, + 151, 154, 10, 48, 2, 191, 215, 17, 80, 2, 245, 132, 12, 2, 82, 79, 18, + 172, 1, 3, 54, 32, 82, 210, 186, 1, 55, 158, 8, 48, 170, 16, 53, 12, 3, + 49, 32, 68, 132, 169, 7, 2, 52, 32, 250, 249, 3, 50, 232, 143, 6, 3, 56, + 32, 81, 241, 2, 2, 51, 32, 2, 171, 131, 12, 65, 8, 206, 186, 1, 49, 188, + 24, 3, 55, 32, 84, 192, 249, 6, 2, 53, 32, 183, 146, 6, 48, 4, 154, 228, + 12, 49, 21, 3, 48, 32, 68, 28, 90, 52, 30, 54, 30, 56, 174, 253, 11, 53, + 158, 2, 49, 30, 51, 166, 1, 50, 235, 163, 3, 55, 4, 234, 154, 19, 55, 3, + 57, 4, 206, 154, 19, 51, 3, 52, 8, 178, 154, 19, 50, 2, 51, 2, 54, 3, 57, + 4, 136, 202, 10, 9, 69, 68, 32, 80, 65, 80, 69, 82, 67, 227, 188, 7, 32, + 5, 195, 233, 17, 84, 98, 96, 7, 76, 69, 84, 84, 69, 82, 32, 169, 234, 6, + 11, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 94, 238, 1, 84, 230, 240, + 2, 68, 178, 159, 10, 85, 150, 149, 2, 78, 138, 30, 69, 234, 139, 3, 67, + 2, 71, 2, 72, 2, 75, 2, 80, 2, 83, 2, 89, 2, 90, 162, 7, 65, 2, 79, 234, + 61, 66, 2, 70, 2, 74, 2, 76, 2, 77, 2, 87, 2, 88, 187, 2, 73, 20, 64, 4, + 79, 78, 69, 32, 214, 206, 18, 83, 138, 69, 72, 187, 2, 65, 12, 48, 4, 77, + 89, 65, 32, 129, 192, 1, 2, 78, 65, 10, 130, 183, 12, 74, 234, 191, 6, + 66, 158, 11, 84, 254, 16, 67, 39, 78, 196, 2, 232, 1, 2, 67, 75, 132, 1, + 3, 71, 73, 67, 176, 6, 3, 78, 71, 32, 222, 4, 84, 116, 3, 86, 69, 32, 70, + 87, 172, 12, 5, 90, 69, 78, 71, 69, 134, 231, 7, 66, 148, 138, 10, 8, 85, + 68, 76, 89, 32, 67, 82, 89, 141, 15, 4, 76, 76, 73, 80, 9, 96, 10, 73, + 78, 71, 45, 83, 72, 73, 70, 84, 32, 153, 20, 9, 32, 87, 73, 84, 72, 32, + 73, 78, 75, 4, 162, 163, 17, 90, 251, 85, 79, 40, 56, 6, 32, 71, 65, 84, + 69, 32, 137, 2, 3, 65, 76, 32, 12, 104, 6, 66, 85, 70, 70, 69, 82, 84, 9, + 73, 78, 86, 69, 82, 84, 69, 68, 32, 182, 130, 18, 65, 159, 85, 79, 5, + 133, 1, 17, 32, 87, 73, 84, 72, 32, 73, 78, 86, 69, 82, 84, 69, 68, 32, + 73, 78, 4, 48, 3, 79, 85, 84, 129, 154, 6, 3, 73, 78, 80, 2, 167, 150, + 18, 80, 28, 36, 3, 65, 78, 68, 85, 2, 79, 82, 15, 33, 6, 32, 87, 73, 84, + 72, 32, 12, 226, 1, 68, 94, 72, 58, 77, 151, 161, 15, 85, 15, 11, 32, 12, + 88, 13, 79, 86, 69, 82, 76, 65, 80, 80, 73, 78, 71, 32, 76, 49, 5, 87, + 73, 84, 72, 32, 2, 249, 255, 17, 7, 79, 71, 73, 67, 65, 76, 32, 10, 26, + 68, 94, 72, 59, 77, 6, 11, 79, 6, 44, 5, 85, 66, 76, 69, 32, 247, 143, + 18, 84, 4, 234, 161, 15, 85, 235, 60, 79, 2, 177, 207, 15, 9, 79, 82, 73, + 90, 79, 78, 84, 65, 76, 2, 225, 218, 7, 5, 73, 68, 68, 76, 69, 34, 66, + 68, 228, 1, 4, 76, 69, 70, 84, 109, 5, 82, 73, 71, 72, 84, 6, 164, 1, 30, + 65, 83, 72, 32, 70, 82, 79, 77, 32, 76, 69, 70, 84, 32, 77, 69, 77, 66, + 69, 82, 32, 79, 70, 32, 68, 79, 85, 66, 76, 69, 166, 144, 16, 73, 219, + 208, 1, 82, 2, 17, 2, 32, 86, 2, 181, 128, 18, 5, 69, 82, 84, 73, 67, 16, + 18, 32, 119, 87, 6, 48, 6, 82, 73, 71, 72, 84, 32, 139, 238, 15, 84, 4, + 246, 227, 15, 68, 215, 138, 2, 65, 12, 26, 87, 139, 244, 8, 32, 10, 29, + 5, 65, 82, 68, 83, 32, 10, 82, 65, 0, 8, 68, 79, 85, 66, 76, 69, 32, 65, + 221, 227, 7, 4, 83, 81, 85, 73, 4, 25, 4, 82, 82, 79, 87, 5, 137, 162, + 16, 2, 32, 70, 6, 92, 5, 73, 79, 78, 32, 66, 144, 237, 17, 9, 32, 79, 70, + 32, 70, 79, 82, 84, 85, 211, 91, 85, 2, 211, 235, 9, 79, 4, 38, 76, 157, + 249, 13, 3, 72, 79, 84, 2, 133, 206, 17, 2, 69, 84, 222, 1, 34, 32, 229, + 1, 3, 69, 82, 32, 14, 138, 1, 66, 196, 195, 10, 11, 68, 79, 85, 66, 76, + 69, 32, 80, 82, 73, 77, 154, 173, 6, 65, 204, 111, 6, 75, 65, 86, 89, 75, + 65, 211, 10, 76, 4, 38, 82, 249, 248, 4, 3, 65, 84, 84, 2, 209, 240, 17, + 7, 73, 71, 72, 84, 78, 69, 83, 208, 1, 158, 1, 72, 140, 2, 5, 76, 69, 70, + 84, 32, 236, 2, 6, 82, 73, 71, 72, 84, 32, 234, 230, 15, 66, 74, 67, 74, + 70, 234, 10, 77, 150, 2, 79, 166, 18, 83, 67, 84, 20, 84, 4, 65, 76, 70, + 32, 241, 184, 7, 11, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 82, 18, 212, + 164, 7, 9, 65, 78, 68, 32, 85, 80, 80, 69, 82, 32, 7, 73, 78, 86, 69, 82, + 83, 69, 194, 201, 8, 72, 230, 1, 86, 170, 26, 77, 130, 3, 76, 22, 82, + 202, 5, 66, 195, 156, 1, 67, 72, 186, 1, 66, 60, 8, 70, 79, 85, 78, 84, + 65, 73, 78, 22, 80, 56, 12, 83, 69, 77, 73, 67, 73, 82, 67, 85, 76, 65, + 82, 154, 1, 81, 246, 2, 84, 248, 191, 11, 3, 67, 82, 65, 139, 177, 4, 79, + 24, 56, 8, 65, 76, 76, 80, 79, 73, 78, 84, 255, 246, 15, 76, 2, 179, 240, + 9, 32, 4, 212, 243, 14, 5, 65, 73, 78, 84, 66, 163, 140, 1, 69, 2, 221, + 193, 7, 5, 32, 65, 78, 84, 73, 74, 110, 81, 166, 2, 83, 82, 84, 238, 176, + 7, 82, 202, 184, 8, 66, 238, 3, 67, 226, 3, 79, 222, 7, 68, 207, 2, 80, + 30, 17, 2, 85, 65, 30, 48, 6, 68, 82, 65, 78, 84, 32, 187, 130, 16, 82, + 28, 74, 70, 60, 2, 78, 69, 50, 83, 226, 253, 15, 67, 226, 1, 77, 143, 1, + 84, 4, 26, 65, 255, 180, 17, 82, 2, 185, 183, 7, 3, 67, 69, 32, 2, 25, 4, + 85, 84, 82, 65, 2, 191, 189, 18, 76, 4, 254, 218, 10, 77, 143, 165, 5, + 84, 6, 148, 190, 7, 11, 69, 77, 73, 67, 73, 82, 67, 85, 76, 65, 82, 231, + 194, 8, 72, 6, 226, 129, 16, 82, 155, 1, 87, 5, 141, 213, 17, 25, 32, 68, + 73, 86, 73, 68, 69, 68, 32, 66, 89, 32, 72, 79, 82, 73, 90, 79, 78, 84, + 65, 76, 32, 82, 85, 6, 18, 71, 23, 78, 2, 215, 241, 13, 71, 4, 188, 213, + 17, 5, 65, 82, 32, 69, 67, 255, 100, 71, 114, 104, 12, 67, 73, 65, 78, + 32, 76, 69, 84, 84, 69, 82, 32, 228, 1, 5, 68, 73, 65, 78, 32, 131, 237, + 17, 73, 58, 206, 1, 77, 194, 137, 5, 75, 206, 203, 9, 66, 50, 84, 234, + 164, 2, 65, 2, 69, 2, 78, 134, 251, 1, 68, 2, 71, 2, 72, 2, 73, 2, 74, 2, + 76, 2, 80, 2, 81, 2, 82, 2, 83, 2, 85, 2, 87, 2, 88, 3, 90, 5, 167, 245, + 18, 77, 54, 88, 7, 76, 69, 84, 84, 69, 82, 32, 189, 213, 14, 9, 84, 82, + 73, 65, 78, 71, 85, 76, 65, 52, 214, 148, 13, 84, 174, 192, 1, 76, 198, + 152, 2, 83, 238, 11, 65, 2, 69, 2, 78, 134, 251, 1, 66, 2, 67, 2, 68, 2, + 70, 2, 71, 2, 73, 2, 75, 2, 77, 2, 79, 2, 81, 2, 82, 2, 85, 2, 86, 3, 89, + 248, 53, 162, 1, 65, 206, 103, 69, 246, 102, 73, 154, 25, 79, 148, 112, + 3, 82, 79, 32, 234, 3, 85, 192, 70, 7, 89, 65, 78, 77, 65, 82, 32, 218, + 145, 15, 86, 198, 61, 77, 27, 87, 204, 23, 186, 1, 71, 62, 72, 254, 10, + 75, 234, 3, 76, 252, 13, 2, 77, 77, 22, 78, 154, 17, 80, 118, 82, 174, 7, + 83, 190, 7, 84, 196, 36, 3, 89, 65, 78, 140, 129, 1, 3, 88, 73, 77, 247, + 230, 15, 67, 6, 128, 226, 17, 4, 73, 67, 32, 87, 238, 88, 78, 155, 53, + 69, 166, 1, 84, 6, 65, 74, 65, 78, 73, 32, 129, 3, 10, 74, 79, 78, 71, + 32, 84, 73, 76, 69, 32, 78, 38, 76, 162, 2, 83, 207, 129, 13, 65, 72, 88, + 6, 69, 84, 84, 69, 82, 32, 161, 144, 12, 10, 73, 71, 65, 84, 85, 82, 69, + 32, 83, 72, 70, 182, 174, 3, 78, 186, 229, 11, 68, 82, 82, 34, 84, 206, + 145, 3, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, + 2, 83, 2, 86, 186, 2, 65, 2, 69, 2, 73, 2, 79, 3, 85, 4, 226, 235, 6, 69, + 229, 236, 7, 5, 73, 71, 78, 32, 78, 88, 236, 1, 2, 66, 65, 38, 69, 46, + 70, 46, 78, 42, 79, 46, 80, 22, 83, 118, 84, 194, 1, 87, 112, 5, 71, 82, + 69, 69, 78, 0, 3, 82, 69, 68, 208, 135, 6, 3, 65, 85, 84, 214, 21, 74, + 197, 158, 11, 11, 67, 72, 82, 89, 83, 65, 78, 84, 72, 69, 77, 4, 138, + 158, 1, 77, 223, 200, 17, 67, 8, 228, 2, 4, 73, 71, 72, 84, 195, 1, 65, + 12, 172, 2, 2, 73, 86, 13, 3, 79, 85, 82, 8, 192, 1, 2, 79, 82, 65, 2, + 73, 78, 8, 218, 1, 78, 189, 223, 11, 3, 82, 67, 72, 2, 159, 191, 17, 76, + 18, 88, 2, 79, 85, 76, 4, 69, 86, 69, 78, 0, 2, 73, 88, 198, 151, 12, 85, + 187, 194, 1, 80, 2, 157, 2, 2, 84, 72, 12, 36, 3, 72, 82, 69, 13, 2, 87, + 79, 6, 11, 69, 6, 25, 4, 32, 79, 70, 32, 6, 46, 67, 229, 246, 6, 5, 66, + 65, 77, 66, 79, 4, 224, 139, 6, 2, 73, 82, 217, 217, 10, 5, 72, 65, 82, + 65, 67, 6, 50, 69, 60, 4, 72, 73, 84, 69, 247, 172, 17, 73, 2, 17, 2, 83, + 84, 2, 17, 2, 32, 87, 2, 147, 215, 17, 73, 2, 17, 2, 32, 68, 2, 183, 172, + 17, 82, 52, 52, 5, 65, 83, 65, 82, 32, 233, 251, 14, 2, 69, 77, 50, 162, + 1, 69, 40, 7, 76, 69, 84, 84, 69, 82, 32, 152, 1, 5, 80, 65, 83, 83, 73, + 32, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 173, 216, 18, 3, 65, + 78, 71, 2, 161, 228, 15, 5, 78, 68, 32, 79, 70, 36, 158, 232, 16, 78, + 246, 247, 1, 66, 2, 67, 2, 68, 2, 71, 2, 74, 2, 75, 2, 76, 2, 77, 2, 80, + 2, 82, 2, 83, 2, 84, 2, 86, 2, 89, 187, 2, 65, 2, 11, 77, 2, 211, 136, + 18, 66, 8, 146, 225, 18, 69, 2, 73, 2, 79, 3, 85, 246, 1, 80, 7, 65, 89, + 65, 76, 65, 77, 32, 176, 11, 2, 69, 32, 225, 1, 3, 84, 69, 83, 236, 1, + 198, 1, 68, 44, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 240, 1, 7, 76, 69, + 84, 84, 69, 82, 32, 164, 5, 7, 78, 85, 77, 66, 69, 82, 32, 36, 5, 83, 73, + 71, 78, 32, 178, 180, 13, 86, 247, 196, 1, 65, 22, 204, 169, 8, 2, 65, + 84, 227, 195, 8, 73, 26, 56, 4, 79, 78, 69, 32, 121, 6, 84, 72, 82, 69, + 69, 32, 18, 82, 84, 146, 135, 5, 83, 230, 249, 7, 70, 62, 79, 130, 216, + 3, 69, 46, 72, 47, 81, 4, 254, 131, 13, 87, 239, 214, 3, 69, 8, 170, 135, + 5, 83, 170, 253, 7, 69, 110, 84, 163, 214, 3, 81, 132, 1, 226, 1, 65, 82, + 67, 158, 1, 68, 78, 84, 82, 86, 218, 130, 13, 78, 146, 251, 1, 76, 38, + 82, 134, 6, 85, 202, 141, 1, 79, 134, 60, 73, 206, 193, 1, 83, 82, 66, 2, + 71, 2, 74, 2, 75, 2, 80, 162, 7, 69, 234, 61, 72, 2, 77, 3, 89, 11, 192, + 151, 16, 7, 82, 67, 72, 65, 73, 67, 32, 254, 194, 2, 65, 2, 73, 3, 85, + 22, 26, 72, 215, 217, 18, 65, 20, 44, 5, 73, 76, 76, 85, 32, 167, 217, + 18, 65, 18, 130, 249, 12, 76, 158, 229, 3, 78, 170, 194, 1, 82, 222, 56, + 75, 2, 77, 3, 89, 10, 220, 220, 10, 4, 79, 84, 32, 82, 182, 180, 7, 68, + 138, 69, 72, 187, 2, 65, 10, 38, 84, 170, 213, 18, 72, 187, 2, 65, 6, + 166, 213, 18, 72, 2, 84, 187, 2, 65, 12, 166, 227, 3, 69, 138, 161, 11, + 79, 139, 211, 3, 65, 6, 154, 131, 13, 79, 131, 128, 4, 84, 18, 50, 67, + 114, 86, 206, 254, 14, 65, 167, 146, 3, 80, 6, 54, 79, 120, 5, 73, 82, + 67, 85, 76, 203, 192, 14, 65, 2, 221, 178, 13, 9, 77, 66, 73, 78, 73, 78, + 71, 32, 65, 6, 60, 9, 69, 82, 84, 73, 67, 65, 76, 32, 66, 255, 145, 18, + 73, 2, 209, 254, 14, 2, 65, 82, 8, 80, 12, 87, 73, 84, 72, 32, 83, 84, + 82, 79, 75, 69, 32, 70, 65, 227, 204, 17, 83, 4, 64, 10, 65, 78, 68, 32, + 77, 65, 76, 69, 32, 65, 227, 204, 17, 83, 2, 25, 4, 78, 68, 32, 70, 2, + 11, 69, 2, 11, 77, 2, 231, 209, 7, 65, 2, 211, 145, 17, 69, 2, 219, 208, + 16, 79, 185, 1, 210, 1, 32, 220, 2, 5, 68, 65, 73, 67, 32, 224, 3, 8, 73, + 67, 72, 65, 69, 65, 78, 32, 222, 8, 83, 216, 165, 2, 3, 85, 65, 76, 190, + 217, 10, 65, 152, 168, 1, 8, 84, 69, 76, 80, 73, 69, 67, 69, 203, 251, 3, + 71, 12, 132, 1, 3, 73, 78, 32, 128, 1, 5, 87, 73, 84, 72, 32, 204, 146, + 2, 3, 68, 65, 78, 253, 252, 12, 8, 65, 78, 68, 32, 87, 79, 77, 65, 4, + 144, 135, 10, 19, 66, 85, 83, 73, 78, 69, 83, 83, 32, 83, 85, 73, 84, 32, + 76, 69, 86, 73, 84, 185, 151, 1, 4, 84, 85, 88, 69, 4, 224, 231, 13, 8, + 71, 85, 65, 32, 80, 73, 32, 77, 149, 157, 3, 4, 84, 85, 82, 66, 58, 112, + 4, 65, 70, 70, 82, 28, 7, 76, 69, 84, 84, 69, 82, 32, 244, 173, 14, 3, + 86, 79, 67, 146, 67, 71, 251, 25, 80, 2, 249, 227, 17, 2, 73, 67, 50, 82, + 65, 176, 1, 2, 68, 85, 2, 85, 28, 3, 72, 65, 76, 22, 73, 183, 132, 18, + 75, 38, 154, 1, 75, 198, 235, 12, 84, 146, 192, 1, 83, 242, 207, 3, 73, + 226, 79, 66, 2, 68, 2, 71, 2, 72, 2, 76, 2, 77, 2, 78, 2, 80, 2, 81, 2, + 82, 3, 90, 5, 235, 200, 18, 83, 2, 153, 131, 8, 2, 83, 72, 2, 187, 200, + 18, 81, 4, 222, 202, 18, 78, 3, 84, 102, 216, 1, 7, 76, 69, 84, 84, 69, + 82, 32, 194, 4, 78, 80, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, + 32, 220, 1, 4, 83, 73, 71, 78, 129, 195, 10, 16, 65, 66, 66, 82, 69, 86, + 73, 65, 84, 73, 79, 78, 32, 77, 65, 82, 72, 218, 1, 65, 46, 66, 38, 68, + 30, 71, 30, 74, 2, 90, 30, 75, 30, 81, 38, 83, 50, 84, 54, 88, 234, 162, + 6, 76, 158, 206, 1, 82, 154, 217, 2, 89, 158, 208, 1, 72, 222, 169, 5, + 78, 134, 2, 87, 218, 103, 70, 2, 80, 171, 4, 77, 6, 150, 203, 10, 76, + 122, 65, 171, 251, 6, 89, 4, 254, 237, 12, 72, 211, 214, 3, 69, 4, 202, + 164, 6, 65, 23, 72, 4, 190, 202, 10, 72, 15, 73, 4, 250, 202, 10, 72, 15, + 65, 4, 194, 161, 12, 72, 15, 65, 4, 158, 164, 6, 72, 151, 253, 5, 79, 8, + 194, 199, 10, 83, 154, 3, 65, 251, 250, 6, 72, 6, 154, 163, 6, 72, 206, + 159, 10, 69, 243, 131, 1, 65, 4, 170, 160, 12, 65, 3, 79, 10, 33, 6, 85, + 77, 66, 69, 82, 32, 10, 130, 202, 10, 79, 58, 84, 135, 198, 2, 70, 14, + 80, 2, 68, 79, 116, 3, 76, 73, 78, 134, 167, 5, 70, 186, 202, 11, 84, + 167, 10, 83, 6, 54, 84, 13, 9, 85, 66, 76, 69, 32, 68, 79, 84, 32, 5, 11, + 32, 2, 25, 4, 87, 73, 84, 72, 2, 199, 192, 2, 73, 2, 155, 203, 3, 69, 2, + 159, 150, 17, 32, 2, 11, 32, 2, 249, 170, 18, 2, 83, 72, 4, 92, 17, 32, + 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 76, 73, 71, 72, 84, 147, 214, + 10, 76, 2, 135, 252, 14, 72, 142, 1, 142, 1, 65, 20, 5, 67, 72, 69, 78, + 32, 144, 163, 6, 4, 82, 73, 65, 71, 193, 157, 11, 13, 84, 73, 65, 76, 32, + 65, 82, 84, 83, 32, 85, 78, 73, 2, 171, 161, 5, 67, 136, 1, 152, 1, 7, + 76, 69, 84, 84, 69, 82, 32, 194, 1, 83, 236, 2, 11, 86, 79, 87, 69, 76, + 32, 83, 73, 71, 78, 32, 170, 237, 17, 72, 225, 5, 4, 77, 65, 82, 75, 60, + 254, 3, 84, 146, 148, 2, 68, 202, 171, 14, 78, 238, 178, 1, 67, 2, 75, 2, + 80, 2, 83, 2, 90, 138, 69, 45, 2, 66, 2, 71, 2, 72, 2, 74, 2, 76, 2, 77, + 2, 82, 2, 87, 2, 89, 187, 2, 65, 62, 96, 4, 73, 71, 78, 32, 37, 16, 85, + 66, 74, 79, 73, 78, 69, 68, 32, 76, 69, 84, 84, 69, 82, 32, 4, 158, 167, + 14, 67, 203, 207, 3, 65, 58, 182, 1, 84, 146, 148, 2, 68, 202, 171, 14, + 78, 238, 178, 1, 67, 2, 75, 2, 80, 2, 83, 2, 90, 138, 69, 66, 2, 71, 2, + 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, 87, 2, 89, 187, 2, 65, 8, 194, 242, + 17, 83, 138, 69, 72, 187, 2, 65, 10, 158, 183, 18, 65, 186, 2, 69, 2, 73, + 2, 79, 3, 85, 156, 1, 120, 11, 65, 82, 65, 77, 32, 71, 79, 78, 68, 73, + 32, 232, 5, 3, 67, 85, 76, 64, 5, 75, 32, 87, 79, 82, 183, 240, 17, 85, + 150, 1, 100, 7, 76, 69, 84, 84, 69, 82, 32, 178, 2, 82, 56, 5, 83, 73, + 71, 78, 32, 82, 86, 223, 194, 16, 68, 94, 210, 1, 74, 42, 84, 154, 219, + 14, 65, 38, 68, 214, 6, 85, 206, 201, 1, 73, 42, 76, 214, 192, 1, 75, 38, + 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 80, 138, 69, 72, 2, 77, 2, 82, 2, + 86, 2, 89, 186, 2, 69, 3, 79, 6, 130, 179, 18, 78, 38, 72, 187, 2, 65, + 10, 246, 237, 17, 84, 138, 69, 72, 2, 82, 187, 2, 65, 4, 32, 2, 65, 45, + 175, 220, 14, 69, 2, 147, 240, 17, 75, 10, 178, 176, 1, 67, 198, 196, 12, + 72, 246, 43, 78, 186, 221, 1, 86, 179, 241, 1, 65, 22, 26, 79, 139, 251, + 15, 73, 20, 45, 9, 87, 69, 76, 32, 83, 73, 71, 78, 32, 20, 74, 86, 154, + 223, 14, 65, 38, 85, 206, 201, 1, 73, 222, 137, 2, 69, 3, 79, 2, 233, + 201, 7, 6, 79, 67, 65, 76, 73, 67, 2, 145, 249, 17, 11, 73, 78, 69, 32, + 79, 82, 68, 73, 78, 65, 76, 2, 207, 158, 17, 75, 226, 15, 76, 10, 72, 69, + 77, 65, 84, 73, 67, 65, 76, 32, 237, 207, 14, 3, 69, 32, 68, 224, 15, + 252, 1, 5, 66, 79, 76, 68, 32, 168, 6, 14, 68, 79, 85, 66, 76, 69, 45, + 83, 84, 82, 85, 67, 75, 32, 238, 1, 70, 164, 2, 7, 73, 84, 65, 76, 73, + 67, 32, 180, 3, 10, 77, 79, 78, 79, 83, 80, 65, 67, 69, 32, 40, 2, 82, + 73, 36, 3, 76, 69, 70, 167, 1, 83, 160, 5, 176, 1, 8, 67, 65, 80, 73, 84, + 65, 76, 32, 130, 2, 83, 210, 14, 73, 222, 3, 69, 38, 75, 38, 78, 30, 80, + 98, 82, 242, 5, 84, 56, 7, 70, 82, 65, 75, 84, 85, 82, 159, 160, 16, 68, + 104, 190, 4, 68, 174, 15, 84, 186, 4, 65, 22, 66, 2, 90, 42, 69, 98, 71, + 22, 73, 22, 75, 34, 76, 22, 79, 50, 80, 42, 82, 22, 83, 70, 85, 142, 197, + 1, 67, 214, 249, 12, 77, 2, 78, 206, 201, 1, 88, 222, 137, 2, 70, 2, 72, + 2, 74, 2, 81, 2, 86, 2, 87, 3, 89, 208, 1, 60, 5, 77, 65, 76, 76, 32, + 213, 25, 5, 67, 82, 73, 80, 84, 104, 250, 1, 68, 230, 19, 65, 22, 66, 2, 90, 42, 69, 38, 70, 62, 71, 22, 73, 22, 75, 34, 76, 22, 79, 50, 80, 42, - 82, 22, 83, 34, 84, 38, 85, 250, 196, 1, 67, 214, 173, 12, 77, 2, 78, - 210, 200, 1, 88, 206, 134, 2, 74, 2, 81, 2, 86, 2, 87, 3, 89, 9, 52, 7, - 79, 84, 76, 69, 83, 83, 32, 131, 173, 14, 69, 4, 154, 209, 17, 73, 3, 74, - 124, 130, 16, 67, 34, 83, 195, 210, 15, 68, 12, 32, 2, 71, 72, 179, 246, - 6, 83, 10, 17, 2, 84, 32, 10, 112, 6, 87, 72, 73, 84, 69, 32, 170, 236, - 5, 68, 234, 99, 65, 197, 164, 7, 9, 70, 76, 65, 84, 84, 69, 78, 69, 68, - 4, 254, 246, 13, 83, 39, 84, 130, 6, 84, 10, 65, 78, 83, 45, 83, 69, 82, - 73, 70, 32, 133, 14, 6, 67, 82, 73, 80, 84, 32, 176, 5, 96, 5, 66, 79, - 76, 68, 32, 176, 12, 6, 73, 84, 65, 76, 73, 67, 34, 67, 34, 83, 195, 210, - 15, 68, 204, 3, 98, 73, 122, 67, 230, 2, 69, 38, 75, 38, 78, 30, 80, 98, - 82, 30, 83, 214, 5, 84, 187, 211, 15, 68, 220, 1, 33, 6, 84, 65, 76, 73, - 67, 32, 220, 1, 74, 67, 230, 2, 69, 38, 75, 38, 78, 30, 80, 98, 82, 30, - 83, 215, 5, 84, 102, 37, 7, 65, 80, 73, 84, 65, 76, 32, 102, 250, 1, 84, - 186, 4, 65, 22, 66, 2, 90, 22, 68, 22, 69, 98, 71, 22, 73, 22, 75, 34, - 76, 22, 79, 50, 80, 42, 82, 22, 83, 70, 85, 250, 196, 1, 67, 214, 173, - 12, 77, 2, 78, 210, 200, 1, 88, 206, 134, 2, 70, 2, 72, 2, 74, 2, 81, 2, - 86, 2, 87, 3, 89, 9, 40, 4, 72, 69, 84, 65, 183, 164, 17, 65, 5, 151, - 182, 16, 32, 2, 185, 212, 15, 4, 80, 83, 73, 76, 2, 17, 2, 65, 80, 2, - 159, 7, 80, 2, 165, 166, 17, 2, 65, 66, 6, 74, 72, 168, 141, 5, 8, 65, - 82, 84, 73, 65, 76, 32, 68, 163, 167, 11, 73, 2, 199, 180, 16, 73, 2, - 177, 180, 16, 2, 72, 79, 102, 29, 5, 77, 65, 76, 76, 32, 102, 246, 1, 65, - 22, 66, 2, 90, 22, 68, 22, 69, 38, 70, 62, 71, 22, 73, 22, 75, 34, 76, - 22, 79, 50, 80, 42, 82, 22, 83, 34, 84, 38, 85, 250, 196, 1, 67, 214, - 173, 12, 77, 2, 78, 210, 200, 1, 88, 206, 134, 2, 72, 2, 74, 2, 81, 2, - 86, 2, 87, 3, 89, 5, 235, 208, 1, 76, 5, 175, 160, 17, 69, 5, 203, 160, - 14, 69, 7, 178, 212, 1, 80, 235, 237, 15, 84, 5, 11, 73, 2, 29, 5, 78, - 65, 76, 32, 83, 2, 227, 1, 73, 5, 219, 248, 14, 65, 5, 147, 159, 17, 79, - 5, 11, 65, 2, 223, 158, 14, 80, 5, 143, 160, 14, 65, 7, 11, 77, 4, 230, - 170, 1, 73, 239, 213, 15, 69, 9, 142, 175, 17, 72, 2, 83, 219, 19, 73, 5, - 203, 163, 17, 72, 5, 11, 73, 2, 143, 161, 17, 71, 7, 190, 157, 14, 72, - 231, 255, 2, 65, 5, 199, 209, 1, 80, 2, 11, 72, 2, 11, 69, 2, 11, 84, 2, - 159, 174, 16, 65, 104, 11, 32, 104, 18, 67, 35, 83, 52, 53, 5, 65, 80, - 73, 84, 65, 52, 21, 3, 77, 65, 76, 52, 183, 216, 11, 76, 82, 76, 8, 67, - 65, 80, 73, 84, 65, 76, 32, 157, 1, 6, 83, 77, 65, 76, 76, 32, 36, 222, - 191, 17, 65, 2, 67, 2, 68, 2, 71, 2, 74, 2, 75, 2, 78, 2, 79, 2, 80, 2, - 81, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, 2, 89, 3, 90, 46, 194, 190, - 17, 65, 2, 66, 2, 67, 2, 68, 2, 70, 2, 72, 2, 73, 2, 74, 2, 75, 2, 76, 2, - 77, 2, 78, 2, 80, 2, 81, 2, 82, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, - 2, 89, 3, 90, 40, 45, 9, 32, 78, 85, 77, 69, 82, 65, 76, 32, 40, 70, 69, - 66, 70, 70, 78, 26, 83, 66, 84, 142, 206, 15, 90, 143, 83, 79, 6, 40, 4, - 73, 71, 72, 84, 243, 224, 9, 76, 5, 231, 234, 15, 69, 8, 30, 73, 105, 3, - 79, 85, 82, 4, 202, 254, 4, 70, 139, 166, 12, 86, 4, 65, 3, 73, 78, 69, - 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 5, 191, 233, 15, 84, 10, 42, 72, - 190, 224, 9, 87, 243, 137, 7, 69, 4, 150, 253, 4, 73, 183, 208, 10, 82, - 248, 8, 234, 1, 65, 192, 4, 9, 67, 72, 65, 78, 73, 67, 65, 76, 32, 34, - 68, 180, 15, 11, 69, 84, 69, 73, 32, 77, 65, 89, 69, 75, 32, 178, 11, 76, - 34, 78, 162, 50, 82, 172, 15, 8, 83, 83, 65, 71, 69, 32, 87, 65, 20, 2, - 84, 82, 183, 183, 16, 77, 26, 72, 6, 83, 85, 82, 69, 68, 32, 249, 141, - 16, 6, 84, 32, 79, 78, 32, 66, 24, 96, 5, 65, 78, 71, 76, 69, 200, 247, - 9, 9, 82, 73, 71, 72, 84, 32, 65, 78, 71, 175, 173, 7, 66, 21, 11, 32, - 18, 212, 1, 39, 87, 73, 84, 72, 32, 79, 80, 69, 78, 32, 65, 82, 77, 32, - 69, 78, 68, 73, 78, 71, 32, 73, 78, 32, 65, 82, 82, 79, 87, 32, 80, 79, - 73, 78, 84, 73, 78, 71, 32, 161, 130, 17, 7, 79, 80, 69, 78, 73, 78, 71, - 16, 62, 68, 24, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 43, 85, 4, 73, 3, - 79, 87, 78, 4, 173, 249, 7, 5, 84, 32, 65, 78, 68, 4, 11, 80, 4, 145, - 190, 11, 3, 32, 65, 78, 4, 254, 180, 16, 65, 211, 56, 76, 250, 1, 56, 9, - 69, 70, 65, 73, 68, 82, 73, 78, 32, 159, 7, 73, 190, 1, 174, 1, 67, 52, - 6, 68, 73, 71, 73, 84, 32, 152, 1, 7, 78, 85, 77, 66, 69, 82, 32, 114, - 83, 132, 255, 6, 12, 69, 88, 67, 76, 65, 77, 65, 84, 73, 79, 78, 32, 215, - 228, 7, 70, 70, 128, 3, 5, 65, 80, 73, 84, 65, 255, 226, 14, 79, 26, 82, - 84, 48, 2, 79, 78, 194, 194, 15, 70, 30, 83, 102, 90, 130, 83, 78, 211, - 110, 69, 8, 44, 3, 72, 82, 69, 209, 176, 16, 2, 87, 79, 4, 207, 176, 16, - 69, 20, 50, 84, 226, 240, 4, 69, 46, 70, 42, 78, 31, 83, 6, 162, 242, 4, - 72, 204, 227, 4, 2, 87, 69, 215, 137, 7, 69, 70, 68, 3, 77, 65, 76, 217, - 183, 9, 8, 89, 77, 66, 79, 76, 32, 65, 73, 68, 45, 9, 76, 32, 76, 69, 84, - 84, 69, 82, 32, 68, 242, 1, 65, 38, 78, 146, 221, 3, 72, 2, 75, 162, 255, - 9, 89, 138, 143, 3, 79, 150, 64, 66, 2, 67, 2, 68, 2, 69, 2, 70, 2, 71, - 2, 73, 2, 74, 2, 76, 2, 77, 2, 80, 2, 81, 2, 82, 2, 83, 2, 84, 2, 85, 2, - 86, 2, 87, 2, 88, 3, 90, 7, 162, 237, 3, 84, 207, 190, 13, 73, 7, 202, - 171, 17, 71, 3, 89, 60, 48, 5, 69, 86, 65, 76, 32, 45, 3, 85, 77, 32, 6, - 138, 211, 10, 69, 206, 140, 4, 67, 115, 81, 54, 210, 1, 66, 46, 70, 164, - 1, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 248, 1, 11, 77, 65, 84, 72, 69, - 77, 65, 84, 73, 67, 65, 22, 83, 104, 4, 84, 72, 82, 69, 186, 212, 14, 86, - 230, 59, 69, 166, 4, 87, 207, 10, 71, 4, 164, 4, 3, 79, 76, 68, 163, 201, - 13, 76, 10, 84, 9, 76, 65, 84, 84, 69, 78, 69, 68, 32, 204, 3, 3, 79, 85, - 82, 219, 145, 15, 73, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 213, - 1, 3, 84, 32, 80, 6, 11, 84, 6, 78, 32, 41, 15, 45, 80, 79, 73, 78, 84, - 73, 78, 71, 32, 65, 78, 71, 76, 69, 4, 36, 5, 67, 85, 82, 76, 89, 59, 80, - 2, 17, 2, 32, 66, 2, 237, 163, 7, 4, 82, 65, 67, 75, 2, 153, 226, 16, 10, - 65, 82, 69, 78, 84, 72, 69, 83, 73, 83, 2, 251, 230, 16, 76, 10, 72, 4, - 77, 65, 76, 76, 154, 181, 14, 72, 188, 91, 2, 73, 88, 183, 2, 65, 2, 133, - 162, 14, 2, 32, 87, 4, 11, 69, 4, 45, 9, 32, 80, 79, 73, 78, 84, 69, 68, - 32, 4, 206, 179, 9, 80, 151, 221, 5, 66, 158, 1, 146, 1, 65, 104, 6, 67, - 72, 69, 73, 75, 72, 34, 76, 144, 6, 8, 83, 89, 76, 76, 65, 66, 76, 69, 0, - 4, 87, 79, 82, 68, 50, 86, 223, 172, 15, 68, 6, 80, 8, 72, 65, 78, 71, - 32, 75, 72, 85, 164, 6, 3, 80, 85, 78, 155, 177, 13, 78, 2, 175, 232, 15, - 68, 4, 130, 210, 16, 65, 255, 59, 69, 94, 52, 6, 69, 84, 84, 69, 82, 32, - 185, 5, 2, 85, 77, 92, 250, 1, 66, 36, 2, 67, 72, 38, 68, 50, 71, 38, 74, - 34, 75, 42, 78, 62, 80, 30, 83, 42, 84, 54, 73, 0, 3, 76, 65, 73, 0, 3, - 77, 73, 84, 230, 210, 12, 72, 166, 10, 82, 2, 87, 188, 144, 2, 2, 65, 84, - 230, 213, 1, 89, 242, 8, 85, 214, 79, 69, 3, 79, 4, 198, 229, 15, 72, - 191, 185, 1, 65, 4, 194, 149, 16, 73, 159, 137, 1, 65, 8, 238, 141, 10, - 72, 178, 135, 6, 73, 235, 65, 68, 4, 190, 141, 10, 72, 183, 141, 7, 79, - 4, 170, 228, 15, 72, 163, 48, 73, 6, 216, 1, 2, 79, 75, 163, 139, 10, 72, - 12, 178, 1, 65, 0, 3, 71, 79, 85, 182, 153, 17, 78, 3, 89, 6, 118, 65, - 179, 226, 15, 72, 6, 238, 139, 17, 65, 162, 14, 72, 3, 83, 10, 48, 2, 73, - 76, 162, 139, 10, 72, 155, 201, 6, 84, 5, 157, 177, 1, 4, 32, 76, 79, 78, - 2, 161, 152, 17, 3, 32, 73, 89, 2, 141, 178, 16, 7, 32, 82, 69, 80, 69, - 84, 73, 30, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 191, 232, 14, - 73, 28, 130, 1, 65, 44, 4, 67, 72, 69, 73, 2, 79, 0, 3, 83, 79, 85, 0, 2, - 89, 69, 22, 73, 38, 85, 206, 231, 10, 78, 187, 129, 4, 86, 8, 218, 136, - 16, 78, 170, 80, 65, 175, 64, 85, 2, 175, 136, 16, 78, 4, 154, 136, 16, - 78, 215, 144, 1, 73, 4, 246, 135, 16, 78, 215, 144, 1, 85, 4, 158, 144, - 16, 84, 183, 56, 79, 174, 3, 160, 1, 11, 68, 69, 32, 75, 73, 75, 65, 75, - 85, 73, 32, 244, 130, 16, 20, 79, 82, 65, 72, 32, 87, 73, 84, 72, 32, 78, - 73, 78, 69, 32, 66, 82, 65, 78, 67, 95, 83, 170, 3, 148, 1, 17, 67, 79, - 77, 66, 73, 78, 73, 78, 71, 32, 78, 85, 77, 66, 69, 82, 32, 164, 1, 10, - 83, 89, 76, 76, 65, 66, 76, 69, 32, 77, 199, 144, 9, 68, 14, 62, 84, 56, - 7, 72, 85, 78, 68, 82, 69, 68, 243, 177, 4, 77, 8, 26, 69, 187, 178, 4, - 72, 6, 26, 78, 187, 230, 13, 69, 4, 140, 178, 4, 2, 32, 84, 191, 226, 12, - 83, 138, 3, 22, 48, 167, 21, 49, 198, 1, 118, 48, 250, 1, 49, 210, 1, 50, - 138, 2, 51, 254, 1, 52, 150, 2, 53, 158, 2, 54, 238, 1, 55, 142, 2, 56, - 187, 2, 57, 18, 142, 1, 49, 34, 50, 22, 51, 22, 52, 218, 149, 2, 53, 142, - 230, 3, 56, 180, 149, 10, 3, 57, 32, 77, 92, 3, 55, 32, 77, 213, 90, 3, - 54, 32, 87, 2, 11, 32, 2, 147, 254, 16, 75, 2, 239, 136, 17, 32, 2, 183, - 239, 11, 32, 2, 11, 32, 2, 203, 253, 16, 87, 20, 130, 1, 49, 22, 54, 18, - 56, 22, 57, 200, 28, 2, 48, 32, 138, 161, 7, 53, 130, 190, 5, 52, 226, - 10, 55, 222, 9, 50, 199, 191, 3, 51, 2, 227, 226, 16, 32, 2, 143, 25, 32, - 2, 139, 170, 13, 32, 2, 191, 209, 12, 32, 20, 106, 49, 22, 50, 22, 51, - 22, 52, 22, 53, 22, 54, 22, 55, 22, 57, 158, 186, 11, 48, 253, 232, 1, 2, - 56, 32, 2, 155, 144, 10, 32, 2, 195, 234, 9, 32, 2, 223, 233, 16, 32, 2, - 215, 233, 11, 32, 2, 131, 191, 12, 32, 2, 215, 235, 16, 32, 2, 143, 185, - 12, 32, 2, 191, 28, 32, 20, 174, 1, 48, 16, 2, 49, 32, 20, 2, 52, 32, 20, - 2, 56, 32, 230, 186, 4, 57, 238, 51, 50, 22, 53, 188, 201, 7, 2, 54, 32, - 244, 211, 3, 3, 55, 32, 78, 213, 90, 3, 51, 32, 89, 2, 187, 37, 32, 2, - 159, 248, 16, 89, 2, 139, 248, 16, 70, 2, 155, 196, 15, 78, 20, 184, 1, - 2, 48, 32, 20, 2, 51, 32, 30, 53, 22, 56, 164, 8, 3, 52, 32, 75, 252, 21, - 3, 54, 32, 72, 230, 2, 55, 170, 205, 4, 49, 212, 164, 4, 3, 57, 32, 87, - 245, 181, 5, 3, 50, 32, 72, 2, 203, 157, 15, 72, 2, 177, 199, 16, 2, 78, - 71, 2, 223, 252, 10, 32, 2, 187, 234, 16, 32, 20, 178, 1, 48, 18, 53, 20, - 2, 54, 32, 20, 2, 56, 32, 22, 57, 172, 190, 9, 2, 50, 32, 236, 4, 2, 51, - 32, 166, 200, 1, 49, 164, 187, 3, 3, 52, 32, 76, 213, 137, 1, 3, 55, 32, - 78, 2, 143, 4, 32, 2, 183, 195, 15, 32, 2, 167, 218, 16, 71, 2, 227, 236, - 12, 78, 2, 177, 182, 15, 2, 32, 77, 20, 192, 1, 2, 50, 32, 22, 54, 242, - 25, 51, 164, 2, 3, 55, 32, 78, 52, 3, 49, 32, 87, 156, 215, 1, 2, 52, 32, - 206, 143, 3, 56, 156, 255, 4, 3, 48, 32, 78, 174, 92, 57, 137, 186, 4, 3, - 53, 32, 75, 2, 159, 216, 16, 77, 2, 195, 254, 9, 32, 20, 184, 1, 3, 52, - 32, 75, 20, 2, 53, 32, 20, 2, 55, 32, 22, 57, 178, 9, 51, 234, 11, 50, - 144, 2, 3, 49, 32, 71, 40, 3, 48, 32, 71, 144, 171, 14, 3, 54, 32, 75, - 149, 84, 3, 56, 32, 70, 2, 251, 222, 16, 80, 2, 251, 236, 16, 70, 2, 199, - 228, 16, 86, 2, 139, 171, 8, 32, 20, 228, 1, 2, 48, 32, 20, 2, 49, 32, - 20, 2, 52, 32, 22, 53, 248, 13, 6, 54, 32, 76, 79, 78, 71, 152, 12, 2, - 57, 32, 160, 167, 3, 3, 51, 32, 72, 132, 252, 8, 4, 50, 32, 78, 71, 164, - 195, 3, 3, 55, 32, 72, 165, 97, 3, 56, 32, 70, 2, 215, 234, 16, 89, 2, - 227, 148, 15, 80, 2, 207, 148, 15, 76, 2, 171, 132, 16, 32, 20, 230, 1, - 53, 164, 14, 4, 48, 32, 78, 71, 172, 8, 3, 51, 32, 71, 244, 243, 11, 2, - 55, 32, 214, 118, 57, 252, 88, 3, 50, 32, 75, 172, 184, 1, 3, 49, 32, 84, - 192, 27, 4, 56, 32, 78, 89, 148, 129, 1, 3, 52, 32, 77, 213, 34, 2, 54, - 32, 2, 155, 207, 16, 32, 196, 1, 118, 48, 154, 2, 49, 210, 2, 50, 170, 2, - 51, 222, 1, 52, 162, 2, 53, 214, 2, 54, 166, 2, 55, 238, 2, 56, 231, 2, - 57, 20, 222, 1, 49, 30, 52, 172, 11, 6, 54, 32, 76, 79, 78, 71, 164, 4, - 6, 53, 32, 76, 79, 78, 71, 24, 2, 51, 32, 162, 229, 4, 50, 230, 152, 2, - 48, 204, 249, 4, 3, 55, 32, 71, 252, 135, 3, 3, 57, 32, 89, 197, 39, 3, - 56, 32, 75, 2, 205, 180, 15, 2, 32, 70, 2, 157, 187, 14, 2, 32, 84, 20, + 82, 22, 83, 34, 84, 38, 85, 142, 197, 1, 67, 214, 249, 12, 77, 2, 78, + 206, 201, 1, 88, 222, 137, 2, 72, 2, 74, 2, 81, 2, 86, 2, 87, 3, 89, 7, + 26, 73, 235, 239, 14, 69, 2, 187, 213, 1, 71, 110, 68, 8, 67, 65, 80, 73, + 84, 65, 76, 32, 162, 23, 83, 223, 159, 16, 68, 38, 154, 168, 18, 65, 2, + 66, 2, 68, 2, 69, 2, 70, 2, 71, 2, 73, 2, 74, 2, 75, 2, 76, 2, 77, 2, 79, + 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, 3, 89, 96, 52, 7, 82, 65, 75, + 84, 85, 82, 32, 219, 142, 7, 65, 94, 52, 8, 67, 65, 80, 73, 84, 65, 76, + 32, 143, 21, 83, 42, 134, 166, 18, 65, 2, 66, 2, 68, 2, 69, 2, 70, 2, 71, + 2, 74, 2, 75, 2, 76, 2, 77, 2, 78, 2, 79, 2, 80, 2, 81, 2, 83, 2, 84, 2, + 85, 2, 86, 2, 87, 2, 88, 3, 89, 222, 1, 100, 6, 83, 77, 65, 76, 76, 32, + 222, 7, 67, 230, 2, 69, 38, 75, 38, 78, 30, 80, 98, 82, 243, 5, 84, 104, + 242, 1, 68, 198, 12, 65, 22, 66, 2, 90, 42, 69, 38, 70, 62, 71, 22, 73, + 22, 75, 34, 76, 22, 79, 50, 80, 42, 82, 22, 83, 34, 84, 38, 85, 142, 197, + 1, 67, 214, 249, 12, 77, 2, 78, 206, 201, 1, 88, 222, 137, 2, 74, 2, 81, + 2, 86, 2, 87, 3, 89, 9, 52, 7, 79, 84, 76, 69, 83, 83, 32, 175, 232, 14, + 69, 4, 186, 161, 18, 73, 3, 74, 124, 130, 16, 67, 34, 83, 223, 159, 16, + 68, 12, 32, 2, 71, 72, 131, 137, 7, 83, 10, 17, 2, 84, 32, 10, 112, 6, + 87, 72, 73, 84, 69, 32, 170, 244, 5, 68, 234, 106, 65, 241, 230, 4, 9, + 70, 76, 65, 84, 84, 69, 78, 69, 68, 4, 130, 165, 14, 83, 39, 84, 130, 6, + 84, 10, 65, 78, 83, 45, 83, 69, 82, 73, 70, 32, 133, 14, 6, 67, 82, 73, + 80, 84, 32, 176, 5, 96, 5, 66, 79, 76, 68, 32, 176, 12, 6, 73, 84, 65, + 76, 73, 67, 34, 67, 34, 83, 223, 159, 16, 68, 204, 3, 98, 73, 122, 67, + 230, 2, 69, 38, 75, 38, 78, 30, 80, 98, 82, 30, 83, 214, 5, 84, 215, 160, + 16, 68, 220, 1, 33, 6, 84, 65, 76, 73, 67, 32, 220, 1, 74, 67, 230, 2, + 69, 38, 75, 38, 78, 30, 80, 98, 82, 30, 83, 215, 5, 84, 102, 37, 7, 65, + 80, 73, 84, 65, 76, 32, 102, 250, 1, 84, 186, 4, 65, 22, 66, 2, 90, 22, + 68, 22, 69, 98, 71, 22, 73, 22, 75, 34, 76, 22, 79, 50, 80, 42, 82, 22, + 83, 70, 85, 142, 197, 1, 67, 214, 249, 12, 77, 2, 78, 206, 201, 1, 88, + 222, 137, 2, 70, 2, 72, 2, 74, 2, 81, 2, 86, 2, 87, 3, 89, 9, 40, 4, 72, + 69, 84, 65, 215, 244, 17, 65, 5, 159, 134, 17, 32, 2, 213, 161, 16, 4, + 80, 83, 73, 76, 2, 17, 2, 65, 80, 2, 159, 7, 80, 2, 197, 246, 17, 2, 65, + 66, 6, 74, 72, 172, 149, 5, 8, 65, 82, 84, 73, 65, 76, 32, 68, 167, 239, + 11, 73, 2, 207, 132, 17, 73, 2, 185, 132, 17, 2, 72, 79, 102, 29, 5, 77, + 65, 76, 76, 32, 102, 246, 1, 65, 22, 66, 2, 90, 22, 68, 22, 69, 38, 70, + 62, 71, 22, 73, 22, 75, 34, 76, 22, 79, 50, 80, 42, 82, 22, 83, 34, 84, + 38, 85, 142, 197, 1, 67, 214, 249, 12, 77, 2, 78, 206, 201, 1, 88, 222, + 137, 2, 72, 2, 74, 2, 81, 2, 86, 2, 87, 3, 89, 5, 251, 188, 14, 76, 5, + 207, 240, 17, 69, 5, 247, 219, 14, 69, 7, 186, 212, 1, 80, 131, 190, 16, + 84, 5, 11, 73, 2, 29, 5, 78, 65, 76, 32, 83, 2, 227, 1, 73, 5, 155, 194, + 15, 65, 5, 179, 239, 17, 79, 5, 11, 65, 2, 139, 218, 14, 80, 5, 187, 219, + 14, 65, 7, 11, 77, 4, 246, 170, 1, 73, 243, 165, 16, 69, 9, 174, 255, 17, + 72, 2, 83, 219, 19, 73, 5, 235, 243, 17, 72, 5, 11, 73, 2, 175, 241, 17, + 71, 7, 234, 216, 14, 72, 219, 148, 3, 65, 5, 207, 209, 1, 80, 2, 11, 72, + 2, 11, 69, 2, 11, 84, 2, 167, 254, 16, 65, 104, 11, 32, 104, 18, 67, 35, + 83, 52, 53, 5, 65, 80, 73, 84, 65, 52, 21, 3, 77, 65, 76, 52, 147, 254, + 11, 76, 82, 76, 8, 67, 65, 80, 73, 84, 65, 76, 32, 157, 1, 6, 83, 77, 65, + 76, 76, 32, 36, 254, 143, 18, 65, 2, 67, 2, 68, 2, 71, 2, 74, 2, 75, 2, + 78, 2, 79, 2, 80, 2, 81, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, 2, 89, + 3, 90, 46, 226, 142, 18, 65, 2, 66, 2, 67, 2, 68, 2, 70, 2, 72, 2, 73, 2, + 74, 2, 75, 2, 76, 2, 77, 2, 78, 2, 80, 2, 81, 2, 82, 2, 83, 2, 84, 2, 85, + 2, 86, 2, 87, 2, 88, 2, 89, 3, 90, 40, 45, 9, 32, 78, 85, 77, 69, 82, 65, + 76, 32, 40, 70, 69, 66, 70, 70, 78, 26, 83, 66, 84, 170, 155, 16, 90, + 251, 85, 79, 6, 40, 4, 73, 71, 72, 84, 215, 247, 9, 76, 5, 147, 184, 16, + 69, 8, 30, 73, 105, 3, 79, 85, 82, 4, 206, 134, 5, 70, 167, 238, 12, 86, + 4, 65, 3, 73, 78, 69, 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 5, 235, + 182, 16, 84, 10, 42, 72, 162, 247, 9, 87, 163, 195, 7, 69, 4, 154, 133, + 5, 73, 207, 149, 11, 82, 252, 8, 234, 1, 65, 192, 4, 9, 67, 72, 65, 78, + 73, 67, 65, 76, 32, 34, 68, 164, 15, 11, 69, 84, 69, 73, 32, 77, 65, 89, + 69, 75, 32, 178, 11, 76, 34, 78, 154, 50, 82, 176, 15, 8, 83, 83, 65, 71, + 69, 32, 87, 65, 20, 2, 84, 82, 235, 135, 17, 77, 26, 72, 6, 83, 85, 82, + 69, 68, 32, 253, 220, 16, 6, 84, 32, 79, 78, 32, 66, 24, 96, 5, 65, 78, + 71, 76, 69, 148, 142, 10, 9, 82, 73, 71, 72, 84, 32, 65, 78, 71, 131, + 231, 7, 66, 21, 11, 32, 18, 212, 1, 39, 87, 73, 84, 72, 32, 79, 80, 69, + 78, 32, 65, 82, 77, 32, 69, 78, 68, 73, 78, 71, 32, 73, 78, 32, 65, 82, + 82, 79, 87, 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 181, 210, 17, 7, 79, + 80, 69, 78, 73, 78, 71, 16, 62, 68, 24, 3, 76, 69, 70, 0, 4, 82, 73, 71, + 72, 43, 85, 4, 73, 3, 79, 87, 78, 4, 137, 144, 8, 5, 84, 32, 65, 78, 68, + 4, 11, 80, 4, 225, 227, 11, 3, 32, 65, 78, 4, 142, 133, 17, 65, 215, 56, + 76, 250, 1, 56, 9, 69, 70, 65, 73, 68, 82, 73, 78, 32, 159, 7, 73, 190, + 1, 174, 1, 67, 52, 6, 68, 73, 71, 73, 84, 32, 152, 1, 7, 78, 85, 77, 66, + 69, 82, 32, 114, 83, 164, 146, 7, 12, 69, 88, 67, 76, 65, 77, 65, 84, 73, + 79, 78, 32, 247, 154, 8, 70, 70, 128, 3, 5, 65, 80, 73, 84, 65, 191, 172, + 15, 79, 26, 82, 84, 48, 2, 79, 78, 222, 143, 16, 70, 30, 83, 102, 90, + 238, 85, 78, 235, 110, 69, 8, 44, 3, 72, 82, 69, 217, 128, 17, 2, 87, 79, + 4, 215, 128, 17, 69, 20, 50, 84, 230, 248, 4, 69, 46, 70, 42, 78, 31, 83, + 6, 166, 250, 4, 72, 172, 242, 4, 2, 87, 69, 135, 195, 7, 69, 70, 68, 3, + 77, 65, 76, 157, 206, 9, 8, 89, 77, 66, 79, 76, 32, 65, 73, 68, 45, 9, + 76, 32, 76, 69, 84, 84, 69, 82, 32, 68, 242, 1, 65, 38, 78, 130, 227, 3, + 72, 2, 75, 198, 197, 10, 89, 138, 147, 3, 79, 162, 64, 66, 2, 67, 2, 68, + 2, 69, 2, 70, 2, 71, 2, 73, 2, 74, 2, 76, 2, 77, 2, 80, 2, 81, 2, 82, 2, + 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, 3, 90, 7, 146, 243, 3, 84, 255, + 136, 14, 73, 7, 234, 251, 17, 71, 3, 89, 60, 48, 5, 69, 86, 65, 76, 32, + 45, 3, 85, 77, 32, 6, 210, 237, 10, 69, 198, 187, 4, 67, 115, 81, 54, + 210, 1, 66, 50, 70, 164, 1, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 248, 1, + 11, 77, 65, 84, 72, 69, 77, 65, 84, 73, 67, 65, 22, 83, 84, 4, 84, 72, + 82, 69, 182, 158, 15, 86, 130, 63, 69, 166, 4, 87, 215, 10, 71, 4, 180, + 156, 6, 3, 79, 76, 68, 255, 225, 7, 76, 10, 84, 9, 76, 65, 84, 84, 69, + 78, 69, 68, 32, 184, 3, 3, 79, 85, 82, 243, 222, 15, 73, 4, 44, 3, 76, + 69, 70, 1, 4, 82, 73, 71, 72, 2, 213, 1, 3, 84, 32, 80, 6, 11, 84, 6, 78, + 32, 41, 15, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 65, 78, 71, 76, 69, + 4, 36, 5, 67, 85, 82, 76, 89, 59, 80, 2, 17, 2, 32, 66, 2, 137, 183, 7, + 4, 82, 65, 67, 75, 2, 169, 178, 17, 10, 65, 82, 69, 78, 84, 72, 69, 83, + 73, 83, 2, 139, 183, 17, 76, 10, 212, 152, 6, 4, 77, 65, 76, 76, 250, + 229, 8, 72, 212, 95, 2, 73, 88, 183, 2, 65, 4, 11, 69, 4, 45, 9, 32, 80, + 79, 73, 78, 84, 69, 68, 32, 4, 162, 202, 9, 80, 219, 147, 6, 66, 158, 1, + 146, 1, 65, 104, 6, 67, 72, 69, 73, 75, 72, 34, 76, 144, 6, 8, 83, 89, + 76, 76, 65, 66, 76, 69, 0, 4, 87, 79, 82, 68, 50, 86, 139, 250, 15, 68, + 6, 80, 8, 72, 65, 78, 71, 32, 75, 72, 85, 164, 6, 3, 80, 85, 78, 147, + 213, 13, 78, 2, 159, 183, 16, 68, 4, 166, 162, 17, 65, 139, 60, 69, 94, + 52, 6, 69, 84, 84, 69, 82, 32, 185, 5, 2, 85, 77, 92, 250, 1, 66, 36, 2, + 67, 72, 38, 68, 50, 71, 38, 74, 34, 75, 42, 78, 62, 80, 30, 83, 42, 84, + 54, 73, 0, 3, 76, 65, 73, 0, 3, 77, 73, 84, 138, 247, 12, 72, 166, 10, + 82, 2, 87, 232, 181, 2, 2, 65, 84, 182, 220, 1, 89, 246, 8, 85, 226, 79, + 69, 3, 79, 4, 182, 180, 16, 72, 255, 186, 1, 65, 4, 218, 229, 16, 73, + 183, 137, 1, 65, 8, 238, 165, 10, 72, 202, 191, 6, 73, 247, 65, 68, 4, + 190, 165, 10, 72, 231, 197, 7, 79, 4, 154, 179, 16, 72, 203, 49, 73, 6, + 216, 1, 2, 79, 75, 163, 163, 10, 72, 12, 178, 1, 65, 0, 3, 71, 79, 85, + 230, 233, 17, 78, 3, 89, 6, 118, 65, 163, 177, 16, 72, 6, 158, 220, 17, + 65, 162, 14, 72, 3, 83, 10, 48, 2, 73, 76, 162, 163, 10, 72, 191, 129, 7, + 84, 5, 181, 177, 1, 4, 32, 76, 79, 78, 2, 209, 232, 17, 3, 32, 73, 89, 2, + 173, 130, 17, 7, 32, 82, 69, 80, 69, 84, 73, 30, 64, 10, 79, 87, 69, 76, + 32, 83, 73, 71, 78, 32, 143, 178, 15, 73, 28, 130, 1, 65, 44, 4, 67, 72, + 69, 73, 2, 79, 0, 3, 83, 79, 85, 0, 2, 89, 69, 22, 73, 38, 85, 194, 131, + 11, 78, 151, 175, 4, 86, 8, 242, 216, 16, 78, 182, 80, 65, 187, 64, 85, + 2, 199, 216, 16, 78, 4, 178, 216, 16, 78, 239, 144, 1, 73, 4, 142, 216, + 16, 78, 239, 144, 1, 85, 4, 182, 224, 16, 84, 195, 56, 79, 178, 3, 160, + 1, 11, 68, 69, 32, 75, 73, 75, 65, 75, 85, 73, 32, 140, 211, 16, 20, 79, + 82, 65, 72, 32, 87, 73, 84, 72, 32, 78, 73, 78, 69, 32, 66, 82, 65, 78, + 67, 95, 83, 174, 3, 148, 1, 17, 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, + 78, 85, 77, 66, 69, 82, 32, 164, 1, 10, 83, 89, 76, 76, 65, 66, 76, 69, + 32, 77, 155, 167, 9, 68, 14, 62, 84, 56, 7, 72, 85, 78, 68, 82, 69, 68, + 135, 186, 4, 77, 8, 26, 69, 207, 186, 4, 72, 6, 26, 78, 167, 161, 14, 69, + 4, 160, 186, 4, 2, 32, 84, 219, 170, 13, 83, 142, 3, 22, 48, 163, 21, 49, + 198, 1, 118, 48, 250, 1, 49, 210, 1, 50, 138, 2, 51, 254, 1, 52, 150, 2, + 53, 158, 2, 54, 242, 1, 55, 134, 2, 56, 187, 2, 57, 18, 142, 1, 49, 34, + 50, 22, 51, 22, 52, 254, 150, 2, 53, 154, 240, 3, 56, 156, 218, 10, 3, + 57, 32, 77, 92, 3, 55, 32, 77, 237, 90, 3, 54, 32, 87, 2, 11, 32, 2, 195, + 206, 17, 75, 2, 159, 217, 17, 32, 2, 215, 148, 12, 32, 2, 11, 32, 2, 251, + 205, 17, 87, 20, 130, 1, 49, 22, 54, 18, 56, 22, 57, 248, 28, 2, 48, 32, + 198, 183, 7, 53, 170, 203, 5, 52, 198, 10, 55, 222, 9, 50, 243, 235, 3, + 51, 2, 147, 179, 17, 32, 2, 167, 25, 32, 2, 163, 204, 13, 32, 2, 227, + 245, 12, 32, 20, 106, 49, 22, 50, 22, 51, 22, 52, 22, 53, 22, 54, 22, 55, + 22, 57, 130, 224, 11, 48, 145, 231, 1, 2, 56, 32, 2, 211, 169, 10, 32, 2, + 247, 128, 10, 32, 2, 143, 186, 17, 32, 2, 247, 142, 12, 32, 2, 167, 227, + 12, 32, 2, 135, 188, 17, 32, 2, 179, 221, 12, 32, 2, 239, 28, 32, 20, + 174, 1, 48, 16, 2, 49, 32, 20, 2, 52, 32, 20, 2, 56, 32, 246, 194, 4, 57, + 182, 51, 50, 22, 53, 136, 230, 7, 2, 54, 32, 232, 255, 3, 3, 55, 32, 78, + 237, 90, 3, 51, 32, 89, 2, 179, 37, 32, 2, 207, 200, 17, 89, 2, 187, 200, + 17, 70, 2, 255, 145, 16, 78, 20, 184, 1, 2, 48, 32, 20, 2, 51, 32, 30, + 53, 22, 56, 160, 8, 3, 52, 32, 75, 128, 22, 3, 54, 32, 72, 222, 2, 55, + 138, 213, 4, 49, 208, 179, 4, 3, 57, 32, 87, 241, 232, 5, 3, 50, 32, 72, + 2, 247, 234, 15, 72, 2, 213, 151, 17, 2, 78, 71, 2, 191, 162, 11, 32, 2, + 235, 186, 17, 32, 20, 178, 1, 48, 18, 53, 20, 2, 54, 32, 20, 2, 56, 32, + 22, 57, 136, 213, 9, 2, 50, 32, 236, 4, 2, 51, 32, 170, 215, 1, 49, 148, + 223, 3, 3, 52, 32, 76, 137, 143, 1, 3, 55, 32, 78, 2, 159, 4, 32, 2, 155, + 145, 16, 32, 2, 215, 170, 17, 71, 2, 247, 144, 13, 78, 2, 237, 131, 16, + 2, 32, 77, 20, 196, 1, 2, 50, 32, 22, 54, 238, 25, 51, 168, 2, 3, 55, 32, + 78, 140, 216, 1, 2, 52, 32, 234, 150, 3, 56, 248, 144, 5, 3, 48, 32, 78, + 172, 89, 3, 49, 32, 87, 158, 13, 57, 225, 227, 4, 3, 53, 32, 75, 2, 203, + 168, 17, 77, 2, 247, 151, 10, 32, 20, 196, 1, 3, 52, 32, 75, 20, 2, 53, + 32, 22, 57, 232, 7, 3, 49, 32, 71, 234, 1, 51, 236, 6, 3, 48, 32, 71, + 146, 5, 50, 236, 224, 10, 2, 55, 32, 252, 149, 4, 3, 54, 32, 75, 241, 87, + 3, 56, 32, 70, 2, 155, 175, 17, 80, 2, 155, 189, 17, 70, 2, 187, 243, 10, + 32, 20, 228, 1, 2, 48, 32, 20, 2, 49, 32, 20, 2, 52, 32, 22, 53, 172, 14, + 6, 54, 32, 76, 79, 78, 71, 224, 11, 2, 57, 32, 168, 173, 3, 3, 51, 32, + 72, 168, 154, 9, 4, 50, 32, 78, 71, 152, 239, 3, 3, 55, 32, 72, 189, 97, + 3, 56, 32, 70, 2, 139, 187, 17, 89, 2, 147, 226, 15, 80, 2, 255, 225, 15, + 76, 2, 207, 212, 16, 32, 20, 230, 1, 53, 216, 14, 4, 48, 32, 78, 71, 244, + 7, 3, 51, 32, 71, 192, 152, 12, 2, 55, 32, 138, 118, 57, 192, 112, 3, 50, + 32, 75, 156, 202, 1, 3, 49, 32, 84, 208, 27, 4, 56, 32, 78, 89, 252, 131, + 1, 3, 52, 32, 77, 225, 34, 2, 54, 32, 2, 207, 159, 17, 32, 200, 1, 118, + 48, 186, 2, 49, 210, 2, 50, 166, 2, 51, 222, 1, 52, 186, 2, 53, 214, 2, + 54, 166, 2, 55, 190, 2, 56, 223, 2, 57, 20, 222, 1, 49, 30, 52, 28, 7, + 53, 32, 76, 79, 78, 71, 32, 12, 2, 51, 32, 184, 11, 6, 54, 32, 76, 79, + 78, 71, 138, 241, 4, 50, 198, 168, 2, 48, 212, 134, 5, 3, 55, 32, 71, + 228, 176, 3, 3, 57, 32, 89, 253, 39, 3, 56, 32, 75, 2, 181, 130, 16, 2, + 32, 70, 2, 241, 132, 15, 2, 32, 84, 2, 11, 77, 2, 203, 132, 15, 66, 20, 208, 1, 6, 48, 32, 76, 79, 78, 71, 22, 51, 34, 54, 22, 56, 32, 2, 57, 32, - 156, 18, 4, 53, 32, 78, 71, 224, 206, 2, 2, 55, 32, 192, 145, 2, 3, 50, - 32, 75, 140, 198, 9, 3, 52, 32, 87, 145, 254, 1, 2, 49, 32, 2, 211, 216, - 15, 32, 2, 11, 32, 2, 131, 227, 16, 74, 2, 207, 240, 15, 32, 2, 11, 32, - 2, 207, 226, 16, 87, 2, 155, 236, 14, 78, 20, 230, 1, 50, 32, 2, 51, 32, - 232, 3, 3, 52, 32, 71, 182, 10, 48, 220, 147, 7, 2, 57, 32, 176, 196, 2, - 4, 56, 32, 72, 79, 248, 207, 2, 5, 55, 32, 78, 71, 71, 236, 90, 2, 53, - 32, 180, 249, 1, 3, 54, 32, 87, 129, 148, 1, 2, 49, 32, 2, 11, 32, 2, - 203, 247, 12, 77, 2, 11, 78, 2, 143, 227, 16, 68, 20, 172, 8, 3, 52, 32, - 78, 204, 2, 2, 56, 32, 224, 233, 4, 3, 50, 32, 75, 194, 195, 2, 49, 2, - 53, 216, 173, 2, 2, 55, 32, 152, 208, 4, 3, 51, 32, 70, 0, 3, 54, 32, 83, - 164, 50, 2, 48, 32, 225, 145, 1, 3, 57, 32, 87, 20, 232, 1, 3, 53, 32, - 77, 22, 54, 224, 2, 3, 57, 32, 78, 172, 4, 6, 50, 32, 76, 79, 78, 71, - 130, 196, 6, 55, 140, 151, 3, 3, 51, 32, 87, 200, 212, 2, 2, 48, 32, 220, - 60, 3, 56, 32, 71, 244, 190, 1, 3, 49, 32, 89, 1, 3, 52, 32, 86, 2, 179, - 223, 16, 66, 2, 129, 191, 15, 3, 32, 78, 71, 20, 192, 1, 2, 50, 32, 34, - 52, 26, 53, 34, 54, 36, 2, 55, 32, 244, 7, 2, 48, 32, 180, 225, 9, 3, 56, - 32, 75, 148, 182, 2, 2, 49, 32, 148, 225, 2, 5, 57, 32, 78, 71, 71, 249, - 204, 1, 2, 51, 32, 2, 11, 78, 2, 207, 238, 16, 74, 2, 213, 5, 2, 32, 77, - 2, 11, 32, 2, 247, 220, 16, 71, 2, 241, 175, 14, 4, 32, 78, 71, 71, 2, - 227, 131, 15, 74, 20, 220, 1, 2, 48, 32, 20, 6, 49, 32, 76, 79, 78, 71, - 30, 56, 204, 2, 2, 55, 32, 128, 170, 9, 3, 52, 32, 78, 200, 48, 4, 54, - 32, 71, 85, 152, 208, 4, 2, 53, 32, 148, 84, 3, 50, 32, 83, 0, 2, 51, 32, - 185, 98, 2, 57, 32, 2, 219, 173, 14, 74, 2, 221, 249, 11, 2, 32, 77, 2, - 183, 170, 12, 32, 20, 198, 1, 50, 46, 52, 40, 6, 53, 32, 76, 79, 78, 71, - 28, 3, 55, 32, 78, 34, 56, 152, 199, 11, 2, 54, 32, 156, 99, 3, 57, 32, - 75, 160, 212, 2, 3, 51, 32, 86, 156, 108, 4, 48, 32, 78, 89, 207, 53, 49, - 2, 11, 32, 2, 11, 77, 2, 183, 171, 14, 66, 2, 17, 2, 32, 77, 2, 227, 204, - 16, 66, 2, 205, 204, 16, 2, 32, 74, 2, 11, 71, 2, 231, 163, 15, 85, 2, - 147, 154, 15, 32, 20, 208, 1, 2, 48, 32, 22, 49, 22, 50, 32, 6, 51, 32, - 76, 79, 78, 71, 34, 56, 252, 241, 8, 2, 53, 32, 152, 126, 3, 52, 32, 78, - 188, 182, 2, 2, 54, 32, 228, 210, 2, 4, 55, 32, 77, 66, 145, 30, 4, 57, - 32, 77, 85, 2, 215, 168, 14, 68, 2, 215, 238, 9, 32, 2, 11, 32, 2, 147, - 213, 16, 86, 2, 253, 222, 14, 3, 32, 78, 71, 2, 17, 2, 32, 77, 2, 239, - 251, 14, 66, 16, 142, 1, 48, 32, 3, 49, 32, 78, 20, 3, 50, 32, 78, 20, 2, - 51, 32, 20, 3, 52, 32, 87, 22, 53, 20, 2, 54, 32, 169, 165, 12, 3, 55, - 32, 70, 2, 11, 32, 2, 191, 250, 14, 71, 2, 171, 250, 14, 68, 2, 227, 193, - 16, 74, 2, 215, 150, 16, 72, 2, 203, 210, 16, 85, 2, 219, 158, 15, 32, 2, - 227, 144, 1, 83, 248, 1, 68, 6, 79, 73, 84, 73, 67, 32, 250, 145, 10, 80, - 241, 70, 2, 67, 85, 244, 1, 104, 8, 67, 85, 82, 83, 73, 86, 69, 32, 221, - 10, 13, 72, 73, 69, 82, 79, 71, 76, 89, 80, 72, 73, 67, 32, 180, 1, 96, - 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 250, 2, 76, 133, 3, 7, 78, 85, 77, - 66, 69, 82, 32, 24, 78, 69, 50, 70, 44, 4, 79, 78, 69, 32, 46, 83, 50, - 84, 61, 3, 78, 73, 78, 4, 160, 1, 2, 76, 69, 93, 4, 73, 71, 72, 84, 4, - 192, 1, 2, 73, 86, 13, 3, 79, 85, 82, 4, 232, 225, 14, 4, 84, 87, 69, 76, - 19, 72, 4, 26, 69, 93, 2, 73, 88, 2, 65, 2, 86, 69, 6, 46, 69, 12, 3, 72, - 82, 69, 13, 2, 87, 79, 2, 23, 78, 2, 11, 69, 2, 197, 227, 14, 5, 32, 84, - 87, 69, 76, 52, 76, 6, 69, 84, 84, 69, 82, 32, 137, 2, 8, 79, 71, 79, 71, - 82, 65, 77, 32, 48, 182, 1, 65, 46, 84, 146, 234, 1, 78, 2, 83, 206, 252, - 12, 72, 138, 176, 1, 75, 254, 68, 66, 2, 68, 2, 76, 2, 77, 2, 80, 2, 81, - 2, 82, 2, 87, 2, 89, 186, 2, 69, 2, 73, 3, 79, 5, 149, 139, 11, 6, 82, - 67, 72, 65, 73, 67, 6, 150, 222, 16, 65, 2, 69, 3, 79, 4, 174, 201, 4, - 73, 153, 248, 11, 2, 82, 77, 104, 92, 5, 69, 73, 71, 72, 84, 30, 70, 92, - 4, 78, 73, 78, 69, 54, 83, 78, 84, 73, 2, 79, 78, 11, 150, 1, 89, 223, 1, - 32, 24, 18, 73, 35, 79, 12, 142, 2, 86, 139, 228, 3, 70, 12, 148, 2, 2, - 85, 82, 227, 227, 3, 82, 11, 28, 2, 84, 89, 223, 1, 32, 2, 231, 141, 6, - 32, 24, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 13, 154, 1, 32, 227, 227, 3, - 84, 28, 34, 72, 50, 87, 159, 140, 6, 69, 12, 32, 2, 82, 69, 215, 227, 3, - 73, 8, 39, 69, 12, 26, 79, 215, 227, 3, 69, 9, 11, 32, 6, 194, 138, 6, - 72, 171, 167, 5, 84, 64, 96, 7, 76, 69, 84, 84, 69, 82, 32, 229, 160, 3, - 11, 83, 89, 77, 66, 79, 76, 32, 86, 73, 68, 74, 60, 174, 1, 66, 2, 82, - 22, 78, 30, 83, 38, 84, 174, 223, 14, 72, 138, 176, 1, 75, 254, 68, 68, - 2, 76, 2, 77, 2, 80, 2, 81, 2, 87, 2, 89, 186, 2, 65, 2, 69, 2, 73, 3, - 79, 4, 179, 159, 3, 65, 8, 158, 159, 3, 65, 3, 69, 6, 130, 159, 3, 65, - 139, 184, 13, 69, 10, 222, 158, 3, 65, 2, 69, 139, 184, 13, 79, 2, 175, - 252, 11, 73, 20, 44, 5, 73, 67, 65, 76, 32, 251, 213, 16, 79, 18, 116, - 10, 76, 79, 78, 71, 32, 79, 86, 69, 82, 32, 74, 80, 26, 84, 168, 1, 7, - 83, 72, 79, 82, 84, 32, 79, 187, 32, 66, 4, 188, 244, 2, 2, 83, 72, 185, - 180, 10, 7, 84, 87, 79, 32, 83, 72, 79, 2, 109, 3, 69, 78, 84, 8, 66, 69, - 34, 82, 41, 10, 87, 79, 32, 83, 72, 79, 82, 84, 83, 32, 2, 17, 2, 84, 82, - 2, 23, 65, 2, 11, 73, 2, 217, 187, 15, 2, 83, 69, 4, 26, 79, 179, 231, 7, - 74, 2, 137, 162, 10, 4, 86, 69, 82, 32, 224, 2, 100, 3, 65, 79, 32, 136, - 18, 2, 67, 82, 142, 1, 68, 118, 76, 130, 1, 78, 245, 2, 4, 82, 82, 79, - 82, 170, 2, 156, 1, 7, 76, 69, 84, 84, 69, 82, 32, 180, 10, 5, 83, 73, - 71, 78, 32, 212, 1, 5, 84, 79, 78, 69, 32, 69, 11, 86, 79, 87, 69, 76, - 32, 83, 73, 71, 78, 32, 178, 1, 214, 1, 65, 110, 66, 34, 68, 106, 71, 34, - 76, 50, 78, 106, 82, 170, 1, 83, 54, 84, 218, 1, 86, 32, 3, 89, 73, 32, - 118, 90, 238, 247, 12, 81, 230, 200, 1, 80, 242, 190, 1, 72, 2, 77, 254, - 68, 70, 2, 75, 2, 87, 3, 88, 10, 52, 7, 82, 67, 72, 65, 73, 67, 32, 179, - 206, 16, 72, 8, 254, 231, 8, 90, 170, 151, 4, 78, 211, 204, 3, 77, 4, - 162, 186, 16, 82, 219, 19, 65, 16, 50, 90, 154, 4, 76, 218, 198, 16, 68, - 187, 2, 65, 8, 206, 185, 16, 89, 162, 17, 72, 2, 90, 187, 2, 65, 6, 190, - 133, 16, 72, 183, 71, 65, 8, 238, 150, 10, 72, 174, 179, 6, 89, 187, 2, - 65, 18, 54, 65, 186, 132, 16, 71, 2, 78, 2, 89, 255, 68, 72, 5, 11, 83, - 2, 169, 159, 13, 4, 65, 76, 73, 90, 16, 60, 9, 69, 70, 79, 82, 77, 69, - 68, 32, 84, 171, 180, 16, 84, 14, 40, 4, 79, 78, 69, 45, 151, 211, 14, - 83, 12, 206, 202, 16, 49, 2, 50, 2, 52, 2, 53, 2, 54, 3, 56, 8, 186, 182, - 16, 89, 162, 17, 72, 2, 83, 187, 2, 65, 32, 78, 76, 20, 4, 79, 78, 69, - 45, 70, 83, 130, 198, 16, 84, 186, 2, 65, 3, 69, 4, 171, 147, 10, 72, 14, - 250, 200, 16, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 8, 254, 197, - 16, 72, 2, 83, 186, 2, 65, 3, 69, 4, 206, 197, 16, 70, 187, 2, 65, 16, - 70, 84, 228, 207, 14, 2, 68, 90, 250, 60, 78, 142, 184, 1, 75, 3, 80, 8, - 234, 255, 15, 83, 254, 68, 84, 187, 2, 65, 16, 50, 90, 142, 255, 15, 83, - 254, 68, 72, 187, 2, 65, 8, 218, 144, 10, 83, 174, 179, 6, 89, 187, 2, - 65, 8, 144, 1, 9, 82, 69, 70, 79, 82, 77, 69, 68, 32, 34, 65, 145, 245, - 13, 18, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 77, 79, 68, 73, 70, 73, - 69, 82, 4, 30, 65, 189, 88, 2, 86, 79, 2, 225, 230, 11, 3, 83, 80, 73, 8, - 130, 172, 14, 66, 168, 74, 3, 84, 79, 80, 182, 81, 65, 143, 82, 82, 104, - 146, 1, 65, 78, 69, 62, 73, 78, 79, 74, 85, 74, 89, 180, 250, 6, 9, 82, - 79, 85, 78, 68, 69, 68, 32, 69, 206, 134, 7, 87, 238, 70, 78, 135, 122, - 86, 19, 206, 202, 14, 78, 130, 184, 1, 69, 134, 63, 72, 146, 1, 65, 2, - 73, 3, 85, 15, 242, 222, 12, 82, 146, 235, 1, 78, 150, 248, 1, 65, 3, 73, - 23, 186, 201, 14, 65, 54, 79, 154, 247, 1, 78, 86, 69, 2, 71, 2, 73, 3, - 85, 13, 42, 69, 230, 192, 16, 71, 2, 79, 3, 85, 4, 226, 192, 16, 82, 3, - 89, 19, 166, 200, 14, 65, 202, 228, 1, 69, 134, 19, 78, 2, 79, 86, 73, 3, - 85, 7, 166, 172, 16, 85, 219, 19, 73, 12, 18, 32, 63, 79, 4, 172, 172, - 15, 4, 79, 78, 32, 85, 13, 4, 68, 65, 83, 72, 8, 190, 189, 11, 83, 158, - 216, 3, 80, 218, 34, 32, 139, 112, 66, 6, 64, 4, 68, 76, 69, 32, 173, - 215, 4, 6, 76, 73, 78, 69, 32, 72, 4, 152, 180, 5, 3, 84, 72, 73, 203, - 185, 10, 68, 8, 76, 6, 73, 84, 65, 82, 89, 32, 172, 153, 10, 3, 75, 89, - 32, 255, 156, 5, 76, 4, 26, 72, 187, 134, 12, 77, 2, 163, 170, 4, 69, 24, - 42, 73, 76, 2, 85, 83, 235, 187, 16, 89, 6, 34, 68, 22, 77, 147, 185, 14, - 66, 2, 195, 137, 6, 73, 2, 187, 212, 6, 73, 16, 46, 32, 201, 142, 10, 5, - 45, 79, 82, 45, 80, 14, 40, 4, 83, 73, 71, 78, 231, 249, 12, 84, 13, 11, - 32, 10, 44, 5, 87, 73, 84, 72, 32, 239, 153, 6, 73, 8, 66, 67, 182, 195, - 4, 68, 206, 135, 8, 82, 21, 4, 70, 65, 76, 76, 2, 129, 195, 11, 3, 79, - 77, 77, 5, 163, 181, 14, 32, 182, 9, 184, 1, 10, 66, 73, 76, 69, 32, 80, - 72, 79, 78, 69, 166, 1, 68, 190, 76, 78, 174, 27, 79, 156, 1, 3, 83, 81, - 85, 38, 84, 250, 1, 85, 242, 140, 11, 89, 217, 178, 2, 5, 86, 73, 69, 32, + 248, 17, 4, 53, 32, 78, 71, 204, 208, 2, 2, 55, 32, 184, 151, 2, 3, 50, + 32, 75, 128, 136, 10, 3, 52, 32, 87, 229, 132, 2, 2, 49, 32, 2, 227, 165, + 16, 32, 2, 11, 32, 2, 151, 179, 17, 74, 2, 203, 192, 16, 32, 2, 11, 32, + 2, 227, 178, 17, 87, 2, 151, 185, 15, 78, 20, 226, 1, 50, 32, 2, 51, 32, + 128, 4, 3, 52, 32, 71, 254, 9, 48, 148, 240, 9, 4, 56, 32, 72, 79, 156, + 220, 2, 5, 55, 32, 78, 71, 71, 224, 88, 2, 53, 32, 220, 55, 2, 57, 32, + 236, 236, 1, 3, 54, 32, 87, 245, 150, 1, 2, 49, 32, 2, 11, 32, 2, 171, + 155, 13, 77, 2, 11, 78, 2, 167, 179, 17, 68, 20, 200, 3, 2, 56, 32, 252, + 4, 3, 52, 32, 78, 216, 243, 4, 3, 50, 32, 75, 210, 210, 2, 49, 2, 53, + 236, 174, 2, 2, 55, 32, 232, 129, 5, 3, 51, 32, 70, 0, 3, 54, 32, 83, + 236, 53, 2, 48, 32, 233, 148, 1, 3, 57, 32, 87, 20, 236, 1, 8, 50, 32, + 76, 79, 78, 71, 32, 77, 20, 3, 53, 32, 77, 22, 54, 224, 2, 3, 57, 32, 78, + 174, 219, 6, 55, 220, 155, 3, 3, 51, 32, 87, 236, 224, 2, 2, 48, 32, 176, + 60, 3, 56, 32, 71, 204, 228, 1, 3, 49, 32, 89, 1, 3, 52, 32, 86, 2, 171, + 164, 17, 66, 2, 179, 175, 17, 66, 2, 213, 141, 16, 3, 32, 78, 71, 20, + 192, 1, 2, 50, 32, 34, 52, 26, 53, 34, 54, 36, 2, 55, 32, 188, 7, 2, 48, + 32, 244, 250, 9, 3, 56, 32, 75, 128, 193, 2, 2, 49, 32, 156, 138, 3, 5, + 57, 32, 78, 71, 71, 253, 207, 1, 2, 51, 32, 2, 11, 78, 2, 207, 190, 17, + 74, 2, 165, 5, 2, 32, 77, 2, 11, 32, 2, 247, 172, 17, 71, 2, 145, 249, + 14, 4, 32, 78, 71, 71, 2, 223, 208, 15, 74, 20, 220, 1, 2, 48, 32, 20, 6, + 49, 32, 76, 79, 78, 71, 30, 56, 156, 2, 2, 55, 32, 220, 192, 9, 3, 52, + 32, 78, 236, 49, 4, 54, 32, 71, 85, 232, 129, 5, 2, 53, 32, 240, 87, 3, + 50, 32, 83, 0, 2, 51, 32, 165, 101, 2, 57, 32, 2, 251, 246, 14, 74, 2, + 241, 157, 12, 2, 32, 77, 2, 171, 206, 12, 32, 24, 198, 1, 50, 2, 52, 36, + 6, 53, 32, 76, 79, 78, 71, 28, 3, 55, 32, 78, 34, 56, 184, 236, 11, 2, + 54, 32, 160, 98, 3, 57, 32, 75, 168, 253, 2, 3, 51, 32, 86, 136, 111, 4, + 48, 32, 78, 89, 219, 53, 49, 4, 181, 246, 14, 4, 32, 77, 66, 79, 2, 253, + 156, 17, 2, 32, 74, 2, 11, 71, 2, 203, 241, 15, 85, 2, 207, 231, 15, 32, + 20, 212, 1, 2, 48, 32, 22, 49, 22, 50, 20, 6, 51, 32, 76, 79, 78, 71, 34, + 56, 216, 136, 9, 2, 53, 32, 252, 128, 1, 3, 52, 32, 78, 168, 193, 2, 2, + 54, 32, 236, 251, 2, 4, 55, 32, 77, 66, 161, 30, 4, 57, 32, 77, 85, 2, + 163, 242, 14, 68, 2, 139, 136, 10, 32, 2, 239, 216, 10, 32, 2, 165, 172, + 15, 3, 32, 78, 71, 2, 17, 2, 32, 77, 2, 163, 201, 15, 66, 16, 142, 1, 48, + 32, 3, 49, 32, 78, 20, 3, 50, 32, 78, 20, 2, 51, 32, 20, 3, 52, 32, 87, + 22, 53, 20, 2, 54, 32, 213, 201, 12, 3, 55, 32, 70, 2, 11, 32, 2, 243, + 199, 15, 71, 2, 223, 199, 15, 68, 2, 155, 146, 17, 74, 2, 131, 231, 16, + 72, 2, 131, 163, 17, 85, 2, 199, 236, 15, 32, 2, 255, 144, 1, 83, 248, 1, + 72, 6, 79, 73, 84, 73, 67, 32, 204, 254, 10, 2, 67, 85, 179, 183, 2, 80, + 244, 1, 104, 8, 67, 85, 82, 83, 73, 86, 69, 32, 221, 10, 13, 72, 73, 69, + 82, 79, 71, 76, 89, 80, 72, 73, 67, 32, 180, 1, 96, 9, 70, 82, 65, 67, + 84, 73, 79, 78, 32, 250, 2, 76, 133, 3, 7, 78, 85, 77, 66, 69, 82, 32, + 24, 78, 69, 50, 70, 44, 4, 79, 78, 69, 32, 46, 83, 50, 84, 61, 3, 78, 73, + 78, 4, 160, 1, 2, 76, 69, 93, 4, 73, 71, 72, 84, 4, 192, 1, 2, 73, 86, + 13, 3, 79, 85, 82, 4, 140, 175, 15, 4, 84, 87, 69, 76, 19, 72, 4, 26, 69, + 93, 2, 73, 88, 2, 65, 2, 86, 69, 6, 46, 69, 12, 3, 72, 82, 69, 13, 2, 87, + 79, 2, 23, 78, 2, 11, 69, 2, 237, 176, 15, 5, 32, 84, 87, 69, 76, 52, 76, + 6, 69, 84, 84, 69, 82, 32, 137, 2, 8, 79, 71, 79, 71, 82, 65, 77, 32, 48, + 182, 1, 65, 46, 84, 182, 235, 1, 78, 2, 83, 218, 200, 13, 72, 130, 179, + 1, 75, 138, 69, 66, 2, 68, 2, 76, 2, 77, 2, 80, 2, 81, 2, 82, 2, 87, 2, + 89, 186, 2, 69, 2, 73, 3, 79, 5, 253, 176, 11, 6, 82, 67, 72, 65, 73, 67, + 6, 202, 174, 17, 65, 2, 69, 3, 79, 4, 138, 209, 4, 73, 241, 192, 12, 2, + 82, 77, 104, 92, 5, 69, 73, 71, 72, 84, 30, 70, 92, 4, 78, 73, 78, 69, + 54, 83, 78, 84, 73, 2, 79, 78, 11, 150, 1, 89, 223, 1, 32, 24, 18, 73, + 35, 79, 12, 142, 2, 86, 207, 235, 3, 70, 12, 148, 2, 2, 85, 82, 167, 235, + 3, 82, 11, 28, 2, 84, 89, 223, 1, 32, 2, 203, 160, 6, 32, 24, 40, 4, 69, + 86, 69, 78, 1, 2, 73, 88, 13, 154, 1, 32, 167, 235, 3, 84, 28, 34, 72, + 50, 87, 131, 159, 6, 69, 12, 32, 2, 82, 69, 155, 235, 3, 73, 8, 39, 69, + 12, 26, 79, 155, 235, 3, 69, 9, 11, 32, 6, 166, 157, 6, 72, 235, 185, 5, + 84, 64, 96, 7, 76, 69, 84, 84, 69, 82, 32, 141, 167, 3, 11, 83, 89, 77, + 66, 79, 76, 32, 86, 73, 68, 74, 60, 174, 1, 66, 2, 82, 22, 78, 30, 83, + 38, 84, 222, 172, 15, 72, 130, 179, 1, 75, 138, 69, 68, 2, 76, 2, 77, 2, + 80, 2, 81, 2, 87, 2, 89, 186, 2, 65, 2, 69, 2, 73, 3, 79, 4, 219, 165, 3, + 65, 8, 198, 165, 3, 65, 3, 69, 6, 170, 165, 3, 65, 151, 130, 14, 69, 10, + 134, 165, 3, 65, 2, 69, 151, 130, 14, 79, 2, 215, 160, 12, 73, 20, 44, 5, + 73, 67, 65, 76, 32, 175, 166, 17, 79, 18, 116, 10, 76, 79, 78, 71, 32, + 79, 86, 69, 82, 32, 74, 80, 26, 84, 168, 1, 7, 83, 72, 79, 82, 84, 32, + 79, 215, 32, 66, 4, 156, 246, 2, 2, 83, 72, 201, 237, 10, 7, 84, 87, 79, + 32, 83, 72, 79, 2, 109, 3, 69, 78, 84, 8, 66, 69, 34, 82, 41, 10, 87, 79, + 32, 83, 72, 79, 82, 84, 83, 32, 2, 17, 2, 84, 82, 2, 23, 65, 2, 11, 73, + 2, 245, 139, 16, 2, 83, 69, 4, 26, 79, 163, 254, 7, 74, 2, 129, 190, 10, + 4, 86, 69, 82, 32, 230, 2, 104, 3, 65, 79, 32, 136, 18, 2, 67, 82, 142, + 1, 68, 142, 1, 76, 130, 1, 78, 245, 2, 4, 82, 82, 79, 82, 170, 2, 156, 1, + 7, 76, 69, 84, 84, 69, 82, 32, 180, 10, 5, 83, 73, 71, 78, 32, 212, 1, 5, + 84, 79, 78, 69, 32, 69, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, + 178, 1, 214, 1, 65, 110, 66, 34, 68, 106, 71, 34, 76, 50, 78, 106, 82, + 170, 1, 83, 54, 84, 218, 1, 86, 32, 3, 89, 73, 32, 118, 90, 246, 165, 13, + 81, 254, 231, 1, 80, 246, 193, 1, 72, 2, 77, 138, 69, 70, 2, 75, 2, 87, + 3, 88, 10, 52, 7, 82, 67, 72, 65, 73, 67, 32, 227, 158, 17, 72, 8, 210, + 254, 8, 90, 222, 174, 4, 78, 251, 238, 3, 77, 4, 210, 138, 17, 82, 219, + 19, 65, 16, 50, 90, 154, 4, 76, 138, 151, 17, 68, 187, 2, 65, 8, 254, + 137, 17, 89, 162, 17, 72, 2, 90, 187, 2, 65, 6, 226, 213, 16, 72, 195, + 71, 65, 8, 222, 178, 10, 72, 238, 231, 6, 89, 187, 2, 65, 18, 54, 65, + 222, 212, 16, 71, 2, 78, 2, 89, 139, 69, 72, 5, 11, 83, 2, 149, 218, 13, + 4, 65, 76, 73, 90, 16, 60, 9, 69, 70, 79, 82, 77, 69, 68, 32, 84, 219, + 132, 17, 84, 14, 40, 4, 79, 78, 69, 45, 195, 160, 15, 83, 12, 254, 154, + 17, 49, 2, 50, 2, 52, 2, 53, 2, 54, 3, 56, 8, 234, 134, 17, 89, 162, 17, + 72, 2, 83, 187, 2, 65, 32, 78, 76, 20, 4, 79, 78, 69, 45, 70, 83, 178, + 150, 17, 84, 186, 2, 65, 3, 69, 4, 155, 175, 10, 72, 14, 170, 153, 17, + 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 8, 174, 150, 17, 72, 2, 83, + 186, 2, 65, 3, 69, 4, 254, 149, 17, 70, 187, 2, 65, 16, 70, 84, 144, 157, + 15, 2, 68, 90, 190, 62, 78, 206, 185, 1, 75, 3, 80, 8, 142, 208, 16, 83, + 138, 69, 84, 187, 2, 65, 16, 50, 90, 178, 207, 16, 83, 138, 69, 72, 187, + 2, 65, 8, 202, 172, 10, 83, 238, 231, 6, 89, 187, 2, 65, 8, 144, 1, 9, + 82, 69, 70, 79, 82, 77, 69, 68, 32, 34, 65, 189, 243, 15, 18, 67, 79, 78, + 83, 79, 78, 65, 78, 84, 32, 77, 79, 68, 73, 70, 73, 69, 82, 4, 30, 65, + 213, 88, 2, 86, 79, 2, 133, 139, 12, 3, 83, 80, 73, 8, 218, 246, 14, 66, + 140, 77, 3, 84, 79, 80, 154, 84, 65, 159, 82, 82, 104, 146, 1, 65, 78, + 69, 62, 73, 78, 79, 74, 85, 74, 89, 160, 145, 7, 9, 82, 79, 85, 78, 68, + 69, 68, 32, 69, 178, 185, 7, 87, 202, 74, 78, 251, 124, 86, 19, 250, 151, + 15, 78, 250, 186, 1, 69, 146, 63, 72, 146, 1, 65, 2, 73, 3, 85, 15, 254, + 191, 13, 82, 178, 215, 1, 78, 154, 251, 1, 65, 3, 73, 23, 230, 150, 15, + 65, 54, 79, 158, 250, 1, 78, 86, 69, 2, 71, 2, 73, 3, 85, 13, 42, 69, + 150, 145, 17, 71, 2, 79, 3, 85, 4, 146, 145, 17, 82, 3, 89, 19, 210, 149, + 15, 65, 206, 231, 1, 69, 134, 19, 78, 2, 79, 86, 73, 3, 85, 7, 214, 252, + 16, 85, 219, 19, 73, 12, 18, 32, 63, 79, 4, 196, 252, 15, 4, 79, 78, 32, + 85, 13, 4, 68, 65, 83, 72, 8, 206, 226, 11, 83, 162, 130, 4, 80, 222, 35, + 32, 163, 112, 66, 12, 64, 4, 68, 76, 69, 32, 189, 223, 4, 6, 76, 73, 78, + 69, 32, 72, 10, 184, 192, 5, 3, 84, 72, 73, 238, 217, 8, 76, 22, 82, 207, + 163, 2, 68, 8, 76, 6, 73, 84, 65, 82, 89, 32, 232, 188, 10, 3, 75, 89, + 32, 195, 201, 5, 76, 4, 26, 72, 199, 170, 12, 77, 2, 227, 177, 4, 69, 24, + 42, 73, 76, 2, 85, 83, 131, 140, 17, 89, 6, 34, 68, 22, 77, 155, 134, 15, + 66, 2, 219, 156, 6, 73, 2, 223, 235, 6, 73, 16, 46, 32, 205, 177, 10, 5, + 45, 79, 82, 45, 80, 14, 40, 4, 83, 73, 71, 78, 203, 167, 13, 84, 13, 11, + 32, 10, 44, 5, 87, 73, 84, 72, 32, 135, 173, 6, 73, 8, 66, 67, 250, 202, + 4, 68, 234, 163, 8, 82, 21, 4, 70, 65, 76, 76, 2, 177, 231, 11, 3, 79, + 77, 77, 5, 171, 130, 15, 32, 184, 9, 184, 1, 10, 66, 73, 76, 69, 32, 80, + 72, 79, 78, 69, 166, 1, 68, 190, 76, 78, 170, 27, 79, 172, 1, 3, 83, 81, + 85, 38, 84, 250, 1, 85, 242, 176, 11, 89, 209, 219, 2, 5, 86, 73, 69, 32, 67, 7, 11, 32, 4, 108, 21, 87, 73, 84, 72, 32, 82, 73, 71, 72, 84, 87, - 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 255, 177, 1, 79, 2, 11, 32, 2, - 213, 132, 16, 2, 65, 84, 154, 6, 58, 69, 74, 73, 149, 75, 7, 85, 76, 79, - 32, 84, 87, 79, 4, 196, 230, 15, 10, 82, 78, 32, 80, 69, 78, 84, 65, 84, + 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 167, 178, 1, 79, 2, 11, 32, 2, + 225, 212, 16, 2, 65, 84, 154, 6, 58, 69, 74, 73, 149, 75, 7, 85, 76, 79, + 32, 84, 87, 79, 4, 208, 182, 16, 10, 82, 78, 32, 80, 69, 78, 84, 65, 84, 72, 159, 18, 76, 148, 6, 42, 32, 221, 1, 5, 70, 73, 69, 82, 32, 158, 1, - 88, 5, 83, 73, 71, 78, 32, 210, 172, 3, 86, 178, 146, 3, 68, 182, 1, 76, - 167, 180, 4, 65, 10, 42, 65, 234, 189, 8, 72, 203, 179, 7, 86, 4, 44, 5, - 82, 68, 72, 65, 67, 163, 239, 15, 78, 2, 201, 239, 15, 3, 65, 78, 68, + 88, 5, 83, 73, 71, 78, 32, 230, 179, 3, 86, 190, 162, 3, 68, 186, 1, 76, + 195, 194, 4, 65, 10, 42, 65, 166, 212, 8, 72, 155, 237, 7, 86, 4, 44, 5, + 82, 68, 72, 65, 67, 175, 191, 16, 78, 2, 213, 191, 16, 3, 65, 78, 68, 246, 4, 92, 12, 66, 82, 69, 86, 69, 32, 87, 73, 84, 72, 32, 73, 89, 7, 76, 69, 84, 84, 69, 82, 32, 2, 33, 6, 78, 86, 69, 82, 84, 69, 2, 21, 3, - 68, 32, 66, 2, 249, 181, 15, 2, 82, 69, 244, 4, 198, 1, 65, 82, 66, 66, - 67, 190, 13, 68, 194, 1, 69, 182, 2, 71, 82, 72, 46, 76, 230, 4, 77, 216, - 3, 7, 79, 80, 69, 78, 32, 83, 72, 22, 80, 38, 82, 182, 4, 83, 214, 33, - 84, 114, 85, 118, 86, 63, 89, 6, 38, 76, 150, 27, 67, 183, 167, 10, 80, - 2, 237, 19, 6, 86, 69, 79, 76, 65, 82, 6, 176, 19, 5, 73, 76, 65, 66, 73, - 201, 45, 4, 69, 71, 73, 78, 160, 1, 240, 1, 7, 65, 80, 73, 84, 65, 76, + 68, 32, 66, 2, 129, 134, 16, 2, 82, 69, 244, 4, 198, 1, 65, 82, 66, 66, + 67, 190, 13, 68, 202, 1, 69, 182, 2, 71, 82, 72, 46, 76, 230, 4, 77, 216, + 3, 7, 79, 80, 69, 78, 32, 83, 72, 22, 80, 38, 82, 182, 4, 83, 206, 33, + 84, 114, 85, 118, 86, 63, 89, 6, 38, 76, 158, 27, 67, 251, 204, 10, 80, + 2, 245, 19, 6, 86, 69, 79, 76, 65, 82, 6, 184, 19, 5, 73, 76, 65, 66, 73, + 193, 45, 4, 69, 71, 73, 78, 160, 1, 240, 1, 7, 65, 80, 73, 84, 65, 76, 32, 192, 2, 7, 69, 78, 84, 82, 69, 68, 32, 100, 13, 72, 73, 78, 69, 83, 69, 32, 84, 79, 78, 69, 32, 89, 108, 8, 89, 82, 73, 76, 76, 73, 67, 32, - 166, 188, 10, 73, 244, 2, 4, 82, 79, 83, 83, 191, 154, 5, 79, 56, 206, 1, - 66, 42, 82, 250, 26, 72, 198, 193, 12, 79, 138, 143, 3, 65, 150, 64, 67, + 242, 225, 10, 73, 244, 2, 4, 82, 79, 83, 83, 255, 196, 5, 79, 56, 206, 1, + 66, 42, 82, 130, 27, 72, 202, 141, 13, 79, 138, 147, 3, 65, 162, 64, 67, 2, 68, 2, 69, 2, 70, 2, 71, 2, 73, 2, 74, 2, 75, 2, 76, 2, 77, 2, 78, 2, - 80, 2, 81, 2, 84, 2, 85, 2, 86, 3, 87, 5, 237, 130, 6, 5, 65, 82, 82, 69, - 68, 7, 41, 8, 69, 86, 69, 82, 83, 69, 68, 32, 4, 170, 171, 16, 69, 3, 78, + 80, 2, 81, 2, 84, 2, 85, 2, 86, 3, 87, 5, 133, 150, 6, 5, 65, 82, 82, 69, + 68, 7, 41, 8, 69, 86, 69, 82, 83, 69, 68, 32, 4, 194, 251, 16, 69, 3, 78, 4, 30, 76, 21, 3, 82, 73, 71, 2, 29, 2, 69, 70, 2, 11, 72, 2, 11, 84, 2, - 173, 26, 2, 32, 72, 16, 36, 3, 65, 78, 71, 1, 2, 73, 78, 8, 11, 32, 8, - 170, 226, 11, 83, 214, 246, 3, 80, 146, 44, 81, 3, 82, 78, 34, 72, 30, - 83, 175, 217, 15, 69, 2, 197, 195, 14, 2, 65, 82, 74, 40, 5, 77, 65, 76, + 181, 26, 2, 32, 72, 16, 36, 3, 65, 78, 71, 1, 2, 73, 78, 8, 11, 32, 8, + 182, 134, 12, 83, 214, 162, 4, 80, 158, 44, 81, 3, 82, 78, 34, 72, 30, + 83, 187, 169, 16, 69, 2, 217, 144, 15, 2, 65, 82, 74, 40, 5, 77, 65, 76, 76, 32, 183, 6, 79, 72, 186, 1, 66, 138, 1, 68, 38, 69, 150, 1, 80, 58, - 83, 94, 84, 34, 89, 158, 163, 15, 90, 246, 63, 73, 138, 19, 67, 2, 71, + 83, 94, 84, 34, 89, 158, 243, 15, 90, 130, 64, 73, 150, 19, 67, 2, 71, 186, 22, 74, 2, 86, 158, 20, 72, 2, 75, 186, 2, 65, 2, 79, 3, 85, 6, 38, - 89, 186, 27, 65, 187, 139, 16, 69, 2, 221, 201, 1, 19, 69, 76, 79, 82, - 85, 83, 83, 73, 65, 78, 45, 85, 75, 82, 65, 73, 78, 73, 65, 4, 206, 190, - 6, 90, 195, 231, 9, 69, 15, 50, 83, 186, 165, 16, 70, 2, 76, 2, 77, 3, - 82, 5, 25, 4, 32, 87, 73, 84, 2, 21, 3, 72, 32, 68, 2, 11, 69, 2, 205, - 223, 12, 3, 83, 67, 69, 4, 26, 65, 191, 164, 16, 69, 2, 241, 247, 15, 2, - 76, 79, 8, 42, 84, 198, 172, 1, 67, 251, 244, 14, 72, 4, 145, 19, 8, 82, - 65, 73, 71, 72, 84, 32, 85, 4, 238, 140, 16, 83, 215, 22, 69, 6, 36, 3, - 69, 82, 85, 255, 162, 16, 85, 5, 37, 7, 32, 87, 73, 84, 72, 32, 66, 2, - 21, 3, 65, 67, 75, 2, 217, 228, 15, 2, 32, 89, 2, 191, 139, 11, 70, 16, - 18, 69, 27, 79, 2, 161, 5, 2, 78, 84, 14, 64, 2, 84, 32, 44, 5, 85, 66, - 76, 69, 32, 137, 52, 2, 87, 78, 6, 178, 192, 9, 83, 194, 144, 4, 72, 43, - 86, 4, 254, 135, 2, 65, 215, 240, 2, 80, 24, 40, 5, 88, 84, 82, 65, 45, - 139, 49, 78, 20, 52, 5, 72, 73, 71, 72, 32, 81, 4, 76, 79, 87, 32, 10, - 156, 1, 9, 69, 88, 84, 82, 65, 45, 76, 79, 87, 154, 7, 68, 70, 76, 79, - 84, 10, 76, 10, 69, 88, 84, 82, 65, 45, 72, 73, 71, 72, 154, 7, 68, 70, - 76, 79, 84, 2, 145, 8, 8, 32, 67, 79, 78, 84, 79, 85, 82, 8, 162, 9, 82, - 226, 3, 76, 237, 238, 14, 9, 69, 79, 82, 71, 73, 65, 78, 32, 78, 10, 248, - 5, 4, 73, 71, 72, 32, 135, 41, 65, 44, 46, 65, 84, 2, 79, 87, 201, 11, 2, - 69, 70, 2, 21, 3, 84, 69, 82, 2, 17, 2, 65, 76, 2, 17, 2, 32, 67, 2, 211, - 237, 14, 76, 36, 86, 32, 245, 231, 11, 15, 69, 82, 32, 82, 73, 71, 72, - 84, 32, 67, 79, 82, 78, 69, 82, 34, 158, 1, 67, 20, 2, 68, 79, 40, 4, 76, - 69, 70, 84, 82, 77, 12, 2, 82, 73, 50, 84, 178, 3, 65, 42, 71, 170, 2, - 73, 136, 196, 13, 2, 85, 80, 235, 180, 1, 86, 2, 131, 173, 10, 73, 6, - 238, 2, 84, 205, 200, 13, 2, 87, 78, 6, 28, 2, 32, 65, 247, 2, 45, 4, 25, - 4, 82, 82, 79, 87, 5, 151, 203, 13, 72, 2, 111, 65, 4, 180, 202, 13, 3, - 71, 72, 84, 255, 205, 2, 78, 4, 194, 2, 79, 155, 128, 14, 73, 18, 18, 65, - 23, 73, 2, 151, 244, 14, 67, 16, 26, 68, 163, 254, 12, 78, 14, 38, 32, - 217, 1, 4, 68, 76, 69, 32, 8, 26, 68, 70, 76, 79, 84, 4, 17, 2, 79, 84, - 4, 25, 4, 84, 69, 68, 32, 4, 18, 76, 79, 84, 2, 25, 4, 69, 70, 84, 45, 2, - 25, 4, 83, 84, 69, 77, 2, 17, 2, 32, 84, 2, 11, 79, 2, 11, 78, 2, 187, - 198, 13, 69, 6, 40, 6, 68, 79, 85, 66, 76, 69, 75, 71, 4, 11, 32, 4, 18, - 65, 43, 71, 2, 11, 67, 2, 229, 170, 10, 2, 85, 84, 2, 11, 82, 2, 179, - 170, 10, 65, 2, 207, 148, 14, 69, 4, 246, 250, 12, 76, 247, 129, 2, 82, - 26, 104, 6, 65, 73, 83, 69, 68, 32, 150, 1, 69, 216, 1, 3, 73, 71, 72, - 173, 222, 8, 5, 72, 79, 84, 73, 67, 10, 70, 68, 30, 73, 234, 186, 9, 69, - 180, 189, 5, 2, 85, 80, 199, 74, 67, 2, 129, 197, 14, 2, 79, 87, 2, 209, - 186, 9, 7, 78, 86, 69, 82, 84, 69, 68, 8, 104, 7, 86, 69, 82, 83, 69, 68, - 32, 133, 28, 14, 84, 82, 79, 70, 76, 69, 88, 32, 67, 76, 73, 67, 75, 32, - 6, 26, 71, 255, 197, 13, 67, 4, 11, 76, 4, 49, 10, 79, 84, 84, 65, 76, - 32, 83, 84, 79, 80, 5, 167, 19, 32, 6, 17, 2, 84, 32, 6, 38, 72, 250, - 133, 13, 84, 231, 59, 65, 2, 185, 168, 7, 3, 65, 76, 70, 156, 2, 138, 1, - 72, 48, 5, 77, 65, 76, 76, 32, 156, 31, 8, 84, 82, 69, 83, 83, 32, 65, - 78, 53, 11, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 4, 216, 130, 8, - 3, 79, 82, 84, 251, 139, 6, 69, 144, 2, 154, 2, 65, 46, 66, 130, 1, 67, - 238, 2, 68, 238, 2, 90, 66, 69, 46, 70, 30, 71, 110, 72, 102, 77, 34, 73, - 34, 74, 98, 76, 176, 3, 7, 78, 32, 87, 73, 84, 72, 32, 30, 79, 94, 80, - 22, 82, 146, 2, 83, 190, 1, 84, 130, 8, 85, 142, 1, 86, 134, 240, 15, 75, - 2, 81, 2, 87, 2, 88, 3, 89, 9, 238, 23, 76, 210, 164, 15, 73, 215, 79, - 69, 11, 46, 65, 50, 79, 222, 8, 32, 227, 221, 15, 69, 2, 17, 2, 82, 82, - 2, 181, 227, 5, 2, 69, 68, 2, 229, 20, 4, 84, 84, 79, 77, 41, 100, 7, 65, - 80, 73, 84, 65, 76, 32, 180, 1, 6, 76, 79, 83, 69, 68, 32, 190, 17, 32, - 187, 227, 15, 72, 30, 114, 73, 214, 6, 71, 238, 16, 76, 250, 218, 15, 79, - 158, 20, 65, 186, 2, 66, 2, 72, 2, 78, 2, 82, 2, 85, 3, 89, 7, 22, 78, - 191, 11, 32, 2, 237, 220, 5, 5, 86, 69, 82, 84, 69, 4, 26, 82, 175, 136, - 9, 79, 2, 229, 20, 9, 69, 86, 69, 82, 83, 69, 68, 32, 79, 23, 104, 6, 32, - 87, 73, 84, 72, 32, 86, 69, 32, 9, 79, 84, 76, 69, 83, 83, 32, 74, 32, - 105, 3, 90, 32, 68, 6, 26, 72, 247, 217, 13, 84, 4, 21, 3, 79, 79, 75, 5, - 201, 160, 13, 3, 32, 65, 78, 4, 238, 15, 90, 131, 210, 15, 76, 4, 33, 6, - 87, 73, 84, 72, 32, 83, 4, 29, 5, 84, 82, 79, 75, 69, 5, 177, 211, 8, 4, - 32, 65, 78, 68, 6, 33, 6, 73, 71, 82, 65, 80, 72, 7, 33, 6, 32, 87, 73, - 84, 72, 32, 4, 138, 14, 67, 219, 4, 82, 11, 190, 131, 16, 83, 2, 84, 2, - 90, 63, 78, 5, 225, 13, 3, 69, 78, 71, 11, 56, 5, 82, 69, 69, 75, 32, - 162, 1, 32, 151, 183, 13, 65, 4, 26, 71, 195, 223, 10, 80, 2, 151, 184, - 13, 65, 11, 40, 6, 32, 87, 73, 84, 72, 32, 39, 69, 4, 142, 178, 14, 72, - 231, 160, 1, 83, 4, 17, 2, 78, 71, 5, 11, 32, 2, 239, 207, 8, 87, 4, 222, - 4, 32, 239, 216, 15, 79, 5, 37, 7, 32, 87, 73, 84, 72, 32, 67, 2, 11, 82, - 2, 181, 212, 13, 6, 79, 83, 83, 69, 68, 45, 25, 116, 6, 32, 87, 73, 84, - 72, 32, 226, 9, 83, 2, 90, 112, 2, 69, 90, 241, 222, 15, 8, 73, 71, 65, - 84, 85, 82, 69, 32, 12, 54, 73, 82, 77, 82, 82, 222, 6, 80, 187, 237, 3, - 66, 2, 49, 10, 78, 86, 69, 82, 84, 69, 68, 32, 76, 65, 2, 189, 172, 12, - 2, 90, 89, 2, 11, 73, 2, 17, 2, 68, 68, 2, 17, 2, 76, 69, 2, 185, 189, - 12, 2, 32, 84, 4, 61, 13, 69, 84, 82, 79, 70, 76, 69, 88, 32, 72, 79, 79, - 75, 5, 217, 12, 4, 32, 65, 78, 68, 4, 222, 11, 82, 207, 1, 76, 9, 18, 32, - 47, 80, 2, 21, 3, 87, 73, 84, 2, 139, 205, 15, 72, 4, 241, 152, 12, 2, - 69, 78, 5, 151, 233, 15, 72, 15, 80, 6, 32, 87, 73, 84, 72, 32, 62, 65, - 41, 8, 69, 86, 69, 82, 83, 69, 68, 32, 4, 26, 70, 239, 206, 13, 84, 2, - 209, 201, 8, 3, 73, 83, 72, 2, 17, 2, 77, 83, 2, 247, 212, 5, 32, 6, 38, - 71, 182, 7, 79, 203, 243, 15, 69, 2, 11, 76, 2, 129, 176, 13, 4, 79, 84, - 84, 65, 13, 72, 6, 32, 87, 73, 84, 72, 32, 34, 67, 61, 6, 73, 68, 69, 87, - 65, 89, 4, 158, 3, 67, 131, 166, 14, 72, 4, 26, 82, 203, 180, 11, 72, 2, - 221, 199, 5, 3, 73, 80, 84, 2, 159, 167, 6, 83, 51, 106, 32, 102, 67, - 116, 2, 69, 83, 44, 2, 79, 80, 42, 83, 96, 6, 85, 82, 78, 69, 68, 32, - 207, 208, 12, 72, 4, 29, 5, 87, 73, 84, 72, 32, 4, 22, 80, 231, 5, 82, 2, - 197, 197, 8, 6, 65, 76, 65, 84, 65, 76, 2, 45, 9, 32, 68, 73, 71, 82, 65, - 80, 72, 32, 2, 25, 4, 87, 73, 84, 72, 2, 17, 2, 32, 67, 2, 195, 137, 3, - 85, 2, 11, 72, 2, 157, 241, 9, 3, 32, 68, 73, 2, 177, 206, 5, 5, 32, 72, - 65, 76, 70, 4, 37, 7, 32, 68, 73, 71, 82, 65, 80, 4, 11, 72, 5, 11, 32, - 2, 153, 3, 4, 87, 73, 84, 72, 32, 86, 65, 50, 77, 74, 79, 42, 82, 214, 1, - 89, 206, 241, 15, 72, 2, 73, 2, 86, 3, 87, 7, 26, 76, 167, 244, 15, 69, - 2, 231, 252, 13, 80, 5, 41, 8, 32, 87, 73, 84, 72, 32, 76, 79, 2, 221, - 133, 4, 2, 78, 71, 2, 11, 80, 2, 189, 133, 6, 2, 69, 78, 9, 33, 6, 32, - 87, 73, 84, 72, 32, 6, 26, 76, 131, 162, 14, 72, 4, 37, 7, 79, 78, 71, - 32, 76, 69, 71, 5, 25, 4, 32, 65, 78, 68, 2, 17, 2, 32, 82, 2, 11, 69, 2, - 213, 191, 8, 7, 84, 82, 79, 70, 76, 69, 88, 5, 29, 5, 32, 87, 73, 84, 72, - 2, 245, 230, 3, 2, 32, 66, 9, 18, 32, 95, 80, 4, 40, 4, 87, 73, 84, 72, - 171, 206, 14, 66, 2, 17, 2, 32, 76, 2, 11, 69, 2, 131, 1, 70, 2, 177, - 160, 15, 2, 83, 73, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 26, 82, 231, - 158, 14, 72, 2, 21, 3, 73, 71, 72, 2, 135, 189, 8, 84, 4, 11, 68, 4, 11, - 32, 4, 182, 208, 1, 72, 35, 76, 4, 24, 2, 72, 65, 31, 84, 2, 25, 4, 76, - 70, 32, 84, 2, 43, 82, 4, 30, 82, 53, 3, 85, 82, 78, 2, 237, 157, 15, 8, - 73, 65, 78, 71, 85, 76, 65, 82, 2, 161, 144, 8, 2, 69, 68, 8, 70, 80, - 140, 245, 9, 7, 78, 65, 83, 80, 73, 82, 65, 223, 247, 5, 83, 4, 11, 32, - 4, 190, 226, 12, 84, 231, 59, 65, 4, 26, 79, 199, 210, 14, 69, 2, 11, 73, - 2, 155, 155, 15, 67, 4, 36, 3, 65, 78, 71, 1, 2, 73, 78, 2, 25, 4, 32, + 89, 198, 27, 65, 199, 219, 16, 69, 2, 221, 202, 1, 19, 69, 76, 79, 82, + 85, 83, 83, 73, 65, 78, 45, 85, 75, 82, 65, 73, 78, 73, 65, 4, 242, 213, + 6, 90, 183, 160, 10, 69, 15, 50, 83, 210, 245, 16, 70, 2, 76, 2, 77, 3, + 82, 5, 25, 4, 32, 87, 73, 84, 2, 21, 3, 72, 32, 68, 2, 11, 69, 2, 177, + 141, 13, 3, 83, 67, 69, 4, 26, 65, 215, 244, 16, 69, 2, 137, 200, 16, 2, + 76, 79, 8, 42, 84, 206, 173, 1, 67, 139, 196, 15, 72, 4, 153, 19, 8, 82, + 65, 73, 71, 72, 84, 32, 85, 4, 134, 221, 16, 83, 215, 22, 69, 6, 36, 3, + 69, 82, 85, 151, 243, 16, 85, 5, 37, 7, 32, 87, 73, 84, 72, 32, 66, 2, + 21, 3, 65, 67, 75, 2, 229, 180, 16, 2, 32, 89, 2, 227, 175, 11, 70, 16, + 18, 69, 27, 79, 2, 169, 5, 2, 78, 84, 14, 64, 2, 84, 32, 52, 5, 85, 66, + 76, 69, 32, 129, 52, 2, 87, 78, 6, 242, 218, 9, 83, 142, 192, 4, 86, 171, + 180, 1, 72, 4, 194, 137, 2, 65, 219, 247, 2, 80, 24, 40, 5, 88, 84, 82, + 65, 45, 131, 49, 78, 20, 52, 5, 72, 73, 71, 72, 32, 81, 4, 76, 79, 87, + 32, 10, 156, 1, 9, 69, 88, 84, 82, 65, 45, 76, 79, 87, 154, 7, 68, 70, + 76, 79, 84, 10, 76, 10, 69, 88, 84, 82, 65, 45, 72, 73, 71, 72, 154, 7, + 68, 70, 76, 79, 84, 2, 145, 8, 8, 32, 67, 79, 78, 84, 79, 85, 82, 8, 162, + 9, 82, 226, 3, 76, 237, 191, 15, 9, 69, 79, 82, 71, 73, 65, 78, 32, 78, + 10, 248, 5, 4, 73, 71, 72, 32, 255, 40, 65, 44, 46, 65, 84, 2, 79, 87, + 201, 11, 2, 69, 70, 2, 21, 3, 84, 69, 82, 2, 17, 2, 65, 76, 2, 17, 2, 32, + 67, 2, 191, 188, 15, 76, 36, 86, 32, 249, 139, 12, 15, 69, 82, 32, 82, + 73, 71, 72, 84, 32, 67, 79, 82, 78, 69, 82, 34, 158, 1, 67, 20, 2, 68, + 79, 40, 4, 76, 69, 70, 84, 82, 77, 12, 2, 82, 73, 50, 84, 178, 3, 65, 42, + 71, 170, 2, 73, 184, 141, 14, 2, 85, 80, 179, 187, 1, 86, 2, 199, 210, + 10, 73, 6, 238, 2, 84, 253, 145, 14, 2, 87, 78, 6, 28, 2, 32, 65, 247, 2, + 45, 4, 25, 4, 82, 82, 79, 87, 5, 199, 148, 14, 72, 2, 111, 65, 4, 228, + 147, 14, 3, 71, 72, 84, 223, 212, 2, 78, 4, 194, 2, 79, 243, 204, 14, 73, + 18, 18, 65, 23, 73, 2, 147, 195, 15, 67, 16, 26, 68, 195, 185, 13, 78, + 14, 38, 32, 217, 1, 4, 68, 76, 69, 32, 8, 26, 68, 70, 76, 79, 84, 4, 17, + 2, 79, 84, 4, 25, 4, 84, 69, 68, 32, 4, 18, 76, 79, 84, 2, 25, 4, 69, 70, + 84, 45, 2, 25, 4, 83, 84, 69, 77, 2, 17, 2, 32, 84, 2, 11, 79, 2, 11, 78, + 2, 199, 196, 15, 69, 6, 40, 6, 68, 79, 85, 66, 76, 69, 75, 71, 4, 11, 32, + 4, 18, 65, 43, 71, 2, 11, 67, 2, 169, 208, 10, 2, 85, 84, 2, 11, 82, 2, + 247, 207, 10, 65, 2, 207, 225, 14, 69, 4, 150, 182, 13, 76, 207, 150, 2, + 82, 26, 104, 6, 65, 73, 83, 69, 68, 32, 150, 1, 69, 216, 1, 3, 73, 71, + 72, 133, 245, 8, 5, 72, 79, 84, 73, 67, 10, 70, 68, 30, 73, 162, 213, 9, + 69, 244, 242, 5, 2, 85, 80, 211, 74, 67, 2, 157, 146, 15, 2, 79, 87, 2, + 137, 213, 9, 7, 78, 86, 69, 82, 84, 69, 68, 8, 104, 7, 86, 69, 82, 83, + 69, 68, 32, 137, 28, 14, 84, 82, 79, 70, 76, 69, 88, 32, 67, 76, 73, 67, + 75, 32, 6, 26, 71, 175, 143, 14, 67, 4, 11, 76, 4, 49, 10, 79, 84, 84, + 65, 76, 32, 83, 84, 79, 80, 5, 171, 19, 32, 6, 17, 2, 84, 32, 6, 38, 72, + 166, 197, 13, 84, 235, 69, 65, 2, 133, 191, 7, 3, 65, 76, 70, 156, 2, + 138, 1, 72, 48, 5, 77, 65, 76, 76, 32, 148, 31, 8, 84, 82, 69, 83, 83, + 32, 65, 78, 53, 11, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 4, 136, + 153, 8, 3, 79, 82, 84, 203, 194, 6, 69, 144, 2, 154, 2, 65, 50, 66, 130, + 1, 67, 238, 2, 68, 238, 2, 90, 66, 69, 46, 70, 30, 71, 110, 72, 102, 77, + 34, 73, 34, 74, 98, 76, 176, 3, 7, 78, 32, 87, 73, 84, 72, 32, 30, 79, + 94, 80, 22, 82, 146, 2, 83, 190, 1, 84, 246, 7, 85, 142, 1, 86, 158, 192, + 16, 75, 2, 81, 2, 87, 2, 88, 3, 89, 9, 238, 131, 13, 76, 214, 136, 3, 73, + 227, 79, 69, 11, 46, 65, 50, 79, 222, 8, 32, 239, 173, 16, 69, 2, 17, 2, + 82, 82, 2, 193, 246, 5, 2, 69, 68, 2, 229, 20, 4, 84, 84, 79, 77, 41, + 100, 7, 65, 80, 73, 84, 65, 76, 32, 180, 1, 6, 76, 79, 83, 69, 68, 32, + 190, 17, 32, 199, 179, 16, 72, 30, 114, 73, 214, 6, 71, 226, 16, 76, 146, + 171, 16, 79, 158, 20, 65, 186, 2, 66, 2, 72, 2, 78, 2, 82, 2, 85, 3, 89, + 7, 22, 78, 191, 11, 32, 2, 249, 239, 5, 5, 86, 69, 82, 84, 69, 4, 26, 82, + 195, 161, 9, 79, 2, 217, 20, 9, 69, 86, 69, 82, 83, 69, 68, 32, 79, 23, + 104, 6, 32, 87, 73, 84, 72, 32, 86, 69, 32, 9, 79, 84, 76, 69, 83, 83, + 32, 74, 32, 105, 3, 90, 32, 68, 6, 26, 72, 163, 163, 14, 84, 4, 21, 3, + 79, 79, 75, 5, 225, 234, 13, 3, 32, 65, 78, 4, 238, 15, 90, 143, 162, 16, + 76, 4, 33, 6, 87, 73, 84, 72, 32, 83, 4, 29, 5, 84, 82, 79, 75, 69, 5, + 133, 234, 8, 4, 32, 65, 78, 68, 6, 33, 6, 73, 71, 82, 65, 80, 72, 7, 33, + 6, 32, 87, 73, 84, 72, 32, 4, 138, 14, 67, 207, 4, 82, 11, 202, 211, 16, + 83, 2, 84, 2, 90, 63, 78, 5, 225, 13, 3, 69, 78, 71, 11, 56, 5, 82, 69, + 69, 75, 32, 162, 1, 32, 195, 128, 14, 65, 4, 26, 71, 191, 132, 11, 80, 2, + 195, 129, 14, 65, 11, 40, 6, 32, 87, 73, 84, 72, 32, 39, 69, 4, 166, 255, + 14, 72, 219, 163, 1, 83, 4, 17, 2, 78, 71, 5, 11, 32, 2, 195, 230, 8, 87, + 4, 222, 4, 32, 251, 168, 16, 79, 5, 37, 7, 32, 87, 73, 84, 72, 32, 67, 2, + 11, 82, 2, 225, 157, 14, 6, 79, 83, 83, 69, 68, 45, 25, 116, 6, 32, 87, + 73, 84, 72, 32, 226, 9, 83, 2, 90, 112, 2, 69, 90, 253, 174, 16, 8, 73, + 71, 65, 84, 85, 82, 69, 32, 12, 54, 73, 82, 77, 82, 82, 222, 6, 80, 243, + 244, 3, 66, 2, 49, 10, 78, 86, 69, 82, 84, 69, 68, 32, 76, 65, 2, 173, + 218, 12, 2, 90, 89, 2, 11, 73, 2, 17, 2, 68, 68, 2, 17, 2, 76, 69, 2, + 145, 235, 12, 2, 32, 84, 4, 61, 13, 69, 84, 82, 79, 70, 76, 69, 88, 32, + 72, 79, 79, 75, 5, 205, 12, 4, 32, 65, 78, 68, 4, 210, 11, 82, 207, 1, + 76, 9, 18, 32, 47, 80, 2, 21, 3, 87, 73, 84, 2, 151, 157, 16, 72, 4, 193, + 186, 12, 2, 69, 78, 5, 163, 185, 16, 72, 15, 80, 6, 32, 87, 73, 84, 72, + 32, 62, 65, 41, 8, 69, 86, 69, 82, 83, 69, 68, 32, 4, 26, 70, 155, 152, + 14, 84, 2, 165, 224, 8, 3, 73, 83, 72, 2, 17, 2, 77, 83, 2, 131, 232, 5, + 32, 6, 38, 71, 170, 7, 79, 227, 195, 16, 69, 2, 11, 76, 2, 173, 249, 13, + 4, 79, 84, 84, 65, 13, 72, 6, 32, 87, 73, 84, 72, 32, 34, 67, 61, 6, 73, + 68, 69, 87, 65, 89, 4, 158, 3, 67, 155, 243, 14, 72, 4, 26, 82, 203, 216, + 11, 72, 2, 233, 218, 5, 3, 73, 80, 84, 2, 231, 189, 6, 83, 51, 106, 32, + 102, 67, 116, 2, 69, 83, 44, 2, 79, 80, 42, 83, 96, 6, 85, 82, 78, 69, + 68, 32, 231, 139, 13, 72, 4, 29, 5, 87, 73, 84, 72, 32, 4, 22, 80, 219, + 5, 82, 2, 153, 220, 8, 6, 65, 76, 65, 84, 65, 76, 2, 45, 9, 32, 68, 73, + 71, 82, 65, 80, 72, 32, 2, 25, 4, 87, 73, 84, 72, 2, 17, 2, 32, 67, 2, + 179, 145, 3, 85, 2, 11, 72, 2, 217, 150, 10, 3, 32, 68, 73, 2, 189, 225, + 5, 5, 32, 72, 65, 76, 70, 4, 37, 7, 32, 68, 73, 71, 82, 65, 80, 4, 11, + 72, 5, 11, 32, 2, 141, 3, 4, 87, 73, 84, 72, 32, 86, 65, 38, 77, 74, 79, + 42, 82, 214, 1, 89, 230, 193, 16, 72, 2, 73, 2, 86, 3, 87, 7, 150, 236, + 12, 76, 183, 216, 3, 69, 5, 41, 8, 32, 87, 73, 84, 72, 32, 76, 79, 2, + 213, 141, 4, 2, 78, 71, 2, 11, 80, 2, 225, 156, 6, 2, 69, 78, 9, 33, 6, + 32, 87, 73, 84, 72, 32, 6, 26, 76, 167, 239, 14, 72, 4, 37, 7, 79, 78, + 71, 32, 76, 69, 71, 5, 25, 4, 32, 65, 78, 68, 2, 17, 2, 32, 82, 2, 11, + 69, 2, 181, 214, 8, 7, 84, 82, 79, 70, 76, 69, 88, 5, 29, 5, 32, 87, 73, + 84, 72, 2, 185, 238, 3, 2, 32, 66, 9, 18, 32, 95, 80, 4, 40, 4, 87, 73, + 84, 72, 179, 159, 15, 66, 2, 17, 2, 32, 76, 2, 11, 69, 2, 131, 1, 70, 2, + 189, 240, 15, 2, 83, 73, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 26, 82, + 139, 236, 14, 72, 2, 21, 3, 73, 71, 72, 2, 231, 211, 8, 84, 4, 11, 68, 4, + 11, 32, 4, 130, 210, 1, 72, 35, 76, 4, 24, 2, 72, 65, 31, 84, 2, 25, 4, + 76, 70, 32, 84, 2, 43, 82, 4, 30, 82, 53, 3, 85, 82, 78, 2, 249, 237, 15, + 8, 73, 65, 78, 71, 85, 76, 65, 82, 2, 253, 166, 8, 2, 69, 68, 8, 70, 80, + 212, 154, 10, 7, 78, 65, 83, 80, 73, 82, 65, 175, 162, 6, 83, 4, 11, 32, + 4, 242, 161, 13, 84, 235, 69, 65, 4, 26, 79, 199, 162, 15, 69, 2, 11, 73, + 2, 167, 235, 15, 67, 4, 36, 3, 65, 78, 71, 1, 2, 73, 78, 2, 25, 4, 32, 68, 69, 80, 2, 21, 3, 65, 82, 84, 2, 21, 3, 73, 78, 71, 2, 17, 2, 32, 84, - 2, 11, 79, 2, 227, 238, 5, 78, 2, 11, 32, 2, 215, 194, 14, 83, 234, 2, - 92, 2, 69, 89, 88, 7, 71, 79, 76, 73, 65, 78, 32, 210, 24, 79, 209, 222, - 11, 3, 75, 69, 89, 6, 26, 32, 251, 171, 15, 45, 4, 184, 223, 11, 6, 87, - 73, 84, 72, 32, 87, 255, 163, 2, 66, 214, 2, 194, 2, 68, 46, 70, 148, 1, + 2, 11, 79, 2, 143, 134, 6, 78, 2, 11, 32, 2, 211, 145, 15, 83, 234, 2, + 92, 2, 69, 89, 88, 7, 71, 79, 76, 73, 65, 78, 32, 206, 24, 79, 177, 130, + 12, 3, 75, 69, 89, 6, 26, 32, 135, 252, 15, 45, 4, 152, 131, 12, 6, 87, + 73, 84, 72, 32, 87, 179, 205, 2, 66, 214, 2, 194, 2, 68, 46, 70, 148, 1, 14, 73, 78, 86, 69, 82, 84, 69, 68, 32, 66, 73, 82, 71, 65, 16, 7, 76, - 69, 84, 84, 69, 82, 32, 142, 16, 83, 132, 1, 7, 82, 79, 84, 65, 84, 69, - 68, 22, 66, 98, 84, 160, 2, 4, 86, 79, 87, 69, 158, 233, 3, 67, 164, 1, - 6, 77, 65, 78, 67, 72, 85, 196, 162, 7, 4, 78, 73, 82, 85, 171, 166, 2, - 69, 22, 236, 20, 3, 79, 85, 66, 183, 227, 13, 73, 12, 112, 19, 82, 69, - 69, 32, 86, 65, 82, 73, 65, 84, 73, 79, 78, 32, 83, 69, 76, 69, 67, 158, - 154, 13, 85, 175, 46, 79, 8, 177, 207, 9, 3, 84, 79, 82, 5, 231, 19, 32, - 136, 2, 130, 2, 65, 244, 4, 2, 67, 72, 88, 2, 77, 65, 174, 2, 83, 250, 1, - 84, 234, 3, 90, 190, 229, 13, 72, 150, 168, 1, 75, 2, 76, 162, 7, 69, 2, - 79, 2, 85, 222, 61, 66, 2, 68, 2, 70, 2, 71, 2, 74, 2, 78, 2, 80, 2, 81, + 69, 84, 84, 69, 82, 32, 138, 16, 83, 132, 1, 7, 82, 79, 84, 65, 84, 69, + 68, 22, 66, 98, 84, 160, 2, 4, 86, 79, 87, 69, 154, 241, 3, 67, 164, 1, + 6, 77, 65, 78, 67, 72, 85, 216, 190, 7, 4, 78, 73, 82, 85, 239, 203, 2, + 69, 22, 232, 20, 3, 79, 85, 66, 207, 176, 14, 73, 12, 112, 19, 82, 69, + 69, 32, 86, 65, 82, 73, 65, 84, 73, 79, 78, 32, 83, 69, 76, 69, 67, 214, + 227, 13, 85, 199, 46, 79, 8, 249, 244, 9, 3, 84, 79, 82, 5, 227, 19, 32, + 136, 2, 130, 2, 65, 244, 4, 2, 67, 72, 88, 2, 77, 65, 174, 2, 83, 246, 1, + 84, 234, 3, 90, 214, 178, 14, 72, 142, 171, 1, 75, 2, 76, 162, 7, 69, 2, + 79, 2, 85, 234, 61, 66, 2, 68, 2, 70, 2, 71, 2, 74, 2, 78, 2, 80, 2, 81, 2, 82, 2, 87, 2, 89, 187, 2, 73, 59, 56, 8, 76, 73, 32, 71, 65, 76, 73, - 32, 207, 225, 15, 78, 54, 174, 1, 65, 52, 6, 86, 73, 83, 65, 82, 71, 22, + 32, 231, 177, 16, 78, 54, 174, 1, 65, 52, 6, 86, 73, 83, 65, 82, 71, 22, 68, 76, 5, 72, 65, 76, 70, 32, 34, 73, 50, 85, 34, 78, 30, 84, 66, 66, - 222, 150, 15, 80, 2, 90, 242, 68, 83, 14, 67, 3, 75, 7, 48, 6, 78, 85, - 83, 86, 65, 82, 191, 224, 15, 72, 2, 131, 134, 9, 65, 8, 26, 65, 215, - 221, 15, 68, 7, 214, 222, 8, 77, 253, 190, 6, 3, 71, 65, 76, 4, 162, 221, - 15, 89, 187, 2, 85, 5, 45, 9, 78, 86, 69, 82, 84, 69, 68, 32, 85, 2, 169, - 156, 15, 3, 66, 65, 68, 4, 178, 220, 15, 71, 3, 78, 8, 60, 6, 72, 82, 69, - 69, 32, 66, 222, 150, 15, 84, 183, 71, 65, 2, 17, 2, 65, 76, 2, 247, 184, - 15, 85, 6, 26, 65, 207, 221, 15, 73, 5, 29, 5, 32, 87, 73, 84, 72, 2, - 201, 143, 14, 2, 32, 84, 41, 29, 5, 78, 67, 72, 85, 32, 38, 104, 9, 65, - 76, 73, 32, 71, 65, 76, 73, 32, 202, 228, 13, 90, 134, 245, 1, 70, 2, 75, - 2, 82, 187, 2, 73, 28, 122, 68, 166, 165, 9, 67, 158, 164, 1, 84, 142, - 154, 3, 66, 2, 71, 2, 74, 2, 76, 138, 176, 1, 90, 254, 4, 78, 247, 63, - 83, 4, 202, 227, 13, 68, 135, 245, 1, 72, 48, 52, 4, 73, 66, 69, 32, 246, - 215, 15, 72, 187, 2, 65, 44, 186, 215, 2, 84, 162, 243, 2, 71, 2, 72, - 218, 156, 6, 73, 190, 251, 1, 67, 2, 83, 246, 7, 82, 222, 158, 1, 65, - 186, 9, 90, 162, 7, 85, 222, 61, 68, 2, 70, 2, 74, 2, 75, 2, 80, 187, 2, - 69, 60, 52, 4, 79, 68, 79, 32, 254, 213, 15, 83, 187, 2, 65, 56, 250, 1, - 65, 98, 68, 34, 74, 34, 78, 176, 163, 1, 8, 76, 79, 78, 71, 32, 86, 79, - 87, 250, 174, 1, 84, 162, 243, 2, 71, 150, 152, 8, 67, 246, 7, 72, 182, - 175, 1, 79, 2, 85, 222, 61, 66, 2, 75, 2, 77, 2, 80, 2, 81, 2, 87, 2, 89, - 186, 2, 69, 3, 73, 6, 56, 8, 76, 73, 32, 71, 65, 76, 73, 32, 171, 213, - 15, 78, 4, 190, 222, 13, 90, 135, 245, 1, 84, 4, 158, 211, 15, 90, 187, - 2, 65, 4, 254, 210, 15, 73, 187, 2, 65, 2, 223, 210, 15, 73, 6, 170, 193, - 15, 72, 162, 17, 82, 187, 2, 65, 8, 128, 1, 4, 87, 73, 82, 76, 149, 181, - 4, 21, 73, 66, 69, 32, 83, 89, 76, 76, 65, 66, 76, 69, 32, 66, 79, 85, - 78, 68, 65, 82, 89, 6, 17, 2, 32, 66, 6, 25, 4, 73, 82, 71, 65, 7, 33, 6, - 32, 87, 73, 84, 72, 32, 4, 150, 2, 68, 171, 141, 15, 79, 6, 152, 1, 3, - 82, 73, 80, 56, 18, 85, 82, 78, 69, 68, 32, 83, 87, 73, 82, 76, 32, 66, - 73, 82, 71, 65, 32, 245, 190, 1, 8, 79, 68, 79, 32, 83, 79, 70, 84, 2, - 209, 141, 15, 9, 76, 69, 32, 66, 73, 82, 71, 65, 32, 2, 33, 6, 87, 73, - 84, 72, 32, 68, 2, 145, 141, 15, 5, 79, 85, 66, 76, 69, 2, 131, 192, 9, - 76, 10, 96, 9, 71, 82, 65, 77, 32, 70, 79, 82, 32, 224, 251, 3, 5, 83, - 84, 65, 66, 76, 235, 166, 9, 82, 6, 26, 89, 243, 136, 12, 69, 4, 190, - 254, 14, 65, 155, 1, 73, 8, 48, 2, 78, 32, 162, 197, 4, 68, 231, 242, 10, - 83, 4, 72, 12, 86, 73, 69, 87, 73, 78, 71, 32, 67, 69, 82, 69, 147, 136, - 12, 67, 2, 177, 174, 15, 2, 77, 79, 4, 230, 129, 12, 73, 139, 204, 3, 69, - 10, 26, 72, 69, 2, 79, 82, 2, 45, 9, 69, 82, 32, 67, 72, 82, 73, 83, 84, - 2, 147, 247, 2, 77, 8, 50, 32, 52, 4, 73, 90, 69, 68, 155, 192, 14, 87, - 4, 230, 173, 8, 66, 133, 232, 5, 4, 83, 67, 79, 79, 2, 205, 168, 3, 7, - 32, 87, 72, 69, 69, 76, 67, 18, 52, 2, 78, 84, 152, 1, 2, 83, 69, 247, - 200, 15, 84, 10, 48, 3, 65, 73, 78, 185, 224, 11, 3, 32, 70, 85, 9, 11, - 32, 6, 222, 166, 9, 82, 24, 5, 67, 65, 66, 76, 69, 137, 235, 1, 6, 66, - 73, 67, 89, 67, 76, 7, 11, 32, 4, 160, 185, 14, 2, 84, 82, 219, 83, 70, - 86, 52, 7, 76, 69, 84, 84, 69, 82, 32, 191, 211, 5, 68, 62, 198, 1, 75, - 62, 77, 34, 78, 34, 79, 30, 80, 34, 84, 246, 104, 68, 220, 231, 7, 2, 72, - 65, 2, 82, 158, 180, 1, 69, 190, 29, 83, 150, 90, 76, 246, 7, 67, 246, - 162, 4, 89, 190, 28, 66, 2, 87, 187, 2, 65, 6, 180, 154, 5, 2, 69, 65, - 182, 238, 5, 72, 231, 190, 4, 79, 4, 230, 180, 15, 65, 215, 1, 73, 4, - 158, 247, 14, 73, 255, 59, 71, 7, 210, 198, 15, 76, 3, 79, 4, 222, 178, - 15, 72, 219, 19, 65, 6, 214, 246, 8, 72, 170, 204, 6, 69, 155, 3, 65, - 204, 4, 44, 2, 76, 84, 238, 6, 83, 159, 175, 13, 67, 102, 36, 4, 65, 78, - 73, 32, 223, 2, 73, 76, 52, 7, 76, 69, 84, 84, 69, 82, 32, 211, 139, 4, - 83, 74, 210, 1, 68, 222, 82, 78, 158, 230, 1, 82, 230, 208, 1, 74, 166, - 209, 7, 84, 206, 160, 3, 66, 2, 67, 2, 71, 2, 75, 2, 80, 254, 68, 72, 2, - 76, 2, 77, 2, 83, 2, 86, 2, 89, 186, 2, 65, 2, 69, 2, 73, 3, 85, 10, 38, - 68, 162, 192, 15, 72, 187, 2, 65, 6, 158, 192, 15, 68, 2, 72, 187, 2, 65, - 26, 56, 2, 80, 76, 236, 2, 3, 83, 69, 84, 183, 174, 14, 77, 18, 50, 69, - 89, 8, 73, 67, 65, 84, 73, 79, 78, 32, 2, 41, 8, 32, 77, 85, 83, 73, 67, - 65, 76, 2, 21, 3, 32, 78, 79, 2, 155, 173, 14, 84, 16, 40, 4, 83, 73, 71, - 78, 191, 192, 15, 88, 15, 11, 32, 12, 48, 3, 73, 78, 32, 81, 5, 87, 73, - 84, 72, 32, 8, 166, 199, 3, 76, 22, 82, 140, 246, 8, 5, 68, 79, 85, 66, - 76, 235, 222, 1, 84, 4, 210, 246, 11, 85, 167, 200, 2, 68, 7, 11, 32, 4, - 232, 82, 5, 77, 85, 76, 84, 73, 207, 209, 11, 85, 226, 3, 32, 2, 73, 67, - 247, 152, 3, 72, 224, 3, 30, 32, 101, 3, 65, 76, 32, 6, 76, 4, 78, 65, - 84, 85, 200, 43, 2, 70, 76, 181, 229, 10, 4, 83, 72, 65, 82, 2, 239, 165, - 10, 82, 218, 3, 64, 8, 75, 69, 89, 66, 79, 65, 82, 68, 74, 83, 155, 239, - 3, 78, 5, 41, 8, 32, 87, 73, 84, 72, 32, 74, 65, 2, 209, 254, 14, 2, 67, - 75, 212, 3, 48, 6, 89, 77, 66, 79, 76, 32, 135, 131, 11, 67, 210, 3, 230, - 2, 66, 238, 1, 67, 226, 9, 68, 242, 2, 69, 162, 1, 70, 166, 2, 71, 112, - 9, 65, 82, 80, 69, 71, 71, 73, 65, 84, 168, 1, 2, 72, 65, 126, 75, 198, - 3, 76, 138, 1, 77, 190, 2, 78, 162, 1, 79, 218, 2, 80, 200, 2, 2, 81, 85, - 146, 2, 82, 190, 2, 83, 230, 5, 84, 182, 10, 86, 42, 88, 42, 87, 180, - 233, 8, 9, 73, 78, 86, 69, 82, 84, 69, 68, 32, 163, 151, 6, 90, 20, 36, - 5, 69, 71, 73, 78, 32, 59, 82, 8, 130, 15, 80, 30, 83, 182, 224, 7, 66, - 187, 173, 7, 84, 12, 24, 2, 65, 67, 35, 69, 4, 218, 130, 15, 75, 143, 53, - 69, 8, 26, 86, 223, 245, 14, 65, 6, 32, 2, 73, 83, 139, 183, 15, 69, 5, - 171, 26, 32, 84, 120, 2, 65, 69, 30, 76, 94, 79, 194, 7, 82, 234, 11, 32, - 224, 32, 7, 73, 82, 67, 76, 69, 32, 88, 145, 200, 5, 2, 85, 84, 2, 197, - 241, 14, 2, 83, 85, 8, 42, 73, 245, 23, 5, 85, 83, 84, 69, 82, 4, 26, 77, - 175, 152, 13, 86, 2, 207, 177, 9, 65, 64, 26, 77, 179, 178, 15, 68, 62, - 64, 7, 66, 73, 78, 73, 78, 71, 32, 185, 251, 5, 3, 77, 79, 78, 60, 202, - 1, 65, 80, 7, 77, 65, 82, 67, 65, 84, 79, 56, 2, 68, 79, 56, 2, 85, 80, - 28, 2, 70, 76, 40, 5, 72, 65, 82, 77, 79, 22, 83, 142, 2, 84, 202, 154, - 7, 66, 140, 180, 3, 2, 76, 79, 183, 222, 4, 82, 6, 76, 5, 67, 67, 69, 78, - 84, 37, 10, 85, 71, 77, 69, 78, 84, 65, 84, 73, 79, 5, 205, 2, 5, 45, 83, - 84, 65, 67, 2, 231, 160, 14, 78, 6, 52, 2, 87, 78, 168, 3, 2, 85, 66, - 199, 145, 15, 73, 2, 137, 151, 14, 2, 32, 66, 12, 136, 76, 2, 65, 71, - 231, 228, 14, 73, 2, 231, 137, 14, 78, 12, 132, 1, 9, 78, 65, 80, 32, 80, - 73, 90, 90, 73, 22, 84, 236, 187, 4, 11, 80, 82, 69, 67, 72, 71, 69, 83, - 65, 78, 71, 255, 238, 5, 77, 2, 191, 206, 11, 67, 6, 44, 5, 65, 67, 67, - 65, 84, 159, 158, 15, 69, 4, 40, 4, 73, 83, 83, 73, 203, 174, 15, 79, 2, - 211, 143, 15, 77, 10, 34, 82, 253, 225, 11, 2, 69, 78, 8, 28, 2, 73, 80, - 191, 6, 69, 2, 133, 205, 11, 6, 76, 69, 32, 84, 79, 78, 4, 22, 79, 191, - 2, 69, 2, 159, 161, 14, 73, 24, 98, 65, 142, 1, 69, 84, 6, 79, 85, 66, - 76, 69, 32, 213, 249, 9, 7, 82, 85, 77, 32, 67, 76, 69, 10, 96, 2, 32, - 67, 20, 2, 77, 80, 196, 29, 4, 83, 72, 69, 68, 233, 228, 14, 5, 76, 32, - 83, 69, 71, 2, 223, 164, 8, 65, 5, 255, 166, 13, 32, 4, 52, 3, 67, 82, - 69, 241, 201, 8, 4, 71, 82, 69, 69, 2, 157, 25, 3, 83, 67, 69, 6, 246, - 21, 83, 250, 6, 66, 131, 244, 4, 70, 14, 32, 3, 78, 68, 32, 155, 16, 73, - 10, 74, 80, 30, 83, 208, 13, 3, 79, 70, 32, 230, 210, 7, 66, 187, 173, 7, - 84, 2, 145, 153, 12, 2, 72, 82, 2, 215, 176, 14, 76, 34, 104, 6, 69, 82, - 77, 65, 84, 65, 22, 73, 122, 79, 102, 32, 144, 36, 3, 85, 83, 65, 157, - 221, 3, 2, 76, 65, 5, 195, 129, 13, 32, 10, 22, 78, 135, 34, 86, 8, 56, - 9, 71, 69, 82, 69, 68, 32, 84, 82, 69, 235, 20, 65, 6, 185, 215, 5, 4, - 77, 79, 76, 79, 6, 208, 25, 3, 85, 82, 45, 147, 201, 13, 82, 18, 54, 32, - 56, 7, 76, 73, 83, 83, 65, 78, 68, 23, 82, 6, 25, 4, 67, 76, 69, 70, 7, - 161, 13, 3, 32, 79, 84, 4, 163, 235, 5, 79, 8, 84, 9, 65, 67, 69, 32, 78, - 79, 84, 69, 32, 37, 8, 69, 71, 79, 82, 73, 65, 78, 32, 4, 132, 196, 8, 2, - 78, 79, 27, 83, 4, 250, 2, 67, 3, 70, 8, 44, 3, 76, 70, 32, 209, 8, 3, - 85, 80, 84, 6, 52, 3, 80, 69, 68, 246, 214, 3, 78, 163, 169, 3, 82, 2, - 135, 175, 5, 65, 24, 48, 6, 73, 69, 86, 65, 78, 32, 147, 255, 13, 79, 22, - 178, 1, 67, 46, 69, 68, 7, 81, 85, 65, 82, 84, 69, 82, 62, 70, 184, 211, - 3, 4, 72, 65, 76, 70, 0, 5, 87, 72, 79, 76, 69, 253, 208, 1, 9, 82, 69, - 67, 73, 84, 65, 84, 73, 86, 2, 11, 32, 2, 11, 67, 2, 171, 144, 5, 76, 6, - 64, 5, 73, 71, 72, 84, 72, 253, 235, 13, 5, 78, 68, 32, 79, 70, 4, 165, - 230, 5, 10, 32, 78, 79, 84, 69, 32, 83, 84, 69, 77, 4, 214, 14, 76, 225, - 196, 3, 4, 73, 78, 65, 76, 8, 44, 4, 79, 78, 71, 65, 213, 13, 2, 69, 70, - 7, 11, 32, 4, 28, 3, 73, 77, 80, 3, 80, 2, 197, 2, 7, 69, 82, 70, 69, 67, - 84, 65, 18, 104, 2, 65, 88, 20, 2, 69, 90, 20, 5, 73, 78, 73, 77, 65, 48, - 3, 79, 79, 78, 25, 4, 85, 76, 84, 73, 2, 135, 253, 14, 73, 2, 143, 255, - 14, 90, 7, 11, 32, 4, 146, 250, 6, 82, 171, 153, 5, 66, 4, 185, 17, 2, - 32, 78, 4, 60, 11, 80, 76, 69, 32, 77, 69, 65, 83, 85, 82, 69, 15, 32, 2, - 11, 32, 2, 139, 249, 6, 82, 10, 120, 4, 69, 66, 69, 78, 208, 26, 3, 85, - 76, 76, 148, 198, 5, 3, 65, 84, 85, 173, 176, 6, 7, 79, 84, 69, 72, 69, - 65, 68, 2, 245, 131, 14, 4, 83, 84, 73, 77, 32, 88, 2, 78, 69, 120, 14, - 82, 78, 65, 77, 69, 78, 84, 32, 83, 84, 82, 79, 75, 69, 107, 84, 6, 92, - 18, 32, 72, 85, 78, 68, 82, 69, 68, 32, 84, 87, 69, 78, 84, 89, 45, 69, - 73, 155, 20, 45, 4, 177, 13, 2, 71, 72, 22, 11, 45, 22, 250, 158, 3, 49, - 206, 250, 11, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 4, - 173, 4, 3, 84, 65, 86, 18, 96, 9, 65, 82, 69, 78, 84, 72, 69, 83, 73, 0, - 2, 76, 85, 18, 69, 142, 1, 79, 247, 145, 13, 73, 2, 239, 22, 83, 6, 68, - 4, 68, 65, 76, 32, 49, 9, 83, 32, 83, 85, 66, 80, 85, 78, 67, 4, 26, 85, - 219, 213, 14, 77, 2, 191, 213, 14, 80, 2, 247, 249, 12, 84, 6, 48, 2, 68, - 65, 237, 5, 5, 82, 82, 69, 67, 84, 2, 175, 147, 13, 84, 12, 76, 6, 65, - 82, 84, 69, 82, 32, 125, 9, 73, 78, 68, 73, 67, 69, 83, 73, 77, 8, 60, 5, - 84, 79, 78, 69, 32, 138, 200, 3, 78, 163, 169, 3, 82, 4, 26, 83, 251, - 250, 4, 70, 2, 205, 148, 15, 3, 72, 65, 82, 4, 17, 2, 65, 32, 4, 138, - 240, 11, 65, 217, 161, 3, 3, 66, 65, 83, 14, 22, 69, 167, 1, 73, 10, 72, - 4, 80, 69, 65, 84, 73, 10, 86, 69, 82, 83, 69, 32, 70, 73, 78, 65, 8, - 152, 195, 5, 9, 69, 68, 32, 70, 73, 71, 85, 82, 69, 191, 130, 8, 32, 2, - 211, 4, 76, 4, 48, 2, 71, 72, 57, 6, 78, 70, 79, 82, 90, 65, 2, 33, 6, - 84, 32, 82, 69, 80, 69, 2, 131, 251, 9, 65, 2, 155, 151, 8, 78, 48, 136, - 1, 6, 67, 65, 78, 68, 73, 67, 62, 69, 166, 1, 72, 54, 73, 252, 1, 6, 81, - 85, 65, 82, 69, 32, 204, 128, 8, 2, 85, 66, 239, 2, 79, 4, 17, 2, 85, 83, - 5, 173, 141, 13, 5, 32, 70, 76, 69, 88, 14, 32, 2, 77, 73, 195, 230, 14, - 71, 12, 64, 6, 66, 82, 69, 86, 73, 83, 1, 6, 77, 73, 78, 73, 77, 65, 6, - 11, 32, 6, 138, 13, 87, 166, 222, 6, 82, 171, 153, 5, 66, 6, 84, 3, 79, - 82, 84, 169, 211, 5, 3, 65, 82, 80, 14, 32, 4, 78, 71, 76, 69, 43, 88, 2, - 17, 2, 32, 66, 2, 155, 169, 13, 65, 12, 18, 45, 79, 84, 4, 242, 7, 76, - 213, 238, 13, 11, 83, 84, 82, 73, 78, 71, 32, 70, 82, 69, 84, 8, 52, 3, - 69, 69, 78, 1, 6, 89, 45, 70, 79, 85, 82, 4, 193, 12, 2, 84, 72, 6, 26, - 78, 135, 140, 15, 66, 4, 229, 9, 7, 79, 84, 69, 72, 69, 65, 68, 60, 132, - 1, 6, 69, 77, 80, 85, 83, 32, 154, 4, 72, 88, 2, 87, 79, 84, 7, 79, 82, - 67, 85, 76, 85, 83, 46, 82, 141, 3, 3, 85, 82, 78, 16, 232, 1, 27, 73, - 77, 80, 69, 82, 70, 69, 67, 84, 85, 77, 32, 67, 85, 77, 32, 80, 82, 79, - 76, 65, 84, 73, 79, 78, 69, 32, 129, 1, 25, 80, 69, 82, 70, 69, 67, 84, - 85, 77, 32, 67, 85, 77, 32, 80, 82, 79, 76, 65, 84, 73, 79, 78, 69, 32, - 10, 60, 10, 73, 77, 80, 69, 82, 70, 69, 67, 84, 65, 131, 1, 80, 9, 169, - 184, 5, 11, 32, 68, 73, 77, 73, 78, 85, 84, 73, 79, 78, 6, 60, 3, 73, 77, - 80, 41, 8, 80, 69, 82, 70, 69, 67, 84, 65, 2, 213, 226, 14, 5, 69, 82, - 70, 69, 67, 5, 145, 163, 8, 12, 32, 68, 73, 77, 73, 78, 85, 84, 73, 79, - 78, 45, 6, 72, 2, 82, 69, 249, 5, 11, 73, 82, 84, 89, 45, 83, 69, 67, 79, - 78, 68, 2, 11, 69, 2, 11, 45, 2, 11, 76, 2, 37, 7, 73, 78, 69, 32, 83, - 84, 65, 2, 147, 132, 14, 70, 5, 173, 168, 11, 6, 32, 82, 69, 83, 85, 80, - 27, 33, 6, 73, 65, 78, 71, 76, 69, 24, 128, 1, 10, 32, 78, 79, 84, 69, - 72, 69, 65, 68, 32, 61, 17, 45, 82, 79, 85, 78, 68, 32, 78, 79, 84, 69, - 72, 69, 65, 68, 32, 68, 20, 58, 68, 24, 3, 85, 80, 32, 38, 82, 25, 3, 76, - 69, 70, 4, 93, 3, 79, 87, 78, 8, 34, 82, 78, 87, 207, 247, 11, 66, 4, 21, - 3, 73, 71, 72, 4, 11, 84, 4, 11, 32, 4, 26, 87, 207, 247, 11, 66, 2, 11, - 72, 2, 187, 189, 13, 73, 7, 11, 32, 4, 222, 160, 8, 83, 223, 224, 6, 85, - 4, 36, 3, 79, 73, 68, 187, 190, 14, 73, 2, 221, 178, 12, 5, 32, 78, 79, - 84, 69, 6, 92, 4, 72, 79, 76, 69, 157, 174, 8, 13, 73, 84, 72, 32, 70, - 73, 78, 71, 69, 82, 78, 65, 73, 4, 11, 32, 4, 246, 178, 3, 78, 163, 169, - 3, 82, 192, 3, 252, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, - 73, 71, 78, 32, 254, 1, 76, 216, 14, 16, 77, 79, 68, 73, 70, 73, 69, 82, - 32, 76, 69, 84, 84, 69, 82, 32, 122, 83, 166, 10, 84, 216, 1, 11, 86, 79, - 87, 69, 76, 32, 83, 73, 71, 78, 32, 223, 241, 12, 68, 16, 66, 77, 165, 1, - 11, 83, 72, 65, 78, 32, 77, 69, 68, 73, 65, 76, 14, 80, 6, 69, 68, 73, - 65, 76, 32, 45, 10, 79, 78, 32, 77, 69, 68, 73, 65, 76, 32, 8, 238, 249, - 14, 72, 2, 82, 2, 87, 3, 89, 6, 194, 249, 14, 76, 2, 77, 3, 78, 2, 219, - 182, 10, 32, 238, 1, 104, 6, 69, 84, 84, 69, 82, 32, 189, 13, 15, 79, 71, - 79, 71, 82, 65, 77, 32, 75, 72, 65, 77, 84, 73, 32, 232, 1, 242, 1, 65, - 50, 69, 146, 1, 71, 50, 75, 254, 1, 77, 134, 1, 78, 58, 82, 86, 83, 130, - 3, 84, 198, 1, 87, 242, 133, 11, 68, 190, 4, 86, 214, 20, 85, 210, 200, - 1, 73, 42, 76, 242, 190, 1, 66, 2, 67, 2, 74, 2, 80, 254, 68, 72, 2, 89, - 187, 2, 79, 7, 140, 174, 14, 4, 73, 84, 79, 78, 207, 74, 85, 9, 77, 17, - 65, 83, 84, 69, 82, 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 78, 32, 6, - 42, 71, 182, 178, 10, 89, 187, 138, 3, 78, 2, 179, 178, 10, 72, 6, 214, - 163, 9, 82, 142, 209, 5, 72, 187, 2, 65, 44, 32, 2, 72, 65, 203, 246, 14, - 65, 43, 25, 4, 77, 84, 73, 32, 40, 134, 1, 68, 34, 84, 162, 191, 8, 78, - 178, 238, 5, 67, 2, 72, 2, 74, 158, 37, 76, 226, 31, 70, 2, 71, 2, 82, 2, - 83, 2, 88, 3, 90, 6, 238, 173, 14, 68, 255, 68, 72, 4, 207, 173, 14, 84, - 12, 36, 3, 79, 78, 32, 203, 244, 14, 65, 10, 60, 2, 66, 66, 206, 252, 12, - 74, 134, 181, 1, 78, 187, 66, 69, 4, 134, 244, 14, 65, 3, 69, 10, 134, - 190, 8, 78, 174, 179, 6, 71, 2, 89, 187, 2, 65, 4, 196, 214, 2, 12, 85, - 77, 65, 73, 32, 80, 65, 76, 65, 85, 78, 71, 239, 156, 12, 65, 50, 90, 72, - 136, 215, 2, 9, 71, 65, 87, 32, 75, 65, 82, 69, 78, 198, 152, 12, 83, - 187, 2, 65, 44, 66, 65, 197, 1, 11, 87, 69, 32, 80, 65, 76, 65, 85, 78, - 71, 32, 41, 17, 2, 78, 32, 38, 134, 1, 78, 234, 248, 12, 74, 2, 80, 2, - 84, 138, 176, 1, 66, 2, 67, 2, 71, 2, 75, 254, 68, 68, 2, 70, 2, 72, 2, - 90, 187, 2, 65, 6, 234, 237, 14, 78, 2, 89, 187, 2, 65, 4, 190, 248, 12, - 67, 3, 83, 36, 38, 65, 134, 168, 14, 84, 255, 68, 72, 31, 41, 8, 73, 32, - 76, 65, 73, 78, 71, 32, 28, 82, 78, 142, 134, 11, 68, 250, 160, 3, 66, 2, - 71, 2, 74, 158, 37, 76, 227, 31, 70, 4, 254, 235, 14, 78, 3, 89, 6, 92, - 17, 69, 83, 84, 69, 82, 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 78, 32, - 191, 237, 14, 65, 4, 190, 168, 10, 80, 195, 205, 2, 84, 6, 170, 225, 13, - 79, 154, 60, 81, 255, 62, 72, 4, 56, 6, 75, 72, 65, 77, 84, 73, 1, 4, 83, - 72, 65, 78, 2, 29, 5, 32, 82, 69, 68, 85, 2, 133, 132, 1, 2, 80, 76, 90, - 88, 4, 73, 71, 78, 32, 232, 6, 6, 89, 77, 66, 79, 76, 32, 145, 143, 9, 3, - 72, 65, 78, 52, 202, 3, 65, 32, 12, 75, 72, 65, 77, 84, 73, 32, 84, 79, - 78, 69, 45, 30, 83, 132, 2, 15, 84, 65, 73, 32, 76, 65, 73, 78, 71, 32, - 84, 79, 78, 69, 45, 28, 22, 87, 69, 83, 84, 69, 82, 78, 32, 80, 87, 79, - 32, 75, 65, 82, 69, 78, 32, 84, 79, 78, 69, 154, 238, 2, 68, 128, 145, 5, - 19, 82, 85, 77, 65, 73, 32, 80, 65, 76, 65, 85, 78, 71, 32, 84, 79, 78, - 69, 45, 156, 151, 3, 9, 80, 65, 79, 32, 75, 65, 82, 69, 78, 160, 91, 6, - 76, 73, 84, 84, 76, 69, 159, 176, 2, 86, 4, 250, 211, 13, 83, 243, 78, - 78, 4, 170, 231, 14, 49, 3, 51, 18, 40, 4, 72, 65, 78, 32, 151, 150, 14, - 69, 16, 84, 8, 67, 79, 85, 78, 67, 73, 76, 32, 84, 5, 84, 79, 78, 69, 45, - 159, 231, 13, 83, 6, 204, 151, 11, 8, 69, 77, 80, 72, 65, 84, 73, 67, - 189, 205, 3, 4, 84, 79, 78, 69, 8, 182, 229, 14, 50, 2, 51, 2, 53, 3, 54, - 4, 138, 229, 14, 50, 3, 53, 10, 11, 45, 10, 226, 228, 14, 49, 2, 50, 2, - 51, 2, 52, 3, 53, 18, 130, 1, 65, 132, 1, 2, 76, 79, 28, 5, 83, 72, 65, - 78, 32, 232, 148, 8, 4, 71, 69, 78, 73, 201, 85, 6, 67, 79, 77, 80, 76, - 69, 8, 84, 5, 73, 84, 79, 78, 32, 217, 152, 6, 10, 70, 79, 82, 69, 77, - 69, 78, 84, 73, 79, 6, 98, 69, 130, 213, 8, 84, 223, 243, 4, 79, 2, 129, - 149, 8, 2, 67, 65, 4, 26, 69, 223, 200, 13, 79, 2, 205, 12, 4, 88, 67, - 76, 65, 24, 140, 1, 20, 79, 78, 69, 32, 77, 65, 82, 75, 32, 83, 71, 65, - 87, 32, 75, 65, 82, 69, 78, 32, 161, 139, 9, 8, 65, 73, 32, 76, 65, 73, - 78, 71, 4, 42, 72, 253, 163, 10, 4, 75, 69, 32, 80, 2, 161, 188, 9, 2, - 65, 84, 56, 154, 2, 65, 76, 9, 71, 69, 66, 65, 32, 75, 65, 82, 69, 20, 6, - 75, 65, 89, 65, 72, 32, 40, 4, 77, 79, 78, 32, 34, 83, 154, 1, 69, 40, - 18, 87, 69, 83, 84, 69, 82, 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 78, - 32, 144, 146, 9, 2, 84, 65, 246, 228, 1, 86, 214, 20, 85, 211, 200, 1, - 73, 8, 26, 73, 195, 221, 14, 65, 7, 25, 4, 84, 79, 78, 32, 4, 215, 214, - 12, 65, 2, 139, 157, 14, 78, 6, 166, 198, 14, 69, 2, 79, 215, 22, 85, 4, - 250, 200, 14, 73, 219, 19, 79, 10, 72, 10, 71, 65, 87, 32, 75, 65, 82, - 69, 78, 32, 21, 4, 72, 65, 78, 32, 2, 255, 182, 14, 69, 8, 54, 69, 20, 5, - 70, 73, 78, 65, 76, 211, 216, 14, 65, 5, 179, 222, 13, 32, 2, 191, 201, - 14, 32, 4, 138, 182, 14, 69, 151, 14, 85, 216, 17, 128, 2, 5, 45, 65, 82, - 89, 32, 214, 4, 65, 242, 16, 66, 30, 69, 206, 31, 73, 136, 1, 3, 75, 79, - 32, 154, 10, 79, 162, 24, 85, 224, 8, 22, 89, 73, 65, 75, 69, 78, 71, 32, - 80, 85, 65, 67, 72, 85, 69, 32, 72, 77, 79, 78, 71, 32, 232, 246, 13, 2, - 78, 66, 27, 76, 32, 130, 1, 67, 114, 84, 46, 83, 160, 1, 5, 85, 78, 73, - 79, 78, 108, 4, 87, 72, 73, 84, 174, 168, 11, 76, 254, 16, 73, 135, 140, - 2, 80, 8, 52, 7, 73, 82, 67, 76, 69, 68, 32, 147, 200, 13, 79, 6, 40, 2, - 80, 76, 14, 84, 167, 244, 7, 68, 2, 35, 85, 2, 21, 3, 73, 77, 69, 2, 147, - 172, 11, 83, 6, 40, 6, 81, 85, 65, 82, 69, 32, 87, 85, 4, 60, 9, 73, 78, - 84, 69, 82, 83, 69, 67, 84, 1, 2, 85, 78, 2, 255, 146, 11, 73, 2, 11, 77, - 2, 235, 168, 11, 77, 7, 69, 15, 32, 79, 80, 69, 82, 65, 84, 79, 82, 32, - 87, 73, 84, 72, 32, 4, 198, 247, 10, 80, 211, 140, 3, 68, 2, 11, 69, 2, - 145, 132, 12, 2, 32, 86, 188, 2, 170, 2, 66, 252, 4, 10, 71, 32, 77, 85, - 78, 68, 65, 82, 73, 32, 154, 4, 73, 52, 2, 78, 68, 216, 4, 7, 84, 73, 79, - 78, 65, 76, 32, 240, 206, 1, 2, 85, 83, 136, 236, 7, 5, 77, 69, 32, 66, - 65, 148, 172, 3, 7, 90, 65, 82, 32, 65, 77, 85, 196, 157, 1, 8, 82, 82, - 79, 87, 32, 78, 79, 45, 219, 62, 75, 82, 52, 7, 65, 84, 65, 69, 65, 78, - 32, 219, 206, 14, 76, 80, 160, 1, 7, 76, 69, 84, 84, 69, 82, 32, 236, 2, - 7, 78, 85, 77, 66, 69, 82, 32, 193, 207, 9, 16, 67, 82, 85, 67, 73, 70, - 79, 82, 77, 32, 78, 85, 77, 66, 69, 82, 62, 236, 1, 6, 70, 73, 78, 65, - 76, 32, 174, 253, 1, 84, 246, 118, 68, 34, 76, 50, 81, 206, 194, 1, 82, - 134, 212, 2, 65, 50, 71, 90, 90, 34, 83, 66, 89, 154, 193, 1, 72, 234, 5, - 75, 130, 76, 66, 190, 173, 4, 78, 254, 1, 87, 202, 103, 80, 171, 4, 77, - 18, 138, 244, 2, 65, 54, 76, 174, 149, 4, 83, 190, 3, 89, 130, 199, 1, - 75, 130, 76, 66, 190, 173, 4, 78, 198, 105, 72, 171, 4, 77, 16, 154, 245, - 2, 84, 186, 151, 4, 79, 143, 211, 5, 70, 84, 84, 7, 76, 69, 84, 84, 69, - 82, 32, 164, 2, 5, 83, 73, 71, 78, 32, 183, 219, 12, 68, 54, 42, 65, 50, - 69, 66, 73, 50, 79, 47, 85, 11, 254, 185, 14, 78, 202, 17, 66, 2, 72, 3, - 74, 15, 150, 211, 12, 78, 230, 110, 76, 242, 108, 84, 174, 28, 71, 3, 77, - 11, 194, 132, 14, 68, 150, 70, 72, 2, 83, 3, 84, 11, 210, 201, 14, 78, - 86, 76, 2, 80, 3, 89, 11, 250, 201, 14, 67, 2, 68, 2, 75, 3, 82, 10, 112, - 2, 77, 85, 20, 3, 83, 85, 84, 172, 165, 2, 2, 73, 75, 184, 129, 9, 2, 79, - 74, 137, 233, 2, 3, 84, 79, 89, 2, 255, 143, 14, 72, 2, 183, 199, 14, 85, - 4, 236, 137, 11, 5, 76, 32, 80, 79, 76, 231, 2, 82, 133, 1, 41, 8, 73, - 78, 65, 71, 65, 82, 73, 32, 130, 1, 140, 1, 7, 76, 69, 84, 84, 69, 82, - 32, 252, 1, 5, 83, 73, 71, 78, 32, 52, 11, 86, 79, 87, 69, 76, 32, 83, - 73, 71, 78, 32, 199, 207, 4, 72, 94, 214, 1, 86, 202, 164, 10, 82, 206, - 55, 65, 38, 68, 46, 84, 230, 24, 85, 210, 200, 1, 73, 42, 76, 246, 189, - 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, - 77, 2, 89, 186, 2, 69, 3, 79, 6, 242, 254, 4, 79, 243, 197, 9, 65, 10, - 202, 205, 9, 83, 134, 144, 1, 65, 243, 163, 3, 86, 24, 210, 213, 4, 80, - 190, 40, 86, 166, 225, 5, 65, 190, 21, 85, 210, 200, 1, 73, 206, 134, 2, - 69, 3, 79, 4, 202, 47, 68, 191, 210, 13, 80, 4, 250, 194, 14, 83, 15, 72, - 246, 4, 216, 1, 3, 71, 65, 84, 128, 7, 6, 73, 84, 72, 69, 82, 32, 194, 3, - 83, 136, 1, 2, 85, 84, 242, 1, 87, 144, 16, 3, 88, 84, 32, 140, 150, 8, - 4, 80, 84, 85, 78, 240, 239, 2, 2, 67, 75, 150, 141, 2, 82, 183, 144, 1, - 76, 154, 1, 152, 1, 4, 73, 86, 69, 32, 245, 160, 12, 27, 69, 68, 32, 68, - 79, 85, 66, 76, 69, 32, 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 82, - 32, 68, 79, 85, 66, 152, 1, 152, 1, 8, 67, 73, 82, 67, 76, 69, 68, 32, - 216, 1, 9, 68, 73, 65, 71, 79, 78, 65, 76, 32, 196, 1, 8, 83, 81, 85, 65, - 82, 69, 68, 32, 147, 170, 8, 65, 78, 104, 5, 68, 73, 71, 73, 84, 28, 7, - 78, 85, 77, 66, 69, 82, 32, 142, 3, 76, 238, 150, 13, 84, 139, 3, 83, 2, - 189, 209, 12, 2, 32, 90, 20, 50, 84, 218, 254, 1, 69, 46, 70, 42, 78, 31, - 83, 6, 154, 128, 2, 72, 47, 87, 6, 38, 77, 234, 254, 12, 67, 191, 2, 68, - 2, 49, 10, 73, 68, 68, 76, 69, 32, 82, 73, 71, 72, 2, 17, 2, 84, 32, 2, - 25, 4, 84, 79, 32, 76, 2, 25, 4, 79, 87, 69, 82, 2, 169, 198, 11, 2, 32, - 67, 66, 118, 76, 248, 229, 2, 4, 67, 82, 79, 83, 142, 138, 9, 81, 146, - 144, 1, 65, 198, 19, 73, 2, 87, 170, 164, 1, 80, 3, 83, 52, 221, 209, 8, - 6, 65, 84, 73, 78, 32, 67, 18, 158, 1, 65, 216, 1, 17, 71, 82, 69, 65, - 84, 69, 82, 45, 84, 72, 65, 78, 32, 78, 79, 82, 32, 37, 14, 76, 69, 83, - 83, 45, 84, 72, 65, 78, 32, 78, 79, 82, 32, 6, 92, 3, 32, 83, 85, 85, 16, - 80, 80, 82, 79, 88, 73, 77, 65, 84, 69, 76, 89, 32, 78, 79, 82, 4, 30, - 66, 1, 3, 80, 69, 82, 2, 253, 223, 6, 8, 83, 69, 84, 32, 79, 70, 32, 78, - 2, 213, 48, 5, 32, 65, 67, 84, 85, 6, 250, 246, 7, 69, 191, 177, 4, 76, - 6, 214, 246, 7, 69, 155, 177, 4, 71, 6, 26, 84, 199, 179, 12, 83, 4, 76, - 5, 73, 78, 71, 32, 68, 145, 176, 10, 8, 32, 87, 73, 84, 72, 32, 69, 71, - 2, 133, 228, 7, 2, 79, 76, 64, 40, 4, 82, 65, 76, 32, 203, 252, 13, 69, - 62, 48, 6, 67, 72, 69, 83, 83, 32, 235, 247, 13, 70, 60, 74, 75, 134, - 218, 12, 66, 38, 69, 130, 5, 80, 22, 81, 38, 82, 131, 2, 84, 20, 44, 5, - 78, 73, 71, 72, 84, 151, 219, 12, 73, 15, 195, 219, 12, 32, 246, 2, 70, - 32, 184, 8, 2, 65, 32, 248, 6, 3, 76, 73, 78, 183, 206, 3, 83, 174, 1, - 90, 77, 64, 4, 83, 72, 69, 81, 20, 8, 84, 65, 73, 32, 76, 85, 69, 32, - 183, 152, 13, 76, 4, 25, 4, 79, 79, 78, 32, 4, 234, 244, 7, 87, 171, 170, - 5, 83, 2, 183, 154, 9, 69, 166, 1, 160, 1, 7, 76, 69, 84, 84, 69, 82, 32, - 244, 2, 8, 83, 73, 71, 78, 32, 76, 65, 69, 22, 84, 76, 11, 86, 79, 87, - 69, 76, 32, 83, 73, 71, 78, 32, 191, 190, 12, 68, 102, 76, 6, 70, 73, 78, - 65, 76, 32, 68, 4, 72, 73, 71, 72, 1, 3, 76, 79, 87, 14, 198, 183, 12, - 78, 150, 248, 1, 66, 2, 68, 2, 75, 2, 77, 3, 86, 44, 11, 32, 44, 146, 1, - 75, 2, 88, 34, 83, 210, 222, 10, 78, 134, 135, 3, 84, 82, 80, 254, 68, - 66, 2, 68, 2, 70, 2, 72, 2, 76, 2, 77, 2, 81, 2, 86, 3, 89, 4, 190, 171, - 14, 86, 187, 2, 65, 4, 158, 171, 14, 85, 187, 2, 65, 5, 183, 173, 14, 86, - 6, 224, 133, 7, 3, 79, 78, 69, 253, 76, 8, 72, 65, 77, 32, 68, 73, 71, - 73, 34, 110, 65, 46, 73, 30, 79, 38, 85, 216, 231, 11, 11, 86, 79, 87, - 69, 76, 32, 83, 72, 79, 82, 84, 167, 195, 2, 69, 8, 238, 184, 10, 65, - 250, 242, 3, 69, 3, 89, 4, 186, 171, 14, 73, 3, 89, 9, 166, 184, 10, 65, - 251, 242, 3, 89, 11, 130, 184, 10, 69, 250, 242, 3, 85, 3, 89, 194, 1, - 182, 1, 68, 106, 71, 72, 7, 76, 69, 84, 84, 69, 82, 32, 214, 2, 83, 190, - 23, 80, 202, 133, 1, 86, 138, 200, 7, 65, 144, 230, 1, 5, 73, 78, 83, 69, - 82, 210, 142, 1, 67, 207, 186, 2, 79, 26, 64, 6, 79, 85, 66, 76, 69, 32, - 214, 211, 4, 65, 239, 230, 7, 73, 4, 198, 211, 4, 68, 231, 137, 7, 67, 2, - 11, 65, 2, 11, 80, 2, 17, 2, 32, 70, 2, 213, 224, 10, 2, 73, 76, 108, - 218, 1, 78, 62, 86, 254, 188, 10, 65, 38, 68, 46, 84, 230, 24, 85, 210, - 200, 1, 73, 202, 190, 1, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 76, - 2, 77, 2, 80, 2, 82, 254, 68, 72, 2, 87, 2, 89, 186, 2, 69, 3, 79, 14, - 210, 222, 13, 71, 2, 89, 254, 68, 72, 2, 78, 187, 2, 65, 10, 26, 69, 207, - 193, 10, 79, 2, 173, 246, 11, 3, 68, 73, 67, 22, 26, 73, 139, 179, 4, 65, - 20, 44, 3, 71, 78, 32, 213, 128, 9, 2, 68, 68, 18, 210, 189, 10, 65, 74, - 67, 98, 78, 230, 179, 1, 74, 228, 2, 5, 70, 73, 78, 65, 76, 50, 85, 211, - 235, 1, 86, 4, 203, 248, 13, 69, 4, 250, 200, 9, 80, 171, 193, 3, 76, 6, - 70, 78, 245, 141, 13, 11, 71, 72, 84, 32, 87, 73, 84, 72, 32, 83, 84, 4, - 156, 217, 7, 7, 69, 32, 80, 79, 73, 78, 84, 131, 199, 6, 74, 124, 152, 1, - 3, 67, 79, 77, 242, 2, 68, 78, 76, 156, 4, 4, 72, 73, 71, 72, 72, 7, 83, - 89, 77, 66, 79, 76, 32, 238, 192, 7, 69, 133, 178, 1, 3, 84, 65, 77, 20, - 52, 7, 66, 73, 78, 73, 78, 71, 32, 143, 158, 14, 77, 18, 116, 5, 76, 79, - 78, 71, 32, 76, 2, 78, 65, 20, 6, 83, 72, 79, 82, 84, 32, 165, 158, 13, - 6, 68, 79, 85, 66, 76, 69, 8, 130, 1, 72, 34, 76, 194, 207, 10, 82, 21, - 7, 68, 69, 83, 67, 69, 78, 68, 2, 155, 172, 10, 83, 6, 34, 72, 34, 76, - 195, 207, 10, 82, 2, 137, 208, 10, 3, 73, 71, 72, 2, 233, 207, 10, 2, 79, - 87, 24, 152, 1, 4, 65, 78, 84, 65, 152, 133, 12, 4, 79, 82, 79, 77, 203, - 41, 73, 70, 76, 4, 65, 74, 65, 78, 32, 6, 69, 84, 84, 69, 82, 32, 173, 3, - 2, 79, 87, 2, 185, 213, 12, 3, 89, 65, 76, 66, 228, 1, 2, 68, 65, 42, 74, - 90, 78, 206, 249, 9, 82, 158, 225, 1, 79, 198, 71, 67, 170, 183, 1, 69, - 238, 18, 71, 242, 42, 66, 2, 70, 2, 72, 2, 75, 2, 76, 2, 77, 2, 80, 2, - 83, 2, 84, 2, 87, 2, 89, 186, 2, 65, 2, 73, 3, 85, 5, 237, 248, 10, 5, - 71, 66, 65, 83, 73, 8, 40, 4, 79, 78, 65, 32, 151, 154, 14, 65, 6, 214, - 162, 12, 67, 134, 245, 1, 74, 3, 82, 11, 26, 65, 1, 2, 89, 65, 5, 173, - 193, 5, 5, 32, 87, 79, 76, 79, 2, 29, 5, 32, 84, 79, 78, 69, 2, 17, 2, - 32, 65, 2, 135, 171, 8, 80, 4, 68, 7, 71, 66, 65, 75, 85, 82, 85, 1, 6, - 79, 79, 32, 68, 69, 78, 2, 159, 199, 12, 78, 180, 1, 90, 32, 156, 3, 2, - 77, 73, 118, 78, 150, 1, 82, 238, 7, 84, 186, 203, 13, 45, 135, 40, 83, - 18, 202, 1, 66, 96, 5, 69, 78, 84, 82, 89, 22, 80, 160, 180, 2, 15, 79, - 78, 69, 32, 85, 78, 68, 69, 82, 32, 69, 73, 71, 72, 84, 220, 242, 3, 9, - 77, 79, 66, 73, 76, 69, 32, 80, 72, 181, 30, 3, 83, 77, 79, 4, 52, 4, 82, - 69, 65, 75, 173, 131, 2, 3, 73, 67, 89, 2, 17, 2, 32, 72, 2, 231, 244, - 12, 69, 5, 151, 142, 13, 32, 4, 60, 6, 69, 68, 69, 83, 84, 82, 161, 167, - 9, 3, 73, 82, 65, 2, 137, 230, 10, 2, 73, 65, 4, 36, 5, 78, 65, 76, 32, - 68, 59, 83, 2, 249, 255, 12, 9, 73, 71, 73, 84, 32, 83, 72, 65, 80, 2, - 195, 215, 10, 77, 6, 38, 45, 185, 183, 9, 3, 70, 79, 82, 4, 76, 8, 66, - 82, 69, 65, 75, 73, 78, 71, 245, 162, 2, 5, 80, 79, 84, 65, 66, 2, 129, - 201, 6, 2, 32, 72, 93, 56, 2, 84, 72, 146, 11, 77, 173, 185, 10, 3, 68, - 73, 67, 84, 78, 32, 213, 131, 11, 13, 69, 65, 83, 84, 45, 80, 79, 73, 78, - 84, 73, 78, 71, 82, 96, 5, 69, 65, 83, 84, 32, 132, 2, 6, 73, 78, 68, 73, - 67, 32, 217, 1, 5, 87, 69, 83, 84, 32, 30, 82, 65, 182, 221, 6, 80, 130, - 1, 84, 246, 160, 4, 66, 38, 68, 18, 83, 203, 43, 87, 12, 40, 4, 82, 82, - 79, 87, 175, 217, 6, 78, 11, 11, 32, 8, 80, 9, 67, 82, 79, 83, 83, 73, - 78, 71, 32, 244, 2, 2, 65, 78, 255, 216, 6, 87, 4, 218, 180, 3, 83, 207, - 166, 3, 78, 20, 50, 80, 60, 3, 81, 85, 65, 66, 82, 215, 126, 70, 2, 37, - 7, 76, 65, 67, 69, 72, 79, 76, 2, 135, 158, 4, 68, 4, 240, 157, 4, 2, 82, - 84, 237, 173, 9, 5, 78, 84, 73, 84, 89, 2, 17, 2, 85, 80, 2, 187, 144, 4, - 69, 32, 82, 65, 218, 217, 6, 80, 130, 1, 84, 246, 160, 4, 66, 38, 68, 18, - 83, 203, 43, 87, 14, 34, 78, 33, 4, 82, 82, 79, 87, 2, 229, 177, 3, 3, - 68, 32, 83, 13, 11, 32, 10, 68, 3, 84, 79, 32, 138, 213, 6, 67, 40, 3, - 65, 78, 68, 219, 2, 87, 4, 146, 214, 6, 67, 229, 228, 4, 4, 76, 79, 78, - 71, 56, 54, 32, 236, 5, 5, 67, 72, 69, 68, 32, 207, 2, 69, 38, 162, 1, - 65, 132, 2, 4, 78, 79, 82, 77, 78, 80, 46, 83, 170, 1, 84, 250, 195, 7, - 69, 196, 25, 7, 73, 68, 69, 78, 84, 73, 67, 214, 151, 4, 71, 38, 76, 243, - 77, 67, 10, 88, 3, 32, 83, 85, 52, 7, 78, 32, 69, 76, 69, 77, 69, 28, 2, - 83, 89, 139, 223, 7, 76, 4, 30, 66, 1, 3, 80, 69, 82, 2, 29, 2, 83, 69, - 2, 11, 78, 2, 243, 119, 84, 2, 37, 7, 77, 80, 84, 79, 84, 73, 67, 2, 17, - 2, 65, 76, 2, 201, 223, 7, 2, 76, 89, 4, 181, 174, 6, 14, 65, 76, 32, 83, - 85, 66, 71, 82, 79, 85, 80, 32, 79, 70, 2, 153, 223, 7, 6, 65, 82, 65, - 76, 76, 69, 6, 48, 6, 81, 85, 65, 82, 69, 32, 195, 254, 12, 73, 4, 68, 5, - 73, 77, 65, 71, 69, 1, 8, 79, 82, 73, 71, 73, 78, 65, 76, 2, 21, 3, 32, - 79, 70, 2, 183, 172, 6, 32, 4, 178, 163, 10, 82, 227, 202, 1, 73, 8, 58, - 76, 40, 4, 82, 73, 71, 72, 133, 1, 3, 85, 80, 80, 4, 36, 2, 69, 70, 133, - 1, 2, 79, 87, 2, 89, 20, 84, 32, 83, 69, 77, 73, 67, 73, 82, 67, 76, 69, - 32, 87, 73, 84, 72, 32, 84, 72, 2, 17, 2, 82, 69, 2, 207, 180, 12, 69, 2, - 11, 69, 2, 41, 8, 82, 32, 82, 73, 71, 72, 84, 45, 2, 153, 131, 3, 6, 83, - 72, 65, 68, 79, 87, 11, 34, 32, 53, 4, 66, 79, 79, 75, 4, 17, 2, 80, 65, - 4, 146, 234, 13, 71, 215, 22, 68, 5, 81, 18, 32, 87, 73, 84, 72, 32, 68, - 69, 67, 79, 82, 65, 84, 73, 86, 69, 32, 67, 2, 223, 251, 3, 79, 186, 6, - 102, 77, 176, 3, 4, 83, 72, 85, 32, 168, 157, 5, 8, 84, 32, 65, 78, 68, - 32, 66, 79, 131, 226, 7, 76, 26, 36, 4, 66, 69, 82, 32, 247, 2, 69, 24, - 58, 69, 50, 70, 42, 83, 66, 84, 57, 4, 78, 73, 78, 69, 4, 204, 1, 3, 73, - 71, 72, 21, 3, 76, 69, 86, 4, 144, 1, 2, 79, 85, 13, 2, 73, 70, 6, 34, - 73, 85, 4, 69, 86, 69, 78, 4, 82, 88, 239, 172, 13, 71, 8, 40, 2, 72, 73, - 46, 69, 21, 2, 87, 69, 2, 11, 82, 2, 17, 2, 84, 69, 2, 11, 69, 2, 143, - 175, 7, 78, 4, 248, 174, 7, 3, 76, 86, 69, 1, 3, 78, 84, 89, 2, 155, 221, - 6, 82, 154, 6, 72, 12, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, - 215, 222, 11, 73, 152, 6, 18, 49, 87, 50, 160, 2, 222, 1, 55, 2, 56, 2, - 57, 2, 65, 2, 66, 2, 67, 2, 68, 2, 69, 3, 70, 248, 3, 138, 1, 48, 2, 49, - 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, - 67, 2, 68, 2, 69, 143, 1, 70, 32, 246, 248, 13, 48, 2, 49, 2, 50, 2, 51, - 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, 68, 2, - 69, 3, 70, 24, 234, 247, 13, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, - 54, 2, 55, 2, 56, 2, 57, 2, 65, 3, 66, 142, 1, 108, 8, 67, 73, 82, 67, - 76, 69, 68, 32, 22, 76, 226, 3, 83, 160, 2, 5, 84, 79, 78, 69, 45, 223, - 129, 12, 68, 2, 211, 243, 13, 67, 92, 88, 6, 69, 84, 84, 69, 82, 32, 141, - 157, 1, 10, 79, 71, 79, 71, 82, 65, 77, 32, 78, 89, 90, 130, 2, 78, 90, - 84, 222, 188, 7, 88, 186, 116, 65, 146, 161, 1, 82, 188, 44, 2, 72, 65, - 226, 180, 1, 79, 222, 56, 68, 2, 77, 2, 80, 146, 198, 1, 69, 222, 61, 67, - 2, 70, 2, 71, 2, 75, 2, 76, 2, 81, 2, 83, 2, 86, 2, 89, 2, 90, 186, 2, - 73, 2, 85, 3, 87, 22, 86, 84, 158, 236, 11, 80, 238, 131, 2, 67, 2, 75, - 2, 81, 2, 82, 2, 89, 187, 2, 65, 6, 134, 240, 13, 83, 2, 88, 187, 2, 65, - 14, 64, 4, 73, 71, 78, 32, 185, 1, 7, 89, 76, 76, 65, 66, 76, 69, 12, 56, - 4, 70, 79, 82, 32, 137, 243, 12, 4, 88, 87, 32, 88, 10, 152, 9, 2, 76, - 79, 190, 147, 3, 84, 132, 89, 8, 73, 78, 86, 69, 82, 84, 69, 66, 154, - 168, 3, 80, 237, 86, 3, 65, 78, 73, 2, 221, 172, 11, 4, 32, 76, 69, 78, - 14, 246, 239, 13, 66, 2, 68, 2, 71, 2, 74, 2, 77, 2, 83, 3, 86, 214, 13, - 178, 2, 66, 226, 1, 67, 224, 5, 4, 70, 70, 73, 67, 54, 71, 248, 6, 4, 73, - 76, 32, 68, 22, 76, 222, 63, 78, 166, 5, 80, 214, 7, 82, 190, 10, 83, - 212, 8, 2, 84, 84, 154, 8, 85, 180, 1, 3, 86, 69, 82, 234, 159, 2, 89, - 138, 180, 5, 72, 190, 233, 1, 75, 174, 230, 1, 68, 230, 61, 77, 246, 9, - 87, 159, 137, 1, 88, 10, 132, 1, 6, 76, 73, 81, 85, 69, 32, 140, 152, 2, - 9, 83, 69, 82, 86, 69, 82, 32, 69, 89, 221, 28, 8, 74, 69, 67, 84, 32, - 82, 69, 80, 6, 160, 177, 4, 13, 65, 78, 71, 76, 69, 32, 79, 80, 69, 78, - 73, 78, 71, 191, 241, 1, 72, 28, 56, 2, 82, 32, 234, 4, 84, 229, 172, 5, - 3, 67, 85, 76, 22, 156, 1, 11, 65, 77, 79, 85, 78, 84, 32, 79, 70, 32, - 67, 22, 66, 198, 1, 67, 118, 68, 106, 70, 0, 10, 73, 78, 86, 69, 82, 84, - 69, 68, 32, 70, 175, 149, 12, 72, 2, 207, 158, 5, 72, 6, 136, 1, 15, 82, - 65, 78, 67, 72, 32, 66, 65, 78, 75, 32, 73, 68, 69, 78, 144, 233, 4, 5, - 69, 76, 84, 32, 66, 201, 225, 5, 3, 79, 87, 32, 2, 21, 3, 84, 73, 70, 2, - 11, 73, 2, 199, 187, 10, 67, 4, 92, 17, 85, 83, 84, 79, 77, 69, 82, 32, - 65, 67, 67, 79, 85, 78, 84, 32, 78, 227, 195, 1, 72, 2, 159, 135, 6, 85, - 4, 44, 5, 79, 85, 66, 76, 69, 171, 129, 12, 65, 2, 11, 32, 2, 11, 66, 2, - 181, 133, 7, 3, 65, 67, 75, 2, 247, 164, 13, 79, 4, 184, 206, 8, 4, 65, - 71, 79, 78, 209, 148, 3, 2, 79, 80, 2, 11, 69, 2, 213, 215, 7, 5, 32, 66, - 85, 73, 76, 60, 48, 4, 72, 65, 77, 32, 177, 225, 13, 2, 79, 78, 58, 122, - 70, 0, 10, 82, 69, 86, 69, 82, 83, 69, 68, 32, 70, 36, 7, 76, 69, 84, 84, - 69, 82, 32, 161, 231, 3, 3, 83, 80, 65, 2, 177, 244, 3, 4, 69, 65, 84, - 72, 52, 196, 1, 2, 65, 73, 22, 66, 2, 80, 34, 67, 36, 2, 69, 65, 64, 2, - 70, 69, 22, 71, 22, 73, 58, 76, 2, 82, 22, 78, 46, 79, 34, 83, 50, 85, - 150, 178, 1, 77, 170, 9, 68, 149, 138, 11, 3, 84, 73, 78, 2, 147, 209, - 13, 76, 2, 11, 69, 2, 199, 226, 11, 73, 4, 138, 239, 7, 69, 255, 237, 3, - 79, 6, 138, 1, 66, 2, 68, 201, 247, 3, 6, 77, 72, 65, 78, 67, 72, 2, 199, - 130, 9, 65, 2, 199, 217, 12, 79, 4, 32, 2, 79, 68, 207, 224, 12, 70, 2, - 235, 224, 7, 72, 2, 135, 195, 11, 85, 4, 176, 204, 12, 3, 71, 69, 65, - 207, 67, 73, 4, 222, 143, 13, 78, 215, 79, 82, 4, 158, 178, 11, 65, 161, - 171, 1, 3, 84, 82, 65, 6, 60, 5, 73, 76, 76, 69, 65, 162, 223, 11, 65, - 135, 255, 1, 82, 2, 207, 142, 13, 78, 2, 155, 182, 12, 82, 220, 7, 64, 7, - 32, 67, 72, 73, 75, 73, 32, 206, 6, 68, 167, 218, 12, 73, 96, 132, 1, 7, - 76, 69, 84, 84, 69, 82, 32, 148, 3, 2, 77, 85, 62, 71, 74, 80, 164, 198, - 3, 2, 82, 69, 214, 163, 8, 68, 243, 167, 1, 65, 60, 50, 65, 98, 69, 54, - 73, 50, 76, 62, 79, 51, 85, 16, 50, 65, 242, 218, 13, 78, 86, 71, 2, 76, - 3, 84, 8, 194, 219, 13, 74, 2, 75, 2, 77, 3, 87, 8, 130, 149, 13, 68, - 198, 13, 82, 210, 56, 78, 3, 80, 8, 154, 201, 13, 78, 202, 17, 72, 2, 82, - 3, 83, 12, 134, 153, 8, 65, 174, 193, 5, 69, 2, 73, 2, 79, 3, 85, 8, 202, - 189, 13, 84, 174, 28, 66, 2, 72, 3, 86, 8, 242, 137, 13, 78, 214, 79, 67, - 2, 68, 3, 89, 4, 56, 2, 45, 71, 221, 242, 11, 6, 32, 84, 84, 85, 68, 68, - 2, 217, 242, 11, 13, 65, 65, 72, 76, 65, 65, 32, 84, 84, 85, 68, 68, 65, - 6, 84, 11, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 137, 197, 6, 4, - 72, 65, 65, 82, 4, 48, 8, 68, 79, 85, 66, 76, 69, 32, 77, 3, 77, 2, 229, - 144, 13, 3, 85, 67, 65, 250, 6, 34, 32, 225, 55, 3, 69, 82, 32, 244, 6, - 240, 2, 8, 67, 72, 73, 78, 69, 83, 69, 32, 44, 10, 72, 85, 78, 71, 65, - 82, 73, 65, 78, 32, 244, 6, 7, 73, 84, 65, 76, 73, 67, 32, 212, 4, 14, - 78, 79, 82, 84, 72, 32, 65, 82, 65, 66, 73, 65, 78, 32, 196, 4, 3, 80, - 69, 82, 164, 12, 2, 83, 79, 144, 13, 14, 84, 85, 82, 75, 73, 67, 32, 76, - 69, 84, 84, 69, 82, 32, 132, 7, 7, 85, 89, 71, 72, 85, 82, 32, 151, 140, - 11, 75, 4, 166, 183, 11, 73, 205, 91, 3, 72, 79, 79, 216, 1, 96, 6, 67, - 65, 80, 73, 84, 65, 0, 4, 83, 77, 65, 76, 221, 5, 7, 78, 85, 77, 66, 69, - 82, 32, 102, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 102, 226, 1, 65, - 54, 69, 164, 2, 10, 78, 73, 75, 79, 76, 83, 66, 85, 82, 71, 0, 9, 82, 85, - 68, 73, 77, 69, 78, 84, 65, 42, 79, 34, 85, 132, 223, 3, 5, 67, 76, 79, - 83, 69, 178, 231, 7, 73, 237, 200, 1, 6, 83, 72, 79, 82, 84, 32, 11, 250, - 149, 12, 77, 146, 116, 78, 150, 70, 65, 3, 75, 63, 174, 1, 77, 22, 78, - 78, 83, 146, 218, 9, 67, 86, 71, 2, 76, 2, 84, 154, 239, 1, 90, 226, 131, - 2, 66, 2, 68, 2, 69, 2, 70, 2, 72, 2, 74, 2, 75, 2, 80, 2, 82, 3, 86, 5, - 187, 206, 13, 80, 11, 34, 84, 134, 206, 13, 67, 3, 89, 5, 165, 145, 2, 5, - 45, 83, 72, 65, 80, 5, 219, 205, 13, 90, 4, 11, 32, 4, 230, 182, 13, 79, - 3, 85, 7, 202, 182, 13, 69, 215, 22, 79, 9, 230, 201, 13, 78, 154, 3, 83, - 3, 85, 12, 210, 4, 70, 234, 237, 5, 79, 171, 137, 6, 84, 78, 80, 7, 76, - 69, 84, 84, 69, 82, 32, 169, 3, 8, 78, 85, 77, 69, 82, 65, 76, 32, 70, - 190, 1, 69, 90, 75, 50, 83, 36, 3, 78, 79, 82, 206, 249, 9, 85, 210, 200, - 1, 73, 198, 134, 1, 80, 2, 84, 254, 82, 67, 186, 22, 66, 2, 68, 2, 72, 2, - 86, 2, 89, 2, 90, 214, 22, 65, 3, 79, 23, 198, 214, 9, 83, 238, 239, 1, - 82, 146, 198, 1, 75, 210, 61, 70, 2, 76, 2, 77, 3, 78, 8, 230, 178, 13, - 72, 214, 22, 65, 2, 69, 3, 85, 4, 32, 2, 79, 85, 151, 178, 13, 72, 2, 29, - 5, 84, 72, 69, 82, 78, 2, 209, 191, 12, 2, 32, 84, 8, 38, 70, 146, 247, - 11, 84, 227, 55, 79, 4, 11, 73, 4, 130, 222, 11, 70, 163, 211, 1, 86, 64, - 76, 7, 76, 69, 84, 84, 69, 82, 32, 209, 3, 7, 78, 85, 77, 66, 69, 82, 32, - 58, 210, 1, 65, 38, 71, 38, 72, 30, 75, 38, 84, 74, 90, 246, 104, 77, - 244, 138, 3, 2, 69, 83, 218, 156, 3, 66, 2, 70, 2, 82, 2, 89, 242, 161, - 3, 78, 246, 215, 1, 76, 150, 55, 68, 146, 1, 81, 254, 2, 87, 255, 55, 83, - 4, 138, 180, 3, 76, 211, 193, 9, 73, 4, 218, 153, 12, 72, 135, 153, 1, - 69, 4, 214, 195, 13, 65, 3, 69, 4, 130, 140, 6, 72, 191, 183, 6, 65, 8, - 34, 72, 246, 194, 13, 65, 3, 69, 4, 230, 186, 12, 65, 143, 136, 1, 69, 4, - 11, 65, 4, 254, 243, 12, 73, 215, 79, 72, 6, 190, 131, 6, 84, 227, 166, - 6, 79, 178, 1, 64, 11, 77, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 183, - 5, 83, 76, 228, 1, 2, 67, 72, 22, 68, 66, 69, 22, 73, 30, 77, 2, 78, 30, - 80, 22, 83, 70, 84, 50, 86, 38, 89, 94, 90, 218, 172, 6, 76, 2, 82, 254, - 209, 2, 71, 230, 255, 1, 79, 190, 198, 1, 66, 242, 40, 65, 242, 31, 75, - 174, 45, 72, 187, 2, 85, 2, 171, 180, 7, 69, 6, 26, 90, 219, 172, 13, 79, - 4, 138, 254, 8, 72, 219, 157, 4, 73, 5, 139, 192, 13, 70, 7, 246, 191, - 13, 65, 3, 69, 2, 249, 168, 13, 2, 69, 78, 2, 223, 174, 6, 69, 6, 26, 72, - 187, 171, 13, 73, 4, 232, 252, 8, 3, 67, 72, 79, 3, 79, 4, 26, 83, 247, - 170, 13, 65, 2, 227, 153, 13, 73, 4, 146, 252, 8, 79, 215, 137, 4, 69, - 14, 60, 2, 69, 82, 226, 136, 8, 65, 174, 163, 5, 82, 203, 17, 85, 7, 210, - 189, 13, 73, 3, 85, 4, 146, 251, 8, 72, 219, 157, 4, 65, 102, 52, 4, 73, - 65, 78, 32, 213, 153, 1, 3, 79, 78, 65, 100, 80, 7, 78, 85, 77, 66, 69, - 82, 32, 80, 5, 83, 73, 71, 78, 32, 139, 154, 10, 87, 10, 42, 84, 186, - 147, 8, 72, 147, 143, 4, 79, 6, 162, 228, 1, 87, 231, 135, 11, 69, 88, - 210, 1, 65, 86, 66, 62, 68, 98, 74, 2, 86, 30, 84, 42, 88, 226, 109, 71, - 2, 75, 2, 78, 2, 82, 250, 228, 8, 77, 214, 156, 3, 83, 206, 69, 67, 2, - 70, 2, 72, 2, 76, 2, 80, 2, 89, 2, 90, 186, 2, 73, 3, 85, 9, 45, 9, 85, - 82, 65, 77, 65, 90, 68, 65, 65, 7, 222, 213, 6, 45, 159, 225, 6, 72, 6, - 38, 65, 165, 250, 9, 3, 85, 85, 77, 5, 175, 182, 13, 71, 10, 34, 65, 178, - 184, 13, 73, 3, 85, 7, 37, 7, 72, 89, 65, 65, 85, 83, 72, 5, 179, 212, 6, - 45, 4, 242, 183, 13, 65, 3, 73, 6, 158, 181, 13, 72, 186, 2, 65, 3, 85, - 4, 132, 137, 11, 8, 83, 72, 65, 65, 89, 65, 84, 72, 171, 174, 2, 65, 144, - 1, 92, 6, 71, 68, 73, 65, 78, 32, 141, 7, 12, 85, 84, 72, 32, 65, 82, 65, - 66, 73, 65, 78, 32, 80, 58, 70, 74, 76, 145, 5, 7, 78, 85, 77, 66, 69, - 82, 32, 2, 11, 82, 2, 157, 155, 11, 10, 65, 67, 84, 73, 79, 78, 32, 79, - 78, 69, 60, 76, 6, 69, 84, 84, 69, 82, 32, 149, 4, 8, 73, 71, 65, 84, 85, - 82, 69, 32, 58, 234, 1, 65, 96, 6, 70, 73, 78, 65, 76, 32, 200, 1, 5, 82, - 69, 83, 72, 45, 230, 214, 1, 76, 178, 151, 4, 71, 90, 90, 34, 83, 66, 89, - 154, 193, 1, 72, 234, 5, 75, 130, 76, 66, 190, 173, 4, 78, 254, 1, 84, 2, - 87, 202, 103, 80, 171, 4, 77, 6, 26, 76, 243, 178, 12, 89, 4, 244, 240, - 5, 8, 84, 69, 82, 78, 65, 84, 69, 32, 239, 199, 1, 69, 18, 116, 3, 78, - 85, 78, 0, 5, 83, 65, 68, 72, 69, 0, 3, 84, 65, 87, 130, 215, 1, 65, 158, - 172, 6, 66, 131, 151, 5, 72, 5, 41, 8, 32, 87, 73, 84, 72, 32, 86, 69, 2, - 209, 199, 5, 4, 82, 84, 73, 67, 2, 193, 214, 1, 6, 65, 89, 73, 78, 45, - 68, 18, 42, 84, 158, 239, 5, 79, 143, 211, 5, 70, 10, 42, 72, 230, 215, - 1, 87, 231, 135, 11, 69, 4, 230, 194, 11, 82, 159, 2, 73, 64, 60, 7, 76, - 69, 84, 84, 69, 82, 32, 245, 3, 3, 78, 85, 77, 58, 202, 1, 65, 38, 68, - 74, 71, 34, 75, 34, 83, 78, 84, 218, 43, 90, 230, 165, 1, 76, 50, 81, - 206, 194, 1, 82, 238, 213, 2, 89, 154, 193, 1, 72, 234, 81, 66, 190, 173, - 4, 78, 254, 1, 87, 202, 103, 70, 171, 4, 77, 4, 202, 155, 3, 76, 211, - 193, 9, 89, 6, 32, 2, 72, 65, 219, 210, 1, 65, 4, 210, 254, 7, 76, 203, - 128, 5, 68, 4, 226, 44, 72, 163, 189, 5, 73, 4, 154, 178, 7, 65, 247, 75, - 72, 8, 26, 65, 239, 171, 12, 72, 6, 226, 163, 7, 77, 186, 218, 5, 68, - 143, 45, 84, 8, 166, 90, 72, 206, 209, 10, 69, 247, 128, 1, 65, 6, 56, 4, - 66, 69, 82, 32, 245, 240, 12, 4, 69, 82, 73, 67, 4, 26, 70, 223, 144, 12, - 79, 2, 207, 190, 11, 73, 146, 1, 80, 7, 79, 82, 75, 72, 79, 78, 32, 197, - 3, 8, 89, 69, 78, 73, 83, 69, 73, 32, 84, 54, 65, 202, 1, 69, 122, 73, - 30, 79, 203, 192, 11, 66, 45, 106, 69, 130, 180, 9, 83, 226, 243, 3, 66, - 2, 68, 2, 71, 2, 76, 2, 78, 2, 81, 2, 82, 2, 84, 3, 89, 20, 222, 167, 13, - 66, 2, 68, 2, 71, 2, 75, 2, 76, 2, 78, 2, 82, 2, 83, 2, 84, 3, 89, 20, - 74, 78, 142, 138, 13, 76, 158, 27, 83, 146, 1, 67, 2, 77, 2, 80, 3, 90, - 8, 182, 166, 13, 67, 2, 71, 2, 84, 3, 89, 7, 138, 166, 13, 67, 3, 81, 13, - 130, 3, 69, 238, 162, 13, 80, 2, 81, 3, 84, 62, 38, 65, 170, 1, 69, 86, - 73, 23, 79, 39, 98, 69, 166, 163, 13, 83, 62, 78, 86, 66, 2, 68, 2, 71, - 2, 76, 2, 81, 2, 82, 2, 84, 3, 89, 17, 158, 172, 11, 78, 150, 248, 1, 66, - 2, 71, 2, 75, 2, 84, 3, 89, 15, 46, 78, 178, 162, 13, 83, 146, 1, 67, 3, - 90, 6, 190, 163, 13, 67, 2, 84, 3, 89, 5, 155, 163, 13, 81, 6, 26, 69, - 239, 162, 13, 81, 5, 235, 162, 13, 75, 52, 148, 1, 10, 67, 79, 77, 66, - 73, 78, 73, 78, 71, 32, 36, 7, 76, 69, 84, 84, 69, 82, 32, 233, 1, 12, - 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 8, 146, 221, 5, 84, 243, - 213, 3, 68, 36, 170, 199, 1, 65, 178, 195, 1, 82, 214, 212, 2, 76, 58, - 90, 34, 83, 66, 89, 252, 195, 1, 6, 70, 73, 78, 65, 76, 32, 0, 6, 71, 73, - 77, 69, 76, 45, 134, 3, 75, 130, 76, 66, 190, 173, 4, 78, 254, 1, 84, 2, - 87, 202, 103, 80, 171, 4, 77, 8, 52, 4, 84, 87, 79, 32, 166, 130, 11, 70, - 179, 122, 66, 4, 150, 138, 12, 66, 71, 68, 6, 170, 180, 1, 87, 152, 140, - 3, 3, 65, 68, 85, 235, 150, 7, 77, 22, 212, 1, 34, 32, 87, 73, 84, 72, - 32, 69, 88, 67, 76, 65, 77, 65, 84, 73, 79, 78, 32, 77, 65, 82, 75, 32, - 87, 73, 84, 72, 32, 76, 69, 70, 84, 32, 82, 44, 7, 67, 79, 77, 73, 78, - 71, 32, 210, 1, 69, 239, 202, 12, 73, 2, 21, 3, 73, 71, 72, 2, 151, 197, - 9, 84, 10, 144, 1, 6, 65, 85, 84, 79, 77, 79, 20, 7, 70, 73, 82, 69, 32, - 69, 78, 20, 6, 80, 79, 76, 73, 67, 69, 184, 212, 2, 2, 84, 65, 191, 195, - 8, 66, 2, 135, 253, 10, 66, 2, 211, 129, 12, 71, 2, 135, 206, 9, 32, 8, - 70, 32, 245, 230, 11, 11, 45, 80, 73, 69, 67, 69, 32, 83, 87, 73, 77, 6, - 40, 4, 68, 79, 84, 32, 147, 234, 9, 66, 4, 26, 79, 235, 235, 9, 76, 2, - 169, 221, 2, 11, 86, 69, 82, 32, 84, 87, 79, 32, 68, 79, 84, 46, 82, 69, - 168, 6, 2, 84, 73, 148, 143, 11, 5, 72, 73, 85, 67, 72, 167, 177, 1, 80, - 36, 70, 78, 181, 5, 12, 82, 65, 84, 73, 78, 71, 32, 83, 89, 83, 84, 69, - 34, 22, 32, 247, 3, 45, 28, 108, 2, 66, 79, 32, 7, 67, 69, 78, 84, 82, - 69, 32, 114, 70, 50, 72, 42, 77, 202, 233, 6, 83, 239, 194, 3, 76, 4, - 190, 147, 13, 79, 155, 3, 88, 8, 50, 84, 238, 130, 11, 66, 222, 2, 65, - 219, 82, 67, 2, 37, 7, 69, 65, 82, 68, 82, 79, 80, 2, 207, 132, 11, 45, - 4, 180, 255, 1, 5, 73, 76, 69, 32, 70, 15, 79, 2, 17, 2, 65, 78, 2, 187, - 251, 9, 68, 4, 57, 12, 65, 73, 76, 66, 79, 88, 32, 87, 73, 84, 72, 32, 4, - 44, 3, 76, 79, 87, 21, 4, 82, 65, 73, 83, 2, 17, 2, 69, 82, 2, 205, 173, - 11, 2, 69, 68, 6, 108, 15, 67, 73, 82, 67, 85, 73, 84, 45, 79, 85, 84, - 80, 85, 84, 32, 217, 242, 11, 6, 79, 85, 84, 76, 73, 78, 4, 18, 72, 3, - 76, 2, 229, 190, 1, 4, 45, 84, 89, 80, 2, 189, 132, 12, 6, 77, 32, 67, - 79, 77, 77, 6, 64, 2, 79, 78, 153, 177, 1, 8, 67, 65, 76, 32, 68, 73, 83, - 67, 2, 235, 250, 10, 32, 204, 1, 132, 1, 3, 65, 78, 71, 66, 73, 212, 8, - 5, 78, 65, 84, 69, 32, 192, 225, 3, 5, 84, 72, 79, 68, 79, 254, 186, 1, - 32, 227, 231, 5, 67, 6, 28, 2, 69, 32, 239, 21, 85, 4, 154, 191, 11, 66, - 211, 73, 72, 188, 1, 48, 5, 71, 73, 78, 65, 76, 21, 3, 89, 65, 32, 2, - 151, 133, 1, 32, 186, 1, 106, 65, 30, 70, 250, 1, 73, 32, 7, 76, 69, 84, - 84, 69, 82, 32, 142, 2, 83, 214, 1, 86, 243, 153, 11, 68, 4, 182, 142, 8, + 234, 230, 15, 80, 2, 90, 254, 68, 83, 14, 67, 3, 75, 7, 48, 6, 78, 85, + 83, 86, 65, 82, 215, 176, 16, 72, 2, 195, 160, 9, 65, 8, 26, 65, 239, + 173, 16, 68, 7, 246, 247, 8, 77, 233, 245, 6, 3, 71, 65, 76, 4, 186, 173, + 16, 89, 187, 2, 85, 5, 45, 9, 78, 86, 69, 82, 84, 69, 68, 32, 85, 2, 181, + 236, 15, 3, 66, 65, 68, 4, 202, 172, 16, 71, 3, 78, 8, 60, 6, 72, 82, 69, + 69, 32, 66, 234, 230, 15, 84, 195, 71, 65, 2, 17, 2, 65, 76, 2, 143, 137, + 16, 85, 6, 26, 65, 231, 173, 16, 73, 5, 29, 5, 32, 87, 73, 84, 72, 2, + 237, 220, 14, 2, 32, 84, 41, 29, 5, 78, 67, 72, 85, 32, 38, 104, 9, 65, + 76, 73, 32, 71, 65, 76, 73, 32, 222, 177, 14, 90, 138, 248, 1, 70, 2, 75, + 2, 82, 187, 2, 73, 28, 122, 68, 254, 192, 9, 67, 226, 222, 2, 84, 134, + 145, 2, 66, 2, 71, 2, 74, 2, 76, 130, 179, 1, 90, 254, 4, 78, 131, 64, + 83, 4, 222, 176, 14, 68, 139, 248, 1, 72, 48, 52, 4, 73, 66, 69, 32, 142, + 168, 16, 72, 187, 2, 65, 44, 242, 221, 5, 71, 2, 72, 158, 173, 6, 73, + 154, 19, 84, 222, 145, 2, 67, 2, 83, 246, 7, 82, 214, 161, 1, 65, 186, 9, + 90, 162, 7, 85, 234, 61, 68, 2, 70, 2, 74, 2, 75, 2, 80, 187, 2, 69, 60, + 52, 4, 79, 68, 79, 32, 154, 166, 16, 83, 187, 2, 65, 56, 250, 1, 65, 98, + 68, 34, 74, 34, 78, 236, 164, 1, 8, 76, 79, 78, 71, 32, 86, 79, 87, 250, + 179, 4, 71, 182, 192, 6, 84, 222, 145, 2, 67, 246, 7, 72, 174, 178, 1, + 79, 2, 85, 234, 61, 66, 2, 75, 2, 77, 2, 80, 2, 81, 2, 87, 2, 89, 186, 2, + 69, 3, 73, 6, 56, 8, 76, 73, 32, 71, 65, 76, 73, 32, 199, 165, 16, 78, 4, + 214, 171, 14, 90, 139, 248, 1, 84, 4, 186, 163, 16, 90, 187, 2, 65, 4, + 154, 163, 16, 73, 187, 2, 65, 2, 251, 162, 16, 73, 6, 198, 145, 16, 72, + 162, 17, 82, 187, 2, 65, 8, 128, 1, 4, 87, 73, 82, 76, 189, 190, 4, 21, + 73, 66, 69, 32, 83, 89, 76, 76, 65, 66, 76, 69, 32, 66, 79, 85, 78, 68, + 65, 82, 89, 6, 17, 2, 32, 66, 6, 25, 4, 73, 82, 71, 65, 7, 33, 6, 32, 87, + 73, 84, 72, 32, 4, 150, 2, 68, 187, 221, 15, 79, 6, 152, 1, 3, 82, 73, + 80, 56, 18, 85, 82, 78, 69, 68, 32, 83, 87, 73, 82, 76, 32, 66, 73, 82, + 71, 65, 32, 197, 192, 1, 8, 79, 68, 79, 32, 83, 79, 70, 84, 2, 225, 221, + 15, 9, 76, 69, 32, 66, 73, 82, 71, 65, 32, 2, 33, 6, 87, 73, 84, 72, 32, + 68, 2, 161, 221, 15, 5, 79, 85, 66, 76, 69, 2, 207, 229, 9, 76, 10, 96, + 9, 71, 82, 65, 77, 32, 70, 79, 82, 32, 208, 131, 4, 5, 83, 84, 65, 66, + 76, 183, 232, 9, 82, 6, 26, 89, 219, 182, 12, 69, 4, 206, 206, 15, 65, + 155, 1, 73, 10, 48, 2, 78, 32, 230, 209, 4, 68, 191, 182, 11, 83, 6, 88, + 12, 86, 73, 69, 87, 73, 78, 71, 32, 67, 69, 82, 69, 234, 181, 12, 67, 85, + 2, 76, 65, 2, 189, 254, 15, 2, 77, 79, 4, 190, 175, 12, 73, 191, 238, 3, + 69, 10, 26, 72, 69, 2, 79, 82, 2, 45, 9, 69, 82, 32, 67, 72, 82, 73, 83, + 84, 2, 255, 254, 2, 77, 8, 50, 32, 52, 4, 73, 90, 69, 68, 143, 144, 15, + 87, 4, 226, 196, 8, 66, 233, 159, 6, 4, 83, 67, 79, 79, 2, 189, 176, 3, + 7, 32, 87, 72, 69, 69, 76, 67, 18, 52, 2, 78, 84, 152, 1, 2, 83, 69, 131, + 153, 16, 84, 10, 48, 3, 65, 73, 78, 141, 132, 12, 3, 32, 70, 85, 9, 11, + 32, 6, 142, 202, 9, 82, 24, 5, 67, 65, 66, 76, 69, 217, 235, 1, 6, 66, + 73, 67, 89, 67, 76, 7, 11, 32, 4, 148, 137, 15, 2, 84, 82, 231, 83, 70, + 86, 52, 7, 76, 69, 84, 84, 69, 82, 32, 211, 234, 5, 68, 62, 198, 1, 75, + 62, 77, 34, 78, 34, 79, 30, 80, 34, 84, 234, 105, 68, 180, 128, 8, 2, 72, + 65, 2, 82, 134, 192, 1, 69, 134, 29, 83, 154, 89, 76, 246, 7, 67, 130, + 207, 4, 89, 190, 28, 66, 2, 87, 187, 2, 65, 6, 192, 173, 5, 2, 69, 65, + 170, 255, 5, 72, 243, 234, 4, 79, 4, 242, 132, 16, 65, 215, 1, 73, 4, + 158, 199, 15, 73, 139, 60, 71, 7, 222, 150, 16, 76, 3, 79, 4, 234, 130, + 16, 72, 219, 19, 65, 6, 178, 144, 9, 72, 218, 130, 7, 69, 155, 3, 65, + 206, 4, 44, 2, 76, 84, 234, 6, 83, 151, 252, 13, 67, 102, 36, 4, 65, 78, + 73, 32, 219, 2, 73, 76, 52, 7, 76, 69, 84, 84, 69, 82, 32, 147, 148, 4, + 83, 74, 206, 1, 68, 222, 83, 78, 166, 236, 1, 82, 214, 198, 9, 74, 178, + 51, 84, 206, 145, 3, 66, 2, 67, 2, 71, 2, 75, 2, 80, 138, 69, 72, 2, 76, + 2, 77, 2, 83, 2, 86, 2, 89, 186, 2, 65, 2, 69, 2, 73, 3, 85, 10, 38, 68, + 178, 144, 16, 72, 187, 2, 65, 6, 174, 144, 16, 68, 2, 72, 187, 2, 65, 26, + 56, 2, 80, 76, 236, 2, 3, 83, 69, 84, 175, 254, 14, 77, 18, 50, 69, 89, + 8, 73, 67, 65, 84, 73, 79, 78, 32, 2, 41, 8, 32, 77, 85, 83, 73, 67, 65, + 76, 2, 21, 3, 32, 78, 79, 2, 147, 253, 14, 84, 16, 40, 4, 83, 73, 71, 78, + 207, 144, 16, 88, 15, 11, 32, 12, 48, 3, 73, 78, 32, 81, 5, 87, 73, 84, + 72, 32, 8, 226, 206, 3, 76, 22, 82, 192, 175, 9, 5, 68, 79, 85, 66, 76, + 247, 236, 1, 84, 4, 174, 164, 12, 85, 195, 234, 2, 68, 7, 11, 32, 4, 232, + 83, 5, 77, 85, 76, 84, 73, 239, 139, 12, 85, 228, 3, 44, 5, 72, 82, 79, + 79, 77, 21, 2, 73, 67, 5, 175, 226, 14, 32, 224, 3, 30, 32, 101, 3, 65, + 76, 32, 6, 76, 4, 78, 65, 84, 85, 200, 43, 2, 70, 76, 161, 137, 11, 4, + 83, 72, 65, 82, 2, 235, 201, 10, 82, 218, 3, 64, 8, 75, 69, 89, 66, 79, + 65, 82, 68, 66, 83, 135, 247, 3, 78, 5, 41, 8, 32, 87, 73, 84, 72, 32, + 74, 65, 2, 143, 239, 13, 67, 212, 3, 48, 6, 89, 77, 66, 79, 76, 32, 243, + 166, 11, 67, 210, 3, 230, 2, 66, 238, 1, 67, 226, 9, 68, 242, 2, 69, 162, + 1, 70, 166, 2, 71, 112, 9, 65, 82, 80, 69, 71, 71, 73, 65, 84, 168, 1, 2, + 72, 65, 126, 75, 198, 3, 76, 138, 1, 77, 190, 2, 78, 162, 1, 79, 218, 2, + 80, 200, 2, 2, 81, 85, 146, 2, 82, 198, 2, 83, 230, 5, 84, 182, 10, 86, + 42, 88, 42, 87, 200, 140, 9, 9, 73, 78, 86, 69, 82, 84, 69, 68, 32, 255, + 195, 6, 90, 20, 36, 5, 69, 71, 73, 78, 32, 59, 82, 8, 130, 15, 80, 30, + 83, 218, 246, 7, 66, 143, 231, 7, 84, 12, 24, 2, 65, 67, 35, 69, 4, 198, + 210, 15, 75, 155, 53, 69, 8, 26, 86, 203, 197, 15, 65, 6, 32, 2, 73, 83, + 131, 135, 16, 69, 5, 171, 26, 32, 84, 120, 2, 65, 69, 30, 76, 94, 79, + 194, 7, 82, 234, 11, 32, 232, 32, 7, 73, 82, 67, 76, 69, 32, 88, 189, + 222, 5, 2, 85, 84, 2, 177, 193, 15, 2, 83, 85, 8, 42, 73, 245, 23, 5, 85, + 83, 84, 69, 82, 4, 26, 77, 223, 225, 13, 86, 2, 247, 214, 9, 65, 64, 26, + 77, 171, 130, 16, 68, 62, 64, 7, 66, 73, 78, 73, 78, 71, 32, 237, 145, 6, + 3, 77, 79, 78, 60, 202, 1, 65, 80, 7, 77, 65, 82, 67, 65, 84, 79, 56, 2, + 68, 79, 56, 2, 85, 80, 28, 2, 70, 76, 40, 5, 72, 65, 82, 77, 79, 22, 83, + 142, 2, 84, 226, 176, 7, 66, 224, 193, 3, 2, 76, 79, 195, 138, 5, 82, 6, + 76, 5, 67, 67, 69, 78, 84, 37, 10, 85, 71, 77, 69, 78, 84, 65, 84, 73, + 79, 5, 205, 2, 5, 45, 83, 84, 65, 67, 2, 199, 240, 14, 78, 6, 52, 2, 87, + 78, 168, 3, 2, 85, 66, 191, 225, 15, 73, 2, 233, 230, 14, 2, 32, 66, 12, + 248, 76, 2, 65, 71, 239, 179, 15, 73, 2, 195, 216, 14, 78, 12, 132, 1, 9, + 78, 65, 80, 32, 80, 73, 90, 90, 73, 22, 84, 224, 205, 4, 11, 80, 82, 69, + 67, 72, 71, 69, 83, 65, 78, 71, 219, 129, 6, 77, 2, 139, 244, 11, 67, 6, + 44, 5, 65, 67, 67, 65, 84, 151, 238, 15, 69, 4, 40, 4, 73, 83, 83, 73, + 195, 254, 15, 79, 2, 203, 223, 15, 77, 10, 34, 82, 193, 143, 12, 2, 69, + 78, 8, 28, 2, 73, 80, 191, 6, 69, 2, 209, 242, 11, 6, 76, 69, 32, 84, 79, + 78, 4, 22, 79, 191, 2, 69, 2, 255, 240, 14, 73, 24, 98, 65, 142, 1, 69, + 84, 6, 79, 85, 66, 76, 69, 32, 225, 158, 10, 7, 82, 85, 77, 32, 67, 76, + 69, 10, 96, 2, 32, 67, 20, 2, 77, 80, 204, 29, 4, 83, 72, 69, 68, 217, + 180, 15, 5, 76, 32, 83, 69, 71, 2, 223, 189, 8, 65, 5, 231, 243, 13, 32, + 4, 52, 3, 67, 82, 69, 145, 228, 8, 4, 71, 82, 69, 69, 2, 165, 25, 3, 83, + 67, 69, 6, 246, 21, 83, 130, 7, 66, 243, 134, 5, 70, 14, 32, 3, 78, 68, + 32, 155, 16, 73, 10, 74, 80, 30, 83, 208, 13, 3, 79, 70, 32, 138, 233, 7, + 66, 143, 231, 7, 84, 2, 229, 211, 12, 2, 72, 82, 2, 191, 128, 15, 76, 34, + 104, 6, 69, 82, 77, 65, 84, 65, 22, 73, 122, 79, 102, 32, 152, 36, 3, 85, + 83, 65, 205, 229, 3, 2, 76, 65, 5, 219, 202, 13, 32, 10, 22, 78, 143, 34, + 86, 8, 56, 9, 71, 69, 82, 69, 68, 32, 84, 82, 69, 243, 20, 65, 6, 237, + 237, 5, 4, 77, 79, 76, 79, 6, 216, 25, 3, 85, 82, 45, 183, 150, 14, 82, + 18, 54, 32, 56, 7, 76, 73, 83, 83, 65, 78, 68, 23, 82, 6, 25, 4, 67, 76, + 69, 70, 7, 161, 13, 3, 32, 79, 84, 4, 215, 129, 6, 79, 8, 84, 9, 65, 67, + 69, 32, 78, 79, 84, 69, 32, 37, 8, 69, 71, 79, 82, 73, 65, 78, 32, 4, + 164, 222, 8, 2, 78, 79, 27, 83, 4, 250, 2, 67, 3, 70, 8, 44, 3, 76, 70, + 32, 209, 8, 3, 85, 80, 84, 6, 52, 3, 80, 69, 68, 226, 222, 3, 78, 207, + 183, 3, 82, 2, 139, 198, 5, 65, 24, 48, 6, 73, 69, 86, 65, 78, 32, 247, + 205, 14, 79, 22, 178, 1, 67, 46, 69, 68, 7, 81, 85, 65, 82, 84, 69, 82, + 62, 70, 164, 219, 3, 4, 72, 65, 76, 70, 0, 5, 87, 72, 79, 76, 69, 157, + 224, 1, 9, 82, 69, 67, 73, 84, 65, 84, 73, 86, 2, 11, 32, 2, 11, 67, 2, + 163, 163, 5, 76, 6, 64, 5, 73, 71, 72, 84, 72, 201, 186, 14, 5, 78, 68, + 32, 79, 70, 4, 217, 252, 5, 10, 32, 78, 79, 84, 69, 32, 83, 84, 69, 77, + 4, 222, 14, 76, 197, 204, 3, 4, 73, 78, 65, 76, 8, 44, 4, 79, 78, 71, 65, + 221, 13, 2, 69, 70, 7, 11, 32, 4, 28, 3, 73, 77, 80, 3, 80, 2, 197, 2, 7, + 69, 82, 70, 69, 67, 84, 65, 18, 104, 2, 65, 88, 20, 2, 69, 90, 20, 5, 73, + 78, 73, 77, 65, 48, 3, 79, 79, 78, 25, 4, 85, 76, 84, 73, 2, 255, 204, + 15, 73, 2, 135, 207, 15, 90, 7, 11, 32, 4, 170, 144, 7, 82, 167, 194, 5, + 66, 4, 193, 17, 2, 32, 78, 4, 60, 11, 80, 76, 69, 32, 77, 69, 65, 83, 85, + 82, 69, 15, 32, 2, 11, 32, 2, 163, 143, 7, 82, 10, 120, 4, 69, 66, 69, + 78, 216, 26, 3, 85, 76, 76, 192, 220, 5, 3, 65, 84, 85, 141, 217, 6, 7, + 79, 84, 69, 72, 69, 65, 68, 2, 213, 211, 14, 4, 83, 84, 73, 77, 32, 88, + 2, 78, 69, 120, 14, 82, 78, 65, 77, 69, 78, 84, 32, 83, 84, 82, 79, 75, + 69, 107, 84, 6, 92, 18, 32, 72, 85, 78, 68, 82, 69, 68, 32, 84, 87, 69, + 78, 84, 89, 45, 69, 73, 163, 20, 45, 4, 185, 13, 2, 71, 72, 22, 11, 45, + 22, 158, 166, 3, 49, 162, 195, 12, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, + 2, 56, 3, 57, 4, 173, 4, 3, 84, 65, 86, 18, 96, 9, 65, 82, 69, 78, 84, + 72, 69, 83, 73, 0, 2, 76, 85, 18, 69, 142, 1, 79, 223, 222, 13, 73, 2, + 247, 22, 83, 6, 68, 4, 68, 65, 76, 32, 49, 9, 83, 32, 83, 85, 66, 80, 85, + 78, 67, 4, 26, 85, 199, 165, 15, 77, 2, 171, 165, 15, 80, 2, 167, 195, + 13, 84, 6, 48, 2, 68, 65, 245, 5, 5, 82, 82, 69, 67, 84, 2, 151, 224, 13, + 84, 12, 76, 6, 65, 82, 84, 69, 82, 32, 125, 9, 73, 78, 68, 73, 67, 69, + 83, 73, 77, 8, 60, 5, 84, 79, 78, 69, 32, 246, 207, 3, 78, 207, 183, 3, + 82, 4, 26, 83, 243, 141, 5, 70, 2, 197, 228, 15, 3, 72, 65, 82, 4, 17, 2, + 65, 32, 4, 142, 171, 12, 65, 205, 182, 3, 3, 66, 65, 83, 14, 22, 69, 175, + 1, 73, 10, 72, 4, 80, 69, 65, 84, 81, 10, 86, 69, 82, 83, 69, 32, 70, 73, + 78, 65, 8, 56, 8, 69, 68, 32, 70, 73, 71, 85, 82, 163, 146, 14, 32, 6, + 147, 217, 5, 69, 2, 211, 4, 76, 4, 48, 2, 71, 72, 57, 6, 78, 70, 79, 82, + 90, 65, 2, 33, 6, 84, 32, 82, 69, 80, 69, 2, 255, 158, 10, 65, 2, 147, + 176, 8, 78, 48, 136, 1, 6, 67, 65, 78, 68, 73, 67, 62, 69, 166, 1, 72, + 54, 73, 252, 1, 6, 81, 85, 65, 82, 69, 32, 212, 153, 8, 2, 85, 66, 243, + 100, 79, 4, 17, 2, 85, 83, 5, 141, 218, 13, 5, 32, 70, 76, 69, 88, 14, + 32, 2, 77, 73, 179, 182, 15, 71, 12, 64, 6, 66, 82, 69, 86, 73, 83, 1, 6, + 77, 73, 78, 73, 77, 65, 6, 11, 32, 6, 138, 13, 87, 182, 244, 6, 82, 167, + 194, 5, 66, 6, 84, 3, 79, 82, 84, 213, 233, 5, 3, 65, 82, 80, 14, 32, 4, + 78, 71, 76, 69, 43, 88, 2, 17, 2, 32, 66, 2, 135, 246, 13, 65, 12, 18, + 45, 79, 84, 4, 242, 7, 76, 173, 190, 14, 11, 83, 84, 82, 73, 78, 71, 32, + 70, 82, 69, 84, 8, 52, 3, 69, 69, 78, 1, 6, 89, 45, 70, 79, 85, 82, 4, + 193, 12, 2, 84, 72, 6, 26, 78, 247, 219, 15, 66, 4, 229, 9, 7, 79, 84, + 69, 72, 69, 65, 68, 60, 132, 1, 6, 69, 77, 80, 85, 83, 32, 154, 4, 72, + 88, 2, 87, 79, 84, 7, 79, 82, 67, 85, 76, 85, 83, 46, 82, 141, 3, 3, 85, + 82, 78, 16, 232, 1, 27, 73, 77, 80, 69, 82, 70, 69, 67, 84, 85, 77, 32, + 67, 85, 77, 32, 80, 82, 79, 76, 65, 84, 73, 79, 78, 69, 32, 129, 1, 25, + 80, 69, 82, 70, 69, 67, 84, 85, 77, 32, 67, 85, 77, 32, 80, 82, 79, 76, + 65, 84, 73, 79, 78, 69, 32, 10, 60, 10, 73, 77, 80, 69, 82, 70, 69, 67, + 84, 65, 131, 1, 80, 9, 213, 206, 5, 11, 32, 68, 73, 77, 73, 78, 85, 84, + 73, 79, 78, 6, 60, 3, 73, 77, 80, 41, 8, 80, 69, 82, 70, 69, 67, 84, 65, + 2, 197, 178, 15, 5, 69, 82, 70, 69, 67, 5, 169, 189, 8, 12, 32, 68, 73, + 77, 73, 78, 85, 84, 73, 79, 78, 45, 6, 72, 2, 82, 69, 249, 5, 11, 73, 82, + 84, 89, 45, 83, 69, 67, 79, 78, 68, 2, 11, 69, 2, 11, 45, 2, 11, 76, 2, + 37, 7, 73, 78, 69, 32, 83, 84, 65, 2, 235, 211, 14, 70, 5, 225, 206, 11, + 6, 32, 82, 69, 83, 85, 80, 27, 33, 6, 73, 65, 78, 71, 76, 69, 24, 128, 1, + 10, 32, 78, 79, 84, 69, 72, 69, 65, 68, 32, 61, 17, 45, 82, 79, 85, 78, + 68, 32, 78, 79, 84, 69, 72, 69, 65, 68, 32, 68, 20, 58, 68, 24, 3, 85, + 80, 32, 38, 82, 25, 3, 76, 69, 70, 4, 93, 3, 79, 87, 78, 8, 34, 82, 78, + 87, 219, 182, 12, 66, 4, 21, 3, 73, 71, 72, 4, 11, 84, 4, 11, 32, 4, 26, + 87, 219, 182, 12, 66, 2, 11, 72, 2, 223, 138, 14, 73, 7, 11, 32, 4, 246, + 186, 8, 83, 183, 150, 7, 85, 4, 36, 3, 79, 73, 68, 159, 142, 15, 73, 2, + 237, 251, 12, 5, 32, 78, 79, 84, 69, 6, 92, 4, 72, 79, 76, 69, 221, 199, + 8, 13, 73, 84, 72, 32, 70, 73, 78, 71, 69, 82, 78, 65, 73, 4, 11, 32, 4, + 218, 186, 3, 78, 207, 183, 3, 82, 232, 3, 224, 2, 15, 67, 79, 78, 83, 79, + 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 254, 1, 76, 212, 14, 16, 77, 79, + 68, 73, 70, 73, 69, 82, 32, 76, 69, 84, 84, 69, 82, 32, 122, 83, 80, 16, + 69, 65, 83, 84, 69, 82, 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 222, 9, + 84, 204, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 160, 175, 9, + 3, 80, 65, 79, 207, 142, 4, 68, 16, 66, 77, 165, 1, 11, 83, 72, 65, 78, + 32, 77, 69, 68, 73, 65, 76, 14, 80, 6, 69, 68, 73, 65, 76, 32, 45, 10, + 79, 78, 32, 77, 69, 68, 73, 65, 76, 32, 8, 250, 200, 15, 72, 2, 82, 2, + 87, 3, 89, 6, 206, 200, 15, 76, 2, 77, 3, 78, 2, 219, 217, 10, 32, 238, + 1, 104, 6, 69, 84, 84, 69, 82, 32, 185, 13, 15, 79, 71, 79, 71, 82, 65, + 77, 32, 75, 72, 65, 77, 84, 73, 32, 232, 1, 238, 1, 65, 50, 69, 146, 1, + 71, 50, 75, 254, 1, 77, 134, 1, 78, 58, 82, 86, 83, 130, 3, 84, 198, 1, + 87, 178, 227, 11, 68, 214, 6, 85, 22, 86, 186, 201, 1, 73, 42, 76, 246, + 193, 1, 66, 2, 67, 2, 74, 2, 80, 138, 69, 72, 2, 89, 187, 2, 79, 7, 144, + 253, 14, 4, 73, 84, 79, 78, 219, 74, 85, 9, 77, 17, 65, 83, 84, 69, 82, + 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 78, 32, 6, 42, 71, 186, 213, 10, + 89, 135, 181, 3, 78, 2, 183, 213, 10, 72, 6, 154, 200, 9, 82, 218, 251, + 5, 72, 187, 2, 65, 44, 32, 2, 72, 65, 219, 197, 15, 65, 43, 25, 4, 77, + 84, 73, 32, 40, 134, 1, 68, 34, 84, 242, 217, 8, 78, 230, 162, 6, 67, 2, + 72, 2, 74, 170, 37, 76, 226, 31, 70, 2, 71, 2, 82, 2, 83, 2, 88, 3, 90, + 6, 242, 252, 14, 68, 139, 69, 72, 4, 211, 252, 14, 84, 12, 36, 3, 79, 78, + 32, 219, 195, 15, 65, 10, 60, 2, 66, 66, 218, 200, 13, 74, 254, 183, 1, + 78, 199, 66, 69, 4, 150, 195, 15, 65, 3, 69, 10, 214, 216, 8, 78, 238, + 231, 6, 71, 2, 89, 187, 2, 65, 4, 252, 220, 2, 12, 85, 77, 65, 73, 32, + 80, 65, 76, 65, 85, 78, 71, 199, 229, 12, 65, 50, 90, 72, 192, 221, 2, 9, + 71, 65, 87, 32, 75, 65, 82, 69, 78, 158, 225, 12, 83, 187, 2, 65, 44, 66, + 65, 197, 1, 11, 87, 69, 32, 80, 65, 76, 65, 85, 78, 71, 32, 41, 17, 2, + 78, 32, 38, 134, 1, 78, 246, 196, 13, 74, 2, 80, 2, 84, 130, 179, 1, 66, + 2, 67, 2, 71, 2, 75, 138, 69, 68, 2, 70, 2, 72, 2, 90, 187, 2, 65, 6, + 250, 188, 15, 78, 2, 89, 187, 2, 65, 4, 202, 196, 13, 67, 3, 83, 36, 38, + 65, 138, 247, 14, 84, 139, 69, 72, 31, 41, 8, 73, 32, 76, 65, 73, 78, 71, + 32, 28, 82, 78, 206, 227, 11, 68, 190, 146, 3, 66, 2, 71, 2, 74, 170, 37, + 76, 227, 31, 70, 4, 142, 187, 15, 78, 3, 89, 6, 92, 17, 69, 83, 84, 69, + 82, 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 78, 32, 207, 188, 15, 65, 4, + 194, 203, 10, 80, 203, 246, 2, 84, 6, 162, 176, 14, 79, 166, 60, 81, 139, + 63, 72, 4, 56, 6, 75, 72, 65, 77, 84, 73, 1, 4, 83, 72, 65, 78, 2, 29, 5, + 32, 82, 69, 68, 85, 2, 193, 132, 1, 2, 80, 76, 90, 76, 2, 72, 65, 20, 4, + 73, 71, 78, 32, 233, 6, 6, 89, 77, 66, 79, 76, 32, 20, 199, 186, 9, 78, + 52, 202, 3, 65, 32, 12, 75, 72, 65, 77, 84, 73, 32, 84, 79, 78, 69, 45, + 30, 83, 132, 2, 15, 84, 65, 73, 32, 76, 65, 73, 78, 71, 32, 84, 79, 78, + 69, 45, 28, 22, 87, 69, 83, 84, 69, 82, 78, 32, 80, 87, 79, 32, 75, 65, + 82, 69, 78, 32, 84, 79, 78, 69, 206, 244, 2, 68, 252, 163, 5, 19, 82, 85, + 77, 65, 73, 32, 80, 65, 76, 65, 85, 78, 71, 32, 84, 79, 78, 69, 45, 204, + 170, 3, 9, 80, 65, 79, 32, 75, 65, 82, 69, 78, 148, 114, 6, 76, 73, 84, + 84, 76, 69, 199, 187, 2, 86, 4, 234, 162, 14, 83, 255, 78, 78, 4, 178, + 182, 15, 49, 3, 51, 18, 40, 4, 72, 65, 78, 32, 147, 229, 14, 69, 16, 84, + 8, 67, 79, 85, 78, 67, 73, 76, 32, 84, 5, 84, 79, 78, 69, 45, 151, 182, + 14, 83, 6, 172, 196, 11, 8, 69, 77, 80, 72, 65, 84, 73, 67, 229, 239, 3, + 4, 84, 79, 78, 69, 8, 190, 180, 15, 50, 2, 51, 2, 53, 3, 54, 4, 146, 180, + 15, 50, 3, 53, 10, 11, 45, 10, 234, 179, 15, 49, 2, 50, 2, 51, 2, 52, 3, + 53, 18, 130, 1, 65, 132, 1, 2, 76, 79, 28, 5, 83, 72, 65, 78, 32, 192, + 173, 8, 4, 71, 69, 78, 73, 169, 97, 6, 67, 79, 77, 80, 76, 69, 8, 84, 5, + 73, 84, 79, 78, 32, 141, 174, 6, 10, 70, 79, 82, 69, 77, 69, 78, 84, 73, + 79, 6, 98, 69, 186, 249, 8, 84, 151, 158, 5, 79, 2, 217, 173, 8, 2, 67, + 65, 4, 26, 69, 207, 151, 14, 79, 2, 201, 12, 4, 88, 67, 76, 65, 24, 140, + 1, 20, 79, 78, 69, 32, 77, 65, 82, 75, 32, 83, 71, 65, 87, 32, 75, 65, + 82, 69, 78, 32, 221, 175, 9, 8, 65, 73, 32, 76, 65, 73, 78, 71, 4, 38, + 72, 129, 206, 8, 3, 75, 69, 32, 2, 139, 208, 8, 65, 56, 150, 2, 65, 76, + 9, 71, 69, 66, 65, 32, 75, 65, 82, 69, 20, 6, 75, 65, 89, 65, 72, 32, 40, + 4, 77, 79, 78, 32, 34, 83, 142, 1, 69, 40, 18, 87, 69, 83, 84, 69, 82, + 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 78, 32, 228, 182, 9, 2, 84, 65, + 142, 160, 2, 85, 22, 86, 187, 201, 1, 73, 8, 26, 73, 219, 172, 15, 65, 7, + 25, 4, 84, 79, 78, 32, 4, 223, 162, 13, 65, 2, 151, 236, 14, 78, 6, 190, + 149, 15, 69, 2, 79, 215, 22, 85, 4, 146, 152, 15, 73, 219, 19, 79, 10, + 80, 4, 72, 65, 78, 32, 157, 204, 8, 10, 71, 65, 87, 32, 75, 65, 82, 69, + 78, 32, 8, 54, 69, 20, 5, 70, 73, 78, 65, 76, 247, 167, 15, 65, 5, 199, + 173, 14, 32, 2, 227, 152, 15, 32, 4, 174, 133, 15, 69, 151, 14, 85, 230, + 17, 152, 2, 5, 45, 65, 82, 89, 32, 214, 4, 65, 222, 16, 66, 30, 69, 146, + 32, 73, 136, 1, 3, 75, 79, 32, 170, 10, 79, 162, 24, 85, 224, 8, 22, 89, + 73, 65, 75, 69, 78, 71, 32, 80, 85, 65, 67, 72, 85, 69, 32, 72, 77, 79, + 78, 71, 32, 216, 135, 2, 2, 80, 78, 220, 189, 12, 2, 78, 66, 27, 76, 32, + 130, 1, 67, 114, 84, 46, 83, 160, 1, 5, 85, 78, 73, 79, 78, 108, 4, 87, + 72, 73, 84, 246, 225, 11, 76, 210, 17, 73, 223, 160, 2, 80, 8, 52, 7, 73, + 82, 67, 76, 69, 68, 32, 135, 151, 14, 79, 6, 40, 2, 80, 76, 14, 84, 219, + 141, 8, 68, 2, 35, 85, 2, 21, 3, 73, 77, 69, 2, 219, 229, 11, 83, 6, 40, + 6, 81, 85, 65, 82, 69, 32, 87, 85, 4, 60, 9, 73, 78, 84, 69, 82, 83, 69, + 67, 84, 1, 2, 85, 78, 2, 215, 191, 11, 73, 2, 11, 77, 2, 179, 226, 11, + 77, 7, 69, 15, 32, 79, 80, 69, 82, 65, 84, 79, 82, 32, 87, 73, 84, 72, + 32, 4, 150, 157, 11, 80, 131, 182, 3, 68, 2, 11, 69, 2, 233, 204, 12, 2, + 32, 86, 188, 2, 170, 2, 66, 252, 4, 10, 71, 32, 77, 85, 78, 68, 65, 82, + 73, 32, 142, 4, 73, 52, 2, 78, 68, 208, 4, 7, 84, 73, 79, 78, 65, 76, 32, + 228, 213, 1, 2, 85, 83, 140, 137, 8, 5, 77, 69, 32, 66, 65, 184, 212, 3, + 7, 90, 65, 82, 32, 65, 77, 85, 188, 160, 1, 8, 82, 82, 79, 87, 32, 78, + 79, 45, 231, 62, 75, 82, 52, 7, 65, 84, 65, 69, 65, 78, 32, 231, 157, 15, + 76, 80, 160, 1, 7, 76, 69, 84, 84, 69, 82, 32, 236, 2, 7, 78, 85, 77, 66, + 69, 82, 32, 133, 243, 9, 16, 67, 82, 85, 67, 73, 70, 79, 82, 77, 32, 78, + 85, 77, 66, 69, 82, 62, 236, 1, 6, 70, 73, 78, 65, 76, 32, 154, 132, 2, + 84, 246, 118, 68, 34, 76, 50, 81, 238, 205, 1, 82, 178, 215, 2, 65, 50, + 71, 90, 90, 34, 83, 66, 89, 158, 208, 1, 72, 234, 5, 75, 198, 75, 66, + 178, 216, 4, 78, 134, 2, 87, 218, 103, 80, 171, 4, 77, 18, 246, 250, 2, + 65, 54, 76, 250, 163, 4, 83, 190, 3, 89, 134, 214, 1, 75, 198, 75, 66, + 178, 216, 4, 78, 222, 105, 72, 171, 4, 77, 16, 134, 252, 2, 84, 134, 166, + 4, 79, 223, 137, 6, 70, 84, 84, 7, 76, 69, 84, 84, 69, 82, 32, 164, 2, 5, + 83, 73, 71, 78, 32, 191, 167, 13, 68, 54, 42, 65, 50, 69, 66, 73, 50, 79, + 47, 85, 11, 138, 137, 15, 78, 202, 17, 66, 2, 72, 3, 74, 15, 158, 159, + 13, 78, 210, 113, 76, 138, 109, 84, 174, 28, 71, 3, 77, 11, 194, 211, 14, + 68, 162, 70, 72, 2, 83, 3, 84, 11, 222, 152, 15, 78, 86, 76, 2, 80, 3, + 89, 11, 134, 153, 15, 67, 2, 68, 2, 75, 3, 82, 10, 100, 2, 77, 85, 20, 3, + 83, 85, 84, 182, 118, 73, 212, 234, 10, 2, 79, 74, 241, 253, 2, 3, 84, + 79, 89, 2, 139, 223, 14, 72, 2, 207, 150, 15, 85, 4, 208, 182, 11, 5, 76, + 32, 80, 79, 76, 187, 15, 82, 133, 1, 41, 8, 73, 78, 65, 71, 65, 82, 73, + 32, 130, 1, 140, 1, 7, 76, 69, 84, 84, 69, 82, 32, 248, 1, 5, 83, 73, 71, + 78, 32, 52, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 239, 229, 4, + 72, 94, 210, 1, 86, 226, 185, 11, 65, 38, 68, 82, 82, 34, 84, 230, 5, 85, + 206, 201, 1, 73, 42, 76, 250, 192, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, + 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 77, 2, 89, 186, 2, 69, 3, 79, 6, + 202, 148, 5, 79, 183, 255, 9, 65, 10, 254, 240, 9, 83, 238, 203, 1, 65, + 231, 147, 3, 86, 24, 250, 235, 4, 80, 238, 39, 86, 246, 171, 6, 65, 38, + 85, 206, 201, 1, 73, 222, 137, 2, 69, 3, 79, 4, 158, 48, 68, 255, 160, + 14, 80, 4, 154, 146, 15, 83, 15, 72, 254, 4, 216, 1, 3, 71, 65, 84, 192, + 7, 6, 73, 84, 72, 69, 82, 32, 194, 3, 83, 136, 1, 2, 85, 84, 242, 1, 87, + 148, 16, 3, 88, 84, 32, 144, 186, 8, 4, 80, 84, 85, 78, 216, 133, 3, 2, + 67, 75, 238, 161, 2, 82, 207, 144, 1, 76, 162, 1, 152, 1, 4, 73, 86, 69, + 32, 161, 233, 12, 27, 69, 68, 32, 68, 79, 85, 66, 76, 69, 32, 86, 69, 82, + 84, 73, 67, 65, 76, 32, 66, 65, 82, 32, 68, 79, 85, 66, 160, 1, 152, 1, + 8, 67, 73, 82, 67, 76, 69, 68, 32, 224, 1, 9, 68, 73, 65, 71, 79, 78, 65, + 76, 32, 184, 1, 8, 83, 81, 85, 65, 82, 69, 68, 32, 231, 206, 8, 65, 78, + 112, 5, 68, 73, 71, 73, 84, 28, 7, 78, 85, 77, 66, 69, 82, 32, 180, 3, 2, + 76, 65, 202, 228, 13, 84, 135, 4, 83, 2, 209, 157, 13, 2, 32, 90, 20, 50, + 84, 214, 133, 2, 69, 46, 70, 42, 78, 31, 83, 6, 150, 135, 2, 72, 47, 87, + 6, 38, 77, 206, 202, 13, 67, 183, 4, 68, 2, 49, 10, 73, 68, 68, 76, 69, + 32, 82, 73, 71, 72, 2, 17, 2, 84, 32, 2, 41, 8, 84, 79, 32, 76, 79, 87, + 69, 82, 2, 153, 137, 12, 2, 32, 67, 74, 142, 1, 76, 70, 85, 230, 154, 12, + 68, 138, 29, 81, 216, 129, 1, 2, 67, 82, 218, 19, 65, 234, 19, 73, 2, 87, + 134, 8, 82, 194, 157, 1, 80, 3, 83, 54, 26, 65, 183, 237, 10, 69, 52, + 129, 246, 8, 5, 84, 73, 78, 32, 67, 2, 171, 233, 13, 80, 18, 158, 1, 65, + 216, 1, 17, 71, 82, 69, 65, 84, 69, 82, 45, 84, 72, 65, 78, 32, 78, 79, + 82, 32, 37, 14, 76, 69, 83, 83, 45, 84, 72, 65, 78, 32, 78, 79, 82, 32, + 6, 92, 3, 32, 83, 85, 85, 16, 80, 80, 82, 79, 88, 73, 77, 65, 84, 69, 76, + 89, 32, 78, 79, 82, 4, 30, 66, 1, 3, 80, 69, 82, 2, 217, 245, 6, 8, 83, + 69, 84, 32, 79, 70, 32, 78, 2, 233, 48, 5, 32, 65, 67, 84, 85, 6, 254, + 144, 8, 69, 131, 227, 4, 76, 6, 218, 144, 8, 69, 223, 226, 4, 71, 6, 26, + 84, 151, 255, 12, 83, 4, 76, 5, 73, 78, 71, 32, 68, 185, 210, 10, 8, 32, + 87, 73, 84, 72, 32, 69, 71, 2, 181, 252, 7, 2, 79, 76, 64, 40, 4, 82, 65, + 76, 32, 159, 203, 14, 69, 62, 48, 6, 67, 72, 69, 83, 83, 32, 191, 198, + 14, 70, 60, 74, 75, 234, 165, 13, 66, 38, 69, 130, 5, 80, 22, 81, 38, 82, + 131, 2, 84, 20, 44, 5, 78, 73, 71, 72, 84, 251, 166, 13, 73, 15, 167, + 167, 13, 32, 246, 2, 70, 32, 184, 8, 2, 65, 32, 252, 6, 3, 76, 73, 78, + 195, 223, 3, 83, 174, 1, 90, 77, 64, 4, 83, 72, 69, 81, 20, 8, 84, 65, + 73, 32, 76, 85, 69, 32, 255, 230, 13, 76, 4, 25, 4, 79, 79, 78, 32, 4, + 138, 143, 8, 87, 211, 222, 5, 83, 2, 163, 189, 9, 69, 166, 1, 160, 1, 7, + 76, 69, 84, 84, 69, 82, 32, 244, 2, 8, 83, 73, 71, 78, 32, 76, 65, 69, + 22, 84, 76, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 155, 138, 13, + 68, 102, 76, 6, 70, 73, 78, 65, 76, 32, 68, 4, 72, 73, 71, 72, 1, 3, 76, + 79, 87, 14, 162, 131, 13, 78, 154, 251, 1, 66, 2, 68, 2, 75, 2, 77, 3, + 86, 44, 11, 32, 44, 146, 1, 75, 2, 88, 34, 83, 138, 139, 11, 78, 162, + 169, 3, 84, 82, 80, 138, 69, 66, 2, 68, 2, 70, 2, 72, 2, 76, 2, 77, 2, + 81, 2, 86, 3, 89, 4, 158, 250, 14, 86, 187, 2, 65, 4, 254, 249, 14, 85, + 187, 2, 65, 5, 151, 252, 14, 86, 6, 252, 154, 7, 3, 79, 78, 69, 233, 80, + 8, 72, 65, 77, 32, 68, 73, 71, 73, 34, 110, 65, 46, 73, 30, 79, 38, 85, + 216, 175, 12, 11, 86, 79, 87, 69, 76, 32, 83, 72, 79, 82, 84, 135, 202, + 2, 69, 8, 146, 219, 10, 65, 182, 159, 4, 69, 3, 89, 4, 154, 250, 14, 73, + 3, 89, 9, 202, 218, 10, 65, 183, 159, 4, 89, 11, 166, 218, 10, 69, 182, + 159, 4, 85, 3, 89, 194, 1, 182, 1, 68, 106, 71, 72, 7, 76, 69, 84, 84, + 69, 82, 32, 214, 2, 83, 194, 23, 80, 162, 139, 1, 86, 182, 230, 7, 65, + 144, 234, 1, 5, 73, 78, 83, 69, 82, 202, 174, 1, 67, 175, 193, 2, 79, 26, + 64, 6, 79, 85, 66, 76, 69, 32, 142, 155, 11, 65, 147, 235, 1, 73, 4, 254, + 154, 11, 68, 175, 138, 1, 67, 2, 11, 65, 2, 11, 80, 2, 17, 2, 32, 70, 2, + 129, 141, 11, 2, 73, 76, 108, 218, 1, 78, 62, 86, 142, 154, 11, 65, 38, + 68, 114, 84, 230, 5, 85, 206, 201, 1, 73, 206, 193, 1, 83, 82, 66, 2, 67, + 2, 71, 2, 74, 2, 75, 2, 76, 2, 77, 2, 80, 2, 82, 138, 69, 72, 2, 87, 2, + 89, 186, 2, 69, 3, 79, 14, 166, 173, 14, 71, 2, 89, 138, 69, 72, 2, 78, + 187, 2, 65, 10, 26, 69, 139, 161, 11, 79, 2, 173, 190, 12, 3, 68, 73, 67, + 22, 26, 73, 247, 200, 4, 65, 20, 44, 3, 71, 78, 32, 165, 164, 9, 2, 68, + 68, 18, 226, 222, 10, 67, 98, 78, 242, 60, 65, 174, 158, 1, 74, 228, 2, + 5, 70, 73, 78, 65, 76, 50, 85, 167, 242, 1, 86, 4, 167, 199, 14, 69, 4, + 202, 235, 9, 80, 159, 237, 3, 76, 6, 70, 78, 185, 220, 13, 11, 71, 72, + 84, 32, 87, 73, 84, 72, 32, 83, 84, 4, 200, 241, 7, 7, 69, 32, 80, 79, + 73, 78, 84, 179, 253, 6, 74, 124, 152, 1, 3, 67, 79, 77, 130, 3, 68, 78, + 76, 156, 4, 4, 72, 73, 71, 72, 72, 7, 83, 89, 77, 66, 79, 76, 32, 226, + 217, 7, 69, 205, 188, 1, 3, 84, 65, 77, 20, 52, 7, 66, 73, 78, 73, 78, + 71, 32, 235, 236, 14, 77, 18, 88, 3, 68, 79, 85, 32, 5, 76, 79, 78, 71, + 32, 78, 78, 33, 6, 83, 72, 79, 82, 84, 32, 2, 225, 237, 13, 3, 66, 76, + 69, 8, 142, 1, 72, 34, 76, 230, 251, 10, 82, 21, 7, 68, 69, 83, 67, 69, + 78, 68, 2, 11, 65, 2, 171, 206, 10, 83, 6, 34, 72, 34, 76, 231, 251, 10, + 82, 2, 173, 252, 10, 3, 73, 71, 72, 2, 141, 252, 10, 2, 79, 87, 24, 152, + 1, 4, 65, 78, 84, 65, 144, 207, 12, 4, 79, 82, 79, 77, 155, 43, 73, 70, + 76, 4, 65, 74, 65, 78, 32, 6, 69, 84, 84, 69, 82, 32, 173, 3, 2, 79, 87, + 2, 185, 161, 13, 3, 89, 65, 76, 66, 228, 1, 2, 68, 65, 42, 74, 90, 78, + 138, 143, 11, 82, 206, 147, 1, 79, 162, 75, 67, 162, 186, 1, 69, 250, 18, + 71, 242, 42, 66, 2, 70, 2, 72, 2, 75, 2, 76, 2, 77, 2, 80, 2, 83, 2, 84, + 2, 87, 2, 89, 186, 2, 65, 2, 73, 3, 85, 5, 197, 178, 11, 5, 71, 66, 65, + 83, 73, 8, 40, 4, 79, 78, 65, 32, 227, 232, 14, 65, 6, 158, 238, 12, 67, + 138, 248, 1, 74, 3, 82, 11, 26, 65, 1, 2, 89, 65, 5, 245, 135, 8, 5, 32, + 87, 79, 76, 79, 2, 29, 5, 32, 84, 79, 78, 69, 2, 17, 2, 32, 65, 2, 135, + 207, 8, 80, 4, 68, 7, 71, 66, 65, 75, 85, 82, 85, 1, 6, 79, 79, 32, 68, + 69, 78, 2, 247, 146, 13, 78, 184, 1, 90, 32, 156, 3, 2, 77, 73, 118, 78, + 150, 1, 82, 238, 7, 84, 250, 153, 14, 45, 147, 40, 83, 18, 202, 1, 66, + 96, 5, 69, 78, 84, 82, 89, 22, 80, 204, 186, 2, 15, 79, 78, 69, 32, 85, + 78, 68, 69, 82, 32, 69, 73, 71, 72, 84, 160, 129, 4, 9, 77, 79, 66, 73, + 76, 69, 32, 80, 72, 189, 30, 3, 83, 77, 79, 4, 52, 4, 82, 69, 65, 75, + 165, 137, 2, 3, 73, 67, 89, 2, 17, 2, 32, 72, 2, 155, 195, 13, 69, 5, + 203, 220, 13, 32, 4, 60, 6, 69, 68, 69, 83, 84, 82, 221, 201, 9, 3, 73, + 82, 65, 2, 145, 159, 11, 2, 73, 65, 4, 36, 5, 78, 65, 76, 32, 68, 59, 83, + 2, 173, 206, 13, 9, 73, 71, 73, 84, 32, 83, 72, 65, 80, 2, 175, 144, 11, + 77, 6, 38, 45, 249, 217, 9, 3, 70, 79, 82, 4, 76, 8, 66, 82, 69, 65, 75, + 73, 78, 71, 161, 169, 2, 5, 80, 79, 84, 65, 66, 2, 249, 221, 6, 2, 32, + 72, 97, 56, 2, 84, 72, 146, 11, 77, 209, 229, 10, 3, 68, 73, 67, 88, 42, + 32, 201, 205, 6, 4, 69, 65, 83, 84, 86, 96, 5, 69, 65, 83, 84, 32, 148, + 2, 6, 73, 78, 68, 73, 67, 32, 221, 1, 5, 87, 69, 83, 84, 32, 32, 82, 65, + 238, 242, 6, 80, 130, 1, 84, 138, 197, 4, 66, 38, 68, 18, 83, 247, 58, + 87, 14, 40, 4, 82, 82, 79, 87, 215, 238, 6, 78, 13, 11, 32, 10, 96, 9, + 67, 82, 79, 83, 83, 73, 78, 71, 32, 248, 2, 2, 65, 78, 162, 238, 6, 87, + 203, 134, 5, 70, 4, 234, 197, 3, 83, 231, 170, 3, 78, 20, 54, 80, 60, 3, + 81, 85, 65, 66, 82, 171, 132, 1, 70, 2, 37, 7, 76, 65, 67, 69, 72, 79, + 76, 2, 243, 179, 4, 68, 4, 220, 179, 4, 2, 82, 84, 209, 230, 9, 5, 78, + 84, 73, 84, 89, 2, 17, 2, 85, 80, 2, 171, 166, 4, 69, 34, 82, 65, 254, + 238, 6, 80, 130, 1, 84, 138, 197, 4, 66, 38, 68, 18, 83, 247, 58, 87, 16, + 34, 78, 33, 4, 82, 82, 79, 87, 2, 241, 194, 3, 3, 68, 32, 83, 15, 11, 32, + 12, 84, 3, 84, 79, 32, 142, 234, 6, 67, 40, 3, 65, 78, 68, 234, 2, 87, + 203, 134, 5, 70, 4, 150, 235, 6, 67, 169, 204, 6, 4, 76, 79, 78, 71, 56, + 54, 32, 236, 5, 5, 67, 72, 69, 68, 32, 207, 2, 69, 38, 162, 1, 65, 132, + 2, 4, 78, 79, 82, 77, 78, 80, 46, 83, 170, 1, 84, 234, 221, 7, 69, 196, + 33, 7, 73, 68, 69, 78, 84, 73, 67, 154, 193, 4, 71, 38, 76, 191, 78, 67, + 10, 88, 3, 32, 83, 85, 52, 7, 78, 32, 69, 76, 69, 77, 69, 28, 2, 83, 89, + 251, 128, 8, 76, 4, 30, 66, 1, 3, 80, 69, 82, 2, 29, 2, 83, 69, 2, 11, + 78, 2, 183, 125, 84, 2, 37, 7, 77, 80, 84, 79, 84, 73, 67, 2, 17, 2, 65, + 76, 2, 185, 129, 8, 2, 76, 89, 4, 253, 195, 6, 14, 65, 76, 32, 83, 85, + 66, 71, 82, 79, 85, 80, 32, 79, 70, 2, 137, 129, 8, 6, 65, 82, 65, 76, + 76, 69, 6, 48, 6, 81, 85, 65, 82, 69, 32, 247, 204, 13, 73, 4, 68, 5, 73, + 77, 65, 71, 69, 1, 8, 79, 82, 73, 71, 73, 78, 65, 76, 2, 21, 3, 32, 79, + 70, 2, 255, 193, 6, 32, 4, 210, 199, 10, 82, 215, 241, 1, 73, 8, 58, 76, + 40, 4, 82, 73, 71, 72, 133, 1, 3, 85, 80, 80, 4, 36, 2, 69, 70, 133, 1, + 2, 79, 87, 2, 89, 20, 84, 32, 83, 69, 77, 73, 67, 73, 82, 67, 76, 69, 32, + 87, 73, 84, 72, 32, 84, 72, 2, 17, 2, 82, 69, 2, 167, 128, 13, 69, 2, 11, + 69, 2, 41, 8, 82, 32, 82, 73, 71, 72, 84, 45, 2, 197, 147, 3, 6, 83, 72, + 65, 68, 79, 87, 11, 34, 32, 53, 4, 66, 79, 79, 75, 4, 17, 2, 80, 65, 4, + 222, 184, 14, 71, 215, 22, 68, 5, 81, 18, 32, 87, 73, 84, 72, 32, 68, 69, + 67, 79, 82, 65, 84, 73, 86, 69, 32, 67, 2, 171, 141, 4, 79, 186, 6, 102, + 77, 176, 3, 4, 83, 72, 85, 32, 172, 178, 5, 8, 84, 32, 65, 78, 68, 32, + 66, 79, 187, 155, 8, 76, 26, 36, 4, 66, 69, 82, 32, 247, 2, 69, 24, 58, + 69, 50, 70, 42, 83, 66, 84, 57, 4, 78, 73, 78, 69, 4, 204, 1, 3, 73, 71, + 72, 21, 3, 76, 69, 86, 4, 144, 1, 2, 79, 85, 13, 2, 73, 70, 6, 34, 73, + 85, 4, 69, 86, 69, 78, 4, 82, 88, 175, 251, 13, 71, 8, 40, 2, 72, 73, 46, + 69, 21, 2, 87, 69, 2, 11, 82, 2, 17, 2, 84, 69, 2, 11, 69, 2, 171, 199, + 7, 78, 4, 148, 199, 7, 3, 76, 86, 69, 1, 3, 78, 84, 89, 2, 215, 242, 6, + 82, 154, 6, 72, 12, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 251, + 166, 12, 73, 152, 6, 18, 49, 87, 50, 160, 2, 222, 1, 55, 2, 56, 2, 57, 2, + 65, 2, 66, 2, 67, 2, 68, 2, 69, 3, 70, 248, 3, 138, 1, 48, 2, 49, 2, 50, + 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, + 68, 2, 69, 143, 1, 70, 32, 194, 199, 14, 48, 2, 49, 2, 50, 2, 51, 2, 52, + 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, 68, 2, 69, 3, + 70, 24, 182, 198, 14, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, + 55, 2, 56, 2, 57, 2, 65, 3, 66, 142, 1, 118, 76, 226, 3, 83, 164, 2, 5, + 84, 79, 78, 69, 45, 148, 225, 7, 8, 67, 73, 82, 67, 76, 69, 68, 32, 155, + 236, 4, 68, 92, 88, 6, 69, 84, 84, 69, 82, 32, 197, 163, 1, 10, 79, 71, + 79, 71, 82, 65, 77, 32, 78, 89, 90, 130, 2, 78, 90, 84, 246, 214, 7, 88, + 162, 126, 65, 232, 203, 1, 2, 72, 65, 242, 70, 82, 206, 147, 1, 79, 174, + 60, 68, 2, 77, 2, 80, 150, 201, 1, 69, 234, 61, 67, 2, 70, 2, 71, 2, 75, + 2, 76, 2, 81, 2, 83, 2, 86, 2, 89, 2, 90, 186, 2, 73, 2, 85, 3, 87, 22, + 86, 84, 230, 183, 12, 80, 254, 134, 2, 67, 2, 75, 2, 81, 2, 82, 2, 89, + 187, 2, 65, 6, 222, 190, 14, 83, 2, 88, 187, 2, 65, 14, 64, 4, 73, 71, + 78, 32, 189, 1, 7, 89, 76, 76, 65, 66, 76, 69, 12, 56, 4, 70, 79, 82, 32, + 209, 193, 13, 4, 88, 87, 32, 88, 10, 156, 9, 2, 76, 79, 194, 164, 3, 84, + 232, 93, 8, 73, 78, 86, 69, 82, 84, 69, 66, 160, 141, 4, 3, 65, 78, 73, + 183, 167, 2, 80, 2, 209, 244, 11, 4, 32, 76, 69, 78, 14, 202, 190, 14, + 66, 2, 68, 2, 71, 2, 74, 2, 77, 2, 83, 3, 86, 250, 14, 178, 2, 66, 226, + 1, 67, 224, 5, 4, 70, 70, 73, 67, 54, 71, 248, 6, 4, 73, 76, 32, 68, 22, + 76, 186, 69, 78, 150, 5, 80, 234, 7, 82, 218, 10, 83, 188, 8, 2, 84, 84, + 154, 8, 85, 248, 1, 3, 86, 69, 82, 198, 170, 2, 89, 202, 198, 5, 72, 238, + 254, 1, 75, 154, 249, 1, 68, 194, 64, 77, 246, 9, 87, 183, 137, 1, 88, + 10, 132, 1, 6, 76, 73, 81, 85, 69, 32, 180, 158, 2, 9, 83, 69, 82, 86, + 69, 82, 32, 69, 89, 189, 29, 8, 74, 69, 67, 84, 32, 82, 69, 80, 6, 176, + 198, 4, 13, 65, 78, 71, 76, 69, 32, 79, 80, 69, 78, 73, 78, 71, 175, 241, + 1, 72, 28, 56, 2, 82, 32, 234, 4, 84, 229, 193, 5, 3, 67, 85, 76, 22, + 156, 1, 11, 65, 77, 79, 85, 78, 84, 32, 79, 70, 32, 67, 22, 66, 198, 1, + 67, 118, 68, 106, 70, 0, 10, 73, 78, 86, 69, 82, 84, 69, 68, 32, 70, 143, + 225, 12, 72, 2, 207, 179, 5, 72, 6, 136, 1, 15, 82, 65, 78, 67, 72, 32, + 66, 65, 78, 75, 32, 73, 68, 69, 78, 160, 254, 4, 5, 69, 76, 84, 32, 66, + 157, 134, 6, 3, 79, 87, 32, 2, 21, 3, 84, 73, 70, 2, 11, 73, 2, 215, 244, + 10, 67, 4, 92, 17, 85, 83, 84, 79, 77, 69, 82, 32, 65, 67, 67, 79, 85, + 78, 84, 32, 78, 155, 202, 1, 72, 2, 183, 156, 6, 85, 4, 44, 5, 79, 85, + 66, 76, 69, 251, 204, 12, 65, 2, 11, 32, 2, 11, 66, 2, 177, 158, 7, 3, + 65, 67, 75, 2, 191, 243, 13, 79, 4, 152, 241, 8, 4, 65, 71, 79, 78, 181, + 189, 3, 2, 79, 80, 2, 11, 69, 2, 221, 251, 7, 5, 32, 66, 85, 73, 76, 60, + 48, 4, 72, 65, 77, 32, 133, 176, 14, 2, 79, 78, 58, 122, 70, 0, 10, 82, + 69, 86, 69, 82, 83, 69, 68, 32, 70, 36, 7, 76, 69, 84, 84, 69, 82, 32, + 137, 253, 3, 3, 83, 80, 65, 2, 149, 138, 4, 4, 69, 65, 84, 72, 52, 196, + 1, 2, 65, 73, 22, 66, 2, 80, 34, 67, 36, 2, 69, 65, 64, 2, 70, 69, 22, + 71, 22, 73, 58, 76, 2, 82, 22, 78, 46, 79, 34, 83, 50, 85, 206, 184, 1, + 77, 170, 9, 68, 153, 210, 11, 3, 84, 73, 78, 2, 231, 159, 14, 76, 2, 11, + 69, 2, 143, 174, 12, 73, 4, 142, 147, 8, 69, 191, 149, 4, 79, 6, 138, 1, + 66, 2, 68, 169, 141, 4, 6, 77, 72, 65, 78, 67, 72, 2, 143, 165, 9, 65, 2, + 131, 168, 13, 79, 4, 32, 2, 79, 68, 139, 175, 13, 70, 2, 239, 132, 8, 72, + 2, 147, 139, 12, 85, 4, 236, 154, 13, 3, 71, 69, 65, 219, 67, 73, 4, 166, + 222, 13, 78, 227, 79, 82, 4, 146, 250, 11, 65, 233, 177, 1, 3, 84, 82, + 65, 6, 60, 5, 73, 76, 76, 69, 65, 234, 170, 12, 65, 147, 130, 2, 82, 2, + 151, 221, 13, 78, 2, 211, 131, 13, 82, 182, 8, 38, 32, 142, 11, 68, 199, + 164, 13, 73, 184, 1, 64, 6, 67, 72, 73, 75, 73, 32, 205, 6, 5, 79, 78, + 65, 76, 32, 96, 132, 1, 7, 76, 69, 84, 84, 69, 82, 32, 148, 3, 2, 77, 85, + 62, 71, 74, 80, 204, 215, 3, 2, 82, 69, 210, 221, 8, 68, 235, 170, 1, 65, + 60, 50, 65, 98, 69, 54, 73, 50, 76, 62, 79, 51, 85, 16, 50, 65, 154, 169, + 14, 78, 86, 71, 2, 76, 3, 84, 8, 234, 169, 14, 74, 2, 75, 2, 77, 3, 87, + 8, 158, 227, 13, 68, 198, 13, 82, 222, 56, 78, 3, 80, 8, 194, 151, 14, + 78, 202, 17, 72, 2, 82, 3, 83, 12, 214, 188, 8, 65, 134, 236, 5, 69, 2, + 73, 2, 79, 3, 85, 8, 242, 139, 14, 84, 174, 28, 66, 2, 72, 3, 86, 8, 142, + 216, 13, 78, 226, 79, 67, 2, 68, 3, 89, 4, 56, 2, 45, 71, 129, 190, 12, + 6, 32, 84, 84, 85, 68, 68, 2, 253, 189, 12, 13, 65, 65, 72, 76, 65, 65, + 32, 84, 84, 85, 68, 68, 65, 6, 84, 11, 85, 78, 67, 84, 85, 65, 84, 73, + 79, 78, 32, 129, 219, 6, 4, 72, 65, 65, 82, 4, 48, 8, 68, 79, 85, 66, 76, + 69, 32, 77, 3, 77, 2, 129, 223, 13, 3, 85, 67, 65, 88, 100, 7, 76, 69, + 84, 84, 69, 82, 32, 192, 2, 5, 83, 73, 71, 78, 32, 142, 183, 8, 65, 191, + 249, 3, 68, 60, 42, 65, 54, 69, 58, 73, 54, 79, 59, 85, 13, 250, 163, 14, + 66, 2, 68, 2, 72, 2, 76, 3, 87, 13, 230, 211, 13, 78, 226, 79, 67, 2, 71, + 2, 72, 3, 83, 13, 174, 195, 8, 84, 226, 223, 5, 68, 2, 78, 3, 80, 13, + 254, 233, 13, 82, 138, 56, 78, 86, 77, 2, 79, 3, 89, 13, 130, 220, 13, + 68, 218, 52, 78, 202, 17, 74, 2, 75, 3, 82, 6, 58, 73, 236, 228, 12, 4, + 72, 79, 68, 68, 219, 151, 1, 77, 2, 243, 181, 1, 75, 252, 6, 34, 32, 153, + 57, 3, 69, 82, 32, 246, 6, 240, 2, 8, 67, 72, 73, 78, 69, 83, 69, 32, 44, + 10, 72, 85, 78, 71, 65, 82, 73, 65, 78, 32, 244, 6, 7, 73, 84, 65, 76, + 73, 67, 32, 212, 4, 14, 78, 79, 82, 84, 72, 32, 65, 82, 65, 66, 73, 65, + 78, 32, 196, 4, 3, 80, 69, 82, 216, 13, 2, 83, 79, 144, 13, 14, 84, 85, + 82, 75, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 132, 7, 7, 85, 89, 71, + 72, 85, 82, 32, 219, 209, 11, 75, 4, 174, 251, 11, 73, 177, 95, 3, 72, + 79, 79, 216, 1, 96, 6, 67, 65, 80, 73, 84, 65, 0, 4, 83, 77, 65, 76, 221, + 5, 7, 78, 85, 77, 66, 69, 82, 32, 102, 45, 9, 76, 32, 76, 69, 84, 84, 69, + 82, 32, 102, 226, 1, 65, 54, 69, 164, 2, 10, 78, 73, 75, 79, 76, 83, 66, + 85, 82, 71, 0, 9, 82, 85, 68, 73, 77, 69, 78, 84, 65, 42, 79, 34, 85, + 192, 240, 3, 5, 67, 76, 79, 83, 69, 150, 157, 8, 73, 241, 203, 1, 6, 83, + 72, 79, 82, 84, 32, 11, 234, 222, 12, 77, 198, 117, 78, 162, 70, 65, 3, + 75, 63, 174, 1, 77, 22, 78, 78, 83, 134, 248, 9, 67, 86, 71, 2, 76, 2, + 84, 198, 152, 2, 90, 242, 134, 2, 66, 2, 68, 2, 69, 2, 70, 2, 72, 2, 74, + 2, 75, 2, 80, 2, 82, 3, 86, 5, 235, 152, 14, 80, 11, 34, 84, 182, 152, + 14, 67, 3, 89, 5, 189, 148, 2, 5, 45, 83, 72, 65, 80, 5, 139, 152, 14, + 90, 4, 11, 32, 4, 150, 129, 14, 79, 3, 85, 7, 250, 128, 14, 69, 215, 22, + 79, 9, 150, 148, 14, 78, 154, 3, 83, 3, 85, 12, 210, 4, 70, 222, 254, 5, + 79, 243, 191, 6, 84, 78, 80, 7, 76, 69, 84, 84, 69, 82, 32, 169, 3, 8, + 78, 85, 77, 69, 82, 65, 76, 32, 70, 190, 1, 69, 90, 75, 50, 83, 36, 3, + 78, 79, 82, 242, 191, 10, 85, 206, 201, 1, 73, 190, 137, 1, 80, 2, 84, + 150, 83, 67, 186, 22, 66, 2, 68, 2, 72, 2, 86, 2, 89, 2, 90, 214, 22, 65, + 3, 79, 23, 186, 244, 9, 83, 154, 153, 2, 82, 150, 201, 1, 75, 222, 61, + 70, 2, 76, 2, 77, 3, 78, 8, 150, 253, 13, 72, 214, 22, 65, 2, 69, 3, 85, + 4, 32, 2, 79, 85, 199, 252, 13, 72, 2, 29, 5, 84, 72, 69, 82, 78, 2, 233, + 137, 13, 2, 32, 84, 8, 38, 70, 206, 190, 12, 84, 191, 58, 79, 4, 11, 73, + 4, 174, 165, 12, 70, 167, 214, 1, 86, 64, 76, 7, 76, 69, 84, 84, 69, 82, + 32, 209, 3, 7, 78, 85, 77, 66, 69, 82, 32, 58, 210, 1, 65, 38, 71, 38, + 72, 30, 75, 38, 84, 74, 90, 134, 107, 77, 208, 153, 3, 2, 69, 83, 226, + 161, 3, 66, 2, 70, 2, 82, 2, 89, 138, 193, 3, 78, 218, 235, 1, 76, 190, + 56, 68, 146, 1, 81, 134, 3, 87, 131, 56, 83, 4, 186, 193, 3, 76, 199, + 254, 9, 73, 4, 230, 226, 12, 72, 171, 154, 1, 69, 4, 134, 142, 14, 65, 3, + 69, 4, 250, 174, 7, 72, 223, 222, 5, 65, 8, 34, 72, 166, 141, 14, 65, 3, + 69, 4, 254, 132, 13, 65, 167, 136, 1, 69, 4, 11, 65, 4, 162, 190, 13, 73, + 227, 79, 72, 6, 154, 148, 6, 84, 159, 224, 6, 79, 180, 1, 64, 11, 77, 73, + 67, 32, 76, 69, 84, 84, 69, 82, 32, 183, 5, 83, 76, 228, 1, 2, 67, 72, + 22, 68, 66, 69, 22, 73, 30, 77, 2, 78, 30, 80, 22, 83, 70, 84, 50, 86, + 38, 89, 94, 90, 218, 190, 6, 76, 2, 82, 162, 222, 2, 71, 146, 165, 2, 79, + 142, 205, 1, 66, 246, 40, 65, 254, 31, 75, 174, 45, 72, 187, 2, 85, 2, + 139, 212, 7, 69, 6, 26, 90, 139, 247, 13, 79, 4, 174, 156, 9, 72, 231, + 201, 4, 73, 5, 187, 138, 14, 70, 7, 166, 138, 14, 65, 3, 69, 2, 169, 243, + 13, 2, 69, 78, 2, 223, 192, 6, 69, 6, 26, 72, 235, 245, 13, 73, 4, 140, + 155, 9, 3, 67, 72, 79, 3, 79, 4, 26, 83, 167, 245, 13, 65, 2, 147, 228, + 13, 73, 4, 182, 154, 9, 79, 215, 181, 4, 69, 14, 60, 2, 69, 82, 166, 168, + 8, 65, 154, 206, 5, 82, 203, 17, 85, 7, 130, 136, 14, 73, 3, 85, 4, 182, + 153, 9, 72, 231, 201, 4, 65, 104, 88, 4, 73, 65, 78, 32, 241, 5, 13, 79, + 78, 65, 76, 32, 67, 79, 77, 80, 85, 84, 69, 82, 100, 80, 7, 78, 85, 77, + 66, 69, 82, 32, 80, 5, 83, 73, 71, 78, 32, 163, 207, 10, 87, 10, 42, 84, + 182, 178, 8, 72, 139, 186, 4, 79, 6, 142, 230, 1, 87, 251, 207, 11, 69, + 88, 210, 1, 65, 86, 66, 62, 68, 98, 74, 2, 86, 30, 84, 42, 88, 210, 111, + 71, 2, 75, 2, 78, 2, 82, 162, 190, 9, 77, 190, 139, 3, 83, 218, 69, 67, + 2, 70, 2, 72, 2, 76, 2, 80, 2, 89, 2, 90, 186, 2, 73, 3, 85, 9, 45, 9, + 85, 82, 65, 77, 65, 90, 68, 65, 65, 7, 146, 234, 6, 45, 247, 150, 7, 72, + 6, 38, 65, 253, 161, 10, 3, 85, 85, 77, 5, 187, 128, 14, 71, 10, 34, 65, + 190, 130, 14, 73, 3, 85, 7, 37, 7, 72, 89, 65, 65, 85, 83, 72, 5, 231, + 232, 6, 45, 4, 254, 129, 14, 65, 3, 73, 6, 170, 255, 13, 72, 186, 2, 65, + 3, 85, 4, 176, 204, 11, 8, 83, 72, 65, 65, 89, 65, 84, 72, 139, 181, 2, + 65, 5, 141, 142, 5, 31, 32, 87, 73, 84, 72, 32, 77, 79, 78, 73, 84, 79, + 82, 32, 73, 78, 32, 80, 79, 82, 84, 82, 65, 73, 84, 32, 79, 82, 73, 69, + 78, 144, 1, 92, 6, 71, 68, 73, 65, 78, 32, 141, 7, 12, 85, 84, 72, 32, + 65, 82, 65, 66, 73, 65, 78, 32, 80, 58, 70, 74, 76, 145, 5, 7, 78, 85, + 77, 66, 69, 82, 32, 2, 11, 82, 2, 241, 221, 11, 10, 65, 67, 84, 73, 79, + 78, 32, 79, 78, 69, 60, 76, 6, 69, 84, 84, 69, 82, 32, 149, 4, 8, 73, 71, + 65, 84, 85, 82, 69, 32, 58, 234, 1, 65, 96, 6, 70, 73, 78, 65, 76, 32, + 200, 1, 5, 82, 69, 83, 72, 45, 194, 215, 1, 76, 254, 165, 4, 71, 90, 90, + 34, 83, 66, 89, 158, 208, 1, 72, 234, 5, 75, 198, 75, 66, 178, 216, 4, + 78, 134, 2, 84, 2, 87, 218, 103, 80, 171, 4, 77, 6, 26, 76, 215, 251, 12, + 89, 4, 156, 128, 6, 8, 84, 69, 82, 78, 65, 84, 69, 32, 243, 214, 1, 69, + 18, 116, 3, 78, 85, 78, 0, 5, 83, 65, 68, 72, 69, 0, 3, 84, 65, 87, 222, + 215, 1, 65, 178, 201, 6, 66, 143, 194, 5, 72, 5, 41, 8, 32, 87, 73, 84, + 72, 32, 86, 69, 2, 241, 214, 5, 4, 82, 84, 73, 67, 2, 157, 215, 1, 6, 65, + 89, 73, 78, 45, 68, 18, 42, 84, 198, 254, 5, 79, 223, 137, 6, 70, 10, 42, + 72, 194, 216, 1, 87, 251, 207, 11, 69, 4, 222, 136, 12, 82, 159, 2, 73, + 64, 60, 7, 76, 69, 84, 84, 69, 82, 32, 245, 3, 3, 78, 85, 77, 58, 202, 1, + 65, 38, 68, 74, 71, 34, 75, 34, 83, 78, 84, 230, 43, 90, 182, 166, 1, 76, + 50, 81, 238, 205, 1, 82, 154, 217, 2, 89, 158, 208, 1, 72, 174, 81, 66, + 178, 216, 4, 78, 134, 2, 87, 218, 103, 70, 171, 4, 77, 4, 198, 167, 3, + 76, 199, 254, 9, 89, 6, 32, 2, 72, 65, 183, 211, 1, 65, 4, 194, 156, 8, + 76, 215, 171, 5, 68, 4, 238, 44, 72, 191, 204, 5, 73, 4, 198, 208, 7, 65, + 187, 75, 72, 8, 26, 65, 211, 244, 12, 72, 6, 142, 194, 7, 77, 138, 133, + 6, 68, 143, 45, 84, 8, 130, 91, 72, 226, 150, 11, 69, 243, 131, 1, 65, 6, + 56, 4, 66, 69, 82, 32, 229, 185, 13, 4, 69, 82, 73, 67, 4, 26, 70, 195, + 217, 12, 79, 2, 199, 132, 12, 73, 146, 1, 80, 7, 79, 82, 75, 72, 79, 78, + 32, 197, 3, 8, 89, 69, 78, 73, 83, 69, 73, 32, 84, 54, 65, 202, 1, 69, + 122, 73, 30, 79, 195, 134, 12, 66, 45, 106, 69, 194, 208, 9, 83, 158, + 160, 4, 66, 2, 68, 2, 71, 2, 76, 2, 78, 2, 81, 2, 82, 2, 84, 3, 89, 20, + 218, 240, 13, 66, 2, 68, 2, 71, 2, 75, 2, 76, 2, 78, 2, 82, 2, 83, 2, 84, + 3, 89, 20, 74, 78, 138, 211, 13, 76, 158, 27, 83, 146, 1, 67, 2, 77, 2, + 80, 3, 90, 8, 178, 239, 13, 67, 2, 71, 2, 84, 3, 89, 7, 134, 239, 13, 67, + 3, 81, 13, 130, 3, 69, 234, 235, 13, 80, 2, 81, 3, 84, 62, 38, 65, 170, + 1, 69, 86, 73, 23, 79, 39, 98, 69, 162, 236, 13, 83, 62, 78, 86, 66, 2, + 68, 2, 71, 2, 76, 2, 81, 2, 82, 2, 84, 3, 89, 17, 150, 242, 11, 78, 154, + 251, 1, 66, 2, 71, 2, 75, 2, 84, 3, 89, 15, 46, 78, 174, 235, 13, 83, + 146, 1, 67, 3, 90, 6, 186, 236, 13, 67, 2, 84, 3, 89, 5, 151, 236, 13, + 81, 6, 26, 69, 235, 235, 13, 81, 5, 231, 235, 13, 75, 52, 148, 1, 10, 67, + 79, 77, 66, 73, 78, 73, 78, 71, 32, 36, 7, 76, 69, 84, 84, 69, 82, 32, + 233, 1, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 8, 186, 236, + 5, 84, 143, 227, 3, 68, 36, 134, 200, 1, 65, 210, 206, 1, 82, 130, 216, + 2, 76, 58, 90, 34, 83, 66, 89, 128, 211, 1, 6, 70, 73, 78, 65, 76, 32, 0, + 6, 71, 73, 77, 69, 76, 45, 134, 3, 75, 198, 75, 66, 178, 216, 4, 78, 134, + 2, 84, 2, 87, 218, 103, 80, 171, 4, 77, 8, 56, 4, 84, 87, 79, 32, 214, + 196, 11, 70, 235, 129, 1, 66, 4, 246, 210, 12, 66, 71, 68, 6, 130, 181, + 1, 87, 240, 154, 3, 3, 65, 68, 85, 231, 205, 7, 77, 22, 212, 1, 34, 32, + 87, 73, 84, 72, 32, 69, 88, 67, 76, 65, 77, 65, 84, 73, 79, 78, 32, 77, + 65, 82, 75, 32, 87, 73, 84, 72, 32, 76, 69, 70, 84, 32, 82, 44, 7, 67, + 79, 77, 73, 78, 71, 32, 194, 1, 69, 235, 147, 13, 73, 2, 21, 3, 73, 71, + 72, 2, 243, 235, 9, 84, 10, 148, 1, 6, 65, 85, 84, 79, 77, 79, 20, 7, 70, + 73, 82, 69, 32, 69, 78, 192, 224, 2, 2, 84, 65, 148, 232, 8, 6, 80, 79, + 76, 73, 67, 69, 155, 21, 66, 2, 135, 191, 11, 66, 2, 175, 202, 12, 71, 8, + 70, 32, 205, 174, 12, 11, 45, 80, 73, 69, 67, 69, 32, 83, 87, 73, 77, 6, + 40, 4, 68, 79, 84, 32, 215, 157, 10, 66, 4, 26, 79, 175, 159, 10, 76, 2, + 177, 233, 2, 11, 86, 69, 82, 32, 84, 87, 79, 32, 68, 79, 84, 46, 82, 69, + 188, 6, 2, 84, 73, 248, 212, 11, 5, 72, 73, 85, 67, 72, 171, 180, 1, 80, + 36, 70, 78, 201, 5, 12, 82, 65, 84, 73, 78, 71, 32, 83, 89, 83, 84, 69, + 34, 22, 32, 139, 4, 45, 28, 108, 2, 66, 79, 32, 7, 67, 69, 78, 84, 82, + 69, 32, 114, 70, 70, 72, 42, 77, 226, 133, 7, 83, 215, 233, 3, 76, 4, + 198, 220, 13, 79, 155, 3, 88, 8, 50, 84, 222, 200, 11, 66, 222, 2, 65, + 191, 82, 67, 2, 37, 7, 69, 65, 82, 68, 82, 79, 80, 2, 191, 202, 11, 45, + 4, 44, 5, 73, 76, 69, 32, 70, 239, 130, 2, 79, 2, 235, 130, 2, 79, 2, 17, + 2, 65, 78, 2, 191, 175, 10, 68, 4, 57, 12, 65, 73, 76, 66, 79, 88, 32, + 87, 73, 84, 72, 32, 4, 44, 3, 76, 79, 87, 21, 4, 82, 65, 73, 83, 2, 17, + 2, 69, 82, 2, 189, 243, 11, 2, 69, 68, 6, 108, 15, 67, 73, 82, 67, 85, + 73, 84, 45, 79, 85, 84, 80, 85, 84, 32, 181, 187, 12, 6, 79, 85, 84, 76, + 73, 78, 4, 18, 72, 3, 76, 2, 173, 191, 1, 4, 45, 84, 89, 80, 2, 153, 205, + 12, 6, 77, 32, 67, 79, 77, 77, 6, 64, 2, 79, 78, 237, 177, 1, 8, 67, 65, + 76, 32, 68, 73, 83, 67, 2, 167, 192, 11, 32, 206, 1, 112, 3, 65, 78, 71, + 66, 73, 212, 8, 5, 78, 65, 84, 69, 32, 32, 3, 84, 72, 79, 202, 171, 5, + 32, 175, 158, 6, 67, 6, 28, 2, 69, 32, 135, 22, 85, 4, 174, 133, 12, 66, + 175, 76, 72, 188, 1, 48, 5, 71, 73, 78, 65, 76, 21, 3, 89, 65, 32, 2, + 203, 133, 1, 32, 186, 1, 106, 65, 30, 70, 250, 1, 73, 32, 7, 76, 69, 84, + 84, 69, 82, 32, 142, 2, 83, 218, 1, 86, 243, 223, 11, 68, 4, 246, 249, 9, 73, 3, 85, 12, 41, 8, 82, 65, 67, 84, 73, 79, 78, 32, 12, 56, 4, 79, 78, - 69, 32, 81, 6, 84, 72, 82, 69, 69, 32, 8, 42, 83, 170, 140, 11, 69, 46, - 72, 43, 81, 2, 177, 141, 11, 4, 73, 88, 84, 69, 4, 26, 83, 187, 142, 11, - 81, 2, 249, 223, 7, 4, 73, 88, 84, 69, 2, 225, 233, 11, 3, 83, 83, 72, - 104, 226, 1, 82, 166, 212, 6, 89, 150, 205, 2, 65, 38, 68, 46, 84, 46, - 86, 186, 24, 85, 210, 200, 1, 73, 42, 76, 246, 189, 1, 78, 46, 83, 82, - 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, 77, 2, 87, 186, 2, - 69, 3, 79, 6, 206, 135, 13, 72, 2, 82, 187, 2, 65, 18, 112, 20, 69, 81, - 85, 69, 78, 67, 69, 32, 70, 79, 82, 32, 76, 69, 84, 84, 69, 82, 32, 82, - 29, 4, 73, 71, 78, 32, 4, 178, 134, 13, 72, 3, 82, 14, 222, 161, 9, 65, - 74, 67, 98, 78, 234, 128, 2, 79, 227, 161, 1, 86, 26, 49, 10, 79, 87, 69, - 76, 32, 83, 73, 71, 78, 32, 26, 254, 162, 9, 65, 106, 86, 214, 20, 85, - 210, 200, 1, 73, 206, 134, 2, 69, 3, 79, 4, 242, 171, 9, 76, 27, 82, 226, - 1, 76, 4, 65, 71, 69, 32, 164, 4, 6, 77, 65, 78, 89, 65, 32, 247, 129, - 13, 67, 144, 1, 56, 6, 67, 65, 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 72, - 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 72, 194, 1, 65, 38, 69, 114, - 75, 42, 79, 22, 84, 178, 204, 6, 72, 170, 190, 4, 67, 2, 68, 2, 71, 138, - 176, 1, 83, 2, 90, 130, 3, 66, 254, 65, 76, 2, 77, 2, 78, 2, 80, 2, 87, - 186, 2, 73, 3, 85, 9, 226, 139, 11, 73, 131, 248, 1, 72, 15, 26, 72, 211, - 179, 12, 73, 10, 50, 84, 182, 139, 11, 67, 134, 245, 1, 75, 3, 80, 4, - 182, 128, 13, 83, 187, 2, 65, 6, 150, 128, 13, 72, 2, 89, 187, 2, 65, 5, - 211, 178, 12, 73, 6, 222, 186, 12, 83, 183, 71, 65, 80, 52, 7, 76, 69, - 84, 84, 69, 82, 32, 163, 147, 11, 68, 60, 246, 1, 65, 38, 67, 22, 68, 38, - 75, 34, 83, 30, 77, 130, 229, 2, 81, 174, 218, 7, 79, 204, 119, 2, 76, - 65, 156, 71, 2, 78, 85, 254, 1, 87, 138, 62, 69, 222, 61, 66, 2, 70, 2, - 71, 2, 72, 2, 74, 2, 82, 2, 84, 2, 88, 2, 89, 186, 2, 73, 3, 85, 7, 158, - 238, 2, 76, 167, 145, 10, 65, 2, 195, 129, 12, 65, 4, 178, 158, 8, 69, - 163, 222, 4, 72, 4, 206, 253, 11, 65, 227, 126, 72, 4, 26, 72, 175, 254, - 12, 65, 2, 239, 254, 11, 73, 124, 68, 11, 79, 77, 65, 78, 32, 83, 73, 89, - 65, 81, 32, 131, 197, 12, 69, 122, 172, 1, 17, 65, 76, 84, 69, 82, 78, - 65, 84, 69, 32, 78, 85, 77, 66, 69, 82, 32, 200, 1, 13, 70, 82, 65, 67, - 84, 73, 79, 78, 32, 79, 78, 69, 32, 48, 3, 77, 65, 82, 47, 78, 26, 54, - 70, 50, 83, 46, 84, 238, 225, 11, 78, 211, 110, 69, 6, 208, 187, 5, 3, - 79, 85, 82, 211, 195, 6, 73, 6, 160, 187, 5, 2, 73, 88, 171, 211, 5, 69, - 10, 138, 173, 2, 69, 12, 2, 87, 79, 183, 225, 8, 72, 4, 26, 83, 163, 250, - 10, 72, 2, 139, 251, 10, 73, 2, 11, 82, 2, 11, 65, 2, 219, 178, 11, 84, - 90, 33, 6, 85, 77, 66, 69, 82, 32, 90, 58, 69, 66, 70, 94, 78, 26, 83, - 78, 84, 131, 157, 5, 79, 10, 25, 4, 73, 71, 72, 84, 11, 138, 171, 2, 89, - 191, 165, 5, 32, 20, 18, 73, 35, 79, 10, 166, 2, 70, 147, 156, 5, 86, 10, - 134, 2, 82, 157, 156, 5, 2, 85, 82, 10, 65, 3, 73, 78, 69, 20, 40, 4, 69, - 86, 69, 78, 1, 2, 73, 88, 11, 166, 1, 84, 223, 205, 7, 32, 24, 34, 72, - 50, 87, 203, 168, 2, 69, 10, 34, 73, 197, 156, 5, 2, 82, 69, 4, 51, 82, - 10, 26, 69, 171, 156, 5, 79, 4, 11, 78, 4, 11, 84, 4, 159, 168, 2, 89, - 12, 34, 84, 133, 222, 10, 2, 78, 67, 10, 42, 66, 37, 6, 76, 73, 78, 69, - 68, 32, 2, 181, 233, 11, 4, 79, 88, 32, 84, 8, 254, 252, 2, 76, 254, 174, - 3, 87, 250, 181, 4, 66, 239, 13, 71, 12, 18, 72, 35, 76, 2, 245, 227, 11, - 3, 69, 65, 84, 10, 32, 2, 65, 80, 227, 218, 11, 73, 9, 29, 5, 80, 73, 78, - 71, 32, 6, 40, 6, 87, 72, 73, 84, 69, 32, 51, 66, 4, 44, 5, 65, 78, 68, - 32, 66, 247, 199, 9, 83, 2, 221, 199, 9, 4, 76, 65, 67, 75, 144, 13, 194, - 1, 65, 174, 65, 68, 30, 69, 194, 13, 72, 134, 25, 73, 206, 4, 76, 150, - 15, 79, 234, 8, 82, 208, 15, 15, 83, 65, 76, 84, 69, 82, 32, 80, 65, 72, - 76, 65, 86, 73, 32, 214, 5, 85, 151, 215, 11, 77, 154, 6, 134, 2, 68, 38, - 71, 212, 1, 11, 72, 65, 87, 72, 32, 72, 77, 79, 78, 71, 32, 138, 23, 76, - 130, 6, 78, 58, 82, 156, 22, 2, 83, 83, 132, 2, 10, 85, 32, 67, 73, 78, - 32, 72, 65, 85, 32, 180, 7, 3, 87, 32, 80, 172, 213, 7, 2, 67, 75, 145, - 217, 4, 4, 80, 69, 82, 67, 5, 157, 184, 1, 4, 68, 73, 78, 71, 14, 26, 69, - 191, 201, 12, 79, 13, 34, 32, 158, 238, 12, 82, 3, 83, 6, 72, 6, 87, 73, - 84, 72, 32, 67, 149, 139, 4, 6, 70, 65, 67, 73, 78, 71, 4, 50, 85, 181, - 197, 7, 6, 73, 82, 67, 76, 69, 68, 2, 255, 227, 11, 82, 254, 1, 190, 1, - 67, 160, 6, 9, 77, 65, 82, 75, 32, 67, 73, 77, 32, 160, 1, 7, 78, 85, 77, - 66, 69, 82, 32, 244, 1, 5, 83, 73, 71, 78, 32, 148, 10, 7, 86, 79, 87, - 69, 76, 32, 75, 227, 233, 10, 68, 78, 92, 9, 76, 65, 78, 32, 83, 73, 71, - 78, 32, 137, 3, 9, 79, 78, 83, 79, 78, 65, 78, 84, 32, 38, 132, 1, 2, 72, - 65, 38, 75, 46, 76, 34, 84, 82, 86, 30, 89, 148, 9, 2, 88, 89, 176, 5, 2, - 80, 72, 174, 1, 70, 165, 2, 2, 77, 85, 4, 162, 186, 2, 87, 187, 175, 10, - 77, 6, 250, 15, 72, 202, 186, 12, 79, 159, 14, 87, 4, 214, 12, 65, 231, - 158, 12, 73, 8, 22, 83, 247, 9, 72, 6, 152, 185, 2, 3, 72, 69, 69, 150, - 245, 8, 65, 3, 87, 4, 226, 184, 2, 65, 3, 87, 4, 198, 184, 2, 65, 223, - 233, 9, 69, 40, 122, 67, 38, 72, 46, 78, 60, 2, 80, 76, 2, 81, 222, 210, - 2, 76, 2, 77, 2, 82, 2, 86, 2, 88, 2, 89, 147, 238, 9, 65, 4, 230, 211, - 2, 72, 147, 238, 9, 65, 6, 194, 211, 2, 76, 2, 78, 147, 238, 9, 65, 12, - 58, 67, 22, 84, 202, 210, 2, 75, 2, 76, 147, 238, 9, 65, 2, 219, 210, 2, - 72, 4, 198, 210, 2, 72, 3, 83, 14, 42, 75, 50, 83, 38, 84, 195, 211, 12, - 72, 4, 26, 72, 143, 167, 12, 69, 2, 183, 138, 6, 65, 4, 138, 171, 11, 85, - 191, 185, 1, 79, 4, 254, 169, 11, 85, 179, 18, 65, 14, 52, 7, 72, 85, 78, - 68, 82, 69, 68, 38, 84, 79, 77, 4, 108, 2, 32, 77, 223, 226, 12, 83, 8, - 24, 2, 69, 78, 51, 82, 6, 26, 32, 243, 226, 12, 83, 4, 18, 66, 35, 84, 2, - 145, 194, 4, 3, 73, 76, 76, 2, 11, 72, 2, 137, 185, 9, 3, 79, 85, 83, 72, - 188, 1, 4, 67, 73, 77, 32, 250, 2, 72, 32, 3, 73, 66, 32, 22, 77, 86, 78, - 50, 84, 124, 4, 86, 79, 83, 32, 198, 1, 88, 208, 1, 6, 90, 87, 74, 32, - 84, 72, 150, 176, 1, 76, 219, 205, 4, 65, 16, 174, 1, 67, 84, 5, 78, 82, - 69, 83, 32, 22, 84, 148, 164, 11, 7, 80, 85, 66, 32, 68, 65, 87, 197, - 185, 1, 16, 72, 65, 73, 83, 32, 76, 85, 83, 32, 78, 84, 79, 71, 32, 78, - 84, 4, 48, 7, 85, 65, 77, 32, 84, 83, 72, 131, 4, 72, 2, 11, 79, 2, 167, - 175, 2, 79, 2, 207, 182, 1, 84, 6, 56, 3, 88, 87, 86, 157, 221, 12, 5, - 83, 79, 86, 32, 82, 5, 213, 131, 6, 4, 32, 67, 72, 87, 4, 214, 72, 78, - 171, 129, 12, 76, 2, 251, 163, 11, 89, 6, 40, 4, 69, 69, 74, 32, 171, - 159, 12, 85, 4, 128, 3, 2, 84, 83, 57, 2, 83, 85, 4, 26, 84, 231, 213, 8, - 81, 2, 251, 172, 2, 85, 6, 196, 1, 7, 88, 72, 69, 69, 74, 32, 67, 216, - 139, 8, 12, 72, 73, 82, 68, 45, 83, 84, 65, 71, 69, 32, 72, 155, 170, 4, - 65, 14, 54, 70, 22, 83, 30, 84, 190, 69, 76, 163, 212, 7, 78, 2, 191, - 200, 12, 69, 2, 177, 128, 6, 2, 69, 69, 6, 42, 72, 29, 6, 83, 72, 65, 66, - 32, 67, 4, 82, 73, 231, 200, 12, 79, 2, 223, 188, 5, 69, 14, 34, 73, 22, - 89, 191, 214, 10, 65, 2, 151, 159, 11, 65, 10, 40, 4, 69, 69, 77, 32, - 139, 186, 12, 79, 8, 84, 3, 78, 84, 88, 128, 254, 5, 2, 84, 79, 142, 158, - 2, 82, 233, 129, 3, 2, 70, 65, 2, 255, 253, 5, 73, 2, 215, 168, 2, 65, - 56, 50, 65, 66, 69, 38, 73, 2, 85, 38, 79, 39, 87, 20, 170, 1, 65, 2, 73, - 2, 85, 2, 87, 158, 214, 12, 66, 3, 86, 8, 106, 69, 158, 214, 12, 66, 3, - 86, 8, 70, 65, 158, 214, 12, 66, 3, 86, 8, 34, 79, 158, 214, 12, 66, 3, - 86, 4, 154, 214, 12, 66, 3, 86, 76, 18, 76, 23, 77, 2, 155, 152, 12, 65, - 74, 74, 32, 104, 6, 89, 82, 69, 78, 69, 32, 241, 249, 3, 4, 83, 32, 85, - 80, 8, 80, 3, 66, 82, 65, 250, 231, 10, 84, 192, 94, 4, 68, 79, 87, 78, - 1, 2, 85, 80, 2, 143, 178, 12, 78, 64, 80, 2, 76, 69, 40, 4, 82, 73, 71, - 72, 253, 2, 7, 78, 85, 77, 66, 69, 82, 32, 48, 38, 70, 89, 5, 84, 84, 69, - 82, 32, 2, 57, 12, 84, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 70, 2, - 137, 209, 5, 2, 76, 69, 46, 224, 1, 5, 70, 73, 78, 65, 76, 30, 84, 246, - 118, 68, 34, 76, 50, 81, 206, 194, 1, 82, 134, 212, 2, 65, 50, 71, 90, - 90, 34, 83, 66, 89, 154, 193, 1, 72, 234, 5, 75, 130, 76, 66, 190, 173, - 4, 78, 254, 1, 87, 202, 103, 80, 171, 4, 77, 2, 209, 208, 11, 2, 32, 78, - 4, 202, 209, 10, 69, 247, 128, 1, 65, 14, 198, 120, 84, 198, 234, 9, 70, - 143, 84, 79, 4, 32, 2, 67, 65, 183, 144, 12, 68, 2, 251, 187, 11, 75, - 184, 2, 94, 65, 216, 1, 11, 69, 78, 84, 72, 69, 83, 73, 90, 69, 68, 32, - 242, 16, 84, 243, 235, 11, 82, 14, 80, 5, 71, 82, 65, 80, 72, 48, 5, 76, - 76, 69, 76, 32, 201, 138, 6, 2, 67, 72, 6, 206, 121, 85, 198, 187, 8, 32, - 175, 219, 2, 79, 6, 44, 5, 87, 73, 84, 72, 32, 191, 174, 12, 84, 4, 214, - 165, 6, 84, 151, 138, 4, 72, 150, 2, 252, 1, 7, 72, 65, 78, 71, 85, 76, - 32, 188, 4, 10, 73, 68, 69, 79, 71, 82, 65, 80, 72, 32, 188, 7, 18, 75, - 79, 82, 69, 65, 78, 32, 67, 72, 65, 82, 65, 67, 84, 69, 82, 32, 79, 44, - 7, 78, 85, 77, 66, 69, 82, 32, 138, 187, 4, 68, 197, 154, 2, 2, 76, 65, - 58, 102, 67, 110, 72, 30, 75, 66, 77, 34, 78, 34, 80, 62, 82, 30, 83, 26, - 84, 73, 5, 73, 69, 85, 78, 71, 10, 34, 72, 33, 4, 73, 69, 85, 67, 4, 141, - 3, 4, 73, 69, 85, 67, 7, 11, 32, 4, 206, 201, 12, 65, 3, 85, 4, 197, 2, - 3, 73, 69, 85, 8, 168, 2, 5, 72, 73, 69, 85, 75, 13, 5, 73, 89, 69, 79, - 75, 4, 245, 1, 4, 73, 69, 85, 77, 4, 213, 1, 4, 73, 69, 85, 78, 8, 168, - 1, 5, 72, 73, 69, 85, 80, 13, 4, 73, 69, 85, 80, 4, 121, 4, 73, 69, 85, - 76, 4, 93, 3, 73, 79, 83, 8, 56, 5, 72, 73, 69, 85, 84, 13, 5, 73, 75, - 69, 85, 84, 4, 11, 72, 5, 167, 196, 12, 32, 72, 148, 1, 2, 65, 76, 30, - 67, 74, 69, 82, 70, 112, 2, 76, 65, 22, 77, 38, 78, 32, 2, 82, 69, 78, - 83, 138, 2, 84, 50, 87, 134, 215, 10, 72, 159, 79, 79, 2, 225, 233, 7, 2, - 76, 73, 4, 32, 2, 79, 78, 199, 192, 10, 65, 2, 137, 212, 7, 4, 71, 82, - 65, 84, 6, 42, 78, 246, 253, 8, 65, 239, 155, 3, 73, 2, 169, 4, 5, 84, - 69, 82, 80, 82, 10, 58, 73, 156, 186, 11, 5, 69, 83, 84, 73, 86, 231, 16, - 79, 6, 208, 2, 3, 78, 65, 78, 158, 170, 12, 82, 3, 86, 2, 179, 138, 12, - 66, 4, 174, 245, 9, 69, 239, 253, 1, 79, 4, 194, 169, 11, 73, 195, 1, 65, - 8, 38, 83, 170, 178, 11, 80, 195, 109, 65, 4, 182, 179, 6, 79, 219, 142, - 6, 84, 18, 58, 69, 34, 79, 22, 80, 34, 84, 50, 85, 163, 180, 11, 73, 4, - 186, 240, 10, 86, 235, 79, 76, 2, 231, 212, 7, 67, 2, 11, 69, 2, 247, - 161, 4, 67, 4, 26, 85, 247, 145, 11, 79, 2, 247, 174, 12, 68, 4, 26, 80, - 147, 192, 12, 78, 2, 21, 3, 69, 82, 86, 2, 131, 183, 11, 73, 6, 162, 211, - 10, 72, 238, 156, 1, 69, 227, 48, 87, 4, 206, 157, 9, 79, 155, 235, 1, - 65, 4, 166, 253, 7, 32, 137, 242, 3, 2, 74, 69, 22, 42, 69, 46, 70, 42, - 78, 30, 83, 51, 84, 4, 216, 1, 3, 73, 71, 72, 243, 225, 4, 76, 4, 160, 1, - 2, 79, 85, 13, 2, 73, 70, 2, 133, 1, 3, 73, 78, 69, 4, 34, 73, 73, 4, 69, - 86, 69, 78, 2, 71, 88, 8, 34, 72, 46, 87, 247, 236, 11, 69, 2, 11, 73, 2, - 11, 82, 2, 219, 235, 10, 84, 4, 11, 69, 4, 198, 210, 10, 78, 179, 109, - 76, 18, 128, 1, 3, 73, 65, 76, 216, 1, 3, 89, 32, 80, 144, 141, 8, 6, 78, - 69, 82, 83, 72, 73, 189, 167, 3, 6, 32, 65, 76, 84, 69, 82, 12, 62, 32, - 189, 122, 10, 76, 89, 45, 82, 69, 67, 89, 67, 76, 69, 10, 38, 68, 41, 5, - 76, 73, 78, 69, 32, 2, 157, 155, 4, 5, 73, 70, 70, 69, 82, 8, 134, 186, - 3, 68, 242, 45, 70, 20, 4, 66, 65, 67, 75, 243, 209, 8, 85, 2, 235, 222, - 3, 79, 10, 98, 69, 52, 9, 73, 86, 69, 45, 80, 85, 76, 76, 45, 89, 9, 80, - 79, 82, 84, 32, 67, 79, 78, 84, 4, 182, 165, 11, 68, 145, 62, 5, 78, 71, - 69, 82, 32, 4, 40, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 233, 155, 5, 6, - 45, 79, 85, 84, 80, 85, 2, 227, 164, 11, 82, 114, 192, 1, 12, 71, 76, 79, - 84, 84, 65, 76, 32, 83, 84, 79, 80, 62, 76, 160, 3, 3, 82, 73, 83, 36, - 14, 77, 73, 68, 45, 76, 69, 86, 69, 76, 32, 84, 79, 78, 69, 57, 7, 83, - 65, 78, 68, 72, 73, 32, 7, 11, 32, 4, 206, 5, 70, 197, 166, 11, 4, 86, - 65, 82, 73, 82, 72, 6, 69, 84, 84, 69, 82, 32, 213, 2, 7, 79, 87, 45, 70, - 65, 76, 76, 74, 206, 1, 70, 226, 241, 6, 73, 2, 85, 166, 242, 1, 78, 214, - 135, 3, 67, 2, 75, 2, 80, 2, 84, 254, 68, 66, 2, 68, 2, 71, 2, 72, 2, 76, - 2, 77, 2, 82, 2, 83, 2, 86, 2, 90, 186, 2, 65, 2, 69, 3, 79, 20, 44, 5, - 73, 78, 65, 76, 32, 223, 178, 12, 65, 18, 198, 186, 10, 78, 150, 248, 1, - 75, 2, 76, 2, 77, 2, 80, 2, 84, 2, 87, 3, 89, 8, 157, 1, 5, 73, 78, 71, - 32, 84, 7, 11, 32, 4, 192, 1, 5, 76, 79, 78, 71, 32, 15, 70, 12, 66, 84, - 73, 12, 71, 76, 79, 84, 84, 65, 76, 32, 83, 84, 79, 80, 8, 21, 3, 79, 78, - 69, 9, 11, 32, 6, 32, 4, 76, 79, 78, 71, 27, 70, 5, 11, 32, 2, 11, 70, 2, - 135, 181, 3, 73, 2, 223, 161, 4, 82, 4, 222, 175, 12, 70, 3, 73, 80, 130, - 1, 65, 122, 78, 162, 1, 82, 222, 9, 83, 44, 3, 84, 82, 73, 254, 16, 68, - 173, 192, 11, 9, 79, 80, 76, 69, 32, 72, 85, 71, 71, 12, 50, 67, 50, 78, - 218, 204, 6, 32, 135, 225, 5, 82, 6, 190, 255, 10, 79, 190, 27, 69, 147, - 147, 1, 72, 2, 199, 153, 11, 85, 10, 118, 71, 20, 3, 83, 73, 86, 190, - 171, 1, 84, 224, 172, 4, 10, 32, 79, 86, 69, 82, 32, 83, 84, 65, 77, 175, - 167, 4, 67, 2, 147, 173, 11, 85, 2, 167, 239, 11, 69, 48, 186, 1, 32, - 116, 10, 80, 69, 78, 68, 73, 67, 85, 76, 65, 82, 42, 83, 198, 136, 7, 67, - 224, 10, 10, 77, 65, 78, 69, 78, 84, 32, 80, 65, 80, 153, 234, 1, 8, 70, - 79, 82, 77, 73, 78, 71, 32, 6, 34, 77, 30, 84, 223, 163, 11, 83, 2, 153, - 230, 1, 2, 73, 76, 2, 189, 196, 10, 8, 69, 78, 32, 84, 72, 79, 85, 83, 5, - 129, 215, 8, 5, 32, 87, 73, 84, 72, 32, 60, 2, 79, 78, 182, 75, 80, 213, - 213, 10, 4, 69, 86, 69, 82, 28, 22, 32, 243, 5, 65, 26, 216, 2, 10, 68, - 79, 73, 78, 71, 32, 67, 65, 82, 84, 32, 3, 73, 78, 32, 108, 5, 87, 73, - 84, 72, 32, 208, 185, 7, 4, 70, 82, 79, 87, 200, 13, 27, 82, 65, 73, 83, + 69, 32, 81, 6, 84, 72, 82, 69, 69, 32, 8, 42, 83, 162, 210, 11, 69, 46, + 72, 47, 81, 2, 173, 211, 11, 4, 73, 88, 84, 69, 4, 26, 83, 183, 212, 11, + 81, 2, 245, 253, 7, 4, 73, 88, 84, 69, 2, 217, 179, 12, 3, 83, 83, 72, + 104, 226, 1, 82, 238, 232, 6, 89, 134, 144, 3, 65, 38, 68, 114, 84, 46, + 86, 186, 5, 85, 206, 201, 1, 73, 42, 76, 250, 192, 1, 78, 46, 83, 82, 66, + 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 77, 2, 87, 186, 2, 69, + 3, 79, 6, 214, 208, 13, 72, 2, 82, 187, 2, 65, 18, 112, 20, 69, 81, 85, + 69, 78, 67, 69, 32, 70, 79, 82, 32, 76, 69, 84, 84, 69, 82, 32, 82, 29, + 4, 73, 71, 78, 32, 4, 186, 207, 13, 72, 3, 82, 14, 150, 189, 9, 67, 98, + 78, 242, 60, 65, 142, 239, 1, 79, 219, 164, 1, 86, 26, 49, 10, 79, 87, + 69, 76, 32, 83, 73, 71, 78, 32, 26, 142, 253, 9, 65, 38, 85, 22, 86, 186, + 201, 1, 73, 222, 137, 2, 69, 3, 79, 4, 174, 215, 6, 76, 243, 30, 82, 4, + 224, 240, 3, 2, 68, 79, 155, 183, 2, 71, 226, 1, 76, 4, 65, 71, 69, 32, + 140, 4, 6, 77, 65, 78, 89, 65, 32, 231, 202, 13, 67, 144, 1, 56, 6, 67, + 65, 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 72, 45, 9, 76, 32, 76, 69, 84, + 84, 69, 82, 32, 72, 194, 1, 65, 38, 69, 90, 75, 42, 79, 22, 84, 226, 224, + 6, 72, 230, 239, 4, 67, 2, 68, 2, 71, 130, 179, 1, 83, 2, 90, 130, 3, 66, + 138, 66, 76, 2, 77, 2, 78, 2, 80, 2, 87, 186, 2, 73, 3, 85, 9, 182, 209, + 11, 73, 135, 251, 1, 72, 15, 26, 72, 159, 252, 12, 73, 10, 222, 191, 9, + 84, 222, 145, 2, 67, 138, 248, 1, 75, 3, 80, 6, 134, 201, 13, 72, 2, 89, + 187, 2, 65, 5, 183, 251, 12, 73, 6, 194, 131, 13, 83, 195, 71, 65, 80, + 52, 7, 76, 69, 84, 84, 69, 82, 32, 143, 217, 11, 68, 60, 246, 1, 65, 38, + 67, 22, 68, 38, 75, 34, 83, 30, 77, 242, 240, 2, 81, 206, 144, 8, 79, + 224, 123, 2, 76, 65, 208, 73, 2, 78, 85, 134, 2, 87, 142, 62, 69, 234, + 61, 66, 2, 70, 2, 71, 2, 72, 2, 74, 2, 82, 2, 84, 2, 88, 2, 89, 186, 2, + 73, 3, 85, 7, 142, 250, 2, 76, 167, 206, 10, 65, 2, 163, 202, 12, 65, 4, + 150, 187, 8, 69, 175, 138, 5, 72, 4, 166, 198, 12, 65, 251, 126, 72, 4, + 26, 72, 159, 199, 13, 65, 2, 199, 199, 12, 73, 124, 68, 11, 79, 77, 65, + 78, 32, 83, 73, 89, 65, 81, 32, 231, 141, 13, 69, 122, 172, 1, 17, 65, + 76, 84, 69, 82, 78, 65, 84, 69, 32, 78, 85, 77, 66, 69, 82, 32, 200, 1, + 13, 70, 82, 65, 67, 84, 73, 79, 78, 32, 79, 78, 69, 32, 48, 3, 77, 65, + 82, 47, 78, 26, 54, 70, 50, 83, 46, 84, 198, 170, 12, 78, 235, 110, 69, + 6, 236, 202, 5, 3, 79, 85, 82, 151, 253, 6, 73, 6, 188, 202, 5, 2, 73, + 88, 251, 137, 6, 69, 10, 170, 184, 2, 69, 12, 2, 87, 79, 131, 156, 9, 72, + 4, 26, 83, 131, 192, 11, 72, 2, 239, 192, 11, 73, 2, 11, 82, 2, 11, 65, + 2, 255, 248, 11, 84, 90, 33, 6, 85, 77, 66, 69, 82, 32, 90, 58, 69, 66, + 70, 94, 78, 26, 83, 78, 84, 183, 172, 5, 79, 10, 25, 4, 73, 71, 72, 84, + 11, 170, 182, 2, 89, 255, 183, 5, 32, 20, 18, 73, 35, 79, 10, 166, 2, 70, + 199, 171, 5, 86, 10, 134, 2, 82, 209, 171, 5, 2, 85, 82, 10, 65, 3, 73, + 78, 69, 20, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 11, 166, 1, 84, 191, + 235, 7, 32, 24, 34, 72, 50, 87, 235, 179, 2, 69, 10, 34, 73, 249, 171, 5, + 2, 82, 69, 4, 51, 82, 10, 26, 69, 223, 171, 5, 79, 4, 11, 78, 4, 11, 84, + 4, 191, 179, 2, 89, 84, 34, 84, 161, 162, 11, 2, 78, 67, 82, 42, 66, 37, + 6, 76, 73, 78, 69, 68, 32, 2, 141, 178, 12, 4, 79, 88, 32, 84, 80, 92, 7, + 76, 65, 84, 73, 78, 32, 67, 222, 189, 6, 87, 146, 233, 4, 66, 246, 13, + 71, 159, 23, 68, 54, 130, 170, 7, 65, 143, 210, 4, 82, 12, 18, 72, 35, + 76, 2, 137, 172, 12, 3, 69, 65, 84, 10, 32, 2, 65, 80, 247, 162, 12, 73, + 9, 29, 5, 80, 73, 78, 71, 32, 6, 40, 6, 87, 72, 73, 84, 69, 32, 51, 66, + 4, 44, 5, 65, 78, 68, 32, 66, 223, 250, 9, 83, 2, 197, 250, 9, 4, 76, 65, + 67, 75, 146, 13, 210, 1, 65, 174, 65, 68, 30, 69, 134, 13, 72, 138, 25, + 73, 206, 4, 76, 148, 15, 2, 78, 80, 54, 79, 234, 8, 82, 208, 15, 15, 83, + 65, 76, 84, 69, 82, 32, 80, 65, 72, 76, 65, 86, 73, 32, 214, 5, 85, 183, + 159, 12, 77, 154, 6, 134, 2, 68, 38, 71, 212, 1, 11, 72, 65, 87, 72, 32, + 72, 77, 79, 78, 71, 32, 134, 23, 76, 130, 6, 78, 58, 82, 160, 22, 2, 83, + 83, 132, 2, 10, 85, 32, 67, 73, 78, 32, 72, 65, 85, 32, 180, 7, 3, 87, + 32, 80, 188, 241, 7, 2, 67, 75, 157, 133, 5, 4, 80, 69, 82, 67, 5, 237, + 184, 1, 4, 68, 73, 78, 71, 14, 26, 69, 219, 145, 13, 79, 13, 34, 32, 186, + 182, 13, 82, 3, 83, 6, 72, 6, 87, 73, 84, 72, 32, 67, 241, 153, 4, 6, 70, + 65, 67, 73, 78, 71, 4, 50, 85, 193, 226, 7, 6, 73, 82, 67, 76, 69, 68, 2, + 131, 172, 12, 82, 254, 1, 190, 1, 67, 160, 6, 9, 77, 65, 82, 75, 32, 67, + 73, 77, 32, 160, 1, 7, 78, 85, 77, 66, 69, 82, 32, 244, 1, 5, 83, 73, 71, + 78, 32, 144, 10, 7, 86, 79, 87, 69, 76, 32, 75, 255, 174, 11, 68, 78, 92, + 9, 76, 65, 78, 32, 83, 73, 71, 78, 32, 137, 3, 9, 79, 78, 83, 79, 78, 65, + 78, 84, 32, 38, 132, 1, 2, 72, 65, 38, 75, 46, 76, 34, 84, 82, 86, 30, + 89, 148, 9, 2, 88, 89, 172, 5, 2, 80, 72, 174, 1, 70, 165, 2, 2, 77, 85, + 4, 190, 197, 2, 87, 187, 236, 10, 77, 6, 246, 15, 72, 234, 130, 13, 79, + 159, 14, 87, 4, 210, 12, 65, 251, 230, 12, 73, 8, 22, 83, 247, 9, 72, 6, + 180, 196, 2, 3, 72, 69, 69, 214, 176, 9, 65, 3, 87, 4, 254, 195, 2, 65, + 3, 87, 4, 226, 195, 2, 65, 211, 166, 10, 69, 40, 122, 67, 38, 72, 46, 78, + 60, 2, 80, 76, 2, 81, 250, 221, 2, 76, 2, 77, 2, 82, 2, 86, 2, 88, 2, 89, + 147, 171, 10, 65, 4, 130, 223, 2, 72, 147, 171, 10, 65, 6, 222, 222, 2, + 76, 2, 78, 147, 171, 10, 65, 12, 58, 67, 22, 84, 230, 221, 2, 75, 2, 76, + 147, 171, 10, 65, 2, 247, 221, 2, 72, 4, 226, 221, 2, 72, 3, 83, 14, 42, + 75, 50, 83, 38, 84, 223, 155, 13, 72, 4, 26, 72, 159, 239, 12, 69, 2, + 251, 156, 6, 65, 4, 230, 241, 11, 85, 255, 186, 1, 79, 4, 218, 240, 11, + 85, 215, 18, 65, 14, 52, 7, 72, 85, 78, 68, 82, 69, 68, 38, 84, 79, 77, + 4, 108, 2, 32, 77, 251, 170, 13, 83, 8, 24, 2, 69, 78, 51, 82, 6, 26, 32, + 143, 171, 13, 83, 4, 18, 66, 35, 84, 2, 205, 208, 4, 3, 73, 76, 76, 2, + 11, 72, 2, 225, 235, 9, 3, 79, 85, 83, 72, 188, 1, 4, 67, 73, 77, 32, + 246, 2, 72, 32, 3, 73, 66, 32, 22, 77, 86, 78, 50, 84, 124, 4, 86, 79, + 83, 32, 198, 1, 88, 208, 1, 6, 90, 87, 74, 32, 84, 72, 238, 176, 1, 76, + 203, 223, 4, 65, 16, 174, 1, 67, 84, 5, 78, 82, 69, 83, 32, 22, 84, 240, + 234, 11, 7, 80, 85, 66, 32, 68, 65, 87, 133, 187, 1, 16, 72, 65, 73, 83, + 32, 76, 85, 83, 32, 78, 84, 79, 71, 32, 78, 84, 4, 48, 7, 85, 65, 77, 32, + 84, 83, 72, 255, 3, 72, 2, 11, 79, 2, 195, 186, 2, 79, 2, 163, 183, 1, + 84, 6, 52, 3, 88, 87, 86, 173, 135, 10, 4, 83, 79, 86, 32, 5, 157, 150, + 6, 4, 32, 67, 72, 87, 4, 158, 72, 78, 131, 202, 12, 76, 2, 219, 234, 11, + 89, 6, 40, 4, 69, 69, 74, 32, 191, 231, 12, 85, 4, 128, 3, 2, 84, 83, 57, + 2, 83, 85, 4, 26, 84, 207, 241, 8, 81, 2, 155, 184, 2, 85, 6, 196, 1, 7, + 88, 72, 69, 69, 74, 32, 67, 236, 167, 8, 12, 72, 73, 82, 68, 45, 83, 84, + 65, 71, 69, 32, 72, 167, 214, 4, 65, 14, 54, 70, 22, 83, 30, 84, 134, 69, + 76, 239, 240, 7, 78, 2, 223, 144, 13, 69, 2, 249, 146, 6, 2, 69, 69, 6, + 42, 72, 29, 6, 83, 72, 65, 66, 32, 67, 4, 82, 73, 135, 145, 13, 79, 2, + 239, 203, 5, 69, 14, 34, 73, 22, 89, 207, 155, 11, 65, 2, 247, 229, 11, + 65, 10, 40, 4, 69, 69, 77, 32, 171, 130, 13, 79, 8, 84, 3, 78, 84, 88, + 200, 144, 6, 2, 84, 79, 218, 167, 2, 82, 181, 172, 3, 2, 70, 65, 2, 199, + 144, 6, 73, 2, 247, 179, 2, 65, 56, 50, 65, 66, 69, 38, 73, 2, 85, 38, + 79, 39, 87, 20, 170, 1, 65, 2, 73, 2, 85, 2, 87, 190, 158, 13, 66, 3, 86, + 8, 106, 69, 190, 158, 13, 66, 3, 86, 8, 70, 65, 190, 158, 13, 66, 3, 86, + 8, 34, 79, 190, 158, 13, 66, 3, 86, 4, 186, 158, 13, 66, 3, 86, 76, 18, + 76, 23, 77, 2, 175, 224, 12, 65, 74, 74, 32, 104, 6, 89, 82, 69, 78, 69, + 32, 201, 136, 4, 4, 83, 32, 85, 80, 8, 80, 3, 66, 82, 65, 150, 173, 11, + 84, 172, 97, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 175, 250, 12, 78, 64, + 80, 2, 76, 69, 40, 4, 82, 73, 71, 72, 253, 2, 7, 78, 85, 77, 66, 69, 82, + 32, 48, 38, 70, 89, 5, 84, 84, 69, 82, 32, 2, 57, 12, 84, 45, 80, 79, 73, + 78, 84, 73, 78, 71, 32, 70, 2, 177, 226, 5, 2, 76, 69, 46, 224, 1, 5, 70, + 73, 78, 65, 76, 30, 84, 246, 118, 68, 34, 76, 50, 81, 238, 205, 1, 82, + 178, 215, 2, 65, 50, 71, 90, 90, 34, 83, 66, 89, 158, 208, 1, 72, 234, 5, + 75, 198, 75, 66, 178, 216, 4, 78, 134, 2, 87, 218, 103, 80, 171, 4, 77, + 2, 217, 152, 12, 2, 32, 78, 4, 222, 150, 11, 69, 243, 131, 1, 65, 14, + 198, 120, 84, 226, 175, 10, 70, 251, 86, 79, 4, 32, 2, 67, 65, 203, 216, + 12, 68, 2, 131, 132, 12, 75, 184, 2, 94, 65, 220, 1, 11, 69, 78, 84, 72, + 69, 83, 73, 90, 69, 68, 32, 242, 16, 84, 131, 180, 12, 82, 14, 80, 5, 71, + 82, 65, 80, 72, 52, 5, 76, 76, 69, 76, 32, 137, 158, 6, 2, 67, 72, 6, + 198, 232, 9, 32, 194, 222, 1, 85, 207, 145, 1, 79, 6, 44, 5, 87, 73, 84, + 72, 32, 219, 246, 12, 84, 4, 150, 193, 6, 84, 131, 176, 4, 72, 150, 2, + 252, 1, 7, 72, 65, 78, 71, 85, 76, 32, 188, 4, 10, 73, 68, 69, 79, 71, + 82, 65, 80, 72, 32, 188, 7, 18, 75, 79, 82, 69, 65, 78, 32, 67, 72, 65, + 82, 65, 67, 84, 69, 82, 32, 79, 44, 7, 78, 85, 77, 66, 69, 82, 32, 202, + 201, 4, 68, 213, 169, 2, 2, 76, 65, 58, 102, 67, 110, 72, 30, 75, 66, 77, + 34, 78, 34, 80, 62, 82, 30, 83, 26, 84, 73, 5, 73, 69, 85, 78, 71, 10, + 34, 72, 33, 4, 73, 69, 85, 67, 4, 141, 3, 4, 73, 69, 85, 67, 7, 11, 32, + 4, 234, 145, 13, 65, 3, 85, 4, 197, 2, 3, 73, 69, 85, 8, 168, 2, 5, 72, + 73, 69, 85, 75, 13, 5, 73, 89, 69, 79, 75, 4, 245, 1, 4, 73, 69, 85, 77, + 4, 213, 1, 4, 73, 69, 85, 78, 8, 168, 1, 5, 72, 73, 69, 85, 80, 13, 4, + 73, 69, 85, 80, 4, 121, 4, 73, 69, 85, 76, 4, 93, 3, 73, 79, 83, 8, 56, + 5, 72, 73, 69, 85, 84, 13, 5, 73, 75, 69, 85, 84, 4, 11, 72, 5, 195, 140, + 13, 32, 72, 148, 1, 2, 65, 76, 30, 67, 74, 69, 82, 70, 112, 2, 76, 65, + 22, 77, 38, 78, 32, 2, 82, 69, 78, 83, 138, 2, 84, 50, 87, 158, 156, 11, + 72, 139, 82, 79, 2, 241, 133, 8, 2, 76, 73, 4, 32, 2, 79, 78, 211, 133, + 11, 65, 2, 185, 240, 7, 4, 71, 82, 65, 84, 6, 42, 78, 222, 163, 9, 65, + 163, 190, 3, 73, 2, 169, 4, 5, 84, 69, 82, 80, 82, 10, 58, 73, 160, 130, + 12, 5, 69, 83, 84, 73, 86, 239, 16, 79, 6, 208, 2, 3, 78, 65, 78, 186, + 242, 12, 82, 3, 86, 2, 195, 210, 12, 66, 4, 234, 182, 10, 69, 195, 132, + 2, 79, 4, 198, 241, 11, 73, 195, 1, 65, 8, 38, 83, 174, 250, 11, 80, 219, + 109, 65, 4, 134, 209, 6, 79, 167, 185, 6, 84, 18, 58, 69, 34, 79, 22, 80, + 34, 84, 50, 85, 167, 252, 11, 73, 4, 226, 181, 11, 86, 199, 82, 76, 2, + 243, 240, 7, 67, 2, 11, 69, 2, 179, 176, 4, 67, 4, 26, 85, 239, 216, 11, + 79, 2, 147, 247, 12, 68, 4, 26, 80, 175, 136, 13, 78, 2, 21, 3, 69, 82, + 86, 2, 135, 255, 11, 73, 6, 186, 152, 11, 72, 230, 159, 1, 69, 239, 48, + 87, 4, 246, 208, 9, 79, 227, 254, 1, 65, 4, 182, 153, 8, 32, 137, 158, 4, + 2, 74, 69, 22, 42, 69, 46, 70, 42, 78, 30, 83, 51, 84, 4, 216, 1, 3, 73, + 71, 72, 211, 240, 4, 76, 4, 160, 1, 2, 79, 85, 13, 2, 73, 70, 2, 133, 1, + 3, 73, 78, 69, 4, 34, 73, 73, 4, 69, 86, 69, 78, 2, 71, 88, 8, 34, 72, + 46, 87, 135, 181, 12, 69, 2, 11, 73, 2, 11, 82, 2, 131, 177, 11, 84, 4, + 11, 69, 4, 222, 151, 11, 78, 167, 112, 76, 18, 128, 1, 3, 73, 65, 76, + 216, 1, 3, 89, 32, 80, 168, 169, 8, 6, 78, 69, 82, 83, 72, 73, 169, 211, + 3, 6, 32, 65, 76, 84, 69, 82, 12, 62, 32, 193, 123, 10, 76, 89, 45, 82, + 69, 67, 89, 67, 76, 69, 10, 38, 68, 41, 5, 76, 73, 78, 69, 32, 2, 217, + 169, 4, 5, 73, 70, 70, 69, 82, 8, 222, 200, 3, 68, 226, 45, 70, 20, 4, + 66, 65, 67, 75, 199, 139, 9, 85, 2, 191, 237, 3, 79, 10, 98, 69, 52, 9, + 73, 86, 69, 45, 80, 85, 76, 76, 45, 89, 9, 80, 79, 82, 84, 32, 67, 79, + 78, 84, 4, 224, 226, 9, 4, 78, 71, 69, 82, 219, 138, 2, 68, 4, 40, 4, 68, + 79, 87, 78, 1, 2, 85, 80, 2, 185, 171, 5, 6, 45, 79, 85, 84, 80, 85, 2, + 231, 236, 11, 82, 114, 192, 1, 12, 71, 76, 79, 84, 84, 65, 76, 32, 83, + 84, 79, 80, 62, 76, 160, 3, 3, 82, 73, 83, 36, 14, 77, 73, 68, 45, 76, + 69, 86, 69, 76, 32, 84, 79, 78, 69, 57, 7, 83, 65, 78, 68, 72, 73, 32, 7, + 11, 32, 4, 206, 5, 70, 201, 238, 11, 4, 86, 65, 82, 73, 82, 72, 6, 69, + 84, 84, 69, 82, 32, 213, 2, 7, 79, 87, 45, 70, 65, 76, 76, 74, 206, 1, + 70, 166, 143, 7, 73, 2, 85, 214, 250, 1, 78, 242, 169, 3, 67, 2, 75, 2, + 80, 2, 84, 138, 69, 66, 2, 68, 2, 71, 2, 72, 2, 76, 2, 77, 2, 82, 2, 83, + 2, 86, 2, 90, 186, 2, 65, 2, 69, 3, 79, 20, 44, 5, 73, 78, 65, 76, 32, + 251, 250, 12, 65, 18, 222, 255, 10, 78, 154, 251, 1, 75, 2, 76, 2, 77, 2, + 80, 2, 84, 2, 87, 3, 89, 8, 157, 1, 5, 73, 78, 71, 32, 84, 7, 11, 32, 4, + 192, 1, 5, 76, 79, 78, 71, 32, 15, 70, 12, 66, 84, 73, 12, 71, 76, 79, + 84, 84, 65, 76, 32, 83, 84, 79, 80, 8, 21, 3, 79, 78, 69, 9, 11, 32, 6, + 32, 4, 76, 79, 78, 71, 27, 70, 5, 11, 32, 2, 11, 70, 2, 223, 195, 3, 73, + 2, 155, 176, 4, 82, 4, 250, 247, 12, 70, 3, 73, 80, 130, 1, 65, 122, 78, + 162, 1, 82, 162, 9, 83, 44, 3, 84, 82, 73, 130, 17, 68, 245, 136, 12, 9, + 79, 80, 76, 69, 32, 72, 85, 71, 71, 12, 50, 67, 50, 78, 190, 234, 6, 32, + 191, 139, 6, 82, 6, 182, 198, 11, 79, 202, 28, 69, 171, 147, 1, 72, 2, + 203, 225, 11, 85, 10, 118, 71, 20, 3, 83, 73, 86, 222, 178, 1, 84, 132, + 184, 4, 10, 32, 79, 86, 69, 82, 32, 83, 84, 65, 77, 167, 214, 4, 67, 2, + 151, 245, 11, 85, 2, 183, 183, 12, 69, 48, 186, 1, 32, 116, 10, 80, 69, + 78, 68, 73, 67, 85, 76, 65, 82, 42, 83, 210, 165, 7, 67, 252, 9, 10, 77, + 65, 78, 69, 78, 84, 32, 80, 65, 80, 201, 128, 2, 8, 70, 79, 82, 77, 73, + 78, 71, 32, 6, 34, 77, 30, 84, 227, 235, 11, 83, 2, 181, 241, 1, 2, 73, + 76, 2, 213, 137, 11, 8, 69, 78, 32, 84, 72, 79, 85, 83, 5, 129, 253, 8, + 5, 32, 87, 73, 84, 72, 32, 60, 2, 79, 78, 178, 75, 80, 221, 157, 11, 4, + 69, 86, 69, 82, 28, 38, 32, 149, 246, 9, 3, 65, 76, 32, 26, 216, 2, 10, + 68, 79, 73, 78, 71, 32, 67, 65, 82, 84, 32, 3, 73, 78, 32, 92, 5, 87, 73, + 84, 72, 32, 220, 213, 7, 4, 70, 82, 79, 87, 204, 13, 27, 82, 65, 73, 83, 73, 78, 71, 32, 66, 79, 84, 72, 32, 72, 65, 78, 68, 83, 32, 73, 78, 32, - 67, 69, 76, 69, 66, 128, 141, 4, 5, 67, 76, 73, 77, 66, 201, 45, 11, 66, - 79, 87, 73, 78, 71, 32, 68, 69, 69, 80, 2, 11, 87, 2, 179, 169, 3, 72, 4, - 76, 7, 83, 84, 69, 65, 77, 89, 32, 217, 211, 11, 6, 76, 79, 84, 85, 83, - 32, 2, 197, 148, 12, 2, 82, 79, 12, 154, 1, 66, 212, 134, 2, 3, 80, 79, - 85, 160, 157, 1, 2, 67, 82, 196, 214, 5, 6, 70, 79, 76, 68, 69, 68, 177, - 168, 2, 8, 72, 69, 65, 68, 83, 67, 65, 82, 4, 36, 3, 76, 79, 78, 159, - 159, 10, 65, 2, 11, 68, 2, 11, 32, 2, 11, 72, 2, 11, 65, 2, 203, 234, 11, - 73, 2, 21, 3, 76, 32, 67, 2, 161, 236, 10, 4, 79, 77, 80, 85, 4, 232, - 230, 8, 2, 69, 84, 251, 180, 2, 79, 2, 217, 227, 8, 2, 32, 68, 140, 2, - 66, 65, 180, 19, 9, 73, 76, 73, 80, 80, 73, 78, 69, 32, 47, 79, 204, 1, + 67, 69, 76, 69, 66, 128, 185, 4, 5, 67, 76, 73, 77, 66, 213, 45, 11, 66, + 79, 87, 73, 78, 71, 32, 68, 69, 69, 80, 2, 11, 87, 2, 251, 183, 3, 72, 4, + 164, 156, 12, 6, 76, 79, 84, 85, 83, 32, 253, 64, 9, 83, 84, 69, 65, 77, + 89, 32, 82, 79, 12, 154, 1, 66, 240, 145, 2, 3, 80, 79, 85, 220, 160, 1, + 2, 67, 82, 196, 250, 5, 6, 70, 79, 76, 68, 69, 68, 221, 189, 2, 8, 72, + 69, 65, 68, 83, 67, 65, 82, 4, 36, 3, 76, 79, 78, 171, 228, 10, 65, 2, + 11, 68, 2, 11, 32, 2, 11, 72, 2, 11, 65, 2, 219, 178, 12, 73, 4, 224, + 153, 9, 2, 69, 84, 195, 202, 2, 79, 2, 253, 137, 9, 2, 32, 68, 140, 2, + 66, 65, 184, 19, 9, 73, 76, 73, 80, 80, 73, 78, 69, 32, 47, 79, 204, 1, 108, 6, 71, 83, 45, 80, 65, 32, 189, 7, 16, 73, 83, 84, 79, 83, 32, 68, 73, 83, 67, 32, 83, 73, 71, 78, 32, 112, 100, 7, 76, 69, 84, 84, 69, 82, 32, 248, 4, 5, 77, 65, 82, 75, 32, 30, 83, 33, 4, 68, 79, 85, 66, 96, - 138, 2, 65, 138, 1, 67, 50, 68, 42, 83, 64, 5, 86, 79, 73, 67, 69, 174, - 204, 8, 71, 218, 134, 3, 78, 82, 84, 46, 75, 2, 80, 2, 90, 162, 7, 69, - 222, 61, 66, 2, 70, 2, 72, 2, 74, 2, 76, 2, 77, 2, 81, 2, 82, 2, 87, 2, + 138, 2, 65, 138, 1, 67, 50, 68, 42, 83, 64, 5, 86, 79, 73, 67, 69, 222, + 242, 8, 71, 246, 168, 3, 78, 82, 84, 46, 75, 2, 80, 2, 90, 162, 7, 69, + 234, 61, 66, 2, 70, 2, 72, 2, 74, 2, 76, 2, 77, 2, 81, 2, 82, 2, 87, 2, 88, 2, 89, 186, 2, 73, 2, 79, 3, 85, 7, 80, 8, 76, 84, 69, 82, 78, 65, - 84, 69, 21, 8, 83, 80, 73, 82, 65, 84, 69, 68, 2, 163, 154, 12, 32, 2, - 11, 32, 2, 167, 154, 12, 70, 6, 26, 65, 251, 153, 12, 72, 5, 159, 182, 8, - 78, 6, 226, 153, 12, 68, 2, 90, 187, 2, 65, 6, 236, 135, 8, 4, 77, 65, - 76, 76, 206, 145, 4, 72, 187, 2, 65, 4, 34, 68, 21, 4, 76, 69, 83, 83, 2, - 211, 163, 10, 32, 2, 243, 247, 8, 32, 4, 130, 212, 11, 68, 59, 83, 10, - 28, 3, 73, 78, 71, 31, 85, 2, 241, 205, 11, 2, 76, 69, 8, 58, 66, 225, - 206, 11, 8, 80, 69, 82, 70, 73, 88, 69, 68, 6, 229, 150, 8, 13, 74, 79, + 84, 69, 21, 8, 83, 80, 73, 82, 65, 84, 69, 68, 2, 251, 226, 12, 32, 2, + 11, 32, 2, 255, 226, 12, 70, 6, 26, 65, 211, 226, 12, 72, 5, 223, 208, 8, + 78, 6, 186, 226, 12, 68, 2, 90, 187, 2, 65, 6, 168, 164, 8, 4, 77, 65, + 76, 76, 234, 189, 4, 72, 187, 2, 65, 4, 34, 68, 21, 4, 76, 69, 83, 83, 2, + 167, 233, 10, 32, 2, 215, 171, 9, 32, 4, 206, 156, 12, 68, 59, 83, 10, + 28, 3, 73, 78, 71, 31, 85, 2, 189, 150, 12, 2, 76, 69, 8, 58, 66, 173, + 151, 12, 8, 80, 69, 82, 70, 73, 88, 69, 68, 6, 133, 179, 8, 13, 74, 79, 73, 78, 69, 68, 32, 76, 69, 84, 84, 69, 82, 92, 238, 1, 66, 146, 1, 67, - 172, 2, 2, 68, 79, 38, 71, 66, 72, 64, 2, 76, 73, 32, 2, 77, 65, 66, 80, - 162, 1, 82, 38, 83, 150, 1, 84, 70, 87, 168, 203, 5, 2, 70, 76, 172, 224, - 1, 2, 79, 88, 188, 193, 3, 2, 69, 65, 142, 7, 65, 135, 1, 86, 10, 52, 2, - 69, 69, 22, 79, 205, 40, 4, 85, 76, 76, 83, 5, 199, 142, 7, 72, 4, 32, 2, - 79, 77, 175, 150, 12, 87, 2, 11, 69, 2, 219, 189, 11, 82, 16, 34, 65, 86, - 72, 22, 76, 23, 79, 6, 194, 200, 5, 80, 200, 192, 3, 8, 82, 80, 69, 78, - 84, 82, 89, 32, 223, 140, 3, 84, 2, 255, 175, 2, 73, 2, 219, 218, 10, 85, - 6, 26, 76, 33, 2, 77, 66, 2, 11, 85, 2, 239, 196, 11, 77, 5, 37, 7, 73, - 78, 73, 78, 71, 32, 79, 2, 237, 172, 9, 5, 66, 76, 73, 81, 85, 4, 222, - 235, 10, 76, 175, 145, 1, 86, 4, 42, 82, 245, 182, 10, 4, 65, 85, 78, 84, - 2, 195, 220, 10, 65, 6, 42, 69, 198, 180, 7, 79, 155, 209, 2, 73, 2, 183, - 221, 11, 76, 4, 242, 128, 12, 76, 203, 17, 68, 4, 40, 2, 78, 65, 181, - 169, 9, 2, 84, 84, 2, 159, 233, 8, 67, 8, 52, 2, 69, 68, 50, 76, 237, - 136, 7, 3, 65, 80, 89, 2, 25, 4, 69, 83, 84, 82, 2, 211, 201, 10, 73, 4, - 184, 167, 4, 2, 85, 77, 229, 211, 2, 3, 65, 78, 69, 4, 230, 132, 10, 79, - 255, 250, 1, 65, 12, 108, 2, 72, 73, 154, 145, 11, 65, 154, 45, 76, 128, - 19, 3, 84, 82, 65, 165, 39, 7, 77, 65, 76, 76, 32, 65, 88, 4, 146, 170, - 2, 69, 151, 229, 9, 80, 6, 200, 165, 4, 5, 65, 84, 84, 79, 79, 250, 164, - 7, 73, 195, 36, 85, 4, 246, 196, 7, 79, 221, 187, 3, 5, 65, 86, 89, 32, - 66, 4, 218, 232, 1, 83, 25, 4, 68, 79, 85, 66, 60, 56, 8, 69, 78, 73, 67, - 73, 65, 78, 32, 183, 138, 10, 76, 58, 92, 7, 76, 69, 84, 84, 69, 82, 32, - 160, 3, 7, 78, 85, 77, 66, 69, 82, 32, 199, 248, 5, 87, 44, 234, 1, 65, - 34, 68, 22, 72, 22, 81, 22, 83, 58, 84, 202, 246, 1, 87, 150, 212, 5, 90, - 130, 157, 1, 89, 200, 188, 1, 2, 82, 79, 128, 92, 3, 71, 65, 77, 134, 8, - 75, 130, 1, 78, 132, 58, 3, 76, 65, 77, 138, 17, 66, 186, 30, 80, 171, 4, - 77, 4, 198, 137, 11, 76, 187, 49, 73, 2, 151, 172, 3, 69, 4, 171, 213, 6, - 69, 2, 255, 136, 11, 79, 6, 130, 253, 9, 65, 186, 141, 1, 72, 165, 124, - 2, 69, 77, 4, 214, 228, 11, 65, 191, 8, 69, 12, 186, 49, 84, 187, 151, 4, + 172, 2, 2, 68, 79, 38, 71, 66, 72, 64, 2, 76, 73, 32, 2, 77, 65, 70, 80, + 162, 1, 82, 38, 83, 150, 1, 84, 70, 87, 160, 223, 5, 2, 70, 76, 252, 232, + 1, 2, 79, 88, 196, 234, 3, 2, 69, 65, 250, 9, 65, 135, 1, 86, 10, 52, 2, + 69, 69, 22, 79, 133, 41, 4, 85, 76, 76, 83, 5, 247, 171, 7, 72, 4, 32, 2, + 79, 77, 135, 223, 12, 87, 2, 11, 69, 2, 163, 134, 12, 82, 16, 34, 65, 86, + 72, 22, 76, 23, 79, 6, 234, 218, 5, 80, 148, 230, 3, 8, 82, 80, 69, 78, + 84, 82, 89, 32, 195, 157, 3, 84, 2, 227, 191, 2, 73, 2, 243, 161, 11, 85, + 6, 26, 76, 33, 2, 77, 66, 2, 11, 85, 2, 187, 141, 12, 77, 5, 37, 7, 73, + 78, 73, 78, 71, 32, 79, 2, 209, 239, 9, 5, 66, 76, 73, 81, 85, 4, 154, + 179, 11, 76, 203, 146, 1, 86, 4, 42, 82, 201, 252, 10, 4, 65, 85, 78, 84, + 2, 239, 163, 11, 65, 6, 42, 69, 146, 209, 7, 79, 143, 250, 2, 73, 2, 131, + 166, 12, 76, 4, 202, 201, 12, 76, 203, 17, 68, 4, 34, 78, 161, 236, 9, 2, + 84, 84, 2, 11, 65, 2, 255, 156, 9, 67, 8, 52, 2, 69, 68, 50, 76, 153, + 166, 7, 3, 65, 80, 89, 2, 25, 4, 69, 83, 84, 82, 2, 219, 143, 11, 73, 4, + 176, 182, 4, 2, 85, 77, 177, 226, 2, 3, 65, 78, 69, 4, 166, 202, 10, 79, + 147, 254, 1, 65, 12, 108, 2, 72, 73, 222, 217, 11, 65, 158, 45, 76, 128, + 19, 3, 84, 82, 65, 177, 39, 7, 77, 65, 76, 76, 32, 65, 88, 4, 242, 185, + 2, 69, 139, 158, 10, 80, 6, 192, 180, 4, 5, 65, 84, 84, 79, 79, 202, 222, + 7, 73, 207, 36, 85, 4, 190, 225, 7, 79, 209, 231, 3, 5, 65, 86, 89, 32, + 66, 4, 174, 244, 1, 83, 25, 4, 68, 79, 85, 66, 60, 56, 8, 69, 78, 73, 67, + 73, 65, 78, 32, 251, 207, 10, 76, 58, 92, 7, 76, 69, 84, 84, 69, 82, 32, + 160, 3, 7, 78, 85, 77, 66, 69, 82, 32, 203, 150, 6, 87, 44, 234, 1, 65, + 34, 68, 22, 72, 22, 81, 22, 83, 58, 84, 158, 130, 2, 87, 138, 229, 5, 90, + 154, 180, 1, 89, 184, 206, 1, 2, 82, 79, 236, 94, 3, 71, 65, 77, 134, 8, + 75, 130, 1, 78, 144, 58, 3, 76, 65, 77, 138, 17, 66, 198, 30, 80, 171, 4, + 77, 4, 130, 210, 11, 76, 199, 49, 73, 2, 163, 187, 3, 69, 4, 147, 243, 6, + 69, 2, 187, 209, 11, 79, 6, 190, 194, 10, 65, 186, 144, 1, 72, 189, 124, + 2, 69, 77, 4, 170, 173, 12, 65, 191, 8, 69, 12, 238, 49, 84, 135, 166, 4, 79, 40, 104, 2, 67, 75, 66, 71, 62, 76, 90, 78, 178, 1, 83, 32, 7, 84, - 67, 72, 70, 79, 82, 75, 243, 132, 12, 69, 5, 17, 2, 85, 80, 2, 21, 3, 32, - 84, 82, 2, 155, 217, 10, 85, 7, 11, 32, 4, 26, 78, 179, 202, 11, 70, 2, - 183, 254, 10, 79, 6, 52, 4, 69, 32, 79, 70, 238, 90, 67, 247, 171, 11, - 76, 2, 11, 32, 2, 139, 198, 9, 80, 14, 68, 2, 67, 72, 46, 69, 186, 149, - 4, 87, 242, 232, 6, 75, 219, 98, 65, 4, 144, 140, 3, 2, 69, 68, 211, 235, - 7, 73, 4, 28, 2, 32, 68, 195, 71, 65, 2, 185, 241, 4, 2, 69, 67, 4, 174, - 241, 10, 67, 139, 1, 84, 5, 165, 186, 9, 10, 32, 87, 73, 84, 72, 32, 84, - 69, 69, 32, 218, 1, 38, 65, 218, 9, 85, 179, 250, 11, 68, 176, 1, 78, 67, + 67, 72, 70, 79, 82, 75, 199, 205, 12, 69, 5, 17, 2, 85, 80, 2, 21, 3, 32, + 84, 82, 2, 203, 160, 11, 85, 7, 11, 32, 4, 26, 78, 251, 146, 12, 70, 2, + 243, 198, 11, 79, 6, 52, 4, 69, 32, 79, 70, 246, 91, 67, 195, 243, 11, + 76, 2, 11, 32, 2, 255, 135, 10, 80, 14, 68, 2, 67, 72, 46, 69, 178, 164, + 4, 87, 182, 162, 7, 75, 243, 98, 65, 4, 160, 155, 3, 2, 69, 68, 255, 164, + 8, 73, 4, 28, 2, 32, 68, 255, 72, 65, 2, 221, 129, 5, 2, 69, 67, 4, 234, + 185, 11, 67, 139, 1, 84, 5, 153, 252, 9, 10, 32, 87, 73, 84, 72, 32, 84, + 69, 69, 32, 218, 1, 38, 65, 218, 9, 85, 135, 195, 12, 68, 176, 1, 78, 67, 128, 1, 12, 78, 67, 75, 32, 67, 79, 78, 83, 84, 65, 78, 84, 83, 89, 6, - 44, 5, 69, 32, 79, 70, 32, 179, 236, 10, 65, 4, 56, 6, 73, 78, 84, 69, - 82, 69, 189, 173, 11, 2, 87, 79, 2, 231, 235, 6, 83, 5, 45, 9, 32, 79, - 86, 69, 82, 32, 84, 87, 79, 2, 11, 32, 2, 159, 238, 11, 80, 166, 1, 96, - 9, 73, 78, 71, 32, 67, 65, 82, 68, 32, 149, 151, 4, 9, 71, 82, 79, 85, + 44, 5, 69, 32, 79, 70, 32, 239, 180, 11, 65, 4, 56, 6, 73, 78, 84, 69, + 82, 69, 129, 246, 11, 2, 87, 79, 2, 199, 136, 7, 83, 5, 45, 9, 32, 79, + 86, 69, 82, 32, 84, 87, 79, 2, 11, 32, 2, 243, 182, 12, 80, 166, 1, 96, + 9, 73, 78, 71, 32, 67, 65, 82, 68, 32, 141, 166, 4, 9, 71, 82, 79, 85, 78, 68, 32, 83, 76, 164, 1, 182, 1, 66, 44, 3, 82, 69, 68, 0, 5, 87, 72, 73, 84, 69, 42, 70, 74, 75, 38, 69, 34, 83, 36, 3, 81, 85, 69, 14, 84, 92, 2, 65, 67, 0, 3, 78, 73, 78, 13, 4, 74, 65, 67, 75, 4, 40, 4, 76, 65, - 67, 75, 203, 208, 10, 65, 2, 17, 2, 32, 74, 2, 203, 225, 1, 79, 18, 30, - 79, 249, 1, 2, 73, 86, 10, 128, 2, 2, 85, 82, 175, 243, 10, 79, 16, 34, + 67, 75, 251, 151, 11, 65, 2, 17, 2, 32, 74, 2, 159, 237, 1, 79, 18, 30, + 79, 249, 1, 2, 73, 86, 10, 128, 2, 2, 85, 82, 235, 187, 11, 79, 16, 34, 78, 185, 1, 3, 73, 78, 71, 8, 181, 1, 4, 73, 71, 72, 84, 16, 32, 2, 69, 86, 117, 2, 73, 88, 8, 91, 69, 66, 78, 69, 12, 3, 72, 82, 69, 12, 2, 87, 79, 161, 1, 5, 82, 85, 77, 80, 45, 8, 23, 78, 8, 11, 69, 8, 25, 4, 32, - 79, 70, 32, 8, 88, 3, 67, 76, 85, 20, 3, 83, 80, 65, 158, 207, 8, 72, - 137, 3, 5, 68, 73, 65, 77, 79, 2, 255, 189, 11, 66, 2, 219, 231, 10, 68, - 42, 90, 50, 250, 192, 9, 49, 214, 185, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, - 2, 56, 3, 57, 7, 202, 250, 11, 48, 3, 49, 41, 46, 83, 160, 4, 2, 84, 79, - 239, 189, 8, 78, 26, 52, 5, 32, 83, 73, 71, 78, 221, 223, 8, 2, 45, 77, - 25, 11, 32, 22, 64, 3, 73, 78, 32, 124, 5, 87, 73, 84, 72, 32, 219, 235, - 3, 65, 6, 34, 76, 22, 82, 247, 212, 10, 84, 2, 41, 2, 69, 70, 2, 21, 3, - 73, 71, 72, 2, 133, 169, 10, 6, 84, 32, 72, 65, 76, 70, 14, 162, 1, 68, - 34, 83, 204, 221, 9, 4, 84, 73, 76, 68, 140, 117, 5, 66, 76, 65, 67, 75, - 185, 35, 16, 67, 73, 82, 67, 85, 77, 70, 76, 69, 88, 32, 65, 67, 67, 69, - 78, 2, 11, 79, 2, 231, 207, 9, 84, 4, 54, 77, 165, 157, 5, 7, 85, 66, 83, - 67, 82, 73, 80, 2, 197, 231, 8, 3, 65, 76, 76, 11, 33, 6, 32, 70, 79, 82, - 77, 32, 8, 154, 136, 10, 70, 71, 84, 58, 232, 1, 5, 76, 73, 67, 69, 32, - 122, 80, 136, 1, 11, 82, 84, 65, 66, 76, 69, 32, 83, 84, 69, 82, 22, 83, - 158, 1, 84, 146, 1, 85, 196, 1, 4, 87, 69, 82, 32, 202, 231, 6, 79, 245, - 204, 4, 11, 67, 75, 69, 84, 32, 67, 65, 76, 67, 85, 76, 6, 52, 3, 67, 65, - 82, 141, 234, 3, 4, 79, 70, 70, 73, 5, 173, 221, 9, 11, 83, 32, 82, 69, - 86, 79, 76, 86, 73, 78, 71, 6, 76, 13, 32, 68, 73, 82, 69, 67, 84, 73, - 79, 78, 65, 76, 32, 187, 203, 1, 67, 4, 166, 117, 73, 185, 162, 6, 6, 70, - 79, 82, 77, 65, 84, 2, 191, 210, 11, 69, 12, 40, 2, 69, 73, 22, 84, 195, - 246, 4, 73, 2, 147, 161, 11, 68, 8, 36, 3, 65, 76, 32, 163, 228, 10, 66, - 6, 130, 202, 1, 72, 217, 180, 6, 4, 77, 65, 82, 75, 8, 66, 65, 210, 247, - 1, 32, 173, 238, 8, 6, 84, 69, 68, 32, 80, 76, 4, 26, 66, 179, 208, 11, - 84, 2, 29, 5, 76, 69, 32, 87, 65, 2, 135, 47, 84, 12, 108, 4, 76, 84, 82, - 89, 28, 7, 82, 73, 78, 71, 32, 76, 73, 28, 2, 84, 73, 250, 135, 10, 78, - 199, 228, 1, 67, 2, 165, 168, 11, 2, 32, 76, 2, 209, 156, 5, 2, 81, 85, - 4, 197, 137, 10, 2, 78, 71, 8, 24, 2, 79, 78, 47, 83, 4, 128, 218, 10, 4, - 45, 79, 70, 70, 15, 32, 4, 160, 213, 8, 3, 76, 69, 69, 219, 132, 2, 89, - 140, 1, 74, 69, 162, 10, 73, 234, 2, 79, 237, 139, 9, 6, 65, 89, 69, 82, - 32, 66, 102, 132, 1, 6, 71, 78, 65, 78, 84, 32, 66, 83, 156, 167, 5, 4, - 67, 69, 68, 69, 144, 193, 1, 5, 86, 73, 79, 85, 83, 157, 33, 2, 84, 90, - 6, 42, 87, 146, 151, 5, 80, 243, 139, 5, 77, 2, 235, 160, 7, 79, 70, 176, + 79, 70, 32, 8, 88, 3, 67, 76, 85, 20, 3, 83, 80, 65, 174, 130, 9, 72, + 137, 3, 5, 68, 73, 65, 77, 79, 2, 199, 134, 12, 66, 2, 151, 176, 11, 68, + 42, 90, 50, 238, 130, 10, 49, 182, 192, 2, 51, 2, 52, 2, 53, 2, 54, 2, + 55, 2, 56, 3, 57, 7, 158, 195, 12, 48, 3, 49, 41, 46, 83, 160, 4, 2, 84, + 79, 227, 240, 8, 78, 26, 52, 5, 32, 83, 73, 71, 78, 193, 147, 9, 2, 45, + 77, 25, 11, 32, 22, 64, 3, 73, 78, 32, 124, 5, 87, 73, 84, 72, 32, 207, + 250, 3, 65, 6, 34, 76, 22, 82, 183, 156, 11, 84, 2, 41, 2, 69, 70, 2, 21, + 3, 73, 71, 72, 2, 229, 238, 10, 6, 84, 32, 72, 65, 76, 70, 14, 162, 1, + 68, 34, 83, 200, 160, 10, 4, 84, 73, 76, 68, 208, 121, 5, 66, 76, 65, 67, + 75, 181, 36, 16, 67, 73, 82, 67, 85, 77, 70, 76, 69, 88, 32, 65, 67, 67, + 69, 78, 2, 11, 79, 2, 219, 145, 10, 84, 4, 54, 77, 161, 176, 5, 7, 85, + 66, 83, 67, 82, 73, 80, 2, 249, 154, 9, 3, 65, 76, 76, 11, 33, 6, 32, 70, + 79, 82, 77, 32, 8, 234, 205, 10, 70, 71, 84, 2, 253, 132, 12, 8, 32, 84, + 82, 65, 78, 83, 73, 83, 58, 232, 1, 5, 76, 73, 67, 69, 32, 122, 80, 136, + 1, 11, 82, 84, 65, 66, 76, 69, 32, 83, 84, 69, 82, 22, 83, 158, 1, 84, + 146, 1, 85, 196, 1, 4, 87, 69, 82, 32, 194, 132, 7, 79, 145, 248, 4, 11, + 67, 75, 69, 84, 32, 67, 65, 76, 67, 85, 76, 6, 52, 3, 67, 65, 82, 209, + 248, 3, 4, 79, 70, 70, 73, 5, 149, 162, 10, 11, 83, 32, 82, 69, 86, 79, + 76, 86, 73, 78, 71, 6, 76, 13, 32, 68, 73, 82, 69, 67, 84, 73, 79, 78, + 65, 76, 32, 219, 214, 1, 67, 4, 166, 127, 73, 205, 180, 6, 6, 70, 79, 82, + 77, 65, 84, 2, 223, 154, 12, 69, 12, 40, 2, 69, 73, 22, 84, 235, 135, 5, + 73, 2, 167, 233, 11, 68, 8, 36, 3, 65, 76, 32, 171, 172, 11, 66, 6, 162, + 213, 1, 72, 157, 197, 6, 4, 77, 65, 82, 75, 8, 66, 65, 250, 134, 2, 32, + 141, 167, 9, 6, 84, 69, 68, 32, 80, 76, 4, 26, 66, 211, 152, 12, 84, 2, + 29, 5, 76, 69, 32, 87, 65, 2, 143, 48, 84, 12, 108, 4, 76, 84, 82, 89, + 28, 7, 82, 73, 78, 71, 32, 76, 73, 28, 2, 84, 73, 150, 205, 10, 78, 203, + 231, 1, 67, 2, 185, 240, 11, 2, 32, 76, 2, 193, 174, 5, 2, 81, 85, 4, + 225, 206, 10, 2, 78, 71, 8, 24, 2, 79, 78, 47, 83, 4, 136, 162, 11, 4, + 45, 79, 70, 70, 15, 32, 4, 212, 136, 9, 3, 76, 69, 69, 175, 153, 2, 89, + 140, 1, 74, 69, 162, 10, 73, 234, 2, 79, 157, 206, 9, 6, 65, 89, 69, 82, + 32, 66, 102, 132, 1, 6, 71, 78, 65, 78, 84, 32, 66, 83, 224, 186, 5, 4, + 67, 69, 68, 69, 204, 202, 1, 5, 86, 73, 79, 85, 83, 177, 32, 2, 84, 90, + 6, 42, 87, 130, 179, 8, 80, 215, 181, 2, 77, 2, 255, 188, 7, 79, 70, 176, 1, 27, 69, 78, 84, 65, 84, 73, 79, 78, 32, 70, 79, 82, 77, 32, 70, 79, - 82, 32, 86, 69, 82, 84, 73, 67, 65, 76, 32, 241, 162, 8, 10, 67, 82, 73, + 82, 32, 86, 69, 82, 84, 73, 67, 65, 76, 32, 221, 200, 8, 10, 67, 82, 73, 80, 84, 73, 79, 78, 32, 84, 68, 198, 1, 67, 22, 69, 46, 72, 50, 73, 94, 76, 188, 1, 6, 82, 73, 71, 72, 84, 32, 192, 2, 6, 87, 65, 86, 89, 32, 76, - 202, 136, 4, 83, 192, 171, 4, 9, 84, 87, 79, 32, 68, 79, 84, 32, 76, 163, - 99, 81, 4, 203, 170, 2, 79, 6, 138, 252, 5, 88, 214, 194, 2, 77, 3, 78, - 2, 233, 215, 8, 7, 79, 82, 73, 90, 79, 78, 84, 4, 49, 10, 68, 69, 79, 71, - 82, 65, 80, 72, 73, 67, 4, 11, 32, 4, 206, 154, 9, 67, 35, 70, 22, 40, 4, - 69, 70, 84, 32, 191, 128, 10, 79, 20, 112, 6, 87, 72, 73, 84, 69, 32, - 142, 1, 66, 42, 68, 218, 93, 67, 146, 6, 65, 218, 164, 7, 80, 154, 2, 83, - 39, 84, 4, 162, 2, 67, 159, 93, 76, 22, 110, 66, 42, 68, 36, 6, 87, 72, - 73, 84, 69, 32, 182, 93, 67, 146, 6, 65, 218, 164, 7, 80, 154, 2, 83, 39, - 84, 2, 177, 94, 6, 76, 65, 67, 75, 32, 76, 2, 209, 99, 5, 79, 85, 66, 76, - 69, 6, 74, 67, 17, 14, 76, 69, 78, 84, 73, 67, 85, 76, 65, 82, 32, 66, - 82, 65, 2, 131, 93, 79, 4, 198, 143, 11, 67, 177, 29, 2, 75, 67, 2, 235, - 252, 9, 79, 22, 46, 78, 156, 1, 2, 86, 65, 171, 201, 11, 77, 10, 34, 84, - 173, 171, 6, 2, 67, 69, 6, 26, 32, 53, 2, 69, 82, 2, 21, 3, 83, 67, 82, - 2, 253, 235, 9, 2, 69, 69, 5, 17, 2, 32, 73, 2, 175, 144, 11, 67, 10, 60, - 5, 67, 89, 32, 77, 69, 29, 6, 84, 69, 32, 85, 83, 69, 2, 241, 132, 7, 2, - 83, 83, 8, 26, 32, 191, 175, 6, 45, 4, 146, 210, 5, 84, 223, 243, 4, 79, - 14, 130, 1, 74, 30, 80, 228, 32, 5, 72, 73, 66, 73, 84, 148, 176, 8, 6, - 66, 73, 78, 71, 32, 67, 133, 210, 1, 5, 83, 69, 82, 80, 73, 2, 181, 144, - 5, 2, 69, 67, 6, 64, 6, 79, 82, 84, 73, 79, 78, 197, 195, 10, 4, 69, 82, - 84, 89, 5, 179, 182, 5, 65, 58, 172, 1, 15, 70, 79, 85, 82, 32, 68, 79, - 84, 83, 32, 87, 73, 84, 72, 32, 32, 7, 76, 69, 84, 84, 69, 82, 32, 230, - 2, 78, 178, 31, 83, 1, 8, 84, 85, 82, 78, 69, 68, 32, 83, 4, 210, 157, - 10, 67, 187, 109, 68, 36, 166, 1, 65, 22, 68, 34, 76, 22, 77, 50, 87, - 238, 150, 4, 71, 90, 90, 34, 83, 66, 89, 154, 193, 1, 72, 234, 5, 75, - 130, 76, 66, 190, 173, 4, 78, 254, 1, 84, 203, 103, 80, 2, 207, 151, 4, - 76, 2, 11, 65, 2, 251, 171, 6, 76, 2, 235, 151, 4, 65, 2, 25, 4, 69, 77, - 45, 81, 2, 195, 223, 5, 79, 2, 37, 7, 65, 87, 45, 65, 89, 73, 78, 2, 141, - 194, 1, 2, 45, 82, 14, 33, 6, 85, 77, 66, 69, 82, 32, 14, 42, 84, 186, - 151, 4, 79, 187, 217, 2, 70, 8, 42, 87, 250, 234, 9, 72, 239, 156, 1, 69, - 4, 182, 237, 9, 69, 131, 234, 1, 79, 18, 252, 1, 4, 78, 67, 84, 85, 106, - 82, 56, 19, 84, 32, 76, 73, 84, 84, 69, 82, 32, 73, 78, 32, 73, 84, 83, - 32, 80, 76, 65, 170, 140, 1, 83, 140, 42, 20, 66, 76, 73, 67, 32, 65, 68, - 68, 82, 69, 83, 83, 32, 76, 79, 85, 68, 83, 80, 69, 194, 157, 10, 49, 3, - 50, 4, 80, 9, 83, 32, 69, 76, 69, 86, 65, 84, 85, 137, 150, 11, 5, 65, - 84, 73, 79, 78, 2, 135, 147, 11, 83, 4, 32, 2, 80, 76, 183, 189, 11, 83, - 2, 227, 204, 10, 69, 2, 11, 67, 2, 215, 192, 10, 69, 40, 98, 65, 148, 6, - 6, 69, 83, 84, 73, 79, 78, 214, 231, 5, 79, 253, 216, 4, 5, 73, 78, 67, - 85, 78, 30, 104, 2, 68, 82, 208, 4, 9, 84, 69, 82, 78, 73, 79, 78, 32, - 73, 48, 4, 82, 84, 69, 82, 223, 170, 10, 79, 24, 56, 4, 65, 78, 84, 32, - 253, 3, 5, 85, 80, 76, 69, 32, 20, 44, 6, 85, 80, 80, 69, 82, 32, 131, 2, - 76, 16, 56, 4, 76, 69, 70, 84, 249, 1, 5, 82, 73, 71, 72, 84, 11, 29, 5, - 32, 65, 78, 68, 32, 8, 108, 6, 76, 79, 87, 69, 82, 32, 53, 17, 85, 80, - 80, 69, 82, 32, 82, 73, 71, 72, 84, 32, 65, 78, 68, 32, 76, 4, 184, 1, 5, - 76, 69, 70, 84, 32, 195, 163, 11, 82, 4, 11, 79, 4, 11, 87, 4, 241, 163, - 11, 2, 69, 82, 7, 69, 15, 32, 65, 78, 68, 32, 76, 79, 87, 69, 82, 32, 76, - 69, 70, 84, 5, 11, 32, 2, 21, 3, 65, 78, 68, 2, 229, 255, 9, 6, 32, 76, - 79, 87, 69, 82, 4, 22, 73, 155, 37, 80, 2, 133, 163, 8, 7, 78, 84, 69, - 71, 82, 65, 76, 2, 17, 2, 32, 78, 2, 195, 136, 10, 79, 6, 34, 32, 185, - 165, 5, 2, 69, 68, 4, 198, 244, 4, 69, 211, 150, 6, 77, 152, 9, 114, 65, - 182, 7, 69, 224, 27, 6, 72, 73, 78, 79, 67, 69, 34, 73, 182, 78, 76, 46, - 79, 246, 18, 85, 151, 198, 10, 83, 60, 110, 67, 104, 2, 68, 73, 130, 1, - 73, 202, 4, 84, 160, 210, 7, 4, 66, 66, 73, 84, 130, 185, 3, 90, 223, 56, - 77, 6, 40, 4, 73, 78, 71, 32, 211, 184, 8, 67, 4, 140, 251, 9, 7, 77, 79, - 84, 79, 82, 67, 89, 203, 44, 67, 8, 66, 79, 245, 255, 7, 10, 67, 65, 76, - 32, 83, 89, 77, 66, 79, 76, 7, 200, 145, 6, 4, 65, 67, 84, 73, 197, 133, - 4, 2, 32, 66, 34, 60, 5, 76, 87, 65, 89, 32, 46, 78, 21, 4, 83, 69, 68, - 32, 4, 228, 189, 8, 2, 84, 82, 247, 231, 1, 67, 5, 203, 173, 10, 66, 26, - 156, 1, 3, 68, 79, 84, 34, 73, 48, 4, 72, 65, 78, 68, 182, 1, 77, 198, - 139, 7, 70, 186, 237, 1, 67, 226, 170, 1, 83, 197, 18, 7, 66, 65, 67, 75, - 32, 79, 70, 5, 29, 5, 84, 69, 68, 32, 73, 2, 165, 39, 8, 78, 84, 69, 82, - 80, 79, 76, 65, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 130, 27, 70, 201, - 176, 2, 28, 80, 65, 82, 84, 32, 66, 69, 84, 87, 69, 69, 78, 32, 77, 73, - 68, 68, 76, 69, 32, 65, 78, 68, 32, 82, 73, 78, 71, 6, 182, 189, 10, 67, - 2, 68, 3, 82, 5, 135, 165, 11, 73, 248, 1, 226, 1, 67, 232, 3, 2, 68, 32, - 64, 2, 71, 73, 144, 1, 5, 74, 65, 78, 71, 32, 254, 4, 76, 32, 8, 77, 73, - 78, 68, 69, 82, 32, 82, 34, 80, 106, 83, 136, 1, 5, 84, 85, 82, 78, 32, - 42, 86, 225, 184, 1, 5, 70, 69, 82, 69, 78, 24, 114, 69, 32, 3, 89, 67, - 76, 158, 176, 5, 79, 181, 237, 3, 13, 82, 69, 65, 84, 73, 79, 78, 65, 76, - 32, 86, 69, 72, 2, 11, 73, 2, 211, 164, 11, 80, 18, 78, 69, 53, 15, 73, - 78, 71, 32, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 2, 29, 5, 68, 32, - 80, 65, 80, 2, 239, 223, 9, 69, 16, 100, 5, 84, 89, 80, 69, 45, 157, 237, - 4, 14, 71, 69, 78, 69, 82, 73, 67, 32, 77, 65, 84, 69, 82, 73, 14, 58, - 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 2, 229, 177, 5, 6, 32, 80, - 76, 65, 83, 84, 4, 42, 65, 229, 233, 4, 4, 71, 73, 70, 84, 2, 143, 173, - 3, 80, 54, 120, 4, 83, 84, 69, 82, 165, 212, 5, 20, 79, 78, 65, 76, 32, - 73, 78, 68, 73, 67, 65, 84, 79, 82, 32, 83, 89, 77, 66, 79, 2, 135, 215, - 9, 69, 74, 128, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, - 71, 78, 32, 44, 7, 76, 69, 84, 84, 69, 82, 32, 150, 2, 83, 35, 86, 8, - 174, 195, 9, 78, 150, 248, 1, 72, 3, 82, 46, 154, 1, 77, 34, 78, 166, - 183, 11, 66, 2, 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, 80, 2, - 82, 2, 83, 2, 84, 2, 87, 2, 89, 187, 2, 65, 4, 194, 183, 11, 66, 187, 2, - 65, 12, 54, 89, 158, 234, 7, 71, 210, 204, 3, 68, 187, 2, 65, 4, 234, - 182, 11, 74, 187, 2, 65, 2, 11, 69, 2, 223, 207, 10, 67, 18, 64, 10, 79, - 87, 69, 76, 32, 83, 73, 71, 78, 32, 171, 134, 9, 73, 16, 54, 69, 190, - 247, 10, 65, 174, 64, 73, 2, 79, 3, 85, 7, 230, 183, 11, 65, 3, 85, 2, - 137, 167, 10, 3, 73, 69, 86, 2, 201, 231, 10, 3, 73, 66, 66, 2, 41, 8, - 76, 65, 67, 69, 77, 69, 78, 84, 2, 17, 2, 32, 67, 2, 253, 255, 9, 5, 72, - 65, 82, 65, 67, 8, 32, 2, 84, 82, 187, 217, 6, 80, 6, 168, 134, 6, 16, - 73, 67, 84, 69, 68, 32, 76, 69, 70, 84, 32, 69, 78, 84, 82, 89, 243, 158, - 5, 79, 6, 162, 162, 10, 83, 142, 104, 76, 31, 82, 70, 64, 4, 69, 82, 83, - 69, 173, 228, 3, 6, 79, 76, 86, 73, 78, 71, 68, 30, 32, 169, 3, 2, 68, - 32, 20, 184, 1, 6, 67, 72, 69, 67, 75, 69, 28, 2, 76, 73, 108, 7, 83, 79, - 76, 73, 68, 85, 83, 196, 193, 7, 16, 84, 73, 76, 68, 69, 32, 79, 80, 69, - 82, 65, 84, 79, 82, 32, 65, 147, 203, 2, 73, 2, 221, 155, 10, 2, 82, 32, - 4, 136, 194, 3, 18, 71, 72, 84, 32, 70, 79, 85, 82, 32, 80, 79, 73, 78, - 84, 69, 68, 32, 80, 239, 221, 1, 78, 9, 11, 32, 6, 200, 135, 5, 9, 80, - 82, 69, 67, 69, 68, 73, 78, 71, 226, 255, 2, 79, 167, 140, 1, 87, 48, + 174, 151, 4, 83, 184, 207, 4, 9, 84, 87, 79, 32, 68, 79, 84, 32, 76, 135, + 114, 81, 4, 167, 185, 2, 79, 6, 222, 153, 6, 88, 174, 216, 2, 77, 3, 78, + 2, 229, 138, 9, 7, 79, 82, 73, 90, 79, 78, 84, 4, 49, 10, 68, 69, 79, 71, + 82, 65, 80, 72, 73, 67, 4, 11, 32, 4, 142, 220, 9, 67, 35, 70, 22, 40, 4, + 69, 70, 84, 32, 219, 197, 10, 79, 20, 112, 6, 87, 72, 73, 84, 69, 32, + 142, 1, 66, 42, 68, 198, 99, 67, 166, 7, 65, 138, 190, 7, 80, 238, 7, 83, + 39, 84, 4, 162, 2, 67, 139, 99, 76, 22, 110, 66, 42, 68, 36, 6, 87, 72, + 73, 84, 69, 32, 162, 99, 67, 166, 7, 65, 138, 190, 7, 80, 238, 7, 83, 39, + 84, 2, 157, 100, 6, 76, 65, 67, 75, 32, 76, 2, 209, 106, 5, 79, 85, 66, + 76, 69, 6, 74, 67, 17, 14, 76, 69, 78, 84, 73, 67, 85, 76, 65, 82, 32, + 66, 82, 65, 2, 239, 98, 79, 4, 218, 215, 11, 67, 177, 29, 2, 75, 67, 2, + 135, 194, 10, 79, 22, 46, 78, 156, 1, 2, 86, 65, 203, 145, 12, 77, 10, + 34, 84, 225, 200, 6, 2, 67, 69, 6, 26, 32, 53, 2, 69, 82, 2, 21, 3, 83, + 67, 82, 2, 153, 177, 10, 2, 69, 69, 5, 17, 2, 32, 73, 2, 195, 216, 11, + 67, 10, 60, 5, 67, 89, 32, 77, 69, 29, 6, 84, 69, 32, 85, 83, 69, 2, 133, + 161, 7, 2, 83, 83, 8, 26, 32, 159, 166, 8, 45, 4, 226, 239, 5, 84, 151, + 158, 5, 79, 14, 130, 1, 74, 30, 80, 236, 33, 5, 72, 73, 66, 73, 84, 200, + 230, 8, 6, 66, 73, 78, 71, 32, 67, 169, 225, 1, 5, 83, 69, 82, 80, 73, 2, + 165, 162, 5, 2, 69, 67, 6, 64, 6, 79, 82, 84, 73, 79, 78, 205, 139, 11, + 4, 69, 82, 84, 89, 5, 247, 209, 5, 65, 58, 172, 1, 15, 70, 79, 85, 82, + 32, 68, 79, 84, 83, 32, 87, 73, 84, 72, 32, 32, 7, 76, 69, 84, 84, 69, + 82, 32, 230, 2, 78, 134, 32, 83, 1, 8, 84, 85, 82, 78, 69, 68, 32, 83, 4, + 190, 226, 10, 67, 227, 112, 68, 36, 166, 1, 65, 22, 68, 34, 76, 22, 77, + 50, 87, 186, 165, 4, 71, 90, 90, 34, 83, 66, 89, 158, 208, 1, 72, 234, 5, + 75, 198, 75, 66, 178, 216, 4, 78, 134, 2, 84, 219, 103, 80, 2, 155, 166, + 4, 76, 2, 11, 65, 2, 143, 201, 6, 76, 2, 183, 166, 4, 65, 2, 25, 4, 69, + 77, 45, 81, 2, 147, 253, 5, 79, 2, 37, 7, 65, 87, 45, 65, 89, 73, 78, 2, + 173, 205, 1, 2, 45, 82, 14, 33, 6, 85, 77, 66, 69, 82, 32, 14, 42, 84, + 134, 166, 4, 79, 131, 231, 2, 70, 8, 42, 87, 150, 176, 10, 72, 231, 159, + 1, 69, 4, 210, 178, 10, 69, 135, 237, 1, 79, 18, 252, 1, 4, 78, 67, 84, + 85, 94, 82, 56, 19, 84, 32, 76, 73, 84, 84, 69, 82, 32, 73, 78, 32, 73, + 84, 83, 32, 80, 76, 65, 214, 151, 1, 83, 140, 42, 20, 66, 76, 73, 67, 32, + 65, 68, 68, 82, 69, 83, 83, 32, 76, 79, 85, 68, 83, 80, 69, 194, 218, 10, + 49, 3, 50, 4, 136, 206, 10, 9, 83, 32, 69, 76, 69, 86, 65, 84, 85, 229, + 144, 1, 5, 65, 84, 73, 79, 78, 4, 32, 2, 80, 76, 227, 133, 12, 83, 2, + 247, 148, 11, 69, 2, 11, 67, 2, 235, 136, 11, 69, 40, 98, 65, 180, 6, 6, + 69, 83, 84, 73, 79, 78, 162, 133, 6, 79, 165, 131, 5, 5, 73, 78, 67, 85, + 78, 30, 104, 2, 68, 82, 240, 4, 9, 84, 69, 82, 78, 73, 79, 78, 32, 73, + 48, 4, 82, 84, 69, 82, 219, 243, 10, 79, 24, 56, 4, 65, 78, 84, 32, 157, + 4, 5, 85, 80, 76, 69, 32, 20, 44, 6, 85, 80, 80, 69, 82, 32, 131, 2, 76, + 16, 56, 4, 76, 69, 70, 84, 249, 1, 5, 82, 73, 71, 72, 84, 11, 29, 5, 32, + 65, 78, 68, 32, 8, 108, 6, 76, 79, 87, 69, 82, 32, 53, 17, 85, 80, 80, + 69, 82, 32, 82, 73, 71, 72, 84, 32, 65, 78, 68, 32, 76, 4, 192, 1, 5, 76, + 69, 70, 84, 32, 231, 235, 11, 82, 4, 11, 79, 4, 11, 87, 4, 157, 236, 11, + 2, 69, 82, 7, 65, 14, 32, 65, 78, 68, 32, 76, 79, 87, 69, 82, 32, 76, 69, + 70, 4, 11, 84, 5, 11, 32, 2, 21, 3, 65, 78, 68, 2, 17, 2, 32, 76, 2, 17, + 2, 79, 87, 2, 237, 196, 10, 2, 69, 82, 4, 22, 73, 223, 37, 80, 2, 205, + 213, 8, 7, 78, 84, 69, 71, 82, 65, 76, 2, 17, 2, 32, 78, 2, 131, 206, 10, + 79, 6, 34, 32, 233, 192, 5, 2, 69, 68, 4, 250, 134, 5, 69, 159, 204, 6, + 77, 220, 9, 114, 65, 142, 8, 69, 204, 27, 6, 72, 73, 78, 79, 67, 69, 34, + 73, 174, 88, 76, 46, 79, 198, 19, 85, 151, 131, 11, 83, 62, 110, 67, 104, + 2, 68, 73, 130, 1, 73, 162, 5, 84, 152, 237, 7, 4, 66, 66, 73, 84, 178, + 229, 3, 90, 235, 56, 77, 6, 40, 4, 73, 78, 71, 32, 187, 235, 8, 67, 4, + 164, 192, 10, 7, 77, 79, 84, 79, 82, 67, 89, 175, 48, 67, 8, 66, 79, 205, + 165, 8, 10, 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 7, 232, 174, 6, 4, + 65, 67, 84, 73, 129, 175, 4, 2, 32, 66, 36, 60, 5, 76, 87, 65, 89, 32, + 46, 78, 21, 4, 83, 69, 68, 32, 4, 140, 245, 8, 2, 84, 82, 203, 249, 1, + 67, 5, 191, 245, 10, 66, 28, 152, 1, 3, 68, 79, 84, 34, 73, 48, 4, 72, + 65, 78, 68, 182, 1, 77, 38, 83, 166, 167, 7, 70, 230, 146, 2, 67, 237, + 195, 1, 7, 66, 65, 67, 75, 32, 79, 70, 5, 29, 5, 84, 69, 68, 32, 73, 2, + 193, 40, 8, 78, 84, 69, 82, 80, 79, 76, 65, 7, 33, 6, 32, 87, 73, 84, 72, + 32, 4, 198, 27, 70, 209, 190, 2, 28, 80, 65, 82, 84, 32, 66, 69, 84, 87, + 69, 69, 78, 32, 77, 73, 68, 68, 76, 69, 32, 65, 78, 68, 32, 82, 73, 78, + 71, 6, 174, 133, 11, 67, 2, 68, 3, 82, 4, 60, 9, 77, 65, 76, 76, 32, 76, + 69, 70, 84, 255, 234, 10, 81, 2, 129, 145, 8, 2, 32, 83, 5, 187, 236, 11, + 73, 250, 1, 226, 1, 67, 132, 4, 2, 68, 32, 64, 2, 71, 73, 144, 1, 5, 74, + 65, 78, 71, 32, 202, 4, 76, 32, 8, 77, 73, 78, 68, 69, 82, 32, 82, 34, + 80, 106, 83, 136, 1, 5, 84, 85, 82, 78, 32, 42, 86, 193, 199, 1, 5, 70, + 69, 82, 69, 78, 26, 114, 69, 60, 3, 89, 67, 76, 230, 204, 5, 79, 149, + 145, 4, 13, 82, 69, 65, 84, 73, 79, 78, 65, 76, 32, 86, 69, 72, 4, 38, + 73, 181, 182, 10, 3, 80, 84, 65, 2, 235, 235, 11, 80, 18, 78, 69, 53, 15, + 73, 78, 71, 32, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 2, 29, 5, 68, + 32, 80, 65, 80, 2, 131, 164, 10, 69, 16, 100, 5, 84, 89, 80, 69, 45, 133, + 254, 4, 14, 71, 69, 78, 69, 82, 73, 67, 32, 77, 65, 84, 69, 82, 73, 14, + 58, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 2, 173, 206, 5, 6, 32, + 80, 76, 65, 83, 84, 4, 42, 65, 165, 251, 4, 4, 71, 73, 70, 84, 2, 199, + 186, 3, 80, 54, 120, 4, 83, 84, 69, 82, 249, 240, 5, 20, 79, 78, 65, 76, + 32, 73, 78, 68, 73, 67, 65, 84, 79, 82, 32, 83, 89, 77, 66, 79, 2, 155, + 155, 10, 69, 74, 128, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, + 73, 71, 78, 32, 44, 7, 76, 69, 84, 84, 69, 82, 32, 226, 1, 83, 35, 86, 8, + 194, 135, 10, 78, 154, 251, 1, 72, 3, 82, 46, 162, 1, 78, 238, 243, 7, + 77, 234, 138, 4, 66, 2, 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, + 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, 89, 187, 2, 65, 12, 206, 244, 7, 89, + 142, 27, 71, 250, 238, 3, 68, 187, 2, 65, 2, 11, 69, 2, 155, 151, 11, 67, + 18, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 151, 199, 9, 73, 16, + 54, 69, 254, 190, 11, 65, 186, 64, 73, 2, 79, 3, 85, 7, 178, 255, 11, 65, + 3, 85, 2, 189, 238, 10, 3, 73, 69, 86, 2, 137, 175, 11, 3, 73, 66, 66, 2, + 41, 8, 76, 65, 67, 69, 77, 69, 78, 84, 2, 17, 2, 32, 67, 2, 157, 198, 10, + 5, 72, 65, 82, 65, 67, 8, 32, 2, 84, 82, 251, 244, 6, 80, 6, 180, 252, 7, + 16, 73, 67, 84, 69, 68, 32, 76, 69, 70, 84, 32, 69, 78, 84, 82, 89, 179, + 240, 3, 79, 6, 214, 233, 10, 83, 166, 104, 76, 31, 82, 70, 64, 4, 69, 82, + 83, 69, 165, 242, 3, 6, 79, 76, 86, 73, 78, 71, 68, 30, 32, 169, 3, 2, + 68, 32, 20, 184, 1, 6, 67, 72, 69, 67, 75, 69, 28, 2, 76, 73, 108, 7, 83, + 79, 76, 73, 68, 85, 83, 212, 220, 7, 16, 84, 73, 76, 68, 69, 32, 79, 80, + 69, 82, 65, 84, 79, 82, 32, 65, 179, 246, 2, 73, 2, 145, 227, 10, 2, 82, + 32, 4, 248, 207, 3, 18, 71, 72, 84, 32, 70, 79, 85, 82, 32, 80, 79, 73, + 78, 84, 69, 68, 32, 80, 251, 236, 1, 78, 9, 11, 32, 6, 184, 162, 5, 9, + 80, 82, 69, 67, 69, 68, 73, 78, 71, 250, 150, 3, 79, 251, 154, 1, 87, 48, 232, 2, 5, 65, 78, 71, 76, 69, 20, 7, 68, 79, 85, 66, 76, 69, 32, 118, - 78, 20, 5, 70, 79, 82, 75, 69, 70, 80, 82, 82, 214, 1, 83, 102, 84, 212, - 201, 6, 30, 72, 65, 78, 68, 32, 87, 73, 84, 72, 32, 77, 73, 68, 68, 76, - 69, 32, 70, 73, 78, 71, 69, 82, 32, 69, 88, 84, 69, 78, 68, 158, 148, 2, - 67, 114, 81, 212, 61, 3, 86, 73, 67, 157, 36, 6, 69, 77, 80, 84, 89, 32, - 5, 159, 213, 3, 32, 6, 40, 3, 80, 82, 73, 41, 3, 83, 84, 82, 4, 17, 2, - 77, 69, 5, 195, 166, 3, 32, 2, 29, 5, 79, 75, 69, 32, 78, 2, 147, 150, 6, - 79, 2, 49, 10, 68, 32, 80, 65, 82, 65, 71, 82, 65, 80, 2, 175, 4, 72, 4, - 36, 3, 73, 76, 67, 151, 148, 10, 82, 2, 11, 82, 2, 253, 164, 10, 2, 79, + 78, 20, 5, 70, 79, 82, 75, 69, 70, 80, 82, 82, 214, 1, 83, 106, 84, 144, + 229, 6, 30, 72, 65, 78, 68, 32, 87, 73, 84, 72, 32, 77, 73, 68, 68, 76, + 69, 32, 70, 73, 78, 71, 69, 82, 32, 69, 88, 84, 69, 78, 68, 202, 185, 2, + 67, 114, 81, 156, 65, 3, 86, 73, 67, 177, 36, 6, 69, 77, 80, 84, 89, 32, + 5, 231, 227, 3, 32, 6, 40, 3, 80, 82, 73, 41, 3, 83, 84, 82, 4, 17, 2, + 77, 69, 5, 179, 180, 3, 32, 2, 29, 5, 79, 75, 69, 32, 78, 2, 235, 177, 6, + 79, 2, 49, 10, 68, 32, 80, 65, 82, 65, 71, 82, 65, 80, 2, 179, 4, 72, 4, + 36, 3, 73, 76, 67, 203, 219, 10, 82, 2, 11, 82, 2, 177, 236, 10, 2, 79, 87, 6, 156, 1, 17, 65, 73, 83, 69, 68, 32, 72, 65, 78, 68, 32, 87, 73, - 84, 72, 32, 70, 252, 96, 8, 79, 84, 65, 84, 69, 68, 32, 70, 229, 149, 6, - 4, 73, 71, 72, 84, 2, 185, 102, 9, 73, 78, 71, 69, 82, 83, 32, 83, 80, 4, - 132, 124, 17, 65, 78, 83, 45, 83, 69, 82, 73, 70, 32, 67, 65, 80, 73, 84, - 65, 76, 151, 248, 6, 69, 10, 88, 4, 73, 76, 68, 69, 28, 7, 82, 73, 80, - 76, 69, 32, 80, 173, 251, 6, 3, 72, 85, 77, 5, 189, 214, 4, 2, 32, 69, 2, - 187, 144, 10, 82, 2, 11, 82, 2, 171, 234, 10, 79, 213, 3, 228, 1, 4, 66, - 66, 79, 78, 164, 1, 3, 67, 69, 32, 60, 3, 71, 72, 84, 236, 71, 2, 78, 71, - 252, 1, 23, 83, 73, 78, 71, 32, 68, 73, 65, 71, 79, 78, 65, 76, 32, 67, - 82, 79, 83, 83, 73, 78, 71, 32, 134, 195, 5, 65, 143, 246, 3, 70, 19, 37, - 7, 32, 65, 82, 82, 79, 87, 32, 16, 80, 3, 76, 69, 70, 0, 4, 82, 73, 71, - 72, 22, 85, 213, 249, 10, 3, 68, 79, 87, 4, 175, 234, 1, 84, 4, 223, 249, - 10, 80, 4, 26, 67, 131, 160, 9, 66, 2, 133, 135, 1, 3, 82, 65, 67, 162, - 3, 110, 32, 158, 31, 45, 132, 8, 11, 72, 65, 78, 68, 32, 73, 78, 84, 69, - 82, 73, 29, 6, 87, 65, 82, 68, 83, 32, 176, 1, 198, 1, 65, 132, 6, 2, 66, - 76, 62, 67, 184, 1, 2, 68, 79, 242, 1, 70, 60, 2, 72, 65, 212, 2, 2, 76, - 79, 66, 78, 82, 79, 122, 80, 168, 1, 2, 82, 65, 58, 83, 154, 6, 84, 194, - 3, 86, 143, 1, 87, 28, 22, 78, 163, 4, 82, 22, 24, 2, 68, 32, 39, 71, 4, - 214, 41, 76, 21, 3, 85, 80, 80, 18, 26, 69, 17, 2, 76, 69, 2, 175, 23, - 82, 17, 11, 32, 14, 154, 1, 66, 44, 8, 68, 79, 84, 84, 69, 68, 32, 83, 2, - 83, 112, 5, 87, 73, 84, 72, 32, 189, 253, 9, 12, 86, 65, 82, 73, 65, 78, - 84, 32, 87, 73, 84, 72, 4, 133, 252, 9, 6, 82, 65, 67, 75, 69, 84, 2, 37, - 7, 85, 66, 83, 84, 73, 84, 85, 2, 25, 4, 84, 73, 79, 78, 2, 21, 3, 32, - 77, 65, 2, 243, 128, 1, 82, 4, 68, 11, 68, 79, 87, 78, 87, 65, 82, 68, - 83, 32, 90, 203, 171, 8, 65, 2, 141, 131, 10, 5, 73, 71, 90, 65, 71, 6, - 18, 67, 79, 82, 2, 41, 8, 32, 71, 82, 69, 65, 84, 69, 82, 2, 189, 20, 4, - 45, 84, 72, 65, 4, 41, 8, 79, 87, 32, 87, 73, 84, 72, 32, 4, 132, 191, 7, - 7, 67, 73, 82, 67, 76, 69, 68, 207, 141, 2, 83, 4, 25, 4, 65, 67, 75, 32, - 4, 194, 22, 76, 243, 172, 7, 84, 12, 38, 85, 230, 21, 79, 251, 222, 2, - 69, 8, 53, 11, 82, 76, 89, 32, 66, 82, 65, 67, 75, 69, 84, 9, 11, 32, 6, - 44, 4, 77, 73, 68, 68, 138, 8, 76, 23, 85, 2, 129, 229, 9, 2, 76, 69, 12, - 38, 84, 41, 5, 85, 66, 76, 69, 32, 2, 173, 14, 6, 84, 69, 68, 32, 83, 85, - 10, 50, 65, 94, 87, 174, 145, 3, 81, 151, 172, 4, 80, 4, 206, 25, 78, - 217, 149, 3, 15, 82, 82, 79, 87, 32, 87, 73, 84, 72, 32, 82, 79, 85, 78, - 68, 2, 203, 19, 73, 6, 26, 73, 227, 241, 2, 76, 4, 146, 146, 8, 86, 163, - 88, 83, 14, 32, 3, 76, 70, 32, 223, 1, 78, 12, 42, 66, 90, 70, 34, 82, - 139, 166, 8, 77, 6, 11, 76, 6, 40, 4, 65, 67, 75, 32, 231, 231, 9, 79, 4, - 166, 199, 9, 67, 167, 11, 83, 2, 11, 79, 2, 147, 242, 8, 76, 2, 33, 6, - 85, 78, 78, 73, 78, 71, 2, 131, 204, 6, 32, 2, 149, 145, 1, 16, 68, 32, - 84, 69, 76, 69, 80, 72, 79, 78, 69, 32, 82, 69, 67, 69, 2, 241, 193, 10, - 11, 87, 32, 80, 65, 82, 65, 80, 72, 82, 65, 83, 2, 197, 4, 16, 79, 82, - 77, 65, 76, 32, 70, 65, 67, 84, 79, 82, 32, 83, 69, 77, 8, 74, 85, 214, - 150, 8, 78, 237, 178, 1, 8, 80, 69, 78, 32, 83, 81, 85, 65, 2, 201, 147, - 10, 6, 84, 69, 82, 32, 74, 79, 8, 49, 10, 65, 82, 69, 78, 84, 72, 69, 83, - 73, 83, 9, 11, 32, 6, 34, 76, 22, 85, 147, 248, 8, 69, 2, 37, 2, 79, 87, - 2, 17, 2, 80, 80, 2, 169, 223, 3, 2, 69, 82, 2, 249, 8, 10, 73, 83, 69, - 68, 32, 79, 77, 73, 83, 83, 40, 62, 45, 70, 69, 78, 73, 68, 2, 80, 69, - 126, 81, 227, 2, 85, 2, 153, 208, 7, 12, 83, 72, 65, 80, 69, 68, 32, 66, - 65, 71, 32, 68, 4, 26, 77, 199, 160, 8, 86, 2, 205, 128, 10, 7, 73, 68, - 73, 82, 69, 67, 84, 4, 134, 136, 3, 78, 193, 180, 7, 8, 68, 69, 87, 65, - 89, 83, 32, 85, 8, 32, 4, 65, 75, 69, 82, 67, 69, 7, 33, 6, 32, 87, 73, - 84, 72, 32, 4, 186, 237, 3, 79, 75, 84, 2, 201, 4, 2, 67, 72, 20, 57, 12, - 85, 65, 82, 69, 32, 66, 82, 65, 67, 75, 69, 84, 21, 11, 32, 18, 84, 3, - 76, 79, 87, 0, 3, 85, 80, 80, 28, 5, 87, 73, 84, 72, 32, 223, 242, 8, 69, - 2, 133, 216, 3, 2, 69, 82, 12, 96, 8, 84, 73, 67, 75, 32, 73, 78, 32, - 162, 5, 81, 214, 189, 7, 85, 158, 97, 68, 195, 183, 2, 83, 4, 132, 215, - 3, 6, 66, 79, 84, 84, 79, 77, 1, 3, 84, 79, 80, 2, 197, 2, 6, 66, 83, 84, - 73, 84, 85, 20, 54, 72, 162, 1, 82, 146, 177, 7, 79, 179, 169, 2, 65, 8, - 54, 73, 60, 5, 79, 85, 71, 72, 84, 131, 155, 8, 82, 2, 201, 197, 1, 10, - 82, 68, 32, 87, 72, 73, 84, 69, 32, 82, 2, 11, 32, 2, 205, 250, 2, 3, 66, - 85, 66, 8, 34, 65, 89, 4, 73, 65, 78, 71, 2, 33, 6, 78, 83, 80, 79, 83, - 73, 2, 11, 84, 2, 17, 2, 73, 79, 2, 191, 181, 10, 78, 6, 32, 2, 76, 69, - 183, 154, 8, 85, 5, 41, 8, 32, 65, 66, 79, 86, 69, 32, 76, 2, 161, 232, - 8, 2, 69, 70, 4, 45, 9, 69, 82, 84, 73, 67, 65, 76, 32, 66, 4, 68, 9, 65, - 82, 32, 87, 73, 84, 72, 32, 81, 245, 236, 9, 2, 79, 88, 2, 215, 149, 8, - 85, 14, 22, 72, 203, 1, 73, 12, 25, 4, 73, 84, 69, 32, 12, 54, 67, 54, - 76, 182, 170, 7, 80, 154, 2, 83, 39, 84, 4, 26, 79, 215, 168, 7, 85, 2, - 65, 3, 82, 78, 69, 2, 41, 8, 69, 78, 84, 73, 67, 85, 76, 65, 2, 231, 177, - 10, 82, 2, 221, 167, 6, 6, 71, 71, 76, 89, 32, 70, 30, 82, 70, 206, 1, - 72, 130, 1, 80, 250, 1, 83, 197, 1, 6, 84, 79, 45, 76, 69, 70, 8, 33, 6, - 65, 67, 73, 78, 71, 32, 8, 144, 1, 14, 65, 82, 77, 69, 78, 73, 65, 78, - 32, 69, 84, 69, 82, 78, 144, 196, 3, 11, 83, 86, 65, 83, 84, 73, 32, 83, - 73, 71, 78, 151, 133, 3, 70, 2, 239, 227, 7, 73, 2, 81, 18, 65, 78, 68, - 69, 68, 32, 73, 78, 84, 69, 82, 76, 65, 67, 69, 68, 32, 80, 2, 21, 3, 69, - 78, 84, 2, 171, 199, 9, 65, 8, 41, 8, 79, 73, 78, 84, 73, 78, 71, 32, 8, - 140, 1, 6, 67, 85, 82, 86, 69, 68, 22, 65, 224, 247, 2, 6, 68, 79, 85, - 66, 76, 69, 253, 239, 6, 10, 77, 65, 71, 78, 73, 70, 89, 73, 78, 71, 2, - 17, 2, 32, 65, 2, 11, 78, 2, 157, 172, 10, 2, 71, 76, 4, 46, 72, 85, 7, - 73, 68, 69, 32, 65, 82, 67, 2, 17, 2, 65, 68, 2, 17, 2, 69, 68, 2, 165, - 222, 9, 6, 32, 87, 72, 73, 84, 69, 2, 11, 32, 2, 193, 168, 8, 8, 67, 76, - 79, 67, 75, 87, 73, 83, 8, 17, 2, 84, 32, 8, 86, 73, 40, 4, 79, 86, 69, - 82, 196, 238, 4, 5, 69, 77, 66, 69, 68, 187, 204, 5, 77, 2, 17, 2, 83, - 79, 2, 207, 129, 1, 76, 2, 179, 146, 3, 82, 2, 249, 236, 9, 2, 79, 82, - 210, 1, 160, 1, 5, 65, 82, 82, 79, 87, 182, 8, 66, 86, 68, 210, 1, 70, - 122, 72, 230, 6, 76, 26, 79, 34, 80, 50, 82, 70, 83, 94, 84, 202, 8, 87, - 226, 254, 7, 67, 47, 81, 71, 26, 32, 159, 205, 8, 45, 66, 94, 65, 170, 2, - 70, 82, 84, 184, 1, 5, 87, 73, 84, 72, 32, 149, 152, 1, 4, 79, 86, 69, - 82, 12, 40, 5, 66, 79, 86, 69, 32, 143, 1, 78, 10, 70, 82, 208, 155, 1, - 5, 83, 72, 79, 82, 84, 250, 180, 3, 65, 55, 84, 4, 37, 7, 69, 86, 69, 82, - 83, 69, 32, 4, 158, 208, 4, 65, 55, 84, 2, 61, 13, 68, 32, 85, 80, 80, - 69, 82, 32, 65, 78, 68, 32, 76, 2, 17, 2, 79, 87, 2, 137, 142, 8, 2, 69, - 82, 4, 37, 7, 82, 79, 77, 32, 66, 65, 82, 5, 189, 1, 6, 32, 84, 79, 32, - 66, 76, 10, 56, 7, 72, 82, 79, 85, 71, 72, 32, 65, 3, 79, 32, 66, 6, 144, - 203, 4, 3, 83, 85, 80, 130, 156, 4, 71, 131, 143, 2, 88, 4, 26, 76, 235, - 188, 10, 65, 2, 213, 171, 9, 3, 65, 67, 75, 38, 140, 1, 6, 67, 79, 82, - 78, 69, 82, 26, 68, 98, 76, 38, 80, 30, 83, 38, 84, 210, 139, 8, 77, 38, - 78, 122, 69, 226, 148, 1, 72, 155, 160, 1, 86, 2, 213, 18, 2, 32, 68, 4, - 11, 79, 4, 40, 4, 84, 84, 69, 68, 203, 177, 7, 85, 2, 17, 2, 32, 83, 2, - 247, 224, 10, 84, 4, 150, 140, 8, 65, 247, 237, 1, 79, 2, 249, 203, 8, 2, - 76, 85, 6, 218, 140, 8, 77, 135, 182, 2, 84, 10, 206, 16, 73, 171, 3, 65, - 8, 58, 65, 128, 12, 5, 79, 84, 84, 79, 77, 219, 128, 8, 76, 2, 145, 2, 2, - 67, 75, 14, 48, 6, 79, 85, 66, 76, 69, 32, 235, 154, 8, 65, 12, 40, 5, - 65, 82, 82, 79, 87, 135, 19, 68, 11, 26, 32, 183, 195, 8, 45, 6, 26, 87, - 243, 144, 8, 70, 4, 25, 4, 73, 84, 72, 32, 4, 190, 191, 10, 86, 79, 83, - 4, 40, 4, 82, 79, 78, 84, 143, 140, 8, 73, 2, 141, 139, 8, 14, 45, 84, - 73, 76, 84, 69, 68, 32, 83, 72, 65, 68, 79, 87, 30, 26, 65, 195, 143, 8, - 69, 26, 48, 6, 82, 80, 79, 79, 78, 32, 147, 168, 10, 78, 24, 88, 8, 79, - 86, 69, 82, 32, 76, 69, 70, 45, 10, 87, 73, 84, 72, 32, 66, 65, 82, 66, - 32, 2, 237, 219, 7, 6, 84, 87, 65, 82, 68, 83, 22, 44, 4, 68, 79, 87, 78, - 173, 1, 2, 85, 80, 10, 26, 32, 247, 148, 8, 87, 8, 76, 8, 65, 66, 79, 86, - 69, 32, 76, 69, 18, 66, 190, 140, 8, 70, 175, 5, 84, 2, 227, 2, 70, 2, - 185, 201, 4, 7, 69, 76, 79, 87, 32, 76, 79, 12, 26, 32, 203, 147, 8, 87, - 10, 60, 6, 65, 66, 79, 86, 69, 32, 178, 139, 8, 70, 175, 5, 84, 6, 22, - 76, 155, 1, 82, 4, 32, 2, 69, 70, 247, 199, 4, 79, 2, 233, 135, 2, 24, - 84, 87, 65, 82, 68, 83, 32, 72, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, - 72, 32, 66, 65, 82, 66, 2, 21, 3, 73, 71, 72, 2, 105, 24, 84, 87, 65, 82, - 68, 83, 32, 72, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, 32, 66, 65, - 82, 66, 2, 11, 32, 2, 163, 232, 1, 68, 2, 141, 1, 2, 69, 70, 2, 221, 141, - 8, 3, 80, 69, 78, 4, 218, 145, 8, 65, 221, 199, 1, 3, 85, 83, 72, 4, 36, - 3, 73, 71, 72, 187, 148, 10, 79, 2, 11, 84, 2, 171, 1, 45, 6, 32, 2, 81, - 85, 235, 136, 8, 65, 4, 26, 73, 131, 137, 8, 65, 2, 245, 144, 8, 2, 71, - 71, 54, 38, 79, 60, 2, 82, 73, 227, 4, 87, 2, 11, 80, 2, 11, 32, 2, 149, - 129, 8, 4, 83, 72, 65, 68, 34, 40, 5, 65, 78, 71, 76, 69, 255, 3, 80, 30, - 56, 8, 45, 72, 69, 65, 68, 69, 68, 32, 251, 149, 8, 32, 28, 52, 5, 65, - 82, 82, 79, 87, 218, 141, 8, 68, 39, 80, 25, 11, 32, 22, 64, 8, 79, 86, - 69, 82, 32, 76, 69, 70, 22, 87, 155, 137, 8, 84, 2, 203, 136, 8, 84, 18, - 25, 4, 73, 84, 72, 32, 18, 128, 1, 7, 68, 79, 85, 66, 76, 69, 32, 36, 7, - 76, 79, 78, 71, 32, 84, 73, 246, 136, 8, 66, 158, 1, 77, 34, 78, 34, 86, - 35, 72, 4, 186, 196, 8, 72, 255, 236, 1, 86, 4, 11, 80, 4, 11, 32, 4, 18, - 68, 35, 85, 2, 197, 137, 8, 3, 79, 87, 78, 2, 167, 137, 8, 80, 4, 21, 3, - 76, 69, 32, 4, 138, 3, 68, 183, 195, 9, 65, 18, 11, 79, 18, 56, 8, 45, - 72, 69, 65, 68, 69, 68, 32, 191, 139, 8, 32, 16, 76, 6, 65, 82, 82, 79, - 87, 32, 213, 1, 8, 84, 82, 73, 80, 76, 69, 32, 68, 14, 44, 5, 87, 73, 84, - 72, 32, 203, 255, 7, 70, 12, 42, 84, 206, 156, 7, 68, 191, 145, 3, 86, 8, - 26, 65, 131, 139, 8, 82, 6, 17, 2, 73, 76, 7, 33, 6, 32, 87, 73, 84, 72, - 32, 4, 246, 155, 7, 68, 191, 145, 3, 86, 2, 153, 127, 2, 65, 83, 8, 58, - 65, 21, 10, 72, 73, 84, 69, 32, 65, 82, 82, 79, 87, 2, 227, 135, 8, 86, - 7, 11, 32, 4, 164, 185, 2, 4, 70, 82, 79, 77, 147, 212, 5, 87, 19, 66, - 32, 136, 1, 6, 69, 68, 32, 80, 76, 65, 21, 3, 73, 78, 71, 12, 82, 66, 22, - 80, 132, 180, 4, 2, 73, 78, 26, 69, 214, 252, 2, 79, 187, 173, 2, 65, 2, - 195, 201, 10, 85, 2, 11, 79, 2, 231, 151, 10, 73, 2, 215, 165, 10, 78, 2, - 189, 190, 3, 2, 32, 66, 4, 24, 2, 70, 65, 75, 83, 2, 17, 2, 76, 76, 2, - 21, 3, 73, 78, 71, 2, 205, 222, 1, 2, 32, 68, 2, 225, 164, 3, 2, 79, 85, - 8, 178, 217, 10, 69, 2, 73, 2, 77, 3, 79, 124, 136, 2, 2, 67, 75, 20, 2, - 76, 76, 134, 2, 79, 56, 4, 77, 65, 78, 32, 192, 8, 2, 83, 69, 20, 6, 84, - 65, 84, 69, 68, 32, 152, 3, 3, 85, 78, 68, 230, 169, 3, 87, 228, 189, 3, - 15, 65, 83, 84, 69, 68, 32, 83, 87, 69, 69, 84, 32, 80, 79, 84, 165, 253, - 1, 2, 66, 79, 5, 207, 186, 10, 69, 10, 130, 1, 69, 64, 4, 32, 79, 70, 32, - 101, 21, 73, 78, 71, 32, 79, 78, 32, 84, 72, 69, 32, 70, 76, 79, 79, 82, - 32, 76, 65, 85, 71, 6, 60, 9, 68, 45, 85, 80, 32, 78, 69, 87, 83, 33, 2, - 82, 32, 2, 11, 80, 2, 207, 250, 1, 65, 4, 28, 3, 67, 79, 65, 23, 83, 2, - 151, 158, 9, 83, 2, 239, 89, 75, 2, 199, 131, 10, 72, 72, 140, 1, 6, 67, - 69, 78, 84, 85, 82, 22, 68, 100, 3, 81, 85, 73, 28, 8, 78, 85, 77, 69, - 82, 65, 76, 32, 182, 4, 83, 114, 85, 159, 179, 7, 65, 2, 223, 187, 5, 73, - 6, 98, 69, 232, 5, 5, 85, 80, 79, 78, 68, 61, 12, 73, 77, 73, 68, 73, 65, - 32, 83, 69, 88, 84, 85, 2, 229, 5, 3, 78, 65, 82, 48, 142, 1, 70, 136, 1, - 3, 79, 78, 69, 134, 1, 83, 66, 84, 152, 14, 10, 82, 69, 86, 69, 82, 83, - 69, 68, 32, 79, 142, 229, 2, 69, 235, 193, 6, 78, 14, 26, 73, 155, 216, - 9, 79, 12, 36, 3, 70, 84, 89, 167, 246, 2, 86, 7, 11, 32, 4, 158, 168, 5, - 84, 185, 169, 4, 5, 69, 65, 82, 76, 89, 11, 11, 32, 8, 50, 72, 41, 8, 84, - 72, 79, 85, 83, 65, 78, 68, 4, 185, 1, 6, 85, 78, 68, 82, 69, 68, 5, 205, - 245, 3, 2, 32, 67, 6, 32, 2, 73, 88, 223, 225, 8, 69, 5, 221, 207, 9, 2, - 32, 76, 10, 42, 69, 190, 244, 2, 87, 135, 237, 5, 72, 4, 11, 78, 5, 11, - 32, 2, 223, 165, 5, 84, 10, 46, 69, 197, 145, 7, 5, 73, 76, 73, 81, 85, - 8, 60, 2, 77, 85, 40, 5, 83, 84, 69, 82, 84, 21, 2, 88, 84, 2, 17, 2, 78, - 67, 2, 239, 144, 7, 73, 2, 231, 178, 7, 73, 4, 18, 65, 23, 85, 2, 203, - 178, 7, 78, 2, 159, 144, 7, 76, 5, 191, 135, 9, 84, 10, 166, 2, 70, 32, + 84, 72, 32, 70, 200, 107, 8, 79, 84, 65, 84, 69, 68, 32, 70, 217, 166, 6, + 4, 73, 71, 72, 84, 2, 133, 113, 9, 73, 78, 71, 69, 82, 83, 32, 83, 80, 4, + 208, 134, 1, 17, 65, 78, 83, 45, 83, 69, 82, 73, 70, 32, 67, 65, 80, 73, + 84, 65, 76, 183, 159, 7, 69, 10, 88, 4, 73, 76, 68, 69, 28, 7, 82, 73, + 80, 76, 69, 32, 80, 241, 150, 7, 3, 72, 85, 77, 5, 213, 231, 4, 2, 32, + 69, 2, 235, 215, 10, 82, 2, 11, 82, 2, 231, 177, 11, 79, 147, 4, 228, 1, + 4, 66, 66, 79, 78, 152, 1, 3, 67, 69, 32, 60, 3, 71, 72, 84, 240, 81, 2, + 78, 71, 252, 1, 23, 83, 73, 78, 71, 32, 68, 73, 65, 71, 79, 78, 65, 76, + 32, 67, 82, 79, 83, 83, 73, 78, 71, 32, 226, 212, 5, 65, 255, 158, 4, 70, + 19, 37, 7, 32, 65, 82, 82, 79, 87, 32, 16, 88, 3, 76, 69, 70, 0, 4, 82, + 73, 71, 72, 158, 193, 4, 85, 141, 128, 7, 3, 68, 79, 87, 4, 171, 248, 1, + 84, 4, 26, 67, 199, 228, 9, 66, 2, 217, 145, 1, 3, 82, 65, 67, 224, 3, + 110, 32, 190, 36, 45, 152, 12, 11, 72, 65, 78, 68, 32, 73, 78, 84, 69, + 82, 73, 29, 6, 87, 65, 82, 68, 83, 32, 202, 1, 166, 2, 65, 132, 6, 2, 66, + 76, 62, 67, 184, 1, 2, 68, 79, 242, 1, 70, 60, 2, 72, 65, 176, 5, 13, 74, + 85, 83, 84, 73, 70, 73, 69, 68, 32, 76, 69, 70, 20, 2, 76, 79, 66, 78, + 82, 79, 122, 80, 148, 1, 2, 82, 65, 58, 83, 154, 6, 84, 160, 5, 9, 86, + 69, 82, 84, 73, 67, 65, 76, 32, 147, 1, 87, 28, 22, 78, 163, 4, 82, 22, + 24, 2, 68, 32, 39, 71, 4, 170, 50, 76, 21, 3, 85, 80, 80, 18, 26, 69, 17, + 2, 76, 69, 2, 203, 26, 82, 17, 11, 32, 14, 154, 1, 66, 44, 8, 68, 79, 84, + 84, 69, 68, 32, 83, 2, 83, 112, 5, 87, 73, 84, 72, 32, 153, 196, 10, 12, + 86, 65, 82, 73, 65, 78, 84, 32, 87, 73, 84, 72, 4, 153, 194, 10, 6, 82, + 65, 67, 75, 69, 84, 2, 37, 7, 85, 66, 83, 84, 73, 84, 85, 2, 25, 4, 84, + 73, 79, 78, 2, 21, 3, 32, 77, 65, 2, 231, 138, 1, 82, 4, 68, 11, 68, 79, + 87, 78, 87, 65, 82, 68, 83, 32, 90, 179, 232, 8, 65, 2, 233, 201, 10, 5, + 73, 71, 90, 65, 71, 6, 18, 67, 79, 82, 2, 41, 8, 32, 71, 82, 69, 65, 84, + 69, 82, 2, 249, 24, 4, 45, 84, 72, 65, 4, 41, 8, 79, 87, 32, 87, 73, 84, + 72, 32, 4, 188, 220, 7, 7, 67, 73, 82, 67, 76, 69, 68, 151, 180, 2, 83, + 4, 25, 4, 65, 67, 75, 32, 4, 130, 27, 76, 139, 205, 7, 84, 12, 38, 85, + 166, 26, 79, 207, 231, 2, 69, 8, 53, 11, 82, 76, 89, 32, 66, 82, 65, 67, + 75, 69, 84, 9, 11, 32, 6, 44, 4, 77, 73, 68, 68, 250, 10, 76, 27, 85, 2, + 201, 170, 10, 2, 76, 69, 12, 38, 84, 41, 5, 85, 66, 76, 69, 32, 2, 137, + 17, 6, 84, 69, 68, 32, 83, 85, 10, 50, 65, 94, 87, 198, 158, 3, 81, 131, + 190, 4, 80, 4, 162, 31, 78, 157, 157, 3, 15, 82, 82, 79, 87, 32, 87, 73, + 84, 72, 32, 82, 79, 85, 78, 68, 2, 139, 24, 73, 6, 26, 73, 247, 254, 2, + 76, 4, 154, 201, 8, 86, 175, 97, 83, 28, 32, 3, 76, 70, 32, 187, 4, 78, + 26, 128, 1, 8, 65, 78, 68, 32, 76, 69, 70, 84, 62, 66, 90, 70, 82, 72, + 50, 82, 58, 84, 70, 87, 246, 13, 76, 22, 85, 215, 212, 8, 77, 2, 29, 5, + 32, 72, 65, 76, 70, 2, 245, 202, 8, 2, 32, 87, 6, 11, 76, 6, 40, 4, 65, + 67, 75, 32, 163, 172, 10, 79, 4, 146, 138, 10, 67, 207, 11, 83, 4, 58, + 79, 221, 152, 3, 8, 76, 89, 73, 78, 71, 32, 83, 65, 2, 175, 187, 9, 76, + 2, 253, 201, 8, 7, 79, 82, 73, 90, 79, 78, 84, 2, 33, 6, 85, 78, 78, 73, + 78, 71, 2, 247, 228, 6, 32, 2, 209, 169, 5, 12, 82, 73, 80, 76, 69, 32, + 68, 65, 83, 72, 32, 72, 2, 189, 151, 10, 4, 72, 73, 84, 69, 2, 173, 152, + 1, 16, 68, 32, 84, 69, 76, 69, 80, 72, 79, 78, 69, 32, 82, 69, 67, 69, 4, + 231, 217, 7, 84, 2, 233, 133, 11, 11, 87, 32, 80, 65, 82, 65, 80, 72, 82, + 65, 83, 2, 177, 4, 16, 79, 82, 77, 65, 76, 32, 70, 65, 67, 84, 79, 82, + 32, 83, 69, 77, 8, 74, 85, 210, 206, 8, 78, 169, 188, 1, 8, 80, 69, 78, + 32, 83, 81, 85, 65, 2, 181, 215, 10, 6, 84, 69, 82, 32, 74, 79, 8, 49, + 10, 65, 82, 69, 78, 84, 72, 69, 83, 73, 83, 9, 11, 32, 6, 34, 76, 26, 85, + 171, 182, 9, 69, 2, 157, 37, 2, 79, 87, 2, 133, 37, 2, 80, 80, 2, 217, + 10, 10, 73, 83, 69, 68, 32, 79, 77, 73, 83, 83, 40, 62, 45, 70, 69, 78, + 73, 68, 2, 80, 69, 126, 81, 227, 2, 85, 2, 253, 241, 7, 12, 83, 72, 65, + 80, 69, 68, 32, 66, 65, 71, 32, 68, 4, 26, 77, 231, 221, 8, 86, 2, 205, + 196, 10, 7, 73, 68, 73, 82, 69, 67, 84, 4, 194, 146, 3, 78, 145, 238, 7, + 8, 68, 69, 87, 65, 89, 83, 32, 85, 8, 32, 4, 65, 75, 69, 82, 67, 69, 7, + 33, 6, 32, 87, 73, 84, 72, 32, 4, 214, 248, 3, 79, 55, 84, 2, 137, 5, 2, + 67, 72, 20, 57, 12, 85, 65, 82, 69, 32, 66, 82, 65, 67, 75, 69, 84, 21, + 11, 32, 18, 84, 3, 76, 79, 87, 0, 3, 85, 80, 80, 28, 5, 87, 73, 84, 72, + 32, 143, 177, 9, 69, 2, 213, 226, 3, 2, 69, 82, 12, 96, 8, 84, 73, 67, + 75, 32, 73, 78, 32, 230, 6, 81, 246, 221, 7, 85, 222, 125, 68, 183, 189, + 2, 83, 4, 212, 225, 3, 6, 66, 79, 84, 84, 79, 77, 1, 3, 84, 79, 80, 2, + 165, 4, 6, 66, 83, 84, 73, 84, 85, 26, 54, 72, 130, 3, 82, 174, 209, 7, + 79, 171, 202, 2, 65, 14, 62, 73, 116, 5, 79, 85, 71, 72, 84, 45, 4, 82, + 69, 69, 32, 4, 21, 3, 82, 68, 32, 4, 68, 4, 73, 78, 68, 85, 185, 207, 1, + 7, 87, 72, 73, 84, 69, 32, 82, 2, 175, 148, 11, 67, 2, 11, 32, 2, 197, + 132, 3, 3, 66, 85, 66, 8, 60, 9, 81, 85, 65, 82, 84, 69, 82, 83, 32, 191, + 215, 8, 69, 6, 34, 76, 22, 85, 179, 221, 8, 66, 2, 37, 2, 79, 87, 2, 17, + 2, 80, 80, 2, 139, 216, 8, 69, 8, 34, 65, 89, 4, 73, 65, 78, 71, 2, 33, + 6, 78, 83, 80, 79, 83, 73, 2, 11, 84, 2, 17, 2, 73, 79, 2, 235, 247, 10, + 78, 6, 32, 2, 76, 69, 199, 214, 8, 85, 5, 41, 8, 32, 65, 66, 79, 86, 69, + 32, 76, 2, 225, 163, 9, 2, 69, 70, 6, 18, 66, 95, 82, 4, 68, 9, 65, 82, + 32, 87, 73, 84, 72, 32, 81, 177, 175, 10, 2, 79, 88, 2, 251, 208, 8, 85, + 2, 241, 173, 9, 3, 85, 76, 69, 14, 22, 72, 203, 1, 73, 12, 25, 4, 73, 84, + 69, 32, 12, 54, 67, 54, 76, 250, 196, 7, 80, 238, 7, 83, 39, 84, 4, 26, + 79, 207, 193, 7, 85, 2, 65, 3, 82, 78, 69, 2, 41, 8, 69, 78, 84, 73, 67, + 85, 76, 65, 2, 143, 244, 10, 82, 2, 133, 190, 6, 6, 71, 71, 76, 89, 32, + 70, 62, 118, 70, 226, 2, 72, 128, 1, 9, 80, 79, 73, 78, 84, 73, 78, 71, + 32, 214, 4, 83, 197, 1, 6, 84, 79, 45, 76, 69, 70, 18, 33, 6, 65, 67, 73, + 78, 71, 32, 18, 116, 14, 65, 82, 77, 69, 78, 73, 65, 78, 32, 69, 84, 69, + 82, 78, 20, 4, 66, 65, 83, 83, 16, 3, 70, 73, 83, 87, 83, 2, 171, 145, 8, + 73, 2, 227, 43, 73, 6, 26, 72, 239, 195, 11, 84, 5, 11, 32, 2, 149, 162, + 8, 6, 87, 73, 84, 72, 32, 79, 8, 236, 203, 3, 10, 86, 65, 83, 84, 73, 32, + 83, 73, 71, 78, 143, 213, 4, 78, 2, 81, 18, 65, 78, 68, 69, 68, 32, 73, + 78, 84, 69, 82, 76, 65, 67, 69, 68, 32, 80, 2, 21, 3, 69, 78, 84, 2, 231, + 134, 10, 65, 30, 90, 65, 30, 67, 94, 68, 58, 77, 58, 82, 182, 1, 83, 74, + 84, 254, 157, 8, 69, 179, 124, 71, 4, 90, 78, 203, 160, 8, 84, 2, 29, 5, + 85, 82, 86, 69, 68, 2, 17, 2, 32, 65, 2, 11, 78, 2, 177, 237, 10, 2, 71, + 76, 4, 248, 254, 2, 5, 79, 85, 66, 76, 69, 131, 177, 6, 73, 2, 129, 168, + 10, 9, 65, 71, 78, 73, 70, 89, 73, 78, 71, 10, 38, 79, 206, 163, 9, 65, + 247, 28, 73, 6, 92, 5, 67, 75, 69, 84, 32, 137, 163, 9, 12, 76, 76, 69, + 82, 32, 67, 79, 65, 83, 84, 69, 82, 4, 228, 43, 3, 66, 79, 79, 179, 189, + 10, 83, 2, 11, 84, 2, 21, 3, 73, 67, 75, 2, 221, 177, 6, 4, 32, 70, 73, + 71, 2, 155, 220, 7, 65, 4, 46, 72, 85, 7, 73, 68, 69, 32, 65, 82, 67, 2, + 17, 2, 65, 68, 2, 17, 2, 69, 68, 2, 173, 156, 10, 6, 32, 87, 72, 73, 84, + 69, 2, 11, 32, 2, 241, 224, 8, 8, 67, 76, 79, 67, 75, 87, 73, 83, 8, 17, + 2, 84, 32, 8, 86, 73, 40, 4, 79, 86, 69, 82, 152, 130, 5, 5, 69, 77, 66, + 69, 68, 251, 246, 5, 77, 2, 17, 2, 83, 79, 2, 131, 135, 1, 76, 2, 247, + 150, 3, 82, 2, 129, 171, 10, 2, 79, 82, 214, 1, 160, 1, 5, 65, 82, 82, + 79, 87, 130, 9, 66, 86, 68, 210, 1, 70, 122, 72, 230, 6, 76, 26, 79, 34, + 80, 50, 82, 70, 83, 94, 84, 206, 8, 87, 190, 182, 8, 67, 47, 81, 75, 26, + 32, 223, 132, 9, 45, 70, 94, 65, 170, 2, 70, 110, 84, 184, 1, 5, 87, 73, + 84, 72, 32, 181, 157, 1, 4, 79, 86, 69, 82, 12, 40, 5, 66, 79, 86, 69, + 32, 143, 1, 78, 10, 70, 82, 140, 161, 1, 5, 83, 72, 79, 82, 84, 130, 193, + 3, 65, 55, 84, 4, 37, 7, 69, 86, 69, 82, 83, 69, 32, 4, 226, 225, 4, 65, + 55, 84, 2, 61, 13, 68, 32, 85, 80, 80, 69, 82, 32, 65, 78, 68, 32, 76, 2, + 17, 2, 79, 87, 2, 169, 198, 8, 2, 69, 82, 6, 25, 4, 82, 79, 77, 32, 6, + 36, 3, 66, 65, 82, 227, 198, 8, 68, 5, 189, 1, 6, 32, 84, 79, 32, 66, 76, + 10, 56, 7, 72, 82, 79, 85, 71, 72, 32, 65, 3, 79, 32, 66, 6, 184, 220, 4, + 3, 83, 85, 80, 198, 197, 4, 71, 155, 146, 2, 88, 4, 26, 76, 227, 250, 10, + 65, 2, 141, 231, 9, 3, 65, 67, 75, 40, 140, 1, 6, 67, 79, 82, 78, 69, 82, + 26, 68, 98, 76, 86, 80, 30, 83, 38, 84, 178, 195, 8, 77, 38, 78, 122, 69, + 226, 151, 1, 72, 143, 163, 1, 86, 2, 133, 19, 2, 32, 68, 4, 11, 79, 4, + 40, 4, 84, 84, 69, 68, 155, 205, 7, 85, 2, 17, 2, 32, 83, 2, 251, 158, + 11, 84, 6, 26, 79, 143, 196, 8, 65, 4, 26, 87, 211, 176, 11, 79, 2, 253, + 196, 3, 2, 69, 82, 2, 237, 130, 9, 2, 76, 85, 6, 186, 196, 8, 77, 251, + 187, 2, 84, 10, 206, 16, 73, 171, 3, 65, 8, 58, 65, 128, 12, 5, 79, 84, + 84, 79, 77, 175, 184, 8, 76, 2, 145, 2, 2, 67, 75, 14, 48, 6, 79, 85, 66, + 76, 69, 32, 207, 210, 8, 65, 12, 40, 5, 65, 82, 82, 79, 87, 135, 19, 68, + 11, 26, 32, 171, 250, 8, 45, 6, 26, 87, 207, 200, 8, 70, 4, 25, 4, 73, + 84, 72, 32, 4, 146, 253, 10, 86, 79, 83, 4, 40, 4, 82, 79, 78, 84, 235, + 195, 8, 73, 2, 233, 194, 8, 14, 45, 84, 73, 76, 84, 69, 68, 32, 83, 72, + 65, 68, 79, 87, 30, 26, 65, 163, 199, 8, 69, 26, 48, 6, 82, 80, 79, 79, + 78, 32, 219, 229, 10, 78, 24, 88, 8, 79, 86, 69, 82, 32, 76, 69, 70, 45, + 10, 87, 73, 84, 72, 32, 66, 65, 82, 66, 32, 2, 157, 132, 8, 6, 84, 87, + 65, 82, 68, 83, 22, 44, 4, 68, 79, 87, 78, 173, 1, 2, 85, 80, 10, 26, 32, + 219, 204, 8, 87, 8, 76, 8, 65, 66, 79, 86, 69, 32, 76, 69, 18, 66, 154, + 196, 8, 70, 179, 5, 84, 2, 227, 2, 70, 2, 177, 218, 4, 7, 69, 76, 79, 87, + 32, 76, 79, 12, 26, 32, 175, 203, 8, 87, 10, 60, 6, 65, 66, 79, 86, 69, + 32, 142, 195, 8, 70, 179, 5, 84, 6, 22, 76, 155, 1, 82, 4, 32, 2, 69, 70, + 239, 216, 4, 79, 2, 253, 139, 2, 24, 84, 87, 65, 82, 68, 83, 32, 72, 65, + 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, 32, 66, 65, 82, 66, 2, 21, 3, 73, + 71, 72, 2, 105, 24, 84, 87, 65, 82, 68, 83, 32, 72, 65, 82, 80, 79, 79, + 78, 32, 87, 73, 84, 72, 32, 66, 65, 82, 66, 2, 11, 32, 2, 179, 236, 1, + 68, 2, 141, 1, 2, 69, 70, 2, 189, 197, 8, 3, 80, 69, 78, 4, 190, 201, 8, + 65, 181, 205, 1, 3, 85, 83, 72, 4, 36, 3, 73, 71, 72, 131, 210, 10, 79, + 2, 11, 84, 2, 171, 1, 45, 6, 32, 2, 81, 85, 203, 192, 8, 65, 4, 26, 73, + 227, 192, 8, 65, 2, 217, 200, 8, 2, 71, 71, 54, 38, 79, 60, 2, 82, 73, + 227, 4, 87, 2, 11, 80, 2, 11, 32, 2, 241, 184, 8, 4, 83, 72, 65, 68, 34, + 40, 5, 65, 78, 71, 76, 69, 255, 3, 80, 30, 56, 8, 45, 72, 69, 65, 68, 69, + 68, 32, 239, 204, 8, 32, 28, 52, 5, 65, 82, 82, 79, 87, 190, 197, 8, 68, + 39, 80, 25, 11, 32, 22, 64, 8, 79, 86, 69, 82, 32, 76, 69, 70, 22, 87, + 251, 192, 8, 84, 2, 171, 192, 8, 84, 18, 25, 4, 73, 84, 72, 32, 18, 128, + 1, 7, 68, 79, 85, 66, 76, 69, 32, 36, 7, 76, 79, 78, 71, 32, 84, 73, 218, + 192, 8, 66, 158, 1, 77, 34, 78, 34, 86, 35, 72, 4, 158, 251, 8, 72, 239, + 243, 1, 86, 4, 11, 80, 4, 11, 32, 4, 18, 68, 35, 85, 2, 169, 193, 8, 3, + 79, 87, 78, 2, 139, 193, 8, 80, 4, 21, 3, 76, 69, 32, 4, 138, 3, 68, 243, + 128, 10, 65, 18, 11, 79, 18, 56, 8, 45, 72, 69, 65, 68, 69, 68, 32, 163, + 195, 8, 32, 16, 76, 6, 65, 82, 82, 79, 87, 32, 213, 1, 8, 84, 82, 73, 80, + 76, 69, 32, 68, 14, 44, 5, 87, 73, 84, 72, 32, 167, 183, 8, 70, 12, 42, + 84, 238, 183, 7, 68, 243, 179, 3, 86, 8, 26, 65, 231, 194, 8, 82, 6, 17, + 2, 73, 76, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 150, 183, 7, 68, 243, + 179, 3, 86, 2, 137, 132, 1, 2, 65, 83, 8, 58, 65, 21, 10, 72, 73, 84, 69, + 32, 65, 82, 82, 79, 87, 2, 195, 191, 8, 86, 7, 11, 32, 4, 148, 189, 2, 4, + 70, 82, 79, 77, 147, 135, 6, 87, 19, 66, 32, 136, 1, 6, 69, 68, 32, 80, + 76, 65, 21, 3, 73, 78, 71, 12, 82, 66, 22, 80, 248, 196, 4, 2, 73, 78, + 26, 69, 238, 147, 3, 79, 239, 194, 2, 65, 2, 147, 135, 11, 85, 2, 11, 79, + 2, 171, 213, 10, 73, 2, 155, 227, 10, 78, 2, 253, 194, 3, 2, 32, 66, 4, + 24, 2, 70, 65, 75, 83, 2, 17, 2, 76, 76, 2, 21, 3, 73, 78, 71, 2, 217, + 226, 1, 2, 32, 68, 2, 233, 168, 3, 2, 79, 85, 8, 130, 151, 11, 69, 2, 73, + 2, 77, 3, 79, 126, 136, 2, 2, 67, 75, 20, 2, 76, 76, 188, 2, 4, 77, 65, + 78, 32, 194, 8, 79, 80, 2, 83, 69, 20, 6, 84, 65, 84, 69, 68, 32, 152, 3, + 3, 85, 78, 68, 214, 173, 3, 87, 200, 204, 3, 15, 65, 83, 84, 69, 68, 32, + 83, 87, 69, 69, 84, 32, 80, 79, 84, 205, 164, 2, 2, 66, 79, 5, 159, 248, + 10, 69, 10, 130, 1, 69, 64, 4, 32, 79, 70, 32, 101, 21, 73, 78, 71, 32, + 79, 78, 32, 84, 72, 69, 32, 70, 76, 79, 79, 82, 32, 76, 65, 85, 71, 6, + 60, 9, 68, 45, 85, 80, 32, 78, 69, 87, 83, 33, 2, 82, 32, 2, 11, 80, 2, + 215, 254, 1, 65, 4, 28, 3, 67, 79, 65, 23, 83, 2, 187, 218, 9, 83, 2, + 211, 94, 75, 2, 139, 193, 10, 72, 72, 140, 1, 6, 67, 69, 78, 84, 85, 82, + 22, 68, 100, 3, 81, 85, 73, 28, 8, 78, 85, 77, 69, 82, 65, 76, 32, 182, + 4, 83, 114, 85, 255, 219, 7, 65, 2, 187, 205, 5, 73, 6, 98, 69, 232, 5, + 5, 85, 80, 79, 78, 68, 61, 12, 73, 77, 73, 68, 73, 65, 32, 83, 69, 88, + 84, 85, 2, 229, 5, 3, 78, 65, 82, 48, 142, 1, 70, 136, 1, 3, 79, 78, 69, + 134, 1, 83, 66, 84, 232, 14, 10, 82, 69, 86, 69, 82, 83, 69, 68, 32, 79, + 210, 232, 2, 69, 143, 251, 6, 78, 14, 26, 73, 219, 149, 10, 79, 12, 36, + 3, 70, 84, 89, 187, 250, 2, 86, 7, 11, 32, 4, 222, 186, 5, 84, 185, 212, + 4, 5, 69, 65, 82, 76, 89, 11, 11, 32, 8, 50, 72, 41, 8, 84, 72, 79, 85, + 83, 65, 78, 68, 4, 185, 1, 6, 85, 78, 68, 82, 69, 68, 5, 197, 253, 3, 2, + 32, 67, 6, 32, 2, 73, 88, 171, 156, 9, 69, 5, 149, 141, 10, 2, 32, 76, + 10, 42, 69, 210, 248, 2, 87, 191, 163, 6, 72, 4, 11, 78, 5, 11, 32, 2, + 159, 184, 5, 84, 10, 46, 69, 181, 185, 7, 5, 73, 76, 73, 81, 85, 8, 60, + 2, 77, 85, 40, 5, 83, 84, 69, 82, 84, 21, 2, 88, 84, 2, 17, 2, 78, 67, 2, + 223, 184, 7, 73, 2, 199, 219, 7, 73, 4, 18, 65, 23, 85, 2, 171, 219, 7, + 78, 2, 143, 184, 7, 76, 4, 48, 6, 84, 32, 86, 69, 71, 69, 147, 209, 9, + 83, 2, 201, 192, 2, 2, 84, 65, 5, 243, 193, 9, 84, 10, 166, 2, 70, 32, 11, 72, 69, 65, 86, 89, 32, 66, 76, 65, 67, 75, 52, 24, 76, 73, 71, 72, 84, 32, 70, 79, 85, 82, 32, 80, 79, 73, 78, 84, 69, 68, 32, 66, 76, 65, 67, 75, 0, 18, 87, 72, 73, 84, 69, 32, 70, 79, 85, 82, 32, 80, 79, 73, 78, 84, 69, 68, 169, 53, 8, 67, 65, 80, 73, 84, 65, 76, 32, 2, 29, 5, 76, - 79, 82, 65, 76, 2, 217, 236, 8, 8, 32, 72, 69, 65, 82, 84, 32, 66, 2, - 225, 132, 9, 2, 32, 67, 16, 74, 32, 89, 14, 69, 68, 32, 83, 89, 77, 66, - 79, 76, 32, 70, 79, 82, 32, 4, 24, 2, 80, 85, 43, 84, 2, 11, 83, 2, 141, - 200, 9, 2, 72, 80, 2, 175, 210, 3, 65, 12, 68, 2, 83, 72, 242, 135, 6, - 67, 254, 153, 4, 70, 2, 76, 147, 17, 88, 4, 40, 4, 85, 65, 78, 71, 195, - 161, 10, 79, 2, 207, 178, 10, 88, 136, 2, 226, 1, 66, 20, 3, 71, 66, 89, + 79, 82, 65, 76, 2, 213, 166, 9, 8, 32, 72, 69, 65, 82, 84, 32, 66, 2, + 149, 191, 9, 2, 32, 67, 16, 74, 32, 89, 14, 69, 68, 32, 83, 89, 77, 66, + 79, 76, 32, 70, 79, 82, 32, 4, 24, 2, 80, 85, 43, 84, 2, 11, 83, 2, 245, + 132, 10, 2, 72, 80, 2, 203, 217, 3, 65, 12, 68, 2, 83, 72, 230, 152, 6, + 67, 138, 198, 4, 70, 2, 76, 147, 17, 88, 4, 40, 4, 85, 65, 78, 71, 195, + 222, 10, 79, 2, 207, 239, 10, 88, 136, 2, 226, 1, 66, 20, 3, 71, 66, 89, 40, 5, 76, 69, 45, 68, 69, 40, 3, 77, 73, 32, 154, 5, 78, 156, 26, 26, 83, 83, 73, 65, 78, 32, 65, 83, 84, 82, 79, 76, 79, 71, 73, 67, 65, 76, - 32, 83, 89, 77, 66, 79, 76, 32, 247, 252, 4, 80, 2, 211, 172, 8, 76, 2, - 205, 191, 8, 5, 32, 70, 79, 79, 84, 2, 11, 76, 2, 161, 228, 5, 2, 65, 89, - 62, 68, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 102, 78, 195, 190, 2, 68, - 8, 40, 4, 79, 78, 69, 32, 203, 229, 8, 84, 6, 34, 84, 150, 194, 8, 72, - 43, 81, 2, 191, 195, 8, 72, 36, 33, 6, 85, 77, 66, 69, 82, 32, 36, 76, 5, + 32, 83, 89, 77, 66, 79, 76, 32, 231, 142, 5, 80, 2, 255, 228, 8, 76, 2, + 189, 249, 8, 5, 32, 70, 79, 79, 84, 2, 11, 76, 2, 149, 245, 5, 2, 65, 89, + 62, 68, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 102, 78, 231, 193, 2, 68, + 8, 40, 4, 79, 78, 69, 32, 199, 159, 9, 84, 6, 34, 84, 134, 252, 8, 72, + 47, 81, 2, 179, 253, 8, 72, 36, 33, 6, 85, 77, 66, 69, 82, 32, 36, 76, 5, 69, 73, 71, 72, 84, 38, 70, 92, 2, 78, 73, 22, 79, 18, 83, 83, 84, 4, - 158, 129, 3, 32, 139, 192, 7, 89, 8, 18, 73, 35, 79, 4, 130, 2, 86, 251, - 212, 8, 70, 4, 136, 2, 2, 85, 82, 211, 212, 8, 82, 4, 77, 2, 78, 69, 2, - 167, 1, 78, 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 4, 206, 255, 2, 32, - 195, 174, 7, 84, 10, 34, 72, 50, 87, 143, 239, 9, 69, 4, 32, 2, 82, 69, - 215, 212, 8, 73, 2, 39, 69, 4, 26, 79, 199, 212, 8, 69, 2, 187, 254, 2, + 202, 132, 3, 32, 223, 249, 7, 89, 8, 18, 73, 35, 79, 4, 130, 2, 86, 247, + 142, 9, 70, 4, 136, 2, 2, 85, 82, 207, 142, 9, 82, 4, 77, 2, 78, 69, 2, + 167, 1, 78, 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 4, 250, 130, 3, 32, + 151, 232, 7, 84, 10, 34, 72, 50, 87, 131, 172, 10, 69, 4, 32, 2, 82, 69, + 211, 142, 9, 73, 2, 39, 69, 4, 26, 79, 195, 142, 9, 69, 2, 231, 129, 3, 32, 182, 1, 32, 3, 73, 67, 32, 147, 25, 78, 178, 1, 220, 1, 6, 66, 69, 76, 71, 84, 72, 20, 4, 67, 82, 79, 83, 20, 7, 76, 69, 84, 84, 69, 82, 32, - 210, 22, 83, 24, 6, 77, 85, 76, 84, 73, 80, 188, 141, 7, 5, 65, 82, 76, - 65, 85, 205, 183, 1, 7, 84, 86, 73, 77, 65, 68, 85, 2, 151, 220, 8, 79, - 2, 167, 143, 7, 83, 166, 1, 202, 4, 65, 110, 67, 98, 68, 126, 69, 82, 70, + 210, 22, 83, 24, 6, 77, 85, 76, 84, 73, 80, 208, 181, 7, 5, 65, 82, 76, + 65, 85, 181, 201, 1, 7, 84, 86, 73, 77, 65, 68, 85, 2, 147, 150, 9, 79, + 2, 227, 182, 7, 83, 166, 1, 202, 4, 65, 110, 67, 98, 68, 126, 69, 82, 70, 222, 1, 71, 104, 2, 72, 65, 50, 73, 220, 1, 5, 74, 69, 82, 65, 78, 34, 75, 58, 76, 234, 1, 79, 128, 1, 13, 82, 65, 73, 68, 79, 32, 82, 65, 68, 32, 82, 69, 73, 34, 83, 176, 2, 16, 66, 69, 82, 75, 65, 78, 65, 78, 32, 66, 69, 79, 82, 67, 32, 66, 144, 1, 12, 78, 65, 85, 68, 73, 90, 32, 78, - 89, 68, 32, 78, 110, 84, 194, 1, 87, 244, 83, 7, 85, 82, 85, 90, 32, 85, - 82, 208, 187, 2, 10, 77, 65, 78, 78, 65, 90, 32, 77, 65, 78, 184, 60, 13, - 80, 69, 82, 84, 72, 79, 32, 80, 69, 79, 82, 84, 72, 218, 246, 2, 89, 166, - 227, 3, 81, 2, 86, 2, 88, 3, 90, 8, 222, 4, 69, 202, 158, 6, 67, 0, 4, - 78, 83, 85, 90, 249, 135, 3, 9, 76, 71, 73, 90, 32, 69, 79, 76, 72, 11, - 46, 69, 30, 65, 197, 239, 6, 3, 87, 69, 79, 4, 26, 65, 247, 181, 10, 78, - 2, 147, 143, 9, 76, 11, 84, 6, 79, 84, 84, 69, 68, 45, 209, 219, 3, 9, - 65, 71, 65, 90, 32, 68, 65, 69, 71, 6, 134, 181, 10, 76, 2, 78, 3, 80, - 11, 228, 70, 7, 72, 87, 65, 90, 32, 69, 72, 174, 181, 9, 65, 194, 55, 84, + 89, 68, 32, 78, 110, 84, 194, 1, 87, 176, 87, 7, 85, 82, 85, 90, 32, 85, + 82, 192, 188, 2, 10, 77, 65, 78, 78, 65, 90, 32, 77, 65, 78, 180, 63, 13, + 80, 69, 82, 84, 72, 79, 32, 80, 69, 79, 82, 84, 72, 142, 189, 3, 89, 202, + 210, 3, 81, 2, 86, 2, 88, 3, 90, 8, 222, 4, 69, 174, 175, 6, 67, 0, 4, + 78, 83, 85, 90, 253, 179, 3, 9, 76, 71, 73, 90, 32, 69, 79, 76, 72, 11, + 46, 69, 30, 65, 145, 138, 7, 3, 87, 69, 79, 4, 26, 65, 247, 242, 10, 78, + 2, 247, 202, 9, 76, 11, 84, 6, 79, 84, 84, 69, 68, 45, 249, 226, 3, 9, + 65, 71, 65, 90, 32, 68, 65, 69, 71, 6, 134, 242, 10, 76, 2, 78, 3, 80, + 11, 240, 74, 7, 72, 87, 65, 90, 32, 69, 72, 150, 238, 9, 65, 206, 55, 84, 63, 78, 12, 120, 13, 82, 65, 78, 75, 83, 32, 67, 65, 83, 75, 69, 84, 32, - 233, 253, 6, 11, 69, 72, 85, 32, 70, 69, 79, 72, 32, 70, 69, 10, 46, 65, - 154, 245, 9, 73, 2, 79, 195, 60, 69, 4, 26, 69, 207, 178, 10, 67, 2, 235, - 139, 9, 83, 9, 26, 69, 207, 249, 9, 65, 4, 52, 7, 66, 79, 32, 71, 89, 70, - 85, 231, 177, 10, 82, 2, 143, 177, 10, 32, 4, 236, 8, 2, 69, 71, 13, 4, - 71, 76, 65, 90, 12, 156, 1, 2, 78, 71, 20, 9, 83, 65, 90, 32, 73, 83, 32, - 73, 83, 20, 5, 87, 65, 90, 32, 69, 136, 247, 9, 10, 67, 69, 76, 65, 78, - 68, 73, 67, 45, 89, 3, 79, 5, 131, 185, 6, 87, 2, 231, 239, 9, 83, 2, - 199, 174, 10, 79, 2, 11, 32, 2, 183, 175, 10, 74, 7, 21, 3, 65, 85, 78, - 4, 242, 171, 10, 32, 155, 3, 65, 12, 120, 15, 65, 85, 75, 65, 90, 32, 76, - 65, 71, 85, 32, 76, 79, 71, 82, 21, 11, 79, 78, 71, 45, 66, 82, 65, 78, - 67, 72, 45, 2, 211, 164, 9, 32, 10, 64, 3, 65, 82, 32, 158, 4, 72, 62, - 77, 66, 79, 179, 239, 9, 89, 2, 195, 150, 10, 65, 15, 150, 5, 83, 0, 12, - 84, 72, 65, 76, 65, 78, 32, 69, 84, 72, 69, 76, 224, 167, 10, 4, 80, 69, - 78, 45, 14, 69, 2, 78, 3, 79, 2, 11, 68, 2, 167, 243, 9, 32, 26, 150, 1, - 72, 244, 2, 18, 73, 71, 69, 76, 32, 76, 79, 78, 71, 45, 66, 82, 65, 78, - 67, 72, 45, 83, 228, 212, 6, 5, 79, 87, 73, 76, 79, 195, 139, 2, 84, 21, - 45, 9, 79, 82, 84, 45, 84, 87, 73, 71, 45, 18, 102, 66, 58, 72, 62, 77, - 30, 78, 38, 79, 42, 83, 186, 1, 84, 156, 146, 6, 2, 65, 82, 183, 219, 3, - 89, 2, 33, 6, 74, 65, 82, 75, 65, 78, 2, 235, 238, 8, 32, 2, 25, 4, 65, - 71, 65, 76, 2, 11, 76, 2, 195, 167, 10, 32, 2, 253, 145, 3, 2, 65, 68, 2, - 205, 216, 9, 4, 65, 85, 68, 32, 2, 17, 2, 83, 83, 2, 247, 136, 10, 32, 2, - 11, 79, 2, 215, 212, 6, 76, 4, 116, 15, 72, 85, 82, 73, 83, 65, 90, 32, - 84, 72, 85, 82, 83, 32, 84, 33, 10, 73, 87, 65, 90, 32, 84, 73, 82, 32, - 84, 2, 11, 72, 2, 167, 200, 5, 79, 2, 17, 2, 89, 82, 2, 223, 137, 10, 32, - 5, 41, 8, 85, 78, 74, 79, 32, 87, 89, 78, 2, 11, 78, 2, 175, 167, 9, 32, - 2, 21, 3, 73, 78, 71, 2, 169, 248, 6, 2, 76, 69, 4, 204, 191, 8, 16, 73, - 78, 71, 32, 83, 72, 73, 82, 84, 32, 87, 73, 84, 72, 32, 83, 219, 172, 1, - 69, 12, 120, 3, 66, 73, 78, 2, 78, 28, 2, 81, 85, 0, 3, 86, 73, 71, 134, - 234, 6, 83, 249, 154, 1, 6, 84, 82, 69, 68, 69, 67, 2, 149, 133, 8, 2, - 79, 86, 2, 237, 132, 8, 2, 73, 78, 214, 37, 244, 1, 2, 32, 73, 22, 65, - 238, 25, 67, 138, 5, 69, 146, 7, 72, 186, 33, 73, 186, 233, 1, 75, 154, - 2, 76, 130, 9, 77, 170, 21, 78, 174, 2, 79, 142, 39, 80, 180, 11, 2, 81, - 85, 158, 67, 83, 38, 84, 142, 16, 85, 150, 35, 87, 186, 1, 89, 187, 251, - 4, 71, 2, 131, 253, 8, 78, 196, 2, 140, 2, 5, 70, 69, 84, 89, 32, 36, 4, - 71, 73, 84, 84, 30, 76, 112, 8, 77, 65, 82, 73, 84, 65, 78, 32, 142, 14, - 78, 190, 3, 84, 100, 2, 85, 82, 132, 236, 2, 2, 73, 76, 198, 87, 88, 180, - 209, 5, 15, 75, 69, 32, 66, 79, 84, 84, 76, 69, 32, 65, 78, 68, 32, 67, - 143, 98, 82, 4, 234, 250, 1, 86, 151, 164, 7, 80, 2, 205, 201, 3, 2, 65, - 82, 6, 18, 84, 75, 85, 4, 36, 3, 32, 83, 72, 163, 253, 8, 73, 2, 11, 65, - 2, 211, 223, 9, 75, 2, 151, 149, 9, 84, 122, 184, 1, 7, 76, 69, 84, 84, - 69, 82, 32, 198, 3, 77, 248, 2, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, - 79, 78, 32, 132, 4, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 155, - 137, 4, 65, 44, 202, 1, 66, 32, 2, 68, 65, 22, 73, 38, 75, 22, 76, 34, - 83, 46, 84, 182, 2, 65, 208, 204, 5, 2, 71, 65, 134, 139, 1, 82, 210, - 237, 1, 90, 190, 78, 77, 172, 1, 2, 81, 85, 118, 78, 218, 6, 89, 235, - 101, 70, 4, 214, 253, 9, 73, 247, 25, 65, 2, 147, 134, 9, 76, 6, 206, - 153, 10, 78, 2, 84, 3, 89, 2, 147, 152, 9, 65, 2, 11, 65, 2, 187, 133, 9, - 66, 4, 152, 7, 3, 73, 78, 71, 171, 202, 8, 72, 6, 178, 151, 9, 65, 238, - 100, 73, 229, 10, 5, 83, 65, 65, 68, 73, 18, 96, 4, 65, 82, 75, 32, 165, - 1, 15, 79, 68, 73, 70, 73, 69, 82, 32, 76, 69, 84, 84, 69, 82, 32, 12, - 82, 68, 40, 2, 73, 78, 90, 69, 242, 2, 78, 173, 249, 7, 5, 79, 67, 67, - 76, 85, 2, 17, 2, 65, 71, 2, 163, 177, 8, 69, 5, 17, 2, 45, 65, 2, 255, - 148, 9, 76, 6, 46, 69, 180, 6, 2, 83, 72, 163, 143, 10, 73, 2, 145, 156, - 9, 11, 80, 69, 78, 84, 72, 69, 84, 73, 67, 32, 89, 28, 130, 1, 65, 154, - 1, 66, 22, 78, 30, 83, 130, 1, 90, 238, 155, 3, 84, 200, 208, 6, 9, 77, - 69, 76, 79, 68, 73, 67, 32, 81, 3, 81, 10, 76, 3, 70, 83, 65, 34, 78, 28, - 2, 84, 77, 145, 238, 9, 4, 82, 75, 65, 65, 2, 11, 65, 2, 179, 147, 10, - 81, 4, 26, 78, 199, 179, 5, 71, 2, 11, 65, 2, 143, 238, 9, 65, 2, 177, - 132, 3, 2, 69, 81, 4, 64, 4, 72, 73, 89, 89, 41, 8, 79, 70, 32, 77, 65, - 83, 72, 70, 2, 17, 2, 65, 65, 2, 167, 162, 8, 76, 2, 139, 254, 8, 65, 4, - 34, 65, 221, 161, 8, 2, 73, 81, 2, 151, 144, 9, 69, 30, 92, 5, 76, 79, - 78, 71, 32, 54, 79, 66, 83, 146, 206, 4, 65, 174, 193, 5, 69, 2, 73, 3, - 85, 10, 130, 207, 4, 65, 174, 193, 5, 69, 2, 73, 3, 85, 7, 41, 8, 86, 69, - 82, 76, 79, 78, 71, 32, 4, 163, 206, 4, 65, 4, 26, 72, 183, 192, 5, 85, - 2, 153, 251, 5, 3, 79, 82, 84, 10, 68, 8, 83, 45, 83, 69, 82, 73, 70, 32, - 145, 236, 9, 3, 68, 87, 73, 8, 44, 6, 72, 69, 65, 86, 89, 32, 139, 2, 73, - 6, 48, 3, 68, 79, 85, 81, 5, 76, 79, 87, 32, 68, 4, 11, 66, 4, 21, 3, 76, - 69, 32, 4, 84, 6, 84, 85, 82, 78, 69, 68, 23, 67, 2, 21, 3, 79, 85, 66, - 2, 17, 2, 76, 69, 2, 17, 2, 32, 67, 2, 33, 6, 79, 77, 77, 65, 32, 81, 2, - 153, 197, 8, 3, 85, 79, 84, 2, 169, 200, 9, 10, 78, 84, 69, 82, 82, 79, - 66, 65, 78, 71, 6, 48, 6, 69, 76, 76, 73, 84, 69, 135, 173, 5, 85, 5, 25, - 4, 32, 65, 78, 84, 2, 251, 232, 6, 69, 166, 1, 52, 7, 65, 83, 72, 84, 82, - 65, 32, 163, 169, 4, 79, 164, 1, 180, 1, 7, 76, 69, 84, 84, 69, 82, 32, - 212, 1, 5, 83, 73, 71, 78, 32, 186, 17, 68, 156, 244, 2, 16, 67, 79, 78, - 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 72, 251, 138, 2, 86, 100, - 150, 160, 6, 65, 38, 68, 46, 84, 46, 86, 186, 24, 85, 158, 144, 1, 79, - 182, 56, 73, 42, 76, 246, 189, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, - 74, 2, 75, 2, 80, 162, 7, 69, 222, 61, 72, 2, 77, 2, 82, 3, 89, 8, 234, - 160, 6, 67, 190, 161, 3, 65, 239, 1, 86, 52, 66, 65, 32, 4, 72, 79, 79, - 76, 46, 79, 74, 82, 143, 133, 10, 73, 4, 190, 242, 8, 76, 215, 18, 82, 5, - 165, 165, 5, 6, 32, 83, 65, 84, 67, 72, 6, 36, 3, 82, 80, 73, 227, 206, - 8, 79, 4, 214, 181, 9, 79, 135, 18, 85, 36, 66, 69, 72, 4, 73, 80, 84, - 32, 166, 243, 1, 85, 175, 140, 6, 79, 4, 36, 3, 87, 68, 82, 207, 180, 9, - 69, 2, 11, 73, 2, 179, 198, 9, 86, 28, 80, 8, 67, 65, 80, 73, 84, 65, 76, - 32, 86, 76, 81, 6, 83, 77, 65, 76, 76, 32, 18, 170, 131, 10, 66, 2, 69, - 2, 70, 2, 72, 2, 73, 2, 76, 2, 77, 2, 80, 3, 82, 2, 37, 7, 73, 71, 65, - 84, 85, 82, 69, 2, 11, 32, 2, 205, 190, 9, 2, 69, 84, 8, 134, 130, 10, - 69, 2, 71, 2, 76, 3, 79, 64, 250, 1, 65, 30, 67, 74, 69, 36, 5, 71, 77, - 69, 78, 84, 28, 2, 77, 73, 174, 1, 82, 158, 1, 83, 68, 2, 84, 32, 200, - 245, 4, 8, 87, 73, 78, 71, 32, 78, 69, 69, 172, 237, 1, 6, 80, 65, 82, - 65, 84, 69, 178, 122, 88, 202, 99, 68, 149, 159, 1, 2, 76, 70, 4, 222, - 255, 9, 76, 3, 84, 6, 34, 84, 157, 200, 5, 2, 79, 78, 4, 194, 165, 6, 73, - 143, 161, 3, 79, 4, 238, 216, 1, 68, 223, 131, 1, 45, 23, 181, 169, 4, 2, - 69, 68, 6, 128, 77, 28, 68, 73, 82, 69, 67, 84, 32, 80, 82, 79, 68, 85, - 67, 84, 32, 87, 73, 84, 72, 32, 66, 79, 84, 84, 79, 77, 32, 67, 172, 147, - 7, 3, 83, 69, 88, 151, 206, 1, 67, 4, 120, 2, 86, 73, 217, 171, 2, 22, - 73, 79, 85, 83, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 83, 89, 77, - 66, 79, 76, 83, 2, 11, 67, 2, 207, 186, 9, 69, 4, 52, 7, 81, 85, 73, 81, - 85, 65, 68, 155, 198, 8, 65, 2, 91, 82, 4, 64, 10, 84, 82, 65, 78, 83, - 77, 73, 84, 32, 83, 147, 158, 6, 77, 2, 11, 84, 2, 179, 182, 8, 65, 240, - 2, 102, 65, 254, 19, 73, 102, 79, 250, 10, 69, 70, 82, 184, 202, 8, 5, - 85, 70, 70, 76, 69, 143, 143, 1, 89, 176, 2, 164, 1, 12, 68, 79, 87, 69, - 68, 32, 87, 72, 73, 84, 69, 32, 68, 9, 76, 76, 79, 87, 32, 80, 65, 78, - 32, 62, 82, 206, 9, 86, 172, 133, 7, 2, 77, 82, 183, 224, 1, 75, 6, 38, - 76, 134, 169, 8, 67, 167, 11, 83, 2, 249, 185, 8, 2, 65, 84, 2, 17, 2, - 79, 70, 2, 17, 2, 32, 70, 2, 227, 213, 6, 79, 194, 1, 40, 4, 65, 68, 65, - 32, 135, 247, 9, 75, 192, 1, 162, 1, 68, 42, 69, 110, 72, 34, 76, 246, 1, - 83, 208, 2, 6, 86, 79, 87, 69, 76, 32, 226, 175, 4, 65, 196, 213, 1, 7, - 67, 79, 78, 84, 73, 78, 85, 235, 217, 3, 79, 24, 158, 32, 79, 66, 65, - 239, 230, 7, 73, 4, 84, 15, 88, 84, 82, 65, 32, 83, 72, 79, 82, 84, 32, - 86, 79, 87, 69, 163, 187, 8, 75, 2, 163, 179, 9, 76, 2, 189, 196, 9, 3, - 69, 65, 68, 96, 33, 6, 69, 84, 84, 69, 82, 32, 96, 178, 139, 6, 65, 38, - 68, 46, 84, 46, 86, 186, 24, 85, 210, 200, 1, 73, 42, 76, 246, 189, 1, - 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, - 77, 2, 82, 2, 89, 186, 2, 69, 3, 79, 30, 70, 65, 38, 69, 56, 4, 73, 71, - 78, 32, 141, 236, 8, 3, 85, 84, 82, 2, 177, 176, 9, 4, 78, 68, 72, 73, 6, - 132, 202, 2, 5, 67, 84, 73, 79, 78, 231, 142, 4, 80, 20, 102, 73, 182, - 249, 4, 83, 134, 144, 1, 65, 74, 67, 98, 78, 230, 179, 1, 74, 150, 3, 85, - 211, 235, 1, 86, 2, 37, 7, 78, 86, 69, 82, 84, 69, 68, 2, 213, 137, 6, 2, - 32, 67, 30, 60, 6, 77, 79, 68, 73, 70, 73, 21, 5, 83, 73, 71, 78, 32, 2, - 227, 252, 5, 69, 28, 86, 80, 226, 137, 6, 65, 106, 86, 214, 20, 85, 210, - 200, 1, 73, 206, 134, 2, 69, 3, 79, 2, 57, 12, 82, 73, 83, 72, 84, 72, - 65, 77, 65, 84, 82, 65, 2, 167, 215, 9, 32, 98, 72, 3, 69, 68, 32, 21, - 11, 73, 65, 78, 32, 76, 69, 84, 84, 69, 82, 32, 2, 171, 176, 9, 73, 96, - 158, 2, 65, 120, 3, 67, 72, 85, 22, 69, 70, 72, 46, 73, 46, 76, 22, 77, - 38, 79, 94, 80, 18, 82, 22, 83, 38, 84, 64, 2, 87, 79, 36, 2, 89, 69, - 208, 221, 4, 2, 74, 85, 162, 186, 2, 68, 202, 13, 90, 150, 84, 70, 182, - 6, 71, 210, 43, 66, 218, 11, 75, 206, 20, 86, 246, 25, 78, 143, 128, 1, - 85, 16, 82, 82, 198, 177, 9, 73, 222, 25, 68, 162, 8, 71, 2, 87, 198, 21, - 83, 147, 1, 72, 4, 166, 222, 8, 82, 239, 139, 1, 69, 2, 199, 199, 9, 82, - 8, 38, 65, 230, 176, 9, 82, 255, 55, 71, 4, 178, 233, 9, 82, 3, 84, 4, - 216, 241, 7, 2, 65, 45, 211, 166, 1, 85, 6, 150, 153, 9, 65, 130, 57, 67, - 215, 22, 70, 2, 143, 228, 7, 79, 4, 178, 136, 5, 69, 167, 200, 3, 73, 12, - 70, 79, 166, 222, 8, 73, 242, 108, 85, 150, 25, 65, 154, 3, 78, 3, 82, 2, - 235, 208, 9, 90, 2, 163, 13, 69, 2, 243, 196, 8, 79, 4, 230, 198, 8, 85, - 163, 160, 1, 79, 6, 26, 72, 159, 202, 9, 79, 4, 186, 238, 5, 73, 199, - 230, 3, 69, 4, 134, 221, 8, 79, 159, 137, 1, 69, 4, 254, 229, 9, 65, 3, - 87, 10, 78, 69, 222, 214, 3, 70, 192, 121, 6, 78, 84, 79, 32, 83, 72, - 251, 148, 5, 80, 2, 255, 158, 9, 76, 44, 236, 1, 2, 79, 84, 32, 6, 80, - 80, 73, 78, 71, 32, 72, 2, 82, 84, 180, 8, 7, 85, 76, 68, 69, 82, 69, 68, - 208, 240, 1, 24, 67, 75, 69, 68, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, - 32, 69, 88, 80, 76, 79, 68, 73, 78, 71, 199, 171, 7, 87, 2, 145, 208, 7, - 3, 73, 78, 71, 4, 36, 3, 84, 82, 79, 239, 220, 5, 66, 2, 11, 76, 2, 247, - 203, 7, 76, 32, 102, 32, 248, 5, 12, 72, 65, 78, 68, 32, 70, 79, 82, 77, - 65, 84, 32, 206, 149, 6, 67, 255, 197, 3, 83, 20, 154, 2, 66, 92, 11, 83, - 76, 65, 78, 84, 69, 68, 32, 78, 79, 82, 196, 1, 22, 82, 73, 71, 72, 84, - 87, 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 65, 66, 79, 86, 69, 28, - 7, 85, 80, 32, 84, 65, 67, 75, 128, 1, 4, 76, 69, 70, 84, 133, 160, 2, 9, - 68, 79, 87, 78, 32, 84, 65, 67, 75, 4, 88, 14, 65, 67, 75, 83, 76, 65, - 78, 84, 69, 68, 32, 83, 79, 85, 33, 4, 69, 78, 84, 32, 2, 11, 84, 2, 255, - 195, 8, 72, 2, 213, 37, 37, 65, 82, 82, 79, 87, 32, 80, 79, 73, 78, 84, + 137, 165, 7, 11, 69, 72, 85, 32, 70, 69, 79, 72, 32, 70, 69, 10, 46, 65, + 142, 178, 10, 73, 2, 79, 207, 60, 69, 4, 26, 69, 207, 239, 10, 67, 2, + 207, 199, 9, 83, 9, 26, 69, 195, 182, 10, 65, 4, 52, 7, 66, 79, 32, 71, + 89, 70, 85, 231, 238, 10, 82, 2, 143, 238, 10, 32, 4, 236, 8, 2, 69, 71, + 13, 4, 71, 76, 65, 90, 12, 156, 1, 2, 78, 71, 20, 9, 83, 65, 90, 32, 73, + 83, 32, 73, 83, 20, 5, 87, 65, 90, 32, 69, 252, 179, 10, 10, 67, 69, 76, + 65, 78, 68, 73, 67, 45, 89, 3, 79, 5, 199, 201, 6, 87, 2, 219, 172, 10, + 83, 2, 199, 235, 10, 79, 2, 11, 32, 2, 183, 236, 10, 74, 7, 21, 3, 65, + 85, 78, 4, 242, 232, 10, 32, 155, 3, 65, 12, 120, 15, 65, 85, 75, 65, 90, + 32, 76, 65, 71, 85, 32, 76, 79, 71, 82, 21, 11, 79, 78, 71, 45, 66, 82, + 65, 78, 67, 72, 45, 2, 187, 225, 9, 32, 10, 64, 3, 65, 82, 32, 158, 4, + 72, 62, 77, 66, 79, 167, 172, 10, 89, 2, 195, 211, 10, 65, 15, 150, 5, + 83, 0, 12, 84, 72, 65, 76, 65, 78, 32, 69, 84, 72, 69, 76, 224, 228, 10, + 4, 80, 69, 78, 45, 14, 69, 2, 78, 3, 79, 2, 11, 68, 2, 155, 176, 10, 32, + 26, 150, 1, 72, 244, 2, 18, 73, 71, 69, 76, 32, 76, 79, 78, 71, 45, 66, + 82, 65, 78, 67, 72, 45, 83, 200, 239, 6, 5, 79, 87, 73, 76, 79, 147, 171, + 2, 84, 21, 45, 9, 79, 82, 84, 45, 84, 87, 73, 71, 45, 18, 102, 66, 58, + 72, 62, 77, 30, 78, 38, 79, 42, 83, 186, 1, 84, 128, 163, 6, 2, 65, 82, + 199, 135, 4, 89, 2, 33, 6, 74, 65, 82, 75, 65, 78, 2, 171, 170, 9, 32, 2, + 25, 4, 65, 71, 65, 76, 2, 11, 76, 2, 195, 228, 10, 32, 2, 169, 150, 3, 2, + 65, 68, 2, 193, 149, 10, 4, 65, 85, 68, 32, 2, 17, 2, 83, 83, 2, 247, + 197, 10, 32, 2, 11, 79, 2, 187, 239, 6, 76, 4, 116, 15, 72, 85, 82, 73, + 83, 65, 90, 32, 84, 72, 85, 82, 83, 32, 84, 33, 10, 73, 87, 65, 90, 32, + 84, 73, 82, 32, 84, 2, 11, 72, 2, 155, 217, 5, 79, 2, 17, 2, 89, 82, 2, + 223, 198, 10, 32, 5, 41, 8, 85, 78, 74, 79, 32, 87, 89, 78, 2, 11, 78, 2, + 159, 228, 9, 32, 2, 21, 3, 73, 78, 71, 2, 229, 159, 7, 2, 76, 69, 4, 200, + 249, 8, 16, 73, 78, 71, 32, 83, 72, 73, 82, 84, 32, 87, 73, 84, 72, 32, + 83, 211, 175, 1, 69, 12, 120, 3, 66, 73, 78, 2, 78, 28, 2, 81, 85, 0, 3, + 86, 73, 71, 166, 145, 7, 83, 229, 169, 1, 6, 84, 82, 69, 68, 69, 67, 2, + 161, 187, 8, 2, 79, 86, 2, 249, 186, 8, 2, 73, 78, 250, 39, 244, 1, 2, + 32, 73, 22, 65, 238, 25, 67, 138, 5, 69, 166, 11, 72, 190, 33, 73, 198, + 232, 1, 75, 154, 2, 76, 142, 9, 77, 166, 21, 78, 174, 2, 79, 158, 39, 80, + 172, 12, 2, 81, 85, 146, 70, 83, 38, 84, 250, 16, 85, 166, 43, 87, 186, + 1, 89, 251, 166, 5, 71, 2, 239, 184, 9, 78, 196, 2, 140, 2, 5, 70, 69, + 84, 89, 32, 36, 4, 71, 73, 84, 84, 30, 76, 112, 8, 77, 65, 82, 73, 84, + 65, 78, 32, 142, 14, 78, 190, 3, 84, 100, 2, 85, 82, 244, 239, 2, 2, 73, + 76, 234, 91, 88, 144, 134, 6, 15, 75, 69, 32, 66, 79, 84, 84, 76, 69, 32, + 65, 78, 68, 32, 67, 159, 98, 82, 4, 138, 254, 1, 86, 223, 221, 7, 80, 2, + 245, 208, 3, 2, 65, 82, 6, 18, 84, 75, 85, 4, 36, 3, 32, 83, 72, 139, + 186, 9, 73, 2, 11, 65, 2, 199, 156, 10, 75, 2, 255, 209, 9, 84, 122, 184, + 1, 7, 76, 69, 84, 84, 69, 82, 32, 198, 3, 77, 248, 2, 12, 80, 85, 78, 67, + 84, 85, 65, 84, 73, 79, 78, 32, 132, 4, 11, 86, 79, 87, 69, 76, 32, 83, + 73, 71, 78, 32, 203, 155, 4, 65, 44, 202, 1, 66, 32, 2, 68, 65, 22, 73, + 38, 75, 22, 76, 34, 83, 46, 84, 182, 2, 65, 196, 221, 5, 2, 71, 65, 222, + 148, 1, 82, 146, 141, 2, 90, 154, 81, 77, 172, 1, 2, 81, 85, 118, 78, + 226, 6, 89, 251, 101, 70, 4, 214, 186, 10, 73, 247, 25, 65, 2, 251, 194, + 9, 76, 6, 206, 214, 10, 78, 2, 84, 3, 89, 2, 251, 212, 9, 65, 2, 11, 65, + 2, 163, 194, 9, 66, 4, 152, 7, 3, 73, 78, 71, 223, 132, 9, 72, 6, 154, + 212, 9, 65, 134, 101, 73, 229, 10, 5, 83, 65, 65, 68, 73, 18, 96, 4, 65, + 82, 75, 32, 165, 1, 15, 79, 68, 73, 70, 73, 69, 82, 32, 76, 69, 84, 84, + 69, 82, 32, 12, 82, 68, 40, 2, 73, 78, 90, 69, 242, 2, 78, 197, 176, 8, + 5, 79, 67, 67, 76, 85, 2, 17, 2, 65, 71, 2, 159, 235, 8, 69, 5, 17, 2, + 45, 65, 2, 231, 209, 9, 76, 6, 46, 69, 180, 6, 2, 83, 72, 163, 204, 10, + 73, 2, 129, 217, 9, 11, 80, 69, 78, 84, 72, 69, 84, 73, 67, 32, 89, 28, + 130, 1, 65, 154, 1, 66, 22, 78, 30, 83, 130, 1, 90, 174, 162, 3, 84, 136, + 135, 7, 9, 77, 69, 76, 79, 68, 73, 67, 32, 81, 3, 81, 10, 76, 3, 70, 83, + 65, 34, 78, 28, 2, 84, 77, 145, 171, 10, 4, 82, 75, 65, 65, 2, 11, 65, 2, + 179, 208, 10, 81, 4, 26, 78, 187, 196, 5, 71, 2, 11, 65, 2, 143, 171, 10, + 65, 2, 201, 138, 3, 2, 69, 81, 4, 64, 4, 72, 73, 89, 89, 41, 8, 79, 70, + 32, 77, 65, 83, 72, 70, 2, 17, 2, 65, 65, 2, 163, 220, 8, 76, 2, 243, + 186, 9, 65, 4, 34, 65, 217, 219, 8, 2, 73, 81, 2, 255, 204, 9, 69, 30, + 92, 5, 76, 79, 78, 71, 32, 54, 79, 66, 83, 186, 224, 4, 65, 134, 236, 5, + 69, 2, 73, 3, 85, 10, 170, 225, 4, 65, 134, 236, 5, 69, 2, 73, 3, 85, 7, + 41, 8, 86, 69, 82, 76, 79, 78, 71, 32, 4, 203, 224, 4, 65, 4, 26, 72, + 171, 209, 5, 85, 2, 253, 139, 6, 3, 79, 82, 84, 10, 68, 8, 83, 45, 83, + 69, 82, 73, 70, 32, 145, 169, 10, 3, 68, 87, 73, 8, 44, 6, 72, 69, 65, + 86, 89, 32, 139, 2, 73, 6, 48, 3, 68, 79, 85, 81, 5, 76, 79, 87, 32, 68, + 4, 11, 66, 4, 21, 3, 76, 69, 32, 4, 84, 6, 84, 85, 82, 78, 69, 68, 23, + 67, 2, 21, 3, 79, 85, 66, 2, 17, 2, 76, 69, 2, 17, 2, 32, 67, 2, 33, 6, + 79, 77, 77, 65, 32, 81, 2, 205, 255, 8, 3, 85, 79, 84, 2, 157, 133, 10, + 10, 78, 84, 69, 82, 82, 79, 66, 65, 78, 71, 6, 48, 6, 69, 76, 76, 73, 84, + 69, 251, 189, 5, 85, 5, 25, 4, 32, 65, 78, 84, 2, 135, 145, 7, 69, 166, + 1, 52, 7, 65, 83, 72, 84, 82, 65, 32, 235, 187, 4, 79, 164, 1, 180, 1, 7, + 76, 69, 84, 84, 69, 82, 32, 212, 1, 5, 83, 73, 71, 78, 32, 194, 21, 68, + 156, 246, 2, 16, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, + 32, 72, 139, 150, 2, 86, 100, 198, 235, 6, 65, 38, 68, 114, 84, 46, 86, + 186, 5, 85, 202, 141, 1, 79, 134, 60, 73, 42, 76, 250, 192, 1, 78, 46, + 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 162, 7, 69, 234, 61, 72, + 2, 77, 2, 82, 3, 89, 8, 210, 175, 6, 67, 202, 207, 3, 65, 239, 1, 86, 52, + 66, 65, 32, 4, 72, 79, 79, 76, 46, 79, 74, 82, 143, 194, 10, 73, 4, 166, + 175, 9, 76, 215, 18, 82, 5, 153, 182, 5, 6, 32, 83, 65, 84, 67, 72, 6, + 36, 3, 82, 80, 73, 183, 138, 9, 79, 4, 202, 242, 9, 79, 135, 18, 85, 36, + 66, 69, 72, 4, 73, 80, 84, 32, 198, 246, 1, 85, 255, 194, 6, 79, 4, 36, + 3, 87, 68, 82, 195, 241, 9, 69, 2, 11, 73, 2, 167, 131, 10, 86, 28, 80, + 8, 67, 65, 80, 73, 84, 65, 76, 32, 86, 76, 81, 6, 83, 77, 65, 76, 76, 32, + 18, 170, 192, 10, 66, 2, 69, 2, 70, 2, 72, 2, 73, 2, 76, 2, 77, 2, 80, 3, + 82, 2, 37, 7, 73, 71, 65, 84, 85, 82, 69, 2, 11, 32, 2, 193, 251, 9, 2, + 69, 84, 8, 134, 191, 10, 69, 2, 71, 2, 76, 3, 79, 220, 1, 130, 2, 65, 30, + 67, 74, 69, 36, 5, 71, 77, 69, 78, 84, 28, 2, 77, 73, 172, 1, 8, 80, 65, + 82, 65, 84, 69, 68, 32, 138, 4, 82, 158, 1, 83, 68, 2, 84, 32, 140, 131, + 5, 8, 87, 73, 78, 71, 32, 78, 69, 69, 146, 140, 3, 88, 254, 104, 68, 213, + 160, 1, 2, 76, 70, 4, 210, 188, 10, 76, 3, 84, 6, 34, 84, 133, 217, 5, 2, + 79, 78, 4, 222, 187, 6, 73, 219, 199, 3, 79, 4, 130, 220, 1, 68, 195, + 132, 1, 45, 23, 221, 187, 4, 2, 69, 68, 6, 176, 80, 28, 68, 73, 82, 69, + 67, 84, 32, 80, 82, 79, 68, 85, 67, 84, 32, 87, 73, 84, 72, 32, 66, 79, + 84, 84, 79, 77, 32, 67, 252, 197, 7, 3, 83, 69, 88, 255, 212, 1, 67, 158, + 1, 48, 6, 66, 76, 79, 67, 75, 32, 191, 166, 9, 83, 156, 1, 88, 9, 81, 85, + 65, 68, 82, 65, 78, 84, 45, 129, 1, 8, 83, 69, 88, 84, 65, 78, 84, 45, + 30, 42, 49, 38, 50, 30, 51, 131, 184, 10, 52, 17, 34, 50, 30, 51, 131, + 184, 10, 52, 9, 26, 51, 131, 184, 10, 52, 5, 255, 183, 10, 52, 126, 58, + 49, 54, 50, 46, 51, 38, 52, 30, 53, 147, 182, 10, 54, 65, 50, 50, 46, 51, + 38, 52, 30, 53, 147, 182, 10, 54, 33, 42, 51, 38, 52, 30, 53, 147, 182, + 10, 54, 17, 34, 52, 30, 53, 147, 182, 10, 54, 9, 26, 53, 147, 182, 10, + 54, 5, 143, 182, 10, 54, 4, 120, 2, 86, 73, 241, 170, 2, 22, 73, 79, 85, + 83, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 83, 89, 77, 66, 79, 76, + 83, 2, 11, 67, 2, 175, 243, 9, 69, 4, 52, 7, 81, 85, 73, 81, 85, 65, 68, + 219, 253, 8, 65, 2, 91, 82, 4, 64, 10, 84, 82, 65, 78, 83, 77, 73, 84, + 32, 83, 195, 173, 6, 77, 2, 11, 84, 2, 211, 236, 8, 65, 242, 2, 102, 65, + 246, 19, 73, 102, 79, 134, 11, 69, 70, 82, 136, 131, 9, 5, 85, 70, 70, + 76, 69, 167, 143, 1, 89, 176, 2, 164, 1, 12, 68, 79, 87, 69, 68, 32, 87, + 72, 73, 84, 69, 32, 56, 9, 76, 76, 79, 87, 32, 80, 65, 78, 32, 62, 82, + 210, 9, 86, 172, 184, 7, 2, 77, 82, 147, 230, 1, 75, 6, 162, 223, 8, 67, + 206, 11, 83, 237, 4, 3, 76, 65, 84, 2, 17, 2, 79, 70, 2, 17, 2, 32, 70, + 2, 231, 249, 6, 79, 194, 1, 40, 4, 65, 68, 65, 32, 255, 175, 10, 75, 192, + 1, 162, 1, 68, 46, 69, 110, 72, 34, 76, 246, 1, 83, 212, 2, 6, 86, 79, + 87, 69, 76, 32, 250, 189, 4, 65, 228, 211, 1, 7, 67, 79, 78, 84, 73, 78, + 85, 163, 134, 4, 79, 24, 238, 209, 6, 79, 66, 65, 147, 235, 1, 73, 4, 84, + 15, 88, 84, 82, 65, 32, 83, 72, 79, 82, 84, 32, 86, 79, 87, 69, 215, 242, + 8, 75, 2, 139, 236, 9, 76, 2, 177, 253, 9, 3, 69, 65, 68, 96, 33, 6, 69, + 84, 84, 69, 82, 32, 96, 214, 210, 6, 65, 38, 68, 114, 84, 46, 86, 186, 5, + 85, 206, 201, 1, 73, 42, 76, 250, 192, 1, 78, 46, 83, 82, 66, 2, 67, 2, + 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 77, 2, 82, 2, 89, 186, 2, 69, 3, + 79, 30, 70, 65, 38, 69, 56, 4, 73, 71, 78, 32, 233, 164, 9, 3, 85, 84, + 82, 2, 153, 233, 9, 4, 78, 68, 72, 73, 6, 180, 201, 2, 5, 67, 84, 73, 79, + 78, 191, 179, 4, 80, 20, 106, 73, 190, 134, 5, 83, 158, 142, 1, 67, 98, + 78, 242, 60, 65, 174, 158, 1, 74, 150, 3, 85, 167, 242, 1, 86, 2, 37, 7, + 78, 86, 69, 82, 84, 69, 68, 2, 173, 148, 6, 2, 32, 67, 30, 60, 6, 77, 79, + 68, 73, 70, 73, 21, 5, 83, 73, 71, 78, 32, 2, 151, 137, 6, 69, 28, 82, + 80, 226, 211, 6, 65, 38, 85, 22, 86, 186, 201, 1, 73, 222, 137, 2, 69, 3, + 79, 2, 57, 12, 82, 73, 83, 72, 84, 72, 65, 77, 65, 84, 82, 65, 2, 155, + 144, 10, 32, 98, 72, 3, 69, 68, 32, 21, 11, 73, 65, 78, 32, 76, 69, 84, + 84, 69, 82, 32, 2, 147, 233, 9, 73, 96, 158, 2, 65, 120, 3, 67, 72, 85, + 22, 69, 70, 72, 46, 73, 46, 76, 22, 77, 38, 79, 94, 80, 18, 82, 22, 83, + 38, 84, 64, 2, 87, 79, 36, 2, 89, 69, 156, 235, 4, 2, 74, 85, 234, 222, + 2, 68, 202, 13, 90, 242, 87, 70, 182, 6, 71, 150, 45, 66, 246, 11, 75, + 218, 21, 86, 246, 25, 78, 167, 128, 1, 85, 16, 82, 82, 174, 234, 9, 73, + 234, 25, 68, 162, 8, 71, 2, 87, 198, 21, 83, 147, 1, 72, 4, 130, 151, 9, + 82, 135, 140, 1, 69, 2, 187, 128, 10, 82, 8, 38, 65, 206, 233, 9, 82, + 139, 56, 71, 4, 166, 162, 10, 82, 3, 84, 4, 200, 167, 8, 2, 65, 45, 203, + 169, 1, 85, 6, 254, 209, 9, 65, 142, 57, 67, 215, 22, 70, 2, 243, 153, 8, + 79, 4, 154, 149, 5, 69, 155, 244, 3, 73, 12, 70, 79, 130, 151, 9, 73, + 138, 109, 85, 150, 25, 65, 154, 3, 78, 3, 82, 2, 223, 137, 10, 90, 2, + 175, 13, 69, 2, 215, 254, 8, 79, 4, 194, 255, 8, 85, 187, 160, 1, 79, 6, + 26, 72, 147, 131, 10, 79, 4, 242, 250, 5, 73, 131, 147, 4, 69, 4, 226, + 149, 9, 79, 183, 137, 1, 69, 4, 242, 158, 10, 65, 3, 87, 10, 78, 69, 134, + 229, 3, 70, 252, 120, 6, 78, 84, 79, 32, 83, 72, 139, 192, 5, 80, 2, 231, + 215, 9, 76, 46, 252, 1, 2, 79, 84, 32, 6, 80, 80, 73, 78, 71, 32, 72, 2, + 82, 84, 176, 8, 7, 85, 76, 68, 69, 82, 69, 68, 220, 239, 1, 24, 67, 75, + 69, 68, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 69, 88, 80, 76, 79, + 68, 73, 78, 71, 158, 150, 3, 86, 251, 206, 4, 87, 2, 221, 133, 8, 3, 73, + 78, 71, 4, 36, 3, 84, 82, 79, 155, 233, 5, 66, 2, 11, 76, 2, 163, 129, 8, + 76, 32, 102, 32, 248, 5, 12, 72, 65, 78, 68, 32, 70, 79, 82, 77, 65, 84, + 32, 254, 171, 6, 67, 179, 232, 3, 83, 20, 154, 2, 66, 92, 11, 83, 76, 65, + 78, 84, 69, 68, 32, 78, 79, 82, 196, 1, 22, 82, 73, 71, 72, 84, 87, 65, + 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 65, 66, 79, 86, 69, 28, 7, 85, + 80, 32, 84, 65, 67, 75, 128, 1, 4, 76, 69, 70, 84, 149, 159, 2, 9, 68, + 79, 87, 78, 32, 84, 65, 67, 75, 4, 88, 14, 65, 67, 75, 83, 76, 65, 78, + 84, 69, 68, 32, 83, 79, 85, 33, 4, 69, 78, 84, 32, 2, 11, 84, 2, 203, + 252, 8, 72, 2, 245, 36, 37, 65, 82, 82, 79, 87, 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 68, 79, 87, 78, 87, 65, 82, 68, 83, 32, 84, 72, 69, 78, - 32, 78, 79, 82, 84, 72, 32, 69, 2, 189, 237, 5, 2, 32, 76, 7, 11, 32, 4, - 76, 13, 65, 66, 79, 86, 69, 32, 83, 72, 79, 82, 84, 32, 68, 191, 131, 2, - 87, 2, 11, 79, 2, 11, 87, 2, 11, 78, 2, 11, 32, 2, 191, 209, 6, 84, 8, + 32, 78, 79, 82, 84, 72, 32, 69, 2, 229, 249, 5, 2, 32, 76, 7, 11, 32, 4, + 76, 13, 65, 66, 79, 86, 69, 32, 83, 72, 79, 82, 84, 32, 68, 159, 131, 2, + 87, 2, 11, 79, 2, 11, 87, 2, 11, 78, 2, 11, 32, 2, 191, 249, 6, 84, 8, 120, 10, 67, 79, 78, 84, 73, 78, 85, 73, 78, 71, 0, 6, 76, 69, 84, 84, - 69, 82, 44, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 245, 201, 8, 6, 32, 79, - 86, 69, 82, 76, 2, 21, 3, 32, 83, 84, 2, 251, 217, 9, 69, 2, 25, 4, 32, - 79, 80, 69, 2, 167, 205, 8, 78, 4, 26, 73, 215, 216, 9, 85, 2, 155, 217, - 9, 77, 159, 14, 174, 1, 68, 132, 20, 2, 71, 78, 232, 181, 1, 6, 77, 73, - 76, 65, 82, 32, 158, 2, 78, 198, 24, 88, 225, 171, 6, 15, 76, 72, 79, 85, - 69, 84, 84, 69, 32, 79, 70, 32, 74, 65, 80, 200, 1, 64, 5, 68, 72, 65, - 77, 32, 225, 17, 6, 69, 87, 65, 89, 83, 32, 184, 1, 194, 1, 68, 106, 69, - 68, 7, 76, 69, 84, 84, 69, 82, 32, 172, 4, 15, 82, 69, 80, 69, 84, 73, - 84, 73, 79, 78, 32, 77, 65, 82, 75, 50, 83, 165, 8, 11, 86, 79, 87, 69, - 76, 32, 83, 73, 71, 78, 32, 4, 18, 79, 67, 65, 2, 11, 85, 2, 11, 66, 2, - 25, 4, 76, 69, 32, 68, 2, 11, 65, 2, 247, 175, 9, 78, 2, 37, 7, 78, 68, - 32, 79, 70, 32, 84, 2, 129, 138, 9, 2, 69, 88, 102, 210, 1, 65, 98, 84, - 198, 233, 5, 68, 90, 86, 186, 24, 85, 210, 200, 1, 73, 158, 190, 1, 78, - 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, 76, 2, - 77, 2, 82, 2, 89, 186, 2, 69, 3, 79, 11, 72, 8, 76, 84, 69, 82, 78, 65, - 84, 69, 134, 210, 9, 65, 2, 73, 3, 85, 2, 155, 173, 9, 32, 14, 134, 1, - 72, 196, 212, 5, 19, 87, 79, 45, 67, 73, 82, 67, 76, 69, 32, 65, 76, 84, - 69, 82, 78, 65, 84, 69, 242, 180, 3, 84, 183, 71, 65, 4, 224, 144, 9, 20, - 82, 69, 69, 45, 67, 73, 82, 67, 76, 69, 32, 65, 76, 84, 69, 82, 78, 65, - 84, 69, 135, 64, 65, 6, 11, 45, 6, 234, 207, 9, 49, 2, 50, 3, 51, 44, 38, - 69, 181, 7, 4, 73, 71, 78, 32, 32, 96, 11, 67, 84, 73, 79, 78, 32, 77, - 65, 82, 75, 32, 177, 6, 8, 80, 65, 82, 65, 84, 79, 82, 32, 28, 80, 11, - 68, 79, 85, 66, 76, 69, 32, 82, 73, 78, 71, 57, 5, 87, 73, 84, 72, 32, 5, - 11, 32, 2, 225, 162, 8, 6, 87, 73, 84, 72, 32, 82, 24, 212, 1, 12, 67, - 73, 82, 67, 76, 69, 83, 32, 65, 78, 68, 32, 116, 5, 81, 85, 65, 68, 82, - 0, 4, 83, 69, 80, 84, 12, 16, 82, 65, 89, 83, 32, 65, 78, 68, 32, 68, 79, - 84, 84, 69, 68, 32, 46, 68, 45, 3, 84, 82, 73, 6, 60, 4, 70, 79, 85, 82, - 0, 3, 84, 87, 79, 163, 160, 8, 82, 2, 169, 160, 6, 8, 32, 69, 78, 67, 76, - 79, 83, 85, 2, 83, 85, 6, 42, 68, 28, 3, 84, 82, 73, 215, 1, 67, 2, 197, - 1, 3, 79, 85, 66, 2, 171, 1, 80, 6, 52, 9, 68, 69, 78, 84, 32, 65, 78, - 68, 32, 103, 80, 4, 116, 6, 68, 79, 84, 84, 69, 68, 49, 14, 85, 45, 83, - 72, 65, 80, 69, 68, 32, 79, 82, 78, 65, 77, 2, 17, 2, 76, 69, 2, 17, 2, - 32, 67, 2, 25, 4, 82, 69, 83, 67, 2, 151, 186, 1, 69, 4, 226, 165, 8, 66, - 255, 81, 68, 12, 246, 208, 4, 83, 206, 144, 1, 67, 98, 78, 222, 160, 3, - 65, 239, 1, 86, 26, 74, 65, 94, 86, 226, 246, 5, 85, 210, 200, 1, 73, - 206, 134, 2, 69, 3, 79, 10, 184, 247, 5, 10, 76, 84, 69, 82, 78, 65, 84, - 69, 32, 85, 158, 207, 3, 65, 2, 73, 3, 85, 4, 11, 79, 4, 33, 6, 67, 65, - 76, 73, 67, 32, 4, 167, 226, 5, 82, 16, 56, 5, 66, 76, 65, 67, 75, 1, 5, - 87, 72, 73, 84, 69, 8, 11, 32, 8, 70, 82, 24, 3, 76, 69, 70, 12, 4, 68, - 79, 87, 78, 1, 2, 85, 80, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 225, 92, 6, - 32, 80, 79, 73, 78, 84, 194, 10, 96, 8, 87, 82, 73, 84, 73, 78, 71, 32, - 217, 238, 1, 10, 32, 79, 70, 32, 84, 72, 69, 32, 72, 79, 192, 10, 136, 3, - 4, 65, 73, 82, 32, 192, 1, 2, 66, 82, 102, 67, 138, 1, 68, 218, 4, 69, - 242, 6, 70, 244, 3, 4, 87, 65, 76, 76, 138, 1, 72, 238, 77, 76, 224, 6, - 2, 77, 79, 222, 42, 78, 230, 1, 82, 202, 7, 83, 162, 5, 84, 180, 10, 5, - 71, 82, 65, 83, 80, 184, 5, 30, 85, 80, 80, 69, 82, 32, 66, 79, 68, 89, - 32, 84, 73, 76, 84, 73, 78, 71, 32, 70, 82, 79, 77, 32, 72, 73, 80, 32, - 74, 79, 211, 179, 4, 80, 8, 48, 4, 66, 76, 79, 87, 29, 4, 83, 85, 67, 75, - 4, 58, 32, 195, 207, 4, 73, 4, 30, 32, 61, 3, 73, 78, 71, 2, 157, 158, 1, - 10, 83, 77, 65, 76, 76, 32, 82, 79, 84, 65, 2, 251, 190, 8, 32, 10, 52, - 5, 69, 65, 84, 72, 32, 149, 170, 1, 2, 85, 83, 4, 152, 163, 2, 2, 69, 88, - 1, 2, 73, 78, 10, 40, 6, 72, 69, 69, 75, 83, 32, 63, 79, 6, 170, 107, 83, - 146, 38, 78, 153, 204, 3, 4, 80, 85, 70, 70, 4, 246, 236, 8, 76, 211, 46, - 77, 28, 108, 15, 82, 69, 65, 77, 89, 32, 69, 89, 69, 66, 82, 79, 87, 83, - 32, 165, 1, 7, 89, 78, 65, 77, 73, 67, 32, 8, 64, 4, 68, 79, 87, 78, 0, - 2, 85, 80, 29, 4, 78, 69, 85, 84, 2, 169, 143, 1, 2, 32, 78, 4, 21, 3, - 82, 65, 76, 4, 11, 32, 4, 194, 58, 68, 247, 255, 8, 85, 20, 180, 1, 11, - 69, 86, 69, 82, 89, 32, 79, 84, 72, 69, 82, 30, 70, 22, 83, 144, 122, 9, - 65, 82, 82, 79, 87, 72, 69, 65, 68, 138, 39, 82, 232, 186, 3, 2, 84, 69, - 21, 4, 71, 82, 65, 68, 2, 137, 161, 8, 2, 32, 84, 2, 135, 241, 5, 65, 6, - 68, 11, 73, 77, 85, 76, 84, 65, 78, 69, 79, 85, 83, 235, 157, 8, 76, 5, - 251, 151, 1, 32, 54, 64, 2, 89, 69, 188, 208, 4, 4, 88, 67, 73, 84, 167, - 216, 3, 65, 50, 166, 1, 32, 56, 15, 66, 82, 79, 87, 83, 32, 83, 84, 82, - 65, 73, 71, 72, 84, 32, 44, 5, 71, 65, 90, 69, 45, 140, 2, 7, 76, 65, 83, - 72, 69, 83, 32, 65, 2, 83, 32, 6, 140, 151, 1, 5, 66, 76, 73, 78, 75, - 131, 224, 4, 87, 6, 186, 53, 68, 170, 84, 78, 207, 171, 8, 85, 18, 100, - 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, 25, 10, 87, 65, 76, 76, - 80, 76, 65, 78, 69, 32, 8, 82, 83, 207, 45, 67, 10, 18, 67, 43, 83, 4, - 254, 45, 85, 241, 95, 3, 73, 82, 67, 6, 37, 7, 84, 82, 65, 73, 71, 72, - 84, 7, 11, 32, 4, 250, 163, 1, 65, 55, 68, 6, 130, 51, 68, 196, 154, 4, - 4, 70, 76, 85, 84, 179, 229, 4, 85, 14, 112, 5, 72, 65, 76, 70, 32, 26, - 67, 28, 4, 87, 73, 68, 69, 246, 92, 79, 233, 244, 3, 6, 83, 81, 85, 69, - 69, 90, 4, 22, 67, 147, 93, 79, 2, 205, 229, 2, 2, 76, 79, 4, 162, 67, - 32, 233, 61, 4, 78, 73, 78, 71, 38, 204, 1, 28, 65, 67, 69, 32, 68, 73, - 82, 69, 67, 84, 73, 79, 78, 32, 80, 79, 83, 73, 84, 73, 79, 78, 32, 78, - 79, 83, 69, 32, 138, 1, 73, 82, 76, 176, 1, 8, 79, 82, 69, 72, 69, 65, - 68, 32, 163, 225, 6, 85, 6, 88, 10, 85, 80, 32, 79, 82, 32, 68, 79, 87, - 78, 13, 8, 70, 79, 82, 87, 65, 82, 68, 32, 5, 11, 32, 2, 169, 212, 4, 3, - 84, 73, 76, 12, 248, 210, 3, 11, 76, 76, 32, 77, 79, 68, 73, 70, 73, 69, - 82, 171, 163, 2, 78, 12, 44, 4, 73, 67, 75, 32, 29, 3, 79, 79, 82, 10, - 174, 141, 1, 76, 35, 83, 2, 169, 176, 8, 20, 80, 76, 65, 78, 69, 32, 83, - 72, 79, 85, 76, 68, 69, 82, 32, 72, 73, 80, 32, 77, 6, 182, 89, 87, 222, - 38, 67, 47, 78, 156, 4, 34, 65, 137, 75, 3, 69, 65, 68, 140, 4, 36, 3, - 78, 68, 45, 211, 242, 8, 73, 138, 4, 92, 5, 65, 78, 71, 76, 69, 138, 5, - 67, 150, 10, 70, 186, 42, 72, 241, 12, 4, 79, 86, 65, 76, 37, 11, 32, 34, - 188, 1, 5, 73, 78, 68, 69, 88, 176, 1, 7, 76, 73, 84, 84, 76, 69, 32, - 136, 1, 5, 82, 73, 78, 71, 32, 173, 66, 18, 77, 73, 68, 68, 76, 69, 32, - 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, 17, 11, 32, 14, 128, 1, 7, - 77, 73, 68, 68, 76, 69, 32, 228, 24, 11, 82, 73, 78, 71, 32, 76, 73, 84, - 84, 76, 69, 241, 42, 5, 84, 72, 85, 77, 66, 4, 178, 70, 76, 183, 144, 8, - 82, 8, 44, 5, 73, 78, 68, 69, 88, 135, 167, 9, 85, 7, 145, 24, 18, 32, - 84, 72, 85, 77, 66, 32, 73, 78, 68, 69, 88, 32, 84, 72, 85, 77, 66, 4, - 108, 22, 68, 79, 87, 78, 32, 77, 73, 68, 68, 76, 69, 32, 84, 72, 85, 77, - 66, 32, 73, 78, 68, 69, 159, 68, 76, 2, 207, 231, 7, 88, 88, 64, 5, 73, - 82, 67, 76, 69, 184, 3, 3, 76, 65, 87, 183, 2, 85, 35, 11, 32, 32, 116, - 5, 73, 78, 68, 69, 88, 200, 1, 7, 76, 73, 84, 84, 76, 69, 32, 36, 7, 77, - 73, 68, 68, 76, 69, 32, 167, 64, 82, 21, 11, 32, 18, 72, 6, 77, 73, 68, - 68, 76, 69, 198, 26, 72, 246, 38, 82, 195, 158, 8, 66, 13, 11, 32, 10, - 64, 5, 67, 82, 79, 83, 83, 194, 64, 84, 90, 76, 183, 144, 8, 82, 4, 138, - 65, 32, 167, 155, 8, 69, 4, 170, 253, 7, 73, 239, 164, 1, 85, 6, 252, 47, - 10, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 247, 241, 8, 85, 17, 11, 32, - 14, 144, 2, 28, 77, 73, 68, 68, 76, 69, 32, 82, 73, 78, 71, 32, 76, 73, - 84, 84, 76, 69, 32, 67, 79, 78, 74, 79, 73, 78, 69, 68, 176, 49, 2, 70, - 79, 194, 11, 78, 170, 1, 84, 237, 118, 23, 73, 78, 68, 69, 88, 32, 84, - 72, 85, 77, 66, 32, 67, 85, 82, 86, 69, 32, 84, 72, 85, 77, 66, 5, 163, - 181, 1, 32, 38, 46, 80, 149, 2, 6, 82, 76, 73, 67, 85, 69, 31, 11, 32, - 28, 188, 1, 5, 73, 78, 68, 69, 88, 56, 19, 70, 73, 86, 69, 32, 70, 73, - 78, 71, 69, 82, 83, 32, 83, 80, 82, 69, 65, 68, 196, 50, 7, 77, 73, 68, - 68, 76, 69, 32, 18, 79, 214, 7, 78, 171, 1, 84, 9, 11, 32, 6, 40, 5, 84, - 72, 85, 77, 66, 247, 58, 82, 5, 215, 46, 32, 9, 11, 32, 6, 72, 5, 73, 78, - 68, 69, 88, 0, 6, 77, 73, 68, 68, 76, 69, 195, 71, 79, 2, 133, 204, 8, - 13, 32, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, 32, 172, 2, 44, 3, - 73, 83, 84, 177, 33, 3, 76, 65, 84, 247, 1, 11, 32, 244, 1, 160, 2, 5, - 73, 78, 68, 69, 88, 192, 16, 7, 76, 73, 84, 84, 76, 69, 32, 196, 2, 7, - 77, 73, 68, 68, 76, 69, 32, 200, 4, 5, 82, 73, 78, 71, 32, 132, 2, 5, 84, - 72, 85, 77, 66, 138, 2, 72, 205, 9, 22, 70, 79, 85, 82, 32, 70, 73, 78, - 71, 69, 82, 83, 32, 67, 79, 78, 74, 79, 73, 78, 69, 68, 137, 1, 11, 32, - 134, 1, 232, 1, 4, 66, 69, 78, 84, 36, 6, 72, 73, 78, 71, 69, 68, 76, 6, - 77, 73, 68, 68, 76, 69, 168, 7, 2, 67, 85, 64, 6, 84, 72, 85, 77, 66, 32, - 160, 5, 16, 85, 80, 32, 77, 73, 68, 68, 76, 69, 32, 72, 73, 78, 71, 69, - 68, 151, 4, 82, 5, 217, 45, 5, 32, 79, 86, 69, 82, 9, 11, 32, 6, 252, 20, - 8, 77, 73, 68, 68, 76, 69, 32, 85, 251, 230, 7, 76, 71, 11, 32, 68, 236, - 1, 4, 66, 69, 78, 84, 42, 67, 244, 1, 10, 85, 80, 32, 83, 80, 82, 69, 65, - 68, 32, 100, 6, 72, 73, 78, 71, 69, 68, 46, 82, 80, 5, 84, 72, 85, 77, - 66, 188, 28, 14, 83, 84, 82, 65, 73, 71, 72, 84, 32, 84, 72, 85, 77, 66, - 231, 17, 76, 5, 229, 92, 6, 32, 84, 72, 85, 77, 66, 28, 68, 8, 79, 78, - 74, 79, 73, 78, 69, 68, 233, 1, 4, 82, 79, 83, 83, 23, 11, 32, 20, 144, - 1, 6, 67, 85, 80, 80, 69, 68, 28, 6, 84, 72, 85, 77, 66, 32, 68, 5, 72, - 73, 78, 71, 69, 166, 22, 73, 165, 7, 6, 77, 73, 68, 68, 76, 69, 5, 11, - 32, 2, 203, 27, 84, 8, 160, 1, 4, 83, 73, 68, 69, 235, 61, 70, 6, 22, 69, - 163, 47, 32, 4, 223, 15, 68, 5, 241, 30, 7, 32, 83, 80, 82, 69, 65, 68, - 8, 32, 3, 73, 78, 71, 247, 19, 65, 7, 11, 32, 4, 138, 36, 67, 199, 168, - 8, 66, 19, 11, 32, 16, 64, 6, 65, 78, 71, 76, 69, 68, 22, 67, 106, 72, - 231, 202, 8, 66, 5, 207, 188, 5, 32, 6, 74, 85, 230, 7, 73, 241, 25, 10, - 79, 78, 74, 79, 73, 78, 69, 68, 32, 72, 2, 217, 174, 4, 2, 80, 80, 4, - 194, 33, 73, 233, 26, 2, 79, 79, 40, 164, 1, 7, 65, 78, 71, 76, 69, 68, - 32, 46, 67, 220, 1, 14, 70, 79, 82, 87, 65, 82, 68, 32, 73, 78, 68, 69, - 88, 32, 32, 4, 72, 79, 79, 75, 53, 4, 83, 73, 68, 69, 4, 128, 1, 2, 73, - 78, 1, 3, 79, 85, 84, 12, 36, 5, 73, 82, 67, 76, 69, 15, 85, 5, 47, 68, - 8, 32, 4, 80, 80, 69, 68, 39, 82, 2, 221, 40, 5, 32, 77, 73, 68, 68, 6, - 56, 9, 86, 69, 32, 84, 72, 85, 77, 66, 32, 143, 37, 76, 4, 206, 160, 1, - 73, 219, 164, 4, 85, 4, 218, 84, 83, 183, 242, 7, 66, 7, 157, 8, 9, 69, - 68, 32, 77, 73, 68, 68, 76, 69, 15, 11, 32, 12, 92, 6, 73, 78, 68, 69, - 88, 32, 156, 13, 5, 84, 72, 85, 77, 66, 193, 8, 4, 66, 79, 84, 72, 4, 26, - 72, 183, 197, 8, 66, 2, 207, 212, 7, 73, 7, 37, 7, 32, 84, 72, 85, 77, - 66, 32, 4, 178, 28, 67, 251, 129, 1, 83, 22, 88, 4, 68, 79, 87, 78, 186, - 1, 84, 158, 6, 82, 130, 21, 73, 170, 167, 8, 66, 147, 67, 85, 9, 11, 32, - 6, 80, 9, 79, 84, 72, 69, 82, 83, 32, 67, 73, 25, 7, 82, 73, 80, 80, 76, - 69, 32, 2, 241, 51, 2, 82, 67, 4, 22, 67, 187, 80, 83, 2, 11, 85, 2, 185, - 166, 4, 2, 82, 86, 4, 192, 35, 6, 79, 85, 67, 72, 69, 83, 39, 72, 30, - 134, 1, 82, 28, 6, 84, 72, 85, 77, 66, 32, 138, 3, 85, 134, 1, 68, 214, - 30, 76, 161, 188, 7, 9, 66, 69, 78, 84, 32, 79, 86, 69, 82, 4, 238, 4, - 65, 235, 29, 73, 16, 80, 7, 65, 78, 71, 76, 69, 68, 32, 126, 67, 124, 4, - 72, 79, 79, 75, 151, 32, 76, 6, 60, 10, 79, 85, 84, 32, 73, 78, 68, 69, - 88, 32, 215, 1, 73, 4, 26, 67, 211, 130, 9, 85, 2, 241, 182, 2, 3, 82, - 79, 83, 6, 76, 12, 73, 82, 67, 76, 69, 68, 32, 73, 78, 68, 69, 88, 45, 3, - 85, 80, 80, 4, 11, 32, 4, 150, 21, 72, 191, 236, 8, 85, 2, 25, 4, 69, 68, - 32, 73, 2, 229, 30, 4, 78, 68, 69, 88, 4, 11, 80, 5, 175, 15, 32, 18, - 102, 68, 20, 6, 77, 73, 68, 68, 76, 69, 42, 82, 194, 29, 84, 90, 76, 182, - 188, 7, 73, 239, 164, 1, 85, 2, 223, 174, 7, 79, 7, 11, 32, 4, 206, 3, - 82, 179, 16, 67, 2, 11, 65, 2, 37, 7, 73, 83, 69, 68, 32, 75, 78, 2, 11, - 85, 2, 11, 67, 2, 211, 221, 7, 75, 35, 11, 32, 32, 148, 1, 8, 66, 69, 84, - 87, 69, 69, 78, 32, 102, 72, 20, 5, 79, 86, 69, 82, 32, 120, 4, 83, 73, - 68, 69, 100, 6, 85, 78, 68, 69, 82, 32, 223, 40, 70, 8, 80, 12, 73, 78, - 68, 69, 88, 32, 77, 73, 68, 68, 76, 69, 246, 20, 77, 159, 6, 82, 5, 151, - 70, 32, 2, 131, 156, 4, 69, 4, 52, 6, 70, 79, 85, 82, 32, 82, 161, 2, 2, - 84, 87, 2, 11, 65, 2, 133, 81, 9, 73, 83, 69, 68, 32, 75, 78, 85, 67, 6, - 11, 32, 6, 38, 68, 190, 15, 67, 199, 168, 8, 66, 2, 25, 4, 73, 65, 71, - 79, 2, 215, 241, 7, 78, 10, 54, 73, 34, 84, 48, 4, 70, 79, 85, 82, 255, - 22, 76, 2, 161, 7, 4, 78, 68, 69, 88, 4, 34, 87, 13, 4, 72, 82, 69, 69, - 2, 11, 79, 2, 153, 235, 7, 5, 32, 70, 73, 78, 71, 55, 11, 32, 52, 194, 1, - 70, 172, 3, 4, 72, 69, 69, 76, 192, 1, 6, 83, 80, 76, 73, 84, 32, 236, 1, - 6, 84, 72, 85, 77, 66, 32, 153, 231, 4, 16, 66, 69, 84, 87, 69, 69, 78, - 32, 80, 65, 76, 77, 32, 70, 65, 67, 24, 140, 1, 18, 73, 86, 69, 32, 70, - 73, 78, 71, 69, 82, 83, 32, 83, 80, 82, 69, 65, 68, 165, 1, 11, 79, 85, - 82, 32, 70, 73, 78, 71, 69, 82, 83, 15, 11, 32, 12, 68, 6, 72, 73, 78, - 71, 69, 68, 42, 84, 186, 2, 70, 143, 176, 8, 66, 7, 11, 32, 4, 190, 4, - 84, 151, 15, 78, 2, 205, 35, 6, 72, 85, 77, 66, 32, 70, 11, 11, 32, 8, - 72, 9, 67, 79, 78, 74, 79, 73, 78, 69, 68, 154, 8, 72, 175, 169, 8, 66, - 5, 153, 216, 8, 3, 32, 83, 80, 11, 11, 32, 8, 96, 19, 70, 73, 86, 69, 32, - 70, 73, 78, 71, 69, 82, 83, 32, 83, 80, 82, 69, 65, 68, 151, 2, 84, 7, - 11, 32, 4, 26, 70, 143, 176, 8, 66, 2, 21, 3, 79, 85, 82, 2, 167, 1, 32, - 10, 72, 6, 67, 69, 78, 84, 82, 69, 96, 5, 73, 78, 68, 69, 88, 171, 16, - 76, 7, 49, 10, 32, 84, 72, 85, 77, 66, 32, 83, 73, 68, 4, 11, 69, 5, 11, - 32, 2, 203, 174, 8, 66, 2, 11, 32, 2, 11, 84, 2, 197, 135, 1, 5, 72, 85, - 77, 66, 32, 6, 130, 31, 70, 170, 104, 83, 207, 166, 7, 66, 82, 48, 4, 73, - 78, 71, 69, 181, 9, 3, 79, 79, 75, 63, 11, 32, 60, 206, 1, 70, 172, 1, 5, - 73, 78, 68, 69, 88, 196, 2, 6, 76, 73, 84, 84, 76, 69, 80, 6, 77, 73, 68, - 68, 76, 69, 30, 79, 64, 4, 82, 73, 78, 71, 124, 6, 84, 72, 85, 77, 66, - 32, 154, 6, 78, 215, 128, 4, 83, 4, 92, 19, 73, 86, 69, 32, 70, 73, 78, - 71, 69, 82, 83, 32, 83, 80, 82, 69, 65, 68, 32, 19, 79, 2, 207, 25, 79, - 2, 249, 1, 11, 85, 82, 32, 70, 73, 78, 71, 69, 82, 83, 32, 23, 11, 32, - 20, 86, 72, 40, 7, 77, 73, 68, 68, 76, 69, 32, 104, 5, 84, 72, 85, 77, - 66, 223, 9, 82, 2, 11, 73, 2, 249, 140, 4, 2, 78, 71, 6, 36, 4, 82, 73, - 78, 71, 207, 10, 76, 5, 11, 32, 2, 11, 67, 2, 21, 3, 79, 78, 74, 2, 191, - 33, 79, 11, 11, 32, 8, 34, 83, 226, 22, 79, 203, 39, 76, 4, 146, 222, 6, - 73, 187, 8, 77, 9, 11, 32, 6, 22, 73, 195, 8, 84, 4, 25, 4, 78, 68, 69, - 88, 5, 151, 8, 32, 5, 11, 32, 2, 175, 8, 82, 8, 21, 3, 80, 69, 78, 9, 11, - 32, 6, 174, 7, 78, 171, 1, 84, 5, 97, 22, 32, 68, 79, 87, 78, 32, 73, 78, - 68, 69, 88, 32, 84, 72, 85, 77, 66, 32, 72, 79, 79, 75, 2, 205, 77, 2, - 32, 77, 6, 68, 9, 66, 69, 84, 87, 69, 69, 78, 32, 77, 53, 4, 83, 73, 68, - 69, 2, 29, 5, 73, 68, 68, 76, 69, 2, 235, 134, 4, 32, 5, 33, 6, 32, 84, - 79, 85, 67, 72, 2, 145, 194, 7, 3, 73, 78, 71, 21, 11, 32, 18, 172, 1, 4, - 67, 85, 82, 76, 28, 18, 73, 78, 68, 69, 88, 32, 82, 73, 78, 71, 32, 76, - 73, 84, 84, 76, 69, 32, 48, 7, 77, 73, 68, 68, 76, 69, 32, 221, 2, 4, 82, - 73, 78, 71, 2, 229, 132, 5, 2, 73, 67, 6, 254, 159, 5, 85, 234, 203, 2, - 79, 239, 41, 73, 8, 104, 21, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, - 32, 67, 79, 78, 74, 79, 73, 78, 69, 68, 139, 2, 84, 7, 231, 213, 2, 32, - 17, 11, 32, 14, 130, 1, 76, 66, 78, 78, 82, 94, 84, 136, 154, 2, 15, 70, - 73, 86, 69, 32, 70, 73, 78, 71, 69, 82, 83, 32, 83, 80, 131, 162, 5, 73, - 2, 21, 3, 73, 84, 84, 2, 17, 2, 76, 69, 2, 215, 236, 7, 32, 2, 11, 79, 2, - 11, 32, 2, 11, 84, 2, 11, 72, 2, 209, 167, 7, 2, 85, 77, 2, 11, 73, 2, - 21, 3, 78, 71, 32, 2, 11, 76, 2, 11, 73, 2, 11, 84, 2, 239, 191, 7, 84, - 4, 29, 5, 72, 85, 77, 66, 32, 4, 206, 14, 70, 171, 104, 83, 17, 11, 32, - 14, 56, 8, 77, 79, 86, 69, 77, 69, 78, 84, 199, 221, 7, 82, 12, 26, 45, - 207, 144, 7, 32, 10, 100, 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, - 25, 10, 87, 65, 76, 76, 80, 76, 65, 78, 69, 32, 4, 62, 67, 235, 40, 83, - 6, 38, 67, 28, 2, 84, 73, 207, 40, 83, 2, 241, 225, 7, 2, 85, 82, 2, 239, - 193, 8, 76, 38, 50, 73, 225, 3, 7, 79, 67, 65, 84, 73, 79, 78, 22, 32, 3, - 77, 66, 32, 171, 1, 80, 16, 56, 4, 67, 79, 77, 66, 29, 6, 76, 69, 78, 71, - 84, 72, 2, 189, 176, 5, 2, 73, 78, 14, 11, 45, 14, 202, 220, 8, 49, 2, - 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 6, 58, 32, 153, 1, 9, 83, 32, 80, - 82, 69, 83, 83, 69, 68, 4, 116, 12, 76, 79, 87, 69, 82, 32, 79, 86, 69, - 82, 32, 85, 213, 193, 7, 11, 85, 80, 80, 69, 82, 32, 79, 86, 69, 82, 32, - 2, 11, 80, 2, 231, 156, 8, 80, 2, 29, 5, 32, 84, 79, 71, 69, 2, 11, 84, - 2, 167, 156, 8, 72, 16, 22, 32, 215, 1, 45, 12, 140, 1, 2, 72, 69, 48, 3, - 84, 79, 82, 152, 217, 6, 3, 68, 69, 80, 0, 3, 87, 73, 68, 241, 106, 10, - 76, 73, 77, 66, 83, 32, 68, 73, 71, 73, 4, 216, 13, 4, 65, 68, 32, 78, - 175, 160, 8, 73, 2, 167, 185, 8, 83, 4, 52, 5, 70, 76, 79, 79, 82, 1, 4, - 87, 65, 76, 76, 2, 149, 153, 8, 5, 80, 76, 65, 78, 69, 156, 3, 64, 4, 85, - 84, 72, 32, 161, 5, 7, 86, 69, 77, 69, 78, 84, 45, 54, 186, 1, 67, 88, 5, - 70, 82, 79, 87, 78, 0, 5, 83, 77, 73, 76, 69, 56, 4, 75, 73, 83, 83, 36, - 5, 79, 80, 69, 78, 32, 192, 1, 5, 84, 69, 78, 83, 69, 165, 29, 5, 87, 82, - 73, 78, 75, 8, 48, 6, 76, 79, 83, 69, 68, 32, 243, 166, 7, 79, 6, 222, 2, - 70, 142, 38, 67, 47, 78, 7, 11, 32, 4, 22, 79, 203, 1, 87, 2, 179, 131, - 7, 80, 7, 11, 32, 4, 166, 1, 87, 83, 70, 18, 100, 4, 79, 86, 65, 76, 0, - 9, 82, 69, 67, 84, 65, 78, 71, 76, 69, 42, 87, 82, 70, 139, 131, 7, 67, - 7, 11, 32, 4, 26, 87, 183, 129, 7, 89, 2, 25, 4, 82, 73, 78, 75, 2, 131, - 243, 3, 76, 7, 11, 32, 4, 18, 70, 43, 83, 2, 17, 2, 79, 82, 2, 155, 187, - 7, 87, 2, 17, 2, 85, 67, 2, 147, 242, 3, 75, 230, 2, 192, 1, 9, 68, 73, - 65, 71, 79, 78, 65, 76, 32, 148, 1, 11, 70, 76, 79, 79, 82, 80, 76, 65, - 78, 69, 32, 184, 14, 6, 72, 73, 78, 71, 69, 32, 189, 2, 10, 87, 65, 76, - 76, 80, 76, 65, 78, 69, 32, 32, 56, 8, 66, 69, 84, 87, 69, 69, 78, 32, - 22, 65, 31, 84, 16, 18, 65, 31, 84, 8, 229, 28, 3, 87, 65, 89, 8, 201, - 28, 6, 79, 87, 65, 82, 68, 83, 150, 1, 208, 2, 24, 65, 82, 77, 32, 67, - 73, 82, 67, 76, 69, 32, 72, 73, 84, 84, 73, 78, 71, 32, 87, 65, 76, 76, - 32, 38, 66, 34, 67, 224, 1, 8, 70, 73, 78, 71, 69, 82, 32, 67, 52, 5, 72, - 85, 77, 80, 32, 184, 3, 5, 76, 79, 79, 80, 32, 198, 1, 83, 108, 7, 84, - 82, 73, 80, 76, 69, 32, 110, 87, 206, 12, 68, 198, 2, 80, 154, 6, 90, - 159, 175, 7, 74, 12, 178, 7, 76, 154, 9, 77, 39, 83, 8, 150, 17, 79, 243, - 172, 7, 69, 28, 86, 72, 20, 5, 85, 82, 86, 69, 32, 196, 29, 5, 79, 82, - 78, 69, 82, 195, 239, 6, 82, 2, 163, 156, 7, 69, 18, 80, 4, 67, 79, 77, - 66, 158, 8, 72, 230, 15, 76, 198, 157, 2, 77, 243, 178, 1, 83, 2, 11, 73, - 2, 187, 234, 3, 78, 6, 128, 9, 6, 73, 82, 67, 76, 69, 83, 239, 20, 79, - 18, 56, 8, 72, 73, 84, 84, 73, 78, 71, 32, 167, 231, 3, 83, 16, 72, 8, - 67, 69, 73, 76, 73, 78, 71, 32, 101, 6, 70, 76, 79, 79, 82, 32, 8, 56, 5, - 76, 65, 82, 71, 69, 1, 5, 83, 77, 65, 76, 76, 4, 11, 32, 4, 238, 54, 84, - 135, 2, 68, 8, 88, 4, 83, 77, 65, 76, 12, 5, 76, 65, 82, 71, 69, 17, 7, - 84, 82, 73, 80, 76, 69, 32, 2, 11, 76, 2, 255, 18, 32, 4, 56, 5, 76, 65, - 82, 71, 69, 1, 5, 83, 77, 65, 76, 76, 2, 157, 53, 2, 32, 84, 18, 56, 8, - 72, 73, 84, 84, 73, 78, 71, 32, 239, 227, 3, 83, 16, 64, 7, 67, 69, 73, - 76, 73, 78, 71, 1, 5, 70, 76, 79, 79, 82, 8, 11, 32, 8, 22, 76, 191, 9, - 83, 4, 249, 22, 4, 65, 82, 71, 69, 12, 44, 6, 72, 65, 75, 73, 78, 71, - 243, 16, 73, 2, 21, 3, 32, 80, 65, 2, 149, 227, 3, 4, 82, 65, 76, 76, 8, - 76, 12, 65, 76, 84, 69, 82, 78, 65, 84, 73, 78, 71, 32, 138, 18, 87, 63, - 83, 4, 134, 18, 87, 159, 21, 77, 18, 80, 4, 65, 86, 69, 32, 169, 1, 11, - 82, 73, 83, 84, 32, 67, 73, 82, 67, 76, 69, 14, 30, 72, 102, 83, 171, 20, - 76, 8, 37, 7, 73, 84, 84, 73, 78, 71, 32, 8, 252, 2, 4, 67, 69, 73, 76, - 25, 5, 70, 76, 79, 79, 82, 4, 138, 251, 4, 78, 195, 193, 1, 77, 4, 209, - 5, 10, 32, 72, 73, 84, 84, 73, 78, 71, 32, 87, 14, 112, 3, 85, 80, 32, - 184, 1, 6, 68, 79, 87, 78, 32, 83, 245, 183, 5, 10, 83, 73, 68, 69, 32, - 84, 79, 32, 83, 73, 10, 40, 5, 68, 79, 87, 78, 32, 143, 1, 83, 8, 68, 8, - 65, 76, 84, 69, 82, 78, 65, 84, 230, 17, 76, 143, 203, 3, 83, 4, 21, 3, - 73, 78, 71, 4, 11, 32, 4, 190, 17, 76, 143, 203, 3, 83, 2, 219, 30, 69, - 162, 1, 148, 2, 11, 65, 82, 77, 32, 67, 73, 82, 67, 76, 69, 32, 98, 66, - 54, 67, 174, 4, 68, 100, 8, 70, 73, 78, 71, 69, 82, 32, 67, 64, 5, 72, - 85, 77, 80, 32, 60, 5, 76, 79, 79, 80, 32, 102, 80, 34, 83, 216, 1, 7, - 84, 82, 73, 80, 76, 69, 32, 218, 1, 87, 202, 2, 90, 159, 175, 7, 74, 8, - 18, 77, 39, 83, 4, 225, 13, 5, 69, 68, 73, 85, 77, 4, 11, 77, 4, 177, 13, - 3, 65, 76, 76, 12, 34, 79, 185, 13, 3, 69, 78, 68, 6, 183, 13, 88, 46, - 100, 6, 79, 82, 78, 69, 82, 32, 88, 4, 85, 82, 86, 69, 232, 11, 4, 72, - 69, 67, 75, 195, 239, 6, 82, 8, 54, 82, 194, 12, 76, 158, 152, 2, 77, - 243, 178, 1, 83, 2, 11, 79, 2, 223, 140, 5, 84, 30, 50, 32, 137, 2, 7, - 68, 32, 67, 82, 79, 83, 83, 26, 66, 72, 68, 2, 84, 72, 133, 5, 7, 81, 85, - 65, 82, 84, 69, 82, 12, 196, 5, 10, 65, 76, 70, 45, 67, 73, 82, 67, 76, - 69, 159, 14, 73, 6, 96, 2, 69, 78, 29, 18, 82, 69, 69, 45, 81, 85, 65, - 82, 84, 69, 82, 32, 67, 73, 82, 67, 76, 69, 2, 11, 32, 2, 131, 1, 83, 4, - 11, 32, 4, 242, 161, 2, 77, 243, 178, 1, 83, 8, 33, 6, 79, 85, 66, 76, - 69, 32, 8, 30, 83, 150, 4, 65, 75, 87, 2, 253, 138, 8, 3, 84, 82, 65, 6, - 32, 3, 73, 82, 67, 151, 9, 79, 4, 173, 7, 3, 76, 69, 83, 10, 142, 8, 76, - 178, 8, 72, 238, 143, 2, 77, 243, 178, 1, 83, 12, 68, 5, 83, 77, 65, 76, - 76, 142, 7, 76, 178, 8, 72, 239, 143, 2, 77, 5, 11, 32, 2, 239, 36, 68, - 6, 181, 6, 4, 69, 65, 75, 83, 12, 22, 73, 243, 36, 72, 10, 29, 5, 78, 71, - 76, 69, 32, 10, 52, 8, 83, 84, 82, 65, 73, 71, 72, 84, 207, 1, 87, 8, 11, - 32, 8, 42, 76, 198, 157, 2, 77, 243, 178, 1, 83, 4, 25, 4, 65, 82, 71, - 69, 5, 191, 149, 8, 83, 8, 26, 65, 74, 87, 63, 83, 4, 49, 10, 76, 84, 69, - 82, 78, 65, 84, 73, 78, 71, 5, 17, 2, 32, 87, 2, 29, 5, 82, 73, 83, 84, - 32, 2, 237, 139, 7, 2, 70, 76, 2, 37, 7, 84, 82, 65, 73, 71, 72, 84, 2, - 171, 20, 32, 26, 104, 4, 65, 86, 69, 32, 185, 1, 17, 82, 73, 83, 84, 32, - 67, 73, 82, 67, 76, 69, 32, 70, 82, 79, 78, 84, 22, 108, 6, 67, 85, 82, - 86, 69, 32, 140, 1, 13, 68, 73, 65, 71, 79, 78, 65, 76, 32, 80, 65, 84, - 72, 235, 8, 72, 12, 48, 4, 68, 79, 85, 66, 1, 4, 84, 82, 73, 80, 6, 85, - 2, 76, 69, 4, 11, 32, 4, 222, 30, 68, 43, 83, 6, 29, 5, 73, 71, 90, 65, - 71, 6, 11, 32, 6, 42, 76, 158, 152, 2, 77, 243, 178, 1, 83, 2, 143, 240, - 6, 65, 10, 40, 4, 79, 83, 69, 32, 215, 253, 6, 69, 8, 26, 67, 46, 78, 35, - 87, 2, 11, 79, 2, 157, 157, 7, 3, 78, 84, 65, 2, 161, 162, 7, 3, 69, 85, - 84, 4, 36, 2, 73, 71, 21, 3, 82, 73, 78, 2, 191, 130, 5, 71, 2, 171, 130, - 5, 75, 72, 56, 7, 79, 84, 65, 84, 73, 79, 78, 225, 22, 2, 85, 66, 66, 60, - 10, 32, 77, 79, 68, 73, 70, 73, 69, 82, 45, 155, 1, 45, 30, 82, 49, 154, - 169, 8, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 14, 150, - 169, 8, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 36, 104, 11, 70, - 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, 133, 2, 10, 87, 65, 76, 76, 80, - 76, 65, 78, 69, 32, 18, 100, 4, 68, 79, 85, 66, 0, 4, 83, 73, 78, 71, 21, - 11, 65, 76, 84, 69, 82, 78, 65, 84, 73, 78, 71, 6, 17, 2, 76, 69, 7, 45, - 9, 32, 72, 73, 84, 84, 73, 78, 71, 32, 4, 32, 2, 67, 69, 33, 2, 70, 76, - 2, 11, 73, 2, 147, 213, 7, 76, 2, 155, 237, 7, 79, 18, 88, 8, 65, 76, 84, - 69, 82, 78, 65, 84, 44, 4, 68, 79, 85, 66, 1, 4, 83, 73, 78, 71, 6, 72, - 4, 73, 78, 71, 32, 191, 164, 8, 69, 6, 17, 2, 76, 69, 7, 11, 32, 4, 11, - 72, 4, 11, 73, 4, 33, 6, 84, 84, 73, 78, 71, 32, 4, 44, 2, 67, 72, 21, 5, - 70, 82, 79, 78, 84, 2, 135, 220, 4, 69, 2, 253, 158, 6, 2, 32, 87, 30, - 172, 1, 8, 72, 79, 85, 76, 68, 69, 82, 32, 196, 1, 7, 81, 85, 69, 69, 90, - 69, 32, 240, 1, 7, 85, 82, 70, 65, 67, 69, 32, 240, 10, 5, 84, 82, 73, - 75, 69, 243, 221, 4, 69, 6, 100, 4, 72, 73, 80, 32, 173, 232, 3, 15, 84, - 73, 76, 84, 73, 78, 71, 32, 70, 82, 79, 77, 32, 87, 65, 4, 48, 4, 80, 79, - 83, 73, 181, 135, 7, 2, 83, 80, 2, 11, 84, 2, 177, 242, 4, 2, 73, 79, 12, - 48, 6, 70, 76, 73, 67, 75, 32, 18, 76, 35, 83, 2, 211, 16, 65, 4, 129, 1, - 4, 65, 82, 71, 69, 6, 34, 69, 65, 4, 77, 65, 76, 76, 2, 17, 2, 81, 85, 2, - 21, 3, 69, 78, 84, 2, 207, 149, 7, 73, 4, 11, 32, 4, 214, 11, 77, 187, 4, - 83, 4, 22, 83, 135, 11, 66, 2, 201, 204, 1, 4, 89, 77, 66, 79, 82, 58, - 69, 218, 2, 79, 137, 8, 6, 82, 65, 86, 69, 76, 45, 20, 76, 3, 69, 84, 72, - 185, 1, 11, 78, 83, 69, 32, 67, 72, 69, 69, 75, 83, 32, 15, 11, 32, 12, - 56, 3, 79, 78, 32, 86, 77, 177, 5, 4, 66, 73, 84, 69, 8, 56, 4, 76, 73, - 80, 83, 1, 6, 84, 79, 78, 71, 85, 69, 5, 11, 32, 2, 11, 77, 2, 245, 180, - 3, 2, 79, 86, 6, 50, 77, 252, 162, 4, 2, 72, 73, 131, 222, 2, 76, 2, 157, - 149, 3, 2, 73, 68, 28, 76, 5, 78, 71, 85, 69, 32, 196, 4, 4, 82, 83, 79, - 45, 129, 2, 2, 85, 67, 16, 192, 2, 7, 67, 69, 78, 84, 82, 69, 32, 52, 14, - 73, 78, 83, 73, 68, 69, 32, 77, 79, 85, 84, 72, 32, 82, 36, 4, 84, 73, - 80, 32, 88, 7, 76, 73, 67, 75, 73, 78, 71, 128, 244, 6, 14, 83, 84, 73, - 67, 75, 73, 78, 71, 32, 79, 85, 84, 32, 70, 133, 159, 1, 17, 77, 79, 86, - 69, 83, 32, 65, 71, 65, 73, 78, 83, 84, 32, 67, 72, 69, 4, 214, 1, 73, - 193, 166, 3, 5, 83, 84, 73, 67, 75, 2, 225, 183, 3, 4, 69, 76, 65, 88, 4, - 84, 7, 66, 69, 84, 87, 69, 69, 78, 41, 10, 84, 79, 85, 67, 72, 73, 78, - 71, 32, 73, 2, 11, 32, 2, 189, 246, 4, 2, 76, 73, 2, 237, 167, 3, 5, 78, - 83, 73, 68, 69, 6, 120, 10, 87, 65, 76, 76, 80, 76, 65, 78, 69, 32, 169, - 186, 3, 14, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, 84, 87, 73, 4, - 108, 8, 67, 85, 82, 86, 69, 68, 32, 66, 241, 200, 1, 13, 83, 84, 82, 65, - 73, 71, 72, 84, 32, 83, 84, 82, 69, 2, 139, 134, 7, 69, 6, 11, 72, 6, 11, - 32, 6, 30, 66, 34, 77, 187, 4, 83, 2, 157, 194, 6, 3, 69, 84, 87, 2, 149, - 2, 3, 85, 76, 84, 34, 100, 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, - 32, 29, 10, 87, 65, 76, 76, 80, 76, 65, 78, 69, 32, 14, 178, 1, 82, 151, - 2, 83, 20, 72, 11, 65, 82, 77, 32, 83, 80, 73, 82, 65, 76, 32, 78, 82, - 151, 2, 83, 6, 30, 84, 134, 2, 68, 43, 83, 2, 11, 82, 2, 11, 73, 2, 183, - 239, 6, 80, 12, 41, 8, 79, 84, 65, 84, 73, 79, 78, 45, 12, 52, 5, 70, 76, - 79, 79, 82, 1, 4, 87, 65, 76, 76, 6, 33, 6, 80, 76, 65, 78, 69, 32, 6, - 26, 65, 54, 68, 43, 83, 2, 29, 5, 76, 84, 69, 82, 78, 2, 247, 180, 3, 65, - 2, 17, 2, 79, 85, 2, 183, 237, 6, 66, 2, 139, 237, 6, 73, 2, 11, 72, 2, - 247, 178, 3, 65, 2, 11, 73, 2, 243, 249, 6, 78, 10, 100, 6, 65, 66, 79, - 86, 69, 32, 162, 1, 79, 197, 133, 2, 10, 77, 73, 78, 85, 83, 32, 83, 73, - 77, 73, 4, 60, 7, 71, 82, 69, 65, 84, 69, 82, 1, 4, 76, 69, 83, 83, 2, - 37, 7, 45, 84, 72, 65, 78, 32, 65, 2, 25, 4, 66, 79, 86, 69, 2, 157, 228, - 1, 2, 32, 69, 4, 203, 252, 5, 82, 250, 1, 64, 3, 71, 76, 69, 212, 4, 5, - 72, 65, 76, 65, 32, 135, 102, 69, 20, 50, 32, 221, 137, 8, 6, 45, 83, 72, - 73, 70, 84, 16, 138, 1, 67, 0, 9, 71, 82, 65, 80, 72, 73, 67, 32, 67, - 116, 2, 72, 73, 74, 76, 36, 4, 82, 73, 71, 72, 145, 208, 4, 4, 83, 72, - 73, 70, 2, 41, 8, 72, 65, 82, 65, 67, 84, 69, 82, 2, 37, 7, 32, 73, 78, - 84, 82, 79, 68, 2, 11, 85, 2, 143, 203, 7, 67, 2, 25, 4, 71, 72, 45, 82, - 2, 185, 1, 7, 69, 86, 69, 82, 83, 69, 68, 4, 32, 2, 69, 70, 109, 2, 79, - 87, 2, 49, 10, 84, 45, 80, 79, 73, 78, 84, 73, 78, 71, 2, 21, 3, 32, 65, - 78, 2, 17, 2, 71, 76, 2, 31, 69, 2, 17, 2, 45, 57, 2, 11, 32, 2, 11, 81, - 2, 205, 161, 2, 2, 85, 79, 228, 1, 168, 2, 8, 65, 82, 67, 72, 65, 73, 67, - 32, 224, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, - 32, 114, 76, 188, 8, 5, 83, 73, 71, 78, 32, 144, 1, 11, 86, 79, 87, 69, - 76, 32, 83, 73, 71, 78, 32, 229, 212, 2, 17, 80, 85, 78, 67, 84, 85, 65, - 84, 73, 79, 78, 32, 75, 85, 78, 68, 68, 40, 46, 68, 109, 7, 78, 85, 77, - 66, 69, 82, 32, 18, 25, 4, 73, 71, 73, 84, 18, 11, 32, 18, 146, 150, 6, - 70, 30, 83, 42, 84, 190, 83, 78, 14, 79, 199, 110, 69, 22, 150, 218, 2, - 79, 134, 189, 3, 69, 30, 70, 42, 78, 38, 83, 39, 84, 6, 18, 82, 63, 89, - 4, 56, 6, 65, 75, 65, 65, 82, 65, 245, 254, 7, 2, 69, 80, 2, 241, 254, 7, - 3, 65, 78, 83, 138, 1, 60, 6, 69, 84, 84, 69, 82, 32, 217, 171, 2, 3, 73, - 84, 72, 118, 246, 1, 65, 106, 69, 34, 73, 62, 85, 34, 77, 220, 1, 4, 68, - 65, 78, 84, 58, 79, 28, 8, 83, 65, 78, 89, 65, 75, 65, 32, 50, 70, 2, 72, - 2, 82, 2, 86, 2, 89, 32, 8, 84, 65, 65, 76, 85, 74, 65, 32, 29, 9, 75, - 65, 78, 84, 65, 74, 65, 32, 78, 34, 102, 69, 204, 1, 2, 76, 80, 144, 2, - 5, 77, 66, 65, 32, 66, 14, 65, 2, 73, 2, 85, 175, 216, 4, 89, 4, 230, 3, - 69, 175, 216, 4, 89, 12, 46, 76, 2, 82, 154, 3, 73, 175, 216, 4, 89, 4, - 11, 85, 4, 138, 3, 85, 175, 216, 4, 89, 28, 42, 65, 177, 1, 5, 85, 85, - 82, 68, 72, 22, 32, 2, 72, 65, 203, 218, 4, 89, 20, 41, 8, 65, 80, 82, - 65, 65, 78, 65, 32, 20, 70, 84, 138, 1, 68, 22, 66, 2, 67, 2, 71, 2, 74, - 2, 75, 3, 80, 4, 154, 1, 84, 15, 65, 6, 25, 4, 65, 74, 65, 32, 6, 102, - 76, 2, 78, 3, 83, 4, 86, 79, 175, 216, 4, 89, 8, 26, 68, 22, 71, 3, 74, - 4, 18, 68, 15, 65, 2, 11, 65, 2, 171, 216, 4, 89, 6, 26, 78, 21, 2, 83, - 65, 2, 89, 2, 65, 65, 4, 68, 11, 78, 89, 79, 79, 71, 65, 32, 78, 65, 65, - 75, 163, 215, 4, 89, 2, 185, 246, 7, 4, 83, 73, 75, 89, 8, 66, 65, 150, - 146, 4, 67, 193, 227, 3, 6, 86, 73, 83, 65, 82, 71, 4, 220, 113, 5, 76, - 45, 76, 65, 75, 245, 131, 7, 6, 78, 85, 83, 86, 65, 82, 34, 56, 5, 68, - 73, 71, 65, 32, 62, 71, 82, 75, 155, 2, 65, 12, 58, 71, 48, 4, 75, 79, - 77, 66, 118, 65, 26, 73, 19, 80, 4, 11, 65, 4, 236, 2, 3, 69, 84, 84, 67, - 89, 2, 11, 85, 2, 131, 244, 7, 86, 16, 52, 5, 69, 84, 84, 73, 32, 85, 4, - 79, 77, 66, 85, 6, 26, 65, 26, 73, 19, 80, 2, 213, 1, 2, 69, 68, 2, 203, - 1, 83, 2, 175, 1, 65, 10, 40, 2, 86, 65, 173, 198, 7, 2, 32, 68, 9, 29, - 5, 32, 72, 65, 65, 32, 6, 62, 65, 0, 6, 68, 73, 71, 65, 32, 65, 85, 3, - 71, 65, 89, 2, 17, 2, 69, 76, 2, 11, 65, 2, 17, 2, 45, 80, 2, 11, 73, 2, - 183, 209, 7, 76, 2, 193, 202, 5, 5, 65, 78, 85, 75, 73, 12, 84, 2, 32, - 80, 206, 2, 45, 137, 223, 5, 10, 84, 69, 69, 78, 32, 80, 79, 73, 78, 84, - 8, 140, 1, 23, 69, 84, 65, 76, 76, 69, 68, 32, 66, 76, 65, 67, 75, 32, - 65, 78, 68, 32, 87, 72, 73, 84, 69, 49, 7, 79, 73, 78, 84, 69, 68, 32, 2, - 25, 4, 32, 70, 76, 79, 2, 207, 172, 6, 82, 6, 78, 80, 150, 221, 5, 66, - 217, 129, 1, 9, 83, 84, 65, 82, 32, 87, 73, 84, 72, 2, 21, 3, 73, 78, 87, - 2, 149, 221, 5, 4, 72, 69, 69, 76, 2, 179, 194, 3, 80, 12, 46, 73, 102, - 85, 213, 215, 6, 3, 65, 84, 69, 4, 56, 8, 32, 65, 78, 68, 32, 83, 75, 73, - 163, 182, 7, 69, 2, 17, 2, 32, 66, 2, 175, 158, 7, 79, 6, 32, 2, 76, 76, - 143, 235, 7, 78, 5, 11, 32, 2, 49, 10, 65, 78, 68, 32, 67, 82, 79, 83, - 83, 66, 2, 11, 79, 2, 247, 217, 6, 78, 42, 46, 65, 198, 4, 69, 210, 1, - 73, 155, 1, 79, 14, 64, 5, 78, 84, 69, 68, 32, 253, 219, 5, 5, 86, 79, - 78, 73, 67, 12, 148, 1, 12, 69, 81, 85, 65, 76, 32, 84, 79, 32, 79, 82, - 32, 229, 1, 19, 78, 79, 82, 84, 72, 32, 65, 82, 82, 79, 87, 32, 87, 73, - 84, 72, 32, 72, 79, 8, 60, 7, 71, 82, 69, 65, 84, 69, 82, 1, 4, 76, 69, - 83, 83, 4, 29, 5, 45, 84, 72, 65, 78, 5, 11, 32, 2, 25, 4, 87, 73, 84, - 72, 2, 25, 4, 32, 68, 79, 84, 2, 17, 2, 32, 73, 2, 11, 78, 2, 11, 83, 2, - 243, 220, 5, 73, 4, 24, 2, 79, 75, 43, 82, 2, 17, 2, 69, 68, 2, 135, 155, - 5, 32, 2, 29, 5, 73, 90, 79, 78, 84, 2, 11, 65, 2, 203, 187, 5, 76, 12, - 80, 2, 69, 80, 180, 214, 7, 9, 85, 84, 72, 32, 79, 82, 32, 83, 80, 203, - 17, 68, 8, 40, 4, 73, 78, 71, 32, 191, 170, 7, 89, 6, 152, 187, 4, 8, 65, - 67, 67, 79, 77, 77, 79, 68, 190, 153, 2, 83, 255, 85, 70, 6, 76, 9, 67, - 69, 32, 79, 70, 32, 80, 73, 90, 21, 6, 71, 72, 84, 76, 89, 32, 2, 247, - 227, 7, 90, 4, 196, 162, 6, 2, 70, 82, 145, 16, 3, 83, 77, 73, 10, 18, - 80, 75, 84, 6, 140, 186, 4, 9, 73, 78, 71, 32, 76, 65, 82, 71, 69, 199, - 171, 3, 69, 4, 26, 32, 243, 228, 7, 72, 2, 11, 77, 2, 193, 203, 6, 3, 65, - 67, 72, 148, 1, 34, 65, 210, 12, 73, 159, 8, 79, 114, 32, 2, 76, 76, 227, - 212, 6, 83, 112, 30, 32, 193, 11, 2, 69, 82, 108, 178, 2, 65, 44, 3, 66, + 69, 82, 40, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 205, 178, 3, 5, 32, 79, + 86, 69, 82, 2, 21, 3, 32, 83, 84, 2, 227, 146, 10, 69, 2, 25, 4, 32, 79, + 80, 69, 2, 247, 133, 9, 78, 4, 26, 73, 191, 145, 10, 85, 2, 131, 146, 10, + 77, 159, 14, 174, 1, 68, 168, 19, 2, 71, 78, 204, 181, 1, 6, 77, 73, 76, + 65, 82, 32, 158, 2, 78, 202, 24, 88, 241, 226, 6, 15, 76, 72, 79, 85, 69, + 84, 84, 69, 32, 79, 70, 32, 74, 65, 80, 200, 1, 64, 5, 68, 72, 65, 77, + 32, 133, 17, 6, 69, 87, 65, 89, 83, 32, 184, 1, 202, 1, 69, 68, 7, 76, + 69, 84, 84, 69, 82, 32, 176, 4, 15, 82, 69, 80, 69, 84, 73, 84, 73, 79, + 78, 32, 77, 65, 82, 75, 50, 83, 164, 8, 11, 86, 79, 87, 69, 76, 32, 83, + 73, 71, 78, 32, 243, 163, 6, 68, 2, 37, 7, 78, 68, 32, 79, 70, 32, 84, 2, + 189, 195, 9, 2, 69, 88, 102, 214, 1, 65, 98, 84, 186, 177, 6, 68, 158, 1, + 86, 186, 5, 85, 206, 201, 1, 73, 162, 193, 1, 78, 46, 83, 82, 66, 2, 67, + 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, 2, 82, 2, 89, 186, + 2, 69, 3, 79, 11, 72, 8, 76, 84, 69, 82, 78, 65, 84, 69, 202, 139, 10, + 65, 2, 73, 3, 85, 2, 223, 230, 9, 32, 14, 134, 1, 72, 204, 225, 5, 19, + 87, 79, 45, 67, 73, 82, 67, 76, 69, 32, 65, 76, 84, 69, 82, 78, 65, 84, + 69, 162, 225, 3, 84, 195, 71, 65, 4, 152, 202, 9, 20, 82, 69, 69, 45, 67, + 73, 82, 67, 76, 69, 32, 65, 76, 84, 69, 82, 78, 65, 84, 69, 147, 64, 65, + 6, 11, 45, 6, 174, 137, 10, 49, 2, 50, 3, 51, 44, 38, 69, 181, 7, 4, 73, + 71, 78, 32, 32, 96, 11, 67, 84, 73, 79, 78, 32, 77, 65, 82, 75, 32, 177, + 6, 8, 80, 65, 82, 65, 84, 79, 82, 32, 28, 80, 11, 68, 79, 85, 66, 76, 69, + 32, 82, 73, 78, 71, 57, 5, 87, 73, 84, 72, 32, 5, 11, 32, 2, 129, 219, 8, + 6, 87, 73, 84, 72, 32, 82, 24, 212, 1, 12, 67, 73, 82, 67, 76, 69, 83, + 32, 65, 78, 68, 32, 116, 5, 81, 85, 65, 68, 82, 0, 4, 83, 69, 80, 84, 12, + 16, 82, 65, 89, 83, 32, 65, 78, 68, 32, 68, 79, 84, 84, 69, 68, 32, 46, + 68, 45, 3, 84, 82, 73, 6, 60, 4, 70, 79, 85, 82, 0, 3, 84, 87, 79, 195, + 216, 8, 82, 2, 169, 196, 6, 8, 32, 69, 78, 67, 76, 79, 83, 85, 2, 83, 85, + 6, 42, 68, 28, 3, 84, 82, 73, 215, 1, 67, 2, 197, 1, 3, 79, 85, 66, 2, + 171, 1, 80, 6, 52, 9, 68, 69, 78, 84, 32, 65, 78, 68, 32, 103, 80, 4, + 116, 6, 68, 79, 84, 84, 69, 68, 49, 14, 85, 45, 83, 72, 65, 80, 69, 68, + 32, 79, 82, 78, 65, 77, 2, 17, 2, 76, 69, 2, 17, 2, 32, 67, 2, 25, 4, 82, + 69, 83, 67, 2, 251, 185, 1, 69, 4, 150, 224, 8, 66, 131, 81, 68, 12, 210, + 222, 4, 83, 158, 142, 1, 67, 98, 78, 234, 206, 3, 65, 239, 1, 86, 26, 74, + 65, 94, 86, 154, 172, 6, 85, 206, 201, 1, 73, 222, 137, 2, 69, 3, 79, 10, + 240, 172, 6, 10, 76, 84, 69, 82, 78, 65, 84, 69, 32, 85, 170, 211, 3, 65, + 2, 73, 3, 85, 4, 11, 79, 4, 33, 6, 67, 65, 76, 73, 67, 32, 4, 199, 172, + 6, 82, 16, 56, 5, 66, 76, 65, 67, 75, 1, 5, 87, 72, 73, 84, 69, 8, 11, + 32, 8, 70, 82, 24, 3, 76, 69, 70, 12, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, + 21, 3, 73, 71, 72, 2, 11, 84, 2, 225, 92, 6, 32, 80, 79, 73, 78, 84, 194, + 10, 96, 8, 87, 82, 73, 84, 73, 78, 71, 32, 133, 239, 1, 10, 32, 79, 70, + 32, 84, 72, 69, 32, 72, 79, 192, 10, 136, 3, 4, 65, 73, 82, 32, 192, 1, + 2, 66, 82, 102, 67, 138, 1, 68, 218, 4, 69, 242, 6, 70, 244, 3, 4, 87, + 65, 76, 76, 138, 1, 72, 234, 77, 76, 212, 6, 2, 77, 79, 222, 42, 78, 218, + 1, 82, 202, 7, 83, 162, 5, 84, 180, 10, 5, 71, 82, 65, 83, 80, 184, 5, + 30, 85, 80, 80, 69, 82, 32, 66, 79, 68, 89, 32, 84, 73, 76, 84, 73, 78, + 71, 32, 70, 82, 79, 77, 32, 72, 73, 80, 32, 74, 79, 195, 197, 4, 80, 8, + 48, 4, 66, 76, 79, 87, 29, 4, 83, 85, 67, 75, 4, 58, 32, 155, 221, 4, 73, + 4, 30, 32, 61, 3, 73, 78, 71, 2, 129, 158, 1, 10, 83, 77, 65, 76, 76, 32, + 82, 79, 84, 65, 2, 167, 248, 8, 32, 10, 52, 5, 69, 65, 84, 72, 32, 249, + 169, 1, 2, 85, 83, 4, 144, 164, 2, 2, 69, 88, 1, 2, 73, 78, 10, 40, 6, + 72, 69, 69, 75, 83, 32, 63, 79, 6, 154, 107, 83, 146, 38, 78, 225, 217, + 3, 4, 80, 85, 70, 70, 4, 174, 166, 9, 76, 223, 46, 77, 28, 108, 15, 82, + 69, 65, 77, 89, 32, 69, 89, 69, 66, 82, 79, 87, 83, 32, 165, 1, 7, 89, + 78, 65, 77, 73, 67, 32, 8, 64, 4, 68, 79, 87, 78, 0, 2, 85, 80, 29, 4, + 78, 69, 85, 84, 2, 153, 143, 1, 2, 32, 78, 4, 21, 3, 82, 65, 76, 4, 11, + 32, 4, 194, 58, 68, 187, 185, 9, 85, 20, 180, 1, 11, 69, 86, 69, 82, 89, + 32, 79, 84, 72, 69, 82, 30, 70, 22, 83, 128, 122, 9, 65, 82, 82, 79, 87, + 72, 69, 65, 68, 254, 38, 82, 188, 200, 3, 2, 84, 69, 21, 4, 71, 82, 65, + 68, 2, 181, 218, 8, 2, 32, 84, 2, 151, 136, 6, 65, 6, 68, 11, 73, 77, 85, + 76, 84, 65, 78, 69, 79, 85, 83, 151, 215, 8, 76, 5, 223, 151, 1, 32, 54, + 64, 2, 89, 69, 244, 221, 4, 4, 88, 67, 73, 84, 155, 132, 4, 65, 50, 166, + 1, 32, 56, 15, 66, 82, 79, 87, 83, 32, 83, 84, 82, 65, 73, 71, 72, 84, + 32, 44, 5, 71, 65, 90, 69, 45, 140, 2, 7, 76, 65, 83, 72, 69, 83, 32, 65, + 2, 83, 32, 6, 240, 150, 1, 5, 66, 76, 73, 78, 75, 175, 247, 4, 87, 6, + 186, 53, 68, 154, 84, 78, 163, 229, 8, 85, 18, 100, 11, 70, 76, 79, 79, + 82, 80, 76, 65, 78, 69, 32, 25, 10, 87, 65, 76, 76, 80, 76, 65, 78, 69, + 32, 8, 82, 83, 207, 45, 67, 10, 18, 67, 43, 83, 4, 254, 45, 85, 213, 95, + 3, 73, 82, 67, 6, 37, 7, 84, 82, 65, 73, 71, 72, 84, 7, 11, 32, 4, 222, + 163, 1, 65, 55, 68, 6, 130, 51, 68, 252, 167, 4, 4, 70, 76, 85, 84, 191, + 145, 5, 85, 14, 112, 5, 72, 65, 76, 70, 32, 26, 67, 28, 4, 87, 73, 68, + 69, 230, 92, 79, 177, 130, 4, 6, 83, 81, 85, 69, 69, 90, 4, 22, 67, 131, + 93, 79, 2, 225, 232, 2, 2, 76, 79, 4, 162, 67, 32, 217, 61, 4, 78, 73, + 78, 71, 38, 204, 1, 28, 65, 67, 69, 32, 68, 73, 82, 69, 67, 84, 73, 79, + 78, 32, 80, 79, 83, 73, 84, 73, 79, 78, 32, 78, 79, 83, 69, 32, 138, 1, + 73, 82, 76, 176, 1, 8, 79, 82, 69, 72, 69, 65, 68, 32, 135, 148, 7, 85, + 6, 88, 10, 85, 80, 32, 79, 82, 32, 68, 79, 87, 78, 13, 8, 70, 79, 82, 87, + 65, 82, 68, 32, 5, 11, 32, 2, 225, 225, 4, 3, 84, 73, 76, 12, 240, 225, + 3, 11, 76, 76, 32, 77, 79, 68, 73, 70, 73, 69, 82, 151, 184, 2, 78, 12, + 44, 4, 73, 67, 75, 32, 29, 3, 79, 79, 82, 10, 146, 141, 1, 76, 35, 83, 2, + 221, 233, 8, 20, 80, 76, 65, 78, 69, 32, 83, 72, 79, 85, 76, 68, 69, 82, + 32, 72, 73, 80, 32, 77, 6, 166, 89, 87, 222, 38, 67, 47, 78, 156, 4, 34, + 65, 133, 75, 3, 69, 65, 68, 140, 4, 36, 3, 78, 68, 45, 139, 172, 9, 73, + 138, 4, 92, 5, 65, 78, 71, 76, 69, 138, 5, 67, 150, 10, 70, 186, 42, 72, + 241, 12, 4, 79, 86, 65, 76, 37, 11, 32, 34, 188, 1, 5, 73, 78, 68, 69, + 88, 176, 1, 7, 76, 73, 84, 84, 76, 69, 32, 136, 1, 5, 82, 73, 78, 71, 32, + 173, 66, 18, 77, 73, 68, 68, 76, 69, 32, 82, 73, 78, 71, 32, 76, 73, 84, + 84, 76, 69, 17, 11, 32, 14, 128, 1, 7, 77, 73, 68, 68, 76, 69, 32, 228, + 24, 11, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, 241, 42, 5, 84, 72, + 85, 77, 66, 4, 174, 70, 76, 243, 201, 8, 82, 8, 44, 5, 73, 78, 68, 69, + 88, 203, 224, 9, 85, 7, 145, 24, 18, 32, 84, 72, 85, 77, 66, 32, 73, 78, + 68, 69, 88, 32, 84, 72, 85, 77, 66, 4, 108, 22, 68, 79, 87, 78, 32, 77, + 73, 68, 68, 76, 69, 32, 84, 72, 85, 77, 66, 32, 73, 78, 68, 69, 155, 68, + 76, 2, 223, 157, 8, 88, 88, 64, 5, 73, 82, 67, 76, 69, 184, 3, 3, 76, 65, + 87, 183, 2, 85, 35, 11, 32, 32, 116, 5, 73, 78, 68, 69, 88, 200, 1, 7, + 76, 73, 84, 84, 76, 69, 32, 36, 7, 77, 73, 68, 68, 76, 69, 32, 163, 64, + 82, 21, 11, 32, 18, 72, 6, 77, 73, 68, 68, 76, 69, 198, 26, 72, 242, 38, + 82, 255, 215, 8, 66, 13, 11, 32, 10, 64, 5, 67, 82, 79, 83, 83, 198, 64, + 84, 82, 76, 243, 201, 8, 82, 4, 134, 65, 32, 227, 212, 8, 69, 4, 210, + 181, 8, 73, 139, 166, 1, 85, 6, 252, 47, 10, 82, 73, 78, 71, 32, 76, 73, + 84, 84, 76, 187, 171, 9, 85, 17, 11, 32, 14, 144, 2, 28, 77, 73, 68, 68, + 76, 69, 32, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, 32, 67, 79, 78, + 74, 79, 73, 78, 69, 68, 176, 49, 2, 70, 79, 198, 11, 78, 162, 1, 84, 217, + 118, 23, 73, 78, 68, 69, 88, 32, 84, 72, 85, 77, 66, 32, 67, 85, 82, 86, + 69, 32, 84, 72, 85, 77, 66, 5, 139, 181, 1, 32, 38, 46, 80, 149, 2, 6, + 82, 76, 73, 67, 85, 69, 31, 11, 32, 28, 188, 1, 5, 73, 78, 68, 69, 88, + 56, 19, 70, 73, 86, 69, 32, 70, 73, 78, 71, 69, 82, 83, 32, 83, 80, 82, + 69, 65, 68, 196, 50, 7, 77, 73, 68, 68, 76, 69, 32, 18, 79, 218, 7, 78, + 163, 1, 84, 9, 11, 32, 6, 40, 5, 84, 72, 85, 77, 66, 243, 58, 82, 5, 215, + 46, 32, 9, 11, 32, 6, 72, 5, 73, 78, 68, 69, 88, 0, 6, 77, 73, 68, 68, + 76, 69, 179, 71, 79, 2, 189, 133, 9, 13, 32, 82, 73, 78, 71, 32, 76, 73, + 84, 84, 76, 69, 32, 172, 2, 44, 3, 73, 83, 84, 177, 33, 3, 76, 65, 84, + 247, 1, 11, 32, 244, 1, 160, 2, 5, 73, 78, 68, 69, 88, 192, 16, 7, 76, + 73, 84, 84, 76, 69, 32, 196, 2, 7, 77, 73, 68, 68, 76, 69, 32, 200, 4, 5, + 82, 73, 78, 71, 32, 132, 2, 5, 84, 72, 85, 77, 66, 138, 2, 72, 205, 9, + 22, 70, 79, 85, 82, 32, 70, 73, 78, 71, 69, 82, 83, 32, 67, 79, 78, 74, + 79, 73, 78, 69, 68, 137, 1, 11, 32, 134, 1, 232, 1, 4, 66, 69, 78, 84, + 36, 6, 72, 73, 78, 71, 69, 68, 76, 6, 77, 73, 68, 68, 76, 69, 168, 7, 2, + 67, 85, 64, 6, 84, 72, 85, 77, 66, 32, 160, 5, 16, 85, 80, 32, 77, 73, + 68, 68, 76, 69, 32, 72, 73, 78, 71, 69, 68, 151, 4, 82, 5, 217, 45, 5, + 32, 79, 86, 69, 82, 9, 11, 32, 6, 252, 20, 8, 77, 73, 68, 68, 76, 69, 32, + 85, 167, 160, 8, 76, 71, 11, 32, 68, 236, 1, 4, 66, 69, 78, 84, 42, 67, + 244, 1, 10, 85, 80, 32, 83, 80, 82, 69, 65, 68, 32, 100, 6, 72, 73, 78, + 71, 69, 68, 46, 82, 80, 5, 84, 72, 85, 77, 66, 188, 28, 14, 83, 84, 82, + 65, 73, 71, 72, 84, 32, 84, 72, 85, 77, 66, 227, 17, 76, 5, 213, 92, 6, + 32, 84, 72, 85, 77, 66, 28, 68, 8, 79, 78, 74, 79, 73, 78, 69, 68, 233, + 1, 4, 82, 79, 83, 83, 23, 11, 32, 20, 144, 1, 6, 67, 85, 80, 80, 69, 68, + 28, 6, 84, 72, 85, 77, 66, 32, 68, 5, 72, 73, 78, 71, 69, 166, 22, 73, + 165, 7, 6, 77, 73, 68, 68, 76, 69, 5, 11, 32, 2, 203, 27, 84, 8, 160, 1, + 4, 83, 73, 68, 69, 219, 61, 70, 6, 22, 69, 159, 47, 32, 4, 223, 15, 68, + 5, 241, 30, 7, 32, 83, 80, 82, 69, 65, 68, 8, 32, 3, 73, 78, 71, 247, 19, + 65, 7, 11, 32, 4, 138, 36, 67, 255, 225, 8, 66, 19, 11, 32, 16, 64, 6, + 65, 78, 71, 76, 69, 68, 22, 67, 106, 72, 159, 132, 9, 66, 5, 247, 211, 5, + 32, 6, 74, 85, 230, 7, 73, 241, 25, 10, 79, 78, 74, 79, 73, 78, 69, 68, + 32, 72, 2, 145, 188, 4, 2, 80, 80, 4, 194, 33, 73, 217, 26, 2, 79, 79, + 40, 164, 1, 7, 65, 78, 71, 76, 69, 68, 32, 46, 67, 220, 1, 14, 70, 79, + 82, 87, 65, 82, 68, 32, 73, 78, 68, 69, 88, 32, 32, 4, 72, 79, 79, 75, + 53, 4, 83, 73, 68, 69, 4, 128, 1, 2, 73, 78, 1, 3, 79, 85, 84, 12, 36, 5, + 73, 82, 67, 76, 69, 15, 85, 5, 47, 68, 8, 32, 4, 80, 80, 69, 68, 39, 82, + 2, 225, 40, 5, 32, 77, 73, 68, 68, 6, 56, 9, 86, 69, 32, 84, 72, 85, 77, + 66, 32, 143, 37, 76, 4, 182, 160, 1, 73, 131, 188, 4, 85, 4, 202, 84, 83, + 255, 171, 8, 66, 7, 157, 8, 9, 69, 68, 32, 77, 73, 68, 68, 76, 69, 15, + 11, 32, 12, 92, 6, 73, 78, 68, 69, 88, 32, 156, 13, 5, 84, 72, 85, 77, + 66, 193, 8, 4, 66, 79, 84, 72, 4, 26, 72, 239, 254, 8, 66, 2, 231, 140, + 8, 73, 7, 37, 7, 32, 84, 72, 85, 77, 66, 32, 4, 178, 28, 67, 227, 129, 1, + 83, 22, 88, 4, 68, 79, 87, 78, 186, 1, 84, 158, 6, 82, 130, 21, 73, 226, + 224, 8, 66, 159, 67, 85, 9, 11, 32, 6, 80, 9, 79, 84, 72, 69, 82, 83, 32, + 67, 73, 25, 7, 82, 73, 80, 80, 76, 69, 32, 2, 225, 51, 2, 82, 67, 4, 22, + 67, 171, 80, 83, 2, 11, 85, 2, 241, 179, 4, 2, 82, 86, 4, 196, 35, 6, 79, + 85, 67, 72, 69, 83, 39, 72, 30, 134, 1, 82, 28, 6, 84, 72, 85, 77, 66, + 32, 138, 3, 85, 134, 1, 68, 210, 30, 76, 205, 244, 7, 9, 66, 69, 78, 84, + 32, 79, 86, 69, 82, 4, 238, 4, 65, 231, 29, 73, 16, 80, 7, 65, 78, 71, + 76, 69, 68, 32, 126, 67, 124, 4, 72, 79, 79, 75, 147, 32, 76, 6, 60, 10, + 79, 85, 84, 32, 73, 78, 68, 69, 88, 32, 215, 1, 73, 4, 26, 67, 151, 188, + 9, 85, 2, 133, 186, 2, 3, 82, 79, 83, 6, 76, 12, 73, 82, 67, 76, 69, 68, + 32, 73, 78, 68, 69, 88, 45, 3, 85, 80, 80, 4, 11, 32, 4, 150, 21, 72, + 131, 166, 9, 85, 2, 25, 4, 69, 68, 32, 73, 2, 233, 30, 4, 78, 68, 69, 88, + 4, 11, 80, 5, 175, 15, 32, 18, 102, 68, 20, 6, 77, 73, 68, 68, 76, 69, + 42, 82, 198, 29, 84, 82, 76, 226, 244, 7, 73, 139, 166, 1, 85, 2, 175, + 229, 7, 79, 7, 11, 32, 4, 206, 3, 82, 179, 16, 67, 2, 11, 65, 2, 37, 7, + 73, 83, 69, 68, 32, 75, 78, 2, 11, 85, 2, 11, 67, 2, 147, 148, 8, 75, 35, + 11, 32, 32, 148, 1, 8, 66, 69, 84, 87, 69, 69, 78, 32, 102, 72, 20, 5, + 79, 86, 69, 82, 32, 120, 4, 83, 73, 68, 69, 100, 6, 85, 78, 68, 69, 82, + 32, 207, 40, 70, 8, 80, 12, 73, 78, 68, 69, 88, 32, 77, 73, 68, 68, 76, + 69, 246, 20, 77, 155, 6, 82, 5, 135, 70, 32, 2, 187, 169, 4, 69, 4, 52, + 6, 70, 79, 85, 82, 32, 82, 161, 2, 2, 84, 87, 2, 11, 65, 2, 233, 80, 9, + 73, 83, 69, 68, 32, 75, 78, 85, 67, 6, 11, 32, 6, 38, 68, 190, 15, 67, + 255, 225, 8, 66, 2, 25, 4, 73, 65, 71, 79, 2, 131, 171, 8, 78, 10, 54, + 73, 34, 84, 48, 4, 70, 79, 85, 82, 131, 23, 76, 2, 161, 7, 4, 78, 68, 69, + 88, 4, 34, 87, 13, 4, 72, 82, 69, 69, 2, 11, 79, 2, 197, 164, 8, 5, 32, + 70, 73, 78, 71, 55, 11, 32, 52, 194, 1, 70, 172, 3, 4, 72, 69, 69, 76, + 192, 1, 6, 83, 80, 76, 73, 84, 32, 236, 1, 6, 84, 72, 85, 77, 66, 32, + 165, 244, 4, 16, 66, 69, 84, 87, 69, 69, 78, 32, 80, 65, 76, 77, 32, 70, + 65, 67, 24, 140, 1, 18, 73, 86, 69, 32, 70, 73, 78, 71, 69, 82, 83, 32, + 83, 80, 82, 69, 65, 68, 165, 1, 11, 79, 85, 82, 32, 70, 73, 78, 71, 69, + 82, 83, 15, 11, 32, 12, 68, 6, 72, 73, 78, 71, 69, 68, 42, 84, 186, 2, + 70, 199, 233, 8, 66, 7, 11, 32, 4, 190, 4, 84, 155, 15, 78, 2, 189, 35, + 6, 72, 85, 77, 66, 32, 70, 11, 11, 32, 8, 72, 9, 67, 79, 78, 74, 79, 73, + 78, 69, 68, 154, 8, 72, 231, 226, 8, 66, 5, 221, 145, 9, 3, 32, 83, 80, + 11, 11, 32, 8, 96, 19, 70, 73, 86, 69, 32, 70, 73, 78, 71, 69, 82, 83, + 32, 83, 80, 82, 69, 65, 68, 151, 2, 84, 7, 11, 32, 4, 26, 70, 199, 233, + 8, 66, 2, 21, 3, 79, 85, 82, 2, 167, 1, 32, 10, 72, 6, 67, 69, 78, 84, + 82, 69, 96, 5, 73, 78, 68, 69, 88, 167, 16, 76, 7, 49, 10, 32, 84, 72, + 85, 77, 66, 32, 83, 73, 68, 4, 11, 69, 5, 11, 32, 2, 131, 232, 8, 66, 2, + 11, 32, 2, 11, 84, 2, 173, 135, 1, 5, 72, 85, 77, 66, 32, 6, 242, 30, 70, + 162, 104, 83, 159, 224, 7, 66, 82, 48, 4, 73, 78, 71, 69, 181, 9, 3, 79, + 79, 75, 63, 11, 32, 60, 206, 1, 70, 172, 1, 5, 73, 78, 68, 69, 88, 196, + 2, 6, 76, 73, 84, 84, 76, 69, 80, 6, 77, 73, 68, 68, 76, 69, 30, 79, 64, + 4, 82, 73, 78, 71, 124, 6, 84, 72, 85, 77, 66, 32, 158, 6, 78, 139, 142, + 4, 83, 4, 92, 19, 73, 86, 69, 32, 70, 73, 78, 71, 69, 82, 83, 32, 83, 80, + 82, 69, 65, 68, 32, 19, 79, 2, 191, 25, 79, 2, 249, 1, 11, 85, 82, 32, + 70, 73, 78, 71, 69, 82, 83, 32, 23, 11, 32, 20, 86, 72, 40, 7, 77, 73, + 68, 68, 76, 69, 32, 104, 5, 84, 72, 85, 77, 66, 219, 9, 82, 2, 11, 73, 2, + 177, 154, 4, 2, 78, 71, 6, 36, 4, 82, 73, 78, 71, 203, 10, 76, 5, 11, 32, + 2, 11, 67, 2, 21, 3, 79, 78, 74, 2, 175, 33, 79, 11, 11, 32, 8, 34, 83, + 210, 22, 79, 203, 39, 76, 4, 190, 148, 7, 73, 195, 8, 77, 9, 11, 32, 6, + 22, 73, 199, 8, 84, 4, 25, 4, 78, 68, 69, 88, 5, 155, 8, 32, 5, 11, 32, + 2, 171, 8, 82, 8, 21, 3, 80, 69, 78, 9, 11, 32, 6, 178, 7, 78, 163, 1, + 84, 5, 97, 22, 32, 68, 79, 87, 78, 32, 73, 78, 68, 69, 88, 32, 84, 72, + 85, 77, 66, 32, 72, 79, 79, 75, 2, 177, 77, 2, 32, 77, 6, 68, 9, 66, 69, + 84, 87, 69, 69, 78, 32, 77, 53, 4, 83, 73, 68, 69, 2, 29, 5, 73, 68, 68, + 76, 69, 2, 163, 148, 4, 32, 5, 33, 6, 32, 84, 79, 85, 67, 72, 2, 185, + 250, 7, 3, 73, 78, 71, 21, 11, 32, 18, 172, 1, 4, 67, 85, 82, 76, 28, 18, + 73, 78, 68, 69, 88, 32, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, 32, + 48, 7, 77, 73, 68, 68, 76, 69, 32, 225, 2, 4, 82, 73, 78, 71, 2, 253, + 147, 5, 2, 73, 67, 6, 142, 183, 5, 85, 142, 238, 2, 79, 243, 41, 73, 8, + 104, 21, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, 32, 67, 79, 78, 74, + 79, 73, 78, 69, 68, 143, 2, 84, 7, 223, 228, 2, 32, 17, 11, 32, 14, 112, + 14, 70, 73, 86, 69, 32, 70, 73, 78, 71, 69, 82, 83, 32, 83, 22, 76, 66, + 78, 70, 82, 94, 84, 183, 244, 7, 73, 2, 215, 160, 2, 80, 2, 21, 3, 73, + 84, 84, 2, 17, 2, 76, 69, 2, 135, 166, 8, 32, 2, 11, 79, 2, 11, 32, 2, + 11, 84, 2, 11, 72, 2, 247, 251, 5, 85, 2, 11, 73, 2, 21, 3, 78, 71, 32, + 2, 11, 76, 2, 11, 73, 2, 11, 84, 2, 179, 246, 7, 84, 4, 29, 5, 72, 85, + 77, 66, 32, 4, 194, 14, 70, 163, 104, 83, 17, 11, 32, 14, 56, 8, 77, 79, + 86, 69, 77, 69, 78, 84, 247, 150, 8, 82, 12, 26, 45, 163, 199, 7, 32, 10, + 100, 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, 25, 10, 87, 65, 76, + 76, 80, 76, 65, 78, 69, 32, 4, 62, 67, 223, 40, 83, 6, 38, 67, 28, 2, 84, + 73, 195, 40, 83, 2, 169, 155, 8, 2, 85, 82, 2, 183, 251, 8, 76, 38, 50, + 73, 225, 3, 7, 79, 67, 65, 84, 73, 79, 78, 22, 32, 3, 77, 66, 32, 171, 1, + 80, 16, 56, 4, 67, 79, 77, 66, 29, 6, 76, 69, 78, 71, 84, 72, 2, 193, + 212, 5, 2, 73, 78, 14, 11, 45, 14, 146, 150, 9, 49, 2, 50, 2, 51, 2, 52, + 2, 53, 2, 54, 3, 55, 6, 58, 32, 153, 1, 9, 83, 32, 80, 82, 69, 83, 83, + 69, 68, 4, 116, 12, 76, 79, 87, 69, 82, 32, 79, 86, 69, 82, 32, 85, 133, + 251, 7, 11, 85, 80, 80, 69, 82, 32, 79, 86, 69, 82, 32, 2, 11, 80, 2, + 163, 214, 8, 80, 2, 29, 5, 32, 84, 79, 71, 69, 2, 11, 84, 2, 227, 213, 8, + 72, 16, 22, 32, 203, 1, 45, 12, 148, 1, 2, 72, 69, 236, 177, 2, 3, 84, + 79, 82, 144, 222, 4, 3, 68, 69, 80, 0, 3, 87, 73, 68, 229, 109, 10, 76, + 73, 77, 66, 83, 32, 68, 73, 71, 73, 4, 196, 13, 4, 65, 68, 32, 78, 131, + 218, 8, 73, 4, 52, 5, 70, 76, 79, 79, 82, 1, 4, 87, 65, 76, 76, 2, 221, + 210, 8, 5, 80, 76, 65, 78, 69, 156, 3, 64, 4, 85, 84, 72, 32, 161, 5, 7, + 86, 69, 77, 69, 78, 84, 45, 54, 186, 1, 67, 88, 5, 70, 82, 79, 87, 78, 0, + 5, 83, 77, 73, 76, 69, 56, 4, 75, 73, 83, 83, 36, 5, 79, 80, 69, 78, 32, + 192, 1, 5, 84, 69, 78, 83, 69, 165, 29, 5, 87, 82, 73, 78, 75, 8, 48, 6, + 76, 79, 83, 69, 68, 32, 163, 223, 7, 79, 6, 222, 2, 70, 142, 38, 67, 47, + 78, 7, 11, 32, 4, 22, 79, 203, 1, 87, 2, 147, 186, 7, 80, 7, 11, 32, 4, + 166, 1, 87, 83, 70, 18, 100, 4, 79, 86, 65, 76, 0, 9, 82, 69, 67, 84, 65, + 78, 71, 76, 69, 42, 87, 82, 70, 235, 185, 7, 67, 7, 11, 32, 4, 26, 87, + 151, 184, 7, 89, 2, 25, 4, 82, 73, 78, 75, 2, 203, 128, 4, 76, 7, 11, 32, + 4, 18, 70, 43, 83, 2, 17, 2, 79, 82, 2, 215, 244, 7, 87, 2, 17, 2, 85, + 67, 2, 219, 255, 3, 75, 230, 2, 192, 1, 9, 68, 73, 65, 71, 79, 78, 65, + 76, 32, 148, 1, 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, 184, 14, + 6, 72, 73, 78, 71, 69, 32, 189, 2, 10, 87, 65, 76, 76, 80, 76, 65, 78, + 69, 32, 32, 56, 8, 66, 69, 84, 87, 69, 69, 78, 32, 22, 65, 31, 84, 16, + 18, 65, 31, 84, 8, 229, 28, 3, 87, 65, 89, 8, 201, 28, 6, 79, 87, 65, 82, + 68, 83, 150, 1, 208, 2, 24, 65, 82, 77, 32, 67, 73, 82, 67, 76, 69, 32, + 72, 73, 84, 84, 73, 78, 71, 32, 87, 65, 76, 76, 32, 38, 66, 34, 67, 224, + 1, 8, 70, 73, 78, 71, 69, 82, 32, 67, 52, 5, 72, 85, 77, 80, 32, 184, 3, + 5, 76, 79, 79, 80, 32, 198, 1, 83, 108, 7, 84, 82, 73, 80, 76, 69, 32, + 110, 87, 206, 12, 68, 198, 2, 80, 154, 6, 90, 227, 232, 7, 74, 12, 178, + 7, 76, 154, 9, 77, 39, 83, 8, 150, 17, 79, 175, 230, 7, 69, 28, 86, 72, + 20, 5, 85, 82, 86, 69, 32, 196, 29, 5, 79, 82, 78, 69, 82, 227, 165, 7, + 82, 2, 211, 212, 7, 69, 18, 80, 4, 67, 79, 77, 66, 158, 8, 72, 230, 15, + 76, 202, 172, 2, 77, 183, 177, 1, 83, 2, 11, 73, 2, 131, 248, 3, 78, 6, + 128, 9, 6, 73, 82, 67, 76, 69, 83, 239, 20, 79, 18, 56, 8, 72, 73, 84, + 84, 73, 78, 71, 32, 239, 244, 3, 83, 16, 72, 8, 67, 69, 73, 76, 73, 78, + 71, 32, 101, 6, 70, 76, 79, 79, 82, 32, 8, 56, 5, 76, 65, 82, 71, 69, 1, + 5, 83, 77, 65, 76, 76, 4, 11, 32, 4, 226, 54, 84, 135, 2, 68, 8, 88, 4, + 83, 77, 65, 76, 12, 5, 76, 65, 82, 71, 69, 17, 7, 84, 82, 73, 80, 76, 69, + 32, 2, 11, 76, 2, 255, 18, 32, 4, 56, 5, 76, 65, 82, 71, 69, 1, 5, 83, + 77, 65, 76, 76, 2, 145, 53, 2, 32, 84, 18, 56, 8, 72, 73, 84, 84, 73, 78, + 71, 32, 183, 241, 3, 83, 16, 64, 7, 67, 69, 73, 76, 73, 78, 71, 1, 5, 70, + 76, 79, 79, 82, 8, 11, 32, 8, 22, 76, 191, 9, 83, 4, 249, 22, 4, 65, 82, + 71, 69, 12, 44, 6, 72, 65, 75, 73, 78, 71, 243, 16, 73, 2, 21, 3, 32, 80, + 65, 2, 221, 240, 3, 4, 82, 65, 76, 76, 8, 76, 12, 65, 76, 84, 69, 82, 78, + 65, 84, 73, 78, 71, 32, 138, 18, 87, 63, 83, 4, 134, 18, 87, 147, 21, 77, + 18, 80, 4, 65, 86, 69, 32, 169, 1, 11, 82, 73, 83, 84, 32, 67, 73, 82, + 67, 76, 69, 14, 30, 72, 102, 83, 171, 20, 76, 8, 37, 7, 73, 84, 84, 73, + 78, 71, 32, 8, 252, 2, 4, 67, 69, 73, 76, 25, 5, 70, 76, 79, 79, 82, 4, + 170, 146, 5, 78, 231, 224, 1, 77, 4, 209, 5, 10, 32, 72, 73, 84, 84, 73, + 78, 71, 32, 87, 14, 112, 3, 85, 80, 32, 184, 1, 6, 68, 79, 87, 78, 32, + 83, 149, 225, 5, 10, 83, 73, 68, 69, 32, 84, 79, 32, 83, 73, 10, 40, 5, + 68, 79, 87, 78, 32, 143, 1, 83, 8, 68, 8, 65, 76, 84, 69, 82, 78, 65, 84, + 230, 17, 76, 215, 216, 3, 83, 4, 21, 3, 73, 78, 71, 4, 11, 32, 4, 190, + 17, 76, 215, 216, 3, 83, 2, 207, 30, 69, 162, 1, 148, 2, 11, 65, 82, 77, + 32, 67, 73, 82, 67, 76, 69, 32, 98, 66, 54, 67, 174, 4, 68, 100, 8, 70, + 73, 78, 71, 69, 82, 32, 67, 64, 5, 72, 85, 77, 80, 32, 60, 5, 76, 79, 79, + 80, 32, 102, 80, 34, 83, 216, 1, 7, 84, 82, 73, 80, 76, 69, 32, 218, 1, + 87, 202, 2, 90, 227, 232, 7, 74, 8, 18, 77, 39, 83, 4, 225, 13, 5, 69, + 68, 73, 85, 77, 4, 11, 77, 4, 177, 13, 3, 65, 76, 76, 12, 34, 79, 185, + 13, 3, 69, 78, 68, 6, 183, 13, 88, 46, 100, 6, 79, 82, 78, 69, 82, 32, + 88, 4, 85, 82, 86, 69, 232, 11, 4, 72, 69, 67, 75, 227, 165, 7, 82, 8, + 54, 82, 194, 12, 76, 162, 167, 2, 77, 183, 177, 1, 83, 2, 11, 79, 2, 239, + 176, 5, 84, 30, 50, 32, 137, 2, 7, 68, 32, 67, 82, 79, 83, 83, 26, 66, + 72, 68, 2, 84, 72, 133, 5, 7, 81, 85, 65, 82, 84, 69, 82, 12, 196, 5, 10, + 65, 76, 70, 45, 67, 73, 82, 67, 76, 69, 147, 14, 73, 6, 96, 2, 69, 78, + 29, 18, 82, 69, 69, 45, 81, 85, 65, 82, 84, 69, 82, 32, 67, 73, 82, 67, + 76, 69, 2, 11, 32, 2, 131, 1, 83, 4, 11, 32, 4, 246, 176, 2, 77, 183, + 177, 1, 83, 8, 33, 6, 79, 85, 66, 76, 69, 32, 8, 30, 83, 150, 4, 65, 75, + 87, 2, 209, 196, 8, 3, 84, 82, 65, 6, 32, 3, 73, 82, 67, 151, 9, 79, 4, + 173, 7, 3, 76, 69, 83, 10, 142, 8, 76, 166, 8, 72, 254, 158, 2, 77, 183, + 177, 1, 83, 12, 68, 5, 83, 77, 65, 76, 76, 142, 7, 76, 166, 8, 72, 255, + 158, 2, 77, 5, 11, 32, 2, 227, 36, 68, 6, 181, 6, 4, 69, 65, 75, 83, 12, + 22, 73, 231, 36, 72, 10, 29, 5, 78, 71, 76, 69, 32, 10, 52, 8, 83, 84, + 82, 65, 73, 71, 72, 84, 207, 1, 87, 8, 11, 32, 8, 42, 76, 202, 172, 2, + 77, 183, 177, 1, 83, 4, 25, 4, 65, 82, 71, 69, 5, 147, 207, 8, 83, 8, 26, + 65, 74, 87, 63, 83, 4, 49, 10, 76, 84, 69, 82, 78, 65, 84, 73, 78, 71, 5, + 17, 2, 32, 87, 2, 29, 5, 82, 73, 83, 84, 32, 2, 165, 196, 7, 2, 70, 76, + 2, 37, 7, 84, 82, 65, 73, 71, 72, 84, 2, 159, 20, 32, 26, 104, 4, 65, 86, + 69, 32, 185, 1, 17, 82, 73, 83, 84, 32, 67, 73, 82, 67, 76, 69, 32, 70, + 82, 79, 78, 84, 22, 108, 6, 67, 85, 82, 86, 69, 32, 140, 1, 13, 68, 73, + 65, 71, 79, 78, 65, 76, 32, 80, 65, 84, 72, 223, 8, 72, 12, 48, 4, 68, + 79, 85, 66, 1, 4, 84, 82, 73, 80, 6, 85, 2, 76, 69, 4, 11, 32, 4, 210, + 30, 68, 43, 83, 6, 29, 5, 73, 71, 90, 65, 71, 6, 11, 32, 6, 42, 76, 162, + 167, 2, 77, 183, 177, 1, 83, 2, 175, 166, 7, 65, 10, 40, 4, 79, 83, 69, + 32, 135, 182, 7, 69, 8, 26, 67, 46, 78, 35, 87, 2, 11, 79, 2, 217, 214, + 7, 3, 78, 84, 65, 2, 221, 219, 7, 3, 69, 85, 84, 4, 44, 3, 82, 73, 78, + 241, 157, 1, 2, 73, 71, 2, 151, 167, 5, 75, 72, 56, 7, 79, 84, 65, 84, + 73, 79, 78, 225, 22, 2, 85, 66, 66, 60, 10, 32, 77, 79, 68, 73, 70, 73, + 69, 82, 45, 155, 1, 45, 30, 82, 49, 250, 226, 8, 50, 2, 51, 2, 52, 2, 53, + 2, 54, 2, 55, 2, 56, 3, 57, 14, 246, 226, 8, 48, 2, 49, 2, 50, 2, 51, 2, + 52, 2, 53, 3, 54, 36, 104, 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, + 32, 133, 2, 10, 87, 65, 76, 76, 80, 76, 65, 78, 69, 32, 18, 100, 4, 68, + 79, 85, 66, 0, 4, 83, 73, 78, 71, 21, 11, 65, 76, 84, 69, 82, 78, 65, 84, + 73, 78, 71, 6, 17, 2, 76, 69, 7, 45, 9, 32, 72, 73, 84, 84, 73, 78, 71, + 32, 4, 32, 2, 67, 69, 33, 2, 70, 76, 2, 11, 73, 2, 231, 142, 8, 76, 2, + 239, 166, 8, 79, 18, 88, 8, 65, 76, 84, 69, 82, 78, 65, 84, 44, 4, 68, + 79, 85, 66, 1, 4, 83, 73, 78, 71, 6, 72, 4, 73, 78, 71, 32, 159, 222, 8, + 69, 6, 17, 2, 76, 69, 7, 11, 32, 4, 11, 72, 4, 11, 73, 4, 33, 6, 84, 84, + 73, 78, 71, 32, 4, 44, 2, 67, 72, 21, 5, 70, 82, 79, 78, 84, 2, 179, 243, + 4, 69, 2, 205, 213, 6, 2, 32, 87, 30, 172, 1, 8, 72, 79, 85, 76, 68, 69, + 82, 32, 196, 1, 7, 81, 85, 69, 69, 90, 69, 32, 240, 1, 7, 85, 82, 70, 65, + 67, 69, 32, 240, 10, 5, 84, 82, 73, 75, 69, 243, 129, 5, 69, 6, 100, 4, + 72, 73, 80, 32, 129, 246, 3, 15, 84, 73, 76, 84, 73, 78, 71, 32, 70, 82, + 79, 77, 32, 87, 65, 4, 48, 4, 80, 79, 83, 73, 253, 192, 7, 2, 83, 80, 2, + 11, 84, 2, 205, 150, 5, 2, 73, 79, 12, 48, 6, 70, 76, 73, 67, 75, 32, 18, + 76, 35, 83, 2, 211, 16, 65, 4, 129, 1, 4, 65, 82, 71, 69, 6, 34, 69, 65, + 4, 77, 65, 76, 76, 2, 17, 2, 81, 85, 2, 21, 3, 69, 78, 84, 2, 151, 207, + 7, 73, 4, 11, 32, 4, 214, 11, 77, 187, 4, 83, 4, 22, 83, 135, 11, 66, 2, + 249, 207, 1, 4, 89, 77, 66, 79, 82, 58, 69, 218, 2, 79, 137, 8, 6, 82, + 65, 86, 69, 76, 45, 20, 76, 3, 69, 84, 72, 185, 1, 11, 78, 83, 69, 32, + 67, 72, 69, 69, 75, 83, 32, 15, 11, 32, 12, 56, 3, 79, 78, 32, 86, 77, + 177, 5, 4, 66, 73, 84, 69, 8, 56, 4, 76, 73, 80, 83, 1, 6, 84, 79, 78, + 71, 85, 69, 5, 11, 32, 2, 11, 77, 2, 201, 194, 3, 2, 79, 86, 6, 50, 77, + 160, 176, 4, 2, 72, 73, 167, 138, 3, 76, 2, 213, 163, 3, 2, 73, 68, 28, + 76, 5, 78, 71, 85, 69, 32, 196, 4, 4, 82, 83, 79, 45, 129, 2, 2, 85, 67, + 16, 192, 2, 7, 67, 69, 78, 84, 82, 69, 32, 52, 14, 73, 78, 83, 73, 68, + 69, 32, 77, 79, 85, 84, 72, 32, 82, 36, 4, 84, 73, 80, 32, 88, 7, 76, 73, + 67, 75, 73, 78, 71, 208, 174, 7, 14, 83, 84, 73, 67, 75, 73, 78, 71, 32, + 79, 85, 84, 32, 70, 149, 158, 1, 17, 77, 79, 86, 69, 83, 32, 65, 71, 65, + 73, 78, 83, 84, 32, 67, 72, 69, 4, 214, 1, 73, 181, 180, 3, 5, 83, 84, + 73, 67, 75, 2, 181, 197, 3, 4, 69, 76, 65, 88, 4, 84, 7, 66, 69, 84, 87, + 69, 69, 78, 41, 10, 84, 79, 85, 67, 72, 73, 78, 71, 32, 73, 2, 11, 32, 2, + 173, 155, 5, 2, 76, 73, 2, 173, 174, 5, 5, 78, 83, 73, 68, 69, 6, 120, + 10, 87, 65, 76, 76, 80, 76, 65, 78, 69, 32, 253, 199, 3, 14, 70, 76, 79, + 79, 82, 80, 76, 65, 78, 69, 32, 84, 87, 73, 4, 108, 8, 67, 85, 82, 86, + 69, 68, 32, 66, 161, 204, 1, 13, 83, 84, 82, 65, 73, 71, 72, 84, 32, 83, + 84, 82, 69, 2, 211, 191, 7, 69, 6, 11, 72, 6, 11, 32, 6, 30, 66, 34, 77, + 187, 4, 83, 2, 137, 249, 6, 3, 69, 84, 87, 2, 149, 2, 3, 85, 76, 84, 34, + 100, 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, 29, 10, 87, 65, 76, + 76, 80, 76, 65, 78, 69, 32, 14, 178, 1, 82, 151, 2, 83, 20, 72, 11, 65, + 82, 77, 32, 83, 80, 73, 82, 65, 76, 32, 78, 82, 151, 2, 83, 6, 30, 84, + 134, 2, 68, 43, 83, 2, 11, 82, 2, 11, 73, 2, 147, 166, 7, 80, 12, 41, 8, + 79, 84, 65, 84, 73, 79, 78, 45, 12, 52, 5, 70, 76, 79, 79, 82, 1, 4, 87, + 65, 76, 76, 6, 33, 6, 80, 76, 65, 78, 69, 32, 6, 26, 65, 54, 68, 43, 83, + 2, 29, 5, 76, 84, 69, 82, 78, 2, 203, 194, 3, 65, 2, 17, 2, 79, 85, 2, + 147, 164, 7, 66, 2, 231, 163, 7, 73, 2, 11, 72, 2, 203, 192, 3, 65, 2, + 11, 73, 2, 187, 179, 7, 78, 10, 100, 6, 65, 66, 79, 86, 69, 32, 162, 1, + 79, 213, 148, 2, 10, 77, 73, 78, 85, 83, 32, 83, 73, 77, 73, 4, 60, 7, + 71, 82, 69, 65, 84, 69, 82, 1, 4, 76, 69, 83, 83, 2, 37, 7, 45, 84, 72, + 65, 78, 32, 65, 2, 25, 4, 66, 79, 86, 69, 2, 161, 241, 1, 2, 32, 69, 4, + 147, 179, 6, 82, 250, 1, 68, 3, 71, 76, 69, 212, 4, 5, 72, 65, 76, 65, + 32, 143, 161, 5, 69, 20, 50, 32, 185, 195, 8, 6, 45, 83, 72, 73, 70, 84, + 16, 138, 1, 67, 0, 9, 71, 82, 65, 80, 72, 73, 67, 32, 67, 116, 2, 72, 73, + 74, 76, 36, 4, 82, 73, 71, 72, 141, 244, 4, 4, 83, 72, 73, 70, 2, 41, 8, + 72, 65, 82, 65, 67, 84, 69, 82, 2, 37, 7, 32, 73, 78, 84, 82, 79, 68, 2, + 11, 85, 2, 223, 132, 8, 67, 2, 25, 4, 71, 72, 45, 82, 2, 185, 1, 7, 69, + 86, 69, 82, 83, 69, 68, 4, 32, 2, 69, 70, 109, 2, 79, 87, 2, 49, 10, 84, + 45, 80, 79, 73, 78, 84, 73, 78, 71, 2, 21, 3, 32, 65, 78, 2, 17, 2, 71, + 76, 2, 31, 69, 2, 17, 2, 45, 57, 2, 11, 32, 2, 11, 81, 2, 233, 176, 2, 2, + 85, 79, 228, 1, 168, 2, 8, 65, 82, 67, 72, 65, 73, 67, 32, 224, 1, 15, + 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 114, 76, 188, + 8, 5, 83, 73, 71, 78, 32, 144, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, + 78, 32, 177, 227, 2, 17, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, + 75, 85, 78, 68, 68, 40, 46, 68, 109, 7, 78, 85, 77, 66, 69, 82, 32, 18, + 25, 4, 73, 71, 73, 84, 18, 11, 32, 18, 234, 204, 6, 70, 30, 83, 42, 84, + 170, 86, 78, 14, 79, 223, 110, 69, 22, 226, 232, 2, 79, 146, 229, 3, 69, + 30, 70, 42, 78, 38, 83, 39, 84, 6, 18, 82, 63, 89, 4, 56, 6, 65, 75, 65, + 65, 82, 65, 209, 184, 8, 2, 69, 80, 2, 205, 184, 8, 3, 65, 78, 83, 138, + 1, 60, 6, 69, 84, 84, 69, 82, 32, 233, 186, 2, 3, 73, 84, 72, 118, 246, + 1, 65, 106, 69, 34, 73, 62, 85, 34, 77, 220, 1, 4, 68, 65, 78, 84, 58, + 79, 28, 8, 83, 65, 78, 89, 65, 75, 65, 32, 50, 70, 2, 72, 2, 82, 2, 86, + 2, 89, 32, 8, 84, 65, 65, 76, 85, 74, 65, 32, 29, 9, 75, 65, 78, 84, 65, + 74, 65, 32, 78, 34, 102, 69, 204, 1, 2, 76, 80, 144, 2, 5, 77, 66, 65, + 32, 66, 14, 65, 2, 73, 2, 85, 151, 253, 4, 89, 4, 230, 3, 69, 151, 253, + 4, 89, 12, 46, 76, 2, 82, 154, 3, 73, 151, 253, 4, 89, 4, 11, 85, 4, 138, + 3, 85, 151, 253, 4, 89, 28, 42, 65, 177, 1, 5, 85, 85, 82, 68, 72, 22, + 32, 2, 72, 65, 179, 255, 4, 89, 20, 41, 8, 65, 80, 82, 65, 65, 78, 65, + 32, 20, 70, 84, 138, 1, 68, 22, 66, 2, 67, 2, 71, 2, 74, 2, 75, 3, 80, 4, + 154, 1, 84, 15, 65, 6, 25, 4, 65, 74, 65, 32, 6, 102, 76, 2, 78, 3, 83, + 4, 86, 79, 151, 253, 4, 89, 8, 26, 68, 22, 71, 3, 74, 4, 18, 68, 15, 65, + 2, 11, 65, 2, 147, 253, 4, 89, 6, 26, 78, 21, 2, 83, 65, 2, 89, 2, 65, + 65, 4, 68, 11, 78, 89, 79, 79, 71, 65, 32, 78, 65, 65, 75, 139, 252, 4, + 89, 2, 149, 176, 8, 4, 83, 73, 75, 89, 8, 66, 65, 218, 157, 4, 67, 217, + 145, 4, 6, 86, 73, 83, 65, 82, 71, 4, 192, 116, 5, 76, 45, 76, 65, 75, + 237, 186, 7, 6, 78, 85, 83, 86, 65, 82, 34, 56, 5, 68, 73, 71, 65, 32, + 62, 71, 82, 75, 155, 2, 65, 12, 58, 71, 48, 4, 75, 79, 77, 66, 118, 65, + 26, 73, 19, 80, 4, 11, 65, 4, 236, 2, 3, 69, 84, 84, 67, 89, 2, 11, 85, + 2, 223, 173, 8, 86, 16, 52, 5, 69, 84, 84, 73, 32, 85, 4, 79, 77, 66, 85, + 6, 26, 65, 26, 73, 19, 80, 2, 213, 1, 2, 69, 68, 2, 203, 1, 83, 2, 175, + 1, 65, 10, 40, 2, 86, 65, 137, 128, 8, 2, 32, 68, 9, 29, 5, 32, 72, 65, + 65, 32, 6, 62, 65, 0, 6, 68, 73, 71, 65, 32, 65, 85, 3, 71, 65, 89, 2, + 17, 2, 69, 76, 2, 11, 65, 2, 17, 2, 45, 80, 2, 11, 73, 2, 147, 139, 8, + 76, 2, 189, 253, 5, 5, 65, 78, 85, 75, 73, 12, 84, 2, 32, 80, 206, 2, 45, + 205, 149, 6, 10, 84, 69, 69, 78, 32, 80, 79, 73, 78, 84, 8, 140, 1, 23, + 69, 84, 65, 76, 76, 69, 68, 32, 66, 76, 65, 67, 75, 32, 65, 78, 68, 32, + 87, 72, 73, 84, 69, 49, 7, 79, 73, 78, 84, 69, 68, 32, 2, 25, 4, 32, 70, + 76, 79, 2, 223, 227, 6, 82, 6, 78, 80, 218, 147, 6, 66, 217, 132, 1, 9, + 83, 84, 65, 82, 32, 87, 73, 84, 72, 2, 21, 3, 73, 78, 87, 2, 217, 147, 6, + 4, 72, 69, 69, 76, 2, 139, 208, 3, 80, 12, 46, 73, 102, 85, 153, 145, 7, + 3, 65, 84, 69, 4, 56, 8, 32, 65, 78, 68, 32, 83, 75, 73, 243, 239, 7, 69, + 2, 17, 2, 32, 66, 2, 255, 215, 7, 79, 6, 32, 2, 76, 76, 235, 164, 8, 78, + 5, 11, 32, 2, 49, 10, 65, 78, 68, 32, 67, 82, 79, 83, 83, 66, 2, 11, 79, + 2, 187, 147, 7, 78, 42, 46, 65, 198, 4, 69, 210, 1, 73, 167, 1, 79, 14, + 64, 5, 78, 84, 69, 68, 32, 193, 146, 6, 5, 86, 79, 78, 73, 67, 12, 148, + 1, 12, 69, 81, 85, 65, 76, 32, 84, 79, 32, 79, 82, 32, 229, 1, 19, 78, + 79, 82, 84, 72, 32, 65, 82, 82, 79, 87, 32, 87, 73, 84, 72, 32, 72, 79, + 8, 60, 7, 71, 82, 69, 65, 84, 69, 82, 1, 4, 76, 69, 83, 83, 4, 29, 5, 45, + 84, 72, 65, 78, 5, 11, 32, 2, 25, 4, 87, 73, 84, 72, 2, 25, 4, 32, 68, + 79, 84, 2, 17, 2, 32, 73, 2, 11, 78, 2, 11, 83, 2, 183, 147, 6, 73, 4, + 24, 2, 79, 75, 43, 82, 2, 17, 2, 69, 68, 2, 131, 206, 5, 32, 2, 29, 5, + 73, 90, 79, 78, 84, 2, 11, 65, 2, 199, 238, 5, 76, 12, 80, 2, 69, 80, + 144, 144, 8, 9, 85, 84, 72, 32, 79, 82, 32, 83, 80, 203, 17, 68, 8, 40, + 4, 73, 78, 71, 32, 143, 228, 7, 89, 6, 176, 223, 4, 8, 65, 67, 67, 79, + 77, 77, 79, 68, 234, 174, 2, 83, 139, 86, 70, 6, 76, 9, 67, 69, 32, 79, + 70, 32, 80, 73, 90, 21, 6, 71, 72, 84, 76, 89, 32, 2, 211, 157, 8, 90, 4, + 40, 2, 83, 77, 173, 217, 6, 2, 70, 82, 2, 215, 234, 6, 73, 10, 18, 80, + 75, 84, 6, 152, 222, 4, 9, 73, 78, 71, 32, 76, 65, 82, 71, 69, 139, 193, + 3, 69, 4, 26, 32, 195, 158, 8, 72, 2, 11, 77, 2, 249, 132, 7, 3, 65, 67, + 72, 150, 1, 34, 65, 158, 13, 73, 207, 7, 79, 116, 32, 2, 76, 76, 155, + 142, 7, 83, 114, 30, 32, 141, 12, 2, 69, 82, 110, 194, 2, 65, 48, 3, 66, 76, 85, 0, 5, 79, 82, 65, 78, 71, 20, 2, 67, 79, 158, 1, 68, 22, 69, 140, 2, 11, 73, 68, 69, 79, 71, 82, 65, 80, 72, 73, 67, 28, 2, 76, 69, 34, 82, - 218, 3, 83, 22, 84, 36, 4, 86, 69, 69, 32, 208, 238, 1, 3, 71, 82, 69, - 34, 72, 138, 4, 80, 154, 79, 78, 202, 203, 2, 70, 83, 81, 6, 226, 167, 4, - 77, 218, 44, 73, 187, 124, 83, 2, 151, 151, 6, 69, 12, 68, 7, 78, 84, 65, - 73, 78, 83, 32, 218, 243, 1, 77, 231, 156, 5, 76, 6, 36, 4, 65, 83, 32, - 77, 175, 1, 87, 2, 11, 69, 2, 11, 77, 2, 251, 161, 7, 66, 2, 235, 243, 1, - 79, 12, 84, 9, 76, 69, 77, 69, 78, 84, 32, 79, 70, 178, 243, 1, 81, 42, - 88, 215, 194, 2, 77, 7, 17, 2, 32, 87, 4, 25, 4, 73, 84, 72, 32, 4, 26, - 86, 223, 196, 4, 79, 2, 181, 192, 5, 21, 69, 82, 84, 73, 67, 65, 76, 32, - 66, 65, 82, 32, 65, 84, 32, 69, 78, 68, 32, 79, 70, 2, 129, 146, 5, 2, - 32, 67, 8, 130, 1, 70, 255, 245, 1, 83, 40, 96, 3, 73, 71, 72, 64, 13, - 79, 77, 65, 78, 32, 78, 85, 77, 69, 82, 65, 76, 32, 155, 201, 5, 69, 6, - 17, 2, 84, 32, 6, 170, 255, 3, 67, 134, 2, 80, 191, 2, 84, 32, 62, 69, - 50, 70, 62, 79, 46, 84, 238, 236, 5, 83, 231, 83, 78, 4, 26, 76, 171, - 176, 7, 73, 2, 255, 237, 5, 69, 8, 26, 73, 251, 225, 6, 79, 6, 154, 26, - 86, 175, 214, 5, 70, 6, 11, 78, 6, 11, 69, 7, 203, 177, 2, 32, 8, 42, 87, - 134, 237, 5, 72, 239, 156, 1, 69, 4, 26, 69, 171, 217, 7, 79, 2, 227, - 220, 6, 76, 2, 183, 163, 4, 69, 4, 178, 235, 3, 87, 211, 215, 1, 73, 2, - 11, 87, 2, 21, 3, 73, 84, 72, 2, 11, 32, 2, 211, 143, 4, 85, 4, 29, 5, - 32, 84, 72, 65, 78, 5, 11, 32, 2, 11, 79, 2, 223, 176, 1, 82, 32, 34, 76, - 161, 207, 6, 2, 82, 75, 30, 40, 4, 73, 78, 71, 32, 255, 214, 7, 69, 28, - 112, 14, 67, 65, 84, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 65, 10, - 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 4, 30, 79, 157, 1, 2, 72, 69, 2, - 213, 231, 2, 3, 80, 69, 78, 24, 86, 72, 104, 10, 79, 80, 69, 78, 32, 77, - 79, 85, 84, 72, 178, 2, 83, 159, 205, 2, 84, 6, 34, 69, 50, 79, 207, 177, - 7, 65, 2, 189, 2, 8, 65, 82, 84, 45, 83, 72, 65, 80, 2, 135, 166, 4, 82, - 9, 29, 5, 32, 65, 78, 68, 32, 6, 26, 67, 58, 83, 63, 84, 2, 17, 2, 79, - 76, 2, 253, 174, 3, 4, 68, 32, 83, 87, 2, 11, 77, 2, 11, 73, 2, 11, 76, - 2, 117, 3, 73, 78, 71, 2, 37, 7, 73, 71, 72, 84, 76, 89, 45, 2, 11, 67, - 2, 21, 3, 76, 79, 83, 2, 17, 2, 69, 68, 2, 149, 190, 6, 3, 32, 69, 89, 8, - 64, 11, 77, 73, 76, 73, 78, 71, 32, 69, 89, 69, 83, 175, 1, 85, 7, 29, 5, - 32, 65, 78, 68, 32, 4, 52, 4, 72, 65, 78, 68, 57, 5, 84, 72, 82, 69, 69, - 2, 169, 226, 2, 9, 32, 67, 79, 86, 69, 82, 73, 78, 71, 2, 225, 163, 4, 2, - 32, 72, 2, 17, 2, 78, 71, 2, 241, 187, 6, 4, 76, 65, 83, 83, 2, 195, 183, - 4, 75, 16, 42, 65, 32, 2, 69, 69, 21, 2, 79, 87, 4, 210, 197, 6, 73, 203, - 114, 75, 2, 199, 198, 6, 90, 10, 100, 7, 32, 67, 65, 80, 80, 69, 68, 28, - 3, 77, 65, 78, 212, 203, 2, 3, 66, 79, 65, 195, 144, 1, 70, 2, 233, 135, - 4, 2, 32, 77, 5, 49, 10, 32, 87, 73, 84, 72, 79, 85, 84, 32, 83, 2, 247, - 178, 6, 78, 151, 3, 162, 2, 67, 48, 2, 70, 84, 228, 1, 6, 71, 68, 73, 65, + 218, 3, 83, 22, 84, 36, 2, 85, 80, 56, 4, 86, 69, 69, 32, 136, 253, 1, 3, + 71, 82, 69, 34, 72, 146, 4, 80, 234, 77, 78, 222, 240, 2, 70, 83, 81, 6, + 194, 203, 4, 77, 214, 49, 73, 135, 138, 1, 83, 2, 135, 206, 6, 69, 12, + 68, 7, 78, 84, 65, 73, 78, 83, 32, 202, 130, 2, 77, 167, 199, 5, 76, 6, + 36, 4, 65, 83, 32, 77, 175, 1, 87, 2, 11, 69, 2, 11, 77, 2, 171, 219, 7, + 66, 2, 219, 130, 2, 79, 12, 84, 9, 76, 69, 77, 69, 78, 84, 32, 79, 70, + 162, 130, 2, 81, 42, 88, 175, 216, 2, 77, 7, 17, 2, 32, 87, 4, 25, 4, 73, + 84, 72, 32, 4, 26, 86, 171, 233, 4, 79, 2, 129, 243, 5, 21, 69, 82, 84, + 73, 67, 65, 76, 32, 66, 65, 82, 32, 65, 84, 32, 69, 78, 68, 32, 79, 70, + 2, 221, 196, 5, 2, 32, 67, 8, 130, 1, 70, 247, 132, 2, 83, 40, 96, 3, 73, + 71, 72, 64, 13, 79, 77, 65, 78, 32, 78, 85, 77, 69, 82, 65, 76, 32, 191, + 255, 5, 69, 6, 17, 2, 84, 32, 6, 170, 143, 4, 67, 210, 3, 80, 147, 8, 84, + 32, 62, 69, 50, 70, 62, 79, 46, 84, 166, 163, 6, 83, 211, 86, 78, 4, 26, + 76, 231, 233, 7, 73, 2, 183, 164, 6, 69, 8, 26, 73, 167, 155, 7, 79, 6, + 130, 26, 86, 255, 140, 6, 70, 6, 11, 78, 6, 11, 69, 7, 247, 191, 2, 32, + 8, 42, 87, 190, 163, 6, 72, 231, 159, 1, 69, 4, 26, 69, 231, 146, 8, 79, + 2, 143, 150, 7, 76, 2, 147, 199, 4, 69, 4, 182, 248, 3, 87, 211, 128, 2, + 73, 2, 185, 243, 4, 9, 45, 80, 79, 73, 78, 84, 73, 78, 71, 2, 11, 87, 2, + 21, 3, 73, 84, 72, 2, 11, 32, 2, 163, 166, 4, 85, 4, 29, 5, 32, 84, 72, + 65, 78, 5, 11, 32, 2, 11, 79, 2, 135, 189, 1, 82, 32, 34, 76, 141, 136, + 7, 2, 82, 75, 30, 40, 4, 73, 78, 71, 32, 131, 144, 8, 69, 28, 112, 14, + 67, 65, 84, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 41, 10, 70, 65, + 67, 69, 32, 87, 73, 84, 72, 32, 4, 160, 1, 2, 72, 69, 163, 236, 4, 79, + 24, 86, 72, 108, 10, 79, 80, 69, 78, 32, 77, 79, 85, 84, 72, 246, 1, 83, + 203, 219, 2, 84, 6, 34, 69, 54, 79, 231, 234, 7, 65, 2, 165, 146, 5, 8, + 65, 82, 84, 45, 83, 72, 65, 80, 2, 219, 201, 4, 82, 9, 29, 5, 32, 65, 78, + 68, 32, 6, 26, 67, 58, 83, 71, 84, 2, 17, 2, 79, 76, 2, 145, 188, 3, 4, + 68, 32, 83, 87, 2, 11, 77, 2, 11, 73, 2, 11, 76, 2, 217, 144, 5, 3, 73, + 78, 71, 2, 37, 7, 73, 71, 72, 84, 76, 89, 45, 2, 231, 143, 5, 67, 8, 64, + 11, 77, 73, 76, 73, 78, 71, 32, 69, 89, 69, 83, 175, 1, 85, 7, 29, 5, 32, + 65, 78, 68, 32, 4, 52, 4, 72, 65, 78, 68, 57, 5, 84, 72, 82, 69, 69, 2, + 221, 232, 4, 9, 32, 67, 79, 86, 69, 82, 73, 78, 71, 2, 241, 199, 4, 2, + 32, 72, 2, 17, 2, 78, 71, 2, 173, 245, 6, 4, 76, 65, 83, 83, 2, 171, 220, + 4, 75, 16, 42, 65, 32, 2, 69, 69, 21, 2, 79, 87, 4, 142, 255, 6, 73, 227, + 114, 75, 2, 131, 128, 7, 90, 10, 100, 7, 32, 67, 65, 80, 80, 69, 68, 28, + 3, 77, 65, 78, 136, 218, 2, 3, 66, 79, 65, 167, 143, 1, 70, 2, 137, 159, + 4, 2, 32, 77, 5, 49, 10, 32, 87, 73, 84, 72, 79, 85, 84, 32, 83, 2, 179, + 236, 6, 78, 155, 3, 162, 2, 67, 48, 2, 70, 84, 228, 1, 6, 71, 68, 73, 65, 78, 32, 244, 10, 3, 76, 73, 68, 140, 2, 12, 79, 78, 32, 87, 73, 84, 72, 32, 82, 73, 71, 72, 20, 11, 82, 65, 32, 83, 79, 77, 80, 69, 78, 71, 32, - 238, 2, 85, 172, 8, 6, 89, 79, 77, 66, 79, 32, 162, 169, 6, 77, 234, 134, - 1, 72, 3, 83, 4, 128, 214, 2, 3, 67, 69, 82, 247, 182, 4, 75, 10, 70, 32, - 124, 9, 87, 65, 82, 69, 45, 70, 85, 78, 67, 155, 196, 5, 66, 6, 58, 72, - 44, 6, 73, 67, 69, 32, 67, 82, 231, 133, 6, 83, 2, 11, 89, 2, 11, 80, 2, - 135, 248, 5, 72, 2, 171, 143, 6, 69, 2, 11, 84, 2, 251, 211, 5, 73, 84, + 234, 2, 85, 216, 8, 6, 89, 79, 77, 66, 79, 32, 182, 226, 6, 77, 130, 135, + 1, 72, 3, 83, 4, 232, 227, 2, 3, 67, 69, 82, 215, 226, 4, 75, 10, 70, 32, + 124, 9, 87, 65, 82, 69, 45, 70, 85, 78, 67, 223, 250, 5, 66, 6, 58, 72, + 44, 6, 73, 67, 69, 32, 67, 82, 239, 188, 6, 83, 2, 11, 89, 2, 11, 80, 2, + 231, 174, 6, 72, 2, 191, 199, 6, 69, 2, 11, 84, 2, 203, 138, 6, 73, 84, 248, 1, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 168, 2, 13, 73, 78, 68, 69, 80, 69, 78, 68, 69, 78, 84, 32, 83, 20, 7, 76, 69, 84, 84, 69, 82, 32, 188, 3, 7, 78, 85, 77, 66, 69, 82, 32, 113, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 22, 140, 1, 3, 72, 79, 79, 20, 4, 76, 79, - 78, 71, 50, 83, 58, 84, 242, 213, 3, 68, 216, 199, 1, 4, 82, 69, 83, 72, - 249, 13, 4, 67, 85, 82, 86, 4, 199, 172, 5, 75, 2, 25, 4, 32, 72, 79, 79, - 2, 147, 158, 5, 75, 2, 21, 3, 84, 82, 79, 2, 11, 75, 2, 219, 157, 5, 69, - 4, 189, 208, 1, 2, 87, 79, 2, 211, 196, 6, 72, 42, 190, 1, 65, 50, 71, - 34, 76, 56, 5, 82, 69, 83, 72, 45, 2, 90, 34, 83, 66, 89, 154, 193, 1, - 72, 234, 5, 75, 130, 76, 66, 2, 70, 190, 173, 4, 78, 254, 1, 84, 2, 87, - 202, 103, 80, 171, 4, 77, 4, 26, 76, 231, 194, 6, 89, 2, 215, 200, 1, 69, - 2, 11, 73, 2, 167, 225, 2, 77, 4, 26, 65, 183, 220, 5, 69, 2, 189, 194, - 1, 2, 77, 69, 2, 11, 65, 2, 235, 193, 6, 89, 6, 26, 65, 191, 193, 6, 72, - 4, 178, 185, 1, 77, 187, 218, 5, 68, 2, 195, 193, 1, 79, 8, 18, 79, 59, - 84, 4, 11, 78, 4, 11, 69, 5, 11, 32, 2, 227, 151, 2, 72, 4, 230, 213, 5, - 87, 187, 154, 1, 69, 10, 66, 67, 0, 6, 72, 65, 76, 70, 32, 67, 53, 4, 84, - 87, 79, 32, 2, 21, 3, 73, 82, 67, 2, 201, 193, 4, 2, 76, 69, 6, 100, 13, + 78, 71, 50, 83, 58, 84, 142, 227, 3, 68, 176, 237, 1, 4, 82, 69, 83, 72, + 129, 15, 4, 67, 85, 82, 86, 4, 195, 224, 5, 75, 2, 25, 4, 32, 72, 79, 79, + 2, 135, 209, 5, 75, 2, 21, 3, 84, 82, 79, 2, 11, 75, 2, 207, 208, 5, 69, + 4, 193, 223, 1, 2, 87, 79, 2, 143, 254, 6, 72, 42, 190, 1, 65, 50, 71, + 34, 76, 56, 5, 82, 69, 83, 72, 45, 2, 90, 34, 83, 66, 89, 158, 208, 1, + 72, 234, 5, 75, 198, 75, 66, 2, 70, 178, 216, 4, 78, 134, 2, 84, 2, 87, + 218, 103, 80, 171, 4, 77, 4, 26, 76, 163, 252, 6, 89, 2, 219, 215, 1, 69, + 2, 11, 73, 2, 239, 238, 2, 77, 4, 26, 65, 135, 147, 6, 69, 2, 193, 209, + 1, 2, 77, 69, 2, 11, 65, 2, 167, 251, 6, 89, 6, 26, 65, 251, 250, 6, 72, + 4, 182, 200, 1, 77, 139, 133, 6, 68, 2, 199, 208, 1, 79, 8, 18, 79, 59, + 84, 4, 11, 78, 4, 11, 69, 5, 11, 32, 2, 167, 166, 2, 72, 4, 182, 140, 6, + 87, 179, 157, 1, 69, 10, 66, 67, 0, 6, 72, 65, 76, 70, 32, 67, 53, 4, 84, + 87, 79, 32, 2, 21, 3, 73, 82, 67, 2, 173, 236, 4, 2, 76, 69, 6, 100, 13, 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 82, 83, 13, 8, 67, 73, 82, - 67, 76, 69, 83, 32, 5, 11, 32, 2, 153, 240, 5, 4, 87, 73, 84, 72, 8, 30, + 67, 76, 69, 83, 32, 5, 11, 32, 2, 249, 166, 6, 4, 87, 73, 84, 72, 8, 30, 32, 189, 1, 2, 85, 83, 4, 93, 21, 81, 85, 73, 76, 84, 32, 83, 81, 85, 65, 82, 69, 32, 79, 82, 78, 65, 77, 69, 78, 84, 5, 11, 32, 2, 11, 73, 2, 17, - 2, 78, 32, 2, 11, 66, 2, 173, 155, 6, 4, 76, 65, 67, 75, 5, 173, 162, 4, - 7, 32, 87, 73, 84, 72, 32, 79, 2, 251, 227, 3, 84, 70, 52, 7, 76, 69, 84, - 84, 69, 82, 32, 207, 204, 5, 68, 50, 190, 1, 69, 32, 2, 77, 65, 30, 78, - 38, 66, 2, 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, 80, 2, 82, 2, - 83, 2, 84, 2, 86, 2, 89, 186, 183, 7, 65, 2, 73, 2, 79, 3, 85, 4, 150, - 184, 7, 69, 147, 1, 72, 4, 134, 185, 7, 69, 3, 72, 6, 34, 71, 2, 89, 187, - 183, 7, 65, 2, 183, 183, 7, 65, 54, 104, 3, 84, 72, 32, 197, 233, 5, 17, - 78, 68, 32, 82, 69, 67, 79, 82, 68, 73, 78, 71, 32, 67, 79, 80, 89, 52, - 60, 5, 69, 65, 83, 84, 32, 237, 2, 5, 87, 69, 83, 84, 32, 28, 96, 5, 65, - 82, 82, 79, 87, 162, 4, 80, 130, 1, 84, 246, 160, 4, 66, 38, 68, 18, 83, - 203, 43, 87, 11, 11, 32, 8, 52, 2, 65, 78, 38, 67, 120, 2, 84, 79, 139, - 2, 87, 2, 237, 2, 5, 68, 32, 83, 79, 85, 2, 37, 7, 82, 79, 83, 83, 73, - 78, 71, 2, 17, 2, 32, 78, 2, 17, 2, 79, 82, 2, 221, 209, 4, 5, 84, 72, - 32, 69, 65, 2, 17, 2, 32, 67, 2, 239, 219, 3, 79, 24, 96, 5, 65, 82, 82, - 79, 87, 182, 1, 80, 130, 1, 84, 246, 160, 4, 66, 38, 68, 18, 83, 203, 43, - 87, 7, 11, 32, 4, 36, 5, 65, 78, 68, 32, 78, 75, 87, 2, 17, 2, 79, 82, 2, - 21, 3, 84, 72, 32, 2, 201, 207, 4, 2, 87, 69, 2, 21, 3, 73, 84, 72, 2, - 11, 32, 2, 183, 225, 5, 72, 6, 41, 8, 79, 73, 78, 84, 73, 78, 71, 32, 6, - 42, 86, 154, 253, 3, 76, 215, 137, 2, 66, 2, 17, 2, 73, 78, 2, 239, 252, - 3, 69, 4, 89, 20, 82, 73, 65, 78, 71, 76, 69, 45, 72, 69, 65, 68, 69, 68, - 32, 65, 82, 82, 79, 87, 5, 11, 32, 2, 147, 214, 4, 84, 166, 1, 212, 3, - 23, 67, 76, 85, 83, 84, 69, 82, 45, 73, 78, 73, 84, 73, 65, 76, 32, 76, - 69, 84, 84, 69, 82, 32, 40, 21, 70, 73, 78, 65, 76, 32, 67, 79, 78, 83, - 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 114, 71, 32, 27, 72, 69, 65, - 68, 32, 77, 65, 82, 75, 32, 87, 73, 84, 72, 32, 77, 79, 79, 78, 32, 65, - 78, 68, 32, 83, 85, 78, 104, 7, 76, 69, 84, 84, 69, 82, 32, 180, 1, 5, - 77, 65, 82, 75, 32, 54, 83, 100, 8, 84, 69, 82, 77, 73, 78, 65, 76, 41, - 6, 86, 79, 87, 69, 76, 32, 8, 246, 228, 6, 83, 254, 68, 76, 3, 82, 24, - 162, 184, 3, 83, 206, 251, 1, 78, 222, 245, 1, 45, 186, 2, 66, 2, 68, 2, - 71, 2, 75, 2, 76, 2, 77, 3, 82, 2, 157, 166, 6, 3, 69, 77, 73, 7, 29, 5, - 32, 65, 78, 68, 32, 4, 50, 70, 1, 8, 84, 82, 73, 80, 76, 69, 32, 70, 2, - 187, 158, 5, 76, 82, 234, 174, 2, 68, 206, 178, 4, 75, 38, 78, 46, 83, - 38, 84, 46, 66, 2, 67, 2, 71, 2, 74, 2, 80, 2, 90, 254, 68, 45, 2, 72, 2, - 76, 2, 77, 2, 82, 2, 86, 2, 89, 187, 2, 65, 8, 158, 153, 3, 80, 214, 200, - 3, 68, 58, 83, 63, 84, 10, 40, 4, 73, 71, 78, 32, 211, 229, 6, 85, 8, - 146, 246, 4, 74, 158, 2, 86, 122, 85, 231, 233, 1, 65, 4, 233, 247, 1, 5, - 32, 77, 65, 82, 75, 22, 44, 5, 83, 73, 71, 78, 32, 251, 228, 6, 76, 20, - 88, 7, 86, 79, 67, 65, 76, 73, 67, 226, 229, 6, 65, 26, 79, 2, 85, 150, - 64, 69, 3, 73, 4, 11, 32, 4, 254, 165, 7, 76, 3, 82, 61, 166, 1, 65, 142, - 1, 69, 136, 5, 13, 72, 69, 82, 73, 67, 65, 76, 32, 65, 78, 71, 76, 69, - 82, 73, 220, 1, 12, 76, 65, 83, 72, 73, 78, 71, 32, 83, 87, 69, 65, 23, - 79, 13, 60, 4, 71, 72, 69, 84, 20, 3, 82, 75, 76, 151, 141, 7, 67, 2, - 163, 144, 7, 84, 6, 26, 73, 239, 159, 5, 69, 2, 165, 156, 6, 2, 78, 71, - 18, 52, 2, 65, 75, 250, 3, 69, 69, 4, 83, 77, 73, 76, 12, 70, 45, 64, 2, - 69, 82, 173, 2, 8, 73, 78, 71, 32, 72, 69, 65, 68, 2, 221, 139, 5, 11, - 78, 79, 45, 69, 86, 73, 76, 32, 77, 79, 78, 9, 33, 6, 32, 87, 73, 84, 72, - 32, 6, 26, 67, 78, 79, 75, 84, 2, 33, 6, 65, 78, 67, 69, 76, 76, 2, 245, - 240, 6, 5, 65, 84, 73, 79, 78, 2, 41, 8, 78, 69, 32, 83, 79, 85, 78, 68, - 2, 133, 184, 5, 2, 32, 87, 2, 25, 4, 72, 82, 69, 69, 2, 157, 140, 6, 10, - 32, 83, 79, 85, 78, 68, 32, 87, 65, 86, 2, 11, 32, 2, 245, 218, 5, 9, 73, - 78, 32, 83, 73, 76, 72, 79, 85, 4, 34, 68, 241, 232, 2, 2, 67, 72, 2, 11, - 66, 2, 255, 138, 6, 79, 2, 211, 151, 6, 79, 7, 45, 9, 32, 79, 80, 69, 78, - 73, 78, 71, 32, 4, 138, 243, 6, 76, 227, 42, 85, 10, 44, 3, 68, 69, 82, - 41, 4, 82, 65, 76, 32, 5, 17, 2, 32, 87, 2, 239, 226, 5, 69, 6, 80, 8, - 67, 65, 76, 69, 78, 68, 65, 82, 0, 4, 78, 79, 84, 69, 29, 2, 83, 72, 2, - 137, 214, 6, 2, 32, 80, 2, 223, 151, 5, 69, 2, 235, 136, 6, 84, 10, 90, - 79, 60, 7, 85, 84, 73, 78, 71, 32, 87, 128, 228, 2, 3, 82, 84, 83, 223, - 130, 3, 78, 4, 144, 84, 7, 76, 32, 79, 70, 32, 84, 72, 251, 198, 6, 78, - 2, 11, 72, 2, 159, 249, 5, 65, 206, 5, 26, 65, 255, 211, 6, 73, 204, 5, - 28, 2, 82, 69, 175, 66, 84, 202, 5, 30, 32, 129, 44, 2, 68, 32, 218, 3, - 246, 1, 65, 116, 2, 86, 32, 58, 66, 158, 1, 67, 170, 1, 68, 102, 69, 194, - 2, 70, 74, 71, 222, 1, 72, 214, 2, 73, 94, 75, 206, 4, 76, 102, 77, 210, - 5, 78, 94, 79, 178, 1, 80, 222, 3, 82, 154, 3, 83, 186, 1, 84, 74, 87, - 218, 6, 89, 175, 158, 6, 85, 14, 114, 32, 152, 71, 2, 78, 80, 236, 142, - 2, 3, 82, 85, 72, 238, 51, 65, 240, 43, 2, 80, 65, 250, 224, 3, 77, 3, - 85, 2, 21, 3, 79, 86, 69, 2, 11, 82, 2, 219, 133, 7, 32, 14, 98, 65, 36, - 4, 85, 83, 83, 89, 178, 136, 3, 73, 212, 64, 2, 79, 82, 202, 39, 69, 207, - 164, 3, 81, 4, 32, 2, 65, 82, 155, 149, 7, 82, 2, 175, 136, 3, 69, 22, - 98, 65, 30, 79, 230, 18, 77, 204, 128, 7, 7, 32, 79, 86, 69, 82, 32, 75, - 74, 85, 14, 67, 3, 68, 4, 158, 148, 7, 76, 3, 80, 5, 17, 2, 82, 80, 2, - 247, 181, 2, 79, 20, 82, 65, 234, 17, 77, 142, 195, 2, 69, 174, 49, 79, - 234, 140, 4, 66, 2, 74, 3, 76, 5, 207, 20, 65, 18, 66, 69, 22, 82, 236, - 23, 5, 83, 85, 75, 85, 85, 183, 250, 6, 86, 2, 235, 162, 5, 75, 12, 52, - 7, 65, 32, 78, 65, 77, 69, 32, 235, 145, 7, 71, 10, 132, 1, 4, 72, 69, - 73, 83, 20, 5, 84, 65, 73, 83, 89, 224, 20, 3, 77, 69, 73, 240, 182, 2, - 3, 82, 69, 73, 1, 4, 83, 89, 79, 85, 2, 135, 253, 6, 69, 2, 227, 235, 6, - 79, 6, 26, 79, 159, 144, 7, 77, 4, 128, 226, 5, 2, 85, 82, 239, 145, 1, - 79, 24, 102, 65, 46, 73, 188, 12, 5, 85, 82, 65, 77, 85, 178, 139, 3, 72, - 186, 244, 3, 80, 186, 2, 66, 3, 89, 6, 166, 191, 6, 82, 210, 46, 78, 147, - 33, 76, 6, 42, 82, 150, 210, 4, 78, 231, 185, 2, 71, 2, 229, 158, 5, 2, - 85, 68, 30, 130, 1, 65, 22, 69, 54, 79, 62, 85, 226, 202, 1, 80, 196, - 184, 5, 10, 73, 82, 65, 71, 65, 78, 65, 32, 72, 79, 234, 8, 71, 3, 90, 5, - 207, 232, 1, 73, 4, 184, 11, 3, 75, 85, 84, 129, 221, 1, 2, 82, 85, 6, - 26, 79, 175, 140, 7, 78, 4, 194, 231, 6, 82, 235, 36, 78, 6, 54, 73, 160, - 17, 4, 65, 82, 65, 68, 131, 179, 5, 82, 2, 203, 191, 3, 73, 14, 54, 78, - 228, 12, 4, 77, 65, 71, 69, 167, 254, 6, 85, 7, 190, 199, 2, 73, 243, - 175, 4, 84, 58, 230, 1, 65, 64, 2, 89, 85, 20, 3, 73, 82, 79, 78, 77, 90, - 79, 60, 2, 85, 82, 144, 8, 2, 69, 69, 202, 135, 3, 72, 202, 237, 2, 67, - 208, 120, 3, 32, 79, 72, 162, 14, 80, 186, 2, 66, 2, 71, 2, 75, 2, 76, 2, - 84, 2, 86, 3, 87, 9, 18, 73, 23, 82, 2, 143, 245, 6, 82, 4, 22, 79, 143, - 20, 65, 2, 159, 204, 4, 82, 9, 252, 16, 3, 77, 69, 69, 248, 2, 2, 87, 65, - 169, 161, 2, 3, 71, 85, 82, 9, 11, 32, 6, 22, 67, 199, 14, 83, 4, 22, 65, - 179, 6, 85, 2, 209, 185, 4, 2, 80, 73, 4, 18, 79, 23, 82, 2, 251, 231, 6, - 80, 2, 151, 204, 5, 85, 4, 176, 154, 5, 4, 85, 90, 69, 73, 239, 66, 79, - 12, 62, 79, 244, 13, 2, 69, 70, 230, 247, 6, 77, 2, 78, 3, 88, 4, 202, - 209, 5, 90, 143, 180, 1, 71, 82, 162, 1, 32, 70, 65, 122, 66, 22, 69, 54, - 73, 110, 77, 68, 2, 85, 32, 78, 86, 2, 87, 162, 137, 3, 72, 214, 237, 2, - 79, 230, 134, 1, 80, 186, 2, 71, 2, 76, 3, 83, 10, 34, 79, 242, 2, 67, - 139, 8, 83, 6, 198, 10, 86, 207, 232, 6, 72, 13, 54, 73, 44, 2, 78, 83, - 146, 82, 82, 159, 185, 4, 72, 4, 252, 150, 5, 2, 75, 85, 175, 199, 1, 82, - 2, 135, 179, 6, 89, 5, 243, 160, 2, 32, 6, 28, 2, 71, 65, 251, 10, 69, 5, - 191, 178, 6, 84, 8, 42, 75, 20, 2, 82, 73, 207, 129, 7, 76, 2, 211, 221, - 5, 85, 5, 11, 66, 2, 11, 65, 2, 203, 244, 2, 65, 7, 11, 32, 4, 22, 67, - 139, 8, 83, 2, 11, 85, 2, 175, 161, 2, 66, 16, 218, 128, 7, 65, 2, 70, 2, - 71, 2, 76, 2, 77, 2, 83, 2, 86, 3, 87, 5, 11, 32, 2, 11, 77, 2, 191, 189, - 6, 69, 16, 70, 65, 238, 10, 79, 178, 244, 6, 70, 2, 77, 2, 83, 2, 86, 3, - 87, 5, 167, 224, 6, 78, 12, 78, 78, 20, 7, 82, 73, 71, 73, 78, 65, 76, - 170, 171, 2, 79, 255, 210, 4, 86, 2, 207, 217, 6, 83, 6, 21, 3, 32, 79, - 70, 7, 25, 4, 32, 79, 82, 32, 4, 214, 86, 78, 51, 69, 46, 118, 65, 82, - 69, 94, 73, 74, 79, 150, 234, 6, 80, 218, 16, 67, 2, 70, 2, 72, 2, 77, 2, - 82, 2, 83, 2, 86, 3, 87, 9, 38, 65, 221, 220, 3, 3, 32, 65, 77, 4, 236, - 1, 2, 83, 69, 227, 213, 6, 84, 8, 34, 69, 22, 78, 231, 220, 6, 83, 2, - 147, 232, 6, 90, 4, 230, 215, 1, 73, 139, 255, 4, 83, 6, 34, 75, 233, 3, - 3, 65, 83, 85, 4, 166, 238, 2, 85, 235, 140, 4, 79, 6, 34, 73, 22, 78, - 21, 2, 83, 73, 2, 191, 174, 3, 78, 2, 191, 219, 6, 68, 2, 133, 193, 6, 4, - 84, 73, 79, 78, 22, 60, 2, 65, 68, 114, 69, 62, 73, 134, 1, 85, 227, 166, - 6, 79, 7, 21, 3, 32, 79, 86, 4, 25, 4, 69, 82, 32, 83, 5, 17, 2, 32, 83, - 2, 11, 81, 2, 153, 153, 2, 2, 85, 65, 4, 36, 3, 78, 84, 79, 191, 211, 6, - 77, 2, 175, 167, 5, 71, 6, 40, 2, 71, 72, 62, 84, 243, 244, 6, 82, 2, - 181, 205, 3, 10, 84, 32, 79, 80, 69, 78, 32, 66, 79, 88, 2, 189, 234, 2, - 2, 84, 79, 4, 172, 1, 2, 85, 66, 195, 185, 4, 80, 14, 82, 65, 72, 3, 69, - 78, 84, 128, 178, 2, 3, 73, 82, 73, 202, 195, 4, 82, 3, 86, 4, 48, 2, 73, - 75, 225, 162, 2, 4, 78, 84, 73, 73, 2, 243, 232, 2, 85, 4, 198, 245, 6, - 73, 3, 79, 6, 42, 65, 146, 254, 2, 72, 159, 167, 3, 79, 2, 241, 191, 6, - 2, 82, 71, 34, 50, 65, 20, 4, 73, 84, 72, 32, 159, 244, 6, 66, 2, 167, - 168, 3, 84, 30, 222, 1, 66, 40, 3, 68, 73, 65, 0, 5, 79, 82, 84, 72, 79, - 84, 2, 72, 79, 42, 76, 126, 84, 28, 6, 85, 80, 80, 69, 82, 32, 186, 1, - 86, 190, 227, 3, 82, 225, 241, 1, 13, 67, 79, 78, 84, 79, 85, 82, 69, 68, - 32, 79, 85, 84, 2, 209, 231, 3, 5, 79, 84, 84, 79, 77, 2, 165, 129, 4, - 16, 71, 79, 78, 65, 76, 32, 67, 82, 79, 83, 83, 72, 65, 84, 67, 72, 2, - 149, 3, 6, 82, 73, 90, 79, 78, 84, 6, 44, 5, 79, 87, 69, 82, 32, 211, - 229, 3, 69, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 189, 1, 3, 84, - 32, 68, 2, 177, 229, 3, 2, 79, 80, 8, 60, 5, 76, 69, 70, 84, 32, 37, 6, - 82, 73, 71, 72, 84, 32, 4, 70, 68, 253, 241, 3, 2, 84, 79, 4, 34, 68, - 205, 253, 3, 2, 84, 79, 2, 141, 228, 3, 7, 73, 65, 71, 79, 78, 65, 76, 2, - 29, 5, 69, 82, 84, 73, 67, 2, 185, 253, 3, 2, 65, 76, 6, 32, 2, 65, 65, - 183, 166, 5, 85, 4, 254, 200, 6, 82, 247, 5, 68, 240, 1, 226, 1, 67, 174, - 9, 68, 34, 70, 88, 3, 82, 73, 83, 142, 1, 72, 66, 75, 106, 76, 166, 1, - 77, 38, 78, 34, 79, 94, 80, 34, 83, 190, 2, 84, 156, 1, 5, 69, 73, 71, - 72, 84, 22, 85, 90, 86, 166, 199, 4, 65, 222, 105, 87, 207, 96, 73, 90, - 128, 1, 21, 74, 75, 32, 85, 78, 73, 70, 73, 69, 68, 32, 73, 68, 69, 79, - 71, 82, 65, 80, 72, 45, 134, 216, 5, 79, 219, 146, 1, 76, 86, 68, 2, 52, - 69, 82, 53, 206, 2, 54, 150, 2, 55, 166, 1, 56, 95, 57, 10, 50, 48, 234, - 4, 65, 154, 190, 5, 56, 207, 96, 50, 4, 222, 233, 6, 48, 3, 57, 30, 150, - 1, 50, 42, 51, 38, 52, 48, 2, 53, 66, 0, 2, 68, 69, 22, 57, 164, 4, 2, - 56, 70, 162, 143, 3, 66, 248, 172, 2, 2, 70, 56, 205, 96, 2, 49, 56, 6, - 214, 4, 55, 194, 157, 6, 49, 3, 52, 4, 150, 148, 3, 70, 143, 173, 2, 67, - 4, 26, 48, 159, 148, 3, 51, 2, 195, 231, 6, 56, 2, 175, 231, 6, 54, 4, - 242, 147, 3, 50, 243, 208, 3, 49, 24, 86, 50, 46, 51, 50, 53, 34, 54, 16, - 2, 55, 48, 28, 2, 70, 49, 129, 2, 2, 69, 56, 6, 70, 57, 238, 145, 3, 53, - 203, 153, 2, 52, 4, 26, 53, 199, 146, 3, 48, 2, 215, 229, 6, 53, 4, 202, - 2, 66, 211, 143, 3, 57, 2, 171, 2, 50, 4, 146, 229, 6, 56, 3, 57, 2, 247, - 228, 6, 52, 12, 74, 53, 38, 57, 12, 2, 49, 50, 20, 2, 68, 52, 157, 225, - 6, 2, 65, 55, 4, 170, 144, 3, 51, 227, 210, 2, 49, 2, 11, 56, 2, 231, - 227, 6, 49, 2, 211, 227, 6, 50, 6, 50, 57, 20, 2, 68, 55, 209, 143, 3, 2, - 67, 65, 2, 159, 143, 3, 69, 2, 247, 226, 6, 48, 4, 204, 156, 6, 2, 49, - 52, 221, 67, 2, 48, 52, 2, 11, 79, 2, 251, 183, 3, 84, 10, 84, 3, 65, 76, - 76, 104, 4, 79, 85, 82, 32, 240, 5, 3, 73, 86, 69, 179, 238, 4, 82, 2, - 57, 12, 73, 78, 71, 32, 68, 73, 65, 71, 79, 78, 65, 76, 2, 11, 32, 2, 11, - 83, 2, 191, 251, 4, 76, 4, 210, 179, 3, 68, 131, 173, 3, 75, 8, 192, 181, - 3, 2, 73, 45, 158, 242, 2, 68, 210, 56, 67, 3, 86, 8, 56, 8, 65, 84, 65, - 75, 65, 78, 65, 32, 239, 205, 6, 69, 6, 150, 172, 6, 75, 202, 28, 68, - 159, 20, 83, 60, 36, 5, 65, 84, 73, 78, 32, 79, 79, 54, 164, 5, 12, 83, - 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 227, 112, 67, 6, 178, 178, 3, - 71, 157, 149, 2, 3, 83, 83, 76, 4, 234, 218, 4, 73, 251, 130, 2, 86, 4, - 174, 223, 5, 69, 147, 126, 71, 4, 140, 3, 15, 78, 69, 32, 72, 85, 78, 68, - 82, 69, 68, 32, 84, 87, 69, 78, 147, 218, 6, 75, 4, 158, 2, 80, 187, 215, - 4, 76, 20, 110, 69, 146, 1, 72, 20, 2, 73, 88, 198, 200, 4, 65, 150, 66, - 77, 130, 47, 81, 234, 98, 79, 210, 61, 68, 3, 83, 4, 56, 7, 67, 79, 78, - 68, 32, 83, 67, 21, 3, 86, 69, 78, 2, 247, 137, 5, 82, 2, 29, 5, 32, 80, - 79, 73, 78, 2, 11, 84, 2, 143, 177, 5, 32, 2, 163, 218, 6, 86, 2, 17, 2, - 84, 89, 2, 239, 217, 6, 32, 8, 44, 4, 72, 82, 69, 69, 22, 87, 243, 4, 73, - 2, 163, 147, 6, 32, 4, 68, 13, 69, 78, 84, 89, 45, 84, 87, 79, 32, 80, - 79, 73, 78, 19, 79, 2, 215, 75, 84, 2, 179, 213, 6, 32, 4, 48, 6, 80, 32, - 87, 73, 84, 72, 243, 145, 6, 72, 2, 17, 2, 32, 69, 2, 235, 108, 88, 4, - 202, 145, 6, 79, 151, 70, 83, 2, 201, 183, 5, 6, 32, 66, 76, 65, 67, 75, - 6, 146, 215, 6, 50, 2, 51, 3, 65, 85, 134, 1, 65, 138, 7, 69, 236, 1, 10, - 73, 67, 75, 32, 70, 73, 71, 85, 82, 69, 182, 1, 79, 90, 82, 234, 2, 85, - 130, 200, 6, 83, 3, 88, 38, 92, 6, 70, 70, 32, 79, 70, 32, 106, 77, 80, - 2, 78, 68, 90, 82, 206, 3, 84, 175, 167, 5, 68, 4, 60, 8, 65, 69, 83, 67, - 85, 76, 65, 80, 21, 3, 72, 69, 82, 2, 219, 209, 4, 73, 2, 211, 192, 5, - 77, 2, 21, 3, 80, 69, 68, 2, 17, 2, 32, 69, 2, 173, 210, 1, 4, 78, 86, - 69, 76, 2, 21, 3, 73, 78, 71, 2, 17, 2, 32, 80, 2, 11, 69, 2, 11, 82, 2, - 171, 131, 6, 83, 24, 42, 32, 205, 1, 5, 84, 32, 79, 70, 32, 12, 82, 69, - 54, 79, 240, 224, 1, 8, 65, 78, 68, 32, 67, 82, 69, 83, 131, 229, 1, 87, - 2, 17, 2, 81, 85, 2, 11, 65, 2, 135, 148, 6, 76, 4, 44, 5, 70, 32, 68, - 65, 86, 143, 167, 3, 80, 2, 255, 138, 6, 73, 12, 66, 71, 30, 83, 40, 4, - 80, 82, 79, 84, 142, 66, 72, 155, 102, 84, 2, 89, 4, 85, 65, 82, 68, 4, - 26, 69, 155, 239, 1, 84, 2, 11, 76, 2, 21, 3, 69, 67, 84, 2, 29, 5, 69, - 68, 32, 65, 82, 2, 135, 205, 6, 69, 4, 144, 229, 4, 10, 85, 69, 32, 79, - 70, 32, 76, 73, 66, 69, 187, 154, 1, 73, 8, 92, 2, 65, 77, 112, 9, 78, - 79, 71, 82, 65, 80, 72, 73, 67, 193, 203, 1, 4, 84, 72, 79, 83, 4, 54, - 32, 173, 196, 5, 7, 73, 78, 71, 32, 66, 79, 87, 2, 33, 6, 76, 79, 67, 79, - 77, 79, 2, 135, 197, 1, 84, 2, 137, 130, 4, 2, 32, 70, 11, 11, 32, 8, 60, - 5, 87, 73, 84, 72, 32, 193, 153, 2, 4, 76, 69, 65, 78, 4, 32, 4, 65, 82, - 77, 83, 51, 68, 2, 25, 4, 32, 82, 65, 73, 2, 167, 236, 1, 83, 2, 143, - 181, 5, 82, 4, 44, 4, 67, 75, 32, 67, 21, 3, 80, 87, 65, 2, 139, 196, 5, - 72, 2, 211, 168, 6, 84, 12, 66, 65, 108, 10, 69, 83, 83, 32, 79, 85, 84, - 76, 73, 78, 79, 73, 6, 44, 4, 73, 71, 72, 84, 45, 3, 87, 66, 69, 4, 188, - 130, 3, 2, 32, 82, 243, 176, 2, 78, 2, 147, 61, 82, 2, 17, 2, 69, 68, 2, - 17, 2, 32, 87, 2, 145, 182, 4, 4, 72, 73, 84, 69, 4, 52, 4, 78, 71, 32, - 84, 233, 7, 4, 67, 84, 76, 89, 2, 17, 2, 69, 82, 2, 169, 143, 6, 3, 77, - 73, 78, 6, 100, 8, 68, 73, 79, 32, 77, 73, 67, 82, 20, 9, 70, 70, 69, 68, - 32, 70, 76, 65, 84, 207, 196, 6, 80, 2, 215, 157, 5, 79, 2, 11, 66, 2, - 231, 248, 3, 82, 202, 2, 150, 1, 66, 196, 2, 5, 67, 67, 69, 69, 68, 192, - 3, 8, 77, 77, 65, 84, 73, 79, 78, 32, 70, 78, 132, 15, 3, 80, 69, 82, - 148, 11, 2, 82, 70, 47, 83, 63, 11, 83, 60, 72, 6, 67, 82, 73, 80, 84, - 32, 124, 3, 69, 84, 32, 93, 3, 84, 73, 84, 30, 222, 22, 69, 162, 1, 80, - 150, 208, 2, 77, 202, 1, 76, 26, 82, 222, 237, 1, 70, 30, 83, 42, 84, 62, - 90, 130, 83, 78, 15, 79, 28, 56, 6, 65, 66, 79, 86, 69, 32, 246, 25, 79, - 159, 3, 87, 6, 142, 24, 83, 203, 139, 5, 82, 2, 235, 254, 4, 85, 22, 11, - 83, 23, 11, 32, 20, 128, 1, 6, 65, 66, 79, 86, 69, 32, 156, 1, 7, 66, 85, - 84, 32, 78, 79, 84, 32, 2, 79, 82, 229, 255, 2, 5, 85, 78, 68, 69, 82, - 12, 100, 4, 78, 79, 84, 32, 28, 12, 83, 73, 78, 71, 76, 69, 45, 76, 73, - 78, 69, 32, 218, 24, 65, 39, 69, 4, 242, 24, 65, 167, 1, 69, 4, 250, 24, - 69, 83, 78, 2, 93, 5, 32, 69, 81, 85, 73, 4, 17, 2, 32, 69, 4, 17, 2, 81, - 85, 4, 22, 73, 167, 25, 65, 2, 173, 25, 6, 86, 65, 76, 69, 78, 84, 6, - 202, 246, 2, 66, 228, 196, 1, 4, 87, 73, 84, 72, 147, 139, 1, 84, 161, 1, - 166, 1, 32, 72, 7, 68, 65, 78, 69, 83, 69, 32, 200, 12, 4, 82, 73, 83, - 69, 200, 166, 2, 14, 83, 69, 84, 32, 79, 86, 69, 82, 32, 66, 85, 73, 76, - 68, 255, 240, 2, 70, 4, 50, 87, 197, 146, 5, 6, 66, 69, 72, 73, 78, 68, - 2, 247, 255, 5, 73, 146, 1, 198, 2, 65, 20, 17, 67, 79, 78, 83, 79, 78, - 65, 78, 84, 32, 83, 73, 71, 78, 32, 80, 65, 172, 1, 7, 76, 69, 84, 84, - 69, 82, 32, 188, 3, 18, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, - 66, 73, 78, 68, 85, 32, 228, 1, 5, 83, 73, 71, 78, 32, 200, 1, 13, 86, - 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 80, 65, 255, 195, 4, 68, 2, 235, - 211, 2, 86, 10, 68, 2, 77, 73, 40, 2, 78, 89, 33, 7, 83, 65, 78, 71, 65, - 78, 32, 2, 17, 2, 78, 71, 2, 167, 176, 5, 75, 4, 142, 5, 65, 163, 146, 1, - 73, 4, 226, 182, 6, 77, 3, 87, 76, 246, 1, 65, 58, 70, 78, 76, 2, 82, 34, - 83, 202, 230, 2, 69, 246, 215, 1, 78, 246, 175, 1, 66, 2, 75, 254, 68, - 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 77, 2, 80, 2, 81, 2, 84, 2, 86, 2, 87, - 2, 88, 2, 89, 2, 90, 186, 2, 73, 2, 79, 3, 85, 7, 128, 247, 5, 6, 82, 67, - 72, 65, 73, 67, 135, 64, 69, 6, 44, 5, 73, 78, 65, 76, 32, 163, 182, 6, - 65, 4, 158, 182, 6, 75, 3, 77, 4, 154, 145, 6, 69, 235, 36, 65, 4, 170, - 179, 6, 89, 187, 2, 65, 16, 90, 66, 2, 68, 2, 75, 12, 3, 76, 69, 85, 38, - 67, 34, 80, 189, 177, 6, 3, 83, 85, 82, 2, 11, 65, 2, 137, 94, 5, 32, 83, - 65, 84, 65, 2, 11, 65, 2, 247, 239, 5, 75, 4, 168, 3, 3, 65, 78, 71, 141, - 238, 5, 3, 85, 82, 78, 10, 28, 2, 80, 65, 211, 82, 86, 8, 28, 3, 77, 65, - 65, 23, 78, 2, 143, 178, 6, 69, 6, 18, 71, 71, 89, 4, 38, 76, 177, 236, - 5, 3, 87, 73, 83, 2, 165, 144, 5, 2, 65, 89, 2, 141, 175, 6, 2, 69, 67, - 12, 18, 77, 23, 78, 2, 179, 244, 2, 69, 10, 96, 3, 69, 85, 76, 34, 79, - 22, 89, 140, 220, 1, 3, 71, 72, 85, 157, 152, 2, 4, 65, 69, 76, 65, 2, - 11, 69, 2, 147, 224, 5, 85, 2, 139, 244, 3, 76, 2, 203, 142, 1, 85, 5, - 173, 130, 3, 13, 32, 79, 86, 69, 82, 32, 77, 79, 85, 78, 84, 65, 73, 72, - 54, 83, 166, 195, 4, 72, 241, 64, 4, 86, 73, 76, 76, 68, 56, 6, 67, 82, - 73, 80, 84, 32, 173, 2, 3, 69, 84, 32, 34, 118, 69, 34, 76, 130, 1, 80, - 150, 208, 2, 77, 226, 1, 82, 222, 237, 1, 70, 30, 83, 42, 84, 62, 90, - 130, 83, 78, 15, 79, 4, 226, 66, 81, 235, 192, 5, 73, 6, 96, 18, 65, 84, - 73, 78, 32, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 135, 210, - 2, 69, 4, 138, 173, 6, 73, 3, 78, 2, 155, 147, 3, 76, 34, 148, 1, 6, 65, - 66, 79, 86, 69, 32, 96, 7, 66, 69, 83, 73, 68, 69, 32, 162, 1, 79, 158, - 3, 87, 153, 148, 4, 9, 80, 82, 69, 67, 69, 68, 73, 78, 71, 6, 26, 83, - 231, 187, 2, 76, 4, 11, 85, 4, 26, 80, 191, 194, 4, 66, 2, 185, 194, 4, - 2, 69, 82, 4, 108, 23, 65, 78, 68, 32, 74, 79, 73, 78, 69, 68, 32, 66, - 89, 32, 68, 65, 83, 72, 32, 87, 73, 84, 72, 23, 83, 2, 17, 2, 32, 83, 2, - 153, 193, 4, 2, 85, 66, 16, 11, 70, 17, 11, 32, 14, 120, 6, 65, 66, 79, - 86, 69, 32, 132, 1, 4, 87, 73, 84, 72, 181, 166, 5, 11, 79, 82, 32, 69, - 81, 85, 65, 76, 32, 84, 79, 8, 34, 65, 38, 69, 18, 84, 67, 78, 2, 11, 76, - 2, 113, 3, 77, 79, 83, 2, 187, 60, 81, 2, 21, 3, 73, 76, 68, 2, 147, 253, - 2, 69, 2, 17, 2, 32, 78, 2, 11, 79, 2, 11, 84, 2, 11, 32, 2, 11, 69, 2, - 17, 2, 81, 85, 2, 11, 65, 2, 11, 76, 2, 179, 218, 2, 32, 6, 25, 4, 73, - 84, 72, 32, 6, 96, 2, 80, 76, 24, 14, 77, 85, 76, 84, 73, 80, 76, 73, 67, - 65, 84, 73, 79, 78, 231, 212, 5, 68, 2, 11, 85, 2, 11, 83, 2, 169, 254, - 3, 5, 32, 83, 73, 71, 78, 4, 196, 160, 4, 2, 65, 67, 211, 203, 1, 69, 4, - 60, 9, 80, 69, 78, 83, 73, 79, 78, 32, 82, 167, 144, 6, 72, 2, 21, 3, 65, - 73, 76, 2, 247, 151, 5, 87, 8, 26, 65, 98, 73, 35, 85, 4, 44, 5, 83, 72, - 32, 65, 77, 135, 163, 6, 78, 2, 173, 223, 5, 7, 80, 69, 82, 83, 65, 78, - 68, 2, 11, 77, 2, 231, 228, 5, 77, 2, 129, 250, 2, 2, 78, 71, 230, 2, 92, - 11, 76, 79, 84, 73, 32, 78, 65, 71, 82, 73, 32, 226, 5, 77, 182, 15, 78, - 97, 2, 82, 73, 90, 180, 1, 7, 76, 69, 84, 84, 69, 82, 32, 168, 2, 11, 80, - 79, 69, 84, 82, 89, 32, 77, 65, 82, 75, 56, 5, 83, 73, 71, 78, 32, 145, - 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 64, 174, 1, 68, 46, - 82, 34, 84, 154, 89, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 246, 165, 5, - 72, 2, 76, 2, 77, 2, 78, 2, 83, 246, 30, 65, 2, 69, 2, 73, 2, 79, 3, 85, - 8, 226, 89, 68, 246, 165, 5, 72, 247, 30, 79, 4, 170, 255, 5, 82, 247, - 30, 79, 8, 150, 89, 84, 246, 165, 5, 72, 247, 30, 79, 8, 11, 45, 8, 198, - 157, 6, 49, 2, 50, 2, 51, 3, 52, 8, 46, 65, 70, 72, 213, 215, 5, 3, 68, - 86, 73, 4, 64, 10, 76, 84, 69, 82, 78, 65, 84, 69, 32, 72, 203, 215, 5, - 78, 2, 129, 137, 2, 2, 65, 83, 10, 150, 253, 5, 79, 246, 30, 65, 2, 69, - 2, 73, 3, 85, 82, 60, 8, 66, 79, 76, 32, 70, 79, 82, 32, 213, 14, 2, 77, - 69, 80, 238, 2, 66, 48, 2, 67, 65, 118, 68, 222, 1, 69, 130, 2, 70, 56, - 8, 72, 79, 82, 73, 90, 79, 78, 84, 0, 6, 86, 69, 82, 84, 73, 67, 44, 3, - 76, 73, 78, 52, 9, 77, 65, 82, 75, 83, 32, 67, 72, 65, 22, 78, 90, 65, - 68, 3, 82, 69, 67, 32, 5, 71, 82, 79, 85, 80, 0, 4, 85, 78, 73, 84, 22, - 83, 205, 3, 15, 84, 89, 80, 69, 32, 65, 32, 69, 76, 69, 67, 84, 82, 79, - 78, 4, 242, 147, 4, 69, 133, 198, 1, 3, 65, 67, 75, 4, 44, 3, 82, 82, 73, - 237, 182, 1, 2, 78, 67, 2, 33, 6, 65, 71, 69, 32, 82, 69, 2, 11, 84, 2, - 135, 185, 1, 85, 14, 24, 2, 65, 84, 55, 69, 2, 161, 3, 9, 65, 32, 76, 73, - 78, 75, 32, 69, 83, 12, 76, 12, 86, 73, 67, 69, 32, 67, 79, 78, 84, 82, - 79, 76, 157, 8, 2, 76, 69, 8, 11, 32, 8, 222, 174, 1, 70, 154, 250, 2, - 84, 203, 83, 79, 12, 22, 78, 207, 1, 83, 10, 48, 5, 68, 32, 79, 70, 32, - 137, 1, 2, 81, 85, 8, 18, 77, 31, 84, 2, 193, 236, 4, 2, 69, 68, 6, 64, - 11, 82, 65, 78, 83, 77, 73, 83, 83, 73, 79, 78, 159, 108, 69, 5, 143, - 171, 3, 32, 2, 155, 7, 73, 2, 205, 206, 4, 2, 67, 65, 4, 36, 2, 73, 76, - 73, 3, 79, 82, 77, 2, 191, 2, 69, 2, 129, 162, 1, 6, 65, 76, 32, 84, 65, - 66, 2, 11, 69, 2, 17, 2, 32, 70, 2, 223, 178, 1, 69, 2, 191, 219, 4, 80, - 6, 26, 69, 175, 141, 4, 85, 4, 56, 8, 71, 65, 84, 73, 86, 69, 32, 65, - 247, 247, 4, 87, 2, 21, 3, 67, 75, 78, 2, 21, 3, 79, 87, 76, 2, 191, 136, - 1, 69, 2, 11, 79, 2, 17, 2, 82, 68, 2, 191, 247, 2, 32, 18, 176, 1, 7, - 65, 77, 65, 82, 73, 84, 65, 60, 3, 72, 73, 70, 52, 8, 84, 65, 82, 84, 32, - 79, 70, 32, 64, 7, 85, 66, 83, 84, 73, 84, 85, 204, 1, 3, 89, 78, 67, - 255, 206, 5, 80, 2, 25, 4, 78, 32, 83, 79, 2, 11, 85, 2, 219, 209, 5, 82, - 4, 17, 2, 84, 32, 4, 226, 148, 5, 79, 239, 41, 73, 4, 22, 72, 155, 102, - 84, 2, 17, 2, 69, 65, 2, 223, 188, 5, 68, 4, 11, 84, 4, 11, 69, 5, 11, - 32, 2, 25, 4, 70, 79, 82, 77, 2, 17, 2, 32, 84, 2, 251, 237, 5, 87, 2, - 137, 207, 5, 2, 73, 67, 2, 11, 84, 2, 235, 250, 5, 82, 7, 38, 67, 145, - 171, 2, 3, 65, 71, 79, 2, 141, 134, 1, 9, 72, 82, 79, 78, 79, 85, 83, 32, - 73, 180, 1, 36, 3, 65, 67, 32, 163, 215, 4, 78, 178, 1, 150, 3, 65, 52, - 4, 66, 65, 82, 82, 32, 2, 67, 79, 80, 13, 68, 79, 84, 84, 69, 68, 32, 90, - 76, 65, 77, 65, 32, 118, 69, 90, 72, 204, 2, 7, 76, 69, 84, 84, 69, 82, - 32, 236, 9, 9, 79, 66, 76, 73, 81, 85, 69, 32, 76, 28, 4, 80, 84, 72, 65, - 0, 4, 90, 81, 65, 80, 110, 82, 88, 2, 83, 85, 174, 2, 84, 216, 205, 4, 3, - 77, 85, 83, 228, 21, 7, 70, 69, 77, 73, 78, 73, 78, 229, 136, 1, 5, 81, - 85, 83, 72, 83, 2, 11, 66, 2, 197, 158, 5, 5, 66, 82, 69, 86, 73, 2, 11, - 69, 2, 167, 134, 6, 75, 6, 38, 78, 165, 17, 4, 76, 79, 78, 32, 2, 17, 2, - 84, 82, 2, 151, 182, 5, 65, 4, 28, 3, 65, 78, 71, 35, 72, 2, 11, 85, 2, - 255, 227, 4, 76, 2, 11, 79, 2, 185, 184, 3, 5, 82, 73, 90, 79, 78, 6, 60, - 10, 78, 68, 32, 79, 70, 32, 80, 65, 82, 65, 151, 14, 83, 2, 201, 11, 2, - 71, 82, 14, 104, 8, 65, 82, 75, 76, 69, 65, 78, 32, 132, 1, 4, 66, 65, - 83, 65, 65, 7, 79, 82, 73, 90, 79, 78, 84, 6, 60, 5, 65, 83, 84, 69, 82, - 40, 4, 77, 69, 84, 79, 3, 79, 2, 17, 2, 73, 83, 2, 219, 128, 4, 67, 2, - 181, 166, 2, 2, 66, 69, 6, 164, 11, 8, 45, 69, 83, 65, 83, 65, 32, 68, - 159, 223, 3, 32, 2, 197, 178, 5, 2, 65, 76, 92, 154, 2, 68, 102, 72, 32, - 3, 76, 65, 77, 34, 77, 204, 1, 2, 80, 69, 142, 1, 82, 78, 83, 144, 1, 8, - 70, 73, 78, 65, 76, 32, 83, 69, 106, 65, 14, 75, 2, 81, 34, 84, 40, 5, - 71, 65, 77, 65, 76, 40, 4, 89, 85, 68, 72, 146, 75, 66, 166, 130, 4, 90, - 154, 43, 78, 254, 1, 87, 159, 126, 69, 4, 76, 14, 79, 84, 76, 69, 83, 83, - 32, 68, 65, 76, 65, 84, 72, 32, 139, 3, 65, 2, 139, 193, 2, 82, 4, 11, - 69, 5, 159, 254, 5, 84, 2, 11, 65, 2, 255, 253, 5, 68, 24, 60, 9, 65, 76, - 65, 89, 65, 76, 65, 77, 32, 231, 237, 5, 73, 22, 78, 76, 22, 78, 158, - 134, 4, 66, 242, 210, 1, 84, 138, 34, 83, 14, 74, 3, 82, 4, 199, 247, 3, - 76, 8, 182, 84, 78, 234, 166, 5, 71, 3, 89, 9, 33, 6, 82, 83, 73, 65, 78, - 32, 6, 50, 66, 16, 3, 68, 72, 65, 17, 3, 71, 72, 65, 2, 131, 79, 72, 2, - 147, 2, 76, 2, 143, 243, 4, 77, 4, 52, 7, 69, 86, 69, 82, 83, 69, 68, - 211, 150, 4, 73, 2, 251, 182, 4, 32, 14, 142, 1, 69, 40, 7, 79, 71, 68, - 73, 65, 78, 32, 64, 12, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, - 166, 250, 4, 72, 177, 82, 2, 65, 68, 2, 17, 2, 77, 75, 2, 175, 251, 3, - 65, 6, 42, 90, 32, 2, 75, 72, 131, 227, 5, 70, 2, 195, 206, 4, 72, 2, 11, - 76, 2, 11, 65, 2, 183, 248, 5, 80, 6, 36, 3, 69, 84, 72, 255, 250, 4, 65, - 5, 237, 85, 6, 32, 71, 65, 82, 83, 72, 5, 215, 203, 5, 32, 4, 225, 223, - 3, 2, 73, 78, 6, 21, 3, 72, 65, 32, 6, 42, 68, 186, 223, 3, 66, 223, 155, - 1, 65, 2, 17, 2, 79, 84, 2, 155, 152, 1, 84, 8, 58, 66, 210, 144, 2, 87, - 217, 189, 1, 4, 85, 75, 75, 65, 4, 173, 204, 3, 2, 65, 83, 14, 88, 8, 66, - 76, 73, 78, 69, 65, 82, 32, 113, 10, 80, 82, 65, 76, 73, 78, 69, 65, 82, - 32, 8, 44, 5, 67, 79, 76, 79, 78, 227, 170, 3, 70, 7, 11, 32, 4, 29, 5, - 83, 75, 69, 87, 69, 4, 171, 202, 5, 68, 6, 44, 5, 67, 79, 76, 79, 78, - 243, 169, 3, 70, 5, 185, 194, 5, 7, 32, 83, 75, 69, 87, 69, 68, 8, 62, - 72, 25, 11, 87, 79, 32, 86, 69, 82, 84, 73, 67, 65, 76, 4, 21, 3, 82, 69, - 69, 4, 25, 4, 32, 68, 79, 84, 4, 239, 218, 3, 83, 148, 41, 102, 45, 58, - 65, 166, 106, 69, 254, 45, 72, 174, 46, 73, 254, 72, 79, 182, 20, 82, - 226, 16, 85, 191, 9, 87, 4, 32, 2, 83, 72, 203, 205, 4, 82, 2, 179, 235, - 4, 73, 244, 26, 182, 1, 66, 82, 71, 192, 17, 2, 73, 32, 182, 29, 75, 152, - 5, 9, 76, 76, 89, 32, 77, 65, 82, 75, 32, 34, 77, 226, 37, 78, 156, 13, - 3, 80, 69, 32, 94, 85, 150, 233, 4, 67, 159, 11, 88, 5, 249, 123, 16, 76, - 69, 32, 84, 69, 78, 78, 73, 83, 32, 80, 65, 68, 68, 76, 69, 144, 2, 78, - 32, 224, 11, 5, 65, 76, 79, 71, 32, 253, 2, 6, 66, 65, 78, 87, 65, 32, - 190, 1, 158, 1, 65, 102, 67, 186, 1, 68, 58, 69, 98, 71, 118, 72, 46, 76, - 222, 3, 80, 54, 81, 62, 82, 106, 83, 194, 77, 78, 166, 213, 1, 84, 166, - 118, 70, 159, 177, 1, 86, 6, 42, 80, 170, 180, 2, 77, 147, 169, 1, 83, 2, - 17, 2, 79, 83, 2, 197, 192, 5, 4, 84, 82, 79, 80, 8, 18, 73, 63, 79, 2, - 25, 4, 82, 67, 85, 77, 2, 213, 2, 4, 70, 76, 69, 88, 6, 26, 77, 231, 156, - 5, 76, 4, 11, 77, 4, 228, 216, 4, 7, 69, 82, 67, 73, 65, 76, 32, 211, - 147, 1, 65, 22, 26, 79, 207, 253, 3, 73, 2, 11, 76, 2, 175, 85, 76, 4, - 18, 81, 43, 88, 2, 17, 2, 85, 65, 2, 207, 209, 2, 76, 2, 221, 129, 5, 4, - 67, 76, 65, 77, 4, 11, 82, 4, 18, 65, 55, 69, 2, 11, 86, 2, 11, 69, 2, - 209, 121, 3, 32, 65, 67, 2, 137, 4, 4, 65, 84, 69, 82, 2, 129, 141, 2, 6, - 89, 80, 72, 69, 78, 45, 114, 38, 65, 246, 2, 69, 183, 129, 4, 79, 104, - 25, 4, 84, 73, 78, 32, 104, 34, 67, 33, 4, 83, 77, 65, 76, 52, 29, 5, 65, - 80, 73, 84, 65, 52, 41, 8, 76, 32, 76, 69, 84, 84, 69, 82, 52, 11, 32, - 52, 254, 231, 5, 65, 2, 66, 2, 67, 2, 68, 2, 69, 2, 70, 2, 71, 2, 72, 2, - 73, 2, 74, 2, 75, 2, 76, 2, 77, 2, 78, 2, 79, 2, 80, 2, 81, 2, 82, 2, 83, - 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, 2, 89, 3, 90, 8, 22, 83, 195, 1, 70, - 2, 11, 83, 2, 145, 64, 3, 45, 84, 72, 4, 26, 69, 247, 203, 2, 76, 2, 185, - 68, 2, 82, 67, 4, 11, 85, 4, 26, 79, 235, 154, 3, 69, 2, 227, 251, 4, 84, - 8, 36, 3, 73, 71, 72, 159, 210, 3, 69, 6, 17, 2, 84, 32, 6, 238, 135, 2, - 67, 134, 2, 80, 155, 2, 83, 6, 174, 174, 2, 69, 202, 165, 1, 79, 155, - 211, 1, 80, 46, 80, 7, 76, 69, 84, 84, 69, 82, 32, 208, 1, 5, 83, 73, 71, - 78, 32, 159, 2, 86, 38, 162, 1, 65, 218, 146, 2, 78, 210, 204, 3, 66, 2, - 68, 2, 71, 2, 72, 2, 75, 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, - 2, 89, 186, 2, 73, 3, 85, 5, 145, 151, 5, 6, 82, 67, 72, 65, 73, 67, 4, - 18, 80, 51, 86, 2, 25, 4, 65, 77, 85, 68, 2, 159, 191, 2, 80, 2, 247, - 174, 3, 73, 36, 48, 7, 76, 69, 84, 84, 69, 82, 32, 147, 1, 86, 32, 158, - 145, 2, 78, 210, 204, 3, 66, 2, 68, 2, 71, 2, 75, 2, 76, 2, 77, 2, 80, 2, - 83, 2, 84, 2, 87, 2, 89, 186, 2, 65, 2, 73, 3, 85, 4, 41, 8, 79, 87, 69, - 76, 32, 83, 73, 71, 4, 11, 78, 4, 179, 158, 5, 32, 212, 3, 112, 10, 76, - 69, 32, 76, 69, 84, 84, 69, 82, 32, 248, 2, 5, 84, 72, 65, 77, 32, 193, - 18, 5, 86, 73, 69, 84, 32, 70, 186, 1, 65, 34, 69, 30, 84, 214, 140, 2, - 78, 138, 144, 1, 79, 206, 247, 1, 75, 2, 80, 162, 7, 85, 222, 61, 70, 2, - 72, 2, 76, 2, 77, 2, 81, 2, 83, 2, 86, 2, 88, 2, 89, 187, 2, 73, 7, 194, - 197, 5, 85, 215, 22, 73, 7, 246, 219, 5, 69, 3, 72, 18, 60, 3, 79, 78, - 69, 234, 147, 5, 83, 254, 68, 72, 187, 2, 65, 10, 11, 45, 10, 142, 219, - 5, 50, 2, 51, 2, 52, 2, 53, 3, 54, 254, 1, 196, 1, 2, 67, 79, 240, 3, 4, - 72, 79, 82, 65, 0, 4, 84, 72, 65, 77, 28, 7, 76, 69, 84, 84, 69, 82, 32, - 220, 4, 5, 83, 73, 71, 78, 32, 217, 5, 11, 86, 79, 87, 69, 76, 32, 83, - 73, 71, 78, 32, 20, 164, 1, 13, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, - 71, 78, 32, 237, 198, 4, 21, 77, 66, 73, 78, 73, 78, 71, 32, 67, 82, 89, - 80, 84, 79, 71, 82, 65, 77, 77, 73, 67, 18, 148, 1, 6, 70, 73, 78, 65, - 76, 32, 22, 76, 56, 16, 72, 73, 71, 72, 32, 82, 65, 84, 72, 65, 32, 79, - 82, 32, 76, 79, 22, 77, 186, 211, 5, 66, 3, 83, 2, 151, 148, 5, 78, 4, - 54, 79, 149, 151, 1, 7, 65, 32, 84, 65, 78, 71, 32, 2, 167, 216, 1, 87, - 6, 48, 6, 69, 68, 73, 65, 76, 32, 191, 213, 5, 65, 4, 130, 211, 5, 76, 3, - 82, 20, 129, 231, 3, 2, 32, 68, 106, 188, 1, 2, 71, 82, 44, 5, 72, 73, - 71, 72, 32, 94, 76, 222, 1, 82, 198, 129, 2, 85, 210, 200, 1, 73, 166, - 15, 78, 182, 216, 1, 79, 162, 8, 69, 158, 20, 66, 2, 68, 2, 77, 2, 87, - 187, 2, 65, 2, 21, 3, 69, 65, 84, 2, 231, 208, 5, 32, 32, 242, 1, 75, 42, - 82, 250, 136, 5, 83, 82, 67, 2, 80, 2, 84, 254, 68, 70, 2, 72, 3, 89, 36, - 60, 3, 79, 87, 32, 234, 145, 5, 65, 194, 41, 85, 159, 20, 76, 28, 86, 75, - 42, 82, 202, 137, 5, 67, 2, 80, 2, 84, 254, 68, 70, 2, 72, 2, 83, 3, 89, - 6, 234, 206, 5, 72, 2, 88, 187, 2, 65, 2, 189, 217, 3, 2, 65, 84, 8, 26, - 65, 243, 185, 5, 85, 7, 138, 206, 5, 78, 3, 84, 50, 182, 1, 72, 34, 75, - 176, 1, 4, 77, 65, 73, 32, 82, 82, 136, 1, 2, 83, 65, 92, 5, 87, 73, 65, - 78, 71, 152, 27, 4, 84, 79, 78, 69, 224, 100, 3, 68, 79, 75, 177, 242, 3, - 2, 67, 65, 4, 130, 254, 4, 65, 167, 63, 79, 14, 52, 4, 72, 85, 69, 78, - 134, 3, 65, 139, 177, 4, 69, 8, 80, 6, 32, 84, 79, 78, 69, 45, 145, 134, - 4, 8, 45, 76, 85, 69, 32, 75, 65, 82, 6, 194, 205, 5, 51, 2, 52, 3, 53, - 8, 56, 4, 75, 65, 78, 71, 194, 136, 1, 89, 235, 138, 3, 83, 5, 243, 141, - 1, 32, 4, 92, 3, 65, 32, 72, 21, 16, 69, 86, 69, 82, 83, 69, 68, 32, 82, - 79, 84, 65, 84, 69, 68, 32, 2, 179, 146, 4, 65, 2, 171, 162, 3, 82, 8, - 48, 3, 84, 75, 65, 226, 131, 4, 87, 139, 119, 75, 4, 17, 2, 65, 78, 5, - 251, 219, 3, 75, 5, 133, 156, 3, 2, 87, 65, 38, 90, 65, 36, 4, 77, 65, - 73, 32, 22, 79, 46, 84, 86, 85, 242, 193, 3, 73, 207, 134, 2, 69, 9, 242, - 201, 5, 65, 2, 69, 3, 73, 2, 255, 181, 4, 83, 11, 218, 176, 3, 65, 226, - 152, 2, 79, 3, 89, 4, 42, 65, 129, 138, 1, 4, 72, 65, 77, 32, 2, 17, 2, - 76, 76, 2, 135, 217, 3, 32, 9, 166, 136, 5, 85, 151, 64, 69, 144, 1, 184, - 1, 7, 76, 69, 84, 84, 69, 82, 32, 188, 2, 5, 77, 65, 73, 32, 75, 32, 7, - 83, 89, 77, 66, 79, 76, 32, 124, 9, 84, 79, 78, 69, 32, 77, 65, 73, 32, - 81, 6, 86, 79, 87, 69, 76, 32, 96, 44, 4, 72, 73, 71, 72, 1, 3, 76, 79, - 87, 48, 11, 32, 48, 154, 1, 75, 30, 67, 2, 80, 2, 84, 34, 78, 214, 165, - 5, 66, 2, 68, 2, 70, 2, 71, 2, 72, 2, 76, 2, 77, 2, 82, 2, 83, 2, 86, 2, - 89, 247, 30, 79, 6, 26, 72, 235, 196, 5, 79, 4, 242, 165, 5, 72, 247, 30, - 79, 6, 210, 165, 5, 71, 2, 89, 247, 30, 79, 4, 178, 243, 4, 65, 183, 52, - 72, 10, 72, 2, 75, 79, 110, 78, 164, 128, 1, 4, 72, 79, 32, 72, 235, 136, - 3, 83, 4, 140, 129, 1, 3, 73, 32, 75, 167, 194, 4, 78, 8, 58, 78, 178, - 134, 1, 84, 186, 255, 1, 83, 203, 185, 2, 69, 2, 151, 134, 3, 85, 26, 50, - 65, 58, 85, 30, 73, 174, 193, 5, 69, 3, 79, 10, 170, 171, 5, 85, 214, 22, - 65, 2, 77, 2, 78, 3, 89, 9, 26, 69, 175, 193, 5, 65, 5, 171, 193, 5, 65, - 138, 1, 52, 3, 82, 73, 32, 173, 180, 4, 4, 69, 79, 85, 84, 136, 1, 82, - 65, 20, 7, 76, 69, 84, 84, 69, 82, 32, 170, 2, 83, 78, 86, 227, 206, 3, - 68, 2, 163, 213, 1, 66, 88, 214, 1, 65, 254, 157, 1, 82, 242, 55, 68, 46, - 84, 230, 24, 85, 210, 200, 1, 73, 158, 190, 1, 78, 126, 66, 2, 67, 2, 71, - 2, 74, 2, 75, 2, 80, 2, 83, 254, 68, 72, 2, 76, 2, 77, 2, 86, 2, 89, 186, - 2, 69, 3, 79, 11, 176, 149, 3, 7, 82, 67, 72, 65, 73, 67, 32, 230, 168, - 2, 65, 2, 73, 3, 85, 8, 25, 4, 73, 71, 78, 32, 8, 226, 215, 1, 78, 222, - 160, 3, 65, 239, 1, 86, 18, 49, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, - 32, 18, 238, 215, 1, 65, 190, 21, 85, 210, 200, 1, 73, 206, 134, 2, 69, - 3, 79, 4, 226, 51, 70, 139, 239, 3, 79, 188, 6, 36, 3, 73, 76, 32, 147, - 154, 4, 65, 186, 6, 170, 3, 65, 102, 67, 194, 2, 68, 72, 3, 87, 69, 84, - 48, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 136, 8, 9, 73, 78, 32, 80, 79, - 83, 83, 69, 83, 22, 76, 204, 2, 7, 78, 85, 77, 66, 69, 82, 32, 152, 1, - 18, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 69, 78, 68, 32, 79, - 70, 54, 82, 42, 83, 188, 14, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, - 32, 78, 84, 228, 1, 2, 89, 69, 184, 143, 4, 5, 77, 79, 78, 84, 72, 147, - 118, 79, 6, 76, 5, 83, 32, 65, 66, 79, 206, 55, 85, 149, 154, 3, 5, 78, - 68, 32, 79, 68, 2, 219, 159, 3, 86, 52, 80, 9, 79, 78, 83, 79, 78, 65, - 78, 84, 32, 192, 21, 3, 85, 82, 82, 143, 10, 82, 48, 130, 1, 75, 22, 76, - 22, 78, 46, 84, 170, 193, 1, 83, 170, 16, 82, 166, 227, 3, 67, 2, 72, 2, - 74, 2, 77, 2, 80, 2, 86, 3, 89, 5, 247, 247, 4, 83, 7, 151, 185, 4, 76, - 11, 158, 189, 3, 78, 130, 248, 1, 71, 3, 89, 5, 243, 180, 5, 84, 26, 68, - 2, 82, 89, 172, 29, 2, 69, 66, 234, 248, 1, 65, 247, 175, 1, 73, 2, 213, - 201, 1, 7, 32, 67, 85, 76, 84, 73, 86, 42, 168, 1, 4, 79, 78, 69, 32, - 252, 4, 6, 84, 72, 82, 69, 69, 32, 165, 172, 5, 22, 68, 79, 87, 78, 83, - 67, 65, 76, 73, 78, 71, 32, 70, 65, 67, 84, 79, 82, 32, 75, 73, 73, 30, - 112, 5, 69, 73, 71, 72, 84, 34, 70, 40, 3, 72, 65, 76, 18, 79, 88, 4, 83, - 73, 88, 84, 110, 84, 187, 175, 3, 81, 4, 242, 3, 73, 215, 173, 5, 72, 4, - 188, 3, 2, 79, 82, 191, 173, 3, 73, 4, 171, 1, 70, 2, 133, 3, 18, 78, 69, - 45, 72, 85, 78, 68, 82, 69, 68, 45, 65, 78, 68, 45, 83, 73, 88, 6, 64, 5, - 69, 69, 78, 84, 72, 165, 233, 1, 5, 89, 45, 70, 79, 85, 4, 11, 45, 4, - 198, 175, 5, 49, 3, 50, 8, 38, 72, 138, 1, 87, 239, 174, 3, 69, 4, 132, - 1, 18, 82, 69, 69, 45, 72, 85, 78, 68, 82, 69, 68, 45, 65, 78, 68, 45, - 84, 87, 249, 242, 3, 8, 73, 82, 84, 89, 45, 83, 69, 67, 2, 17, 2, 69, 78, - 2, 17, 2, 84, 73, 2, 207, 174, 3, 69, 10, 58, 69, 28, 4, 83, 73, 88, 84, - 82, 84, 163, 174, 3, 81, 2, 129, 1, 3, 73, 71, 72, 4, 50, 69, 149, 175, - 3, 6, 89, 45, 70, 79, 85, 82, 2, 145, 175, 3, 2, 69, 78, 2, 21, 3, 87, - 69, 78, 2, 221, 174, 3, 3, 84, 73, 69, 2, 247, 209, 1, 83, 72, 48, 6, 69, - 84, 84, 69, 82, 32, 187, 197, 3, 65, 70, 198, 1, 78, 138, 24, 84, 222, - 22, 76, 210, 90, 82, 206, 55, 65, 182, 25, 85, 158, 144, 1, 79, 182, 56, - 73, 202, 190, 1, 83, 242, 7, 69, 222, 61, 67, 2, 72, 2, 74, 2, 75, 2, 77, - 2, 80, 2, 86, 3, 89, 10, 46, 78, 234, 166, 5, 71, 2, 89, 187, 2, 65, 4, - 230, 166, 5, 78, 187, 2, 65, 8, 38, 79, 230, 215, 3, 84, 167, 74, 83, 4, - 11, 78, 4, 17, 2, 69, 32, 4, 18, 72, 31, 84, 2, 213, 72, 3, 85, 78, 68, - 2, 229, 238, 1, 3, 72, 79, 85, 2, 17, 2, 32, 84, 2, 11, 69, 2, 155, 139, - 5, 88, 2, 17, 2, 85, 80, 2, 199, 143, 3, 69, 194, 4, 152, 1, 5, 65, 76, - 84, 32, 80, 20, 4, 73, 71, 78, 32, 206, 4, 80, 28, 9, 84, 65, 82, 84, 73, - 78, 71, 32, 70, 41, 8, 89, 76, 76, 65, 66, 76, 69, 32, 2, 207, 175, 3, - 65, 40, 90, 65, 48, 3, 67, 69, 86, 34, 75, 80, 2, 77, 85, 102, 85, 14, - 80, 118, 86, 163, 65, 78, 4, 216, 2, 4, 65, 90, 72, 65, 195, 221, 4, 78, - 2, 11, 73, 2, 215, 255, 4, 84, 6, 38, 85, 157, 255, 4, 3, 65, 65, 67, 4, - 18, 90, 87, 82, 2, 151, 144, 5, 72, 6, 60, 4, 75, 75, 85, 82, 16, 2, 84, - 72, 21, 3, 85, 86, 85, 2, 163, 83, 85, 2, 203, 241, 2, 65, 2, 75, 90, 8, - 26, 65, 255, 210, 4, 79, 6, 34, 84, 230, 232, 3, 65, 15, 78, 2, 11, 72, - 2, 17, 2, 65, 75, 2, 163, 253, 4, 75, 10, 38, 65, 210, 82, 69, 143, 140, - 4, 73, 4, 176, 103, 3, 82, 65, 65, 213, 202, 2, 6, 75, 65, 73, 89, 65, - 82, 2, 11, 69, 2, 159, 10, 78, 2, 17, 2, 82, 79, 2, 243, 153, 4, 77, 148, - 4, 122, 75, 170, 1, 76, 174, 1, 78, 194, 1, 82, 90, 83, 186, 1, 84, 86, - 67, 2, 72, 2, 74, 2, 77, 2, 80, 2, 86, 3, 89, 46, 88, 2, 83, 83, 150, - 186, 1, 65, 190, 21, 85, 158, 144, 1, 79, 182, 56, 73, 187, 198, 1, 69, - 24, 154, 182, 1, 65, 182, 25, 85, 158, 144, 1, 79, 182, 56, 73, 187, 198, - 1, 69, 66, 82, 76, 246, 184, 1, 65, 190, 21, 85, 158, 144, 1, 79, 182, - 56, 73, 187, 198, 1, 69, 44, 250, 4, 76, 250, 179, 1, 65, 190, 21, 85, - 158, 144, 1, 79, 182, 56, 73, 187, 198, 1, 69, 110, 102, 78, 190, 3, 71, - 2, 89, 250, 179, 1, 65, 190, 21, 85, 158, 144, 1, 79, 182, 56, 73, 187, - 198, 1, 69, 44, 186, 3, 78, 250, 179, 1, 65, 190, 21, 85, 158, 144, 1, - 79, 182, 56, 73, 187, 198, 1, 69, 44, 226, 2, 82, 250, 179, 1, 65, 190, - 21, 85, 158, 144, 1, 79, 182, 56, 73, 187, 198, 1, 69, 68, 94, 72, 174, - 1, 83, 250, 179, 1, 65, 190, 21, 85, 158, 144, 1, 79, 182, 56, 73, 187, - 198, 1, 69, 24, 162, 181, 1, 65, 190, 21, 85, 158, 144, 1, 79, 230, 2, - 82, 210, 53, 73, 187, 198, 1, 69, 44, 82, 84, 250, 179, 1, 65, 190, 21, - 85, 158, 144, 1, 79, 182, 56, 73, 187, 198, 1, 69, 22, 246, 179, 1, 65, - 190, 21, 85, 158, 144, 1, 79, 182, 56, 73, 187, 198, 1, 69, 6, 68, 2, 79, - 84, 33, 11, 82, 65, 68, 73, 84, 73, 79, 78, 65, 76, 32, 2, 11, 65, 2, - 199, 144, 4, 76, 4, 24, 2, 67, 82, 55, 78, 2, 17, 2, 69, 68, 2, 11, 73, - 2, 247, 143, 4, 84, 2, 11, 85, 2, 17, 2, 77, 66, 2, 23, 69, 2, 11, 65, 2, - 171, 143, 4, 82, 184, 13, 36, 5, 65, 66, 65, 84, 65, 35, 71, 2, 11, 32, - 2, 135, 169, 3, 84, 182, 13, 54, 69, 20, 3, 83, 65, 32, 241, 6, 3, 85, - 84, 32, 2, 215, 251, 3, 82, 178, 1, 52, 7, 76, 69, 84, 84, 69, 82, 32, - 143, 166, 3, 68, 158, 1, 210, 1, 65, 58, 70, 54, 72, 62, 76, 58, 77, 54, - 78, 50, 83, 126, 85, 114, 69, 2, 73, 2, 79, 2, 86, 158, 198, 4, 84, 82, - 67, 2, 68, 2, 71, 2, 75, 2, 80, 254, 68, 66, 2, 82, 2, 87, 2, 88, 2, 89, - 3, 90, 16, 174, 4, 87, 162, 142, 5, 67, 2, 81, 2, 88, 3, 90, 4, 168, 193, - 4, 5, 73, 78, 65, 76, 32, 239, 80, 65, 6, 26, 84, 203, 145, 5, 65, 4, - 142, 143, 5, 84, 187, 2, 65, 4, 192, 236, 3, 5, 79, 78, 71, 32, 85, 231, - 164, 1, 65, 10, 238, 144, 5, 65, 2, 67, 2, 81, 2, 88, 3, 90, 8, 130, 142, - 5, 71, 2, 72, 2, 89, 187, 2, 65, 8, 26, 72, 243, 143, 5, 65, 6, 40, 4, - 79, 82, 84, 32, 199, 143, 5, 65, 4, 208, 152, 1, 2, 85, 69, 229, 234, 2, - 2, 65, 87, 32, 58, 73, 54, 69, 162, 142, 5, 67, 2, 81, 2, 88, 3, 90, 16, - 50, 85, 162, 142, 5, 67, 2, 81, 2, 88, 3, 90, 8, 158, 142, 5, 67, 2, 81, + 2, 78, 32, 2, 11, 66, 2, 233, 212, 6, 4, 76, 65, 67, 75, 5, 145, 199, 4, + 7, 32, 87, 73, 84, 72, 32, 79, 2, 179, 251, 3, 84, 70, 52, 7, 76, 69, 84, + 84, 69, 82, 32, 159, 131, 6, 68, 50, 198, 1, 69, 32, 2, 77, 65, 30, 78, + 186, 146, 1, 66, 2, 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, 80, + 2, 82, 2, 83, 2, 84, 2, 86, 2, 89, 242, 222, 6, 65, 2, 73, 2, 79, 3, 85, + 4, 226, 241, 7, 69, 147, 1, 72, 4, 210, 242, 7, 69, 3, 72, 6, 182, 146, + 1, 71, 2, 89, 243, 222, 6, 65, 58, 104, 3, 84, 72, 32, 169, 160, 6, 17, + 78, 68, 32, 82, 69, 67, 79, 82, 68, 73, 78, 71, 32, 67, 79, 80, 89, 56, + 60, 5, 69, 65, 83, 84, 32, 253, 2, 5, 87, 69, 83, 84, 32, 30, 96, 5, 65, + 82, 82, 79, 87, 194, 4, 80, 130, 1, 84, 138, 197, 4, 66, 38, 68, 18, 83, + 247, 58, 87, 13, 11, 32, 10, 68, 2, 65, 78, 38, 67, 120, 2, 84, 79, 154, + 2, 87, 203, 134, 5, 70, 2, 253, 2, 5, 68, 32, 83, 79, 85, 2, 37, 7, 82, + 79, 83, 83, 73, 78, 71, 2, 17, 2, 32, 78, 2, 17, 2, 79, 82, 2, 173, 133, + 5, 5, 84, 72, 32, 69, 65, 2, 17, 2, 32, 67, 2, 243, 241, 3, 79, 26, 96, + 5, 65, 82, 82, 79, 87, 198, 1, 80, 130, 1, 84, 138, 197, 4, 66, 38, 68, + 18, 83, 247, 58, 87, 9, 11, 32, 6, 52, 5, 65, 78, 68, 32, 78, 74, 87, + 203, 134, 5, 70, 2, 17, 2, 79, 82, 2, 21, 3, 84, 72, 32, 2, 137, 131, 5, + 2, 87, 69, 2, 21, 3, 73, 84, 72, 2, 11, 32, 2, 251, 151, 6, 72, 6, 41, 8, + 79, 73, 78, 84, 73, 78, 71, 32, 6, 42, 86, 242, 160, 4, 76, 155, 158, 2, + 66, 2, 17, 2, 73, 78, 2, 199, 160, 4, 69, 4, 73, 16, 82, 73, 65, 78, 71, + 76, 69, 45, 72, 69, 65, 68, 69, 68, 32, 65, 4, 25, 4, 82, 82, 79, 87, 5, + 11, 32, 2, 203, 137, 5, 84, 166, 1, 220, 3, 23, 67, 76, 85, 83, 84, 69, + 82, 45, 73, 78, 73, 84, 73, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 40, + 21, 70, 73, 78, 65, 76, 32, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, + 73, 71, 78, 32, 112, 27, 72, 69, 65, 68, 32, 77, 65, 82, 75, 32, 87, 73, + 84, 72, 32, 77, 79, 79, 78, 32, 65, 78, 68, 32, 83, 85, 78, 104, 7, 76, + 69, 84, 84, 69, 82, 32, 180, 1, 5, 77, 65, 82, 75, 32, 54, 83, 100, 8, + 84, 69, 82, 77, 73, 78, 65, 76, 40, 6, 86, 79, 87, 69, 76, 32, 207, 132, + 4, 71, 8, 142, 158, 7, 83, 138, 69, 76, 3, 82, 24, 138, 197, 3, 83, 134, + 165, 2, 78, 226, 248, 1, 45, 186, 2, 66, 2, 68, 2, 71, 2, 75, 2, 76, 2, + 77, 3, 82, 7, 29, 5, 32, 65, 78, 68, 32, 4, 50, 70, 1, 8, 84, 82, 73, 80, + 76, 69, 32, 70, 2, 235, 212, 5, 76, 82, 230, 188, 2, 68, 138, 222, 4, 75, + 38, 78, 46, 83, 38, 84, 46, 66, 2, 67, 2, 71, 2, 74, 2, 80, 2, 90, 138, + 69, 45, 2, 72, 2, 76, 2, 77, 2, 82, 2, 86, 2, 89, 187, 2, 65, 8, 222, + 140, 4, 80, 206, 142, 3, 68, 58, 83, 63, 84, 10, 40, 4, 73, 71, 78, 32, + 139, 159, 7, 85, 8, 246, 168, 5, 74, 158, 2, 86, 122, 85, 187, 240, 1, + 65, 4, 237, 223, 3, 5, 32, 77, 65, 82, 75, 22, 44, 5, 83, 73, 71, 78, 32, + 179, 158, 7, 76, 20, 88, 7, 86, 79, 67, 65, 76, 73, 67, 154, 159, 7, 65, + 26, 79, 2, 85, 162, 64, 69, 3, 73, 4, 11, 32, 4, 194, 223, 7, 76, 3, 82, + 71, 122, 65, 206, 1, 69, 168, 5, 13, 72, 69, 82, 73, 67, 65, 76, 32, 65, + 78, 71, 76, 69, 82, 73, 220, 1, 2, 76, 65, 91, 79, 17, 48, 4, 71, 72, 69, + 84, 22, 82, 147, 199, 7, 67, 2, 159, 202, 7, 84, 10, 24, 2, 75, 76, 59, + 83, 6, 26, 73, 191, 214, 5, 69, 2, 237, 213, 6, 2, 78, 71, 4, 17, 2, 69, + 32, 4, 214, 53, 72, 135, 3, 86, 22, 104, 2, 65, 75, 230, 3, 69, 68, 4, + 83, 77, 73, 76, 141, 214, 3, 9, 67, 75, 76, 69, 32, 70, 73, 76, 76, 12, + 70, 45, 64, 2, 69, 82, 153, 2, 8, 73, 78, 71, 32, 72, 69, 65, 68, 2, 161, + 193, 5, 11, 78, 79, 45, 69, 86, 73, 76, 32, 77, 79, 78, 9, 33, 6, 32, 87, + 73, 84, 72, 32, 6, 26, 67, 78, 79, 55, 84, 2, 33, 6, 65, 78, 67, 69, 76, + 76, 2, 241, 169, 7, 5, 65, 84, 73, 79, 78, 2, 209, 186, 4, 8, 78, 69, 32, + 83, 79, 85, 78, 68, 2, 25, 4, 72, 82, 69, 69, 2, 149, 197, 6, 10, 32, 83, + 79, 85, 78, 68, 32, 87, 65, 86, 2, 11, 32, 2, 185, 145, 6, 9, 73, 78, 32, + 83, 73, 76, 72, 79, 85, 4, 34, 68, 245, 245, 2, 2, 67, 72, 2, 11, 66, 2, + 247, 195, 6, 79, 2, 203, 208, 6, 79, 7, 45, 9, 32, 79, 80, 69, 78, 73, + 78, 71, 32, 4, 154, 172, 7, 76, 227, 42, 85, 10, 44, 3, 68, 69, 82, 41, + 4, 82, 65, 76, 32, 5, 17, 2, 32, 87, 2, 191, 154, 6, 69, 6, 80, 8, 67, + 65, 76, 69, 78, 68, 65, 82, 0, 4, 78, 79, 84, 69, 29, 2, 83, 72, 2, 141, + 143, 7, 2, 32, 80, 2, 223, 205, 5, 69, 4, 64, 10, 83, 72, 73, 78, 71, 32, + 83, 87, 69, 65, 211, 156, 6, 84, 2, 159, 193, 6, 84, 10, 90, 79, 60, 7, + 85, 84, 73, 78, 71, 32, 87, 192, 240, 2, 3, 82, 84, 83, 191, 173, 3, 78, + 4, 240, 87, 7, 76, 32, 79, 70, 32, 84, 72, 231, 251, 6, 78, 2, 11, 72, 2, + 231, 174, 6, 65, 222, 5, 26, 65, 191, 140, 7, 73, 220, 5, 28, 2, 82, 69, + 163, 69, 84, 218, 5, 30, 32, 245, 46, 2, 68, 32, 234, 3, 250, 1, 65, 104, + 2, 86, 32, 94, 66, 158, 1, 67, 170, 1, 68, 102, 69, 194, 2, 70, 146, 2, + 71, 222, 1, 72, 214, 2, 73, 94, 75, 190, 4, 76, 102, 77, 210, 5, 78, 94, + 79, 178, 1, 80, 222, 3, 82, 154, 3, 83, 206, 2, 84, 74, 87, 230, 6, 89, + 251, 211, 6, 85, 16, 102, 32, 58, 80, 132, 73, 2, 78, 80, 144, 153, 2, 3, + 82, 85, 72, 194, 51, 65, 162, 185, 4, 77, 3, 85, 2, 21, 3, 79, 86, 69, 2, + 11, 82, 2, 175, 190, 7, 32, 4, 234, 184, 2, 69, 231, 138, 1, 65, 14, 98, + 65, 36, 4, 85, 83, 83, 89, 170, 148, 3, 73, 216, 74, 2, 79, 82, 138, 53, + 69, 195, 185, 3, 81, 4, 32, 2, 65, 82, 203, 205, 7, 82, 2, 167, 148, 3, + 69, 22, 98, 65, 30, 79, 158, 20, 77, 196, 183, 7, 7, 32, 79, 86, 69, 82, + 32, 75, 74, 85, 14, 67, 3, 68, 4, 206, 204, 7, 76, 3, 80, 5, 17, 2, 82, + 80, 2, 155, 194, 2, 79, 20, 82, 65, 162, 19, 77, 250, 205, 2, 69, 130, + 49, 79, 162, 185, 4, 66, 2, 74, 3, 76, 5, 135, 22, 65, 18, 66, 69, 22, + 82, 164, 25, 5, 83, 85, 75, 85, 85, 175, 177, 7, 86, 2, 151, 216, 5, 75, + 12, 52, 7, 65, 32, 78, 65, 77, 69, 32, 155, 202, 7, 71, 10, 132, 1, 4, + 72, 69, 73, 83, 20, 5, 84, 65, 73, 83, 89, 152, 22, 3, 77, 69, 73, 220, + 193, 2, 3, 82, 69, 73, 1, 4, 83, 89, 79, 85, 2, 183, 181, 7, 69, 2, 147, + 164, 7, 79, 12, 26, 79, 207, 200, 7, 77, 10, 60, 9, 85, 82, 32, 67, 79, + 82, 78, 69, 82, 227, 171, 7, 79, 8, 26, 32, 243, 199, 7, 83, 6, 128, 1, + 11, 66, 76, 65, 67, 75, 32, 84, 82, 73, 65, 78, 216, 62, 6, 68, 73, 65, + 71, 79, 78, 229, 199, 3, 5, 83, 65, 76, 84, 73, 2, 167, 137, 4, 71, 24, + 102, 65, 46, 73, 172, 12, 5, 85, 82, 65, 77, 85, 238, 149, 3, 72, 246, + 160, 4, 80, 186, 2, 66, 3, 89, 6, 130, 246, 6, 82, 222, 46, 78, 147, 33, + 76, 6, 42, 82, 158, 130, 5, 78, 199, 192, 2, 71, 2, 201, 210, 5, 2, 85, + 68, 30, 130, 1, 65, 22, 69, 54, 79, 62, 85, 242, 214, 1, 80, 156, 227, 5, + 10, 73, 82, 65, 71, 65, 78, 65, 32, 72, 79, 234, 8, 71, 3, 90, 5, 167, + 244, 1, 73, 4, 168, 11, 3, 75, 85, 84, 233, 232, 1, 2, 82, 85, 6, 26, 79, + 151, 195, 7, 78, 4, 170, 158, 7, 82, 235, 36, 78, 6, 54, 73, 144, 17, 4, + 65, 82, 65, 68, 175, 231, 5, 82, 2, 255, 211, 3, 73, 14, 54, 78, 212, 12, + 4, 77, 65, 71, 69, 159, 181, 7, 85, 7, 154, 210, 2, 73, 255, 219, 4, 84, + 58, 230, 1, 65, 48, 2, 89, 85, 20, 3, 73, 82, 79, 78, 77, 90, 79, 60, 2, + 85, 82, 144, 8, 2, 69, 69, 134, 146, 3, 72, 238, 153, 3, 67, 232, 120, 3, + 32, 79, 72, 162, 14, 80, 186, 2, 66, 2, 71, 2, 75, 2, 76, 2, 84, 2, 86, + 3, 87, 9, 22, 82, 131, 98, 73, 4, 22, 79, 163, 21, 65, 2, 183, 252, 4, + 82, 9, 252, 16, 3, 77, 69, 69, 140, 4, 2, 87, 65, 129, 171, 2, 3, 71, 85, + 82, 9, 11, 32, 6, 22, 67, 199, 14, 83, 4, 22, 65, 179, 6, 85, 2, 233, + 233, 4, 2, 80, 73, 4, 18, 79, 23, 82, 2, 243, 158, 7, 80, 2, 207, 129, 6, + 85, 4, 164, 206, 5, 4, 85, 90, 69, 73, 215, 68, 79, 12, 62, 79, 244, 13, + 2, 69, 70, 222, 174, 7, 77, 2, 78, 3, 88, 4, 150, 135, 6, 90, 187, 181, + 1, 71, 82, 162, 1, 32, 70, 65, 122, 66, 22, 69, 54, 73, 110, 77, 68, 2, + 85, 32, 78, 86, 2, 87, 222, 147, 3, 72, 250, 153, 3, 79, 254, 134, 1, 80, + 186, 2, 71, 2, 76, 3, 83, 10, 34, 79, 242, 2, 67, 139, 8, 83, 6, 198, 10, + 86, 199, 159, 7, 72, 13, 54, 73, 44, 2, 78, 83, 206, 84, 82, 215, 234, 4, + 72, 4, 240, 202, 5, 2, 75, 85, 179, 202, 1, 82, 2, 243, 233, 6, 89, 5, + 223, 171, 2, 32, 6, 28, 2, 71, 65, 251, 10, 69, 5, 171, 233, 6, 84, 8, + 42, 75, 20, 2, 82, 73, 199, 184, 7, 76, 2, 183, 147, 6, 85, 5, 11, 66, 2, + 11, 65, 2, 139, 255, 2, 65, 7, 11, 32, 4, 22, 67, 139, 8, 83, 2, 11, 85, + 2, 155, 172, 2, 66, 16, 210, 183, 7, 65, 2, 70, 2, 71, 2, 76, 2, 77, 2, + 83, 2, 86, 3, 87, 5, 11, 32, 2, 11, 77, 2, 171, 244, 6, 69, 16, 70, 65, + 130, 12, 79, 150, 170, 7, 70, 2, 77, 2, 83, 2, 86, 3, 87, 5, 159, 151, 7, + 78, 12, 78, 78, 20, 7, 82, 73, 71, 73, 78, 65, 76, 150, 182, 2, 79, 139, + 255, 4, 86, 2, 199, 144, 7, 83, 6, 21, 3, 32, 79, 70, 7, 25, 4, 32, 79, + 82, 32, 4, 242, 96, 78, 51, 69, 46, 118, 65, 82, 69, 94, 73, 74, 79, 142, + 161, 7, 80, 218, 16, 67, 2, 70, 2, 72, 2, 77, 2, 82, 2, 83, 2, 86, 3, 87, + 9, 38, 65, 229, 254, 3, 3, 32, 65, 77, 4, 236, 1, 2, 83, 69, 219, 140, 7, + 84, 8, 34, 69, 22, 78, 223, 147, 7, 83, 2, 139, 159, 7, 90, 4, 206, 227, + 1, 73, 155, 170, 5, 83, 6, 34, 75, 233, 3, 3, 65, 83, 85, 4, 230, 248, 2, + 85, 163, 185, 4, 79, 6, 34, 73, 22, 78, 21, 2, 83, 73, 2, 131, 195, 3, + 78, 2, 183, 146, 7, 68, 2, 241, 247, 6, 4, 84, 73, 79, 78, 22, 60, 2, 65, + 68, 114, 69, 62, 73, 134, 1, 85, 207, 221, 6, 79, 7, 21, 3, 32, 79, 86, + 4, 25, 4, 69, 82, 32, 83, 5, 17, 2, 32, 83, 2, 11, 81, 2, 133, 164, 2, 2, + 85, 65, 4, 36, 3, 78, 84, 79, 183, 138, 7, 77, 2, 179, 219, 5, 71, 6, 40, + 2, 71, 72, 62, 84, 235, 171, 7, 82, 2, 233, 238, 3, 10, 84, 32, 79, 80, + 69, 78, 32, 66, 79, 88, 2, 253, 244, 2, 2, 84, 79, 4, 228, 1, 2, 85, 66, + 163, 233, 4, 80, 22, 138, 1, 65, 72, 3, 69, 78, 84, 28, 11, 80, 73, 82, + 65, 76, 32, 70, 82, 79, 77, 32, 152, 188, 2, 3, 73, 82, 73, 214, 239, 4, + 82, 3, 86, 4, 48, 2, 73, 75, 149, 173, 2, 4, 78, 84, 73, 73, 2, 251, 242, + 2, 85, 4, 134, 172, 7, 73, 3, 79, 8, 18, 66, 43, 84, 4, 201, 128, 7, 5, + 79, 84, 84, 79, 77, 4, 11, 79, 4, 151, 128, 7, 80, 6, 42, 65, 186, 135, + 3, 72, 207, 211, 3, 79, 2, 201, 245, 6, 2, 82, 71, 34, 50, 65, 20, 4, 73, + 84, 72, 32, 131, 170, 7, 66, 2, 215, 187, 3, 84, 30, 214, 1, 66, 40, 3, + 68, 73, 65, 0, 5, 79, 82, 84, 72, 79, 94, 72, 54, 76, 126, 84, 28, 6, 85, + 80, 80, 69, 82, 32, 186, 1, 86, 178, 136, 4, 82, 173, 130, 2, 13, 67, 79, + 78, 84, 79, 85, 82, 69, 68, 32, 79, 85, 84, 2, 217, 140, 4, 5, 79, 84, + 84, 79, 77, 2, 11, 71, 2, 213, 175, 4, 15, 79, 78, 65, 76, 32, 67, 82, + 79, 83, 83, 72, 65, 84, 67, 72, 2, 11, 79, 2, 149, 3, 6, 82, 73, 90, 79, + 78, 84, 6, 44, 5, 79, 87, 69, 82, 32, 199, 138, 4, 69, 4, 44, 3, 76, 69, + 70, 1, 4, 82, 73, 71, 72, 2, 189, 1, 3, 84, 32, 68, 2, 165, 138, 4, 2, + 79, 80, 8, 60, 5, 76, 69, 70, 84, 32, 37, 6, 82, 73, 71, 72, 84, 32, 4, + 70, 68, 185, 153, 4, 2, 84, 79, 4, 34, 68, 245, 171, 4, 2, 84, 79, 2, + 129, 137, 4, 7, 73, 65, 71, 79, 78, 65, 76, 2, 29, 5, 69, 82, 84, 73, 67, + 2, 225, 171, 4, 2, 65, 76, 6, 32, 2, 65, 65, 195, 217, 5, 85, 4, 214, + 254, 6, 82, 247, 5, 68, 240, 1, 226, 1, 67, 174, 9, 68, 34, 70, 88, 3, + 82, 73, 83, 142, 1, 72, 66, 75, 106, 76, 166, 1, 77, 38, 78, 34, 79, 94, + 80, 34, 83, 190, 2, 84, 156, 1, 5, 69, 73, 71, 72, 84, 22, 85, 90, 86, + 230, 249, 4, 65, 218, 107, 87, 223, 97, 73, 90, 128, 1, 21, 74, 75, 32, + 85, 78, 73, 70, 73, 69, 68, 32, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, + 198, 141, 6, 79, 243, 146, 1, 76, 86, 68, 2, 52, 69, 82, 53, 206, 2, 54, + 150, 2, 55, 166, 1, 56, 95, 57, 10, 50, 48, 234, 4, 65, 214, 242, 5, 56, + 223, 97, 50, 4, 182, 159, 7, 48, 3, 57, 30, 150, 1, 50, 42, 51, 38, 52, + 48, 2, 53, 66, 0, 2, 68, 69, 22, 57, 164, 4, 2, 56, 70, 222, 162, 3, 66, + 248, 205, 2, 2, 70, 56, 221, 97, 2, 49, 56, 6, 214, 4, 55, 142, 211, 6, + 49, 3, 52, 4, 210, 167, 3, 70, 143, 206, 2, 67, 4, 26, 48, 219, 167, 3, + 51, 2, 155, 157, 7, 56, 2, 135, 157, 7, 54, 4, 174, 167, 3, 50, 143, 243, + 3, 49, 24, 86, 50, 46, 51, 50, 53, 34, 54, 16, 2, 55, 48, 28, 2, 70, 49, + 129, 2, 2, 69, 56, 6, 70, 57, 170, 165, 3, 53, 167, 186, 2, 52, 4, 26, + 53, 131, 166, 3, 48, 2, 175, 155, 7, 53, 4, 202, 2, 66, 143, 163, 3, 57, + 2, 171, 2, 50, 4, 234, 154, 7, 56, 3, 57, 2, 207, 154, 7, 52, 12, 74, 53, + 38, 57, 12, 2, 49, 50, 20, 2, 68, 52, 245, 150, 7, 2, 65, 55, 4, 230, + 163, 3, 51, 231, 244, 2, 49, 2, 11, 56, 2, 191, 153, 7, 49, 2, 171, 153, + 7, 50, 6, 50, 57, 20, 2, 68, 55, 141, 163, 3, 2, 67, 65, 2, 219, 162, 3, + 69, 2, 207, 152, 7, 48, 4, 152, 210, 6, 2, 49, 52, 233, 67, 2, 48, 52, 2, + 11, 79, 2, 143, 216, 3, 84, 10, 84, 3, 65, 76, 76, 104, 4, 79, 85, 82, + 32, 240, 5, 3, 73, 86, 69, 135, 161, 5, 82, 2, 57, 12, 73, 78, 71, 32, + 68, 73, 65, 71, 79, 78, 65, 76, 2, 11, 32, 2, 11, 83, 2, 147, 174, 5, 76, + 4, 230, 211, 3, 68, 199, 194, 3, 75, 8, 212, 213, 3, 2, 73, 45, 214, 135, + 3, 68, 222, 56, 67, 3, 86, 8, 56, 8, 65, 84, 65, 75, 65, 78, 65, 32, 199, + 131, 7, 69, 6, 226, 225, 6, 75, 214, 28, 68, 159, 20, 83, 60, 36, 5, 65, + 84, 73, 78, 32, 79, 79, 54, 164, 5, 12, 83, 77, 65, 76, 76, 32, 76, 69, + 84, 84, 69, 82, 239, 123, 67, 6, 198, 210, 3, 71, 201, 170, 2, 3, 83, 83, + 76, 4, 178, 141, 5, 73, 139, 134, 2, 86, 4, 246, 148, 6, 69, 163, 126, + 71, 4, 140, 3, 15, 78, 69, 32, 72, 85, 78, 68, 82, 69, 68, 32, 84, 87, + 69, 78, 235, 143, 7, 75, 4, 158, 2, 80, 131, 138, 5, 76, 20, 110, 69, + 146, 1, 72, 20, 2, 73, 88, 134, 251, 4, 65, 186, 66, 77, 222, 49, 81, + 246, 98, 79, 222, 61, 68, 3, 83, 4, 56, 7, 67, 79, 78, 68, 32, 83, 67, + 21, 3, 86, 69, 78, 2, 219, 188, 5, 82, 2, 29, 5, 32, 80, 79, 73, 78, 2, + 11, 84, 2, 203, 229, 5, 32, 2, 251, 143, 7, 86, 2, 17, 2, 84, 89, 2, 199, + 143, 7, 32, 8, 44, 4, 72, 82, 69, 69, 22, 87, 243, 4, 73, 2, 239, 200, 6, + 32, 4, 68, 13, 69, 78, 84, 89, 45, 84, 87, 79, 32, 80, 79, 73, 78, 19, + 79, 2, 223, 86, 84, 2, 139, 139, 7, 32, 4, 48, 6, 80, 32, 87, 73, 84, 72, + 191, 199, 6, 72, 2, 17, 2, 32, 69, 2, 247, 119, 88, 4, 150, 199, 6, 79, + 163, 70, 83, 2, 137, 237, 5, 6, 32, 66, 76, 65, 67, 75, 6, 234, 140, 7, + 50, 2, 51, 3, 65, 93, 134, 1, 65, 178, 6, 69, 236, 1, 10, 73, 67, 75, 32, + 70, 73, 71, 85, 82, 69, 182, 1, 79, 90, 82, 174, 4, 85, 238, 252, 6, 83, + 3, 88, 38, 92, 6, 70, 70, 32, 79, 70, 32, 106, 77, 82, 82, 206, 3, 84, + 210, 133, 3, 78, 243, 214, 2, 68, 4, 60, 8, 65, 69, 83, 67, 85, 76, 65, + 80, 21, 3, 72, 69, 82, 2, 163, 132, 5, 73, 2, 147, 246, 5, 77, 2, 21, 3, + 80, 69, 68, 2, 17, 2, 32, 69, 2, 229, 220, 1, 4, 78, 86, 69, 76, 24, 42, + 32, 205, 1, 5, 84, 32, 79, 70, 32, 12, 82, 69, 54, 79, 180, 235, 1, 8, + 65, 78, 68, 32, 67, 82, 69, 83, 139, 128, 2, 87, 2, 17, 2, 81, 85, 2, 11, + 65, 2, 171, 202, 6, 76, 4, 44, 5, 70, 32, 68, 65, 86, 251, 199, 3, 80, 2, + 163, 193, 6, 73, 12, 66, 71, 30, 83, 40, 4, 80, 82, 79, 84, 242, 77, 72, + 215, 101, 84, 2, 89, 4, 85, 65, 82, 68, 4, 26, 69, 191, 249, 1, 84, 2, + 11, 76, 2, 21, 3, 69, 67, 84, 2, 29, 5, 69, 68, 32, 65, 82, 2, 183, 131, + 7, 69, 4, 188, 152, 5, 10, 85, 69, 32, 79, 70, 32, 76, 73, 66, 69, 179, + 157, 1, 73, 8, 92, 2, 65, 77, 112, 9, 78, 79, 71, 82, 65, 80, 72, 73, 67, + 209, 214, 1, 4, 84, 72, 79, 83, 4, 54, 32, 197, 250, 5, 7, 73, 78, 71, + 32, 66, 79, 87, 2, 33, 6, 76, 79, 67, 79, 77, 79, 2, 143, 208, 1, 84, 2, + 217, 177, 4, 2, 32, 70, 11, 11, 32, 8, 60, 5, 87, 73, 84, 72, 32, 233, + 163, 2, 4, 76, 69, 65, 78, 4, 32, 4, 65, 82, 77, 83, 51, 68, 2, 25, 4, + 32, 82, 65, 73, 2, 203, 246, 1, 83, 2, 167, 235, 5, 82, 4, 44, 4, 67, 75, + 32, 67, 21, 3, 80, 87, 65, 2, 163, 250, 5, 72, 2, 131, 223, 6, 84, 20, + 66, 65, 108, 10, 69, 83, 83, 32, 79, 85, 84, 76, 73, 78, 79, 73, 6, 44, + 4, 73, 71, 72, 84, 45, 3, 87, 66, 69, 4, 184, 150, 3, 2, 32, 82, 143, + 211, 2, 78, 2, 243, 72, 82, 2, 17, 2, 69, 68, 2, 17, 2, 32, 87, 2, 169, + 233, 4, 4, 72, 73, 84, 69, 12, 76, 4, 78, 71, 32, 84, 52, 4, 80, 69, 68, + 32, 241, 8, 4, 67, 84, 76, 89, 2, 17, 2, 69, 82, 2, 181, 197, 6, 3, 77, + 73, 78, 8, 70, 68, 24, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 13, 2, 85, + 80, 2, 33, 3, 79, 87, 78, 2, 11, 84, 2, 11, 45, 2, 169, 216, 5, 8, 80, + 79, 73, 78, 84, 73, 78, 71, 6, 100, 8, 68, 73, 79, 32, 77, 73, 67, 82, + 20, 9, 70, 70, 69, 68, 32, 70, 76, 65, 84, 187, 249, 6, 80, 2, 167, 209, + 5, 79, 2, 11, 66, 2, 243, 166, 4, 82, 162, 3, 150, 1, 66, 212, 2, 5, 67, + 67, 69, 69, 68, 192, 3, 8, 77, 77, 65, 84, 73, 79, 78, 32, 70, 78, 204, + 22, 3, 80, 69, 82, 204, 11, 2, 82, 70, 47, 83, 63, 11, 83, 60, 76, 6, 67, + 82, 73, 80, 84, 32, 136, 1, 3, 69, 84, 32, 93, 3, 84, 73, 84, 30, 118, + 76, 186, 29, 69, 166, 1, 80, 22, 82, 218, 211, 2, 77, 246, 149, 2, 70, + 30, 83, 42, 84, 62, 90, 238, 85, 78, 15, 79, 2, 207, 30, 69, 28, 56, 6, + 65, 66, 79, 86, 69, 32, 246, 33, 79, 159, 3, 87, 6, 142, 32, 83, 143, + 184, 5, 82, 2, 251, 176, 5, 85, 22, 11, 83, 23, 11, 32, 20, 128, 1, 6, + 65, 66, 79, 86, 69, 32, 156, 1, 7, 66, 85, 84, 32, 78, 79, 84, 32, 2, 79, + 82, 141, 146, 3, 5, 85, 78, 68, 69, 82, 12, 100, 4, 78, 79, 84, 32, 28, + 12, 83, 73, 78, 71, 76, 69, 45, 76, 73, 78, 69, 32, 218, 32, 65, 39, 69, + 4, 242, 32, 65, 167, 1, 69, 4, 250, 32, 69, 83, 78, 2, 93, 5, 32, 69, 81, + 85, 73, 4, 17, 2, 32, 69, 4, 17, 2, 81, 85, 4, 22, 73, 167, 33, 65, 2, + 173, 33, 6, 86, 65, 76, 69, 78, 84, 6, 242, 136, 3, 66, 136, 228, 1, 4, + 87, 73, 84, 72, 147, 142, 1, 84, 249, 1, 194, 1, 32, 72, 7, 68, 65, 78, + 69, 83, 69, 32, 204, 12, 4, 82, 73, 83, 69, 72, 5, 85, 87, 65, 82, 32, + 132, 174, 2, 14, 83, 69, 84, 32, 79, 86, 69, 82, 32, 66, 85, 73, 76, 68, + 159, 157, 3, 70, 4, 50, 87, 225, 197, 5, 6, 66, 69, 72, 73, 78, 68, 2, + 171, 180, 6, 73, 146, 1, 198, 2, 65, 20, 17, 67, 79, 78, 83, 79, 78, 65, + 78, 84, 32, 83, 73, 71, 78, 32, 80, 65, 172, 1, 7, 76, 69, 84, 84, 69, + 82, 32, 188, 3, 18, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 66, + 73, 78, 68, 85, 32, 228, 1, 5, 83, 73, 71, 78, 32, 204, 1, 13, 86, 79, + 87, 69, 76, 32, 83, 73, 71, 78, 32, 80, 65, 183, 245, 4, 68, 2, 171, 152, + 3, 86, 10, 68, 2, 77, 73, 40, 2, 78, 89, 33, 7, 83, 65, 78, 71, 65, 78, + 32, 2, 17, 2, 78, 71, 2, 207, 228, 5, 75, 4, 142, 5, 65, 211, 155, 1, 73, + 4, 162, 235, 6, 77, 3, 87, 76, 246, 1, 65, 58, 70, 78, 76, 2, 82, 34, 83, + 254, 150, 3, 69, 254, 216, 1, 78, 238, 178, 1, 66, 2, 75, 138, 69, 67, 2, + 68, 2, 71, 2, 72, 2, 74, 2, 77, 2, 80, 2, 81, 2, 84, 2, 86, 2, 87, 2, 88, + 2, 89, 2, 90, 186, 2, 73, 2, 79, 3, 85, 7, 180, 171, 6, 6, 82, 67, 72, + 65, 73, 67, 147, 64, 69, 6, 44, 5, 73, 78, 65, 76, 32, 227, 234, 6, 65, + 4, 222, 234, 6, 75, 3, 77, 4, 218, 197, 6, 69, 235, 36, 65, 4, 234, 231, + 6, 89, 187, 2, 65, 16, 90, 66, 2, 68, 2, 75, 12, 3, 76, 69, 85, 38, 67, + 34, 80, 253, 229, 6, 3, 83, 85, 82, 2, 11, 65, 2, 253, 103, 5, 32, 83, + 65, 84, 65, 2, 11, 65, 2, 171, 164, 6, 75, 4, 172, 3, 3, 65, 78, 71, 189, + 162, 6, 3, 85, 82, 78, 10, 32, 2, 80, 65, 235, 145, 3, 86, 8, 28, 3, 77, + 65, 65, 23, 78, 2, 203, 230, 6, 69, 6, 18, 71, 71, 89, 4, 38, 76, 225, + 160, 6, 3, 87, 73, 83, 2, 209, 197, 5, 2, 65, 89, 2, 201, 227, 6, 2, 69, + 67, 12, 18, 77, 23, 78, 2, 187, 134, 3, 69, 10, 96, 3, 69, 85, 76, 34, + 79, 22, 89, 188, 228, 1, 3, 71, 72, 85, 201, 189, 2, 4, 65, 69, 76, 65, + 2, 11, 69, 2, 195, 148, 6, 85, 2, 231, 161, 4, 76, 2, 247, 151, 1, 85, 5, + 165, 161, 3, 13, 32, 79, 86, 69, 82, 32, 77, 79, 85, 78, 84, 65, 73, 88, + 88, 7, 76, 69, 84, 84, 69, 82, 32, 184, 6, 6, 83, 73, 71, 78, 32, 80, + 135, 236, 4, 68, 66, 142, 2, 65, 66, 67, 66, 68, 44, 3, 72, 65, 77, 22, + 74, 34, 75, 42, 78, 38, 79, 2, 85, 34, 80, 38, 82, 20, 4, 83, 72, 89, 69, + 34, 84, 112, 3, 86, 65, 82, 230, 169, 4, 71, 148, 57, 4, 76, 79, 65, 67, + 136, 89, 2, 73, 77, 210, 13, 89, 150, 27, 66, 214, 89, 69, 203, 28, 77, + 6, 42, 80, 238, 215, 5, 65, 255, 134, 1, 86, 2, 251, 248, 1, 80, 4, 40, + 2, 72, 69, 253, 204, 6, 2, 65, 82, 2, 243, 207, 5, 76, 4, 22, 69, 155, + 95, 79, 2, 219, 204, 6, 86, 2, 171, 193, 6, 83, 2, 11, 89, 2, 239, 222, + 6, 65, 6, 194, 172, 6, 76, 146, 48, 73, 99, 72, 4, 138, 190, 5, 71, 171, + 160, 1, 65, 2, 11, 84, 2, 147, 144, 1, 84, 4, 198, 189, 5, 72, 175, 161, + 1, 73, 2, 243, 185, 6, 69, 4, 242, 199, 6, 76, 215, 22, 82, 8, 42, 69, + 22, 72, 209, 187, 6, 2, 65, 83, 2, 171, 142, 1, 78, 4, 26, 65, 167, 185, + 5, 69, 2, 243, 201, 6, 82, 2, 255, 218, 6, 67, 2, 175, 190, 6, 86, 72, + 54, 83, 182, 237, 4, 72, 209, 66, 4, 86, 73, 76, 76, 68, 56, 6, 67, 82, + 73, 80, 84, 32, 229, 2, 3, 69, 84, 32, 34, 114, 69, 34, 76, 134, 1, 80, + 22, 82, 218, 211, 2, 77, 246, 149, 2, 70, 30, 83, 42, 84, 62, 90, 238, + 85, 78, 15, 79, 4, 174, 69, 81, 183, 235, 5, 73, 6, 88, 18, 65, 84, 73, + 78, 32, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 31, 69, 4, + 170, 218, 6, 73, 3, 78, 2, 55, 70, 2, 191, 171, 3, 76, 2, 21, 3, 73, 71, + 72, 2, 11, 84, 2, 139, 215, 2, 32, 34, 148, 1, 6, 65, 66, 79, 86, 69, 32, + 96, 7, 66, 69, 83, 73, 68, 69, 32, 162, 1, 79, 158, 3, 87, 221, 189, 4, + 9, 80, 82, 69, 67, 69, 68, 73, 78, 71, 6, 26, 83, 135, 188, 2, 76, 4, 11, + 85, 4, 26, 80, 151, 236, 4, 66, 2, 145, 236, 4, 2, 69, 82, 4, 108, 23, + 65, 78, 68, 32, 74, 79, 73, 78, 69, 68, 32, 66, 89, 32, 68, 65, 83, 72, + 32, 87, 73, 84, 72, 23, 83, 2, 17, 2, 32, 83, 2, 241, 234, 4, 2, 85, 66, + 16, 11, 70, 17, 11, 32, 14, 120, 6, 65, 66, 79, 86, 69, 32, 132, 1, 4, + 87, 73, 84, 72, 249, 210, 5, 11, 79, 82, 32, 69, 81, 85, 65, 76, 32, 84, + 79, 8, 34, 65, 38, 69, 18, 84, 67, 78, 2, 11, 76, 2, 113, 3, 77, 79, 83, + 2, 203, 62, 81, 2, 21, 3, 73, 76, 68, 2, 171, 148, 3, 69, 2, 17, 2, 32, + 78, 2, 11, 79, 2, 11, 84, 2, 11, 32, 2, 11, 69, 2, 17, 2, 81, 85, 2, 11, + 65, 2, 11, 76, 2, 219, 228, 2, 32, 6, 25, 4, 73, 84, 72, 32, 6, 96, 2, + 80, 76, 24, 14, 77, 85, 76, 84, 73, 80, 76, 73, 67, 65, 84, 73, 79, 78, + 183, 129, 6, 68, 2, 11, 85, 2, 11, 83, 2, 165, 164, 4, 5, 32, 83, 73, 71, + 78, 4, 144, 202, 4, 2, 65, 67, 215, 206, 1, 69, 4, 60, 9, 80, 69, 78, 83, + 73, 79, 78, 32, 82, 131, 189, 6, 72, 2, 21, 3, 65, 73, 76, 2, 187, 196, + 5, 87, 8, 26, 65, 98, 73, 35, 85, 4, 44, 5, 83, 72, 32, 65, 77, 227, 207, + 6, 78, 2, 253, 139, 6, 7, 80, 69, 82, 83, 65, 78, 68, 2, 11, 77, 2, 183, + 145, 6, 77, 2, 233, 145, 3, 2, 78, 71, 236, 2, 92, 11, 76, 79, 84, 73, + 32, 78, 65, 71, 82, 73, 32, 226, 5, 77, 194, 17, 78, 97, 2, 82, 73, 90, + 180, 1, 7, 76, 69, 84, 84, 69, 82, 32, 168, 2, 11, 80, 79, 69, 84, 82, + 89, 32, 77, 65, 82, 75, 56, 5, 83, 73, 71, 78, 32, 145, 1, 11, 86, 79, + 87, 69, 76, 32, 83, 73, 71, 78, 32, 64, 174, 1, 68, 46, 82, 34, 84, 166, + 91, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 198, 208, 5, 72, 2, 76, 2, 77, + 2, 78, 2, 83, 246, 30, 65, 2, 69, 2, 73, 2, 79, 3, 85, 8, 238, 91, 68, + 198, 208, 5, 72, 247, 30, 79, 4, 134, 172, 6, 82, 247, 30, 79, 8, 162, + 91, 84, 198, 208, 5, 72, 247, 30, 79, 8, 11, 45, 8, 162, 202, 6, 49, 2, + 50, 2, 51, 3, 52, 8, 46, 65, 70, 72, 165, 132, 6, 3, 68, 86, 73, 4, 64, + 10, 76, 84, 69, 82, 78, 65, 84, 69, 32, 72, 155, 132, 6, 78, 2, 193, 137, + 2, 2, 65, 83, 10, 242, 169, 6, 79, 246, 30, 65, 2, 69, 2, 73, 3, 85, 88, + 60, 8, 66, 79, 76, 32, 70, 79, 82, 32, 225, 16, 2, 77, 69, 86, 238, 2, + 66, 48, 2, 67, 65, 118, 68, 234, 3, 69, 130, 2, 70, 56, 8, 72, 79, 82, + 73, 90, 79, 78, 84, 0, 6, 86, 69, 82, 84, 73, 67, 44, 3, 76, 73, 78, 52, + 9, 77, 65, 82, 75, 83, 32, 67, 72, 65, 22, 78, 90, 65, 68, 3, 82, 69, 67, + 32, 5, 71, 82, 79, 85, 80, 0, 4, 85, 78, 73, 84, 22, 83, 205, 3, 15, 84, + 89, 80, 69, 32, 65, 32, 69, 76, 69, 67, 84, 82, 79, 78, 4, 190, 189, 4, + 69, 137, 201, 1, 3, 65, 67, 75, 4, 44, 3, 82, 82, 73, 189, 183, 1, 2, 78, + 67, 2, 33, 6, 65, 71, 69, 32, 82, 69, 2, 11, 84, 2, 215, 185, 1, 85, 20, + 24, 2, 65, 84, 55, 69, 2, 173, 5, 9, 65, 32, 76, 73, 78, 75, 32, 69, 83, + 18, 84, 4, 76, 69, 84, 69, 133, 2, 12, 86, 73, 67, 69, 32, 67, 79, 78, + 84, 82, 79, 76, 11, 11, 32, 8, 160, 1, 11, 82, 69, 67, 84, 65, 78, 71, + 85, 76, 65, 82, 0, 6, 83, 81, 85, 65, 82, 69, 154, 9, 70, 129, 185, 5, + 11, 77, 69, 68, 73, 85, 77, 32, 83, 72, 65, 68, 2, 61, 13, 32, 67, 72, + 69, 67, 75, 69, 82, 32, 66, 79, 65, 82, 2, 227, 193, 5, 68, 8, 11, 32, 8, + 162, 173, 1, 70, 162, 163, 3, 84, 183, 86, 79, 12, 22, 78, 207, 1, 83, + 10, 48, 5, 68, 32, 79, 70, 32, 137, 1, 2, 81, 85, 8, 18, 77, 31, 84, 2, + 245, 149, 5, 2, 69, 68, 6, 64, 11, 82, 65, 78, 83, 77, 73, 83, 83, 73, + 79, 78, 223, 107, 69, 5, 235, 207, 3, 32, 2, 155, 7, 73, 2, 209, 246, 4, + 2, 67, 65, 4, 36, 2, 73, 76, 73, 3, 79, 82, 77, 2, 191, 2, 69, 2, 229, + 160, 1, 6, 65, 76, 32, 84, 65, 66, 2, 11, 69, 2, 17, 2, 32, 70, 2, 163, + 177, 1, 69, 2, 227, 132, 5, 80, 6, 26, 69, 239, 180, 4, 85, 4, 56, 8, 71, + 65, 84, 73, 86, 69, 32, 65, 175, 162, 5, 87, 2, 21, 3, 67, 75, 78, 2, 21, + 3, 79, 87, 76, 2, 231, 135, 1, 69, 2, 11, 79, 2, 17, 2, 82, 68, 2, 163, + 141, 3, 32, 18, 180, 1, 7, 65, 77, 65, 82, 73, 84, 65, 60, 3, 72, 73, 70, + 52, 8, 84, 65, 82, 84, 32, 79, 70, 32, 64, 8, 85, 66, 83, 84, 73, 84, 85, + 84, 200, 1, 3, 89, 78, 67, 195, 249, 5, 80, 2, 25, 4, 78, 32, 83, 79, 2, + 11, 85, 2, 155, 252, 5, 82, 4, 17, 2, 84, 32, 4, 158, 191, 5, 79, 243, + 41, 73, 4, 22, 72, 215, 101, 84, 2, 17, 2, 69, 65, 2, 159, 231, 5, 68, 4, + 11, 69, 5, 11, 32, 2, 11, 70, 2, 21, 3, 79, 82, 77, 2, 17, 2, 32, 84, 2, + 203, 152, 6, 87, 2, 205, 249, 5, 2, 73, 67, 2, 11, 84, 2, 187, 165, 6, + 82, 7, 38, 67, 181, 171, 2, 3, 65, 71, 79, 2, 181, 133, 1, 9, 72, 82, 79, + 78, 79, 85, 83, 32, 73, 180, 1, 36, 3, 65, 67, 32, 199, 128, 5, 78, 178, + 1, 150, 3, 65, 52, 4, 66, 65, 82, 82, 32, 2, 67, 79, 80, 13, 68, 79, 84, + 84, 69, 68, 32, 90, 76, 65, 77, 65, 32, 118, 69, 90, 72, 204, 2, 7, 76, + 69, 84, 84, 69, 82, 32, 236, 9, 9, 79, 66, 76, 73, 81, 85, 69, 32, 76, + 28, 4, 80, 84, 72, 65, 0, 4, 90, 81, 65, 80, 110, 82, 88, 2, 83, 85, 174, + 2, 84, 140, 247, 4, 3, 77, 85, 83, 232, 22, 7, 70, 69, 77, 73, 78, 73, + 78, 253, 136, 1, 5, 81, 85, 83, 72, 83, 2, 11, 66, 2, 133, 201, 5, 5, 66, + 82, 69, 86, 73, 2, 11, 69, 2, 247, 176, 6, 75, 6, 38, 78, 165, 17, 4, 76, + 79, 78, 32, 2, 17, 2, 84, 82, 2, 219, 224, 5, 65, 4, 28, 3, 65, 78, 71, + 35, 72, 2, 11, 85, 2, 191, 143, 5, 76, 2, 11, 79, 2, 169, 220, 3, 5, 82, + 73, 90, 79, 78, 6, 60, 10, 78, 68, 32, 79, 70, 32, 80, 65, 82, 65, 151, + 14, 83, 2, 201, 11, 2, 71, 82, 14, 104, 8, 65, 82, 75, 76, 69, 65, 78, + 32, 132, 1, 4, 66, 65, 83, 65, 65, 7, 79, 82, 73, 90, 79, 78, 84, 6, 60, + 5, 65, 83, 84, 69, 82, 40, 4, 77, 69, 84, 79, 3, 79, 2, 17, 2, 73, 83, 2, + 155, 168, 4, 67, 2, 201, 167, 2, 2, 66, 69, 6, 164, 11, 8, 45, 69, 83, + 65, 83, 65, 32, 68, 151, 132, 4, 32, 2, 137, 221, 5, 2, 65, 76, 92, 154, + 2, 68, 102, 72, 32, 3, 76, 65, 77, 34, 77, 204, 1, 2, 80, 69, 142, 1, 82, + 78, 83, 144, 1, 8, 70, 73, 78, 65, 76, 32, 83, 69, 106, 65, 14, 75, 2, + 81, 34, 84, 40, 5, 71, 65, 77, 65, 76, 40, 4, 89, 85, 68, 72, 214, 74, + 66, 142, 172, 4, 90, 166, 44, 78, 134, 2, 87, 175, 126, 69, 4, 76, 14, + 79, 84, 76, 69, 83, 83, 32, 68, 65, 76, 65, 84, 72, 32, 139, 3, 65, 2, + 167, 201, 2, 82, 4, 11, 69, 5, 239, 168, 6, 84, 2, 11, 65, 2, 207, 168, + 6, 68, 24, 60, 9, 65, 76, 65, 89, 65, 76, 65, 77, 32, 183, 152, 6, 73, + 22, 78, 76, 22, 78, 234, 173, 4, 66, 246, 213, 1, 84, 138, 34, 83, 14, + 74, 3, 82, 4, 135, 159, 4, 76, 8, 246, 83, 78, 250, 209, 5, 71, 3, 89, 9, + 33, 6, 82, 83, 73, 65, 78, 32, 6, 50, 66, 16, 3, 68, 72, 65, 17, 3, 71, + 72, 65, 2, 199, 78, 72, 2, 147, 2, 76, 2, 199, 157, 5, 77, 4, 52, 7, 69, + 86, 69, 82, 83, 69, 68, 159, 190, 4, 73, 2, 255, 222, 4, 32, 14, 142, 1, + 69, 40, 7, 79, 71, 68, 73, 65, 78, 32, 64, 12, 85, 80, 69, 82, 83, 67, + 82, 73, 80, 84, 32, 65, 222, 164, 5, 72, 201, 82, 2, 65, 68, 2, 17, 2, + 77, 75, 2, 243, 162, 4, 65, 6, 42, 90, 32, 2, 75, 72, 211, 141, 6, 70, 2, + 239, 247, 4, 72, 2, 11, 76, 2, 11, 65, 2, 135, 163, 6, 80, 6, 36, 3, 69, + 84, 72, 191, 165, 5, 65, 5, 173, 85, 6, 32, 71, 65, 82, 83, 72, 5, 167, + 246, 5, 32, 4, 217, 132, 4, 2, 73, 78, 6, 21, 3, 72, 65, 32, 6, 42, 68, + 178, 132, 4, 66, 167, 161, 1, 65, 2, 17, 2, 79, 84, 2, 223, 150, 1, 84, + 8, 58, 66, 162, 203, 2, 87, 249, 166, 1, 4, 85, 75, 75, 65, 4, 157, 240, + 3, 2, 65, 83, 14, 88, 8, 66, 76, 73, 78, 69, 65, 82, 32, 113, 10, 80, 82, + 65, 76, 73, 78, 69, 65, 82, 32, 8, 44, 5, 67, 79, 76, 79, 78, 211, 206, + 3, 70, 7, 11, 32, 4, 29, 5, 83, 75, 69, 87, 69, 4, 251, 244, 5, 68, 6, + 44, 5, 67, 79, 76, 79, 78, 227, 205, 3, 70, 5, 253, 236, 5, 7, 32, 83, + 75, 69, 87, 69, 68, 8, 62, 72, 25, 11, 87, 79, 32, 86, 69, 82, 84, 73, + 67, 65, 76, 4, 21, 3, 82, 69, 69, 4, 25, 4, 32, 68, 79, 84, 4, 231, 255, + 3, 83, 190, 43, 102, 45, 58, 65, 206, 105, 69, 154, 45, 72, 182, 46, 73, + 230, 69, 79, 170, 33, 82, 214, 16, 85, 147, 22, 87, 4, 32, 2, 83, 72, + 255, 246, 4, 82, 2, 235, 149, 5, 73, 244, 26, 182, 1, 66, 82, 71, 196, + 17, 2, 73, 32, 170, 29, 75, 144, 5, 9, 76, 76, 89, 32, 77, 65, 82, 75, + 32, 34, 77, 178, 37, 78, 132, 13, 3, 80, 69, 32, 94, 85, 190, 148, 5, 67, + 159, 11, 88, 5, 221, 122, 16, 76, 69, 32, 84, 69, 78, 78, 73, 83, 32, 80, + 65, 68, 68, 76, 69, 144, 2, 78, 32, 240, 11, 5, 65, 76, 79, 71, 32, 241, + 2, 6, 66, 65, 78, 87, 65, 32, 190, 1, 162, 1, 65, 102, 67, 186, 1, 68, + 58, 69, 98, 71, 118, 72, 46, 76, 230, 3, 80, 58, 81, 62, 82, 106, 83, + 142, 76, 78, 230, 222, 1, 84, 250, 145, 1, 70, 231, 183, 1, 86, 6, 42, + 80, 150, 201, 2, 77, 219, 187, 1, 83, 2, 17, 2, 79, 83, 2, 145, 235, 5, + 4, 84, 82, 79, 80, 8, 18, 73, 63, 79, 2, 25, 4, 82, 67, 85, 77, 2, 213, + 2, 4, 70, 76, 69, 88, 6, 26, 77, 167, 199, 5, 76, 4, 11, 77, 4, 152, 131, + 5, 7, 69, 82, 67, 73, 65, 76, 32, 235, 147, 1, 65, 22, 26, 79, 151, 165, + 4, 73, 2, 11, 76, 2, 235, 84, 76, 4, 18, 81, 43, 88, 2, 17, 2, 85, 65, 2, + 171, 231, 2, 76, 2, 153, 172, 5, 4, 67, 76, 65, 77, 4, 11, 82, 4, 18, 65, + 55, 69, 2, 11, 86, 2, 11, 69, 2, 177, 120, 3, 32, 65, 67, 2, 145, 4, 4, + 65, 84, 69, 82, 2, 145, 142, 2, 6, 89, 80, 72, 69, 78, 45, 114, 38, 65, + 254, 2, 69, 247, 168, 4, 79, 104, 25, 4, 84, 73, 78, 32, 104, 34, 67, 41, + 4, 83, 77, 65, 76, 52, 11, 65, 52, 25, 4, 80, 73, 84, 65, 52, 41, 8, 76, + 32, 76, 69, 84, 84, 69, 82, 52, 11, 32, 52, 194, 146, 6, 65, 2, 66, 2, + 67, 2, 68, 2, 69, 2, 70, 2, 71, 2, 72, 2, 73, 2, 74, 2, 75, 2, 76, 2, 77, + 2, 78, 2, 79, 2, 80, 2, 81, 2, 82, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, + 88, 2, 89, 3, 90, 8, 22, 83, 199, 1, 70, 2, 11, 83, 2, 197, 63, 3, 45, + 84, 72, 4, 26, 69, 203, 225, 2, 76, 2, 11, 82, 2, 227, 67, 67, 4, 11, 85, + 4, 26, 79, 203, 190, 3, 69, 2, 147, 166, 5, 84, 8, 36, 3, 73, 71, 72, + 199, 249, 3, 69, 6, 17, 2, 84, 32, 6, 242, 136, 2, 67, 210, 3, 80, 239, + 7, 83, 6, 142, 195, 2, 69, 146, 184, 1, 79, 167, 214, 1, 80, 46, 80, 7, + 76, 69, 84, 84, 69, 82, 32, 208, 1, 5, 83, 73, 71, 78, 32, 147, 2, 86, + 38, 162, 1, 65, 242, 154, 2, 78, 250, 238, 3, 66, 2, 68, 2, 71, 2, 72, 2, + 75, 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, 89, 186, 2, 73, + 3, 85, 5, 197, 193, 5, 6, 82, 67, 72, 65, 73, 67, 4, 26, 80, 199, 181, 2, + 86, 2, 25, 4, 65, 77, 85, 68, 2, 227, 212, 2, 80, 36, 48, 7, 76, 69, 84, + 84, 69, 82, 32, 147, 1, 86, 32, 194, 153, 2, 78, 250, 238, 3, 66, 2, 68, + 2, 71, 2, 75, 2, 76, 2, 77, 2, 80, 2, 83, 2, 84, 2, 87, 2, 89, 186, 2, + 65, 2, 73, 3, 85, 4, 41, 8, 79, 87, 69, 76, 32, 83, 73, 71, 4, 11, 78, 4, + 243, 200, 5, 32, 212, 3, 112, 10, 76, 69, 32, 76, 69, 84, 84, 69, 82, 32, + 248, 2, 5, 84, 72, 65, 77, 32, 189, 18, 5, 86, 73, 69, 84, 32, 70, 186, + 1, 65, 34, 69, 30, 84, 250, 148, 2, 78, 210, 171, 1, 79, 162, 254, 1, 75, + 2, 80, 162, 7, 85, 234, 61, 70, 2, 72, 2, 76, 2, 77, 2, 81, 2, 83, 2, 86, + 2, 88, 2, 89, 187, 2, 73, 7, 142, 240, 5, 85, 215, 22, 73, 7, 194, 134, + 6, 69, 3, 72, 18, 60, 3, 79, 78, 69, 170, 190, 5, 83, 138, 69, 72, 187, + 2, 65, 10, 11, 45, 10, 218, 133, 6, 50, 2, 51, 2, 52, 2, 53, 3, 54, 254, + 1, 196, 1, 2, 67, 79, 240, 3, 4, 72, 79, 82, 65, 0, 4, 84, 72, 65, 77, + 28, 7, 76, 69, 84, 84, 69, 82, 32, 220, 4, 5, 83, 73, 71, 78, 32, 213, 5, + 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 20, 164, 1, 13, 78, 83, + 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 161, 241, 4, 21, 77, 66, 73, + 78, 73, 78, 71, 32, 67, 82, 89, 80, 84, 79, 71, 82, 65, 77, 77, 73, 67, + 18, 148, 1, 6, 70, 73, 78, 65, 76, 32, 22, 76, 56, 16, 72, 73, 71, 72, + 32, 82, 65, 84, 72, 65, 32, 79, 82, 32, 76, 79, 22, 77, 134, 254, 5, 66, + 3, 83, 2, 215, 190, 5, 78, 4, 54, 79, 213, 149, 1, 7, 65, 32, 84, 65, 78, + 71, 32, 2, 187, 214, 1, 87, 6, 48, 6, 69, 68, 73, 65, 76, 32, 139, 128, + 6, 65, 4, 206, 253, 5, 76, 3, 82, 20, 201, 142, 4, 2, 32, 68, 106, 188, + 1, 2, 71, 82, 44, 5, 72, 73, 71, 72, 32, 94, 76, 222, 1, 82, 134, 168, 2, + 85, 206, 201, 1, 73, 178, 15, 78, 186, 219, 1, 79, 162, 8, 69, 158, 20, + 66, 2, 68, 2, 77, 2, 87, 187, 2, 65, 2, 21, 3, 69, 65, 84, 2, 179, 251, + 5, 32, 32, 242, 1, 75, 42, 82, 186, 179, 5, 83, 82, 67, 2, 80, 2, 84, + 138, 69, 70, 2, 72, 3, 89, 36, 60, 3, 79, 87, 32, 170, 188, 5, 65, 206, + 41, 85, 159, 20, 76, 28, 86, 75, 42, 82, 138, 180, 5, 67, 2, 80, 2, 84, + 138, 69, 70, 2, 72, 2, 83, 3, 89, 6, 182, 249, 5, 72, 2, 88, 187, 2, 65, + 2, 133, 129, 4, 2, 65, 84, 8, 26, 65, 191, 228, 5, 85, 7, 214, 248, 5, + 78, 3, 84, 50, 178, 1, 72, 34, 75, 176, 1, 4, 77, 65, 73, 32, 82, 82, + 136, 1, 2, 83, 65, 92, 5, 87, 73, 65, 78, 71, 188, 126, 3, 68, 79, 75, + 224, 117, 3, 84, 79, 78, 205, 168, 3, 2, 67, 65, 4, 198, 168, 5, 65, 179, + 63, 79, 14, 52, 4, 72, 85, 69, 78, 134, 3, 65, 195, 219, 4, 69, 8, 80, 6, + 32, 84, 79, 78, 69, 45, 149, 174, 4, 8, 45, 76, 85, 69, 32, 75, 65, 82, + 6, 146, 248, 5, 51, 2, 52, 3, 53, 8, 56, 4, 75, 65, 78, 71, 134, 135, 1, + 89, 183, 181, 3, 83, 5, 183, 140, 1, 32, 4, 92, 3, 65, 32, 72, 21, 16, + 69, 86, 69, 82, 83, 69, 68, 32, 82, 79, 84, 65, 84, 69, 68, 32, 2, 195, + 187, 4, 65, 2, 155, 198, 3, 82, 8, 48, 3, 84, 75, 65, 230, 171, 4, 87, + 203, 121, 75, 4, 17, 2, 65, 78, 5, 199, 131, 4, 75, 5, 245, 191, 3, 2, + 87, 65, 38, 90, 65, 36, 4, 77, 65, 73, 32, 22, 79, 46, 84, 86, 85, 178, + 233, 3, 73, 223, 137, 2, 69, 9, 194, 244, 5, 65, 2, 69, 3, 73, 2, 183, + 224, 4, 83, 11, 210, 213, 3, 65, 186, 158, 2, 79, 3, 89, 4, 42, 65, 197, + 136, 1, 4, 72, 65, 77, 32, 2, 17, 2, 76, 76, 2, 211, 128, 4, 32, 9, 234, + 178, 5, 85, 163, 64, 69, 144, 1, 184, 1, 7, 76, 69, 84, 84, 69, 82, 32, + 188, 2, 5, 77, 65, 73, 32, 75, 32, 7, 83, 89, 77, 66, 79, 76, 32, 116, 9, + 84, 79, 78, 69, 32, 77, 65, 73, 32, 81, 6, 86, 79, 87, 69, 76, 32, 96, + 44, 4, 72, 73, 71, 72, 1, 3, 76, 79, 87, 48, 11, 32, 48, 154, 1, 75, 30, + 67, 2, 80, 2, 84, 34, 78, 166, 208, 5, 66, 2, 68, 2, 70, 2, 71, 2, 72, 2, + 76, 2, 77, 2, 82, 2, 83, 2, 86, 2, 89, 247, 30, 79, 6, 26, 72, 187, 239, + 5, 79, 4, 194, 208, 5, 72, 247, 30, 79, 6, 162, 208, 5, 71, 2, 89, 247, + 30, 79, 4, 246, 157, 5, 65, 195, 52, 72, 10, 68, 2, 75, 79, 106, 78, 240, + 126, 4, 72, 79, 32, 72, 183, 179, 3, 83, 4, 212, 127, 3, 73, 32, 75, 179, + 238, 4, 78, 8, 58, 78, 254, 132, 1, 84, 230, 164, 2, 83, 171, 192, 2, 69, + 2, 143, 170, 3, 85, 26, 50, 65, 58, 85, 30, 73, 134, 236, 5, 69, 3, 79, + 10, 130, 214, 5, 85, 214, 22, 65, 2, 77, 2, 78, 3, 89, 9, 26, 69, 135, + 236, 5, 65, 5, 131, 236, 5, 65, 138, 1, 52, 3, 82, 73, 32, 237, 222, 4, + 4, 69, 79, 85, 84, 136, 1, 82, 65, 20, 7, 76, 69, 84, 84, 69, 82, 32, + 166, 2, 83, 78, 86, 187, 246, 3, 68, 2, 195, 211, 1, 66, 88, 210, 1, 65, + 250, 142, 2, 68, 82, 82, 34, 84, 230, 5, 85, 206, 201, 1, 73, 162, 193, + 1, 78, 126, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 2, 83, 138, 69, 72, 2, + 76, 2, 77, 2, 86, 2, 89, 186, 2, 69, 3, 79, 11, 172, 185, 3, 7, 82, 67, + 72, 65, 73, 67, 32, 198, 175, 2, 65, 2, 73, 3, 85, 8, 25, 4, 73, 71, 78, + 32, 8, 166, 212, 1, 78, 234, 206, 3, 65, 239, 1, 86, 18, 49, 10, 79, 87, + 69, 76, 32, 83, 73, 71, 78, 32, 18, 214, 147, 2, 65, 38, 85, 206, 201, 1, + 73, 222, 137, 2, 69, 3, 79, 4, 154, 51, 70, 155, 154, 4, 79, 188, 6, 36, + 3, 73, 76, 32, 239, 193, 4, 65, 186, 6, 154, 3, 65, 106, 67, 194, 2, 68, + 72, 3, 87, 69, 84, 48, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 232, 7, 9, + 73, 78, 32, 80, 79, 83, 83, 69, 83, 22, 76, 200, 2, 7, 78, 85, 77, 66, + 69, 82, 32, 152, 1, 18, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, + 69, 78, 68, 32, 79, 70, 54, 82, 42, 83, 166, 14, 84, 228, 1, 7, 86, 79, + 87, 69, 76, 32, 83, 100, 2, 89, 69, 176, 186, 4, 5, 77, 79, 78, 84, 72, + 171, 118, 79, 6, 80, 5, 83, 32, 65, 66, 79, 242, 132, 2, 85, 217, 244, 1, + 5, 78, 68, 32, 79, 68, 2, 243, 197, 3, 86, 52, 80, 9, 79, 78, 83, 79, 78, + 65, 78, 84, 32, 156, 21, 3, 85, 82, 82, 171, 9, 82, 48, 130, 1, 75, 22, + 76, 22, 78, 46, 84, 218, 191, 1, 83, 194, 77, 82, 202, 210, 3, 67, 2, 72, + 2, 74, 2, 77, 2, 80, 2, 86, 3, 89, 5, 215, 162, 5, 83, 7, 243, 227, 4, + 76, 11, 134, 229, 3, 78, 134, 251, 1, 71, 3, 89, 5, 223, 223, 5, 84, 26, + 68, 2, 82, 89, 164, 28, 2, 69, 66, 238, 143, 2, 65, 227, 193, 1, 73, 2, + 137, 200, 1, 7, 32, 67, 85, 76, 84, 73, 86, 42, 168, 1, 4, 79, 78, 69, + 32, 220, 4, 6, 84, 72, 82, 69, 69, 32, 177, 215, 5, 22, 68, 79, 87, 78, + 83, 67, 65, 76, 73, 78, 71, 32, 70, 65, 67, 84, 79, 82, 32, 75, 73, 73, + 30, 112, 5, 69, 73, 71, 72, 84, 34, 70, 40, 3, 72, 65, 76, 22, 79, 88, 4, + 83, 73, 88, 84, 74, 84, 187, 215, 3, 81, 4, 210, 3, 73, 227, 216, 5, 72, + 4, 156, 3, 2, 79, 82, 187, 213, 3, 73, 4, 215, 218, 1, 70, 2, 225, 2, 18, + 78, 69, 45, 72, 85, 78, 68, 82, 69, 68, 45, 65, 78, 68, 45, 83, 73, 88, + 6, 232, 217, 1, 5, 69, 69, 78, 84, 72, 177, 24, 5, 89, 45, 70, 79, 85, 8, + 38, 72, 138, 1, 87, 239, 214, 3, 69, 4, 132, 1, 18, 82, 69, 69, 45, 72, + 85, 78, 68, 82, 69, 68, 45, 65, 78, 68, 45, 84, 87, 201, 156, 4, 8, 73, + 82, 84, 89, 45, 83, 69, 67, 2, 17, 2, 69, 78, 2, 17, 2, 84, 73, 2, 207, + 214, 3, 69, 10, 58, 69, 28, 4, 83, 73, 88, 84, 82, 84, 163, 214, 3, 81, + 2, 129, 1, 3, 73, 71, 72, 4, 50, 69, 149, 215, 3, 6, 89, 45, 70, 79, 85, + 82, 2, 145, 215, 3, 2, 69, 78, 2, 21, 3, 87, 69, 78, 2, 221, 214, 3, 3, + 84, 73, 69, 2, 171, 214, 1, 83, 72, 48, 6, 69, 84, 84, 69, 82, 32, 195, + 237, 3, 65, 70, 194, 1, 78, 158, 201, 1, 84, 166, 49, 65, 82, 76, 38, 82, + 134, 6, 85, 202, 141, 1, 79, 134, 60, 73, 206, 193, 1, 83, 242, 7, 69, + 234, 61, 67, 2, 72, 2, 74, 2, 75, 2, 77, 2, 80, 2, 86, 3, 89, 10, 46, 78, + 250, 209, 5, 71, 2, 89, 187, 2, 65, 4, 246, 209, 5, 78, 187, 2, 65, 8, + 38, 79, 130, 128, 4, 84, 131, 77, 83, 4, 11, 78, 4, 17, 2, 69, 32, 4, 18, + 72, 31, 84, 2, 217, 71, 3, 85, 78, 68, 2, 149, 132, 2, 3, 72, 79, 85, 2, + 17, 2, 32, 84, 2, 11, 69, 2, 171, 182, 5, 88, 2, 17, 2, 85, 80, 2, 131, + 182, 3, 69, 194, 4, 152, 1, 5, 65, 76, 84, 32, 80, 20, 4, 73, 71, 78, 32, + 206, 4, 80, 28, 9, 84, 65, 82, 84, 73, 78, 71, 32, 70, 41, 8, 89, 76, 76, + 65, 66, 76, 69, 32, 2, 219, 215, 3, 65, 40, 90, 65, 48, 3, 67, 69, 86, + 34, 75, 80, 2, 77, 85, 102, 85, 14, 80, 118, 86, 167, 64, 78, 4, 216, 2, + 4, 65, 90, 72, 65, 199, 136, 5, 78, 2, 11, 73, 2, 231, 170, 5, 84, 6, 38, + 85, 173, 170, 5, 3, 65, 65, 67, 4, 18, 90, 87, 82, 2, 167, 187, 5, 72, 6, + 60, 4, 75, 75, 85, 82, 16, 2, 84, 72, 21, 3, 85, 86, 85, 2, 167, 82, 85, + 2, 251, 149, 3, 65, 2, 75, 90, 8, 26, 65, 131, 254, 4, 79, 6, 34, 84, + 182, 146, 4, 65, 15, 78, 2, 11, 72, 2, 17, 2, 65, 75, 2, 179, 168, 5, 75, + 10, 38, 65, 214, 81, 69, 143, 184, 4, 73, 4, 180, 102, 3, 82, 65, 65, + 221, 243, 2, 6, 75, 65, 73, 89, 65, 82, 2, 11, 69, 2, 187, 9, 78, 2, 17, + 2, 82, 79, 2, 235, 196, 4, 77, 148, 4, 130, 1, 75, 166, 1, 76, 166, 1, + 78, 186, 1, 82, 86, 83, 178, 1, 84, 214, 2, 67, 2, 72, 2, 74, 2, 77, 2, + 80, 2, 86, 3, 89, 46, 84, 2, 83, 83, 174, 246, 1, 65, 38, 85, 202, 141, + 1, 79, 134, 60, 73, 191, 201, 1, 69, 24, 214, 239, 1, 65, 250, 6, 85, + 202, 141, 1, 79, 134, 60, 73, 191, 201, 1, 69, 66, 78, 76, 146, 245, 1, + 65, 38, 85, 202, 141, 1, 79, 134, 60, 73, 191, 201, 1, 69, 44, 226, 6, + 76, 174, 238, 1, 65, 38, 85, 202, 141, 1, 79, 134, 60, 73, 191, 201, 1, + 69, 110, 98, 78, 174, 5, 71, 2, 89, 174, 238, 1, 65, 38, 85, 202, 141, 1, + 79, 134, 60, 73, 191, 201, 1, 69, 44, 170, 5, 78, 174, 238, 1, 65, 38, + 85, 202, 141, 1, 79, 134, 60, 73, 191, 201, 1, 69, 44, 214, 4, 82, 174, + 238, 1, 65, 38, 85, 202, 141, 1, 79, 134, 60, 73, 191, 201, 1, 69, 68, + 90, 72, 170, 3, 83, 174, 238, 1, 65, 38, 85, 202, 141, 1, 79, 134, 60, + 73, 191, 201, 1, 69, 24, 210, 241, 1, 65, 38, 85, 202, 141, 1, 79, 230, + 2, 82, 162, 57, 73, 191, 201, 1, 69, 44, 210, 2, 84, 174, 238, 1, 65, 38, + 85, 202, 141, 1, 79, 134, 60, 73, 191, 201, 1, 69, 6, 68, 2, 79, 84, 33, + 11, 82, 65, 68, 73, 84, 73, 79, 78, 65, 76, 32, 2, 11, 65, 2, 163, 188, + 4, 76, 4, 24, 2, 67, 82, 55, 78, 2, 17, 2, 69, 68, 2, 11, 73, 2, 211, + 187, 4, 84, 2, 11, 85, 2, 17, 2, 77, 66, 2, 123, 69, 22, 25, 4, 73, 71, + 78, 32, 22, 170, 238, 1, 65, 38, 85, 202, 141, 1, 79, 134, 60, 73, 191, + 201, 1, 69, 2, 11, 65, 2, 163, 186, 4, 82, 184, 13, 36, 5, 65, 66, 65, + 84, 65, 35, 71, 2, 11, 32, 2, 147, 209, 3, 84, 182, 13, 54, 69, 20, 3, + 83, 65, 32, 217, 6, 3, 85, 84, 32, 2, 207, 166, 4, 82, 178, 1, 52, 7, 76, + 69, 84, 84, 69, 82, 32, 155, 206, 3, 68, 158, 1, 210, 1, 65, 58, 70, 54, + 72, 38, 76, 58, 77, 54, 78, 50, 83, 126, 85, 114, 69, 2, 73, 2, 79, 2, + 86, 186, 241, 4, 84, 82, 67, 2, 68, 2, 71, 2, 75, 2, 80, 138, 69, 66, 2, + 82, 2, 87, 2, 88, 2, 89, 3, 90, 16, 150, 4, 87, 202, 185, 5, 67, 2, 81, + 2, 88, 3, 90, 4, 172, 236, 4, 5, 73, 78, 65, 76, 32, 251, 80, 65, 6, 174, + 177, 1, 84, 199, 139, 4, 65, 4, 204, 150, 4, 5, 79, 78, 71, 32, 85, 131, + 166, 1, 65, 10, 150, 188, 5, 65, 2, 67, 2, 81, 2, 88, 3, 90, 8, 170, 185, + 5, 71, 2, 72, 2, 89, 187, 2, 65, 8, 26, 72, 155, 187, 5, 65, 6, 40, 4, + 79, 82, 84, 32, 239, 186, 5, 65, 4, 188, 151, 1, 2, 85, 69, 137, 151, 3, + 2, 65, 87, 32, 58, 73, 54, 69, 202, 185, 5, 67, 2, 81, 2, 88, 3, 90, 16, + 50, 85, 202, 185, 5, 67, 2, 81, 2, 88, 3, 90, 8, 198, 185, 5, 67, 2, 81, 2, 88, 3, 90, 130, 12, 64, 10, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, - 143, 241, 2, 73, 128, 12, 70, 48, 178, 1, 49, 2, 50, 2, 51, 2, 52, 2, 53, + 143, 150, 3, 73, 128, 12, 70, 48, 178, 1, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 95, 55, 198, 1, 86, 48, 242, 1, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, - 54, 2, 55, 2, 56, 3, 57, 18, 130, 140, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, + 54, 2, 55, 2, 56, 3, 57, 18, 170, 183, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 200, 1, 150, 1, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 138, 1, 58, 48, 2, 49, 2, 50, 2, - 51, 2, 52, 2, 53, 95, 54, 20, 146, 138, 5, 48, 2, 49, 2, 50, 2, 51, 2, - 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 182, 137, 5, 48, 2, 49, 2, 50, + 51, 2, 52, 2, 53, 95, 54, 20, 186, 181, 5, 48, 2, 49, 2, 50, 2, 51, 2, + 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 222, 180, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 4, 48, 6, 67, 65, 82, 84, 82, - 73, 21, 2, 68, 82, 2, 187, 212, 3, 68, 2, 215, 139, 4, 73, 2, 155, 133, - 3, 82, 142, 3, 154, 1, 65, 152, 2, 5, 68, 68, 89, 32, 66, 22, 76, 234, + 73, 21, 2, 68, 82, 2, 183, 254, 3, 68, 2, 239, 182, 4, 73, 2, 179, 173, + 3, 82, 142, 3, 154, 1, 65, 152, 2, 5, 68, 68, 89, 32, 66, 22, 76, 166, 14, 78, 144, 1, 5, 83, 84, 32, 84, 85, 21, 12, 84, 82, 65, 71, 82, 65, 77, 32, 70, 79, 82, 32, 10, 68, 9, 67, 85, 80, 32, 87, 73, 84, 72, 79, - 58, 82, 171, 181, 4, 80, 2, 33, 6, 85, 84, 32, 72, 65, 78, 2, 195, 228, - 3, 68, 6, 72, 9, 45, 79, 70, 70, 32, 67, 65, 76, 69, 29, 5, 68, 82, 79, - 80, 45, 2, 221, 226, 3, 2, 78, 68, 4, 234, 243, 2, 83, 245, 112, 4, 66, - 65, 82, 66, 2, 147, 226, 3, 69, 216, 1, 38, 69, 225, 2, 4, 85, 71, 85, - 32, 16, 52, 6, 80, 72, 79, 78, 69, 32, 246, 1, 83, 35, 86, 12, 132, 1, 3, - 82, 69, 67, 252, 151, 1, 3, 76, 79, 67, 242, 227, 2, 83, 165, 116, 13, - 79, 78, 32, 84, 79, 80, 32, 79, 70, 32, 77, 79, 68, 6, 36, 5, 69, 73, 86, - 69, 82, 51, 79, 5, 29, 5, 32, 87, 73, 84, 72, 2, 159, 39, 32, 2, 243, - 221, 2, 82, 2, 11, 67, 2, 211, 188, 3, 79, 2, 247, 231, 2, 73, 200, 1, - 162, 1, 65, 40, 15, 70, 82, 65, 67, 84, 73, 79, 78, 32, 68, 73, 71, 73, - 84, 32, 164, 2, 2, 76, 69, 156, 4, 5, 83, 73, 71, 78, 32, 198, 2, 86, - 175, 136, 3, 68, 2, 11, 73, 2, 245, 189, 4, 2, 32, 76, 14, 74, 84, 40, 2, - 79, 78, 81, 10, 90, 69, 82, 79, 32, 70, 79, 82, 32, 79, 8, 36, 3, 72, 82, - 69, 13, 2, 87, 79, 4, 11, 69, 4, 29, 5, 32, 70, 79, 82, 32, 4, 34, 79, - 21, 4, 69, 86, 69, 78, 2, 17, 2, 68, 68, 2, 49, 10, 32, 80, 79, 87, 69, - 82, 83, 32, 79, 70, 2, 201, 22, 2, 32, 70, 114, 44, 5, 84, 84, 69, 82, - 32, 159, 187, 4, 78, 112, 214, 1, 68, 54, 76, 38, 78, 106, 82, 34, 84, - 242, 144, 1, 65, 126, 86, 186, 24, 85, 158, 144, 1, 79, 182, 56, 73, 202, - 190, 1, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 162, 7, 69, 222, - 61, 72, 2, 77, 3, 89, 10, 234, 179, 4, 68, 254, 68, 72, 2, 90, 187, 2, - 65, 6, 198, 244, 2, 76, 167, 134, 2, 65, 10, 42, 65, 230, 247, 4, 71, 2, - 78, 3, 89, 5, 41, 8, 75, 65, 65, 82, 65, 32, 80, 79, 2, 163, 37, 76, 6, - 194, 89, 82, 159, 160, 4, 65, 10, 138, 178, 4, 84, 254, 68, 72, 2, 83, - 187, 2, 65, 20, 86, 67, 194, 1, 83, 248, 35, 3, 84, 85, 85, 142, 108, 65, - 170, 1, 78, 203, 162, 3, 86, 6, 60, 9, 79, 77, 66, 73, 78, 73, 78, 71, - 32, 219, 145, 1, 65, 4, 70, 65, 185, 250, 3, 11, 67, 65, 78, 68, 82, 65, - 66, 73, 78, 68, 85, 2, 33, 6, 78, 85, 83, 86, 65, 82, 2, 147, 250, 3, 65, - 2, 11, 73, 2, 169, 189, 3, 3, 68, 68, 72, 30, 49, 10, 79, 87, 69, 76, 32, - 83, 73, 71, 78, 32, 30, 186, 145, 1, 65, 106, 86, 214, 20, 85, 158, 144, - 1, 79, 182, 56, 73, 187, 198, 1, 69, 6, 80, 10, 78, 73, 83, 32, 82, 65, - 67, 81, 85, 69, 150, 221, 2, 71, 219, 151, 2, 84, 2, 11, 84, 2, 25, 4, - 32, 65, 78, 68, 2, 251, 239, 2, 32, 2, 219, 221, 4, 66, 162, 1, 210, 2, - 65, 134, 1, 66, 174, 1, 67, 130, 3, 68, 198, 3, 69, 154, 2, 70, 182, 1, - 71, 198, 1, 72, 64, 8, 89, 79, 85, 84, 72, 70, 85, 76, 52, 2, 73, 78, 46, - 75, 98, 76, 146, 1, 77, 134, 1, 79, 62, 80, 142, 1, 82, 210, 1, 83, 224, - 1, 14, 86, 65, 83, 84, 78, 69, 83, 83, 32, 79, 82, 32, 87, 65, 12, 2, 87, - 65, 202, 94, 85, 143, 234, 3, 74, 8, 88, 4, 67, 67, 85, 77, 22, 83, 132, - 21, 2, 68, 86, 157, 175, 1, 5, 71, 71, 82, 65, 86, 2, 183, 176, 1, 85, 2, - 183, 173, 4, 67, 6, 92, 7, 79, 76, 68, 32, 82, 69, 83, 32, 5, 82, 65, 78, - 67, 72, 221, 230, 3, 3, 65, 82, 82, 2, 153, 159, 4, 3, 79, 76, 85, 2, 11, - 73, 2, 133, 157, 1, 3, 78, 71, 32, 22, 54, 72, 20, 3, 76, 79, 83, 102, - 79, 211, 248, 1, 69, 2, 219, 186, 3, 65, 6, 26, 69, 143, 206, 3, 85, 4, - 26, 68, 207, 215, 3, 78, 2, 21, 3, 32, 77, 79, 2, 243, 238, 2, 85, 12, - 28, 3, 77, 80, 76, 35, 78, 4, 242, 17, 73, 143, 139, 4, 69, 8, 32, 4, 83, - 84, 65, 78, 23, 84, 2, 183, 219, 4, 67, 6, 46, 69, 20, 3, 82, 65, 82, - 211, 221, 3, 65, 2, 247, 155, 4, 78, 2, 171, 129, 3, 73, 20, 80, 4, 65, - 82, 75, 69, 22, 69, 206, 1, 73, 74, 85, 237, 204, 4, 3, 79, 85, 66, 2, - 199, 154, 4, 78, 6, 132, 1, 19, 70, 69, 67, 84, 73, 86, 69, 78, 69, 83, - 83, 32, 79, 82, 32, 68, 73, 83, 84, 34, 80, 133, 7, 6, 67, 73, 83, 73, - 86, 69, 2, 11, 79, 2, 211, 153, 4, 82, 2, 17, 2, 65, 82, 2, 143, 10, 84, - 8, 68, 6, 70, 70, 73, 67, 85, 76, 34, 77, 161, 12, 4, 86, 69, 82, 71, 2, - 11, 84, 2, 171, 213, 3, 73, 4, 140, 1, 2, 73, 78, 243, 150, 4, 77, 14, - 100, 5, 77, 66, 69, 76, 76, 34, 78, 124, 4, 88, 72, 65, 85, 236, 106, 3, - 84, 69, 82, 239, 242, 2, 65, 2, 189, 164, 4, 3, 73, 83, 72, 6, 80, 4, 68, - 69, 65, 86, 20, 4, 76, 65, 82, 71, 129, 233, 2, 4, 67, 79, 85, 78, 2, - 175, 238, 3, 79, 2, 183, 163, 4, 69, 2, 135, 150, 4, 83, 12, 70, 79, 76, - 3, 85, 76, 76, 196, 5, 3, 65, 73, 76, 147, 181, 4, 76, 4, 18, 76, 35, 83, - 2, 225, 148, 4, 3, 76, 79, 87, 2, 205, 4, 2, 84, 69, 4, 142, 150, 3, 32, - 211, 56, 78, 10, 96, 8, 65, 84, 72, 69, 82, 73, 78, 71, 22, 79, 64, 3, - 82, 69, 65, 65, 5, 85, 65, 82, 68, 69, 5, 227, 228, 3, 32, 2, 41, 8, 73, - 78, 71, 32, 84, 79, 32, 77, 2, 211, 174, 4, 69, 2, 75, 84, 4, 48, 2, 65, - 82, 33, 6, 79, 76, 68, 73, 78, 71, 2, 11, 68, 2, 187, 204, 3, 78, 2, 11, - 32, 2, 195, 216, 1, 66, 4, 26, 67, 211, 164, 4, 78, 2, 143, 5, 82, 4, 60, - 7, 69, 69, 80, 73, 78, 71, 32, 225, 140, 4, 2, 73, 78, 2, 11, 83, 2, 151, - 221, 2, 77, 6, 18, 65, 107, 69, 4, 60, 3, 66, 79, 85, 21, 8, 87, 32, 79, - 82, 32, 77, 79, 68, 2, 243, 143, 4, 82, 2, 187, 215, 3, 69, 2, 139, 144, - 4, 71, 6, 26, 65, 30, 69, 47, 73, 2, 153, 143, 4, 2, 83, 83, 2, 11, 65, - 2, 11, 83, 2, 191, 191, 3, 85, 2, 11, 82, 2, 171, 153, 4, 69, 4, 204, - 162, 3, 7, 78, 32, 84, 72, 69, 32, 86, 139, 107, 80, 8, 54, 65, 64, 4, - 69, 78, 69, 84, 165, 98, 2, 85, 82, 4, 22, 84, 243, 2, 67, 2, 17, 2, 84, - 69, 2, 183, 142, 4, 82, 2, 195, 177, 1, 82, 12, 30, 69, 157, 1, 2, 73, - 84, 10, 34, 76, 22, 83, 231, 186, 4, 65, 2, 251, 204, 1, 69, 6, 18, 73, - 51, 80, 4, 30, 68, 137, 1, 2, 83, 84, 2, 147, 1, 69, 2, 11, 79, 2, 199, - 211, 3, 78, 2, 255, 210, 3, 85, 10, 34, 69, 64, 2, 73, 78, 23, 84, 2, 11, - 86, 2, 17, 2, 69, 82, 2, 11, 65, 2, 211, 158, 4, 78, 2, 183, 138, 4, 75, - 6, 42, 79, 237, 219, 2, 4, 82, 69, 78, 71, 4, 26, 80, 131, 196, 4, 86, 2, - 11, 80, 2, 211, 166, 3, 65, 2, 39, 83, 4, 26, 73, 231, 183, 4, 84, 2, - 147, 137, 4, 84, 224, 2, 86, 65, 180, 31, 2, 69, 82, 246, 1, 73, 206, 1, - 79, 100, 3, 82, 69, 69, 179, 8, 85, 146, 2, 44, 4, 65, 78, 65, 32, 145, - 10, 2, 73, 32, 100, 122, 65, 38, 69, 76, 7, 76, 69, 84, 84, 69, 82, 32, - 154, 7, 79, 76, 3, 73, 66, 73, 0, 3, 85, 66, 85, 45, 2, 83, 85, 4, 184, - 8, 3, 65, 66, 65, 3, 66, 6, 58, 66, 0, 3, 69, 66, 69, 245, 7, 4, 89, 66, - 69, 89, 2, 243, 7, 69, 78, 202, 1, 65, 42, 68, 82, 70, 2, 81, 14, 71, 50, - 72, 38, 75, 62, 76, 32, 3, 77, 69, 69, 20, 2, 67, 72, 2, 74, 2, 80, 18, - 78, 42, 83, 94, 84, 102, 86, 2, 87, 42, 90, 130, 225, 2, 66, 2, 82, 3, - 89, 4, 252, 1, 2, 76, 73, 191, 174, 4, 73, 6, 30, 65, 29, 3, 72, 65, 65, - 4, 218, 2, 65, 255, 1, 86, 2, 227, 175, 4, 76, 2, 123, 65, 6, 110, 65, - 86, 78, 245, 173, 4, 3, 72, 65, 73, 4, 178, 228, 2, 72, 147, 237, 1, 65, - 4, 26, 65, 247, 227, 2, 72, 2, 11, 65, 2, 199, 174, 4, 70, 4, 18, 65, 35, - 72, 2, 11, 65, 2, 147, 174, 4, 77, 2, 211, 1, 65, 4, 224, 173, 4, 2, 79, - 79, 191, 34, 65, 8, 32, 2, 65, 65, 18, 72, 23, 69, 2, 163, 16, 68, 4, 18, - 69, 87, 65, 2, 243, 172, 4, 69, 10, 62, 65, 16, 3, 72, 65, 65, 190, 225, - 2, 84, 203, 239, 1, 79, 2, 131, 1, 86, 5, 155, 172, 4, 76, 2, 17, 2, 65, - 65, 2, 243, 171, 4, 86, 6, 26, 65, 175, 208, 4, 79, 4, 26, 86, 147, 208, - 4, 65, 2, 21, 3, 73, 89, 65, 2, 159, 188, 4, 78, 6, 48, 3, 65, 66, 79, - 14, 66, 1, 3, 79, 66, 79, 2, 23, 65, 2, 11, 79, 2, 11, 70, 2, 11, 73, 2, - 167, 187, 4, 76, 2, 211, 206, 3, 75, 174, 1, 26, 67, 159, 224, 2, 68, - 154, 1, 132, 1, 9, 72, 65, 82, 65, 67, 84, 69, 82, 32, 245, 162, 4, 17, - 85, 82, 82, 69, 78, 67, 89, 32, 83, 89, 77, 66, 79, 76, 32, 66, 65, 152, - 1, 176, 2, 6, 66, 79, 32, 66, 65, 73, 16, 6, 67, 72, 79, 32, 67, 72, 56, - 3, 68, 79, 32, 44, 2, 70, 79, 76, 3, 72, 79, 32, 66, 75, 190, 1, 76, 138, - 1, 77, 252, 1, 7, 65, 78, 71, 75, 72, 65, 78, 46, 78, 158, 1, 80, 234, 1, - 82, 70, 83, 238, 2, 84, 224, 2, 3, 87, 79, 32, 34, 89, 145, 226, 3, 2, - 79, 32, 2, 143, 12, 77, 8, 204, 210, 2, 2, 65, 78, 170, 167, 1, 73, 155, - 58, 79, 4, 164, 165, 4, 3, 67, 72, 65, 227, 33, 68, 6, 32, 2, 32, 70, 21, - 2, 78, 71, 4, 219, 209, 2, 65, 2, 147, 130, 3, 77, 4, 40, 4, 78, 79, 75, - 72, 223, 199, 4, 72, 2, 239, 197, 4, 85, 14, 40, 2, 72, 79, 229, 9, 3, - 79, 32, 75, 12, 26, 32, 239, 206, 3, 77, 10, 36, 2, 75, 72, 57, 3, 82, - 65, 75, 8, 158, 9, 87, 150, 171, 3, 85, 254, 67, 79, 255, 59, 65, 2, 135, - 239, 3, 72, 8, 76, 2, 79, 32, 224, 11, 8, 65, 75, 75, 72, 65, 78, 71, 89, - 139, 187, 4, 85, 4, 32, 2, 67, 72, 207, 245, 3, 76, 2, 171, 164, 4, 85, - 16, 28, 2, 65, 73, 235, 52, 79, 14, 42, 32, 176, 1, 3, 84, 65, 73, 19, - 89, 10, 76, 4, 67, 72, 65, 84, 44, 5, 72, 65, 78, 45, 65, 22, 84, 191, - 193, 4, 69, 2, 11, 84, 2, 11, 65, 2, 195, 194, 4, 87, 2, 151, 177, 3, 75, - 4, 222, 165, 4, 72, 159, 11, 82, 2, 203, 2, 75, 2, 193, 154, 4, 2, 65, - 77, 8, 60, 3, 71, 79, 32, 32, 3, 73, 75, 72, 29, 3, 79, 32, 78, 2, 11, - 78, 2, 211, 158, 4, 71, 2, 237, 166, 4, 2, 65, 72, 4, 182, 243, 3, 69, - 215, 79, 85, 12, 68, 6, 65, 73, 89, 65, 78, 78, 22, 72, 249, 159, 4, 3, - 79, 32, 80, 2, 203, 174, 4, 79, 8, 36, 3, 73, 78, 84, 21, 2, 79, 32, 2, - 255, 156, 4, 72, 6, 44, 2, 80, 72, 161, 5, 4, 83, 65, 77, 80, 4, 182, - 240, 3, 85, 155, 1, 65, 4, 32, 2, 79, 32, 227, 192, 4, 85, 2, 11, 82, 2, - 155, 190, 4, 85, 36, 44, 4, 65, 82, 65, 32, 225, 1, 2, 79, 32, 28, 62, - 65, 130, 1, 85, 134, 184, 2, 73, 206, 134, 2, 69, 3, 79, 13, 64, 6, 73, - 32, 77, 65, 73, 77, 142, 191, 4, 65, 2, 69, 3, 77, 4, 26, 65, 191, 247, - 2, 85, 2, 11, 76, 2, 139, 171, 4, 65, 9, 186, 254, 3, 69, 151, 64, 85, 8, - 24, 2, 82, 85, 23, 83, 2, 187, 170, 4, 83, 6, 230, 155, 4, 65, 226, 31, - 85, 187, 2, 79, 18, 30, 72, 133, 2, 2, 79, 32, 14, 48, 6, 65, 78, 84, 72, - 65, 75, 21, 2, 79, 32, 2, 175, 169, 3, 72, 12, 80, 8, 78, 65, 78, 71, 77, - 79, 78, 84, 20, 4, 80, 72, 85, 84, 13, 2, 84, 72, 2, 163, 157, 4, 72, 2, - 123, 72, 8, 34, 65, 234, 234, 3, 79, 3, 85, 4, 158, 244, 2, 72, 183, 199, - 1, 78, 4, 38, 84, 165, 140, 2, 3, 80, 65, 84, 2, 147, 156, 4, 65, 2, 11, - 87, 2, 243, 233, 2, 65, 6, 30, 65, 45, 3, 79, 32, 89, 2, 21, 3, 77, 65, - 75, 2, 231, 242, 2, 75, 4, 154, 233, 3, 73, 215, 77, 65, 10, 30, 69, 145, - 1, 2, 77, 79, 6, 18, 32, 107, 70, 4, 84, 11, 68, 79, 69, 83, 32, 78, 79, - 84, 32, 69, 88, 201, 164, 3, 4, 69, 88, 73, 83, 2, 147, 113, 73, 2, 171, - 152, 3, 79, 4, 46, 77, 157, 145, 3, 5, 68, 89, 78, 65, 77, 2, 187, 129, - 3, 69, 10, 18, 78, 95, 82, 8, 26, 32, 191, 175, 3, 75, 6, 26, 83, 255, - 177, 2, 71, 4, 218, 165, 2, 65, 211, 212, 1, 80, 2, 21, 3, 68, 32, 80, 2, - 25, 4, 76, 65, 67, 69, 2, 17, 2, 32, 77, 2, 143, 163, 3, 69, 4, 56, 4, - 85, 71, 72, 84, 173, 162, 3, 4, 78, 71, 32, 83, 2, 185, 164, 1, 5, 32, - 66, 65, 76, 76, 46, 22, 32, 199, 3, 45, 30, 154, 1, 68, 90, 76, 114, 82, - 130, 131, 1, 66, 86, 67, 242, 5, 83, 170, 149, 1, 80, 253, 21, 15, 78, - 69, 84, 87, 79, 82, 75, 69, 68, 32, 67, 79, 77, 80, 85, 4, 64, 10, 73, - 77, 69, 78, 83, 73, 79, 78, 65, 76, 219, 134, 1, 79, 2, 131, 146, 3, 32, - 6, 76, 12, 73, 78, 69, 83, 32, 67, 79, 78, 86, 69, 82, 71, 237, 127, 2, - 69, 70, 4, 217, 135, 4, 3, 73, 78, 71, 10, 36, 4, 65, 89, 83, 32, 147, - 127, 73, 8, 230, 153, 2, 66, 222, 155, 1, 65, 242, 81, 76, 31, 82, 16, - 44, 2, 68, 32, 254, 3, 80, 131, 133, 1, 69, 12, 196, 2, 12, 84, 79, 80, - 45, 76, 73, 71, 72, 84, 69, 68, 32, 80, 17, 76, 69, 70, 84, 45, 76, 73, - 71, 72, 84, 69, 68, 32, 68, 79, 87, 78, 0, 16, 82, 73, 71, 72, 84, 45, - 76, 73, 71, 72, 84, 69, 68, 32, 85, 80, 209, 223, 1, 25, 66, 79, 84, 84, - 79, 77, 45, 76, 73, 71, 72, 84, 69, 68, 32, 82, 73, 71, 72, 84, 87, 65, - 82, 68, 83, 6, 76, 4, 76, 69, 70, 84, 69, 11, 82, 73, 71, 72, 84, 87, 65, - 82, 68, 83, 32, 2, 11, 87, 2, 25, 4, 65, 82, 68, 83, 2, 181, 200, 1, 2, - 32, 69, 4, 154, 200, 1, 69, 139, 23, 65, 2, 129, 239, 3, 5, 69, 82, 45, - 69, 77, 8, 34, 77, 85, 4, 78, 68, 69, 82, 4, 21, 3, 66, 83, 32, 4, 38, - 85, 149, 182, 2, 3, 68, 79, 87, 2, 199, 165, 3, 80, 4, 212, 128, 3, 10, - 32, 67, 76, 79, 85, 68, 32, 65, 78, 68, 141, 45, 2, 83, 84, 230, 5, 204, - 1, 6, 66, 69, 84, 65, 78, 32, 240, 45, 6, 69, 32, 79, 86, 69, 82, 72, 7, - 70, 73, 78, 65, 71, 72, 32, 150, 9, 71, 152, 1, 3, 76, 68, 69, 248, 2, 2, - 77, 69, 56, 2, 78, 89, 78, 82, 223, 154, 3, 67, 160, 3, 228, 2, 18, 65, - 83, 84, 82, 79, 76, 79, 71, 73, 67, 65, 76, 32, 83, 73, 71, 78, 32, 172, - 1, 18, 67, 65, 78, 84, 73, 76, 76, 65, 84, 73, 79, 78, 32, 83, 73, 71, - 78, 32, 208, 1, 6, 68, 73, 71, 73, 84, 32, 100, 9, 75, 85, 32, 82, 85, - 32, 75, 72, 65, 26, 76, 204, 3, 5, 77, 65, 82, 75, 32, 246, 18, 83, 181, - 15, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 6, 46, 83, 129, 41, - 6, 45, 75, 72, 89, 85, 68, 4, 104, 8, 68, 79, 78, 71, 32, 84, 83, 72, - 229, 21, 13, 71, 82, 65, 32, 71, 67, 65, 78, 32, 45, 67, 72, 65, 2, 159, - 32, 85, 8, 144, 1, 5, 72, 69, 65, 86, 89, 0, 5, 76, 73, 71, 72, 84, 40, - 7, 83, 66, 85, 66, 32, 45, 67, 189, 255, 3, 8, 67, 65, 78, 71, 32, 84, - 69, 45, 2, 17, 2, 32, 66, 2, 231, 144, 3, 69, 2, 251, 154, 3, 72, 40, - 156, 182, 2, 4, 72, 65, 76, 70, 82, 70, 30, 83, 42, 84, 62, 90, 130, 83, - 78, 14, 79, 199, 110, 69, 5, 245, 10, 2, 32, 66, 92, 96, 6, 69, 84, 84, - 69, 82, 32, 181, 2, 13, 79, 71, 79, 84, 89, 80, 69, 32, 83, 73, 71, 78, - 32, 88, 226, 1, 75, 50, 82, 208, 213, 3, 10, 70, 73, 88, 69, 68, 45, 70, - 79, 82, 77, 202, 1, 68, 86, 78, 46, 83, 38, 84, 46, 66, 2, 67, 2, 71, 2, - 80, 2, 90, 254, 68, 45, 2, 72, 2, 74, 2, 76, 2, 77, 2, 87, 2, 89, 187, 2, - 65, 8, 134, 158, 4, 83, 14, 72, 2, 75, 187, 2, 65, 4, 226, 157, 4, 82, - 187, 2, 65, 4, 248, 25, 3, 76, 72, 65, 13, 4, 67, 72, 65, 68, 80, 250, 2, - 66, 176, 1, 8, 77, 78, 89, 65, 77, 32, 89, 73, 114, 67, 172, 2, 13, 89, - 73, 71, 32, 77, 71, 79, 32, 84, 83, 72, 69, 71, 190, 1, 71, 212, 2, 9, - 65, 78, 71, 32, 75, 72, 65, 78, 71, 58, 72, 48, 2, 73, 78, 246, 1, 78, - 220, 1, 2, 80, 65, 30, 82, 118, 83, 46, 84, 36, 4, 76, 69, 65, 68, 233, - 250, 1, 17, 68, 69, 76, 73, 77, 73, 84, 69, 82, 32, 84, 83, 72, 69, 71, - 32, 66, 10, 52, 9, 75, 65, 45, 32, 83, 72, 79, 71, 32, 27, 83, 4, 142, 1, - 71, 67, 89, 6, 34, 75, 229, 21, 3, 68, 85, 83, 4, 56, 6, 65, 45, 32, 83, - 72, 79, 89, 4, 85, 82, 32, 89, 2, 21, 3, 71, 32, 71, 2, 41, 8, 73, 32, - 77, 71, 79, 32, 82, 71, 2, 163, 211, 2, 89, 2, 221, 2, 2, 73, 71, 12, 84, - 5, 65, 82, 69, 84, 32, 240, 1, 2, 72, 69, 29, 7, 76, 79, 83, 73, 78, 71, - 32, 6, 112, 12, 45, 68, 90, 85, 68, 32, 82, 84, 65, 71, 83, 32, 97, 12, - 89, 73, 71, 32, 77, 71, 79, 32, 80, 72, 85, 82, 4, 42, 66, 37, 6, 77, 69, - 32, 76, 79, 78, 2, 33, 6, 90, 72, 73, 32, 77, 73, 2, 231, 22, 71, 2, 225, - 3, 3, 32, 83, 72, 2, 137, 142, 2, 2, 32, 77, 4, 68, 13, 66, 82, 68, 65, - 32, 82, 78, 89, 73, 78, 71, 32, 89, 3, 89, 2, 213, 5, 11, 73, 71, 32, 77, - 71, 79, 32, 83, 71, 65, 66, 12, 68, 4, 84, 69, 82, 32, 141, 2, 8, 85, 71, - 32, 82, 84, 65, 71, 83, 8, 56, 8, 89, 73, 71, 32, 77, 71, 79, 32, 211, - 207, 3, 84, 6, 68, 4, 45, 85, 77, 32, 117, 9, 84, 82, 85, 78, 67, 65, 84, - 69, 68, 4, 88, 7, 82, 78, 65, 77, 32, 66, 67, 245, 2, 10, 71, 84, 69, 82, - 32, 84, 83, 72, 69, 71, 2, 241, 2, 2, 65, 68, 2, 203, 145, 4, 32, 4, 21, - 3, 32, 71, 89, 4, 130, 196, 3, 79, 135, 18, 65, 2, 17, 2, 65, 76, 2, 213, - 238, 3, 2, 65, 78, 6, 92, 6, 73, 84, 73, 65, 76, 32, 165, 204, 3, 11, 84, - 69, 82, 83, 89, 76, 76, 65, 66, 73, 67, 4, 68, 13, 66, 82, 68, 65, 32, - 82, 78, 89, 73, 78, 71, 32, 89, 3, 89, 2, 53, 11, 73, 71, 32, 77, 71, 79, - 32, 77, 68, 85, 78, 2, 151, 240, 3, 32, 10, 72, 10, 71, 65, 83, 32, 66, - 90, 85, 78, 71, 32, 77, 4, 89, 73, 83, 32, 4, 56, 3, 83, 71, 79, 245, - 237, 3, 5, 78, 89, 73, 32, 90, 2, 151, 10, 82, 6, 44, 5, 84, 83, 72, 69, - 71, 163, 201, 3, 83, 5, 147, 201, 3, 32, 2, 233, 234, 3, 2, 76, 85, 4, - 212, 200, 3, 8, 71, 89, 65, 32, 71, 82, 65, 77, 1, 14, 73, 78, 32, 67, - 72, 69, 78, 32, 83, 80, 85, 78, 71, 83, 4, 224, 199, 3, 4, 66, 82, 85, + 58, 82, 199, 224, 4, 80, 2, 33, 6, 85, 84, 32, 72, 65, 78, 2, 231, 140, + 4, 68, 6, 72, 9, 45, 79, 70, 70, 32, 67, 65, 76, 69, 29, 5, 68, 82, 79, + 80, 45, 2, 245, 142, 4, 2, 78, 68, 4, 250, 155, 3, 83, 245, 115, 4, 66, + 65, 82, 66, 2, 171, 142, 4, 69, 216, 1, 38, 69, 213, 2, 4, 85, 71, 85, + 32, 16, 60, 6, 80, 72, 79, 78, 69, 32, 246, 1, 83, 151, 179, 2, 86, 12, + 132, 1, 3, 82, 69, 67, 228, 150, 1, 3, 76, 79, 67, 146, 144, 3, 83, 189, + 116, 13, 79, 78, 32, 84, 79, 80, 32, 79, 70, 32, 77, 79, 68, 6, 36, 5, + 69, 73, 86, 69, 82, 51, 79, 5, 29, 5, 32, 87, 73, 84, 72, 2, 179, 38, 32, + 2, 255, 140, 3, 82, 2, 11, 67, 2, 167, 229, 3, 79, 200, 1, 162, 1, 65, + 20, 15, 70, 82, 65, 67, 84, 73, 79, 78, 32, 68, 73, 71, 73, 84, 32, 164, + 2, 2, 76, 69, 252, 3, 5, 83, 73, 71, 78, 32, 198, 2, 86, 147, 177, 3, 68, + 2, 247, 205, 1, 73, 14, 74, 84, 40, 2, 79, 78, 81, 10, 90, 69, 82, 79, + 32, 70, 79, 82, 32, 79, 8, 36, 3, 72, 82, 69, 13, 2, 87, 79, 4, 11, 69, + 4, 29, 5, 32, 70, 79, 82, 32, 4, 34, 79, 21, 4, 69, 86, 69, 78, 2, 17, 2, + 68, 68, 2, 49, 10, 32, 80, 79, 87, 69, 82, 83, 32, 79, 70, 2, 133, 22, 2, + 32, 70, 114, 44, 5, 84, 84, 69, 82, 32, 219, 230, 4, 78, 112, 214, 1, 68, + 54, 78, 106, 82, 38, 84, 138, 203, 1, 65, 82, 76, 114, 86, 186, 5, 85, + 202, 141, 1, 79, 134, 60, 73, 206, 193, 1, 83, 82, 66, 2, 67, 2, 71, 2, + 74, 2, 75, 2, 80, 162, 7, 69, 234, 61, 72, 2, 77, 3, 89, 10, 166, 223, 4, + 68, 138, 69, 72, 2, 90, 187, 2, 65, 10, 42, 65, 210, 163, 5, 71, 2, 78, + 3, 89, 5, 41, 8, 75, 65, 65, 82, 65, 32, 80, 79, 2, 131, 37, 76, 6, 158, + 204, 1, 82, 175, 217, 3, 65, 10, 230, 221, 4, 84, 138, 69, 72, 2, 83, + 187, 2, 65, 20, 86, 67, 194, 1, 83, 212, 35, 3, 84, 85, 85, 170, 107, 78, + 242, 60, 65, 231, 147, 3, 86, 6, 60, 9, 79, 77, 66, 73, 78, 73, 78, 71, + 32, 171, 143, 1, 65, 4, 70, 65, 145, 166, 4, 11, 67, 65, 78, 68, 82, 65, + 66, 73, 78, 68, 85, 2, 33, 6, 78, 85, 83, 86, 65, 82, 2, 235, 165, 4, 65, + 2, 11, 73, 2, 209, 231, 3, 3, 68, 68, 72, 30, 49, 10, 79, 87, 69, 76, 32, + 83, 73, 71, 78, 32, 30, 174, 206, 1, 65, 38, 85, 22, 86, 182, 141, 1, 79, + 134, 60, 73, 191, 201, 1, 69, 6, 80, 10, 78, 73, 83, 32, 82, 65, 67, 81, + 85, 69, 174, 132, 3, 71, 175, 156, 2, 84, 2, 11, 84, 2, 25, 4, 32, 65, + 78, 68, 2, 215, 152, 3, 32, 2, 199, 137, 5, 66, 162, 1, 210, 2, 65, 134, + 1, 66, 174, 1, 67, 222, 2, 68, 202, 3, 69, 154, 2, 70, 182, 1, 71, 198, + 1, 72, 64, 8, 89, 79, 85, 84, 72, 70, 85, 76, 52, 2, 73, 78, 46, 75, 98, + 76, 146, 1, 77, 134, 1, 79, 62, 80, 142, 1, 82, 210, 1, 83, 224, 1, 14, + 86, 65, 83, 84, 78, 69, 83, 83, 32, 79, 82, 32, 87, 65, 12, 2, 87, 65, + 154, 94, 85, 203, 150, 4, 74, 8, 88, 4, 67, 67, 85, 77, 22, 83, 228, 20, + 2, 68, 86, 229, 197, 1, 5, 71, 71, 82, 65, 86, 2, 239, 185, 1, 85, 2, + 151, 217, 4, 67, 6, 92, 7, 79, 76, 68, 32, 82, 69, 83, 32, 5, 82, 65, 78, + 67, 72, 177, 146, 4, 3, 65, 82, 82, 2, 249, 202, 4, 3, 79, 76, 85, 2, 11, + 73, 2, 213, 166, 1, 3, 78, 71, 32, 22, 54, 72, 20, 3, 76, 79, 83, 66, 79, + 175, 152, 2, 69, 2, 155, 229, 3, 65, 6, 26, 69, 227, 249, 3, 85, 4, 230, + 248, 1, 68, 215, 138, 2, 78, 12, 28, 3, 77, 80, 76, 35, 78, 4, 246, 17, + 73, 143, 183, 4, 69, 8, 32, 4, 83, 84, 65, 78, 23, 84, 2, 199, 135, 5, + 67, 6, 46, 69, 20, 3, 82, 65, 82, 203, 137, 4, 65, 2, 251, 199, 4, 78, 2, + 183, 170, 3, 73, 20, 80, 4, 65, 82, 75, 69, 22, 69, 210, 1, 73, 74, 85, + 249, 248, 4, 3, 79, 85, 66, 2, 203, 198, 4, 78, 6, 132, 1, 19, 70, 69, + 67, 84, 73, 86, 69, 78, 69, 83, 83, 32, 79, 82, 32, 68, 73, 83, 84, 34, + 80, 137, 7, 6, 67, 73, 83, 73, 86, 69, 2, 11, 79, 2, 215, 197, 4, 82, 2, + 11, 65, 2, 11, 82, 2, 143, 10, 84, 8, 68, 6, 70, 70, 73, 67, 85, 76, 34, + 77, 161, 12, 4, 86, 69, 82, 71, 2, 11, 84, 2, 159, 129, 4, 73, 4, 140, 1, + 2, 73, 78, 243, 194, 4, 77, 14, 100, 5, 77, 66, 69, 76, 76, 34, 78, 124, + 4, 88, 72, 65, 85, 188, 106, 3, 84, 69, 82, 147, 159, 3, 65, 2, 189, 208, + 4, 3, 73, 83, 72, 6, 80, 4, 68, 69, 65, 86, 20, 4, 76, 65, 82, 71, 129, + 146, 3, 4, 67, 79, 85, 78, 2, 171, 154, 4, 79, 2, 183, 207, 4, 69, 2, + 135, 194, 4, 83, 12, 70, 79, 76, 3, 85, 76, 76, 196, 5, 3, 65, 73, 76, + 159, 225, 4, 76, 4, 18, 76, 35, 83, 2, 225, 192, 4, 3, 76, 79, 87, 2, + 205, 4, 2, 84, 69, 4, 166, 191, 3, 32, 175, 59, 78, 10, 96, 8, 65, 84, + 72, 69, 82, 73, 78, 71, 22, 79, 64, 3, 82, 69, 65, 65, 5, 85, 65, 82, 68, + 69, 5, 215, 144, 4, 32, 2, 41, 8, 73, 78, 71, 32, 84, 79, 32, 77, 2, 211, + 218, 4, 69, 2, 75, 84, 4, 48, 2, 65, 82, 33, 6, 79, 76, 68, 73, 78, 71, + 2, 11, 68, 2, 175, 248, 3, 78, 2, 11, 32, 2, 235, 243, 1, 66, 4, 26, 67, + 211, 208, 4, 78, 2, 143, 5, 82, 4, 60, 7, 69, 69, 80, 73, 78, 71, 32, + 221, 184, 4, 2, 73, 78, 2, 11, 83, 2, 147, 134, 3, 77, 6, 18, 65, 107, + 69, 4, 60, 3, 66, 79, 85, 21, 8, 87, 32, 79, 82, 32, 77, 79, 68, 2, 243, + 187, 4, 82, 2, 175, 131, 4, 69, 2, 139, 188, 4, 71, 6, 26, 65, 30, 69, + 47, 73, 2, 153, 187, 4, 2, 83, 83, 2, 11, 65, 2, 11, 83, 2, 179, 235, 3, + 85, 2, 11, 82, 2, 171, 197, 4, 69, 4, 164, 203, 3, 7, 78, 32, 84, 72, 69, + 32, 86, 179, 110, 80, 8, 54, 65, 64, 4, 69, 78, 69, 84, 245, 97, 2, 85, + 82, 4, 22, 84, 243, 2, 67, 2, 17, 2, 84, 69, 2, 183, 186, 4, 82, 2, 139, + 200, 1, 82, 12, 30, 69, 157, 1, 2, 73, 84, 10, 34, 76, 22, 83, 243, 230, + 4, 65, 2, 227, 227, 1, 69, 6, 18, 73, 51, 80, 4, 30, 68, 137, 1, 2, 83, + 84, 2, 147, 1, 69, 2, 11, 79, 2, 187, 255, 3, 78, 2, 243, 254, 3, 85, 10, + 34, 69, 64, 2, 73, 78, 23, 84, 2, 11, 86, 2, 17, 2, 69, 82, 2, 11, 65, 2, + 211, 202, 4, 78, 2, 183, 182, 4, 75, 6, 42, 79, 237, 132, 3, 4, 82, 69, + 78, 71, 4, 26, 80, 143, 240, 4, 86, 2, 11, 80, 2, 179, 209, 3, 65, 2, 39, + 83, 4, 26, 73, 243, 227, 4, 84, 2, 147, 181, 4, 84, 224, 2, 86, 65, 180, + 31, 2, 69, 82, 246, 1, 73, 206, 1, 79, 100, 3, 82, 69, 69, 187, 8, 85, + 146, 2, 44, 4, 65, 78, 65, 32, 145, 10, 2, 73, 32, 100, 122, 65, 38, 69, + 76, 7, 76, 69, 84, 84, 69, 82, 32, 154, 7, 79, 76, 3, 73, 66, 73, 0, 3, + 85, 66, 85, 45, 2, 83, 85, 4, 184, 8, 3, 65, 66, 65, 3, 66, 6, 58, 66, 0, + 3, 69, 66, 69, 245, 7, 4, 89, 66, 69, 89, 2, 243, 7, 69, 78, 202, 1, 65, + 42, 68, 82, 70, 2, 81, 14, 71, 50, 72, 38, 75, 62, 76, 32, 3, 77, 69, 69, + 20, 2, 67, 72, 2, 74, 2, 80, 18, 78, 42, 83, 94, 84, 102, 86, 2, 87, 42, + 90, 138, 138, 3, 66, 2, 82, 3, 89, 4, 252, 1, 2, 76, 73, 203, 218, 4, 73, + 6, 30, 65, 29, 3, 72, 65, 65, 4, 218, 2, 65, 255, 1, 86, 2, 239, 219, 4, + 76, 2, 123, 65, 6, 110, 65, 86, 78, 129, 218, 4, 3, 72, 65, 73, 4, 186, + 141, 3, 72, 151, 240, 1, 65, 4, 26, 65, 255, 140, 3, 72, 2, 11, 65, 2, + 211, 218, 4, 70, 4, 18, 65, 35, 72, 2, 11, 65, 2, 159, 218, 4, 77, 2, + 211, 1, 65, 4, 236, 217, 4, 2, 79, 79, 191, 34, 65, 8, 32, 2, 65, 65, 18, + 72, 23, 69, 2, 163, 16, 68, 4, 18, 69, 87, 65, 2, 255, 216, 4, 69, 10, + 62, 65, 16, 3, 72, 65, 65, 198, 138, 3, 84, 207, 242, 1, 79, 2, 131, 1, + 86, 5, 167, 216, 4, 76, 2, 17, 2, 65, 65, 2, 255, 215, 4, 86, 6, 26, 65, + 187, 252, 4, 79, 4, 26, 86, 159, 252, 4, 65, 2, 21, 3, 73, 89, 65, 2, + 171, 232, 4, 78, 6, 48, 3, 65, 66, 79, 14, 66, 1, 3, 79, 66, 79, 2, 23, + 65, 2, 11, 79, 2, 11, 70, 2, 11, 73, 2, 179, 231, 4, 76, 2, 199, 250, 3, + 75, 174, 1, 26, 67, 167, 137, 3, 68, 154, 1, 132, 1, 9, 72, 65, 82, 65, + 67, 84, 69, 82, 32, 129, 207, 4, 17, 85, 82, 82, 69, 78, 67, 89, 32, 83, + 89, 77, 66, 79, 76, 32, 66, 65, 152, 1, 176, 2, 6, 66, 79, 32, 66, 65, + 73, 16, 6, 67, 72, 79, 32, 67, 72, 56, 3, 68, 79, 32, 44, 2, 70, 79, 76, + 3, 72, 79, 32, 66, 75, 190, 1, 76, 138, 1, 77, 252, 1, 7, 65, 78, 71, 75, + 72, 65, 78, 46, 78, 158, 1, 80, 234, 1, 82, 70, 83, 238, 2, 84, 224, 2, + 3, 87, 79, 32, 34, 89, 141, 142, 4, 2, 79, 32, 2, 143, 12, 77, 8, 212, + 251, 2, 2, 65, 78, 162, 170, 1, 73, 167, 58, 79, 4, 176, 209, 4, 3, 67, + 72, 65, 227, 33, 68, 6, 32, 2, 32, 70, 21, 2, 78, 71, 4, 227, 250, 2, 65, + 2, 211, 171, 3, 77, 4, 40, 4, 78, 79, 75, 72, 235, 243, 4, 72, 2, 251, + 241, 4, 85, 14, 40, 2, 72, 79, 229, 9, 3, 79, 32, 75, 12, 26, 32, 235, + 250, 3, 77, 10, 36, 2, 75, 72, 57, 3, 82, 65, 75, 8, 158, 9, 87, 138, + 215, 3, 85, 138, 68, 79, 139, 60, 65, 2, 131, 155, 4, 72, 8, 76, 2, 79, + 32, 224, 11, 8, 65, 75, 75, 72, 65, 78, 71, 89, 151, 231, 4, 85, 4, 32, + 2, 67, 72, 207, 161, 4, 76, 2, 183, 208, 4, 85, 16, 28, 2, 65, 73, 219, + 52, 79, 14, 42, 32, 176, 1, 3, 84, 65, 73, 19, 89, 10, 76, 4, 67, 72, 65, + 84, 44, 5, 72, 65, 78, 45, 65, 22, 84, 203, 237, 4, 69, 2, 11, 84, 2, 11, + 65, 2, 207, 238, 4, 87, 2, 139, 221, 3, 75, 4, 234, 209, 4, 72, 159, 11, + 82, 2, 203, 2, 75, 2, 205, 198, 4, 2, 65, 77, 8, 60, 3, 71, 79, 32, 32, + 3, 73, 75, 72, 29, 3, 79, 32, 78, 2, 11, 78, 2, 223, 202, 4, 71, 2, 249, + 210, 4, 2, 65, 72, 4, 182, 159, 4, 69, 227, 79, 85, 12, 68, 6, 65, 73, + 89, 65, 78, 78, 22, 72, 133, 204, 4, 3, 79, 32, 80, 2, 215, 218, 4, 79, + 8, 36, 3, 73, 78, 84, 21, 2, 79, 32, 2, 139, 201, 4, 72, 6, 44, 2, 80, + 72, 161, 5, 4, 83, 65, 77, 80, 4, 182, 156, 4, 85, 155, 1, 65, 4, 32, 2, + 79, 32, 239, 236, 4, 85, 2, 11, 82, 2, 167, 234, 4, 85, 36, 44, 4, 65, + 82, 65, 32, 225, 1, 2, 79, 32, 28, 62, 65, 130, 1, 85, 130, 225, 2, 73, + 222, 137, 2, 69, 3, 79, 13, 64, 6, 73, 32, 77, 65, 73, 77, 154, 235, 4, + 65, 2, 69, 3, 77, 4, 26, 65, 255, 160, 3, 85, 2, 11, 76, 2, 151, 215, 4, + 65, 9, 186, 170, 4, 69, 163, 64, 85, 8, 24, 2, 82, 85, 23, 83, 2, 199, + 214, 4, 83, 6, 242, 199, 4, 65, 226, 31, 85, 187, 2, 79, 18, 30, 72, 133, + 2, 2, 79, 32, 14, 48, 6, 65, 78, 84, 72, 65, 75, 21, 2, 79, 32, 2, 163, + 213, 3, 72, 12, 80, 8, 78, 65, 78, 71, 77, 79, 78, 84, 20, 4, 80, 72, 85, + 84, 13, 2, 84, 72, 2, 175, 201, 4, 72, 2, 123, 72, 8, 34, 65, 234, 150, + 4, 79, 3, 85, 4, 222, 157, 3, 72, 131, 202, 1, 78, 4, 38, 84, 209, 177, + 2, 3, 80, 65, 84, 2, 159, 200, 4, 65, 2, 11, 87, 2, 139, 147, 3, 65, 6, + 30, 65, 45, 3, 79, 32, 89, 2, 21, 3, 77, 65, 75, 2, 167, 156, 3, 75, 4, + 154, 149, 4, 73, 227, 77, 65, 10, 30, 69, 145, 1, 2, 77, 79, 6, 18, 32, + 107, 70, 4, 84, 11, 68, 79, 69, 83, 32, 78, 79, 84, 32, 69, 88, 189, 208, + 3, 4, 69, 88, 73, 83, 2, 235, 122, 73, 2, 159, 196, 3, 79, 4, 46, 77, + 141, 188, 3, 5, 68, 89, 78, 65, 77, 2, 155, 172, 3, 69, 10, 18, 78, 95, + 82, 8, 26, 32, 179, 219, 3, 75, 6, 26, 83, 251, 218, 2, 71, 4, 206, 206, + 2, 65, 223, 215, 1, 80, 2, 21, 3, 68, 32, 80, 2, 25, 4, 76, 65, 67, 69, + 2, 17, 2, 32, 77, 2, 131, 207, 3, 69, 4, 56, 4, 85, 71, 72, 84, 161, 206, + 3, 4, 78, 71, 32, 83, 2, 161, 187, 1, 5, 32, 66, 65, 76, 76, 46, 22, 32, + 207, 3, 45, 30, 154, 1, 68, 90, 76, 118, 82, 198, 153, 1, 66, 86, 67, + 194, 6, 83, 134, 167, 1, 80, 137, 22, 15, 78, 69, 84, 87, 79, 82, 75, 69, + 68, 32, 67, 79, 77, 80, 85, 4, 64, 10, 73, 77, 69, 78, 83, 73, 79, 78, + 65, 76, 163, 157, 1, 79, 2, 139, 187, 3, 32, 6, 80, 12, 73, 78, 69, 83, + 32, 67, 79, 78, 86, 69, 82, 71, 149, 150, 1, 2, 69, 70, 4, 225, 179, 4, + 3, 73, 78, 71, 10, 40, 4, 65, 89, 83, 32, 183, 149, 1, 73, 8, 146, 192, + 2, 66, 166, 161, 1, 65, 130, 82, 76, 31, 82, 16, 44, 2, 68, 32, 254, 3, + 80, 147, 156, 1, 69, 12, 196, 2, 12, 84, 79, 80, 45, 76, 73, 71, 72, 84, + 69, 68, 32, 80, 17, 76, 69, 70, 84, 45, 76, 73, 71, 72, 84, 69, 68, 32, + 68, 79, 87, 78, 0, 16, 82, 73, 71, 72, 84, 45, 76, 73, 71, 72, 84, 69, + 68, 32, 85, 80, 245, 132, 2, 25, 66, 79, 84, 84, 79, 77, 45, 76, 73, 71, + 72, 84, 69, 68, 32, 82, 73, 71, 72, 84, 87, 65, 82, 68, 83, 6, 76, 4, 76, + 69, 70, 84, 69, 11, 82, 73, 71, 72, 84, 87, 65, 82, 68, 83, 32, 2, 11, + 87, 2, 25, 4, 65, 82, 68, 83, 2, 197, 238, 1, 2, 32, 69, 4, 170, 238, 1, + 69, 159, 22, 65, 2, 249, 154, 4, 5, 69, 82, 45, 69, 77, 8, 34, 77, 85, 4, + 78, 68, 69, 82, 4, 21, 3, 66, 83, 32, 4, 38, 85, 149, 223, 2, 3, 68, 79, + 87, 2, 179, 209, 3, 80, 4, 180, 171, 3, 10, 32, 67, 76, 79, 85, 68, 32, + 65, 78, 68, 161, 46, 2, 83, 84, 230, 5, 204, 1, 6, 66, 69, 84, 65, 78, + 32, 184, 45, 6, 69, 32, 79, 86, 69, 82, 72, 7, 70, 73, 78, 65, 71, 72, + 32, 150, 9, 71, 152, 1, 3, 76, 68, 69, 252, 2, 2, 77, 69, 56, 2, 78, 89, + 78, 82, 139, 199, 3, 67, 160, 3, 228, 2, 18, 65, 83, 84, 82, 79, 76, 79, + 71, 73, 67, 65, 76, 32, 83, 73, 71, 78, 32, 172, 1, 18, 67, 65, 78, 84, + 73, 76, 76, 65, 84, 73, 79, 78, 32, 83, 73, 71, 78, 32, 208, 1, 6, 68, + 73, 71, 73, 84, 32, 100, 9, 75, 85, 32, 82, 85, 32, 75, 72, 65, 26, 76, + 176, 3, 5, 77, 65, 82, 75, 32, 222, 18, 83, 181, 15, 11, 86, 79, 87, 69, + 76, 32, 83, 73, 71, 78, 32, 6, 46, 83, 205, 40, 6, 45, 75, 72, 89, 85, + 68, 4, 104, 8, 68, 79, 78, 71, 32, 84, 83, 72, 205, 21, 13, 71, 82, 65, + 32, 71, 67, 65, 78, 32, 45, 67, 72, 65, 2, 235, 31, 85, 8, 144, 1, 5, 72, + 69, 65, 86, 89, 0, 5, 76, 73, 71, 72, 84, 40, 7, 83, 66, 85, 66, 32, 45, + 67, 193, 171, 4, 8, 67, 65, 78, 71, 32, 84, 69, 45, 2, 17, 2, 32, 66, 2, + 211, 188, 3, 69, 2, 231, 198, 3, 72, 40, 156, 223, 2, 4, 72, 65, 76, 70, + 82, 70, 30, 83, 42, 84, 62, 90, 238, 85, 78, 14, 79, 223, 110, 69, 5, + 221, 10, 2, 32, 66, 92, 96, 6, 69, 84, 84, 69, 82, 32, 153, 2, 13, 79, + 71, 79, 84, 89, 80, 69, 32, 83, 73, 71, 78, 32, 88, 230, 1, 75, 162, 115, + 82, 212, 142, 3, 10, 70, 73, 88, 69, 68, 45, 70, 79, 82, 77, 202, 1, 68, + 86, 78, 46, 83, 38, 84, 46, 66, 2, 67, 2, 71, 2, 80, 2, 90, 138, 69, 45, + 2, 72, 2, 74, 2, 76, 2, 77, 2, 87, 2, 89, 187, 2, 65, 8, 134, 202, 4, 83, + 14, 72, 2, 75, 187, 2, 65, 4, 224, 25, 3, 76, 72, 65, 13, 4, 67, 72, 65, + 68, 80, 254, 2, 66, 176, 1, 8, 77, 78, 89, 65, 77, 32, 89, 73, 114, 67, + 172, 2, 13, 89, 73, 71, 32, 77, 71, 79, 32, 84, 83, 72, 69, 71, 190, 1, + 71, 212, 2, 9, 65, 78, 71, 32, 75, 72, 65, 78, 71, 58, 72, 48, 2, 73, 78, + 246, 1, 78, 222, 1, 82, 118, 83, 46, 84, 36, 4, 76, 69, 65, 68, 212, 100, + 2, 80, 65, 181, 191, 1, 17, 68, 69, 76, 73, 77, 73, 84, 69, 82, 32, 84, + 83, 72, 69, 71, 32, 66, 10, 52, 9, 75, 65, 45, 32, 83, 72, 79, 71, 32, + 27, 83, 4, 142, 1, 71, 67, 89, 6, 34, 75, 201, 21, 3, 68, 85, 83, 4, 56, + 6, 65, 45, 32, 83, 72, 79, 89, 4, 85, 82, 32, 89, 2, 21, 3, 71, 32, 71, + 2, 41, 8, 73, 32, 77, 71, 79, 32, 82, 71, 2, 243, 252, 2, 89, 2, 221, 2, + 2, 73, 71, 12, 84, 5, 65, 82, 69, 84, 32, 240, 1, 2, 72, 69, 29, 7, 76, + 79, 83, 73, 78, 71, 32, 6, 112, 12, 45, 68, 90, 85, 68, 32, 82, 84, 65, + 71, 83, 32, 97, 12, 89, 73, 71, 32, 77, 71, 79, 32, 80, 72, 85, 82, 4, + 42, 66, 37, 6, 77, 69, 32, 76, 79, 78, 2, 33, 6, 90, 72, 73, 32, 77, 73, + 2, 203, 22, 71, 2, 225, 3, 3, 32, 83, 72, 2, 149, 183, 2, 2, 32, 77, 4, + 68, 13, 66, 82, 68, 65, 32, 82, 78, 89, 73, 78, 71, 32, 89, 3, 89, 2, + 213, 5, 11, 73, 71, 32, 77, 71, 79, 32, 83, 71, 65, 66, 12, 68, 4, 84, + 69, 82, 32, 141, 2, 8, 85, 71, 32, 82, 84, 65, 71, 83, 8, 56, 8, 89, 73, + 71, 32, 77, 71, 79, 32, 227, 251, 3, 84, 6, 68, 4, 45, 85, 77, 32, 117, + 9, 84, 82, 85, 78, 67, 65, 84, 69, 68, 4, 88, 7, 82, 78, 65, 77, 32, 66, + 67, 245, 2, 10, 71, 84, 69, 82, 32, 84, 83, 72, 69, 71, 2, 241, 2, 2, 65, + 68, 2, 231, 189, 4, 32, 4, 21, 3, 32, 71, 89, 4, 146, 240, 3, 79, 135, + 18, 65, 2, 17, 2, 65, 76, 2, 241, 154, 4, 2, 65, 78, 6, 92, 6, 73, 84, + 73, 65, 76, 32, 181, 248, 3, 11, 84, 69, 82, 83, 89, 76, 76, 65, 66, 73, + 67, 4, 68, 13, 66, 82, 68, 65, 32, 82, 78, 89, 73, 78, 71, 32, 89, 3, 89, + 2, 53, 11, 73, 71, 32, 77, 71, 79, 32, 77, 68, 85, 78, 2, 179, 156, 4, + 32, 10, 72, 10, 71, 65, 83, 32, 66, 90, 85, 78, 71, 32, 77, 4, 89, 73, + 83, 32, 4, 56, 3, 83, 71, 79, 145, 154, 4, 5, 78, 89, 73, 32, 90, 2, 251, + 9, 82, 6, 44, 5, 84, 83, 72, 69, 71, 179, 245, 3, 83, 5, 163, 245, 3, 32, + 4, 128, 245, 3, 8, 71, 89, 65, 32, 71, 82, 65, 77, 1, 14, 73, 78, 32, 67, + 72, 69, 78, 32, 83, 80, 85, 78, 71, 83, 4, 140, 244, 3, 4, 66, 82, 85, 76, 39, 72, 6, 32, 4, 82, 65, 73, 76, 55, 83, 2, 225, 7, 9, 73, 78, 71, - 32, 77, 67, 72, 65, 78, 4, 56, 5, 65, 32, 45, 80, 72, 165, 198, 3, 3, 72, - 69, 71, 2, 255, 231, 3, 82, 156, 1, 84, 4, 73, 71, 78, 32, 228, 6, 9, 85, + 32, 77, 67, 72, 65, 78, 4, 56, 5, 65, 32, 45, 80, 72, 209, 242, 3, 3, 72, + 69, 71, 2, 183, 148, 4, 82, 156, 1, 84, 4, 73, 71, 78, 32, 228, 6, 9, 85, 66, 74, 79, 73, 78, 69, 68, 32, 143, 4, 89, 42, 176, 1, 4, 71, 82, 85, 32, 96, 2, 76, 67, 30, 77, 36, 11, 78, 89, 73, 32, 90, 76, 65, 32, 78, 65, 65, 22, 82, 252, 2, 2, 89, 65, 130, 4, 73, 165, 4, 5, 83, 78, 65, 32, 76, 4, 40, 3, 67, 65, 78, 1, 3, 77, 69, 68, 2, 25, 4, 32, 82, 71, 89, 2, 169, 4, 2, 73, 78, 4, 238, 3, 73, 179, 4, 69, 4, 136, 4, 2, 65, 82, 231, - 3, 67, 2, 159, 228, 3, 32, 20, 116, 4, 68, 69, 76, 32, 240, 1, 10, 74, - 69, 83, 32, 83, 85, 32, 78, 71, 65, 137, 192, 3, 6, 78, 65, 77, 32, 66, - 67, 16, 52, 5, 68, 75, 65, 82, 32, 53, 4, 78, 65, 71, 32, 8, 94, 71, 129, - 161, 2, 6, 82, 68, 69, 76, 32, 78, 8, 42, 71, 69, 6, 82, 68, 69, 76, 32, - 68, 6, 46, 67, 228, 233, 1, 2, 78, 89, 251, 116, 83, 2, 251, 133, 4, 73, - 2, 147, 228, 2, 75, 2, 147, 154, 2, 32, 4, 18, 78, 71, 82, 2, 11, 71, 2, - 21, 3, 32, 82, 84, 2, 11, 65, 2, 255, 199, 3, 71, 2, 17, 2, 32, 84, 2, - 187, 241, 2, 83, 94, 68, 7, 76, 69, 84, 84, 69, 82, 32, 145, 2, 5, 83, + 3, 67, 2, 215, 144, 4, 32, 20, 116, 4, 68, 69, 76, 32, 240, 1, 10, 74, + 69, 83, 32, 83, 85, 32, 78, 71, 65, 181, 236, 3, 6, 78, 65, 77, 32, 66, + 67, 16, 52, 5, 68, 75, 65, 82, 32, 53, 4, 78, 65, 71, 32, 8, 94, 71, 181, + 202, 2, 6, 82, 68, 69, 76, 32, 78, 8, 42, 71, 69, 6, 82, 68, 69, 76, 32, + 68, 6, 46, 67, 212, 143, 2, 2, 78, 89, 167, 122, 83, 2, 179, 178, 4, 73, + 2, 187, 145, 3, 75, 2, 199, 195, 2, 32, 4, 18, 78, 71, 82, 2, 11, 71, 2, + 21, 3, 32, 82, 84, 2, 11, 65, 2, 171, 244, 3, 71, 2, 17, 2, 32, 84, 2, + 219, 157, 3, 83, 94, 68, 7, 76, 69, 84, 84, 69, 82, 32, 145, 2, 5, 83, 73, 71, 78, 32, 88, 220, 1, 10, 70, 73, 88, 69, 68, 45, 70, 79, 82, 77, - 234, 185, 3, 68, 50, 75, 38, 78, 46, 83, 38, 84, 46, 66, 2, 67, 2, 71, 2, - 80, 2, 90, 254, 68, 45, 2, 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, 87, 2, 89, - 187, 2, 65, 6, 11, 32, 6, 166, 128, 4, 82, 2, 87, 3, 89, 6, 38, 73, 50, + 150, 230, 3, 68, 50, 75, 38, 78, 46, 83, 38, 84, 46, 66, 2, 67, 2, 71, 2, + 80, 2, 90, 138, 69, 45, 2, 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, 87, 2, 89, + 187, 2, 65, 6, 11, 32, 6, 222, 172, 4, 82, 2, 87, 3, 89, 6, 38, 73, 50, 77, 33, 3, 76, 67, 69, 2, 45, 9, 78, 86, 69, 82, 84, 69, 68, 32, 77, 2, - 11, 67, 2, 45, 2, 72, 85, 2, 25, 4, 32, 84, 83, 65, 2, 11, 32, 2, 231, - 185, 2, 67, 20, 60, 6, 76, 76, 65, 66, 76, 69, 21, 5, 77, 66, 79, 76, 32, - 2, 227, 239, 3, 32, 18, 104, 4, 68, 82, 73, 76, 32, 6, 78, 79, 82, 32, - 66, 85, 130, 1, 80, 93, 7, 82, 68, 79, 32, 82, 74, 69, 2, 11, 32, 2, 211, - 218, 3, 66, 9, 11, 32, 6, 72, 4, 66, 90, 72, 73, 0, 4, 71, 83, 85, 77, 1, - 4, 78, 89, 73, 83, 2, 217, 209, 1, 5, 32, 45, 75, 72, 89, 4, 52, 6, 65, - 68, 77, 65, 32, 71, 21, 3, 72, 85, 82, 2, 187, 182, 2, 68, 2, 251, 88, - 32, 5, 245, 195, 2, 6, 32, 82, 71, 89, 65, 32, 30, 120, 8, 82, 69, 86, - 69, 82, 83, 69, 68, 182, 24, 86, 214, 20, 85, 158, 144, 1, 79, 182, 56, - 73, 186, 198, 1, 69, 223, 61, 65, 4, 213, 245, 1, 2, 32, 73, 2, 25, 4, - 32, 73, 78, 70, 2, 11, 73, 2, 11, 78, 2, 223, 145, 2, 73, 118, 216, 1, 9, - 67, 79, 78, 83, 79, 78, 65, 78, 84, 20, 7, 76, 69, 84, 84, 69, 82, 32, - 236, 6, 20, 77, 79, 68, 73, 70, 73, 69, 82, 32, 76, 69, 84, 84, 69, 82, - 32, 76, 65, 66, 73, 37, 8, 83, 69, 80, 65, 82, 65, 84, 79, 2, 211, 187, - 3, 32, 112, 106, 65, 108, 17, 66, 69, 82, 66, 69, 82, 32, 65, 67, 65, 68, + 11, 67, 2, 45, 2, 72, 85, 2, 25, 4, 32, 84, 83, 65, 2, 11, 32, 2, 211, + 227, 2, 67, 20, 60, 6, 76, 76, 65, 66, 76, 69, 21, 5, 77, 66, 79, 76, 32, + 2, 155, 156, 4, 32, 18, 104, 4, 68, 82, 73, 76, 32, 6, 78, 79, 82, 32, + 66, 85, 130, 1, 80, 93, 7, 82, 68, 79, 32, 82, 74, 69, 2, 11, 32, 2, 139, + 135, 4, 66, 9, 11, 32, 6, 72, 4, 66, 90, 72, 73, 0, 4, 71, 83, 85, 77, 1, + 4, 78, 89, 73, 83, 2, 177, 247, 1, 5, 32, 45, 75, 72, 89, 4, 52, 6, 65, + 68, 77, 65, 32, 71, 21, 3, 72, 85, 82, 2, 167, 224, 2, 68, 2, 191, 112, + 32, 5, 237, 238, 2, 6, 32, 82, 71, 89, 65, 32, 30, 116, 8, 82, 69, 86, + 69, 82, 83, 69, 68, 186, 85, 85, 22, 86, 182, 141, 1, 79, 134, 60, 73, + 190, 201, 1, 69, 235, 61, 65, 4, 129, 159, 2, 2, 32, 73, 2, 25, 4, 32, + 73, 78, 70, 2, 11, 73, 2, 11, 78, 2, 151, 187, 2, 73, 118, 216, 1, 9, 67, + 79, 78, 83, 79, 78, 65, 78, 84, 20, 7, 76, 69, 84, 84, 69, 82, 32, 236, + 6, 20, 77, 79, 68, 73, 70, 73, 69, 82, 32, 76, 69, 84, 84, 69, 82, 32, + 76, 65, 66, 73, 37, 8, 83, 69, 80, 65, 82, 65, 84, 79, 2, 131, 232, 3, + 32, 112, 106, 65, 108, 17, 66, 69, 82, 66, 69, 82, 32, 65, 67, 65, 68, 69, 77, 89, 32, 89, 65, 30, 84, 223, 1, 89, 4, 84, 6, 89, 69, 82, 32, 89, - 65, 241, 246, 3, 9, 72, 65, 71, 71, 65, 82, 32, 89, 65, 2, 251, 246, 3, - 71, 4, 246, 247, 3, 72, 3, 74, 18, 92, 11, 65, 87, 69, 76, 76, 69, 77, - 69, 84, 32, 89, 33, 8, 85, 65, 82, 69, 71, 32, 89, 65, 2, 11, 65, 2, 239, - 246, 3, 90, 16, 62, 71, 190, 2, 75, 210, 242, 3, 90, 62, 78, 86, 72, 3, - 81, 4, 154, 246, 3, 72, 3, 78, 86, 54, 65, 210, 2, 69, 250, 242, 3, 73, + 65, 173, 163, 4, 9, 72, 65, 71, 71, 65, 82, 32, 89, 65, 2, 183, 163, 4, + 71, 4, 178, 164, 4, 72, 3, 74, 18, 92, 11, 65, 87, 69, 76, 76, 69, 77, + 69, 84, 32, 89, 33, 8, 85, 65, 82, 69, 71, 32, 89, 65, 2, 11, 65, 2, 171, + 163, 4, 90, 16, 62, 71, 190, 2, 75, 142, 159, 4, 90, 62, 78, 86, 72, 3, + 81, 4, 214, 162, 4, 72, 3, 78, 86, 54, 65, 210, 2, 69, 182, 159, 4, 73, 2, 79, 3, 85, 77, 190, 1, 68, 30, 71, 2, 75, 14, 66, 2, 72, 22, 83, 30, - 84, 30, 90, 242, 15, 82, 150, 226, 3, 67, 146, 1, 65, 2, 70, 2, 74, 2, - 76, 2, 77, 2, 78, 2, 80, 2, 81, 2, 86, 2, 87, 3, 89, 9, 38, 68, 227, 243, - 3, 72, 7, 11, 72, 5, 223, 243, 3, 72, 7, 202, 243, 3, 72, 3, 83, 7, 174, - 243, 3, 72, 3, 84, 7, 146, 243, 3, 72, 3, 90, 5, 247, 242, 3, 89, 2, 189, - 137, 3, 4, 65, 76, 73, 90, 2, 135, 177, 3, 82, 6, 76, 2, 69, 82, 21, 13, - 72, 84, 32, 84, 82, 73, 70, 79, 76, 73, 65, 84, 69, 5, 211, 180, 3, 32, - 2, 33, 6, 32, 83, 78, 79, 87, 70, 2, 167, 43, 76, 19, 11, 32, 16, 72, 8, - 79, 80, 69, 82, 65, 84, 79, 82, 229, 1, 5, 87, 73, 84, 72, 32, 11, 11, - 32, 8, 38, 65, 97, 5, 87, 73, 84, 72, 32, 4, 25, 4, 66, 79, 86, 69, 4, - 11, 32, 4, 26, 76, 231, 207, 2, 82, 2, 249, 207, 2, 2, 69, 70, 4, 26, 82, - 219, 238, 2, 68, 2, 17, 2, 73, 83, 2, 173, 161, 2, 3, 73, 78, 71, 6, 26, - 68, 183, 196, 1, 82, 4, 11, 79, 4, 207, 213, 1, 84, 6, 22, 82, 155, 98, - 83, 2, 11, 32, 2, 175, 133, 1, 67, 5, 21, 3, 32, 84, 87, 2, 21, 3, 79, - 32, 68, 2, 249, 63, 3, 79, 84, 83, 170, 1, 100, 5, 72, 85, 84, 65, 32, - 156, 9, 11, 79, 78, 73, 65, 78, 32, 83, 73, 71, 78, 32, 223, 210, 2, 69, - 164, 1, 178, 1, 65, 88, 7, 76, 69, 84, 84, 69, 82, 32, 232, 2, 2, 83, 73, - 140, 2, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 142, 247, 1, 68, - 216, 149, 1, 2, 71, 86, 239, 71, 79, 4, 18, 66, 51, 78, 2, 29, 5, 66, 82, - 69, 86, 73, 2, 203, 16, 65, 2, 215, 214, 3, 74, 94, 202, 1, 65, 38, 68, - 46, 84, 46, 86, 186, 24, 85, 210, 200, 1, 73, 158, 190, 1, 78, 46, 83, - 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 254, 68, 72, 2, 76, 2, 77, 2, - 82, 2, 89, 186, 2, 69, 3, 79, 9, 206, 232, 3, 65, 2, 73, 3, 85, 8, 246, - 160, 3, 68, 254, 68, 72, 187, 2, 65, 8, 202, 160, 3, 84, 254, 68, 72, - 187, 2, 65, 10, 238, 3, 79, 231, 227, 3, 65, 12, 21, 3, 71, 78, 32, 12, - 42, 65, 74, 67, 98, 78, 203, 162, 3, 86, 4, 26, 86, 247, 161, 3, 78, 2, - 21, 3, 65, 71, 82, 2, 255, 238, 1, 65, 2, 11, 65, 2, 11, 78, 2, 25, 4, - 68, 82, 65, 66, 2, 11, 73, 2, 11, 78, 2, 243, 192, 3, 68, 2, 11, 85, 2, - 239, 192, 3, 75, 30, 78, 65, 38, 83, 70, 86, 214, 20, 85, 210, 200, 1, - 73, 206, 134, 2, 69, 3, 79, 6, 214, 228, 3, 65, 2, 73, 3, 85, 4, 25, 4, - 72, 79, 82, 84, 4, 11, 32, 4, 138, 228, 3, 69, 3, 79, 8, 11, 79, 8, 33, - 6, 67, 65, 76, 73, 67, 32, 8, 26, 82, 139, 231, 2, 76, 5, 163, 227, 3, - 82, 4, 128, 174, 3, 8, 67, 65, 80, 73, 84, 65, 76, 32, 227, 24, 69, 138, - 1, 210, 1, 77, 18, 78, 34, 79, 80, 2, 80, 32, 152, 9, 23, 82, 84, 79, 73, + 84, 30, 90, 138, 77, 82, 186, 209, 3, 67, 146, 1, 65, 2, 70, 2, 74, 2, + 76, 2, 77, 2, 78, 2, 80, 2, 81, 2, 86, 2, 87, 3, 89, 9, 38, 68, 159, 160, + 4, 72, 7, 11, 72, 5, 155, 160, 4, 72, 7, 134, 160, 4, 72, 3, 83, 7, 234, + 159, 4, 72, 3, 84, 7, 206, 159, 4, 72, 3, 90, 5, 179, 159, 4, 89, 2, 233, + 181, 3, 4, 65, 76, 73, 90, 2, 183, 221, 3, 82, 6, 76, 2, 69, 82, 21, 13, + 72, 84, 32, 84, 82, 73, 70, 79, 76, 73, 65, 84, 69, 5, 131, 225, 3, 32, + 2, 33, 6, 32, 83, 78, 79, 87, 70, 2, 175, 53, 76, 19, 11, 32, 16, 72, 8, + 79, 80, 69, 82, 65, 84, 79, 82, 233, 1, 5, 87, 73, 84, 72, 32, 11, 11, + 32, 8, 38, 65, 101, 5, 87, 73, 84, 72, 32, 4, 25, 4, 66, 79, 86, 69, 4, + 11, 32, 4, 26, 76, 139, 252, 2, 82, 2, 11, 69, 2, 147, 252, 2, 70, 4, 26, + 82, 251, 154, 3, 68, 2, 17, 2, 73, 83, 2, 241, 202, 2, 3, 73, 78, 71, 6, + 26, 68, 143, 234, 1, 82, 4, 11, 79, 4, 175, 252, 1, 84, 6, 22, 82, 239, + 125, 83, 2, 11, 32, 2, 243, 171, 1, 67, 5, 21, 3, 32, 84, 87, 2, 21, 3, + 79, 32, 68, 2, 237, 86, 3, 79, 84, 83, 170, 1, 100, 5, 72, 85, 84, 65, + 32, 184, 6, 11, 79, 78, 73, 65, 78, 32, 83, 73, 71, 78, 32, 227, 129, 3, + 69, 164, 1, 178, 1, 65, 88, 7, 76, 69, 84, 84, 69, 82, 32, 204, 1, 2, 83, + 73, 200, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 162, 162, 2, + 68, 204, 152, 1, 2, 71, 86, 255, 71, 79, 4, 18, 66, 51, 78, 2, 29, 5, 66, + 82, 69, 86, 73, 2, 171, 22, 65, 2, 143, 131, 4, 74, 94, 178, 60, 65, 38, + 68, 114, 84, 46, 86, 186, 5, 85, 206, 201, 1, 73, 162, 193, 1, 78, 46, + 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, + 2, 82, 2, 89, 186, 2, 69, 3, 79, 12, 21, 3, 71, 78, 32, 12, 46, 67, 98, + 78, 242, 60, 65, 231, 147, 3, 86, 2, 11, 65, 2, 11, 78, 2, 25, 4, 68, 82, + 65, 66, 2, 11, 73, 2, 11, 78, 2, 139, 239, 3, 68, 2, 11, 85, 2, 135, 239, + 3, 75, 30, 78, 83, 166, 63, 65, 38, 85, 22, 86, 186, 201, 1, 73, 222, + 137, 2, 69, 3, 79, 4, 25, 4, 72, 79, 82, 84, 4, 11, 32, 4, 198, 146, 4, + 69, 3, 79, 4, 144, 221, 3, 8, 67, 65, 80, 73, 84, 65, 76, 32, 239, 24, + 69, 146, 2, 140, 2, 12, 68, 72, 82, 73, 32, 76, 69, 84, 84, 69, 82, 32, + 182, 4, 77, 18, 78, 34, 79, 80, 2, 80, 32, 168, 17, 23, 82, 84, 79, 73, 83, 69, 32, 83, 72, 69, 76, 76, 32, 66, 82, 65, 67, 75, 69, 84, 69, 68, - 32, 130, 4, 84, 198, 246, 1, 73, 169, 67, 5, 75, 89, 79, 32, 84, 2, 239, - 20, 65, 2, 11, 71, 2, 135, 202, 3, 85, 6, 32, 2, 84, 72, 131, 212, 2, 76, - 5, 11, 66, 2, 11, 82, 2, 239, 250, 1, 85, 40, 140, 1, 4, 65, 82, 67, 32, - 162, 2, 67, 44, 2, 72, 65, 218, 1, 80, 126, 76, 22, 82, 138, 1, 83, 38, - 84, 81, 7, 87, 73, 84, 72, 32, 85, 80, 6, 180, 1, 19, 65, 78, 84, 73, 67, - 76, 79, 67, 75, 87, 73, 83, 69, 32, 65, 82, 82, 79, 87, 73, 21, 67, 76, - 79, 67, 75, 87, 73, 83, 69, 32, 65, 82, 82, 79, 87, 32, 87, 73, 84, 72, - 32, 5, 29, 5, 32, 87, 73, 84, 72, 2, 17, 2, 32, 80, 2, 143, 218, 1, 76, - 2, 11, 77, 2, 227, 217, 1, 73, 2, 11, 85, 2, 213, 137, 3, 3, 82, 76, 89, - 12, 36, 3, 76, 70, 32, 247, 219, 3, 84, 10, 50, 66, 46, 76, 26, 82, 114, - 83, 143, 214, 1, 73, 2, 11, 76, 2, 145, 140, 2, 3, 65, 67, 75, 2, 11, 69, - 2, 35, 70, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 17, 2, 32, 80, 2, 33, 6, - 65, 82, 69, 78, 84, 72, 2, 167, 189, 1, 69, 2, 11, 69, 2, 11, 67, 2, 11, - 84, 2, 11, 73, 2, 191, 227, 1, 79, 6, 41, 2, 69, 70, 6, 21, 3, 73, 71, - 72, 6, 17, 2, 84, 32, 6, 42, 67, 249, 133, 3, 4, 72, 65, 76, 70, 4, 26, - 79, 191, 223, 2, 82, 2, 211, 154, 3, 82, 2, 185, 133, 3, 4, 81, 85, 65, - 82, 2, 11, 79, 2, 149, 133, 3, 12, 82, 84, 79, 73, 83, 69, 32, 83, 72, - 69, 76, 76, 2, 29, 5, 87, 65, 82, 68, 83, 2, 17, 2, 32, 65, 2, 141, 218, - 2, 4, 82, 82, 79, 87, 20, 188, 1, 22, 67, 74, 75, 32, 85, 78, 73, 70, 73, - 69, 68, 32, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 161, 2, 19, 76, 65, - 84, 73, 78, 32, 67, 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 18, - 40, 2, 52, 69, 34, 53, 54, 54, 87, 55, 4, 202, 1, 48, 203, 172, 2, 56, 4, - 30, 50, 141, 1, 2, 66, 56, 2, 159, 142, 3, 68, 6, 48, 2, 50, 53, 22, 53, - 249, 172, 2, 2, 55, 50, 2, 235, 211, 3, 51, 2, 67, 53, 4, 32, 2, 48, 66, - 21, 2, 54, 68, 2, 167, 211, 3, 57, 2, 147, 211, 3, 55, 2, 11, 82, 2, 163, - 149, 3, 32, 64, 48, 6, 65, 76, 32, 82, 85, 78, 21, 2, 79, 32, 2, 235, - 216, 2, 79, 62, 72, 7, 76, 69, 84, 84, 69, 82, 32, 229, 2, 6, 83, 73, 71, - 78, 32, 82, 60, 202, 1, 66, 102, 73, 22, 78, 138, 144, 1, 69, 198, 71, - 67, 170, 183, 1, 65, 222, 61, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, - 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, 89, 186, 2, 79, 3, 85, 12, 52, - 7, 82, 69, 65, 84, 72, 89, 32, 203, 207, 3, 65, 10, 42, 73, 158, 144, 1, - 69, 175, 168, 2, 65, 5, 155, 207, 3, 85, 4, 206, 204, 3, 71, 187, 2, 65, - 2, 17, 2, 73, 83, 2, 21, 3, 73, 78, 71, 2, 11, 32, 2, 135, 165, 2, 84, - 98, 46, 65, 138, 2, 73, 246, 11, 79, 195, 1, 85, 16, 66, 67, 36, 2, 68, - 69, 34, 77, 32, 2, 78, 83, 243, 252, 2, 73, 4, 222, 200, 1, 75, 239, 203, - 1, 84, 2, 153, 16, 4, 32, 77, 65, 82, 5, 11, 32, 2, 175, 170, 2, 67, 4, - 44, 2, 80, 76, 33, 5, 86, 69, 82, 83, 65, 2, 11, 85, 2, 147, 173, 3, 84, - 2, 231, 48, 76, 66, 156, 1, 3, 65, 78, 71, 176, 3, 8, 68, 69, 78, 84, 32, - 69, 77, 66, 20, 9, 71, 82, 65, 77, 32, 70, 79, 82, 32, 140, 2, 4, 80, 76, - 69, 32, 255, 244, 2, 67, 16, 48, 2, 76, 69, 245, 1, 5, 85, 76, 65, 82, - 32, 10, 44, 6, 32, 87, 73, 84, 72, 32, 163, 1, 45, 8, 80, 9, 83, 69, 82, - 73, 70, 83, 32, 65, 84, 54, 85, 150, 136, 2, 82, 147, 64, 68, 2, 17, 2, - 32, 66, 2, 145, 184, 3, 3, 79, 84, 84, 2, 173, 47, 2, 78, 68, 2, 177, - 168, 2, 4, 72, 69, 65, 68, 6, 68, 9, 70, 76, 65, 71, 32, 79, 78, 32, 80, - 34, 82, 131, 235, 1, 66, 2, 11, 79, 2, 155, 171, 3, 83, 2, 11, 85, 2, - 199, 137, 3, 76, 2, 231, 180, 3, 76, 16, 66, 69, 34, 72, 34, 76, 22, 77, - 46, 84, 42, 87, 227, 179, 1, 70, 2, 11, 65, 2, 171, 199, 1, 82, 2, 11, - 69, 2, 163, 217, 1, 65, 2, 163, 150, 3, 65, 2, 21, 3, 79, 85, 78, 2, 159, - 154, 2, 84, 2, 17, 2, 72, 85, 2, 183, 161, 1, 78, 4, 198, 142, 2, 65, - 239, 40, 73, 30, 176, 2, 3, 67, 79, 76, 24, 20, 72, 79, 82, 73, 90, 79, - 78, 84, 65, 76, 32, 66, 65, 82, 32, 87, 73, 84, 72, 32, 98, 80, 34, 84, - 20, 13, 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 82, 32, 44, 9, 83, - 79, 76, 73, 68, 85, 83, 32, 66, 232, 8, 2, 68, 65, 224, 168, 1, 6, 78, - 69, 83, 84, 69, 68, 163, 11, 73, 2, 145, 24, 2, 79, 78, 4, 34, 68, 33, 4, - 84, 82, 73, 80, 2, 17, 2, 79, 85, 2, 11, 66, 2, 133, 145, 3, 2, 76, 69, - 4, 218, 190, 1, 76, 143, 107, 82, 2, 171, 171, 1, 73, 8, 42, 66, 74, 68, - 198, 161, 1, 82, 111, 87, 2, 29, 5, 73, 78, 65, 82, 89, 2, 21, 3, 32, 82, - 69, 2, 251, 19, 76, 2, 25, 4, 69, 76, 73, 77, 2, 183, 137, 2, 73, 10, 24, - 2, 76, 76, 35, 80, 5, 229, 188, 1, 3, 69, 89, 66, 6, 44, 5, 73, 67, 65, - 76, 32, 191, 173, 3, 72, 4, 18, 68, 47, 70, 2, 11, 82, 2, 11, 73, 2, 191, - 187, 3, 78, 2, 167, 217, 1, 73, 6, 18, 69, 79, 77, 5, 205, 153, 3, 14, - 32, 76, 73, 71, 72, 84, 32, 77, 79, 79, 78, 32, 65, 82, 2, 195, 136, 3, - 80, 56, 80, 3, 71, 82, 73, 22, 82, 148, 165, 2, 5, 77, 66, 76, 69, 82, - 151, 150, 1, 76, 2, 255, 181, 2, 75, 50, 50, 75, 84, 4, 78, 69, 68, 32, - 171, 154, 2, 84, 4, 48, 6, 73, 83, 72, 32, 76, 73, 167, 170, 3, 69, 2, - 11, 82, 2, 247, 180, 2, 65, 44, 238, 1, 65, 80, 6, 66, 76, 65, 67, 75, - 32, 40, 7, 87, 72, 73, 84, 69, 32, 83, 22, 67, 50, 68, 96, 2, 78, 79, 32, - 2, 79, 75, 30, 83, 185, 146, 3, 21, 71, 82, 69, 69, 75, 32, 83, 77, 65, - 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 73, 79, 4, 26, 77, 147, 152, 2, - 78, 2, 17, 2, 80, 69, 2, 11, 82, 2, 179, 171, 2, 83, 4, 18, 80, 23, 83, - 2, 223, 129, 2, 69, 2, 207, 131, 2, 72, 4, 144, 3, 5, 65, 80, 73, 84, 65, - 167, 106, 79, 6, 30, 65, 33, 3, 73, 71, 73, 2, 11, 71, 2, 147, 250, 2, - 71, 4, 129, 203, 1, 3, 84, 32, 84, 6, 162, 2, 82, 183, 174, 2, 84, 2, - 177, 209, 1, 2, 32, 72, 14, 128, 1, 18, 65, 78, 83, 45, 83, 69, 82, 73, - 70, 32, 67, 65, 80, 73, 84, 65, 76, 32, 38, 69, 32, 3, 77, 65, 76, 33, 2, - 79, 85, 6, 254, 181, 3, 71, 2, 76, 3, 89, 2, 11, 77, 2, 203, 229, 2, 73, - 2, 11, 76, 2, 163, 180, 2, 32, 4, 21, 3, 84, 72, 32, 4, 32, 2, 69, 65, 1, - 2, 87, 69, 2, 53, 11, 83, 84, 32, 80, 79, 73, 78, 84, 73, 78, 71, 2, 17, - 2, 32, 76, 2, 251, 178, 2, 69, 36, 58, 69, 52, 8, 73, 83, 84, 69, 68, 32, - 82, 73, 63, 79, 2, 17, 2, 76, 86, 2, 209, 159, 1, 3, 69, 32, 80, 2, 17, - 2, 71, 72, 2, 153, 93, 6, 84, 87, 65, 82, 68, 83, 32, 30, 32, 249, 9, 2, - 45, 69, 30, 200, 2, 24, 65, 83, 84, 69, 82, 73, 83, 75, 83, 32, 65, 76, - 73, 71, 78, 69, 68, 32, 86, 69, 82, 84, 73, 67, 34, 66, 86, 67, 116, 3, - 68, 79, 84, 226, 1, 72, 44, 14, 73, 78, 84, 69, 82, 83, 69, 67, 84, 73, - 78, 71, 32, 76, 92, 6, 74, 79, 73, 78, 69, 68, 64, 8, 76, 79, 71, 73, 67, - 65, 76, 32, 98, 77, 0, 3, 87, 79, 77, 119, 83, 2, 11, 65, 2, 171, 140, 3, - 76, 2, 29, 5, 85, 84, 84, 79, 78, 2, 17, 2, 32, 77, 2, 11, 79, 2, 147, - 166, 2, 85, 2, 93, 21, 79, 78, 83, 69, 67, 85, 84, 73, 86, 69, 32, 69, - 81, 85, 65, 76, 83, 32, 83, 73, 71, 2, 195, 240, 2, 78, 6, 18, 32, 55, - 83, 4, 22, 76, 131, 1, 80, 2, 225, 137, 1, 2, 69, 65, 2, 53, 11, 32, 79, - 86, 69, 82, 32, 79, 78, 69, 32, 68, 2, 11, 79, 2, 11, 84, 2, 17, 2, 32, - 80, 2, 29, 5, 85, 78, 67, 84, 85, 2, 239, 219, 2, 65, 2, 11, 69, 2, 11, - 65, 2, 243, 151, 2, 82, 4, 17, 2, 79, 71, 4, 25, 4, 73, 67, 65, 76, 4, - 11, 32, 4, 214, 157, 2, 65, 147, 85, 79, 2, 17, 2, 32, 83, 2, 21, 3, 81, - 85, 65, 2, 139, 151, 2, 82, 4, 30, 79, 13, 3, 65, 78, 68, 2, 11, 82, 2, - 11, 32, 2, 11, 79, 2, 11, 80, 2, 183, 17, 69, 2, 11, 69, 2, 45, 9, 78, - 32, 72, 79, 76, 68, 73, 78, 71, 2, 11, 32, 2, 11, 72, 2, 11, 65, 2, 207, - 204, 1, 78, 2, 49, 10, 80, 69, 69, 67, 72, 32, 66, 85, 66, 66, 2, 239, - 148, 2, 76, 2, 11, 77, 2, 133, 195, 1, 2, 32, 68, 140, 3, 140, 1, 8, 71, - 65, 82, 73, 84, 73, 67, 32, 148, 6, 7, 77, 66, 82, 69, 76, 76, 65, 166, - 1, 78, 154, 8, 80, 174, 148, 1, 82, 135, 131, 2, 83, 62, 48, 7, 76, 69, - 84, 84, 69, 82, 32, 159, 5, 87, 60, 206, 1, 65, 28, 2, 81, 79, 22, 66, - 22, 68, 50, 71, 44, 2, 72, 79, 22, 75, 34, 76, 32, 2, 82, 65, 22, 83, 74, - 84, 74, 89, 22, 90, 186, 161, 2, 78, 178, 91, 80, 246, 5, 87, 202, 12, - 77, 174, 18, 73, 3, 85, 4, 26, 76, 143, 213, 2, 73, 2, 167, 162, 3, 80, - 2, 255, 255, 2, 69, 4, 26, 69, 247, 154, 2, 72, 2, 207, 255, 2, 76, 4, - 214, 248, 1, 72, 153, 137, 1, 2, 65, 77, 5, 163, 161, 3, 84, 4, 186, 162, - 2, 65, 215, 126, 72, 2, 11, 65, 2, 151, 254, 2, 77, 2, 203, 171, 1, 83, - 8, 38, 65, 146, 163, 2, 72, 215, 90, 83, 4, 246, 139, 3, 68, 239, 13, 77, - 6, 38, 72, 218, 133, 3, 69, 175, 28, 79, 2, 11, 65, 2, 179, 231, 1, 78, - 2, 207, 219, 2, 79, 4, 130, 253, 2, 69, 207, 36, 85, 2, 21, 3, 79, 82, - 68, 2, 25, 4, 32, 68, 73, 86, 2, 139, 125, 73, 7, 11, 32, 4, 84, 4, 79, - 78, 32, 71, 45, 13, 87, 73, 84, 72, 32, 82, 65, 73, 78, 32, 68, 82, 79, - 2, 11, 82, 2, 11, 79, 2, 143, 146, 2, 85, 2, 139, 226, 2, 80, 32, 160, 1, - 3, 65, 77, 85, 20, 7, 67, 69, 82, 84, 65, 73, 78, 34, 68, 62, 73, 241, 5, - 18, 77, 65, 82, 82, 73, 69, 68, 32, 80, 65, 82, 84, 78, 69, 82, 83, 72, - 73, 2, 227, 141, 2, 83, 2, 11, 84, 2, 155, 151, 2, 89, 4, 26, 69, 199, - 138, 2, 79, 2, 11, 82, 2, 151, 130, 3, 84, 22, 60, 2, 79, 78, 230, 3, 84, - 106, 86, 201, 43, 3, 67, 79, 82, 15, 11, 32, 12, 160, 1, 6, 65, 66, 79, - 86, 69, 32, 108, 22, 66, 69, 83, 73, 68, 69, 32, 65, 78, 68, 32, 74, 79, - 73, 78, 69, 68, 32, 87, 73, 84, 72, 41, 5, 87, 73, 84, 72, 32, 4, 52, 9, - 66, 65, 82, 32, 65, 66, 79, 86, 69, 23, 73, 2, 17, 2, 32, 73, 2, 229, 39, - 4, 78, 84, 69, 82, 2, 17, 2, 32, 85, 2, 139, 202, 2, 78, 6, 66, 77, 58, - 79, 217, 224, 2, 8, 76, 79, 71, 73, 67, 65, 76, 32, 2, 11, 73, 2, 11, 78, - 2, 11, 85, 2, 223, 146, 2, 83, 2, 11, 86, 2, 229, 73, 2, 69, 82, 4, 18, - 32, 67, 69, 2, 11, 83, 2, 17, 2, 69, 80, 2, 11, 65, 2, 219, 223, 2, 82, - 2, 171, 133, 2, 68, 2, 57, 12, 69, 82, 83, 65, 76, 32, 82, 69, 67, 89, - 67, 76, 2, 17, 2, 73, 78, 2, 199, 132, 2, 71, 2, 179, 132, 2, 80, 164, 2, - 170, 1, 32, 128, 8, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 248, 3, - 4, 80, 69, 82, 32, 168, 29, 8, 83, 73, 68, 69, 45, 68, 79, 87, 21, 6, 87, - 65, 82, 68, 83, 32, 38, 140, 1, 5, 65, 82, 82, 79, 87, 240, 1, 5, 66, 65, - 82, 66, 32, 228, 1, 5, 68, 79, 87, 78, 32, 210, 1, 70, 30, 82, 89, 4, 84, - 65, 67, 75, 8, 60, 7, 32, 84, 72, 82, 79, 85, 71, 21, 4, 72, 69, 65, 68, - 2, 143, 197, 1, 72, 7, 11, 32, 4, 112, 22, 66, 69, 84, 87, 69, 69, 78, - 32, 84, 87, 79, 32, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 159, 134, 2, - 73, 2, 181, 254, 1, 2, 32, 66, 8, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, - 72, 4, 57, 12, 84, 32, 68, 79, 87, 78, 32, 66, 65, 82, 66, 32, 4, 44, 3, - 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 11, 84, 2, 25, 4, 32, 72, 65, 82, 2, - 11, 80, 2, 207, 193, 2, 79, 14, 76, 5, 65, 82, 82, 79, 87, 62, 66, 38, - 68, 18, 83, 202, 43, 87, 215, 9, 84, 5, 37, 7, 32, 87, 73, 84, 72, 32, - 66, 2, 175, 135, 2, 65, 2, 209, 245, 1, 4, 76, 65, 67, 75, 2, 147, 44, - 79, 2, 131, 50, 65, 2, 11, 73, 2, 155, 98, 83, 2, 53, 11, 73, 71, 72, 84, - 32, 68, 73, 65, 71, 79, 78, 2, 225, 113, 4, 65, 76, 32, 69, 5, 29, 5, 32, - 87, 73, 84, 72, 2, 33, 6, 32, 67, 73, 82, 67, 76, 2, 159, 145, 2, 69, 14, - 96, 8, 77, 73, 76, 73, 84, 65, 82, 89, 20, 6, 83, 77, 65, 76, 76, 32, 22, - 65, 66, 82, 35, 84, 2, 37, 2, 32, 65, 4, 18, 65, 67, 82, 2, 11, 73, 2, - 11, 82, 2, 17, 2, 80, 76, 2, 179, 243, 1, 65, 2, 11, 69, 2, 199, 232, 1, - 68, 4, 37, 7, 82, 73, 65, 78, 71, 76, 69, 4, 11, 32, 4, 11, 87, 4, 25, 4, - 73, 84, 72, 32, 4, 18, 76, 27, 82, 2, 11, 69, 2, 35, 70, 2, 21, 3, 73, - 71, 72, 2, 11, 84, 2, 17, 2, 32, 72, 2, 21, 3, 65, 76, 70, 2, 17, 2, 32, - 66, 2, 11, 76, 2, 207, 219, 1, 65, 110, 172, 1, 4, 65, 78, 68, 32, 186, - 2, 66, 74, 70, 36, 5, 72, 65, 76, 70, 32, 148, 3, 5, 76, 69, 70, 84, 32, - 226, 5, 79, 60, 6, 82, 73, 71, 72, 84, 32, 218, 12, 83, 63, 84, 8, 34, - 76, 53, 4, 82, 73, 71, 72, 6, 48, 2, 69, 70, 149, 1, 5, 79, 87, 69, 82, - 32, 2, 53, 11, 84, 32, 65, 78, 68, 32, 76, 79, 87, 69, 82, 2, 217, 25, - 19, 32, 84, 82, 73, 65, 78, 71, 85, 76, 65, 82, 32, 84, 72, 82, 69, 69, - 32, 81, 4, 28, 2, 84, 82, 179, 29, 79, 2, 205, 3, 7, 73, 65, 78, 71, 85, - 76, 65, 2, 17, 2, 76, 65, 2, 17, 2, 68, 69, 2, 177, 208, 1, 3, 32, 83, - 67, 2, 17, 2, 73, 86, 2, 191, 22, 69, 12, 96, 5, 66, 76, 79, 67, 75, 108, - 8, 73, 78, 86, 69, 82, 83, 69, 32, 254, 19, 77, 227, 160, 1, 67, 5, 209, - 20, 23, 32, 65, 78, 68, 32, 76, 79, 87, 69, 82, 32, 72, 65, 76, 70, 32, - 73, 78, 86, 69, 82, 83, 69, 4, 100, 21, 77, 69, 68, 73, 85, 77, 32, 83, - 72, 65, 68, 69, 32, 65, 78, 68, 32, 76, 79, 87, 69, 51, 87, 2, 11, 82, 2, - 141, 26, 5, 32, 72, 65, 76, 70, 2, 21, 3, 72, 73, 84, 2, 147, 179, 1, 69, - 34, 106, 66, 238, 3, 67, 54, 84, 176, 9, 13, 79, 82, 32, 76, 79, 87, 69, - 82, 32, 82, 73, 71, 72, 143, 1, 81, 22, 65, 14, 76, 79, 67, 75, 32, 68, - 73, 65, 71, 79, 78, 65, 76, 32, 22, 144, 1, 6, 76, 79, 87, 69, 82, 32, - 173, 6, 24, 85, 80, 80, 69, 82, 32, 77, 73, 68, 68, 76, 69, 32, 76, 69, - 70, 84, 32, 84, 79, 32, 85, 80, 80, 18, 176, 1, 10, 67, 69, 78, 84, 82, - 69, 32, 84, 79, 32, 36, 8, 76, 69, 70, 84, 32, 84, 79, 32, 169, 8, 18, - 77, 73, 68, 68, 76, 69, 32, 76, 69, 70, 84, 32, 84, 79, 32, 85, 80, 80, - 6, 70, 76, 193, 6, 3, 85, 80, 80, 6, 34, 76, 153, 7, 3, 85, 80, 80, 2, - 217, 7, 2, 79, 87, 2, 29, 5, 79, 82, 78, 69, 82, 2, 139, 218, 1, 32, 6, - 22, 79, 187, 12, 82, 2, 129, 12, 11, 32, 76, 79, 87, 69, 82, 32, 82, 73, - 71, 72, 4, 11, 78, 4, 17, 2, 69, 32, 4, 150, 15, 81, 139, 4, 69, 40, 128, - 1, 2, 66, 76, 186, 6, 68, 100, 12, 79, 82, 32, 76, 79, 87, 69, 82, 32, - 76, 69, 70, 106, 80, 38, 81, 104, 2, 83, 72, 51, 84, 22, 61, 13, 79, 67, - 75, 32, 68, 73, 65, 71, 79, 78, 65, 76, 32, 22, 140, 1, 24, 76, 79, 87, - 69, 82, 32, 77, 73, 68, 68, 76, 69, 32, 76, 69, 70, 84, 32, 84, 79, 32, - 76, 79, 87, 57, 6, 85, 80, 80, 69, 82, 32, 4, 21, 3, 69, 82, 32, 4, 246, - 3, 67, 211, 202, 2, 82, 18, 176, 1, 10, 67, 69, 78, 84, 82, 69, 32, 84, - 79, 32, 92, 8, 76, 69, 70, 84, 32, 84, 79, 32, 141, 1, 18, 77, 73, 68, - 68, 76, 69, 32, 76, 69, 70, 84, 32, 84, 79, 32, 76, 79, 87, 6, 32, 3, 76, - 79, 87, 139, 1, 85, 4, 21, 3, 69, 82, 32, 4, 142, 2, 77, 171, 202, 2, 82, - 6, 28, 3, 76, 79, 87, 51, 85, 4, 21, 3, 69, 82, 32, 4, 142, 1, 67, 43, - 77, 2, 17, 2, 80, 80, 2, 17, 2, 69, 82, 2, 117, 2, 32, 77, 6, 21, 3, 69, - 82, 32, 6, 34, 67, 42, 77, 171, 202, 2, 82, 2, 11, 69, 2, 241, 212, 1, 2, - 78, 84, 2, 25, 4, 73, 68, 68, 76, 2, 203, 166, 1, 69, 2, 65, 14, 82, 79, - 80, 45, 83, 72, 65, 68, 79, 87, 69, 68, 32, 87, 2, 225, 181, 1, 3, 72, - 73, 84, 2, 69, 15, 84, 32, 67, 85, 82, 76, 89, 32, 66, 82, 65, 67, 75, - 69, 84, 2, 11, 32, 2, 179, 162, 2, 83, 2, 11, 69, 2, 249, 69, 2, 78, 67, - 2, 81, 18, 85, 65, 68, 82, 65, 78, 84, 32, 67, 73, 82, 67, 85, 76, 65, - 82, 32, 65, 2, 163, 203, 1, 82, 4, 245, 97, 8, 65, 68, 79, 87, 69, 68, - 32, 87, 6, 18, 79, 107, 82, 2, 49, 10, 32, 76, 79, 87, 69, 82, 32, 76, - 69, 70, 2, 11, 84, 2, 11, 32, 2, 11, 70, 2, 163, 108, 73, 4, 25, 4, 73, - 65, 78, 71, 4, 40, 4, 85, 76, 65, 82, 171, 217, 2, 76, 2, 17, 2, 32, 77, - 2, 41, 8, 69, 68, 73, 85, 77, 32, 83, 72, 2, 199, 98, 65, 2, 17, 2, 69, - 86, 2, 17, 2, 69, 78, 2, 117, 2, 32, 69, 6, 52, 2, 72, 82, 129, 1, 6, 82, - 73, 65, 78, 71, 85, 4, 21, 3, 69, 69, 32, 4, 18, 69, 35, 81, 2, 65, 5, - 73, 71, 72, 84, 72, 2, 33, 6, 85, 65, 82, 84, 69, 82, 2, 231, 4, 83, 2, - 45, 9, 76, 65, 82, 32, 79, 78, 69, 32, 81, 2, 165, 4, 6, 85, 65, 82, 84, - 69, 82, 2, 203, 175, 2, 78, 128, 1, 146, 1, 65, 174, 6, 66, 154, 1, 68, - 50, 70, 82, 72, 146, 4, 67, 46, 81, 42, 82, 22, 83, 102, 84, 142, 7, 80, - 173, 3, 6, 87, 72, 73, 84, 69, 32, 32, 34, 78, 33, 4, 82, 82, 79, 87, 2, - 11, 67, 2, 207, 166, 2, 79, 31, 11, 32, 28, 134, 1, 65, 180, 1, 14, 76, - 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 70, 32, 68, 32, 5, 87, 73, 84, - 72, 32, 214, 8, 70, 175, 5, 84, 2, 41, 8, 78, 68, 32, 82, 73, 71, 72, 84, - 2, 17, 2, 32, 79, 2, 25, 4, 78, 69, 32, 69, 2, 29, 5, 73, 71, 72, 84, 72, - 2, 11, 32, 2, 11, 66, 2, 11, 76, 2, 239, 185, 1, 79, 2, 233, 200, 1, 3, - 79, 87, 78, 20, 74, 68, 58, 76, 42, 77, 38, 78, 58, 83, 66, 69, 246, 12, - 84, 139, 59, 72, 2, 21, 3, 79, 85, 66, 2, 11, 76, 2, 131, 183, 2, 69, 2, - 11, 65, 2, 233, 23, 3, 82, 71, 69, 2, 205, 23, 5, 69, 68, 73, 85, 77, 2, - 25, 4, 79, 84, 67, 72, 2, 11, 69, 2, 247, 56, 68, 4, 11, 77, 4, 25, 4, - 65, 76, 76, 32, 4, 22, 69, 203, 22, 84, 2, 237, 22, 10, 81, 85, 73, 76, - 65, 84, 69, 82, 65, 76, 4, 11, 76, 4, 25, 4, 65, 67, 75, 32, 4, 44, 5, - 67, 73, 82, 67, 76, 143, 202, 1, 65, 2, 25, 4, 69, 68, 32, 87, 2, 11, 72, - 2, 229, 14, 2, 73, 84, 4, 22, 79, 187, 13, 65, 2, 169, 14, 2, 85, 66, 2, - 11, 73, 2, 37, 7, 78, 71, 69, 82, 45, 80, 79, 2, 205, 200, 1, 2, 83, 84, - 20, 88, 17, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, 32, 66, 65, 82, - 66, 32, 255, 2, 69, 16, 56, 4, 76, 69, 70, 84, 245, 1, 5, 82, 73, 71, 72, - 84, 10, 22, 32, 231, 9, 87, 8, 60, 7, 66, 69, 83, 73, 68, 69, 32, 206, 1, - 70, 175, 5, 84, 4, 40, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 193, 146, 1, - 23, 87, 65, 82, 68, 83, 32, 72, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, - 72, 32, 66, 65, 82, 66, 6, 22, 32, 243, 7, 87, 4, 22, 70, 175, 5, 84, 2, - 229, 15, 3, 82, 79, 77, 4, 25, 4, 65, 86, 89, 32, 4, 26, 67, 191, 196, 1, - 65, 2, 185, 10, 7, 79, 77, 80, 82, 69, 83, 83, 2, 133, 9, 6, 85, 65, 68, - 82, 85, 80, 2, 195, 139, 2, 79, 4, 30, 65, 53, 3, 81, 85, 65, 2, 153, - 195, 1, 8, 78, 83, 45, 83, 69, 82, 73, 70, 2, 255, 8, 82, 36, 36, 2, 82, - 73, 225, 7, 2, 87, 79, 30, 40, 5, 65, 78, 71, 76, 69, 151, 7, 80, 28, 52, - 8, 45, 72, 69, 65, 68, 69, 68, 32, 199, 13, 32, 26, 48, 5, 65, 82, 82, - 79, 87, 170, 5, 68, 39, 80, 23, 11, 32, 20, 92, 17, 76, 69, 70, 84, 87, - 65, 82, 68, 83, 32, 79, 70, 32, 68, 79, 87, 78, 98, 84, 19, 87, 2, 37, 7, - 87, 65, 82, 68, 83, 32, 84, 2, 37, 7, 82, 73, 65, 78, 71, 76, 69, 2, 215, - 5, 45, 2, 187, 10, 79, 16, 25, 4, 73, 84, 72, 32, 16, 114, 66, 28, 6, 76, - 79, 78, 71, 32, 84, 130, 1, 77, 34, 78, 34, 86, 34, 72, 149, 57, 6, 68, - 79, 85, 66, 76, 69, 2, 149, 2, 3, 79, 76, 68, 4, 17, 2, 73, 80, 4, 11, - 32, 4, 34, 76, 21, 4, 82, 73, 71, 72, 2, 17, 2, 69, 70, 2, 11, 84, 2, 11, - 87, 2, 135, 123, 65, 2, 121, 5, 69, 68, 73, 85, 77, 2, 89, 5, 65, 82, 82, - 79, 87, 2, 29, 5, 69, 82, 89, 32, 72, 2, 25, 4, 69, 65, 86, 89, 2, 237, - 171, 2, 4, 32, 83, 72, 65, 2, 11, 65, 2, 249, 1, 2, 83, 72, 2, 11, 65, 2, - 25, 4, 73, 82, 69, 68, 2, 29, 5, 32, 65, 82, 82, 79, 2, 239, 151, 2, 87, - 2, 11, 76, 2, 227, 186, 1, 69, 6, 74, 32, 61, 14, 45, 72, 69, 65, 68, 69, - 68, 32, 65, 82, 82, 79, 87, 32, 2, 25, 4, 72, 69, 65, 68, 2, 11, 69, 2, - 219, 185, 1, 68, 4, 42, 87, 129, 132, 1, 4, 70, 82, 79, 77, 2, 33, 6, 73, - 84, 72, 32, 84, 82, 2, 61, 13, 73, 65, 78, 71, 76, 69, 32, 65, 82, 82, - 79, 87, 72, 2, 169, 118, 2, 69, 65, 18, 88, 5, 65, 82, 82, 79, 87, 133, - 4, 12, 68, 79, 85, 66, 76, 69, 32, 65, 82, 82, 79, 87, 15, 11, 32, 12, - 104, 8, 70, 82, 79, 77, 32, 66, 65, 82, 40, 11, 79, 78, 32, 80, 69, 68, - 69, 83, 84, 65, 76, 183, 1, 87, 5, 93, 7, 32, 87, 73, 84, 72, 32, 72, 7, - 33, 6, 32, 87, 73, 84, 72, 32, 4, 18, 72, 43, 86, 2, 69, 7, 79, 82, 73, - 90, 79, 78, 84, 2, 29, 5, 69, 82, 84, 73, 67, 2, 17, 2, 65, 76, 2, 11, - 32, 2, 151, 173, 1, 66, 2, 29, 5, 73, 84, 72, 73, 78, 2, 17, 2, 32, 84, - 2, 37, 7, 82, 73, 65, 78, 71, 76, 69, 2, 11, 32, 2, 11, 65, 2, 25, 4, 82, - 82, 79, 87, 2, 11, 72, 2, 239, 135, 2, 69, 5, 45, 9, 32, 79, 78, 32, 80, - 69, 68, 69, 83, 2, 163, 196, 1, 84, 226, 15, 86, 65, 254, 16, 83, 174, 3, - 69, 162, 42, 73, 178, 8, 79, 134, 2, 84, 21, 2, 85, 76, 218, 8, 116, 2, - 73, 32, 128, 16, 17, 82, 73, 65, 84, 73, 79, 78, 32, 83, 69, 76, 69, 67, - 84, 79, 82, 45, 173, 42, 2, 77, 80, 216, 4, 54, 67, 34, 70, 82, 81, 40, - 2, 83, 89, 243, 91, 68, 2, 11, 79, 2, 135, 170, 2, 77, 2, 11, 85, 2, 11, - 76, 2, 11, 76, 2, 11, 32, 2, 11, 83, 2, 195, 209, 1, 84, 2, 17, 2, 85, - 69, 2, 135, 225, 1, 83, 190, 4, 68, 7, 76, 76, 65, 66, 76, 69, 32, 197, - 11, 5, 77, 66, 79, 76, 32, 164, 4, 214, 1, 68, 70, 66, 2, 83, 2, 84, 2, - 90, 70, 71, 122, 72, 82, 75, 134, 1, 76, 130, 1, 77, 86, 78, 134, 3, 67, - 2, 70, 2, 74, 2, 80, 2, 82, 2, 86, 2, 89, 78, 87, 50, 69, 34, 79, 158, - 70, 65, 2, 73, 3, 85, 42, 66, 72, 162, 8, 79, 238, 254, 1, 69, 150, 64, - 65, 2, 73, 3, 85, 28, 230, 7, 72, 58, 79, 238, 254, 1, 69, 150, 64, 65, - 2, 73, 3, 85, 34, 62, 66, 202, 6, 69, 86, 79, 130, 191, 2, 65, 2, 73, 3, - 85, 18, 106, 79, 222, 5, 69, 214, 191, 2, 65, 2, 73, 3, 85, 24, 50, 79, - 222, 5, 69, 214, 71, 65, 2, 73, 3, 85, 7, 174, 197, 2, 78, 3, 79, 34, 70, - 80, 206, 5, 79, 130, 71, 65, 238, 183, 1, 69, 150, 64, 73, 3, 85, 18, - 246, 4, 69, 86, 79, 130, 71, 65, 130, 248, 1, 73, 3, 85, 16, 54, 69, 218, - 4, 79, 130, 191, 2, 65, 2, 73, 3, 85, 7, 26, 78, 191, 195, 2, 69, 2, 21, - 3, 71, 84, 72, 2, 183, 133, 2, 69, 42, 214, 3, 66, 0, 2, 71, 66, 58, 79, - 238, 254, 1, 69, 150, 64, 65, 2, 73, 3, 85, 90, 90, 68, 174, 1, 71, 126, - 74, 2, 89, 58, 79, 238, 254, 1, 69, 150, 64, 65, 2, 73, 3, 85, 24, 54, - 79, 150, 129, 2, 69, 150, 64, 65, 2, 73, 3, 85, 15, 36, 3, 76, 69, 32, - 131, 193, 2, 79, 10, 54, 83, 214, 161, 2, 68, 190, 28, 70, 2, 75, 3, 77, - 2, 211, 161, 2, 79, 25, 42, 71, 182, 240, 1, 65, 2, 69, 3, 79, 16, 50, - 69, 86, 79, 130, 191, 2, 65, 2, 73, 3, 85, 7, 210, 191, 2, 69, 3, 78, 14, - 54, 79, 238, 254, 1, 69, 150, 64, 65, 2, 73, 3, 85, 5, 255, 190, 2, 79, - 28, 46, 69, 34, 79, 158, 70, 65, 2, 73, 3, 85, 9, 186, 70, 69, 131, 248, - 1, 78, 9, 154, 70, 79, 131, 248, 1, 78, 26, 66, 68, 62, 70, 30, 74, 22, - 75, 50, 78, 22, 84, 199, 227, 1, 66, 6, 26, 79, 179, 236, 1, 65, 4, 174, - 236, 1, 79, 251, 49, 45, 4, 74, 69, 251, 185, 2, 65, 2, 243, 235, 1, 79, - 4, 26, 69, 199, 235, 1, 85, 2, 195, 235, 1, 69, 2, 195, 168, 2, 73, 6, - 154, 235, 1, 73, 2, 79, 183, 78, 65, 128, 4, 74, 49, 94, 50, 98, 51, 2, - 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 223, 1, 182, 1, 48, 2, 49, 2, 50, - 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 137, 1, 90, 48, 2, 49, - 2, 50, 2, 51, 2, 52, 94, 53, 250, 184, 2, 54, 2, 55, 2, 56, 3, 57, 23, - 210, 185, 2, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, - 3, 57, 17, 246, 184, 2, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, - 180, 1, 116, 4, 68, 73, 67, 32, 222, 19, 82, 208, 148, 1, 13, 67, 84, 79, - 82, 32, 79, 82, 32, 67, 82, 79, 83, 83, 195, 106, 83, 86, 60, 5, 83, 73, - 71, 78, 32, 153, 10, 5, 84, 79, 78, 69, 32, 48, 218, 2, 65, 216, 1, 17, - 68, 79, 85, 66, 76, 69, 32, 65, 78, 85, 83, 86, 65, 82, 65, 32, 65, 98, - 74, 52, 6, 78, 73, 72, 83, 72, 86, 22, 82, 240, 1, 8, 72, 69, 88, 73, 70, - 79, 82, 77, 22, 76, 52, 4, 84, 73, 82, 89, 22, 85, 60, 8, 86, 73, 83, 65, - 82, 71, 65, 32, 237, 6, 17, 89, 65, 74, 85, 82, 86, 69, 68, 73, 67, 32, - 77, 73, 68, 76, 73, 78, 14, 76, 8, 78, 85, 83, 86, 65, 82, 65, 32, 212, - 1, 3, 84, 73, 75, 151, 2, 82, 10, 134, 1, 65, 24, 4, 66, 65, 72, 73, 24, - 9, 85, 66, 72, 65, 89, 65, 84, 79, 32, 201, 4, 10, 86, 65, 77, 65, 71, - 79, 77, 85, 75, 72, 2, 21, 3, 78, 84, 65, 2, 21, 3, 82, 71, 79, 2, 11, - 77, 2, 163, 9, 85, 2, 151, 239, 1, 82, 2, 33, 6, 73, 72, 86, 65, 77, 85, - 2, 151, 3, 76, 2, 235, 174, 2, 65, 8, 144, 1, 15, 69, 86, 69, 82, 83, 69, - 68, 32, 86, 73, 83, 65, 82, 71, 65, 36, 9, 79, 84, 65, 84, 69, 68, 32, - 65, 82, 57, 5, 84, 72, 65, 78, 71, 4, 11, 32, 4, 218, 6, 65, 23, 85, 2, - 25, 4, 68, 72, 65, 86, 2, 245, 236, 1, 2, 73, 83, 2, 17, 2, 32, 76, 2, - 21, 3, 79, 78, 71, 2, 145, 234, 1, 2, 32, 65, 2, 203, 171, 2, 65, 2, 37, - 7, 80, 65, 68, 72, 77, 65, 78, 2, 203, 171, 2, 73, 10, 40, 3, 65, 78, 85, - 2, 85, 179, 9, 83, 4, 25, 4, 68, 65, 84, 84, 4, 11, 65, 5, 25, 4, 32, 87, - 73, 84, 2, 11, 72, 2, 11, 32, 2, 11, 84, 2, 11, 65, 2, 215, 163, 1, 73, - 38, 128, 2, 5, 67, 65, 78, 68, 82, 16, 2, 68, 79, 96, 2, 75, 65, 136, 1, - 4, 80, 82, 69, 78, 16, 2, 82, 73, 106, 84, 164, 1, 11, 89, 65, 74, 85, - 82, 86, 69, 68, 73, 67, 32, 180, 1, 12, 65, 84, 72, 65, 82, 86, 65, 86, - 69, 68, 73, 67, 213, 224, 1, 2, 83, 72, 4, 251, 17, 65, 6, 40, 5, 85, 66, - 76, 69, 32, 191, 3, 84, 4, 22, 82, 211, 5, 83, 2, 11, 73, 2, 251, 1, 78, - 4, 56, 3, 82, 83, 72, 17, 7, 84, 72, 65, 75, 65, 32, 65, 2, 235, 110, 65, - 2, 17, 2, 78, 85, 2, 17, 2, 68, 65, 2, 171, 132, 2, 84, 2, 167, 49, 75, - 4, 82, 78, 237, 2, 15, 71, 86, 69, 68, 73, 67, 32, 75, 65, 83, 72, 77, - 73, 82, 73, 2, 139, 171, 1, 71, 6, 42, 72, 24, 4, 82, 73, 80, 76, 19, 87, - 2, 49, 3, 82, 69, 69, 2, 219, 2, 69, 2, 11, 79, 2, 25, 4, 32, 68, 79, 84, - 2, 11, 83, 2, 11, 32, 2, 159, 14, 66, 8, 176, 1, 10, 65, 71, 71, 82, 65, - 86, 65, 84, 69, 68, 22, 73, 105, 27, 75, 65, 84, 72, 65, 75, 65, 32, 73, - 78, 68, 69, 80, 69, 78, 68, 69, 78, 84, 32, 83, 86, 65, 82, 73, 84, 65, - 2, 17, 2, 32, 73, 2, 49, 10, 78, 68, 69, 80, 69, 78, 68, 69, 78, 84, 2, - 17, 2, 32, 83, 2, 221, 255, 1, 3, 86, 65, 82, 5, 37, 7, 32, 83, 67, 72, - 82, 79, 69, 2, 143, 230, 1, 68, 90, 62, 83, 16, 6, 84, 73, 67, 65, 76, - 32, 221, 13, 2, 89, 32, 2, 171, 84, 73, 62, 208, 2, 4, 66, 65, 82, 32, - 166, 3, 69, 62, 70, 36, 11, 73, 68, 69, 79, 71, 82, 65, 80, 72, 73, 67, - 48, 12, 75, 65, 78, 65, 32, 82, 69, 80, 69, 65, 84, 32, 252, 1, 4, 76, - 73, 78, 69, 238, 1, 77, 88, 17, 79, 78, 69, 32, 69, 73, 71, 72, 84, 72, - 32, 66, 76, 79, 67, 75, 45, 62, 84, 192, 73, 3, 83, 73, 88, 241, 32, 5, - 90, 73, 71, 90, 65, 8, 108, 6, 66, 69, 83, 73, 68, 69, 60, 8, 68, 79, 85, - 66, 76, 69, 32, 76, 20, 4, 84, 82, 73, 80, 139, 1, 87, 2, 17, 2, 32, 82, - 2, 21, 3, 73, 71, 72, 2, 187, 123, 84, 2, 69, 2, 69, 70, 2, 25, 4, 76, - 69, 32, 82, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 29, 5, 32, 84, 85, 82, - 78, 2, 11, 83, 2, 11, 84, 2, 219, 124, 73, 2, 21, 3, 73, 84, 72, 2, 17, - 2, 32, 72, 2, 153, 237, 1, 7, 79, 82, 73, 90, 79, 78, 84, 2, 25, 4, 76, - 76, 73, 80, 2, 11, 83, 2, 155, 223, 1, 73, 2, 11, 79, 2, 129, 79, 2, 85, - 82, 2, 17, 2, 32, 73, 2, 237, 178, 1, 2, 84, 69, 10, 120, 4, 77, 65, 82, - 75, 45, 22, 87, 73, 84, 72, 32, 86, 79, 73, 67, 69, 68, 32, 83, 79, 85, - 78, 68, 32, 77, 65, 82, 75, 7, 11, 32, 4, 50, 85, 21, 3, 76, 79, 87, 5, - 17, 2, 32, 85, 2, 17, 2, 80, 80, 2, 17, 2, 69, 82, 2, 129, 26, 2, 32, 72, - 11, 11, 32, 8, 38, 69, 49, 5, 87, 73, 84, 72, 32, 2, 25, 4, 88, 84, 69, - 78, 2, 255, 200, 1, 83, 6, 40, 4, 67, 73, 82, 67, 207, 135, 1, 77, 4, 11, - 76, 4, 11, 69, 4, 11, 32, 4, 26, 66, 223, 155, 1, 65, 2, 11, 69, 2, 159, - 126, 76, 2, 65, 14, 65, 76, 69, 32, 87, 73, 84, 72, 32, 83, 84, 82, 79, - 75, 2, 239, 144, 1, 69, 12, 194, 151, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, - 3, 55, 10, 32, 2, 65, 66, 106, 73, 19, 82, 6, 18, 32, 35, 85, 2, 11, 75, - 2, 251, 132, 2, 69, 4, 33, 6, 76, 65, 84, 73, 79, 78, 5, 199, 45, 32, 2, - 135, 9, 76, 2, 29, 5, 65, 70, 70, 73, 67, 2, 249, 234, 1, 2, 32, 76, 26, - 80, 6, 72, 69, 65, 86, 89, 32, 176, 5, 3, 77, 85, 67, 233, 9, 3, 66, 79, - 76, 20, 62, 69, 190, 1, 70, 38, 82, 82, 83, 246, 1, 87, 207, 10, 71, 4, - 25, 4, 73, 71, 72, 84, 4, 11, 32, 4, 22, 80, 223, 2, 83, 2, 25, 4, 79, - 73, 78, 84, 2, 17, 2, 69, 68, 2, 17, 2, 32, 66, 2, 25, 4, 76, 65, 67, 75, - 2, 11, 32, 2, 139, 79, 83, 2, 11, 73, 2, 185, 1, 2, 86, 69, 2, 11, 69, 2, - 29, 5, 86, 69, 82, 83, 69, 2, 17, 2, 32, 83, 2, 231, 1, 79, 6, 30, 65, - 42, 73, 147, 1, 79, 2, 11, 76, 2, 11, 84, 2, 147, 113, 73, 2, 11, 88, 2, - 11, 32, 2, 11, 83, 2, 21, 3, 80, 79, 75, 2, 17, 2, 69, 68, 2, 11, 32, 2, - 11, 65, 2, 11, 83, 2, 129, 78, 3, 84, 69, 82, 2, 165, 13, 3, 76, 73, 68, - 4, 21, 3, 72, 73, 84, 4, 11, 69, 4, 11, 32, 4, 190, 64, 67, 199, 46, 83, - 4, 11, 72, 4, 11, 32, 4, 18, 71, 39, 76, 2, 69, 6, 82, 69, 65, 84, 69, - 82, 2, 11, 69, 2, 11, 83, 2, 11, 83, 2, 17, 2, 45, 84, 2, 239, 70, 72, - 160, 1, 140, 1, 9, 66, 82, 65, 84, 73, 79, 78, 32, 77, 34, 67, 32, 3, 68, - 69, 79, 126, 69, 218, 1, 79, 22, 82, 21, 7, 84, 72, 75, 85, 81, 73, 32, - 2, 11, 79, 2, 159, 246, 1, 68, 2, 221, 126, 4, 84, 79, 82, 89, 6, 30, 32, - 77, 3, 67, 65, 83, 4, 18, 67, 43, 71, 2, 17, 2, 65, 77, 2, 195, 199, 1, - 69, 2, 143, 116, 65, 2, 147, 71, 83, 6, 168, 1, 31, 84, 78, 65, 77, 69, - 83, 69, 32, 65, 76, 84, 69, 82, 78, 65, 84, 69, 32, 82, 69, 65, 68, 73, - 78, 71, 32, 77, 65, 82, 75, 32, 193, 105, 5, 87, 68, 65, 84, 65, 4, 26, - 78, 199, 135, 2, 67, 2, 143, 126, 72, 2, 175, 138, 1, 76, 2, 227, 234, 1, - 71, 140, 1, 56, 6, 67, 65, 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 70, 45, - 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 70, 230, 1, 66, 34, 69, 22, 73, - 22, 76, 34, 78, 254, 133, 1, 67, 2, 68, 2, 83, 2, 84, 214, 56, 72, 226, - 48, 70, 2, 74, 2, 77, 2, 80, 2, 82, 2, 86, 2, 88, 2, 90, 158, 20, 71, 2, - 75, 2, 81, 186, 2, 65, 2, 79, 2, 85, 3, 89, 4, 150, 240, 1, 66, 215, 22, - 69, 5, 203, 134, 2, 73, 5, 227, 239, 1, 74, 4, 234, 131, 2, 76, 187, 2, - 65, 4, 174, 239, 1, 74, 215, 22, 69, 8, 28, 3, 73, 68, 69, 59, 76, 2, 21, - 3, 68, 32, 71, 2, 177, 71, 4, 82, 69, 69, 75, 6, 46, 67, 20, 3, 76, 69, - 89, 41, 2, 85, 77, 2, 203, 219, 1, 65, 2, 11, 66, 2, 11, 65, 2, 147, 123, - 76, 2, 11, 69, 2, 17, 2, 32, 73, 2, 201, 122, 4, 78, 84, 69, 71, 5, 223, - 131, 2, 83, 40, 70, 67, 45, 13, 71, 65, 82, 32, 70, 82, 65, 67, 84, 73, - 79, 78, 32, 2, 11, 65, 2, 11, 78, 2, 155, 197, 1, 85, 38, 106, 70, 96, 4, - 79, 78, 69, 32, 174, 2, 84, 80, 7, 83, 69, 86, 69, 78, 32, 69, 145, 33, - 3, 90, 69, 82, 6, 56, 4, 73, 86, 69, 32, 249, 3, 5, 79, 85, 82, 32, 70, - 4, 158, 3, 69, 113, 3, 83, 73, 88, 18, 66, 69, 28, 2, 70, 73, 18, 72, 30, - 78, 14, 81, 30, 83, 55, 84, 2, 193, 1, 3, 73, 71, 72, 2, 167, 1, 70, 2, - 11, 65, 2, 131, 127, 76, 2, 111, 73, 2, 165, 73, 3, 85, 65, 82, 4, 24, 2, - 69, 86, 15, 73, 2, 43, 69, 2, 43, 88, 4, 18, 69, 35, 72, 2, 11, 78, 2, - 243, 253, 1, 84, 2, 155, 104, 73, 10, 48, 5, 72, 82, 69, 69, 32, 93, 3, - 87, 79, 32, 6, 26, 69, 26, 81, 67, 70, 2, 109, 3, 73, 71, 72, 2, 21, 3, - 85, 65, 82, 2, 151, 111, 84, 4, 22, 70, 211, 32, 84, 2, 11, 73, 2, 11, - 70, 2, 205, 191, 1, 2, 84, 72, 156, 6, 86, 65, 242, 23, 69, 158, 3, 72, - 142, 66, 73, 234, 8, 79, 134, 6, 82, 167, 144, 1, 74, 204, 2, 122, 78, - 220, 5, 2, 88, 73, 158, 1, 82, 210, 10, 84, 194, 1, 86, 252, 70, 2, 70, - 70, 233, 78, 6, 83, 84, 69, 66, 65, 83, 122, 36, 4, 67, 72, 79, 32, 183, - 5, 73, 118, 100, 7, 76, 69, 84, 84, 69, 82, 32, 252, 3, 3, 78, 71, 85, - 16, 5, 84, 79, 78, 69, 32, 243, 7, 68, 88, 210, 1, 65, 38, 79, 34, 69, - 22, 73, 22, 76, 50, 78, 42, 84, 50, 85, 22, 89, 138, 175, 1, 75, 2, 80, - 2, 83, 254, 68, 66, 2, 67, 2, 68, 2, 70, 2, 71, 2, 72, 2, 74, 2, 77, 2, - 82, 2, 86, 2, 87, 3, 90, 13, 34, 65, 206, 247, 1, 78, 87, 85, 7, 11, 78, - 5, 147, 248, 1, 71, 5, 255, 247, 1, 78, 5, 151, 247, 1, 78, 4, 26, 76, - 191, 247, 1, 65, 2, 131, 245, 1, 72, 6, 238, 244, 1, 71, 2, 89, 187, 2, - 65, 8, 198, 244, 1, 72, 2, 82, 2, 83, 187, 2, 65, 5, 251, 166, 1, 69, 4, - 170, 245, 1, 73, 147, 1, 65, 2, 179, 111, 78, 8, 40, 3, 75, 79, 73, 1, 3, - 84, 85, 80, 5, 135, 226, 1, 78, 4, 21, 3, 78, 71, 32, 4, 76, 8, 67, 82, - 69, 83, 67, 69, 78, 84, 1, 7, 71, 73, 66, 66, 79, 85, 83, 2, 21, 3, 32, - 77, 79, 2, 11, 79, 2, 175, 97, 78, 170, 1, 72, 9, 65, 78, 71, 32, 67, 73, - 84, 73, 32, 249, 108, 4, 78, 73, 78, 71, 168, 1, 128, 1, 6, 67, 65, 80, - 73, 84, 65, 0, 4, 83, 77, 65, 76, 190, 4, 68, 156, 2, 7, 78, 85, 77, 66, - 69, 82, 32, 171, 219, 1, 79, 64, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, - 32, 64, 174, 1, 65, 38, 69, 42, 72, 74, 78, 50, 79, 22, 83, 50, 85, 30, - 89, 170, 42, 84, 248, 107, 2, 86, 73, 206, 51, 66, 2, 80, 246, 5, 75, - 158, 11, 73, 2, 87, 162, 17, 68, 3, 71, 9, 162, 240, 1, 78, 86, 77, 3, - 84, 7, 11, 78, 4, 198, 240, 1, 78, 3, 89, 8, 38, 79, 210, 151, 1, 73, - 231, 31, 65, 4, 178, 183, 1, 82, 223, 25, 76, 4, 26, 71, 223, 158, 1, 85, - 2, 143, 237, 1, 65, 5, 159, 169, 1, 68, 4, 26, 83, 175, 219, 1, 73, 2, - 155, 202, 1, 85, 4, 238, 238, 1, 67, 3, 85, 8, 34, 85, 178, 238, 1, 65, - 3, 79, 5, 175, 238, 1, 74, 20, 11, 73, 20, 11, 71, 20, 17, 2, 73, 84, 20, - 11, 32, 20, 66, 70, 30, 83, 42, 84, 62, 90, 130, 83, 78, 14, 79, 199, - 110, 69, 4, 218, 112, 73, 131, 4, 79, 4, 22, 69, 227, 96, 73, 2, 247, 27, - 86, 4, 26, 72, 207, 205, 1, 87, 2, 11, 82, 2, 223, 213, 1, 69, 2, 11, 69, - 2, 159, 205, 1, 82, 18, 42, 69, 30, 70, 42, 78, 38, 83, 39, 84, 2, 221, - 1, 3, 73, 71, 72, 4, 22, 73, 139, 1, 79, 2, 171, 1, 70, 2, 17, 2, 73, 78, - 2, 135, 1, 69, 4, 92, 2, 69, 86, 25, 2, 73, 88, 6, 34, 72, 26, 87, 187, - 154, 1, 69, 2, 11, 73, 2, 35, 82, 2, 11, 69, 2, 11, 78, 2, 171, 216, 1, - 84, 12, 32, 2, 69, 82, 175, 232, 1, 67, 10, 34, 32, 173, 153, 1, 2, 77, - 69, 8, 80, 3, 67, 76, 79, 22, 87, 224, 197, 1, 5, 66, 85, 70, 70, 65, 1, - 2, 80, 79, 2, 183, 179, 1, 83, 2, 235, 107, 65, 20, 60, 2, 69, 32, 124, - 4, 73, 78, 71, 32, 161, 1, 2, 89, 32, 6, 182, 2, 68, 157, 186, 1, 23, 65, - 82, 82, 79, 87, 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 68, 73, 82, 69, - 67, 84, 76, 89, 6, 64, 5, 66, 76, 65, 67, 75, 0, 5, 87, 72, 73, 84, 69, - 55, 72, 2, 17, 2, 32, 70, 2, 11, 76, 2, 171, 229, 1, 65, 2, 11, 65, 2, - 11, 78, 2, 235, 94, 68, 8, 26, 68, 34, 76, 43, 79, 2, 11, 65, 2, 139, - 228, 1, 83, 4, 22, 79, 223, 75, 73, 2, 183, 75, 87, 2, 11, 86, 2, 11, 69, - 2, 155, 75, 82, 14, 48, 3, 65, 82, 89, 70, 68, 70, 73, 171, 1, 83, 4, 11, - 32, 4, 32, 2, 67, 65, 219, 166, 1, 70, 2, 191, 166, 1, 84, 4, 26, 71, - 175, 146, 1, 68, 2, 149, 67, 6, 69, 45, 84, 65, 73, 76, 4, 116, 17, 69, - 82, 83, 84, 82, 65, 83, 83, 32, 69, 76, 76, 73, 80, 84, 73, 67, 185, 43, - 7, 71, 72, 84, 32, 76, 73, 70, 2, 17, 2, 32, 70, 2, 157, 145, 1, 2, 85, - 78, 2, 37, 7, 84, 32, 83, 89, 82, 73, 65, 2, 179, 35, 67, 180, 2, 52, 3, - 69, 69, 76, 100, 3, 73, 84, 69, 219, 62, 65, 7, 60, 7, 32, 79, 70, 32, - 68, 72, 65, 21, 4, 67, 72, 65, 73, 2, 143, 191, 1, 82, 2, 251, 76, 82, - 172, 2, 54, 32, 181, 63, 8, 45, 70, 69, 65, 84, 72, 69, 82, 170, 2, 148, - 2, 18, 65, 82, 82, 79, 87, 32, 83, 72, 65, 70, 84, 32, 87, 73, 68, 84, - 72, 32, 114, 66, 46, 67, 194, 15, 68, 166, 5, 69, 50, 70, 178, 3, 72, - 222, 3, 76, 218, 3, 77, 226, 1, 78, 34, 80, 146, 1, 81, 78, 82, 250, 1, - 83, 142, 12, 84, 228, 3, 2, 85, 80, 217, 3, 3, 86, 69, 82, 4, 22, 84, - 251, 67, 79, 2, 11, 87, 2, 21, 3, 79, 32, 84, 2, 17, 2, 72, 73, 2, 11, - 82, 2, 247, 158, 1, 68, 2, 11, 85, 2, 11, 76, 2, 143, 167, 1, 76, 92, - 146, 1, 72, 168, 10, 5, 73, 82, 67, 76, 69, 162, 3, 76, 25, 20, 79, 78, - 67, 65, 86, 69, 45, 83, 73, 68, 69, 68, 32, 68, 73, 65, 77, 79, 78, 68, - 66, 25, 4, 69, 83, 83, 32, 66, 66, 66, 38, 69, 118, 75, 142, 4, 80, 22, - 81, 38, 82, 131, 2, 84, 6, 241, 5, 5, 73, 83, 72, 79, 80, 4, 45, 9, 81, - 85, 73, 72, 79, 80, 80, 69, 82, 5, 17, 2, 32, 82, 2, 129, 6, 8, 79, 84, - 65, 84, 69, 68, 32, 78, 26, 38, 73, 25, 5, 78, 73, 71, 72, 84, 6, 177, 4, - 2, 78, 71, 21, 22, 32, 151, 3, 45, 12, 41, 8, 82, 79, 84, 65, 84, 69, 68, - 32, 12, 104, 2, 70, 79, 0, 15, 79, 78, 69, 32, 72, 85, 78, 68, 82, 69, - 68, 32, 84, 72, 73, 18, 84, 215, 3, 78, 2, 207, 1, 82, 6, 148, 1, 11, 87, - 79, 32, 72, 85, 78, 68, 82, 69, 68, 32, 133, 3, 20, 72, 82, 69, 69, 32, - 72, 85, 78, 68, 82, 69, 68, 32, 70, 73, 70, 84, 69, 69, 78, 4, 36, 4, 84, - 87, 69, 78, 175, 2, 83, 2, 217, 2, 7, 84, 89, 45, 70, 73, 86, 69, 6, 166, - 3, 66, 94, 81, 47, 82, 6, 41, 2, 65, 87, 6, 21, 3, 85, 69, 69, 6, 35, 78, - 6, 21, 3, 79, 79, 75, 7, 45, 9, 32, 82, 79, 84, 65, 84, 69, 68, 32, 4, - 70, 78, 25, 13, 84, 87, 79, 32, 72, 85, 78, 68, 82, 69, 68, 32, 83, 2, - 49, 3, 73, 78, 69, 2, 25, 4, 69, 86, 69, 78, 2, 17, 2, 84, 89, 2, 253, - 62, 6, 32, 68, 69, 71, 82, 69, 12, 33, 6, 85, 82, 78, 69, 68, 32, 12, 42, - 66, 30, 75, 34, 80, 34, 81, 47, 82, 2, 225, 88, 3, 73, 83, 72, 4, 222, - 128, 1, 73, 159, 38, 78, 2, 11, 65, 2, 203, 129, 1, 87, 2, 11, 85, 2, 11, - 69, 2, 159, 129, 1, 69, 2, 255, 166, 1, 79, 19, 11, 32, 16, 100, 16, 67, - 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, 65, 67, 75, 121, 5, 87, - 73, 84, 72, 32, 2, 17, 2, 32, 83, 2, 11, 77, 2, 21, 3, 65, 76, 76, 2, 11, - 32, 2, 11, 67, 2, 11, 73, 2, 11, 82, 2, 203, 45, 67, 14, 56, 2, 68, 79, - 70, 84, 148, 30, 2, 76, 79, 231, 1, 85, 4, 18, 84, 35, 87, 2, 11, 32, 2, - 179, 163, 1, 82, 2, 179, 51, 78, 2, 11, 87, 2, 11, 79, 2, 11, 32, 2, 155, - 57, 68, 2, 141, 26, 2, 85, 66, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 148, - 29, 2, 76, 69, 49, 2, 82, 73, 28, 76, 6, 73, 65, 77, 79, 78, 68, 216, 2, - 3, 79, 87, 78, 177, 1, 2, 82, 65, 15, 11, 32, 12, 160, 1, 17, 67, 79, 78, - 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, 65, 67, 75, 32, 128, 1, 9, 87, - 73, 84, 72, 32, 67, 69, 78, 84, 198, 22, 83, 173, 18, 2, 73, 78, 6, 74, - 83, 0, 6, 86, 69, 82, 89, 32, 83, 29, 6, 77, 69, 68, 73, 85, 77, 2, 25, - 4, 77, 65, 76, 76, 2, 181, 14, 2, 32, 68, 2, 11, 82, 2, 11, 69, 2, 179, - 56, 68, 10, 96, 10, 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 53, 10, 45, - 80, 79, 73, 78, 84, 73, 78, 71, 32, 6, 250, 34, 66, 24, 5, 76, 69, 70, - 84, 32, 51, 73, 4, 138, 36, 83, 51, 84, 4, 33, 6, 85, 71, 72, 84, 83, 32, - 4, 22, 77, 191, 118, 75, 2, 223, 119, 65, 2, 29, 5, 88, 67, 76, 65, 77, - 2, 155, 14, 65, 14, 74, 76, 144, 2, 11, 79, 85, 82, 32, 80, 79, 73, 78, - 84, 69, 68, 71, 82, 8, 28, 2, 65, 71, 175, 1, 79, 5, 149, 1, 34, 32, 87, - 73, 84, 72, 32, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 77, 73, 68, - 68, 76, 69, 32, 66, 76, 65, 67, 75, 32, 83, 84, 82, 73, 2, 155, 174, 1, - 80, 4, 26, 82, 227, 134, 1, 87, 2, 17, 2, 69, 84, 2, 215, 173, 1, 84, 4, - 11, 32, 4, 18, 67, 23, 83, 2, 223, 195, 1, 85, 2, 187, 33, 84, 2, 183, - 59, 79, 16, 34, 65, 150, 1, 69, 219, 1, 79, 2, 25, 4, 82, 68, 32, 83, 2, - 45, 9, 72, 69, 76, 76, 32, 70, 76, 79, 80, 2, 17, 2, 80, 89, 2, 17, 2, - 32, 68, 2, 11, 73, 2, 135, 191, 1, 83, 10, 22, 65, 247, 10, 88, 8, 30, - 82, 29, 3, 86, 89, 32, 4, 11, 84, 5, 183, 14, 32, 4, 74, 67, 41, 14, 83, - 65, 76, 84, 73, 82, 69, 32, 87, 73, 84, 72, 32, 82, 2, 21, 3, 72, 69, 67, - 2, 155, 127, 75, 2, 139, 18, 79, 4, 152, 30, 10, 82, 73, 90, 79, 78, 84, - 65, 76, 32, 69, 229, 10, 2, 85, 82, 18, 170, 1, 65, 116, 3, 69, 70, 84, - 165, 9, 31, 79, 90, 69, 78, 71, 69, 32, 67, 79, 78, 84, 65, 73, 78, 73, - 78, 71, 32, 66, 76, 65, 67, 75, 32, 83, 77, 65, 76, 76, 32, 76, 4, 24, 2, - 82, 71, 19, 84, 2, 199, 29, 69, 2, 11, 73, 2, 11, 78, 2, 17, 2, 32, 67, - 2, 11, 82, 2, 159, 39, 79, 12, 58, 32, 77, 10, 45, 80, 79, 73, 78, 84, - 73, 78, 71, 32, 6, 44, 6, 76, 65, 78, 69, 32, 77, 223, 22, 80, 2, 11, 69, - 2, 219, 8, 82, 6, 30, 80, 166, 24, 83, 51, 84, 2, 187, 5, 79, 12, 68, 6, - 69, 68, 73, 85, 77, 32, 121, 7, 79, 79, 78, 32, 83, 69, 76, 10, 30, 68, - 54, 83, 211, 6, 76, 2, 11, 73, 2, 11, 65, 2, 11, 77, 2, 163, 45, 79, 6, - 202, 24, 84, 186, 1, 77, 59, 81, 2, 11, 69, 2, 139, 184, 1, 78, 2, 11, - 73, 2, 163, 186, 1, 66, 6, 18, 65, 87, 69, 2, 37, 7, 82, 65, 76, 76, 69, - 76, 79, 2, 11, 71, 2, 11, 82, 2, 227, 168, 1, 65, 4, 11, 78, 4, 154, 2, - 84, 191, 45, 78, 2, 21, 3, 85, 69, 83, 2, 145, 117, 9, 84, 73, 79, 78, - 32, 77, 65, 82, 75, 14, 36, 4, 73, 71, 72, 84, 179, 22, 69, 12, 60, 10, - 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 195, 17, 32, 8, 30, 80, 178, 19, - 83, 51, 84, 4, 18, 69, 55, 79, 2, 11, 78, 2, 11, 84, 2, 11, 65, 2, 147, - 103, 71, 2, 11, 73, 2, 11, 78, 2, 235, 120, 84, 56, 118, 67, 32, 3, 69, - 83, 65, 18, 72, 58, 77, 170, 1, 80, 68, 5, 81, 85, 65, 82, 69, 168, 6, 2, - 85, 78, 163, 10, 84, 2, 133, 39, 4, 73, 83, 83, 79, 2, 155, 36, 77, 2, - 21, 3, 79, 71, 73, 2, 137, 120, 4, 32, 80, 73, 69, 8, 32, 4, 65, 76, 76, - 32, 119, 73, 6, 18, 76, 71, 83, 2, 11, 79, 2, 11, 90, 2, 11, 69, 2, 11, - 78, 2, 159, 157, 1, 71, 4, 182, 17, 84, 243, 1, 81, 2, 187, 43, 76, 2, - 21, 3, 65, 68, 69, 2, 11, 32, 2, 11, 83, 2, 203, 150, 1, 85, 29, 11, 32, + 32, 130, 4, 84, 222, 149, 2, 73, 149, 70, 5, 75, 89, 79, 32, 84, 104, + 242, 1, 71, 42, 74, 30, 77, 34, 78, 70, 72, 34, 80, 34, 83, 210, 50, 82, + 206, 147, 1, 79, 134, 60, 69, 42, 76, 198, 2, 65, 178, 191, 1, 67, 2, 68, + 2, 75, 2, 84, 2, 88, 2, 90, 138, 69, 66, 2, 70, 2, 81, 2, 86, 186, 2, 73, + 2, 85, 3, 89, 6, 170, 139, 4, 72, 2, 74, 187, 2, 65, 4, 186, 141, 4, 65, + 3, 89, 4, 230, 138, 4, 66, 187, 2, 65, 14, 66, 71, 190, 194, 2, 74, 194, + 130, 1, 88, 138, 69, 68, 187, 2, 65, 4, 130, 138, 4, 74, 187, 2, 65, 4, + 226, 137, 4, 83, 187, 2, 65, 10, 54, 72, 198, 193, 2, 75, 202, 199, 1, + 84, 187, 2, 65, 4, 138, 137, 4, 84, 187, 2, 65, 2, 231, 28, 65, 2, 11, + 71, 2, 179, 244, 3, 85, 6, 32, 2, 84, 72, 151, 254, 2, 76, 5, 11, 66, 2, + 11, 82, 2, 151, 162, 2, 85, 72, 252, 1, 4, 65, 82, 67, 32, 162, 2, 67, + 44, 2, 72, 65, 166, 3, 80, 140, 3, 13, 74, 85, 83, 84, 73, 70, 73, 69, + 68, 32, 76, 79, 87, 84, 5, 76, 69, 70, 84, 32, 228, 1, 6, 82, 73, 71, 72, + 84, 32, 170, 2, 83, 38, 84, 81, 7, 87, 73, 84, 72, 32, 85, 80, 6, 180, 1, + 19, 65, 78, 84, 73, 67, 76, 79, 67, 75, 87, 73, 83, 69, 32, 65, 82, 82, + 79, 87, 73, 21, 67, 76, 79, 67, 75, 87, 73, 83, 69, 32, 65, 82, 82, 79, + 87, 32, 87, 73, 84, 72, 32, 5, 29, 5, 32, 87, 73, 84, 72, 2, 17, 2, 32, + 80, 2, 187, 128, 2, 76, 2, 11, 77, 2, 143, 128, 2, 73, 2, 11, 85, 2, 133, + 179, 3, 3, 82, 76, 89, 34, 36, 3, 76, 70, 32, 179, 133, 4, 84, 32, 70, + 70, 186, 1, 76, 22, 82, 178, 2, 83, 250, 5, 66, 231, 243, 1, 73, 8, 136, + 1, 15, 79, 82, 87, 65, 82, 68, 45, 70, 65, 67, 73, 78, 71, 32, 82, 157, + 2, 13, 76, 65, 73, 76, 73, 78, 71, 32, 82, 79, 66, 79, 84, 4, 22, 85, + 243, 1, 79, 2, 203, 197, 3, 78, 8, 41, 2, 69, 70, 8, 21, 3, 73, 71, 72, + 8, 11, 84, 8, 54, 32, 69, 9, 45, 70, 65, 67, 73, 78, 71, 32, 82, 2, 11, + 80, 2, 33, 6, 65, 82, 69, 78, 84, 72, 2, 207, 222, 1, 69, 6, 38, 79, 21, + 5, 85, 78, 78, 69, 82, 2, 155, 177, 3, 66, 4, 29, 5, 32, 70, 82, 65, 77, + 4, 11, 69, 4, 11, 45, 4, 134, 129, 4, 49, 3, 50, 4, 18, 69, 59, 84, 2, + 11, 67, 2, 11, 84, 2, 11, 73, 2, 147, 135, 2, 79, 2, 17, 2, 65, 78, 2, + 11, 68, 2, 21, 3, 73, 78, 71, 2, 17, 2, 32, 80, 2, 11, 69, 2, 11, 82, 2, + 207, 175, 3, 83, 4, 17, 2, 69, 82, 4, 33, 6, 32, 72, 65, 76, 70, 32, 4, + 250, 3, 66, 139, 105, 87, 10, 180, 1, 13, 74, 85, 83, 84, 73, 70, 73, 69, + 68, 32, 76, 79, 87, 126, 67, 50, 72, 225, 214, 2, 21, 66, 76, 65, 67, 75, + 32, 76, 69, 70, 84, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 83, 2, 237, + 1, 7, 69, 82, 32, 82, 73, 71, 72, 8, 78, 67, 50, 72, 33, 13, 74, 85, 83, + 84, 73, 70, 73, 69, 68, 32, 76, 79, 87, 4, 26, 79, 243, 130, 3, 82, 2, + 139, 190, 3, 82, 2, 253, 168, 3, 3, 65, 76, 70, 2, 33, 6, 69, 82, 32, 76, + 69, 70, 2, 53, 11, 84, 32, 81, 85, 65, 82, 84, 69, 82, 32, 66, 2, 11, 76, + 2, 209, 168, 2, 3, 65, 67, 75, 2, 201, 167, 3, 4, 81, 85, 65, 82, 2, 11, + 79, 2, 165, 167, 3, 12, 82, 84, 79, 73, 83, 69, 32, 83, 72, 69, 76, 76, + 2, 29, 5, 87, 65, 82, 68, 83, 2, 17, 2, 32, 65, 2, 153, 252, 2, 4, 82, + 82, 79, 87, 20, 188, 1, 22, 67, 74, 75, 32, 85, 78, 73, 70, 73, 69, 68, + 32, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 161, 2, 19, 76, 65, 84, 73, + 78, 32, 67, 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 18, 40, 2, + 52, 69, 34, 53, 54, 54, 87, 55, 4, 202, 1, 48, 203, 205, 2, 56, 4, 30, + 50, 141, 1, 2, 66, 56, 2, 175, 176, 3, 68, 6, 48, 2, 50, 53, 22, 53, 249, + 205, 2, 2, 55, 50, 2, 135, 246, 3, 51, 2, 67, 53, 4, 32, 2, 48, 66, 21, + 2, 54, 68, 2, 195, 245, 3, 57, 2, 175, 245, 3, 55, 2, 11, 82, 2, 179, + 183, 3, 32, 64, 48, 6, 65, 76, 32, 82, 85, 78, 21, 2, 79, 32, 2, 247, + 250, 2, 79, 62, 72, 7, 76, 69, 84, 84, 69, 82, 32, 217, 2, 6, 83, 73, 71, + 78, 32, 82, 60, 206, 1, 66, 106, 78, 138, 30, 73, 202, 141, 1, 69, 162, + 75, 67, 162, 186, 1, 65, 234, 61, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, + 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, 89, 186, 2, 79, 3, 85, 12, + 52, 7, 82, 69, 65, 84, 72, 89, 32, 227, 241, 3, 65, 10, 182, 30, 73, 202, + 141, 1, 69, 143, 175, 2, 65, 4, 246, 238, 3, 71, 187, 2, 65, 2, 17, 2, + 73, 83, 2, 21, 3, 73, 78, 71, 2, 11, 32, 2, 147, 198, 2, 84, 98, 46, 65, + 254, 1, 73, 246, 11, 79, 195, 1, 85, 16, 66, 67, 36, 2, 68, 69, 34, 77, + 20, 2, 78, 83, 155, 159, 3, 73, 4, 246, 231, 1, 75, 243, 206, 1, 84, 2, + 141, 16, 4, 32, 77, 65, 82, 5, 131, 212, 1, 32, 4, 44, 2, 80, 76, 33, 5, + 86, 69, 82, 83, 65, 2, 11, 85, 2, 199, 207, 3, 84, 2, 171, 62, 76, 66, + 156, 1, 3, 65, 78, 71, 176, 3, 8, 68, 69, 78, 84, 32, 69, 77, 66, 20, 9, + 71, 82, 65, 77, 32, 70, 79, 82, 32, 140, 2, 4, 80, 76, 69, 32, 167, 151, + 3, 67, 16, 48, 2, 76, 69, 245, 1, 5, 85, 76, 65, 82, 32, 10, 44, 6, 32, + 87, 73, 84, 72, 32, 163, 1, 45, 8, 80, 9, 83, 69, 82, 73, 70, 83, 32, 65, + 84, 54, 85, 138, 168, 2, 82, 187, 66, 68, 2, 17, 2, 32, 66, 2, 197, 218, + 3, 3, 79, 84, 84, 2, 241, 60, 2, 78, 68, 2, 205, 202, 2, 4, 72, 69, 65, + 68, 6, 68, 9, 70, 76, 65, 71, 32, 79, 78, 32, 80, 34, 82, 179, 138, 2, + 66, 2, 11, 79, 2, 207, 205, 3, 83, 2, 11, 85, 2, 239, 171, 3, 76, 2, 155, + 215, 3, 76, 16, 66, 69, 34, 72, 34, 76, 22, 77, 46, 84, 42, 87, 255, 210, + 1, 70, 2, 11, 65, 2, 211, 230, 1, 82, 2, 11, 69, 2, 211, 248, 1, 65, 2, + 215, 184, 3, 65, 2, 21, 3, 79, 85, 78, 2, 175, 187, 2, 84, 2, 17, 2, 72, + 85, 2, 215, 199, 1, 78, 4, 206, 175, 2, 65, 131, 42, 73, 30, 176, 2, 3, + 67, 79, 76, 24, 20, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 66, 65, + 82, 32, 87, 73, 84, 72, 32, 98, 80, 34, 84, 20, 13, 86, 69, 82, 84, 73, + 67, 65, 76, 32, 66, 65, 82, 32, 44, 9, 83, 79, 76, 73, 68, 85, 83, 32, + 66, 188, 21, 2, 68, 65, 168, 187, 1, 6, 78, 69, 83, 84, 69, 68, 171, 11, + 73, 2, 129, 37, 2, 79, 78, 4, 34, 68, 33, 4, 84, 82, 73, 80, 2, 17, 2, + 79, 85, 2, 11, 66, 2, 185, 179, 3, 2, 76, 69, 4, 254, 221, 1, 76, 135, + 110, 82, 2, 167, 202, 1, 73, 8, 42, 66, 74, 68, 134, 189, 1, 82, 115, 87, + 2, 29, 5, 73, 78, 65, 82, 89, 2, 21, 3, 32, 82, 69, 2, 235, 32, 76, 2, + 25, 4, 69, 76, 73, 77, 2, 191, 170, 2, 73, 10, 24, 2, 76, 76, 35, 80, 5, + 137, 220, 1, 3, 69, 89, 66, 6, 44, 5, 73, 67, 65, 76, 32, 243, 207, 3, + 72, 4, 18, 68, 47, 70, 2, 11, 82, 2, 11, 73, 2, 243, 221, 3, 78, 2, 215, + 248, 1, 73, 6, 18, 69, 79, 77, 5, 129, 188, 3, 14, 32, 76, 73, 71, 72, + 84, 32, 77, 79, 79, 78, 32, 65, 82, 2, 235, 170, 3, 80, 216, 1, 76, 3, + 71, 82, 73, 22, 76, 214, 12, 82, 221, 186, 2, 5, 77, 66, 76, 69, 82, 2, + 155, 216, 2, 75, 162, 1, 68, 11, 85, 45, 84, 73, 71, 65, 76, 65, 82, 73, + 32, 179, 222, 3, 73, 160, 1, 122, 65, 42, 67, 30, 68, 106, 71, 32, 7, 76, + 69, 84, 84, 69, 82, 32, 178, 3, 82, 32, 5, 83, 73, 71, 78, 32, 131, 3, + 86, 2, 11, 85, 2, 177, 155, 3, 2, 32, 76, 2, 245, 158, 3, 2, 79, 78, 4, + 18, 79, 67, 65, 2, 11, 85, 2, 11, 66, 2, 25, 4, 76, 69, 32, 68, 2, 11, + 65, 2, 159, 183, 3, 78, 2, 129, 215, 2, 3, 69, 77, 73, 100, 206, 1, 65, + 38, 68, 46, 76, 38, 82, 34, 84, 46, 86, 186, 5, 85, 206, 201, 1, 73, 162, + 193, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 206, 40, + 79, 162, 8, 69, 158, 20, 72, 2, 77, 3, 89, 9, 158, 218, 3, 65, 2, 73, 3, + 85, 8, 186, 146, 3, 68, 138, 69, 72, 187, 2, 65, 6, 154, 208, 1, 76, 183, + 137, 2, 65, 4, 242, 214, 3, 82, 187, 2, 65, 8, 202, 145, 3, 84, 138, 69, + 72, 187, 2, 65, 10, 214, 5, 79, 139, 211, 3, 65, 2, 11, 69, 2, 243, 221, + 1, 80, 18, 174, 1, 65, 72, 6, 76, 79, 79, 80, 69, 68, 40, 2, 79, 77, 0, + 5, 83, 72, 82, 73, 73, 48, 13, 67, 65, 78, 68, 82, 65, 32, 65, 78, 85, + 78, 65, 83, 22, 80, 179, 146, 3, 86, 4, 26, 86, 235, 145, 3, 78, 2, 21, + 3, 65, 71, 82, 2, 251, 219, 1, 65, 2, 17, 2, 32, 86, 2, 187, 157, 1, 73, + 2, 17, 2, 32, 80, 2, 25, 4, 85, 83, 72, 80, 2, 231, 204, 3, 73, 2, 237, + 176, 3, 2, 76, 85, 30, 56, 10, 69, 68, 73, 67, 32, 84, 79, 78, 69, 32, + 35, 79, 4, 226, 164, 1, 65, 235, 4, 83, 26, 45, 9, 87, 69, 76, 32, 83, + 73, 71, 78, 32, 26, 70, 65, 38, 85, 22, 86, 186, 201, 1, 73, 234, 234, 1, + 79, 163, 8, 69, 6, 202, 211, 3, 65, 2, 73, 3, 85, 5, 167, 211, 3, 85, 8, + 11, 79, 8, 33, 6, 67, 65, 76, 73, 67, 32, 8, 26, 82, 159, 214, 2, 76, 5, + 199, 210, 3, 82, 50, 50, 75, 84, 4, 78, 69, 68, 32, 135, 173, 2, 84, 4, + 48, 6, 73, 83, 72, 32, 76, 73, 135, 192, 3, 69, 2, 11, 82, 2, 191, 202, + 2, 65, 44, 238, 1, 65, 80, 6, 66, 76, 65, 67, 75, 32, 40, 7, 87, 72, 73, + 84, 69, 32, 83, 22, 67, 50, 68, 96, 2, 78, 79, 32, 2, 79, 75, 30, 83, + 153, 168, 3, 21, 71, 82, 69, 69, 75, 32, 83, 77, 65, 76, 76, 32, 76, 69, + 84, 84, 69, 82, 32, 73, 79, 4, 26, 77, 239, 170, 2, 78, 2, 17, 2, 80, 69, + 2, 11, 82, 2, 251, 192, 2, 83, 4, 18, 80, 23, 83, 2, 147, 150, 2, 69, 2, + 131, 152, 2, 72, 4, 144, 3, 5, 65, 80, 73, 84, 65, 167, 121, 79, 6, 30, + 65, 33, 3, 73, 71, 73, 2, 11, 71, 2, 231, 143, 3, 71, 4, 221, 221, 1, 3, + 84, 32, 84, 6, 162, 2, 82, 255, 195, 2, 84, 2, 141, 228, 1, 2, 32, 72, + 14, 128, 1, 18, 65, 78, 83, 45, 83, 69, 82, 73, 70, 32, 67, 65, 80, 73, + 84, 65, 76, 32, 38, 69, 32, 3, 77, 65, 76, 33, 2, 79, 85, 6, 222, 203, 3, + 71, 2, 76, 3, 89, 2, 11, 77, 2, 159, 251, 2, 73, 2, 11, 76, 2, 235, 201, + 2, 32, 4, 21, 3, 84, 72, 32, 4, 32, 2, 69, 65, 1, 2, 87, 69, 2, 53, 11, + 83, 84, 32, 80, 79, 73, 78, 84, 73, 78, 71, 2, 17, 2, 32, 76, 2, 195, + 200, 2, 69, 38, 58, 69, 52, 8, 73, 83, 84, 69, 68, 32, 82, 73, 63, 79, 2, + 17, 2, 76, 86, 2, 153, 178, 1, 3, 69, 32, 80, 2, 17, 2, 71, 72, 2, 137, + 109, 6, 84, 87, 65, 82, 68, 83, 34, 30, 32, 229, 10, 2, 45, 69, 32, 134, + 3, 66, 86, 67, 116, 3, 68, 79, 84, 226, 1, 72, 44, 14, 73, 78, 84, 69, + 82, 83, 69, 67, 84, 73, 78, 71, 32, 76, 92, 6, 74, 79, 73, 78, 69, 68, + 64, 8, 76, 79, 71, 73, 67, 65, 76, 32, 98, 77, 0, 3, 87, 79, 77, 116, 13, + 82, 73, 78, 71, 83, 32, 65, 76, 73, 71, 78, 69, 68, 48, 24, 65, 83, 84, + 69, 82, 73, 83, 75, 83, 32, 65, 76, 73, 71, 78, 69, 68, 32, 86, 69, 82, + 84, 73, 67, 35, 83, 2, 29, 5, 85, 84, 84, 79, 78, 2, 17, 2, 32, 77, 2, + 11, 79, 2, 191, 187, 2, 85, 2, 93, 21, 79, 78, 83, 69, 67, 85, 84, 73, + 86, 69, 32, 69, 81, 85, 65, 76, 83, 32, 83, 73, 71, 2, 251, 133, 3, 78, + 6, 18, 32, 55, 83, 4, 22, 76, 131, 1, 80, 2, 145, 163, 1, 2, 69, 65, 2, + 53, 11, 32, 79, 86, 69, 82, 32, 79, 78, 69, 32, 68, 2, 11, 79, 2, 11, 84, + 2, 17, 2, 32, 80, 2, 29, 5, 85, 78, 67, 84, 85, 2, 167, 241, 2, 65, 2, + 11, 69, 2, 11, 65, 2, 159, 173, 2, 82, 4, 17, 2, 79, 71, 4, 25, 4, 73, + 67, 65, 76, 4, 11, 32, 4, 130, 179, 2, 65, 159, 85, 79, 2, 17, 2, 32, 83, + 2, 21, 3, 81, 85, 65, 2, 183, 172, 2, 82, 4, 30, 79, 13, 3, 65, 78, 68, + 2, 11, 82, 2, 11, 32, 2, 11, 79, 2, 11, 80, 2, 143, 18, 69, 2, 11, 69, 2, + 45, 9, 78, 32, 72, 79, 76, 68, 73, 78, 71, 2, 11, 32, 2, 11, 72, 2, 11, + 65, 2, 143, 223, 1, 78, 2, 45, 9, 32, 72, 79, 82, 73, 90, 79, 78, 84, 2, + 11, 65, 2, 219, 154, 3, 76, 2, 49, 10, 80, 69, 69, 67, 72, 32, 66, 85, + 66, 66, 2, 203, 169, 2, 76, 2, 11, 77, 2, 245, 212, 1, 2, 32, 68, 248, 3, + 140, 1, 8, 71, 65, 82, 73, 84, 73, 67, 32, 152, 6, 7, 77, 66, 82, 69, 76, + 76, 65, 166, 1, 78, 158, 8, 80, 138, 166, 1, 82, 151, 134, 2, 83, 62, 48, + 7, 76, 69, 84, 84, 69, 82, 32, 159, 5, 87, 60, 206, 1, 65, 28, 2, 81, 79, + 22, 66, 22, 68, 50, 71, 44, 2, 72, 79, 22, 75, 34, 76, 32, 2, 82, 65, 22, + 83, 74, 84, 74, 89, 22, 90, 150, 182, 2, 78, 202, 91, 80, 246, 5, 87, + 202, 12, 77, 174, 18, 73, 3, 85, 4, 26, 76, 247, 233, 2, 73, 2, 155, 183, + 3, 80, 2, 243, 148, 3, 69, 4, 26, 69, 211, 175, 2, 72, 2, 195, 148, 3, + 76, 4, 166, 140, 2, 72, 189, 138, 1, 2, 65, 77, 5, 151, 182, 3, 84, 4, + 150, 183, 2, 65, 239, 126, 72, 2, 11, 65, 2, 139, 147, 3, 77, 2, 187, + 189, 1, 83, 8, 38, 65, 238, 183, 2, 72, 239, 90, 83, 4, 234, 160, 3, 68, + 239, 13, 77, 6, 38, 72, 206, 154, 3, 69, 175, 28, 79, 2, 11, 65, 2, 231, + 250, 1, 78, 2, 183, 240, 2, 79, 4, 246, 145, 3, 69, 207, 36, 85, 2, 21, + 3, 79, 82, 68, 2, 25, 4, 32, 68, 73, 86, 2, 235, 149, 1, 73, 7, 11, 32, + 4, 84, 4, 79, 78, 32, 71, 45, 13, 87, 73, 84, 72, 32, 82, 65, 73, 78, 32, + 68, 82, 79, 2, 11, 82, 2, 11, 79, 2, 231, 166, 2, 85, 2, 239, 246, 2, 80, + 32, 160, 1, 3, 65, 77, 85, 20, 7, 67, 69, 82, 84, 65, 73, 78, 34, 68, 62, + 73, 245, 5, 18, 77, 65, 82, 82, 73, 69, 68, 32, 80, 65, 82, 84, 78, 69, + 82, 83, 72, 73, 2, 187, 162, 2, 83, 2, 11, 84, 2, 243, 171, 2, 89, 4, 26, + 69, 159, 159, 2, 79, 2, 11, 82, 2, 135, 151, 3, 84, 22, 60, 2, 79, 78, + 234, 3, 84, 106, 86, 173, 58, 3, 67, 79, 82, 15, 11, 32, 12, 160, 1, 6, + 65, 66, 79, 86, 69, 32, 108, 22, 66, 69, 83, 73, 68, 69, 32, 65, 78, 68, + 32, 74, 79, 73, 78, 69, 68, 32, 87, 73, 84, 72, 41, 5, 87, 73, 84, 72, + 32, 4, 52, 9, 66, 65, 82, 32, 65, 66, 79, 86, 69, 23, 73, 2, 17, 2, 32, + 73, 2, 161, 49, 4, 78, 84, 69, 82, 2, 17, 2, 32, 85, 2, 239, 222, 2, 78, + 6, 66, 77, 58, 79, 189, 245, 2, 8, 76, 79, 71, 73, 67, 65, 76, 32, 2, 11, + 73, 2, 11, 78, 2, 11, 85, 2, 183, 167, 2, 83, 2, 11, 86, 2, 209, 140, 2, + 2, 69, 82, 4, 18, 32, 67, 69, 2, 11, 83, 2, 17, 2, 69, 80, 2, 11, 65, 2, + 187, 244, 2, 82, 2, 255, 153, 2, 68, 2, 57, 12, 69, 82, 83, 65, 76, 32, + 82, 69, 67, 89, 67, 76, 2, 17, 2, 73, 78, 2, 155, 153, 2, 71, 2, 135, + 153, 2, 80, 144, 3, 130, 1, 32, 246, 7, 45, 180, 8, 4, 80, 69, 82, 32, + 132, 40, 8, 83, 73, 68, 69, 45, 68, 79, 87, 21, 6, 87, 65, 82, 68, 83, + 32, 38, 140, 1, 5, 65, 82, 82, 79, 87, 224, 1, 5, 66, 65, 82, 66, 32, + 228, 1, 5, 68, 79, 87, 78, 32, 210, 1, 70, 30, 82, 93, 4, 84, 65, 67, 75, + 8, 64, 4, 72, 69, 65, 68, 137, 51, 7, 32, 84, 72, 82, 79, 85, 71, 7, 11, + 32, 4, 112, 22, 66, 69, 84, 87, 69, 69, 78, 32, 84, 87, 79, 32, 72, 79, + 82, 73, 90, 79, 78, 84, 65, 76, 171, 155, 2, 73, 2, 193, 147, 2, 2, 32, + 66, 8, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 4, 57, 12, 84, 32, 68, + 79, 87, 78, 32, 66, 65, 82, 66, 32, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, + 71, 72, 2, 11, 84, 2, 25, 4, 32, 72, 65, 82, 2, 11, 80, 2, 231, 214, 2, + 79, 14, 76, 5, 65, 82, 82, 79, 87, 62, 66, 38, 68, 18, 83, 246, 58, 87, + 219, 9, 84, 5, 37, 7, 32, 87, 73, 84, 72, 32, 66, 2, 187, 156, 2, 65, 2, + 221, 138, 2, 4, 76, 65, 67, 75, 2, 191, 59, 79, 2, 179, 65, 65, 2, 11, + 73, 2, 223, 112, 83, 2, 53, 11, 73, 71, 72, 84, 32, 68, 73, 65, 71, 79, + 78, 2, 189, 128, 1, 4, 65, 76, 32, 69, 5, 29, 5, 32, 87, 73, 84, 72, 2, + 33, 6, 32, 67, 73, 82, 67, 76, 2, 175, 166, 2, 69, 32, 58, 70, 225, 1, 9, + 80, 79, 73, 78, 84, 73, 78, 71, 32, 4, 41, 8, 65, 67, 73, 78, 71, 32, 83, + 78, 4, 65, 14, 65, 75, 69, 32, 72, 69, 65, 68, 32, 87, 73, 84, 72, 32, 4, + 42, 79, 25, 6, 67, 76, 79, 83, 69, 68, 2, 21, 3, 80, 69, 78, 2, 21, 3, + 32, 77, 79, 2, 135, 159, 1, 85, 28, 130, 1, 65, 86, 69, 62, 70, 20, 8, + 77, 73, 76, 73, 84, 65, 82, 89, 26, 82, 88, 6, 83, 77, 65, 76, 76, 32, + 118, 84, 255, 121, 71, 4, 22, 84, 159, 2, 73, 2, 37, 7, 79, 77, 73, 67, + 32, 66, 79, 2, 219, 227, 1, 77, 2, 29, 5, 78, 69, 82, 71, 89, 2, 213, + 179, 1, 2, 32, 87, 2, 143, 158, 3, 82, 2, 129, 1, 2, 32, 65, 8, 64, 5, + 79, 67, 75, 69, 84, 118, 69, 222, 129, 1, 65, 247, 28, 73, 2, 239, 200, + 2, 32, 4, 18, 65, 67, 82, 2, 11, 73, 2, 11, 82, 2, 17, 2, 80, 76, 2, 255, + 131, 2, 65, 2, 11, 69, 2, 151, 248, 1, 68, 4, 37, 7, 82, 73, 65, 78, 71, + 76, 69, 4, 11, 32, 4, 11, 87, 4, 25, 4, 73, 84, 72, 32, 4, 18, 76, 27, + 82, 2, 11, 69, 2, 35, 70, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 17, 2, 32, + 72, 2, 21, 3, 65, 76, 70, 2, 17, 2, 32, 66, 2, 11, 76, 2, 143, 235, 1, + 65, 200, 1, 192, 1, 4, 65, 78, 68, 32, 210, 2, 66, 74, 67, 74, 70, 36, 5, + 72, 65, 76, 70, 32, 200, 4, 5, 76, 69, 70, 84, 32, 254, 5, 77, 150, 2, + 79, 60, 6, 82, 73, 71, 72, 84, 32, 234, 17, 83, 67, 84, 8, 34, 76, 53, 4, + 82, 73, 71, 72, 6, 48, 2, 69, 70, 173, 1, 5, 79, 87, 69, 82, 32, 2, 53, + 11, 84, 32, 65, 78, 68, 32, 76, 79, 87, 69, 82, 2, 201, 35, 25, 32, 84, + 82, 73, 65, 78, 71, 85, 76, 65, 82, 32, 84, 72, 82, 69, 69, 32, 81, 85, + 65, 82, 84, 69, 82, 4, 28, 2, 84, 82, 231, 39, 79, 2, 225, 4, 7, 73, 65, + 78, 71, 85, 76, 65, 2, 17, 2, 76, 65, 2, 17, 2, 68, 69, 2, 185, 223, 1, + 3, 32, 83, 67, 10, 33, 6, 69, 78, 84, 82, 69, 32, 10, 174, 12, 76, 22, + 82, 251, 21, 79, 2, 17, 2, 73, 86, 2, 179, 31, 69, 20, 140, 1, 5, 66, 76, + 79, 67, 75, 110, 72, 32, 8, 73, 78, 86, 69, 82, 83, 69, 32, 198, 1, 86, + 170, 26, 77, 130, 3, 76, 22, 82, 139, 162, 1, 67, 5, 225, 28, 23, 32, 65, + 78, 68, 32, 76, 79, 87, 69, 82, 32, 72, 65, 76, 70, 32, 73, 78, 86, 69, + 82, 83, 69, 2, 225, 18, 4, 69, 65, 86, 89, 4, 100, 21, 77, 69, 68, 73, + 85, 77, 32, 83, 72, 65, 68, 69, 32, 65, 78, 68, 32, 76, 79, 87, 69, 51, + 87, 2, 11, 82, 2, 185, 35, 5, 32, 72, 65, 76, 70, 2, 21, 3, 72, 73, 84, + 2, 191, 191, 1, 69, 2, 29, 5, 69, 82, 84, 73, 67, 2, 197, 115, 14, 65, + 76, 32, 76, 73, 78, 69, 32, 87, 73, 84, 72, 32, 84, 62, 50, 66, 238, 3, + 67, 54, 79, 74, 84, 179, 13, 81, 22, 65, 14, 76, 79, 67, 75, 32, 68, 73, + 65, 71, 79, 78, 65, 76, 32, 22, 144, 1, 6, 76, 79, 87, 69, 82, 32, 233, + 8, 24, 85, 80, 80, 69, 82, 32, 77, 73, 68, 68, 76, 69, 32, 76, 69, 70, + 84, 32, 84, 79, 32, 85, 80, 80, 18, 176, 1, 10, 67, 69, 78, 84, 82, 69, + 32, 84, 79, 32, 36, 8, 76, 69, 70, 84, 32, 84, 79, 32, 229, 10, 18, 77, + 73, 68, 68, 76, 69, 32, 76, 69, 70, 84, 32, 84, 79, 32, 85, 80, 80, 6, + 70, 76, 253, 8, 3, 85, 80, 80, 6, 34, 76, 213, 9, 3, 85, 80, 80, 2, 149, + 10, 2, 79, 87, 2, 29, 5, 79, 82, 78, 69, 82, 2, 155, 232, 1, 32, 4, 182, + 12, 78, 53, 12, 82, 32, 76, 79, 87, 69, 82, 32, 82, 73, 71, 72, 8, 34, + 79, 166, 19, 82, 155, 1, 87, 2, 237, 18, 11, 32, 76, 79, 87, 69, 82, 32, + 82, 73, 71, 72, 12, 33, 6, 73, 68, 68, 76, 69, 32, 12, 52, 7, 67, 69, 78, + 84, 82, 69, 32, 74, 76, 23, 82, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, + 72, 2, 117, 3, 84, 32, 79, 4, 41, 2, 69, 70, 4, 21, 3, 73, 71, 72, 4, 17, + 2, 84, 32, 4, 30, 79, 253, 17, 2, 84, 87, 2, 139, 9, 78, 4, 11, 78, 4, + 17, 2, 69, 32, 4, 150, 21, 81, 147, 4, 69, 68, 84, 2, 66, 76, 186, 6, 68, + 114, 79, 222, 1, 80, 38, 81, 224, 4, 2, 83, 72, 51, 84, 22, 61, 13, 79, + 67, 75, 32, 68, 73, 65, 71, 79, 78, 65, 76, 32, 22, 140, 1, 24, 76, 79, + 87, 69, 82, 32, 77, 73, 68, 68, 76, 69, 32, 76, 69, 70, 84, 32, 84, 79, + 32, 76, 79, 87, 57, 6, 85, 80, 80, 69, 82, 32, 4, 21, 3, 69, 82, 32, 4, + 246, 3, 67, 135, 215, 2, 82, 18, 176, 1, 10, 67, 69, 78, 84, 82, 69, 32, + 84, 79, 32, 92, 8, 76, 69, 70, 84, 32, 84, 79, 32, 141, 1, 18, 77, 73, + 68, 68, 76, 69, 32, 76, 69, 70, 84, 32, 84, 79, 32, 76, 79, 87, 6, 32, 3, + 76, 79, 87, 139, 1, 85, 4, 21, 3, 69, 82, 32, 4, 142, 2, 77, 223, 214, 2, + 82, 6, 28, 3, 76, 79, 87, 51, 85, 4, 21, 3, 69, 82, 32, 4, 142, 1, 67, + 43, 77, 2, 17, 2, 80, 80, 2, 17, 2, 69, 82, 2, 117, 2, 32, 77, 6, 21, 3, + 69, 82, 32, 6, 34, 67, 42, 77, 223, 214, 2, 82, 2, 11, 69, 2, 141, 225, + 1, 2, 78, 84, 2, 25, 4, 73, 68, 68, 76, 2, 139, 176, 1, 69, 2, 57, 12, + 82, 79, 80, 45, 83, 72, 65, 68, 79, 87, 69, 68, 2, 17, 2, 32, 87, 2, 213, + 190, 1, 3, 72, 73, 84, 4, 62, 78, 53, 11, 82, 32, 76, 79, 87, 69, 82, 32, + 76, 69, 70, 2, 225, 16, 9, 69, 32, 83, 73, 88, 84, 69, 69, 78, 2, 69, 15, + 84, 32, 67, 85, 82, 76, 89, 32, 66, 82, 65, 67, 75, 69, 84, 2, 11, 32, 2, + 219, 173, 2, 83, 2, 11, 69, 2, 205, 74, 2, 78, 67, 26, 17, 2, 85, 65, 26, + 44, 6, 68, 82, 65, 78, 84, 32, 255, 3, 82, 24, 90, 67, 112, 10, 70, 65, + 67, 69, 32, 87, 73, 84, 72, 32, 114, 77, 76, 2, 83, 84, 67, 84, 14, 68, + 7, 73, 82, 67, 85, 76, 65, 82, 145, 167, 1, 4, 72, 69, 83, 83, 2, 17, 2, + 32, 65, 2, 147, 212, 1, 82, 4, 34, 67, 45, 4, 79, 80, 69, 78, 2, 21, 3, + 76, 79, 83, 2, 17, 2, 69, 68, 2, 177, 231, 1, 3, 32, 69, 89, 2, 25, 4, + 73, 67, 82, 79, 2, 11, 67, 2, 233, 194, 1, 4, 79, 77, 80, 85, 2, 41, 8, + 65, 78, 68, 73, 78, 71, 32, 75, 2, 195, 207, 2, 78, 2, 21, 3, 69, 76, 69, + 2, 11, 86, 2, 187, 90, 73, 2, 181, 167, 1, 3, 84, 69, 82, 4, 153, 102, 8, + 65, 68, 79, 87, 69, 68, 32, 87, 8, 30, 79, 106, 82, 155, 1, 87, 2, 49, + 10, 32, 76, 79, 87, 69, 82, 32, 76, 69, 70, 2, 11, 84, 2, 11, 32, 2, 11, + 70, 2, 195, 112, 73, 4, 25, 4, 73, 65, 78, 71, 4, 40, 4, 85, 76, 65, 82, + 219, 224, 2, 76, 2, 17, 2, 32, 77, 2, 41, 8, 69, 68, 73, 85, 77, 32, 83, + 72, 2, 223, 102, 65, 2, 21, 3, 69, 76, 70, 2, 11, 84, 2, 163, 164, 1, 72, + 2, 17, 2, 69, 86, 2, 17, 2, 69, 78, 2, 145, 1, 2, 32, 69, 10, 64, 5, 72, + 82, 69, 69, 32, 193, 1, 6, 82, 73, 65, 78, 71, 85, 8, 54, 69, 49, 9, 81, + 85, 65, 82, 84, 69, 82, 83, 32, 2, 29, 5, 73, 71, 72, 84, 72, 2, 243, 5, + 83, 6, 30, 76, 22, 82, 203, 5, 66, 2, 41, 2, 69, 70, 2, 21, 3, 73, 71, + 72, 2, 43, 84, 2, 17, 2, 76, 65, 2, 11, 82, 2, 17, 2, 32, 79, 2, 25, 4, + 78, 69, 32, 81, 2, 185, 4, 6, 85, 65, 82, 84, 69, 82, 2, 199, 181, 2, 78, + 128, 1, 154, 1, 65, 184, 6, 2, 66, 76, 150, 1, 68, 50, 70, 82, 72, 150, + 4, 67, 46, 81, 42, 82, 22, 83, 102, 84, 146, 7, 80, 173, 3, 6, 87, 72, + 73, 84, 69, 32, 32, 34, 78, 33, 4, 82, 82, 79, 87, 2, 11, 67, 2, 195, + 172, 2, 79, 31, 11, 32, 28, 134, 1, 65, 192, 1, 14, 76, 69, 70, 84, 87, + 65, 82, 68, 83, 32, 79, 70, 32, 68, 32, 5, 87, 73, 84, 72, 32, 210, 8, + 70, 179, 5, 84, 2, 41, 8, 78, 68, 32, 82, 73, 71, 72, 84, 2, 17, 2, 32, + 79, 2, 25, 4, 78, 69, 32, 69, 2, 21, 3, 73, 71, 72, 2, 17, 2, 84, 72, 2, + 11, 32, 2, 11, 66, 2, 11, 76, 2, 191, 190, 1, 79, 2, 197, 206, 1, 3, 79, + 87, 78, 20, 74, 68, 58, 76, 42, 77, 38, 78, 58, 83, 66, 69, 250, 12, 84, + 139, 58, 72, 2, 21, 3, 79, 85, 66, 2, 11, 76, 2, 247, 188, 2, 69, 2, 11, + 65, 2, 253, 22, 3, 82, 71, 69, 2, 225, 22, 5, 69, 68, 73, 85, 77, 2, 25, + 4, 79, 84, 67, 72, 2, 11, 69, 2, 139, 56, 68, 4, 11, 77, 4, 25, 4, 65, + 76, 76, 32, 4, 22, 69, 223, 21, 84, 2, 129, 22, 10, 81, 85, 73, 76, 65, + 84, 69, 82, 65, 76, 4, 25, 4, 65, 67, 75, 32, 4, 26, 67, 139, 208, 1, 65, + 2, 25, 4, 73, 82, 67, 76, 2, 25, 4, 69, 68, 32, 87, 2, 11, 72, 2, 237, + 14, 2, 73, 84, 4, 22, 79, 195, 13, 65, 2, 177, 14, 2, 85, 66, 2, 11, 73, + 2, 37, 7, 78, 71, 69, 82, 45, 80, 79, 2, 173, 206, 1, 2, 83, 84, 20, 88, + 17, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, 32, 66, 65, 82, 66, 32, + 131, 3, 69, 16, 56, 4, 76, 69, 70, 84, 245, 1, 5, 82, 73, 71, 72, 84, 10, + 22, 32, 239, 9, 87, 8, 60, 7, 66, 69, 83, 73, 68, 69, 32, 206, 1, 70, + 179, 5, 84, 4, 40, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 197, 149, 1, 23, + 87, 65, 82, 68, 83, 32, 72, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, + 32, 66, 65, 82, 66, 6, 22, 32, 251, 7, 87, 4, 22, 70, 179, 5, 84, 2, 217, + 195, 1, 3, 82, 79, 77, 4, 25, 4, 65, 86, 89, 32, 4, 26, 67, 155, 202, 1, + 65, 2, 189, 10, 7, 79, 77, 80, 82, 69, 83, 83, 2, 137, 9, 6, 85, 65, 68, + 82, 85, 80, 2, 171, 145, 2, 79, 4, 30, 65, 53, 3, 81, 85, 65, 2, 245, + 200, 1, 8, 78, 83, 45, 83, 69, 82, 73, 70, 2, 131, 9, 82, 36, 36, 2, 82, + 73, 229, 7, 2, 87, 79, 30, 40, 5, 65, 78, 71, 76, 69, 155, 7, 80, 28, 52, + 8, 45, 72, 69, 65, 68, 69, 68, 32, 219, 12, 32, 26, 48, 5, 65, 82, 82, + 79, 87, 174, 5, 68, 39, 80, 23, 11, 32, 20, 92, 17, 76, 69, 70, 84, 87, + 65, 82, 68, 83, 32, 79, 70, 32, 68, 79, 87, 78, 98, 84, 23, 87, 2, 37, 7, + 87, 65, 82, 68, 83, 32, 84, 2, 37, 7, 82, 73, 65, 78, 71, 76, 69, 2, 219, + 5, 45, 2, 171, 190, 1, 79, 16, 25, 4, 73, 84, 72, 32, 16, 114, 66, 28, 6, + 76, 79, 78, 71, 32, 84, 130, 1, 77, 34, 78, 34, 86, 34, 72, 149, 56, 6, + 68, 79, 85, 66, 76, 69, 2, 149, 2, 3, 79, 76, 68, 4, 17, 2, 73, 80, 4, + 11, 32, 4, 34, 76, 21, 4, 82, 73, 71, 72, 2, 17, 2, 69, 70, 2, 11, 84, 2, + 11, 87, 2, 243, 125, 65, 2, 121, 5, 69, 68, 73, 85, 77, 2, 89, 5, 65, 82, + 82, 79, 87, 2, 29, 5, 69, 82, 89, 32, 72, 2, 25, 4, 69, 65, 86, 89, 2, + 221, 177, 2, 4, 32, 83, 72, 65, 2, 11, 65, 2, 249, 1, 2, 83, 72, 2, 11, + 65, 2, 25, 4, 73, 82, 69, 68, 2, 29, 5, 32, 65, 82, 82, 79, 2, 211, 157, + 2, 87, 2, 11, 76, 2, 187, 192, 1, 69, 6, 74, 32, 61, 14, 45, 72, 69, 65, + 68, 69, 68, 32, 65, 82, 82, 79, 87, 32, 2, 25, 4, 72, 69, 65, 68, 2, 11, + 69, 2, 179, 191, 1, 68, 4, 42, 87, 253, 134, 1, 4, 70, 82, 79, 77, 2, 33, + 6, 73, 84, 72, 32, 84, 82, 2, 61, 13, 73, 65, 78, 71, 76, 69, 32, 65, 82, + 82, 79, 87, 72, 2, 149, 121, 2, 69, 65, 18, 88, 5, 65, 82, 82, 79, 87, + 149, 3, 12, 68, 79, 85, 66, 76, 69, 32, 65, 82, 82, 79, 87, 15, 11, 32, + 12, 108, 11, 79, 78, 32, 80, 69, 68, 69, 83, 84, 65, 76, 106, 87, 217, + 179, 1, 8, 70, 82, 79, 77, 32, 66, 65, 82, 7, 33, 6, 32, 87, 73, 84, 72, + 32, 4, 26, 86, 171, 180, 1, 72, 2, 205, 180, 1, 5, 69, 82, 84, 73, 67, 2, + 29, 5, 73, 84, 72, 73, 78, 2, 17, 2, 32, 84, 2, 37, 7, 82, 73, 65, 78, + 71, 76, 69, 2, 11, 32, 2, 11, 65, 2, 25, 4, 82, 82, 79, 87, 2, 11, 72, 2, + 195, 142, 2, 69, 5, 45, 9, 32, 79, 78, 32, 80, 69, 68, 69, 83, 2, 235, + 202, 1, 84, 240, 15, 86, 65, 254, 16, 83, 174, 3, 69, 234, 45, 73, 186, + 8, 79, 134, 2, 84, 21, 2, 85, 76, 218, 8, 116, 2, 73, 32, 128, 16, 17, + 82, 73, 65, 84, 73, 79, 78, 32, 83, 69, 76, 69, 67, 84, 79, 82, 45, 245, + 45, 2, 77, 80, 216, 4, 54, 67, 34, 70, 82, 81, 40, 2, 83, 89, 207, 95, + 68, 2, 11, 79, 2, 231, 176, 2, 77, 2, 11, 85, 2, 11, 76, 2, 11, 76, 2, + 11, 32, 2, 11, 83, 2, 147, 216, 1, 84, 2, 17, 2, 85, 69, 2, 215, 231, 1, + 83, 190, 4, 68, 7, 76, 76, 65, 66, 76, 69, 32, 197, 11, 5, 77, 66, 79, + 76, 32, 164, 4, 214, 1, 68, 70, 66, 2, 83, 2, 84, 2, 90, 70, 71, 122, 72, + 82, 75, 134, 1, 76, 130, 1, 77, 86, 78, 134, 3, 67, 2, 70, 2, 74, 2, 80, + 2, 82, 2, 86, 2, 89, 78, 87, 50, 69, 34, 79, 250, 73, 65, 2, 73, 3, 85, + 42, 66, 72, 162, 8, 79, 194, 133, 2, 69, 162, 64, 65, 2, 73, 3, 85, 28, + 230, 7, 72, 58, 79, 194, 133, 2, 69, 162, 64, 65, 2, 73, 3, 85, 34, 62, + 66, 202, 6, 69, 86, 79, 226, 197, 2, 65, 2, 73, 3, 85, 18, 106, 79, 222, + 5, 69, 182, 198, 2, 65, 2, 73, 3, 85, 24, 50, 79, 222, 5, 69, 178, 75, + 65, 2, 73, 3, 85, 7, 142, 204, 2, 78, 3, 79, 34, 70, 80, 206, 5, 79, 222, + 74, 65, 230, 186, 1, 69, 162, 64, 73, 3, 85, 18, 246, 4, 69, 86, 79, 222, + 74, 65, 134, 251, 1, 73, 3, 85, 16, 54, 69, 218, 4, 79, 226, 197, 2, 65, + 2, 73, 3, 85, 7, 26, 78, 159, 202, 2, 69, 2, 21, 3, 71, 84, 72, 2, 139, + 140, 2, 69, 42, 214, 3, 66, 0, 2, 71, 66, 58, 79, 194, 133, 2, 69, 162, + 64, 65, 2, 73, 3, 85, 90, 90, 68, 174, 1, 71, 126, 74, 2, 89, 58, 79, + 194, 133, 2, 69, 162, 64, 65, 2, 73, 3, 85, 24, 54, 79, 234, 135, 2, 69, + 162, 64, 65, 2, 73, 3, 85, 15, 36, 3, 76, 69, 32, 227, 199, 2, 79, 10, + 54, 83, 182, 168, 2, 68, 190, 28, 70, 2, 75, 3, 77, 2, 179, 168, 2, 79, + 25, 42, 71, 138, 247, 1, 65, 2, 69, 3, 79, 16, 50, 69, 86, 79, 226, 197, + 2, 65, 2, 73, 3, 85, 7, 178, 198, 2, 69, 3, 78, 14, 54, 79, 194, 133, 2, + 69, 162, 64, 65, 2, 73, 3, 85, 5, 223, 197, 2, 79, 28, 46, 69, 34, 79, + 250, 73, 65, 2, 73, 3, 85, 9, 150, 74, 69, 135, 251, 1, 78, 9, 246, 73, + 79, 135, 251, 1, 78, 26, 66, 68, 62, 70, 30, 74, 22, 75, 50, 78, 22, 84, + 151, 234, 1, 66, 6, 26, 79, 135, 243, 1, 65, 4, 130, 243, 1, 79, 135, 50, + 45, 4, 74, 69, 219, 192, 2, 65, 2, 199, 242, 1, 79, 4, 26, 69, 155, 242, + 1, 85, 2, 151, 242, 1, 69, 2, 163, 175, 2, 73, 6, 238, 241, 1, 73, 2, 79, + 195, 78, 65, 128, 4, 74, 49, 94, 50, 98, 51, 2, 52, 2, 53, 2, 54, 2, 55, + 2, 56, 3, 57, 223, 1, 182, 1, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, + 54, 2, 55, 2, 56, 3, 57, 137, 1, 90, 48, 2, 49, 2, 50, 2, 51, 2, 52, 94, + 53, 218, 191, 2, 54, 2, 55, 2, 56, 3, 57, 23, 178, 192, 2, 48, 2, 49, 2, + 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 17, 214, 191, 2, 48, + 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 194, 1, 116, 4, 68, 73, 67, 32, + 206, 19, 82, 168, 155, 1, 13, 67, 84, 79, 82, 32, 79, 82, 32, 67, 82, 79, + 83, 83, 219, 106, 83, 86, 60, 5, 83, 73, 71, 78, 32, 153, 10, 5, 84, 79, + 78, 69, 32, 48, 218, 2, 65, 216, 1, 17, 68, 79, 85, 66, 76, 69, 32, 65, + 78, 85, 83, 86, 65, 82, 65, 32, 65, 98, 74, 52, 6, 78, 73, 72, 83, 72, + 86, 22, 82, 240, 1, 8, 72, 69, 88, 73, 70, 79, 82, 77, 22, 76, 52, 4, 84, + 73, 82, 89, 22, 85, 60, 8, 86, 73, 83, 65, 82, 71, 65, 32, 237, 6, 17, + 89, 65, 74, 85, 82, 86, 69, 68, 73, 67, 32, 77, 73, 68, 76, 73, 78, 14, + 76, 8, 78, 85, 83, 86, 65, 82, 65, 32, 212, 1, 3, 84, 73, 75, 151, 2, 82, + 10, 134, 1, 65, 24, 4, 66, 65, 72, 73, 24, 9, 85, 66, 72, 65, 89, 65, 84, + 79, 32, 201, 4, 10, 86, 65, 77, 65, 71, 79, 77, 85, 75, 72, 2, 21, 3, 78, + 84, 65, 2, 21, 3, 82, 71, 79, 2, 11, 77, 2, 163, 9, 85, 2, 235, 245, 1, + 82, 2, 33, 6, 73, 72, 86, 65, 77, 85, 2, 151, 3, 76, 2, 203, 181, 2, 65, + 8, 144, 1, 15, 69, 86, 69, 82, 83, 69, 68, 32, 86, 73, 83, 65, 82, 71, + 65, 36, 9, 79, 84, 65, 84, 69, 68, 32, 65, 82, 57, 5, 84, 72, 65, 78, 71, + 4, 11, 32, 4, 218, 6, 65, 23, 85, 2, 25, 4, 68, 72, 65, 86, 2, 201, 243, + 1, 2, 73, 83, 2, 17, 2, 32, 76, 2, 21, 3, 79, 78, 71, 2, 229, 240, 1, 2, + 32, 65, 2, 171, 178, 2, 65, 2, 37, 7, 80, 65, 68, 72, 77, 65, 78, 2, 171, + 178, 2, 73, 10, 40, 3, 65, 78, 85, 2, 85, 179, 9, 83, 4, 25, 4, 68, 65, + 84, 84, 4, 11, 65, 5, 25, 4, 32, 87, 73, 84, 2, 11, 72, 2, 11, 32, 2, 11, + 84, 2, 11, 65, 2, 159, 170, 1, 73, 38, 128, 2, 5, 67, 65, 78, 68, 82, 16, + 2, 68, 79, 96, 2, 75, 65, 136, 1, 4, 80, 82, 69, 78, 16, 2, 82, 73, 106, + 84, 164, 1, 11, 89, 65, 74, 85, 82, 86, 69, 68, 73, 67, 32, 180, 1, 12, + 65, 84, 72, 65, 82, 86, 65, 86, 69, 68, 73, 67, 169, 231, 1, 2, 83, 72, + 4, 131, 19, 65, 6, 40, 5, 85, 66, 76, 69, 32, 191, 3, 84, 4, 22, 82, 211, + 5, 83, 2, 11, 73, 2, 251, 1, 78, 4, 56, 3, 82, 83, 72, 17, 7, 84, 72, 65, + 75, 65, 32, 65, 2, 139, 116, 65, 2, 17, 2, 78, 85, 2, 17, 2, 68, 65, 2, + 139, 139, 2, 84, 2, 131, 53, 75, 4, 82, 78, 237, 2, 15, 71, 86, 69, 68, + 73, 67, 32, 75, 65, 83, 72, 77, 73, 82, 73, 2, 219, 177, 1, 71, 6, 42, + 72, 24, 4, 82, 73, 80, 76, 19, 87, 2, 49, 3, 82, 69, 69, 2, 219, 2, 69, + 2, 11, 79, 2, 25, 4, 32, 68, 79, 84, 2, 11, 83, 2, 11, 32, 2, 167, 15, + 66, 8, 176, 1, 10, 65, 71, 71, 82, 65, 86, 65, 84, 69, 68, 22, 73, 105, + 27, 75, 65, 84, 72, 65, 75, 65, 32, 73, 78, 68, 69, 80, 69, 78, 68, 69, + 78, 84, 32, 83, 86, 65, 82, 73, 84, 65, 2, 17, 2, 32, 73, 2, 49, 10, 78, + 68, 69, 80, 69, 78, 68, 69, 78, 84, 2, 17, 2, 32, 83, 2, 189, 134, 2, 3, + 86, 65, 82, 5, 241, 10, 7, 32, 83, 67, 72, 82, 79, 69, 104, 62, 83, 16, + 6, 84, 73, 67, 65, 76, 32, 181, 17, 2, 89, 32, 2, 167, 88, 73, 76, 200, + 2, 4, 66, 65, 82, 32, 174, 3, 67, 42, 69, 62, 70, 38, 71, 32, 11, 73, 68, + 69, 79, 71, 82, 65, 80, 72, 73, 67, 48, 12, 75, 65, 78, 65, 32, 82, 69, + 80, 69, 65, 84, 32, 254, 1, 76, 194, 3, 77, 88, 17, 79, 78, 69, 32, 69, + 73, 71, 72, 84, 72, 32, 66, 76, 79, 67, 75, 45, 62, 82, 158, 1, 84, 218, + 1, 90, 173, 72, 3, 83, 73, 88, 8, 108, 6, 66, 69, 83, 73, 68, 69, 64, 8, + 68, 79, 85, 66, 76, 69, 32, 76, 20, 4, 84, 82, 73, 80, 143, 1, 87, 2, 17, + 2, 32, 82, 2, 21, 3, 73, 71, 72, 2, 159, 129, 1, 84, 2, 69, 2, 69, 70, 2, + 25, 4, 76, 69, 32, 82, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 29, 5, 32, 84, + 85, 82, 78, 2, 11, 83, 2, 11, 84, 2, 203, 128, 1, 73, 2, 21, 3, 73, 84, + 72, 2, 17, 2, 32, 72, 2, 137, 244, 1, 7, 79, 82, 73, 90, 79, 78, 84, 2, + 141, 235, 1, 5, 65, 80, 65, 67, 73, 2, 25, 4, 76, 76, 73, 80, 2, 11, 83, + 2, 215, 229, 1, 73, 2, 11, 79, 2, 213, 82, 2, 85, 82, 2, 221, 155, 1, 3, + 79, 45, 75, 2, 17, 2, 32, 73, 2, 133, 185, 1, 2, 84, 69, 10, 120, 4, 77, + 65, 82, 75, 45, 22, 87, 73, 84, 72, 32, 86, 79, 73, 67, 69, 68, 32, 83, + 79, 85, 78, 68, 32, 77, 65, 82, 75, 7, 11, 32, 4, 50, 85, 21, 3, 76, 79, + 87, 5, 17, 2, 32, 85, 2, 17, 2, 80, 80, 2, 17, 2, 69, 82, 2, 153, 29, 2, + 32, 72, 16, 30, 65, 33, 3, 73, 78, 69, 2, 11, 68, 2, 151, 226, 1, 68, 15, + 11, 32, 12, 38, 69, 49, 5, 87, 73, 84, 72, 32, 2, 25, 4, 88, 84, 69, 78, + 2, 219, 206, 1, 83, 10, 56, 4, 67, 73, 82, 67, 98, 70, 26, 84, 151, 140, + 1, 77, 4, 11, 76, 4, 11, 69, 4, 11, 32, 4, 26, 66, 167, 161, 1, 65, 2, + 11, 69, 2, 223, 131, 1, 76, 2, 57, 3, 79, 85, 82, 2, 11, 72, 2, 21, 3, + 82, 69, 69, 2, 45, 9, 32, 84, 73, 67, 75, 32, 77, 65, 82, 2, 167, 223, 1, + 75, 2, 65, 14, 65, 76, 69, 32, 87, 73, 84, 72, 32, 83, 84, 82, 79, 75, 2, + 171, 149, 1, 69, 12, 150, 156, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, + 4, 42, 65, 57, 6, 69, 83, 73, 83, 84, 79, 2, 25, 4, 67, 73, 78, 71, 2, + 11, 32, 2, 207, 121, 67, 2, 21, 3, 82, 32, 83, 2, 11, 69, 2, 159, 215, 1, + 71, 10, 32, 2, 65, 66, 106, 73, 19, 82, 6, 18, 32, 35, 85, 2, 11, 75, 2, + 179, 136, 2, 69, 4, 33, 6, 76, 65, 84, 73, 79, 78, 5, 251, 45, 32, 2, + 167, 9, 76, 2, 29, 5, 65, 70, 70, 73, 67, 2, 177, 238, 1, 2, 32, 76, 2, + 233, 106, 4, 73, 71, 90, 65, 26, 80, 6, 72, 69, 65, 86, 89, 32, 176, 5, + 3, 77, 85, 67, 241, 9, 3, 66, 79, 76, 20, 62, 69, 190, 1, 70, 38, 82, 82, + 83, 246, 1, 87, 215, 10, 71, 4, 25, 4, 73, 71, 72, 84, 4, 11, 32, 4, 22, + 80, 223, 2, 83, 2, 25, 4, 79, 73, 78, 84, 2, 17, 2, 69, 68, 2, 17, 2, 32, + 66, 2, 25, 4, 76, 65, 67, 75, 2, 11, 32, 2, 215, 79, 83, 2, 11, 73, 2, + 185, 1, 2, 86, 69, 2, 11, 69, 2, 29, 5, 86, 69, 82, 83, 69, 2, 17, 2, 32, + 83, 2, 231, 1, 79, 6, 30, 65, 42, 73, 147, 1, 79, 2, 11, 76, 2, 11, 84, + 2, 147, 116, 73, 2, 11, 88, 2, 11, 32, 2, 11, 83, 2, 21, 3, 80, 79, 75, + 2, 17, 2, 69, 68, 2, 11, 32, 2, 11, 65, 2, 11, 83, 2, 205, 78, 3, 84, 69, + 82, 2, 173, 13, 3, 76, 73, 68, 4, 21, 3, 72, 73, 84, 4, 11, 69, 4, 11, + 32, 4, 226, 64, 67, 163, 49, 83, 4, 11, 72, 4, 11, 32, 4, 18, 71, 39, 76, + 2, 69, 6, 82, 69, 65, 84, 69, 82, 2, 11, 69, 2, 11, 83, 2, 11, 83, 2, 17, + 2, 45, 84, 2, 187, 71, 72, 160, 1, 140, 1, 9, 66, 82, 65, 84, 73, 79, 78, + 32, 77, 34, 67, 36, 3, 68, 69, 79, 126, 69, 222, 1, 79, 22, 82, 21, 7, + 84, 72, 75, 85, 81, 73, 32, 2, 11, 79, 2, 183, 249, 1, 68, 2, 221, 129, + 1, 4, 84, 79, 82, 89, 6, 30, 32, 77, 3, 67, 65, 83, 4, 18, 67, 43, 71, 2, + 17, 2, 65, 77, 2, 203, 202, 1, 69, 2, 139, 119, 65, 2, 219, 71, 83, 6, + 168, 1, 31, 84, 78, 65, 77, 69, 83, 69, 32, 65, 76, 84, 69, 82, 78, 65, + 84, 69, 32, 82, 69, 65, 68, 73, 78, 71, 32, 77, 65, 82, 75, 32, 189, 108, + 5, 87, 68, 65, 84, 65, 4, 26, 78, 219, 138, 2, 67, 2, 139, 129, 1, 72, 2, + 167, 141, 1, 76, 2, 243, 237, 1, 71, 140, 1, 56, 6, 67, 65, 80, 73, 84, + 65, 1, 4, 83, 77, 65, 76, 70, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, + 70, 230, 1, 66, 34, 69, 22, 73, 22, 76, 34, 78, 246, 136, 1, 67, 2, 68, + 2, 83, 2, 84, 226, 56, 72, 238, 48, 70, 2, 74, 2, 77, 2, 80, 2, 82, 2, + 86, 2, 88, 2, 90, 158, 20, 71, 2, 75, 2, 81, 186, 2, 65, 2, 79, 2, 85, 3, + 89, 4, 166, 243, 1, 66, 215, 22, 69, 5, 219, 137, 2, 73, 5, 243, 242, 1, + 74, 4, 250, 134, 2, 76, 187, 2, 65, 4, 190, 242, 1, 74, 215, 22, 69, 8, + 28, 3, 73, 68, 69, 59, 76, 2, 21, 3, 68, 32, 71, 2, 141, 71, 4, 82, 69, + 69, 75, 6, 46, 67, 20, 3, 76, 69, 89, 41, 2, 85, 77, 2, 219, 222, 1, 65, + 2, 11, 66, 2, 11, 65, 2, 139, 126, 76, 2, 11, 69, 2, 17, 2, 32, 73, 2, + 193, 125, 4, 78, 84, 69, 71, 5, 239, 134, 2, 83, 40, 70, 67, 45, 13, 71, + 65, 82, 32, 70, 82, 65, 67, 84, 73, 79, 78, 32, 2, 11, 65, 2, 11, 78, 2, + 159, 200, 1, 85, 38, 106, 70, 96, 4, 79, 78, 69, 32, 178, 2, 84, 80, 7, + 83, 69, 86, 69, 78, 32, 69, 153, 33, 3, 90, 69, 82, 6, 56, 4, 73, 86, 69, + 32, 253, 3, 5, 79, 85, 82, 32, 70, 4, 162, 3, 69, 113, 3, 83, 73, 88, 18, + 66, 69, 28, 2, 70, 73, 18, 72, 34, 78, 14, 81, 30, 83, 55, 84, 2, 197, 1, + 3, 73, 71, 72, 2, 171, 1, 70, 2, 11, 65, 2, 251, 129, 1, 76, 2, 111, 73, + 2, 133, 75, 3, 85, 65, 82, 4, 24, 2, 69, 86, 15, 73, 2, 43, 69, 2, 43, + 88, 4, 18, 69, 35, 72, 2, 11, 78, 2, 255, 128, 2, 84, 2, 143, 107, 73, + 10, 48, 5, 72, 82, 69, 69, 32, 93, 3, 87, 79, 32, 6, 26, 69, 26, 81, 67, + 70, 2, 109, 3, 73, 71, 72, 2, 21, 3, 85, 65, 82, 2, 139, 114, 84, 4, 22, + 70, 219, 32, 84, 2, 11, 73, 2, 11, 70, 2, 205, 194, 1, 2, 84, 72, 164, 6, + 86, 65, 250, 23, 69, 158, 3, 72, 250, 68, 73, 234, 8, 79, 134, 6, 82, + 191, 144, 1, 74, 204, 2, 114, 70, 18, 78, 220, 5, 2, 88, 73, 158, 1, 82, + 210, 10, 84, 194, 1, 86, 221, 152, 1, 6, 83, 84, 69, 66, 65, 83, 2, 147, + 90, 70, 122, 36, 4, 67, 72, 79, 32, 183, 5, 73, 118, 100, 7, 76, 69, 84, + 84, 69, 82, 32, 252, 3, 3, 78, 71, 85, 16, 5, 84, 79, 78, 69, 32, 243, 7, + 68, 88, 210, 1, 65, 38, 79, 34, 69, 22, 73, 22, 76, 50, 78, 42, 84, 50, + 85, 22, 89, 130, 178, 1, 75, 2, 80, 2, 83, 138, 69, 66, 2, 67, 2, 68, 2, + 70, 2, 71, 2, 72, 2, 74, 2, 77, 2, 82, 2, 86, 2, 87, 3, 90, 13, 34, 65, + 210, 250, 1, 78, 87, 85, 7, 11, 78, 5, 151, 251, 1, 71, 5, 131, 251, 1, + 78, 5, 155, 250, 1, 78, 4, 26, 76, 195, 250, 1, 65, 2, 135, 248, 1, 72, + 6, 242, 247, 1, 71, 2, 89, 187, 2, 65, 8, 202, 247, 1, 72, 2, 82, 2, 83, + 187, 2, 65, 5, 243, 169, 1, 69, 4, 174, 248, 1, 73, 147, 1, 65, 2, 159, + 114, 78, 8, 40, 3, 75, 79, 73, 1, 3, 84, 85, 80, 5, 139, 229, 1, 78, 4, + 21, 3, 78, 71, 32, 4, 76, 8, 67, 82, 69, 83, 67, 69, 78, 84, 1, 7, 71, + 73, 66, 66, 79, 85, 83, 2, 21, 3, 32, 77, 79, 2, 11, 79, 2, 155, 100, 78, + 170, 1, 72, 9, 65, 78, 71, 32, 67, 73, 84, 73, 32, 229, 111, 4, 78, 73, + 78, 71, 168, 1, 128, 1, 6, 67, 65, 80, 73, 84, 65, 0, 4, 83, 77, 65, 76, + 190, 4, 68, 156, 2, 7, 78, 85, 77, 66, 69, 82, 32, 175, 222, 1, 79, 64, + 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 64, 174, 1, 65, 38, 69, 42, + 72, 74, 78, 50, 79, 22, 83, 50, 85, 30, 89, 226, 42, 84, 180, 110, 2, 86, + 73, 222, 51, 66, 2, 80, 246, 5, 75, 158, 11, 73, 2, 87, 162, 17, 68, 3, + 71, 9, 166, 243, 1, 78, 86, 77, 3, 84, 7, 11, 78, 4, 202, 243, 1, 78, 3, + 89, 8, 38, 79, 198, 154, 1, 73, 235, 31, 65, 4, 170, 186, 1, 82, 235, 25, + 76, 4, 26, 71, 215, 161, 1, 85, 2, 147, 240, 1, 65, 5, 151, 172, 1, 68, + 4, 26, 83, 179, 222, 1, 73, 2, 159, 205, 1, 85, 4, 242, 241, 1, 67, 3, + 85, 8, 34, 85, 182, 241, 1, 65, 3, 79, 5, 179, 241, 1, 74, 20, 11, 73, + 20, 11, 71, 20, 17, 2, 73, 84, 20, 11, 32, 20, 66, 70, 30, 83, 42, 84, + 62, 90, 238, 85, 78, 14, 79, 223, 110, 69, 4, 206, 115, 73, 131, 4, 79, + 4, 22, 69, 207, 99, 73, 2, 135, 28, 86, 4, 26, 72, 211, 208, 1, 87, 2, + 11, 82, 2, 227, 216, 1, 69, 2, 11, 69, 2, 163, 208, 1, 82, 18, 42, 69, + 30, 70, 42, 78, 38, 83, 39, 84, 2, 221, 1, 3, 73, 71, 72, 4, 22, 73, 139, + 1, 79, 2, 171, 1, 70, 2, 17, 2, 73, 78, 2, 135, 1, 69, 4, 92, 2, 69, 86, + 25, 2, 73, 88, 6, 34, 72, 26, 87, 179, 157, 1, 69, 2, 11, 73, 2, 35, 82, + 2, 11, 69, 2, 11, 78, 2, 175, 219, 1, 84, 12, 32, 2, 69, 82, 179, 235, 1, + 67, 10, 34, 32, 165, 156, 1, 2, 77, 69, 8, 80, 3, 67, 76, 79, 22, 87, + 228, 200, 1, 5, 66, 85, 70, 70, 65, 1, 2, 80, 79, 2, 175, 182, 1, 83, 2, + 223, 110, 65, 20, 60, 2, 69, 32, 124, 4, 73, 78, 71, 32, 161, 1, 2, 89, + 32, 6, 182, 2, 68, 161, 189, 1, 23, 65, 82, 82, 79, 87, 32, 80, 79, 73, + 78, 84, 73, 78, 71, 32, 68, 73, 82, 69, 67, 84, 76, 89, 6, 64, 5, 66, 76, + 65, 67, 75, 0, 5, 87, 72, 73, 84, 69, 55, 72, 2, 17, 2, 32, 70, 2, 11, + 76, 2, 175, 232, 1, 65, 2, 11, 65, 2, 11, 78, 2, 215, 97, 68, 8, 26, 68, + 34, 76, 43, 79, 2, 11, 65, 2, 143, 231, 1, 83, 4, 22, 79, 203, 78, 73, 2, + 163, 78, 87, 2, 11, 86, 2, 11, 69, 2, 135, 78, 82, 14, 48, 3, 65, 82, 89, + 70, 68, 70, 73, 171, 1, 83, 4, 11, 32, 4, 32, 2, 67, 65, 211, 169, 1, 70, + 2, 183, 169, 1, 84, 4, 26, 71, 167, 149, 1, 68, 2, 129, 70, 6, 69, 45, + 84, 65, 73, 76, 4, 116, 17, 69, 82, 83, 84, 82, 65, 83, 83, 32, 69, 76, + 76, 73, 80, 84, 73, 67, 145, 45, 7, 71, 72, 84, 32, 76, 73, 70, 2, 17, 2, + 32, 70, 2, 149, 148, 1, 2, 85, 78, 2, 37, 7, 84, 32, 83, 89, 82, 73, 65, + 2, 131, 35, 67, 188, 2, 52, 3, 69, 69, 76, 100, 3, 73, 84, 69, 219, 62, + 65, 7, 60, 7, 32, 79, 70, 32, 68, 72, 65, 21, 4, 67, 72, 65, 73, 2, 147, + 194, 1, 82, 2, 231, 79, 82, 180, 2, 54, 32, 161, 66, 8, 45, 70, 69, 65, + 84, 72, 69, 82, 178, 2, 148, 2, 18, 65, 82, 82, 79, 87, 32, 83, 72, 65, + 70, 84, 32, 87, 73, 68, 84, 72, 32, 114, 66, 46, 67, 250, 15, 68, 166, 5, + 69, 50, 70, 178, 3, 72, 246, 3, 76, 210, 4, 77, 222, 1, 78, 34, 80, 146, + 1, 81, 78, 82, 142, 2, 83, 158, 12, 84, 228, 3, 2, 85, 80, 149, 4, 3, 86, + 69, 82, 4, 22, 84, 231, 70, 79, 2, 11, 87, 2, 21, 3, 79, 32, 84, 2, 17, + 2, 72, 73, 2, 11, 82, 2, 239, 161, 1, 68, 2, 11, 85, 2, 11, 76, 2, 135, + 170, 1, 76, 94, 154, 1, 72, 176, 10, 5, 73, 82, 67, 76, 69, 162, 3, 76, + 24, 20, 79, 78, 67, 65, 86, 69, 45, 83, 73, 68, 69, 68, 32, 68, 73, 65, + 77, 79, 78, 68, 79, 82, 66, 25, 4, 69, 83, 83, 32, 66, 66, 66, 38, 69, + 118, 75, 142, 4, 80, 22, 81, 38, 82, 131, 2, 84, 6, 241, 5, 5, 73, 83, + 72, 79, 80, 4, 45, 9, 81, 85, 73, 72, 79, 80, 80, 69, 82, 5, 17, 2, 32, + 82, 2, 129, 6, 8, 79, 84, 65, 84, 69, 68, 32, 78, 26, 38, 73, 25, 5, 78, + 73, 71, 72, 84, 6, 177, 4, 2, 78, 71, 21, 22, 32, 151, 3, 45, 12, 41, 8, + 82, 79, 84, 65, 84, 69, 68, 32, 12, 104, 2, 70, 79, 0, 15, 79, 78, 69, + 32, 72, 85, 78, 68, 82, 69, 68, 32, 84, 72, 73, 18, 84, 215, 3, 78, 2, + 207, 1, 82, 6, 148, 1, 11, 87, 79, 32, 72, 85, 78, 68, 82, 69, 68, 32, + 133, 3, 20, 72, 82, 69, 69, 32, 72, 85, 78, 68, 82, 69, 68, 32, 70, 73, + 70, 84, 69, 69, 78, 4, 36, 4, 84, 87, 69, 78, 175, 2, 83, 2, 217, 2, 7, + 84, 89, 45, 70, 73, 86, 69, 6, 174, 3, 66, 94, 81, 47, 82, 6, 41, 2, 65, + 87, 6, 21, 3, 85, 69, 69, 6, 35, 78, 6, 21, 3, 79, 79, 75, 7, 45, 9, 32, + 82, 79, 84, 65, 84, 69, 68, 32, 4, 70, 78, 25, 13, 84, 87, 79, 32, 72, + 85, 78, 68, 82, 69, 68, 32, 83, 2, 49, 3, 73, 78, 69, 2, 25, 4, 69, 86, + 69, 78, 2, 17, 2, 84, 89, 2, 225, 65, 6, 32, 68, 69, 71, 82, 69, 12, 29, + 5, 85, 82, 78, 69, 68, 12, 11, 32, 12, 42, 66, 30, 75, 34, 80, 34, 81, + 47, 82, 2, 197, 91, 3, 73, 83, 72, 4, 198, 131, 1, 73, 171, 38, 78, 2, + 11, 65, 2, 179, 132, 1, 87, 2, 11, 85, 2, 11, 69, 2, 135, 132, 1, 69, 2, + 243, 169, 1, 79, 19, 11, 32, 16, 100, 16, 67, 79, 78, 84, 65, 73, 78, 73, + 78, 71, 32, 66, 76, 65, 67, 75, 121, 5, 87, 73, 84, 72, 32, 2, 17, 2, 32, + 83, 2, 11, 77, 2, 21, 3, 65, 76, 76, 2, 11, 32, 2, 11, 67, 2, 11, 73, 2, + 11, 82, 2, 187, 45, 67, 14, 56, 2, 68, 79, 70, 84, 228, 31, 2, 76, 79, + 231, 1, 85, 4, 18, 84, 35, 87, 2, 11, 32, 2, 167, 166, 1, 82, 2, 143, 54, + 78, 2, 11, 87, 2, 11, 79, 2, 11, 32, 2, 247, 59, 68, 2, 209, 27, 2, 85, + 66, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 228, 30, 2, 76, 69, 49, 2, 82, + 73, 2, 17, 2, 79, 83, 2, 227, 141, 1, 83, 28, 76, 6, 73, 65, 77, 79, 78, + 68, 216, 2, 3, 79, 87, 78, 177, 1, 2, 82, 65, 15, 11, 32, 12, 160, 1, 17, + 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, 65, 67, 75, 32, 128, + 1, 9, 87, 73, 84, 72, 32, 67, 69, 78, 84, 226, 23, 83, 197, 19, 2, 73, + 78, 6, 74, 83, 0, 6, 86, 69, 82, 89, 32, 83, 29, 6, 77, 69, 68, 73, 85, + 77, 2, 25, 4, 77, 65, 76, 76, 2, 197, 15, 2, 32, 68, 2, 11, 82, 2, 11, + 69, 2, 231, 58, 68, 10, 96, 10, 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, + 53, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 6, 170, 36, 66, 24, 5, + 76, 69, 70, 84, 32, 51, 73, 4, 194, 37, 83, 51, 84, 4, 33, 6, 85, 71, 72, + 84, 83, 32, 4, 22, 77, 255, 120, 75, 2, 159, 122, 65, 2, 29, 5, 88, 67, + 76, 65, 77, 2, 167, 15, 65, 14, 74, 76, 144, 2, 11, 79, 85, 82, 32, 80, + 79, 73, 78, 84, 69, 68, 71, 82, 8, 28, 2, 65, 71, 175, 1, 79, 5, 149, 1, + 34, 32, 87, 73, 84, 72, 32, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, + 77, 73, 68, 68, 76, 69, 32, 66, 76, 65, 67, 75, 32, 83, 84, 82, 73, 2, + 231, 176, 1, 80, 4, 26, 82, 163, 137, 1, 87, 2, 17, 2, 69, 84, 2, 163, + 176, 1, 84, 4, 11, 32, 4, 18, 67, 23, 83, 2, 171, 198, 1, 85, 2, 247, 36, + 84, 2, 235, 61, 79, 16, 34, 65, 150, 1, 69, 231, 1, 79, 2, 25, 4, 82, 68, + 32, 83, 2, 45, 9, 72, 69, 76, 76, 32, 70, 76, 79, 80, 2, 17, 2, 80, 89, + 2, 17, 2, 32, 68, 2, 11, 73, 2, 211, 193, 1, 83, 10, 22, 65, 151, 12, 88, + 8, 30, 82, 29, 3, 86, 89, 32, 4, 11, 84, 5, 211, 15, 32, 4, 74, 67, 53, + 14, 83, 65, 76, 84, 73, 82, 69, 32, 87, 73, 84, 72, 32, 82, 2, 17, 2, 72, + 69, 2, 11, 67, 2, 211, 129, 1, 75, 2, 167, 19, 79, 4, 60, 8, 82, 73, 90, + 79, 78, 84, 65, 76, 233, 42, 2, 85, 82, 2, 205, 31, 2, 32, 69, 22, 42, + 65, 116, 3, 69, 70, 84, 175, 1, 79, 4, 24, 2, 82, 71, 19, 84, 2, 227, 32, + 69, 2, 11, 73, 2, 11, 78, 2, 17, 2, 32, 67, 2, 11, 82, 2, 187, 42, 79, + 12, 58, 32, 77, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 6, 44, 6, 76, + 65, 78, 69, 32, 77, 247, 24, 80, 2, 11, 69, 2, 227, 10, 82, 6, 150, 2, + 80, 206, 24, 83, 51, 84, 6, 160, 1, 4, 87, 69, 82, 32, 221, 8, 30, 90, + 69, 78, 71, 69, 32, 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, + 65, 67, 75, 32, 83, 77, 65, 76, 76, 32, 76, 4, 44, 3, 76, 69, 70, 1, 4, + 82, 73, 71, 72, 2, 11, 84, 2, 17, 2, 32, 80, 2, 203, 5, 79, 12, 68, 6, + 69, 68, 73, 85, 77, 32, 117, 7, 79, 79, 78, 32, 83, 69, 76, 10, 30, 68, + 54, 83, 227, 6, 76, 2, 11, 73, 2, 11, 65, 2, 11, 77, 2, 199, 46, 79, 6, + 246, 26, 84, 50, 77, 59, 81, 2, 11, 69, 2, 203, 185, 1, 78, 2, 11, 73, 2, + 227, 187, 1, 66, 6, 18, 65, 87, 69, 2, 37, 7, 82, 65, 76, 76, 69, 76, 79, + 2, 11, 71, 2, 11, 82, 2, 163, 170, 1, 65, 4, 11, 78, 4, 174, 2, 84, 211, + 46, 78, 2, 21, 3, 85, 69, 83, 2, 197, 118, 9, 84, 73, 79, 78, 32, 77, 65, + 82, 75, 14, 34, 69, 25, 4, 73, 71, 72, 84, 2, 129, 21, 2, 67, 84, 12, 60, + 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 211, 17, 32, 8, 30, 80, 202, + 19, 83, 51, 84, 4, 18, 69, 55, 79, 2, 11, 78, 2, 11, 84, 2, 11, 65, 2, + 179, 104, 71, 2, 11, 73, 2, 11, 78, 2, 139, 122, 84, 56, 118, 67, 32, 3, + 69, 83, 65, 18, 72, 58, 77, 166, 1, 80, 68, 5, 81, 85, 65, 82, 69, 180, + 6, 2, 85, 78, 183, 12, 84, 2, 153, 40, 4, 73, 83, 83, 79, 2, 175, 37, 77, + 2, 21, 3, 79, 71, 73, 2, 169, 121, 4, 32, 80, 73, 69, 8, 32, 4, 65, 76, + 76, 32, 115, 73, 6, 18, 76, 71, 83, 2, 11, 79, 2, 11, 90, 2, 11, 69, 2, + 11, 78, 2, 203, 158, 1, 71, 4, 210, 19, 84, 107, 81, 2, 211, 44, 76, 2, + 21, 3, 65, 68, 69, 2, 11, 32, 2, 11, 83, 2, 251, 151, 1, 85, 29, 11, 32, 26, 114, 66, 36, 17, 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, - 65, 67, 75, 32, 73, 5, 87, 73, 84, 72, 32, 2, 17, 2, 85, 84, 2, 251, 97, - 84, 6, 184, 16, 4, 86, 69, 82, 89, 22, 83, 37, 6, 77, 69, 68, 73, 85, 77, - 18, 150, 1, 76, 50, 82, 214, 1, 85, 144, 1, 17, 86, 69, 82, 84, 73, 67, - 65, 76, 32, 66, 73, 83, 69, 67, 84, 73, 78, 221, 18, 6, 67, 69, 78, 84, - 82, 69, 6, 18, 69, 15, 79, 2, 67, 70, 4, 247, 1, 87, 4, 18, 73, 115, 79, - 2, 17, 2, 71, 72, 2, 33, 6, 84, 87, 65, 82, 68, 83, 2, 11, 32, 2, 11, 84, - 2, 11, 73, 2, 179, 171, 1, 67, 2, 29, 5, 85, 78, 68, 69, 68, 2, 21, 3, - 32, 67, 79, 2, 181, 31, 2, 82, 78, 4, 17, 2, 80, 80, 4, 21, 3, 69, 82, - 32, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 33, 6, 84, 32, 81, 85, - 65, 68, 2, 163, 35, 82, 2, 159, 19, 71, 11, 11, 32, 8, 84, 12, 66, 69, - 72, 73, 78, 68, 32, 67, 76, 79, 85, 68, 69, 5, 87, 73, 84, 72, 32, 5, 29, - 5, 32, 87, 73, 84, 72, 2, 17, 2, 32, 82, 2, 243, 43, 65, 4, 38, 82, 29, - 5, 83, 77, 65, 76, 76, 2, 11, 65, 2, 155, 109, 89, 2, 25, 4, 32, 67, 76, - 79, 2, 171, 100, 85, 10, 38, 79, 54, 69, 62, 82, 203, 1, 87, 2, 49, 10, - 85, 67, 72, 84, 79, 78, 69, 32, 84, 69, 2, 17, 2, 76, 69, 2, 11, 80, 2, - 11, 72, 2, 147, 16, 79, 4, 148, 1, 4, 65, 80, 69, 90, 33, 28, 73, 65, 78, - 71, 76, 69, 32, 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 83, 77, 65, - 76, 76, 32, 87, 72, 73, 84, 2, 11, 73, 2, 155, 151, 1, 85, 2, 255, 3, 69, - 2, 11, 79, 2, 85, 19, 45, 87, 65, 89, 32, 76, 69, 70, 84, 32, 87, 65, 89, - 32, 84, 82, 65, 70, 70, 2, 11, 73, 2, 223, 166, 1, 67, 12, 62, 32, 185, - 1, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 4, 11, 80, 4, 41, 8, 79, - 73, 78, 84, 73, 78, 71, 32, 4, 18, 66, 75, 73, 2, 21, 3, 65, 67, 75, 2, - 25, 4, 72, 65, 78, 68, 2, 17, 2, 32, 73, 2, 17, 2, 78, 68, 2, 215, 24, - 69, 8, 68, 4, 67, 72, 69, 86, 18, 83, 73, 7, 84, 82, 73, 65, 78, 71, 76, - 2, 171, 84, 82, 2, 25, 4, 77, 65, 76, 76, 2, 17, 2, 32, 84, 2, 129, 2, 2, - 82, 73, 4, 11, 69, 5, 11, 32, 2, 11, 87, 2, 145, 18, 3, 73, 84, 72, 8, - 44, 6, 84, 73, 67, 65, 76, 32, 183, 1, 89, 6, 26, 66, 18, 69, 51, 82, 2, - 215, 105, 65, 2, 11, 76, 2, 17, 2, 76, 73, 2, 131, 25, 80, 2, 11, 69, 2, - 17, 2, 67, 84, 2, 11, 65, 2, 11, 78, 2, 11, 71, 2, 207, 138, 1, 76, 2, - 17, 2, 32, 83, 2, 11, 77, 2, 21, 3, 65, 76, 76, 2, 17, 2, 32, 83, 2, 11, - 81, 2, 11, 85, 2, 11, 65, 2, 203, 137, 1, 82, 2, 11, 69, 2, 11, 68, 2, - 17, 2, 32, 82, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 11, 87, 2, 241, 4, 4, - 65, 82, 68, 83, 100, 140, 1, 10, 68, 69, 45, 72, 69, 65, 68, 69, 68, 32, - 132, 4, 4, 71, 71, 76, 89, 124, 6, 76, 84, 69, 68, 32, 70, 42, 78, 189, - 1, 2, 82, 69, 80, 128, 1, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 12, 4, 68, - 79, 87, 78, 0, 2, 85, 80, 32, 3, 78, 79, 82, 1, 3, 83, 79, 85, 10, 11, - 84, 10, 109, 5, 87, 65, 82, 68, 83, 20, 21, 3, 84, 72, 32, 20, 32, 2, 69, - 65, 1, 2, 87, 69, 10, 17, 2, 83, 84, 10, 11, 32, 10, 110, 72, 0, 6, 86, - 69, 82, 89, 32, 72, 28, 5, 76, 73, 71, 72, 84, 0, 6, 77, 69, 68, 73, 85, - 77, 23, 66, 2, 25, 4, 69, 65, 86, 89, 2, 17, 2, 32, 66, 2, 21, 3, 65, 82, - 66, 2, 11, 32, 2, 11, 65, 2, 11, 82, 2, 11, 82, 2, 251, 27, 79, 2, 17, 2, - 32, 86, 2, 11, 69, 2, 33, 6, 82, 84, 73, 67, 65, 76, 2, 11, 32, 2, 11, - 76, 2, 11, 73, 2, 191, 130, 1, 78, 2, 11, 76, 2, 11, 79, 2, 135, 91, 87, - 12, 46, 68, 106, 69, 186, 15, 75, 139, 136, 1, 71, 6, 22, 32, 131, 26, - 79, 4, 44, 2, 67, 72, 217, 15, 4, 66, 76, 79, 87, 2, 11, 73, 2, 255, 128, - 1, 77, 2, 11, 32, 2, 121, 3, 71, 76, 65, 4, 36, 5, 68, 32, 75, 69, 89, - 51, 76, 2, 17, 2, 66, 79, 2, 11, 65, 2, 191, 80, 82, 2, 11, 69, 2, 231, - 88, 83, 30, 66, 77, 158, 3, 82, 226, 11, 78, 214, 64, 79, 129, 9, 2, 76, - 70, 14, 36, 2, 65, 78, 161, 2, 2, 69, 78, 13, 72, 12, 32, 87, 73, 84, 72, - 32, 66, 85, 78, 78, 89, 32, 29, 2, 83, 32, 2, 11, 69, 2, 167, 6, 65, 8, - 48, 2, 66, 79, 28, 2, 67, 76, 54, 72, 19, 83, 2, 11, 79, 2, 195, 86, 84, - 2, 11, 79, 2, 11, 84, 2, 11, 72, 2, 143, 86, 69, 2, 163, 119, 65, 2, 17, - 2, 65, 78, 2, 131, 10, 68, 2, 11, 83, 2, 11, 32, 2, 11, 83, 2, 11, 89, 2, - 17, 2, 77, 66, 2, 187, 9, 79, 10, 72, 2, 68, 32, 156, 1, 3, 76, 68, 32, - 32, 2, 82, 73, 195, 144, 1, 77, 4, 56, 9, 83, 69, 80, 65, 82, 65, 84, 79, - 82, 191, 83, 74, 2, 17, 2, 32, 77, 2, 21, 3, 73, 68, 68, 2, 11, 76, 2, - 11, 69, 2, 11, 32, 2, 195, 64, 68, 2, 11, 77, 2, 199, 144, 1, 65, 2, 11, - 69, 2, 159, 83, 68, 10, 56, 7, 65, 80, 80, 69, 68, 32, 80, 30, 69, 163, - 1, 73, 2, 201, 76, 3, 82, 69, 83, 6, 48, 2, 65, 84, 80, 3, 83, 84, 76, - 163, 108, 78, 2, 11, 72, 2, 11, 32, 2, 11, 80, 2, 25, 4, 82, 79, 68, 85, - 2, 171, 114, 67, 2, 11, 69, 2, 235, 80, 82, 2, 11, 84, 2, 11, 73, 2, 17, - 2, 78, 71, 2, 17, 2, 32, 72, 2, 11, 65, 2, 203, 71, 78, 34, 76, 2, 32, - 73, 138, 1, 45, 28, 7, 73, 65, 78, 71, 81, 73, 32, 143, 83, 79, 2, 53, - 11, 78, 32, 65, 32, 82, 69, 67, 84, 65, 78, 71, 2, 11, 76, 2, 11, 69, 2, - 11, 32, 2, 11, 66, 2, 11, 79, 2, 139, 140, 1, 88, 2, 11, 82, 2, 163, 122, - 65, 28, 48, 5, 66, 76, 65, 67, 75, 1, 3, 82, 69, 68, 14, 11, 32, 14, 130, - 1, 67, 72, 4, 69, 76, 69, 80, 28, 4, 71, 69, 78, 69, 46, 72, 36, 4, 83, - 79, 76, 68, 169, 9, 6, 77, 65, 78, 68, 65, 82, 4, 24, 2, 65, 78, 19, 72, - 2, 155, 58, 78, 2, 189, 57, 3, 65, 82, 73, 2, 11, 72, 2, 179, 70, 65, 2, - 11, 82, 2, 11, 65, 2, 155, 137, 1, 76, 2, 17, 2, 79, 82, 2, 159, 114, 83, - 2, 131, 75, 73, 242, 19, 50, 65, 58, 69, 218, 11, 73, 217, 35, 2, 79, 45, - 2, 17, 2, 87, 78, 2, 11, 73, 2, 229, 74, 2, 78, 71, 98, 60, 4, 76, 76, - 79, 87, 62, 78, 53, 5, 90, 73, 68, 73, 32, 2, 17, 2, 32, 72, 2, 11, 69, - 2, 11, 65, 2, 203, 106, 82, 2, 11, 32, 2, 11, 83, 2, 11, 73, 2, 239, 54, - 71, 94, 112, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 76, 5, 72, 89, - 80, 72, 69, 17, 7, 76, 69, 84, 84, 69, 82, 32, 4, 44, 3, 77, 65, 68, 13, - 4, 72, 65, 77, 90, 2, 11, 68, 2, 203, 67, 65, 2, 207, 27, 78, 88, 250, 1, - 67, 50, 77, 18, 68, 42, 69, 58, 72, 30, 75, 22, 71, 2, 81, 32, 3, 76, 65, - 77, 98, 78, 18, 80, 30, 83, 66, 84, 28, 2, 86, 65, 118, 87, 14, 79, 18, - 88, 52, 3, 89, 79, 84, 154, 1, 90, 130, 53, 82, 226, 48, 66, 254, 5, 85, - 162, 14, 70, 3, 74, 6, 22, 72, 251, 113, 73, 4, 22, 72, 227, 113, 73, 2, - 223, 113, 73, 4, 11, 65, 4, 154, 130, 1, 68, 3, 76, 8, 42, 76, 130, 50, - 89, 214, 79, 84, 3, 87, 2, 71, 73, 4, 254, 111, 65, 147, 15, 72, 4, 18, - 72, 15, 65, 2, 11, 65, 2, 139, 129, 1, 70, 5, 11, 32, 2, 11, 87, 2, 21, - 3, 73, 84, 72, 2, 17, 2, 32, 68, 2, 11, 79, 2, 179, 3, 84, 2, 195, 48, - 85, 4, 178, 105, 72, 215, 22, 69, 8, 46, 72, 234, 47, 73, 194, 9, 65, - 151, 70, 69, 2, 231, 47, 73, 4, 214, 104, 72, 215, 22, 65, 5, 37, 7, 32, - 65, 76, 84, 69, 82, 78, 2, 21, 3, 65, 84, 69, 2, 11, 32, 2, 11, 70, 2, - 11, 79, 2, 211, 109, 82, 2, 11, 65, 2, 143, 126, 87, 4, 22, 72, 235, 125, - 65, 2, 11, 69, 2, 135, 46, 89, 5, 37, 7, 32, 87, 73, 84, 72, 32, 67, 2, - 45, 9, 73, 82, 67, 85, 77, 70, 76, 69, 88, 2, 11, 32, 2, 11, 65, 2, 11, - 66, 2, 11, 79, 2, 239, 101, 86, 6, 22, 65, 159, 124, 69, 5, 155, 124, 76, - 140, 19, 34, 32, 161, 35, 3, 78, 32, 89, 138, 19, 88, 8, 82, 65, 68, 73, - 67, 65, 76, 32, 181, 7, 9, 83, 89, 76, 76, 65, 66, 76, 69, 32, 110, 170, - 1, 66, 42, 67, 86, 68, 42, 71, 74, 72, 66, 74, 66, 75, 30, 76, 30, 77, - 26, 78, 74, 80, 26, 83, 74, 84, 30, 86, 30, 89, 30, 90, 246, 35, 81, 186, - 49, 87, 235, 30, 79, 4, 22, 66, 243, 64, 85, 2, 147, 93, 85, 12, 42, 85, - 18, 89, 162, 98, 72, 203, 22, 73, 2, 247, 120, 79, 7, 242, 120, 80, 3, - 84, 4, 22, 68, 199, 120, 85, 2, 243, 63, 85, 10, 42, 71, 222, 91, 79, - 162, 28, 69, 15, 65, 4, 146, 89, 85, 235, 30, 79, 8, 22, 88, 227, 88, 77, - 6, 222, 88, 85, 202, 2, 73, 163, 28, 79, 8, 22, 74, 151, 119, 79, 6, 230, - 90, 85, 218, 5, 73, 215, 22, 89, 4, 190, 90, 73, 175, 28, 69, 6, 186, 54, - 73, 199, 7, 89, 4, 166, 118, 79, 15, 73, 8, 30, 89, 26, 90, 183, 90, 66, - 4, 238, 117, 73, 3, 79, 2, 215, 117, 85, 4, 166, 89, 85, 3, 89, 10, 22, - 72, 207, 97, 83, 8, 210, 60, 85, 166, 28, 65, 162, 28, 79, 15, 89, 4, - 198, 88, 65, 175, 28, 85, 4, 134, 60, 85, 199, 56, 69, 4, 142, 88, 73, - 175, 28, 79, 10, 54, 85, 220, 62, 2, 90, 73, 226, 24, 79, 175, 28, 65, 4, - 230, 115, 80, 3, 82, 156, 18, 134, 2, 66, 134, 1, 67, 162, 1, 68, 110, - 70, 50, 71, 150, 1, 72, 138, 3, 73, 134, 1, 74, 98, 75, 54, 76, 62, 77, - 134, 1, 78, 234, 4, 80, 54, 81, 2, 89, 46, 82, 162, 1, 83, 134, 1, 84, - 102, 86, 82, 87, 58, 88, 50, 90, 140, 2, 2, 85, 79, 66, 65, 2, 79, 107, - 69, 132, 1, 66, 66, 238, 21, 85, 150, 1, 69, 26, 73, 42, 65, 2, 79, 67, - 89, 64, 234, 21, 85, 150, 1, 69, 26, 73, 42, 65, 2, 79, 3, 89, 122, 66, - 72, 238, 20, 85, 150, 1, 69, 26, 73, 42, 65, 2, 79, 67, 89, 54, 46, 85, - 146, 22, 65, 2, 69, 2, 79, 67, 89, 19, 142, 22, 79, 106, 82, 214, 88, 80, - 3, 88, 106, 58, 68, 138, 15, 85, 134, 5, 73, 94, 69, 66, 65, 3, 79, 54, - 210, 19, 85, 58, 73, 94, 69, 66, 65, 3, 79, 42, 182, 20, 79, 66, 65, 2, - 73, 2, 89, 67, 85, 116, 58, 71, 214, 15, 85, 146, 4, 73, 42, 65, 2, 69, - 3, 79, 56, 50, 73, 162, 15, 85, 186, 4, 65, 2, 69, 3, 79, 13, 150, 19, - 69, 254, 89, 84, 3, 88, 246, 1, 78, 73, 30, 76, 58, 77, 54, 78, 110, 88, - 50, 85, 254, 15, 69, 66, 65, 3, 79, 6, 198, 19, 69, 215, 88, 84, 64, 238, - 16, 85, 58, 73, 94, 69, 2, 79, 66, 65, 67, 89, 58, 182, 16, 85, 58, 73, - 158, 1, 65, 2, 79, 35, 89, 42, 46, 79, 34, 85, 202, 16, 69, 26, 73, 43, - 65, 6, 226, 106, 80, 2, 84, 3, 88, 6, 238, 17, 79, 215, 88, 84, 46, 46, - 85, 254, 15, 69, 26, 73, 42, 65, 3, 79, 8, 187, 16, 79, 19, 42, 84, 130, - 16, 69, 190, 89, 80, 3, 88, 5, 11, 69, 2, 11, 82, 2, 11, 65, 2, 11, 84, - 2, 11, 73, 2, 11, 79, 2, 187, 39, 78, 106, 50, 74, 190, 10, 85, 146, 4, - 73, 42, 79, 67, 89, 50, 158, 13, 85, 174, 1, 73, 42, 79, 3, 89, 56, 242, - 12, 85, 58, 73, 158, 1, 65, 2, 69, 3, 79, 70, 218, 9, 85, 250, 3, 69, 26, - 73, 42, 65, 2, 79, 67, 89, 106, 70, 71, 218, 8, 85, 158, 3, 73, 158, 1, - 65, 2, 79, 2, 89, 107, 69, 44, 186, 11, 85, 150, 1, 69, 66, 65, 2, 79, - 105, 2, 73, 69, 248, 2, 102, 66, 54, 68, 94, 71, 90, 74, 46, 82, 50, 89, - 78, 90, 134, 7, 85, 58, 73, 94, 65, 2, 69, 67, 79, 54, 202, 10, 73, 158, - 1, 65, 2, 79, 66, 85, 3, 89, 46, 46, 73, 198, 10, 69, 66, 65, 2, 79, 67, - 85, 13, 234, 11, 69, 214, 88, 80, 2, 84, 3, 88, 34, 60, 2, 85, 79, 218, - 9, 69, 0, 2, 73, 69, 66, 65, 3, 79, 7, 210, 99, 84, 3, 88, 50, 230, 1, - 85, 242, 7, 73, 42, 79, 67, 89, 46, 146, 9, 79, 66, 65, 2, 69, 66, 85, 3, - 89, 38, 30, 85, 222, 8, 73, 43, 79, 15, 194, 8, 79, 254, 89, 80, 2, 84, - 3, 88, 56, 62, 85, 206, 4, 79, 178, 2, 73, 158, 1, 65, 66, 89, 43, 69, - 15, 254, 8, 79, 2, 82, 214, 88, 80, 3, 88, 60, 150, 6, 85, 58, 73, 158, - 1, 65, 2, 79, 67, 89, 56, 254, 2, 85, 146, 4, 73, 42, 79, 67, 89, 100, - 58, 82, 254, 4, 85, 150, 1, 69, 66, 65, 2, 79, 67, 89, 48, 46, 85, 162, - 6, 69, 2, 79, 66, 89, 43, 65, 17, 134, 7, 79, 2, 82, 214, 88, 80, 2, 84, - 3, 88, 176, 1, 70, 83, 158, 3, 72, 50, 85, 58, 73, 94, 69, 66, 65, 2, 79, - 67, 89, 56, 130, 4, 73, 94, 69, 66, 65, 2, 79, 2, 85, 67, 89, 56, 46, 85, - 158, 3, 73, 94, 69, 66, 65, 3, 79, 21, 182, 4, 79, 106, 82, 214, 88, 80, - 2, 84, 3, 88, 60, 54, 69, 166, 3, 73, 42, 65, 2, 79, 66, 85, 3, 89, 4, - 134, 93, 80, 3, 88, 28, 38, 85, 206, 2, 69, 2, 79, 67, 65, 9, 203, 2, 79, - 40, 210, 2, 73, 42, 79, 66, 89, 41, 2, 85, 79, 178, 1, 66, 72, 50, 85, - 58, 73, 42, 90, 54, 69, 66, 65, 2, 79, 67, 89, 54, 46, 85, 214, 1, 65, 2, - 69, 2, 79, 67, 89, 19, 146, 1, 79, 170, 1, 82, 214, 88, 80, 2, 84, 3, 88, - 15, 90, 69, 254, 89, 80, 2, 84, 3, 88, 58, 50, 69, 2, 79, 26, 73, 42, 65, - 34, 85, 35, 89, 7, 250, 89, 80, 3, 88, 17, 38, 69, 190, 89, 80, 2, 84, 3, - 88, 9, 186, 89, 80, 2, 84, 3, 88, 11, 70, 82, 214, 88, 80, 3, 88, 13, 38, - 82, 214, 88, 80, 2, 84, 3, 88, 5, 211, 88, 88, 2, 215, 7, 65, 2, 191, 57, - 89, 180, 4, 252, 1, 10, 32, 78, 79, 84, 65, 84, 73, 79, 78, 32, 220, 6, - 16, 65, 78, 65, 66, 65, 90, 65, 82, 32, 83, 81, 85, 65, 82, 69, 32, 210, - 15, 69, 180, 2, 6, 73, 80, 80, 69, 82, 45, 96, 8, 78, 65, 77, 69, 78, 78, - 89, 32, 176, 33, 3, 79, 77, 66, 235, 26, 87, 26, 144, 1, 9, 66, 65, 71, - 32, 77, 69, 77, 66, 69, 38, 82, 104, 6, 68, 79, 77, 65, 73, 78, 60, 3, - 76, 69, 70, 154, 1, 83, 145, 2, 3, 84, 89, 80, 2, 11, 82, 2, 209, 83, 2, - 83, 72, 8, 100, 4, 65, 78, 71, 69, 60, 3, 73, 71, 72, 221, 1, 11, 69, 76, - 65, 84, 73, 79, 78, 65, 76, 32, 67, 2, 173, 3, 11, 32, 65, 78, 84, 73, - 82, 69, 83, 84, 82, 73, 4, 17, 2, 84, 32, 4, 60, 4, 73, 77, 65, 71, 13, - 7, 66, 73, 78, 68, 73, 78, 71, 2, 11, 69, 2, 25, 4, 32, 66, 82, 65, 2, - 11, 67, 2, 175, 29, 75, 8, 44, 6, 67, 72, 69, 77, 65, 32, 211, 1, 80, 6, - 18, 67, 71, 80, 2, 17, 2, 79, 77, 2, 11, 80, 2, 11, 79, 2, 11, 83, 2, - 107, 73, 4, 30, 73, 41, 3, 82, 79, 74, 2, 11, 80, 2, 11, 73, 2, 151, 80, - 78, 2, 11, 69, 2, 11, 67, 2, 11, 84, 2, 87, 73, 2, 255, 51, 79, 2, 11, - 69, 2, 11, 32, 2, 11, 67, 2, 11, 79, 2, 11, 76, 2, 11, 79, 2, 211, 79, - 78, 144, 1, 240, 1, 2, 67, 76, 68, 7, 73, 78, 73, 84, 73, 65, 76, 204, 2, - 14, 70, 73, 78, 65, 76, 32, 67, 79, 78, 83, 79, 78, 65, 78, 16, 7, 76, - 69, 84, 84, 69, 82, 32, 148, 3, 5, 77, 65, 82, 75, 32, 206, 1, 83, 181, - 3, 6, 86, 79, 87, 69, 76, 32, 14, 64, 5, 79, 83, 73, 78, 71, 137, 1, 6, - 85, 83, 84, 69, 82, 45, 4, 11, 32, 4, 64, 12, 68, 79, 85, 66, 76, 69, 45, - 76, 73, 78, 69, 68, 23, 72, 2, 17, 2, 32, 72, 2, 17, 2, 69, 65, 2, 215, - 10, 68, 10, 96, 13, 70, 73, 78, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, - 41, 7, 73, 78, 73, 84, 73, 65, 76, 8, 226, 72, 76, 2, 82, 2, 86, 3, 89, - 2, 37, 7, 32, 76, 69, 84, 84, 69, 82, 2, 151, 6, 32, 2, 131, 9, 84, 82, - 166, 1, 68, 50, 75, 38, 78, 46, 83, 38, 84, 46, 66, 2, 67, 2, 71, 2, 80, - 2, 90, 254, 68, 45, 2, 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, 86, 2, 89, 187, - 2, 65, 12, 206, 1, 68, 2, 90, 254, 68, 72, 187, 2, 65, 6, 142, 70, 83, - 14, 72, 187, 2, 65, 8, 246, 69, 71, 2, 78, 2, 89, 187, 2, 65, 6, 202, 69, - 72, 2, 83, 187, 2, 65, 12, 42, 83, 2, 84, 254, 68, 72, 187, 2, 65, 4, - 250, 68, 72, 187, 2, 65, 8, 50, 68, 58, 83, 40, 4, 76, 79, 78, 71, 23, - 84, 2, 29, 5, 79, 85, 66, 76, 69, 2, 11, 32, 2, 11, 83, 2, 11, 72, 2, 11, - 65, 2, 147, 70, 68, 2, 17, 2, 32, 84, 2, 17, 2, 83, 72, 2, 135, 69, 69, - 14, 36, 4, 73, 71, 78, 32, 255, 2, 85, 12, 54, 65, 72, 6, 67, 65, 78, 68, - 82, 65, 167, 1, 86, 2, 11, 78, 2, 11, 85, 2, 17, 2, 83, 86, 2, 11, 65, 2, - 251, 65, 82, 6, 36, 5, 66, 73, 78, 68, 85, 15, 32, 5, 11, 32, 2, 25, 4, - 87, 73, 84, 72, 2, 17, 2, 32, 79, 2, 21, 3, 82, 78, 65, 2, 11, 77, 2, 11, - 69, 2, 227, 38, 78, 4, 11, 73, 4, 18, 82, 19, 83, 2, 207, 33, 65, 2, 11, - 65, 2, 11, 82, 2, 255, 63, 71, 2, 151, 4, 66, 20, 38, 76, 109, 5, 83, 73, - 71, 78, 32, 2, 17, 2, 69, 78, 2, 11, 71, 2, 11, 84, 2, 11, 72, 2, 11, 32, - 2, 11, 77, 2, 11, 65, 2, 251, 61, 82, 18, 86, 65, 26, 79, 2, 85, 16, 8, - 82, 69, 86, 69, 82, 83, 69, 68, 134, 64, 69, 3, 73, 4, 170, 64, 73, 3, - 85, 5, 147, 64, 69, 2, 171, 44, 32, 12, 72, 2, 66, 82, 16, 9, 82, 79, 32, - 87, 73, 68, 84, 72, 32, 203, 1, 85, 2, 147, 2, 65, 8, 32, 2, 78, 79, 86, - 83, 31, 74, 4, 26, 45, 73, 2, 78, 45, 2, 29, 5, 66, 82, 69, 65, 75, 2, - 11, 32, 2, 11, 83, 2, 163, 1, 80, 2, 11, 74, 2, 11, 79, 2, 11, 73, 2, 11, - 78, 2, 143, 5, 69, 2, 207, 61, 83, 2, 21, 3, 77, 79, 85, 2, 17, 2, 84, - 72, 2, 11, 32, 2, 11, 70, 2, 11, 65, 2, 155, 38, 67, 242, 2, 168, 1, 10, - 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 212, 18, 6, 78, 69, 85, 77, 69, - 32, 181, 38, 17, 80, 82, 73, 90, 78, 65, 75, 32, 77, 79, 68, 73, 70, 73, - 69, 82, 32, 128, 1, 144, 2, 17, 76, 79, 87, 69, 82, 32, 84, 79, 78, 65, - 76, 32, 82, 65, 78, 71, 69, 88, 5, 77, 65, 82, 75, 32, 148, 3, 18, 65, - 84, 84, 65, 67, 72, 73, 78, 71, 32, 86, 69, 82, 84, 73, 67, 65, 76, 221, - 11, 17, 84, 79, 78, 65, 76, 32, 82, 65, 78, 71, 69, 32, 77, 65, 82, 75, - 32, 2, 33, 6, 32, 73, 78, 68, 73, 67, 2, 11, 65, 2, 11, 84, 2, 11, 79, 2, - 207, 56, 82, 118, 182, 2, 67, 142, 1, 68, 104, 8, 71, 79, 82, 65, 90, 68, - 79, 32, 34, 78, 106, 75, 114, 79, 72, 2, 80, 79, 140, 2, 8, 77, 65, 76, - 79, 32, 80, 79, 86, 100, 2, 82, 65, 50, 83, 186, 1, 84, 108, 7, 86, 89, - 83, 79, 75, 79, 32, 234, 1, 90, 224, 10, 2, 76, 79, 224, 22, 4, 85, 68, - 65, 82, 129, 6, 4, 66, 79, 82, 90, 6, 60, 6, 72, 65, 83, 72, 75, 65, 29, - 5, 85, 82, 86, 69, 68, 5, 185, 47, 3, 32, 80, 79, 2, 17, 2, 32, 79, 2, - 11, 77, 2, 223, 24, 69, 4, 228, 10, 13, 69, 77, 69, 83, 84, 86, 69, 78, - 78, 89, 32, 90, 65, 217, 14, 6, 86, 79, 69, 84, 79, 67, 10, 30, 78, 89, - 3, 86, 89, 83, 8, 29, 5, 73, 90, 75, 79, 32, 8, 152, 8, 8, 83, 32, 75, - 82, 89, 90, 72, 69, 35, 79, 2, 155, 20, 79, 8, 56, 4, 82, 89, 90, 72, - 138, 6, 65, 233, 39, 2, 85, 80, 5, 21, 3, 32, 79, 78, 2, 11, 32, 2, 191, - 7, 76, 6, 208, 5, 3, 84, 83, 69, 200, 13, 5, 66, 76, 65, 67, 72, 131, 31, - 78, 18, 22, 68, 247, 1, 86, 6, 60, 7, 67, 72, 65, 83, 72, 73, 69, 189, - 22, 3, 86, 69, 82, 5, 29, 5, 32, 87, 73, 84, 72, 2, 17, 2, 32, 86, 2, 29, - 5, 69, 82, 84, 73, 67, 2, 17, 2, 65, 76, 2, 11, 32, 2, 11, 83, 2, 11, 84, - 2, 11, 82, 2, 11, 79, 2, 131, 25, 75, 12, 29, 5, 89, 83, 72, 69, 32, 12, - 22, 83, 251, 3, 79, 8, 154, 3, 32, 229, 2, 4, 84, 82, 65, 78, 4, 28, 2, - 90, 83, 183, 5, 86, 2, 219, 37, 69, 10, 136, 1, 2, 75, 79, 16, 16, 84, - 82, 65, 78, 78, 79, 32, 77, 65, 76, 79, 32, 80, 79, 86, 89, 236, 1, 5, - 82, 69, 68, 78, 69, 231, 26, 79, 2, 239, 42, 66, 2, 11, 83, 2, 183, 22, - 72, 10, 50, 79, 16, 5, 83, 65, 84, 65, 32, 139, 36, 73, 2, 187, 28, 67, - 6, 158, 1, 79, 201, 24, 3, 83, 32, 75, 10, 24, 2, 83, 32, 95, 79, 6, 11, - 75, 6, 44, 6, 72, 79, 75, 72, 76, 79, 247, 24, 82, 4, 11, 77, 4, 17, 2, - 32, 79, 4, 11, 78, 4, 11, 32, 4, 18, 76, 31, 82, 2, 11, 69, 2, 179, 14, - 70, 2, 11, 73, 2, 11, 71, 2, 139, 14, 72, 6, 18, 65, 31, 69, 2, 249, 25, - 3, 68, 69, 82, 4, 22, 86, 239, 10, 76, 2, 199, 38, 79, 6, 60, 5, 77, 82, - 65, 67, 72, 18, 83, 1, 4, 84, 82, 69, 83, 2, 155, 10, 78, 2, 17, 2, 86, - 69, 2, 155, 6, 84, 232, 1, 128, 2, 2, 67, 72, 38, 68, 218, 1, 70, 28, 10, - 71, 79, 76, 85, 66, 67, 72, 73, 75, 32, 158, 1, 75, 160, 2, 6, 77, 69, - 67, 72, 73, 75, 184, 1, 2, 78, 69, 18, 79, 138, 2, 80, 146, 2, 82, 114, - 83, 180, 20, 9, 86, 82, 65, 75, 72, 73, 89, 65, 32, 151, 2, 90, 4, 246, - 11, 69, 209, 10, 2, 65, 83, 10, 74, 69, 98, 85, 16, 9, 86, 65, 32, 86, - 32, 67, 72, 69, 76, 187, 27, 79, 4, 76, 2, 82, 66, 213, 2, 12, 77, 69, - 83, 84, 86, 69, 78, 78, 89, 32, 75, 76, 2, 195, 34, 73, 2, 203, 34, 68, - 2, 11, 78, 2, 231, 36, 85, 2, 11, 73, 2, 147, 34, 84, 10, 78, 84, 38, 83, - 240, 3, 5, 77, 82, 65, 67, 72, 145, 14, 4, 66, 79, 82, 90, 4, 32, 3, 82, - 69, 83, 251, 1, 73, 2, 21, 3, 86, 69, 84, 2, 231, 17, 76, 14, 76, 4, 72, - 65, 77, 73, 18, 76, 40, 3, 79, 66, 89, 16, 2, 82, 89, 87, 85, 2, 219, 3, - 76, 2, 11, 89, 2, 11, 85, 2, 151, 33, 67, 2, 223, 31, 76, 6, 28, 2, 85, - 75, 219, 32, 90, 5, 21, 3, 32, 84, 73, 2, 11, 75, 2, 251, 15, 72, 2, 11, - 70, 2, 11, 73, 2, 11, 83, 2, 215, 30, 77, 11, 11, 32, 8, 44, 7, 75, 76, - 89, 85, 67, 72, 69, 83, 80, 6, 64, 9, 78, 69, 80, 79, 83, 84, 79, 89, 65, - 14, 80, 163, 14, 86, 2, 39, 78, 2, 25, 4, 79, 86, 79, 68, 2, 143, 14, 78, - 2, 223, 22, 77, 14, 40, 2, 66, 76, 41, 4, 83, 79, 75, 65, 2, 11, 65, 2, - 11, 75, 2, 243, 30, 79, 13, 11, 32, 10, 30, 75, 142, 26, 84, 39, 83, 6, - 104, 11, 76, 89, 85, 67, 72, 69, 86, 65, 89, 65, 32, 185, 25, 10, 82, 89, - 85, 75, 79, 86, 65, 89, 65, 32, 4, 202, 16, 78, 251, 8, 83, 14, 58, 65, - 88, 8, 69, 82, 69, 86, 79, 68, 75, 65, 27, 79, 6, 44, 3, 82, 65, 75, 222, - 19, 76, 211, 5, 85, 2, 11, 76, 2, 11, 73, 2, 171, 28, 84, 5, 153, 15, 2, - 32, 78, 4, 68, 5, 68, 67, 72, 65, 83, 245, 9, 7, 76, 75, 85, 76, 73, 90, - 77, 2, 11, 72, 2, 219, 4, 73, 4, 64, 11, 69, 86, 69, 82, 83, 69, 68, 32, - 67, 72, 69, 139, 26, 79, 2, 25, 4, 76, 89, 85, 83, 2, 215, 17, 84, 126, - 96, 9, 75, 65, 77, 69, 89, 84, 83, 65, 32, 148, 2, 8, 76, 79, 90, 72, 73, - 84, 73, 69, 119, 84, 22, 128, 1, 13, 68, 86, 79, 69, 67, 72, 69, 76, 78, - 65, 89, 65, 32, 44, 7, 75, 76, 89, 85, 67, 72, 69, 74, 84, 218, 18, 77, - 119, 83, 8, 218, 10, 75, 110, 78, 178, 8, 80, 75, 83, 6, 40, 5, 86, 65, - 89, 65, 32, 243, 10, 78, 4, 178, 15, 84, 183, 4, 83, 4, 162, 15, 73, 147, - 4, 82, 9, 11, 32, 6, 48, 2, 83, 32, 25, 6, 90, 65, 75, 82, 89, 84, 4, - 166, 4, 75, 95, 90, 2, 11, 79, 2, 211, 22, 69, 96, 92, 4, 65, 84, 89, 65, - 172, 4, 6, 79, 80, 73, 84, 83, 65, 189, 1, 5, 82, 69, 76, 65, 32, 23, 11, - 32, 20, 76, 2, 83, 32, 236, 2, 9, 90, 65, 75, 82, 89, 84, 65, 89, 65, - 159, 5, 78, 14, 160, 1, 14, 68, 86, 85, 77, 89, 65, 32, 90, 65, 80, 89, - 65, 84, 89, 28, 7, 75, 82, 89, 90, 72, 69, 77, 24, 2, 82, 79, 17, 8, 90, - 65, 80, 89, 65, 84, 79, 89, 2, 11, 77, 2, 215, 19, 73, 5, 189, 1, 2, 32, - 73, 2, 203, 2, 71, 7, 21, 3, 32, 73, 32, 4, 54, 75, 37, 9, 80, 79, 68, - 67, 72, 65, 83, 72, 73, 2, 11, 82, 2, 21, 3, 89, 90, 72, 2, 211, 1, 69, - 5, 17, 2, 32, 83, 2, 17, 2, 32, 90, 2, 29, 5, 65, 80, 89, 65, 84, 2, 11, - 79, 2, 199, 17, 89, 7, 11, 32, 4, 68, 6, 83, 32, 79, 67, 72, 75, 29, 7, - 87, 73, 84, 72, 32, 83, 79, 2, 11, 79, 2, 215, 16, 77, 2, 45, 9, 82, 79, - 67, 72, 89, 65, 32, 78, 79, 2, 11, 90, 2, 163, 7, 72, 68, 140, 1, 9, 68, - 86, 79, 69, 67, 72, 69, 76, 78, 162, 1, 75, 78, 78, 174, 1, 71, 152, 3, - 8, 77, 82, 65, 67, 72, 78, 79, 84, 42, 80, 75, 84, 10, 18, 79, 71, 65, 6, - 64, 7, 80, 79, 86, 79, 68, 78, 65, 189, 5, 4, 75, 82, 89, 90, 4, 17, 2, - 89, 65, 5, 17, 2, 32, 75, 2, 145, 5, 4, 76, 89, 85, 67, 26, 48, 6, 76, - 89, 85, 67, 72, 69, 77, 2, 82, 89, 4, 22, 78, 203, 6, 80, 2, 157, 8, 9, - 69, 80, 79, 83, 84, 79, 89, 65, 78, 22, 48, 7, 85, 75, 79, 86, 65, 89, - 65, 195, 3, 90, 21, 11, 32, 18, 54, 71, 236, 2, 5, 84, 82, 89, 65, 83, - 179, 2, 80, 14, 21, 3, 82, 79, 77, 14, 32, 4, 78, 65, 89, 65, 47, 79, 5, - 237, 1, 7, 32, 87, 73, 84, 72, 32, 83, 10, 92, 10, 75, 82, 89, 90, 72, - 69, 86, 65, 89, 65, 17, 9, 80, 79, 86, 79, 68, 78, 65, 89, 65, 5, 175, 2, - 32, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 24, 2, 68, 79, 23, 83, 2, 41, 2, - 85, 66, 2, 21, 3, 73, 78, 71, 2, 133, 6, 6, 76, 69, 32, 90, 65, 80, 2, - 175, 6, 75, 2, 237, 5, 3, 72, 69, 86, 2, 11, 73, 2, 11, 75, 2, 187, 5, - 72, 6, 22, 79, 187, 3, 82, 4, 28, 2, 76, 85, 187, 1, 86, 2, 163, 1, 80, - 8, 80, 5, 82, 89, 65, 83, 79, 153, 2, 10, 73, 75, 72, 65, 89, 65, 32, 80, - 85, 84, 6, 62, 80, 44, 4, 83, 84, 82, 69, 173, 1, 4, 71, 76, 65, 83, 2, - 17, 2, 79, 86, 2, 193, 1, 2, 79, 68, 2, 171, 1, 76, 16, 88, 12, 75, 76, - 89, 85, 67, 72, 69, 86, 65, 89, 65, 32, 38, 77, 46, 80, 38, 84, 39, 83, - 8, 34, 77, 46, 80, 38, 84, 39, 83, 2, 25, 4, 82, 65, 67, 72, 2, 247, 1, - 78, 2, 11, 82, 2, 205, 1, 2, 79, 83, 2, 11, 82, 2, 11, 69, 2, 11, 83, 2, - 157, 1, 4, 86, 69, 84, 76, 6, 30, 65, 121, 3, 77, 69, 89, 4, 32, 4, 78, - 79, 90, 72, 31, 80, 2, 11, 69, 2, 151, 3, 75, 2, 17, 2, 89, 65, 2, 11, - 84, 2, 11, 65, 2, 35, 89, 2, 11, 84, 2, 11, 83, 2, 183, 2, 65, 10, 108, - 11, 68, 73, 82, 69, 67, 84, 73, 79, 78, 32, 70, 28, 3, 75, 82, 89, 28, 5, - 76, 69, 86, 69, 76, 35, 82, 2, 11, 76, 2, 159, 1, 73, 2, 11, 90, 2, 143, - 1, 72, 4, 11, 45, 4, 114, 50, 3, 51, 2, 11, 79, 2, 83, 71, 8, 26, 78, 34, - 83, 15, 74, 4, 18, 66, 27, 74, 2, 11, 83, 2, 11, 80, 3, 0, + 65, 67, 75, 32, 85, 5, 87, 73, 84, 72, 32, 2, 17, 2, 85, 84, 2, 159, 99, + 84, 6, 54, 86, 178, 17, 83, 37, 6, 77, 69, 68, 73, 85, 77, 2, 153, 17, 3, + 69, 82, 89, 18, 150, 1, 76, 50, 82, 214, 1, 85, 144, 1, 17, 86, 69, 82, + 84, 73, 67, 65, 76, 32, 66, 73, 83, 69, 67, 84, 73, 78, 233, 19, 6, 67, + 69, 78, 84, 82, 69, 6, 18, 69, 15, 79, 2, 67, 70, 4, 247, 1, 87, 4, 18, + 73, 115, 79, 2, 17, 2, 71, 72, 2, 33, 6, 84, 87, 65, 82, 68, 83, 2, 11, + 32, 2, 11, 84, 2, 11, 73, 2, 215, 172, 1, 67, 2, 29, 5, 85, 78, 68, 69, + 68, 2, 21, 3, 32, 67, 79, 2, 193, 32, 2, 82, 78, 4, 17, 2, 80, 80, 4, 21, + 3, 69, 82, 32, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 33, 6, 84, + 32, 81, 85, 65, 68, 2, 175, 36, 82, 2, 171, 20, 71, 11, 11, 32, 8, 84, + 12, 66, 69, 72, 73, 78, 68, 32, 67, 76, 79, 85, 68, 69, 5, 87, 73, 84, + 72, 32, 5, 29, 5, 32, 87, 73, 84, 72, 2, 17, 2, 32, 82, 2, 255, 44, 65, + 4, 38, 82, 29, 5, 83, 77, 65, 76, 76, 2, 11, 65, 2, 179, 110, 89, 2, 11, + 32, 2, 21, 3, 67, 76, 79, 2, 187, 101, 85, 10, 38, 79, 54, 69, 62, 82, + 203, 1, 87, 2, 49, 10, 85, 67, 72, 84, 79, 78, 69, 32, 84, 69, 2, 17, 2, + 76, 69, 2, 11, 80, 2, 11, 72, 2, 151, 17, 79, 4, 148, 1, 4, 65, 80, 69, + 90, 33, 28, 73, 65, 78, 71, 76, 69, 32, 67, 79, 78, 84, 65, 73, 78, 73, + 78, 71, 32, 83, 77, 65, 76, 76, 32, 87, 72, 73, 84, 2, 11, 73, 2, 183, + 152, 1, 85, 2, 135, 4, 69, 2, 11, 79, 2, 85, 19, 45, 87, 65, 89, 32, 76, + 69, 70, 84, 32, 87, 65, 89, 32, 84, 82, 65, 70, 70, 2, 11, 73, 2, 251, + 167, 1, 67, 12, 62, 32, 185, 1, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, + 32, 4, 11, 80, 4, 41, 8, 79, 73, 78, 84, 73, 78, 71, 32, 4, 18, 66, 75, + 73, 2, 21, 3, 65, 67, 75, 2, 25, 4, 72, 65, 78, 68, 2, 17, 2, 32, 73, 2, + 17, 2, 78, 68, 2, 219, 25, 69, 8, 54, 67, 42, 83, 125, 7, 84, 82, 73, 65, + 78, 71, 76, 2, 21, 3, 72, 69, 86, 2, 179, 85, 82, 2, 25, 4, 77, 65, 76, + 76, 2, 17, 2, 32, 84, 2, 17, 2, 82, 73, 2, 11, 65, 2, 11, 78, 2, 11, 71, + 2, 211, 141, 1, 76, 4, 11, 69, 5, 11, 32, 2, 11, 87, 2, 217, 18, 3, 73, + 84, 72, 10, 44, 6, 84, 73, 67, 65, 76, 32, 255, 1, 89, 8, 62, 69, 48, 9, + 82, 69, 67, 84, 65, 78, 71, 76, 69, 127, 66, 2, 11, 76, 2, 17, 2, 76, 73, + 2, 183, 25, 80, 5, 37, 7, 32, 87, 73, 84, 72, 32, 72, 2, 37, 7, 79, 82, + 73, 90, 79, 78, 84, 2, 17, 2, 65, 76, 2, 11, 32, 2, 11, 66, 2, 219, 104, + 65, 2, 17, 2, 32, 83, 2, 11, 77, 2, 21, 3, 65, 76, 76, 2, 17, 2, 32, 83, + 2, 11, 81, 2, 11, 85, 2, 11, 65, 2, 227, 137, 1, 82, 2, 11, 69, 2, 11, + 68, 2, 17, 2, 32, 82, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 11, 87, 2, 241, + 4, 4, 65, 82, 68, 83, 100, 140, 1, 10, 68, 69, 45, 72, 69, 65, 68, 69, + 68, 32, 132, 4, 4, 71, 71, 76, 89, 124, 6, 76, 84, 69, 68, 32, 70, 42, + 78, 189, 1, 2, 82, 69, 80, 128, 1, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, + 12, 4, 68, 79, 87, 78, 0, 2, 85, 80, 32, 3, 78, 79, 82, 1, 3, 83, 79, 85, + 10, 11, 84, 10, 109, 5, 87, 65, 82, 68, 83, 20, 21, 3, 84, 72, 32, 20, + 32, 2, 69, 65, 1, 2, 87, 69, 10, 17, 2, 83, 84, 10, 11, 32, 10, 110, 72, + 0, 6, 86, 69, 82, 89, 32, 72, 28, 5, 76, 73, 71, 72, 84, 0, 6, 77, 69, + 68, 73, 85, 77, 23, 66, 2, 25, 4, 69, 65, 86, 89, 2, 17, 2, 32, 66, 2, + 21, 3, 65, 82, 66, 2, 11, 32, 2, 11, 65, 2, 11, 82, 2, 11, 82, 2, 131, + 28, 79, 2, 17, 2, 32, 86, 2, 11, 69, 2, 33, 6, 82, 84, 73, 67, 65, 76, 2, + 11, 32, 2, 11, 76, 2, 11, 73, 2, 215, 130, 1, 78, 2, 11, 76, 2, 11, 79, + 2, 147, 91, 87, 12, 46, 68, 106, 69, 186, 15, 75, 163, 136, 1, 71, 6, 22, + 32, 139, 26, 79, 4, 44, 2, 67, 72, 217, 15, 4, 66, 76, 79, 87, 2, 11, 73, + 2, 151, 129, 1, 77, 2, 11, 32, 2, 121, 3, 71, 76, 65, 4, 36, 5, 68, 32, + 75, 69, 89, 51, 76, 2, 17, 2, 66, 79, 2, 11, 65, 2, 203, 80, 82, 2, 11, + 69, 2, 243, 88, 83, 30, 66, 77, 158, 3, 82, 226, 11, 78, 226, 64, 79, + 129, 9, 2, 76, 70, 14, 36, 2, 65, 78, 161, 2, 2, 69, 78, 13, 72, 12, 32, + 87, 73, 84, 72, 32, 66, 85, 78, 78, 89, 32, 29, 2, 83, 32, 2, 11, 69, 2, + 167, 6, 65, 8, 48, 2, 66, 79, 28, 2, 67, 76, 54, 72, 19, 83, 2, 11, 79, + 2, 207, 86, 84, 2, 11, 79, 2, 11, 84, 2, 11, 72, 2, 155, 86, 69, 2, 187, + 119, 65, 2, 17, 2, 65, 78, 2, 131, 10, 68, 2, 11, 83, 2, 11, 32, 2, 11, + 83, 2, 11, 89, 2, 17, 2, 77, 66, 2, 187, 9, 79, 10, 72, 2, 68, 32, 156, + 1, 3, 76, 68, 32, 32, 2, 82, 73, 219, 144, 1, 77, 4, 56, 9, 83, 69, 80, + 65, 82, 65, 84, 79, 82, 203, 83, 74, 2, 17, 2, 32, 77, 2, 21, 3, 73, 68, + 68, 2, 11, 76, 2, 11, 69, 2, 11, 32, 2, 207, 64, 68, 2, 11, 77, 2, 223, + 144, 1, 65, 2, 11, 69, 2, 171, 83, 68, 10, 56, 7, 65, 80, 80, 69, 68, 32, + 80, 30, 69, 163, 1, 73, 2, 213, 76, 3, 82, 69, 83, 6, 48, 2, 65, 84, 80, + 3, 83, 84, 76, 187, 108, 78, 2, 11, 72, 2, 11, 32, 2, 11, 80, 2, 25, 4, + 82, 79, 68, 85, 2, 195, 114, 67, 2, 11, 69, 2, 247, 80, 82, 2, 11, 84, 2, + 11, 73, 2, 17, 2, 78, 71, 2, 17, 2, 32, 72, 2, 11, 65, 2, 215, 71, 78, + 34, 76, 2, 32, 73, 138, 1, 45, 28, 7, 73, 65, 78, 71, 81, 73, 32, 155, + 83, 79, 2, 53, 11, 78, 32, 65, 32, 82, 69, 67, 84, 65, 78, 71, 2, 11, 76, + 2, 11, 69, 2, 11, 32, 2, 11, 66, 2, 11, 79, 2, 163, 140, 1, 88, 2, 11, + 82, 2, 187, 122, 65, 28, 48, 5, 66, 76, 65, 67, 75, 1, 3, 82, 69, 68, 14, + 11, 32, 14, 130, 1, 67, 72, 4, 69, 76, 69, 80, 28, 4, 71, 69, 78, 69, 46, + 72, 36, 4, 83, 79, 76, 68, 169, 9, 6, 77, 65, 78, 68, 65, 82, 4, 24, 2, + 65, 78, 19, 72, 2, 167, 58, 78, 2, 201, 57, 3, 65, 82, 73, 2, 11, 72, 2, + 191, 70, 65, 2, 11, 82, 2, 11, 65, 2, 179, 137, 1, 76, 2, 17, 2, 79, 82, + 2, 183, 114, 83, 2, 143, 75, 73, 242, 19, 50, 65, 58, 69, 226, 11, 73, + 217, 35, 2, 79, 45, 2, 17, 2, 87, 78, 2, 11, 73, 2, 241, 74, 2, 78, 71, + 98, 60, 4, 76, 76, 79, 87, 62, 78, 53, 5, 90, 73, 68, 73, 32, 2, 17, 2, + 32, 72, 2, 11, 69, 2, 11, 65, 2, 227, 106, 82, 2, 11, 32, 2, 11, 83, 2, + 11, 73, 2, 251, 54, 71, 94, 112, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, + 32, 76, 5, 72, 89, 80, 72, 69, 17, 7, 76, 69, 84, 84, 69, 82, 32, 4, 44, + 3, 77, 65, 68, 13, 4, 72, 65, 77, 90, 2, 11, 68, 2, 215, 67, 65, 2, 215, + 27, 78, 88, 250, 1, 67, 50, 77, 18, 68, 42, 69, 58, 72, 30, 75, 22, 71, + 2, 81, 32, 3, 76, 65, 77, 98, 78, 18, 80, 30, 83, 66, 84, 28, 2, 86, 65, + 126, 87, 14, 79, 18, 88, 52, 3, 89, 79, 84, 154, 1, 90, 134, 53, 82, 238, + 48, 66, 254, 5, 85, 162, 14, 70, 3, 74, 6, 22, 72, 147, 114, 73, 4, 22, + 72, 251, 113, 73, 2, 247, 113, 73, 4, 11, 65, 4, 178, 130, 1, 68, 3, 76, + 8, 42, 76, 142, 50, 89, 226, 79, 84, 3, 87, 2, 71, 73, 4, 150, 112, 65, + 147, 15, 72, 4, 18, 72, 15, 65, 2, 11, 65, 2, 163, 129, 1, 70, 5, 11, 32, + 2, 11, 87, 2, 21, 3, 73, 84, 72, 2, 17, 2, 32, 68, 2, 11, 79, 2, 187, 3, + 84, 2, 207, 48, 85, 4, 202, 105, 72, 215, 22, 69, 8, 46, 72, 246, 47, 73, + 194, 9, 65, 163, 70, 69, 2, 243, 47, 73, 4, 238, 104, 72, 215, 22, 65, 5, + 37, 7, 32, 65, 76, 84, 69, 82, 78, 2, 17, 2, 65, 84, 2, 11, 69, 2, 11, + 32, 2, 11, 70, 2, 11, 79, 2, 227, 109, 82, 2, 11, 65, 2, 159, 126, 87, 4, + 22, 72, 251, 125, 65, 2, 11, 69, 2, 139, 46, 89, 5, 37, 7, 32, 87, 73, + 84, 72, 32, 67, 2, 45, 9, 73, 82, 67, 85, 77, 70, 76, 69, 88, 2, 11, 32, + 2, 11, 65, 2, 11, 66, 2, 11, 79, 2, 255, 101, 86, 6, 22, 65, 175, 124, + 69, 5, 171, 124, 76, 140, 19, 34, 32, 161, 35, 3, 78, 32, 89, 138, 19, + 88, 8, 82, 65, 68, 73, 67, 65, 76, 32, 181, 7, 9, 83, 89, 76, 76, 65, 66, + 76, 69, 32, 110, 170, 1, 66, 42, 67, 86, 68, 42, 71, 74, 72, 66, 74, 66, + 75, 30, 76, 30, 77, 26, 78, 74, 80, 26, 83, 74, 84, 30, 86, 30, 89, 30, + 90, 250, 35, 81, 198, 49, 87, 235, 30, 79, 4, 22, 66, 247, 64, 85, 2, + 163, 93, 85, 12, 42, 85, 18, 89, 178, 98, 72, 203, 22, 73, 2, 135, 121, + 79, 7, 130, 121, 80, 3, 84, 4, 22, 68, 215, 120, 85, 2, 247, 63, 85, 10, + 42, 71, 238, 91, 79, 162, 28, 69, 15, 65, 4, 162, 89, 85, 235, 30, 79, 8, + 22, 88, 243, 88, 77, 6, 238, 88, 85, 202, 2, 73, 163, 28, 79, 8, 22, 74, + 167, 119, 79, 6, 246, 90, 85, 218, 5, 73, 215, 22, 89, 4, 206, 90, 73, + 175, 28, 69, 6, 190, 54, 73, 199, 7, 89, 4, 182, 118, 79, 15, 73, 8, 30, + 89, 26, 90, 199, 90, 66, 4, 254, 117, 73, 3, 79, 2, 231, 117, 85, 4, 182, + 89, 85, 3, 89, 10, 22, 72, 223, 97, 83, 8, 214, 60, 85, 178, 28, 65, 162, + 28, 79, 15, 89, 4, 214, 88, 65, 175, 28, 85, 4, 138, 60, 85, 211, 56, 69, + 4, 158, 88, 73, 175, 28, 79, 10, 54, 85, 224, 62, 2, 90, 73, 238, 24, 79, + 175, 28, 65, 4, 246, 115, 80, 3, 82, 156, 18, 134, 2, 66, 134, 1, 67, + 162, 1, 68, 110, 70, 50, 71, 150, 1, 72, 138, 3, 73, 134, 1, 74, 98, 75, + 54, 76, 62, 77, 134, 1, 78, 234, 4, 80, 54, 81, 2, 89, 46, 82, 162, 1, + 83, 134, 1, 84, 102, 86, 82, 87, 58, 88, 50, 90, 140, 2, 2, 85, 79, 66, + 65, 2, 79, 107, 69, 132, 1, 66, 66, 238, 21, 85, 150, 1, 69, 26, 73, 42, + 65, 2, 79, 67, 89, 64, 234, 21, 85, 150, 1, 69, 26, 73, 42, 65, 2, 79, 3, + 89, 122, 66, 72, 238, 20, 85, 150, 1, 69, 26, 73, 42, 65, 2, 79, 67, 89, + 54, 46, 85, 146, 22, 65, 2, 69, 2, 79, 67, 89, 19, 142, 22, 79, 106, 82, + 230, 88, 80, 3, 88, 106, 58, 68, 138, 15, 85, 134, 5, 73, 94, 69, 66, 65, + 3, 79, 54, 210, 19, 85, 58, 73, 94, 69, 66, 65, 3, 79, 42, 182, 20, 79, + 66, 65, 2, 73, 2, 89, 67, 85, 116, 58, 71, 214, 15, 85, 146, 4, 73, 42, + 65, 2, 69, 3, 79, 56, 50, 73, 162, 15, 85, 186, 4, 65, 2, 69, 3, 79, 13, + 150, 19, 69, 142, 90, 84, 3, 88, 246, 1, 78, 73, 30, 76, 58, 77, 54, 78, + 110, 88, 50, 85, 254, 15, 69, 66, 65, 3, 79, 6, 198, 19, 69, 231, 88, 84, + 64, 238, 16, 85, 58, 73, 94, 69, 2, 79, 66, 65, 67, 89, 58, 182, 16, 85, + 58, 73, 158, 1, 65, 2, 79, 35, 89, 42, 46, 79, 34, 85, 202, 16, 69, 26, + 73, 43, 65, 6, 242, 106, 80, 2, 84, 3, 88, 6, 238, 17, 79, 231, 88, 84, + 46, 46, 85, 254, 15, 69, 26, 73, 42, 65, 3, 79, 8, 187, 16, 79, 19, 42, + 84, 130, 16, 69, 206, 89, 80, 3, 88, 5, 11, 69, 2, 11, 82, 2, 11, 65, 2, + 11, 84, 2, 11, 73, 2, 11, 79, 2, 191, 39, 78, 106, 50, 74, 190, 10, 85, + 146, 4, 73, 42, 79, 67, 89, 50, 158, 13, 85, 174, 1, 73, 42, 79, 3, 89, + 56, 242, 12, 85, 58, 73, 158, 1, 65, 2, 69, 3, 79, 70, 218, 9, 85, 250, + 3, 69, 26, 73, 42, 65, 2, 79, 67, 89, 106, 70, 71, 218, 8, 85, 158, 3, + 73, 158, 1, 65, 2, 79, 2, 89, 107, 69, 44, 186, 11, 85, 150, 1, 69, 66, + 65, 2, 79, 105, 2, 73, 69, 248, 2, 102, 66, 54, 68, 94, 71, 90, 74, 46, + 82, 50, 89, 78, 90, 134, 7, 85, 58, 73, 94, 65, 2, 69, 67, 79, 54, 202, + 10, 73, 158, 1, 65, 2, 79, 66, 85, 3, 89, 46, 46, 73, 198, 10, 69, 66, + 65, 2, 79, 67, 85, 13, 234, 11, 69, 230, 88, 80, 2, 84, 3, 88, 34, 60, 2, + 85, 79, 218, 9, 69, 0, 2, 73, 69, 66, 65, 3, 79, 7, 226, 99, 84, 3, 88, + 50, 230, 1, 85, 242, 7, 73, 42, 79, 67, 89, 46, 146, 9, 79, 66, 65, 2, + 69, 66, 85, 3, 89, 38, 30, 85, 222, 8, 73, 43, 79, 15, 194, 8, 79, 142, + 90, 80, 2, 84, 3, 88, 56, 62, 85, 206, 4, 79, 178, 2, 73, 158, 1, 65, 66, + 89, 43, 69, 15, 254, 8, 79, 2, 82, 230, 88, 80, 3, 88, 60, 150, 6, 85, + 58, 73, 158, 1, 65, 2, 79, 67, 89, 56, 254, 2, 85, 146, 4, 73, 42, 79, + 67, 89, 100, 58, 82, 254, 4, 85, 150, 1, 69, 66, 65, 2, 79, 67, 89, 48, + 46, 85, 162, 6, 69, 2, 79, 66, 89, 43, 65, 17, 134, 7, 79, 2, 82, 230, + 88, 80, 2, 84, 3, 88, 176, 1, 70, 83, 158, 3, 72, 50, 85, 58, 73, 94, 69, + 66, 65, 2, 79, 67, 89, 56, 130, 4, 73, 94, 69, 66, 65, 2, 79, 2, 85, 67, + 89, 56, 46, 85, 158, 3, 73, 94, 69, 66, 65, 3, 79, 21, 182, 4, 79, 106, + 82, 230, 88, 80, 2, 84, 3, 88, 60, 54, 69, 166, 3, 73, 42, 65, 2, 79, 66, + 85, 3, 89, 4, 150, 93, 80, 3, 88, 28, 38, 85, 206, 2, 69, 2, 79, 67, 65, + 9, 203, 2, 79, 40, 210, 2, 73, 42, 79, 66, 89, 41, 2, 85, 79, 178, 1, 66, + 72, 50, 85, 58, 73, 42, 90, 54, 69, 66, 65, 2, 79, 67, 89, 54, 46, 85, + 214, 1, 65, 2, 69, 2, 79, 67, 89, 19, 146, 1, 79, 170, 1, 82, 230, 88, + 80, 2, 84, 3, 88, 15, 90, 69, 142, 90, 80, 2, 84, 3, 88, 58, 50, 69, 2, + 79, 26, 73, 42, 65, 34, 85, 35, 89, 7, 138, 90, 80, 3, 88, 17, 38, 69, + 206, 89, 80, 2, 84, 3, 88, 9, 202, 89, 80, 2, 84, 3, 88, 11, 70, 82, 230, + 88, 80, 3, 88, 13, 38, 82, 230, 88, 80, 2, 84, 3, 88, 5, 227, 88, 88, 2, + 219, 7, 65, 2, 207, 57, 89, 180, 4, 252, 1, 10, 32, 78, 79, 84, 65, 84, + 73, 79, 78, 32, 224, 6, 16, 65, 78, 65, 66, 65, 90, 65, 82, 32, 83, 81, + 85, 65, 82, 69, 32, 210, 15, 69, 180, 2, 6, 73, 80, 80, 69, 82, 45, 96, + 8, 78, 65, 77, 69, 78, 78, 89, 32, 188, 33, 3, 79, 77, 66, 235, 26, 87, + 26, 144, 1, 9, 66, 65, 71, 32, 77, 69, 77, 66, 69, 42, 82, 104, 6, 68, + 79, 77, 65, 73, 78, 60, 3, 76, 69, 70, 154, 1, 83, 145, 2, 3, 84, 89, 80, + 2, 11, 82, 2, 11, 83, 2, 215, 83, 72, 8, 100, 4, 65, 78, 71, 69, 60, 3, + 73, 71, 72, 221, 1, 11, 69, 76, 65, 84, 73, 79, 78, 65, 76, 32, 67, 2, + 173, 3, 11, 32, 65, 78, 84, 73, 82, 69, 83, 84, 82, 73, 4, 17, 2, 84, 32, + 4, 60, 4, 73, 77, 65, 71, 13, 7, 66, 73, 78, 68, 73, 78, 71, 2, 11, 69, + 2, 25, 4, 32, 66, 82, 65, 2, 11, 67, 2, 175, 29, 75, 8, 44, 6, 67, 72, + 69, 77, 65, 32, 211, 1, 80, 6, 18, 67, 71, 80, 2, 17, 2, 79, 77, 2, 11, + 80, 2, 11, 79, 2, 11, 83, 2, 107, 73, 4, 30, 73, 41, 3, 82, 79, 74, 2, + 11, 80, 2, 11, 73, 2, 163, 80, 78, 2, 11, 69, 2, 11, 67, 2, 11, 84, 2, + 87, 73, 2, 139, 52, 79, 2, 11, 69, 2, 11, 32, 2, 11, 67, 2, 11, 79, 2, + 11, 76, 2, 11, 79, 2, 223, 79, 78, 144, 1, 240, 1, 2, 67, 76, 68, 7, 73, + 78, 73, 84, 73, 65, 76, 204, 2, 14, 70, 73, 78, 65, 76, 32, 67, 79, 78, + 83, 79, 78, 65, 78, 16, 7, 76, 69, 84, 84, 69, 82, 32, 148, 3, 5, 77, 65, + 82, 75, 32, 206, 1, 83, 181, 3, 6, 86, 79, 87, 69, 76, 32, 14, 64, 5, 79, + 83, 73, 78, 71, 137, 1, 6, 85, 83, 84, 69, 82, 45, 4, 11, 32, 4, 64, 12, + 68, 79, 85, 66, 76, 69, 45, 76, 73, 78, 69, 68, 23, 72, 2, 17, 2, 32, 72, + 2, 17, 2, 69, 65, 2, 215, 10, 68, 10, 96, 13, 70, 73, 78, 65, 76, 32, 76, + 69, 84, 84, 69, 82, 32, 41, 7, 73, 78, 73, 84, 73, 65, 76, 8, 238, 72, + 76, 2, 82, 2, 86, 3, 89, 2, 37, 7, 32, 76, 69, 84, 84, 69, 82, 2, 151, 6, + 32, 2, 131, 9, 84, 82, 166, 1, 68, 50, 75, 38, 78, 46, 83, 38, 84, 46, + 66, 2, 67, 2, 71, 2, 80, 2, 90, 138, 69, 45, 2, 72, 2, 74, 2, 76, 2, 77, + 2, 82, 2, 86, 2, 89, 187, 2, 65, 12, 206, 1, 68, 2, 90, 138, 69, 72, 187, + 2, 65, 6, 154, 70, 83, 14, 72, 187, 2, 65, 8, 130, 70, 71, 2, 78, 2, 89, + 187, 2, 65, 6, 214, 69, 72, 2, 83, 187, 2, 65, 12, 42, 83, 2, 84, 138, + 69, 72, 187, 2, 65, 4, 134, 69, 72, 187, 2, 65, 8, 50, 68, 58, 83, 40, 4, + 76, 79, 78, 71, 23, 84, 2, 29, 5, 79, 85, 66, 76, 69, 2, 11, 32, 2, 11, + 83, 2, 11, 72, 2, 11, 65, 2, 159, 70, 68, 2, 17, 2, 32, 84, 2, 17, 2, 83, + 72, 2, 147, 69, 69, 14, 36, 4, 73, 71, 78, 32, 255, 2, 85, 12, 54, 65, + 72, 6, 67, 65, 78, 68, 82, 65, 167, 1, 86, 2, 11, 78, 2, 11, 85, 2, 17, + 2, 83, 86, 2, 11, 65, 2, 135, 66, 82, 6, 36, 5, 66, 73, 78, 68, 85, 15, + 32, 5, 11, 32, 2, 25, 4, 87, 73, 84, 72, 2, 17, 2, 32, 79, 2, 21, 3, 82, + 78, 65, 2, 11, 77, 2, 11, 69, 2, 239, 38, 78, 4, 11, 73, 4, 18, 82, 19, + 83, 2, 219, 33, 65, 2, 11, 65, 2, 11, 82, 2, 139, 64, 71, 2, 151, 4, 66, + 20, 38, 76, 109, 5, 83, 73, 71, 78, 32, 2, 17, 2, 69, 78, 2, 11, 71, 2, + 11, 84, 2, 11, 72, 2, 11, 32, 2, 11, 77, 2, 11, 65, 2, 135, 62, 82, 18, + 86, 65, 26, 79, 2, 85, 16, 8, 82, 69, 86, 69, 82, 83, 69, 68, 146, 64, + 69, 3, 73, 4, 182, 64, 73, 3, 85, 5, 159, 64, 69, 2, 183, 44, 32, 12, 72, + 2, 66, 82, 16, 9, 82, 79, 32, 87, 73, 68, 84, 72, 32, 203, 1, 85, 2, 147, + 2, 65, 8, 32, 2, 78, 79, 86, 83, 31, 74, 4, 26, 45, 73, 2, 78, 45, 2, 29, + 5, 66, 82, 69, 65, 75, 2, 11, 32, 2, 11, 83, 2, 163, 1, 80, 2, 11, 74, 2, + 11, 79, 2, 11, 73, 2, 11, 78, 2, 143, 5, 69, 2, 219, 61, 83, 2, 21, 3, + 77, 79, 85, 2, 17, 2, 84, 72, 2, 11, 32, 2, 11, 70, 2, 11, 65, 2, 167, + 38, 67, 242, 2, 168, 1, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 224, + 18, 6, 78, 69, 85, 77, 69, 32, 181, 38, 17, 80, 82, 73, 90, 78, 65, 75, + 32, 77, 79, 68, 73, 70, 73, 69, 82, 32, 128, 1, 144, 2, 17, 76, 79, 87, + 69, 82, 32, 84, 79, 78, 65, 76, 32, 82, 65, 78, 71, 69, 88, 5, 77, 65, + 82, 75, 32, 148, 3, 18, 65, 84, 84, 65, 67, 72, 73, 78, 71, 32, 86, 69, + 82, 84, 73, 67, 65, 76, 233, 11, 17, 84, 79, 78, 65, 76, 32, 82, 65, 78, + 71, 69, 32, 77, 65, 82, 75, 32, 2, 33, 6, 32, 73, 78, 68, 73, 67, 2, 11, + 65, 2, 11, 84, 2, 11, 79, 2, 219, 56, 82, 118, 182, 2, 67, 142, 1, 68, + 104, 8, 71, 79, 82, 65, 90, 68, 79, 32, 34, 78, 106, 75, 114, 79, 72, 2, + 80, 79, 152, 2, 8, 77, 65, 76, 79, 32, 80, 79, 86, 100, 2, 82, 65, 50, + 83, 186, 1, 84, 108, 7, 86, 89, 83, 79, 75, 79, 32, 234, 1, 90, 224, 10, + 2, 76, 79, 224, 22, 4, 85, 68, 65, 82, 129, 6, 4, 66, 79, 82, 90, 6, 60, + 6, 72, 65, 83, 72, 75, 65, 29, 5, 85, 82, 86, 69, 68, 5, 197, 47, 3, 32, + 80, 79, 2, 17, 2, 32, 79, 2, 11, 77, 2, 235, 24, 69, 4, 240, 10, 13, 69, + 77, 69, 83, 84, 86, 69, 78, 78, 89, 32, 90, 65, 217, 14, 6, 86, 79, 69, + 84, 79, 67, 10, 30, 78, 89, 3, 86, 89, 83, 8, 29, 5, 73, 90, 75, 79, 32, + 8, 164, 8, 8, 83, 32, 75, 82, 89, 90, 72, 69, 35, 79, 2, 167, 20, 79, 8, + 56, 4, 82, 89, 90, 72, 150, 6, 65, 233, 39, 2, 85, 80, 5, 21, 3, 32, 79, + 78, 2, 11, 32, 2, 203, 7, 76, 6, 220, 5, 3, 84, 83, 69, 200, 13, 5, 66, + 76, 65, 67, 72, 131, 31, 78, 18, 22, 68, 131, 2, 86, 6, 60, 7, 67, 72, + 65, 83, 72, 73, 69, 201, 22, 3, 86, 69, 82, 5, 17, 2, 32, 87, 2, 21, 3, + 73, 84, 72, 2, 17, 2, 32, 86, 2, 29, 5, 69, 82, 84, 73, 67, 2, 17, 2, 65, + 76, 2, 11, 32, 2, 11, 83, 2, 11, 84, 2, 11, 82, 2, 11, 79, 2, 131, 25, + 75, 12, 29, 5, 89, 83, 72, 69, 32, 12, 22, 83, 251, 3, 79, 8, 154, 3, 32, + 229, 2, 4, 84, 82, 65, 78, 4, 28, 2, 90, 83, 183, 5, 86, 2, 219, 37, 69, + 10, 136, 1, 2, 75, 79, 16, 16, 84, 82, 65, 78, 78, 79, 32, 77, 65, 76, + 79, 32, 80, 79, 86, 89, 236, 1, 5, 82, 69, 68, 78, 69, 231, 26, 79, 2, + 239, 42, 66, 2, 11, 83, 2, 183, 22, 72, 10, 50, 79, 16, 5, 83, 65, 84, + 65, 32, 139, 36, 73, 2, 187, 28, 67, 6, 158, 1, 79, 201, 24, 3, 83, 32, + 75, 10, 24, 2, 83, 32, 95, 79, 6, 11, 75, 6, 44, 6, 72, 79, 75, 72, 76, + 79, 247, 24, 82, 4, 11, 77, 4, 17, 2, 32, 79, 4, 11, 78, 4, 11, 32, 4, + 18, 76, 31, 82, 2, 11, 69, 2, 179, 14, 70, 2, 11, 73, 2, 11, 71, 2, 139, + 14, 72, 6, 18, 65, 31, 69, 2, 249, 25, 3, 68, 69, 82, 4, 22, 86, 239, 10, + 76, 2, 199, 38, 79, 6, 60, 5, 77, 82, 65, 67, 72, 18, 83, 1, 4, 84, 82, + 69, 83, 2, 155, 10, 78, 2, 17, 2, 86, 69, 2, 155, 6, 84, 232, 1, 128, 2, + 2, 67, 72, 38, 68, 218, 1, 70, 28, 10, 71, 79, 76, 85, 66, 67, 72, 73, + 75, 32, 158, 1, 75, 160, 2, 6, 77, 69, 67, 72, 73, 75, 184, 1, 2, 78, 69, + 18, 79, 138, 2, 80, 146, 2, 82, 114, 83, 180, 20, 9, 86, 82, 65, 75, 72, + 73, 89, 65, 32, 151, 2, 90, 4, 246, 11, 69, 209, 10, 2, 65, 83, 10, 74, + 69, 98, 85, 16, 9, 86, 65, 32, 86, 32, 67, 72, 69, 76, 187, 27, 79, 4, + 76, 2, 82, 66, 213, 2, 12, 77, 69, 83, 84, 86, 69, 78, 78, 89, 32, 75, + 76, 2, 195, 34, 73, 2, 203, 34, 68, 2, 11, 78, 2, 231, 36, 85, 2, 11, 73, + 2, 147, 34, 84, 10, 78, 84, 38, 83, 240, 3, 5, 77, 82, 65, 67, 72, 145, + 14, 4, 66, 79, 82, 90, 4, 32, 3, 82, 69, 83, 251, 1, 73, 2, 21, 3, 86, + 69, 84, 2, 231, 17, 76, 14, 76, 4, 72, 65, 77, 73, 18, 76, 40, 3, 79, 66, + 89, 16, 2, 82, 89, 87, 85, 2, 219, 3, 76, 2, 11, 89, 2, 11, 85, 2, 151, + 33, 67, 2, 223, 31, 76, 6, 28, 2, 85, 75, 219, 32, 90, 5, 21, 3, 32, 84, + 73, 2, 11, 75, 2, 251, 15, 72, 2, 11, 70, 2, 11, 73, 2, 11, 83, 2, 215, + 30, 77, 11, 11, 32, 8, 44, 7, 75, 76, 89, 85, 67, 72, 69, 83, 80, 6, 64, + 9, 78, 69, 80, 79, 83, 84, 79, 89, 65, 14, 80, 163, 14, 86, 2, 39, 78, 2, + 25, 4, 79, 86, 79, 68, 2, 143, 14, 78, 2, 223, 22, 77, 14, 40, 2, 66, 76, + 41, 4, 83, 79, 75, 65, 2, 11, 65, 2, 11, 75, 2, 243, 30, 79, 13, 11, 32, + 10, 30, 75, 142, 26, 84, 39, 83, 6, 104, 11, 76, 89, 85, 67, 72, 69, 86, + 65, 89, 65, 32, 185, 25, 10, 82, 89, 85, 75, 79, 86, 65, 89, 65, 32, 4, + 202, 16, 78, 251, 8, 83, 14, 58, 65, 88, 8, 69, 82, 69, 86, 79, 68, 75, + 65, 27, 79, 6, 44, 3, 82, 65, 75, 222, 19, 76, 211, 5, 85, 2, 11, 76, 2, + 11, 73, 2, 171, 28, 84, 5, 153, 15, 2, 32, 78, 4, 68, 5, 68, 67, 72, 65, + 83, 245, 9, 7, 76, 75, 85, 76, 73, 90, 77, 2, 11, 72, 2, 219, 4, 73, 4, + 64, 11, 69, 86, 69, 82, 83, 69, 68, 32, 67, 72, 69, 139, 26, 79, 2, 25, + 4, 76, 89, 85, 83, 2, 215, 17, 84, 126, 96, 9, 75, 65, 77, 69, 89, 84, + 83, 65, 32, 148, 2, 8, 76, 79, 90, 72, 73, 84, 73, 69, 119, 84, 22, 128, + 1, 13, 68, 86, 79, 69, 67, 72, 69, 76, 78, 65, 89, 65, 32, 44, 7, 75, 76, + 89, 85, 67, 72, 69, 74, 84, 218, 18, 77, 119, 83, 8, 218, 10, 75, 110, + 78, 178, 8, 80, 75, 83, 6, 40, 5, 86, 65, 89, 65, 32, 243, 10, 78, 4, + 178, 15, 84, 183, 4, 83, 4, 162, 15, 73, 147, 4, 82, 9, 11, 32, 6, 48, 2, + 83, 32, 25, 6, 90, 65, 75, 82, 89, 84, 4, 166, 4, 75, 95, 90, 2, 11, 79, + 2, 211, 22, 69, 96, 92, 4, 65, 84, 89, 65, 172, 4, 6, 79, 80, 73, 84, 83, + 65, 189, 1, 5, 82, 69, 76, 65, 32, 23, 11, 32, 20, 76, 2, 83, 32, 236, 2, + 9, 90, 65, 75, 82, 89, 84, 65, 89, 65, 159, 5, 78, 14, 160, 1, 14, 68, + 86, 85, 77, 89, 65, 32, 90, 65, 80, 89, 65, 84, 89, 28, 7, 75, 82, 89, + 90, 72, 69, 77, 24, 2, 82, 79, 17, 8, 90, 65, 80, 89, 65, 84, 79, 89, 2, + 11, 77, 2, 215, 19, 73, 5, 189, 1, 2, 32, 73, 2, 203, 2, 71, 7, 21, 3, + 32, 73, 32, 4, 54, 75, 37, 9, 80, 79, 68, 67, 72, 65, 83, 72, 73, 2, 11, + 82, 2, 21, 3, 89, 90, 72, 2, 211, 1, 69, 5, 17, 2, 32, 83, 2, 17, 2, 32, + 90, 2, 29, 5, 65, 80, 89, 65, 84, 2, 11, 79, 2, 199, 17, 89, 7, 11, 32, + 4, 68, 6, 83, 32, 79, 67, 72, 75, 29, 7, 87, 73, 84, 72, 32, 83, 79, 2, + 11, 79, 2, 215, 16, 77, 2, 45, 9, 82, 79, 67, 72, 89, 65, 32, 78, 79, 2, + 11, 90, 2, 163, 7, 72, 68, 140, 1, 9, 68, 86, 79, 69, 67, 72, 69, 76, 78, + 162, 1, 75, 78, 78, 174, 1, 71, 152, 3, 8, 77, 82, 65, 67, 72, 78, 79, + 84, 42, 80, 75, 84, 10, 18, 79, 71, 65, 6, 64, 7, 80, 79, 86, 79, 68, 78, + 65, 189, 5, 4, 75, 82, 89, 90, 4, 17, 2, 89, 65, 5, 17, 2, 32, 75, 2, + 145, 5, 4, 76, 89, 85, 67, 26, 48, 6, 76, 89, 85, 67, 72, 69, 77, 2, 82, + 89, 4, 22, 78, 203, 6, 80, 2, 157, 8, 9, 69, 80, 79, 83, 84, 79, 89, 65, + 78, 22, 48, 7, 85, 75, 79, 86, 65, 89, 65, 195, 3, 90, 21, 11, 32, 18, + 54, 71, 236, 2, 5, 84, 82, 89, 65, 83, 179, 2, 80, 14, 21, 3, 82, 79, 77, + 14, 32, 4, 78, 65, 89, 65, 47, 79, 5, 237, 1, 7, 32, 87, 73, 84, 72, 32, + 83, 10, 92, 10, 75, 82, 89, 90, 72, 69, 86, 65, 89, 65, 17, 9, 80, 79, + 86, 79, 68, 78, 65, 89, 65, 5, 175, 2, 32, 7, 33, 6, 32, 87, 73, 84, 72, + 32, 4, 24, 2, 68, 79, 23, 83, 2, 41, 2, 85, 66, 2, 21, 3, 73, 78, 71, 2, + 133, 6, 6, 76, 69, 32, 90, 65, 80, 2, 175, 6, 75, 2, 237, 5, 3, 72, 69, + 86, 2, 11, 73, 2, 11, 75, 2, 187, 5, 72, 6, 22, 79, 187, 3, 82, 4, 28, 2, + 76, 85, 187, 1, 86, 2, 163, 1, 80, 8, 80, 5, 82, 89, 65, 83, 79, 153, 2, + 10, 73, 75, 72, 65, 89, 65, 32, 80, 85, 84, 6, 62, 80, 44, 4, 83, 84, 82, + 69, 173, 1, 4, 71, 76, 65, 83, 2, 17, 2, 79, 86, 2, 193, 1, 2, 79, 68, 2, + 171, 1, 76, 16, 88, 12, 75, 76, 89, 85, 67, 72, 69, 86, 65, 89, 65, 32, + 38, 77, 46, 80, 38, 84, 39, 83, 8, 34, 77, 46, 80, 38, 84, 39, 83, 2, 25, + 4, 82, 65, 67, 72, 2, 247, 1, 78, 2, 11, 82, 2, 205, 1, 2, 79, 83, 2, 11, + 82, 2, 11, 69, 2, 11, 83, 2, 157, 1, 4, 86, 69, 84, 76, 6, 30, 65, 121, + 3, 77, 69, 89, 4, 32, 4, 78, 79, 90, 72, 31, 80, 2, 11, 69, 2, 151, 3, + 75, 2, 17, 2, 89, 65, 2, 11, 84, 2, 11, 65, 2, 35, 89, 2, 11, 84, 2, 11, + 83, 2, 183, 2, 65, 10, 108, 11, 68, 73, 82, 69, 67, 84, 73, 79, 78, 32, + 70, 28, 3, 75, 82, 89, 28, 5, 76, 69, 86, 69, 76, 35, 82, 2, 11, 76, 2, + 159, 1, 73, 2, 11, 90, 2, 143, 1, 72, 4, 11, 45, 4, 114, 50, 3, 51, 2, + 11, 79, 2, 83, 71, 8, 26, 78, 34, 83, 15, 74, 4, 18, 66, 27, 74, 2, 11, + 83, 2, 11, 80, 3, 0, }; static const unsigned int dawg_pos_to_codepoint[] = { @@ -8959,7 +9250,7 @@ static const unsigned int dawg_pos_to_codepoint[] = { 71458, 71459, 71462, 71464, 71483, 71482, 71477, 71476, 71479, 71478, 71475, 71474, 71472, 71481, 71473, 71480, 9992, 128747, 128748, 128874, 128822, 128823, 128837, 128776, 128777, 128774, 128775, 128773, 128811, - 128859, 128829, 128826, 128855, 128769, 128875, 128876, 128830, 128834, + 128859, 128826, 128829, 128855, 128769, 128875, 128876, 128830, 128834, 128835, 128836, 128783, 128857, 128848, 128846, 128844, 128805, 128800, 128803, 128804, 128798, 128869, 128870, 128871, 128872, 128873, 128787, 128865, 128866, 128864, 128880, 128794, 128841, 128883, 128882, 128854, @@ -8971,525 +9262,558 @@ static const unsigned int dawg_pos_to_codepoint[] = { 128862, 128781, 128839, 128795, 128852, 128847, 128831, 128832, 128840, 128809, 128849, 128845, 128778, 128815, 128779, 128780, 128790, 128791, 128808, 128772, 128842, 128853, 128771, 128838, 128770, 9879, 8501, - 983054, 8780, 9006, 983203, 8776, 10863, 8778, 9095, 9941, 128126, 9200, - 38, 127994, 127944, 128657, 10815, 82970, 82971, 82964, 82965, 82966, - 82967, 82968, 82969, 82972, 82973, 82974, 82994, 82995, 82996, 82987, - 82988, 82992, 82993, 82986, 82989, 82990, 82991, 82997, 82998, 82999, - 83016, 83017, 83018, 83019, 83010, 83011, 83012, 83013, 83014, 83015, - 83020, 83021, 83022, 83050, 83051, 83052, 83053, 83043, 83044, 83045, - 83046, 83047, 83048, 83049, 83054, 82984, 82985, 82975, 82976, 82977, - 82978, 82979, 82980, 82981, 82982, 82983, 82953, 82954, 82955, 82956, - 82957, 82958, 82959, 82960, 82961, 82962, 82963, 82944, 82945, 82946, - 82947, 82948, 82949, 82950, 82951, 82952, 83000, 83001, 83002, 83003, - 83004, 83005, 83006, 83007, 83008, 83009, 83023, 83024, 83025, 83026, - 83027, 83028, 83029, 83030, 83031, 83032, 83033, 83034, 83035, 83036, - 83037, 83038, 83039, 83040, 83041, 83042, 83062, 83063, 83064, 83065, - 83070, 83071, 83072, 83073, 83066, 83067, 83068, 83055, 83056, 83057, - 83058, 83059, 83060, 83061, 83069, 83074, 83075, 83076, 83077, 83078, - 83083, 83084, 83079, 83080, 83081, 83082, 83085, 83086, 83087, 83088, - 83094, 83095, 83089, 83090, 83091, 83092, 83093, 83096, 83097, 83098, - 83099, 83105, 83106, 83100, 83101, 83102, 83103, 83104, 83107, 83108, - 83109, 83110, 83111, 83112, 83113, 83114, 83115, 83116, 83117, 83118, - 83119, 83120, 83121, 83122, 83123, 83124, 83125, 83126, 83127, 83128, - 83129, 83130, 83131, 83132, 83133, 83134, 83135, 83136, 83137, 83138, - 83139, 83140, 83141, 83142, 83143, 83144, 83145, 83146, 83147, 83148, - 83149, 83150, 83151, 83152, 83153, 83154, 83155, 83156, 83157, 83158, - 83159, 83160, 83161, 83162, 83163, 83164, 83165, 83166, 83167, 83168, - 83169, 83170, 83173, 83174, 83175, 83180, 83181, 83183, 83184, 83171, - 83172, 83176, 83177, 83178, 83179, 83182, 83190, 83191, 83192, 83193, - 83185, 83186, 83187, 83188, 83189, 83194, 83195, 83196, 83274, 83275, - 83280, 83281, 83270, 83271, 83272, 83273, 83276, 83277, 83278, 83279, - 83268, 83269, 83259, 83260, 83261, 83262, 83263, 83264, 83265, 83266, - 83267, 83204, 83205, 83197, 83198, 83199, 83200, 83201, 83202, 83203, - 83206, 83207, 83245, 83246, 83238, 83239, 83240, 83241, 83242, 83243, - 83244, 83247, 83248, 83208, 83209, 83210, 83211, 83212, 83213, 83214, - 83215, 83216, 83217, 83218, 83219, 83220, 83221, 83222, 83223, 83224, - 83225, 83226, 83227, 83228, 83229, 83230, 83231, 83232, 83233, 83234, - 83235, 83236, 83237, 83249, 83250, 83251, 83252, 83253, 83254, 83255, - 83256, 83257, 83258, 83291, 83292, 83282, 83283, 83284, 83285, 83286, - 83287, 83288, 83289, 83290, 83312, 83313, 83303, 83304, 83305, 83306, - 83307, 83308, 83309, 83310, 83311, 83348, 83349, 83339, 83340, 83341, - 83342, 83343, 83344, 83345, 83346, 83347, 83322, 83323, 83324, 83325, - 83316, 83317, 83318, 83314, 83315, 83319, 83320, 83321, 83326, 83327, - 83328, 83354, 83355, 83359, 83360, 83350, 83351, 83352, 83353, 83356, - 83357, 83358, 83361, 83377, 83378, 83374, 83375, 83381, 83382, 83373, - 83376, 83379, 83380, 83383, 83384, 83385, 83389, 83386, 83387, 83388, - 83390, 83391, 83392, 83393, 83394, 83395, 83363, 83364, 83362, 83365, - 83366, 83367, 83368, 83369, 83370, 83371, 83372, 83293, 83294, 83295, - 83296, 83297, 83298, 83299, 83300, 83301, 83302, 83329, 83330, 83331, - 83332, 83333, 83334, 83335, 83336, 83337, 83338, 83406, 83407, 83408, - 83409, 83410, 83411, 83412, 83413, 83414, 83415, 83416, 83447, 83448, - 83455, 83456, 83449, 83450, 83451, 83452, 83453, 83454, 83457, 83458, - 83489, 83490, 83491, 83492, 83493, 83494, 83495, 83496, 83396, 83397, - 83398, 83399, 83400, 83401, 83402, 83403, 83404, 83405, 83417, 83418, - 83419, 83420, 83421, 83422, 83423, 83424, 83425, 83426, 83427, 83428, - 83429, 83430, 83431, 83432, 83433, 83434, 83435, 83436, 83437, 83438, - 83439, 83440, 83441, 83442, 83443, 83444, 83445, 83446, 83459, 83460, - 83461, 83462, 83463, 83464, 83465, 83466, 83467, 83468, 83469, 83470, - 83471, 83472, 83473, 83474, 83475, 83476, 83477, 83478, 83479, 83480, - 83481, 83482, 83483, 83484, 83485, 83486, 83487, 83488, 83526, 83497, - 83498, 83499, 83500, 83501, 83502, 83503, 83504, 83505, 83506, 83507, - 83508, 83509, 83510, 83511, 83512, 83513, 83514, 83515, 83516, 83517, - 83518, 83519, 83520, 83521, 83522, 83523, 83524, 83525, 129728, 8736, - 10654, 10660, 128551, 8491, 128162, 128544, 128028, 11150, 11148, 11149, - 11151, 11119, 8630, 10560, 8755, 128260, 10226, 8634, 10769, 128246, - 9875, 10193, 9765, 9021, 9055, 9061, 9033, 9052, 9022, 9066, 9058, 9067, - 9042, 9049, 9035, 9034, 9038, 9062, 9073, 9046, 9050, 9075, 9080, 9014, - 9082, 9078, 9077, 9081, 9060, 9051, 9063, 9029, 9109, 9020, 9056, 9017, - 9018, 9036, 9047, 9044, 9037, 9043, 9040, 9027, 9031, 9072, 9071, 9016, - 9026, 9025, 9028, 9032, 9019, 9054, 9048, 9030, 9076, 9070, 9023, 9059, - 9069, 9015, 9079, 9024, 9074, 9057, 9041, 9045, 9053, 9039, 9065, 9064, - 9068, 39, 11236, 8773, 8786, 10864, 8774, 8784, 983195, 983196, 2277, - 2280, 2276, 2279, 2278, 2281, 1548, 1615, 65144, 65145, 2302, 1612, - 65138, 1549, 2206, 2299, 2300, 2274, 1643, 1771, 1770, 1757, 1565, 1614, - 1630, 2293, 2292, 65142, 65143, 1611, 65136, 1538, 1645, 1748, 2207, - 1621, 1620, 1616, 2294, 65146, 65147, 1613, 65140, 2258, 2255, 2254, - 2257, 2236, 2244, 2235, 2237, 1593, 1886, 2227, 1696, 1887, 1885, 65226, - 65227, 65225, 65228, 1575, 2165, 2176, 2173, 2161, 2174, 2171, 2167, - 2177, 2169, 2166, 2168, 2178, 2164, 2160, 2162, 2175, 2172, 1571, 65156, - 65155, 1573, 65160, 65159, 1651, 1650, 1908, 1907, 1570, 65154, 65153, - 2163, 2170, 1649, 64337, 64336, 1609, 65264, 65263, 65166, 65165, 1749, - 1576, 2230, 1878, 2208, 1874, 1875, 1872, 1876, 1877, 1873, 2209, 65168, - 65169, 65167, 65170, 1664, 64347, 64348, 64346, 64349, 1659, 64339, - 64340, 64338, 64341, 1583, 1674, 1675, 1882, 1774, 1679, 2222, 1881, - 1680, 1673, 65194, 65193, 1676, 64389, 64388, 1590, 1787, 65214, 65215, - 65213, 65216, 1677, 64387, 64386, 1672, 64393, 64392, 1646, 1697, 1647, - 1668, 64371, 64372, 64370, 64373, 1678, 64391, 64390, 1740, 1911, 1910, - 1909, 1599, 1598, 1597, 64509, 64510, 64508, 64511, 1601, 1699, 2212, - 1698, 1889, 1701, 1888, 65234, 65235, 65233, 65236, 1711, 1716, 1714, - 1712, 2224, 64403, 64404, 64402, 64405, 1594, 2243, 1788, 65230, 65231, - 65229, 65232, 1715, 64407, 64408, 64406, 64409, 2248, 1581, 1916, 2186, - 1903, 1906, 1902, 1880, 1669, 1666, 1879, 1665, 65186, 65187, 65185, - 65188, 1569, 65152, 1607, 1729, 1730, 64423, 64424, 64422, 64425, 1791, - 1728, 64421, 64420, 1726, 64427, 64428, 64426, 64429, 65258, 65259, - 65257, 65260, 1652, 1653, 1656, 1654, 1580, 2210, 2246, 2245, 65182, - 65183, 65181, 65184, 1688, 64395, 64394, 1603, 1919, 1710, 2228, 1708, - 1707, 65242, 65243, 65241, 65244, 1568, 1705, 1892, 1596, 1891, 2189, - 1595, 2242, 1890, 64399, 64400, 64398, 64401, 1733, 64481, 64480, 1737, - 64483, 64482, 1582, 65190, 65191, 65189, 65192, 1604, 2214, 1718, 2247, - 1717, 1720, 1719, 1898, 65246, 65247, 65245, 65248, 2221, 1605, 2215, - 1894, 1893, 65250, 65251, 65249, 65252, 1564, 1709, 1713, 64411, 64412, - 64410, 64413, 64468, 64469, 64467, 64470, 1667, 64375, 64376, 64374, - 64377, 1606, 1722, 64415, 64414, 1896, 1897, 1725, 1895, 2185, 1721, - 1724, 65254, 65255, 65253, 65256, 1662, 2231, 2238, 64343, 64344, 64342, - 64345, 1702, 64367, 64368, 64366, 64369, 1602, 2213, 2229, 1703, 1704, - 65238, 65239, 65237, 65240, 1585, 1905, 2233, 1682, 1685, 1883, 1899, - 1687, 1775, 1900, 1689, 1684, 1686, 2218, 1683, 65198, 65197, 1681, - 64397, 64396, 1723, 64417, 64418, 64416, 64419, 2220, 1589, 2223, 1694, - 1693, 65210, 65211, 65209, 65212, 1587, 1690, 1918, 1904, 1691, 1692, - 1901, 1884, 1917, 65202, 65203, 65201, 65204, 1588, 1786, 65206, 65207, - 65205, 65208, 1648, 2225, 1706, 1591, 2211, 2188, 1695, 2187, 65218, - 65219, 65217, 65220, 1670, 2241, 1727, 64379, 64380, 64378, 64381, 1671, - 64383, 64384, 64382, 64385, 1578, 1577, 65172, 65171, 1731, 65176, 2232, - 2239, 1661, 1660, 65174, 65175, 65173, 1663, 64355, 64356, 64354, 64357, - 1584, 65196, 65195, 1579, 65178, 65179, 65177, 65180, 2182, 1657, 2240, - 64359, 64360, 64358, 64361, 1658, 64351, 64352, 64350, 64353, 1735, 1655, - 64477, 64472, 64471, 64488, 64489, 1739, 1700, 64363, 64364, 64362, - 64365, 64479, 64478, 1608, 2219, 1743, 1913, 1912, 1572, 65158, 65157, - 1738, 1732, 65262, 65261, 1610, 1746, 1915, 1914, 1747, 64433, 64432, - 64431, 64430, 1574, 65162, 65163, 65161, 65164, 1742, 2216, 2234, 2217, - 1741, 1745, 65266, 65267, 65265, 65268, 1736, 64476, 64475, 1734, 64474, - 64473, 1592, 65222, 65223, 65221, 65224, 1586, 65200, 65199, 2226, 1744, - 64485, 64486, 64484, 64487, 2297, 2295, 64888, 64887, 64886, 64950, - 64699, 64554, 64964, 64885, 64698, 64553, 64787, 64759, 64788, 64760, - 64842, 64839, 64841, 64840, 64845, 65015, 64656, 64605, 64828, 64829, - 65010, 65011, 65023, 64962, 64669, 64518, 64672, 64738, 64926, 64670, - 64519, 64622, 64521, 64668, 64517, 64620, 64671, 64520, 64737, 64621, - 64619, 64618, 64623, 64522, 65021, 64878, 64939, 64693, 64547, 64880, - 64879, 64694, 64548, 64803, 64775, 64692, 64546, 64695, 64549, 64812, - 64784, 64804, 64776, 64893, 64892, 64704, 64559, 64636, 64561, 64702, - 64557, 64961, 64705, 64560, 64703, 64558, 64637, 64562, 64889, 64891, - 64890, 64701, 64556, 64789, 64761, 64700, 64555, 64790, 64762, 64859, - 64858, 64682, 64536, 64795, 64767, 64959, 64681, 64535, 64796, 64768, - 64915, 64916, 64728, 64594, 64595, 64596, 64727, 64593, 64729, 64934, - 64958, 64679, 64533, 64857, 64856, 64935, 64933, 64680, 64534, 64797, - 64769, 64798, 64770, 65019, 64643, 64573, 64640, 64567, 64708, 64568, - 64641, 64711, 64571, 64747, 64710, 64570, 64709, 64569, 64963, 64955, - 64951, 64642, 64712, 64572, 64748, 64644, 64574, 64538, 64799, 64771, - 64683, 64537, 64684, 64539, 64800, 64772, 65272, 65271, 65274, 65273, - 65270, 65269, 64646, 64579, 65276, 65275, 64898, 64949, 64896, 64897, - 64714, 64576, 64717, 64899, 64900, 64954, 64956, 64940, 64713, 64575, - 64902, 64901, 64715, 64577, 64904, 64903, 64941, 64645, 64716, 64578, - 64749, 64647, 64580, 64585, 64648, 64905, 64906, 64907, 64719, 64582, - 64910, 64911, 64953, 64720, 64583, 64909, 64914, 64908, 64960, 64718, - 64581, 64945, 64649, 64721, 64584, 64586, 65012, 64918, 64917, 64947, - 64723, 64588, 64726, 64751, 64952, 64957, 64921, 64920, 64919, 64967, - 64722, 64587, 64923, 64922, 64652, 64725, 64590, 64750, 64654, 64591, - 64653, 64651, 64724, 64589, 64650, 64655, 64592, 64895, 64948, 64894, - 64946, 64707, 64564, 64638, 64565, 64706, 64563, 64639, 64566, 65009, - 64843, 64833, 64835, 64836, 64837, 64834, 64832, 64847, 65014, 64604, - 64869, 64868, 64937, 64689, 64544, 64965, 64870, 64691, 64545, 64801, - 64773, 64690, 64811, 64783, 64802, 64774, 65013, 64975, 65008, 65017, - 65018, 64844, 64838, 64860, 64686, 64541, 64821, 64817, 64744, 64862, - 64861, 64685, 64540, 64820, 64936, 64966, 64687, 64542, 64822, 64864, - 64863, 64865, 64867, 64866, 64688, 64543, 64743, 64791, 64763, 64810, - 64782, 64792, 64764, 64606, 64609, 64755, 64607, 64610, 64756, 64611, - 64608, 64754, 64818, 64746, 64872, 64871, 64938, 64806, 64814, 64778, - 64824, 64873, 64805, 64813, 64777, 64823, 64875, 64874, 64877, 64876, - 64808, 64816, 64780, 64745, 64793, 64765, 64807, 64815, 64779, 64825, - 64809, 64781, 64794, 64766, 65022, 64846, 64882, 64881, 64883, 64884, - 64819, 64551, 64826, 64785, 64757, 64696, 64550, 64786, 64758, 64851, - 64850, 64849, 64674, 64524, 64677, 64740, 64930, 64852, 64929, 64675, - 64525, 64928, 64848, 64927, 64673, 64523, 64932, 64853, 64855, 64854, - 64931, 64626, 64676, 64526, 64739, 64628, 64527, 64627, 64625, 64624, - 64629, 64528, 64603, 64529, 64634, 64531, 64632, 64678, 64530, 64741, - 64633, 64631, 64630, 64635, 64532, 64742, 65016, 64661, 64601, 64616, - 64515, 64491, 64490, 64493, 64492, 64503, 64504, 64502, 64667, 64736, - 64664, 64513, 64663, 64512, 64665, 64614, 64666, 64514, 64735, 64615, - 64499, 64498, 64495, 64494, 64617, 64516, 64501, 64500, 64613, 64612, - 64497, 64496, 64942, 64731, 64598, 64734, 64753, 64660, 64658, 64943, - 64730, 64597, 64732, 64599, 64925, 64924, 64944, 64659, 64733, 64600, - 64752, 64657, 64662, 64602, 64506, 64507, 64505, 64697, 64552, 64827, - 2204, 1619, 1624, 2303, 126492, 126494, 126493, 126495, 126644, 126638, - 126641, 126647, 126648, 126646, 126632, 126645, 126630, 126650, 126626, - 126636, 126631, 126625, 126640, 126643, 126633, 126651, 126637, 126635, - 126649, 126627, 126642, 126639, 126629, 126489, 126467, 126518, 126517, - 126516, 126510, 126513, 126503, 126500, 126519, 126506, 126498, 126508, - 126497, 126512, 126505, 126523, 126509, 126507, 126514, 126511, 126521, - 126612, 126606, 126609, 126615, 126592, 126607, 126599, 126596, 126616, - 126614, 126600, 126613, 126598, 126618, 126594, 126604, 126593, 126608, - 126611, 126601, 126619, 126605, 126603, 126617, 126595, 126610, 126597, - 126475, 126704, 126705, 126588, 126590, 126585, 126582, 126568, 126581, - 126580, 126574, 126577, 126567, 126564, 126583, 126570, 126562, 126572, - 126586, 126561, 126576, 126569, 126587, 126573, 126578, 126575, 126484, - 126478, 126481, 126557, 126559, 126553, 126548, 126542, 126545, 126551, - 126530, 126535, 126537, 126555, 126541, 126539, 126546, 126543, 126472, - 126488, 126486, 126485, 126464, 126479, 126487, 126474, 126470, 126490, - 126466, 126476, 126471, 126465, 126480, 126483, 126473, 126491, 126477, - 126482, 126469, 1541, 1536, 2290, 2289, 2288, 1550, 2192, 1769, 2193, - 1642, 2184, 1544, 1629, 2296, 2301, 2298, 1772, 2183, 1623, 983633, - 983635, 983640, 983634, 983638, 983636, 983639, 983637, 983641, 1563, - 1617, 65148, 65149, 1554, 1555, 1552, 1537, 1539, 1540, 1790, 1789, 1553, - 1551, 1556, 2249, 1560, 1761, 2250, 2272, 1558, 983202, 1751, 1750, 1753, - 1752, 1762, 1764, 1768, 2264, 2273, 1756, 2261, 1755, 1557, 2200, 2260, - 2266, 2268, 2267, 2269, 2252, 2271, 2270, 2291, 1767, 2251, 1760, 1759, - 1559, 2253, 1754, 2263, 2262, 2265, 2202, 2201, 69375, 2203, 69373, - 69374, 2259, 1773, 1763, 1562, 1561, 1766, 1765, 1622, 1618, 65150, - 65151, 2256, 2205, 64444, 64435, 64434, 64441, 64440, 64439, 64438, - 64446, 64445, 64437, 64436, 64449, 64448, 64443, 64442, 64450, 64447, - 1758, 1600, 2179, 2180, 65137, 2181, 65139, 2285, 2282, 2286, 2283, 2287, - 2284, 1566, 2275, 1644, 1627, 1626, 1628, 2190, 1631, 1567, 1625, 1545, - 1546, 1542, 1543, 1637, 1636, 1639, 1638, 1635, 1634, 1632, 1641, 1633, - 1640, 1375, 1370, 1359, 1337, 1349, 1362, 1347, 1353, 1342, 1361, 1360, - 1356, 1333, 1335, 1336, 1346, 1331, 1355, 1345, 1364, 1343, 1363, 1354, - 1351, 1357, 1329, 1358, 1352, 1340, 1366, 1341, 1339, 1330, 1348, 1350, - 1338, 1334, 1344, 1332, 1365, 1373, 1371, 1372, 1395, 1401, 1390, 1409, - 1408, 1404, 1381, 1383, 1384, 1394, 1379, 1403, 1393, 1412, 1391, 1411, - 1402, 1399, 1405, 1376, 1407, 1385, 1377, 1406, 1400, 1397, 1416, 1410, - 1388, 1414, 1389, 1387, 1378, 1396, 1398, 1386, 1382, 1392, 1380, 1413, - 1415, 64279, 64277, 64275, 64276, 64278, 1369, 1418, 1423, 1417, 1374, - 129201, 10549, 10548, 129200, 10550, 10551, 129968, 128667, 127912, 9800, - 8978, 9738, 65948, 42, 8727, 8258, 128562, 11225, 9954, 8771, 8870, - 128095, 9883, 128663, 127975, 128762, 8371, 127814, 68352, 68353, 68357, - 68355, 68358, 68359, 68356, 68354, 68373, 68374, 68372, 68393, 68405, - 68388, 68387, 68386, 68391, 68390, 68389, 68371, 68370, 68369, 68403, - 68401, 68404, 68399, 68394, 68395, 68378, 68381, 68377, 68366, 68367, - 68362, 68363, 68364, 68365, 68385, 68384, 68380, 68379, 68402, 68400, - 68360, 68361, 68375, 68383, 68376, 68368, 68398, 68392, 68382, 68397, - 68396, 68409, 129361, 1547, 129518, 8525, 9810, 129683, 128118, 128036, - 127868, 128124, 128700, 128281, 128386, 11101, 11099, 983056, 10155, - 128043, 129363, 127992, 129441, 129366, 129391, 128708, 7005, 7007, 7006, - 6917, 6918, 6987, 6988, 6928, 6953, 6954, 6936, 6937, 6948, 6944, 6943, - 6949, 6984, 6927, 6933, 6934, 6919, 6920, 6929, 6930, 6921, 6922, 6938, - 6939, 6931, 6981, 6932, 6982, 6958, 6925, 6926, 6950, 6945, 6935, 6940, - 6951, 6952, 6957, 6923, 6924, 6962, 6960, 6961, 6946, 6942, 6941, 6947, - 6983, 6985, 6986, 6963, 6955, 6959, 6956, 7022, 7025, 7021, 7024, 7023, - 7026, 7020, 7019, 7027, 7012, 7013, 7018, 7015, 7017, 7016, 7010, 7014, - 7009, 7011, 7032, 7036, 7033, 7034, 7035, 7031, 7030, 7029, 7028, 7003, - 7038, 7008, 7002, 7037, 6915, 6913, 6912, 6916, 6914, 6964, 6972, 6973, - 6970, 6971, 6968, 6969, 6974, 6975, 6977, 6976, 6965, 6978, 6979, 6966, - 6967, 6980, 7004, 6997, 6996, 6999, 6998, 6995, 6994, 6992, 7001, 6993, - 7000, 127880, 10057, 9744, 128503, 128505, 128499, 128501, 11197, 9745, - 9746, 128502, 128500, 10007, 129526, 129648, 42737, 42736, 42741, 42740, - 42733, 42699, 42713, 42712, 42692, 42706, 42683, 42719, 42734, 42735, - 42682, 42729, 42657, 42725, 42659, 42670, 42718, 42666, 42716, 42701, - 42675, 42671, 42722, 42720, 42727, 42723, 42702, 42726, 42677, 42707, - 42708, 42709, 42686, 42694, 42674, 42731, 42695, 42685, 42684, 42691, - 42673, 42664, 42715, 42703, 92193, 92188, 92161, 92199, 92240, 92179, - 92211, 92219, 92213, 92243, 92207, 92216, 92183, 92184, 92238, 92171, - 92174, 92203, 92186, 92210, 92230, 92175, 92221, 92232, 92246, 92198, - 92214, 92190, 92176, 92197, 92196, 92164, 92245, 92212, 92201, 92160, - 92182, 92173, 92229, 92227, 92180, 92237, 92241, 92223, 92185, 92194, - 92178, 92239, 92225, 92233, 92167, 92191, 92231, 92244, 92204, 92226, - 92236, 92200, 92189, 92187, 92162, 92163, 92169, 92170, 92202, 92205, - 92222, 92168, 92165, 92215, 92224, 92208, 92220, 92235, 92177, 92181, - 92195, 92234, 92209, 92166, 92172, 92206, 92192, 92228, 92217, 92242, - 92218, 92262, 92271, 92272, 92270, 92294, 92253, 92301, 92256, 92273, - 92259, 92251, 92297, 92300, 92296, 92295, 92255, 92252, 92292, 92286, - 92268, 92290, 92281, 92267, 92284, 92287, 92282, 92283, 92277, 92298, - 92276, 92302, 92247, 92299, 92288, 92269, 92260, 92261, 92257, 92274, - 92289, 92263, 92279, 92265, 92266, 92250, 92249, 92285, 92248, 92264, - 92280, 92258, 92254, 92278, 92291, 92293, 92275, 92310, 92320, 92312, - 92394, 92393, 92319, 92384, 92321, 92386, 92361, 92373, 92366, 92328, - 92329, 92357, 92342, 92397, 92365, 92376, 92339, 92382, 92363, 92318, - 92387, 92383, 92315, 92385, 92354, 92311, 92381, 92343, 92326, 92364, - 92324, 92391, 92344, 92390, 92338, 92396, 92308, 92349, 92375, 92378, - 92317, 92331, 92362, 92336, 92341, 92370, 92347, 92307, 92303, 92309, - 92395, 92368, 92356, 92367, 92360, 92389, 92333, 92359, 92374, 92351, - 92325, 92371, 92350, 92345, 92372, 92314, 92340, 92323, 92304, 92313, - 92316, 92398, 92399, 92380, 92330, 92379, 92392, 92335, 92332, 92346, - 92400, 92358, 92334, 92353, 92337, 92306, 92369, 92388, 92322, 92305, - 92327, 92377, 92352, 92348, 92355, 92436, 92517, 92488, 92434, 92415, - 92431, 92454, 92460, 92420, 92489, 92422, 92474, 92479, 92449, 92502, - 92439, 92484, 92495, 92464, 92511, 92406, 92446, 92497, 92426, 92482, - 92448, 92515, 92401, 92478, 92427, 92496, 92458, 92424, 92445, 92404, - 92466, 92444, 92438, 92442, 92441, 92510, 92499, 92425, 92437, 92440, - 92471, 92465, 92409, 92483, 92475, 92467, 92485, 92457, 92432, 92476, - 92435, 92412, 92414, 92430, 92407, 92403, 92405, 92487, 92418, 92486, - 92408, 92447, 92459, 92480, 92472, 92505, 92514, 92450, 92463, 92410, - 92493, 92507, 92503, 92462, 92506, 92443, 92509, 92469, 92461, 92490, - 92512, 92494, 92455, 92417, 92501, 92413, 92508, 92500, 92504, 92473, - 92419, 92498, 92423, 92516, 92428, 92452, 92451, 92481, 92416, 92456, - 92433, 92477, 92492, 92491, 92513, 92411, 92402, 92421, 92453, 92468, - 92470, 92429, 92661, 92626, 92616, 92596, 92612, 92603, 92673, 92651, - 92627, 92597, 92585, 92578, 92615, 92565, 92552, 92602, 92674, 92570, - 92646, 92599, 92532, 92587, 92538, 92620, 92670, 92665, 92575, 92521, - 92633, 92595, 92523, 92556, 92539, 92664, 92653, 92667, 92600, 92542, - 92555, 92668, 92520, 92582, 92591, 92581, 92551, 92654, 92611, 92614, - 92666, 92671, 92617, 92637, 92562, 92518, 92592, 92558, 92528, 92607, - 92535, 92557, 92563, 92550, 92624, 92640, 92657, 92531, 92601, 92553, - 92543, 92619, 92598, 92541, 92544, 92579, 92658, 92554, 92628, 92648, - 92547, 92527, 92545, 92540, 92604, 92622, 92589, 92608, 92583, 92609, - 92662, 92613, 92584, 92625, 92634, 92524, 92536, 92605, 92647, 92548, - 92663, 92593, 92621, 92568, 92610, 92576, 92529, 92618, 92649, 92561, - 92656, 92526, 92655, 92546, 92534, 92560, 92580, 92659, 92660, 92638, - 92571, 92525, 92549, 92559, 92635, 92577, 92530, 92636, 92669, 92572, - 92672, 92537, 92519, 92630, 92586, 92569, 92566, 92573, 92652, 92522, - 92574, 92650, 92533, 92567, 92623, 92606, 92639, 92564, 92590, 92642, - 92643, 92594, 92641, 92645, 92644, 92588, 92629, 92632, 92631, 92710, - 92695, 92694, 92726, 92675, 92719, 92677, 92718, 92682, 92717, 92689, - 92720, 92724, 92685, 92722, 92723, 92711, 92712, 92698, 92688, 92697, - 92696, 92702, 92687, 92704, 92681, 92708, 92703, 92706, 92714, 92709, - 92679, 92721, 92684, 92683, 92707, 92691, 92713, 92700, 92693, 92727, - 92690, 92692, 92686, 92680, 92725, 92699, 92701, 92705, 92716, 92728, - 92678, 92715, 92676, 42693, 42698, 42711, 42696, 42667, 42717, 42704, - 42661, 42721, 42669, 42668, 42705, 42700, 42680, 42678, 42710, 42688, - 42681, 42732, 42676, 42679, 42730, 42728, 42672, 42662, 42724, 42687, - 42689, 42690, 42697, 42714, 42660, 42656, 42665, 42663, 42658, 42738, - 42742, 42739, 42743, 127974, 128183, 128180, 128182, 128181, 127820, - 129685, 128202, 129532, 128136, 129530, 127936, 92916, 92912, 92915, - 92913, 92914, 92887, 92894, 92908, 92880, 92907, 92893, 92886, 92888, - 92881, 92906, 92896, 92891, 92902, 92900, 92885, 92890, 92884, 92904, - 92905, 92899, 92889, 92897, 92892, 92895, 92882, 92898, 92883, 92901, - 92903, 92909, 92917, 9918, 129415, 7152, 7153, 7124, 7108, 7114, 7130, - 7139, 7127, 7138, 7133, 7136, 7113, 7111, 7117, 7119, 7107, 7135, 7125, - 7112, 7123, 7129, 7116, 7132, 7105, 7126, 7128, 7110, 7109, 7137, 7121, - 7118, 7106, 7120, 7134, 7122, 7115, 7131, 7104, 7140, 7141, 7154, 7155, - 7167, 7165, 7166, 7164, 7142, 7150, 7151, 7144, 7147, 7149, 7143, 7145, - 7146, 7148, 128704, 128705, 128267, 127900, 127901, 9835, 9836, 129492, - 128059, 127958, 128147, 129451, 129752, 127866, 129714, 983055, 128276, - 129745, 128277, 9086, 128718, 2519, 2557, 2432, 2548, 2552, 2551, 2550, - 2549, 2553, 2454, 2510, 983661, 2453, 2480, 2545, 2544, 2525, 2524, 2556, - 2443, 2528, 2444, 2529, 2527, 2479, 2437, 2438, 2448, 2452, 2466, 2465, - 2471, 2470, 2464, 2463, 2469, 2468, 2441, 2442, 2439, 2440, 2457, 2467, - 2462, 2472, 2486, 2487, 2488, 2477, 2476, 2459, 2458, 2456, 2455, 2461, - 2460, 2475, 2474, 2489, 2482, 2478, 2447, 2451, 2547, 2546, 983651, - 983650, 983652, 2558, 2493, 2434, 2433, 2492, 2509, 2435, 2554, 2494, - 2504, 2508, 2499, 2500, 2530, 2531, 2497, 2498, 2495, 2496, 2503, 2507, - 2539, 2538, 2541, 2540, 2537, 2536, 2534, 2543, 2535, 2542, 2555, 11102, - 127857, 9004, 9187, 8812, 8502, 8757, 129475, 128719, 72710, 72711, - 72712, 72746, 72704, 72705, 72715, 72717, 72731, 72730, 72736, 72735, - 72729, 72728, 72734, 72733, 72708, 72709, 72706, 72707, 72722, 72732, - 72727, 72737, 72747, 72748, 72749, 72741, 72740, 72724, 72723, 72721, - 72720, 72726, 72725, 72719, 72718, 72739, 72738, 72750, 72745, 72742, - 72744, 72743, 72714, 72716, 72801, 72810, 72806, 72797, 72807, 72798, - 72802, 72811, 72800, 72809, 72799, 72808, 72796, 72805, 72804, 72795, - 72803, 72794, 72768, 72765, 72764, 72767, 72766, 72756, 72757, 72758, - 72751, 72761, 72763, 72754, 72755, 72752, 72753, 72760, 72762, 72770, - 72769, 72789, 72788, 72791, 72790, 72787, 72786, 72784, 72793, 72785, - 72792, 72771, 72772, 72773, 72812, 128692, 128690, 10745, 10744, 129506, - 127921, 127874, 128038, 8383, 129766, 9763, 128089, 129452, 9679, 9864, - 10733, 9865, 9210, 11176, 11177, 11178, 11179, 11180, 11182, 11181, - 11183, 9960, 10028, 9821, 129554, 129596, 129609, 129612, 9818, 129551, - 129593, 9822, 129543, 129564, 129585, 129597, 129606, 129555, 129619, - 129617, 129618, 9823, 129556, 129598, 9819, 129552, 129594, 9820, 129553, - 129595, 129575, 129572, 129576, 129577, 129573, 129574, 9827, 9670, - 11201, 10070, 10730, 11230, 9830, 128419, 128899, 9196, 9662, 9660, - 11167, 127778, 9922, 9923, 10047, 9873, 10022, 128447, 128420, 9829, - 11042, 128426, 11052, 10711, 9194, 9198, 128896, 9668, 9666, 9664, 11164, - 8268, 9944, 128412, 9754, 9699, 9698, 10731, 11044, 11035, 9207, 11206, - 11045, 9204, 11207, 11047, 9205, 11208, 9206, 11205, 128921, 128927, - 9726, 9724, 9912, 128392, 9648, 11039, 127986, 9193, 9197, 9654, 9199, - 128898, 11091, 9658, 9656, 10145, 10148, 11166, 8269, 128413, 9755, - 127990, 9644, 11049, 11050, 11089, 9642, 9787, 9632, 11200, 9209, 128306, - 9728, 128369, 9927, 9984, 128900, 128909, 9986, 9751, 9824, 9733, 128919, - 128925, 128908, 9951, 128383, 9742, 9942, 128418, 128897, 9195, 9652, - 9650, 9700, 9701, 11165, 9851, 11054, 9646, 128920, 128926, 11037, 11204, - 10707, 10002, 10067, 8493, 8460, 8465, 8476, 8488, 10164, 10166, 10165, - 9250, 129792, 129794, 129798, 129806, 129821, 129836, 129813, 129844, - 129829, 129802, 129817, 129848, 129832, 129810, 129840, 129825, 129796, - 129804, 129819, 129850, 129834, 129842, 129827, 129800, 129815, 129846, - 129831, 129808, 129838, 129823, 129793, 129797, 129805, 129820, 129851, - 129835, 129812, 129843, 129828, 129801, 129816, 129847, 129809, 129839, - 129824, 129795, 129803, 129818, 129849, 129833, 129811, 129841, 129826, - 129799, 129814, 129845, 129830, 129807, 129837, 129822, 127804, 128033, - 128216, 128153, 129744, 128939, 128951, 128957, 128945, 128902, 128912, - 128932, 128366, 128278, 128209, 128218, 129667, 12731, 12727, 12726, - 12724, 12725, 12570, 12574, 12718, 12576, 12719, 12578, 12580, 12713, - 12735, 12720, 12584, 12715, 12572, 12579, 12581, 12709, 12708, 12573, - 12575, 12582, 12557, 12728, 12588, 12707, 12732, 12583, 12714, 12723, - 12589, 12716, 12712, 12585, 12555, 12587, 12717, 12591, 12571, 12722, - 12711, 12590, 12734, 12721, 12710, 12577, 12567, 12563, 12705, 12730, - 12558, 12733, 12568, 12564, 12556, 12729, 12569, 12565, 12549, 12704, - 12560, 12706, 12553, 12552, 12559, 12551, 12550, 12561, 12566, 12554, - 12586, 12562, 11211, 11867, 11868, 8993, 9141, 9142, 10555, 9183, 9181, - 8990, 8973, 11812, 8991, 8972, 11813, 9185, 127870, 128144, 127893, - 128335, 129379, 127923, 8904, 10705, 10706, 127993, 9574, 9559, 9556, - 9577, 9565, 9562, 9553, 9580, 9571, 9568, 9552, 9511, 9490, 9503, 9486, - 9537, 9520, 9513, 9489, 9505, 9485, 9543, 9519, 9558, 9555, 9573, 9557, - 9554, 9572, 9592, 9598, 9593, 9599, 9499, 9531, 9495, 9551, 9549, 9595, - 9523, 9491, 9487, 9483, 9481, 9479, 9477, 9475, 9547, 9515, 9507, 9473, - 9594, 9541, 9525, 9517, 9533, 9530, 9522, 9546, 9539, 9582, 9581, 9583, - 9584, 129959, 129964, 129954, 129958, 129965, 129955, 129952, 129956, - 129963, 129960, 129953, 129961, 129957, 129962, 9586, 9585, 9587, 129966, - 9550, 9548, 9591, 9516, 9488, 9484, 9588, 9596, 9482, 9480, 9478, 9476, - 9589, 9597, 9524, 9496, 9492, 9474, 9532, 9508, 9500, 9472, 129967, 9590, - 9542, 9526, 9518, 9534, 9529, 9521, 9545, 9540, 9536, 9510, 9498, 9502, - 9494, 9528, 9544, 9514, 9497, 9506, 9493, 9527, 9564, 9561, 9576, 9563, - 9560, 9575, 9570, 9567, 9579, 9538, 9512, 9504, 9535, 9509, 9501, 9569, - 9566, 9578, 129354, 983263, 128163, 128102, 128713, 128023, 129460, - 69649, 69685, 69749, 69745, 69746, 69687, 69686, 69637, 69638, 69648, - 69650, 69664, 69663, 69669, 69668, 69662, 69661, 69667, 69666, 69643, - 69644, 69645, 69646, 69679, 69641, 69642, 69639, 69640, 69684, 69678, - 69655, 69665, 69660, 69670, 69680, 69681, 69682, 69674, 69673, 69657, - 69656, 69654, 69653, 69659, 69658, 69652, 69651, 69672, 69671, 69683, - 69675, 69677, 69676, 69647, 69721, 69730, 69726, 69717, 69727, 69718, - 69722, 69731, 69720, 69729, 69719, 69728, 69716, 69725, 69724, 69715, - 69723, 69714, 69732, 69733, 69759, 69709, 69707, 69706, 69705, 69708, - 69744, 69632, 69635, 69634, 69636, 69633, 69700, 69747, 69748, 69689, - 69688, 69699, 69701, 69694, 69695, 69696, 69697, 69692, 69693, 69690, - 69691, 69698, 69702, 69704, 69703, 69739, 69738, 69741, 69740, 69737, - 69736, 69734, 69743, 69735, 69742, 10241, 10243, 10247, 10255, 10271, - 10303, 10367, 10495, 10431, 10335, 10463, 10399, 10287, 10351, 10479, - 10415, 10319, 10447, 10383, 10263, 10295, 10359, 10487, 10423, 10327, - 10455, 10391, 10279, 10343, 10471, 10407, 10311, 10439, 10375, 10251, - 10267, 10299, 10363, 10491, 10427, 10331, 10459, 10395, 10283, 10347, - 10475, 10411, 10315, 10443, 10379, 10259, 10291, 10355, 10483, 10419, - 10323, 10451, 10387, 10275, 10339, 10467, 10403, 10307, 10435, 10371, - 10245, 10253, 10269, 10301, 10365, 10493, 10429, 10333, 10461, 10397, - 10285, 10349, 10477, 10413, 10317, 10445, 10381, 10261, 10293, 10357, - 10485, 10421, 10325, 10453, 10389, 10277, 10341, 10469, 10405, 10309, - 10437, 10373, 10249, 10265, 10297, 10361, 10489, 10425, 10329, 10457, - 10393, 10281, 10345, 10473, 10409, 10313, 10441, 10377, 10257, 10289, - 10353, 10481, 10417, 10321, 10449, 10385, 10273, 10337, 10465, 10401, - 10305, 10433, 10369, 10242, 10246, 10254, 10270, 10302, 10366, 10494, - 10430, 10334, 10462, 10398, 10286, 10350, 10478, 10414, 10318, 10446, - 10382, 10262, 10294, 10358, 10486, 10422, 10326, 10454, 10390, 10278, - 10342, 10470, 10406, 10310, 10438, 10374, 10250, 10266, 10298, 10362, - 10490, 10426, 10330, 10458, 10394, 10282, 10346, 10474, 10410, 10314, - 10442, 10378, 10258, 10290, 10354, 10482, 10418, 10322, 10450, 10386, - 10274, 10338, 10466, 10402, 10306, 10434, 10370, 10244, 10252, 10268, - 10300, 10364, 10492, 10428, 10332, 10460, 10396, 10284, 10348, 10476, - 10412, 10316, 10444, 10380, 10260, 10292, 10356, 10484, 10420, 10324, - 10452, 10388, 10276, 10340, 10468, 10404, 10308, 10436, 10372, 10248, - 10264, 10296, 10360, 10488, 10424, 10328, 10456, 10392, 10280, 10344, - 10472, 10408, 10312, 10440, 10376, 10256, 10288, 10352, 10480, 10416, - 10320, 10448, 10384, 10272, 10336, 10464, 10400, 10304, 10432, 10368, - 10240, 129504, 983125, 129329, 127838, 728, 127753, 128112, 128188, - 129650, 129521, 9099, 166, 128148, 129382, 129294, 129529, 129483, - 129767, 128027, 6663, 6662, 6659, 6658, 6671, 6670, 6667, 6666, 6661, - 6668, 6665, 6657, 6678, 6669, 6656, 6674, 6660, 6673, 6676, 6664, 6675, - 6672, 6677, 6683, 6681, 6679, 6682, 6680, 6687, 6686, 5957, 5960, 5962, - 5959, 5956, 5969, 5955, 5966, 5963, 5961, 5965, 5968, 5958, 5967, 5964, - 5952, 5953, 5954, 5970, 5971, 8226, 8729, 128363, 128364, 9678, 128652, - 128101, 128100, 128655, 129480, 129419, 127959, 127791, 129699, 118939, - 118940, 118944, 118943, 118941, 118942, 118938, 118945, 118882, 118876, - 118824, 118835, 118797, 118866, 118957, 118801, 118802, 118865, 118818, - 118916, 118819, 118917, 118899, 118935, 119018, 119022, 119021, 119020, - 119023, 119019, 119017, 118986, 118984, 118985, 118843, 118887, 118808, - 118870, 119003, 119002, 119004, 119005, 118937, 118991, 118995, 118990, - 118992, 118994, 118993, 118930, 118933, 118931, 118932, 119014, 118918, - 118912, 119015, 118785, 118831, 118907, 118966, 118900, 118880, 118798, - 118888, 118884, 118869, 118960, 118959, 118958, 118836, 118969, 118977, - 118978, 118971, 118976, 118975, 118973, 118970, 118987, 118979, 118980, - 118972, 118983, 983272, 118982, 118974, 118988, 118981, 119000, 119001, - 118929, 118928, 118806, 119029, 118927, 118898, 118964, 118965, 118853, - 118968, 118837, 118967, 118936, 118956, 118810, 118854, 118847, 118839, - 118906, 118791, 118800, 119024, 119026, 118862, 118812, 119025, 119027, - 118863, 118811, 118820, 119028, 118911, 118842, 118850, 118921, 118858, - 118913, 118914, 118915, 118834, 118860, 118868, 118796, 118881, 118922, - 118926, 118925, 118924, 118923, 118830, 118877, 118949, 118947, 118948, - 118963, 118955, 118962, 118946, 118961, 118953, 118952, 118951, 118950, - 118954, 118871, 118805, 118855, 118828, 118901, 118787, 118788, 118856, - 118816, 118875, 118845, 118879, 118793, 118846, 118878, 118814, 118822, - 118873, 118859, 118857, 118849, 118840, 118861, 118786, 118895, 118841, - 118874, 118892, 118897, 118807, 118784, 118821, 118844, 118825, 118889, - 119010, 119012, 119013, 119011, 119006, 119008, 119009, 119007, 118910, - 118815, 118852, 119016, 118826, 118885, 118827, 118803, 118886, 118792, - 118813, 118920, 118799, 118833, 118829, 118904, 118902, 118903, 118905, - 118804, 118934, 118919, 118832, 118893, 118838, 118851, 118883, 118894, - 118891, 118896, 118823, 118789, 118790, 118872, 118817, 118809, 118908, - 118909, 118996, 118998, 118997, 118999, 118989, 118794, 118795, 118867, - 118864, 118848, 118890, 983262, 983126, 983057, 9764, 8454, 129305, - 128197, 128247, 128248, 127957, 983098, 5130, 5131, 5122, 6322, 5148, - 5551, 5310, 5382, 5166, 6321, 5759, 5559, 5556, 5557, 5558, 5567, 5564, - 5565, 5566, 5563, 5560, 5561, 5562, 5555, 5552, 5553, 5554, 5384, 5439, - 6387, 6388, 5281, 5264, 6382, 6389, 5203, 5674, 5675, 5677, 5676, 5673, - 5672, 5706, 5707, 5709, 5708, 5705, 5704, 5204, 5574, 5575, 5577, 5576, - 5573, 5572, 6384, 6381, 5620, 6383, 5617, 5618, 5619, 5616, 5615, 5195, - 5592, 5593, 5595, 5594, 5591, 5590, 5174, 5175, 5129, 5703, 5662, 5663, - 5665, 5664, 5661, 5660, 5655, 5656, 6386, 5659, 5657, 5654, 5652, 5633, - 5629, 5630, 5632, 5631, 5628, 5627, 5623, 5624, 5626, 5625, 5622, 5621, - 5636, 5637, 5639, 5329, 5638, 5635, 5634, 5722, 5718, 5719, 5721, 5720, - 5717, 5716, 5712, 5713, 5715, 5714, 5711, 5710, 5614, 5610, 5611, 5613, - 5612, 5609, 5608, 5702, 5698, 5699, 5701, 5700, 5697, 5696, 5686, 5687, - 5689, 5688, 5685, 5684, 5692, 5693, 5695, 5694, 5691, 5690, 5737, 5738, - 5740, 5739, 5736, 5735, 5604, 5605, 5607, 5606, 5603, 5602, 5598, 5599, - 5601, 5600, 5597, 5596, 5725, 5726, 5728, 5727, 5724, 5723, 5680, 5681, - 5683, 5682, 5679, 5678, 5668, 5669, 5671, 5670, 5667, 5666, 5731, 5732, - 5734, 5733, 5730, 5729, 5642, 5643, 5645, 5644, 5641, 5640, 5580, 5581, - 5583, 5582, 5579, 5578, 5586, 5587, 5589, 5588, 5585, 5584, 5648, 5649, - 5651, 5650, 5647, 5646, 5128, 5265, 5258, 5276, 5278, 5272, 5274, 5268, - 5270, 5266, 5741, 5261, 5262, 5259, 5260, 5257, 5121, 6364, 5163, 5469, - 5465, 5466, 5460, 5461, 5158, 5157, 5162, 5155, 5156, 6367, 6366, 5160, - 5152, 5151, 5153, 5154, 5161, 5159, 5462, 5742, 5463, 5464, 5467, 5459, - 5120, 5501, 5123, 5124, 5164, 5251, 5252, 5246, 5248, 6329, 5242, 5244, - 5238, 5240, 5236, 5234, 5235, 5228, 6328, 5231, 5232, 5229, 5230, 5227, - 5354, 5542, 5540, 5541, 5538, 5539, 5536, 5537, 5338, 5339, 5332, 6333, - 5350, 5352, 5346, 5348, 5342, 5344, 5340, 5335, 5336, 5333, 5334, 5331, - 5307, 5283, 5356, 5458, 5287, 5288, 5385, 5290, 5291, 5284, 6330, 5302, - 5304, 5298, 5300, 5294, 5296, 5292, 5285, 5286, 5309, 5328, 5473, 5475, - 5471, 5319, 5386, 5390, 5391, 5388, 5389, 5380, 5387, 5142, 5147, 5280, - 5250, 5306, 5327, 5221, 5437, 72372, 72373, 72370, 72371, 72368, 72369, - 72378, 72379, 72376, 72377, 72374, 72375, 5320, 5313, 6332, 5525, 5523, - 5524, 5518, 5744, 5521, 5522, 5519, 5520, 5526, 5749, 5750, 5747, 5748, - 5745, 5746, 5499, 5497, 5498, 5495, 5496, 5493, 5494, 5492, 5500, 5316, - 5317, 6331, 5323, 5325, 6346, 6348, 6342, 6344, 5321, 5314, 5315, 5312, - 5330, 5509, 5507, 5508, 5502, 5743, 5505, 5506, 5503, 5504, 5125, 6361, - 6347, 6349, 6343, 6345, 6362, 6363, 6359, 6358, 6360, 6356, 6357, 5165, - 5126, 6320, 5193, 5184, 5186, 6326, 5188, 5190, 5180, 5182, 5178, 5176, - 5177, 5168, 6325, 5171, 5172, 6324, 5169, 5170, 5167, 5456, 6368, 5443, - 6355, 5454, 6353, 6354, 6351, 6352, 6350, 5451, 5452, 5445, 6341, 5448, - 5449, 5446, 5447, 5442, 5381, 5364, 6335, 5570, 6380, 5571, 5568, 5569, - 5653, 6385, 5658, 5529, 6379, 6378, 5530, 5527, 5528, 5441, 5282, 5311, - 5365, 5358, 5413, 5405, 5407, 6338, 5409, 5411, 5401, 5403, 5399, 5395, - 5396, 6336, 5397, 5398, 6337, 5393, 5394, 5392, 5361, 5256, 5253, 5254, - 5255, 5362, 6334, 5383, 5376, 5378, 5372, 5374, 5368, 5370, 5366, 72383, - 72380, 72381, 72382, 5359, 5360, 5357, 5222, 5482, 5550, 5548, 5549, - 5546, 5547, 5544, 5545, 5543, 6372, 5480, 6371, 5478, 5479, 5476, 5477, - 5472, 5474, 5470, 5512, 6377, 6376, 5513, 5510, 5511, 5487, 5486, 6375, - 5485, 6374, 6373, 5483, 5484, 5226, 5223, 5224, 5225, 5205, 5206, 5197, - 6327, 5217, 5219, 5213, 5215, 5209, 5211, 5207, 5491, 5488, 5489, 5490, - 5200, 5201, 5198, 5199, 5196, 5132, 5355, 5351, 5353, 5347, 5349, 5343, - 5345, 5341, 5453, 6370, 5450, 6369, 5444, 5308, 5303, 5305, 5299, 5301, - 5295, 5297, 5293, 5194, 5189, 5191, 5185, 5187, 5181, 5183, 5179, 5440, - 5434, 5436, 5430, 5432, 5426, 5428, 5424, 5324, 5326, 5322, 5457, 5455, - 5517, 5514, 5515, 5516, 5410, 5412, 5406, 5408, 5402, 5404, 5400, 5377, - 5379, 5373, 5375, 5369, 5371, 5367, 5277, 5279, 5273, 5275, 5269, 5271, - 5267, 5247, 5249, 5243, 5245, 5239, 5241, 5237, 5481, 5218, 5220, 5214, - 5216, 5210, 5212, 5208, 5468, 5144, 5146, 5139, 5141, 5135, 5137, 5133, - 6365, 5138, 5140, 5535, 5756, 5757, 5754, 5755, 5752, 5753, 5751, 5534, - 5531, 5532, 5533, 5758, 5143, 5145, 6323, 5134, 5136, 5438, 5192, 5173, - 5263, 5233, 5337, 5289, 5318, 5363, 5202, 5420, 5127, 5149, 5421, 5422, - 5415, 6340, 5418, 5419, 6339, 5433, 5435, 5429, 5431, 5425, 5427, 5423, - 5416, 5417, 5414, 5150, 983097, 983171, 917631, 128473, 9803, 128367, - 127852, 129387, 128758, 11839, 9809, 128199, 128450, 128451, 8248, 8257, - 8453, 66225, 66246, 66211, 66214, 66254, 66218, 66250, 66251, 66252, - 66253, 66229, 66238, 66244, 66227, 66224, 66222, 66223, 66242, 66243, - 66232, 66221, 66247, 66230, 66226, 66239, 66212, 66248, 66256, 66235, - 66208, 66215, 66210, 66220, 66234, 66255, 66240, 66241, 66236, 66237, - 66231, 66209, 66213, 66249, 66233, 66245, 66217, 66219, 66216, 66228, - 127904, 711, 127887, 129690, 983073, 129365, 9936, 128008, 128049, + 983054, 128126, 117837, 117836, 117844, 117845, 117842, 117843, 117839, + 117838, 117841, 117840, 8780, 9006, 983203, 8776, 10863, 8778, 9095, + 9941, 9200, 38, 127994, 127944, 128657, 10815, 82970, 82971, 82964, + 82965, 82966, 82967, 82968, 82969, 82972, 82973, 82974, 82994, 82995, + 82996, 82987, 82988, 82992, 82993, 82986, 82989, 82990, 82991, 82997, + 82998, 82999, 83016, 83017, 83018, 83019, 83010, 83011, 83012, 83013, + 83014, 83015, 83020, 83021, 83022, 83050, 83051, 83052, 83053, 83043, + 83044, 83045, 83046, 83047, 83048, 83049, 83054, 82984, 82985, 82975, + 82976, 82977, 82978, 82979, 82980, 82981, 82982, 82983, 82953, 82954, + 82955, 82956, 82957, 82958, 82959, 82960, 82961, 82962, 82963, 82944, + 82945, 82946, 82947, 82948, 82949, 82950, 82951, 82952, 83000, 83001, + 83002, 83003, 83004, 83005, 83006, 83007, 83008, 83009, 83023, 83024, + 83025, 83026, 83027, 83028, 83029, 83030, 83031, 83032, 83033, 83034, + 83035, 83036, 83037, 83038, 83039, 83040, 83041, 83042, 83062, 83063, + 83064, 83065, 83070, 83071, 83072, 83073, 83066, 83067, 83068, 83055, + 83056, 83057, 83058, 83059, 83060, 83061, 83069, 83074, 83075, 83076, + 83077, 83078, 83083, 83084, 83079, 83080, 83081, 83082, 83085, 83086, + 83087, 83088, 83094, 83095, 83089, 83090, 83091, 83092, 83093, 83096, + 83097, 83098, 83099, 83105, 83106, 83100, 83101, 83102, 83103, 83104, + 83107, 83108, 83109, 83110, 83111, 83112, 83113, 83114, 83115, 83116, + 83117, 83118, 83119, 83120, 83121, 83122, 83123, 83124, 83125, 83126, + 83127, 83128, 83129, 83130, 83131, 83132, 83133, 83134, 83135, 83136, + 83137, 83138, 83139, 83140, 83141, 83142, 83143, 83144, 83145, 83146, + 83147, 83148, 83149, 83150, 83151, 83152, 83153, 83154, 83155, 83156, + 83157, 83158, 83159, 83160, 83161, 83162, 83163, 83164, 83165, 83166, + 83167, 83168, 83169, 83170, 83173, 83174, 83175, 83180, 83181, 83183, + 83184, 83171, 83172, 83176, 83177, 83178, 83179, 83182, 83190, 83191, + 83192, 83193, 83185, 83186, 83187, 83188, 83189, 83194, 83195, 83196, + 83274, 83275, 83280, 83281, 83270, 83271, 83272, 83273, 83276, 83277, + 83278, 83279, 83268, 83269, 83259, 83260, 83261, 83262, 83263, 83264, + 83265, 83266, 83267, 83204, 83205, 83197, 83198, 83199, 83200, 83201, + 83202, 83203, 83206, 83207, 83245, 83246, 83238, 83239, 83240, 83241, + 83242, 83243, 83244, 83247, 83248, 83208, 83209, 83210, 83211, 83212, + 83213, 83214, 83215, 83216, 83217, 83218, 83219, 83220, 83221, 83222, + 83223, 83224, 83225, 83226, 83227, 83228, 83229, 83230, 83231, 83232, + 83233, 83234, 83235, 83236, 83237, 83249, 83250, 83251, 83252, 83253, + 83254, 83255, 83256, 83257, 83258, 83291, 83292, 83282, 83283, 83284, + 83285, 83286, 83287, 83288, 83289, 83290, 83312, 83313, 83303, 83304, + 83305, 83306, 83307, 83308, 83309, 83310, 83311, 83348, 83349, 83339, + 83340, 83341, 83342, 83343, 83344, 83345, 83346, 83347, 83322, 83323, + 83324, 83325, 83316, 83317, 83318, 83314, 83315, 83319, 83320, 83321, + 83326, 83327, 83328, 83354, 83355, 83359, 83360, 83350, 83351, 83352, + 83353, 83356, 83357, 83358, 83361, 83377, 83378, 83374, 83375, 83381, + 83382, 83373, 83376, 83379, 83380, 83383, 83384, 83385, 83389, 83386, + 83387, 83388, 83390, 83391, 83392, 83393, 83394, 83395, 83363, 83364, + 83362, 83365, 83366, 83367, 83368, 83369, 83370, 83371, 83372, 83293, + 83294, 83295, 83296, 83297, 83298, 83299, 83300, 83301, 83302, 83329, + 83330, 83331, 83332, 83333, 83334, 83335, 83336, 83337, 83338, 83406, + 83407, 83408, 83409, 83410, 83411, 83412, 83413, 83414, 83415, 83416, + 83447, 83448, 83455, 83456, 83449, 83450, 83451, 83452, 83453, 83454, + 83457, 83458, 83489, 83490, 83491, 83492, 83493, 83494, 83495, 83496, + 83396, 83397, 83398, 83399, 83400, 83401, 83402, 83403, 83404, 83405, + 83417, 83418, 83419, 83420, 83421, 83422, 83423, 83424, 83425, 83426, + 83427, 83428, 83429, 83430, 83431, 83432, 83433, 83434, 83435, 83436, + 83437, 83438, 83439, 83440, 83441, 83442, 83443, 83444, 83445, 83446, + 83459, 83460, 83461, 83462, 83463, 83464, 83465, 83466, 83467, 83468, + 83469, 83470, 83471, 83472, 83473, 83474, 83475, 83476, 83477, 83478, + 83479, 83480, 83481, 83482, 83483, 83484, 83485, 83486, 83487, 83488, + 83526, 83497, 83498, 83499, 83500, 83501, 83502, 83503, 83504, 83505, + 83506, 83507, 83508, 83509, 83510, 83511, 83512, 83513, 83514, 83515, + 83516, 83517, 83518, 83519, 83520, 83521, 83522, 83523, 83524, 83525, + 129728, 8736, 10654, 10660, 128551, 8491, 128162, 128544, 128028, 117768, + 128246, 11150, 11148, 11149, 11151, 11119, 8630, 10560, 8755, 128260, + 10226, 8634, 10769, 9875, 10193, 9765, 9021, 9055, 9061, 9033, 9052, + 9022, 9066, 9058, 9067, 9042, 9049, 9035, 9034, 9038, 9062, 9073, 9046, + 9050, 9075, 9080, 9014, 9082, 9078, 9077, 9081, 9060, 9051, 9063, 9029, + 9109, 9020, 9056, 9017, 9018, 9036, 9047, 9044, 9037, 9043, 9040, 9027, + 9031, 9072, 9071, 9016, 9026, 9025, 9028, 9032, 9019, 9054, 9048, 9030, + 9076, 9070, 9023, 9059, 9069, 9015, 9079, 9024, 9074, 9057, 9041, 9045, + 9053, 9039, 9065, 9064, 9068, 39, 11236, 8773, 8786, 10864, 8774, 8784, + 983195, 983196, 69372, 1548, 2277, 2280, 2276, 2279, 2278, 2281, 1615, + 65144, 65145, 2302, 1612, 65138, 1549, 2206, 2299, 2300, 1643, 2274, + 1771, 1770, 1565, 1757, 1614, 1630, 2293, 2292, 65142, 65143, 1611, + 65136, 1538, 1645, 1748, 2207, 1621, 1620, 1616, 2294, 65146, 65147, + 1613, 65140, 2258, 2255, 2254, 2257, 2236, 2244, 2235, 2237, 1593, 1886, + 2227, 1696, 1887, 1885, 65226, 65227, 65225, 65228, 1575, 2165, 2176, + 2173, 2161, 2174, 2171, 2167, 2177, 2169, 2166, 2168, 2178, 2164, 2160, + 2162, 2175, 2172, 1571, 65156, 65155, 1573, 65160, 65159, 1651, 1650, + 1908, 1907, 1570, 65154, 65153, 2163, 2170, 1649, 64337, 64336, 1609, + 65264, 65263, 65166, 65165, 1749, 1576, 2230, 1878, 2208, 1874, 1875, + 1872, 1876, 1877, 1873, 2209, 65168, 65169, 65167, 65170, 1664, 64347, + 64348, 64346, 64349, 1659, 64339, 64340, 64338, 64341, 1583, 1882, 1774, + 1679, 2222, 69314, 1881, 1674, 1675, 1680, 1673, 65194, 65193, 1676, + 64389, 64388, 1590, 1787, 65214, 65215, 65213, 65216, 1677, 64387, 64386, + 1672, 64393, 64392, 1646, 1697, 1647, 1668, 64371, 64372, 64370, 64373, + 1678, 64391, 64390, 1740, 1911, 1910, 1909, 1599, 1598, 1597, 64509, + 64510, 64508, 64511, 1601, 1699, 2212, 1698, 1889, 1701, 1888, 65234, + 65235, 65233, 65236, 1711, 1716, 1714, 1712, 2224, 64403, 64404, 64402, + 64405, 1594, 2243, 1788, 65230, 65231, 65229, 65232, 1715, 64407, 64408, + 64406, 64409, 2248, 1581, 1916, 2186, 1903, 1906, 1902, 1880, 1669, 1666, + 1879, 1665, 65186, 65187, 65185, 65188, 1569, 65152, 1607, 1729, 1730, + 64423, 64424, 64422, 64425, 1791, 1728, 64421, 64420, 1726, 64427, 64428, + 64426, 64429, 65258, 65259, 65257, 65260, 1652, 1653, 1656, 1654, 1580, + 2210, 2246, 2245, 65182, 65183, 65181, 65184, 1688, 64395, 64394, 1603, + 69316, 1919, 1710, 2228, 1708, 1707, 65242, 65243, 65241, 65244, 1568, + 1705, 1892, 1596, 1891, 2189, 1595, 2242, 1890, 64399, 64400, 64398, + 64401, 1733, 64481, 64480, 1737, 64483, 64482, 1582, 65190, 65191, 65189, + 65192, 1604, 2214, 1718, 2247, 1717, 1720, 1719, 1898, 65246, 65247, + 65245, 65248, 2221, 1605, 2215, 1894, 1893, 65250, 65251, 65249, 65252, + 1564, 1709, 1713, 64411, 64412, 64410, 64413, 64468, 64469, 64467, 64470, + 1667, 64375, 64376, 64374, 64377, 1606, 1722, 64415, 64414, 1896, 1897, + 1725, 1895, 2185, 1721, 1724, 65254, 65255, 65253, 65256, 1662, 2231, + 2238, 64343, 64344, 64342, 64345, 1702, 64367, 64368, 64366, 64369, 1602, + 2213, 2229, 1703, 1704, 65238, 65239, 65237, 65240, 1585, 1905, 2233, + 1682, 1685, 1883, 1899, 1687, 1775, 1900, 1689, 1684, 1686, 2218, 1683, + 65198, 65197, 1681, 64397, 64396, 1723, 64417, 64418, 64416, 64419, 2220, + 1589, 2223, 1694, 1693, 65210, 65211, 65209, 65212, 1587, 1690, 1918, + 1904, 1691, 1692, 1901, 1884, 1917, 65202, 65203, 65201, 65204, 1588, + 1786, 65206, 65207, 65205, 65208, 1648, 2225, 1706, 1591, 69315, 2211, + 2188, 1695, 2187, 65218, 65219, 65217, 65220, 1670, 2241, 1727, 64379, + 64380, 64378, 64381, 1671, 64383, 64384, 64382, 64385, 1578, 1577, 65172, + 65171, 1731, 65176, 2232, 2239, 1661, 1660, 65174, 65175, 65173, 1663, + 64355, 64356, 64354, 64357, 1584, 65196, 65195, 1579, 65178, 65179, + 65177, 65180, 2182, 1657, 2240, 64359, 64360, 64358, 64361, 1658, 64351, + 64352, 64350, 64353, 1735, 1655, 64477, 64472, 64471, 64488, 64489, 1739, + 1700, 64363, 64364, 64362, 64365, 64479, 64478, 1608, 2219, 1743, 1913, + 1912, 1572, 65158, 65157, 1738, 1732, 65262, 65261, 1610, 1746, 1915, + 1914, 1747, 64433, 64432, 64431, 64430, 1574, 65162, 65163, 65161, 65164, + 1742, 2216, 2234, 2217, 1741, 1745, 65266, 65267, 65265, 65268, 1736, + 64476, 64475, 1734, 64474, 64473, 1592, 65222, 65223, 65221, 65224, 1586, + 65200, 65199, 2226, 1744, 64485, 64486, 64484, 64487, 2297, 2295, 64888, + 64887, 64886, 64950, 64699, 64554, 64964, 64885, 64698, 64553, 64787, + 64759, 64788, 64760, 64842, 64839, 64841, 64840, 64845, 65015, 64656, + 64605, 64828, 64829, 65010, 65011, 65023, 64962, 64669, 64518, 64672, + 64738, 64926, 64670, 64519, 64622, 64521, 64668, 64517, 64620, 64671, + 64520, 64737, 64621, 64619, 64618, 64623, 64522, 65021, 64878, 64939, + 64693, 64547, 64880, 64879, 64694, 64548, 64803, 64775, 64692, 64546, + 64695, 64549, 64812, 64784, 64804, 64776, 64893, 64892, 64704, 64559, + 64636, 64561, 64702, 64557, 64961, 64705, 64560, 64703, 64558, 64637, + 64562, 64889, 64891, 64890, 64701, 64556, 64789, 64761, 64700, 64555, + 64790, 64762, 64859, 64858, 64682, 64536, 64795, 64767, 64959, 64681, + 64535, 64796, 64768, 64915, 64916, 64728, 64594, 64595, 64596, 64727, + 64593, 64729, 64934, 64958, 64679, 64533, 64857, 64856, 64935, 64933, + 64680, 64534, 64797, 64769, 64798, 64770, 65019, 64643, 64573, 64640, + 64567, 64708, 64568, 64641, 64711, 64571, 64747, 64710, 64570, 64709, + 64569, 64963, 64955, 64951, 64642, 64712, 64572, 64748, 64644, 64574, + 64538, 64799, 64771, 64683, 64537, 64684, 64539, 64800, 64772, 65272, + 65271, 65274, 65273, 65270, 65269, 64646, 64579, 65276, 65275, 64898, + 64949, 64896, 64897, 64714, 64576, 64717, 64899, 64900, 64954, 64956, + 64940, 64713, 64575, 64902, 64901, 64715, 64577, 64904, 64903, 64941, + 64645, 64716, 64578, 64749, 64647, 64580, 64585, 64648, 64905, 64906, + 64907, 64719, 64582, 64910, 64911, 64953, 64720, 64583, 64909, 64914, + 64908, 64960, 64718, 64581, 64945, 64649, 64721, 64584, 64586, 65012, + 64918, 64917, 64947, 64723, 64588, 64726, 64751, 64952, 64957, 64921, + 64920, 64919, 64967, 64722, 64587, 64923, 64922, 64652, 64725, 64590, + 64750, 64654, 64591, 64653, 64651, 64724, 64589, 64650, 64655, 64592, + 64895, 64948, 64894, 64946, 64707, 64564, 64638, 64565, 64706, 64563, + 64639, 64566, 65009, 64843, 64833, 64835, 64836, 64837, 64834, 64832, + 64847, 65014, 64604, 64869, 64868, 64937, 64689, 64544, 64965, 64870, + 64691, 64545, 64801, 64773, 64690, 64811, 64783, 64802, 64774, 65013, + 64975, 65008, 65017, 65018, 64844, 64838, 64860, 64686, 64541, 64821, + 64817, 64744, 64862, 64861, 64685, 64540, 64820, 64936, 64966, 64687, + 64542, 64822, 64864, 64863, 64865, 64867, 64866, 64688, 64543, 64743, + 64791, 64763, 64810, 64782, 64792, 64764, 64606, 64609, 64755, 64607, + 64610, 64756, 64611, 64608, 64754, 64818, 64746, 64872, 64871, 64938, + 64806, 64814, 64778, 64824, 64873, 64805, 64813, 64777, 64823, 64875, + 64874, 64877, 64876, 64808, 64816, 64780, 64745, 64793, 64765, 64807, + 64815, 64779, 64825, 64809, 64781, 64794, 64766, 65022, 64846, 64882, + 64881, 64883, 64884, 64819, 64551, 64826, 64785, 64757, 64696, 64550, + 64786, 64758, 64851, 64850, 64849, 64674, 64524, 64677, 64740, 64930, + 64852, 64929, 64675, 64525, 64928, 64848, 64927, 64673, 64523, 64932, + 64853, 64855, 64854, 64931, 64626, 64676, 64526, 64739, 64628, 64527, + 64627, 64625, 64624, 64629, 64528, 64603, 64529, 64634, 64531, 64632, + 64678, 64530, 64741, 64633, 64631, 64630, 64635, 64532, 64742, 65016, + 64661, 64601, 64616, 64515, 64491, 64490, 64493, 64492, 64503, 64504, + 64502, 64667, 64736, 64664, 64513, 64663, 64512, 64665, 64614, 64666, + 64514, 64735, 64615, 64499, 64498, 64495, 64494, 64617, 64516, 64501, + 64500, 64613, 64612, 64497, 64496, 64942, 64731, 64598, 64734, 64753, + 64660, 64658, 64943, 64730, 64597, 64732, 64599, 64925, 64924, 64944, + 64659, 64733, 64600, 64752, 64657, 64662, 64602, 64506, 64507, 64505, + 64697, 64552, 64827, 2204, 1619, 1624, 2303, 126492, 126494, 126493, + 126495, 126644, 126638, 126641, 126647, 126648, 126646, 126632, 126645, + 126630, 126650, 126626, 126636, 126625, 126640, 126643, 126633, 126631, + 126651, 126637, 126635, 126649, 126627, 126642, 126639, 126629, 126489, + 126467, 126518, 126517, 126516, 126510, 126513, 126503, 126500, 126519, + 126506, 126498, 126508, 126497, 126512, 126505, 126523, 126509, 126507, + 126514, 126511, 126521, 126612, 126606, 126609, 126615, 126592, 126607, + 126599, 126596, 126616, 126614, 126600, 126613, 126598, 126618, 126594, + 126604, 126593, 126608, 126611, 126601, 126619, 126605, 126603, 126617, + 126595, 126610, 126597, 126475, 126704, 126705, 126588, 126590, 126585, + 126582, 126568, 126581, 126580, 126574, 126577, 126567, 126564, 126583, + 126570, 126562, 126572, 126561, 126576, 126569, 126586, 126587, 126573, + 126578, 126575, 126484, 126478, 126481, 126557, 126559, 126553, 126548, + 126542, 126545, 126551, 126530, 126537, 126535, 126555, 126541, 126539, + 126546, 126543, 126472, 126488, 126486, 126485, 126464, 126479, 126487, + 126474, 126470, 126490, 126466, 126476, 126465, 126480, 126483, 126473, + 126471, 126491, 126477, 126482, 126469, 1541, 1536, 2290, 2289, 2288, + 1642, 2199, 1550, 2192, 1769, 2193, 2184, 1544, 1629, 2296, 2301, 2298, + 1772, 2183, 1623, 983633, 983635, 983640, 983634, 983638, 983636, 983639, + 983637, 983641, 1563, 1617, 65148, 65149, 1554, 1555, 1552, 1537, 1539, + 1540, 1790, 1789, 1553, 1551, 1556, 2249, 1560, 1761, 2250, 2272, 1558, + 983202, 1751, 1750, 1753, 1752, 1762, 1764, 1768, 2264, 2273, 1756, 2261, + 1755, 1557, 2200, 2260, 2266, 2268, 2267, 2269, 2252, 2271, 2270, 2291, + 1767, 2251, 1760, 1759, 1559, 2253, 1754, 2263, 2262, 2265, 2202, 2201, + 69375, 2203, 69373, 69374, 2259, 1773, 1763, 1562, 1561, 1766, 1765, + 1622, 1618, 65150, 65151, 2256, 2205, 64444, 64435, 64434, 64441, 64440, + 64439, 64438, 64446, 64445, 64437, 64436, 64449, 64448, 64443, 64442, + 64450, 64447, 1758, 1600, 2179, 2180, 65137, 2181, 65139, 2285, 2282, + 2286, 2283, 2287, 2284, 1566, 2275, 1644, 1627, 1626, 1628, 2190, 1631, + 1567, 1625, 1545, 1546, 1542, 1543, 1637, 1636, 1639, 1638, 1635, 1634, + 1632, 1641, 1633, 1640, 1375, 1370, 1359, 1337, 1349, 1362, 1347, 1353, + 1342, 1361, 1360, 1356, 1333, 1335, 1336, 1346, 1331, 1355, 1345, 1364, + 1343, 1363, 1354, 1351, 1357, 1329, 1358, 1352, 1340, 1366, 1341, 1339, + 1330, 1348, 1350, 1338, 1334, 1344, 1332, 1365, 1373, 1372, 1371, 1395, + 1401, 1390, 1409, 1408, 1404, 1381, 1383, 1384, 1394, 1379, 1403, 1393, + 1412, 1391, 1411, 1402, 1399, 1405, 1376, 1407, 1385, 1377, 1406, 1400, + 1397, 1416, 1410, 1388, 1414, 1389, 1387, 1378, 1396, 1398, 1386, 1382, + 1392, 1380, 1413, 1415, 64279, 64277, 64275, 64276, 64278, 1369, 1418, + 1423, 1417, 1374, 129201, 10549, 10548, 129200, 10550, 10551, 129968, + 128667, 127912, 9800, 8978, 9738, 65948, 42, 8727, 8258, 128562, 11225, + 9954, 8771, 8870, 128095, 9883, 128663, 127975, 128762, 8371, 127814, + 68352, 68353, 68357, 68355, 68358, 68359, 68356, 68354, 68373, 68374, + 68372, 68393, 68405, 68388, 68387, 68386, 68391, 68390, 68389, 68371, + 68370, 68369, 68403, 68401, 68404, 68399, 68394, 68395, 68378, 68381, + 68377, 68366, 68367, 68362, 68363, 68364, 68365, 68385, 68384, 68380, + 68379, 68402, 68400, 68360, 68361, 68375, 68383, 68376, 68368, 68398, + 68392, 68382, 68397, 68396, 68409, 129361, 1547, 129518, 8525, 9810, + 129683, 128118, 128036, 127868, 128124, 128700, 128281, 128386, 11101, + 11099, 983056, 10155, 128043, 129363, 127992, 129441, 129366, 129391, + 128708, 7007, 7005, 7006, 6991, 6990, 6917, 6918, 6987, 6988, 6928, 6953, + 6954, 6936, 6937, 6948, 6944, 6943, 6949, 6984, 6927, 6933, 6934, 6919, + 6920, 6929, 6930, 6921, 6922, 6938, 6939, 6931, 6981, 6932, 6982, 6958, + 6925, 6926, 6950, 6945, 6935, 6940, 6951, 6952, 6957, 6923, 6924, 6962, + 6960, 6961, 6946, 6942, 6941, 6947, 6983, 6985, 6986, 6963, 6955, 6959, + 6956, 7022, 7025, 7021, 7024, 7023, 7026, 7020, 7019, 7027, 7012, 7013, + 7018, 7015, 7017, 7016, 7010, 7014, 7009, 7011, 7032, 7036, 7033, 7034, + 7035, 7031, 7030, 7029, 7028, 7003, 7038, 7008, 7002, 7037, 7039, 6915, + 6913, 6912, 6914, 6916, 6964, 6972, 6973, 6970, 6971, 6968, 6969, 6974, + 6975, 6977, 6976, 6965, 6978, 6979, 6966, 6967, 6980, 7004, 6997, 6996, + 6999, 6998, 6995, 6994, 6992, 7001, 6993, 7000, 127880, 10057, 9744, + 128503, 128505, 128499, 128501, 11197, 9745, 9746, 128502, 128500, 10007, + 129526, 129648, 42737, 42736, 42741, 42740, 42733, 42699, 42713, 42712, + 42692, 42706, 42683, 42719, 42734, 42735, 42682, 42729, 42657, 42725, + 42659, 42670, 42718, 42666, 42716, 42701, 42675, 42671, 42722, 42720, + 42727, 42723, 42702, 42726, 42677, 42707, 42708, 42709, 42686, 42694, + 42674, 42731, 42695, 42685, 42684, 42691, 42673, 42664, 42715, 42703, + 92193, 92188, 92161, 92199, 92240, 92179, 92211, 92219, 92213, 92243, + 92207, 92216, 92183, 92184, 92238, 92171, 983268, 92174, 92203, 92186, + 92210, 92230, 92175, 92221, 92232, 92246, 92198, 92214, 92190, 92176, + 92197, 92196, 92164, 92245, 92212, 92201, 92160, 92182, 92173, 92229, + 92227, 92180, 92237, 92241, 92223, 92185, 92194, 92178, 92239, 92225, + 92233, 92167, 92191, 92231, 92244, 92204, 92226, 92236, 92200, 92189, + 92187, 92162, 92163, 92169, 92170, 92202, 92205, 92222, 92168, 92165, + 92215, 92224, 92208, 92220, 92235, 92177, 92181, 92195, 92234, 92209, + 92166, 92172, 92206, 92192, 92228, 92217, 92242, 92218, 92262, 92271, + 92272, 92270, 92294, 92253, 92301, 92256, 92273, 92259, 92251, 92297, + 92300, 92296, 92295, 92255, 92252, 92292, 92286, 92268, 92290, 92281, + 92267, 92284, 92287, 92282, 92283, 92277, 92298, 92276, 92302, 92247, + 92299, 92288, 92269, 92260, 92261, 92257, 92274, 92289, 92263, 92279, + 92265, 92266, 92250, 92249, 92285, 92248, 92264, 92280, 92258, 92254, + 92278, 92291, 92293, 92275, 92310, 92320, 92312, 92394, 92393, 92319, + 92384, 92321, 92386, 92361, 92373, 92366, 92328, 92329, 92357, 92342, + 92397, 92365, 92376, 92339, 92382, 92363, 92318, 92387, 92383, 92315, + 92385, 92354, 92311, 92381, 92343, 92326, 92364, 92324, 92391, 92344, + 92390, 92338, 92396, 92308, 92349, 92375, 92378, 92317, 92331, 92362, + 92336, 92341, 92370, 92347, 92307, 92303, 92309, 92395, 92368, 92356, + 92367, 92360, 92389, 92333, 92359, 92374, 92351, 92325, 92371, 92350, + 92345, 92372, 92314, 92340, 92323, 92304, 92313, 92316, 92398, 92399, + 92380, 92330, 92379, 92392, 92335, 92332, 92346, 92400, 92358, 92334, + 92353, 92337, 92306, 92369, 92388, 92322, 92305, 92327, 92377, 92352, + 92348, 92355, 92436, 92517, 92488, 92434, 92415, 92431, 92454, 92460, + 92420, 92489, 92422, 92474, 92479, 92449, 92502, 92439, 92484, 92495, + 92464, 92511, 92406, 92446, 92497, 92426, 92482, 92448, 92515, 92401, + 92478, 92427, 92496, 92458, 92424, 92445, 92404, 92466, 92444, 92438, + 92442, 92441, 92510, 92499, 92425, 92437, 92440, 92471, 92465, 92409, + 92483, 92475, 92467, 92485, 92457, 92432, 92476, 92435, 92412, 92414, + 92430, 92407, 92403, 92405, 92487, 92418, 92486, 92408, 92447, 92459, + 92480, 92472, 92505, 92514, 92450, 92463, 92410, 92493, 92507, 92503, + 92462, 92506, 92443, 92509, 92469, 92461, 92490, 92512, 92494, 92455, + 92417, 92501, 92413, 92508, 92500, 92504, 92473, 92419, 92498, 92423, + 92516, 92428, 92452, 92451, 92481, 92416, 92456, 92433, 92477, 92492, + 92491, 92513, 92411, 92402, 92421, 92453, 92468, 92470, 92429, 92661, + 92626, 92616, 92596, 92612, 92603, 92673, 92651, 92627, 92597, 92585, + 92578, 92615, 92565, 92552, 92602, 92674, 92570, 92646, 92599, 92532, + 92587, 92538, 92620, 92670, 92665, 92575, 92521, 92633, 92595, 92523, + 92556, 92539, 92664, 92653, 92667, 92600, 92542, 92555, 92668, 92520, + 92582, 92591, 92581, 92551, 92654, 92611, 92614, 92666, 92671, 92617, + 92637, 92562, 92518, 92592, 92558, 92528, 92607, 92535, 92557, 92563, + 92550, 92624, 92640, 92657, 92531, 92601, 92553, 92543, 92619, 92598, + 92541, 92544, 92579, 92658, 92554, 92628, 92648, 92547, 92527, 92545, + 92540, 92604, 92622, 92589, 92608, 92583, 92609, 92662, 92613, 92584, + 92625, 92634, 92524, 92536, 92605, 92647, 92548, 92663, 92593, 92621, + 92568, 92610, 92576, 92529, 92618, 92649, 92561, 92656, 92526, 92655, + 92546, 92534, 92560, 92580, 92659, 92660, 92638, 92571, 92525, 92549, + 92559, 92635, 92577, 92530, 92636, 92669, 92572, 92672, 92537, 92519, + 92630, 92586, 92569, 92566, 92573, 92652, 92522, 92574, 92650, 92533, + 92567, 92623, 92606, 92639, 92564, 92590, 92642, 92643, 92594, 92641, + 92645, 92644, 92588, 92629, 92632, 92631, 92710, 92695, 92694, 92726, + 92675, 92719, 92677, 92718, 92682, 92717, 92689, 92720, 92724, 92685, + 92722, 92723, 92711, 92712, 92698, 92688, 92697, 92696, 92702, 92687, + 92704, 92681, 92708, 92703, 92706, 92714, 92709, 92679, 92721, 92684, + 92683, 92707, 92691, 92713, 92700, 92693, 92727, 92690, 92692, 92686, + 92680, 92725, 92699, 92701, 92705, 92716, 92728, 92678, 92715, 92676, + 42693, 42698, 42711, 42696, 42667, 42717, 42704, 42661, 42721, 42669, + 42668, 42705, 42700, 42680, 42678, 42710, 42688, 42681, 42732, 42676, + 42679, 42730, 42728, 42672, 42662, 42724, 42687, 42689, 42690, 42697, + 42714, 42660, 42656, 42665, 42663, 42658, 42738, 42742, 42739, 42743, + 127974, 128183, 128180, 128182, 128181, 127820, 129685, 128202, 129532, + 128136, 129530, 127936, 92916, 92912, 92915, 92913, 92914, 92887, 92894, + 92908, 92880, 92907, 92893, 92886, 92888, 92881, 92906, 92896, 92891, + 92902, 92900, 92885, 92890, 92884, 92904, 92905, 92899, 92889, 92897, + 92892, 92895, 92882, 92898, 92883, 92901, 92903, 92909, 92917, 9918, + 129415, 7152, 7153, 7124, 7108, 7114, 7130, 7139, 7127, 7138, 7133, 7136, + 7113, 7111, 7117, 7119, 7107, 7135, 7125, 7112, 7123, 7129, 7116, 7132, + 7105, 7126, 7128, 7110, 7109, 7137, 7121, 7118, 7106, 7120, 7134, 7122, + 7115, 7131, 7104, 7140, 7141, 7154, 7155, 7167, 7165, 7166, 7164, 7142, + 7150, 7151, 7144, 7147, 7149, 7143, 7145, 7146, 7148, 128704, 128705, + 128267, 127900, 127901, 9835, 9836, 129492, 128059, 127958, 128147, + 129451, 129752, 127866, 129714, 983055, 128276, 129745, 128277, 9086, + 128718, 2557, 2432, 2519, 2548, 2552, 2551, 2550, 2549, 2553, 2454, 2510, + 983661, 2453, 2480, 2545, 2544, 2525, 2524, 2556, 2443, 2528, 2444, 2529, + 2527, 2479, 2437, 2438, 2448, 2452, 2466, 2465, 2471, 2470, 2464, 2463, + 2469, 2468, 2441, 2442, 2439, 2440, 2457, 2467, 2462, 2472, 2486, 2487, + 2488, 2477, 2476, 2459, 2458, 2456, 2455, 2461, 2460, 2475, 2474, 2489, + 2482, 2478, 2447, 2451, 2547, 2546, 983651, 983650, 983652, 2558, 2433, + 2492, 2493, 2434, 2509, 2435, 2554, 2494, 2504, 2508, 2497, 2498, 2499, + 2500, 2530, 2531, 2495, 2496, 2503, 2507, 2539, 2538, 2541, 2540, 2537, + 2536, 2534, 2543, 2535, 2542, 2555, 11102, 127857, 9004, 9187, 8812, + 8502, 8757, 129475, 128719, 72710, 72711, 72712, 72746, 72704, 72705, + 72715, 72717, 72731, 72730, 72736, 72735, 72729, 72728, 72734, 72733, + 72708, 72709, 72706, 72707, 72722, 72732, 72727, 72737, 72747, 72748, + 72749, 72741, 72740, 72724, 72723, 72721, 72720, 72726, 72725, 72719, + 72718, 72739, 72738, 72750, 72745, 72742, 72744, 72743, 72714, 72716, + 72801, 72810, 72806, 72797, 72807, 72798, 72802, 72811, 72800, 72809, + 72799, 72808, 72796, 72805, 72804, 72795, 72803, 72794, 72764, 72768, + 72765, 72767, 72766, 72756, 72757, 72758, 72751, 72761, 72763, 72754, + 72755, 72752, 72753, 72760, 72762, 72770, 72769, 72789, 72788, 72791, + 72790, 72787, 72786, 72784, 72793, 72785, 72792, 72771, 72772, 72773, + 72812, 128692, 128690, 10745, 10744, 129506, 127921, 127874, 128038, + 8383, 129766, 9763, 128089, 129452, 9679, 9864, 10733, 9865, 9210, 11176, + 11177, 11178, 11179, 11180, 11182, 11181, 11183, 9960, 10028, 9821, + 129554, 129596, 129609, 129612, 9818, 129551, 129593, 9822, 129543, + 129564, 129585, 129597, 129606, 129555, 129619, 129617, 129618, 9823, + 129556, 129598, 9819, 129552, 129594, 9820, 129553, 129595, 129575, + 129572, 129576, 129577, 129573, 129574, 9827, 9670, 11201, 10070, 10730, + 11230, 9830, 128419, 128899, 9196, 9662, 9660, 11167, 127778, 9922, 9923, + 10047, 9873, 10022, 128447, 128420, 9829, 11042, 128426, 11052, 10711, + 11044, 117867, 117870, 117869, 117868, 11035, 9194, 9198, 128896, 9668, + 9666, 9664, 11164, 8268, 9944, 128412, 9754, 9699, 9698, 10731, 9207, + 11206, 11045, 9204, 11207, 11047, 9205, 11208, 9206, 11205, 128921, + 128927, 9726, 9724, 9912, 117871, 10002, 128392, 9648, 11039, 127986, + 118451, 128413, 9755, 9193, 9197, 9654, 9199, 128898, 11091, 9658, 9656, + 10145, 10148, 11166, 8269, 127990, 9644, 11049, 11050, 11089, 9642, + 129997, 9787, 9632, 11200, 9209, 128306, 9728, 128369, 9927, 9984, + 128900, 128909, 9986, 9751, 9824, 9733, 128919, 128925, 128908, 9951, + 128383, 9742, 9942, 128418, 128897, 9195, 9652, 9650, 9700, 9701, 11165, + 9851, 9646, 11054, 128920, 128926, 11037, 11204, 10707, 10067, 8493, + 8460, 8465, 8476, 8488, 10164, 10166, 10165, 9250, 118018, 118039, + 118128, 118187, 118218, 118159, 118054, 118112, 118234, 118084, 118202, + 118143, 118031, 118062, 118120, 118240, 118179, 118090, 118210, 118151, + 118047, 118104, 118226, 118165, 118076, 118194, 118135, 118021, 118035, + 118066, 118124, 118243, 118183, 118093, 118214, 118155, 118050, 118108, + 118230, 118168, 118080, 118198, 118139, 118028, 118058, 118116, 118237, + 118175, 118087, 118206, 118147, 118043, 118100, 118222, 118162, 118072, + 118190, 118131, 118029, 118060, 118118, 118177, 118208, 118149, 118045, + 118102, 118224, 118023, 118037, 118068, 118126, 118244, 118185, 118095, + 118216, 118157, 118052, 118110, 118232, 118170, 118082, 118200, 118141, + 118074, 118192, 118133, 118020, 118033, 118064, 118122, 118242, 118181, + 118092, 118212, 118153, 118048, 118106, 118228, 118167, 118078, 118196, + 118137, 118026, 118056, 118114, 118235, 118173, 118085, 118204, 118145, + 118041, 118098, 118220, 118160, 118070, 118188, 118129, 118034, 118065, + 118123, 118182, 118213, 118154, 118049, 118107, 118229, 118079, 118197, + 118138, 118017, 118024, 118038, 118069, 118127, 118245, 118186, 118096, + 118217, 118158, 118053, 118111, 118233, 118171, 118083, 118201, 118142, + 118030, 118061, 118119, 118239, 118178, 118089, 118209, 118150, 118046, + 118103, 118225, 118164, 118075, 118193, 118134, 118027, 118057, 118115, + 118236, 118174, 118086, 118205, 118146, 118042, 118099, 118221, 118161, + 118071, 118189, 118130, 118025, 118055, 118113, 118172, 118203, 118144, + 118040, 118097, 118219, 118016, 118022, 118036, 118067, 118125, 118184, + 118094, 118215, 118156, 118051, 118109, 118231, 118169, 118081, 118199, + 118140, 118059, 118117, 118238, 118176, 118088, 118207, 118148, 118044, + 118101, 118223, 118163, 118073, 118191, 118132, 118019, 118105, 118227, + 118166, 118032, 118063, 118121, 118241, 118180, 118091, 118211, 118152, + 118077, 118195, 118136, 129792, 129794, 129798, 129806, 129821, 129836, + 129813, 129844, 129829, 129802, 129817, 129848, 129832, 129810, 129840, + 129825, 129796, 129804, 129819, 129850, 129834, 129842, 129827, 129800, + 129815, 129846, 129831, 129808, 129838, 129823, 129793, 129801, 129816, + 129847, 129797, 129805, 129820, 129851, 129835, 129812, 129843, 129828, + 129809, 129839, 129824, 129795, 129803, 129818, 129849, 129833, 129811, + 129841, 129826, 129799, 129814, 129845, 129830, 129807, 129837, 129822, + 127804, 128033, 128216, 128153, 129744, 128939, 128951, 128957, 128945, + 128902, 128912, 128932, 128366, 128278, 128209, 128218, 129667, 12731, + 12727, 12726, 12724, 12725, 12570, 12574, 12718, 12576, 12719, 12578, + 12580, 12713, 12735, 12720, 12584, 12715, 12572, 12579, 12581, 12709, + 12708, 12573, 12575, 12582, 12557, 12728, 12588, 12707, 12732, 12583, + 12714, 12723, 12589, 12716, 12712, 12585, 12555, 12587, 12717, 12591, + 12571, 12722, 12711, 12590, 12734, 12721, 12710, 12577, 12567, 12563, + 12705, 12730, 12558, 12733, 12568, 12564, 12556, 12729, 12569, 12565, + 12549, 12704, 12560, 12706, 12553, 12552, 12559, 12551, 12550, 12561, + 12566, 12554, 12586, 12562, 118257, 118258, 118259, 118260, 117854, + 11867, 117853, 118253, 118255, 11868, 117855, 118249, 118251, 118247, + 11211, 8993, 130029, 8990, 8973, 11812, 8991, 8972, 11813, 130030, 9141, + 9142, 10555, 9183, 9181, 130026, 130018, 9185, 127870, 128144, 127893, + 128335, 129379, 127923, 8904, 10705, 10706, 127993, 118282, 117792, + 117791, 118281, 9574, 9559, 9556, 9577, 9565, 9562, 9553, 9580, 9571, + 9568, 9552, 9511, 9490, 9503, 9486, 9537, 9520, 9513, 9489, 9505, 9485, + 9543, 9519, 9558, 9555, 9573, 9557, 9554, 9572, 9592, 9598, 9593, 9599, + 9499, 9531, 9495, 9475, 9547, 9515, 9507, 9595, 9523, 9491, 9487, 9551, + 9549, 9483, 9481, 9479, 9477, 9473, 9594, 9541, 9525, 9517, 9533, 9530, + 9522, 9546, 9539, 9582, 9581, 9583, 9584, 130010, 130014, 129954, 129958, + 130003, 129959, 129964, 129965, 129955, 130000, 129952, 129956, 129963, + 129960, 129953, 129961, 129957, 129962, 130007, 130005, 130004, 130012, + 9586, 130008, 130011, 130002, 130015, 130006, 9585, 130009, 130001, + 130013, 9587, 129966, 9591, 9516, 9488, 9484, 9550, 9548, 9472, 117787, + 117788, 129967, 9588, 9596, 9482, 9480, 9478, 9476, 117789, 9589, 9597, + 9524, 9496, 9492, 9474, 118297, 118295, 118296, 118294, 9532, 9508, 9500, + 117790, 9590, 9542, 9526, 9518, 9534, 9529, 9521, 9545, 9540, 9536, 9510, + 9498, 9502, 9494, 9528, 9544, 9514, 9497, 9506, 9493, 9527, 9564, 9561, + 9576, 9563, 9560, 9575, 9570, 9567, 9579, 9538, 9512, 9504, 9535, 9509, + 9501, 9569, 9566, 9578, 129354, 983263, 128163, 128102, 128713, 128023, + 129460, 69649, 69685, 69749, 69745, 69746, 69687, 69686, 69637, 69638, + 69648, 69650, 69664, 69663, 69669, 69668, 69662, 69661, 69667, 69666, + 69643, 69644, 69645, 69646, 69679, 69641, 69642, 69639, 69640, 69684, + 69678, 69655, 69665, 69660, 69670, 69680, 69681, 69682, 69674, 69673, + 69657, 69656, 69654, 69653, 69659, 69658, 69652, 69651, 69672, 69671, + 69683, 69675, 69677, 69676, 69647, 69721, 69730, 69726, 69717, 69727, + 69718, 69722, 69731, 69720, 69729, 69719, 69728, 69716, 69725, 69724, + 69715, 69723, 69714, 69732, 69733, 69759, 69709, 69707, 69706, 69705, + 69708, 69632, 69744, 69635, 69634, 69636, 69633, 69700, 69747, 69748, + 69689, 69688, 69699, 69701, 69692, 69693, 69694, 69695, 69696, 69697, + 69690, 69691, 69698, 69702, 69704, 69703, 69739, 69738, 69741, 69740, + 69737, 69736, 69734, 69743, 69735, 69742, 10241, 10243, 10247, 10255, + 10271, 10303, 10367, 10495, 10431, 10335, 10463, 10399, 10287, 10351, + 10479, 10415, 10319, 10447, 10383, 10263, 10295, 10359, 10487, 10423, + 10327, 10455, 10391, 10279, 10343, 10471, 10407, 10311, 10439, 10375, + 10251, 10267, 10299, 10363, 10491, 10427, 10331, 10459, 10395, 10283, + 10347, 10475, 10411, 10315, 10443, 10379, 10259, 10291, 10355, 10483, + 10419, 10323, 10451, 10387, 10275, 10339, 10467, 10403, 10307, 10435, + 10371, 10245, 10253, 10269, 10301, 10365, 10493, 10429, 10333, 10461, + 10397, 10285, 10349, 10477, 10413, 10317, 10445, 10381, 10261, 10293, + 10357, 10485, 10421, 10325, 10453, 10389, 10277, 10341, 10469, 10405, + 10309, 10437, 10373, 10249, 10265, 10297, 10361, 10489, 10425, 10329, + 10457, 10393, 10281, 10345, 10473, 10409, 10313, 10441, 10377, 10257, + 10289, 10353, 10481, 10417, 10321, 10449, 10385, 10273, 10337, 10465, + 10401, 10305, 10433, 10369, 10242, 10246, 10254, 10270, 10302, 10366, + 10494, 10430, 10334, 10462, 10398, 10286, 10350, 10478, 10414, 10318, + 10446, 10382, 10262, 10294, 10358, 10486, 10422, 10326, 10454, 10390, + 10278, 10342, 10470, 10406, 10310, 10438, 10374, 10250, 10266, 10298, + 10362, 10490, 10426, 10330, 10458, 10394, 10282, 10346, 10474, 10410, + 10314, 10442, 10378, 10258, 10290, 10354, 10482, 10418, 10322, 10450, + 10386, 10274, 10338, 10466, 10402, 10306, 10434, 10370, 10244, 10252, + 10268, 10300, 10364, 10492, 10428, 10332, 10460, 10396, 10284, 10348, + 10476, 10412, 10316, 10444, 10380, 10260, 10292, 10356, 10484, 10420, + 10324, 10452, 10388, 10276, 10340, 10468, 10404, 10308, 10436, 10372, + 10248, 10264, 10296, 10360, 10488, 10424, 10328, 10456, 10392, 10280, + 10344, 10472, 10408, 10312, 10440, 10376, 10256, 10288, 10352, 10480, + 10416, 10320, 10448, 10384, 10272, 10336, 10464, 10400, 10304, 10432, + 10368, 10240, 129504, 983125, 129329, 127838, 728, 127753, 128112, + 128188, 129650, 129521, 9099, 166, 128148, 129382, 129294, 129529, + 129483, 129767, 128027, 6663, 6662, 6659, 6658, 6671, 6670, 6667, 6666, + 6661, 6668, 6665, 6657, 6678, 6669, 6656, 6674, 6660, 6673, 6676, 6664, + 6675, 6672, 6677, 6683, 6681, 6679, 6682, 6680, 6687, 6686, 5957, 5960, + 5962, 5959, 5956, 5969, 5955, 5966, 5963, 5961, 5965, 5968, 5958, 5967, + 5964, 5952, 5953, 5954, 5970, 5971, 8226, 8729, 128363, 128364, 9678, + 128652, 128101, 128100, 128655, 129480, 129419, 127959, 127791, 129699, + 118939, 118940, 118944, 118943, 118941, 118942, 118938, 118945, 118882, + 118876, 118824, 118835, 118797, 118866, 118957, 118801, 118802, 118865, + 118818, 118916, 118819, 118917, 118899, 118935, 119018, 119022, 119021, + 119020, 119023, 119019, 119017, 118986, 118984, 118985, 118843, 118887, + 118808, 118870, 119003, 119002, 119004, 119005, 118937, 118991, 118995, + 118990, 118992, 118994, 118993, 118930, 118933, 118931, 118932, 119014, + 118918, 118912, 119015, 118785, 118831, 118907, 118966, 118900, 118880, + 118798, 118888, 118884, 118869, 118960, 118959, 118958, 118836, 118969, + 118977, 118978, 118971, 118976, 118975, 118973, 118970, 118987, 118979, + 118980, 118972, 118983, 983274, 118982, 118974, 118988, 118981, 119000, + 119001, 118929, 118928, 118806, 119029, 118927, 118898, 118964, 118965, + 118853, 118968, 118837, 118967, 118936, 118956, 118810, 118854, 118847, + 118839, 118906, 118791, 118800, 119024, 119026, 118862, 118812, 119025, + 119027, 118863, 118811, 118820, 119028, 118911, 118842, 118850, 118921, + 118858, 118913, 118914, 118915, 118834, 118860, 118868, 118796, 118881, + 118922, 118926, 118925, 118924, 118923, 118830, 118877, 118949, 118947, + 118948, 118963, 118955, 118962, 118946, 118961, 118953, 118952, 118951, + 118950, 118954, 118871, 118805, 118855, 118828, 118901, 118787, 118788, + 118856, 118816, 118875, 118845, 118879, 118793, 118846, 118878, 118814, + 118822, 118873, 118859, 118857, 118849, 118840, 118861, 118786, 118895, + 118841, 118874, 118892, 118897, 118807, 118784, 118821, 118844, 118825, + 118889, 119010, 119012, 119013, 119011, 119006, 119008, 119009, 119007, + 118910, 118815, 118852, 119016, 118826, 118885, 118827, 118803, 118886, + 118792, 118813, 118920, 118799, 118833, 118829, 118904, 118902, 118903, + 118905, 118804, 118934, 118919, 118832, 118893, 118838, 118851, 118883, + 118894, 118891, 118896, 118823, 118789, 118790, 118872, 118817, 118809, + 118908, 118909, 118996, 118998, 118997, 118999, 118989, 118794, 118795, + 118867, 118864, 118848, 118890, 983262, 983126, 983057, 9764, 8454, + 129305, 128197, 128247, 128248, 127957, 983098, 5130, 5131, 5122, 6322, + 5148, 5551, 5310, 5382, 5166, 6321, 5759, 5559, 5556, 5557, 5558, 5567, + 5564, 5565, 5566, 5563, 5560, 5561, 5562, 5555, 5552, 5553, 5554, 5384, + 5439, 6387, 6388, 5281, 5264, 6382, 6389, 5203, 5674, 5675, 5677, 5676, + 5673, 5672, 5706, 5707, 5709, 5708, 5705, 5704, 5204, 5574, 5575, 5577, + 5576, 5573, 5572, 6384, 6381, 5620, 6383, 5617, 5618, 5619, 5616, 5615, + 5195, 5592, 5593, 5595, 5594, 5591, 5590, 5174, 5175, 5129, 5703, 5662, + 5663, 5665, 5664, 5661, 5660, 5655, 5656, 6386, 5659, 5657, 5654, 5652, + 5633, 5629, 5630, 5632, 5631, 5628, 5627, 5623, 5624, 5626, 5625, 5622, + 5621, 5636, 5637, 5639, 5329, 5638, 5635, 5634, 5722, 5718, 5719, 5721, + 5720, 5717, 5716, 5712, 5713, 5715, 5714, 5711, 5710, 5614, 5610, 5611, + 5613, 5612, 5609, 5608, 5702, 5698, 5699, 5701, 5700, 5697, 5696, 5686, + 5687, 5689, 5688, 5685, 5684, 5692, 5693, 5695, 5694, 5691, 5690, 5737, + 5738, 5740, 5739, 5736, 5735, 5604, 5605, 5607, 5606, 5603, 5602, 5598, + 5599, 5601, 5600, 5597, 5596, 5725, 5726, 5728, 5727, 5724, 5723, 5680, + 5681, 5683, 5682, 5679, 5678, 5668, 5669, 5671, 5670, 5667, 5666, 5731, + 5732, 5734, 5733, 5730, 5729, 5642, 5643, 5645, 5644, 5641, 5640, 5580, + 5581, 5583, 5582, 5579, 5578, 5586, 5587, 5589, 5588, 5585, 5584, 5648, + 5649, 5651, 5650, 5647, 5646, 5128, 5265, 5258, 5276, 5278, 5272, 5274, + 5268, 5270, 5266, 5741, 5261, 5262, 5259, 5260, 5257, 5121, 6364, 5163, + 5469, 5465, 5466, 5460, 5461, 5158, 5157, 5162, 5155, 5156, 6367, 6366, + 5160, 5152, 5151, 5153, 5154, 5161, 5159, 5462, 5742, 5463, 5464, 5467, + 5459, 5120, 5501, 5123, 5124, 5164, 5251, 5252, 5246, 5248, 6329, 5242, + 5244, 5238, 5240, 5236, 5234, 5235, 5228, 6328, 5231, 5232, 5229, 5230, + 5227, 5354, 5542, 5540, 5541, 5538, 5539, 5536, 5537, 5338, 5339, 5332, + 6333, 5350, 5352, 5346, 5348, 5342, 5344, 5340, 5335, 5336, 5333, 5334, + 5331, 5307, 5283, 5356, 5458, 5287, 5288, 5385, 5290, 5291, 5284, 6330, + 5302, 5304, 5298, 5300, 5294, 5296, 5292, 5285, 5286, 5309, 5328, 5473, + 5475, 5471, 5319, 5386, 5390, 5391, 5388, 5389, 5380, 5387, 5142, 5147, + 5280, 5250, 5306, 5327, 5221, 5437, 72372, 72373, 72370, 72371, 72368, + 72369, 72378, 72379, 72376, 72377, 72374, 72375, 5320, 5313, 6332, 5525, + 5523, 5524, 5518, 5744, 5521, 5522, 5519, 5520, 5526, 5749, 5750, 5747, + 5748, 5745, 5746, 5499, 5497, 5498, 5495, 5496, 5493, 5494, 5492, 5500, + 5316, 5317, 6331, 5323, 5325, 6346, 6348, 6342, 6344, 5321, 5314, 5315, + 5312, 5330, 5509, 5507, 5508, 5502, 5743, 5505, 5506, 5503, 5504, 5125, + 6361, 6347, 6349, 6343, 6345, 6362, 6363, 6359, 6358, 6360, 6356, 6357, + 5165, 5126, 6320, 5193, 5184, 5186, 6326, 5188, 5190, 5180, 5182, 5178, + 5176, 5177, 5168, 6325, 5171, 5172, 6324, 5169, 5170, 5167, 5456, 6368, + 5443, 6355, 5454, 6353, 6354, 6351, 6352, 6350, 5451, 5452, 5445, 6341, + 5448, 5449, 5446, 5447, 5442, 5381, 5364, 6335, 5570, 6380, 5571, 5568, + 5569, 5653, 6385, 5658, 5529, 6379, 6378, 5530, 5527, 5528, 5441, 5282, + 5311, 5365, 5358, 5413, 5405, 5407, 6338, 5409, 5411, 5401, 5403, 5399, + 5395, 5396, 6336, 5397, 5398, 6337, 5393, 5394, 5392, 5361, 5256, 5253, + 5254, 5255, 5362, 6334, 5383, 5376, 5378, 5372, 5374, 5368, 5370, 5366, + 72383, 72380, 72381, 72382, 5359, 5360, 5357, 5222, 5482, 5550, 5548, + 5549, 5546, 5547, 5544, 5545, 5543, 6372, 5480, 6371, 5478, 5479, 5476, + 5477, 5472, 5474, 5470, 5512, 6377, 6376, 5513, 5510, 5511, 5487, 5486, + 6375, 5485, 6374, 6373, 5483, 5484, 5226, 5223, 5224, 5225, 5205, 5206, + 5197, 6327, 5217, 5219, 5213, 5215, 5209, 5211, 5207, 5491, 5488, 5489, + 5490, 5200, 5201, 5198, 5199, 5196, 5132, 5355, 5351, 5353, 5347, 5349, + 5343, 5345, 5341, 5453, 6370, 5450, 6369, 5444, 5308, 5303, 5305, 5299, + 5301, 5295, 5297, 5293, 5194, 5189, 5191, 5185, 5187, 5181, 5183, 5179, + 5440, 5434, 5436, 5430, 5432, 5426, 5428, 5424, 5324, 5326, 5322, 5457, + 5455, 5517, 5514, 5515, 5516, 5410, 5412, 5406, 5408, 5402, 5404, 5400, + 5377, 5379, 5373, 5375, 5369, 5371, 5367, 5277, 5279, 5273, 5275, 5269, + 5271, 5267, 5247, 5249, 5243, 5245, 5239, 5241, 5237, 5481, 5218, 5220, + 5214, 5216, 5210, 5212, 5208, 5468, 5144, 5146, 5139, 5141, 5135, 5137, + 5133, 6365, 5138, 5140, 5535, 5756, 5757, 5754, 5755, 5752, 5753, 5751, + 5534, 5531, 5532, 5533, 5758, 5143, 5145, 6323, 5134, 5136, 5438, 5192, + 5173, 5263, 5233, 5337, 5289, 5318, 5363, 5202, 5420, 5127, 5149, 5421, + 5422, 5415, 6340, 5418, 5419, 6339, 5433, 5435, 5429, 5431, 5425, 5427, + 5423, 5416, 5417, 5414, 5150, 983097, 983171, 917631, 128473, 9803, + 128367, 127852, 129387, 128758, 11839, 9809, 128199, 128450, 128451, + 8248, 8257, 8453, 66225, 66246, 66211, 66214, 66254, 66218, 66250, 66251, + 66252, 66253, 66229, 66238, 66244, 66227, 66224, 66222, 66223, 66242, + 66243, 66232, 66221, 66247, 66230, 66226, 66239, 66212, 66248, 66256, + 66235, 66208, 66215, 66210, 66220, 66234, 66255, 66240, 66241, 66236, + 66237, 66231, 66209, 66213, 66249, 66233, 66245, 66217, 66219, 66216, + 66228, 127904, 711, 127887, 129690, 983073, 129365, 9936, 128008, 128049, 128569, 128572, 66888, 66864, 66912, 66882, 66873, 66902, 66889, 66890, 66911, 66891, 66895, 66901, 66881, 66867, 66870, 66868, 66904, 66866, 66876, 66910, 66879, 66883, 66897, 66915, 66884, 66878, 66885, 66914, @@ -9540,12 +9864,12 @@ static const unsigned int dawg_pos_to_codepoint[] = { 127247, 8857, 8861, 12905, 12919, 12904, 12918, 12903, 12917, 12926, 12909, 12923, 12906, 12920, 12896, 12910, 12900, 12914, 12897, 12911, 12908, 12922, 12901, 12915, 12899, 12913, 12902, 12916, 12907, 12921, - 12898, 12912, 9097, 10162, 127343, 12975, 127569, 12959, 127568, 12963, + 12898, 12912, 9097, 127343, 10162, 12975, 127569, 12959, 127568, 12963, 12951, 12962, 12965, 12973, 12943, 12957, 12935, 12950, 12939, 12932, 12955, 12931, 12964, 12946, 12869, 12871, 12952, 12966, 12967, 12969, 12942, 12954, 12938, 12976, 12936, 12948, 12868, 12974, 12961, 12970, 12968, 12953, 12934, 12972, 12956, 12944, 12870, 12947, 12949, 12971, - 12945, 12933, 12930, 12937, 12929, 12941, 12940, 12960, 12958, 12928, + 12945, 12933, 12958, 12930, 12937, 12929, 12941, 12940, 12960, 12928, 127275, 127276, 128712, 13033, 13036, 13034, 13037, 13035, 13013, 13016, 13014, 13017, 13015, 13038, 13041, 13039, 13042, 13040, 13028, 13031, 13029, 13032, 13030, 13046, 13049, 13047, 13050, 13048, 13018, 13021, @@ -9682,30 +10006,30 @@ static const unsigned int dawg_pos_to_codepoint[] = { 12009, 11951, 11925, 11924, 11922, 11949, 11948, 11917, 11916, 11958, 11932, 12017, 11911, 11923, 11969, 11982, 11981, 11938, 11937, 11972, 11971, 11956, 11955, 11954, 11953, 11913, 11912, 11961, 12752, 12743, - 12748, 12757, 12741, 12750, 12769, 12747, 12749, 12744, 12742, 12746, - 12768, 12758, 12754, 12763, 12770, 12764, 12753, 12740, 12767, 12760, - 12759, 12745, 12766, 12762, 12755, 12761, 12736, 12765, 12739, 12737, - 12738, 12756, 12751, 12771, 128079, 127916, 127963, 128385, 129346, - 127867, 128203, 128346, 128358, 128343, 128355, 128340, 128352, 128339, - 128351, 128344, 128356, 128336, 128348, 128342, 128354, 128341, 128353, - 128345, 128357, 128347, 128359, 128337, 128349, 128338, 128350, 10561, - 8754, 128259, 10227, 128472, 128257, 128258, 8631, 11118, 8635, 8753, - 10959, 10961, 10960, 10962, 127746, 10828, 10832, 128234, 128235, 128272, - 128213, 10829, 8272, 9729, 127786, 127785, 127784, 127783, 129313, 9114, - 129715, 127864, 129381, 58, 8788, 8353, 128165, 7625, 7623, 769, 791, - 833, 8410, 8404, 8423, 857, 8432, 7677, 844, 774, 7627, 814, 810, 838, - 70459, 780, 812, 784, 8409, 8405, 787, 789, 806, 65062, 65069, 42609, - 1160, 11774, 11744, 11768, 11747, 11757, 11765, 42654, 11751, 11752, - 11753, 11756, 42613, 11775, 11772, 42655, 11767, 11754, 42619, 11763, - 11762, 42618, 42615, 42612, 42617, 11770, 42614, 11771, 11773, 11769, - 11759, 42616, 11760, 11758, 11748, 11749, 11761, 11746, 11764, 11755, - 11745, 11750, 11766, 42621, 1156, 1158, 1159, 1157, 42608, 42610, 1155, - 65070, 65071, 1161, 42620, 123023, 42607, 770, 813, 807, 43248, 43244, - 43245, 43246, 43247, 43242, 43243, 43249, 43237, 43236, 43239, 43238, - 43235, 43234, 43232, 43241, 43233, 43240, 7675, 776, 6833, 804, 775, - 7672, 856, 803, 7674, 7617, 7616, 861, 860, 865, 7676, 862, 863, 6840, - 831, 6858, 6857, 6844, 866, 858, 864, 65058, 65059, 8422, 840, 782, 779, - 783, 819, 7629, 6832, 798, 6835, 8413, 8416, 8418, 8414, 8419, 8420, + 12748, 12768, 12772, 12757, 12741, 12750, 12769, 12747, 12749, 12744, + 12742, 12746, 12758, 12754, 12763, 12770, 12764, 12753, 12740, 12767, + 12760, 12759, 12745, 12773, 12766, 12762, 12755, 12761, 12736, 12765, + 12739, 12737, 12738, 12756, 12751, 12771, 128079, 127916, 127963, 128385, + 129346, 127867, 128203, 128346, 128358, 128343, 128355, 128340, 128352, + 128339, 128351, 128344, 128356, 128336, 128348, 128342, 128354, 128341, + 128353, 128345, 128357, 128347, 128359, 128337, 128349, 128338, 128350, + 10561, 8754, 128259, 10227, 128472, 128257, 128258, 8631, 11118, 8635, + 8753, 10959, 10961, 10960, 10962, 127746, 10828, 10832, 128234, 128235, + 128272, 128213, 10829, 8272, 9729, 127786, 127785, 127784, 127783, + 129313, 9114, 129715, 127864, 129381, 58, 8788, 8353, 128165, 7625, 7623, + 769, 791, 833, 8410, 8404, 8423, 857, 8432, 7677, 844, 774, 7627, 814, + 810, 838, 70459, 780, 812, 784, 8409, 8405, 787, 789, 806, 65062, 65069, + 42609, 1160, 11774, 11744, 11768, 11747, 11757, 11765, 42654, 11751, + 11752, 11753, 11756, 42613, 11775, 11772, 42655, 11767, 11754, 42619, + 11763, 11762, 42618, 42615, 42612, 42617, 11770, 42614, 11771, 11773, + 11769, 11759, 42616, 11760, 11758, 11748, 11749, 11761, 11746, 11764, + 11755, 11745, 11750, 11766, 42621, 1156, 1158, 1159, 1157, 42608, 42610, + 1155, 65070, 65071, 1161, 42620, 123023, 42607, 770, 813, 807, 43248, + 43244, 43245, 43246, 43247, 43242, 43243, 43249, 43237, 43236, 43239, + 43238, 43235, 43234, 43232, 43241, 43233, 43240, 7675, 776, 6833, 804, + 775, 7672, 856, 803, 7674, 7617, 7616, 861, 860, 865, 7676, 862, 863, + 6840, 831, 6858, 6857, 6844, 866, 858, 864, 65058, 65059, 8422, 840, 782, + 779, 783, 819, 7629, 6832, 798, 6835, 8413, 8416, 8418, 8414, 8419, 8420, 8415, 839, 850, 8412, 122892, 122884, 122891, 122890, 122921, 122919, 122889, 122916, 122900, 122907, 122910, 122901, 122908, 122880, 122920, 122881, 122909, 122903, 122922, 122883, 122895, 122894, 122896, 122898, @@ -9735,10 +10059,10 @@ static const unsigned int dawg_pos_to_codepoint[] = { 996, 11434, 11472, 11414, 11422, 11478, 11468, 11470, 11476, 11474, 11482, 11460, 11480, 11454, 11462, 11444, 11486, 11488, 11484, 11490, 11440, 1004, 994, 11456, 11402, 11428, 11408, 11430, 11404, 11394, 11438, - 11424, 11410, 11426, 11400, 11416, 11418, 11432, 66272, 66298, 66289, - 66295, 66286, 66294, 66285, 66299, 66290, 66291, 66297, 66288, 66296, - 66287, 66293, 66284, 66292, 66283, 66282, 66277, 66276, 66279, 66278, - 66275, 66274, 66281, 66273, 66280, 11517, 11518, 11514, 11515, 11516, + 11424, 11410, 11426, 11400, 11416, 11418, 11432, 66298, 66289, 66295, + 66286, 66294, 66285, 66299, 66290, 66291, 66297, 66288, 66296, 66287, + 66293, 66284, 66292, 66283, 66282, 66277, 66276, 66279, 66278, 66275, + 66274, 66281, 66273, 66280, 66272, 11517, 11518, 11514, 11515, 11516, 11513, 11465, 11393, 11507, 11500, 11502, 11447, 11453, 11459, 11467, 11443, 11449, 11451, 11399, 1007, 1003, 11397, 1001, 11407, 999, 11437, 11413, 11421, 997, 11435, 11473, 11415, 11423, 11479, 11469, 11471, @@ -9749,7 +10073,7 @@ static const unsigned int dawg_pos_to_codepoint[] = { 8792, 129720, 9012, 9013, 119661, 119660, 119663, 119662, 119659, 119658, 119665, 119657, 119664, 119652, 119651, 119654, 119653, 119650, 119649, 119656, 119648, 119655, 128715, 128145, 128004, 128046, 9904, 129689, - 129509, 983074, 128179, 127769, 129431, 127951, 9769, 9768, 11857, 11856, + 129509, 983074, 128179, 127769, 129431, 127951, 9769, 9768, 11856, 11857, 128322, 128321, 10060, 127370, 127884, 9876, 9932, 128010, 129360, 128081, 8354, 129660, 128575, 128546, 128302, 129408, 8731, 74794, 74771, 74861, 74780, 74820, 74821, 74765, 74758, 74853, 74856, 74855, 74854, @@ -9815,462 +10139,871 @@ static const unsigned int dawg_pos_to_codepoint[] = { 74117, 74116, 74118, 74630, 74119, 74962, 74963, 74631, 74122, 74123, 74120, 74121, 74626, 74107, 74106, 74628, 74108, 74109, 74124, 74125, 74126, 74131, 74132, 74128, 74129, 74130, 74133, 74134, 74135, 74136, - 74137, 74138, 74139, 74140, 74141, 74142, 74607, 74127, 74144, 74146, - 74147, 74145, 74152, 74153, 74150, 74151, 74148, 74149, 74154, 74155, - 74157, 74158, 74163, 74164, 74165, 74160, 74161, 74156, 74159, 74162, - 74143, 74166, 74167, 74168, 74169, 74170, 74171, 74172, 74175, 74173, - 74174, 74176, 74177, 74182, 74183, 74180, 74181, 74632, 74186, 74184, - 74185, 74188, 74190, 74189, 74187, 74194, 74195, 74193, 74191, 74192, - 74196, 74197, 74198, 74199, 74200, 74201, 74202, 74205, 74206, 74207, - 74208, 74204, 74209, 74211, 74210, 74212, 74213, 74215, 74214, 74216, - 74218, 74217, 74964, 74178, 74179, 74203, 74219, 74220, 74223, 74224, - 74221, 74222, 74966, 74967, 74974, 74973, 74972, 74969, 74970, 74971, - 74975, 74968, 74965, 74977, 74976, 74980, 74981, 74982, 74984, 74985, - 74978, 74979, 74983, 74986, 74987, 74988, 74989, 74990, 74991, 74993, - 74997, 74996, 74998, 74995, 74994, 74992, 74999, 75000, 75003, 75004, - 75005, 75006, 75001, 75002, 75009, 75015, 75016, 75019, 75017, 75018, - 75013, 75012, 75010, 75011, 75014, 75021, 75030, 75029, 75027, 75024, - 75025, 75028, 75022, 75026, 75023, 75008, 75020, 75031, 75032, 75007, - 74226, 74227, 74228, 74229, 74230, 74225, 74231, 74233, 74234, 74232, - 74235, 74237, 74258, 74259, 75033, 74261, 74633, 74260, 74240, 74634, - 74241, 74243, 75035, 74246, 74247, 74245, 74248, 74249, 74250, 74251, - 75036, 75037, 74254, 74255, 74635, 74256, 75038, 74242, 74252, 74253, - 75034, 74238, 74239, 74244, 74257, 74263, 74265, 74264, 74266, 74269, - 74270, 74271, 74236, 74262, 74267, 74268, 74272, 74273, 74274, 74278, - 74279, 74275, 74276, 74277, 74280, 74281, 74636, 74282, 75039, 74283, - 74284, 74290, 74294, 74295, 75041, 75040, 74292, 74293, 74291, 74296, - 74297, 74298, 74299, 74300, 74637, 74301, 74286, 74287, 74285, 74289, - 74288, 74302, 74304, 74307, 74305, 74306, 74308, 74310, 74309, 74311, - 74303, 74638, 74312, 74314, 74313, 74315, 74316, 74319, 74321, 74320, - 74639, 74322, 74324, 74325, 74323, 74642, 75043, 74326, 75044, 75046, - 75047, 74327, 75048, 74329, 74328, 75049, 74330, 74332, 74333, 74331, - 75050, 75051, 74334, 75052, 74335, 75042, 74641, 75045, 74640, 74317, - 74336, 74318, 74337, 74338, 983266, 983265, 74643, 74339, 74347, 74348, - 74342, 74343, 74341, 74344, 74340, 74346, 74345, 74349, 74355, 74354, - 74350, 74352, 74358, 74359, 74353, 74357, 74351, 74356, 74360, 74361, - 74362, 74363, 74364, 74365, 74366, 74644, 74367, 74375, 74376, 74368, - 74369, 74373, 74374, 74370, 74371, 74372, 74377, 74378, 74379, 74380, - 74381, 74382, 74645, 74383, 74384, 74385, 74386, 74387, 74389, 74408, - 75053, 74388, 74646, 74395, 74394, 75055, 74400, 74399, 75056, 74401, - 74406, 74402, 74403, 74404, 74405, 74391, 74392, 74396, 74398, 75054, - 74397, 74393, 74390, 74407, 74409, 74410, 74411, 74412, 74413, 74414, - 74421, 74422, 74419, 74417, 74420, 74416, 74418, 74415, 74423, 75057, - 74424, 74425, 74426, 75058, 74428, 74429, 75059, 75060, 75061, 74427, - 74432, 74434, 74433, 74430, 74431, 74435, 74437, 74436, 74438, 74441, - 74440, 74444, 74445, 74446, 74448, 74447, 74443, 74449, 74442, 74439, - 74451, 74453, 74452, 74450, 74454, 74455, 74456, 74457, 75063, 75062, - 74458, 74459, 75064, 74460, 74461, 74462, 74463, 74465, 74464, 74466, - 74468, 74469, 74471, 74472, 74473, 74474, 74467, 74470, 74475, 74477, - 74478, 74479, 74476, 74480, 74481, 74482, 74483, 74488, 74486, 74487, - 74485, 74489, 74484, 74490, 75065, 74491, 74494, 74497, 74499, 74500, - 74498, 74495, 74647, 74496, 74501, 74504, 75066, 75067, 74505, 74506, - 74502, 74503, 74492, 74493, 74507, 74510, 74512, 74511, 74649, 74509, - 74508, 74515, 74516, 74522, 74523, 74519, 74520, 74517, 74518, 74521, - 74524, 74534, 74535, 74525, 74526, 74648, 74527, 74528, 74529, 74531, - 74532, 74533, 74530, 74536, 74538, 74537, 74539, 75068, 74540, 74541, - 74542, 74545, 74546, 74547, 75069, 74544, 74543, 74549, 74550, 74551, - 74552, 74553, 75070, 74555, 74556, 74558, 74557, 74559, 74560, 74562, - 74564, 74563, 75072, 74566, 75071, 74570, 74569, 74572, 74574, 74573, - 74554, 74567, 74571, 74565, 74561, 74568, 74575, 74576, 74548, 74577, - 74581, 74579, 74580, 74578, 74584, 74583, 74582, 74586, 74587, 74588, - 74585, 74513, 74514, 74589, 74591, 74590, 74593, 75073, 74592, 74595, - 74598, 74599, 74596, 74601, 74597, 74600, 74602, 75074, 74603, 75075, - 74604, 74605, 74606, 74594, 9982, 129380, 11232, 129473, 8911, 8910, - 10160, 9130, 129356, 128177, 164, 127835, 10081, 127854, 128707, 129362, - 129385, 67594, 67595, 67596, 67597, 67598, 67599, 67600, 67601, 67602, - 67603, 67604, 67605, 67606, 67607, 67608, 67609, 67610, 67611, 67612, - 67613, 67614, 67615, 67616, 67617, 67618, 67619, 67620, 67621, 67622, - 67623, 67624, 67625, 67626, 67627, 67628, 67629, 67630, 67631, 67632, - 67633, 67634, 67635, 67636, 67637, 67589, 67592, 67644, 67647, 67639, - 67640, 67584, 67585, 67586, 67587, 67588, 77712, 77713, 77714, 77715, - 77716, 77717, 77718, 77719, 77722, 77723, 77720, 77721, 77724, 77725, - 77726, 77727, 77728, 77729, 77730, 77731, 77732, 77733, 77734, 77735, - 77736, 77737, 77738, 77739, 77740, 77741, 77742, 77743, 77744, 77745, - 77746, 77747, 77748, 77749, 77750, 77751, 77752, 77753, 77754, 77755, - 77756, 77757, 77758, 77773, 77774, 77768, 77769, 77770, 77771, 77772, - 77775, 77776, 77777, 77788, 77789, 77790, 77791, 77792, 77793, 77794, - 77795, 77796, 77759, 77760, 77761, 77762, 77763, 77764, 77765, 77766, - 77767, 77778, 77779, 77780, 77781, 77782, 77783, 77784, 77785, 77786, - 77787, 77797, 77798, 77799, 77800, 77801, 77802, 77803, 77804, 77805, - 77806, 77807, 77808, 77809, 77810, 1126, 1033, 1300, 42574, 1034, 1055, - 1190, 1316, 1136, 1146, 42564, 42592, 42580, 1296, 1302, 1058, 42634, - 1196, 1035, 42640, 42638, 1062, 42642, 42636, 1059, 1266, 1264, 1262, - 1144, 1028, 1040, 1234, 1232, 1212, 1214, 1248, 1192, 1310, 1256, 1258, - 1184, 42602, 1130, 42572, 42586, 1030, 1041, 1063, 1268, 1206, 1208, - 42584, 42650, 42630, 1026, 42568, 42604, 42648, 1029, 42562, 1322, 42632, - 1039, 42626, 1324, 42624, 1044, 1069, 1051, 1312, 1326, 1221, 1298, 1053, - 1314, 1186, 1320, 1225, 1223, 1056, 1166, 1260, 1057, 1194, 1052, 1229, - 1060, 1043, 1172, 1168, 1270, 1170, 1274, 1027, 1061, 1202, 1276, 1278, - 1066, 42644, 1048, 1252, 1037, 1250, 1045, 1024, 1238, 1025, 42588, 1128, - 1132, 42578, 42582, 1124, 42566, 1140, 1142, 1050, 1178, 1219, 1180, - 1182, 1286, 1282, 1280, 1288, 1290, 1292, 1294, 1284, 1152, 1227, 1036, - 1134, 42600, 42570, 1054, 1120, 1148, 1254, 1150, 1240, 1242, 1049, 1162, - 1038, 1210, 1318, 1065, 42646, 1064, 42596, 42598, 1068, 42594, 1198, - 1200, 1164, 1071, 1304, 1122, 1067, 1272, 42576, 1031, 42590, 1070, 1047, - 1246, 1176, 42560, 1046, 1244, 1174, 1217, 42628, 1138, 1032, 1042, 1308, - 1306, 1236, 1204, 1188, 42622, 42606, 1216, 7467, 42623, 1072, 1235, - 1233, 1213, 1215, 1249, 1193, 1311, 1257, 1259, 1185, 42603, 1131, 42573, - 42587, 1110, 1073, 1095, 1269, 1207, 1209, 42585, 42651, 42631, 1106, - 42569, 42605, 42649, 1109, 42563, 1323, 42633, 1119, 42627, 1325, 42625, - 1076, 1101, 1083, 1313, 1327, 1222, 1299, 1085, 1315, 1187, 1321, 1226, - 1224, 1088, 1167, 1261, 1089, 1195, 1084, 1230, 1092, 1075, 1173, 1169, - 1271, 1171, 1275, 1107, 1093, 1203, 1277, 1279, 1098, 42645, 1080, 1253, - 1117, 1251, 1077, 1104, 1239, 1105, 42589, 1129, 1133, 42579, 42583, - 1125, 42567, 1141, 1143, 1082, 1179, 1220, 1181, 1183, 1287, 1283, 1281, - 1289, 1291, 1293, 1295, 1285, 1153, 1228, 1116, 1135, 1127, 7297, 1113, - 1301, 42601, 42571, 42575, 7298, 1114, 1086, 1121, 1149, 1255, 1151, - 1087, 1191, 1317, 1231, 1137, 42565, 42593, 42581, 1297, 1147, 7296, - 1303, 1241, 1243, 1081, 1163, 1118, 1211, 1319, 1097, 42647, 1096, 42597, - 42599, 1100, 42595, 1199, 1201, 1165, 7302, 7303, 7300, 1090, 42635, - 1197, 1115, 42641, 42639, 1094, 7301, 42643, 42637, 1091, 1267, 1265, - 1263, 1145, 1108, 7304, 7299, 1309, 1103, 1305, 1123, 1099, 1273, 42577, - 1111, 42591, 1102, 1079, 1247, 1177, 42561, 1078, 1245, 1175, 1218, - 42629, 1139, 1112, 1074, 1307, 1237, 1205, 1189, 122984, 122962, 122986, - 122985, 122965, 122976, 122971, 122974, 122964, 122983, 122977, 122981, - 122982, 122980, 122978, 122967, 122968, 122969, 122966, 122979, 122973, - 122963, 122970, 122961, 122972, 122975, 1154, 9005, 127744, 983201, - 983188, 983172, 8224, 11830, 11831, 128481, 128131, 127841, 128374, 9619, - 11843, 128168, 10143, 65101, 65097, 8504, 983081, 983084, 983086, 983088, - 983090, 983162, 127795, 9110, 9192, 128475, 8451, 176, 8457, 983120, - 128666, 8796, 983119, 9161, 9159, 9153, 9156, 9162, 9160, 9154, 9157, - 9163, 9150, 9158, 9164, 9151, 9152, 9155, 127980, 9739, 66589, 66591, - 66596, 66597, 66587, 66585, 66594, 66595, 66593, 66599, 66562, 66563, - 66564, 66565, 66561, 66560, 66568, 66569, 66570, 66571, 66567, 66566, - 66598, 66573, 66588, 66579, 66592, 66590, 66581, 66578, 66580, 66582, - 66577, 66586, 66575, 66584, 66583, 66574, 66572, 66576, 66629, 66631, - 66636, 66637, 66627, 66625, 66634, 66635, 66633, 66639, 66602, 66603, - 66604, 66605, 66601, 66600, 66608, 66609, 66610, 66611, 66607, 66606, - 66638, 66613, 66628, 66619, 66632, 66630, 66621, 66618, 66620, 66622, - 66617, 66626, 66615, 66624, 66623, 66614, 66612, 66616, 127964, 127965, - 128421, 128468, 2388, 2416, 43258, 2387, 43257, 72448, 72449, 43259, - 2309, 2310, 2320, 2324, 2421, 43262, 2330, 2418, 2317, 2321, 2331, 2396, - 2430, 2338, 2337, 2343, 2342, 2429, 2394, 2328, 2427, 2327, 2426, 2361, - 2350, 2424, 2308, 2318, 2322, 2358, 2359, 2360, 2323, 2420, 2419, 2431, - 2349, 2348, 2333, 2428, 2332, 2393, 2326, 2325, 2397, 2353, 2352, 2399, - 2351, 2313, 2314, 2423, 2422, 2345, 2339, 2329, 2334, 2344, 2356, 2355, - 2354, 2336, 2335, 2341, 2340, 2315, 2400, 2316, 2401, 2357, 2311, 2312, - 2347, 2346, 2425, 2395, 2398, 2392, 2319, 983644, 983643, 983646, 983647, - 983649, 983648, 983642, 983645, 72450, 72451, 72452, 72453, 2305, 43255, - 43251, 43254, 43253, 72455, 72454, 72456, 43256, 43250, 43260, 43252, - 2304, 2365, 2306, 72457, 2364, 2417, 2381, 2307, 2386, 2385, 2366, 2376, - 2380, 2383, 43263, 2389, 2373, 2377, 2379, 2363, 2362, 2382, 2369, 2370, - 2391, 2390, 2374, 2378, 2371, 2372, 2402, 2403, 2367, 2368, 2375, 2405, - 2404, 2411, 2410, 2413, 2412, 2409, 2408, 2406, 2415, 2407, 2414, 43261, - 2384, 983089, 983161, 983087, 983085, 983083, 127962, 129487, 129420, - 11033, 11032, 11030, 11031, 128924, 128160, 8900, 8960, 168, 9856, 9857, - 9858, 9859, 9860, 9861, 128754, 53, 127238, 9356, 52, 127237, 9355, 57, - 127242, 9360, 49, 127234, 9352, 55, 127240, 9358, 54, 127239, 9357, 51, - 127236, 9354, 50, 127235, 9353, 48, 127233, 127232, 56, 127241, 9359, - 119557, 119556, 119555, 9868, 9871, 9870, 9869, 119553, 119554, 10131, - 10126, 10125, 10128, 10127, 10124, 10123, 127244, 10130, 10122, 10129, - 10111, 10106, 10105, 10108, 10107, 10104, 10103, 10110, 10102, 10109, - 10121, 10116, 10115, 10118, 10117, 10114, 10113, 127243, 10120, 10112, - 10119, 9107, 127919, 128549, 128542, 9933, 9090, 129400, 72004, 72021, - 72020, 72023, 72022, 72019, 72018, 72016, 72025, 72017, 72024, 71964, - 71958, 71963, 71974, 71973, 71936, 71937, 71961, 71960, 71966, 71965, - 71940, 71941, 71938, 71939, 71982, 71976, 71952, 71962, 71957, 71967, - 71978, 71979, 71980, 71971, 71970, 71954, 71953, 71951, 71950, 71949, - 71948, 71969, 71968, 71981, 71955, 71972, 71975, 71977, 71983, 71942, - 71945, 71997, 71996, 72003, 71995, 71984, 71991, 71987, 71988, 71985, - 71986, 71989, 71992, 71998, 72002, 72000, 72005, 72006, 71999, 72001, - 8725, 247, 8903, 129343, 8739, 9902, 129684, 128171, 128565, 12291, 8783, - 9009, 128462, 128441, 128442, 128443, 8939, 8941, 8716, 8740, 10990, - 8832, 8928, 8876, 8833, 8929, 8878, 128021, 71723, 71716, 71680, 71681, - 71687, 71689, 71703, 71702, 71708, 71707, 71701, 71700, 71706, 71705, - 71684, 71685, 71682, 71683, 71694, 71704, 71699, 71709, 71719, 71720, - 71721, 71713, 71712, 71696, 71695, 71693, 71692, 71698, 71697, 71691, - 71690, 71711, 71710, 71722, 71717, 71714, 71718, 71715, 71686, 71688, - 71729, 71730, 71724, 71732, 71734, 71727, 71728, 71725, 71726, 71731, - 71733, 71739, 71738, 71735, 71737, 71736, 128054, 128044, 36, 127025, - 127026, 127027, 127028, 127029, 127030, 127031, 127032, 127033, 127034, - 127035, 127036, 127037, 127038, 127039, 127040, 127041, 127042, 127043, - 127044, 127045, 127046, 127047, 127048, 127049, 127050, 127051, 127052, - 127053, 127054, 127055, 127056, 127057, 127058, 127059, 127060, 127061, - 127062, 127063, 127064, 127065, 127066, 127067, 127068, 127069, 127070, - 127071, 127072, 127073, 127024, 127075, 127076, 127077, 127078, 127079, - 127080, 127081, 127082, 127083, 127084, 127085, 127086, 127087, 127088, - 127089, 127090, 127091, 127092, 127093, 127094, 127095, 127096, 127097, - 127098, 127099, 127100, 127101, 127102, 127103, 127104, 127105, 127106, - 127107, 127108, 127109, 127110, 127111, 127112, 127113, 127114, 127115, - 127116, 127117, 127118, 127119, 127120, 127121, 127122, 127123, 127074, - 129743, 8363, 8724, 8760, 8901, 729, 9676, 8284, 11850, 11034, 129765, - 11795, 11784, 10649, 11798, 9470, 9465, 9464, 9467, 9466, 9463, 9462, - 9469, 9461, 9468, 10175, 10868, 10986, 8225, 8223, 11840, 8914, 8748, - 11842, 8222, 8215, 10835, 10836, 10645, 10913, 10915, 10914, 10939, 8243, - 12318, 10746, 10830, 10831, 11849, 10988, 11844, 10940, 8913, 8912, - 11005, 10987, 8915, 9208, 10981, 8875, 10979, 8214, 11799, 10646, 733, - 8252, 8263, 65100, 10908, 10907, 11002, 11001, 10906, 10905, 8510, 8473, - 8511, 8450, 8461, 8469, 8474, 8477, 8484, 8518, 8519, 8520, 8521, 8517, - 8509, 8508, 8512, 10719, 9890, 9891, 11260, 127849, 8868, 10993, 10623, - 8945, 8964, 128315, 128317, 10728, 10729, 129288, 129290, 129289, 129291, - 8595, 8629, 8671, 129035, 129031, 129179, 129043, 129027, 129047, 8626, - 8627, 10504, 8693, 129975, 8615, 10515, 11796, 11107, 11139, 11123, - 129067, 11168, 11169, 129063, 129059, 129075, 129071, 11133, 11085, - 11117, 11143, 129171, 10507, 8609, 11247, 8681, 129175, 8623, 11147, - 11015, 8659, 8675, 129079, 10597, 10607, 10593, 10585, 8643, 10589, - 10581, 8642, 129091, 129095, 129087, 10225, 128623, 129107, 129083, 8650, - 128687, 128330, 128682, 129444, 8367, 10139, 128009, 128050, 128042, - 129656, 128167, 129316, 129345, 9946, 128087, 113784, 113788, 113785, - 113783, 113782, 113786, 113787, 113795, 113798, 113793, 113800, 113781, - 113794, 113792, 113799, 113797, 113796, 113776, 113808, 113817, 113811, - 113814, 113809, 113816, 113779, 113810, 113815, 113813, 113812, 113778, - 113777, 113780, 113729, 113733, 113672, 113677, 113683, 113735, 113739, - 113746, 113668, 113678, 113674, 113726, 113691, 113699, 113700, 113695, - 113709, 113712, 113713, 113705, 113711, 113669, 113725, 113679, 113684, - 113670, 113743, 113749, 113687, 113689, 113693, 113707, 113697, 113703, - 113690, 113694, 113708, 113698, 113704, 113764, 113763, 113762, 113761, - 113732, 113753, 113731, 113755, 113754, 113666, 113766, 113765, 113676, - 113675, 113741, 113750, 113680, 113688, 113692, 113696, 113710, 113727, - 113728, 113716, 113717, 113714, 113715, 113701, 113702, 113724, 113723, - 113706, 113742, 113740, 113767, 113769, 113730, 113768, 113682, 113685, - 113752, 113737, 113667, 113719, 113718, 113681, 113745, 113748, 113751, - 113738, 113673, 113770, 113720, 113757, 113760, 113722, 113759, 113756, - 113721, 113758, 113665, 113747, 113664, 113686, 113734, 113736, 113744, - 113671, 113821, 113822, 113823, 113820, 129375, 129414, 129516, 128192, - 983082, 128066, 127805, 127806, 129467, 9793, 127758, 127759, 127757, - 9178, 9841, 129413, 11790, 77830, 77831, 77832, 77828, 77829, 77824, - 77825, 77826, 77827, 77833, 77834, 77835, 77840, 77841, 77844, 77845, - 77836, 77837, 77838, 77839, 77842, 77843, 77846, 77847, 77869, 77870, - 77872, 77873, 77874, 77875, 77877, 77878, 77871, 77876, 77879, 77880, - 77881, 77882, 77860, 77861, 77858, 77859, 77862, 77863, 77864, 77865, - 77866, 77867, 77868, 77903, 77848, 77849, 77850, 77851, 77852, 77853, - 77854, 77855, 77856, 77857, 77883, 77884, 77885, 77886, 77887, 77888, - 77889, 77890, 77891, 77892, 77893, 77894, 77895, 77896, 77897, 77898, - 77899, 77900, 77901, 77902, 78867, 78868, 78869, 78861, 78862, 78863, - 78864, 78865, 78866, 78870, 78871, 78892, 78893, 78894, 78872, 78873, - 78874, 78875, 78876, 78877, 78878, 78879, 78880, 78881, 78882, 78883, - 78884, 78885, 78886, 78887, 78888, 78889, 78890, 78891, 78903, 78910, - 78908, 77908, 77909, 77904, 77905, 77906, 77907, 77910, 77911, 77912, - 77913, 77915, 77916, 77917, 77918, 77914, 77919, 77920, 77921, 77922, - 77923, 77924, 77925, 77926, 77927, 77928, 77929, 77930, 77931, 77932, - 77933, 77934, 77935, 77936, 77937, 77938, 77939, 77940, 77941, 77949, - 77950, 77942, 77943, 77944, 77945, 77946, 77947, 77948, 77951, 77974, - 77975, 77978, 77979, 77973, 77976, 77977, 77980, 77981, 77982, 77983, - 77984, 77991, 77992, 77994, 77995, 77985, 77986, 77987, 77988, 77989, - 77990, 77993, 77996, 77997, 77998, 77999, 78000, 78001, 78002, 78003, - 78004, 78005, 78006, 78008, 78009, 78011, 78012, 78007, 78010, 78013, - 78014, 78015, 78016, 78017, 78025, 78026, 78027, 78028, 78029, 78030, - 78031, 78032, 78033, 78018, 78019, 78020, 78021, 78022, 78023, 78024, - 77969, 77970, 77962, 77963, 77964, 77965, 77966, 77967, 77968, 77971, - 77972, 77952, 77953, 77954, 77955, 77956, 77957, 77958, 77959, 77960, - 77961, 78041, 78042, 78043, 78044, 78034, 78035, 78036, 78037, 78038, - 78039, 78040, 78057, 78058, 78066, 78067, 78059, 78060, 78061, 78062, - 78063, 78064, 78065, 78068, 78073, 78074, 78069, 78070, 78071, 78072, - 78075, 78076, 78077, 78051, 78052, 78053, 78054, 78045, 78046, 78047, - 78048, 78049, 78050, 78055, 78056, 78904, 78911, 78909, 78078, 78079, - 78080, 78081, 78082, 78083, 78084, 78085, 78086, 78087, 78091, 78092, - 78088, 78089, 78090, 78093, 78094, 78095, 78096, 78097, 78098, 78111, - 78112, 78118, 78119, 78120, 78121, 78110, 78113, 78114, 78115, 78116, - 78117, 78122, 78128, 78129, 78130, 78131, 78132, 78133, 78123, 78124, - 78125, 78126, 78127, 78134, 78135, 78137, 78138, 78139, 78140, 78136, - 78141, 78142, 78100, 78101, 78099, 78102, 78103, 78104, 78105, 78106, - 78107, 78108, 78109, 78913, 78150, 78151, 78152, 78148, 78149, 78143, - 78144, 78145, 78146, 78147, 78153, 78154, 78156, 78157, 78155, 78158, - 78159, 78160, 78161, 78162, 78163, 78164, 78165, 78184, 78185, 78186, - 78187, 78178, 78179, 78180, 78181, 78182, 78183, 78188, 78189, 78193, - 78194, 78196, 78197, 78190, 78191, 78192, 78195, 78198, 78199, 78200, - 78201, 78166, 78167, 78173, 78174, 78168, 78169, 78170, 78171, 78172, - 78175, 78176, 78177, 78202, 78203, 78204, 78205, 78206, 78212, 78213, - 78207, 78208, 78209, 78210, 78211, 78214, 78215, 78914, 78916, 78897, - 78220, 78221, 78225, 78226, 78216, 78217, 78218, 78219, 78222, 78223, - 78224, 78227, 78228, 78229, 78230, 78231, 78232, 78233, 78234, 78907, - 78901, 78899, 78906, 78900, 78898, 78905, 78244, 78245, 78249, 78250, - 78243, 78246, 78247, 78248, 78251, 78252, 78915, 78253, 78254, 78255, - 78257, 78258, 78256, 78259, 78260, 78261, 78262, 78263, 78264, 78268, - 78269, 78270, 78271, 78272, 78273, 78274, 78275, 78276, 78265, 78266, - 78279, 78280, 78281, 78282, 78283, 78284, 78267, 78277, 78278, 78285, - 78286, 78289, 78290, 78292, 78293, 78297, 78298, 78287, 78288, 78291, - 78294, 78295, 78296, 78299, 78304, 78305, 78306, 78301, 78302, 78300, - 78303, 78307, 78308, 78309, 78310, 78311, 78312, 78313, 78314, 78315, - 78316, 78317, 78318, 78933, 78928, 78920, 78924, 78932, 78926, 78921, - 78929, 78925, 78923, 78931, 78919, 78927, 78922, 78930, 78912, 78336, - 78337, 78338, 78328, 78329, 78330, 78331, 78332, 78333, 78334, 78335, - 78339, 78354, 78355, 78356, 78357, 78358, 78359, 78361, 78362, 78351, - 78352, 78353, 78360, 78363, 78364, 78345, 78346, 78340, 78341, 78342, - 78343, 78344, 78347, 78348, 78349, 78350, 78365, 78366, 78367, 78319, - 78320, 78321, 78322, 78323, 78324, 78325, 78326, 78327, 78372, 78373, - 78368, 78369, 78370, 78371, 78374, 78375, 78376, 78377, 78385, 78386, - 78378, 78379, 78380, 78381, 78382, 78383, 78384, 78387, 78388, 78389, - 78399, 78400, 78401, 78402, 78409, 78410, 78403, 78404, 78405, 78406, - 78407, 78408, 78411, 78414, 78415, 78412, 78413, 78390, 78391, 78392, - 78393, 78394, 78395, 78396, 78397, 78398, 78423, 78424, 78425, 78426, - 78427, 78428, 78429, 78416, 78417, 78421, 78422, 78418, 78419, 78420, - 78430, 78431, 78432, 78433, 78434, 78435, 78436, 78445, 78446, 78437, - 78438, 78439, 78440, 78441, 78442, 78443, 78444, 78447, 78448, 78452, - 78453, 78454, 78455, 78459, 78460, 78449, 78450, 78451, 78456, 78457, - 78458, 78469, 78470, 78471, 78472, 78473, 78461, 78462, 78465, 78466, - 78463, 78464, 78467, 78468, 78474, 78475, 78476, 78487, 78488, 78489, - 78490, 78477, 78478, 78479, 78480, 78481, 78482, 78483, 78484, 78485, - 78486, 78902, 78491, 78492, 78494, 78495, 78493, 78496, 78497, 78498, - 78499, 78500, 78501, 78502, 78503, 78514, 78515, 78516, 78512, 78513, - 78511, 78517, 78518, 78519, 78520, 78521, 78522, 78523, 78524, 78530, - 78531, 78525, 78526, 78527, 78528, 78529, 78532, 78533, 78534, 78535, - 78536, 78537, 78538, 78539, 78540, 78541, 78542, 78543, 78544, 78546, - 78547, 78551, 78552, 78545, 78548, 78549, 78550, 78553, 78554, 78555, - 78560, 78561, 78562, 78565, 78566, 78556, 78557, 78558, 78559, 78563, - 78564, 78567, 78568, 78575, 78576, 78577, 78569, 78570, 78571, 78572, - 78573, 78574, 78578, 78579, 78580, 78586, 78587, 78581, 78582, 78583, - 78584, 78585, 78588, 78589, 78590, 78591, 78592, 78593, 78594, 78595, - 78596, 78597, 78598, 78601, 78602, 78606, 78607, 78608, 78609, 78610, - 78611, 78599, 78600, 78603, 78604, 78605, 78613, 78614, 78619, 78620, - 78612, 78615, 78616, 78617, 78618, 78621, 78622, 78623, 78636, 78637, - 78638, 78639, 78634, 78635, 78640, 78641, 78642, 78624, 78625, 78626, - 78627, 78628, 78629, 78630, 78631, 78632, 78633, 78917, 78648, 78649, - 78650, 78643, 78644, 78645, 78646, 78647, 78651, 78652, 78653, 78667, - 78668, 78674, 78675, 78664, 78665, 78666, 78669, 78670, 78671, 78672, - 78673, 78678, 78679, 78676, 78677, 78680, 78681, 78682, 78683, 78684, - 78685, 78686, 78687, 78688, 78689, 78654, 78655, 78656, 78657, 78658, - 78659, 78660, 78661, 78662, 78663, 78706, 78707, 78708, 78690, 78691, - 78692, 78693, 78694, 78695, 78696, 78697, 78698, 78699, 78700, 78701, - 78702, 78703, 78704, 78705, 78709, 78710, 78712, 78713, 78714, 78715, - 78895, 78716, 78717, 78718, 78711, 78719, 78720, 78721, 78722, 78723, - 78724, 78725, 78726, 78727, 78728, 78729, 78730, 78731, 78732, 78733, - 78734, 78735, 78736, 78737, 78738, 78741, 78742, 78747, 78748, 78749, - 78750, 78739, 78740, 78743, 78744, 78745, 78746, 78751, 78752, 78753, - 78754, 78756, 78757, 78761, 78762, 78755, 78758, 78759, 78760, 78763, - 78764, 78765, 78766, 78896, 78769, 78770, 78776, 78777, 78767, 78768, - 78771, 78772, 78773, 78774, 78775, 78778, 78779, 78783, 78784, 78787, - 78788, 78789, 78790, 78780, 78781, 78782, 78785, 78786, 78791, 78796, - 78797, 78792, 78793, 78794, 78795, 78798, 78918, 78802, 78803, 78804, - 78806, 78807, 78809, 78810, 78799, 78800, 78801, 78805, 78808, 78811, - 78812, 78813, 78814, 78815, 78816, 78817, 78818, 78819, 78821, 78822, - 78823, 78824, 78825, 78826, 78827, 78828, 78829, 78830, 78831, 78832, - 78820, 78833, 78834, 78835, 78836, 78842, 78843, 78844, 78845, 78846, - 78847, 78848, 78849, 78850, 78851, 78852, 78853, 78854, 78855, 78856, - 78857, 78858, 78859, 78860, 78837, 78838, 78839, 78840, 78841, 78235, - 78236, 78237, 78238, 78239, 78240, 78241, 78242, 78504, 78505, 78506, - 78507, 78508, 78509, 78510, 129370, 10037, 10039, 10036, 10049, 10058, - 10035, 9834, 66854, 66853, 66827, 66826, 66833, 66832, 66821, 66837, - 66836, 66835, 66842, 66841, 66824, 66823, 66819, 66818, 66822, 66820, - 66855, 66831, 66844, 66843, 66846, 66845, 66852, 66851, 66817, 66825, - 66828, 66830, 66834, 66839, 66840, 66848, 66849, 66816, 66829, 66838, - 66847, 66850, 128268, 128294, 128161, 8961, 9191, 8712, 8946, 8953, 8947, - 8952, 8950, 8949, 10969, 10194, 128024, 128727, 69608, 69621, 69603, - 69611, 69618, 69619, 69600, 69615, 69602, 69606, 69614, 69617, 69620, - 69609, 69604, 69607, 69610, 69601, 69613, 69605, 69616, 69612, 69622, - 129501, 983101, 129456, 129457, 129459, 129458, 127995, 127996, 127997, - 127998, 127999, 128453, 128454, 128455, 129721, 128460, 128461, 8709, - 10675, 10676, 10674, 10673, 128459, 9091, 8193, 8212, 8195, 8192, 8211, - 8194, 983179, 8718, 983178, 983135, 983099, 983048, 983095, 983046, - 983064, 128282, 983051, 983050, 9993, 128388, 128233, 9094, 983067, - 983100, 983049, 8926, 8927, 8925, 8924, 8797, 8917, 61, 10865, 10867, - 11072, 10609, 10723, 10724, 10926, 11257, 10871, 10854, 10862, 8789, - 10872, 8781, 8794, 10738, 10736, 10734, 10739, 10737, 10735, 11249, - 11248, 9003, 8998, 983105, 983104, 8494, 8793, 983136, 4957, 4959, 4958, - 4963, 4965, 4978, 4988, 4980, 4979, 4987, 4985, 4982, 4981, 4986, 4984, - 4983, 4968, 4966, 4964, 4960, 4999, 4998, 4711, 4997, 43816, 43819, - 43821, 43820, 43818, 43822, 43817, 4704, 4707, 4710, 11653, 4709, 4708, - 4706, 4705, 43808, 43811, 43813, 43812, 43810, 43814, 43809, 11704, - 11707, 11709, 11708, 11706, 11710, 11705, 11688, 11691, 11693, 11692, - 11690, 11694, 11689, 4904, 4907, 4910, 11664, 4909, 4908, 4911, 4906, - 4905, 4728, 4731, 4734, 11655, 4733, 4732, 4735, 4730, 4729, 43789, - 43788, 43787, 43786, 43790, 43785, 4856, 4859, 4862, 11661, 4861, 4860, - 4863, 4858, 4857, 43797, 43796, 43795, 43794, 43798, 43793, 4848, 4851, - 4854, 11660, 4853, 4852, 4855, 4850, 4849, 5003, 5002, 4943, 5001, 4936, - 4939, 4941, 4940, 4954, 4938, 4942, 4937, 4873, 124916, 124915, 124924, - 124923, 124910, 124909, 124926, 124925, 124922, 124921, 124920, 124919, - 124918, 124917, 124914, 124913, 124912, 124904, 11667, 4895, 11670, - 11669, 11668, 4888, 4891, 4893, 4892, 4890, 4894, 4889, 4768, 4771, 4774, - 11658, 4773, 4772, 4775, 4770, 4769, 4880, 4883, 4885, 4884, 4882, 11736, - 11739, 11741, 11740, 11738, 11742, 11737, 4872, 4875, 4878, 4879, 4877, - 4876, 4874, 124907, 124906, 4631, 124905, 124896, 124899, 124901, 124900, - 124898, 124902, 124897, 4624, 4627, 4629, 4628, 4626, 4630, 4625, 4608, - 4611, 4614, 4615, 4613, 4612, 4610, 4609, 4800, 4803, 4805, 4804, 4802, - 4792, 4795, 4797, 4796, 4794, 4798, 4793, 4784, 4787, 4789, 4788, 4786, - 11720, 11723, 11725, 11724, 11722, 11726, 11721, 4776, 4779, 4782, 4783, - 4781, 4780, 4778, 4777, 4995, 4994, 4639, 4993, 4632, 4635, 4638, 11649, - 4637, 4636, 4953, 4634, 4633, 4760, 4763, 4766, 11657, 4765, 4764, 4767, - 4762, 4761, 4752, 4755, 4758, 11656, 4757, 4756, 4759, 4754, 4753, 4912, - 4816, 4819, 4821, 4820, 4818, 4822, 4817, 4915, 4918, 11665, 4917, 4916, - 4919, 4914, 4913, 5007, 5006, 4951, 5005, 4944, 4947, 4950, 11666, 4949, - 4948, 4946, 4945, 4696, 4699, 4701, 4700, 4698, 4688, 4691, 4693, 4692, - 4690, 4694, 4689, 4680, 4683, 4685, 4684, 4682, 11712, 11715, 11717, - 11716, 11714, 11718, 11713, 4672, 4675, 4678, 4679, 4677, 4676, 4674, - 4673, 4648, 4651, 4654, 11650, 4653, 4652, 4655, 4952, 4650, 4649, 4661, - 4996, 5000, 4992, 5004, 4660, 4664, 4667, 4670, 11652, 4669, 4668, 4671, - 4666, 4665, 4640, 4643, 4645, 4644, 4647, 4642, 4646, 4641, 11680, 11683, - 11685, 11684, 11682, 11686, 11681, 4656, 4659, 4662, 11651, 4663, 4658, - 4657, 4896, 4899, 4902, 11663, 4901, 4900, 4903, 4898, 4897, 43781, - 43780, 43779, 43778, 43782, 43777, 4928, 4931, 4934, 4935, 4933, 4932, - 4930, 4929, 4920, 4923, 4925, 4924, 4927, 4922, 4926, 4921, 4720, 4723, - 4726, 11654, 4725, 4724, 4727, 4722, 4721, 4864, 4867, 4870, 11662, 4869, - 4868, 4871, 4866, 4865, 4616, 4619, 4622, 11648, 4621, 4620, 4623, 4618, - 4617, 4808, 4811, 4814, 4815, 4813, 4812, 4810, 4809, 4840, 4843, 4846, - 4847, 4845, 4844, 4842, 4841, 4744, 4747, 4749, 4748, 4746, 11728, 11731, - 11733, 11732, 11730, 11734, 11729, 4736, 4739, 4742, 4743, 4741, 4740, - 4738, 4737, 4832, 4835, 4837, 4836, 4839, 4834, 4838, 4833, 11696, 11699, - 11701, 11700, 11698, 11702, 11697, 4824, 4827, 4830, 11659, 4829, 4828, - 4831, 4826, 4825, 4712, 4715, 4717, 4716, 4719, 4714, 4718, 4713, 5009, - 5016, 5012, 5015, 5013, 5017, 5010, 5011, 5014, 5008, 4973, 4972, 4975, - 4974, 4971, 4970, 4977, 4969, 4976, 4962, 4967, 4961, 983096, 983047, - 127984, 127972, 8352, 8364, 8455, 8265, 33, 8761, 128942, 128954, 128948, - 128905, 128915, 128935, 128125, 1781, 1780, 1783, 1782, 1779, 1778, 1776, - 1785, 1777, 1784, 128529, 128065, 128083, 128064, 128231, 9167, 127794, - 983180, 128523, 128561, 129312, 128531, 129301, 128567, 129488, 128582, - 128558, 129326, 128560, 129762, 129320, 128581, 129395, 129763, 129402, - 128539, 128540, 128541, 128514, 129298, 129323, 128580, 128548, 129764, - 129396, 128566, 129318, 128134, 128536, 129401, 8507, 127981, 10540, - 10543, 9950, 127810, 129478, 128439, 128224, 128106, 9771, 127877, - 129498, 129718, 128552, 170, 9792, 127905, 9972, 129338, 8210, 8199, - 128193, 128452, 983107, 128253, 127902, 128293, 128658, 129519, 127879, - 127878, 129512, 9789, 127771, 127763, 8296, 129351, 128031, 127907, 9673, - 127845, 128074, 8281, 11821, 127953, 10765, 129407, 129747, 9189, 9971, - 129449, 128170, 9884, 10086, 9880, 8277, 127924, 128190, 128563, 129672, - 129712, 129359, 128760, 128389, 127787, 127745, 129709, 128448, 129462, - 128099, 127860, 127869, 11792, 10972, 983071, 8704, 8873, 129376, 10021, - 11156, 8280, 8283, 10019, 127808, 10018, 128966, 8732, 8197, 9970, + 74137, 983267, 74138, 74139, 74140, 74141, 74142, 74607, 74127, 74144, + 74146, 74147, 74145, 74152, 74153, 74150, 74151, 74148, 74149, 74154, + 74155, 74157, 74158, 74163, 74164, 74165, 74160, 74161, 74156, 74159, + 74162, 74143, 74166, 74167, 74168, 74169, 74170, 74171, 74172, 74175, + 74173, 74174, 74176, 74177, 74182, 74183, 74180, 74181, 74632, 74186, + 74184, 74185, 74188, 74190, 74189, 74187, 74194, 74195, 74193, 74191, + 74192, 74196, 74197, 74198, 74199, 74200, 74201, 74202, 74205, 74206, + 74207, 74208, 74204, 74209, 74211, 74210, 74212, 74213, 74215, 74214, + 74216, 74218, 74217, 74964, 74178, 74179, 74203, 74219, 74220, 74223, + 74224, 74221, 74222, 74966, 74967, 74974, 74973, 74972, 74969, 74970, + 74971, 74975, 74968, 74965, 74977, 74976, 74980, 74981, 74982, 74984, + 74985, 74978, 74979, 74983, 74986, 74987, 74988, 74989, 74990, 74991, + 74993, 74997, 74996, 74998, 74995, 74994, 74992, 74999, 75000, 75003, + 75004, 75005, 75006, 75001, 75002, 75009, 75015, 75016, 75019, 75017, + 75018, 75013, 75012, 75010, 75011, 75014, 75021, 75030, 75029, 75027, + 75024, 75025, 75028, 75022, 75026, 75023, 75008, 75020, 75031, 75032, + 75007, 74226, 74227, 74228, 74229, 74230, 74225, 74231, 74233, 74234, + 74232, 74235, 74237, 74258, 74259, 75033, 74261, 74633, 74260, 74240, + 74634, 74241, 74243, 75035, 74246, 74247, 74245, 74248, 74249, 74250, + 74251, 75036, 75037, 74254, 74255, 74635, 74256, 75038, 74242, 74252, + 74253, 75034, 74238, 74239, 74244, 74257, 74263, 74265, 74264, 74266, + 74269, 74270, 74271, 74236, 74262, 74267, 74268, 74272, 74273, 74274, + 74278, 74279, 74275, 74276, 74277, 74280, 74281, 74636, 74282, 75039, + 74283, 74284, 74290, 74294, 74295, 75041, 75040, 74292, 74293, 74291, + 74296, 74297, 74298, 74299, 74300, 74637, 74301, 74286, 74287, 74285, + 74289, 74288, 74302, 74304, 74307, 74305, 74306, 74308, 74310, 74309, + 74311, 74303, 74638, 74312, 74314, 74313, 74315, 74316, 74319, 74321, + 74320, 74639, 74322, 74324, 74325, 74323, 74642, 75043, 74326, 75044, + 75046, 75047, 74327, 75048, 74329, 74328, 75049, 74330, 74332, 74333, + 74331, 75050, 75051, 74334, 75052, 74335, 75042, 74641, 75045, 74640, + 74317, 74336, 74318, 74337, 74338, 983266, 983265, 74643, 74339, 74347, + 74348, 74342, 74343, 74341, 74344, 74340, 74346, 74345, 74349, 74355, + 74354, 74350, 74352, 74358, 74359, 74353, 74357, 74351, 74356, 74360, + 74361, 74362, 74363, 74364, 74365, 74366, 74644, 74367, 74375, 74376, + 74368, 74369, 74373, 74374, 74370, 74371, 74372, 74377, 74378, 74379, + 74380, 74381, 74382, 74645, 74383, 74384, 74385, 74386, 74387, 74389, + 74408, 75053, 74388, 74646, 74395, 74394, 75055, 74400, 74399, 75056, + 74401, 74406, 74402, 74403, 74404, 74405, 74391, 74392, 74396, 74398, + 75054, 74397, 74393, 74390, 74407, 74409, 74410, 74411, 74412, 74413, + 74414, 74421, 74422, 74419, 74417, 74420, 74416, 74418, 74415, 74423, + 75057, 74424, 74425, 74426, 75058, 74428, 74429, 75059, 75060, 75061, + 74427, 74432, 74434, 74433, 74430, 74431, 74435, 74437, 74436, 74438, + 74441, 74440, 74444, 74445, 74446, 74448, 74447, 74443, 74449, 74442, + 74439, 74451, 74453, 74452, 74450, 74454, 74455, 74456, 74457, 75063, + 75062, 74458, 74459, 75064, 74460, 74461, 74462, 74463, 74465, 74464, + 74466, 74468, 74469, 74471, 74472, 74473, 74474, 74467, 74470, 74475, + 74477, 74478, 74479, 74476, 74480, 74481, 74482, 74483, 74488, 74486, + 74487, 74485, 74489, 74484, 74490, 75065, 74491, 74494, 74497, 74499, + 74500, 74498, 74495, 74647, 74496, 74501, 74504, 75066, 75067, 74505, + 74506, 74502, 74503, 74492, 74493, 74507, 74510, 74512, 74511, 74649, + 74509, 74508, 74515, 74516, 74522, 74523, 74519, 74520, 74517, 74518, + 74521, 74524, 74534, 74535, 74525, 74526, 74648, 74527, 74528, 74529, + 74531, 74532, 74533, 74530, 74536, 74538, 74537, 74539, 75068, 74540, + 74541, 74542, 74545, 74546, 74547, 75069, 74544, 74543, 74549, 74550, + 74551, 74552, 74553, 75070, 74555, 74556, 74558, 74557, 74559, 74560, + 74562, 74564, 74563, 75072, 74566, 75071, 74570, 74569, 74572, 74574, + 74573, 74554, 74567, 74571, 74565, 74561, 74568, 74575, 74576, 74548, + 74577, 74581, 74579, 74580, 74578, 74584, 74583, 74582, 74586, 74587, + 74588, 74585, 74513, 74514, 74589, 74591, 74590, 74593, 75073, 74592, + 74595, 74598, 74599, 74596, 74601, 74597, 74600, 74602, 75074, 74603, + 75075, 74604, 74605, 74606, 74594, 9982, 129380, 11232, 129473, 8911, + 8910, 10160, 9130, 129356, 128177, 164, 127835, 10081, 128707, 127854, + 129362, 129385, 67594, 67595, 67596, 67597, 67598, 67599, 67600, 67601, + 67602, 67603, 67604, 67605, 67606, 67607, 67608, 67609, 67610, 67611, + 67612, 67613, 67614, 67615, 67616, 67617, 67618, 67619, 67620, 67621, + 67622, 67623, 67624, 67625, 67626, 67627, 67628, 67629, 67630, 67631, + 67632, 67633, 67634, 67635, 67636, 67637, 67589, 67592, 67644, 67647, + 67639, 67640, 67584, 67585, 67586, 67587, 67588, 77712, 77713, 77714, + 77715, 77716, 77717, 77718, 77719, 77722, 77723, 77720, 77721, 77724, + 77725, 77726, 77727, 77728, 77729, 77730, 77731, 77732, 77733, 77734, + 77735, 77736, 77737, 77738, 77739, 77740, 77741, 77742, 77743, 77744, + 77745, 77746, 77747, 77748, 77749, 77750, 77751, 77752, 77753, 77754, + 77755, 77756, 77757, 77758, 77773, 77774, 77768, 77769, 77770, 77771, + 77772, 77775, 77776, 77777, 77788, 77789, 77790, 77791, 77792, 77793, + 77794, 77795, 77796, 77759, 77760, 77761, 77762, 77763, 77764, 77765, + 77766, 77767, 77778, 77779, 77780, 77781, 77782, 77783, 77784, 77785, + 77786, 77787, 77797, 77798, 77799, 77800, 77801, 77802, 77803, 77804, + 77805, 77806, 77807, 77808, 77809, 77810, 1126, 1033, 1300, 42574, 1034, + 1055, 1190, 1316, 1136, 1146, 42564, 42592, 42580, 1296, 1302, 1058, + 42634, 1196, 1035, 42640, 42638, 1062, 42642, 7305, 42636, 1059, 1266, + 1264, 1262, 1144, 1028, 1040, 1234, 1232, 1212, 1214, 1248, 1192, 1310, + 1256, 1258, 1184, 42602, 1130, 42572, 42586, 1030, 1041, 1063, 1268, + 1206, 1208, 42584, 42650, 42630, 1026, 42568, 42604, 42648, 1029, 42562, + 1322, 42632, 1039, 42626, 1324, 42624, 1044, 1069, 1051, 1312, 1326, + 1221, 1298, 1053, 1314, 1186, 1320, 1225, 1223, 1056, 1166, 1260, 1057, + 1194, 1052, 1229, 1060, 1043, 1172, 1168, 1270, 1170, 1274, 1027, 1061, + 1202, 1276, 1278, 1066, 42644, 1048, 1252, 1037, 1250, 1045, 1024, 1238, + 1025, 42588, 1128, 1132, 42578, 42582, 1124, 42566, 1140, 1142, 1050, + 1178, 1219, 1180, 1182, 1286, 1282, 1280, 1288, 1290, 1292, 1294, 1284, + 1152, 1227, 1036, 1134, 42600, 42570, 1054, 1120, 1148, 1254, 1150, 1240, + 1242, 1049, 1162, 1038, 1210, 1318, 1065, 42646, 1064, 42596, 42598, + 1068, 42594, 1198, 1200, 1164, 1071, 1304, 1122, 1067, 1272, 42576, 1031, + 42590, 1070, 1047, 1246, 1176, 42560, 1046, 1244, 1174, 1217, 42628, + 1138, 1032, 1042, 1308, 1306, 1236, 1204, 1188, 42622, 42606, 1216, 7467, + 42623, 1072, 1235, 1233, 1213, 1215, 1249, 1193, 1311, 1257, 1259, 1185, + 42603, 1131, 42573, 42587, 1110, 1073, 1095, 1269, 1207, 1209, 42585, + 42651, 42631, 1106, 42569, 42605, 42649, 1109, 42563, 1323, 42633, 1119, + 42627, 1325, 42625, 1076, 1101, 1083, 1313, 1327, 1222, 1299, 1085, 1315, + 1187, 1321, 1226, 1224, 1088, 1167, 1261, 1089, 1195, 1084, 1230, 1092, + 1075, 1173, 1169, 1271, 1171, 1275, 1107, 1093, 1203, 1277, 1279, 1098, + 42645, 1080, 1253, 1117, 1251, 1077, 1104, 1239, 1105, 42589, 1129, 1133, + 42579, 42583, 1125, 42567, 1141, 1143, 1082, 1179, 1220, 1181, 1183, + 1287, 1283, 1281, 1289, 1291, 1293, 1295, 1285, 1153, 1228, 1116, 1135, + 1127, 7297, 1113, 1301, 42601, 42571, 42575, 7298, 1114, 1086, 1121, + 1149, 1255, 1151, 1087, 1191, 1317, 1231, 1137, 42565, 42593, 42581, + 1297, 1147, 7296, 1303, 1241, 1243, 1081, 1163, 1118, 1211, 1319, 1097, + 42647, 1096, 42597, 42599, 1100, 42595, 1199, 1201, 1165, 7302, 7303, + 7300, 1090, 42635, 1197, 1115, 42641, 42639, 1094, 7301, 42643, 7306, + 42637, 1091, 1267, 1265, 1263, 1145, 1108, 7304, 7299, 1309, 1103, 1305, + 1123, 1099, 1273, 42577, 1111, 42591, 1102, 1079, 1247, 1177, 42561, + 1078, 1245, 1175, 1218, 42629, 1139, 1112, 1074, 1307, 1237, 1205, 1189, + 122984, 122962, 122986, 122985, 122965, 122976, 122971, 122974, 122964, + 122983, 122977, 122981, 122982, 122980, 122978, 122967, 122968, 122969, + 122966, 122979, 122973, 122963, 122970, 122961, 122972, 122975, 1154, + 9005, 127744, 983201, 983188, 983172, 8224, 11830, 11831, 128481, 128131, + 127841, 128374, 9619, 11843, 128168, 10143, 65101, 65097, 8504, 983081, + 983084, 983086, 983088, 983090, 983162, 9110, 9192, 127795, 128475, 8451, + 176, 8457, 983120, 128666, 8796, 983119, 9161, 9159, 9153, 9156, 9162, + 9160, 9154, 9157, 9164, 9151, 9163, 9150, 9158, 9152, 9155, 117829, + 117828, 127980, 9739, 66589, 66591, 66596, 66597, 66587, 66585, 66594, + 66595, 66593, 66599, 66562, 66563, 66564, 66565, 66561, 66560, 66568, + 66569, 66570, 66571, 66567, 66566, 66598, 66573, 66588, 66579, 66592, + 66590, 66581, 66578, 66580, 66582, 66577, 66586, 66575, 66584, 66583, + 66574, 66572, 66576, 66629, 66631, 66636, 66637, 66627, 66625, 66634, + 66635, 66633, 66639, 66602, 66603, 66604, 66605, 66601, 66600, 66608, + 66609, 66610, 66611, 66607, 66606, 66638, 66613, 66628, 66619, 66632, + 66630, 66621, 66618, 66620, 66622, 66617, 66626, 66615, 66624, 66623, + 66614, 66612, 66616, 127964, 127965, 128421, 128468, 2388, 2416, 43258, + 2387, 43257, 72448, 72449, 43259, 2309, 2310, 2320, 2324, 2421, 43262, + 2330, 2418, 2317, 2321, 2331, 2396, 2430, 2338, 2337, 2343, 2342, 2429, + 2394, 2328, 2427, 2327, 2426, 2361, 2350, 2424, 2308, 2318, 2322, 2358, + 2359, 2360, 2323, 2420, 2419, 2431, 2349, 2348, 2393, 2326, 2325, 2397, + 2353, 2352, 2399, 2351, 2313, 2314, 2423, 2422, 2345, 2339, 2329, 2334, + 2344, 2333, 2428, 2332, 2356, 2355, 2354, 2336, 2335, 2341, 2340, 2315, + 2400, 2316, 2401, 2357, 2311, 2312, 2347, 2346, 2425, 2395, 2398, 2392, + 2319, 983644, 983643, 983646, 983647, 983649, 983648, 983642, 983645, + 72450, 72451, 72452, 72453, 2305, 43255, 43251, 43254, 43253, 72455, + 72454, 72456, 43250, 43260, 2304, 72457, 2364, 2365, 2306, 43252, 43256, + 2417, 2381, 2307, 2386, 2385, 2366, 2376, 2380, 2383, 43263, 2389, 2373, + 2377, 2379, 2363, 2362, 2382, 2369, 2370, 2391, 2390, 2374, 2378, 2371, + 2372, 2402, 2403, 2367, 2368, 2375, 2405, 2404, 2411, 2410, 2413, 2412, + 2409, 2408, 2406, 2415, 2407, 2414, 43261, 2384, 983089, 983161, 983087, + 983085, 983083, 127962, 129487, 129420, 11033, 11032, 11030, 11031, + 128924, 128160, 8900, 8960, 168, 117827, 9856, 9857, 9858, 9859, 9860, + 9861, 128754, 53, 127238, 9356, 52, 127237, 9355, 57, 127242, 9360, 49, + 127234, 9352, 55, 127240, 9358, 54, 127239, 9357, 51, 127236, 9354, 50, + 127235, 9353, 48, 127233, 127232, 56, 127241, 9359, 119557, 119556, + 119555, 9868, 9871, 9870, 9869, 119553, 119554, 10131, 10126, 10125, + 10128, 10127, 10124, 10123, 127244, 10130, 10122, 10129, 10111, 10106, + 10105, 10108, 10107, 10104, 10103, 10110, 10102, 10109, 10121, 10116, + 10115, 10118, 10117, 10114, 10113, 127243, 10120, 10112, 10119, 9107, + 127919, 128549, 128542, 9933, 9090, 129400, 72004, 72021, 72020, 72023, + 72022, 72019, 72018, 72016, 72025, 72017, 72024, 71964, 71958, 71963, + 71974, 71973, 71936, 71937, 71961, 71960, 71966, 71965, 71940, 71941, + 71938, 71939, 71982, 71976, 71952, 71962, 71957, 71967, 71978, 71979, + 71980, 71971, 71970, 71954, 71953, 71951, 71950, 71949, 71948, 71969, + 71968, 71981, 71955, 71972, 71975, 71977, 71983, 71942, 71945, 71997, + 71996, 72003, 71995, 71984, 71991, 71987, 71988, 71985, 71986, 71989, + 71992, 71998, 72002, 72000, 72005, 72006, 71999, 72001, 8725, 247, 8903, + 129343, 8739, 9902, 129684, 128171, 128565, 12291, 8783, 9009, 128462, + 128441, 128442, 128443, 8939, 8941, 8716, 8740, 10990, 8832, 8928, 8876, + 8833, 8929, 8878, 128021, 71680, 71681, 71687, 71689, 71703, 71702, + 71708, 71707, 71723, 71716, 71701, 71700, 71706, 71705, 71684, 71685, + 71682, 71683, 71694, 71704, 71699, 71709, 71719, 71720, 71721, 71713, + 71712, 71696, 71695, 71693, 71692, 71698, 71697, 71691, 71690, 71711, + 71710, 71722, 71717, 71714, 71718, 71715, 71686, 71688, 71729, 71730, + 71724, 71732, 71734, 71727, 71728, 71725, 71726, 71731, 71733, 71739, + 71738, 71735, 71737, 71736, 128054, 128044, 36, 127025, 127026, 127027, + 127028, 127029, 127030, 127031, 127032, 127033, 127034, 127035, 127036, + 127037, 127038, 127039, 127040, 127041, 127042, 127043, 127044, 127045, + 127046, 127047, 127048, 127049, 127050, 127051, 127052, 127053, 127054, + 127055, 127056, 127057, 127058, 127059, 127060, 127061, 127062, 127063, + 127064, 127065, 127066, 127067, 127068, 127069, 127070, 127071, 127072, + 127073, 127024, 127075, 127076, 127077, 127078, 127079, 127080, 127081, + 127082, 127083, 127084, 127085, 127086, 127087, 127088, 127089, 127090, + 127091, 127092, 127093, 127094, 127095, 127096, 127097, 127098, 127099, + 127100, 127101, 127102, 127103, 127104, 127105, 127106, 127107, 127108, + 127109, 127110, 127111, 127112, 127113, 127114, 127115, 127116, 127117, + 127118, 127119, 127120, 127121, 127122, 127123, 127074, 129743, 8363, + 8724, 8760, 8901, 729, 9676, 8284, 11850, 11034, 129765, 11795, 11784, + 10649, 11798, 9470, 9465, 9464, 9467, 9466, 9463, 9462, 9469, 9461, 9468, + 10175, 10868, 10986, 8225, 8223, 11840, 8914, 8748, 11842, 8222, 8215, + 10835, 10836, 10645, 10913, 10915, 10914, 10939, 8243, 12318, 10746, + 10830, 10831, 11849, 10988, 11844, 10940, 8913, 8912, 11005, 10987, 8915, + 9208, 10981, 8875, 10979, 8214, 11799, 10646, 733, 8252, 8263, 65100, + 10908, 10907, 11002, 11001, 10906, 10905, 8510, 8473, 8511, 8450, 8461, + 8469, 8474, 8477, 8484, 8518, 8519, 8520, 8521, 8517, 8509, 8508, 8512, + 10719, 9890, 9891, 11260, 127849, 8868, 10993, 10623, 8945, 8964, 117764, + 128317, 117859, 118264, 117883, 118268, 117849, 128315, 117914, 117864, + 10728, 10729, 117875, 117879, 129288, 129290, 129289, 129291, 8595, 8629, + 8671, 129035, 129031, 129179, 129043, 129027, 129047, 8626, 8627, 10504, + 8693, 129975, 8615, 10515, 11796, 11015, 129203, 11147, 11107, 11139, + 11123, 129067, 11168, 11169, 129063, 129059, 129075, 129071, 11133, + 11085, 11117, 11143, 129171, 10507, 8609, 11247, 8681, 129175, 8623, + 8659, 8675, 129079, 10597, 10607, 10593, 10585, 8643, 10589, 10581, 8642, + 129091, 129095, 129087, 10225, 128623, 129107, 129083, 8650, 128687, + 128330, 128682, 129444, 8367, 10139, 128009, 128050, 128042, 129656, + 128167, 129316, 129345, 9946, 128087, 113784, 113788, 113785, 113783, + 113782, 113786, 113787, 113795, 113798, 113793, 113800, 113781, 113794, + 113792, 113799, 113797, 113796, 113776, 113808, 113817, 113811, 113814, + 113809, 113816, 113779, 113810, 113815, 113813, 113812, 113778, 113777, + 113780, 113729, 113733, 113672, 113677, 113683, 113735, 113739, 113746, + 113668, 113678, 113674, 113726, 113691, 113699, 113700, 113695, 113709, + 113712, 113713, 113705, 113711, 113669, 113725, 113679, 113684, 113670, + 113743, 113749, 113687, 113689, 113693, 113707, 113697, 113703, 113690, + 113694, 113708, 113698, 113704, 113764, 113763, 113762, 113761, 113732, + 113753, 113731, 113755, 113754, 113666, 113766, 113765, 113676, 113675, + 113741, 113750, 113680, 113688, 113692, 113696, 113710, 113727, 113728, + 113716, 113717, 113714, 113715, 113701, 113702, 113724, 113723, 113706, + 113742, 113740, 113767, 113769, 113730, 113768, 113682, 113685, 113752, + 113737, 113667, 113719, 113718, 113681, 113745, 113748, 113751, 113738, + 113673, 113770, 113720, 113757, 113760, 113722, 113759, 113756, 113721, + 113758, 113665, 113747, 113664, 113686, 113734, 113736, 113744, 113671, + 113821, 113822, 113823, 113820, 129375, 129414, 129516, 128192, 983082, + 128066, 127805, 127806, 129467, 9793, 127758, 127759, 127757, 9178, 9841, + 129413, 11790, 77830, 77831, 77832, 77828, 77829, 77824, 77825, 77826, + 77827, 77833, 77834, 77835, 77840, 77841, 77844, 77845, 77836, 77837, + 77838, 77839, 77842, 77843, 77846, 77847, 77869, 77870, 77872, 77873, + 77874, 77875, 77877, 77878, 77871, 77876, 77879, 77880, 77881, 77882, + 77860, 77861, 77858, 77859, 77862, 77863, 77864, 77865, 77866, 77867, + 77868, 77903, 77848, 77849, 77850, 77851, 77852, 77853, 77854, 77855, + 77856, 77857, 77883, 77884, 77885, 77886, 77887, 77888, 77889, 77890, + 77891, 77892, 77893, 77894, 77895, 77896, 77897, 77898, 77899, 77900, + 77901, 77902, 78867, 78868, 78869, 78861, 78862, 78863, 78864, 78865, + 78866, 78870, 78871, 78892, 78893, 78894, 78872, 78873, 78874, 78875, + 78876, 78877, 78878, 78879, 78880, 78881, 78882, 78883, 78884, 78885, + 78886, 78887, 78888, 78889, 78890, 78891, 78910, 78908, 78903, 77908, + 77909, 77904, 77905, 77906, 77907, 77910, 77911, 77912, 77913, 77915, + 77916, 77917, 77918, 77914, 77919, 77920, 77921, 77922, 77923, 77924, + 77925, 77926, 77927, 77928, 77929, 77930, 77931, 77932, 77933, 77934, + 77935, 77936, 77937, 77938, 77939, 77940, 77941, 77949, 77950, 77942, + 77943, 77944, 77945, 77946, 77947, 77948, 77951, 77974, 77975, 77978, + 77979, 77973, 77976, 77977, 77980, 77981, 77982, 77983, 77984, 77991, + 77992, 77994, 77995, 77985, 77986, 77987, 77988, 77989, 77990, 77993, + 77996, 77997, 77998, 77999, 78000, 78001, 78002, 78003, 78004, 78005, + 78006, 78008, 78009, 78011, 78012, 78007, 78010, 78013, 78014, 78015, + 78016, 78017, 78025, 78026, 78027, 78028, 78029, 78030, 78031, 78032, + 78033, 78018, 78019, 78020, 78021, 78022, 78023, 78024, 77969, 77970, + 77962, 77963, 77964, 77965, 77966, 77967, 77968, 77971, 77972, 77952, + 77953, 77954, 77955, 77956, 77957, 77958, 77959, 77960, 77961, 78041, + 78042, 78043, 78044, 78034, 78035, 78036, 78037, 78038, 78039, 78040, + 78057, 78058, 78066, 78067, 78059, 78060, 78061, 78062, 78063, 78064, + 78065, 78068, 78073, 78074, 78069, 78070, 78071, 78072, 78075, 78076, + 78077, 78051, 78052, 78053, 78054, 78045, 78046, 78047, 78048, 78049, + 78050, 78055, 78056, 78911, 78909, 78904, 78078, 78079, 78080, 78081, + 78082, 78083, 78084, 78085, 78086, 78087, 78091, 78092, 78088, 78089, + 78090, 78093, 78094, 78095, 78096, 78097, 78098, 78111, 78112, 78118, + 78119, 78120, 78121, 78110, 78113, 78114, 78115, 78116, 78117, 78122, + 78128, 78129, 78130, 78131, 78132, 78133, 78123, 78124, 78125, 78126, + 78127, 78134, 78135, 78137, 78138, 78139, 78140, 78136, 78141, 78142, + 78100, 78101, 78099, 78102, 78103, 78104, 78105, 78106, 78107, 78108, + 78109, 78913, 78150, 78151, 78152, 78148, 78149, 78143, 78144, 78145, + 78146, 78147, 78153, 78154, 78156, 78157, 78155, 78158, 78159, 78160, + 78161, 78162, 78163, 78164, 78165, 78184, 78185, 78186, 78187, 78178, + 78179, 78180, 78181, 78182, 78183, 78188, 78189, 78193, 78194, 78196, + 78197, 78190, 78191, 78192, 78195, 78198, 78199, 78200, 78201, 78166, + 78167, 78173, 78174, 78168, 78169, 78170, 78171, 78172, 78175, 78176, + 78177, 78202, 78203, 78204, 78205, 78206, 78212, 78213, 78207, 78208, + 78209, 78210, 78211, 78214, 78215, 78914, 78916, 78897, 78220, 78221, + 78225, 78226, 78216, 78217, 78218, 78219, 78222, 78223, 78224, 78227, + 78228, 78229, 78230, 78231, 78232, 78233, 78234, 78907, 78901, 78899, + 78906, 78900, 78898, 78905, 78244, 78245, 78249, 78250, 78243, 78246, + 78247, 78248, 78251, 78252, 78915, 78253, 78254, 78255, 78257, 78258, + 78256, 78259, 78260, 78261, 78262, 78263, 78264, 78268, 78269, 78270, + 78271, 78272, 78273, 78274, 78275, 78276, 78265, 78266, 78279, 78280, + 78281, 78282, 78283, 78284, 78267, 78277, 78278, 78285, 78286, 78289, + 78290, 78292, 78293, 78297, 78298, 78287, 78288, 78291, 78294, 78295, + 78296, 78299, 78304, 78305, 78306, 78301, 78302, 78300, 78303, 78307, + 78308, 78309, 78310, 78311, 78312, 78313, 78314, 78315, 78316, 78317, + 78318, 78933, 78928, 78920, 78924, 78932, 78926, 78921, 78929, 78925, + 78923, 78931, 78919, 78927, 78922, 78930, 78912, 78336, 78337, 78338, + 78328, 78329, 78330, 78331, 78332, 78333, 78334, 78335, 78339, 78354, + 78355, 78356, 78357, 78358, 78359, 78361, 78362, 78351, 78352, 78353, + 78360, 78363, 78364, 78345, 78346, 78340, 78341, 78342, 78343, 78344, + 78347, 78348, 78349, 78350, 78365, 78366, 78367, 78319, 78320, 78321, + 78322, 78323, 78324, 78325, 78326, 78327, 78372, 78373, 78368, 78369, + 78370, 78371, 78374, 78375, 78376, 78377, 78385, 78386, 78378, 78379, + 78380, 78381, 78382, 78383, 78384, 78387, 78388, 78389, 78399, 78400, + 78401, 78402, 78409, 78410, 78403, 78404, 78405, 78406, 78407, 78408, + 78411, 78414, 78415, 78412, 78413, 78390, 78391, 78392, 78393, 78394, + 78395, 78396, 78397, 78398, 78423, 78424, 78425, 78426, 78427, 78428, + 78429, 78416, 78417, 78421, 78422, 78418, 78419, 78420, 78430, 78431, + 78432, 78433, 78434, 78435, 78436, 78445, 78446, 78437, 78438, 78439, + 78440, 78441, 78442, 78443, 78444, 78447, 78448, 78452, 78453, 78454, + 78455, 78459, 78460, 78449, 78450, 78451, 78456, 78457, 78458, 78469, + 78470, 78471, 78472, 78473, 78461, 78462, 78465, 78466, 78463, 78464, + 78467, 78468, 78474, 78475, 78476, 78487, 78488, 78489, 78490, 78477, + 78478, 78479, 78480, 78481, 78482, 78483, 78484, 78485, 78486, 78902, + 78491, 78492, 78494, 78495, 78493, 78496, 78497, 78498, 78499, 78500, + 78501, 78502, 78503, 78514, 78515, 78516, 78512, 78513, 78511, 78517, + 78518, 78519, 78520, 78521, 78522, 78523, 78524, 78530, 78531, 78525, + 78526, 78527, 78528, 78529, 78532, 78533, 78534, 78535, 78536, 78537, + 78538, 78539, 78540, 78541, 78542, 78543, 78544, 78546, 78547, 78551, + 78552, 78545, 78548, 78549, 78550, 78553, 78554, 78555, 78560, 78561, + 78562, 78565, 78566, 78556, 78557, 78558, 78559, 78563, 78564, 78567, + 78568, 78575, 78576, 78577, 78569, 78570, 78571, 78572, 78573, 78574, + 78578, 78579, 78580, 78586, 78587, 78581, 78582, 78583, 78584, 78585, + 78588, 78589, 78590, 78591, 78592, 78593, 78594, 78595, 78596, 78597, + 78598, 78601, 78602, 78606, 78607, 78608, 78609, 78610, 78611, 78599, + 78600, 78603, 78604, 78605, 78613, 78614, 78619, 78620, 78612, 78615, + 78616, 78617, 78618, 78621, 78622, 78623, 78636, 78637, 78638, 78639, + 78634, 78635, 78640, 78641, 78642, 78624, 78625, 78626, 78627, 78628, + 78629, 78630, 78631, 78632, 78633, 78917, 78648, 78649, 78650, 78643, + 78644, 78645, 78646, 78647, 78651, 78652, 78653, 78667, 78668, 78674, + 78675, 78664, 78665, 78666, 78669, 78670, 78671, 78672, 78673, 78678, + 78679, 78676, 78677, 78680, 78681, 78682, 78683, 78684, 78685, 78686, + 78687, 78688, 78689, 78654, 78655, 78656, 78657, 78658, 78659, 78660, + 78661, 78662, 78663, 78706, 78707, 78708, 78690, 78691, 78692, 78693, + 78694, 78695, 78696, 78697, 78698, 78699, 78700, 78701, 78702, 78703, + 78704, 78705, 78709, 78710, 78712, 78713, 78714, 78715, 78895, 78716, + 78717, 78718, 78711, 78719, 78720, 78721, 78722, 78723, 78724, 78725, + 78726, 78727, 78728, 78729, 78730, 78731, 78732, 78733, 78734, 78735, + 78736, 78737, 78738, 78741, 78742, 78747, 78748, 78749, 78750, 78739, + 78740, 78743, 78744, 78745, 78746, 78751, 78752, 78753, 78754, 78756, + 78757, 78761, 78762, 78755, 78758, 78759, 78760, 78763, 78764, 78765, + 78766, 78896, 78769, 78770, 78776, 78777, 78767, 78768, 78771, 78772, + 78773, 78774, 78775, 78778, 78779, 78783, 78784, 78787, 78788, 78789, + 78790, 78780, 78781, 78782, 78785, 78786, 78791, 78796, 78797, 78792, + 78793, 78794, 78795, 78798, 78918, 78802, 78803, 78804, 78806, 78807, + 78809, 78810, 78799, 78800, 78801, 78805, 78808, 78811, 78812, 78813, + 78814, 78815, 78816, 78817, 78818, 78819, 78821, 78822, 78823, 78824, + 78825, 78826, 78827, 78828, 78829, 78830, 78831, 78832, 78820, 78833, + 78834, 78835, 78836, 78842, 78843, 78844, 78845, 78846, 78847, 78848, + 78849, 78850, 78851, 78852, 78853, 78854, 78855, 78856, 78857, 78858, + 78859, 78860, 78837, 78838, 78839, 78840, 78841, 78235, 78236, 78237, + 78238, 78239, 78240, 78241, 78242, 78504, 78505, 78506, 78507, 78508, + 78509, 78510, 78944, 78945, 78946, 78947, 78948, 78949, 78950, 78951, + 78952, 78953, 78954, 78955, 78956, 78957, 78958, 78959, 78960, 78961, + 78962, 78963, 78964, 78965, 78966, 78967, 78968, 78969, 78970, 78971, + 78972, 78973, 78974, 78975, 78976, 78977, 78978, 78979, 78980, 78981, + 78982, 78983, 78984, 78985, 78986, 78987, 78988, 78989, 78990, 78991, + 78992, 78993, 78994, 78995, 78996, 78997, 78998, 78999, 79000, 79001, + 79002, 79003, 79004, 79005, 79006, 79007, 79008, 79009, 79010, 79011, + 79012, 79013, 79014, 79015, 79016, 79017, 79018, 79019, 79020, 79021, + 79022, 79023, 79024, 79025, 79026, 79027, 79028, 79029, 79030, 79031, + 79032, 79033, 79034, 79035, 79036, 79037, 79038, 79039, 79040, 79041, + 79042, 79043, 79044, 79045, 79046, 79047, 79048, 79049, 79050, 79051, + 79052, 79053, 79054, 79055, 79056, 79057, 79058, 79059, 79060, 79061, + 79062, 79063, 79064, 79065, 79066, 79067, 79068, 79069, 79070, 79071, + 79072, 79073, 79074, 79075, 79076, 79077, 79078, 79079, 79080, 79081, + 79082, 79083, 79084, 79085, 79086, 79087, 79088, 79089, 79090, 79091, + 79092, 79093, 79094, 79095, 79096, 79097, 79098, 79099, 79100, 79101, + 79102, 79103, 79104, 79105, 79106, 79107, 79108, 79109, 79110, 79111, + 79112, 79113, 79114, 79115, 79116, 79117, 79118, 79119, 79120, 79121, + 79122, 79123, 79124, 79125, 79126, 79127, 79128, 79129, 79130, 79131, + 79132, 79133, 79134, 79135, 79136, 79137, 79138, 79139, 79140, 79141, + 79142, 79143, 79144, 79145, 79146, 79147, 79148, 79149, 79150, 79151, + 79152, 79153, 79154, 79155, 79156, 79157, 79158, 79159, 79160, 79161, + 79162, 79163, 79164, 79165, 79166, 79167, 79168, 79169, 79170, 79171, + 79172, 79173, 79174, 79175, 79176, 79177, 79178, 79179, 79180, 79181, + 79182, 79183, 79184, 79185, 79186, 79187, 79188, 79189, 79190, 79191, + 79192, 79193, 79194, 79195, 79196, 79197, 79198, 79199, 79200, 79201, + 79202, 79203, 79204, 79205, 79206, 79207, 79208, 79209, 79210, 79211, + 79212, 79213, 79214, 79215, 79216, 79217, 79218, 79219, 79220, 79221, + 79222, 79223, 79224, 79225, 79226, 79227, 79228, 79229, 79230, 79231, + 79232, 79233, 79234, 79235, 79236, 79237, 79238, 79239, 79240, 79241, + 79242, 79243, 79244, 79245, 79246, 79247, 79248, 79249, 79250, 79251, + 79252, 79253, 79254, 79255, 79256, 79257, 79258, 79259, 79260, 79261, + 79262, 79263, 79264, 79265, 79266, 79267, 79268, 79269, 79270, 79271, + 79272, 79273, 79274, 79275, 79276, 79277, 79278, 79279, 79280, 79281, + 79282, 79283, 79284, 79285, 79286, 79287, 79288, 79289, 79290, 79291, + 79292, 79293, 79294, 79295, 79296, 79297, 79298, 79299, 79300, 79301, + 79302, 79303, 79304, 79305, 79306, 79307, 79308, 79309, 79310, 79311, + 79312, 79313, 79314, 79315, 79316, 79317, 79318, 79319, 79320, 79321, + 79322, 79323, 79324, 79325, 79326, 79327, 79328, 79329, 79330, 79331, + 79332, 79333, 79334, 79335, 79336, 79337, 79338, 79339, 79340, 79341, + 79342, 79343, 79344, 79345, 79346, 79347, 79348, 79349, 79350, 79351, + 79352, 79353, 79354, 79355, 79356, 79357, 79358, 79359, 79360, 79361, + 79362, 79363, 79364, 79365, 79366, 79367, 79368, 79369, 79370, 79371, + 79372, 79373, 79374, 79375, 79376, 79377, 79378, 79379, 79380, 79381, + 79382, 79383, 79384, 79385, 79386, 79387, 79388, 79389, 79390, 79391, + 79392, 79393, 79394, 79395, 79396, 79397, 79398, 79399, 79400, 79401, + 79402, 79403, 79404, 79405, 79406, 79407, 79408, 79409, 79410, 79411, + 79412, 79413, 79414, 79415, 79416, 79417, 79418, 79419, 79420, 79421, + 79422, 79423, 79424, 79425, 79426, 79427, 79428, 79429, 79430, 79431, + 79432, 79433, 79434, 79435, 79436, 79437, 79438, 79439, 79440, 79441, + 79442, 79443, 79444, 79445, 79446, 79447, 79448, 79449, 79450, 79451, + 79452, 79453, 79454, 79455, 79456, 79457, 79458, 79459, 79460, 79461, + 79462, 79463, 79464, 79465, 79466, 79467, 79468, 79469, 79470, 79471, + 79472, 79473, 79474, 79475, 79476, 79477, 79478, 79479, 79480, 79481, + 79482, 79483, 79484, 79485, 79486, 79487, 79488, 79489, 79490, 79491, + 79492, 79493, 79494, 79495, 79496, 79497, 79498, 79499, 79500, 79501, + 79502, 79503, 79504, 79505, 79506, 79507, 79508, 79509, 79510, 79511, + 79512, 79513, 79514, 79515, 79516, 79517, 79518, 79519, 79520, 79521, + 79522, 79523, 79524, 79525, 79526, 79527, 79528, 79529, 79530, 79531, + 79532, 79533, 79534, 79535, 79536, 79537, 79538, 79539, 79540, 79541, + 79542, 79543, 79544, 79545, 79546, 79547, 79548, 79549, 79550, 79551, + 79552, 79553, 79554, 79555, 79556, 79557, 79558, 79559, 79560, 79561, + 79562, 79563, 79564, 79565, 79566, 79567, 79568, 79569, 79570, 79571, + 79572, 79573, 79574, 79575, 79576, 79577, 79578, 79579, 79580, 79581, + 79582, 79583, 79584, 79585, 79586, 79587, 79588, 79589, 79590, 79591, + 79592, 79593, 79594, 79595, 79596, 79597, 79598, 79599, 79600, 79601, + 79602, 79603, 79604, 79605, 79606, 79607, 79608, 79609, 79610, 79611, + 79612, 79613, 79614, 79615, 79616, 79617, 79618, 79619, 79620, 79621, + 79622, 79623, 79624, 79625, 79626, 79627, 79628, 79629, 79630, 79631, + 79632, 79633, 79634, 79635, 79636, 79637, 79638, 79639, 79640, 79641, + 79642, 79643, 79644, 79645, 79646, 79647, 79648, 79649, 79650, 79651, + 79652, 79653, 79654, 79655, 79656, 79657, 79658, 79659, 79660, 79661, + 79662, 79663, 79664, 79665, 79666, 79667, 79668, 79669, 79670, 79671, + 79672, 79673, 79674, 79675, 79676, 79677, 79678, 79679, 79680, 79681, + 79682, 79683, 79684, 79685, 79686, 79687, 79688, 79689, 79690, 79691, + 79692, 79693, 79694, 79695, 79696, 79697, 79698, 79699, 79700, 79701, + 79702, 79703, 79704, 79705, 79706, 79707, 79708, 79709, 79710, 79711, + 79712, 79713, 79714, 79715, 79716, 79717, 79718, 79719, 79720, 79721, + 79722, 79723, 79724, 79725, 79726, 79727, 79728, 79729, 79730, 79731, + 79732, 79733, 79734, 79735, 79736, 79737, 79738, 79739, 79740, 79741, + 79742, 79743, 79744, 79745, 79746, 79747, 79748, 79749, 79750, 79751, + 79752, 79753, 79754, 79755, 79756, 79757, 79758, 79759, 79760, 79761, + 79762, 79763, 79764, 79765, 79766, 79767, 79768, 79769, 79770, 79771, + 79772, 79773, 79774, 79775, 79776, 79777, 79778, 79779, 79780, 79781, + 79782, 79783, 79784, 79785, 79786, 79787, 79788, 79789, 79790, 79791, + 79792, 79793, 79794, 79795, 79796, 79797, 79798, 79799, 79800, 79801, + 79802, 79803, 79804, 79805, 79806, 79807, 79808, 79809, 79810, 79811, + 79812, 79813, 79814, 79815, 79816, 79817, 79818, 79819, 79820, 79821, + 79822, 79823, 79824, 79825, 79826, 79827, 79828, 79829, 79830, 79831, + 79832, 79833, 79834, 79835, 79836, 79837, 79838, 79839, 79840, 79841, + 79842, 79843, 79844, 79845, 79846, 79847, 79848, 79849, 79850, 79851, + 79852, 79853, 79854, 79855, 79856, 79857, 79858, 79859, 79860, 79861, + 79862, 79863, 79864, 79865, 79866, 79867, 79868, 79869, 79870, 79871, + 79872, 79873, 79874, 79875, 79876, 79877, 79878, 79879, 79880, 79881, + 79882, 79883, 79884, 79885, 79886, 79887, 79888, 79889, 79890, 79891, + 79892, 79893, 79894, 79895, 79896, 79897, 79898, 79899, 79900, 79901, + 79902, 79903, 79904, 79905, 79906, 79907, 79908, 79909, 79910, 79911, + 79912, 79913, 79914, 79915, 79916, 79917, 79918, 79919, 79920, 79921, + 79922, 79923, 79924, 79925, 79926, 79927, 79928, 79929, 79930, 79931, + 79932, 79933, 79934, 79935, 79936, 79937, 79938, 79939, 79940, 79941, + 79942, 79943, 79944, 79945, 79946, 79947, 79948, 79949, 79950, 79951, + 79952, 79953, 79954, 79955, 79956, 79957, 79958, 79959, 79960, 79961, + 79962, 79963, 79964, 79965, 79966, 79967, 79968, 79969, 79970, 79971, + 79972, 79973, 79974, 79975, 79976, 79977, 79978, 79979, 79980, 79981, + 79982, 79983, 79984, 79985, 79986, 79987, 79988, 79989, 79990, 79991, + 79992, 79993, 79994, 79995, 79996, 79997, 79998, 79999, 80000, 80001, + 80002, 80003, 80004, 80005, 80006, 80007, 80008, 80009, 80010, 80011, + 80012, 80013, 80014, 80015, 80016, 80017, 80018, 80019, 80020, 80021, + 80022, 80023, 80024, 80025, 80026, 80027, 80028, 80029, 80030, 80031, + 80032, 80033, 80034, 80035, 80036, 80037, 80038, 80039, 80040, 80041, + 80042, 80043, 80044, 80045, 80046, 80047, 80048, 80049, 80050, 80051, + 80052, 80053, 80054, 80055, 80056, 80057, 80058, 80059, 80060, 80061, + 80062, 80063, 80064, 80065, 80066, 80067, 80068, 80069, 80070, 80071, + 80072, 80073, 80074, 80075, 80076, 80077, 80078, 80079, 80080, 80081, + 80082, 80083, 80084, 80085, 80086, 80087, 80088, 80089, 80090, 80091, + 80092, 80093, 80094, 80095, 80096, 80097, 80098, 80099, 80100, 80101, + 80102, 80103, 80104, 80105, 80106, 80107, 80108, 80109, 80110, 80111, + 80112, 80113, 80114, 80115, 80116, 80117, 80118, 80119, 80120, 80121, + 80122, 80123, 80124, 80125, 80126, 80127, 80128, 80129, 80130, 80131, + 80132, 80133, 80134, 80135, 80136, 80137, 80138, 80139, 80140, 80141, + 80142, 80143, 80144, 80145, 80146, 80147, 80148, 80149, 80150, 80151, + 80152, 80153, 80154, 80155, 80156, 80157, 80158, 80159, 80160, 80161, + 80162, 80163, 80164, 80165, 80166, 80167, 80168, 80169, 80170, 80171, + 80172, 80173, 80174, 80175, 80176, 80177, 80178, 80179, 80180, 80181, + 80182, 80183, 80184, 80185, 80186, 80187, 80188, 80189, 80190, 80191, + 80192, 80193, 80194, 80195, 80196, 80197, 80198, 80199, 80200, 80201, + 80202, 80203, 80204, 80205, 80206, 80207, 80208, 80209, 80210, 80211, + 80212, 80213, 80214, 80215, 80216, 80217, 80218, 80219, 80220, 80221, + 80222, 80223, 80224, 80225, 80226, 80227, 80228, 80229, 80230, 80231, + 80232, 80233, 80234, 80235, 80236, 80237, 80238, 80239, 80240, 80241, + 80242, 80243, 80244, 80245, 80246, 80247, 80248, 80249, 80250, 80251, + 80252, 80253, 80254, 80255, 80256, 80257, 80258, 80259, 80260, 80261, + 80262, 80263, 80264, 80265, 80266, 80267, 80268, 80269, 80270, 80271, + 80272, 80273, 80274, 80275, 80276, 80277, 80278, 80279, 80280, 80281, + 80282, 80283, 80284, 80285, 80286, 80287, 80288, 80289, 80290, 80291, + 80292, 80293, 80294, 80295, 80296, 80297, 80298, 80299, 80300, 80301, + 80302, 80303, 80304, 80305, 80306, 80307, 80308, 80309, 80310, 80311, + 80312, 80313, 80314, 80315, 80316, 80317, 80318, 80319, 80320, 80321, + 80322, 80323, 80324, 80325, 80326, 80327, 80328, 80329, 80330, 80331, + 80332, 80333, 80334, 80335, 80336, 80337, 80338, 80339, 80340, 80341, + 80342, 80343, 80344, 80345, 80346, 80347, 80348, 80349, 80350, 80351, + 80352, 80353, 80354, 80355, 80356, 80357, 80358, 80359, 80360, 80361, + 80362, 80363, 80364, 80365, 80366, 80367, 80368, 80369, 80370, 80371, + 80372, 80373, 80374, 80375, 80376, 80377, 80378, 80379, 80380, 80381, + 80382, 80383, 80384, 80385, 80386, 80387, 80388, 80389, 80390, 80391, + 80392, 80393, 80394, 80395, 80396, 80397, 80398, 80399, 80400, 80401, + 80402, 80403, 80404, 80405, 80406, 80407, 80408, 80409, 80410, 80411, + 80412, 80413, 80414, 80415, 80416, 80417, 80418, 80419, 80420, 80421, + 80422, 80423, 80424, 80425, 80426, 80427, 80428, 80429, 80430, 80431, + 80432, 80433, 80434, 80435, 80436, 80437, 80438, 80439, 80440, 80441, + 80442, 80443, 80444, 80445, 80446, 80447, 80448, 80449, 80450, 80451, + 80452, 80453, 80454, 80455, 80456, 80457, 80458, 80459, 80460, 80461, + 80462, 80463, 80464, 80465, 80466, 80467, 80468, 80469, 80470, 80471, + 80472, 80473, 80474, 80475, 80476, 80477, 80478, 80479, 80480, 80481, + 80482, 80483, 80484, 80485, 80486, 80487, 80488, 80489, 80490, 80491, + 80492, 80493, 80494, 80495, 80496, 80497, 80498, 80499, 80500, 80501, + 80502, 80503, 80504, 80505, 80506, 80507, 80508, 80509, 80510, 80511, + 80512, 80513, 80514, 80515, 80516, 80517, 80518, 80519, 80520, 80521, + 80522, 80523, 80524, 80525, 80526, 80527, 80528, 80529, 80530, 80531, + 80532, 80533, 80534, 80535, 80536, 80537, 80538, 80539, 80540, 80541, + 80542, 80543, 80544, 80545, 80546, 80547, 80548, 80549, 80550, 80551, + 80552, 80553, 80554, 80555, 80556, 80557, 80558, 80559, 80560, 80561, + 80562, 80563, 80564, 80565, 80566, 80567, 80568, 80569, 80570, 80571, + 80572, 80573, 80574, 80575, 80576, 80577, 80578, 80579, 80580, 80581, + 80582, 80583, 80584, 80585, 80586, 80587, 80588, 80589, 80590, 80591, + 80592, 80593, 80594, 80595, 80596, 80597, 80598, 80599, 80600, 80601, + 80602, 80603, 80604, 80605, 80606, 80607, 80608, 80609, 80610, 80611, + 80612, 80613, 80614, 80615, 80616, 80617, 80618, 80619, 80620, 80621, + 80622, 80623, 80624, 80625, 80626, 80627, 80628, 80629, 80630, 80631, + 80632, 80633, 80634, 80635, 80636, 80637, 80638, 80639, 80640, 80641, + 80642, 80643, 80644, 80645, 80646, 80647, 80648, 80649, 80650, 80651, + 80652, 80653, 80654, 80655, 80656, 80657, 80658, 80659, 80660, 80661, + 80662, 80663, 80664, 80665, 80666, 80667, 80668, 80669, 80670, 80671, + 80672, 80673, 80674, 80675, 80676, 80677, 80678, 80679, 80680, 80681, + 80682, 80683, 80684, 80685, 80686, 80687, 80688, 80689, 80690, 80691, + 80692, 80693, 80694, 80695, 80696, 80697, 80698, 80699, 80700, 80701, + 80702, 80703, 80704, 80705, 80706, 80707, 80708, 80709, 80710, 80711, + 80712, 80713, 80714, 80715, 80716, 80717, 80718, 80719, 80720, 80721, + 80722, 80723, 80724, 80725, 80726, 80727, 80728, 80729, 80730, 80731, + 80732, 80733, 80734, 80735, 80736, 80737, 80738, 80739, 80740, 80741, + 80742, 80743, 80744, 80745, 80746, 80747, 80748, 80749, 80750, 80751, + 80752, 80753, 80754, 80755, 80756, 80757, 80758, 80759, 80760, 80761, + 80762, 80763, 80764, 80765, 80766, 80767, 80768, 80769, 80770, 80771, + 80772, 80773, 80774, 80775, 80776, 80777, 80778, 80779, 80780, 80781, + 80782, 80783, 80784, 80785, 80786, 80787, 80788, 80789, 80790, 80791, + 80792, 80793, 80794, 80795, 80796, 80797, 80798, 80799, 80800, 80801, + 80802, 80803, 80804, 80805, 80806, 80807, 80808, 80809, 80810, 80811, + 80812, 80813, 80814, 80815, 80816, 80817, 80818, 80819, 80820, 80821, + 80822, 80823, 80824, 80825, 80826, 80827, 80828, 80829, 80830, 80831, + 80832, 80833, 80834, 80835, 80836, 80837, 80838, 80839, 80840, 80841, + 80842, 80843, 80844, 80845, 80846, 80847, 80848, 80849, 80850, 80851, + 80852, 80853, 80854, 80855, 80856, 80857, 80858, 80859, 80860, 80861, + 80862, 80863, 80864, 80865, 80866, 80867, 80868, 80869, 80870, 80871, + 80872, 80873, 80874, 80875, 80876, 80877, 80878, 80879, 80880, 80881, + 80882, 80883, 80884, 80885, 80886, 80887, 80888, 80889, 80890, 80891, + 80892, 80893, 80894, 80895, 80896, 80897, 80898, 80899, 80900, 80901, + 80902, 80903, 80904, 80905, 80906, 80907, 80908, 80909, 80910, 80911, + 80912, 80913, 80914, 80915, 80916, 80917, 80918, 80919, 80920, 80921, + 80922, 80923, 80924, 80925, 80926, 80927, 80928, 80929, 80930, 80931, + 80932, 80933, 80934, 80935, 80936, 80937, 80938, 80939, 80940, 80941, + 80942, 80943, 80944, 80945, 80946, 80947, 80948, 80949, 80950, 80951, + 80952, 80953, 80954, 80955, 80956, 80957, 80958, 80959, 80960, 80961, + 80962, 80963, 80964, 80965, 80966, 80967, 80968, 80969, 80970, 80971, + 80972, 80973, 80974, 80975, 80976, 80977, 80978, 80979, 80980, 80981, + 80982, 80983, 80984, 80985, 80986, 80987, 80988, 80989, 80990, 80991, + 80992, 80993, 80994, 80995, 80996, 80997, 80998, 80999, 81000, 81001, + 81002, 81003, 81004, 81005, 81006, 81007, 81008, 81009, 81010, 81011, + 81012, 81013, 81014, 81015, 81016, 81017, 81018, 81019, 81020, 81021, + 81022, 81023, 81024, 81025, 81026, 81027, 81028, 81029, 81030, 81031, + 81032, 81033, 81034, 81035, 81036, 81037, 81038, 81039, 81040, 81041, + 81042, 81043, 81044, 81045, 81046, 81047, 81048, 81049, 81050, 81051, + 81052, 81053, 81054, 81055, 81056, 81057, 81058, 81059, 81060, 81061, + 81062, 81063, 81064, 81065, 81066, 81067, 81068, 81069, 81070, 81071, + 81072, 81073, 81074, 81075, 81076, 81077, 81078, 81079, 81080, 81081, + 81082, 81083, 81084, 81085, 81086, 81087, 81088, 81089, 81090, 81091, + 81092, 81093, 81094, 81095, 81096, 81097, 81098, 81099, 81100, 81101, + 81102, 81103, 81104, 81105, 81106, 81107, 81108, 81109, 81110, 81111, + 81112, 81113, 81114, 81115, 81116, 81117, 81118, 81119, 81120, 81121, + 81122, 81123, 81124, 81125, 81126, 81127, 81128, 81129, 81130, 81131, + 81132, 81133, 81134, 81135, 81136, 81137, 81138, 81139, 81140, 81141, + 81142, 81143, 81144, 81145, 81146, 81147, 81148, 81149, 81150, 81151, + 81152, 81153, 81154, 81155, 81156, 81157, 81158, 81159, 81160, 81161, + 81162, 81163, 81164, 81165, 81166, 81167, 81168, 81169, 81170, 81171, + 81172, 81173, 81174, 81175, 81176, 81177, 81178, 81179, 81180, 81181, + 81182, 81183, 81184, 81185, 81186, 81187, 81188, 81189, 81190, 81191, + 81192, 81193, 81194, 81195, 81196, 81197, 81198, 81199, 81200, 81201, + 81202, 81203, 81204, 81205, 81206, 81207, 81208, 81209, 81210, 81211, + 81212, 81213, 81214, 81215, 81216, 81217, 81218, 81219, 81220, 81221, + 81222, 81223, 81224, 81225, 81226, 81227, 81228, 81229, 81230, 81231, + 81232, 81233, 81234, 81235, 81236, 81237, 81238, 81239, 81240, 81241, + 81242, 81243, 81244, 81245, 81246, 81247, 81248, 81249, 81250, 81251, + 81252, 81253, 81254, 81255, 81256, 81257, 81258, 81259, 81260, 81261, + 81262, 81263, 81264, 81265, 81266, 81267, 81268, 81269, 81270, 81271, + 81272, 81273, 81274, 81275, 81276, 81277, 81278, 81279, 81280, 81281, + 81282, 81283, 81284, 81285, 81286, 81287, 81288, 81289, 81290, 81291, + 81292, 81293, 81294, 81295, 81296, 81297, 81298, 81299, 81300, 81301, + 81302, 81303, 81304, 81305, 81306, 81307, 81308, 81309, 81310, 81311, + 81312, 81313, 81314, 81315, 81316, 81317, 81318, 81319, 81320, 81321, + 81322, 81323, 81324, 81325, 81326, 81327, 81328, 81329, 81330, 81331, + 81332, 81333, 81334, 81335, 81336, 81337, 81338, 81339, 81340, 81341, + 81342, 81343, 81344, 81345, 81346, 81347, 81348, 81349, 81350, 81351, + 81352, 81353, 81354, 81355, 81356, 81357, 81358, 81359, 81360, 81361, + 81362, 81363, 81364, 81365, 81366, 81367, 81368, 81369, 81370, 81371, + 81372, 81373, 81374, 81375, 81376, 81377, 81378, 81379, 81380, 81381, + 81382, 81383, 81384, 81385, 81386, 81387, 81388, 81389, 81390, 81391, + 81392, 81393, 81394, 81395, 81396, 81397, 81398, 81399, 81400, 81401, + 81402, 81403, 81404, 81405, 81406, 81407, 81408, 81409, 81410, 81411, + 81412, 81413, 81414, 81415, 81416, 81417, 81418, 81419, 81420, 81421, + 81422, 81423, 81424, 81425, 81426, 81427, 81428, 81429, 81430, 81431, + 81432, 81433, 81434, 81435, 81436, 81437, 81438, 81439, 81440, 81441, + 81442, 81443, 81444, 81445, 81446, 81447, 81448, 81449, 81450, 81451, + 81452, 81453, 81454, 81455, 81456, 81457, 81458, 81459, 81460, 81461, + 81462, 81463, 81464, 81465, 81466, 81467, 81468, 81469, 81470, 81471, + 81472, 81473, 81474, 81475, 81476, 81477, 81478, 81479, 81480, 81481, + 81482, 81483, 81484, 81485, 81486, 81487, 81488, 81489, 81490, 81491, + 81492, 81493, 81494, 81495, 81496, 81497, 81498, 81499, 81500, 81501, + 81502, 81503, 81504, 81505, 81506, 81507, 81508, 81509, 81510, 81511, + 81512, 81513, 81514, 81515, 81516, 81517, 81518, 81519, 81520, 81521, + 81522, 81523, 81524, 81525, 81526, 81527, 81528, 81529, 81530, 81531, + 81532, 81533, 81534, 81535, 81536, 81537, 81538, 81539, 81540, 81541, + 81542, 81543, 81544, 81545, 81546, 81547, 81548, 81549, 81550, 81551, + 81552, 81553, 81554, 81555, 81556, 81557, 81558, 81559, 81560, 81561, + 81562, 81563, 81564, 81565, 81566, 81567, 81568, 81569, 81570, 81571, + 81572, 81573, 81574, 81575, 81576, 81577, 81578, 81579, 81580, 81581, + 81582, 81583, 81584, 81585, 81586, 81587, 81588, 81589, 81590, 81591, + 81592, 81593, 81594, 81595, 81596, 81597, 81598, 81599, 81600, 81601, + 81602, 81603, 81604, 81605, 81606, 81607, 81608, 81609, 81610, 81611, + 81612, 81613, 81614, 81615, 81616, 81617, 81618, 81619, 81620, 81621, + 81622, 81623, 81624, 81625, 81626, 81627, 81628, 81629, 81630, 81631, + 81632, 81633, 81634, 81635, 81636, 81637, 81638, 81639, 81640, 81641, + 81642, 81643, 81644, 81645, 81646, 81647, 81648, 81649, 81650, 81651, + 81652, 81653, 81654, 81655, 81656, 81657, 81658, 81659, 81660, 81661, + 81662, 81663, 81664, 81665, 81666, 81667, 81668, 81669, 81670, 81671, + 81672, 81673, 81674, 81675, 81676, 81677, 81678, 81679, 81680, 81681, + 81682, 81683, 81684, 81685, 81686, 81687, 81688, 81689, 81690, 81691, + 81692, 81693, 81694, 81695, 81696, 81697, 81698, 81699, 81700, 81701, + 81702, 81703, 81704, 81705, 81706, 81707, 81708, 81709, 81710, 81711, + 81712, 81713, 81714, 81715, 81716, 81717, 81718, 81719, 81720, 81721, + 81722, 81723, 81724, 81725, 81726, 81727, 81728, 81729, 81730, 81731, + 81732, 81733, 81734, 81735, 81736, 81737, 81738, 81739, 81740, 81741, + 81742, 81743, 81744, 81745, 81746, 81747, 81748, 81749, 81750, 81751, + 81752, 81753, 81754, 81755, 81756, 81757, 81758, 81759, 81760, 81761, + 81762, 81763, 81764, 81765, 81766, 81767, 81768, 81769, 81770, 81771, + 81772, 81773, 81774, 81775, 81776, 81777, 81778, 81779, 81780, 81781, + 81782, 81783, 81784, 81785, 81786, 81787, 81788, 81789, 81790, 81791, + 81792, 81793, 81794, 81795, 81796, 81797, 81798, 81799, 81800, 81801, + 81802, 81803, 81804, 81805, 81806, 81807, 81808, 81809, 81810, 81811, + 81812, 81813, 81814, 81815, 81816, 81817, 81818, 81819, 81820, 81821, + 81822, 81823, 81824, 81825, 81826, 81827, 81828, 81829, 81830, 81831, + 81832, 81833, 81834, 81835, 81836, 81837, 81838, 81839, 81840, 81841, + 81842, 81843, 81844, 81845, 81846, 81847, 81848, 81849, 81850, 81851, + 81852, 81853, 81854, 81855, 81856, 81857, 81858, 81859, 81860, 81861, + 81862, 81863, 81864, 81865, 81866, 81867, 81868, 81869, 81870, 81871, + 81872, 81873, 81874, 81875, 81876, 81877, 81878, 81879, 81880, 81881, + 81882, 81883, 81884, 81885, 81886, 81887, 81888, 81889, 81890, 81891, + 81892, 81893, 81894, 81895, 81896, 81897, 81898, 81899, 81900, 81901, + 81902, 81903, 81904, 81905, 81906, 81907, 81908, 81909, 81910, 81911, + 81912, 81913, 81914, 81915, 81916, 81917, 81918, 81919, 82928, 82929, + 82930, 82931, 82932, 82933, 82934, 82935, 82936, 82937, 82938, 82688, + 82689, 82690, 82691, 82692, 82693, 82694, 82695, 82696, 82697, 82698, + 82699, 82700, 82701, 82702, 82703, 82704, 82705, 82706, 82707, 82708, + 82709, 82710, 82711, 82712, 82713, 82714, 82715, 82716, 82717, 82718, + 82719, 82720, 82721, 82722, 82723, 82724, 82725, 82726, 82727, 82728, + 82729, 82730, 82731, 82732, 82733, 82734, 82735, 82736, 82737, 82738, + 82739, 82740, 82741, 82742, 82743, 82744, 82745, 82746, 82747, 82748, + 82749, 82750, 82751, 82752, 82753, 82754, 82755, 82756, 82757, 82758, + 82759, 82760, 82761, 82762, 82763, 82764, 82765, 82766, 82767, 82768, + 82769, 82770, 82771, 82772, 82773, 82774, 82775, 82776, 82777, 82778, + 82779, 82780, 82781, 82782, 82783, 82784, 82785, 82786, 82787, 82788, + 82789, 82790, 82791, 82792, 82793, 82794, 82795, 82796, 82797, 82798, + 82799, 82800, 82801, 82802, 82803, 82804, 82805, 82806, 82807, 82808, + 82809, 82810, 82811, 82812, 82813, 82814, 82815, 82816, 82817, 82818, + 82819, 82820, 82821, 82822, 82823, 82824, 82825, 82826, 82827, 82828, + 82829, 82830, 82831, 82832, 82833, 82834, 82835, 82836, 82837, 82838, + 82839, 82840, 82841, 82842, 82843, 82844, 82845, 82846, 82847, 82848, + 82849, 82850, 82851, 82852, 82853, 82854, 82855, 82856, 82857, 82858, + 82859, 82860, 82861, 82862, 82863, 82864, 82865, 82866, 82867, 82868, + 82869, 82870, 82871, 82872, 82873, 82874, 82875, 82876, 82877, 82878, + 82879, 82880, 82881, 82882, 82883, 82884, 82885, 82886, 82887, 82888, + 82889, 82890, 82891, 82892, 82893, 82894, 82895, 82896, 82897, 82898, + 82899, 82900, 82901, 82902, 82903, 82904, 82905, 82906, 82907, 82908, + 82909, 82910, 82911, 82912, 82913, 82914, 82915, 82916, 82917, 82918, + 82919, 82920, 82921, 82922, 82923, 82924, 82925, 82926, 82927, 81920, + 81921, 81922, 81923, 81924, 81925, 81926, 81927, 81928, 81929, 81930, + 81931, 81932, 81933, 81934, 81935, 81936, 81937, 81938, 81939, 81940, + 81941, 81942, 81943, 81944, 81945, 81946, 81947, 81948, 81949, 81950, + 81951, 81952, 81953, 81954, 81955, 81956, 81957, 81958, 81959, 81960, + 81961, 81962, 81963, 81964, 81965, 81966, 81967, 81968, 81969, 81970, + 81971, 81972, 81973, 81974, 81975, 81976, 81977, 81978, 81979, 81980, + 81981, 81982, 81983, 81984, 81985, 81986, 81987, 81988, 81989, 81990, + 81991, 81992, 81993, 81994, 81995, 81996, 81997, 81998, 81999, 82000, + 82001, 82002, 82003, 82004, 82005, 82006, 82007, 82008, 82009, 82010, + 82011, 82012, 82013, 82014, 82015, 82016, 82017, 82018, 82019, 82020, + 82021, 82022, 82023, 82024, 82025, 82026, 82027, 82028, 82029, 82030, + 82031, 82032, 82033, 82034, 82035, 82036, 82037, 82038, 82039, 82040, + 82041, 82042, 82043, 82044, 82045, 82046, 82047, 82048, 82049, 82050, + 82051, 82052, 82053, 82054, 82055, 82056, 82057, 82058, 82059, 82060, + 82061, 82062, 82063, 82064, 82065, 82066, 82067, 82068, 82069, 82070, + 82071, 82072, 82073, 82074, 82075, 82076, 82077, 82078, 82079, 82080, + 82081, 82082, 82083, 82084, 82085, 82086, 82087, 82088, 82089, 82090, + 82091, 82092, 82093, 82094, 82095, 82096, 82097, 82098, 82099, 82100, + 82101, 82102, 82103, 82104, 82105, 82106, 82107, 82108, 82109, 82110, + 82111, 82112, 82113, 82114, 82115, 82116, 82117, 82118, 82119, 82120, + 82121, 82122, 82123, 82124, 82125, 82126, 82127, 82128, 82129, 82130, + 82131, 82132, 82133, 82134, 82135, 82136, 82137, 82138, 82139, 82140, + 82141, 82142, 82143, 82144, 82145, 82146, 82147, 82148, 82149, 82150, + 82151, 82152, 82153, 82154, 82155, 82156, 82157, 82158, 82159, 82160, + 82161, 82162, 82163, 82164, 82165, 82166, 82167, 82168, 82169, 82170, + 82171, 82172, 82173, 82174, 82175, 82176, 82177, 82178, 82179, 82180, + 82181, 82182, 82183, 82184, 82185, 82186, 82187, 82188, 82189, 82190, + 82191, 82192, 82193, 82194, 82195, 82196, 82197, 82198, 82199, 82200, + 82201, 82202, 82203, 82204, 82205, 82206, 82207, 82208, 82209, 82210, + 82211, 82212, 82213, 82214, 82215, 82216, 82217, 82218, 82219, 82220, + 82221, 82222, 82223, 82224, 82225, 82226, 82227, 82228, 82229, 82230, + 82231, 82232, 82233, 82234, 82235, 82236, 82237, 82238, 82239, 82240, + 82241, 82242, 82243, 82244, 82245, 82246, 82247, 82248, 82249, 82250, + 82251, 82252, 82253, 82254, 82255, 82256, 82257, 82258, 82259, 82260, + 82261, 82262, 82263, 82264, 82265, 82266, 82267, 82268, 82269, 82270, + 82271, 82272, 82273, 82274, 82275, 82276, 82277, 82278, 82279, 82280, + 82281, 82282, 82283, 82284, 82285, 82286, 82287, 82288, 82289, 82290, + 82291, 82292, 82293, 82294, 82295, 82296, 82297, 82298, 82299, 82300, + 82301, 82302, 82303, 82304, 82305, 82306, 82307, 82308, 82309, 82310, + 82311, 82312, 82313, 82314, 82315, 82316, 82317, 82318, 82319, 82320, + 82321, 82322, 82323, 82324, 82325, 82326, 82327, 82328, 82329, 82330, + 82331, 82332, 82333, 82334, 82335, 82336, 82337, 82338, 82339, 82340, + 82341, 82342, 82343, 82344, 82345, 82346, 82347, 82348, 82349, 82350, + 82351, 82352, 82353, 82354, 82355, 82356, 82357, 82358, 82359, 82360, + 82361, 82362, 82363, 82364, 82365, 82366, 82367, 82368, 82369, 82370, + 82371, 82372, 82373, 82374, 82375, 82376, 82377, 82378, 82379, 82380, + 82381, 82382, 82383, 82384, 82385, 82386, 82387, 82388, 82389, 82390, + 82391, 82392, 82393, 82394, 82395, 82396, 82397, 82398, 82399, 82400, + 82401, 82402, 82403, 82404, 82405, 82406, 82407, 82408, 82409, 82410, + 82411, 82412, 82413, 82414, 82415, 82416, 82417, 82418, 82419, 82420, + 82421, 82422, 82423, 82424, 82425, 82426, 82427, 82428, 82429, 82430, + 82431, 82432, 82433, 82434, 82435, 82436, 82437, 82438, 82439, 82440, + 82441, 82442, 82443, 82444, 82445, 82446, 82447, 82448, 82449, 82450, + 82451, 82452, 82453, 82454, 82455, 82456, 82457, 82458, 82459, 82460, + 82461, 82462, 82463, 82464, 82465, 82466, 82467, 82468, 82469, 82470, + 82471, 82472, 82473, 82474, 82475, 82476, 82477, 82478, 82479, 82480, + 82481, 82482, 82483, 82484, 82485, 82486, 82487, 82488, 82489, 82490, + 82491, 82492, 82493, 82494, 82495, 82496, 82497, 82498, 82499, 82500, + 82501, 82502, 82503, 82504, 82505, 82506, 82507, 82508, 82509, 82510, + 82511, 82512, 82513, 82514, 82515, 82516, 82517, 82518, 82519, 82520, + 82521, 82522, 82523, 82524, 82525, 82526, 82527, 82528, 82529, 82530, + 82531, 82532, 82533, 82534, 82535, 82536, 82537, 82538, 82539, 82540, + 82541, 82542, 82543, 82544, 82545, 82546, 82547, 82548, 82549, 82550, + 82551, 82552, 82553, 82554, 82555, 82556, 82557, 82558, 82559, 82560, + 82561, 82562, 82563, 82564, 82565, 82566, 82567, 82568, 82569, 82570, + 82571, 82572, 82573, 82574, 82575, 82576, 82577, 82578, 82579, 82580, + 82581, 82582, 82583, 82584, 82585, 82586, 82587, 82588, 82589, 82590, + 82591, 82592, 82593, 82594, 82595, 82596, 82597, 82598, 82599, 82600, + 82601, 82602, 82603, 82604, 82605, 82606, 82607, 82608, 82609, 82610, + 82611, 82612, 82613, 82614, 82615, 82616, 82617, 82618, 82619, 82620, + 82621, 82622, 82623, 82624, 82625, 82626, 82627, 82628, 82629, 82630, + 82631, 82632, 82633, 82634, 82635, 82636, 82637, 82638, 82639, 82640, + 82641, 82642, 82643, 82644, 82645, 82646, 82647, 82648, 82649, 82650, + 82651, 82652, 82653, 82654, 82655, 82656, 82657, 82658, 82659, 82660, + 82661, 82662, 82663, 82664, 82665, 82666, 82667, 82668, 82669, 82670, + 82671, 82672, 82673, 82674, 82675, 82676, 82677, 82678, 82679, 82680, + 82681, 82682, 82683, 82684, 82685, 82686, 82687, 129370, 10037, 10039, + 10036, 10049, 117865, 117866, 10058, 10035, 9834, 66854, 66853, 66827, + 66826, 66833, 66832, 66821, 66837, 66836, 66835, 66842, 66841, 66824, + 66823, 66819, 66818, 66822, 66820, 66855, 66831, 66844, 66843, 66846, + 66845, 66852, 66851, 66817, 66825, 66828, 66830, 66834, 66839, 66840, + 66848, 66849, 66816, 66829, 66838, 66847, 66850, 128268, 128294, 128161, + 8961, 9191, 8712, 8946, 8953, 8947, 8952, 8950, 8949, 10969, 10194, + 128024, 128727, 69608, 69621, 69603, 69611, 69618, 69619, 69600, 69615, + 69602, 69606, 69614, 69617, 69620, 69609, 69604, 69607, 69610, 69601, + 69613, 69605, 69616, 69612, 69622, 129501, 983101, 129456, 129457, + 129459, 129458, 127995, 127996, 127997, 127998, 127999, 128453, 128454, + 128455, 129721, 128460, 128461, 8709, 10675, 10676, 10674, 10673, 128459, + 9091, 8193, 8212, 8195, 8192, 8211, 8194, 983179, 8718, 983178, 983135, + 983099, 983048, 983095, 983046, 983064, 128282, 983051, 983050, 9993, + 128388, 128233, 9094, 983067, 983100, 983049, 8926, 8927, 8925, 8924, + 8797, 8917, 61, 10865, 10867, 11072, 10609, 10723, 10724, 10926, 11257, + 10871, 10854, 10862, 8789, 10872, 8781, 8794, 10738, 10736, 10734, 10739, + 10737, 10735, 11249, 11248, 9003, 8998, 983105, 983104, 8494, 8793, + 983136, 4957, 4959, 4958, 4963, 4965, 4978, 4988, 4980, 4979, 4987, 4985, + 4982, 4981, 4986, 4984, 4983, 4968, 4966, 4964, 4960, 4999, 4998, 4711, + 4997, 43816, 43819, 43821, 43820, 43818, 43822, 43817, 4704, 4707, 4710, + 11653, 4709, 4708, 4706, 4705, 43808, 43811, 43813, 43812, 43810, 43814, + 43809, 11704, 11707, 11709, 11708, 11706, 11710, 11705, 11688, 11691, + 11693, 11692, 11690, 11694, 11689, 4904, 4907, 4910, 11664, 4909, 4908, + 4911, 4906, 4905, 4728, 4731, 4734, 11655, 4733, 4732, 4735, 4730, 4729, + 43789, 43788, 43787, 43786, 43790, 43785, 4856, 4859, 4862, 11661, 4861, + 4860, 4863, 4858, 4857, 43797, 43796, 43795, 43794, 43798, 43793, 4848, + 4851, 4854, 11660, 4853, 4852, 4855, 4850, 4849, 5003, 5002, 4943, 5001, + 4936, 4939, 4941, 4940, 4954, 4938, 4942, 4937, 4873, 124916, 124915, + 124924, 124923, 124910, 124909, 124926, 124925, 124922, 124921, 124920, + 124919, 124918, 124917, 124914, 124913, 124912, 124904, 11667, 4895, + 11670, 11669, 11668, 4888, 4891, 4893, 4892, 4890, 4894, 4889, 4768, + 4771, 4774, 11658, 4773, 4772, 4775, 4770, 4769, 4880, 4883, 4885, 4884, + 4882, 11736, 11739, 11741, 11740, 11738, 11742, 11737, 4872, 4875, 4878, + 4879, 4877, 4876, 4874, 124907, 124906, 4631, 124905, 124896, 124899, + 124901, 124900, 124898, 124902, 124897, 4624, 4627, 4629, 4628, 4626, + 4630, 4625, 4608, 4611, 4614, 4615, 4613, 4612, 4610, 4609, 4800, 4803, + 4805, 4804, 4802, 4792, 4795, 4797, 4796, 4794, 4798, 4793, 4784, 4787, + 4789, 4788, 4786, 11720, 11723, 11725, 11724, 11722, 11726, 11721, 4776, + 4779, 4782, 4783, 4781, 4780, 4778, 4777, 4995, 4994, 4639, 4993, 4632, + 4635, 4638, 11649, 4637, 4636, 4953, 4634, 4633, 4760, 4763, 4766, 11657, + 4765, 4764, 4767, 4762, 4761, 4752, 4755, 4758, 11656, 4757, 4756, 4759, + 4754, 4753, 4912, 4816, 4819, 4821, 4820, 4818, 4822, 4817, 4915, 4918, + 11665, 4917, 4916, 4919, 4914, 4913, 5007, 5006, 4951, 5005, 4944, 4947, + 4950, 11666, 4949, 4948, 4946, 4945, 4696, 4699, 4701, 4700, 4698, 4688, + 4691, 4693, 4692, 4690, 4694, 4689, 4680, 4683, 4685, 4684, 4682, 11712, + 11715, 11717, 11716, 11714, 11718, 11713, 4672, 4675, 4678, 4679, 4677, + 4676, 4674, 4673, 4648, 4651, 4654, 11650, 4653, 4652, 4655, 4952, 4650, + 4649, 4661, 4996, 5000, 4992, 5004, 4660, 4664, 4667, 4670, 11652, 4669, + 4668, 4671, 4666, 4665, 4640, 4643, 4645, 4644, 4647, 4642, 4646, 4641, + 11680, 11683, 11685, 11684, 11682, 11686, 11681, 4656, 4659, 4662, 11651, + 4663, 4658, 4657, 4896, 4899, 4902, 11663, 4901, 4900, 4903, 4898, 4897, + 43781, 43780, 43779, 43778, 43782, 43777, 4928, 4931, 4934, 4935, 4933, + 4932, 4930, 4929, 4920, 4923, 4925, 4924, 4927, 4922, 4926, 4921, 4720, + 4723, 4726, 11654, 4725, 4724, 4727, 4722, 4721, 4864, 4867, 4870, 11662, + 4869, 4868, 4871, 4866, 4865, 4616, 4619, 4622, 11648, 4621, 4620, 4623, + 4618, 4617, 4808, 4811, 4814, 4815, 4813, 4812, 4810, 4809, 4840, 4843, + 4846, 4847, 4845, 4844, 4842, 4841, 4744, 4747, 4749, 4748, 4746, 11728, + 11731, 11733, 11732, 11730, 11734, 11729, 4736, 4739, 4742, 4743, 4741, + 4740, 4738, 4737, 4832, 4835, 4837, 4836, 4839, 4834, 4838, 4833, 11696, + 11699, 11701, 11700, 11698, 11702, 11697, 4824, 4827, 4830, 11659, 4829, + 4828, 4831, 4826, 4825, 4712, 4715, 4717, 4716, 4719, 4714, 4718, 4713, + 5009, 5016, 5012, 5015, 5013, 5017, 5010, 5011, 5014, 5008, 4973, 4972, + 4975, 4974, 4971, 4970, 4977, 4969, 4976, 4962, 4967, 4961, 983096, + 983047, 127984, 127972, 8352, 8364, 8455, 8265, 33, 8761, 118269, 118270, + 118271, 118274, 128529, 128942, 128954, 128948, 128905, 128915, 128935, + 128125, 1781, 1780, 1783, 1782, 1779, 1778, 1776, 1785, 1777, 1784, + 128065, 128083, 128064, 128231, 9167, 127794, 983180, 128523, 128561, + 129312, 128531, 129301, 128567, 129488, 128582, 128558, 129326, 128560, + 129762, 129320, 128581, 129395, 129763, 129402, 128539, 128540, 128541, + 128514, 129298, 129769, 129323, 128580, 128548, 129764, 129396, 128566, + 129318, 128134, 128536, 129401, 8507, 127981, 10540, 10543, 9950, 127810, + 129478, 128439, 128224, 128106, 9771, 127877, 129498, 128552, 129718, + 170, 9792, 127905, 9972, 129338, 8210, 8199, 128193, 128452, 983107, + 128253, 127902, 129734, 10765, 128293, 128658, 129519, 127879, 127878, + 129512, 9789, 127771, 127763, 8296, 129351, 128031, 127907, 9673, 127845, + 128074, 8281, 11821, 127953, 129407, 129747, 9189, 117910, 9971, 129449, + 128170, 9884, 10086, 9880, 8277, 127924, 128190, 128563, 129672, 129712, + 128760, 117834, 117835, 129359, 128389, 127787, 127745, 129709, 128448, + 129462, 128099, 127860, 127869, 11792, 10972, 983071, 8704, 8873, 129376, + 10021, 11156, 8280, 8283, 10019, 127808, 10018, 128966, 8732, 8197, 9970, 129749, 129418, 8260, 8543, 128444, 128446, 128445, 127839, 8355, 129398, 10156, 128037, 8994, 128550, 128056, 127844, 127773, 127765, 10199, 9608, - 46, 65342, 65312, 65292, 65306, 65504, 65371, 65375, 65288, 65339, 65308, + 46, 65342, 65312, 65292, 65306, 65504, 65375, 65371, 65288, 65339, 65308, 65313, 65314, 65315, 65316, 65317, 65318, 65319, 65320, 65321, 65322, 65323, 65324, 65325, 65326, 65327, 65328, 65329, 65330, 65331, 65332, 65333, 65334, 65335, 65336, 65337, 65338, 65345, 65346, 65347, 65348, 65349, 65350, 65351, 65352, 65353, 65354, 65355, 65356, 65357, 65358, 65359, 65360, 65361, 65362, 65363, 65364, 65365, 65366, 65367, 65368, - 65369, 65370, 65343, 65506, 65283, 65505, 65285, 65291, 65373, 65376, + 65369, 65370, 65343, 65506, 65283, 65505, 65285, 65291, 65376, 65373, 65289, 65341, 65340, 65307, 65295, 65509, 65507, 65287, 65286, 65290, 65284, 65301, 65300, 65303, 65302, 65299, 65298, 65296, 65305, 65297, 65304, 65309, 65281, 65344, 65310, 65293, 65282, 65311, 65510, 65374, - 65508, 65294, 65372, 8289, 9905, 9981, 9179, 983215, 983216, 983217, - 983219, 983108, 983236, 983072, 129476, 127922, 9881, 9965, 9966, 128142, - 9802, 8782, 8785, 8762, 4301, 4256, 4288, 4292, 4290, 4293, 4289, 4281, - 4285, 4284, 4282, 4278, 4258, 4287, 4283, 4277, 4265, 4276, 4270, 4280, - 4273, 4271, 4262, 4263, 4274, 4266, 4272, 4257, 4267, 4286, 4268, 4279, - 4261, 4259, 4260, 4264, 4269, 4275, 4295, 4291, 11565, 11520, 11552, - 11556, 11554, 11557, 11553, 11545, 11549, 11548, 11546, 11542, 11522, - 11551, 11547, 11541, 11529, 11540, 11534, 11544, 11537, 11535, 11526, - 11527, 11538, 11530, 11536, 11521, 11531, 11550, 11532, 11543, 11525, - 11523, 11524, 11528, 11533, 11539, 11559, 11555, 983955, 4323, 4349, - 4346, 4304, 4329, 4333, 4332, 4330, 4344, 4308, 4326, 4306, 4340, 4350, - 4336, 4338, 4341, 4337, 4335, 4331, 4325, 4313, 4351, 4314, 4324, 4318, - 4328, 4321, 4345, 4311, 4322, 4319, 4310, 4320, 4305, 4315, 4334, 4316, - 4327, 4309, 4307, 4312, 4317, 4343, 4339, 4342, 7357, 7354, 7312, 7337, - 7341, 7340, 7338, 7352, 7316, 7334, 7314, 7348, 7358, 7344, 7346, 7349, - 7345, 7343, 7339, 7333, 7321, 7359, 7322, 7332, 7326, 7336, 7329, 7353, - 7319, 7330, 7327, 7318, 7328, 7313, 7323, 7342, 7324, 7335, 7317, 7315, - 7320, 7325, 7331, 7351, 7347, 7350, 4347, 8368, 12307, 129502, 8503, - 129754, 128103, 128714, 129426, 11264, 11304, 11265, 11311, 11293, 11276, - 11268, 11271, 11287, 11306, 11267, 11275, 11274, 11305, 11303, 11307, - 11273, 11310, 11278, 11279, 11280, 11281, 11289, 11282, 11290, 11283, - 11291, 11308, 11294, 11284, 11298, 11300, 11301, 11285, 11309, 11292, - 11266, 11269, 11296, 11295, 11297, 11302, 11299, 11272, 11270, 11288, - 11286, 11277, 11312, 11352, 11313, 11359, 11341, 11324, 11316, 11319, - 11335, 11354, 11315, 11323, 11322, 11353, 11351, 11355, 11321, 11358, - 11326, 11327, 11328, 11329, 11337, 11330, 11338, 11331, 11339, 11356, - 11342, 11332, 11346, 11348, 11349, 11333, 11357, 11340, 11314, 11317, - 11344, 11343, 11345, 11350, 11347, 11320, 11318, 11336, 11334, 11325, - 129371, 127760, 127775, 129508, 10726, 129349, 128016, 66356, 66352, - 66376, 66359, 66358, 66375, 66378, 66369, 66365, 66368, 66357, 66370, - 66360, 66372, 66373, 66367, 66374, 66353, 66377, 66355, 66364, 66371, - 66361, 66363, 66366, 66354, 66362, 129405, 129421, 128893, 129727, + 65294, 65508, 65372, 8289, 9905, 118280, 9981, 9179, 983215, 983216, + 983217, 983219, 983108, 983236, 983072, 68972, 68971, 68973, 68970, + 68964, 68965, 68959, 68961, 68948, 68945, 68954, 68960, 68953, 68963, + 68949, 68947, 68952, 68946, 68962, 68958, 68950, 68957, 68951, 68955, + 68956, 68944, 68996, 68997, 68991, 68993, 68980, 68977, 68986, 68992, + 68985, 68995, 68981, 68979, 68984, 68978, 68994, 68990, 68982, 68989, + 68983, 68987, 68988, 68976, 68943, 68969, 68941, 68938, 68939, 68940, + 68942, 68975, 68974, 69006, 69007, 68933, 68932, 68935, 68934, 68931, + 68930, 68928, 68937, 68929, 68936, 129476, 127922, 9881, 9965, 9966, + 128142, 9802, 8782, 8785, 8762, 4301, 4256, 4288, 4292, 4290, 4293, 4289, + 4281, 4285, 4284, 4282, 4278, 4258, 4287, 4283, 4277, 4265, 4276, 4270, + 4280, 4273, 4271, 4262, 4263, 4274, 4266, 4272, 4257, 4267, 4286, 4268, + 4279, 4261, 4259, 4260, 4264, 4269, 4275, 4295, 4291, 11565, 11520, + 11552, 11556, 11554, 11557, 11553, 11545, 11549, 11548, 11546, 11542, + 11522, 11551, 11547, 11541, 11529, 11540, 11534, 11544, 11537, 11535, + 11526, 11527, 11538, 11530, 11536, 11521, 11531, 11550, 11532, 11543, + 11525, 11523, 11524, 11528, 11533, 11539, 11559, 11555, 983955, 4323, + 4349, 4346, 4304, 4329, 4333, 4332, 4330, 4344, 4308, 4326, 4306, 4340, + 4350, 4336, 4338, 4341, 4337, 4335, 4331, 4325, 4313, 4351, 4314, 4324, + 4318, 4328, 4321, 4345, 4311, 4322, 4319, 4310, 4320, 4305, 4315, 4334, + 4316, 4327, 4309, 4307, 4312, 4317, 4343, 4339, 4342, 7357, 7354, 7312, + 7337, 7341, 7340, 7338, 7352, 7316, 7334, 7314, 7348, 7358, 7344, 7346, + 7349, 7345, 7343, 7339, 7333, 7321, 7359, 7322, 7332, 7326, 7336, 7329, + 7353, 7319, 7330, 7327, 7318, 7328, 7313, 7323, 7342, 7324, 7335, 7317, + 7315, 7320, 7325, 7331, 7351, 7347, 7350, 4347, 8368, 12307, 129502, + 8503, 129754, 128103, 128714, 129426, 11264, 11304, 11265, 11311, 11293, + 11276, 11268, 11271, 11287, 11306, 11267, 11275, 11274, 11305, 11303, + 11307, 11273, 11310, 11278, 11279, 11280, 11281, 11289, 11282, 11290, + 11283, 11291, 11308, 11294, 11284, 11298, 11300, 11301, 11285, 11309, + 11292, 11266, 11269, 11296, 11295, 11297, 11302, 11299, 11272, 11270, + 11288, 11286, 11277, 11312, 11352, 11313, 11359, 11341, 11324, 11316, + 11319, 11335, 11354, 11315, 11323, 11322, 11353, 11351, 11355, 11321, + 11358, 11326, 11327, 11328, 11329, 11337, 11330, 11338, 11331, 11339, + 11356, 11342, 11332, 11346, 11348, 11349, 11333, 11357, 11340, 11314, + 11317, 11344, 11343, 11345, 11350, 11347, 11320, 11318, 11336, 11334, + 11325, 129371, 127760, 127775, 129508, 10726, 129349, 128016, 66356, + 66352, 66376, 66359, 66358, 66375, 66378, 66369, 66365, 66368, 66357, + 66370, 66360, 66372, 66373, 66367, 66374, 66353, 66377, 66355, 66364, + 66371, 66361, 66363, 66366, 66354, 66362, 129421, 129405, 128893, 129727, 127948, 127891, 70495, 70494, 70411, 70496, 70412, 70497, 70453, 70405, 70406, 70416, 70420, 70434, 70433, 70439, 70438, 70432, 70431, 70437, 70436, 70409, 70410, 70407, 70408, 70451, 70450, 70425, 70435, 70430, 70440, 70454, 70455, 70456, 70445, 70444, 70427, 70426, 70424, 70423, 70429, 70428, 70422, 70421, 70443, 70442, 70419, 70415, 70457, 70446, - 70448, 70447, 70400, 70401, 70493, 70461, 70402, 70460, 70477, 70403, - 70462, 70472, 70476, 70467, 70468, 70498, 70499, 70465, 70466, 70463, - 70464, 70475, 70471, 70487, 70480, 96, 127815, 10896, 10894, 10900, + 70448, 70447, 70400, 70401, 70460, 70461, 70402, 70493, 70477, 70403, + 70487, 70462, 70472, 70476, 70465, 70466, 70467, 70468, 70498, 70499, + 70463, 70464, 70475, 70471, 70480, 96, 127815, 10896, 10894, 10900, 10892, 10898, 10616, 10890, 10888, 10917, 8935, 8809, 10878, 10882, 10884, 10880, 10886, 8819, 8805, 8823, 10916, 8807, 8923, 10919, 10921, 10874, 10876, 8919, 62, 65860, 65863, 65878, 65866, 65873, 65859, 65861, @@ -10318,192 +11051,199 @@ static const unsigned int dawg_pos_to_codepoint[] = { 119322, 119323, 119324, 119296, 119305, 119306, 119307, 119308, 119309, 119310, 119311, 119312, 119313, 119314, 119298, 119299, 119301, 119302, 119303, 119304, 890, 65913, 976, 65928, 65930, 894, 127823, 128215, - 128154, 129367, 129654, 128512, 129322, 128513, 129321, 128568, 128556, + 128154, 129367, 129654, 128512, 129322, 129321, 128513, 128568, 128556, 983110, 11218, 128151, 8370, 128130, 127928, 129454, 2693, 2694, 2704, 2708, 2722, 2721, 2727, 2726, 2720, 2719, 2725, 2724, 2699, 2784, 2700, 2785, 2741, 2697, 2698, 2695, 2696, 2739, 2738, 2809, 2713, 2723, 2718, 2728, 2742, 2743, 2744, 2733, 2732, 2715, 2714, 2712, 2711, 2717, 2716, 2710, 2709, 2731, 2730, 2745, 2734, 2736, 2735, 2703, 2707, 2814, 2689, - 2812, 2815, 2813, 2811, 2810, 2749, 2690, 2748, 2765, 2691, 2757, 2761, - 2750, 2760, 2764, 2755, 2756, 2786, 2787, 2753, 2754, 2751, 2752, 2759, + 2812, 2815, 2813, 2811, 2810, 2748, 2749, 2690, 2765, 2691, 2757, 2761, + 2750, 2760, 2764, 2753, 2754, 2755, 2756, 2786, 2787, 2751, 2752, 2759, 2763, 2701, 2705, 2800, 2801, 2795, 2794, 2797, 2796, 2793, 2792, 2790, - 2799, 2791, 2798, 2768, 73056, 73057, 73064, 73067, 73091, 73090, 73081, - 73080, 73086, 73085, 73076, 73075, 73060, 73061, 73092, 73082, 73058, + 2799, 2791, 2798, 2768, 73092, 73082, 73056, 73057, 73064, 73067, 73091, + 73090, 73081, 73080, 73086, 73085, 73076, 73075, 73060, 73061, 73058, 73059, 73087, 73077, 73071, 73070, 73084, 73083, 73079, 73078, 73089, 73088, 73074, 73073, 73094, 73093, 73066, 73063, 73095, 73072, 73096, 73097, 73069, 73068, 73110, 73109, 73098, 73105, 73108, 73101, 73102, 73099, 73100, 73107, 73104, 73111, 73125, 73124, 73127, 73126, 73123, 73122, 73120, 73129, 73121, 73128, 73112, 2678, 2673, 2650, 2584, 2583, - 2649, 2582, 2581, 2652, 2608, 2565, 2566, 2576, 2580, 2594, 2593, 2599, - 2598, 2592, 2591, 2597, 2596, 2569, 2570, 2567, 2568, 2611, 2610, 2585, + 2649, 2582, 2581, 2565, 2566, 2576, 2580, 2594, 2593, 2599, 2598, 2652, + 2608, 2592, 2591, 2597, 2596, 2569, 2570, 2567, 2568, 2611, 2610, 2585, 2595, 2590, 2600, 2605, 2604, 2587, 2586, 2589, 2588, 2603, 2602, 2614, 2616, 2579, 2575, 2654, 2617, 2606, 2613, 2607, 2651, 983656, 983655, 983654, 983653, 983658, 983657, 2561, 2562, 2641, 2620, 2677, 2637, 2563, 2622, 2632, 2636, 2625, 2626, 2623, 2624, 2635, 2631, 2672, 2674, 2676, - 2667, 2666, 2669, 2668, 2665, 2664, 2662, 2671, 2663, 2670, 2675, 128123, - 983111, 129710, 8202, 128135, 65467, 65441, 65443, 65444, 65445, 65446, - 65469, 65458, 65460, 65449, 65454, 65455, 65452, 65451, 65450, 65456, - 65453, 65465, 65442, 65459, 65448, 65462, 65461, 65483, 65482, 65476, - 65477, 65499, 65490, 65495, 65466, 65464, 65479, 65478, 65498, 65500, - 65463, 65457, 65468, 65447, 65493, 65492, 65485, 65486, 65494, 65470, - 65474, 65475, 65484, 65487, 65491, 65440, 65388, 65390, 65389, 65391, - 65383, 65386, 65384, 65387, 65385, 65403, 65406, 65404, 65407, 65405, - 65437, 65413, 65416, 65414, 65417, 65415, 65418, 65421, 65419, 65422, - 65420, 65398, 65401, 65399, 65402, 65400, 65423, 65426, 65424, 65427, - 65425, 65431, 65434, 65432, 65435, 65433, 65408, 65411, 65409, 65412, - 65410, 65428, 65430, 65429, 65436, 65382, 65393, 65396, 65394, 65397, - 65395, 65439, 65438, 65381, 65392, 65378, 65513, 65379, 65515, 65514, - 65512, 65380, 65377, 65517, 65518, 65516, 128296, 128736, 9773, 9874, - 128057, 129708, 127828, 129310, 129776, 129342, 128092, 129309, 4366, - 4434, 4435, 4431, 4413, 4412, 4436, 4430, 4433, 4415, 4414, 4437, 4432, - 4364, 4429, 4363, 4427, 4420, 4422, 43382, 4425, 4424, 4419, 4426, 4418, - 4417, 43383, 4421, 4379, 4439, 4395, 4396, 4381, 4352, 4442, 4367, 4358, - 43375, 4380, 43376, 43377, 4354, 4444, 4371, 4374, 4373, 4445, 4443, - 4369, 4438, 43386, 4359, 4385, 43378, 4387, 4390, 4386, 4388, 4389, 4394, - 43379, 4382, 4392, 4391, 4393, 4384, 4383, 43380, 4416, 4357, 43371, - 43374, 43364, 43365, 43370, 43367, 43372, 43368, 43373, 4376, 43369, - 43366, 4378, 4361, 4402, 4403, 4410, 4404, 4400, 4408, 4397, 4407, 4406, - 4401, 4409, 4399, 4398, 4411, 4405, 4365, 43384, 4362, 43381, 4377, - 43385, 4356, 43388, 4353, 4372, 4360, 4440, 4423, 4355, 4446, 43360, - 43363, 4375, 43361, 43362, 4368, 4428, 4441, 4370, 43387, 4447, 12335, - 12334, 4541, 55288, 55287, 4542, 4546, 4598, 4599, 4597, 4600, 4540, - 4591, 4588, 4589, 55261, 4596, 4582, 4578, 4520, 4605, 4547, 4522, 4548, - 4604, 4602, 4603, 4606, 4543, 4535, 4572, 55265, 4575, 55263, 4574, 4573, - 4571, 4576, 55266, 4570, 55262, 4577, 4523, 55243, 55244, 4524, 4553, - 4550, 4549, 4525, 4552, 4551, 4587, 55284, 55283, 4545, 55291, 4595, - 55290, 4536, 4579, 55268, 4537, 55271, 4580, 55273, 55272, 55269, 55267, - 4581, 4527, 4528, 55254, 4556, 4565, 4568, 4529, 4561, 55256, 4562, 4530, - 55258, 55257, 4564, 4563, 4533, 4567, 55253, 4566, 4531, 4558, 4559, - 4532, 4569, 55260, 55259, 4557, 4534, 4538, 55275, 4583, 4585, 55280, - 55279, 4586, 55278, 55274, 55281, 4584, 55282, 4560, 55255, 55245, 55246, - 983213, 4539, 55276, 55277, 55264, 55289, 4521, 4607, 55270, 4590, 4526, - 4555, 55248, 55249, 55252, 55251, 55250, 4554, 55247, 4544, 4592, 983214, - 983211, 983212, 4593, 55285, 55286, 4594, 4601, 4449, 4510, 55238, 4511, - 55237, 4513, 4512, 4515, 4470, 4471, 4450, 4454, 4453, 4476, 4474, 4475, - 4467, 55227, 55226, 4502, 55225, 55228, 4501, 4469, 4504, 4509, 4505, - 55229, 55230, 55232, 55231, 55234, 55235, 4506, 55233, 4508, 55236, 4507, - 4514, 4457, 4482, 55217, 4481, 55216, 4518, 4519, 4480, 4479, 4483, 4460, - 4462, 4492, 55221, 55222, 4491, 4489, 4490, 4493, 4451, 4473, 4472, 4516, - 4452, 4456, 4455, 4517, 4477, 4478, 4468, 4503, 4461, 4484, 4485, 4486, - 55218, 55219, 55220, 4488, 4487, 4466, 4498, 4497, 4496, 4495, 4494, - 55223, 4500, 55224, 4499, 4464, 4463, 4458, 4459, 4465, 4448, 12623, - 12685, 12686, 12624, 12618, 12616, 12628, 12627, 12641, 12643, 12615, - 12676, 12664, 12665, 12657, 12619, 12593, 12595, 12609, 12654, 12656, - 12655, 12596, 12597, 12646, 12598, 12648, 12647, 12610, 12612, 12660, - 12661, 12663, 12659, 12662, 12658, 12621, 12671, 12601, 12602, 12649, - 12603, 12607, 12604, 12651, 12652, 12606, 12650, 12653, 12608, 12605, - 12620, 12599, 12613, 12670, 12666, 12667, 12669, 12668, 12617, 12594, - 12645, 12611, 12600, 12677, 12614, 12672, 12638, 12637, 12632, 12633, - 12639, 12630, 12629, 12678, 12673, 12675, 12674, 12635, 12679, 12680, - 12681, 12640, 12683, 12682, 12684, 12625, 12626, 12642, 12622, 12631, - 12634, 12636, 12644, 12323, 12346, 12345, 12322, 12344, 12325, 12324, - 12327, 12326, 12329, 12321, 12328, 68875, 68874, 68887, 68889, 68872, - 68881, 68868, 68867, 68877, 68876, 68890, 68891, 68885, 68880, 68879, - 68865, 68870, 68873, 68882, 68871, 68869, 68883, 68884, 68866, 68892, - 68886, 68888, 68878, 68864, 68899, 68898, 68901, 68903, 68902, 68900, - 68893, 68896, 68894, 68897, 68895, 68917, 68916, 68919, 68918, 68915, - 68914, 68912, 68921, 68913, 68920, 5925, 5928, 5930, 5927, 5924, 5937, - 5923, 5934, 5931, 5929, 5933, 5936, 5926, 5935, 5932, 5920, 5921, 5922, - 5940, 5938, 5939, 128587, 128035, 67808, 67823, 67814, 67816, 67829, + 2667, 2666, 2669, 2668, 2665, 2664, 2662, 2671, 2663, 2670, 2675, 90412, + 90414, 90411, 90410, 90373, 90388, 90382, 90381, 90387, 90386, 90380, + 90379, 90385, 90384, 90392, 90391, 90375, 90374, 90372, 90371, 90377, + 90376, 90370, 90369, 90390, 90389, 90378, 90396, 90393, 90395, 90397, + 90383, 90394, 90368, 90413, 90415, 90398, 90405, 90408, 90401, 90402, + 90406, 90407, 90399, 90400, 90403, 90404, 90409, 90421, 90420, 90423, + 90422, 90419, 90418, 90416, 90425, 90417, 90424, 128123, 983111, 129710, + 8202, 128135, 65467, 65441, 65443, 65444, 65445, 65446, 65469, 65458, + 65460, 65449, 65454, 65455, 65452, 65451, 65450, 65456, 65453, 65465, + 65442, 65459, 65448, 65462, 65461, 65483, 65482, 65476, 65477, 65499, + 65490, 65495, 65466, 65464, 65479, 65478, 65498, 65500, 65463, 65457, + 65468, 65447, 65493, 65492, 65485, 65486, 65494, 65470, 65474, 65475, + 65484, 65487, 65491, 65440, 65388, 65390, 65389, 65391, 65383, 65386, + 65384, 65387, 65385, 65403, 65406, 65404, 65407, 65405, 65437, 65413, + 65416, 65414, 65417, 65415, 65418, 65421, 65419, 65422, 65420, 65398, + 65401, 65399, 65402, 65400, 65423, 65426, 65424, 65427, 65425, 65431, + 65434, 65432, 65435, 65433, 65408, 65411, 65409, 65412, 65410, 65428, + 65430, 65429, 65436, 65382, 65393, 65396, 65394, 65397, 65395, 65439, + 65438, 65381, 65392, 65378, 65513, 65379, 65515, 65512, 65514, 65380, + 65377, 65517, 65518, 65516, 128296, 128736, 9773, 9874, 128057, 129708, + 127828, 129310, 129776, 129342, 128092, 129309, 4366, 4434, 4435, 4431, + 4413, 4412, 4436, 4430, 4433, 4415, 4414, 4437, 4432, 4364, 4429, 4363, + 4427, 4420, 4422, 43382, 4425, 4424, 4419, 4426, 4418, 4417, 43383, 4421, + 4379, 4439, 4395, 4396, 4381, 4352, 4442, 4367, 4358, 43375, 4380, 43376, + 43377, 4354, 4444, 4371, 4374, 4373, 4445, 4443, 4369, 4438, 43386, 4359, + 4385, 43378, 4387, 4390, 4386, 4388, 4389, 4394, 43379, 4382, 4392, 4391, + 4393, 4384, 4383, 43380, 4416, 4357, 43371, 43374, 43364, 43365, 43370, + 43367, 43372, 43368, 43373, 4376, 43369, 43366, 4378, 4361, 4402, 4403, + 4410, 4404, 4400, 4408, 4397, 4407, 4406, 4401, 4409, 4399, 4398, 4411, + 4405, 4365, 43384, 4362, 43381, 4377, 43385, 4356, 43388, 4353, 4372, + 4360, 4440, 4423, 4355, 4446, 43360, 43363, 4375, 43361, 43362, 4368, + 4428, 4441, 4370, 43387, 4447, 12335, 12334, 4541, 55288, 55287, 4542, + 4546, 4598, 4599, 4597, 4600, 4540, 4591, 4588, 4589, 55261, 4596, 4582, + 4578, 4520, 4605, 4547, 4522, 4548, 4604, 4602, 4603, 4606, 4543, 4535, + 4572, 55265, 4575, 55263, 4574, 4573, 4571, 4576, 55266, 4570, 55262, + 4577, 4523, 55243, 55244, 4524, 4553, 4550, 4549, 4525, 4552, 4551, 4587, + 55284, 55283, 4545, 55291, 4595, 55290, 4536, 4579, 55268, 4537, 55271, + 4580, 55273, 55272, 55269, 55267, 4581, 4527, 4528, 55254, 4556, 4565, + 4568, 4529, 4561, 55256, 4562, 4530, 55258, 55257, 4564, 4563, 4533, + 4567, 55253, 4566, 4531, 4558, 4559, 4532, 4569, 55260, 55259, 4557, + 4534, 4538, 55275, 4583, 4585, 55280, 55279, 4586, 55278, 55274, 55281, + 4584, 55282, 4560, 55255, 55245, 55246, 983213, 4539, 55276, 55277, + 55264, 55289, 4521, 4607, 55270, 4590, 4526, 4555, 55248, 55249, 55252, + 55251, 55250, 4554, 55247, 4544, 4592, 983214, 983211, 983212, 4593, + 55285, 55286, 4594, 4601, 4449, 4510, 55238, 4511, 55237, 4513, 4512, + 4515, 4470, 4471, 4450, 4454, 4453, 4476, 4474, 4475, 4467, 55227, 55226, + 4502, 55225, 55228, 4501, 4469, 4504, 4509, 4505, 55229, 55230, 55232, + 55231, 55234, 55235, 4506, 55233, 4508, 55236, 4507, 4514, 4457, 4482, + 55217, 4481, 55216, 4518, 4519, 4480, 4479, 4483, 4460, 4462, 4492, + 55221, 55222, 4491, 4489, 4490, 4493, 4451, 4473, 4472, 4516, 4452, 4456, + 4455, 4517, 4477, 4478, 4468, 4503, 4461, 4484, 4485, 4486, 55218, 55219, + 55220, 4488, 4487, 4466, 4498, 4497, 4496, 4495, 4494, 55223, 4500, + 55224, 4499, 4464, 4463, 4458, 4459, 4465, 4448, 12623, 12685, 12686, + 12624, 12618, 12616, 12628, 12627, 12641, 12643, 12615, 12676, 12664, + 12665, 12657, 12619, 12593, 12595, 12609, 12654, 12656, 12655, 12596, + 12597, 12646, 12598, 12648, 12647, 12610, 12612, 12660, 12661, 12663, + 12659, 12662, 12658, 12621, 12671, 12601, 12602, 12649, 12603, 12607, + 12604, 12651, 12652, 12606, 12650, 12653, 12608, 12605, 12620, 12599, + 12613, 12670, 12666, 12667, 12669, 12668, 12617, 12594, 12645, 12611, + 12600, 12677, 12614, 12672, 12638, 12637, 12632, 12633, 12639, 12630, + 12629, 12678, 12673, 12675, 12674, 12635, 12679, 12680, 12681, 12640, + 12683, 12682, 12684, 12625, 12626, 12642, 12622, 12631, 12634, 12636, + 12644, 12323, 12346, 12345, 12322, 12344, 12325, 12324, 12327, 12326, + 12329, 12321, 12328, 68875, 68874, 68887, 68889, 68872, 68881, 68868, + 68867, 68877, 68876, 68890, 68891, 68885, 68880, 68879, 68865, 68870, + 68873, 68882, 68871, 68869, 68883, 68884, 68866, 68892, 68886, 68888, + 68878, 68864, 68899, 68898, 68901, 68903, 68902, 68900, 68893, 68896, + 68894, 68897, 68895, 68917, 68916, 68919, 68918, 68915, 68914, 68912, + 68921, 68913, 68920, 5925, 5928, 5930, 5927, 5924, 5937, 5923, 5934, + 5931, 5929, 5933, 5936, 5926, 5935, 5932, 5920, 5921, 5922, 5940, 5938, + 5939, 128587, 128436, 129673, 128035, 67808, 67823, 67814, 67816, 67829, 67819, 67826, 67811, 67810, 67822, 67825, 67828, 67817, 67812, 67815, 67818, 67809, 67821, 67813, 67824, 67820, 67835, 67839, 67838, 67837, - 67836, 128891, 128436, 11233, 129702, 9980, 127911, 127892, 128157, - 128152, 128159, 129782, 128585, 129180, 129183, 129182, 129181, 128628, - 10033, 10008, 10149, 10150, 10084, 10167, 10169, 10168, 10054, 10004, - 11096, 9955, 11095, 11097, 10152, 10144, 10135, 129055, 129051, 10077, - 10078, 128178, 128977, 10040, 128975, 10059, 128958, 10071, 10082, - 129008, 10020, 128970, 128946, 129943, 10083, 11093, 128327, 10096, - 10094, 129052, 129048, 10080, 10079, 10157, 128627, 10006, 10134, 10012, - 11094, 10030, 10097, 10095, 10137, 129054, 129050, 10140, 128635, 128940, - 10075, 10076, 128972, 128952, 128607, 128615, 10136, 128605, 128613, - 128625, 10056, 128606, 128614, 10138, 128604, 128612, 10051, 10045, - 10171, 10142, 128980, 128979, 129053, 129049, 10158, 9947, 128903, - 128913, 10132, 10173, 128633, 10133, 10074, 10010, 1442, 1453, 1447, - 1436, 1437, 1438, 1445, 1446, 1443, 1444, 1433, 1441, 1425, 1439, 1448, - 1426, 1427, 1449, 1440, 1435, 1430, 1450, 1434, 1428, 1429, 1432, 1454, - 1431, 1451, 1452, 1488, 64304, 64302, 64303, 64288, 64297, 1506, 1489, - 64332, 64305, 1499, 64333, 64315, 1508, 64334, 64324, 1498, 64314, 1507, - 64323, 1509, 1503, 1501, 1511, 64327, 1492, 64308, 1495, 1504, 64320, - 1494, 64310, 1505, 64321, 1513, 64329, 64300, 64301, 64298, 64299, 1512, - 64328, 1496, 64312, 1514, 64330, 1510, 64326, 1491, 64307, 1490, 64306, - 1500, 64316, 1502, 64318, 1493, 64331, 64309, 64293, 64289, 64296, 64295, - 64290, 64292, 64291, 64294, 1497, 64285, 64313, 64335, 1520, 1522, 64287, - 1521, 1477, 1476, 1455, 1468, 1458, 1459, 1457, 1460, 1465, 1466, 1463, - 64286, 1464, 1479, 1467, 1471, 1462, 1473, 1456, 1474, 1461, 1469, 1524, - 1523, 1472, 1475, 1478, 1470, 1519, 9937, 9096, 11263, 128641, 110597, - 110594, 110595, 110596, 110778, 110779, 110780, 110781, 110782, 110783, - 110784, 110785, 110750, 110759, 110760, 110751, 110752, 110753, 110754, - 110755, 110756, 110757, 110758, 110768, 110769, 110770, 110771, 110772, - 110773, 110774, 110775, 110776, 110777, 110761, 110762, 110763, 110764, - 110765, 110766, 110767, 110615, 110624, 110625, 110626, 110616, 110617, - 110618, 110619, 110620, 110621, 110622, 110623, 110651, 110648, 110649, - 110650, 110627, 110628, 110629, 110630, 110631, 110632, 110633, 110634, - 110642, 110643, 110644, 110645, 110646, 110647, 110635, 110636, 110637, - 110638, 110639, 110640, 110641, 110806, 110804, 110805, 110807, 110808, - 110809, 110810, 110811, 110812, 110786, 110787, 110788, 110789, 110790, - 110791, 110792, 110793, 110794, 110795, 110796, 110797, 110798, 110799, - 110800, 110801, 110802, 110803, 110744, 110738, 110739, 110740, 110741, - 110742, 110743, 110734, 110727, 110728, 110729, 110730, 110731, 110732, - 110733, 110718, 110719, 110720, 110721, 110722, 110723, 110724, 110725, - 110726, 110745, 110746, 110747, 110748, 110749, 110735, 110736, 110737, - 110877, 110878, 110850, 110851, 110852, 110853, 110854, 110855, 110840, - 110841, 110842, 110843, 110844, 110845, 110833, 110834, 110835, 110836, - 110837, 110838, 110839, 110829, 110830, 110831, 110832, 110846, 110847, - 110848, 110849, 110652, 110653, 110654, 110655, 110656, 110657, 110658, - 110659, 110666, 110667, 110668, 110669, 110670, 110671, 110672, 110673, - 110660, 110661, 110662, 110663, 110664, 110665, 110674, 110675, 110676, - 110677, 110678, 110679, 110680, 110681, 110682, 110683, 110684, 110685, - 110702, 110703, 110704, 110705, 110706, 110707, 110708, 110709, 110710, - 110717, 110711, 110712, 110713, 110714, 110715, 110716, 110701, 110697, - 110698, 110699, 110700, 110690, 110691, 110692, 110693, 110694, 110695, - 110696, 110686, 110687, 110688, 110689, 110856, 110857, 110858, 110859, - 110860, 110861, 110862, 110863, 110864, 110865, 110870, 110871, 110872, - 110873, 110874, 110875, 110876, 110866, 110867, 110868, 110869, 110818, - 110813, 110814, 110815, 110816, 110817, 110823, 110824, 110825, 110826, - 110827, 110828, 110819, 110820, 110821, 110822, 983271, 110607, 110608, - 110609, 110610, 110611, 110602, 110603, 110604, 110605, 110606, 110612, - 110613, 110614, 110598, 110599, 110600, 110601, 8889, 127807, 19922, - 19966, 19958, 19967, 19924, 19946, 19923, 19909, 19947, 19944, 19943, - 19956, 19906, 19962, 19935, 19939, 19920, 19916, 19948, 19917, 19937, - 19931, 19929, 19925, 19911, 19964, 19928, 19945, 19934, 19918, 19930, - 19942, 19950, 19941, 19949, 19938, 19914, 19927, 19936, 19952, 19965, - 19912, 19915, 19926, 19954, 19910, 19932, 19953, 19904, 19933, 19940, - 19905, 19951, 19959, 19957, 19960, 19955, 19961, 19913, 19908, 19921, - 19963, 19907, 19919, 129428, 128262, 9889, 983123, 128644, 128645, - 128096, 12447, 12354, 110879, 110593, 12430, 110929, 110928, 110930, - 12419, 12423, 12421, 12437, 12438, 110898, 12387, 12353, 12359, 12355, - 12361, 12357, 12373, 12379, 12375, 12381, 12377, 12403, 983997, 984000, - 983998, 984001, 983999, 12400, 12409, 12412, 12406, 12435, 12394, 12397, - 12395, 12398, 12396, 12384, 12391, 12386, 12393, 12389, 12364, 12370, - 12366, 12372, 12368, 12399, 12408, 12402, 12411, 12405, 12363, 12369, - 12365, 12371, 12367, 12414, 12417, 12415, 12418, 12416, 12401, 12410, - 12404, 12413, 12407, 12425, 12428, 12426, 12429, 12427, 12383, 12390, - 12385, 12392, 12388, 12374, 12380, 12376, 12382, 12378, 12431, 12433, - 12432, 12434, 12420, 12424, 12422, 12436, 12360, 12356, 12362, 12358, - 12446, 12445, 9964, 128725, 129406, 127802, 129435, 128616, 128617, - 128371, 128029, 127855, 11203, 11043, 8213, 9135, 129921, 129910, 129911, - 129912, 129913, 129914, 129915, 9146, 9147, 9148, 9149, 983059, 983141, - 983138, 11134, 128677, 8230, 9897, 128014, 127943, 128052, 127798, 9749, - 9832, 127789, 127976, 8987, 9203, 8962, 127969, 127968, 127960, 127973, - 128298, 8763, 129693, 983124, 983060, 983142, 983139, 128559, 128175, - 129303, 128726, 8208, 11802, 8259, 8231, 45, 11794, 9102, 11226, 129723, - 8372, 127848, 129482, 127954, 9976, 8801, 10725, 10855, 129706, 12696, - 12700, 12693, 12697, 12695, 12703, 12692, 12699, 12691, 12694, 12688, - 12698, 12690, 12689, 12701, 12702, 12294, 12289, 12275, 12273, 12274, - 12272, 12283, 12282, 12285, 12279, 12280, 12281, 12278, 12277, 12284, - 12783, 12287, 12276, 12286, 12332, 12295, 119670, 119669, 119668, 119667, - 119666, 12999, 12995, 13309, 13310, 13292, 13282, 13299, 13304, 13303, - 13306, 13305, 13302, 13301, 13308, 13300, 13307, 13291, 13281, 13289, - 13287, 13297, 13290, 13294, 13284, 13283, 13293, 13288, 13298, 13286, - 13296, 13285, 13295, 13280, 13003, 13164, 13168, 13167, 13166, 13165, - 13156, 13146, 13157, 13147, 13154, 13152, 13162, 13155, 13159, 13149, - 13148, 13158, 13153, 13163, 13151, 13161, 13150, 13160, 13144, 13145, - 12992, 12997, 12998, 12993, 12994, 12996, 13002, 13000, 13001, 12343, - 12351, 12333, 12331, 12330, 12350, 12290, 12293, 12288, 8887, 8787, - 128127, 67676, 67673, 67675, 67679, 67674, 67672, 67677, 67678, 67656, - 67669, 67651, 67659, 67666, 67667, 67648, 67663, 67650, 67654, 67662, - 67665, 67668, 67657, 67652, 67655, 67658, 67649, 67661, 67653, 67664, - 67660, 67671, 128232, 10716, 128474, 10721, 8710, 983130, 983129, 129781, + 67836, 128891, 11233, 129702, 9980, 127911, 127892, 128157, 128152, + 128159, 129782, 128585, 129180, 129183, 129182, 129181, 128628, 10033, + 10008, 10149, 10150, 10084, 10167, 10169, 10168, 10054, 10004, 11096, + 9955, 11095, 11097, 10152, 10144, 10135, 129055, 129051, 10077, 10078, + 128178, 128977, 10040, 128975, 10059, 128958, 10071, 10082, 129008, + 10020, 128970, 128946, 129943, 10083, 11093, 128327, 10096, 10094, + 129052, 129048, 10080, 10079, 10157, 128627, 10006, 10134, 10012, 11094, + 10030, 10097, 10095, 10137, 129054, 129050, 10140, 128635, 128940, 10075, + 10076, 128972, 128952, 128607, 128615, 10136, 128605, 128613, 128625, + 10056, 128606, 128614, 10138, 128604, 128612, 10051, 10045, 10171, 10142, + 128980, 128979, 129053, 129049, 10158, 128913, 118277, 9947, 128903, + 10132, 10173, 128633, 10133, 10074, 10010, 1442, 1453, 1447, 1436, 1437, + 1438, 1445, 1446, 1443, 1444, 1433, 1441, 1425, 1439, 1448, 1426, 1427, + 1449, 1440, 1435, 1430, 1450, 1434, 1428, 1429, 1432, 1454, 1431, 1451, + 1452, 1488, 64304, 64302, 64303, 64288, 64297, 1506, 1489, 64332, 64305, + 1499, 64333, 64315, 1508, 64334, 64324, 1498, 64314, 1507, 64323, 1509, + 1503, 1501, 1511, 64327, 1492, 64308, 1495, 1504, 64320, 1494, 64310, + 1505, 64321, 1513, 64329, 64300, 64301, 64298, 64299, 1512, 64328, 1496, + 64312, 1514, 64330, 1510, 64326, 1491, 64307, 1490, 64306, 1500, 64316, + 1502, 64318, 1493, 64331, 64309, 64293, 64289, 64296, 64295, 64290, + 64292, 64291, 64294, 1497, 64285, 64313, 64335, 1520, 1522, 64287, 1521, + 1477, 1476, 1455, 1468, 1458, 1459, 1457, 1460, 1465, 1466, 1463, 1464, + 1479, 1467, 1471, 1462, 1473, 1456, 1474, 1461, 64286, 1469, 1524, 1523, + 1472, 1475, 1478, 1470, 1519, 9937, 9096, 11263, 128641, 110597, 110594, + 110595, 110596, 110778, 110779, 110780, 110781, 110782, 110783, 110784, + 110785, 110750, 110759, 110760, 110751, 110752, 110753, 110754, 110755, + 110756, 110757, 110758, 110768, 110769, 110770, 110771, 110772, 110773, + 110774, 110775, 110776, 110777, 110761, 110762, 110763, 110764, 110765, + 110766, 110767, 110615, 110624, 110625, 110626, 110616, 110617, 110618, + 110619, 110620, 110621, 110622, 110623, 110651, 110648, 110649, 110650, + 110627, 110628, 110629, 110630, 110631, 110632, 110633, 110634, 110642, + 110643, 110644, 110645, 110646, 110647, 110635, 110636, 110637, 110638, + 110639, 110640, 110641, 110806, 110804, 110805, 110807, 110808, 110809, + 110810, 110811, 110812, 110786, 110787, 110788, 110789, 110790, 110791, + 110792, 110793, 110794, 110795, 110796, 110797, 110798, 110799, 110800, + 110801, 110802, 110803, 110744, 110738, 110739, 110740, 110741, 110742, + 110743, 110734, 110727, 110728, 110729, 110730, 110731, 110732, 110733, + 110718, 110719, 110720, 110721, 110722, 110723, 110724, 110725, 110726, + 110745, 110746, 110747, 110748, 110749, 110735, 110736, 110737, 110877, + 110878, 110850, 110851, 110852, 110853, 110854, 110855, 110840, 110841, + 110842, 110843, 110844, 110845, 110833, 110834, 110835, 110836, 110837, + 110838, 110839, 110829, 110830, 110831, 110832, 110846, 110847, 110848, + 110849, 110652, 110653, 110654, 110655, 110656, 110657, 110658, 110659, + 110666, 110667, 110668, 110669, 110670, 110671, 110672, 110673, 110660, + 110661, 110662, 110663, 110664, 110665, 110674, 110675, 110676, 110677, + 110678, 110679, 110680, 110681, 110682, 110683, 110684, 110685, 110702, + 110703, 110704, 110705, 110706, 110707, 110708, 110709, 110710, 110717, + 110711, 110712, 110713, 110714, 110715, 110716, 110701, 110697, 110698, + 110699, 110700, 110690, 110691, 110692, 110693, 110694, 110695, 110696, + 110686, 110687, 110688, 110689, 110856, 110857, 110858, 110859, 110860, + 110861, 110862, 110863, 110864, 110865, 110870, 110871, 110872, 110873, + 110874, 110875, 110876, 110866, 110867, 110868, 110869, 110818, 110813, + 110814, 110815, 110816, 110817, 110823, 110824, 110825, 110826, 110827, + 110828, 110819, 110820, 110821, 110822, 983273, 110607, 110608, 110609, + 110610, 110611, 110602, 110603, 110604, 110605, 110606, 110612, 110613, + 110614, 110598, 110599, 110600, 110601, 8889, 127807, 19922, 19966, + 19958, 19967, 19924, 19946, 19923, 19909, 19947, 19944, 19943, 19956, + 19906, 19962, 19935, 19939, 19920, 19916, 19948, 19917, 19937, 19931, + 19929, 19925, 19911, 19928, 19964, 19945, 19934, 19918, 19930, 19942, + 19950, 19941, 19949, 19938, 19914, 19927, 19936, 19952, 19965, 19912, + 19915, 19926, 19954, 19910, 19932, 19953, 19904, 19933, 19940, 19905, + 19951, 19959, 19957, 19960, 19955, 19961, 19913, 19908, 19921, 19963, + 19907, 19919, 129428, 128262, 9889, 983123, 128644, 128645, 128096, + 12447, 12354, 110879, 110593, 12430, 110929, 110928, 110930, 12419, + 12423, 12421, 12437, 12438, 110898, 12387, 12353, 12359, 12355, 12361, + 12357, 12373, 12379, 12375, 12381, 12377, 12403, 983997, 984000, 983998, + 984001, 983999, 12400, 12409, 12412, 12406, 12435, 12394, 12397, 12395, + 12398, 12396, 12384, 12391, 12386, 12393, 12389, 12364, 12370, 12366, + 12372, 12368, 12399, 12408, 12402, 12411, 12405, 12363, 12369, 12365, + 12371, 12367, 12414, 12417, 12415, 12418, 12416, 12401, 12410, 12404, + 12413, 12407, 12425, 12428, 12426, 12429, 12427, 12383, 12390, 12385, + 12392, 12388, 12374, 12380, 12376, 12382, 12378, 12431, 12433, 12432, + 12434, 12420, 12424, 12422, 12436, 12360, 12356, 12362, 12358, 12446, + 12445, 9964, 128725, 129406, 127802, 129435, 128616, 128617, 128371, + 128029, 127855, 11203, 11043, 8213, 118290, 118287, 117905, 9135, 117893, + 129921, 129910, 129911, 129912, 129913, 129914, 129915, 9146, 9147, 9148, + 9149, 983059, 983141, 983138, 11134, 128677, 117779, 8230, 9897, 117915, + 117769, 118448, 128014, 127943, 128052, 127798, 9749, 9832, 127789, + 127976, 8987, 9203, 8962, 127969, 127968, 127960, 127973, 128298, 8763, + 129693, 983124, 983060, 983142, 983139, 128559, 128175, 129303, 128726, + 8208, 11802, 8259, 8231, 45, 11794, 9102, 11226, 129723, 8372, 127848, + 129482, 127954, 9976, 8801, 10725, 10855, 129706, 12696, 12700, 12693, + 12697, 12695, 12703, 12692, 12699, 12691, 12694, 12688, 12698, 12690, + 12689, 12701, 12702, 12294, 12289, 12275, 12273, 12274, 12272, 12283, + 12282, 12285, 12279, 12280, 12281, 12278, 12277, 12284, 12783, 12287, + 12276, 12286, 12332, 12295, 119670, 119669, 119668, 119667, 119666, + 12999, 12995, 13309, 13310, 13292, 13282, 13299, 13304, 13303, 13306, + 13305, 13302, 13301, 13308, 13300, 13307, 13291, 13281, 13289, 13287, + 13297, 13290, 13294, 13284, 13283, 13293, 13288, 13298, 13286, 13296, + 13285, 13295, 13280, 13003, 13164, 13168, 13167, 13166, 13165, 13156, + 13146, 13157, 13147, 13154, 13152, 13162, 13155, 13159, 13149, 13148, + 13158, 13153, 13163, 13151, 13161, 13150, 13160, 13144, 13145, 12992, + 12997, 12998, 12993, 12994, 12996, 13002, 13000, 13001, 12343, 12351, + 12333, 12331, 12330, 12350, 12290, 12293, 12288, 8887, 8787, 128127, + 67676, 67673, 67675, 67679, 67674, 67672, 67677, 67678, 67656, 67669, + 67651, 67659, 67666, 67667, 67648, 67663, 67650, 67654, 67662, 67665, + 67668, 67657, 67652, 67655, 67658, 67649, 67661, 67653, 67664, 67660, + 67671, 128232, 10716, 128474, 10721, 8710, 983130, 983129, 129781, 126132, 126112, 126126, 126125, 126127, 126131, 126130, 126129, 126113, 126114, 126110, 126111, 126072, 126081, 126108, 126090, 126099, 126078, 126105, 126069, 126087, 126096, 126077, 126104, 126068, 126086, 126095, @@ -10521,235 +11261,248 @@ static const unsigned int dawg_pos_to_codepoint[] = { 68428, 68445, 68441, 68442, 68444, 68440, 68446, 68447, 68443, 9088, 8747, 10767, 10773, 10776, 10780, 10778, 10775, 10777, 10779, 10766, 9134, 65529, 65531, 65530, 9892, 8745, 10825, 10823, 10820, 10819, 10816, - 10827, 8253, 8890, 10812, 129942, 129969, 129972, 9689, 129936, 9688, - 11800, 11845, 11846, 8766, 9959, 161, 8487, 8276, 191, 8290, 8292, 8291, - 128229, 127982, 129311, 127875, 127983, 127971, 12292, 9979, 127886, - 128122, 128121, 128304, 43453, 43454, 43455, 43457, 43421, 43422, 43426, - 43427, 43398, 43397, 43399, 43407, 43408, 43409, 43412, 43402, 43403, - 43418, 43416, 43428, 43423, 43431, 43432, 43413, 43414, 43410, 43411, - 43429, 43430, 43401, 43435, 43436, 43441, 43439, 43440, 43424, 43425, - 43419, 43420, 43415, 43417, 43396, 43405, 43442, 43437, 43433, 43438, - 43434, 43404, 43406, 43400, 43458, 43466, 43467, 43459, 43465, 43461, - 43464, 43468, 43463, 43486, 43462, 43487, 43460, 43471, 43456, 43469, - 43393, 43443, 43392, 43395, 43394, 43448, 43449, 43444, 43450, 43445, - 43446, 43447, 43452, 43451, 43477, 43476, 43479, 43478, 43475, 43474, - 43472, 43481, 43473, 43480, 129753, 129724, 128086, 128377, 10781, - 129337, 9795, 9909, 129513, 69786, 69787, 69785, 69793, 69792, 69763, - 69764, 69770, 69772, 69784, 69783, 69791, 69790, 69767, 69768, 69765, - 69766, 69777, 69789, 69782, 69794, 69804, 69805, 69806, 69798, 69797, - 69779, 69778, 69776, 69775, 69781, 69780, 69774, 69773, 69796, 69795, - 69788, 69801, 69807, 69802, 69799, 69803, 69800, 69769, 69771, 69760, - 69818, 69761, 69817, 69762, 69822, 69823, 69825, 69824, 69826, 69808, - 69814, 69816, 69811, 69812, 69809, 69810, 69813, 69815, 69821, 69837, - 69819, 69820, 12142, 12164, 12060, 12157, 12100, 12149, 12184, 12191, - 12227, 12068, 12174, 12234, 12205, 12134, 12168, 12219, 12189, 12088, - 12090, 12096, 12160, 12182, 12224, 12190, 12147, 12114, 12118, 12058, - 12176, 12075, 12112, 12045, 12170, 12124, 12070, 12194, 12109, 12229, - 12196, 12139, 12056, 12099, 12034, 12084, 12136, 12120, 12044, 12111, - 12094, 12125, 12243, 12238, 12082, 12159, 12063, 12215, 12062, 12042, - 12241, 12067, 12235, 12043, 12140, 12119, 12207, 12123, 12133, 12222, - 12117, 12226, 12245, 12214, 12217, 12236, 12155, 12188, 12113, 12065, - 12198, 12066, 12146, 12171, 12225, 12200, 12121, 12093, 12095, 12221, - 12092, 12216, 12231, 12054, 12218, 12179, 12037, 12173, 12072, 12046, - 12127, 12152, 12049, 12074, 12107, 12208, 12212, 12041, 12210, 12131, - 12033, 12039, 12199, 12085, 12128, 12161, 12162, 12233, 12165, 12192, - 12077, 12201, 12061, 12105, 12040, 12240, 12102, 12153, 12032, 12080, - 12167, 12048, 12156, 12059, 12126, 12158, 12050, 12183, 12204, 12097, - 12239, 12053, 12078, 12150, 12071, 12187, 12186, 12223, 12228, 12104, - 12098, 12064, 12036, 12057, 12163, 12178, 12185, 12154, 12083, 12203, - 12087, 12135, 12151, 12202, 12035, 12122, 12141, 12180, 12144, 12076, - 12052, 12115, 12091, 12108, 12169, 12143, 12148, 12130, 12089, 12211, - 12073, 12101, 12138, 12103, 12209, 12047, 12220, 12172, 12129, 12166, - 12242, 12237, 12145, 12106, 12081, 12244, 12038, 12086, 12055, 12181, - 12197, 12193, 12175, 12116, 12110, 12177, 12137, 12230, 12213, 12195, - 12069, 12079, 12206, 12051, 12232, 12132, 129432, 983205, 3251, 3250, - 3240, 3293, 3225, 3235, 3230, 3249, 3248, 3205, 3206, 3216, 3220, 3234, - 3233, 3239, 3238, 3232, 3231, 3237, 3236, 3211, 3296, 3212, 3297, 3253, - 3209, 3210, 3218, 3219, 3207, 3208, 3254, 3255, 3256, 3245, 3244, 3227, - 3226, 3224, 3223, 3229, 3228, 3222, 3221, 3243, 3242, 3214, 3215, 3294, - 3257, 3246, 3247, 3285, 3315, 3201, 3200, 3204, 3261, 3202, 3260, 3313, - 3314, 3277, 3203, 3286, 3262, 3272, 3276, 3267, 3268, 3298, 3299, 3265, - 3266, 3274, 3275, 3263, 3264, 3270, 3271, 3307, 3306, 3309, 3308, 3305, - 3304, 3302, 3311, 3303, 3310, 12450, 984009, 984008, 984007, 984010, - 110881, 110880, 110882, 110592, 12499, 984002, 984005, 984003, 984006, - 984004, 12496, 12505, 12508, 12502, 12511, 110583, 110584, 110585, - 110586, 110587, 110589, 110590, 110576, 110577, 110578, 110579, 110581, - 110582, 12510, 12513, 12514, 12512, 12531, 12490, 12493, 12491, 12494, - 12492, 12789, 12792, 12790, 12793, 12791, 12795, 12798, 12796, 12799, - 12797, 12533, 12534, 110933, 12784, 12787, 12483, 12526, 110949, 110948, - 110950, 12515, 12519, 12517, 110951, 12788, 12785, 12786, 12794, 12449, - 12455, 12451, 12457, 12453, 12469, 12475, 12471, 12477, 12473, 12480, - 12487, 12482, 12489, 12485, 12460, 12466, 12462, 12468, 12464, 12495, - 12504, 12498, 12507, 12501, 12459, 12465, 12461, 12467, 12463, 12497, - 12506, 12500, 12509, 12503, 12521, 12524, 12522, 12525, 12523, 12479, - 12486, 12481, 12488, 12484, 12535, 12537, 12536, 12538, 12532, 12470, - 12476, 12472, 12478, 12474, 12527, 12529, 12528, 12530, 12516, 12520, - 12518, 12456, 12452, 12458, 12454, 12542, 12543, 12541, 12539, 12448, - 12540, 12444, 12443, 73476, 73477, 73487, 73523, 73498, 73497, 73503, - 73502, 73508, 73507, 73501, 73500, 73506, 73505, 73482, 73483, 73484, - 73485, 73480, 73481, 73478, 73479, 73494, 73504, 73499, 73509, 73519, - 73520, 73521, 73513, 73512, 73496, 73495, 73493, 73492, 73491, 73490, - 73511, 73510, 73522, 73517, 73514, 73516, 73518, 73515, 73486, 73488, - 73551, 73548, 73546, 73545, 73549, 73543, 73541, 73544, 73550, 73542, - 73547, 73474, 73537, 73472, 73475, 73473, 73525, 73524, 73535, 73530, - 73534, 73536, 73528, 73529, 73526, 73527, 73540, 73539, 73557, 73556, - 73559, 73558, 73555, 73554, 73552, 73561, 73553, 73560, 73538, 43283, - 43295, 43299, 43301, 43277, 43281, 43284, 43275, 43274, 43286, 43285, - 43279, 43278, 43294, 43282, 43289, 43297, 43288, 43276, 43292, 43287, - 43290, 43296, 43293, 43291, 43280, 43298, 43300, 43310, 43311, 43308, - 43309, 43307, 43303, 43305, 43304, 43302, 43306, 43269, 43268, 43271, - 43270, 43267, 43266, 43264, 43273, 43265, 43272, 119496, 119506, 119499, - 119503, 119493, 119492, 119502, 119497, 119507, 119495, 119505, 119494, - 119504, 119501, 119491, 119500, 119490, 119498, 119488, 119489, 128331, - 8490, 128273, 9000, 128422, 983552, 983553, 983559, 983558, 983561, - 983560, 983557, 983556, 983554, 983563, 983555, 983562, 128287, 68163, - 68162, 68161, 68160, 68113, 68146, 68112, 68147, 68148, 68123, 68122, - 68128, 68127, 68126, 68121, 68131, 68125, 68124, 68130, 68129, 68141, - 68142, 68143, 68135, 68134, 68118, 68117, 68115, 68114, 68133, 68132, - 68149, 68140, 68145, 68119, 68139, 68136, 68138, 68137, 68144, 68096, - 68165, 68164, 68166, 68167, 68179, 68178, 68183, 68176, 68182, 68181, - 68184, 68180, 68177, 68152, 68153, 68109, 68154, 68111, 68110, 68099, - 68101, 68097, 68102, 68098, 68108, 68159, 68168, 129711, 101120, 101121, - 101122, 101123, 101124, 101125, 101126, 101127, 101128, 101129, 101130, - 101131, 101132, 101133, 101134, 101135, 101136, 101137, 101138, 101139, - 101140, 101141, 101142, 101143, 101144, 101145, 101146, 101147, 101148, - 101149, 101150, 101151, 101152, 101153, 101154, 101155, 101156, 101157, - 101158, 101159, 101160, 101161, 101162, 101163, 101164, 101165, 101166, - 101167, 101168, 101169, 101170, 101171, 101172, 101173, 101174, 101175, - 101176, 101177, 101178, 101179, 101180, 101181, 101182, 101183, 101184, - 101185, 101186, 101187, 101188, 101189, 101190, 101191, 101192, 101193, - 101194, 101195, 101196, 101197, 101198, 101199, 101200, 101201, 101202, - 101203, 101204, 101205, 101206, 101207, 101208, 101209, 101210, 101211, - 101212, 101213, 101214, 101215, 101216, 101217, 101218, 101219, 101220, - 101221, 101222, 101223, 101224, 101225, 101226, 101227, 101228, 101229, - 101230, 101231, 101232, 101233, 101234, 101235, 101236, 101237, 101238, - 101239, 101240, 101241, 101242, 101243, 101244, 101245, 101246, 101247, - 101248, 101249, 101250, 101251, 101252, 101253, 101254, 101255, 101256, - 101257, 101258, 101259, 101260, 101261, 101262, 101263, 101264, 101265, - 101266, 101267, 101268, 101269, 101270, 101271, 101272, 101273, 101274, - 101275, 101276, 101277, 101278, 101279, 101280, 101281, 101282, 101283, - 101284, 101285, 101286, 101287, 101288, 101289, 101290, 101291, 101292, - 101293, 101294, 101295, 101296, 101297, 101298, 101299, 101300, 101301, - 101302, 101303, 101304, 101305, 101306, 101307, 101308, 101309, 101310, - 101311, 101312, 101313, 101314, 101315, 101316, 101317, 101318, 101319, - 101320, 101321, 101322, 101323, 101324, 101325, 101326, 101327, 101328, - 101329, 101330, 101331, 101332, 101333, 101334, 101335, 101336, 101337, - 101338, 101339, 101340, 101341, 101342, 101343, 101344, 101345, 101346, - 101347, 101348, 101349, 101350, 101351, 101352, 101353, 101354, 101355, - 101356, 101357, 101358, 101359, 101360, 101361, 101362, 101363, 101364, - 101365, 101366, 101367, 101368, 101369, 101370, 101371, 101372, 101373, - 101374, 101375, 101584, 101585, 101586, 101587, 101588, 101589, 101376, - 101377, 101378, 101379, 101380, 101381, 101382, 101383, 101384, 101385, - 101386, 101387, 101388, 101389, 101390, 101391, 101392, 101393, 101394, - 101395, 101396, 101397, 101398, 101399, 101400, 101401, 101402, 101403, - 101404, 101405, 101406, 101407, 101408, 101409, 101410, 101411, 101412, - 101413, 101414, 101415, 101416, 101417, 101418, 101419, 101420, 101421, - 101422, 101423, 101424, 101425, 101426, 101427, 101428, 101429, 101430, - 101431, 101432, 101433, 101434, 101435, 101436, 101437, 101438, 101439, - 101440, 101441, 101442, 101443, 101444, 101445, 101446, 101447, 101448, - 101449, 101450, 101451, 101452, 101453, 101454, 101455, 101456, 101457, - 101458, 101459, 101460, 101461, 101462, 101463, 101464, 101465, 101466, - 101467, 101468, 101469, 101470, 101471, 101472, 101473, 101474, 101475, - 101476, 101477, 101478, 101479, 101480, 101481, 101482, 101483, 101484, - 101485, 101486, 101487, 101488, 101489, 101490, 101491, 101492, 101493, - 101494, 101495, 101496, 101497, 101498, 101499, 101500, 101501, 101502, - 101503, 101504, 101505, 101506, 101507, 101508, 101509, 101510, 101511, - 101512, 101513, 101514, 101515, 101516, 101517, 101518, 101519, 101520, - 101521, 101522, 101523, 101524, 101525, 101526, 101527, 101528, 101529, - 101530, 101531, 101532, 101533, 101534, 101535, 101536, 101537, 101538, - 101539, 101540, 101541, 101542, 101543, 101544, 101545, 101546, 101547, - 101548, 101549, 101550, 101551, 101552, 101553, 101554, 101555, 101556, - 101557, 101558, 101559, 101560, 101561, 101562, 101563, 101564, 101565, - 101566, 101567, 101568, 101569, 101570, 101571, 101572, 101573, 101574, - 101575, 101576, 101577, 101578, 101579, 101580, 101581, 101582, 101583, - 94180, 983960, 983965, 983970, 983975, 983962, 983964, 983961, 983963, - 983957, 983959, 983956, 983958, 983977, 983979, 983978, 983972, 983974, - 983967, 983969, 983971, 983973, 983966, 983968, 983989, 983983, 983985, - 983986, 983987, 983980, 983982, 983984, 983981, 983976, 983988, 6107, - 6052, 6064, 6051, 6067, 6066, 6065, 6055, 6057, 6058, 6056, 6053, 6054, - 6063, 983994, 983991, 983992, 983993, 6061, 6062, 6059, 6060, 6022, 6024, - 6021, 6023, 6017, 6019, 6016, 6018, 6020, 6030, 6025, 6035, 6037, 6039, - 6038, 6046, 6045, 6047, 6032, 6034, 6027, 6029, 6031, 6033, 6026, 6028, - 6049, 6043, 6040, 6042, 6044, 6041, 6036, 6048, 6050, 6108, 6109, 6095, - 6096, 6099, 6091, 6101, 6104, 6102, 6098, 6094, 6100, 6106, 6092, 6087, - 6093, 6090, 6103, 6105, 6088, 6086, 6089, 6097, 6652, 6636, 6655, 6639, - 6653, 6637, 6654, 6638, 6651, 6635, 6650, 6634, 6133, 6137, 6136, 6134, - 6135, 6130, 6132, 6131, 6129, 6128, 6624, 6648, 6632, 6649, 6633, 6647, - 6631, 6646, 6630, 6645, 6629, 6642, 6626, 6640, 6643, 6627, 6644, 6628, - 6641, 6625, 6069, 6068, 6070, 983996, 6082, 6083, 6085, 6071, 6080, 6072, - 6078, 983995, 6084, 6073, 6079, 6074, 983990, 6075, 6077, 6076, 6081, - 6117, 6116, 6119, 6118, 6115, 6114, 6112, 6121, 6113, 6120, 70204, 70201, - 70200, 70208, 70185, 70178, 70179, 70177, 70155, 70156, 70154, 70172, - 70167, 70166, 70173, 70171, 70161, 70160, 70144, 70145, 70149, 70151, - 70165, 70164, 70170, 70169, 70187, 70183, 70157, 70168, 70163, 70174, - 70159, 70158, 70153, 70152, 70176, 70175, 70186, 70180, 70207, 70182, - 70184, 70181, 70148, 70146, 70150, 70147, 70199, 70206, 70197, 70198, - 70196, 70203, 70209, 70188, 70193, 70195, 70189, 70190, 70192, 70194, - 70191, 70202, 70205, 70357, 70358, 70356, 70333, 70334, 70332, 70340, - 70339, 70338, 70345, 70347, 70344, 70352, 70351, 70346, 70361, 70320, - 70321, 70327, 70329, 70343, 70342, 70350, 70349, 70324, 70325, 70322, - 70323, 70335, 70348, 70341, 70353, 70337, 70336, 70331, 70330, 70355, - 70354, 70364, 70365, 70366, 70362, 70359, 70363, 70360, 70326, 70328, - 70378, 70377, 70367, 70368, 70374, 70376, 70371, 70372, 70369, 70370, - 70373, 70375, 70389, 70388, 70391, 70390, 70387, 70386, 70384, 70393, - 70385, 70392, 128143, 128535, 128537, 128538, 128573, 128139, 129373, - 8365, 128088, 129665, 129486, 129698, 12927, 128040, 11235, 129404, - 127991, 129357, 128030, 129692, 3805, 3804, 983206, 983207, 3743, 3741, - 3807, 3806, 3714, 3716, 3713, 983209, 3747, 3749, 3730, 3729, 3736, 3728, - 3727, 3731, 3726, 3744, 3721, 3718, 3724, 3756, 3740, 3742, 3739, 3752, - 3753, 3754, 3722, 3734, 3735, 3733, 3755, 3758, 3719, 3725, 3737, 3738, - 3720, 3732, 3745, 983208, 3751, 3746, 3757, 3773, 3772, 3770, 3785, 3786, - 3787, 3784, 3760, 3762, 3780, 3763, 3779, 3761, 3771, 3766, 3767, 3768, - 3769, 3776, 3777, 3764, 3765, 3778, 3782, 3790, 3759, 3797, 3796, 3799, - 3798, 3795, 3794, 3792, 3801, 3793, 3800, 3788, 3789, 128996, 129003, - 128309, 128311, 128998, 128994, 129001, 68413, 68415, 128992, 128310, - 128999, 68412, 68414, 11004, 10201, 10200, 10782, 128995, 129002, 128308, - 128997, 128993, 129000, 9711, 10923, 10925, 8382, 9790, 127772, 127767, - 65, 258, 7858, 7856, 7854, 7862, 7860, 550, 480, 7840, 512, 196, 478, - 197, 506, 7680, 256, 983564, 194, 7848, 7846, 7844, 7852, 7850, 461, - 7842, 260, 983590, 983592, 192, 193, 514, 195, 570, 198, 508, 482, 42946, - 11373, 393, 42808, 42810, 42802, 42804, 42806, 42812, 66, 386, 42902, - 7686, 7684, 7682, 385, 579, 42822, 42932, 67, 199, 7688, 264, 268, 262, - 42948, 391, 42898, 266, 571, 42960, 42796, 42798, 42862, 42931, 68, 498, - 453, 42951, 272, 7696, 7698, 270, 395, 7694, 7692, 7690, 394, 497, 452, - 69, 552, 7708, 202, 983586, 7874, 7872, 7870, 983584, 7878, 7876, 7704, - 282, 278, 983598, 983600, 7864, 516, 203, 7868, 7706, 274, 7700, 7702, - 983570, 983572, 983574, 7866, 280, 983594, 983596, 200, 201, 518, 276, - 582, 439, 494, 440, 42786, 42788, 42858, 208, 425, 330, 70, 401, 7710, - 42904, 71, 290, 284, 486, 500, 286, 7712, 42912, 403, 288, 484, 577, - 42938, 42940, 42942, 404, 983199, 72, 7722, 7720, 292, 542, 7718, 11367, - 7716, 7714, 42922, 294, 11381, 502, 42790, 73, 520, 7882, 304, 207, 7726, - 206, 463, 298, 983566, 296, 7724, 7880, 302, 983604, 983606, 204, 205, - 522, 300, 407, 42873, 42875, 42877, 42882, 42884, 42886, 406, 42860, 74, - 308, 42930, 983608, 584, 75, 310, 488, 42818, 11369, 7730, 42816, 42820, - 7728, 7732, 42914, 408, 76, 42925, 573, 11360, 7734, 7736, 11362, 319, - 456, 321, 315, 7740, 317, 42824, 313, 7738, 983610, 455, 77, 7742, 7746, - 7744, 983612, 11374, 7930, 7932, 42966, 78, 325, 7754, 327, 459, 544, - 7752, 413, 504, 323, 42896, 7750, 7748, 42916, 209, 458, 79, 42826, - 42828, 332, 7760, 7762, 415, 212, 7892, 7890, 7888, 7896, 7894, 465, 214, - 554, 558, 560, 7884, 524, 336, 490, 492, 216, 510, 213, 7758, 7756, 556, - 983576, 983578, 983580, 416, 7902, 7900, 7898, 7906, 7904, 7886, 210, - 211, 526, 334, 42944, 400, 390, 42934, 418, 42830, 546, 80, 42834, 11363, - 42832, 42836, 7764, 420, 7766, 81, 42840, 42838, 82, 342, 344, 7770, - 7772, 7768, 528, 11364, 983614, 340, 7774, 530, 42918, 588, 42842, 42997, - 42923, 42814, 398, 42844, 83, 352, 7782, 350, 536, 348, 346, 7780, 7778, - 7784, 7776, 42953, 11390, 983582, 42920, 42949, 586, 42926, 42891, 7838, - 42968, 42924, 399, 84, 354, 7792, 538, 356, 574, 7788, 7786, 7790, 430, - 428, 358, 42878, 11375, 11376, 42893, 42928, 42880, 412, 42929, 581, 222, - 42852, 42854, 388, 423, 444, 42794, 42792, 85, 219, 7798, 467, 220, 473, - 475, 471, 469, 7794, 532, 368, 7908, 431, 7916, 7914, 7912, 7920, 7918, - 7910, 362, 7802, 983568, 983620, 983622, 370, 983616, 983618, 360, 7800, - 7796, 217, 218, 534, 364, 366, 42936, 580, 433, 86, 42846, 7806, 7804, - 434, 42850, 42906, 42908, 42910, 42856, 42848, 87, 372, 7812, 7816, 7814, - 7808, 7810, 11378, 503, 88, 7820, 7818, 89, 374, 376, 7924, 7822, 7922, - 435, 7926, 7934, 221, 562, 7928, 590, 540, 90, 7824, 381, 377, 11371, - 7826, 379, 7828, 11391, 437, 42950, 548, 306, 338, 10013, 43007, 43005, - 43006, 43003, 43004, 42999, 450, 7461, 684, 664, 685, 662, 122638, 446, - 426, 674, 451, 122634, 7431, 7430, 7459, 618, 641, 671, 122628, 7436, - 7439, 7440, 630, 7445, 640, 7438, 7449, 43846, 42870, 7451, 11387, + 10827, 8253, 8890, 10812, 117901, 117903, 9688, 129942, 129969, 129972, + 9689, 129936, 11800, 11845, 11846, 8766, 9959, 161, 8487, 8276, 191, + 8290, 8292, 8291, 128229, 127982, 129311, 127875, 127983, 127971, 12292, + 9979, 127886, 128304, 128122, 128121, 43453, 43454, 43455, 43457, 43421, + 43422, 43426, 43427, 43398, 43397, 43399, 43407, 43408, 43409, 43412, + 43402, 43403, 43418, 43416, 43428, 43423, 43431, 43432, 43413, 43414, + 43410, 43411, 43429, 43430, 43401, 43435, 43436, 43441, 43439, 43440, + 43424, 43425, 43419, 43420, 43415, 43417, 43396, 43405, 43442, 43437, + 43433, 43438, 43434, 43404, 43406, 43400, 43458, 43466, 43467, 43459, + 43465, 43461, 43464, 43468, 43463, 43486, 43462, 43487, 43460, 43471, + 43456, 43469, 43393, 43443, 43392, 43395, 43394, 43448, 43449, 43444, + 43450, 43445, 43446, 43447, 43452, 43451, 43477, 43476, 43479, 43478, + 43475, 43474, 43472, 43481, 43473, 43480, 129753, 129724, 128086, 128377, + 10781, 129337, 9795, 9909, 129513, 69786, 69787, 69785, 69793, 69792, + 69763, 69764, 69770, 69772, 69784, 69783, 69791, 69790, 69767, 69768, + 69765, 69766, 69777, 69789, 69782, 69794, 69804, 69805, 69806, 69798, + 69797, 69779, 69778, 69776, 69775, 69781, 69780, 69774, 69773, 69796, + 69795, 69788, 69801, 69807, 69802, 69799, 69803, 69800, 69769, 69771, + 69760, 69818, 69761, 69817, 69762, 69822, 69823, 69825, 69824, 69826, + 69808, 69814, 69816, 69811, 69812, 69809, 69810, 69813, 69815, 69821, + 69837, 69819, 69820, 12142, 12164, 12060, 12157, 12100, 12149, 12184, + 12191, 12227, 12068, 12174, 12234, 12205, 12134, 12168, 12219, 12189, + 12088, 12090, 12096, 12160, 12182, 12224, 12190, 12147, 12114, 12118, + 12058, 12176, 12075, 12112, 12045, 12170, 12124, 12070, 12194, 12109, + 12229, 12196, 12139, 12056, 12099, 12034, 12084, 12136, 12120, 12044, + 12111, 12094, 12125, 12243, 12238, 12082, 12159, 12063, 12215, 12062, + 12042, 12241, 12067, 12235, 12043, 12140, 12119, 12207, 12123, 12133, + 12222, 12117, 12226, 12245, 12214, 12217, 12236, 12155, 12188, 12113, + 12065, 12198, 12066, 12146, 12171, 12225, 12200, 12121, 12093, 12095, + 12221, 12092, 12216, 12231, 12054, 12218, 12179, 12037, 12173, 12072, + 12046, 12127, 12152, 12049, 12074, 12107, 12208, 12212, 12041, 12210, + 12131, 12033, 12039, 12199, 12085, 12128, 12161, 12162, 12233, 12165, + 12192, 12077, 12201, 12061, 12105, 12040, 12240, 12102, 12153, 12032, + 12080, 12167, 12048, 12156, 12059, 12126, 12158, 12050, 12183, 12204, + 12097, 12239, 12053, 12078, 12150, 12071, 12187, 12186, 12223, 12228, + 12104, 12098, 12064, 12036, 12057, 12163, 12178, 12185, 12154, 12203, + 12083, 12087, 12135, 12151, 12202, 12035, 12122, 12141, 12180, 12144, + 12076, 12052, 12115, 12091, 12108, 12169, 12143, 12148, 12130, 12089, + 12211, 12073, 12101, 12138, 12103, 12209, 12047, 12220, 12172, 12129, + 12166, 12242, 12237, 12145, 12106, 12081, 12244, 12038, 12086, 12055, + 12181, 12197, 12193, 12175, 12116, 12110, 12177, 12137, 12230, 12213, + 12195, 12069, 12079, 12206, 12051, 12232, 12132, 129432, 3240, 3293, + 3225, 3235, 3230, 3205, 3206, 3216, 3220, 3234, 3233, 3239, 3238, 983205, + 3251, 3250, 3249, 3248, 3232, 3231, 3237, 3236, 3211, 3296, 3212, 3297, + 3253, 3209, 3210, 3218, 3219, 3207, 3208, 3254, 3255, 3256, 3245, 3244, + 3227, 3226, 3224, 3223, 3229, 3228, 3222, 3221, 3243, 3242, 3214, 3215, + 3294, 3257, 3246, 3247, 3285, 3315, 3201, 3200, 3204, 3260, 3261, 3202, + 3313, 3314, 3277, 3203, 3286, 3262, 3272, 3276, 3265, 3266, 3267, 3268, + 3298, 3299, 3274, 3275, 3263, 3264, 3270, 3271, 3307, 3306, 3309, 3308, + 3305, 3304, 3302, 3311, 3303, 3310, 12450, 984009, 984008, 984007, + 984010, 110881, 110880, 110882, 110592, 12499, 984002, 984005, 984003, + 984006, 984004, 12496, 12505, 12508, 12502, 12511, 110583, 110584, + 110585, 110586, 110587, 110589, 110590, 110576, 110577, 110578, 110579, + 110581, 110582, 12510, 12513, 12514, 12512, 12531, 12490, 12493, 12491, + 12494, 12492, 12789, 12792, 12790, 12793, 12791, 12795, 12798, 12796, + 12799, 12797, 12533, 12534, 110933, 12784, 12787, 12483, 12526, 110949, + 110948, 110950, 12515, 12519, 12517, 110951, 12788, 12785, 12786, 12794, + 12449, 12455, 12451, 12457, 12453, 12469, 12475, 12471, 12477, 12473, + 12480, 12487, 12482, 12489, 12485, 12460, 12466, 12462, 12468, 12464, + 12495, 12504, 12498, 12507, 12501, 12459, 12465, 12461, 12467, 12463, + 12497, 12506, 12500, 12509, 12503, 12521, 12524, 12522, 12525, 12523, + 12479, 12486, 12481, 12488, 12484, 12535, 12537, 12536, 12538, 12532, + 12470, 12476, 12472, 12478, 12474, 12527, 12529, 12528, 12530, 12516, + 12520, 12518, 12456, 12452, 12458, 12454, 12542, 12543, 12541, 12539, + 12448, 12540, 12444, 12443, 73476, 73477, 73487, 73523, 73498, 73497, + 73503, 73502, 73508, 73507, 73501, 73500, 73506, 73505, 73480, 73481, + 73482, 73483, 73484, 73485, 73478, 73479, 73494, 73504, 73499, 73509, + 73519, 73520, 73521, 73513, 73512, 73496, 73495, 73493, 73492, 73491, + 73490, 73511, 73510, 73522, 73517, 73514, 73516, 73518, 73515, 73486, + 73488, 73551, 73548, 73546, 73545, 73549, 73543, 73541, 73544, 73550, + 73542, 73547, 73537, 73472, 73562, 73474, 73475, 73473, 73525, 73524, + 73535, 73530, 73534, 73536, 73528, 73529, 73526, 73527, 73540, 73539, + 73557, 73556, 73559, 73558, 73555, 73554, 73552, 73561, 73553, 73560, + 73538, 43299, 43301, 43283, 43295, 43277, 43281, 43284, 43275, 43274, + 43286, 43285, 43279, 43278, 43294, 43282, 43289, 43297, 43288, 43276, + 43292, 43287, 43290, 43296, 43293, 43291, 43280, 43298, 43300, 43310, + 43311, 43308, 43309, 43307, 43303, 43305, 43304, 43302, 43306, 43269, + 43268, 43271, 43270, 43267, 43266, 43264, 43273, 43265, 43272, 119496, + 119506, 119499, 119503, 119493, 119492, 119502, 119497, 119507, 119495, + 119505, 119494, 119504, 119501, 119491, 119500, 119490, 119498, 119488, + 119489, 128331, 8490, 128273, 9000, 128422, 983552, 983553, 983559, + 983558, 983561, 983560, 983557, 983556, 983554, 983563, 983555, 983562, + 128287, 118449, 68163, 68162, 68161, 68160, 68113, 68146, 68112, 68147, + 68148, 68123, 68122, 68128, 68127, 68126, 68121, 68131, 68125, 68124, + 68130, 68129, 68141, 68142, 68143, 68135, 68134, 68118, 68117, 68115, + 68114, 68133, 68132, 68149, 68140, 68145, 68119, 68139, 68136, 68138, + 68137, 68144, 68096, 68165, 68164, 68166, 68167, 68179, 68178, 68183, + 68176, 68182, 68181, 68184, 68180, 68177, 68152, 68153, 68109, 68154, + 68111, 68110, 68099, 68101, 68097, 68102, 68098, 68108, 68159, 68168, + 129711, 101120, 101121, 101122, 101123, 101124, 101125, 101126, 101127, + 101128, 101129, 101130, 101131, 101132, 101133, 101134, 101135, 101136, + 101137, 101138, 101139, 101140, 101141, 101142, 101143, 101144, 101145, + 101146, 101147, 101148, 101149, 101150, 101151, 101152, 101153, 101154, + 101155, 101156, 101157, 101158, 101159, 101160, 101161, 101162, 101163, + 101164, 101165, 101166, 101167, 101168, 101169, 101170, 101171, 101172, + 101173, 101174, 101175, 101176, 101177, 101178, 101179, 101180, 101181, + 101182, 101183, 101184, 101185, 101186, 101187, 101188, 101189, 101190, + 101191, 101192, 101193, 101194, 101195, 101196, 101197, 101198, 101199, + 101200, 101201, 101202, 101203, 101204, 101205, 101206, 101207, 101208, + 101209, 101210, 101211, 101212, 101213, 101214, 101215, 101216, 101217, + 101218, 101219, 101220, 101221, 101222, 101223, 101224, 101225, 101226, + 101227, 101228, 101229, 101230, 101231, 101232, 101233, 101234, 101235, + 101236, 101237, 101238, 101239, 101240, 101241, 101242, 101243, 101244, + 101245, 101246, 101247, 101248, 101249, 101250, 101251, 101252, 101253, + 101254, 101255, 101256, 101257, 101258, 101259, 101260, 101261, 101262, + 101263, 101264, 101265, 101266, 101267, 101268, 101269, 101270, 101271, + 101272, 101273, 101274, 101275, 101276, 101277, 101278, 101279, 101280, + 101281, 101282, 101283, 101284, 101285, 101286, 101287, 101288, 101289, + 101290, 101291, 101292, 101293, 101294, 101295, 101296, 101297, 101298, + 101299, 101300, 101301, 101302, 101303, 101304, 101305, 101306, 101307, + 101308, 101309, 101310, 101311, 101312, 101313, 101314, 101315, 101316, + 101317, 101318, 101319, 101320, 101321, 101322, 101323, 101324, 101325, + 101326, 101327, 101328, 101329, 101330, 101331, 101332, 101333, 101334, + 101335, 101336, 101337, 101338, 101339, 101340, 101341, 101342, 101343, + 101344, 101345, 101346, 101347, 101348, 101349, 101350, 101351, 101352, + 101353, 101354, 101355, 101356, 101357, 101358, 101359, 101360, 101361, + 101362, 101363, 101364, 101365, 101366, 101367, 101368, 101369, 101370, + 101371, 101372, 101373, 101374, 101375, 101584, 101585, 101586, 101587, + 101588, 101589, 101376, 101377, 101378, 101379, 101380, 101381, 101382, + 101383, 101384, 101385, 101386, 101387, 101388, 101389, 101390, 101391, + 101392, 101393, 101394, 101395, 101396, 101397, 101398, 101399, 101400, + 101401, 101402, 101403, 101404, 101405, 101406, 101407, 101408, 101409, + 101410, 101411, 101412, 101413, 101414, 101415, 101416, 101417, 101418, + 101419, 101420, 101421, 101422, 101423, 101424, 101425, 101426, 101427, + 101428, 101429, 101430, 101431, 101432, 101433, 101434, 101435, 101436, + 101437, 101438, 101439, 101440, 101441, 101442, 101443, 101444, 101445, + 101446, 101447, 101448, 101449, 101450, 101451, 101452, 101453, 101454, + 101455, 101456, 101457, 101458, 101459, 101460, 101461, 101462, 101463, + 101464, 101465, 101466, 101467, 101468, 101469, 101470, 101471, 101472, + 101473, 101474, 101475, 101476, 101477, 101478, 101479, 101480, 101481, + 101482, 101483, 101484, 101485, 101486, 101487, 101488, 101489, 101490, + 101491, 101492, 101493, 101494, 101495, 101496, 101497, 101498, 101499, + 101500, 101501, 101502, 101503, 101504, 101505, 101506, 101507, 101508, + 101509, 101510, 101511, 101512, 101513, 101514, 101515, 101516, 101517, + 101518, 101519, 101520, 101521, 101522, 101523, 101524, 101525, 101526, + 101527, 101528, 101529, 101530, 101531, 101532, 101533, 101534, 101535, + 101536, 101537, 101538, 101539, 101540, 101541, 101542, 101543, 101544, + 101545, 101546, 101547, 101548, 101549, 101550, 101551, 101552, 101553, + 101554, 101555, 101556, 101557, 101558, 101559, 101560, 101561, 101562, + 101563, 101564, 101565, 101566, 101567, 101568, 101569, 101570, 101571, + 101572, 101573, 101574, 101575, 101576, 101577, 101578, 101579, 101580, + 101581, 101582, 101583, 101631, 94180, 983960, 983965, 983970, 983975, + 983962, 983964, 983961, 983963, 983957, 983959, 983956, 983958, 983977, + 983979, 983978, 983972, 983974, 983967, 983969, 983971, 983973, 983966, + 983968, 983989, 983983, 983985, 983986, 983987, 983980, 983982, 983984, + 983981, 983976, 983988, 6107, 6052, 6064, 6051, 6067, 6066, 6065, 6055, + 6057, 6058, 6056, 6053, 6054, 6063, 983994, 983991, 983992, 983993, 6061, + 6062, 6059, 6060, 6022, 6024, 6021, 6023, 6017, 6019, 6016, 6018, 6020, + 6030, 6025, 6035, 6037, 6039, 6038, 6046, 6045, 6047, 6032, 6034, 6027, + 6029, 6031, 6033, 6026, 6028, 6049, 6043, 6040, 6042, 6044, 6041, 6036, + 6048, 6050, 6108, 6109, 6095, 6096, 6099, 6091, 6101, 6104, 6102, 6098, + 6094, 6100, 6106, 6092, 6087, 6093, 6090, 6103, 6105, 6088, 6086, 6089, + 6097, 6652, 6636, 6655, 6639, 6653, 6637, 6654, 6638, 6651, 6635, 6650, + 6634, 6133, 6137, 6136, 6134, 6135, 6130, 6132, 6131, 6129, 6128, 6624, + 6648, 6632, 6649, 6633, 6647, 6631, 6646, 6630, 6645, 6629, 6642, 6626, + 6640, 6643, 6627, 6644, 6628, 6641, 6625, 6069, 6068, 6070, 983996, 6082, + 6083, 6085, 6071, 6080, 6072, 6078, 983995, 6084, 6073, 6079, 6074, + 983990, 6075, 6077, 6076, 6081, 6117, 6116, 6119, 6118, 6115, 6114, 6112, + 6121, 6113, 6120, 70204, 70201, 70200, 70208, 70185, 70178, 70179, 70177, + 70155, 70156, 70154, 70172, 70167, 70166, 70173, 70171, 70161, 70160, + 70144, 70145, 70149, 70151, 70165, 70164, 70170, 70169, 70187, 70183, + 70157, 70168, 70163, 70174, 70159, 70158, 70153, 70152, 70176, 70175, + 70186, 70180, 70207, 70182, 70184, 70181, 70148, 70146, 70150, 70147, + 70199, 70206, 70198, 70197, 70196, 70203, 70209, 70188, 70193, 70195, + 70189, 70190, 70192, 70194, 70191, 70202, 70205, 70357, 70358, 70356, + 70333, 70334, 70332, 70345, 70347, 70344, 70352, 70351, 70340, 70339, + 70338, 70320, 70321, 70327, 70329, 70346, 70361, 70343, 70342, 70350, + 70349, 70324, 70325, 70322, 70323, 70335, 70348, 70341, 70353, 70337, + 70336, 70331, 70330, 70355, 70354, 70364, 70365, 70366, 70362, 70359, + 70363, 70360, 70326, 70328, 70377, 70378, 70367, 70368, 70374, 70376, + 70371, 70372, 70369, 70370, 70373, 70375, 70389, 70388, 70391, 70390, + 70387, 70386, 70384, 70393, 70385, 70392, 93521, 93520, 93525, 93524, + 93519, 93518, 93523, 93522, 93512, 93517, 93526, 93530, 93529, 93514, + 93513, 93511, 93510, 93516, 93515, 93509, 93508, 93528, 93527, 93537, + 93536, 93538, 93534, 93531, 93533, 93535, 93532, 93507, 93505, 93549, + 93548, 93504, 93547, 93506, 93539, 93544, 93546, 93541, 93542, 93543, + 93540, 93545, 93551, 93550, 93557, 93556, 93559, 93558, 93555, 93554, + 93552, 93561, 93553, 93560, 128143, 128535, 128537, 128538, 128573, + 128139, 129373, 8365, 128088, 129665, 129486, 129698, 12927, 128040, + 11235, 129404, 127991, 129357, 128030, 129692, 3805, 3804, 983206, + 983207, 3743, 3741, 3807, 3806, 3714, 3716, 3713, 983209, 3747, 3749, + 3730, 3729, 3736, 3728, 3727, 3731, 3726, 3744, 3721, 3718, 3724, 3756, + 3740, 3742, 3739, 3752, 3753, 3754, 3722, 3734, 3735, 3733, 3755, 3758, + 3719, 3725, 3737, 3738, 3720, 3732, 3745, 983208, 3751, 3746, 3757, 3773, + 3772, 3770, 3785, 3786, 3787, 3784, 3760, 3762, 3780, 3763, 3779, 3761, + 3771, 3766, 3767, 3768, 3769, 3776, 3777, 3764, 3765, 3778, 3782, 3790, + 3759, 3797, 3796, 3799, 3798, 3795, 3794, 3792, 3801, 3793, 3800, 3788, + 3789, 128996, 129003, 128309, 128311, 128998, 128994, 129001, 68413, + 68415, 128992, 128310, 128999, 68412, 68414, 118324, 118319, 118322, + 118323, 118303, 118304, 118336, 118331, 118328, 118315, 118314, 118306, + 118316, 118318, 118334, 118335, 118333, 118327, 118339, 118341, 118342, + 118340, 118320, 118338, 118332, 118302, 118325, 118309, 118317, 118307, + 118313, 118326, 118329, 118312, 118330, 118352, 118348, 118351, 118350, + 118347, 118344, 118345, 118343, 118349, 118346, 118305, 118321, 118301, + 118299, 118298, 118310, 118311, 118308, 118300, 118337, 11004, 10201, + 10200, 10782, 128995, 129002, 128308, 128997, 128993, 129000, 9711, + 10923, 10925, 8382, 9790, 127772, 127767, 65, 258, 7858, 7856, 7854, + 7862, 7860, 550, 480, 7840, 512, 196, 478, 197, 506, 7680, 256, 983564, + 194, 7848, 7846, 7844, 7852, 7850, 461, 7842, 260, 983590, 983592, 192, + 193, 514, 195, 570, 198, 508, 482, 42946, 393, 11373, 42808, 42810, + 42802, 42804, 42806, 42812, 66, 386, 42902, 7686, 7684, 7682, 385, 579, + 42822, 42932, 67, 199, 7688, 264, 268, 262, 42948, 391, 42898, 266, 571, + 42960, 42796, 42798, 42862, 42931, 68, 498, 453, 42951, 272, 7696, 7698, + 270, 395, 7694, 7692, 7690, 394, 497, 452, 69, 552, 7708, 202, 983586, + 7874, 7872, 7870, 983584, 7878, 7876, 7704, 282, 278, 983598, 983600, + 7864, 516, 203, 7868, 7706, 274, 7700, 7702, 983570, 983572, 983574, + 7866, 280, 983594, 983596, 200, 201, 518, 276, 582, 439, 494, 440, 42786, + 42788, 42858, 208, 425, 330, 70, 401, 7710, 42904, 71, 290, 284, 486, + 500, 286, 7712, 42912, 403, 288, 484, 577, 42938, 42940, 42942, 404, + 983199, 72, 7722, 7720, 292, 542, 7718, 11367, 7716, 7714, 42922, 294, + 11381, 502, 42790, 73, 520, 7882, 304, 207, 7726, 206, 463, 298, 983566, + 296, 7724, 7880, 302, 983604, 983606, 204, 205, 522, 300, 407, 42873, + 42875, 42877, 42882, 42884, 42886, 406, 42860, 74, 308, 42930, 983608, + 584, 75, 310, 488, 42818, 11369, 7730, 42816, 42820, 7728, 7732, 42914, + 408, 76, 42925, 573, 11360, 7734, 7736, 11362, 319, 456, 321, 315, 7740, + 317, 42824, 313, 7738, 983610, 42970, 42972, 455, 77, 7742, 7746, 7744, + 983612, 11374, 7930, 7932, 42966, 78, 325, 7754, 327, 459, 544, 7752, + 413, 504, 323, 42896, 7750, 7748, 42916, 209, 458, 79, 42826, 42828, 332, + 7760, 7762, 415, 212, 7892, 7890, 7888, 7896, 7894, 465, 214, 554, 558, + 560, 7884, 524, 336, 490, 492, 216, 510, 213, 7758, 7756, 556, 983576, + 983578, 983580, 416, 7902, 7900, 7898, 7906, 7904, 7886, 210, 211, 526, + 334, 42944, 400, 390, 42934, 418, 42830, 546, 80, 42834, 11363, 42832, + 42836, 7764, 420, 7766, 81, 42840, 42838, 82, 342, 344, 7770, 7772, 7768, + 528, 11364, 983614, 340, 7774, 530, 42918, 588, 42842, 42997, 42923, + 42814, 398, 42844, 42955, 83, 352, 7782, 350, 536, 348, 346, 7780, 7778, + 7784, 7776, 42956, 42953, 11390, 983582, 42920, 42949, 586, 42926, 42891, + 7838, 42968, 42924, 399, 84, 354, 7792, 538, 356, 574, 7788, 7786, 7790, + 430, 428, 358, 42878, 11375, 11376, 42893, 42928, 42880, 412, 42929, 581, + 222, 42852, 42854, 388, 423, 444, 42794, 42792, 85, 219, 7798, 467, 220, + 473, 475, 471, 469, 7794, 532, 368, 7908, 431, 7916, 7914, 7912, 7920, + 7918, 7910, 362, 7802, 983568, 983620, 983622, 370, 983616, 983618, 360, + 7800, 7796, 217, 218, 534, 364, 366, 42936, 580, 433, 86, 42846, 7806, + 7804, 434, 42850, 42906, 42908, 42910, 42856, 42848, 87, 372, 7812, 7816, + 7814, 7808, 7810, 11378, 503, 88, 7820, 7818, 89, 374, 376, 7924, 7822, + 7922, 435, 7926, 7934, 221, 562, 7928, 590, 540, 90, 7824, 381, 377, + 11371, 7826, 379, 7828, 11391, 437, 42950, 548, 306, 338, 10013, 43007, + 43005, 43006, 43003, 43004, 42999, 450, 7461, 684, 664, 685, 662, 122638, + 446, 426, 674, 451, 122634, 7431, 7430, 7459, 618, 641, 671, 122628, + 7436, 7439, 7440, 630, 7445, 640, 7438, 7449, 43846, 42870, 7451, 11387, 122626, 122640, 43002, 7450, 665, 7427, 610, 667, 7424, 7425, 7428, 7429, 42800, 668, 7434, 7435, 7437, 628, 7448, 42927, 42801, 7452, 7456, 7457, - 655, 7458, 663, 122639, 42895, 447, 443, 448, 449, 660, 673, 661, 7460, + 655, 7458, 663, 122639, 42895, 443, 447, 448, 449, 660, 673, 661, 7460, 422, 7547, 7550, 97, 259, 7859, 7857, 7855, 7863, 7861, 551, 481, 7841, 513, 228, 479, 229, 507, 7681, 7834, 7567, 257, 983565, 226, 7849, 7847, 7845, 7853, 7851, 462, 7843, 261, 983591, 983593, 224, 225, 515, 227, @@ -10776,400 +11529,411 @@ static const unsigned int dawg_pos_to_codepoint[] = { 669, 496, 983609, 585, 107, 311, 489, 42819, 11370, 7731, 42817, 42821, 7729, 7733, 7556, 42915, 409, 312, 108, 620, 122643, 410, 316, 7741, 564, 318, 7735, 7737, 43832, 11361, 619, 43833, 320, 122662, 42825, 314, 7739, - 43831, 621, 42894, 122641, 7557, 983611, 322, 411, 622, 122629, 43829, - 383, 7836, 7835, 7837, 682, 683, 42866, 457, 109, 7743, 43834, 7535, - 7558, 7747, 7745, 983613, 625, 7931, 7933, 42967, 42867, 110, 326, 7755, - 43835, 565, 328, 414, 7753, 626, 122663, 7536, 505, 324, 42897, 7751, - 7749, 7559, 627, 42917, 241, 329, 983589, 42868, 460, 111, 244, 7893, - 7891, 7889, 7897, 7895, 466, 246, 555, 559, 561, 7885, 525, 337, 42827, - 11386, 42829, 333, 7761, 7763, 491, 493, 248, 511, 245, 7759, 7757, 557, - 983577, 983579, 983581, 417, 7903, 7901, 7899, 7907, 7905, 7887, 242, - 243, 527, 335, 122651, 42945, 596, 983625, 983626, 7575, 43839, 43874, - 603, 7571, 42935, 419, 42831, 547, 112, 42835, 7549, 42833, 42837, 7765, - 7537, 7560, 421, 7767, 632, 113, 42841, 672, 587, 42839, 569, 114, 343, - 43849, 345, 7771, 7773, 7769, 529, 638, 7539, 122646, 7775, 636, 637, - 983615, 122664, 7538, 341, 531, 7561, 42919, 589, 43847, 42843, 42998, - 604, 7572, 605, 639, 122625, 600, 122631, 8580, 42815, 122627, 42869, - 42845, 612, 115, 347, 7781, 353, 7783, 351, 537, 349, 122654, 7779, 7785, - 7777, 42954, 575, 983583, 122665, 7540, 7562, 42921, 642, 42892, 43872, - 601, 983629, 983630, 7573, 602, 43851, 43852, 609, 43830, 223, 7441, - 7442, 7443, 7455, 7454, 7453, 42969, 645, 43845, 116, 355, 7793, 539, - 566, 357, 11366, 7831, 7789, 7787, 122666, 7541, 7791, 427, 429, 122633, - 648, 359, 679, 122647, 122652, 7546, 254, 42853, 42855, 389, 424, 445, - 7446, 42795, 397, 613, 686, 687, 7433, 42879, 7444, 43842, 43841, 43843, - 43844, 7432, 633, 43880, 122645, 634, 122632, 11385, 635, 647, 122637, - 652, 983627, 983628, 592, 594, 7426, 623, 624, 654, 122630, 43857, 477, - 7543, 670, 42881, 653, 42871, 680, 678, 43879, 11383, 42793, 117, 649, - 43855, 251, 7799, 468, 252, 474, 476, 472, 470, 7795, 533, 369, 7909, - 432, 7917, 7915, 7913, 7921, 7919, 7911, 363, 7803, 983569, 983621, - 983623, 371, 983617, 983619, 7577, 367, 361, 7801, 7797, 249, 43854, - 42937, 250, 535, 365, 43858, 650, 7551, 7531, 43856, 42872, 43875, 118, - 42847, 7807, 7564, 11380, 11377, 7805, 651, 42851, 42907, 42909, 42911, - 42857, 42849, 119, 373, 7813, 7817, 7815, 7809, 7811, 7832, 11379, 120, - 7821, 7819, 43863, 43864, 43865, 43862, 7565, 121, 375, 255, 7925, 7823, - 7923, 436, 7927, 7935, 43866, 591, 253, 563, 7929, 7833, 541, 122, 378, - 7825, 657, 382, 11372, 7827, 380, 7829, 576, 438, 7542, 7566, 656, 549, - 64256, 64259, 64260, 64257, 64258, 307, 64261, 64262, 339, 8347, 8340, - 8336, 8337, 8341, 7522, 11388, 8342, 8343, 8344, 8345, 8338, 8346, 7523, - 8348, 7524, 7525, 8339, 917505, 127811, 129388, 129897, 129916, 129899, - 129917, 129947, 10203, 10202, 128494, 12296, 10641, 10643, 11058, 11056, - 10576, 10571, 10570, 10574, 12304, 10647, 123, 9128, 9129, 9127, 12300, - 8968, 9948, 10714, 12298, 8220, 11816, 11780, 129287, 129284, 129285, - 129283, 129286, 9686, 11240, 9612, 129977, 129970, 129940, 129932, - 128379, 128709, 11804, 10204, 10197, 9614, 9615, 8596, 8700, 8697, 8622, - 10568, 8660, 10500, 8654, 8621, 11012, 8703, 11020, 129112, 11108, 11788, - 10181, 8907, 9609, 8216, 11814, 128488, 91, 9123, 9121, 10639, 10637, - 8261, 10635, 11863, 11861, 9122, 11778, 10703, 129900, 11785, 129985, - 128492, 9613, 9610, 12308, 8867, 12302, 10627, 12310, 10629, 12314, - 12312, 10712, 128398, 9611, 10620, 8970, 8905, 40, 9117, 9115, 9116, - 11808, 9144, 10553, 10154, 1422, 4054, 4056, 129307, 9958, 10748, 9001, - 171, 128269, 8294, 8237, 8234, 8206, 8592, 10563, 11074, 11083, 11082, - 10611, 129973, 10618, 10615, 11070, 8676, 8633, 10525, 11064, 8698, - 129032, 8619, 10566, 129040, 129024, 8602, 11024, 11025, 8610, 11066, - 11065, 129028, 129176, 129044, 8617, 8695, 8612, 10527, 8646, 10521, - 129192, 129184, 11144, 11013, 10603, 10599, 10590, 10582, 8637, 10594, - 10602, 10598, 10586, 10578, 8636, 8651, 129778, 129088, 129092, 11104, - 11136, 11130, 983241, 11174, 11172, 129064, 129060, 129056, 129072, - 129068, 11120, 11114, 11140, 129168, 10510, 8666, 129186, 11067, 11069, - 11068, 11244, 11061, 11060, 11062, 11063, 8606, 8678, 129172, 8604, 8656, - 10498, 8653, 10502, 10523, 10508, 8672, 129194, 129076, 129188, 8701, - 8647, 129783, 129190, 128620, 8668, 129080, 129104, 129084, 11077, 9804, - 128006, 7213, 7221, 7216, 7220, 7215, 7214, 7217, 7218, 7219, 7170, 7169, - 7168, 7184, 7183, 7182, 7247, 7193, 7180, 7188, 7187, 7186, 7185, 7172, - 7171, 7198, 7197, 7190, 7189, 7173, 7177, 7181, 7192, 7191, 7246, 7245, - 7179, 7178, 7175, 7174, 7201, 7200, 7176, 7196, 7195, 7199, 7202, 7194, - 7203, 7227, 7231, 7230, 7228, 7229, 7223, 7222, 7205, 7204, 7210, 7211, - 7208, 7209, 7206, 7212, 7207, 7237, 7236, 7239, 7238, 7235, 7234, 7232, - 7241, 7233, 7240, 10897, 10895, 10893, 10899, 10891, 10614, 10889, 10887, - 8922, 8934, 8808, 10918, 10920, 10885, 10877, 10881, 10883, 10879, 8818, - 8804, 8822, 8806, 10873, 10875, 8918, 60, 128210, 127898, 127819, 129461, - 128626, 128955, 128969, 128943, 11212, 128964, 10099, 128648, 10098, - 9617, 128937, 128949, 128978, 128960, 129653, 128910, 10072, 128930, - 128504, 9735, 128498, 128497, 6429, 6404, 6403, 6412, 6430, 6411, 6421, - 6410, 6405, 6415, 6425, 6426, 6427, 6419, 6418, 6407, 6406, 6414, 6413, - 6409, 6408, 6402, 6401, 6417, 6416, 6428, 6423, 6420, 6422, 6424, 6458, - 6457, 6459, 6464, 6449, 6452, 6450, 6448, 6456, 6454, 6453, 6455, 6451, - 6442, 6443, 6441, 6432, 6436, 6438, 6440, 6437, 6439, 6435, 6433, 6434, - 6400, 6468, 6469, 6475, 6474, 6477, 6476, 6473, 6472, 6470, 6479, 6471, - 6478, 13007, 10770, 10771, 10772, 983062, 8232, 983068, 983143, 67143, - 67146, 67151, 67165, 67166, 67167, 67157, 67158, 67159, 67160, 67161, - 67162, 67163, 67164, 67171, 67172, 67173, 67168, 67169, 67170, 67174, - 67175, 67176, 67177, 67178, 67179, 67230, 67231, 67180, 67181, 67182, - 67183, 67184, 67185, 67186, 67187, 67188, 67189, 67190, 67191, 67192, - 67193, 67194, 67195, 67196, 67197, 67198, 67199, 67200, 67201, 67202, - 67203, 67204, 67205, 67206, 67207, 67208, 67209, 67210, 67211, 67212, - 67213, 67214, 67215, 67216, 67217, 67218, 67219, 67220, 67221, 67222, - 67223, 67224, 67225, 67226, 67227, 67228, 67229, 67232, 67233, 67234, - 67235, 67236, 67237, 67238, 67239, 67240, 67241, 67242, 67243, 67244, - 67245, 67246, 67247, 67248, 67249, 67250, 67251, 67252, 67253, 67254, - 67255, 67256, 67257, 67258, 67259, 67260, 67261, 67262, 67263, 67264, - 67274, 67275, 67276, 67277, 67278, 67279, 67280, 67281, 67282, 67283, - 67284, 67285, 67286, 67287, 67288, 67289, 67290, 67291, 67292, 67293, - 67294, 67295, 67296, 67297, 67298, 67299, 67300, 67301, 67302, 67303, - 67304, 67265, 67266, 67267, 67268, 67269, 67270, 67271, 67272, 67273, - 67325, 67326, 67327, 67328, 67329, 67330, 67305, 67306, 67307, 67308, - 67309, 67310, 67311, 67312, 67313, 67314, 67315, 67316, 67317, 67318, - 67319, 67320, 67321, 67322, 67323, 67324, 67331, 67332, 67333, 67334, - 67335, 67336, 67337, 67338, 67349, 67350, 67351, 67352, 67353, 67354, - 67355, 67356, 67357, 67358, 67359, 67360, 67361, 67362, 67363, 67364, - 67365, 67366, 67367, 67368, 67378, 67379, 67380, 67381, 67382, 67369, - 67370, 67371, 67372, 67373, 67374, 67375, 67376, 67377, 67339, 67340, - 67341, 67342, 67343, 67344, 67345, 67346, 67347, 67348, 67401, 67404, - 67403, 67402, 67400, 67398, 67393, 67397, 67395, 67394, 67399, 67392, - 67396, 67406, 67408, 67409, 67410, 67407, 67405, 67411, 67413, 67412, - 67424, 67425, 67426, 67427, 67428, 67429, 67430, 67431, 67081, 67082, - 67083, 67084, 67085, 67087, 67088, 67089, 67090, 67091, 67092, 67093, - 67094, 67086, 67095, 67096, 67097, 67098, 67100, 67101, 67102, 67103, - 67104, 67105, 67106, 67107, 67108, 67109, 67110, 67111, 67112, 67113, - 67114, 67115, 67116, 67117, 67118, 67119, 67120, 67121, 67122, 67123, - 67124, 67125, 67126, 67127, 67128, 67129, 67130, 67131, 67132, 67133, - 67134, 67135, 67136, 67137, 67138, 67139, 67140, 67141, 67142, 67072, - 67073, 67074, 67075, 67076, 67077, 67078, 67079, 67080, 67145, 67147, - 67148, 67149, 67150, 67154, 67155, 67144, 67152, 67153, 67156, 67099, - 65667, 65668, 65669, 65670, 65671, 65672, 65673, 65675, 65674, 65677, - 65676, 65665, 65664, 65666, 65681, 65679, 65680, 65682, 65678, 65686, - 65685, 65687, 65693, 65690, 65691, 65692, 65694, 65703, 65696, 65695, - 65697, 65698, 65699, 65701, 65702, 65707, 65706, 65704, 65705, 65708, - 65709, 65710, 65711, 65712, 65713, 65719, 65717, 65714, 65715, 65716, - 65718, 65720, 65721, 65722, 65723, 65724, 65725, 65726, 65727, 65728, - 65729, 65731, 65730, 65732, 65733, 65737, 65734, 65735, 65736, 65738, - 65739, 65740, 65741, 65743, 65742, 65744, 65745, 65747, 65748, 65752, - 65749, 65750, 65751, 65753, 65754, 65755, 65756, 65757, 65779, 65780, - 65781, 65782, 65783, 65784, 65785, 65759, 65760, 65761, 65762, 65763, - 65764, 65765, 65766, 65767, 65768, 65769, 65770, 65771, 65772, 65773, - 65774, 65775, 65776, 65777, 65778, 65758, 65786, 65683, 65684, 65689, - 65688, 65700, 65746, 65561, 65543, 65587, 65582, 65589, 65536, 65541, - 65579, 65566, 65571, 65596, 65569, 65584, 65544, 65559, 65540, 65557, - 65560, 65580, 65606, 65600, 65599, 65577, 65562, 65538, 65573, 65563, - 65609, 65588, 65549, 65568, 65537, 65581, 65574, 65601, 65542, 65593, - 65583, 65594, 65552, 65547, 65605, 65578, 65545, 65585, 65586, 65546, - 65570, 65591, 65564, 65565, 65607, 65610, 65611, 65553, 65590, 65550, - 65539, 65576, 65608, 65551, 65554, 65592, 65603, 65597, 65567, 65572, - 65558, 65555, 65612, 65602, 65556, 65613, 65604, 65620, 65621, 65623, - 65624, 65626, 65627, 65628, 65629, 65622, 65616, 65617, 65619, 65618, - 65625, 128391, 128279, 128482, 128132, 42237, 42235, 42232, 42234, 42236, - 42233, 42206, 42205, 42197, 42196, 42204, 42195, 42228, 42229, 42230, - 42224, 42225, 42213, 42208, 42203, 42202, 42221, 42198, 42216, 42214, - 42200, 42199, 42194, 42193, 42219, 42210, 73648, 42220, 42211, 42212, - 42222, 42223, 42227, 42231, 42192, 42217, 42201, 42209, 42207, 42218, - 42215, 42226, 42238, 42239, 8356, 8374, 129409, 129422, 9806, 128274, - 983079, 983076, 128271, 8743, 10848, 10846, 10833, 10844, 10842, 10847, - 8744, 10841, 10851, 10850, 10834, 10845, 10843, 10982, 10188, 129688, - 10234, 10231, 10206, 10229, 10235, 10232, 10237, 11059, 10230, 10236, - 10233, 10238, 10239, 10205, 129524, 128884, 129719, 128140, 127977, - 128261, 129707, 12319, 8270, 11847, 11848, 95, 9691, 129935, 9604, 9697, - 128394, 129852, 129853, 129870, 129872, 129868, 129856, 129871, 129869, - 129854, 129873, 129855, 128395, 128396, 128393, 10559, 129951, 9722, - 128397, 9695, 10558, 128318, 10065, 129950, 9727, 129863, 129865, 129867, - 129864, 129861, 129866, 129859, 129862, 129860, 129857, 129858, 10195, - 10063, 9998, 9694, 9987, 9605, 9602, 9601, 9607, 9603, 9606, 129903, - 9674, 10208, 129438, 128557, 127853, 129523, 128886, 129729, 66190, - 66192, 66187, 66196, 66178, 66179, 66199, 66185, 66200, 66176, 66201, - 66177, 66202, 66191, 66193, 66181, 66180, 66203, 66182, 66186, 66189, - 66195, 66188, 66197, 66198, 66194, 66183, 66204, 66184, 67887, 67892, - 67881, 67895, 67891, 67886, 67872, 67893, 67876, 67894, 67883, 67896, - 67873, 67897, 67875, 67889, 67874, 67878, 67880, 67882, 67884, 67890, - 67885, 67888, 67877, 67879, 67903, 129317, 983226, 983234, 983224, - 983229, 8468, 129433, 983065, 129668, 129522, 129497, 69986, 69981, - 69991, 70002, 69997, 69985, 69984, 69990, 69989, 69983, 69982, 69988, - 69987, 69995, 69994, 69978, 69977, 69976, 69975, 69980, 69979, 69974, - 69973, 69993, 69992, 70001, 69998, 69996, 70000, 69999, 69968, 69971, - 69969, 69972, 69970, 70006, 70005, 70003, 70004, 127012, 127019, 127008, - 126990, 126999, 126976, 127005, 126987, 126996, 127004, 126986, 126995, - 126979, 127009, 126991, 127000, 127001, 126983, 126992, 127011, 127010, - 126977, 127007, 126989, 126998, 127006, 126988, 126997, 127015, 127014, - 127003, 126985, 126994, 127002, 126984, 126993, 126978, 126982, 127017, - 126981, 126980, 127016, 127018, 127013, 73464, 73442, 73451, 73448, - 73444, 73449, 73447, 73441, 73450, 73440, 73454, 73445, 73443, 73453, - 73456, 73446, 73455, 73452, 73457, 73463, 73461, 73459, 73462, 73460, - 73458, 128892, 3415, 3449, 3435, 3434, 3437, 3436, 3433, 3432, 3430, - 3439, 3431, 3438, 3419, 3420, 3446, 3417, 3422, 3416, 3447, 3444, 3443, - 3448, 3418, 3421, 3445, 3333, 3423, 3334, 3344, 3348, 3453, 3454, 3414, - 3451, 3450, 3452, 3455, 3412, 3413, 3355, 3354, 3406, 3362, 3361, 3367, - 3366, 3360, 3386, 3359, 3365, 3364, 3332, 3339, 3424, 3340, 3425, 3381, - 3369, 3363, 3353, 3358, 3368, 3380, 3379, 3378, 3377, 3376, 3337, 3338, - 3346, 3347, 3335, 3336, 3382, 3383, 3384, 3373, 3372, 3352, 3351, 3357, - 3356, 3350, 3349, 3371, 3370, 3342, 3343, 3385, 3374, 3375, 3441, 3442, - 3440, 3328, 3388, 3329, 3387, 3405, 3331, 3389, 3330, 3407, 3390, 3400, - 3404, 3395, 3396, 3426, 3427, 3393, 3394, 3402, 3403, 3391, 3392, 3398, - 3399, 9895, 9894, 9893, 9794, 10016, 129443, 128104, 128372, 129333, - 128114, 128115, 128378, 128107, 2122, 2121, 2133, 2120, 2126, 2132, 2129, - 2136, 2113, 2115, 2114, 2116, 2123, 2124, 2125, 2128, 2130, 2131, 2118, - 2134, 2117, 2112, 2127, 2119, 2135, 2139, 2138, 2142, 2137, 68288, 68314, - 68313, 68290, 68289, 68293, 68308, 68292, 68291, 68300, 68299, 68298, - 68297, 68306, 68304, 68320, 68318, 68323, 68312, 68317, 68322, 68309, - 68302, 68324, 68305, 68319, 68307, 68321, 68303, 68294, 68301, 68311, - 68295, 68316, 68315, 68310, 68331, 68335, 68334, 68333, 68332, 68340, - 68339, 68338, 68342, 68337, 68341, 68336, 68296, 68326, 68325, 128094, - 129469, 8380, 128368, 129389, 9967, 127809, 129671, 72835, 72834, 72827, - 72826, 72836, 72828, 72821, 72825, 72829, 72823, 72822, 72819, 72818, - 72831, 72830, 72844, 72845, 72838, 72839, 72840, 72832, 72820, 72846, - 72824, 72843, 72833, 72842, 72837, 72841, 72847, 72886, 72885, 72867, - 72866, 72859, 72858, 72868, 72860, 72853, 72857, 72861, 72855, 72854, - 72851, 72850, 72863, 72862, 72876, 72877, 72870, 72871, 72864, 72852, - 72878, 72856, 72875, 72865, 72874, 72869, 72873, 72879, 72880, 72883, - 72881, 72884, 72882, 72816, 72817, 9901, 129355, 73007, 72980, 72979, - 72983, 72982, 72988, 73008, 72987, 72960, 72961, 72968, 72971, 72985, - 72984, 72990, 72989, 72964, 72965, 72962, 72963, 73005, 72999, 73006, - 72973, 72972, 72976, 72986, 72981, 72991, 73001, 73002, 73003, 72995, - 72994, 72978, 72977, 72975, 72974, 72993, 72992, 73004, 72996, 72998, - 73000, 72997, 72966, 72969, 73031, 73030, 73027, 73028, 73026, 73025, - 73024, 73014, 73009, 73020, 73023, 73012, 73013, 73010, 73011, 73018, - 73021, 73029, 73045, 73044, 73047, 73046, 73043, 73042, 73040, 73049, - 73041, 73048, 186, 127405, 12348, 119811, 120778, 120491, 119827, 120495, - 120505, 120507, 119808, 120488, 119809, 120489, 119833, 120493, 119812, - 120492, 120494, 119814, 120490, 119816, 120496, 119818, 120497, 119819, - 120498, 119822, 120502, 120512, 119823, 120509, 120511, 120503, 119825, - 120504, 119826, 120506, 119828, 120508, 119810, 120510, 119820, 120499, - 119821, 120500, 119831, 120501, 119813, 119815, 119817, 119824, 119829, - 119830, 119832, 119837, 120779, 120517, 119834, 120514, 119835, 120515, - 119859, 120519, 119838, 120518, 120520, 119839, 120531, 119840, 120516, - 119842, 120522, 119844, 120523, 119845, 120524, 119848, 120528, 120538, - 119849, 120535, 120537, 120529, 119851, 120530, 119852, 120532, 119853, - 120521, 120533, 119854, 120534, 119836, 120536, 119846, 120525, 119847, - 120526, 119857, 120527, 119841, 119843, 119850, 119855, 119856, 119858, - 120016, 120017, 120018, 120019, 120020, 120021, 120022, 120023, 120024, - 120025, 120026, 120027, 120028, 120029, 120030, 120031, 120032, 120033, - 120034, 120035, 120036, 120037, 120038, 120039, 120040, 120041, 120042, - 120043, 120044, 120045, 120046, 120047, 120048, 120049, 120050, 120051, - 120052, 120053, 120054, 120055, 120056, 120057, 120058, 120059, 120060, - 120061, 120062, 120063, 120064, 120065, 120066, 120067, 119931, 120611, - 120621, 120623, 119912, 120604, 119913, 120605, 119937, 120609, 119915, - 120607, 119916, 120608, 120610, 119918, 120606, 119920, 120612, 119922, - 120613, 119923, 120614, 119926, 120618, 120628, 119927, 120625, 120627, - 120619, 119929, 120620, 119930, 120622, 119932, 120624, 119914, 120626, - 119924, 120615, 119925, 120616, 119935, 120617, 119917, 119919, 119921, - 119928, 119933, 119934, 119936, 120656, 120658, 120629, 120659, 120655, - 120661, 120660, 119938, 120630, 119939, 120631, 119963, 120635, 119941, - 120633, 119942, 120634, 120636, 119943, 120647, 119944, 120632, 119946, - 120638, 119948, 120639, 119949, 120640, 119952, 120644, 120654, 119953, - 120651, 120653, 120645, 119955, 120646, 119956, 120648, 119957, 120637, - 120649, 119958, 120650, 119940, 120652, 119950, 120641, 119951, 120642, - 119961, 120643, 119945, 119947, 119954, 119959, 119960, 119962, 120657, - 120540, 120542, 120513, 120543, 120539, 120545, 120544, 120541, 120172, - 120173, 120174, 120175, 120176, 120177, 120178, 120179, 120180, 120181, - 120182, 120183, 120184, 120185, 120186, 120187, 120188, 120189, 120190, - 120191, 120192, 120193, 120194, 120195, 120196, 120197, 120198, 120199, - 120200, 120201, 120202, 120203, 120204, 120205, 120206, 120207, 120208, - 120209, 120210, 120211, 120212, 120213, 120214, 120215, 120216, 120217, - 120218, 120219, 120220, 120221, 120222, 120223, 120787, 120786, 120789, - 120788, 120785, 120784, 120782, 120791, 120783, 120790, 120120, 120121, - 120123, 120124, 120125, 120126, 120128, 120129, 120130, 120131, 120132, - 120134, 120138, 120139, 120140, 120141, 120142, 120143, 120144, 120146, - 120147, 120148, 120149, 120150, 120151, 120152, 120153, 120154, 120155, - 120156, 120157, 120158, 120159, 120160, 120161, 120162, 120163, 120164, - 120165, 120166, 120167, 120168, 120169, 120170, 120171, 120797, 120796, - 120799, 120798, 120795, 120794, 120792, 120801, 120793, 120800, 120068, - 120069, 120071, 120072, 120073, 120074, 120077, 120078, 120079, 120080, - 120081, 120082, 120083, 120084, 120086, 120087, 120088, 120089, 120090, - 120091, 120092, 120094, 120095, 120096, 120097, 120098, 120099, 120100, - 120101, 120102, 120103, 120104, 120105, 120106, 120107, 120108, 120109, - 120110, 120111, 120112, 120113, 120114, 120115, 120116, 120117, 120118, - 120119, 10189, 119889, 120484, 120485, 120575, 119886, 120572, 119887, - 120573, 119911, 120577, 119890, 120576, 120578, 119891, 120589, 119892, - 120574, 119894, 120580, 119896, 120581, 119897, 120582, 119900, 120586, - 120596, 119901, 120593, 120595, 120587, 119903, 120588, 119904, 120590, - 119905, 120579, 120591, 119906, 120592, 119888, 120594, 119898, 120583, - 119899, 120584, 119909, 120585, 119895, 119902, 119907, 119908, 119910, - 119879, 120553, 120563, 120565, 119860, 120546, 119861, 120547, 119885, - 120551, 119863, 120549, 119864, 120550, 120552, 119866, 120548, 119868, - 120554, 119870, 120555, 119871, 120556, 119874, 120560, 120570, 119875, - 120567, 120569, 120561, 119877, 120562, 119878, 120564, 119880, 120566, - 119862, 120568, 119872, 120557, 119873, 120558, 119883, 120559, 119865, - 119867, 119869, 119876, 119881, 119882, 119884, 120598, 120600, 120571, - 120601, 120597, 120603, 120602, 120599, 120432, 120433, 120434, 120435, - 120436, 120437, 120438, 120439, 120440, 120441, 120442, 120443, 120444, - 120445, 120446, 120447, 120448, 120449, 120450, 120451, 120452, 120453, - 120454, 120455, 120456, 120457, 120458, 120459, 120460, 120461, 120462, - 120463, 120464, 120465, 120466, 120467, 120468, 120469, 120470, 120471, - 120472, 120473, 120474, 120475, 120476, 120477, 120478, 120479, 120480, - 120481, 120482, 120483, 120827, 120826, 120829, 120828, 120825, 120824, - 120822, 120831, 120823, 120830, 10215, 10221, 10219, 10217, 10223, 10187, - 10214, 10220, 10218, 10216, 10222, 120399, 120727, 120737, 120739, - 120380, 120720, 120381, 120721, 120405, 120725, 120383, 120723, 120384, - 120724, 120726, 120386, 120722, 120388, 120728, 120390, 120729, 120391, - 120730, 120394, 120734, 120744, 120395, 120741, 120743, 120735, 120397, - 120736, 120398, 120738, 120400, 120740, 120382, 120742, 120392, 120731, - 120393, 120732, 120403, 120733, 120385, 120387, 120389, 120396, 120401, - 120402, 120404, 120772, 120774, 120745, 120775, 120771, 120777, 120776, - 120406, 120746, 120407, 120747, 120431, 120751, 120409, 120749, 120410, - 120750, 120752, 120411, 120763, 120412, 120748, 120414, 120754, 120416, - 120755, 120417, 120756, 120420, 120760, 120770, 120421, 120767, 120769, - 120761, 120423, 120762, 120424, 120764, 120425, 120753, 120765, 120426, - 120766, 120408, 120768, 120418, 120757, 120419, 120758, 120429, 120759, - 120413, 120415, 120422, 120427, 120428, 120430, 120773, 120295, 120669, - 120679, 120681, 120276, 120662, 120277, 120663, 120301, 120667, 120279, - 120665, 120280, 120666, 120668, 120282, 120664, 120284, 120670, 120286, - 120671, 120287, 120672, 120290, 120676, 120686, 120291, 120683, 120685, - 120677, 120293, 120678, 120294, 120680, 120296, 120682, 120278, 120684, - 120288, 120673, 120289, 120674, 120299, 120675, 120281, 120283, 120285, - 120292, 120297, 120298, 120300, 120714, 120716, 120687, 120717, 120713, - 120719, 120718, 120302, 120688, 120303, 120689, 120327, 120693, 120305, - 120691, 120306, 120692, 120694, 120307, 120705, 120308, 120690, 120310, - 120696, 120312, 120697, 120313, 120698, 120316, 120702, 120712, 120317, - 120709, 120711, 120703, 120319, 120704, 120320, 120706, 120321, 120695, - 120707, 120322, 120708, 120304, 120710, 120314, 120699, 120315, 120700, - 120325, 120701, 120309, 120311, 120318, 120323, 120324, 120326, 120715, - 120817, 120816, 120819, 120818, 120815, 120814, 120812, 120821, 120813, - 120820, 120328, 120329, 120330, 120331, 120332, 120333, 120334, 120335, - 120336, 120337, 120338, 120339, 120340, 120341, 120342, 120343, 120344, - 120345, 120346, 120347, 120348, 120349, 120350, 120351, 120352, 120353, - 120354, 120355, 120356, 120357, 120358, 120359, 120360, 120361, 120362, - 120363, 120364, 120365, 120366, 120367, 120368, 120369, 120370, 120371, - 120372, 120373, 120374, 120375, 120376, 120377, 120378, 120379, 120224, - 120225, 120226, 120227, 120228, 120229, 120230, 120231, 120232, 120233, - 120234, 120235, 120236, 120237, 120238, 120239, 120240, 120241, 120242, - 120243, 120244, 120245, 120246, 120247, 120248, 120249, 120250, 120251, - 120252, 120253, 120254, 120255, 120256, 120257, 120258, 120259, 120260, - 120261, 120262, 120263, 120264, 120265, 120266, 120267, 120268, 120269, - 120270, 120271, 120272, 120273, 120274, 120275, 120807, 120806, 120809, - 120808, 120805, 120804, 120802, 120811, 120803, 120810, 119964, 119966, - 119967, 119970, 119973, 119974, 119977, 119978, 119979, 119980, 119982, - 119983, 119984, 119985, 119986, 119987, 119988, 119989, 119990, 119991, - 119992, 119993, 119995, 119997, 119998, 119999, 120000, 120001, 120002, - 120003, 120005, 120006, 120007, 120008, 120009, 120010, 120011, 120012, - 120013, 120014, 120015, 129481, 119528, 119538, 119531, 119535, 119525, - 119524, 119534, 119529, 119539, 119527, 119537, 119526, 119536, 119533, - 119523, 119532, 119522, 119530, 119520, 119521, 128470, 175, 8737, 10667, - 10666, 10671, 10669, 10670, 10668, 10665, 10664, 10651, 10653, 8798, - 127830, 129470, 129471, 93773, 93764, 93790, 93787, 983268, 93783, - 983267, 93782, 93772, 93766, 93791, 93779, 93789, 93786, 93776, 93777, - 93785, 93775, 93770, 93769, 93771, 93774, 93780, 93760, 93767, 93781, - 93788, 93761, 93768, 93778, 93762, 93763, 93784, 93765, 93847, 93827, - 93846, 93826, 93845, 93825, 93844, 93829, 93828, 93831, 93830, 93824, - 93833, 93832, 93837, 93836, 93834, 93842, 93835, 93838, 93839, 93843, - 93840, 93841, 93805, 93796, 93822, 93819, 983270, 93815, 983269, 93814, - 93804, 93798, 93823, 93811, 93821, 93818, 93808, 93809, 93817, 93807, - 93802, 93801, 93803, 93806, 93812, 93792, 93799, 93813, 93820, 93793, - 93800, 93810, 93794, 93795, 93816, 93797, 93849, 93850, 93848, 11859, - 11852, 11860, 128901, 9899, 10090, 10091, 128967, 128965, 128944, 10100, - 10088, 10092, 10101, 10089, 10093, 8287, 9900, 9618, 128971, 128950, - 128938, 128963, 128961, 10073, 128974, 128956, 9898, 128911, 128931, - 43761, 44013, 43762, 43760, 44011, 43994, 43989, 43974, 43746, 43993, - 43991, 43751, 43750, 43992, 43986, 43987, 43990, 43968, 43995, 43976, - 43973, 43999, 43977, 44001, 43752, 43747, 43972, 43998, 43984, 43969, - 43753, 43754, 43975, 44000, 43978, 43749, 43748, 43983, 44002, 43970, - 43996, 43971, 43997, 43981, 43988, 43979, 43985, 43980, 43982, 43744, - 43745, 44012, 43763, 43764, 44005, 43757, 43759, 43758, 44009, 44003, - 44007, 44006, 44004, 43755, 44008, 43756, 44010, 43765, 43766, 44021, - 44020, 44023, 44022, 44019, 44018, 44016, 44025, 44017, 44024, 129760, - 127816, 125140, 125137, 125136, 125139, 125141, 125138, 125142, 124928, - 124929, 124930, 124936, 124937, 124949, 124950, 124948, 124938, 124956, - 124990, 124992, 124974, 124955, 124964, 124963, 124991, 124957, 124962, - 124976, 124996, 124997, 124998, 124982, 124983, 124984, 125004, 124975, - 125003, 125005, 125011, 125018, 125028, 125029, 125012, 125019, 125020, - 125027, 125013, 125035, 125048, 124942, 124934, 125090, 125046, 125078, - 125033, 124946, 125037, 125070, 125000, 125095, 125121, 124951, 125044, - 125041, 125072, 124987, 125066, 125076, 125074, 125009, 125107, 124945, - 125108, 125068, 125124, 125002, 124931, 125089, 125022, 125119, 124980, - 124986, 125080, 125099, 125100, 124933, 125021, 125015, 125071, 124985, - 125117, 125056, 124993, 125039, 125049, 125043, 125024, 124932, 125047, - 125097, 124959, 125069, 125088, 124999, 125123, 124952, 125036, 125026, - 125001, 125057, 125085, 124960, 125073, 124966, 125098, 125014, 125091, - 124989, 125007, 124978, 124940, 125106, 125050, 125030, 125092, 124941, - 125060, 125077, 125102, 125094, 125053, 125103, 125040, 125055, 125104, - 124939, 125017, 125112, 124961, 125087, 124970, 124971, 124969, 125023, - 124979, 125042, 124947, 125075, 125051, 125111, 125086, 124968, 124944, - 125038, 125096, 125016, 125118, 125109, 124953, 125059, 125052, 125006, - 124958, 125093, 125115, 125054, 124988, 125008, 125084, 125061, 125064, - 125120, 125063, 124967, 124977, 124965, 125031, 125081, 125082, 125010, - 125067, 124973, 125032, 124935, 125116, 125122, 125101, 124994, 124995, - 125113, 125058, 125079, 125114, 125065, 125034, 125083, 124954, 125062, - 125105, 125110, 125045, 124943, 124972, 124981, 125025, 125131, 125130, - 125133, 125132, 125129, 125128, 125135, 125127, 125134, 128334, 128697, - 68028, 68093, 68090, 68089, 68086, 68029, 68092, 68091, 68095, 68088, - 68087, 68094, 68000, 68016, 68020, 68021, 68022, 68009, 68010, 68015, - 68017, 68014, 68013, 68018, 68006, 68023, 68012, 68008, 68007, 68019, - 68011, 68005, 68004, 68001, 68002, 68003, 68031, 68030, 68039, 68075, - 68057, 68084, 68066, 68036, 68054, 68081, 68063, 68045, 68072, 68035, - 68053, 68080, 68062, 68044, 68071, 68040, 68076, 68058, 68085, 68067, - 68038, 68056, 68083, 68065, 68047, 68074, 68037, 68055, 68082, 68064, - 68046, 68073, 68034, 68052, 68079, 68061, 68043, 68070, 68033, 68051, - 68078, 68060, 68042, 68069, 68041, 68068, 68032, 68050, 68077, 68059, - 67974, 67975, 67982, 67983, 67978, 67979, 67980, 67981, 67987, 67988, - 67989, 67992, 67993, 67994, 67995, 67996, 67986, 67985, 67990, 67997, - 67984, 67977, 67976, 67991, 67973, 67972, 67968, 67969, 67970, 67971, - 67998, 67999, 129500, 9791, 983173, 9170, 9172, 9177, 9176, 9175, 9173, - 9174, 9171, 9169, 128647, 128221, 94015, 93989, 93971, 93958, 94019, - 94021, 93953, 94023, 93999, 93995, 94008, 93981, 93979, 93967, 93963, - 93993, 93992, 93983, 93977, 93976, 93975, 93974, 93968, 94032, 93988, - 93987, 93973, 93972, 93997, 93996, 93969, 94106, 94107, 94108, 94109, - 94110, 94111, 94002, 94026, 94022, 94003, 94004, 94010, 93980, 93978, - 94099, 94100, 94101, 94102, 94103, 94104, 94105, 93998, 93994, 94007, - 94025, 93966, 93962, 94024, 93961, 93960, 94000, 94009, 93964, 93965, - 94001, 93970, 93984, 93954, 94017, 94014, 94016, 94013, 94006, 94012, - 94005, 94011, 93986, 93985, 93955, 93952, 94020, 93990, 93957, 93956, - 93959, 93982, 94018, 93991, 94035, 94034, 94033, 94031, 94098, 94096, - 94097, 94095, 94036, 94039, 94040, 94067, 94068, 94038, 94037, 94073, - 94075, 94045, 94071, 94069, 94046, 94047, 94085, 94074, 94049, 94050, - 94051, 94052, 94053, 94086, 94057, 94054, 94084, 94055, 94056, 94041, - 94082, 94048, 94081, 94042, 94076, 94058, 94059, 94060, 94061, 94063, - 94064, 94079, 94087, 94062, 94065, 94080, 94066, 94072, 94070, 94044, - 94043, 94077, 94078, 94083, 983239, 983240, 128300, 127908, 181, 129440, - 129986, 183, 8943, 129686, 127894, 127756, 8357, 128189, 128469, 128656, + 43831, 621, 42894, 122641, 7557, 983611, 322, 42971, 411, 622, 122629, + 43829, 383, 7836, 7835, 7837, 682, 683, 42866, 457, 109, 7743, 43834, + 7535, 7558, 7747, 7745, 983613, 625, 7931, 7933, 42967, 42867, 110, 326, + 7755, 43835, 565, 328, 414, 7753, 626, 122663, 7536, 505, 324, 42897, + 7751, 7749, 7559, 627, 42917, 241, 329, 983589, 42868, 460, 111, 244, + 7893, 7891, 7889, 7897, 7895, 466, 246, 555, 559, 561, 7885, 525, 337, + 42827, 11386, 42829, 333, 7761, 7763, 491, 493, 248, 511, 245, 7759, + 7757, 557, 983577, 983579, 983581, 417, 7903, 7901, 7899, 7907, 7905, + 7887, 242, 243, 527, 335, 122651, 42945, 596, 983625, 983626, 7575, + 43839, 43874, 603, 7571, 42935, 419, 42831, 547, 112, 42835, 7549, 42833, + 42837, 7765, 7537, 7560, 421, 7767, 632, 113, 42841, 672, 587, 42839, + 569, 114, 343, 43849, 345, 7771, 7773, 7769, 529, 638, 7539, 122646, + 7775, 636, 637, 983615, 122664, 7538, 341, 531, 7561, 42919, 589, 43847, + 42843, 42998, 604, 7572, 605, 639, 122625, 600, 122631, 8580, 42815, + 122627, 42869, 42845, 612, 115, 347, 7781, 353, 7783, 351, 537, 349, + 122654, 7779, 7785, 7777, 42957, 42954, 575, 983583, 122665, 7540, 7562, + 42921, 642, 42892, 43872, 601, 983629, 983630, 7573, 602, 43851, 43852, + 609, 43830, 223, 7441, 7442, 7443, 7455, 7454, 7453, 42969, 645, 43845, + 116, 355, 7793, 539, 566, 357, 11366, 7831, 7789, 7787, 122666, 7541, + 7791, 427, 429, 122633, 648, 359, 679, 122647, 122652, 7546, 254, 42853, + 42855, 389, 424, 445, 7446, 42795, 397, 613, 686, 687, 7433, 42879, 7444, + 43842, 43841, 43843, 43844, 7432, 633, 43880, 122645, 634, 122632, 11385, + 635, 647, 122637, 652, 983627, 983628, 592, 594, 7426, 623, 624, 654, + 122630, 43857, 477, 7543, 670, 42881, 653, 42871, 680, 678, 43879, 11383, + 42793, 117, 649, 43855, 251, 7799, 468, 252, 474, 476, 472, 470, 7795, + 533, 369, 7909, 432, 7917, 7915, 7913, 7921, 7919, 7911, 363, 7803, + 983569, 983621, 983623, 371, 983617, 983619, 7577, 367, 361, 7801, 7797, + 249, 43854, 42937, 250, 535, 365, 43858, 650, 7551, 7531, 43856, 42872, + 43875, 118, 42847, 7807, 7564, 11380, 11377, 7805, 651, 42851, 42907, + 42909, 42911, 42857, 42849, 119, 373, 7813, 7817, 7815, 7809, 7811, 7832, + 11379, 120, 7821, 7819, 43863, 43864, 43865, 43862, 7565, 121, 375, 255, + 7925, 7823, 7923, 436, 7927, 7935, 43866, 591, 253, 563, 7929, 7833, 541, + 122, 378, 7825, 657, 382, 11372, 7827, 380, 7829, 576, 438, 7542, 7566, + 656, 549, 64256, 64259, 64260, 64257, 64258, 307, 64261, 64262, 339, + 8347, 8340, 8336, 8337, 8341, 7522, 11388, 8342, 8343, 8344, 8345, 8338, + 8346, 7523, 8348, 7524, 7525, 8339, 917505, 129726, 127811, 129388, + 129897, 129916, 129947, 10203, 10202, 129899, 129917, 117902, 128494, + 12296, 10641, 10643, 11058, 11056, 10576, 10571, 10570, 10574, 12304, + 10647, 123, 9128, 9129, 9127, 12300, 8968, 9948, 10714, 12298, 8220, + 11816, 11780, 129287, 129284, 129285, 129283, 129286, 9686, 11240, 9612, + 129977, 117924, 118288, 129970, 118285, 118283, 118435, 118440, 129940, + 129932, 128379, 128709, 11804, 10204, 9614, 9615, 129999, 10197, 8596, + 8700, 8697, 8622, 10568, 8660, 10500, 8654, 8621, 11012, 8703, 11020, + 129112, 11108, 11788, 10181, 8907, 9609, 8216, 11814, 128488, 91, 9123, + 9121, 10639, 10637, 8261, 10635, 11863, 11861, 9122, 11778, 10703, + 129900, 11785, 117771, 129985, 128492, 118434, 118441, 9610, 9613, 12308, + 129998, 8867, 12302, 10627, 12310, 10629, 12314, 12312, 10712, 128398, + 9611, 10620, 8970, 130027, 130019, 8905, 40, 9117, 9115, 9116, 11808, + 9144, 9001, 117856, 118265, 10748, 171, 117774, 128269, 117920, 117846, + 117922, 117911, 117861, 117762, 117918, 117880, 10553, 10154, 1422, + 117832, 117906, 117908, 129307, 4054, 4056, 117872, 117876, 9958, 8294, + 8237, 8234, 8206, 8592, 10563, 11074, 11083, 11082, 10611, 129973, 10618, + 10615, 11070, 8676, 8633, 10525, 129032, 8619, 11064, 8698, 10566, + 129040, 129024, 8602, 11024, 11025, 8610, 11066, 11065, 129028, 129176, + 129044, 8617, 8695, 8612, 10527, 129216, 8646, 10521, 129192, 129184, + 11144, 11013, 10603, 10599, 10590, 10582, 8637, 10594, 10602, 10598, + 10586, 10578, 8636, 8651, 129778, 129088, 129092, 11104, 11136, 11130, + 983241, 11174, 11172, 129064, 129060, 129056, 129072, 129068, 11120, + 11114, 11140, 129168, 10510, 8666, 129186, 11067, 11069, 11068, 11244, + 11061, 11060, 11062, 11063, 8606, 8678, 129172, 8604, 8656, 10498, 8653, + 10502, 10523, 10508, 8672, 129194, 129076, 129188, 8701, 8647, 129783, + 129190, 128620, 8668, 129080, 129104, 129084, 11077, 9804, 128006, 7213, + 7221, 7216, 7220, 7215, 7214, 7217, 7218, 7219, 7170, 7169, 7168, 7184, + 7183, 7182, 7247, 7193, 7180, 7188, 7187, 7186, 7185, 7172, 7171, 7198, + 7197, 7190, 7189, 7173, 7177, 7181, 7192, 7191, 7246, 7245, 7179, 7178, + 7175, 7174, 7201, 7200, 7176, 7196, 7195, 7199, 7202, 7194, 7203, 7227, + 7231, 7230, 7228, 7229, 7223, 7222, 7205, 7204, 7210, 7211, 7208, 7209, + 7206, 7212, 7207, 7237, 7236, 7239, 7238, 7235, 7234, 7232, 7241, 7233, + 7240, 10897, 10895, 10893, 10899, 10891, 10614, 10889, 10887, 8922, 8934, + 8808, 10918, 10920, 10885, 10877, 10881, 10883, 10879, 8818, 8804, 8822, + 8806, 10873, 10875, 8918, 60, 128210, 127898, 127819, 129461, 128626, + 128955, 128969, 128943, 11212, 128964, 10099, 128648, 10098, 9617, + 128937, 128949, 128978, 128960, 129653, 128910, 10072, 128930, 128504, + 9735, 128498, 128497, 6429, 6404, 6403, 6412, 6430, 6411, 6421, 6410, + 6405, 6415, 6425, 6426, 6427, 6419, 6418, 6407, 6406, 6414, 6413, 6409, + 6408, 6402, 6401, 6417, 6416, 6428, 6423, 6420, 6422, 6424, 6458, 6457, + 6459, 6464, 6449, 6452, 6450, 6448, 6456, 6454, 6453, 6455, 6451, 6442, + 6443, 6441, 6432, 6436, 6438, 6440, 6437, 6439, 6435, 6433, 6434, 6400, + 6468, 6469, 6475, 6474, 6477, 6476, 6473, 6472, 6470, 6479, 6471, 6478, + 13007, 10770, 10771, 10772, 983062, 8232, 983068, 983143, 67143, 67146, + 67151, 67165, 67166, 67167, 67157, 67158, 67159, 67160, 67161, 67162, + 67163, 67164, 67171, 67172, 67173, 67168, 67169, 67170, 67174, 67175, + 67176, 67177, 67178, 67179, 67230, 67231, 67180, 67181, 67182, 67183, + 67184, 67185, 67186, 67187, 67188, 67189, 67190, 67191, 67192, 67193, + 67194, 67195, 67196, 67197, 67198, 67199, 67200, 67201, 67202, 67203, + 67204, 67205, 67206, 67207, 67208, 67209, 67210, 67211, 67212, 67213, + 67214, 67215, 67216, 67217, 67218, 67219, 67220, 67221, 67222, 67223, + 67224, 67225, 67226, 67227, 67228, 67229, 67232, 67233, 67234, 67235, + 67236, 67237, 67238, 67239, 67240, 67241, 67242, 67243, 67244, 67245, + 67246, 67247, 67248, 67249, 67250, 67251, 67252, 67253, 67254, 67255, + 67256, 67257, 67258, 67259, 67260, 67261, 67262, 67263, 67264, 67274, + 67275, 67276, 67277, 67278, 67279, 67280, 67281, 67282, 67283, 67284, + 67285, 67286, 67287, 67288, 67289, 67290, 67291, 67292, 67293, 67294, + 67295, 67296, 67297, 67298, 67299, 67300, 67301, 67302, 67303, 67304, + 67265, 67266, 67267, 67268, 67269, 67270, 67271, 67272, 67273, 67325, + 67326, 67327, 67328, 67329, 67330, 67305, 67306, 67307, 67308, 67309, + 67310, 67311, 67312, 67313, 67314, 67315, 67316, 67317, 67318, 67319, + 67320, 67321, 67322, 67323, 67324, 67331, 67332, 67333, 67334, 67335, + 67336, 67337, 67338, 67349, 67350, 67351, 67352, 67353, 67354, 67355, + 67356, 67357, 67358, 67359, 67360, 67361, 67362, 67363, 67364, 67365, + 67366, 67367, 67368, 67378, 67379, 67380, 67381, 67382, 67369, 67370, + 67371, 67372, 67373, 67374, 67375, 67376, 67377, 67339, 67340, 67341, + 67342, 67343, 67344, 67345, 67346, 67347, 67348, 67401, 67404, 67403, + 67402, 67400, 67398, 67393, 67397, 67395, 67394, 67399, 67392, 67396, + 67406, 67408, 67409, 67410, 67407, 67405, 67411, 67413, 67412, 67424, + 67425, 67426, 67427, 67428, 67429, 67430, 67431, 67081, 67082, 67083, + 67084, 67085, 67087, 67088, 67089, 67090, 67091, 67092, 67093, 67094, + 67086, 67095, 67096, 67097, 67098, 67100, 67101, 67102, 67103, 67104, + 67105, 67106, 67107, 67108, 67109, 67110, 67111, 67112, 67113, 67114, + 67115, 67116, 67117, 67118, 67119, 67120, 67121, 67122, 67123, 67124, + 67125, 67126, 67127, 67128, 67129, 67130, 67131, 67132, 67133, 67134, + 67135, 67136, 67137, 67138, 67139, 67140, 67141, 67142, 67072, 67073, + 67074, 67075, 67076, 67077, 67078, 67079, 67080, 67145, 67147, 67148, + 67149, 67150, 67154, 67155, 67144, 67152, 67153, 67156, 67099, 65667, + 65668, 65669, 65670, 65671, 65672, 65673, 65675, 65674, 65677, 65676, + 65665, 65664, 65666, 65681, 65679, 65680, 65682, 65678, 65686, 65685, + 65687, 65693, 65690, 65691, 65692, 65694, 65703, 65696, 65695, 65697, + 65698, 65699, 65701, 65702, 65707, 65706, 65704, 65705, 65708, 65709, + 65710, 65711, 65712, 65713, 65719, 65717, 65714, 65715, 65716, 65718, + 65720, 65721, 65722, 65723, 65724, 65725, 65726, 65727, 65728, 65729, + 65731, 65730, 65732, 65733, 65737, 65734, 65735, 65736, 65738, 65739, + 65740, 65741, 65743, 65742, 65744, 65745, 65747, 65748, 65752, 65749, + 65750, 65751, 65753, 65754, 65755, 65756, 65757, 65779, 65780, 65781, + 65782, 65783, 65784, 65785, 65759, 65760, 65761, 65762, 65763, 65764, + 65765, 65766, 65767, 65768, 65769, 65770, 65771, 65772, 65773, 65774, + 65775, 65776, 65777, 65778, 65758, 65786, 65683, 65684, 65689, 65688, + 65700, 65746, 65561, 65543, 65587, 65582, 65589, 65536, 65541, 65579, + 65566, 65571, 65596, 65569, 65584, 65544, 65559, 65540, 65557, 65560, + 65580, 65606, 65600, 65599, 65577, 65562, 65538, 65573, 65563, 65609, + 65588, 65549, 65568, 65537, 65581, 65574, 65601, 65542, 65593, 65583, + 65594, 65552, 65547, 65605, 65578, 65545, 65585, 65586, 65546, 65570, + 65591, 65564, 65565, 65607, 65610, 65611, 65553, 65590, 65550, 65539, + 65576, 65608, 65551, 65554, 65592, 65603, 65597, 65567, 65572, 65558, + 65555, 65612, 65602, 65556, 65613, 65604, 65620, 65621, 65623, 65624, + 65626, 65627, 65628, 65629, 65622, 65616, 65617, 65619, 65618, 65625, + 128391, 128279, 128482, 128132, 42237, 42235, 42232, 42234, 42236, 42233, + 42206, 42205, 42197, 42196, 42204, 42195, 42228, 42229, 42230, 42213, + 42208, 42224, 42225, 42203, 42202, 42221, 42198, 42216, 42214, 42200, + 42199, 42194, 42193, 42219, 42210, 73648, 42220, 42211, 42212, 42222, + 42223, 42227, 42231, 42192, 42217, 42201, 42209, 42207, 42218, 42215, + 42226, 42238, 42239, 8356, 8374, 129409, 129422, 9806, 128274, 983079, + 983076, 128271, 117785, 117786, 117784, 117783, 117782, 117781, 8743, + 10848, 10846, 10833, 10844, 10842, 10847, 8744, 10841, 10851, 10850, + 10834, 10845, 10843, 10982, 10188, 129688, 10234, 10231, 10206, 10229, + 10235, 10232, 10237, 11059, 10230, 10236, 10233, 10238, 10239, 10205, + 129524, 128884, 129719, 128140, 127977, 128261, 129707, 12319, 8270, + 11847, 11848, 95, 118273, 9691, 118276, 118291, 129935, 118436, 118447, + 9604, 9697, 117765, 128394, 129852, 129853, 129870, 129872, 129868, + 129856, 129871, 129869, 129854, 129873, 129855, 128395, 128396, 128393, + 10559, 117934, 117936, 117932, 117930, 117972, 9695, 117960, 117948, + 117964, 117968, 117952, 117956, 117944, 117940, 117817, 129951, 9722, + 117820, 128397, 118428, 117935, 117937, 117933, 117931, 117973, 9694, + 117961, 117949, 117965, 117969, 117953, 117957, 117945, 117941, 117818, + 10558, 128318, 10065, 129950, 9727, 117823, 117767, 129863, 129865, + 129867, 129864, 129861, 129866, 129859, 129862, 129860, 129857, 129858, + 10195, 118431, 10063, 9998, 9987, 118429, 117821, 118430, 117822, 130021, + 9605, 118425, 118426, 118424, 117816, 118427, 117819, 9602, 9601, 9607, + 9603, 118437, 118446, 9606, 129903, 9674, 10208, 129438, 128557, 127853, + 129523, 128886, 129729, 66190, 66192, 66187, 66196, 66178, 66179, 66199, + 66185, 66200, 66176, 66201, 66177, 66202, 66191, 66193, 66181, 66180, + 66203, 66182, 66186, 66189, 66195, 66188, 66197, 66198, 66194, 66183, + 66204, 66184, 67887, 67892, 67881, 67895, 67891, 67886, 67872, 67893, + 67876, 67894, 67883, 67896, 67873, 67897, 67875, 67889, 67874, 67878, + 67880, 67882, 67884, 67890, 67885, 67888, 67877, 67879, 67903, 129317, + 983226, 983234, 983224, 983229, 8468, 129433, 983065, 129668, 129522, + 129497, 69986, 69981, 69991, 69985, 69984, 69990, 69989, 70002, 69997, + 69983, 69982, 69988, 69987, 69995, 69994, 69978, 69977, 69976, 69975, + 69980, 69979, 69974, 69973, 69993, 69992, 70001, 69998, 69996, 70000, + 69999, 69968, 69971, 69969, 69972, 69970, 70006, 70005, 70003, 70004, + 127012, 127019, 127008, 126990, 126999, 126976, 127005, 126987, 126996, + 127004, 126986, 126995, 126979, 127009, 126991, 127000, 127001, 126983, + 126992, 127011, 127010, 126977, 127007, 126989, 126998, 127006, 126988, + 126997, 127015, 127014, 127003, 126985, 126994, 127002, 126984, 126993, + 126978, 126982, 127017, 126981, 126980, 127016, 127018, 127013, 73464, + 73442, 73451, 73448, 73444, 73449, 73447, 73441, 73450, 73440, 73454, + 73445, 73443, 73453, 73456, 73446, 73455, 73452, 73457, 73463, 73461, + 73459, 73462, 73460, 73458, 128892, 3449, 3435, 3434, 3437, 3436, 3433, + 3432, 3430, 3439, 3431, 3438, 3419, 3420, 3446, 3417, 3422, 3416, 3447, + 3444, 3443, 3448, 3418, 3421, 3445, 3333, 3423, 3334, 3344, 3348, 3453, + 3454, 3414, 3451, 3450, 3452, 3455, 3412, 3413, 3355, 3354, 3406, 3362, + 3361, 3367, 3366, 3360, 3386, 3359, 3365, 3364, 3332, 3339, 3424, 3340, + 3425, 3381, 3369, 3363, 3353, 3358, 3368, 3380, 3379, 3378, 3377, 3376, + 3337, 3338, 3346, 3347, 3335, 3336, 3382, 3383, 3384, 3373, 3372, 3352, + 3351, 3357, 3356, 3350, 3349, 3371, 3370, 3342, 3343, 3385, 3374, 3375, + 3441, 3442, 3440, 3328, 3388, 3329, 3387, 3405, 3331, 3389, 3330, 3407, + 3390, 3400, 3404, 3393, 3394, 3395, 3396, 3426, 3427, 3402, 3403, 3391, + 3392, 3398, 3399, 3415, 9895, 9894, 9893, 9794, 10016, 129443, 128104, + 128372, 129333, 128114, 128115, 128378, 128107, 2137, 2122, 2121, 2133, + 2120, 2126, 2132, 2129, 2136, 2113, 2115, 2114, 2116, 2123, 2124, 2125, + 2128, 2130, 2131, 2118, 2134, 2117, 2112, 2127, 2119, 2135, 2138, 2139, + 2142, 68288, 68314, 68313, 68290, 68289, 68293, 68308, 68292, 68291, + 68300, 68299, 68298, 68297, 68306, 68304, 68320, 68318, 68323, 68312, + 68317, 68322, 68309, 68302, 68324, 68305, 68319, 68307, 68321, 68303, + 68294, 68301, 68311, 68295, 68316, 68315, 68310, 68331, 68335, 68334, + 68333, 68332, 68340, 68339, 68338, 68342, 68337, 68341, 68336, 68296, + 68326, 68325, 128094, 129469, 8380, 128368, 129389, 9967, 127809, 129671, + 72835, 72834, 72827, 72826, 72836, 72828, 72821, 72825, 72829, 72823, + 72822, 72819, 72818, 72831, 72830, 72844, 72845, 72838, 72839, 72840, + 72832, 72820, 72846, 72824, 72843, 72833, 72842, 72837, 72841, 72847, + 72886, 72885, 72867, 72866, 72859, 72858, 72868, 72860, 72853, 72857, + 72861, 72855, 72854, 72851, 72850, 72863, 72862, 72876, 72877, 72870, + 72871, 72864, 72852, 72878, 72856, 72875, 72865, 72874, 72869, 72873, + 72879, 72880, 72883, 72881, 72884, 72882, 72816, 72817, 9901, 129355, + 73007, 72980, 72979, 72983, 72982, 72988, 73008, 72987, 72960, 72961, + 72968, 72971, 72985, 72984, 72990, 72989, 72964, 72965, 72962, 72963, + 73005, 72999, 73006, 72973, 72972, 72976, 72986, 72981, 72991, 73001, + 73002, 73003, 72995, 72994, 72978, 72977, 72975, 72974, 72993, 72992, + 73004, 72996, 72998, 73000, 72997, 72966, 72969, 73031, 73030, 73027, + 73028, 73026, 73025, 73024, 73014, 73009, 73020, 73023, 73012, 73013, + 73010, 73011, 73018, 73021, 73029, 73045, 73044, 73047, 73046, 73043, + 73042, 73040, 73049, 73041, 73048, 186, 127405, 12348, 119811, 120778, + 120491, 119827, 120495, 120505, 120507, 119808, 120488, 119809, 120489, + 119833, 120493, 119812, 120492, 120494, 119814, 120490, 119816, 120496, + 119818, 120497, 119819, 120498, 119822, 120502, 120512, 119823, 120509, + 120511, 120503, 119825, 120504, 119826, 120506, 119828, 120508, 119810, + 120510, 119820, 120499, 119821, 120500, 119831, 120501, 119813, 119815, + 119817, 119824, 119829, 119830, 119832, 119837, 120779, 120517, 119834, + 120514, 119835, 120515, 119859, 120519, 119838, 120518, 120520, 119839, + 120531, 119840, 120516, 119842, 120522, 119844, 120523, 119845, 120524, + 119848, 120528, 120538, 119849, 120535, 120537, 120529, 119851, 120530, + 119852, 120532, 119853, 120521, 120533, 119854, 120534, 119836, 120536, + 119846, 120525, 119847, 120526, 119857, 120527, 119841, 119843, 119850, + 119855, 119856, 119858, 120016, 120017, 120018, 120019, 120020, 120021, + 120022, 120023, 120024, 120025, 120026, 120027, 120028, 120029, 120030, + 120031, 120032, 120033, 120034, 120035, 120036, 120037, 120038, 120039, + 120040, 120041, 120042, 120043, 120044, 120045, 120046, 120047, 120048, + 120049, 120050, 120051, 120052, 120053, 120054, 120055, 120056, 120057, + 120058, 120059, 120060, 120061, 120062, 120063, 120064, 120065, 120066, + 120067, 119931, 120611, 120621, 120623, 119912, 120604, 119913, 120605, + 119937, 120609, 119915, 120607, 119916, 120608, 120610, 119918, 120606, + 119920, 120612, 119922, 120613, 119923, 120614, 119926, 120618, 120628, + 119927, 120625, 120627, 120619, 119929, 120620, 119930, 120622, 119932, + 120624, 119914, 120626, 119924, 120615, 119925, 120616, 119935, 120617, + 119917, 119919, 119921, 119928, 119933, 119934, 119936, 120656, 120658, + 120629, 120659, 120655, 120661, 120660, 119938, 120630, 119939, 120631, + 119963, 120635, 119941, 120633, 119942, 120634, 120636, 119943, 120647, + 119944, 120632, 119946, 120638, 119948, 120639, 119949, 120640, 119952, + 120644, 120654, 119953, 120651, 120653, 120645, 119955, 120646, 119956, + 120648, 119957, 120637, 120649, 119958, 120650, 119940, 120652, 119950, + 120641, 119951, 120642, 119961, 120643, 119945, 119947, 119954, 119959, + 119960, 119962, 120657, 120540, 120542, 120513, 120543, 120539, 120545, + 120544, 120541, 120172, 120173, 120174, 120175, 120176, 120177, 120178, + 120179, 120180, 120181, 120182, 120183, 120184, 120185, 120186, 120187, + 120188, 120189, 120190, 120191, 120192, 120193, 120194, 120195, 120196, + 120197, 120198, 120199, 120200, 120201, 120202, 120203, 120204, 120205, + 120206, 120207, 120208, 120209, 120210, 120211, 120212, 120213, 120214, + 120215, 120216, 120217, 120218, 120219, 120220, 120221, 120222, 120223, + 120787, 120786, 120789, 120788, 120785, 120784, 120782, 120791, 120783, + 120790, 120120, 120121, 120123, 120124, 120125, 120126, 120128, 120129, + 120130, 120131, 120132, 120134, 120138, 120139, 120140, 120141, 120142, + 120143, 120144, 120146, 120147, 120148, 120149, 120150, 120151, 120152, + 120153, 120154, 120155, 120156, 120157, 120158, 120159, 120160, 120161, + 120162, 120163, 120164, 120165, 120166, 120167, 120168, 120169, 120170, + 120171, 120797, 120796, 120799, 120798, 120795, 120794, 120792, 120801, + 120793, 120800, 120068, 120069, 120071, 120072, 120073, 120074, 120077, + 120078, 120079, 120080, 120081, 120082, 120083, 120084, 120086, 120087, + 120088, 120089, 120090, 120091, 120092, 120094, 120095, 120096, 120097, + 120098, 120099, 120100, 120101, 120102, 120103, 120104, 120105, 120106, + 120107, 120108, 120109, 120110, 120111, 120112, 120113, 120114, 120115, + 120116, 120117, 120118, 120119, 10189, 119889, 120484, 120485, 120575, + 119886, 120572, 119887, 120573, 119911, 120577, 119890, 120576, 120578, + 119891, 120589, 119892, 120574, 119894, 120580, 119896, 120581, 119897, + 120582, 119900, 120586, 120596, 119901, 120593, 120595, 120587, 119903, + 120588, 119904, 120590, 119905, 120579, 120591, 119906, 120592, 119888, + 120594, 119898, 120583, 119899, 120584, 119909, 120585, 119895, 119902, + 119907, 119908, 119910, 119879, 120553, 120563, 120565, 119860, 120546, + 119861, 120547, 119885, 120551, 119863, 120549, 119864, 120550, 120552, + 119866, 120548, 119868, 120554, 119870, 120555, 119871, 120556, 119874, + 120560, 120570, 119875, 120567, 120569, 120561, 119877, 120562, 119878, + 120564, 119880, 120566, 119862, 120568, 119872, 120557, 119873, 120558, + 119883, 120559, 119865, 119867, 119869, 119876, 119881, 119882, 119884, + 120598, 120600, 120571, 120601, 120597, 120603, 120602, 120599, 120432, + 120433, 120434, 120435, 120436, 120437, 120438, 120439, 120440, 120441, + 120442, 120443, 120444, 120445, 120446, 120447, 120448, 120449, 120450, + 120451, 120452, 120453, 120454, 120455, 120456, 120457, 120458, 120459, + 120460, 120461, 120462, 120463, 120464, 120465, 120466, 120467, 120468, + 120469, 120470, 120471, 120472, 120473, 120474, 120475, 120476, 120477, + 120478, 120479, 120480, 120481, 120482, 120483, 120827, 120826, 120829, + 120828, 120825, 120824, 120822, 120831, 120823, 120830, 10215, 10221, + 10219, 10217, 10223, 10187, 10214, 10220, 10218, 10216, 10222, 120399, + 120727, 120737, 120739, 120380, 120720, 120381, 120721, 120405, 120725, + 120383, 120723, 120384, 120724, 120726, 120386, 120722, 120388, 120728, + 120390, 120729, 120391, 120730, 120394, 120734, 120744, 120395, 120741, + 120743, 120735, 120397, 120736, 120398, 120738, 120400, 120740, 120382, + 120742, 120392, 120731, 120393, 120732, 120403, 120733, 120385, 120387, + 120389, 120396, 120401, 120402, 120404, 120772, 120774, 120745, 120775, + 120771, 120777, 120776, 120406, 120746, 120407, 120747, 120431, 120751, + 120409, 120749, 120410, 120750, 120752, 120411, 120763, 120412, 120748, + 120414, 120754, 120416, 120755, 120417, 120756, 120420, 120760, 120770, + 120421, 120767, 120769, 120761, 120423, 120762, 120424, 120764, 120425, + 120753, 120765, 120426, 120766, 120408, 120768, 120418, 120757, 120419, + 120758, 120429, 120759, 120413, 120415, 120422, 120427, 120428, 120430, + 120773, 120295, 120669, 120679, 120681, 120276, 120662, 120277, 120663, + 120301, 120667, 120279, 120665, 120280, 120666, 120668, 120282, 120664, + 120284, 120670, 120286, 120671, 120287, 120672, 120290, 120676, 120686, + 120291, 120683, 120685, 120677, 120293, 120678, 120294, 120680, 120296, + 120682, 120278, 120684, 120288, 120673, 120289, 120674, 120299, 120675, + 120281, 120283, 120285, 120292, 120297, 120298, 120300, 120714, 120716, + 120687, 120717, 120713, 120719, 120718, 120302, 120688, 120303, 120689, + 120327, 120693, 120305, 120691, 120306, 120692, 120694, 120307, 120705, + 120308, 120690, 120310, 120696, 120312, 120697, 120313, 120698, 120316, + 120702, 120712, 120317, 120709, 120711, 120703, 120319, 120704, 120320, + 120706, 120321, 120695, 120707, 120322, 120708, 120304, 120710, 120314, + 120699, 120315, 120700, 120325, 120701, 120309, 120311, 120318, 120323, + 120324, 120326, 120715, 120817, 120816, 120819, 120818, 120815, 120814, + 120812, 120821, 120813, 120820, 120328, 120329, 120330, 120331, 120332, + 120333, 120334, 120335, 120336, 120337, 120338, 120339, 120340, 120341, + 120342, 120343, 120344, 120345, 120346, 120347, 120348, 120349, 120350, + 120351, 120352, 120353, 120354, 120355, 120356, 120357, 120358, 120359, + 120360, 120361, 120362, 120363, 120364, 120365, 120366, 120367, 120368, + 120369, 120370, 120371, 120372, 120373, 120374, 120375, 120376, 120377, + 120378, 120379, 120224, 120225, 120226, 120227, 120228, 120229, 120230, + 120231, 120232, 120233, 120234, 120235, 120236, 120237, 120238, 120239, + 120240, 120241, 120242, 120243, 120244, 120245, 120246, 120247, 120248, + 120249, 120250, 120251, 120252, 120253, 120254, 120255, 120256, 120257, + 120258, 120259, 120260, 120261, 120262, 120263, 120264, 120265, 120266, + 120267, 120268, 120269, 120270, 120271, 120272, 120273, 120274, 120275, + 120807, 120806, 120809, 120808, 120805, 120804, 120802, 120811, 120803, + 120810, 119964, 119966, 119967, 119970, 119973, 119974, 119977, 119978, + 119979, 119980, 119982, 119983, 119984, 119985, 119986, 119987, 119988, + 119989, 119990, 119991, 119992, 119993, 119995, 119997, 119998, 119999, + 120000, 120001, 120002, 120003, 120005, 120006, 120007, 120008, 120009, + 120010, 120011, 120012, 120013, 120014, 120015, 129481, 119528, 119538, + 119531, 119535, 119525, 119524, 119534, 119529, 119539, 119527, 119537, + 119526, 119536, 119533, 119523, 119532, 119522, 119530, 119520, 119521, + 128470, 175, 8737, 10667, 10666, 10671, 10669, 10670, 10668, 10665, + 10664, 10651, 10653, 8798, 127830, 129470, 129471, 93773, 93764, 93790, + 93787, 983270, 93783, 983269, 93782, 93772, 93766, 93791, 93779, 93789, + 93786, 93776, 93777, 93785, 93775, 93770, 93769, 93771, 93774, 93780, + 93760, 93767, 93781, 93788, 93761, 93768, 93778, 93762, 93763, 93784, + 93765, 93847, 93827, 93846, 93826, 93845, 93825, 93844, 93829, 93828, + 93831, 93830, 93824, 93833, 93832, 93837, 93836, 93834, 93842, 93835, + 93838, 93839, 93843, 93840, 93841, 93805, 93796, 93822, 93819, 983272, + 93815, 983271, 93814, 93804, 93798, 93823, 93811, 93821, 93818, 93808, + 93809, 93817, 93807, 93802, 93801, 93803, 93806, 93812, 93792, 93799, + 93813, 93820, 93793, 93800, 93810, 93794, 93795, 93816, 93797, 93849, + 93850, 93848, 11859, 11852, 11860, 128901, 9899, 10090, 10091, 128967, + 128965, 128944, 10100, 10088, 10092, 10101, 10089, 10093, 8287, 9900, + 9618, 128971, 128950, 128938, 128963, 128961, 10073, 128974, 128956, + 9898, 128911, 128931, 43761, 44013, 43762, 43760, 44011, 43994, 43989, + 43974, 43746, 43993, 43991, 43751, 43750, 43992, 43986, 43987, 43990, + 43968, 43995, 43976, 43973, 43999, 43977, 44001, 43752, 43747, 43972, + 43998, 43984, 43969, 43753, 43754, 43975, 44000, 43978, 43749, 43748, + 43983, 44002, 43970, 43996, 43971, 43997, 43981, 43988, 43979, 43985, + 43980, 43982, 43744, 43745, 44012, 43763, 43764, 44005, 43757, 43759, + 43758, 44009, 44003, 44007, 44006, 44004, 43755, 44008, 43756, 44010, + 43765, 43766, 44021, 44020, 44023, 44022, 44019, 44018, 44016, 44025, + 44017, 44024, 129760, 127816, 125140, 125137, 125136, 125139, 125141, + 125138, 125142, 124928, 124929, 124930, 124936, 124937, 124949, 124950, + 124948, 124938, 124956, 124990, 124992, 124974, 124955, 124964, 124963, + 124991, 124957, 124962, 124976, 124996, 124997, 124998, 124982, 124983, + 124984, 125004, 124975, 125003, 125005, 125011, 125018, 125028, 125029, + 125012, 125019, 125020, 125027, 125013, 125035, 125048, 124942, 124934, + 125090, 125046, 125078, 125033, 124946, 125037, 125070, 125000, 125095, + 125121, 124951, 125044, 125041, 125072, 124987, 125066, 125076, 125074, + 125009, 125107, 125108, 125068, 125124, 124945, 125002, 124931, 125089, + 125022, 124980, 125099, 124986, 125100, 125080, 125119, 124933, 125021, + 125015, 125071, 124985, 125117, 125056, 124993, 125039, 125049, 125043, + 125024, 124932, 125047, 125097, 124959, 125069, 125088, 124999, 125123, + 124952, 125036, 125026, 125001, 125085, 124960, 125057, 125073, 124966, + 125098, 125014, 125091, 124989, 125007, 124978, 124940, 125106, 125050, + 125030, 125092, 124941, 125060, 125077, 125102, 125094, 125053, 125040, + 125055, 125104, 125103, 124939, 125017, 124961, 125112, 125087, 124970, + 124971, 124969, 125023, 124979, 125042, 124947, 125086, 125075, 125051, + 125111, 124968, 124944, 125038, 125096, 125016, 125118, 125109, 124953, + 125059, 125052, 125006, 124958, 125093, 125115, 125054, 124988, 125008, + 125084, 125061, 125064, 125120, 125063, 124967, 124977, 124965, 125031, + 983275, 125081, 125082, 983276, 125010, 125067, 124973, 125032, 124935, + 125116, 125122, 125101, 124994, 124995, 125113, 125058, 125079, 125114, + 125065, 125034, 125083, 124954, 125062, 125105, 125110, 125045, 124943, + 124972, 124981, 125025, 125131, 125130, 125133, 125132, 125129, 125128, + 125135, 125127, 125134, 128334, 128697, 68028, 68093, 68090, 68089, + 68086, 68029, 68092, 68091, 68095, 68088, 68087, 68094, 68000, 68016, + 68020, 68021, 68022, 68009, 68010, 68015, 68017, 68014, 68013, 68018, + 68006, 68023, 68012, 68008, 68007, 68019, 68011, 68005, 68004, 68001, + 68002, 68003, 68031, 68030, 68039, 68075, 68057, 68084, 68066, 68036, + 68054, 68081, 68063, 68045, 68072, 68035, 68053, 68080, 68062, 68044, + 68071, 68040, 68076, 68058, 68085, 68067, 68038, 68056, 68083, 68065, + 68047, 68074, 68037, 68055, 68082, 68064, 68046, 68073, 68034, 68052, + 68079, 68061, 68043, 68070, 68033, 68051, 68078, 68060, 68042, 68069, + 68041, 68068, 68032, 68050, 68077, 68059, 67974, 67975, 67982, 67983, + 67978, 67979, 67980, 67981, 67987, 67988, 67989, 67992, 67993, 67994, + 67995, 67996, 67986, 67985, 67990, 67997, 67984, 67977, 67976, 67991, + 67973, 67972, 67968, 67969, 67970, 67971, 67998, 67999, 9791, 129500, + 983173, 9170, 9172, 9177, 9176, 9175, 9173, 9174, 9171, 9169, 128647, + 128221, 94015, 93989, 93971, 93958, 94019, 94021, 93953, 94023, 93999, + 93995, 94008, 93981, 93979, 93967, 93963, 93993, 93992, 93983, 93977, + 93976, 93975, 93974, 93968, 94032, 93988, 93987, 93973, 93972, 93997, + 93996, 93969, 94106, 94107, 94108, 94109, 94110, 94111, 94002, 94026, + 94022, 94003, 94004, 94010, 93980, 93978, 94099, 94100, 94101, 94102, + 94103, 94104, 94105, 93998, 93994, 94007, 94025, 93966, 93962, 94024, + 93961, 93960, 94000, 94009, 93964, 93965, 94001, 93970, 93984, 93954, + 94017, 94014, 94016, 94013, 94006, 94012, 94005, 94011, 93986, 93985, + 93955, 93952, 94020, 93990, 93957, 93956, 93959, 93982, 94018, 93991, + 94035, 94034, 94033, 94031, 94098, 94096, 94097, 94095, 94036, 94039, + 94040, 94067, 94068, 94038, 94037, 94073, 94075, 94045, 94071, 94069, + 94046, 94047, 94085, 94074, 94049, 94050, 94051, 94052, 94053, 94086, + 94057, 94054, 94084, 94055, 94056, 94041, 94082, 94048, 94081, 94042, + 94076, 94058, 94059, 94060, 94061, 94063, 94064, 94079, 94087, 94062, + 94065, 94080, 94066, 94072, 94070, 94044, 94043, 94077, 94078, 94083, + 983239, 983240, 128300, 127908, 181, 129440, 117772, 129986, 130022, + 130023, 183, 8943, 129686, 127894, 127756, 8357, 128189, 128469, 128656, 8722, 10793, 10794, 10796, 10795, 10810, 8770, 8723, 10751, 129694, 129705, 128241, 128242, 128244, 129339, 8871, 71232, 71229, 71236, 71231, - 71230, 71216, 71226, 71228, 71221, 71222, 71223, 71224, 71219, 71220, + 71230, 71216, 71226, 71228, 71219, 71220, 71221, 71222, 71223, 71224, 71217, 71218, 71225, 71227, 71234, 71233, 71253, 71252, 71255, 71254, 71251, 71250, 71248, 71257, 71249, 71256, 71168, 71169, 71179, 71181, 71195, 71194, 71200, 71199, 71193, 71192, 71198, 71197, 71174, 71175, @@ -11185,7 +11949,7 @@ static const unsigned int dawg_pos_to_codepoint[] = { 122941, 122959, 122989, 122955, 122950, 122948, 122944, 122951, 122988, 122953, 122934, 122935, 122936, 122933, 122949, 122931, 122957, 122930, 122947, 122937, 122928, 122940, 122945, 42653, 7544, 710, 735, 42889, - 67510, 42776, 42777, 42775, 750, 698, 725, 709, 984011, 42765, 42760, + 67510, 42776, 42775, 42777, 750, 698, 725, 709, 984011, 42765, 42760, 42770, 741, 984012, 42769, 42764, 42774, 745, 762, 764, 715, 704, 67507, 4348, 42766, 42761, 42771, 742, 721, 67511, 42888, 42768, 42763, 751, 767, 753, 42773, 717, 754, 755, 744, 759, 719, 718, 42783, 752, 716, @@ -11211,191 +11975,194 @@ static const unsigned int dawg_pos_to_codepoint[] = { 6287, 6278, 6285, 6284, 6288, 6277, 6291, 6290, 6293, 6294, 6292, 6283, 6281, 6185, 6196, 6264, 6210, 6190, 6303, 6305, 6307, 6300, 6302, 6304, 6312, 6298, 6301, 6314, 6308, 6309, 6299, 6306, 6263, 6262, 6260, 6261, - 6259, 6254, 6248, 6244, 6252, 6245, 6253, 6238, 6239, 6257, 6247, 6256, + 6259, 6244, 6252, 6245, 6253, 6238, 6239, 6254, 6248, 6257, 6247, 6256, 6242, 6258, 6255, 6241, 6240, 6249, 6251, 6250, 6243, 6246, 6237, 6193, - 6192, 6297, 6296, 6218, 6236, 6225, 6234, 6227, 6235, 6211, 6228, 6224, - 6222, 6232, 6226, 6233, 6214, 6216, 6215, 6217, 6219, 6231, 6223, 6220, + 6192, 6297, 6296, 6218, 6236, 6225, 6234, 6227, 6235, 6211, 6222, 6232, + 6228, 6224, 6226, 6233, 6214, 6216, 6215, 6217, 6219, 6231, 6223, 6220, 6221, 6230, 6229, 6212, 6213, 6204, 6194, 6209, 6207, 6205, 6206, 6203, 6202, 6208, 6191, 6177, 6183, 6179, 6181, 6180, 6182, 6186, 6195, 6201, 6189, 6197, 6184, 6187, 6188, 6199, 6200, 6198, 6178, 71273, 71275, 71274, 6151, 71265, 71270, 71269, 6144, 71268, 71264, 71267, 71276, 6150, 6158, 6148, 6146, 6152, 6153, 6154, 6145, 9866, 9867, 119552, 9101, - 128669, 128018, 128053, 127889, 129390, 128496, 129742, 129439, 128332, - 129334, 128741, 128757, 129468, 128739, 9968, 128670, 128672, 128693, - 128507, 128001, 129700, 128045, 128068, 128511, 127909, 92748, 92744, - 92761, 92750, 92739, 92751, 92737, 92754, 92749, 92753, 92743, 92752, - 92757, 92766, 92736, 92741, 92746, 92764, 92745, 92765, 92755, 92760, - 92758, 92756, 92763, 92762, 92747, 92738, 92740, 92759, 92742, 92783, - 92782, 92773, 92772, 92775, 92774, 92771, 92770, 92768, 92777, 92769, - 92776, 70291, 70292, 70290, 70297, 70296, 70293, 70287, 70298, 70312, - 70311, 70306, 70285, 70284, 70289, 70288, 70295, 70294, 70303, 70301, - 70283, 70282, 70280, 70278, 70277, 70276, 70300, 70299, 70310, 70307, - 70304, 70309, 70308, 70305, 70272, 70275, 70273, 70274, 70313, 127926, - 215, 10804, 10805, 10807, 10811, 10801, 10800, 10005, 8844, 8845, 8846, - 8888, 9838, 9837, 9839, 127929, 127896, 119161, 119159, 119155, 119157, - 119061, 119060, 119224, 119235, 119132, 119058, 119059, 119255, 119253, - 119130, 119131, 119163, 119169, 119149, 119167, 119168, 119210, 119178, - 119173, 119211, 119150, 119151, 119152, 119153, 119154, 119175, 119212, - 119213, 119166, 119164, 119141, 119142, 119176, 119179, 119143, 119144, - 119145, 119165, 119177, 119170, 119174, 119092, 119052, 119247, 119186, - 119073, 119109, 119093, 119050, 119220, 119221, 119044, 119049, 119187, - 119209, 119082, 119041, 119083, 119077, 119078, 119162, 119160, 119208, - 119156, 119158, 119136, 119102, 119056, 119057, 119146, 119147, 119148, - 119042, 119066, 119065, 119069, 119185, 119074, 119075, 119076, 119231, - 119232, 119085, 119084, 119070, 119071, 119072, 119218, 119217, 119189, - 119188, 119248, 119249, 119172, 119171, 119216, 119134, 119100, 119206, - 119262, 119270, 119271, 119263, 119268, 119269, 119272, 119264, 119267, - 119266, 119265, 119274, 119223, 119234, 119233, 119046, 119222, 119184, - 119227, 119237, 119228, 119122, 119123, 119081, 119098, 119207, 119129, - 119087, 119086, 119128, 119140, 119106, 119062, 119195, 119204, 119205, - 119196, 119197, 119198, 119199, 119200, 119201, 119202, 119203, 119094, - 119095, 119126, 119108, 119215, 119214, 119261, 119252, 119257, 119258, - 119183, 119090, 119091, 119135, 119101, 119096, 119097, 119053, 119054, - 119055, 119048, 119043, 119047, 119180, 119254, 119259, 119225, 119236, - 119226, 119229, 119238, 119230, 119051, 119045, 119089, 119088, 119040, - 119067, 119068, 119137, 119103, 119139, 119105, 119110, 119111, 119250, - 119181, 119273, 119243, 119244, 119245, 119246, 119242, 119240, 119239, - 119241, 119064, 119138, 119104, 119063, 119256, 119260, 119190, 119118, - 119119, 119120, 119121, 119112, 119113, 119116, 119117, 119114, 119115, - 119124, 119125, 119191, 119193, 119194, 119127, 119251, 119107, 119133, - 119099, 119219, 119192, 119182, 127932, 127925, 127812, 8811, 8810, 4158, - 4156, 4157, 4155, 4192, 4191, 4190, 4226, 4129, 43642, 4138, 4135, 4208, - 4207, 4206, 4159, 4099, 4098, 4097, 43625, 43624, 43626, 43623, 43622, - 43621, 43627, 43618, 43617, 43630, 43629, 43620, 43619, 983244, 43631, - 43616, 43635, 43628, 43633, 43634, 4096, 4188, 4189, 4187, 4186, 4136, - 4121, 4106, 4111, 4100, 4105, 4116, 4238, 4123, 4176, 43491, 4218, 4220, - 43490, 4221, 4224, 43492, 4223, 43489, 4216, 43488, 4215, 4214, 4213, - 4219, 4222, 4225, 4217, 4130, 43646, 43647, 4193, 4177, 4126, 4112, - 43503, 43495, 43502, 43501, 43516, 43515, 43518, 43517, 43498, 43497, - 43500, 43499, 43514, 43496, 4108, 4107, 4113, 4198, 4197, 4125, 4110, - 4109, 4115, 4114, 4178, 4179, 4180, 4181, 4133, 4134, 4131, 4132, 4128, - 4124, 4120, 4119, 4102, 4101, 4104, 4103, 4118, 4117, 4127, 4122, 4137, - 43636, 43637, 43638, 43632, 43494, 4154, 4150, 4250, 4251, 4237, 4235, - 4236, 4231, 4232, 4233, 4234, 43493, 4171, 43644, 43645, 4201, 4202, - 4203, 4204, 4205, 4151, 4239, 43643, 4170, 4153, 4152, 43639, 43641, - 43640, 4174, 4172, 4255, 4254, 4175, 4173, 4245, 4244, 4247, 4246, 4243, - 4242, 4240, 4249, 4241, 4248, 4195, 4196, 43509, 43508, 43511, 43510, - 43507, 43506, 43504, 43513, 43505, 43512, 4146, 4252, 4253, 4140, 4209, - 4212, 4210, 4211, 4147, 4148, 4194, 4228, 4229, 4230, 4227, 4145, 4149, - 4199, 4200, 4139, 4182, 4183, 4184, 4185, 4143, 4144, 4141, 4142, 4165, - 4164, 4167, 4166, 4163, 4162, 4160, 4169, 4161, 4168, 983218, 983232, - 983174, 10753, 10754, 10752, 8720, 10761, 10757, 10758, 8721, 8899, - 10756, 10755, 11007, 8896, 8897, 8898, 8719, 67712, 67728, 67740, 67724, - 67726, 67714, 67732, 67718, 67730, 67723, 67742, 67717, 67729, 67738, - 67739, 67713, 67735, 67716, 67721, 67734, 67737, 67741, 67725, 67719, - 67722, 67727, 67715, 67733, 67720, 67736, 67731, 67758, 67752, 67753, - 67757, 67751, 67759, 67756, 67754, 67755, 8711, 124117, 124120, 124119, - 124121, 124118, 124132, 124136, 124133, 124138, 124137, 124134, 124135, - 124122, 124124, 124126, 124123, 124125, 124112, 124116, 124114, 124113, - 124115, 124127, 124128, 124129, 124130, 124131, 124140, 124143, 124142, - 124139, 124141, 124149, 124148, 124151, 124150, 124147, 124146, 124144, - 124153, 124145, 124152, 128133, 8358, 8892, 72102, 72103, 72138, 72144, - 72136, 72096, 72097, 72107, 72109, 72123, 72122, 72128, 72127, 72121, - 72120, 72126, 72125, 72100, 72101, 72098, 72099, 72143, 72137, 72114, - 72124, 72119, 72129, 72139, 72140, 72141, 72133, 72132, 72116, 72115, - 72113, 72112, 72118, 72117, 72111, 72110, 72131, 72130, 72142, 72134, - 72135, 72106, 72108, 72162, 72161, 72158, 72160, 72159, 72164, 72150, - 72151, 72145, 72155, 72157, 72148, 72149, 72146, 72147, 72154, 72156, - 72163, 8302, 127966, 129314, 128219, 129535, 8239, 983092, 983197, - 983128, 9471, 9453, 9460, 9452, 9458, 9451, 9454, 9455, 9459, 9456, 9457, - 127312, 127313, 127314, 127315, 127316, 127317, 127318, 127319, 127320, - 127321, 127322, 127323, 127324, 127325, 127326, 127327, 127328, 127329, - 127330, 127331, 127332, 127333, 127334, 127335, 127336, 127337, 128982, - 128984, 129982, 129981, 129983, 127344, 127345, 127346, 127347, 127348, - 127349, 127350, 127351, 127352, 127353, 127354, 127355, 127356, 127357, - 127358, 127359, 127360, 127361, 127362, 127363, 127364, 127365, 127366, - 127367, 127368, 127369, 10062, 129988, 127374, 127371, 127375, 127372, - 127373, 983091, 8879, 8840, 8841, 8775, 8821, 8817, 8825, 8820, 8816, - 8824, 129670, 129722, 11228, 129540, 129544, 129565, 129586, 129603, - 129607, 129561, 129536, 129557, 129599, 129539, 129560, 129602, 129610, - 129613, 129541, 129562, 129604, 129537, 129558, 129600, 129538, 129559, - 129601, 129581, 129578, 129582, 129583, 129579, 129580, 128528, 9906, - 127770, 127761, 8362, 6595, 6594, 6599, 6598, 6597, 6596, 6593, 6566, - 6530, 6567, 6531, 6570, 6537, 6532, 6544, 6543, 6536, 6542, 6549, 6548, - 6562, 6561, 6554, 6560, 6556, 6550, 6528, 6555, 6538, 6568, 6533, 6569, - 6534, 6571, 6540, 6535, 6547, 6546, 6539, 6545, 6552, 6551, 6565, 6564, - 6557, 6563, 6559, 6553, 6529, 6558, 6541, 6622, 6623, 6600, 6601, 6618, - 6577, 6587, 6582, 6586, 6578, 6592, 6583, 6584, 6590, 6589, 6579, 6585, - 6591, 6580, 6588, 6576, 6581, 6613, 6612, 6615, 6614, 6611, 6610, 6608, - 6617, 6609, 6616, 983063, 70732, 70746, 70731, 70741, 70740, 70743, - 70742, 70739, 70738, 70736, 70745, 70737, 70744, 70734, 70675, 70674, - 70681, 70680, 70692, 70686, 70691, 70751, 70662, 70663, 70664, 70665, - 70656, 70657, 70667, 70669, 70685, 70684, 70690, 70689, 70683, 70682, - 70688, 70687, 70660, 70661, 70658, 70659, 70705, 70706, 70707, 70696, - 70695, 70677, 70676, 70673, 70672, 70679, 70678, 70671, 70670, 70703, - 70702, 70698, 70697, 70694, 70693, 70701, 70700, 70708, 70704, 70699, - 70666, 70668, 70727, 70724, 70723, 70726, 70752, 70728, 70753, 70722, - 70725, 70730, 70750, 70747, 70709, 70719, 70721, 70714, 70715, 70716, - 70717, 70712, 70713, 70710, 70711, 70718, 70720, 70735, 70749, 70733, - 70729, 11154, 11155, 128240, 9112, 983131, 9798, 11209, 128084, 129299, - 983132, 128985, 129399, 127747, 2031, 2032, 2033, 2030, 2034, 2027, 2028, - 2029, 2035, 2040, 2045, 2046, 1989, 1988, 1991, 1990, 1987, 1986, 1984, - 1993, 1985, 1992, 2042, 2008, 2001, 2025, 2024, 2026, 2006, 2002, 2019, - 2016, 2018, 2023, 2010, 2009, 2000, 1999, 2007, 1997, 1995, 2012, 2003, - 2013, 2020, 2014, 2015, 2017, 2004, 2011, 2005, 2021, 2022, 1994, 1996, - 1998, 2037, 2036, 2039, 2038, 2041, 2047, 983127, 128691, 9940, 128683, - 128695, 128370, 128286, 128245, 128685, 8303, 65934, 8209, 128689, 10973, - 8893, 8599, 10542, 10545, 10536, 10532, 10530, 128602, 128594, 128610, - 11111, 11127, 11016, 8663, 129109, 11008, 43063, 43062, 43065, 43064, - 43059, 43060, 43057, 43056, 43061, 43058, 10529, 8598, 8689, 8632, 10546, - 10535, 10531, 128600, 128592, 128608, 11110, 11126, 11017, 8662, 129108, - 11009, 128746, 8882, 8884, 8379, 8836, 8837, 8713, 8772, 8777, 8938, - 8940, 8742, 8930, 8931, 172, 8877, 8769, 8813, 8800, 8802, 8815, 8814, - 9083, 128323, 10159, 128324, 10161, 128456, 128457, 128458, 128211, - 128212, 160, 128067, 9369, 9362, 9365, 9366, 9367, 35, 9368, 9364, 9361, - 9363, 9371, 9370, 8470, 110960, 110961, 110962, 110963, 110964, 110965, - 110966, 110967, 110968, 110969, 110970, 110971, 110972, 110973, 110974, - 110975, 110976, 110977, 110978, 110979, 110980, 110981, 110982, 110983, - 110984, 110985, 110986, 110987, 110988, 110989, 110990, 110991, 110992, - 110993, 110994, 110995, 110996, 110997, 110998, 110999, 111000, 111001, - 111002, 111003, 111004, 111005, 111006, 111007, 111008, 111009, 111010, - 111011, 111012, 111013, 111014, 111015, 111016, 111017, 111018, 111019, - 111020, 111021, 111022, 111023, 111024, 111025, 111026, 111027, 111028, - 111029, 111030, 111031, 111032, 111033, 111034, 111035, 111036, 111037, - 111038, 111039, 111040, 111041, 111042, 111043, 111044, 111045, 111046, - 111047, 111048, 111049, 111050, 111051, 111052, 111053, 111054, 111055, - 111056, 111057, 111058, 111059, 111060, 111061, 111062, 111063, 111064, - 111065, 111066, 111067, 111068, 111069, 111070, 111071, 111072, 111073, - 111074, 111075, 111076, 111077, 111078, 111079, 111080, 111081, 111082, - 111083, 111084, 111085, 111086, 111087, 111088, 111089, 111090, 111091, - 111092, 111093, 111094, 111095, 111096, 111097, 111098, 111099, 111100, - 111101, 111102, 111103, 111104, 111105, 111106, 111107, 111108, 111109, - 111110, 111111, 111112, 111113, 111114, 111115, 111116, 111117, 111118, - 111119, 111120, 111121, 111122, 111123, 111124, 111125, 111126, 111127, - 111128, 111129, 111130, 111131, 111132, 111133, 111134, 111135, 111136, - 111137, 111138, 111139, 111140, 111141, 111142, 111143, 111144, 111145, - 111146, 111147, 111148, 111149, 111150, 111151, 111152, 111153, 111154, - 111155, 111156, 111157, 111158, 111159, 111160, 111161, 111162, 111163, - 111164, 111165, 111166, 111167, 111168, 111169, 111170, 111171, 111172, - 111173, 111174, 111175, 111176, 111177, 111178, 111179, 111180, 111181, - 111182, 111183, 111184, 111185, 111186, 111187, 111188, 111189, 111190, - 111191, 111192, 111193, 111194, 111195, 111196, 111197, 111198, 111199, - 111200, 111201, 111202, 111203, 111204, 111205, 111206, 111207, 111208, - 111209, 111210, 111211, 111212, 111213, 111214, 111215, 111216, 111217, - 111218, 111219, 111220, 111221, 111222, 111223, 111224, 111225, 111226, - 111227, 111228, 111229, 111230, 111231, 111232, 111233, 111234, 111235, - 111236, 111237, 111238, 111239, 111240, 111241, 111242, 111243, 111244, - 111245, 111246, 111247, 111248, 111249, 111250, 111251, 111252, 111253, - 111254, 111255, 111256, 111257, 111258, 111259, 111260, 111261, 111262, - 111263, 111264, 111265, 111266, 111267, 111268, 111269, 111270, 111271, - 111272, 111273, 111274, 111275, 111276, 111277, 111278, 111279, 111280, - 111281, 111282, 111283, 111284, 111285, 111286, 111287, 111288, 111289, - 111290, 111291, 111292, 111293, 111294, 111295, 111296, 111297, 111298, - 111299, 111300, 111301, 111302, 111303, 111304, 111305, 111306, 111307, - 111308, 111309, 111310, 111311, 111312, 111313, 111314, 111315, 111316, - 111317, 111318, 111319, 111320, 111321, 111322, 111323, 111324, 111325, - 111326, 111327, 111328, 111329, 111330, 111331, 111332, 111333, 111334, - 111335, 111336, 111337, 111338, 111339, 111340, 111341, 111342, 111343, - 111344, 111345, 111346, 111347, 111348, 111349, 111350, 111351, 111352, - 111353, 111354, 111355, 94177, 128297, 983041, 983040, 123215, 123149, - 123155, 123138, 123166, 123164, 123148, 123143, 123161, 123153, 123152, - 123141, 123137, 123156, 123139, 123163, 123142, 123172, 123173, 123171, - 123158, 123140, 123167, 123176, 123177, 123165, 123151, 123168, 123136, - 123169, 123162, 123178, 123179, 123144, 123157, 123170, 123150, 123145, - 123159, 123146, 123154, 123160, 123147, 123174, 123175, 123180, 123214, - 123193, 123192, 123195, 123191, 123194, 123196, 123197, 123184, 123190, - 123189, 123186, 123185, 123188, 123187, 123205, 123204, 123207, 123206, - 123203, 123202, 123200, 123209, 123201, 123208, 983231, 983066, 10663, + 128669, 128018, 128053, 127889, 129390, 118261, 128496, 129742, 129439, + 128332, 129334, 128741, 128757, 129468, 128739, 9968, 128670, 128672, + 128693, 128507, 128001, 129700, 128045, 128068, 128511, 127909, 92748, + 92744, 92761, 92750, 92739, 92751, 92737, 92754, 92749, 92753, 92743, + 92752, 92757, 92766, 92736, 92741, 92746, 92764, 92745, 92765, 92755, + 92760, 92758, 92756, 92763, 92762, 92747, 92738, 92740, 92759, 92742, + 92783, 92782, 92773, 92772, 92775, 92774, 92771, 92770, 92768, 92777, + 92769, 92776, 70291, 70292, 70290, 70297, 70296, 70293, 70287, 70298, + 70312, 70311, 70306, 70285, 70284, 70289, 70288, 70295, 70294, 70303, + 70301, 70283, 70282, 70280, 70278, 70277, 70276, 70300, 70299, 70310, + 70307, 70304, 70309, 70308, 70305, 70272, 70275, 70273, 70274, 70313, + 127926, 215, 10804, 10805, 10807, 10811, 10801, 10800, 10005, 8844, 8845, + 8846, 8888, 127812, 117860, 9838, 9837, 9839, 127929, 127896, 119161, + 119159, 119155, 119157, 119061, 119060, 119224, 119235, 119132, 119058, + 119059, 119255, 119253, 119130, 119131, 119163, 119169, 119149, 119167, + 119168, 119210, 119178, 119173, 119211, 119150, 119151, 119152, 119153, + 119154, 119175, 119212, 119213, 119166, 119164, 119141, 119142, 119176, + 119179, 119143, 119144, 119145, 119165, 119177, 119170, 119174, 119092, + 119052, 119247, 119186, 119073, 119109, 119093, 119050, 119220, 119221, + 119044, 119049, 119187, 119209, 119082, 119041, 119083, 119077, 119078, + 119162, 119160, 119208, 119156, 119158, 119136, 119102, 119056, 119057, + 119146, 119147, 119148, 119042, 119066, 119065, 119069, 119185, 119074, + 119075, 119076, 119231, 119232, 119085, 119084, 119070, 119071, 119072, + 119218, 119217, 119189, 119188, 119248, 119249, 119172, 119171, 119216, + 119134, 119100, 119206, 119262, 119270, 119271, 119263, 119268, 119269, + 119272, 119264, 119267, 119266, 119265, 119274, 119223, 119234, 119233, + 119046, 119222, 119184, 119227, 119237, 119228, 119122, 119123, 119081, + 119098, 119207, 119129, 119087, 119086, 119128, 119140, 119106, 119062, + 119195, 119204, 119205, 119196, 119197, 119198, 119199, 119200, 119201, + 119202, 119203, 119094, 119095, 119126, 119108, 119215, 119214, 119261, + 119252, 119257, 119258, 119183, 119090, 119091, 119135, 119101, 119096, + 119097, 119053, 119054, 119055, 119048, 119043, 119047, 119180, 119254, + 119259, 119225, 119236, 119226, 119229, 119238, 119230, 119051, 119045, + 119089, 119088, 119040, 119067, 119068, 119137, 119103, 119139, 119105, + 119110, 119111, 119250, 119181, 119273, 119243, 119244, 119245, 119246, + 119242, 119240, 119239, 119241, 119064, 119138, 119104, 119063, 119256, + 119260, 119190, 119118, 119119, 119120, 119121, 119112, 119113, 119116, + 119117, 119114, 119115, 119124, 119125, 119191, 119193, 119194, 119127, + 119251, 119107, 119133, 119099, 119219, 119192, 119182, 127932, 127925, + 8811, 8810, 4158, 4156, 4157, 4155, 4192, 4191, 4190, 4226, 4129, 43642, + 4138, 4135, 4208, 4207, 4206, 4159, 4099, 4098, 4097, 43625, 43624, + 43626, 43623, 43622, 43621, 43627, 43618, 43617, 43630, 43629, 43620, + 43619, 983244, 43631, 43616, 43635, 43628, 43633, 43634, 4096, 4188, + 4189, 4187, 4186, 4136, 4121, 4106, 4111, 4100, 4105, 4116, 4238, 4123, + 4176, 43491, 4218, 4220, 43490, 4221, 4224, 43492, 4223, 43489, 4216, + 43488, 4215, 4214, 4213, 4219, 4222, 4225, 4217, 4130, 43646, 43647, + 4193, 4177, 4126, 4112, 43503, 43495, 43502, 43501, 43516, 43515, 43518, + 43517, 43498, 43497, 43500, 43499, 43514, 43496, 4108, 4107, 4113, 4198, + 4197, 4125, 4110, 4109, 4115, 4114, 4133, 4134, 4178, 4179, 4180, 4181, + 4131, 4132, 4128, 4124, 4120, 4119, 4102, 4101, 4104, 4103, 4118, 4117, + 4127, 4122, 4137, 43636, 43637, 43638, 43632, 43494, 4245, 4244, 4247, + 4246, 4243, 4242, 4240, 4249, 4241, 4248, 4154, 4150, 4250, 4251, 4237, + 4235, 4236, 4231, 4232, 4233, 4234, 43493, 4171, 43644, 43645, 4201, + 4202, 4203, 4204, 4205, 4151, 4239, 43643, 4170, 4153, 4152, 43639, + 43641, 43640, 4174, 4172, 4255, 4254, 4175, 4173, 71391, 71390, 71393, + 71392, 71389, 71388, 71386, 71395, 71387, 71394, 4195, 4196, 43509, + 43508, 43511, 43510, 43507, 43506, 43504, 43513, 43505, 43512, 4146, + 4252, 4253, 4140, 4209, 4212, 4210, 4211, 4147, 4148, 4228, 4229, 4230, + 4227, 4194, 4145, 4149, 4199, 4200, 4139, 4143, 4144, 4182, 4183, 4184, + 4185, 4141, 4142, 71381, 71380, 71383, 71382, 71379, 71378, 71376, 71385, + 71377, 71384, 4165, 4164, 4167, 4166, 4163, 4162, 4160, 4169, 4161, 4168, + 983218, 983232, 983174, 10753, 10754, 10752, 8720, 10761, 10757, 10758, + 8721, 8899, 10756, 10755, 11007, 8896, 8897, 8898, 8719, 67712, 67728, + 67740, 67724, 67726, 67714, 67732, 67718, 67730, 67723, 67742, 67717, + 67729, 67738, 67739, 67713, 67735, 67716, 67721, 67734, 67737, 67741, + 67725, 67719, 67722, 67727, 67715, 67733, 67720, 67736, 67731, 67758, + 67752, 67753, 67757, 67751, 67759, 67756, 67754, 67755, 8711, 124117, + 124120, 124119, 124121, 124118, 124132, 124136, 124133, 124138, 124137, + 124134, 124135, 124122, 124124, 124126, 124123, 124125, 124112, 124116, + 124114, 124113, 124115, 124127, 124128, 124129, 124130, 124131, 124140, + 124143, 124142, 124139, 124141, 124149, 124148, 124151, 124150, 124147, + 124146, 124144, 124153, 124145, 124152, 128133, 8358, 8892, 72102, 72103, + 72138, 72096, 72097, 72107, 72109, 72123, 72122, 72128, 72127, 72144, + 72136, 72121, 72120, 72126, 72125, 72100, 72101, 72098, 72099, 72143, + 72137, 72114, 72124, 72119, 72129, 72139, 72140, 72141, 72133, 72132, + 72116, 72115, 72113, 72112, 72118, 72117, 72111, 72110, 72131, 72130, + 72142, 72134, 72135, 72106, 72108, 72162, 72161, 72158, 72160, 72159, + 72164, 72150, 72151, 72145, 72155, 72157, 72148, 72149, 72146, 72147, + 72154, 72156, 72163, 8302, 127966, 129314, 128219, 129535, 8239, 983092, + 983197, 983128, 9471, 9453, 9460, 9452, 9458, 9451, 9454, 9455, 9459, + 9456, 9457, 127312, 127313, 127314, 127315, 127316, 127317, 127318, + 127319, 127320, 127321, 127322, 127323, 127324, 127325, 127326, 127327, + 127328, 127329, 127330, 127331, 127332, 127333, 127334, 127335, 127336, + 127337, 128982, 128984, 129982, 129981, 129983, 127344, 127345, 127346, + 127347, 127348, 127349, 127350, 127351, 127352, 127353, 127354, 127355, + 127356, 127357, 127358, 127359, 127360, 127361, 127362, 127363, 127364, + 127365, 127366, 127367, 127368, 127369, 129204, 129205, 129207, 129988, + 10062, 127374, 127371, 127375, 129206, 127372, 127373, 983091, 8879, + 8840, 8841, 8775, 8821, 8817, 8825, 8820, 8816, 8824, 129670, 129722, + 11228, 129540, 129544, 129565, 129586, 129603, 129607, 129561, 129536, + 129557, 129599, 129539, 129560, 129602, 129610, 129613, 129541, 129562, + 129604, 129537, 129558, 129600, 129538, 129559, 129601, 129581, 129578, + 129582, 129583, 129579, 129580, 128528, 9906, 127770, 127761, 8362, 6595, + 6594, 6599, 6598, 6597, 6596, 6593, 6566, 6530, 6567, 6531, 6570, 6537, + 6532, 6544, 6543, 6536, 6542, 6549, 6548, 6562, 6561, 6554, 6560, 6556, + 6550, 6528, 6555, 6538, 6568, 6533, 6569, 6534, 6571, 6540, 6535, 6547, + 6546, 6539, 6545, 6552, 6551, 6565, 6564, 6557, 6563, 6559, 6553, 6529, + 6558, 6541, 6622, 6623, 6600, 6601, 6618, 6577, 6587, 6582, 6586, 6578, + 6592, 6583, 6584, 6590, 6589, 6579, 6585, 6591, 6580, 6588, 6576, 6581, + 6613, 6612, 6615, 6614, 6611, 6610, 6608, 6617, 6609, 6616, 983063, + 70732, 70746, 70731, 70741, 70740, 70743, 70742, 70739, 70738, 70736, + 70745, 70737, 70744, 70734, 70675, 70674, 70681, 70680, 70692, 70686, + 70691, 70751, 70662, 70663, 70664, 70665, 70656, 70657, 70667, 70669, + 70685, 70684, 70690, 70689, 70683, 70682, 70688, 70687, 70660, 70661, + 70658, 70659, 70705, 70706, 70707, 70696, 70695, 70677, 70676, 70673, + 70672, 70679, 70678, 70671, 70670, 70703, 70702, 70698, 70697, 70694, + 70693, 70701, 70700, 70708, 70704, 70699, 70666, 70668, 70723, 70726, + 70727, 70724, 70752, 70728, 70753, 70722, 70725, 70730, 70750, 70747, + 70709, 70719, 70721, 70712, 70713, 70714, 70715, 70716, 70717, 70710, + 70711, 70718, 70720, 70735, 70749, 70733, 70729, 11154, 11155, 128240, + 9112, 983131, 9798, 11209, 128084, 129299, 983132, 128985, 129399, + 127747, 2035, 2031, 2032, 2033, 2030, 2034, 2027, 2028, 2029, 2040, 2045, + 2046, 1989, 1988, 1991, 1990, 1987, 1986, 1984, 1993, 1985, 1992, 2042, + 2008, 2001, 2025, 2024, 2026, 2006, 2002, 2019, 2016, 2018, 2023, 2010, + 2009, 2000, 1999, 2007, 1997, 1995, 2012, 2003, 2013, 2020, 2014, 2015, + 2017, 2004, 2011, 2005, 2021, 2022, 1994, 1996, 1998, 2037, 2036, 2039, + 2038, 2041, 2047, 983127, 128691, 9940, 128683, 128695, 128370, 128286, + 128245, 128685, 8303, 65934, 8209, 128689, 10973, 8893, 8599, 10542, + 10545, 10536, 10532, 129209, 10530, 128602, 128594, 128610, 11111, 11127, + 11016, 8663, 129109, 11008, 43063, 43062, 43065, 43064, 43059, 43060, + 43057, 43056, 43061, 43058, 10529, 8598, 8689, 8632, 10546, 10535, 10531, + 129208, 128600, 128592, 128608, 11110, 11126, 11017, 8662, 129108, 11009, + 128746, 8882, 8884, 8379, 8836, 8837, 8713, 8772, 8777, 8938, 8940, 8742, + 8930, 8931, 172, 8877, 8769, 8813, 8800, 8802, 8815, 8814, 9083, 128323, + 10159, 128324, 10161, 128456, 128457, 128458, 128211, 128212, 160, + 128067, 9369, 9362, 9365, 9366, 9367, 35, 9368, 9364, 9361, 9363, 9371, + 9370, 8470, 110960, 110961, 110962, 110963, 110964, 110965, 110966, + 110967, 110968, 110969, 110970, 110971, 110972, 110973, 110974, 110975, + 110976, 110977, 110978, 110979, 110980, 110981, 110982, 110983, 110984, + 110985, 110986, 110987, 110988, 110989, 110990, 110991, 110992, 110993, + 110994, 110995, 110996, 110997, 110998, 110999, 111000, 111001, 111002, + 111003, 111004, 111005, 111006, 111007, 111008, 111009, 111010, 111011, + 111012, 111013, 111014, 111015, 111016, 111017, 111018, 111019, 111020, + 111021, 111022, 111023, 111024, 111025, 111026, 111027, 111028, 111029, + 111030, 111031, 111032, 111033, 111034, 111035, 111036, 111037, 111038, + 111039, 111040, 111041, 111042, 111043, 111044, 111045, 111046, 111047, + 111048, 111049, 111050, 111051, 111052, 111053, 111054, 111055, 111056, + 111057, 111058, 111059, 111060, 111061, 111062, 111063, 111064, 111065, + 111066, 111067, 111068, 111069, 111070, 111071, 111072, 111073, 111074, + 111075, 111076, 111077, 111078, 111079, 111080, 111081, 111082, 111083, + 111084, 111085, 111086, 111087, 111088, 111089, 111090, 111091, 111092, + 111093, 111094, 111095, 111096, 111097, 111098, 111099, 111100, 111101, + 111102, 111103, 111104, 111105, 111106, 111107, 111108, 111109, 111110, + 111111, 111112, 111113, 111114, 111115, 111116, 111117, 111118, 111119, + 111120, 111121, 111122, 111123, 111124, 111125, 111126, 111127, 111128, + 111129, 111130, 111131, 111132, 111133, 111134, 111135, 111136, 111137, + 111138, 111139, 111140, 111141, 111142, 111143, 111144, 111145, 111146, + 111147, 111148, 111149, 111150, 111151, 111152, 111153, 111154, 111155, + 111156, 111157, 111158, 111159, 111160, 111161, 111162, 111163, 111164, + 111165, 111166, 111167, 111168, 111169, 111170, 111171, 111172, 111173, + 111174, 111175, 111176, 111177, 111178, 111179, 111180, 111181, 111182, + 111183, 111184, 111185, 111186, 111187, 111188, 111189, 111190, 111191, + 111192, 111193, 111194, 111195, 111196, 111197, 111198, 111199, 111200, + 111201, 111202, 111203, 111204, 111205, 111206, 111207, 111208, 111209, + 111210, 111211, 111212, 111213, 111214, 111215, 111216, 111217, 111218, + 111219, 111220, 111221, 111222, 111223, 111224, 111225, 111226, 111227, + 111228, 111229, 111230, 111231, 111232, 111233, 111234, 111235, 111236, + 111237, 111238, 111239, 111240, 111241, 111242, 111243, 111244, 111245, + 111246, 111247, 111248, 111249, 111250, 111251, 111252, 111253, 111254, + 111255, 111256, 111257, 111258, 111259, 111260, 111261, 111262, 111263, + 111264, 111265, 111266, 111267, 111268, 111269, 111270, 111271, 111272, + 111273, 111274, 111275, 111276, 111277, 111278, 111279, 111280, 111281, + 111282, 111283, 111284, 111285, 111286, 111287, 111288, 111289, 111290, + 111291, 111292, 111293, 111294, 111295, 111296, 111297, 111298, 111299, + 111300, 111301, 111302, 111303, 111304, 111305, 111306, 111307, 111308, + 111309, 111310, 111311, 111312, 111313, 111314, 111315, 111316, 111317, + 111318, 111319, 111320, 111321, 111322, 111323, 111324, 111325, 111326, + 111327, 111328, 111329, 111330, 111331, 111332, 111333, 111334, 111335, + 111336, 111337, 111338, 111339, 111340, 111341, 111342, 111343, 111344, + 111345, 111346, 111347, 111348, 111349, 111350, 111351, 111352, 111353, + 111354, 111355, 94177, 128297, 983041, 983040, 123149, 123155, 123138, + 123166, 123164, 123148, 123143, 123161, 123153, 123152, 123141, 123137, + 123156, 123139, 123163, 123142, 123172, 123173, 123140, 123167, 123171, + 123158, 123176, 123177, 123165, 123151, 123168, 123136, 123169, 123162, + 123178, 123179, 123144, 123157, 123170, 123150, 123145, 123159, 123146, + 123154, 123160, 123147, 123174, 123175, 123180, 123214, 123193, 123192, + 123195, 123194, 123191, 123196, 123197, 123184, 123190, 123189, 123186, + 123185, 123188, 123187, 123215, 123205, 123204, 123207, 123206, 123203, + 123202, 123200, 123209, 123201, 123208, 117776, 983231, 983066, 10663, 10662, 11869, 9215, 65532, 9287, 9286, 9284, 9285, 9289, 9281, 9290, 9288, 9282, 9283, 9280, 128721, 128025, 128885, 127970, 5787, 5788, 5776, 5761, 5786, 5770, 5769, 5781, 5779, 5785, 5763, 5772, 5780, 5784, 5762, @@ -11404,34 +12171,39 @@ static const unsigned int dawg_pos_to_codepoint[] = { 7280, 7282, 7281, 7279, 7271, 7270, 7272, 7269, 7258, 7263, 7278, 7268, 7283, 7273, 7284, 7285, 7287, 7286, 7276, 7274, 7275, 7277, 7290, 7288, 7289, 7295, 7294, 7292, 7291, 7253, 7252, 7255, 7254, 7251, 7250, 7248, - 7257, 7249, 7256, 7293, 94179, 94178, 68736, 68739, 68744, 68737, 68756, - 68745, 68760, 68769, 68761, 68775, 68785, 68741, 68762, 68772, 68773, - 68740, 68777, 68742, 68749, 68750, 68758, 68759, 68774, 68776, 68783, - 68784, 68738, 68743, 68747, 68748, 68751, 68754, 68755, 68768, 68770, - 68782, 68765, 68780, 68766, 68781, 68763, 68767, 68764, 68778, 68757, - 68786, 68779, 68746, 68752, 68753, 68771, 68800, 68803, 68808, 68801, - 68820, 68809, 68824, 68833, 68825, 68839, 68849, 68805, 68826, 68836, - 68837, 68804, 68841, 68806, 68813, 68814, 68822, 68823, 68838, 68840, - 68847, 68848, 68802, 68807, 68811, 68812, 68815, 68818, 68819, 68832, - 68834, 68846, 68829, 68844, 68830, 68845, 68827, 68831, 68828, 68842, - 68821, 68850, 68843, 68810, 68816, 68817, 68835, 68861, 68859, 68858, - 68862, 68863, 68860, 66308, 66324, 66318, 66335, 66323, 66331, 66327, - 66330, 66315, 66316, 66317, 66329, 66314, 66306, 66322, 66351, 66321, - 66350, 66326, 66334, 66313, 66333, 66328, 66320, 66312, 66325, 66332, - 66305, 66307, 66311, 66309, 66349, 66310, 66304, 66319, 66339, 66337, - 66338, 66336, 68241, 68242, 68246, 68244, 68226, 68224, 68237, 68235, - 68249, 68251, 68247, 68233, 68248, 68252, 68227, 68234, 68230, 68239, - 68232, 68240, 68231, 68250, 68236, 68225, 68243, 68245, 68228, 68229, - 68238, 68255, 68254, 68253, 66404, 66390, 66392, 66387, 66388, 66411, - 66393, 66421, 66418, 66396, 66397, 66399, 66406, 66405, 66401, 66413, - 66402, 66398, 66414, 66415, 66416, 66408, 66420, 66417, 66407, 66419, - 66389, 66391, 66395, 66400, 66386, 66409, 66410, 66385, 66384, 66394, - 66412, 66403, 66516, 66514, 66515, 66517, 66513, 66464, 66504, 66505, - 66506, 66482, 66510, 66511, 66477, 66508, 66509, 66478, 66479, 66473, - 66474, 66490, 66491, 66480, 66475, 66476, 66507, 66471, 66469, 66470, - 66467, 66468, 66484, 66485, 66492, 66493, 66486, 66487, 66488, 66497, - 66498, 66495, 66472, 66483, 66499, 66494, 66481, 66489, 66496, 66465, - 66466, 66512, 128435, 69414, 69395, 69376, 69394, 69391, 69392, 69398, + 7257, 7249, 7256, 7293, 124374, 124376, 124375, 124377, 124378, 124379, + 124392, 124396, 124395, 124397, 124394, 124393, 124380, 124381, 124383, + 124384, 124385, 124382, 124368, 124371, 124370, 124369, 124372, 124373, + 124386, 124388, 124390, 124389, 124387, 124391, 124399, 124400, 124398, + 124415, 124406, 124405, 124408, 124407, 124404, 124403, 124401, 124410, + 124402, 124409, 94179, 94178, 68736, 68739, 68744, 68737, 68756, 68745, + 68760, 68769, 68761, 68775, 68785, 68741, 68762, 68772, 68773, 68740, + 68777, 68742, 68749, 68750, 68758, 68759, 68774, 68776, 68783, 68784, + 68738, 68743, 68747, 68748, 68751, 68754, 68755, 68768, 68770, 68782, + 68765, 68780, 68766, 68781, 68763, 68767, 68764, 68778, 68757, 68786, + 68779, 68746, 68752, 68753, 68771, 68800, 68803, 68808, 68801, 68820, + 68809, 68824, 68833, 68825, 68839, 68849, 68805, 68826, 68836, 68837, + 68804, 68841, 68806, 68813, 68814, 68822, 68823, 68838, 68840, 68847, + 68848, 68802, 68807, 68811, 68812, 68815, 68818, 68819, 68832, 68834, + 68846, 68829, 68844, 68830, 68845, 68827, 68831, 68828, 68842, 68821, + 68850, 68843, 68810, 68816, 68817, 68835, 68861, 68859, 68858, 68862, + 68863, 68860, 66308, 66324, 66318, 66335, 66323, 66331, 66327, 66330, + 66315, 66316, 66317, 66329, 66314, 66306, 66322, 66351, 66321, 66350, + 66326, 66334, 66313, 66333, 66328, 66320, 66312, 66325, 66332, 66305, + 66307, 66311, 66309, 66349, 66310, 66304, 66319, 66339, 66337, 66338, + 66336, 68241, 68242, 68246, 68244, 68226, 68224, 68237, 68235, 68249, + 68251, 68247, 68233, 68248, 68252, 68227, 68234, 68230, 68239, 68232, + 68240, 68231, 68250, 68236, 68225, 68243, 68245, 68228, 68229, 68238, + 68255, 68254, 68253, 66404, 66390, 66392, 66387, 66388, 66411, 66393, + 66421, 66418, 66396, 66397, 66399, 66406, 66405, 66401, 66413, 66402, + 66398, 66414, 66415, 66416, 66408, 66420, 66417, 66407, 66419, 66389, + 66391, 66395, 66400, 66386, 66409, 66410, 66385, 66384, 66394, 66412, + 66403, 66516, 66514, 66515, 66517, 66513, 66464, 66504, 66505, 66506, + 66482, 66510, 66511, 66477, 66508, 66509, 66478, 66479, 66473, 66474, + 66490, 66491, 66480, 66475, 66476, 66507, 66471, 66469, 66470, 66467, + 66468, 66484, 66485, 66492, 66493, 66486, 66487, 66488, 66497, 66498, + 66495, 66472, 66483, 66499, 66494, 66481, 66489, 66496, 66465, 66466, + 66512, 128435, 118450, 69414, 69395, 69376, 69394, 69391, 69392, 69398, 69399, 69403, 69404, 69377, 69379, 69382, 69400, 69388, 69380, 69384, 69393, 69397, 69401, 69386, 69381, 69385, 69387, 69378, 69390, 69402, 69383, 69396, 69389, 69415, 69407, 69412, 69411, 69406, 69410, 69405, @@ -11449,7 +12221,7 @@ static const unsigned int dawg_pos_to_codepoint[] = { 69507, 69506, 69488, 69502, 69496, 69505, 69492, 69499, 69501, 69503, 69494, 69493, 69490, 69495, 69489, 69498, 69504, 69491, 69500, 69497, 69511, 69512, 69513, 69510, 128477, 128117, 129491, 128116, 129746, - 128283, 128664, 128753, 128660, 128662, 128653, 11819, 8228, 128431, + 128283, 128664, 128753, 128662, 128660, 128653, 11819, 8228, 128431, 129649, 129477, 128214, 9251, 10044, 10027, 10034, 10011, 128194, 128449, 128080, 128237, 128236, 10180, 10179, 128275, 9103, 9104, 10174, 983191, 8997, 128191, 128440, 9934, 9741, 128217, 129505, 129447, 8886, 2902, @@ -11458,214 +12230,230 @@ static const unsigned int dawg_pos_to_codepoint[] = { 2852, 2827, 2912, 2828, 2913, 2869, 2825, 2826, 2823, 2824, 2867, 2866, 2841, 2851, 2846, 2856, 2870, 2871, 2872, 2861, 2860, 2843, 2842, 2840, 2839, 2845, 2844, 2838, 2837, 2859, 2858, 2873, 2862, 2929, 2831, 2835, - 983660, 983659, 2877, 2818, 2817, 2876, 2901, 2893, 2819, 2878, 2888, - 2892, 2883, 2884, 2914, 2915, 2881, 2882, 2879, 2880, 2887, 2891, 2923, + 983660, 983659, 2817, 2876, 2877, 2818, 2901, 2893, 2819, 2878, 2888, + 2892, 2881, 2882, 2883, 2884, 2914, 2915, 2879, 2880, 2887, 2891, 2923, 2922, 2925, 2924, 2921, 2920, 2918, 2927, 2919, 2926, 64830, 64831, 9766, - 10183, 128895, 66736, 66737, 66738, 66739, 66743, 66763, 66761, 66742, - 66749, 66757, 66744, 66768, 66750, 66748, 66754, 66755, 66764, 66762, - 66760, 66746, 66745, 66741, 66765, 66769, 66759, 66758, 66771, 66770, - 66740, 66751, 66752, 66753, 66756, 66767, 66747, 66766, 66776, 66777, - 66778, 66779, 66783, 66803, 66801, 66782, 66789, 66797, 66784, 66808, - 66790, 66788, 66794, 66795, 66804, 66802, 66800, 66786, 66785, 66781, - 66805, 66809, 66799, 66798, 66811, 66810, 66780, 66791, 66792, 66793, - 66796, 66807, 66787, 66806, 66710, 66688, 66715, 66699, 66694, 66698, - 66703, 66693, 66697, 66696, 66705, 66702, 66713, 66717, 66704, 66706, - 66707, 66711, 66716, 66689, 66701, 66700, 66708, 66691, 66695, 66690, - 66692, 66709, 66712, 66714, 66725, 66724, 66727, 66726, 66723, 66722, - 66720, 66729, 66721, 66728, 983192, 126257, 126264, 126258, 126259, - 126265, 126260, 126263, 126267, 126255, 126266, 126256, 126262, 126261, - 126269, 126268, 126254, 126216, 126225, 126252, 126234, 126243, 126222, - 126249, 126213, 126231, 126240, 126221, 126248, 126212, 126230, 126239, - 126217, 126226, 126253, 126235, 126244, 126215, 126224, 126251, 126233, - 126242, 126214, 126223, 126250, 126232, 126241, 126220, 126247, 126211, - 126229, 126238, 126219, 126246, 126210, 126228, 126237, 126218, 126245, - 126209, 126227, 126236, 129446, 128228, 10015, 9885, 10029, 10009, 8485, - 129397, 128471, 11195, 11194, 11196, 8254, 129450, 8486, 128076, 127842, - 128329, 129417, 128002, 983122, 983121, 128463, 128195, 128479, 128196, - 128223, 128464, 128724, 93059, 93065, 93064, 93058, 93070, 93056, 93055, - 93053, 93062, 93069, 93061, 93066, 93071, 93068, 93054, 93057, 93063, - 93067, 93060, 92967, 92975, 92965, 92969, 92959, 92968, 92971, 92957, - 92962, 92960, 92972, 92970, 92963, 92958, 92966, 92961, 92956, 92974, - 92964, 92973, 92979, 92978, 92980, 92977, 92976, 92982, 92981, 93023, - 93020, 93024, 93021, 93019, 93025, 93022, 93043, 92985, 93047, 93044, - 93045, 92997, 93046, 93042, 93032, 93029, 92995, 93038, 92993, 93041, - 93035, 93033, 93037, 93030, 93039, 92987, 92992, 92986, 92983, 92984, - 93027, 92994, 93034, 92988, 92990, 92989, 92991, 93028, 92996, 93031, - 93040, 93036, 92954, 92955, 92938, 92939, 92932, 92933, 92942, 92943, - 92950, 92951, 92928, 92929, 92936, 92937, 92948, 92949, 92930, 92931, - 92944, 92945, 92934, 92935, 92940, 92941, 92946, 92947, 92952, 92953, - 93013, 93012, 93015, 93014, 93011, 93010, 93008, 93017, 93009, 93016, - 9908, 11801, 127796, 129779, 129780, 67703, 67693, 67688, 67702, 67683, - 67691, 67699, 67700, 67680, 67696, 67682, 67686, 67695, 67698, 67701, - 67689, 67684, 67687, 67690, 67681, 67694, 67685, 67697, 67692, 67704, - 67711, 67706, 67707, 67710, 67709, 67708, 67705, 129330, 129374, 128060, - 11853, 8233, 11791, 10995, 10994, 8741, 129666, 12809, 12823, 12808, - 12822, 12828, 12813, 12827, 12810, 12824, 12800, 12814, 12804, 12818, - 12801, 12815, 12812, 12826, 12805, 12819, 12803, 12817, 12806, 12820, - 12811, 12825, 12802, 12816, 12807, 12821, 12863, 12855, 12858, 12861, - 12847, 12839, 12854, 12843, 12836, 12864, 12835, 12856, 12846, 12842, - 12840, 12852, 12862, 12865, 12857, 12867, 12838, 12866, 12851, 12853, - 12859, 12849, 12860, 12848, 12837, 12834, 12841, 12833, 12845, 12844, - 12850, 12832, 12830, 12829, 9349, 9342, 9345, 9346, 9350, 9347, 9348, - 9344, 9351, 9343, 9341, 9336, 9335, 9338, 9337, 9334, 9333, 9340, 9332, - 9339, 127248, 127249, 127250, 127251, 127252, 127253, 127254, 127255, - 127256, 127257, 127258, 127259, 127260, 127261, 127262, 127263, 127264, - 127265, 127266, 127267, 127268, 127269, 127270, 127271, 127272, 127273, - 9372, 9373, 9374, 9375, 9376, 9377, 9378, 9379, 9380, 9381, 9382, 9383, - 9384, 9385, 9386, 9387, 9388, 9389, 9390, 9391, 9392, 9393, 9394, 9395, - 9396, 9397, 8706, 983147, 983146, 983149, 983150, 9853, 127881, 12880, - 12349, 129436, 11261, 128755, 9105, 9106, 128706, 72437, 72440, 72432, - 72416, 72419, 72413, 72417, 72415, 72412, 72414, 72418, 72420, 72403, - 72407, 72411, 72409, 72410, 72391, 72400, 72404, 72397, 72394, 72385, - 72401, 72384, 72399, 72398, 72396, 72388, 72393, 72392, 72386, 72387, - 72402, 72395, 72390, 72389, 72405, 72406, 72408, 72436, 72435, 72438, - 72439, 72422, 72421, 72424, 72425, 72431, 72433, 72434, 72428, 72427, - 72429, 72430, 72423, 72426, 128062, 128230, 128206, 983228, 983237, - 129434, 9774, 127825, 129372, 129755, 127824, 128039, 128532, 9956, - 128390, 9999, 8240, 8241, 8524, 10178, 10977, 129336, 129494, 129496, - 128113, 9977, 128590, 129733, 128591, 129493, 128589, 128588, 129495, - 128583, 128187, 8966, 128547, 37, 9854, 127917, 8359, 8369, 129515, - 128694, 129730, 43101, 43117, 43120, 43076, 43123, 43077, 43115, 43090, - 43082, 43094, 43098, 43099, 43119, 43118, 43109, 43074, 43075, 43116, - 43079, 43083, 43089, 43088, 43114, 43113, 43081, 43080, 43073, 43072, - 43085, 43084, 43092, 43093, 43104, 43110, 43086, 43108, 43100, 43078, - 43097, 43087, 43106, 43096, 43091, 43107, 43095, 43102, 43105, 43103, - 43127, 43126, 43124, 43121, 43111, 43112, 43122, 43125, 66033, 66023, - 66017, 66010, 66027, 66003, 66018, 66028, 66004, 66012, 66022, 66020, - 66045, 66019, 66031, 66041, 66007, 66006, 66025, 66026, 66038, 66016, - 66013, 66014, 66000, 66001, 66034, 66036, 66037, 66029, 66011, 66024, - 66015, 66021, 66042, 66043, 66002, 66008, 66032, 66005, 66044, 66040, - 66039, 66030, 66009, 66035, 5941, 5942, 67840, 67855, 67843, 67844, - 67847, 67858, 67857, 67860, 67854, 67861, 67848, 67845, 67846, 67849, - 67859, 67842, 67850, 67853, 67851, 67841, 67856, 67852, 67864, 67866, - 67867, 67863, 67862, 67865, 67871, 11227, 9935, 128763, 128022, 128061, - 128055, 128169, 182, 128138, 129292, 129295, 127885, 127821, 10031, - 129655, 129669, 9811, 128299, 8916, 10970, 129383, 8984, 128720, 129703, - 8462, 8463, 127183, 127136, 127167, 127199, 127188, 127140, 127156, - 127172, 127200, 127189, 127141, 127157, 127173, 127196, 127148, 127164, - 127180, 127198, 127150, 127166, 127182, 127192, 127144, 127160, 127176, - 127191, 127143, 127159, 127175, 127190, 127142, 127158, 127174, 127197, - 127149, 127165, 127181, 127194, 127146, 127162, 127178, 127187, 127139, - 127155, 127171, 127186, 127138, 127154, 127170, 127202, 127220, 127221, - 127201, 127210, 127211, 127212, 127213, 127214, 127215, 127216, 127217, - 127218, 127219, 127203, 127204, 127205, 127206, 127207, 127208, 127209, - 127185, 127137, 127153, 127169, 127193, 127145, 127161, 127177, 127195, - 127147, 127163, 127179, 128733, 983151, 43, 10797, 10798, 10809, 10789, - 10786, 10791, 10790, 10788, 10792, 10787, 10866, 177, 9799, 11222, 11221, - 11220, 11219, 129696, 983148, 128659, 128680, 128110, 8297, 8236, 127871, - 128254, 11239, 128239, 12306, 12320, 128238, 8982, 128688, 129364, - 127858, 129716, 127831, 129751, 128574, 128545, 163, 128093, 9212, 9213, - 9214, 9211, 128041, 128425, 129328, 129732, 129731, 65043, 65040, 65045, - 65073, 65074, 65049, 65041, 65042, 65091, 65047, 65083, 65085, 65089, - 65079, 65087, 65077, 65095, 65081, 65075, 65084, 65086, 65092, 983261, - 65048, 65090, 65080, 65088, 65078, 65096, 65082, 65076, 65044, 65072, - 65046, 8478, 8826, 10937, 10933, 10927, 10929, 10935, 10931, 8936, 8830, - 8828, 8880, 9111, 129384, 9113, 128424, 128438, 129332, 128120, 983193, - 983166, 983163, 983164, 983167, 8242, 8965, 8759, 8733, 8522, 128711, - 129455, 11224, 128255, 68507, 68508, 68480, 68483, 68490, 68491, 68485, - 68482, 68486, 68493, 68495, 68496, 68488, 68484, 68487, 68489, 68481, - 68492, 68497, 68494, 68526, 68522, 68523, 68525, 68521, 68527, 68524, - 68505, 68506, 11854, 8200, 128156, 128091, 128686, 128204, 128226, - 983165, 983168, 983194, 9624, 9625, 9626, 9627, 9628, 9629, 9630, 9631, - 9622, 9623, 10764, 8279, 10774, 9833, 128894, 8264, 63, 8799, 34, 9915, - 127949, 127950, 129437, 128251, 9762, 128280, 9143, 128740, 128643, 9926, - 127752, 11827, 11783, 11782, 9995, 128400, 128406, 127338, 127339, - 127340, 9994, 11828, 11787, 129306, 128000, 8758, 128007, 128048, 129682, - 128015, 129534, 9852, 9843, 9844, 9845, 9846, 9847, 9848, 9849, 9850, - 983113, 128665, 127822, 129511, 174, 127462, 127463, 127464, 127465, - 127466, 127467, 127468, 127469, 127470, 127471, 127472, 127473, 127474, - 127475, 127476, 127477, 127478, 127479, 127480, 127481, 127482, 127483, - 127484, 127485, 127486, 127487, 43344, 43343, 43346, 43345, 43330, 43320, - 43333, 43323, 43331, 43314, 43332, 43317, 43319, 43321, 43316, 43313, - 43329, 43322, 43312, 43326, 43318, 43325, 43324, 43315, 43328, 43327, - 43334, 43359, 43337, 43342, 43341, 43338, 43340, 43335, 43339, 43336, - 43347, 128524, 127895, 65533, 9952, 9953, 128699, 8479, 9166, 11152, - 11153, 128639, 128968, 983152, 92, 10184, 10741, 10743, 11073, 11079, - 983153, 10659, 10661, 8246, 12317, 10989, 8976, 11793, 8267, 8245, - 128401, 9753, 11262, 8515, 8271, 8765, 8909, 8247, 128402, 128403, - 128405, 11841, 11822, 128404, 10672, 128158, 8251, 129423, 983154, - 127872, 11190, 11188, 11191, 11189, 11186, 11187, 11184, 11185, 127832, - 127833, 129919, 129918, 128495, 8735, 12297, 10642, 11777, 11776, 9084, - 8894, 10652, 10644, 10228, 8692, 12305, 10648, 125, 9132, 9133, 9131, - 12301, 8969, 11781, 12299, 10608, 10715, 8221, 11817, 129929, 10621, - 8971, 9687, 11241, 9616, 129978, 129971, 129933, 128381, 11805, 8906, - 10198, 129927, 9621, 129980, 41, 9120, 9118, 9119, 11789, 10182, 8908, - 129931, 8217, 11815, 128360, 128361, 128362, 128489, 93, 9126, 9124, - 10638, 10640, 8262, 10636, 11864, 11862, 9125, 11779, 129987, 128493, - 129928, 129930, 11786, 8895, 10702, 129902, 12309, 8866, 11809, 9145, - 12303, 10628, 12311, 10630, 12315, 12313, 10713, 1421, 4053, 4055, - 129308, 9957, 10749, 9002, 187, 128270, 10153, 10552, 8295, 8238, 8235, - 8207, 10813, 8594, 11080, 11084, 10562, 10613, 10612, 129974, 8614, - 10528, 11076, 11075, 10567, 10526, 8677, 8628, 10513, 8699, 129034, 8620, - 10565, 129042, 129026, 8603, 11022, 11023, 8611, 10517, 10516, 129030, - 129178, 129046, 8618, 8696, 8644, 10522, 129193, 129185, 11146, 11157, - 8658, 10499, 8655, 10503, 10524, 10509, 8674, 129195, 129078, 8652, - 10601, 10605, 10591, 10583, 8641, 10600, 10604, 10596, 10587, 10579, - 8640, 129777, 129090, 129094, 129191, 8702, 8649, 129784, 129189, 128622, - 8669, 129082, 129106, 129187, 11106, 11138, 11132, 983242, 11175, 11173, - 129066, 129062, 129058, 129074, 129070, 11122, 11116, 11142, 129170, - 10511, 8667, 10518, 10520, 10519, 11246, 10497, 10496, 10501, 10512, - 8608, 8605, 8680, 8688, 129174, 129086, 11078, 128141, 128735, 11824, - 8790, 8791, 8728, 730, 129680, 128365, 10539, 10544, 65020, 129350, - 983227, 983235, 983225, 983230, 129704, 128640, 128478, 127906, 128764, - 129531, 129315, 128019, 65947, 65942, 65945, 65940, 65943, 8556, 8583, - 8582, 8548, 8558, 8577, 8547, 8544, 8557, 8584, 8559, 8576, 8549, 8581, - 8550, 8553, 8578, 8555, 8545, 8546, 8579, 8554, 8551, 8552, 65938, 65944, - 65936, 65939, 65941, 65937, 65946, 127801, 127989, 10087, 10085, 11213, - 11215, 8506, 128205, 128907, 127588, 127586, 127589, 127584, 127585, - 127587, 128675, 127840, 129302, 8381, 127945, 10740, 69245, 69243, 69244, - 69246, 69241, 69232, 69238, 69229, 69237, 69228, 69242, 69233, 69234, - 69240, 69231, 69239, 69230, 69236, 69227, 69235, 69226, 69225, 69220, - 69219, 69222, 69221, 69218, 69217, 69224, 69216, 69223, 5872, 5869, 5803, - 5802, 5800, 5833, 5837, 5860, 5811, 5859, 5858, 5841, 5851, 5824, 5844, - 5854, 5826, 5846, 5856, 5799, 5814, 5880, 5879, 5877, 5876, 5878, 5792, - 5813, 5815, 5828, 5816, 5819, 5818, 5853, 5852, 5825, 5831, 5864, 5857, - 5827, 5873, 5812, 5810, 5850, 5829, 5820, 5848, 5804, 5862, 5806, 5801, - 5855, 5845, 5807, 5808, 5875, 5809, 5874, 5843, 5821, 5849, 5823, 5805, - 5836, 5840, 5830, 5863, 5835, 5834, 5861, 5842, 5822, 5798, 5839, 5797, - 5817, 5794, 5847, 5832, 5796, 5795, 5865, 5793, 5866, 5838, 5867, 5868, - 5870, 5871, 127933, 127939, 11254, 11252, 11253, 11251, 11255, 11256, - 8360, 983114, 10700, 129466, 129527, 9808, 129474, 9747, 129761, 2049, - 2053, 2051, 2063, 2055, 2052, 2058, 2059, 2062, 2068, 2069, 2056, 2065, - 2048, 2050, 2067, 2054, 2060, 2066, 2061, 2057, 2064, 2073, 2070, 2071, - 2075, 2093, 2072, 2074, 2084, 2088, 2097, 2110, 2098, 2100, 2108, 2099, - 2096, 2101, 2109, 2106, 2104, 2107, 2103, 2105, 2082, 2079, 2076, 2089, - 2086, 2091, 2081, 2078, 2085, 2092, 2083, 2080, 2077, 2090, 2087, 2102, - 128630, 128631, 128632, 128634, 129386, 128752, 128225, 9796, 43138, - 43139, 43150, 43153, 43167, 43166, 43172, 43171, 43165, 43164, 43170, - 43169, 43144, 43145, 43146, 43147, 43182, 43142, 43143, 43151, 43152, - 43140, 43141, 43187, 43181, 43158, 43168, 43163, 43173, 43183, 43184, - 43185, 43177, 43176, 43160, 43159, 43157, 43156, 43162, 43161, 43155, - 43154, 43175, 43174, 43148, 43149, 43186, 43178, 43180, 43179, 43205, - 43136, 43204, 43137, 43215, 43214, 43221, 43220, 43223, 43222, 43219, - 43218, 43216, 43225, 43217, 43224, 43188, 43189, 43200, 43203, 43194, - 43195, 43196, 43197, 43192, 43193, 43201, 43202, 43190, 43191, 43198, - 43199, 129429, 9973, 127927, 127862, 129403, 9878, 129507, 127979, - 127890, 129410, 9807, 128756, 129691, 128437, 8492, 8496, 8497, 8459, - 8464, 8466, 8499, 8472, 8475, 128624, 8495, 8458, 8467, 8500, 8456, - 128220, 983186, 129453, 128186, 167, 8980, 129352, 127793, 128584, 8979, - 130037, 130036, 130039, 130038, 130035, 130034, 130032, 130041, 130033, - 130040, 10802, 9914, 59, 8480, 129324, 9916, 65093, 983169, 8726, 129697, - 11259, 9913, 11250, 129331, 10014, 10061, 10032, 129368, 70086, 70085, - 70101, 70100, 70103, 70102, 70099, 70098, 70096, 70105, 70097, 70104, - 70092, 70106, 70108, 70019, 70020, 70030, 70032, 70046, 70045, 70051, - 70050, 70044, 70043, 70049, 70048, 70025, 70026, 70027, 70028, 70062, - 70023, 70024, 70021, 70022, 70061, 70060, 70037, 70047, 70042, 70052, - 70063, 70064, 70065, 70056, 70055, 70039, 70038, 70036, 70035, 70041, - 70040, 70034, 70033, 70054, 70053, 70066, 70057, 70059, 70058, 70029, - 70031, 70089, 70110, 70111, 70088, 70095, 70107, 70081, 70017, 70016, - 70090, 70082, 70083, 70080, 70018, 70093, 70091, 70094, 70067, 70077, - 70079, 70072, 70073, 70074, 70075, 70070, 70071, 70068, 70069, 70076, - 70078, 70087, 70109, 70084, 129416, 127847, 66684, 66680, 66682, 66665, - 66673, 66679, 66664, 66669, 66647, 66685, 66672, 66683, 66663, 66659, - 66649, 66686, 66674, 66662, 66660, 66656, 66661, 66677, 66678, 66668, - 66676, 66666, 66681, 66640, 66670, 66646, 66645, 66644, 66654, 66641, - 66667, 66658, 66648, 66687, 66657, 66651, 66655, 66643, 66652, 66650, - 66642, 66653, 66671, 66675, 9752, 129768, 128737, 983075, 983078, 9961, - 128674, 127776, 128722, 128717, 11087, 11103, 11086, 10564, 10976, 10985, - 10984, 10974, 10975, 10983, 113825, 113824, 113826, 113827, 127856, - 129651, 9085, 129327, 128703, 128017, 129424, 129335, 10722, 983198, - 983080, 71107, 71106, 71113, 71040, 71131, 71041, 71051, 71053, 71128, + 117826, 10183, 128895, 66736, 66737, 66738, 66739, 66743, 66763, 66761, + 66742, 66749, 66757, 66744, 66768, 66750, 66748, 66754, 66755, 66764, + 66762, 66760, 66746, 66745, 66741, 66765, 66769, 66759, 66758, 66771, + 66770, 66740, 66751, 66752, 66753, 66756, 66767, 66747, 66766, 66776, + 66777, 66778, 66779, 66783, 66803, 66801, 66782, 66789, 66797, 66784, + 66808, 66790, 66788, 66794, 66795, 66804, 66802, 66800, 66786, 66785, + 66781, 66805, 66809, 66799, 66798, 66811, 66810, 66780, 66791, 66792, + 66793, 66796, 66807, 66787, 66806, 66710, 66688, 66715, 66699, 66694, + 66698, 66703, 66693, 66697, 66696, 66705, 66702, 66713, 66717, 66704, + 66706, 66707, 66711, 66716, 66689, 66701, 66700, 66708, 66691, 66695, + 66690, 66692, 66709, 66712, 66714, 66725, 66724, 66727, 66726, 66723, + 66722, 66720, 66729, 66721, 66728, 983192, 126257, 126264, 126258, + 126259, 126265, 126260, 126263, 126267, 126255, 126266, 126256, 126262, + 126261, 126269, 126268, 126254, 126216, 126225, 126252, 126234, 126243, + 126222, 126249, 126213, 126231, 126240, 126221, 126248, 126212, 126230, + 126239, 126217, 126226, 126253, 126235, 126244, 126215, 126224, 126251, + 126233, 126242, 126214, 126223, 126250, 126232, 126241, 126220, 126247, + 126211, 126229, 126238, 126219, 126246, 126210, 126228, 126237, 126218, + 126245, 126209, 126227, 126236, 129446, 128228, 117974, 117975, 117976, + 117977, 117978, 117979, 117980, 117981, 117982, 117983, 117984, 117985, + 117986, 117987, 117988, 117989, 117990, 117991, 117992, 117993, 117994, + 117995, 117996, 117997, 117998, 117999, 10015, 9885, 10029, 10009, + 118005, 118004, 118007, 118006, 118003, 118002, 118000, 118009, 118001, + 118008, 8485, 129397, 128471, 11195, 11194, 11196, 8254, 129450, 8486, + 128076, 127842, 128329, 129417, 128002, 983122, 983121, 128463, 128195, + 128479, 128196, 128223, 128464, 128724, 93059, 93065, 93064, 93058, + 93070, 93056, 93055, 93053, 93062, 93069, 93061, 93066, 93071, 93068, + 93054, 93057, 93063, 93067, 93060, 92967, 92975, 92965, 92969, 92959, + 92968, 92971, 92957, 92962, 92960, 92972, 92970, 92963, 92958, 92966, + 92961, 92956, 92974, 92964, 92973, 92979, 92978, 92980, 92977, 92976, + 92982, 92981, 93023, 93020, 93024, 93021, 93019, 93025, 93022, 93043, + 92985, 93047, 93044, 93045, 92997, 93046, 93042, 93032, 93029, 92995, + 93038, 92993, 93041, 93035, 93033, 93037, 93030, 93039, 92987, 92992, + 92986, 92983, 92984, 93027, 92994, 93034, 92988, 92990, 92989, 92991, + 93028, 92996, 93031, 93040, 93036, 92954, 92955, 92938, 92939, 92932, + 92933, 92942, 92943, 92950, 92951, 92928, 92929, 92936, 92937, 92948, + 92949, 92930, 92931, 92944, 92945, 92934, 92935, 92940, 92941, 92946, + 92947, 92952, 92953, 93013, 93012, 93015, 93014, 93011, 93010, 93008, + 93017, 93009, 93016, 9908, 11801, 127796, 129779, 129780, 67703, 67693, + 67688, 67702, 67683, 67691, 67699, 67700, 67680, 67696, 67682, 67686, + 67695, 67698, 67701, 67689, 67684, 67687, 67690, 67681, 67694, 67685, + 67697, 67692, 67704, 67711, 67706, 67707, 67710, 67709, 67708, 67705, + 129330, 129374, 128060, 8233, 11853, 11791, 10995, 10994, 8741, 129666, + 12809, 12823, 12808, 12822, 12828, 12813, 12827, 12810, 12824, 12800, + 12814, 12804, 12818, 12801, 12815, 12812, 12826, 12805, 12819, 12803, + 12817, 12806, 12820, 12811, 12825, 12802, 12816, 12807, 12821, 12863, + 12855, 12858, 12861, 12847, 12839, 12854, 12843, 12836, 12864, 12835, + 12856, 12846, 12842, 12840, 12852, 12862, 12865, 12857, 12867, 12838, + 12866, 12851, 12853, 12859, 12849, 12860, 12848, 12837, 12834, 12841, + 12833, 12845, 12844, 12850, 12832, 12830, 12829, 9349, 9342, 9345, 9346, + 9350, 9347, 9348, 9344, 9351, 9343, 9341, 9336, 9335, 9338, 9337, 9334, + 9333, 9340, 9332, 9339, 127248, 127249, 127250, 127251, 127252, 127253, + 127254, 127255, 127256, 127257, 127258, 127259, 127260, 127261, 127262, + 127263, 127264, 127265, 127266, 127267, 127268, 127269, 127270, 127271, + 127272, 127273, 9372, 9373, 9374, 9375, 9376, 9377, 9378, 9379, 9380, + 9381, 9382, 9383, 9384, 9385, 9386, 9387, 9388, 9389, 9390, 9391, 9392, + 9393, 9394, 9395, 9396, 9397, 8706, 983147, 983146, 983149, 983150, 9853, + 127881, 12880, 12349, 129436, 128755, 11261, 9105, 9106, 128706, 72437, + 72440, 72432, 72416, 72419, 72413, 72417, 72415, 72412, 72414, 72418, + 72420, 72403, 72407, 72411, 72409, 72410, 72391, 72400, 72404, 72397, + 72394, 72385, 72401, 72384, 72399, 72398, 72396, 72388, 72393, 72392, + 72386, 72387, 72402, 72395, 72390, 72389, 72405, 72406, 72408, 72436, + 72435, 72438, 72439, 72422, 72421, 72424, 72425, 72431, 72433, 72434, + 72428, 72427, 72429, 72430, 72423, 72426, 128062, 128230, 128206, 983228, + 983237, 129434, 9774, 127825, 129372, 129755, 127824, 128039, 128532, + 9956, 128390, 9999, 8240, 8241, 8524, 10178, 10977, 129336, 129496, + 129494, 128113, 9977, 128590, 129733, 128591, 129493, 128589, 128588, + 129495, 128583, 128187, 8966, 128547, 37, 9854, 127917, 8359, 8369, + 129515, 128694, 129730, 43101, 43117, 43120, 43076, 43123, 43077, 43115, + 43090, 43082, 43094, 43098, 43099, 43119, 43118, 43109, 43074, 43075, + 43116, 43079, 43083, 43089, 43088, 43114, 43113, 43081, 43080, 43073, + 43072, 43085, 43084, 43092, 43093, 43104, 43110, 43086, 43108, 43100, + 43078, 43097, 43087, 43106, 43096, 43091, 43107, 43095, 43102, 43105, + 43103, 43127, 43126, 43124, 43121, 43111, 43112, 43122, 43125, 66033, + 66023, 66017, 66010, 66027, 66003, 66018, 66028, 66004, 66012, 66022, + 66020, 66045, 66019, 66031, 66041, 66007, 66006, 66025, 66026, 66038, + 66016, 66013, 66014, 66000, 66001, 66034, 66036, 66037, 66029, 66011, + 66024, 66015, 66021, 66042, 66043, 66002, 66008, 66032, 66005, 66044, + 66040, 66039, 66030, 66009, 66035, 5941, 5942, 67840, 67855, 67843, + 67844, 67847, 67858, 67857, 67860, 67854, 67861, 67848, 67845, 67846, + 67849, 67859, 67842, 67850, 67853, 67851, 67841, 67856, 67852, 67864, + 67866, 67867, 67863, 67862, 67865, 67871, 11227, 9935, 128763, 128022, + 128061, 128055, 128169, 182, 128138, 129292, 129295, 127885, 127821, + 10031, 129655, 129669, 9811, 128299, 8916, 10970, 129383, 8984, 128720, + 129703, 8462, 8463, 127183, 127136, 127167, 127199, 127188, 127140, + 127156, 127172, 127200, 127189, 127141, 127157, 127173, 127196, 127148, + 127164, 127180, 127198, 127150, 127166, 127182, 127192, 127144, 127160, + 127176, 127191, 127143, 127159, 127175, 127190, 127142, 127158, 127174, + 127197, 127149, 127165, 127181, 127194, 127146, 127162, 127178, 127187, + 127139, 127155, 127171, 127186, 127138, 127154, 127170, 127202, 127220, + 127221, 127201, 127210, 127211, 127212, 127213, 127214, 127215, 127216, + 127217, 127218, 127219, 127203, 127204, 127205, 127206, 127207, 127208, + 127209, 127185, 127137, 127153, 127169, 127193, 127145, 127161, 127177, + 127195, 127147, 127163, 127179, 128733, 983151, 43, 10797, 10798, 10809, + 10789, 10786, 10791, 10790, 10788, 10792, 10787, 10866, 177, 9799, 11222, + 11221, 11220, 11219, 129696, 983148, 117777, 128659, 128680, 128110, + 8297, 8236, 127871, 128254, 11239, 128239, 12306, 12320, 128238, 8982, + 128688, 129364, 127858, 129716, 127831, 129751, 128574, 128545, 163, + 128093, 9212, 9213, 9214, 9211, 128041, 128425, 129328, 129732, 129731, + 65043, 65040, 65045, 65073, 65074, 65049, 65041, 65042, 65091, 65047, + 65083, 65085, 65089, 65079, 65087, 65077, 65095, 65081, 65075, 65084, + 65086, 65092, 983261, 65048, 65090, 65080, 65088, 65078, 65096, 65082, + 65076, 65044, 65072, 65046, 8478, 8826, 10937, 10933, 10927, 10929, + 10935, 10931, 8936, 8830, 8828, 8880, 9111, 129384, 9113, 128424, 128438, + 129332, 128120, 983193, 983166, 983163, 983164, 983167, 8242, 8965, 8759, + 8733, 8522, 128711, 129455, 11224, 128255, 68507, 68508, 68480, 68483, + 68490, 68491, 68485, 68482, 68486, 68493, 68495, 68496, 68488, 68484, + 68487, 68489, 68481, 68492, 68497, 68494, 68526, 68522, 68523, 68525, + 68521, 68527, 68524, 68505, 68506, 11854, 8200, 128156, 128091, 128686, + 128204, 128226, 983165, 983168, 983194, 9624, 9625, 9626, 9627, 9628, + 9629, 9630, 9631, 9622, 9623, 10764, 8279, 10774, 9833, 128894, 8264, 63, + 8799, 34, 9915, 127949, 127950, 129437, 128251, 9762, 128280, 9143, + 128740, 128643, 9926, 127752, 11827, 11783, 11782, 9995, 128400, 128406, + 127338, 127339, 127340, 129996, 11787, 9994, 11828, 129306, 128000, 8758, + 128007, 128048, 129682, 128015, 129534, 117778, 9852, 9843, 9844, 9845, + 9846, 9847, 9848, 9849, 9850, 983113, 128665, 127822, 129511, 174, + 127462, 127463, 127464, 127465, 127466, 127467, 127468, 127469, 127470, + 127471, 127472, 127473, 127474, 127475, 127476, 127477, 127478, 127479, + 127480, 127481, 127482, 127483, 127484, 127485, 127486, 127487, 43344, + 43343, 43346, 43345, 43333, 43323, 43331, 43314, 43332, 43317, 43330, + 43320, 43319, 43321, 43316, 43313, 43329, 43322, 43312, 43326, 43318, + 43325, 43324, 43315, 43328, 43327, 43334, 43359, 43337, 43342, 43341, + 43338, 43340, 43335, 43339, 43336, 43347, 128524, 127895, 65533, 9952, + 9953, 128699, 8479, 9166, 11152, 11153, 128639, 128968, 983152, 92, + 10184, 10741, 10743, 11073, 11079, 983153, 10659, 10661, 8246, 12317, + 10989, 8976, 11793, 8267, 8245, 128401, 9753, 11262, 8515, 8271, 8765, + 8909, 8247, 128402, 128403, 128405, 11841, 11822, 128404, 10672, 128158, + 8251, 129423, 983154, 127872, 11190, 11188, 11191, 11189, 11186, 11187, + 11184, 11185, 127832, 127833, 129919, 129918, 128495, 8735, 12297, 10642, + 11777, 11776, 9084, 8894, 10652, 10644, 10228, 8692, 12305, 10648, 125, + 9132, 9133, 9131, 12301, 8969, 11781, 12299, 10608, 10715, 8221, 11817, + 129929, 10621, 8971, 118272, 9687, 11241, 9616, 129978, 117925, 118289, + 129971, 118286, 118284, 118432, 118443, 129933, 128381, 130025, 130017, + 11805, 8906, 10198, 129927, 9621, 129980, 41, 9120, 9118, 9119, 11789, + 10182, 8908, 129931, 8217, 11815, 128360, 128361, 128362, 128489, 93, + 9126, 9124, 10638, 10640, 8262, 10636, 11864, 11862, 9125, 11779, 117773, + 129987, 128493, 118433, 118442, 129930, 129928, 11786, 8895, 10702, + 129902, 12309, 8866, 11809, 9145, 117766, 12303, 10628, 12311, 10630, + 12315, 12313, 10713, 1421, 117833, 117907, 117909, 129308, 4053, 4055, + 117874, 117878, 9957, 9002, 117858, 10749, 187, 117775, 128270, 117921, + 117848, 117923, 117913, 117863, 117763, 117919, 117882, 117761, 10153, + 10552, 8295, 8238, 8235, 8207, 10813, 8594, 11080, 11084, 10562, 10613, + 10612, 129974, 8614, 10528, 129217, 11076, 11075, 10567, 10526, 8677, + 8628, 10513, 8699, 129202, 8620, 129034, 10565, 129042, 129026, 8603, + 11022, 11023, 8611, 10517, 10516, 129030, 129178, 129046, 8618, 8696, + 8644, 10522, 129193, 129185, 11146, 11157, 8658, 10499, 8655, 10503, + 10524, 10509, 8674, 129195, 129078, 8652, 10601, 10605, 10591, 10583, + 8641, 10600, 10604, 10596, 10587, 10579, 8640, 129777, 129090, 129094, + 129191, 8702, 8649, 129784, 129189, 128622, 8669, 129082, 129106, 129187, + 11106, 11138, 11132, 983242, 11175, 11173, 129066, 129062, 129058, + 129074, 129070, 11122, 11116, 11142, 129170, 10511, 8667, 10518, 10520, + 10519, 11246, 10497, 10496, 10501, 10512, 8608, 8605, 8680, 8688, 129174, + 129086, 11078, 128141, 128735, 11824, 8790, 8791, 8728, 730, 129680, + 128365, 10539, 10544, 65020, 129350, 983227, 983235, 983225, 983230, + 129704, 128640, 128478, 127906, 128764, 129531, 129315, 65947, 65942, + 65945, 65940, 65943, 8556, 8583, 8582, 8548, 8558, 8577, 8547, 8544, + 8557, 8584, 8559, 8576, 8549, 8581, 8550, 8553, 8578, 8555, 8545, 8546, + 8579, 8554, 8551, 8552, 65938, 65944, 65936, 65939, 65941, 65937, 65946, + 129756, 128019, 127801, 127989, 10087, 10085, 11213, 11215, 8506, 128205, + 128907, 127588, 127586, 127589, 127584, 127585, 127587, 128675, 127840, + 129302, 8381, 127945, 10740, 69245, 69243, 69244, 69246, 69241, 69232, + 69238, 69229, 69237, 69228, 69242, 69233, 69234, 69240, 69231, 69239, + 69230, 69236, 69227, 69235, 69226, 69225, 69220, 69219, 69222, 69221, + 69218, 69217, 69224, 69216, 69223, 5872, 5869, 5803, 5802, 5800, 5833, + 5837, 5860, 5811, 5859, 5858, 5841, 5851, 5824, 5844, 5854, 5826, 5846, + 5856, 5799, 5814, 5880, 5879, 5877, 5876, 5878, 5792, 5813, 5815, 5828, + 5816, 5819, 5818, 5853, 5852, 5825, 5831, 5864, 5857, 5827, 5873, 5812, + 5810, 5850, 5829, 5820, 5848, 5804, 5862, 5806, 5801, 5855, 5845, 5807, + 5808, 5875, 5809, 5874, 5843, 5821, 5849, 5823, 5805, 5836, 5840, 5830, + 5863, 5835, 5834, 5861, 5842, 5822, 5798, 5839, 5797, 5817, 5794, 5847, + 5832, 5796, 5795, 5865, 5793, 5866, 5838, 5867, 5868, 5870, 5871, 127933, + 127939, 11254, 11252, 11253, 11251, 11255, 11256, 8360, 983114, 10700, + 129466, 129527, 9808, 129474, 9747, 129761, 2049, 2053, 2051, 2063, 2055, + 2052, 2058, 2059, 2062, 2068, 2069, 2056, 2065, 2048, 2050, 2067, 2054, + 2060, 2066, 2061, 2057, 2064, 2073, 2070, 2071, 2075, 2093, 2072, 2074, + 2084, 2088, 2097, 2110, 2098, 2100, 2108, 2099, 2096, 2101, 2109, 2106, + 2104, 2107, 2103, 2105, 2082, 2079, 2076, 2089, 2086, 2091, 2081, 2078, + 2085, 2092, 2083, 2080, 2077, 2090, 2087, 2102, 128630, 128631, 128632, + 128634, 129386, 128752, 128225, 9796, 43138, 43139, 43150, 43153, 43167, + 43166, 43172, 43171, 43165, 43164, 43170, 43169, 43144, 43145, 43146, + 43147, 43182, 43142, 43143, 43151, 43152, 43140, 43141, 43187, 43181, + 43158, 43168, 43163, 43173, 43183, 43184, 43185, 43177, 43176, 43160, + 43159, 43157, 43156, 43162, 43161, 43155, 43154, 43175, 43174, 43148, + 43149, 43186, 43178, 43180, 43179, 43205, 43136, 43204, 43137, 43215, + 43214, 43221, 43220, 43223, 43222, 43219, 43218, 43216, 43225, 43217, + 43224, 43188, 43189, 43200, 43203, 43192, 43193, 43194, 43195, 43196, + 43197, 43201, 43202, 43190, 43191, 43198, 43199, 129429, 9973, 127927, + 127862, 129403, 9878, 129507, 127979, 127890, 129410, 9807, 128756, + 129691, 128437, 8492, 8496, 8497, 8459, 8464, 8466, 8499, 8472, 8475, + 128624, 8495, 8458, 8467, 8500, 8456, 128220, 983186, 129453, 128186, + 167, 8980, 129352, 127793, 128584, 8979, 130037, 130036, 130039, 130038, + 130035, 130034, 130032, 130041, 130033, 130040, 10802, 9914, 59, 117793, + 117795, 117799, 117807, 117803, 117797, 117805, 117801, 117794, 117798, + 117806, 117802, 117796, 117804, 117800, 118353, 118355, 118359, 118367, + 118383, 118415, 118399, 118375, 118407, 118391, 118363, 118379, 118411, + 118395, 118371, 118403, 118387, 118357, 118365, 118381, 118413, 118397, + 118373, 118405, 118389, 118361, 118377, 118409, 118393, 118369, 118401, + 118385, 118354, 118358, 118366, 118382, 118414, 118398, 118374, 118406, + 118390, 118362, 118378, 118410, 118394, 118370, 118402, 118386, 118356, + 118364, 118380, 118412, 118396, 118372, 118404, 118388, 118360, 118376, + 118408, 118392, 118368, 118400, 118384, 11259, 8480, 129324, 9916, 65093, + 983169, 8726, 129697, 9913, 11250, 129331, 10061, 10032, 10014, 129368, + 70086, 70085, 70101, 70100, 70103, 70102, 70099, 70098, 70096, 70105, + 70097, 70104, 70092, 70106, 70108, 70019, 70020, 70030, 70032, 70046, + 70045, 70051, 70050, 70044, 70043, 70049, 70048, 70025, 70026, 70027, + 70028, 70062, 70023, 70024, 70021, 70022, 70061, 70060, 70037, 70047, + 70042, 70052, 70063, 70064, 70065, 70056, 70055, 70039, 70038, 70036, + 70035, 70041, 70040, 70034, 70033, 70054, 70053, 70066, 70057, 70059, + 70058, 70029, 70031, 70089, 70110, 70111, 70088, 70095, 70107, 70016, + 70090, 70081, 70017, 70082, 70083, 70080, 70018, 70093, 70091, 70094, + 70067, 70077, 70079, 70070, 70071, 70072, 70073, 70074, 70075, 70068, + 70069, 70076, 70078, 70087, 70109, 70084, 129416, 127847, 66684, 66680, + 66682, 66665, 66673, 66679, 66664, 66669, 66647, 66685, 66672, 66683, + 66663, 66659, 66649, 66686, 66674, 66662, 66660, 66656, 66661, 66677, + 66678, 66668, 66676, 66666, 66681, 66640, 66670, 66646, 66645, 66644, + 66654, 66641, 66667, 66658, 66648, 66687, 66657, 66651, 66655, 66643, + 66652, 66650, 66642, 66653, 66671, 66675, 9752, 129768, 128737, 983075, + 983078, 9961, 128674, 127776, 128722, 128717, 11087, 11103, 11086, 10564, + 10976, 10985, 10984, 10974, 10975, 10983, 113825, 113824, 113826, 113827, + 127856, 129651, 9085, 129327, 129679, 128703, 128017, 129424, 129335, + 10722, 983198, 983080, 71113, 71040, 71131, 71041, 71051, 71053, 71128, 71070, 71129, 71130, 71065, 71064, 71069, 71067, 71066, 71072, 71071, 71046, 71047, 71048, 71049, 71082, 71044, 71045, 71042, 71043, 71058, 71068, 71063, 71073, 71083, 71084, 71085, 71077, 71076, 71060, 71059, @@ -11674,138 +12462,139 @@ static const unsigned int dawg_pos_to_codepoint[] = { 71127, 71126, 71125, 71123, 71124, 71117, 71118, 71116, 71121, 71115, 71114, 71122, 71109, 71108, 71105, 71100, 71104, 71101, 71103, 71102, 71132, 71133, 71087, 71097, 71099, 71092, 71093, 71090, 71091, 71088, - 71089, 71096, 71098, 128411, 128410, 128417, 128416, 128409, 128408, - 128415, 128414, 121399, 121397, 121400, 121398, 121402, 121401, 121104, - 121103, 121102, 121388, 121387, 121386, 121482, 121479, 121358, 121359, - 121357, 121360, 121341, 121335, 121339, 121340, 121336, 121334, 121333, - 121338, 121337, 121342, 121368, 121367, 121373, 121356, 121355, 121354, - 121380, 121382, 121381, 121384, 121383, 121385, 121377, 121379, 121378, - 121375, 121376, 121374, 121371, 121369, 121366, 121370, 121372, 121364, - 121365, 121452, 121392, 121352, 121353, 121351, 121499, 121500, 121501, - 121502, 121503, 121470, 121117, 121115, 121119, 121118, 121116, 121456, - 121363, 121362, 121361, 121480, 121455, 120965, 120837, 121000, 120969, - 121027, 121026, 121076, 121074, 121075, 120995, 120991, 120990, 120982, - 121005, 121011, 121044, 121042, 121043, 120950, 120833, 120847, 121002, - 120859, 120863, 120997, 120967, 120845, 121019, 120839, 120993, 120980, - 121038, 121045, 121031, 121009, 120934, 121046, 121047, 120905, 120936, - 120937, 120935, 121065, 120941, 120834, 121068, 121069, 121024, 120915, - 120916, 121040, 120940, 120944, 120946, 120942, 120945, 120947, 120943, - 120949, 121020, 121039, 120948, 121091, 120832, 120838, 120861, 120843, - 120852, 120870, 120844, 120846, 120848, 120865, 120853, 120856, 120885, - 120877, 120878, 120879, 120884, 120857, 120882, 120854, 120855, 120858, - 120883, 121001, 120871, 120850, 120867, 120966, 120972, 120971, 120849, - 120862, 120895, 120894, 120872, 120873, 120893, 120875, 120874, 120866, - 120864, 120996, 120842, 120891, 120890, 121067, 120892, 120887, 121064, - 121066, 121063, 121060, 121061, 121062, 120899, 120881, 121052, 121059, - 121057, 121054, 121055, 121056, 121058, 120851, 120869, 120868, 120841, - 121018, 120974, 120977, 120976, 120975, 120985, 120986, 120983, 120992, - 120988, 120984, 120978, 121032, 121037, 120897, 120896, 120898, 120889, - 120888, 120886, 120880, 121035, 121030, 121033, 121028, 121036, 120860, - 121003, 121012, 121014, 121013, 121007, 121016, 121008, 121015, 121006, - 121077, 121083, 120876, 121084, 121085, 121078, 121090, 121087, 121079, - 121080, 121081, 120840, 121086, 121088, 121089, 120979, 121082, 121092, - 120906, 120922, 120908, 120919, 120920, 120921, 120914, 120910, 120912, - 120900, 120903, 120904, 120902, 120901, 120924, 120909, 120911, 120913, - 120926, 120930, 120931, 120932, 120929, 120933, 120928, 120925, 120927, - 120923, 120957, 120917, 120907, 120836, 121051, 120968, 120973, 120998, - 121072, 121053, 121073, 121070, 121071, 121025, 120970, 120994, 120989, - 120987, 121029, 121041, 120955, 120961, 120956, 120959, 120999, 121004, - 120964, 120960, 120963, 120962, 120958, 120939, 120938, 121023, 121021, - 121022, 121050, 121048, 121049, 121034, 121017, 120951, 120981, 120953, - 121010, 120954, 120952, 120918, 120835, 121451, 121343, 121349, 121347, - 121348, 121346, 121345, 121350, 121344, 121462, 121463, 121464, 121465, - 121466, 121467, 121468, 121469, 121428, 121429, 121427, 121476, 121473, - 121477, 121475, 121474, 121478, 121472, 121471, 121404, 121405, 121403, - 121430, 121409, 121411, 121410, 121406, 121408, 121407, 121421, 121423, - 121422, 121415, 121416, 121417, 121418, 121419, 121420, 121414, 121413, - 121412, 121424, 121425, 121426, 121432, 121431, 121183, 121184, 121182, - 121181, 121187, 121188, 121186, 121185, 121175, 121176, 121174, 121173, - 121179, 121180, 121178, 121177, 121324, 121321, 121323, 121320, 121322, - 121319, 121210, 121209, 121208, 121203, 121207, 121305, 121272, 121271, - 121287, 121286, 121303, 121304, 121302, 121301, 121206, 121205, 121204, - 121198, 121332, 121331, 121129, 121276, 121274, 121275, 121273, 121288, - 121289, 121291, 121290, 121306, 121280, 121278, 121279, 121277, 121295, - 121293, 121294, 121292, 121307, 121314, 121191, 121192, 121190, 121189, - 121193, 121202, 121201, 121200, 121199, 121282, 121281, 121297, 121296, - 121308, 121309, 121310, 121328, 121327, 121194, 121196, 121197, 121195, - 121216, 121215, 121214, 121213, 121212, 121211, 121449, 121125, 121126, - 121121, 121122, 121123, 121124, 121127, 121318, 121316, 121317, 121315, - 121156, 121155, 121154, 121146, 121145, 121144, 121150, 121149, 121148, - 121147, 121230, 121231, 121229, 121228, 121261, 121254, 121247, 121233, - 121232, 121226, 121227, 121225, 121224, 121249, 121248, 121153, 121152, - 121151, 121139, 121135, 121137, 121138, 121136, 121330, 121329, 121128, - 121236, 121262, 121255, 121235, 121234, 121237, 121240, 121239, 121263, - 121256, 121238, 121162, 121161, 121160, 121132, 121133, 121131, 121130, - 121134, 121253, 121142, 121143, 121141, 121140, 121243, 121242, 121241, - 121246, 121245, 121244, 121270, 121269, 121268, 121264, 121257, 121326, - 121325, 121159, 121158, 121157, 121448, 121394, 121393, 121396, 121395, - 121450, 121513, 121514, 121515, 121516, 121517, 121518, 121519, 121505, - 121506, 121507, 121508, 121509, 121510, 121511, 121512, 121312, 121284, - 121299, 121311, 121283, 121298, 121313, 121285, 121300, 121267, 121260, - 121252, 121251, 121266, 121259, 121250, 121265, 121258, 121107, 121106, - 121105, 121454, 121453, 121457, 121120, 121112, 121110, 121114, 121113, - 121111, 121108, 121109, 121101, 121100, 121099, 121481, 121441, 121445, - 121446, 121443, 121444, 121442, 121447, 121390, 121389, 121391, 121440, - 121439, 121437, 121435, 121436, 121434, 121433, 121438, 121459, 121458, - 121460, 121095, 121094, 121093, 121219, 121218, 121217, 121222, 121221, - 121220, 121223, 121172, 121171, 121170, 121168, 121167, 121166, 121165, - 121164, 121163, 121169, 121098, 121097, 121096, 121461, 121483, 129304, - 10912, 10911, 10910, 10909, 10860, 983185, 983183, 8219, 8249, 8218, - 8250, 983158, 983155, 983156, 983159, 70117, 70116, 70119, 70118, 70115, - 70114, 70121, 70113, 70120, 70131, 70132, 70129, 70126, 70125, 70130, - 70128, 70127, 70124, 70123, 70122, 983953, 983954, 983952, 3464, 3463, - 3495, 3501, 3497, 3503, 3510, 3488, 3484, 3490, 3482, 3508, 3513, 3462, - 3475, 3478, 3461, 3474, 3473, 3472, 3471, 3470, 3469, 3466, 3465, 3468, - 3467, 3496, 3502, 3498, 3504, 3511, 3489, 3485, 3491, 3483, 3509, 3512, - 3525, 3499, 3522, 3517, 3505, 3523, 3477, 3476, 3500, 3507, 3487, 3494, - 3526, 3524, 3515, 3520, 3514, 3492, 3493, 3521, 3486, 3563, 3562, 3565, - 3564, 3561, 3560, 3558, 3567, 3559, 3566, 3530, 3458, 3457, 3459, 3570, - 3571, 3546, 3537, 3539, 3542, 3544, 3551, 3536, 3538, 3540, 3545, 3548, - 3549, 3550, 3547, 3535, 3572, 8767, 10046, 128973, 10038, 128303, 8198, - 10042, 128510, 127935, 9975, 128128, 9760, 129448, 128761, 10902, 10904, - 10901, 10903, 11098, 11100, 42611, 128716, 128164, 128564, 128554, - 128373, 128759, 127829, 128577, 128578, 10840, 10839, 9011, 127920, - 129445, 65120, 128745, 65121, 128313, 128312, 8717, 8956, 8958, 65131, - 65104, 65109, 65129, 8714, 8948, 8951, 65126, 65111, 65112, 65105, 65115, - 65113, 65117, 65124, 65116, 65114, 65118, 8570, 8567, 8564, 8574, 8572, - 8563, 8560, 8573, 8575, 8571, 8561, 8562, 8569, 8566, 8565, 8568, 65128, - 65108, 68411, 732, 10849, 65125, 65123, 65130, 65122, 65119, 65106, - 65110, 10922, 10924, 10803, 128570, 128571, 128525, 128520, 128519, - 128515, 128517, 128516, 128518, 128522, 129325, 129392, 128526, 129394, - 8995, 128527, 128684, 128012, 128013, 129319, 127956, 9731, 9924, 127938, - 10052, 983077, 9917, 129510, 173, 127846, 128428, 9108, 129358, 69453, - 69452, 69454, 69456, 69447, 69449, 69446, 69448, 69455, 69451, 69450, - 69445, 69424, 69437, 69426, 69433, 69444, 69440, 69429, 69436, 69439, - 69441, 69431, 69427, 69430, 69432, 69425, 69443, 69435, 69442, 69428, - 69438, 69434, 69457, 69460, 69459, 69458, 69463, 69465, 69461, 69462, - 69464, 128618, 128619, 47, 10742, 128284, 69859, 69863, 69864, 69846, - 69847, 69857, 69849, 69842, 69843, 69844, 69845, 69854, 69856, 69855, - 69848, 69851, 69853, 69840, 69841, 69850, 69852, 69858, 69860, 69862, - 69861, 69877, 69876, 69879, 69878, 69875, 69874, 69872, 69881, 69873, - 69880, 8600, 10537, 10541, 8690, 10533, 128603, 128595, 128611, 11112, - 11128, 11018, 8664, 129110, 11010, 8601, 10538, 10534, 128601, 128593, - 128609, 11113, 11129, 11019, 8665, 129111, 11011, 8471, 72328, 72329, - 72327, 72326, 72340, 72339, 72334, 72332, 72341, 72335, 72333, 72330, - 72331, 72338, 72336, 72337, 72344, 72352, 72351, 72350, 72297, 72296, - 72302, 72311, 72301, 72323, 72285, 72284, 72288, 72298, 72293, 72303, - 72319, 72320, 72321, 72310, 72309, 72295, 72294, 72300, 72299, 72307, - 72306, 72290, 72289, 72287, 72286, 72292, 72291, 72305, 72304, 72312, - 72313, 72314, 72322, 72317, 72308, 72316, 72318, 72315, 72272, 72349, - 72348, 72347, 72346, 72324, 72343, 72325, 72342, 72345, 72353, 72354, - 72282, 72281, 72279, 72280, 72277, 72278, 72275, 72274, 72276, 72273, - 72283, 8384, 983043, 983182, 983118, 983177, 127837, 128150, 10055, - 10024, 32, 128586, 128264, 128263, 128265, 128266, 128483, 128676, - 128172, 8375, 8738, 10656, 10657, 128375, 128376, 128467, 128466, 128026, - 128166, 129525, 129348, 128051, 127941, 129533, 13279, 13058, 13057, - 13059, 13056, 13250, 13171, 13278, 13101, 13172, 13108, 13105, 13118, - 13116, 13251, 13192, 8851, 13255, 13183, 13213, 13220, 13216, 13254, - 8852, 13252, 13253, 13170, 13092, 13175, 13177, 13176, 13093, 13094, - 13256, 127376, 13207, 13064, 13179, 13181, 13182, 13055, 13180, 13005, - 13063, 13006, 9974, 9165, 13209, 13070, 13071, 13311, 13075, 13073, + 71089, 71096, 71098, 71107, 71106, 128411, 128410, 128417, 128416, + 128409, 128408, 128415, 128414, 121399, 121397, 121400, 121398, 121402, + 121401, 121104, 121103, 121102, 121388, 121387, 121386, 121482, 121479, + 121358, 121359, 121357, 121360, 121341, 121335, 121339, 121340, 121336, + 121334, 121333, 121338, 121337, 121342, 121368, 121367, 121373, 121356, + 121355, 121354, 121380, 121382, 121381, 121384, 121383, 121385, 121377, + 121379, 121378, 121375, 121376, 121374, 121371, 121369, 121366, 121370, + 121372, 121364, 121365, 121452, 121392, 121352, 121353, 121351, 121499, + 121500, 121501, 121502, 121503, 121470, 121117, 121115, 121119, 121118, + 121116, 121456, 121363, 121362, 121361, 121480, 121455, 120965, 120837, + 121000, 120969, 121027, 121026, 121076, 121074, 121075, 120995, 120991, + 120990, 120982, 121005, 121011, 121044, 121042, 121043, 120950, 120833, + 120847, 121002, 120859, 120863, 120997, 120967, 120845, 121019, 120839, + 120993, 120980, 121038, 121045, 121031, 121009, 120934, 121046, 121047, + 120905, 120936, 120937, 120935, 121065, 120941, 120834, 121068, 121069, + 121024, 120915, 120916, 121040, 120940, 120944, 120946, 120942, 120945, + 120947, 120943, 120949, 121020, 121039, 120948, 121091, 120832, 120838, + 120861, 120843, 120852, 120870, 120844, 120846, 120848, 120865, 120853, + 120856, 120885, 120877, 120878, 120879, 120884, 120857, 120882, 120854, + 120855, 120858, 120883, 121001, 120871, 120850, 120867, 120966, 120972, + 120971, 120849, 120862, 120895, 120894, 120872, 120873, 120893, 120875, + 120874, 120866, 120864, 120996, 120842, 120891, 120890, 121067, 120892, + 120887, 121064, 121066, 121063, 121060, 121061, 121062, 120899, 120881, + 121052, 121059, 121057, 121054, 121055, 121056, 121058, 120851, 120869, + 120868, 120841, 121018, 120974, 120977, 120976, 120975, 120985, 120986, + 120983, 120992, 120988, 120984, 120978, 121032, 121037, 120897, 120896, + 120898, 120889, 120888, 120886, 120880, 121035, 121030, 121033, 121028, + 121036, 120860, 121003, 121012, 121014, 121013, 121007, 121016, 121008, + 121015, 121006, 121077, 121083, 120876, 121084, 121085, 121078, 121090, + 121087, 121079, 121080, 121081, 120840, 121086, 121088, 121089, 120979, + 121082, 121092, 120906, 120922, 120908, 120919, 120920, 120921, 120914, + 120910, 120912, 120900, 120903, 120904, 120902, 120901, 120924, 120909, + 120911, 120913, 120926, 120930, 120931, 120932, 120929, 120933, 120928, + 120925, 120927, 120923, 120957, 120917, 120907, 120836, 121051, 120968, + 120973, 120998, 121072, 121053, 121073, 121070, 121071, 121025, 120970, + 120994, 120989, 120987, 121029, 121041, 120955, 120961, 120956, 120959, + 120999, 121004, 120964, 120960, 120963, 120962, 120958, 120939, 120938, + 121023, 121021, 121022, 121050, 121048, 121049, 121034, 121017, 120951, + 120918, 120981, 120953, 121010, 120954, 120952, 120835, 121451, 121343, + 121349, 121347, 121348, 121346, 121345, 121350, 121344, 121462, 121463, + 121464, 121465, 121466, 121467, 121468, 121469, 121428, 121429, 121427, + 121476, 121473, 121477, 121475, 121474, 121478, 121472, 121471, 121404, + 121405, 121403, 121430, 121409, 121411, 121410, 121406, 121408, 121407, + 121421, 121423, 121422, 121415, 121416, 121417, 121418, 121419, 121420, + 121414, 121413, 121412, 121424, 121425, 121426, 121432, 121431, 121183, + 121184, 121182, 121181, 121187, 121188, 121186, 121185, 121175, 121176, + 121174, 121173, 121179, 121180, 121178, 121177, 121324, 121321, 121323, + 121320, 121322, 121319, 121210, 121209, 121208, 121203, 121207, 121305, + 121272, 121271, 121287, 121286, 121303, 121304, 121302, 121301, 121206, + 121205, 121204, 121198, 121332, 121331, 121129, 121276, 121274, 121275, + 121273, 121288, 121289, 121291, 121290, 121306, 121280, 121278, 121279, + 121277, 121295, 121293, 121294, 121292, 121307, 121314, 121191, 121192, + 121190, 121189, 121193, 121202, 121201, 121200, 121199, 121282, 121281, + 121297, 121296, 121308, 121309, 121310, 121328, 121327, 121194, 121196, + 121197, 121195, 121216, 121215, 121214, 121213, 121212, 121211, 121449, + 121125, 121126, 121121, 121122, 121123, 121124, 121127, 121318, 121316, + 121317, 121315, 121156, 121155, 121154, 121146, 121145, 121144, 121150, + 121149, 121148, 121147, 121230, 121231, 121229, 121228, 121261, 121254, + 121247, 121233, 121232, 121226, 121227, 121225, 121224, 121249, 121248, + 121153, 121152, 121151, 121139, 121135, 121137, 121138, 121136, 121330, + 121329, 121128, 121236, 121262, 121255, 121235, 121234, 121237, 121240, + 121239, 121263, 121256, 121238, 121162, 121161, 121160, 121132, 121133, + 121131, 121130, 121134, 121253, 121142, 121143, 121141, 121140, 121243, + 121242, 121241, 121246, 121245, 121244, 121270, 121269, 121268, 121264, + 121257, 121326, 121325, 121159, 121158, 121157, 121448, 121394, 121393, + 121395, 121396, 121450, 121513, 121514, 121515, 121516, 121517, 121518, + 121519, 121505, 121506, 121507, 121508, 121509, 121510, 121511, 121512, + 121312, 121284, 121299, 121311, 121283, 121298, 121313, 121285, 121300, + 121267, 121260, 121252, 121251, 121266, 121259, 121250, 121265, 121258, + 121107, 121106, 121105, 121454, 121453, 121457, 121120, 121112, 121110, + 121114, 121113, 121111, 121108, 121109, 121101, 121100, 121099, 121481, + 121441, 121445, 121446, 121443, 121444, 121442, 121447, 121390, 121389, + 121391, 121440, 121439, 121437, 121435, 121436, 121434, 121433, 121438, + 121459, 121458, 121460, 121095, 121094, 121093, 121219, 121218, 121217, + 121222, 121221, 121220, 121223, 121172, 121171, 121170, 121168, 121167, + 121166, 121165, 121164, 121163, 121169, 121098, 121097, 121096, 121461, + 121483, 129304, 10912, 10911, 10910, 10909, 10860, 983185, 983183, 8219, + 8249, 8218, 8250, 983158, 983155, 983156, 983159, 70117, 70116, 70119, + 70118, 70115, 70114, 70121, 70113, 70120, 70131, 70132, 70129, 70126, + 70125, 70130, 70128, 70127, 70124, 70123, 70122, 983953, 983954, 983952, + 3464, 3463, 3495, 3501, 3497, 3503, 3510, 3488, 3484, 3490, 3482, 3508, + 3513, 3462, 3475, 3478, 3461, 3474, 3473, 3472, 3471, 3470, 3469, 3466, + 3465, 3468, 3467, 3496, 3502, 3498, 3504, 3511, 3489, 3485, 3491, 3483, + 3509, 3512, 3525, 3499, 3522, 3517, 3505, 3523, 3477, 3476, 3500, 3507, + 3487, 3494, 3526, 3524, 3515, 3520, 3514, 3492, 3493, 3521, 3486, 3563, + 3562, 3565, 3564, 3561, 3560, 3558, 3567, 3559, 3566, 3530, 3458, 3457, + 3459, 3570, 3571, 3546, 3537, 3539, 3542, 3544, 3551, 3536, 3538, 3540, + 3545, 3548, 3549, 3550, 3547, 3535, 3572, 8767, 10046, 128973, 10038, + 128303, 8198, 10042, 128510, 127935, 9975, 128128, 9760, 129448, 128761, + 10902, 10904, 10901, 10903, 11098, 11100, 42611, 128716, 128164, 128564, + 128554, 128373, 128759, 127829, 128578, 128577, 10840, 10839, 9011, + 127920, 129445, 65120, 128745, 65121, 128313, 128312, 8717, 8956, 8958, + 65131, 65104, 65109, 65129, 8714, 8948, 8951, 65126, 65111, 65112, 65105, + 65115, 65113, 65117, 65124, 65116, 65114, 65118, 8570, 8567, 8564, 8574, + 8572, 8563, 8560, 8573, 8575, 8571, 8561, 8562, 8569, 8566, 8565, 8568, + 65128, 65108, 68411, 732, 118266, 10849, 65125, 65123, 65130, 65122, + 65119, 65106, 65110, 10922, 10924, 10803, 128571, 128570, 128525, 128520, + 128519, 128515, 128517, 128516, 128518, 128522, 129325, 129392, 128526, + 129394, 8995, 128527, 128684, 128012, 128013, 129319, 127956, 9731, 9924, + 127938, 10052, 983077, 9917, 129510, 173, 127846, 128428, 9108, 129358, + 69453, 69452, 69454, 69456, 69447, 69449, 69446, 69448, 69455, 69451, + 69450, 69445, 69424, 69437, 69426, 69433, 69444, 69440, 69429, 69436, + 69439, 69441, 69431, 69427, 69430, 69432, 69425, 69443, 69435, 69442, + 69428, 69438, 69434, 69457, 69460, 69459, 69458, 69463, 69465, 69461, + 69462, 69464, 128618, 128619, 47, 10742, 128284, 69859, 69863, 69864, + 69846, 69847, 69857, 69849, 69842, 69843, 69844, 69845, 69854, 69856, + 69855, 69848, 69851, 69853, 69840, 69841, 69850, 69852, 69858, 69860, + 69862, 69861, 69877, 69876, 69879, 69878, 69875, 69874, 69872, 69881, + 69873, 69880, 8600, 10537, 10541, 8690, 10533, 129210, 128603, 128595, + 128611, 11112, 11128, 11018, 8664, 129110, 11010, 8601, 10538, 10534, + 129211, 128601, 128593, 128609, 11113, 11129, 11019, 8665, 129111, 11011, + 8471, 72328, 72329, 72327, 72326, 72340, 72339, 72334, 72332, 72341, + 72335, 72333, 72330, 72331, 72338, 72336, 72337, 72352, 72351, 72350, + 72297, 72296, 72302, 72311, 72301, 72323, 72285, 72284, 72288, 72298, + 72293, 72303, 72319, 72320, 72321, 72310, 72309, 72295, 72294, 72300, + 72299, 72307, 72306, 72290, 72289, 72287, 72286, 72292, 72291, 72305, + 72304, 72312, 72313, 72314, 72322, 72317, 72308, 72316, 72318, 72315, + 72272, 72349, 72348, 72347, 72346, 72324, 72343, 72325, 72342, 72345, + 72353, 72354, 72282, 72281, 72279, 72280, 72277, 72278, 72275, 72274, + 72276, 72273, 72283, 72344, 8384, 983043, 983182, 983118, 983177, 127837, + 128150, 10055, 10024, 117824, 117825, 32, 128586, 128264, 128263, 128265, + 128266, 128483, 128676, 128172, 8375, 117830, 117831, 8738, 10656, 10657, + 128375, 128376, 128467, 128466, 128026, 128166, 129759, 129525, 129348, + 128051, 127941, 129533, 13279, 117900, 13056, 13058, 13057, 13059, 13250, + 13171, 13278, 13101, 13172, 13108, 13105, 13118, 13116, 13251, 13192, + 8851, 13255, 13183, 13213, 13220, 13216, 13254, 8852, 13252, 13253, + 13170, 13092, 13175, 13177, 13176, 13093, 13094, 13256, 127376, 13207, + 13064, 13179, 13181, 13182, 13055, 13180, 13005, 13063, 13006, 117899, + 117898, 117897, 9974, 9165, 13209, 13070, 13071, 13311, 13075, 13073, 13072, 13080, 13081, 13203, 13228, 13191, 13257, 13258, 13098, 13110, 13113, 13121, 13122, 13119, 13107, 13106, 13109, 13259, 13169, 127488, - 13004, 13200, 13260, 13060, 13061, 8847, 8932, 8849, 13178, 13188, 13067, - 13069, 13068, 13074, 13076, 13078, 13079, 13077, 13214, 13262, 13222, + 13004, 13200, 13260, 13060, 13061, 8847, 8932, 8849, 13178, 13188, 13069, + 13068, 13067, 13074, 13076, 13078, 13079, 13077, 13214, 13262, 13222, 13218, 13086, 13085, 13082, 13083, 13084, 13201, 13193, 13248, 13226, 13189, 13199, 13261, 13208, 13263, 13240, 13246, 8977, 13266, 10957, 13264, 13265, 13267, 13223, 13224, 13249, 13221, 13217, 13187, 13123, @@ -11817,28 +12606,29 @@ static const unsigned int dawg_pos_to_codepoint[] = { 13184, 13115, 13112, 13114, 13111, 13103, 13104, 13102, 13117, 13120, 11216, 13273, 13174, 13194, 13271, 13272, 13274, 13232, 13236, 13242, 13229, 13230, 13231, 13142, 13141, 10958, 13137, 13138, 13140, 13139, - 8730, 13087, 13088, 13090, 13091, 13089, 13275, 13276, 128918, 13204, - 13095, 13143, 11027, 9641, 9638, 9636, 11029, 9706, 9703, 11026, 9705, - 9639, 11028, 9640, 9637, 9704, 10720, 13277, 13135, 13134, 13136, 13062, - 127529, 127530, 127512, 127508, 127533, 127545, 127520, 127516, 127534, - 127506, 127540, 127525, 127546, 127532, 127511, 127509, 127524, 127505, - 127517, 127518, 127527, 127537, 127504, 127528, 127535, 127519, 127515, - 127513, 127543, 127542, 127526, 127541, 127544, 127522, 127538, 127514, - 127521, 127539, 127510, 127536, 127523, 127547, 127531, 127378, 127377, - 8865, 10693, 11820, 127390, 127392, 127379, 10692, 127400, 127399, - 127398, 127306, 127489, 127507, 127490, 9919, 127397, 127280, 127281, - 127282, 127283, 127284, 127285, 127286, 127287, 127288, 127289, 127290, - 127291, 127292, 127293, 127294, 127295, 127296, 127297, 127298, 127299, - 127300, 127301, 127302, 127303, 127304, 127305, 10190, 10191, 127401, - 8863, 127307, 127381, 127382, 127396, 127383, 127310, 8862, 127388, - 127393, 127402, 127395, 9949, 10695, 10696, 127384, 127308, 127309, - 127387, 127394, 127389, 8864, 127391, 127385, 127403, 127404, 127386, - 10694, 127311, 127380, 10151, 129425, 983157, 983160, 983134, 983190, - 9877, 9882, 128387, 129485, 8795, 10017, 8902, 9770, 11242, 11243, - 983175, 983133, 983181, 983176, 983042, 983044, 128509, 128649, 127967, - 128642, 127836, 11836, 129658, 129989, 129990, 129993, 129991, 129992, - 128480, 9201, 128207, 9188, 127827, 10025, 983189, 8803, 127897, 129369, - 128723, 983170, 983045, 983103, 8332, 8328, 8330, 8331, 8333, 8334, 8325, + 8730, 13087, 13088, 13090, 13091, 117887, 117886, 117884, 117885, 13089, + 13275, 13276, 128918, 13204, 13095, 13143, 11027, 9641, 9638, 9636, + 11029, 9706, 9703, 11026, 9705, 9639, 11028, 9640, 9637, 9704, 10720, + 13277, 13135, 13134, 13136, 13062, 127529, 127530, 127512, 127508, + 127533, 127545, 127520, 127516, 127534, 127506, 127540, 127525, 127546, + 127532, 127511, 127509, 127524, 127505, 127517, 127518, 127527, 127537, + 127504, 127528, 127535, 127519, 127515, 127513, 127543, 127542, 127526, + 127541, 127544, 127522, 127538, 127514, 127521, 127539, 127510, 127536, + 127523, 127547, 127531, 127378, 127377, 8865, 10693, 11820, 127390, + 127392, 127379, 10692, 127400, 127399, 127398, 127306, 127489, 127507, + 127490, 9919, 127397, 127280, 127281, 127282, 127283, 127284, 127285, + 127286, 127287, 127288, 127289, 127290, 127291, 127292, 127293, 127294, + 127295, 127296, 127297, 127298, 127299, 127300, 127301, 127302, 127303, + 127304, 127305, 10190, 10191, 127401, 8863, 127307, 127381, 127382, + 127396, 127383, 127310, 8862, 127388, 127393, 127402, 127395, 9949, + 10695, 10696, 127384, 127308, 127309, 127387, 127394, 127389, 8864, + 127391, 127385, 127403, 127404, 127386, 10694, 127311, 127380, 10151, + 129425, 983157, 983160, 983134, 983190, 9877, 9882, 128387, 8795, 10017, + 8902, 9770, 11242, 11243, 983175, 983133, 983181, 983176, 983042, 983044, + 128509, 128649, 129485, 127967, 128642, 127836, 11836, 129658, 129989, + 129990, 129993, 129991, 129992, 128480, 9201, 128207, 9188, 127827, + 10025, 983189, 117891, 117888, 117890, 117889, 8803, 127897, 129369, + 128723, 983170, 983045, 983103, 8333, 8332, 8328, 8330, 8334, 8331, 8325, 8324, 8327, 8326, 8323, 8322, 8320, 8329, 8321, 10963, 10965, 10617, 8834, 10953, 10949, 10951, 10955, 8842, 8838, 10947, 10943, 10945, 10941, 983102, 8827, 10938, 10934, 10928, 10930, 10936, 10932, 8937, 8831, 8829, @@ -11849,406 +12639,433 @@ static const unsigned int dawg_pos_to_codepoint[] = { 7056, 7044, 7047, 7045, 7367, 7366, 7365, 7364, 7363, 7361, 7362, 7360, 7082, 7041, 7042, 7040, 7083, 7080, 7081, 7079, 7077, 7076, 7078, 7093, 7092, 7095, 7094, 7091, 7090, 7088, 7097, 7089, 7096, 127749, 127748, - 127751, 127803, 8316, 8312, 8305, 8319, 8317, 8314, 8315, 8318, 8309, - 8308, 8311, 8310, 179, 178, 8304, 8313, 185, 10966, 10964, 10619, 10968, - 10967, 8835, 10954, 10950, 10952, 10956, 8843, 8839, 10948, 10944, 10946, - 10942, 10185, 129464, 129465, 8751, 127940, 128671, 127843, 128629, - 129442, 127946, 8275, 43027, 43026, 43031, 43030, 43040, 43038, 43025, - 43024, 43029, 43028, 43036, 43035, 43021, 43020, 43018, 43017, 43023, - 43022, 43016, 43015, 43034, 43033, 43042, 43039, 43037, 43032, 43041, - 43008, 43012, 43009, 43013, 43011, 43048, 43049, 43050, 43051, 43052, - 43019, 43014, 43010, 43047, 43043, 43046, 43044, 43045, 9223, 9224, 9229, - 9240, 9232, 9236, 9235, 9234, 9233, 9249, 9253, 9241, 9220, 9239, 9219, - 9221, 9243, 9244, 9228, 9225, 9227, 9226, 128325, 9237, 9252, 9216, 9222, - 9246, 9245, 9247, 8527, 9230, 9231, 9217, 9218, 9242, 9254, 9238, 9248, - 11159, 9007, 983094, 983093, 128333, 1807, 1866, 1802, 1798, 1799, 1849, - 1848, 1792, 1854, 1853, 1805, 1804, 1803, 1852, 1851, 1850, 1797, 1814, - 1813, 1815, 1818, 1824, 2153, 2152, 2149, 2148, 2144, 2146, 2150, 2147, - 2154, 2145, 2151, 1825, 1830, 1837, 1839, 1838, 1831, 1834, 1827, 1869, - 1870, 1871, 1809, 1835, 1832, 1828, 1808, 1823, 1833, 1819, 1820, 1836, - 1811, 1812, 1821, 1822, 1810, 1817, 1826, 1816, 1829, 1864, 1863, 1842, - 1841, 1840, 1845, 1844, 1843, 1847, 1846, 1855, 1858, 1796, 983204, 1801, - 1794, 1795, 1800, 1793, 1862, 1861, 1860, 1859, 1865, 1856, 1857, 128137, - 983184, 128085, 129430, 983061, 127955, 917543, 917542, 917546, 917598, - 917568, 917548, 917562, 917540, 917557, 917556, 917559, 917558, 917555, - 917554, 917552, 917561, 917553, 917560, 917565, 917537, 917600, 917566, - 917549, 917569, 917570, 917571, 917572, 917573, 917574, 917575, 917576, - 917577, 917578, 917579, 917580, 917581, 917582, 917583, 917584, 917585, - 917586, 917587, 917588, 917589, 917590, 917591, 917592, 917593, 917594, - 917601, 917602, 917603, 917604, 917605, 917606, 917607, 917608, 917609, - 917610, 917611, 917612, 917613, 917614, 917615, 917616, 917617, 917618, - 917619, 917620, 917621, 917622, 917623, 917624, 917625, 917626, 917564, - 917627, 917544, 917595, 917599, 917541, 917547, 917538, 917567, 917629, - 917545, 917597, 917596, 917563, 917551, 917536, 917539, 917630, 917550, - 917628, 5888, 5919, 5893, 5896, 5898, 5895, 5892, 5905, 5891, 5902, 5899, - 5897, 5901, 5904, 5894, 5903, 5900, 5889, 5890, 5909, 5908, 5906, 5907, - 5989, 5992, 5994, 5991, 5988, 5987, 5998, 5995, 5993, 6000, 5990, 5999, - 5996, 5984, 5985, 5986, 6002, 6003, 6499, 6508, 6509, 6507, 6501, 6502, - 6512, 6513, 6514, 6515, 6516, 6497, 6483, 6487, 6486, 6482, 6498, 6505, - 6504, 6496, 6480, 6490, 6489, 6503, 6506, 6492, 6494, 6488, 6491, 6495, - 6484, 6493, 6481, 6485, 6500, 6745, 6746, 6743, 6747, 6742, 6741, 6748, - 6749, 6750, 6783, 6789, 6788, 6791, 6790, 6787, 6786, 6784, 6793, 6785, - 6792, 6805, 6804, 6807, 6806, 6803, 6802, 6800, 6809, 6801, 6808, 6740, - 6689, 6690, 6688, 6702, 6726, 6727, 6728, 6696, 6695, 6713, 6712, 6707, - 6706, 6714, 6729, 6720, 6693, 6692, 6691, 6704, 6699, 6697, 6717, 6715, - 6709, 6708, 6716, 6732, 6698, 6719, 6723, 6739, 6724, 6730, 6721, 6705, - 6701, 6722, 6735, 6736, 6733, 6734, 6694, 6700, 6710, 6738, 6737, 6711, - 6703, 6718, 6725, 6731, 6828, 6820, 6775, 6776, 6777, 6780, 6824, 6825, - 6819, 6772, 6744, 6823, 6779, 6778, 6822, 6826, 6827, 6818, 6752, 6816, - 6817, 6773, 6774, 6821, 6829, 6753, 6755, 6767, 6769, 6754, 6763, 6764, - 6771, 6768, 6765, 6756, 6770, 6761, 6762, 6760, 6759, 6757, 6758, 6766, - 43653, 43651, 43649, 43661, 43659, 43679, 43677, 43671, 43669, 43657, - 43665, 43673, 43675, 43667, 43681, 43655, 43693, 43689, 43683, 43687, - 43663, 43691, 43685, 43695, 43652, 43650, 43648, 43660, 43658, 43678, - 43676, 43670, 43668, 43656, 43664, 43672, 43674, 43666, 43680, 43654, - 43692, 43688, 43682, 43686, 43662, 43690, 43684, 43694, 43696, 43703, - 43743, 43739, 43740, 43742, 43741, 43712, 43713, 43714, 43711, 43707, - 43697, 43710, 43709, 43708, 43700, 43699, 43705, 43706, 43698, 43704, - 43701, 43702, 71353, 71296, 71352, 71297, 71303, 71305, 71338, 71332, - 71319, 71318, 71324, 71323, 71317, 71316, 71322, 71321, 71300, 71301, - 71298, 71299, 71310, 71320, 71315, 71325, 71329, 71328, 71312, 71311, - 71309, 71308, 71314, 71313, 71307, 71306, 71327, 71326, 71335, 71336, - 71337, 71333, 71330, 71334, 71331, 71302, 71304, 71351, 71339, 71350, - 71340, 71341, 71347, 71349, 71344, 71345, 71342, 71343, 71346, 71348, - 71365, 71364, 71367, 71366, 71363, 71362, 71360, 71369, 71361, 71368, - 129377, 119672, 119671, 3064, 3031, 73707, 983662, 983685, 983674, - 983677, 983676, 983669, 983667, 983679, 983663, 983665, 983668, 983666, - 983683, 983681, 983682, 983673, 983678, 983664, 983684, 983680, 983671, - 983670, 983675, 983672, 73706, 3063, 73701, 3062, 3059, 3051, 3050, 3053, - 3052, 3049, 3048, 3046, 3055, 3047, 3054, 73700, 73666, 73676, 73668, - 73679, 73681, 73682, 73665, 73673, 73674, 73667, 73664, 73669, 73672, - 73675, 73680, 73670, 73678, 73671, 73677, 73683, 73684, 73710, 2985, - 2979, 2969, 2974, 2984, 2975, 2980, 2996, 2995, 2994, 2993, 2992, 2949, - 2950, 2960, 2964, 2953, 2954, 2962, 2963, 2951, 2952, 2998, 2999, 3000, - 2958, 2959, 2970, 3001, 2972, 2965, 2990, 2986, 2997, 2991, 73702, 3057, - 3058, 3056, 3066, 73727, 3065, 73703, 73687, 2946, 73686, 73698, 73690, - 73693, 73692, 73712, 73689, 73688, 73691, 73697, 73694, 73695, 73696, - 73713, 73699, 3021, 2947, 73685, 73708, 73711, 983939, 983940, 983947, - 983950, 983943, 983944, 983948, 983949, 983941, 983942, 983945, 983946, - 983686, 983693, 983696, 983689, 983690, 983694, 983695, 983687, 983688, - 983691, 983692, 983840, 983847, 983850, 983843, 983844, 983848, 983849, - 983841, 983842, 983845, 983846, 983851, 983858, 983861, 983854, 983855, - 983859, 983860, 983852, 983853, 983856, 983857, 983818, 983825, 983828, - 983821, 983822, 983826, 983827, 983819, 983820, 983823, 983824, 983873, - 983880, 983883, 983876, 983877, 983881, 983882, 983874, 983875, 983878, - 983879, 983741, 983748, 983751, 983744, 983745, 983749, 983750, 983742, - 983743, 983746, 983747, 983697, 983704, 983707, 983700, 983701, 983705, - 983706, 983698, 983699, 983702, 983703, 983719, 983726, 983729, 983722, - 983723, 983727, 983728, 983720, 983721, 983724, 983725, 983763, 983770, - 983773, 983766, 983767, 983771, 983772, 983764, 983765, 983768, 983769, - 983862, 983869, 983872, 983865, 983866, 983870, 983871, 983863, 983864, - 983867, 983868, 983807, 983814, 983817, 983810, 983811, 983815, 983816, - 983808, 983809, 983812, 983813, 983895, 983902, 983905, 983898, 983899, - 983903, 983904, 983951, 983896, 983897, 983900, 983901, 983906, 983913, - 983916, 983909, 983910, 983914, 983915, 983907, 983908, 983911, 983912, - 983917, 983924, 983927, 983920, 983921, 983925, 983926, 983918, 983919, - 983922, 983923, 983730, 983737, 983740, 983733, 983734, 983738, 983739, - 983731, 983732, 983735, 983736, 983752, 983759, 983762, 983755, 983756, - 983760, 983761, 983753, 983754, 983757, 983758, 983708, 983715, 983718, - 983711, 983712, 983716, 983717, 983709, 983710, 983713, 983714, 983928, - 983935, 983938, 983931, 983932, 983936, 983937, 983929, 983930, 983933, - 983934, 983884, 983891, 983894, 983887, 983888, 983892, 983893, 983885, - 983886, 983889, 983890, 983785, 983792, 983795, 983788, 983789, 983793, - 983794, 983786, 983787, 983790, 983791, 983774, 983781, 983784, 983777, - 983778, 983782, 983783, 983775, 983776, 983779, 983780, 983829, 983836, - 983839, 983832, 983833, 983837, 983838, 983830, 983831, 983834, 983835, - 983796, 983803, 983806, 983799, 983800, 983804, 983805, 983797, 983798, - 983801, 983802, 3006, 3016, 3020, 3009, 3010, 3018, 3019, 3007, 3008, - 3014, 3015, 73709, 73704, 73705, 3061, 3060, 3024, 129748, 127883, - 127818, 92809, 92810, 92811, 92808, 92789, 92790, 92791, 92788, 92816, - 92859, 92856, 92847, 92845, 92817, 92846, 92843, 92829, 92830, 92831, - 92828, 92835, 92851, 92840, 92844, 92818, 92819, 92852, 92836, 92825, - 92826, 92827, 92824, 92813, 92814, 92815, 92812, 92820, 92822, 92823, - 92821, 92805, 92806, 92807, 92804, 92797, 92798, 92799, 92796, 92801, - 92802, 92803, 92800, 92785, 92786, 92787, 92784, 92793, 92794, 92795, - 92792, 92857, 92854, 92848, 92861, 92853, 92860, 92849, 92855, 92834, - 92833, 92832, 92841, 92839, 92842, 92850, 92838, 92858, 92837, 92862, - 92869, 92868, 92871, 92870, 92867, 92866, 92864, 92873, 92865, 92872, - 100352, 100353, 100354, 100355, 100356, 100357, 100358, 100359, 100360, - 100361, 100362, 100363, 100364, 100365, 100366, 100367, 100368, 100369, - 100370, 100371, 100372, 100373, 100374, 100375, 100376, 100377, 100378, - 100379, 100380, 100381, 100382, 100383, 100384, 100385, 100386, 100387, - 100388, 100389, 100390, 100391, 100392, 100393, 100394, 100395, 100396, - 100397, 100398, 100399, 100400, 100401, 100402, 100403, 100404, 100405, - 100406, 100407, 100408, 100409, 100410, 100411, 100412, 100413, 100414, - 100415, 100416, 100417, 100418, 100419, 100420, 100421, 100422, 100423, - 100424, 100425, 100426, 100427, 100428, 100429, 100430, 100431, 100432, - 100433, 100434, 100435, 100436, 100437, 100438, 100439, 100440, 100441, - 100442, 100443, 100444, 100445, 100446, 100447, 100448, 100449, 100450, - 100451, 100452, 100453, 100454, 100455, 100456, 100457, 100458, 100459, - 100460, 100461, 100462, 100463, 100464, 100465, 100466, 100467, 100468, - 100469, 100470, 100471, 100472, 100473, 100474, 100475, 100476, 100477, - 100478, 100479, 100480, 100481, 100482, 100483, 100484, 100485, 100486, - 100487, 100488, 100489, 100490, 100491, 100492, 100493, 100494, 100495, - 100496, 100497, 100498, 100499, 100500, 100501, 100502, 100503, 100504, - 100505, 100506, 100507, 100508, 100509, 100510, 100511, 100512, 100513, - 100514, 100515, 100516, 100517, 100518, 100519, 100520, 100521, 100522, - 100523, 100524, 100525, 100526, 100527, 100528, 100529, 100530, 100531, - 100532, 100533, 100534, 100535, 100536, 100537, 100538, 100539, 100540, - 100541, 100542, 100543, 100544, 100545, 100546, 100547, 100548, 100549, - 100550, 100551, 100552, 100553, 100554, 100555, 100556, 100557, 100558, - 100559, 100560, 100561, 100562, 100563, 100564, 100565, 100566, 100567, - 100568, 100569, 100570, 100571, 100572, 100573, 100574, 100575, 100576, - 100577, 100578, 100579, 100580, 100581, 100582, 100583, 100584, 100585, - 100586, 100587, 100588, 100589, 100590, 100591, 100592, 100593, 100594, - 100595, 100596, 100597, 100598, 100599, 100600, 100601, 100602, 100603, - 100604, 100605, 100606, 100607, 100608, 100609, 100610, 100611, 100612, - 100613, 100614, 100615, 100616, 100617, 100618, 100619, 100620, 100621, - 100622, 100623, 100624, 100625, 100626, 100627, 100628, 100629, 100630, - 100631, 100632, 100633, 100634, 100635, 100636, 100637, 100638, 100639, - 100640, 100641, 100642, 100643, 100644, 100645, 100646, 100647, 100648, - 100649, 100650, 100651, 100652, 100653, 100654, 100655, 100656, 100657, - 100658, 100659, 100660, 100661, 100662, 100663, 100664, 100665, 100666, - 100667, 100668, 100669, 100670, 100671, 100672, 100673, 100674, 100675, - 100676, 100677, 100678, 100679, 100680, 100681, 100682, 100683, 100684, - 100685, 100686, 100687, 100688, 100689, 100690, 100691, 100692, 100693, - 100694, 100695, 100696, 100697, 100698, 100699, 100700, 100701, 100702, - 100703, 100704, 100705, 100706, 100707, 100708, 100709, 100710, 100711, - 100712, 100713, 100714, 100715, 100716, 100717, 100718, 100719, 100720, - 100721, 100722, 100723, 100724, 100725, 100726, 100727, 100728, 100729, - 100730, 100731, 100732, 100733, 100734, 100735, 100736, 100737, 100738, - 100739, 100740, 100741, 100742, 100743, 100744, 100745, 100746, 100747, - 100748, 100749, 100750, 100751, 100752, 100753, 100754, 100755, 100756, - 100757, 100758, 100759, 100760, 100761, 100762, 100763, 100764, 100765, - 100766, 100767, 100768, 100769, 100770, 100771, 100772, 100773, 100774, - 100775, 100776, 100777, 100778, 100779, 100780, 100781, 100782, 100783, - 100784, 100785, 100786, 100787, 100788, 100789, 100790, 100791, 100792, - 100793, 100794, 100795, 100796, 100797, 100798, 100799, 100800, 100801, - 100802, 100803, 100804, 100805, 100806, 100807, 100808, 100809, 100810, - 100811, 100812, 100813, 100814, 100815, 100816, 100817, 100818, 100819, - 100820, 100821, 100822, 100823, 100824, 100825, 100826, 100827, 100828, - 100829, 100830, 100831, 100832, 100833, 100834, 100835, 100836, 100837, - 100838, 100839, 100840, 100841, 100842, 100843, 100844, 100845, 100846, - 100847, 100848, 100849, 100850, 100851, 100852, 100853, 100854, 100855, - 100856, 100857, 100858, 100859, 100860, 100861, 100862, 100863, 100864, - 100865, 100866, 100867, 100868, 100869, 100870, 100871, 100872, 100873, - 100874, 100875, 100876, 100877, 100878, 100879, 100880, 100881, 100882, - 100883, 100884, 100885, 100886, 100887, 100888, 100889, 100890, 100891, - 100892, 100893, 100894, 100895, 100896, 100897, 100898, 100899, 100900, - 100901, 100902, 100903, 100904, 100905, 100906, 100907, 100908, 100909, - 100910, 100911, 100912, 100913, 100914, 100915, 100916, 100917, 100918, - 100919, 100920, 100921, 100922, 100923, 100924, 100925, 100926, 100927, - 100928, 100929, 100930, 100931, 100932, 100933, 100934, 100935, 100936, - 100937, 100938, 100939, 100940, 100941, 100942, 100943, 100944, 100945, - 100946, 100947, 100948, 100949, 100950, 100951, 100952, 100953, 100954, - 100955, 100956, 100957, 100958, 100959, 100960, 100961, 100962, 100963, - 100964, 100965, 100966, 100967, 100968, 100969, 100970, 100971, 100972, - 100973, 100974, 100975, 100976, 100977, 100978, 100979, 100980, 100981, - 100982, 100983, 100984, 100985, 100986, 100987, 100988, 100989, 100990, - 100991, 100992, 100993, 100994, 100995, 100996, 100997, 100998, 100999, - 101000, 101001, 101002, 101003, 101004, 101005, 101006, 101007, 101008, - 101009, 101010, 101011, 101012, 101013, 101014, 101015, 101016, 101017, - 101018, 101019, 101020, 101021, 101022, 101023, 101024, 101025, 101026, - 101027, 101028, 101029, 101030, 101031, 101032, 101033, 101034, 101035, - 101036, 101037, 101038, 101039, 101040, 101041, 101042, 101043, 101044, - 101045, 101046, 101047, 101048, 101049, 101050, 101051, 101052, 101053, - 101054, 101055, 101056, 101057, 101058, 101059, 101060, 101061, 101062, - 101063, 101064, 101065, 101066, 101067, 101068, 101069, 101070, 101071, - 101072, 101073, 101074, 101075, 101076, 101077, 101078, 101079, 101080, - 101081, 101082, 101083, 101084, 101085, 101086, 101087, 101088, 101089, - 101090, 101091, 101092, 101093, 101094, 101095, 101096, 101097, 101098, - 101099, 101100, 101101, 101102, 101103, 101104, 101105, 101106, 101107, - 101108, 101109, 101110, 101111, 101112, 101113, 101114, 101115, 101116, - 101117, 101118, 101119, 94176, 128429, 9991, 9801, 127790, 128661, - 127861, 128198, 10043, 10170, 129750, 129528, 128222, 128380, 8981, 9990, - 8481, 128384, 128301, 128250, 3158, 3195, 3198, 3194, 3197, 3193, 3196, - 3192, 3106, 3105, 3111, 3161, 3110, 3124, 3123, 3122, 3112, 3165, 3097, - 3107, 3102, 3162, 3121, 3120, 3104, 3103, 3109, 3160, 3108, 3077, 3078, - 3088, 3092, 3083, 3168, 3084, 3169, 3125, 3081, 3082, 3090, 3091, 3079, - 3080, 3126, 3127, 3128, 3117, 3116, 3099, 3098, 3096, 3095, 3101, 3100, - 3094, 3093, 3115, 3114, 3086, 3087, 3129, 3118, 3119, 3157, 3076, 3072, - 3073, 3191, 3199, 3133, 3074, 3132, 3149, 3075, 3134, 3144, 3148, 3139, - 3140, 3170, 3171, 3137, 3138, 3146, 3147, 3135, 3136, 3142, 3143, 3179, - 3178, 3181, 3180, 3177, 3176, 3174, 3183, 3175, 3182, 127934, 8376, 9978, - 129514, 119617, 119564, 119577, 119633, 119587, 119566, 119561, 119585, - 119613, 119590, 119631, 119634, 119630, 119608, 119582, 119563, 119573, - 119558, 119624, 119567, 119623, 119586, 119636, 119612, 119625, 119568, - 119584, 119619, 119618, 119583, 119603, 119600, 119626, 119610, 119580, - 119576, 119638, 119559, 119595, 119632, 119606, 119592, 119615, 119599, - 119602, 119614, 119629, 119574, 119569, 119570, 119622, 119562, 119591, - 119637, 119597, 119589, 119616, 119609, 119560, 119635, 119565, 119604, - 119588, 119571, 119594, 119578, 119596, 119579, 119598, 119572, 119605, - 119627, 119621, 119628, 119601, 119593, 119607, 119575, 119620, 119611, - 119581, 1959, 1958, 1964, 1961, 1965, 1927, 1954, 1951, 1937, 1931, 1930, - 1956, 1934, 1935, 1955, 1945, 1920, 1926, 1946, 1933, 1925, 1929, 1943, - 1942, 1941, 1922, 1969, 1950, 1949, 1921, 1936, 1939, 1932, 1947, 1944, - 1952, 1928, 1957, 1938, 1948, 1953, 1924, 1923, 1940, 1967, 1966, 1963, - 1960, 1962, 1968, 3610, 3592, 3594, 3593, 3596, 3598, 3604, 3613, 3615, - 3663, 3630, 3627, 3588, 3587, 3589, 3586, 3590, 3675, 3585, 3628, 3621, - 3653, 3622, 3659, 3633, 3657, 3658, 3656, 3655, 3654, 3617, 3674, 3591, - 3661, 3603, 3609, 3631, 3642, 3612, 3614, 3616, 3611, 3619, 3620, 3632, - 3652, 3651, 3634, 3649, 3635, 3640, 3638, 3639, 3641, 3636, 3637, 3648, - 3650, 3625, 3624, 3626, 3595, 3660, 3601, 3602, 3607, 3600, 3608, 3606, - 3605, 3599, 3623, 3662, 3597, 3618, 3629, 3647, 3669, 3668, 3671, 3670, - 3667, 3666, 3664, 3673, 3665, 3672, 8708, 8707, 8756, 127777, 10727, - 128936, 8201, 128929, 129300, 129353, 128173, 129652, 10176, 8278, 9887, - 9886, 11057, 128485, 128484, 128486, 128487, 8694, 128433, 10870, 128491, - 128962, 128423, 11160, 11162, 10146, 11163, 11161, 10147, 8196, 11835, - 128077, 128078, 9928, 9736, 3865, 3863, 3864, 4032, 4033, 4035, 4034, - 3886, 3885, 3888, 3887, 3884, 3883, 3891, 3890, 3882, 3889, 3877, 3876, - 3879, 3878, 3875, 3874, 3872, 3881, 3873, 3880, 4030, 4031, 3945, 3905, - 3947, 3904, 3948, 3938, 3946, 3917, 3916, 3932, 3931, 3922, 3921, 3908, - 3918, 3913, 3923, 3940, 3941, 3942, 3930, 3929, 3915, 3914, 3920, 3919, - 3927, 3926, 3910, 3909, 3907, 3906, 3925, 3924, 3934, 3935, 3936, 3943, - 3911, 3939, 3928, 3933, 3937, 3944, 3862, 3861, 983210, 3850, 4048, 3849, - 3892, 4049, 3894, 3859, 3846, 3896, 4052, 3845, 3847, 3842, 3843, 3841, - 3860, 3898, 3899, 3900, 3901, 3972, 4051, 3844, 3851, 3895, 3893, 4050, - 3856, 3854, 3973, 3858, 3857, 3848, 3853, 4058, 3897, 3855, 4057, 3852, - 3978, 3979, 3974, 3976, 3903, 3977, 3970, 3866, 3867, 3868, 3871, 3869, - 3870, 4047, 4046, 3966, 3967, 3975, 3902, 3980, 3971, 4028, 4026, 4027, - 3997, 3996, 4012, 4011, 4002, 4001, 4025, 3985, 3984, 3988, 3998, 3993, - 4003, 4020, 4021, 4022, 4010, 4009, 3995, 3994, 4000, 3999, 4007, 4006, - 3990, 3989, 3987, 3986, 4005, 4004, 4014, 4015, 4016, 4023, 3991, 4019, - 4008, 4018, 4013, 4017, 4024, 3983, 3982, 3981, 3840, 4036, 4041, 4044, - 4043, 4042, 4038, 4040, 4037, 4039, 3968, 3969, 3958, 3959, 3960, 3961, - 3956, 3957, 3964, 3965, 3954, 3955, 3962, 3963, 3953, 10717, 11647, - 11608, 11595, 11585, 11573, 11620, 11607, 11600, 11582, 11590, 11596, - 11601, 11586, 11592, 11568, 11575, 11577, 11578, 11576, 11571, 11606, - 11572, 11581, 11589, 11583, 11569, 11570, 11584, 11587, 11609, 11611, - 11610, 11612, 11613, 11615, 11619, 11594, 11621, 11604, 11605, 11614, - 11588, 11580, 11574, 11597, 11598, 11599, 11602, 11591, 11616, 11617, - 11618, 11622, 11579, 11593, 11623, 11603, 11631, 11632, 128005, 128047, - 10053, 126, 8764, 11081, 10610, 10859, 10858, 11807, 11806, 11803, 9202, - 10708, 10709, 10750, 68410, 70854, 70784, 70785, 70786, 70796, 70798, - 70812, 70811, 70817, 70816, 70810, 70809, 70815, 70814, 70791, 70792, - 70793, 70794, 70827, 70789, 70790, 70787, 70788, 70803, 70813, 70808, - 70818, 70828, 70829, 70830, 70822, 70821, 70805, 70804, 70802, 70801, - 70807, 70806, 70800, 70799, 70820, 70819, 70831, 70826, 70823, 70825, - 70824, 70795, 70797, 70852, 70848, 70847, 70851, 70850, 70849, 70832, - 70843, 70846, 70842, 70845, 70837, 70838, 70839, 70840, 70835, 70836, - 70833, 70834, 70841, 70844, 70869, 70868, 70871, 70870, 70867, 70866, - 70864, 70873, 70865, 70872, 70853, 70855, 11858, 8266, 128555, 127915, - 127813, 128069, 129463, 129701, 129520, 10554, 10557, 10556, 9182, 11210, - 11865, 11866, 11833, 8992, 127913, 9180, 8988, 8975, 11810, 8989, 8974, - 11811, 9140, 9184, 128285, 127553, 127554, 127559, 127555, 127557, - 127560, 127552, 127556, 127558, 127274, 9008, 123554, 123556, 123559, - 123561, 123564, 123537, 123553, 123555, 123544, 123543, 123558, 123560, - 123546, 123565, 123563, 123539, 123541, 123550, 123549, 123540, 123552, - 123542, 123536, 123551, 123545, 123538, 123548, 123547, 123562, 123557, - 123566, 128701, 128508, 128434, 128668, 8482, 128650, 128651, 11223, - 10971, 128646, 10701, 10699, 128710, 10698, 10141, 128681, 128208, 8227, - 128305, 9783, 9776, 9777, 9782, 9779, 9781, 9780, 9778, 10998, 10856, - 10857, 10747, 8244, 8779, 10996, 10624, 8874, 10997, 11003, 11851, 11000, - 10999, 8749, 8285, 129484, 128654, 127865, 128032, 127942, 8872, 11231, - 127930, 8366, 8378, 129411, 8523, 10658, 11202, 9930, 9929, 8498, 11826, + 72648, 72662, 72661, 72669, 72652, 72640, 72663, 72651, 72655, 72672, + 72646, 72667, 72666, 72653, 72657, 72645, 72665, 72649, 72644, 72658, + 72668, 72670, 72664, 72671, 72641, 72659, 72650, 72656, 72643, 72660, + 72654, 72642, 72647, 72673, 72693, 72692, 72695, 72694, 72691, 72690, + 72688, 72697, 72689, 72696, 127751, 127803, 8316, 8312, 8305, 8319, 8317, + 8314, 8318, 8315, 8309, 8308, 8311, 8310, 179, 178, 8304, 8313, 185, + 10966, 10964, 10619, 10968, 10967, 8835, 10954, 10950, 10952, 10956, + 8843, 8839, 10948, 10944, 10946, 10942, 10185, 129464, 129465, 8751, + 127940, 128671, 127843, 128629, 129442, 127946, 8275, 43027, 43026, + 43031, 43030, 43040, 43038, 43025, 43024, 43029, 43028, 43036, 43035, + 43021, 43020, 43018, 43017, 43023, 43022, 43016, 43015, 43034, 43033, + 43042, 43039, 43037, 43032, 43041, 43008, 43012, 43009, 43013, 43011, + 43048, 43049, 43050, 43051, 43052, 43019, 43014, 43010, 43047, 43043, + 43046, 43044, 43045, 9223, 9224, 9229, 9240, 9232, 9249, 9256, 9255, + 9253, 9257, 9236, 9235, 9234, 9233, 9241, 9220, 9239, 9219, 9221, 9243, + 9244, 9228, 9225, 9227, 9226, 128325, 9237, 9252, 9216, 9222, 9246, 9245, + 9247, 8527, 9230, 9231, 9217, 9218, 9242, 9254, 9238, 9248, 11159, 9007, + 983094, 983093, 128333, 1807, 1866, 1802, 1798, 1799, 1849, 1848, 1792, + 1854, 1853, 1805, 1804, 1803, 1852, 1851, 1850, 1797, 1814, 1813, 1815, + 1818, 1824, 2153, 2152, 2149, 2148, 2144, 2146, 2150, 2147, 2154, 2145, + 2151, 1825, 1830, 1837, 1839, 1838, 1831, 1834, 1827, 1869, 1870, 1871, + 1809, 1835, 1832, 1828, 1808, 1823, 1833, 1819, 1820, 1836, 1811, 1812, + 1821, 1822, 1810, 1817, 1826, 1816, 1829, 1864, 1863, 1842, 1841, 1840, + 1845, 1844, 1843, 1847, 1846, 1855, 1858, 1796, 983204, 1801, 1794, 1795, + 1800, 1793, 1862, 1861, 1860, 1859, 1865, 1856, 1857, 128137, 983184, + 128085, 129430, 983061, 127955, 917543, 917542, 917546, 917598, 917568, + 917548, 917562, 917540, 917557, 917556, 917559, 917558, 917555, 917554, + 917552, 917561, 917553, 917560, 917565, 917537, 917600, 917566, 917549, + 917569, 917570, 917571, 917572, 917573, 917574, 917575, 917576, 917577, + 917578, 917579, 917580, 917581, 917582, 917583, 917584, 917585, 917586, + 917587, 917588, 917589, 917590, 917591, 917592, 917593, 917594, 917601, + 917602, 917603, 917604, 917605, 917606, 917607, 917608, 917609, 917610, + 917611, 917612, 917613, 917614, 917615, 917616, 917617, 917618, 917619, + 917620, 917621, 917622, 917623, 917624, 917625, 917626, 917564, 917627, + 917544, 917595, 917599, 917541, 917547, 917538, 917567, 917629, 917545, + 917597, 917596, 917563, 917551, 917536, 917539, 917630, 917550, 917628, + 5888, 5919, 5893, 5896, 5898, 5895, 5892, 5905, 5891, 5902, 5899, 5897, + 5901, 5904, 5894, 5903, 5900, 5889, 5890, 5909, 5908, 5906, 5907, 5989, + 5992, 5994, 5991, 5988, 5987, 5998, 5995, 5993, 6000, 5990, 5999, 5996, + 5984, 5985, 5986, 6002, 6003, 6499, 6508, 6509, 6507, 6501, 6502, 6512, + 6513, 6514, 6515, 6516, 6497, 6483, 6487, 6486, 6482, 6498, 6505, 6504, + 6496, 6480, 6490, 6489, 6503, 6506, 6492, 6494, 6488, 6491, 6495, 6484, + 6493, 6481, 6485, 6500, 6745, 6746, 6743, 6747, 6742, 6741, 6748, 6749, + 6750, 6783, 6789, 6788, 6791, 6790, 6787, 6786, 6784, 6793, 6785, 6792, + 6805, 6804, 6807, 6806, 6803, 6802, 6800, 6809, 6801, 6808, 6740, 6689, + 6690, 6688, 6702, 6726, 6727, 6728, 6696, 6695, 6713, 6712, 6707, 6706, + 6714, 6729, 6720, 6693, 6692, 6691, 6704, 6699, 6697, 6717, 6715, 6709, + 6708, 6716, 6732, 6698, 6719, 6723, 6739, 6724, 6730, 6721, 6705, 6701, + 6722, 6735, 6736, 6733, 6734, 6694, 6700, 6710, 6738, 6737, 6711, 6703, + 6718, 6725, 6731, 6828, 6820, 6775, 6776, 6777, 6780, 6824, 6825, 6819, + 6772, 6744, 6823, 6779, 6778, 6822, 6826, 6827, 6818, 6752, 6816, 6817, + 6821, 6773, 6774, 6829, 6753, 6755, 6767, 6769, 6754, 6763, 6764, 6771, + 6768, 6765, 6756, 6770, 6761, 6762, 6760, 6759, 6757, 6758, 6766, 43653, + 43651, 43649, 43661, 43659, 43679, 43677, 43671, 43669, 43657, 43665, + 43673, 43675, 43667, 43681, 43655, 43693, 43689, 43683, 43687, 43663, + 43691, 43685, 43695, 43652, 43650, 43648, 43660, 43658, 43678, 43676, + 43670, 43668, 43656, 43664, 43672, 43674, 43666, 43680, 43654, 43692, + 43688, 43682, 43686, 43662, 43690, 43684, 43694, 43696, 43703, 43743, + 43739, 43740, 43742, 43741, 43712, 43713, 43714, 43711, 43707, 43697, + 43710, 43709, 43708, 43700, 43699, 43705, 43706, 43698, 43704, 43701, + 43702, 71353, 71296, 71352, 71297, 71303, 71305, 71319, 71318, 71324, + 71323, 71338, 71332, 71317, 71316, 71322, 71321, 71300, 71301, 71298, + 71299, 71310, 71320, 71315, 71325, 71329, 71328, 71312, 71311, 71309, + 71308, 71314, 71313, 71307, 71306, 71327, 71326, 71335, 71336, 71337, + 71333, 71330, 71334, 71331, 71302, 71304, 71351, 71339, 71350, 71340, + 71341, 71347, 71349, 71344, 71345, 71342, 71343, 71346, 71348, 71365, + 71364, 71367, 71366, 71363, 71362, 71360, 71369, 71361, 71368, 129377, + 119672, 119671, 3064, 3031, 73707, 983662, 983685, 983674, 983677, + 983676, 983669, 983667, 983679, 983663, 983665, 983668, 983666, 983683, + 983681, 983682, 983673, 983678, 983664, 983684, 983680, 983671, 983670, + 983675, 983672, 73706, 3063, 73701, 3062, 3059, 3051, 3050, 3053, 3052, + 3049, 3048, 3046, 3055, 3047, 3054, 73700, 73666, 73676, 73668, 73679, + 73681, 73682, 73665, 73673, 73674, 73667, 73664, 73669, 73672, 73675, + 73680, 73670, 73678, 73671, 73677, 73683, 73684, 73710, 2985, 2979, 2969, + 2974, 2984, 2975, 2980, 2949, 2950, 2960, 2964, 2996, 2995, 2994, 2993, + 2992, 2953, 2954, 2962, 2963, 2951, 2952, 2998, 2999, 3000, 2958, 2959, + 2970, 3001, 2972, 2965, 2990, 2986, 2997, 2991, 73702, 3057, 3058, 3056, + 3066, 73727, 3065, 73703, 73687, 2946, 73686, 73698, 73690, 73693, 73692, + 73712, 73689, 73688, 73691, 73697, 73694, 73695, 73696, 73713, 73699, + 3021, 2947, 73685, 73708, 73711, 983939, 983940, 983947, 983950, 983943, + 983944, 983948, 983949, 983941, 983942, 983945, 983946, 983686, 983693, + 983696, 983689, 983690, 983694, 983695, 983687, 983688, 983691, 983692, + 983840, 983847, 983850, 983843, 983844, 983848, 983849, 983841, 983842, + 983845, 983846, 983851, 983858, 983861, 983854, 983855, 983859, 983860, + 983852, 983853, 983856, 983857, 983818, 983825, 983828, 983821, 983822, + 983826, 983827, 983819, 983820, 983823, 983824, 983873, 983880, 983883, + 983876, 983877, 983881, 983882, 983874, 983875, 983878, 983879, 983741, + 983748, 983751, 983744, 983745, 983749, 983750, 983742, 983743, 983746, + 983747, 983697, 983704, 983707, 983700, 983701, 983705, 983706, 983698, + 983699, 983702, 983703, 983719, 983726, 983729, 983722, 983723, 983727, + 983728, 983720, 983721, 983724, 983725, 983763, 983770, 983773, 983766, + 983767, 983771, 983772, 983764, 983765, 983768, 983769, 983862, 983869, + 983872, 983865, 983866, 983870, 983871, 983863, 983864, 983867, 983868, + 983807, 983814, 983817, 983810, 983811, 983815, 983816, 983808, 983809, + 983812, 983813, 983895, 983902, 983905, 983898, 983899, 983903, 983904, + 983951, 983896, 983897, 983900, 983901, 983906, 983913, 983916, 983909, + 983910, 983914, 983915, 983907, 983908, 983911, 983912, 983917, 983924, + 983927, 983920, 983921, 983925, 983926, 983918, 983919, 983922, 983923, + 983730, 983737, 983740, 983733, 983734, 983738, 983739, 983731, 983732, + 983735, 983736, 983752, 983759, 983762, 983755, 983756, 983760, 983761, + 983753, 983754, 983757, 983758, 983708, 983715, 983718, 983711, 983712, + 983716, 983717, 983709, 983710, 983713, 983714, 983928, 983935, 983938, + 983931, 983932, 983936, 983937, 983929, 983930, 983933, 983934, 983884, + 983891, 983894, 983887, 983888, 983892, 983893, 983885, 983886, 983889, + 983890, 983785, 983792, 983795, 983788, 983789, 983793, 983794, 983786, + 983787, 983790, 983791, 983774, 983781, 983784, 983777, 983778, 983782, + 983783, 983775, 983776, 983779, 983780, 983829, 983836, 983839, 983832, + 983833, 983837, 983838, 983830, 983831, 983834, 983835, 983796, 983803, + 983806, 983799, 983800, 983804, 983805, 983797, 983798, 983801, 983802, + 73709, 73704, 73705, 3006, 3016, 3020, 3009, 3010, 3018, 3019, 3007, + 3008, 3014, 3015, 3061, 3060, 3024, 129748, 127883, 127818, 92809, 92810, + 92811, 92808, 92789, 92790, 92791, 92788, 92816, 92859, 92856, 92847, + 92845, 92817, 92846, 92843, 92829, 92830, 92831, 92828, 92835, 92851, + 92840, 92844, 92818, 92819, 92852, 92836, 92825, 92826, 92827, 92824, + 92813, 92814, 92815, 92812, 92820, 92822, 92823, 92821, 92805, 92806, + 92807, 92804, 92797, 92798, 92799, 92796, 92801, 92802, 92803, 92800, + 92785, 92786, 92787, 92784, 92793, 92794, 92795, 92792, 92857, 92854, + 92848, 92861, 92853, 92860, 92849, 92855, 92834, 92833, 92832, 92841, + 92839, 92842, 92850, 92838, 92858, 92837, 92862, 92869, 92868, 92871, + 92870, 92867, 92866, 92864, 92873, 92865, 92872, 100352, 100353, 100354, + 100355, 100356, 100357, 100358, 100359, 100360, 100361, 100362, 100363, + 100364, 100365, 100366, 100367, 100368, 100369, 100370, 100371, 100372, + 100373, 100374, 100375, 100376, 100377, 100378, 100379, 100380, 100381, + 100382, 100383, 100384, 100385, 100386, 100387, 100388, 100389, 100390, + 100391, 100392, 100393, 100394, 100395, 100396, 100397, 100398, 100399, + 100400, 100401, 100402, 100403, 100404, 100405, 100406, 100407, 100408, + 100409, 100410, 100411, 100412, 100413, 100414, 100415, 100416, 100417, + 100418, 100419, 100420, 100421, 100422, 100423, 100424, 100425, 100426, + 100427, 100428, 100429, 100430, 100431, 100432, 100433, 100434, 100435, + 100436, 100437, 100438, 100439, 100440, 100441, 100442, 100443, 100444, + 100445, 100446, 100447, 100448, 100449, 100450, 100451, 100452, 100453, + 100454, 100455, 100456, 100457, 100458, 100459, 100460, 100461, 100462, + 100463, 100464, 100465, 100466, 100467, 100468, 100469, 100470, 100471, + 100472, 100473, 100474, 100475, 100476, 100477, 100478, 100479, 100480, + 100481, 100482, 100483, 100484, 100485, 100486, 100487, 100488, 100489, + 100490, 100491, 100492, 100493, 100494, 100495, 100496, 100497, 100498, + 100499, 100500, 100501, 100502, 100503, 100504, 100505, 100506, 100507, + 100508, 100509, 100510, 100511, 100512, 100513, 100514, 100515, 100516, + 100517, 100518, 100519, 100520, 100521, 100522, 100523, 100524, 100525, + 100526, 100527, 100528, 100529, 100530, 100531, 100532, 100533, 100534, + 100535, 100536, 100537, 100538, 100539, 100540, 100541, 100542, 100543, + 100544, 100545, 100546, 100547, 100548, 100549, 100550, 100551, 100552, + 100553, 100554, 100555, 100556, 100557, 100558, 100559, 100560, 100561, + 100562, 100563, 100564, 100565, 100566, 100567, 100568, 100569, 100570, + 100571, 100572, 100573, 100574, 100575, 100576, 100577, 100578, 100579, + 100580, 100581, 100582, 100583, 100584, 100585, 100586, 100587, 100588, + 100589, 100590, 100591, 100592, 100593, 100594, 100595, 100596, 100597, + 100598, 100599, 100600, 100601, 100602, 100603, 100604, 100605, 100606, + 100607, 100608, 100609, 100610, 100611, 100612, 100613, 100614, 100615, + 100616, 100617, 100618, 100619, 100620, 100621, 100622, 100623, 100624, + 100625, 100626, 100627, 100628, 100629, 100630, 100631, 100632, 100633, + 100634, 100635, 100636, 100637, 100638, 100639, 100640, 100641, 100642, + 100643, 100644, 100645, 100646, 100647, 100648, 100649, 100650, 100651, + 100652, 100653, 100654, 100655, 100656, 100657, 100658, 100659, 100660, + 100661, 100662, 100663, 100664, 100665, 100666, 100667, 100668, 100669, + 100670, 100671, 100672, 100673, 100674, 100675, 100676, 100677, 100678, + 100679, 100680, 100681, 100682, 100683, 100684, 100685, 100686, 100687, + 100688, 100689, 100690, 100691, 100692, 100693, 100694, 100695, 100696, + 100697, 100698, 100699, 100700, 100701, 100702, 100703, 100704, 100705, + 100706, 100707, 100708, 100709, 100710, 100711, 100712, 100713, 100714, + 100715, 100716, 100717, 100718, 100719, 100720, 100721, 100722, 100723, + 100724, 100725, 100726, 100727, 100728, 100729, 100730, 100731, 100732, + 100733, 100734, 100735, 100736, 100737, 100738, 100739, 100740, 100741, + 100742, 100743, 100744, 100745, 100746, 100747, 100748, 100749, 100750, + 100751, 100752, 100753, 100754, 100755, 100756, 100757, 100758, 100759, + 100760, 100761, 100762, 100763, 100764, 100765, 100766, 100767, 100768, + 100769, 100770, 100771, 100772, 100773, 100774, 100775, 100776, 100777, + 100778, 100779, 100780, 100781, 100782, 100783, 100784, 100785, 100786, + 100787, 100788, 100789, 100790, 100791, 100792, 100793, 100794, 100795, + 100796, 100797, 100798, 100799, 100800, 100801, 100802, 100803, 100804, + 100805, 100806, 100807, 100808, 100809, 100810, 100811, 100812, 100813, + 100814, 100815, 100816, 100817, 100818, 100819, 100820, 100821, 100822, + 100823, 100824, 100825, 100826, 100827, 100828, 100829, 100830, 100831, + 100832, 100833, 100834, 100835, 100836, 100837, 100838, 100839, 100840, + 100841, 100842, 100843, 100844, 100845, 100846, 100847, 100848, 100849, + 100850, 100851, 100852, 100853, 100854, 100855, 100856, 100857, 100858, + 100859, 100860, 100861, 100862, 100863, 100864, 100865, 100866, 100867, + 100868, 100869, 100870, 100871, 100872, 100873, 100874, 100875, 100876, + 100877, 100878, 100879, 100880, 100881, 100882, 100883, 100884, 100885, + 100886, 100887, 100888, 100889, 100890, 100891, 100892, 100893, 100894, + 100895, 100896, 100897, 100898, 100899, 100900, 100901, 100902, 100903, + 100904, 100905, 100906, 100907, 100908, 100909, 100910, 100911, 100912, + 100913, 100914, 100915, 100916, 100917, 100918, 100919, 100920, 100921, + 100922, 100923, 100924, 100925, 100926, 100927, 100928, 100929, 100930, + 100931, 100932, 100933, 100934, 100935, 100936, 100937, 100938, 100939, + 100940, 100941, 100942, 100943, 100944, 100945, 100946, 100947, 100948, + 100949, 100950, 100951, 100952, 100953, 100954, 100955, 100956, 100957, + 100958, 100959, 100960, 100961, 100962, 100963, 100964, 100965, 100966, + 100967, 100968, 100969, 100970, 100971, 100972, 100973, 100974, 100975, + 100976, 100977, 100978, 100979, 100980, 100981, 100982, 100983, 100984, + 100985, 100986, 100987, 100988, 100989, 100990, 100991, 100992, 100993, + 100994, 100995, 100996, 100997, 100998, 100999, 101000, 101001, 101002, + 101003, 101004, 101005, 101006, 101007, 101008, 101009, 101010, 101011, + 101012, 101013, 101014, 101015, 101016, 101017, 101018, 101019, 101020, + 101021, 101022, 101023, 101024, 101025, 101026, 101027, 101028, 101029, + 101030, 101031, 101032, 101033, 101034, 101035, 101036, 101037, 101038, + 101039, 101040, 101041, 101042, 101043, 101044, 101045, 101046, 101047, + 101048, 101049, 101050, 101051, 101052, 101053, 101054, 101055, 101056, + 101057, 101058, 101059, 101060, 101061, 101062, 101063, 101064, 101065, + 101066, 101067, 101068, 101069, 101070, 101071, 101072, 101073, 101074, + 101075, 101076, 101077, 101078, 101079, 101080, 101081, 101082, 101083, + 101084, 101085, 101086, 101087, 101088, 101089, 101090, 101091, 101092, + 101093, 101094, 101095, 101096, 101097, 101098, 101099, 101100, 101101, + 101102, 101103, 101104, 101105, 101106, 101107, 101108, 101109, 101110, + 101111, 101112, 101113, 101114, 101115, 101116, 101117, 101118, 101119, + 94176, 128429, 9991, 9801, 127790, 128661, 127861, 128198, 10043, 10170, + 129750, 129528, 128222, 128380, 8981, 9990, 8481, 128384, 128301, 128250, + 3158, 3195, 3198, 3194, 3197, 3193, 3196, 3192, 3106, 3105, 3111, 3161, + 3110, 3112, 3165, 3097, 3107, 3102, 3162, 3121, 3120, 3104, 3103, 3109, + 3160, 3108, 3077, 3078, 3088, 3092, 3124, 3123, 3122, 3083, 3168, 3084, + 3169, 3125, 3081, 3082, 3090, 3091, 3079, 3080, 3126, 3127, 3128, 3117, + 3116, 3099, 3098, 3096, 3095, 3101, 3100, 3094, 3093, 3115, 3114, 3086, + 3087, 3129, 3118, 3119, 3157, 3076, 3072, 3073, 3191, 3199, 3132, 3133, + 3074, 3149, 3075, 3134, 3144, 3148, 3137, 3138, 3139, 3140, 3170, 3171, + 3146, 3147, 3135, 3136, 3142, 3143, 3179, 3178, 3181, 3180, 3177, 3176, + 3174, 3183, 3175, 3182, 127934, 8376, 9978, 129514, 119617, 119564, + 119577, 119633, 119587, 119566, 119561, 119585, 119613, 119590, 119631, + 119634, 119630, 119608, 119582, 119563, 119573, 119558, 119624, 119567, + 119623, 119586, 119636, 119612, 119625, 119568, 119584, 119619, 119618, + 119583, 119603, 119600, 119626, 119610, 119580, 119576, 119638, 119559, + 119595, 119632, 119606, 119592, 119615, 119599, 119602, 119614, 119629, + 119574, 119569, 119570, 119622, 119562, 119591, 119637, 119597, 119589, + 119616, 119609, 119560, 119635, 119565, 119604, 119588, 119571, 119594, + 119578, 119596, 119579, 119598, 119572, 119605, 119627, 119621, 119628, + 119601, 119593, 119607, 119575, 119620, 119611, 119581, 1959, 1958, 1964, + 1961, 1965, 1927, 1954, 1951, 1937, 1931, 1930, 1956, 1934, 1935, 1955, + 1945, 1920, 1926, 1946, 1933, 1925, 1929, 1943, 1942, 1941, 1922, 1969, + 1950, 1949, 1921, 1936, 1939, 1932, 1947, 1944, 1952, 1928, 1957, 1938, + 1948, 1953, 1924, 1923, 1940, 1967, 1966, 1963, 1960, 1962, 1968, 3610, + 3592, 3594, 3593, 3596, 3598, 3604, 3613, 3615, 3663, 3630, 3627, 3588, + 3587, 3589, 3586, 3590, 3675, 3585, 3628, 3621, 3653, 3622, 3659, 3633, + 3657, 3658, 3656, 3655, 3654, 3617, 3674, 3591, 3661, 3603, 3609, 3631, + 3642, 3612, 3614, 3616, 3611, 3619, 3620, 3632, 3652, 3651, 3634, 3649, + 3635, 3640, 3638, 3639, 3641, 3636, 3637, 3648, 3650, 3625, 3624, 3626, + 3595, 3660, 3601, 3602, 3607, 3600, 3608, 3606, 3605, 3599, 3623, 3662, + 3597, 3618, 3629, 3647, 3669, 3668, 3671, 3670, 3667, 3666, 3664, 3673, + 3665, 3672, 8708, 8707, 8756, 127777, 10727, 128936, 8201, 128929, + 129300, 129353, 128173, 129652, 10176, 8278, 9887, 9886, 11057, 128485, + 128484, 128486, 128487, 8694, 128433, 10870, 128491, 128962, 128423, + 11160, 11162, 10146, 11163, 11161, 10147, 8196, 11835, 128077, 128078, + 9928, 9736, 3865, 3863, 3864, 4032, 4033, 4035, 4034, 3886, 3885, 3888, + 3887, 3884, 3883, 3891, 3890, 3882, 3889, 3877, 3876, 3879, 3878, 3875, + 3874, 3872, 3881, 3873, 3880, 4030, 4031, 3945, 3905, 3947, 3904, 3948, + 3938, 3946, 3917, 3916, 3932, 3931, 3922, 3921, 3908, 3918, 3913, 3923, + 3940, 3941, 3942, 3930, 3929, 3915, 3914, 3920, 3919, 3927, 3926, 3910, + 3909, 3907, 3906, 3925, 3924, 3934, 3935, 3936, 3943, 3911, 3939, 3928, + 3933, 3937, 3944, 3862, 3861, 983210, 3850, 4048, 3849, 3892, 4049, 3894, + 3859, 3846, 3896, 4052, 3845, 3847, 3842, 3843, 3841, 3860, 3898, 3899, + 3900, 3901, 3972, 4051, 3844, 3851, 3895, 3893, 4050, 3856, 3854, 3858, + 3857, 3848, 3853, 4058, 3897, 3855, 4057, 3973, 3852, 3978, 3979, 3974, + 3976, 3903, 3977, 3970, 3866, 3867, 3868, 3871, 3869, 3870, 4047, 4046, + 3966, 3967, 3975, 3902, 3980, 3971, 4028, 4026, 4027, 3997, 3996, 4012, + 4011, 4002, 4001, 4025, 3985, 3984, 3988, 3998, 3993, 4003, 4020, 4021, + 4022, 4010, 4009, 3995, 3994, 4000, 3999, 4007, 4006, 3990, 3989, 3987, + 3986, 4005, 4004, 4014, 4015, 4016, 4023, 3991, 4019, 4008, 4018, 4013, + 4017, 4024, 3983, 3982, 3981, 3840, 4036, 4041, 4044, 4043, 4042, 4038, + 4040, 4037, 4039, 3968, 3969, 3956, 3957, 3958, 3959, 3960, 3961, 3964, + 3965, 3954, 3955, 3962, 3963, 3953, 10717, 11647, 11608, 11595, 11585, + 11573, 11620, 11607, 11600, 11582, 11590, 11596, 11601, 11586, 11592, + 11568, 11575, 11577, 11578, 11576, 11571, 11606, 11572, 11581, 11589, + 11583, 11569, 11570, 11584, 11587, 11609, 11611, 11610, 11612, 11613, + 11615, 11619, 11594, 11621, 11604, 11605, 11614, 11588, 11580, 11574, + 11597, 11598, 11599, 11602, 11591, 11616, 11617, 11618, 11622, 11579, + 11593, 11623, 11603, 11631, 11632, 128005, 128047, 10053, 126, 8764, + 11081, 10610, 10859, 10858, 11807, 11806, 11803, 9202, 10708, 10709, + 10750, 68410, 70854, 70784, 70785, 70786, 70796, 70798, 70812, 70811, + 70817, 70816, 70810, 70809, 70815, 70814, 70791, 70792, 70793, 70794, + 70827, 70789, 70790, 70787, 70788, 70803, 70813, 70808, 70818, 70828, + 70829, 70830, 70822, 70821, 70805, 70804, 70802, 70801, 70807, 70806, + 70800, 70799, 70820, 70819, 70831, 70826, 70823, 70825, 70824, 70795, + 70797, 70847, 70851, 70852, 70848, 70850, 70849, 70842, 70845, 70832, + 70843, 70846, 70835, 70836, 70837, 70838, 70839, 70840, 70833, 70834, + 70841, 70844, 70869, 70868, 70871, 70870, 70867, 70866, 70864, 70873, + 70865, 70872, 70853, 70855, 11858, 8266, 128555, 127915, 67054, 67022, + 67020, 67027, 67051, 67011, 67031, 67023, 67021, 67033, 67049, 67047, + 67015, 67032, 67025, 67024, 67058, 67035, 67041, 67040, 67056, 67055, + 67039, 67038, 67037, 67034, 67059, 67018, 67017, 67030, 67029, 67008, + 67009, 67013, 67012, 67016, 67014, 67057, 67028, 67043, 67042, 67048, + 67046, 67053, 67052, 67010, 67019, 67036, 67045, 67026, 67044, 67050, + 127813, 128069, 129463, 129701, 129520, 10554, 10557, 10556, 9182, + 118256, 117851, 118262, 118263, 11865, 117850, 118252, 118254, 11866, + 117852, 118248, 118250, 11833, 118246, 11210, 8992, 127913, 9180, 130024, + 130016, 130031, 8988, 8975, 11810, 118279, 8989, 8974, 11811, 130028, + 9140, 9184, 128285, 127553, 127554, 127559, 127555, 127557, 127560, + 127552, 127556, 127558, 127274, 9008, 123554, 123556, 123559, 123561, + 123564, 123537, 123544, 123543, 123553, 123555, 123558, 123560, 123546, + 123565, 123563, 123539, 123541, 123550, 123549, 123540, 123552, 123542, + 123536, 123551, 123545, 123538, 123548, 123547, 123562, 123557, 123566, + 128701, 128508, 128434, 128668, 8482, 128650, 128651, 11223, 10971, + 128646, 10701, 10699, 128710, 10698, 10141, 128681, 128208, 8227, 128305, + 9783, 9776, 9777, 9782, 9779, 9781, 9780, 9778, 10998, 10856, 10857, + 10747, 8244, 8779, 10996, 10624, 8874, 10997, 11003, 11851, 11000, 10999, + 8749, 8285, 129484, 128654, 127865, 128032, 127942, 8872, 11231, 127930, + 8366, 70601, 70608, 70613, 70612, 70610, 70528, 70529, 70542, 70545, + 70559, 70558, 70564, 70563, 70581, 70579, 70573, 70580, 70572, 70557, + 70556, 70562, 70561, 70534, 70535, 70536, 70537, 70574, 70532, 70533, + 70530, 70531, 70550, 70560, 70555, 70565, 70575, 70576, 70577, 70569, + 70568, 70552, 70551, 70549, 70548, 70554, 70553, 70547, 70546, 70567, + 70566, 70544, 70539, 70578, 70570, 70571, 70609, 70583, 70604, 70607, + 70615, 70616, 70602, 70611, 70606, 70605, 70626, 70625, 70584, 70597, + 70600, 70587, 70588, 70589, 70590, 70591, 70592, 70585, 70586, 70599, + 70594, 127799, 8378, 129411, 8523, 10658, 11202, 9930, 9929, 8498, 11826, 11832, 8587, 8586, 128598, 128596, 8985, 128399, 8513, 8514, 8516, 11829, - 8526, 128599, 128597, 8489, 128034, 129347, 127799, 10041, 128256, 8273, - 128432, 10869, 8229, 8282, 11818, 128149, 10837, 10838, 10697, 10760, - 10759, 128108, 128109, 128490, 11834, 66432, 66451, 66454, 66433, 66436, + 8526, 128599, 128597, 8489, 128034, 129347, 10041, 128256, 128432, 10869, + 8229, 8282, 11818, 128149, 10837, 10838, 10697, 10760, 10759, 128108, + 128109, 117896, 8273, 128490, 11834, 66432, 66451, 66454, 66433, 66436, 66447, 66457, 66434, 66437, 66440, 66443, 66435, 66445, 66455, 66453, 66450, 66444, 66461, 66456, 66441, 66458, 66442, 66439, 66449, 66448, 66452, 66438, 66446, 66459, 66460, 66463, 9730, 9969, 9748, 128530, 11217, 8255, 9100, 8746, 10824, 10822, 10826, 10817, 10818, 10821, - 983116, 11258, 9842, 129412, 9903, 10685, 8963, 8996, 11193, 10577, + 983116, 11258, 9842, 129412, 9903, 8963, 8996, 11193, 10685, 10577, 10573, 10572, 10575, 8597, 8616, 11021, 8661, 129113, 8691, 11109, 10622, - 8944, 8869, 10207, 128742, 128744, 128316, 128743, 128314, 9709, 9710, - 129898, 129946, 129920, 129896, 9985, 129924, 9600, 129937, 129938, 9690, - 129934, 9696, 129885, 129887, 129889, 129886, 129883, 129888, 129881, - 129884, 129882, 129879, 129880, 10196, 129944, 129948, 9720, 9136, 9692, - 129922, 9620, 129874, 129875, 129892, 129894, 129890, 129878, 129893, - 129891, 129876, 129895, 129877, 10064, 9137, 10000, 9693, 128319, 10066, - 129945, 129949, 9721, 129926, 129923, 129925, 129901, 128579, 11797, - 8593, 129976, 8645, 8670, 129033, 129029, 129177, 129041, 129025, 129045, - 8624, 8625, 10505, 8613, 10514, 11145, 11014, 8657, 8673, 129077, 10606, - 10595, 10592, 10584, 8639, 10588, 10580, 8638, 129089, 129093, 129085, - 10224, 128621, 129105, 129081, 11105, 11137, 11121, 129065, 11170, 11171, - 129061, 129057, 129073, 129069, 11131, 11115, 11141, 129169, 10506, 8607, - 11245, 10569, 8648, 8679, 8682, 11192, 8683, 8684, 8685, 129173, 8686, - 8687, 9797, 983117, 42509, 42510, 42511, 42446, 42370, 42486, 42257, - 42333, 42294, 42407, 42445, 42369, 42485, 42256, 42332, 42293, 42406, - 42449, 42373, 42489, 42260, 42336, 42297, 42410, 42434, 42359, 42473, - 42246, 42321, 42283, 42396, 42435, 42360, 42474, 42247, 42322, 42284, - 42397, 42452, 42376, 42492, 42263, 42339, 42300, 42413, 42451, 42375, - 42491, 42262, 42338, 42299, 42412, 42444, 42368, 42484, 42255, 42331, - 42292, 42405, 42443, 42367, 42483, 42254, 42330, 42291, 42404, 42454, - 42378, 42494, 42265, 42341, 42302, 42415, 42453, 42377, 42493, 42264, - 42340, 42301, 42414, 42439, 42440, 42364, 42479, 42251, 42480, 42327, - 42288, 42401, 42502, 42272, 42503, 42461, 42385, 42349, 42309, 42422, - 42429, 42430, 42355, 42468, 42242, 42469, 42316, 42317, 42278, 42279, - 42391, 42392, 42476, 42249, 42477, 42437, 42362, 42324, 42325, 42286, - 42399, 42459, 42383, 42346, 42347, 42499, 42270, 42307, 42420, 42487, - 42508, 42258, 42447, 42371, 42334, 42295, 42408, 42436, 42361, 42475, - 42248, 42323, 42285, 42398, 42438, 42363, 42478, 42250, 42326, 42287, - 42400, 42462, 42386, 42504, 42273, 42350, 42310, 42423, 42450, 42514, - 42539, 42512, 42513, 42538, 42374, 42490, 42261, 42337, 42298, 42411, - 42507, 42500, 42271, 42501, 42460, 42384, 42348, 42308, 42421, 42315, - 42467, 42428, 42457, 42381, 42497, 42268, 42344, 42305, 42418, 42464, - 42388, 42506, 42275, 42352, 42312, 42425, 42463, 42387, 42505, 42274, - 42351, 42311, 42424, 42455, 42379, 42495, 42266, 42342, 42303, 42416, - 42441, 42365, 42481, 42252, 42328, 42289, 42402, 42456, 42380, 42496, - 42267, 42343, 42304, 42417, 42433, 42358, 42472, 42245, 42320, 42282, - 42395, 42448, 42372, 42488, 42259, 42335, 42296, 42409, 42442, 42366, - 42482, 42253, 42329, 42290, 42403, 42458, 42382, 42498, 42269, 42345, - 42306, 42419, 42470, 42243, 42244, 42471, 42431, 42356, 42357, 42432, - 42318, 42319, 42280, 42281, 42393, 42394, 42465, 42240, 42241, 42466, - 42426, 42353, 42354, 42427, 42313, 42314, 42276, 42277, 42389, 42390, - 42523, 42526, 42522, 42515, 42520, 42527, 42516, 42524, 42518, 42517, - 42525, 42521, 42519, 42533, 42532, 42535, 42534, 42531, 42530, 42528, - 42537, 42529, 42536, 65024, 65033, 917843, 917844, 917845, 917846, - 917847, 917848, 917849, 917850, 917851, 917852, 65034, 917853, 917854, - 917855, 917856, 917857, 917858, 917859, 917860, 917861, 917862, 65035, - 917863, 917864, 917865, 917866, 917867, 917868, 917869, 917870, 917871, - 917872, 65036, 917873, 917874, 917875, 917876, 917877, 917878, 917879, - 917880, 917881, 917882, 65037, 917883, 917884, 917885, 917886, 917887, - 917888, 917889, 917890, 917891, 917892, 65038, 917893, 917894, 917895, - 917896, 917897, 917898, 917899, 917900, 917901, 917902, 65039, 917903, - 917904, 917905, 917906, 917907, 917908, 917909, 917910, 917911, 917912, - 917760, 917913, 917914, 917915, 917916, 917917, 917918, 917919, 917920, - 917921, 917922, 917761, 917923, 917924, 917925, 917926, 917927, 917928, - 917929, 917930, 917931, 917932, 917762, 917933, 917934, 917935, 917936, - 917937, 917938, 917939, 917940, 917941, 917942, 65025, 917763, 917943, - 917944, 917945, 917946, 917947, 917948, 917949, 917950, 917951, 917952, - 917764, 917953, 917954, 917955, 917956, 917957, 917958, 917959, 917960, - 917961, 917962, 917765, 917963, 917964, 917965, 917966, 917967, 917968, - 917969, 917970, 917971, 917972, 917766, 917973, 917974, 917975, 917976, - 917977, 917978, 917979, 917980, 917981, 917982, 917767, 917983, 917984, - 917985, 917986, 917987, 917988, 917989, 917990, 917991, 917992, 917768, - 917993, 917994, 917995, 917996, 917997, 917998, 917999, 917769, 917770, - 917771, 917772, 65026, 917773, 917774, 917775, 917776, 917777, 917778, - 917779, 917780, 917781, 917782, 65027, 917783, 917784, 917785, 917786, - 917787, 917788, 917789, 917790, 917791, 917792, 65028, 917793, 917794, - 917795, 917796, 917797, 917798, 917799, 917800, 917801, 917802, 65029, - 917803, 917804, 917805, 917806, 917807, 917808, 917809, 917810, 917811, - 917812, 65030, 917813, 917814, 917815, 917816, 917817, 917818, 917819, - 917820, 917821, 917822, 65031, 917823, 917824, 917825, 917826, 917827, - 917828, 917829, 917830, 917831, 917832, 65032, 917833, 917834, 917835, - 917836, 917837, 917838, 917839, 917840, 917841, 917842, 129499, 983245, - 983254, 983356, 983357, 983358, 983359, 983360, 983361, 983362, 983363, - 983364, 983365, 983255, 983366, 983367, 983368, 983369, 983370, 983371, - 983372, 983373, 983374, 983375, 983256, 983376, 983377, 983378, 983379, - 983380, 983381, 983382, 983383, 983384, 983385, 983257, 983386, 983387, - 983388, 983389, 983390, 983391, 983392, 983393, 983394, 983395, 983258, - 983396, 983397, 983398, 983399, 983400, 983401, 983402, 983403, 983404, - 983405, 983259, 983406, 983407, 983408, 983409, 983410, 983411, 983412, - 983413, 983414, 983415, 983260, 983416, 983417, 983418, 983419, 983420, - 983421, 983422, 983423, 983424, 983425, 983273, 983426, 983427, 983428, - 983429, 983430, 983431, 983432, 983433, 983434, 983435, 983274, 983436, - 983437, 983438, 983439, 983440, 983441, 983442, 983443, 983444, 983445, - 983275, 983446, 983447, 983448, 983449, 983450, 983451, 983452, 983453, - 983454, 983455, 983246, 983276, 983456, 983457, 983458, 983459, 983460, - 983461, 983462, 983463, 983464, 983465, 983277, 983466, 983467, 983468, - 983469, 983470, 983471, 983472, 983473, 983474, 983475, 983278, 983476, - 983477, 983478, 983479, 983480, 983481, 983482, 983483, 983484, 983485, - 983279, 983486, 983487, 983488, 983489, 983490, 983491, 983492, 983493, - 983494, 983495, 983280, 983496, 983497, 983498, 983499, 983500, 983501, - 983502, 983503, 983504, 983505, 983281, 983506, 983507, 983508, 983509, - 983510, 983511, 983512, 983282, 983283, 983284, 983285, 983247, 983286, - 983287, 983288, 983289, 983290, 983291, 983292, 983293, 983294, 983295, - 983248, 983296, 983297, 983298, 983299, 983300, 983301, 983302, 983303, - 983304, 983305, 983249, 983306, 983307, 983308, 983309, 983310, 983311, - 983312, 983313, 983314, 983315, 983250, 983316, 983317, 983318, 983319, - 983320, 983321, 983322, 983323, 983324, 983325, 983251, 983326, 983327, - 983328, 983329, 983330, 983331, 983332, 983333, 983334, 983335, 983252, - 983336, 983337, 983338, 983339, 983340, 983341, 983342, 983343, 983344, - 983345, 983253, 983346, 983347, 983348, 983349, 983350, 983351, 983352, - 983353, 983354, 983355, 7401, 7402, 7409, 7403, 7404, 7415, 7410, 7418, - 7413, 7379, 7398, 7396, 7411, 7408, 7406, 7407, 7405, 7414, 7397, 7400, - 7395, 7399, 7394, 7380, 7384, 7412, 7417, 7386, 7389, 7376, 7388, 7378, - 7416, 7392, 7391, 7387, 7390, 7381, 7382, 7383, 7385, 7393, 7377, 8483, - 10704, 10980, 10978, 10186, 8942, 8286, 12347, 12337, 12339, 12341, - 12338, 12340, 124, 9168, 10992, 10991, 9087, 9896, 129904, 129905, - 129906, 129907, 129908, 129909, 11135, 983069, 983144, 11823, 128678, - 11837, 10650, 128976, 128959, 128947, 128637, 128941, 128953, 128636, + 8944, 8869, 10207, 117873, 117877, 117857, 128743, 117881, 118267, + 128742, 117847, 128314, 117912, 117862, 128744, 128316, 9709, 9710, + 117760, 129898, 129946, 129920, 129896, 9985, 118417, 117809, 118418, + 117810, 130020, 129924, 9600, 129937, 118275, 129938, 9690, 118292, + 129934, 118439, 118444, 9696, 129885, 129887, 129889, 129886, 129883, + 129888, 129881, 129884, 129882, 129879, 129880, 10196, 118416, 9136, + 129944, 129948, 9720, 117808, 9692, 117958, 117946, 117962, 117966, + 117950, 117954, 117928, 117926, 117942, 117970, 117938, 117813, 118421, + 118422, 118420, 117812, 118423, 117815, 129922, 9620, 129874, 129875, + 129892, 129894, 129890, 129878, 129893, 129891, 129876, 129895, 129877, + 10064, 118419, 9137, 10000, 9693, 117959, 117947, 117963, 117967, 117951, + 117955, 117929, 117927, 117943, 117971, 117939, 117814, 128319, 10066, + 129945, 129949, 9721, 117811, 129926, 129923, 118438, 118445, 129925, + 129901, 128579, 11797, 8593, 129976, 8645, 8670, 129033, 129029, 129177, + 129041, 129025, 129045, 8624, 8625, 10505, 8613, 10514, 11145, 11014, + 8657, 8673, 129077, 10606, 10595, 10592, 10584, 8639, 10588, 10580, 8638, + 129089, 129093, 129085, 10224, 128621, 129105, 129081, 11105, 11137, + 11121, 129065, 11170, 11171, 129061, 129057, 129073, 129069, 11131, + 11115, 11141, 129169, 10506, 8607, 11245, 10569, 8648, 8679, 8683, 8685, + 8684, 129173, 8682, 11192, 8686, 8687, 9797, 983117, 42509, 42510, 42511, + 42446, 42370, 42486, 42257, 42333, 42294, 42407, 42445, 42369, 42485, + 42256, 42332, 42293, 42406, 42449, 42373, 42489, 42260, 42336, 42297, + 42410, 42434, 42359, 42473, 42246, 42321, 42283, 42396, 42435, 42360, + 42474, 42247, 42322, 42284, 42397, 42452, 42376, 42492, 42263, 42339, + 42300, 42413, 42451, 42375, 42491, 42262, 42338, 42299, 42412, 42444, + 42368, 42484, 42255, 42331, 42292, 42405, 42443, 42367, 42483, 42254, + 42330, 42291, 42404, 42454, 42378, 42494, 42265, 42341, 42302, 42415, + 42453, 42377, 42493, 42264, 42340, 42301, 42414, 42439, 42440, 42364, + 42479, 42251, 42480, 42327, 42288, 42401, 42502, 42272, 42503, 42461, + 42385, 42349, 42309, 42422, 42429, 42430, 42355, 42468, 42242, 42469, + 42316, 42317, 42278, 42279, 42391, 42392, 42476, 42249, 42477, 42437, + 42362, 42324, 42325, 42286, 42399, 42459, 42383, 42346, 42347, 42499, + 42270, 42307, 42420, 42487, 42508, 42258, 42447, 42371, 42334, 42295, + 42408, 42436, 42361, 42475, 42248, 42323, 42285, 42398, 42438, 42363, + 42478, 42250, 42326, 42287, 42400, 42462, 42386, 42504, 42273, 42350, + 42310, 42423, 42450, 42514, 42539, 42512, 42513, 42538, 42374, 42490, + 42261, 42337, 42298, 42411, 42507, 42500, 42271, 42501, 42460, 42384, + 42348, 42308, 42421, 42315, 42467, 42428, 42457, 42381, 42497, 42268, + 42344, 42305, 42418, 42464, 42388, 42506, 42275, 42352, 42312, 42425, + 42463, 42387, 42505, 42274, 42351, 42311, 42424, 42455, 42379, 42495, + 42266, 42342, 42303, 42416, 42441, 42365, 42481, 42252, 42328, 42289, + 42402, 42456, 42380, 42496, 42267, 42343, 42304, 42417, 42433, 42358, + 42472, 42245, 42320, 42282, 42395, 42448, 42372, 42488, 42259, 42335, + 42296, 42409, 42442, 42366, 42482, 42253, 42329, 42290, 42403, 42458, + 42382, 42498, 42269, 42345, 42306, 42419, 42470, 42243, 42244, 42471, + 42431, 42356, 42357, 42432, 42318, 42319, 42280, 42281, 42393, 42394, + 42465, 42240, 42241, 42466, 42426, 42353, 42354, 42427, 42313, 42314, + 42276, 42277, 42389, 42390, 42523, 42526, 42522, 42515, 42520, 42527, + 42516, 42524, 42518, 42517, 42525, 42521, 42519, 42533, 42532, 42535, + 42534, 42531, 42530, 42528, 42537, 42529, 42536, 65024, 65033, 917843, + 917844, 917845, 917846, 917847, 917848, 917849, 917850, 917851, 917852, + 65034, 917853, 917854, 917855, 917856, 917857, 917858, 917859, 917860, + 917861, 917862, 65035, 917863, 917864, 917865, 917866, 917867, 917868, + 917869, 917870, 917871, 917872, 65036, 917873, 917874, 917875, 917876, + 917877, 917878, 917879, 917880, 917881, 917882, 65037, 917883, 917884, + 917885, 917886, 917887, 917888, 917889, 917890, 917891, 917892, 65038, + 917893, 917894, 917895, 917896, 917897, 917898, 917899, 917900, 917901, + 917902, 65039, 917903, 917904, 917905, 917906, 917907, 917908, 917909, + 917910, 917911, 917912, 917760, 917913, 917914, 917915, 917916, 917917, + 917918, 917919, 917920, 917921, 917922, 917761, 917923, 917924, 917925, + 917926, 917927, 917928, 917929, 917930, 917931, 917932, 917762, 917933, + 917934, 917935, 917936, 917937, 917938, 917939, 917940, 917941, 917942, + 65025, 917763, 917943, 917944, 917945, 917946, 917947, 917948, 917949, + 917950, 917951, 917952, 917764, 917953, 917954, 917955, 917956, 917957, + 917958, 917959, 917960, 917961, 917962, 917765, 917963, 917964, 917965, + 917966, 917967, 917968, 917969, 917970, 917971, 917972, 917766, 917973, + 917974, 917975, 917976, 917977, 917978, 917979, 917980, 917981, 917982, + 917767, 917983, 917984, 917985, 917986, 917987, 917988, 917989, 917990, + 917991, 917992, 917768, 917993, 917994, 917995, 917996, 917997, 917998, + 917999, 917769, 917770, 917771, 917772, 65026, 917773, 917774, 917775, + 917776, 917777, 917778, 917779, 917780, 917781, 917782, 65027, 917783, + 917784, 917785, 917786, 917787, 917788, 917789, 917790, 917791, 917792, + 65028, 917793, 917794, 917795, 917796, 917797, 917798, 917799, 917800, + 917801, 917802, 65029, 917803, 917804, 917805, 917806, 917807, 917808, + 917809, 917810, 917811, 917812, 65030, 917813, 917814, 917815, 917816, + 917817, 917818, 917819, 917820, 917821, 917822, 65031, 917823, 917824, + 917825, 917826, 917827, 917828, 917829, 917830, 917831, 917832, 65032, + 917833, 917834, 917835, 917836, 917837, 917838, 917839, 917840, 917841, + 917842, 129499, 983245, 983254, 983360, 983361, 983362, 983363, 983364, + 983365, 983366, 983367, 983368, 983369, 983255, 983370, 983371, 983372, + 983373, 983374, 983375, 983376, 983377, 983378, 983379, 983256, 983380, + 983381, 983382, 983383, 983384, 983385, 983386, 983387, 983388, 983389, + 983257, 983390, 983391, 983392, 983393, 983394, 983395, 983396, 983397, + 983398, 983399, 983258, 983400, 983401, 983402, 983403, 983404, 983405, + 983406, 983407, 983408, 983409, 983259, 983410, 983411, 983412, 983413, + 983414, 983415, 983416, 983417, 983418, 983419, 983260, 983420, 983421, + 983422, 983423, 983424, 983425, 983426, 983427, 983428, 983429, 983277, + 983430, 983431, 983432, 983433, 983434, 983435, 983436, 983437, 983438, + 983439, 983278, 983440, 983441, 983442, 983443, 983444, 983445, 983446, + 983447, 983448, 983449, 983279, 983450, 983451, 983452, 983453, 983454, + 983455, 983456, 983457, 983458, 983459, 983246, 983280, 983460, 983461, + 983462, 983463, 983464, 983465, 983466, 983467, 983468, 983469, 983281, + 983470, 983471, 983472, 983473, 983474, 983475, 983476, 983477, 983478, + 983479, 983282, 983480, 983481, 983482, 983483, 983484, 983485, 983486, + 983487, 983488, 983489, 983283, 983490, 983491, 983492, 983493, 983494, + 983495, 983496, 983497, 983498, 983499, 983284, 983500, 983501, 983502, + 983503, 983504, 983505, 983506, 983507, 983508, 983509, 983285, 983510, + 983511, 983512, 983513, 983514, 983515, 983516, 983286, 983287, 983288, + 983289, 983247, 983290, 983291, 983292, 983293, 983294, 983295, 983296, + 983297, 983298, 983299, 983248, 983300, 983301, 983302, 983303, 983304, + 983305, 983306, 983307, 983308, 983309, 983249, 983310, 983311, 983312, + 983313, 983314, 983315, 983316, 983317, 983318, 983319, 983250, 983320, + 983321, 983322, 983323, 983324, 983325, 983326, 983327, 983328, 983329, + 983251, 983330, 983331, 983332, 983333, 983334, 983335, 983336, 983337, + 983338, 983339, 983252, 983340, 983341, 983342, 983343, 983344, 983345, + 983346, 983347, 983348, 983349, 983253, 983350, 983351, 983352, 983353, + 983354, 983355, 983356, 983357, 983358, 983359, 7401, 7402, 7409, 7403, + 7404, 7415, 7410, 7418, 7413, 7379, 7398, 7396, 7411, 7408, 7406, 7407, + 7405, 7414, 7397, 7400, 7395, 7399, 7394, 7380, 7384, 7412, 7417, 7386, + 7389, 7376, 7388, 7378, 7416, 7392, 7391, 7387, 7390, 7381, 7382, 7383, + 7385, 7393, 7377, 8483, 10704, 10980, 10978, 10186, 117780, 8942, 8286, + 117917, 12347, 12337, 12339, 12341, 12338, 12340, 117892, 124, 9168, + 10992, 10991, 117904, 118293, 9087, 9896, 129904, 129905, 129906, 129907, + 129908, 129909, 117916, 117770, 11135, 983069, 983144, 11823, 128678, + 10650, 11837, 128976, 128959, 128947, 128637, 128941, 128953, 128636, 128904, 128914, 128934, 8921, 8920, 128933, 10799, 9910, 128243, 9996, 128249, 127918, 128252, 94193, 94192, 8983, 127931, 9805, 66929, 66930, 66936, 66935, 66942, 66943, 66947, 66946, 66950, 66949, 66932, 66931, @@ -12260,211 +13077,211 @@ static const unsigned int dawg_pos_to_codepoint[] = { 67004, 66977, 66984, 66992, 66967, 66990, 66999, 67003, 129979, 127755, 127952, 8752, 983070, 983145, 11238, 8541, 8538, 8536, 8539, 8533, 189, 8529, 188, 8528, 8537, 8530, 8531, 8540, 190, 8535, 8534, 8532, 8542, - 8585, 123585, 123584, 123624, 123619, 123620, 123606, 123605, 123622, - 123618, 123611, 123623, 123612, 123621, 123615, 123592, 123613, 123625, - 123593, 123596, 123617, 123616, 123595, 123614, 123626, 123627, 123590, - 123609, 123604, 123591, 123594, 123599, 123598, 123586, 123587, 123588, - 123597, 123589, 123610, 123600, 123608, 123607, 123603, 123602, 123601, - 123647, 123630, 123631, 123628, 123629, 123637, 123636, 123639, 123638, - 123635, 123634, 123632, 123641, 123633, 123640, 127768, 127766, 127762, - 127764, 71841, 71850, 71862, 71861, 71848, 71856, 71853, 71867, 71866, - 71865, 71868, 71840, 71859, 71849, 71857, 71869, 71870, 71855, 71847, - 71843, 71854, 71844, 71845, 71858, 71871, 71863, 71864, 71852, 71846, - 71842, 71860, 71851, 71873, 71882, 71894, 71893, 71880, 71888, 71885, - 71899, 71898, 71897, 71900, 71872, 71891, 71881, 71889, 71901, 71902, - 71887, 71879, 71875, 71886, 71876, 71877, 71890, 71903, 71895, 71896, - 71884, 71878, 71874, 71892, 71883, 71909, 71908, 71911, 71910, 71907, - 71906, 71904, 71913, 71905, 71912, 71921, 71918, 71917, 71922, 71920, - 71919, 71916, 71915, 71914, 71935, 9888, 128702, 127754, 128003, 129341, - 127817, 8986, 12316, 11071, 10547, 127988, 127987, 128075, 12336, 65103, - 8967, 65099, 129479, 128465, 128576, 128553, 10172, 128146, 983238, + 8585, 129479, 123585, 123584, 123624, 123619, 123620, 123606, 123605, + 123622, 123618, 123611, 123623, 123612, 123621, 123615, 123592, 123613, + 123625, 123593, 123596, 123617, 123616, 123595, 123614, 123626, 123627, + 123590, 123609, 123604, 123591, 123594, 123599, 123598, 123586, 123587, + 123588, 123597, 123589, 123610, 123600, 123608, 123607, 123603, 123602, + 123601, 123647, 123630, 123631, 123628, 123629, 123637, 123636, 123639, + 123638, 123635, 123634, 123632, 123641, 123633, 123640, 127768, 127766, + 127762, 127764, 71841, 71850, 71862, 71861, 71848, 71856, 71853, 71867, + 71866, 71865, 71868, 71840, 71859, 71849, 71857, 71869, 71870, 71855, + 71847, 71843, 71854, 71844, 71845, 71858, 71871, 71863, 71864, 71852, + 71846, 71842, 71860, 71851, 71873, 71882, 71894, 71893, 71880, 71888, + 71885, 71899, 71898, 71897, 71900, 71872, 71891, 71881, 71889, 71901, + 71902, 71887, 71879, 71875, 71886, 71876, 71877, 71890, 71903, 71895, + 71896, 71884, 71878, 71874, 71892, 71883, 71909, 71908, 71911, 71910, + 71907, 71906, 71904, 71913, 71905, 71912, 71921, 71918, 71917, 71922, + 71920, 71919, 71916, 71915, 71914, 71935, 9888, 128702, 127754, 128003, + 129341, 127817, 8986, 12316, 11071, 10547, 127988, 127987, 128075, 12336, + 65103, 8967, 65099, 128465, 128576, 128553, 10172, 128146, 983238, 127947, 9840, 128734, 9784, 9855, 129197, 129196, 9702, 9815, 129548, 129590, 129608, 129611, 9812, 129545, 129587, 9816, 129542, 129563, 129584, 129591, 129605, 129549, 129616, 129614, 129615, 9817, 129550, 129592, 9813, 129546, 129588, 9814, 129547, 129589, 129569, 129566, 129570, 129571, 129567, 129568, 9675, 128906, 9862, 10732, 9863, 9717, - 9718, 9716, 9719, 9831, 10209, 10210, 10211, 9671, 9672, 128922, 128923, - 10192, 9826, 9931, 128071, 128407, 9759, 9663, 9661, 9920, 9921, 10069, - 9872, 9983, 10048, 128174, 11214, 10023, 9785, 128427, 129293, 9825, - 9989, 129984, 11041, 11053, 10710, 11036, 128326, 9945, 128072, 9756, - 9669, 9667, 9665, 128928, 11046, 11088, 9725, 9723, 11048, 11229, 10001, - 9649, 11040, 127985, 10068, 11092, 9659, 9657, 9655, 128073, 9758, 9645, - 9988, 65094, 9750, 11051, 11090, 9643, 9786, 9828, 9633, 128307, 128916, - 9635, 128917, 10212, 9713, 9714, 10213, 9634, 9712, 9715, 9707, 9093, - 127779, 127781, 127782, 9788, 127780, 9734, 128382, 9743, 9186, 10177, - 9943, 128070, 9757, 129994, 9653, 9651, 9708, 11006, 11055, 9647, 11038, - 10163, 128011, 129144, 129152, 129120, 129136, 129128, 129146, 129154, - 129122, 129138, 129130, 129147, 129155, 129123, 129139, 129131, 129145, - 129153, 129121, 129137, 129129, 129149, 129157, 129125, 129141, 129133, - 129148, 129156, 129124, 129140, 129132, 129150, 129158, 129126, 129142, - 129134, 129151, 129159, 129127, 129143, 129135, 11838, 129344, 127888, - 127788, 129695, 127863, 128521, 129725, 128430, 128732, 128105, 128111, - 128098, 128090, 128082, 128097, 128698, 11825, 8288, 128506, 128543, - 129713, 8361, 129717, 128058, 127873, 8768, 129340, 128295, 9997, 983233, - 8999, 129659, 129644, 129643, 129641, 129639, 129642, 129645, 129640, - 129637, 129636, 129634, 129632, 129635, 129638, 129633, 8891, 129393, - 128155, 165, 69292, 69291, 69293, 69256, 69255, 69254, 69281, 69268, - 69259, 69248, 69271, 69289, 69286, 69287, 69257, 69278, 69277, 69279, - 69276, 69280, 69296, 69282, 69251, 69250, 69266, 69265, 69267, 69253, - 69252, 69269, 69274, 69275, 69284, 69285, 69272, 69258, 69288, 69297, - 69263, 69260, 69270, 69262, 69261, 69249, 69283, 69273, 69264, 42139, - 42149, 42173, 42172, 42132, 42147, 42179, 42174, 42148, 42169, 42150, - 42134, 42166, 42135, 42145, 42143, 42137, 42175, 42157, 42154, 42167, - 42165, 42163, 42130, 42182, 42129, 42171, 42138, 42140, 42136, 42131, - 42151, 42164, 42181, 42142, 42156, 42170, 42176, 42178, 42160, 42133, - 42144, 42152, 42159, 42161, 42158, 42141, 42146, 42177, 42180, 42155, - 42162, 42128, 42168, 42153, 41070, 41059, 41060, 41058, 41073, 41072, - 41071, 41068, 41069, 41066, 41067, 41065, 41048, 41052, 41053, 41050, - 41051, 41049, 41046, 41047, 41056, 41057, 41054, 41055, 41063, 41064, - 41061, 41062, 41076, 41077, 41074, 41075, 41006, 40995, 40996, 40994, - 41009, 41008, 41007, 41004, 41005, 41002, 41003, 41001, 40984, 40988, - 40989, 40986, 40987, 40985, 40982, 40983, 40992, 40993, 40990, 40991, - 40999, 41000, 40997, 40998, 41012, 41015, 41014, 41013, 41010, 41011, - 41842, 41831, 41832, 41829, 41830, 41845, 41844, 41843, 41841, 41827, - 41828, 41825, 41826, 41839, 41840, 41837, 41838, 41835, 41836, 41833, - 41834, 41848, 41851, 41850, 41849, 41846, 41847, 41670, 41659, 41660, - 41658, 41673, 41672, 41671, 41668, 41669, 41666, 41667, 41665, 41648, - 41652, 41653, 41650, 41651, 41649, 41646, 41647, 41656, 41657, 41654, - 41655, 41663, 41664, 41661, 41662, 41676, 41679, 41678, 41677, 41674, - 41675, 41293, 41282, 41283, 41281, 41296, 41295, 41294, 41291, 41292, - 41272, 41275, 41276, 41274, 41273, 41270, 41271, 41289, 41290, 41288, - 41279, 41280, 41277, 41278, 41286, 41287, 41284, 41285, 41238, 41228, - 41227, 41241, 41240, 41239, 41236, 41237, 41218, 41221, 41222, 41220, - 41219, 41216, 41217, 41234, 41235, 41233, 41225, 41226, 41223, 41224, - 41231, 41232, 41229, 41230, 41174, 41175, 41173, 41171, 41172, 41169, - 41170, 41167, 41168, 41165, 41166, 41184, 41185, 41182, 41183, 41178, - 41181, 41180, 41179, 41176, 41177, 41494, 41496, 41497, 41495, 41492, - 41493, 41516, 41504, 41505, 41502, 41503, 41519, 41518, 41517, 41514, - 41515, 41500, 41501, 41498, 41499, 41512, 41513, 41510, 41511, 41508, - 41509, 41506, 41507, 41460, 41448, 41449, 41446, 41447, 41463, 41462, - 41461, 41458, 41459, 41436, 41440, 41441, 41438, 41439, 41437, 41434, - 41435, 41444, 41445, 41442, 41443, 41456, 41457, 41454, 41455, 41452, - 41453, 41450, 41451, 41584, 41583, 41582, 41389, 41379, 41380, 41378, - 41392, 41391, 41390, 41387, 41388, 41369, 41372, 41373, 41371, 41370, - 41367, 41368, 41385, 41386, 41384, 41382, 41383, 41381, 41376, 41377, - 41374, 41375, 41395, 41398, 41397, 41396, 41393, 41394, 41125, 41117, - 41118, 41116, 41128, 41127, 41126, 41123, 41124, 41107, 41110, 41111, - 41109, 41108, 41105, 41106, 41114, 41115, 41112, 41113, 41121, 41122, - 41119, 41120, 41130, 41133, 41132, 41131, 41129, 41336, 41334, 41335, - 41333, 41332, 41340, 41338, 41339, 41337, 41322, 41326, 41327, 41324, - 41325, 41323, 41320, 41321, 41330, 41331, 41328, 41329, 41556, 41557, - 41554, 41555, 41563, 41564, 41562, 41544, 41548, 41549, 41546, 41547, - 41545, 41542, 41543, 41552, 41553, 41550, 41551, 41560, 41561, 41558, - 41559, 41591, 41592, 41589, 41590, 41598, 41599, 41597, 41587, 41588, - 41585, 41586, 41595, 41596, 41593, 41594, 40962, 40960, 983243, 40966, - 40967, 40964, 40965, 40963, 40961, 42025, 42017, 42018, 42016, 42028, - 42027, 42026, 42023, 42024, 42010, 42014, 42015, 42012, 42013, 42011, - 42008, 42009, 42021, 42022, 42019, 42020, 42031, 42032, 42029, 42030, - 41970, 41962, 41963, 41960, 41961, 41973, 41972, 41971, 41968, 41969, - 41954, 41958, 41959, 41956, 41957, 41955, 41952, 41953, 41966, 41967, - 41964, 41965, 41976, 41979, 41978, 41977, 41974, 41975, 41488, 41476, - 41477, 41475, 41491, 41490, 41489, 41486, 41487, 41466, 41469, 41470, - 41468, 41467, 41464, 41465, 41473, 41474, 41471, 41472, 41484, 41485, - 41482, 41483, 41480, 41481, 41478, 41479, 41424, 41413, 41414, 41411, - 41412, 41427, 41426, 41425, 41422, 41423, 41420, 41421, 41419, 41401, - 41405, 41406, 41403, 41404, 41402, 41399, 41400, 41409, 41410, 41407, - 41408, 41417, 41418, 41415, 41416, 41430, 41433, 41432, 41431, 41428, - 41429, 41538, 41527, 41528, 41526, 41541, 41540, 41539, 41536, 41537, - 41534, 41535, 41533, 41524, 41525, 41522, 41523, 41531, 41532, 41529, - 41530, 41521, 41520, 41157, 41147, 41148, 41145, 41146, 41160, 41159, - 41158, 41155, 41156, 41136, 41139, 41140, 41138, 41137, 41134, 41135, - 41143, 41144, 41141, 41142, 41151, 41152, 41149, 41150, 41163, 41164, - 41161, 41162, 41154, 41153, 41080, 41083, 41084, 41082, 41081, 41078, - 41079, 41087, 41088, 41085, 41086, 41091, 41092, 41089, 41090, 41095, - 41098, 41097, 41096, 41093, 41094, 41101, 41104, 41103, 41102, 41099, - 41100, 41299, 41302, 41301, 41300, 41297, 41298, 41312, 41313, 41311, - 41305, 41306, 41303, 41304, 41309, 41310, 41307, 41308, 41316, 41319, - 41318, 41317, 41314, 41315, 41574, 41572, 41573, 41580, 41581, 41579, - 41566, 41567, 41565, 41570, 41571, 41568, 41569, 41577, 41578, 41575, - 41576, 42048, 42042, 42041, 42051, 42050, 42049, 42047, 42035, 42039, - 42040, 42037, 42038, 42036, 42033, 42034, 42045, 42046, 42043, 42044, - 42054, 42057, 42056, 42055, 42052, 42053, 41881, 41882, 41880, 41878, - 41879, 41876, 41877, 41885, 41886, 41883, 41884, 41889, 41892, 41891, - 41890, 41887, 41888, 41895, 41898, 41897, 41896, 41893, 41894, 42075, - 42067, 42068, 42066, 42076, 42073, 42074, 42060, 42064, 42065, 42062, - 42063, 42061, 42058, 42059, 42071, 42072, 42069, 42070, 41727, 41721, - 41720, 41730, 41729, 41728, 41726, 41723, 41722, 41711, 41714, 41715, - 41713, 41712, 41709, 41710, 41718, 41719, 41716, 41717, 41733, 41736, - 41735, 41734, 41731, 41732, 41725, 41724, 41363, 41352, 41353, 41351, - 41366, 41365, 41364, 41361, 41362, 41343, 41346, 41347, 41345, 41344, - 41341, 41342, 41349, 41350, 41348, 41359, 41360, 41358, 41356, 41357, - 41354, 41355, 41036, 41028, 41029, 41027, 41039, 41038, 41037, 41034, - 41035, 41018, 41021, 41022, 41020, 41019, 41016, 41017, 41025, 41026, - 41023, 41024, 41032, 41033, 41030, 41031, 41042, 41045, 41044, 41043, - 41040, 41041, 41998, 41990, 41991, 41988, 41989, 42001, 42000, 41999, - 41996, 41997, 41982, 41986, 41987, 41984, 41985, 41983, 41980, 41981, - 41994, 41995, 41992, 41993, 42004, 42007, 42006, 42005, 42002, 42003, - 42115, 42107, 42108, 42105, 42106, 42118, 42117, 42116, 42113, 42114, - 42099, 42103, 42104, 42101, 42102, 42100, 42097, 42098, 42111, 42112, - 42109, 42110, 42121, 42124, 42123, 42122, 42119, 42120, 41866, 41855, - 41854, 41869, 41868, 41867, 41864, 41865, 41862, 41863, 41860, 41861, - 41858, 41859, 41856, 41857, 41872, 41875, 41874, 41873, 41870, 41871, - 41853, 41852, 41942, 41931, 41932, 41930, 41945, 41944, 41943, 41940, - 41941, 41938, 41939, 41937, 41928, 41929, 41926, 41927, 41935, 41936, - 41933, 41934, 41948, 41951, 41950, 41949, 41946, 41947, 41772, 41775, - 41776, 41774, 41773, 41770, 41771, 41786, 41787, 41785, 41779, 41780, - 41777, 41778, 41783, 41784, 41781, 41782, 41790, 41791, 41788, 41789, - 41794, 41797, 41796, 41795, 41792, 41793, 41916, 41904, 41905, 41903, - 41919, 41918, 41917, 41914, 41915, 41901, 41902, 41899, 41900, 41912, - 41913, 41910, 41911, 41908, 41909, 41906, 41907, 41922, 41925, 41924, - 41923, 41920, 41921, 41760, 41749, 41750, 41748, 41763, 41762, 41761, - 41758, 41759, 41739, 41742, 41743, 41741, 41740, 41737, 41738, 41756, - 41757, 41755, 41746, 41747, 41744, 41745, 41753, 41754, 41751, 41752, - 41766, 41769, 41768, 41767, 41764, 41765, 41266, 41255, 41256, 41253, - 41254, 41269, 41268, 41267, 41264, 41265, 41244, 41247, 41248, 41246, - 41245, 41242, 41243, 41262, 41263, 41261, 41251, 41252, 41249, 41250, - 41259, 41260, 41257, 41258, 41203, 41202, 41188, 41192, 41193, 41190, - 41191, 41189, 41186, 41187, 41196, 41197, 41194, 41195, 41200, 41201, - 41198, 41199, 41206, 41209, 41208, 41207, 41204, 41205, 41212, 41215, - 41214, 41213, 41210, 41211, 40981, 41605, 41606, 41604, 41611, 41612, - 41610, 41608, 41609, 41607, 41602, 41603, 41600, 41601, 42079, 42083, - 42084, 42081, 42082, 42080, 42077, 42078, 42089, 42090, 42087, 42088, - 42093, 42096, 42095, 42094, 42091, 42092, 42086, 42085, 41815, 41803, - 41804, 41802, 41818, 41817, 41816, 41813, 41814, 41800, 41801, 41798, - 41799, 41811, 41812, 41809, 41810, 41807, 41808, 41805, 41806, 41821, - 41824, 41823, 41822, 41819, 41820, 41636, 41625, 41626, 41624, 41639, - 41638, 41637, 41634, 41635, 41615, 41618, 41619, 41617, 41616, 41613, - 41614, 41696, 41697, 41695, 41693, 41694, 41692, 41682, 41686, 41687, - 41684, 41685, 41683, 41680, 41681, 41690, 41691, 41688, 41689, 41699, - 41702, 41701, 41700, 41698, 41705, 41708, 41707, 41706, 41703, 41704, - 41632, 41633, 41631, 41622, 41623, 41620, 41621, 41629, 41630, 41627, - 41628, 41642, 41645, 41644, 41643, 41640, 41641, 40973, 40974, 40972, - 40970, 40971, 40968, 40969, 40977, 40978, 40975, 40976, 40980, 40979, - 9775, 129664, 8959, 10853, 10632, 10634, 10814, 10852, 10631, 10633, - 10783, 10784, 10785, 10625, 10626, 72262, 72256, 72253, 72252, 72254, - 72251, 72250, 72261, 72255, 72243, 72215, 72214, 72230, 72229, 72220, - 72219, 72242, 72204, 72203, 72207, 72216, 72211, 72221, 72238, 72239, - 72240, 72228, 72227, 72213, 72212, 72218, 72217, 72225, 72224, 72209, - 72208, 72206, 72205, 72223, 72222, 72231, 72232, 72233, 72241, 72210, - 72236, 72226, 72235, 72237, 72234, 72192, 72259, 72258, 72260, 72257, - 72248, 72245, 72246, 72247, 72244, 72249, 72263, 72202, 72199, 72200, - 72198, 72197, 72195, 72194, 72201, 72196, 72193, 129427, 65279, 8204, - 8203, 8205, 11234, 129296, 118593, 118584, 118585, 118591, 118580, - 118589, 118528, 118540, 118531, 118543, 118559, 118529, 118541, 118532, - 118544, 118592, 118573, 118569, 118568, 118581, 118586, 118561, 118582, - 118583, 118566, 118538, 118550, 118555, 118556, 118535, 118547, 118537, - 118549, 118553, 118558, 118534, 118546, 118572, 118562, 118571, 118554, - 118533, 118545, 118587, 118588, 118530, 118542, 118552, 118563, 118539, - 118551, 118557, 118536, 118548, 118579, 118570, 118560, 118567, 118565, - 118564, 118590, 118576, 118577, 118578, 118619, 118639, 118637, 118645, - 118721, 118611, 118635, 118659, 118626, 118623, 118625, 118624, 118622, - 118638, 118612, 118660, 118608, 118609, 118657, 118719, 118695, 118699, - 118698, 118697, 118696, 118722, 118720, 118703, 118708, 118707, 118706, - 118705, 118704, 118610, 118620, 118723, 118616, 118617, 118640, 118672, - 118636, 118658, 118652, 118651, 118649, 118650, 118648, 118646, 118647, - 118644, 118643, 118641, 118642, 118653, 118656, 118654, 118655, 118662, - 118670, 118664, 118666, 118669, 118663, 118665, 118671, 118667, 118668, - 118673, 118614, 118615, 118618, 118685, 118687, 118684, 118683, 118686, - 118681, 118680, 118709, 118713, 118711, 118716, 118717, 118714, 118715, - 118712, 118718, 118710, 118676, 118679, 118690, 118688, 118693, 118694, - 118691, 118692, 118689, 118675, 118677, 118678, 118674, 118701, 118702, - 118700, 118682, 118632, 118631, 118634, 118633, 118628, 118627, 118630, - 118629, 118613, 118621, 118661, 118596, 118597, 118594, 118595, 118598, - 129503, 983264, 983222, 983221, 983223, + 9718, 9716, 9719, 9831, 10209, 10210, 10211, 129995, 9671, 9672, 128922, + 128923, 10192, 9826, 9931, 128071, 128407, 9759, 9663, 9661, 9920, 9921, + 10069, 9872, 9983, 10048, 128174, 11214, 10023, 9785, 128427, 129293, + 9825, 9989, 129984, 11041, 11053, 10710, 11036, 128326, 9945, 128072, + 9756, 9669, 9667, 9665, 117894, 117895, 128928, 11046, 11088, 9725, 9723, + 11048, 11229, 10001, 9649, 11040, 127985, 10068, 9645, 11092, 9659, 9657, + 9655, 128073, 9758, 9988, 65094, 9750, 11051, 11090, 9643, 9786, 9828, + 9633, 128307, 128916, 9635, 128917, 10212, 9713, 9714, 10213, 9634, 9712, + 9715, 9707, 9093, 127779, 127781, 127782, 9788, 127780, 9734, 128382, + 9743, 9186, 10177, 9943, 128070, 9757, 129994, 9653, 9651, 9708, 11055, + 9647, 118278, 11006, 11038, 10163, 128011, 129144, 129152, 129120, + 129136, 129128, 129146, 129154, 129122, 129138, 129130, 129147, 129155, + 129123, 129139, 129131, 129145, 129153, 129121, 129137, 129129, 129149, + 129157, 129125, 129141, 129133, 129148, 129156, 129124, 129140, 129132, + 129150, 129158, 129126, 129142, 129134, 129151, 129159, 129127, 129143, + 129135, 11838, 129344, 127888, 127788, 129695, 127863, 128521, 129725, + 128430, 128732, 128105, 128111, 128098, 128090, 128082, 128097, 128698, + 11825, 8288, 128506, 128543, 129713, 8361, 129717, 128058, 127873, 8768, + 129340, 128295, 9997, 983233, 8999, 129659, 129644, 129643, 129641, + 129639, 129642, 129645, 129640, 129637, 129636, 129634, 129632, 129635, + 129638, 129633, 8891, 129393, 128155, 165, 69292, 69291, 69293, 69256, + 69255, 69254, 69281, 69268, 69259, 69248, 69271, 69289, 69286, 69287, + 69257, 69278, 69277, 69279, 69276, 69280, 69296, 69282, 69251, 69250, + 69266, 69265, 69267, 69253, 69252, 69269, 69274, 69275, 69284, 69285, + 69272, 69258, 69288, 69297, 69263, 69260, 69270, 69262, 69261, 69249, + 69283, 69273, 69264, 42139, 42149, 42173, 42172, 42132, 42147, 42179, + 42174, 42148, 42169, 42150, 42134, 42166, 42135, 42145, 42143, 42137, + 42175, 42157, 42154, 42167, 42165, 42163, 42130, 42182, 42129, 42171, + 42138, 42140, 42136, 42131, 42151, 42164, 42181, 42142, 42156, 42170, + 42176, 42178, 42160, 42133, 42144, 42152, 42159, 42161, 42158, 42141, + 42146, 42177, 42180, 42155, 42162, 42128, 42168, 42153, 41070, 41059, + 41060, 41058, 41073, 41072, 41071, 41068, 41069, 41066, 41067, 41065, + 41048, 41052, 41053, 41050, 41051, 41049, 41046, 41047, 41056, 41057, + 41054, 41055, 41063, 41064, 41061, 41062, 41076, 41077, 41074, 41075, + 41006, 40995, 40996, 40994, 41009, 41008, 41007, 41004, 41005, 41002, + 41003, 41001, 40984, 40988, 40989, 40986, 40987, 40985, 40982, 40983, + 40992, 40993, 40990, 40991, 40999, 41000, 40997, 40998, 41012, 41015, + 41014, 41013, 41010, 41011, 41842, 41831, 41832, 41829, 41830, 41845, + 41844, 41843, 41841, 41827, 41828, 41825, 41826, 41839, 41840, 41837, + 41838, 41835, 41836, 41833, 41834, 41848, 41851, 41850, 41849, 41846, + 41847, 41670, 41659, 41660, 41658, 41673, 41672, 41671, 41668, 41669, + 41666, 41667, 41665, 41648, 41652, 41653, 41650, 41651, 41649, 41646, + 41647, 41656, 41657, 41654, 41655, 41663, 41664, 41661, 41662, 41676, + 41679, 41678, 41677, 41674, 41675, 41293, 41282, 41283, 41281, 41296, + 41295, 41294, 41291, 41292, 41272, 41275, 41276, 41274, 41273, 41270, + 41271, 41289, 41290, 41288, 41279, 41280, 41277, 41278, 41286, 41287, + 41284, 41285, 41238, 41228, 41227, 41241, 41240, 41239, 41236, 41237, + 41218, 41221, 41222, 41220, 41219, 41216, 41217, 41234, 41235, 41233, + 41225, 41226, 41223, 41224, 41231, 41232, 41229, 41230, 41174, 41175, + 41173, 41171, 41172, 41169, 41170, 41167, 41168, 41165, 41166, 41184, + 41185, 41182, 41183, 41178, 41181, 41180, 41179, 41176, 41177, 41494, + 41496, 41497, 41495, 41492, 41493, 41516, 41504, 41505, 41502, 41503, + 41519, 41518, 41517, 41514, 41515, 41500, 41501, 41498, 41499, 41512, + 41513, 41510, 41511, 41508, 41509, 41506, 41507, 41460, 41448, 41449, + 41446, 41447, 41463, 41462, 41461, 41458, 41459, 41436, 41440, 41441, + 41438, 41439, 41437, 41434, 41435, 41444, 41445, 41442, 41443, 41456, + 41457, 41454, 41455, 41452, 41453, 41450, 41451, 41584, 41583, 41582, + 41389, 41379, 41380, 41378, 41392, 41391, 41390, 41387, 41388, 41369, + 41372, 41373, 41371, 41370, 41367, 41368, 41385, 41386, 41384, 41382, + 41383, 41381, 41376, 41377, 41374, 41375, 41395, 41398, 41397, 41396, + 41393, 41394, 41125, 41117, 41118, 41116, 41128, 41127, 41126, 41123, + 41124, 41107, 41110, 41111, 41109, 41108, 41105, 41106, 41114, 41115, + 41112, 41113, 41121, 41122, 41119, 41120, 41130, 41133, 41132, 41131, + 41129, 41336, 41334, 41335, 41333, 41332, 41340, 41338, 41339, 41337, + 41322, 41326, 41327, 41324, 41325, 41323, 41320, 41321, 41330, 41331, + 41328, 41329, 41556, 41557, 41554, 41555, 41563, 41564, 41562, 41544, + 41548, 41549, 41546, 41547, 41545, 41542, 41543, 41552, 41553, 41550, + 41551, 41560, 41561, 41558, 41559, 41591, 41592, 41589, 41590, 41598, + 41599, 41597, 41587, 41588, 41585, 41586, 41595, 41596, 41593, 41594, + 40962, 40960, 983243, 40966, 40967, 40964, 40965, 40963, 40961, 42025, + 42017, 42018, 42016, 42028, 42027, 42026, 42023, 42024, 42010, 42014, + 42015, 42012, 42013, 42011, 42008, 42009, 42021, 42022, 42019, 42020, + 42031, 42032, 42029, 42030, 41970, 41962, 41963, 41960, 41961, 41973, + 41972, 41971, 41968, 41969, 41954, 41958, 41959, 41956, 41957, 41955, + 41952, 41953, 41966, 41967, 41964, 41965, 41976, 41979, 41978, 41977, + 41974, 41975, 41488, 41476, 41477, 41475, 41491, 41490, 41489, 41486, + 41487, 41466, 41469, 41470, 41468, 41467, 41464, 41465, 41473, 41474, + 41471, 41472, 41484, 41485, 41482, 41483, 41480, 41481, 41478, 41479, + 41424, 41413, 41414, 41411, 41412, 41427, 41426, 41425, 41422, 41423, + 41420, 41421, 41419, 41401, 41405, 41406, 41403, 41404, 41402, 41399, + 41400, 41409, 41410, 41407, 41408, 41417, 41418, 41415, 41416, 41430, + 41433, 41432, 41431, 41428, 41429, 41538, 41527, 41528, 41526, 41541, + 41540, 41539, 41536, 41537, 41534, 41535, 41533, 41524, 41525, 41522, + 41523, 41531, 41532, 41529, 41530, 41521, 41520, 41157, 41147, 41148, + 41145, 41146, 41160, 41159, 41158, 41155, 41156, 41136, 41139, 41140, + 41138, 41137, 41134, 41135, 41143, 41144, 41141, 41142, 41151, 41152, + 41149, 41150, 41163, 41164, 41161, 41162, 41154, 41153, 41080, 41083, + 41084, 41082, 41081, 41078, 41079, 41087, 41088, 41085, 41086, 41091, + 41092, 41089, 41090, 41095, 41098, 41097, 41096, 41093, 41094, 41101, + 41104, 41103, 41102, 41099, 41100, 41299, 41302, 41301, 41300, 41297, + 41298, 41312, 41313, 41311, 41305, 41306, 41303, 41304, 41309, 41310, + 41307, 41308, 41316, 41319, 41318, 41317, 41314, 41315, 41574, 41572, + 41573, 41580, 41581, 41579, 41566, 41567, 41565, 41570, 41571, 41568, + 41569, 41577, 41578, 41575, 41576, 42048, 42042, 42041, 42051, 42050, + 42049, 42047, 42035, 42039, 42040, 42037, 42038, 42036, 42033, 42034, + 42045, 42046, 42043, 42044, 42054, 42057, 42056, 42055, 42052, 42053, + 41881, 41882, 41880, 41878, 41879, 41876, 41877, 41885, 41886, 41883, + 41884, 41889, 41892, 41891, 41890, 41887, 41888, 41895, 41898, 41897, + 41896, 41893, 41894, 42075, 42067, 42068, 42066, 42076, 42073, 42074, + 42060, 42064, 42065, 42062, 42063, 42061, 42058, 42059, 42071, 42072, + 42069, 42070, 41727, 41721, 41720, 41730, 41729, 41728, 41726, 41723, + 41722, 41711, 41714, 41715, 41713, 41712, 41709, 41710, 41718, 41719, + 41716, 41717, 41733, 41736, 41735, 41734, 41731, 41732, 41725, 41724, + 41363, 41352, 41353, 41351, 41366, 41365, 41364, 41361, 41362, 41343, + 41346, 41347, 41345, 41344, 41341, 41342, 41349, 41350, 41348, 41359, + 41360, 41358, 41356, 41357, 41354, 41355, 41036, 41028, 41029, 41027, + 41039, 41038, 41037, 41034, 41035, 41018, 41021, 41022, 41020, 41019, + 41016, 41017, 41025, 41026, 41023, 41024, 41032, 41033, 41030, 41031, + 41042, 41045, 41044, 41043, 41040, 41041, 41998, 41990, 41991, 41988, + 41989, 42001, 42000, 41999, 41996, 41997, 41982, 41986, 41987, 41984, + 41985, 41983, 41980, 41981, 41994, 41995, 41992, 41993, 42004, 42007, + 42006, 42005, 42002, 42003, 42115, 42107, 42108, 42105, 42106, 42118, + 42117, 42116, 42113, 42114, 42099, 42103, 42104, 42101, 42102, 42100, + 42097, 42098, 42111, 42112, 42109, 42110, 42121, 42124, 42123, 42122, + 42119, 42120, 41866, 41855, 41854, 41869, 41868, 41867, 41864, 41865, + 41862, 41863, 41860, 41861, 41858, 41859, 41856, 41857, 41872, 41875, + 41874, 41873, 41870, 41871, 41853, 41852, 41942, 41931, 41932, 41930, + 41945, 41944, 41943, 41940, 41941, 41938, 41939, 41937, 41928, 41929, + 41926, 41927, 41935, 41936, 41933, 41934, 41948, 41951, 41950, 41949, + 41946, 41947, 41772, 41775, 41776, 41774, 41773, 41770, 41771, 41786, + 41787, 41785, 41779, 41780, 41777, 41778, 41783, 41784, 41781, 41782, + 41790, 41791, 41788, 41789, 41794, 41797, 41796, 41795, 41792, 41793, + 41916, 41904, 41905, 41903, 41919, 41918, 41917, 41914, 41915, 41901, + 41902, 41899, 41900, 41912, 41913, 41910, 41911, 41908, 41909, 41906, + 41907, 41922, 41925, 41924, 41923, 41920, 41921, 41760, 41749, 41750, + 41748, 41763, 41762, 41761, 41758, 41759, 41739, 41742, 41743, 41741, + 41740, 41737, 41738, 41756, 41757, 41755, 41746, 41747, 41744, 41745, + 41753, 41754, 41751, 41752, 41766, 41769, 41768, 41767, 41764, 41765, + 41266, 41255, 41256, 41253, 41254, 41269, 41268, 41267, 41264, 41265, + 41244, 41247, 41248, 41246, 41245, 41242, 41243, 41262, 41263, 41261, + 41251, 41252, 41249, 41250, 41259, 41260, 41257, 41258, 41203, 41202, + 41188, 41192, 41193, 41190, 41191, 41189, 41186, 41187, 41196, 41197, + 41194, 41195, 41200, 41201, 41198, 41199, 41206, 41209, 41208, 41207, + 41204, 41205, 41212, 41215, 41214, 41213, 41210, 41211, 40981, 41605, + 41606, 41604, 41611, 41612, 41610, 41608, 41609, 41607, 41602, 41603, + 41600, 41601, 42079, 42083, 42084, 42081, 42082, 42080, 42077, 42078, + 42089, 42090, 42087, 42088, 42093, 42096, 42095, 42094, 42091, 42092, + 42086, 42085, 41815, 41803, 41804, 41802, 41818, 41817, 41816, 41813, + 41814, 41800, 41801, 41798, 41799, 41811, 41812, 41809, 41810, 41807, + 41808, 41805, 41806, 41821, 41824, 41823, 41822, 41819, 41820, 41636, + 41625, 41626, 41624, 41639, 41638, 41637, 41634, 41635, 41615, 41618, + 41619, 41617, 41616, 41613, 41614, 41696, 41697, 41695, 41693, 41694, + 41692, 41682, 41686, 41687, 41684, 41685, 41683, 41680, 41681, 41690, + 41691, 41688, 41689, 41699, 41702, 41701, 41700, 41698, 41705, 41708, + 41707, 41706, 41703, 41704, 41632, 41633, 41631, 41622, 41623, 41620, + 41621, 41629, 41630, 41627, 41628, 41642, 41645, 41644, 41643, 41640, + 41641, 40973, 40974, 40972, 40970, 40971, 40968, 40969, 40977, 40978, + 40975, 40976, 40980, 40979, 9775, 129664, 8959, 10853, 10632, 10634, + 10814, 10852, 10631, 10633, 10783, 10784, 10785, 10625, 10626, 72262, + 72256, 72253, 72252, 72254, 72251, 72250, 72261, 72255, 72243, 72215, + 72214, 72230, 72229, 72220, 72219, 72242, 72204, 72203, 72207, 72216, + 72211, 72221, 72238, 72239, 72240, 72228, 72227, 72213, 72212, 72218, + 72217, 72225, 72224, 72209, 72208, 72206, 72205, 72223, 72222, 72231, + 72232, 72233, 72241, 72210, 72236, 72226, 72235, 72237, 72234, 72192, + 72259, 72258, 72260, 72257, 72248, 72245, 72246, 72247, 72244, 72249, + 72263, 72202, 72199, 72200, 72198, 72197, 72195, 72194, 72201, 72196, + 72193, 129427, 65279, 8204, 8203, 8205, 11234, 129296, 118593, 118584, + 118585, 118591, 118580, 118589, 118528, 118540, 118531, 118543, 118559, + 118529, 118541, 118532, 118544, 118592, 118573, 118569, 118568, 118581, + 118586, 118561, 118582, 118583, 118566, 118538, 118550, 118555, 118556, + 118535, 118547, 118537, 118549, 118553, 118558, 118534, 118546, 118572, + 118562, 118571, 118554, 118533, 118545, 118587, 118588, 118530, 118542, + 118552, 118563, 118539, 118551, 118557, 118536, 118548, 118579, 118570, + 118560, 118567, 118565, 118564, 118590, 118576, 118577, 118578, 118619, + 118639, 118637, 118645, 118721, 118611, 118635, 118659, 118626, 118623, + 118625, 118624, 118622, 118638, 118612, 118660, 118608, 118609, 118657, + 118719, 118695, 118699, 118698, 118697, 118696, 118722, 118720, 118703, + 118708, 118707, 118706, 118705, 118704, 118610, 118620, 118723, 118616, + 118617, 118640, 118672, 118636, 118658, 118652, 118651, 118649, 118650, + 118648, 118646, 118647, 118644, 118643, 118641, 118642, 118653, 118656, + 118654, 118655, 118662, 118670, 118664, 118666, 118669, 118663, 118665, + 118671, 118667, 118668, 118673, 118614, 118615, 118618, 118685, 118687, + 118684, 118683, 118686, 118681, 118680, 118709, 118713, 118711, 118716, + 118717, 118714, 118715, 118712, 118718, 118710, 118676, 118679, 118690, + 118688, 118693, 118694, 118691, 118692, 118689, 118675, 118677, 118678, + 118674, 118701, 118702, 118700, 118682, 118632, 118631, 118634, 118633, + 118628, 118627, 118630, 118629, 118613, 118621, 118661, 118596, 118597, + 118594, 118595, 118598, 129503, 983264, 983222, 983221, 983223, }; #define DAWG_CODEPOINT_TO_POS_SHIFT 8 -#define DAWG_CODEPOINT_TO_POS_NOTFOUND 35762 +#define DAWG_CODEPOINT_TO_POS_NOTFOUND 40951 static const unsigned char dawg_codepoint_to_pos_index1[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, @@ -12483,18 +13300,20 @@ static const unsigned char dawg_codepoint_to_pos_index1[] = { 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 52, 52, 52, 52, 52, 52, 52, 52, 52, 112, 113, - 114, 115, 116, 117, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 52, 52, 118, 119, 120, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, + 128, 129, 130, 131, 132, 133, 134, 135, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 136, 52, 52, 52, 52, 52, 52, 137, 138, 139, 140, 52, 141, 142, 143, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 52, 52, 52, 121, 122, 123, 124, 52, 52, 125, 126, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 144, 145, 146, 147, 148, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 127, 128, 129, 130, 131, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 149, 150, 151, 152, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 153, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, + 166, 167, 168, 52, 52, 52, 52, 169, 170, 171, 172, 52, 173, 174, 52, 175, + 176, 177, 52, 52, 178, 179, 180, 52, 181, 182, 183, 184, 185, 186, 187, + 188, 189, 190, 191, 192, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 52, 52, 52, 52, 132, 133, 134, 135, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 136, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 52, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 52, 52, - 52, 52, 149, 150, 151, 152, 52, 153, 52, 52, 154, 155, 156, 52, 52, 157, - 158, 159, 52, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, @@ -12507,9 +13326,9 @@ static const unsigned char dawg_codepoint_to_pos_index1[] = { 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 193, 194, 195, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 172, 173, 174, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, @@ -12664,9 +13483,8 @@ static const unsigned char dawg_codepoint_to_pos_index1[] = { 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 196, 197, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 175, - 176, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, @@ -12679,8 +13497,8 @@ static const unsigned char dawg_codepoint_to_pos_index1[] = { 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 198, 199, 200, 201, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 52, 52, 52, 177, 178, 179, 180, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, @@ -12708,4467 +13526,5003 @@ static const unsigned char dawg_codepoint_to_pos_index1[] = { 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, }; static const unsigned short dawg_codepoint_to_pos_index2[] = { - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 29174, 13037, 26950, 24736, 10779, 26523, 358, 1041, 19550, - 27163, 2511, 26774, 8150, 15996, 13206, 29017, 10563, 10548, 10560, - 10557, 10542, 10539, 10554, 10551, 10566, 10545, 7753, 27749, 19776, - 12475, 13751, 26948, 8149, 18159, 18205, 18215, 18231, 18246, 18291, - 18295, 18312, 18326, 18355, 18360, 18372, 18390, 18399, 18415, 18465, - 18473, 18476, 18496, 18519, 18548, 18587, 18598, 18607, 18610, 18624, - 19518, 27073, 27177, 6518, 20535, 13722, 18716, 18766, 18786, 18809, - 18845, 18904, 18912, 18930, 18949, 18986, 18992, 19006, 19044, 19057, - 19081, 19138, 19149, 19155, 19193, 19234, 19307, 19355, 19369, 19378, - 19386, 19402, 19466, 33652, 27135, 32321, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 24729, - 16349, 5830, 26815, 9647, 34209, 4636, 27731, 10531, 8344, 13111, 19565, - 24711, 28968, 26996, 22169, 10248, 26786, 29740, 29739, 4, 22895, 26671, - 22898, 5826, 29743, 21136, 27214, 33780, 33778, 33786, 16352, 18188, - 18189, 18177, 18191, 18170, 18172, 18193, 18216, 18277, 18278, 18249, - 18264, 18342, 18343, 18332, 18330, 18288, 18413, 18454, 18455, 18422, - 18440, 18429, 23603, 18438, 18579, 18580, 18549, 18552, 18619, 18540, - 19224, 18747, 18748, 18736, 18750, 18727, 18729, 18753, 18787, 18878, - 18879, 18848, 18863, 18967, 18968, 18950, 18952, 18903, 19076, 19120, - 19121, 19082, 19106, 19089, 10690, 19104, 19342, 19345, 19310, 19313, - 19397, 19256, 19388, 18175, 18734, 18160, 18717, 18185, 18744, 18220, - 18792, 18218, 18789, 18224, 18797, 18219, 18791, 18238, 18817, 18235, - 18811, 18267, 18867, 18280, 18881, 18259, 18858, 18274, 18875, 18258, - 18857, 18297, 18914, 18300, 18917, 18304, 18922, 18296, 18913, 18315, - 18933, 18322, 18942, 18336, 18964, 18334, 18959, 18345, 18970, 18339, - 18961, 18329, 18833, 18636, 19422, 18356, 18987, 18361, 18993, 19005, - 18386, 19023, 18382, 19010, 18384, 19013, 18379, 19020, 18381, 19031, - 18408, 19069, 18400, 19058, 18402, 19062, 19077, 18290, 18886, 18418, - 19099, 18457, 19123, 18435, 19095, 18637, 19425, 18485, 19172, 18477, - 19156, 18478, 19158, 18502, 19194, 18501, 19200, 18499, 19198, 18497, - 19196, 18520, 19235, 18523, 19239, 18530, 19251, 18576, 19339, 18568, - 19329, 18582, 19347, 18583, 19338, 18559, 19320, 18573, 19334, 18599, - 19370, 18611, 19387, 18612, 18627, 19403, 18630, 19409, 18626, 19406, - 19036, 18775, 18211, 18206, 18767, 18543, 19259, 18460, 18222, 18795, - 18198, 18243, 18239, 18812, 19264, 18494, 18518, 18459, 18292, 18907, - 18303, 18310, 18948, 18353, 18346, 18371, 19004, 19009, 19032, 18537, - 18406, 19063, 18421, 18447, 19113, 18462, 19135, 18471, 19146, 18713, - 18544, 19260, 18289, 18653, 19247, 18529, 19248, 18528, 18561, 19322, - 18586, 18591, 18616, 19392, 18633, 19412, 18282, 18284, 18896, 18901, - 18706, 18545, 19261, 18652, 18705, 18707, 18708, 18645, 18655, 18245, - 18233, 18839, 18389, 18380, 19043, 18414, 18403, 19080, 18183, 18742, - 18333, 18951, 18428, 19088, 18551, 19312, 18556, 19317, 18555, 19316, - 18553, 19314, 18554, 19315, 19296, 18171, 18728, 18167, 18724, 18195, - 18756, 18305, 18923, 18298, 18915, 18362, 18994, 18436, 19102, 18437, - 19103, 18283, 18898, 18989, 18244, 18232, 18838, 18299, 18916, 18324, - 18606, 18407, 19068, 18173, 18730, 18194, 18755, 18439, 19105, 18169, - 18726, 18190, 18749, 18263, 18862, 18279, 18880, 18327, 18958, 18344, - 18969, 18434, 19094, 18456, 19122, 18482, 19162, 18487, 19173, 18558, - 19319, 18581, 19346, 18500, 19199, 18522, 19237, 18623, 19401, 18316, - 18934, 18404, 18816, 18464, 19137, 18635, 19416, 18166, 18723, 18247, - 18846, 18430, 19090, 18443, 19109, 18431, 19091, 18432, 19092, 18620, - 19398, 19012, 19061, 19238, 18830, 18843, 19154, 18192, 18225, 18798, - 18374, 18524, 19206, 19411, 18306, 18924, 18212, 18585, 18539, 18281, - 18883, 18359, 18991, 18512, 19152, 18489, 19176, 18622, 19396, 19288, - 18758, 19289, 18774, 19126, 18790, 18813, 18821, 19185, 19215, 19219, - 19132, 19180, 19182, 18805, 18831, 18921, 19222, 18682, 18928, 19192, - 19265, 18941, 18947, 18971, 18982, 18660, 19018, 19007, 19026, 19033, - 19291, 19292, 19052, 19065, 19074, 18693, 18778, 18667, 18804, 19148, - 19276, 19279, 19282, 19167, 19168, 19163, 19183, 18669, 18661, 19212, - 18889, 18832, 19232, 18893, 19283, 19250, 19308, 19349, 19362, 19285, - 19300, 19293, 18700, 19415, 19405, 18895, 18897, 18709, 18711, 18650, - 18702, 18648, 18680, 18803, 18683, 18689, 18988, 19298, 18662, 19151, - 18710, 18654, 18840, 18826, 18841, 19303, 19252, 19302, 18910, 19040, - 19041, 18647, 18649, 19266, 19267, 23222, 23223, 23231, 23253, 23280, - 23283, 23178, 23300, 23302, 23151, 23094, 23308, 23005, 23159, 23161, - 23137, 23110, 23157, 23139, 23163, 23310, 23096, 23086, 5759, 23314, - 23140, 23004, 23109, 23135, 23126, 23132, 23131, 23307, 23117, 23038, - 23037, 23309, 23095, 23150, 23148, 4629, 10885, 27334, 25266, 28928, - 10944, 23164, 23087, 23221, 23233, 23260, 23301, 23257, 23101, 23116, - 23144, 23129, 23106, 23316, 23315, 23313, 23311, 23093, 23122, 23134, - 23124, 23127, 23128, 23147, 23146, 23145, 23130, 23156, 23007, 23107, - 23008, 23108, 23166, 23149, 23123, 7961, 7759, 7843, 8125, 8067, 8087, - 7769, 7868, 7865, 7974, 8112, 7894, 7775, 8140, 7893, 7895, 7777, 7977, - 8133, 7780, 8095, 7781, 7962, 7760, 8052, 8108, 8046, 7976, 8049, 8135, - 7899, 8092, 8076, 8091, 8096, 7871, 7867, 8111, 7782, 7845, 8085, 8139, - 7772, 7980, 7776, 7844, 7771, 7978, 8129, 8072, 8066, 7896, 8128, 8116, - 8064, 8115, 8063, 8103, 7979, 8119, 8123, 8147, 8141, 7882, 7963, 7761, - 7971, 7970, 7973, 7972, 7773, 7908, 7892, 8045, 8078, 7975, 7768, 8053, - 8134, 7966, 8099, 8050, 7909, 8146, 8041, 8100, 8098, 8104, 7870, 7765, - 7887, 8114, 7876, 7875, 7879, 7880, 7888, 7877, 7886, 7993, 8001, 8006, - 8014, 8017, 7999, 8031, 8033, 8035, 8020, 8023, 8038, 8039, 13936, 14200, - 13831, 14067, 14018, 14014, 13925, 14182, 35762, 35762, 14257, 14207, - 14208, 14206, 14262, 13939, 35762, 35762, 35762, 35762, 14222, 13952, - 13829, 13805, 13862, 13852, 13876, 35762, 13908, 35762, 13923, 13898, - 14114, 13808, 13935, 13933, 13931, 13853, 13937, 13832, 13929, 13863, - 13932, 13938, 13940, 13941, 13942, 13899, 13928, 13909, 35762, 13911, - 13930, 13914, 13926, 13934, 13927, 13878, 13868, 13919, 14064, 14079, - 14104, 14123, 14134, 14039, 14199, 14197, 14069, 14070, 14201, 14080, - 14194, 14105, 14145, 14202, 14203, 14204, 14205, 14172, 14185, 14186, - 14196, 14192, 14195, 14125, 14183, 14198, 14184, 14147, 14110, 14130, - 14181, 14143, 14171, 13947, 14259, 14218, 14226, 14224, 14225, 14034, - 14035, 13999, 14010, 14066, 14009, 14191, 14012, 14068, 14011, 14146, - 14008, 14189, 8225, 8319, 8203, 8297, 8199, 8293, 8197, 8291, 8195, 8289, - 8224, 8318, 8194, 8288, 13998, 14037, 14015, 14013, 13948, 14016, 14038, - 13913, 14193, 13943, 13912, 14190, 14036, 13945, 13946, 13944, 9913, - 9915, 9862, 9901, 9837, 9866, 9853, 9972, 9985, 9809, 9812, 9826, 9940, - 9910, 9953, 9870, 9838, 9854, 9986, 9895, 9874, 9912, 9979, 9975, 9908, - 9951, 9925, 9876, 9892, 9881, 9944, 9813, 9887, 9890, 9823, 9832, 9894, - 9902, 9829, 9855, 9958, 9956, 9906, 9969, 9961, 9875, 9974, 9966, 9997, - 10013, 10186, 10054, 10033, 10071, 10179, 10175, 10067, 10129, 10084, - 10035, 10051, 10040, 10110, 10115, 10046, 10049, 10147, 10157, 10053, - 10061, 10153, 10014, 10136, 10134, 10065, 10169, 10139, 10034, 10174, - 10166, 10072, 10074, 10021, 10060, 10162, 10025, 10012, 10172, 10185, - 10103, 10109, 10150, 10099, 10069, 10131, 10029, 9945, 10111, 9968, - 10168, 9921, 10080, 9808, 10101, 9917, 10076, 9850, 10009, 9918, 10077, - 9941, 10100, 9816, 10119, 9984, 10184, 9923, 10082, 9924, 10083, 9836, - 10161, 9817, 10124, 9946, 10112, 9948, 10114, 9938, 10097, 10217, 7836, - 7830, 7833, 7831, 7832, 7786, 7839, 9952, 10130, 9965, 10143, 9888, - 10047, 9897, 10056, 9899, 10058, 9896, 10055, 9981, 10181, 9977, 10177, - 9926, 10085, 9928, 10087, 9929, 10088, 9848, 10007, 9883, 10042, 9991, - 10190, 9814, 10116, 9844, 10003, 9891, 10050, 9825, 10149, 9963, 10141, - 9964, 10142, 9903, 10062, 9990, 10189, 9857, 10016, 9858, 10017, 9954, - 10132, 9841, 10000, 9842, 10001, 9994, 9982, 10182, 9927, 10086, 9879, - 10038, 9886, 10045, 9885, 10044, 9939, 10098, 9893, 10052, 10118, 9840, - 9999, 9839, 9998, 9989, 10188, 9914, 10073, 9949, 10127, 9950, 10128, - 9980, 10180, 9976, 10176, 9843, 10002, 9911, 10070, 9909, 10068, 9947, - 10113, 9846, 10005, 9847, 10006, 9889, 10048, 9835, 10160, 9834, 10159, - 9833, 10158, 9856, 10015, 9898, 10057, 9970, 10170, 9900, 10059, 9904, - 10063, 9905, 10064, 9932, 10091, 9931, 10090, 9937, 10096, 9930, 10089, - 9933, 10092, 9934, 10093, 9935, 10094, 9936, 10095, 9821, 10123, 9880, - 10039, 9810, 10104, 9822, 10126, 9967, 10167, 9988, 10187, 9987, 10165, - 9845, 10004, 9877, 10036, 9882, 10041, 9815, 10117, 9955, 10133, 9884, - 10043, 9868, 10027, 9872, 10031, 9878, 10037, 35762, 2429, 2436, 2420, - 2442, 2416, 2440, 2417, 2418, 2407, 2439, 2435, 2432, 2434, 2412, 2424, - 2441, 2422, 2419, 2410, 2437, 2408, 2438, 2427, 2431, 2411, 2426, 2421, - 2415, 2428, 2430, 2406, 2414, 2413, 2409, 2425, 2423, 2443, 2433, 35762, - 35762, 2493, 2405, 2445, 2446, 2444, 2497, 2404, 2466, 2469, 2479, 2457, - 2485, 2453, 2483, 2454, 2455, 2468, 2482, 2478, 2475, 2477, 2449, 2461, - 2484, 2459, 2456, 2447, 2480, 2472, 2481, 2464, 2471, 2448, 2463, 2458, - 2452, 2465, 2470, 2467, 2451, 2450, 2474, 2462, 2460, 2486, 2476, 2487, - 2473, 2496, 2494, 35762, 35762, 27207, 19558, 2495, 35762, 15349, 15352, - 15353, 15360, 15361, 15357, 15364, 15362, 15347, 15359, 15356, 15340, - 15341, 15342, 15350, 15355, 15348, 15337, 15345, 15346, 15343, 15344, - 15339, 15351, 15354, 15358, 15365, 15366, 15338, 15363, 15444, 15460, - 15448, 15446, 15447, 15449, 15462, 15458, 15452, 15454, 15450, 15451, - 15456, 15445, 15463, 15469, 15457, 15466, 15459, 15461, 15467, 15443, - 15442, 15468, 15455, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 15367, 15374, 15417, 15415, 15392, 15423, 15397, 15394, 15409, - 15434, 15383, 15377, 15419, 15389, 15421, 15388, 15395, 15399, 15373, - 15385, 15380, 15387, 15413, 15390, 15407, 15401, 15411, 35762, 35762, - 35762, 35762, 15470, 15438, 15441, 15439, 15465, 15464, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 2249, - 2283, 1081, 2284, 2285, 2248, 2392, 2393, 2259, 2390, 2391, 2582, 1056, - 1063, 2253, 2289, 2282, 2288, 2280, 2281, 2290, 2310, 2296, 2325, 2292, - 2341, 2340, 2276, 1367, 1072, 2380, 2388, 1322, 1275, 1139, 1129, 1554, - 1132, 1570, 1111, 1153, 1497, 1496, 1517, 1301, 1260, 1341, 1178, 1514, - 1419, 1596, 1453, 1466, 1445, 1193, 1475, 1591, 1101, 1247, 1328, 1325, - 1222, 1221, 1220, 2368, 1227, 1410, 1312, 1346, 1359, 1383, 1277, 1549, - 1147, 1561, 1079, 1061, 1091, 1073, 1057, 1087, 2277, 2345, 2102, 1086, - 1085, 2344, 2266, 2103, 2389, 2384, 2383, 2385, 2260, 1074, 2387, 2400, - 2402, 2399, 2398, 2395, 2394, 2397, 2396, 2403, 2401, 2257, 1068, 2382, - 1082, 1205, 1207, 1472, 1144, 1136, 1135, 1297, 1298, 1300, 1535, 1299, - 1523, 1529, 1173, 1505, 1504, 1398, 1509, 1168, 1270, 1268, 1378, 1208, - 1267, 1484, 1491, 1202, 1187, 1179, 1180, 1190, 1199, 1213, 1183, 1186, - 1436, 1422, 1433, 1430, 1423, 1431, 1426, 1309, 1429, 1454, 1457, 1458, - 1448, 1447, 1478, 1104, 1206, 1230, 1228, 1542, 1232, 1405, 1413, 1414, - 1323, 1474, 1317, 1316, 1368, 1314, 1238, 1241, 1369, 1240, 1254, 1239, - 1350, 1348, 1352, 1351, 1392, 1384, 1439, 1393, 1389, 1288, 1486, 1285, - 1278, 1279, 1500, 1558, 1335, 1588, 1534, 1585, 1338, 1557, 1541, 1216, - 1579, 1575, 1551, 1600, 1580, 1562, 1565, 1083, 1152, 2299, 2298, 2301, - 2300, 2327, 2309, 2307, 1071, 2367, 2324, 2323, 2293, 2302, 2339, 2303, - 2343, 2342, 2321, 2304, 2255, 1070, 1069, 2264, 2338, 1182, 1427, 13052, - 13054, 13051, 13050, 13047, 13046, 13049, 13048, 13055, 13053, 1467, - 1194, 1249, 2287, 2286, 1284, 29867, 29941, 29938, 29939, 29935, 29876, - 29863, 29864, 29940, 29937, 29862, 29872, 29871, 29870, 35762, 29860, - 29908, 29904, 29918, 29914, 29915, 29878, 29877, 29879, 29921, 29919, - 29880, 29911, 29912, 29916, 29917, 29909, 29881, 29893, 29920, 29900, - 29907, 29922, 29894, 29898, 29906, 29910, 29899, 29905, 29913, 29895, - 29897, 29896, 29927, 29926, 29925, 29930, 29929, 29928, 29932, 29931, - 29866, 29865, 29875, 29874, 29873, 29869, 29868, 29933, 29947, 29948, - 29934, 29945, 29944, 29943, 29942, 29924, 29923, 29946, 29861, 35762, - 35762, 29901, 29902, 29903, 1159, 1162, 1157, 1158, 1160, 1161, 1155, - 1269, 1266, 1185, 1181, 1424, 1460, 1106, 1102, 1105, 1233, 1231, 1330, - 1326, 1324, 1362, 1361, 1390, 1387, 1388, 1353, 1425, 1428, 1459, 1265, - 1263, 1456, 1420, 1264, 1138, 1137, 1219, 1218, 1217, 1553, 1552, 1564, - 1563, 1261, 1461, 1455, 1313, 31890, 31903, 31899, 31916, 31915, 31894, - 31891, 31879, 31910, 31895, 31884, 31883, 31906, 31893, 31886, 31887, - 31904, 31882, 31912, 31905, 31917, 31898, 31897, 31896, 31908, 31889, - 31892, 31907, 31913, 31902, 31901, 31881, 31909, 31914, 31880, 31888, - 31885, 31911, 31875, 31874, 31921, 31877, 31922, 31920, 31876, 31878, - 31919, 31918, 31923, 31900, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 24597, 24599, - 24596, 24595, 24592, 24591, 24594, 24593, 24600, 24598, 24632, 24619, - 24633, 24618, 24634, 24616, 24615, 24603, 24608, 24621, 24627, 24629, - 24607, 24617, 24602, 24614, 24613, 24628, 24620, 24622, 24624, 24625, - 24610, 24626, 24611, 24609, 24623, 24630, 24631, 24612, 24605, 24604, - 24606, 24584, 24585, 24586, 24582, 24579, 24580, 24581, 24583, 24587, - 24636, 24635, 24638, 24637, 24588, 24639, 24601, 35762, 35762, 24589, - 24590, 24640, 27560, 27547, 27561, 27549, 27552, 27548, 27563, 27551, - 27558, 27567, 27553, 27554, 27564, 27566, 27555, 27550, 27568, 27559, - 27565, 27562, 27556, 27557, 27570, 27571, 27574, 27569, 27575, 27572, - 27594, 27604, 27599, 27593, 27603, 27598, 27592, 27602, 27576, 27600, - 27596, 27606, 27577, 27595, 27605, 27597, 27601, 27573, 35762, 35762, - 27584, 27578, 27580, 27583, 27581, 27585, 27607, 27590, 27588, 27591, - 27587, 27589, 27582, 27586, 27579, 35762, 20924, 20911, 20913, 20912, - 20914, 20923, 20921, 20926, 20906, 20904, 20903, 20915, 20916, 20917, - 20907, 20925, 20918, 20909, 20919, 20920, 20908, 20905, 20922, 20927, - 20910, 20931, 20929, 20928, 35762, 35762, 20930, 35762, 29886, 29891, - 29887, 29889, 29885, 29884, 29888, 29892, 29883, 29882, 29890, 35762, - 35762, 35762, 35762, 35762, 1125, 1115, 1126, 1142, 1124, 1112, 1121, - 1118, 1122, 1120, 1143, 1117, 1128, 1114, 1116, 1127, 1113, 1119, 1123, - 2369, 2370, 2372, 1522, 2265, 2258, 1391, 1262, 1479, 1477, 1327, 2386, - 35762, 2254, 2256, 35762, 35762, 35762, 35762, 35762, 35762, 2311, 2332, - 2331, 2334, 2101, 2349, 1064, 1084, 1156, 1163, 1302, 1476, 1229, 1411, - 1347, 1360, 1576, 1578, 1432, 1550, 1444, 1358, 1184, 1446, 1242, 1473, - 1599, 1103, 1315, 1412, 1154, 1399, 1502, 1421, 1577, 1099, 1097, 1100, - 1400, 1503, 1524, 1485, 1329, 1248, 1098, 1304, 1303, 1349, 1259, 2291, - 2294, 2322, 2317, 2326, 1095, 1094, 2348, 1096, 1093, 2337, 2312, 2308, - 2329, 2328, 2305, 2330, 2313, 2315, 2314, 2316, 2319, 2318, 2295, 2306, - 1067, 2381, 1052, 1050, 1054, 1053, 1051, 1055, 2375, 2377, 2379, 2374, - 2376, 2378, 2252, 2251, 2250, 2320, 1076, 1075, 1088, 1606, 2261, 1605, - 2263, 1065, 1066, 2262, 1060, 2104, 10466, 10454, 10468, 10473, 10389, - 10363, 10364, 10433, 10434, 10412, 10413, 10428, 10430, 10371, 10390, - 10441, 10365, 10372, 10391, 10395, 10366, 10406, 10405, 10384, 10382, - 10418, 10369, 10373, 10403, 10401, 10419, 10425, 10424, 10377, 10376, - 10417, 10427, 10426, 10379, 10378, 10420, 10416, 10436, 10435, 10400, - 10399, 10387, 10411, 10409, 10408, 10423, 10422, 10421, 10432, 10392, - 10393, 10394, 10386, 10486, 10485, 10470, 10467, 10476, 10498, 10499, - 10488, 10489, 10494, 10495, 10482, 10492, 10500, 10477, 10483, 10493, - 10484, 10478, 10472, 10487, 10479, 10514, 10475, 10474, 10358, 10355, - 10481, 10491, 10490, 10440, 10404, 10381, 10438, 10374, 10407, 10439, - 10410, 10429, 10431, 10496, 10497, 10502, 10501, 10509, 10511, 10508, - 10507, 10504, 10503, 10506, 10505, 10512, 10510, 10356, 10471, 10370, - 10397, 10396, 10367, 10415, 10414, 10388, 10437, 10385, 10383, 10402, - 10380, 10375, 10398, 3530, 3599, 3598, 3602, 35762, 3553, 3554, 3567, - 3568, 3565, 3566, 3547, 3549, 35762, 35762, 3589, 3555, 35762, 35762, - 3590, 3556, 3540, 3537, 3581, 3580, 3569, 3579, 3578, 3583, 3582, 3571, - 3562, 3561, 3558, 3557, 3570, 3564, 3563, 3560, 3559, 3572, 35762, 3585, - 3584, 3577, 3576, 3588, 3552, 3541, 35762, 3587, 35762, 35762, 35762, - 3573, 3574, 3575, 3586, 35762, 35762, 3600, 3597, 3604, 3613, 3614, 3611, - 3612, 3607, 3608, 35762, 35762, 3615, 3605, 35762, 35762, 3616, 3606, - 3601, 3538, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 3528, - 35762, 35762, 35762, 35762, 3545, 3544, 35762, 3551, 3548, 3550, 3609, - 3610, 35762, 35762, 3623, 3625, 3622, 3621, 3618, 3617, 3620, 3619, 3626, - 3624, 3543, 3542, 3592, 3591, 3531, 3535, 3534, 3533, 3532, 3536, 3603, - 3627, 3546, 3529, 3596, 35762, 35762, 14491, 14492, 14497, 35762, 14445, - 14446, 14459, 14460, 14457, 14458, 35762, 35762, 35762, 35762, 14478, - 14447, 35762, 35762, 14477, 14448, 14442, 14441, 14439, 14438, 14463, - 14470, 14469, 14472, 14471, 14465, 14454, 14453, 14450, 14449, 14464, - 14456, 14455, 14452, 14451, 14466, 35762, 14474, 14473, 14468, 14467, - 14481, 14483, 14444, 35762, 14462, 14461, 35762, 14482, 14475, 35762, - 14476, 14480, 35762, 35762, 14494, 35762, 14498, 14503, 14504, 14501, - 14502, 35762, 35762, 35762, 35762, 14506, 14499, 35762, 35762, 14505, - 14500, 14496, 35762, 35762, 35762, 14493, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 14440, 14437, 14484, 14443, 35762, 14479, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 14516, 14518, 14515, 14514, - 14511, 14510, 14513, 14512, 14519, 14517, 14507, 14436, 14508, 14520, - 14509, 14495, 14435, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 14331, 14338, 14341, 35762, 14281, 14282, 14300, - 14301, 14298, 14299, 14293, 14295, 14357, 35762, 14328, 14283, 14358, - 35762, 14329, 14284, 14321, 14320, 14317, 14316, 14305, 14315, 14314, - 14319, 14318, 14307, 14290, 14289, 14286, 14285, 14306, 14292, 14291, - 14288, 14287, 14308, 35762, 14323, 14322, 14313, 14312, 14325, 14327, - 14326, 35762, 14303, 14302, 35762, 14297, 14309, 14310, 14311, 14324, - 35762, 35762, 14339, 14337, 14344, 14353, 14354, 14351, 14352, 14347, - 14348, 14342, 35762, 14355, 14345, 14343, 35762, 14356, 14346, 14340, - 35762, 35762, 14371, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 14294, 14296, - 14349, 14350, 35762, 35762, 14367, 14369, 14366, 14365, 14362, 14361, - 14364, 14363, 14370, 14368, 14359, 14360, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 14304, 14336, 14335, 14332, 14334, 14330, 14333, - 35762, 25865, 25864, 25869, 35762, 25814, 25815, 25833, 25834, 25831, - 25832, 25826, 25828, 35762, 35762, 25859, 25816, 35762, 35762, 25860, - 25817, 25853, 25852, 25849, 25848, 25837, 25847, 25846, 25851, 25850, - 25839, 25823, 25822, 25819, 25818, 25838, 25825, 25824, 25821, 25820, - 25840, 35762, 25855, 25854, 25845, 25844, 25857, 25813, 25811, 35762, - 25836, 25835, 35762, 25830, 25841, 25842, 25843, 25856, 35762, 35762, - 25866, 25863, 25870, 25879, 25880, 25877, 25878, 25873, 25874, 35762, - 35762, 25881, 25871, 35762, 35762, 25882, 25872, 25868, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 25867, 25800, 25801, 35762, 35762, - 35762, 35762, 25810, 25809, 35762, 25812, 25827, 25829, 25875, 25876, - 35762, 35762, 25889, 25891, 25888, 25887, 25884, 25883, 25886, 25885, - 25892, 25890, 25808, 25858, 25805, 25804, 25807, 25802, 25803, 25806, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 30505, 30522, 35762, 30473, 30474, 30481, 30482, 30477, 30478, 35762, - 35762, 35762, 30486, 30487, 30475, 35762, 30479, 30480, 30476, 30491, - 35762, 35762, 35762, 30463, 30488, 35762, 30490, 35762, 30464, 30466, - 35762, 35762, 35762, 30462, 30467, 35762, 35762, 35762, 30465, 30461, - 30493, 35762, 35762, 35762, 30492, 30495, 30472, 30471, 30470, 30469, - 30468, 30494, 30483, 30484, 30485, 30489, 35762, 35762, 35762, 35762, - 30792, 30799, 30800, 30795, 30796, 35762, 35762, 35762, 30801, 30802, - 30793, 35762, 30797, 30798, 30794, 30521, 35762, 35762, 30808, 35762, - 35762, 35762, 35762, 35762, 35762, 30397, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 30434, 30436, 30433, 30432, 30429, 30428, 30431, 30430, 30437, 30435, - 30499, 30497, 30498, 30427, 30807, 30806, 30426, 30424, 30396, 30502, - 30500, 35762, 35762, 35762, 35762, 35762, 31755, 31756, 31760, 31763, - 31754, 31718, 31719, 31731, 31732, 31727, 31728, 31722, 31724, 35762, - 31748, 31749, 31720, 35762, 31729, 31730, 31721, 31745, 31744, 31741, - 31740, 31707, 31739, 31738, 31743, 31742, 31709, 31714, 31713, 31698, - 31697, 31708, 31717, 31715, 31701, 31699, 31705, 35762, 31747, 31746, - 31737, 31736, 31751, 31752, 31712, 31711, 31704, 31703, 31702, 31726, - 31733, 31734, 31735, 31750, 35762, 35762, 31761, 31759, 31764, 31775, - 31776, 31771, 31772, 31767, 31768, 35762, 31777, 31778, 31765, 35762, - 31773, 31774, 31766, 31762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 31753, 31689, 35762, 31716, 31700, 31710, 35762, 35762, 31706, - 35762, 35762, 31723, 31725, 31769, 31770, 35762, 35762, 31785, 31787, - 31784, 31783, 31780, 31779, 31782, 31781, 31788, 31786, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 31757, 31696, 31694, 31692, 31690, - 31695, 31693, 31691, 31758, 16808, 16807, 16811, 16816, 16809, 16761, - 16762, 16782, 16783, 16778, 16779, 16773, 16775, 35762, 16799, 16800, - 16763, 35762, 16780, 16781, 16764, 16796, 16795, 16792, 16791, 16756, - 16790, 16789, 16794, 16793, 16758, 16770, 16769, 16766, 16765, 16757, - 16772, 16771, 16768, 16767, 16754, 35762, 16798, 16797, 16788, 16787, - 16803, 16804, 16760, 16759, 16753, 16752, 35762, 16777, 16784, 16785, - 16786, 16802, 35762, 35762, 16812, 16810, 16818, 16829, 16830, 16825, - 16826, 16821, 16822, 35762, 16831, 16832, 16819, 35762, 16827, 16828, - 16820, 16815, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 16805, - 16817, 35762, 35762, 35762, 35762, 35762, 35762, 16755, 16801, 35762, - 16774, 16776, 16823, 16824, 35762, 35762, 16839, 16841, 16838, 16837, - 16834, 16833, 16836, 16835, 16842, 16840, 35762, 16813, 16814, 16806, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 20866, 20868, 20873, 20871, 20823, 20797, 20799, 20843, - 20844, 20839, 20840, 20824, 20826, 35762, 20858, 20859, 20800, 35762, - 20841, 20842, 20801, 20855, 20854, 20851, 20850, 20831, 20812, 20811, - 20853, 20852, 20832, 20820, 20818, 20815, 20814, 20830, 20822, 20821, - 20817, 20816, 20833, 20829, 20857, 20856, 20849, 20848, 20861, 20862, - 20838, 20837, 20836, 20835, 20834, 20828, 20845, 20846, 20847, 20860, - 20819, 20869, 20867, 20872, 20875, 20886, 20887, 20882, 20883, 20878, - 20879, 35762, 20888, 20889, 20876, 35762, 20884, 20885, 20877, 20870, - 20813, 20874, 35762, 35762, 35762, 35762, 20809, 20810, 20804, 20772, - 20789, 20787, 20794, 20784, 20785, 20795, 20788, 20798, 20825, 20827, - 20880, 20881, 35762, 35762, 20780, 20782, 20779, 20778, 20775, 20774, - 20777, 20776, 20783, 20781, 20865, 20863, 20864, 20792, 20791, 20796, - 20786, 20790, 20793, 20773, 20806, 20805, 20807, 20802, 20803, 20808, - 35762, 28828, 28827, 28829, 35762, 28773, 28770, 28758, 28757, 28781, - 28780, 28783, 28782, 28779, 28778, 28777, 28776, 28775, 28774, 28771, - 28802, 28801, 28772, 35762, 35762, 35762, 28767, 28792, 28765, 28790, - 28815, 28805, 28764, 28789, 28766, 28791, 28812, 28813, 28806, 28759, - 28784, 28761, 28786, 28796, 28803, 28760, 28785, 28762, 28787, 28799, - 35762, 28804, 28768, 28793, 28763, 28788, 28794, 28769, 28811, 28809, - 35762, 28798, 35762, 35762, 28810, 28814, 28797, 28800, 28808, 28795, - 28807, 35762, 35762, 35762, 28826, 35762, 35762, 35762, 35762, 28846, - 28838, 28833, 28839, 28834, 28840, 35762, 28835, 35762, 28836, 28841, - 28832, 28845, 28842, 28843, 28844, 28837, 35762, 35762, 35762, 35762, - 35762, 35762, 28822, 28824, 28821, 28820, 28817, 28816, 28819, 28818, - 28825, 28823, 35762, 35762, 28830, 28831, 28847, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 31942, - 31939, 31937, 31936, 31938, 31940, 31956, 31925, 31927, 31926, 31985, - 31928, 31997, 31929, 31994, 31990, 31987, 31988, 31958, 31930, 31993, - 31992, 31989, 31991, 31959, 31924, 31965, 31962, 31931, 31963, 31932, - 31964, 31954, 31998, 31966, 31967, 31944, 31946, 31995, 31983, 31982, - 31984, 31935, 31943, 31999, 31934, 31960, 31968, 31948, 31971, 31973, - 31978, 31979, 31975, 31976, 31974, 31977, 31961, 35762, 35762, 35762, - 35762, 32000, 31980, 31972, 31981, 31970, 31969, 31945, 31953, 31952, - 31951, 31949, 31950, 31947, 31986, 31957, 31996, 31933, 32007, 32009, - 32006, 32005, 32002, 32001, 32004, 32003, 32010, 32008, 31955, 31941, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 18051, 18049, 35762, - 18050, 35762, 18064, 18079, 18083, 18063, 18073, 35762, 18065, 18080, - 18061, 18059, 18058, 18056, 18055, 18060, 18084, 18076, 18074, 18075, - 18057, 18081, 18082, 18069, 18067, 18046, 18068, 18045, 18062, 18085, - 18088, 18053, 35762, 18054, 35762, 18087, 18070, 18071, 18072, 18077, - 18066, 18089, 18078, 18115, 18097, 18102, 18098, 18100, 18110, 18111, - 18104, 18105, 18106, 18107, 18092, 18103, 18091, 18090, 35762, 35762, - 18108, 18109, 18112, 18101, 18099, 35762, 18113, 35762, 18096, 18093, - 18094, 18095, 18126, 18127, 18114, 35762, 18122, 18124, 18121, 18120, - 18117, 18116, 18119, 18118, 18125, 18123, 35762, 35762, 18042, 18041, - 18048, 18047, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 32233, 32140, 32138, 32139, 32148, 32136, - 32133, 32137, 32158, 32128, 32126, 32149, 32164, 32159, 32154, 32162, - 32153, 32157, 32156, 32132, 32141, 32124, 32123, 32051, 32052, 32050, - 32172, 32173, 32174, 32176, 32177, 32175, 32073, 32075, 32072, 32071, - 32068, 32067, 32070, 32069, 32076, 32074, 32065, 32062, 32061, 32058, - 32057, 32060, 32059, 32066, 32064, 32063, 32129, 32151, 32131, 32150, - 32134, 32161, 32142, 32143, 32144, 32145, 32183, 32169, 32082, 32080, - 32110, 32109, 32092, 32108, 32107, 32117, 35762, 32094, 32102, 32101, - 32087, 32086, 32093, 32104, 32103, 32091, 32090, 32095, 32112, 32111, - 32106, 32105, 32119, 32100, 32099, 32089, 32088, 32120, 32113, 32114, - 32115, 32121, 32084, 32118, 32096, 32097, 32098, 32116, 32122, 32079, - 32085, 32081, 32083, 35762, 35762, 35762, 35762, 32257, 32253, 32254, - 32249, 32250, 32245, 32246, 32247, 32248, 32255, 32256, 32251, 32252, - 32180, 32181, 32243, 32244, 32171, 32185, 32146, 32155, 32167, 32182, - 32168, 32170, 32165, 32166, 32184, 32232, 32231, 32230, 32197, 32196, - 32216, 32215, 32198, 32214, 32213, 32223, 35762, 32200, 32208, 32207, - 32190, 32189, 32199, 32210, 32209, 32194, 32193, 32201, 32218, 32217, - 32212, 32211, 32225, 32206, 32205, 32192, 32191, 32227, 32219, 32220, - 32221, 32228, 32226, 32224, 32202, 32203, 32204, 32222, 32229, 32195, - 32187, 32188, 32186, 35762, 32077, 32078, 32053, 32054, 32056, 32055, - 32234, 32241, 32239, 32242, 32240, 32235, 32238, 32237, 32236, 35762, - 32179, 32178, 32127, 32130, 32152, 32147, 32135, 27208, 19559, 27209, - 19560, 32163, 32160, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 23897, 23876, 23875, 23874, 23906, 23974, 23973, 23976, 23975, 23907, - 23904, 23952, 23951, 23958, 23957, 23905, 23936, 23953, 23960, 23959, - 23908, 23978, 23977, 23972, 23971, 23903, 23980, 23910, 23970, 23956, - 23935, 23979, 23969, 23866, 23930, 23967, 23968, 23965, 23966, 23869, - 23902, 23981, 23868, 24063, 24047, 24070, 24071, 24068, 24069, 24059, - 24044, 24052, 24053, 24060, 23988, 24007, 24012, 24011, 23987, 23861, - 23859, 23860, 23858, 23873, 24078, 24080, 24077, 24076, 24073, 24072, - 24075, 24074, 24081, 24079, 24010, 23999, 24017, 24021, 24016, 24020, - 23911, 23934, 23961, 23962, 23963, 23964, 24064, 24065, 24066, 24067, - 23901, 23900, 23898, 23899, 23864, 23863, 23862, 23933, 24054, 24032, - 24033, 23955, 23954, 24061, 24062, 24002, 24003, 24004, 24005, 24006, - 23872, 23871, 23870, 24048, 24050, 24051, 24049, 23925, 23924, 23923, - 23921, 23929, 23913, 23926, 23914, 23916, 23927, 23919, 23917, 23928, - 23865, 24058, 24055, 24056, 24057, 23994, 23995, 23996, 23997, 23992, - 23993, 23991, 23909, 24008, 24028, 24030, 24027, 24026, 24023, 24022, - 24025, 24024, 24031, 24029, 23989, 23990, 24045, 24046, 24019, 24018, - 13332, 13358, 13343, 13364, 13365, 13363, 13353, 13354, 13366, 13347, - 13356, 13359, 13361, 13367, 13349, 13352, 13357, 13351, 13355, 13368, - 13348, 13346, 13342, 13362, 13350, 13338, 13341, 13345, 13340, 13339, - 13360, 13344, 13333, 13337, 13335, 13370, 13334, 13336, 35762, 13369, - 35762, 35762, 35762, 35762, 35762, 13331, 35762, 35762, 13415, 13446, - 13423, 13452, 13421, 13451, 13444, 13441, 13453, 13433, 13435, 13447, - 13449, 13454, 13437, 13443, 13445, 13439, 13442, 13412, 13436, 13432, - 13422, 13450, 13438, 13416, 13419, 13431, 13418, 13417, 13448, 13430, - 13426, 13429, 13427, 13456, 13424, 13428, 13457, 13455, 13420, 13440, - 13414, 13504, 23112, 13413, 13425, 13434, 14693, 14767, 14701, 14772, - 14765, 14729, 14696, 14711, 14769, 14743, 14761, 14675, 14673, 14759, - 14660, 14695, 14779, 14708, 14782, 14703, 14768, 14705, 14704, 14776, - 14739, 14763, 14742, 14688, 14698, 14692, 14721, 14726, 14725, 14712, - 14716, 14714, 14717, 14718, 14715, 14723, 14722, 14724, 14719, 14690, - 14691, 14750, 14756, 14755, 14748, 14753, 14744, 14745, 14747, 14758, - 14752, 14751, 14749, 14754, 14746, 14757, 14665, 14664, 14670, 14669, - 14728, 14685, 14684, 14682, 14677, 14687, 14678, 14771, 14681, 14680, - 14683, 14676, 14780, 14674, 14667, 14663, 14672, 14668, 14661, 14662, - 14666, 14671, 14709, 14689, 14770, 14781, 14694, 14707, 14702, 14706, - 14773, 14784, 15022, 14928, 14938, 14986, 14990, 14940, 14939, 14992, - 14991, 14967, 15019, 15020, 14977, 14998, 14978, 15018, 15017, 15021, - 15007, 14944, 14996, 14951, 14936, 14937, 14988, 14987, 14942, 14943, - 14941, 14994, 14995, 14975, 14974, 14970, 14968, 14976, 14999, 15000, - 15001, 15006, 15005, 14983, 14984, 14982, 14979, 14985, 15012, 15011, - 15010, 15009, 15008, 15016, 15014, 14950, 14947, 14997, 14952, 14954, - 14961, 14965, 14963, 14953, 14929, 14931, 14934, 14933, 14966, 14935, - 14989, 14993, 14972, 14973, 14804, 14905, 14807, 14827, 14830, 14834, - 14909, 14855, 14856, 14861, 14865, 14874, 14877, 14870, 14882, 14814, - 14844, 14847, 14883, 14900, 14796, 14787, 14790, 14813, 14918, 14840, - 14791, 14806, 14808, 14833, 14832, 14836, 14835, 14831, 14916, 14910, - 14858, 14881, 14875, 14876, 14895, 14862, 14864, 14869, 14868, 14859, - 14873, 14871, 14860, 14878, 14824, 14821, 14815, 14820, 14819, 14817, - 14822, 14826, 14803, 14845, 14849, 14854, 14802, 14885, 14893, 14886, - 14889, 14837, 14798, 14799, 14908, 14797, 14919, 14923, 14926, 14842, - 14801, 14794, 14792, 14793, 14795, 14927, 14810, 14811, 14809, 14805, - 14812, 14906, 12703, 12710, 12709, 12704, 12708, 12707, 12705, 12706, - 12930, 12938, 12937, 12931, 12935, 12934, 12932, 12936, 12696, 12702, - 12700, 12697, 12699, 12698, 12701, 12687, 12747, 12755, 12754, 12748, - 12752, 12751, 12749, 12745, 12859, 12866, 12864, 12860, 12862, 12861, - 12865, 12863, 12834, 12843, 12842, 12835, 12839, 12838, 12836, 12840, - 12874, 12880, 12879, 12875, 12849, 12844, 12876, 12878, 12850, 12858, - 12857, 12851, 12855, 12854, 12852, 12856, 12826, 12833, 12832, 12827, - 12831, 12830, 12828, 12829, 12814, 35762, 12818, 12815, 12817, 12816, - 35762, 35762, 12807, 12813, 12811, 12808, 12810, 12809, 12812, 35762, - 12802, 35762, 12806, 12803, 12805, 12804, 35762, 35762, 12537, 12544, - 12543, 12538, 12542, 12541, 12539, 12528, 12999, 13006, 13004, 13000, - 13002, 13001, 13005, 13003, 12912, 12920, 12919, 12913, 12917, 12916, - 12914, 12918, 12575, 12583, 12582, 12576, 12580, 12579, 12577, 12581, - 12967, 12974, 12973, 12968, 12972, 12971, 12969, 12970, 12955, 35762, - 12959, 12956, 12958, 12957, 35762, 35762, 12765, 12773, 12772, 12766, - 12770, 12769, 12767, 12771, 12756, 12764, 12763, 12757, 12761, 12760, - 12758, 12762, 12657, 12665, 12664, 12658, 12662, 12661, 12659, 12663, - 12735, 12742, 12741, 12736, 12740, 12739, 12737, 12738, 12723, 35762, - 12727, 12724, 12726, 12725, 35762, 35762, 12716, 12722, 12720, 12717, - 12719, 12718, 12721, 35762, 12711, 35762, 12715, 12712, 12714, 12713, - 35762, 35762, 12939, 12946, 12945, 12940, 12944, 12943, 12941, 12942, - 12775, 12781, 12779, 12776, 12778, 12777, 12780, 35762, 12990, 12998, - 12997, 12991, 12995, 12994, 12992, 12996, 12975, 12982, 12980, 12976, - 12978, 12977, 12981, 12979, 12947, 12954, 12953, 12948, 12952, 12951, - 12949, 12950, 12605, 12613, 12612, 12606, 12610, 12609, 12607, 12611, - 12590, 12598, 12597, 12591, 12595, 12594, 12592, 12596, 12921, 12929, - 12928, 12922, 12926, 12925, 12923, 12927, 12678, 12626, 12684, 12679, - 12683, 12682, 12680, 12681, 12666, 35762, 12670, 12667, 12669, 12668, - 35762, 35762, 12650, 12656, 12654, 12651, 12653, 12652, 12655, 12646, - 12881, 12889, 12888, 12882, 12886, 12885, 12883, 12887, 12566, 12574, - 12573, 12567, 12571, 12570, 12568, 12572, 12774, 12789, 12788, 12782, - 12786, 12785, 12783, 12787, 12904, 12911, 12909, 12905, 12907, 12906, - 12910, 12908, 12896, 12903, 12902, 12897, 12901, 12900, 12898, 12899, - 12618, 12625, 12623, 12619, 12621, 12620, 12624, 12616, 12794, 12801, - 12800, 12795, 12799, 12798, 12796, 12792, 12841, 12753, 12622, 35762, - 35762, 12506, 12508, 12507, 12525, 13028, 13026, 12509, 12524, 12510, - 12523, 13027, 12522, 13024, 13022, 13021, 13018, 13017, 13020, 13019, - 13025, 13023, 12511, 12514, 12513, 12518, 12517, 12521, 12520, 12516, - 12519, 12515, 12512, 35762, 35762, 35762, 12847, 12746, 12744, 12743, - 12845, 12529, 12527, 12526, 12846, 12617, 12615, 12614, 12848, 12793, - 12791, 12790, 13016, 13007, 13013, 13014, 13009, 13011, 13015, 13010, - 13008, 13012, 35762, 35762, 35762, 35762, 35762, 35762, 6081, 6082, 6083, - 6084, 6085, 6086, 6044, 6080, 6045, 6046, 6047, 6048, 6049, 6009, 6010, - 6011, 6012, 6013, 6014, 6050, 6051, 6052, 6053, 6054, 6055, 6056, 6057, - 6058, 6059, 6060, 6015, 6008, 6016, 6017, 6018, 6019, 6020, 6021, 6062, - 6063, 6064, 6065, 6066, 6067, 6023, 6022, 6024, 6025, 6026, 6027, 6028, - 6002, 6041, 6003, 6042, 6004, 6043, 6005, 6006, 6007, 6001, 6029, 6030, - 6031, 6032, 6033, 6034, 6035, 6036, 6037, 6038, 6039, 6040, 6068, 6069, - 6070, 6071, 6072, 6073, 6074, 6075, 6076, 6077, 6078, 6079, 6061, 35762, - 35762, 6161, 6162, 6163, 6164, 6165, 6147, 35762, 35762, 5223, 5195, - 4968, 5225, 5226, 5375, 5389, 5672, 5179, 5039, 4966, 4967, 5549, 5639, - 5659, 5637, 5660, 5638, 5641, 5635, 5642, 5636, 5304, 5656, 5633, 5657, - 5634, 5305, 4970, 5673, 5691, 5212, 5211, 5213, 5214, 5206, 5207, 5204, - 5203, 5216, 5210, 5215, 5205, 5197, 5227, 5388, 4974, 5409, 5402, 5407, - 5408, 5404, 5405, 5663, 5037, 5038, 5400, 5401, 5399, 5578, 5397, 5576, - 5398, 5577, 5392, 5574, 5393, 5575, 5395, 5572, 5396, 5573, 5662, 5391, - 5571, 5030, 5548, 5531, 5546, 5547, 5544, 5545, 5670, 5001, 5014, 5529, - 5530, 5539, 5631, 5537, 5629, 5538, 5630, 5535, 5627, 5536, 5628, 5533, - 5625, 5534, 5626, 5310, 5491, 5526, 5527, 5528, 5525, 5246, 5240, 5244, - 5245, 5242, 5243, 5665, 5238, 5239, 5237, 5623, 5235, 5621, 5236, 5622, - 5233, 5619, 5234, 5620, 5230, 5617, 5231, 5618, 5307, 5228, 5229, 5471, - 5472, 5473, 5470, 5194, 5181, 5192, 5193, 5190, 5191, 5664, 4998, 5180, - 5188, 5616, 5186, 5614, 5187, 5615, 5184, 5612, 5185, 5613, 5182, 5610, - 5183, 5611, 5306, 4997, 5447, 5272, 5280, 5289, 5290, 5275, 5276, 5667, - 5278, 5279, 5288, 5570, 5286, 5568, 5287, 5569, 5284, 5566, 5285, 5567, - 5282, 5564, 5283, 5565, 5308, 5271, 5563, 5291, 4972, 5448, 5364, 5325, - 5362, 5363, 5352, 5353, 5668, 5296, 5324, 5361, 5589, 5355, 5587, 5356, - 5588, 5309, 5292, 5070, 5365, 5270, 5257, 5268, 5269, 5266, 5267, 5666, - 5255, 5256, 5265, 5557, 5263, 5555, 5264, 5556, 5261, 5553, 5262, 5554, - 5259, 5551, 5260, 5552, 5247, 5550, 5273, 5490, 5450, 5488, 5489, 5469, - 5474, 5669, 5430, 5449, 5483, 5609, 5481, 5607, 5482, 5608, 5479, 5605, - 5480, 5606, 5477, 5603, 5478, 5604, 5302, 5429, 4973, 5476, 4993, 5277, - 5297, 5303, 5300, 5301, 5298, 5299, 5468, 5466, 5467, 5460, 5461, 5463, - 5464, 5459, 5602, 5457, 5600, 5458, 5601, 5452, 5598, 5453, 5599, 5455, - 5596, 5456, 5597, 5451, 5690, 5676, 5688, 5689, 5678, 5679, 5671, 5674, - 5675, 5687, 5586, 5685, 5584, 5686, 5585, 5683, 5582, 5684, 5583, 5681, - 5580, 5682, 5581, 5311, 5661, 4994, 5579, 5446, 5428, 5412, 5562, 5422, - 5426, 5427, 5424, 5425, 5560, 5420, 5421, 5558, 5414, 5591, 5410, 5590, - 5274, 5222, 5201, 5202, 5217, 5219, 5220, 5199, 5200, 5221, 5632, 5198, - 5510, 5295, 5508, 5293, 5509, 5294, 5506, 5507, 5504, 5505, 5502, 5624, - 5492, 5523, 5524, 5520, 5518, 5517, 5541, 5542, 5543, 5540, 5350, 5348, - 5349, 5346, 5347, 5344, 5345, 5343, 5351, 5224, 5369, 5373, 5374, 5371, - 5372, 5367, 5368, 5366, 5515, 5516, 5511, 5514, 5593, 5594, 5595, 5592, - 5330, 5334, 5335, 5332, 5333, 5328, 5329, 5327, 5336, 5444, 5445, 5440, - 5443, 5652, 5653, 5654, 5651, 5643, 5253, 5254, 5251, 5252, 5249, 5250, - 5248, 5500, 5498, 5499, 5496, 5497, 5494, 5495, 5493, 4971, 4990, 4991, - 4992, 4989, 4978, 4979, 4980, 4977, 4986, 4987, 4988, 4985, 4982, 4983, - 4984, 4981, 5435, 5436, 5432, 5434, 5020, 5019, 5015, 5016, 5018, 5017, - 5166, 5165, 5161, 5162, 5164, 5163, 5172, 5171, 5167, 5168, 5170, 5169, - 5036, 5035, 5031, 5032, 5034, 5033, 5130, 5129, 5125, 5126, 5128, 5127, - 5124, 5123, 5119, 5120, 5122, 5121, 5093, 5092, 5088, 5089, 5091, 5090, - 5087, 5029, 5028, 5025, 5026, 5027, 5023, 5066, 5065, 5061, 5062, 5064, - 5063, 5060, 5059, 5055, 5056, 5058, 5057, 5054, 5073, 5072, 5067, 5068, - 5071, 5069, 5160, 5159, 5155, 5156, 5158, 5157, 5178, 5177, 5173, 5174, - 5176, 5175, 5053, 5437, 5052, 5047, 5048, 5051, 5439, 5050, 5046, 5045, - 5041, 5042, 5044, 5043, 5148, 5147, 5143, 5144, 5146, 5145, 5007, 5006, - 5002, 5003, 5005, 5004, 5142, 5141, 5137, 5138, 5140, 5139, 5106, 5105, - 5101, 5102, 5104, 5103, 5112, 5111, 5107, 5108, 5110, 5109, 5100, 5099, - 5095, 5096, 5098, 5097, 5094, 5040, 5013, 5012, 5008, 5009, 5011, 5010, - 5086, 5085, 5081, 5082, 5084, 5083, 5080, 5079, 5075, 5076, 5078, 5077, - 5074, 5136, 5135, 5131, 5132, 5134, 5133, 5154, 5153, 5149, 5150, 5152, - 5151, 5118, 5117, 5113, 5114, 5116, 5115, 5189, 5218, 5370, 5331, 5341, - 5342, 5339, 5340, 5337, 5338, 5650, 5648, 5649, 5646, 5647, 5644, 5645, - 5655, 4976, 25265, 25240, 25251, 25247, 25257, 25254, 25260, 25263, - 25264, 25243, 25242, 25262, 25248, 25253, 25258, 25252, 25239, 25255, - 25261, 25245, 25249, 25244, 25256, 25259, 25250, 25246, 25241, 25237, - 25238, 35762, 35762, 35762, 27467, 27523, 27517, 27521, 27520, 27515, - 27513, 27460, 27445, 27491, 27444, 27443, 27488, 27503, 27490, 27494, - 27495, 27497, 27483, 27449, 27482, 27468, 27461, 27469, 27471, 27516, - 27473, 27472, 27486, 27500, 27512, 27502, 27454, 27476, 27457, 27480, - 27470, 27485, 27506, 27477, 27519, 27446, 27509, 27508, 27504, 27447, - 27525, 27514, 27505, 27452, 27511, 27499, 27455, 27493, 27458, 27518, - 27487, 27501, 27484, 27453, 27475, 27474, 27456, 27492, 27459, 27479, - 27451, 27450, 27448, 27510, 27489, 27507, 27478, 27522, 27524, 27526, - 27527, 27442, 27528, 27529, 27441, 27481, 27498, 27496, 27465, 27464, - 27466, 27463, 27462, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 30050, 30067, 30068, 30058, 30056, 30052, 30064, 30055, 30053, 30061, - 30054, 30060, 30066, 30062, 30059, 30065, 30063, 30057, 30071, 30072, - 30070, 30069, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 30051, 15194, 15195, 15196, 15185, 15183, 15179, 15191, 15182, - 15180, 15188, 15181, 15187, 15193, 15189, 15186, 15192, 15190, 15184, - 15198, 15199, 15197, 26633, 26634, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 4689, 4690, 4691, 4680, 4678, 4674, 4686, - 4677, 4675, 4683, 4676, 4682, 4688, 4684, 4681, 4687, 4685, 4679, 4692, - 4693, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 30086, 30087, 30088, 30078, 30077, 30073, 30083, - 30076, 30074, 30081, 30075, 30080, 30085, 35762, 30079, 30084, 30082, - 35762, 30089, 30090, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 17762, 17760, 17763, 17761, 17764, - 17758, 17756, 17759, 17757, 17766, 17780, 17776, 17781, 17777, 17765, - 17778, 17774, 17779, 17775, 17767, 17788, 17768, 17770, 17769, 17784, - 17787, 17785, 17783, 17786, 17772, 17771, 17773, 17789, 17782, 17790, - 17737, 17735, 17745, 17746, 17741, 17744, 17742, 17743, 17754, 17755, - 17752, 17753, 17747, 17736, 17740, 17739, 17738, 17857, 17856, 17858, - 17863, 17865, 17869, 17871, 17873, 17875, 17874, 17866, 17870, 17864, - 17876, 17860, 17861, 17868, 17862, 17811, 17805, 17810, 17812, 17807, - 17796, 17804, 17806, 17801, 17793, 17794, 17813, 17800, 17795, 17802, - 17797, 17799, 17808, 17798, 17809, 17803, 17734, 17791, 17792, 35762, - 35762, 17883, 17885, 17882, 17881, 17878, 17877, 17880, 17879, 17886, - 17884, 35762, 35762, 35762, 35762, 35762, 35762, 17835, 17834, 17831, - 17833, 17832, 17826, 17829, 17830, 17828, 17827, 35762, 35762, 35762, - 35762, 35762, 35762, 23479, 23491, 23487, 23336, 23486, 23337, 23484, - 23475, 23488, 23489, 23490, 23335, 23334, 23333, 23485, 23332, 23328, - 23330, 23327, 23326, 23323, 23322, 23325, 23324, 23331, 23329, 35762, - 35762, 35762, 35762, 35762, 35762, 23340, 23454, 23471, 23456, 23458, - 23457, 23459, 23455, 23465, 23368, 23460, 23466, 23467, 23463, 23372, - 23453, 23415, 23414, 23445, 23461, 23369, 23464, 23470, 23468, 23469, - 23462, 23451, 23450, 23444, 23448, 23449, 23447, 23452, 23446, 23371, - 23424, 23442, 23443, 23431, 23433, 23432, 23434, 23418, 23435, 23438, - 23439, 23427, 23437, 23426, 23420, 23429, 23422, 23425, 23441, 23440, - 23436, 23428, 23430, 23421, 23423, 23419, 23413, 23398, 23399, 23407, - 23406, 23403, 23411, 23394, 23396, 23412, 23401, 23393, 23408, 23410, - 23409, 23395, 23397, 23392, 23405, 23402, 23400, 23404, 23391, 23389, - 23390, 23388, 23387, 23370, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 23342, 23344, 23346, 23353, 23352, 23360, 23356, 23341, 23351, - 23367, 23354, 23366, 23358, 23357, 23348, 23355, 23359, 23345, 23362, - 23361, 23365, 23363, 23364, 23343, 23417, 23416, 23380, 23385, 23376, - 23381, 23377, 23373, 23378, 23374, 23386, 23375, 23383, 23384, 23350, - 23349, 23379, 23347, 23382, 35762, 35762, 35762, 35762, 35762, 5390, - 4975, 4969, 5658, 5406, 5403, 5394, 5532, 5241, 5232, 5281, 5354, 5326, - 5258, 5475, 5431, 5462, 5465, 5454, 5680, 5677, 5423, 5359, 5379, 5360, - 5380, 5357, 5377, 5358, 5378, 5419, 5417, 5418, 5415, 5416, 5413, 5386, - 5387, 5384, 5383, 5385, 5376, 5381, 5382, 5196, 5640, 5209, 5208, 5411, - 5561, 5559, 5503, 5501, 5522, 5521, 5519, 5513, 5512, 5442, 5441, 5433, - 5022, 4999, 5024, 5021, 5438, 5049, 4995, 4996, 5000, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 19858, 19825, - 19824, 19805, 19804, 19811, 19819, 19818, 19823, 19822, 19810, 19808, - 19806, 19821, 19820, 19812, 19827, 19826, 19817, 19816, 19830, 19809, - 19831, 19829, 19832, 19813, 19814, 19815, 19828, 19803, 19807, 35762, - 19849, 19856, 19857, 19855, 19850, 19853, 19851, 19854, 19852, 19848, - 19846, 19847, 35762, 35762, 35762, 35762, 19840, 19837, 19839, 19845, - 19838, 19843, 19842, 19844, 19841, 19834, 19833, 19835, 35762, 35762, - 35762, 35762, 19836, 35762, 35762, 35762, 19859, 19860, 19867, 19869, - 19866, 19865, 19862, 19861, 19864, 19863, 19870, 19868, 30111, 30123, - 30106, 30103, 30121, 30124, 30105, 30104, 30118, 30113, 30112, 30119, - 30116, 30122, 30117, 30120, 30110, 30102, 30107, 30091, 30125, 30095, - 30096, 30114, 30109, 30108, 30115, 30094, 30092, 30093, 35762, 35762, - 30097, 30098, 30099, 30100, 30101, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 24411, 24433, 24393, 24395, - 24398, 24415, 24417, 24420, 24401, 24397, 24413, 24423, 24419, 24435, - 24402, 24400, 24399, 24424, 24422, 24421, 24404, 24403, 24410, 24426, - 24425, 24432, 24407, 24412, 24409, 24429, 24434, 24431, 24408, 24406, - 24405, 24430, 24428, 24427, 24392, 24394, 24414, 24416, 24396, 24418, - 35762, 35762, 35762, 35762, 24456, 24441, 24445, 24451, 24454, 24457, - 24443, 24447, 24448, 24452, 24444, 24442, 24455, 24450, 24449, 24453, - 24446, 24391, 24386, 24385, 24390, 24389, 24388, 24387, 24438, 24439, - 35762, 35762, 35762, 35762, 35762, 35762, 24464, 24466, 24463, 24462, - 24459, 24458, 24461, 24460, 24467, 24465, 24440, 35762, 35762, 35762, - 24436, 24437, 17836, 17855, 17848, 17851, 17853, 17846, 17844, 17842, - 17838, 17840, 17825, 17823, 17815, 17819, 17821, 17817, 17849, 17854, - 17847, 17850, 17852, 17845, 17843, 17841, 17837, 17839, 17824, 17822, - 17814, 17818, 17820, 17816, 4658, 4655, 4647, 4646, 4660, 4652, 4645, - 4644, 4663, 4654, 4651, 4650, 4653, 4657, 4649, 4648, 4665, 4661, 4659, - 4664, 4662, 4666, 4656, 4669, 4671, 4668, 4670, 4667, 35762, 35762, 4673, - 4672, 30159, 30157, 30158, 30175, 30174, 30173, 30199, 30165, 30164, - 30178, 30185, 30177, 30200, 30193, 30160, 30205, 30176, 30192, 30169, - 30168, 30182, 30181, 30201, 30204, 30167, 30166, 30170, 30180, 30183, - 30179, 30206, 30186, 30172, 30191, 30194, 30187, 30189, 30207, 30161, - 30162, 30163, 30171, 30190, 30208, 30184, 30197, 30198, 30195, 30196, - 30203, 30202, 30188, 30156, 30131, 30130, 30128, 30219, 30126, 30127, - 30129, 30132, 30133, 30134, 35762, 30227, 30234, 30238, 30235, 30244, - 30250, 30251, 30249, 30248, 30246, 30247, 30239, 30240, 30243, 30252, - 30236, 30242, 30237, 30245, 30241, 30218, 30230, 30231, 30211, 30212, - 30213, 30222, 30221, 30214, 35762, 35762, 30135, 30142, 30144, 30141, - 30140, 30137, 30136, 30139, 30138, 30145, 30143, 35762, 35762, 35762, - 35762, 35762, 35762, 30152, 30154, 30151, 30150, 30147, 30146, 30149, - 30148, 30155, 30153, 35762, 35762, 35762, 35762, 35762, 35762, 30228, - 30229, 30226, 30217, 30210, 30232, 30223, 30220, 30215, 30216, 30224, - 30225, 30209, 30233, 35762, 35762, 7898, 7866, 7982, 7900, 8132, 8145, - 8144, 8084, 7881, 8061, 8120, 8090, 7885, 8089, 8088, 8030, 8024, 8047, - 8106, 8048, 8107, 8118, 8077, 7981, 8093, 7884, 7883, 8130, 8008, 8009, - 8010, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 2698, 2697, 2700, 2696, 2699, 2608, 2609, 2625, 2626, 2629, 2630, 2647, - 2648, 2638, 2639, 2622, 2612, 2627, 2628, 2633, 2635, 2623, 2624, 2642, - 2615, 2616, 2631, 2632, 2643, 2654, 2653, 2619, 2618, 2641, 2652, 2655, - 2617, 2620, 2640, 2644, 2645, 2613, 2614, 2660, 2662, 2646, 2637, 2661, - 2650, 2651, 2649, 2659, 2701, 2712, 2715, 2716, 2706, 2707, 2704, 2705, - 2702, 2703, 2708, 2709, 2711, 2710, 2713, 2714, 2717, 2634, 2636, 2656, - 2621, 2657, 2658, 2610, 2611, 35762, 35762, 35762, 2725, 2727, 2724, - 2723, 2720, 2719, 2722, 2721, 2728, 2726, 2694, 2691, 2718, 2605, 2607, - 2606, 2693, 2680, 2678, 2681, 2672, 2673, 2679, 2675, 2677, 2676, 2674, - 2670, 2669, 2665, 2663, 2667, 2666, 2664, 2668, 2671, 2690, 2689, 2688, - 2687, 2682, 2684, 2685, 2686, 2683, 2695, 2692, 35762, 29705, 29703, - 29704, 29656, 29691, 29693, 29658, 29692, 29668, 29669, 29676, 29684, - 29679, 29670, 29677, 29681, 29690, 29671, 29685, 29678, 29672, 29683, - 29661, 29686, 29674, 29682, 29689, 29665, 29663, 29687, 29667, 29688, - 29680, 29651, 29652, 29653, 29711, 29710, 29712, 29709, 29707, 29708, - 29702, 29706, 29654, 29655, 29675, 29666, 29719, 29721, 29718, 29717, - 29714, 29713, 29716, 29715, 29722, 29720, 29650, 29664, 29662, 29673, - 29659, 29660, 3488, 3474, 3482, 3466, 3454, 3478, 3477, 3463, 3469, 3462, - 3455, 3486, 3472, 3464, 3481, 3465, 3483, 3480, 3485, 3470, 3453, 3468, - 3475, 3458, 3476, 3471, 3456, 3487, 3473, 3460, 3484, 3467, 3461, 3479, - 3459, 3457, 3489, 3490, 3497, 3503, 3500, 3504, 3505, 3501, 3506, 3502, - 3498, 3499, 3451, 3452, 3491, 3492, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 3496, 3494, 3495, 3493, 19688, 19687, 19686, 19700, - 19699, 19705, 19715, 19714, 19718, 19706, 19713, 19712, 19694, 19707, - 19691, 19690, 19689, 19698, 19697, 19696, 19695, 19704, 19703, 19709, - 19708, 19693, 19723, 19720, 19719, 19702, 19701, 19721, 19717, 19716, - 19722, 19724, 19733, 19732, 19738, 19740, 19736, 19737, 19734, 19735, - 19739, 19677, 19682, 19681, 19679, 19683, 19684, 19685, 19680, 19678, - 19731, 19730, 35762, 35762, 35762, 19725, 19728, 19729, 19727, 19726, - 19747, 19749, 19746, 19745, 19742, 19741, 19744, 19743, 19750, 19748, - 35762, 35762, 35762, 19711, 19710, 19692, 25311, 25313, 25310, 25309, - 25306, 25305, 25308, 25307, 25314, 25312, 25284, 25275, 25273, 25272, - 25274, 25285, 25269, 25268, 25270, 25271, 25287, 25283, 25281, 25280, - 25282, 25289, 25295, 25296, 25294, 25297, 25286, 25279, 25276, 25278, - 25277, 25288, 25290, 25291, 25293, 25292, 25299, 25300, 25298, 25304, - 25303, 25315, 25302, 25301, 10125, 10102, 10108, 10164, 10146, 10154, - 10144, 10145, 10163, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 13460, 13491, 13468, 13497, 13466, 13496, 13489, 13486, 13498, 13478, - 13480, 13492, 13494, 13499, 13482, 13488, 13490, 13484, 13487, 13500, - 13481, 13477, 13467, 13495, 13483, 13461, 13464, 13476, 13463, 13462, - 13493, 13475, 13471, 13474, 13472, 13502, 13469, 13473, 13503, 13501, - 13465, 13485, 13459, 35762, 35762, 13458, 13470, 13479, 29701, 29699, - 29700, 29698, 29697, 29696, 29695, 29694, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 33625, 33638, 33627, 33605, 33619, 33633, - 33634, 33635, 33620, 33636, 33623, 33631, 33626, 33624, 33632, 33630, - 33629, 33637, 33618, 33616, 33607, 33614, 33606, 33617, 33615, 33596, - 33597, 33599, 33600, 33612, 33610, 33611, 33609, 33598, 33602, 33608, - 33621, 33604, 33613, 33601, 33628, 33622, 33603, 35762, 35762, 35762, - 35762, 35762, 18684, 18685, 19290, 18681, 18686, 18687, 18658, 18657, - 19275, 19268, 18690, 18691, 18664, 18692, 18670, 18665, 18666, 19225, - 19226, 19227, 19270, 18668, 19262, 18784, 18694, 18671, 18679, 18674, - 18697, 19230, 19229, 19228, 18698, 18699, 18701, 18659, 18712, 18646, - 14004, 14007, 14003, 14006, 14002, 9995, 23018, 23019, 23009, 23010, - 23021, 23022, 23012, 23024, 23014, 23025, 23026, 23027, 23028, 23029, - 23030, 23013, 23016, 23017, 23031, 23011, 23033, 23034, 23036, 23167, - 23274, 23168, 23276, 23171, 23196, 23210, 23264, 23249, 23279, 23217, - 23287, 23298, 23227, 23214, 23247, 23250, 23271, 23173, 23251, 23266, - 23291, 23265, 23277, 23295, 23169, 23175, 23218, 23201, 23219, 23195, - 19431, 19439, 19441, 19442, 14213, 14209, 14212, 14211, 14210, 19351, - 18770, 18819, 18905, 19047, 19067, 19144, 19171, 19164, 19209, 19245, - 19413, 19297, 23085, 18976, 19255, 18714, 18983, 19140, 18715, 19350, - 18771, 18823, 18906, 18919, 19002, 19029, 19048, 19073, 19145, 19174, - 19210, 18892, 19358, 19385, 19414, 18733, 18759, 18822, 18882, 19133, - 19181, 19218, 18973, 19129, 18894, 19337, 18900, 23275, 23176, 23194, - 23212, 23258, 23215, 23202, 23263, 23286, 23229, 23230, 23177, 23179, - 23232, 23236, 23238, 23182, 23228, 23278, 23246, 23245, 23188, 23172, - 23252, 23262, 23211, 23267, 23293, 23294, 23190, 23297, 23288, 23207, - 23209, 23208, 23213, 23290, 7874, 7873, 8122, 8121, 8074, 7965, 8073, - 7758, 7964, 7757, 8022, 7770, 8075, 7897, 8086, 8113, 7983, 8137, 8138, - 8005, 7996, 7997, 7998, 8000, 8007, 8003, 8032, 7988, 8034, 8011, 7989, - 7990, 8036, 7991, 7992, 8021, 8025, 8013, 8040, 7995, 8027, 8028, 8026, - 8004, 8012, 8016, 8037, 8002, 8019, 8029, 7994, 8015, 8018, 8136, 7987, - 7986, 7869, 8142, 7872, 7864, 7878, 7767, 8042, 8097, 18174, 18731, - 18210, 18773, 18209, 18772, 18208, 18769, 18217, 18788, 18242, 18825, - 18241, 18824, 18240, 18820, 18236, 18814, 18237, 18815, 18268, 18868, - 18269, 18869, 18257, 18856, 18266, 18866, 18248, 18847, 18293, 18908, - 18301, 18918, 18320, 18938, 18319, 18937, 18317, 18935, 18314, 18932, - 18313, 18931, 18337, 18965, 18331, 18953, 18368, 19000, 18365, 18997, - 18369, 19001, 18376, 19014, 18377, 19015, 18387, 19024, 18383, 19011, - 18391, 19045, 18393, 19050, 18392, 19049, 18411, 19072, 18410, 19071, - 18405, 19064, 18401, 19059, 18442, 19108, 18441, 19107, 18419, 19100, - 18420, 19101, 18470, 19143, 18472, 19147, 18481, 19161, 18479, 19159, - 18480, 19160, 18486, 19166, 18506, 19204, 18504, 19202, 18503, 19195, - 18498, 19197, 18505, 19203, 18526, 19243, 18525, 19242, 18527, 19246, - 18521, 19236, 18557, 19318, 18578, 19341, 18550, 19311, 18577, 19340, - 18569, 19330, 18590, 19361, 18589, 19357, 18603, 19374, 18604, 19375, - 18600, 19371, 18602, 19373, 18601, 19372, 18609, 19380, 18608, 19379, - 18614, 19390, 18625, 19404, 18629, 19408, 18631, 19410, 18939, 19241, - 19376, 19400, 18732, 19038, 19037, 19039, 18515, 18829, 18168, 18725, - 18184, 18743, 18180, 18739, 18179, 18738, 18178, 18737, 18182, 18741, - 18181, 18740, 18163, 18720, 18162, 18719, 18161, 18718, 18165, 18722, - 18164, 18721, 18262, 18861, 18273, 18874, 18265, 18865, 18253, 18852, - 18252, 18851, 18251, 18850, 18256, 18855, 18255, 18854, 18338, 18966, - 18328, 18957, 18433, 19093, 18453, 19119, 18425, 19085, 18424, 19084, - 18423, 19083, 18427, 19087, 18426, 19086, 18450, 19116, 18449, 19115, - 18448, 19114, 18452, 19118, 18451, 19117, 18560, 19321, 18567, 19328, - 18564, 19325, 18563, 19324, 18562, 19323, 18566, 19327, 18565, 19326, - 18615, 19391, 18613, 19389, 18617, 19393, 18621, 19399, 18396, 19053, - 18397, 19054, 18618, 19394, 14051, 14043, 14056, 14048, 14052, 14044, - 14054, 14046, 13817, 13809, 13820, 13812, 13818, 13810, 13822, 13814, - 14074, 14071, 14076, 14073, 14075, 14072, 35762, 35762, 13857, 13854, - 13859, 13856, 13858, 13855, 35762, 35762, 14089, 14081, 14094, 14086, - 14090, 14082, 14092, 14084, 13841, 13833, 13844, 13836, 13842, 13834, - 13846, 13838, 14115, 14106, 14118, 14109, 14117, 14108, 14116, 14107, - 13869, 13864, 13872, 13867, 13871, 13866, 13870, 13865, 14176, 14173, - 14178, 14175, 14177, 14174, 35762, 35762, 13903, 13900, 13905, 13902, - 13904, 13901, 35762, 35762, 14135, 14126, 14138, 14129, 14137, 14128, - 14136, 14127, 35762, 13915, 35762, 13918, 35762, 13917, 35762, 13916, - 14156, 14148, 14161, 14153, 14157, 14149, 14159, 14151, 13887, 13879, - 13890, 13882, 13888, 13880, 13892, 13884, 14041, 14061, 14078, 14077, - 14101, 14099, 14121, 14122, 14180, 14179, 14141, 14142, 14168, 14166, - 35762, 35762, 14058, 14050, 14057, 14049, 14053, 14045, 14055, 14047, - 13824, 13816, 13821, 13813, 13819, 13811, 13823, 13815, 14096, 14088, - 14095, 14087, 14091, 14083, 14093, 14085, 13848, 13840, 13845, 13837, - 13843, 13835, 13847, 13839, 14163, 14155, 14162, 14154, 14158, 14150, - 14160, 14152, 13894, 13886, 13891, 13883, 13889, 13881, 13893, 13885, - 14040, 14065, 14042, 14063, 14062, 35762, 14059, 14060, 13826, 13830, - 13827, 13828, 13825, 14000, 14028, 14029, 14033, 13949, 14102, 14103, - 14100, 35762, 14097, 14098, 13861, 13860, 13851, 13850, 13849, 14032, - 14031, 14030, 14120, 14124, 14113, 14112, 35762, 35762, 14119, 14111, - 13873, 13877, 13874, 13875, 35762, 13957, 13956, 13955, 14140, 14144, - 14133, 14132, 14188, 14187, 14139, 14131, 13920, 13924, 13921, 13922, - 13910, 13951, 13950, 14227, 35762, 35762, 14169, 14170, 14167, 35762, - 14164, 14165, 13907, 13906, 13897, 13896, 13895, 14025, 13954, 35762, - 12447, 12444, 12449, 12446, 32044, 13184, 28853, 13117, 26923, 32017, - 14524, 35568, 35567, 35569, 19570, 27221, 15992, 24652, 13116, 12448, - 12445, 15946, 10941, 10915, 19515, 27171, 28728, 28726, 19475, 27145, - 10914, 10909, 10223, 10908, 4694, 32505, 25769, 32572, 15964, 15995, - 19876, 26269, 19569, 27220, 26798, 19568, 27219, 24257, 26502, 26503, - 26884, 10923, 32519, 27088, 27082, 27096, 5706, 28727, 28729, 27105, - 10945, 16335, 26084, 32621, 5992, 5707, 2513, 15994, 13188, 19523, 27182, - 10946, 26947, 13036, 32418, 27087, 3828, 3867, 20532, 27093, 7742, 32569, - 8148, 29770, 16351, 13152, 32024, 26943, 13177, 13139, 32573, 13178, - 10887, 32530, 33645, 22296, 34177, 13310, 16353, 16355, 16354, 35762, - 19567, 27218, 13132, 26797, 16249, 7, 16248, 6, 24252, 24650, 29741, - 29729, 35762, 35762, 29736, 29735, 29738, 29737, 29728, 29742, 29732, - 29733, 29727, 29731, 29734, 29730, 29615, 29617, 29614, 29613, 29610, - 29609, 29612, 29611, 29604, 29616, 29605, 29606, 29603, 29607, 29608, - 35762, 19428, 19429, 19437, 19443, 19427, 19430, 19433, 19434, 19435, - 19436, 19438, 19426, 19440, 35762, 35762, 35762, 13033, 7755, 8394, - 13194, 20484, 22903, 24185, 26526, 27538, 34181, 24384, 10881, 13034, - 18028, 32539, 11052, 13505, 26527, 14277, 2524, 16001, 5825, 20485, - 29183, 31790, 16239, 32540, 24700, 20985, 27407, 18155, 3742, 29165, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 8051, 8105, 8065, 8117, 7763, 7779, - 8044, 8102, 8110, 7778, 7762, 8124, 7910, 7901, 7904, 7907, 7902, 8054, - 7903, 7905, 7906, 8094, 7891, 7764, 8131, 8143, 8056, 8062, 8109, 8055, - 8043, 8101, 7766, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 0, 100, 10957, 10247, - 5829, 5708, 4959, 13035, 27726, 10249, 27723, 27715, 3917, 10958, 26688, - 26689, 27716, 3918, 27717, 27724, 20657, 10959, 24743, 29081, 27719, - 10955, 10960, 27720, 3919, 10961, 26860, 27066, 27750, 31685, 32492, - 33639, 10962, 26078, 26086, 16350, 3920, 32563, 17143, 951, 27712, 3916, - 12503, 27722, 27713, 27714, 32547, 27718, 27725, 346, 3633, 13508, 10236, - 16246, 27395, 13096, 10969, 10968, 10954, 10956, 10970, 32556, 32557, - 27092, 32558, 10967, 10963, 10964, 10965, 10966, 26888, 32542, 26504, - 2584, 32560, 29846, 33781, 33779, 33783, 33784, 33789, 33777, 33788, - 33787, 33775, 33782, 33774, 33776, 33785, 33773, 33790, 13189, 27365, - 27376, 27377, 27364, 27361, 27370, 27372, 27380, 27381, 27373, 27379, - 27375, 27358, 27366, 27362, 27368, 28915, 28919, 28920, 28914, 28911, - 28923, 28922, 28910, 28924, 28921, 28909, 28918, 28913, 28916, 28912, - 28917, 27369, 27363, 27374, 27378, 19187, 27371, 27360, 27359, 27367, - 33791, 32551, 32550, 35762, 35762, 35762, 35762, 19571, 32718, 27223, - 10989, 19497, 32643, 24682, 24656, 29055, 29069, 19591, 27245, 19654, - 27322, 19651, 32768, 27321, 11022, 19594, 27248, 19602, 32731, 27230, - 11003, 32644, 19600, 27254, 19587, 27241, 19505, 19500, 11026, 32728, - 32729, 10998, 10999, 27237, 10990, 960, 7726, 24684, 19582, 965, 7728, - 19620, 19614, 32745, 32742, 27282, 27276, 11039, 11036, 27256, 32720, - 19604, 19666, 32771, 27288, 11047, 19621, 27271, 19657, 19504, 27264, - 19655, 32735, 27262, 11029, 19502, 32646, 24694, 24668, 29066, 29078, - 19641, 27312, 19670, 27292, 32721, 10991, 19661, 32736, 27268, 11030, - 19581, 27236, 19652, 32772, 27323, 11024, 32773, 32775, 32776, 32777, - 32779, 32780, 27324, 24683, 29058, 32648, 27132, 11001, 32032, 19601, - 27255, 19499, 19585, 27239, 19498, 19665, 27287, 19507, 13172, 8153, - 26414, 32012, 32011, 12437, 16167, 24141, 12386, 24703, 28895, 8161, - 10707, 28888, 12451, 24100, 24088, 24092, 22907, 22914, 10882, 10689, - 27755, 2512, 27333, 4695, 29403, 8400, 13183, 26887, 16240, 27122, 947, - 22170, 29184, 10693, 10708, 26273, 24708, 20493, 20500, 16328, 32623, - 16313, 10912, 32529, 8167, 29763, 33769, 7729, 7720, 962, 32013, 3634, - 26977, 26886, 10883, 13038, 13330, 15982, 32322, 27094, 16347, 28848, - 34185, 24713, 22913, 2517, 24704, 1043, 1046, 24340, 351, 24705, 353, - 32520, 348, 12489, 13328, 10699, 1047, 13329, 1044, 16130, 7754, 12487, - 27331, 27332, 8346, 12504, 12490, 29565, 10252, 12473, 22181, 26949, - 24715, 16006, 24716, 29596, 19770, 13741, 19772, 13744, 19761, 13734, - 23857, 23856, 3632, 24714, 24718, 24717, 24345, 24342, 19769, 13740, - 24344, 24341, 19771, 13742, 24346, 24343, 26861, 29633, 26870, 29642, - 26869, 29641, 10710, 10713, 29621, 29749, 24701, 24702, 29627, 29755, - 24338, 24339, 29626, 29754, 23611, 23612, 23613, 29276, 29365, 29278, - 29367, 29214, 29221, 6503, 6449, 6508, 6241, 6254, 6504, 6230, 6513, - 6255, 29532, 29525, 29546, 29480, 27197, 19537, 10976, 32652, 2518, - 22922, 32536, 13173, 32523, 10939, 10712, 24712, 10715, 24337, 26871, - 29643, 24698, 8162, 24699, 8163, 25799, 16129, 23614, 15761, 16336, - 34206, 24186, 24655, 27128, 27193, 24097, 24098, 24099, 24093, 10529, - 10884, 29567, 10691, 4099, 19549, 27158, 19513, 27169, 27095, 9642, 9641, - 10933, 10932, 10911, 10936, 26682, 12474, 19775, 13750, 33682, 33681, - 19759, 13745, 12472, 12471, 12469, 12470, 10711, 10714, 24709, 24710, - 29277, 29366, 19760, 13733, 26868, 29640, 24706, 10705, 24707, 10706, - 33644, 22899, 32651, 10979, 12387, 12389, 28896, 12392, 12391, 28897, - 12390, 12388, 8164, 8165, 28889, 8166, 28890, 35480, 10530, 12384, 15976, - 32636, 10980, 26885, 26521, 33954, 19471, 27140, 19548, 27149, 4090, - 4087, 32441, 32438, 27085, 29309, 2508, 27736, 27732, 31683, 26806, - 33693, 26685, 32554, 33945, 15974, 32437, 32440, 4086, 4089, 32434, 4080, - 13198, 28954, 32637, 25791, 12500, 34190, 17145, 19564, 27213, 12499, - 3630, 10218, 349, 29856, 32456, 10700, 8172, 28880, 8348, 8349, 991, - 1029, 1015, 1003, 1004, 1020, 1001, 971, 976, 1026, 1031, 1017, 1016, - 1011, 1018, 999, 1023, 1012, 1019, 974, 983, 982, 1005, 1008, 984, 1037, - 1010, 1034, 980, 1009, 1007, 1035, 987, 1006, 1022, 981, 988, 997, 975, - 1036, 1021, 972, 1002, 1033, 978, 1027, 996, 973, 985, 998, 1039, 1038, - 977, 979, 1040, 1028, 1025, 1014, 1013, 986, 1032, 989, 1024, 994, 993, - 1030, 990, 995, 992, 24719, 27127, 27937, 3526, 33656, 16312, 8170, - 10615, 12443, 8152, 34095, 12465, 354, 15472, 6285, 6507, 4635, 32622, - 23495, 15998, 25787, 25788, 26426, 26427, 10610, 28971, 1000, 10244, - 26872, 24569, 26874, 7749, 19552, 19553, 19551, 27165, 27166, 27164, - 19520, 19527, 19519, 27179, 27186, 27178, 19469, 19467, 19468, 9644, - 27138, 27136, 27137, 16323, 15947, 32688, 32704, 29646, 29644, 32443, - 4081, 4082, 26958, 19555, 27199, 15955, 15956, 15957, 15958, 10263, - 10266, 10267, 10256, 10260, 10268, 10257, 10261, 10264, 10255, 10259, - 10254, 10258, 10262, 10265, 29244, 27067, 13061, 33653, 22739, 22731, - 22738, 22732, 22736, 22737, 22735, 22734, 22733, 11219, 13313, 32436, - 4085, 32429, 4084, 32444, 4092, 34104, 3631, 29592, 13145, 8, 12385, - 10245, 3856, 3821, 3900, 3804, 3857, 3822, 3859, 357, 29590, 32330, - 15975, 3840, 3843, 3845, 3837, 10937, 3879, 3751, 26820, 26817, 26818, - 26819, 25220, 29841, 29849, 29850, 29830, 29828, 29831, 29842, 29816, - 29817, 29835, 29837, 29836, 29834, 29818, 29847, 29848, 29820, 29824, - 29823, 29822, 29821, 29839, 29853, 29829, 29819, 29827, 29851, 29832, - 29833, 29844, 29843, 29845, 29854, 29825, 3924, 25774, 29840, 29826, - 29852, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 25232, 25227, 25230, 25231, - 25224, 25225, 25223, 25222, 25229, 25226, 25228, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 6251, 6248, 6247, - 6244, 6243, 6246, 6245, 6252, 6250, 6472, 6452, 6497, 6485, 6467, 6455, - 6471, 6469, 6451, 6498, 6486, 26360, 26358, 26357, 26354, 26353, 26356, - 26355, 26361, 26359, 26352, 26343, 26351, 26349, 26344, 26345, 26347, - 26348, 26342, 26346, 26350, 10550, 10562, 10559, 10544, 10541, 10556, - 10553, 10568, 10547, 24739, 24732, 24740, 24738, 24733, 24734, 24735, - 24737, 24731, 24742, 24741, 26388, 26389, 26390, 26391, 26392, 26393, - 26394, 26395, 26396, 26397, 26398, 26399, 26400, 26401, 26402, 26403, - 26404, 26405, 26406, 26407, 26408, 26409, 26410, 26411, 26412, 26413, - 6395, 6396, 6397, 6398, 6399, 6400, 6401, 6402, 6403, 6404, 6405, 6406, - 6407, 6408, 6409, 6410, 6411, 6412, 6413, 6414, 6415, 6416, 6417, 6418, - 6419, 6420, 6421, 6422, 6423, 6424, 6425, 6426, 6427, 6428, 6429, 6430, - 6431, 6432, 6433, 6434, 6435, 6436, 6437, 6438, 6439, 6440, 6441, 6442, - 6443, 6444, 6445, 6446, 6249, 24266, 24264, 24262, 24267, 24268, 24270, - 24271, 24265, 24269, 24263, 10903, 10901, 10900, 10897, 10896, 10899, - 10898, 10904, 10902, 10895, 24261, 4206, 4153, 4202, 4149, 4196, 4148, - 4195, 4147, 4194, 4146, 4193, 4145, 4190, 4123, 4117, 4144, 4189, 4121, - 4115, 4143, 4201, 4227, 4221, 4138, 4200, 4225, 4219, 4136, 4205, 4243, - 4220, 4116, 4240, 4122, 4226, 4152, 4204, 4242, 4218, 4114, 4239, 4120, - 4224, 4151, 4188, 4157, 4211, 4125, 4119, 4214, 4160, 4142, 4199, 4156, - 4210, 4228, 4222, 4213, 4159, 4137, 4203, 4158, 4212, 4241, 4217, 4118, - 4238, 4162, 4216, 4155, 4209, 4124, 4223, 4215, 4161, 4150, 4186, 4140, - 4185, 4139, 4113, 4109, 4130, 4127, 4105, 4129, 4126, 4104, 4233, 4230, - 4108, 4232, 4229, 4107, 4245, 4236, 4112, 4244, 4235, 4111, 4131, 4128, - 4103, 4234, 4231, 4106, 4246, 4237, 4110, 4164, 4163, 4165, 4166, 4182, - 4181, 4183, 4191, 4197, 4208, 4187, 4132, 4134, 4154, 4141, 4192, 4198, - 4133, 4135, 32667, 20583, 20582, 20585, 20538, 20581, 20586, 20584, - 13205, 19514, 19535, 19546, 19485, 19534, 19495, 19496, 27152, 19790, - 22298, 10230, 32691, 27161, 26940, 26941, 26932, 26933, 26934, 26935, - 26936, 26937, 26938, 26939, 3877, 34082, 34091, 34085, 29418, 29427, - 29417, 29424, 29426, 29416, 3875, 34079, 3871, 34073, 3908, 34115, 3853, - 34063, 3902, 34111, 3901, 34110, 3858, 34070, 3863, 34069, 3862, 34068, - 3806, 34028, 3805, 34027, 3826, 34054, 3825, 34053, 3824, 34052, 3796, - 34017, 34018, 13136, 20588, 34004, 10886, 6224, 4698, 3747, 6216, 6225, - 6217, 6222, 6221, 6223, 19483, 27150, 16343, 16341, 32670, 20536, 32689, - 32706, 20579, 20559, 32672, 20539, 3833, 3832, 3903, 3904, 33970, 29421, - 29428, 29423, 29420, 34094, 34112, 32659, 32660, 18152, 34092, 34088, - 34089, 34093, 34011, 34009, 34010, 34012, 32687, 32711, 20557, 34059, - 3850, 34058, 3849, 20564, 3881, 7743, 32616, 28961, 8155, 3890, 34101, - 19800, 32049, 29647, 2509, 10270, 8173, 25795, 3896, 34103, 2731, 2737, - 2738, 27545, 32618, 15970, 34076, 3888, 27911, 27090, 3831, 3869, 34051, - 34108, 34072, 34026, 28859, 5822, 26956, 3744, 4958, 970, 25895, 6179, - 8381, 8380, 29568, 13106, 102, 14650, 26492, 35478, 32508, 32509, 32514, - 32511, 32513, 32512, 32510, 32507, 33966, 34038, 34080, 3876, 34099, - 13129, 18156, 22729, 13112, 11215, 20893, 16465, 27615, 32781, 24571, - 26787, 2507, 31672, 13327, 5696, 19675, 33695, 20488, 27708, 27543, 5702, - 2585, 26680, 33976, 33992, 33995, 33971, 33979, 33989, 3767, 3783, 3786, - 3762, 3770, 3780, 3889, 34041, 34022, 3795, 34081, 3816, 3801, 34013, - 15971, 26945, 12340, 3512, 3513, 23616, 23615, 23617, 33964, 11220, - 32632, 26984, 26985, 26986, 26987, 26988, 26989, 26990, 26991, 3906, - 26983, 26419, 26524, 33967, 10532, 10533, 10534, 10535, 10536, 10537, - 34006, 34008, 3748, 3750, 23492, 23493, 10572, 10575, 10574, 10573, - 34032, 3812, 14651, 968, 8389, 29561, 27703, 345, 13151, 13323, 29562, - 2520, 13149, 26075, 32026, 32025, 33939, 15829, 10972, 10973, 16327, - 20892, 20891, 20890, 33657, 15965, 22307, 22284, 22297, 21059, 10694, - 32634, 8372, 13311, 24381, 5832, 26228, 16466, 33685, 6182, 3851, 27758, - 27748, 26951, 27752, 28966, 3449, 29494, 34029, 34030, 3809, 3810, 28962, - 29649, 26961, 3883, 32048, 32546, 32545, 34023, 8390, 10614, 25794, - 26665, 5764, 15471, 6237, 5833, 24643, 355, 3897, 34106, 3829, 34049, - 11061, 15328, 19472, 29537, 13100, 3894, 27063, 27064, 2516, 15256, - 26499, 27211, 19562, 16348, 3760, 27916, 6214, 5824, 15934, 13324, 13325, - 20988, 23510, 32617, 13185, 13146, 13114, 27699, 29243, 28857, 16005, - 26511, 31791, 16363, 15232, 13312, 9637, 34033, 3884, 32665, 3887, 20580, - 34074, 34042, 31684, 31671, 226, 12462, 26972, 26966, 33687, 34188, - 20578, 26501, 32705, 34062, 3914, 5998, 15254, 23610, 15290, 2741, 15246, - 26077, 15336, 25778, 15292, 18638, 27761, 26074, 20894, 29566, 13181, - 13179, 15275, 13175, 3813, 34037, 29173, 29594, 6510, 25776, 3761, 26076, - 15294, 26677, 27763, 15245, 25777, 12339, 12336, 12334, 28851, 12335, - 15268, 32567, 28854, 31677, 25775, 15320, 28849, 3811, 34034, 12337, - 6499, 15319, 28964, 32320, 15253, 29172, 15313, 2730, 12338, 15270, 8386, - 27762, 24329, 20577, 32703, 20562, 32708, 3915, 34066, 34031, 3798, - 15272, 19797, 22304, 15335, 15303, 15304, 15264, 15265, 15287, 15286, - 9649, 15273, 15279, 15249, 27392, 13150, 27391, 22291, 22294, 22285, - 22286, 22292, 22295, 15283, 15296, 15282, 15295, 19789, 19787, 22290, - 22293, 10597, 10595, 10594, 10591, 10590, 10593, 10592, 10598, 10596, - 10589, 10608, 10605, 10604, 10601, 10600, 10603, 10602, 10609, 10607, - 10599, 10587, 10584, 10583, 10580, 10579, 10582, 10581, 10588, 10586, - 10578, 15331, 15334, 15291, 15261, 15309, 15297, 15316, 11053, 15300, - 32502, 15322, 10233, 15260, 3864, 32040, 32043, 3865, 15247, 15248, - 29555, 15259, 27216, 19557, 2597, 13196, 15288, 15327, 24721, 9643, - 24723, 6286, 34117, 3921, 3923, 3922, 15250, 15252, 15251, 31678, 15321, - 33960, 15332, 25789, 10905, 32023, 34105, 26505, 25785, 25784, 19512, - 27168, 25896, 27074, 29760, 33643, 21756, 20508, 21577, 29522, 29523, - 34021, 969, 12394, 20576, 32684, 19494, 27159, 13204, 18144, 18143, - 19453, 19452, 19493, 20523, 20512, 32653, 20589, 34014, 34015, 34016, - 34087, 34090, 21757, 21751, 21760, 21754, 21759, 21753, 21758, 21752, - 21761, 21755, 32749, 11043, 964, 7722, 27131, 20513, 20518, 20511, 20515, - 20520, 20510, 20514, 20519, 20516, 20521, 20522, 4624, 4369, 4497, 4370, - 4561, 4434, 4498, 4371, 4593, 4466, 4530, 4403, 4562, 4435, 4499, 4372, - 4609, 4482, 4546, 4419, 4578, 4451, 4515, 4388, 4594, 4467, 4531, 4404, - 4563, 4436, 4500, 4373, 4617, 4490, 4554, 4427, 4586, 4459, 4523, 4396, - 4602, 4475, 4539, 4412, 4571, 4444, 4508, 4381, 4610, 4483, 4547, 4420, - 4579, 4452, 4516, 4389, 4595, 4468, 4532, 4405, 4564, 4437, 4501, 4374, - 4621, 4494, 4558, 4431, 4590, 4463, 4527, 4400, 4606, 4479, 4543, 4416, - 4575, 4448, 4512, 4385, 4614, 4487, 4551, 4424, 4583, 4456, 4520, 4393, - 4599, 4472, 4536, 4409, 4568, 4441, 4505, 4378, 4618, 4491, 4555, 4428, - 4587, 4460, 4524, 4397, 4603, 4476, 4540, 4413, 4572, 4445, 4509, 4382, - 4611, 4484, 4548, 4421, 4580, 4453, 4517, 4390, 4596, 4469, 4533, 4406, - 4565, 4438, 4502, 4375, 4623, 4496, 4560, 4433, 4592, 4465, 4529, 4402, - 4608, 4481, 4545, 4418, 4577, 4450, 4514, 4387, 4616, 4489, 4553, 4426, - 4585, 4458, 4522, 4395, 4601, 4474, 4538, 4411, 4570, 4443, 4507, 4380, - 4620, 4493, 4557, 4430, 4589, 4462, 4526, 4399, 4605, 4478, 4542, 4415, - 4574, 4447, 4511, 4384, 4613, 4486, 4550, 4423, 4582, 4455, 4519, 4392, - 4598, 4471, 4535, 4408, 4567, 4440, 4504, 4377, 4622, 4495, 4559, 4432, - 4591, 4464, 4528, 4401, 4607, 4480, 4544, 4417, 4576, 4449, 4513, 4386, - 4615, 4488, 4552, 4425, 4584, 4457, 4521, 4394, 4600, 4473, 4537, 4410, - 4569, 4442, 4506, 4379, 4619, 4492, 4556, 4429, 4588, 4461, 4525, 4398, - 4604, 4477, 4541, 4414, 4573, 4446, 4510, 4383, 4612, 4485, 4549, 4422, - 4581, 4454, 4518, 4391, 4597, 4470, 4534, 4407, 4566, 4439, 4503, 4376, - 27318, 27317, 19656, 27263, 19503, 27319, 19658, 27265, 11000, 32730, - 32767, 11021, 19660, 27267, 19640, 27311, 27320, 27238, 32732, 11004, - 27250, 27249, 27313, 27315, 27314, 19605, 27257, 19659, 27266, 19583, - 27235, 19603, 27231, 24681, 24661, 24687, 24660, 29059, 29071, 24686, - 24659, 29056, 29070, 27337, 13098, 29057, 24657, 13099, 27338, 24658, - 24685, 33948, 2500, 2499, 2502, 2503, 27217, 19556, 32426, 4083, 32428, - 32427, 20560, 20555, 961, 7719, 27226, 19572, 27924, 27242, 19588, 27234, - 19501, 32770, 19462, 19461, 32641, 32640, 19463, 32642, 19460, 32639, - 19619, 27281, 32744, 11038, 19613, 27275, 32741, 11035, 19618, 27280, - 32743, 11037, 19612, 27274, 32740, 11034, 19615, 32739, 27279, 11032, - 19617, 19611, 27277, 27272, 19616, 19610, 27278, 27273, 32738, 11033, - 27143, 12479, 32324, 19576, 27228, 27227, 19756, 19579, 13729, 29620, - 19578, 29746, 19547, 27148, 32650, 10978, 32522, 35491, 35492, 19539, - 27201, 19541, 27203, 35486, 35482, 35487, 35483, 19524, 27183, 19522, - 27180, 19521, 27181, 19456, 27124, 19457, 27130, 10918, 10943, 19465, - 27134, 10893, 33670, 22179, 27129, 22180, 948, 5, 29185, 29186, 32543, - 27080, 949, 27081, 25218, 25217, 22178, 22177, 22172, 22171, 22176, - 22174, 22175, 22173, 27103, 12441, 12440, 12438, 12439, 6226, 6514, 6501, - 6505, 6502, 6227, 6219, 6228, 32635, 6509, 6232, 6447, 6515, 6218, 6220, - 29486, 29481, 29552, 29538, 29539, 32578, 32501, 32499, 27540, 32498, - 27194, 19529, 33640, 4100, 4101, 3913, 32331, 32332, 34046, 3820, 19544, - 27206, 19473, 27144, 16164, 32258, 16241, 10971, 29429, 16166, 27943, - 12480, 12481, 16007, 13613, 32015, 10983, 10984, 3799, 3834, 34007, 3749, - 12493, 12496, 12492, 12495, 12491, 12494, 27409, 27075, 29018, 27076, - 3737, 3736, 10925, 32518, 19563, 27212, 32333, 22915, 24087, 24085, - 24086, 24095, 24094, 24090, 24091, 32580, 32579, 24089, 23317, 29645, - 26942, 13142, 16322, 16314, 6519, 966, 19872, 19873, 19874, 16315, 26944, - 16319, 16316, 16320, 16318, 16321, 16317, 16463, 18145, 35488, 35489, - 35490, 26779, 26784, 26782, 26778, 26781, 26780, 26783, 22908, 22909, - 22911, 22910, 26775, 26776, 33684, 23609, 23608, 27747, 28939, 23604, - 23605, 6448, 23606, 6242, 26777, 22912, 23607, 16337, 27222, 35484, 362, - 16333, 32627, 32628, 16332, 16331, 32629, 32625, 16330, 32624, 16329, - 32626, 16334, 7735, 7741, 10926, 10927, 7736, 20496, 20504, 10916, 10917, - 32576, 32577, 28879, 28878, 20501, 20498, 20506, 20497, 20505, 20495, - 20499, 20494, 28929, 20503, 20502, 35485, 35481, 12485, 16008, 32516, - 32517, 32326, 32325, 28723, 8174, 12486, 352, 1045, 12476, 26785, 12477, - 10906, 32571, 32034, 12484, 12488, 19773, 13748, 19774, 13749, 19765, - 13735, 19768, 13738, 19766, 13736, 19767, 13737, 19764, 13739, 19758, - 13731, 19757, 13730, 19755, 13727, 19753, 13725, 19752, 13724, 19751, - 13728, 19754, 13726, 28864, 28862, 28865, 28863, 10953, 10952, 10949, - 10948, 28722, 28721, 28720, 28719, 10919, 10921, 10920, 13743, 13732, - 19762, 13746, 19763, 13747, 28937, 18153, 28938, 18154, 12482, 26864, - 29636, 26865, 29637, 26867, 29639, 26863, 29635, 26866, 29638, 26862, - 29634, 10922, 10931, 29631, 29759, 29629, 29757, 29630, 29758, 29628, - 29756, 29623, 29751, 29624, 29752, 29622, 29750, 29625, 29753, 29311, - 29398, 7730, 7732, 7731, 7733, 29618, 29745, 29619, 29744, 29748, 29747, - 12393, 26683, 32496, 13170, 24654, 27928, 27929, 27925, 26506, 33642, - 10940, 33641, 10938, 20507, 27930, 27927, 27926, 10907, 10935, 10929, - 27084, 10709, 33655, 33654, 10977, 26272, 26271, 32521, 32524, 32515, - 32528, 32527, 10951, 10950, 32525, 18142, 10934, 34113, 24096, 24670, - 24696, 29068, 29080, 19506, 19609, 32734, 11028, 24667, 24693, 29065, - 29077, 19508, 32645, 27246, 27247, 19592, 19593, 29422, 29415, 29425, - 29419, 10525, 10526, 10524, 10523, 10889, 3836, 34047, 3911, 34116, 3854, - 34064, 34044, 3817, 15945, 3835, 3839, 34056, 3842, 34060, 3872, 3873, - 34077, 3819, 34045, 3907, 34114, 19459, 32027, 19458, 20517, 19648, - 19647, 19649, 19650, 19584, 19596, 19595, 19643, 19645, 19644, 19580, - 33947, 12478, 27077, 19573, 27233, 27232, 19674, 27327, 27078, 27224, - 32323, 19575, 19574, 27225, 11017, 27923, 27921, 34057, 3874, 34078, - 3861, 34067, 15280, 15293, 15257, 15255, 15258, 28866, 2595, 28867, 2594, - 3628, 27922, 19625, 32753, 27296, 11006, 19510, 32649, 24691, 24665, - 29063, 29075, 19637, 32764, 27308, 11018, 7727, 959, 19636, 32755, 27307, - 11008, 35762, 35762, 24692, 24666, 29064, 29076, 19627, 32763, 27298, - 11016, 15962, 33664, 19626, 32754, 27297, 11007, 19638, 32765, 27309, - 11019, 19608, 32733, 27260, 11027, 956, 957, 955, 958, 27068, 27069, - 24566, 24567, 13176, 27261, 35762, 29855, 32038, 32042, 32039, 32041, - 3827, 3905, 3866, 3807, 11010, 11011, 32757, 32758, 19630, 27301, 19629, - 27300, 3752, 3753, 3754, 3755, 3756, 3758, 3757, 3759, 27115, 27116, - 27113, 27114, 27110, 27112, 27109, 27111, 32774, 32638, 26082, 26081, - 26083, 2736, 6517, 6231, 3878, 3797, 32544, 15944, 3912, 3846, 3838, - 3841, 3844, 24572, 32430, 4077, 19785, 27393, 34036, 27394, 29383, 32620, - 14275, 26791, 26790, 26789, 26788, 32495, 26891, 2515, 15999, 26664, - 24349, 34061, 3800, 32537, 9639, 15230, 35570, 18035, 1042, 97, 33772, - 26801, 19484, 27151, 29569, 29570, 19646, 32769, 27316, 11023, 12498, - 12497, 27759, 27535, 27533, 27534, 27532, 27536, 27537, 12483, 32631, - 27757, 10974, 26424, 27091, 15473, 13513, 13515, 13549, 13523, 13519, - 13550, 13557, 13520, 13556, 13529, 13525, 13524, 13518, 13560, 13531, - 13532, 13533, 13534, 13536, 13538, 13542, 13546, 13559, 13521, 13558, - 13535, 13537, 13539, 13548, 13517, 13541, 13552, 13551, 13553, 13543, - 13555, 13544, 13545, 13554, 13527, 13514, 13526, 13522, 13528, 13540, - 13547, 13530, 13516, 13561, 13563, 13597, 13571, 13567, 13598, 13605, - 13568, 13604, 13577, 13573, 13572, 13566, 13608, 13579, 13580, 13581, - 13582, 13584, 13586, 13590, 13594, 13607, 13569, 13606, 13583, 13585, - 13587, 13596, 13565, 13589, 13600, 13599, 13601, 13591, 13603, 13592, - 13593, 13602, 13575, 13562, 13574, 13570, 13576, 13588, 13595, 13578, - 13564, 18375, 19017, 18378, 18467, 18483, 18751, 19240, 18318, 18936, - 18364, 18996, 18628, 19407, 18197, 18395, 18532, 18533, 19360, 18605, - 19377, 19359, 18323, 18943, 19305, 18864, 19281, 19097, 18675, 19432, - 23035, 18508, 18632, 8182, 8276, 8232, 8326, 8196, 8290, 8193, 8287, - 8237, 8331, 8227, 8321, 8231, 8325, 8198, 8292, 8229, 8323, 8235, 8329, - 8201, 8295, 8206, 8300, 8238, 8332, 8239, 8333, 8202, 8296, 8207, 8301, - 8234, 8328, 8236, 8330, 8228, 8322, 8230, 8324, 8240, 8334, 8204, 8298, - 8200, 8294, 8233, 8327, 8223, 8317, 8190, 8284, 8218, 8312, 8186, 8280, - 8191, 8285, 8192, 8286, 8187, 8281, 8216, 8310, 8226, 8320, 8188, 8282, - 8214, 8308, 8217, 8311, 8181, 8275, 8189, 8283, 8209, 8303, 8210, 8304, - 8205, 8299, 8212, 8306, 8211, 8305, 8208, 8302, 8215, 8309, 8213, 8307, - 8221, 8315, 8219, 8313, 8220, 8314, 8222, 8316, 8336, 8337, 8338, 8340, - 8341, 8335, 8339, 8184, 8278, 8185, 8279, 8180, 8179, 8178, 8183, 8277, - 35762, 35762, 35762, 35762, 35762, 8274, 8271, 8272, 8273, 8269, 8270, - 8342, 13372, 13398, 13383, 13404, 13405, 13403, 13393, 13394, 13406, - 13387, 13396, 13399, 13401, 13407, 13389, 13392, 13397, 13391, 13395, - 13408, 13388, 13386, 13382, 13402, 13390, 13378, 13381, 13385, 13380, - 13379, 13400, 13384, 13373, 13377, 13375, 13410, 13374, 13376, 35762, - 13409, 35762, 35762, 35762, 35762, 35762, 13371, 35762, 35762, 32273, - 32284, 32285, 32278, 32280, 32263, 32302, 32274, 32277, 32275, 32276, - 32312, 32301, 32281, 32267, 32283, 32286, 32262, 32271, 32287, 32300, - 32282, 32268, 32307, 32272, 32313, 32295, 32261, 32269, 32303, 32304, - 32305, 32266, 32270, 32306, 32315, 32297, 32298, 32279, 32265, 32260, - 32288, 32290, 32289, 32291, 32292, 32299, 32293, 32308, 32309, 32310, - 32294, 32264, 32296, 32311, 32314, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 32316, 32317, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 32259, 12933, - 12750, 12837, 12877, 12853, 12540, 12915, 12578, 12768, 12759, 12660, - 12993, 12608, 12593, 12924, 12884, 12569, 12784, 12797, 12645, 12649, - 12648, 12647, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 12867, 12873, 12871, 12868, 12870, 12869, 12872, 35762, 12559, - 12565, 12563, 12560, 12562, 12561, 12564, 35762, 12983, 12989, 12987, - 12984, 12986, 12985, 12988, 35762, 12552, 12558, 12556, 12553, 12555, - 12554, 12557, 35762, 12819, 12825, 12823, 12820, 12822, 12821, 12824, - 35762, 12728, 12734, 12732, 12729, 12731, 12730, 12733, 35762, 12960, - 12966, 12964, 12961, 12963, 12962, 12965, 35762, 12671, 12677, 12675, - 12672, 12674, 12673, 12676, 35762, 7788, 7826, 7823, 7790, 7820, 7821, - 7827, 7794, 7795, 7796, 7803, 7825, 7797, 7791, 7819, 7816, 7818, 7822, - 7806, 7805, 7824, 7792, 7828, 7802, 7789, 7815, 7811, 7813, 7800, 7814, - 7787, 7799, 27126, 27125, 19528, 27187, 19477, 27141, 26965, 26964, - 10892, 19531, 27192, 26974, 19511, 27167, 11222, 26270, 13169, 27086, - 15997, 10891, 11005, 32717, 10894, 10942, 16344, 26229, 15993, 32329, - 19492, 27157, 32328, 32327, 19554, 27198, 32439, 32442, 4088, 4091, - 19516, 27172, 19476, 27146, 32574, 25768, 29482, 13140, 27101, 33667, - 27330, 34176, 32548, 26963, 26973, 32559, 10224, 10225, 32549, 32433, - 32584, 32045, 29582, 33669, 34159, 5701, 10910, 27100, 10913, 10231, - 10930, 16345, 16346, 20533, 20534, 10928, 10888, 32526, 22281, 26268, - 26922, 8345, 8383, 8382, 32417, 22280, 22282, 19526, 27185, 19525, 27184, - 32431, 32432, 4078, 4079, 25219, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 7615, - 7568, 7618, 7617, 7616, 7611, 7539, 7636, 7650, 7649, 7572, 7619, 7632, - 7631, 7601, 7600, 7599, 7598, 7628, 7637, 7627, 7626, 7587, 7586, 7590, - 7614, 35762, 7571, 7634, 7608, 7573, 7606, 7566, 7642, 7641, 7580, 7610, - 7609, 7620, 7570, 7574, 7595, 7537, 7579, 7630, 7629, 7542, 7625, 7553, - 7648, 7647, 7646, 7645, 7603, 7633, 7613, 7578, 7651, 7541, 7540, 7602, - 7607, 7584, 7583, 7582, 7638, 7569, 7644, 7643, 7557, 7621, 7589, 7556, - 7555, 7581, 7565, 7622, 7640, 7639, 7567, 7549, 7597, 7596, 7552, 7550, - 7605, 7604, 7612, 7543, 7559, 7551, 7561, 7547, 7577, 7576, 7575, 7545, - 7588, 7564, 7538, 7585, 7546, 7563, 7554, 7623, 7624, 7548, 7594, 7544, - 7592, 7560, 7593, 7562, 7635, 7591, 7558, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 16662, 16644, - 16578, 16698, 16686, 16630, 16730, 16645, 16658, 16641, 16593, 16597, - 16582, 16567, 16633, 16719, 16665, 16636, 16670, 16747, 16704, 16675, - 16627, 16732, 16576, 16687, 16563, 16667, 16538, 16656, 16592, 16590, - 16685, 16613, 16615, 16595, 16545, 16744, 16570, 16678, 16632, 16714, - 16637, 16565, 16703, 16654, 16676, 16745, 16663, 16728, 16588, 16692, - 16579, 16647, 16731, 16694, 16553, 16712, 16554, 16706, 16624, 16621, - 16584, 16622, 16555, 16673, 16684, 16577, 16540, 16715, 16660, 16717, - 16683, 16657, 16727, 16638, 16707, 16572, 16738, 16583, 16566, 16612, - 16561, 16705, 16737, 16604, 16562, 16599, 16581, 16620, 16699, 16601, - 16569, 16585, 16668, 16634, 16648, 16722, 16711, 16643, 16749, 16602, - 16549, 16695, 16580, 16740, 16716, 16575, 16598, 16700, 16536, 16709, - 16702, 16726, 16616, 16560, 16710, 16541, 16677, 16696, 16635, 16661, - 16691, 16610, 16666, 16539, 16669, 16589, 16556, 16649, 16650, 16688, - 16537, 16652, 16723, 16664, 16550, 16708, 16568, 16617, 16721, 16631, - 16546, 16736, 16564, 16739, 16689, 16629, 16701, 16733, 16557, 16671, - 16542, 16690, 16680, 16679, 16611, 16552, 16559, 16543, 16653, 16735, - 16571, 16743, 16574, 16734, 16614, 16646, 16619, 16655, 16697, 16693, - 16672, 16548, 16746, 16600, 16639, 16718, 16642, 16713, 16640, 16742, - 16607, 16591, 16625, 16608, 16628, 16551, 16720, 16623, 16603, 16681, - 16558, 16618, 16605, 16544, 16682, 16573, 16741, 16626, 16748, 16651, - 16547, 16596, 16609, 16725, 16587, 16674, 16659, 16594, 16724, 16586, - 16729, 16606, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 16031, 16029, - 16030, 16028, 16043, 16039, 16038, 16035, 16036, 16037, 16033, 16032, - 16040, 16034, 16044, 16042, 16128, 16027, 16126, 10698, 16362, 16127, - 16026, 16046, 19455, 27123, 19474, 27142, 19470, 27139, 19538, 27200, - 19464, 27133, 26803, 13506, 19536, 27196, 19540, 27202, 19543, 27205, - 19542, 27204, 33946, 27083, 10924, 20531, 26804, 15127, 15120, 15117, - 15123, 15122, 15125, 15124, 15128, 15126, 16124, 16123, 16045, 16122, - 14786, 14785, 33952, 33647, 33650, 33648, 33651, 33649, 6500, 16120, - 15121, 15119, 15118, 33646, 21138, 26422, 16125, 16121, 35762, 15849, - 15835, 15851, 15929, 15853, 15931, 15850, 15928, 15852, 15930, 15890, - 15880, 15892, 15882, 15894, 15884, 15891, 15881, 15893, 15883, 15854, - 15915, 15856, 15917, 15858, 15919, 15855, 15916, 15857, 15918, 15910, - 15875, 15912, 15877, 15848, 15914, 15879, 15911, 15876, 15913, 15878, - 15870, 15872, 15874, 15871, 15873, 15885, 15865, 15900, 15887, 15859, - 15902, 15889, 15868, 15904, 15886, 15866, 15901, 15888, 15867, 15903, - 15895, 15897, 15899, 15896, 15898, 15842, 15924, 15844, 15926, 15843, - 15925, 15905, 15907, 15909, 15906, 15908, 15838, 15920, 15922, 15921, - 15923, 15869, 15927, 15845, 15846, 35762, 35762, 7985, 7984, 16987, - 16986, 15933, 15932, 15834, 16984, 16914, 16843, 16916, 16977, 16918, - 16979, 16915, 16976, 16917, 16978, 16939, 16929, 16941, 16931, 16943, - 16933, 16940, 16930, 16942, 16932, 16919, 16964, 16921, 16966, 16923, - 16968, 16920, 16965, 16922, 16967, 16954, 16924, 16956, 16926, 16901, - 16958, 16928, 16955, 16925, 16957, 16927, 16881, 16883, 16885, 16882, - 16884, 16934, 16858, 16944, 16936, 16852, 16946, 16938, 16861, 16948, - 16935, 16859, 16945, 16937, 16860, 16947, 16876, 16862, 16879, 16877, - 16878, 16906, 16973, 16908, 16975, 16907, 16974, 16949, 16951, 16953, - 16950, 16952, 16902, 16969, 16971, 16970, 16972, 16880, 16963, 16896, - 16897, 16959, 16961, 16960, 16962, 16983, 16985, 16982, 16980, 16981, - 35762, 35762, 35762, 35762, 35762, 4063, 4071, 4070, 4068, 4067, 4074, - 4039, 4059, 4027, 4055, 4069, 4065, 4072, 4076, 4052, 4058, 4062, 4073, - 4051, 4057, 4061, 4007, 4043, 4019, 4024, 4008, 4025, 4010, 4050, 4012, - 4020, 4013, 4021, 4026, 4032, 4017, 4038, 4075, 4040, 4029, 4035, 4046, - 4042, 35762, 15039, 15083, 15040, 15045, 15046, 15048, 15075, 15086, - 15061, 15062, 15064, 15066, 15073, 15069, 15065, 15072, 15041, 15051, - 15085, 15052, 15076, 15088, 15033, 15028, 15082, 15027, 15038, 15074, - 15059, 15112, 15023, 15026, 15109, 15110, 15030, 15029, 15096, 15095, - 15113, 15092, 15093, 15114, 15101, 15115, 15091, 15090, 15094, 15105, - 15031, 15111, 15032, 15116, 15084, 15047, 15050, 15049, 15063, 15070, - 15067, 15068, 15071, 15042, 15044, 15043, 15037, 15058, 15056, 15053, - 15054, 15057, 15055, 15035, 15036, 15078, 15079, 15081, 15080, 15077, - 15060, 15089, 15098, 15100, 15099, 15034, 15087, 15097, 15102, 15103, - 15104, 15107, 15106, 15108, 15024, 15025, 35762, 16020, 16023, 16022, - 16018, 16016, 16012, 16019, 16014, 16010, 16013, 16021, 16017, 16011, - 16024, 16025, 16015, 4064, 4053, 4066, 4030, 4023, 4022, 4049, 4045, - 4037, 4014, 4033, 4018, 4036, 4041, 4009, 4011, 4016, 4048, 4044, 4034, - 4005, 4006, 4004, 4003, 4028, 4060, 4054, 4002, 4031, 4056, 4047, 4015, - 7680, 7683, 7684, 7682, 7671, 7656, 7662, 7653, 7661, 7675, 7663, 7659, - 7654, 7660, 7657, 7686, 7652, 7670, 7666, 7678, 7685, 7655, 7665, 7674, - 7673, 7679, 7677, 7667, 7669, 7681, 7676, 7672, 7664, 7658, 7668, 7687, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 16041, 16899, 16911, 16912, 16900, 16910, 16886, 16888, 16890, - 16887, 16889, 16913, 16891, 16893, 16895, 16892, 16894, 26284, 26288, - 26300, 26294, 26286, 26292, 26296, 26302, 26277, 26275, 26282, 26298, - 26290, 26280, 26285, 26289, 26301, 26295, 26287, 26293, 26297, 26303, - 26278, 26276, 26283, 26299, 26291, 26281, 26279, 26341, 26340, 35762, - 26339, 26335, 26333, 26314, 26312, 26332, 26324, 26309, 26318, 26334, - 26317, 26311, 26337, 26336, 26316, 26308, 26331, 26329, 26338, 26326, - 26319, 26327, 26310, 26305, 26315, 26322, 26306, 26328, 26330, 26307, - 26320, 26304, 26313, 26321, 26325, 26323, 6319, 6307, 6329, 6308, 6473, - 6487, 6475, 6457, 6454, 6470, 6468, 6450, 26421, 6488, 6494, 6493, 6490, - 6489, 6492, 6491, 6496, 6495, 6474, 6476, 6482, 6481, 6478, 6477, 6267, - 6271, 6283, 6277, 6269, 6275, 6279, 6260, 6258, 6256, 6265, 6281, 6273, - 6263, 6268, 6272, 6284, 6278, 6270, 6276, 6280, 6261, 6259, 6257, 6266, - 6282, 6274, 6264, 6394, 6393, 6262, 18033, 6342, 6337, 6335, 6304, 6302, - 6334, 6325, 6299, 6317, 6336, 6315, 6301, 6339, 6338, 6313, 6297, 6328, - 6333, 6306, 6330, 6318, 6331, 6300, 6293, 6309, 6324, 6314, 6303, 6327, - 6298, 6341, 6290, 6340, 6321, 6294, 6292, 6305, 6295, 6310, 6311, 6323, - 6312, 6322, 6332, 6326, 6296, 6320, 6288, 6316, 6480, 6479, 6484, 6483, - 6456, 6458, 6464, 6463, 6460, 6459, 6462, 6461, 6466, 6465, 6453, 16111, - 16114, 16115, 16053, 16116, 16112, 16113, 16052, 16118, 16119, 16117, - 16085, 29271, 29240, 29242, 19871, 6388, 6390, 6392, 6389, 6391, 6351, - 6353, 6355, 6352, 6354, 6371, 6373, 6375, 6372, 6374, 6376, 6378, 6380, - 6377, 6379, 6361, 6363, 6365, 6362, 6364, 6346, 6348, 6350, 6347, 6349, - 6356, 6358, 6360, 6357, 6359, 6385, 6387, 6386, 6366, 6368, 6370, 6367, - 6369, 6381, 6383, 6382, 6384, 29238, 29202, 29200, 29199, 29201, 29274, - 29275, 29434, 29241, 29234, 29364, 29368, 29281, 29283, 29282, 29246, - 29247, 29251, 29250, 29284, 29249, 29285, 29288, 29286, 29287, 29252, - 29253, 29295, 29296, 29297, 29294, 29293, 29404, 29405, 29408, 29406, - 29407, 29225, 29229, 29230, 29413, 29357, 29358, 29259, 29371, 29372, - 29206, 29380, 29378, 29379, 29209, 29266, 29265, 29208, 29267, 29260, - 29377, 29375, 29261, 29376, 29374, 29211, 29381, 29210, 29264, 29382, - 29262, 29263, 29321, 29322, 29325, 29324, 29323, 29331, 29332, 29333, - 29328, 29329, 29330, 29432, 29431, 29433, 29399, 29400, 29402, 29401, - 29397, 29396, 29414, 16109, 16110, 16092, 16094, 16101, 16100, 16107, - 16105, 16096, 16103, 16095, 16098, 16091, 16093, 16102, 16099, 16108, - 16106, 16097, 16104, 16086, 16090, 16089, 16088, 16087, 29269, 29224, - 29204, 29207, 29369, 29385, 29226, 29228, 29227, 29279, 29235, 29239, - 29236, 29237, 29216, 29373, 29356, 29338, 29320, 29280, 29302, 29326, - 29256, 29213, 29299, 29386, 29359, 29339, 29340, 29353, 29303, 29272, - 29298, 29350, 29254, 29412, 29341, 29354, 29233, 29305, 29245, 29360, - 29342, 29335, 29217, 29289, 29337, 29219, 29319, 29292, 29336, 29218, - 29318, 29291, 29315, 29316, 29370, 29301, 29352, 29255, 29393, 29394, - 29395, 29390, 29361, 29343, 29355, 29391, 29362, 29344, 29346, 29307, - 29347, 29392, 29363, 29345, 29348, 29308, 29349, 29300, 29317, 29203, - 29212, 29222, 29223, 29220, 29215, 29231, 29257, 29258, 29268, 29273, - 29304, 29290, 29306, 29312, 29313, 29310, 29314, 29327, 29334, 29351, - 29387, 29388, 29384, 29389, 29409, 29410, 29430, 29205, 29198, 16084, - 16069, 16057, 16076, 16075, 16082, 16080, 16071, 16078, 16070, 16073, - 16068, 16056, 16077, 16074, 16083, 16081, 16072, 16079, 16058, 16066, - 16064, 16063, 16060, 16059, 16062, 16061, 16067, 16065, 16054, 16055, - 29248, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 15811, - 15814, 15775, 15825, 15822, 15770, 15808, 15787, 15804, 15821, 15799, - 15805, 15780, 15782, 15792, 15826, 15779, 15823, 15763, 15769, 15767, - 15786, 15806, 15800, 15789, 15785, 15793, 15784, 15809, 15812, 15791, - 15777, 15801, 15783, 15798, 15778, 15813, 15796, 15794, 15773, 15772, - 15790, 15768, 15771, 15781, 15797, 15795, 15815, 15802, 15810, 15807, - 15819, 15774, 15817, 15765, 15816, 15818, 15820, 15776, 15824, 15788, - 15803, 15764, 15766, 34695, 34702, 34694, 34701, 34699, 34700, 34697, - 34698, 35470, 35471, 35468, 35469, 35467, 35465, 35466, 35474, 35475, - 35472, 35473, 35477, 35476, 35342, 34362, 34363, 34356, 34361, 34359, - 34360, 34357, 34358, 34366, 34367, 34364, 34365, 34347, 34345, 34346, - 34370, 34371, 34368, 34369, 34355, 34353, 34354, 34351, 34352, 34344, - 34350, 34349, 34348, 34376, 34377, 34372, 34375, 34374, 34373, 35074, - 35075, 35069, 35073, 35072, 35070, 35071, 35078, 35079, 35076, 35077, - 35063, 35061, 35062, 35082, 35083, 35080, 35081, 35067, 35068, 35060, - 35066, 35065, 35064, 35088, 35089, 35084, 35087, 35086, 35085, 34330, - 34331, 34324, 34329, 34327, 34328, 34325, 34326, 34334, 34335, 34332, - 34333, 34315, 34313, 34314, 34338, 34339, 34336, 34337, 34323, 34321, - 34322, 34319, 34320, 34312, 34318, 34317, 34316, 34342, 34343, 34340, - 34341, 34877, 34878, 34872, 34876, 34875, 34873, 34874, 34881, 34882, - 34879, 34880, 34885, 34886, 34883, 34884, 34891, 34892, 34887, 34890, - 34889, 34888, 34897, 34898, 34893, 34896, 34895, 34894, 34620, 34621, - 34615, 34619, 34618, 34616, 34617, 34624, 34625, 34622, 34623, 34609, - 34607, 34608, 34628, 34629, 34626, 34627, 34613, 34614, 34606, 34612, - 34611, 34610, 34634, 34630, 34633, 34632, 34631, 34856, 34857, 34851, - 34855, 34854, 34852, 34853, 34860, 34861, 34858, 34859, 34844, 34845, - 34842, 34843, 34864, 34865, 34862, 34863, 34871, 34870, 34849, 34850, - 34841, 34848, 34847, 34846, 34868, 34869, 34866, 34867, 34501, 34502, - 34499, 34500, 34497, 34498, 34495, 34496, 34494, 34492, 34493, 34511, - 34512, 34507, 34510, 34509, 34508, 34505, 34506, 34503, 34504, 35320, - 35321, 35314, 35319, 35317, 35318, 35315, 35316, 35324, 35325, 35322, - 35323, 35328, 35329, 35326, 35327, 35313, 35312, 35334, 35335, 35330, - 35333, 35332, 35331, 35340, 35341, 35336, 35339, 35338, 35337, 34479, - 34480, 34474, 34478, 34477, 34475, 34476, 34486, 34487, 34484, 34485, - 34468, 34467, 34490, 34491, 34488, 34489, 34483, 34481, 34482, 34472, - 34473, 34466, 34471, 34470, 34469, 35299, 35300, 35294, 35298, 35297, - 35295, 35296, 35306, 35307, 35304, 35305, 35287, 35288, 35285, 35286, - 35310, 35311, 35308, 35309, 35303, 35301, 35302, 35292, 35293, 35284, - 35291, 35290, 35289, 34453, 34454, 34448, 34452, 34451, 34449, 34450, - 34460, 34461, 34458, 34459, 34442, 34440, 34441, 34464, 34465, 34462, - 34463, 34457, 34455, 34456, 34446, 34447, 34439, 34445, 34444, 34443, - 34903, 34904, 34899, 34902, 34901, 34900, 34910, 34911, 34908, 34909, - 34914, 34915, 34912, 34913, 34907, 34905, 34906, 34920, 34921, 34916, - 34919, 34918, 34917, 34650, 34651, 34644, 34649, 34647, 34648, 34645, - 34646, 34654, 34655, 34652, 34653, 34639, 34638, 34636, 34637, 34635, - 34643, 34641, 34642, 34640, 35048, 35049, 35043, 35047, 35046, 35044, - 35045, 35052, 35050, 35051, 35037, 35035, 35036, 35058, 35059, 35056, - 35057, 35055, 35053, 35054, 35041, 35042, 35034, 35040, 35039, 35038, - 34588, 34589, 34583, 34587, 34586, 34584, 34585, 34598, 34599, 34596, - 34597, 34577, 34575, 34576, 34595, 34593, 34594, 34592, 34590, 34591, - 34581, 34582, 34574, 34580, 34579, 34578, 34604, 34605, 34600, 34603, - 34602, 34601, 34803, 34804, 34797, 34802, 34800, 34801, 34798, 34799, - 34807, 34808, 34805, 34806, 34787, 34788, 34785, 34786, 34811, 34812, - 34809, 34810, 34796, 34794, 34795, 34792, 34793, 34784, 34791, 34790, - 34789, 34817, 34818, 34813, 34816, 34815, 34814, 34557, 34558, 34551, - 34556, 34554, 34555, 34552, 34553, 34561, 34562, 34559, 34560, 34544, - 34545, 34542, 34543, 34569, 34570, 34567, 34568, 34565, 34566, 34563, - 34564, 34549, 34550, 34541, 34548, 34547, 34546, 34770, 34771, 34765, - 34769, 34768, 34766, 34767, 34774, 34775, 34772, 34773, 34759, 34757, - 34758, 34782, 34783, 34780, 34781, 34778, 34779, 34776, 34777, 34763, - 34764, 34756, 34762, 34761, 34760, 34517, 34518, 34513, 34516, 34514, - 34515, 34531, 34532, 34529, 34530, 34522, 34523, 34520, 34521, 34539, - 34540, 34537, 34538, 34535, 34536, 34533, 34534, 34527, 34528, 34519, - 34526, 34525, 34524, 34840, 34839, 34833, 34834, 34831, 34832, 34822, - 34820, 34821, 34837, 34838, 34835, 34836, 34830, 34828, 34829, 34826, - 34827, 34819, 34825, 34824, 34823, 34669, 34670, 34663, 34668, 34666, - 34667, 34664, 34665, 34673, 34674, 34671, 34672, 34658, 34659, 34656, - 34657, 34677, 34678, 34675, 34676, 34662, 34660, 34661, 34930, 34928, - 34929, 34933, 34934, 34931, 34932, 34923, 34924, 34922, 34937, 34938, - 34935, 34936, 34927, 34925, 34926, 34573, 34572, 34571, 34688, 34689, - 34686, 34687, 34681, 34682, 34679, 34680, 34692, 34693, 34690, 34691, - 34685, 34683, 34684, 35354, 35355, 35352, 35353, 35345, 35343, 35344, - 35351, 35349, 35350, 35348, 35346, 35347, 35417, 35418, 35412, 35416, - 35415, 35413, 35414, 35453, 35454, 35451, 35452, 35406, 35404, 35405, - 35457, 35458, 35455, 35456, 35450, 35448, 35449, 35410, 35411, 35403, - 35409, 35408, 35407, 35463, 35464, 35459, 35462, 35461, 35460, 34423, - 34424, 34417, 34422, 34420, 34421, 34418, 34419, 34427, 34428, 34425, - 34426, 34408, 34406, 34407, 34431, 34432, 34429, 34430, 34416, 34414, - 34415, 34412, 34413, 34405, 34411, 34410, 34409, 34437, 34438, 34433, - 34436, 34435, 34434, 35431, 35432, 35425, 35430, 35428, 35429, 35426, - 35427, 35435, 35436, 35433, 35434, 35424, 35422, 35423, 35421, 35419, - 35420, 35441, 35437, 35440, 35439, 35438, 35446, 35447, 35442, 35445, - 35444, 35443, 35020, 35021, 35015, 35019, 35018, 35016, 35017, 35024, - 35025, 35022, 35023, 35008, 35007, 35014, 35013, 35033, 35032, 35012, - 35006, 35011, 35010, 35009, 35030, 35031, 35026, 35029, 35028, 35027, - 35265, 35266, 35260, 35264, 35263, 35261, 35262, 35272, 35273, 35270, - 35271, 35254, 35252, 35253, 35276, 35277, 35274, 35275, 35269, 35267, - 35268, 35258, 35259, 35251, 35257, 35256, 35255, 35282, 35283, 35278, - 35281, 35280, 35279, 35201, 35202, 35196, 35200, 35199, 35197, 35198, - 35208, 35209, 35206, 35207, 35212, 35213, 35210, 35211, 35205, 35203, - 35204, 35216, 35217, 35214, 35215, 35222, 35223, 35218, 35221, 35220, - 35219, 35387, 35388, 35385, 35386, 35379, 35377, 35378, 35395, 35396, - 35393, 35394, 35391, 35392, 35389, 35390, 35383, 35384, 35376, 35382, - 35381, 35380, 35401, 35402, 35397, 35400, 35399, 35398, 34389, 34390, - 34387, 34388, 34381, 34382, 34379, 34380, 34397, 34398, 34395, 34396, - 34393, 34394, 34391, 34392, 34386, 34378, 34385, 34384, 34383, 34403, - 34404, 34399, 34402, 34401, 34400, 35169, 35168, 35148, 35147, 35160, - 35161, 35158, 35159, 35156, 35157, 35154, 35155, 35152, 35153, 35146, - 35151, 35150, 35149, 35166, 35167, 35162, 35165, 35164, 35163, 34969, - 34970, 34967, 34968, 34966, 34964, 34965, 34973, 34974, 34971, 34972, - 34979, 34980, 34975, 34978, 34977, 34976, 34985, 34986, 34981, 34984, - 34983, 34982, 35235, 35236, 35233, 35234, 35227, 35225, 35226, 35243, - 35244, 35241, 35242, 35239, 35240, 35237, 35238, 35231, 35232, 35224, - 35230, 35229, 35228, 35249, 35250, 35245, 35248, 35247, 35246, 35184, - 35185, 35182, 35183, 35173, 35171, 35172, 35188, 35189, 35186, 35187, - 35181, 35179, 35180, 35177, 35178, 35170, 35176, 35175, 35174, 35194, - 35195, 35190, 35193, 35192, 35191, 34744, 34745, 34738, 34743, 34741, - 34742, 34739, 34740, 34731, 34732, 34729, 34730, 34748, 34749, 34746, - 34747, 34736, 34737, 34728, 34735, 34734, 34733, 34754, 34755, 34750, - 34753, 34752, 34751, 35106, 35107, 35100, 35105, 35103, 35104, 35101, - 35102, 35093, 35094, 35091, 35092, 35110, 35111, 35108, 35109, 35098, - 35099, 35090, 35097, 35096, 35095, 35116, 35117, 35112, 35115, 35114, - 35113, 34718, 34719, 34712, 34717, 34715, 34716, 34713, 34714, 34706, - 34704, 34705, 34722, 34723, 34720, 34721, 34710, 34711, 34703, 34709, - 34708, 34707, 34726, 34727, 34724, 34725, 34952, 34953, 34946, 34951, - 34949, 34950, 34947, 34948, 34941, 34940, 34956, 34957, 34954, 34955, - 34945, 34939, 34944, 34943, 34942, 34962, 34963, 34958, 34961, 34960, - 34959, 35000, 35001, 34994, 34999, 34997, 34998, 34995, 34996, 34990, - 34988, 34989, 35004, 35005, 35002, 35003, 34992, 34993, 34987, 34991, - 35362, 35363, 35356, 35361, 35359, 35360, 35357, 35358, 35375, 35374, - 35366, 35367, 35364, 35365, 35372, 35373, 35368, 35371, 35370, 35369, - 35134, 35135, 35128, 35133, 35131, 35132, 35129, 35130, 35121, 35122, - 35119, 35120, 35138, 35139, 35136, 35137, 35126, 35127, 35118, 35125, - 35124, 35123, 35144, 35145, 35140, 35143, 35142, 35141, 35762, 35762, - 35762, 34309, 34282, 34280, 34287, 34261, 34297, 34268, 34270, 34286, - 34273, 34284, 34257, 34285, 34303, 34291, 34272, 34298, 34271, 34304, - 34262, 34265, 34258, 34267, 34288, 34299, 34311, 34276, 34307, 34292, - 34275, 34302, 34300, 34296, 34301, 34308, 34279, 34289, 34278, 34269, - 34277, 34310, 34266, 34293, 34283, 34260, 34259, 34264, 34274, 34294, - 34305, 34295, 34263, 34306, 34290, 34281, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 20474, 20463, 20462, 20446, 20444, - 20443, 20457, 20461, 20460, 20476, 20455, 20454, 20445, 20442, 20441, - 20478, 20453, 20477, 20465, 20468, 20469, 20452, 20459, 20480, 20458, - 20475, 20479, 20464, 20467, 20456, 20470, 20471, 20450, 20451, 20481, - 20472, 20447, 20448, 20449, 20473, 20437, 20440, 20438, 20436, 20439, - 20435, 20482, 20483, 33047, 33048, 32884, 33033, 33034, 33007, 32810, - 32817, 32920, 32893, 32927, 32867, 32993, 33021, 32845, 32838, 32796, - 32789, 32911, 33014, 32803, 32946, 32831, 32824, 32859, 32852, 32986, - 33000, 32965, 33028, 32906, 32952, 32873, 32934, 32979, 32972, 33056, - 33057, 32888, 32889, 33042, 33043, 33009, 32812, 32819, 32922, 32899, - 32929, 32870, 32995, 33023, 32847, 32840, 32798, 32791, 32915, 33016, - 32805, 32948, 32833, 32826, 32861, 32854, 32988, 33002, 32967, 33030, - 32907, 32957, 32878, 32936, 32981, 32974, 33054, 33055, 32959, 32886, - 32887, 33040, 33041, 33008, 32811, 32818, 32921, 32897, 32898, 32928, - 32869, 32994, 33022, 32846, 32839, 32797, 32790, 32914, 33015, 32804, - 32947, 32832, 32825, 32860, 32853, 32987, 33001, 32966, 33029, 32903, - 32904, 32956, 32877, 32935, 32980, 32973, 33051, 33052, 32882, 33037, - 33038, 33005, 32808, 32815, 32918, 32896, 32925, 32865, 32991, 33019, - 32843, 32836, 32794, 32787, 32913, 33012, 32801, 32944, 32829, 32822, - 32857, 32850, 32984, 32998, 32963, 33026, 32902, 32955, 32876, 32932, - 32977, 32970, 33058, 33059, 32890, 32891, 33044, 33045, 33010, 32813, - 32820, 32923, 32900, 32930, 32871, 32996, 33024, 32848, 32841, 32799, - 32792, 32916, 33017, 32806, 32949, 32834, 32827, 32862, 32855, 32989, - 33003, 32968, 33031, 32908, 32958, 32879, 32937, 32982, 32975, 33050, - 33053, 32961, 32880, 32881, 33036, 33039, 33004, 32807, 32814, 32917, - 32895, 32924, 32863, 32864, 32990, 33018, 32842, 32835, 32793, 32786, - 32912, 33011, 32800, 32938, 32828, 32821, 32856, 32849, 32983, 32997, - 32962, 33025, 32901, 32954, 32875, 32931, 32976, 32969, 33046, 33049, - 32960, 32883, 32885, 33032, 33035, 33006, 32809, 32816, 32919, 32892, - 32894, 32926, 32866, 32868, 32992, 33020, 32844, 32837, 32795, 32788, - 32909, 33013, 32802, 32945, 32830, 32823, 32858, 32851, 32985, 32999, - 32964, 33027, 32905, 32951, 32953, 32872, 32874, 32933, 32978, 32971, - 32950, 32910, 32783, 32784, 32785, 32941, 32942, 32939, 33063, 33066, - 33069, 33068, 33072, 33064, 33071, 33062, 33060, 33067, 33070, 33061, - 33065, 33079, 33081, 33078, 33077, 33074, 33073, 33076, 33075, 33082, - 33080, 32943, 32940, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 9978, 10178, 9867, 10026, 9818, 10120, 9922, 10081, - 9863, 10022, 9943, 10106, 9851, 10010, 9811, 10107, 9971, 10171, 9919, - 10078, 9820, 10122, 9920, 10079, 9859, 10018, 9852, 10011, 9916, 10075, - 9973, 10173, 9819, 10121, 9962, 10140, 9959, 10137, 9960, 10138, 9942, - 10105, 9849, 10008, 9864, 10023, 9993, 7842, 7834, 7785, 7835, 28868, - 7809, 7798, 7812, 7808, 7817, 7810, 7807, 7804, 7840, 7829, 9992, 9996, - 9873, 10032, 9871, 10030, 9983, 10183, 9861, 10020, 9869, 10028, 9824, - 10148, 9831, 10156, 9828, 10152, 9827, 10151, 9830, 10155, 9907, 10066, - 9957, 10135, 9865, 10024, 9860, 10019, 23047, 23084, 7793, 7801, 3393, - 2760, 3396, 2762, 3392, 3368, 3385, 3395, 2789, 3394, 2765, 3365, 3371, - 3370, 2763, 2769, 3384, 2788, 2782, 2768, 3380, 2776, 3375, 3381, 3374, - 3378, 2758, 2754, 2786, 2785, 2780, 3387, 3377, 3388, 3389, 2787, 2752, - 3361, 2781, 2784, 3364, 3390, 3362, 2749, 3373, 2767, 2774, 2791, 3367, - 3372, 2753, 2777, 2778, 2779, 3376, 3363, 2751, 2750, 3391, 2790, 2766, - 3366, 2764, 2755, 2771, 3369, 2770, 2773, 3386, 2761, 2775, 2772, 3383, - 2759, 3382, 2783, 3379, 2748, 2756, 2757, 2745, 2744, 3397, 3399, 2747, - 2746, 3398, 3400, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 23044, 23040, 23043, 23039, 23045, 23041, 23046, 23042, 23099, 23114, - 23142, 23121, 23104, 23098, 23113, 23141, 23120, 23103, 23100, 23115, - 23143, 23125, 23105, 23092, 23090, 23091, 23136, 23155, 23152, 23154, - 23153, 23133, 23303, 23304, 18285, 18884, 18286, 18885, 18325, 18946, - 18547, 19306, 18546, 19263, 18227, 18806, 18228, 18807, 18688, 18696, - 18201, 18762, 18202, 18763, 18203, 18764, 18199, 18760, 18200, 18761, - 18204, 18765, 18493, 19188, 18366, 18998, 18363, 18995, 18367, 18999, - 18213, 18783, 18385, 19022, 18416, 19096, 18417, 19098, 18463, 19136, - 18468, 19141, 18466, 19139, 18469, 19142, 18475, 19153, 18474, 19150, - 18490, 19178, 18495, 19191, 18588, 19356, 18597, 19368, 18592, 19363, - 18541, 19257, 18542, 19258, 18596, 19367, 18287, 18902, 18354, 18985, - 18229, 18808, 23312, 18844, 19042, 19056, 19079, 19190, 18673, 19301, - 19353, 18347, 18974, 18348, 18975, 18349, 18531, 19269, 18536, 19299, - 18350, 18977, 18351, 18978, 18352, 18979, 23119, 23088, 23165, 18514, - 19213, 18534, 19027, 18704, 18409, 19070, 18223, 18796, 18793, 18940, - 18207, 18768, 18294, 18909, 18593, 19364, 18594, 19365, 18595, 19366, - 18302, 18920, 18370, 19003, 18412, 19075, 18488, 19175, 18510, 19211, - 18321, 18492, 18517, 18373, 18513, 18695, 18535, 18538, 18357, 18230, - 18214, 18785, 18461, 19134, 18584, 19344, 18307, 18925, 18308, 18926, - 18309, 18927, 18458, 19125, 18196, 18757, 18221, 18511, 18634, 18234, - 18810, 18507, 19205, 35762, 35762, 35762, 35762, 35762, 18226, 18802, - 35762, 18837, 35762, 18834, 18398, 19055, 18516, 19231, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 23020, 23023, 23032, 18491, 19179, 18644, 23015, 23244, - 18678, 18642, 18643, 18640, 18641, 18639, 29798, 29800, 29810, 29802, - 29799, 29801, 29809, 29790, 29789, 29786, 29785, 29808, 29784, 29783, - 29788, 29787, 29778, 29777, 29772, 29771, 29780, 29779, 29774, 29773, - 29796, 29792, 29791, 29782, 29781, 29795, 29776, 29794, 29775, 29797, - 29793, 29812, 29814, 29815, 29813, 29811, 29803, 29804, 29805, 29806, - 29807, 35762, 35762, 35762, 24678, 24677, 24680, 24675, 24676, 24679, - 24672, 24671, 24674, 24673, 35762, 35762, 35762, 35762, 35762, 35762, - 26558, 26557, 26546, 26547, 26534, 26536, 26568, 26549, 26556, 26555, - 26539, 26550, 26560, 26559, 26565, 26570, 26552, 26551, 26538, 26573, - 26561, 26562, 26540, 26575, 26572, 26569, 26541, 26542, 26567, 26531, - 26576, 26578, 26563, 26577, 26571, 26574, 26566, 26545, 26564, 26583, - 26584, 26554, 26553, 26537, 26548, 26532, 26544, 26543, 26533, 26582, - 26585, 26535, 26581, 26586, 26580, 26579, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 27667, 27669, 27616, 27617, 27637, 27638, - 27633, 27634, 27628, 27629, 27630, 27631, 27660, 27661, 27618, 27635, - 27636, 27619, 27657, 27656, 27653, 27652, 27641, 27651, 27650, 27655, - 27654, 27643, 27625, 27624, 27621, 27620, 27642, 27627, 27626, 27623, - 27622, 27644, 27659, 27658, 27649, 27648, 27663, 27665, 27664, 27640, - 27632, 27645, 27646, 27647, 27662, 27639, 27682, 27683, 27694, 27695, - 27690, 27691, 27686, 27687, 27688, 27689, 27696, 27697, 27684, 27692, - 27693, 27685, 27668, 27666, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 27671, 27670, 27678, 27680, 27677, 27676, 27673, 27672, - 27675, 27674, 27681, 27679, 35762, 35762, 35762, 35762, 35762, 35762, - 7860, 7862, 7859, 7858, 7855, 7854, 7857, 7856, 7863, 7861, 7851, 7852, - 7847, 7848, 7849, 7850, 7846, 7853, 10463, 10456, 10465, 10458, 10457, - 10455, 10462, 10359, 10357, 10362, 10464, 10513, 10368, 10480, 17118, - 17120, 17117, 17116, 17113, 17112, 17115, 17114, 17121, 17119, 17082, - 17081, 17092, 17078, 17086, 17085, 17099, 17079, 17088, 17074, 17080, - 17084, 17083, 17094, 17091, 17089, 17095, 17098, 17093, 17097, 17087, - 17075, 17096, 17090, 17100, 17076, 17101, 17077, 17110, 17107, 17109, - 17108, 17111, 17106, 17104, 17105, 17102, 17103, 27041, 27038, 27032, - 27046, 27037, 27034, 27043, 27035, 27028, 27036, 27040, 27030, 27045, - 27044, 27042, 27048, 27047, 27039, 27027, 27031, 27033, 27029, 27049, - 27056, 27058, 27051, 27054, 27057, 27055, 27053, 27052, 27024, 27023, - 27026, 27025, 27059, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 27050, 14774, 14777, 14778, 14775, 14732, - 14733, 14741, 14735, 14737, 14740, 14734, 14730, 14736, 14738, 14731, - 14697, 14699, 14700, 14713, 14720, 14727, 14762, 14679, 14686, 14760, - 14764, 14710, 14783, 14766, 35762, 35762, 35762, 16437, 16435, 16439, - 16438, 16409, 16377, 16376, 16378, 16418, 16397, 16383, 16384, 16416, - 16410, 16417, 16379, 16380, 16381, 16393, 16394, 16382, 16391, 16392, - 16407, 16386, 16408, 16385, 16405, 16406, 16372, 16373, 16388, 16403, - 16404, 16374, 16375, 16387, 16395, 16396, 16389, 16390, 16413, 16415, - 16398, 16399, 16412, 16414, 16401, 16402, 16400, 16411, 16436, 16442, - 16444, 16445, 16446, 16440, 16441, 16443, 16448, 16447, 16368, 16369, - 16370, 16433, 16371, 16419, 16422, 16431, 16424, 16429, 16427, 16425, - 16423, 16420, 16421, 16426, 16434, 35762, 16432, 16455, 16457, 16454, - 16453, 16450, 16449, 16452, 16451, 16458, 16456, 35762, 35762, 35762, - 35762, 16428, 16430, 23922, 23920, 23915, 23912, 23918, 23998, 23986, - 23938, 23950, 23946, 23945, 23948, 23947, 23940, 23939, 23937, 24040, - 24042, 24039, 24038, 24035, 24034, 24037, 24036, 24043, 24041, 23949, - 23942, 23941, 23944, 23943, 35762, 5944, 5962, 5964, 5961, 5945, 5963, - 5953, 5952, 5949, 5948, 5924, 5925, 5947, 5946, 5951, 5950, 5926, 5928, - 5927, 5955, 5954, 5941, 5940, 5929, 5930, 5939, 5935, 5934, 5933, 5938, - 5937, 5931, 5932, 5936, 5960, 5958, 5957, 5959, 5942, 5943, 5956, 5969, - 5972, 5973, 5978, 5976, 5975, 5974, 5970, 5971, 5977, 5912, 5910, 5909, - 5911, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 5918, 5917, 5914, 5906, 5916, 5922, 5913, 5920, 5923, 5921, 5919, 5915, - 5908, 5907, 35762, 35762, 5985, 5987, 5984, 5983, 5980, 5979, 5982, 5981, - 5988, 5986, 35762, 35762, 5965, 5967, 5966, 5968, 23892, 23885, 23884, - 23889, 23888, 23882, 23881, 23880, 23878, 23877, 23879, 23883, 23894, - 23887, 23886, 23891, 23985, 23895, 23896, 23893, 23982, 23983, 23984, - 24013, 24015, 24014, 23867, 24009, 24000, 24001, 23931, 23932, 30279, - 30255, 30278, 30254, 30277, 30253, 30292, 30268, 30286, 30262, 30281, - 30257, 30280, 30256, 30297, 30273, 30287, 30263, 30290, 30266, 30285, - 30261, 30284, 30260, 30288, 30264, 30289, 30265, 30283, 30259, 30282, - 30258, 30291, 30267, 30295, 30271, 30299, 30275, 30296, 30272, 30294, - 30270, 30298, 30274, 30293, 30269, 30300, 30276, 30301, 30313, 30321, - 30318, 30317, 30323, 30324, 30302, 30322, 30319, 30320, 30312, 30316, - 30315, 30314, 30311, 30308, 30309, 30310, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 30304, 30305, 30307, 30306, 30303, 22359, 22360, 22318, 22335, 22346, - 22345, 22322, 22321, 22334, 22340, 22341, 22373, 22375, 22365, 22367, - 22366, 22313, 22310, 22312, 22362, 22363, 22377, 22378, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 12895, 12893, - 12892, 12891, 12890, 12894, 35762, 35762, 12589, 12587, 12586, 12585, - 12584, 12588, 35762, 35762, 12604, 12602, 12601, 12600, 12599, 12603, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 12545, - 12551, 12549, 12546, 12548, 12547, 12550, 35762, 12530, 12536, 12534, - 12531, 12533, 12532, 12535, 35762, 18776, 18752, 18782, 18777, 18873, - 19035, 19223, 19025, 19016, 19019, 19046, 19060, 18887, 18780, 18781, - 19130, 18981, 19272, 19271, 19273, 19274, 19233, 18672, 19177, 18835, - 19157, 18836, 19220, 19221, 18779, 19343, 19309, 19352, 19295, 19348, - 18799, 18800, 18801, 19384, 19381, 19382, 19383, 19395, 23002, 23225, - 23234, 23235, 23292, 19214, 18984, 19131, 19354, 18980, 14005, 18842, - 19304, 19277, 23289, 23138, 23162, 35762, 35762, 35762, 35762, 6167, - 6168, 6169, 6170, 6171, 6172, 6130, 6166, 6131, 6132, 6133, 6134, 6135, - 6095, 6096, 6097, 6098, 6099, 6100, 6136, 6137, 6138, 6139, 6140, 6141, - 6142, 6143, 6144, 6145, 6146, 6101, 6094, 6102, 6103, 6104, 6105, 6106, - 6107, 6148, 6149, 6150, 6151, 6152, 6153, 6109, 6108, 6110, 6111, 6112, - 6113, 6114, 6088, 6127, 6089, 6128, 6090, 6129, 6091, 6092, 6093, 6087, - 6115, 6116, 6117, 6118, 6119, 6120, 6121, 6122, 6123, 6124, 6125, 6126, - 6154, 6155, 6156, 6157, 6158, 6159, 6160, 22327, 22339, 22349, 22351, - 22336, 22330, 22317, 22342, 22329, 22332, 22344, 22355, 22357, 22353, - 22358, 22347, 22338, 22356, 22324, 22325, 22354, 22316, 22326, 22320, - 22323, 22319, 22315, 22328, 22350, 22352, 22337, 22331, 22343, 22333, - 22348, 22369, 22372, 22364, 22371, 22370, 22374, 22368, 22376, 22314, - 22361, 22311, 35762, 35762, 22385, 22387, 22384, 22383, 22380, 22379, - 22382, 22381, 22388, 22386, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 14971, 14969, 15002, 15003, - 15004, 14980, 14981, 15013, 15015, 14948, 14946, 14945, 14949, 14955, - 14956, 14958, 14957, 14962, 14959, 14960, 14964, 14932, 14930, 35762, - 35762, 35762, 35762, 14828, 14829, 14897, 14898, 14917, 14911, 14912, - 14915, 14914, 14913, 14872, 14857, 14896, 14863, 14867, 14866, 14880, - 14879, 14800, 14825, 14818, 14903, 14816, 14823, 14853, 14846, 14852, - 14907, 14848, 14851, 14850, 14891, 14884, 14901, 14902, 14890, 14888, - 14887, 14892, 14894, 14839, 14838, 14924, 14925, 14789, 14788, 14904, - 14843, 14841, 35762, 35762, 35762, 35762, 7281, 7282, 7283, 7284, 7285, - 7286, 7287, 7288, 7289, 7290, 7291, 7292, 7293, 7294, 7295, 7296, 7297, - 7298, 7299, 7300, 7301, 7302, 7303, 7304, 7305, 7306, 7307, 7308, 7309, - 7310, 7311, 7312, 7313, 7314, 7315, 7316, 7317, 7318, 7319, 7320, 7321, - 7322, 7323, 7324, 7325, 7326, 7327, 7328, 7329, 7330, 7331, 7332, 7333, - 7334, 7335, 7336, 7337, 7338, 7339, 7340, 7341, 7342, 7343, 7344, 7345, - 7346, 7347, 7348, 7349, 7350, 7351, 7352, 7353, 7354, 7355, 7356, 7357, - 7358, 7359, 7360, 7361, 7362, 7363, 7364, 7365, 7366, 7367, 7368, 7369, - 7370, 7371, 7372, 7373, 7374, 7375, 7376, 7377, 7378, 7379, 7380, 7381, - 7382, 7383, 7384, 7385, 7386, 7387, 7388, 7389, 7390, 7391, 7392, 7393, - 7394, 7395, 7396, 7397, 7398, 7399, 7400, 7401, 7402, 7403, 7404, 7405, - 7406, 7407, 7408, 7409, 7410, 7411, 7412, 7413, 7414, 7415, 7416, 7417, - 7418, 7419, 7420, 7421, 7422, 7423, 7424, 7425, 7426, 7427, 7428, 7429, - 7430, 7431, 7432, 7433, 7434, 7435, 7436, 7437, 7438, 7439, 7440, 7441, - 7442, 7443, 7444, 7445, 7446, 7447, 7448, 7449, 7450, 7451, 7452, 7453, - 7454, 7455, 7456, 7457, 7458, 7459, 7460, 7461, 7462, 7463, 7464, 7465, - 7466, 7467, 7468, 7469, 7470, 7471, 7472, 7473, 7474, 7475, 7476, 7477, - 7478, 7479, 7480, 7481, 7482, 7483, 7484, 7485, 7486, 7487, 7488, 7489, - 7490, 7491, 7492, 7493, 7494, 7495, 7496, 7497, 7498, 7499, 7500, 7501, - 7502, 7503, 7504, 7505, 7506, 7507, 7508, 7509, 7510, 7511, 7512, 7513, - 7514, 7515, 7516, 7517, 7518, 7519, 7520, 7521, 7522, 7523, 7524, 7525, - 7526, 7527, 7528, 7529, 7530, 7531, 7532, 7533, 7534, 7535, 7536, 7079, - 7080, 7081, 7082, 7083, 7084, 7085, 7086, 7087, 7088, 7089, 7090, 7091, - 7092, 7093, 7094, 7095, 7096, 7097, 7098, 7099, 7100, 7101, 7102, 7103, - 7104, 7105, 7106, 7107, 7108, 7109, 7110, 7111, 7112, 7113, 7114, 7115, - 7116, 7117, 7118, 7119, 7120, 7121, 7122, 7123, 7124, 7125, 7126, 7127, - 7128, 7129, 7130, 7131, 7132, 7133, 7134, 7135, 7136, 7137, 7138, 7139, - 7140, 7141, 7142, 7143, 7144, 7145, 7146, 7147, 7148, 7149, 7150, 7151, - 7152, 7153, 7154, 7155, 7156, 7157, 7158, 7159, 7160, 7161, 7162, 7163, - 7164, 7165, 7166, 7167, 7168, 7169, 7170, 7171, 7172, 7173, 7174, 7065, - 7066, 7067, 7068, 7069, 7070, 7071, 7072, 7073, 7074, 7075, 7076, 7077, - 7078, 35762, 35762, 7175, 7176, 7177, 7178, 7179, 7180, 7181, 7182, 7183, - 7184, 7185, 7186, 7187, 7188, 7189, 7190, 7191, 7192, 7193, 7194, 7195, - 7196, 7197, 7198, 7199, 7200, 7201, 7202, 7203, 7204, 7205, 7206, 7207, - 7208, 7209, 7210, 7211, 7212, 7213, 7214, 7215, 7216, 7217, 7218, 7219, - 7220, 7221, 7222, 7223, 7224, 7225, 7226, 7227, 7228, 7229, 7230, 7231, - 7232, 7233, 7234, 7235, 7236, 7237, 7238, 7239, 7240, 7241, 7242, 7243, - 7244, 7245, 7246, 7247, 7248, 7249, 7250, 7251, 7252, 7253, 7254, 7255, - 7256, 7257, 7258, 7259, 7260, 7261, 7262, 7263, 7264, 7265, 7266, 7267, - 7268, 7269, 7270, 7271, 7272, 7273, 7274, 7275, 7276, 7277, 7278, 7279, - 7280, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 19417, - 19420, 19421, 19418, 19419, 19423, 19424, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 2490, 2491, 2489, - 2492, 2488, 35762, 35762, 35762, 35762, 35762, 15435, 15453, 15440, - 15371, 15427, 15430, 15432, 15431, 15426, 15433, 15429, 15428, 15372, - 15405, 15406, 15403, 15404, 15369, 15370, 15368, 15376, 15418, 15416, - 15393, 15425, 15398, 35762, 15410, 15436, 15384, 15379, 15420, 35762, - 15422, 35762, 15396, 15400, 35762, 15386, 15382, 35762, 15414, 15391, - 15408, 15402, 15412, 15424, 15375, 15378, 15381, 15437, 1146, 1145, 1176, - 1174, 1175, 1177, 1403, 1401, 1402, 1404, 1171, 1169, 1170, 1172, 1532, - 1530, 1531, 1533, 1512, 1510, 1511, 1513, 1527, 1525, 1526, 1528, 1545, - 1543, 1544, 1546, 1408, 1406, 1407, 1409, 1211, 1209, 1210, 1212, 1381, - 1379, 1380, 1382, 1489, 1487, 1488, 1490, 1494, 1492, 1493, 1495, 1201, - 1200, 1192, 1191, 1215, 1214, 1204, 1203, 1311, 1310, 1438, 1437, 1333, - 1331, 1332, 1334, 1245, 1243, 1244, 1246, 1257, 1255, 1256, 1258, 1372, - 1370, 1371, 1373, 1386, 1385, 1442, 1440, 1441, 1443, 1287, 1286, 1282, - 1280, 1281, 1283, 1291, 1289, 1290, 1292, 1569, 1568, 1567, 1566, 2352, - 2351, 2360, 2359, 2356, 2355, 2354, 2353, 2364, 2363, 2350, 2358, 2357, - 2366, 2362, 2361, 2365, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 1376, - 1374, 1375, 1377, 1538, 1537, 1590, 1589, 1587, 1586, 1536, 1548, 1547, - 1337, 1336, 1340, 1339, 1603, 1601, 1602, 1604, 1539, 1540, 2043, 2042, - 2045, 2044, 2064, 2063, 2072, 2071, 2062, 2061, 2068, 2067, 2048, 2046, - 2047, 2097, 2095, 2096, 1225, 1223, 1224, 1226, 2054, 2052, 2058, 2041, - 2066, 1645, 1636, 1641, 1648, 1643, 1654, 2006, 1994, 2001, 2014, 2017, - 2022, 2024, 2029, 2026, 2035, 1723, 1729, 1708, 1703, 1762, 1758, 1764, - 1913, 1906, 1918, 1926, 1885, 1889, 1667, 1659, 1663, 1669, 1987, 1982, - 2099, 1616, 1612, 1697, 1693, 1681, 1686, 1677, 1684, 1679, 1688, 1867, - 1863, 1865, 1869, 1738, 1740, 1748, 1746, 1743, 1754, 1736, 1757, 1790, - 1782, 1794, 1800, 1774, 1803, 1821, 1810, 1815, 1825, 1804, 1826, 1842, - 1832, 1854, 1847, 1850, 1857, 1718, 1714, 1715, 1716, 2082, 2075, 2084, - 2090, 2039, 2094, 2023, 1880, 1628, 1934, 1937, 1941, 1935, 1938, 1940, - 2070, 2069, 2056, 2060, 2040, 2065, 1652, 1651, 1646, 1650, 1642, 1653, - 2020, 2019, 2012, 2018, 2016, 2021, 2033, 2032, 2027, 2031, 2025, 2034, - 1678, 1687, 1864, 1868, 1737, 1741, 1752, 1735, 1756, 1798, 1773, 1802, - 1805, 1823, 1855, 1852, 1845, 1851, 1849, 1856, 1627, 2092, 2079, 2088, - 2078, 2038, 2093, 2053, 2051, 2055, 2057, 2049, 1644, 1635, 1640, 1647, - 1637, 2005, 1993, 2000, 2013, 1995, 2028, 1722, 1728, 1707, 1702, 1761, - 1763, 1912, 1905, 1917, 1925, 1884, 1892, 1888, 1666, 1658, 1662, 1668, - 1986, 2098, 1615, 1611, 1696, 1692, 1680, 1685, 1676, 1683, 1866, 1862, - 1739, 1747, 1745, 1742, 1753, 1789, 1781, 1793, 1799, 1783, 1820, 1809, - 1814, 1824, 1841, 1831, 1853, 1846, 1833, 1717, 1713, 1719, 2081, 2074, - 2083, 2089, 2076, 2059, 2050, 1649, 1638, 2015, 1996, 2030, 2036, 1927, - 1909, 1964, 1944, 1744, 1755, 1801, 1848, 1834, 2091, 2077, 1942, 1936, - 1939, 1985, 1989, 1618, 1620, 1695, 1699, 1929, 1933, 1966, 1974, 1705, - 1710, 1731, 1733, 1760, 1766, 1891, 1896, 1665, 1673, 1955, 1950, 1969, - 1963, 1972, 1931, 1894, 1671, 1984, 1988, 1617, 1619, 1694, 1698, 1928, - 1932, 1965, 1973, 1704, 1709, 1730, 1732, 1759, 1765, 1890, 1895, 1664, - 1672, 1953, 1948, 1967, 1961, 1971, 1930, 1893, 1670, 1954, 1949, 1968, - 1962, 1908, 1943, 1981, 1914, 1907, 1919, 1956, 1951, 1970, 1983, 2100, - 1629, 1630, 25893, 25894, 1877, 1872, 1876, 1873, 1874, 1875, 1903, 1622, - 1624, 1623, 1621, 1871, 1902, 1625, 1976, 1878, 2003, 1992, 1991, 1990, - 1998, 2008, 2010, 2009, 1725, 1724, 1701, 1700, 1904, 1911, 1910, 1921, - 1920, 1922, 1924, 1923, 1882, 1881, 1887, 1946, 1945, 1952, 1958, 1957, - 1960, 1959, 1656, 1661, 1660, 1978, 1977, 1979, 1980, 1614, 1609, 1608, - 1607, 1689, 1691, 1690, 1675, 1674, 1860, 1858, 1779, 1780, 1777, 1784, - 1785, 1792, 1791, 1796, 1795, 1806, 1807, 1808, 1818, 1816, 1811, 1812, - 35762, 35762, 1817, 1711, 1712, 1829, 1828, 1839, 1838, 1837, 1844, 1843, - 2086, 2085, 1639, 2004, 2002, 1999, 1997, 2011, 2007, 1727, 1720, 1726, - 1915, 1883, 1947, 1657, 1788, 1797, 2073, 2080, 2087, 1822, 1861, 1830, - 1859, 1778, 1610, 1751, 1835, 1813, 1786, 1750, 1787, 1836, 1721, 1706, - 1819, 1682, 1634, 1749, 1613, 1886, 1916, 1840, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 1898, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 1899, 1870, 1631, 1632, - 1827, 1897, 1879, 1626, 2037, 1900, 1901, 1734, 27339, 1655, 1975, 1633, - 33083, 33194, 33262, 33273, 33284, 33295, 33306, 33317, 33328, 33084, - 33095, 33106, 33117, 33128, 33139, 33150, 26827, 26832, 26833, 26826, - 26857, 26828, 26859, 26835, 26849, 26831, 35762, 35762, 35762, 35762, - 35762, 35762, 8057, 8059, 7889, 7890, 8068, 8070, 7783, 8058, 8060, 8126, - 8127, 8069, 8071, 7784, 7837, 7838, 26858, 26829, 26830, 26844, 26856, - 26841, 26853, 26839, 26851, 26843, 26855, 26836, 26845, 26837, 26846, - 26840, 26852, 26838, 26850, 26834, 26847, 27753, 34075, 26842, 26854, - 10235, 5828, 33955, 10947, 10234, 5827, 33953, 28892, 28901, 28935, - 35762, 28926, 28893, 28936, 28899, 28900, 28903, 28907, 28902, 28906, - 28904, 28908, 28934, 28883, 28885, 28933, 28931, 28905, 28930, 28898, - 35762, 28925, 28894, 28932, 28891, 35762, 35762, 35762, 35762, 1080, - 2371, 1062, 2373, 1092, 35762, 1077, 1078, 1058, 1059, 1089, 1090, 2278, - 2279, 2346, 2347, 1276, 1141, 1140, 1131, 1130, 1556, 1555, 1134, 1133, - 1573, 1571, 1572, 1574, 1151, 1150, 1166, 1164, 1165, 1167, 1499, 1498, - 1508, 1506, 1507, 1501, 1520, 1518, 1519, 1521, 1307, 1305, 1306, 1308, - 1273, 1271, 1272, 1274, 1344, 1342, 1343, 1345, 1189, 1188, 1516, 1515, - 1435, 1434, 1598, 1597, 1464, 1462, 1463, 1465, 1470, 1468, 1469, 1471, - 1451, 1449, 1450, 1452, 1197, 1195, 1196, 1198, 1482, 1480, 1481, 1483, - 1594, 1592, 1593, 1595, 1109, 1107, 1108, 1110, 1252, 1250, 1251, 1253, - 1236, 1234, 1235, 1237, 1417, 1415, 1416, 1418, 1320, 1318, 1319, 1321, - 1356, 1354, 1355, 1357, 1365, 1363, 1364, 1366, 1396, 1394, 1395, 1397, - 1295, 1293, 1294, 1296, 1560, 1559, 1149, 1148, 1583, 1581, 1582, 1584, - 1772, 1771, 1768, 1767, 1770, 1769, 1776, 1775, 35762, 35762, 35566, - 35762, 13299, 13303, 13271, 13287, 13273, 13285, 13284, 13214, 13277, - 13286, 13274, 13209, 13302, 13308, 13281, 13294, 13296, 13293, 13292, - 13289, 13288, 13291, 13290, 13297, 13295, 13210, 13280, 13216, 13298, - 13301, 13304, 13208, 13217, 13218, 13219, 13220, 13221, 13222, 13223, - 13224, 13225, 13226, 13227, 13228, 13229, 13230, 13231, 13232, 13233, - 13234, 13235, 13236, 13237, 13238, 13239, 13240, 13241, 13242, 13215, - 13279, 13278, 13207, 13269, 13300, 13243, 13244, 13245, 13246, 13247, - 13248, 13249, 13250, 13251, 13252, 13253, 13254, 13255, 13256, 13257, - 13258, 13259, 13260, 13261, 13262, 13263, 13264, 13265, 13266, 13267, - 13268, 13212, 13309, 13275, 13306, 13213, 13276, 14644, 14637, 14639, - 14643, 14635, 14627, 14582, 14584, 14586, 14583, 14585, 14578, 14580, - 14579, 14581, 14636, 14628, 14630, 14632, 14629, 14631, 14603, 14605, - 14607, 14604, 14606, 14587, 14589, 14591, 14588, 14590, 14618, 14620, - 14622, 14619, 14621, 14593, 14595, 14597, 14594, 14596, 14598, 14600, - 14602, 14599, 14601, 14608, 14610, 14612, 14609, 14611, 14623, 14625, - 14624, 14613, 14615, 14617, 14614, 14616, 14626, 14592, 14634, 14633, - 14577, 14527, 14544, 14528, 14529, 14530, 14531, 14565, 14546, 14535, - 14540, 14539, 14538, 14542, 14536, 14537, 14541, 14563, 14533, 14545, - 14534, 14548, 14547, 14562, 14557, 14543, 14556, 14526, 14564, 14532, - 14571, 35762, 35762, 35762, 14572, 14573, 14551, 14552, 14559, 14558, - 35762, 35762, 14550, 14549, 14574, 14568, 14569, 14575, 35762, 35762, - 14554, 14576, 14567, 14566, 14570, 14555, 35762, 35762, 14560, 14553, - 14561, 35762, 35762, 35762, 13211, 13272, 13270, 13283, 13307, 13282, - 13305, 35762, 14642, 14638, 14641, 14640, 14647, 14645, 14646, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 16324, - 16326, 16325, 25221, 27062, 35762, 35762, 20348, 20374, 20367, 20400, - 20358, 20349, 20378, 20344, 20356, 20386, 20389, 20383, 35762, 20372, - 20399, 20403, 20382, 20397, 20404, 20411, 20414, 20359, 20410, 20357, - 20360, 20343, 20366, 20369, 20392, 20393, 20351, 20408, 20373, 20354, - 20390, 20352, 20409, 20368, 20376, 35762, 20401, 20365, 20385, 20350, - 20361, 20375, 20346, 20380, 20355, 20387, 20388, 20345, 20371, 20347, - 20398, 20391, 20405, 20379, 20381, 35762, 20353, 20407, 35762, 20364, - 20363, 20377, 20413, 20406, 20416, 20384, 20362, 20394, 20402, 20370, - 20395, 20396, 20412, 20415, 35762, 35762, 20426, 20427, 20429, 20428, - 20417, 20418, 20425, 20419, 20420, 20430, 20421, 20422, 20423, 20424, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 20232, 20231, 20233, 20220, 20221, 20222, - 20223, 20224, 20225, 20226, 20228, 20227, 20230, 20229, 20238, 20235, - 20236, 20234, 20237, 20337, 20338, 20240, 20239, 20241, 20340, 20339, - 20243, 20244, 20245, 20242, 20246, 20249, 20248, 20250, 20251, 20252, - 20341, 20253, 20254, 20247, 20257, 20258, 20256, 20255, 20259, 20260, - 20261, 20262, 20263, 20264, 20267, 20268, 20269, 20266, 20270, 20265, - 20271, 20272, 20273, 20274, 20275, 20276, 20277, 20278, 20279, 20280, - 20282, 20281, 20283, 20284, 20286, 20287, 20288, 20285, 20289, 20290, - 20291, 20292, 20294, 20293, 20295, 20296, 20342, 20297, 20298, 20300, - 20301, 20302, 20299, 20303, 20304, 20305, 20306, 20307, 20335, 20315, - 20316, 20317, 20318, 20319, 20320, 20321, 20322, 20323, 20324, 20325, - 20326, 20327, 20328, 20329, 20330, 20331, 20332, 20333, 20334, 20308, - 20309, 20310, 20311, 20312, 20313, 20314, 20336, 35762, 35762, 35762, - 35762, 35762, 112, 113, 159, 35762, 35762, 35762, 35762, 156, 151, 146, - 126, 121, 139, 134, 114, 129, 154, 149, 144, 124, 119, 140, 135, 115, - 130, 157, 152, 147, 127, 122, 142, 137, 117, 132, 158, 153, 148, 128, - 123, 143, 138, 118, 133, 155, 150, 145, 125, 120, 141, 136, 116, 131, - 35762, 35762, 35762, 111, 107, 109, 110, 108, 103, 104, 105, 106, 13770, - 13767, 13771, 13757, 13752, 13758, 13761, 13753, 13763, 13772, 13755, - 13765, 13759, 13768, 13762, 13764, 13774, 13756, 13766, 13760, 13769, - 13773, 13754, 13775, 13787, 13796, 13786, 13782, 13795, 13777, 13783, - 13799, 13803, 13804, 13785, 13788, 13794, 13793, 13801, 13802, 13784, - 13791, 13797, 13792, 13781, 13800, 13789, 13776, 13778, 13798, 13790, - 13779, 13780, 14022, 14023, 14221, 14217, 14258, 14223, 13953, 14027, - 14220, 14216, 13959, 13958, 14019, 14001, 14017, 14026, 14021, 13807, - 13806, 14260, 14219, 14261, 14024, 14215, 13997, 24651, 35762, 27384, - 27387, 27382, 27385, 27356, 27386, 27354, 27357, 27383, 27355, 27388, - 27353, 2510, 35762, 35762, 35762, 14214, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 26611, 26612, 26623, 26592, 26595, 26626, 26604, - 26603, 26624, 26631, 26590, 26617, 26596, 26609, 26610, 26619, 26608, - 26589, 26593, 26600, 26598, 26620, 26597, 26588, 26618, 26605, 26606, - 26591, 26594, 26616, 26630, 26601, 26625, 26587, 26613, 26632, 26614, - 26615, 26607, 26629, 26628, 26602, 26621, 26622, 26627, 26599, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 20605, - 20607, 20600, 20601, 20612, 20611, 20614, 20622, 20624, 20603, 20615, - 20598, 20618, 20616, 20596, 20609, 20597, 20610, 20621, 20617, 20599, - 20619, 20620, 20602, 20604, 20606, 20608, 20613, 20623, 35762, 35762, - 35762, 5738, 5749, 5740, 5711, 5734, 5750, 5712, 5739, 5756, 5754, 5714, - 5755, 5741, 5729, 5724, 5725, 5723, 5709, 5732, 5722, 5757, 5719, 5731, - 5748, 5728, 5752, 5742, 5737, 5746, 5747, 5720, 5733, 5744, 5745, 5726, - 5727, 5721, 5753, 5710, 5730, 5735, 5751, 5715, 5716, 5717, 5718, 5713, - 5743, 5736, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 8241, 8267, 8265, 8264, - 8261, 8260, 8263, 8262, 8268, 8266, 8259, 8258, 8256, 8247, 8245, 8254, - 8252, 8243, 8249, 8250, 8257, 8255, 8246, 8244, 8253, 8251, 8242, 8248, - 35762, 35762, 35762, 35762, 25459, 25453, 25439, 25454, 25426, 25456, - 25458, 25455, 25450, 25446, 25438, 25434, 25435, 25436, 25428, 25460, - 25449, 25442, 25440, 25430, 25427, 25451, 25444, 25432, 25448, 25437, - 25433, 25431, 25452, 25447, 25445, 25429, 25464, 25462, 25463, 25461, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 25457, - 25443, 25441, 13617, 13633, 13641, 13635, 13616, 13626, 13620, 13619, - 13628, 13638, 13642, 13639, 13636, 13624, 13640, 13631, 13625, 13623, - 13627, 13637, 13629, 13630, 13632, 13621, 13618, 13634, 13622, 35762, - 35762, 35762, 35762, 35762, 25531, 25530, 25527, 25500, 25501, 25523, - 25498, 25524, 25499, 25503, 25532, 25525, 25506, 25507, 25514, 25508, - 25526, 25511, 25513, 25534, 25497, 25510, 25509, 25521, 25518, 25528, - 25529, 25502, 25533, 25512, 25515, 25516, 25517, 25520, 25505, 25522, - 25519, 25504, 8083, 8081, 8079, 8080, 8082, 35762, 35762, 35762, 35762, - 35762, 32585, 32588, 32592, 32596, 32589, 32593, 32611, 32607, 32594, - 32604, 32606, 32595, 32601, 32597, 32612, 32590, 32609, 32608, 32600, - 32586, 32610, 32599, 32587, 32598, 32603, 32591, 32605, 32613, 32614, - 32602, 35762, 32615, 25540, 25582, 25583, 25563, 25564, 25561, 25562, - 25560, 25575, 25552, 25553, 25557, 25558, 25547, 25550, 25551, 25556, - 25579, 25544, 25576, 25565, 25566, 25569, 25570, 25571, 25580, 25554, - 25555, 25567, 25568, 25578, 25574, 25581, 25572, 25573, 25577, 35762, - 35762, 35762, 35762, 25541, 25542, 25543, 25559, 25548, 25549, 25545, - 25546, 25584, 25539, 25536, 25537, 25535, 25538, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 10286, - 10285, 10281, 10282, 10283, 10284, 10292, 10291, 10287, 10288, 10289, - 10290, 10309, 10294, 10308, 10305, 10310, 10303, 10300, 10296, 10301, - 10299, 10302, 10307, 10306, 10276, 10304, 10275, 10295, 10271, 10298, - 10272, 10297, 10279, 10277, 10278, 10273, 10274, 10293, 10280, 10326, - 10325, 10321, 10322, 10323, 10324, 10332, 10331, 10327, 10328, 10329, - 10330, 10349, 10334, 10348, 10345, 10350, 10343, 10340, 10336, 10341, - 10339, 10342, 10347, 10346, 10316, 10344, 10315, 10335, 10311, 10338, - 10312, 10337, 10319, 10317, 10318, 10313, 10314, 10333, 10320, 27890, - 27896, 27907, 27904, 27894, 27893, 27892, 27871, 27899, 27877, 27906, - 27902, 27905, 27908, 27895, 27903, 27882, 27901, 27898, 27876, 27881, - 27883, 27880, 27875, 27869, 27866, 27888, 27897, 27886, 27870, 27891, - 27909, 27873, 27867, 27879, 27910, 27887, 27884, 27885, 27868, 27864, - 27889, 27865, 27874, 27863, 27872, 27878, 27900, 25971, 25989, 25995, - 25993, 25996, 25977, 25974, 25994, 25979, 25978, 25975, 25973, 25991, - 25990, 25981, 25976, 25984, 25980, 25985, 25986, 25992, 25997, 25970, - 25987, 25998, 25982, 25999, 25972, 25988, 25983, 35762, 35762, 26006, - 26008, 26005, 26004, 26001, 26000, 26003, 26002, 26009, 26007, 35762, - 35762, 35762, 35762, 35762, 35762, 25898, 25899, 25900, 25901, 25926, - 25919, 25905, 25902, 25908, 25918, 25917, 25932, 25911, 25906, 25910, - 25927, 25928, 25929, 25912, 25913, 25930, 25907, 25923, 25922, 25916, - 25904, 25915, 25903, 25914, 25920, 25933, 25931, 25909, 25921, 25925, - 25924, 35762, 35762, 35762, 35762, 25934, 25935, 25936, 25937, 25962, - 25955, 25941, 25938, 25944, 25954, 25953, 25968, 25947, 25942, 25946, - 25963, 25964, 25965, 25948, 25949, 25966, 25943, 25959, 25958, 25952, - 25940, 25951, 25939, 25950, 25956, 25969, 25967, 25945, 25957, 25961, - 25960, 35762, 35762, 35762, 35762, 12376, 12367, 12356, 12355, 12358, - 12347, 12357, 12354, 12353, 12368, 12344, 12343, 12369, 12377, 12370, - 12360, 12346, 12345, 12371, 12350, 12349, 12348, 12378, 12372, 12373, - 12352, 12351, 12362, 12361, 12364, 12363, 12379, 12374, 12375, 12380, - 12366, 12365, 12342, 12341, 12359, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 5770, 5819, 5786, 5782, 5784, 5809, 5783, 5807, - 5804, 5773, 5806, 5808, 5787, 5798, 5794, 5789, 5817, 5781, 5772, 5790, - 5793, 5795, 5820, 5811, 5769, 5775, 5776, 5778, 5812, 5810, 5815, 5779, - 5799, 5791, 5818, 5803, 5814, 5780, 5774, 5797, 5785, 5816, 5801, 5813, - 5802, 5800, 5788, 5777, 5771, 5805, 5796, 5792, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 5821, 33727, - 33696, 33697, 33707, 33706, 33709, 33708, 33699, 33698, 33716, 33724, - 35762, 33715, 33714, 33700, 33701, 33717, 33725, 33703, 33702, 33718, - 33705, 33704, 33728, 33719, 33726, 33720, 35762, 33711, 33710, 33713, - 33712, 33729, 33721, 33722, 35762, 33730, 33723, 35762, 33762, 33731, - 33732, 33742, 33741, 33744, 33743, 33734, 33733, 33751, 33759, 35762, - 33750, 33749, 33735, 33736, 33752, 33760, 33738, 33737, 33753, 33740, - 33739, 33763, 33754, 33761, 33755, 35762, 33746, 33745, 33748, 33747, - 33764, 33756, 33757, 35762, 33765, 33758, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 20199, 20200, 20201, 20202, 20203, 20204, 20205, - 20206, 20207, 20138, 20139, 20140, 20141, 20142, 20151, 20143, 20144, - 20145, 20146, 20147, 20148, 20149, 20150, 20152, 20153, 20154, 20155, - 20219, 20156, 20157, 20158, 20159, 20160, 20161, 20162, 20163, 20164, - 20165, 20166, 20167, 20168, 20169, 20170, 20171, 20172, 20173, 20174, - 20175, 20176, 20177, 20178, 20179, 20180, 20181, 20182, 20183, 20184, - 20185, 20186, 20187, 20188, 20189, 20190, 20191, 20192, 20193, 20194, - 20195, 20196, 20197, 20198, 19879, 20215, 20208, 19880, 20209, 20210, - 20211, 20212, 19881, 20216, 20217, 20213, 20214, 20218, 19885, 19886, - 19887, 19888, 19889, 19890, 19891, 19892, 19882, 19883, 19884, 19896, - 19897, 19898, 19893, 19894, 19895, 19899, 19900, 19901, 19902, 19903, - 19904, 19907, 19908, 19909, 19910, 19911, 19912, 19913, 19914, 19915, - 19916, 19917, 19918, 19919, 19920, 19921, 19922, 19923, 19924, 19925, - 19926, 19927, 19928, 19929, 19930, 19931, 19932, 19933, 19934, 19935, - 19936, 19937, 19938, 19939, 19940, 19941, 19942, 19943, 19944, 19945, - 19946, 19947, 19948, 19949, 19950, 19951, 19952, 19953, 19954, 19955, - 19956, 19905, 19906, 19957, 19958, 19959, 19960, 19961, 19962, 19963, - 19964, 19965, 19966, 19967, 19968, 19969, 19970, 19971, 19972, 19973, - 19974, 19975, 19976, 19977, 19978, 19979, 19980, 19981, 19982, 19983, - 19984, 19985, 19986, 19987, 19988, 19989, 20021, 20022, 20023, 20024, - 20025, 20026, 20027, 20028, 20029, 19990, 19991, 19992, 19993, 19994, - 19995, 19996, 19997, 19998, 19999, 20000, 20001, 20002, 20003, 20004, - 20005, 20006, 20007, 20008, 20009, 20010, 20011, 20012, 20013, 20014, - 20015, 20016, 20017, 20018, 20019, 20020, 20036, 20037, 20038, 20039, - 20040, 20041, 20042, 20043, 20044, 20045, 20046, 20047, 20048, 20049, - 20050, 20051, 20052, 20053, 20054, 20055, 20030, 20031, 20032, 20033, - 20034, 20035, 20056, 20057, 20058, 20059, 20060, 20061, 20062, 20063, - 20098, 20099, 20100, 20101, 20102, 20103, 20104, 20105, 20106, 20107, - 20064, 20065, 20066, 20067, 20068, 20069, 20070, 20071, 20072, 20073, - 20074, 20075, 20076, 20077, 20078, 20079, 20080, 20081, 20082, 20083, - 20089, 20090, 20091, 20092, 20093, 20094, 20095, 20096, 20097, 20084, - 20085, 20086, 20087, 20088, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 20119, 20114, 20117, 20116, 20120, 20115, 20113, - 20118, 20112, 20108, 20111, 20110, 20109, 20126, 20121, 20125, 20122, - 20123, 20124, 20127, 20129, 20128, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 20130, 20131, 20132, 20133, 20134, - 20135, 20136, 20137, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 23185, 23306, 23305, - 23170, 23186, 23174, 35762, 23204, 23206, 23205, 23200, 23199, 23197, - 23198, 23259, 23192, 23216, 23256, 23180, 23220, 23181, 23224, 23187, - 23226, 23203, 23240, 23241, 23239, 23183, 23237, 23242, 23243, 23284, - 23285, 23248, 23184, 23193, 23299, 23281, 23282, 23255, 23254, 23189, - 23269, 23272, 23273, 23270, 23268, 23296, 35762, 23191, 23111, 23158, - 23006, 23089, 23118, 23003, 23160, 23261, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 9704, 9705, 9706, 9707, 9708, 9698, - 35762, 35762, 9699, 35762, 9654, 9655, 9656, 9657, 9658, 9659, 9660, - 9661, 9662, 9663, 9664, 9665, 9666, 9667, 9668, 9669, 9670, 9671, 9672, - 9673, 9674, 9675, 9676, 9677, 9678, 9679, 9680, 9681, 9682, 9683, 9684, - 9685, 9686, 9687, 9688, 9689, 9690, 9691, 9692, 9693, 9694, 9695, 9696, - 9697, 35762, 9702, 9703, 35762, 35762, 35762, 9700, 35762, 35762, 9701, - 16146, 16157, 16148, 16142, 16154, 16159, 16149, 16155, 16140, 16153, - 16156, 16143, 16161, 16158, 16150, 16147, 16160, 16151, 16144, 16145, - 16152, 16141, 35762, 16162, 16137, 16133, 16136, 16134, 16132, 16138, - 16139, 16135, 26241, 26252, 26243, 26237, 26249, 26254, 26244, 26250, - 26235, 26248, 26251, 26238, 26256, 26234, 26253, 26245, 26242, 26255, - 26246, 26239, 26240, 26247, 26236, 26233, 26257, 26264, 26259, 26260, - 26263, 26262, 26261, 26258, 24101, 24116, 24106, 24127, 24118, 24112, - 24108, 24124, 24129, 24119, 24125, 24110, 24104, 24123, 24105, 24126, - 24102, 24113, 24109, 24131, 24107, 24128, 24120, 24117, 24130, 24121, - 24114, 24115, 24103, 24122, 24111, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 24136, 24133, 24134, 24139, 24140, 24138, 24135, - 24132, 24137, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 15202, 15218, 15210, 15209, 15215, 15220, 15204, 15216, 15205, 15214, - 15217, 15207, 15222, 15219, 15211, 15203, 15221, 15212, 15208, 35762, - 15213, 15206, 35762, 35762, 35762, 35762, 35762, 15223, 15227, 15226, - 15225, 15224, 26635, 26654, 26650, 26637, 26638, 26646, 26647, 26639, - 26645, 26648, 26651, 26653, 26656, 26652, 26643, 26636, 26655, 26641, - 26640, 26649, 26642, 26644, 26661, 26660, 26657, 26662, 26658, 26659, - 35762, 35762, 35762, 26663, 20631, 20637, 20641, 20639, 20633, 20649, - 20642, 20650, 20643, 20627, 20644, 20635, 20645, 20647, 20630, 20625, - 20648, 20640, 20646, 20629, 20626, 20632, 20634, 20628, 20636, 20638, - 35762, 35762, 35762, 35762, 35762, 20651, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 22722, 22723, 22724, 22725, 22721, 22720, 22696, 22697, 22718, 22717, - 22700, 22701, 22702, 22703, 22698, 22699, 22716, 22713, 22712, 22704, - 22705, 22706, 22714, 22719, 22707, 22708, 22709, 22710, 22711, 22715, - 22726, 22727, 22618, 22639, 22640, 22641, 22638, 22637, 22630, 22634, - 22633, 22623, 22624, 22636, 22632, 22628, 22627, 22625, 22619, 22626, - 22629, 22635, 22620, 22621, 22622, 22631, 35762, 35762, 35762, 35762, - 22606, 22611, 22643, 22642, 22692, 22684, 22678, 22655, 22649, 22672, - 22666, 22644, 22661, 22690, 22688, 22682, 22659, 22653, 22676, 22670, - 35762, 35762, 22693, 22685, 22679, 22656, 22650, 22673, 22667, 22646, - 22663, 22695, 22687, 22681, 22658, 22652, 22675, 22669, 22648, 22665, - 22691, 22689, 22683, 22660, 22654, 22677, 22671, 22645, 22662, 22694, - 22686, 22680, 22657, 22651, 22674, 22668, 22647, 22664, 22610, 22616, - 22615, 22609, 22608, 22613, 22612, 22607, 22617, 22614, 17200, 17222, - 17224, 17220, 35762, 17221, 17223, 35762, 35762, 35762, 35762, 35762, - 17225, 17216, 17219, 17218, 17166, 17164, 17188, 17187, 35762, 17186, - 17185, 17194, 35762, 17174, 17170, 17169, 17177, 17176, 17173, 17172, - 17171, 17179, 17178, 17175, 17190, 17189, 17184, 17183, 17196, 17198, - 17197, 17195, 17192, 17180, 17181, 17182, 17199, 17193, 17165, 17167, - 17168, 17191, 35762, 35762, 17214, 17215, 17217, 35762, 35762, 35762, - 35762, 17226, 17163, 17162, 17161, 17160, 17202, 17201, 17203, 17204, - 17227, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 17208, 17213, - 17206, 17205, 17212, 17210, 17209, 17207, 17211, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 25648, 25644, 25649, 25654, 25645, 25652, - 25638, 25646, 25650, 25642, 25637, 25633, 25651, 25634, 25636, 25635, - 25653, 25626, 25627, 25629, 25632, 25630, 25631, 25641, 25643, 25628, - 25647, 25640, 25639, 25656, 25655, 25657, 25470, 25488, 25469, 25479, - 25491, 25492, 25481, 25485, 25483, 25476, 25480, 25472, 25487, 25471, - 25493, 25482, 25484, 25465, 25466, 25489, 25468, 25490, 25467, 25475, - 25477, 25473, 25486, 25474, 25478, 25496, 25495, 25494, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 20932, 20936, 20935, 20940, 20939, 20937, 20961, 20964, 20980, 20944, - 20943, 20942, 20941, 20962, 20954, 20960, 20946, 20956, 20945, 20958, - 20938, 20953, 20967, 20963, 20950, 20934, 20933, 20966, 20965, 20951, - 20948, 20957, 20947, 20959, 20952, 20949, 20955, 20982, 20981, 35762, - 35762, 35762, 35762, 20968, 20972, 20971, 20970, 20969, 20979, 20977, - 20975, 20974, 20973, 20978, 20976, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 2526, 2527, 2533, 2529, 2532, 2528, 2530, - 2531, 2569, 2570, 2559, 2560, 2561, 2562, 2557, 2558, 2574, 2547, 2546, - 2545, 2536, 2534, 2535, 2571, 2573, 2556, 2554, 2566, 2565, 2555, 2577, - 2572, 2564, 2563, 2541, 2540, 2539, 2544, 2543, 2542, 2576, 2537, 2552, - 2553, 2579, 2578, 2575, 2551, 2568, 2549, 2567, 2548, 2550, 2538, 35762, - 35762, 35762, 2580, 32334, 28927, 18140, 18135, 18141, 18136, 16288, - 16299, 16290, 16284, 16296, 16301, 16291, 16297, 16282, 16295, 16298, - 16285, 16303, 16300, 16292, 16289, 16302, 16293, 16286, 16287, 16294, - 16283, 35762, 35762, 16308, 16305, 16306, 16311, 16307, 16304, 16309, - 16310, 16257, 16271, 16262, 16258, 16268, 16261, 16263, 16269, 16255, - 16267, 16270, 16259, 16260, 16272, 16264, 16273, 16265, 16266, 16256, - 35762, 35762, 35762, 35762, 35762, 16278, 16275, 16276, 16281, 16277, - 16274, 16279, 16280, 26895, 26909, 26900, 26896, 26906, 26899, 26901, - 26907, 26905, 26908, 26897, 26898, 26910, 26902, 26912, 26903, 26904, - 26911, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 26920, 26921, - 26893, 26894, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 26917, 26914, 26915, 26919, 26916, 26913, - 26918, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 25658, 25700, 25701, 25690, 25726, 25719, 25693, 25694, 25728, - 25671, 25711, 25659, 25704, 25673, 25713, 25661, 25705, 25672, 25712, - 25660, 25689, 25725, 25679, 25718, 25668, 25708, 25662, 25706, 25695, - 25729, 25674, 25714, 25663, 25684, 25687, 25675, 25664, 25702, 25682, - 25721, 25680, 25720, 25683, 25722, 25710, 25681, 25703, 25688, 25696, - 25691, 25686, 25724, 25676, 25715, 25692, 25727, 25697, 25730, 25677, - 25716, 25665, 25669, 25666, 25670, 25709, 25685, 25723, 25678, 25717, - 25667, 25707, 25698, 25699, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 25318, - 25321, 25344, 25319, 25333, 25329, 25335, 25345, 25320, 25323, 25365, - 25346, 25347, 25336, 25337, 25348, 25366, 25367, 25349, 25350, 25322, - 25362, 25338, 25339, 25324, 25326, 25330, 25358, 25360, 25354, 25356, - 25359, 25351, 25325, 25352, 25368, 25331, 25332, 25340, 25327, 25341, - 25334, 25361, 25364, 25355, 25357, 25353, 25342, 25343, 25328, 25363, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 25369, 25372, 25395, 25370, 25384, 25380, 25386, - 25396, 25371, 25374, 25416, 25397, 25398, 25387, 25388, 25399, 25417, - 25418, 25400, 25401, 25373, 25413, 25389, 25390, 25375, 25377, 25381, - 25409, 25411, 25405, 25407, 25410, 25402, 25376, 25403, 25419, 25382, - 25383, 25391, 25378, 25392, 25385, 25412, 25415, 25406, 25408, 25404, - 25393, 25394, 25379, 25414, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 25422, 25421, 25425, 25420, 25423, 25424, 15157, 15144, 15152, - 15136, 15135, 15149, 15145, 15148, 15133, 15146, 15130, 15129, 15138, - 15137, 15156, 15143, 15142, 15134, 15147, 15150, 15151, 15141, 15154, - 15131, 15155, 15132, 15139, 15140, 15153, 15164, 15166, 15168, 15165, - 15167, 15159, 15158, 15163, 15160, 15162, 15161, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 15175, 15177, 15174, 15173, 15170, - 15169, 15172, 15171, 15178, 15176, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 27439, - 27437, 27436, 27433, 27432, 27435, 27434, 27440, 27438, 27431, 27430, - 27428, 27419, 27417, 27426, 27424, 27415, 27421, 27422, 27429, 27427, - 27418, 27416, 27425, 27423, 27414, 27420, 27411, 27412, 27410, 27413, - 35762, 34219, 34253, 34233, 34232, 34238, 34237, 34215, 34214, 34213, - 34224, 34245, 34218, 34249, 34252, 34251, 34248, 34256, 34235, 34234, - 34236, 34217, 34239, 34250, 34220, 34244, 34255, 34240, 34241, 34228, - 34226, 34225, 34227, 34229, 34216, 34231, 34254, 34242, 34243, 34222, - 34223, 34246, 34221, 35762, 34211, 34210, 34212, 35762, 35762, 34230, - 34247, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 2335, 2336, 2333, 25588, 25596, - 25610, 25597, 25601, 25607, 25598, 25613, 25602, 25608, 25606, 25609, - 25600, 25615, 25611, 25590, 25591, 25603, 25589, 25587, 25614, 25604, - 25592, 25593, 25599, 25605, 25612, 25594, 25595, 25622, 25620, 25617, - 25625, 25624, 25621, 25619, 25618, 25623, 25586, 25616, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 28985, 28999, 28987, 28996, - 29003, 28991, 28997, 28995, 28998, 28988, 29005, 29001, 28992, 28986, - 29004, 28993, 28990, 28994, 29002, 29000, 28989, 28984, 28979, 28977, - 28980, 28978, 28983, 28982, 28974, 28973, 28975, 28981, 28976, 29006, - 29009, 29008, 29007, 29012, 29013, 29010, 29014, 29011, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 25735, 25747, 25745, 25750, 25739, 25744, 25743, 25746, 25737, 25752, - 25748, 25740, 25751, 25741, 25736, 25742, 25749, 25738, 25734, 25733, - 25732, 25731, 25756, 25753, 25754, 25755, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 6190, 6184, 6198, 6192, 6187, 6195, 6201, - 6183, 6193, 6196, 6194, 6197, 6188, 6203, 6199, 6185, 6191, 6202, 6189, - 6186, 6200, 6208, 6205, 6206, 6210, 6207, 6204, 6209, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 12403, 12414, - 12405, 12399, 12411, 12416, 12406, 12412, 12397, 12410, 12413, 12400, - 12418, 12415, 12407, 12404, 12417, 12408, 12401, 12402, 12409, 12398, - 12419, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 4335, 4339, 4337, 4336, 4338, 4261, 4262, 4280, 4281, 4278, 4279, 4273, - 4274, 4275, 4276, 4307, 4263, 4254, 4264, 4300, 4299, 4296, 4295, 4284, - 4294, 4293, 4298, 4297, 4286, 4270, 4269, 4266, 4265, 4285, 4272, 4271, - 4268, 4267, 4287, 4302, 4301, 4292, 4291, 4304, 4306, 4305, 4283, 4277, - 4288, 4289, 4290, 4303, 4282, 4255, 4260, 4259, 4344, 4343, 4353, 4354, - 4351, 4352, 4347, 4348, 4349, 4350, 4355, 4345, 4340, 4346, 4356, 4358, - 4357, 4332, 4331, 4330, 4333, 4329, 35762, 35762, 35762, 35762, 4325, - 4323, 4320, 4311, 4313, 4318, 4316, 4308, 4314, 4324, 4322, 4321, 4310, - 4312, 4319, 4317, 4309, 4315, 4326, 4327, 4365, 4367, 4364, 4363, 4360, - 4359, 4362, 4361, 4368, 4366, 4334, 4257, 4258, 4341, 4342, 4256, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 4328, 16513, - 16515, 16517, 16473, 16474, 16483, 16484, 16481, 16482, 16511, 16475, - 16512, 16476, 16501, 16500, 16497, 16496, 16485, 16495, 16494, 16499, - 16498, 16487, 16478, 16477, 16470, 16468, 16469, 16504, 16486, 16480, - 16479, 16472, 16471, 16488, 16503, 16502, 16493, 16492, 16508, 16510, - 16505, 16507, 16509, 16489, 16490, 16491, 16506, 16523, 16528, 16529, - 16526, 16527, 16530, 16524, 16531, 16525, 16516, 16514, 16534, 16535, - 16532, 16518, 16519, 16521, 16520, 16522, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 16533, 35762, 35762, 29037, - 29038, 29027, 29028, 29029, 29030, 29023, 29024, 29034, 29026, 29039, - 29035, 29040, 29036, 29031, 29033, 29032, 29025, 29041, 29020, 29042, - 29044, 29043, 29021, 29022, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 29051, 29053, 29050, 29049, 29046, 29045, 29048, 29047, 29054, - 29052, 35762, 35762, 35762, 35762, 35762, 35762, 5873, 5875, 5874, 5869, - 5871, 5872, 5870, 5858, 5857, 5854, 5853, 5839, 5852, 5851, 5856, 5855, - 5841, 5844, 5843, 5836, 5835, 5840, 5846, 5845, 5838, 5837, 5842, 5862, - 5861, 5850, 5849, 5864, 5847, 5848, 5865, 5860, 5868, 5866, 5863, 5877, - 5885, 5886, 5881, 5882, 5883, 5879, 5887, 5880, 5888, 5905, 5904, 5889, - 5903, 35762, 5898, 5900, 5897, 5896, 5893, 5892, 5895, 5894, 5901, 5899, - 5876, 5891, 5890, 5902, 5859, 5878, 5884, 5867, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 20693, 20695, 20697, 20694, 20696, - 20685, 20684, 20681, 20680, 20679, 20678, 20683, 20682, 20664, 20673, - 20672, 20669, 20668, 20663, 20675, 20674, 20671, 20670, 20665, 20687, - 20686, 20677, 20676, 20690, 20667, 20689, 20692, 20691, 20688, 20666, - 20700, 20701, 20699, 20698, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 27836, 27835, 27841, 27780, 27781, 27799, 27800, - 27797, 27798, 27792, 27793, 27794, 27795, 27826, 27782, 27827, 27783, - 27819, 27818, 27815, 27814, 27803, 27813, 27812, 27817, 27816, 27805, - 27789, 27788, 27785, 27784, 27804, 27791, 27790, 27787, 27786, 27806, - 27821, 27820, 27811, 27810, 27823, 27825, 27824, 27802, 27801, 27796, - 27807, 27808, 27809, 27822, 27845, 27854, 27855, 27852, 27853, 27848, - 27849, 27850, 27851, 27856, 27846, 27857, 27847, 27840, 27834, 27838, - 27839, 27860, 27766, 27765, 27858, 27831, 27828, 27837, 27843, 27777, - 27842, 27844, 27832, 27773, 27775, 27772, 27771, 27768, 27767, 27770, - 27769, 27776, 27774, 27778, 27833, 27779, 27859, 27829, 27830, 35762, - 28741, 28739, 28738, 28735, 28734, 28737, 28736, 28742, 28740, 28753, - 28752, 28751, 28747, 28746, 28750, 28749, 28745, 28748, 28743, 28744, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 17905, 17906, 17932, 17934, 17931, 17907, 17933, 17908, 17922, - 17921, 17897, 17895, 17896, 17915, 17920, 17919, 17904, 17903, 35762, - 17917, 17910, 17909, 17900, 17899, 17916, 17912, 17911, 17902, 17898, - 17901, 17918, 17924, 17923, 17894, 17892, 17893, 17926, 17930, 17928, - 17914, 17929, 17891, 17925, 17913, 17942, 17945, 17946, 17949, 17947, - 17943, 17948, 17944, 17939, 17937, 17938, 17935, 17889, 17888, 17950, - 17940, 17887, 17951, 17936, 17927, 17890, 17941, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 23597, - 23599, 23600, 23598, 23588, 23587, 23586, 35762, 23585, 35762, 23584, - 23583, 23576, 23575, 35762, 23570, 23578, 23577, 23566, 23564, 23565, - 23569, 23580, 23579, 23568, 23567, 23571, 23590, 23589, 23582, 35762, - 23581, 23593, 23596, 23574, 23592, 23595, 23594, 23591, 23573, 23572, - 23601, 35762, 35762, 35762, 35762, 35762, 35762, 17968, 17969, 17978, - 17979, 17976, 17977, 17997, 17970, 17998, 17971, 17987, 17986, 17957, - 17955, 17956, 17980, 17985, 17984, 17960, 17959, 17958, 17982, 17973, - 17972, 17963, 17961, 17966, 17962, 17981, 17975, 17974, 17965, 17964, - 17983, 17989, 17988, 17954, 17952, 17953, 17994, 17996, 17967, 17993, - 17995, 17990, 17991, 17992, 18001, 18002, 18007, 18008, 18005, 18006, - 18009, 18003, 18010, 18004, 18000, 17999, 35762, 35762, 35762, 35762, - 35762, 18017, 18019, 18016, 18015, 18012, 18011, 18014, 18013, 18020, - 18018, 35762, 35762, 35762, 35762, 35762, 35762, 13699, 13700, 13703, - 13706, 35762, 13656, 13657, 13670, 13671, 13668, 13669, 13651, 13653, - 35762, 35762, 13694, 13658, 35762, 35762, 13693, 13659, 13690, 13689, - 13686, 13685, 13674, 13684, 13683, 13688, 13687, 13676, 13665, 13664, - 13661, 13660, 13675, 13667, 13666, 13663, 13662, 13677, 35762, 13692, - 13691, 13682, 13681, 13696, 13698, 13697, 35762, 13673, 13672, 35762, - 13655, 13678, 13679, 13680, 13695, 35762, 7774, 13704, 13702, 13707, - 13716, 13717, 13714, 13715, 13710, 13711, 35762, 35762, 13719, 13708, - 35762, 35762, 13718, 13709, 13705, 35762, 35762, 13721, 35762, 35762, - 35762, 35762, 35762, 35762, 13720, 35762, 35762, 35762, 35762, 35762, - 13701, 13650, 13649, 13652, 13654, 13712, 13713, 35762, 35762, 7954, - 7955, 7953, 7952, 7951, 7950, 7949, 35762, 35762, 35762, 7960, 7957, - 7958, 7956, 7959, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 24495, 24496, 24509, 24510, 24507, 24508, 24491, 24492, 24493, - 24494, 24535, 24497, 24536, 24498, 24523, 24522, 24519, 24518, 24484, - 24483, 24517, 24516, 24521, 24520, 24486, 24485, 24504, 24503, 24500, - 24499, 24488, 24506, 24505, 24502, 24501, 24489, 24487, 24529, 24528, - 24515, 24514, 24527, 24526, 24534, 24531, 24530, 24525, 24524, 24533, - 24511, 24512, 24513, 24532, 24549, 24558, 24559, 24556, 24557, 24552, - 24553, 24554, 24555, 24560, 24550, 24561, 24551, 24544, 24539, 24538, - 24545, 24540, 24537, 24542, 24565, 24546, 24471, 24469, 24564, 24482, - 24562, 24478, 24480, 24477, 24476, 24473, 24472, 24475, 24474, 24481, - 24479, 24470, 24548, 35762, 24563, 24547, 24490, 24541, 24543, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 32336, - 32337, 32338, 32356, 32357, 32354, 32355, 32349, 32350, 32351, 32352, - 32382, 32339, 32383, 32340, 32374, 32373, 32370, 32369, 32358, 32368, - 32367, 32372, 32371, 32360, 32346, 32345, 32342, 32341, 32359, 32348, - 32347, 32344, 32343, 32361, 32376, 32375, 32366, 32365, 32379, 32381, - 32380, 32378, 32353, 32362, 32363, 32364, 32377, 32390, 32401, 32402, - 32399, 32400, 32395, 32396, 32397, 32398, 32403, 32393, 32391, 32404, - 32394, 32392, 32386, 32385, 32389, 32388, 32387, 32384, 32415, 32335, - 32416, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 32411, - 32413, 32410, 32409, 32406, 32405, 32408, 32407, 32414, 32412, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 27949, 27951, 27972, 27973, 27970, - 27971, 27965, 27966, 27967, 27968, 27998, 27952, 27999, 27953, 27990, - 27989, 27986, 27985, 27974, 27984, 27983, 27988, 27987, 27976, 27959, - 27958, 27962, 27961, 27975, 27960, 27955, 27964, 27963, 27977, 27992, - 27991, 27982, 27981, 27995, 27997, 27996, 27994, 27969, 27978, 27979, - 27980, 27993, 28027, 28034, 28035, 28032, 28033, 28030, 28031, 35762, - 35762, 28036, 28028, 28037, 28029, 28020, 28022, 28024, 28023, 28021, - 28019, 27947, 27946, 28018, 28017, 28000, 28001, 28002, 27948, 28015, - 28014, 28012, 28010, 28011, 28003, 28004, 28013, 28016, 28008, 28009, - 28007, 28006, 28005, 27954, 27956, 27957, 27950, 28025, 28026, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 22953, 22954, 22972, 22973, 22970, 22971, 22965, - 22966, 22967, 22968, 22999, 22955, 23000, 22956, 22992, 22991, 22988, - 22987, 22976, 22986, 22985, 22990, 22989, 22978, 22962, 22961, 22958, - 22957, 22977, 22964, 22963, 22960, 22959, 22979, 22994, 22993, 22984, - 22983, 22996, 22998, 22997, 22975, 22969, 22980, 22981, 22982, 22995, - 22974, 22928, 22937, 22938, 22935, 22936, 22931, 22932, 22933, 22934, - 22939, 22929, 22940, 22930, 22924, 22927, 22926, 22923, 22942, 22941, - 23001, 22925, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 22949, 22951, 22948, 22947, 22944, 22943, 22946, - 22945, 22952, 22950, 35762, 35762, 35762, 35762, 35762, 35762, 23481, - 23476, 23321, 23482, 23480, 23478, 23477, 23338, 23339, 23472, 23474, - 23473, 23483, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 30326, 30328, 30343, 30344, 30341, 30342, 30368, 30329, 30369, - 30330, 30358, 30357, 30354, 30353, 30345, 30352, 30351, 30356, 30355, - 30347, 30338, 30337, 30334, 30333, 30346, 30340, 30339, 30336, 30335, - 30348, 30360, 30359, 30350, 30349, 30365, 30367, 30332, 30364, 30366, - 30361, 30362, 30363, 30331, 30371, 30373, 30374, 30379, 30380, 30377, - 30378, 30381, 30375, 30382, 30376, 30372, 30370, 30327, 30325, 35762, - 35762, 35762, 35762, 35762, 35762, 30389, 30391, 30388, 30387, 30384, - 30383, 30386, 30385, 30392, 30390, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 191, 190, - 178, 181, 175, 167, 193, 192, 183, 195, 189, 184, 174, 196, 177, 197, - 180, 194, 164, 171, 170, 187, 166, 186, 182, 188, 165, 35762, 35762, 162, - 163, 161, 203, 204, 210, 211, 208, 209, 212, 207, 213, 205, 206, 200, - 35762, 35762, 35762, 35762, 222, 224, 221, 220, 217, 216, 219, 218, 225, - 223, 215, 214, 198, 199, 201, 202, 185, 173, 172, 169, 168, 179, 176, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 10719, 10720, 10733, 10734, 10731, - 10732, 10759, 10721, 10760, 10722, 10751, 10750, 10747, 10746, 10735, - 10745, 10744, 10749, 10748, 10737, 10728, 10727, 10724, 10723, 10736, - 10730, 10729, 10726, 10725, 10738, 10753, 10752, 10743, 10742, 10756, - 10758, 10718, 10755, 10757, 10739, 10740, 10741, 10754, 10717, 10763, - 10768, 10769, 10766, 10767, 10761, 10762, 10770, 10764, 10771, 10765, - 10774, 10776, 10775, 10773, 10772, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 33866, 33855, 33884, 33874, 33876, - 33877, 33883, 33873, 33859, 33868, 33856, 33886, 33882, 33861, 33875, - 33872, 33860, 33869, 33878, 33867, 33885, 33858, 33857, 33880, 33881, - 33864, 33863, 33862, 33865, 33870, 33871, 33879, 33898, 33887, 33916, - 33906, 33908, 33909, 33915, 33905, 33891, 33900, 33888, 33918, 33914, - 33893, 33907, 33904, 33892, 33901, 33910, 33899, 33917, 33890, 33889, - 33912, 33913, 33896, 33895, 33894, 33897, 33902, 33903, 33911, 33925, - 33927, 33924, 33923, 33920, 33919, 33922, 33921, 33928, 33926, 33937, - 33936, 33935, 33931, 33930, 33934, 33933, 33929, 33932, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 33938, 10633, 10634, 10641, 10642, 10639, 10640, 10668, 35762, 35762, - 10669, 35762, 35762, 10659, 10658, 10657, 10656, 10645, 10655, 10654, - 10663, 35762, 10647, 10629, 35762, 10636, 10635, 10646, 10630, 10628, - 10638, 10637, 10648, 10661, 10660, 10653, 10652, 10664, 10632, 10631, - 10665, 10644, 10666, 10649, 10650, 10651, 10662, 10643, 10667, 10674, - 10678, 10679, 10676, 10677, 10680, 35762, 10675, 10681, 35762, 35762, - 10673, 10671, 10670, 10682, 10687, 10684, 10688, 10683, 10672, 10617, - 10685, 10686, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 10624, 10626, 10623, 10622, 10619, 10618, 10621, 10620, 10627, - 10625, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 24192, 24193, 24206, 24207, 24204, 24205, 24187, 24188, 35762, - 35762, 24232, 24194, 24233, 24195, 24226, 24225, 24222, 24221, 24210, - 24220, 24219, 24224, 24223, 24212, 24201, 24200, 24197, 24196, 24211, - 24203, 24202, 24199, 24198, 24213, 24228, 24227, 24218, 24217, 24230, - 24231, 24191, 24209, 24189, 24214, 24215, 24216, 24229, 24208, 24190, - 24242, 24247, 24248, 24245, 24246, 24240, 24241, 35762, 35762, 24249, - 24243, 24250, 24244, 24236, 24238, 24237, 24235, 24234, 24251, 24239, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35543, 35564, 35561, - 35560, 35563, 35559, 35558, 35556, 35557, 35562, 35555, 35511, 35510, - 35530, 35529, 35512, 35528, 35527, 35537, 35514, 35522, 35521, 35504, - 35503, 35513, 35524, 35523, 35508, 35507, 35515, 35532, 35531, 35526, - 35525, 35539, 35520, 35519, 35506, 35505, 35533, 35534, 35535, 35542, - 35540, 35538, 35541, 35516, 35517, 35518, 35536, 35509, 35502, 35552, - 35549, 35550, 35551, 35548, 35553, 35499, 35498, 35496, 35495, 35497, - 35501, 35494, 35547, 35545, 35544, 35546, 35500, 35493, 35554, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 29142, 29163, 29161, - 29160, 29162, 29158, 29159, 29156, 29157, 29155, 29154, 29164, 29109, - 29108, 29128, 29127, 29110, 29126, 29125, 29130, 29129, 29112, 29120, - 29119, 29103, 29102, 29111, 29122, 29121, 29106, 29104, 29113, 29132, - 29131, 29124, 29123, 29138, 29118, 29117, 29105, 29133, 29134, 29135, - 29141, 29139, 29137, 29140, 29114, 29115, 29116, 29136, 29107, 29147, - 29149, 29085, 29084, 29082, 29083, 29093, 29094, 29089, 29092, 29088, - 29091, 29096, 29097, 29095, 29087, 29086, 29090, 29150, 29148, 29098, - 29151, 29146, 29145, 29144, 29143, 29101, 29100, 29099, 29152, 29153, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 5316, 5317, 5314, 5315, 5312, 5313, 5322, 5323, - 5320, 5321, 5318, 5319, 5485, 5486, 5487, 5484, 26453, 26451, 26460, - 26461, 26457, 26465, 26464, 26446, 26459, 26458, 26450, 26463, 26456, - 26449, 26455, 26454, 26447, 26452, 26462, 26441, 26448, 26466, 26467, - 26442, 26468, 26444, 26445, 26443, 26437, 26434, 26438, 26436, 26432, - 26435, 26439, 26433, 26440, 26474, 26473, 26484, 26475, 26476, 26485, - 26481, 26480, 26482, 26483, 26477, 26431, 26478, 26479, 26470, 26469, - 26429, 26471, 26472, 26430, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 10360, 10361, 10450, 10451, 10452, 10453, 10460, 10459, 10461, - 10469, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 3641, 3642, 3655, 3656, - 3653, 3654, 3637, 3638, 3639, 35762, 3681, 3643, 3682, 3644, 3673, 3672, - 3669, 3668, 3657, 3667, 3666, 3671, 3670, 3659, 3650, 3649, 3646, 3645, - 3658, 3652, 3651, 3648, 3647, 3660, 3675, 3674, 3665, 3664, 3678, 3680, - 3679, 3677, 3640, 3661, 3662, 3663, 3676, 3709, 3714, 3715, 3712, 3713, - 3706, 3707, 3708, 35762, 3716, 3710, 3717, 3711, 3703, 3702, 3705, 3704, - 3701, 3719, 3718, 3730, 3731, 3732, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 3726, 3728, 3725, 3724, 3721, 3720, - 3723, 3722, 3729, 3727, 3700, 3698, 3695, 3686, 3688, 3693, 3691, 3683, - 3689, 3699, 3697, 3696, 3685, 3687, 3694, 3692, 3684, 3690, 3733, 35762, - 35762, 35762, 21057, 21058, 21003, 21002, 21012, 20997, 21001, 21000, - 21014, 20998, 20994, 20993, 20996, 20999, 21005, 21004, 21011, 21016, - 20992, 20991, 20995, 21018, 21008, 21009, 21010, 21019, 21017, 21015, - 21006, 21007, 21013, 21020, 35762, 35762, 21035, 21034, 21043, 21029, - 21033, 21032, 21045, 21030, 21026, 21025, 21028, 21031, 21037, 21036, - 21042, 21047, 21024, 21023, 21027, 21049, 21040, 21041, 35762, 21050, - 21048, 21046, 21038, 21039, 21044, 21051, 21052, 21054, 21056, 21053, - 21055, 21022, 21021, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 21069, 21070, 21079, 21080, - 21077, 21078, 21106, 35762, 21071, 21107, 35762, 21072, 21085, 21084, - 21098, 21097, 21086, 21096, 21095, 21063, 21062, 21088, 21065, 21064, - 21074, 21073, 21087, 21068, 21066, 21076, 21075, 21089, 21100, 21099, - 21094, 21093, 21102, 21105, 21103, 21082, 21104, 21090, 21091, 21092, - 21101, 21081, 21083, 21061, 21067, 21116, 21121, 21122, 21119, 21120, - 21115, 35762, 35762, 35762, 21123, 35762, 21117, 21124, 35762, 21118, - 21114, 21113, 21112, 21110, 21111, 21125, 21109, 21108, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 21132, 21134, 21131, 21130, - 21127, 21126, 21129, 21128, 21135, 21133, 35762, 35762, 35762, 35762, - 35762, 35762, 14372, 14373, 14388, 14389, 14384, 14385, 35762, 14405, - 14374, 35762, 14404, 14375, 14411, 14410, 14393, 14392, 14407, 14401, - 14400, 14383, 14382, 14391, 14397, 14396, 14379, 14378, 14387, 14395, - 14394, 14381, 14380, 14390, 14399, 14398, 14377, 14376, 14386, 14403, - 14402, 14406, 14408, 14409, 14414, 14419, 14420, 14417, 14418, 35762, - 14422, 14415, 35762, 14421, 14416, 14413, 14412, 14423, 14434, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 14430, 14432, 14429, 14428, - 14425, 14424, 14427, 14426, 14433, 14431, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 20755, 20753, 20747, 20758, - 20750, 20757, 20761, 20752, 20749, 20751, 20754, 20748, 20763, 20759, - 20756, 20762, 20760, 20764, 20770, 20767, 20769, 20766, 20768, 20765, - 20746, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 17048, 17050, - 17046, 17049, 16988, 16989, 17008, 17009, 17006, 17007, 17002, 17003, - 17004, 17005, 17033, 16990, 17034, 35762, 17024, 17023, 17022, 17021, - 17010, 17020, 17019, 16993, 16992, 17012, 16999, 16998, 16995, 16994, - 17011, 17001, 17000, 16997, 16996, 17013, 17026, 17025, 17018, 17017, - 17029, 17032, 17030, 17028, 17031, 17014, 17015, 17016, 17027, 16991, - 17052, 17051, 17059, 17060, 17057, 17058, 17054, 35762, 35762, 35762, - 17055, 17053, 17056, 17047, 17073, 17062, 17061, 17041, 17044, 17040, - 17042, 17038, 17037, 17045, 17036, 17039, 17043, 17035, 17069, 17071, - 17068, 17067, 17064, 17063, 17066, 17065, 17072, 17070, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 20466, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 30449, 30445, 30439, 30448, 30441, 30450, 30454, 30456, 30451, 30446, - 30447, 30452, 30440, 30457, 30455, 30442, 30453, 30443, 30444, 30458, - 30459, 30523, 30506, 30504, 30513, 30512, 30508, 30514, 30510, 30509, - 30516, 30517, 30518, 30515, 30507, 30520, 30438, 30425, 30496, 30503, - 30804, 30805, 30423, 30398, 30524, 30803, 30460, 30525, 30511, 30519, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 30501, 8517, 8525, 8523, 8519, 8524, 8520, 8518, - 8521, 8522, 8586, 8526, 8535, 8534, 8528, 8527, 8539, 8529, 8530, 8538, - 8532, 8533, 8540, 8541, 8546, 8543, 8542, 8544, 8545, 8548, 8550, 8552, - 8551, 8553, 8559, 8556, 8557, 8561, 8554, 8555, 8560, 8558, 8563, 8562, - 8564, 8566, 8567, 8571, 8570, 8568, 8569, 8572, 8585, 8573, 8574, 8575, - 8584, 8576, 8580, 8581, 8579, 8577, 8578, 8583, 8582, 8587, 8588, 8599, - 8590, 8594, 8595, 8596, 8597, 8598, 8600, 8603, 8602, 8601, 8604, 8606, - 8607, 8608, 8609, 8610, 8611, 8612, 8613, 8614, 8615, 8616, 8617, 8618, - 8619, 8620, 8621, 8622, 8623, 8638, 8624, 8625, 8635, 8629, 8626, 8627, - 8628, 8636, 8633, 8637, 8634, 8630, 8632, 8645, 8641, 8642, 8643, 8646, - 8657, 8647, 8650, 8651, 8653, 8654, 8655, 8658, 8661, 8659, 8660, 8662, - 8663, 8665, 8666, 8695, 8702, 8696, 8697, 8698, 8699, 8700, 8701, 8703, - 8705, 8704, 8706, 8709, 8710, 8713, 8707, 8708, 8714, 8759, 8758, 8760, - 8715, 8718, 8719, 8720, 8716, 8717, 8721, 8724, 8722, 8725, 8727, 8736, - 8737, 8738, 8739, 8757, 8740, 8741, 8754, 8755, 8753, 8742, 8743, 8744, - 8745, 8746, 8747, 8748, 8751, 8752, 8761, 8851, 8762, 8763, 8765, 8764, - 8771, 8766, 8768, 8770, 8774, 8773, 8775, 8776, 8783, 8777, 8778, 8782, - 8786, 8787, 8784, 8785, 8793, 8790, 8794, 8795, 8796, 8797, 8798, 8800, - 8801, 8803, 8802, 8805, 8804, 8807, 8806, 8808, 8809, 8811, 8812, 8816, - 8818, 8822, 8823, 8836, 8828, 8829, 8824, 8825, 8826, 8830, 8834, 8831, - 8832, 8833, 8837, 8838, 8840, 8841, 8842, 8843, 8844, 8845, 8855, 8846, - 8847, 8850, 8849, 8848, 8852, 8853, 8854, 8856, 8857, 8860, 8861, 8862, - 8863, 8864, 8866, 8865, 8882, 8873, 8874, 8867, 8868, 8870, 8871, 8869, - 8872, 8881, 8875, 8880, 8878, 8877, 8879, 8884, 8903, 8885, 8886, 8887, - 8890, 8889, 8891, 8892, 8894, 8895, 8896, 8904, 8897, 8898, 8899, 8902, - 8901, 8900, 8905, 8906, 8908, 8909, 8910, 8911, 8913, 8917, 8914, 8918, - 8916, 8915, 8919, 8920, 8921, 8922, 8926, 8925, 8923, 8924, 8927, 8928, - 8930, 8949, 8951, 8931, 8933, 8932, 8934, 8935, 8938, 8939, 8937, 8936, - 8940, 8941, 8942, 8943, 8946, 8944, 8945, 8947, 8948, 8952, 8953, 8950, - 8954, 8955, 8956, 8957, 8959, 8962, 8961, 8963, 8964, 8966, 8967, 8968, - 8972, 8971, 8969, 8970, 8973, 8977, 8976, 8975, 8978, 8979, 8981, 8982, - 8986, 8983, 8984, 8989, 8987, 8990, 8991, 8993, 8992, 8994, 8995, 9017, - 9016, 9019, 9020, 9001, 9002, 8999, 9000, 8998, 8996, 9004, 9003, 9005, - 9007, 9013, 9014, 9011, 9012, 9021, 9022, 9023, 9040, 9026, 9027, 9028, - 9024, 9025, 9029, 9030, 9031, 9032, 9033, 9034, 9035, 9036, 9037, 9038, - 9063, 9041, 9044, 9042, 9043, 9049, 9050, 9047, 9048, 9045, 9046, 9051, - 9052, 9060, 9053, 9054, 9061, 9058, 9059, 9062, 9055, 9056, 9057, 9064, - 9065, 9066, 9067, 9068, 9069, 9070, 9072, 9073, 9071, 9074, 9075, 9116, - 9117, 9078, 9079, 9076, 9077, 9082, 9083, 9081, 9087, 9084, 9086, 9085, - 9091, 9092, 9090, 9088, 9089, 9093, 9094, 9095, 9096, 9097, 9098, 9099, - 9118, 9104, 9100, 9101, 9102, 9103, 9105, 9107, 9106, 9108, 9109, 9111, - 9110, 9112, 9114, 9113, 9119, 9120, 9123, 9124, 9121, 9122, 9198, 9193, - 9194, 9195, 9196, 9197, 9199, 9202, 9200, 9201, 9203, 9245, 9204, 9234, - 9235, 9211, 9213, 9230, 9214, 9236, 9218, 9216, 9217, 9219, 9220, 9221, - 9222, 9231, 9232, 9225, 9226, 9228, 9237, 9205, 9206, 9210, 9208, 9246, - 9238, 9240, 9239, 9241, 9247, 9248, 9242, 9243, 9244, 9249, 9250, 9251, - 9254, 9255, 9256, 9252, 9253, 9257, 9258, 9260, 9262, 9263, 9281, 9279, - 9280, 9283, 9282, 9264, 9271, 9269, 9270, 9265, 9266, 9272, 9273, 9274, - 9275, 9276, 9278, 9284, 9293, 9285, 9287, 9288, 9286, 9289, 9291, 9290, - 9292, 9295, 9297, 9296, 9298, 9299, 9332, 9334, 9300, 9302, 9301, 9304, - 9307, 9305, 9306, 9310, 9314, 9317, 9316, 9319, 9322, 9320, 9321, 9325, - 9327, 9333, 9335, 9336, 9340, 9347, 9345, 9343, 9344, 9346, 9349, 9348, - 9341, 9342, 9350, 9353, 9359, 9354, 9357, 9352, 9351, 9360, 9358, 9355, - 9356, 9361, 9362, 9363, 9364, 9365, 9366, 9367, 9369, 9372, 9373, 9376, - 9377, 9378, 9374, 9375, 9370, 9371, 9379, 9380, 9381, 9382, 9383, 9384, - 9386, 9387, 9388, 9389, 9390, 9394, 9391, 9415, 9408, 9409, 9414, 9397, - 9396, 9410, 9413, 9411, 9400, 9399, 9402, 9404, 9405, 9406, 9407, 9403, - 9416, 9392, 9417, 9418, 9419, 9420, 9421, 9422, 9430, 9428, 9426, 9429, - 9425, 9427, 9423, 9424, 9431, 9433, 9434, 9435, 9442, 9437, 9438, 9446, - 9447, 9443, 9445, 9444, 9448, 9450, 9449, 9451, 9462, 9453, 9452, 9461, - 9459, 9454, 9455, 9456, 9458, 9457, 9460, 9466, 9463, 9465, 9464, 9467, - 9468, 9469, 9470, 9473, 9474, 9476, 9477, 9478, 9479, 9481, 9480, 9482, - 9489, 9483, 9484, 9490, 9485, 9486, 9487, 9488, 9491, 9495, 9492, 9493, - 9494, 9496, 9497, 9498, 9499, 9505, 9503, 9501, 9502, 9500, 9504, 9506, - 9508, 9525, 9526, 9509, 9514, 9516, 9510, 9513, 9511, 9512, 9517, 9523, - 9524, 9518, 9521, 9522, 9527, 9533, 9532, 9528, 9530, 9529, 9614, 9615, - 9534, 9535, 9540, 9541, 9538, 9539, 9542, 9536, 9537, 9543, 9546, 9547, - 9549, 9550, 9551, 9555, 9552, 9553, 9554, 9544, 9545, 9556, 9558, 9557, - 9559, 9561, 9562, 9563, 9569, 9568, 9564, 9565, 9566, 9601, 9570, 9571, - 9572, 9573, 9574, 9593, 9576, 9577, 9579, 9578, 9580, 9581, 9597, 9582, - 9584, 9583, 9596, 9586, 9594, 9598, 9589, 9588, 9595, 9590, 9592, 9591, - 9599, 9600, 9602, 9606, 9604, 9605, 9603, 9609, 9608, 9607, 9613, 9610, - 9611, 9612, 9616, 9618, 9617, 9621, 9619, 9636, 9622, 9625, 9627, 9623, - 9624, 9628, 9626, 9629, 9631, 9633, 9634, 9635, 9039, 8536, 8547, 8565, - 8631, 8640, 8656, 8664, 8756, 8749, 8767, 8769, 8859, 8883, 8929, 8958, - 8960, 8974, 8980, 9015, 8988, 9018, 8997, 9006, 9010, 9080, 9209, 9212, - 9227, 9259, 9277, 9294, 9303, 9331, 9329, 9308, 9339, 9368, 9385, 9395, - 9515, 9548, 9531, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 8501, 8496, 8433, 8419, 8480, 8476, 8408, - 8453, 8500, 8441, 8425, 8486, 8475, 8407, 8452, 8439, 8423, 8482, 8472, - 8402, 8449, 8462, 8506, 8498, 8435, 8421, 8484, 8474, 8404, 8451, 8463, - 8507, 8499, 8436, 8422, 8508, 8490, 8491, 8437, 8413, 8479, 8471, 8401, - 8448, 8466, 8509, 8492, 8493, 8438, 8414, 8477, 8478, 8461, 8504, 8487, - 8488, 8428, 8418, 8494, 8495, 8429, 8432, 8430, 8431, 8485, 8470, 8468, - 8469, 8405, 8406, 8444, 8446, 8447, 8445, 8502, 8497, 8434, 8420, 8481, - 8460, 8503, 8489, 8426, 8427, 8416, 8417, 8443, 8442, 8456, 8505, 8465, - 8511, 8415, 8464, 8510, 8457, 8458, 8454, 8455, 8459, 8467, 8409, 8412, - 8411, 8410, 8440, 8424, 8483, 8473, 8403, 8450, 35762, 8516, 8515, 8514, - 8512, 8513, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 8537, 8531, 8549, 8589, 8591, 8592, 8593, 8605, - 8644, 8639, 8649, 8648, 8652, 8669, 8667, 8668, 8670, 8671, 8689, 8675, - 8672, 8673, 8674, 8691, 8692, 8690, 8679, 8678, 8676, 8677, 8682, 8680, - 8681, 8683, 8684, 8685, 8686, 8693, 8694, 8688, 8687, 8712, 8711, 8723, - 8726, 8731, 8735, 8728, 8732, 8733, 8729, 8730, 8734, 8750, 8772, 8779, - 8780, 8781, 8788, 8789, 8792, 8791, 8799, 8810, 8813, 8814, 8815, 8817, - 8819, 8820, 8821, 8827, 8835, 8839, 8858, 8876, 8888, 8893, 8907, 8912, - 8965, 8985, 9008, 9009, 9115, 9135, 9125, 9126, 9134, 9130, 9131, 9132, - 9129, 9128, 9127, 9133, 9137, 9136, 9143, 9144, 9138, 9139, 9140, 9145, - 9141, 9142, 9146, 9147, 9148, 9149, 9150, 9151, 9158, 9152, 9157, 9156, - 9154, 9153, 9155, 9159, 9160, 9165, 9166, 9161, 9162, 9163, 9164, 9192, - 9188, 9167, 9175, 9176, 9174, 9173, 9177, 9168, 9169, 9171, 9172, 9170, - 9189, 9178, 9185, 9187, 9182, 9183, 9186, 9181, 9184, 9180, 9179, 9190, - 9191, 9207, 9233, 9215, 9223, 9224, 9229, 9261, 9268, 9267, 9328, 9309, - 9311, 9330, 9312, 9313, 9315, 9318, 9323, 9324, 9326, 9393, 9412, 9398, - 9401, 9432, 9436, 9439, 9440, 9441, 9472, 9471, 9475, 9507, 9519, 9520, - 9560, 9567, 9575, 9587, 9585, 9620, 9630, 9632, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 9709, - 9710, 9711, 9712, 9713, 9714, 9715, 9716, 9719, 9720, 9717, 9718, 9721, - 9722, 9723, 9724, 9725, 9726, 9727, 9728, 9729, 9730, 9731, 9732, 9733, - 9734, 9735, 9736, 9737, 9738, 9739, 9740, 9741, 9742, 9743, 9744, 9745, - 9746, 9747, 9748, 9749, 9750, 9751, 9752, 9753, 9754, 9755, 9775, 9776, - 9777, 9778, 9779, 9780, 9781, 9782, 9783, 9758, 9759, 9760, 9761, 9762, - 9756, 9757, 9763, 9764, 9765, 9784, 9785, 9786, 9787, 9788, 9789, 9790, - 9791, 9792, 9793, 9766, 9767, 9768, 9769, 9770, 9771, 9772, 9773, 9774, - 9794, 9795, 9796, 9797, 9798, 9799, 9800, 9801, 9802, 9803, 9804, 9805, - 9806, 9807, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 11228, 11229, 11230, 11231, 11226, - 11227, 11223, 11224, 11225, 11232, 11233, 11234, 11239, 11240, 11241, - 11242, 11235, 11236, 11243, 11244, 11237, 11238, 11245, 11246, 11273, - 11274, 11275, 11276, 11277, 11278, 11279, 11280, 11281, 11282, 11263, - 11264, 11261, 11262, 11265, 11266, 11267, 11268, 11269, 11270, 11271, - 11247, 11248, 11255, 11249, 11250, 11251, 11252, 11256, 11253, 11254, - 11257, 11258, 11259, 11260, 11283, 11284, 11285, 11286, 11287, 11288, - 11289, 11290, 11291, 11292, 11293, 11294, 11295, 11296, 11297, 11298, - 11299, 11300, 11301, 11302, 11272, 11342, 11343, 11344, 11345, 11340, - 11341, 11346, 11347, 11348, 11349, 11354, 11350, 11351, 11352, 11353, - 11355, 11356, 11357, 11358, 11359, 11360, 11361, 11362, 11363, 11364, - 11365, 11366, 11367, 11368, 11369, 11370, 11371, 11372, 11373, 11374, - 11375, 11376, 11377, 11380, 11381, 11382, 11383, 11384, 11385, 11386, - 11378, 11379, 11387, 11460, 11461, 11462, 11463, 11464, 11465, 11466, - 11467, 11468, 11469, 11451, 11452, 11453, 11454, 11455, 11456, 11457, - 11449, 11450, 11458, 11459, 11392, 11388, 11389, 11393, 11394, 11390, - 11391, 11395, 11396, 11397, 11398, 11399, 11404, 11405, 11406, 11407, - 11408, 11409, 11400, 11401, 11410, 11402, 11403, 11411, 11412, 11413, - 11414, 11415, 11416, 11417, 11418, 11419, 11420, 11421, 11426, 11422, - 11423, 11427, 11424, 11425, 11428, 11429, 11430, 11431, 11432, 11442, - 11443, 11444, 11445, 11446, 11447, 11448, 11433, 11434, 11435, 11436, - 11437, 11438, 11439, 11440, 11441, 11474, 11475, 11476, 11477, 11478, - 11479, 11480, 11470, 11471, 11472, 11473, 11506, 11507, 11508, 11509, - 11510, 11511, 11502, 11503, 11504, 11505, 11512, 11513, 11481, 11482, - 11485, 11486, 11487, 11488, 11489, 11490, 11491, 11483, 11484, 11492, - 11495, 11496, 11497, 11498, 11493, 11494, 11499, 11500, 11501, 11517, - 11518, 11519, 11520, 11521, 11522, 11523, 11524, 11525, 11526, 11529, - 11530, 11531, 11527, 11528, 11532, 11533, 11534, 11535, 11536, 11537, - 11573, 11571, 11572, 11574, 11575, 11576, 11577, 11578, 11579, 11580, - 11581, 11544, 11538, 11539, 11545, 11546, 11547, 11548, 11549, 11540, - 11541, 11542, 11543, 11550, 11557, 11558, 11559, 11560, 11561, 11551, - 11552, 11553, 11554, 11555, 11556, 11562, 11563, 11568, 11564, 11565, - 11566, 11567, 11569, 11570, 11588, 11589, 11590, 11591, 11592, 11586, - 11587, 11583, 11584, 11585, 11593, 11594, 11597, 11595, 11596, 11598, - 11599, 11600, 11601, 11602, 11603, 11604, 11605, 11630, 11631, 11634, - 11635, 11636, 11637, 11638, 11632, 11633, 11639, 11640, 11641, 11610, - 11611, 11612, 11613, 11614, 11615, 11606, 11607, 11608, 11609, 11616, - 11617, 11622, 11623, 11624, 11618, 11619, 11625, 11620, 11621, 11626, - 11627, 11628, 11629, 11642, 11643, 11644, 11645, 11646, 11649, 11650, - 11651, 11652, 11653, 11647, 11648, 11654, 11655, 11663, 11664, 11665, - 11666, 11659, 11660, 11667, 11668, 11669, 11661, 11662, 11670, 11671, - 11672, 11673, 11674, 11675, 11676, 11677, 12318, 12319, 12320, 12321, - 12322, 12323, 12324, 12325, 11689, 11685, 11686, 11690, 11691, 11692, - 11687, 11688, 11693, 11694, 11696, 11697, 11698, 11701, 11699, 11700, - 11702, 11703, 11704, 11705, 11706, 11707, 11717, 11718, 11725, 11708, - 11709, 11710, 11711, 11712, 11713, 11714, 11715, 11716, 11726, 11727, - 11719, 11720, 11721, 11722, 11723, 11724, 11728, 11729, 11736, 11737, - 11730, 11731, 11738, 11732, 11733, 11739, 11740, 11741, 11734, 11735, - 11742, 11748, 11746, 11747, 11749, 11743, 11744, 11745, 11750, 11751, - 11752, 11753, 11754, 11755, 11756, 11757, 11758, 11759, 11760, 11761, - 11818, 11819, 11820, 11821, 11822, 11823, 11824, 11825, 11826, 11781, - 11782, 11783, 11784, 11785, 11786, 11787, 11788, 11778, 11779, 11780, - 11789, 11806, 11807, 11808, 11809, 11810, 11804, 11805, 11811, 11812, - 11813, 11814, 11798, 11799, 11800, 11790, 11791, 11792, 11793, 11794, - 11795, 11801, 11796, 11797, 11802, 11803, 11815, 11816, 11817, 11829, - 11830, 11831, 11832, 11827, 11828, 11833, 11834, 11835, 11836, 11839, - 11840, 11841, 11842, 11843, 11844, 11845, 11837, 11838, 11846, 11847, - 11848, 11866, 11867, 11868, 11869, 11870, 11871, 11872, 11873, 11874, - 11849, 11850, 11851, 11852, 11855, 11856, 11857, 11858, 11859, 11860, - 11853, 11854, 11861, 11864, 11865, 11862, 11863, 11882, 11883, 11886, - 11887, 11888, 11884, 11885, 11875, 11876, 11877, 11878, 11879, 11880, - 11881, 11889, 11890, 11891, 11892, 11893, 11894, 11895, 11898, 11899, - 11900, 11901, 11902, 11903, 11904, 11905, 11896, 11897, 11906, 11907, - 11914, 11915, 11916, 11908, 11909, 11910, 11911, 11917, 11918, 11919, - 11912, 11913, 11925, 11926, 11929, 11930, 11927, 11928, 11931, 11932, - 11920, 11921, 11922, 11923, 11924, 11933, 11934, 11935, 11940, 11941, - 11942, 11943, 11944, 11945, 11946, 11947, 11948, 11949, 11936, 11937, - 11938, 11939, 11951, 11952, 11955, 11953, 11954, 11956, 11957, 11958, - 11959, 11960, 11961, 11962, 11963, 12326, 12327, 12328, 12329, 12330, - 12331, 12332, 11969, 11967, 11968, 11964, 11965, 11966, 11970, 11971, - 11972, 11973, 11974, 11975, 11976, 11977, 11980, 11981, 11982, 11983, - 11984, 11978, 11979, 11985, 11986, 11987, 11988, 11989, 11990, 11991, - 11992, 11993, 11994, 11995, 11996, 11997, 12002, 11998, 11999, 12003, - 12004, 12005, 12000, 12001, 12006, 12007, 12008, 12014, 12015, 12016, - 12017, 12009, 12010, 12011, 12018, 12019, 12012, 12013, 12020, 12021, - 12025, 12026, 12027, 12028, 12029, 12030, 12022, 12023, 12024, 12031, - 12032, 12033, 12036, 12037, 12038, 12039, 12040, 12034, 12035, 12041, - 12042, 12043, 12044, 12045, 12046, 12047, 12048, 12049, 12050, 12051, - 12060, 12061, 12052, 12053, 12062, 12063, 12064, 12054, 12055, 12056, - 12057, 12058, 12059, 12069, 12065, 12066, 12070, 12071, 12072, 12073, - 12067, 12068, 12074, 12075, 12076, 12086, 12087, 12088, 12089, 12090, - 12091, 12092, 12093, 12094, 12095, 12081, 12082, 12077, 12078, 12079, - 12080, 12083, 12084, 12085, 12100, 12101, 12102, 12103, 12104, 12097, - 12098, 12099, 12105, 12106, 12107, 12134, 12135, 12136, 12137, 12138, - 12139, 12140, 12141, 12142, 12143, 12112, 12113, 12114, 12108, 12109, - 12115, 12116, 12117, 12118, 12119, 12110, 12111, 12122, 12123, 12120, - 12121, 12124, 12125, 12126, 12127, 12128, 12129, 12130, 12131, 12132, - 12133, 12147, 12148, 12149, 12150, 12151, 12152, 12153, 12154, 12155, - 12156, 12157, 12158, 12159, 12160, 12161, 12162, 12144, 12145, 12146, - 12163, 12164, 12173, 12165, 12166, 12167, 12168, 12170, 12171, 12172, - 12174, 12175, 12176, 12177, 12178, 12179, 12180, 12181, 12182, 12183, - 12184, 12185, 12186, 12187, 12188, 12189, 12190, 12191, 12192, 12193, - 12200, 12201, 12194, 12195, 12202, 12203, 12204, 12205, 12196, 12197, - 12198, 12199, 12206, 12207, 12208, 12209, 12214, 12210, 12211, 12215, - 12216, 12217, 12212, 12213, 12218, 12219, 12220, 12221, 12227, 12228, - 12223, 12224, 12229, 12230, 12231, 12232, 12233, 12225, 12226, 12234, - 12235, 12242, 12243, 12244, 12236, 12237, 12245, 12246, 12238, 12239, - 12240, 12241, 12247, 12250, 12251, 12252, 12253, 12248, 12249, 12254, - 12263, 12264, 12265, 12256, 12257, 12258, 12266, 12259, 12260, 12267, - 12261, 12262, 12268, 12269, 12270, 12271, 12272, 12273, 12274, 12275, - 12276, 12289, 12277, 12278, 12279, 12280, 12281, 12282, 12283, 12284, - 12285, 12286, 12287, 12288, 12290, 12291, 12292, 12293, 12313, 12314, - 12315, 12316, 12317, 12294, 12295, 12296, 12297, 12298, 12299, 12300, - 12301, 12302, 12303, 12304, 12305, 12306, 12307, 12308, 12309, 12310, - 12311, 12312, 11306, 11307, 11308, 11309, 11310, 11311, 11303, 11304, - 11305, 11312, 11313, 11317, 11318, 11319, 11320, 11321, 11322, 11323, - 11324, 11325, 11326, 11327, 11328, 11329, 11330, 11331, 11332, 11333, - 11334, 11335, 11336, 11314, 11315, 11316, 12169, 12222, 11658, 11683, - 11680, 11682, 11679, 11950, 11337, 11514, 11684, 11681, 11678, 11339, - 11516, 11338, 11515, 11777, 11582, 11656, 11695, 11657, 12096, 12255, - 11773, 11764, 11768, 11775, 11771, 11765, 11770, 11767, 11774, 11763, - 11769, 11776, 11772, 11766, 11762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 435, 436, 437, 438, 439, 440, 441, - 442, 443, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 365, - 366, 367, 368, 369, 370, 363, 364, 371, 372, 373, 415, 416, 417, 418, - 419, 420, 421, 422, 423, 413, 414, 381, 377, 378, 382, 383, 384, 379, - 380, 374, 375, 376, 385, 386, 387, 444, 445, 446, 447, 448, 449, 450, - 451, 452, 453, 392, 393, 394, 395, 396, 397, 388, 389, 390, 391, 398, - 399, 400, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, - 466, 467, 468, 469, 470, 471, 472, 473, 405, 406, 407, 408, 409, 410, - 411, 401, 402, 403, 404, 412, 485, 486, 487, 488, 489, 490, 491, 474, - 475, 476, 477, 482, 483, 484, 492, 478, 479, 480, 481, 493, 494, 495, - 496, 497, 500, 501, 502, 503, 498, 499, 504, 505, 506, 507, 510, 511, - 512, 513, 514, 508, 509, 515, 516, 517, 518, 521, 522, 523, 524, 525, - 519, 520, 526, 527, 528, 529, 530, 531, 532, 533, 534, 535, 536, 537, - 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, - 552, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, - 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, - 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 597, 598, 590, 591, - 592, 599, 600, 601, 602, 593, 594, 603, 595, 596, 608, 609, 610, 611, - 612, 604, 605, 606, 607, 613, 614, 615, 641, 642, 643, 644, 645, 646, - 647, 639, 640, 648, 649, 661, 662, 663, 664, 665, 666, 667, 668, 669, - 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, - 684, 685, 686, 687, 688, 689, 690, 652, 653, 654, 655, 656, 657, 658, - 650, 651, 659, 660, 691, 692, 693, 694, 695, 696, 697, 698, 699, 700, - 630, 631, 632, 633, 634, 635, 636, 637, 638, 628, 629, 620, 621, 622, - 623, 616, 617, 624, 625, 626, 627, 618, 619, 703, 704, 705, 706, 707, - 708, 709, 710, 711, 701, 702, 795, 796, 797, 798, 799, 800, 801, 802, - 803, 804, 714, 715, 716, 717, 718, 719, 720, 721, 722, 712, 713, 741, - 742, 738, 739, 740, 743, 744, 745, 734, 735, 736, 737, 746, 747, 748, - 805, 806, 807, 808, 809, 810, 811, 812, 813, 814, 725, 726, 727, 728, - 729, 730, 731, 732, 733, 723, 724, 753, 754, 755, 756, 749, 750, 757, - 758, 759, 751, 752, 760, 786, 784, 785, 787, 788, 789, 790, 791, 792, - 793, 794, 767, 763, 764, 768, 761, 762, 769, 770, 765, 766, 771, 772, - 773, 775, 776, 777, 774, 778, 779, 780, 781, 782, 783, 846, 847, 848, - 849, 850, 851, 852, 853, 854, 855, 815, 816, 817, 818, 819, 820, 821, - 822, 823, 824, 825, 856, 857, 858, 859, 860, 861, 862, 863, 864, 865, - 866, 867, 868, 869, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, - 880, 881, 882, 883, 884, 885, 826, 827, 830, 831, 832, 833, 834, 835, - 828, 829, 836, 837, 886, 887, 888, 889, 890, 891, 892, 893, 894, 895, - 896, 897, 898, 899, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, - 910, 911, 912, 913, 914, 915, 838, 839, 840, 841, 842, 843, 844, 845, - 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 930, - 931, 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, 944, - 945, 916, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 2827, 2794, 2852, 2853, 2823, - 2860, 2871, 2842, 2859, 2854, 2855, 2807, 2872, 2829, 2808, 2813, 2820, - 2866, 2838, 2797, 2832, 2867, 2828, 2804, 2805, 2836, 2810, 2851, 2793, - 2850, 2819, 2843, 2874, 2792, 2837, 2868, 2822, 2821, 2817, 2795, 2849, - 2826, 2856, 2809, 2846, 2857, 2873, 2802, 2863, 2870, 2811, 2798, 2825, - 2800, 2818, 2861, 2803, 2876, 2878, 2799, 2864, 2814, 2858, 2835, 2862, - 2840, 2847, 2831, 2875, 2830, 2812, 2844, 2815, 2841, 2869, 2865, 2848, - 2833, 2806, 2839, 2796, 2834, 2877, 2801, 2845, 2824, 2816, 2910, 2926, - 2924, 2923, 2889, 2895, 2884, 2930, 2894, 2886, 2916, 2929, 2888, 2914, - 2915, 2879, 2919, 2927, 2921, 2922, 2901, 2898, 2913, 2882, 2880, 2881, - 2887, 2917, 2934, 2908, 2906, 2931, 2920, 2928, 2900, 2904, 2905, 2902, - 2925, 2897, 2903, 2912, 2918, 2899, 2932, 2896, 2933, 2883, 2893, 2892, - 2890, 2907, 2911, 2891, 2885, 2909, 2986, 3006, 3027, 3023, 2985, 2974, - 2987, 2935, 2963, 2937, 3007, 3003, 2960, 3008, 2978, 2957, 2940, 2936, - 2942, 3026, 3005, 2968, 2998, 2966, 3028, 2947, 2948, 3012, 2979, 3016, - 2994, 3020, 3015, 2981, 3022, 2972, 2954, 3004, 2982, 2950, 2965, 2970, - 3001, 3017, 2984, 3031, 2975, 3000, 2997, 3030, 3021, 2962, 3032, 2990, - 2949, 3019, 2995, 2992, 2944, 2980, 2956, 2967, 2952, 2946, 2991, 2989, - 3024, 2983, 2999, 3002, 2945, 2996, 2976, 2953, 3029, 2977, 3013, 3011, - 2964, 2955, 2959, 2941, 2961, 2943, 2958, 3025, 2993, 2971, 2969, 3014, - 2939, 2938, 2988, 2973, 2951, 3009, 3010, 3018, 3060, 3144, 3093, 3067, - 3094, 3053, 3092, 3098, 3080, 3107, 3143, 3089, 3123, 3090, 3037, 3136, - 3121, 3096, 3128, 3041, 3145, 3043, 3130, 3065, 3075, 3056, 3062, 3132, - 3149, 3091, 3038, 3086, 3138, 3036, 3088, 3033, 3076, 3070, 3048, 3077, - 3072, 3071, 3113, 3069, 3066, 3054, 3099, 3058, 3046, 3105, 3134, 3133, - 3146, 3039, 3120, 3137, 3085, 3064, 3100, 3040, 3116, 3111, 3106, 3051, - 3079, 3068, 3083, 3147, 3115, 3148, 3078, 3102, 3127, 3044, 3082, 3087, - 3139, 3061, 3045, 3101, 3135, 3057, 3081, 3049, 3084, 3097, 3095, 3035, - 3042, 3117, 3141, 3140, 3108, 3119, 3050, 3063, 3055, 3129, 3074, 3125, - 3122, 3047, 3110, 3126, 3103, 3112, 3109, 3124, 3114, 3073, 3052, 3118, - 3142, 3104, 3059, 3131, 3034, 3203, 3280, 3190, 3177, 3287, 3180, 3243, - 3269, 3259, 3229, 3206, 3254, 3274, 3215, 3170, 3290, 3262, 3208, 3244, - 3279, 3172, 3182, 3231, 3221, 3187, 3218, 3222, 3230, 3261, 3228, 3247, - 3270, 3211, 3194, 3164, 3217, 3225, 3188, 3181, 3209, 3205, 3271, 3263, - 3257, 3202, 3210, 3295, 3163, 3284, 3291, 3251, 3283, 3167, 3268, 3277, - 3285, 3288, 3176, 3253, 3273, 3161, 3223, 3264, 3193, 3191, 3236, 3240, - 3160, 3282, 3171, 3303, 3234, 3296, 3192, 3204, 3249, 3299, 3179, 3153, - 3159, 3220, 3169, 3186, 3216, 3165, 3155, 3232, 3245, 3293, 3207, 3235, - 3237, 3252, 3196, 3154, 3239, 3197, 3162, 3152, 3200, 3255, 3219, 3173, - 3250, 3233, 3292, 3212, 3241, 3151, 3158, 3226, 3304, 3281, 3306, 3305, - 3178, 3242, 3272, 3275, 3201, 3267, 3294, 3213, 3300, 3297, 3298, 3302, - 3301, 3168, 3246, 3227, 3256, 3289, 3157, 3286, 3184, 3195, 3260, 3258, - 3214, 3224, 3265, 3266, 3150, 3238, 3248, 3183, 3175, 3198, 3185, 3189, - 3276, 3174, 3199, 3278, 3156, 3166, 3311, 3360, 3313, 3358, 3338, 3351, - 3332, 3315, 3341, 3340, 3320, 3350, 3330, 3326, 3317, 3348, 3343, 3349, - 3346, 3309, 3308, 3328, 3327, 3325, 3353, 3345, 3354, 3329, 3334, 3331, - 3355, 3335, 3342, 3333, 3337, 3307, 3323, 3324, 3344, 3336, 3359, 3356, - 3316, 3314, 3312, 3318, 3339, 3321, 3322, 3319, 3352, 3310, 3347, 3357, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 23535, 23527, 23548, - 23525, 23549, 23536, 23551, 23531, 23522, 23539, 23537, 23547, 23521, - 23529, 23524, 23526, 23532, 23530, 23528, 23541, 23544, 23533, 23543, - 23550, 23542, 23523, 23546, 23545, 23538, 23540, 23534, 35762, 23560, - 23562, 23559, 23558, 23555, 23554, 23557, 23556, 23563, 23561, 35762, - 35762, 35762, 35762, 23553, 23552, 30867, 30864, 30865, 30866, 30819, - 30816, 30817, 30818, 30871, 30868, 30869, 30870, 30859, 30856, 30857, - 30858, 30863, 30860, 30861, 30862, 30855, 30852, 30853, 30854, 30815, - 30812, 30813, 30814, 30847, 30844, 30845, 30846, 30820, 30825, 30836, - 30837, 30848, 30851, 30849, 30850, 30843, 30840, 30841, 30842, 30831, - 30828, 30829, 30830, 30882, 30881, 30880, 30832, 30839, 30889, 30887, - 30884, 30834, 30883, 30885, 30827, 30835, 30824, 30826, 30823, 30874, - 30878, 30886, 30833, 30838, 30876, 30873, 30879, 30822, 30872, 30888, - 30821, 30877, 30875, 30890, 35762, 30897, 30899, 30896, 30895, 30892, - 30891, 30894, 30893, 30900, 30898, 35762, 35762, 35762, 35762, 35762, - 35762, 3421, 3426, 3442, 3444, 3434, 3432, 3424, 3418, 3425, 3438, 3433, - 3429, 3440, 3423, 3419, 3441, 3428, 3439, 3443, 3437, 3431, 3445, 3430, - 3446, 3435, 3436, 3427, 3422, 3420, 3447, 35762, 35762, 3414, 3416, 3417, - 3415, 3413, 3448, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 26200, 26201, 26206, 26207, 26194, 26195, 26210, 26211, - 26202, 26203, 26192, 26193, 26212, 26213, 26196, 26197, 26208, 26209, - 26214, 26215, 26204, 26205, 26198, 26199, 26216, 26217, 26190, 26191, - 26136, 26127, 26133, 26124, 26129, 26135, 26128, 26132, 26138, 26122, - 26134, 26120, 26125, 26123, 26131, 26126, 26130, 26139, 26137, 26121, - 26144, 26143, 26141, 26140, 26142, 26146, 26145, 26176, 26177, 26155, - 26175, 26173, 26181, 26183, 26182, 26184, 26174, 26166, 26179, 26164, - 26186, 26159, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 26224, 26226, 26223, 26222, 26219, 26218, 26221, 26220, - 26227, 26225, 35762, 26151, 26148, 26150, 26153, 26147, 26149, 26152, - 35762, 26178, 26185, 26163, 26171, 26187, 26162, 26169, 26180, 26168, - 26189, 26170, 26165, 26172, 26188, 26167, 26161, 26154, 26157, 26158, - 26160, 26156, 35762, 35762, 35762, 35762, 35762, 26108, 26115, 26107, - 26106, 26116, 26104, 26101, 26119, 26111, 26109, 26117, 26103, 26102, - 26112, 26118, 26114, 26110, 26105, 26113, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 22208, 22212, 22215, 22216, 22186, 22218, 22194, 22209, - 22213, 22204, 22203, 22205, 22193, 22185, 22206, 22202, 22199, 22200, - 22214, 22196, 22207, 22210, 22192, 22190, 22217, 22201, 22198, 22188, - 22211, 22197, 22187, 22195, 22266, 22270, 22273, 22274, 22244, 22276, - 22252, 22267, 22271, 22262, 22261, 22263, 22251, 22243, 22264, 22260, - 22257, 22258, 22272, 22254, 22265, 22268, 22250, 22248, 22275, 22259, - 22256, 22246, 22269, 22255, 22245, 22253, 22230, 22224, 22222, 22220, - 22227, 22226, 22229, 22228, 22232, 22231, 22235, 22237, 22234, 22233, - 22238, 22239, 22241, 22242, 22236, 22240, 22225, 22223, 22221, 22219, - 22279, 22277, 22278, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 22822, 22748, 22810, 22821, 22826, 22825, - 22745, 22827, 22802, 22801, 22799, 22756, 22805, 22806, 22798, 22755, - 22764, 22772, 22808, 22744, 22769, 22768, 22763, 22762, 22761, 22760, - 22786, 22754, 22785, 22753, 22828, 22759, 22809, 22820, 22819, 22767, - 22766, 22743, 22824, 22830, 22758, 22757, 22795, 22751, 22771, 22770, - 22794, 22750, 22803, 22807, 22779, 22782, 22783, 22817, 22815, 22796, - 22752, 22804, 22784, 22818, 22816, 22814, 22812, 22742, 22813, 22811, - 22829, 22746, 22823, 22747, 22781, 22749, 22800, 22797, 22780, 35762, - 35762, 35762, 35762, 22834, 22765, 22833, 22832, 22831, 22839, 22845, - 22844, 22840, 22841, 22866, 22870, 22887, 22886, 22848, 22851, 22852, - 22868, 22855, 22856, 22857, 22858, 22859, 22862, 22864, 22865, 22861, - 22872, 22873, 22874, 22875, 22880, 22876, 22877, 22881, 22883, 22842, - 22843, 22850, 22885, 22849, 22884, 22846, 22854, 22847, 22871, 22888, - 22889, 22878, 22882, 22869, 22867, 22890, 22863, 22853, 22860, 22879, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 22838, 22836, 22837, - 22835, 22787, 22788, 22789, 22790, 22791, 22792, 22793, 22773, 22774, - 22775, 22776, 22777, 22778, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 31669, 25140, - 25317, 25316, 17699, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 33692, 33691, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 30901, 30902, 30903, 30904, 30905, 30906, 30907, 30908, 30909, 30910, - 30911, 30912, 30913, 30914, 30915, 30916, 30917, 30918, 30919, 30920, - 30921, 30922, 30923, 30924, 30925, 30926, 30927, 30928, 30929, 30930, - 30931, 30932, 30933, 30934, 30935, 30936, 30937, 30938, 30939, 30940, - 30941, 30942, 30943, 30944, 30945, 30946, 30947, 30948, 30949, 30950, - 30951, 30952, 30953, 30954, 30955, 30956, 30957, 30958, 30959, 30960, - 30961, 30962, 30963, 30964, 30965, 30966, 30967, 30968, 30969, 30970, - 30971, 30972, 30973, 30974, 30975, 30976, 30977, 30978, 30979, 30980, - 30981, 30982, 30983, 30984, 30985, 30986, 30987, 30988, 30989, 30990, - 30991, 30992, 30993, 30994, 30995, 30996, 30997, 30998, 30999, 31000, - 31001, 31002, 31003, 31004, 31005, 31006, 31007, 31008, 31009, 31010, - 31011, 31012, 31013, 31014, 31015, 31016, 31017, 31018, 31019, 31020, - 31021, 31022, 31023, 31024, 31025, 31026, 31027, 31028, 31029, 31030, - 31031, 31032, 31033, 31034, 31035, 31036, 31037, 31038, 31039, 31040, - 31041, 31042, 31043, 31044, 31045, 31046, 31047, 31048, 31049, 31050, - 31051, 31052, 31053, 31054, 31055, 31056, 31057, 31058, 31059, 31060, - 31061, 31062, 31063, 31064, 31065, 31066, 31067, 31068, 31069, 31070, - 31071, 31072, 31073, 31074, 31075, 31076, 31077, 31078, 31079, 31080, - 31081, 31082, 31083, 31084, 31085, 31086, 31087, 31088, 31089, 31090, - 31091, 31092, 31093, 31094, 31095, 31096, 31097, 31098, 31099, 31100, - 31101, 31102, 31103, 31104, 31105, 31106, 31107, 31108, 31109, 31110, - 31111, 31112, 31113, 31114, 31115, 31116, 31117, 31118, 31119, 31120, - 31121, 31122, 31123, 31124, 31125, 31126, 31127, 31128, 31129, 31130, - 31131, 31132, 31133, 31134, 31135, 31136, 31137, 31138, 31139, 31140, - 31141, 31142, 31143, 31144, 31145, 31146, 31147, 31148, 31149, 31150, - 31151, 31152, 31153, 31154, 31155, 31156, 31157, 31158, 31159, 31160, - 31161, 31162, 31163, 31164, 31165, 31166, 31167, 31168, 31169, 31170, - 31171, 31172, 31173, 31174, 31175, 31176, 31177, 31178, 31179, 31180, - 31181, 31182, 31183, 31184, 31185, 31186, 31187, 31188, 31189, 31190, - 31191, 31192, 31193, 31194, 31195, 31196, 31197, 31198, 31199, 31200, - 31201, 31202, 31203, 31204, 31205, 31206, 31207, 31208, 31209, 31210, - 31211, 31212, 31213, 31214, 31215, 31216, 31217, 31218, 31219, 31220, - 31221, 31222, 31223, 31224, 31225, 31226, 31227, 31228, 31229, 31230, - 31231, 31232, 31233, 31234, 31235, 31236, 31237, 31238, 31239, 31240, - 31241, 31242, 31243, 31244, 31245, 31246, 31247, 31248, 31249, 31250, - 31251, 31252, 31253, 31254, 31255, 31256, 31257, 31258, 31259, 31260, - 31261, 31262, 31263, 31264, 31265, 31266, 31267, 31268, 31269, 31270, - 31271, 31272, 31273, 31274, 31275, 31276, 31277, 31278, 31279, 31280, - 31281, 31282, 31283, 31284, 31285, 31286, 31287, 31288, 31289, 31290, - 31291, 31292, 31293, 31294, 31295, 31296, 31297, 31298, 31299, 31300, - 31301, 31302, 31303, 31304, 31305, 31306, 31307, 31308, 31309, 31310, - 31311, 31312, 31313, 31314, 31315, 31316, 31317, 31318, 31319, 31320, - 31321, 31322, 31323, 31324, 31325, 31326, 31327, 31328, 31329, 31330, - 31331, 31332, 31333, 31334, 31335, 31336, 31337, 31338, 31339, 31340, - 31341, 31342, 31343, 31344, 31345, 31346, 31347, 31348, 31349, 31350, - 31351, 31352, 31353, 31354, 31355, 31356, 31357, 31358, 31359, 31360, - 31361, 31362, 31363, 31364, 31365, 31366, 31367, 31368, 31369, 31370, - 31371, 31372, 31373, 31374, 31375, 31376, 31377, 31378, 31379, 31380, - 31381, 31382, 31383, 31384, 31385, 31386, 31387, 31388, 31389, 31390, - 31391, 31392, 31393, 31394, 31395, 31396, 31397, 31398, 31399, 31400, - 31401, 31402, 31403, 31404, 31405, 31406, 31407, 31408, 31409, 31410, - 31411, 31412, 31413, 31414, 31415, 31416, 31417, 31418, 31419, 31420, - 31421, 31422, 31423, 31424, 31425, 31426, 31427, 31428, 31429, 31430, - 31431, 31432, 31433, 31434, 31435, 31436, 31437, 31438, 31439, 31440, - 31441, 31442, 31443, 31444, 31445, 31446, 31447, 31448, 31449, 31450, - 31451, 31452, 31453, 31454, 31455, 31456, 31457, 31458, 31459, 31460, - 31461, 31462, 31463, 31464, 31465, 31466, 31467, 31468, 31469, 31470, - 31471, 31472, 31473, 31474, 31475, 31476, 31477, 31478, 31479, 31480, - 31481, 31482, 31483, 31484, 31485, 31486, 31487, 31488, 31489, 31490, - 31491, 31492, 31493, 31494, 31495, 31496, 31497, 31498, 31499, 31500, - 31501, 31502, 31503, 31504, 31505, 31506, 31507, 31508, 31509, 31510, - 31511, 31512, 31513, 31514, 31515, 31516, 31517, 31518, 31519, 31520, - 31521, 31522, 31523, 31524, 31525, 31526, 31527, 31528, 31529, 31530, - 31531, 31532, 31533, 31534, 31535, 31536, 31537, 31538, 31539, 31540, - 31541, 31542, 31543, 31544, 31545, 31546, 31547, 31548, 31549, 31550, - 31551, 31552, 31553, 31554, 31555, 31556, 31557, 31558, 31559, 31560, - 31561, 31562, 31563, 31564, 31565, 31566, 31567, 31568, 31569, 31570, - 31571, 31572, 31573, 31574, 31575, 31576, 31577, 31578, 31579, 31580, - 31581, 31582, 31583, 31584, 31585, 31586, 31587, 31588, 31589, 31590, - 31591, 31592, 31593, 31594, 31595, 31596, 31597, 31598, 31599, 31600, - 31601, 31602, 31603, 31604, 31605, 31606, 31607, 31608, 31609, 31610, - 31611, 31612, 31613, 31614, 31615, 31616, 31617, 31618, 31619, 31620, - 31621, 31622, 31623, 31624, 31625, 31626, 31627, 31628, 31629, 31630, - 31631, 31632, 31633, 31634, 31635, 31636, 31637, 31638, 31639, 31640, - 31641, 31642, 31643, 31644, 31645, 31646, 31647, 31648, 31649, 31650, - 31651, 31652, 31653, 31654, 31655, 31656, 31657, 31658, 31659, 31660, - 31661, 31662, 31663, 31664, 31665, 31666, 31667, 31668, 17229, 17230, - 17231, 17232, 17233, 17234, 17235, 17236, 17237, 17238, 17239, 17240, - 17241, 17242, 17243, 17244, 17245, 17246, 17247, 17248, 17249, 17250, - 17251, 17252, 17253, 17254, 17255, 17256, 17257, 17258, 17259, 17260, - 17261, 17262, 17263, 17264, 17265, 17266, 17267, 17268, 17269, 17270, - 17271, 17272, 17273, 17274, 17275, 17276, 17277, 17278, 17279, 17280, - 17281, 17282, 17283, 17284, 17285, 17286, 17287, 17288, 17289, 17290, - 17291, 17292, 17293, 17294, 17295, 17296, 17297, 17298, 17299, 17300, - 17301, 17302, 17303, 17304, 17305, 17306, 17307, 17308, 17309, 17310, - 17311, 17312, 17313, 17314, 17315, 17316, 17317, 17318, 17319, 17320, - 17321, 17322, 17323, 17324, 17325, 17326, 17327, 17328, 17329, 17330, - 17331, 17332, 17333, 17334, 17335, 17336, 17337, 17338, 17339, 17340, - 17341, 17342, 17343, 17344, 17345, 17346, 17347, 17348, 17349, 17350, - 17351, 17352, 17353, 17354, 17355, 17356, 17357, 17358, 17359, 17360, - 17361, 17362, 17363, 17364, 17365, 17366, 17367, 17368, 17369, 17370, - 17371, 17372, 17373, 17374, 17375, 17376, 17377, 17378, 17379, 17380, - 17381, 17382, 17383, 17384, 17385, 17386, 17387, 17388, 17389, 17390, - 17391, 17392, 17393, 17394, 17395, 17396, 17397, 17398, 17399, 17400, - 17401, 17402, 17403, 17404, 17405, 17406, 17407, 17408, 17409, 17410, - 17411, 17412, 17413, 17414, 17415, 17416, 17417, 17418, 17419, 17420, - 17421, 17422, 17423, 17424, 17425, 17426, 17427, 17428, 17429, 17430, - 17431, 17432, 17433, 17434, 17435, 17436, 17437, 17438, 17439, 17440, - 17441, 17442, 17443, 17444, 17445, 17446, 17447, 17448, 17449, 17450, - 17451, 17452, 17453, 17454, 17455, 17456, 17457, 17458, 17459, 17460, - 17461, 17462, 17463, 17464, 17465, 17466, 17467, 17468, 17469, 17470, - 17471, 17472, 17473, 17474, 17475, 17476, 17477, 17478, 17479, 17480, - 17481, 17482, 17483, 17484, 17491, 17492, 17493, 17494, 17495, 17496, - 17497, 17498, 17499, 17500, 17501, 17502, 17503, 17504, 17505, 17506, - 17507, 17508, 17509, 17510, 17511, 17512, 17513, 17514, 17515, 17516, - 17517, 17518, 17519, 17520, 17521, 17522, 17523, 17524, 17525, 17526, - 17527, 17528, 17529, 17530, 17531, 17532, 17533, 17534, 17535, 17536, - 17537, 17538, 17539, 17540, 17541, 17542, 17543, 17544, 17545, 17546, - 17547, 17548, 17549, 17550, 17551, 17552, 17553, 17554, 17555, 17556, - 17557, 17558, 17559, 17560, 17561, 17562, 17563, 17564, 17565, 17566, - 17567, 17568, 17569, 17570, 17571, 17572, 17573, 17574, 17575, 17576, - 17577, 17578, 17579, 17580, 17581, 17582, 17583, 17584, 17585, 17586, - 17587, 17588, 17589, 17590, 17591, 17592, 17593, 17594, 17595, 17596, - 17597, 17598, 17599, 17600, 17601, 17602, 17603, 17604, 17605, 17606, - 17607, 17608, 17609, 17610, 17611, 17612, 17613, 17614, 17615, 17616, - 17617, 17618, 17619, 17620, 17621, 17622, 17623, 17624, 17625, 17626, - 17627, 17628, 17629, 17630, 17631, 17632, 17633, 17634, 17635, 17636, - 17637, 17638, 17639, 17640, 17641, 17642, 17643, 17644, 17645, 17646, - 17647, 17648, 17649, 17650, 17651, 17652, 17653, 17654, 17655, 17656, - 17657, 17658, 17659, 17660, 17661, 17662, 17663, 17664, 17665, 17666, - 17667, 17668, 17669, 17670, 17671, 17672, 17673, 17674, 17675, 17676, - 17677, 17678, 17679, 17680, 17681, 17682, 17683, 17684, 17685, 17686, - 17687, 17688, 17689, 17690, 17691, 17692, 17693, 17694, 17695, 17696, - 17697, 17698, 17485, 17486, 17487, 17488, 17489, 17490, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 16870, 16871, 16872, 16873, 35762, 16874, 16875, 16863, 16864, 16865, - 16866, 16867, 35762, 16868, 16869, 35762, 16851, 15837, 15476, 15477, - 15478, 15475, 15757, 15758, 15759, 15760, 15749, 15750, 15751, 15752, - 15753, 15744, 15745, 15746, 15747, 15748, 15754, 15755, 15756, 15515, - 15519, 15520, 15521, 15522, 15523, 15524, 15525, 15526, 15516, 15517, - 15518, 15531, 15532, 15533, 15534, 15535, 15536, 15537, 15538, 15545, - 15546, 15547, 15548, 15549, 15550, 15551, 15539, 15540, 15541, 15542, - 15543, 15544, 15528, 15529, 15530, 15527, 15640, 15641, 15642, 15643, - 15644, 15645, 15646, 15647, 15656, 15657, 15658, 15659, 15660, 15661, - 15648, 15649, 15650, 15651, 15652, 15653, 15654, 15655, 15662, 15663, - 15664, 15665, 15666, 15667, 15668, 15669, 15670, 15671, 15672, 15673, - 15702, 15703, 15704, 15705, 15695, 15696, 15697, 15698, 15699, 15700, - 15701, 15691, 15692, 15693, 15694, 15690, 15674, 15675, 15676, 15677, - 15678, 15679, 15680, 15681, 15682, 15684, 15685, 15686, 15687, 15688, - 15689, 15683, 15594, 15595, 15596, 15597, 15598, 15599, 15600, 15601, - 15602, 15587, 15588, 15589, 15590, 15591, 15592, 15593, 15586, 15608, - 15609, 15610, 15580, 15581, 15582, 15583, 15584, 15585, 15579, 15603, - 15604, 15605, 15606, 15607, 15487, 15490, 15491, 15492, 15493, 15494, - 15495, 15496, 15497, 15488, 15489, 15508, 15509, 15510, 15511, 15512, - 15513, 15514, 15498, 15499, 15500, 15501, 15502, 15503, 15504, 15505, - 15506, 15507, 15479, 15480, 15481, 15482, 15483, 15484, 15485, 15486, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 34087, 17354, 31745, 29447, 11088, 31317, 368, 1052, 24153, + 31969, 2527, 31568, 8453, 20460, 17532, 33926, 10872, 10857, 10869, + 10866, 10851, 10848, 10863, 10860, 10875, 10854, 8056, 32578, 24396, + 16792, 18147, 31743, 8452, 22741, 22787, 22797, 22813, 22828, 22873, + 22877, 22894, 22908, 22937, 22942, 22954, 22974, 22983, 22999, 23049, + 23057, 23060, 23081, 23105, 23134, 23173, 23184, 23193, 23196, 23210, + 24115, 31870, 31983, 6819, 25161, 18118, 23302, 23352, 23372, 23395, + 23431, 23490, 23498, 23516, 23535, 23572, 23578, 23592, 23631, 23644, + 23668, 23725, 23736, 23742, 23780, 23822, 23895, 23943, 23957, 23966, + 23974, 23990, 24056, 38833, 31932, 37296, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 29440, + 20815, 6131, 31610, 9951, 39398, 4937, 32560, 10839, 8647, 17433, 24163, + 29422, 33877, 31793, 26847, 10554, 31580, 34712, 34711, 4, 27575, 31465, + 27581, 6127, 34715, 25814, 32030, 38965, 38963, 38971, 20818, 22770, + 22771, 22759, 22773, 22752, 22754, 22775, 22798, 22859, 22860, 22831, + 22846, 22924, 22925, 22914, 22912, 22870, 22997, 23038, 23039, 23006, + 23024, 23013, 28287, 23022, 23165, 23166, 23135, 23138, 23205, 23126, + 23812, 23333, 23334, 23322, 23336, 23313, 23315, 23339, 23373, 23464, + 23465, 23434, 23449, 23553, 23554, 23536, 23538, 23489, 23663, 23707, + 23708, 23669, 23693, 23676, 10999, 23691, 23930, 23933, 23898, 23901, + 23985, 23844, 23976, 22757, 23320, 22742, 23303, 22767, 23330, 22802, + 23378, 22800, 23375, 22806, 23383, 22801, 23377, 22820, 23403, 22817, + 23397, 22849, 23453, 22862, 23467, 22841, 23444, 22856, 23461, 22840, + 23443, 22879, 23500, 22882, 23503, 22886, 23508, 22878, 23499, 22897, + 23519, 22904, 23528, 22918, 23550, 22916, 23545, 22927, 23556, 22921, + 23547, 22911, 23419, 23222, 24010, 22938, 23573, 22943, 23579, 23591, + 22968, 23609, 22964, 23596, 22966, 23599, 22961, 23606, 22963, 23617, + 22992, 23656, 22984, 23645, 22986, 23649, 23664, 22872, 23472, 23002, + 23686, 23041, 23710, 23019, 23682, 23223, 24013, 23069, 23759, 23061, + 23743, 23062, 23745, 23087, 23781, 23086, 23787, 23084, 23785, 23082, + 23783, 23106, 23823, 23109, 23827, 23116, 23839, 23162, 23927, 23154, + 23917, 23168, 23935, 23169, 23926, 23145, 23908, 23159, 23922, 23185, + 23958, 23197, 23975, 23198, 23213, 23991, 23216, 23997, 23212, 23994, + 23623, 23361, 22793, 22788, 23353, 23129, 23847, 23044, 22804, 23381, + 22779, 22825, 22821, 23398, 23852, 23078, 23104, 23043, 22874, 23493, + 22885, 22892, 23534, 22935, 22928, 22953, 23590, 23595, 23619, 23123, + 22990, 23650, 23005, 23031, 23700, 23046, 23722, 23055, 23733, 23299, + 23130, 23848, 22871, 23239, 23835, 23115, 23836, 23114, 23147, 23910, + 23172, 23177, 23202, 23980, 23219, 24000, 22864, 22866, 23482, 23487, + 23291, 23131, 23849, 23238, 23292, 23293, 23294, 23231, 23241, 22827, + 22815, 23425, 22973, 22962, 23630, 22998, 22987, 23667, 22765, 23328, + 22915, 23537, 23012, 23675, 23137, 23900, 23142, 23905, 23141, 23904, + 23139, 23902, 23140, 23903, 23884, 22753, 23314, 22749, 23310, 22777, + 23342, 22887, 23509, 22880, 23501, 22944, 23580, 23020, 23689, 23021, + 23690, 22865, 23484, 23575, 22826, 22814, 23424, 22881, 23502, 22906, + 23192, 22991, 23655, 22755, 23316, 22776, 23341, 23023, 23692, 22751, + 23312, 22772, 23335, 22845, 23448, 22861, 23466, 22909, 23544, 22926, + 23555, 23018, 23681, 23040, 23709, 23066, 23749, 23071, 23760, 23144, + 23907, 23167, 23934, 23085, 23786, 23108, 23825, 23209, 23989, 22898, + 23520, 22988, 23402, 23048, 23724, 23221, 24004, 22748, 23309, 22829, + 23432, 23014, 23677, 23027, 23696, 23015, 23678, 23016, 23679, 23206, + 23986, 23598, 23648, 23826, 23416, 23429, 23741, 22774, 22807, 23384, + 22956, 23110, 23794, 23999, 22888, 23510, 22794, 23171, 23125, 22863, + 23469, 22941, 23577, 23098, 23739, 23073, 23763, 23208, 23984, 23876, + 23344, 23877, 23360, 23713, 23376, 23399, 23407, 23772, 23803, 23807, + 23719, 23767, 23769, 23391, 23417, 23507, 23810, 23268, 23514, 23779, + 23853, 23527, 23533, 23557, 23568, 23246, 23604, 23593, 23612, 23620, + 23879, 23880, 23639, 23652, 23661, 23279, 23364, 23253, 23390, 23735, + 23864, 23867, 23870, 23754, 23755, 23750, 23770, 23255, 23247, 23800, + 23475, 23418, 23820, 23479, 23871, 23838, 23896, 23937, 23950, 23873, + 23888, 23881, 23286, 24003, 23993, 23481, 23483, 23295, 23297, 23236, + 23288, 23234, 23266, 23389, 23269, 23275, 23574, 23886, 23248, 23738, + 23296, 23240, 23426, 23412, 23427, 23891, 23840, 23890, 23496, 23627, + 23628, 23233, 23235, 23854, 23855, 27905, 27906, 27914, 27936, 27963, + 27966, 27861, 27983, 27985, 27834, 27777, 27991, 27688, 27842, 27844, + 27820, 27793, 27840, 27822, 27846, 27993, 27779, 27769, 6060, 27997, + 27823, 27687, 27792, 27818, 27809, 27815, 27814, 27990, 27800, 27721, + 27720, 27992, 27778, 27833, 27831, 4930, 11194, 32162, 29978, 33836, + 11253, 27847, 27770, 27904, 27916, 27943, 27984, 27940, 27784, 27799, + 27827, 27812, 27789, 27999, 27998, 27996, 27994, 27776, 27805, 27817, + 27807, 27810, 27811, 27830, 27829, 27828, 27813, 27839, 27690, 27790, + 27691, 27791, 27849, 27832, 27806, 8264, 8062, 8146, 8428, 8370, 8390, + 8072, 8171, 8168, 8277, 8415, 8197, 8078, 8443, 8196, 8198, 8080, 8280, + 8436, 8083, 8398, 8084, 8265, 8063, 8355, 8411, 8349, 8279, 8352, 8438, + 8202, 8395, 8379, 8394, 8399, 8174, 8170, 8414, 8085, 8148, 8388, 8442, + 8075, 8283, 8079, 8147, 8074, 8281, 8432, 8375, 8369, 8199, 8431, 8419, + 8367, 8418, 8366, 8406, 8282, 8422, 8426, 8450, 8444, 8185, 8266, 8064, + 8274, 8273, 8276, 8275, 8076, 8211, 8195, 8348, 8381, 8278, 8071, 8356, + 8437, 8269, 8402, 8353, 8212, 8449, 8344, 8403, 8401, 8407, 8173, 8068, + 8190, 8417, 8179, 8178, 8182, 8183, 8191, 8180, 8189, 8296, 8304, 8309, + 8317, 8320, 8302, 8334, 8336, 8338, 8323, 8326, 8341, 8342, 18332, 18596, + 18227, 18463, 18414, 18410, 18321, 18578, 40951, 40951, 18653, 18603, + 18604, 18602, 18658, 18335, 40951, 40951, 40951, 40951, 18618, 18348, + 18225, 18201, 18258, 18248, 18272, 40951, 18304, 40951, 18319, 18294, + 18510, 18204, 18331, 18329, 18327, 18249, 18333, 18228, 18325, 18259, + 18328, 18334, 18336, 18337, 18338, 18295, 18324, 18305, 40951, 18307, + 18326, 18310, 18322, 18330, 18323, 18274, 18264, 18315, 18460, 18475, + 18500, 18519, 18530, 18435, 18595, 18593, 18465, 18466, 18597, 18476, + 18590, 18501, 18541, 18598, 18599, 18600, 18601, 18568, 18581, 18582, + 18592, 18588, 18591, 18521, 18579, 18594, 18580, 18543, 18506, 18526, + 18577, 18539, 18567, 18343, 18655, 18614, 18622, 18620, 18621, 18430, + 18431, 18395, 18406, 18462, 18405, 18587, 18408, 18464, 18407, 18542, + 18404, 18585, 8528, 8622, 8506, 8600, 8502, 8596, 8500, 8594, 8498, 8592, + 8527, 8621, 8497, 8591, 18394, 18433, 18411, 18409, 18344, 18412, 18434, + 18309, 18589, 18339, 18308, 18586, 18432, 18341, 18342, 18340, 10218, + 10220, 10167, 10206, 10142, 10171, 10158, 10277, 10290, 10113, 10116, + 10130, 10245, 10215, 10258, 10175, 10143, 10159, 10291, 10200, 10179, + 10217, 10284, 10280, 10213, 10256, 10230, 10181, 10197, 10186, 10249, + 10117, 10192, 10195, 10127, 10137, 10199, 10207, 10133, 10160, 10263, + 10261, 10211, 10274, 10266, 10180, 10279, 10271, 10302, 10318, 10492, + 10359, 10338, 10376, 10485, 10481, 10372, 10434, 10389, 10340, 10356, + 10345, 10415, 10420, 10351, 10354, 10452, 10463, 10358, 10366, 10458, + 10319, 10441, 10439, 10370, 10475, 10444, 10339, 10480, 10472, 10377, + 10379, 10326, 10365, 10468, 10330, 10317, 10478, 10491, 10408, 10414, + 10455, 10404, 10374, 10436, 10334, 10250, 10416, 10273, 10474, 10226, + 10385, 10112, 10406, 10222, 10381, 10155, 10314, 10223, 10382, 10246, + 10405, 10120, 10424, 10289, 10490, 10228, 10387, 10229, 10388, 10141, + 10467, 10121, 10429, 10251, 10417, 10253, 10419, 10243, 10402, 10523, + 8139, 8133, 8136, 8134, 8135, 8089, 8142, 10257, 10435, 10270, 10448, + 10193, 10352, 10202, 10361, 10204, 10363, 10201, 10360, 10286, 10487, + 10282, 10483, 10231, 10390, 10233, 10392, 10234, 10393, 10153, 10312, + 10188, 10347, 10296, 10496, 10118, 10421, 10149, 10308, 10196, 10355, + 10129, 10454, 10268, 10446, 10269, 10447, 10208, 10367, 10295, 10495, + 10162, 10321, 10163, 10322, 10259, 10437, 10146, 10305, 10147, 10306, + 10299, 10287, 10488, 10232, 10391, 10184, 10343, 10191, 10350, 10190, + 10349, 10244, 10403, 10198, 10357, 10423, 10145, 10304, 10144, 10303, + 10294, 10494, 10219, 10378, 10254, 10432, 10255, 10433, 10285, 10486, + 10281, 10482, 10148, 10307, 10216, 10375, 10214, 10373, 10252, 10418, + 10151, 10310, 10152, 10311, 10194, 10353, 10140, 10466, 10139, 10465, + 10138, 10464, 10161, 10320, 10203, 10362, 10275, 10476, 10205, 10364, + 10209, 10368, 10210, 10369, 10237, 10396, 10236, 10395, 10242, 10401, + 10235, 10394, 10238, 10397, 10239, 10398, 10240, 10399, 10241, 10400, + 10125, 10428, 10185, 10344, 10114, 10409, 10126, 10431, 10272, 10473, + 10293, 10493, 10292, 10471, 10150, 10309, 10182, 10341, 10187, 10346, + 10119, 10422, 10260, 10438, 10189, 10348, 10173, 10332, 10177, 10336, + 10183, 10342, 40951, 2445, 2452, 2436, 2458, 2432, 2456, 2433, 2434, + 2423, 2455, 2451, 2448, 2450, 2428, 2440, 2457, 2438, 2435, 2426, 2453, + 2424, 2454, 2443, 2447, 2427, 2442, 2437, 2431, 2444, 2446, 2422, 2430, + 2429, 2425, 2441, 2439, 2459, 2449, 40951, 40951, 2509, 2421, 2462, 2461, + 2460, 2513, 2420, 2482, 2485, 2495, 2473, 2501, 2469, 2499, 2470, 2471, + 2484, 2498, 2494, 2491, 2493, 2465, 2477, 2500, 2475, 2472, 2463, 2496, + 2488, 2497, 2480, 2487, 2464, 2479, 2474, 2468, 2481, 2486, 2483, 2467, + 2466, 2490, 2478, 2476, 2502, 2492, 2503, 2489, 2512, 2510, 40951, 40951, + 32017, 24176, 2511, 40951, 19805, 19808, 19809, 19816, 19817, 19813, + 19820, 19818, 19803, 19815, 19812, 19796, 19797, 19798, 19806, 19811, + 19804, 19793, 19801, 19802, 19799, 19800, 19795, 19807, 19810, 19814, + 19821, 19822, 19794, 19819, 19900, 19915, 19904, 19902, 19903, 19905, + 19917, 19913, 19908, 19909, 19906, 19907, 19911, 19901, 19919, 19925, + 19912, 19922, 19914, 19916, 19923, 19899, 19898, 19924, 19910, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 19823, 19830, 19873, + 19871, 19848, 19879, 19853, 19850, 19865, 19890, 19839, 19833, 19875, + 19845, 19877, 19844, 19851, 19855, 19829, 19841, 19836, 19843, 19869, + 19846, 19863, 19857, 19867, 40951, 40951, 40951, 40951, 19926, 19894, + 19897, 19895, 19921, 19920, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 2264, 2299, 1093, 2300, 2301, 2263, + 2408, 2409, 2275, 2406, 2407, 2598, 1062, 1075, 2270, 2305, 2298, 2304, + 2296, 2297, 2306, 2326, 2312, 2341, 2308, 2357, 2356, 2292, 1381, 1083, + 2396, 2404, 1336, 1288, 1151, 1141, 1569, 1144, 1585, 1123, 1165, 1512, + 1511, 1532, 1314, 1273, 1355, 1190, 1529, 1433, 1611, 1467, 1480, 1459, + 1206, 1489, 1606, 1113, 1260, 1342, 1339, 1235, 1234, 1233, 2384, 1240, + 1424, 1325, 1360, 1373, 1397, 1290, 1564, 1159, 1576, 1091, 1073, 1103, + 1085, 1069, 1099, 2293, 2361, 2117, 1098, 1097, 2360, 2282, 2118, 2405, + 2400, 2399, 2401, 2276, 1086, 2403, 2416, 2418, 2415, 2414, 2411, 2410, + 2413, 2412, 2419, 2417, 2268, 1079, 2398, 1094, 1218, 1220, 1486, 1156, + 1148, 1147, 1310, 1311, 1313, 1550, 1312, 1538, 1544, 1185, 1520, 1519, + 1412, 1524, 1180, 1283, 1281, 1392, 1221, 1280, 1499, 1506, 1215, 1200, + 1197, 1198, 1203, 1212, 1226, 1193, 1199, 1450, 1436, 1447, 1444, 1437, + 1445, 1440, 1322, 1443, 1468, 1471, 1472, 1462, 1461, 1493, 1116, 1219, + 1243, 1241, 1557, 1245, 1419, 1427, 1428, 1337, 1488, 1331, 1330, 1382, + 1328, 1251, 1254, 1383, 1253, 1267, 1252, 1364, 1362, 1366, 1365, 1406, + 1398, 1453, 1407, 1403, 1301, 1501, 1298, 1291, 1292, 1515, 1573, 1349, + 1603, 1549, 1600, 1352, 1572, 1556, 1229, 1594, 1590, 1566, 1615, 1595, + 1577, 1580, 1095, 1164, 2315, 2314, 2317, 2316, 2343, 2325, 2323, 1084, + 2383, 2340, 2339, 2309, 2318, 2355, 2319, 2359, 2358, 2337, 2320, 2272, + 1082, 1081, 2280, 2354, 1192, 1441, 17374, 17376, 17373, 17372, 17369, + 17368, 17371, 17370, 17377, 17375, 1481, 1207, 1262, 2303, 2302, 1297, + 34842, 34916, 34913, 34914, 34910, 34851, 34838, 34839, 34915, 34912, + 34837, 34847, 34846, 34845, 40951, 34835, 34883, 34879, 34893, 34889, + 34890, 34853, 34852, 34854, 34896, 34894, 34855, 34886, 34887, 34891, + 34892, 34884, 34856, 34868, 34895, 34875, 34882, 34897, 34869, 34873, + 34881, 34885, 34874, 34880, 34888, 34870, 34872, 34871, 34902, 34901, + 34900, 34905, 34904, 34903, 34907, 34906, 34841, 34840, 34850, 34849, + 34848, 34844, 34843, 34908, 34922, 34923, 34909, 34920, 34919, 34918, + 34917, 34899, 34898, 34921, 34836, 40951, 40951, 34876, 34877, 34878, + 1171, 1174, 1169, 1170, 1172, 1173, 1167, 1282, 1279, 1196, 1191, 1438, + 1474, 1118, 1114, 1117, 1246, 1244, 1344, 1340, 1338, 1376, 1375, 1404, + 1401, 1402, 1367, 1439, 1442, 1473, 1278, 1276, 1470, 1434, 1277, 1150, + 1149, 1232, 1231, 1230, 1568, 1567, 1579, 1578, 1274, 1475, 1469, 1327, + 36865, 36878, 36874, 36891, 36890, 36869, 36866, 36854, 36885, 36870, + 36859, 36858, 36881, 36868, 36861, 36862, 36879, 36857, 36887, 36880, + 36892, 36873, 36872, 36871, 36883, 36864, 36867, 36882, 36888, 36877, + 36876, 36856, 36884, 36889, 36855, 36863, 36860, 36886, 36850, 36849, + 36896, 36852, 36897, 36895, 36851, 36853, 36894, 36893, 36898, 36875, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 29306, 29308, 29305, 29304, 29301, 29300, + 29303, 29302, 29309, 29307, 29341, 29328, 29342, 29327, 29343, 29325, + 29324, 29312, 29317, 29330, 29336, 29338, 29316, 29326, 29311, 29323, + 29322, 29337, 29329, 29331, 29333, 29334, 29319, 29335, 29320, 29318, + 29332, 29339, 29340, 29321, 29314, 29313, 29315, 29294, 29295, 29296, + 29292, 29289, 29290, 29291, 29293, 29288, 29345, 29344, 29347, 29346, + 29297, 29348, 29310, 40951, 40951, 29298, 29299, 29349, 32389, 32376, + 32390, 32378, 32381, 32377, 32392, 32380, 32387, 32396, 32382, 32383, + 32393, 32395, 32384, 32379, 32397, 32388, 32394, 32391, 32385, 32386, + 32399, 32400, 32403, 32398, 32404, 32401, 32423, 32433, 32428, 32422, + 32432, 32427, 32421, 32431, 32405, 32429, 32425, 32435, 32406, 32424, + 32434, 32426, 32430, 32402, 40951, 40951, 32413, 32407, 32409, 32412, + 32410, 32414, 32436, 32419, 32417, 32420, 32416, 32418, 32411, 32415, + 32408, 40951, 25603, 25590, 25592, 25591, 25593, 25602, 25600, 25605, + 25585, 25583, 25582, 25594, 25595, 25596, 25586, 25604, 25597, 25588, + 25598, 25599, 25587, 25584, 25601, 25606, 25589, 25581, 25607, 25608, + 40951, 40951, 25609, 40951, 34861, 34866, 34862, 34864, 34860, 34859, + 34863, 34867, 34858, 34857, 34865, 40951, 40951, 40951, 40951, 40951, + 1137, 1127, 1138, 1154, 1136, 1124, 1133, 1130, 1134, 1132, 1155, 1129, + 1140, 1126, 1128, 1139, 1125, 1131, 1135, 2385, 2386, 2388, 1537, 2281, + 2274, 1405, 1275, 1494, 1492, 1341, 2402, 40951, 2271, 2273, 40951, + 40951, 40951, 40951, 40951, 2269, 2327, 2348, 2347, 2350, 2116, 2365, + 1076, 1096, 1168, 1175, 1315, 1491, 1242, 1425, 1361, 1374, 1591, 1593, + 1446, 1565, 1458, 1372, 1194, 1460, 1255, 1487, 1614, 1115, 1329, 1426, + 1166, 1413, 1517, 1435, 1592, 1111, 1109, 1112, 1414, 1518, 1539, 1500, + 1343, 1261, 1110, 1317, 1316, 1363, 1272, 2307, 2310, 2338, 2333, 2342, + 1107, 1106, 2364, 1108, 1105, 2353, 2328, 2324, 2345, 2344, 2321, 2346, + 2329, 2331, 2330, 2332, 2335, 2334, 2311, 2322, 1080, 2397, 1065, 1063, + 1067, 1066, 1064, 1068, 2391, 2393, 2395, 2390, 2392, 2394, 2267, 2266, + 2265, 2336, 1088, 1087, 1100, 1621, 2277, 1620, 2279, 1077, 1078, 2278, + 1072, 2119, 10772, 10762, 10776, 10781, 10697, 10671, 10672, 10741, + 10742, 10717, 10718, 10736, 10738, 10679, 10698, 10749, 10673, 10680, + 10699, 10703, 10674, 10711, 10710, 10692, 10690, 10723, 10677, 10681, + 10728, 10726, 10724, 10733, 10732, 10685, 10684, 10722, 10735, 10734, + 10687, 10686, 10725, 10721, 10744, 10743, 10708, 10707, 10695, 10716, + 10714, 10713, 10731, 10730, 10729, 10740, 10700, 10701, 10702, 10694, + 10794, 10793, 10774, 10775, 10784, 10806, 10807, 10796, 10797, 10802, + 10803, 10790, 10800, 10808, 10785, 10791, 10801, 10792, 10786, 10780, + 10795, 10787, 10822, 10783, 10782, 10666, 10663, 10789, 10799, 10798, + 10748, 10709, 10689, 10746, 10682, 10712, 10747, 10715, 10737, 10739, + 10804, 10805, 10810, 10809, 10817, 10819, 10816, 10815, 10812, 10811, + 10814, 10813, 10820, 10818, 10664, 10779, 10678, 10705, 10704, 10675, + 10720, 10719, 10696, 10745, 10693, 10691, 10727, 10688, 10683, 10706, + 3549, 3617, 3620, 3622, 40951, 3573, 3574, 3587, 3588, 3585, 3586, 3567, + 3569, 40951, 40951, 3609, 3575, 40951, 40951, 3610, 3576, 3560, 3557, + 3601, 3600, 3589, 3599, 3598, 3603, 3602, 3591, 3582, 3581, 3578, 3577, + 3590, 3584, 3583, 3580, 3579, 3592, 40951, 3605, 3604, 3597, 3596, 3608, + 3572, 3561, 40951, 3607, 40951, 40951, 40951, 3593, 3594, 3595, 3606, + 40951, 40951, 3618, 3619, 3624, 3633, 3634, 3627, 3628, 3629, 3630, + 40951, 40951, 3635, 3625, 40951, 40951, 3636, 3626, 3621, 3558, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 3550, 40951, 40951, + 40951, 40951, 3565, 3564, 40951, 3571, 3568, 3570, 3631, 3632, 40951, + 40951, 3643, 3645, 3642, 3641, 3638, 3637, 3640, 3639, 3646, 3644, 3563, + 3562, 3612, 3611, 3551, 3555, 3554, 3553, 3552, 3556, 3623, 3647, 3566, + 3548, 3616, 40951, 40951, 18887, 18888, 18893, 40951, 18839, 18840, + 18855, 18856, 18853, 18854, 40951, 40951, 40951, 40951, 18874, 18841, + 40951, 40951, 18873, 18842, 18838, 18837, 18835, 18834, 18859, 18866, + 18865, 18868, 18867, 18861, 18850, 18849, 18844, 18843, 18860, 18852, + 18851, 18846, 18845, 18862, 40951, 18870, 18869, 18864, 18863, 18877, + 18879, 18848, 40951, 18858, 18857, 40951, 18878, 18871, 40951, 18872, + 18876, 40951, 40951, 18890, 40951, 18894, 18899, 18900, 18897, 18898, + 40951, 40951, 40951, 40951, 18902, 18895, 40951, 40951, 18901, 18896, + 18892, 40951, 40951, 40951, 18889, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 18836, 18833, 18880, 18847, 40951, 18875, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 18912, 18914, 18911, 18910, 18907, + 18906, 18909, 18908, 18915, 18913, 18903, 18832, 18904, 18916, 18905, + 18891, 18831, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 18727, 18735, 18737, 40951, 18677, 18678, 18696, 18697, + 18694, 18695, 18689, 18691, 18753, 40951, 18724, 18679, 18754, 40951, + 18725, 18680, 18717, 18716, 18713, 18712, 18701, 18711, 18710, 18715, + 18714, 18703, 18686, 18685, 18682, 18681, 18702, 18688, 18687, 18684, + 18683, 18704, 40951, 18719, 18718, 18709, 18708, 18721, 18723, 18722, + 40951, 18699, 18698, 40951, 18693, 18705, 18706, 18707, 18720, 40951, + 40951, 18733, 18734, 18740, 18749, 18750, 18743, 18744, 18745, 18746, + 18738, 40951, 18751, 18741, 18739, 40951, 18752, 18742, 18736, 40951, + 40951, 18767, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 18690, 18692, 18747, + 18748, 40951, 40951, 18763, 18765, 18762, 18761, 18758, 18757, 18760, + 18759, 18766, 18764, 18755, 18756, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 18700, 18732, 18731, 18728, 18730, 18726, 18729, 40951, + 30620, 30623, 30626, 40951, 30571, 30572, 30590, 30591, 30588, 30589, + 30583, 30585, 40951, 40951, 30616, 30573, 40951, 40951, 30617, 30574, + 30610, 30609, 30606, 30605, 30594, 30604, 30603, 30608, 30607, 30596, + 30580, 30579, 30576, 30575, 30595, 30582, 30581, 30578, 30577, 30597, + 40951, 30612, 30611, 30602, 30601, 30614, 30570, 30568, 40951, 30593, + 30592, 40951, 30587, 30598, 30599, 30600, 30613, 40951, 40951, 30621, + 30622, 30627, 30636, 30637, 30630, 30631, 30632, 30633, 40951, 40951, + 30638, 30628, 40951, 40951, 30639, 30629, 30625, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 30624, 30557, 30558, 40951, 40951, 40951, + 40951, 30567, 30566, 40951, 30569, 30584, 30586, 30634, 30635, 40951, + 40951, 30646, 30648, 30645, 30644, 30641, 30640, 30643, 30642, 30649, + 30647, 30565, 30615, 30562, 30561, 30564, 30559, 30560, 30563, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 35480, + 35497, 40951, 35443, 35444, 35456, 35457, 35452, 35453, 40951, 40951, + 40951, 35461, 35462, 35445, 40951, 35454, 35455, 35446, 35466, 40951, + 40951, 40951, 35438, 35463, 40951, 35465, 40951, 35439, 35441, 40951, + 40951, 40951, 35437, 35442, 40951, 40951, 40951, 35440, 35436, 35468, + 40951, 40951, 40951, 35467, 35470, 35451, 35450, 35449, 35448, 35447, + 35469, 35458, 35459, 35460, 35464, 40951, 40951, 40951, 40951, 35770, + 35777, 35778, 35773, 35774, 40951, 40951, 40951, 35779, 35780, 35771, + 40951, 35775, 35776, 35772, 35496, 40951, 40951, 35783, 40951, 40951, + 40951, 40951, 40951, 40951, 35372, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 35409, + 35411, 35408, 35407, 35404, 35403, 35406, 35405, 35412, 35410, 35474, + 35472, 35473, 35402, 35782, 35781, 35401, 35399, 35371, 35477, 35475, + 40951, 40951, 40951, 40951, 40951, 36730, 36731, 36736, 36738, 36729, + 36690, 36691, 36706, 36707, 36702, 36703, 36697, 36699, 40951, 36723, + 36724, 36692, 40951, 36704, 36705, 36693, 36720, 36719, 36716, 36715, + 36679, 36714, 36713, 36718, 36717, 36681, 36686, 36685, 36673, 36672, + 36680, 36689, 36687, 36676, 36674, 36677, 40951, 36722, 36721, 36712, + 36711, 36726, 36727, 36684, 36683, 36696, 36695, 36694, 36701, 36708, + 36709, 36710, 36725, 40951, 40951, 36734, 36735, 36739, 36750, 36751, + 36742, 36743, 36744, 36745, 40951, 36752, 36753, 36740, 40951, 36748, + 36749, 36741, 36737, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 36728, 36664, 40951, 36688, 36675, 36682, 40951, 40951, 36678, 40951, + 40951, 36698, 36700, 36746, 36747, 40951, 40951, 36760, 36762, 36759, + 36758, 36755, 36754, 36757, 36756, 36763, 36761, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 36732, 36671, 36669, 36667, 36665, 36670, + 36668, 36666, 36733, 21274, 21273, 21278, 21282, 21275, 21222, 21223, + 21248, 21249, 21244, 21245, 21239, 21241, 40951, 21265, 21266, 21224, + 40951, 21246, 21247, 21225, 21262, 21261, 21258, 21257, 21219, 21256, + 21255, 21260, 21259, 21221, 21236, 21235, 21227, 21226, 21220, 21238, + 21237, 21229, 21228, 21217, 40951, 21264, 21263, 21254, 21253, 21269, + 21270, 21234, 21233, 21232, 21231, 40951, 21243, 21250, 21251, 21252, + 21268, 40951, 40951, 21276, 21277, 21284, 21295, 21296, 21287, 21288, + 21289, 21290, 40951, 21297, 21298, 21285, 40951, 21293, 21294, 21286, + 21281, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 21271, 21283, + 40951, 40951, 40951, 40951, 40951, 40951, 21218, 21267, 40951, 21240, + 21242, 21291, 21292, 40951, 40951, 21305, 21307, 21304, 21303, 21300, + 21299, 21302, 21301, 21308, 21306, 40951, 21279, 21280, 21272, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 25543, 25545, 25550, 25548, 25500, 25474, 25476, 25520, 25521, + 25516, 25517, 25501, 25503, 40951, 25535, 25536, 25477, 40951, 25518, + 25519, 25478, 25532, 25531, 25528, 25527, 25508, 25489, 25488, 25530, + 25529, 25509, 25497, 25495, 25492, 25491, 25507, 25499, 25498, 25494, + 25493, 25510, 25506, 25534, 25533, 25526, 25525, 25538, 25539, 25515, + 25514, 25513, 25512, 25511, 25505, 25522, 25523, 25524, 25537, 25496, + 25546, 25544, 25549, 25552, 25563, 25564, 25555, 25556, 25557, 25558, + 40951, 25565, 25566, 25553, 40951, 25561, 25562, 25554, 25547, 25490, + 25551, 40951, 40951, 40951, 40951, 25486, 25487, 25481, 25567, 25466, + 25464, 25471, 25461, 25462, 25472, 25465, 25475, 25502, 25504, 25559, + 25560, 40951, 40951, 25457, 25459, 25456, 25455, 25452, 25451, 25454, + 25453, 25460, 25458, 25542, 25540, 25541, 25469, 25468, 25473, 25463, + 25467, 25470, 25450, 25483, 25482, 25484, 25479, 25480, 25485, 40951, + 33736, 33735, 33737, 40951, 33681, 33678, 33666, 33665, 33689, 33688, + 33691, 33690, 33687, 33686, 33685, 33684, 33683, 33682, 33679, 33710, + 33709, 33680, 40951, 40951, 40951, 33675, 33700, 33673, 33698, 33723, + 33713, 33672, 33697, 33674, 33699, 33720, 33721, 33714, 33667, 33692, + 33669, 33694, 33704, 33711, 33668, 33693, 33670, 33695, 33707, 40951, + 33712, 33676, 33701, 33671, 33696, 33702, 33677, 33719, 33717, 40951, + 33706, 40951, 40951, 33718, 33722, 33705, 33708, 33716, 33703, 33715, + 40951, 40951, 40951, 33734, 40951, 40951, 40951, 40951, 33754, 33746, + 33741, 33747, 33742, 33748, 40951, 33743, 40951, 33744, 33749, 33740, + 33753, 33750, 33751, 33752, 33745, 40951, 40951, 40951, 40951, 40951, + 40951, 33730, 33732, 33729, 33728, 33725, 33724, 33727, 33726, 33733, + 33731, 40951, 40951, 33738, 33739, 33755, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 36917, 36914, + 36912, 36911, 36913, 36915, 36931, 36900, 36902, 36901, 36960, 36903, + 36972, 36904, 36969, 36965, 36962, 36963, 36933, 36905, 36968, 36967, + 36964, 36966, 36934, 36899, 36940, 36937, 36906, 36938, 36907, 36939, + 36929, 36973, 36941, 36942, 36919, 36921, 36970, 36958, 36957, 36959, + 36910, 36918, 36974, 36909, 36935, 36943, 36923, 36946, 36948, 36953, + 36954, 36950, 36951, 36949, 36952, 36936, 40951, 40951, 40951, 40951, + 36975, 36955, 36947, 36956, 36945, 36944, 36920, 36928, 36927, 36926, + 36924, 36925, 36922, 36961, 36932, 36971, 36908, 36982, 36984, 36981, + 36980, 36977, 36976, 36979, 36978, 36985, 36983, 36930, 36916, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 22578, 22576, 40951, 22577, + 40951, 22591, 22606, 22610, 22590, 22600, 40951, 22592, 22607, 22588, + 22586, 22585, 22583, 22582, 22587, 22611, 22603, 22601, 22602, 22584, + 22608, 22609, 22596, 22594, 22573, 22595, 22572, 22589, 22612, 22615, + 22580, 40951, 22581, 40951, 22614, 22597, 22598, 22599, 22604, 22593, + 22616, 22605, 22642, 22624, 22629, 22625, 22627, 22637, 22638, 22631, + 22632, 22633, 22634, 22619, 22630, 22618, 22617, 40951, 40951, 22635, + 22636, 22639, 22628, 22626, 40951, 22640, 40951, 22623, 22620, 22621, + 22622, 22653, 22654, 22641, 40951, 22649, 22651, 22648, 22647, 22644, + 22643, 22646, 22645, 22652, 22650, 40951, 40951, 22569, 22568, 22575, + 22574, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 37208, 37115, 37113, 37114, 37123, 37111, 37108, + 37112, 37132, 37103, 37101, 37124, 37139, 37133, 37129, 37136, 37128, + 37131, 37130, 37107, 37116, 37099, 37098, 37026, 37027, 37025, 37147, + 37148, 37149, 37151, 37152, 37150, 37048, 37050, 37047, 37046, 37043, + 37042, 37045, 37044, 37051, 37049, 37040, 37037, 37036, 37033, 37032, + 37035, 37034, 37041, 37039, 37038, 37104, 37126, 37106, 37125, 37109, + 37135, 37117, 37118, 37119, 37120, 37158, 37144, 37057, 37055, 37085, + 37084, 37067, 37083, 37082, 37092, 40951, 37069, 37077, 37076, 37062, + 37061, 37068, 37079, 37078, 37066, 37065, 37070, 37087, 37086, 37081, + 37080, 37094, 37075, 37074, 37064, 37063, 37095, 37088, 37089, 37090, + 37096, 37059, 37093, 37071, 37072, 37073, 37091, 37097, 37054, 37060, + 37056, 37058, 40951, 40951, 40951, 40951, 37232, 37228, 37229, 37220, + 37221, 37222, 37223, 37224, 37225, 37230, 37231, 37226, 37227, 37155, + 37156, 37218, 37219, 37146, 37160, 37121, 37138, 37142, 37157, 37143, + 37145, 37140, 37141, 37159, 37207, 37206, 37205, 37172, 37171, 37191, + 37190, 37173, 37189, 37188, 37198, 40951, 37175, 37183, 37182, 37165, + 37164, 37174, 37185, 37184, 37169, 37168, 37176, 37193, 37192, 37187, + 37186, 37200, 37181, 37180, 37167, 37166, 37202, 37194, 37195, 37196, + 37203, 37201, 37199, 37177, 37178, 37179, 37197, 37204, 37170, 37162, + 37163, 37161, 40951, 37052, 37053, 37028, 37029, 37031, 37030, 37209, + 37216, 37214, 37217, 37215, 37210, 37213, 37212, 37211, 40951, 37154, + 37153, 37102, 37105, 37127, 37122, 37110, 32022, 24181, 32023, 24182, + 37137, 37134, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 28582, + 28561, 28560, 28559, 28591, 28659, 28658, 28661, 28660, 28592, 28589, + 28637, 28636, 28643, 28642, 28590, 28621, 28638, 28645, 28644, 28593, + 28663, 28662, 28657, 28656, 28588, 28665, 28595, 28655, 28641, 28620, + 28664, 28654, 28551, 28615, 28652, 28653, 28646, 28647, 28554, 28587, + 28666, 28553, 28758, 28742, 28765, 28766, 28759, 28760, 28754, 28739, + 28747, 28748, 28755, 28683, 28702, 28707, 28706, 28682, 28546, 28544, + 28545, 28543, 28558, 28783, 28785, 28782, 28781, 28778, 28777, 28780, + 28779, 28786, 28784, 28705, 28694, 28712, 28716, 28711, 28715, 28596, + 28619, 28648, 28649, 28650, 28651, 28761, 28762, 28763, 28764, 28586, + 28585, 28583, 28584, 28549, 28548, 28547, 28618, 28753, 28727, 28728, + 28640, 28639, 28756, 28757, 28697, 28698, 28699, 28700, 28701, 28557, + 28556, 28555, 28743, 28745, 28746, 28744, 28610, 28609, 28608, 28606, + 28614, 28598, 28611, 28599, 28601, 28612, 28604, 28602, 28613, 28550, + 28752, 28749, 28750, 28751, 28689, 28690, 28691, 28692, 28687, 28688, + 28686, 28594, 28703, 28678, 28680, 28677, 28676, 28673, 28672, 28675, + 28674, 28681, 28679, 28684, 28685, 28740, 28741, 28714, 28713, 17728, + 17754, 17739, 17760, 17761, 17759, 17749, 17750, 17762, 17743, 17752, + 17755, 17757, 17763, 17745, 17748, 17753, 17747, 17751, 17764, 17744, + 17742, 17738, 17758, 17746, 17734, 17737, 17741, 17736, 17735, 17756, + 17740, 17729, 17733, 17731, 17766, 17730, 17732, 40951, 17765, 40951, + 40951, 40951, 40951, 40951, 17727, 40951, 40951, 17811, 17842, 17819, + 17848, 17817, 17847, 17840, 17837, 17849, 17829, 17831, 17843, 17845, + 17850, 17833, 17839, 17841, 17835, 17838, 17808, 17832, 17828, 17818, + 17846, 17834, 17812, 17815, 17827, 17814, 17813, 17844, 17826, 17822, + 17825, 17823, 17852, 17820, 17824, 17853, 17851, 17816, 17836, 17810, + 17900, 27795, 17809, 17821, 17830, 19147, 19221, 19155, 19226, 19219, + 19183, 19150, 19165, 19223, 19197, 19215, 19129, 19127, 19213, 19114, + 19149, 19233, 19162, 19236, 19157, 19222, 19159, 19158, 19230, 19193, + 19217, 19196, 19142, 19152, 19146, 19175, 19180, 19179, 19166, 19170, + 19168, 19171, 19172, 19169, 19177, 19176, 19178, 19173, 19144, 19145, + 19204, 19210, 19209, 19202, 19207, 19198, 19199, 19201, 19212, 19206, + 19205, 19203, 19208, 19200, 19211, 19119, 19118, 19124, 19123, 19182, + 19139, 19138, 19136, 19131, 19141, 19132, 19225, 19135, 19134, 19137, + 19130, 19234, 19128, 19121, 19117, 19126, 19122, 19115, 19116, 19120, + 19125, 19163, 19143, 19224, 19235, 19148, 19161, 19156, 19160, 19227, + 19238, 19476, 19382, 19392, 19440, 19444, 19394, 19393, 19446, 19445, + 19421, 19473, 19474, 19431, 19452, 19432, 19472, 19471, 19475, 19461, + 19398, 19450, 19405, 19390, 19391, 19442, 19441, 19396, 19397, 19395, + 19448, 19449, 19429, 19428, 19424, 19422, 19430, 19453, 19454, 19455, + 19460, 19459, 19437, 19438, 19436, 19433, 19439, 19466, 19465, 19464, + 19463, 19462, 19470, 19468, 19404, 19401, 19451, 19406, 19408, 19415, + 19419, 19417, 19407, 19383, 19385, 19388, 19387, 19420, 19389, 19443, + 19447, 19426, 19427, 19258, 19359, 19261, 19281, 19284, 19288, 19363, + 19309, 19310, 19315, 19319, 19328, 19331, 19324, 19336, 19268, 19298, + 19301, 19337, 19354, 19250, 19241, 19244, 19267, 19372, 19294, 19245, + 19260, 19262, 19287, 19286, 19290, 19289, 19285, 19370, 19364, 19312, + 19335, 19329, 19330, 19349, 19316, 19318, 19323, 19322, 19313, 19327, + 19325, 19314, 19332, 19278, 19275, 19269, 19274, 19273, 19271, 19276, + 19280, 19257, 19299, 19303, 19308, 19256, 19339, 19347, 19340, 19343, + 19291, 19252, 19253, 19362, 19251, 19373, 19377, 19380, 19296, 19255, + 19248, 19246, 19247, 19249, 19381, 19264, 19265, 19263, 19259, 19266, + 19360, 17020, 17027, 17026, 17021, 17025, 17024, 17022, 17023, 17247, + 17255, 17254, 17248, 17252, 17251, 17249, 17253, 17013, 17019, 17017, + 17014, 17016, 17015, 17018, 17004, 17064, 17072, 17071, 17065, 17069, + 17068, 17066, 17062, 17176, 17183, 17181, 17177, 17179, 17178, 17182, + 17180, 17151, 17160, 17159, 17152, 17156, 17155, 17153, 17157, 17191, + 17197, 17196, 17192, 17166, 17161, 17193, 17195, 17167, 17175, 17174, + 17168, 17172, 17171, 17169, 17173, 17143, 17150, 17149, 17144, 17148, + 17147, 17145, 17146, 17131, 40951, 17135, 17132, 17134, 17133, 40951, + 40951, 17124, 17130, 17128, 17125, 17127, 17126, 17129, 40951, 17119, + 40951, 17123, 17120, 17122, 17121, 40951, 40951, 16854, 16861, 16860, + 16855, 16859, 16858, 16856, 16845, 17316, 17323, 17321, 17317, 17319, + 17318, 17322, 17320, 17229, 17237, 17236, 17230, 17234, 17233, 17231, + 17235, 16892, 16900, 16899, 16893, 16897, 16896, 16894, 16898, 17284, + 17291, 17290, 17285, 17289, 17288, 17286, 17287, 17272, 40951, 17276, + 17273, 17275, 17274, 40951, 40951, 17082, 17090, 17089, 17083, 17087, + 17086, 17084, 17088, 17073, 17081, 17080, 17074, 17078, 17077, 17075, + 17079, 16974, 16982, 16981, 16975, 16979, 16978, 16976, 16980, 17052, + 17059, 17058, 17053, 17057, 17056, 17054, 17055, 17040, 40951, 17044, + 17041, 17043, 17042, 40951, 40951, 17033, 17039, 17037, 17034, 17036, + 17035, 17038, 40951, 17028, 40951, 17032, 17029, 17031, 17030, 40951, + 40951, 17256, 17263, 17262, 17257, 17261, 17260, 17258, 17259, 17092, + 17098, 17096, 17093, 17095, 17094, 17097, 40951, 17307, 17315, 17314, + 17308, 17312, 17311, 17309, 17313, 17292, 17299, 17297, 17293, 17295, + 17294, 17298, 17296, 17264, 17271, 17270, 17265, 17269, 17268, 17266, + 17267, 16922, 16930, 16929, 16923, 16927, 16926, 16924, 16928, 16907, + 16915, 16914, 16908, 16912, 16911, 16909, 16913, 17238, 17246, 17245, + 17239, 17243, 17242, 17240, 17244, 16995, 16943, 17001, 16996, 17000, + 16999, 16997, 16998, 16983, 40951, 16987, 16984, 16986, 16985, 40951, + 40951, 16967, 16973, 16971, 16968, 16970, 16969, 16972, 16963, 17198, + 17206, 17205, 17199, 17203, 17202, 17200, 17204, 16883, 16891, 16890, + 16884, 16888, 16887, 16885, 16889, 17091, 17106, 17105, 17099, 17103, + 17102, 17100, 17104, 17221, 17228, 17226, 17222, 17224, 17223, 17227, + 17225, 17213, 17220, 17219, 17214, 17218, 17217, 17215, 17216, 16935, + 16942, 16940, 16936, 16938, 16937, 16941, 16933, 17111, 17118, 17117, + 17112, 17116, 17115, 17113, 17109, 17158, 17070, 16939, 40951, 40951, + 16823, 16825, 16824, 16842, 17345, 17343, 16826, 16841, 16827, 16840, + 17344, 16839, 17341, 17339, 17338, 17335, 17334, 17337, 17336, 17342, + 17340, 16828, 16831, 16830, 16835, 16834, 16838, 16837, 16833, 16836, + 16832, 16829, 40951, 40951, 40951, 17164, 17063, 17061, 17060, 17162, + 16846, 16844, 16843, 17163, 16934, 16932, 16931, 17165, 17110, 17108, + 17107, 17333, 17324, 17330, 17331, 17326, 17328, 17332, 17327, 17325, + 17329, 40951, 40951, 40951, 40951, 40951, 40951, 6382, 6383, 6384, 6385, + 6386, 6387, 6345, 6381, 6346, 6347, 6348, 6349, 6350, 6310, 6311, 6312, + 6313, 6314, 6315, 6351, 6352, 6353, 6354, 6355, 6356, 6357, 6358, 6359, + 6360, 6361, 6316, 6309, 6317, 6318, 6319, 6320, 6321, 6322, 6363, 6364, + 6365, 6366, 6367, 6368, 6324, 6323, 6325, 6326, 6327, 6328, 6329, 6303, + 6342, 6304, 6343, 6305, 6344, 6306, 6307, 6308, 6302, 6330, 6331, 6332, + 6333, 6334, 6335, 6336, 6337, 6338, 6339, 6340, 6341, 6369, 6370, 6371, + 6372, 6373, 6374, 6375, 6376, 6377, 6378, 6379, 6380, 6362, 40951, 40951, + 6462, 6463, 6464, 6465, 6466, 6448, 40951, 40951, 5524, 5496, 5269, 5526, + 5527, 5676, 5690, 5973, 5480, 5340, 5267, 5268, 5850, 5940, 5960, 5938, + 5961, 5939, 5942, 5936, 5943, 5937, 5605, 5957, 5934, 5958, 5935, 5606, + 5271, 5974, 5992, 5513, 5512, 5514, 5515, 5507, 5508, 5505, 5504, 5517, + 5511, 5516, 5506, 5498, 5528, 5689, 5275, 5710, 5703, 5708, 5709, 5705, + 5706, 5964, 5338, 5339, 5701, 5702, 5700, 5879, 5698, 5877, 5699, 5878, + 5693, 5875, 5694, 5876, 5696, 5873, 5697, 5874, 5963, 5692, 5872, 5331, + 5849, 5832, 5847, 5848, 5845, 5846, 5971, 5302, 5315, 5830, 5831, 5840, + 5932, 5838, 5930, 5839, 5931, 5836, 5928, 5837, 5929, 5834, 5926, 5835, + 5927, 5611, 5792, 5827, 5828, 5829, 5826, 5547, 5541, 5545, 5546, 5543, + 5544, 5966, 5539, 5540, 5538, 5924, 5536, 5922, 5537, 5923, 5534, 5920, + 5535, 5921, 5531, 5918, 5532, 5919, 5608, 5529, 5530, 5772, 5773, 5774, + 5771, 5495, 5482, 5493, 5494, 5491, 5492, 5965, 5299, 5481, 5489, 5917, + 5487, 5915, 5488, 5916, 5485, 5913, 5486, 5914, 5483, 5911, 5484, 5912, + 5607, 5298, 5748, 5573, 5581, 5590, 5591, 5576, 5577, 5968, 5579, 5580, + 5589, 5871, 5587, 5869, 5588, 5870, 5585, 5867, 5586, 5868, 5583, 5865, + 5584, 5866, 5609, 5572, 5864, 5592, 5273, 5749, 5665, 5626, 5663, 5664, + 5653, 5654, 5969, 5597, 5625, 5662, 5890, 5656, 5888, 5657, 5889, 5610, + 5593, 5371, 5666, 5571, 5558, 5569, 5570, 5567, 5568, 5967, 5556, 5557, + 5566, 5858, 5564, 5856, 5565, 5857, 5562, 5854, 5563, 5855, 5560, 5852, + 5561, 5853, 5548, 5851, 5574, 5791, 5751, 5789, 5790, 5770, 5775, 5970, + 5731, 5750, 5784, 5910, 5782, 5908, 5783, 5909, 5780, 5906, 5781, 5907, + 5778, 5904, 5779, 5905, 5603, 5730, 5274, 5777, 5294, 5578, 5598, 5604, + 5601, 5602, 5599, 5600, 5769, 5767, 5768, 5761, 5762, 5764, 5765, 5760, + 5903, 5758, 5901, 5759, 5902, 5753, 5899, 5754, 5900, 5756, 5897, 5757, + 5898, 5752, 5991, 5977, 5989, 5990, 5979, 5980, 5972, 5975, 5976, 5988, + 5887, 5986, 5885, 5987, 5886, 5984, 5883, 5985, 5884, 5982, 5881, 5983, + 5882, 5612, 5962, 5295, 5880, 5747, 5729, 5713, 5863, 5723, 5727, 5728, + 5725, 5726, 5861, 5721, 5722, 5859, 5715, 5892, 5711, 5891, 5575, 5523, + 5502, 5503, 5518, 5520, 5521, 5500, 5501, 5522, 5933, 5499, 5811, 5596, + 5809, 5594, 5810, 5595, 5807, 5808, 5805, 5806, 5803, 5925, 5793, 5824, + 5825, 5821, 5819, 5818, 5842, 5843, 5844, 5841, 5651, 5649, 5650, 5647, + 5648, 5645, 5646, 5644, 5652, 5525, 5670, 5674, 5675, 5672, 5673, 5668, + 5669, 5667, 5816, 5817, 5812, 5815, 5894, 5895, 5896, 5893, 5631, 5635, + 5636, 5633, 5634, 5629, 5630, 5628, 5637, 5745, 5746, 5741, 5744, 5953, + 5954, 5955, 5952, 5944, 5554, 5555, 5552, 5553, 5550, 5551, 5549, 5801, + 5799, 5800, 5797, 5798, 5795, 5796, 5794, 5272, 5291, 5292, 5293, 5290, + 5279, 5280, 5281, 5278, 5287, 5288, 5289, 5286, 5283, 5284, 5285, 5282, + 5736, 5737, 5733, 5735, 5321, 5320, 5316, 5317, 5319, 5318, 5467, 5466, + 5462, 5463, 5465, 5464, 5473, 5472, 5468, 5469, 5471, 5470, 5337, 5336, + 5332, 5333, 5335, 5334, 5431, 5430, 5426, 5427, 5429, 5428, 5425, 5424, + 5420, 5421, 5423, 5422, 5394, 5393, 5389, 5390, 5392, 5391, 5388, 5330, + 5329, 5326, 5327, 5328, 5324, 5367, 5366, 5362, 5363, 5365, 5364, 5361, + 5360, 5356, 5357, 5359, 5358, 5355, 5374, 5373, 5368, 5369, 5372, 5370, + 5461, 5460, 5456, 5457, 5459, 5458, 5479, 5478, 5474, 5475, 5477, 5476, + 5354, 5738, 5353, 5348, 5349, 5352, 5740, 5351, 5347, 5346, 5342, 5343, + 5345, 5344, 5449, 5448, 5444, 5445, 5447, 5446, 5308, 5307, 5303, 5304, + 5306, 5305, 5443, 5442, 5438, 5439, 5441, 5440, 5407, 5406, 5402, 5403, + 5405, 5404, 5413, 5412, 5408, 5409, 5411, 5410, 5401, 5400, 5396, 5397, + 5399, 5398, 5395, 5341, 5314, 5313, 5309, 5310, 5312, 5311, 5387, 5386, + 5382, 5383, 5385, 5384, 5381, 5380, 5376, 5377, 5379, 5378, 5375, 5437, + 5436, 5432, 5433, 5435, 5434, 5455, 5454, 5450, 5451, 5453, 5452, 5419, + 5418, 5414, 5415, 5417, 5416, 5490, 5519, 5671, 5632, 5642, 5643, 5640, + 5641, 5638, 5639, 5951, 5949, 5950, 5947, 5948, 5945, 5946, 5956, 5277, + 29977, 29952, 29963, 29959, 29969, 29966, 29972, 29975, 29976, 29955, + 29954, 29974, 29960, 29965, 29970, 29964, 29951, 29967, 29973, 29957, + 29961, 29956, 29968, 29971, 29962, 29958, 29953, 29949, 29950, 40951, + 40951, 40951, 32296, 32352, 32346, 32350, 32349, 32344, 32342, 32289, + 32274, 32320, 32273, 32272, 32317, 32332, 32319, 32323, 32324, 32326, + 32312, 32278, 32311, 32297, 32290, 32298, 32300, 32345, 32302, 32301, + 32315, 32329, 32341, 32331, 32283, 32305, 32286, 32309, 32299, 32314, + 32335, 32306, 32348, 32275, 32338, 32337, 32333, 32276, 32354, 32343, + 32334, 32281, 32340, 32328, 32284, 32322, 32287, 32347, 32316, 32330, + 32313, 32282, 32304, 32303, 32285, 32321, 32288, 32308, 32280, 32279, + 32277, 32339, 32318, 32336, 32307, 32351, 32353, 32355, 32356, 32271, + 32357, 32358, 32270, 32310, 32327, 32325, 32294, 32293, 32295, 32292, + 32291, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 35025, 35042, + 35043, 35033, 35031, 35027, 35039, 35030, 35028, 35036, 35029, 35035, + 35041, 35037, 35034, 35040, 35038, 35032, 35046, 35047, 35045, 35044, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 35026, + 19648, 19649, 19650, 19639, 19637, 19633, 19645, 19636, 19634, 19642, + 19635, 19641, 19647, 19643, 19640, 19646, 19644, 19638, 19652, 19653, + 19651, 31427, 31428, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 4990, 4991, 4992, 4981, 4979, 4975, 4987, 4978, 4976, 4984, + 4977, 4983, 4989, 4985, 4982, 4988, 4986, 4980, 4993, 4994, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 35061, 35062, 35063, 35053, 35052, 35048, 35058, 35051, 35049, 35056, + 35050, 35055, 35060, 40951, 35054, 35059, 35057, 40951, 35064, 35065, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 22231, 22229, 22232, 22230, 22233, 22227, 22225, 22228, + 22226, 22235, 22249, 22245, 22250, 22246, 22234, 22247, 22243, 22248, + 22244, 22236, 22257, 22237, 22239, 22238, 22253, 22256, 22254, 22252, + 22255, 22241, 22240, 22242, 22258, 22251, 22259, 22206, 22204, 22214, + 22215, 22210, 22213, 22211, 22212, 22223, 22224, 22221, 22222, 22216, + 22205, 22209, 22208, 22207, 22326, 22325, 22327, 22332, 22334, 22338, + 22340, 22342, 22344, 22343, 22335, 22339, 22333, 22345, 22329, 22330, + 22337, 22331, 22280, 22274, 22279, 22281, 22276, 22265, 22273, 22275, + 22270, 22262, 22263, 22282, 22269, 22264, 22271, 22266, 22268, 22277, + 22267, 22278, 22272, 22203, 22260, 22261, 40951, 40951, 22352, 22354, + 22351, 22350, 22347, 22346, 22349, 22348, 22355, 22353, 40951, 40951, + 40951, 40951, 40951, 40951, 22304, 22303, 22300, 22302, 22301, 22295, + 22298, 22299, 22297, 22296, 40951, 40951, 40951, 40951, 40951, 40951, + 28162, 28174, 28170, 28019, 28169, 28020, 28167, 28158, 28171, 28172, + 28173, 28018, 28017, 28016, 28168, 28015, 28011, 28013, 28010, 28009, + 28006, 28005, 28008, 28007, 28014, 28012, 40951, 40951, 40951, 40951, + 40951, 40951, 28023, 28137, 28154, 28139, 28141, 28140, 28142, 28138, + 28148, 28051, 28143, 28149, 28150, 28146, 28055, 28136, 28098, 28097, + 28128, 28144, 28052, 28147, 28153, 28151, 28152, 28145, 28134, 28133, + 28127, 28131, 28132, 28130, 28135, 28129, 28054, 28107, 28125, 28126, + 28114, 28116, 28115, 28117, 28101, 28118, 28121, 28122, 28108, 28120, + 28111, 28103, 28112, 28105, 28110, 28124, 28123, 28119, 28109, 28113, + 28104, 28106, 28102, 28096, 28079, 28080, 28090, 28089, 28086, 28094, + 28075, 28077, 28095, 28084, 28082, 28091, 28093, 28092, 28076, 28078, + 28081, 28088, 28085, 28083, 28087, 28074, 28072, 28073, 28071, 28070, + 28053, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 28025, 28027, + 28029, 28036, 28035, 28043, 28039, 28024, 28034, 28050, 28037, 28049, + 28041, 28040, 28031, 28038, 28042, 28028, 28045, 28044, 28048, 28046, + 28047, 28026, 28100, 28099, 28063, 28068, 28059, 28064, 28060, 28056, + 28061, 28057, 28069, 28058, 28066, 28067, 28033, 28032, 28062, 28030, + 28065, 40951, 40951, 40951, 40951, 40951, 5691, 5276, 5270, 5959, 5707, + 5704, 5695, 5833, 5542, 5533, 5582, 5655, 5627, 5559, 5776, 5732, 5763, + 5766, 5755, 5981, 5978, 5724, 5660, 5680, 5661, 5681, 5658, 5678, 5659, + 5679, 5720, 5718, 5719, 5716, 5717, 5714, 5687, 5688, 5685, 5684, 5686, + 5677, 5682, 5683, 5497, 5941, 5510, 5509, 5712, 5862, 5860, 5804, 5802, + 5823, 5822, 5820, 5814, 5813, 5743, 5742, 5734, 5323, 5300, 5325, 5322, + 5739, 5350, 5296, 5297, 5301, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 24478, 24445, 24444, 24425, 24424, 24431, + 24439, 24438, 24443, 24442, 24430, 24428, 24426, 24441, 24440, 24432, + 24447, 24446, 24437, 24436, 24450, 24429, 24451, 24449, 24452, 24433, + 24434, 24435, 24448, 24423, 24427, 40951, 24469, 24476, 24477, 24475, + 24470, 24473, 24471, 24474, 24472, 24468, 24466, 24467, 40951, 40951, + 40951, 40951, 24460, 24457, 24459, 24465, 24458, 24463, 24462, 24464, + 24461, 24454, 24453, 24455, 40951, 40951, 40951, 40951, 24456, 40951, + 40951, 40951, 24479, 24480, 24487, 24489, 24486, 24485, 24482, 24481, + 24484, 24483, 24490, 24488, 35086, 35098, 35081, 35078, 35096, 35099, + 35080, 35079, 35093, 35088, 35087, 35094, 35091, 35097, 35092, 35095, + 35085, 35077, 35082, 35066, 35100, 35070, 35071, 35089, 35084, 35083, + 35090, 35069, 35067, 35068, 40951, 40951, 35072, 35073, 35074, 35075, + 35076, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 29120, 29142, 29102, 29104, 29107, 29124, 29126, 29129, + 29110, 29106, 29122, 29132, 29128, 29144, 29111, 29109, 29108, 29133, + 29131, 29130, 29113, 29112, 29119, 29135, 29134, 29141, 29116, 29121, + 29118, 29138, 29143, 29140, 29117, 29115, 29114, 29139, 29137, 29136, + 29101, 29103, 29123, 29125, 29105, 29127, 40951, 40951, 40951, 40951, + 29165, 29150, 29154, 29160, 29163, 29166, 29152, 29156, 29157, 29161, + 29153, 29151, 29164, 29159, 29158, 29162, 29155, 29100, 29095, 29094, + 29099, 29098, 29097, 29096, 29147, 29148, 40951, 40951, 40951, 40951, + 40951, 40951, 29173, 29175, 29172, 29171, 29168, 29167, 29170, 29169, + 29176, 29174, 29149, 40951, 40951, 40951, 29145, 29146, 22305, 22324, + 22317, 22320, 22322, 22315, 22313, 22311, 22307, 22309, 22294, 22292, + 22284, 22288, 22290, 22286, 22318, 22323, 22316, 22319, 22321, 22314, + 22312, 22310, 22306, 22308, 22293, 22291, 22283, 22287, 22289, 22285, + 4959, 4956, 4948, 4947, 4961, 4953, 4946, 4945, 4964, 4955, 4952, 4951, + 4954, 4958, 4950, 4949, 4966, 4962, 4960, 4965, 4963, 4967, 4957, 4970, + 4972, 4969, 4971, 4968, 40951, 40951, 4974, 4973, 35134, 35132, 35133, + 35150, 35149, 35148, 35174, 35140, 35139, 35153, 35160, 35152, 35175, + 35168, 35135, 35180, 35151, 35167, 35144, 35143, 35157, 35156, 35176, + 35179, 35142, 35141, 35145, 35155, 35158, 35154, 35181, 35161, 35147, + 35166, 35169, 35162, 35164, 35182, 35136, 35137, 35138, 35146, 35165, + 35183, 35159, 35172, 35173, 35170, 35171, 35178, 35177, 35163, 35131, + 35106, 35105, 35103, 35194, 35101, 35102, 35104, 35107, 35108, 35109, + 40951, 35202, 35209, 35213, 35210, 35219, 35225, 35226, 35224, 35223, + 35221, 35222, 35214, 35215, 35218, 35227, 35211, 35217, 35212, 35220, + 35216, 35193, 35206, 35207, 35186, 35187, 35188, 35197, 35196, 35189, + 40951, 40951, 35110, 35117, 35119, 35116, 35115, 35112, 35111, 35114, + 35113, 35120, 35118, 40951, 40951, 40951, 40951, 40951, 40951, 35127, + 35129, 35126, 35125, 35122, 35121, 35124, 35123, 35130, 35128, 40951, + 40951, 40951, 40951, 40951, 40951, 35203, 35204, 35201, 35192, 35185, + 35205, 35198, 35195, 35190, 35191, 35199, 35200, 35184, 35208, 40951, + 40951, 8201, 8169, 8285, 8203, 8435, 8448, 8447, 8387, 8184, 8364, 8423, + 8393, 8188, 8392, 8391, 8333, 8327, 8350, 8409, 8351, 8410, 8421, 8380, + 8284, 8396, 8187, 8186, 8433, 8311, 8312, 8313, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 2717, 2716, 2718, 2715, 2719, + 2626, 2627, 2643, 2644, 2647, 2648, 2665, 2666, 2656, 2657, 2640, 2630, + 2645, 2646, 2651, 2653, 2641, 2642, 2660, 2633, 2634, 2649, 2650, 2661, + 2672, 2671, 2637, 2636, 2659, 2670, 2673, 2635, 2638, 2658, 2662, 2663, + 2631, 2632, 2678, 2680, 2664, 2655, 2679, 2668, 2669, 2667, 2677, 2720, + 2731, 2734, 2735, 2725, 2726, 2723, 2724, 2721, 2722, 2727, 2728, 2730, + 2729, 2732, 2733, 2736, 2652, 2654, 2674, 2639, 2675, 2676, 2628, 2629, + 40951, 2625, 2624, 2744, 2746, 2743, 2742, 2739, 2738, 2741, 2740, 2747, + 2745, 2712, 2709, 2737, 2622, 2623, 2621, 2711, 2698, 2696, 2699, 2690, + 2691, 2697, 2693, 2695, 2694, 2692, 2688, 2687, 2683, 2681, 2685, 2684, + 2682, 2686, 2689, 2708, 2707, 2706, 2705, 2700, 2702, 2703, 2704, 2701, + 2713, 2710, 2714, 34633, 34631, 34632, 34584, 34619, 34621, 34586, 34620, + 34596, 34597, 34604, 34612, 34607, 34598, 34605, 34609, 34618, 34599, + 34613, 34606, 34600, 34611, 34589, 34614, 34602, 34610, 34617, 34593, + 34591, 34615, 34595, 34616, 34608, 34579, 34580, 34581, 34639, 34638, + 34640, 34637, 34635, 34636, 34630, 34634, 34582, 34583, 34603, 34594, + 34647, 34649, 34646, 34645, 34642, 34641, 34644, 34643, 34650, 34648, + 34578, 34592, 34590, 34601, 34587, 34588, 3508, 3494, 3502, 3486, 3474, + 3498, 3497, 3483, 3489, 3482, 3475, 3506, 3492, 3484, 3501, 3485, 3503, + 3500, 3505, 3490, 3473, 3488, 3495, 3478, 3496, 3491, 3476, 3507, 3493, + 3480, 3504, 3487, 3481, 3499, 3479, 3477, 3509, 3510, 3517, 3523, 3520, + 3524, 3525, 3521, 3526, 3522, 3518, 3519, 3471, 3472, 3511, 3512, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 3516, 3514, 3515, 3513, + 24308, 24307, 24306, 24320, 24319, 24325, 24335, 24334, 24338, 24326, + 24333, 24332, 24314, 24327, 24311, 24310, 24309, 24318, 24317, 24316, + 24315, 24324, 24323, 24329, 24328, 24313, 24343, 24340, 24339, 24322, + 24321, 24341, 24337, 24336, 24342, 24344, 24353, 24352, 24358, 24360, + 24356, 24357, 24354, 24355, 24359, 24297, 24302, 24301, 24299, 24303, + 24304, 24305, 24300, 24298, 24351, 24350, 40951, 40951, 40951, 24345, + 24348, 24349, 24347, 24346, 24367, 24369, 24366, 24365, 24362, 24361, + 24364, 24363, 24370, 24368, 40951, 40951, 40951, 24331, 24330, 24312, + 30023, 30025, 30022, 30021, 30018, 30017, 30020, 30019, 30026, 30024, + 29996, 29987, 29985, 29984, 29986, 29997, 29981, 29980, 29982, 29983, + 29999, 29995, 29993, 29992, 29994, 30001, 30007, 30008, 30006, 30009, + 29998, 29991, 29988, 29990, 29989, 30000, 30002, 30003, 30005, 30004, + 30011, 30012, 30010, 30016, 30015, 30027, 30014, 30013, 10430, 10407, + 10413, 10470, 10451, 10459, 10449, 10450, 10469, 10135, 10461, 40951, + 40951, 40951, 40951, 40951, 17856, 17887, 17864, 17893, 17862, 17892, + 17885, 17882, 17894, 17874, 17876, 17888, 17890, 17895, 17878, 17884, + 17886, 17880, 17883, 17896, 17877, 17873, 17863, 17891, 17879, 17857, + 17860, 17872, 17859, 17858, 17889, 17871, 17867, 17870, 17868, 17898, + 17865, 17869, 17899, 17897, 17861, 17881, 17855, 40951, 40951, 17854, + 17866, 17875, 34629, 34627, 34628, 34626, 34625, 34624, 34623, 34622, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 38803, 38816, + 38805, 38783, 38797, 38811, 38812, 38813, 38798, 38814, 38801, 38809, + 38804, 38802, 38810, 38808, 38807, 38815, 38796, 38794, 38785, 38792, + 38784, 38795, 38793, 38774, 38775, 38777, 38778, 38790, 38788, 38789, + 38787, 38776, 38780, 38786, 38799, 38782, 38791, 38779, 38806, 38800, + 38781, 40951, 40951, 40951, 40951, 40951, 23270, 23271, 23878, 23267, + 23272, 23273, 23244, 23243, 23863, 23856, 23276, 23277, 23250, 23278, + 23256, 23251, 23252, 23813, 23814, 23815, 23858, 23254, 23850, 23370, + 23280, 23257, 23265, 23260, 23283, 23818, 23817, 23816, 23284, 23285, + 23287, 23245, 23298, 23232, 18400, 18403, 18399, 18402, 18398, 10300, + 27701, 27702, 27692, 27693, 27704, 27705, 27695, 27707, 27697, 27708, + 27709, 27710, 27711, 27712, 27713, 27696, 27699, 27700, 27714, 27694, + 27716, 27717, 27719, 27850, 27957, 27851, 27959, 27854, 27879, 27893, + 27947, 27932, 27962, 27900, 27970, 27981, 27910, 27897, 27930, 27933, + 27954, 27856, 27934, 27949, 27974, 27948, 27960, 27978, 27852, 27858, + 27901, 27884, 27902, 27878, 24019, 24027, 24029, 24030, 18609, 18605, + 18608, 18607, 18606, 23939, 23356, 23405, 23491, 23634, 23654, 23731, + 23758, 23751, 23797, 23833, 24001, 23885, 27768, 23562, 23843, 23300, + 23569, 23727, 23301, 23938, 23357, 23409, 23492, 23505, 23588, 23615, + 23635, 23660, 23732, 23761, 23798, 23478, 23946, 23973, 24002, 23319, + 23345, 23408, 23468, 23720, 23768, 23806, 23559, 23716, 23480, 23925, + 23486, 27958, 27859, 27877, 27895, 27941, 27898, 27885, 27946, 27969, + 27912, 27913, 27860, 27862, 27915, 27919, 27921, 27865, 27911, 27961, + 27929, 27928, 27871, 27855, 27935, 27945, 27894, 27950, 27976, 27977, + 27873, 27980, 27971, 27890, 27892, 27891, 27896, 27973, 8177, 8176, 8425, + 8424, 8377, 8268, 8376, 8061, 8267, 8060, 8325, 8073, 8378, 8200, 8389, + 8416, 8286, 8440, 8441, 8308, 8299, 8300, 8301, 8303, 8310, 8306, 8335, + 8291, 8337, 8314, 8292, 8293, 8339, 8294, 8295, 8324, 8328, 8316, 8343, + 8298, 8330, 8331, 8329, 8307, 8315, 8319, 8340, 8305, 8322, 8332, 8297, + 8318, 8321, 8439, 8290, 8289, 8172, 8445, 8175, 8167, 8181, 8070, 8345, + 8400, 22756, 23317, 22792, 23359, 22791, 23358, 22790, 23355, 22799, + 23374, 22824, 23411, 22823, 23410, 22822, 23406, 22818, 23400, 22819, + 23401, 22850, 23454, 22851, 23455, 22839, 23442, 22848, 23452, 22830, + 23433, 22875, 23494, 22883, 23504, 22902, 23524, 22901, 23523, 22899, + 23521, 22896, 23518, 22895, 23517, 22919, 23551, 22913, 23539, 22950, + 23586, 22947, 23583, 22951, 23587, 22958, 23600, 22959, 23601, 22969, + 23610, 22965, 23597, 22975, 23632, 22977, 23637, 22976, 23636, 22995, + 23659, 22994, 23658, 22989, 23651, 22985, 23646, 23026, 23695, 23025, + 23694, 23003, 23687, 23004, 23688, 23054, 23730, 23056, 23734, 23065, + 23748, 23063, 23746, 23064, 23747, 23070, 23753, 23091, 23791, 23089, + 23789, 23088, 23782, 23083, 23784, 23090, 23790, 23112, 23831, 23111, + 23830, 23113, 23834, 23107, 23824, 23143, 23906, 23164, 23929, 23136, + 23899, 23163, 23928, 23155, 23918, 23176, 23949, 23175, 23945, 23189, + 23962, 23190, 23963, 23186, 23959, 23188, 23961, 23187, 23960, 23195, + 23968, 23194, 23967, 23200, 23978, 23211, 23992, 23215, 23996, 23217, + 23998, 23525, 23829, 23964, 23988, 23318, 23625, 23624, 23626, 23101, + 23415, 22750, 23311, 22766, 23329, 22762, 23325, 22761, 23324, 22760, + 23323, 22764, 23327, 22763, 23326, 22745, 23306, 22744, 23305, 22743, + 23304, 22747, 23308, 22746, 23307, 22844, 23447, 22855, 23460, 22847, + 23451, 22835, 23438, 22834, 23437, 22833, 23436, 22838, 23441, 22837, + 23440, 22920, 23552, 22910, 23543, 23017, 23680, 23037, 23706, 23009, + 23672, 23008, 23671, 23007, 23670, 23011, 23674, 23010, 23673, 23034, + 23703, 23033, 23702, 23032, 23701, 23036, 23705, 23035, 23704, 23146, + 23909, 23153, 23916, 23150, 23913, 23149, 23912, 23148, 23911, 23152, + 23915, 23151, 23914, 23201, 23979, 23199, 23977, 23203, 23981, 23207, + 23987, 22980, 23640, 22981, 23641, 23204, 23982, 18447, 18439, 18452, + 18444, 18448, 18440, 18450, 18442, 18213, 18205, 18216, 18208, 18214, + 18206, 18218, 18210, 18470, 18467, 18472, 18469, 18471, 18468, 40951, + 40951, 18253, 18250, 18255, 18252, 18254, 18251, 40951, 40951, 18485, + 18477, 18490, 18482, 18486, 18478, 18488, 18480, 18237, 18229, 18240, + 18232, 18238, 18230, 18242, 18234, 18511, 18502, 18514, 18505, 18513, + 18504, 18512, 18503, 18265, 18260, 18268, 18263, 18267, 18262, 18266, + 18261, 18572, 18569, 18574, 18571, 18573, 18570, 40951, 40951, 18299, + 18296, 18301, 18298, 18300, 18297, 40951, 40951, 18531, 18522, 18534, + 18525, 18533, 18524, 18532, 18523, 40951, 18311, 40951, 18314, 40951, + 18313, 40951, 18312, 18552, 18544, 18557, 18549, 18553, 18545, 18555, + 18547, 18283, 18275, 18286, 18278, 18284, 18276, 18288, 18280, 18437, + 18457, 18474, 18473, 18497, 18495, 18517, 18518, 18576, 18575, 18537, + 18538, 18564, 18562, 40951, 40951, 18454, 18446, 18453, 18445, 18449, + 18441, 18451, 18443, 18220, 18212, 18217, 18209, 18215, 18207, 18219, + 18211, 18492, 18484, 18491, 18483, 18487, 18479, 18489, 18481, 18244, + 18236, 18241, 18233, 18239, 18231, 18243, 18235, 18559, 18551, 18558, + 18550, 18554, 18546, 18556, 18548, 18290, 18282, 18287, 18279, 18285, + 18277, 18289, 18281, 18436, 18461, 18438, 18459, 18458, 40951, 18455, + 18456, 18222, 18226, 18223, 18224, 18221, 18396, 18424, 18425, 18429, + 18345, 18498, 18499, 18496, 40951, 18493, 18494, 18257, 18256, 18247, + 18246, 18245, 18428, 18427, 18426, 18516, 18520, 18509, 18508, 40951, + 40951, 18515, 18507, 18269, 18273, 18270, 18271, 40951, 18353, 18352, + 18351, 18536, 18540, 18529, 18528, 18584, 18583, 18535, 18527, 18316, + 18320, 18317, 18318, 18306, 18347, 18346, 18623, 40951, 40951, 18565, + 18566, 18563, 40951, 18560, 18561, 18303, 18302, 18293, 18292, 18291, + 18421, 18350, 40951, 16764, 16761, 16766, 16763, 37019, 17510, 33761, + 17439, 31718, 36992, 18978, 40757, 40756, 40758, 24189, 32047, 20456, + 29361, 17438, 16765, 16762, 20402, 11250, 11224, 24112, 31977, 33636, + 33634, 24065, 31942, 11223, 11218, 10529, 11217, 4995, 37548, 30526, + 37694, 20425, 20459, 24496, 31062, 24188, 32046, 31593, 24187, 32045, + 28962, 31296, 31297, 31679, 11232, 37562, 31885, 31879, 31893, 6007, + 33635, 33637, 31902, 11254, 20799, 30878, 37745, 6293, 6008, 2529, 20458, + 17514, 24120, 31988, 11255, 31742, 17353, 37393, 31884, 3854, 3896, + 25158, 31890, 8045, 37706, 8451, 34742, 20817, 17476, 36999, 31738, + 17503, 17463, 37695, 17504, 11196, 37573, 38824, 26974, 39366, 17636, + 20819, 20821, 20820, 40951, 24186, 32044, 17456, 31592, 20713, 7, 20712, + 6, 28957, 29359, 34713, 34701, 40951, 40951, 34708, 34707, 34710, 34709, + 34700, 34714, 34704, 34706, 34699, 34703, 34705, 34702, 34543, 34545, + 34542, 34541, 34538, 34537, 34540, 34539, 34533, 34544, 34534, 34536, + 34532, 34531, 34535, 40951, 24016, 24017, 24025, 24031, 24015, 24018, + 24021, 24022, 24023, 24024, 24026, 24014, 24028, 40951, 40951, 40951, + 17350, 8058, 8697, 17520, 25104, 27586, 28890, 31320, 32367, 39370, + 29093, 11190, 17351, 22555, 37582, 11372, 17901, 31321, 18673, 2540, + 20465, 6126, 25105, 34096, 36765, 20703, 37664, 29411, 25663, 32236, + 22737, 3762, 34076, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 8354, 8408, 8368, + 8420, 8066, 8082, 8347, 8405, 8413, 8081, 8065, 8427, 8213, 8204, 8207, + 8210, 8205, 8357, 8206, 8208, 8209, 8397, 8194, 8067, 8434, 8446, 8359, + 8365, 8412, 8358, 8346, 8404, 8069, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 0, + 100, 11266, 10553, 6130, 6009, 5260, 17352, 32555, 10555, 32552, 32544, + 3944, 11267, 31482, 31483, 32545, 3945, 32546, 32553, 25335, 11268, + 29454, 33992, 32548, 11264, 11269, 32549, 3946, 11270, 31655, 31863, + 32658, 36660, 37535, 38817, 11271, 30872, 30880, 20816, 3947, 37687, + 21610, 961, 32541, 3943, 16820, 32551, 32542, 32543, 37671, 32547, 32554, + 346, 3653, 17904, 10542, 20710, 32224, 17418, 11278, 11277, 11263, 11265, + 11279, 37680, 37681, 31889, 37682, 11276, 11272, 11273, 11274, 11275, + 31683, 37666, 31298, 2600, 37684, 34821, 38966, 38964, 38968, 38969, + 38974, 38962, 38973, 38972, 38960, 38967, 38959, 38961, 38970, 38958, + 38975, 17515, 32192, 32203, 32204, 32191, 32188, 32197, 32199, 32207, + 32208, 32200, 32206, 32202, 32185, 32193, 32189, 32195, 33823, 33827, + 33828, 33822, 33819, 33831, 33830, 33818, 33832, 33829, 33817, 33826, + 33821, 33824, 33820, 33825, 32196, 32190, 32201, 32205, 23774, 32198, + 32187, 32186, 32194, 38976, 37675, 37674, 40951, 40951, 40951, 40951, + 24190, 37896, 32049, 11308, 24094, 37767, 29392, 29365, 33964, 33979, + 24210, 32073, 24274, 32150, 24271, 37946, 32149, 11344, 24213, 32076, + 24221, 37909, 32056, 11322, 37768, 24219, 32082, 24204, 32068, 24102, + 24097, 11348, 37906, 37907, 11317, 11318, 32064, 11309, 972, 8029, 29394, + 24201, 977, 8031, 24240, 24234, 37923, 37920, 32110, 32104, 11359, 11356, + 32084, 37898, 24224, 24286, 37949, 32116, 11367, 24241, 32099, 24277, + 24101, 32092, 24275, 37913, 32090, 11349, 24099, 37770, 29405, 29378, + 33976, 33989, 24261, 32140, 24290, 32120, 37899, 11310, 24281, 37914, + 32096, 11350, 24200, 32063, 24272, 37950, 32151, 11346, 37955, 37951, + 37953, 37952, 37957, 37958, 32152, 29393, 33967, 37772, 31929, 11320, + 37007, 24220, 32083, 24096, 24206, 32066, 24095, 24285, 32115, 24104, + 17498, 8456, 31208, 36987, 36986, 16754, 20631, 28846, 16703, 29414, + 33803, 8464, 11016, 33796, 16768, 28805, 28793, 28797, 27590, 27597, + 11191, 10998, 32663, 2528, 32161, 4996, 34323, 8703, 17509, 31682, 20704, + 31919, 957, 26848, 34099, 11002, 11017, 31067, 29419, 25119, 25126, + 20792, 37747, 20777, 11221, 37572, 8470, 34735, 38954, 8032, 8023, 974, + 36988, 3654, 31773, 31681, 11192, 17355, 17726, 20446, 37297, 31891, + 20813, 33756, 39374, 29424, 27596, 2533, 29415, 1054, 1057, 29049, 362, + 29416, 364, 37563, 359, 16806, 17724, 11008, 1058, 17725, 1055, 20594, + 8057, 16804, 32159, 32160, 8649, 16821, 16807, 34488, 10558, 16790, + 26859, 31744, 29426, 20470, 29427, 34524, 24390, 18137, 24392, 18140, + 24381, 18130, 28542, 28541, 3652, 29425, 29429, 29428, 29054, 29051, + 24389, 18136, 29053, 29050, 24391, 18138, 29055, 29052, 31656, 34561, + 31665, 34570, 31664, 34569, 11019, 11022, 34549, 34721, 29412, 29413, + 34555, 34727, 29047, 29048, 34554, 34726, 28295, 28296, 28297, 34196, + 34285, 34198, 34287, 34131, 34138, 6804, 6750, 6809, 6542, 6555, 6805, + 6531, 6814, 6556, 34456, 34449, 34470, 34404, 32006, 24138, 11285, 37776, + 2534, 27605, 37579, 17499, 37566, 11248, 11021, 29423, 11024, 29046, + 31666, 34571, 29409, 8465, 29410, 8466, 30556, 20593, 28298, 20217, + 20800, 39395, 28891, 29364, 31925, 32002, 28802, 28803, 28804, 28798, + 10837, 11193, 34490, 11000, 4372, 24152, 31964, 24110, 31975, 31892, + 9946, 9945, 11242, 11241, 11220, 11245, 31476, 16791, 24395, 18146, + 38867, 38866, 24379, 18141, 16789, 16788, 16786, 16787, 11020, 11023, + 29420, 29421, 34197, 34286, 24380, 18129, 31663, 34568, 29417, 11014, + 29418, 11015, 38823, 27582, 37775, 11288, 16704, 16706, 33804, 16709, + 16708, 33805, 16707, 16705, 8467, 8468, 33797, 8469, 33798, 40669, 10838, + 16701, 20440, 37759, 11289, 31680, 31315, 39140, 24061, 31937, 24149, + 31946, 4355, 4352, 37483, 37479, 31882, 34229, 2524, 32565, 32561, 36658, + 31601, 38878, 31479, 37678, 39131, 20438, 37478, 37482, 4351, 4354, + 37472, 4349, 17524, 33863, 37760, 30548, 16817, 39379, 21612, 24159, + 32027, 16816, 3650, 10524, 360, 34831, 37499, 11009, 8475, 33788, 8651, + 8652, 1002, 1040, 1026, 1014, 1015, 1031, 1012, 982, 987, 1037, 1042, + 1028, 1027, 1022, 1029, 1010, 1034, 1023, 1030, 985, 994, 993, 1016, + 1019, 995, 1048, 1021, 1045, 991, 1020, 1018, 1046, 998, 1017, 1033, 992, + 999, 1008, 986, 1047, 1032, 983, 1013, 1044, 989, 1038, 1007, 984, 996, + 1009, 1050, 1049, 988, 990, 1051, 1039, 1036, 1025, 1024, 997, 1043, + 1000, 1035, 1005, 1004, 1041, 1001, 1006, 1003, 29430, 31924, 32844, + 3546, 38839, 20776, 8473, 10924, 16760, 8455, 39283, 16782, 365, 19928, + 6586, 6808, 4936, 37746, 28178, 20462, 30544, 30545, 31220, 31221, 10919, + 33880, 1011, 10549, 31667, 29278, 31669, 8052, 24155, 24156, 24154, + 31971, 31972, 31970, 24117, 24124, 24116, 31985, 31992, 31984, 24059, + 24057, 24058, 9948, 31935, 31933, 31934, 20787, 20406, 37828, 37867, + 34574, 34572, 37486, 4358, 4359, 31753, 24158, 32008, 20415, 20416, + 20417, 20418, 10571, 10569, 10573, 10562, 10566, 10574, 10563, 10567, + 10572, 10561, 10565, 10560, 10564, 10570, 10568, 34164, 31864, 17382, + 38834, 27419, 27411, 27418, 27412, 27416, 27417, 27415, 27414, 27413, + 11539, 17640, 37474, 4362, 37456, 4361, 37487, 4365, 39292, 3651, 34516, + 17468, 8, 16702, 10550, 3885, 3847, 3928, 3824, 3886, 3848, 3888, 367, + 34514, 37305, 20439, 3864, 3867, 3869, 3861, 11246, 3907, 3771, 31615, + 31612, 31613, 31614, 29932, 34816, 34824, 34825, 34805, 34803, 34806, + 34817, 34788, 34789, 34810, 34812, 34811, 34809, 34790, 34822, 34823, + 34792, 34801, 34800, 34799, 34798, 34814, 34828, 34804, 34791, 34802, + 34826, 34807, 34808, 34819, 34818, 34820, 34829, 34793, 3951, 30531, + 34815, 34796, 34827, 34795, 34794, 34797, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 29944, 29939, + 29942, 29943, 29936, 29937, 29935, 29934, 29941, 29938, 29940, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 6552, 6549, 6548, 6545, 6544, 6547, 6546, 6553, 6551, 6773, 6753, 6798, + 6786, 6768, 6756, 6772, 6770, 6752, 6799, 6787, 31154, 31152, 31151, + 31148, 31147, 31150, 31149, 31155, 31153, 31146, 31137, 31145, 31143, + 31138, 31139, 31141, 31142, 31136, 31140, 31144, 10859, 10871, 10868, + 10853, 10850, 10865, 10862, 10877, 10856, 29450, 29443, 29451, 29449, + 29444, 29445, 29446, 29448, 29442, 29453, 29452, 31182, 31183, 31184, + 31185, 31186, 31187, 31188, 31189, 31190, 31191, 31192, 31193, 31194, + 31195, 31196, 31197, 31198, 31199, 31200, 31201, 31202, 31203, 31204, + 31205, 31206, 31207, 6696, 6697, 6698, 6699, 6700, 6701, 6702, 6703, + 6704, 6705, 6706, 6707, 6708, 6709, 6710, 6711, 6712, 6713, 6714, 6715, + 6716, 6717, 6718, 6719, 6720, 6721, 6722, 6723, 6724, 6725, 6726, 6727, + 6728, 6729, 6730, 6731, 6732, 6733, 6734, 6735, 6736, 6737, 6738, 6739, + 6740, 6741, 6742, 6743, 6744, 6745, 6746, 6747, 6550, 28971, 28969, + 28967, 28972, 28973, 28975, 28976, 28970, 28974, 28968, 11212, 11210, + 11209, 11206, 11205, 11208, 11207, 11213, 11211, 11204, 28966, 4484, + 4430, 4500, 4416, 4493, 4429, 4492, 4428, 4491, 4427, 4490, 4426, 4481, + 4400, 4394, 4423, 4480, 4398, 4392, 4422, 4499, 4528, 4522, 4415, 4498, + 4526, 4520, 4413, 4507, 4544, 4521, 4393, 4541, 4399, 4527, 4419, 4506, + 4543, 4519, 4391, 4540, 4397, 4525, 4418, 4479, 4434, 4512, 4402, 4396, + 4515, 4437, 4421, 4497, 4433, 4511, 4529, 4523, 4514, 4436, 4414, 4505, + 4435, 4513, 4542, 4518, 4395, 4539, 4439, 4517, 4432, 4510, 4401, 4524, + 4516, 4438, 4417, 4483, 4425, 4482, 4424, 4390, 4386, 4407, 4404, 4382, + 4406, 4403, 4381, 4534, 4531, 4385, 4533, 4530, 4384, 4546, 4537, 4389, + 4545, 4536, 4388, 4408, 4405, 4380, 4535, 4532, 4383, 4547, 4538, 4387, + 4441, 4440, 4442, 4443, 4472, 4466, 4476, 4488, 4495, 4509, 4478, 4409, + 4411, 4431, 4420, 4489, 4496, 4410, 4412, 37805, 25259, 25258, 25261, + 25169, 25251, 25264, 25260, 17531, 24111, 24134, 24147, 24075, 24135, + 24090, 24091, 31950, 24410, 26976, 10536, 37853, 31967, 31735, 31736, + 31727, 31728, 31729, 31730, 31731, 31732, 31733, 31734, 3905, 39270, + 39279, 39273, 34342, 34351, 34341, 34348, 34350, 34340, 3902, 39267, + 3898, 39255, 3935, 39302, 3879, 39251, 3930, 39299, 3929, 39298, 3887, + 39259, 3892, 39258, 3891, 39257, 3826, 39214, 3825, 39213, 3852, 39240, + 3851, 39239, 3850, 39238, 3816, 39203, 39204, 17460, 25266, 39189, 11195, + 6525, 4999, 3767, 6517, 6526, 6518, 6523, 6522, 6524, 24073, 31948, + 20804, 20808, 37809, 25163, 37833, 37869, 25213, 25193, 37814, 25170, + 3859, 3858, 3931, 3932, 39155, 34345, 34352, 34347, 34344, 39282, 39300, + 37791, 37792, 22734, 39280, 39276, 39277, 39281, 39196, 39194, 39195, + 39197, 37831, 37886, 25204, 39247, 3874, 39246, 3873, 25227, 3909, 8046, + 37740, 33870, 8458, 3918, 39289, 24420, 37024, 34575, 2525, 10578, 8476, + 30552, 3924, 39291, 2750, 2756, 2757, 32374, 37742, 20434, 39264, 3916, + 32818, 31887, 3857, 3884, 39237, 39296, 39261, 39212, 33767, 6123, 31751, + 3764, 5259, 981, 30652, 6480, 8684, 8683, 34491, 17428, 102, 19104, + 31286, 40667, 37551, 37552, 37557, 37554, 37556, 37555, 37553, 37550, + 39151, 39224, 39268, 3904, 39287, 17453, 22738, 27408, 17434, 11535, + 25571, 20931, 32444, 37959, 29280, 31581, 2523, 36647, 17723, 5997, + 24295, 38880, 25108, 32537, 32372, 6003, 2601, 31474, 39161, 39177, + 39180, 39156, 39164, 39174, 3787, 3803, 3806, 3782, 3790, 3800, 3917, + 39227, 39208, 3815, 39269, 3836, 3821, 39198, 20435, 31740, 16657, 3532, + 3533, 28302, 28301, 28303, 39149, 11540, 37756, 31781, 31782, 31783, + 31784, 31785, 31786, 31787, 31788, 3934, 31780, 31213, 31318, 39152, + 10841, 10842, 10843, 10844, 10845, 10846, 39191, 39193, 3768, 3770, + 28175, 28176, 10881, 10884, 10883, 10882, 39218, 3832, 19105, 979, 8692, + 34485, 32532, 345, 17475, 17719, 34486, 2536, 17473, 30859, 37001, 37000, + 39125, 20285, 11281, 11282, 20791, 25570, 25569, 25568, 38840, 20426, + 26985, 26962, 26975, 25737, 11003, 37758, 8675, 17637, 29090, 6133, + 31022, 20932, 38870, 6483, 3875, 32665, 32577, 31746, 32660, 33875, 3469, + 34418, 39215, 39216, 3829, 3830, 33871, 34577, 31756, 3911, 37023, 37670, + 37669, 39209, 8693, 10923, 30551, 31459, 6065, 19927, 6538, 6134, 29352, + 366, 3925, 39294, 3855, 39235, 11381, 19785, 24062, 34461, 17422, 3922, + 31860, 31861, 2532, 19711, 31293, 32026, 24185, 20814, 3780, 32823, 6515, + 6125, 20390, 17720, 17721, 25666, 28194, 37741, 17511, 17470, 17436, + 32528, 34163, 33765, 20469, 31305, 36766, 20829, 19687, 17639, 9941, + 39219, 3912, 37798, 3915, 25245, 39262, 39228, 36659, 36646, 226, 16779, + 31769, 31761, 38872, 39377, 25244, 31295, 37868, 39250, 3877, 6299, + 19709, 28294, 19745, 2760, 19701, 30861, 19792, 30535, 19747, 23224, + 32670, 30858, 25572, 34489, 17507, 17505, 19730, 17501, 3833, 39223, + 34084, 34518, 6811, 30533, 3781, 30860, 19749, 31471, 32669, 19700, + 30534, 16656, 16651, 16649, 33759, 16650, 19723, 37690, 33762, 36652, + 30532, 19775, 33757, 3831, 39220, 16652, 6800, 19774, 33873, 37295, + 19708, 34083, 19768, 2749, 16655, 19725, 8689, 32668, 29038, 25243, + 37865, 25225, 37883, 3942, 39254, 39217, 3818, 19727, 24417, 26982, + 19791, 19758, 19759, 19719, 19720, 19742, 19741, 9953, 19728, 19734, + 19704, 32221, 17474, 32220, 26969, 26972, 26963, 26964, 26970, 26973, + 19738, 19751, 19737, 19750, 24409, 24407, 26968, 26971, 10906, 10904, + 10903, 10900, 10899, 10902, 10901, 10907, 10905, 10898, 10917, 10914, + 10913, 10910, 10909, 10912, 10911, 10918, 10916, 10908, 10896, 10893, + 10892, 10889, 10888, 10891, 10890, 10897, 10895, 10887, 19787, 19790, + 19746, 19716, 19764, 19752, 19771, 11373, 19755, 37545, 19777, 10539, + 19715, 3893, 37015, 37018, 3894, 19702, 19703, 34479, 19714, 32042, + 24175, 2613, 17522, 19743, 19782, 29432, 9947, 29434, 6588, 39306, 3948, + 3950, 3949, 19705, 19707, 19706, 36653, 19776, 39145, 19788, 30546, + 11214, 36998, 39293, 31299, 30542, 30541, 24109, 31974, 30654, 31871, + 34732, 38821, 26434, 25134, 26255, 34446, 34447, 39207, 980, 16711, + 25241, 37826, 24093, 31965, 17530, 22726, 22725, 24040, 24039, 24089, + 25149, 25138, 37777, 25267, 39199, 39200, 39201, 39275, 39278, 26435, + 26429, 26438, 26432, 26437, 26431, 26436, 26430, 26439, 26433, 37927, + 11363, 976, 8025, 31928, 25139, 25144, 25137, 25141, 25146, 25136, 25140, + 25145, 25142, 25147, 25148, 4925, 4670, 4798, 4671, 4862, 4735, 4799, + 4672, 4894, 4767, 4831, 4704, 4863, 4736, 4800, 4673, 4910, 4783, 4847, + 4720, 4879, 4752, 4816, 4689, 4895, 4768, 4832, 4705, 4864, 4737, 4801, + 4674, 4918, 4791, 4855, 4728, 4887, 4760, 4824, 4697, 4903, 4776, 4840, + 4713, 4872, 4745, 4809, 4682, 4911, 4784, 4848, 4721, 4880, 4753, 4817, + 4690, 4896, 4769, 4833, 4706, 4865, 4738, 4802, 4675, 4922, 4795, 4859, + 4732, 4891, 4764, 4828, 4701, 4907, 4780, 4844, 4717, 4876, 4749, 4813, + 4686, 4915, 4788, 4852, 4725, 4884, 4757, 4821, 4694, 4900, 4773, 4837, + 4710, 4869, 4742, 4806, 4679, 4919, 4792, 4856, 4729, 4888, 4761, 4825, + 4698, 4904, 4777, 4841, 4714, 4873, 4746, 4810, 4683, 4912, 4785, 4849, + 4722, 4881, 4754, 4818, 4691, 4897, 4770, 4834, 4707, 4866, 4739, 4803, + 4676, 4924, 4797, 4861, 4734, 4893, 4766, 4830, 4703, 4909, 4782, 4846, + 4719, 4878, 4751, 4815, 4688, 4917, 4790, 4854, 4727, 4886, 4759, 4823, + 4696, 4902, 4775, 4839, 4712, 4871, 4744, 4808, 4681, 4921, 4794, 4858, + 4731, 4890, 4763, 4827, 4700, 4906, 4779, 4843, 4716, 4875, 4748, 4812, + 4685, 4914, 4787, 4851, 4724, 4883, 4756, 4820, 4693, 4899, 4772, 4836, + 4709, 4868, 4741, 4805, 4678, 4923, 4796, 4860, 4733, 4892, 4765, 4829, + 4702, 4908, 4781, 4845, 4718, 4877, 4750, 4814, 4687, 4916, 4789, 4853, + 4726, 4885, 4758, 4822, 4695, 4901, 4774, 4838, 4711, 4870, 4743, 4807, + 4680, 4920, 4793, 4857, 4730, 4889, 4762, 4826, 4699, 4905, 4778, 4842, + 4715, 4874, 4747, 4811, 4684, 4913, 4786, 4850, 4723, 4882, 4755, 4819, + 4692, 4898, 4771, 4835, 4708, 4867, 4740, 4804, 4677, 32146, 32145, + 24276, 32091, 24100, 32147, 24278, 32093, 11319, 37908, 37945, 11343, + 24280, 32095, 24260, 32139, 32148, 32065, 37910, 11323, 32078, 32077, + 32141, 32143, 32142, 24225, 32085, 24279, 32094, 24202, 32062, 24222, + 32057, 29391, 29371, 29397, 29369, 33968, 33981, 29396, 29368, 33965, + 33980, 32165, 17420, 33966, 29366, 17421, 32166, 29367, 29395, 39134, + 2516, 2515, 2518, 2519, 32043, 24174, 37453, 4360, 37455, 37454, 25223, + 25187, 973, 8022, 32052, 24191, 32831, 32070, 24207, 32061, 24098, 37948, + 24052, 24051, 37765, 37764, 24053, 37766, 24050, 37763, 24239, 32109, + 37922, 11358, 24233, 32103, 37919, 11355, 24238, 32108, 37921, 11357, + 24232, 32102, 37918, 11354, 24235, 37917, 32107, 11352, 24237, 24231, + 32105, 32100, 24236, 24230, 32106, 32101, 37916, 11353, 31940, 16796, + 37299, 24195, 32054, 32053, 24376, 24198, 18125, 34548, 24197, 34718, + 24148, 31945, 37774, 11287, 37565, 40680, 40681, 24140, 32011, 24142, + 32013, 40675, 40671, 40676, 40672, 24121, 31989, 24119, 31986, 24118, + 31987, 24046, 31921, 24047, 31927, 11227, 11252, 24055, 31931, 11202, + 38854, 26857, 31926, 26858, 958, 5, 34100, 34101, 37667, 31877, 959, + 31878, 29930, 29929, 26856, 26855, 26850, 26849, 26854, 26852, 26853, + 26851, 31900, 16758, 16757, 16755, 16756, 6527, 6815, 6802, 6806, 6803, + 6528, 6520, 6529, 37762, 6810, 6533, 6748, 6816, 6519, 6521, 34410, + 34405, 34476, 34462, 34463, 37700, 37544, 37542, 32369, 37541, 32003, + 24126, 38818, 4373, 4374, 3941, 37306, 37307, 39232, 3840, 24145, 32016, + 24063, 31941, 20628, 37233, 20705, 11280, 34353, 20630, 32851, 16797, + 16798, 20471, 18009, 36990, 11300, 11301, 3819, 3860, 39192, 3769, 16810, + 16813, 16809, 16812, 16808, 16811, 32238, 31872, 33927, 31873, 3757, + 3756, 11234, 37561, 24162, 32029, 37308, 27598, 28792, 28790, 28791, + 28800, 28799, 28795, 28796, 37702, 37701, 28794, 28000, 34573, 31737, + 17446, 20786, 20778, 6820, 978, 24492, 24493, 24494, 20779, 31739, 20783, + 20780, 20784, 20782, 20785, 20781, 20929, 22727, 40677, 40678, 40679, + 31573, 31578, 31576, 31572, 31575, 31574, 31577, 27591, 27592, 27594, + 27593, 31569, 31570, 38869, 28293, 28292, 32576, 33848, 28288, 28289, + 6749, 28290, 6543, 31571, 27595, 28291, 20801, 32048, 40673, 372, 20797, + 37751, 37752, 20796, 20795, 37753, 37749, 20794, 37748, 20793, 37750, + 20798, 8038, 8044, 11235, 11236, 8039, 25122, 25130, 11225, 11226, 37698, + 37699, 33787, 33786, 25127, 25124, 25132, 25123, 25131, 25121, 25125, + 25120, 33838, 25129, 25128, 40674, 40670, 16802, 20472, 37559, 37560, + 37301, 37300, 33631, 8477, 16803, 363, 1056, 16793, 31579, 16794, 11215, + 37693, 37009, 16801, 16805, 24393, 18144, 24394, 18145, 24385, 18131, + 24388, 18134, 24386, 18132, 24387, 18133, 24384, 18135, 24378, 18127, + 24377, 18126, 24375, 18123, 24373, 18121, 24372, 18120, 24371, 18124, + 24374, 18122, 33772, 33770, 33773, 33771, 11262, 11261, 11258, 11257, + 33630, 33629, 33628, 33627, 11228, 11230, 11229, 18139, 18128, 24382, + 18142, 24383, 18143, 33846, 22735, 33847, 22736, 16799, 31659, 34564, + 31660, 34565, 31662, 34567, 31658, 34563, 31661, 34566, 31657, 34562, + 11231, 11240, 34559, 34731, 34557, 34729, 34558, 34730, 34556, 34728, + 34551, 34723, 34552, 34724, 34550, 34722, 34553, 34725, 34231, 34318, + 8033, 8035, 8034, 8036, 34546, 34717, 34547, 34716, 34720, 34719, 16710, + 31477, 37539, 17496, 29363, 32835, 32836, 32832, 31300, 38820, 11249, + 38819, 11247, 25133, 32837, 32834, 32833, 11216, 11244, 11238, 31881, + 11018, 38836, 38835, 11286, 31066, 31065, 37564, 37567, 37558, 37571, + 37570, 11260, 11259, 37568, 22724, 11243, 39304, 28801, 29380, 29407, + 33978, 33991, 24103, 24229, 37912, 11325, 29377, 29404, 33975, 33988, + 24105, 37769, 32074, 32075, 24211, 24212, 34346, 34339, 34349, 34343, + 10833, 10834, 10832, 10831, 11198, 3846, 39233, 3939, 39305, 3880, 39252, + 39230, 3837, 20401, 3841, 3863, 39244, 3866, 39248, 3899, 3900, 39265, + 3839, 39231, 3936, 39301, 24049, 37002, 24048, 25143, 24268, 24267, + 24269, 24270, 24205, 24215, 24214, 24263, 24265, 24264, 24199, 39133, + 16795, 31874, 24192, 32060, 32059, 24294, 32155, 31875, 32050, 37298, + 24194, 24193, 32051, 11339, 32830, 32828, 39245, 3901, 39266, 3890, + 39256, 19735, 19748, 19712, 19710, 19713, 33774, 2611, 33775, 2610, 3648, + 32829, 24245, 37931, 32124, 11328, 24107, 37773, 29402, 29375, 33973, + 33986, 24257, 37942, 32136, 11340, 8030, 971, 24256, 37933, 32135, 11330, + 40951, 40951, 29403, 29376, 33974, 33987, 24247, 37941, 32126, 11338, + 20422, 38849, 24246, 37932, 32125, 11329, 24258, 37943, 32137, 11341, + 24228, 37911, 32088, 11327, 968, 969, 967, 970, 31865, 31866, 29275, + 29276, 17502, 32089, 40951, 34830, 37013, 37017, 37014, 37016, 3853, + 3933, 3895, 3827, 11332, 11333, 37935, 37936, 24250, 32129, 24249, 32128, + 3772, 3773, 3774, 3775, 3776, 3778, 3777, 3779, 31912, 31913, 31910, + 31911, 31907, 31909, 31906, 31908, 37956, 37761, 30876, 30875, 30877, + 2755, 6818, 6532, 3906, 3817, 37668, 20400, 3940, 3870, 3862, 3865, 3868, + 29281, 37471, 4348, 24405, 32222, 39222, 32223, 34303, 37744, 18671, + 31585, 31584, 31583, 31582, 37538, 31686, 2531, 20463, 31458, 29058, + 39249, 3820, 37580, 9943, 19685, 40759, 22562, 1053, 97, 38957, 31596, + 24074, 31949, 34492, 34493, 24266, 37947, 32144, 11345, 16815, 16814, + 32666, 32364, 32362, 32363, 32361, 32365, 32366, 16800, 37755, 32657, + 11283, 31219, 31888, 19929, 17909, 17911, 17945, 17919, 17915, 17946, + 17953, 17916, 17952, 17925, 17921, 17920, 17914, 17956, 17927, 17928, + 17929, 17930, 17932, 17934, 17938, 17942, 17955, 17917, 17954, 17931, + 17933, 17935, 17944, 17913, 17937, 17948, 17947, 17949, 17939, 17951, + 17940, 17941, 17950, 17923, 17910, 17922, 17918, 17924, 17936, 17943, + 17926, 17912, 17957, 17959, 17993, 17967, 17963, 17994, 18001, 17964, + 18000, 17973, 17969, 17968, 17962, 18004, 17975, 17976, 17977, 17978, + 17980, 17982, 17986, 17990, 18003, 17965, 18002, 17979, 17981, 17983, + 17992, 17961, 17985, 17996, 17995, 17997, 17987, 17999, 17988, 17989, + 17998, 17971, 17958, 17970, 17966, 17972, 17984, 17991, 17974, 17960, + 22957, 23603, 22960, 23051, 23067, 23337, 23828, 22900, 23522, 22946, + 23582, 23214, 23995, 22780, 22979, 23118, 23119, 23948, 23191, 23965, + 23947, 22905, 23529, 23893, 23450, 23869, 23684, 23261, 24020, 27718, + 23094, 23218, 8485, 8579, 8535, 8629, 8499, 8593, 8496, 8590, 8540, 8634, + 8530, 8624, 8534, 8628, 8501, 8595, 8532, 8626, 8538, 8632, 8504, 8598, + 8509, 8603, 8541, 8635, 8542, 8636, 8505, 8599, 8510, 8604, 8537, 8631, + 8539, 8633, 8531, 8625, 8533, 8627, 8543, 8637, 8507, 8601, 8503, 8597, + 8536, 8630, 8526, 8620, 8493, 8587, 8521, 8615, 8489, 8583, 8494, 8588, + 8495, 8589, 8490, 8584, 8519, 8613, 8529, 8623, 8491, 8585, 8517, 8611, + 8520, 8614, 8484, 8578, 8492, 8586, 8512, 8606, 8513, 8607, 8508, 8602, + 8515, 8609, 8514, 8608, 8511, 8605, 8518, 8612, 8516, 8610, 8524, 8618, + 8522, 8616, 8523, 8617, 8525, 8619, 8639, 8640, 8641, 8643, 8644, 8638, + 8642, 8487, 8581, 8488, 8582, 8483, 8482, 8481, 8486, 8580, 40951, 40951, + 40951, 40951, 40951, 8577, 8574, 8575, 8576, 8572, 8573, 8645, 17768, + 17794, 17779, 17800, 17801, 17799, 17789, 17790, 17802, 17783, 17792, + 17795, 17797, 17803, 17785, 17788, 17793, 17787, 17791, 17804, 17784, + 17782, 17778, 17798, 17786, 17774, 17777, 17781, 17776, 17775, 17796, + 17780, 17769, 17773, 17771, 17806, 17770, 17772, 40951, 17805, 40951, + 40951, 40951, 40951, 40951, 17767, 40951, 40951, 37248, 37259, 37260, + 37253, 37255, 37238, 37277, 37249, 37252, 37250, 37251, 37287, 37276, + 37256, 37242, 37258, 37261, 37237, 37246, 37262, 37275, 37257, 37243, + 37282, 37247, 37288, 37270, 37236, 37244, 37278, 37279, 37280, 37241, + 37245, 37281, 37290, 37272, 37273, 37254, 37240, 37235, 37263, 37265, + 37264, 37266, 37267, 37274, 37268, 37283, 37284, 37285, 37269, 37239, + 37271, 37286, 37289, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 37291, 37292, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 37234, 17250, 17067, 17154, + 17194, 17170, 16857, 17232, 16895, 17085, 17076, 16977, 17310, 16925, + 16910, 17241, 17201, 16886, 17101, 17114, 16962, 16966, 16965, 16964, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 17184, + 17190, 17188, 17185, 17187, 17186, 17189, 40951, 16876, 16882, 16880, + 16877, 16879, 16878, 16881, 40951, 17300, 17306, 17304, 17301, 17303, + 17302, 17305, 40951, 16869, 16875, 16873, 16870, 16872, 16871, 16874, + 40951, 17136, 17142, 17140, 17137, 17139, 17138, 17141, 40951, 17045, + 17051, 17049, 17046, 17048, 17047, 17050, 40951, 17277, 17283, 17281, + 17278, 17280, 17279, 17282, 40951, 16988, 16994, 16992, 16989, 16991, + 16990, 16993, 40951, 8091, 8129, 8126, 8093, 8123, 8124, 8130, 8097, + 8098, 8099, 8106, 8128, 8100, 8094, 8122, 8119, 8121, 8125, 8109, 8108, + 8127, 8095, 8131, 8105, 8092, 8118, 8114, 8116, 8103, 8117, 8090, 8102, + 31923, 31922, 24125, 31993, 24067, 31938, 31760, 31759, 11201, 24128, + 32001, 31768, 24108, 31973, 11542, 31064, 17495, 31883, 20461, 11200, + 11324, 37895, 11203, 11251, 20810, 31023, 20457, 37304, 24088, 31963, + 37303, 37302, 24157, 32007, 37480, 37484, 4353, 4356, 24113, 31978, + 24066, 31943, 37696, 30525, 34406, 17464, 31898, 38852, 32158, 39365, + 37672, 31758, 31770, 37683, 10530, 10531, 37673, 37469, 37708, 37020, + 34506, 38855, 39348, 6002, 11219, 31897, 11222, 10537, 11239, 20811, + 20812, 25159, 25160, 11237, 11197, 37569, 26959, 31063, 31717, 8648, + 8685, 8686, 37392, 26958, 26960, 24123, 31991, 24122, 31990, 37461, + 37465, 4339, 4343, 29931, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 7916, 7869, 7919, + 7918, 7917, 7912, 7840, 7937, 7951, 7950, 7873, 7920, 7933, 7932, 7902, + 7901, 7900, 7899, 7929, 7938, 7928, 7927, 7888, 7887, 7891, 7915, 40951, + 7872, 7935, 7909, 7874, 7907, 7867, 7943, 7942, 7881, 7911, 7910, 7921, + 7871, 7875, 7896, 7838, 7880, 7931, 7930, 7843, 7926, 7854, 7949, 7948, + 7947, 7946, 7904, 7934, 7914, 7879, 7952, 7842, 7841, 7903, 7908, 7885, + 7884, 7883, 7939, 7870, 7945, 7944, 7858, 7922, 7890, 7857, 7856, 7882, + 7866, 7923, 7941, 7940, 7868, 7850, 7898, 7897, 7853, 7851, 7906, 7905, + 7913, 7844, 7860, 7852, 7862, 7848, 7878, 7877, 7876, 7846, 7889, 7865, + 7839, 7886, 7847, 7864, 7855, 7924, 7925, 7849, 7895, 7845, 7893, 7861, + 7894, 7863, 7936, 7892, 7859, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 21128, 21110, 21044, 21164, + 21152, 21096, 21196, 21111, 21124, 21107, 21059, 21063, 21048, 21033, + 21099, 21185, 21131, 21102, 21136, 21213, 21170, 21141, 21093, 21198, + 21042, 21153, 21029, 21133, 21004, 21122, 21058, 21056, 21151, 21079, + 21081, 21061, 21011, 21210, 21036, 21144, 21098, 21180, 21103, 21031, + 21169, 21120, 21142, 21211, 21129, 21194, 21054, 21159, 21045, 21113, + 21197, 21160, 21019, 21178, 21020, 21172, 21090, 21087, 21050, 21088, + 21021, 21139, 21150, 21043, 21006, 21181, 21126, 21183, 21149, 21123, + 21193, 21104, 21173, 21038, 21204, 21049, 21032, 21078, 21027, 21171, + 21203, 21070, 21028, 21065, 21047, 21086, 21165, 21067, 21035, 21051, + 21134, 21100, 21114, 21188, 21177, 21109, 21215, 21068, 21015, 21161, + 21046, 21206, 21182, 21041, 21064, 21166, 21002, 21175, 21168, 21192, + 21082, 21026, 21176, 21007, 21143, 21162, 21101, 21127, 21157, 21076, + 21132, 21005, 21135, 21055, 21022, 21115, 21116, 21154, 21003, 21118, + 21189, 21130, 21016, 21174, 21034, 21083, 21187, 21097, 21012, 21202, + 21030, 21205, 21155, 21095, 21167, 21199, 21023, 21137, 21008, 21156, + 21146, 21145, 21077, 21018, 21025, 21009, 21119, 21201, 21037, 21209, + 21040, 21200, 21080, 21112, 21085, 21121, 21163, 21158, 21138, 21014, + 21212, 21066, 21105, 21184, 21108, 21179, 21106, 21208, 21073, 21057, + 21091, 21074, 21094, 21017, 21186, 21089, 21069, 21147, 21024, 21084, + 21071, 21010, 21148, 21039, 21207, 21092, 21214, 21117, 21013, 21062, + 21075, 21191, 21053, 21140, 21125, 21060, 21190, 21052, 21195, 21072, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 20495, 20493, 20494, 20492, + 20507, 20503, 20502, 20499, 20500, 20501, 20497, 20496, 20504, 20498, + 20508, 20506, 20592, 20491, 20590, 11007, 20828, 20591, 20490, 20510, + 24045, 31920, 24064, 31939, 24060, 31936, 24139, 32010, 24054, 31930, + 31598, 17902, 24136, 32005, 24141, 32012, 24144, 32015, 24143, 32014, + 39132, 31880, 11233, 25157, 31599, 19581, 19574, 19571, 19577, 19576, + 19579, 19578, 19582, 19580, 20588, 20587, 20509, 20586, 19240, 19239, + 39138, 38827, 38830, 38828, 38831, 38829, 6801, 20584, 19575, 19573, + 19572, 38826, 25816, 31216, 20589, 20585, 40951, 20305, 20291, 20307, + 20385, 20309, 20387, 20306, 20384, 20308, 20386, 20346, 20336, 20348, + 20338, 20350, 20340, 20347, 20337, 20349, 20339, 20310, 20371, 20312, + 20373, 20314, 20375, 20311, 20372, 20313, 20374, 20366, 20331, 20368, + 20333, 20304, 20370, 20335, 20367, 20332, 20369, 20334, 20326, 20328, + 20330, 20327, 20329, 20341, 20321, 20356, 20343, 20315, 20358, 20345, + 20324, 20360, 20342, 20322, 20357, 20344, 20323, 20359, 20351, 20353, + 20355, 20352, 20354, 20298, 20380, 20300, 20382, 20299, 20381, 20361, + 20363, 20365, 20362, 20364, 20294, 20376, 20378, 20377, 20379, 20325, + 20383, 20301, 20302, 40951, 40951, 8288, 8287, 21453, 21452, 20389, + 20388, 20290, 21450, 21380, 21309, 21382, 21443, 21384, 21445, 21381, + 21442, 21383, 21444, 21405, 21395, 21407, 21397, 21409, 21399, 21406, + 21396, 21408, 21398, 21385, 21430, 21387, 21432, 21389, 21434, 21386, + 21431, 21388, 21433, 21420, 21390, 21422, 21392, 21367, 21424, 21394, + 21421, 21391, 21423, 21393, 21347, 21349, 21351, 21348, 21350, 21400, + 21324, 21410, 21402, 21318, 21412, 21404, 21327, 21414, 21401, 21325, + 21411, 21403, 21326, 21413, 21342, 21328, 21345, 21343, 21344, 21372, + 21439, 21374, 21441, 21373, 21440, 21415, 21417, 21419, 21416, 21418, + 21368, 21435, 21437, 21436, 21438, 21346, 21429, 21362, 21363, 21425, + 21427, 21426, 21428, 21449, 21451, 21448, 21446, 21447, 40951, 40951, + 40951, 40951, 40951, 4320, 4328, 4327, 4325, 4324, 4331, 4296, 4316, + 4284, 4312, 4326, 4322, 4329, 4333, 4309, 4315, 4319, 4330, 4308, 4314, + 4318, 4264, 4300, 4276, 4281, 4265, 4282, 4267, 4307, 4269, 4277, 4270, + 4278, 4283, 4289, 4274, 4295, 4332, 4297, 4286, 4292, 4303, 4299, 40951, + 19493, 19537, 19494, 19499, 19500, 19502, 19529, 19540, 19515, 19516, + 19518, 19520, 19527, 19523, 19519, 19526, 19495, 19505, 19539, 19506, + 19530, 19542, 19487, 19482, 19536, 19481, 19492, 19528, 19513, 19566, + 19477, 19480, 19563, 19564, 19484, 19483, 19550, 19549, 19567, 19546, + 19547, 19568, 19555, 19569, 19545, 19544, 19548, 19559, 19485, 19565, + 19486, 19570, 19538, 19501, 19504, 19503, 19517, 19524, 19521, 19522, + 19525, 19496, 19498, 19497, 19491, 19512, 19510, 19507, 19508, 19511, + 19509, 19489, 19490, 19532, 19533, 19535, 19534, 19531, 19514, 19543, + 19552, 19554, 19553, 19488, 19541, 19551, 19556, 19557, 19558, 19561, + 19560, 19562, 19478, 19479, 40951, 20484, 20487, 20486, 20482, 20480, + 20476, 20483, 20478, 20474, 20477, 20485, 20481, 20475, 20488, 20489, + 20479, 4321, 4310, 4323, 4287, 4280, 4279, 4306, 4302, 4294, 4271, 4290, + 4275, 4293, 4298, 4266, 4268, 4273, 4305, 4301, 4291, 4262, 4263, 4261, + 4260, 4285, 4317, 4311, 4259, 4288, 4313, 4304, 4272, 7983, 7986, 7987, + 7985, 7973, 7959, 7965, 7954, 7964, 7977, 7966, 7962, 7955, 7963, 7960, + 7989, 7953, 7972, 7968, 7981, 7988, 7958, 7967, 7976, 7975, 7982, 7980, + 7969, 7971, 7984, 7979, 7974, 7956, 7961, 7970, 7990, 7957, 7978, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 20505, 21365, + 21377, 21378, 21366, 21376, 21352, 21354, 21356, 21353, 21355, 21379, + 21357, 21359, 21361, 21358, 21360, 31078, 31082, 31094, 31088, 31080, + 31086, 31090, 31096, 31071, 31069, 31076, 31092, 31084, 31074, 31079, + 31083, 31095, 31089, 31081, 31087, 31091, 31097, 31072, 31070, 31077, + 31093, 31085, 31075, 31073, 31135, 31134, 40951, 31133, 31129, 31127, + 31108, 31106, 31126, 31118, 31103, 31112, 31128, 31111, 31105, 31131, + 31130, 31110, 31102, 31125, 31123, 31132, 31120, 31113, 31121, 31104, + 31099, 31109, 31116, 31100, 31122, 31124, 31101, 31114, 31098, 31107, + 31115, 31119, 31117, 6620, 6608, 6630, 6609, 6774, 6788, 6776, 6758, + 6755, 6771, 6769, 6751, 31215, 6789, 6795, 6794, 6791, 6790, 6793, 6792, + 6797, 6796, 6775, 6777, 6783, 6782, 6779, 6778, 6568, 6572, 6584, 6578, + 6570, 6576, 6580, 6561, 6559, 6557, 6566, 6582, 6574, 6564, 6569, 6573, + 6585, 6579, 6571, 6577, 6581, 6562, 6560, 6558, 6567, 6583, 6575, 6565, + 6695, 6694, 6563, 22560, 6643, 6639, 6637, 6605, 6603, 6635, 6626, 6600, + 6618, 6638, 6616, 6602, 6641, 6640, 6614, 6598, 6629, 6634, 6607, 6631, + 6619, 6632, 6601, 6594, 6610, 6625, 6615, 6604, 6628, 6599, 6636, 6591, + 6642, 6622, 6595, 6593, 6606, 6596, 6611, 6612, 6624, 6613, 6623, 6633, + 6627, 6597, 6621, 6589, 6617, 6781, 6780, 6785, 6784, 6757, 6759, 6765, + 6764, 6761, 6760, 6763, 6762, 6767, 6766, 6754, 20575, 20578, 20579, + 20517, 20580, 20576, 20577, 20516, 20582, 20583, 20581, 20549, 34191, + 34157, 34159, 24491, 6689, 6691, 6693, 6690, 6692, 6652, 6654, 6656, + 6653, 6655, 6672, 6674, 6676, 6673, 6675, 6677, 6679, 6681, 6678, 6680, + 6662, 6664, 6666, 6663, 6665, 6647, 6649, 6651, 6648, 6650, 6657, 6659, + 6661, 6658, 6660, 6686, 6688, 6687, 6667, 6669, 6671, 6668, 6670, 6682, + 6684, 6683, 6685, 34155, 34116, 34118, 34117, 34119, 34194, 34195, 34358, + 34158, 34151, 34284, 34288, 34203, 34202, 34201, 34166, 34167, 34171, + 34170, 34204, 34169, 34205, 34208, 34206, 34207, 34172, 34173, 34215, + 34216, 34217, 34214, 34213, 34324, 34325, 34332, 34326, 34327, 34142, + 34146, 34147, 34337, 34277, 34278, 34179, 34291, 34292, 34123, 34300, + 34298, 34299, 34126, 34186, 34185, 34125, 34187, 34180, 34297, 34295, + 34181, 34296, 34294, 34128, 34301, 34127, 34184, 34302, 34182, 34183, + 34241, 34242, 34245, 34244, 34243, 34251, 34252, 34253, 34248, 34249, + 34250, 34356, 34355, 34357, 34319, 34320, 34322, 34321, 34317, 34316, + 34338, 20573, 20574, 20556, 20558, 20565, 20564, 20571, 20569, 20560, + 20567, 20559, 20562, 20555, 20557, 20566, 20563, 20572, 20570, 20561, + 20568, 20550, 20554, 20553, 20552, 20551, 34189, 34141, 34121, 34124, + 34289, 34305, 34143, 34145, 34144, 34199, 34152, 34156, 34153, 34154, + 34133, 34293, 34276, 34258, 34240, 34200, 34222, 34246, 34176, 34130, + 34219, 34306, 34279, 34259, 34260, 34273, 34223, 34192, 34218, 34270, + 34174, 34336, 34261, 34274, 34150, 34225, 34165, 34280, 34262, 34255, + 34134, 34209, 34257, 34136, 34239, 34212, 34256, 34135, 34238, 34211, + 34235, 34236, 34290, 34221, 34272, 34175, 34313, 34314, 34315, 34310, + 34281, 34263, 34275, 34311, 34282, 34264, 34266, 34227, 34267, 34312, + 34283, 34265, 34268, 34228, 34269, 34220, 34237, 34120, 34129, 34139, + 34140, 34137, 34132, 34148, 34177, 34178, 34188, 34193, 34224, 34210, + 34226, 34232, 34233, 34230, 34234, 34247, 34254, 34271, 34307, 34308, + 34304, 34309, 34333, 34334, 34354, 34122, 34114, 20548, 20533, 20521, + 20540, 20539, 20546, 20544, 20535, 20542, 20534, 20537, 20532, 20520, + 20541, 20538, 20547, 20545, 20536, 20543, 20522, 20530, 20528, 20527, + 20524, 20523, 20526, 20525, 20531, 20529, 20518, 20519, 34168, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 20267, 20270, 20231, + 20281, 20278, 20226, 20264, 20243, 20260, 20277, 20255, 20261, 20236, + 20238, 20248, 20282, 20235, 20279, 20219, 20225, 20223, 20242, 20262, + 20256, 20244, 20241, 20249, 20240, 20265, 20268, 20247, 20233, 20257, + 20239, 20254, 20234, 20269, 20252, 20250, 20229, 20228, 20246, 20224, + 20227, 20237, 20253, 20251, 20271, 20258, 20266, 20263, 20275, 20230, + 20273, 20221, 20272, 20274, 20276, 20232, 20280, 20245, 20259, 20220, + 20222, 39884, 39891, 39883, 39890, 39888, 39889, 39886, 39887, 40659, + 40660, 40657, 40658, 40656, 40654, 40655, 40663, 40664, 40661, 40662, + 40666, 40665, 40531, 39551, 39552, 39545, 39550, 39548, 39549, 39546, + 39547, 39555, 39556, 39553, 39554, 39536, 39534, 39535, 39559, 39560, + 39557, 39558, 39544, 39542, 39543, 39540, 39541, 39533, 39539, 39538, + 39537, 39565, 39566, 39561, 39564, 39563, 39562, 40263, 40264, 40258, + 40262, 40261, 40259, 40260, 40267, 40268, 40265, 40266, 40252, 40250, + 40251, 40271, 40272, 40269, 40270, 40256, 40257, 40249, 40255, 40254, + 40253, 40277, 40278, 40273, 40276, 40275, 40274, 39519, 39520, 39513, + 39518, 39516, 39517, 39514, 39515, 39523, 39524, 39521, 39522, 39504, + 39502, 39503, 39527, 39528, 39525, 39526, 39512, 39510, 39511, 39508, + 39509, 39501, 39507, 39506, 39505, 39531, 39532, 39529, 39530, 40066, + 40067, 40061, 40065, 40064, 40062, 40063, 40070, 40071, 40068, 40069, + 40074, 40075, 40072, 40073, 40080, 40081, 40076, 40079, 40078, 40077, + 40086, 40087, 40082, 40085, 40084, 40083, 39809, 39810, 39804, 39808, + 39807, 39805, 39806, 39813, 39814, 39811, 39812, 39798, 39796, 39797, + 39817, 39818, 39815, 39816, 39802, 39803, 39795, 39801, 39800, 39799, + 39823, 39819, 39822, 39821, 39820, 40045, 40046, 40040, 40044, 40043, + 40041, 40042, 40049, 40050, 40047, 40048, 40033, 40034, 40031, 40032, + 40053, 40054, 40051, 40052, 40060, 40059, 40038, 40039, 40030, 40037, + 40036, 40035, 40057, 40058, 40055, 40056, 39690, 39691, 39688, 39689, + 39686, 39687, 39684, 39685, 39683, 39681, 39682, 39700, 39701, 39696, + 39699, 39698, 39697, 39694, 39695, 39692, 39693, 40509, 40510, 40503, + 40508, 40506, 40507, 40504, 40505, 40513, 40514, 40511, 40512, 40517, + 40518, 40515, 40516, 40502, 40501, 40523, 40524, 40519, 40522, 40521, + 40520, 40529, 40530, 40525, 40528, 40527, 40526, 39668, 39669, 39663, + 39667, 39666, 39664, 39665, 39675, 39676, 39673, 39674, 39657, 39656, + 39679, 39680, 39677, 39678, 39672, 39670, 39671, 39661, 39662, 39655, + 39660, 39659, 39658, 40488, 40489, 40483, 40487, 40486, 40484, 40485, + 40495, 40496, 40493, 40494, 40476, 40477, 40474, 40475, 40499, 40500, + 40497, 40498, 40492, 40490, 40491, 40481, 40482, 40473, 40480, 40479, + 40478, 39642, 39643, 39637, 39641, 39640, 39638, 39639, 39649, 39650, + 39647, 39648, 39631, 39629, 39630, 39653, 39654, 39651, 39652, 39646, + 39644, 39645, 39635, 39636, 39628, 39634, 39633, 39632, 40092, 40093, + 40088, 40091, 40090, 40089, 40099, 40100, 40097, 40098, 40103, 40104, + 40101, 40102, 40096, 40094, 40095, 40109, 40110, 40105, 40108, 40107, + 40106, 39839, 39840, 39833, 39838, 39836, 39837, 39834, 39835, 39843, + 39844, 39841, 39842, 39828, 39827, 39825, 39826, 39824, 39832, 39830, + 39831, 39829, 40237, 40238, 40232, 40236, 40235, 40233, 40234, 40241, + 40239, 40240, 40226, 40224, 40225, 40247, 40248, 40245, 40246, 40244, + 40242, 40243, 40230, 40231, 40223, 40229, 40228, 40227, 39777, 39778, + 39772, 39776, 39775, 39773, 39774, 39787, 39788, 39785, 39786, 39766, + 39764, 39765, 39784, 39782, 39783, 39781, 39779, 39780, 39770, 39771, + 39763, 39769, 39768, 39767, 39793, 39794, 39789, 39792, 39791, 39790, + 39992, 39993, 39986, 39991, 39989, 39990, 39987, 39988, 39996, 39997, + 39994, 39995, 39976, 39977, 39974, 39975, 40000, 40001, 39998, 39999, + 39985, 39983, 39984, 39981, 39982, 39973, 39980, 39979, 39978, 40006, + 40007, 40002, 40005, 40004, 40003, 39746, 39747, 39740, 39745, 39743, + 39744, 39741, 39742, 39750, 39751, 39748, 39749, 39733, 39734, 39731, + 39732, 39758, 39759, 39756, 39757, 39754, 39755, 39752, 39753, 39738, + 39739, 39730, 39737, 39736, 39735, 39959, 39960, 39954, 39958, 39957, + 39955, 39956, 39963, 39964, 39961, 39962, 39948, 39946, 39947, 39971, + 39972, 39969, 39970, 39967, 39968, 39965, 39966, 39952, 39953, 39945, + 39951, 39950, 39949, 39706, 39707, 39702, 39705, 39703, 39704, 39720, + 39721, 39718, 39719, 39711, 39712, 39709, 39710, 39728, 39729, 39726, + 39727, 39724, 39725, 39722, 39723, 39716, 39717, 39708, 39715, 39714, + 39713, 40029, 40028, 40022, 40023, 40020, 40021, 40011, 40009, 40010, + 40026, 40027, 40024, 40025, 40019, 40017, 40018, 40015, 40016, 40008, + 40014, 40013, 40012, 39858, 39859, 39852, 39857, 39855, 39856, 39853, + 39854, 39862, 39863, 39860, 39861, 39847, 39848, 39845, 39846, 39866, + 39867, 39864, 39865, 39851, 39849, 39850, 40119, 40117, 40118, 40122, + 40123, 40120, 40121, 40112, 40113, 40111, 40126, 40127, 40124, 40125, + 40116, 40114, 40115, 39762, 39761, 39760, 39877, 39878, 39875, 39876, + 39870, 39871, 39868, 39869, 39881, 39882, 39879, 39880, 39874, 39872, + 39873, 40543, 40544, 40541, 40542, 40534, 40532, 40533, 40540, 40538, + 40539, 40537, 40535, 40536, 40606, 40607, 40601, 40605, 40604, 40602, + 40603, 40642, 40643, 40640, 40641, 40595, 40593, 40594, 40646, 40647, + 40644, 40645, 40639, 40637, 40638, 40599, 40600, 40592, 40598, 40597, + 40596, 40652, 40653, 40648, 40651, 40650, 40649, 39612, 39613, 39606, + 39611, 39609, 39610, 39607, 39608, 39616, 39617, 39614, 39615, 39597, + 39595, 39596, 39620, 39621, 39618, 39619, 39605, 39603, 39604, 39601, + 39602, 39594, 39600, 39599, 39598, 39626, 39627, 39622, 39625, 39624, + 39623, 40620, 40621, 40614, 40619, 40617, 40618, 40615, 40616, 40624, + 40625, 40622, 40623, 40613, 40611, 40612, 40610, 40608, 40609, 40630, + 40626, 40629, 40628, 40627, 40635, 40636, 40631, 40634, 40633, 40632, + 40209, 40210, 40204, 40208, 40207, 40205, 40206, 40213, 40214, 40211, + 40212, 40197, 40196, 40203, 40202, 40222, 40221, 40201, 40195, 40200, + 40199, 40198, 40219, 40220, 40215, 40218, 40217, 40216, 40454, 40455, + 40449, 40453, 40452, 40450, 40451, 40461, 40462, 40459, 40460, 40443, + 40441, 40442, 40465, 40466, 40463, 40464, 40458, 40456, 40457, 40447, + 40448, 40440, 40446, 40445, 40444, 40471, 40472, 40467, 40470, 40469, + 40468, 40390, 40391, 40385, 40389, 40388, 40386, 40387, 40397, 40398, + 40395, 40396, 40401, 40402, 40399, 40400, 40394, 40392, 40393, 40405, + 40406, 40403, 40404, 40411, 40412, 40407, 40410, 40409, 40408, 40576, + 40577, 40574, 40575, 40568, 40566, 40567, 40584, 40585, 40582, 40583, + 40580, 40581, 40578, 40579, 40572, 40573, 40565, 40571, 40570, 40569, + 40590, 40591, 40586, 40589, 40588, 40587, 39578, 39579, 39576, 39577, + 39570, 39571, 39568, 39569, 39586, 39587, 39584, 39585, 39582, 39583, + 39580, 39581, 39575, 39567, 39574, 39573, 39572, 39592, 39593, 39588, + 39591, 39590, 39589, 40358, 40357, 40337, 40336, 40349, 40350, 40347, + 40348, 40345, 40346, 40343, 40344, 40341, 40342, 40335, 40340, 40339, + 40338, 40355, 40356, 40351, 40354, 40353, 40352, 40158, 40159, 40156, + 40157, 40155, 40153, 40154, 40162, 40163, 40160, 40161, 40168, 40169, + 40164, 40167, 40166, 40165, 40174, 40175, 40170, 40173, 40172, 40171, + 40424, 40425, 40422, 40423, 40416, 40414, 40415, 40432, 40433, 40430, + 40431, 40428, 40429, 40426, 40427, 40420, 40421, 40413, 40419, 40418, + 40417, 40438, 40439, 40434, 40437, 40436, 40435, 40373, 40374, 40371, + 40372, 40362, 40360, 40361, 40377, 40378, 40375, 40376, 40370, 40368, + 40369, 40366, 40367, 40359, 40365, 40364, 40363, 40383, 40384, 40379, + 40382, 40381, 40380, 39933, 39934, 39927, 39932, 39930, 39931, 39928, + 39929, 39920, 39921, 39918, 39919, 39937, 39938, 39935, 39936, 39925, + 39926, 39917, 39924, 39923, 39922, 39943, 39944, 39939, 39942, 39941, + 39940, 40295, 40296, 40289, 40294, 40292, 40293, 40290, 40291, 40282, + 40283, 40280, 40281, 40299, 40300, 40297, 40298, 40287, 40288, 40279, + 40286, 40285, 40284, 40305, 40306, 40301, 40304, 40303, 40302, 39907, + 39908, 39901, 39906, 39904, 39905, 39902, 39903, 39895, 39893, 39894, + 39911, 39912, 39909, 39910, 39899, 39900, 39892, 39898, 39897, 39896, + 39915, 39916, 39913, 39914, 40141, 40142, 40135, 40140, 40138, 40139, + 40136, 40137, 40130, 40129, 40145, 40146, 40143, 40144, 40134, 40128, + 40133, 40132, 40131, 40151, 40152, 40147, 40150, 40149, 40148, 40189, + 40190, 40183, 40188, 40186, 40187, 40184, 40185, 40179, 40177, 40178, + 40193, 40194, 40191, 40192, 40181, 40182, 40176, 40180, 40551, 40552, + 40545, 40550, 40548, 40549, 40546, 40547, 40564, 40563, 40555, 40556, + 40553, 40554, 40561, 40562, 40557, 40560, 40559, 40558, 40323, 40324, + 40317, 40322, 40320, 40321, 40318, 40319, 40310, 40311, 40308, 40309, + 40327, 40328, 40325, 40326, 40315, 40316, 40307, 40314, 40313, 40312, + 40333, 40334, 40329, 40332, 40331, 40330, 40951, 40951, 40951, 39498, + 39471, 39469, 39476, 39450, 39486, 39457, 39459, 39475, 39462, 39473, + 39446, 39474, 39492, 39480, 39461, 39487, 39460, 39493, 39451, 39454, + 39447, 39456, 39477, 39488, 39500, 39465, 39496, 39481, 39464, 39491, + 39489, 39485, 39490, 39497, 39468, 39478, 39467, 39458, 39466, 39499, + 39455, 39482, 39472, 39449, 39448, 39453, 39463, 39483, 39494, 39484, + 39452, 39495, 39479, 39470, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 25094, 25083, 25082, 25066, 25064, 25063, 25077, + 25081, 25080, 25096, 25075, 25074, 25065, 25062, 25061, 25098, 25071, + 25097, 25085, 25088, 25089, 25070, 25079, 25100, 25078, 25095, 25099, + 25084, 25087, 25076, 25090, 25091, 25072, 25073, 25101, 25092, 25067, + 25068, 25069, 25093, 25057, 25060, 25058, 25056, 25059, 25055, 25102, + 25103, 38225, 38226, 38062, 38211, 38212, 38185, 37988, 37995, 38098, + 38071, 38105, 38045, 38171, 38199, 38023, 38016, 37974, 37967, 38089, + 38192, 37981, 38124, 38009, 38002, 38037, 38030, 38164, 38178, 38143, + 38206, 38084, 38130, 38051, 38112, 38157, 38150, 38234, 38235, 38066, + 38067, 38220, 38221, 38187, 37990, 37997, 38100, 38077, 38107, 38048, + 38173, 38201, 38025, 38018, 37976, 37969, 38093, 38194, 37983, 38126, + 38011, 38004, 38039, 38032, 38166, 38180, 38145, 38208, 38085, 38135, + 38056, 38114, 38159, 38152, 38232, 38233, 38137, 38064, 38065, 38218, + 38219, 38186, 37989, 37996, 38099, 38075, 38076, 38106, 38047, 38172, + 38200, 38024, 38017, 37975, 37968, 38092, 38193, 37982, 38125, 38010, + 38003, 38038, 38031, 38165, 38179, 38144, 38207, 38081, 38082, 38134, + 38055, 38113, 38158, 38151, 38229, 38230, 38060, 38215, 38216, 38183, + 37986, 37993, 38096, 38074, 38103, 38043, 38169, 38197, 38021, 38014, + 37972, 37965, 38091, 38190, 37979, 38122, 38007, 38000, 38035, 38028, + 38162, 38176, 38141, 38204, 38080, 38133, 38054, 38110, 38155, 38148, + 38236, 38237, 38068, 38069, 38222, 38223, 38188, 37991, 37998, 38101, + 38078, 38108, 38049, 38174, 38202, 38026, 38019, 37977, 37970, 38094, + 38195, 37984, 38127, 38012, 38005, 38040, 38033, 38167, 38181, 38146, + 38209, 38086, 38136, 38057, 38115, 38160, 38153, 38228, 38231, 38139, + 38058, 38059, 38214, 38217, 38182, 37985, 37992, 38095, 38073, 38102, + 38041, 38042, 38168, 38196, 38020, 38013, 37971, 37964, 38090, 38189, + 37978, 38116, 38006, 37999, 38034, 38027, 38161, 38175, 38140, 38203, + 38079, 38132, 38053, 38109, 38154, 38147, 38224, 38227, 38138, 38061, + 38063, 38210, 38213, 38184, 37987, 37994, 38097, 38070, 38072, 38104, + 38044, 38046, 38170, 38198, 38022, 38015, 37973, 37966, 38087, 38191, + 37980, 38123, 38008, 38001, 38036, 38029, 38163, 38177, 38142, 38205, + 38083, 38129, 38131, 38050, 38052, 38111, 38156, 38149, 38128, 38088, + 37961, 37962, 37963, 38119, 38120, 38117, 38241, 38244, 38247, 38246, + 38250, 38242, 38249, 38240, 38238, 38245, 38248, 38239, 38243, 38257, + 38259, 38256, 38255, 38252, 38251, 38254, 38253, 38260, 38258, 38121, + 38118, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 10283, 10484, 10172, 10331, 10122, 10425, 10227, 10386, 10168, + 10327, 10248, 10411, 10156, 10315, 10115, 10412, 10276, 10477, 10224, + 10383, 10124, 10427, 10225, 10384, 10164, 10323, 10157, 10316, 10221, + 10380, 10278, 10479, 10123, 10426, 10267, 10445, 10264, 10442, 10265, + 10443, 10247, 10410, 10154, 10313, 10169, 10328, 10298, 8145, 8137, 8088, + 8138, 33776, 8112, 8101, 8115, 8111, 8120, 8113, 8110, 8107, 8143, 8132, + 10297, 10301, 10178, 10337, 10176, 10335, 10288, 10489, 10166, 10325, + 10174, 10333, 10128, 10453, 10136, 10462, 10132, 10457, 10131, 10456, + 10134, 10460, 10212, 10371, 10262, 10440, 10170, 10329, 10165, 10324, + 27730, 27767, 8096, 8104, 3413, 2779, 3416, 2781, 3412, 3388, 3405, 3415, + 2808, 3414, 2784, 3385, 3391, 3390, 2782, 2788, 3404, 2807, 2801, 2787, + 3400, 2795, 3395, 3401, 3394, 3398, 2777, 2773, 2805, 2804, 2799, 3407, + 3397, 3408, 3409, 2806, 2771, 3381, 2800, 2803, 3384, 3410, 3382, 2768, + 3393, 2786, 2793, 2810, 3387, 3392, 2772, 2796, 2797, 2798, 3396, 3383, + 2770, 2769, 3411, 2809, 2785, 3386, 2783, 2774, 2790, 3389, 2789, 2792, + 3406, 2780, 2794, 2791, 3403, 2778, 3402, 2802, 3399, 2767, 2775, 2776, + 2764, 2763, 3417, 3419, 2766, 2765, 3418, 3420, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 27727, 27723, 27726, 27722, 27728, + 27724, 27729, 27725, 27782, 27797, 27825, 27804, 27787, 27781, 27796, + 27824, 27803, 27786, 27783, 27798, 27826, 27808, 27788, 27774, 27773, + 27775, 27819, 27838, 27835, 27837, 27836, 27816, 27986, 27987, 22867, + 23470, 22868, 23471, 22907, 23532, 23133, 23894, 23132, 23851, 22809, + 23392, 22810, 23393, 23274, 23282, 22783, 23348, 22784, 23349, 22785, + 23350, 22781, 23346, 22782, 23347, 22786, 23351, 23077, 23775, 22948, + 23584, 22945, 23581, 22949, 23585, 22795, 23369, 22967, 23608, 23000, + 23683, 23001, 23685, 23047, 23723, 23052, 23728, 23050, 23726, 23053, + 23729, 23059, 23740, 23058, 23737, 23074, 23765, 23079, 23778, 23174, + 23944, 23183, 23956, 23178, 23951, 23127, 23845, 23128, 23846, 23182, + 23955, 22869, 23488, 22936, 23571, 22811, 23394, 27995, 23430, 23629, + 23643, 23666, 23777, 23259, 23889, 23941, 22929, 23560, 22930, 23561, + 22931, 23117, 23857, 23122, 23887, 22932, 23563, 22933, 23564, 22934, + 23565, 27802, 27771, 27848, 23100, 23801, 23120, 23613, 23290, 22993, + 23657, 22805, 23382, 23379, 23526, 22789, 23354, 22876, 23495, 23179, + 23952, 23180, 23953, 23181, 23954, 22884, 23506, 22952, 23589, 22996, + 23662, 23072, 23762, 23096, 23799, 22903, 23076, 23103, 22955, 23099, + 23281, 23121, 23124, 22939, 22812, 22796, 23371, 23045, 23721, 23170, + 23932, 22889, 23511, 22890, 23512, 22891, 23513, 23042, 23712, 22778, + 23343, 22803, 23097, 23220, 22816, 23396, 23093, 23793, 23080, 23092, + 23792, 40951, 40951, 22808, 23388, 40951, 23423, 40951, 23420, 22982, + 23642, 23102, 23819, 22971, 23618, 22972, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 27703, 27706, 27715, + 23075, 23766, 23230, 27698, 27927, 23264, 23228, 23229, 23226, 23227, + 23225, 34770, 34772, 34782, 34774, 34771, 34773, 34781, 34762, 34761, + 34758, 34757, 34780, 34756, 34755, 34760, 34759, 34750, 34749, 34744, + 34743, 34752, 34751, 34746, 34745, 34768, 34764, 34763, 34754, 34753, + 34767, 34748, 34766, 34747, 34769, 34765, 34784, 34786, 34787, 34785, + 34783, 34775, 34776, 34777, 34778, 34779, 40951, 40951, 40951, 29388, + 29387, 29390, 29385, 29386, 29389, 29382, 29381, 29384, 29383, 40951, + 40951, 40951, 40951, 40951, 40951, 31352, 31351, 31340, 31341, 31328, + 31330, 31362, 31343, 31350, 31349, 31333, 31344, 31354, 31353, 31359, + 31364, 31346, 31345, 31332, 31367, 31355, 31356, 31334, 31369, 31366, + 31363, 31335, 31336, 31361, 31325, 31370, 31372, 31357, 31371, 31365, + 31368, 31360, 31339, 31358, 31377, 31378, 31348, 31347, 31331, 31342, + 31326, 31338, 31337, 31327, 31376, 31379, 31329, 31375, 31380, 31374, + 31373, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 32496, + 32498, 32445, 32446, 32466, 32467, 32462, 32463, 32457, 32458, 32459, + 32460, 32489, 32490, 32447, 32464, 32465, 32448, 32486, 32485, 32482, + 32481, 32470, 32480, 32479, 32484, 32483, 32472, 32454, 32453, 32450, + 32449, 32471, 32456, 32455, 32452, 32451, 32473, 32488, 32487, 32478, + 32477, 32492, 32494, 32493, 32469, 32461, 32474, 32475, 32476, 32491, + 32468, 32511, 32512, 32523, 32524, 32515, 32516, 32517, 32518, 32519, + 32520, 32525, 32526, 32513, 32521, 32522, 32514, 32497, 32495, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 32500, 32499, 32507, + 32509, 32506, 32505, 32502, 32501, 32504, 32503, 32510, 32508, 40951, + 40951, 40951, 40951, 40951, 40951, 8163, 8165, 8162, 8161, 8158, 8157, + 8160, 8159, 8166, 8164, 8154, 8155, 8150, 8151, 8152, 8153, 8149, 8156, + 10770, 10764, 10777, 10766, 10765, 10763, 10778, 10667, 10665, 10670, + 10771, 10821, 10676, 10788, 21585, 21587, 21584, 21583, 21580, 21579, + 21582, 21581, 21588, 21586, 21549, 21548, 21559, 21545, 21553, 21552, + 21566, 21546, 21555, 21543, 21547, 21551, 21550, 21561, 21558, 21556, + 21562, 21565, 21560, 21564, 21554, 21544, 21563, 21557, 21567, 21541, + 21568, 21542, 21577, 21574, 21576, 21575, 21578, 21573, 21571, 21572, + 21569, 21570, 31838, 31835, 31827, 31843, 31834, 31829, 31840, 31832, + 31831, 31833, 31837, 31825, 31842, 31841, 31839, 31845, 31844, 31836, + 31830, 31826, 31828, 31824, 31846, 31853, 31855, 31848, 31851, 31854, + 31852, 31850, 31849, 31821, 31820, 31823, 31822, 31856, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 31847, + 19228, 19231, 19232, 19229, 19186, 19187, 19195, 19189, 19191, 19194, + 19188, 19184, 19190, 19192, 19185, 19151, 19153, 19154, 19167, 19174, + 19181, 19216, 19133, 19140, 19214, 19218, 19164, 19237, 19220, 40951, + 40951, 40951, 20903, 20901, 20905, 20904, 20875, 20843, 20842, 20844, + 20884, 20863, 20849, 20850, 20882, 20876, 20883, 20845, 20846, 20847, + 20859, 20860, 20848, 20857, 20858, 20873, 20852, 20874, 20851, 20871, + 20872, 20838, 20839, 20854, 20869, 20870, 20840, 20841, 20853, 20861, + 20862, 20855, 20856, 20879, 20881, 20864, 20865, 20878, 20880, 20867, + 20868, 20866, 20877, 20902, 20908, 20910, 20911, 20912, 20906, 20907, + 20909, 20914, 20913, 20834, 20835, 20836, 20899, 20837, 20885, 20888, + 20897, 20890, 20895, 20893, 20891, 20889, 20886, 20887, 20892, 20900, + 40951, 20898, 20921, 20923, 20920, 20919, 20916, 20915, 20918, 20917, + 20924, 20922, 40951, 40951, 40951, 40951, 20894, 20896, 28607, 28605, + 28600, 28597, 28603, 28693, 28671, 28623, 28635, 28631, 28630, 28633, + 28632, 28625, 28624, 28622, 28735, 28737, 28734, 28733, 28730, 28729, + 28732, 28731, 28738, 28736, 28634, 28627, 28626, 28629, 28628, 40951, + 6245, 6263, 6265, 6262, 6246, 6264, 6254, 6253, 6250, 6249, 6225, 6226, + 6248, 6247, 6252, 6251, 6227, 6229, 6228, 6256, 6255, 6242, 6241, 6230, + 6231, 6240, 6236, 6235, 6234, 6239, 6238, 6232, 6233, 6237, 6261, 6259, + 6258, 6260, 6243, 6244, 6257, 6270, 6273, 6274, 6279, 6277, 6276, 6275, + 6271, 6272, 6278, 6213, 6211, 6210, 6212, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 6219, 6218, 6215, 6207, 6217, 6223, + 6214, 6221, 6224, 6222, 6220, 6216, 6209, 6208, 40951, 40951, 6286, 6288, + 6285, 6284, 6281, 6280, 6283, 6282, 6289, 6287, 40951, 40951, 6266, 6268, + 6267, 6269, 28577, 28570, 28569, 28574, 28573, 28567, 28566, 28565, + 28563, 28562, 28564, 28568, 28579, 28572, 28571, 28576, 28670, 28580, + 28581, 28578, 28667, 28668, 28669, 28708, 28710, 28709, 28552, 28704, + 28695, 28696, 28616, 28617, 35254, 35230, 35253, 35229, 35252, 35228, + 35267, 35243, 35261, 35237, 35256, 35232, 35255, 35231, 35272, 35248, + 35262, 35238, 35265, 35241, 35260, 35236, 35259, 35235, 35263, 35239, + 35264, 35240, 35258, 35234, 35257, 35233, 35266, 35242, 35270, 35246, + 35274, 35250, 35271, 35247, 35269, 35245, 35273, 35249, 35268, 35244, + 35275, 35251, 35276, 35288, 35296, 35293, 35292, 35298, 35299, 35277, + 35297, 35294, 35295, 35287, 35291, 35290, 35289, 35286, 35283, 35284, + 35285, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 35279, 35280, 35282, 35281, 35278, + 27037, 27038, 26996, 27013, 27024, 27023, 27000, 26999, 27012, 27018, + 27019, 27051, 27053, 27043, 27045, 27044, 26991, 26988, 26990, 27040, + 27041, 27055, 27056, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 17212, 17210, 17209, 17208, 17207, 17211, 40951, + 40951, 16906, 16904, 16903, 16902, 16901, 16905, 40951, 40951, 16921, + 16919, 16918, 16917, 16916, 16920, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 16862, 16868, 16866, 16863, 16865, 16864, + 16867, 40951, 16847, 16853, 16851, 16848, 16850, 16849, 16852, 40951, + 23362, 23338, 23368, 23363, 23459, 23622, 23811, 23611, 23602, 23605, + 23633, 23647, 23473, 23366, 23367, 23717, 23567, 23860, 23859, 23861, + 23862, 23821, 23258, 23764, 23421, 23744, 23422, 23808, 23809, 23365, + 23931, 23897, 23940, 23883, 23936, 23385, 23386, 23387, 23972, 23969, + 23970, 23971, 23983, 27685, 27908, 27917, 27918, 27975, 23802, 23570, + 23718, 23942, 23566, 18401, 23428, 23892, 23865, 27972, 27821, 27845, + 40951, 40951, 40951, 40951, 6468, 6469, 6470, 6471, 6472, 6473, 6431, + 6467, 6432, 6433, 6434, 6435, 6436, 6396, 6397, 6398, 6399, 6400, 6401, + 6437, 6438, 6439, 6440, 6441, 6442, 6443, 6444, 6445, 6446, 6447, 6402, + 6395, 6403, 6404, 6405, 6406, 6407, 6408, 6449, 6450, 6451, 6452, 6453, + 6454, 6410, 6409, 6411, 6412, 6413, 6414, 6415, 6389, 6428, 6390, 6429, + 6391, 6430, 6392, 6393, 6394, 6388, 6416, 6417, 6418, 6419, 6420, 6421, + 6422, 6423, 6424, 6425, 6426, 6427, 6455, 6456, 6457, 6458, 6459, 6460, + 6461, 27005, 27017, 27027, 27029, 27014, 27008, 26995, 27020, 27007, + 27010, 27022, 27033, 27035, 27031, 27036, 27025, 27016, 27034, 27002, + 27003, 27032, 26994, 27004, 26998, 27001, 26997, 26993, 27006, 27028, + 27030, 27015, 27009, 27021, 27011, 27026, 27047, 27050, 27042, 27049, + 27048, 27052, 27046, 27054, 26992, 27039, 26989, 40951, 40951, 27063, + 27065, 27062, 27061, 27058, 27057, 27060, 27059, 27066, 27064, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 19425, 19423, 19456, 19457, 19458, 19434, 19435, 19467, 19469, + 19402, 19400, 19399, 19403, 19409, 19410, 19412, 19411, 19416, 19413, + 19414, 19418, 19386, 19384, 40951, 40951, 40951, 40951, 19282, 19283, + 19351, 19352, 19371, 19365, 19366, 19369, 19368, 19367, 19326, 19311, + 19350, 19317, 19321, 19320, 19334, 19333, 19254, 19279, 19272, 19357, + 19270, 19277, 19307, 19300, 19306, 19361, 19302, 19305, 19304, 19345, + 19338, 19355, 19356, 19344, 19342, 19341, 19346, 19348, 19293, 19292, + 19378, 19379, 19243, 19242, 19358, 19297, 19295, 40951, 40951, 40951, + 40951, 7582, 7583, 7584, 7585, 7586, 7587, 7588, 7589, 7590, 7591, 7592, + 7593, 7594, 7595, 7596, 7597, 7598, 7599, 7600, 7601, 7602, 7603, 7604, + 7605, 7606, 7607, 7608, 7609, 7610, 7611, 7612, 7613, 7614, 7615, 7616, + 7617, 7618, 7619, 7620, 7621, 7622, 7623, 7624, 7625, 7626, 7627, 7628, + 7629, 7630, 7631, 7632, 7633, 7634, 7635, 7636, 7637, 7638, 7639, 7640, + 7641, 7642, 7643, 7644, 7645, 7646, 7647, 7648, 7649, 7650, 7651, 7652, + 7653, 7654, 7655, 7656, 7657, 7658, 7659, 7660, 7661, 7662, 7663, 7664, + 7665, 7666, 7667, 7668, 7669, 7670, 7671, 7672, 7673, 7674, 7675, 7676, + 7677, 7678, 7679, 7680, 7681, 7682, 7683, 7684, 7685, 7686, 7687, 7688, + 7689, 7690, 7691, 7692, 7693, 7694, 7695, 7696, 7697, 7698, 7699, 7700, + 7701, 7702, 7703, 7704, 7705, 7706, 7707, 7708, 7709, 7710, 7711, 7712, + 7713, 7714, 7715, 7716, 7717, 7718, 7719, 7720, 7721, 7722, 7723, 7724, + 7725, 7726, 7727, 7728, 7729, 7730, 7731, 7732, 7733, 7734, 7735, 7736, + 7737, 7738, 7739, 7740, 7741, 7742, 7743, 7744, 7745, 7746, 7747, 7748, + 7749, 7750, 7751, 7752, 7753, 7754, 7755, 7756, 7757, 7758, 7759, 7760, + 7761, 7762, 7763, 7764, 7765, 7766, 7767, 7768, 7769, 7770, 7771, 7772, + 7773, 7774, 7775, 7776, 7777, 7778, 7779, 7780, 7781, 7782, 7783, 7784, + 7785, 7786, 7787, 7788, 7789, 7790, 7791, 7792, 7793, 7794, 7795, 7796, + 7797, 7798, 7799, 7800, 7801, 7802, 7803, 7804, 7805, 7806, 7807, 7808, + 7809, 7810, 7811, 7812, 7813, 7814, 7815, 7816, 7817, 7818, 7819, 7820, + 7821, 7822, 7823, 7824, 7825, 7826, 7827, 7828, 7829, 7830, 7831, 7832, + 7833, 7834, 7835, 7836, 7837, 7380, 7381, 7382, 7383, 7384, 7385, 7386, + 7387, 7388, 7389, 7390, 7391, 7392, 7393, 7394, 7395, 7396, 7397, 7398, + 7399, 7400, 7401, 7402, 7403, 7404, 7405, 7406, 7407, 7408, 7409, 7410, + 7411, 7412, 7413, 7414, 7415, 7416, 7417, 7418, 7419, 7420, 7421, 7422, + 7423, 7424, 7425, 7426, 7427, 7428, 7429, 7430, 7431, 7432, 7433, 7434, + 7435, 7436, 7437, 7438, 7439, 7440, 7441, 7442, 7443, 7444, 7445, 7446, + 7447, 7448, 7449, 7450, 7451, 7452, 7453, 7454, 7455, 7456, 7457, 7458, + 7459, 7460, 7461, 7462, 7463, 7464, 7465, 7466, 7467, 7468, 7469, 7470, + 7471, 7472, 7473, 7474, 7475, 7366, 7367, 7368, 7369, 7370, 7371, 7372, + 7373, 7374, 7375, 7376, 7377, 7378, 7379, 40951, 40951, 7476, 7477, 7478, + 7479, 7480, 7481, 7482, 7483, 7484, 7485, 7486, 7487, 7488, 7489, 7490, + 7491, 7492, 7493, 7494, 7495, 7496, 7497, 7498, 7499, 7500, 7501, 7502, + 7503, 7504, 7505, 7506, 7507, 7508, 7509, 7510, 7511, 7512, 7513, 7514, + 7515, 7516, 7517, 7518, 7519, 7520, 7521, 7522, 7523, 7524, 7525, 7526, + 7527, 7528, 7529, 7530, 7531, 7532, 7533, 7534, 7535, 7536, 7537, 7538, + 7539, 7540, 7541, 7542, 7543, 7544, 7545, 7546, 7547, 7548, 7549, 7550, + 7551, 7552, 7553, 7554, 7555, 7556, 7557, 7558, 7559, 7560, 7561, 7562, + 7563, 7564, 7565, 7566, 7567, 7568, 7569, 7570, 7571, 7572, 7573, 7574, + 7575, 7576, 7577, 7578, 7579, 7580, 7581, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 24005, 24008, 24009, 24006, 24007, 24011, + 24012, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 2506, 2507, 2505, 2508, 2504, 40951, 40951, 40951, + 40951, 40951, 19891, 19918, 19896, 19827, 19883, 19886, 19888, 19887, + 19882, 19889, 19885, 19884, 19828, 19861, 19862, 19859, 19860, 19825, + 19826, 19824, 19832, 19874, 19872, 19849, 19881, 19854, 40951, 19866, + 19892, 19840, 19835, 19876, 40951, 19878, 40951, 19852, 19856, 40951, + 19842, 19838, 40951, 19870, 19847, 19864, 19858, 19868, 19880, 19831, + 19834, 19837, 19893, 1158, 1157, 1188, 1186, 1187, 1189, 1417, 1415, + 1416, 1418, 1183, 1181, 1182, 1184, 1547, 1545, 1546, 1548, 1527, 1525, + 1526, 1528, 1542, 1540, 1541, 1543, 1560, 1558, 1559, 1561, 1422, 1420, + 1421, 1423, 1224, 1222, 1223, 1225, 1395, 1393, 1394, 1396, 1504, 1502, + 1503, 1505, 1509, 1507, 1508, 1510, 1214, 1213, 1205, 1204, 1228, 1227, + 1217, 1216, 1324, 1323, 1452, 1451, 1347, 1345, 1346, 1348, 1258, 1256, + 1257, 1259, 1270, 1268, 1269, 1271, 1386, 1384, 1385, 1387, 1400, 1399, + 1456, 1454, 1455, 1457, 1300, 1299, 1295, 1293, 1294, 1296, 1304, 1302, + 1303, 1305, 1584, 1583, 1582, 1581, 2368, 2367, 2376, 2375, 2372, 2371, + 2370, 2369, 2380, 2379, 2366, 2374, 2373, 2382, 2378, 2377, 2381, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 1390, 1388, 1389, 1391, 1553, 1552, + 1605, 1604, 1602, 1601, 1551, 1563, 1562, 1351, 1350, 1354, 1353, 1618, + 1616, 1617, 1619, 1554, 1555, 2058, 2057, 2060, 2059, 2079, 2078, 2087, + 2086, 2077, 2076, 2083, 2082, 2063, 2061, 2062, 2112, 2110, 2111, 1238, + 1236, 1237, 1239, 2069, 2067, 2073, 2056, 2081, 1660, 1651, 1656, 1663, + 1658, 1669, 2021, 2009, 2016, 2029, 2032, 2037, 2039, 2044, 2041, 2050, + 1738, 1744, 1723, 1718, 1777, 1773, 1779, 1928, 1921, 1933, 1941, 1900, + 1904, 1682, 1674, 1678, 1684, 2002, 1997, 2114, 1631, 1627, 1712, 1708, + 1696, 1701, 1692, 1699, 1694, 1703, 1882, 1878, 1880, 1884, 1753, 1755, + 1763, 1761, 1758, 1769, 1751, 1772, 1805, 1797, 1809, 1815, 1789, 1818, + 1836, 1825, 1830, 1840, 1819, 1841, 1857, 1847, 1869, 1862, 1865, 1872, + 1733, 1729, 1730, 1731, 2097, 2090, 2099, 2105, 2054, 2109, 2038, 1895, + 1643, 1949, 1952, 1956, 1950, 1953, 1955, 2085, 2084, 2071, 2075, 2055, + 2080, 1667, 1666, 1661, 1665, 1657, 1668, 2035, 2034, 2027, 2033, 2031, + 2036, 2048, 2047, 2042, 2046, 2040, 2049, 1693, 1702, 1879, 1883, 1752, + 1756, 1767, 1750, 1771, 1813, 1788, 1817, 1820, 1838, 1870, 1867, 1860, + 1866, 1864, 1871, 1642, 2107, 2094, 2103, 2093, 2053, 2108, 2068, 2066, + 2070, 2072, 2064, 1659, 1650, 1655, 1662, 1652, 2020, 2008, 2015, 2028, + 2010, 2043, 1737, 1743, 1722, 1717, 1776, 1778, 1927, 1920, 1932, 1940, + 1899, 1907, 1903, 1681, 1673, 1677, 1683, 2001, 2113, 1630, 1626, 1711, + 1707, 1695, 1700, 1691, 1698, 1881, 1877, 1754, 1762, 1760, 1757, 1768, + 1804, 1796, 1808, 1814, 1798, 1835, 1824, 1829, 1839, 1856, 1846, 1868, + 1861, 1848, 1732, 1728, 1734, 2096, 2089, 2098, 2104, 2091, 2074, 2065, + 1664, 1653, 2030, 2011, 2045, 2051, 1942, 1924, 1979, 1959, 1759, 1770, + 1816, 1863, 1849, 2106, 2092, 1957, 1951, 1954, 2000, 2004, 1633, 1635, + 1710, 1714, 1944, 1948, 1981, 1989, 1720, 1725, 1746, 1748, 1775, 1781, + 1906, 1911, 1680, 1688, 1970, 1965, 1984, 1978, 1987, 1946, 1909, 1686, + 1999, 2003, 1632, 1634, 1709, 1713, 1943, 1947, 1980, 1988, 1719, 1724, + 1745, 1747, 1774, 1780, 1905, 1910, 1679, 1687, 1968, 1963, 1982, 1976, + 1986, 1945, 1908, 1685, 1969, 1964, 1983, 1977, 1923, 1958, 1996, 1929, + 1922, 1934, 1971, 1966, 1985, 1998, 2115, 1644, 1645, 30650, 30651, 1892, + 1887, 1891, 1888, 1889, 1890, 1918, 1637, 1639, 1638, 1636, 1886, 1917, + 1640, 1991, 1893, 2018, 2007, 2006, 2005, 2013, 2023, 2025, 2024, 1740, + 1739, 1716, 1715, 1919, 1926, 1925, 1936, 1935, 1937, 1939, 1938, 1897, + 1896, 1902, 1961, 1960, 1967, 1973, 1972, 1975, 1974, 1671, 1676, 1675, + 1993, 1992, 1994, 1995, 1629, 1624, 1623, 1622, 1704, 1706, 1705, 1690, + 1689, 1875, 1873, 1794, 1795, 1792, 1799, 1800, 1807, 1806, 1811, 1810, + 1821, 1822, 1823, 1833, 1831, 1826, 1827, 40951, 40951, 1832, 1726, 1727, + 1844, 1843, 1854, 1853, 1852, 1859, 1858, 2101, 2100, 1654, 2019, 2017, + 2014, 2012, 2026, 2022, 1742, 1735, 1741, 1930, 1898, 1962, 1672, 1803, + 1812, 2088, 2095, 2102, 1837, 1876, 1845, 1874, 1793, 1625, 1766, 1850, + 1828, 1801, 1765, 1802, 1851, 1736, 1721, 1834, 1697, 1649, 1764, 1628, + 1901, 1931, 1855, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 1913, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 1914, 1885, 1646, 1647, 1842, 1912, 1894, 1641, 2052, 1915, + 1916, 1749, 32167, 1670, 1990, 1648, 38261, 38372, 38440, 38451, 38462, + 38473, 38484, 38495, 38506, 38262, 38273, 38284, 38295, 38306, 38317, + 38328, 31622, 31627, 31628, 31621, 31652, 31623, 31654, 31630, 31644, + 31626, 40951, 40951, 40951, 40951, 40951, 40951, 8360, 8362, 8192, 8193, + 8371, 8373, 8086, 8361, 8363, 8429, 8430, 8372, 8374, 8087, 8140, 8141, + 31653, 31624, 31625, 31639, 31651, 31636, 31648, 31634, 31646, 31638, + 31650, 31631, 31640, 31632, 31641, 31635, 31647, 31633, 31645, 31629, + 31642, 32661, 39263, 31637, 31649, 10541, 6129, 39141, 11256, 10540, + 6128, 39139, 33800, 33809, 33844, 40951, 33834, 33801, 33845, 33807, + 33808, 33811, 33815, 33810, 33814, 33812, 33816, 33843, 33791, 33793, + 33842, 33840, 33813, 33839, 33806, 40951, 33833, 33802, 33841, 33799, + 40951, 40951, 40951, 40951, 1092, 2387, 1074, 2389, 1104, 40951, 1089, + 1090, 1070, 1071, 1101, 1102, 2294, 2295, 2362, 2363, 1289, 1153, 1152, + 1143, 1142, 1571, 1570, 1146, 1145, 1588, 1586, 1587, 1589, 1163, 1162, + 1178, 1176, 1177, 1179, 1514, 1513, 1523, 1521, 1522, 1516, 1535, 1533, + 1534, 1536, 1320, 1318, 1319, 1321, 1286, 1284, 1285, 1287, 1358, 1356, + 1357, 1359, 1202, 1201, 1531, 1530, 1449, 1448, 1613, 1612, 1478, 1476, + 1477, 1479, 1484, 1482, 1483, 1485, 1465, 1463, 1464, 1466, 1210, 1208, + 1209, 1211, 1497, 1495, 1496, 1498, 1609, 1607, 1608, 1610, 1121, 1119, + 1120, 1122, 1265, 1263, 1264, 1266, 1249, 1247, 1248, 1250, 1431, 1429, + 1430, 1432, 1334, 1332, 1333, 1335, 1370, 1368, 1369, 1371, 1379, 1377, + 1378, 1380, 1410, 1408, 1409, 1411, 1308, 1306, 1307, 1309, 1575, 1574, + 1161, 1160, 1598, 1596, 1597, 1599, 1787, 1786, 1783, 1782, 1785, 1784, + 1791, 1790, 40951, 40951, 40755, 40951, 17625, 17629, 17597, 17613, + 17599, 17611, 17610, 17540, 17603, 17612, 17600, 17535, 17628, 17633, + 17607, 17620, 17622, 17619, 17618, 17615, 17614, 17617, 17616, 17623, + 17621, 17536, 17606, 17542, 17624, 17627, 17630, 17534, 17543, 17544, + 17545, 17546, 17547, 17548, 17549, 17550, 17551, 17552, 17553, 17554, + 17555, 17556, 17557, 17558, 17559, 17560, 17561, 17562, 17563, 17564, + 17565, 17566, 17567, 17568, 17541, 17605, 17604, 17533, 17595, 17626, + 17569, 17570, 17571, 17572, 17573, 17574, 17575, 17576, 17577, 17578, + 17579, 17580, 17581, 17582, 17583, 17584, 17585, 17586, 17587, 17588, + 17589, 17590, 17591, 17592, 17593, 17594, 17539, 17635, 17602, 17632, + 17538, 17601, 19098, 19091, 19093, 19097, 19089, 19081, 19036, 19038, + 19040, 19037, 19039, 19032, 19034, 19033, 19035, 19090, 19082, 19084, + 19086, 19083, 19085, 19057, 19059, 19061, 19058, 19060, 19041, 19043, + 19045, 19042, 19044, 19072, 19074, 19076, 19073, 19075, 19047, 19049, + 19051, 19048, 19050, 19052, 19054, 19056, 19053, 19055, 19062, 19064, + 19066, 19063, 19065, 19077, 19079, 19078, 19067, 19069, 19071, 19068, + 19070, 19080, 19046, 19088, 19087, 19031, 18981, 18998, 18982, 18983, + 18984, 18985, 19019, 19000, 18989, 18994, 18993, 18992, 18996, 18990, + 18991, 18995, 19017, 18987, 18999, 18988, 19002, 19001, 19016, 19011, + 18997, 19010, 18980, 19018, 18986, 19025, 40951, 40951, 40951, 19026, + 19027, 19005, 19006, 19013, 19012, 40951, 40951, 19004, 19003, 19028, + 19022, 19023, 19029, 40951, 40951, 19008, 19030, 19021, 19020, 19024, + 19009, 40951, 40951, 19014, 19007, 19015, 40951, 40951, 40951, 17537, + 17598, 17596, 17609, 17634, 17608, 17631, 40951, 19095, 19092, 19096, + 19094, 19101, 19099, 19100, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 20788, 20790, 20789, 29933, 31859, 40951, + 40951, 24968, 24994, 24987, 25020, 24978, 24969, 24998, 24964, 24976, + 25006, 25009, 25003, 40951, 24992, 25019, 25023, 25002, 25017, 25024, + 25031, 25034, 24979, 25030, 24977, 24980, 24963, 24986, 24989, 25012, + 25013, 24971, 25028, 24993, 24974, 25010, 24972, 25029, 24988, 24996, + 40951, 25021, 24985, 25005, 24970, 24981, 24995, 24966, 25000, 24975, + 25007, 25008, 24965, 24991, 24967, 25018, 25011, 25025, 24999, 25001, + 40951, 24973, 25027, 40951, 24984, 24983, 24997, 25033, 25026, 25036, + 25004, 24982, 25014, 25022, 24990, 25015, 25016, 25032, 25035, 40951, + 40951, 25046, 25047, 25049, 25048, 25037, 25038, 25045, 25039, 25040, + 25050, 25041, 25042, 25043, 25044, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 24852, + 24851, 24853, 24840, 24841, 24842, 24843, 24844, 24845, 24846, 24848, + 24847, 24850, 24849, 24858, 24855, 24856, 24854, 24857, 24957, 24958, + 24860, 24859, 24861, 24960, 24959, 24863, 24864, 24865, 24862, 24866, + 24869, 24868, 24870, 24871, 24872, 24961, 24873, 24874, 24867, 24877, + 24878, 24876, 24875, 24879, 24880, 24881, 24882, 24883, 24884, 24887, + 24888, 24889, 24886, 24890, 24885, 24891, 24892, 24893, 24894, 24895, + 24896, 24897, 24898, 24899, 24900, 24902, 24901, 24903, 24904, 24906, + 24907, 24908, 24905, 24909, 24910, 24911, 24912, 24914, 24913, 24915, + 24916, 24962, 24917, 24918, 24920, 24921, 24922, 24919, 24923, 24924, + 24925, 24926, 24927, 24955, 24935, 24936, 24937, 24938, 24939, 24940, + 24941, 24942, 24943, 24944, 24945, 24946, 24947, 24948, 24949, 24950, + 24951, 24952, 24953, 24954, 24928, 24929, 24930, 24931, 24932, 24933, + 24934, 24956, 40951, 40951, 40951, 40951, 40951, 112, 113, 159, 40951, + 40951, 40951, 40951, 156, 151, 146, 126, 121, 139, 134, 114, 129, 154, + 149, 144, 124, 119, 140, 135, 115, 130, 157, 152, 147, 127, 122, 142, + 137, 117, 132, 158, 153, 148, 128, 123, 143, 138, 118, 133, 155, 150, + 145, 125, 120, 141, 136, 116, 131, 40951, 40951, 40951, 111, 107, 109, + 110, 108, 103, 104, 105, 106, 18166, 18163, 18167, 18153, 18148, 18154, + 18157, 18149, 18159, 18168, 18151, 18161, 18155, 18164, 18158, 18160, + 18170, 18152, 18162, 18156, 18165, 18169, 18150, 18171, 18183, 18192, + 18182, 18178, 18191, 18173, 18179, 18195, 18199, 18200, 18181, 18184, + 18190, 18189, 18197, 18198, 18180, 18187, 18193, 18188, 18177, 18196, + 18185, 18172, 18174, 18194, 18186, 18175, 18176, 18418, 18419, 18617, + 18613, 18654, 18619, 18349, 18423, 18616, 18612, 18355, 18354, 18415, + 18397, 18413, 18422, 18417, 18203, 18202, 18656, 18615, 18657, 18420, + 18611, 18393, 29360, 40951, 32211, 32214, 32209, 32212, 32183, 32213, + 32181, 32184, 32210, 32182, 32215, 32180, 2526, 40951, 40951, 40951, + 18610, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 31405, 31406, + 31417, 31386, 31389, 31420, 31398, 31397, 31418, 31425, 31384, 31411, + 31390, 31403, 31404, 31413, 31402, 31383, 31387, 31394, 31392, 31414, + 31391, 31382, 31412, 31399, 31400, 31385, 31388, 31410, 31424, 31395, + 31419, 31381, 31407, 31426, 31408, 31409, 31401, 31423, 31422, 31396, + 31415, 31416, 31421, 31393, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 25283, 25285, 25278, 25279, 25290, 25289, + 25292, 25300, 25302, 25281, 25293, 25276, 25296, 25294, 25274, 25287, + 25275, 25288, 25299, 25295, 25277, 25297, 25298, 25280, 25282, 25284, + 25286, 25291, 25301, 40951, 40951, 40951, 6039, 6050, 6041, 6012, 6035, + 6051, 6013, 6040, 6057, 6055, 6015, 6056, 6042, 6030, 6025, 6026, 6024, + 6010, 6033, 6023, 6058, 6020, 6032, 6049, 6029, 6053, 6043, 6038, 6047, + 6048, 6021, 6034, 6045, 6046, 6027, 6028, 6022, 6054, 6011, 6031, 6036, + 6052, 6016, 6017, 6018, 6019, 6014, 6044, 6037, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 8571, 8569, 8567, 8566, 8563, 8562, 8565, 8564, 8570, 8568, + 8561, 8560, 8558, 8549, 8547, 8556, 8554, 8545, 8551, 8552, 8559, 8557, + 8548, 8546, 8555, 8553, 8544, 8550, 40951, 40951, 40951, 40951, 30215, + 30209, 30195, 30210, 30182, 30212, 30214, 30211, 30206, 30202, 30194, + 30190, 30191, 30192, 30184, 30216, 30205, 30198, 30196, 30186, 30183, + 30207, 30200, 30188, 30204, 30193, 30189, 30187, 30208, 30203, 30201, + 30185, 30220, 30218, 30219, 30217, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 30213, 30199, 30197, 18013, 18029, 18037, + 18031, 18012, 18022, 18016, 18015, 18024, 18034, 18038, 18035, 18032, + 18020, 18036, 18027, 18021, 18019, 18023, 18033, 18025, 18026, 18028, + 18017, 18014, 18030, 18018, 40951, 40951, 40951, 40951, 40951, 30287, + 30286, 30283, 30256, 30257, 30279, 30254, 30280, 30255, 30259, 30288, + 30281, 30262, 30263, 30270, 30264, 30282, 30267, 30269, 30290, 30253, + 30266, 30265, 30277, 30274, 30284, 30285, 30258, 30289, 30268, 30271, + 30272, 30273, 30276, 30261, 30278, 30275, 30260, 8386, 8384, 8382, 8383, + 8385, 40951, 40951, 40951, 40951, 40951, 37709, 37712, 37716, 37720, + 37713, 37717, 37735, 37731, 37718, 37728, 37730, 37719, 37725, 37721, + 37736, 37714, 37733, 37732, 37724, 37710, 37734, 37723, 37711, 37722, + 37727, 37715, 37729, 37737, 37738, 37726, 40951, 37739, 30296, 30338, + 30339, 30319, 30320, 30317, 30318, 30316, 30331, 30308, 30309, 30313, + 30314, 30303, 30306, 30307, 30312, 30335, 30300, 30332, 30321, 30322, + 30325, 30326, 30327, 30336, 30310, 30311, 30323, 30324, 30334, 30330, + 30337, 30328, 30329, 30333, 40951, 40951, 40951, 40951, 30297, 30298, + 30299, 30315, 30304, 30305, 30301, 30302, 30340, 30295, 30292, 30293, + 30291, 30294, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 10594, 10593, 10589, 10590, 10591, 10592, + 10600, 10599, 10595, 10596, 10597, 10598, 10617, 10602, 10616, 10613, + 10618, 10611, 10608, 10604, 10609, 10607, 10610, 10615, 10614, 10584, + 10612, 10583, 10603, 10579, 10606, 10580, 10605, 10587, 10585, 10586, + 10581, 10582, 10601, 10588, 10634, 10633, 10629, 10630, 10631, 10632, + 10640, 10639, 10635, 10636, 10637, 10638, 10657, 10642, 10656, 10653, + 10658, 10651, 10648, 10644, 10649, 10647, 10650, 10655, 10654, 10624, + 10652, 10623, 10643, 10619, 10646, 10620, 10645, 10627, 10625, 10626, + 10621, 10622, 10641, 10628, 32797, 32803, 32814, 32811, 32801, 32800, + 32799, 32778, 32806, 32784, 32813, 32809, 32812, 32815, 32802, 32810, + 32789, 32808, 32805, 32783, 32788, 32790, 32787, 32782, 32776, 32773, + 32795, 32804, 32793, 32777, 32798, 32816, 32780, 32774, 32786, 32817, + 32794, 32791, 32792, 32775, 32771, 32796, 32772, 32781, 32770, 32779, + 32785, 32807, 30729, 30747, 30753, 30751, 30754, 30735, 30732, 30752, + 30737, 30736, 30733, 30731, 30749, 30748, 30739, 30734, 30742, 30738, + 30743, 30744, 30750, 30755, 30728, 30745, 30756, 30740, 30757, 30730, + 30746, 30741, 40951, 40951, 30764, 30766, 30763, 30762, 30759, 30758, + 30761, 30760, 30767, 30765, 40951, 40951, 40951, 40951, 40951, 40951, + 30656, 30657, 30658, 30659, 30684, 30677, 30663, 30660, 30666, 30676, + 30675, 30690, 30669, 30664, 30668, 30685, 30686, 30687, 30670, 30671, + 30688, 30665, 30681, 30680, 30674, 30662, 30673, 30661, 30672, 30678, + 30691, 30689, 30667, 30679, 30683, 30682, 40951, 40951, 40951, 40951, + 30692, 30693, 30694, 30695, 30720, 30713, 30699, 30696, 30702, 30712, + 30711, 30726, 30705, 30700, 30704, 30721, 30722, 30723, 30706, 30707, + 30724, 30701, 30717, 30716, 30710, 30698, 30709, 30697, 30708, 30714, + 30727, 30725, 30703, 30715, 30719, 30718, 40951, 40951, 40951, 40951, + 16693, 16684, 16673, 16672, 16675, 16664, 16674, 16671, 16670, 16685, + 16661, 16660, 16686, 16694, 16687, 16677, 16663, 16662, 16688, 16667, + 16666, 16665, 16695, 16689, 16690, 16669, 16668, 16679, 16678, 16681, + 16680, 16696, 16691, 16692, 16697, 16683, 16682, 16659, 16658, 16676, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 6071, 6120, 6087, + 6083, 6085, 6110, 6084, 6108, 6105, 6074, 6107, 6109, 6088, 6099, 6095, + 6090, 6118, 6082, 6073, 6091, 6094, 6096, 6121, 6112, 6070, 6076, 6077, + 6079, 6113, 6111, 6116, 6080, 6100, 6092, 6119, 6104, 6115, 6081, 6075, + 6098, 6086, 6117, 6102, 6114, 6103, 6101, 6089, 6078, 6072, 6106, 6097, + 6093, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 6122, 38912, 38881, 38882, 38892, 38891, 38894, 38893, + 38884, 38883, 38901, 38909, 40951, 38900, 38899, 38885, 38886, 38902, + 38910, 38888, 38887, 38903, 38890, 38889, 38913, 38904, 38911, 38905, + 40951, 38896, 38895, 38898, 38897, 38914, 38906, 38907, 40951, 38915, + 38908, 40951, 38947, 38916, 38917, 38927, 38926, 38929, 38928, 38919, + 38918, 38936, 38944, 40951, 38935, 38934, 38920, 38921, 38937, 38945, + 38923, 38922, 38938, 38925, 38924, 38948, 38939, 38946, 38940, 40951, + 38931, 38930, 38933, 38932, 38949, 38941, 38942, 40951, 38950, 38943, + 40951, 40951, 40951, 37427, 37428, 37441, 37401, 37430, 37429, 37432, + 37408, 37431, 37424, 37423, 37442, 37398, 37404, 37397, 37403, 37411, + 37410, 37445, 37399, 37434, 37426, 37425, 37402, 37409, 37405, 37421, + 37413, 37443, 37420, 37419, 37418, 37415, 37414, 37436, 37435, 37446, + 37444, 37438, 37407, 37437, 37406, 37447, 37400, 37440, 37439, 37396, + 37417, 37416, 37433, 37412, 37422, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 24819, 24820, 24821, + 24822, 24823, 24824, 24825, 24826, 24827, 24758, 24759, 24760, 24761, + 24762, 24771, 24763, 24764, 24765, 24766, 24767, 24768, 24769, 24770, + 24772, 24773, 24774, 24775, 24839, 24776, 24777, 24778, 24779, 24780, + 24781, 24782, 24783, 24784, 24785, 24786, 24787, 24788, 24789, 24790, + 24791, 24792, 24793, 24794, 24795, 24796, 24797, 24798, 24799, 24800, + 24801, 24802, 24803, 24804, 24805, 24806, 24807, 24808, 24809, 24810, + 24811, 24812, 24813, 24814, 24815, 24816, 24817, 24818, 24499, 24835, + 24828, 24500, 24829, 24830, 24831, 24832, 24501, 24836, 24837, 24833, + 24834, 24838, 24505, 24506, 24507, 24508, 24509, 24510, 24511, 24512, + 24502, 24503, 24504, 24516, 24517, 24518, 24513, 24514, 24515, 24519, + 24520, 24521, 24522, 24523, 24524, 24527, 24528, 24529, 24530, 24531, + 24532, 24533, 24534, 24535, 24536, 24537, 24538, 24539, 24540, 24541, + 24542, 24543, 24544, 24545, 24546, 24547, 24548, 24549, 24550, 24551, + 24552, 24553, 24554, 24555, 24556, 24557, 24558, 24559, 24560, 24561, + 24562, 24563, 24564, 24565, 24566, 24567, 24568, 24569, 24570, 24571, + 24572, 24573, 24574, 24575, 24576, 24525, 24526, 24577, 24578, 24579, + 24580, 24581, 24582, 24583, 24584, 24585, 24586, 24587, 24588, 24589, + 24590, 24591, 24592, 24593, 24594, 24595, 24596, 24597, 24598, 24599, + 24600, 24601, 24602, 24603, 24604, 24605, 24606, 24607, 24608, 24609, + 24641, 24642, 24643, 24644, 24645, 24646, 24647, 24648, 24649, 24610, + 24611, 24612, 24613, 24614, 24615, 24616, 24617, 24618, 24619, 24620, + 24621, 24622, 24623, 24624, 24625, 24626, 24627, 24628, 24629, 24630, + 24631, 24632, 24633, 24634, 24635, 24636, 24637, 24638, 24639, 24640, + 24656, 24657, 24658, 24659, 24660, 24661, 24662, 24663, 24664, 24665, + 24666, 24667, 24668, 24669, 24670, 24671, 24672, 24673, 24674, 24675, + 24650, 24651, 24652, 24653, 24654, 24655, 24676, 24677, 24678, 24679, + 24680, 24681, 24682, 24683, 24718, 24719, 24720, 24721, 24722, 24723, + 24724, 24725, 24726, 24727, 24684, 24685, 24686, 24687, 24688, 24689, + 24690, 24691, 24692, 24693, 24694, 24695, 24696, 24697, 24698, 24699, + 24700, 24701, 24702, 24703, 24709, 24710, 24711, 24712, 24713, 24714, + 24715, 24716, 24717, 24704, 24705, 24706, 24707, 24708, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 24739, 24734, 24737, + 24736, 24740, 24735, 24733, 24738, 24732, 24728, 24731, 24730, 24729, + 24746, 24741, 24745, 24742, 24743, 24744, 24747, 24749, 24748, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 24750, + 24751, 24752, 24753, 24754, 24755, 24756, 24757, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 27868, 27989, 27988, 27853, 27869, 27857, 40951, 27887, 27889, + 27888, 27883, 27882, 27880, 27881, 27942, 27875, 27899, 27939, 27863, + 27903, 27864, 27907, 27870, 27909, 27886, 27923, 27924, 27922, 27866, + 27920, 27925, 27926, 27967, 27968, 27931, 27867, 27876, 27982, 27964, + 27965, 27938, 27937, 27872, 27952, 27955, 27956, 27953, 27951, 27979, + 40951, 27874, 27794, 27841, 27689, 27772, 27801, 27686, 27843, 27944, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 10008, + 10009, 10010, 10011, 10012, 10002, 40951, 40951, 10003, 40951, 9958, + 9959, 9960, 9961, 9962, 9963, 9964, 9965, 9966, 9967, 9968, 9969, 9970, + 9971, 9972, 9973, 9974, 9975, 9976, 9977, 9978, 9979, 9980, 9981, 9982, + 9983, 9984, 9985, 9986, 9987, 9988, 9989, 9990, 9991, 9992, 9993, 9994, + 9995, 9996, 9997, 9998, 9999, 10000, 10001, 40951, 10006, 10007, 40951, + 40951, 40951, 10004, 40951, 40951, 10005, 20610, 20621, 20612, 20606, + 20618, 20623, 20613, 20619, 20604, 20617, 20620, 20607, 20625, 20622, + 20614, 20611, 20624, 20615, 20608, 20609, 20616, 20605, 40951, 20626, + 20601, 20597, 20600, 20598, 20596, 20602, 20603, 20599, 31035, 31046, + 31037, 31031, 31043, 31048, 31038, 31044, 31029, 31042, 31045, 31032, + 31050, 31028, 31047, 31039, 31036, 31049, 31040, 31033, 31034, 31041, + 31030, 31027, 31051, 31058, 31053, 31054, 31057, 31056, 31055, 31052, + 28806, 28821, 28811, 28832, 28823, 28817, 28813, 28829, 28834, 28824, + 28830, 28815, 28809, 28828, 28810, 28831, 28807, 28818, 28814, 28836, + 28812, 28833, 28825, 28822, 28835, 28826, 28819, 28820, 28808, 28827, + 28816, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 28841, + 28838, 28839, 28844, 28845, 28843, 28840, 28837, 28842, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 19658, 19674, 19666, 19665, + 19671, 19676, 19660, 19672, 19661, 19670, 19673, 19663, 19678, 19675, + 19667, 19659, 19677, 19668, 19664, 40951, 19669, 19662, 40951, 40951, + 40951, 40951, 40951, 19679, 19683, 19682, 19681, 19680, 31429, 31448, + 31444, 31431, 31432, 31440, 31441, 31433, 31439, 31442, 31445, 31447, + 31450, 31446, 31437, 31430, 31449, 31435, 31434, 31443, 31436, 31438, + 31455, 31454, 31451, 31456, 31452, 31453, 40951, 40951, 40951, 31457, + 25309, 25315, 25319, 25317, 25311, 25327, 25320, 25328, 25321, 25305, + 25322, 25313, 25323, 25325, 25308, 25303, 25326, 25318, 25324, 25307, + 25304, 25310, 25312, 25306, 25314, 25316, 40951, 40951, 40951, 40951, + 40951, 25329, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 27402, 27403, 27404, 27405, + 27401, 27400, 27376, 27377, 27398, 27397, 27380, 27381, 27382, 27383, + 27378, 27379, 27396, 27393, 27392, 27384, 27385, 27386, 27394, 27399, + 27387, 27388, 27389, 27390, 27391, 27395, 27406, 27407, 27298, 27319, + 27320, 27321, 27318, 27317, 27310, 27314, 27313, 27303, 27304, 27316, + 27312, 27308, 27307, 27305, 27299, 27306, 27309, 27315, 27300, 27301, + 27302, 27311, 40951, 40951, 40951, 40951, 27286, 27291, 27323, 27322, + 27372, 27364, 27358, 27335, 27329, 27352, 27346, 27324, 27341, 27370, + 27368, 27362, 27339, 27333, 27356, 27350, 40951, 40951, 27373, 27365, + 27359, 27336, 27330, 27353, 27347, 27326, 27343, 27375, 27367, 27361, + 27338, 27332, 27355, 27349, 27328, 27345, 27371, 27369, 27363, 27340, + 27334, 27357, 27351, 27325, 27342, 27374, 27366, 27360, 27337, 27331, + 27354, 27348, 27327, 27344, 27290, 27296, 27295, 27289, 27288, 27293, + 27292, 27287, 27297, 27294, 21668, 21690, 21692, 21688, 40951, 21689, + 21691, 40951, 40951, 40951, 40951, 40951, 21693, 21684, 21687, 21686, + 21634, 21632, 21656, 21655, 40951, 21654, 21653, 21662, 40951, 21642, + 21638, 21637, 21645, 21644, 21641, 21640, 21639, 21647, 21646, 21643, + 21658, 21657, 21652, 21651, 21664, 21666, 21665, 21663, 21660, 21648, + 21649, 21650, 21667, 21661, 21633, 21635, 21636, 21659, 40951, 40951, + 21682, 21683, 21685, 40951, 40951, 40951, 40951, 21694, 21631, 21630, + 21629, 21628, 21670, 21669, 21671, 21672, 21695, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 21676, 21681, 21674, 21673, 21680, 21678, + 21677, 21675, 21679, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 30405, 30401, 30406, 30411, 30402, 30409, 30395, 30403, 30407, 30399, + 30394, 30390, 30408, 30391, 30393, 30392, 30410, 30383, 30384, 30386, + 30389, 30387, 30388, 30398, 30400, 30385, 30404, 30397, 30396, 30413, + 30412, 30414, 30226, 30244, 30225, 30235, 30247, 30248, 30237, 30241, + 30239, 30232, 30236, 30228, 30243, 30227, 30249, 30238, 30240, 30221, + 30222, 30245, 30224, 30246, 30223, 30231, 30233, 30229, 30242, 30230, + 30234, 30252, 30251, 30250, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 25610, 25614, 25613, 25618, + 25617, 25615, 25639, 25642, 25658, 25622, 25621, 25620, 25619, 25640, + 25632, 25638, 25624, 25634, 25623, 25636, 25616, 25631, 25645, 25641, + 25628, 25612, 25611, 25644, 25643, 25629, 25626, 25635, 25625, 25637, + 25630, 25627, 25633, 25660, 25659, 40951, 40951, 40951, 40951, 25646, + 25650, 25649, 25648, 25647, 25657, 25655, 25653, 25652, 25651, 25656, + 25654, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 2542, 2543, 2549, 2545, 2548, 2544, 2546, 2547, 2585, 2586, 2575, 2576, + 2577, 2578, 2573, 2574, 2590, 2563, 2562, 2561, 2552, 2550, 2551, 2587, + 2589, 2572, 2570, 2582, 2581, 2571, 2593, 2588, 2580, 2579, 2557, 2556, + 2555, 2560, 2559, 2558, 2592, 2553, 2568, 2569, 2595, 2594, 2591, 2567, + 2584, 2565, 2583, 2564, 2566, 2554, 40951, 40951, 40951, 2596, 37309, + 33835, 22667, 22662, 22668, 22663, 20752, 20763, 20754, 20748, 20760, + 20765, 20755, 20761, 20746, 20759, 20762, 20749, 20767, 20764, 20756, + 20753, 20766, 20757, 20750, 20751, 20758, 20747, 40951, 40951, 20772, + 20769, 20770, 20775, 20771, 20768, 20773, 20774, 20721, 20735, 20726, + 20722, 20732, 20725, 20727, 20733, 20719, 20731, 20734, 20723, 20724, + 20736, 20728, 20737, 20729, 20730, 20720, 40951, 40951, 40951, 40951, + 40951, 20742, 20739, 20740, 20745, 20741, 20738, 20743, 20744, 31690, + 31704, 31695, 31691, 31701, 31694, 31696, 31702, 31700, 31703, 31692, + 31693, 31705, 31697, 31707, 31698, 31699, 31706, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 31715, 31716, 31688, 31689, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 31712, 31709, 31710, 31714, 31711, 31708, 31713, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 30415, 30457, 30458, + 30447, 30483, 30476, 30450, 30451, 30485, 30428, 30468, 30416, 30461, + 30430, 30470, 30418, 30462, 30429, 30469, 30417, 30446, 30482, 30436, + 30475, 30425, 30465, 30419, 30463, 30452, 30486, 30431, 30471, 30420, + 30441, 30444, 30432, 30421, 30459, 30439, 30478, 30437, 30477, 30440, + 30479, 30467, 30438, 30460, 30445, 30453, 30448, 30443, 30481, 30433, + 30472, 30449, 30484, 30454, 30487, 30434, 30473, 30422, 30426, 30423, + 30427, 30466, 30442, 30480, 30435, 30474, 30424, 30464, 30455, 30456, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 30074, 30077, 30100, 30075, 30089, + 30085, 30091, 30101, 30076, 30079, 30121, 30102, 30103, 30092, 30093, + 30104, 30122, 30123, 30105, 30106, 30078, 30118, 30094, 30095, 30080, + 30082, 30086, 30114, 30116, 30110, 30112, 30115, 30107, 30081, 30108, + 30124, 30087, 30088, 30096, 30083, 30097, 30090, 30117, 30120, 30111, + 30113, 30109, 30098, 30099, 30084, 30119, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 30125, + 30128, 30151, 30126, 30140, 30136, 30142, 30152, 30127, 30130, 30172, + 30153, 30154, 30143, 30144, 30155, 30173, 30174, 30156, 30157, 30129, + 30169, 30145, 30146, 30131, 30133, 30137, 30165, 30167, 30161, 30163, + 30166, 30158, 30132, 30159, 30175, 30138, 30139, 30147, 30134, 30148, + 30141, 30168, 30171, 30162, 30164, 30160, 30149, 30150, 30135, 30170, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 30178, 30177, 30181, + 30176, 30179, 30180, 19611, 19598, 19606, 19590, 19589, 19603, 19599, + 19602, 19587, 19600, 19584, 19583, 19592, 19591, 19610, 19597, 19596, + 19588, 19601, 19604, 19605, 19595, 19608, 19585, 19609, 19586, 19593, + 19594, 19607, 19618, 19620, 19622, 19619, 19621, 19613, 19612, 19617, + 19614, 19616, 19615, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 19629, 19631, 19628, 19627, 19624, 19623, 19626, 19625, 19632, + 19630, 40951, 40951, 40951, 40951, 40951, 40951, 17713, 17715, 17712, + 17711, 17708, 17707, 17710, 17709, 17716, 17714, 17699, 17700, 17701, + 17698, 17702, 17696, 17673, 17657, 17665, 17663, 17656, 17662, 17668, + 17670, 17664, 17660, 17658, 17671, 17672, 17669, 17667, 17654, 17659, + 17655, 17666, 17661, 17652, 17653, 40951, 40951, 40951, 17697, 17651, + 17649, 17648, 17650, 17704, 17703, 17695, 17679, 17687, 17685, 17678, + 17684, 17690, 17692, 17686, 17682, 17680, 17693, 17694, 17691, 17689, + 17676, 17681, 17677, 17688, 17683, 17674, 17675, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 17705, 17706, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 32268, 32266, 32265, 32262, 32261, + 32264, 32263, 32269, 32267, 32260, 32259, 32257, 32248, 32246, 32255, + 32253, 32244, 32250, 32251, 32258, 32256, 32247, 32245, 32254, 32252, + 32243, 32249, 32240, 32241, 32239, 32242, 40951, 39408, 39442, 39422, + 39421, 39427, 39426, 39404, 39403, 39402, 39413, 39434, 39407, 39438, + 39441, 39440, 39437, 39445, 39424, 39423, 39425, 39406, 39428, 39439, + 39409, 39433, 39444, 39429, 39430, 39417, 39415, 39414, 39416, 39418, + 39405, 39420, 39443, 39431, 39432, 39411, 39412, 39435, 39410, 40951, + 39400, 39399, 39401, 40951, 40951, 39419, 39436, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 1195, 1490, 1326, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 1061, 2351, 2352, 2349, 30345, 30353, 30367, 30354, 30358, 30364, 30355, + 30370, 30359, 30365, 30363, 30366, 30357, 30372, 30368, 30347, 30348, + 30360, 30346, 30344, 30371, 30361, 30349, 30350, 30356, 30362, 30369, + 30351, 30352, 30379, 30377, 30374, 30382, 30381, 30378, 30376, 30375, + 30380, 30343, 30373, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 33894, 33908, 33896, 33905, 33912, 33900, 33906, 33904, 33907, + 33897, 33914, 33910, 33901, 33895, 33913, 33902, 33899, 33903, 33911, + 33909, 33898, 33893, 33888, 33886, 33889, 33887, 33892, 33891, 33883, + 33882, 33884, 33890, 33885, 33915, 33918, 33917, 33916, 33921, 33922, + 33919, 33923, 33920, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 30492, 30504, 30502, 30507, 30496, + 30501, 30500, 30503, 30494, 30509, 30505, 30497, 30508, 30498, 30493, + 30499, 30506, 30495, 30491, 30490, 30489, 30488, 30513, 30510, 30511, + 30512, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 6491, + 6485, 6499, 6493, 6488, 6496, 6502, 6484, 6494, 6497, 6495, 6498, 6489, + 6504, 6500, 6486, 6492, 6503, 6490, 6487, 6501, 6509, 6506, 6507, 6511, + 6508, 6505, 6510, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 16720, 16731, 16722, 16716, 16728, 16733, 16723, 16729, + 16714, 16727, 16730, 16717, 16735, 16732, 16724, 16721, 16734, 16725, + 16718, 16719, 16726, 16715, 16736, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 4635, 4640, 4638, 4637, 4639, 4562, 4563, + 4581, 4582, 4579, 4580, 4574, 4575, 4576, 4577, 4608, 4564, 4555, 4565, + 4601, 4600, 4597, 4596, 4585, 4595, 4594, 4599, 4598, 4587, 4571, 4570, + 4567, 4566, 4586, 4573, 4572, 4569, 4568, 4588, 4603, 4602, 4593, 4592, + 4605, 4607, 4606, 4584, 4578, 4589, 4590, 4591, 4604, 4583, 4556, 4561, + 4560, 4645, 4644, 4654, 4655, 4648, 4649, 4650, 4651, 4652, 4653, 4656, + 4646, 4641, 4647, 4657, 4659, 4658, 4633, 4632, 4631, 4634, 4630, 40951, + 40951, 40951, 40951, 4626, 4624, 4621, 4612, 4614, 4619, 4617, 4609, + 4615, 4625, 4623, 4622, 4611, 4613, 4620, 4618, 4610, 4616, 4627, 4628, + 4666, 4668, 4665, 4664, 4661, 4660, 4663, 4662, 4669, 4667, 4636, 4558, + 4559, 4642, 4643, 4557, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 4629, 20979, 20981, 20983, 20939, 20940, 20949, 20950, + 20947, 20948, 20977, 20941, 20978, 20942, 20967, 20966, 20963, 20962, + 20951, 20961, 20960, 20965, 20964, 20953, 20944, 20943, 20936, 20934, + 20935, 20970, 20952, 20946, 20945, 20938, 20937, 20954, 20969, 20968, + 20959, 20958, 20974, 20976, 20971, 20973, 20975, 20955, 20956, 20957, + 20972, 20989, 20994, 20995, 20992, 20993, 20996, 20990, 20997, 20991, + 20982, 20980, 21000, 21001, 20998, 20984, 20985, 20987, 20986, 20988, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 20999, 40951, 40951, 33946, 33947, 33936, 33937, 33938, 33939, 33932, + 33933, 33943, 33935, 33948, 33944, 33949, 33945, 33940, 33942, 33941, + 33934, 33950, 33929, 33951, 33953, 33952, 33930, 33931, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 33960, 33962, 33959, 33958, 33955, + 33954, 33957, 33956, 33963, 33961, 40951, 40951, 40951, 40951, 40951, + 40951, 6174, 6176, 6175, 6170, 6172, 6173, 6171, 6159, 6158, 6155, 6154, + 6140, 6153, 6152, 6157, 6156, 6142, 6145, 6144, 6137, 6136, 6141, 6147, + 6146, 6139, 6138, 6143, 6163, 6162, 6151, 6150, 6165, 6148, 6149, 6166, + 6161, 6169, 6167, 6164, 6178, 6186, 6187, 6182, 6183, 6184, 6180, 6188, + 6181, 6189, 6206, 6205, 6190, 6204, 40951, 6199, 6201, 6198, 6197, 6194, + 6193, 6196, 6195, 6202, 6200, 6177, 6192, 6191, 6203, 6160, 6179, 6185, + 6168, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 25371, + 25373, 25375, 25372, 25374, 25363, 25362, 25359, 25358, 25357, 25356, + 25361, 25360, 25342, 25351, 25350, 25345, 25344, 25341, 25353, 25352, + 25347, 25346, 25343, 25365, 25364, 25355, 25354, 25368, 25349, 25367, + 25370, 25369, 25366, 25348, 25378, 25379, 25377, 25376, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 32741, 32744, 32748, + 32687, 32688, 32706, 32707, 32704, 32705, 32699, 32700, 32701, 32702, + 32733, 32689, 32734, 32690, 32726, 32725, 32722, 32721, 32710, 32720, + 32719, 32724, 32723, 32712, 32696, 32695, 32692, 32691, 32711, 32698, + 32697, 32694, 32693, 32713, 32728, 32727, 32718, 32717, 32730, 32732, + 32731, 32709, 32708, 32703, 32714, 32715, 32716, 32729, 32752, 32761, + 32762, 32755, 32756, 32757, 32758, 32759, 32760, 32763, 32753, 32764, + 32754, 32747, 32743, 32745, 32746, 32767, 32673, 32672, 32765, 32738, + 32735, 32742, 32750, 32684, 32749, 32751, 32739, 32680, 32682, 32679, + 32678, 32675, 32674, 32677, 32676, 32683, 32681, 32685, 32740, 32686, + 32766, 32736, 32737, 40951, 33649, 33647, 33646, 33643, 33642, 33645, + 33644, 33650, 33648, 33661, 33660, 33659, 33655, 33654, 33658, 33657, + 33653, 33656, 33651, 33652, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 22374, 22375, 22401, 22403, 22400, + 22376, 22402, 22377, 22391, 22390, 22366, 22364, 22365, 22384, 22389, + 22388, 22373, 22372, 40951, 22386, 22379, 22378, 22369, 22368, 22385, + 22381, 22380, 22371, 22367, 22370, 22387, 22393, 22392, 22363, 22361, + 22362, 22395, 22399, 22397, 22383, 22398, 22360, 22394, 22382, 22411, + 22414, 22415, 22418, 22416, 22412, 22417, 22413, 22408, 22407, 22406, + 22404, 22358, 22357, 22419, 22409, 22356, 22420, 22405, 22396, 22359, + 22410, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 28281, 28283, 28284, 28282, 28272, 28271, 28270, + 40951, 28269, 40951, 28268, 28267, 28260, 28259, 40951, 28254, 28262, + 28261, 28250, 28248, 28249, 28253, 28264, 28263, 28252, 28251, 28255, + 28274, 28273, 28266, 40951, 28265, 28277, 28280, 28258, 28276, 28279, + 28278, 28275, 28257, 28256, 28285, 40951, 40951, 40951, 40951, 40951, + 40951, 22435, 22436, 22447, 22448, 22445, 22446, 22466, 22437, 22467, + 22438, 22456, 22455, 22426, 22424, 22425, 22449, 22454, 22453, 22434, + 22433, 22432, 22451, 22442, 22441, 22429, 22427, 22439, 22428, 22450, + 22444, 22443, 22431, 22430, 22452, 22458, 22457, 22423, 22421, 22422, + 22463, 22465, 22440, 22462, 22464, 22459, 22460, 22461, 22470, 22471, + 22476, 22477, 22474, 22475, 22478, 22472, 22479, 22473, 22468, 22469, + 40951, 40951, 40951, 40951, 40951, 22486, 22488, 22485, 22484, 22481, + 22480, 22483, 22482, 22489, 22487, 40951, 40951, 40951, 40951, 40951, + 40951, 18095, 18096, 18099, 18102, 40951, 18052, 18053, 18066, 18067, + 18064, 18065, 18047, 18049, 40951, 40951, 18090, 18054, 40951, 40951, + 18089, 18055, 18086, 18085, 18082, 18081, 18070, 18080, 18079, 18084, + 18083, 18072, 18061, 18060, 18057, 18056, 18071, 18063, 18062, 18059, + 18058, 18073, 40951, 18088, 18087, 18078, 18077, 18092, 18094, 18093, + 40951, 18069, 18068, 40951, 18051, 18074, 18075, 18076, 18091, 40951, + 8077, 18097, 18098, 18104, 18113, 18114, 18107, 18108, 18109, 18110, + 40951, 40951, 18116, 18105, 40951, 40951, 18115, 18106, 18101, 40951, + 40951, 18117, 40951, 40951, 40951, 40951, 40951, 40951, 18103, 40951, + 40951, 40951, 40951, 40951, 18100, 18046, 18045, 18048, 18050, 18111, + 18112, 40951, 40951, 8257, 8258, 8256, 8255, 8254, 8253, 8252, 40951, + 40951, 40951, 8263, 8260, 8261, 8259, 8262, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 37588, 37589, 37612, + 37613, 37610, 37611, 37605, 37606, 37607, 37608, 40951, 37634, 40951, + 40951, 37590, 40951, 37633, 37591, 37630, 37629, 37626, 37625, 37614, + 37624, 37623, 37628, 37627, 37616, 37602, 37601, 37593, 37592, 37615, + 37604, 37603, 37595, 37594, 37617, 37632, 37631, 37622, 37621, 37636, + 37637, 37600, 37598, 37609, 37618, 37619, 37620, 37635, 37597, 37599, + 37596, 40951, 37639, 37650, 37659, 37660, 37653, 37654, 37655, 37656, + 37657, 37658, 40951, 37662, 40951, 40951, 37651, 40951, 37661, 37652, + 37583, 37644, 40951, 37640, 37647, 37646, 37641, 37584, 37638, 37587, + 37645, 37586, 37585, 40951, 37642, 37643, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 37649, 37648, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 29204, 29205, 29218, 29219, 29216, + 29217, 29200, 29201, 29202, 29203, 29244, 29206, 29245, 29207, 29232, + 29231, 29228, 29227, 29193, 29192, 29226, 29225, 29230, 29229, 29195, + 29194, 29213, 29212, 29209, 29208, 29197, 29215, 29214, 29211, 29210, + 29198, 29196, 29238, 29237, 29224, 29223, 29236, 29235, 29243, 29240, + 29239, 29234, 29233, 29242, 29220, 29221, 29222, 29241, 29258, 29267, + 29268, 29261, 29262, 29263, 29264, 29265, 29266, 29269, 29259, 29270, + 29260, 29253, 29246, 29249, 29254, 29247, 29248, 29251, 29274, 29255, + 29180, 29178, 29273, 29191, 29271, 29187, 29189, 29186, 29185, 29182, + 29181, 29184, 29183, 29190, 29188, 29179, 29257, 40951, 29272, 29256, + 29199, 29250, 29252, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 37311, 37312, 37313, 37331, 37332, 37329, 37330, + 37324, 37325, 37326, 37327, 37357, 37314, 37358, 37315, 37349, 37348, + 37345, 37344, 37333, 37343, 37342, 37347, 37346, 37335, 37321, 37320, + 37317, 37316, 37334, 37323, 37322, 37319, 37318, 37336, 37351, 37350, + 37341, 37340, 37354, 37356, 37355, 37353, 37328, 37337, 37338, 37339, + 37352, 37367, 37376, 37377, 37370, 37371, 37372, 37373, 37374, 37375, + 37378, 37365, 37368, 37379, 37366, 37369, 37359, 37362, 37364, 37363, + 37360, 37361, 37390, 37310, 37391, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 37386, 37388, 37385, 37384, 37381, 37380, 37383, + 37382, 37389, 37387, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 32855, + 32857, 32878, 32879, 32876, 32877, 32871, 32872, 32873, 32874, 32904, + 32858, 32905, 32859, 32896, 32895, 32892, 32891, 32880, 32890, 32889, + 32894, 32893, 32882, 32865, 32864, 32868, 32867, 32881, 32866, 32861, + 32870, 32869, 32883, 32898, 32897, 32888, 32887, 32901, 32903, 32902, + 32900, 32875, 32884, 32885, 32886, 32899, 32933, 32940, 32941, 32938, + 32939, 32936, 32937, 40951, 40951, 32942, 32934, 32943, 32935, 32926, + 32928, 32930, 32929, 32927, 32925, 32945, 32944, 32924, 32923, 32906, + 32907, 32908, 32854, 32921, 32920, 32918, 32916, 32917, 32909, 32910, + 32919, 32922, 32914, 32915, 32913, 32912, 32911, 32860, 32862, 32863, + 32856, 32931, 32932, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 27636, 27637, 27655, + 27656, 27653, 27654, 27648, 27649, 27650, 27651, 27682, 27638, 27683, + 27639, 27675, 27674, 27671, 27670, 27659, 27669, 27668, 27673, 27672, + 27661, 27645, 27644, 27641, 27640, 27660, 27647, 27646, 27643, 27642, + 27662, 27677, 27676, 27667, 27666, 27679, 27681, 27680, 27658, 27652, + 27663, 27664, 27665, 27678, 27657, 27611, 27620, 27621, 27614, 27615, + 27616, 27617, 27618, 27619, 27622, 27612, 27623, 27613, 27607, 27610, + 27609, 27606, 27625, 27624, 27684, 27608, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 27632, 27634, 27631, + 27630, 27627, 27626, 27629, 27628, 27635, 27633, 40951, 40951, 40951, + 40951, 40951, 40951, 28164, 28159, 28004, 28165, 28163, 28161, 28160, + 28021, 28022, 28155, 28157, 28156, 28166, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 35301, 35303, 35318, 35319, 35316, + 35317, 35343, 35304, 35344, 35305, 35333, 35332, 35329, 35328, 35320, + 35327, 35326, 35331, 35330, 35322, 35313, 35312, 35307, 35306, 35321, + 35315, 35314, 35309, 35308, 35323, 35335, 35334, 35325, 35324, 35340, + 35342, 35311, 35339, 35341, 35336, 35337, 35338, 35310, 35346, 35348, + 35349, 35354, 35355, 35352, 35353, 35356, 35350, 35357, 35351, 35347, + 35345, 35302, 35300, 40951, 40951, 40951, 40951, 40951, 40951, 35364, + 35366, 35363, 35362, 35359, 35358, 35361, 35360, 35367, 35365, 40951, + 40951, 40951, 40951, 40951, 40951, 28773, 28775, 28772, 28771, 28768, + 28767, 28770, 28769, 28776, 28774, 28723, 28725, 28722, 28721, 28718, + 28717, 28720, 28719, 28726, 28724, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 191, 190, 178, 181, 175, 167, 193, 192, 183, 195, + 189, 184, 174, 196, 177, 197, 180, 194, 164, 171, 170, 187, 166, 186, + 182, 188, 165, 40951, 40951, 162, 163, 161, 203, 204, 210, 211, 208, 209, + 212, 207, 213, 205, 206, 200, 40951, 40951, 40951, 40951, 222, 224, 221, + 220, 217, 216, 219, 218, 225, 223, 215, 214, 198, 199, 201, 202, 185, + 173, 172, 169, 168, 179, 176, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 11026, + 11027, 11042, 11043, 11040, 11041, 11068, 11028, 11069, 11029, 11060, + 11059, 11056, 11055, 11044, 11054, 11053, 11058, 11057, 11046, 11037, + 11036, 11031, 11030, 11045, 11039, 11038, 11033, 11032, 11047, 11062, + 11061, 11052, 11051, 11065, 11067, 11035, 11064, 11066, 11048, 11049, + 11050, 11063, 11034, 11072, 11077, 11078, 11075, 11076, 11070, 11071, + 11079, 11073, 11080, 11074, 11083, 11085, 11084, 11082, 11081, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 39052, + 39041, 39070, 39060, 39062, 39063, 39069, 39059, 39045, 39054, 39042, + 39072, 39068, 39047, 39061, 39058, 39046, 39055, 39064, 39053, 39071, + 39044, 39043, 39066, 39067, 39050, 39049, 39048, 39051, 39056, 39057, + 39065, 39084, 39073, 39102, 39092, 39094, 39095, 39101, 39091, 39077, + 39086, 39074, 39104, 39100, 39079, 39093, 39090, 39078, 39087, 39096, + 39085, 39103, 39076, 39075, 39098, 39099, 39082, 39081, 39080, 39083, + 39088, 39089, 39097, 39111, 39113, 39110, 39109, 39106, 39105, 39108, + 39107, 39114, 39112, 39123, 39122, 39121, 39117, 39116, 39120, 39119, + 39115, 39118, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 39124, 10942, 10943, 10950, 10951, 10948, + 10949, 10977, 40951, 40951, 10978, 40951, 40951, 10968, 10967, 10966, + 10965, 10954, 10964, 10963, 10972, 40951, 10956, 10938, 40951, 10945, + 10944, 10955, 10939, 10937, 10947, 10946, 10957, 10970, 10969, 10962, + 10961, 10973, 10941, 10940, 10974, 10953, 10975, 10958, 10959, 10960, + 10971, 10952, 10976, 10983, 10987, 10988, 10985, 10986, 10989, 40951, + 10984, 10990, 40951, 40951, 10982, 10980, 10979, 10991, 10996, 10993, + 10997, 10992, 10981, 10926, 10994, 10995, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 10933, 10935, 10932, 10931, 10928, + 10927, 10930, 10929, 10936, 10934, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 28895, 28896, 28911, 28912, 28909, + 28910, 28892, 28893, 40951, 40951, 28937, 28897, 28938, 28898, 28931, + 28930, 28927, 28926, 28915, 28925, 28924, 28929, 28928, 28917, 28906, + 28905, 28900, 28899, 28916, 28908, 28907, 28902, 28901, 28918, 28933, + 28932, 28923, 28922, 28935, 28936, 28904, 28914, 28894, 28919, 28920, + 28921, 28934, 28913, 28903, 28947, 28952, 28953, 28950, 28951, 28945, + 28946, 40951, 40951, 28954, 28948, 28955, 28949, 28941, 28943, 28942, + 28940, 28939, 28956, 28944, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40732, 40753, 40750, 40749, 40752, 40748, 40747, 40745, 40746, + 40751, 40744, 40700, 40699, 40719, 40718, 40701, 40717, 40716, 40726, + 40703, 40711, 40710, 40693, 40692, 40702, 40713, 40712, 40697, 40696, + 40704, 40721, 40720, 40715, 40714, 40728, 40709, 40708, 40695, 40694, + 40722, 40723, 40724, 40731, 40729, 40727, 40730, 40705, 40706, 40707, + 40725, 40698, 40691, 40741, 40738, 40739, 40740, 40737, 40742, 40688, + 40687, 40685, 40684, 40686, 40690, 40683, 40736, 40734, 40733, 40735, + 40689, 40682, 40743, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 34052, 34073, 34071, 34070, 34072, 34068, 34069, 34066, 34067, + 34065, 34064, 34074, 34019, 34018, 34038, 34037, 34020, 34036, 34035, + 34040, 34039, 34022, 34030, 34029, 34013, 34012, 34021, 34032, 34031, + 34016, 34014, 34023, 34042, 34041, 34034, 34033, 34048, 34028, 34027, + 34015, 34043, 34044, 34045, 34051, 34049, 34047, 34050, 34024, 34025, + 34026, 34046, 34017, 34057, 34059, 33996, 33995, 33993, 33994, 34004, + 34005, 34000, 34003, 33999, 34002, 34007, 34008, 34006, 33998, 33997, + 34001, 34060, 34058, 34075, 34061, 34056, 34055, 34054, 34053, 34011, + 34010, 34009, 34062, 34063, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 5617, 5618, 5615, 5616, + 5613, 5614, 5623, 5624, 5621, 5622, 5619, 5620, 5786, 5787, 5788, 5785, + 31247, 31245, 31254, 31255, 31251, 31259, 31258, 31240, 31253, 31252, + 31244, 31257, 31250, 31243, 31249, 31248, 31241, 31246, 31256, 31235, + 31242, 31260, 31261, 31236, 31262, 31238, 31239, 31237, 31231, 31228, + 31232, 31230, 31226, 31229, 31233, 31227, 31234, 31268, 31267, 31278, + 31269, 31270, 31279, 31275, 31274, 31276, 31277, 31271, 31225, 31272, + 31273, 31264, 31263, 31223, 31265, 31266, 31224, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 10668, 10669, 10758, 10759, 10760, 10761, + 10768, 10767, 10769, 10773, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 34658, 34677, 34684, 34681, + 34671, 34668, 34663, 34685, 34653, 34670, 34679, 34660, 34657, 34666, + 34683, 34661, 34680, 34667, 34672, 34678, 34682, 34655, 34654, 34659, + 34675, 34669, 34665, 34664, 34673, 34656, 34674, 34676, 34662, 34686, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 34693, 34695, 34692, 34691, 34688, 34687, + 34690, 34689, 34696, 34694, 40951, 40951, 40951, 40951, 40951, 40951, + 3661, 3662, 3675, 3676, 3673, 3674, 3657, 3658, 3659, 40951, 3701, 3663, + 3702, 3664, 3693, 3692, 3689, 3688, 3677, 3687, 3686, 3691, 3690, 3679, + 3670, 3669, 3666, 3665, 3678, 3672, 3671, 3668, 3667, 3680, 3695, 3694, + 3685, 3684, 3698, 3700, 3699, 3697, 3660, 3681, 3682, 3683, 3696, 3729, + 3734, 3735, 3732, 3733, 3726, 3727, 3728, 40951, 3736, 3730, 3737, 3731, + 3721, 3723, 3725, 3724, 3722, 3739, 3738, 3750, 3751, 3752, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 3746, 3748, 3745, + 3744, 3741, 3740, 3743, 3742, 3749, 3747, 3720, 3718, 3715, 3706, 3708, + 3713, 3711, 3703, 3709, 3719, 3717, 3716, 3705, 3707, 3714, 3712, 3704, + 3710, 3753, 40951, 40951, 40951, 25735, 25736, 25681, 25680, 25690, + 25675, 25679, 25678, 25692, 25676, 25672, 25671, 25674, 25677, 25683, + 25682, 25689, 25694, 25670, 25669, 25673, 25696, 25686, 25687, 25688, + 25697, 25695, 25693, 25684, 25685, 25691, 25698, 40951, 40951, 25713, + 25712, 25721, 25707, 25711, 25710, 25723, 25708, 25704, 25703, 25706, + 25709, 25715, 25714, 25720, 25725, 25702, 25701, 25705, 25727, 25718, + 25719, 40951, 25728, 25726, 25724, 25716, 25717, 25722, 25729, 25730, + 25732, 25734, 25731, 25733, 25700, 25699, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 25747, + 25748, 25757, 25758, 25755, 25756, 25784, 40951, 25749, 25785, 40951, + 25750, 25763, 25762, 25776, 25775, 25764, 25774, 25773, 25741, 25740, + 25766, 25743, 25742, 25752, 25751, 25765, 25746, 25744, 25754, 25753, + 25767, 25778, 25777, 25772, 25771, 25780, 25783, 25781, 25760, 25782, + 25768, 25769, 25770, 25779, 25759, 25761, 25739, 25745, 25794, 25799, + 25800, 25797, 25798, 25793, 40951, 40951, 40951, 25801, 40951, 25795, + 25802, 40951, 25796, 25792, 25791, 25790, 25788, 25789, 25803, 25787, + 25786, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 25810, + 25812, 25809, 25808, 25805, 25804, 25807, 25806, 25813, 25811, 40951, + 40951, 40951, 40951, 40951, 40951, 18770, 18771, 18784, 18785, 18782, + 18783, 40951, 18801, 18772, 40951, 18800, 18773, 18807, 18806, 18789, + 18788, 18803, 18797, 18796, 18781, 18780, 18787, 18793, 18792, 18777, + 18776, 18769, 18791, 18790, 18779, 18778, 18786, 18795, 18794, 18775, + 18774, 18768, 18799, 18798, 18802, 18804, 18805, 18810, 18815, 18816, + 18813, 18814, 40951, 18818, 18811, 40951, 18817, 18812, 18809, 18808, + 18819, 18830, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 18826, + 18828, 18825, 18824, 18821, 18820, 18823, 18822, 18829, 18827, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 25433, + 25431, 25425, 25436, 25428, 25435, 25439, 25430, 25427, 25429, 25432, + 25426, 25441, 25437, 25434, 25440, 25438, 25442, 25448, 25445, 25447, + 25444, 25446, 25443, 25424, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 21513, 21517, 21515, 21516, 21454, 21455, 21474, 21475, 21468, + 21469, 21470, 21471, 21472, 21473, 21499, 21456, 21500, 40951, 21490, + 21489, 21488, 21487, 21476, 21486, 21485, 21459, 21458, 21478, 21465, + 21464, 21461, 21460, 21477, 21467, 21466, 21463, 21462, 21479, 21492, + 21491, 21484, 21483, 21495, 21498, 21496, 21494, 21497, 21480, 21481, + 21482, 21493, 21457, 21519, 21518, 21526, 21527, 21524, 21525, 21521, + 40951, 40951, 40951, 21522, 21520, 21523, 21512, 21540, 21529, 21528, + 21507, 21510, 21506, 21508, 21504, 21503, 21511, 21502, 21505, 21509, + 21501, 21536, 21538, 21535, 21534, 21531, 21530, 21533, 21532, 21539, + 21537, 21514, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 25086, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 35424, 35420, 35414, 35423, 35416, 35425, 35429, + 35431, 35426, 35421, 35422, 35427, 35415, 35432, 35430, 35417, 35428, + 35418, 35419, 35433, 35434, 35498, 35481, 35479, 35488, 35487, 35483, + 35489, 35485, 35484, 35491, 35492, 35493, 35490, 35482, 35495, 35413, + 35400, 35471, 35478, 35768, 35769, 35398, 35373, 35499, 35767, 35435, + 35500, 35486, 35494, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 35476, 8820, 8828, 8826, 8822, + 8827, 8823, 8821, 8824, 8825, 8889, 8829, 8838, 8837, 8831, 8830, 8842, + 8832, 8833, 8841, 8835, 8836, 8843, 8844, 8849, 8846, 8845, 8847, 8848, + 8851, 8853, 8855, 8854, 8856, 8862, 8859, 8860, 8864, 8857, 8858, 8863, + 8861, 8866, 8865, 8867, 8869, 8870, 8874, 8873, 8871, 8872, 8875, 8888, + 8876, 8877, 8878, 8887, 8879, 8883, 8884, 8882, 8880, 8881, 8886, 8885, + 8890, 8891, 8902, 8893, 8897, 8898, 8899, 8900, 8901, 8903, 8906, 8905, + 8904, 8907, 8909, 8910, 8911, 8912, 8913, 8914, 8915, 8916, 8917, 8918, + 8919, 8920, 8921, 8922, 8923, 8924, 8925, 8926, 8941, 8927, 8928, 8938, + 8932, 8929, 8930, 8931, 8939, 8936, 8940, 8937, 8933, 8935, 8948, 8944, + 8945, 8946, 8949, 8960, 8950, 8953, 8954, 8956, 8957, 8958, 8961, 8964, + 8962, 8963, 8965, 8966, 8968, 8969, 8998, 9005, 8999, 9000, 9001, 9002, + 9003, 9004, 9006, 9008, 9007, 9009, 9012, 9013, 9016, 9010, 9011, 9017, + 9062, 9061, 9063, 9018, 9021, 9022, 9023, 9019, 9020, 9024, 9027, 9025, + 9028, 9030, 9039, 9040, 9041, 9042, 9060, 9043, 9044, 9057, 9058, 9056, + 9045, 9046, 9047, 9048, 9049, 9050, 9051, 9054, 9055, 9064, 9154, 9065, + 9066, 9068, 9067, 9074, 9069, 9071, 9073, 9077, 9076, 9078, 9079, 9086, + 9080, 9081, 9085, 9089, 9090, 9087, 9088, 9096, 9093, 9097, 9098, 9099, + 9100, 9101, 9103, 9104, 9106, 9105, 9108, 9107, 9110, 9109, 9111, 9112, + 9114, 9115, 9119, 9121, 9125, 9126, 9139, 9131, 9132, 9127, 9128, 9129, + 9133, 9137, 9134, 9135, 9136, 9140, 9141, 9143, 9144, 9145, 9146, 9147, + 9148, 9158, 9149, 9150, 9153, 9152, 9151, 9155, 9156, 9157, 9159, 9160, + 9163, 9164, 9165, 9166, 9167, 9169, 9168, 9185, 9176, 9177, 9170, 9171, + 9173, 9174, 9172, 9175, 9184, 9178, 9183, 9181, 9180, 9182, 9187, 9206, + 9188, 9189, 9190, 9193, 9192, 9194, 9195, 9197, 9198, 9199, 9207, 9200, + 9201, 9202, 9205, 9204, 9203, 9208, 9209, 9211, 9212, 9213, 9214, 9216, + 9220, 9217, 9221, 9219, 9218, 9222, 9223, 9224, 9225, 9229, 9228, 9226, + 9227, 9230, 9231, 9233, 9252, 9254, 9234, 9236, 9235, 9237, 9238, 9241, + 9242, 9240, 9239, 9243, 9244, 9245, 9246, 9249, 9247, 9248, 9250, 9251, + 9255, 9256, 9253, 9257, 9258, 9259, 9260, 9262, 9265, 9264, 9266, 9267, + 9269, 9270, 9271, 9275, 9274, 9272, 9273, 9276, 9280, 9279, 9278, 9281, + 9282, 9284, 9285, 9289, 9286, 9287, 9292, 9290, 9293, 9294, 9296, 9295, + 9297, 9298, 9320, 9319, 9322, 9323, 9304, 9305, 9302, 9303, 9301, 9299, + 9307, 9306, 9308, 9310, 9316, 9317, 9314, 9315, 9324, 9325, 9326, 9344, + 9329, 9330, 9331, 9327, 9328, 9332, 9333, 9334, 9335, 9336, 9338, 9339, + 9340, 9341, 9342, 9367, 9345, 9348, 9346, 9347, 9353, 9354, 9351, 9352, + 9349, 9350, 9355, 9356, 9364, 9357, 9358, 9365, 9362, 9363, 9366, 9359, + 9360, 9361, 9368, 9369, 9370, 9371, 9372, 9373, 9374, 9376, 9377, 9375, + 9378, 9379, 9420, 9421, 9382, 9383, 9380, 9381, 9386, 9387, 9385, 9391, + 9388, 9390, 9389, 9395, 9396, 9394, 9392, 9393, 9397, 9398, 9399, 9400, + 9401, 9402, 9403, 9422, 9408, 9404, 9405, 9406, 9407, 9409, 9411, 9410, + 9412, 9413, 9415, 9414, 9416, 9418, 9417, 9423, 9424, 9427, 9428, 9425, + 9426, 9502, 9497, 9498, 9499, 9500, 9501, 9503, 9506, 9504, 9505, 9507, + 9549, 9508, 9538, 9539, 9515, 9517, 9534, 9518, 9540, 9522, 9520, 9521, + 9523, 9524, 9525, 9526, 9535, 9536, 9529, 9530, 9532, 9541, 9509, 9510, + 9514, 9512, 9550, 9542, 9544, 9543, 9545, 9551, 9552, 9546, 9547, 9548, + 9553, 9554, 9555, 9558, 9559, 9560, 9556, 9557, 9561, 9562, 9564, 9566, + 9567, 9585, 9583, 9584, 9587, 9586, 9568, 9575, 9573, 9574, 9569, 9570, + 9576, 9577, 9578, 9579, 9580, 9582, 9588, 9597, 9589, 9591, 9592, 9590, + 9593, 9595, 9594, 9596, 9599, 9601, 9600, 9602, 9603, 9636, 9638, 9604, + 9606, 9605, 9608, 9611, 9609, 9610, 9614, 9618, 9621, 9620, 9623, 9626, + 9624, 9625, 9629, 9631, 9637, 9639, 9640, 9644, 9651, 9649, 9647, 9648, + 9650, 9653, 9652, 9645, 9646, 9654, 9657, 9663, 9658, 9661, 9656, 9655, + 9664, 9662, 9659, 9660, 9665, 9666, 9667, 9668, 9669, 9670, 9671, 9673, + 9676, 9677, 9680, 9681, 9682, 9678, 9679, 9674, 9675, 9683, 9684, 9685, + 9686, 9687, 9688, 9690, 9691, 9692, 9693, 9694, 9698, 9695, 9719, 9712, + 9713, 9718, 9701, 9700, 9714, 9717, 9715, 9704, 9703, 9706, 9708, 9709, + 9710, 9711, 9707, 9720, 9696, 9721, 9722, 9723, 9724, 9725, 9726, 9734, + 9732, 9730, 9733, 9729, 9731, 9727, 9728, 9735, 9737, 9738, 9739, 9746, + 9741, 9742, 9750, 9751, 9747, 9749, 9748, 9752, 9754, 9753, 9755, 9766, + 9757, 9756, 9765, 9763, 9758, 9759, 9760, 9762, 9761, 9764, 9770, 9767, + 9769, 9768, 9771, 9772, 9773, 9774, 9777, 9778, 9780, 9781, 9782, 9783, + 9785, 9784, 9786, 9793, 9787, 9788, 9794, 9789, 9790, 9791, 9792, 9795, + 9799, 9796, 9797, 9798, 9800, 9801, 9802, 9803, 9809, 9807, 9805, 9806, + 9804, 9808, 9810, 9812, 9829, 9830, 9813, 9818, 9820, 9814, 9817, 9815, + 9816, 9821, 9827, 9828, 9822, 9825, 9826, 9831, 9837, 9836, 9832, 9834, + 9833, 9918, 9919, 9838, 9839, 9844, 9845, 9842, 9843, 9846, 9840, 9841, + 9847, 9850, 9851, 9853, 9854, 9855, 9859, 9856, 9857, 9858, 9848, 9849, + 9860, 9862, 9861, 9863, 9865, 9866, 9867, 9873, 9872, 9868, 9869, 9870, + 9905, 9874, 9875, 9876, 9877, 9878, 9897, 9880, 9881, 9883, 9882, 9884, + 9885, 9901, 9886, 9888, 9887, 9900, 9890, 9898, 9902, 9893, 9892, 9899, + 9894, 9896, 9895, 9903, 9904, 9906, 9910, 9908, 9909, 9907, 9913, 9912, + 9911, 9917, 9914, 9915, 9916, 9920, 9922, 9921, 9925, 9923, 9940, 9926, + 9929, 9931, 9927, 9928, 9932, 9930, 9933, 9935, 9937, 9938, 9939, 9343, + 8839, 8850, 8868, 8934, 8943, 8959, 8967, 9059, 9052, 9070, 9072, 9162, + 9186, 9232, 9261, 9263, 9277, 9283, 9318, 9291, 9321, 9300, 9309, 9313, + 9384, 9513, 9516, 9531, 9563, 9581, 9598, 9607, 9635, 9633, 9612, 9643, + 9672, 9689, 9699, 9819, 9852, 9835, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 8804, 8799, 8736, 8722, + 8783, 8779, 8711, 8756, 8803, 8744, 8728, 8789, 8778, 8710, 8755, 8742, + 8726, 8785, 8775, 8705, 8752, 8765, 8809, 8801, 8738, 8724, 8787, 8777, + 8707, 8754, 8766, 8810, 8802, 8739, 8725, 8811, 8793, 8794, 8740, 8716, + 8782, 8774, 8704, 8751, 8769, 8812, 8795, 8796, 8741, 8717, 8780, 8781, + 8764, 8807, 8790, 8791, 8731, 8721, 8797, 8798, 8732, 8735, 8733, 8734, + 8788, 8773, 8771, 8772, 8708, 8709, 8747, 8749, 8750, 8748, 8805, 8800, + 8737, 8723, 8784, 8763, 8806, 8792, 8729, 8730, 8719, 8720, 8746, 8745, + 8759, 8808, 8768, 8814, 8718, 8767, 8813, 8760, 8761, 8757, 8758, 8762, + 8770, 8712, 8715, 8714, 8713, 8743, 8727, 8786, 8776, 8706, 8753, 40951, + 8819, 8818, 8817, 8815, 8816, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 8840, 8834, 8852, 8892, 8894, 8895, + 8896, 8908, 8947, 8942, 8952, 8951, 8955, 8972, 8970, 8971, 8973, 8974, + 8992, 8978, 8975, 8976, 8977, 8994, 8995, 8993, 8982, 8981, 8979, 8980, + 8985, 8983, 8984, 8986, 8987, 8988, 8989, 8996, 8997, 8991, 8990, 9015, + 9014, 9026, 9029, 9034, 9038, 9031, 9035, 9036, 9032, 9033, 9037, 9053, + 9075, 9082, 9083, 9084, 9091, 9092, 9095, 9094, 9102, 9113, 9116, 9117, + 9118, 9120, 9122, 9123, 9124, 9130, 9138, 9142, 9161, 9179, 9191, 9196, + 9210, 9215, 9268, 9288, 9311, 9312, 9419, 9439, 9429, 9430, 9438, 9434, + 9435, 9436, 9433, 9432, 9431, 9437, 9441, 9440, 9447, 9448, 9442, 9443, + 9444, 9449, 9445, 9446, 9450, 9451, 9452, 9453, 9454, 9455, 9462, 9456, + 9461, 9460, 9458, 9457, 9459, 9463, 9464, 9469, 9470, 9465, 9466, 9467, + 9468, 9496, 9492, 9471, 9479, 9480, 9478, 9477, 9481, 9472, 9473, 9475, + 9476, 9474, 9493, 9482, 9489, 9491, 9486, 9487, 9490, 9485, 9488, 9484, + 9483, 9494, 9495, 9511, 9537, 9519, 9527, 9528, 9533, 9565, 9572, 9571, + 9632, 9613, 9615, 9634, 9616, 9617, 9619, 9622, 9627, 9628, 9630, 9697, + 9716, 9702, 9705, 9736, 9740, 9743, 9744, 9745, 9776, 9775, 9779, 9811, + 9823, 9824, 9864, 9871, 9879, 9891, 9889, 9924, 9934, 9936, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 10013, 10014, 10015, 10016, 10017, 10018, 10019, 10020, 10023, 10024, + 10021, 10022, 10025, 10026, 10027, 10028, 10029, 10030, 10031, 10032, + 10033, 10034, 10035, 10036, 10037, 10038, 10039, 10040, 10041, 10042, + 10043, 10044, 10045, 10046, 10047, 10048, 10049, 10050, 10051, 10052, + 10053, 10054, 10055, 10056, 10057, 10058, 10059, 10079, 10080, 10081, + 10082, 10083, 10084, 10085, 10086, 10087, 10062, 10063, 10064, 10065, + 10066, 10060, 10061, 10067, 10068, 10069, 10088, 10089, 10090, 10091, + 10092, 10093, 10094, 10095, 10096, 10097, 10070, 10071, 10072, 10073, + 10074, 10075, 10076, 10077, 10078, 10098, 10099, 10100, 10101, 10102, + 10103, 10104, 10105, 10106, 10107, 10108, 10109, 10110, 10111, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 11548, 11549, 11550, 11551, 11546, 11547, 11543, 11544, + 11545, 11552, 11553, 11554, 11559, 11560, 11561, 11562, 11555, 11556, + 11563, 11564, 11557, 11558, 11565, 11566, 11593, 11594, 11595, 11596, + 11597, 11598, 11599, 11600, 11601, 11602, 11583, 11584, 11581, 11582, + 11585, 11586, 11587, 11588, 11589, 11590, 11591, 11567, 11568, 11575, + 11569, 11570, 11571, 11572, 11576, 11573, 11574, 11577, 11578, 11579, + 11580, 11603, 11604, 11605, 11606, 11607, 11608, 11609, 11610, 11611, + 11612, 11613, 11614, 11615, 11616, 11617, 11618, 11619, 11620, 11621, + 11622, 11592, 11662, 11663, 11664, 11665, 11660, 11661, 11666, 11667, + 11668, 11669, 11674, 11670, 11671, 11672, 11673, 11675, 11676, 11677, + 11678, 11679, 11680, 11681, 11682, 11683, 11684, 11685, 11686, 11687, + 11688, 11689, 11690, 11691, 11692, 11693, 11694, 11695, 11696, 11697, + 11700, 11701, 11702, 11703, 11704, 11705, 11706, 11698, 11699, 11707, + 11780, 11781, 11782, 11783, 11784, 11785, 11786, 11787, 11788, 11789, + 11771, 11772, 11773, 11774, 11775, 11776, 11777, 11769, 11770, 11778, + 11779, 11712, 11708, 11709, 11713, 11714, 11710, 11711, 11715, 11716, + 11717, 11718, 11719, 11724, 11725, 11726, 11727, 11728, 11729, 11720, + 11721, 11730, 11722, 11723, 11731, 11732, 11733, 11734, 11735, 11736, + 11737, 11738, 11739, 11740, 11741, 11746, 11742, 11743, 11747, 11744, + 11745, 11748, 11749, 11750, 11751, 11752, 11762, 11763, 11764, 11765, + 11766, 11767, 11768, 11753, 11754, 11755, 11756, 11757, 11758, 11759, + 11760, 11761, 11794, 11795, 11796, 11797, 11798, 11799, 11800, 11790, + 11791, 11792, 11793, 11826, 11827, 11828, 11829, 11830, 11831, 11822, + 11823, 11824, 11825, 11832, 11833, 11801, 11802, 11805, 11806, 11807, + 11808, 11809, 11810, 11811, 11803, 11804, 11812, 11815, 11816, 11817, + 11818, 11813, 11814, 11819, 11820, 11821, 11837, 11838, 11839, 11840, + 11841, 11842, 11843, 11844, 11845, 11846, 11849, 11850, 11851, 11847, + 11848, 11852, 11853, 11854, 11855, 11856, 11857, 11893, 11891, 11892, + 11894, 11895, 11896, 11897, 11898, 11899, 11900, 11901, 11864, 11858, + 11859, 11865, 11866, 11867, 11868, 11869, 11860, 11861, 11862, 11863, + 11870, 11877, 11878, 11879, 11880, 11881, 11871, 11872, 11873, 11874, + 11875, 11876, 11882, 11883, 11888, 11884, 11885, 11886, 11887, 11889, + 11890, 11908, 11909, 11910, 11911, 11912, 11906, 11907, 11903, 11904, + 11905, 11913, 11914, 11917, 11915, 11916, 11918, 11919, 11920, 11921, + 11922, 11923, 11924, 11925, 11950, 11951, 11954, 11955, 11956, 11957, + 11958, 11952, 11953, 11959, 11960, 11961, 11930, 11931, 11932, 11933, + 11934, 11935, 11926, 11927, 11928, 11929, 11936, 11937, 11942, 11943, + 11944, 11938, 11939, 11945, 11940, 11941, 11946, 11947, 11948, 11949, + 11962, 11963, 11964, 11965, 11966, 11969, 11970, 11971, 11972, 11973, + 11967, 11968, 11974, 11975, 11983, 11984, 11985, 11986, 11979, 11980, + 11987, 11988, 11989, 11981, 11982, 11990, 11991, 11992, 11993, 11994, + 11995, 11996, 11997, 12638, 12639, 12640, 12641, 12642, 12643, 12644, + 12645, 12009, 12005, 12006, 12010, 12011, 12012, 12007, 12008, 12013, + 12014, 12016, 12017, 12018, 12021, 12019, 12020, 12022, 12023, 12024, + 12025, 12026, 12027, 12037, 12038, 12045, 12028, 12029, 12030, 12031, + 12032, 12033, 12034, 12035, 12036, 12046, 12047, 12039, 12040, 12041, + 12042, 12043, 12044, 12048, 12049, 12056, 12057, 12050, 12051, 12058, + 12052, 12053, 12059, 12060, 12061, 12054, 12055, 12062, 12068, 12066, + 12067, 12069, 12063, 12064, 12065, 12070, 12071, 12072, 12073, 12074, + 12075, 12076, 12077, 12078, 12079, 12080, 12081, 12138, 12139, 12140, + 12141, 12142, 12143, 12144, 12145, 12146, 12101, 12102, 12103, 12104, + 12105, 12106, 12107, 12108, 12098, 12099, 12100, 12109, 12126, 12127, + 12128, 12129, 12130, 12124, 12125, 12131, 12132, 12133, 12134, 12118, + 12119, 12120, 12110, 12111, 12112, 12113, 12114, 12115, 12121, 12116, + 12117, 12122, 12123, 12135, 12136, 12137, 12149, 12150, 12151, 12152, + 12147, 12148, 12153, 12154, 12155, 12156, 12159, 12160, 12161, 12162, + 12163, 12164, 12165, 12157, 12158, 12166, 12167, 12168, 12186, 12187, + 12188, 12189, 12190, 12191, 12192, 12193, 12194, 12169, 12170, 12171, + 12172, 12175, 12176, 12177, 12178, 12179, 12180, 12173, 12174, 12181, + 12184, 12185, 12182, 12183, 12202, 12203, 12206, 12207, 12208, 12204, + 12205, 12195, 12196, 12197, 12198, 12199, 12200, 12201, 12209, 12210, + 12211, 12212, 12213, 12214, 12215, 12218, 12219, 12220, 12221, 12222, + 12223, 12224, 12225, 12216, 12217, 12226, 12227, 12234, 12235, 12236, + 12228, 12229, 12230, 12231, 12237, 12238, 12239, 12232, 12233, 12245, + 12246, 12249, 12250, 12247, 12248, 12251, 12252, 12240, 12241, 12242, + 12243, 12244, 12253, 12254, 12255, 12260, 12261, 12262, 12263, 12264, + 12265, 12266, 12267, 12268, 12269, 12256, 12257, 12258, 12259, 12271, + 12272, 12275, 12273, 12274, 12276, 12277, 12278, 12279, 12280, 12281, + 12282, 12283, 12646, 12647, 12648, 12649, 12650, 12651, 12652, 12289, + 12287, 12288, 12284, 12285, 12286, 12290, 12291, 12292, 12293, 12294, + 12295, 12296, 12297, 12300, 12301, 12302, 12303, 12304, 12298, 12299, + 12305, 12306, 12307, 12308, 12309, 12310, 12311, 12312, 12313, 12314, + 12315, 12316, 12317, 12322, 12318, 12319, 12323, 12324, 12325, 12320, + 12321, 12326, 12327, 12328, 12334, 12335, 12336, 12337, 12329, 12330, + 12331, 12338, 12339, 12332, 12333, 12340, 12341, 12345, 12346, 12347, + 12348, 12349, 12350, 12342, 12343, 12344, 12351, 12352, 12353, 12356, + 12357, 12358, 12359, 12360, 12354, 12355, 12361, 12362, 12363, 12364, + 12365, 12366, 12367, 12368, 12369, 12370, 12371, 12380, 12381, 12372, + 12373, 12382, 12383, 12384, 12374, 12375, 12376, 12377, 12378, 12379, + 12389, 12385, 12386, 12390, 12391, 12392, 12393, 12387, 12388, 12394, + 12395, 12396, 12406, 12407, 12408, 12409, 12410, 12411, 12412, 12413, + 12414, 12415, 12401, 12402, 12397, 12398, 12399, 12400, 12403, 12404, + 12405, 12420, 12421, 12422, 12423, 12424, 12417, 12418, 12419, 12425, + 12426, 12427, 12454, 12455, 12456, 12457, 12458, 12459, 12460, 12461, + 12462, 12463, 12432, 12433, 12434, 12428, 12429, 12435, 12436, 12437, + 12438, 12439, 12430, 12431, 12442, 12443, 12440, 12441, 12444, 12445, + 12446, 12447, 12448, 12449, 12450, 12451, 12452, 12453, 12467, 12468, + 12469, 12470, 12471, 12472, 12473, 12474, 12475, 12476, 12477, 12478, + 12479, 12480, 12481, 12482, 12464, 12465, 12466, 12483, 12484, 12493, + 12485, 12486, 12487, 12488, 12490, 12491, 12492, 12494, 12495, 12496, + 12497, 12498, 12499, 12500, 12501, 12502, 12503, 12504, 12505, 12506, + 12507, 12508, 12509, 12510, 12511, 12512, 12513, 12520, 12521, 12514, + 12515, 12522, 12523, 12524, 12525, 12516, 12517, 12518, 12519, 12526, + 12527, 12528, 12529, 12534, 12530, 12531, 12535, 12536, 12537, 12532, + 12533, 12538, 12539, 12540, 12541, 12547, 12548, 12543, 12544, 12549, + 12550, 12551, 12552, 12553, 12545, 12546, 12554, 12555, 12562, 12563, + 12564, 12556, 12557, 12565, 12566, 12558, 12559, 12560, 12561, 12567, + 12570, 12571, 12572, 12573, 12568, 12569, 12574, 12583, 12584, 12585, + 12576, 12577, 12578, 12586, 12579, 12580, 12587, 12581, 12582, 12588, + 12589, 12590, 12591, 12592, 12593, 12594, 12595, 12596, 12609, 12597, + 12598, 12599, 12600, 12601, 12602, 12603, 12604, 12605, 12606, 12607, + 12608, 12610, 12611, 12612, 12613, 12633, 12634, 12635, 12636, 12637, + 12614, 12615, 12616, 12617, 12618, 12619, 12620, 12621, 12622, 12623, + 12624, 12625, 12626, 12627, 12628, 12629, 12630, 12631, 12632, 11626, + 11627, 11628, 11629, 11630, 11631, 11623, 11624, 11625, 11632, 11633, + 11637, 11638, 11639, 11640, 11641, 11642, 11643, 11644, 11645, 11646, + 11647, 11648, 11649, 11650, 11651, 11652, 11653, 11654, 11655, 11656, + 11634, 11635, 11636, 12489, 12542, 11978, 12003, 12000, 12002, 11999, + 12270, 11659, 11836, 12004, 12001, 11998, 11658, 11835, 11657, 11834, + 12097, 11902, 11976, 12015, 11977, 12416, 12575, 12093, 12084, 12088, + 12095, 12091, 12085, 12090, 12087, 12094, 12083, 12089, 12096, 12092, + 12086, 12082, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 12653, 12654, 12655, 12656, 12657, 12658, 12659, 12660, + 12661, 12662, 12663, 12664, 12665, 12666, 12667, 12668, 12669, 12670, + 12671, 12672, 12673, 12674, 12675, 12676, 12677, 12678, 12679, 12680, + 12681, 12682, 12683, 12684, 12685, 12686, 12687, 12688, 12689, 12690, + 12691, 12692, 12693, 12694, 12695, 12696, 12697, 12698, 12699, 12700, + 12701, 12702, 12703, 12704, 12705, 12706, 12707, 12708, 12709, 12710, + 12711, 12712, 12713, 12714, 12715, 12716, 12717, 12718, 12719, 12720, + 12721, 12722, 12723, 12724, 12725, 12726, 12727, 12728, 12729, 12730, + 12731, 12732, 12733, 12734, 12735, 12736, 12737, 12738, 12739, 12740, + 12741, 12742, 12743, 12744, 12745, 12746, 12747, 12748, 12749, 12750, + 12751, 12752, 12753, 12754, 12755, 12756, 12757, 12758, 12759, 12760, + 12761, 12762, 12763, 12764, 12765, 12766, 12767, 12768, 12769, 12770, + 12771, 12772, 12773, 12774, 12775, 12776, 12777, 12778, 12779, 12780, + 12781, 12782, 12783, 12784, 12785, 12786, 12787, 12788, 12789, 12790, + 12791, 12792, 12793, 12794, 12795, 12796, 12797, 12798, 12799, 12800, + 12801, 12802, 12803, 12804, 12805, 12806, 12807, 12808, 12809, 12810, + 12811, 12812, 12813, 12814, 12815, 12816, 12817, 12818, 12819, 12820, + 12821, 12822, 12823, 12824, 12825, 12826, 12827, 12828, 12829, 12830, + 12831, 12832, 12833, 12834, 12835, 12836, 12837, 12838, 12839, 12840, + 12841, 12842, 12843, 12844, 12845, 12846, 12847, 12848, 12849, 12850, + 12851, 12852, 12853, 12854, 12855, 12856, 12857, 12858, 12859, 12860, + 12861, 12862, 12863, 12864, 12865, 12866, 12867, 12868, 12869, 12870, + 12871, 12872, 12873, 12874, 12875, 12876, 12877, 12878, 12879, 12880, + 12881, 12882, 12883, 12884, 12885, 12886, 12887, 12888, 12889, 12890, + 12891, 12892, 12893, 12894, 12895, 12896, 12897, 12898, 12899, 12900, + 12901, 12902, 12903, 12904, 12905, 12906, 12907, 12908, 12909, 12910, + 12911, 12912, 12913, 12914, 12915, 12916, 12917, 12918, 12919, 12920, + 12921, 12922, 12923, 12924, 12925, 12926, 12927, 12928, 12929, 12930, + 12931, 12932, 12933, 12934, 12935, 12936, 12937, 12938, 12939, 12940, + 12941, 12942, 12943, 12944, 12945, 12946, 12947, 12948, 12949, 12950, + 12951, 12952, 12953, 12954, 12955, 12956, 12957, 12958, 12959, 12960, + 12961, 12962, 12963, 12964, 12965, 12966, 12967, 12968, 12969, 12970, + 12971, 12972, 12973, 12974, 12975, 12976, 12977, 12978, 12979, 12980, + 12981, 12982, 12983, 12984, 12985, 12986, 12987, 12988, 12989, 12990, + 12991, 12992, 12993, 12994, 12995, 12996, 12997, 12998, 12999, 13000, + 13001, 13002, 13003, 13004, 13005, 13006, 13007, 13008, 13009, 13010, + 13011, 13012, 13013, 13014, 13015, 13016, 13017, 13018, 13019, 13020, + 13021, 13022, 13023, 13024, 13025, 13026, 13027, 13028, 13029, 13030, + 13031, 13032, 13033, 13034, 13035, 13036, 13037, 13038, 13039, 13040, + 13041, 13042, 13043, 13044, 13045, 13046, 13047, 13048, 13049, 13050, + 13051, 13052, 13053, 13054, 13055, 13056, 13057, 13058, 13059, 13060, + 13061, 13062, 13063, 13064, 13065, 13066, 13067, 13068, 13069, 13070, + 13071, 13072, 13073, 13074, 13075, 13076, 13077, 13078, 13079, 13080, + 13081, 13082, 13083, 13084, 13085, 13086, 13087, 13088, 13089, 13090, + 13091, 13092, 13093, 13094, 13095, 13096, 13097, 13098, 13099, 13100, + 13101, 13102, 13103, 13104, 13105, 13106, 13107, 13108, 13109, 13110, + 13111, 13112, 13113, 13114, 13115, 13116, 13117, 13118, 13119, 13120, + 13121, 13122, 13123, 13124, 13125, 13126, 13127, 13128, 13129, 13130, + 13131, 13132, 13133, 13134, 13135, 13136, 13137, 13138, 13139, 13140, + 13141, 13142, 13143, 13144, 13145, 13146, 13147, 13148, 13149, 13150, + 13151, 13152, 13153, 13154, 13155, 13156, 13157, 13158, 13159, 13160, + 13161, 13162, 13163, 13164, 13165, 13166, 13167, 13168, 13169, 13170, + 13171, 13172, 13173, 13174, 13175, 13176, 13177, 13178, 13179, 13180, + 13181, 13182, 13183, 13184, 13185, 13186, 13187, 13188, 13189, 13190, + 13191, 13192, 13193, 13194, 13195, 13196, 13197, 13198, 13199, 13200, + 13201, 13202, 13203, 13204, 13205, 13206, 13207, 13208, 13209, 13210, + 13211, 13212, 13213, 13214, 13215, 13216, 13217, 13218, 13219, 13220, + 13221, 13222, 13223, 13224, 13225, 13226, 13227, 13228, 13229, 13230, + 13231, 13232, 13233, 13234, 13235, 13236, 13237, 13238, 13239, 13240, + 13241, 13242, 13243, 13244, 13245, 13246, 13247, 13248, 13249, 13250, + 13251, 13252, 13253, 13254, 13255, 13256, 13257, 13258, 13259, 13260, + 13261, 13262, 13263, 13264, 13265, 13266, 13267, 13268, 13269, 13270, + 13271, 13272, 13273, 13274, 13275, 13276, 13277, 13278, 13279, 13280, + 13281, 13282, 13283, 13284, 13285, 13286, 13287, 13288, 13289, 13290, + 13291, 13292, 13293, 13294, 13295, 13296, 13297, 13298, 13299, 13300, + 13301, 13302, 13303, 13304, 13305, 13306, 13307, 13308, 13309, 13310, + 13311, 13312, 13313, 13314, 13315, 13316, 13317, 13318, 13319, 13320, + 13321, 13322, 13323, 13324, 13325, 13326, 13327, 13328, 13329, 13330, + 13331, 13332, 13333, 13334, 13335, 13336, 13337, 13338, 13339, 13340, + 13341, 13342, 13343, 13344, 13345, 13346, 13347, 13348, 13349, 13350, + 13351, 13352, 13353, 13354, 13355, 13356, 13357, 13358, 13359, 13360, + 13361, 13362, 13363, 13364, 13365, 13366, 13367, 13368, 13369, 13370, + 13371, 13372, 13373, 13374, 13375, 13376, 13377, 13378, 13379, 13380, + 13381, 13382, 13383, 13384, 13385, 13386, 13387, 13388, 13389, 13390, + 13391, 13392, 13393, 13394, 13395, 13396, 13397, 13398, 13399, 13400, + 13401, 13402, 13403, 13404, 13405, 13406, 13407, 13408, 13409, 13410, + 13411, 13412, 13413, 13414, 13415, 13416, 13417, 13418, 13419, 13420, + 13421, 13422, 13423, 13424, 13425, 13426, 13427, 13428, 13429, 13430, + 13431, 13432, 13433, 13434, 13435, 13436, 13437, 13438, 13439, 13440, + 13441, 13442, 13443, 13444, 13445, 13446, 13447, 13448, 13449, 13450, + 13451, 13452, 13453, 13454, 13455, 13456, 13457, 13458, 13459, 13460, + 13461, 13462, 13463, 13464, 13465, 13466, 13467, 13468, 13469, 13470, + 13471, 13472, 13473, 13474, 13475, 13476, 13477, 13478, 13479, 13480, + 13481, 13482, 13483, 13484, 13485, 13486, 13487, 13488, 13489, 13490, + 13491, 13492, 13493, 13494, 13495, 13496, 13497, 13498, 13499, 13500, + 13501, 13502, 13503, 13504, 13505, 13506, 13507, 13508, 13509, 13510, + 13511, 13512, 13513, 13514, 13515, 13516, 13517, 13518, 13519, 13520, + 13521, 13522, 13523, 13524, 13525, 13526, 13527, 13528, 13529, 13530, + 13531, 13532, 13533, 13534, 13535, 13536, 13537, 13538, 13539, 13540, + 13541, 13542, 13543, 13544, 13545, 13546, 13547, 13548, 13549, 13550, + 13551, 13552, 13553, 13554, 13555, 13556, 13557, 13558, 13559, 13560, + 13561, 13562, 13563, 13564, 13565, 13566, 13567, 13568, 13569, 13570, + 13571, 13572, 13573, 13574, 13575, 13576, 13577, 13578, 13579, 13580, + 13581, 13582, 13583, 13584, 13585, 13586, 13587, 13588, 13589, 13590, + 13591, 13592, 13593, 13594, 13595, 13596, 13597, 13598, 13599, 13600, + 13601, 13602, 13603, 13604, 13605, 13606, 13607, 13608, 13609, 13610, + 13611, 13612, 13613, 13614, 13615, 13616, 13617, 13618, 13619, 13620, + 13621, 13622, 13623, 13624, 13625, 13626, 13627, 13628, 13629, 13630, + 13631, 13632, 13633, 13634, 13635, 13636, 13637, 13638, 13639, 13640, + 13641, 13642, 13643, 13644, 13645, 13646, 13647, 13648, 13649, 13650, + 13651, 13652, 13653, 13654, 13655, 13656, 13657, 13658, 13659, 13660, + 13661, 13662, 13663, 13664, 13665, 13666, 13667, 13668, 13669, 13670, + 13671, 13672, 13673, 13674, 13675, 13676, 13677, 13678, 13679, 13680, + 13681, 13682, 13683, 13684, 13685, 13686, 13687, 13688, 13689, 13690, + 13691, 13692, 13693, 13694, 13695, 13696, 13697, 13698, 13699, 13700, + 13701, 13702, 13703, 13704, 13705, 13706, 13707, 13708, 13709, 13710, + 13711, 13712, 13713, 13714, 13715, 13716, 13717, 13718, 13719, 13720, + 13721, 13722, 13723, 13724, 13725, 13726, 13727, 13728, 13729, 13730, + 13731, 13732, 13733, 13734, 13735, 13736, 13737, 13738, 13739, 13740, + 13741, 13742, 13743, 13744, 13745, 13746, 13747, 13748, 13749, 13750, + 13751, 13752, 13753, 13754, 13755, 13756, 13757, 13758, 13759, 13760, + 13761, 13762, 13763, 13764, 13765, 13766, 13767, 13768, 13769, 13770, + 13771, 13772, 13773, 13774, 13775, 13776, 13777, 13778, 13779, 13780, + 13781, 13782, 13783, 13784, 13785, 13786, 13787, 13788, 13789, 13790, + 13791, 13792, 13793, 13794, 13795, 13796, 13797, 13798, 13799, 13800, + 13801, 13802, 13803, 13804, 13805, 13806, 13807, 13808, 13809, 13810, + 13811, 13812, 13813, 13814, 13815, 13816, 13817, 13818, 13819, 13820, + 13821, 13822, 13823, 13824, 13825, 13826, 13827, 13828, 13829, 13830, + 13831, 13832, 13833, 13834, 13835, 13836, 13837, 13838, 13839, 13840, + 13841, 13842, 13843, 13844, 13845, 13846, 13847, 13848, 13849, 13850, + 13851, 13852, 13853, 13854, 13855, 13856, 13857, 13858, 13859, 13860, + 13861, 13862, 13863, 13864, 13865, 13866, 13867, 13868, 13869, 13870, + 13871, 13872, 13873, 13874, 13875, 13876, 13877, 13878, 13879, 13880, + 13881, 13882, 13883, 13884, 13885, 13886, 13887, 13888, 13889, 13890, + 13891, 13892, 13893, 13894, 13895, 13896, 13897, 13898, 13899, 13900, + 13901, 13902, 13903, 13904, 13905, 13906, 13907, 13908, 13909, 13910, + 13911, 13912, 13913, 13914, 13915, 13916, 13917, 13918, 13919, 13920, + 13921, 13922, 13923, 13924, 13925, 13926, 13927, 13928, 13929, 13930, + 13931, 13932, 13933, 13934, 13935, 13936, 13937, 13938, 13939, 13940, + 13941, 13942, 13943, 13944, 13945, 13946, 13947, 13948, 13949, 13950, + 13951, 13952, 13953, 13954, 13955, 13956, 13957, 13958, 13959, 13960, + 13961, 13962, 13963, 13964, 13965, 13966, 13967, 13968, 13969, 13970, + 13971, 13972, 13973, 13974, 13975, 13976, 13977, 13978, 13979, 13980, + 13981, 13982, 13983, 13984, 13985, 13986, 13987, 13988, 13989, 13990, + 13991, 13992, 13993, 13994, 13995, 13996, 13997, 13998, 13999, 14000, + 14001, 14002, 14003, 14004, 14005, 14006, 14007, 14008, 14009, 14010, + 14011, 14012, 14013, 14014, 14015, 14016, 14017, 14018, 14019, 14020, + 14021, 14022, 14023, 14024, 14025, 14026, 14027, 14028, 14029, 14030, + 14031, 14032, 14033, 14034, 14035, 14036, 14037, 14038, 14039, 14040, + 14041, 14042, 14043, 14044, 14045, 14046, 14047, 14048, 14049, 14050, + 14051, 14052, 14053, 14054, 14055, 14056, 14057, 14058, 14059, 14060, + 14061, 14062, 14063, 14064, 14065, 14066, 14067, 14068, 14069, 14070, + 14071, 14072, 14073, 14074, 14075, 14076, 14077, 14078, 14079, 14080, + 14081, 14082, 14083, 14084, 14085, 14086, 14087, 14088, 14089, 14090, + 14091, 14092, 14093, 14094, 14095, 14096, 14097, 14098, 14099, 14100, + 14101, 14102, 14103, 14104, 14105, 14106, 14107, 14108, 14109, 14110, + 14111, 14112, 14113, 14114, 14115, 14116, 14117, 14118, 14119, 14120, + 14121, 14122, 14123, 14124, 14125, 14126, 14127, 14128, 14129, 14130, + 14131, 14132, 14133, 14134, 14135, 14136, 14137, 14138, 14139, 14140, + 14141, 14142, 14143, 14144, 14145, 14146, 14147, 14148, 14149, 14150, + 14151, 14152, 14153, 14154, 14155, 14156, 14157, 14158, 14159, 14160, + 14161, 14162, 14163, 14164, 14165, 14166, 14167, 14168, 14169, 14170, + 14171, 14172, 14173, 14174, 14175, 14176, 14177, 14178, 14179, 14180, + 14181, 14182, 14183, 14184, 14185, 14186, 14187, 14188, 14189, 14190, + 14191, 14192, 14193, 14194, 14195, 14196, 14197, 14198, 14199, 14200, + 14201, 14202, 14203, 14204, 14205, 14206, 14207, 14208, 14209, 14210, + 14211, 14212, 14213, 14214, 14215, 14216, 14217, 14218, 14219, 14220, + 14221, 14222, 14223, 14224, 14225, 14226, 14227, 14228, 14229, 14230, + 14231, 14232, 14233, 14234, 14235, 14236, 14237, 14238, 14239, 14240, + 14241, 14242, 14243, 14244, 14245, 14246, 14247, 14248, 14249, 14250, + 14251, 14252, 14253, 14254, 14255, 14256, 14257, 14258, 14259, 14260, + 14261, 14262, 14263, 14264, 14265, 14266, 14267, 14268, 14269, 14270, + 14271, 14272, 14273, 14274, 14275, 14276, 14277, 14278, 14279, 14280, + 14281, 14282, 14283, 14284, 14285, 14286, 14287, 14288, 14289, 14290, + 14291, 14292, 14293, 14294, 14295, 14296, 14297, 14298, 14299, 14300, + 14301, 14302, 14303, 14304, 14305, 14306, 14307, 14308, 14309, 14310, + 14311, 14312, 14313, 14314, 14315, 14316, 14317, 14318, 14319, 14320, + 14321, 14322, 14323, 14324, 14325, 14326, 14327, 14328, 14329, 14330, + 14331, 14332, 14333, 14334, 14335, 14336, 14337, 14338, 14339, 14340, + 14341, 14342, 14343, 14344, 14345, 14346, 14347, 14348, 14349, 14350, + 14351, 14352, 14353, 14354, 14355, 14356, 14357, 14358, 14359, 14360, + 14361, 14362, 14363, 14364, 14365, 14366, 14367, 14368, 14369, 14370, + 14371, 14372, 14373, 14374, 14375, 14376, 14377, 14378, 14379, 14380, + 14381, 14382, 14383, 14384, 14385, 14386, 14387, 14388, 14389, 14390, + 14391, 14392, 14393, 14394, 14395, 14396, 14397, 14398, 14399, 14400, + 14401, 14402, 14403, 14404, 14405, 14406, 14407, 14408, 14409, 14410, + 14411, 14412, 14413, 14414, 14415, 14416, 14417, 14418, 14419, 14420, + 14421, 14422, 14423, 14424, 14425, 14426, 14427, 14428, 14429, 14430, + 14431, 14432, 14433, 14434, 14435, 14436, 14437, 14438, 14439, 14440, + 14441, 14442, 14443, 14444, 14445, 14446, 14447, 14448, 14449, 14450, + 14451, 14452, 14453, 14454, 14455, 14456, 14457, 14458, 14459, 14460, + 14461, 14462, 14463, 14464, 14465, 14466, 14467, 14468, 14469, 14470, + 14471, 14472, 14473, 14474, 14475, 14476, 14477, 14478, 14479, 14480, + 14481, 14482, 14483, 14484, 14485, 14486, 14487, 14488, 14489, 14490, + 14491, 14492, 14493, 14494, 14495, 14496, 14497, 14498, 14499, 14500, + 14501, 14502, 14503, 14504, 14505, 14506, 14507, 14508, 14509, 14510, + 14511, 14512, 14513, 14514, 14515, 14516, 14517, 14518, 14519, 14520, + 14521, 14522, 14523, 14524, 14525, 14526, 14527, 14528, 14529, 14530, + 14531, 14532, 14533, 14534, 14535, 14536, 14537, 14538, 14539, 14540, + 14541, 14542, 14543, 14544, 14545, 14546, 14547, 14548, 14549, 14550, + 14551, 14552, 14553, 14554, 14555, 14556, 14557, 14558, 14559, 14560, + 14561, 14562, 14563, 14564, 14565, 14566, 14567, 14568, 14569, 14570, + 14571, 14572, 14573, 14574, 14575, 14576, 14577, 14578, 14579, 14580, + 14581, 14582, 14583, 14584, 14585, 14586, 14587, 14588, 14589, 14590, + 14591, 14592, 14593, 14594, 14595, 14596, 14597, 14598, 14599, 14600, + 14601, 14602, 14603, 14604, 14605, 14606, 14607, 14608, 14609, 14610, + 14611, 14612, 14613, 14614, 14615, 14616, 14617, 14618, 14619, 14620, + 14621, 14622, 14623, 14624, 14625, 14626, 14627, 14628, 14629, 14630, + 14631, 14632, 14633, 14634, 14635, 14636, 14637, 14638, 14639, 14640, + 14641, 14642, 14643, 14644, 14645, 14646, 14647, 14648, 14649, 14650, + 14651, 14652, 14653, 14654, 14655, 14656, 14657, 14658, 14659, 14660, + 14661, 14662, 14663, 14664, 14665, 14666, 14667, 14668, 14669, 14670, + 14671, 14672, 14673, 14674, 14675, 14676, 14677, 14678, 14679, 14680, + 14681, 14682, 14683, 14684, 14685, 14686, 14687, 14688, 14689, 14690, + 14691, 14692, 14693, 14694, 14695, 14696, 14697, 14698, 14699, 14700, + 14701, 14702, 14703, 14704, 14705, 14706, 14707, 14708, 14709, 14710, + 14711, 14712, 14713, 14714, 14715, 14716, 14717, 14718, 14719, 14720, + 14721, 14722, 14723, 14724, 14725, 14726, 14727, 14728, 14729, 14730, + 14731, 14732, 14733, 14734, 14735, 14736, 14737, 14738, 14739, 14740, + 14741, 14742, 14743, 14744, 14745, 14746, 14747, 14748, 14749, 14750, + 14751, 14752, 14753, 14754, 14755, 14756, 14757, 14758, 14759, 14760, + 14761, 14762, 14763, 14764, 14765, 14766, 14767, 14768, 14769, 14770, + 14771, 14772, 14773, 14774, 14775, 14776, 14777, 14778, 14779, 14780, + 14781, 14782, 14783, 14784, 14785, 14786, 14787, 14788, 14789, 14790, + 14791, 14792, 14793, 14794, 14795, 14796, 14797, 14798, 14799, 14800, + 14801, 14802, 14803, 14804, 14805, 14806, 14807, 14808, 14809, 14810, + 14811, 14812, 14813, 14814, 14815, 14816, 14817, 14818, 14819, 14820, + 14821, 14822, 14823, 14824, 14825, 14826, 14827, 14828, 14829, 14830, + 14831, 14832, 14833, 14834, 14835, 14836, 14837, 14838, 14839, 14840, + 14841, 14842, 14843, 14844, 14845, 14846, 14847, 14848, 14849, 14850, + 14851, 14852, 14853, 14854, 14855, 14856, 14857, 14858, 14859, 14860, + 14861, 14862, 14863, 14864, 14865, 14866, 14867, 14868, 14869, 14870, + 14871, 14872, 14873, 14874, 14875, 14876, 14877, 14878, 14879, 14880, + 14881, 14882, 14883, 14884, 14885, 14886, 14887, 14888, 14889, 14890, + 14891, 14892, 14893, 14894, 14895, 14896, 14897, 14898, 14899, 14900, + 14901, 14902, 14903, 14904, 14905, 14906, 14907, 14908, 14909, 14910, + 14911, 14912, 14913, 14914, 14915, 14916, 14917, 14918, 14919, 14920, + 14921, 14922, 14923, 14924, 14925, 14926, 14927, 14928, 14929, 14930, + 14931, 14932, 14933, 14934, 14935, 14936, 14937, 14938, 14939, 14940, + 14941, 14942, 14943, 14944, 14945, 14946, 14947, 14948, 14949, 14950, + 14951, 14952, 14953, 14954, 14955, 14956, 14957, 14958, 14959, 14960, + 14961, 14962, 14963, 14964, 14965, 14966, 14967, 14968, 14969, 14970, + 14971, 14972, 14973, 14974, 14975, 14976, 14977, 14978, 14979, 14980, + 14981, 14982, 14983, 14984, 14985, 14986, 14987, 14988, 14989, 14990, + 14991, 14992, 14993, 14994, 14995, 14996, 14997, 14998, 14999, 15000, + 15001, 15002, 15003, 15004, 15005, 15006, 15007, 15008, 15009, 15010, + 15011, 15012, 15013, 15014, 15015, 15016, 15017, 15018, 15019, 15020, + 15021, 15022, 15023, 15024, 15025, 15026, 15027, 15028, 15029, 15030, + 15031, 15032, 15033, 15034, 15035, 15036, 15037, 15038, 15039, 15040, + 15041, 15042, 15043, 15044, 15045, 15046, 15047, 15048, 15049, 15050, + 15051, 15052, 15053, 15054, 15055, 15056, 15057, 15058, 15059, 15060, + 15061, 15062, 15063, 15064, 15065, 15066, 15067, 15068, 15069, 15070, + 15071, 15072, 15073, 15074, 15075, 15076, 15077, 15078, 15079, 15080, + 15081, 15082, 15083, 15084, 15085, 15086, 15087, 15088, 15089, 15090, + 15091, 15092, 15093, 15094, 15095, 15096, 15097, 15098, 15099, 15100, + 15101, 15102, 15103, 15104, 15105, 15106, 15107, 15108, 15109, 15110, + 15111, 15112, 15113, 15114, 15115, 15116, 15117, 15118, 15119, 15120, + 15121, 15122, 15123, 15124, 15125, 15126, 15127, 15128, 15129, 15130, + 15131, 15132, 15133, 15134, 15135, 15136, 15137, 15138, 15139, 15140, + 15141, 15142, 15143, 15144, 15145, 15146, 15147, 15148, 15149, 15150, + 15151, 15152, 15153, 15154, 15155, 15156, 15157, 15158, 15159, 15160, + 15161, 15162, 15163, 15164, 15165, 15166, 15167, 15168, 15169, 15170, + 15171, 15172, 15173, 15174, 15175, 15176, 15177, 15178, 15179, 15180, + 15181, 15182, 15183, 15184, 15185, 15186, 15187, 15188, 15189, 15190, + 15191, 15192, 15193, 15194, 15195, 15196, 15197, 15198, 15199, 15200, + 15201, 15202, 15203, 15204, 15205, 15206, 15207, 15208, 15209, 15210, + 15211, 15212, 15213, 15214, 15215, 15216, 15217, 15218, 15219, 15220, + 15221, 15222, 15223, 15224, 15225, 15226, 15227, 15228, 15229, 15230, + 15231, 15232, 15233, 15234, 15235, 15236, 15237, 15238, 15239, 15240, + 15241, 15242, 15243, 15244, 15245, 15246, 15247, 15248, 15249, 15250, + 15251, 15252, 15253, 15254, 15255, 15256, 15257, 15258, 15259, 15260, + 15261, 15262, 15263, 15264, 15265, 15266, 15267, 15268, 15269, 15270, + 15271, 15272, 15273, 15274, 15275, 15276, 15277, 15278, 15279, 15280, + 15281, 15282, 15283, 15284, 15285, 15286, 15287, 15288, 15289, 15290, + 15291, 15292, 15293, 15294, 15295, 15296, 15297, 15298, 15299, 15300, + 15301, 15302, 15303, 15304, 15305, 15306, 15307, 15308, 15309, 15310, + 15311, 15312, 15313, 15314, 15315, 15316, 15317, 15318, 15319, 15320, + 15321, 15322, 15323, 15324, 15325, 15326, 15327, 15328, 15329, 15330, + 15331, 15332, 15333, 15334, 15335, 15336, 15337, 15338, 15339, 15340, + 15341, 15342, 15343, 15344, 15345, 15346, 15347, 15348, 15349, 15350, + 15351, 15352, 15353, 15354, 15355, 15356, 15357, 15358, 15359, 15360, + 15361, 15362, 15363, 15364, 15365, 15366, 15367, 15368, 15369, 15370, + 15371, 15372, 15373, 15374, 15375, 15376, 15377, 15378, 15379, 15380, + 15381, 15382, 15383, 15384, 15385, 15386, 15387, 15388, 15389, 15390, + 15391, 15392, 15393, 15394, 15395, 15396, 15397, 15398, 15399, 15400, + 15401, 15402, 15403, 15404, 15405, 15406, 15407, 15408, 15409, 15410, + 15411, 15412, 15413, 15414, 15415, 15416, 15417, 15418, 15419, 15420, + 15421, 15422, 15423, 15424, 15425, 15426, 15427, 15428, 15429, 15430, + 15431, 15432, 15433, 15434, 15435, 15436, 15437, 15438, 15439, 15440, + 15441, 15442, 15443, 15444, 15445, 15446, 15447, 15448, 15449, 15450, + 15451, 15452, 15453, 15454, 15455, 15456, 15457, 15458, 15459, 15460, + 15461, 15462, 15463, 15464, 15465, 15466, 15467, 15468, 15469, 15470, + 15471, 15472, 15473, 15474, 15475, 15476, 15477, 15478, 15479, 15480, + 15481, 15482, 15483, 15484, 15485, 15486, 15487, 15488, 15489, 15490, + 15491, 15492, 15493, 15494, 15495, 15496, 15497, 15498, 15499, 15500, + 15501, 15502, 15503, 15504, 15505, 15506, 15507, 15508, 15509, 15510, + 15511, 15512, 15513, 15514, 15515, 15516, 15517, 15518, 15519, 15520, + 15521, 15522, 15523, 15524, 15525, 15526, 15527, 15528, 15529, 15530, + 15531, 15532, 15533, 15534, 15535, 15536, 15537, 15538, 15539, 15540, + 15541, 15542, 15543, 15544, 15545, 15546, 15547, 15548, 15549, 15550, + 15551, 15552, 15553, 15554, 15555, 15556, 15557, 15558, 15559, 15560, 15561, 15562, 15563, 15564, 15565, 15566, 15567, 15568, 15569, 15570, - 15571, 15572, 15573, 15574, 15575, 15576, 15577, 15578, 15553, 15554, - 15552, 15555, 15556, 15557, 15558, 15559, 15560, 15728, 15729, 15730, - 15731, 15732, 15727, 15739, 15740, 15741, 15742, 15733, 15734, 15735, - 15736, 15737, 15738, 15632, 15633, 15634, 15635, 15625, 15626, 15627, - 15628, 15629, 15630, 15631, 15619, 15620, 15621, 15622, 15623, 15624, - 15636, 15637, 15638, 15639, 15613, 15614, 15615, 15616, 15617, 15618, - 15706, 15707, 15708, 15709, 15710, 15711, 15712, 15713, 15714, 15715, - 15723, 15724, 15725, 15726, 15716, 15717, 15718, 15719, 15720, 15721, - 15722, 15611, 15612, 15836, 16849, 16848, 16850, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 15847, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 15840, 15839, 15841, 35762, 35762, 16898, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 16904, 16903, 16905, 16909, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 24744, 24745, 24746, 24747, 24748, 24749, - 24750, 24751, 24752, 24753, 24754, 24755, 24756, 24757, 24758, 24759, - 24760, 24761, 24762, 24763, 24764, 24765, 24766, 24767, 24768, 24769, - 24770, 24771, 24772, 24773, 24774, 24775, 24776, 24777, 24778, 24779, - 24780, 24781, 24782, 24783, 24784, 24785, 24786, 24787, 24788, 24789, - 24790, 24791, 24792, 24793, 24794, 24795, 24796, 24797, 24798, 24799, - 24800, 24801, 24802, 24803, 24804, 24805, 24806, 24807, 24808, 24809, - 24810, 24811, 24812, 24813, 24814, 24815, 24816, 24817, 24818, 24819, - 24820, 24821, 24822, 24823, 24824, 24825, 24826, 24827, 24828, 24829, - 24830, 24831, 24832, 24833, 24834, 24835, 24836, 24837, 24838, 24839, - 24840, 24841, 24842, 24843, 24844, 24845, 24846, 24847, 24848, 24849, - 24850, 24851, 24852, 24853, 24854, 24855, 24856, 24857, 24858, 24859, - 24860, 24861, 24862, 24863, 24864, 24865, 24866, 24867, 24868, 24869, - 24870, 24871, 24872, 24873, 24874, 24875, 24876, 24877, 24878, 24879, - 24880, 24881, 24882, 24883, 24884, 24885, 24886, 24887, 24888, 24889, - 24890, 24891, 24892, 24893, 24894, 24895, 24896, 24897, 24898, 24899, - 24900, 24901, 24902, 24903, 24904, 24905, 24906, 24907, 24908, 24909, - 24910, 24911, 24912, 24913, 24914, 24915, 24916, 24917, 24918, 24919, - 24920, 24921, 24922, 24923, 24924, 24925, 24926, 24927, 24928, 24929, - 24930, 24931, 24932, 24933, 24934, 24935, 24936, 24937, 24938, 24939, - 24940, 24941, 24942, 24943, 24944, 24945, 24946, 24947, 24948, 24949, - 24950, 24951, 24952, 24953, 24954, 24955, 24956, 24957, 24958, 24959, - 24960, 24961, 24962, 24963, 24964, 24965, 24966, 24967, 24968, 24969, - 24970, 24971, 24972, 24973, 24974, 24975, 24976, 24977, 24978, 24979, - 24980, 24981, 24982, 24983, 24984, 24985, 24986, 24987, 24988, 24989, - 24990, 24991, 24992, 24993, 24994, 24995, 24996, 24997, 24998, 24999, - 25000, 25001, 25002, 25003, 25004, 25005, 25006, 25007, 25008, 25009, - 25010, 25011, 25012, 25013, 25014, 25015, 25016, 25017, 25018, 25019, - 25020, 25021, 25022, 25023, 25024, 25025, 25026, 25027, 25028, 25029, - 25030, 25031, 25032, 25033, 25034, 25035, 25036, 25037, 25038, 25039, - 25040, 25041, 25042, 25043, 25044, 25045, 25046, 25047, 25048, 25049, - 25050, 25051, 25052, 25053, 25054, 25055, 25056, 25057, 25058, 25059, - 25060, 25061, 25062, 25063, 25064, 25065, 25066, 25067, 25068, 25069, - 25070, 25071, 25072, 25073, 25074, 25075, 25076, 25077, 25078, 25079, - 25080, 25081, 25082, 25083, 25084, 25085, 25086, 25087, 25088, 25089, - 25090, 25091, 25092, 25093, 25094, 25095, 25096, 25097, 25098, 25099, - 25100, 25101, 25102, 25103, 25104, 25105, 25106, 25107, 25108, 25109, - 25110, 25111, 25112, 25113, 25114, 25115, 25116, 25117, 25118, 25119, - 25120, 25121, 25122, 25123, 25124, 25125, 25126, 25127, 25128, 25129, - 25130, 25131, 25132, 25133, 25134, 25135, 25136, 25137, 25138, 25139, - 35762, 35762, 35762, 35762, 11196, 11194, 11143, 11176, 11103, 11116, - 11120, 11201, 11097, 11184, 11105, 11147, 11146, 11098, 11104, 11118, - 11150, 11179, 11172, 11099, 11119, 11173, 11197, 11123, 11151, 11124, - 11129, 11107, 11152, 11125, 11130, 11110, 11153, 11127, 11132, 11108, - 11109, 11161, 11162, 11128, 11133, 11114, 11165, 11126, 11131, 11111, - 11154, 11115, 11112, 11113, 11159, 11160, 11157, 11158, 11178, 11177, - 11186, 11192, 11189, 11164, 11163, 11117, 11106, 11155, 11156, 11095, - 11170, 11140, 11138, 11096, 11198, 11100, 11199, 11175, 11183, 11101, - 11167, 11148, 11166, 11121, 11200, 11180, 11102, 11195, 11181, 11122, - 11149, 11182, 11174, 11139, 11142, 11141, 11191, 11187, 11193, 11190, - 11188, 11137, 11136, 11135, 11134, 11145, 11144, 11168, 11171, 11169, - 11185, 35762, 35762, 35762, 35762, 35762, 11080, 11093, 11092, 11087, - 11094, 11074, 11067, 11066, 11063, 11065, 11068, 11069, 11064, 35762, - 35762, 35762, 11076, 11072, 11075, 11070, 11079, 11078, 11071, 11077, - 11073, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 11081, 11085, - 11088, 11083, 11091, 11090, 11084, 11089, 11086, 11082, 35762, 35762, - 11205, 11202, 11203, 11204, 27932, 27931, 27933, 27934, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35578, 35583, 35617, 35580, 35585, 35613, 35607, 35601, 35624, 35603, - 35597, 35621, 35579, 35584, 35618, 35581, 35586, 35614, 35608, 35602, - 35625, 35604, 35598, 35622, 35619, 35605, 35612, 35599, 35600, 35623, - 35606, 35582, 35628, 35593, 35610, 35620, 35631, 35630, 35596, 35629, - 35590, 35589, 35627, 35611, 35609, 35588, 35762, 35762, 35633, 35634, - 35635, 35626, 35576, 35591, 35594, 35595, 35573, 35574, 35592, 35615, - 35616, 35577, 35632, 35575, 35587, 35572, 35754, 35755, 35752, 35753, - 35756, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35652, 35653, 35669, 35641, 35650, 35749, 35704, 35705, 35672, 35673, - 35706, 35636, 35670, 35750, 35648, 35645, 35647, 35646, 35644, 35746, - 35745, 35748, 35747, 35742, 35741, 35744, 35743, 35642, 35676, 35638, - 35649, 35637, 35674, 35687, 35688, 35686, 35685, 35639, 35683, 35684, - 35682, 35680, 35681, 35679, 35678, 35689, 35691, 35692, 35690, 35654, - 35677, 35643, 35651, 35751, 35693, 35698, 35695, 35699, 35696, 35701, - 35702, 35697, 35694, 35700, 35675, 35703, 35736, 35733, 35724, 35734, - 35735, 35725, 35713, 35712, 35740, 35710, 35709, 35707, 35711, 35708, - 35727, 35732, 35726, 35730, 35731, 35728, 35729, 35656, 35660, 35659, - 35658, 35657, 35739, 35737, 35738, 35663, 35668, 35667, 35666, 35665, - 35664, 35714, 35723, 35716, 35721, 35715, 35719, 35720, 35717, 35718, - 35722, 35655, 35662, 35640, 35661, 35671, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 4893, 4765, 4886, 4868, 4869, - 4937, 4938, 4818, 4915, 4875, 4949, 4950, 4841, 4720, 4771, 4918, 4819, - 4723, 4724, 4913, 4925, 4864, 4801, 4892, 4744, 4941, 4813, 4827, 4823, - 4916, 4878, 4907, 4871, 4940, 4726, 4728, 4828, 4894, 4879, 4936, 4718, - 4896, 4910, 4912, 4866, 4920, 4848, 4766, 4928, 4919, 4838, 4719, 4778, - 4809, 4930, 4816, 4884, 4888, 4831, 4742, 4895, 4873, 4876, 4815, 4953, - 4883, 4832, 4931, 4908, 4807, 4814, 4865, 4870, 4882, 4834, 4881, 4839, - 4885, 4822, 4826, 4952, 4725, 4721, 4951, 4840, 4774, 4745, 4863, 4939, - 4880, 4889, 4872, 4717, 4849, 4877, 4874, 4770, 4842, 4716, 4932, 4773, - 4911, 4914, 4743, 4772, 4897, 4954, 4934, 4890, 4929, 4933, 4887, 4935, - 4891, 4804, 4730, 4769, 4867, 4922, 4923, 4921, 4924, 4817, 4767, 4942, - 4943, 4906, 4830, 4763, 4835, 4836, 4837, 4727, 4729, 4762, 4927, 4917, - 4833, 4843, 4847, 4846, 4845, 4844, 4803, 4800, 4799, 4757, 4759, 4760, - 4758, 4926, 4731, 4811, 4750, 4714, 4708, 4709, 4712, 4713, 4711, 4710, - 4715, 4856, 4851, 4852, 4850, 4861, 4860, 4859, 4858, 4862, 4854, 4812, - 4722, 4777, 4776, 4775, 4857, 4855, 4853, 4805, 4806, 4768, 4810, 4808, - 4779, 4786, 4782, 4790, 4785, 4794, 4784, 4783, 4780, 4781, 4788, 4789, - 4796, 4793, 4791, 4740, 4741, 4739, 4787, 4795, 4948, 4753, 4751, 4754, - 4756, 4755, 4752, 4944, 4946, 4945, 4947, 4797, 4798, 4747, 4746, 4748, - 4749, 4902, 4905, 4903, 4904, 4898, 4901, 4899, 4900, 4761, 4764, 4909, - 4738, 4732, 4737, 4735, 4734, 4733, 4736, 4820, 4824, 4821, 4825, 4829, - 4802, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 23803, 23680, 23696, 23788, 23675, 23800, 23738, 23789, 23787, - 23676, 23672, 23799, 23666, 23784, 23785, 23786, 23691, 23692, 23629, - 23630, 23625, 23624, 23755, 23826, 23823, 23698, 23697, 23804, 23805, - 23699, 23708, 23709, 23710, 23669, 23701, 23702, 23703, 23682, 23683, - 35762, 35762, 23746, 23679, 23681, 23707, 23706, 23751, 23750, 23802, - 23801, 23778, 23779, 23665, 23671, 23767, 23768, 23782, 23783, 23747, - 23849, 23721, 23781, 23690, 23807, 23825, 23809, 23754, 23847, 23770, - 23670, 23810, 23811, 23834, 23835, 23838, 23839, 23836, 23837, 23830, - 23831, 23832, 23833, 23744, 23745, 23840, 23841, 23769, 23845, 23752, - 23749, 23633, 23634, 23628, 23848, 23720, 23780, 23689, 23806, 23824, - 23808, 23753, 23654, 23655, 23658, 23659, 23660, 23693, 23694, 23695, - 23637, 23644, 23645, 23646, 23647, 23648, 23622, 23687, 23623, 23688, - 23621, 23685, 23620, 23684, 23635, 23653, 23661, 23652, 23638, 23639, - 23636, 23663, 23718, 23717, 23642, 23664, 23649, 23656, 23662, 23641, - 23657, 23790, 23813, 23852, 23777, 23740, 23700, 23668, 23677, 23714, - 23713, 23829, 23842, 23851, 23843, 23844, 23756, 23759, 23760, 23761, - 23762, 23763, 23764, 23765, 23766, 23757, 23758, 23722, 23748, 23686, - 23678, 23640, 23643, 23650, 23651, 23772, 23771, 23719, 23712, 23711, - 23850, 23673, 23674, 23739, 23735, 23626, 23793, 23795, 23741, 23743, - 23796, 23798, 23704, 23705, 23737, 23736, 23627, 23794, 23742, 23797, - 23821, 23820, 23822, 23819, 23815, 23816, 23817, 23818, 23667, 23715, - 23716, 23812, 23846, 23774, 23632, 23791, 23631, 23827, 23775, 23776, - 23792, 23828, 23773, 23723, 23726, 23730, 23733, 23732, 23731, 23727, - 23728, 23724, 23725, 23729, 23814, 23734, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 14240, 14228, 14251, - 14252, 14234, 14253, 14254, 14255, 14256, 14241, 14242, 14243, 14244, - 14245, 14246, 14247, 14248, 14249, 14250, 14229, 14230, 14231, 14232, - 14233, 14235, 14236, 14237, 14238, 14239, 13960, 13968, 13981, 13989, - 13995, 13996, 13961, 13962, 13963, 13964, 13965, 13966, 13967, 13969, - 13970, 13971, 13972, 13973, 13974, 13975, 13976, 13977, 13978, 13979, - 13980, 13982, 13983, 13984, 13985, 13986, 13987, 13988, 13990, 13991, - 13992, 13993, 13994, 7968, 7967, 7969, 14020, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 17140, 17141, - 17138, 17136, 17127, 17126, 17133, 17131, 17122, 17129, 17139, 17124, - 17137, 17135, 17128, 17125, 17134, 17132, 17123, 17130, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 22166, 22167, 22164, 22162, 22153, 22152, 22159, 22157, 22148, 22155, - 22165, 22150, 22163, 22161, 22154, 22151, 22160, 22158, 22149, 22156, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 23494, 10576, 10577, 10571, 10570, 10569, 31810, 31830, - 31851, 31799, 31844, 31808, 31794, 31853, 31798, 31812, 31818, 31841, - 31842, 31856, 31862, 31809, 31840, 31870, 31828, 31795, 31858, 31860, - 31827, 31873, 31807, 31822, 31819, 31800, 31814, 31797, 31855, 31848, - 31802, 31845, 31834, 31868, 31857, 31831, 31859, 31847, 31861, 31836, - 31824, 31867, 31837, 31823, 31854, 31863, 31833, 31869, 31806, 31850, - 31826, 31872, 31816, 31801, 31838, 31835, 31849, 31793, 31821, 31820, - 31871, 31865, 31843, 31813, 31811, 31817, 31825, 31864, 31866, 31839, - 31805, 31803, 31832, 31796, 31804, 31852, 31815, 31846, 31829, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 8366, 8364, 8363, - 8360, 8359, 8362, 8361, 8367, 8365, 8357, 8355, 8354, 8351, 8350, 8353, - 8352, 8358, 8356, 16051, 16050, 16049, 16048, 16047, 30395, 30394, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 21146, 21148, 21176, 21139, 21152, 21184, - 21155, 21185, 21157, 21186, 21159, 21161, 21178, 21180, 21163, 21166, - 21187, 21170, 21172, 21142, 21174, 21188, 21189, 21182, 21190, 21150, - 21194, 21196, 21229, 21191, 21200, 21203, 21205, 21237, 21207, 21238, - 21209, 21211, 21231, 21233, 21213, 21216, 21239, 21220, 21222, 21224, - 21227, 21240, 21241, 21235, 21242, 21198, 21634, 21636, 21666, 21640, - 21642, 21674, 21645, 21675, 21647, 21676, 21649, 21651, 21668, 21670, - 21653, 21656, 21677, 21660, 21662, 21630, 21664, 21678, 21679, 21672, - 21680, 21638, 21582, 21584, 21617, 21578, 21588, 21591, 21593, 35762, - 21595, 21625, 21597, 21599, 21619, 21621, 21601, 21604, 21626, 21608, - 21610, 21612, 21615, 21627, 21628, 21623, 21629, 21586, 21299, 21301, - 21331, 21305, 21307, 21339, 21310, 21340, 21312, 21341, 21314, 21316, - 21333, 21335, 21318, 21321, 21342, 21325, 21327, 21295, 21329, 21343, - 21344, 21337, 21345, 21303, 21353, 21355, 21390, 21359, 21361, 21364, - 21366, 21398, 21368, 21399, 21370, 21372, 21392, 21394, 21374, 21377, - 21400, 21381, 21383, 21385, 21388, 21401, 21402, 21396, 21403, 21357, - 22106, 35762, 22107, 22108, 35762, 35762, 22109, 35762, 35762, 22110, - 22111, 35762, 35762, 22112, 22113, 22114, 22115, 35762, 22116, 22117, - 22118, 22119, 22120, 22121, 22122, 22123, 22124, 22125, 22126, 22127, - 35762, 22128, 35762, 22129, 22130, 22131, 22132, 22133, 22134, 22135, - 35762, 22136, 22137, 22138, 22139, 22140, 22141, 22142, 22143, 22144, - 22145, 22146, 21243, 21244, 21245, 21246, 21247, 21248, 21249, 21250, - 21251, 21252, 21253, 21254, 21255, 21256, 21257, 21258, 21259, 21260, - 21261, 21262, 21263, 21264, 21265, 21266, 21267, 21268, 21269, 21270, - 21271, 21272, 21273, 21274, 21275, 21276, 21277, 21278, 21279, 21280, - 21281, 21282, 21283, 21284, 21285, 21286, 21287, 21288, 21289, 21290, - 21291, 21292, 21293, 21294, 21530, 21531, 35762, 21532, 21533, 21534, - 21535, 35762, 35762, 21536, 21537, 21538, 21539, 21540, 21541, 21542, - 21543, 35762, 21544, 21545, 21546, 21547, 21548, 21549, 21550, 35762, - 21551, 21552, 21553, 21554, 21555, 21556, 21557, 21558, 21559, 21560, - 21561, 21562, 21563, 21564, 21565, 21566, 21567, 21568, 21569, 21570, - 21571, 21572, 21573, 21574, 21575, 21576, 21475, 21476, 35762, 21477, - 21478, 21479, 21480, 35762, 21481, 21482, 21483, 21484, 21485, 35762, - 21486, 35762, 35762, 35762, 21487, 21488, 21489, 21490, 21491, 21492, - 21493, 35762, 21494, 21495, 21496, 21497, 21498, 21499, 21500, 21501, - 21502, 21503, 21504, 21505, 21506, 21507, 21508, 21509, 21510, 21511, - 21512, 21513, 21514, 21515, 21516, 21517, 21518, 21519, 21413, 21414, - 21415, 21416, 21417, 21418, 21419, 21420, 21421, 21422, 21423, 21424, - 21425, 21426, 21427, 21428, 21429, 21430, 21431, 21432, 21433, 21434, - 21435, 21436, 21437, 21438, 21439, 21440, 21441, 21442, 21443, 21444, - 21445, 21446, 21447, 21448, 21449, 21450, 21451, 21452, 21453, 21454, - 21455, 21456, 21457, 21458, 21459, 21460, 21461, 21462, 21463, 21464, - 22044, 22045, 22046, 22047, 22048, 22049, 22050, 22051, 22052, 22053, - 22054, 22055, 22056, 22057, 22058, 22059, 22060, 22061, 22062, 22063, - 22064, 22065, 22066, 22067, 22068, 22069, 22070, 22071, 22072, 22073, - 22074, 22075, 22076, 22077, 22078, 22079, 22080, 22081, 22082, 22083, - 22084, 22085, 22086, 22087, 22088, 22089, 22090, 22091, 22092, 22093, - 22094, 22095, 21876, 21878, 21908, 21882, 21884, 21916, 21887, 21917, - 21889, 21918, 21891, 21893, 21910, 21912, 21895, 21898, 21919, 21902, - 21904, 21872, 21906, 21920, 21921, 21914, 21922, 21880, 21930, 21932, - 21967, 21936, 21938, 21941, 21943, 21975, 21945, 21976, 21947, 21949, - 21969, 21971, 21951, 21954, 21977, 21958, 21960, 21962, 21965, 21978, - 21979, 21973, 21980, 21934, 21992, 21993, 21994, 21995, 21996, 21997, - 21998, 21999, 22000, 22001, 22002, 22003, 22004, 22005, 22006, 22007, - 22008, 22009, 22010, 22011, 22012, 22013, 22014, 22015, 22016, 22017, - 22018, 22019, 22020, 22021, 22022, 22023, 22024, 22025, 22026, 22027, - 22028, 22029, 22030, 22031, 22032, 22033, 22034, 22035, 22036, 22037, - 22038, 22039, 22040, 22041, 22042, 22043, 21766, 21768, 21798, 21772, - 21774, 21806, 21777, 21807, 21779, 21808, 21781, 21783, 21800, 21802, - 21785, 21788, 21809, 21792, 21794, 21762, 21796, 21810, 21811, 21804, - 21812, 21770, 21820, 21822, 21857, 21826, 21828, 21831, 21833, 21865, - 21835, 21866, 21837, 21839, 21859, 21861, 21841, 21844, 21867, 21848, - 21850, 21852, 21855, 21868, 21869, 21863, 21870, 21824, 21689, 21690, - 21691, 21692, 21693, 21694, 21695, 21696, 21697, 21698, 21699, 21700, - 21701, 21702, 21703, 21704, 21705, 21706, 21707, 21708, 21709, 21710, - 21711, 21712, 21713, 21714, 21715, 21716, 21717, 21718, 21719, 21720, - 21721, 21722, 21723, 21724, 21725, 21726, 21727, 21728, 21729, 21730, - 21731, 21732, 21733, 21734, 21735, 21736, 21737, 21738, 21739, 21740, - 21579, 21580, 35762, 35762, 21147, 21149, 21156, 21141, 21153, 21151, - 21154, 21143, 21158, 21160, 21162, 21179, 21181, 21183, 21164, 21169, - 21171, 21144, 21173, 21145, 21175, 21167, 21177, 21168, 21165, 21407, - 21195, 21197, 21206, 21193, 21201, 21199, 21202, 21225, 21208, 21210, - 21212, 21232, 21234, 21236, 21214, 21219, 21221, 21204, 21223, 21226, - 21228, 21217, 21230, 21218, 21215, 21409, 21405, 21412, 21406, 21408, - 21411, 21410, 21635, 21637, 21646, 21641, 21643, 21639, 21644, 21631, - 21648, 21650, 21652, 21669, 21671, 21673, 21654, 21659, 21661, 21632, - 21663, 21633, 21665, 21657, 21667, 21658, 21655, 21683, 21583, 21585, - 21594, 21581, 21589, 21587, 21590, 21613, 21596, 21598, 21600, 21620, - 21622, 21624, 21602, 21607, 21609, 21592, 21611, 21614, 21616, 21605, - 21618, 21606, 21603, 21685, 21681, 21688, 21682, 21684, 21687, 21686, - 21300, 21302, 21311, 21306, 21308, 21304, 21309, 21296, 21313, 21315, - 21317, 21334, 21336, 21338, 21319, 21324, 21326, 21297, 21328, 21298, - 21330, 21322, 21332, 21323, 21320, 21348, 21354, 21356, 21367, 21360, - 21362, 21358, 21363, 21386, 21369, 21371, 21373, 21393, 21395, 21397, - 21375, 21380, 21382, 21365, 21384, 21387, 21389, 21378, 21391, 21379, - 21376, 21350, 21346, 21404, 21347, 21349, 21352, 21351, 21877, 21879, - 21888, 21883, 21885, 21881, 21886, 21873, 21890, 21892, 21894, 21911, - 21913, 21915, 21896, 21901, 21903, 21874, 21905, 21875, 21907, 21899, - 21909, 21900, 21897, 21925, 21931, 21933, 21944, 21937, 21939, 21935, - 21940, 21963, 21946, 21948, 21950, 21970, 21972, 21974, 21952, 21957, - 21959, 21942, 21961, 21964, 21966, 21955, 21968, 21956, 21953, 21927, - 21923, 21981, 21924, 21926, 21929, 21928, 21767, 21769, 21778, 21773, - 21775, 21771, 21776, 21763, 21780, 21782, 21784, 21801, 21803, 21805, - 21786, 21791, 21793, 21764, 21795, 21765, 21797, 21789, 21799, 21790, - 21787, 21815, 21821, 21823, 21834, 21827, 21829, 21825, 21830, 21853, - 21836, 21838, 21840, 21860, 21862, 21864, 21842, 21847, 21849, 21832, - 21851, 21854, 21856, 21845, 21858, 21846, 21843, 21817, 21813, 21871, - 21814, 21816, 21819, 21818, 21140, 21192, 35762, 35762, 21471, 21473, - 21470, 21469, 21466, 21465, 21468, 21467, 21474, 21472, 21526, 21528, - 21525, 21524, 21521, 21520, 21523, 21522, 21529, 21527, 22102, 22104, - 22101, 22100, 22097, 22096, 22099, 22098, 22105, 22103, 21988, 21990, - 21987, 21986, 21983, 21982, 21985, 21984, 21991, 21989, 21747, 21749, - 21746, 21745, 21742, 21741, 21744, 21743, 21750, 21748, 28184, 28140, - 28165, 28381, 28336, 28122, 28185, 28149, 28298, 28250, 28226, 28187, - 28190, 28147, 28191, 28141, 28192, 28214, 28209, 28247, 28188, 28194, - 28203, 28204, 28195, 28201, 28205, 28143, 28277, 28186, 28215, 28144, - 28224, 28193, 28223, 28210, 28249, 28248, 28189, 28208, 28218, 28219, - 28222, 28221, 28289, 28197, 28198, 28199, 28271, 28239, 28202, 28206, - 28200, 28196, 28270, 28231, 28269, 28268, 28228, 28227, 28230, 28220, - 28217, 28216, 28266, 28265, 28267, 28238, 28314, 28318, 28317, 28315, - 28316, 28159, 28305, 28335, 28307, 28320, 28312, 28321, 28313, 28322, - 28311, 28169, 28170, 28334, 28380, 28308, 28309, 28310, 28306, 28332, - 28319, 28330, 28323, 28331, 28329, 28327, 28324, 28325, 28326, 28328, - 28156, 28162, 28160, 28161, 28365, 28364, 28172, 28164, 28175, 28178, - 28173, 28176, 28174, 28177, 28182, 28179, 28139, 28374, 28379, 28376, - 28378, 28353, 28355, 28333, 28363, 28356, 28360, 28354, 28362, 28361, - 28359, 28121, 28211, 28146, 28338, 28124, 28347, 28213, 28212, 28339, - 28252, 28255, 28254, 28253, 28262, 28302, 28151, 28375, 28133, 28258, - 28261, 28256, 28257, 28350, 28260, 28349, 28132, 28131, 28259, 28150, - 28348, 28130, 28225, 28145, 28340, 28357, 28123, 28207, 28142, 28278, - 28358, 28134, 28286, 28282, 28284, 28155, 28377, 28135, 28279, 28281, - 28280, 28285, 28283, 28373, 28251, 28148, 28180, 28367, 28368, 28366, - 28168, 28346, 28126, 28125, 28275, 28351, 28273, 28154, 28263, 28274, - 28372, 28272, 28276, 28264, 28152, 28181, 28171, 28352, 28137, 28138, - 28136, 28153, 28157, 28158, 28370, 28371, 28369, 28337, 28240, 28342, - 28243, 28244, 28245, 28242, 28246, 28241, 28235, 28236, 28237, 28234, - 28232, 28163, 28233, 28229, 28166, 28167, 28344, 28345, 28341, 28343, - 28128, 28129, 28127, 28287, 28292, 28295, 28296, 28297, 28303, 28288, - 28290, 28291, 28299, 28294, 28300, 28301, 28293, 28183, 28304, 28695, - 28694, 28693, 28715, 28714, 28713, 28670, 28669, 28668, 28054, 28053, - 28052, 28656, 28655, 28654, 28666, 28667, 28662, 28665, 28661, 28664, - 28663, 28111, 28114, 28110, 28113, 28112, 28660, 28530, 28531, 28532, - 28533, 28528, 28529, 28534, 28574, 28479, 28592, 28591, 28589, 28590, - 28593, 28568, 28571, 28569, 28570, 28567, 28598, 28597, 28595, 28596, - 28544, 28543, 28542, 28548, 28547, 28546, 28545, 28566, 28565, 28564, - 28541, 28540, 28539, 28614, 28613, 28612, 28588, 28587, 28586, 28711, - 28710, 28709, 28708, 28707, 28706, 28712, 28705, 28704, 28703, 28448, - 28447, 28445, 28446, 28452, 28451, 28449, 28450, 28440, 28439, 28437, - 28438, 28444, 28443, 28441, 28442, 28502, 28501, 28499, 28500, 28503, - 28517, 28520, 28518, 28519, 28476, 28507, 28506, 28505, 28504, 28462, - 28475, 28474, 28473, 28463, 28461, 28460, 28459, 28526, 28525, 28524, - 28523, 28522, 28521, 28698, 28697, 28696, 28701, 28700, 28699, 28702, - 28561, 28560, 28558, 28559, 28552, 28551, 28549, 28550, 28557, 28556, - 28579, 28578, 28575, 28580, 28585, 28582, 28581, 28601, 28600, 28599, - 28604, 28603, 28602, 28555, 28563, 28562, 28651, 28648, 28647, 28594, - 28554, 28577, 28584, 28609, 28653, 28650, 28646, 28553, 28576, 28583, - 28608, 28652, 28649, 28645, 28607, 28606, 28605, 28466, 28465, 28483, - 28481, 28482, 28480, 28492, 28490, 28491, 28489, 28509, 28508, 28640, - 28637, 28643, 28468, 28467, 28484, 28485, 28487, 28486, 28496, 28494, - 28495, 28493, 28511, 28510, 28641, 28638, 28644, 28472, 28471, 28469, - 28470, 28464, 28488, 28497, 28512, 28513, 28514, 28639, 28636, 28642, - 28498, 28538, 28536, 28537, 28535, 28458, 28456, 28454, 28457, 28455, - 28453, 28611, 28610, 28516, 28515, 28573, 28572, 28478, 28477, 28070, - 28069, 28065, 28068, 28072, 28071, 28066, 28067, 28064, 28073, 28383, - 28390, 28388, 28387, 28385, 28386, 28384, 28389, 28103, 28101, 28102, - 28079, 28078, 28077, 28062, 28060, 28061, 28063, 28118, 28117, 28116, - 28097, 28098, 28094, 28075, 28074, 28093, 28095, 28092, 28096, 28076, - 28091, 28089, 28090, 28086, 28088, 28087, 28080, 28082, 28081, 28084, - 28083, 28085, 28057, 28056, 28055, 28680, 28679, 28681, 28100, 28617, - 28616, 28619, 28618, 28047, 28049, 28046, 28048, 28051, 28050, 28412, - 28410, 28411, 28417, 28419, 28418, 28414, 28416, 28415, 28431, 28430, - 28429, 28423, 28424, 28425, 28426, 28427, 28428, 28420, 28422, 28421, - 28432, 28433, 28434, 28401, 28399, 28400, 28413, 28436, 28435, 28688, - 28687, 28685, 28686, 28684, 28689, 28683, 28682, 28672, 28677, 28675, - 28676, 28673, 28674, 28678, 28615, 28527, 28620, 28382, 28099, 28658, - 28657, 28120, 28115, 28659, 28691, 28690, 28692, 28716, 28391, 28392, - 28393, 28394, 28395, 28396, 28397, 28398, 28109, 28409, 28408, 28403, - 28406, 28405, 28402, 28404, 28407, 28059, 28119, 28671, 28058, 28717, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 28104, 28105, 28106, 28107, 28108, - 35762, 28628, 28629, 28630, 28631, 28632, 28633, 28634, 28635, 28621, - 28622, 28623, 28624, 28625, 28626, 28627, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 18911, 19184, 18676, 19189, - 18663, 19034, 19294, 19186, 19280, 19249, 18656, 18890, 18891, 19284, - 18651, 18703, 18677, 19028, 18827, 19008, 18888, 19278, 19165, 19253, - 18899, 18828, 18972, 19124, 19254, 18794, 19201, 35762, 35762, 35762, - 35762, 35762, 35762, 18818, 19021, 19066, 19170, 19208, 19244, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 7924, 7926, 7936, 7930, 7912, 7937, 7944, 35762, 7943, - 7917, 7914, 7913, 7911, 7948, 7932, 7931, 7933, 7947, 7934, 7935, 7919, - 7922, 7946, 7928, 7945, 35762, 35762, 7920, 7923, 7927, 7921, 7939, 7938, - 7940, 35762, 7942, 7918, 35762, 7941, 7916, 7925, 7915, 7929, 35762, - 35762, 35762, 35762, 35762, 23081, 23050, 23078, 23076, 23052, 23074, - 23071, 23072, 23073, 23080, 23057, 23058, 23082, 23061, 23059, 23054, - 23067, 23083, 23056, 23079, 23066, 23075, 23065, 23068, 23053, 23070, - 23051, 23064, 23048, 23077, 23049, 23062, 23060, 10214, 10192, 10212, - 10199, 10195, 10209, 10206, 10207, 10208, 10213, 10197, 10215, 10211, - 10198, 10216, 10196, 10201, 10205, 10210, 10204, 10202, 10203, 10200, - 10191, 10194, 10193, 23055, 23069, 23063, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 7841, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 25172, 25156, 25147, 25158, 25165, 25155, 25160, 25151, - 25177, 25181, 25183, 25186, 25150, 25145, 25180, 25170, 25154, 25153, - 25184, 25146, 25157, 25178, 25164, 25182, 25185, 25152, 25174, 25159, - 25149, 25169, 25148, 25166, 25171, 25173, 25179, 25163, 25161, 25162, - 25187, 25188, 25167, 25168, 25175, 25176, 25189, 35762, 35762, 35762, - 25198, 25202, 25201, 25204, 25203, 25200, 25199, 25194, 25192, 25191, - 25195, 25193, 25196, 25197, 35762, 35762, 25211, 25213, 25210, 25209, - 25206, 25205, 25208, 25207, 25214, 25212, 35762, 35762, 35762, 35762, - 25190, 25144, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 32479, 32462, 32482, 32472, 32476, 32473, 32478, 32466, - 32465, 32481, 32469, 32484, 32483, 32475, 32474, 32480, 32477, 32463, - 32457, 32464, 32458, 32486, 32467, 32459, 32468, 32460, 32485, 32471, - 32461, 32470, 32487, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 33793, 33792, 33824, 33825, 33826, 33828, 33817, 33820, 33806, 33809, - 33821, 33813, 33810, 33827, 33823, 33822, 33830, 33835, 33834, 33833, - 33819, 33798, 33797, 33832, 33831, 33818, 33829, 33801, 33803, 33807, - 33814, 33805, 33812, 33811, 33800, 33795, 33796, 33804, 33799, 33802, - 33794, 33808, 33815, 33816, 33839, 33840, 33837, 33838, 33847, 33849, - 33846, 33845, 33842, 33841, 33844, 33843, 33850, 33848, 35762, 35762, - 35762, 35762, 35762, 33836, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 24159, 24162, 24161, 24163, 24160, 24142, 24146, 24144, - 24143, 24145, 24154, 24157, 24155, 24158, 24156, 24164, 24165, 24166, - 24167, 24168, 24147, 24149, 24152, 24153, 24148, 24151, 24150, 24172, - 24169, 24173, 24171, 24170, 24180, 24182, 24179, 24178, 24175, 24174, - 24177, 24176, 24183, 24181, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 12689, 12695, 12693, 12690, 12692, 12691, - 12694, 35762, 12644, 12688, 12686, 12685, 35762, 12632, 12631, 35762, - 12643, 12642, 12641, 12628, 12627, 12640, 12639, 12638, 12637, 12636, - 12635, 12630, 12629, 12634, 12633, 35762, 22398, 22399, 22400, 22466, - 22487, 22475, 22440, 22573, 22401, 22402, 22406, 22525, 22510, 22515, - 22439, 22591, 22542, 22461, 22445, 22536, 22405, 22403, 22404, 22451, - 22495, 22548, 22586, 22411, 22407, 22415, 22552, 22490, 22501, 22528, - 22416, 22413, 22412, 22565, 22503, 22563, 22541, 22532, 22530, 22531, - 22592, 22571, 22410, 22425, 22417, 22564, 22509, 22534, 22470, 22593, - 22421, 22422, 22423, 22479, 22471, 22455, 22556, 22507, 22408, 22414, - 22409, 22482, 22577, 22578, 22418, 22419, 22420, 22493, 22448, 22498, - 22465, 22426, 22424, 22427, 22551, 22508, 22557, 22459, 22569, 22428, - 22432, 22436, 22505, 22477, 22545, 22526, 22429, 22433, 22434, 22476, - 22468, 22533, 22486, 22594, 22497, 22435, 22430, 22431, 22513, 22566, - 22572, 22444, 22584, 22437, 22496, 22446, 22543, 22483, 22522, 22453, - 22535, 22485, 22452, 22590, 22442, 22488, 22438, 22484, 22512, 22538, - 22550, 22520, 22555, 22523, 22481, 22499, 22580, 22549, 22516, 22559, - 22587, 22562, 22560, 22583, 22456, 22570, 22463, 22491, 22447, 22478, - 22454, 22502, 22458, 22537, 22457, 22517, 22443, 22581, 22472, 22567, - 22568, 22585, 22558, 22500, 22540, 22529, 22492, 22467, 22441, 22506, - 22514, 22553, 22519, 22449, 22544, 22489, 22504, 22473, 22474, 22576, - 22518, 22521, 22524, 22588, 22511, 22460, 22462, 22547, 22589, 22539, - 22527, 22579, 22582, 22554, 22574, 22480, 22546, 22469, 22561, 22450, - 22575, 22494, 22464, 35762, 35762, 22602, 22600, 22599, 22596, 22595, - 22598, 22597, 22603, 22601, 22393, 22392, 22396, 22394, 22391, 22395, - 22397, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 27, 9, 24, 14, 29, 21, 34, 28, 37, 39, 35, 40, 41, 10, 30, - 32, 18, 15, 31, 42, 13, 25, 36, 23, 12, 20, 33, 19, 38, 17, 11, 26, 16, - 22, 62, 44, 59, 49, 64, 56, 69, 63, 72, 74, 70, 75, 76, 45, 65, 67, 53, - 50, 66, 77, 48, 60, 71, 58, 47, 55, 68, 54, 73, 52, 46, 61, 51, 57, 85, - 86, 79, 84, 43, 78, 83, 82, 35762, 35762, 35762, 35762, 93, 95, 92, 91, - 88, 87, 90, 89, 96, 94, 35762, 35762, 35762, 35762, 80, 81, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 16234, 16220, 16215, 16195, 16190, 16208, 16203, 16183, 16198, - 16223, 16218, 16213, 16193, 16188, 16209, 16204, 16184, 16199, 16235, - 16221, 16216, 16196, 16191, 16211, 16206, 16186, 16201, 16236, 16222, - 16217, 16197, 16192, 16212, 16207, 16187, 16202, 16224, 16219, 16214, - 16194, 16189, 16210, 16205, 16185, 16200, 16181, 16182, 16172, 16179, - 16180, 16232, 16230, 16229, 16226, 16225, 16228, 16227, 16233, 16231, - 16238, 16174, 16173, 16175, 16237, 16178, 16177, 16176, 16171, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 26069, 26064, 26059, 26039, 26034, - 26052, 26047, 26027, 26042, 26067, 26062, 26057, 26037, 26032, 26053, - 26048, 26028, 26043, 26070, 26065, 26060, 26040, 26035, 26055, 26050, - 26030, 26045, 26071, 26066, 26061, 26041, 26036, 26056, 26051, 26031, - 26046, 26068, 26063, 26058, 26038, 26033, 26054, 26049, 26029, 26044, - 26026, 26019, 26021, 26011, 26013, 26014, 26016, 26023, 26022, 26017, - 26012, 26015, 26020, 26018, 26025, 26024, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 2231, 2240, 2237, 2135, 35762, 2247, 2235, 2239, 2227, 2243, 2234, 2183, - 2238, 2245, 2210, 2232, 2241, 2211, 2246, 2242, 2209, 2230, 2229, 2233, - 2228, 2134, 2236, 2244, 2105, 2107, 2106, 2108, 35762, 2147, 2145, 35762, - 2142, 35762, 35762, 2141, 35762, 2149, 2144, 2152, 2146, 2151, 2139, - 2154, 2148, 2140, 2153, 35762, 2138, 2137, 2136, 2143, 35762, 2155, - 35762, 2150, 35762, 35762, 35762, 35762, 35762, 35762, 2219, 35762, - 35762, 35762, 35762, 2220, 35762, 2221, 35762, 2224, 35762, 2223, 2216, - 2226, 35762, 2217, 2225, 35762, 2215, 35762, 35762, 2218, 35762, 2214, - 35762, 2222, 35762, 2212, 35762, 2213, 35762, 2202, 2199, 35762, 2196, - 35762, 35762, 2195, 2190, 2204, 2198, 35762, 2200, 2206, 2193, 2208, - 2203, 2194, 2207, 35762, 2192, 2191, 2189, 2197, 35762, 2188, 2201, 2205, - 2186, 35762, 2187, 35762, 2160, 2172, 2170, 2180, 2163, 2182, 2168, 2162, - 2166, 2175, 35762, 2178, 2171, 2177, 2157, 2161, 2173, 2158, 2181, 2174, - 2156, 2167, 2165, 2159, 2164, 2179, 2169, 2176, 35762, 35762, 35762, - 35762, 35762, 2122, 2119, 2130, 35762, 2133, 2117, 2121, 2115, 2125, - 35762, 2128, 2120, 2127, 2110, 2132, 2123, 2111, 2131, 2124, 2109, 2116, - 2114, 2112, 2113, 2129, 2118, 2126, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 2184, 2185, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 20707, 20723, 20738, 20714, 20742, 20741, 20739, - 20719, 20736, 20733, 20712, 20709, 20728, 20725, 20705, 20716, 20720, - 20737, 20734, 20713, 20710, 20729, 20726, 20706, 20717, 20718, 20735, - 20732, 20711, 20708, 20727, 20724, 20704, 20715, 20722, 20721, 20702, - 20745, 20731, 20730, 20743, 20740, 20744, 20703, 35762, 35762, 35762, - 35762, 10829, 10780, 10781, 10782, 10783, 10784, 10785, 10786, 10787, - 10788, 10789, 10790, 10791, 10792, 10793, 10794, 10795, 10796, 10797, - 10798, 10799, 10800, 10801, 10802, 10803, 10804, 10805, 10806, 10807, - 10808, 10809, 10810, 10811, 10812, 10813, 10814, 10815, 10816, 10817, - 10818, 10819, 10820, 10821, 10822, 10823, 10824, 10825, 10826, 10827, - 10828, 10879, 10830, 10831, 10832, 10833, 10834, 10835, 10836, 10837, - 10838, 10839, 10840, 10841, 10842, 10843, 10844, 10845, 10846, 10847, - 10848, 10849, 10850, 10851, 10852, 10853, 10854, 10855, 10856, 10857, - 10858, 10859, 10860, 10861, 10862, 10863, 10864, 10865, 10866, 10867, - 10868, 10869, 10870, 10871, 10872, 10873, 10874, 10875, 10876, 10877, - 10878, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 26691, 26761, 26736, 26732, 26695, 26700, 26720, - 26716, 26712, 26765, 26728, 26769, 26704, 26724, 26708, 35762, 35762, - 26762, 26737, 26733, 26696, 26701, 26721, 26717, 26713, 26766, 26729, - 26770, 26705, 26725, 26709, 26692, 35762, 26763, 26738, 26734, 26697, - 26702, 26722, 26718, 26714, 26767, 26730, 26771, 26706, 26726, 26710, - 26690, 35762, 26760, 26735, 26731, 26694, 26699, 26719, 26715, 26711, - 26764, 26727, 26768, 26703, 26723, 26707, 26693, 26698, 26742, 26739, - 26753, 26754, 26755, 26756, 26757, 26758, 26759, 26743, 26744, 26745, - 26746, 26747, 26748, 26749, 26750, 26751, 26752, 26740, 26741, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 10565, - 10564, 10549, 10561, 10558, 10543, 10540, 10555, 10552, 10567, 10546, - 10606, 10585, 6512, 6229, 6253, 26362, 26363, 26364, 26365, 26366, 26367, - 26368, 26369, 26370, 26371, 26372, 26373, 26374, 26375, 26376, 26377, - 26378, 26379, 26380, 26381, 26382, 26383, 26384, 26385, 26386, 26387, - 32455, 6343, 6344, 6240, 6511, 8343, 29496, 29497, 29498, 29499, 29500, + 15571, 15572, 15573, 15574, 15575, 15576, 15577, 15578, 15579, 15580, + 15581, 15582, 15583, 15584, 15585, 15586, 15587, 15588, 15589, 15590, + 15591, 15592, 15593, 15594, 15595, 15596, 15597, 15598, 15599, 15600, + 15601, 15602, 15603, 15604, 15605, 15606, 15607, 15608, 15609, 15610, + 15611, 15612, 15613, 15614, 15615, 15616, 15617, 15618, 15619, 15620, + 15621, 15622, 15623, 15624, 15625, 15626, 15627, 15628, 15880, 15881, + 15882, 15883, 15884, 15885, 15886, 15887, 15888, 15889, 15890, 15891, + 15892, 15893, 15894, 15895, 15896, 15897, 15898, 15899, 15900, 15901, + 15902, 15903, 15904, 15905, 15906, 15907, 15908, 15909, 15910, 15911, + 15912, 15913, 15914, 15915, 15916, 15917, 15918, 15919, 15920, 15921, + 15922, 15923, 15924, 15925, 15926, 15927, 15928, 15929, 15930, 15931, + 15932, 15933, 15934, 15935, 15936, 15937, 15938, 15939, 15940, 15941, + 15942, 15943, 15944, 15945, 15946, 15947, 15948, 15949, 15950, 15951, + 15952, 15953, 15954, 15955, 15956, 15957, 15958, 15959, 15960, 15961, + 15962, 15963, 15964, 15965, 15966, 15967, 15968, 15969, 15970, 15971, + 15972, 15973, 15974, 15975, 15976, 15977, 15978, 15979, 15980, 15981, + 15982, 15983, 15984, 15985, 15986, 15987, 15988, 15989, 15990, 15991, + 15992, 15993, 15994, 15995, 15996, 15997, 15998, 15999, 16000, 16001, + 16002, 16003, 16004, 16005, 16006, 16007, 16008, 16009, 16010, 16011, + 16012, 16013, 16014, 16015, 16016, 16017, 16018, 16019, 16020, 16021, + 16022, 16023, 16024, 16025, 16026, 16027, 16028, 16029, 16030, 16031, + 16032, 16033, 16034, 16035, 16036, 16037, 16038, 16039, 16040, 16041, + 16042, 16043, 16044, 16045, 16046, 16047, 16048, 16049, 16050, 16051, + 16052, 16053, 16054, 16055, 16056, 16057, 16058, 16059, 16060, 16061, + 16062, 16063, 16064, 16065, 16066, 16067, 16068, 16069, 16070, 16071, + 16072, 16073, 16074, 16075, 16076, 16077, 16078, 16079, 16080, 16081, + 16082, 16083, 16084, 16085, 16086, 16087, 16088, 16089, 16090, 16091, + 16092, 16093, 16094, 16095, 16096, 16097, 16098, 16099, 16100, 16101, + 16102, 16103, 16104, 16105, 16106, 16107, 16108, 16109, 16110, 16111, + 16112, 16113, 16114, 16115, 16116, 16117, 16118, 16119, 16120, 16121, + 16122, 16123, 16124, 16125, 16126, 16127, 16128, 16129, 16130, 16131, + 16132, 16133, 16134, 16135, 16136, 16137, 16138, 16139, 16140, 16141, + 16142, 16143, 16144, 16145, 16146, 16147, 16148, 16149, 16150, 16151, + 16152, 16153, 16154, 16155, 16156, 16157, 16158, 16159, 16160, 16161, + 16162, 16163, 16164, 16165, 16166, 16167, 16168, 16169, 16170, 16171, + 16172, 16173, 16174, 16175, 16176, 16177, 16178, 16179, 16180, 16181, + 16182, 16183, 16184, 16185, 16186, 16187, 16188, 16189, 16190, 16191, + 16192, 16193, 16194, 16195, 16196, 16197, 16198, 16199, 16200, 16201, + 16202, 16203, 16204, 16205, 16206, 16207, 16208, 16209, 16210, 16211, + 16212, 16213, 16214, 16215, 16216, 16217, 16218, 16219, 16220, 16221, + 16222, 16223, 16224, 16225, 16226, 16227, 16228, 16229, 16230, 16231, + 16232, 16233, 16234, 16235, 16236, 16237, 16238, 16239, 16240, 16241, + 16242, 16243, 16244, 16245, 16246, 16247, 16248, 16249, 16250, 16251, + 16252, 16253, 16254, 16255, 16256, 16257, 16258, 16259, 16260, 16261, + 16262, 16263, 16264, 16265, 16266, 16267, 16268, 16269, 16270, 16271, + 16272, 16273, 16274, 16275, 16276, 16277, 16278, 16279, 16280, 16281, + 16282, 16283, 16284, 16285, 16286, 16287, 16288, 16289, 16290, 16291, + 16292, 16293, 16294, 16295, 16296, 16297, 16298, 16299, 16300, 16301, + 16302, 16303, 16304, 16305, 16306, 16307, 16308, 16309, 16310, 16311, + 16312, 16313, 16314, 16315, 16316, 16317, 16318, 16319, 16320, 16321, + 16322, 16323, 16324, 16325, 16326, 16327, 16328, 16329, 16330, 16331, + 16332, 16333, 16334, 16335, 16336, 16337, 16338, 16339, 16340, 16341, + 16342, 16343, 16344, 16345, 16346, 16347, 16348, 16349, 16350, 16351, + 16352, 16353, 16354, 16355, 16356, 16357, 16358, 16359, 16360, 16361, + 16362, 16363, 16364, 16365, 16366, 16367, 16368, 16369, 16370, 16371, + 16372, 16373, 16374, 16375, 16376, 16377, 16378, 16379, 16380, 16381, + 16382, 16383, 16384, 16385, 16386, 16387, 16388, 16389, 16390, 16391, + 16392, 16393, 16394, 16395, 16396, 16397, 16398, 16399, 16400, 16401, + 16402, 16403, 16404, 16405, 16406, 16407, 16408, 16409, 16410, 16411, + 16412, 16413, 16414, 16415, 16416, 16417, 16418, 16419, 16420, 16421, + 16422, 16423, 16424, 16425, 16426, 16427, 16428, 16429, 16430, 16431, + 16432, 16433, 16434, 16435, 16436, 16437, 16438, 16439, 16440, 16441, + 16442, 16443, 16444, 16445, 16446, 16447, 16448, 16449, 16450, 16451, + 16452, 16453, 16454, 16455, 16456, 16457, 16458, 16459, 16460, 16461, + 16462, 16463, 16464, 16465, 16466, 16467, 16468, 16469, 16470, 16471, + 16472, 16473, 16474, 16475, 16476, 16477, 16478, 16479, 16480, 16481, + 16482, 16483, 16484, 16485, 16486, 16487, 16488, 16489, 16490, 16491, + 16492, 16493, 16494, 16495, 16496, 16497, 16498, 16499, 16500, 16501, + 16502, 16503, 16504, 16505, 16506, 16507, 16508, 16509, 16510, 16511, + 16512, 16513, 16514, 16515, 16516, 16517, 16518, 16519, 16520, 16521, + 16522, 16523, 16524, 16525, 16526, 16527, 16528, 16529, 16530, 16531, + 16532, 16533, 16534, 16535, 16536, 16537, 16538, 16539, 16540, 16541, + 16542, 16543, 16544, 16545, 16546, 16547, 16548, 16549, 16550, 16551, + 16552, 16553, 16554, 16555, 16556, 16557, 16558, 16559, 16560, 16561, + 16562, 16563, 16564, 16565, 16566, 16567, 16568, 16569, 16570, 16571, + 16572, 16573, 16574, 16575, 16576, 16577, 16578, 16579, 16580, 16581, + 16582, 16583, 16584, 16585, 16586, 16587, 16588, 16589, 16590, 16591, + 16592, 16593, 16594, 16595, 16596, 16597, 16598, 16599, 16600, 16601, + 16602, 16603, 16604, 16605, 16606, 16607, 16608, 16609, 16610, 16611, + 16612, 16613, 16614, 16615, 16616, 16617, 16618, 16619, 16620, 16621, + 16622, 16623, 16624, 16625, 16626, 16627, 16628, 16629, 16630, 16631, + 16632, 16633, 16634, 16635, 16636, 16637, 16638, 16639, 16640, 16641, + 16642, 16643, 16644, 16645, 16646, 16647, 15640, 15641, 15642, 15643, + 15644, 15645, 15646, 15647, 15648, 15649, 15650, 15651, 15652, 15653, + 15654, 15655, 15656, 15657, 15658, 15659, 15660, 15661, 15662, 15663, + 15664, 15665, 15666, 15667, 15668, 15669, 15670, 15671, 15672, 15673, + 15674, 15675, 15676, 15677, 15678, 15679, 15680, 15681, 15682, 15683, + 15684, 15685, 15686, 15687, 15688, 15689, 15690, 15691, 15692, 15693, + 15694, 15695, 15696, 15697, 15698, 15699, 15700, 15701, 15702, 15703, + 15704, 15705, 15706, 15707, 15708, 15709, 15710, 15711, 15712, 15713, + 15714, 15715, 15716, 15717, 15718, 15719, 15720, 15721, 15722, 15723, + 15724, 15725, 15726, 15727, 15728, 15729, 15730, 15731, 15732, 15733, + 15734, 15735, 15736, 15737, 15738, 15739, 15740, 15741, 15742, 15743, + 15744, 15745, 15746, 15747, 15748, 15749, 15750, 15751, 15752, 15753, + 15754, 15755, 15756, 15757, 15758, 15759, 15760, 15761, 15762, 15763, + 15764, 15765, 15766, 15767, 15768, 15769, 15770, 15771, 15772, 15773, + 15774, 15775, 15776, 15777, 15778, 15779, 15780, 15781, 15782, 15783, + 15784, 15785, 15786, 15787, 15788, 15789, 15790, 15791, 15792, 15793, + 15794, 15795, 15796, 15797, 15798, 15799, 15800, 15801, 15802, 15803, + 15804, 15805, 15806, 15807, 15808, 15809, 15810, 15811, 15812, 15813, + 15814, 15815, 15816, 15817, 15818, 15819, 15820, 15821, 15822, 15823, + 15824, 15825, 15826, 15827, 15828, 15829, 15830, 15831, 15832, 15833, + 15834, 15835, 15836, 15837, 15838, 15839, 15840, 15841, 15842, 15843, + 15844, 15845, 15846, 15847, 15848, 15849, 15850, 15851, 15852, 15853, + 15854, 15855, 15856, 15857, 15858, 15859, 15860, 15861, 15862, 15863, + 15864, 15865, 15866, 15867, 15868, 15869, 15870, 15871, 15872, 15873, + 15874, 15875, 15876, 15877, 15878, 15879, 15629, 15630, 15631, 15632, + 15633, 15634, 15635, 15636, 15637, 15638, 15639, 40951, 40951, 40951, + 40951, 40951, 445, 446, 447, 448, 449, 450, 451, 452, 453, 434, 435, 436, + 437, 438, 439, 440, 441, 442, 443, 444, 375, 376, 377, 378, 379, 380, + 373, 374, 381, 382, 383, 425, 426, 427, 428, 429, 430, 431, 432, 433, + 423, 424, 391, 387, 388, 392, 393, 394, 389, 390, 384, 385, 386, 395, + 396, 397, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 402, 403, + 404, 405, 406, 407, 398, 399, 400, 401, 408, 409, 410, 464, 465, 466, + 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, + 481, 482, 483, 415, 416, 417, 418, 419, 420, 421, 411, 412, 413, 414, + 422, 495, 496, 497, 498, 499, 500, 501, 484, 485, 486, 487, 492, 493, + 494, 502, 488, 489, 490, 491, 503, 504, 505, 506, 507, 510, 511, 512, + 513, 508, 509, 514, 515, 516, 517, 520, 521, 522, 523, 524, 518, 519, + 525, 526, 527, 528, 531, 532, 533, 534, 535, 529, 530, 536, 537, 538, + 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, + 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, 566, + 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, + 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, + 595, 596, 597, 598, 599, 607, 608, 600, 601, 602, 609, 610, 611, 612, + 603, 604, 613, 605, 606, 618, 619, 620, 621, 622, 614, 615, 616, 617, + 623, 624, 625, 651, 652, 653, 654, 655, 656, 657, 649, 650, 658, 659, + 671, 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, 684, + 685, 686, 687, 688, 689, 690, 691, 692, 693, 694, 695, 696, 697, 698, + 699, 700, 662, 663, 664, 665, 666, 667, 668, 660, 661, 669, 670, 701, + 702, 703, 704, 705, 706, 707, 708, 709, 710, 640, 641, 642, 643, 644, + 645, 646, 647, 648, 638, 639, 630, 631, 632, 633, 626, 627, 634, 635, + 636, 637, 628, 629, 713, 714, 715, 716, 717, 718, 719, 720, 721, 711, + 712, 805, 806, 807, 808, 809, 810, 811, 812, 813, 814, 724, 725, 726, + 727, 728, 729, 730, 731, 732, 722, 723, 751, 752, 748, 749, 750, 753, + 754, 755, 744, 745, 746, 747, 756, 757, 758, 815, 816, 817, 818, 819, + 820, 821, 822, 823, 824, 735, 736, 737, 738, 739, 740, 741, 742, 743, + 733, 734, 763, 764, 765, 766, 759, 760, 767, 768, 769, 761, 762, 770, + 796, 794, 795, 797, 798, 799, 800, 801, 802, 803, 804, 777, 773, 774, + 778, 771, 772, 779, 780, 775, 776, 781, 782, 783, 785, 786, 787, 784, + 788, 789, 790, 791, 792, 793, 856, 857, 858, 859, 860, 861, 862, 863, + 864, 865, 825, 826, 827, 828, 829, 830, 831, 832, 833, 834, 835, 866, + 867, 868, 869, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, 880, + 881, 882, 883, 884, 885, 886, 887, 888, 889, 890, 891, 892, 893, 894, + 895, 836, 837, 840, 841, 842, 843, 844, 845, 838, 839, 846, 847, 896, + 897, 898, 899, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, 910, + 911, 912, 913, 914, 915, 916, 917, 918, 919, 920, 921, 922, 923, 924, + 925, 848, 849, 850, 851, 852, 853, 854, 855, 927, 928, 929, 930, 931, + 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, 944, 945, + 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 926, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 18950, 18940, 18939, 18936, 18935, 18921, 18934, + 18933, 18938, 18937, 18943, 18928, 18927, 18924, 18923, 18948, 18930, + 18929, 18926, 18925, 18922, 18942, 18941, 18932, 18931, 18945, 18949, + 18946, 18944, 18947, 18953, 18960, 18961, 18956, 18957, 18962, 18963, + 18954, 18958, 18959, 18955, 18964, 18920, 18919, 18917, 18951, 18918, + 18952, 18971, 18973, 18970, 18969, 18966, 18965, 18968, 18967, 18974, + 18972, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 2847, + 2813, 2872, 2873, 2843, 2880, 2891, 2862, 2879, 2874, 2875, 2826, 2892, + 2849, 2828, 2833, 2840, 2886, 2858, 2816, 2852, 2887, 2848, 2823, 2824, + 2856, 2830, 2871, 2812, 2870, 2839, 2863, 2894, 2811, 2857, 2888, 2842, + 2841, 2837, 2814, 2869, 2846, 2876, 2829, 2866, 2877, 2893, 2821, 2883, + 2890, 2831, 2817, 2845, 2819, 2838, 2881, 2822, 2896, 2898, 2818, 2884, + 2834, 2878, 2855, 2882, 2860, 2867, 2851, 2895, 2850, 2832, 2864, 2835, + 2861, 2889, 2885, 2868, 2853, 2825, 2859, 2815, 2854, 2897, 2820, 2865, + 2844, 2836, 2930, 2946, 2944, 2943, 2909, 2915, 2904, 2950, 2914, 2906, + 2936, 2949, 2908, 2934, 2935, 2899, 2939, 2947, 2941, 2942, 2921, 2918, + 2933, 2902, 2900, 2901, 2907, 2937, 2954, 2928, 2926, 2951, 2940, 2948, + 2920, 2924, 2925, 2922, 2945, 2917, 2923, 2932, 2938, 2919, 2952, 2916, + 2953, 2903, 2913, 2912, 2910, 2927, 2931, 2911, 2905, 2929, 3006, 3026, + 3047, 3043, 3005, 2994, 3007, 2955, 2983, 2957, 3027, 3023, 2980, 3028, + 2998, 2977, 2960, 2956, 2962, 3046, 3025, 2988, 3018, 2986, 3048, 2967, + 2968, 3032, 2999, 3036, 3014, 3040, 3035, 3001, 3042, 2992, 2974, 3024, + 3002, 2970, 2985, 2990, 3021, 3037, 3004, 3051, 2995, 3020, 3017, 3050, + 3041, 2982, 3052, 3010, 2969, 3039, 3015, 3012, 2964, 3000, 2976, 2987, + 2972, 2966, 3011, 3009, 3044, 3003, 3019, 3022, 2965, 3016, 2996, 2973, + 3049, 2997, 3033, 3031, 2984, 2975, 2979, 2961, 2981, 2963, 2978, 3045, + 3013, 2991, 2989, 3034, 2959, 2958, 3008, 2993, 2971, 3029, 3030, 3038, + 3080, 3164, 3113, 3087, 3114, 3073, 3112, 3118, 3100, 3127, 3163, 3109, + 3143, 3110, 3057, 3156, 3141, 3116, 3148, 3061, 3165, 3063, 3150, 3085, + 3095, 3076, 3082, 3152, 3169, 3111, 3058, 3106, 3158, 3056, 3108, 3053, + 3096, 3090, 3068, 3097, 3092, 3091, 3133, 3089, 3086, 3074, 3119, 3078, + 3066, 3125, 3154, 3153, 3166, 3059, 3140, 3157, 3105, 3084, 3120, 3060, + 3136, 3131, 3126, 3071, 3099, 3088, 3103, 3167, 3135, 3168, 3098, 3122, + 3147, 3064, 3102, 3107, 3159, 3081, 3065, 3121, 3155, 3077, 3101, 3069, + 3104, 3117, 3115, 3055, 3062, 3137, 3161, 3160, 3128, 3139, 3070, 3083, + 3075, 3149, 3094, 3145, 3142, 3067, 3130, 3146, 3123, 3132, 3129, 3144, + 3134, 3093, 3072, 3138, 3162, 3124, 3079, 3151, 3054, 3223, 3300, 3210, + 3197, 3307, 3200, 3263, 3289, 3279, 3249, 3226, 3274, 3294, 3235, 3190, + 3310, 3282, 3228, 3264, 3299, 3192, 3202, 3251, 3241, 3207, 3238, 3242, + 3250, 3281, 3248, 3267, 3290, 3231, 3214, 3184, 3237, 3245, 3208, 3201, + 3229, 3225, 3291, 3283, 3277, 3222, 3230, 3315, 3183, 3304, 3311, 3271, + 3303, 3187, 3288, 3297, 3305, 3308, 3196, 3273, 3293, 3181, 3243, 3284, + 3213, 3211, 3256, 3260, 3180, 3302, 3191, 3323, 3254, 3316, 3212, 3224, + 3269, 3319, 3199, 3173, 3179, 3240, 3189, 3206, 3236, 3185, 3175, 3252, + 3265, 3313, 3227, 3255, 3257, 3272, 3216, 3174, 3259, 3217, 3182, 3172, + 3220, 3275, 3239, 3193, 3270, 3253, 3312, 3232, 3261, 3171, 3178, 3246, + 3324, 3301, 3326, 3325, 3198, 3262, 3292, 3295, 3221, 3287, 3314, 3233, + 3320, 3317, 3318, 3322, 3321, 3188, 3266, 3247, 3276, 3309, 3177, 3306, + 3204, 3215, 3280, 3278, 3234, 3244, 3285, 3286, 3170, 3258, 3268, 3203, + 3195, 3218, 3205, 3209, 3296, 3194, 3219, 3298, 3176, 3186, 3331, 3380, + 3333, 3378, 3358, 3371, 3352, 3335, 3361, 3360, 3340, 3370, 3350, 3346, + 3337, 3368, 3363, 3369, 3366, 3329, 3328, 3348, 3347, 3345, 3373, 3365, + 3374, 3349, 3354, 3351, 3375, 3355, 3362, 3353, 3357, 3327, 3343, 3344, + 3364, 3356, 3379, 3376, 3336, 3334, 3332, 3338, 3359, 3341, 3342, 3339, + 3372, 3330, 3367, 3377, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 28219, 28211, 28232, 28209, 28233, 28220, 28235, 28215, 28206, 28223, + 28221, 28231, 28205, 28213, 28208, 28210, 28216, 28214, 28212, 28225, + 28228, 28217, 28227, 28234, 28226, 28207, 28230, 28229, 28222, 28224, + 28218, 40951, 28244, 28246, 28243, 28242, 28239, 28238, 28241, 28240, + 28247, 28245, 40951, 40951, 40951, 40951, 28237, 28236, 35842, 35839, + 35840, 35841, 35794, 35791, 35792, 35793, 35846, 35843, 35844, 35845, + 35834, 35831, 35832, 35833, 35838, 35835, 35836, 35837, 35830, 35827, + 35828, 35829, 35790, 35787, 35788, 35789, 35822, 35819, 35820, 35821, + 35795, 35800, 35811, 35812, 35823, 35826, 35824, 35825, 35818, 35815, + 35816, 35817, 35806, 35803, 35804, 35805, 35857, 35856, 35855, 35807, + 35814, 35864, 35862, 35859, 35809, 35858, 35860, 35802, 35810, 35799, + 35801, 35798, 35849, 35853, 35861, 35808, 35813, 35851, 35848, 35854, + 35797, 35847, 35863, 35796, 35852, 35850, 35865, 40951, 35872, 35874, + 35871, 35870, 35867, 35866, 35869, 35868, 35875, 35873, 40951, 40951, + 40951, 40951, 40951, 40951, 3441, 3446, 3462, 3464, 3454, 3452, 3444, + 3438, 3445, 3458, 3453, 3449, 3460, 3443, 3439, 3461, 3448, 3459, 3463, + 3457, 3451, 3465, 3450, 3466, 3455, 3456, 3447, 3442, 3440, 3467, 40951, + 40951, 3434, 3436, 3437, 3435, 3433, 3468, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 30994, 30995, 31000, 31001, + 30988, 30989, 31004, 31005, 30996, 30997, 30986, 30987, 31006, 31007, + 30990, 30991, 31002, 31003, 31008, 31009, 30998, 30999, 30992, 30993, + 31010, 31011, 30984, 30985, 30930, 30921, 30927, 30918, 30923, 30929, + 30922, 30926, 30932, 30916, 30928, 30914, 30919, 30917, 30925, 30920, + 30924, 30933, 30931, 30915, 30938, 30937, 30935, 30934, 30936, 30940, + 30939, 30970, 30971, 30949, 30969, 30967, 30975, 30977, 30976, 30978, + 30968, 30960, 30973, 30958, 30980, 30953, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 31018, 31020, 31017, 31016, + 31013, 31012, 31015, 31014, 31021, 31019, 40951, 30945, 30942, 30944, + 30947, 30941, 30943, 30946, 40951, 30972, 30979, 30957, 30965, 30981, + 30956, 30963, 30974, 30962, 30983, 30964, 30959, 30966, 30982, 30961, + 30955, 30948, 30951, 30952, 30954, 30950, 40951, 40951, 40951, 40951, + 40951, 30902, 30909, 30901, 30900, 30910, 30898, 30895, 30913, 30905, + 30903, 30911, 30897, 30896, 30906, 30912, 30908, 30904, 30899, 30907, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 22525, 22522, 22527, 22521, + 22510, 22509, 22506, 22505, 22498, 22504, 22503, 22508, 22507, 22499, + 22495, 22494, 22491, 22490, 22497, 22496, 22493, 22492, 22500, 22512, + 22511, 22502, 22501, 22517, 22520, 22518, 22516, 22519, 22514, 22513, + 22515, 22528, 22534, 22531, 22532, 22533, 22529, 22535, 22530, 22526, + 22524, 22523, 22537, 22536, 22544, 22546, 22543, 22542, 22539, 22538, + 22541, 22540, 22547, 22545, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 26886, 26890, 26893, 26894, 26864, 26896, 26872, 26887, + 26891, 26882, 26881, 26883, 26871, 26863, 26884, 26880, 26877, 26878, + 26892, 26874, 26885, 26888, 26870, 26868, 26895, 26879, 26876, 26866, + 26889, 26875, 26865, 26873, 26944, 26948, 26951, 26952, 26922, 26954, + 26930, 26945, 26949, 26940, 26939, 26941, 26929, 26921, 26942, 26938, + 26935, 26936, 26950, 26932, 26943, 26946, 26928, 26926, 26953, 26937, + 26934, 26924, 26947, 26933, 26923, 26931, 26908, 26902, 26900, 26898, + 26905, 26904, 26907, 26906, 26910, 26909, 26913, 26915, 26912, 26911, + 26916, 26917, 26919, 26920, 26914, 26918, 26903, 26901, 26899, 26897, + 26957, 26955, 26956, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 27502, 27428, 27490, 27501, 27506, 27505, + 27425, 27507, 27482, 27481, 27479, 27436, 27485, 27486, 27478, 27435, + 27444, 27452, 27488, 27424, 27449, 27448, 27443, 27442, 27441, 27440, + 27466, 27434, 27465, 27433, 27508, 27439, 27489, 27500, 27499, 27447, + 27446, 27423, 27504, 27510, 27438, 27437, 27475, 27431, 27451, 27450, + 27474, 27430, 27483, 27487, 27459, 27462, 27463, 27497, 27495, 27476, + 27432, 27484, 27464, 27498, 27496, 27494, 27492, 27422, 27493, 27491, + 27509, 27426, 27503, 27427, 27461, 27429, 27480, 27477, 27460, 40951, + 40951, 40951, 40951, 27514, 27445, 27513, 27512, 27511, 27519, 27525, + 27524, 27520, 27521, 27546, 27550, 27567, 27566, 27528, 27531, 27532, + 27548, 27535, 27536, 27537, 27538, 27539, 27542, 27544, 27545, 27541, + 27552, 27553, 27554, 27555, 27560, 27556, 27557, 27561, 27563, 27522, + 27523, 27530, 27565, 27529, 27564, 27526, 27534, 27527, 27551, 27568, + 27569, 27558, 27562, 27549, 27547, 27570, 27543, 27533, 27540, 27559, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 27518, 27516, 27517, + 27515, 27467, 27468, 27469, 27470, 27471, 27472, 27473, 27453, 27454, + 27455, 27456, 27457, 27458, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 36644, 29851, + 30073, 30072, 22168, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 38877, 38876, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 35876, 35877, 35878, 35879, 35880, 35881, 35882, 35883, 35884, 35885, + 35886, 35887, 35888, 35889, 35890, 35891, 35892, 35893, 35894, 35895, + 35896, 35897, 35898, 35899, 35900, 35901, 35902, 35903, 35904, 35905, + 35906, 35907, 35908, 35909, 35910, 35911, 35912, 35913, 35914, 35915, + 35916, 35917, 35918, 35919, 35920, 35921, 35922, 35923, 35924, 35925, + 35926, 35927, 35928, 35929, 35930, 35931, 35932, 35933, 35934, 35935, + 35936, 35937, 35938, 35939, 35940, 35941, 35942, 35943, 35944, 35945, + 35946, 35947, 35948, 35949, 35950, 35951, 35952, 35953, 35954, 35955, + 35956, 35957, 35958, 35959, 35960, 35961, 35962, 35963, 35964, 35965, + 35966, 35967, 35968, 35969, 35970, 35971, 35972, 35973, 35974, 35975, + 35976, 35977, 35978, 35979, 35980, 35981, 35982, 35983, 35984, 35985, + 35986, 35987, 35988, 35989, 35990, 35991, 35992, 35993, 35994, 35995, + 35996, 35997, 35998, 35999, 36000, 36001, 36002, 36003, 36004, 36005, + 36006, 36007, 36008, 36009, 36010, 36011, 36012, 36013, 36014, 36015, + 36016, 36017, 36018, 36019, 36020, 36021, 36022, 36023, 36024, 36025, + 36026, 36027, 36028, 36029, 36030, 36031, 36032, 36033, 36034, 36035, + 36036, 36037, 36038, 36039, 36040, 36041, 36042, 36043, 36044, 36045, + 36046, 36047, 36048, 36049, 36050, 36051, 36052, 36053, 36054, 36055, + 36056, 36057, 36058, 36059, 36060, 36061, 36062, 36063, 36064, 36065, + 36066, 36067, 36068, 36069, 36070, 36071, 36072, 36073, 36074, 36075, + 36076, 36077, 36078, 36079, 36080, 36081, 36082, 36083, 36084, 36085, + 36086, 36087, 36088, 36089, 36090, 36091, 36092, 36093, 36094, 36095, + 36096, 36097, 36098, 36099, 36100, 36101, 36102, 36103, 36104, 36105, + 36106, 36107, 36108, 36109, 36110, 36111, 36112, 36113, 36114, 36115, + 36116, 36117, 36118, 36119, 36120, 36121, 36122, 36123, 36124, 36125, + 36126, 36127, 36128, 36129, 36130, 36131, 36132, 36133, 36134, 36135, + 36136, 36137, 36138, 36139, 36140, 36141, 36142, 36143, 36144, 36145, + 36146, 36147, 36148, 36149, 36150, 36151, 36152, 36153, 36154, 36155, + 36156, 36157, 36158, 36159, 36160, 36161, 36162, 36163, 36164, 36165, + 36166, 36167, 36168, 36169, 36170, 36171, 36172, 36173, 36174, 36175, + 36176, 36177, 36178, 36179, 36180, 36181, 36182, 36183, 36184, 36185, + 36186, 36187, 36188, 36189, 36190, 36191, 36192, 36193, 36194, 36195, + 36196, 36197, 36198, 36199, 36200, 36201, 36202, 36203, 36204, 36205, + 36206, 36207, 36208, 36209, 36210, 36211, 36212, 36213, 36214, 36215, + 36216, 36217, 36218, 36219, 36220, 36221, 36222, 36223, 36224, 36225, + 36226, 36227, 36228, 36229, 36230, 36231, 36232, 36233, 36234, 36235, + 36236, 36237, 36238, 36239, 36240, 36241, 36242, 36243, 36244, 36245, + 36246, 36247, 36248, 36249, 36250, 36251, 36252, 36253, 36254, 36255, + 36256, 36257, 36258, 36259, 36260, 36261, 36262, 36263, 36264, 36265, + 36266, 36267, 36268, 36269, 36270, 36271, 36272, 36273, 36274, 36275, + 36276, 36277, 36278, 36279, 36280, 36281, 36282, 36283, 36284, 36285, + 36286, 36287, 36288, 36289, 36290, 36291, 36292, 36293, 36294, 36295, + 36296, 36297, 36298, 36299, 36300, 36301, 36302, 36303, 36304, 36305, + 36306, 36307, 36308, 36309, 36310, 36311, 36312, 36313, 36314, 36315, + 36316, 36317, 36318, 36319, 36320, 36321, 36322, 36323, 36324, 36325, + 36326, 36327, 36328, 36329, 36330, 36331, 36332, 36333, 36334, 36335, + 36336, 36337, 36338, 36339, 36340, 36341, 36342, 36343, 36344, 36345, + 36346, 36347, 36348, 36349, 36350, 36351, 36352, 36353, 36354, 36355, + 36356, 36357, 36358, 36359, 36360, 36361, 36362, 36363, 36364, 36365, + 36366, 36367, 36368, 36369, 36370, 36371, 36372, 36373, 36374, 36375, + 36376, 36377, 36378, 36379, 36380, 36381, 36382, 36383, 36384, 36385, + 36386, 36387, 36388, 36389, 36390, 36391, 36392, 36393, 36394, 36395, + 36396, 36397, 36398, 36399, 36400, 36401, 36402, 36403, 36404, 36405, + 36406, 36407, 36408, 36409, 36410, 36411, 36412, 36413, 36414, 36415, + 36416, 36417, 36418, 36419, 36420, 36421, 36422, 36423, 36424, 36425, + 36426, 36427, 36428, 36429, 36430, 36431, 36432, 36433, 36434, 36435, + 36436, 36437, 36438, 36439, 36440, 36441, 36442, 36443, 36444, 36445, + 36446, 36447, 36448, 36449, 36450, 36451, 36452, 36453, 36454, 36455, + 36456, 36457, 36458, 36459, 36460, 36461, 36462, 36463, 36464, 36465, + 36466, 36467, 36468, 36469, 36470, 36471, 36472, 36473, 36474, 36475, + 36476, 36477, 36478, 36479, 36480, 36481, 36482, 36483, 36484, 36485, + 36486, 36487, 36488, 36489, 36490, 36491, 36492, 36493, 36494, 36495, + 36496, 36497, 36498, 36499, 36500, 36501, 36502, 36503, 36504, 36505, + 36506, 36507, 36508, 36509, 36510, 36511, 36512, 36513, 36514, 36515, + 36516, 36517, 36518, 36519, 36520, 36521, 36522, 36523, 36524, 36525, + 36526, 36527, 36528, 36529, 36530, 36531, 36532, 36533, 36534, 36535, + 36536, 36537, 36538, 36539, 36540, 36541, 36542, 36543, 36544, 36545, + 36546, 36547, 36548, 36549, 36550, 36551, 36552, 36553, 36554, 36555, + 36556, 36557, 36558, 36559, 36560, 36561, 36562, 36563, 36564, 36565, + 36566, 36567, 36568, 36569, 36570, 36571, 36572, 36573, 36574, 36575, + 36576, 36577, 36578, 36579, 36580, 36581, 36582, 36583, 36584, 36585, + 36586, 36587, 36588, 36589, 36590, 36591, 36592, 36593, 36594, 36595, + 36596, 36597, 36598, 36599, 36600, 36601, 36602, 36603, 36604, 36605, + 36606, 36607, 36608, 36609, 36610, 36611, 36612, 36613, 36614, 36615, + 36616, 36617, 36618, 36619, 36620, 36621, 36622, 36623, 36624, 36625, + 36626, 36627, 36628, 36629, 36630, 36631, 36632, 36633, 36634, 36635, + 36636, 36637, 36638, 36639, 36640, 36641, 36642, 36643, 21697, 21698, + 21699, 21700, 21701, 21702, 21703, 21704, 21705, 21706, 21707, 21708, + 21709, 21710, 21711, 21712, 21713, 21714, 21715, 21716, 21717, 21718, + 21719, 21720, 21721, 21722, 21723, 21724, 21725, 21726, 21727, 21728, + 21729, 21730, 21731, 21732, 21733, 21734, 21735, 21736, 21737, 21738, + 21739, 21740, 21741, 21742, 21743, 21744, 21745, 21746, 21747, 21748, + 21749, 21750, 21751, 21752, 21753, 21754, 21755, 21756, 21757, 21758, + 21759, 21760, 21761, 21762, 21763, 21764, 21765, 21766, 21767, 21768, + 21769, 21770, 21771, 21772, 21773, 21774, 21775, 21776, 21777, 21778, + 21779, 21780, 21781, 21782, 21783, 21784, 21785, 21786, 21787, 21788, + 21789, 21790, 21791, 21792, 21793, 21794, 21795, 21796, 21797, 21798, + 21799, 21800, 21801, 21802, 21803, 21804, 21805, 21806, 21807, 21808, + 21809, 21810, 21811, 21812, 21813, 21814, 21815, 21816, 21817, 21818, + 21819, 21820, 21821, 21822, 21823, 21824, 21825, 21826, 21827, 21828, + 21829, 21830, 21831, 21832, 21833, 21834, 21835, 21836, 21837, 21838, + 21839, 21840, 21841, 21842, 21843, 21844, 21845, 21846, 21847, 21848, + 21849, 21850, 21851, 21852, 21853, 21854, 21855, 21856, 21857, 21858, + 21859, 21860, 21861, 21862, 21863, 21864, 21865, 21866, 21867, 21868, + 21869, 21870, 21871, 21872, 21873, 21874, 21875, 21876, 21877, 21878, + 21879, 21880, 21881, 21882, 21883, 21884, 21885, 21886, 21887, 21888, + 21889, 21890, 21891, 21892, 21893, 21894, 21895, 21896, 21897, 21898, + 21899, 21900, 21901, 21902, 21903, 21904, 21905, 21906, 21907, 21908, + 21909, 21910, 21911, 21912, 21913, 21914, 21915, 21916, 21917, 21918, + 21919, 21920, 21921, 21922, 21923, 21924, 21925, 21926, 21927, 21928, + 21929, 21930, 21931, 21932, 21933, 21934, 21935, 21936, 21937, 21938, + 21939, 21940, 21941, 21942, 21943, 21944, 21945, 21946, 21947, 21948, + 21949, 21950, 21951, 21952, 21959, 21960, 21961, 21962, 21963, 21964, + 21965, 21966, 21967, 21968, 21969, 21970, 21971, 21972, 21973, 21974, + 21975, 21976, 21977, 21978, 21979, 21980, 21981, 21982, 21983, 21984, + 21985, 21986, 21987, 21988, 21989, 21990, 21991, 21992, 21993, 21994, + 21995, 21996, 21997, 21998, 21999, 22000, 22001, 22002, 22003, 22004, + 22005, 22006, 22007, 22008, 22009, 22010, 22011, 22012, 22013, 22014, + 22015, 22016, 22017, 22018, 22019, 22020, 22021, 22022, 22023, 22024, + 22025, 22026, 22027, 22028, 22029, 22030, 22031, 22032, 22033, 22034, + 22035, 22036, 22037, 22038, 22039, 22040, 22041, 22042, 22043, 22044, + 22045, 22046, 22047, 22048, 22049, 22050, 22051, 22052, 22053, 22054, + 22055, 22056, 22057, 22058, 22059, 22060, 22061, 22062, 22063, 22064, + 22065, 22066, 22067, 22068, 22069, 22070, 22071, 22072, 22073, 22074, + 22075, 22076, 22077, 22078, 22079, 22080, 22081, 22082, 22083, 22084, + 22085, 22086, 22087, 22088, 22089, 22090, 22091, 22092, 22093, 22094, + 22095, 22096, 22097, 22098, 22099, 22100, 22101, 22102, 22103, 22104, + 22105, 22106, 22107, 22108, 22109, 22110, 22111, 22112, 22113, 22114, + 22115, 22116, 22117, 22118, 22119, 22120, 22121, 22122, 22123, 22124, + 22125, 22126, 22127, 22128, 22129, 22130, 22131, 22132, 22133, 22134, + 22135, 22136, 22137, 22138, 22139, 22140, 22141, 22142, 22143, 22144, + 22145, 22146, 22147, 22148, 22149, 22150, 22151, 22152, 22153, 22154, + 22155, 22156, 22157, 22158, 22159, 22160, 22161, 22162, 22163, 22164, + 22165, 22166, 21953, 21954, 21955, 21956, 21957, 21958, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 22167, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 21336, 21337, 21338, 21339, 40951, 21340, 21341, 21329, 21330, 21331, + 21332, 21333, 40951, 21334, 21335, 40951, 21317, 20293, 19932, 19933, + 19934, 19931, 20213, 20214, 20215, 20216, 20205, 20206, 20207, 20208, + 20209, 20200, 20201, 20202, 20203, 20204, 20210, 20211, 20212, 19971, + 19975, 19976, 19977, 19978, 19979, 19980, 19981, 19982, 19972, 19973, + 19974, 19987, 19988, 19989, 19990, 19991, 19992, 19993, 19994, 20001, + 20002, 20003, 20004, 20005, 20006, 20007, 19995, 19996, 19997, 19998, + 19999, 20000, 19984, 19985, 19986, 19983, 20096, 20097, 20098, 20099, + 20100, 20101, 20102, 20103, 20112, 20113, 20114, 20115, 20116, 20117, + 20104, 20105, 20106, 20107, 20108, 20109, 20110, 20111, 20118, 20119, + 20120, 20121, 20122, 20123, 20124, 20125, 20126, 20127, 20128, 20129, + 20158, 20159, 20160, 20161, 20151, 20152, 20153, 20154, 20155, 20156, + 20157, 20147, 20148, 20149, 20150, 20146, 20130, 20131, 20132, 20133, + 20134, 20135, 20136, 20137, 20138, 20140, 20141, 20142, 20143, 20144, + 20145, 20139, 20050, 20051, 20052, 20053, 20054, 20055, 20056, 20057, + 20058, 20043, 20044, 20045, 20046, 20047, 20048, 20049, 20042, 20064, + 20065, 20066, 20036, 20037, 20038, 20039, 20040, 20041, 20035, 20059, + 20060, 20061, 20062, 20063, 19943, 19946, 19947, 19948, 19949, 19950, + 19951, 19952, 19953, 19944, 19945, 19964, 19965, 19966, 19967, 19968, + 19969, 19970, 19954, 19955, 19956, 19957, 19958, 19959, 19960, 19961, + 19962, 19963, 19935, 19936, 19937, 19938, 19939, 19940, 19941, 19942, + 20017, 20018, 20019, 20020, 20021, 20022, 20023, 20024, 20025, 20026, + 20027, 20028, 20029, 20030, 20031, 20032, 20033, 20034, 20009, 20010, + 20008, 20011, 20012, 20013, 20014, 20015, 20016, 20184, 20185, 20186, + 20187, 20188, 20183, 20195, 20196, 20197, 20198, 20189, 20190, 20191, + 20192, 20193, 20194, 20088, 20089, 20090, 20091, 20081, 20082, 20083, + 20084, 20085, 20086, 20087, 20075, 20076, 20077, 20078, 20079, 20080, + 20092, 20093, 20094, 20095, 20069, 20070, 20071, 20072, 20073, 20074, + 20162, 20163, 20164, 20165, 20166, 20167, 20168, 20169, 20170, 20171, + 20179, 20180, 20181, 20182, 20172, 20173, 20174, 20175, 20176, 20177, + 20178, 20067, 20068, 20292, 21315, 21314, 21316, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 20303, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 20296, 20295, 20297, 40951, 40951, 21364, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 21370, 21369, 21371, 21375, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 29455, 29456, 29457, 29458, 29459, 29460, + 29461, 29462, 29463, 29464, 29465, 29466, 29467, 29468, 29469, 29470, + 29471, 29472, 29473, 29474, 29475, 29476, 29477, 29478, 29479, 29480, + 29481, 29482, 29483, 29484, 29485, 29486, 29487, 29488, 29489, 29490, + 29491, 29492, 29493, 29494, 29495, 29496, 29497, 29498, 29499, 29500, 29501, 29502, 29503, 29504, 29505, 29506, 29507, 29508, 29509, 29510, 29511, 29512, 29513, 29514, 29515, 29516, 29517, 29518, 29519, 29520, - 29521, 29490, 29526, 29541, 29542, 29531, 29553, 24272, 24273, 24274, - 24275, 24276, 24277, 24278, 24279, 24280, 24281, 24282, 24283, 24284, - 24285, 24286, 24287, 24288, 24289, 24290, 24291, 24292, 24293, 24294, - 24295, 24296, 24297, 26969, 26970, 26971, 6239, 6238, 6287, 24303, 24304, - 24305, 24306, 24307, 24308, 24309, 24310, 24311, 24312, 24313, 24314, - 24315, 24316, 24317, 24318, 24319, 24320, 24321, 24322, 24323, 24324, - 24325, 24326, 24327, 24328, 8387, 24332, 24334, 24335, 24331, 24333, - 29232, 29479, 29478, 29485, 29554, 29527, 29528, 29530, 29540, 29548, - 29551, 29543, 29533, 29545, 29483, 29547, 29484, 29534, 29544, 29536, - 29529, 29495, 29489, 29488, 29487, 29524, 29535, 29549, 29550, 21137, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 26997, 26998, 26999, 27000, - 27001, 27002, 27003, 27004, 27005, 27006, 27007, 27008, 27009, 27010, - 27011, 27012, 27013, 27014, 27015, 27016, 27017, 27018, 27019, 27020, - 27021, 27022, 29270, 29491, 29493, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 29457, 29452, - 29444, 29492, 29438, 29450, 29473, 29449, 29437, 29462, 29470, 29461, - 29442, 29453, 29454, 29460, 29441, 29471, 29468, 29475, 29451, 29446, - 29465, 29455, 29458, 29435, 29436, 29477, 29448, 29439, 29443, 29459, - 29474, 29456, 29469, 29472, 29445, 29466, 29464, 29463, 29467, 29440, - 29447, 29476, 35762, 35762, 35762, 35762, 32452, 32446, 32447, 32449, - 32453, 32450, 32454, 32448, 32451, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 6291, 6289, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 27401, 27402, - 27399, 27403, 27398, 27400, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 10219, 13162, - 7734, 24578, 29724, 29723, 6522, 29725, 26962, 4630, 33941, 33767, 22902, - 11218, 11216, 11217, 13610, 24383, 33853, 13131, 33854, 13203, 33852, - 18158, 33851, 8377, 24382, 13130, 18157, 13202, 29648, 13611, 27918, - 32014, 3808, 34096, 34100, 34097, 34098, 7747, 7746, 7745, 7744, 13161, - 34162, 15972, 31673, 4706, 6175, 27734, 13062, 10243, 26230, 5823, 15969, - 32566, 6173, 27389, 15937, 29726, 3985, 11212, 11213, 15762, 13180, - 20989, 13101, 19445, 23855, 32421, 2525, 13723, 22390, 33944, 30811, - 19779, 3406, 26676, 26994, 14263, 26496, 26493, 6174, 29593, 14654, - 28875, 22182, 26811, 27117, 27118, 8177, 9648, 29581, 29170, 4628, 13193, - 27405, 10228, 26088, 29766, 13201, 13137, 28969, 27862, 16002, 10975, - 8176, 6212, 5698, 20592, 9650, 15943, 27935, 3629, 26809, 8175, 13167, - 31675, 27701, 34164, 7751, 32533, 3520, 7693, 2589, 13168, 4093, 26799, - 27108, 34184, 3740, 16359, 6213, 13107, 13127, 13126, 2729, 26420, 8156, - 30810, 8388, 26675, 16364, 5760, 34161, 23499, 27706, 13648, 15234, 4095, - 22901, 27061, 23619, 29597, 19778, 8168, 3510, 3511, 13122, 98, 5758, - 13113, 27348, 13135, 22894, 23520, 6215, 15233, 2506, 32435, 6520, 32420, - 7689, 26525, 33689, 10611, 28881, 3739, 13322, 4098, 13153, 23854, 23602, - 27700, 14279, 23618, 32538, 33694, 23853, 27530, 31789, 28856, 3412, - 6176, 28963, 27531, 29764, 29196, 32535, 15967, 360, 27408, 29769, 33963, - 13647, 26952, 26953, 8379, 33768, 13141, 16004, 29954, 28960, 4964, 3516, - 4705, 15979, 6521, 10520, 7690, 10351, 10352, 24253, 29579, 15978, 15977, - 25236, 16361, 13032, 15980, 3401, 2522, 15973, 20528, 8171, 27705, 10269, - 13097, 16357, 16360, 13031, 34065, 3855, 33950, 33949, 27390, 3870, - 18037, 2600, 4102, 359, 12426, 12427, 12428, 12429, 12430, 26976, 23515, - 26091, 33942, 8370, 32318, 19676, 26978, 5765, 11054, 8391, 34118, 28957, - 28958, 15966, 26981, 13615, 27940, 23497, 27352, 6181, 10716, 26667, - 4252, 12395, 25234, 29191, 4643, 954, 15942, 18039, 13134, 32534, 3986, - 32564, 15201, 2588, 13197, 3741, 26497, 18034, 26821, 11056, 2598, 10778, - 23517, 8371, 32319, 26979, 5766, 11055, 29195, 15968, 23498, 10777, - 26669, 13200, 14652, 34183, 3515, 26267, 26668, 26486, 6180, 13059, - 13057, 11211, 24730, 23518, 32422, 34107, 34024, 34050, 34071, 13138, - 33951, 26087, 32046, 32047, 7688, 25781, 8393, 34173, 13058, 24573, - 29951, 16461, 11062, 18029, 3745, 34172, 26925, 14658, 26816, 20983, - 2519, 15833, 34174, 34171, 13166, 4701, 4700, 4250, 13510, 20896, 34169, - 13105, 20902, 32581, 32582, 26796, 34170, 4631, 26510, 20899, 20900, - 25760, 25758, 2587, 8159, 26878, 16366, 16365, 14521, 2590, 13045, 356, - 16131, 28858, 16247, 14278, 10227, 20434, 24184, 13093, 14525, 3410, - 29949, 26672, 18026, 20527, 27328, 13326, 18021, 4094, 8369, 33961, 3517, - 4637, 32575, 29171, 14276, 15236, 3988, 14265, 34208, 26924, 15235, - 27104, 15237, 10528, 12383, 952, 4249, 28870, 7756, 29192, 11058, 10232, - 26670, 13148, 10696, 29182, 32021, 34035, 15989, 23319, 9646, 15266, - 8376, 3403, 3405, 3404, 3402, 23318, 5994, 27730, 26520, 4632, 22904, - 13154, 25792, 11209, 13118, 25779, 26095, 26097, 4961, 31676, 5703, 5993, - 5995, 3408, 7694, 26927, 27396, 26488, 29591, 32504, 3999, 19777, 24727, - 24728, 7740, 25773, 14264, 3987, 25796, 4000, 24255, 27727, 22741, 31681, - 26098, 13104, 27614, 26928, 6000, 26073, 16356, 26487, 13060, 16163, - 12464, 7737, 7738, 25783, 25782, 26805, 26802, 24568, 22918, 22919, - 33686, 22920, 24648, 967, 4962, 4963, 33688, 31688, 26955, 33690, 13121, - 26800, 26892, 32568, 7724, 7725, 7721, 963, 20529, 15828, 29177, 29176, - 29178, 29179, 3509, 12381, 19566, 27215, 20492, 7739, 17144, 20489, - 25786, 3523, 3525, 3998, 20432, 26957, 2592, 12459, 25762, 29019, 32445, - 24647, 17159, 16250, 16251, 16254, 16253, 16252, 13123, 12382, 34187, - 14648, 25141, 15981, 26681, 22893, 31687, 8398, 28852, 16367, 32506, - 3880, 34083, 18148, 18130, 18138, 18131, 28887, 28886, 32658, 10981, - 32656, 10982, 20561, 32707, 6236, 8385, 8384, 24720, 24722, 29838, 34048, - 15281, 5831, 26089, 11049, 17142, 23504, 29859, 22604, 4096, 7705, 7715, - 7717, 7701, 7699, 7709, 7707, 7697, 7703, 7711, 7695, 7713, 7706, 7716, - 7718, 7702, 7700, 7710, 7708, 7698, 7704, 7712, 7696, 7714, 27173, 27174, - 27175, 4696, 4697, 27336, 3997, 5697, 20986, 3882, 24646, 15941, 20897, - 28873, 10229, 29187, 29188, 16462, 20901, 19490, 31682, 27156, 34102, - 3895, 31686, 7691, 2593, 29563, 12463, 13160, 26500, 20431, 3852, 20554, - 20540, 20552, 20553, 20558, 19545, 32555, 26967, 27089, 27097, 27098, - 27102, 27099, 26968, 34025, 28043, 28042, 28039, 28038, 3830, 3868, - 28045, 28044, 28041, 28040, 3898, 3802, 3815, 10353, 17146, 32037, 26875, - 26822, 3818, 34039, 28970, 31670, 34167, 25770, 32570, 32033, 32490, - 25585, 15229, 27711, 26876, 13103, 25793, 10702, 10703, 10704, 13190, - 13192, 13191, 3814, 13164, 25780, 5704, 5705, 13119, 12431, 12432, 12433, - 24724, 24725, 24726, 12442, 12435, 12436, 10701, 26094, 26099, 33957, - 29190, 29189, 10354, 22905, 22168, 26080, 7723, 5695, 16165, 10246, 8151, - 25757, 27347, 26096, 29589, 10226, 20433, 29180, 32029, 32028, 32030, - 32031, 19517, 27176, 32583, 32035, 19533, 27189, 19454, 27121, 23501, - 19802, 19801, 2734, 2740, 2735, 2739, 2732, 19799, 2733, 34178, 23514, - 32489, 29577, 28855, 23519, 14268, 14270, 13084, 28945, 28947, 28946, - 28948, 28944, 28943, 34165, 28949, 13064, 27060, 28942, 28952, 28955, - 24380, 13056, 32619, 13067, 26498, 8157, 8158, 18022, 13094, 18023, - 18024, 13081, 13082, 13083, 10613, 34179, 953, 26814, 8397, 26522, 13088, - 10612, 13199, 950, 13110, 33959, 28872, 32419, 14273, 20591, 13072, - 15988, 13074, 13065, 2514, 13155, 28871, 10697, 13091, 13069, 14272, - 5767, 28940, 28941, 5768, 18025, 26813, 8396, 33958, 28876, 28877, 32716, - 13087, 13077, 13071, 26519, 27735, 15239, 29175, 15200, 26517, 26516, - 26512, 26514, 24689, 29073, 24663, 29061, 32553, 32562, 32552, 32561, - 24688, 29072, 24662, 29060, 15317, 15310, 15314, 15307, 24690, 29074, - 24664, 29062, 15318, 15311, 15315, 15308, 15939, 15940, 29015, 29016, - 19669, 32750, 27291, 11044, 27721, 15312, 19781, 15289, 15244, 29767, - 27608, 27609, 27610, 15333, 27611, 15301, 33677, 33674, 5996, 27070, - 27346, 15474, 29580, 26960, 15831, 15832, 32497, 22740, 19788, 29578, - 32493, 32494, 4699, 25767, 32532, 4702, 22906, 361, 13124, 26794, 25765, - 31674, 25766, 2521, 25763, 26993, 10251, 2505, 32491, 23496, 23511, - 29765, 23512, 160, 27917, 27404, 29181, 15963, 33668, 8160, 26795, 32503, - 11050, 24644, 28956, 24649, 26926, 11048, 26807, 24653, 3735, 24642, - 3734, 23513, 26529, 24645, 6178, 22605, 34175, 27065, 2591, 32488, 33940, - 27939, 3507, 3508, 26428, 9651, 2604, 19491, 32500, 26889, 6345, 4251, - 13511, 8368, 28869, 27920, 3527, 3636, 26686, 25233, 27919, 29599, 26100, - 15935, 15991, 12396, 35762, 35762, 35762, 35762, 34168, 26772, 33965, - 27329, 14649, 27913, 25267, 23509, 26959, 23506, 32654, 32657, 32655, - 28884, 24697, 227, 228, 35762, 35762, 35762, 27613, 25764, 10538, 26425, - 27709, 23507, 5700, 28874, 13159, 28861, 2523, 26666, 27349, 35762, - 35762, 35762, 295, 243, 344, 342, 339, 237, 235, 236, 233, 234, 332, 334, + 29521, 29522, 29523, 29524, 29525, 29526, 29527, 29528, 29529, 29530, + 29531, 29532, 29533, 29534, 29535, 29536, 29537, 29538, 29539, 29540, + 29541, 29542, 29543, 29544, 29545, 29546, 29547, 29548, 29549, 29550, + 29551, 29552, 29553, 29554, 29555, 29556, 29557, 29558, 29559, 29560, + 29561, 29562, 29563, 29564, 29565, 29566, 29567, 29568, 29569, 29570, + 29571, 29572, 29573, 29574, 29575, 29576, 29577, 29578, 29579, 29580, + 29581, 29582, 29583, 29584, 29585, 29586, 29587, 29588, 29589, 29590, + 29591, 29592, 29593, 29594, 29595, 29596, 29597, 29598, 29599, 29600, + 29601, 29602, 29603, 29604, 29605, 29606, 29607, 29608, 29609, 29610, + 29611, 29612, 29613, 29614, 29615, 29616, 29617, 29618, 29619, 29620, + 29621, 29622, 29623, 29624, 29625, 29626, 29627, 29628, 29629, 29630, + 29631, 29632, 29633, 29634, 29635, 29636, 29637, 29638, 29639, 29640, + 29641, 29642, 29643, 29644, 29645, 29646, 29647, 29648, 29649, 29650, + 29651, 29652, 29653, 29654, 29655, 29656, 29657, 29658, 29659, 29660, + 29661, 29662, 29663, 29664, 29665, 29666, 29667, 29668, 29669, 29670, + 29671, 29672, 29673, 29674, 29675, 29676, 29677, 29678, 29679, 29680, + 29681, 29682, 29683, 29684, 29685, 29686, 29687, 29688, 29689, 29690, + 29691, 29692, 29693, 29694, 29695, 29696, 29697, 29698, 29699, 29700, + 29701, 29702, 29703, 29704, 29705, 29706, 29707, 29708, 29709, 29710, + 29711, 29712, 29713, 29714, 29715, 29716, 29717, 29718, 29719, 29720, + 29721, 29722, 29723, 29724, 29725, 29726, 29727, 29728, 29729, 29730, + 29731, 29732, 29733, 29734, 29735, 29736, 29737, 29738, 29739, 29740, + 29741, 29742, 29743, 29744, 29745, 29746, 29747, 29748, 29749, 29750, + 29751, 29752, 29753, 29754, 29755, 29756, 29757, 29758, 29759, 29760, + 29761, 29762, 29763, 29764, 29765, 29766, 29767, 29768, 29769, 29770, + 29771, 29772, 29773, 29774, 29775, 29776, 29777, 29778, 29779, 29780, + 29781, 29782, 29783, 29784, 29785, 29786, 29787, 29788, 29789, 29790, + 29791, 29792, 29793, 29794, 29795, 29796, 29797, 29798, 29799, 29800, + 29801, 29802, 29803, 29804, 29805, 29806, 29807, 29808, 29809, 29810, + 29811, 29812, 29813, 29814, 29815, 29816, 29817, 29818, 29819, 29820, + 29821, 29822, 29823, 29824, 29825, 29826, 29827, 29828, 29829, 29830, + 29831, 29832, 29833, 29834, 29835, 29836, 29837, 29838, 29839, 29840, + 29841, 29842, 29843, 29844, 29845, 29846, 29847, 29848, 29849, 29850, + 40951, 40951, 40951, 40951, 11516, 11514, 11463, 11496, 11423, 11436, + 11440, 11521, 11417, 11504, 11425, 11467, 11466, 11418, 11424, 11438, + 11470, 11499, 11492, 11419, 11439, 11493, 11517, 11443, 11471, 11444, + 11449, 11427, 11472, 11445, 11450, 11430, 11473, 11447, 11452, 11428, + 11429, 11481, 11482, 11448, 11453, 11434, 11485, 11446, 11451, 11431, + 11474, 11435, 11432, 11433, 11479, 11480, 11477, 11478, 11498, 11497, + 11506, 11512, 11509, 11484, 11483, 11437, 11426, 11475, 11476, 11415, + 11490, 11460, 11458, 11416, 11518, 11420, 11519, 11495, 11503, 11421, + 11487, 11468, 11486, 11441, 11520, 11500, 11422, 11515, 11501, 11442, + 11469, 11502, 11494, 11459, 11462, 11461, 11511, 11507, 11513, 11510, + 11508, 11457, 11456, 11455, 11454, 11465, 11464, 11488, 11491, 11489, + 11505, 40951, 40951, 40951, 40951, 40951, 11400, 11413, 11412, 11407, + 11414, 11394, 11387, 11386, 11383, 11385, 11388, 11389, 11384, 40951, + 40951, 40951, 11396, 11392, 11395, 11390, 11399, 11398, 11391, 11397, + 11393, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 11401, 11405, + 11408, 11403, 11411, 11410, 11404, 11409, 11406, 11402, 40951, 40951, + 11525, 11522, 11523, 11524, 32839, 32838, 32840, 32841, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 37793, 32041, 24171, 32038, 11290, 25171, 32009, 25229, 965, 20428, + 38848, 24129, 27577, 31994, 24164, 32031, 29926, 31588, 31779, 20424, + 38822, 25118, 25117, 25116, 25115, 25113, 25114, 4485, 4486, 4494, 4508, + 4378, 4377, 32579, 32587, 32580, 32591, 32584, 32588, 32581, 32593, + 32586, 32590, 32583, 32592, 32585, 32589, 32582, 37832, 37800, 37802, + 37887, 37849, 37845, 37881, 37851, 25255, 25202, 25222, 25257, 25205, + 25247, 25249, 25228, 34085, 34086, 30653, 10840, 10576, 10575, 34097, + 34098, 24177, 32018, 17483, 17484, 350, 349, 356, 355, 358, 357, 353, + 354, 351, 352, 24167, 37785, 32034, 11296, 37462, 37458, 37466, 4340, + 4338, 4344, 24160, 37780, 32028, 11292, 28300, 24170, 37788, 32037, + 11299, 16653, 16654, 3842, 3845, 3844, 3843, 3876, 24183, 37778, 32024, + 11302, 24184, 37779, 32025, 11303, 24173, 37782, 32040, 11294, 34330, + 34331, 34329, 34328, 34521, 34523, 34522, 34520, 38832, 20407, 39241, + 39242, 37705, 34162, 34161, 34160, 34115, 20802, 24043, 20803, 38837, + 20405, 24178, 32019, 24179, 32020, 17469, 24169, 37787, 32036, 11298, + 20427, 38847, 38825, 24172, 32039, 24166, 32033, 24168, 32035, 24077, + 31952, 37841, 37877, 37840, 37876, 25191, 25211, 25190, 25210, 25188, + 25208, 25189, 25209, 37844, 37880, 25201, 25221, 37842, 37878, 25200, + 25220, 37835, 37871, 25195, 25215, 37838, 37874, 25198, 25218, 37839, + 37875, 25199, 25219, 37834, 37870, 25194, 25214, 37836, 37872, 25196, + 25216, 37837, 37873, 25197, 25217, 37843, 37879, 25192, 25212, 30832, + 30833, 30834, 30835, 30836, 30837, 30838, 30839, 30840, 30841, 30842, + 30843, 30844, 30845, 30846, 30847, 30848, 30849, 30850, 30851, 30852, + 30853, 30854, 30855, 30856, 30857, 30868, 30870, 30867, 30866, 30863, + 30862, 30865, 30864, 30871, 30869, 40951, 40951, 40951, 40951, 40951, + 40951, 4137, 4081, 3952, 4167, 4038, 3979, 4138, 4019, 4082, 4128, 4054, + 4113, 3995, 4010, 4098, 3964, 4171, 4039, 4069, 3980, 4139, 4020, 4083, + 3953, 4134, 4062, 4121, 4003, 4160, 4016, 4106, 3972, 4047, 4075, 3988, + 4146, 4028, 4091, 3958, 4129, 4055, 4114, 3996, 4153, 4011, 4099, 3965, + 4172, 4040, 4070, 3981, 4140, 4021, 4084, 4066, 4125, 4007, 4164, 4035, + 4110, 3976, 4179, 4051, 4078, 3992, 4150, 4032, 4095, 3961, 4059, 4118, + 4000, 4157, 4103, 3969, 4176, 4044, 3985, 4143, 4025, 4088, 4135, 4063, + 4122, 4004, 4161, 4017, 4107, 3973, 4168, 4048, 4076, 3989, 4147, 4029, + 4092, 3959, 4130, 4056, 4115, 3997, 4154, 4012, 4100, 3966, 4173, 4041, + 4071, 3982, 4141, 4022, 4085, 3954, 4068, 4127, 4009, 4166, 4037, 4112, + 3978, 4181, 4053, 4080, 3994, 4152, 4034, 4097, 3963, 4133, 4061, 4120, + 4002, 4159, 4015, 4105, 3971, 4178, 4046, 4074, 3987, 4145, 4027, 4090, + 3957, 4065, 4124, 4006, 4163, 4109, 3975, 4170, 4050, 3991, 4149, 4031, + 4094, 4131, 4058, 4117, 3999, 4156, 4013, 4102, 3968, 4175, 4043, 4072, + 3984, 4142, 4024, 4087, 3955, 4067, 4126, 4008, 4165, 4036, 4111, 3977, + 4180, 4052, 4079, 3993, 4151, 4033, 4096, 3962, 4132, 4060, 4119, 4001, + 4158, 4014, 4104, 3970, 4177, 4045, 4073, 3986, 4144, 4026, 4089, 3956, + 4136, 4064, 4123, 4005, 4162, 4018, 4108, 3974, 4169, 4049, 4077, 3990, + 4148, 4030, 4093, 3960, 4057, 4116, 3998, 4155, 4101, 3967, 4174, 4042, + 3983, 4023, 4086, 37470, 4347, 37467, 4345, 37468, 4346, 37463, 4341, + 37464, 4342, 37457, 4334, 4335, 4336, 4337, 28184, 37459, 37460, 11293, + 24161, 33837, 37783, 11295, 17356, 17357, 17358, 31947, 25162, 17359, + 37807, 25164, 19784, 39303, 37481, 17638, 4379, 4376, 24081, 31956, + 24080, 31955, 20404, 24078, 31953, 20403, 25165, 37810, 38838, 4504, + 4502, 4503, 4501, 22718, 22717, 22722, 22716, 22694, 22673, 22674, 22714, + 22680, 22698, 22721, 22696, 22719, 22720, 22702, 22699, 22679, 22678, + 22681, 22697, 22682, 22670, 22691, 22715, 22671, 22672, 22669, 22695, + 22700, 22686, 22677, 22701, 22703, 22676, 22693, 22685, 22683, 22684, + 22675, 22723, 22692, 22687, 22690, 22688, 22689, 22711, 22709, 22710, + 22713, 22708, 22705, 22712, 22707, 22706, 22704, 32594, 32626, 32595, + 32642, 32611, 32627, 32596, 32650, 32619, 32635, 32604, 32643, 32612, + 32628, 32597, 32654, 32623, 32639, 32608, 32647, 32616, 32632, 32601, + 32651, 32620, 32636, 32605, 32644, 32613, 32629, 32598, 32656, 32625, + 32641, 32610, 32649, 32618, 32634, 32603, 32653, 32622, 32638, 32607, + 32646, 32615, 32631, 32600, 32655, 32624, 32640, 32609, 32648, 32617, + 32633, 32602, 32652, 32621, 32637, 32606, 32645, 32614, 32630, 32599, + 37827, 37799, 37801, 37866, 37848, 37846, 37847, 37850, 25254, 25252, + 25253, 25256, 25207, 25246, 25248, 25242, 31957, 31997, 24132, 24082, + 25167, 25262, 37890, 37812, 24083, 24133, 31998, 31958, 37813, 37891, + 25263, 25168, 20429, 21627, 30342, 3882, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40767, 40772, 40806, 40769, 40774, 40802, 40796, 40790, + 40813, 40792, 40786, 40810, 40768, 40773, 40807, 40770, 40775, 40803, + 40797, 40791, 40814, 40793, 40787, 40811, 40808, 40794, 40801, 40788, + 40789, 40812, 40795, 40771, 40817, 40782, 40799, 40809, 40820, 40819, + 40785, 40818, 40779, 40778, 40816, 40800, 40798, 40777, 40951, 40951, + 40822, 40823, 40824, 40815, 40765, 40780, 40783, 40784, 40762, 40763, + 40781, 40804, 40805, 40766, 40821, 40764, 40776, 40761, 40943, 40944, + 40941, 40942, 40945, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40841, 40842, 40858, 40830, 40839, 40938, 40893, 40894, + 40861, 40862, 40895, 40825, 40859, 40939, 40837, 40834, 40836, 40835, + 40833, 40935, 40934, 40937, 40936, 40931, 40930, 40933, 40932, 40831, + 40865, 40827, 40838, 40826, 40863, 40876, 40877, 40875, 40874, 40828, + 40872, 40873, 40871, 40869, 40870, 40868, 40867, 40878, 40880, 40881, + 40879, 40843, 40866, 40832, 40840, 40940, 40882, 40887, 40884, 40888, + 40885, 40890, 40891, 40886, 40883, 40889, 40864, 40892, 40925, 40922, + 40913, 40923, 40924, 40914, 40902, 40901, 40929, 40899, 40898, 40896, + 40900, 40897, 40916, 40921, 40915, 40919, 40920, 40917, 40918, 40845, + 40849, 40848, 40847, 40846, 40928, 40926, 40927, 40852, 40857, 40856, + 40855, 40854, 40853, 40903, 40912, 40905, 40910, 40904, 40908, 40909, + 40906, 40907, 40911, 40844, 40851, 40829, 40850, 40860, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 5194, 5066, 5187, + 5169, 5170, 5238, 5239, 5119, 5216, 5176, 5250, 5251, 5142, 5021, 5072, + 5219, 5120, 5024, 5025, 5214, 5226, 5165, 5102, 5193, 5045, 5242, 5114, + 5128, 5124, 5217, 5179, 5208, 5172, 5241, 5027, 5029, 5129, 5195, 5180, + 5237, 5019, 5197, 5211, 5213, 5167, 5221, 5149, 5067, 5229, 5220, 5139, + 5020, 5079, 5110, 5231, 5117, 5185, 5189, 5132, 5043, 5196, 5174, 5177, + 5116, 5254, 5184, 5133, 5232, 5209, 5108, 5115, 5166, 5171, 5183, 5135, + 5182, 5140, 5186, 5123, 5127, 5253, 5026, 5022, 5252, 5141, 5075, 5046, + 5164, 5240, 5181, 5190, 5173, 5018, 5150, 5178, 5175, 5071, 5143, 5017, + 5233, 5074, 5212, 5215, 5044, 5073, 5198, 5255, 5235, 5191, 5230, 5234, + 5188, 5236, 5192, 5105, 5031, 5070, 5168, 5223, 5224, 5222, 5225, 5118, + 5068, 5243, 5244, 5207, 5131, 5064, 5136, 5137, 5138, 5028, 5030, 5063, + 5228, 5218, 5134, 5144, 5148, 5147, 5146, 5145, 5104, 5101, 5100, 5058, + 5060, 5061, 5059, 5227, 5032, 5112, 5051, 5015, 5009, 5010, 5013, 5014, + 5012, 5011, 5016, 5157, 5152, 5153, 5151, 5162, 5161, 5160, 5159, 5163, + 5155, 5113, 5023, 5078, 5077, 5076, 5158, 5156, 5154, 5106, 5107, 5069, + 5111, 5109, 5080, 5087, 5083, 5091, 5086, 5095, 5085, 5084, 5081, 5082, + 5089, 5090, 5097, 5094, 5092, 5041, 5042, 5040, 5088, 5096, 5249, 5054, + 5052, 5055, 5057, 5056, 5053, 5245, 5247, 5246, 5248, 5098, 5099, 5048, + 5047, 5049, 5050, 5203, 5206, 5204, 5205, 5199, 5202, 5200, 5201, 5062, + 5065, 5210, 5039, 5033, 5038, 5036, 5035, 5034, 5037, 5121, 5125, 5122, + 5126, 5130, 5103, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 28489, 28366, 28382, 28474, 28361, 28486, 28424, 28475, + 28473, 28362, 28358, 28485, 28352, 28470, 28471, 28472, 28377, 28378, + 28315, 28316, 28311, 28310, 28441, 28512, 28509, 28384, 28383, 28490, + 28491, 28385, 28394, 28395, 28396, 28355, 28387, 28388, 28389, 28368, + 28369, 40951, 40951, 28432, 28365, 28367, 28393, 28392, 28437, 28436, + 28488, 28487, 28464, 28465, 28351, 28357, 28453, 28454, 28468, 28469, + 28433, 28535, 28407, 28467, 28376, 28493, 28511, 28495, 28440, 28533, + 28456, 28356, 28496, 28497, 28520, 28521, 28524, 28525, 28522, 28523, + 28516, 28517, 28518, 28519, 28430, 28431, 28526, 28527, 28455, 28531, + 28438, 28435, 28319, 28320, 28314, 28534, 28406, 28466, 28375, 28492, + 28510, 28494, 28439, 28340, 28341, 28344, 28345, 28346, 28379, 28380, + 28381, 28323, 28330, 28331, 28332, 28333, 28334, 28308, 28373, 28309, + 28374, 28307, 28371, 28306, 28370, 28321, 28339, 28347, 28338, 28324, + 28325, 28322, 28349, 28404, 28403, 28328, 28350, 28335, 28342, 28348, + 28327, 28343, 28476, 28499, 28538, 28463, 28426, 28386, 28354, 28363, + 28400, 28399, 28515, 28528, 28537, 28529, 28530, 28442, 28445, 28446, + 28447, 28448, 28449, 28450, 28451, 28452, 28443, 28444, 28408, 28434, + 28372, 28364, 28326, 28329, 28336, 28337, 28458, 28457, 28405, 28398, + 28397, 28536, 28359, 28360, 28425, 28421, 28312, 28479, 28481, 28427, + 28429, 28482, 28484, 28390, 28391, 28423, 28422, 28313, 28480, 28428, + 28483, 28507, 28506, 28508, 28505, 28501, 28502, 28503, 28504, 28353, + 28401, 28402, 28498, 28532, 28460, 28318, 28477, 28317, 28513, 28461, + 28462, 28478, 28514, 28459, 28409, 28412, 28416, 28419, 28418, 28417, + 28413, 28414, 28410, 28411, 28415, 28500, 28420, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 18636, 18624, + 18647, 18648, 18630, 18649, 18650, 18651, 18652, 18637, 18638, 18639, + 18640, 18641, 18642, 18643, 18644, 18645, 18646, 18625, 18626, 18627, + 18628, 18629, 18631, 18632, 18633, 18634, 18635, 18356, 18364, 18377, + 18385, 18391, 18392, 18357, 18358, 18359, 18360, 18361, 18362, 18363, + 18365, 18366, 18367, 18368, 18369, 18370, 18371, 18372, 18373, 18374, + 18375, 18376, 18378, 18379, 18380, 18381, 18382, 18383, 18384, 18386, + 18387, 18388, 18389, 18390, 8271, 8270, 8272, 18416, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 21607, + 21608, 21605, 21603, 21594, 21593, 21600, 21598, 21589, 21596, 21606, + 21591, 21604, 21602, 21595, 21592, 21601, 21599, 21590, 21597, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 26844, 26845, 26842, 26840, 26831, 26830, 26837, 26835, 26826, + 26833, 26843, 26828, 26841, 26839, 26832, 26829, 26838, 26836, 26827, + 26834, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 28177, 10885, 10886, 10880, 10879, 10878, 36785, + 36805, 36826, 36774, 36819, 36783, 36769, 36828, 36773, 36787, 36793, + 36816, 36817, 36831, 36837, 36784, 36815, 36845, 36803, 36770, 36833, + 36835, 36802, 36848, 36782, 36797, 36794, 36775, 36789, 36772, 36830, + 36823, 36777, 36820, 36809, 36843, 36832, 36806, 36834, 36822, 36836, + 36811, 36799, 36842, 36812, 36798, 36829, 36838, 36808, 36844, 36781, + 36825, 36801, 36847, 36791, 36776, 36813, 36810, 36824, 36768, 36796, + 36795, 36846, 36840, 36818, 36788, 36786, 36792, 36800, 36839, 36841, + 36814, 36780, 36778, 36807, 36771, 36779, 36827, 36790, 36821, 36804, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 8669, + 8667, 8666, 8663, 8662, 8665, 8664, 8670, 8668, 8660, 8658, 8657, 8654, + 8653, 8656, 8655, 8661, 8659, 20515, 20514, 20513, 20512, 20511, 35370, + 35369, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 25824, 25826, 25854, 25817, + 25830, 25862, 25833, 25863, 25835, 25864, 25837, 25839, 25856, 25858, + 25841, 25844, 25865, 25848, 25850, 25820, 25852, 25866, 25867, 25860, + 25868, 25828, 25872, 25874, 25907, 25869, 25878, 25881, 25883, 25915, + 25885, 25916, 25887, 25889, 25909, 25911, 25891, 25894, 25917, 25898, + 25900, 25902, 25905, 25918, 25919, 25913, 25920, 25876, 26312, 26314, + 26344, 26318, 26320, 26352, 26323, 26353, 26325, 26354, 26327, 26329, + 26346, 26348, 26331, 26334, 26355, 26338, 26340, 26308, 26342, 26356, + 26357, 26350, 26358, 26316, 26260, 26262, 26295, 26256, 26266, 26269, + 26271, 40951, 26273, 26303, 26275, 26277, 26297, 26299, 26279, 26282, + 26304, 26286, 26288, 26290, 26293, 26305, 26306, 26301, 26307, 26264, + 25977, 25979, 26009, 25983, 25985, 26017, 25988, 26018, 25990, 26019, + 25992, 25994, 26011, 26013, 25996, 25999, 26020, 26003, 26005, 25973, + 26007, 26021, 26022, 26015, 26023, 25981, 26031, 26033, 26068, 26037, + 26039, 26042, 26044, 26076, 26046, 26077, 26048, 26050, 26070, 26072, + 26052, 26055, 26078, 26059, 26061, 26063, 26066, 26079, 26080, 26074, + 26081, 26035, 26784, 40951, 26785, 26786, 40951, 40951, 26787, 40951, + 40951, 26788, 26789, 40951, 40951, 26790, 26791, 26792, 26793, 40951, + 26794, 26795, 26796, 26797, 26798, 26799, 26800, 26801, 26802, 26803, + 26804, 26805, 40951, 26806, 40951, 26807, 26808, 26809, 26810, 26811, + 26812, 26813, 40951, 26814, 26815, 26816, 26817, 26818, 26819, 26820, + 26821, 26822, 26823, 26824, 25921, 25922, 25923, 25924, 25925, 25926, + 25927, 25928, 25929, 25930, 25931, 25932, 25933, 25934, 25935, 25936, + 25937, 25938, 25939, 25940, 25941, 25942, 25943, 25944, 25945, 25946, + 25947, 25948, 25949, 25950, 25951, 25952, 25953, 25954, 25955, 25956, + 25957, 25958, 25959, 25960, 25961, 25962, 25963, 25964, 25965, 25966, + 25967, 25968, 25969, 25970, 25971, 25972, 26208, 26209, 40951, 26210, + 26211, 26212, 26213, 40951, 40951, 26214, 26215, 26216, 26217, 26218, + 26219, 26220, 26221, 40951, 26222, 26223, 26224, 26225, 26226, 26227, + 26228, 40951, 26229, 26230, 26231, 26232, 26233, 26234, 26235, 26236, + 26237, 26238, 26239, 26240, 26241, 26242, 26243, 26244, 26245, 26246, + 26247, 26248, 26249, 26250, 26251, 26252, 26253, 26254, 26153, 26154, + 40951, 26155, 26156, 26157, 26158, 40951, 26159, 26160, 26161, 26162, + 26163, 40951, 26164, 40951, 40951, 40951, 26165, 26166, 26167, 26168, + 26169, 26170, 26171, 40951, 26172, 26173, 26174, 26175, 26176, 26177, + 26178, 26179, 26180, 26181, 26182, 26183, 26184, 26185, 26186, 26187, + 26188, 26189, 26190, 26191, 26192, 26193, 26194, 26195, 26196, 26197, + 26091, 26092, 26093, 26094, 26095, 26096, 26097, 26098, 26099, 26100, + 26101, 26102, 26103, 26104, 26105, 26106, 26107, 26108, 26109, 26110, + 26111, 26112, 26113, 26114, 26115, 26116, 26117, 26118, 26119, 26120, + 26121, 26122, 26123, 26124, 26125, 26126, 26127, 26128, 26129, 26130, + 26131, 26132, 26133, 26134, 26135, 26136, 26137, 26138, 26139, 26140, + 26141, 26142, 26722, 26723, 26724, 26725, 26726, 26727, 26728, 26729, + 26730, 26731, 26732, 26733, 26734, 26735, 26736, 26737, 26738, 26739, + 26740, 26741, 26742, 26743, 26744, 26745, 26746, 26747, 26748, 26749, + 26750, 26751, 26752, 26753, 26754, 26755, 26756, 26757, 26758, 26759, + 26760, 26761, 26762, 26763, 26764, 26765, 26766, 26767, 26768, 26769, + 26770, 26771, 26772, 26773, 26554, 26556, 26586, 26560, 26562, 26594, + 26565, 26595, 26567, 26596, 26569, 26571, 26588, 26590, 26573, 26576, + 26597, 26580, 26582, 26550, 26584, 26598, 26599, 26592, 26600, 26558, + 26608, 26610, 26645, 26614, 26616, 26619, 26621, 26653, 26623, 26654, + 26625, 26627, 26647, 26649, 26629, 26632, 26655, 26636, 26638, 26640, + 26643, 26656, 26657, 26651, 26658, 26612, 26670, 26671, 26672, 26673, + 26674, 26675, 26676, 26677, 26678, 26679, 26680, 26681, 26682, 26683, + 26684, 26685, 26686, 26687, 26688, 26689, 26690, 26691, 26692, 26693, + 26694, 26695, 26696, 26697, 26698, 26699, 26700, 26701, 26702, 26703, + 26704, 26705, 26706, 26707, 26708, 26709, 26710, 26711, 26712, 26713, + 26714, 26715, 26716, 26717, 26718, 26719, 26720, 26721, 26444, 26446, + 26476, 26450, 26452, 26484, 26455, 26485, 26457, 26486, 26459, 26461, + 26478, 26480, 26463, 26466, 26487, 26470, 26472, 26440, 26474, 26488, + 26489, 26482, 26490, 26448, 26498, 26500, 26535, 26504, 26506, 26509, + 26511, 26543, 26513, 26544, 26515, 26517, 26537, 26539, 26519, 26522, + 26545, 26526, 26528, 26530, 26533, 26546, 26547, 26541, 26548, 26502, + 26367, 26368, 26369, 26370, 26371, 26372, 26373, 26374, 26375, 26376, + 26377, 26378, 26379, 26380, 26381, 26382, 26383, 26384, 26385, 26386, + 26387, 26388, 26389, 26390, 26391, 26392, 26393, 26394, 26395, 26396, + 26397, 26398, 26399, 26400, 26401, 26402, 26403, 26404, 26405, 26406, + 26407, 26408, 26409, 26410, 26411, 26412, 26413, 26414, 26415, 26416, + 26417, 26418, 26257, 26258, 40951, 40951, 25825, 25827, 25834, 25819, + 25831, 25829, 25832, 25821, 25836, 25838, 25840, 25857, 25859, 25861, + 25842, 25847, 25849, 25822, 25851, 25823, 25853, 25845, 25855, 25846, + 25843, 26085, 25873, 25875, 25884, 25871, 25879, 25877, 25880, 25903, + 25886, 25888, 25890, 25910, 25912, 25914, 25892, 25897, 25899, 25882, + 25901, 25904, 25906, 25895, 25908, 25896, 25893, 26087, 26083, 26090, + 26084, 26086, 26089, 26088, 26313, 26315, 26324, 26319, 26321, 26317, + 26322, 26309, 26326, 26328, 26330, 26347, 26349, 26351, 26332, 26337, + 26339, 26310, 26341, 26311, 26343, 26335, 26345, 26336, 26333, 26361, + 26261, 26263, 26272, 26259, 26267, 26265, 26268, 26291, 26274, 26276, + 26278, 26298, 26300, 26302, 26280, 26285, 26287, 26270, 26289, 26292, + 26294, 26283, 26296, 26284, 26281, 26363, 26359, 26366, 26360, 26362, + 26365, 26364, 25978, 25980, 25989, 25984, 25986, 25982, 25987, 25974, + 25991, 25993, 25995, 26012, 26014, 26016, 25997, 26002, 26004, 25975, + 26006, 25976, 26008, 26000, 26010, 26001, 25998, 26026, 26032, 26034, + 26045, 26038, 26040, 26036, 26041, 26064, 26047, 26049, 26051, 26071, + 26073, 26075, 26053, 26058, 26060, 26043, 26062, 26065, 26067, 26056, + 26069, 26057, 26054, 26028, 26024, 26082, 26025, 26027, 26030, 26029, + 26555, 26557, 26566, 26561, 26563, 26559, 26564, 26551, 26568, 26570, + 26572, 26589, 26591, 26593, 26574, 26579, 26581, 26552, 26583, 26553, + 26585, 26577, 26587, 26578, 26575, 26603, 26609, 26611, 26622, 26615, + 26617, 26613, 26618, 26641, 26624, 26626, 26628, 26648, 26650, 26652, + 26630, 26635, 26637, 26620, 26639, 26642, 26644, 26633, 26646, 26634, + 26631, 26605, 26601, 26659, 26602, 26604, 26607, 26606, 26445, 26447, + 26456, 26451, 26453, 26449, 26454, 26441, 26458, 26460, 26462, 26479, + 26481, 26483, 26464, 26469, 26471, 26442, 26473, 26443, 26475, 26467, + 26477, 26468, 26465, 26493, 26499, 26501, 26512, 26505, 26507, 26503, + 26508, 26531, 26514, 26516, 26518, 26538, 26540, 26542, 26520, 26525, + 26527, 26510, 26529, 26532, 26534, 26523, 26536, 26524, 26521, 26495, + 26491, 26549, 26492, 26494, 26497, 26496, 25818, 25870, 40951, 40951, + 26149, 26151, 26148, 26147, 26144, 26143, 26146, 26145, 26152, 26150, + 26204, 26206, 26203, 26202, 26199, 26198, 26201, 26200, 26207, 26205, + 26780, 26782, 26779, 26778, 26775, 26774, 26777, 26776, 26783, 26781, + 26666, 26668, 26665, 26664, 26661, 26660, 26663, 26662, 26669, 26667, + 26425, 26427, 26424, 26423, 26420, 26419, 26422, 26421, 26428, 26426, + 33092, 33048, 33073, 33289, 33244, 33030, 33093, 33057, 33206, 33158, + 33134, 33095, 33098, 33055, 33099, 33049, 33100, 33122, 33117, 33155, + 33096, 33102, 33111, 33112, 33103, 33109, 33113, 33051, 33185, 33094, + 33123, 33052, 33132, 33101, 33131, 33118, 33157, 33156, 33097, 33116, + 33126, 33127, 33130, 33129, 33197, 33105, 33106, 33107, 33179, 33147, + 33110, 33114, 33108, 33104, 33178, 33139, 33177, 33176, 33136, 33135, + 33138, 33128, 33125, 33124, 33174, 33173, 33175, 33146, 33222, 33226, + 33225, 33223, 33224, 33067, 33213, 33243, 33215, 33228, 33220, 33229, + 33221, 33230, 33219, 33077, 33078, 33242, 33283, 33216, 33217, 33218, + 33214, 33240, 33227, 33238, 33231, 33239, 33237, 33235, 33232, 33233, + 33234, 33236, 33064, 33070, 33068, 33069, 33273, 33272, 33080, 33072, + 33083, 33086, 33081, 33084, 33082, 33085, 33090, 33087, 33047, 33282, + 33288, 33285, 33287, 33261, 33263, 33241, 33271, 33264, 33268, 33262, + 33270, 33269, 33267, 33029, 33119, 33054, 33246, 33032, 33255, 33121, + 33120, 33247, 33160, 33163, 33162, 33161, 33170, 33210, 33059, 33284, + 33041, 33166, 33169, 33164, 33165, 33258, 33168, 33257, 33040, 33039, + 33167, 33058, 33256, 33038, 33133, 33053, 33248, 33265, 33031, 33115, + 33050, 33186, 33266, 33042, 33194, 33190, 33192, 33063, 33286, 33043, + 33187, 33189, 33188, 33193, 33191, 33281, 33159, 33056, 33088, 33275, + 33276, 33274, 33076, 33254, 33034, 33033, 33183, 33259, 33181, 33062, + 33171, 33182, 33280, 33180, 33184, 33172, 33060, 33089, 33079, 33260, + 33045, 33046, 33044, 33061, 33065, 33066, 33278, 33279, 33277, 33245, + 33148, 33250, 33151, 33152, 33153, 33150, 33154, 33149, 33143, 33144, + 33145, 33142, 33140, 33071, 33141, 33137, 33074, 33075, 33252, 33253, + 33249, 33251, 33036, 33037, 33035, 33195, 33200, 33203, 33204, 33205, + 33211, 33196, 33198, 33199, 33207, 33202, 33208, 33209, 33201, 33091, + 33212, 33603, 33602, 33601, 33623, 33622, 33621, 33578, 33577, 33576, + 32962, 32961, 32960, 33564, 33563, 33562, 33574, 33575, 33570, 33573, + 33569, 33572, 33571, 33019, 33022, 33018, 33021, 33020, 33568, 33438, + 33439, 33440, 33441, 33436, 33437, 33442, 33482, 33387, 33500, 33499, + 33497, 33498, 33501, 33476, 33479, 33477, 33478, 33475, 33506, 33505, + 33503, 33504, 33452, 33451, 33450, 33456, 33455, 33454, 33453, 33474, + 33473, 33472, 33449, 33448, 33447, 33522, 33521, 33520, 33496, 33495, + 33494, 33619, 33618, 33617, 33616, 33615, 33614, 33620, 33613, 33612, + 33611, 33356, 33355, 33353, 33354, 33360, 33359, 33357, 33358, 33348, + 33347, 33345, 33346, 33352, 33351, 33349, 33350, 33410, 33409, 33407, + 33408, 33411, 33425, 33428, 33426, 33427, 33384, 33415, 33414, 33413, + 33412, 33370, 33383, 33382, 33381, 33371, 33369, 33368, 33367, 33434, + 33433, 33432, 33431, 33430, 33429, 33606, 33605, 33604, 33609, 33608, + 33607, 33610, 33469, 33468, 33466, 33467, 33460, 33459, 33457, 33458, + 33465, 33464, 33487, 33486, 33483, 33488, 33493, 33490, 33489, 33509, + 33508, 33507, 33512, 33511, 33510, 33463, 33471, 33470, 33559, 33556, + 33555, 33502, 33462, 33485, 33492, 33517, 33561, 33558, 33554, 33461, + 33484, 33491, 33516, 33560, 33557, 33553, 33515, 33514, 33513, 33374, + 33373, 33391, 33389, 33390, 33388, 33400, 33398, 33399, 33397, 33417, + 33416, 33548, 33545, 33551, 33376, 33375, 33392, 33393, 33395, 33394, + 33404, 33402, 33403, 33401, 33419, 33418, 33549, 33546, 33552, 33380, + 33379, 33377, 33378, 33372, 33396, 33405, 33420, 33421, 33422, 33547, + 33544, 33550, 33406, 33446, 33444, 33445, 33443, 33366, 33364, 33362, + 33365, 33363, 33361, 33519, 33518, 33424, 33423, 33481, 33480, 33386, + 33385, 32978, 32977, 32973, 32976, 32980, 32979, 32974, 32975, 32972, + 32981, 33291, 33298, 33296, 33295, 33293, 33294, 33292, 33297, 33011, + 33009, 33010, 32987, 32986, 32985, 32970, 32968, 32969, 32971, 33026, + 33025, 33024, 33005, 33006, 33002, 32983, 32982, 33001, 33003, 33000, + 33004, 32984, 32999, 32997, 32998, 32994, 32996, 32995, 32988, 32990, + 32989, 32992, 32991, 32993, 32965, 32964, 32963, 33588, 33587, 33589, + 33008, 33525, 33524, 33526, 33527, 32955, 32957, 32954, 32956, 32959, + 32958, 33320, 33318, 33319, 33325, 33327, 33326, 33322, 33324, 33323, + 33339, 33338, 33337, 33331, 33332, 33333, 33334, 33335, 33336, 33328, + 33330, 33329, 33340, 33341, 33342, 33309, 33307, 33308, 33321, 33344, + 33343, 33596, 33595, 33593, 33594, 33592, 33597, 33591, 33590, 33580, + 33585, 33583, 33584, 33581, 33582, 33586, 33523, 33435, 33528, 33290, + 33007, 33566, 33565, 33028, 33023, 33567, 33599, 33598, 33600, 33624, + 33299, 33300, 33301, 33302, 33303, 33304, 33305, 33306, 33017, 33317, + 33316, 33311, 33314, 33313, 33310, 33312, 33315, 32967, 33027, 33579, + 32966, 33625, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 33012, 33013, 33014, + 33015, 33016, 40951, 33536, 33537, 33538, 33539, 33540, 33541, 33542, + 33543, 33529, 33530, 33531, 33532, 33533, 33534, 33535, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 23497, 23771, + 23262, 23776, 23249, 23621, 23882, 23773, 23868, 23837, 23242, 23476, + 23477, 23872, 23237, 23289, 23263, 23614, 23413, 23594, 23474, 23866, + 23752, 23841, 23485, 23414, 23558, 23711, 23842, 23380, 23788, 40951, + 40951, 40951, 40951, 40951, 40951, 23404, 23607, 23653, 23757, 23796, + 23832, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 8227, 8229, 8239, 8233, 8215, 8240, 8247, + 40951, 8246, 8220, 8217, 8216, 8214, 8251, 8235, 8234, 8236, 8250, 8237, + 8238, 8222, 8225, 8249, 8231, 8248, 40951, 40951, 8223, 8226, 8230, 8224, + 8242, 8241, 8243, 40951, 8245, 8221, 40951, 8244, 8219, 8228, 8218, 8232, + 40951, 40951, 40951, 40951, 40951, 27764, 27733, 27761, 27759, 27735, + 27757, 27754, 27755, 27756, 27763, 27740, 27741, 27765, 27744, 27742, + 27737, 27750, 27766, 27739, 27762, 27749, 27758, 27748, 27751, 27736, + 27753, 27734, 27747, 27731, 27760, 27732, 27745, 27743, 10520, 10498, + 10518, 10505, 10501, 10515, 10512, 10513, 10514, 10519, 10503, 10521, + 10517, 10504, 10522, 10502, 10507, 10511, 10516, 10510, 10508, 10509, + 10506, 10497, 10500, 10499, 27738, 27752, 27746, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 8144, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 29882, 29866, 29857, 29868, 29873, 29865, 29870, + 29861, 29887, 29891, 29893, 29896, 29860, 29855, 29890, 29880, 29864, + 29863, 29894, 29856, 29867, 29888, 29876, 29892, 29895, 29862, 29884, + 29869, 29859, 29879, 29858, 29874, 29881, 29883, 29889, 29875, 29871, + 29872, 29897, 29898, 29877, 29878, 29885, 29886, 29899, 40951, 40951, + 40951, 29908, 29912, 29911, 29914, 29913, 29910, 29909, 29905, 29902, + 29901, 29904, 29903, 29906, 29907, 40951, 40951, 29922, 29924, 29921, + 29920, 29917, 29916, 29919, 29918, 29925, 29923, 40951, 40951, 40951, + 40951, 29900, 29915, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 37522, 37505, 37525, 37515, 37519, 37516, 37521, + 37507, 37506, 37524, 37512, 37527, 37526, 37518, 37517, 37523, 37520, + 37508, 37500, 37509, 37501, 37529, 37510, 37502, 37511, 37503, 37528, + 37514, 37504, 37513, 37530, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 38979, 38978, 39010, 39011, 39012, 39014, 39003, 39006, 38992, + 38995, 39007, 38999, 38996, 39013, 39009, 39008, 39016, 39021, 39020, + 39019, 39005, 38984, 38983, 39018, 39017, 39004, 39015, 38987, 38989, + 38993, 39000, 38991, 38998, 38997, 38986, 38981, 38982, 38990, 38985, + 38988, 38980, 38994, 39001, 39002, 39025, 39026, 39023, 39024, 39033, + 39035, 39032, 39031, 39028, 39027, 39030, 39029, 39036, 39034, 40951, + 40951, 40951, 40951, 40951, 39022, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 28864, 28867, 28866, 28868, 28865, 28847, 28851, + 28849, 28848, 28850, 28859, 28862, 28860, 28863, 28861, 28869, 28870, + 28871, 28872, 28873, 28852, 28854, 28857, 28858, 28853, 28856, 28855, + 28877, 28874, 28878, 28876, 28875, 28885, 28887, 28884, 28883, 28880, + 28879, 28882, 28881, 28888, 28886, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 30046, + 30049, 30048, 30047, 30050, 30051, 30028, 30030, 30029, 30031, 30032, + 30033, 30040, 30041, 30045, 30042, 30043, 30044, 30052, 30056, 30053, + 30055, 30054, 30057, 30034, 30039, 30038, 30036, 30035, 30037, 30060, + 30058, 30059, 30068, 30070, 30067, 30066, 30063, 30062, 30065, 30064, + 30071, 30069, 40951, 40951, 40951, 40951, 30061, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 17006, 17012, 17010, 17007, 17009, 17008, 17011, 40951, 16961, + 17005, 17003, 17002, 40951, 16949, 16948, 40951, 16960, 16959, 16958, + 16945, 16944, 16957, 16956, 16955, 16954, 16953, 16952, 16947, 16946, + 16951, 16950, 40951, 27076, 27077, 27078, 27144, 27165, 27153, 27118, + 27253, 27079, 27080, 27084, 27203, 27188, 27193, 27117, 27271, 27220, + 27142, 27123, 27214, 27083, 27081, 27082, 27129, 27173, 27226, 27266, + 27089, 27085, 27093, 27230, 27168, 27178, 27205, 27094, 27091, 27090, + 27243, 27181, 27241, 27219, 27210, 27208, 27209, 27272, 27251, 27088, + 27103, 27095, 27242, 27187, 27212, 27147, 27273, 27099, 27100, 27101, + 27157, 27149, 27133, 27234, 27185, 27086, 27092, 27087, 27160, 27257, + 27258, 27096, 27097, 27098, 27171, 27126, 27176, 27143, 27104, 27102, + 27105, 27229, 27186, 27235, 27137, 27249, 27106, 27110, 27114, 27183, + 27155, 27223, 27204, 27107, 27111, 27112, 27154, 27146, 27211, 27164, + 27274, 27175, 27113, 27108, 27109, 27191, 27244, 27252, 27122, 27264, + 27115, 27174, 27124, 27221, 27161, 27199, 27131, 27213, 27163, 27130, + 27270, 27120, 27166, 27116, 27162, 27190, 27217, 27228, 27198, 27233, + 27200, 27159, 27179, 27260, 27227, 27194, 27237, 27267, 27240, 27238, + 27263, 27134, 27250, 27140, 27169, 27125, 27156, 27132, 27180, 27136, + 27216, 27135, 27195, 27121, 27261, 27151, 27246, 27247, 27265, 27236, + 27177, 27215, 27207, 27170, 27145, 27119, 27184, 27192, 27231, 27197, + 27127, 27222, 27167, 27182, 27148, 27150, 27256, 27196, 27202, 27201, + 27268, 27189, 27138, 27139, 27225, 27269, 27218, 27206, 27259, 27262, + 27232, 27254, 27158, 27224, 27152, 27239, 27128, 27255, 27172, 27141, + 40951, 40951, 27282, 27280, 27279, 27276, 27275, 27278, 27277, 27283, + 27281, 27071, 27070, 27074, 27072, 27069, 27073, 27075, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 27, 9, 24, + 14, 29, 21, 34, 28, 37, 39, 35, 40, 41, 10, 30, 32, 18, 15, 31, 42, 13, + 25, 36, 23, 12, 20, 33, 19, 38, 17, 11, 26, 16, 22, 62, 44, 59, 49, 64, + 56, 69, 63, 72, 74, 70, 75, 76, 45, 65, 67, 53, 50, 66, 77, 48, 60, 71, + 58, 47, 55, 68, 54, 73, 52, 46, 61, 51, 57, 85, 86, 79, 84, 43, 78, 83, + 82, 40951, 40951, 40951, 40951, 93, 95, 92, 91, 88, 87, 90, 89, 96, 94, + 40951, 40951, 40951, 40951, 80, 81, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 20698, 20684, + 20679, 20659, 20654, 20672, 20667, 20647, 20662, 20687, 20682, 20677, + 20657, 20652, 20673, 20668, 20648, 20663, 20699, 20685, 20680, 20660, + 20655, 20675, 20670, 20650, 20665, 20700, 20686, 20681, 20661, 20656, + 20676, 20671, 20651, 20666, 20688, 20683, 20678, 20658, 20653, 20674, + 20669, 20649, 20664, 20645, 20646, 20636, 20643, 20644, 20696, 20694, + 20693, 20690, 20689, 20692, 20691, 20697, 20695, 20702, 20638, 20637, + 20639, 20701, 20642, 20641, 20640, 20635, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 30827, 30822, 30817, 30797, 30792, 30810, 30805, 30785, + 30800, 30825, 30820, 30815, 30795, 30790, 30811, 30806, 30786, 30801, + 30828, 30823, 30818, 30798, 30793, 30813, 30808, 30788, 30803, 30829, + 30824, 30819, 30799, 30794, 30814, 30809, 30789, 30804, 30826, 30821, + 30816, 30796, 30791, 30812, 30807, 30787, 30802, 30784, 30777, 30779, + 30769, 30771, 30772, 30774, 30781, 30780, 30775, 30770, 30773, 30778, + 30776, 30783, 30782, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 2246, 2254, 2252, 2150, + 40951, 2262, 2250, 2258, 2242, 2257, 2249, 2198, 2253, 2260, 2225, 2247, + 2255, 2226, 2261, 2256, 2224, 2245, 2244, 2248, 2243, 2149, 2251, 2259, + 2120, 2122, 2121, 2123, 40951, 2162, 2160, 40951, 2157, 40951, 40951, + 2156, 40951, 2164, 2159, 2167, 2161, 2166, 2154, 2169, 2163, 2155, 2168, + 40951, 2153, 2152, 2151, 2158, 40951, 2170, 40951, 2165, 40951, 40951, + 40951, 40951, 40951, 40951, 2234, 40951, 40951, 40951, 40951, 2236, + 40951, 2235, 40951, 2239, 40951, 2238, 2231, 2241, 40951, 2232, 2240, + 40951, 2230, 40951, 40951, 2233, 40951, 2229, 40951, 2237, 40951, 2227, + 40951, 2228, 40951, 2216, 2214, 40951, 2211, 40951, 40951, 2210, 2205, + 2218, 2213, 40951, 2215, 2221, 2208, 2223, 2217, 2209, 2222, 40951, 2207, + 2206, 2204, 2212, 40951, 2203, 2219, 2220, 2201, 40951, 2202, 40951, + 2175, 2187, 2185, 2195, 2178, 2197, 2183, 2177, 2181, 2190, 40951, 2193, + 2186, 2192, 2172, 2176, 2188, 2173, 2196, 2189, 2171, 2182, 2180, 2174, + 2179, 2194, 2184, 2191, 40951, 40951, 40951, 40951, 40951, 2136, 2134, + 2145, 40951, 2148, 2132, 2140, 2130, 2139, 40951, 2143, 2135, 2142, 2125, + 2147, 2137, 2126, 2146, 2138, 2124, 2131, 2129, 2127, 2128, 2144, 2133, + 2141, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 2199, 2200, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 25385, + 25401, 25416, 25392, 25420, 25419, 25417, 25397, 25414, 25411, 25390, + 25387, 25406, 25403, 25383, 25394, 25398, 25415, 25412, 25391, 25388, + 25407, 25404, 25384, 25395, 25396, 25413, 25410, 25389, 25386, 25405, + 25402, 25382, 25393, 25400, 25399, 25380, 25423, 25409, 25408, 25421, + 25418, 25422, 25381, 40951, 40951, 40951, 40951, 11138, 11089, 11090, + 11091, 11092, 11093, 11094, 11095, 11096, 11097, 11098, 11099, 11100, + 11101, 11102, 11103, 11104, 11105, 11106, 11107, 11108, 11109, 11110, + 11111, 11112, 11113, 11114, 11115, 11116, 11117, 11118, 11119, 11120, + 11121, 11122, 11123, 11124, 11125, 11126, 11127, 11128, 11129, 11130, + 11131, 11132, 11133, 11134, 11135, 11136, 11137, 11188, 11139, 11140, + 11141, 11142, 11143, 11144, 11145, 11146, 11147, 11148, 11149, 11150, + 11151, 11152, 11153, 11154, 11155, 11156, 11157, 11158, 11159, 11160, + 11161, 11162, 11163, 11164, 11165, 11166, 11167, 11168, 11169, 11170, + 11171, 11172, 11173, 11174, 11175, 11176, 11177, 11178, 11179, 11180, + 11181, 11182, 11183, 11184, 11185, 11186, 11187, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 31485, + 31555, 31530, 31526, 31489, 31494, 31514, 31510, 31506, 31559, 31522, + 31563, 31498, 31518, 31502, 40951, 40951, 31556, 31531, 31527, 31490, + 31495, 31515, 31511, 31507, 31560, 31523, 31564, 31499, 31519, 31503, + 31486, 40951, 31557, 31532, 31528, 31491, 31496, 31516, 31512, 31508, + 31561, 31524, 31565, 31500, 31520, 31504, 31484, 40951, 31554, 31529, + 31525, 31488, 31493, 31513, 31509, 31505, 31558, 31521, 31562, 31497, + 31517, 31501, 31487, 31492, 31536, 31533, 31547, 31548, 31549, 31550, + 31551, 31552, 31553, 31537, 31538, 31539, 31540, 31541, 31542, 31543, + 31544, 31545, 31546, 31534, 31535, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 10874, 10873, 10858, 10870, 10867, + 10852, 10849, 10864, 10861, 10876, 10855, 10915, 10894, 6813, 6530, 6554, + 31156, 31157, 31158, 31159, 31160, 31161, 31162, 31163, 31164, 31165, + 31166, 31167, 31168, 31169, 31170, 31171, 31172, 31173, 31174, 31175, + 31176, 31177, 31178, 31179, 31180, 31181, 37498, 6644, 6645, 6541, 6812, + 8646, 34420, 34421, 34422, 34423, 34424, 34425, 34426, 34427, 34428, + 34429, 34430, 34431, 34432, 34433, 34434, 34435, 34436, 34437, 34438, + 34439, 34440, 34441, 34442, 34443, 34444, 34445, 34414, 34450, 34465, + 34466, 34455, 34477, 28977, 28978, 28979, 28980, 28981, 28982, 28983, + 28984, 28985, 28986, 28987, 28988, 28989, 28990, 28991, 28992, 28993, + 28994, 28995, 28996, 28997, 28998, 28999, 29000, 29001, 29002, 31764, + 31765, 31766, 6540, 6539, 6587, 29008, 29009, 29010, 29011, 29012, 29013, + 29014, 29015, 29016, 29017, 29018, 29019, 29020, 29021, 29022, 29023, + 29024, 29025, 29026, 29027, 29028, 29029, 29030, 29031, 29032, 29033, + 8690, 29040, 29043, 29044, 29039, 29041, 34149, 34403, 34402, 34409, + 34478, 34451, 34452, 34454, 34464, 34472, 34475, 34467, 34457, 34469, + 34407, 34471, 34408, 34458, 34468, 34460, 34453, 34419, 34413, 34412, + 34411, 34448, 34459, 34473, 34474, 25815, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 31794, 31795, 31796, 31797, 31798, 31799, 31800, 31801, + 31802, 31803, 31804, 31805, 31806, 31807, 31808, 31809, 31810, 31811, + 31812, 31813, 31814, 31815, 31816, 31817, 31818, 31819, 34190, 34415, + 34417, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 34381, 34376, 34368, 34416, 34362, 34374, + 34397, 34373, 34361, 34386, 34394, 34385, 34366, 34377, 34378, 34384, + 34365, 34395, 34392, 34399, 34375, 34370, 34389, 34379, 34382, 34359, + 34360, 34401, 34372, 34363, 34367, 34383, 34398, 34380, 34393, 34396, + 34369, 34390, 34388, 34387, 34391, 34364, 34371, 34400, 40951, 40951, + 40951, 40951, 37495, 37489, 37490, 37492, 37496, 37493, 37497, 37491, + 37494, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 6592, 6590, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 32230, 32231, 32228, 32232, 32227, 32229, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 10525, 17488, 8037, 29287, 34652, 34651, + 6823, 34697, 31757, 4931, 39127, 38952, 27585, 11538, 11536, 11537, + 18006, 29092, 39039, 17455, 39040, 17529, 39038, 22740, 39037, 8680, + 29091, 17454, 22739, 17528, 34576, 18007, 32825, 36989, 3828, 39284, + 39288, 39285, 39286, 8050, 8049, 8048, 8047, 17487, 39351, 20436, 36648, + 5007, 6476, 32563, 17383, 10551, 31024, 6124, 20433, 37663, 6474, 32218, + 20393, 34698, 4242, 11532, 11533, 20218, 17506, 25667, 17423, 24034, + 28299, 37448, 2541, 18119, 27068, 39130, 35786, 24399, 3426, 31470, + 31791, 18659, 31290, 31287, 6475, 34517, 19108, 33783, 26860, 31606, + 31914, 31915, 8480, 9952, 34505, 34081, 4929, 17519, 32234, 10534, 30882, + 34738, 17527, 17461, 33878, 32769, 20466, 11284, 8479, 6513, 5999, 25270, + 9955, 20399, 32842, 3649, 31604, 8478, 17493, 36650, 32530, 39353, 8054, + 37576, 3540, 7996, 2605, 17494, 4366, 31594, 31905, 39373, 3760, 20825, + 6514, 17429, 17451, 17450, 2748, 31214, 8459, 35785, 8691, 31469, 20830, + 6061, 39350, 28182, 32535, 18044, 19689, 4368, 27584, 31858, 28305, + 34525, 24398, 8471, 3530, 3531, 17444, 98, 6059, 17435, 32176, 17459, + 27574, 28204, 6516, 19688, 2522, 37473, 6821, 37395, 7992, 31319, 38874, + 10920, 33789, 3759, 17718, 4371, 17477, 28540, 28286, 32529, 18675, + 28304, 37581, 38879, 28539, 32359, 36764, 33764, 3432, 6477, 33872, + 32360, 34736, 34112, 37578, 20431, 370, 32237, 34741, 39148, 18043, + 31747, 31748, 8682, 38953, 17465, 20468, 34929, 33869, 5265, 3536, 5006, + 20443, 6822, 10828, 7993, 10659, 10660, 28958, 34503, 20442, 20441, + 29948, 20827, 17349, 20444, 3421, 2538, 20437, 25154, 8474, 32534, 10577, + 17419, 20823, 20826, 17348, 39253, 3881, 39136, 39135, 32219, 3897, + 22564, 2616, 4375, 369, 16743, 16744, 16745, 16746, 16747, 31772, 28199, + 30885, 39128, 8673, 37293, 24296, 31774, 6066, 11374, 8694, 39307, 33866, + 33867, 20430, 31777, 18011, 32848, 28180, 32217, 6482, 11025, 31461, + 4553, 16712, 29946, 34106, 4944, 964, 20398, 22566, 17458, 37577, 4243, + 37688, 19657, 2604, 17523, 3761, 31291, 22561, 31616, 11376, 2614, 11087, + 28201, 8674, 37294, 31775, 6067, 11375, 34111, 20432, 28181, 11086, + 31463, 17526, 19106, 39372, 3535, 31061, 31462, 31280, 6481, 17380, + 17378, 11531, 29441, 28202, 37449, 39295, 39210, 39236, 39260, 17462, + 39137, 30881, 37021, 37022, 7991, 30538, 8696, 39362, 17379, 29282, + 34926, 20927, 11382, 22556, 3765, 39361, 31720, 19112, 31611, 25661, + 2535, 20289, 39363, 39360, 17492, 5002, 5001, 4551, 17906, 25574, 39358, + 17427, 25580, 37703, 37704, 31591, 39359, 4932, 31304, 25577, 25578, + 30517, 30515, 2603, 8462, 31673, 20833, 20832, 18975, 2606, 17367, 348, + 20595, 33766, 20711, 18674, 10533, 25054, 28889, 17415, 18979, 3430, + 34924, 31466, 22553, 25153, 32156, 17722, 22548, 4367, 8672, 39146, 3537, + 4938, 37697, 34082, 18672, 19691, 4245, 18661, 39397, 31719, 19690, + 31901, 19692, 10836, 16700, 962, 4550, 33778, 8059, 34107, 11378, 10538, + 31464, 17472, 11005, 34095, 36996, 39221, 20453, 28002, 9950, 19721, + 8679, 3423, 3425, 3424, 3422, 28001, 6295, 32559, 31314, 4933, 27587, + 17478, 30549, 11529, 17440, 30536, 30889, 30891, 5262, 36651, 6004, 6294, + 6296, 3428, 7997, 31722, 32225, 31282, 34515, 37547, 4256, 24397, 29438, + 29439, 8043, 30530, 18660, 4244, 30553, 4257, 28960, 32556, 27421, 36656, + 30892, 17426, 32443, 31723, 6301, 30831, 20822, 31281, 17381, 20627, + 16781, 8040, 8041, 30540, 30539, 31600, 31597, 29277, 27601, 27602, + 38871, 27603, 29357, 966, 5263, 5264, 38873, 36663, 31750, 38875, 17443, + 31595, 31687, 37691, 8027, 8028, 8024, 975, 25155, 20284, 34090, 34089, + 34091, 34092, 3529, 16698, 24165, 32032, 25112, 8042, 21611, 25109, + 30543, 3543, 3545, 4255, 25052, 31752, 2608, 16776, 30519, 33928, 37488, + 29356, 21626, 20714, 20715, 20718, 20717, 20716, 17447, 16699, 39376, + 19102, 29852, 20445, 31475, 27573, 36662, 8701, 33760, 20831, 37549, + 3908, 39271, 22730, 22657, 22665, 22658, 33795, 33794, 37786, 11297, + 37790, 11291, 25224, 37882, 6537, 8688, 8687, 29431, 29433, 34813, 39234, + 19736, 6132, 30883, 11369, 21609, 28188, 34834, 27284, 4369, 8008, 8018, + 8020, 8004, 8002, 8012, 8010, 8000, 8006, 8014, 7998, 8016, 8009, 8019, + 8021, 8005, 8003, 8013, 8011, 8001, 8007, 8015, 7999, 8017, 31979, 31980, + 31981, 4997, 4998, 32164, 4254, 5998, 25664, 3910, 29355, 20397, 25575, + 33781, 10535, 34102, 34103, 20928, 25579, 24086, 36657, 31960, 39290, + 3923, 36661, 7994, 2609, 34487, 16780, 17486, 31294, 25051, 3878, 25186, + 25172, 25184, 25185, 25206, 24146, 37679, 31762, 31886, 31894, 31895, + 31899, 31896, 31763, 39211, 32951, 32950, 32947, 32946, 3856, 3883, + 32953, 32952, 32949, 32948, 3926, 3822, 3835, 10661, 21613, 37012, 31670, + 31617, 3838, 39225, 33879, 36645, 39356, 30527, 37692, 37008, 37533, + 30341, 19655, 32540, 31671, 17425, 30550, 11011, 11012, 11013, 17516, + 17518, 17517, 3834, 17490, 30537, 6005, 6006, 17441, 16748, 16749, 16750, + 29435, 29436, 29437, 16759, 16752, 16753, 11010, 30888, 30893, 39142, + 34105, 34104, 10662, 27588, 26846, 30874, 8026, 5996, 20629, 10552, 8454, + 30514, 32175, 30890, 34513, 10532, 25053, 34093, 37004, 37003, 37005, + 37006, 24114, 31982, 37707, 37010, 24131, 31996, 24044, 31918, 28185, + 24422, 24421, 2753, 2759, 2754, 2758, 2751, 24419, 2752, 39367, 28198, + 37532, 34500, 33763, 28203, 18664, 18667, 17405, 33854, 33856, 33855, + 33857, 33853, 33852, 39354, 33858, 17385, 31857, 33851, 33861, 33864, + 29089, 17360, 37743, 17388, 31292, 8460, 8461, 22549, 17416, 22550, + 22551, 17402, 17403, 17404, 10922, 39368, 963, 31609, 8700, 31316, 17410, + 10921, 17525, 960, 17431, 39144, 33780, 37394, 18669, 25269, 17393, + 20452, 17395, 17386, 2530, 17479, 33779, 11006, 17413, 17390, 18668, + 6068, 33850, 33849, 6069, 22552, 31608, 8699, 39143, 33785, 33784, 37894, + 17409, 17398, 17392, 31313, 32564, 19694, 34088, 19654, 31311, 31310, + 31306, 31308, 29400, 33984, 29373, 33971, 37677, 37686, 37676, 37685, + 29399, 33983, 29372, 33970, 19772, 19765, 19769, 19762, 29401, 33985, + 29374, 33972, 19773, 19766, 19770, 19763, 20395, 20396, 33924, 33925, + 24289, 37928, 32119, 11364, 32550, 19767, 24401, 19744, 19699, 34739, + 32437, 32438, 32439, 19789, 32440, 19756, 38862, 38859, 6297, 31867, + 32174, 19930, 34504, 31755, 20287, 20288, 37540, 27420, 24408, 34501, + 37536, 37537, 5000, 30524, 37575, 5003, 27589, 371, 17448, 31589, 30523, + 36649, 30522, 2537, 30520, 31790, 10557, 2521, 37534, 28179, 28195, + 34737, 28196, 160, 32824, 32233, 34094, 20423, 38853, 8463, 31590, 37546, + 11370, 29353, 33865, 29358, 31721, 11368, 31602, 29362, 3755, 29351, + 3754, 28197, 31323, 29354, 6479, 27285, 39364, 31862, 2607, 37531, 39126, + 32847, 3527, 3528, 31222, 9954, 2620, 24087, 37543, 31684, 6646, 4552, + 17907, 8671, 33777, 32827, 3547, 3656, 31480, 29945, 32826, 34527, 30894, + 20391, 20455, 16713, 40951, 40951, 40951, 40951, 39357, 31566, 39150, + 32157, 19103, 32820, 29979, 28193, 31754, 28190, 37784, 37781, 37789, + 33792, 29408, 227, 228, 40951, 40951, 40951, 32442, 30521, 10847, 31218, + 32538, 28191, 6001, 33782, 17482, 33769, 2539, 31460, 32177, 40951, + 40951, 40951, 295, 243, 344, 342, 339, 237, 235, 236, 233, 234, 332, 334, 335, 321, 288, 250, 281, 282, 283, 265, 309, 286, 336, 337, 307, 308, 270, 323, 276, 277, 259, 300, 256, 278, 319, 257, 258, 255, 310, 317, 338, 329, 279, 238, 318, 311, 316, 333, 298, 299, 297, 301, 302, 303, - 230, 231, 284, 312, 241, 304, 305, 240, 246, 326, 327, 296, 247, 248, + 230, 231, 284, 312, 240, 304, 305, 241, 246, 326, 327, 296, 247, 248, 249, 232, 343, 322, 328, 271, 340, 289, 254, 331, 253, 325, 252, 330, 313, 280, 324, 341, 274, 242, 291, 251, 290, 239, 314, 315, 320, 294, 268, 266, 267, 293, 292, 260, 261, 262, 263, 264, 229, 244, 245, 306, - 275, 287, 269, 285, 273, 272, 20525, 25235, 20594, 35762, 35762, 35762, - 35762, 15228, 20771, 13645, 26946, 25897, 3823, 3899, 3860, 3803, 3885, - 22283, 3994, 15329, 33678, 13042, 34005, 27397, 3893, 3886, 19796, 22308, - 3995, 15330, 33679, 13043, 34084, 34086, 29411, 3891, 3909, 3847, 34019, - 34020, 10527, 3892, 3910, 3848, 34055, 32018, 19798, 22309, 3996, 33683, - 33680, 13044, 32016, 19791, 22301, 3990, 15302, 33675, 13039, 19784, - 22289, 3993, 15277, 33673, 13041, 19792, 22300, 3991, 15306, 33676, - 13040, 19782, 22306, 3992, 15271, 33672, 19794, 22303, 32036, 22302, - 19786, 22288, 13182, 22287, 27071, 19783, 15276, 22299, 15305, 28850, - 22305, 15269, 33671, 15267, 19793, 15324, 15323, 6506, 24298, 6516, - 24299, 24576, 35762, 35762, 35762, 35762, 35762, 35762, 18137, 18150, - 18133, 18146, 18128, 18149, 18132, 18139, 18151, 18134, 18147, 18129, - 35762, 35762, 35762, 35762, 15274, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 19590, 32726, 27244, 10996, 19597, 32723, 27251, 10993, 19586, 32722, - 27240, 10992, 35762, 35762, 35762, 35762, 19589, 32725, 27243, 10995, - 19599, 32727, 27253, 10997, 15285, 15326, 15299, 15263, 15284, 15325, - 15298, 15262, 19633, 32760, 27304, 11013, 19632, 32759, 27303, 11012, - 19631, 32756, 27302, 11009, 19635, 32762, 27306, 11015, 19634, 32761, - 27305, 11014, 19663, 32737, 27270, 11031, 19671, 32752, 27293, 11046, - 19673, 32748, 27326, 11042, 19623, 32746, 27284, 11040, 19624, 32747, - 27285, 11041, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 19672, 32751, 27294, 11045, 24695, 24669, 29067, 29079, 19509, 32647, - 35762, 35762, 35762, 35762, 35762, 35762, 34121, 34136, 34126, 34131, - 34146, 34141, 34151, 34156, 34123, 34138, 34128, 34133, 34148, 34143, - 34153, 34158, 34122, 34137, 34127, 34132, 34147, 34142, 34152, 34157, - 34119, 34134, 34124, 34129, 34144, 34139, 34149, 34154, 34120, 34135, - 34125, 34130, 34145, 34140, 34150, 34155, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 19639, 32766, 27310, 11020, 19653, 32778, - 27325, 11025, 19598, 32724, 27252, 10994, 15240, 15243, 15242, 15241, - 19607, 27259, 19642, 27295, 19664, 27290, 19668, 27286, 19606, 27258, - 19662, 27269, 33969, 33968, 35762, 35762, 2501, 2498, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 6234, 6235, 6233, 19481, 19479, - 19480, 19482, 19478, 10985, 10987, 10986, 10988, 26673, 34040, 4639, - 26674, 35571, 23320, 13085, 24574, 32019, 13068, 27406, 15990, 28718, - 4960, 26975, 19561, 27210, 14659, 14655, 16358, 13066, 7748, 24254, - 27351, 11059, 20652, 13092, 28959, 13076, 14271, 14269, 13086, 27751, - 28950, 13073, 27938, 26823, 4627, 26265, 27760, 26877, 20898, 23505, - 27942, 26507, 16464, 13115, 22921, 34186, 33943, 14657, 10692, 34160, - 11060, 7692, 32565, 29194, 13614, 27340, 13133, 27733, 32020, 4247, - 21060, 9645, 18038, 28972, 13158, 8392, 2581, 9652, 2599, 26808, 5763, - 2602, 14266, 27764, 29598, 12333, 13609, 26494, 18027, 26266, 11206, - 13174, 30393, 6211, 4097, 9638, 7752, 4638, 26684, 26873, 9653, 27612, - 5699, 19446, 20987, 23500, 2603, 28951, 34207, 28953, 13078, 13090, - 26079, 13195, 24577, 10616, 13095, 13080, 27702, 18036, 13643, 15936, - 13143, 8399, 20486, 27707, 32541, 32633, 11221, 11207, 3450, 27861, - 26090, 13187, 4704, 10522, 13644, 20487, 27106, 27941, 29556, 13512, - 35565, 15827, 27698, 29952, 8378, 16750, 20658, 26491, 15938, 26423, - 26954, 20590, 23503, 22896, 2601, 29768, 20895, 11051, 28882, 26072, - 25798, 28860, 13147, 26085, 3518, 3746, 27729, 14280, 26890, 12422, - 12423, 12425, 12424, 4253, 19780, 13165, 32423, 29761, 29762, 27541, - 11214, 23508, 20984, 22183, 22184, 5999, 9640, 27544, 3635, 13321, 25772, - 13102, 33956, 4703, 22147, 16003, 4641, 32531, 29564, 18031, 10521, - 13070, 101, 6177, 25759, 3514, 26515, 26508, 26518, 26509, 20662, 13108, - 33339, 22728, 12420, 13507, 35757, 4625, 25797, 3738, 27704, 13612, 8374, - 28967, 26995, 13128, 16467, 31792, 26528, 11208, 8154, 2583, 13125, - 32425, 4634, 20661, 20593, 20524, 29193, 2742, 27542, 31680, 4640, 3411, - 27350, 3409, 29197, 26982, 24256, 24357, 24368, 24371, 24360, 24350, - 24365, 33980, 3771, 24351, 33977, 33993, 33996, 33972, 33985, 33990, - 3768, 3784, 3787, 3763, 3776, 3781, 24358, 24369, 24372, 24361, 24356, - 24366, 33981, 3772, 24352, 33999, 34002, 34003, 33998, 34000, 34001, - 3790, 3793, 3794, 3789, 3791, 3792, 24375, 24378, 24379, 24374, 24376, - 24377, 33982, 3773, 24353, 33978, 33994, 33997, 33973, 33983, 33991, - 3769, 3785, 3788, 3764, 3774, 3782, 24359, 24370, 24373, 24362, 24354, - 24367, 33984, 3775, 24355, 33974, 3765, 24363, 33975, 3766, 24364, 33987, - 33988, 33986, 3778, 3779, 3777, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 34202, 34205, 34201, 34203, - 34200, 34199, 34204, 34195, 34198, 34194, 34196, 34193, 34192, 34197, - 35762, 35762, 2743, 25771, 4633, 27936, 32022, 19795, 14267, 26678, - 11057, 99, 29583, 34191, 8395, 35762, 35762, 35762, 35479, 18030, 26274, - 4001, 20660, 26679, 24347, 20990, 13156, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 27335, 5834, 26980, 2586, 10695, 3407, 22900, 1, - 20509, 8373, 5761, 27710, 18040, 15983, 22916, 34163, 26792, 27756, - 18032, 4707, 23516, 32424, 15231, 26687, 27345, 22917, 16009, 20530, - 14653, 13163, 14523, 17228, 13157, 34180, 3521, 7750, 26810, 34182, - 13109, 20526, 8347, 12434, 24348, 16000, 16460, 34166, 35762, 13646, 946, - 20595, 26530, 26825, 26824, 26513, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 23502, 10880, 3989, 3524, 25761, 13144, 30809, - 13186, 31679, 26812, 3519, 16459, 13509, 26495, 35762, 35762, 35762, - 35762, 22389, 27546, 13075, 13079, 13089, 10890, 3743, 4642, 27912, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 14656, 27283, 19622, - 26231, 26232, 16170, 15238, 19667, 27289, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 3925, 3955, 3926, 3970, 3941, 3956, 3927, 3978, - 3948, 3964, 3934, 3971, 3942, 3957, 3928, 3982, 3952, 3967, 3938, 3975, - 3961, 3931, 3979, 3949, 3965, 3935, 3972, 3943, 3958, 3929, 3984, 3954, - 3969, 3940, 3977, 3947, 3963, 3933, 3981, 3951, 3937, 3974, 3945, 3960, - 3930, 3983, 3953, 3968, 3939, 3976, 3946, 3962, 3932, 3980, 3950, 3966, - 3936, 3973, 3944, 3959, 20541, 20542, 20549, 20551, 20546, 20574, 20575, - 20571, 20573, 20569, 20572, 20565, 20568, 20566, 20570, 20567, 20545, - 20548, 20543, 20547, 20544, 20550, 32692, 32693, 32700, 32702, 32697, - 32682, 32683, 32679, 32681, 32677, 32680, 32673, 32676, 32674, 32678, - 32675, 32696, 32699, 32694, 32698, 32695, 32701, 32664, 19447, 32661, - 19449, 19530, 32715, 27195, 20587, 33658, 33659, 33660, 33661, 33662, - 33663, 15949, 15950, 15951, 15952, 15953, 15954, 19448, 19450, 27120, - 27119, 32663, 15948, 32690, 32713, 32666, 32714, 32712, 27160, 27190, - 27147, 27191, 27170, 19489, 27155, 32671, 20537, 16342, 32668, 32669, - 35762, 19488, 5997, 16338, 15278, 32685, 32709, 32662, 19451, 32686, - 32710, 20563, 20556, 4173, 4177, 4169, 4172, 4174, 4179, 4170, 4167, - 4176, 4178, 4180, 4175, 4168, 4171, 4184, 4207, 2504, 16339, 19487, - 27154, 16340, 19577, 27229, 11002, 32719, 19486, 27153, 33766, 27162, - 24301, 24300, 24302, 34043, 19532, 22897, 27188, 24330, 29584, 29585, - 29587, 29588, 29586, 34109, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 27743, 27745, 27742, 27741, 27738, 27737, 27740, 27739, 27746, - 27744, 35762, 35762, 35762, 35762, 35762, 35762, 6553, 6554, 6555, 6556, - 6557, 6558, 6559, 6560, 6561, 6562, 6563, 6564, 6565, 6566, 6567, 6568, - 6569, 6570, 6571, 6572, 6573, 6574, 6575, 6576, 6577, 6578, 6579, 6580, - 6581, 6582, 6583, 6584, 6585, 6586, 6587, 6588, 6589, 6590, 6591, 6592, - 6593, 6594, 6595, 6596, 6597, 6598, 6599, 6600, 6601, 6602, 6603, 6604, - 6605, 6606, 6607, 6608, 6609, 6610, 6611, 6612, 6613, 6614, 6615, 6616, - 6617, 6618, 6619, 6620, 6621, 6622, 6623, 6624, 6625, 6626, 6627, 6628, - 6629, 6630, 6631, 6632, 6633, 6634, 6635, 6636, 6637, 6638, 6639, 6640, - 6641, 6642, 6643, 6644, 6645, 6646, 6647, 6648, 6649, 6650, 6651, 6652, - 6653, 6654, 6655, 6656, 6657, 6658, 6659, 6660, 6661, 6662, 6663, 6664, - 6665, 6666, 6667, 6668, 6669, 6670, 6671, 6672, 6673, 6674, 6675, 6676, - 6677, 6678, 6679, 6680, 6681, 6682, 6683, 6684, 6685, 6686, 6687, 6688, - 6689, 6690, 6691, 6692, 6693, 6694, 6695, 6696, 6697, 6698, 6699, 6700, - 6701, 6702, 6703, 6704, 6705, 6706, 6707, 6708, 6709, 6710, 6711, 6712, - 6713, 6714, 6715, 6716, 6717, 6718, 6719, 6720, 6721, 6722, 6723, 6724, - 6725, 6726, 6727, 6728, 6729, 6730, 6731, 6732, 6733, 6734, 6735, 6736, - 6737, 6738, 6739, 6740, 6741, 6742, 6743, 6744, 6745, 6746, 6747, 6748, - 6749, 6750, 6751, 6752, 6753, 6754, 6755, 6756, 6757, 6758, 6759, 6760, - 6761, 6762, 6763, 6764, 6765, 6766, 6767, 6768, 6769, 6770, 6771, 6772, - 6773, 6774, 6775, 6776, 6777, 6778, 6779, 6780, 6781, 6782, 6783, 6784, - 6785, 6786, 6787, 6788, 6789, 6790, 6791, 6792, 6793, 6794, 6795, 6796, - 6797, 6798, 6799, 6800, 6801, 6802, 6803, 6804, 6805, 6806, 6807, 6808, - 6809, 6810, 6811, 6812, 6813, 6814, 6815, 6816, 6817, 6818, 6819, 6820, - 6821, 6822, 6823, 6824, 6825, 6826, 6827, 6828, 6829, 6830, 6831, 6832, - 6833, 6834, 6835, 6836, 6837, 6838, 6839, 6840, 6841, 6842, 6843, 6844, - 6845, 6846, 6847, 6848, 6849, 6850, 6851, 6852, 6853, 6854, 6855, 6856, - 6857, 6858, 6859, 6860, 6861, 6862, 6863, 6864, 6865, 6866, 6867, 6868, - 6869, 6870, 6871, 6872, 6873, 6874, 6875, 6876, 6877, 6878, 6879, 6880, - 6881, 6882, 6883, 6884, 6885, 6886, 6887, 6888, 6889, 6890, 6891, 6892, - 6893, 6894, 6895, 6896, 6897, 6898, 6899, 6900, 6901, 6902, 6903, 6904, - 6905, 6906, 6907, 6908, 6909, 6910, 6911, 6912, 6913, 6914, 6915, 6916, - 6917, 6918, 6919, 6920, 6921, 6922, 6923, 6924, 6925, 6926, 6927, 6928, - 6929, 6930, 6931, 6932, 6933, 6934, 6935, 6936, 6937, 6938, 6939, 6940, - 6941, 6942, 6943, 6944, 6945, 6946, 6947, 6948, 6949, 6950, 6951, 6952, - 6953, 6954, 6955, 6956, 6957, 6958, 6959, 6960, 6961, 6962, 6963, 6964, - 6965, 6966, 6967, 6968, 6969, 6970, 6971, 6972, 6973, 6974, 6975, 6976, - 6977, 6978, 6979, 6980, 6981, 6982, 6983, 6984, 6985, 6986, 6987, 6988, - 6989, 6990, 6991, 6992, 6993, 6994, 6995, 6996, 6997, 6998, 6999, 7000, - 7001, 7002, 7003, 7004, 7005, 7006, 7007, 7008, 7009, 7010, 7011, 7012, - 7013, 7014, 7015, 7016, 7017, 7018, 7019, 7020, 7021, 7022, 7023, 7024, - 7025, 7026, 7027, 7028, 7029, 7030, 7031, 7032, 7033, 7034, 7035, 7036, - 7037, 7038, 7039, 7040, 7041, 7042, 7043, 7044, 7045, 7046, 7047, 7048, - 7049, 7050, 7051, 7052, 7053, 7054, 7055, 7056, 7057, 7058, 7059, 7060, - 7061, 7062, 7063, 7064, 6537, 6538, 6539, 6540, 6541, 6542, 6543, 6544, - 6545, 6546, 6547, 6548, 6549, 6550, 6551, 6552, 6523, 6524, 6525, 6526, - 6527, 6528, 6529, 6530, 6531, 6532, 6533, 6534, 6535, 6536, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 19444, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 30045, 29974, 30037, 30046, - 29962, 30035, 29956, 29955, 30032, 30040, 29957, 30036, 29960, 29977, - 30048, 30044, 29969, 29971, 29968, 29967, 29964, 29963, 29966, 29965, - 29972, 29970, 29961, 30043, 30030, 29973, 29976, 30038, 29959, 29978, - 29979, 29980, 29981, 29982, 29983, 29984, 29985, 29986, 29987, 29988, - 29989, 29990, 29991, 29992, 29993, 29994, 29995, 29996, 29997, 29998, - 29999, 30000, 30001, 30002, 30003, 30033, 30042, 30041, 29958, 30034, - 29975, 30004, 30005, 30006, 30007, 30008, 30009, 30010, 30011, 30012, - 30013, 30014, 30015, 30016, 30017, 30018, 30019, 30020, 30021, 30022, - 30023, 30024, 30025, 30026, 30027, 30028, 30029, 30031, 30049, 30039, - 30047, 5694, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 33161, 33172, 33183, 33195, 33206, 33217, 33228, 33239, 33250, 33258, - 33259, 33260, 33261, 33263, 33264, 33265, 33266, 33267, 33268, 33269, - 33270, 33271, 33272, 33274, 33275, 33276, 33277, 33278, 33279, 33280, - 33281, 33282, 33283, 33285, 33286, 33287, 33288, 33289, 33290, 33291, - 33292, 33293, 33294, 33296, 33297, 33298, 33299, 33300, 33301, 33302, - 33303, 33304, 33305, 33307, 33308, 33309, 33310, 33311, 33312, 33313, - 33314, 33315, 33316, 33318, 33319, 33320, 33321, 33322, 33323, 33324, - 33325, 33326, 33327, 33329, 33330, 33331, 33332, 33333, 33334, 33335, - 33336, 33337, 33338, 33085, 33086, 33087, 33088, 33089, 33090, 33091, - 33092, 33093, 33094, 33096, 33097, 33098, 33099, 33100, 33101, 33102, - 33103, 33104, 33105, 33107, 33108, 33109, 33110, 33111, 33112, 33113, - 33114, 33115, 33116, 33118, 33119, 33120, 33121, 33122, 33123, 33124, - 33125, 33126, 33127, 33129, 33130, 33131, 33132, 33133, 33134, 33135, - 33136, 33137, 33138, 33140, 33141, 33142, 33143, 33144, 33145, 33146, - 33147, 33148, 33149, 33151, 33152, 33153, 33154, 33155, 33156, 33157, - 33158, 33159, 33160, 33162, 33163, 33164, 33165, 33166, 33167, 33168, - 33169, 33170, 33171, 33173, 33174, 33175, 33176, 33177, 33178, 33179, - 33180, 33181, 33182, 33184, 33185, 33186, 33187, 33188, 33189, 33190, - 33191, 33192, 33193, 33196, 33197, 33198, 33199, 33200, 33201, 33202, - 33203, 33204, 33205, 33207, 33208, 33209, 33210, 33211, 33212, 33213, - 33214, 33215, 33216, 33218, 33219, 33220, 33221, 33222, 33223, 33224, - 33225, 33226, 33227, 33229, 33230, 33231, 33232, 33233, 33234, 33235, - 33236, 33237, 33238, 33240, 33241, 33242, 33243, 33244, 33245, 33246, - 33247, 33248, 33249, 33251, 33252, 33253, 33254, 33255, 33256, 33257, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 25143, 25142, 29575, 29166, - 29576, 29601, 12457, 13030, 12455, 12468, 12461, 12460, 3, 2, 347, 3522, - 2596, 4957, 5989, 15959, 15985, 29953, 19875, 24468, 12458, 20659, 25216, - 12466, 19877, 33665, 33770, 13171, 13320, 5762, 8375, 27914, 20491, - 28965, 27915, 20490, 27945, 10237, 11210, 10519, 10238, 10518, 10239, - 10517, 10240, 10515, 10241, 24336, 24258, 29858, 29857, 12456, 13029, - 5692, 4965, 12454, 12467, 12421, 29632, 29602, 12502, 12501, 16242, - 13120, 13318, 16243, 14274, 14522, 16244, 26992, 27539, 16245, 32630, - 32782, 29168, 10253, 10250, 26093, 26092, 15830, 15984, 4626, 4956, - 24641, 24260, 16169, 16168, 24570, 24575, 29572, 29559, 12453, 12505, - 5991, 15961, 15987, 5990, 15960, 15986, 19878, 33666, 33771, 26416, - 26415, 26793, 26417, 26418, 26773, 27072, 27079, 27107, 28731, 28732, - 29557, 28730, 28733, 29558, 10516, 10242, 26881, 26882, 26929, 26880, - 26883, 26930, 27754, 29600, 5693, 10222, 22730, 24084, 29571, 29574, - 29169, 12452, 12450, 13063, 29573, 29167, 28725, 29950, 28724, 27728, - 8169, 10221, 29595, 29560, 25790, 26010, 26879, 26931, 1048, 1049, 24259, - 27944, 18311, 18929, 10220, 2297, 350, 29936, 16751, 18043, 18044, 18086, - 18052, 32125, 14921, 14922, 14899, 14920, 13314, 13315, 13316, 24082, - 13317, 29657, 35760, 35759, 35761, 20655, 27343, 20653, 27341, 26489, - 20656, 27344, 25215, 24083, 34189, 20654, 27342, 13319, 26490, 33962, - 22891, 22892, 19628, 27299, 34696, 23890, 33340, 33451, 33519, 33530, - 33541, 33552, 33563, 33574, 33585, 33341, 33352, 33363, 33374, 33385, - 33396, 33407, 26848, 4955, 4248, 35758, 9338, 9337, 22191, 22189, 22249, - 22247, 15743, 4792, 33418, 33429, 33440, 33452, 33463, 33474, 33485, - 33496, 33507, 33515, 33516, 33517, 33518, 33520, 33521, 33522, 33523, - 33524, 33525, 33526, 33527, 33528, 33529, 33531, 33532, 33533, 33534, - 33535, 33536, 33537, 33538, 33539, 33540, 33542, 33543, 33544, 33545, - 33546, 33547, 33548, 33549, 33550, 33551, 33553, 33554, 33555, 33556, - 33557, 33558, 33559, 33560, 33561, 33562, 33564, 33565, 33566, 33567, - 33568, 33569, 33570, 33571, 33572, 33573, 33575, 33576, 33577, 33578, - 33579, 33580, 33581, 33582, 33583, 33584, 33586, 33587, 33588, 33589, - 33590, 33591, 33592, 33593, 33594, 33595, 33342, 33343, 33344, 33345, - 33346, 33347, 33348, 33349, 33350, 33351, 33353, 33354, 33355, 33356, - 33357, 33358, 33359, 33360, 33361, 33362, 33364, 33365, 33366, 33367, - 33368, 33369, 33370, 33371, 33372, 33373, 33375, 33376, 33377, 33378, - 33379, 33380, 33381, 33382, 33383, 33384, 33386, 33387, 33388, 33389, - 33390, 33391, 33392, 33393, 33394, 33395, 33397, 33398, 33399, 33400, - 33401, 33402, 33403, 33404, 33405, 33406, 33408, 33409, 33410, 33411, - 33412, 33413, 33414, 33415, 33416, 33417, 33419, 33420, 33421, 33422, - 33423, 33424, 33425, 33426, 33427, 33428, 33430, 33431, 33432, 33433, - 33434, 33435, 33436, 33437, 33438, 33439, 33441, 33442, 33443, 33444, - 33445, 33446, 33447, 33448, 33449, 33450, 33453, 33454, 33455, 33456, - 33457, 33458, 33459, 33460, 33461, 33462, 33464, 33465, 33466, 33467, - 33468, 33469, 33470, 33471, 33472, 33473, 33475, 33476, 33477, 33478, - 33479, 33480, 33481, 33482, 33483, 33484, 33486, 33487, 33488, 33489, - 33490, 33491, 33492, 33493, 33494, 33495, 33497, 33498, 33499, 33500, - 33501, 33502, 33503, 33504, 33505, 33506, 33508, 33509, 33510, 33511, - 33512, 33513, 33514, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 17147, 17148, 17155, 17157, 17154, 17153, 17150, 17149, - 17152, 17151, 17158, 17156, 18176, 18735, 18335, 18960, 18570, 19331, - 18270, 18870, 18271, 18871, 18272, 18872, 18444, 19110, 18445, 19111, - 18446, 19112, 18509, 19207, 18254, 18853, 18250, 18849, 18955, 19078, - 18186, 18745, 18187, 18746, 18275, 18876, 18276, 18877, 18260, 18859, - 18261, 18860, 18954, 18956, 18340, 18962, 18341, 18963, 18358, 18990, - 18388, 19030, 18394, 19051, 18484, 19169, 18574, 19335, 18575, 19336, - 18571, 19332, 18572, 19333, 18754, 19127, 19128, 19286, 19287, 19216, - 19217, 18944, 18945, 2267, 2270, 2268, 2272, 2274, 2271, 2273, 2269, - 2275, 10448, 10443, 10442, 10449, 10444, 10445, 10447, 10446, 3594, 3593, - 3595, 14488, 14487, 14486, 14485, 14490, 14489, 25862, 25861, 3539, - 30399, 30407, 30416, 30408, 30410, 30405, 30409, 30404, 30420, 30419, - 30422, 30414, 30401, 30421, 30403, 30402, 30415, 30406, 30418, 30412, - 30413, 30411, 30417, 30400, 30538, 30545, 30546, 30541, 30542, 30547, - 30548, 30539, 30543, 30544, 30540, 30604, 30611, 30612, 30607, 30608, - 30613, 30614, 30605, 30609, 30610, 30606, 30715, 30722, 30723, 30718, - 30719, 30724, 30725, 30716, 30720, 30721, 30717, 30615, 30622, 30623, - 30618, 30619, 30624, 30625, 30616, 30620, 30621, 30617, 30693, 30700, - 30701, 30696, 30697, 30702, 30703, 30694, 30698, 30699, 30695, 30593, - 30600, 30601, 30596, 30597, 30602, 30603, 30594, 30598, 30599, 30595, - 30704, 30711, 30712, 30707, 30708, 30713, 30714, 30705, 30709, 30710, - 30706, 30626, 30633, 30634, 30629, 30630, 30635, 30636, 30627, 30631, - 30632, 30628, 30759, 30766, 30767, 30762, 30763, 30768, 30769, 30760, - 30764, 30765, 30761, 30748, 30755, 30756, 30751, 30752, 30757, 30758, - 30749, 30753, 30754, 30750, 30781, 30788, 30789, 30784, 30785, 30790, - 30791, 30782, 30786, 30787, 30783, 30648, 30655, 30656, 30651, 30652, - 30657, 30658, 30649, 30653, 30654, 30650, 30571, 30578, 30579, 30574, - 30575, 30580, 30581, 30572, 30576, 30577, 30573, 30770, 30777, 30778, - 30773, 30774, 30779, 30780, 30771, 30775, 30776, 30772, 30549, 30556, - 30557, 30552, 30553, 30558, 30559, 30550, 30554, 30555, 30551, 30560, - 30567, 30568, 30563, 30564, 30569, 30570, 30561, 30565, 30566, 30562, - 30637, 30644, 30645, 30640, 30641, 30646, 30647, 30638, 30642, 30643, - 30639, 30582, 30589, 30590, 30585, 30586, 30591, 30592, 30583, 30587, - 30588, 30584, 30737, 30744, 30745, 30740, 30741, 30746, 30747, 30738, - 30742, 30743, 30739, 30659, 30667, 30668, 30662, 30663, 30669, 30670, - 30660, 30664, 30665, 30661, 30671, 30678, 30679, 30674, 30675, 30680, - 30681, 30672, 30676, 30677, 30673, 30682, 30689, 30690, 30685, 30686, - 30691, 30692, 30683, 30687, 30688, 30684, 30726, 30733, 30734, 30729, - 30730, 30735, 30736, 30727, 30731, 30732, 30728, 30526, 30527, 30534, - 30535, 30530, 30531, 30536, 30537, 30528, 30532, 30533, 30529, 30666, - 28756, 28754, 28755, 13411, 17710, 17708, 17711, 17709, 17700, 17706, - 17704, 17707, 17705, 17701, 17721, 17717, 17722, 17718, 17702, 17719, - 17715, 17720, 17716, 17703, 17732, 17712, 17714, 17713, 17728, 17731, - 17729, 17724, 17730, 17725, 17726, 17727, 17733, 17723, 17872, 17749, - 17750, 17751, 17748, 17867, 17859, 15860, 15862, 15864, 15861, 15863, - 16853, 16855, 16857, 16854, 16856, 16846, 16845, 16844, 16847, 23097, - 23102, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, 35762, - 35762, 35762, + 275, 287, 269, 285, 273, 272, 25151, 29947, 25272, 40951, 40951, 40951, + 40951, 19684, 25449, 18041, 31741, 30655, 3849, 3927, 3889, 3823, 3913, + 26961, 4251, 19786, 38863, 17364, 39190, 32226, 3921, 3914, 24416, 26986, + 4252, 19783, 38864, 17365, 39272, 39274, 34335, 3919, 3937, 3871, 39205, + 39206, 10835, 3920, 3938, 3872, 39243, 36993, 24418, 26987, 4253, 38868, + 38865, 17366, 36991, 24411, 26979, 4247, 19757, 38860, 17361, 24404, + 26967, 4250, 19732, 38858, 17363, 24412, 26978, 4248, 19761, 38861, + 17362, 24402, 26984, 4249, 19726, 38857, 24414, 26981, 37011, 26980, + 24406, 26966, 17508, 26965, 31868, 24403, 19731, 26977, 19760, 33758, + 26983, 19724, 38856, 19722, 24413, 19779, 19778, 6807, 29003, 6817, + 29004, 29285, 40951, 40951, 40951, 40951, 40951, 40951, 22664, 22732, + 22660, 22728, 22655, 22731, 22659, 22666, 22733, 22661, 22729, 22656, + 40951, 40951, 40951, 40951, 19729, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 24209, 37904, 32072, 11315, 24216, 37901, 32079, 11312, 24203, 37900, + 32069, 11311, 40951, 40951, 40951, 40951, 24208, 37903, 32071, 11314, + 24218, 37905, 32081, 11316, 19740, 19781, 19754, 19718, 19739, 19780, + 19753, 19717, 24253, 37938, 32132, 11335, 24252, 37937, 32131, 11334, + 24251, 37934, 32130, 11331, 24255, 37940, 32134, 11337, 24254, 37939, + 32133, 11336, 24283, 37915, 32098, 11351, 24291, 37930, 32121, 11366, + 24293, 37926, 32154, 11362, 24243, 37924, 32112, 11360, 24244, 37925, + 32113, 11361, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 24292, 37929, 32122, 11365, 29406, 29379, 33977, 33990, 24106, 37771, + 40951, 40951, 40951, 40951, 40951, 40951, 39310, 39325, 39315, 39320, + 39335, 39330, 39340, 39345, 39312, 39327, 39317, 39322, 39337, 39332, + 39342, 39347, 39311, 39326, 39316, 39321, 39336, 39331, 39341, 39346, + 39308, 39323, 39313, 39318, 39333, 39328, 39338, 39343, 39309, 39324, + 39314, 39319, 39334, 39329, 39339, 39344, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 24259, 37944, 32138, 11342, 24273, 37954, + 32153, 11347, 24217, 37902, 32080, 11313, 19695, 19698, 19697, 19696, + 24227, 32087, 24262, 32123, 24284, 32118, 24288, 32114, 24226, 32086, + 24282, 32097, 39154, 39153, 40951, 40951, 2517, 2514, 32067, 11326, + 29034, 29035, 29042, 29036, 29398, 29370, 33969, 33982, 40951, 40951, + 40951, 40951, 24223, 32058, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 6535, 6536, 6534, 24071, 24069, + 24070, 24072, 24068, 11304, 11306, 11305, 11307, 31467, 39226, 4940, + 31468, 40760, 28003, 17406, 29283, 36994, 17389, 32235, 20454, 33626, + 5261, 31771, 24180, 32021, 19113, 19109, 20824, 17387, 8051, 28959, + 32179, 11379, 25330, 17414, 33868, 17397, 18666, 18665, 17408, 32659, + 33859, 17394, 32845, 31618, 4928, 31059, 32667, 31672, 25576, 28189, + 32850, 31301, 20930, 17437, 27604, 39375, 39129, 19111, 11001, 39349, + 11380, 7995, 37689, 34110, 18010, 32168, 17457, 32562, 36995, 4548, + 25738, 9949, 22565, 33881, 17485, 8695, 2597, 9956, 2615, 31603, 6064, + 2618, 18662, 32671, 34526, 16648, 18005, 31288, 22554, 31060, 11526, + 17500, 35368, 6512, 4370, 9942, 8055, 4939, 31478, 31668, 9957, 32441, + 6000, 24035, 25665, 28183, 2619, 33860, 39396, 33862, 17399, 17412, + 30873, 17521, 29286, 10925, 17417, 17401, 32531, 22563, 18040, 20392, + 17466, 8702, 25106, 32536, 37665, 37757, 11541, 11527, 3470, 32768, + 30884, 17513, 5005, 10830, 18039, 25107, 31903, 32849, 34480, 17908, + 40754, 20283, 32527, 34927, 8681, 21216, 25336, 31285, 20394, 31217, + 31749, 25268, 28187, 27576, 2617, 34740, 25573, 11371, 33790, 30830, + 30555, 33768, 17471, 30879, 3538, 3766, 32558, 18676, 31685, 16739, + 16740, 16742, 16741, 4554, 24400, 17491, 37450, 34733, 34734, 32370, + 11534, 28192, 25662, 26861, 26862, 6300, 9944, 32373, 3655, 17717, 30529, + 17424, 38977, 5004, 26825, 20467, 4942, 37574, 34502, 22558, 10829, + 17391, 101, 6478, 30516, 3534, 31309, 31303, 31312, 31302, 25340, 17430, + 38517, 27409, 16737, 17903, 40946, 4926, 30554, 3758, 32533, 18008, 8677, + 33876, 31792, 17452, 20933, 36767, 31322, 11528, 8457, 2599, 17449, + 37452, 4935, 25339, 25271, 25150, 34109, 2761, 32371, 36655, 4941, 3431, + 32178, 3429, 34113, 31778, 28961, 29066, 29077, 29080, 29069, 29059, + 29074, 39165, 3791, 29060, 39162, 39178, 39181, 39157, 39170, 39175, + 3788, 3804, 3807, 3783, 3796, 3801, 29067, 29078, 29081, 29070, 29065, + 29075, 39166, 3792, 29061, 39184, 39187, 39188, 39183, 39185, 39186, + 3810, 3813, 3814, 3809, 3811, 3812, 29084, 29087, 29088, 29083, 29085, + 29086, 39167, 3793, 29062, 39163, 39179, 39182, 39158, 39168, 39176, + 3789, 3805, 3808, 3784, 3794, 3802, 29068, 29079, 29082, 29071, 29063, + 29076, 39169, 3795, 29064, 39159, 3785, 29072, 39160, 3786, 29073, 39172, + 39173, 39171, 3798, 3799, 3797, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 39391, 39394, 39390, 39392, + 39389, 39388, 39393, 39384, 39387, 39383, 39385, 39382, 39381, 39386, + 40951, 40951, 2762, 30528, 4934, 32843, 36997, 24415, 18663, 31472, + 11377, 99, 34507, 39380, 8698, 40951, 40951, 40951, 40668, 22557, 31068, + 4258, 25338, 31473, 29056, 25668, 17480, 19656, 40951, 40951, 40951, + 40951, 40951, 32846, 32163, 6135, 31776, 2602, 11004, 3427, 27583, 1, + 25135, 8676, 6062, 32539, 22567, 20447, 27599, 39352, 31586, 32664, + 22559, 5008, 28200, 37451, 19686, 31481, 32173, 27600, 20473, 25156, + 19107, 17489, 18977, 21696, 17481, 39369, 3541, 8053, 31605, 39371, + 17432, 25152, 8650, 16751, 29057, 20464, 20926, 39355, 24033, 18042, 956, + 25273, 31324, 31620, 31619, 31307, 17445, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 28186, 11189, 4246, 3544, 30518, 17467, 35784, + 17512, 36654, 31607, 3539, 20925, 17905, 31289, 32216, 40951, 40951, + 34108, 27067, 32375, 17396, 17400, 17411, 11199, 3763, 4943, 32819, + 17407, 40951, 40951, 40951, 40951, 40951, 40951, 19110, 32111, 24242, + 31025, 31026, 20634, 19693, 24287, 32117, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 4182, 4212, 4183, 4227, 4198, 4216, 4184, 4235, + 4205, 4213, 4191, 4228, 4199, 4217, 4185, 4239, 4209, 4224, 4195, 4232, + 4221, 4188, 4236, 4206, 4214, 4192, 4229, 4200, 4218, 4186, 4241, 4211, + 4226, 4197, 4234, 4204, 4223, 4190, 4238, 4208, 4194, 4231, 4202, 4220, + 4187, 4240, 4210, 4225, 4196, 4233, 4203, 4222, 4189, 4237, 4207, 4215, + 4193, 4230, 4201, 4219, 25173, 25174, 25181, 25183, 25178, 25239, 25240, + 25236, 25238, 25234, 25237, 25230, 25233, 25231, 25235, 25232, 25177, + 25180, 25175, 25179, 25176, 25182, 37854, 37855, 37862, 37864, 37859, + 37824, 37825, 37821, 37823, 37819, 37822, 37815, 37818, 37816, 37820, + 37817, 37858, 37861, 37856, 37860, 37857, 37863, 37797, 24036, 37794, + 24041, 24127, 37893, 32004, 25265, 38841, 38842, 38843, 38844, 38845, + 38846, 20409, 20410, 20411, 20412, 20413, 20414, 24037, 24042, 31917, + 31916, 37796, 20408, 37852, 37889, 37804, 37892, 37888, 31966, 32000, + 31944, 31999, 31976, 24085, 31959, 37811, 25166, 20809, 37806, 37808, + 40951, 24084, 6298, 20805, 19733, 37829, 37884, 37795, 24038, 37830, + 37885, 25226, 25203, 4454, 4458, 4446, 4452, 4455, 4460, 4447, 4449, + 4457, 4459, 4461, 4456, 4450, 4451, 4477, 4487, 2520, 20806, 24079, + 31954, 20807, 24196, 32055, 11321, 37897, 24076, 31951, 38951, 31968, + 29006, 29005, 29007, 39229, 24130, 27578, 31995, 29037, 34508, 34509, + 34511, 34512, 34510, 39297, 39202, 31767, 3903, 24137, 24092, 4453, 4474, + 4469, 4448, 4464, 4463, 4471, 4462, 4467, 4473, 4444, 4468, 4465, 4475, + 4445, 4470, 37476, 31962, 4364, 24151, 37803, 25250, 27579, 27580, 37475, + 31961, 4363, 24150, 37485, 4350, 4357, 37477, 32572, 32574, 32571, 32570, + 32567, 32566, 32569, 32568, 32575, 32573, 40951, 40951, 40951, 40951, + 40951, 40951, 6854, 6855, 6856, 6857, 6858, 6859, 6860, 6861, 6862, 6863, + 6864, 6865, 6866, 6867, 6868, 6869, 6870, 6871, 6872, 6873, 6874, 6875, + 6876, 6877, 6878, 6879, 6880, 6881, 6882, 6883, 6884, 6885, 6886, 6887, + 6888, 6889, 6890, 6891, 6892, 6893, 6894, 6895, 6896, 6897, 6898, 6899, + 6900, 6901, 6902, 6903, 6904, 6905, 6906, 6907, 6908, 6909, 6910, 6911, + 6912, 6913, 6914, 6915, 6916, 6917, 6918, 6919, 6920, 6921, 6922, 6923, + 6924, 6925, 6926, 6927, 6928, 6929, 6930, 6931, 6932, 6933, 6934, 6935, + 6936, 6937, 6938, 6939, 6940, 6941, 6942, 6943, 6944, 6945, 6946, 6947, + 6948, 6949, 6950, 6951, 6952, 6953, 6954, 6955, 6956, 6957, 6958, 6959, + 6960, 6961, 6962, 6963, 6964, 6965, 6966, 6967, 6968, 6969, 6970, 6971, + 6972, 6973, 6974, 6975, 6976, 6977, 6978, 6979, 6980, 6981, 6982, 6983, + 6984, 6985, 6986, 6987, 6988, 6989, 6990, 6991, 6992, 6993, 6994, 6995, + 6996, 6997, 6998, 6999, 7000, 7001, 7002, 7003, 7004, 7005, 7006, 7007, + 7008, 7009, 7010, 7011, 7012, 7013, 7014, 7015, 7016, 7017, 7018, 7019, + 7020, 7021, 7022, 7023, 7024, 7025, 7026, 7027, 7028, 7029, 7030, 7031, + 7032, 7033, 7034, 7035, 7036, 7037, 7038, 7039, 7040, 7041, 7042, 7043, + 7044, 7045, 7046, 7047, 7048, 7049, 7050, 7051, 7052, 7053, 7054, 7055, + 7056, 7057, 7058, 7059, 7060, 7061, 7062, 7063, 7064, 7065, 7066, 7067, + 7068, 7069, 7070, 7071, 7072, 7073, 7074, 7075, 7076, 7077, 7078, 7079, + 7080, 7081, 7082, 7083, 7084, 7085, 7086, 7087, 7088, 7089, 7090, 7091, + 7092, 7093, 7094, 7095, 7096, 7097, 7098, 7099, 7100, 7101, 7102, 7103, + 7104, 7105, 7106, 7107, 7108, 7109, 7110, 7111, 7112, 7113, 7114, 7115, + 7116, 7117, 7118, 7119, 7120, 7121, 7122, 7123, 7124, 7125, 7126, 7127, + 7128, 7129, 7130, 7131, 7132, 7133, 7134, 7135, 7136, 7137, 7138, 7139, + 7140, 7141, 7142, 7143, 7144, 7145, 7146, 7147, 7148, 7149, 7150, 7151, + 7152, 7153, 7154, 7155, 7156, 7157, 7158, 7159, 7160, 7161, 7162, 7163, + 7164, 7165, 7166, 7167, 7168, 7169, 7170, 7171, 7172, 7173, 7174, 7175, + 7176, 7177, 7178, 7179, 7180, 7181, 7182, 7183, 7184, 7185, 7186, 7187, + 7188, 7189, 7190, 7191, 7192, 7193, 7194, 7195, 7196, 7197, 7198, 7199, + 7200, 7201, 7202, 7203, 7204, 7205, 7206, 7207, 7208, 7209, 7210, 7211, + 7212, 7213, 7214, 7215, 7216, 7217, 7218, 7219, 7220, 7221, 7222, 7223, + 7224, 7225, 7226, 7227, 7228, 7229, 7230, 7231, 7232, 7233, 7234, 7235, + 7236, 7237, 7238, 7239, 7240, 7241, 7242, 7243, 7244, 7245, 7246, 7247, + 7248, 7249, 7250, 7251, 7252, 7253, 7254, 7255, 7256, 7257, 7258, 7259, + 7260, 7261, 7262, 7263, 7264, 7265, 7266, 7267, 7268, 7269, 7270, 7271, + 7272, 7273, 7274, 7275, 7276, 7277, 7278, 7279, 7280, 7281, 7282, 7283, + 7284, 7285, 7286, 7287, 7288, 7289, 7290, 7291, 7292, 7293, 7294, 7295, + 7296, 7297, 7298, 7299, 7300, 7301, 7302, 7303, 7304, 7305, 7306, 7307, + 7308, 7309, 7310, 7311, 7312, 7313, 7314, 7315, 7316, 7317, 7318, 7319, + 7320, 7321, 7322, 7323, 7324, 7325, 7326, 7327, 7328, 7329, 7330, 7331, + 7332, 7333, 7334, 7335, 7336, 7337, 7338, 7339, 7340, 7341, 7342, 7343, + 7344, 7345, 7346, 7347, 7348, 7349, 7350, 7351, 7352, 7353, 7354, 7355, + 7356, 7357, 7358, 7359, 7360, 7361, 7362, 7363, 7364, 7365, 6838, 6839, + 6840, 6841, 6842, 6843, 6844, 6845, 6846, 6847, 6848, 6849, 6850, 6851, + 6852, 6853, 6824, 6825, 6826, 6827, 6828, 6829, 6830, 6831, 6832, 6833, + 6834, 6835, 6836, 6837, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 24032, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 35020, 34949, 35012, 35021, 34937, 35010, 34931, 34930, 35007, + 35015, 34932, 35011, 34935, 34952, 35023, 35019, 34944, 34946, 34943, + 34942, 34939, 34938, 34941, 34940, 34947, 34945, 34936, 35018, 35005, + 34948, 34951, 35013, 34934, 34953, 34954, 34955, 34956, 34957, 34958, + 34959, 34960, 34961, 34962, 34963, 34964, 34965, 34966, 34967, 34968, + 34969, 34970, 34971, 34972, 34973, 34974, 34975, 34976, 34977, 34978, + 35008, 35017, 35016, 34933, 35009, 34950, 34979, 34980, 34981, 34982, + 34983, 34984, 34985, 34986, 34987, 34988, 34989, 34990, 34991, 34992, + 34993, 34994, 34995, 34996, 34997, 34998, 34999, 35000, 35001, 35002, + 35003, 35004, 35006, 35024, 35014, 35022, 5995, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 38339, 38350, 38361, 38373, 38384, + 38395, 38406, 38417, 38428, 38436, 38437, 38438, 38439, 38441, 38442, + 38443, 38444, 38445, 38446, 38447, 38448, 38449, 38450, 38452, 38453, + 38454, 38455, 38456, 38457, 38458, 38459, 38460, 38461, 38463, 38464, + 38465, 38466, 38467, 38468, 38469, 38470, 38471, 38472, 38474, 38475, + 38476, 38477, 38478, 38479, 38480, 38481, 38482, 38483, 38485, 38486, + 38487, 38488, 38489, 38490, 38491, 38492, 38493, 38494, 38496, 38497, + 38498, 38499, 38500, 38501, 38502, 38503, 38504, 38505, 38507, 38508, + 38509, 38510, 38511, 38512, 38513, 38514, 38515, 38516, 38263, 38264, + 38265, 38266, 38267, 38268, 38269, 38270, 38271, 38272, 38274, 38275, + 38276, 38277, 38278, 38279, 38280, 38281, 38282, 38283, 38285, 38286, + 38287, 38288, 38289, 38290, 38291, 38292, 38293, 38294, 38296, 38297, + 38298, 38299, 38300, 38301, 38302, 38303, 38304, 38305, 38307, 38308, + 38309, 38310, 38311, 38312, 38313, 38314, 38315, 38316, 38318, 38319, + 38320, 38321, 38322, 38323, 38324, 38325, 38326, 38327, 38329, 38330, + 38331, 38332, 38333, 38334, 38335, 38336, 38337, 38338, 38340, 38341, + 38342, 38343, 38344, 38345, 38346, 38347, 38348, 38349, 38351, 38352, + 38353, 38354, 38355, 38356, 38357, 38358, 38359, 38360, 38362, 38363, + 38364, 38365, 38366, 38367, 38368, 38369, 38370, 38371, 38374, 38375, + 38376, 38377, 38378, 38379, 38380, 38381, 38382, 38383, 38385, 38386, + 38387, 38388, 38389, 38390, 38391, 38392, 38393, 38394, 38396, 38397, + 38398, 38399, 38400, 38401, 38402, 38403, 38404, 38405, 38407, 38408, + 38409, 38410, 38411, 38412, 38413, 38414, 38415, 38416, 38418, 38419, + 38420, 38421, 38422, 38423, 38424, 38425, 38426, 38427, 38429, 38430, + 38431, 38432, 38433, 38434, 38435, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 29854, 29853, 34498, 34077, 34499, 34529, 16774, 17347, 16772, + 16785, 16778, 16777, 3, 2, 347, 3542, 2612, 5258, 6290, 20419, 20449, + 34928, 24495, 29177, 16775, 25337, 29928, 16783, 24497, 38850, 38955, + 17497, 17647, 6063, 8678, 32821, 25111, 33874, 32822, 25110, 32853, + 10543, 11530, 10827, 10544, 10826, 10545, 10825, 10546, 10823, 10547, + 29045, 28963, 34833, 34832, 16773, 17346, 5993, 5266, 16771, 16784, + 16738, 34560, 34530, 16819, 16818, 20706, 17442, 17645, 20707, 18670, + 18976, 20708, 31789, 32368, 20709, 37754, 37960, 34079, 10559, 10556, + 30887, 30886, 20286, 20448, 4927, 5257, 29350, 28965, 20633, 20632, + 29279, 29284, 34495, 34483, 16770, 16822, 6292, 20421, 20451, 6291, + 20420, 20450, 24498, 38851, 38956, 31210, 31209, 31587, 31211, 31212, + 31567, 31869, 31876, 31904, 33639, 33640, 34481, 33638, 33641, 34482, + 10824, 10548, 31676, 31677, 31724, 31675, 31678, 31725, 32662, 34528, + 5994, 10528, 27410, 28789, 34494, 34497, 34080, 16769, 16767, 17384, + 34496, 34078, 33633, 34925, 33632, 32557, 8472, 10527, 34519, 34484, + 30547, 30768, 31674, 31726, 1059, 1060, 28964, 32852, 22893, 23515, + 10526, 2313, 361, 34911, 21230, 22570, 22571, 22613, 22579, 37100, 19375, + 19376, 19353, 19374, 17641, 17642, 17643, 28787, 17644, 34585, 40949, + 40948, 40950, 25333, 32171, 25331, 32169, 31283, 25334, 32172, 29927, + 28788, 39378, 25332, 32170, 17646, 31284, 39147, 27571, 27572, 24248, + 32127, 39885, 28575, 38518, 38629, 38697, 38708, 38719, 38730, 38741, + 38752, 38763, 38519, 38530, 38541, 38552, 38563, 38574, 38585, 31643, + 5256, 4549, 40947, 9642, 9641, 9337, 2827, 26869, 26867, 26927, 26925, + 20199, 5093, 27245, 27248, 38596, 38607, 38618, 38630, 38641, 38652, + 38663, 38674, 38685, 38693, 38694, 38695, 38696, 38698, 38699, 38700, + 38701, 38702, 38703, 38704, 38705, 38706, 38707, 38709, 38710, 38711, + 38712, 38713, 38714, 38715, 38716, 38717, 38718, 38720, 38721, 38722, + 38723, 38724, 38725, 38726, 38727, 38728, 38729, 38731, 38732, 38733, + 38734, 38735, 38736, 38737, 38738, 38739, 38740, 38742, 38743, 38744, + 38745, 38746, 38747, 38748, 38749, 38750, 38751, 38753, 38754, 38755, + 38756, 38757, 38758, 38759, 38760, 38761, 38762, 38764, 38765, 38766, + 38767, 38768, 38769, 38770, 38771, 38772, 38773, 38520, 38521, 38522, + 38523, 38524, 38525, 38526, 38527, 38528, 38529, 38531, 38532, 38533, + 38534, 38535, 38536, 38537, 38538, 38539, 38540, 38542, 38543, 38544, + 38545, 38546, 38547, 38548, 38549, 38550, 38551, 38553, 38554, 38555, + 38556, 38557, 38558, 38559, 38560, 38561, 38562, 38564, 38565, 38566, + 38567, 38568, 38569, 38570, 38571, 38572, 38573, 38575, 38576, 38577, + 38578, 38579, 38580, 38581, 38582, 38583, 38584, 38586, 38587, 38588, + 38589, 38590, 38591, 38592, 38593, 38594, 38595, 38597, 38598, 38599, + 38600, 38601, 38602, 38603, 38604, 38605, 38606, 38608, 38609, 38610, + 38611, 38612, 38613, 38614, 38615, 38616, 38617, 38619, 38620, 38621, + 38622, 38623, 38624, 38625, 38626, 38627, 38628, 38631, 38632, 38633, + 38634, 38635, 38636, 38637, 38638, 38639, 38640, 38642, 38643, 38644, + 38645, 38646, 38647, 38648, 38649, 38650, 38651, 38653, 38654, 38655, + 38656, 38657, 38658, 38659, 38660, 38661, 38662, 38664, 38665, 38666, + 38667, 38668, 38669, 38670, 38671, 38672, 38673, 38675, 38676, 38677, + 38678, 38679, 38680, 38681, 38682, 38683, 38684, 38686, 38687, 38688, + 38689, 38690, 38691, 38692, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 21614, + 21615, 21622, 21624, 21621, 21620, 21617, 21616, 21619, 21618, 21625, + 21623, 22758, 23321, 22917, 23546, 23156, 23919, 22852, 23456, 22853, + 23457, 22854, 23458, 23028, 23697, 23029, 23698, 23030, 23699, 23095, + 23795, 22836, 23439, 22832, 23435, 23541, 23665, 22768, 23331, 22769, + 23332, 22857, 23462, 22858, 23463, 22842, 23445, 22843, 23446, 23540, + 23542, 22922, 23548, 22923, 23549, 22940, 23576, 22970, 23616, 22978, + 23638, 23068, 23756, 23160, 23923, 23161, 23924, 23157, 23920, 23158, + 23921, 23340, 23714, 23715, 23874, 23875, 23804, 23805, 23530, 23531, + 2283, 2286, 2284, 2288, 2290, 2287, 2289, 2285, 2291, 10756, 10751, + 10750, 10757, 10752, 10753, 10755, 10754, 3614, 3613, 3615, 18884, 18883, + 18882, 18881, 18886, 18885, 30619, 30618, 3559, 35374, 35382, 35391, + 35383, 35385, 35380, 35384, 35379, 35395, 35394, 35397, 35389, 35376, + 35396, 35378, 35377, 35390, 35381, 35393, 35387, 35388, 35386, 35392, + 35375, 35513, 35520, 35521, 35516, 35517, 35522, 35523, 35514, 35518, + 35519, 35515, 35579, 35586, 35587, 35582, 35583, 35588, 35589, 35580, + 35584, 35585, 35581, 35690, 35697, 35698, 35693, 35694, 35699, 35700, + 35691, 35695, 35696, 35692, 35590, 35597, 35598, 35593, 35594, 35599, + 35600, 35591, 35595, 35596, 35592, 35668, 35675, 35676, 35671, 35672, + 35677, 35678, 35669, 35673, 35674, 35670, 35568, 35575, 35576, 35571, + 35572, 35577, 35578, 35569, 35573, 35574, 35570, 35679, 35686, 35687, + 35682, 35683, 35688, 35689, 35680, 35684, 35685, 35681, 35601, 35608, + 35609, 35604, 35605, 35610, 35611, 35602, 35606, 35607, 35603, 35734, + 35741, 35742, 35737, 35738, 35743, 35744, 35735, 35739, 35740, 35736, + 35723, 35730, 35731, 35726, 35727, 35732, 35733, 35724, 35728, 35729, + 35725, 35756, 35763, 35764, 35759, 35760, 35765, 35766, 35757, 35761, + 35762, 35758, 35623, 35630, 35631, 35626, 35627, 35632, 35633, 35624, + 35628, 35629, 35625, 35546, 35553, 35554, 35549, 35550, 35555, 35556, + 35547, 35551, 35552, 35548, 35745, 35752, 35753, 35748, 35749, 35754, + 35755, 35746, 35750, 35751, 35747, 35524, 35531, 35532, 35527, 35528, + 35533, 35534, 35525, 35529, 35530, 35526, 35535, 35542, 35543, 35538, + 35539, 35544, 35545, 35536, 35540, 35541, 35537, 35612, 35619, 35620, + 35615, 35616, 35621, 35622, 35613, 35617, 35618, 35614, 35557, 35564, + 35565, 35560, 35561, 35566, 35567, 35558, 35562, 35563, 35559, 35712, + 35719, 35720, 35715, 35716, 35721, 35722, 35713, 35717, 35718, 35714, + 35634, 35642, 35643, 35637, 35638, 35644, 35645, 35635, 35639, 35640, + 35636, 35646, 35653, 35654, 35649, 35650, 35655, 35656, 35647, 35651, + 35652, 35648, 35657, 35664, 35665, 35660, 35661, 35666, 35667, 35658, + 35662, 35663, 35659, 35701, 35708, 35709, 35704, 35705, 35710, 35711, + 35702, 35706, 35707, 35703, 35501, 35502, 35509, 35510, 35505, 35506, + 35511, 35512, 35503, 35507, 35508, 35504, 35641, 33664, 33662, 33663, + 17807, 22179, 22177, 22180, 22178, 22169, 22175, 22173, 22176, 22174, + 22170, 22190, 22186, 22191, 22187, 22171, 22188, 22184, 22189, 22185, + 22172, 22201, 22181, 22183, 22182, 22197, 22200, 22198, 22193, 22199, + 22194, 22195, 22196, 22202, 22192, 22341, 22218, 22219, 22220, 22217, + 22336, 22328, 20316, 20318, 20320, 20317, 20319, 21319, 21321, 21323, + 21320, 21322, 21312, 21311, 21310, 21313, 27780, 27785, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, }; static const unsigned int aliases_start = 0xf0000; -static const unsigned int aliases_end = 0xf01d9; +static const unsigned int aliases_end = 0xf01dd; static const unsigned int name_aliases[] = { 0x0000, 0x0000, @@ -17397,12 +18751,16 @@ static const unsigned int name_aliases[] = { 0xFEFF, 0x122D4, 0x122D5, + 0x12327, + 0x1680B, 0x16E56, 0x16E57, 0x16E76, 0x16E77, 0x1B001, 0x1D0C5, + 0x1E899, + 0x1E89A, 0xE0100, 0xE0101, 0xE0102, diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 0bb5e12d7c3dd9..d86741e1dfc18c 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -62,11 +62,10 @@ pass */ +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -396,6 +395,7 @@ xx_modexec(PyObject *m) static PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_modexec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/xxlimited_35.c b/Modules/xxlimited_35.c index 754a368f77e940..1063e54217b746 100644 --- a/Modules/xxlimited_35.c +++ b/Modules/xxlimited_35.c @@ -5,10 +5,10 @@ * See the xxlimited module for an extension module template. */ +// Test the limited C API version 3.5 #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -#define Py_LIMITED_API 0x03050000 +# define Py_LIMITED_API 0x03050000 #endif #include "Python.h" @@ -297,6 +297,10 @@ xx_modexec(PyObject *m) static PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_modexec}, +#ifdef Py_GIL_DISABLED + // These definitions are in the limited API, but not until 3.13. + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, +#endif {0, NULL} }; diff --git a/Modules/xxmodule.c b/Modules/xxmodule.c index 1e4e0ea3743ce3..a46bf8f0e64ee2 100644 --- a/Modules/xxmodule.c +++ b/Modules/xxmodule.c @@ -384,6 +384,7 @@ xx_exec(PyObject *m) static struct PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/xxsubtype.c b/Modules/xxsubtype.c index 560f43e5b3a643..9c548f44558d41 100644 --- a/Modules/xxsubtype.c +++ b/Modules/xxsubtype.c @@ -288,6 +288,7 @@ xxsubtype_exec(PyObject* m) static struct PyModuleDef_Slot xxsubtype_slots[] = { {Py_mod_exec, xxsubtype_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index fe9a6d8d4150ab..b90665ae7ef64a 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -221,6 +221,8 @@ typedef struct PyThread_type_lock lock; } compobject; +#define _compobject_CAST(op) ((compobject *)op) + static void zlib_error(zlibstate *state, z_stream zst, int err, const char *msg) { @@ -267,12 +269,12 @@ newcompobject(PyTypeObject *type) self->eof = 0; self->is_initialised = 0; self->zdict = NULL; - self->unused_data = PyBytes_FromStringAndSize("", 0); + self->unused_data = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); if (self->unused_data == NULL) { Py_DECREF(self); return NULL; } - self->unconsumed_tail = PyBytes_FromStringAndSize("", 0); + self->unconsumed_tail = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); if (self->unconsumed_tail == NULL) { Py_DECREF(self); return NULL; @@ -489,8 +491,8 @@ zlib_decompress_impl(PyObject *module, Py_buffer *data, int wbits, Py_END_ALLOW_THREADS switch (err) { - case Z_OK: /* fall through */ - case Z_BUF_ERROR: /* fall through */ + case Z_OK: _Py_FALLTHROUGH; + case Z_BUF_ERROR: _Py_FALLTHROUGH; case Z_STREAM_END: break; case Z_MEM_ERROR: @@ -706,7 +708,7 @@ zlib_decompressobj_impl(PyObject *module, int wbits, PyObject *zdict) static void Dealloc(compobject *self) { - PyObject *type = (PyObject *)Py_TYPE(self); + PyTypeObject *type = Py_TYPE(self); PyThread_free_lock(self->lock); Py_XDECREF(self->unused_data); Py_XDECREF(self->unconsumed_tail); @@ -716,18 +718,20 @@ Dealloc(compobject *self) } static void -Comp_dealloc(compobject *self) +Comp_dealloc(PyObject *op) { + compobject *self = _compobject_CAST(op); if (self->is_initialised) - deflateEnd(&self->zst); + (void)deflateEnd(&self->zst); Dealloc(self); } static void -Decomp_dealloc(compobject *self) +Decomp_dealloc(PyObject *op) { + compobject *self = _compobject_CAST(op); if (self->is_initialised) - inflateEnd(&self->zst); + (void)inflateEnd(&self->zst); Dealloc(self); } @@ -915,8 +919,8 @@ zlib_Decompress_decompress_impl(compobject *self, PyTypeObject *cls, Py_END_ALLOW_THREADS switch (err) { - case Z_OK: /* fall through */ - case Z_BUF_ERROR: /* fall through */ + case Z_OK: _Py_FALLTHROUGH; + case Z_BUF_ERROR: _Py_FALLTHROUGH; case Z_STREAM_END: break; default: @@ -1293,8 +1297,8 @@ zlib_Decompress_flush_impl(compobject *self, PyTypeObject *cls, Py_END_ALLOW_THREADS switch (err) { - case Z_OK: /* fall through */ - case Z_BUF_ERROR: /* fall through */ + case Z_OK: _Py_FALLTHROUGH; + case Z_BUF_ERROR: _Py_FALLTHROUGH; case Z_STREAM_END: break; default: @@ -1495,8 +1499,8 @@ decompress_buf(ZlibDecompressor *self, Py_ssize_t max_length) err = inflate(&self->zst, Z_SYNC_FLUSH); Py_END_ALLOW_THREADS switch (err) { - case Z_OK: /* fall through */ - case Z_BUF_ERROR: /* fall through */ + case Z_OK: _Py_FALLTHROUGH; + case Z_BUF_ERROR: _Py_FALLTHROUGH; case Z_STREAM_END: break; default: @@ -2106,6 +2110,7 @@ zlib_exec(PyObject *mod) static PyModuleDef_Slot zlib_slots[] = { {Py_mod_exec, zlib_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Objects/abstract.c b/Objects/abstract.c index 8357175aa5591e..c92ef10aa79648 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -862,7 +862,7 @@ PyObject_Format(PyObject *obj, PyObject *format_spec) /* If no format_spec is provided, use an empty string */ if (format_spec == NULL) { - empty = PyUnicode_New(0, 0); + empty = Py_GetConstant(Py_CONSTANT_EMPTY_STR); format_spec = empty; } @@ -1000,20 +1000,6 @@ binary_op(PyObject *v, PyObject *w, const int op_slot, const char *op_name) PyObject *result = BINARY_OP1(v, w, op_slot, op_name); if (result == Py_NotImplemented) { Py_DECREF(result); - - if (op_slot == NB_SLOT(nb_rshift) && - PyCFunction_CheckExact(v) && - strcmp(((PyCFunctionObject *)v)->m_ml->ml_name, "print") == 0) - { - PyErr_Format(PyExc_TypeError, - "unsupported operand type(s) for %.100s: " - "'%.100s' and '%.100s'. Did you mean \"print(, " - "file=)\"?", - op_name, - Py_TYPE(v)->tp_name, - Py_TYPE(w)->tp_name); - return NULL; - } return binop_type_error(v, w, op_name); } return result; @@ -1521,7 +1507,6 @@ PyNumber_Long(PyObject *o) { PyObject *result; PyNumberMethods *m; - PyObject *trunc_func; Py_buffer view; if (o == NULL) { @@ -1563,37 +1548,6 @@ PyNumber_Long(PyObject *o) if (m && m->nb_index) { return PyNumber_Index(o); } - trunc_func = _PyObject_LookupSpecial(o, &_Py_ID(__trunc__)); - if (trunc_func) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "The delegation of int() to __trunc__ is deprecated.", 1)) { - Py_DECREF(trunc_func); - return NULL; - } - result = _PyObject_CallNoArgs(trunc_func); - Py_DECREF(trunc_func); - if (result == NULL || PyLong_CheckExact(result)) { - return result; - } - if (PyLong_Check(result)) { - Py_SETREF(result, _PyLong_Copy((PyLongObject *)result)); - return result; - } - /* __trunc__ is specified to return an Integral type, - but int() needs to return an int. */ - if (!PyIndex_Check(result)) { - PyErr_Format( - PyExc_TypeError, - "__trunc__ returned non-Integral (type %.200s)", - Py_TYPE(result)->tp_name); - Py_DECREF(result); - return NULL; - } - Py_SETREF(result, PyNumber_Index(result)); - return result; - } - if (PyErr_Occurred()) - return NULL; if (PyUnicode_Check(o)) /* The below check is done in PyLong_FromUnicodeObject(). */ @@ -2039,9 +1993,6 @@ PyObject * PySequence_Tuple(PyObject *v) { PyObject *it; /* iter(v) */ - Py_ssize_t n; /* guess for result tuple size */ - PyObject *result = NULL; - Py_ssize_t j; if (v == NULL) { return null_error(); @@ -2063,58 +2014,54 @@ PySequence_Tuple(PyObject *v) if (it == NULL) return NULL; - /* Guess result size and allocate space. */ - n = PyObject_LengthHint(v, 10); - if (n == -1) - goto Fail; - result = PyTuple_New(n); - if (result == NULL) - goto Fail; - - /* Fill the tuple. */ - for (j = 0; ; ++j) { + Py_ssize_t n; + PyObject *buffer[8]; + for (n = 0; n < 8; n++) { PyObject *item = PyIter_Next(it); if (item == NULL) { - if (PyErr_Occurred()) - goto Fail; - break; - } - if (j >= n) { - size_t newn = (size_t)n; - /* The over-allocation strategy can grow a bit faster - than for lists because unlike lists the - over-allocation isn't permanent -- we reclaim - the excess before the end of this routine. - So, grow by ten and then add 25%. - */ - newn += 10u; - newn += newn >> 2; - if (newn > PY_SSIZE_T_MAX) { - /* Check for overflow */ - PyErr_NoMemory(); - Py_DECREF(item); - goto Fail; + if (PyErr_Occurred()) { + goto fail; } - n = (Py_ssize_t)newn; - if (_PyTuple_Resize(&result, n) != 0) { - Py_DECREF(item); - goto Fail; + Py_DECREF(it); + return _PyTuple_FromArraySteal(buffer, n); + } + buffer[n] = item; + } + PyListObject *list = (PyListObject *)PyList_New(16); + if (list == NULL) { + goto fail; + } + assert(n == 8); + Py_SET_SIZE(list, n); + for (Py_ssize_t j = 0; j < n; j++) { + PyList_SET_ITEM(list, j, buffer[j]); + } + for (;;) { + PyObject *item = PyIter_Next(it); + if (item == NULL) { + if (PyErr_Occurred()) { + Py_DECREF(list); + Py_DECREF(it); + return NULL; } + break; + } + if (_PyList_AppendTakeRef(list, item) < 0) { + Py_DECREF(list); + Py_DECREF(it); + return NULL; } - PyTuple_SET_ITEM(result, j, item); } - - /* Cut tuple back if guess was too large. */ - if (j < n && - _PyTuple_Resize(&result, j) != 0) - goto Fail; - Py_DECREF(it); - return result; - -Fail: - Py_XDECREF(result); + PyObject *res = _PyList_AsTupleAndClear(list); + Py_DECREF(list); + return res; +fail: Py_DECREF(it); + while (n > 0) { + n--; + Py_DECREF(buffer[n]); + } return NULL; } @@ -2173,7 +2120,7 @@ PySequence_Fast(PyObject *v, const char *m) PY_ITERSEARCH_COUNT: -1 if error, else # of times obj appears in seq. PY_ITERSEARCH_INDEX: 0-based index of first occurrence of obj in seq; set ValueError and return -1 if none found; also return -1 on error. - Py_ITERSEARCH_CONTAINS: return 1 if obj in seq, else 0; -1 on error. + PY_ITERSEARCH_CONTAINS: return 1 if obj in seq, else 0; -1 on error. */ Py_ssize_t _PySequence_IterSearch(PyObject *seq, PyObject *obj, int operation) @@ -2190,7 +2137,15 @@ _PySequence_IterSearch(PyObject *seq, PyObject *obj, int operation) it = PyObject_GetIter(seq); if (it == NULL) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { - type_error("argument of type '%.200s' is not iterable", seq); + if (operation == PY_ITERSEARCH_CONTAINS) { + type_error( + "argument of type '%.200s' is not a container or iterable", + seq + ); + } + else { + type_error("argument of type '%.200s' is not iterable", seq); + } } return -1; } @@ -2905,7 +2860,50 @@ PyAIter_Check(PyObject *obj) tp->tp_as_async->am_anext != &_PyObject_NextNotImplemented); } +static int +iternext(PyObject *iter, PyObject **item) +{ + iternextfunc tp_iternext = Py_TYPE(iter)->tp_iternext; + if ((*item = tp_iternext(iter))) { + return 1; + } + + PyThreadState *tstate = _PyThreadState_GET(); + /* When the iterator is exhausted it must return NULL; + * a StopIteration exception may or may not be set. */ + if (!_PyErr_Occurred(tstate)) { + return 0; + } + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + _PyErr_Clear(tstate); + return 0; + } + + /* Error case: an exception (different than StopIteration) is set. */ + return -1; +} + +/* Return 1 and set 'item' to the next item of 'iter' on success. + * Return 0 and set 'item' to NULL when there are no remaining values. + * Return -1, set 'item' to NULL and set an exception on error. + */ +int +PyIter_NextItem(PyObject *iter, PyObject **item) +{ + assert(iter != NULL); + assert(item != NULL); + + if (Py_TYPE(iter)->tp_iternext == NULL) { + *item = NULL; + PyErr_Format(PyExc_TypeError, "expected an iterator, got '%T'", iter); + return -1; + } + + return iternext(iter, item); +} + /* Return next item. + * * If an error occurs, return NULL. PyErr_Occurred() will be true. * If the iteration terminates normally, return NULL and clear the * PyExc_StopIteration exception (if it was set). PyErr_Occurred() @@ -2915,17 +2913,9 @@ PyAIter_Check(PyObject *obj) PyObject * PyIter_Next(PyObject *iter) { - PyObject *result; - result = (*Py_TYPE(iter)->tp_iternext)(iter); - if (result == NULL) { - PyThreadState *tstate = _PyThreadState_GET(); - if (_PyErr_Occurred(tstate) - && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) - { - _PyErr_Clear(tstate); - } - } - return result; + PyObject *item; + (void)iternext(iter, &item); + return item; } PySendResult diff --git a/Objects/boolobject.c b/Objects/boolobject.c index fb48dcbeca7850..a88a8ad0cfd560 100644 --- a/Objects/boolobject.c +++ b/Objects/boolobject.c @@ -71,8 +71,8 @@ static PyObject * bool_invert(PyObject *v) { if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Bitwise inversion '~' on bool is deprecated. This " - "returns the bitwise inversion of the underlying int " + "Bitwise inversion '~' on bool is deprecated and will be removed in " + "Python 3.16. This returns the bitwise inversion of the underlying int " "object and is usually not what you expect from negating " "a bool. Use the 'not' operator for boolean negation or " "~int(x) if you really want the bitwise inversion of the " diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 5e3b3affbc76c5..21584332e0e443 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -42,24 +42,27 @@ _getbytevalue(PyObject* arg, int *value) } static int -bytearray_getbuffer(PyByteArrayObject *obj, Py_buffer *view, int flags) +bytearray_getbuffer(PyObject *self, Py_buffer *view, int flags) { - void *ptr; + PyByteArrayObject *obj = _PyByteArray_CAST(self); if (view == NULL) { PyErr_SetString(PyExc_BufferError, "bytearray_getbuffer: view==NULL argument is obsolete"); return -1; } - ptr = (void *) PyByteArray_AS_STRING(obj); - /* cannot fail if view != NULL and readonly == 0 */ - (void)PyBuffer_FillInfo(view, (PyObject*)obj, ptr, Py_SIZE(obj), 0, flags); + + void *ptr = (void *) PyByteArray_AS_STRING(obj); + if (PyBuffer_FillInfo(view, (PyObject*)obj, ptr, Py_SIZE(obj), 0, flags) < 0) { + return -1; + } obj->ob_exports++; return 0; } static void -bytearray_releasebuffer(PyByteArrayObject *obj, Py_buffer *view) +bytearray_releasebuffer(PyObject *self, Py_buffer *view) { + PyByteArrayObject *obj = _PyByteArray_CAST(self); obj->ob_exports--; assert(obj->ob_exports >= 0); } @@ -286,46 +289,53 @@ PyByteArray_Concat(PyObject *a, PyObject *b) /* Functions stuffed into the type object */ static Py_ssize_t -bytearray_length(PyByteArrayObject *self) +bytearray_length(PyObject *op) { + PyByteArrayObject *self = _PyByteArray_CAST(op); return Py_SIZE(self); } static PyObject * -bytearray_iconcat(PyByteArrayObject *self, PyObject *other) +bytearray_iconcat(PyObject *op, PyObject *other) { - Py_ssize_t size; - Py_buffer vo; + PyByteArrayObject *self = _PyByteArray_CAST(op); + Py_buffer vo; if (PyObject_GetBuffer(other, &vo, PyBUF_SIMPLE) != 0) { PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", Py_TYPE(other)->tp_name, Py_TYPE(self)->tp_name); return NULL; } - size = Py_SIZE(self); + Py_ssize_t size = Py_SIZE(self); if (size > PY_SSIZE_T_MAX - vo.len) { PyBuffer_Release(&vo); return PyErr_NoMemory(); } + if (PyByteArray_Resize((PyObject *)self, size + vo.len) < 0) { PyBuffer_Release(&vo); return NULL; } + memcpy(PyByteArray_AS_STRING(self) + size, vo.buf, vo.len); PyBuffer_Release(&vo); return Py_NewRef(self); } static PyObject * -bytearray_repeat(PyByteArrayObject *self, Py_ssize_t count) +bytearray_repeat(PyObject *op, Py_ssize_t count) { - if (count < 0) + PyByteArrayObject *self = _PyByteArray_CAST(op); + if (count < 0) { count = 0; + } const Py_ssize_t mysize = Py_SIZE(self); - if (count > 0 && mysize > PY_SSIZE_T_MAX / count) + if (count > 0 && mysize > PY_SSIZE_T_MAX / count) { return PyErr_NoMemory(); + } Py_ssize_t size = mysize * count; + PyByteArrayObject* result = (PyByteArrayObject *)PyByteArray_FromStringAndSize(NULL, size); const char* buf = PyByteArray_AS_STRING(self); if (result != NULL && size != 0) { @@ -335,20 +345,24 @@ bytearray_repeat(PyByteArrayObject *self, Py_ssize_t count) } static PyObject * -bytearray_irepeat(PyByteArrayObject *self, Py_ssize_t count) +bytearray_irepeat(PyObject *op, Py_ssize_t count) { - if (count < 0) + PyByteArrayObject *self = _PyByteArray_CAST(op); + if (count < 0) { count = 0; + } else if (count == 1) { return Py_NewRef(self); } const Py_ssize_t mysize = Py_SIZE(self); - if (count > 0 && mysize > PY_SSIZE_T_MAX / count) + if (count > 0 && mysize > PY_SSIZE_T_MAX / count) { return PyErr_NoMemory(); + } const Py_ssize_t size = mysize * count; - if (PyByteArray_Resize((PyObject *)self, size) < 0) + if (PyByteArray_Resize((PyObject *)self, size) < 0) { return NULL; + } char* buf = PyByteArray_AS_STRING(self); _PyBytes_Repeat(buf, size, buf, mysize); @@ -357,8 +371,9 @@ bytearray_irepeat(PyByteArrayObject *self, Py_ssize_t count) } static PyObject * -bytearray_getitem(PyByteArrayObject *self, Py_ssize_t i) +bytearray_getitem(PyObject *op, Py_ssize_t i) { + PyByteArrayObject *self = _PyByteArray_CAST(op); if (i < 0 || i >= Py_SIZE(self)) { PyErr_SetString(PyExc_IndexError, "bytearray index out of range"); return NULL; @@ -367,8 +382,9 @@ bytearray_getitem(PyByteArrayObject *self, Py_ssize_t i) } static PyObject * -bytearray_subscript(PyByteArrayObject *self, PyObject *index) +bytearray_subscript(PyObject *op, PyObject *index) { + PyByteArrayObject *self = _PyByteArray_CAST(op); if (_PyIndex_Check(index)) { Py_ssize_t i = PyNumber_AsSsize_t(index, PyExc_IndexError); @@ -559,12 +575,13 @@ bytearray_setslice(PyByteArrayObject *self, Py_ssize_t lo, Py_ssize_t hi, } static int -bytearray_setitem(PyByteArrayObject *self, Py_ssize_t i, PyObject *value) +bytearray_setitem(PyObject *op, Py_ssize_t i, PyObject *value) { - int ival = -1; + PyByteArrayObject *self = _PyByteArray_CAST(op); // GH-91153: We need to do this *before* the size check, in case value has a // nasty __index__ method that changes the size of the bytearray: + int ival = -1; if (value && !_getbytevalue(value, &ival)) { return -1; } @@ -588,11 +605,11 @@ bytearray_setitem(PyByteArrayObject *self, Py_ssize_t i, PyObject *value) } static int -bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *values) +bytearray_ass_subscript(PyObject *op, PyObject *index, PyObject *values) { - Py_ssize_t start, stop, step, slicelen, needed; - char *buf, *bytes; - buf = PyByteArray_AS_STRING(self); + PyByteArrayObject *self = _PyByteArray_CAST(op); + Py_ssize_t start, stop, step, slicelen; + char *buf = PyByteArray_AS_STRING(self); if (_PyIndex_Check(index)) { Py_ssize_t i = PyNumber_AsSsize_t(index, PyExc_IndexError); @@ -645,6 +662,8 @@ bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *valu return -1; } + char *bytes; + Py_ssize_t needed; if (values == NULL) { bytes = NULL; needed = 0; @@ -661,7 +680,7 @@ bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *valu values = PyByteArray_FromObject(values); if (values == NULL) return -1; - err = bytearray_ass_subscript(self, index, values); + err = bytearray_ass_subscript((PyObject*)self, index, values); Py_DECREF(values); return err; } @@ -670,10 +689,14 @@ bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *valu bytes = PyByteArray_AS_STRING(values); needed = Py_SIZE(values); } + /* Make sure b[5:2] = ... inserts before 5, not before 2. */ if ((step < 0 && start < stop) || (step > 0 && start > stop)) + { stop = start; + } + if (step == 1) { return bytearray_setslice_linear(self, start, stop, bytes, needed); } @@ -785,7 +808,7 @@ bytearray___init___impl(PyByteArrayObject *self, PyObject *arg, if (encoded == NULL) return -1; assert(PyBytes_Check(encoded)); - new = bytearray_iconcat(self, encoded); + new = bytearray_iconcat((PyObject*)self, encoded); Py_DECREF(encoded); if (new == NULL) return -1; @@ -926,8 +949,9 @@ bytearray___init___impl(PyByteArrayObject *self, PyObject *arg, /* Mostly copied from string_repr, but without the "smart quote" functionality. */ static PyObject * -bytearray_repr(PyByteArrayObject *self) +bytearray_repr(PyObject *op) { + PyByteArrayObject *self = _PyByteArray_CAST(op); const char *className = _PyType_Name(Py_TYPE(self)); const char *quote_prefix = "(b"; const char *quote_postfix = ")"; @@ -1021,7 +1045,7 @@ bytearray_str(PyObject *op) return NULL; } } - return bytearray_repr((PyByteArrayObject*)op); + return bytearray_repr(op); } static PyObject * @@ -1080,8 +1104,9 @@ bytearray_richcompare(PyObject *self, PyObject *other, int op) } static void -bytearray_dealloc(PyByteArrayObject *self) +bytearray_dealloc(PyObject *op) { + PyByteArrayObject *self = _PyByteArray_CAST(op); if (self->ob_exports > 0) { PyErr_SetString(PyExc_SystemError, "deallocated bytearray object has exported buffers"); @@ -1121,16 +1146,44 @@ bytearray_dealloc(PyByteArrayObject *self) #include "stringlib/transmogrify.h" +/*[clinic input] +@text_signature "($self, sub[, start[, end]], /)" +bytearray.find + + sub: object + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytes. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytes. + / + +Return the lowest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start:end]. + +Return -1 on failure. +[clinic start generated code]*/ + static PyObject * -bytearray_find(PyByteArrayObject *self, PyObject *args) +bytearray_find_impl(PyByteArrayObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=413e1cab2ae87da0 input=793dfad803e2952f]*/ { - return _Py_bytes_find(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_find(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + sub, start, end); } +/*[clinic input] +bytearray.count = bytearray.find + +Return the number of non-overlapping occurrences of subsection 'sub' in bytes B[start:end]. +[clinic start generated code]*/ + static PyObject * -bytearray_count(PyByteArrayObject *self, PyObject *args) +bytearray_count_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=a21ee2692e4f1233 input=4deb529db38deda8]*/ { - return _Py_bytes_count(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_count(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + sub, start, end); } /*[clinic input] @@ -1162,40 +1215,111 @@ bytearray_copy_impl(PyByteArrayObject *self) PyByteArray_GET_SIZE(self)); } +/*[clinic input] +bytearray.index = bytearray.find + +Return the lowest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start:end]. + +Raise ValueError if the subsection is not found. +[clinic start generated code]*/ + static PyObject * -bytearray_index(PyByteArrayObject *self, PyObject *args) +bytearray_index_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=067a1e78efc672a7 input=8cbaf6836dbd2a9a]*/ { - return _Py_bytes_index(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_index(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + sub, start, end); } +/*[clinic input] +bytearray.rfind = bytearray.find + +Return the highest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start:end]. + +Return -1 on failure. +[clinic start generated code]*/ + static PyObject * -bytearray_rfind(PyByteArrayObject *self, PyObject *args) +bytearray_rfind_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=51bf886f932b283c input=eaa107468a158423]*/ { - return _Py_bytes_rfind(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_rfind(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + sub, start, end); } +/*[clinic input] +bytearray.rindex = bytearray.find + +Return the highest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start:end]. + +Raise ValueError if the subsection is not found. +[clinic start generated code]*/ + static PyObject * -bytearray_rindex(PyByteArrayObject *self, PyObject *args) +bytearray_rindex_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=38e1cf66bafb08b9 input=81cf49d0af4d5bd0]*/ { - return _Py_bytes_rindex(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_rindex(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + sub, start, end); } static int bytearray_contains(PyObject *self, PyObject *arg) { - return _Py_bytes_contains(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), arg); + return _Py_bytes_contains(PyByteArray_AS_STRING(self), + PyByteArray_GET_SIZE(self), + arg); } +/*[clinic input] +@text_signature "($self, prefix[, start[, end]], /)" +bytearray.startswith + + prefix as subobj: object + A bytes or a tuple of bytes to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytearray. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytearray. + / + +Return True if the bytearray starts with the specified prefix, False otherwise. +[clinic start generated code]*/ + static PyObject * -bytearray_startswith(PyByteArrayObject *self, PyObject *args) +bytearray_startswith_impl(PyByteArrayObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=a3d9b6d44d3662a6 input=76385e0b376b45c1]*/ { - return _Py_bytes_startswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_startswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + subobj, start, end); } +/*[clinic input] +@text_signature "($self, suffix[, start[, end]], /)" +bytearray.endswith + + suffix as subobj: object + A bytes or a tuple of bytes to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytearray. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytearray. + / + +Return True if the bytearray ends with the specified suffix, False otherwise. +[clinic start generated code]*/ + static PyObject * -bytearray_endswith(PyByteArrayObject *self, PyObject *args) +bytearray_endswith_impl(PyByteArrayObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=e75ea8c227954caa input=9b8baa879aa3d74b]*/ { - return _Py_bytes_endswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), args); + return _Py_bytes_endswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + subobj, start, end); } /*[clinic input] @@ -1989,8 +2113,9 @@ PyDoc_STRVAR(alloc_doc, Return the number of bytes actually allocated."); static PyObject * -bytearray_alloc(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored)) +bytearray_alloc(PyObject *op, PyObject *Py_UNUSED(ignored)) { + PyByteArrayObject *self = _PyByteArray_CAST(op); return PyLong_FromSsize_t(self->ob_alloc); } @@ -2165,32 +2290,31 @@ bytearray_sizeof_impl(PyByteArrayObject *self) } static PySequenceMethods bytearray_as_sequence = { - (lenfunc)bytearray_length, /* sq_length */ - (binaryfunc)PyByteArray_Concat, /* sq_concat */ - (ssizeargfunc)bytearray_repeat, /* sq_repeat */ - (ssizeargfunc)bytearray_getitem, /* sq_item */ + bytearray_length, /* sq_length */ + PyByteArray_Concat, /* sq_concat */ + bytearray_repeat, /* sq_repeat */ + bytearray_getitem, /* sq_item */ 0, /* sq_slice */ - (ssizeobjargproc)bytearray_setitem, /* sq_ass_item */ + bytearray_setitem, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)bytearray_contains, /* sq_contains */ - (binaryfunc)bytearray_iconcat, /* sq_inplace_concat */ - (ssizeargfunc)bytearray_irepeat, /* sq_inplace_repeat */ + bytearray_contains, /* sq_contains */ + bytearray_iconcat, /* sq_inplace_concat */ + bytearray_irepeat, /* sq_inplace_repeat */ }; static PyMappingMethods bytearray_as_mapping = { - (lenfunc)bytearray_length, - (binaryfunc)bytearray_subscript, - (objobjargproc)bytearray_ass_subscript, + bytearray_length, + bytearray_subscript, + bytearray_ass_subscript, }; static PyBufferProcs bytearray_as_buffer = { - (getbufferproc)bytearray_getbuffer, - (releasebufferproc)bytearray_releasebuffer, + bytearray_getbuffer, + bytearray_releasebuffer, }; -static PyMethodDef -bytearray_methods[] = { - {"__alloc__", (PyCFunction)bytearray_alloc, METH_NOARGS, alloc_doc}, +static PyMethodDef bytearray_methods[] = { + {"__alloc__", bytearray_alloc, METH_NOARGS, alloc_doc}, BYTEARRAY_REDUCE_METHODDEF BYTEARRAY_REDUCE_EX_METHODDEF BYTEARRAY_SIZEOF_METHODDEF @@ -2200,18 +2324,15 @@ bytearray_methods[] = { STRINGLIB_CENTER_METHODDEF BYTEARRAY_CLEAR_METHODDEF BYTEARRAY_COPY_METHODDEF - {"count", (PyCFunction)bytearray_count, METH_VARARGS, - _Py_count__doc__}, + BYTEARRAY_COUNT_METHODDEF BYTEARRAY_DECODE_METHODDEF - {"endswith", (PyCFunction)bytearray_endswith, METH_VARARGS, - _Py_endswith__doc__}, + BYTEARRAY_ENDSWITH_METHODDEF STRINGLIB_EXPANDTABS_METHODDEF BYTEARRAY_EXTEND_METHODDEF - {"find", (PyCFunction)bytearray_find, METH_VARARGS, - _Py_find__doc__}, + BYTEARRAY_FIND_METHODDEF BYTEARRAY_FROMHEX_METHODDEF BYTEARRAY_HEX_METHODDEF - {"index", (PyCFunction)bytearray_index, METH_VARARGS, _Py_index__doc__}, + BYTEARRAY_INDEX_METHODDEF BYTEARRAY_INSERT_METHODDEF {"isalnum", stringlib_isalnum, METH_NOARGS, _Py_isalnum__doc__}, @@ -2241,16 +2362,15 @@ bytearray_methods[] = { BYTEARRAY_REMOVEPREFIX_METHODDEF BYTEARRAY_REMOVESUFFIX_METHODDEF BYTEARRAY_REVERSE_METHODDEF - {"rfind", (PyCFunction)bytearray_rfind, METH_VARARGS, _Py_rfind__doc__}, - {"rindex", (PyCFunction)bytearray_rindex, METH_VARARGS, _Py_rindex__doc__}, + BYTEARRAY_RFIND_METHODDEF + BYTEARRAY_RINDEX_METHODDEF STRINGLIB_RJUST_METHODDEF BYTEARRAY_RPARTITION_METHODDEF BYTEARRAY_RSPLIT_METHODDEF BYTEARRAY_RSTRIP_METHODDEF BYTEARRAY_SPLIT_METHODDEF BYTEARRAY_SPLITLINES_METHODDEF - {"startswith", (PyCFunction)bytearray_startswith, METH_VARARGS , - _Py_startswith__doc__}, + BYTEARRAY_STARTSWITH_METHODDEF BYTEARRAY_STRIP_METHODDEF {"swapcase", stringlib_swapcase, METH_NOARGS, _Py_swapcase__doc__}, @@ -2298,12 +2418,12 @@ PyTypeObject PyByteArray_Type = { "bytearray", sizeof(PyByteArrayObject), 0, - (destructor)bytearray_dealloc, /* tp_dealloc */ + bytearray_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)bytearray_repr, /* tp_repr */ + bytearray_repr, /* tp_repr */ &bytearray_as_number, /* tp_as_number */ &bytearray_as_sequence, /* tp_as_sequence */ &bytearray_as_mapping, /* tp_as_mapping */ @@ -2318,7 +2438,7 @@ PyTypeObject PyByteArray_Type = { bytearray_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ - (richcmpfunc)bytearray_richcompare, /* tp_richcompare */ + bytearray_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ bytearray_iter, /* tp_iter */ 0, /* tp_iternext */ @@ -2333,7 +2453,8 @@ PyTypeObject PyByteArray_Type = { (initproc)bytearray___init__, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ PyType_GenericNew, /* tp_new */ - PyObject_Del, /* tp_free */ + PyObject_Free, /* tp_free */ + .tp_version_tag = _Py_TYPE_VERSION_BYTEARRAY, }; /*********************** Bytearray Iterator ****************************/ @@ -2344,24 +2465,29 @@ typedef struct { PyByteArrayObject *it_seq; /* Set to NULL when iterator is exhausted */ } bytesiterobject; +#define _bytesiterobject_CAST(op) ((bytesiterobject *)(op)) + static void -bytearrayiter_dealloc(bytesiterobject *it) +bytearrayiter_dealloc(PyObject *self) { + bytesiterobject *it = _bytesiterobject_CAST(self); _PyObject_GC_UNTRACK(it); Py_XDECREF(it->it_seq); PyObject_GC_Del(it); } static int -bytearrayiter_traverse(bytesiterobject *it, visitproc visit, void *arg) +bytearrayiter_traverse(PyObject *self, visitproc visit, void *arg) { + bytesiterobject *it = _bytesiterobject_CAST(self); Py_VISIT(it->it_seq); return 0; } static PyObject * -bytearrayiter_next(bytesiterobject *it) +bytearrayiter_next(PyObject *self) { + bytesiterobject *it = _bytesiterobject_CAST(self); PyByteArrayObject *seq; assert(it != NULL); @@ -2381,8 +2507,9 @@ bytearrayiter_next(bytesiterobject *it) } static PyObject * -bytearrayiter_length_hint(bytesiterobject *it, PyObject *Py_UNUSED(ignored)) +bytearrayiter_length_hint(PyObject *self, PyObject *Py_UNUSED(ignored)) { + bytesiterobject *it = _bytesiterobject_CAST(self); Py_ssize_t len = 0; if (it->it_seq) { len = PyByteArray_GET_SIZE(it->it_seq) - it->it_index; @@ -2397,14 +2524,14 @@ PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); static PyObject * -bytearrayiter_reduce(bytesiterobject *it, PyObject *Py_UNUSED(ignored)) +bytearrayiter_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter)); /* _PyEval_GetBuiltin can invoke arbitrary code, * call must be before access of iterator pointers. * see issue #101765 */ - + bytesiterobject *it = _bytesiterobject_CAST(self); if (it->it_seq != NULL) { return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index); } else { @@ -2413,11 +2540,13 @@ bytearrayiter_reduce(bytesiterobject *it, PyObject *Py_UNUSED(ignored)) } static PyObject * -bytearrayiter_setstate(bytesiterobject *it, PyObject *state) +bytearrayiter_setstate(PyObject *self, PyObject *state) { Py_ssize_t index = PyLong_AsSsize_t(state); if (index == -1 && PyErr_Occurred()) return NULL; + + bytesiterobject *it = _bytesiterobject_CAST(self); if (it->it_seq != NULL) { if (index < 0) index = 0; @@ -2431,11 +2560,11 @@ bytearrayiter_setstate(bytesiterobject *it, PyObject *state) PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); static PyMethodDef bytearrayiter_methods[] = { - {"__length_hint__", (PyCFunction)bytearrayiter_length_hint, METH_NOARGS, + {"__length_hint__", bytearrayiter_length_hint, METH_NOARGS, length_hint_doc}, - {"__reduce__", (PyCFunction)bytearrayiter_reduce, METH_NOARGS, + {"__reduce__", bytearrayiter_reduce, METH_NOARGS, bytearray_reduce__doc__}, - {"__setstate__", (PyCFunction)bytearrayiter_setstate, METH_O, + {"__setstate__", bytearrayiter_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -2446,7 +2575,7 @@ PyTypeObject PyByteArrayIter_Type = { sizeof(bytesiterobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)bytearrayiter_dealloc, /* tp_dealloc */ + bytearrayiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -2463,12 +2592,12 @@ PyTypeObject PyByteArrayIter_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)bytearrayiter_traverse, /* tp_traverse */ + bytearrayiter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)bytearrayiter_next, /* tp_iternext */ + bytearrayiter_next, /* tp_iternext */ bytearrayiter_methods, /* tp_methods */ 0, }; diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index c1bc6383df30ce..c239ae18a593e3 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -92,57 +92,6 @@ _Py_bytes_isalnum(const char *cptr, Py_ssize_t len) } -PyDoc_STRVAR_shared(_Py_isascii__doc__, -"B.isascii() -> bool\n\ -\n\ -Return True if B is empty or all characters in B are ASCII,\n\ -False otherwise."); - -// Optimization is copied from ascii_decode in unicodeobject.c -/* Mask to quickly check whether a C 'size_t' contains a - non-ASCII, UTF8-encoded char. */ -#if (SIZEOF_SIZE_T == 8) -# define ASCII_CHAR_MASK 0x8080808080808080ULL -#elif (SIZEOF_SIZE_T == 4) -# define ASCII_CHAR_MASK 0x80808080U -#else -# error C 'size_t' size should be either 4 or 8! -#endif - -PyObject* -_Py_bytes_isascii(const char *cptr, Py_ssize_t len) -{ - const char *p = cptr; - const char *end = p + len; - - while (p < end) { - /* Fast path, see in STRINGLIB(utf8_decode) in stringlib/codecs.h - for an explanation. */ - if (_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T)) { - /* Help allocation */ - const char *_p = p; - while (_p + SIZEOF_SIZE_T <= end) { - size_t value = *(const size_t *) _p; - if (value & ASCII_CHAR_MASK) { - Py_RETURN_FALSE; - } - _p += SIZEOF_SIZE_T; - } - p = _p; - if (_p == end) - break; - } - if ((unsigned char)*p & 0x80) { - Py_RETURN_FALSE; - } - p++; - } - Py_RETURN_TRUE; -} - -#undef ASCII_CHAR_MASK - - PyDoc_STRVAR_shared(_Py_isdigit__doc__, "B.isdigit() -> bool\n\ \n\ @@ -438,6 +387,7 @@ _Py_bytes_maketrans(Py_buffer *frm, Py_buffer *to) #include "stringlib/fastsearch.h" #include "stringlib/count.h" #include "stringlib/find.h" +#include "stringlib/find_max_char.h" /* Wraps stringlib_parse_args_finds() and additionally checks the first @@ -453,31 +403,21 @@ stringlib_parse_args_finds(). */ Py_LOCAL_INLINE(int) -parse_args_finds_byte(const char *function_name, PyObject *args, - PyObject **subobj, char *byte, - Py_ssize_t *start, Py_ssize_t *end) +parse_args_finds_byte(const char *function_name, PyObject **subobj, char *byte) { - PyObject *tmp_subobj; - Py_ssize_t ival; - - if(!stringlib_parse_args_finds(function_name, args, &tmp_subobj, - start, end)) - return 0; - - if (PyObject_CheckBuffer(tmp_subobj)) { - *subobj = tmp_subobj; + if (PyObject_CheckBuffer(*subobj)) { return 1; } - if (!_PyIndex_Check(tmp_subobj)) { + if (!_PyIndex_Check(*subobj)) { PyErr_Format(PyExc_TypeError, "argument should be integer or bytes-like object, " "not '%.200s'", - Py_TYPE(tmp_subobj)->tp_name); + Py_TYPE(*subobj)->tp_name); return 0; } - ival = PyNumber_AsSsize_t(tmp_subobj, NULL); + Py_ssize_t ival = PyNumber_AsSsize_t(*subobj, NULL); if (ival == -1 && PyErr_Occurred()) { return 0; } @@ -492,35 +432,40 @@ parse_args_finds_byte(const char *function_name, PyObject *args, } /* helper macro to fixup start/end slice values */ -#define ADJUST_INDICES(start, end, len) \ - if (end > len) \ - end = len; \ - else if (end < 0) { \ - end += len; \ - if (end < 0) \ - end = 0; \ - } \ - if (start < 0) { \ - start += len; \ - if (start < 0) \ - start = 0; \ - } +#define ADJUST_INDICES(start, end, len) \ + do { \ + if (end > len) { \ + end = len; \ + } \ + else if (end < 0) { \ + end += len; \ + if (end < 0) { \ + end = 0; \ + } \ + } \ + if (start < 0) { \ + start += len; \ + if (start < 0) { \ + start = 0; \ + } \ + } \ + } while (0) Py_LOCAL_INLINE(Py_ssize_t) find_internal(const char *str, Py_ssize_t len, - const char *function_name, PyObject *args, int dir) + const char *function_name, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end, + int dir) { - PyObject *subobj; char byte; Py_buffer subbuf; const char *sub; Py_ssize_t sub_len; - Py_ssize_t start = 0, end = PY_SSIZE_T_MAX; Py_ssize_t res; - if (!parse_args_finds_byte(function_name, args, - &subobj, &byte, &start, &end)) + if (!parse_args_finds_byte(function_name, &subobj, &byte)) { return -2; + } if (subobj) { if (PyObject_GetBuffer(subobj, &subbuf, PyBUF_SIMPLE) != 0) @@ -566,37 +511,21 @@ find_internal(const char *str, Py_ssize_t len, return res; } -PyDoc_STRVAR_shared(_Py_find__doc__, -"B.find(sub[, start[, end]]) -> int\n\ -\n\ -Return the lowest index in B where subsection sub is found,\n\ -such that sub is contained within B[start,end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Return -1 on failure."); - PyObject * -_Py_bytes_find(const char *str, Py_ssize_t len, PyObject *args) +_Py_bytes_find(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) { - Py_ssize_t result = find_internal(str, len, "find", args, +1); + Py_ssize_t result = find_internal(str, len, "find", sub, start, end, +1); if (result == -2) return NULL; return PyLong_FromSsize_t(result); } -PyDoc_STRVAR_shared(_Py_index__doc__, -"B.index(sub[, start[, end]]) -> int\n\ -\n\ -Return the lowest index in B where subsection sub is found,\n\ -such that sub is contained within B[start,end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Raises ValueError when the subsection is not found."); - PyObject * -_Py_bytes_index(const char *str, Py_ssize_t len, PyObject *args) +_Py_bytes_index(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) { - Py_ssize_t result = find_internal(str, len, "index", args, +1); + Py_ssize_t result = find_internal(str, len, "index", sub, start, end, +1); if (result == -2) return NULL; if (result == -1) { @@ -607,37 +536,21 @@ _Py_bytes_index(const char *str, Py_ssize_t len, PyObject *args) return PyLong_FromSsize_t(result); } -PyDoc_STRVAR_shared(_Py_rfind__doc__, -"B.rfind(sub[, start[, end]]) -> int\n\ -\n\ -Return the highest index in B where subsection sub is found,\n\ -such that sub is contained within B[start,end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Return -1 on failure."); - PyObject * -_Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *args) +_Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) { - Py_ssize_t result = find_internal(str, len, "rfind", args, -1); + Py_ssize_t result = find_internal(str, len, "rfind", sub, start, end, -1); if (result == -2) return NULL; return PyLong_FromSsize_t(result); } -PyDoc_STRVAR_shared(_Py_rindex__doc__, -"B.rindex(sub[, start[, end]]) -> int\n\ -\n\ -Return the highest index in B where subsection sub is found,\n\ -such that sub is contained within B[start,end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Raise ValueError when the subsection is not found."); - PyObject * -_Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *args) +_Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) { - Py_ssize_t result = find_internal(str, len, "rindex", args, -1); + Py_ssize_t result = find_internal(str, len, "rindex", sub, start, end, -1); if (result == -2) return NULL; if (result == -1) { @@ -648,28 +561,20 @@ _Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *args) return PyLong_FromSsize_t(result); } -PyDoc_STRVAR_shared(_Py_count__doc__, -"B.count(sub[, start[, end]]) -> int\n\ -\n\ -Return the number of non-overlapping occurrences of subsection sub in\n\ -bytes B[start:end]. Optional arguments start and end are interpreted\n\ -as in slice notation."); - PyObject * -_Py_bytes_count(const char *str, Py_ssize_t len, PyObject *args) +_Py_bytes_count(const char *str, Py_ssize_t len, PyObject *sub_obj, + Py_ssize_t start, Py_ssize_t end) { - PyObject *sub_obj; const char *sub; Py_ssize_t sub_len; char byte; - Py_ssize_t start = 0, end = PY_SSIZE_T_MAX; Py_buffer vsub; PyObject *count_obj; - if (!parse_args_finds_byte("count", args, - &sub_obj, &byte, &start, &end)) + if (!parse_args_finds_byte("count", &sub_obj, &byte)) { return NULL; + } if (sub_obj) { if (PyObject_GetBuffer(sub_obj, &vsub, PyBUF_SIMPLE) != 0) @@ -771,66 +676,65 @@ tailmatch(const char *str, Py_ssize_t len, PyObject *substr, static PyObject * _Py_bytes_tailmatch(const char *str, Py_ssize_t len, - const char *function_name, PyObject *args, + const char *function_name, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end, int direction) { - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; - PyObject *subobj = NULL; - int result; - - if (!stringlib_parse_args_finds(function_name, args, &subobj, &start, &end)) - return NULL; if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - result = tailmatch(str, len, PyTuple_GET_ITEM(subobj, i), - start, end, direction); - if (result == -1) + PyObject *item = PyTuple_GET_ITEM(subobj, i); + int result = tailmatch(str, len, item, start, end, direction); + if (result < 0) { return NULL; + } else if (result) { Py_RETURN_TRUE; } } Py_RETURN_FALSE; } - result = tailmatch(str, len, subobj, start, end, direction); + int result = tailmatch(str, len, subobj, start, end, direction); if (result == -1) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) + if (PyErr_ExceptionMatches(PyExc_TypeError)) { PyErr_Format(PyExc_TypeError, "%s first arg must be bytes or a tuple of bytes, " "not %s", function_name, Py_TYPE(subobj)->tp_name); + } return NULL; } - else - return PyBool_FromLong(result); + return PyBool_FromLong(result); } -PyDoc_STRVAR_shared(_Py_startswith__doc__, -"B.startswith(prefix[, start[, end]]) -> bool\n\ -\n\ -Return True if B starts with the specified prefix, False otherwise.\n\ -With optional start, test B beginning at that position.\n\ -With optional end, stop comparing B at that position.\n\ -prefix can also be a tuple of bytes to try."); +PyObject * +_Py_bytes_startswith(const char *str, Py_ssize_t len, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end) +{ + return _Py_bytes_tailmatch(str, len, "startswith", subobj, start, end, -1); +} PyObject * -_Py_bytes_startswith(const char *str, Py_ssize_t len, PyObject *args) +_Py_bytes_endswith(const char *str, Py_ssize_t len, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end) { - return _Py_bytes_tailmatch(str, len, "startswith", args, -1); + return _Py_bytes_tailmatch(str, len, "endswith", subobj, start, end, +1); } -PyDoc_STRVAR_shared(_Py_endswith__doc__, -"B.endswith(suffix[, start[, end]]) -> bool\n\ +PyDoc_STRVAR_shared(_Py_isascii__doc__, +"B.isascii() -> bool\n\ \n\ -Return True if B ends with the specified suffix, False otherwise.\n\ -With optional start, test B beginning at that position.\n\ -With optional end, stop comparing B at that position.\n\ -suffix can also be a tuple of bytes to try."); +Return True if B is empty or all characters in B are ASCII,\n\ +False otherwise."); -PyObject * -_Py_bytes_endswith(const char *str, Py_ssize_t len, PyObject *args) +PyObject* +_Py_bytes_isascii(const char *cptr, Py_ssize_t len) { - return _Py_bytes_tailmatch(str, len, "endswith", args, +1); + const char *p = cptr; + const char *end = p + len; + Py_ssize_t max_char = stringlib_find_max_char(cptr, end); + if (max_char > 127) { + Py_RETURN_FALSE; + } + Py_RETURN_TRUE; } diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 26227dd251122d..b3d1c425ad18b7 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -51,26 +51,53 @@ static inline PyObject* bytes_get_empty(void) } +static inline void +set_ob_shash(PyBytesObject *a, Py_hash_t hash) +{ +_Py_COMP_DIAG_PUSH +_Py_COMP_DIAG_IGNORE_DEPR_DECLS +#ifdef Py_GIL_DISABLED + _Py_atomic_store_ssize_relaxed(&a->ob_shash, hash); +#else + a->ob_shash = hash; +#endif +_Py_COMP_DIAG_POP +} + +static inline Py_hash_t +get_ob_shash(PyBytesObject *a) +{ +_Py_COMP_DIAG_PUSH +_Py_COMP_DIAG_IGNORE_DEPR_DECLS +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&a->ob_shash); +#else + return a->ob_shash; +#endif +_Py_COMP_DIAG_POP +} + + /* - For PyBytes_FromString(), the parameter `str' points to a null-terminated - string containing exactly `size' bytes. + For PyBytes_FromString(), the parameter 'str' points to a null-terminated + string containing exactly 'size' bytes. - For PyBytes_FromStringAndSize(), the parameter `str' is - either NULL or else points to a string containing at least `size' bytes. - For PyBytes_FromStringAndSize(), the string in the `str' parameter does + For PyBytes_FromStringAndSize(), the parameter 'str' is + either NULL or else points to a string containing at least 'size' bytes. + For PyBytes_FromStringAndSize(), the string in the 'str' parameter does not have to be null-terminated. (Therefore it is safe to construct a - substring by calling `PyBytes_FromStringAndSize(origstring, substrlen)'.) - If `str' is NULL then PyBytes_FromStringAndSize() will allocate `size+1' + substring by calling 'PyBytes_FromStringAndSize(origstring, substrlen)'.) + If 'str' is NULL then PyBytes_FromStringAndSize() will allocate 'size+1' bytes (setting the last byte to the null terminating character) and you can - fill in the data yourself. If `str' is non-NULL then the resulting + fill in the data yourself. If 'str' is non-NULL then the resulting PyBytes object must be treated as immutable and you must not fill in nor alter the data yourself, since the strings may be shared. - The PyObject member `op->ob_size', which denotes the number of "extra + The PyObject member 'op->ob_size', which denotes the number of "extra items" in a variable-size object, will contain the number of bytes allocated for string data, not counting the null terminating character. - It is therefore equal to the `size' parameter (for - PyBytes_FromStringAndSize()) or the length of the string in the `str' + It is therefore equal to the 'size' parameter (for + PyBytes_FromStringAndSize()) or the length of the string in the 'str' parameter (for PyBytes_FromString()). */ static PyObject * @@ -98,10 +125,7 @@ _PyBytes_FromSize(Py_ssize_t size, int use_calloc) return PyErr_NoMemory(); } _PyObject_InitVar((PyVarObject*)op, &PyBytes_Type, size); -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - op->ob_shash = -1; -_Py_COMP_DIAG_POP + set_ob_shash(op, -1); if (!use_calloc) { op->ob_sval[size] = '\0'; } @@ -165,10 +189,7 @@ PyBytes_FromString(const char *str) return PyErr_NoMemory(); } _PyObject_InitVar((PyVarObject*)op, &PyBytes_Type, size); -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - op->ob_shash = -1; -_Py_COMP_DIAG_POP + set_ob_shash(op, -1); memcpy(op->ob_sval, str, size+1); return (PyObject *) op; } @@ -477,21 +498,32 @@ formatlong(PyObject *v, int flags, int prec, int type) static int byte_converter(PyObject *arg, char *p) { - if (PyBytes_Check(arg) && PyBytes_GET_SIZE(arg) == 1) { + if (PyBytes_Check(arg)) { + if (PyBytes_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "%%c requires an integer in range(256) or " + "a single byte, not a bytes object of length %zd", + PyBytes_GET_SIZE(arg)); + return 0; + } *p = PyBytes_AS_STRING(arg)[0]; return 1; } - else if (PyByteArray_Check(arg) && PyByteArray_GET_SIZE(arg) == 1) { + else if (PyByteArray_Check(arg)) { + if (PyByteArray_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "%%c requires an integer in range(256) or " + "a single byte, not a bytearray object of length %zd", + PyByteArray_GET_SIZE(arg)); + return 0; + } *p = PyByteArray_AS_STRING(arg)[0]; return 1; } - else { + else if (PyIndex_Check(arg)) { int overflow; long ival = PyLong_AsLongAndOverflow(arg, &overflow); if (ival == -1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) { - goto onError; - } return 0; } if (!(0 <= ival && ival <= 255)) { @@ -503,9 +535,9 @@ byte_converter(PyObject *arg, char *p) *p = (char)ival; return 1; } - onError: - PyErr_SetString(PyExc_TypeError, - "%c requires an integer in range(256) or a single byte"); + PyErr_Format(PyExc_TypeError, + "%%c requires an integer in range(256) or a single byte, not %T", + arg); return 0; } @@ -1173,7 +1205,8 @@ PyObject *PyBytes_DecodeEscape(const char *s, unsigned char c = *first_invalid_escape; if ('4' <= c && c <= '7') { if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "invalid octal escape sequence '\\%.3s'", + "b\"\\%.3s\" is an invalid octal escape sequence. " + "Such sequences will not work in the future. ", first_invalid_escape) < 0) { Py_DECREF(result); @@ -1182,7 +1215,8 @@ PyObject *PyBytes_DecodeEscape(const char *s, } else { if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "invalid escape sequence '\\%c'", + "b\"\\%c\" is an invalid escape sequence. " + "Such sequences will not work in the future. ", c) < 0) { Py_DECREF(result); @@ -1191,7 +1225,6 @@ PyObject *PyBytes_DecodeEscape(const char *s, } } return result; - } /* -------------------------------------------------------------------- */ /* object api */ @@ -1394,8 +1427,9 @@ bytes_str(PyObject *op) } static Py_ssize_t -bytes_length(PyBytesObject *a) +bytes_length(PyObject *self) { + PyBytesObject *a = _PyBytes_CAST(self); return Py_SIZE(a); } @@ -1445,11 +1479,9 @@ bytes_concat(PyObject *a, PyObject *b) } static PyObject * -bytes_repeat(PyBytesObject *a, Py_ssize_t n) +bytes_repeat(PyObject *self, Py_ssize_t n) { - Py_ssize_t size; - PyBytesObject *op; - size_t nbytes; + PyBytesObject *a = _PyBytes_CAST(self); if (n < 0) n = 0; /* watch out for overflows: the size can overflow int, @@ -1460,25 +1492,22 @@ bytes_repeat(PyBytesObject *a, Py_ssize_t n) "repeated bytes are too long"); return NULL; } - size = Py_SIZE(a) * n; + Py_ssize_t size = Py_SIZE(a) * n; if (size == Py_SIZE(a) && PyBytes_CheckExact(a)) { return Py_NewRef(a); } - nbytes = (size_t)size; + size_t nbytes = (size_t)size; if (nbytes + PyBytesObject_SIZE <= nbytes) { PyErr_SetString(PyExc_OverflowError, "repeated bytes are too long"); return NULL; } - op = (PyBytesObject *)PyObject_Malloc(PyBytesObject_SIZE + nbytes); + PyBytesObject *op = PyObject_Malloc(PyBytesObject_SIZE + nbytes); if (op == NULL) { return PyErr_NoMemory(); } _PyObject_InitVar((PyVarObject*)op, &PyBytes_Type, size); -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - op->ob_shash = -1; -_Py_COMP_DIAG_POP + set_ob_shash(op, -1); op->ob_sval[size] = '\0'; _PyBytes_Repeat(op->ob_sval, size, a->ob_sval, Py_SIZE(a)); @@ -1493,8 +1522,9 @@ bytes_contains(PyObject *self, PyObject *arg) } static PyObject * -bytes_item(PyBytesObject *a, Py_ssize_t i) +bytes_item(PyObject *self, Py_ssize_t i) { + PyBytesObject *a = _PyBytes_CAST(self); if (i < 0 || i >= Py_SIZE(a)) { PyErr_SetString(PyExc_IndexError, "index out of range"); return NULL; @@ -1520,21 +1550,17 @@ bytes_compare_eq(PyBytesObject *a, PyBytesObject *b) } static PyObject* -bytes_richcompare(PyBytesObject *a, PyBytesObject *b, int op) +bytes_richcompare(PyObject *aa, PyObject *bb, int op) { - int c; - Py_ssize_t len_a, len_b; - Py_ssize_t min_len; - /* Make sure both arguments are strings. */ - if (!(PyBytes_Check(a) && PyBytes_Check(b))) { + if (!(PyBytes_Check(aa) && PyBytes_Check(bb))) { if (_Py_GetConfig()->bytes_warning && (op == Py_EQ || op == Py_NE)) { - if (PyUnicode_Check(a) || PyUnicode_Check(b)) { + if (PyUnicode_Check(aa) || PyUnicode_Check(bb)) { if (PyErr_WarnEx(PyExc_BytesWarning, "Comparison between bytes and string", 1)) return NULL; } - if (PyLong_Check(a) || PyLong_Check(b)) { + if (PyLong_Check(aa) || PyLong_Check(bb)) { if (PyErr_WarnEx(PyExc_BytesWarning, "Comparison between bytes and int", 1)) return NULL; @@ -1542,7 +1568,10 @@ bytes_richcompare(PyBytesObject *a, PyBytesObject *b, int op) } Py_RETURN_NOTIMPLEMENTED; } - else if (a == b) { + + PyBytesObject *a = _PyBytes_CAST(aa); + PyBytesObject *b = _PyBytes_CAST(bb); + if (a == b) { switch (op) { case Py_EQ: case Py_LE: @@ -1564,38 +1593,42 @@ bytes_richcompare(PyBytesObject *a, PyBytesObject *b, int op) return PyBool_FromLong(eq); } else { - len_a = Py_SIZE(a); - len_b = Py_SIZE(b); - min_len = Py_MIN(len_a, len_b); + Py_ssize_t len_a = Py_SIZE(a); + Py_ssize_t len_b = Py_SIZE(b); + Py_ssize_t min_len = Py_MIN(len_a, len_b); + int c; if (min_len > 0) { c = Py_CHARMASK(*a->ob_sval) - Py_CHARMASK(*b->ob_sval); if (c == 0) c = memcmp(a->ob_sval, b->ob_sval, min_len); } - else + else { c = 0; - if (c != 0) + } + if (c != 0) { Py_RETURN_RICHCOMPARE(c, 0, op); + } Py_RETURN_RICHCOMPARE(len_a, len_b, op); } } static Py_hash_t -bytes_hash(PyBytesObject *a) +bytes_hash(PyObject *self) { -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - if (a->ob_shash == -1) { + PyBytesObject *a = _PyBytes_CAST(self); + Py_hash_t hash = get_ob_shash(a); + if (hash == -1) { /* Can't fail */ - a->ob_shash = _Py_HashBytes(a->ob_sval, Py_SIZE(a)); + hash = Py_HashBuffer(a->ob_sval, Py_SIZE(a)); + set_ob_shash(a, hash); } - return a->ob_shash; -_Py_COMP_DIAG_POP + return hash; } static PyObject* -bytes_subscript(PyBytesObject* self, PyObject* item) +bytes_subscript(PyObject *op, PyObject* item) { + PyBytesObject *self = _PyBytes_CAST(op); if (_PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) @@ -1623,7 +1656,7 @@ bytes_subscript(PyBytesObject* self, PyObject* item) &stop, step); if (slicelength <= 0) { - return PyBytes_FromStringAndSize("", 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } else if (start == 0 && step == 1 && slicelength == PyBytes_GET_SIZE(self) && @@ -1659,31 +1692,32 @@ bytes_subscript(PyBytesObject* self, PyObject* item) } static int -bytes_buffer_getbuffer(PyBytesObject *self, Py_buffer *view, int flags) +bytes_buffer_getbuffer(PyObject *op, Py_buffer *view, int flags) { + PyBytesObject *self = _PyBytes_CAST(op); return PyBuffer_FillInfo(view, (PyObject*)self, (void *)self->ob_sval, Py_SIZE(self), 1, flags); } static PySequenceMethods bytes_as_sequence = { - (lenfunc)bytes_length, /*sq_length*/ - (binaryfunc)bytes_concat, /*sq_concat*/ - (ssizeargfunc)bytes_repeat, /*sq_repeat*/ - (ssizeargfunc)bytes_item, /*sq_item*/ + bytes_length, /*sq_length*/ + bytes_concat, /*sq_concat*/ + bytes_repeat, /*sq_repeat*/ + bytes_item, /*sq_item*/ 0, /*sq_slice*/ 0, /*sq_ass_item*/ 0, /*sq_ass_slice*/ - (objobjproc)bytes_contains /*sq_contains*/ + bytes_contains /*sq_contains*/ }; static PyMappingMethods bytes_as_mapping = { - (lenfunc)bytes_length, - (binaryfunc)bytes_subscript, + bytes_length, + bytes_subscript, 0, }; static PyBufferProcs bytes_as_buffer = { - (getbufferproc)bytes_buffer_getbuffer, + bytes_buffer_getbuffer, NULL, }; @@ -1856,37 +1890,95 @@ bytes_join(PyBytesObject *self, PyObject *iterable_of_bytes) } PyObject * -_PyBytes_Join(PyObject *sep, PyObject *x) +PyBytes_Join(PyObject *sep, PyObject *iterable) { - assert(sep != NULL && PyBytes_Check(sep)); - assert(x != NULL); - return bytes_join((PyBytesObject*)sep, x); + if (sep == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + if (!PyBytes_Check(sep)) { + PyErr_Format(PyExc_TypeError, + "sep: expected bytes, got %T", sep); + return NULL; + } + + return stringlib_bytes_join(sep, iterable); } +/*[clinic input] +@text_signature "($self, sub[, start[, end]], /)" +bytes.find + + sub: object + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytes. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytes. + / + +Return the lowest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start,end]. + +Return -1 on failure. +[clinic start generated code]*/ + static PyObject * -bytes_find(PyBytesObject *self, PyObject *args) +bytes_find_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=d5961a1c77b472a1 input=3171e62a8ae7f240]*/ { - return _Py_bytes_find(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_find(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + sub, start, end); } +/*[clinic input] +bytes.index = bytes.find + +Return the lowest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start,end]. + +Raise ValueError if the subsection is not found. +[clinic start generated code]*/ + static PyObject * -bytes_index(PyBytesObject *self, PyObject *args) +bytes_index_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=0da25cc74683ba42 input=aa34ad71ba0bafe3]*/ { - return _Py_bytes_index(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_index(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + sub, start, end); } +/*[clinic input] +bytes.rfind = bytes.find + +Return the highest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start,end]. + +Return -1 on failure. +[clinic start generated code]*/ static PyObject * -bytes_rfind(PyBytesObject *self, PyObject *args) +bytes_rfind_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=51b60fa4ad011c09 input=864c3e7f3010b33c]*/ { - return _Py_bytes_rfind(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_rfind(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + sub, start, end); } +/*[clinic input] +bytes.rindex = bytes.find + +Return the highest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start,end]. + +Raise ValueError if the subsection is not found. +[clinic start generated code]*/ static PyObject * -bytes_rindex(PyBytesObject *self, PyObject *args) +bytes_rindex_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=42bf674e0a0aabf6 input=21051fc5cfeacf2c]*/ { - return _Py_bytes_rindex(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_rindex(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + sub, start, end); } @@ -2023,10 +2115,19 @@ bytes_rstrip_impl(PyBytesObject *self, PyObject *bytes) } +/*[clinic input] +bytes.count = bytes.find + +Return the number of non-overlapping occurrences of subsection 'sub' in bytes B[start:end]. +[clinic start generated code]*/ + static PyObject * -bytes_count(PyBytesObject *self, PyObject *args) +bytes_count_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=9848140b9be17d0f input=b6e4a5ed515e1e59]*/ { - return _Py_bytes_count(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_count(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + sub, start, end); } @@ -2285,16 +2386,52 @@ bytes_removesuffix_impl(PyBytesObject *self, Py_buffer *suffix) return PyBytes_FromStringAndSize(self_start, self_len); } +/*[clinic input] +@text_signature "($self, prefix[, start[, end]], /)" +bytes.startswith + + prefix as subobj: object + A bytes or a tuple of bytes to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytes. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytes. + / + +Return True if the bytes starts with the specified prefix, False otherwise. +[clinic start generated code]*/ + static PyObject * -bytes_startswith(PyBytesObject *self, PyObject *args) +bytes_startswith_impl(PyBytesObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end) +/*[clinic end generated code: output=b1e8da1cbd528e8c input=8a4165df8adfa6c9]*/ { - return _Py_bytes_startswith(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_startswith(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + subobj, start, end); } +/*[clinic input] +@text_signature "($self, suffix[, start[, end]], /)" +bytes.endswith + + suffix as subobj: object + A bytes or a tuple of bytes to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the bytes. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the bytes. + / + +Return True if the bytes ends with the specified suffix, False otherwise. +[clinic start generated code]*/ + static PyObject * -bytes_endswith(PyBytesObject *self, PyObject *args) +bytes_endswith_impl(PyBytesObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=038b633111f3629d input=b5c3407a2a5c9aac]*/ { - return _Py_bytes_endswith(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), args); + return _Py_bytes_endswith(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + subobj, start, end); } @@ -2424,7 +2561,12 @@ _PyBytes_FromHex(PyObject *string, int use_bytearray) bot = _PyLong_DigitValue[*str]; if (bot >= 16) { - invalid_char = str - PyUnicode_1BYTE_DATA(string); + /* Check if we had a second digit */ + if (str >= end){ + invalid_char = -1; + } else { + invalid_char = str - PyUnicode_1BYTE_DATA(string); + } goto error; } str++; @@ -2435,9 +2577,14 @@ _PyBytes_FromHex(PyObject *string, int use_bytearray) return _PyBytesWriter_Finish(&writer, buf); error: - PyErr_Format(PyExc_ValueError, - "non-hexadecimal number found in " - "fromhex() arg at position %zd", invalid_char); + if (invalid_char == -1) { + PyErr_SetString(PyExc_ValueError, + "fromhex() arg must contain an even number of hexadecimal digits"); + } else { + PyErr_Format(PyExc_ValueError, + "non-hexadecimal number found in " + "fromhex() arg at position %zd", invalid_char); + } _PyBytesWriter_Dealloc(&writer); return NULL; } @@ -2488,17 +2635,14 @@ bytes_methods[] = { {"capitalize", stringlib_capitalize, METH_NOARGS, _Py_capitalize__doc__}, STRINGLIB_CENTER_METHODDEF - {"count", (PyCFunction)bytes_count, METH_VARARGS, - _Py_count__doc__}, + BYTES_COUNT_METHODDEF BYTES_DECODE_METHODDEF - {"endswith", (PyCFunction)bytes_endswith, METH_VARARGS, - _Py_endswith__doc__}, + BYTES_ENDSWITH_METHODDEF STRINGLIB_EXPANDTABS_METHODDEF - {"find", (PyCFunction)bytes_find, METH_VARARGS, - _Py_find__doc__}, + BYTES_FIND_METHODDEF BYTES_FROMHEX_METHODDEF BYTES_HEX_METHODDEF - {"index", (PyCFunction)bytes_index, METH_VARARGS, _Py_index__doc__}, + BYTES_INDEX_METHODDEF {"isalnum", stringlib_isalnum, METH_NOARGS, _Py_isalnum__doc__}, {"isalpha", stringlib_isalpha, METH_NOARGS, @@ -2524,16 +2668,15 @@ bytes_methods[] = { BYTES_REPLACE_METHODDEF BYTES_REMOVEPREFIX_METHODDEF BYTES_REMOVESUFFIX_METHODDEF - {"rfind", (PyCFunction)bytes_rfind, METH_VARARGS, _Py_rfind__doc__}, - {"rindex", (PyCFunction)bytes_rindex, METH_VARARGS, _Py_rindex__doc__}, + BYTES_RFIND_METHODDEF + BYTES_RINDEX_METHODDEF STRINGLIB_RJUST_METHODDEF BYTES_RPARTITION_METHODDEF BYTES_RSPLIT_METHODDEF BYTES_RSTRIP_METHODDEF BYTES_SPLIT_METHODDEF BYTES_SPLITLINES_METHODDEF - {"startswith", (PyCFunction)bytes_startswith, METH_VARARGS, - _Py_startswith__doc__}, + BYTES_STARTSWITH_METHODDEF BYTES_STRIP_METHODDEF {"swapcase", stringlib_swapcase, METH_NOARGS, _Py_swapcase__doc__}, @@ -2879,10 +3022,7 @@ bytes_alloc(PyTypeObject *self, Py_ssize_t nitems) if (obj == NULL) { return NULL; } -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - obj->ob_shash = -1; -_Py_COMP_DIAG_POP + set_ob_shash(obj, -1); return (PyObject*)obj; } @@ -2899,11 +3039,8 @@ bytes_subtype_new(PyTypeObject *type, PyObject *tmp) if (pnew != NULL) { memcpy(PyBytes_AS_STRING(pnew), PyBytes_AS_STRING(tmp), n+1); -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - ((PyBytesObject *)pnew)->ob_shash = - ((PyBytesObject *)tmp)->ob_shash; -_Py_COMP_DIAG_POP + set_ob_shash((PyBytesObject *)pnew, + get_ob_shash((PyBytesObject *)tmp)); } return pnew; } @@ -2933,11 +3070,11 @@ PyTypeObject PyBytes_Type = { 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)bytes_repr, /* tp_repr */ + bytes_repr, /* tp_repr */ &bytes_as_number, /* tp_as_number */ &bytes_as_sequence, /* tp_as_sequence */ &bytes_as_mapping, /* tp_as_mapping */ - (hashfunc)bytes_hash, /* tp_hash */ + bytes_hash, /* tp_hash */ 0, /* tp_call */ bytes_str, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -2949,7 +3086,7 @@ PyTypeObject PyBytes_Type = { bytes_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ - (richcmpfunc)bytes_richcompare, /* tp_richcompare */ + bytes_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ bytes_iter, /* tp_iter */ 0, /* tp_iternext */ @@ -2964,7 +3101,8 @@ PyTypeObject PyBytes_Type = { 0, /* tp_init */ bytes_alloc, /* tp_alloc */ bytes_new, /* tp_new */ - PyObject_Del, /* tp_free */ + PyObject_Free, /* tp_free */ + .tp_version_tag = _Py_TYPE_VERSION_BYTES, }; void @@ -3025,11 +3163,9 @@ PyBytes_ConcatAndDel(PyObject **pv, PyObject *w) /* The following function breaks the notion that bytes are immutable: - it changes the size of a bytes object. We get away with this only if there - is only one module referencing the object. You can also think of it + it changes the size of a bytes object. You can think of it as creating a new bytes object and destroying the old one, only - more efficiently. In any case, don't use this if the bytes object may - already be known to some other part of the code... + more efficiently. Note that if there's not enough memory to resize the bytes object, the original bytes object at *pv is deallocated, *pv is set to NULL, an "out of memory" exception is set, and -1 is returned. Else (on success) 0 is @@ -3045,36 +3181,49 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) PyBytesObject *sv; v = *pv; if (!PyBytes_Check(v) || newsize < 0) { - goto error; + *pv = 0; + Py_DECREF(v); + PyErr_BadInternalCall(); + return -1; } - if (Py_SIZE(v) == newsize) { + Py_ssize_t oldsize = PyBytes_GET_SIZE(v); + if (oldsize == newsize) { /* return early if newsize equals to v->ob_size */ return 0; } - if (Py_SIZE(v) == 0) { - if (newsize == 0) { - return 0; - } + if (oldsize == 0) { *pv = _PyBytes_FromSize(newsize, 0); Py_DECREF(v); return (*pv == NULL) ? -1 : 0; } - if (Py_REFCNT(v) != 1) { - goto error; - } if (newsize == 0) { *pv = bytes_get_empty(); Py_DECREF(v); return 0; } + if (Py_REFCNT(v) != 1) { + if (oldsize < newsize) { + *pv = _PyBytes_FromSize(newsize, 0); + if (*pv) { + memcpy(PyBytes_AS_STRING(*pv), PyBytes_AS_STRING(v), oldsize); + } + } + else { + *pv = PyBytes_FromStringAndSize(PyBytes_AS_STRING(v), newsize); + } + Py_DECREF(v); + return (*pv == NULL) ? -1 : 0; + } + #ifdef Py_TRACE_REFS _Py_ForgetReference(v); #endif + _PyReftracerTrack(v, PyRefTracer_DESTROY); *pv = (PyObject *) PyObject_Realloc(v, PyBytesObject_SIZE + newsize); if (*pv == NULL) { #ifdef Py_REF_DEBUG - _Py_DecRefTotal(_PyInterpreterState_GET()); + _Py_DecRefTotal(_PyThreadState_GET()); #endif PyObject_Free(v); PyErr_NoMemory(); @@ -3084,16 +3233,8 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) sv = (PyBytesObject *) *pv; Py_SET_SIZE(sv, newsize); sv->ob_sval[newsize] = '\0'; -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - sv->ob_shash = -1; /* invalidate cached hash value */ -_Py_COMP_DIAG_POP + set_ob_shash(sv, -1); /* invalidate cached hash value */ return 0; -error: - *pv = 0; - Py_DECREF(v); - PyErr_BadInternalCall(); - return -1; } @@ -3105,24 +3246,29 @@ typedef struct { PyBytesObject *it_seq; /* Set to NULL when iterator is exhausted */ } striterobject; +#define _striterobject_CAST(op) ((striterobject *)(op)) + static void -striter_dealloc(striterobject *it) +striter_dealloc(PyObject *op) { + striterobject *it = _striterobject_CAST(op); _PyObject_GC_UNTRACK(it); Py_XDECREF(it->it_seq); PyObject_GC_Del(it); } static int -striter_traverse(striterobject *it, visitproc visit, void *arg) +striter_traverse(PyObject *op, visitproc visit, void *arg) { + striterobject *it = _striterobject_CAST(op); Py_VISIT(it->it_seq); return 0; } static PyObject * -striter_next(striterobject *it) +striter_next(PyObject *op) { + striterobject *it = _striterobject_CAST(op); PyBytesObject *seq; assert(it != NULL); @@ -3142,8 +3288,9 @@ striter_next(striterobject *it) } static PyObject * -striter_len(striterobject *it, PyObject *Py_UNUSED(ignored)) +striter_len(PyObject *op, PyObject *Py_UNUSED(ignored)) { + striterobject *it = _striterobject_CAST(op); Py_ssize_t len = 0; if (it->it_seq) len = PyBytes_GET_SIZE(it->it_seq) - it->it_index; @@ -3154,14 +3301,14 @@ PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); static PyObject * -striter_reduce(striterobject *it, PyObject *Py_UNUSED(ignored)) +striter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) { PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter)); /* _PyEval_GetBuiltin can invoke arbitrary code, * call must be before access of iterator pointers. * see issue #101765 */ - + striterobject *it = _striterobject_CAST(op); if (it->it_seq != NULL) { return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index); } else { @@ -3172,11 +3319,12 @@ striter_reduce(striterobject *it, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); static PyObject * -striter_setstate(striterobject *it, PyObject *state) +striter_setstate(PyObject *op, PyObject *state) { Py_ssize_t index = PyLong_AsSsize_t(state); if (index == -1 && PyErr_Occurred()) return NULL; + striterobject *it = _striterobject_CAST(op); if (it->it_seq != NULL) { if (index < 0) index = 0; @@ -3190,12 +3338,9 @@ striter_setstate(striterobject *it, PyObject *state) PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); static PyMethodDef striter_methods[] = { - {"__length_hint__", (PyCFunction)striter_len, METH_NOARGS, - length_hint_doc}, - {"__reduce__", (PyCFunction)striter_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)striter_setstate, METH_O, - setstate_doc}, + {"__length_hint__", striter_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", striter_reduce, METH_NOARGS, reduce_doc}, + {"__setstate__", striter_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -3205,7 +3350,7 @@ PyTypeObject PyBytesIter_Type = { sizeof(striterobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)striter_dealloc, /* tp_dealloc */ + striter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -3222,12 +3367,12 @@ PyTypeObject PyBytesIter_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)striter_traverse, /* tp_traverse */ + striter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)striter_next, /* tp_iternext */ + striter_next, /* tp_iternext */ striter_methods, /* tp_methods */ 0, }; diff --git a/Objects/capsule.c b/Objects/capsule.c index 555979dab2b789..16ae65905ef5ac 100644 --- a/Objects/capsule.c +++ b/Objects/capsule.c @@ -18,6 +18,8 @@ typedef struct { } PyCapsule; +#define _PyCapsule_CAST(op) ((PyCapsule *)(op)) + static int _is_legal_capsule(PyObject *op, const char *invalid_capsule) @@ -284,7 +286,7 @@ PyCapsule_Import(const char *name, int no_block) static void capsule_dealloc(PyObject *op) { - PyCapsule *capsule = (PyCapsule *)op; + PyCapsule *capsule = _PyCapsule_CAST(op); PyObject_GC_UnTrack(op); if (capsule->destructor) { capsule->destructor(op); @@ -296,7 +298,7 @@ capsule_dealloc(PyObject *op) static PyObject * capsule_repr(PyObject *o) { - PyCapsule *capsule = (PyCapsule *)o; + PyCapsule *capsule = _PyCapsule_CAST(o); const char *name; const char *quote; @@ -314,24 +316,27 @@ capsule_repr(PyObject *o) static int -capsule_traverse(PyCapsule *capsule, visitproc visit, void *arg) +capsule_traverse(PyObject *self, visitproc visit, void *arg) { // Capsule object is only tracked by the GC - // if _PyCapsule_SetTraverse() is called - assert(capsule->traverse_func != NULL); - - return capsule->traverse_func((PyObject*)capsule, visit, arg); + // if _PyCapsule_SetTraverse() is called, but + // this can still be manually triggered by gc.get_referents() + PyCapsule *capsule = _PyCapsule_CAST(self); + if (capsule->traverse_func != NULL) { + return capsule->traverse_func(self, visit, arg); + } + return 0; } static int -capsule_clear(PyCapsule *capsule) +capsule_clear(PyObject *self) { // Capsule object is only tracked by the GC // if _PyCapsule_SetTraverse() is called + PyCapsule *capsule = _PyCapsule_CAST(self); assert(capsule->clear_func != NULL); - - return capsule->clear_func((PyObject*)capsule); + return capsule->clear_func(self); } @@ -354,8 +359,8 @@ PyTypeObject PyCapsule_Type = { .tp_dealloc = capsule_dealloc, .tp_repr = capsule_repr, .tp_doc = PyCapsule_Type__doc__, - .tp_traverse = (traverseproc)capsule_traverse, - .tp_clear = (inquiry)capsule_clear, + .tp_traverse = capsule_traverse, + .tp_clear = capsule_clear, }; diff --git a/Objects/cellobject.c b/Objects/cellobject.c index f1a43be38b2b58..ec2eeb1a855b63 100644 --- a/Objects/cellobject.c +++ b/Objects/cellobject.c @@ -1,9 +1,12 @@ /* Cell object implementation */ #include "Python.h" +#include "pycore_cell.h" // PyCell_GetRef() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" +#define _PyCell_CAST(op) _Py_CAST(PyCellObject*, (op)) + PyObject * PyCell_New(PyObject *obj) { @@ -56,8 +59,7 @@ PyCell_Get(PyObject *op) PyErr_BadInternalCall(); return NULL; } - PyObject *value = PyCell_GET(op); - return Py_XNewRef(value); + return PyCell_GetRef((PyCellObject *)op); } int @@ -67,20 +69,30 @@ PyCell_Set(PyObject *op, PyObject *value) PyErr_BadInternalCall(); return -1; } - PyObject *old_value = PyCell_GET(op); - PyCell_SET(op, Py_XNewRef(value)); - Py_XDECREF(old_value); + PyCell_SetTakeRef((PyCellObject *)op, Py_XNewRef(value)); return 0; } static void -cell_dealloc(PyCellObject *op) +cell_dealloc(PyObject *self) { + PyCellObject *op = _PyCell_CAST(self); _PyObject_GC_UNTRACK(op); Py_XDECREF(op->ob_ref); PyObject_GC_Del(op); } +static PyObject * +cell_compare_impl(PyObject *a, PyObject *b, int op) +{ + if (a != NULL && b != NULL) { + return PyObject_RichCompare(a, b, op); + } + else { + Py_RETURN_RICHCOMPARE(b == NULL, a == NULL, op); + } +} + static PyObject * cell_richcompare(PyObject *a, PyObject *b, int op) { @@ -91,62 +103,69 @@ cell_richcompare(PyObject *a, PyObject *b, int op) if (!PyCell_Check(a) || !PyCell_Check(b)) { Py_RETURN_NOTIMPLEMENTED; } + PyObject *a_ref = PyCell_GetRef((PyCellObject *)a); + PyObject *b_ref = PyCell_GetRef((PyCellObject *)b); /* compare cells by contents; empty cells come before anything else */ - a = ((PyCellObject *)a)->ob_ref; - b = ((PyCellObject *)b)->ob_ref; - if (a != NULL && b != NULL) - return PyObject_RichCompare(a, b, op); + PyObject *res = cell_compare_impl(a_ref, b_ref, op); - Py_RETURN_RICHCOMPARE(b == NULL, a == NULL, op); + Py_XDECREF(a_ref); + Py_XDECREF(b_ref); + return res; } static PyObject * -cell_repr(PyCellObject *op) +cell_repr(PyObject *self) { - if (op->ob_ref == NULL) - return PyUnicode_FromFormat("", op); - - return PyUnicode_FromFormat("", - op, Py_TYPE(op->ob_ref)->tp_name, - op->ob_ref); + PyObject *ref = PyCell_GetRef((PyCellObject *)self); + if (ref == NULL) { + return PyUnicode_FromFormat("", self); + } + PyObject *res = PyUnicode_FromFormat("", + self, Py_TYPE(ref)->tp_name, ref); + Py_DECREF(ref); + return res; } static int -cell_traverse(PyCellObject *op, visitproc visit, void *arg) +cell_traverse(PyObject *self, visitproc visit, void *arg) { + PyCellObject *op = _PyCell_CAST(self); Py_VISIT(op->ob_ref); return 0; } static int -cell_clear(PyCellObject *op) +cell_clear(PyObject *self) { + PyCellObject *op = _PyCell_CAST(self); Py_CLEAR(op->ob_ref); return 0; } static PyObject * -cell_get_contents(PyCellObject *op, void *closure) +cell_get_contents(PyObject *self, void *closure) { - if (op->ob_ref == NULL) - { + PyCellObject *op = _PyCell_CAST(self); + PyObject *res = PyCell_GetRef(op); + if (res == NULL) { PyErr_SetString(PyExc_ValueError, "Cell is empty"); return NULL; } - return Py_NewRef(op->ob_ref); + return res; } static int -cell_set_contents(PyCellObject *op, PyObject *obj, void *Py_UNUSED(ignored)) +cell_set_contents(PyObject *self, PyObject *obj, void *Py_UNUSED(ignored)) { - Py_XSETREF(op->ob_ref, Py_XNewRef(obj)); + PyCellObject *cell = _PyCell_CAST(self); + Py_XINCREF(obj); + PyCell_SetTakeRef((PyCellObject *)cell, obj); return 0; } static PyGetSetDef cell_getsetlist[] = { - {"cell_contents", (getter)cell_get_contents, - (setter)cell_set_contents, NULL}, + {"cell_contents", cell_get_contents, cell_set_contents, NULL}, {NULL} /* sentinel */ }; @@ -155,12 +174,12 @@ PyTypeObject PyCell_Type = { "cell", sizeof(PyCellObject), 0, - (destructor)cell_dealloc, /* tp_dealloc */ + cell_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)cell_repr, /* tp_repr */ + cell_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -172,8 +191,8 @@ PyTypeObject PyCell_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ cell_new_doc, /* tp_doc */ - (traverseproc)cell_traverse, /* tp_traverse */ - (inquiry)cell_clear, /* tp_clear */ + cell_traverse, /* tp_traverse */ + cell_clear, /* tp_clear */ cell_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ @@ -188,6 +207,6 @@ PyTypeObject PyCell_Type = { 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ - (newfunc)cell_new, /* tp_new */ + cell_new, /* tp_new */ 0, /* tp_free */ }; diff --git a/Objects/classobject.c b/Objects/classobject.c index d7e520f556d9a0..58e1d17977322e 100644 --- a/Objects/classobject.c +++ b/Objects/classobject.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_VectorcallTstate() #include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_freelist.h" #include "pycore_object.h" #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() @@ -10,6 +11,7 @@ #include "clinic/classobject.c.h" +#define _PyMethodObject_CAST(op) _Py_CAST(PyMethodObject*, (op)) #define TP_DESCR_GET(t) ((t)->tp_descr_get) /*[clinic input] @@ -111,9 +113,12 @@ PyMethod_New(PyObject *func, PyObject *self) PyErr_BadInternalCall(); return NULL; } - PyMethodObject *im = PyObject_GC_New(PyMethodObject, &PyMethod_Type); + PyMethodObject *im = _Py_FREELIST_POP(PyMethodObject, pymethodobjects); if (im == NULL) { - return NULL; + im = PyObject_GC_New(PyMethodObject, &PyMethod_Type); + if (im == NULL) { + return NULL; + } } im->im_weakreflist = NULL; im->im_func = Py_NewRef(func); @@ -166,13 +171,14 @@ static PyMemberDef method_memberlist[] = { should only be used for the class, not for instances */ static PyObject * -method_get_doc(PyMethodObject *im, void *context) +method_get_doc(PyObject *self, void *context) { + PyMethodObject *im = _PyMethodObject_CAST(self); return PyObject_GetAttr(im->im_func, &_Py_ID(__doc__)); } static PyGetSetDef method_getset[] = { - {"__doc__", (getter)method_get_doc, NULL, NULL}, + {"__doc__", method_get_doc, NULL, NULL}, {0} }; @@ -188,15 +194,18 @@ method_getattro(PyObject *obj, PyObject *name) if (PyType_Ready(tp) < 0) return NULL; } - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); } if (descr != NULL) { descrgetfunc f = TP_DESCR_GET(Py_TYPE(descr)); - if (f != NULL) - return f(descr, obj, (PyObject *)Py_TYPE(obj)); + if (f != NULL) { + PyObject *res = f(descr, obj, (PyObject *)Py_TYPE(obj)); + Py_DECREF(descr); + return res; + } else { - return Py_NewRef(descr); + return descr; } } @@ -232,14 +241,16 @@ method_new_impl(PyTypeObject *type, PyObject *function, PyObject *instance) } static void -method_dealloc(PyMethodObject *im) +method_dealloc(PyObject *self) { + PyMethodObject *im = _PyMethodObject_CAST(self); _PyObject_GC_UNTRACK(im); if (im->im_weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *)im); Py_DECREF(im->im_func); Py_XDECREF(im->im_self); - PyObject_GC_Del(im); + assert(Py_IS_TYPE(self, &PyMethod_Type)); + _Py_FREELIST_FREE(pymethodobjects, (PyObject *)im, PyObject_GC_Del); } static PyObject * @@ -271,8 +282,9 @@ method_richcompare(PyObject *self, PyObject *other, int op) } static PyObject * -method_repr(PyMethodObject *a) +method_repr(PyObject *op) { + PyMethodObject *a = _PyMethodObject_CAST(op); PyObject *self = a->im_self; PyObject *func = a->im_func; PyObject *funcname, *result; @@ -298,22 +310,26 @@ method_repr(PyMethodObject *a) } static Py_hash_t -method_hash(PyMethodObject *a) +method_hash(PyObject *self) { - Py_hash_t x, y; - x = _Py_HashPointer(a->im_self); - y = PyObject_Hash(a->im_func); - if (y == -1) + PyMethodObject *a = _PyMethodObject_CAST(self); + Py_hash_t x = PyObject_GenericHash(a->im_self); + Py_hash_t y = PyObject_Hash(a->im_func); + if (y == -1) { return -1; + } + x = x ^ y; - if (x == -1) + if (x == -1) { x = -2; + } return x; } static int -method_traverse(PyMethodObject *im, visitproc visit, void *arg) +method_traverse(PyObject *self, visitproc visit, void *arg) { + PyMethodObject *im = _PyMethodObject_CAST(self); Py_VISIT(im->im_func); Py_VISIT(im->im_self); return 0; @@ -330,17 +346,17 @@ PyTypeObject PyMethod_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "method", .tp_basicsize = sizeof(PyMethodObject), - .tp_dealloc = (destructor)method_dealloc, + .tp_dealloc = method_dealloc, .tp_vectorcall_offset = offsetof(PyMethodObject, vectorcall), - .tp_repr = (reprfunc)method_repr, - .tp_hash = (hashfunc)method_hash, + .tp_repr = method_repr, + .tp_hash = method_hash, .tp_call = PyVectorcall_Call, .tp_getattro = method_getattro, .tp_setattro = PyObject_GenericSetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_VECTORCALL, .tp_doc = method_new__doc__, - .tp_traverse = (traverseproc)method_traverse, + .tp_traverse = method_traverse, .tp_richcompare = method_richcompare, .tp_weaklistoffset = offsetof(PyMethodObject, im_weakreflist), .tp_methods = method_methods, @@ -396,7 +412,7 @@ instancemethod_get_doc(PyObject *self, void *context) } static PyGetSetDef instancemethod_getset[] = { - {"__doc__", (getter)instancemethod_get_doc, NULL, NULL}, + {"__doc__", instancemethod_get_doc, NULL, NULL}, {0} }; @@ -410,14 +426,17 @@ instancemethod_getattro(PyObject *self, PyObject *name) if (PyType_Ready(tp) < 0) return NULL; } - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); if (descr != NULL) { descrgetfunc f = TP_DESCR_GET(Py_TYPE(descr)); - if (f != NULL) - return f(descr, self, (PyObject *)Py_TYPE(self)); + if (f != NULL) { + PyObject *res = f(descr, self, (PyObject *)Py_TYPE(self)); + Py_DECREF(descr); + return res; + } else { - return Py_NewRef(descr); + return descr; } } @@ -531,7 +550,7 @@ PyTypeObject PyInstanceMethod_Type = { .tp_name = "instancemethod", .tp_basicsize = sizeof(PyInstanceMethodObject), .tp_dealloc = instancemethod_dealloc, - .tp_repr = (reprfunc)instancemethod_repr, + .tp_repr = instancemethod_repr, .tp_call = instancemethod_call, .tp_getattro = instancemethod_getattro, .tp_setattro = PyObject_GenericSetAttr, diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index d95245067e2608..91cf5363e639d1 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -50,7 +50,8 @@ bytearray___init__(PyObject *self, PyObject *args, PyObject *kwargs) const char *encoding = NULL; const char *errors = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 3, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -101,6 +102,106 @@ bytearray___init__(PyObject *self, PyObject *args, PyObject *kwargs) return return_value; } +PyDoc_STRVAR(bytearray_find__doc__, +"find($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the lowest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start:end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Return -1 on failure."); + +#define BYTEARRAY_FIND_METHODDEF \ + {"find", _PyCFunction_CAST(bytearray_find), METH_FASTCALL, bytearray_find__doc__}, + +static PyObject * +bytearray_find_impl(PyByteArrayObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytearray_find(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("find", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_find_impl((PyByteArrayObject *)self, sub, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytearray_count__doc__, +"count($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the number of non-overlapping occurrences of subsection \'sub\' in bytes B[start:end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes."); + +#define BYTEARRAY_COUNT_METHODDEF \ + {"count", _PyCFunction_CAST(bytearray_count), METH_FASTCALL, bytearray_count__doc__}, + +static PyObject * +bytearray_count_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytearray_count(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("count", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_count_impl((PyByteArrayObject *)self, sub, start, end); + +exit: + return return_value; +} + PyDoc_STRVAR(bytearray_clear__doc__, "clear($self, /)\n" "--\n" @@ -114,9 +215,9 @@ static PyObject * bytearray_clear_impl(PyByteArrayObject *self); static PyObject * -bytearray_clear(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored)) +bytearray_clear(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return bytearray_clear_impl(self); + return bytearray_clear_impl((PyByteArrayObject *)self); } PyDoc_STRVAR(bytearray_copy__doc__, @@ -132,9 +233,264 @@ static PyObject * bytearray_copy_impl(PyByteArrayObject *self); static PyObject * -bytearray_copy(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored)) +bytearray_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return bytearray_copy_impl((PyByteArrayObject *)self); +} + +PyDoc_STRVAR(bytearray_index__doc__, +"index($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the lowest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start:end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Raise ValueError if the subsection is not found."); + +#define BYTEARRAY_INDEX_METHODDEF \ + {"index", _PyCFunction_CAST(bytearray_index), METH_FASTCALL, bytearray_index__doc__}, + +static PyObject * +bytearray_index_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytearray_index(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("index", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_index_impl((PyByteArrayObject *)self, sub, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytearray_rfind__doc__, +"rfind($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the highest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start:end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Return -1 on failure."); + +#define BYTEARRAY_RFIND_METHODDEF \ + {"rfind", _PyCFunction_CAST(bytearray_rfind), METH_FASTCALL, bytearray_rfind__doc__}, + +static PyObject * +bytearray_rfind_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytearray_rfind(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("rfind", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_rfind_impl((PyByteArrayObject *)self, sub, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytearray_rindex__doc__, +"rindex($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the highest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start:end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Raise ValueError if the subsection is not found."); + +#define BYTEARRAY_RINDEX_METHODDEF \ + {"rindex", _PyCFunction_CAST(bytearray_rindex), METH_FASTCALL, bytearray_rindex__doc__}, + +static PyObject * +bytearray_rindex_impl(PyByteArrayObject *self, PyObject *sub, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytearray_rindex(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("rindex", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_rindex_impl((PyByteArrayObject *)self, sub, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytearray_startswith__doc__, +"startswith($self, prefix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the bytearray starts with the specified prefix, False otherwise.\n" +"\n" +" prefix\n" +" A bytes or a tuple of bytes to try.\n" +" start\n" +" Optional start position. Default: start of the bytearray.\n" +" end\n" +" Optional stop position. Default: end of the bytearray."); + +#define BYTEARRAY_STARTSWITH_METHODDEF \ + {"startswith", _PyCFunction_CAST(bytearray_startswith), METH_FASTCALL, bytearray_startswith__doc__}, + +static PyObject * +bytearray_startswith_impl(PyByteArrayObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytearray_startswith(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("startswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_startswith_impl((PyByteArrayObject *)self, subobj, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytearray_endswith__doc__, +"endswith($self, suffix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the bytearray ends with the specified suffix, False otherwise.\n" +"\n" +" suffix\n" +" A bytes or a tuple of bytes to try.\n" +" start\n" +" Optional start position. Default: start of the bytearray.\n" +" end\n" +" Optional stop position. Default: end of the bytearray."); + +#define BYTEARRAY_ENDSWITH_METHODDEF \ + {"endswith", _PyCFunction_CAST(bytearray_endswith), METH_FASTCALL, bytearray_endswith__doc__}, + +static PyObject * +bytearray_endswith_impl(PyByteArrayObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytearray_endswith(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { - return bytearray_copy_impl(self); + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("endswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytearray_endswith_impl((PyByteArrayObject *)self, subobj, start, end); + +exit: + return return_value; } PyDoc_STRVAR(bytearray_removeprefix__doc__, @@ -154,7 +510,7 @@ static PyObject * bytearray_removeprefix_impl(PyByteArrayObject *self, Py_buffer *prefix); static PyObject * -bytearray_removeprefix(PyByteArrayObject *self, PyObject *arg) +bytearray_removeprefix(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; Py_buffer prefix = {NULL, NULL}; @@ -162,7 +518,7 @@ bytearray_removeprefix(PyByteArrayObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &prefix, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = bytearray_removeprefix_impl(self, &prefix); + return_value = bytearray_removeprefix_impl((PyByteArrayObject *)self, &prefix); exit: /* Cleanup for prefix */ @@ -190,7 +546,7 @@ static PyObject * bytearray_removesuffix_impl(PyByteArrayObject *self, Py_buffer *suffix); static PyObject * -bytearray_removesuffix(PyByteArrayObject *self, PyObject *arg) +bytearray_removesuffix(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; Py_buffer suffix = {NULL, NULL}; @@ -198,7 +554,7 @@ bytearray_removesuffix(PyByteArrayObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &suffix, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = bytearray_removesuffix_impl(self, &suffix); + return_value = bytearray_removesuffix_impl((PyByteArrayObject *)self, &suffix); exit: /* Cleanup for suffix */ @@ -229,7 +585,7 @@ bytearray_translate_impl(PyByteArrayObject *self, PyObject *table, PyObject *deletechars); static PyObject * -bytearray_translate(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +bytearray_translate(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -262,7 +618,8 @@ bytearray_translate(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t n PyObject *table; PyObject *deletechars = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -272,7 +629,7 @@ bytearray_translate(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t n } deletechars = args[1]; skip_optional_pos: - return_value = bytearray_translate_impl(self, table, deletechars); + return_value = bytearray_translate_impl((PyByteArrayObject *)self, table, deletechars); exit: return return_value; @@ -347,7 +704,7 @@ bytearray_replace_impl(PyByteArrayObject *self, Py_buffer *old, Py_buffer *new, Py_ssize_t count); static PyObject * -bytearray_replace(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +bytearray_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_buffer old = {NULL, NULL}; @@ -379,7 +736,7 @@ bytearray_replace(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nar count = ival; } skip_optional: - return_value = bytearray_replace_impl(self, &old, &new, count); + return_value = bytearray_replace_impl((PyByteArrayObject *)self, &old, &new, count); exit: /* Cleanup for old */ @@ -416,7 +773,7 @@ bytearray_split_impl(PyByteArrayObject *self, PyObject *sep, Py_ssize_t maxsplit); static PyObject * -bytearray_split(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +bytearray_split(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -449,7 +806,8 @@ bytearray_split(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs PyObject *sep = Py_None; Py_ssize_t maxsplit = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -475,7 +833,7 @@ bytearray_split(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs maxsplit = ival; } skip_optional_pos: - return_value = bytearray_split_impl(self, sep, maxsplit); + return_value = bytearray_split_impl((PyByteArrayObject *)self, sep, maxsplit); exit: return return_value; @@ -538,7 +896,7 @@ bytearray_rsplit_impl(PyByteArrayObject *self, PyObject *sep, Py_ssize_t maxsplit); static PyObject * -bytearray_rsplit(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +bytearray_rsplit(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -571,7 +929,8 @@ bytearray_rsplit(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t narg PyObject *sep = Py_None; Py_ssize_t maxsplit = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -597,7 +956,7 @@ bytearray_rsplit(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t narg maxsplit = ival; } skip_optional_pos: - return_value = bytearray_rsplit_impl(self, sep, maxsplit); + return_value = bytearray_rsplit_impl((PyByteArrayObject *)self, sep, maxsplit); exit: return return_value; @@ -616,9 +975,9 @@ static PyObject * bytearray_reverse_impl(PyByteArrayObject *self); static PyObject * -bytearray_reverse(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored)) +bytearray_reverse(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return bytearray_reverse_impl(self); + return bytearray_reverse_impl((PyByteArrayObject *)self); } PyDoc_STRVAR(bytearray_insert__doc__, @@ -639,7 +998,7 @@ static PyObject * bytearray_insert_impl(PyByteArrayObject *self, Py_ssize_t index, int item); static PyObject * -bytearray_insert(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +bytearray_insert(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t index; @@ -663,7 +1022,7 @@ bytearray_insert(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t narg if (!_getbytevalue(args[1], &item)) { goto exit; } - return_value = bytearray_insert_impl(self, index, item); + return_value = bytearray_insert_impl((PyByteArrayObject *)self, index, item); exit: return return_value; @@ -685,7 +1044,7 @@ static PyObject * bytearray_append_impl(PyByteArrayObject *self, int item); static PyObject * -bytearray_append(PyByteArrayObject *self, PyObject *arg) +bytearray_append(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; int item; @@ -693,7 +1052,7 @@ bytearray_append(PyByteArrayObject *self, PyObject *arg) if (!_getbytevalue(arg, &item)) { goto exit; } - return_value = bytearray_append_impl(self, item); + return_value = bytearray_append_impl((PyByteArrayObject *)self, item); exit: return return_value; @@ -730,7 +1089,7 @@ static PyObject * bytearray_pop_impl(PyByteArrayObject *self, Py_ssize_t index); static PyObject * -bytearray_pop(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +bytearray_pop(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t index = -1; @@ -754,7 +1113,7 @@ bytearray_pop(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) index = ival; } skip_optional: - return_value = bytearray_pop_impl(self, index); + return_value = bytearray_pop_impl((PyByteArrayObject *)self, index); exit: return return_value; @@ -776,7 +1135,7 @@ static PyObject * bytearray_remove_impl(PyByteArrayObject *self, int value); static PyObject * -bytearray_remove(PyByteArrayObject *self, PyObject *arg) +bytearray_remove(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; int value; @@ -784,7 +1143,7 @@ bytearray_remove(PyByteArrayObject *self, PyObject *arg) if (!_getbytevalue(arg, &value)) { goto exit; } - return_value = bytearray_remove_impl(self, value); + return_value = bytearray_remove_impl((PyByteArrayObject *)self, value); exit: return return_value; @@ -805,7 +1164,7 @@ static PyObject * bytearray_strip_impl(PyByteArrayObject *self, PyObject *bytes); static PyObject * -bytearray_strip(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +bytearray_strip(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *bytes = Py_None; @@ -818,7 +1177,7 @@ bytearray_strip(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs } bytes = args[0]; skip_optional: - return_value = bytearray_strip_impl(self, bytes); + return_value = bytearray_strip_impl((PyByteArrayObject *)self, bytes); exit: return return_value; @@ -839,7 +1198,7 @@ static PyObject * bytearray_lstrip_impl(PyByteArrayObject *self, PyObject *bytes); static PyObject * -bytearray_lstrip(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +bytearray_lstrip(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *bytes = Py_None; @@ -852,7 +1211,7 @@ bytearray_lstrip(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t narg } bytes = args[0]; skip_optional: - return_value = bytearray_lstrip_impl(self, bytes); + return_value = bytearray_lstrip_impl((PyByteArrayObject *)self, bytes); exit: return return_value; @@ -873,7 +1232,7 @@ static PyObject * bytearray_rstrip_impl(PyByteArrayObject *self, PyObject *bytes); static PyObject * -bytearray_rstrip(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +bytearray_rstrip(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *bytes = Py_None; @@ -886,7 +1245,7 @@ bytearray_rstrip(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t narg } bytes = args[0]; skip_optional: - return_value = bytearray_rstrip_impl(self, bytes); + return_value = bytearray_rstrip_impl((PyByteArrayObject *)self, bytes); exit: return return_value; @@ -915,7 +1274,7 @@ bytearray_decode_impl(PyByteArrayObject *self, const char *encoding, const char *errors); static PyObject * -bytearray_decode(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +bytearray_decode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -948,7 +1307,8 @@ bytearray_decode(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t narg const char *encoding = NULL; const char *errors = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -987,7 +1347,7 @@ bytearray_decode(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t narg goto exit; } skip_optional_pos: - return_value = bytearray_decode_impl(self, encoding, errors); + return_value = bytearray_decode_impl((PyByteArrayObject *)self, encoding, errors); exit: return return_value; @@ -1022,7 +1382,7 @@ static PyObject * bytearray_splitlines_impl(PyByteArrayObject *self, int keepends); static PyObject * -bytearray_splitlines(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +bytearray_splitlines(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -1054,7 +1414,8 @@ bytearray_splitlines(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; int keepends = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1066,7 +1427,7 @@ bytearray_splitlines(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t goto exit; } skip_optional_pos: - return_value = bytearray_splitlines_impl(self, keepends); + return_value = bytearray_splitlines_impl((PyByteArrayObject *)self, keepends); exit: return return_value; @@ -1134,7 +1495,7 @@ static PyObject * bytearray_hex_impl(PyByteArrayObject *self, PyObject *sep, int bytes_per_sep); static PyObject * -bytearray_hex(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +bytearray_hex(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -1167,7 +1528,8 @@ bytearray_hex(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *sep = NULL; int bytes_per_sep = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1185,7 +1547,7 @@ bytearray_hex(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs, goto exit; } skip_optional_pos: - return_value = bytearray_hex_impl(self, sep, bytes_per_sep); + return_value = bytearray_hex_impl((PyByteArrayObject *)self, sep, bytes_per_sep); exit: return return_value; @@ -1204,9 +1566,9 @@ static PyObject * bytearray_reduce_impl(PyByteArrayObject *self); static PyObject * -bytearray_reduce(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored)) +bytearray_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return bytearray_reduce_impl(self); + return bytearray_reduce_impl((PyByteArrayObject *)self); } PyDoc_STRVAR(bytearray_reduce_ex__doc__, @@ -1222,7 +1584,7 @@ static PyObject * bytearray_reduce_ex_impl(PyByteArrayObject *self, int proto); static PyObject * -bytearray_reduce_ex(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t nargs) +bytearray_reduce_ex(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int proto = 0; @@ -1238,7 +1600,7 @@ bytearray_reduce_ex(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t n goto exit; } skip_optional: - return_value = bytearray_reduce_ex_impl(self, proto); + return_value = bytearray_reduce_ex_impl((PyByteArrayObject *)self, proto); exit: return return_value; @@ -1257,8 +1619,8 @@ static PyObject * bytearray_sizeof_impl(PyByteArrayObject *self); static PyObject * -bytearray_sizeof(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored)) +bytearray_sizeof(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return bytearray_sizeof_impl(self); + return bytearray_sizeof_impl((PyByteArrayObject *)self); } -/*[clinic end generated code: output=0797a5e03cda2a16 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bc8bec8514102bf3 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/bytesobject.c.h b/Objects/clinic/bytesobject.c.h index 1e45be3e7aefb3..9aef736428ad0e 100644 --- a/Objects/clinic/bytesobject.c.h +++ b/Objects/clinic/bytesobject.c.h @@ -22,9 +22,9 @@ static PyObject * bytes___bytes___impl(PyBytesObject *self); static PyObject * -bytes___bytes__(PyBytesObject *self, PyObject *Py_UNUSED(ignored)) +bytes___bytes__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return bytes___bytes___impl(self); + return bytes___bytes___impl((PyBytesObject *)self); } PyDoc_STRVAR(bytes_split__doc__, @@ -48,7 +48,7 @@ static PyObject * bytes_split_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t maxsplit); static PyObject * -bytes_split(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +bytes_split(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -81,7 +81,8 @@ bytes_split(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *sep = Py_None; Py_ssize_t maxsplit = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -107,7 +108,7 @@ bytes_split(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje maxsplit = ival; } skip_optional_pos: - return_value = bytes_split_impl(self, sep, maxsplit); + return_value = bytes_split_impl((PyBytesObject *)self, sep, maxsplit); exit: return return_value; @@ -133,7 +134,7 @@ static PyObject * bytes_partition_impl(PyBytesObject *self, Py_buffer *sep); static PyObject * -bytes_partition(PyBytesObject *self, PyObject *arg) +bytes_partition(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; Py_buffer sep = {NULL, NULL}; @@ -141,7 +142,7 @@ bytes_partition(PyBytesObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &sep, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = bytes_partition_impl(self, &sep); + return_value = bytes_partition_impl((PyBytesObject *)self, &sep); exit: /* Cleanup for sep */ @@ -172,7 +173,7 @@ static PyObject * bytes_rpartition_impl(PyBytesObject *self, Py_buffer *sep); static PyObject * -bytes_rpartition(PyBytesObject *self, PyObject *arg) +bytes_rpartition(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; Py_buffer sep = {NULL, NULL}; @@ -180,7 +181,7 @@ bytes_rpartition(PyBytesObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &sep, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = bytes_rpartition_impl(self, &sep); + return_value = bytes_rpartition_impl((PyBytesObject *)self, &sep); exit: /* Cleanup for sep */ @@ -214,7 +215,7 @@ static PyObject * bytes_rsplit_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t maxsplit); static PyObject * -bytes_rsplit(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +bytes_rsplit(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -247,7 +248,8 @@ bytes_rsplit(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *sep = Py_None; Py_ssize_t maxsplit = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -273,7 +275,7 @@ bytes_rsplit(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, PyObj maxsplit = ival; } skip_optional_pos: - return_value = bytes_rsplit_impl(self, sep, maxsplit); + return_value = bytes_rsplit_impl((PyBytesObject *)self, sep, maxsplit); exit: return return_value; @@ -294,6 +296,210 @@ PyDoc_STRVAR(bytes_join__doc__, #define BYTES_JOIN_METHODDEF \ {"join", (PyCFunction)bytes_join, METH_O, bytes_join__doc__}, +PyDoc_STRVAR(bytes_find__doc__, +"find($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the lowest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start,end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Return -1 on failure."); + +#define BYTES_FIND_METHODDEF \ + {"find", _PyCFunction_CAST(bytes_find), METH_FASTCALL, bytes_find__doc__}, + +static PyObject * +bytes_find_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytes_find(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("find", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_find_impl((PyBytesObject *)self, sub, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytes_index__doc__, +"index($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the lowest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start,end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Raise ValueError if the subsection is not found."); + +#define BYTES_INDEX_METHODDEF \ + {"index", _PyCFunction_CAST(bytes_index), METH_FASTCALL, bytes_index__doc__}, + +static PyObject * +bytes_index_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytes_index(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("index", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_index_impl((PyBytesObject *)self, sub, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytes_rfind__doc__, +"rfind($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the highest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start,end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Return -1 on failure."); + +#define BYTES_RFIND_METHODDEF \ + {"rfind", _PyCFunction_CAST(bytes_rfind), METH_FASTCALL, bytes_rfind__doc__}, + +static PyObject * +bytes_rfind_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytes_rfind(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("rfind", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_rfind_impl((PyBytesObject *)self, sub, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytes_rindex__doc__, +"rindex($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the highest index in B where subsection \'sub\' is found, such that \'sub\' is contained within B[start,end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes.\n" +"\n" +"Raise ValueError if the subsection is not found."); + +#define BYTES_RINDEX_METHODDEF \ + {"rindex", _PyCFunction_CAST(bytes_rindex), METH_FASTCALL, bytes_rindex__doc__}, + +static PyObject * +bytes_rindex_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytes_rindex(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("rindex", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_rindex_impl((PyBytesObject *)self, sub, start, end); + +exit: + return return_value; +} + PyDoc_STRVAR(bytes_strip__doc__, "strip($self, bytes=None, /)\n" "--\n" @@ -309,7 +515,7 @@ static PyObject * bytes_strip_impl(PyBytesObject *self, PyObject *bytes); static PyObject * -bytes_strip(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) +bytes_strip(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *bytes = Py_None; @@ -322,7 +528,7 @@ bytes_strip(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) } bytes = args[0]; skip_optional: - return_value = bytes_strip_impl(self, bytes); + return_value = bytes_strip_impl((PyBytesObject *)self, bytes); exit: return return_value; @@ -343,7 +549,7 @@ static PyObject * bytes_lstrip_impl(PyBytesObject *self, PyObject *bytes); static PyObject * -bytes_lstrip(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) +bytes_lstrip(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *bytes = Py_None; @@ -356,7 +562,7 @@ bytes_lstrip(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) } bytes = args[0]; skip_optional: - return_value = bytes_lstrip_impl(self, bytes); + return_value = bytes_lstrip_impl((PyBytesObject *)self, bytes); exit: return return_value; @@ -377,7 +583,7 @@ static PyObject * bytes_rstrip_impl(PyBytesObject *self, PyObject *bytes); static PyObject * -bytes_rstrip(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) +bytes_rstrip(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *bytes = Py_None; @@ -390,7 +596,56 @@ bytes_rstrip(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) } bytes = args[0]; skip_optional: - return_value = bytes_rstrip_impl(self, bytes); + return_value = bytes_rstrip_impl((PyBytesObject *)self, bytes); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytes_count__doc__, +"count($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the number of non-overlapping occurrences of subsection \'sub\' in bytes B[start:end].\n" +"\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes."); + +#define BYTES_COUNT_METHODDEF \ + {"count", _PyCFunction_CAST(bytes_count), METH_FASTCALL, bytes_count__doc__}, + +static PyObject * +bytes_count_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytes_count(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *sub; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("count", nargs, 1, 3)) { + goto exit; + } + sub = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_count_impl((PyBytesObject *)self, sub, start, end); exit: return return_value; @@ -416,7 +671,7 @@ bytes_translate_impl(PyBytesObject *self, PyObject *table, PyObject *deletechars); static PyObject * -bytes_translate(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +bytes_translate(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -449,7 +704,8 @@ bytes_translate(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, Py PyObject *table; PyObject *deletechars = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -459,7 +715,7 @@ bytes_translate(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, Py } deletechars = args[1]; skip_optional_pos: - return_value = bytes_translate_impl(self, table, deletechars); + return_value = bytes_translate_impl((PyBytesObject *)self, table, deletechars); exit: return return_value; @@ -534,7 +790,7 @@ bytes_replace_impl(PyBytesObject *self, Py_buffer *old, Py_buffer *new, Py_ssize_t count); static PyObject * -bytes_replace(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) +bytes_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_buffer old = {NULL, NULL}; @@ -566,7 +822,7 @@ bytes_replace(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs) count = ival; } skip_optional: - return_value = bytes_replace_impl(self, &old, &new, count); + return_value = bytes_replace_impl((PyBytesObject *)self, &old, &new, count); exit: /* Cleanup for old */ @@ -597,7 +853,7 @@ static PyObject * bytes_removeprefix_impl(PyBytesObject *self, Py_buffer *prefix); static PyObject * -bytes_removeprefix(PyBytesObject *self, PyObject *arg) +bytes_removeprefix(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; Py_buffer prefix = {NULL, NULL}; @@ -605,7 +861,7 @@ bytes_removeprefix(PyBytesObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &prefix, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = bytes_removeprefix_impl(self, &prefix); + return_value = bytes_removeprefix_impl((PyBytesObject *)self, &prefix); exit: /* Cleanup for prefix */ @@ -633,7 +889,7 @@ static PyObject * bytes_removesuffix_impl(PyBytesObject *self, Py_buffer *suffix); static PyObject * -bytes_removesuffix(PyBytesObject *self, PyObject *arg) +bytes_removesuffix(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; Py_buffer suffix = {NULL, NULL}; @@ -641,7 +897,7 @@ bytes_removesuffix(PyBytesObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &suffix, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = bytes_removesuffix_impl(self, &suffix); + return_value = bytes_removesuffix_impl((PyBytesObject *)self, &suffix); exit: /* Cleanup for suffix */ @@ -652,6 +908,108 @@ bytes_removesuffix(PyBytesObject *self, PyObject *arg) return return_value; } +PyDoc_STRVAR(bytes_startswith__doc__, +"startswith($self, prefix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the bytes starts with the specified prefix, False otherwise.\n" +"\n" +" prefix\n" +" A bytes or a tuple of bytes to try.\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes."); + +#define BYTES_STARTSWITH_METHODDEF \ + {"startswith", _PyCFunction_CAST(bytes_startswith), METH_FASTCALL, bytes_startswith__doc__}, + +static PyObject * +bytes_startswith_impl(PyBytesObject *self, PyObject *subobj, + Py_ssize_t start, Py_ssize_t end); + +static PyObject * +bytes_startswith(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("startswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_startswith_impl((PyBytesObject *)self, subobj, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(bytes_endswith__doc__, +"endswith($self, suffix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the bytes ends with the specified suffix, False otherwise.\n" +"\n" +" suffix\n" +" A bytes or a tuple of bytes to try.\n" +" start\n" +" Optional start position. Default: start of the bytes.\n" +" end\n" +" Optional stop position. Default: end of the bytes."); + +#define BYTES_ENDSWITH_METHODDEF \ + {"endswith", _PyCFunction_CAST(bytes_endswith), METH_FASTCALL, bytes_endswith__doc__}, + +static PyObject * +bytes_endswith_impl(PyBytesObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +bytes_endswith(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("endswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = bytes_endswith_impl((PyBytesObject *)self, subobj, start, end); + +exit: + return return_value; +} + PyDoc_STRVAR(bytes_decode__doc__, "decode($self, /, encoding=\'utf-8\', errors=\'strict\')\n" "--\n" @@ -675,7 +1033,7 @@ bytes_decode_impl(PyBytesObject *self, const char *encoding, const char *errors); static PyObject * -bytes_decode(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +bytes_decode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -708,7 +1066,8 @@ bytes_decode(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, PyObj const char *encoding = NULL; const char *errors = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -747,7 +1106,7 @@ bytes_decode(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, PyObj goto exit; } skip_optional_pos: - return_value = bytes_decode_impl(self, encoding, errors); + return_value = bytes_decode_impl((PyBytesObject *)self, encoding, errors); exit: return return_value; @@ -769,7 +1128,7 @@ static PyObject * bytes_splitlines_impl(PyBytesObject *self, int keepends); static PyObject * -bytes_splitlines(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +bytes_splitlines(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -801,7 +1160,8 @@ bytes_splitlines(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, P Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; int keepends = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -813,7 +1173,7 @@ bytes_splitlines(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, P goto exit; } skip_optional_pos: - return_value = bytes_splitlines_impl(self, keepends); + return_value = bytes_splitlines_impl((PyBytesObject *)self, keepends); exit: return return_value; @@ -881,7 +1241,7 @@ static PyObject * bytes_hex_impl(PyBytesObject *self, PyObject *sep, int bytes_per_sep); static PyObject * -bytes_hex(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +bytes_hex(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -914,7 +1274,8 @@ bytes_hex(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *sep = NULL; int bytes_per_sep = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -932,7 +1293,7 @@ bytes_hex(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject goto exit; } skip_optional_pos: - return_value = bytes_hex_impl(self, sep, bytes_per_sep); + return_value = bytes_hex_impl((PyBytesObject *)self, sep, bytes_per_sep); exit: return return_value; @@ -979,7 +1340,8 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) const char *encoding = NULL; const char *errors = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 3, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -1029,4 +1391,4 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=8a49dbbd78914a6f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=96fe2d6ef9ac8f6a input=a9049054013a1b77]*/ diff --git a/Objects/clinic/classobject.c.h b/Objects/clinic/classobject.c.h index 3e149c97324a6a..5934f1c2a41669 100644 --- a/Objects/clinic/classobject.c.h +++ b/Objects/clinic/classobject.c.h @@ -16,9 +16,9 @@ static PyObject * method___reduce___impl(PyMethodObject *self); static PyObject * -method___reduce__(PyMethodObject *self, PyObject *Py_UNUSED(ignored)) +method___reduce__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return method___reduce___impl(self); + return method___reduce___impl((PyMethodObject *)self); } PyDoc_STRVAR(method_new__doc__, @@ -82,4 +82,4 @@ instancemethod_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=5a5e3f2d0726f189 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ab546abf90aac94e input=a9049054013a1b77]*/ diff --git a/Objects/clinic/codeobject.c.h b/Objects/clinic/codeobject.c.h index 68e2d7f528c061..2184742cc0d99d 100644 --- a/Objects/clinic/codeobject.c.h +++ b/Objects/clinic/codeobject.c.h @@ -174,7 +174,7 @@ code_replace_impl(PyCodeObject *self, int co_argcount, PyObject *co_exceptiontable); static PyObject * -code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +code_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -204,26 +204,27 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje #undef KWTUPLE PyObject *argsbuf[18]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - int co_argcount = self->co_argcount; - int co_posonlyargcount = self->co_posonlyargcount; - int co_kwonlyargcount = self->co_kwonlyargcount; - int co_nlocals = self->co_nlocals; - int co_stacksize = self->co_stacksize; - int co_flags = self->co_flags; - int co_firstlineno = self->co_firstlineno; + int co_argcount = ((PyCodeObject *)self)->co_argcount; + int co_posonlyargcount = ((PyCodeObject *)self)->co_posonlyargcount; + int co_kwonlyargcount = ((PyCodeObject *)self)->co_kwonlyargcount; + int co_nlocals = ((PyCodeObject *)self)->co_nlocals; + int co_stacksize = ((PyCodeObject *)self)->co_stacksize; + int co_flags = ((PyCodeObject *)self)->co_flags; + int co_firstlineno = ((PyCodeObject *)self)->co_firstlineno; PyObject *co_code = NULL; - PyObject *co_consts = self->co_consts; - PyObject *co_names = self->co_names; + PyObject *co_consts = ((PyCodeObject *)self)->co_consts; + PyObject *co_names = ((PyCodeObject *)self)->co_names; PyObject *co_varnames = NULL; PyObject *co_freevars = NULL; PyObject *co_cellvars = NULL; - PyObject *co_filename = self->co_filename; - PyObject *co_name = self->co_name; - PyObject *co_qualname = self->co_qualname; - PyObject *co_linetable = self->co_linetable; - PyObject *co_exceptiontable = self->co_exceptiontable; + PyObject *co_filename = ((PyCodeObject *)self)->co_filename; + PyObject *co_name = ((PyCodeObject *)self)->co_name; + PyObject *co_qualname = ((PyCodeObject *)self)->co_qualname; + PyObject *co_linetable = ((PyCodeObject *)self)->co_linetable; + PyObject *co_exceptiontable = ((PyCodeObject *)self)->co_exceptiontable; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -399,7 +400,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje } co_exceptiontable = args[17]; skip_optional_kwonly: - return_value = code_replace_impl(self, co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, co_stacksize, co_flags, co_firstlineno, co_code, co_consts, co_names, co_varnames, co_freevars, co_cellvars, co_filename, co_name, co_qualname, co_linetable, co_exceptiontable); + return_value = code_replace_impl((PyCodeObject *)self, co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, co_stacksize, co_flags, co_firstlineno, co_code, co_consts, co_names, co_varnames, co_freevars, co_cellvars, co_filename, co_name, co_qualname, co_linetable, co_exceptiontable); exit: return return_value; @@ -420,7 +421,7 @@ static PyObject * code__varname_from_oparg_impl(PyCodeObject *self, int oparg); static PyObject * -code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +code__varname_from_oparg(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -451,7 +452,8 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n PyObject *argsbuf[1]; int oparg; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -459,9 +461,9 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n if (oparg == -1 && PyErr_Occurred()) { goto exit; } - return_value = code__varname_from_oparg_impl(self, oparg); + return_value = code__varname_from_oparg_impl((PyCodeObject *)self, oparg); exit: return return_value; } -/*[clinic end generated code: output=d604263a3ca72a0f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=73861c79e93aaee5 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/complexobject.c.h b/Objects/clinic/complexobject.c.h index 49b50304021f7b..e00da1d960c54d 100644 --- a/Objects/clinic/complexobject.c.h +++ b/Objects/clinic/complexobject.c.h @@ -21,9 +21,9 @@ static PyObject * complex_conjugate_impl(PyComplexObject *self); static PyObject * -complex_conjugate(PyComplexObject *self, PyObject *Py_UNUSED(ignored)) +complex_conjugate(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return complex_conjugate_impl(self); + return complex_conjugate_impl((PyComplexObject *)self); } PyDoc_STRVAR(complex___getnewargs____doc__, @@ -38,9 +38,9 @@ static PyObject * complex___getnewargs___impl(PyComplexObject *self); static PyObject * -complex___getnewargs__(PyComplexObject *self, PyObject *Py_UNUSED(ignored)) +complex___getnewargs__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return complex___getnewargs___impl(self); + return complex___getnewargs___impl((PyComplexObject *)self); } PyDoc_STRVAR(complex___format____doc__, @@ -56,7 +56,7 @@ static PyObject * complex___format___impl(PyComplexObject *self, PyObject *format_spec); static PyObject * -complex___format__(PyComplexObject *self, PyObject *arg) +complex___format__(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; PyObject *format_spec; @@ -66,7 +66,7 @@ complex___format__(PyComplexObject *self, PyObject *arg) goto exit; } format_spec = arg; - return_value = complex___format___impl(self, format_spec); + return_value = complex___format___impl((PyComplexObject *)self, format_spec); exit: return return_value; @@ -85,18 +85,21 @@ static PyObject * complex___complex___impl(PyComplexObject *self); static PyObject * -complex___complex__(PyComplexObject *self, PyObject *Py_UNUSED(ignored)) +complex___complex__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return complex___complex___impl(self); + return complex___complex___impl((PyComplexObject *)self); } PyDoc_STRVAR(complex_new__doc__, "complex(real=0, imag=0)\n" "--\n" "\n" -"Create a complex number from a real part and an optional imaginary part.\n" +"Create a complex number from a string or numbers.\n" "\n" -"This is equivalent to (real + imag*1j) where imag defaults to 0."); +"If a string is given, parse it as a complex number.\n" +"If a single number is given, convert it to a complex number.\n" +"If the \'real\' or \'imag\' arguments are given, create a complex number\n" +"with the specified real and imaginary components."); static PyObject * complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i); @@ -137,7 +140,8 @@ complex_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *r = NULL; PyObject *i = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -157,4 +161,13 @@ complex_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=04e6261649967b30 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(complex_from_number__doc__, +"from_number($type, number, /)\n" +"--\n" +"\n" +"Convert number to a complex floating-point number."); + +#define COMPLEX_FROM_NUMBER_METHODDEF \ + {"from_number", (PyCFunction)complex_from_number, METH_O|METH_CLASS, complex_from_number__doc__}, +/*[clinic end generated code: output=252cddef7f9169a0 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/descrobject.c.h b/Objects/clinic/descrobject.c.h index 02fb440d9c83af..2f96adb5094fe3 100644 --- a/Objects/clinic/descrobject.c.h +++ b/Objects/clinic/descrobject.c.h @@ -8,6 +8,12 @@ preserve #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +PyDoc_STRVAR(mappingproxy_new__doc__, +"mappingproxy(mapping)\n" +"--\n" +"\n" +"Read-only proxy of a mapping."); + static PyObject * mappingproxy_new_impl(PyTypeObject *type, PyObject *mapping); @@ -45,7 +51,8 @@ mappingproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t nargs = PyTuple_GET_SIZE(args); PyObject *mapping; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -135,7 +142,8 @@ property_init(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *fdel = NULL; PyObject *doc = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 4, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -167,4 +175,4 @@ property_init(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=a4664ccf3da10f5a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2f43f9297a36aa40 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/dictobject.c.h b/Objects/clinic/dictobject.c.h index fb46c4c64334f9..c66916bb33aa37 100644 --- a/Objects/clinic/dictobject.c.h +++ b/Objects/clinic/dictobject.c.h @@ -52,9 +52,9 @@ static PyObject * dict_copy_impl(PyDictObject *self); static PyObject * -dict_copy(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +dict_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return dict_copy_impl(self); + return dict_copy_impl((PyDictObject *)self); } PyDoc_STRVAR(dict___contains____doc__, @@ -79,7 +79,7 @@ static PyObject * dict_get_impl(PyDictObject *self, PyObject *key, PyObject *default_value); static PyObject * -dict_get(PyDictObject *self, PyObject *const *args, Py_ssize_t nargs) +dict_get(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *key; @@ -94,9 +94,7 @@ dict_get(PyDictObject *self, PyObject *const *args, Py_ssize_t nargs) } default_value = args[1]; skip_optional: - Py_BEGIN_CRITICAL_SECTION(self); - return_value = dict_get_impl(self, key, default_value); - Py_END_CRITICAL_SECTION(); + return_value = dict_get_impl((PyDictObject *)self, key, default_value); exit: return return_value; @@ -118,7 +116,7 @@ dict_setdefault_impl(PyDictObject *self, PyObject *key, PyObject *default_value); static PyObject * -dict_setdefault(PyDictObject *self, PyObject *const *args, Py_ssize_t nargs) +dict_setdefault(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *key; @@ -134,7 +132,7 @@ dict_setdefault(PyDictObject *self, PyObject *const *args, Py_ssize_t nargs) default_value = args[1]; skip_optional: Py_BEGIN_CRITICAL_SECTION(self); - return_value = dict_setdefault_impl(self, key, default_value); + return_value = dict_setdefault_impl((PyDictObject *)self, key, default_value); Py_END_CRITICAL_SECTION(); exit: @@ -154,9 +152,9 @@ static PyObject * dict_clear_impl(PyDictObject *self); static PyObject * -dict_clear(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +dict_clear(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return dict_clear_impl(self); + return dict_clear_impl((PyDictObject *)self); } PyDoc_STRVAR(dict_pop__doc__, @@ -175,7 +173,7 @@ static PyObject * dict_pop_impl(PyDictObject *self, PyObject *key, PyObject *default_value); static PyObject * -dict_pop(PyDictObject *self, PyObject *const *args, Py_ssize_t nargs) +dict_pop(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *key; @@ -190,7 +188,7 @@ dict_pop(PyDictObject *self, PyObject *const *args, Py_ssize_t nargs) } default_value = args[1]; skip_optional: - return_value = dict_pop_impl(self, key, default_value); + return_value = dict_pop_impl((PyDictObject *)self, key, default_value); exit: return return_value; @@ -212,12 +210,12 @@ static PyObject * dict_popitem_impl(PyDictObject *self); static PyObject * -dict_popitem(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +dict_popitem(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = dict_popitem_impl(self); + return_value = dict_popitem_impl((PyDictObject *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -236,9 +234,9 @@ static PyObject * dict___sizeof___impl(PyDictObject *self); static PyObject * -dict___sizeof__(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +dict___sizeof__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return dict___sizeof___impl(self); + return dict___sizeof___impl((PyDictObject *)self); } PyDoc_STRVAR(dict___reversed____doc__, @@ -254,9 +252,9 @@ static PyObject * dict___reversed___impl(PyDictObject *self); static PyObject * -dict___reversed__(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +dict___reversed__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return dict___reversed___impl(self); + return dict___reversed___impl((PyDictObject *)self); } PyDoc_STRVAR(dict_keys__doc__, @@ -272,9 +270,9 @@ static PyObject * dict_keys_impl(PyDictObject *self); static PyObject * -dict_keys(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +dict_keys(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return dict_keys_impl(self); + return dict_keys_impl((PyDictObject *)self); } PyDoc_STRVAR(dict_items__doc__, @@ -290,9 +288,9 @@ static PyObject * dict_items_impl(PyDictObject *self); static PyObject * -dict_items(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +dict_items(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return dict_items_impl(self); + return dict_items_impl((PyDictObject *)self); } PyDoc_STRVAR(dict_values__doc__, @@ -308,8 +306,8 @@ static PyObject * dict_values_impl(PyDictObject *self); static PyObject * -dict_values(PyDictObject *self, PyObject *Py_UNUSED(ignored)) +dict_values(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return dict_values_impl(self); + return dict_values_impl((PyDictObject *)self); } -/*[clinic end generated code: output=f3dd5f3fb8122aef input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0f04bf0e7e6b130f input=a9049054013a1b77]*/ diff --git a/Objects/clinic/enumobject.c.h b/Objects/clinic/enumobject.c.h index 09774a73c8145c..998b2eb121c6d9 100644 --- a/Objects/clinic/enumobject.c.h +++ b/Objects/clinic/enumobject.c.h @@ -62,7 +62,8 @@ enum_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *iterable; PyObject *start = 0; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -107,4 +108,4 @@ reversed_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=5c48a9a482a52e91 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2e339ade8bedb3a0 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/exceptions.c.h b/Objects/clinic/exceptions.c.h new file mode 100644 index 00000000000000..8699df07495ad8 --- /dev/null +++ b/Objects/clinic/exceptions.c.h @@ -0,0 +1,383 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_BadArgument() + +PyDoc_STRVAR(BaseException___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n"); + +#define BASEEXCEPTION___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)BaseException___reduce__, METH_NOARGS, BaseException___reduce____doc__}, + +static PyObject * +BaseException___reduce___impl(PyBaseExceptionObject *self); + +static PyObject * +BaseException___reduce__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException___reduce___impl((PyBaseExceptionObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(BaseException___setstate____doc__, +"__setstate__($self, state, /)\n" +"--\n" +"\n"); + +#define BASEEXCEPTION___SETSTATE___METHODDEF \ + {"__setstate__", (PyCFunction)BaseException___setstate__, METH_O, BaseException___setstate____doc__}, + +static PyObject * +BaseException___setstate___impl(PyBaseExceptionObject *self, PyObject *state); + +static PyObject * +BaseException___setstate__(PyBaseExceptionObject *self, PyObject *state) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException___setstate___impl((PyBaseExceptionObject *)self, state); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(BaseException_with_traceback__doc__, +"with_traceback($self, tb, /)\n" +"--\n" +"\n" +"Set self.__traceback__ to tb and return self."); + +#define BASEEXCEPTION_WITH_TRACEBACK_METHODDEF \ + {"with_traceback", (PyCFunction)BaseException_with_traceback, METH_O, BaseException_with_traceback__doc__}, + +static PyObject * +BaseException_with_traceback_impl(PyBaseExceptionObject *self, PyObject *tb); + +static PyObject * +BaseException_with_traceback(PyBaseExceptionObject *self, PyObject *tb) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException_with_traceback_impl((PyBaseExceptionObject *)self, tb); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(BaseException_add_note__doc__, +"add_note($self, note, /)\n" +"--\n" +"\n" +"Add a note to the exception"); + +#define BASEEXCEPTION_ADD_NOTE_METHODDEF \ + {"add_note", (PyCFunction)BaseException_add_note, METH_O, BaseException_add_note__doc__}, + +static PyObject * +BaseException_add_note_impl(PyBaseExceptionObject *self, PyObject *note); + +static PyObject * +BaseException_add_note(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *note; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("add_note", "argument", "str", arg); + goto exit; + } + note = arg; + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException_add_note_impl((PyBaseExceptionObject *)self, note); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +#if !defined(BaseException_args_DOCSTR) +# define BaseException_args_DOCSTR NULL +#endif +#if defined(BASEEXCEPTION_ARGS_GETSETDEF) +# undef BASEEXCEPTION_ARGS_GETSETDEF +# define BASEEXCEPTION_ARGS_GETSETDEF {"args", (getter)BaseException_args_get, (setter)BaseException_args_set, BaseException_args_DOCSTR}, +#else +# define BASEEXCEPTION_ARGS_GETSETDEF {"args", (getter)BaseException_args_get, NULL, BaseException_args_DOCSTR}, +#endif + +static PyObject * +BaseException_args_get_impl(PyBaseExceptionObject *self); + +static PyObject * +BaseException_args_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException_args_get_impl((PyBaseExceptionObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(BaseException_args_DOCSTR) +# define BaseException_args_DOCSTR NULL +#endif +#if defined(BASEEXCEPTION_ARGS_GETSETDEF) +# undef BASEEXCEPTION_ARGS_GETSETDEF +# define BASEEXCEPTION_ARGS_GETSETDEF {"args", (getter)BaseException_args_get, (setter)BaseException_args_set, BaseException_args_DOCSTR}, +#else +# define BASEEXCEPTION_ARGS_GETSETDEF {"args", NULL, (setter)BaseException_args_set, NULL}, +#endif + +static int +BaseException_args_set_impl(PyBaseExceptionObject *self, PyObject *value); + +static int +BaseException_args_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException_args_set_impl((PyBaseExceptionObject *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(BaseException___traceback___DOCSTR) +# define BaseException___traceback___DOCSTR NULL +#endif +#if defined(BASEEXCEPTION___TRACEBACK___GETSETDEF) +# undef BASEEXCEPTION___TRACEBACK___GETSETDEF +# define BASEEXCEPTION___TRACEBACK___GETSETDEF {"__traceback__", (getter)BaseException___traceback___get, (setter)BaseException___traceback___set, BaseException___traceback___DOCSTR}, +#else +# define BASEEXCEPTION___TRACEBACK___GETSETDEF {"__traceback__", (getter)BaseException___traceback___get, NULL, BaseException___traceback___DOCSTR}, +#endif + +static PyObject * +BaseException___traceback___get_impl(PyBaseExceptionObject *self); + +static PyObject * +BaseException___traceback___get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException___traceback___get_impl((PyBaseExceptionObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(BaseException___traceback___DOCSTR) +# define BaseException___traceback___DOCSTR NULL +#endif +#if defined(BASEEXCEPTION___TRACEBACK___GETSETDEF) +# undef BASEEXCEPTION___TRACEBACK___GETSETDEF +# define BASEEXCEPTION___TRACEBACK___GETSETDEF {"__traceback__", (getter)BaseException___traceback___get, (setter)BaseException___traceback___set, BaseException___traceback___DOCSTR}, +#else +# define BASEEXCEPTION___TRACEBACK___GETSETDEF {"__traceback__", NULL, (setter)BaseException___traceback___set, NULL}, +#endif + +static int +BaseException___traceback___set_impl(PyBaseExceptionObject *self, + PyObject *value); + +static int +BaseException___traceback___set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException___traceback___set_impl((PyBaseExceptionObject *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(BaseException___context___DOCSTR) +# define BaseException___context___DOCSTR NULL +#endif +#if defined(BASEEXCEPTION___CONTEXT___GETSETDEF) +# undef BASEEXCEPTION___CONTEXT___GETSETDEF +# define BASEEXCEPTION___CONTEXT___GETSETDEF {"__context__", (getter)BaseException___context___get, (setter)BaseException___context___set, BaseException___context___DOCSTR}, +#else +# define BASEEXCEPTION___CONTEXT___GETSETDEF {"__context__", (getter)BaseException___context___get, NULL, BaseException___context___DOCSTR}, +#endif + +static PyObject * +BaseException___context___get_impl(PyBaseExceptionObject *self); + +static PyObject * +BaseException___context___get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException___context___get_impl((PyBaseExceptionObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(BaseException___context___DOCSTR) +# define BaseException___context___DOCSTR NULL +#endif +#if defined(BASEEXCEPTION___CONTEXT___GETSETDEF) +# undef BASEEXCEPTION___CONTEXT___GETSETDEF +# define BASEEXCEPTION___CONTEXT___GETSETDEF {"__context__", (getter)BaseException___context___get, (setter)BaseException___context___set, BaseException___context___DOCSTR}, +#else +# define BASEEXCEPTION___CONTEXT___GETSETDEF {"__context__", NULL, (setter)BaseException___context___set, NULL}, +#endif + +static int +BaseException___context___set_impl(PyBaseExceptionObject *self, + PyObject *value); + +static int +BaseException___context___set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException___context___set_impl((PyBaseExceptionObject *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(BaseException___cause___DOCSTR) +# define BaseException___cause___DOCSTR NULL +#endif +#if defined(BASEEXCEPTION___CAUSE___GETSETDEF) +# undef BASEEXCEPTION___CAUSE___GETSETDEF +# define BASEEXCEPTION___CAUSE___GETSETDEF {"__cause__", (getter)BaseException___cause___get, (setter)BaseException___cause___set, BaseException___cause___DOCSTR}, +#else +# define BASEEXCEPTION___CAUSE___GETSETDEF {"__cause__", (getter)BaseException___cause___get, NULL, BaseException___cause___DOCSTR}, +#endif + +static PyObject * +BaseException___cause___get_impl(PyBaseExceptionObject *self); + +static PyObject * +BaseException___cause___get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException___cause___get_impl((PyBaseExceptionObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if !defined(BaseException___cause___DOCSTR) +# define BaseException___cause___DOCSTR NULL +#endif +#if defined(BASEEXCEPTION___CAUSE___GETSETDEF) +# undef BASEEXCEPTION___CAUSE___GETSETDEF +# define BASEEXCEPTION___CAUSE___GETSETDEF {"__cause__", (getter)BaseException___cause___get, (setter)BaseException___cause___set, BaseException___cause___DOCSTR}, +#else +# define BASEEXCEPTION___CAUSE___GETSETDEF {"__cause__", NULL, (setter)BaseException___cause___set, NULL}, +#endif + +static int +BaseException___cause___set_impl(PyBaseExceptionObject *self, + PyObject *value); + +static int +BaseException___cause___set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseException___cause___set_impl((PyBaseExceptionObject *)self, value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(BaseExceptionGroup_derive__doc__, +"derive($self, excs, /)\n" +"--\n" +"\n"); + +#define BASEEXCEPTIONGROUP_DERIVE_METHODDEF \ + {"derive", (PyCFunction)BaseExceptionGroup_derive, METH_O, BaseExceptionGroup_derive__doc__}, + +static PyObject * +BaseExceptionGroup_derive_impl(PyBaseExceptionGroupObject *self, + PyObject *excs); + +static PyObject * +BaseExceptionGroup_derive(PyBaseExceptionGroupObject *self, PyObject *excs) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseExceptionGroup_derive_impl((PyBaseExceptionGroupObject *)self, excs); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(BaseExceptionGroup_split__doc__, +"split($self, matcher_value, /)\n" +"--\n" +"\n"); + +#define BASEEXCEPTIONGROUP_SPLIT_METHODDEF \ + {"split", (PyCFunction)BaseExceptionGroup_split, METH_O, BaseExceptionGroup_split__doc__}, + +static PyObject * +BaseExceptionGroup_split_impl(PyBaseExceptionGroupObject *self, + PyObject *matcher_value); + +static PyObject * +BaseExceptionGroup_split(PyBaseExceptionGroupObject *self, PyObject *matcher_value) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseExceptionGroup_split_impl((PyBaseExceptionGroupObject *)self, matcher_value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(BaseExceptionGroup_subgroup__doc__, +"subgroup($self, matcher_value, /)\n" +"--\n" +"\n"); + +#define BASEEXCEPTIONGROUP_SUBGROUP_METHODDEF \ + {"subgroup", (PyCFunction)BaseExceptionGroup_subgroup, METH_O, BaseExceptionGroup_subgroup__doc__}, + +static PyObject * +BaseExceptionGroup_subgroup_impl(PyBaseExceptionGroupObject *self, + PyObject *matcher_value); + +static PyObject * +BaseExceptionGroup_subgroup(PyBaseExceptionGroupObject *self, PyObject *matcher_value) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = BaseExceptionGroup_subgroup_impl((PyBaseExceptionGroupObject *)self, matcher_value); + Py_END_CRITICAL_SECTION(); + + return return_value; +} +/*[clinic end generated code: output=19aed708dcaf7184 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/floatobject.c.h b/Objects/clinic/floatobject.c.h index 10f6149cc88c22..dd29135590a6a6 100644 --- a/Objects/clinic/floatobject.c.h +++ b/Objects/clinic/floatobject.c.h @@ -197,7 +197,7 @@ PyDoc_STRVAR(float_new__doc__, "float(x=0, /)\n" "--\n" "\n" -"Convert a string or number to a floating point number, if possible."); +"Convert a string or number to a floating-point number, if possible."); static PyObject * float_new_impl(PyTypeObject *type, PyObject *x); @@ -227,6 +227,15 @@ float_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return return_value; } +PyDoc_STRVAR(float_from_number__doc__, +"from_number($type, number, /)\n" +"--\n" +"\n" +"Convert real number to a floating-point number."); + +#define FLOAT_FROM_NUMBER_METHODDEF \ + {"from_number", (PyCFunction)float_from_number, METH_O|METH_CLASS, float_from_number__doc__}, + PyDoc_STRVAR(float___getnewargs____doc__, "__getnewargs__($self, /)\n" "--\n" @@ -256,7 +265,7 @@ PyDoc_STRVAR(float___getformat____doc__, "It exists mainly to be used in Python\'s test suite.\n" "\n" "This function returns whichever of \'unknown\', \'IEEE, big-endian\' or \'IEEE,\n" -"little-endian\' best describes the format of floating point numbers used by the\n" +"little-endian\' best describes the format of floating-point numbers used by the\n" "C type named by typestr."); #define FLOAT___GETFORMAT___METHODDEF \ @@ -318,4 +327,4 @@ float___format__(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=c79743c8551c30d9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=366cea9463cc5bf6 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/funcobject.c.h b/Objects/clinic/funcobject.c.h index 8f20bda26438cf..3f95a1db7b2788 100644 --- a/Objects/clinic/funcobject.c.h +++ b/Objects/clinic/funcobject.c.h @@ -73,7 +73,8 @@ func_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *closure = Py_None; PyObject *kwdefaults = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 2, 6, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 2, /*maxpos*/ 6, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -115,4 +116,4 @@ func_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=10947342188f38a9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bad4e19757dd26c3 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/listobject.c.h b/Objects/clinic/listobject.c.h index a61550a49b66fc..a29ed9f7088700 100644 --- a/Objects/clinic/listobject.c.h +++ b/Objects/clinic/listobject.c.h @@ -23,7 +23,7 @@ static PyObject * list_insert_impl(PyListObject *self, Py_ssize_t index, PyObject *object); static PyObject * -list_insert(PyListObject *self, PyObject *const *args, Py_ssize_t nargs) +list_insert(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t index; @@ -46,7 +46,7 @@ list_insert(PyListObject *self, PyObject *const *args, Py_ssize_t nargs) } object = args[1]; Py_BEGIN_CRITICAL_SECTION(self); - return_value = list_insert_impl(self, index, object); + return_value = list_insert_impl((PyListObject *)self, index, object); Py_END_CRITICAL_SECTION(); exit: @@ -66,12 +66,12 @@ static PyObject * py_list_clear_impl(PyListObject *self); static PyObject * -py_list_clear(PyListObject *self, PyObject *Py_UNUSED(ignored)) +py_list_clear(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = py_list_clear_impl(self); + return_value = py_list_clear_impl((PyListObject *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -90,12 +90,12 @@ static PyObject * list_copy_impl(PyListObject *self); static PyObject * -list_copy(PyListObject *self, PyObject *Py_UNUSED(ignored)) +list_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = list_copy_impl(self); + return_value = list_copy_impl((PyListObject *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -119,35 +119,20 @@ list_append(PyListObject *self, PyObject *object) PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = list_append_impl(self, object); + return_value = list_append_impl((PyListObject *)self, object); Py_END_CRITICAL_SECTION(); return return_value; } -PyDoc_STRVAR(py_list_extend__doc__, +PyDoc_STRVAR(list_extend__doc__, "extend($self, iterable, /)\n" "--\n" "\n" "Extend list by appending elements from the iterable."); -#define PY_LIST_EXTEND_METHODDEF \ - {"extend", (PyCFunction)py_list_extend, METH_O, py_list_extend__doc__}, - -static PyObject * -py_list_extend_impl(PyListObject *self, PyObject *iterable); - -static PyObject * -py_list_extend(PyListObject *self, PyObject *iterable) -{ - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION2(self, iterable); - return_value = py_list_extend_impl(self, iterable); - Py_END_CRITICAL_SECTION2(); - - return return_value; -} +#define LIST_EXTEND_METHODDEF \ + {"extend", (PyCFunction)list_extend, METH_O, list_extend__doc__}, PyDoc_STRVAR(list_pop__doc__, "pop($self, index=-1, /)\n" @@ -164,7 +149,7 @@ static PyObject * list_pop_impl(PyListObject *self, Py_ssize_t index); static PyObject * -list_pop(PyListObject *self, PyObject *const *args, Py_ssize_t nargs) +list_pop(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; Py_ssize_t index = -1; @@ -189,7 +174,7 @@ list_pop(PyListObject *self, PyObject *const *args, Py_ssize_t nargs) } skip_optional: Py_BEGIN_CRITICAL_SECTION(self); - return_value = list_pop_impl(self, index); + return_value = list_pop_impl((PyListObject *)self, index); Py_END_CRITICAL_SECTION(); exit: @@ -217,7 +202,7 @@ static PyObject * list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse); static PyObject * -list_sort(PyListObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +list_sort(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -250,7 +235,8 @@ list_sort(PyListObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *keyfunc = Py_None; int reverse = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -268,7 +254,9 @@ list_sort(PyListObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject goto exit; } skip_optional_kwonly: - return_value = list_sort_impl(self, keyfunc, reverse); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = list_sort_impl((PyListObject *)self, keyfunc, reverse); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -287,12 +275,12 @@ static PyObject * list_reverse_impl(PyListObject *self); static PyObject * -list_reverse(PyListObject *self, PyObject *Py_UNUSED(ignored)) +list_reverse(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = list_reverse_impl(self); + return_value = list_reverse_impl((PyListObject *)self); Py_END_CRITICAL_SECTION(); return return_value; @@ -314,7 +302,7 @@ list_index_impl(PyListObject *self, PyObject *value, Py_ssize_t start, Py_ssize_t stop); static PyObject * -list_index(PyListObject *self, PyObject *const *args, Py_ssize_t nargs) +list_index(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *value; @@ -338,7 +326,7 @@ list_index(PyListObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: - return_value = list_index_impl(self, value, start, stop); + return_value = list_index_impl((PyListObject *)self, value, start, stop); exit: return return_value; @@ -373,7 +361,7 @@ list_remove(PyListObject *self, PyObject *value) PyObject *return_value = NULL; Py_BEGIN_CRITICAL_SECTION(self); - return_value = list_remove_impl(self, value); + return_value = list_remove_impl((PyListObject *)self, value); Py_END_CRITICAL_SECTION(); return return_value; @@ -430,9 +418,9 @@ static PyObject * list___sizeof___impl(PyListObject *self); static PyObject * -list___sizeof__(PyListObject *self, PyObject *Py_UNUSED(ignored)) +list___sizeof__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return list___sizeof___impl(self); + return list___sizeof___impl((PyListObject *)self); } PyDoc_STRVAR(list___reversed____doc__, @@ -448,8 +436,8 @@ static PyObject * list___reversed___impl(PyListObject *self); static PyObject * -list___reversed__(PyListObject *self, PyObject *Py_UNUSED(ignored)) +list___reversed__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return list___reversed___impl(self); + return list___reversed___impl((PyListObject *)self); } -/*[clinic end generated code: output=26dfb2c9846348f9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=35c43dc33f9ba521 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/longobject.c.h b/Objects/clinic/longobject.c.h index 4a3d71c6111af5..c025f74af5cb58 100644 --- a/Objects/clinic/longobject.c.h +++ b/Objects/clinic/longobject.c.h @@ -48,7 +48,8 @@ long_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *x = NULL; PyObject *obase = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -116,7 +117,7 @@ int___format__(PyObject *self, PyObject *arg) } PyDoc_STRVAR(int___round____doc__, -"__round__($self, ndigits=, /)\n" +"__round__($self, ndigits=None, /)\n" "--\n" "\n" "Rounding an Integral returns itself.\n" @@ -133,7 +134,7 @@ static PyObject * int___round__(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *o_ndigits = NULL; + PyObject *o_ndigits = Py_None; if (!_PyArg_CheckPositional("__round__", nargs, 0, 1)) { goto exit; @@ -267,7 +268,7 @@ PyDoc_STRVAR(int_to_bytes__doc__, " the most significant byte is at the beginning of the byte array. If\n" " byteorder is \'little\', the most significant byte is at the end of the\n" " byte array. To request the native byte order of the host system, use\n" -" `sys.byteorder\' as the byte order value. Default is to use \'big\'.\n" +" sys.byteorder as the byte order value. Default is to use \'big\'.\n" " signed\n" " Determines whether two\'s complement is used to represent the integer.\n" " If signed is False and a negative integer is given, an OverflowError\n" @@ -315,7 +316,8 @@ int_to_bytes(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject * PyObject *byteorder = NULL; int is_signed = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -380,7 +382,7 @@ PyDoc_STRVAR(int_from_bytes__doc__, " the most significant byte is at the beginning of the byte array. If\n" " byteorder is \'little\', the most significant byte is at the end of the\n" " byte array. To request the native byte order of the host system, use\n" -" `sys.byteorder\' as the byte order value. Default is to use \'big\'.\n" +" sys.byteorder as the byte order value. Default is to use \'big\'.\n" " signed\n" " Indicates whether two\'s complement is used to represent the integer."); @@ -426,7 +428,8 @@ int_from_bytes(PyTypeObject *type, PyObject *const *args, Py_ssize_t nargs, PyOb PyObject *byteorder = NULL; int is_signed = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -476,4 +479,4 @@ int_is_integer(PyObject *self, PyObject *Py_UNUSED(ignored)) { return int_is_integer_impl(self); } -/*[clinic end generated code: output=7e6e57246e55911f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fb96bd642a643f75 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index f199434dacb9e8..4706c92051926c 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -51,7 +51,8 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t nargs = PyTuple_GET_SIZE(args); PyObject *object; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -107,7 +108,8 @@ memoryview__from_flags(PyTypeObject *type, PyObject *const *args, Py_ssize_t nar PyObject *object; int flags; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -135,9 +137,9 @@ static PyObject * memoryview_release_impl(PyMemoryViewObject *self); static PyObject * -memoryview_release(PyMemoryViewObject *self, PyObject *Py_UNUSED(ignored)) +memoryview_release(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return memoryview_release_impl(self); + return memoryview_release_impl((PyMemoryViewObject *)self); } PyDoc_STRVAR(memoryview_cast__doc__, @@ -154,7 +156,7 @@ memoryview_cast_impl(PyMemoryViewObject *self, PyObject *format, PyObject *shape); static PyObject * -memoryview_cast(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +memoryview_cast(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -187,7 +189,8 @@ memoryview_cast(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t narg PyObject *format; PyObject *shape = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -201,7 +204,7 @@ memoryview_cast(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t narg } shape = args[1]; skip_optional_pos: - return_value = memoryview_cast_impl(self, format, shape); + return_value = memoryview_cast_impl((PyMemoryViewObject *)self, format, shape); exit: return return_value; @@ -220,9 +223,9 @@ static PyObject * memoryview_toreadonly_impl(PyMemoryViewObject *self); static PyObject * -memoryview_toreadonly(PyMemoryViewObject *self, PyObject *Py_UNUSED(ignored)) +memoryview_toreadonly(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return memoryview_toreadonly_impl(self); + return memoryview_toreadonly_impl((PyMemoryViewObject *)self); } PyDoc_STRVAR(memoryview_tolist__doc__, @@ -238,9 +241,9 @@ static PyObject * memoryview_tolist_impl(PyMemoryViewObject *self); static PyObject * -memoryview_tolist(PyMemoryViewObject *self, PyObject *Py_UNUSED(ignored)) +memoryview_tolist(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return memoryview_tolist_impl(self); + return memoryview_tolist_impl((PyMemoryViewObject *)self); } PyDoc_STRVAR(memoryview_tobytes__doc__, @@ -262,7 +265,7 @@ static PyObject * memoryview_tobytes_impl(PyMemoryViewObject *self, const char *order); static PyObject * -memoryview_tobytes(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +memoryview_tobytes(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -294,7 +297,8 @@ memoryview_tobytes(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t n Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; const char *order = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -320,7 +324,7 @@ memoryview_tobytes(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t n goto exit; } skip_optional_pos: - return_value = memoryview_tobytes_impl(self, order); + return_value = memoryview_tobytes_impl((PyMemoryViewObject *)self, order); exit: return return_value; @@ -357,7 +361,7 @@ memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, int bytes_per_sep); static PyObject * -memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +memoryview_hex(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -390,7 +394,8 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs PyObject *sep = NULL; int bytes_per_sep = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -408,9 +413,64 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs goto exit; } skip_optional_pos: - return_value = memoryview_hex_impl(self, sep, bytes_per_sep); + return_value = memoryview_hex_impl((PyMemoryViewObject *)self, sep, bytes_per_sep); exit: return return_value; } -/*[clinic end generated code: output=7e76a09106921ba2 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(memoryview_count__doc__, +"count($self, value, /)\n" +"--\n" +"\n" +"Count the number of occurrences of a value."); + +#define MEMORYVIEW_COUNT_METHODDEF \ + {"count", (PyCFunction)memoryview_count, METH_O, memoryview_count__doc__}, + +PyDoc_STRVAR(memoryview_index__doc__, +"index($self, value, start=0, stop=sys.maxsize, /)\n" +"--\n" +"\n" +"Return the index of the first occurrence of a value.\n" +"\n" +"Raises ValueError if the value is not present."); + +#define MEMORYVIEW_INDEX_METHODDEF \ + {"index", _PyCFunction_CAST(memoryview_index), METH_FASTCALL, memoryview_index__doc__}, + +static PyObject * +memoryview_index_impl(PyMemoryViewObject *self, PyObject *value, + Py_ssize_t start, Py_ssize_t stop); + +static PyObject * +memoryview_index(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *value; + Py_ssize_t start = 0; + Py_ssize_t stop = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("index", nargs, 1, 3)) { + goto exit; + } + value = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndexNotNone(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndexNotNone(args[2], &stop)) { + goto exit; + } +skip_optional: + return_value = memoryview_index_impl((PyMemoryViewObject *)self, value, start, stop); + +exit: + return return_value; +} +/*[clinic end generated code: output=2ef6c061d9c4e3dc input=a9049054013a1b77]*/ diff --git a/Objects/clinic/moduleobject.c.h b/Objects/clinic/moduleobject.c.h index 3c0bbe22d5ebab..5535a2f1178b9f 100644 --- a/Objects/clinic/moduleobject.c.h +++ b/Objects/clinic/moduleobject.c.h @@ -55,7 +55,8 @@ module___init__(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *name; PyObject *doc = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -74,4 +75,4 @@ module___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=e8a71bfbed774c15 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d7da56d5c2eb6d30 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/odictobject.c.h b/Objects/clinic/odictobject.c.h index 5ef5380656905a..44d89c4e0ef2f7 100644 --- a/Objects/clinic/odictobject.c.h +++ b/Objects/clinic/odictobject.c.h @@ -54,7 +54,8 @@ OrderedDict_fromkeys(PyTypeObject *type, PyObject *const *args, Py_ssize_t nargs PyObject *seq; PyObject *value = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -86,7 +87,7 @@ OrderedDict_setdefault_impl(PyODictObject *self, PyObject *key, PyObject *default_value); static PyObject * -OrderedDict_setdefault(PyODictObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +OrderedDict_setdefault(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -119,7 +120,8 @@ OrderedDict_setdefault(PyODictObject *self, PyObject *const *args, Py_ssize_t na PyObject *key; PyObject *default_value = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -129,7 +131,7 @@ OrderedDict_setdefault(PyODictObject *self, PyObject *const *args, Py_ssize_t na } default_value = args[1]; skip_optional_pos: - return_value = OrderedDict_setdefault_impl(self, key, default_value); + return_value = OrderedDict_setdefault_impl((PyODictObject *)self, key, default_value); exit: return return_value; @@ -152,7 +154,7 @@ OrderedDict_pop_impl(PyODictObject *self, PyObject *key, PyObject *default_value); static PyObject * -OrderedDict_pop(PyODictObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +OrderedDict_pop(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -185,7 +187,8 @@ OrderedDict_pop(PyODictObject *self, PyObject *const *args, Py_ssize_t nargs, Py PyObject *key; PyObject *default_value = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -195,7 +198,7 @@ OrderedDict_pop(PyODictObject *self, PyObject *const *args, Py_ssize_t nargs, Py } default_value = args[1]; skip_optional_pos: - return_value = OrderedDict_pop_impl(self, key, default_value); + return_value = OrderedDict_pop_impl((PyODictObject *)self, key, default_value); exit: return return_value; @@ -216,7 +219,7 @@ static PyObject * OrderedDict_popitem_impl(PyODictObject *self, int last); static PyObject * -OrderedDict_popitem(PyODictObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +OrderedDict_popitem(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -248,7 +251,8 @@ OrderedDict_popitem(PyODictObject *self, PyObject *const *args, Py_ssize_t nargs Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; int last = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -260,7 +264,7 @@ OrderedDict_popitem(PyODictObject *self, PyObject *const *args, Py_ssize_t nargs goto exit; } skip_optional_pos: - return_value = OrderedDict_popitem_impl(self, last); + return_value = OrderedDict_popitem_impl((PyODictObject *)self, last); exit: return return_value; @@ -281,7 +285,7 @@ static PyObject * OrderedDict_move_to_end_impl(PyODictObject *self, PyObject *key, int last); static PyObject * -OrderedDict_move_to_end(PyODictObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +OrderedDict_move_to_end(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -314,7 +318,8 @@ OrderedDict_move_to_end(PyODictObject *self, PyObject *const *args, Py_ssize_t n PyObject *key; int last = 1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -327,9 +332,9 @@ OrderedDict_move_to_end(PyODictObject *self, PyObject *const *args, Py_ssize_t n goto exit; } skip_optional_pos: - return_value = OrderedDict_move_to_end_impl(self, key, last); + return_value = OrderedDict_move_to_end_impl((PyODictObject *)self, key, last); exit: return return_value; } -/*[clinic end generated code: output=eff78d2a3f9379bd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=55bd390bb516e997 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/setobject.c.h b/Objects/clinic/setobject.c.h index f3c96995ede60d..bf7e604e4b0a46 100644 --- a/Objects/clinic/setobject.c.h +++ b/Objects/clinic/setobject.c.h @@ -2,7 +2,7 @@ preserve [clinic start generated code]*/ -#include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() PyDoc_STRVAR(set_pop__doc__, "pop($self, /)\n" @@ -19,9 +19,15 @@ static PyObject * set_pop_impl(PySetObject *so); static PyObject * -set_pop(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set_pop(PyObject *so, PyObject *Py_UNUSED(ignored)) { - return set_pop_impl(so); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_pop_impl((PySetObject *)so); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(set_update__doc__, @@ -34,28 +40,20 @@ PyDoc_STRVAR(set_update__doc__, {"update", _PyCFunction_CAST(set_update), METH_FASTCALL, set_update__doc__}, static PyObject * -set_update_impl(PySetObject *so, PyObject *args); +set_update_impl(PySetObject *so, PyObject * const *others, + Py_ssize_t others_length); static PyObject * -set_update(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +set_update(PyObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; - - if (!_PyArg_CheckPositional("update", nargs, 0, PY_SSIZE_T_MAX)) { - goto exit; - } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = set_update_impl(so, __clinic_args); - -exit: - Py_XDECREF(__clinic_args); + PyObject * const *others; + Py_ssize_t others_length; + + others = args; + others_length = nargs; + return_value = set_update_impl((PySetObject *)so, others, others_length); + return return_value; } @@ -72,9 +70,15 @@ static PyObject * set_copy_impl(PySetObject *so); static PyObject * -set_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set_copy(PyObject *so, PyObject *Py_UNUSED(ignored)) { - return set_copy_impl(so); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_copy_impl((PySetObject *)so); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(frozenset_copy__doc__, @@ -90,9 +94,15 @@ static PyObject * frozenset_copy_impl(PySetObject *so); static PyObject * -frozenset_copy(PySetObject *so, PyObject *Py_UNUSED(ignored)) +frozenset_copy(PyObject *so, PyObject *Py_UNUSED(ignored)) { - return frozenset_copy_impl(so); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = frozenset_copy_impl((PySetObject *)so); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(set_clear__doc__, @@ -108,9 +118,15 @@ static PyObject * set_clear_impl(PySetObject *so); static PyObject * -set_clear(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set_clear(PyObject *so, PyObject *Py_UNUSED(ignored)) { - return set_clear_impl(so); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_clear_impl((PySetObject *)so); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(set_union__doc__, @@ -123,28 +139,20 @@ PyDoc_STRVAR(set_union__doc__, {"union", _PyCFunction_CAST(set_union), METH_FASTCALL, set_union__doc__}, static PyObject * -set_union_impl(PySetObject *so, PyObject *args); +set_union_impl(PySetObject *so, PyObject * const *others, + Py_ssize_t others_length); static PyObject * -set_union(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +set_union(PyObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; - - if (!_PyArg_CheckPositional("union", nargs, 0, PY_SSIZE_T_MAX)) { - goto exit; - } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = set_union_impl(so, __clinic_args); - -exit: - Py_XDECREF(__clinic_args); + PyObject * const *others; + Py_ssize_t others_length; + + others = args; + others_length = nargs; + return_value = set_union_impl((PySetObject *)so, others, others_length); + return return_value; } @@ -158,28 +166,20 @@ PyDoc_STRVAR(set_intersection_multi__doc__, {"intersection", _PyCFunction_CAST(set_intersection_multi), METH_FASTCALL, set_intersection_multi__doc__}, static PyObject * -set_intersection_multi_impl(PySetObject *so, PyObject *args); +set_intersection_multi_impl(PySetObject *so, PyObject * const *others, + Py_ssize_t others_length); static PyObject * -set_intersection_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +set_intersection_multi(PyObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; - - if (!_PyArg_CheckPositional("intersection", nargs, 0, PY_SSIZE_T_MAX)) { - goto exit; - } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = set_intersection_multi_impl(so, __clinic_args); - -exit: - Py_XDECREF(__clinic_args); + PyObject * const *others; + Py_ssize_t others_length; + + others = args; + others_length = nargs; + return_value = set_intersection_multi_impl((PySetObject *)so, others, others_length); + return return_value; } @@ -193,28 +193,20 @@ PyDoc_STRVAR(set_intersection_update_multi__doc__, {"intersection_update", _PyCFunction_CAST(set_intersection_update_multi), METH_FASTCALL, set_intersection_update_multi__doc__}, static PyObject * -set_intersection_update_multi_impl(PySetObject *so, PyObject *args); +set_intersection_update_multi_impl(PySetObject *so, PyObject * const *others, + Py_ssize_t others_length); static PyObject * -set_intersection_update_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +set_intersection_update_multi(PyObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; - - if (!_PyArg_CheckPositional("intersection_update", nargs, 0, PY_SSIZE_T_MAX)) { - goto exit; - } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = set_intersection_update_multi_impl(so, __clinic_args); - -exit: - Py_XDECREF(__clinic_args); + PyObject * const *others; + Py_ssize_t others_length; + + others = args; + others_length = nargs; + return_value = set_intersection_update_multi_impl((PySetObject *)so, others, others_length); + return return_value; } @@ -227,6 +219,21 @@ PyDoc_STRVAR(set_isdisjoint__doc__, #define SET_ISDISJOINT_METHODDEF \ {"isdisjoint", (PyCFunction)set_isdisjoint, METH_O, set_isdisjoint__doc__}, +static PyObject * +set_isdisjoint_impl(PySetObject *so, PyObject *other); + +static PyObject * +set_isdisjoint(PySetObject *so, PyObject *other) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION2(so, other); + return_value = set_isdisjoint_impl((PySetObject *)so, other); + Py_END_CRITICAL_SECTION2(); + + return return_value; +} + PyDoc_STRVAR(set_difference_update__doc__, "difference_update($self, /, *others)\n" "--\n" @@ -237,28 +244,20 @@ PyDoc_STRVAR(set_difference_update__doc__, {"difference_update", _PyCFunction_CAST(set_difference_update), METH_FASTCALL, set_difference_update__doc__}, static PyObject * -set_difference_update_impl(PySetObject *so, PyObject *args); +set_difference_update_impl(PySetObject *so, PyObject * const *others, + Py_ssize_t others_length); static PyObject * -set_difference_update(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +set_difference_update(PyObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; - - if (!_PyArg_CheckPositional("difference_update", nargs, 0, PY_SSIZE_T_MAX)) { - goto exit; - } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = set_difference_update_impl(so, __clinic_args); - -exit: - Py_XDECREF(__clinic_args); + PyObject * const *others; + Py_ssize_t others_length; + + others = args; + others_length = nargs; + return_value = set_difference_update_impl((PySetObject *)so, others, others_length); + return return_value; } @@ -272,28 +271,20 @@ PyDoc_STRVAR(set_difference_multi__doc__, {"difference", _PyCFunction_CAST(set_difference_multi), METH_FASTCALL, set_difference_multi__doc__}, static PyObject * -set_difference_multi_impl(PySetObject *so, PyObject *args); +set_difference_multi_impl(PySetObject *so, PyObject * const *others, + Py_ssize_t others_length); static PyObject * -set_difference_multi(PySetObject *so, PyObject *const *args, Py_ssize_t nargs) +set_difference_multi(PyObject *so, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *__clinic_args = NULL; - - if (!_PyArg_CheckPositional("difference", nargs, 0, PY_SSIZE_T_MAX)) { - goto exit; - } - __clinic_args = PyTuple_New(nargs - 0); - if (!__clinic_args) { - goto exit; - } - for (Py_ssize_t i = 0; i < nargs - 0; ++i) { - PyTuple_SET_ITEM(__clinic_args, i, Py_NewRef(args[0 + i])); - } - return_value = set_difference_multi_impl(so, __clinic_args); - -exit: - Py_XDECREF(__clinic_args); + PyObject * const *others; + Py_ssize_t others_length; + + others = args; + others_length = nargs; + return_value = set_difference_multi_impl((PySetObject *)so, others, others_length); + return return_value; } @@ -315,6 +306,21 @@ PyDoc_STRVAR(set_symmetric_difference__doc__, #define SET_SYMMETRIC_DIFFERENCE_METHODDEF \ {"symmetric_difference", (PyCFunction)set_symmetric_difference, METH_O, set_symmetric_difference__doc__}, +static PyObject * +set_symmetric_difference_impl(PySetObject *so, PyObject *other); + +static PyObject * +set_symmetric_difference(PySetObject *so, PyObject *other) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION2(so, other); + return_value = set_symmetric_difference_impl((PySetObject *)so, other); + Py_END_CRITICAL_SECTION2(); + + return return_value; +} + PyDoc_STRVAR(set_issubset__doc__, "issubset($self, other, /)\n" "--\n" @@ -324,6 +330,21 @@ PyDoc_STRVAR(set_issubset__doc__, #define SET_ISSUBSET_METHODDEF \ {"issubset", (PyCFunction)set_issubset, METH_O, set_issubset__doc__}, +static PyObject * +set_issubset_impl(PySetObject *so, PyObject *other); + +static PyObject * +set_issubset(PySetObject *so, PyObject *other) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION2(so, other); + return_value = set_issubset_impl((PySetObject *)so, other); + Py_END_CRITICAL_SECTION2(); + + return return_value; +} + PyDoc_STRVAR(set_issuperset__doc__, "issuperset($self, other, /)\n" "--\n" @@ -333,6 +354,21 @@ PyDoc_STRVAR(set_issuperset__doc__, #define SET_ISSUPERSET_METHODDEF \ {"issuperset", (PyCFunction)set_issuperset, METH_O, set_issuperset__doc__}, +static PyObject * +set_issuperset_impl(PySetObject *so, PyObject *other); + +static PyObject * +set_issuperset(PySetObject *so, PyObject *other) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION2(so, other); + return_value = set_issuperset_impl((PySetObject *)so, other); + Py_END_CRITICAL_SECTION2(); + + return return_value; +} + PyDoc_STRVAR(set_add__doc__, "add($self, object, /)\n" "--\n" @@ -344,6 +380,21 @@ PyDoc_STRVAR(set_add__doc__, #define SET_ADD_METHODDEF \ {"add", (PyCFunction)set_add, METH_O, set_add__doc__}, +static PyObject * +set_add_impl(PySetObject *so, PyObject *key); + +static PyObject * +set_add(PySetObject *so, PyObject *key) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_add_impl((PySetObject *)so, key); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(set___contains____doc__, "__contains__($self, object, /)\n" "--\n" @@ -353,6 +404,21 @@ PyDoc_STRVAR(set___contains____doc__, #define SET___CONTAINS___METHODDEF \ {"__contains__", (PyCFunction)set___contains__, METH_O|METH_COEXIST, set___contains____doc__}, +static PyObject * +set___contains___impl(PySetObject *so, PyObject *key); + +static PyObject * +set___contains__(PySetObject *so, PyObject *key) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set___contains___impl((PySetObject *)so, key); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(set_remove__doc__, "remove($self, object, /)\n" "--\n" @@ -364,6 +430,21 @@ PyDoc_STRVAR(set_remove__doc__, #define SET_REMOVE_METHODDEF \ {"remove", (PyCFunction)set_remove, METH_O, set_remove__doc__}, +static PyObject * +set_remove_impl(PySetObject *so, PyObject *key); + +static PyObject * +set_remove(PySetObject *so, PyObject *key) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_remove_impl((PySetObject *)so, key); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(set_discard__doc__, "discard($self, object, /)\n" "--\n" @@ -376,6 +457,21 @@ PyDoc_STRVAR(set_discard__doc__, #define SET_DISCARD_METHODDEF \ {"discard", (PyCFunction)set_discard, METH_O, set_discard__doc__}, +static PyObject * +set_discard_impl(PySetObject *so, PyObject *key); + +static PyObject * +set_discard(PySetObject *so, PyObject *key) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set_discard_impl((PySetObject *)so, key); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(set___reduce____doc__, "__reduce__($self, /)\n" "--\n" @@ -389,9 +485,15 @@ static PyObject * set___reduce___impl(PySetObject *so); static PyObject * -set___reduce__(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set___reduce__(PyObject *so, PyObject *Py_UNUSED(ignored)) { - return set___reduce___impl(so); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set___reduce___impl((PySetObject *)so); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(set___sizeof____doc__, @@ -407,8 +509,14 @@ static PyObject * set___sizeof___impl(PySetObject *so); static PyObject * -set___sizeof__(PySetObject *so, PyObject *Py_UNUSED(ignored)) +set___sizeof__(PyObject *so, PyObject *Py_UNUSED(ignored)) { - return set___sizeof___impl(so); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(so); + return_value = set___sizeof___impl((PySetObject *)so); + Py_END_CRITICAL_SECTION(); + + return return_value; } -/*[clinic end generated code: output=34a30591148da884 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=83b7742a762ce465 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/structseq.c.h b/Objects/clinic/structseq.c.h index cec49ebccccb24..0afb21ad918669 100644 --- a/Objects/clinic/structseq.c.h +++ b/Objects/clinic/structseq.c.h @@ -47,7 +47,8 @@ structseq_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *arg; PyObject *dict = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -62,4 +63,4 @@ structseq_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=5bf39b3f06a34ce4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ef2406240ce0ad3f input=a9049054013a1b77]*/ diff --git a/Objects/clinic/tupleobject.c.h b/Objects/clinic/tupleobject.c.h index 5d6a2c481a5f2a..40ffd4c1755769 100644 --- a/Objects/clinic/tupleobject.c.h +++ b/Objects/clinic/tupleobject.c.h @@ -20,7 +20,7 @@ tuple_index_impl(PyTupleObject *self, PyObject *value, Py_ssize_t start, Py_ssize_t stop); static PyObject * -tuple_index(PyTupleObject *self, PyObject *const *args, Py_ssize_t nargs) +tuple_index(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *value; @@ -44,7 +44,7 @@ tuple_index(PyTupleObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: - return_value = tuple_index_impl(self, value, start, stop); + return_value = tuple_index_impl((PyTupleObject *)self, value, start, stop); exit: return return_value; @@ -110,8 +110,8 @@ static PyObject * tuple___getnewargs___impl(PyTupleObject *self); static PyObject * -tuple___getnewargs__(PyTupleObject *self, PyObject *Py_UNUSED(ignored)) +tuple___getnewargs__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return tuple___getnewargs___impl(self); + return tuple___getnewargs___impl((PyTupleObject *)self); } -/*[clinic end generated code: output=a6a9abba5d121f4c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=779cb4a13db67397 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/typeobject.c.h b/Objects/clinic/typeobject.c.h index 1fa153598db213..5e8187b3f5b748 100644 --- a/Objects/clinic/typeobject.c.h +++ b/Objects/clinic/typeobject.c.h @@ -22,7 +22,7 @@ type___instancecheck__(PyTypeObject *self, PyObject *instance) PyObject *return_value = NULL; int _return_value; - _return_value = type___instancecheck___impl(self, instance); + _return_value = type___instancecheck___impl((PyTypeObject *)self, instance); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -50,7 +50,7 @@ type___subclasscheck__(PyTypeObject *self, PyObject *subclass) PyObject *return_value = NULL; int _return_value; - _return_value = type___subclasscheck___impl(self, subclass); + _return_value = type___subclasscheck___impl((PyTypeObject *)self, subclass); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -73,9 +73,9 @@ static PyObject * type_mro_impl(PyTypeObject *self); static PyObject * -type_mro(PyTypeObject *self, PyObject *Py_UNUSED(ignored)) +type_mro(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return type_mro_impl(self); + return type_mro_impl((PyTypeObject *)self); } PyDoc_STRVAR(type___subclasses____doc__, @@ -91,9 +91,9 @@ static PyObject * type___subclasses___impl(PyTypeObject *self); static PyObject * -type___subclasses__(PyTypeObject *self, PyObject *Py_UNUSED(ignored)) +type___subclasses__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return type___subclasses___impl(self); + return type___subclasses___impl((PyTypeObject *)self); } PyDoc_STRVAR(type___dir____doc__, @@ -109,9 +109,9 @@ static PyObject * type___dir___impl(PyTypeObject *self); static PyObject * -type___dir__(PyTypeObject *self, PyObject *Py_UNUSED(ignored)) +type___dir__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return type___dir___impl(self); + return type___dir___impl((PyTypeObject *)self); } PyDoc_STRVAR(type___sizeof____doc__, @@ -127,9 +127,9 @@ static PyObject * type___sizeof___impl(PyTypeObject *self); static PyObject * -type___sizeof__(PyTypeObject *self, PyObject *Py_UNUSED(ignored)) +type___sizeof__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return type___sizeof___impl(self); + return type___sizeof___impl((PyTypeObject *)self); } PyDoc_STRVAR(object___getstate____doc__, @@ -262,4 +262,4 @@ object___dir__(PyObject *self, PyObject *Py_UNUSED(ignored)) { return object___dir___impl(self); } -/*[clinic end generated code: output=b56c87f9cace1921 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f7db85fd11818c63 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/typevarobject.c.h b/Objects/clinic/typevarobject.c.h index 2bb0a98a2ed64c..e50ed7d95b9b2a 100644 --- a/Objects/clinic/typevarobject.c.h +++ b/Objects/clinic/typevarobject.c.h @@ -6,19 +6,19 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif -#include "pycore_modsupport.h" // _PyArg_UnpackKeywordsWithVararg() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(typevar_new__doc__, -"typevar(name, *constraints, bound=None, covariant=False,\n" -" contravariant=False, infer_variance=False)\n" +"typevar(name, *constraints, bound=None, default=typing.NoDefault,\n" +" covariant=False, contravariant=False, infer_variance=False)\n" "--\n" "\n" "Create a TypeVar."); static PyObject * typevar_new_impl(PyTypeObject *type, PyObject *name, PyObject *constraints, - PyObject *bound, int covariant, int contravariant, - int infer_variance); + PyObject *bound, PyObject *default_value, int covariant, + int contravariant, int infer_variance); static PyObject * typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -26,14 +26,14 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 5 + #define NUM_KEYWORDS 6 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), }, + .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(default), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -42,7 +42,7 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "bound", "covariant", "contravariant", "infer_variance", NULL}; + static const char * const _keywords[] = {"name", "bound", "default", "covariant", "contravariant", "infer_variance", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "typevar", @@ -56,11 +56,13 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *name; PyObject *constraints = NULL; PyObject *bound = Py_None; + PyObject *default_value = &_Py_NoDefaultStruct; int covariant = 0; int contravariant = 0; int infer_variance = 0; - fastargs = _PyArg_UnpackKeywordsWithVararg(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, 1, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 1, argsbuf); if (!fastargs) { goto exit; } @@ -69,12 +71,17 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto exit; } name = fastargs[0]; - constraints = fastargs[1]; if (!noptargs) { goto skip_optional_kwonly; } + if (fastargs[1]) { + bound = fastargs[1]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } if (fastargs[2]) { - bound = fastargs[2]; + default_value = fastargs[2]; if (!--noptargs) { goto skip_optional_kwonly; } @@ -102,10 +109,16 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto exit; } skip_optional_kwonly: - return_value = typevar_new_impl(type, name, constraints, bound, covariant, contravariant, infer_variance); + constraints = PyTuple_GetSlice(args, 1, PY_SSIZE_T_MAX); + if (!constraints) { + goto exit; + } + return_value = typevar_new_impl(type, name, constraints, bound, default_value, covariant, contravariant, infer_variance); exit: + /* Cleanup for constraints */ Py_XDECREF(constraints); + return return_value; } @@ -117,6 +130,36 @@ PyDoc_STRVAR(typevar_typing_subst__doc__, #define TYPEVAR_TYPING_SUBST_METHODDEF \ {"__typing_subst__", (PyCFunction)typevar_typing_subst, METH_O, typevar_typing_subst__doc__}, +PyDoc_STRVAR(typevar_typing_prepare_subst__doc__, +"__typing_prepare_subst__($self, alias, args, /)\n" +"--\n" +"\n"); + +#define TYPEVAR_TYPING_PREPARE_SUBST_METHODDEF \ + {"__typing_prepare_subst__", _PyCFunction_CAST(typevar_typing_prepare_subst), METH_FASTCALL, typevar_typing_prepare_subst__doc__}, + +static PyObject * +typevar_typing_prepare_subst_impl(typevarobject *self, PyObject *alias, + PyObject *args); + +static PyObject * +typevar_typing_prepare_subst(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *alias; + PyObject *__clinic_args; + + if (!_PyArg_CheckPositional("__typing_prepare_subst__", nargs, 2, 2)) { + goto exit; + } + alias = args[0]; + __clinic_args = args[1]; + return_value = typevar_typing_prepare_subst_impl((typevarobject *)self, alias, __clinic_args); + +exit: + return return_value; +} + PyDoc_STRVAR(typevar_reduce__doc__, "__reduce__($self, /)\n" "--\n" @@ -129,9 +172,26 @@ static PyObject * typevar_reduce_impl(typevarobject *self); static PyObject * -typevar_reduce(typevarobject *self, PyObject *Py_UNUSED(ignored)) +typevar_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return typevar_reduce_impl((typevarobject *)self); +} + +PyDoc_STRVAR(typevar_has_default__doc__, +"has_default($self, /)\n" +"--\n" +"\n"); + +#define TYPEVAR_HAS_DEFAULT_METHODDEF \ + {"has_default", (PyCFunction)typevar_has_default, METH_NOARGS, typevar_has_default__doc__}, + +static PyObject * +typevar_has_default_impl(typevarobject *self); + +static PyObject * +typevar_has_default(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return typevar_reduce_impl(self); + return typevar_has_default_impl((typevarobject *)self); } PyDoc_STRVAR(paramspecargs_new__doc__, @@ -177,7 +237,8 @@ paramspecargs_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t nargs = PyTuple_GET_SIZE(args); PyObject *origin; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -231,7 +292,8 @@ paramspeckwargs_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t nargs = PyTuple_GET_SIZE(args); PyObject *origin; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -243,15 +305,16 @@ paramspeckwargs_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(paramspec_new__doc__, -"paramspec(name, *, bound=None, covariant=False, contravariant=False,\n" -" infer_variance=False)\n" +"paramspec(name, *, bound=None, default=typing.NoDefault,\n" +" covariant=False, contravariant=False, infer_variance=False)\n" "--\n" "\n" "Create a ParamSpec object."); static PyObject * paramspec_new_impl(PyTypeObject *type, PyObject *name, PyObject *bound, - int covariant, int contravariant, int infer_variance); + PyObject *default_value, int covariant, int contravariant, + int infer_variance); static PyObject * paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -259,14 +322,14 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 5 + #define NUM_KEYWORDS 6 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), }, + .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(default), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -275,24 +338,26 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "bound", "covariant", "contravariant", "infer_variance", NULL}; + static const char * const _keywords[] = {"name", "bound", "default", "covariant", "contravariant", "infer_variance", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "paramspec", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[5]; + PyObject *argsbuf[6]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *name; PyObject *bound = Py_None; + PyObject *default_value = &_Py_NoDefaultStruct; int covariant = 0; int contravariant = 0; int infer_variance = 0; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -311,7 +376,13 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } } if (fastargs[2]) { - covariant = PyObject_IsTrue(fastargs[2]); + default_value = fastargs[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[3]) { + covariant = PyObject_IsTrue(fastargs[3]); if (covariant < 0) { goto exit; } @@ -319,8 +390,8 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - if (fastargs[3]) { - contravariant = PyObject_IsTrue(fastargs[3]); + if (fastargs[4]) { + contravariant = PyObject_IsTrue(fastargs[4]); if (contravariant < 0) { goto exit; } @@ -328,12 +399,12 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - infer_variance = PyObject_IsTrue(fastargs[4]); + infer_variance = PyObject_IsTrue(fastargs[5]); if (infer_variance < 0) { goto exit; } skip_optional_kwonly: - return_value = paramspec_new_impl(type, name, bound, covariant, contravariant, infer_variance); + return_value = paramspec_new_impl(type, name, bound, default_value, covariant, contravariant, infer_variance); exit: return return_value; @@ -360,7 +431,7 @@ paramspec_typing_prepare_subst_impl(paramspecobject *self, PyObject *alias, PyObject *args); static PyObject * -paramspec_typing_prepare_subst(paramspecobject *self, PyObject *const *args, Py_ssize_t nargs) +paramspec_typing_prepare_subst(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *alias; @@ -371,7 +442,7 @@ paramspec_typing_prepare_subst(paramspecobject *self, PyObject *const *args, Py_ } alias = args[0]; __clinic_args = args[1]; - return_value = paramspec_typing_prepare_subst_impl(self, alias, __clinic_args); + return_value = paramspec_typing_prepare_subst_impl((paramspecobject *)self, alias, __clinic_args); exit: return return_value; @@ -389,19 +460,37 @@ static PyObject * paramspec_reduce_impl(paramspecobject *self); static PyObject * -paramspec_reduce(paramspecobject *self, PyObject *Py_UNUSED(ignored)) +paramspec_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return paramspec_reduce_impl((paramspecobject *)self); +} + +PyDoc_STRVAR(paramspec_has_default__doc__, +"has_default($self, /)\n" +"--\n" +"\n"); + +#define PARAMSPEC_HAS_DEFAULT_METHODDEF \ + {"has_default", (PyCFunction)paramspec_has_default, METH_NOARGS, paramspec_has_default__doc__}, + +static PyObject * +paramspec_has_default_impl(paramspecobject *self); + +static PyObject * +paramspec_has_default(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return paramspec_reduce_impl(self); + return paramspec_has_default_impl((paramspecobject *)self); } PyDoc_STRVAR(typevartuple__doc__, -"typevartuple(name)\n" +"typevartuple(name, *, default=typing.NoDefault)\n" "--\n" "\n" "Create a new TypeVarTuple with the given name."); static PyObject * -typevartuple_impl(PyTypeObject *type, PyObject *name); +typevartuple_impl(PyTypeObject *type, PyObject *name, + PyObject *default_value); static PyObject * typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -409,14 +498,14 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 2 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), }, + .ob_item = { &_Py_ID(name), &_Py_ID(default), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -425,19 +514,22 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", NULL}; + static const char * const _keywords[] = {"name", "default", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "typevartuple", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[1]; + PyObject *argsbuf[2]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *name; + PyObject *default_value = &_Py_NoDefaultStruct; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -446,7 +538,12 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto exit; } name = fastargs[0]; - return_value = typevartuple_impl(type, name); + if (!noptargs) { + goto skip_optional_kwonly; + } + default_value = fastargs[1]; +skip_optional_kwonly: + return_value = typevartuple_impl(type, name, default_value); exit: return return_value; @@ -473,7 +570,7 @@ typevartuple_typing_prepare_subst_impl(typevartupleobject *self, PyObject *alias, PyObject *args); static PyObject * -typevartuple_typing_prepare_subst(typevartupleobject *self, PyObject *const *args, Py_ssize_t nargs) +typevartuple_typing_prepare_subst(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *alias; @@ -484,7 +581,7 @@ typevartuple_typing_prepare_subst(typevartupleobject *self, PyObject *const *arg } alias = args[0]; __clinic_args = args[1]; - return_value = typevartuple_typing_prepare_subst_impl(self, alias, __clinic_args); + return_value = typevartuple_typing_prepare_subst_impl((typevartupleobject *)self, alias, __clinic_args); exit: return return_value; @@ -502,9 +599,26 @@ static PyObject * typevartuple_reduce_impl(typevartupleobject *self); static PyObject * -typevartuple_reduce(typevartupleobject *self, PyObject *Py_UNUSED(ignored)) +typevartuple_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return typevartuple_reduce_impl((typevartupleobject *)self); +} + +PyDoc_STRVAR(typevartuple_has_default__doc__, +"has_default($self, /)\n" +"--\n" +"\n"); + +#define TYPEVARTUPLE_HAS_DEFAULT_METHODDEF \ + {"has_default", (PyCFunction)typevartuple_has_default, METH_NOARGS, typevartuple_has_default__doc__}, + +static PyObject * +typevartuple_has_default_impl(typevartupleobject *self); + +static PyObject * +typevartuple_has_default(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return typevartuple_reduce_impl(self); + return typevartuple_has_default_impl((typevartupleobject *)self); } PyDoc_STRVAR(typealias_reduce__doc__, @@ -519,9 +633,9 @@ static PyObject * typealias_reduce_impl(typealiasobject *self); static PyObject * -typealias_reduce(typealiasobject *self, PyObject *Py_UNUSED(ignored)) +typealias_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return typealias_reduce_impl(self); + return typealias_reduce_impl((typealiasobject *)self); } PyDoc_STRVAR(typealias_new__doc__, @@ -571,7 +685,8 @@ typealias_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *value; PyObject *type_params = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 2, 2, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -591,4 +706,4 @@ typealias_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=5a582d9d89ad787b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f499d959a942c599 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h index 3e5167d9242fe4..5c6a425b0f803a 100644 --- a/Objects/clinic/unicodeobject.c.h +++ b/Objects/clinic/unicodeobject.c.h @@ -22,9 +22,9 @@ static PyObject * EncodingMap_size_impl(struct encoding_map *self); static PyObject * -EncodingMap_size(struct encoding_map *self, PyObject *Py_UNUSED(ignored)) +EncodingMap_size(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return EncodingMap_size_impl(self); + return EncodingMap_size_impl((struct encoding_map *)self); } PyDoc_STRVAR(unicode_title__doc__, @@ -136,6 +136,61 @@ unicode_center(PyObject *self, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(unicode_count__doc__, +"count($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the number of non-overlapping occurrences of substring sub in string S[start:end].\n" +"\n" +"Optional arguments start and end are interpreted as in slice notation."); + +#define UNICODE_COUNT_METHODDEF \ + {"count", _PyCFunction_CAST(unicode_count), METH_FASTCALL, unicode_count__doc__}, + +static Py_ssize_t +unicode_count_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_count(PyObject *str, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *substr; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + Py_ssize_t _return_value; + + if (!_PyArg_CheckPositional("count", nargs, 1, 3)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("count", "argument 1", "str", args[0]); + goto exit; + } + substr = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + _return_value = unicode_count_impl(str, substr, start, end); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(unicode_encode__doc__, "encode($self, /, encoding=\'utf-8\', errors=\'strict\')\n" "--\n" @@ -191,7 +246,8 @@ unicode_encode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject const char *encoding = NULL; const char *errors = NULL; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -283,7 +339,8 @@ unicode_expandtabs(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyOb Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; int tabsize = 8; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -301,6 +358,118 @@ unicode_expandtabs(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyOb return return_value; } +PyDoc_STRVAR(unicode_find__doc__, +"find($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end].\n" +"\n" +"Optional arguments start and end are interpreted as in slice notation.\n" +"Return -1 on failure."); + +#define UNICODE_FIND_METHODDEF \ + {"find", _PyCFunction_CAST(unicode_find), METH_FASTCALL, unicode_find__doc__}, + +static Py_ssize_t +unicode_find_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_find(PyObject *str, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *substr; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + Py_ssize_t _return_value; + + if (!_PyArg_CheckPositional("find", nargs, 1, 3)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("find", "argument 1", "str", args[0]); + goto exit; + } + substr = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + _return_value = unicode_find_impl(str, substr, start, end); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(unicode_index__doc__, +"index($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end].\n" +"\n" +"Optional arguments start and end are interpreted as in slice notation.\n" +"Raises ValueError when the substring is not found."); + +#define UNICODE_INDEX_METHODDEF \ + {"index", _PyCFunction_CAST(unicode_index), METH_FASTCALL, unicode_index__doc__}, + +static Py_ssize_t +unicode_index_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_index(PyObject *str, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *substr; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + Py_ssize_t _return_value; + + if (!_PyArg_CheckPositional("index", nargs, 1, 3)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("index", "argument 1", "str", args[0]); + goto exit; + } + substr = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + _return_value = unicode_index_impl(str, substr, start, end); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(unicode_isascii__doc__, "isascii($self, /)\n" "--\n" @@ -791,7 +960,8 @@ unicode_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObjec PyObject *new; Py_ssize_t count = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -892,6 +1062,118 @@ unicode_removesuffix(PyObject *self, PyObject *arg) return return_value; } +PyDoc_STRVAR(unicode_rfind__doc__, +"rfind($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the highest index in S where substring sub is found, such that sub is contained within S[start:end].\n" +"\n" +"Optional arguments start and end are interpreted as in slice notation.\n" +"Return -1 on failure."); + +#define UNICODE_RFIND_METHODDEF \ + {"rfind", _PyCFunction_CAST(unicode_rfind), METH_FASTCALL, unicode_rfind__doc__}, + +static Py_ssize_t +unicode_rfind_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_rfind(PyObject *str, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *substr; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + Py_ssize_t _return_value; + + if (!_PyArg_CheckPositional("rfind", nargs, 1, 3)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("rfind", "argument 1", "str", args[0]); + goto exit; + } + substr = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + _return_value = unicode_rfind_impl(str, substr, start, end); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(unicode_rindex__doc__, +"rindex($self, sub[, start[, end]], /)\n" +"--\n" +"\n" +"Return the highest index in S where substring sub is found, such that sub is contained within S[start:end].\n" +"\n" +"Optional arguments start and end are interpreted as in slice notation.\n" +"Raises ValueError when the substring is not found."); + +#define UNICODE_RINDEX_METHODDEF \ + {"rindex", _PyCFunction_CAST(unicode_rindex), METH_FASTCALL, unicode_rindex__doc__}, + +static Py_ssize_t +unicode_rindex_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_rindex(PyObject *str, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *substr; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + Py_ssize_t _return_value; + + if (!_PyArg_CheckPositional("rindex", nargs, 1, 3)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("rindex", "argument 1", "str", args[0]); + goto exit; + } + substr = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + _return_value = unicode_rindex_impl(str, substr, start, end); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(unicode_rjust__doc__, "rjust($self, width, fillchar=\' \', /)\n" "--\n" @@ -1003,7 +1285,8 @@ unicode_split(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *sep = Py_None; Py_ssize_t maxsplit = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1125,7 +1408,8 @@ unicode_rsplit(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *sep = Py_None; Py_ssize_t maxsplit = -1; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1205,7 +1489,8 @@ unicode_splitlines(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyOb Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; int keepends = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1369,6 +1654,108 @@ unicode_zfill(PyObject *self, PyObject *arg) return return_value; } +PyDoc_STRVAR(unicode_startswith__doc__, +"startswith($self, prefix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the string starts with the specified prefix, False otherwise.\n" +"\n" +" prefix\n" +" A string or a tuple of strings to try.\n" +" start\n" +" Optional start position. Default: start of the string.\n" +" end\n" +" Optional stop position. Default: end of the string."); + +#define UNICODE_STARTSWITH_METHODDEF \ + {"startswith", _PyCFunction_CAST(unicode_startswith), METH_FASTCALL, unicode_startswith__doc__}, + +static PyObject * +unicode_startswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_startswith(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("startswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = unicode_startswith_impl(self, subobj, start, end); + +exit: + return return_value; +} + +PyDoc_STRVAR(unicode_endswith__doc__, +"endswith($self, suffix[, start[, end]], /)\n" +"--\n" +"\n" +"Return True if the string ends with the specified suffix, False otherwise.\n" +"\n" +" suffix\n" +" A string or a tuple of strings to try.\n" +" start\n" +" Optional start position. Default: start of the string.\n" +" end\n" +" Optional stop position. Default: end of the string."); + +#define UNICODE_ENDSWITH_METHODDEF \ + {"endswith", _PyCFunction_CAST(unicode_endswith), METH_FASTCALL, unicode_endswith__doc__}, + +static PyObject * +unicode_endswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +unicode_endswith(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *subobj; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("endswith", nargs, 1, 3)) { + goto exit; + } + subobj = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndex(args[2], &end)) { + goto exit; + } +skip_optional: + return_value = unicode_endswith_impl(self, subobj, start, end); + +exit: + return return_value; +} + PyDoc_STRVAR(unicode___format____doc__, "__format__($self, format_spec, /)\n" "--\n" @@ -1457,7 +1844,8 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) const char *encoding = NULL; const char *errors = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 3, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -1507,4 +1895,4 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=1aab29bab5201c78 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4d1cecd6d08498a4 input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 30336fa86111a7..7eea61968bc4d9 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -5,14 +5,23 @@ #include "pycore_code.h" // _PyCodeConstructor #include "pycore_frame.h" // FRAME_SPECIALS_SIZE +#include "pycore_hashtable.h" // _Py_hashtable_t +#include "pycore_index_pool.h" // _PyIndexPool +#include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // PyInterpreterState.co_extra_freefuncs +#include "pycore_object.h" // _PyObject_SetDeferredRefcount +#include "pycore_object_stack.h" #include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches #include "pycore_opcode_utils.h" // RESUME_AT_FUNC_START +#include "pycore_pymem.h" // _PyMem_FreeDelayed #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_uniqueid.h" // _PyObject_AssignUniqueId() #include "clinic/codeobject.c.h" +#define INITIAL_SPECIALIZED_CODE_SIZE 16 + static const char * code_event_name(PyCodeEvent event) { switch (event) { @@ -99,10 +108,16 @@ PyCode_ClearWatcher(int watcher_id) * generic helpers ******************/ -/* all_name_chars(s): true iff s matches [a-zA-Z0-9_]* */ +#define _PyCodeObject_CAST(op) (assert(PyCode_Check(op)), (PyCodeObject *)(op)) + static int -all_name_chars(PyObject *o) +should_intern_string(PyObject *o) { +#ifdef Py_GIL_DISABLED + // The free-threaded build interns (and immortalizes) all string constants + return 1; +#else + // compute if s matches [a-zA-Z0-9_] const unsigned char *s, *e; if (!PyUnicode_IS_ASCII(o)) @@ -115,11 +130,17 @@ all_name_chars(PyObject *o) return 0; } return 1; +#endif } +#ifdef Py_GIL_DISABLED +static PyObject *intern_one_constant(PyObject *op); +#endif + static int intern_strings(PyObject *tuple) { + PyInterpreterState *interp = _PyInterpreterState_GET(); Py_ssize_t i; for (i = PyTuple_GET_SIZE(tuple); --i >= 0; ) { @@ -129,21 +150,24 @@ intern_strings(PyObject *tuple) "non-string found in code slot"); return -1; } - PyUnicode_InternInPlace(&_PyTuple_ITEMS(tuple)[i]); + _PyUnicode_InternImmortal(interp, &_PyTuple_ITEMS(tuple)[i]); } return 0; } -/* Intern selected string constants */ +/* Intern constants. In the default build, this interns selected string + constants. In the free-threaded build, this also interns non-string + constants. */ static int -intern_string_constants(PyObject *tuple, int *modified) +intern_constants(PyObject *tuple, int *modified) { + PyInterpreterState *interp = _PyInterpreterState_GET(); for (Py_ssize_t i = PyTuple_GET_SIZE(tuple); --i >= 0; ) { PyObject *v = PyTuple_GET_ITEM(tuple, i); if (PyUnicode_CheckExact(v)) { - if (all_name_chars(v)) { + if (should_intern_string(v)) { PyObject *w = v; - PyUnicode_InternInPlace(&v); + _PyUnicode_InternMortal(interp, &v); if (w != v) { PyTuple_SET_ITEM(tuple, i, v); if (modified) { @@ -153,7 +177,7 @@ intern_string_constants(PyObject *tuple, int *modified) } } else if (PyTuple_CheckExact(v)) { - if (intern_string_constants(v, NULL) < 0) { + if (intern_constants(v, NULL) < 0) { return -1; } } @@ -164,7 +188,7 @@ intern_string_constants(PyObject *tuple, int *modified) return -1; } int tmp_modified = 0; - if (intern_string_constants(tmp, &tmp_modified) < 0) { + if (intern_constants(tmp, &tmp_modified) < 0) { Py_DECREF(tmp); return -1; } @@ -183,6 +207,56 @@ intern_string_constants(PyObject *tuple, int *modified) } Py_DECREF(tmp); } +#ifdef Py_GIL_DISABLED + else if (PySlice_Check(v)) { + PySliceObject *slice = (PySliceObject *)v; + PyObject *tmp = PyTuple_New(3); + if (tmp == NULL) { + return -1; + } + PyTuple_SET_ITEM(tmp, 0, Py_NewRef(slice->start)); + PyTuple_SET_ITEM(tmp, 1, Py_NewRef(slice->stop)); + PyTuple_SET_ITEM(tmp, 2, Py_NewRef(slice->step)); + int tmp_modified = 0; + if (intern_constants(tmp, &tmp_modified) < 0) { + Py_DECREF(tmp); + return -1; + } + if (tmp_modified) { + v = PySlice_New(PyTuple_GET_ITEM(tmp, 0), + PyTuple_GET_ITEM(tmp, 1), + PyTuple_GET_ITEM(tmp, 2)); + if (v == NULL) { + Py_DECREF(tmp); + return -1; + } + PyTuple_SET_ITEM(tuple, i, v); + Py_DECREF(slice); + if (modified) { + *modified = 1; + } + } + Py_DECREF(tmp); + } + + // Intern non-string constants in the free-threaded build + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + if (!_Py_IsImmortal(v) && !PyCode_Check(v) && + !PyUnicode_CheckExact(v) && !tstate->suppress_co_const_immortalization) + { + PyObject *interned = intern_one_constant(v); + if (interned == NULL) { + return -1; + } + else if (interned != v) { + PyTuple_SET_ITEM(tuple, i, interned); + Py_SETREF(v, interned); + if (modified) { + *modified = 1; + } + } + } +#endif } return 0; } @@ -230,21 +304,32 @@ validate_and_copy_tuple(PyObject *tup) } static int -init_co_cached(PyCodeObject *self) { - if (self->_co_cached == NULL) { - self->_co_cached = PyMem_New(_PyCoCached, 1); - if (self->_co_cached == NULL) { +init_co_cached(PyCodeObject *self) +{ + _PyCoCached *cached = FT_ATOMIC_LOAD_PTR(self->_co_cached); + if (cached != NULL) { + return 0; + } + + Py_BEGIN_CRITICAL_SECTION(self); + cached = self->_co_cached; + if (cached == NULL) { + cached = PyMem_New(_PyCoCached, 1); + if (cached == NULL) { PyErr_NoMemory(); - return -1; } - self->_co_cached->_co_code = NULL; - self->_co_cached->_co_cellvars = NULL; - self->_co_cached->_co_freevars = NULL; - self->_co_cached->_co_varnames = NULL; + else { + cached->_co_code = NULL; + cached->_co_cellvars = NULL; + cached->_co_freevars = NULL; + cached->_co_varnames = NULL; + FT_ATOMIC_STORE_PTR(self->_co_cached, cached); + } } - return 0; - + Py_END_CRITICAL_SECTION(); + return cached != NULL ? 0 : -1; } + /****************** * _PyCode_New() ******************/ @@ -373,9 +458,15 @@ _PyCode_Validate(struct _PyCodeConstructor *con) return 0; } -extern void _PyCode_Quicken(PyCodeObject *code); +extern void +_PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, PyObject *consts, + int enable_counters); -static void +#ifdef Py_GIL_DISABLED +static _PyCodeArray * _PyCodeArray_New(Py_ssize_t size); +#endif + +static int init_code(PyCodeObject *co, struct _PyCodeConstructor *con) { int nlocalsplus = (int)PyTuple_GET_SIZE(con->localsplusnames); @@ -386,9 +477,13 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) con->stacksize = 1; } + PyInterpreterState *interp = _PyInterpreterState_GET(); co->co_filename = Py_NewRef(con->filename); co->co_name = Py_NewRef(con->name); co->co_qualname = Py_NewRef(con->qualname); + _PyUnicode_InternMortal(interp, &co->co_filename); + _PyUnicode_InternMortal(interp, &co->co_name); + _PyUnicode_InternMortal(interp, &co->co_qualname); co->co_flags = con->flags; co->co_firstlineno = con->firstlineno; @@ -414,11 +509,16 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_framesize = nlocalsplus + con->stacksize + FRAME_SPECIALS_SIZE; co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; - PyInterpreterState *interp = _PyInterpreterState_GET(); - co->co_version = interp->next_func_version; - if (interp->next_func_version != 0) { - interp->next_func_version++; - } +#ifdef Py_GIL_DISABLED + PyMutex_Lock(&interp->func_state.mutex); +#endif + co->co_version = interp->func_state.next_version; + if (interp->func_state.next_version != 0) { + interp->func_state.next_version++; + } +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&interp->func_state.mutex); +#endif co->_co_monitoring = NULL; co->_co_instrumentation_version = 0; /* not set */ @@ -429,14 +529,27 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) memcpy(_PyCode_CODE(co), PyBytes_AS_STRING(con->code), PyBytes_GET_SIZE(con->code)); +#ifdef Py_GIL_DISABLED + co->co_tlbc = _PyCodeArray_New(INITIAL_SPECIALIZED_CODE_SIZE); + if (co->co_tlbc == NULL) { + return -1; + } + co->co_tlbc->entries[0] = co->co_code_adaptive; +#endif int entry_point = 0; while (entry_point < Py_SIZE(co) && _PyCode_CODE(co)[entry_point].op.code != RESUME) { entry_point++; } co->_co_firsttraceable = entry_point; - _PyCode_Quicken(co); +#ifdef Py_GIL_DISABLED + _PyCode_Quicken(_PyCode_CODE(co), Py_SIZE(co), co->co_consts, + interp->config.tlbc_enabled); +#else + _PyCode_Quicken(_PyCode_CODE(co), Py_SIZE(co), co->co_consts, 1); +#endif notify_code_watchers(PY_CODE_EVENT_CREATE, co); + return 0; } static int @@ -490,7 +603,7 @@ get_line_delta(const uint8_t *ptr) static PyObject * remove_column_info(PyObject *locations) { - int offset = 0; + Py_ssize_t offset = 0; const uint8_t *data = (const uint8_t *)PyBytes_AS_STRING(locations); PyObject *res = PyBytes_FromStringAndSize(NULL, 32); if (res == NULL) { @@ -530,18 +643,41 @@ remove_column_info(PyObject *locations) return res; } -/* The caller is responsible for ensuring that the given data is valid. */ - -PyCodeObject * -_PyCode_New(struct _PyCodeConstructor *con) +static int +intern_code_constants(struct _PyCodeConstructor *con) { +#ifdef Py_GIL_DISABLED + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _py_code_state *state = &interp->code_state; + PyMutex_Lock(&state->mutex); +#endif if (intern_strings(con->names) < 0) { - return NULL; + goto error; } - if (intern_string_constants(con->consts, NULL) < 0) { - return NULL; + if (intern_constants(con->consts, NULL) < 0) { + goto error; } if (intern_strings(con->localsplusnames) < 0) { + goto error; + } +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&state->mutex); +#endif + return 0; + +error: +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&state->mutex); +#endif + return -1; +} + +/* The caller is responsible for ensuring that the given data is valid. */ + +PyCodeObject * +_PyCode_New(struct _PyCodeConstructor *con) +{ + if (intern_code_constants(con) < 0) { return NULL; } @@ -557,13 +693,27 @@ _PyCode_New(struct _PyCodeConstructor *con) } Py_ssize_t size = PyBytes_GET_SIZE(con->code) / sizeof(_Py_CODEUNIT); - PyCodeObject *co = PyObject_NewVar(PyCodeObject, &PyCode_Type, size); + PyCodeObject *co; +#ifdef Py_GIL_DISABLED + co = PyObject_GC_NewVar(PyCodeObject, &PyCode_Type, size); +#else + co = PyObject_NewVar(PyCodeObject, &PyCode_Type, size); +#endif if (co == NULL) { Py_XDECREF(replacement_locations); PyErr_NoMemory(); return NULL; } - init_code(co, con); + + if (init_code(co, con) < 0) { + Py_DECREF(co); + return NULL; + } + +#ifdef Py_GIL_DISABLED + co->_co_unique_id = _PyObject_AssignUniqueId((PyObject *)co); + _PyObject_GC_TRACK(co); +#endif Py_XDECREF(replacement_locations); return co; } @@ -750,7 +900,7 @@ PyUnstable_Code_New(int argcount, int kwonlyargcount, static const uint8_t assert0[6] = { RESUME, RESUME_AT_FUNC_START, - LOAD_ASSERTION_ERROR, 0, + LOAD_COMMON_CONSTANT, CONSTANT_ASSERTIONERROR, RAISE_VARARGS, 1 }; @@ -829,6 +979,9 @@ PyCode_Addr2Line(PyCodeObject *co, int addrq) if (addrq < 0) { return co->co_firstlineno; } + if (co->_co_monitoring && co->_co_monitoring->lines) { + return _Py_Instrumentation_GetLine(co, addrq/sizeof(_Py_CODEUNIT)); + } assert(addrq >= 0 && addrq < _PyCode_NBYTES(co)); PyCodeAddressRange bounds; _PyCode_InitAddressRange(co, &bounds); @@ -1163,8 +1316,9 @@ typedef struct { static void -lineiter_dealloc(lineiterator *li) +lineiter_dealloc(PyObject *self) { + lineiterator *li = (lineiterator*)self; Py_DECREF(li->li_code); Py_TYPE(li)->tp_free(li); } @@ -1178,8 +1332,9 @@ _source_offset_converter(int *value) { } static PyObject * -lineiter_next(lineiterator *li) +lineiter_next(PyObject *self) { + lineiterator *li = (lineiterator*)self; PyCodeAddressRange *bounds = &li->li_line; if (!_PyLineTable_NextAddressRange(bounds)) { return NULL; @@ -1203,7 +1358,7 @@ PyTypeObject _PyLineIterator = { sizeof(lineiterator), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)lineiter_dealloc, /* tp_dealloc */ + lineiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -1225,7 +1380,7 @@ PyTypeObject _PyLineIterator = { 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)lineiter_next, /* tp_iternext */ + lineiter_next, /* tp_iternext */ 0, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ @@ -1237,7 +1392,7 @@ PyTypeObject _PyLineIterator = { 0, /* tp_init */ 0, /* tp_alloc */ 0, /* tp_new */ - PyObject_Del, /* tp_free */ + PyObject_Free, /* tp_free */ }; static lineiterator * @@ -1264,15 +1419,17 @@ typedef struct { } positionsiterator; static void -positionsiter_dealloc(positionsiterator* pi) +positionsiter_dealloc(PyObject *self) { + positionsiterator *pi = (positionsiterator*)self; Py_DECREF(pi->pi_code); Py_TYPE(pi)->tp_free(pi); } static PyObject* -positionsiter_next(positionsiterator* pi) +positionsiter_next(PyObject *self) { + positionsiterator *pi = (positionsiterator*)self; if (pi->pi_offset >= pi->pi_range.ar_end) { assert(pi->pi_offset == pi->pi_range.ar_end); if (at_end(&pi->pi_range)) { @@ -1294,7 +1451,7 @@ PyTypeObject _PyPositionsIterator = { sizeof(positionsiterator), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)positionsiter_dealloc, /* tp_dealloc */ + positionsiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -1316,7 +1473,7 @@ PyTypeObject _PyPositionsIterator = { 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)positionsiter_next, /* tp_iternext */ + positionsiter_next, /* tp_iternext */ 0, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ @@ -1328,12 +1485,13 @@ PyTypeObject _PyPositionsIterator = { 0, /* tp_init */ 0, /* tp_alloc */ 0, /* tp_new */ - PyObject_Del, /* tp_free */ + PyObject_Free, /* tp_free */ }; static PyObject* -code_positionsiterator(PyCodeObject* code, PyObject* Py_UNUSED(args)) +code_positionsiterator(PyObject *self, PyObject* Py_UNUSED(args)) { + PyCodeObject *code = (PyCodeObject*)self; positionsiterator* pi = (positionsiterator*)PyType_GenericAlloc(&_PyPositionsIterator, 0); if (pi == NULL) { return NULL; @@ -1429,16 +1587,21 @@ get_cached_locals(PyCodeObject *co, PyObject **cached_field, { assert(cached_field != NULL); assert(co->_co_cached != NULL); - if (*cached_field != NULL) { - return Py_NewRef(*cached_field); + PyObject *varnames = FT_ATOMIC_LOAD_PTR(*cached_field); + if (varnames != NULL) { + return Py_NewRef(varnames); } - assert(*cached_field == NULL); - PyObject *varnames = get_localsplus_names(co, kind, num); + + Py_BEGIN_CRITICAL_SECTION(co); + varnames = *cached_field; if (varnames == NULL) { - return NULL; + varnames = get_localsplus_names(co, kind, num); + if (varnames != NULL) { + FT_ATOMIC_STORE_PTR(*cached_field, varnames); + } } - *cached_field = Py_NewRef(varnames); - return varnames; + Py_END_CRITICAL_SECTION(); + return Py_XNewRef(varnames); } PyObject * @@ -1486,13 +1649,16 @@ PyCode_GetFreevars(PyCodeObject *code) return _PyCode_GetFreevars(code); } +#ifdef _Py_TIER2 + static void clear_executors(PyCodeObject *co) { assert(co->co_executors); for (int i = 0; i < co->co_executors->size; i++) { if (co->co_executors->executors[i]) { - _Py_ExecutorClear(co->co_executors->executors[i]); + _Py_ExecutorDetach(co->co_executors->executors[i]); + assert(co->co_executors->executors[i] == NULL); } } PyMem_Free(co->co_executors); @@ -1505,20 +1671,17 @@ _PyCode_Clear_Executors(PyCodeObject *code) clear_executors(code); } +#endif + static void deopt_code(PyCodeObject *code, _Py_CODEUNIT *instructions) { Py_ssize_t len = Py_SIZE(code); for (int i = 0; i < len; i++) { - int opcode = _Py_GetBaseOpcode(code, i); - if (opcode == ENTER_EXECUTOR) { - _PyExecutorObject *exec = code->co_executors->executors[instructions[i].op.arg]; - opcode = _PyOpcode_Deopt[exec->vm_data.opcode]; - instructions[i].op.arg = exec->vm_data.oparg; - } - assert(opcode != ENTER_EXECUTOR); - int caches = _PyOpcode_Caches[opcode]; - instructions[i].op.code = opcode; + _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(code, i); + assert(inst.op.code < MIN_SPECIALIZED_OPCODE); + int caches = _PyOpcode_Caches[inst.op.code]; + instructions[i] = inst; for (int j = 1; j <= caches; j++) { instructions[i+j].cache = 0; } @@ -1532,18 +1695,26 @@ _PyCode_GetCode(PyCodeObject *co) if (init_co_cached(co)) { return NULL; } - if (co->_co_cached->_co_code != NULL) { - return Py_NewRef(co->_co_cached->_co_code); + + _PyCoCached *cached = co->_co_cached; + PyObject *code = FT_ATOMIC_LOAD_PTR(cached->_co_code); + if (code != NULL) { + return Py_NewRef(code); } - PyObject *code = PyBytes_FromStringAndSize((const char *)_PyCode_CODE(co), - _PyCode_NBYTES(co)); + + Py_BEGIN_CRITICAL_SECTION(co); + code = cached->_co_code; if (code == NULL) { - return NULL; + code = PyBytes_FromStringAndSize((const char *)_PyCode_CODE(co), + _PyCode_NBYTES(co)); + if (code != NULL) { + deopt_code(co, (_Py_CODEUNIT *)PyBytes_AS_STRING(code)); + assert(cached->_co_code == NULL); + FT_ATOMIC_STORE_PTR(cached->_co_code, code); + } } - deopt_code(co, (_Py_CODEUNIT *)PyBytes_AS_STRING(code)); - assert(co->_co_cached->_co_code == NULL); - co->_co_cached->_co_code = Py_NewRef(code); - return code; + Py_END_CRITICAL_SECTION(); + return Py_XNewRef(code); } PyObject * @@ -1699,17 +1870,20 @@ free_monitoring_data(_PyCoMonitoringData *data) } static void -code_dealloc(PyCodeObject *co) +code_dealloc(PyObject *self) { - assert(Py_REFCNT(co) == 0); - Py_SET_REFCNT(co, 1); + PyCodeObject *co = _PyCodeObject_CAST(self); + _PyObject_ResurrectStart(self); notify_code_watchers(PY_CODE_EVENT_DESTROY, co); - if (Py_REFCNT(co) > 1) { - Py_SET_REFCNT(co, Py_REFCNT(co) - 1); + if (_PyObject_ResurrectEnd(self)) { return; } - Py_SET_REFCNT(co, 0); +#ifdef Py_GIL_DISABLED + PyObject_GC_UnTrack(co); +#endif + + _PyFunction_ClearCodeByVersion(co->co_version); if (co->co_extra != NULL) { PyInterpreterState *interp = _PyInterpreterState_GET(); _PyCodeObjectExtra *co_extra = co->co_extra; @@ -1724,9 +1898,11 @@ code_dealloc(PyCodeObject *co) PyMem_Free(co_extra); } +#ifdef _Py_TIER2 if (co->co_executors != NULL) { clear_executors(co); } +#endif Py_XDECREF(co->co_consts); Py_XDECREF(co->co_names); @@ -1737,6 +1913,9 @@ code_dealloc(PyCodeObject *co) Py_XDECREF(co->co_qualname); Py_XDECREF(co->co_linetable); Py_XDECREF(co->co_exceptiontable); +#ifdef Py_GIL_DISABLED + assert(co->_co_unique_id == _Py_INVALID_UNIQUE_ID); +#endif if (co->_co_cached != NULL) { Py_XDECREF(co->_co_cached->_co_code); Py_XDECREF(co->_co_cached->_co_cellvars); @@ -1745,15 +1924,37 @@ code_dealloc(PyCodeObject *co) PyMem_Free(co->_co_cached); } if (co->co_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject*)co); + PyObject_ClearWeakRefs(self); } free_monitoring_data(co->_co_monitoring); +#ifdef Py_GIL_DISABLED + // The first element always points to the mutable bytecode at the end of + // the code object, which will be freed when the code object is freed. + for (Py_ssize_t i = 1; i < co->co_tlbc->size; i++) { + char *entry = co->co_tlbc->entries[i]; + if (entry != NULL) { + PyMem_Free(entry); + } + } + PyMem_Free(co->co_tlbc); +#endif PyObject_Free(co); } +#ifdef Py_GIL_DISABLED +static int +code_traverse(PyObject *self, visitproc visit, void *arg) +{ + PyCodeObject *co = _PyCodeObject_CAST(self); + Py_VISIT(co->co_consts); + return 0; +} +#endif + static PyObject * -code_repr(PyCodeObject *co) +code_repr(PyObject *self) { + PyCodeObject *co = _PyCodeObject_CAST(self); int lineno; if (co->co_firstlineno != 0) lineno = co->co_firstlineno; @@ -1804,33 +2005,12 @@ code_richcompare(PyObject *self, PyObject *other, int op) goto unequal; } for (int i = 0; i < Py_SIZE(co); i++) { - _Py_CODEUNIT co_instr = _PyCode_CODE(co)[i]; - _Py_CODEUNIT cp_instr = _PyCode_CODE(cp)[i]; - uint8_t co_code = _Py_GetBaseOpcode(co, i); - uint8_t co_arg = co_instr.op.arg; - uint8_t cp_code = _Py_GetBaseOpcode(cp, i); - uint8_t cp_arg = cp_instr.op.arg; - - if (co_code == ENTER_EXECUTOR) { - const int exec_index = co_arg; - _PyExecutorObject *exec = co->co_executors->executors[exec_index]; - co_code = _PyOpcode_Deopt[exec->vm_data.opcode]; - co_arg = exec->vm_data.oparg; - } - assert(co_code != ENTER_EXECUTOR); - - if (cp_code == ENTER_EXECUTOR) { - const int exec_index = cp_arg; - _PyExecutorObject *exec = cp->co_executors->executors[exec_index]; - cp_code = _PyOpcode_Deopt[exec->vm_data.opcode]; - cp_arg = exec->vm_data.oparg; - } - assert(cp_code != ENTER_EXECUTOR); - - if (co_code != cp_code || co_arg != cp_arg) { + _Py_CODEUNIT co_instr = _Py_GetBaseCodeUnit(co, i); + _Py_CODEUNIT cp_instr = _Py_GetBaseCodeUnit(cp, i); + if (co_instr.cache != cp_instr.cache) { goto unequal; } - i += _PyOpcode_Caches[co_code]; + i += _PyOpcode_Caches[co_instr.op.code]; } /* compare constants */ @@ -1881,12 +2061,13 @@ code_richcompare(PyObject *self, PyObject *other, int op) } static Py_hash_t -code_hash(PyCodeObject *co) +code_hash(PyObject *self) { + PyCodeObject *co = _PyCodeObject_CAST(self); Py_uhash_t uhash = 20221211; #define SCRAMBLE_IN(H) do { \ uhash ^= (Py_uhash_t)(H); \ - uhash *= _PyHASH_MULTIPLIER; \ + uhash *= PyHASH_MULTIPLIER; \ } while (0) #define SCRAMBLE_IN_HASH(EXPR) do { \ Py_hash_t h = PyObject_Hash(EXPR); \ @@ -1909,22 +2090,10 @@ code_hash(PyCodeObject *co) SCRAMBLE_IN(co->co_firstlineno); SCRAMBLE_IN(Py_SIZE(co)); for (int i = 0; i < Py_SIZE(co); i++) { - _Py_CODEUNIT co_instr = _PyCode_CODE(co)[i]; - uint8_t co_code = co_instr.op.code; - uint8_t co_arg = co_instr.op.arg; - if (co_code == ENTER_EXECUTOR) { - _PyExecutorObject *exec = co->co_executors->executors[co_arg]; - assert(exec != NULL); - assert(exec->vm_data.opcode != ENTER_EXECUTOR); - co_code = _PyOpcode_Deopt[exec->vm_data.opcode]; - co_arg = exec->vm_data.oparg; - } - else { - co_code = _Py_GetBaseOpcode(co, i); - } - SCRAMBLE_IN(co_code); - SCRAMBLE_IN(co_arg); - i += _PyOpcode_Caches[co_code]; + _Py_CODEUNIT co_instr = _Py_GetBaseCodeUnit(co, i); + SCRAMBLE_IN(co_instr.op.code); + SCRAMBLE_IN(co_instr.op.arg); + i += _PyOpcode_Caches[co_instr.op.code]; } if ((Py_hash_t)uhash == -1) { return -2; @@ -1955,8 +2124,9 @@ static PyMemberDef code_memberlist[] = { static PyObject * -code_getlnotab(PyCodeObject *code, void *closure) +code_getlnotab(PyObject *self, void *closure) { + PyCodeObject *code = _PyCodeObject_CAST(self); if (PyErr_WarnEx(PyExc_DeprecationWarning, "co_lnotab is deprecated, use co_lines instead.", 1) < 0) { @@ -1966,51 +2136,57 @@ code_getlnotab(PyCodeObject *code, void *closure) } static PyObject * -code_getvarnames(PyCodeObject *code, void *closure) +code_getvarnames(PyObject *self, void *closure) { + PyCodeObject *code = _PyCodeObject_CAST(self); return _PyCode_GetVarnames(code); } static PyObject * -code_getcellvars(PyCodeObject *code, void *closure) +code_getcellvars(PyObject *self, void *closure) { + PyCodeObject *code = _PyCodeObject_CAST(self); return _PyCode_GetCellvars(code); } static PyObject * -code_getfreevars(PyCodeObject *code, void *closure) +code_getfreevars(PyObject *self, void *closure) { + PyCodeObject *code = _PyCodeObject_CAST(self); return _PyCode_GetFreevars(code); } static PyObject * -code_getcodeadaptive(PyCodeObject *code, void *closure) +code_getcodeadaptive(PyObject *self, void *closure) { + PyCodeObject *code = _PyCodeObject_CAST(self); return PyBytes_FromStringAndSize(code->co_code_adaptive, _PyCode_NBYTES(code)); } static PyObject * -code_getcode(PyCodeObject *code, void *closure) +code_getcode(PyObject *self, void *closure) { + PyCodeObject *code = _PyCodeObject_CAST(self); return _PyCode_GetCode(code); } static PyGetSetDef code_getsetlist[] = { - {"co_lnotab", (getter)code_getlnotab, NULL, NULL}, - {"_co_code_adaptive", (getter)code_getcodeadaptive, NULL, NULL}, + {"co_lnotab", code_getlnotab, NULL, NULL}, + {"_co_code_adaptive", code_getcodeadaptive, NULL, NULL}, // The following old names are kept for backward compatibility. - {"co_varnames", (getter)code_getvarnames, NULL, NULL}, - {"co_cellvars", (getter)code_getcellvars, NULL, NULL}, - {"co_freevars", (getter)code_getfreevars, NULL, NULL}, - {"co_code", (getter)code_getcode, NULL, NULL}, + {"co_varnames", code_getvarnames, NULL, NULL}, + {"co_cellvars", code_getcellvars, NULL, NULL}, + {"co_freevars", code_getfreevars, NULL, NULL}, + {"co_code", code_getcode, NULL, NULL}, {0} }; static PyObject * -code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args)) +code_sizeof(PyObject *self, PyObject *Py_UNUSED(args)) { + PyCodeObject *co = _PyCodeObject_CAST(self); size_t res = _PyObject_VAR_SIZE(Py_TYPE(co), Py_SIZE(co)); _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra*) co->co_extra; if (co_extra != NULL) { @@ -2021,34 +2197,42 @@ code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args)) } static PyObject * -code_linesiterator(PyCodeObject *code, PyObject *Py_UNUSED(args)) +code_linesiterator(PyObject *self, PyObject *Py_UNUSED(args)) { + PyCodeObject *code = _PyCodeObject_CAST(self); return (PyObject *)new_linesiterator(code); } +static PyObject * +code_branchesiterator(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyCodeObject *code = _PyCodeObject_CAST(self); + return _PyInstrumentation_BranchesIterator(code); +} + /*[clinic input] @text_signature "($self, /, **changes)" code.replace * - co_argcount: int(c_default="self->co_argcount") = unchanged - co_posonlyargcount: int(c_default="self->co_posonlyargcount") = unchanged - co_kwonlyargcount: int(c_default="self->co_kwonlyargcount") = unchanged - co_nlocals: int(c_default="self->co_nlocals") = unchanged - co_stacksize: int(c_default="self->co_stacksize") = unchanged - co_flags: int(c_default="self->co_flags") = unchanged - co_firstlineno: int(c_default="self->co_firstlineno") = unchanged + co_argcount: int(c_default="((PyCodeObject *)self)->co_argcount") = unchanged + co_posonlyargcount: int(c_default="((PyCodeObject *)self)->co_posonlyargcount") = unchanged + co_kwonlyargcount: int(c_default="((PyCodeObject *)self)->co_kwonlyargcount") = unchanged + co_nlocals: int(c_default="((PyCodeObject *)self)->co_nlocals") = unchanged + co_stacksize: int(c_default="((PyCodeObject *)self)->co_stacksize") = unchanged + co_flags: int(c_default="((PyCodeObject *)self)->co_flags") = unchanged + co_firstlineno: int(c_default="((PyCodeObject *)self)->co_firstlineno") = unchanged co_code: object(subclass_of="&PyBytes_Type", c_default="NULL") = unchanged - co_consts: object(subclass_of="&PyTuple_Type", c_default="self->co_consts") = unchanged - co_names: object(subclass_of="&PyTuple_Type", c_default="self->co_names") = unchanged + co_consts: object(subclass_of="&PyTuple_Type", c_default="((PyCodeObject *)self)->co_consts") = unchanged + co_names: object(subclass_of="&PyTuple_Type", c_default="((PyCodeObject *)self)->co_names") = unchanged co_varnames: object(subclass_of="&PyTuple_Type", c_default="NULL") = unchanged co_freevars: object(subclass_of="&PyTuple_Type", c_default="NULL") = unchanged co_cellvars: object(subclass_of="&PyTuple_Type", c_default="NULL") = unchanged - co_filename: unicode(c_default="self->co_filename") = unchanged - co_name: unicode(c_default="self->co_name") = unchanged - co_qualname: unicode(c_default="self->co_qualname") = unchanged - co_linetable: object(subclass_of="&PyBytes_Type", c_default="self->co_linetable") = unchanged - co_exceptiontable: object(subclass_of="&PyBytes_Type", c_default="self->co_exceptiontable") = unchanged + co_filename: unicode(c_default="((PyCodeObject *)self)->co_filename") = unchanged + co_name: unicode(c_default="((PyCodeObject *)self)->co_name") = unchanged + co_qualname: unicode(c_default="((PyCodeObject *)self)->co_qualname") = unchanged + co_linetable: object(subclass_of="&PyBytes_Type", c_default="((PyCodeObject *)self)->co_linetable") = unchanged + co_exceptiontable: object(subclass_of="&PyBytes_Type", c_default="((PyCodeObject *)self)->co_exceptiontable") = unchanged Return a copy of the code object with new values for the specified fields. [clinic start generated code]*/ @@ -2063,7 +2247,7 @@ code_replace_impl(PyCodeObject *self, int co_argcount, PyObject *co_filename, PyObject *co_name, PyObject *co_qualname, PyObject *co_linetable, PyObject *co_exceptiontable) -/*[clinic end generated code: output=e75c48a15def18b9 input=18e280e07846c122]*/ +/*[clinic end generated code: output=e75c48a15def18b9 input=a455a89c57ac9d42]*/ { #define CHECK_INT_ARG(ARG) \ if (ARG < 0) { \ @@ -2164,12 +2348,14 @@ code__varname_from_oparg_impl(PyCodeObject *self, int oparg) /* XXX code objects need to participate in GC? */ static struct PyMethodDef code_methods[] = { - {"__sizeof__", (PyCFunction)code_sizeof, METH_NOARGS}, - {"co_lines", (PyCFunction)code_linesiterator, METH_NOARGS}, - {"co_positions", (PyCFunction)code_positionsiterator, METH_NOARGS}, + {"__sizeof__", code_sizeof, METH_NOARGS}, + {"co_lines", code_linesiterator, METH_NOARGS}, + {"co_branches", code_branchesiterator, METH_NOARGS}, + {"co_positions", code_positionsiterator, METH_NOARGS}, CODE_REPLACE_METHODDEF CODE__VARNAME_FROM_OPARG_METHODDEF - {"__replace__", _PyCFunction_CAST(code_replace), METH_FASTCALL|METH_KEYWORDS}, + {"__replace__", _PyCFunction_CAST(code_replace), METH_FASTCALL|METH_KEYWORDS, + PyDoc_STR("__replace__($self, /, **changes)\n--\n\nThe same as replace().")}, {NULL, NULL} /* sentinel */ }; @@ -2179,24 +2365,32 @@ PyTypeObject PyCode_Type = { "code", offsetof(PyCodeObject, co_code_adaptive), sizeof(_Py_CODEUNIT), - (destructor)code_dealloc, /* tp_dealloc */ + code_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)code_repr, /* tp_repr */ + code_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ - (hashfunc)code_hash, /* tp_hash */ + code_hash, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ +#ifdef Py_GIL_DISABLED + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ +#else Py_TPFLAGS_DEFAULT, /* tp_flags */ +#endif code_new__doc__, /* tp_doc */ +#ifdef Py_GIL_DISABLED + code_traverse, /* tp_traverse */ +#else 0, /* tp_traverse */ +#endif 0, /* tp_clear */ code_richcompare, /* tp_richcompare */ offsetof(PyCodeObject, co_weakreflist), /* tp_weaklistoffset */ @@ -2336,6 +2530,40 @@ _PyCode_ConstantKey(PyObject *op) Py_DECREF(set); return key; } + else if (PySlice_Check(op)) { + PySliceObject *slice = (PySliceObject *)op; + PyObject *start_key = NULL; + PyObject *stop_key = NULL; + PyObject *step_key = NULL; + key = NULL; + + start_key = _PyCode_ConstantKey(slice->start); + if (start_key == NULL) { + goto slice_exit; + } + + stop_key = _PyCode_ConstantKey(slice->stop); + if (stop_key == NULL) { + goto slice_exit; + } + + step_key = _PyCode_ConstantKey(slice->step); + if (step_key == NULL) { + goto slice_exit; + } + + PyObject *slice_key = PySlice_New(start_key, stop_key, step_key); + if (slice_key == NULL) { + goto slice_exit; + } + + key = PyTuple_Pack(2, slice_key, op); + Py_DECREF(slice_key); + slice_exit: + Py_XDECREF(start_key); + Py_XDECREF(stop_key); + Py_XDECREF(step_key); + } else { /* for other types, use the object identifier as a unique identifier * to ensure that they are seen as unequal. */ @@ -2349,48 +2577,449 @@ _PyCode_ConstantKey(PyObject *op) return key; } +#ifdef Py_GIL_DISABLED +static PyObject * +intern_one_constant(PyObject *op) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + _Py_hashtable_t *consts = interp->code_state.constants; + + assert(!PyUnicode_CheckExact(op)); // strings are interned separately + + _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(consts, op); + if (entry == NULL) { + if (_Py_hashtable_set(consts, op, op) != 0) { + return NULL; + } + +#ifdef Py_REF_DEBUG + Py_ssize_t refcnt = Py_REFCNT(op); + if (refcnt != 1) { + // Adjust the reftotal to account for the fact that we only + // restore a single reference in _PyCode_Fini. + _Py_AddRefTotal(_PyThreadState_GET(), -(refcnt - 1)); + } +#endif + + _Py_SetImmortal(op); + return op; + } + + assert(_Py_IsImmortal(entry->value)); + return (PyObject *)entry->value; +} + +static int +compare_constants(const void *key1, const void *key2) { + PyObject *op1 = (PyObject *)key1; + PyObject *op2 = (PyObject *)key2; + if (op1 == op2) { + return 1; + } + if (Py_TYPE(op1) != Py_TYPE(op2)) { + return 0; + } + // We compare container contents by identity because we have already + // internalized the items. + if (PyTuple_CheckExact(op1)) { + Py_ssize_t size = PyTuple_GET_SIZE(op1); + if (size != PyTuple_GET_SIZE(op2)) { + return 0; + } + for (Py_ssize_t i = 0; i < size; i++) { + if (PyTuple_GET_ITEM(op1, i) != PyTuple_GET_ITEM(op2, i)) { + return 0; + } + } + return 1; + } + else if (PyFrozenSet_CheckExact(op1)) { + if (PySet_GET_SIZE(op1) != PySet_GET_SIZE(op2)) { + return 0; + } + Py_ssize_t pos1 = 0, pos2 = 0; + PyObject *obj1, *obj2; + Py_hash_t hash1, hash2; + while ((_PySet_NextEntry(op1, &pos1, &obj1, &hash1)) && + (_PySet_NextEntry(op2, &pos2, &obj2, &hash2))) + { + if (obj1 != obj2) { + return 0; + } + } + return 1; + } + else if (PySlice_Check(op1)) { + PySliceObject *s1 = (PySliceObject *)op1; + PySliceObject *s2 = (PySliceObject *)op2; + return (s1->start == s2->start && + s1->stop == s2->stop && + s1->step == s2->step); + } + else if (PyBytes_CheckExact(op1) || PyLong_CheckExact(op1)) { + return PyObject_RichCompareBool(op1, op2, Py_EQ); + } + else if (PyFloat_CheckExact(op1)) { + // Ensure that, for example, +0.0 and -0.0 are distinct + double f1 = PyFloat_AS_DOUBLE(op1); + double f2 = PyFloat_AS_DOUBLE(op2); + return memcmp(&f1, &f2, sizeof(double)) == 0; + } + else if (PyComplex_CheckExact(op1)) { + Py_complex c1 = ((PyComplexObject *)op1)->cval; + Py_complex c2 = ((PyComplexObject *)op2)->cval; + return memcmp(&c1, &c2, sizeof(Py_complex)) == 0; + } + _Py_FatalErrorFormat("unexpected type in compare_constants: %s", + Py_TYPE(op1)->tp_name); + return 0; +} + +static Py_uhash_t +hash_const(const void *key) +{ + PyObject *op = (PyObject *)key; + if (PySlice_Check(op)) { + PySliceObject *s = (PySliceObject *)op; + PyObject *data[3] = { s->start, s->stop, s->step }; + return Py_HashBuffer(&data, sizeof(data)); + } + else if (PyTuple_CheckExact(op)) { + Py_ssize_t size = PyTuple_GET_SIZE(op); + PyObject **data = _PyTuple_ITEMS(op); + return Py_HashBuffer(data, sizeof(PyObject *) * size); + } + Py_hash_t h = PyObject_Hash(op); + if (h == -1) { + // This should never happen: all the constants we support have + // infallible hash functions. + Py_FatalError("code: hash failed"); + } + return (Py_uhash_t)h; +} + +static int +clear_containers(_Py_hashtable_t *ht, const void *key, const void *value, + void *user_data) +{ + // First clear containers to avoid recursive deallocation later on in + // destroy_key. + PyObject *op = (PyObject *)key; + if (PyTuple_CheckExact(op)) { + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(op); i++) { + Py_CLEAR(_PyTuple_ITEMS(op)[i]); + } + } + else if (PySlice_Check(op)) { + PySliceObject *slice = (PySliceObject *)op; + Py_SETREF(slice->start, Py_None); + Py_SETREF(slice->stop, Py_None); + Py_SETREF(slice->step, Py_None); + } + else if (PyFrozenSet_CheckExact(op)) { + _PySet_ClearInternal((PySetObject *)op); + } + return 0; +} + +static void +destroy_key(void *key) +{ + _Py_ClearImmortal(key); +} +#endif + +PyStatus +_PyCode_Init(PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + struct _py_code_state *state = &interp->code_state; + state->constants = _Py_hashtable_new_full(&hash_const, &compare_constants, + &destroy_key, NULL, NULL); + if (state->constants == NULL) { + return _PyStatus_NO_MEMORY(); + } +#endif + return _PyStatus_OK(); +} + void -_PyStaticCode_Fini(PyCodeObject *co) +_PyCode_Fini(PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + // Free interned constants + struct _py_code_state *state = &interp->code_state; + if (state->constants) { + _Py_hashtable_foreach(state->constants, &clear_containers, NULL); + _Py_hashtable_destroy(state->constants); + state->constants = NULL; + } + _PyIndexPool_Fini(&interp->tlbc_indices); +#endif +} + +#ifdef Py_GIL_DISABLED + +// Thread-local bytecode (TLBC) +// +// Each thread specializes a thread-local copy of the bytecode, created on the +// first RESUME, in free-threaded builds. All copies of the bytecode for a code +// object are stored in the `co_tlbc` array. Threads reserve a globally unique +// index identifying its copy of the bytecode in all `co_tlbc` arrays at thread +// creation and release the index at thread destruction. The first entry in +// every `co_tlbc` array always points to the "main" copy of the bytecode that +// is stored at the end of the code object. This ensures that no bytecode is +// copied for programs that do not use threads. +// +// Thread-local bytecode can be disabled at runtime by providing either `-X +// tlbc=0` or `PYTHON_TLBC=0`. Disabling thread-local bytecode also disables +// specialization. All threads share the main copy of the bytecode when +// thread-local bytecode is disabled. +// +// Concurrent modifications to the bytecode made by the specializing +// interpreter and instrumentation use atomics, with specialization taking care +// not to overwrite an instruction that was instrumented concurrently. + +int32_t +_Py_ReserveTLBCIndex(PyInterpreterState *interp) +{ + if (interp->config.tlbc_enabled) { + return _PyIndexPool_AllocIndex(&interp->tlbc_indices); + } + // All threads share the main copy of the bytecode when TLBC is disabled + return 0; +} + +void +_Py_ClearTLBCIndex(_PyThreadStateImpl *tstate) { - if (co->co_executors != NULL) { - clear_executors(co); + PyInterpreterState *interp = ((PyThreadState *)tstate)->interp; + if (interp->config.tlbc_enabled) { + _PyIndexPool_FreeIndex(&interp->tlbc_indices, tstate->tlbc_index); } - deopt_code(co, _PyCode_CODE(co)); - PyMem_Free(co->co_extra); - if (co->_co_cached != NULL) { - Py_CLEAR(co->_co_cached->_co_code); - Py_CLEAR(co->_co_cached->_co_cellvars); - Py_CLEAR(co->_co_cached->_co_freevars); - Py_CLEAR(co->_co_cached->_co_varnames); - PyMem_Free(co->_co_cached); - co->_co_cached = NULL; +} + +static _PyCodeArray * +_PyCodeArray_New(Py_ssize_t size) +{ + _PyCodeArray *arr = PyMem_Calloc( + 1, offsetof(_PyCodeArray, entries) + sizeof(void *) * size); + if (arr == NULL) { + PyErr_NoMemory(); + return NULL; } - co->co_extra = NULL; - if (co->co_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *)co); - co->co_weakreflist = NULL; + arr->size = size; + return arr; +} + +static void +copy_code(_Py_CODEUNIT *dst, PyCodeObject *co) +{ + int code_len = (int) Py_SIZE(co); + for (int i = 0; i < code_len; i += _PyInstruction_GetLength(co, i)) { + dst[i] = _Py_GetBaseCodeUnit(co, i); } - free_monitoring_data(co->_co_monitoring); - co->_co_monitoring = NULL; + _PyCode_Quicken(dst, code_len, co->co_consts, 1); } -int -_PyStaticCode_Init(PyCodeObject *co) +static Py_ssize_t +get_pow2_greater(Py_ssize_t initial, Py_ssize_t limit) { - int res = intern_strings(co->co_names); - if (res < 0) { - return -1; + // initial must be a power of two + assert(!(initial & (initial - 1))); + Py_ssize_t res = initial; + while (res && res < limit) { + res <<= 1; } - res = intern_string_constants(co->co_consts, NULL); - if (res < 0) { - return -1; + return res; +} + +static _Py_CODEUNIT * +create_tlbc_lock_held(PyCodeObject *co, Py_ssize_t idx) +{ + _PyCodeArray *tlbc = co->co_tlbc; + if (idx >= tlbc->size) { + Py_ssize_t new_size = get_pow2_greater(tlbc->size, idx + 1); + if (!new_size) { + PyErr_NoMemory(); + return NULL; + } + _PyCodeArray *new_tlbc = _PyCodeArray_New(new_size); + if (new_tlbc == NULL) { + return NULL; + } + memcpy(new_tlbc->entries, tlbc->entries, tlbc->size * sizeof(void *)); + _Py_atomic_store_ptr_release(&co->co_tlbc, new_tlbc); + _PyMem_FreeDelayed(tlbc); + tlbc = new_tlbc; } - res = intern_strings(co->co_localsplusnames); - if (res < 0) { + char *bc = PyMem_Calloc(1, _PyCode_NBYTES(co)); + if (bc == NULL) { + PyErr_NoMemory(); + return NULL; + } + copy_code((_Py_CODEUNIT *) bc, co); + assert(tlbc->entries[idx] == NULL); + tlbc->entries[idx] = bc; + return (_Py_CODEUNIT *) bc; +} + +static _Py_CODEUNIT * +get_tlbc_lock_held(PyCodeObject *co) +{ + _PyCodeArray *tlbc = co->co_tlbc; + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)PyThreadState_GET(); + int32_t idx = tstate->tlbc_index; + if (idx < tlbc->size && tlbc->entries[idx] != NULL) { + return (_Py_CODEUNIT *)tlbc->entries[idx]; + } + return create_tlbc_lock_held(co, idx); +} + +_Py_CODEUNIT * +_PyCode_GetTLBC(PyCodeObject *co) +{ + _Py_CODEUNIT *result; + Py_BEGIN_CRITICAL_SECTION(co); + result = get_tlbc_lock_held(co); + Py_END_CRITICAL_SECTION(); + return result; +} + +// My kingdom for a bitset +struct flag_set { + uint8_t *flags; + Py_ssize_t size; +}; + +static inline int +flag_is_set(struct flag_set *flags, Py_ssize_t idx) +{ + assert(idx >= 0); + return (idx < flags->size) && flags->flags[idx]; +} + +// Set the flag for each tlbc index in use +static int +get_indices_in_use(PyInterpreterState *interp, struct flag_set *in_use) +{ + assert(interp->stoptheworld.world_stopped); + assert(in_use->flags == NULL); + int32_t max_index = 0; + _Py_FOR_EACH_TSTATE_BEGIN(interp, p) { + int32_t idx = ((_PyThreadStateImpl *) p)->tlbc_index; + if (idx > max_index) { + max_index = idx; + } + } + _Py_FOR_EACH_TSTATE_END(interp); + in_use->size = (size_t) max_index + 1; + in_use->flags = PyMem_Calloc(in_use->size, sizeof(*in_use->flags)); + if (in_use->flags == NULL) { return -1; } - _PyCode_Quicken(co); + _Py_FOR_EACH_TSTATE_BEGIN(interp, p) { + in_use->flags[((_PyThreadStateImpl *) p)->tlbc_index] = 1; + } + _Py_FOR_EACH_TSTATE_END(interp); + return 0; +} + +struct get_code_args { + _PyObjectStack code_objs; + struct flag_set indices_in_use; + int err; +}; + +static void +clear_get_code_args(struct get_code_args *args) +{ + if (args->indices_in_use.flags != NULL) { + PyMem_Free(args->indices_in_use.flags); + args->indices_in_use.flags = NULL; + } + _PyObjectStack_Clear(&args->code_objs); +} + +static inline int +is_bytecode_unused(_PyCodeArray *tlbc, Py_ssize_t idx, + struct flag_set *indices_in_use) +{ + assert(idx > 0 && idx < tlbc->size); + return tlbc->entries[idx] != NULL && !flag_is_set(indices_in_use, idx); +} + +static int +get_code_with_unused_tlbc(PyObject *obj, struct get_code_args *args) +{ + if (!PyCode_Check(obj)) { + return 1; + } + PyCodeObject *co = (PyCodeObject *) obj; + _PyCodeArray *tlbc = co->co_tlbc; + // The first index always points at the main copy of the bytecode embedded + // in the code object. + for (Py_ssize_t i = 1; i < tlbc->size; i++) { + if (is_bytecode_unused(tlbc, i, &args->indices_in_use)) { + if (_PyObjectStack_Push(&args->code_objs, obj) < 0) { + args->err = -1; + return 0; + } + return 1; + } + } + return 1; +} + +static void +free_unused_bytecode(PyCodeObject *co, struct flag_set *indices_in_use) +{ + _PyCodeArray *tlbc = co->co_tlbc; + // The first index always points at the main copy of the bytecode embedded + // in the code object. + for (Py_ssize_t i = 1; i < tlbc->size; i++) { + if (is_bytecode_unused(tlbc, i, indices_in_use)) { + PyMem_Free(tlbc->entries[i]); + tlbc->entries[i] = NULL; + } + } +} + +int +_Py_ClearUnusedTLBC(PyInterpreterState *interp) +{ + struct get_code_args args = { + .code_objs = {NULL}, + .indices_in_use = {NULL, 0}, + .err = 0, + }; + _PyEval_StopTheWorld(interp); + // Collect in-use tlbc indices + if (get_indices_in_use(interp, &args.indices_in_use) < 0) { + goto err; + } + // Collect code objects that have bytecode not in use by any thread + _PyGC_VisitObjectsWorldStopped( + interp, (gcvisitobjects_t)get_code_with_unused_tlbc, &args); + if (args.err < 0) { + goto err; + } + // Free unused bytecode. This must happen outside of gc_visit_heaps; it is + // unsafe to allocate or free any mimalloc managed memory when it's + // running. + PyObject *obj; + while ((obj = _PyObjectStack_Pop(&args.code_objs)) != NULL) { + free_unused_bytecode((PyCodeObject*) obj, &args.indices_in_use); + } + _PyEval_StartTheWorld(interp); + clear_get_code_args(&args); return 0; + +err: + _PyEval_StartTheWorld(interp); + clear_get_code_args(&args); + PyErr_NoMemory(); + return -1; } -#define MAX_CODE_UNITS_PER_LOC_ENTRY 8 +#endif diff --git a/Objects/complexobject.c b/Objects/complexobject.c index d8b0e84da5df4a..5d9b3c9f0e3e76 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -8,11 +8,14 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_complexobject.h" // _PyComplex_FormatAdvancedWriter() +#include "pycore_floatobject.h" // _Py_convert_int_to_double() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_object.h" // _PyObject_Init() #include "pycore_pymath.h" // _Py_ADJUST_ERANGE2() +#define _PyComplexObject_CAST(op) ((PyComplexObject *)(op)) + /*[clinic input] class complex "PyComplexObject *" "&PyComplex_Type" @@ -34,6 +37,20 @@ _Py_c_sum(Py_complex a, Py_complex b) return r; } +Py_complex +_Py_cr_sum(Py_complex a, double b) +{ + Py_complex r = a; + r.real += b; + return r; +} + +static inline Py_complex +_Py_rc_sum(double a, Py_complex b) +{ + return _Py_cr_sum(b, a); +} + Py_complex _Py_c_diff(Py_complex a, Py_complex b) { @@ -43,6 +60,23 @@ _Py_c_diff(Py_complex a, Py_complex b) return r; } +Py_complex +_Py_cr_diff(Py_complex a, double b) +{ + Py_complex r = a; + r.real -= b; + return r; +} + +Py_complex +_Py_rc_diff(double a, Py_complex b) +{ + Py_complex r; + r.real = a - b.real; + r.imag = -b.imag; + return r; +} + Py_complex _Py_c_neg(Py_complex a) { @@ -53,14 +87,81 @@ _Py_c_neg(Py_complex a) } Py_complex -_Py_c_prod(Py_complex a, Py_complex b) +_Py_c_prod(Py_complex z, Py_complex w) { - Py_complex r; - r.real = a.real*b.real - a.imag*b.imag; - r.imag = a.real*b.imag + a.imag*b.real; + double a = z.real, b = z.imag, c = w.real, d = w.imag; + double ac = a*c, bd = b*d, ad = a*d, bc = b*c; + Py_complex r = {ac - bd, ad + bc}; + + /* Recover infinities that computed as nan+nanj. See e.g. the C11, + Annex G.5.1, routine _Cmultd(). */ + if (isnan(r.real) && isnan(r.imag)) { + int recalc = 0; + + if (isinf(a) || isinf(b)) { /* z is infinite */ + /* "Box" the infinity and change nans in the other factor to 0 */ + a = copysign(isinf(a) ? 1.0 : 0.0, a); + b = copysign(isinf(b) ? 1.0 : 0.0, b); + if (isnan(c)) { + c = copysign(0.0, c); + } + if (isnan(d)) { + d = copysign(0.0, d); + } + recalc = 1; + } + if (isinf(c) || isinf(d)) { /* w is infinite */ + /* "Box" the infinity and change nans in the other factor to 0 */ + c = copysign(isinf(c) ? 1.0 : 0.0, c); + d = copysign(isinf(d) ? 1.0 : 0.0, d); + if (isnan(a)) { + a = copysign(0.0, a); + } + if (isnan(b)) { + b = copysign(0.0, b); + } + recalc = 1; + } + if (!recalc && (isinf(ac) || isinf(bd) || isinf(ad) || isinf(bc))) { + /* Recover infinities from overflow by changing nans to 0 */ + if (isnan(a)) { + a = copysign(0.0, a); + } + if (isnan(b)) { + b = copysign(0.0, b); + } + if (isnan(c)) { + c = copysign(0.0, c); + } + if (isnan(d)) { + d = copysign(0.0, d); + } + recalc = 1; + } + if (recalc) { + r.real = Py_INFINITY*(a*c - b*d); + r.imag = Py_INFINITY*(a*d + b*c); + } + } + return r; } +Py_complex +_Py_cr_prod(Py_complex a, double b) +{ + Py_complex r = a; + r.real *= b; + r.imag *= b; + return r; +} + +static inline Py_complex +_Py_rc_prod(double a, Py_complex b) +{ + return _Py_cr_prod(b, a); +} + /* Avoid bad optimization on Windows ARM64 until the compiler is fixed */ #ifdef _M_ARM64 #pragma optimize("", off) @@ -88,8 +189,7 @@ _Py_c_quot(Py_complex a, Py_complex b) * numerators and denominator by whichever of {b.real, b.imag} has * larger magnitude. The earliest reference I found was to CACM * Algorithm 116 (Complex Division, Robert L. Smith, Stanford - * University). As usual, though, we're still ignoring all IEEE - * endcases. + * University). */ Py_complex r; /* the result */ const double abs_breal = b.real < 0 ? -b.real : b.real; @@ -120,6 +220,86 @@ _Py_c_quot(Py_complex a, Py_complex b) /* At least one of b.real or b.imag is a NaN */ r.real = r.imag = Py_NAN; } + + /* Recover infinities and zeros that computed as nan+nanj. See e.g. + the C11, Annex G.5.2, routine _Cdivd(). */ + if (isnan(r.real) && isnan(r.imag)) { + if ((isinf(a.real) || isinf(a.imag)) + && isfinite(b.real) && isfinite(b.imag)) + { + const double x = copysign(isinf(a.real) ? 1.0 : 0.0, a.real); + const double y = copysign(isinf(a.imag) ? 1.0 : 0.0, a.imag); + r.real = Py_INFINITY * (x*b.real + y*b.imag); + r.imag = Py_INFINITY * (y*b.real - x*b.imag); + } + else if ((isinf(abs_breal) || isinf(abs_bimag)) + && isfinite(a.real) && isfinite(a.imag)) + { + const double x = copysign(isinf(b.real) ? 1.0 : 0.0, b.real); + const double y = copysign(isinf(b.imag) ? 1.0 : 0.0, b.imag); + r.real = 0.0 * (a.real*x + a.imag*y); + r.imag = 0.0 * (a.imag*x - a.real*y); + } + } + + return r; +} + +Py_complex +_Py_cr_quot(Py_complex a, double b) +{ + Py_complex r = a; + if (b) { + r.real /= b; + r.imag /= b; + } + else { + errno = EDOM; + r.real = r.imag = 0.0; + } + return r; +} + +/* an equivalent of _Py_c_quot() function, when 1st argument is real */ +Py_complex +_Py_rc_quot(double a, Py_complex b) +{ + Py_complex r; + const double abs_breal = b.real < 0 ? -b.real : b.real; + const double abs_bimag = b.imag < 0 ? -b.imag : b.imag; + + if (abs_breal >= abs_bimag) { + if (abs_breal == 0.0) { + errno = EDOM; + r.real = r.imag = 0.0; + } + else { + const double ratio = b.imag / b.real; + const double denom = b.real + b.imag * ratio; + r.real = a / denom; + r.imag = (-a * ratio) / denom; + } + } + else if (abs_bimag >= abs_breal) { + const double ratio = b.real / b.imag; + const double denom = b.real * ratio + b.imag; + assert(b.imag != 0.0); + r.real = (a * ratio) / denom; + r.imag = (-a) / denom; + } + else { + r.real = r.imag = Py_NAN; + } + + if (isnan(r.real) && isnan(r.imag) && isfinite(a) + && (isinf(abs_breal) || isinf(abs_bimag))) + { + const double x = copysign(isinf(b.real) ? 1.0 : 0.0, b.real); + const double y = copysign(isinf(b.imag) ? 1.0 : 0.0, b.imag); + r.real = 0.0 * (a*x); + r.imag = 0.0 * (-a*y); + } + return r; } #ifdef _M_ARM64 @@ -147,11 +327,13 @@ _Py_c_pow(Py_complex a, Py_complex b) at = atan2(a.imag, a.real); phase = at*b.real; if (b.imag != 0.0) { - len /= exp(at*b.imag); + len *= exp(-at*b.imag); phase += b.imag*log(vabs); } r.real = len*cos(phase); r.imag = len*sin(phase); + + _Py_ADJUST_ERANGE2(r.real, r.imag); } return r; } @@ -188,16 +370,16 @@ _Py_c_abs(Py_complex z) /* sets errno = ERANGE on overflow; otherwise errno = 0 */ double result; - if (!Py_IS_FINITE(z.real) || !Py_IS_FINITE(z.imag)) { + if (!isfinite(z.real) || !isfinite(z.imag)) { /* C99 rules: if either the real or the imaginary part is an infinity, return infinity, even if the other part is a NaN. */ - if (Py_IS_INFINITY(z.real)) { + if (isinf(z.real)) { result = fabs(z.real); errno = 0; return result; } - if (Py_IS_INFINITY(z.imag)) { + if (isinf(z.imag)) { result = fabs(z.imag); errno = 0; return result; @@ -207,7 +389,7 @@ _Py_c_abs(Py_complex z) return Py_NAN; } result = hypot(z.real, z.imag); - if (!Py_IS_FINITE(result)) + if (!isfinite(result)) errno = ERANGE; else errno = 0; @@ -373,11 +555,12 @@ PyComplex_AsCComplex(PyObject *op) } static PyObject * -complex_repr(PyComplexObject *v) +complex_repr(PyObject *op) { int precision = 0; char format_code = 'r'; PyObject *result = NULL; + PyComplexObject *v = _PyComplexObject_CAST(op); /* If these are non-NULL, they'll need to be freed. */ char *pre = NULL; @@ -429,13 +612,14 @@ complex_repr(PyComplexObject *v) } static Py_hash_t -complex_hash(PyComplexObject *v) +complex_hash(PyObject *op) { Py_uhash_t hashreal, hashimag, combined; - hashreal = (Py_uhash_t)_Py_HashDouble((PyObject *) v, v->cval.real); + PyComplexObject *v = _PyComplexObject_CAST(op); + hashreal = (Py_uhash_t)_Py_HashDouble(op, v->cval.real); if (hashreal == (Py_uhash_t)-1) return -1; - hashimag = (Py_uhash_t)_Py_HashDouble((PyObject *)v, v->cval.imag); + hashimag = (Py_uhash_t)_Py_HashDouble(op, v->cval.imag); if (hashimag == (Py_uhash_t)-1) return -1; /* Note: if the imaginary part is 0, hashimag is 0 now, @@ -451,83 +635,90 @@ complex_hash(PyComplexObject *v) } /* This macro may return! */ -#define TO_COMPLEX(obj, c) \ - if (PyComplex_Check(obj)) \ - c = ((PyComplexObject *)(obj))->cval; \ - else if (to_complex(&(obj), &(c)) < 0) \ +#define TO_COMPLEX(obj, c) \ + if (PyComplex_Check(obj)) \ + c = ((PyComplexObject *)(obj))->cval; \ + else if (real_to_complex(&(obj), &(c)) < 0) \ return (obj) static int -to_complex(PyObject **pobj, Py_complex *pc) +real_to_double(PyObject **pobj, double *dbl) { PyObject *obj = *pobj; - pc->real = pc->imag = 0.0; - if (PyLong_Check(obj)) { - pc->real = PyLong_AsDouble(obj); - if (pc->real == -1.0 && PyErr_Occurred()) { - *pobj = NULL; - return -1; - } - return 0; - } if (PyFloat_Check(obj)) { - pc->real = PyFloat_AsDouble(obj); - return 0; + *dbl = PyFloat_AS_DOUBLE(obj); + } + else if (_Py_convert_int_to_double(pobj, dbl) < 0) { + return -1; } - *pobj = Py_NewRef(Py_NotImplemented); - return -1; + return 0; } - -static PyObject * -complex_add(PyObject *v, PyObject *w) +static int +real_to_complex(PyObject **pobj, Py_complex *pc) { - Py_complex result; - Py_complex a, b; - TO_COMPLEX(v, a); - TO_COMPLEX(w, b); - result = _Py_c_sum(a, b); - return PyComplex_FromCComplex(result); + pc->imag = 0.0; + return real_to_double(pobj, &(pc->real)); } -static PyObject * -complex_sub(PyObject *v, PyObject *w) -{ - Py_complex result; - Py_complex a, b; - TO_COMPLEX(v, a); - TO_COMPLEX(w, b); - result = _Py_c_diff(a, b); - return PyComplex_FromCComplex(result); -} - -static PyObject * -complex_mul(PyObject *v, PyObject *w) -{ - Py_complex result; - Py_complex a, b; - TO_COMPLEX(v, a); - TO_COMPLEX(w, b); - result = _Py_c_prod(a, b); - return PyComplex_FromCComplex(result); -} - -static PyObject * -complex_div(PyObject *v, PyObject *w) -{ - Py_complex quot; - Py_complex a, b; - TO_COMPLEX(v, a); - TO_COMPLEX(w, b); - errno = 0; - quot = _Py_c_quot(a, b); - if (errno == EDOM) { - PyErr_SetString(PyExc_ZeroDivisionError, "complex division by zero"); - return NULL; - } - return PyComplex_FromCComplex(quot); -} +/* Complex arithmetic rules implement special mixed-mode case where combining + a pure-real (float or int) value and a complex value is performed directly + without first coercing the real value to a complex value. + + Let us consider the addition as an example, assuming that ints are implicitly + converted to floats. We have the following rules (up to variants with changed + order of operands): + + complex(a, b) + complex(c, d) = complex(a + c, b + d) + float(a) + complex(b, c) = complex(a + b, c) + + Similar rules are implemented for subtraction, multiplication and division. + See C11's Annex G, sections G.5.1 and G.5.2. + */ + +#define COMPLEX_BINOP(NAME, FUNC) \ + static PyObject * \ + complex_##NAME(PyObject *v, PyObject *w) \ + { \ + Py_complex a; \ + errno = 0; \ + if (PyComplex_Check(w)) { \ + Py_complex b = ((PyComplexObject *)w)->cval; \ + if (PyComplex_Check(v)) { \ + a = ((PyComplexObject *)v)->cval; \ + a = _Py_c_##FUNC(a, b); \ + } \ + else if (real_to_double(&v, &a.real) < 0) { \ + return v; \ + } \ + else { \ + a = _Py_rc_##FUNC(a.real, b); \ + } \ + } \ + else if (!PyComplex_Check(v)) { \ + Py_RETURN_NOTIMPLEMENTED; \ + } \ + else { \ + a = ((PyComplexObject *)v)->cval; \ + double b; \ + if (real_to_double(&w, &b) < 0) { \ + return w; \ + } \ + a = _Py_cr_##FUNC(a, b); \ + } \ + if (errno == EDOM) { \ + PyErr_SetString(PyExc_ZeroDivisionError, \ + "division by zero"); \ + return NULL; \ + } \ + return PyComplex_FromCComplex(a); \ + } + +COMPLEX_BINOP(add, sum) +COMPLEX_BINOP(mul, prod) +COMPLEX_BINOP(sub, diff) +COMPLEX_BINOP(div, quot) static PyObject * complex_pow(PyObject *v, PyObject *w, PyObject *z) @@ -546,15 +737,15 @@ complex_pow(PyObject *v, PyObject *w, PyObject *z) // a faster and more accurate algorithm. if (b.imag == 0.0 && b.real == floor(b.real) && fabs(b.real) <= 100.0) { p = c_powi(a, (long)b.real); + _Py_ADJUST_ERANGE2(p.real, p.imag); } else { p = _Py_c_pow(a, b); } - _Py_ADJUST_ERANGE2(p.real, p.imag); if (errno == EDOM) { PyErr_SetString(PyExc_ZeroDivisionError, - "0.0 to a negative or complex power"); + "zero to a negative or complex power"); return NULL; } else if (errno == ERANGE) { @@ -566,8 +757,9 @@ complex_pow(PyObject *v, PyObject *w, PyObject *z) } static PyObject * -complex_neg(PyComplexObject *v) +complex_neg(PyObject *op) { + PyComplexObject *v = _PyComplexObject_CAST(op); Py_complex neg; neg.real = -v->cval.real; neg.imag = -v->cval.imag; @@ -575,22 +767,20 @@ complex_neg(PyComplexObject *v) } static PyObject * -complex_pos(PyComplexObject *v) +complex_pos(PyObject *op) { + PyComplexObject *v = _PyComplexObject_CAST(op); if (PyComplex_CheckExact(v)) { return Py_NewRef(v); } - else - return PyComplex_FromCComplex(v->cval); + return PyComplex_FromCComplex(v->cval); } static PyObject * -complex_abs(PyComplexObject *v) +complex_abs(PyObject *op) { - double result; - - result = _Py_c_abs(v->cval); - + PyComplexObject *v = _PyComplexObject_CAST(op); + double result = _Py_c_abs(v->cval); if (errno == ERANGE) { PyErr_SetString(PyExc_OverflowError, "absolute value too large"); @@ -600,8 +790,9 @@ complex_abs(PyComplexObject *v) } static int -complex_bool(PyComplexObject *v) +complex_bool(PyObject *op) { + PyComplexObject *v = _PyComplexObject_CAST(op); return v->cval.real != 0.0 || v->cval.imag != 0.0; } @@ -736,22 +927,6 @@ complex___complex___impl(PyComplexObject *self) } -static PyMethodDef complex_methods[] = { - COMPLEX_CONJUGATE_METHODDEF - COMPLEX___COMPLEX___METHODDEF - COMPLEX___GETNEWARGS___METHODDEF - COMPLEX___FORMAT___METHODDEF - {NULL, NULL} /* sentinel */ -}; - -static PyMemberDef complex_members[] = { - {"real", Py_T_DOUBLE, offsetof(PyComplexObject, cval.real), Py_READONLY, - "the real part of a complex number"}, - {"imag", Py_T_DOUBLE, offsetof(PyComplexObject, cval.imag), Py_READONLY, - "the imaginary part of a complex number"}, - {0}, -}; - static PyObject * complex_from_string_inner(const char *s, Py_ssize_t len, void *type) { @@ -894,8 +1069,8 @@ complex_subtype_from_string(PyTypeObject *type, PyObject *v) } else { PyErr_Format(PyExc_TypeError, - "complex() argument must be a string or a number, not '%.200s'", - Py_TYPE(v)->tp_name); + "complex() argument must be a string or a number, not %T", + v); return NULL; } @@ -905,20 +1080,94 @@ complex_subtype_from_string(PyTypeObject *type, PyObject *v) return result; } +/* The constructor should only accept a string as a positional argument, + * not as by the 'real' keyword. But Argument Clinic does not allow + * to distinguish between argument passed positionally and by keyword. + * So the constructor must be split into two parts: actual_complex_new() + * handles the case of no arguments and one positional argument, and calls + * complex_new(), implemented with Argument Clinic, to handle the remaining + * cases: 'real' and 'imag' arguments. This separation is well suited + * for different constructor roles: converting a string or number to a complex + * number and constructing a complex number from real and imaginary parts. + */ +static PyObject * +actual_complex_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *res = NULL; + PyNumberMethods *nbr; + + if (PyTuple_GET_SIZE(args) > 1 || (kwargs != NULL && PyDict_GET_SIZE(kwargs))) { + return complex_new(type, args, kwargs); + } + if (!PyTuple_GET_SIZE(args)) { + return complex_subtype_from_doubles(type, 0, 0); + } + + PyObject *arg = PyTuple_GET_ITEM(args, 0); + /* Special-case for a single argument when type(arg) is complex. */ + if (PyComplex_CheckExact(arg) && type == &PyComplex_Type) { + /* Note that we can't know whether it's safe to return + a complex *subclass* instance as-is, hence the restriction + to exact complexes here. If either the input or the + output is a complex subclass, it will be handled below + as a non-orthogonal vector. */ + return Py_NewRef(arg); + } + if (PyUnicode_Check(arg)) { + return complex_subtype_from_string(type, arg); + } + PyObject *tmp = try_complex_special_method(arg); + if (tmp) { + Py_complex c = ((PyComplexObject*)tmp)->cval; + res = complex_subtype_from_doubles(type, c.real, c.imag); + Py_DECREF(tmp); + } + else if (PyErr_Occurred()) { + return NULL; + } + else if (PyComplex_Check(arg)) { + /* Note that if arg is of a complex subtype, we're only + retaining its real & imag parts here, and the return + value is (properly) of the builtin complex type. */ + Py_complex c = ((PyComplexObject*)arg)->cval; + res = complex_subtype_from_doubles(type, c.real, c.imag); + } + else if ((nbr = Py_TYPE(arg)->tp_as_number) != NULL && + (nbr->nb_float != NULL || nbr->nb_index != NULL)) + { + /* The argument really is entirely real, and contributes + nothing in the imaginary direction. + Just treat it as a double. */ + double r = PyFloat_AsDouble(arg); + if (r != -1.0 || !PyErr_Occurred()) { + res = complex_subtype_from_doubles(type, r, 0); + } + } + else { + PyErr_Format(PyExc_TypeError, + "complex() argument must be a string or a number, not %T", + arg); + } + return res; +} + /*[clinic input] @classmethod complex.__new__ as complex_new real as r: object(c_default="NULL") = 0 imag as i: object(c_default="NULL") = 0 -Create a complex number from a real part and an optional imaginary part. +Create a complex number from a string or numbers. -This is equivalent to (real + imag*1j) where imag defaults to 0. +If a string is given, parse it as a complex number. +If a single number is given, convert it to a complex number. +If the 'real' or 'imag' arguments are given, create a complex number +with the specified real and imaginary components. [clinic start generated code]*/ static PyObject * complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) -/*[clinic end generated code: output=b6c7dd577b537dc1 input=f4c667f2596d4fd1]*/ +/*[clinic end generated code: output=b6c7dd577b537dc1 input=ff4268dc540958a4]*/ { PyObject *tmp; PyNumberMethods *nbr, *nbi = NULL; @@ -930,32 +1179,10 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) if (r == NULL) { r = _PyLong_GetZero(); } + PyObject *orig_r = r; - /* Special-case for a single argument when type(arg) is complex. */ - if (PyComplex_CheckExact(r) && i == NULL && - type == &PyComplex_Type) { - /* Note that we can't know whether it's safe to return - a complex *subclass* instance as-is, hence the restriction - to exact complexes here. If either the input or the - output is a complex subclass, it will be handled below - as a non-orthogonal vector. */ - return Py_NewRef(r); - } - if (PyUnicode_Check(r)) { - if (i != NULL) { - PyErr_SetString(PyExc_TypeError, - "complex() can't take second arg" - " if first is a string"); - return NULL; - } - return complex_subtype_from_string(type, r); - } - if (i != NULL && PyUnicode_Check(i)) { - PyErr_SetString(PyExc_TypeError, - "complex() second arg can't be a string"); - return NULL; - } - + /* DEPRECATED: The call of try_complex_special_method() for the "real" + * part will be dropped after the end of the deprecation period. */ tmp = try_complex_special_method(r); if (tmp) { r = tmp; @@ -970,9 +1197,8 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) (nbr->nb_float == NULL && nbr->nb_index == NULL && !PyComplex_Check(r))) { PyErr_Format(PyExc_TypeError, - "complex() first argument must be a string or a number, " - "not '%.200s'", - Py_TYPE(r)->tp_name); + "complex() argument 'real' must be a real number, not %T", + r); if (own_r) { Py_DECREF(r); } @@ -984,9 +1210,8 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) (nbi->nb_float == NULL && nbi->nb_index == NULL && !PyComplex_Check(i))) { PyErr_Format(PyExc_TypeError, - "complex() second argument must be a number, " - "not '%.200s'", - Py_TYPE(i)->tp_name); + "complex() argument 'imag' must be a real number, not %T", + i); if (own_r) { Py_DECREF(r); } @@ -998,6 +1223,7 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) both be treated as numbers, and the constructor should return a complex number equal to (real + imag*1j). + The following is DEPRECATED: Note that we do NOT assume the input to already be in canonical form; the "real" and "imag" parts might themselves be complex numbers, which slightly complicates the code below. */ @@ -1008,19 +1234,27 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) cr = ((PyComplexObject*)r)->cval; cr_is_complex = 1; if (own_r) { + /* r was a newly created complex number, rather + than the original "real" argument. */ Py_DECREF(r); } + nbr = Py_TYPE(orig_r)->tp_as_number; + if (nbr == NULL || + (nbr->nb_float == NULL && nbr->nb_index == NULL)) + { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "complex() argument 'real' must be a real number, not %T", + orig_r)) { + return NULL; + } + } } else { /* The "real" part really is entirely real, and contributes nothing in the imaginary direction. Just treat it as a double. */ tmp = PyNumber_Float(r); - if (own_r) { - /* r was a newly created complex number, rather - than the original "real" argument. */ - Py_DECREF(r); - } + assert(!own_r); if (tmp == NULL) return NULL; assert(PyFloat_Check(tmp)); @@ -1032,6 +1266,11 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) ci.real = cr.imag; } else if (PyComplex_Check(i)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "complex() argument 'imag' must be a real number, not %T", + i)) { + return NULL; + } ci = ((PyComplexObject*)i)->cval; ci_is_complex = 1; } else { @@ -1057,17 +1296,63 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) return complex_subtype_from_doubles(type, cr.real, ci.real); } +/*[clinic input] +@classmethod +complex.from_number + + number: object + / + +Convert number to a complex floating-point number. +[clinic start generated code]*/ + +static PyObject * +complex_from_number(PyTypeObject *type, PyObject *number) +/*[clinic end generated code: output=658a7a5fb0de074d input=3f8bdd3a2bc3facd]*/ +{ + if (PyComplex_CheckExact(number) && type == &PyComplex_Type) { + Py_INCREF(number); + return number; + } + Py_complex cv = PyComplex_AsCComplex(number); + if (cv.real == -1.0 && PyErr_Occurred()) { + return NULL; + } + PyObject *result = PyComplex_FromCComplex(cv); + if (type != &PyComplex_Type && result != NULL) { + Py_SETREF(result, PyObject_CallOneArg((PyObject *)type, result)); + } + return result; +} + +static PyMethodDef complex_methods[] = { + COMPLEX_FROM_NUMBER_METHODDEF + COMPLEX_CONJUGATE_METHODDEF + COMPLEX___COMPLEX___METHODDEF + COMPLEX___GETNEWARGS___METHODDEF + COMPLEX___FORMAT___METHODDEF + {NULL, NULL} /* sentinel */ +}; + +static PyMemberDef complex_members[] = { + {"real", Py_T_DOUBLE, offsetof(PyComplexObject, cval.real), Py_READONLY, + "the real part of a complex number"}, + {"imag", Py_T_DOUBLE, offsetof(PyComplexObject, cval.imag), Py_READONLY, + "the imaginary part of a complex number"}, + {0}, +}; + static PyNumberMethods complex_as_number = { - (binaryfunc)complex_add, /* nb_add */ - (binaryfunc)complex_sub, /* nb_subtract */ - (binaryfunc)complex_mul, /* nb_multiply */ + complex_add, /* nb_add */ + complex_sub, /* nb_subtract */ + complex_mul, /* nb_multiply */ 0, /* nb_remainder */ 0, /* nb_divmod */ - (ternaryfunc)complex_pow, /* nb_power */ - (unaryfunc)complex_neg, /* nb_negative */ - (unaryfunc)complex_pos, /* nb_positive */ - (unaryfunc)complex_abs, /* nb_absolute */ - (inquiry)complex_bool, /* nb_bool */ + complex_pow, /* nb_power */ + complex_neg, /* nb_negative */ + complex_pos, /* nb_positive */ + complex_abs, /* nb_absolute */ + complex_bool, /* nb_bool */ 0, /* nb_invert */ 0, /* nb_lshift */ 0, /* nb_rshift */ @@ -1088,7 +1373,7 @@ static PyNumberMethods complex_as_number = { 0, /* nb_inplace_xor */ 0, /* nb_inplace_or */ 0, /* nb_floor_divide */ - (binaryfunc)complex_div, /* nb_true_divide */ + complex_div, /* nb_true_divide */ 0, /* nb_inplace_floor_divide */ 0, /* nb_inplace_true_divide */ }; @@ -1103,11 +1388,11 @@ PyTypeObject PyComplex_Type = { 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)complex_repr, /* tp_repr */ + complex_repr, /* tp_repr */ &complex_as_number, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ - (hashfunc)complex_hash, /* tp_hash */ + complex_hash, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -1131,6 +1416,7 @@ PyTypeObject PyComplex_Type = { 0, /* tp_dictoffset */ 0, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ - complex_new, /* tp_new */ - PyObject_Del, /* tp_free */ + actual_complex_new, /* tp_new */ + PyObject_Free, /* tp_free */ + .tp_version_tag = _Py_TYPE_VERSION_COMPLEX, }; diff --git a/Objects/descrobject.c b/Objects/descrobject.c index df546a090c28e4..238becee241d1d 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -909,6 +909,7 @@ descr_new(PyTypeObject *descrtype, PyTypeObject *type, const char *name) descr = (PyDescrObject *)PyType_GenericAlloc(descrtype, 0); if (descr != NULL) { + _PyObject_SetDeferredRefcount((PyObject *)descr); descr->d_type = (PyTypeObject*)Py_XNewRef(type); descr->d_name = PyUnicode_InternFromString(name); if (descr->d_name == NULL) { @@ -1165,8 +1166,8 @@ mappingproxy_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) static PyMethodDef mappingproxy_methods[] = { {"get", _PyCFunction_CAST(mappingproxy_get), METH_FASTCALL, - PyDoc_STR("D.get(k[,d]) -> D[k] if k in D, else d." - " d defaults to None.")}, + PyDoc_STR("get($self, key, default=None, /)\n--\n\n" + "Return the value for key if key is in the mapping, else default.")}, {"keys", mappingproxy_keys, METH_NOARGS, PyDoc_STR("D.keys() -> a set-like object providing a view on D's keys")}, {"values", mappingproxy_values, METH_NOARGS, @@ -1254,11 +1255,12 @@ mappingproxy.__new__ as mappingproxy_new mapping: object +Read-only proxy of a mapping. [clinic start generated code]*/ static PyObject * mappingproxy_new_impl(PyTypeObject *type, PyObject *mapping) -/*[clinic end generated code: output=65f27f02d5b68fa7 input=d2d620d4f598d4f8]*/ +/*[clinic end generated code: output=65f27f02d5b68fa7 input=c156df096ef7590c]*/ { mappingproxyobject *mappingproxy; @@ -1346,8 +1348,8 @@ wrapper_hash(PyObject *self) { wrapperobject *wp = (wrapperobject *)self; Py_hash_t x, y; - x = _Py_HashPointer(wp->self); - y = _Py_HashPointer(wp->descr); + x = PyObject_GenericHash(wp->self); + y = Py_HashPointer(wp->descr); x = x ^ y; if (x == -1) x = -2; @@ -1506,6 +1508,8 @@ PyWrapper_New(PyObject *d, PyObject *self) /* A built-in 'property' type */ +#define _propertyobject_CAST(op) ((propertyobject *)(op)) + /* class property(object): @@ -1857,22 +1861,9 @@ property_init_impl(propertyobject *self, PyObject *fget, PyObject *fset, /* if no docstring given and the getter has one, use that one */ else if (fget != NULL) { int rc = PyObject_GetOptionalAttr(fget, &_Py_ID(__doc__), &prop_doc); - if (rc <= 0) { + if (rc < 0) { return rc; } - if (!Py_IS_TYPE(self, &PyProperty_Type) && - prop_doc != NULL && prop_doc != Py_None) { - // This oddity preserves the long existing behavior of surfacing - // an AttributeError when using a dict-less (__slots__) property - // subclass as a decorator on a getter method with a docstring. - // See PropertySubclassTest.test_slots_docstring_copy_exception. - int err = PyObject_SetAttr( - (PyObject *)self, &_Py_ID(__doc__), prop_doc); - if (err < 0) { - Py_DECREF(prop_doc); // release our new reference. - return -1; - } - } if (prop_doc == Py_None) { prop_doc = NULL; Py_DECREF(Py_None); @@ -1900,7 +1891,9 @@ property_init_impl(propertyobject *self, PyObject *fget, PyObject *fset, Py_DECREF(prop_doc); if (err < 0) { assert(PyErr_Occurred()); - if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + if (!self->getter_doc && + PyErr_ExceptionMatches(PyExc_AttributeError)) + { PyErr_Clear(); // https://github.com/python/cpython/issues/98963#issuecomment-1574413319 // Python silently dropped this doc assignment through 3.11. @@ -1920,8 +1913,9 @@ property_init_impl(propertyobject *self, PyObject *fget, PyObject *fset, } static PyObject * -property_get__name__(propertyobject *prop, void *Py_UNUSED(ignored)) +property_get__name__(PyObject *op, void *Py_UNUSED(ignored)) { + propertyobject *prop = _propertyobject_CAST(op); PyObject *name; if (property_name(prop, &name) < 0) { return NULL; @@ -1934,16 +1928,17 @@ property_get__name__(propertyobject *prop, void *Py_UNUSED(ignored)) } static int -property_set__name__(propertyobject *prop, PyObject *value, - void *Py_UNUSED(ignored)) +property_set__name__(PyObject *op, PyObject *value, void *Py_UNUSED(ignored)) { + propertyobject *prop = _propertyobject_CAST(op); Py_XSETREF(prop->prop_name, Py_XNewRef(value)); return 0; } static PyObject * -property_get___isabstractmethod__(propertyobject *prop, void *closure) +property_get___isabstractmethod__(PyObject *op, void *closure) { + propertyobject *prop = _propertyobject_CAST(op); int res = _PyObject_IsAbstract(prop->prop_get); if (res == -1) { return NULL; @@ -1971,9 +1966,8 @@ property_get___isabstractmethod__(propertyobject *prop, void *closure) } static PyGetSetDef property_getsetlist[] = { - {"__name__", (getter)property_get__name__, (setter)property_set__name__}, - {"__isabstractmethod__", - (getter)property_get___isabstractmethod__, NULL, + {"__name__", property_get__name__, property_set__name__, NULL, NULL}, + {"__isabstractmethod__", property_get___isabstractmethod__, NULL, NULL, NULL}, {NULL} /* Sentinel */ @@ -2024,7 +2018,7 @@ PyTypeObject PyDictProxy_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_MAPPING, /* tp_flags */ - 0, /* tp_doc */ + mappingproxy_new__doc__, /* tp_doc */ mappingproxy_traverse, /* tp_traverse */ 0, /* tp_clear */ mappingproxy_richcompare, /* tp_richcompare */ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 39778df603f43f..733a10a2e80b18 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -34,7 +34,7 @@ As of Python 3.6, this is compact and ordered. Basic idea is described here: dk_indices is actual hashtable. It holds index in entries, or DKIX_EMPTY(-1) or DKIX_DUMMY(-2). -Size of indices is dk_size. Type of each index in indices is vary on dk_size: +Size of indices is dk_size. Type of each index in indices varies with dk_size: * int8 for dk_size <= 128 * int16 for 256 <= dk_size <= 2**15 @@ -154,13 +154,31 @@ ASSERT_DICT_LOCKED(PyObject *op) _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); } #define ASSERT_DICT_LOCKED(op) ASSERT_DICT_LOCKED(_Py_CAST(PyObject*, op)) +#define ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op) \ + if (!_PyInterpreterState_GET()->stoptheworld.world_stopped) { \ + ASSERT_DICT_LOCKED(op); \ + } +#define ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(op) \ + if (!_PyInterpreterState_GET()->stoptheworld.world_stopped) { \ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); \ + } + #define IS_DICT_SHARED(mp) _PyObject_GC_IS_SHARED(mp) #define SET_DICT_SHARED(mp) _PyObject_GC_SET_SHARED(mp) #define LOAD_INDEX(keys, size, idx) _Py_atomic_load_int##size##_relaxed(&((const int##size##_t*)keys->dk_indices)[idx]); #define STORE_INDEX(keys, size, idx, value) _Py_atomic_store_int##size##_relaxed(&((int##size##_t*)keys->dk_indices)[idx], (int##size##_t)value); #define ASSERT_OWNED_OR_SHARED(mp) \ assert(_Py_IsOwnedByCurrentThread((PyObject *)mp) || IS_DICT_SHARED(mp)); -#define LOAD_KEYS_NENTRIES(d) + +#define LOCK_KEYS_IF_SPLIT(keys, kind) \ + if (kind == DICT_KEYS_SPLIT) { \ + LOCK_KEYS(keys); \ + } + +#define UNLOCK_KEYS_IF_SPLIT(keys, kind) \ + if (kind == DICT_KEYS_SPLIT) { \ + UNLOCK_KEYS(keys); \ + } static inline Py_ssize_t load_keys_nentries(PyDictObject *mp) @@ -183,9 +201,6 @@ set_values(PyDictObject *mp, PyDictValues *values) _Py_atomic_store_ptr_release(&mp->ma_values, values); } -// Defined until we get QSBR -#define _PyMem_FreeQsbr PyMem_Free - #define LOCK_KEYS(keys) PyMutex_LockFlags(&keys->dk_mutex, _Py_LOCK_DONT_DETACH) #define UNLOCK_KEYS(keys) PyMutex_Unlock(&keys->dk_mutex) @@ -196,7 +211,10 @@ set_values(PyDictObject *mp, PyDictValues *values) #define INCREF_KEYS(dk) _Py_atomic_add_ssize(&dk->dk_refcnt, 1) // Dec refs the keys object, giving the previous value #define DECREF_KEYS(dk) _Py_atomic_add_ssize(&dk->dk_refcnt, -1) -#define LOAD_KEYS_NENTIRES(keys) _Py_atomic_load_ssize_relaxed(&keys->dk_nentries) +#define LOAD_KEYS_NENTRIES(keys) _Py_atomic_load_ssize_relaxed(&keys->dk_nentries) + +#define INCREF_KEYS_FT(dk) dictkeys_incref(dk) +#define DECREF_KEYS_FT(dk, shared) dictkeys_decref(_PyInterpreterState_GET(), dk, shared) static inline void split_keys_entry_added(PyDictKeysObject *keys) { @@ -211,6 +229,8 @@ static inline void split_keys_entry_added(PyDictKeysObject *keys) #else /* Py_GIL_DISABLED */ #define ASSERT_DICT_LOCKED(op) +#define ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op) +#define ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(op) #define LOCK_KEYS(keys) #define UNLOCK_KEYS(keys) #define ASSERT_KEYS_LOCKED(keys) @@ -218,7 +238,11 @@ static inline void split_keys_entry_added(PyDictKeysObject *keys) #define STORE_SHARED_KEY(key, value) key = value #define INCREF_KEYS(dk) dk->dk_refcnt++ #define DECREF_KEYS(dk) dk->dk_refcnt-- -#define LOAD_KEYS_NENTIRES(keys) keys->dk_nentries +#define LOAD_KEYS_NENTRIES(keys) keys->dk_nentries +#define INCREF_KEYS_FT(dk) +#define DECREF_KEYS_FT(dk, shared) +#define LOCK_KEYS_IF_SPLIT(keys, kind) +#define UNLOCK_KEYS_IF_SPLIT(keys, kind) #define IS_DICT_SHARED(mp) (false) #define SET_DICT_SHARED(mp) #define LOAD_INDEX(keys, size, idx) ((const int##size##_t*)(keys->dk_indices))[idx] @@ -364,70 +388,36 @@ static int dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_value, PyObject **result, int incref_result); -#include "clinic/dictobject.c.h" - - -#ifdef WITH_FREELISTS -static struct _Py_dict_freelist * -get_dict_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - return &freelists->dicts; -} - -static struct _Py_dictkeys_freelist * -get_dictkeys_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - return &freelists->dictkeys; -} +#ifndef NDEBUG +static int _PyObject_InlineValuesConsistencyCheck(PyObject *obj); #endif +#include "clinic/dictobject.c.h" -void -_PyDict_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) -{ -#ifdef WITH_FREELISTS - struct _Py_dict_freelist *freelist = &freelists->dicts; - while (freelist->numfree > 0) { - PyDictObject *op = freelist->items[--freelist->numfree]; - assert(PyDict_CheckExact(op)); - PyObject_GC_Del(op); - } - struct _Py_dictkeys_freelist *keys_freelist = &freelists->dictkeys; - while (keys_freelist->numfree > 0) { - PyMem_Free(keys_freelist->items[--keys_freelist->numfree]); - } - if (is_finalization) { - freelist->numfree = -1; - keys_freelist->numfree = -1; - } -#endif -} static inline Py_hash_t unicode_get_hash(PyObject *o) { assert(PyUnicode_CheckExact(o)); - return _PyASCIIObject_CAST(o)->hash; + return FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyASCIIObject_CAST(o)->hash); } /* Print summary info about the state of the optimized allocator */ void _PyDict_DebugMallocStats(FILE *out) { -#ifdef WITH_FREELISTS - struct _Py_dict_freelist *dict_freelist = get_dict_freelist(); _PyDebugAllocatorStats(out, "free PyDictObject", - dict_freelist->numfree, sizeof(PyDictObject)); - struct _Py_dictkeys_freelist *dictkeys_freelist = get_dictkeys_freelist(); + _Py_FREELIST_SIZE(dicts), + sizeof(PyDictObject)); _PyDebugAllocatorStats(out, "free PyDictKeysObject", - dictkeys_freelist->numfree, sizeof(PyDictKeysObject)); -#endif + _Py_FREELIST_SIZE(dictkeys), + sizeof(PyDictKeysObject)); } #define DK_MASK(dk) (DK_SIZE(dk)-1) +#define _Py_DICT_IMMORTAL_INITIAL_REFCNT PY_SSIZE_T_MIN + static void free_keys_object(PyDictKeysObject *keys, bool use_qsbr); /* PyDictKeysObject has refcounts like PyObject does, so we have the @@ -440,11 +430,12 @@ static void free_keys_object(PyDictKeysObject *keys, bool use_qsbr); static inline void dictkeys_incref(PyDictKeysObject *dk) { - if (dk->dk_refcnt == _Py_IMMORTAL_REFCNT) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) < 0) { + assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) == _Py_DICT_IMMORTAL_INITIAL_REFCNT); return; } #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif INCREF_KEYS(dk); } @@ -452,12 +443,13 @@ dictkeys_incref(PyDictKeysObject *dk) static inline void dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk, bool use_qsbr) { - if (dk->dk_refcnt == _Py_IMMORTAL_REFCNT) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) < 0) { + assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) == _Py_DICT_IMMORTAL_INITIAL_REFCNT); return; } - assert(dk->dk_refcnt > 0); + assert(FT_ATOMIC_LOAD_SSIZE(dk->dk_refcnt) > 0); #ifdef Py_REF_DEBUG - _Py_DecRefTotal(_PyInterpreterState_GET()); + _Py_DecRefTotal(_PyThreadState_GET()); #endif if (DECREF_KEYS(dk) == 1) { if (DK_IS_UNICODE(dk)) { @@ -596,11 +588,14 @@ estimate_log2_keysize(Py_ssize_t n) /* This immutable, empty PyDictKeysObject is used for PyDict_Clear() * (which cannot fail and thus can do no allocation). + * + * See https://github.com/python/cpython/pull/127568#discussion_r1868070614 + * for the rationale of using dk_log2_index_bytes=3 instead of 0. */ static PyDictKeysObject empty_keys_struct = { - _Py_IMMORTAL_REFCNT, /* dk_refcnt */ + _Py_DICT_IMMORTAL_INITIAL_REFCNT, /* dk_refcnt */ 0, /* dk_log2_size */ - 0, /* dk_log2_index_bytes */ + 3, /* dk_log2_index_bytes */ DICT_KEYS_UNICODE, /* dk_kind */ #ifdef Py_GIL_DISABLED {0}, /* dk_mutex */ @@ -627,8 +622,9 @@ static inline int get_index_from_order(PyDictObject *mp, Py_ssize_t i) { assert(mp->ma_used <= SHARED_KEYS_MAX_SIZE); - assert(i < (((char *)mp->ma_values)[-2])); - return ((char *)mp->ma_values)[-3-i]; + assert(i < mp->ma_values->size); + uint8_t *array = get_insertion_order_array(mp->ma_values); + return array[i]; } #ifdef DEBUG_PYDICT @@ -651,6 +647,8 @@ dump_entries(PyDictKeysObject *dk) int _PyDict_CheckConsistency(PyObject *op, int check_content) { + ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op); + #define CHECK(expr) \ do { if (!(expr)) { _PyObject_ASSERT_FAILED_MSG(op, Py_STRINGIFY(expr)); } } while (0) @@ -662,10 +660,15 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) int splitted = _PyDict_HasSplitTable(mp); Py_ssize_t usable = USABLE_FRACTION(DK_SIZE(keys)); + // In the free-threaded build, shared keys may be concurrently modified, + // so use atomic loads. + Py_ssize_t dk_usable = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_usable); + Py_ssize_t dk_nentries = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_nentries); + CHECK(0 <= mp->ma_used && mp->ma_used <= usable); - CHECK(0 <= keys->dk_usable && keys->dk_usable <= usable); - CHECK(0 <= keys->dk_nentries && keys->dk_nentries <= usable); - CHECK(keys->dk_usable + keys->dk_nentries <= usable); + CHECK(0 <= dk_usable && dk_usable <= usable); + CHECK(0 <= dk_nentries && dk_nentries <= usable); + CHECK(dk_usable + dk_nentries <= usable); if (!splitted) { /* combined table */ @@ -675,9 +678,14 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) else { CHECK(keys->dk_kind == DICT_KEYS_SPLIT); CHECK(mp->ma_used <= SHARED_KEYS_MAX_SIZE); + if (mp->ma_values->embedded) { + CHECK(mp->ma_values->embedded == 1); + CHECK(mp->ma_values->valid == 1); + } } if (check_content) { + LOCK_KEYS_IF_SPLIT(keys, keys->dk_kind); for (Py_ssize_t i=0; i < DK_SIZE(keys); i++) { Py_ssize_t ix = dictkeys_get_index(keys, i); CHECK(DKIX_DUMMY <= ix && ix <= usable); @@ -733,6 +741,7 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) CHECK(mp->ma_values->values[index] != NULL); } } + UNLOCK_KEYS_IF_SPLIT(keys, keys->dk_kind); } return 1; @@ -743,7 +752,6 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) static PyDictKeysObject* new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) { - PyDictKeysObject *dk; Py_ssize_t usable; int log2_bytes; size_t entry_size = unicode ? sizeof(PyDictUnicodeEntry) : sizeof(PyDictKeyEntry); @@ -766,15 +774,11 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) log2_bytes = log2_size + 2; } -#ifdef WITH_FREELISTS - struct _Py_dictkeys_freelist *freelist = get_dictkeys_freelist(); - if (log2_size == PyDict_LOG_MINSIZE && unicode && freelist->numfree > 0) { - dk = freelist->items[--freelist->numfree]; - OBJECT_STAT_INC(from_freelist); + PyDictKeysObject *dk = NULL; + if (log2_size == PyDict_LOG_MINSIZE && unicode) { + dk = _Py_FREELIST_POP_MEM(dictkeys); } - else -#endif - { + if (dk == NULL) { dk = PyMem_Malloc(sizeof(PyDictKeysObject) + ((size_t)1 << log2_bytes) + entry_size * usable); @@ -784,7 +788,7 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) } } #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif dk->dk_refcnt = 1; dk->dk_log2_size = log2_size; @@ -806,51 +810,56 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr) { #ifdef Py_GIL_DISABLED if (use_qsbr) { - _PyMem_FreeQsbr(keys); + _PyMem_FreeDelayed(keys); return; } #endif -#ifdef WITH_FREELISTS - struct _Py_dictkeys_freelist *freelist = get_dictkeys_freelist(); - if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE - && freelist->numfree < PyDict_MAXFREELIST - && freelist->numfree >= 0 - && DK_IS_UNICODE(keys)) { - freelist->items[freelist->numfree++] = keys; - OBJECT_STAT_INC(to_freelist); - return; + if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE && keys->dk_kind == DICT_KEYS_UNICODE) { + _Py_FREELIST_FREE(dictkeys, keys, PyMem_Free); } -#endif - PyMem_Free(keys); + else { + PyMem_Free(keys); + } +} + +static size_t +values_size_from_count(size_t count) +{ + assert(count >= 1); + size_t suffix_size = _Py_SIZE_ROUND_UP(count, sizeof(PyObject *)); + assert(suffix_size < 128); + assert(suffix_size % sizeof(PyObject *) == 0); + return (count + 1) * sizeof(PyObject *) + suffix_size; } +#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys) + static inline PyDictValues* new_values(size_t size) { - assert(size >= 1); - size_t prefix_size = _Py_SIZE_ROUND_UP(size+2, sizeof(PyObject *)); - assert(prefix_size < 256); - size_t n = prefix_size + size * sizeof(PyObject *); - uint8_t *mem = PyMem_Malloc(n); - if (mem == NULL) { + size_t n = values_size_from_count(size); + PyDictValues *res = (PyDictValues *)PyMem_Malloc(n); + if (res == NULL) { return NULL; } - assert(prefix_size % sizeof(PyObject *) == 0); - mem[prefix_size-1] = (uint8_t)prefix_size; - return (PyDictValues*)(mem + prefix_size); + res->embedded = 0; + res->size = 0; + assert(size < 256); + res->capacity = (uint8_t)size; + return res; } static inline void free_values(PyDictValues *values, bool use_qsbr) { - int prefix_size = DICT_VALUES_SIZE(values); + assert(values->embedded == 0); #ifdef Py_GIL_DISABLED if (use_qsbr) { - _PyMem_FreeQsbr(((char *)values)-prefix_size); + _PyMem_FreeDelayed(values); return; } #endif - PyMem_Free(((char *)values)-prefix_size); + PyMem_Free(values); } /* Consumes a reference to the keys object */ @@ -859,20 +868,9 @@ new_dict(PyInterpreterState *interp, PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free_values_on_failure) { - PyDictObject *mp; assert(keys != NULL); -#ifdef WITH_FREELISTS - struct _Py_dict_freelist *freelist = get_dict_freelist(); - if (freelist->numfree > 0) { - mp = freelist->items[--freelist->numfree]; - assert (mp != NULL); - assert (Py_IS_TYPE(mp, &PyDict_Type)); - OBJECT_STAT_INC(from_freelist); - _Py_NewReference((PyObject *)mp); - } - else -#endif - { + PyDictObject *mp = _Py_FREELIST_POP(PyDictObject, dicts); + if (mp == NULL) { mp = PyObject_GC_New(PyDictObject, &PyDict_Type); if (mp == NULL) { dictkeys_decref(interp, keys, false); @@ -882,43 +880,25 @@ new_dict(PyInterpreterState *interp, return NULL; } } + assert(Py_IS_TYPE(mp, &PyDict_Type)); mp->ma_keys = keys; mp->ma_values = values; mp->ma_used = used; - mp->ma_version_tag = DICT_NEXT_VERSION(interp); + mp->_ma_watcher_tag = 0; ASSERT_CONSISTENT(mp); + _PyObject_GC_TRACK(mp); return (PyObject *)mp; } -static inline size_t -shared_keys_usable_size(PyDictKeysObject *keys) -{ -#ifdef Py_GIL_DISABLED - // dk_usable will decrease for each instance that is created and each - // value that is added. dk_nentries will increase for each value that - // is added. We want to always return the right value or larger. - // We therefore increase dk_nentries first and we decrease dk_usable - // second, and conversely here we read dk_usable first and dk_entries - // second (to avoid the case where we read entries before the increment - // and read usable after the decrement) - return (size_t)(_Py_atomic_load_ssize_acquire(&keys->dk_usable) + - _Py_atomic_load_ssize_acquire(&keys->dk_nentries)); -#else - return (size_t)keys->dk_nentries + (size_t)keys->dk_usable; -#endif -} - -/* Consumes a reference to the keys object */ static PyObject * new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) { size_t size = shared_keys_usable_size(keys); PyDictValues *values = new_values(size); if (values == NULL) { - dictkeys_decref(interp, keys, false); return PyErr_NoMemory(); } - ((char *)values)[-2] = 0; + dictkeys_incref(keys); for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } @@ -980,7 +960,7 @@ clone_combined_dict_keys(PyDictObject *orig) we have it now; calling dictkeys_incref would be an error as keys->dk_refcnt is already set to 1 (after memcpy). */ #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif return keys; } @@ -1017,7 +997,7 @@ lookdict_index(PyDictKeysObject *k, Py_hash_t hash, Py_ssize_t index) static inline Py_ALWAYS_INLINE Py_ssize_t do_lookup(PyDictObject *mp, PyDictKeysObject *dk, PyObject *key, Py_hash_t hash, - Py_ssize_t (*check_lookup)(PyDictObject *, PyDictKeysObject *, void *, Py_ssize_t ix, PyObject *key, Py_hash_t)) + int (*check_lookup)(PyDictObject *, PyDictKeysObject *, void *, Py_ssize_t ix, PyObject *key, Py_hash_t)) { void *ep0 = _DK_ENTRIES(dk); size_t mask = DK_MASK(dk); @@ -1027,7 +1007,7 @@ do_lookup(PyDictObject *mp, PyDictKeysObject *dk, PyObject *key, Py_hash_t hash, for (;;) { ix = dictkeys_get_index(dk, i); if (ix >= 0) { - Py_ssize_t cmp = check_lookup(mp, dk, ep0, ix, key, hash); + int cmp = check_lookup(mp, dk, ep0, ix, key, hash); if (cmp < 0) { return cmp; } else if (cmp) { @@ -1043,7 +1023,7 @@ do_lookup(PyDictObject *mp, PyDictKeysObject *dk, PyObject *key, Py_hash_t hash, // Manual loop unrolling ix = dictkeys_get_index(dk, i); if (ix >= 0) { - Py_ssize_t cmp = check_lookup(mp, dk, ep0, ix, key, hash); + int cmp = check_lookup(mp, dk, ep0, ix, key, hash); if (cmp < 0) { return cmp; } else if (cmp) { @@ -1059,7 +1039,7 @@ do_lookup(PyDictObject *mp, PyDictKeysObject *dk, PyObject *key, Py_hash_t hash, Py_UNREACHABLE(); } -static inline Py_ALWAYS_INLINE Py_ssize_t +static inline int compare_unicode_generic(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { @@ -1067,7 +1047,6 @@ compare_unicode_generic(PyDictObject *mp, PyDictKeysObject *dk, assert(ep->me_key != NULL); assert(PyUnicode_CheckExact(ep->me_key)); assert(!PyUnicode_CheckExact(key)); - // TODO: Thread safety if (unicode_get_hash(ep->me_key) == hash) { PyObject *startkey = ep->me_key; @@ -1095,15 +1074,16 @@ unicodekeys_lookup_generic(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key return do_lookup(mp, dk, key, hash, compare_unicode_generic); } -static inline Py_ALWAYS_INLINE Py_ssize_t +static inline int compare_unicode_unicode(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; - assert(ep->me_key != NULL); - assert(PyUnicode_CheckExact(ep->me_key)); - if (ep->me_key == key || - (unicode_get_hash(ep->me_key) == hash && unicode_eq(ep->me_key, key))) { + PyObject *ep_key = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key); + assert(ep_key != NULL); + assert(PyUnicode_CheckExact(ep_key)); + if (ep_key == key || + (unicode_get_hash(ep_key) == hash && unicode_eq(ep_key, key))) { return 1; } return 0; @@ -1115,7 +1095,7 @@ unicodekeys_lookup_unicode(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) return do_lookup(NULL, dk, key, hash, compare_unicode_unicode); } -static inline Py_ALWAYS_INLINE Py_ssize_t +static inline int compare_generic(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { @@ -1149,6 +1129,53 @@ dictkeys_generic_lookup(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, P return do_lookup(mp, dk, key, hash, compare_generic); } +static bool +check_keys_unicode(PyDictKeysObject *dk, PyObject *key) +{ + return PyUnicode_CheckExact(key) && (dk->dk_kind != DICT_KEYS_GENERAL); +} + +static Py_ssize_t +hash_unicode_key(PyObject *key) +{ + assert(PyUnicode_CheckExact(key)); + Py_hash_t hash = unicode_get_hash(key); + if (hash == -1) { + hash = PyUnicode_Type.tp_hash(key); + assert(hash != -1); + } + return hash; +} + +#ifdef Py_GIL_DISABLED +static Py_ssize_t +unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key, + Py_hash_t hash); +#endif + +static Py_ssize_t +unicodekeys_lookup_split(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + Py_ssize_t ix; + assert(dk->dk_kind == DICT_KEYS_SPLIT); + assert(PyUnicode_CheckExact(key)); + +#ifdef Py_GIL_DISABLED + // A split dictionaries keys can be mutated by other dictionaries + // but if we have a unicode key we can avoid locking the shared + // keys. + ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash); + if (ix == DKIX_KEY_CHANGED) { + LOCK_KEYS(dk); + ix = unicodekeys_lookup_unicode(dk, key, hash); + UNLOCK_KEYS(dk); + } +#else + ix = unicodekeys_lookup_unicode(dk, key, hash); +#endif + return ix; +} + /* Lookup a string in a (all unicode) dict keys. * Returns DKIX_ERROR if key is not a string, * or if the dict keys is not all strings. @@ -1158,10 +1185,36 @@ dictkeys_generic_lookup(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, P Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dk, PyObject *key) { - DictKeysKind kind = dk->dk_kind; - if (!PyUnicode_CheckExact(key) || kind == DICT_KEYS_GENERAL) { + if (!check_keys_unicode(dk, key)) { + return DKIX_ERROR; + } + Py_hash_t hash = hash_unicode_key(key); + return unicodekeys_lookup_unicode(dk, key, hash); +} + +Py_ssize_t +_PyDictKeys_StringLookupAndVersion(PyDictKeysObject *dk, PyObject *key, uint32_t *version) +{ + if (!check_keys_unicode(dk, key)) { return DKIX_ERROR; } + Py_ssize_t ix; + Py_hash_t hash = hash_unicode_key(key); + LOCK_KEYS(dk); + ix = unicodekeys_lookup_unicode(dk, key, hash); + *version = _PyDictKeys_GetVersionForCurrentState(_PyInterpreterState_GET(), dk); + UNLOCK_KEYS(dk); + return ix; +} + +/* Like _PyDictKeys_StringLookup() but only works on split keys. Note + * that in free-threaded builds this locks the keys object as required. + */ +Py_ssize_t +_PyDictKeys_StringLookupSplit(PyDictKeysObject* dk, PyObject *key) +{ + assert(dk->dk_kind == DICT_KEYS_SPLIT); + assert(PyUnicode_CheckExact(key)); Py_hash_t hash = unicode_get_hash(key); if (hash == -1) { hash = PyUnicode_Type.tp_hash(key); @@ -1170,7 +1223,7 @@ _PyDictKeys_StringLookup(PyDictKeysObject* dk, PyObject *key) return DKIX_ERROR; } } - return unicodekeys_lookup_unicode(dk, key, hash); + return unicodekeys_lookup_split(dk, key, hash); } /* @@ -1194,17 +1247,33 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu PyDictKeysObject *dk; DictKeysKind kind; Py_ssize_t ix; - // TODO: Thread safety + + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(mp); start: dk = mp->ma_keys; kind = dk->dk_kind; if (kind != DICT_KEYS_GENERAL) { if (PyUnicode_CheckExact(key)) { +#ifdef Py_GIL_DISABLED + if (kind == DICT_KEYS_SPLIT) { + ix = unicodekeys_lookup_split(dk, key, hash); + } + else { + ix = unicodekeys_lookup_unicode(dk, key, hash); + } +#else ix = unicodekeys_lookup_unicode(dk, key, hash); +#endif } else { + INCREF_KEYS_FT(dk); + LOCK_KEYS_IF_SPLIT(dk, kind); + ix = unicodekeys_lookup_generic(mp, dk, key, hash); + + UNLOCK_KEYS_IF_SPLIT(dk, kind); + DECREF_KEYS_FT(dk, IS_DICT_SHARED(mp)); if (ix == DKIX_KEY_CHANGED) { goto start; } @@ -1238,10 +1307,10 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu return ix; } +#ifdef Py_GIL_DISABLED static inline void ensure_shared_on_read(PyDictObject *mp) { -#ifdef Py_GIL_DISABLED if (!_Py_IsOwnedByCurrentThread((PyObject *)mp) && !IS_DICT_SHARED(mp)) { // The first time we access a dict from a non-owning thread we mark it // as shared. This ensures that a concurrent resize operation will @@ -1253,8 +1322,8 @@ ensure_shared_on_read(PyDictObject *mp) } Py_END_CRITICAL_SECTION(); } -#endif } +#endif static inline void ensure_shared_on_resize(PyDictObject *mp) @@ -1276,10 +1345,24 @@ ensure_shared_on_resize(PyDictObject *mp) #endif } +static inline void +ensure_shared_on_keys_version_assignment(PyDictObject *mp) +{ + ASSERT_DICT_LOCKED((PyObject *) mp); + #ifdef Py_GIL_DISABLED + if (!IS_DICT_SHARED(mp)) { + // This ensures that a concurrent resize operation will delay + // freeing the old keys or values using QSBR, which is necessary to + // safely allow concurrent reads without locking. + SET_DICT_SHARED(mp); + } + #endif +} + #ifdef Py_GIL_DISABLED -static inline Py_ALWAYS_INLINE -Py_ssize_t compare_unicode_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, +static inline Py_ALWAYS_INLINE int +compare_unicode_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; @@ -1288,7 +1371,7 @@ Py_ssize_t compare_unicode_generic_threadsafe(PyDictObject *mp, PyDictKeysObject assert(!PyUnicode_CheckExact(key)); if (startkey != NULL) { - if (!_Py_TryIncref(&ep->me_key, startkey)) { + if (!_Py_TryIncrefCompare(&ep->me_key, startkey)) { return DKIX_KEY_CHANGED; } @@ -1321,7 +1404,7 @@ unicodekeys_lookup_generic_threadsafe(PyDictObject *mp, PyDictKeysObject* dk, Py return do_lookup(mp, dk, key, hash, compare_unicode_generic_threadsafe); } -static inline Py_ALWAYS_INLINE Py_ssize_t +static inline Py_ALWAYS_INLINE int compare_unicode_unicode_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { @@ -1336,7 +1419,7 @@ compare_unicode_unicode_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, return unicode_get_hash(startkey) == hash && unicode_eq(startkey, key); } else { - if (!_Py_TryIncref(&ep->me_key, startkey)) { + if (!_Py_TryIncrefCompare(&ep->me_key, startkey)) { return DKIX_KEY_CHANGED; } if (unicode_get_hash(startkey) == hash && unicode_eq(startkey, key)) { @@ -1355,8 +1438,8 @@ unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key, Py_ha return do_lookup(NULL, dk, key, hash, compare_unicode_unicode_threadsafe); } -static inline Py_ALWAYS_INLINE -Py_ssize_t compare_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, +static inline Py_ALWAYS_INLINE int +compare_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { PyDictKeyEntry *ep = &((PyDictKeyEntry *)ep0)[ix]; @@ -1366,7 +1449,7 @@ Py_ssize_t compare_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, } Py_ssize_t ep_hash = _Py_atomic_load_ssize_relaxed(&ep->me_hash); if (ep_hash == hash) { - if (startkey == NULL || !_Py_TryIncref(&ep->me_key, startkey)) { + if (startkey == NULL || !_Py_TryIncrefCompare(&ep->me_key, startkey)) { return DKIX_KEY_CHANGED; } int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); @@ -1392,7 +1475,7 @@ dictkeys_generic_lookup_threadsafe(PyDictObject *mp, PyDictKeysObject* dk, PyObj return do_lookup(mp, dk, key, hash, compare_generic_threadsafe); } -static Py_ssize_t +Py_ssize_t _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr) { PyDictKeysObject *dk; @@ -1422,7 +1505,7 @@ _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyOb if (values == NULL) goto read_failed; - uint8_t capacity = _Py_atomic_load_uint8_relaxed(&DICT_VALUES_SIZE(values)); + uint8_t capacity = _Py_atomic_load_uint8_relaxed(&values->capacity); if (ix >= (Py_ssize_t)capacity) goto read_failed; @@ -1484,12 +1567,75 @@ _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyOb *value_addr = value; if (value != NULL) { assert(ix >= 0); - Py_INCREF(value); + _Py_NewRefWithLock(value); } Py_END_CRITICAL_SECTION(); return ix; } +Py_ssize_t +_Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject *key, Py_hash_t hash, _PyStackRef *value_addr) +{ + PyDictKeysObject *dk = _Py_atomic_load_ptr(&mp->ma_keys); + if (dk->dk_kind == DICT_KEYS_UNICODE && PyUnicode_CheckExact(key)) { + Py_ssize_t ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash); + if (ix == DKIX_EMPTY) { + *value_addr = PyStackRef_NULL; + return ix; + } + else if (ix >= 0) { + PyObject **addr_of_value = &DK_UNICODE_ENTRIES(dk)[ix].me_value; + PyObject *value = _Py_atomic_load_ptr(addr_of_value); + if (value == NULL) { + *value_addr = PyStackRef_NULL; + return DKIX_EMPTY; + } + if (_Py_IsImmortal(value) || _PyObject_HasDeferredRefcount(value)) { + *value_addr = (_PyStackRef){ .bits = (uintptr_t)value | Py_TAG_DEFERRED }; + return ix; + } + if (_Py_TryIncrefCompare(addr_of_value, value)) { + *value_addr = PyStackRef_FromPyObjectSteal(value); + return ix; + } + } + } + + PyObject *obj; + Py_ssize_t ix = _Py_dict_lookup_threadsafe(mp, key, hash, &obj); + if (ix >= 0 && obj != NULL) { + *value_addr = PyStackRef_FromPyObjectSteal(obj); + } + else { + *value_addr = PyStackRef_NULL; + } + return ix; +} + +#else // Py_GIL_DISABLED + +Py_ssize_t +_Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr) +{ + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, value_addr); + Py_XNewRef(*value_addr); + return ix; +} + +Py_ssize_t +_Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject *key, Py_hash_t hash, _PyStackRef *value_addr) +{ + PyObject *val; + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &val); + if (val == NULL) { + *value_addr = PyStackRef_NULL; + } + else { + *value_addr = PyStackRef_FromPyObjectNew(val); + } + return ix; +} + #endif int @@ -1507,59 +1653,24 @@ _PyDict_HasOnlyStringKeys(PyObject *dict) return 1; } -#define MAINTAIN_TRACKING(mp, key, value) \ - do { \ - if (!_PyObject_GC_IS_TRACKED(mp)) { \ - if (_PyObject_GC_MAY_BE_TRACKED(key) || \ - _PyObject_GC_MAY_BE_TRACKED(value)) { \ - _PyObject_GC_TRACK(mp); \ - } \ - } \ - } while(0) - void -_PyDict_MaybeUntrack(PyObject *op) +_PyDict_EnablePerThreadRefcounting(PyObject *op) { - PyDictObject *mp; - PyObject *value; - Py_ssize_t i, numentries; - - if (!PyDict_CheckExact(op) || !_PyObject_GC_IS_TRACKED(op)) + assert(PyDict_Check(op)); +#ifdef Py_GIL_DISABLED + Py_ssize_t id = _PyObject_AssignUniqueId(op); + if (id == _Py_INVALID_UNIQUE_ID) { return; - - mp = (PyDictObject *) op; - numentries = mp->ma_keys->dk_nentries; - if (_PyDict_HasSplitTable(mp)) { - for (i = 0; i < numentries; i++) { - if ((value = mp->ma_values->values[i]) == NULL) - continue; - if (_PyObject_GC_MAY_BE_TRACKED(value)) { - return; - } - } } - else { - if (DK_IS_UNICODE(mp->ma_keys)) { - PyDictUnicodeEntry *ep0 = DK_UNICODE_ENTRIES(mp->ma_keys); - for (i = 0; i < numentries; i++) { - if ((value = ep0[i].me_value) == NULL) - continue; - if (_PyObject_GC_MAY_BE_TRACKED(value)) - return; - } - } - else { - PyDictKeyEntry *ep0 = DK_ENTRIES(mp->ma_keys); - for (i = 0; i < numentries; i++) { - if ((value = ep0[i].me_value) == NULL) - continue; - if (_PyObject_GC_MAY_BE_TRACKED(value) || - _PyObject_GC_MAY_BE_TRACKED(ep0[i].me_key)) - return; - } - } + if ((uint64_t)id >= (uint64_t)DICT_UNIQUE_ID_MAX) { + _PyObject_ReleaseUniqueId(id); + return; } - _PyObject_GC_UNTRACK(op); + + PyDictObject *mp = (PyDictObject *)op; + assert((mp->_ma_watcher_tag >> DICT_UNIQUE_ID_SHIFT) == 0); + mp->_ma_watcher_tag += (uint64_t)id << DICT_UNIQUE_ID_SHIFT; +#endif } static inline int @@ -1597,31 +1708,6 @@ insertion_resize(PyInterpreterState *interp, PyDictObject *mp, int unicode) return dictresize(interp, mp, calculate_log2_keysize(GROWTH_RATE(mp)), unicode); } -static Py_ssize_t -insert_into_splitdictkeys(PyDictKeysObject *keys, PyObject *name, Py_hash_t hash) -{ - assert(PyUnicode_CheckExact(name)); - ASSERT_KEYS_LOCKED(keys); - - Py_ssize_t ix = unicodekeys_lookup_unicode(keys, name, hash); - if (ix == DKIX_EMPTY) { - if (keys->dk_usable <= 0) { - return DKIX_EMPTY; - } - /* Insert into new slot. */ - keys->dk_version = 0; - Py_ssize_t hashpos = find_empty_slot(keys, hash); - ix = keys->dk_nentries; - PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(keys)[ix]; - dictkeys_set_index(keys, hashpos, ix); - assert(ep->me_key == NULL); - ep->me_key = Py_NewRef(name); - split_keys_entry_added(keys); - } - assert (ix < SHARED_KEYS_MAX_SIZE); - return ix; -} - static inline int insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, Py_hash_t hash, PyObject *key, PyObject *value) @@ -1633,6 +1719,9 @@ insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, } } + _PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value); + FT_ATOMIC_STORE_UINT32_RELAXED(mp->ma_keys->dk_version, 0); + Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash); dictkeys_set_index(mp->ma_keys, hashpos, mp->ma_keys->dk_nentries); @@ -1655,39 +1744,57 @@ insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, return 0; } -static int -insert_split_dict(PyInterpreterState *interp, PyDictObject *mp, - Py_hash_t hash, PyObject *key, PyObject *value) +static Py_ssize_t +insert_split_key(PyDictKeysObject *keys, PyObject *key, Py_hash_t hash) { - PyDictKeysObject *keys = mp->ma_keys; - LOCK_KEYS(keys); - if (keys->dk_usable <= 0) { - /* Need to resize. */ - UNLOCK_KEYS(keys); - int ins = insertion_resize(interp, mp, 1); - if (ins < 0) { - return -1; - } - assert(!_PyDict_HasSplitTable(mp)); - return insert_combined_dict(interp, mp, hash, key, value); - } - - Py_ssize_t hashpos = find_empty_slot(keys, hash); - dictkeys_set_index(keys, hashpos, keys->dk_nentries); + assert(PyUnicode_CheckExact(key)); + Py_ssize_t ix; - PyDictUnicodeEntry *ep; - ep = &DK_UNICODE_ENTRIES(keys)[keys->dk_nentries]; - STORE_SHARED_KEY(ep->me_key, key); - Py_ssize_t index = keys->dk_nentries; - _PyDictValues_AddToInsertionOrder(mp->ma_values, index); - assert (mp->ma_values->values[index] == NULL); - STORE_SPLIT_VALUE(mp, index, value); +#ifdef Py_GIL_DISABLED + ix = unicodekeys_lookup_unicode_threadsafe(keys, key, hash); + if (ix >= 0) { + return ix; + } +#endif - split_keys_entry_added(keys); - assert(keys->dk_usable >= 0); + LOCK_KEYS(keys); + ix = unicodekeys_lookup_unicode(keys, key, hash); + if (ix == DKIX_EMPTY && keys->dk_usable > 0) { + // Insert into new slot + FT_ATOMIC_STORE_UINT32_RELAXED(keys->dk_version, 0); + Py_ssize_t hashpos = find_empty_slot(keys, hash); + ix = keys->dk_nentries; + dictkeys_set_index(keys, hashpos, ix); + PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(keys)[ix]; + STORE_SHARED_KEY(ep->me_key, Py_NewRef(key)); + split_keys_entry_added(keys); + } + assert (ix < SHARED_KEYS_MAX_SIZE); UNLOCK_KEYS(keys); - return 0; + return ix; +} + +static void +insert_split_value(PyInterpreterState *interp, PyDictObject *mp, PyObject *key, PyObject *value, Py_ssize_t ix) +{ + assert(PyUnicode_CheckExact(key)); + ASSERT_DICT_LOCKED(mp); + PyObject *old_value = mp->ma_values->values[ix]; + if (old_value == NULL) { + _PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value); + STORE_SPLIT_VALUE(mp, ix, Py_NewRef(value)); + _PyDictValues_AddToInsertionOrder(mp->ma_values, ix); + STORE_USED(mp, mp->ma_used + 1); + } + else { + _PyDict_NotifyEvent(interp, PyDict_EVENT_MODIFIED, mp, key, value); + STORE_SPLIT_VALUE(mp, ix, Py_NewRef(value)); + // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, + // when dict only holds the strong reference to value in ep->me_value. + Py_DECREF(old_value); + } + ASSERT_CONSISTENT(mp); } /* @@ -1710,55 +1817,49 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); } - Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &old_value); - if (ix == DKIX_ERROR) - goto Fail; + if (_PyDict_HasSplitTable(mp)) { + Py_ssize_t ix = insert_split_key(mp->ma_keys, key, hash); + if (ix != DKIX_EMPTY) { + insert_split_value(interp, mp, key, value, ix); + Py_DECREF(key); + Py_DECREF(value); + return 0; + } + + /* No space in shared keys. Resize and continue below. */ + if (insertion_resize(interp, mp, 1) < 0) { + goto Fail; + } + } - MAINTAIN_TRACKING(mp, key, value); + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &old_value); + if (ix == DKIX_ERROR) + goto Fail; if (ix == DKIX_EMPTY) { - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, value); + assert(!_PyDict_HasSplitTable(mp)); /* Insert into new slot. */ - mp->ma_keys->dk_version = 0; assert(old_value == NULL); - - if (!_PyDict_HasSplitTable(mp)) { - if (insert_combined_dict(interp, mp, hash, key, value) < 0) { - goto Fail; - } - } - else { - if (insert_split_dict(interp, mp, hash, key, value) < 0) - goto Fail; + if (insert_combined_dict(interp, mp, hash, key, value) < 0) { + goto Fail; } - - mp->ma_used++; - mp->ma_version_tag = new_version; + STORE_USED(mp, mp->ma_used + 1); ASSERT_CONSISTENT(mp); return 0; } if (old_value != value) { - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_MODIFIED, mp, key, value); - if (_PyDict_HasSplitTable(mp)) { - mp->ma_values->values[ix] = value; - if (old_value == NULL) { - _PyDictValues_AddToInsertionOrder(mp->ma_values, ix); - mp->ma_used++; - } + _PyDict_NotifyEvent(interp, PyDict_EVENT_MODIFIED, mp, key, value); + assert(old_value != NULL); + assert(!_PyDict_HasSplitTable(mp)); + if (DK_IS_UNICODE(mp->ma_keys)) { + PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; + STORE_VALUE(ep, value); } else { - assert(old_value != NULL); - if (DK_IS_UNICODE(mp->ma_keys)) { - DK_UNICODE_ENTRIES(mp->ma_keys)[ix].me_value = value; - } - else { - DK_ENTRIES(mp->ma_keys)[ix].me_value = value; - } + PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; + STORE_VALUE(ep, value); } - mp->ma_version_tag = new_version; } Py_XDECREF(old_value); /* which **CAN** re-enter (see issue #22653) */ ASSERT_CONSISTENT(mp); @@ -1780,9 +1881,6 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, assert(mp->ma_keys == Py_EMPTY_KEYS); ASSERT_DICT_LOCKED(mp); - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, value); - int unicode = PyUnicode_CheckExact(key); PyDictKeysObject *newkeys = new_keys_object( interp, PyDict_LOG_MINSIZE, unicode); @@ -1791,26 +1889,25 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, Py_DECREF(value); return -1; } + _PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value); + /* We don't decref Py_EMPTY_KEYS here because it is immortal. */ assert(mp->ma_values == NULL); - MAINTAIN_TRACKING(mp, key, value); - size_t hashpos = (size_t)hash & (PyDict_MINSIZE-1); dictkeys_set_index(newkeys, hashpos, 0); if (unicode) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(newkeys); ep->me_key = key; - ep->me_value = value; + STORE_VALUE(ep, value); } else { PyDictKeyEntry *ep = DK_ENTRIES(newkeys); ep->me_key = key; ep->me_hash = hash; - ep->me_value = value; + STORE_VALUE(ep, value); } - mp->ma_used++; - mp->ma_version_tag = new_version; + STORE_USED(mp, mp->ma_used + 1); newkeys->dk_usable--; newkeys->dk_nentries++; // We store the keys last so no one can see them in a partially inconsistent @@ -1818,11 +1915,7 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, // the case where we're inserting from the non-owner thread. We don't use // set_keys here because the transition from empty to non-empty is safe // as the empty keys will never be freed. -#ifdef Py_GIL_DISABLED - _Py_atomic_store_ptr_release(&mp->ma_keys, newkeys); -#else - mp->ma_keys = newkeys; -#endif + FT_ATOMIC_STORE_PTR_RELEASE(mp->ma_keys, newkeys); return 0; } @@ -1860,6 +1953,16 @@ build_indices_unicode(PyDictKeysObject *keys, PyDictUnicodeEntry *ep, Py_ssize_t } } +static void +invalidate_and_clear_inline_values(PyDictValues *values) +{ + assert(values->embedded); + FT_ATOMIC_STORE_UINT8(values->valid, 0); + for (int i = 0; i < values->capacity; i++) { + FT_ATOMIC_STORE_PTR_RELEASE(values->values[i], NULL); + } +} + /* Restructure the table by allocating a new table and reinserting all items again. When entries have been deleted, the new table may @@ -1867,7 +1970,7 @@ actually be smaller than the old one. If a table is split (its keys and hashes are shared, its values are not), then the values are temporarily copied into the table, it is resized as a combined table, then the me_value slots in the old table are NULLed out. -After resizing a table is always combined. +After resizing, a table is always combined. This function supports: - Unicode split -> Unicode combined or Generic @@ -1948,7 +2051,14 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, set_keys(mp, newkeys); dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); set_values(mp, NULL); - free_values(oldvalues, IS_DICT_SHARED(mp)); + if (oldvalues->embedded) { + assert(oldvalues->embedded == 1); + assert(oldvalues->valid == 1); + invalidate_and_clear_inline_values(oldvalues); + } + else { + free_values(oldvalues, IS_DICT_SHARED(mp)); + } } else { // oldkeys is combined. if (oldkeys->dk_kind == DICT_KEYS_GENERAL) { @@ -2005,7 +2115,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, if (oldkeys != Py_EMPTY_KEYS) { #ifdef Py_REF_DEBUG - _Py_DecRefTotal(_PyInterpreterState_GET()); + _Py_DecRefTotal(_PyThreadState_GET()); #endif assert(oldkeys->dk_kind != DICT_KEYS_SPLIT); assert(oldkeys->dk_refcnt == 1); @@ -2111,13 +2221,10 @@ dict_getitem(PyObject *op, PyObject *key, const char *warnmsg) } PyDictObject *mp = (PyDictObject *)op; - Py_hash_t hash; - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) { - PyErr_FormatUnraisable(warnmsg); - return NULL; - } + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + PyErr_FormatUnraisable(warnmsg); + return NULL; } PyThreadState *tstate = _PyThreadState_GET(); @@ -2166,12 +2273,9 @@ _PyDict_LookupIndex(PyDictObject *mp, PyObject *key) assert(PyDict_CheckExact((PyObject*)mp)); assert(PyUnicode_CheckExact(key)); - Py_hash_t hash = unicode_get_hash(key); + Py_hash_t hash = _PyObject_HashFast(key); if (hash == -1) { - hash = PyObject_Hash(key); - if (hash == -1) { - return -1; - } + return -1; } return _Py_dict_lookup(mp, key, hash, &value); @@ -2208,15 +2312,36 @@ _PyDict_GetItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) * exception occurred. */ int -_PyDict_GetItemRef_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash, PyObject **result) +_PyDict_GetItemRef_KnownHash_LockHeld(PyDictObject *op, PyObject *key, + Py_hash_t hash, PyObject **result) { - PyDictObject*mp = (PyDictObject *)op; + PyObject *value; + Py_ssize_t ix = _Py_dict_lookup(op, key, hash, &value); + assert(ix >= 0 || value == NULL); + if (ix == DKIX_ERROR) { + *result = NULL; + return -1; + } + if (value == NULL) { + *result = NULL; + return 0; // missing key + } + *result = Py_NewRef(value); + return 1; // key is present +} +/* Gets an item and provides a new reference if the value is present. + * Returns 1 if the key is present, 0 if the key is missing, and -1 if an + * exception occurred. +*/ +int +_PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result) +{ PyObject *value; #ifdef Py_GIL_DISABLED - Py_ssize_t ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); + Py_ssize_t ix = _Py_dict_lookup_threadsafe(op, key, hash, &value); #else - Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); + Py_ssize_t ix = _Py_dict_lookup(op, key, hash, &value); #endif assert(ix >= 0 || value == NULL); if (ix == DKIX_ERROR) { @@ -2244,19 +2369,41 @@ PyDict_GetItemRef(PyObject *op, PyObject *key, PyObject **result) return -1; } - Py_hash_t hash; - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) - { - hash = PyObject_Hash(key); - if (hash == -1) { - *result = NULL; - return -1; - } + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + *result = NULL; + return -1; } - return _PyDict_GetItemRef_KnownHash(op, key, hash, result); + return _PyDict_GetItemRef_KnownHash((PyDictObject *)op, key, hash, result); } +int +_PyDict_GetItemRef_Unicode_LockHeld(PyDictObject *op, PyObject *key, PyObject **result) +{ + ASSERT_DICT_LOCKED(op); + assert(PyUnicode_CheckExact(key)); + + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + *result = NULL; + return -1; + } + + PyObject *value; + Py_ssize_t ix = _Py_dict_lookup(op, key, hash, &value); + assert(ix >= 0 || value == NULL); + if (ix == DKIX_ERROR) { + *result = NULL; + return -1; + } + if (value == NULL) { + *result = NULL; + return 0; // missing key + } + *result = Py_NewRef(value); + return 1; // key is present +} /* Variant of PyDict_GetItem() that doesn't suppress exceptions. This returns NULL *with* an exception set if an exception occurred. @@ -2274,12 +2421,9 @@ PyDict_GetItemWithError(PyObject *op, PyObject *key) PyErr_BadInternalCall(); return NULL; } - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) - { - hash = PyObject_Hash(key); - if (hash == -1) { - return NULL; - } + hash = _PyObject_HashFast(key); + if (hash == -1) { + return NULL; } #ifdef Py_GIL_DISABLED @@ -2296,7 +2440,7 @@ PyObject * _PyDict_GetItemWithError(PyObject *dp, PyObject *kv) { assert(PyUnicode_CheckExact(kv)); - Py_hash_t hash = kv->ob_type->tp_hash(kv); + Py_hash_t hash = Py_TYPE(kv)->tp_hash(kv); if (hash == -1) { return NULL; } @@ -2337,34 +2481,94 @@ _PyDict_GetItemStringWithError(PyObject *v, const char *key) * Raise an exception and return NULL if an error occurred (ex: computing the * key hash failed, key comparison failed, ...). Return NULL if the key doesn't * exist. Return the value if the key exists. + * + * Returns a new reference. */ PyObject * _PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) { - // TODO: Thread safety Py_ssize_t ix; Py_hash_t hash; PyObject *value; - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return NULL; + hash = _PyObject_HashFast(key); + if (hash == -1) { + return NULL; } /* namespace 1: globals */ - ix = _Py_dict_lookup(globals, key, hash, &value); + ix = _Py_dict_lookup_threadsafe(globals, key, hash, &value); if (ix == DKIX_ERROR) return NULL; if (ix != DKIX_EMPTY && value != NULL) return value; /* namespace 2: builtins */ - ix = _Py_dict_lookup(builtins, key, hash, &value); + ix = _Py_dict_lookup_threadsafe(builtins, key, hash, &value); assert(ix >= 0 || value == NULL); return value; } +void +_PyDict_LoadGlobalStackRef(PyDictObject *globals, PyDictObject *builtins, PyObject *key, _PyStackRef *res) +{ + Py_ssize_t ix; + Py_hash_t hash; + + hash = _PyObject_HashFast(key); + if (hash == -1) { + *res = PyStackRef_NULL; + return; + } + + /* namespace 1: globals */ + ix = _Py_dict_lookup_threadsafe_stackref(globals, key, hash, res); + if (ix == DKIX_ERROR) { + return; + } + if (ix != DKIX_EMPTY && !PyStackRef_IsNull(*res)) { + return; + } + + /* namespace 2: builtins */ + ix = _Py_dict_lookup_threadsafe_stackref(builtins, key, hash, res); + assert(ix >= 0 || PyStackRef_IsNull(*res)); +} + +PyObject * +_PyDict_LoadBuiltinsFromGlobals(PyObject *globals) +{ + if (!PyDict_Check(globals)) { + PyErr_BadInternalCall(); + return NULL; + } + + PyDictObject *mp = (PyDictObject *)globals; + PyObject *key = &_Py_ID(__builtins__); + Py_hash_t hash = unicode_get_hash(key); + + // Use the stackref variant to avoid reference count contention on the + // builtins module in the free threading build. It's important not to + // make any escaping calls between the lookup and the `PyStackRef_CLOSE()` + // because the `ref` is not visible to the GC. + _PyStackRef ref; + Py_ssize_t ix = _Py_dict_lookup_threadsafe_stackref(mp, key, hash, &ref); + if (ix == DKIX_ERROR) { + return NULL; + } + if (PyStackRef_IsNull(ref)) { + return Py_NewRef(PyEval_GetBuiltins()); + } + PyObject *builtins = PyStackRef_AsPyObjectBorrow(ref); + if (PyModule_Check(builtins)) { + builtins = _PyModule_GetDict(builtins); + assert(builtins != NULL); + } + _Py_INCREF_BUILTINS(builtins); + PyStackRef_CLOSE(ref); + return builtins; +} + /* Consumes references to key and value */ static int setitem_take2_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) @@ -2374,14 +2578,11 @@ setitem_take2_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) assert(key); assert(value); assert(PyDict_Check(mp)); - Py_hash_t hash; - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) { - Py_DECREF(key); - Py_DECREF(value); - return -1; - } + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + Py_DECREF(key); + Py_DECREF(value); + return -1; } PyInterpreterState *interp = _PyInterpreterState_GET(); @@ -2433,11 +2634,21 @@ setitem_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) int -_PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, - Py_hash_t hash) +_PyDict_SetItem_KnownHash_LockHeld(PyDictObject *mp, PyObject *key, PyObject *value, + Py_hash_t hash) { - PyDictObject *mp; + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (mp->ma_keys == Py_EMPTY_KEYS) { + return insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); + } + /* insertdict() handles any resizing that might be necessary */ + return insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); +} +int +_PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, + Py_hash_t hash) +{ if (!PyDict_Check(op)) { PyErr_BadInternalCall(); return -1; @@ -2445,21 +2656,10 @@ _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, assert(key); assert(value); assert(hash != -1); - mp = (PyDictObject *)op; int res; - PyInterpreterState *interp = _PyInterpreterState_GET(); - - Py_BEGIN_CRITICAL_SECTION(mp); - - if (mp->ma_keys == Py_EMPTY_KEYS) { - res = insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); - } - else { - /* insertdict() handles any resizing that might be necessary */ - res = insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); - } - + Py_BEGIN_CRITICAL_SECTION(op); + res = _PyDict_SetItem_KnownHash_LockHeld((PyDictObject *)op, key, value, hash); Py_END_CRITICAL_SECTION(); return res; } @@ -2467,22 +2667,24 @@ _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, static void delete_index_from_values(PyDictValues *values, Py_ssize_t ix) { - uint8_t *size_ptr = ((uint8_t *)values)-2; - int size = *size_ptr; + uint8_t *array = get_insertion_order_array(values); + int size = values->size; + assert(size <= values->capacity); int i; - for (i = 1; size_ptr[-i] != ix; i++) { - assert(i <= size); + for (i = 0; array[i] != ix; i++) { + assert(i < size); } - assert(i <= size); + assert(i < size); + size--; for (; i < size; i++) { - size_ptr[-i] = size_ptr[-i-1]; + array[i] = array[i+1]; } - *size_ptr = size -1; + values->size = size; } -static int +static void delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, - PyObject *old_value, uint64_t new_version) + PyObject *old_value) { PyObject *old_key; @@ -2491,18 +2693,17 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, Py_ssize_t hashpos = lookdict_index(mp->ma_keys, hash, ix); assert(hashpos >= 0); - mp->ma_used--; - mp->ma_version_tag = new_version; + STORE_USED(mp, mp->ma_used - 1); if (_PyDict_HasSplitTable(mp)) { assert(old_value == mp->ma_values->values[ix]); - mp->ma_values->values[ix] = NULL; + STORE_SPLIT_VALUE(mp, ix, NULL); assert(ix < SHARED_KEYS_MAX_SIZE); /* Update order */ delete_index_from_values(mp->ma_values, ix); ASSERT_CONSISTENT(mp); } else { - mp->ma_keys->dk_version = 0; + FT_ATOMIC_STORE_UINT32_RELAXED(mp->ma_keys->dk_version, 0); dictkeys_set_index(mp->ma_keys, hashpos, DKIX_DUMMY); if (DK_IS_UNICODE(mp->ma_keys)) { PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; @@ -2522,18 +2723,15 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, Py_DECREF(old_value); ASSERT_CONSISTENT(mp); - return 0; } int PyDict_DelItem(PyObject *op, PyObject *key) { - Py_hash_t hash; assert(key); - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return -1; + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + return -1; } return _PyDict_DelItem_KnownHash(op, key, hash); @@ -2565,9 +2763,9 @@ delitem_knownhash_lock_held(PyObject *op, PyObject *key, Py_hash_t hash) } PyInterpreterState *interp = _PyInterpreterState_GET(); - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_DELETED, mp, key, NULL); - return delitem_common(mp, hash, ix, old_value, new_version); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, mp, key, NULL); + delitem_common(mp, hash, ix, old_value); + return 0; } int @@ -2582,9 +2780,10 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) static int delitemif_lock_held(PyObject *op, PyObject *key, - int (*predicate)(PyObject *value)) + int (*predicate)(PyObject *value, void *arg), + void *arg) { - Py_ssize_t hashpos, ix; + Py_ssize_t ix; PyDictObject *mp; Py_hash_t hash; PyObject *old_value; @@ -2592,35 +2791,28 @@ delitemif_lock_held(PyObject *op, PyObject *key, ASSERT_DICT_LOCKED(op); - if (!PyDict_Check(op)) { - PyErr_BadInternalCall(); - return -1; - } assert(key); hash = PyObject_Hash(key); if (hash == -1) return -1; mp = (PyDictObject *)op; ix = _Py_dict_lookup(mp, key, hash, &old_value); - if (ix == DKIX_ERROR) + if (ix == DKIX_ERROR) { return -1; + } if (ix == DKIX_EMPTY || old_value == NULL) { - _PyErr_SetKeyError(key); - return -1; + return 0; } - res = predicate(old_value); + res = predicate(old_value, arg); if (res == -1) return -1; - hashpos = lookdict_index(mp->ma_keys, hash, ix); - assert(hashpos >= 0); - if (res > 0) { PyInterpreterState *interp = _PyInterpreterState_GET(); - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_DELETED, mp, key, NULL); - return delitem_common(mp, hashpos, ix, old_value, new_version); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, mp, key, NULL); + delitem_common(mp, hash, ix, old_value); + return 1; } else { return 0; } @@ -2632,11 +2824,13 @@ delitemif_lock_held(PyObject *op, PyObject *key, */ int _PyDict_DelItemIf(PyObject *op, PyObject *key, - int (*predicate)(PyObject *value)) + int (*predicate)(PyObject *value, void *arg), + void *arg) { + assert(PyDict_Check(op)); int res; Py_BEGIN_CRITICAL_SECTION(op); - res = delitemif_lock_held(op, key, predicate); + res = delitemif_lock_held(op, key, predicate, arg); Py_END_CRITICAL_SECTION(); return res; } @@ -2661,27 +2855,30 @@ clear_lock_held(PyObject *op) } /* Empty the dict... */ PyInterpreterState *interp = _PyInterpreterState_GET(); - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_CLEARED, mp, NULL, NULL); + _PyDict_NotifyEvent(interp, PyDict_EVENT_CLEARED, mp, NULL, NULL); // We don't inc ref empty keys because they're immortal ensure_shared_on_resize(mp); - - set_keys(mp, Py_EMPTY_KEYS); - set_values(mp, NULL); - mp->ma_used = 0; - mp->ma_version_tag = new_version; - /* ...then clear the keys and values */ - if (oldvalues != NULL) { - n = oldkeys->dk_nentries; - for (i = 0; i < n; i++) - Py_CLEAR(oldvalues->values[i]); - free_values(oldvalues, IS_DICT_SHARED(mp)); - dictkeys_decref(interp, oldkeys, false); - } - else { + STORE_USED(mp, 0); + if (oldvalues == NULL) { + set_keys(mp, Py_EMPTY_KEYS); assert(oldkeys->dk_refcnt == 1); dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); } + else { + n = oldkeys->dk_nentries; + for (i = 0; i < n; i++) { + Py_CLEAR(oldvalues->values[i]); + } + if (oldvalues->embedded) { + oldvalues->size = 0; + } + else { + set_values(mp, NULL); + set_keys(mp, Py_EMPTY_KEYS); + free_values(oldvalues, IS_DICT_SHARED(mp)); + dictkeys_decref(interp, oldkeys, false); + } + } ASSERT_CONSISTENT(mp); } @@ -2710,8 +2907,6 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, if (!PyDict_Check(op)) return 0; - ASSERT_DICT_LOCKED(op); - mp = (PyDictObject *)op; i = *ppos; if (_PyDict_HasSplitTable(mp)) { @@ -2784,11 +2979,7 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, int PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) { - int res; - Py_BEGIN_CRITICAL_SECTION(op); - res = _PyDict_Next(op, ppos, pkey, pvalue, NULL); - Py_END_CRITICAL_SECTION(); - return res; + return _PyDict_Next(op, ppos, pkey, pvalue, NULL); } @@ -2826,9 +3017,8 @@ _PyDict_Pop_KnownHash(PyDictObject *mp, PyObject *key, Py_hash_t hash, assert(old_value != NULL); PyInterpreterState *interp = _PyInterpreterState_GET(); - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_DELETED, mp, key, NULL); - delitem_common(mp, hash, ix, Py_NewRef(old_value), new_version); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, mp, key, NULL); + delitem_common(mp, hash, ix, Py_NewRef(old_value)); ASSERT_CONSISTENT(mp); if (result) { @@ -2861,15 +3051,12 @@ pop_lock_held(PyObject *op, PyObject *key, PyObject **result) return 0; } - Py_hash_t hash; - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) { - if (result) { - *result = NULL; - } - return -1; + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + if (result) { + *result = NULL; } + return -1; } return _PyDict_Pop_KnownHash(dict, key, hash, result); } @@ -2903,8 +3090,8 @@ PyDict_PopString(PyObject *op, const char *key, PyObject **result) } -PyObject * -_PyDict_Pop(PyObject *dict, PyObject *key, PyObject *default_value) +static PyObject * +dict_pop_default(PyObject *dict, PyObject *key, PyObject *default_value) { PyObject *result; if (PyDict_Pop(dict, key, &result) == 0) { @@ -2917,6 +3104,12 @@ _PyDict_Pop(PyObject *dict, PyObject *key, PyObject *default_value) return result; } +PyObject * +_PyDict_Pop(PyObject *dict, PyObject *key, PyObject *default_value) +{ + return dict_pop_default(dict, key, default_value); +} + static PyDictObject * dict_dict_fromkeys(PyInterpreterState *interp, PyDictObject *mp, PyObject *iterable, PyObject *value) @@ -2958,8 +3151,9 @@ dict_set_fromkeys(PyInterpreterState *interp, PyDictObject *mp, return NULL; } - while (_PySet_NextEntry(iterable, &pos, &key, &hash)) { - if (insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value))) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(iterable); + while (_PySet_NextEntryRef(iterable, &pos, &key, &hash)) { + if (insertdict(interp, mp, key, hash, Py_NewRef(value))) { Py_DECREF(mp); return NULL; } @@ -3017,7 +3211,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) goto dict_iter_exit; } } -dict_iter_exit: +dict_iter_exit:; Py_END_CRITICAL_SECTION(); } else { while ((key = PyIter_Next(it)) != NULL) { @@ -3046,14 +3240,11 @@ dict_dealloc(PyObject *self) { PyDictObject *mp = (PyDictObject *)self; PyInterpreterState *interp = _PyInterpreterState_GET(); - assert(Py_REFCNT(mp) == 0); - Py_SET_REFCNT(mp, 1); + _PyObject_ResurrectStart(self); _PyDict_NotifyEvent(interp, PyDict_EVENT_DEALLOCATED, mp, NULL, NULL); - if (Py_REFCNT(mp) > 1) { - Py_SET_REFCNT(mp, Py_REFCNT(mp) - 1); + if (_PyObject_ResurrectEnd(self)) { return; } - Py_SET_REFCNT(mp, 0); PyDictValues *values = mp->ma_values; PyDictKeysObject *keys = mp->ma_keys; Py_ssize_t i, n; @@ -3062,26 +3253,22 @@ dict_dealloc(PyObject *self) PyObject_GC_UnTrack(mp); Py_TRASHCAN_BEGIN(mp, dict_dealloc) if (values != NULL) { - for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) { - Py_XDECREF(values->values[i]); + if (values->embedded == 0) { + for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) { + Py_XDECREF(values->values[i]); + } + free_values(values, false); } - free_values(values, false); dictkeys_decref(interp, keys, false); } else if (keys != NULL) { assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS); dictkeys_decref(interp, keys, false); } -#ifdef WITH_FREELISTS - struct _Py_dict_freelist *freelist = get_dict_freelist(); - if (freelist->numfree < PyDict_MAXFREELIST && freelist->numfree >=0 && - Py_IS_TYPE(mp, &PyDict_Type)) { - freelist->items[freelist->numfree++] = mp; - OBJECT_STAT_INC(to_freelist); + if (Py_IS_TYPE(mp, &PyDict_Type)) { + _Py_FREELIST_FREE(dicts, mp, Py_TYPE(mp)->tp_free); } - else -#endif - { + else { Py_TYPE(mp)->tp_free((PyObject *)mp); } Py_TRASHCAN_END @@ -3092,14 +3279,12 @@ static PyObject * dict_repr_lock_held(PyObject *self) { PyDictObject *mp = (PyDictObject *)self; - Py_ssize_t i; PyObject *key = NULL, *value = NULL; - _PyUnicodeWriter writer; - int first; + ASSERT_DICT_LOCKED(mp); - i = Py_ReprEnter((PyObject *)mp); - if (i != 0) { - return i > 0 ? PyUnicode_FromString("{...}") : NULL; + int res = Py_ReprEnter((PyObject *)mp); + if (res != 0) { + return (res > 0 ? PyUnicode_FromString("{...}") : NULL); } if (mp->ma_used == 0) { @@ -3107,66 +3292,70 @@ dict_repr_lock_held(PyObject *self) return PyUnicode_FromString("{}"); } - _PyUnicodeWriter_Init(&writer); - writer.overallocate = 1; - /* "{" + "1: 2" + ", 3: 4" * (len - 1) + "}" */ - writer.min_length = 1 + 4 + (2 + 4) * (mp->ma_used - 1) + 1; + // "{" + "1: 2" + ", 3: 4" * (len - 1) + "}" + Py_ssize_t prealloc = 1 + 4 + 6 * (mp->ma_used - 1) + 1; + PyUnicodeWriter *writer = PyUnicodeWriter_Create(prealloc); + if (writer == NULL) { + goto error; + } - if (_PyUnicodeWriter_WriteChar(&writer, '{') < 0) + if (PyUnicodeWriter_WriteChar(writer, '{') < 0) { goto error; + } /* Do repr() on each key+value pair, and insert ": " between them. Note that repr may mutate the dict. */ - i = 0; - first = 1; + Py_ssize_t i = 0; + int first = 1; while (_PyDict_Next((PyObject *)mp, &i, &key, &value, NULL)) { - PyObject *s; - int res; - - /* Prevent repr from deleting key or value during key format. */ + // Prevent repr from deleting key or value during key format. Py_INCREF(key); Py_INCREF(value); if (!first) { - if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) + // Write ", " + if (PyUnicodeWriter_WriteChar(writer, ',') < 0) { + goto error; + } + if (PyUnicodeWriter_WriteChar(writer, ' ') < 0) { goto error; + } } first = 0; - s = PyObject_Repr(key); - if (s == NULL) - goto error; - res = _PyUnicodeWriter_WriteStr(&writer, s); - Py_DECREF(s); - if (res < 0) + // Write repr(key) + if (PyUnicodeWriter_WriteRepr(writer, key) < 0) { goto error; + } - if (_PyUnicodeWriter_WriteASCIIString(&writer, ": ", 2) < 0) + // Write ": " + if (PyUnicodeWriter_WriteChar(writer, ':') < 0) { goto error; - - s = PyObject_Repr(value); - if (s == NULL) + } + if (PyUnicodeWriter_WriteChar(writer, ' ') < 0) { goto error; - res = _PyUnicodeWriter_WriteStr(&writer, s); - Py_DECREF(s); - if (res < 0) + } + + // Write repr(value) + if (PyUnicodeWriter_WriteRepr(writer, value) < 0) { goto error; + } Py_CLEAR(key); Py_CLEAR(value); } - writer.overallocate = 0; - if (_PyUnicodeWriter_WriteChar(&writer, '}') < 0) + if (PyUnicodeWriter_WriteChar(writer, '}') < 0) { goto error; + } Py_ReprLeave((PyObject *)mp); - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); error: Py_ReprLeave((PyObject *)mp); - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); Py_XDECREF(key); Py_XDECREF(value); return NULL; @@ -3185,8 +3374,7 @@ dict_repr(PyObject *self) static Py_ssize_t dict_length(PyObject *self) { - PyDictObject *mp = (PyDictObject *)self; - return _Py_atomic_load_ssize_relaxed(&mp->ma_used); + return FT_ATOMIC_LOAD_SSIZE_RELAXED(((PyDictObject *)self)->ma_used); } static PyObject * @@ -3197,16 +3385,11 @@ dict_subscript(PyObject *self, PyObject *key) Py_hash_t hash; PyObject *value; - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return NULL; + hash = _PyObject_HashFast(key); + if (hash == -1) { + return NULL; } -#ifdef Py_GIL_DISABLED ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); -#else - ix = _Py_dict_lookup(mp, key, hash, &value); -#endif if (ix == DKIX_ERROR) return NULL; if (ix == DKIX_EMPTY || value == NULL) { @@ -3226,11 +3409,7 @@ dict_subscript(PyObject *self, PyObject *key) _PyErr_SetKeyError(key); return NULL; } -#ifdef Py_GIL_DISABLED return value; -#else - return Py_NewRef(value); -#endif } static int @@ -3586,6 +3765,9 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) static int dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *other, int override) { + ASSERT_DICT_LOCKED(mp); + ASSERT_DICT_LOCKED(other); + if (other == mp || other->ma_used == 0) /* a.update(a) or a.update({}); nothing to do */ return 0; @@ -3598,12 +3780,13 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe PyDictKeysObject *okeys = other->ma_keys; // If other is clean, combined, and just allocated, just clone it. - if (other->ma_values == NULL && - other->ma_used == okeys->dk_nentries && - (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || - USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) { - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL); + if (mp->ma_values == NULL && + other->ma_values == NULL && + other->ma_used == okeys->dk_nentries && + (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || + USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used) + ) { + _PyDict_NotifyEvent(interp, PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL); PyDictKeysObject *keys = clone_combined_dict_keys(other); if (keys == NULL) return -1; @@ -3611,13 +3794,7 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe ensure_shared_on_resize(mp); dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp)); mp->ma_keys = keys; - if (_PyDict_HasSplitTable(mp)) { - free_values(mp->ma_values, IS_DICT_SHARED(mp)); - mp->ma_values = NULL; - } - - mp->ma_used = other->ma_used; - mp->ma_version_tag = new_version; + STORE_USED(mp, other->ma_used); ASSERT_CONSISTENT(mp); if (_PyObject_GC_IS_TRACKED(other) && !_PyObject_GC_IS_TRACKED(mp)) { @@ -3819,6 +3996,27 @@ dict_copy_impl(PyDictObject *self) return PyDict_Copy((PyObject *)self); } +/* Copies the values, but does not change the reference + * counts of the objects in the array. + * Return NULL, but does *not* set an exception on failure */ +static PyDictValues * +copy_values(PyDictValues *values) +{ + PyDictValues *newvalues = new_values(values->capacity); + if (newvalues == NULL) { + return NULL; + } + newvalues->size = values->size; + uint8_t *values_order = get_insertion_order_array(values); + uint8_t *new_values_order = get_insertion_order_array(newvalues); + memcpy(new_values_order, values_order, values->capacity); + for (int i = 0; i < values->capacity; i++) { + newvalues->values[i] = values->values[i]; + } + assert(newvalues->embedded == 0); + return newvalues; +} + static PyObject * copy_lock_held(PyObject *o) { @@ -3836,28 +4034,24 @@ copy_lock_held(PyObject *o) if (_PyDict_HasSplitTable(mp)) { PyDictObject *split_copy; - size_t size = shared_keys_usable_size(mp->ma_keys); - PyDictValues *newvalues = new_values(size); - if (newvalues == NULL) + PyDictValues *newvalues = copy_values(mp->ma_values); + if (newvalues == NULL) { return PyErr_NoMemory(); + } split_copy = PyObject_GC_New(PyDictObject, &PyDict_Type); if (split_copy == NULL) { free_values(newvalues, false); return NULL; } - size_t prefix_size = ((uint8_t *)newvalues)[-1]; - memcpy(((char *)newvalues)-prefix_size, ((char *)mp->ma_values)-prefix_size, prefix_size-1); + for (size_t i = 0; i < newvalues->capacity; i++) { + Py_XINCREF(newvalues->values[i]); + } split_copy->ma_values = newvalues; split_copy->ma_keys = mp->ma_keys; split_copy->ma_used = mp->ma_used; - split_copy->ma_version_tag = DICT_NEXT_VERSION(interp); + split_copy->_ma_watcher_tag = 0; dictkeys_incref(mp->ma_keys); - for (size_t i = 0; i < size; i++) { - PyObject *value = mp->ma_values->values[i]; - split_copy->ma_values->values[i] = Py_XNewRef(value); - } - if (_PyObject_GC_IS_TRACKED(mp)) - _PyObject_GC_TRACK(split_copy); + _PyObject_GC_TRACK(split_copy); return (PyObject *)split_copy; } @@ -3892,11 +4086,6 @@ copy_lock_held(PyObject *o) new->ma_used = mp->ma_used; ASSERT_CONSISTENT(new); - if (_PyObject_GC_IS_TRACKED(mp)) { - /* Maintain tracking. */ - _PyObject_GC_TRACK(new); - } - return (PyObject *)new; } @@ -3933,7 +4122,7 @@ PyDict_Size(PyObject *mp) PyErr_BadInternalCall(); return -1; } - return ((PyDictObject *)mp)->ma_used; + return FT_ATOMIC_LOAD_SSIZE_RELAXED(((PyDictObject *)mp)->ma_used); } /* Return 1 if dicts equal, 0 if not, -1 if error. @@ -3952,7 +4141,7 @@ dict_equal_lock_held(PyDictObject *a, PyDictObject *b) /* can't be equal if # of entries differ */ return 0; /* Same # of entries -- check all of 'em. Exit early on any diff. */ - for (i = 0; i < LOAD_KEYS_NENTIRES(a->ma_keys); i++) { + for (i = 0; i < LOAD_KEYS_NENTRIES(a->ma_keys); i++) { PyObject *key, *aval; Py_hash_t hash; if (DK_IS_UNICODE(a->ma_keys)) { @@ -4059,7 +4248,6 @@ dict___contains__(PyDictObject *self, PyObject *key) } /*[clinic input] -@critical_section dict.get key: object @@ -4071,35 +4259,23 @@ Return the value for key if key is in the dictionary, else default. static PyObject * dict_get_impl(PyDictObject *self, PyObject *key, PyObject *default_value) -/*[clinic end generated code: output=bba707729dee05bf input=a631d3f18f584c60]*/ +/*[clinic end generated code: output=bba707729dee05bf input=279ddb5790b6b107]*/ { PyObject *val = NULL; Py_hash_t hash; Py_ssize_t ix; - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return NULL; + hash = _PyObject_HashFast(key); + if (hash == -1) { + return NULL; } -#ifdef Py_GIL_DISABLED ix = _Py_dict_lookup_threadsafe(self, key, hash, &val); -#else - ix = _Py_dict_lookup(self, key, hash, &val); -#endif if (ix == DKIX_ERROR) return NULL; -#ifdef Py_GIL_DISABLED if (ix == DKIX_EMPTY || val == NULL) { val = Py_NewRef(default_value); } return val; -#else - if (ix == DKIX_EMPTY || val == NULL) { - val = default_value; - } - return Py_NewRef(val); -#endif } static int @@ -4121,14 +4297,12 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu return -1; } - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) { - if (result) { - *result = NULL; - } - return -1; + hash = _PyObject_HashFast(key); + if (hash == -1) { + if (result) { + *result = NULL; } + return -1; } if (mp->ma_keys == Py_EMPTY_KEYS) { @@ -4154,6 +4328,29 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu } } + if (_PyDict_HasSplitTable(mp)) { + Py_ssize_t ix = insert_split_key(mp->ma_keys, key, hash); + if (ix != DKIX_EMPTY) { + PyObject *value = mp->ma_values->values[ix]; + int already_present = value != NULL; + if (!already_present) { + insert_split_value(interp, mp, key, default_value, ix); + value = default_value; + } + if (result) { + *result = incref_result ? Py_NewRef(value) : value; + } + return already_present; + } + + /* No space in shared keys. Resize and continue below. */ + if (insertion_resize(interp, mp, 1) < 0) { + goto error; + } + } + + assert(!_PyDict_HasSplitTable(mp)); + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); if (ix == DKIX_ERROR) { if (result) { @@ -4163,35 +4360,18 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu } if (ix == DKIX_EMPTY) { - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, default_value); - mp->ma_keys->dk_version = 0; + assert(!_PyDict_HasSplitTable(mp)); value = default_value; - if (!_PyDict_HasSplitTable(mp)) { - if (insert_combined_dict(interp, mp, hash, Py_NewRef(key), Py_NewRef(value)) < 0) { - Py_DECREF(key); - Py_DECREF(value); - if (result) { - *result = NULL; - } - return -1; - } - } - else { - if (insert_split_dict(interp, mp, hash, Py_NewRef(key), Py_NewRef(value)) < 0) { - Py_DECREF(key); - Py_DECREF(value); - if (result) { - *result = NULL; - } - return -1; + if (insert_combined_dict(interp, mp, hash, Py_NewRef(key), Py_NewRef(value)) < 0) { + Py_DECREF(key); + Py_DECREF(value); + if (result) { + *result = NULL; } } - MAINTAIN_TRACKING(mp, key, value); - mp->ma_used++; - mp->ma_version_tag = new_version; + STORE_USED(mp, mp->ma_used + 1); assert(mp->ma_keys->dk_usable >= 0); ASSERT_CONSISTENT(mp); if (result) { @@ -4199,29 +4379,19 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu } return 0; } - else if (value == NULL) { - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, default_value); - value = default_value; - assert(_PyDict_HasSplitTable(mp)); - assert(mp->ma_values->values[ix] == NULL); - MAINTAIN_TRACKING(mp, key, value); - mp->ma_values->values[ix] = Py_NewRef(value); - _PyDictValues_AddToInsertionOrder(mp->ma_values, ix); - mp->ma_used++; - mp->ma_version_tag = new_version; - ASSERT_CONSISTENT(mp); - if (result) { - *result = incref_result ? Py_NewRef(value) : value; - } - return 0; - } + assert(value != NULL); ASSERT_CONSISTENT(mp); if (result) { *result = incref_result ? Py_NewRef(value) : value; } return 1; + +error: + if (result) { + *result = NULL; + } + return -1; } int @@ -4300,7 +4470,7 @@ static PyObject * dict_pop_impl(PyDictObject *self, PyObject *key, PyObject *default_value) /*[clinic end generated code: output=3abb47b89f24c21c input=e221baa01044c44c]*/ { - return _PyDict_Pop((PyObject*)self, key, default_value); + return dict_pop_default((PyObject*)self, key, default_value); } /*[clinic input] @@ -4319,9 +4489,10 @@ dict_popitem_impl(PyDictObject *self) { Py_ssize_t i, j; PyObject *res; - uint64_t new_version; PyInterpreterState *interp = _PyInterpreterState_GET(); + ASSERT_DICT_LOCKED(self); + /* Allocate the result tuple before checking the size. Believe it * or not, this allocation could trigger a garbage collection which * could empty the dict, so if we checked the size first and that @@ -4346,7 +4517,7 @@ dict_popitem_impl(PyDictObject *self) return NULL; } } - self->ma_keys->dk_version = 0; + FT_ATOMIC_STORE_UINT32_RELAXED(self->ma_keys->dk_version, 0); /* Pop last item */ PyObject *key, *value; @@ -4360,8 +4531,7 @@ dict_popitem_impl(PyDictObject *self) assert(i >= 0); key = ep0[i].me_key; - new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_DELETED, self, key, NULL); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, self, key, NULL); hash = unicode_get_hash(key); value = ep0[i].me_value; ep0[i].me_key = NULL; @@ -4376,8 +4546,7 @@ dict_popitem_impl(PyDictObject *self) assert(i >= 0); key = ep0[i].me_key; - new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_DELETED, self, key, NULL); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DELETED, self, key, NULL); hash = ep0[i].me_hash; value = ep0[i].me_value; ep0[i].me_key = NULL; @@ -4395,7 +4564,6 @@ dict_popitem_impl(PyDictObject *self) /* We can't dk_usable++ since there is DKIX_DUMMY in indices */ STORE_KEYS_NENTRIES(self->ma_keys, i); STORE_USED(self, self->ma_used - 1); - self->ma_version_tag = new_version; ASSERT_CONSISTENT(self); return res; } @@ -4409,8 +4577,10 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) if (DK_IS_UNICODE(keys)) { if (_PyDict_HasSplitTable(mp)) { - for (i = 0; i < n; i++) { - Py_VISIT(mp->ma_values->values[i]); + if (!mp->ma_values->embedded) { + for (i = 0; i < n; i++) { + Py_VISIT(mp->ma_values->values[i]); + } } } else { @@ -4522,8 +4692,8 @@ PyDoc_STRVAR(getitem__doc__, "__getitem__($self, key, /)\n--\n\nReturn self[key]."); PyDoc_STRVAR(update__doc__, -"D.update([E, ]**F) -> None. Update D from dict/iterable E and F.\n\ -If E is present and has a .keys() method, then does: for k in E: D[k] = E[k]\n\ +"D.update([E, ]**F) -> None. Update D from mapping/iterable E and F.\n\ +If E is present and has a .keys() method, then does: for k in E.keys(): D[k] = E[k]\n\ If E is present and lacks a .keys() method, then does: for k, v in E: D[k] = v\n\ In either case, this is followed by: for k in F: D[k] = F[k]"); @@ -4555,12 +4725,10 @@ static PyMethodDef mapp_methods[] = { int PyDict_Contains(PyObject *op, PyObject *key) { - Py_hash_t hash; + Py_hash_t hash = _PyObject_HashFast(key); - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return -1; + if (hash == -1) { + return -1; } return _PyDict_Contains_KnownHash(op, key, hash); @@ -4646,22 +4814,13 @@ dict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyDictObject *d = (PyDictObject *)self; d->ma_used = 0; - d->ma_version_tag = DICT_NEXT_VERSION( - _PyInterpreterState_GET()); + d->_ma_watcher_tag = 0; dictkeys_incref(Py_EMPTY_KEYS); d->ma_keys = Py_EMPTY_KEYS; d->ma_values = NULL; ASSERT_CONSISTENT(d); - - if (type != &PyDict_Type) { - // Don't track if a subclass tp_alloc is PyType_GenericAlloc() - if (!_PyObject_GC_IS_TRACKED(d)) { - _PyObject_GC_TRACK(d); - } - } - else { - // _PyType_AllocNoTrack() does not track the created object - assert(!_PyObject_GC_IS_TRACKED(d)); + if (!_PyObject_GC_IS_TRACKED(d)) { + _PyObject_GC_TRACK(d); } return self; } @@ -4764,6 +4923,7 @@ PyTypeObject PyDict_Type = { dict_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ .tp_vectorcall = dict_vectorcall, + .tp_version_tag = _Py_TYPE_VERSION_DICT, }; /* For backward compatibility with old dictionary interface */ @@ -4817,7 +4977,8 @@ PyDict_SetItemString(PyObject *v, const char *key, PyObject *item) kv = PyUnicode_FromString(key); if (kv == NULL) return -1; - PyUnicode_InternInPlace(&kv); /* XXX Should we really? */ + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternImmortal(interp, &kv); /* XXX Should we really? */ err = PyDict_SetItem(v, kv, item); Py_DECREF(kv); return err; @@ -4859,19 +5020,21 @@ typedef struct { static PyObject * dictiter_new(PyDictObject *dict, PyTypeObject *itertype) { + Py_ssize_t used; dictiterobject *di; di = PyObject_GC_New(dictiterobject, itertype); if (di == NULL) { return NULL; } di->di_dict = (PyDictObject*)Py_NewRef(dict); - di->di_used = dict->ma_used; - di->len = dict->ma_used; + used = FT_ATOMIC_LOAD_SSIZE_RELAXED(dict->ma_used); + di->di_used = used; + di->len = used; if (itertype == &PyDictRevIterKey_Type || itertype == &PyDictRevIterItem_Type || itertype == &PyDictRevIterValue_Type) { if (_PyDict_HasSplitTable(dict)) { - di->di_pos = dict->ma_used - 1; + di->di_pos = used - 1; } else { di->di_pos = load_keys_nentries(dict) - 1; @@ -4920,8 +5083,8 @@ dictiter_len(PyObject *self, PyObject *Py_UNUSED(ignored)) { dictiterobject *di = (dictiterobject *)self; Py_ssize_t len = 0; - if (di->di_dict != NULL && di->di_used == di->di_dict->ma_used) - len = di->len; + if (di->di_dict != NULL && di->di_used == FT_ATOMIC_LOAD_SSIZE_RELAXED(di->di_dict->ma_used)) + len = FT_ATOMIC_LOAD_SSIZE_RELAXED(di->len); return PyLong_FromSize_t(len); } @@ -5204,6 +5367,7 @@ dictiter_iternextitem_lock_held(PyDictObject *d, PyObject *self, Py_ssize_t i; assert (PyDict_Check(d)); + ASSERT_DICT_LOCKED(d); if (di->di_used != d->ma_used) { PyErr_SetString(PyExc_RuntimeError, @@ -5273,7 +5437,7 @@ dictiter_iternextitem_lock_held(PyDictObject *d, PyObject *self, #ifdef Py_GIL_DISABLED // Grabs the key and/or value from the provided locations and if successful -// returns them with an increased reference count. If either one is unsucessful +// returns them with an increased reference count. If either one is unsuccessful // nothing is incref'd and returns -1. static int acquire_key_value(PyObject **key_loc, PyObject *value, PyObject **value_loc, @@ -5287,7 +5451,7 @@ acquire_key_value(PyObject **key_loc, PyObject *value, PyObject **value_loc, } if (out_value) { - if (!_Py_TryIncref(value_loc, value)) { + if (!_Py_TryIncrefCompare(value_loc, value)) { if (out_key) { Py_DECREF(*out_key); } @@ -5299,16 +5463,11 @@ acquire_key_value(PyObject **key_loc, PyObject *value, PyObject **value_loc, return 0; } -static Py_ssize_t -load_values_used_size(PyDictValues *values) -{ - return (Py_ssize_t)_Py_atomic_load_uint8(&DICT_VALUES_USED_SIZE(values)); -} - static int dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, PyObject **out_key, PyObject **out_value) { + int res; dictiterobject *di = (dictiterobject *)self; Py_ssize_t i; PyDictKeysObject *k; @@ -5333,7 +5492,7 @@ dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, goto concurrent_modification; } - Py_ssize_t used = load_values_used_size(values); + Py_ssize_t used = (Py_ssize_t)_Py_atomic_load_uint8(&values->size); if (i >= used) { goto fail; } @@ -5404,7 +5563,6 @@ dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, Py_DECREF(d); return -1; - int res; try_locked: Py_BEGIN_CRITICAL_SECTION(d); res = dictiter_iternextitem_lock_held(d, self, out_key, out_value); @@ -5724,7 +5882,7 @@ dictview_len(PyObject *self) _PyDictViewObject *dv = (_PyDictViewObject *)self; Py_ssize_t len = 0; if (dv->dv_dict != NULL) - len = dv->dv_dict->ma_used; + len = FT_ATOMIC_LOAD_SSIZE_RELAXED(dv->dv_dict->ma_used); return len; } @@ -6525,9 +6683,10 @@ dictvalues_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) /* Returns NULL if cannot allocate a new PyDictKeysObject, but does not set an error */ PyDictKeysObject * -_PyDict_NewKeysForClass(void) +_PyDict_NewKeysForClass(PyHeapTypeObject *cls) { PyInterpreterState *interp = _PyInterpreterState_GET(); + PyDictKeysObject *keys = new_keys_object( interp, NEXT_LOG2_SHARED_KEYS_MAX_SIZE, 1); if (keys == NULL) { @@ -6539,18 +6698,32 @@ _PyDict_NewKeysForClass(void) keys->dk_usable = SHARED_KEYS_MAX_SIZE; keys->dk_kind = DICT_KEYS_SPLIT; } + if (cls->ht_type.tp_dict) { + PyObject *attrs = PyDict_GetItem(cls->ht_type.tp_dict, &_Py_ID(__static_attributes__)); + if (attrs != NULL && PyTuple_Check(attrs)) { + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(attrs); i++) { + PyObject *key = PyTuple_GET_ITEM(attrs, i); + Py_hash_t hash; + if (PyUnicode_CheckExact(key) && (hash = unicode_get_hash(key)) != -1) { + if (insert_split_key(keys, key, hash) == DKIX_EMPTY) { + break; + } + } + } + } + } return keys; } -#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys) - -int +void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp) { assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictKeysObject *keys = CACHED_KEYS(tp); assert(keys != NULL); + OBJECT_STAT_INC(inline_values); #ifdef Py_GIL_DISABLED Py_ssize_t usable = _Py_atomic_load_ssize_relaxed(&keys->dk_usable); if (usable > 1) { @@ -6566,124 +6739,111 @@ _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp) } #endif size_t size = shared_keys_usable_size(keys); - PyDictValues *values = new_values(size); - if (values == NULL) { - PyErr_NoMemory(); - return -1; - } - assert(((uint8_t *)values)[-1] >= (size + 2)); - ((uint8_t *)values)[-2] = 0; + PyDictValues *values = _PyObject_InlineValues(obj); + assert(size < 256); + values->capacity = (uint8_t)size; + values->size = 0; + values->embedded = 1; + values->valid = 1; for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } - _PyDictOrValues_SetValues(_PyObject_DictOrValuesPointer(obj), values); - return 0; -} - -int -_PyObject_InitializeDict(PyObject *obj) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyTypeObject *tp = Py_TYPE(obj); - if (tp->tp_dictoffset == 0) { - return 0; - } - if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - OBJECT_STAT_INC(new_values); - return _PyObject_InitInlineValues(obj, tp); - } - PyObject *dict; - if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { - dictkeys_incref(CACHED_KEYS(tp)); - dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); - } - else { - dict = PyDict_New(); - } - if (dict == NULL) { - return -1; - } - PyObject **dictptr = _PyObject_ComputedDictPointer(obj); - *dictptr = dict; - return 0; + _PyObject_ManagedDictPointer(obj)->dict = NULL; } -static PyObject * +static PyDictObject * make_dict_from_instance_attributes(PyInterpreterState *interp, PyDictKeysObject *keys, PyDictValues *values) { dictkeys_incref(keys); Py_ssize_t used = 0; - Py_ssize_t track = 0; size_t size = shared_keys_usable_size(keys); for (size_t i = 0; i < size; i++) { PyObject *val = values->values[i]; if (val != NULL) { used += 1; - track += _PyObject_GC_MAY_BE_TRACKED(val); } } - PyObject *res = new_dict(interp, keys, values, used, 0); - if (track && res) { - _PyObject_GC_TRACK(res); - } + PyDictObject *res = (PyDictObject *)new_dict(interp, keys, values, used, 0); return res; } -PyObject * -_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values) +PyDictObject * +_PyObject_MaterializeManagedDict_LockHeld(PyObject *obj) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); + ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(obj); + OBJECT_STAT_INC(dict_materialized_on_request); - return make_dict_from_instance_attributes(interp, keys, values); + + PyDictValues *values = _PyObject_InlineValues(obj); + PyDictObject *dict; + if (values->valid) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); + dict = make_dict_from_instance_attributes(interp, keys, values); + } + else { + dict = (PyDictObject *)PyDict_New(); + } + FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict, + dict); + return dict; } -// Return true if the dict was dematerialized, false otherwise. -bool -_PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv) +PyDictObject * +_PyObject_MaterializeManagedDict(PyObject *obj) { - assert(_PyObject_DictOrValuesPointer(obj) == dorv); - assert(!_PyDictOrValues_IsValues(*dorv)); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv); - if (dict == NULL) { - return false; - } - // It's likely that this dict still shares its keys (if it was materialized - // on request and not heavily modified): - if (!PyDict_CheckExact(dict)) { - return false; + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict != NULL) { + return dict; } - assert(_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_HEAPTYPE)); - if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) || - !has_unique_reference((PyObject *)dict)) - { - return false; + + Py_BEGIN_CRITICAL_SECTION(obj); + +#ifdef Py_GIL_DISABLED + dict = _PyObject_GetManagedDict(obj); + if (dict != NULL) { + // We raced with another thread creating the dict + goto exit; } - ensure_shared_on_resize(dict); - - assert(dict->ma_values); - // We have an opportunity to do something *really* cool: dematerialize it! - _PyDictKeys_DecRef(dict->ma_keys); - _PyDictOrValues_SetValues(dorv, dict->ma_values); - OBJECT_STAT_INC(dict_dematerialized); - // Don't try this at home, kids: - dict->ma_keys = NULL; - dict->ma_values = NULL; - Py_DECREF(dict); - return true; +#endif + dict = _PyObject_MaterializeManagedDict_LockHeld(obj); + +#ifdef Py_GIL_DISABLED +exit: +#endif + Py_END_CRITICAL_SECTION(); + return dict; } int -_PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, +_PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value) +{ + if (value == NULL) { + Py_hash_t hash = _PyObject_HashFast(name); + if (hash == -1) { + return -1; + } + return delitem_knownhash_lock_held((PyObject *)dict, name, hash); + } else { + return setitem_lock_held(dict, name, value); + } +} + +// Called with either the object's lock or the dict's lock held +// depending on whether or not a dict has been materialized for +// the object. +static int +store_instance_attr_lock_held(PyObject *obj, PyDictValues *values, PyObject *name, PyObject *value) { - PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); assert(keys != NULL); assert(values != NULL); - assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); Py_ssize_t ix = DKIX_EMPTY; + PyDictObject *dict = _PyObject_GetManagedDict(obj); + assert(dict == NULL || ((PyDictObject *)dict)->ma_values == values); if (PyUnicode_CheckExact(name)) { Py_hash_t hash = unicode_get_hash(name); if (hash == -1) { @@ -6691,18 +6851,7 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, assert(hash != -1); } -#ifdef Py_GIL_DISABLED - // Try a thread-safe lookup to see if the index is already allocated - ix = unicodekeys_lookup_unicode_threadsafe(keys, name, hash); - if (ix == DKIX_EMPTY) { - // Lock keys and do insert - LOCK_KEYS(keys); - ix = insert_into_splitdictkeys(keys, name, hash); - UNLOCK_KEYS(keys); - } -#else - ix = insert_into_splitdictkeys(keys, name, hash); -#endif + ix = insert_split_key(keys, name, hash); #ifdef Py_STATS if (ix == DKIX_EMPTY) { @@ -6720,84 +6869,244 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, } #endif } + if (ix == DKIX_EMPTY) { - PyObject *dict = make_dict_from_instance_attributes( - interp, keys, values); + int res; if (dict == NULL) { - return -1; - } - _PyObject_DictOrValuesPointer(obj)->dict = dict; - if (value == NULL) { - return PyDict_DelItem(dict, name); - } - else { - return PyDict_SetItem(dict, name, value); + // Make the dict but don't publish it in the object + // so that no one else will see it. + dict = make_dict_from_instance_attributes(PyInterpreterState_Get(), keys, values); + if (dict == NULL || + _PyDict_SetItem_LockHeld(dict, name, value) < 0) { + Py_XDECREF(dict); + return -1; + } + + FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict, + (PyDictObject *)dict); + return 0; } + + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(dict); + + res = _PyDict_SetItem_LockHeld(dict, name, value); + return res; } + PyObject *old_value = values->values[ix]; - values->values[ix] = Py_XNewRef(value); + if (old_value == NULL && value == NULL) { + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute '%U'", + Py_TYPE(obj)->tp_name, name); + return -1; + } + + if (dict) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyDict_WatchEvent event = (old_value == NULL ? PyDict_EVENT_ADDED : + value == NULL ? PyDict_EVENT_DELETED : + PyDict_EVENT_MODIFIED); + _PyDict_NotifyEvent(interp, event, dict, name, value); + } + + FT_ATOMIC_STORE_PTR_RELEASE(values->values[ix], Py_XNewRef(value)); + if (old_value == NULL) { - if (value == NULL) { - PyErr_Format(PyExc_AttributeError, - "'%.100s' object has no attribute '%U'", - Py_TYPE(obj)->tp_name, name); - return -1; - } _PyDictValues_AddToInsertionOrder(values, ix); + if (dict) { + assert(dict->ma_values == values); + STORE_USED(dict, dict->ma_used + 1); + } } else { if (value == NULL) { delete_index_from_values(values, ix); + if (dict) { + assert(dict->ma_values == values); + STORE_USED(dict, dict->ma_used - 1); + } } Py_DECREF(old_value); } return 0; } -/* Sanity check for managed dicts */ -#if 0 -#define CHECK(val) assert(val); if (!(val)) { return 0; } - -int -_PyObject_ManagedDictValidityCheck(PyObject *obj) +static inline int +store_instance_attr_dict(PyObject *obj, PyDictObject *dict, PyObject *name, PyObject *value) { - PyTypeObject *tp = Py_TYPE(obj); - CHECK(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - int size = ((uint8_t *)values)[-2]; - int count = 0; - PyDictKeysObject *keys = CACHED_KEYS(tp); - for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - if (values->values[i] != NULL) { - count++; - } - } - CHECK(size == count); + PyDictValues *values = _PyObject_InlineValues(obj); + int res; + Py_BEGIN_CRITICAL_SECTION(dict); + if (dict->ma_values == values) { + res = store_instance_attr_lock_held(obj, values, name, value); } else { - if (dorv_ptr->dict != NULL) { - CHECK(PyDict_Check(dorv_ptr->dict)); + res = _PyDict_SetItem_LockHeld(dict, name, value); + } + Py_END_CRITICAL_SECTION(); + return res; +} + +int +_PyObject_StoreInstanceAttribute(PyObject *obj, PyObject *name, PyObject *value) +{ + PyDictValues *values = _PyObject_InlineValues(obj); + if (!FT_ATOMIC_LOAD_UINT8(values->valid)) { + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + dict = (PyDictObject *)PyObject_GenericGetDict(obj, NULL); + if (dict == NULL) { + return -1; + } + int res = store_instance_attr_dict(obj, dict, name, value); + Py_DECREF(dict); + return res; + } + return store_instance_attr_dict(obj, dict, name, value); + } + +#ifdef Py_GIL_DISABLED + // We have a valid inline values, at least for now... There are two potential + // races with having the values become invalid. One is the dictionary + // being detached from the object. The other is if someone is inserting + // into the dictionary directly and therefore causing it to resize. + // + // If we haven't materialized the dictionary yet we lock on the object, which + // will also be used to prevent the dictionary from being materialized while + // we're doing the insertion. If we race and the dictionary gets created + // then we'll need to release the object lock and lock the dictionary to + // prevent resizing. + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + int res; + Py_BEGIN_CRITICAL_SECTION(obj); + dict = _PyObject_GetManagedDict(obj); + + if (dict == NULL) { + res = store_instance_attr_lock_held(obj, values, name, value); + } + Py_END_CRITICAL_SECTION(); + + if (dict == NULL) { + return res; + } + } + return store_instance_attr_dict(obj, dict, name, value); +#else + return store_instance_attr_lock_held(obj, values, name, value); +#endif +} + +/* Sanity check for managed dicts */ +#if 0 +#define CHECK(val) assert(val); if (!(val)) { return 0; } + +int +_PyObject_ManagedDictValidityCheck(PyObject *obj) +{ + PyTypeObject *tp = Py_TYPE(obj); + CHECK(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj); + if (_PyManagedDictPointer_IsValues(*managed_dict)) { + PyDictValues *values = _PyManagedDictPointer_GetValues(*managed_dict); + int size = ((uint8_t *)values)[-2]; + int count = 0; + PyDictKeysObject *keys = CACHED_KEYS(tp); + for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { + if (values->values[i] != NULL) { + count++; + } + } + CHECK(size == count); + } + else { + if (managed_dict->dict != NULL) { + CHECK(PyDict_Check(managed_dict->dict)); } } return 1; } #endif -PyObject * -_PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, - PyObject *name) +// Attempts to get an instance attribute from the inline values. Returns true +// if successful, or false if the caller needs to lookup in the dictionary. +bool +_PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr) { assert(PyUnicode_CheckExact(name)); + PyDictValues *values = _PyObject_InlineValues(obj); + if (!FT_ATOMIC_LOAD_UINT8(values->valid)) { + return false; + } + PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); assert(keys != NULL); - Py_ssize_t ix = _PyDictKeys_StringLookup(keys, name); + Py_ssize_t ix = _PyDictKeys_StringLookupSplit(keys, name); if (ix == DKIX_EMPTY) { - return NULL; + *attr = NULL; + return true; } + +#ifdef Py_GIL_DISABLED + PyObject *value = _Py_atomic_load_ptr_acquire(&values->values[ix]); + if (value == NULL) { + if (FT_ATOMIC_LOAD_UINT8(values->valid)) { + *attr = NULL; + return true; + } + } + else if (_Py_TryIncrefCompare(&values->values[ix], value)) { + *attr = value; + return true; + } + + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + // No dict, lock the object to prevent one from being + // materialized... + bool success = false; + Py_BEGIN_CRITICAL_SECTION(obj); + + dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + // Still no dict, we can read from the values + assert(values->valid); + value = values->values[ix]; + *attr = _Py_XNewRefWithLock(value); + success = true; + } + + Py_END_CRITICAL_SECTION(); + + if (success) { + return true; + } + } + + // We have a dictionary, we'll need to lock it to prevent + // the values from being resized. + assert(dict != NULL); + + bool success; + Py_BEGIN_CRITICAL_SECTION(dict); + + if (dict->ma_values == values && FT_ATOMIC_LOAD_UINT8(values->valid)) { + value = _Py_atomic_load_ptr_relaxed(&values->values[ix]); + *attr = _Py_XNewRefWithLock(value); + success = true; + } else { + // Caller needs to lookup from the dictionary + success = false; + } + + Py_END_CRITICAL_SECTION(); + + return success; +#else PyObject *value = values->values[ix]; - return Py_XNewRef(value); + *attr = Py_XNewRef(value); + return true; +#endif } int @@ -6807,45 +7116,31 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj) if (tp->tp_dictoffset == 0) { return 1; } - PyObject *dict; - if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(dorv)) { + PyDictObject *dict; + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictValues *values = _PyObject_InlineValues(obj); + if (FT_ATOMIC_LOAD_UINT8(values->valid)) { PyDictKeysObject *keys = CACHED_KEYS(tp); for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - if (_PyDictOrValues_GetValues(dorv)->values[i] != NULL) { + if (FT_ATOMIC_LOAD_PTR_RELAXED(values->values[i]) != NULL) { return 0; } } return 1; } - dict = _PyDictOrValues_GetDict(dorv); + dict = _PyObject_GetManagedDict(obj); + } + else if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + dict = _PyObject_GetManagedDict(obj); } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); - dict = *dictptr; + dict = (PyDictObject *)*dictptr; } if (dict == NULL) { return 1; } - return ((PyDictObject *)dict)->ma_used == 0; -} - -void -_PyObject_FreeInstanceAttributes(PyObject *self) -{ - PyTypeObject *tp = Py_TYPE(self); - assert(Py_TYPE(self)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); - if (!_PyDictOrValues_IsValues(dorv)) { - return; - } - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyDictKeysObject *keys = CACHED_KEYS(tp); - for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - Py_XDECREF(values->values[i]); - } - free_values(values, IS_DICT_SHARED((PyDictObject*)self)); + return FT_ATOMIC_LOAD_SSIZE_RELAXED(((PyDictObject *)dict)->ma_used) == 0; } int @@ -6855,74 +7150,327 @@ PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { return 0; } - assert(tp->tp_dictoffset); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(dorv)) { - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyDictKeysObject *keys = CACHED_KEYS(tp); - for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - Py_VISIT(values->values[i]); + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictValues *values = _PyObject_InlineValues(obj); + if (values->valid) { + for (Py_ssize_t i = 0; i < values->capacity; i++) { + Py_VISIT(values->values[i]); + } + return 0; } } + Py_VISIT(_PyObject_ManagedDictPointer(obj)->dict); + return 0; +} + +static void +set_dict_inline_values(PyObject *obj, PyDictObject *new_dict) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); + + PyDictValues *values = _PyObject_InlineValues(obj); + + Py_XINCREF(new_dict); + FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict, new_dict); + + if (values->valid) { + FT_ATOMIC_STORE_UINT8(values->valid, 0); + for (Py_ssize_t i = 0; i < values->capacity; i++) { + Py_CLEAR(values->values[i]); + } + } +} + +#ifdef Py_GIL_DISABLED + +// Trys and sets the dictionary for an object in the easy case when our current +// dictionary is either completely not materialized or is a dictionary which +// does not point at the inline values. +static bool +try_set_dict_inline_only_or_other_dict(PyObject *obj, PyObject *new_dict, PyDictObject **cur_dict) +{ + bool replaced = false; + Py_BEGIN_CRITICAL_SECTION(obj); + + PyDictObject *dict = *cur_dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + // We only have inline values, we can just completely replace them. + set_dict_inline_values(obj, (PyDictObject *)new_dict); + replaced = true; + goto exit_lock; + } + + if (FT_ATOMIC_LOAD_PTR_RELAXED(dict->ma_values) != _PyObject_InlineValues(obj)) { + // We have a materialized dict which doesn't point at the inline values, + // We get to simply swap dictionaries and free the old dictionary. + FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict, + (PyDictObject *)Py_XNewRef(new_dict)); + replaced = true; + goto exit_lock; + } else { - PyObject *dict = _PyDictOrValues_GetDict(dorv); - Py_VISIT(dict); + // We have inline values, we need to lock the dict and the object + // at the same time to safely dematerialize them. To do that while releasing + // the object lock we need a strong reference to the current dictionary. + Py_INCREF(dict); + } +exit_lock: + Py_END_CRITICAL_SECTION(); + return replaced; +} + +// Replaces a dictionary that is probably the dictionary which has been +// materialized and points at the inline values. We could have raced +// and replaced it with another dictionary though. +static int +replace_dict_probably_inline_materialized(PyObject *obj, PyDictObject *inline_dict, + PyDictObject *cur_dict, PyObject *new_dict) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); + + if (cur_dict == inline_dict) { + assert(FT_ATOMIC_LOAD_PTR_RELAXED(inline_dict->ma_values) == _PyObject_InlineValues(obj)); + + int err = _PyDict_DetachFromObject(inline_dict, obj); + if (err != 0) { + assert(new_dict == NULL); + return err; + } } + + FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict, + (PyDictObject *)Py_XNewRef(new_dict)); return 0; } +#endif + +static void +decref_maybe_delay(PyObject *obj, bool delay) +{ + if (delay) { + _PyObject_XDecRefDelayed(obj); + } + else { + Py_XDECREF(obj); + } +} + +static int +set_or_clear_managed_dict(PyObject *obj, PyObject *new_dict, bool clear) +{ + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); +#ifndef NDEBUG + Py_BEGIN_CRITICAL_SECTION(obj); + assert(_PyObject_InlineValuesConsistencyCheck(obj)); + Py_END_CRITICAL_SECTION(); +#endif + int err = 0; + PyTypeObject *tp = Py_TYPE(obj); + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { +#ifdef Py_GIL_DISABLED + PyDictObject *prev_dict; + if (!try_set_dict_inline_only_or_other_dict(obj, new_dict, &prev_dict)) { + // We had a materialized dictionary which pointed at the inline + // values. We need to lock both the object and the dict at the + // same time to safely replace it. We can't merely lock the dictionary + // while the object is locked because it could suspend the object lock. + PyDictObject *cur_dict; + + assert(prev_dict != NULL); + Py_BEGIN_CRITICAL_SECTION2(obj, prev_dict); + + // We could have had another thread race in between the call to + // try_set_dict_inline_only_or_other_dict where we locked the object + // and when we unlocked and re-locked the dictionary. + cur_dict = _PyObject_GetManagedDict(obj); + + err = replace_dict_probably_inline_materialized(obj, prev_dict, + cur_dict, new_dict); + + Py_END_CRITICAL_SECTION2(); + + // Decref for the dictionary we incref'd in try_set_dict_inline_only_or_other_dict + // while the object was locked + decref_maybe_delay((PyObject *)prev_dict, + !clear && prev_dict != cur_dict); + if (err != 0) { + return err; + } + + prev_dict = cur_dict; + } + + if (prev_dict != NULL) { + // decref for the dictionary that we replaced + decref_maybe_delay((PyObject *)prev_dict, !clear); + } + + return 0; +#else + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + set_dict_inline_values(obj, (PyDictObject *)new_dict); + return 0; + } + if (_PyDict_DetachFromObject(dict, obj) == 0) { + _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)Py_XNewRef(new_dict); + Py_DECREF(dict); + return 0; + } + assert(new_dict == NULL); + return -1; +#endif + } + else { + PyDictObject *dict; + + Py_BEGIN_CRITICAL_SECTION(obj); + + dict = _PyObject_ManagedDictPointer(obj)->dict; + + FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict, + (PyDictObject *)Py_XNewRef(new_dict)); + + Py_END_CRITICAL_SECTION(); + decref_maybe_delay((PyObject *)dict, !clear); + } + assert(_PyObject_InlineValuesConsistencyCheck(obj)); + return err; +} + +int +_PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict) +{ + return set_or_clear_managed_dict(obj, new_dict, false); +} + void PyObject_ClearManagedDict(PyObject *obj) { - PyTypeObject *tp = Py_TYPE(obj); - if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { - return; + if (set_or_clear_managed_dict(obj, NULL, true) < 0) { + /* Must be out of memory */ + assert(PyErr_Occurred() == PyExc_MemoryError); + PyErr_WriteUnraisable(NULL); + /* Clear the dict */ + PyDictObject *dict = _PyObject_GetManagedDict(obj); + Py_BEGIN_CRITICAL_SECTION2(dict, obj); + dict = _PyObject_ManagedDictPointer(obj)->dict; + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyDictKeysObject *oldkeys = dict->ma_keys; + set_keys(dict, Py_EMPTY_KEYS); + dict->ma_values = NULL; + dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(dict)); + STORE_USED(dict, 0); + set_dict_inline_values(obj, NULL); + Py_END_CRITICAL_SECTION2(); } - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - PyDictKeysObject *keys = CACHED_KEYS(tp); - for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - Py_CLEAR(values->values[i]); +} + +int +_PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj) +{ + ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(obj); + assert(_PyObject_ManagedDictPointer(obj)->dict == mp); + assert(_PyObject_InlineValuesConsistencyCheck(obj)); + + if (FT_ATOMIC_LOAD_PTR_RELAXED(mp->ma_values) != _PyObject_InlineValues(obj)) { + return 0; + } + + // We could be called with an unlocked dict when the caller knows the + // values are already detached, so we assert after inline values check. + ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(mp); + assert(mp->ma_values->embedded == 1); + assert(mp->ma_values->valid == 1); + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + + PyDictValues *values = copy_values(mp->ma_values); + + if (values == NULL) { + PyErr_NoMemory(); + return -1; + } + mp->ma_values = values; + + invalidate_and_clear_inline_values(_PyObject_InlineValues(obj)); + + assert(_PyObject_InlineValuesConsistencyCheck(obj)); + ASSERT_CONSISTENT(mp); + return 0; +} + +static inline PyObject * +ensure_managed_dict(PyObject *obj) +{ + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + PyTypeObject *tp = Py_TYPE(obj); + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(obj)->valid)) { + dict = _PyObject_MaterializeManagedDict(obj); + } + else { +#ifdef Py_GIL_DISABLED + // Check again that we're not racing with someone else creating the dict + Py_BEGIN_CRITICAL_SECTION(obj); + dict = _PyObject_GetManagedDict(obj); + if (dict != NULL) { + goto done; + } +#endif + dict = (PyDictObject *)new_dict_with_shared_keys(_PyInterpreterState_GET(), + CACHED_KEYS(tp)); + FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict, + (PyDictObject *)dict); + +#ifdef Py_GIL_DISABLED +done: + Py_END_CRITICAL_SECTION(); +#endif } - dorv_ptr->dict = NULL; - free_values(values, IS_DICT_SHARED((PyDictObject*)obj)); } - else { - PyObject *dict = dorv_ptr->dict; - if (dict) { - dorv_ptr->dict = NULL; - Py_DECREF(dict); + return (PyObject *)dict; +} + +static inline PyObject * +ensure_nonmanaged_dict(PyObject *obj, PyObject **dictptr) +{ + PyDictKeysObject *cached; + + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*dictptr); + if (dict == NULL) { +#ifdef Py_GIL_DISABLED + Py_BEGIN_CRITICAL_SECTION(obj); + dict = *dictptr; + if (dict != NULL) { + goto done; + } +#endif + PyTypeObject *tp = Py_TYPE(obj); + if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && (cached = CACHED_KEYS(tp))) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(!_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)); + dict = new_dict_with_shared_keys(interp, cached); + } + else { + dict = PyDict_New(); } + FT_ATOMIC_STORE_PTR_RELEASE(*dictptr, dict); +#ifdef Py_GIL_DISABLED +done: + Py_END_CRITICAL_SECTION(); +#endif } + return dict; } PyObject * PyObject_GenericGetDict(PyObject *obj, void *context) { - PyObject *dict; - PyInterpreterState *interp = _PyInterpreterState_GET(); PyTypeObject *tp = Py_TYPE(obj); if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - OBJECT_STAT_INC(dict_materialized_on_request); - dict = make_dict_from_instance_attributes( - interp, CACHED_KEYS(tp), values); - if (dict != NULL) { - dorv_ptr->dict = dict; - } - } - else { - dict = _PyDictOrValues_GetDict(*dorv_ptr); - if (dict == NULL) { - dictkeys_incref(CACHED_KEYS(tp)); - OBJECT_STAT_INC(dict_materialized_on_request); - dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); - dorv_ptr->dict = dict; - } - } + return Py_XNewRef(ensure_managed_dict(obj)); } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); @@ -6931,64 +7479,28 @@ PyObject_GenericGetDict(PyObject *obj, void *context) "This object has no __dict__"); return NULL; } - dict = *dictptr; - if (dict == NULL) { - PyTypeObject *tp = Py_TYPE(obj); - if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { - dictkeys_incref(CACHED_KEYS(tp)); - *dictptr = dict = new_dict_with_shared_keys( - interp, CACHED_KEYS(tp)); - } - else { - *dictptr = dict = PyDict_New(); - } - } + + return Py_XNewRef(ensure_nonmanaged_dict(obj, dictptr)); } - return Py_XNewRef(dict); } int -_PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, +_PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, PyObject *key, PyObject *value) { PyObject *dict; int res; - PyDictKeysObject *cached; - PyInterpreterState *interp = _PyInterpreterState_GET(); assert(dictptr != NULL); - if ((tp->tp_flags & Py_TPFLAGS_HEAPTYPE) && (cached = CACHED_KEYS(tp))) { - assert(dictptr != NULL); - dict = *dictptr; - if (dict == NULL) { - assert(!_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)); - dictkeys_incref(cached); - dict = new_dict_with_shared_keys(interp, cached); - if (dict == NULL) - return -1; - *dictptr = dict; - } - if (value == NULL) { - res = PyDict_DelItem(dict, key); - } - else { - res = PyDict_SetItem(dict, key, value); - } - } else { - dict = *dictptr; - if (dict == NULL) { - dict = PyDict_New(); - if (dict == NULL) - return -1; - *dictptr = dict; - } - if (value == NULL) { - res = PyDict_DelItem(dict, key); - } else { - res = PyDict_SetItem(dict, key, value); - } + dict = ensure_nonmanaged_dict(obj, dictptr); + if (dict == NULL) { + return -1; } + + Py_BEGIN_CRITICAL_SECTION(dict); + res = _PyDict_SetItem_LockHeld((PyDictObject *)dict, key, value); ASSERT_CONSISTENT(dict); + Py_END_CRITICAL_SECTION(); return res; } @@ -6999,20 +7511,54 @@ _PyDictKeys_DecRef(PyDictKeysObject *keys) dictkeys_decref(interp, keys, false); } -uint32_t _PyDictKeys_GetVersionForCurrentState(PyInterpreterState *interp, - PyDictKeysObject *dictkeys) +static inline uint32_t +get_next_dict_keys_version(PyInterpreterState *interp) { - if (dictkeys->dk_version != 0) { - return dictkeys->dk_version; - } +#ifdef Py_GIL_DISABLED + uint32_t v; + do { + v = _Py_atomic_load_uint32_relaxed( + &interp->dict_state.next_keys_version); + if (v == 0) { + return 0; + } + } while (!_Py_atomic_compare_exchange_uint32( + &interp->dict_state.next_keys_version, &v, v + 1)); +#else if (interp->dict_state.next_keys_version == 0) { return 0; } uint32_t v = interp->dict_state.next_keys_version++; - dictkeys->dk_version = v; +#endif return v; } +// In free-threaded builds the caller must ensure that the keys object is not +// being mutated concurrently by another thread. +uint32_t +_PyDictKeys_GetVersionForCurrentState(PyInterpreterState *interp, + PyDictKeysObject *dictkeys) +{ + uint32_t dk_version = FT_ATOMIC_LOAD_UINT32_RELAXED(dictkeys->dk_version); + if (dk_version != 0) { + return dk_version; + } + dk_version = get_next_dict_keys_version(interp); + FT_ATOMIC_STORE_UINT32_RELAXED(dictkeys->dk_version, dk_version); + return dk_version; +} + +uint32_t +_PyDict_GetKeysVersionForCurrentState(PyInterpreterState *interp, + PyDictObject *dict) +{ + ASSERT_DICT_LOCKED((PyObject *) dict); + uint32_t dk_version = + _PyDictKeys_GetVersionForCurrentState(interp, dict->ma_keys); + ensure_shared_on_keys_version_assignment(dict); + return dk_version; +} + static inline int validate_watcher_id(PyInterpreterState *interp, int watcher_id) { @@ -7038,7 +7584,7 @@ PyDict_Watch(int watcher_id, PyObject* dict) if (validate_watcher_id(interp, watcher_id)) { return -1; } - ((PyDictObject*)dict)->ma_version_tag |= (1LL << watcher_id); + ((PyDictObject*)dict)->_ma_watcher_tag |= (1LL << watcher_id); return 0; } @@ -7053,7 +7599,7 @@ PyDict_Unwatch(int watcher_id, PyObject* dict) if (validate_watcher_id(interp, watcher_id)) { return -1; } - ((PyDictObject*)dict)->ma_version_tag &= ~(1LL << watcher_id); + ((PyDictObject*)dict)->_ma_watcher_tag &= ~(1LL << watcher_id); return 0; } @@ -7121,3 +7667,24 @@ _PyDict_SendEvent(int watcher_bits, watcher_bits >>= 1; } } + +#ifndef NDEBUG +static int +_PyObject_InlineValuesConsistencyCheck(PyObject *obj) +{ + if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) { + return 1; + } + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + return 1; + } + if (dict->ma_values == _PyObject_InlineValues(obj) || + _PyObject_InlineValues(obj)->valid == 0) { + return 1; + } + assert(0); + return 0; +} +#endif diff --git a/Objects/enumobject.c b/Objects/enumobject.c index 556666779d8522..eb8952675269a2 100644 --- a/Objects/enumobject.c +++ b/Objects/enumobject.c @@ -23,6 +23,7 @@ typedef struct { PyObject* one; /* borrowed reference */ } enumobject; +#define _enumobject_CAST(op) ((enumobject *)(op)) /*[clinic input] @classmethod @@ -150,8 +151,9 @@ enumerate_vectorcall(PyObject *type, PyObject *const *args, } static void -enum_dealloc(enumobject *en) +enum_dealloc(PyObject *op) { + enumobject *en = _enumobject_CAST(op); PyObject_GC_UnTrack(en); Py_XDECREF(en->en_sit); Py_XDECREF(en->en_result); @@ -160,8 +162,9 @@ enum_dealloc(enumobject *en) } static int -enum_traverse(enumobject *en, visitproc visit, void *arg) +enum_traverse(PyObject *op, visitproc visit, void *arg) { + enumobject *en = _enumobject_CAST(op); Py_VISIT(en->en_sit); Py_VISIT(en->en_result); Py_VISIT(en->en_longindex); @@ -220,8 +223,9 @@ enum_next_long(enumobject *en, PyObject* next_item) } static PyObject * -enum_next(enumobject *en) +enum_next(PyObject *op) { + enumobject *en = _enumobject_CAST(op); PyObject *next_index; PyObject *next_item; PyObject *result = en->en_result; @@ -270,8 +274,9 @@ enum_next(enumobject *en) } static PyObject * -enum_reduce(enumobject *en, PyObject *Py_UNUSED(ignored)) +enum_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) { + enumobject *en = _enumobject_CAST(op); if (en->en_longindex != NULL) return Py_BuildValue("O(OO)", Py_TYPE(en), en->en_sit, en->en_longindex); else @@ -281,7 +286,7 @@ enum_reduce(enumobject *en, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); static PyMethodDef enum_methods[] = { - {"__reduce__", (PyCFunction)enum_reduce, METH_NOARGS, reduce_doc}, + {"__reduce__", enum_reduce, METH_NOARGS, reduce_doc}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* sentinel */ @@ -293,7 +298,7 @@ PyTypeObject PyEnum_Type = { sizeof(enumobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)enum_dealloc, /* tp_dealloc */ + enum_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -311,12 +316,12 @@ PyTypeObject PyEnum_Type = { Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /* tp_flags */ enum_new__doc__, /* tp_doc */ - (traverseproc)enum_traverse, /* tp_traverse */ + enum_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)enum_next, /* tp_iternext */ + enum_next, /* tp_iternext */ enum_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ @@ -329,7 +334,7 @@ PyTypeObject PyEnum_Type = { PyType_GenericAlloc, /* tp_alloc */ enum_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ - .tp_vectorcall = (vectorcallfunc)enumerate_vectorcall + .tp_vectorcall = enumerate_vectorcall }; /* Reversed Object ***************************************************************/ @@ -340,6 +345,8 @@ typedef struct { PyObject* seq; } reversedobject; +#define _reversedobject_CAST(op) ((reversedobject *)(op)) + /*[clinic input] @classmethod reversed.__new__ as reversed_new @@ -411,23 +418,26 @@ reversed_vectorcall(PyObject *type, PyObject * const*args, } static void -reversed_dealloc(reversedobject *ro) +reversed_dealloc(PyObject *op) { + reversedobject *ro = _reversedobject_CAST(op); PyObject_GC_UnTrack(ro); Py_XDECREF(ro->seq); Py_TYPE(ro)->tp_free(ro); } static int -reversed_traverse(reversedobject *ro, visitproc visit, void *arg) +reversed_traverse(PyObject *op, visitproc visit, void *arg) { + reversedobject *ro = _reversedobject_CAST(op); Py_VISIT(ro->seq); return 0; } static PyObject * -reversed_next(reversedobject *ro) +reversed_next(PyObject *op) { + reversedobject *ro = _reversedobject_CAST(op); PyObject *item; Py_ssize_t index = ro->index; @@ -447,8 +457,9 @@ reversed_next(reversedobject *ro) } static PyObject * -reversed_len(reversedobject *ro, PyObject *Py_UNUSED(ignored)) +reversed_len(PyObject *op, PyObject *Py_UNUSED(ignored)) { + reversedobject *ro = _reversedobject_CAST(op); Py_ssize_t position, seqsize; if (ro->seq == NULL) @@ -463,8 +474,9 @@ reversed_len(reversedobject *ro, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); static PyObject * -reversed_reduce(reversedobject *ro, PyObject *Py_UNUSED(ignored)) +reversed_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) { + reversedobject *ro = _reversedobject_CAST(op); if (ro->seq) return Py_BuildValue("O(O)n", Py_TYPE(ro), ro->seq, ro->index); else @@ -472,8 +484,9 @@ reversed_reduce(reversedobject *ro, PyObject *Py_UNUSED(ignored)) } static PyObject * -reversed_setstate(reversedobject *ro, PyObject *state) +reversed_setstate(PyObject *op, PyObject *state) { + reversedobject *ro = _reversedobject_CAST(op); Py_ssize_t index = PyLong_AsSsize_t(state); if (index == -1 && PyErr_Occurred()) return NULL; @@ -493,9 +506,9 @@ reversed_setstate(reversedobject *ro, PyObject *state) PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); static PyMethodDef reversediter_methods[] = { - {"__length_hint__", (PyCFunction)reversed_len, METH_NOARGS, length_hint_doc}, - {"__reduce__", (PyCFunction)reversed_reduce, METH_NOARGS, reduce_doc}, - {"__setstate__", (PyCFunction)reversed_setstate, METH_O, setstate_doc}, + {"__length_hint__", reversed_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", reversed_reduce, METH_NOARGS, reduce_doc}, + {"__setstate__", reversed_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -505,7 +518,7 @@ PyTypeObject PyReversed_Type = { sizeof(reversedobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)reversed_dealloc, /* tp_dealloc */ + reversed_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -523,12 +536,12 @@ PyTypeObject PyReversed_Type = { Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /* tp_flags */ reversed_new__doc__, /* tp_doc */ - (traverseproc)reversed_traverse,/* tp_traverse */ + reversed_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)reversed_next, /* tp_iternext */ + reversed_next, /* tp_iternext */ reversediter_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ @@ -541,5 +554,5 @@ PyTypeObject PyReversed_Type = { PyType_GenericAlloc, /* tp_alloc */ reversed_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ - .tp_vectorcall = (vectorcallfunc)reversed_vectorcall, + .tp_vectorcall = reversed_vectorcall, }; diff --git a/Objects/exception_handling_notes.txt b/Objects/exception_handling_notes.txt deleted file mode 100644 index 387ef935ce739e..00000000000000 --- a/Objects/exception_handling_notes.txt +++ /dev/null @@ -1,182 +0,0 @@ -Description of exception handling in Python 3.11 ------------------------------------------------- - -Python 3.11 uses what is known as "zero-cost" exception handling. -Prior to 3.11, exceptions were handled by a runtime stack of "blocks". - -In zero-cost exception handling, the cost of supporting exceptions is minimized. -In the common case (where no exception is raised) the cost is reduced -to zero (or close to zero). -The cost of raising an exception is increased, but not by much. - -The following code: - -def f(): - try: - g(0) - except: - return "fail" - -compiles as follows in 3.10: - - 2 0 SETUP_FINALLY 7 (to 16) - - 3 2 LOAD_GLOBAL 0 (g) - 4 LOAD_CONST 1 (0) - 6 CALL_NO_KW 1 - 8 POP_TOP - 10 POP_BLOCK - 12 LOAD_CONST 0 (None) - 14 RETURN_VALUE - - 4 >> 16 POP_TOP - 18 POP_TOP - 20 POP_TOP - - 5 22 POP_EXCEPT - 24 LOAD_CONST 3 ('fail') - 26 RETURN_VALUE - -Note the explicit instructions to push and pop from the "block" stack: -SETUP_FINALLY and POP_BLOCK. - -In 3.11, the SETUP_FINALLY and POP_BLOCK are eliminated, replaced with -a table to determine where to jump to when an exception is raised. - - 1 0 RESUME 0 - - 2 2 NOP - - 3 4 LOAD_GLOBAL 1 (g + NULL) - 16 LOAD_CONST 1 (0) - 18 PRECALL 1 - 22 CALL 1 - 32 POP_TOP - 34 LOAD_CONST 0 (None) - 36 RETURN_VALUE - >> 38 PUSH_EXC_INFO - - 4 40 POP_TOP - - 5 42 POP_EXCEPT - 44 LOAD_CONST 2 ('fail') - 46 RETURN_VALUE - >> 48 COPY 3 - 50 POP_EXCEPT - 52 RERAISE 1 -ExceptionTable: - 4 to 32 -> 38 [0] - 38 to 40 -> 48 [1] lasti - -(Note this code is from 3.11, later versions may have slightly different bytecode.) - -If an instruction raises an exception then its offset is used to find the target to jump to. -For example, the CALL at offset 22, falls into the range 4 to 32. -So, if g() raises an exception, then control jumps to offset 38. - - -Unwinding ---------- - -When an exception is raised, the current instruction offset is used to find following: -target to jump to, stack depth, and 'lasti', which determines whether the instruction -offset of the raising instruction should be pushed. - -This information is stored in the exception table, described below. - -If there is no relevant entry, the exception bubbles up to the caller. - -If there is an entry, then: - 1. pop values from the stack until it matches the stack depth for the handler. - 2. if 'lasti' is true, then push the offset that the exception was raised at. - 3. push the exception to the stack. - 4. jump to the target offset and resume execution. - - -Format of the exception table ------------------------------ - -Conceptually, the exception table consists of a sequence of 5-tuples: - 1. start-offset (inclusive) - 2. end-offset (exclusive) - 3. target - 4. stack-depth - 5. push-lasti (boolean) - -All offsets and lengths are in instructions, not bytes. - -We want the format to be compact, but quickly searchable. -For it to be compact, it needs to have variable sized entries so that we can store common (small) offsets compactly, but handle large offsets if needed. -For it to be searchable quickly, we need to support binary search giving us log(n) performance in all cases. -Binary search typically assumes fixed size entries, but that is not necessary, as long as we can identify the start of an entry. - -It is worth noting that the size (end-start) is always smaller than the end, so we encode the entries as: - start, size, target, depth, push-lasti - -Also, sizes are limited to 2**30 as the code length cannot exceed 2**31 and each instruction takes 2 bytes. -It also happens that depth is generally quite small. - -So, we need to encode: - start (up to 30 bits) - size (up to 30 bits) - target (up to 30 bits) - depth (up to ~8 bits) - lasti (1 bit) - -We need a marker for the start of the entry, so the first byte of entry will have the most significant bit set. -Since the most significant bit is reserved for marking the start of an entry, we have 7 bits per byte to encode offsets. -Encoding uses a standard varint encoding, but with only 7 bits instead of the usual 8. -The 8 bits of a bit are (msb left) SXdddddd where S is the start bit. X is the extend bit meaning that the next byte is required to extend the offset. - -In addition, we will combine depth and lasti into a single value, ((depth<<1)+lasti), before encoding. - -For example, the exception entry: - start: 20 - end: 28 - target: 100 - depth: 3 - lasti: False - -is encoded first by converting to the more compact four value form: - start: 20 - size: 8 - target: 100 - depth<<1+lasti: 6 - -which is then encoded as: - 148 (MSB + 20 for start) - 8 (size) - 65 (Extend bit + 1) - 36 (Remainder of target, 100 == (1<<6)+36) - 6 - -for a total of five bytes. - - - -Script to parse the exception table ------------------------------------ - -def parse_varint(iterator): - b = next(iterator) - val = b & 63 - while b&64: - val <<= 6 - b = next(iterator) - val |= b&63 - return val - -def parse_exception_table(code): - iterator = iter(code.co_exceptiontable) - try: - while True: - start = parse_varint(iterator)*2 - length = parse_varint(iterator)*2 - end = start + length - 2 # Present as inclusive, not exclusive - target = parse_varint(iterator)*2 - dl = parse_varint(iterator) - depth = dl >> 1 - lasti = bool(dl&1) - yield start, end, target, depth, lasti - except StopIteration: - return diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 63c461d34fb4ff..ea2733435fc3ec 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -16,6 +16,14 @@ #include "osdefs.h" // SEP +#include "clinic/exceptions.c.h" + +/*[clinic input] +class BaseException "PyBaseExceptionObject *" "&PyExc_BaseException" +class BaseExceptionGroup "PyBaseExceptionGroupObject *" "&PyExc_BaseExceptionGroup" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b7c45e78cff8edc3]*/ + /* Compatibility aliases */ PyObject *PyExc_EnvironmentError = NULL; // borrowed ref @@ -78,6 +86,40 @@ BaseException_init(PyBaseExceptionObject *self, PyObject *args, PyObject *kwds) return 0; } + +static PyObject * +BaseException_vectorcall(PyObject *type_obj, PyObject * const*args, + size_t nargsf, PyObject *kwnames) +{ + PyTypeObject *type = _PyType_CAST(type_obj); + if (!_PyArg_NoKwnames(type->tp_name, kwnames)) { + return NULL; + } + + PyBaseExceptionObject *self; + self = (PyBaseExceptionObject *)type->tp_alloc(type, 0); + if (!self) { + return NULL; + } + + // The dict is created on the fly in PyObject_GenericSetAttr() + self->dict = NULL; + self->notes = NULL; + self->traceback = NULL; + self->cause = NULL; + self->context = NULL; + self->suppress_context = 0; + + self->args = _PyTuple_FromArray(args, PyVectorcall_NARGS(nargsf)); + if (!self->args) { + Py_DECREF(self); + return NULL; + } + + return (PyObject *)self; +} + + static int BaseException_clear(PyBaseExceptionObject *self) { @@ -118,30 +160,50 @@ BaseException_traverse(PyBaseExceptionObject *self, visitproc visit, void *arg) static PyObject * BaseException_str(PyBaseExceptionObject *self) { + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(self); switch (PyTuple_GET_SIZE(self->args)) { case 0: - return PyUnicode_FromString(""); + res = Py_GetConstant(Py_CONSTANT_EMPTY_STR); + break; case 1: - return PyObject_Str(PyTuple_GET_ITEM(self->args, 0)); + res = PyObject_Str(PyTuple_GET_ITEM(self->args, 0)); + break; default: - return PyObject_Str(self->args); + res = PyObject_Str(self->args); + break; } + Py_END_CRITICAL_SECTION(); + return res; } static PyObject * BaseException_repr(PyBaseExceptionObject *self) { + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(self); const char *name = _PyType_Name(Py_TYPE(self)); - if (PyTuple_GET_SIZE(self->args) == 1) - return PyUnicode_FromFormat("%s(%R)", name, + if (PyTuple_GET_SIZE(self->args) == 1) { + res = PyUnicode_FromFormat("%s(%R)", name, PyTuple_GET_ITEM(self->args, 0)); - else - return PyUnicode_FromFormat("%s%R", name, self->args); + } + else { + res = PyUnicode_FromFormat("%s%R", name, self->args); + } + Py_END_CRITICAL_SECTION(); + return res; } /* Pickling support */ + +/*[clinic input] +@critical_section +BaseException.__reduce__ +[clinic start generated code]*/ + static PyObject * -BaseException_reduce(PyBaseExceptionObject *self, PyObject *Py_UNUSED(ignored)) +BaseException___reduce___impl(PyBaseExceptionObject *self) +/*[clinic end generated code: output=af87c1247ef98748 input=283be5a10d9c964f]*/ { if (self->args && self->dict) return PyTuple_Pack(3, Py_TYPE(self), self->args, self->dict); @@ -154,8 +216,17 @@ BaseException_reduce(PyBaseExceptionObject *self, PyObject *Py_UNUSED(ignored)) * all their attributes in the __dict__. Code is taken from cPickle's * load_build function. */ + +/*[clinic input] +@critical_section +BaseException.__setstate__ + state: object + / +[clinic start generated code]*/ + static PyObject * -BaseException_setstate(PyObject *self, PyObject *state) +BaseException___setstate___impl(PyBaseExceptionObject *self, PyObject *state) +/*[clinic end generated code: output=f3834889950453ab input=5524b61cfe9b9856]*/ { PyObject *d_key, *d_value; Py_ssize_t i = 0; @@ -168,7 +239,7 @@ BaseException_setstate(PyObject *self, PyObject *state) while (PyDict_Next(state, &i, &d_key, &d_value)) { Py_INCREF(d_key); Py_INCREF(d_value); - int res = PyObject_SetAttr(self, d_key, d_value); + int res = PyObject_SetAttr((PyObject *)self, d_key, d_value); Py_DECREF(d_value); Py_DECREF(d_key); if (res < 0) { @@ -179,18 +250,26 @@ BaseException_setstate(PyObject *self, PyObject *state) Py_RETURN_NONE; } + +/*[clinic input] +@critical_section +BaseException.with_traceback + tb: object + / + +Set self.__traceback__ to tb and return self. +[clinic start generated code]*/ + static PyObject * -BaseException_with_traceback(PyObject *self, PyObject *tb) { - if (PyException_SetTraceback(self, tb)) +BaseException_with_traceback_impl(PyBaseExceptionObject *self, PyObject *tb) +/*[clinic end generated code: output=81e92f2387927f10 input=b5fb64d834717e36]*/ +{ + if (BaseException___traceback___set_impl(self, tb) < 0){ return NULL; - + } return Py_NewRef(self); } -PyDoc_STRVAR(with_traceback_doc, -"Exception.with_traceback(tb) --\n\ - set self.__traceback__ to tb and return self."); - static inline PyBaseExceptionObject* _PyBaseExceptionObject_cast(PyObject *exc) { @@ -198,18 +277,21 @@ _PyBaseExceptionObject_cast(PyObject *exc) return (PyBaseExceptionObject *)exc; } +/*[clinic input] +@critical_section +BaseException.add_note + note: object(subclass_of="&PyUnicode_Type") + / + +Add a note to the exception +[clinic start generated code]*/ + static PyObject * -BaseException_add_note(PyObject *self, PyObject *note) +BaseException_add_note_impl(PyBaseExceptionObject *self, PyObject *note) +/*[clinic end generated code: output=fb7cbcba611c187b input=e60a6b6e9596acaf]*/ { - if (!PyUnicode_Check(note)) { - PyErr_Format(PyExc_TypeError, - "note must be a str, not '%s'", - Py_TYPE(note)->tp_name); - return NULL; - } - PyObject *notes; - if (PyObject_GetOptionalAttr(self, &_Py_ID(__notes__), ¬es) < 0) { + if (PyObject_GetOptionalAttr((PyObject *)self, &_Py_ID(__notes__), ¬es) < 0) { return NULL; } if (notes == NULL) { @@ -217,7 +299,7 @@ BaseException_add_note(PyObject *self, PyObject *note) if (notes == NULL) { return NULL; } - if (PyObject_SetAttr(self, &_Py_ID(__notes__), notes) < 0) { + if (PyObject_SetAttr((PyObject *)self, &_Py_ID(__notes__), notes) < 0) { Py_DECREF(notes); return NULL; } @@ -235,22 +317,23 @@ BaseException_add_note(PyObject *self, PyObject *note) Py_RETURN_NONE; } -PyDoc_STRVAR(add_note_doc, -"Exception.add_note(note) --\n\ - add a note to the exception"); - static PyMethodDef BaseException_methods[] = { - {"__reduce__", (PyCFunction)BaseException_reduce, METH_NOARGS }, - {"__setstate__", (PyCFunction)BaseException_setstate, METH_O }, - {"with_traceback", (PyCFunction)BaseException_with_traceback, METH_O, - with_traceback_doc}, - {"add_note", (PyCFunction)BaseException_add_note, METH_O, - add_note_doc}, - {NULL, NULL, 0, NULL}, + BASEEXCEPTION___REDUCE___METHODDEF + BASEEXCEPTION___SETSTATE___METHODDEF + BASEEXCEPTION_WITH_TRACEBACK_METHODDEF + BASEEXCEPTION_ADD_NOTE_METHODDEF + {NULL, NULL, 0, NULL}, }; +/*[clinic input] +@critical_section +@getter +BaseException.args +[clinic start generated code]*/ + static PyObject * -BaseException_get_args(PyBaseExceptionObject *self, void *Py_UNUSED(ignored)) +BaseException_args_get_impl(PyBaseExceptionObject *self) +/*[clinic end generated code: output=e02e34e35cf4d677 input=64282386e4d7822d]*/ { if (self->args == NULL) { Py_RETURN_NONE; @@ -258,23 +341,37 @@ BaseException_get_args(PyBaseExceptionObject *self, void *Py_UNUSED(ignored)) return Py_NewRef(self->args); } +/*[clinic input] +@critical_section +@setter +BaseException.args +[clinic start generated code]*/ + static int -BaseException_set_args(PyBaseExceptionObject *self, PyObject *val, void *Py_UNUSED(ignored)) +BaseException_args_set_impl(PyBaseExceptionObject *self, PyObject *value) +/*[clinic end generated code: output=331137e11d8f9e80 input=2400047ea5970a84]*/ { PyObject *seq; - if (val == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_TypeError, "args may not be deleted"); return -1; } - seq = PySequence_Tuple(val); + seq = PySequence_Tuple(value); if (!seq) return -1; Py_XSETREF(self->args, seq); return 0; } +/*[clinic input] +@critical_section +@getter +BaseException.__traceback__ +[clinic start generated code]*/ + static PyObject * -BaseException_get_tb(PyBaseExceptionObject *self, void *Py_UNUSED(ignored)) +BaseException___traceback___get_impl(PyBaseExceptionObject *self) +/*[clinic end generated code: output=17cf874a52339398 input=a2277f0de62170cf]*/ { if (self->traceback == NULL) { Py_RETURN_NONE; @@ -282,17 +379,26 @@ BaseException_get_tb(PyBaseExceptionObject *self, void *Py_UNUSED(ignored)) return Py_NewRef(self->traceback); } + +/*[clinic input] +@critical_section +@setter +BaseException.__traceback__ +[clinic start generated code]*/ + static int -BaseException_set_tb(PyBaseExceptionObject *self, PyObject *tb, void *Py_UNUSED(ignored)) +BaseException___traceback___set_impl(PyBaseExceptionObject *self, + PyObject *value) +/*[clinic end generated code: output=a82c86d9f29f48f0 input=12676035676badad]*/ { - if (tb == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_TypeError, "__traceback__ may not be deleted"); return -1; } - if (PyTraceBack_Check(tb)) { - Py_XSETREF(self->traceback, Py_NewRef(tb)); + if (PyTraceBack_Check(value)) { + Py_XSETREF(self->traceback, Py_NewRef(value)); } - else if (tb == Py_None) { + else if (value == Py_None) { Py_CLEAR(self->traceback); } else { @@ -303,73 +409,100 @@ BaseException_set_tb(PyBaseExceptionObject *self, PyObject *tb, void *Py_UNUSED( return 0; } +/*[clinic input] +@critical_section +@getter +BaseException.__context__ +[clinic start generated code]*/ + static PyObject * -BaseException_get_context(PyObject *self, void *Py_UNUSED(ignored)) +BaseException___context___get_impl(PyBaseExceptionObject *self) +/*[clinic end generated code: output=6ec5d296ce8d1c93 input=b2d22687937e66ab]*/ { - PyObject *res = PyException_GetContext(self); - if (res) - return res; /* new reference already returned above */ - Py_RETURN_NONE; + if (self->context == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(self->context); } +/*[clinic input] +@critical_section +@setter +BaseException.__context__ +[clinic start generated code]*/ + static int -BaseException_set_context(PyObject *self, PyObject *arg, void *Py_UNUSED(ignored)) +BaseException___context___set_impl(PyBaseExceptionObject *self, + PyObject *value) +/*[clinic end generated code: output=b4cb52dcca1da3bd input=c0971adf47fa1858]*/ { - if (arg == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_TypeError, "__context__ may not be deleted"); return -1; - } else if (arg == Py_None) { - arg = NULL; - } else if (!PyExceptionInstance_Check(arg)) { + } else if (value == Py_None) { + value = NULL; + } else if (!PyExceptionInstance_Check(value)) { PyErr_SetString(PyExc_TypeError, "exception context must be None " "or derive from BaseException"); return -1; } else { - /* PyException_SetContext steals this reference */ - Py_INCREF(arg); + Py_INCREF(value); } - PyException_SetContext(self, arg); + Py_XSETREF(self->context, value); return 0; } +/*[clinic input] +@critical_section +@getter +BaseException.__cause__ +[clinic start generated code]*/ + static PyObject * -BaseException_get_cause(PyObject *self, void *Py_UNUSED(ignored)) +BaseException___cause___get_impl(PyBaseExceptionObject *self) +/*[clinic end generated code: output=987f6c4d8a0bdbab input=40e0eac427b6e602]*/ { - PyObject *res = PyException_GetCause(self); - if (res) - return res; /* new reference already returned above */ - Py_RETURN_NONE; + if (self->cause == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(self->cause); } +/*[clinic input] +@critical_section +@setter +BaseException.__cause__ +[clinic start generated code]*/ + static int -BaseException_set_cause(PyObject *self, PyObject *arg, void *Py_UNUSED(ignored)) +BaseException___cause___set_impl(PyBaseExceptionObject *self, + PyObject *value) +/*[clinic end generated code: output=6161315398aaf541 input=e1b403c0bde3f62a]*/ { - if (arg == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_TypeError, "__cause__ may not be deleted"); return -1; - } else if (arg == Py_None) { - arg = NULL; - } else if (!PyExceptionInstance_Check(arg)) { + } else if (value == Py_None) { + value = NULL; + } else if (!PyExceptionInstance_Check(value)) { PyErr_SetString(PyExc_TypeError, "exception cause must be None " "or derive from BaseException"); return -1; } else { /* PyException_SetCause steals this reference */ - Py_INCREF(arg); + Py_INCREF(value); } - PyException_SetCause(self, arg); + PyException_SetCause((PyObject *)self, value); return 0; } static PyGetSetDef BaseException_getset[] = { {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict}, - {"args", (getter)BaseException_get_args, (setter)BaseException_set_args}, - {"__traceback__", (getter)BaseException_get_tb, (setter)BaseException_set_tb}, - {"__context__", BaseException_get_context, - BaseException_set_context, PyDoc_STR("exception context")}, - {"__cause__", BaseException_get_cause, - BaseException_set_cause, PyDoc_STR("exception cause")}, + BASEEXCEPTION_ARGS_GETSETDEF + BASEEXCEPTION___TRACEBACK___GETSETDEF + BASEEXCEPTION___CONTEXT___GETSETDEF + BASEEXCEPTION___CAUSE___GETSETDEF {NULL}, }; @@ -377,59 +510,81 @@ static PyGetSetDef BaseException_getset[] = { PyObject * PyException_GetTraceback(PyObject *self) { - PyBaseExceptionObject *base_self = _PyBaseExceptionObject_cast(self); - return Py_XNewRef(base_self->traceback); + PyObject *traceback; + Py_BEGIN_CRITICAL_SECTION(self); + traceback = Py_XNewRef(_PyBaseExceptionObject_cast(self)->traceback); + Py_END_CRITICAL_SECTION(); + return traceback; } int PyException_SetTraceback(PyObject *self, PyObject *tb) { - return BaseException_set_tb(_PyBaseExceptionObject_cast(self), tb, NULL); + int res; + Py_BEGIN_CRITICAL_SECTION(self); + res = BaseException___traceback___set_impl(_PyBaseExceptionObject_cast(self), tb); + Py_END_CRITICAL_SECTION(); + return res; } PyObject * PyException_GetCause(PyObject *self) { - PyObject *cause = _PyBaseExceptionObject_cast(self)->cause; - return Py_XNewRef(cause); + PyObject *cause; + Py_BEGIN_CRITICAL_SECTION(self); + cause = Py_XNewRef(_PyBaseExceptionObject_cast(self)->cause); + Py_END_CRITICAL_SECTION(); + return cause; } /* Steals a reference to cause */ void PyException_SetCause(PyObject *self, PyObject *cause) { + Py_BEGIN_CRITICAL_SECTION(self); PyBaseExceptionObject *base_self = _PyBaseExceptionObject_cast(self); base_self->suppress_context = 1; Py_XSETREF(base_self->cause, cause); + Py_END_CRITICAL_SECTION(); } PyObject * PyException_GetContext(PyObject *self) { - PyObject *context = _PyBaseExceptionObject_cast(self)->context; - return Py_XNewRef(context); + PyObject *context; + Py_BEGIN_CRITICAL_SECTION(self); + context = Py_XNewRef(_PyBaseExceptionObject_cast(self)->context); + Py_END_CRITICAL_SECTION(); + return context; } /* Steals a reference to context */ void PyException_SetContext(PyObject *self, PyObject *context) { + Py_BEGIN_CRITICAL_SECTION(self); Py_XSETREF(_PyBaseExceptionObject_cast(self)->context, context); + Py_END_CRITICAL_SECTION(); } PyObject * PyException_GetArgs(PyObject *self) { - PyObject *args = _PyBaseExceptionObject_cast(self)->args; - return Py_NewRef(args); + PyObject *args; + Py_BEGIN_CRITICAL_SECTION(self); + args = Py_NewRef(_PyBaseExceptionObject_cast(self)->args); + Py_END_CRITICAL_SECTION(); + return args; } void PyException_SetArgs(PyObject *self, PyObject *args) { + Py_BEGIN_CRITICAL_SECTION(self); Py_INCREF(args); Py_XSETREF(_PyBaseExceptionObject_cast(self)->args, args); + Py_END_CRITICAL_SECTION(); } const char * @@ -486,6 +641,7 @@ static PyTypeObject _PyExc_BaseException = { (initproc)BaseException_init, /* tp_init */ 0, /* tp_alloc */ BaseException_new, /* tp_new */ + .tp_vectorcall = BaseException_vectorcall, }; /* the CPython API expects exceptions to be (PyObject *) - both a hold-over from the previous implementation and also allowing Python objects to be used @@ -510,10 +666,10 @@ static PyTypeObject _PyExc_ ## EXCNAME = { \ }; \ PyObject *PyExc_ ## EXCNAME = (PyObject *)&_PyExc_ ## EXCNAME -#define MiddlingExtendsException(EXCBASE, EXCNAME, EXCSTORE, EXCDOC) \ -static PyTypeObject _PyExc_ ## EXCNAME = { \ +#define MiddlingExtendsExceptionEx(EXCBASE, EXCNAME, PYEXCNAME, EXCSTORE, EXCDOC) \ +PyTypeObject _PyExc_ ## EXCNAME = { \ PyVarObject_HEAD_INIT(NULL, 0) \ - # EXCNAME, \ + # PYEXCNAME, \ sizeof(Py ## EXCSTORE ## Object), \ 0, (destructor)EXCSTORE ## _dealloc, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ 0, 0, 0, 0, 0, \ @@ -522,8 +678,12 @@ static PyTypeObject _PyExc_ ## EXCNAME = { \ (inquiry)EXCSTORE ## _clear, 0, 0, 0, 0, 0, 0, 0, &_ ## EXCBASE, \ 0, 0, 0, offsetof(Py ## EXCSTORE ## Object, dict), \ (initproc)EXCSTORE ## _init, 0, 0, \ -}; \ -PyObject *PyExc_ ## EXCNAME = (PyObject *)&_PyExc_ ## EXCNAME +}; + +#define MiddlingExtendsException(EXCBASE, EXCNAME, EXCSTORE, EXCDOC) \ + static MiddlingExtendsExceptionEx( \ + EXCBASE, EXCNAME, EXCNAME, EXCSTORE, EXCDOC); \ + PyObject *PyExc_ ## EXCNAME = (PyObject *)&_PyExc_ ## EXCNAME #define ComplexExtendsException(EXCBASE, EXCNAME, EXCSTORE, EXCNEW, \ EXCMETHODS, EXCMEMBERS, EXCGETSET, \ @@ -875,10 +1035,18 @@ BaseExceptionGroup_str(PyBaseExceptionGroupObject *self) self->msg, num_excs, num_excs > 1 ? "s" : ""); } +/*[clinic input] +@critical_section +BaseExceptionGroup.derive + excs: object + / +[clinic start generated code]*/ + static PyObject * -BaseExceptionGroup_derive(PyObject *self_, PyObject *excs) +BaseExceptionGroup_derive_impl(PyBaseExceptionGroupObject *self, + PyObject *excs) +/*[clinic end generated code: output=4307564218dfbf06 input=f72009d38e98cec1]*/ { - PyBaseExceptionGroupObject *self = _PyBaseExceptionGroupObject_cast(self_); PyObject *init_args = PyTuple_Pack(2, self->msg, excs); if (!init_args) { return NULL; @@ -1171,8 +1339,17 @@ exceptiongroup_split_recursive(PyObject *exc, return retval; } +/*[clinic input] +@critical_section +BaseExceptionGroup.split + matcher_value: object + / +[clinic start generated code]*/ + static PyObject * -BaseExceptionGroup_split(PyObject *self, PyObject *matcher_value) +BaseExceptionGroup_split_impl(PyBaseExceptionGroupObject *self, + PyObject *matcher_value) +/*[clinic end generated code: output=d74db579da4df6e2 input=0c5cfbfed57e0052]*/ { _exceptiongroup_split_matcher_type matcher_type; if (get_matcher_type(matcher_value, &matcher_type) < 0) { @@ -1182,7 +1359,7 @@ BaseExceptionGroup_split(PyObject *self, PyObject *matcher_value) _exceptiongroup_split_result split_result; bool construct_rest = true; if (exceptiongroup_split_recursive( - self, matcher_type, matcher_value, + (PyObject *)self, matcher_type, matcher_value, construct_rest, &split_result) < 0) { return NULL; } @@ -1197,8 +1374,17 @@ BaseExceptionGroup_split(PyObject *self, PyObject *matcher_value) return result; } +/*[clinic input] +@critical_section +BaseExceptionGroup.subgroup + matcher_value: object + / +[clinic start generated code]*/ + static PyObject * -BaseExceptionGroup_subgroup(PyObject *self, PyObject *matcher_value) +BaseExceptionGroup_subgroup_impl(PyBaseExceptionGroupObject *self, + PyObject *matcher_value) +/*[clinic end generated code: output=07dbec8f77d4dd8e input=988ffdd755a151ce]*/ { _exceptiongroup_split_matcher_type matcher_type; if (get_matcher_type(matcher_value, &matcher_type) < 0) { @@ -1208,7 +1394,7 @@ BaseExceptionGroup_subgroup(PyObject *self, PyObject *matcher_value) _exceptiongroup_split_result split_result; bool construct_rest = false; if (exceptiongroup_split_recursive( - self, matcher_type, matcher_value, + (PyObject *)self, matcher_type, matcher_value, construct_rest, &split_result) < 0) { return NULL; } @@ -1474,9 +1660,9 @@ static PyMemberDef BaseExceptionGroup_members[] = { static PyMethodDef BaseExceptionGroup_methods[] = { {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, - {"derive", (PyCFunction)BaseExceptionGroup_derive, METH_O}, - {"split", (PyCFunction)BaseExceptionGroup_split, METH_O}, - {"subgroup", (PyCFunction)BaseExceptionGroup_subgroup, METH_O}, + BASEEXCEPTIONGROUP_DERIVE_METHODDEF + BASEEXCEPTIONGROUP_SPLIT_METHODDEF + BASEEXCEPTIONGROUP_SUBGROUP_METHODDEF {NULL} }; @@ -2573,8 +2759,8 @@ MiddlingExtendsException(PyExc_IndentationError, TabError, SyntaxError, /* * IncompleteInputError extends SyntaxError */ -MiddlingExtendsException(PyExc_SyntaxError, IncompleteInputError, SyntaxError, - "incomplete input."); +MiddlingExtendsExceptionEx(PyExc_SyntaxError, IncompleteInputError, _IncompleteInputError, + SyntaxError, "incomplete input."); /* * LookupError extends Exception @@ -2628,280 +2814,623 @@ SimpleExtendsException(PyExc_Exception, ValueError, SimpleExtendsException(PyExc_ValueError, UnicodeError, "Unicode related error."); + +/* + * Check the validity of 'attr' as a unicode or bytes object depending + * on 'as_bytes' and return a new reference on it if it is the case. + * + * The 'name' is the attribute name and is only used for error reporting. + * + * On success, this returns a strong reference on 'attr'. + * On failure, this sets a TypeError and returns NULL. + */ static PyObject * -get_string(PyObject *attr, const char *name) +as_unicode_error_attribute(PyObject *attr, const char *name, int as_bytes) { - if (!attr) { - PyErr_Format(PyExc_TypeError, "%.200s attribute not set", name); + assert(as_bytes == 0 || as_bytes == 1); + if (attr == NULL) { + PyErr_Format(PyExc_TypeError, "%s attribute not set", name); return NULL; } - - if (!PyBytes_Check(attr)) { - PyErr_Format(PyExc_TypeError, "%.200s attribute must be bytes", name); + if (!(as_bytes ? PyBytes_Check(attr) : PyUnicode_Check(attr))) { + PyErr_Format(PyExc_TypeError, + "%s attribute must be %s", + name, + as_bytes ? "bytes" : "unicode"); return NULL; } return Py_NewRef(attr); } -static PyObject * -get_unicode(PyObject *attr, const char *name) + +#define PyUnicodeError_Check(PTR) \ + PyObject_TypeCheck((PTR), (PyTypeObject *)PyExc_UnicodeError) +#define PyUnicodeError_CAST(PTR) \ + (assert(PyUnicodeError_Check(PTR)), ((PyUnicodeErrorObject *)(PTR))) + + +/* class names to use when reporting errors */ +#define Py_UNICODE_ENCODE_ERROR_NAME "UnicodeEncodeError" +#define Py_UNICODE_DECODE_ERROR_NAME "UnicodeDecodeError" +#define Py_UNICODE_TRANSLATE_ERROR_NAME "UnicodeTranslateError" + + +/* + * Check that 'self' is a UnicodeError object. + * + * On success, this returns 0. + * On failure, this sets a TypeError exception and returns -1. + * + * The 'expect_type' is the name of the expected type, which is + * only used for error reporting. + * + * As an implementation detail, the `PyUnicode*Error_*` functions + * currently allow *any* subclass of UnicodeError as 'self'. + * + * Use one of the `Py_UNICODE_*_ERROR_NAME` macros to avoid typos. + */ +static inline int +check_unicode_error_type(PyObject *self, const char *expect_type) { - if (!attr) { - PyErr_Format(PyExc_TypeError, "%.200s attribute not set", name); - return NULL; + assert(self != NULL); + if (!PyUnicodeError_Check(self)) { + PyErr_Format(PyExc_TypeError, + "expecting a %s object, got %T", expect_type, self); + return -1; } + return 0; +} - if (!PyUnicode_Check(attr)) { - PyErr_Format(PyExc_TypeError, - "%.200s attribute must be unicode", name); - return NULL; + +// --- PyUnicodeEncodeObject: internal helpers -------------------------------- +// +// In the helpers below, the caller is responsible to ensure that 'self' +// is a PyUnicodeErrorObject, although this is verified on DEBUG builds +// through PyUnicodeError_CAST(). + +/* + * Return the underlying (str) 'encoding' attribute of a UnicodeError object. + */ +static inline PyObject * +unicode_error_get_encoding_impl(PyObject *self) +{ + assert(self != NULL); + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); + return as_unicode_error_attribute(exc->encoding, "encoding", false); +} + + +/* + * Return the underlying 'object' attribute of a UnicodeError object + * as a bytes or a string instance, depending on the 'as_bytes' flag. + */ +static inline PyObject * +unicode_error_get_object_impl(PyObject *self, int as_bytes) +{ + assert(self != NULL); + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); + return as_unicode_error_attribute(exc->object, "object", as_bytes); +} + + +/* + * Return the underlying (str) 'reason' attribute of a UnicodeError object. + */ +static inline PyObject * +unicode_error_get_reason_impl(PyObject *self) +{ + assert(self != NULL); + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); + return as_unicode_error_attribute(exc->reason, "reason", false); +} + + +/* + * Set the underlying (str) 'reason' attribute of a UnicodeError object. + * + * Return 0 on success and -1 on failure. + */ +static inline int +unicode_error_set_reason_impl(PyObject *self, const char *reason) +{ + assert(self != NULL); + PyObject *value = PyUnicode_FromString(reason); + if (value == NULL) { + return -1; } - return Py_NewRef(attr); + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); + Py_XSETREF(exc->reason, value); + return 0; } -static int -set_unicodefromstring(PyObject **attr, const char *value) + +/* + * Set the 'start' attribute of a UnicodeError object. + * + * Return 0 on success and -1 on failure. + */ +static inline int +unicode_error_set_start_impl(PyObject *self, Py_ssize_t start) +{ + assert(self != NULL); + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); + exc->start = start; + return 0; +} + + +/* + * Set the 'end' attribute of a UnicodeError object. + * + * Return 0 on success and -1 on failure. + */ +static inline int +unicode_error_set_end_impl(PyObject *self, Py_ssize_t end) +{ + assert(self != NULL); + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); + exc->end = end; + return 0; +} + +// --- PyUnicodeEncodeObject: internal getters -------------------------------- + +/* + * Adjust the (inclusive) 'start' value of a UnicodeError object. + * + * The 'start' can be negative or not, but when adjusting the value, + * we clip it in [0, max(0, objlen - 1)] and do not interpret it as + * a relative offset. + * + * This function always succeeds. + */ +static Py_ssize_t +unicode_error_adjust_start(Py_ssize_t start, Py_ssize_t objlen) { - PyObject *obj = PyUnicode_FromString(value); - if (!obj) + assert(objlen >= 0); + if (start < 0) { + start = 0; + } + if (start >= objlen) { + start = objlen == 0 ? 0 : objlen - 1; + } + return start; +} + + +/* Assert some properties of the adjusted 'start' value. */ +#ifndef NDEBUG +static void +assert_adjusted_unicode_error_start(Py_ssize_t start, Py_ssize_t objlen) +{ + assert(objlen >= 0); + /* in the future, `min_start` may be something else */ + Py_ssize_t min_start = 0; + assert(start >= min_start); + /* in the future, `max_start` may be something else */ + Py_ssize_t max_start = Py_MAX(min_start, objlen - 1); + assert(start <= max_start); +} +#else +#define assert_adjusted_unicode_error_start(...) +#endif + + +/* + * Adjust the (exclusive) 'end' value of a UnicodeError object. + * + * The 'end' can be negative or not, but when adjusting the value, + * we clip it in [min(1, objlen), max(min(1, objlen), objlen)] and + * do not interpret it as a relative offset. + * + * This function always succeeds. + */ +static Py_ssize_t +unicode_error_adjust_end(Py_ssize_t end, Py_ssize_t objlen) +{ + assert(objlen >= 0); + if (end < 1) { + end = 1; + } + if (end > objlen) { + end = objlen; + } + return end; +} + + +/* Assert some properties of the adjusted 'end' value. */ +#ifndef NDEBUG +static void +assert_adjusted_unicode_error_end(Py_ssize_t end, Py_ssize_t objlen) +{ + assert(objlen >= 0); + /* in the future, `min_end` may be something else */ + Py_ssize_t min_end = Py_MIN(1, objlen); + assert(end >= min_end); + /* in the future, `max_end` may be something else */ + Py_ssize_t max_end = Py_MAX(min_end, objlen); + assert(end <= max_end); +} +#else +#define assert_adjusted_unicode_error_end(...) +#endif + + +/* + * Adjust the length of the range described by a UnicodeError object. + * + * The 'start' and 'end' arguments must have been obtained by + * unicode_error_adjust_start() and unicode_error_adjust_end(). + * + * The result is clipped in [0, objlen]. By construction, it + * will always be smaller than 'objlen' as 'start' and 'end' + * are smaller than 'objlen'. + */ +static Py_ssize_t +unicode_error_adjust_len(Py_ssize_t start, Py_ssize_t end, Py_ssize_t objlen) +{ + assert_adjusted_unicode_error_start(start, objlen); + assert_adjusted_unicode_error_end(end, objlen); + Py_ssize_t ranlen = end - start; + assert(ranlen <= objlen); + return ranlen < 0 ? 0 : ranlen; +} + + +/* Assert some properties of the adjusted range 'len' value. */ +#ifndef NDEBUG +static void +assert_adjusted_unicode_error_len(Py_ssize_t ranlen, Py_ssize_t objlen) +{ + assert(objlen >= 0); + assert(ranlen >= 0); + assert(ranlen <= objlen); +} +#else +#define assert_adjusted_unicode_error_len(...) +#endif + + +/* + * Get various common parameters of a UnicodeError object. + * + * The caller is responsible to ensure that 'self' is a PyUnicodeErrorObject, + * although this condition is verified by this function on DEBUG builds. + * + * Return 0 on success and -1 on failure. + * + * Output parameters: + * + * obj A strong reference to the 'object' attribute. + * objlen The 'object' length. + * start The clipped 'start' attribute. + * end The clipped 'end' attribute. + * slen The length of the slice described by the clipped 'start' + * and 'end' values. It always lies in [0, objlen]. + * + * An output parameter can be NULL to indicate that + * the corresponding value does not need to be stored. + * + * Input parameter: + * + * as_bytes If true, the error's 'object' attribute must be a `bytes`, + * i.e. 'self' is a `UnicodeDecodeError` instance. Otherwise, + * the 'object' attribute must be a string. + * + * A TypeError is raised if the 'object' type is incompatible. + */ +int +_PyUnicodeError_GetParams(PyObject *self, + PyObject **obj, Py_ssize_t *objlen, + Py_ssize_t *start, Py_ssize_t *end, Py_ssize_t *slen, + int as_bytes) +{ + assert(self != NULL); + assert(as_bytes == 0 || as_bytes == 1); + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); + PyObject *r = as_unicode_error_attribute(exc->object, "object", as_bytes); + if (r == NULL) { return -1; - Py_XSETREF(*attr, obj); + } + + Py_ssize_t n = as_bytes ? PyBytes_GET_SIZE(r) : PyUnicode_GET_LENGTH(r); + if (objlen != NULL) { + *objlen = n; + } + + Py_ssize_t start_value = -1; + if (start != NULL || slen != NULL) { + start_value = unicode_error_adjust_start(exc->start, n); + } + if (start != NULL) { + assert_adjusted_unicode_error_start(start_value, n); + *start = start_value; + } + + Py_ssize_t end_value = -1; + if (end != NULL || slen != NULL) { + end_value = unicode_error_adjust_end(exc->end, n); + } + if (end != NULL) { + assert_adjusted_unicode_error_end(end_value, n); + *end = end_value; + } + + if (slen != NULL) { + *slen = unicode_error_adjust_len(start_value, end_value, n); + assert_adjusted_unicode_error_len(*slen, n); + } + + if (obj != NULL) { + *obj = r; + } + else { + Py_DECREF(r); + } return 0; } + +// --- PyUnicodeEncodeObject: 'encoding' getters ------------------------------ +// Note: PyUnicodeTranslateError does not have an 'encoding' attribute. + PyObject * -PyUnicodeEncodeError_GetEncoding(PyObject *exc) +PyUnicodeEncodeError_GetEncoding(PyObject *self) { - return get_unicode(((PyUnicodeErrorObject *)exc)->encoding, "encoding"); + int rc = check_unicode_error_type(self, Py_UNICODE_ENCODE_ERROR_NAME); + return rc < 0 ? NULL : unicode_error_get_encoding_impl(self); } + PyObject * -PyUnicodeDecodeError_GetEncoding(PyObject *exc) +PyUnicodeDecodeError_GetEncoding(PyObject *self) { - return get_unicode(((PyUnicodeErrorObject *)exc)->encoding, "encoding"); + int rc = check_unicode_error_type(self, Py_UNICODE_DECODE_ERROR_NAME); + return rc < 0 ? NULL : unicode_error_get_encoding_impl(self); } + +// --- PyUnicodeEncodeObject: 'object' getters -------------------------------- + PyObject * -PyUnicodeEncodeError_GetObject(PyObject *exc) +PyUnicodeEncodeError_GetObject(PyObject *self) { - return get_unicode(((PyUnicodeErrorObject *)exc)->object, "object"); + int rc = check_unicode_error_type(self, Py_UNICODE_ENCODE_ERROR_NAME); + return rc < 0 ? NULL : unicode_error_get_object_impl(self, false); } + PyObject * -PyUnicodeDecodeError_GetObject(PyObject *exc) +PyUnicodeDecodeError_GetObject(PyObject *self) { - return get_string(((PyUnicodeErrorObject *)exc)->object, "object"); + int rc = check_unicode_error_type(self, Py_UNICODE_DECODE_ERROR_NAME); + return rc < 0 ? NULL : unicode_error_get_object_impl(self, true); } + PyObject * -PyUnicodeTranslateError_GetObject(PyObject *exc) +PyUnicodeTranslateError_GetObject(PyObject *self) +{ + int rc = check_unicode_error_type(self, Py_UNICODE_TRANSLATE_ERROR_NAME); + return rc < 0 ? NULL : unicode_error_get_object_impl(self, false); +} + + +// --- PyUnicodeEncodeObject: 'start' getters --------------------------------- + +/* + * Specialization of _PyUnicodeError_GetParams() for the 'start' attribute. + * + * The caller is responsible to ensure that 'self' is a PyUnicodeErrorObject, + * although this condition is verified by this function on DEBUG builds. + */ +static inline int +unicode_error_get_start_impl(PyObject *self, Py_ssize_t *start, int as_bytes) { - return get_unicode(((PyUnicodeErrorObject *)exc)->object, "object"); + assert(self != NULL); + return _PyUnicodeError_GetParams(self, NULL, NULL, + start, NULL, NULL, + as_bytes); } + int -PyUnicodeEncodeError_GetStart(PyObject *exc, Py_ssize_t *start) +PyUnicodeEncodeError_GetStart(PyObject *self, Py_ssize_t *start) { - Py_ssize_t size; - PyObject *obj = get_unicode(((PyUnicodeErrorObject *)exc)->object, - "object"); - if (!obj) - return -1; - *start = ((PyUnicodeErrorObject *)exc)->start; - size = PyUnicode_GET_LENGTH(obj); - if (*start<0) - *start = 0; /*XXX check for values <0*/ - if (*start>=size) - *start = size-1; - Py_DECREF(obj); - return 0; + int rc = check_unicode_error_type(self, Py_UNICODE_ENCODE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_get_start_impl(self, start, false); } int -PyUnicodeDecodeError_GetStart(PyObject *exc, Py_ssize_t *start) +PyUnicodeDecodeError_GetStart(PyObject *self, Py_ssize_t *start) { - Py_ssize_t size; - PyObject *obj = get_string(((PyUnicodeErrorObject *)exc)->object, "object"); - if (!obj) - return -1; - size = PyBytes_GET_SIZE(obj); - *start = ((PyUnicodeErrorObject *)exc)->start; - if (*start<0) - *start = 0; - if (*start>=size) - *start = size-1; - Py_DECREF(obj); - return 0; + int rc = check_unicode_error_type(self, Py_UNICODE_DECODE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_get_start_impl(self, start, true); } int -PyUnicodeTranslateError_GetStart(PyObject *exc, Py_ssize_t *start) +PyUnicodeTranslateError_GetStart(PyObject *self, Py_ssize_t *start) { - return PyUnicodeEncodeError_GetStart(exc, start); + int rc = check_unicode_error_type(self, Py_UNICODE_TRANSLATE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_get_start_impl(self, start, false); } +// --- PyUnicodeEncodeObject: 'start' setters --------------------------------- + int -PyUnicodeEncodeError_SetStart(PyObject *exc, Py_ssize_t start) +PyUnicodeEncodeError_SetStart(PyObject *self, Py_ssize_t start) { - ((PyUnicodeErrorObject *)exc)->start = start; - return 0; + int rc = check_unicode_error_type(self, Py_UNICODE_ENCODE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_set_start_impl(self, start); } int -PyUnicodeDecodeError_SetStart(PyObject *exc, Py_ssize_t start) +PyUnicodeDecodeError_SetStart(PyObject *self, Py_ssize_t start) { - ((PyUnicodeErrorObject *)exc)->start = start; - return 0; + int rc = check_unicode_error_type(self, Py_UNICODE_DECODE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_set_start_impl(self, start); } int -PyUnicodeTranslateError_SetStart(PyObject *exc, Py_ssize_t start) +PyUnicodeTranslateError_SetStart(PyObject *self, Py_ssize_t start) { - ((PyUnicodeErrorObject *)exc)->start = start; - return 0; + int rc = check_unicode_error_type(self, Py_UNICODE_TRANSLATE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_set_start_impl(self, start); +} + + +// --- PyUnicodeEncodeObject: 'end' getters ----------------------------------- + +/* + * Specialization of _PyUnicodeError_GetParams() for the 'end' attribute. + * + * The caller is responsible to ensure that 'self' is a PyUnicodeErrorObject, + * although this condition is verified by this function on DEBUG builds. + */ +static inline int +unicode_error_get_end_impl(PyObject *self, Py_ssize_t *end, int as_bytes) +{ + assert(self != NULL); + return _PyUnicodeError_GetParams(self, NULL, NULL, + NULL, end, NULL, + as_bytes); } int -PyUnicodeEncodeError_GetEnd(PyObject *exc, Py_ssize_t *end) +PyUnicodeEncodeError_GetEnd(PyObject *self, Py_ssize_t *end) { - Py_ssize_t size; - PyObject *obj = get_unicode(((PyUnicodeErrorObject *)exc)->object, - "object"); - if (!obj) - return -1; - *end = ((PyUnicodeErrorObject *)exc)->end; - size = PyUnicode_GET_LENGTH(obj); - if (*end<1) - *end = 1; - if (*end>size) - *end = size; - Py_DECREF(obj); - return 0; + int rc = check_unicode_error_type(self, Py_UNICODE_ENCODE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_get_end_impl(self, end, false); } int -PyUnicodeDecodeError_GetEnd(PyObject *exc, Py_ssize_t *end) +PyUnicodeDecodeError_GetEnd(PyObject *self, Py_ssize_t *end) { - Py_ssize_t size; - PyObject *obj = get_string(((PyUnicodeErrorObject *)exc)->object, "object"); - if (!obj) - return -1; - size = PyBytes_GET_SIZE(obj); - *end = ((PyUnicodeErrorObject *)exc)->end; - if (*end<1) - *end = 1; - if (*end>size) - *end = size; - Py_DECREF(obj); - return 0; + int rc = check_unicode_error_type(self, Py_UNICODE_DECODE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_get_end_impl(self, end, true); } int -PyUnicodeTranslateError_GetEnd(PyObject *exc, Py_ssize_t *end) +PyUnicodeTranslateError_GetEnd(PyObject *self, Py_ssize_t *end) { - return PyUnicodeEncodeError_GetEnd(exc, end); + int rc = check_unicode_error_type(self, Py_UNICODE_TRANSLATE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_get_end_impl(self, end, false); } +// --- PyUnicodeEncodeObject: 'end' setters ----------------------------------- + int -PyUnicodeEncodeError_SetEnd(PyObject *exc, Py_ssize_t end) +PyUnicodeEncodeError_SetEnd(PyObject *self, Py_ssize_t end) { - ((PyUnicodeErrorObject *)exc)->end = end; - return 0; + int rc = check_unicode_error_type(self, Py_UNICODE_ENCODE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_set_end_impl(self, end); } int -PyUnicodeDecodeError_SetEnd(PyObject *exc, Py_ssize_t end) +PyUnicodeDecodeError_SetEnd(PyObject *self, Py_ssize_t end) { - ((PyUnicodeErrorObject *)exc)->end = end; - return 0; + int rc = check_unicode_error_type(self, Py_UNICODE_DECODE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_set_end_impl(self, end); } int -PyUnicodeTranslateError_SetEnd(PyObject *exc, Py_ssize_t end) +PyUnicodeTranslateError_SetEnd(PyObject *self, Py_ssize_t end) { - ((PyUnicodeErrorObject *)exc)->end = end; - return 0; + int rc = check_unicode_error_type(self, Py_UNICODE_TRANSLATE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_set_end_impl(self, end); } + +// --- PyUnicodeEncodeObject: 'reason' getters -------------------------------- + PyObject * -PyUnicodeEncodeError_GetReason(PyObject *exc) +PyUnicodeEncodeError_GetReason(PyObject *self) { - return get_unicode(((PyUnicodeErrorObject *)exc)->reason, "reason"); + int rc = check_unicode_error_type(self, Py_UNICODE_ENCODE_ERROR_NAME); + return rc < 0 ? NULL : unicode_error_get_reason_impl(self); } PyObject * -PyUnicodeDecodeError_GetReason(PyObject *exc) +PyUnicodeDecodeError_GetReason(PyObject *self) { - return get_unicode(((PyUnicodeErrorObject *)exc)->reason, "reason"); + int rc = check_unicode_error_type(self, Py_UNICODE_DECODE_ERROR_NAME); + return rc < 0 ? NULL : unicode_error_get_reason_impl(self); } PyObject * -PyUnicodeTranslateError_GetReason(PyObject *exc) +PyUnicodeTranslateError_GetReason(PyObject *self) { - return get_unicode(((PyUnicodeErrorObject *)exc)->reason, "reason"); + int rc = check_unicode_error_type(self, Py_UNICODE_TRANSLATE_ERROR_NAME); + return rc < 0 ? NULL : unicode_error_get_reason_impl(self); } +// --- PyUnicodeEncodeObject: 'reason' setters -------------------------------- + int -PyUnicodeEncodeError_SetReason(PyObject *exc, const char *reason) +PyUnicodeEncodeError_SetReason(PyObject *self, const char *reason) { - return set_unicodefromstring(&((PyUnicodeErrorObject *)exc)->reason, - reason); + int rc = check_unicode_error_type(self, Py_UNICODE_ENCODE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_set_reason_impl(self, reason); } int -PyUnicodeDecodeError_SetReason(PyObject *exc, const char *reason) +PyUnicodeDecodeError_SetReason(PyObject *self, const char *reason) { - return set_unicodefromstring(&((PyUnicodeErrorObject *)exc)->reason, - reason); + int rc = check_unicode_error_type(self, Py_UNICODE_DECODE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_set_reason_impl(self, reason); } int -PyUnicodeTranslateError_SetReason(PyObject *exc, const char *reason) +PyUnicodeTranslateError_SetReason(PyObject *self, const char *reason) { - return set_unicodefromstring(&((PyUnicodeErrorObject *)exc)->reason, - reason); + int rc = check_unicode_error_type(self, Py_UNICODE_TRANSLATE_ERROR_NAME); + return rc < 0 ? -1 : unicode_error_set_reason_impl(self, reason); } static int -UnicodeError_clear(PyUnicodeErrorObject *self) +UnicodeError_clear(PyObject *self) { - Py_CLEAR(self->encoding); - Py_CLEAR(self->object); - Py_CLEAR(self->reason); + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); + Py_CLEAR(exc->encoding); + Py_CLEAR(exc->object); + Py_CLEAR(exc->reason); return BaseException_clear((PyBaseExceptionObject *)self); } static void -UnicodeError_dealloc(PyUnicodeErrorObject *self) +UnicodeError_dealloc(PyObject *self) { + PyTypeObject *type = Py_TYPE(self); _PyObject_GC_UNTRACK(self); - UnicodeError_clear(self); - Py_TYPE(self)->tp_free((PyObject *)self); + (void)UnicodeError_clear(self); + type->tp_free(self); } static int -UnicodeError_traverse(PyUnicodeErrorObject *self, visitproc visit, void *arg) +UnicodeError_traverse(PyObject *self, visitproc visit, void *arg) { - Py_VISIT(self->encoding); - Py_VISIT(self->object); - Py_VISIT(self->reason); + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); + Py_VISIT(exc->encoding); + Py_VISIT(exc->object); + Py_VISIT(exc->reason); return BaseException_traverse((PyBaseExceptionObject *)self, visit, arg); } @@ -2927,74 +3456,80 @@ static PyMemberDef UnicodeError_members[] = { static int UnicodeEncodeError_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyUnicodeErrorObject *err; - - if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) + if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) { return -1; + } - err = (PyUnicodeErrorObject *)self; - - Py_CLEAR(err->encoding); - Py_CLEAR(err->object); - Py_CLEAR(err->reason); + PyObject *encoding = NULL, *object = NULL, *reason = NULL; // borrowed + Py_ssize_t start = -1, end = -1; if (!PyArg_ParseTuple(args, "UUnnU", - &err->encoding, &err->object, - &err->start, &err->end, &err->reason)) { - err->encoding = err->object = err->reason = NULL; + &encoding, &object, &start, &end, &reason)) + { return -1; } - Py_INCREF(err->encoding); - Py_INCREF(err->object); - Py_INCREF(err->reason); - + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); + Py_XSETREF(exc->encoding, Py_NewRef(encoding)); + Py_XSETREF(exc->object, Py_NewRef(object)); + exc->start = start; + exc->end = end; + Py_XSETREF(exc->reason, Py_NewRef(reason)); return 0; } static PyObject * UnicodeEncodeError_str(PyObject *self) { - PyUnicodeErrorObject *uself = (PyUnicodeErrorObject *)self; + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); PyObject *result = NULL; PyObject *reason_str = NULL; PyObject *encoding_str = NULL; - if (!uself->object) + if (exc->object == NULL) { /* Not properly initialized. */ - return PyUnicode_FromString(""); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); + } /* Get reason and encoding as strings, which they might not be if they've been modified after we were constructed. */ - reason_str = PyObject_Str(uself->reason); - if (reason_str == NULL) + reason_str = PyObject_Str(exc->reason); + if (reason_str == NULL) { goto done; - encoding_str = PyObject_Str(uself->encoding); - if (encoding_str == NULL) + } + encoding_str = PyObject_Str(exc->encoding); + if (encoding_str == NULL) { goto done; + } + + Py_ssize_t len = PyUnicode_GET_LENGTH(exc->object); + Py_ssize_t start = exc->start, end = exc->end; - if (uself->start < PyUnicode_GET_LENGTH(uself->object) && uself->end == uself->start+1) { - Py_UCS4 badchar = PyUnicode_ReadChar(uself->object, uself->start); + if ((start >= 0 && start < len) && (end >= 0 && end <= len) && end == start + 1) { + Py_UCS4 badchar = PyUnicode_ReadChar(exc->object, start); const char *fmt; - if (badchar <= 0xff) + if (badchar <= 0xff) { fmt = "'%U' codec can't encode character '\\x%02x' in position %zd: %U"; - else if (badchar <= 0xffff) + } + else if (badchar <= 0xffff) { fmt = "'%U' codec can't encode character '\\u%04x' in position %zd: %U"; - else + } + else { fmt = "'%U' codec can't encode character '\\U%08x' in position %zd: %U"; + } result = PyUnicode_FromFormat( fmt, encoding_str, (int)badchar, - uself->start, + start, reason_str); } else { result = PyUnicode_FromFormat( "'%U' codec can't encode characters in position %zd-%zd: %U", encoding_str, - uself->start, - uself->end-1, + start, + end - 1, reason_str); } done: @@ -3025,84 +3560,87 @@ PyObject *PyExc_UnicodeEncodeError = (PyObject *)&_PyExc_UnicodeEncodeError; static int UnicodeDecodeError_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyUnicodeErrorObject *ude; - - if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) + if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) { return -1; + } - ude = (PyUnicodeErrorObject *)self; - - Py_CLEAR(ude->encoding); - Py_CLEAR(ude->object); - Py_CLEAR(ude->reason); + PyObject *encoding = NULL, *object = NULL, *reason = NULL; // borrowed + Py_ssize_t start = -1, end = -1; if (!PyArg_ParseTuple(args, "UOnnU", - &ude->encoding, &ude->object, - &ude->start, &ude->end, &ude->reason)) { - ude->encoding = ude->object = ude->reason = NULL; - return -1; + &encoding, &object, &start, &end, &reason)) + { + return -1; } - Py_INCREF(ude->encoding); - Py_INCREF(ude->object); - Py_INCREF(ude->reason); - - if (!PyBytes_Check(ude->object)) { + if (PyBytes_Check(object)) { + Py_INCREF(object); // make 'object' a strong reference + } + else { Py_buffer view; - if (PyObject_GetBuffer(ude->object, &view, PyBUF_SIMPLE) != 0) - goto error; - Py_XSETREF(ude->object, PyBytes_FromStringAndSize(view.buf, view.len)); + if (PyObject_GetBuffer(object, &view, PyBUF_SIMPLE) != 0) { + return -1; + } + // 'object' is borrowed, so we can re-use the variable + object = PyBytes_FromStringAndSize(view.buf, view.len); PyBuffer_Release(&view); - if (!ude->object) - goto error; + if (object == NULL) { + return -1; + } } - return 0; -error: - Py_CLEAR(ude->encoding); - Py_CLEAR(ude->object); - Py_CLEAR(ude->reason); - return -1; + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); + Py_XSETREF(exc->encoding, Py_NewRef(encoding)); + Py_XSETREF(exc->object, object /* already a strong reference */); + exc->start = start; + exc->end = end; + Py_XSETREF(exc->reason, Py_NewRef(reason)); + return 0; } static PyObject * UnicodeDecodeError_str(PyObject *self) { - PyUnicodeErrorObject *uself = (PyUnicodeErrorObject *)self; + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); PyObject *result = NULL; PyObject *reason_str = NULL; PyObject *encoding_str = NULL; - if (!uself->object) + if (exc->object == NULL) { /* Not properly initialized. */ - return PyUnicode_FromString(""); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); + } /* Get reason and encoding as strings, which they might not be if they've been modified after we were constructed. */ - reason_str = PyObject_Str(uself->reason); - if (reason_str == NULL) + reason_str = PyObject_Str(exc->reason); + if (reason_str == NULL) { goto done; - encoding_str = PyObject_Str(uself->encoding); - if (encoding_str == NULL) + } + encoding_str = PyObject_Str(exc->encoding); + if (encoding_str == NULL) { goto done; + } + + Py_ssize_t len = PyBytes_GET_SIZE(exc->object); + Py_ssize_t start = exc->start, end = exc->end; - if (uself->start < PyBytes_GET_SIZE(uself->object) && uself->end == uself->start+1) { - int byte = (int)(PyBytes_AS_STRING(((PyUnicodeErrorObject *)self)->object)[uself->start]&0xff); + if ((start >= 0 && start < len) && (end >= 0 && end <= len) && end == start + 1) { + int badbyte = (int)(PyBytes_AS_STRING(exc->object)[start] & 0xff); result = PyUnicode_FromFormat( "'%U' codec can't decode byte 0x%02x in position %zd: %U", encoding_str, - byte, - uself->start, + badbyte, + start, reason_str); } else { result = PyUnicode_FromFormat( "'%U' codec can't decode bytes in position %zd-%zd: %U", encoding_str, - uself->start, - uself->end-1, - reason_str - ); + start, + end - 1, + reason_str); } done: Py_XDECREF(reason_str); @@ -3139,25 +3677,24 @@ PyUnicodeDecodeError_Create( */ static int -UnicodeTranslateError_init(PyUnicodeErrorObject *self, PyObject *args, - PyObject *kwds) +UnicodeTranslateError_init(PyObject *self, PyObject *args, PyObject *kwds) { - if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) + if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) { return -1; + } - Py_CLEAR(self->object); - Py_CLEAR(self->reason); + PyObject *object = NULL, *reason = NULL; // borrowed + Py_ssize_t start = -1, end = -1; - if (!PyArg_ParseTuple(args, "UnnU", - &self->object, - &self->start, &self->end, &self->reason)) { - self->object = self->reason = NULL; + if (!PyArg_ParseTuple(args, "UnnU", &object, &start, &end, &reason)) { return -1; } - Py_INCREF(self->object); - Py_INCREF(self->reason); - + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); + Py_XSETREF(exc->object, Py_NewRef(object)); + exc->start = start; + exc->end = end; + Py_XSETREF(exc->reason, Py_NewRef(reason)); return 0; } @@ -3165,42 +3702,49 @@ UnicodeTranslateError_init(PyUnicodeErrorObject *self, PyObject *args, static PyObject * UnicodeTranslateError_str(PyObject *self) { - PyUnicodeErrorObject *uself = (PyUnicodeErrorObject *)self; + PyUnicodeErrorObject *exc = PyUnicodeError_CAST(self); PyObject *result = NULL; PyObject *reason_str = NULL; - if (!uself->object) + if (exc->object == NULL) { /* Not properly initialized. */ - return PyUnicode_FromString(""); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); + } /* Get reason as a string, which it might not be if it's been modified after we were constructed. */ - reason_str = PyObject_Str(uself->reason); - if (reason_str == NULL) + reason_str = PyObject_Str(exc->reason); + if (reason_str == NULL) { goto done; + } + + Py_ssize_t len = PyUnicode_GET_LENGTH(exc->object); + Py_ssize_t start = exc->start, end = exc->end; - if (uself->start < PyUnicode_GET_LENGTH(uself->object) && uself->end == uself->start+1) { - Py_UCS4 badchar = PyUnicode_ReadChar(uself->object, uself->start); + if ((start >= 0 && start < len) && (end >= 0 && end <= len) && end == start + 1) { + Py_UCS4 badchar = PyUnicode_ReadChar(exc->object, start); const char *fmt; - if (badchar <= 0xff) + if (badchar <= 0xff) { fmt = "can't translate character '\\x%02x' in position %zd: %U"; - else if (badchar <= 0xffff) + } + else if (badchar <= 0xffff) { fmt = "can't translate character '\\u%04x' in position %zd: %U"; - else + } + else { fmt = "can't translate character '\\U%08x' in position %zd: %U"; + } result = PyUnicode_FromFormat( fmt, (int)badchar, - uself->start, - reason_str - ); - } else { + start, + reason_str); + } + else { result = PyUnicode_FromFormat( "can't translate characters in position %zd-%zd: %U", - uself->start, - uself->end-1, - reason_str - ); + start, + end - 1, + reason_str); } done: Py_XDECREF(reason_str); @@ -3248,7 +3792,7 @@ SimpleExtendsException(PyExc_Exception, ArithmeticError, * FloatingPointError extends ArithmeticError */ SimpleExtendsException(PyExc_ArithmeticError, FloatingPointError, - "Floating point operation failed."); + "Floating-point operation failed."); /* @@ -3348,8 +3892,9 @@ _PyErr_NoMemory(PyThreadState *tstate) } static void -MemoryError_dealloc(PyBaseExceptionObject *self) +MemoryError_dealloc(PyObject *obj) { + PyBaseExceptionObject *self = (PyBaseExceptionObject *)obj; _PyObject_GC_UNTRACK(self); BaseException_clear(self); @@ -3408,7 +3953,7 @@ PyTypeObject _PyExc_MemoryError = { PyVarObject_HEAD_INIT(NULL, 0) "MemoryError", sizeof(PyBaseExceptionObject), - 0, (destructor)MemoryError_dealloc, 0, 0, 0, 0, 0, 0, 0, + 0, MemoryError_dealloc, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, PyDoc_STR("Out of memory."), (traverseproc)BaseException_traverse, @@ -3640,7 +4185,7 @@ static struct static_exception static_exceptions[] = { // Level 4: Other subclasses ITEM(IndentationError), // base: SyntaxError(Exception) - ITEM(IncompleteInputError), // base: SyntaxError(Exception) + {&_PyExc_IncompleteInputError, "_IncompleteInputError"}, // base: SyntaxError(Exception) ITEM(IndexError), // base: LookupError(Exception) ITEM(KeyError), // base: LookupError(Exception) ITEM(ModuleNotFoundError), // base: ImportError(Exception) @@ -3675,6 +4220,11 @@ _PyExc_InitTypes(PyInterpreterState *interp) if (_PyStaticType_InitBuiltin(interp, exc) < 0) { return -1; } + if (exc->tp_new == BaseException_new + && exc->tp_init == (initproc)BaseException_init) + { + exc->tp_vectorcall = BaseException_vectorcall; + } } return 0; } @@ -3685,7 +4235,7 @@ _PyExc_FiniTypes(PyInterpreterState *interp) { for (Py_ssize_t i=Py_ARRAY_LENGTH(static_exceptions) - 1; i >= 0; i--) { PyTypeObject *exc = static_exceptions[i].exc; - _PyStaticType_Dealloc(interp, exc); + _PyStaticType_FiniBuiltin(interp, exc); } } diff --git a/Objects/fileobject.c b/Objects/fileobject.c index e30ab952dff571..c377d1bb28b56f 100644 --- a/Objects/fileobject.c +++ b/Objects/fileobject.c @@ -80,13 +80,7 @@ PyFile_GetLine(PyObject *f, int n) "EOF when reading a line"); } else if (s[len-1] == '\n') { - if (Py_REFCNT(result) == 1) - _PyBytes_Resize(&result, len-1); - else { - PyObject *v; - v = PyBytes_FromStringAndSize(s, len-1); - Py_SETREF(result, v); - } + (void) _PyBytes_Resize(&result, len-1); } } if (n < 0 && result != NULL && PyUnicode_Check(result)) { @@ -468,7 +462,7 @@ PyTypeObject PyStdPrinter_Type = { 0, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ 0, /* tp_new */ - PyObject_Del, /* tp_free */ + PyObject_Free, /* tp_free */ }; diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 37d2d312a6a0b7..7ca43033d722ab 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -7,8 +7,8 @@ #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_dtoa.h" // _Py_dg_dtoa() #include "pycore_floatobject.h" // _PyFloat_FormatAdvancedWriter() +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_initconfig.h" // _PyStatus_OK() -#include "pycore_interp.h" // _Py_float_freelist #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_Init(), _PyDebugAllocatorStats() @@ -26,16 +26,6 @@ class float "PyObject *" "&PyFloat_Type" #include "clinic/floatobject.c.h" -#ifdef WITH_FREELISTS -static struct _Py_float_freelist * -get_float_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - assert(freelists != NULL); - return &freelists->floats; -} -#endif - double PyFloat_GetMax(void) @@ -98,10 +88,18 @@ PyFloat_GetInfo(void) return NULL; } -#define SetIntFlag(flag) \ - PyStructSequence_SET_ITEM(floatinfo, pos++, PyLong_FromLong(flag)) -#define SetDblFlag(flag) \ - PyStructSequence_SET_ITEM(floatinfo, pos++, PyFloat_FromDouble(flag)) +#define SetFlag(CALL) \ + do { \ + PyObject *flag = (CALL); \ + if (flag == NULL) { \ + Py_CLEAR(floatinfo); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(floatinfo, pos++, flag); \ + } while (0) + +#define SetIntFlag(FLAG) SetFlag(PyLong_FromLong((FLAG))) +#define SetDblFlag(FLAG) SetFlag(PyFloat_FromDouble((FLAG))) SetDblFlag(DBL_MAX); SetIntFlag(DBL_MAX_EXP); @@ -116,39 +114,61 @@ PyFloat_GetInfo(void) SetIntFlag(FLT_ROUNDS); #undef SetIntFlag #undef SetDblFlag +#undef SetFlag - if (PyErr_Occurred()) { - Py_CLEAR(floatinfo); - return NULL; - } return floatinfo; } PyObject * PyFloat_FromDouble(double fval) { - PyFloatObject *op; -#ifdef WITH_FREELISTS - struct _Py_float_freelist *float_freelist = get_float_freelist(); - op = float_freelist->items; - if (op != NULL) { - float_freelist->items = (PyFloatObject *) Py_TYPE(op); - float_freelist->numfree--; - OBJECT_STAT_INC(from_freelist); - } - else -#endif - { + PyFloatObject *op = _Py_FREELIST_POP(PyFloatObject, floats); + if (op == NULL) { op = PyObject_Malloc(sizeof(PyFloatObject)); if (!op) { return PyErr_NoMemory(); } + _PyObject_Init((PyObject*)op, &PyFloat_Type); } - _PyObject_Init((PyObject*)op, &PyFloat_Type); op->ob_fval = fval; return (PyObject *) op; } +#ifdef Py_GIL_DISABLED + +PyObject *_PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyStackRef right, double value) +{ + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + return PyFloat_FromDouble(value); +} + +#else // Py_GIL_DISABLED + +PyObject *_PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyStackRef right, double value) +{ + PyObject *left_o = PyStackRef_AsPyObjectSteal(left); + PyObject *right_o = PyStackRef_AsPyObjectSteal(right); + if (Py_REFCNT(left_o) == 1) { + ((PyFloatObject *)left_o)->ob_fval = value; + _Py_DECREF_SPECIALIZED(right_o, _PyFloat_ExactDealloc); + return left_o; + } + else if (Py_REFCNT(right_o) == 1) { + ((PyFloatObject *)right_o)->ob_fval = value; + _Py_DECREF_NO_DEALLOC(left_o); + return right_o; + } + else { + PyObject *result = PyFloat_FromDouble(value); + _Py_DECREF_NO_DEALLOC(left_o); + _Py_DECREF_NO_DEALLOC(right_o); + return result; + } +} + +#endif // Py_GIL_DISABLED + static PyObject * float_from_string_inner(const char *s, Py_ssize_t len, void *obj) { @@ -243,35 +263,17 @@ void _PyFloat_ExactDealloc(PyObject *obj) { assert(PyFloat_CheckExact(obj)); - PyFloatObject *op = (PyFloatObject *)obj; -#ifdef WITH_FREELISTS - struct _Py_float_freelist *float_freelist = get_float_freelist(); - if (float_freelist->numfree >= PyFloat_MAXFREELIST || float_freelist->numfree < 0) { - PyObject_Free(op); - return; - } - float_freelist->numfree++; - Py_SET_TYPE(op, (PyTypeObject *)float_freelist->items); - float_freelist->items = op; - OBJECT_STAT_INC(to_freelist); -#else - PyObject_Free(op); -#endif + _Py_FREELIST_FREE(floats, obj, PyObject_Free); } static void float_dealloc(PyObject *op) { assert(PyFloat_Check(op)); -#ifdef WITH_FREELISTS - if (PyFloat_CheckExact(op)) { + if (PyFloat_CheckExact(op)) _PyFloat_ExactDealloc(op); - } else -#endif - { Py_TYPE(op)->tp_free(op); - } } double @@ -339,16 +341,16 @@ PyFloat_AsDouble(PyObject *op) obj is not of float or int type, Py_NotImplemented is incref'ed, stored in obj, and returned from the function invoking this macro. */ -#define CONVERT_TO_DOUBLE(obj, dbl) \ - if (PyFloat_Check(obj)) \ - dbl = PyFloat_AS_DOUBLE(obj); \ - else if (convert_to_double(&(obj), &(dbl)) < 0) \ +#define CONVERT_TO_DOUBLE(obj, dbl) \ + if (PyFloat_Check(obj)) \ + dbl = PyFloat_AS_DOUBLE(obj); \ + else if (_Py_convert_int_to_double(&(obj), &(dbl)) < 0) \ return obj; /* Methods */ -static int -convert_to_double(PyObject **v, double *dbl) +int +_Py_convert_int_to_double(PyObject **v, double *dbl) { PyObject *obj = *v; @@ -413,7 +415,7 @@ float_richcompare(PyObject *v, PyObject *w, int op) if (PyFloat_Check(w)) j = PyFloat_AS_DOUBLE(w); - else if (!Py_IS_FINITE(i)) { + else if (!isfinite(i)) { if (PyLong_Check(w)) /* If i is an infinity, its magnitude exceeds any * finite integer, so it doesn't matter which int we @@ -426,10 +428,10 @@ float_richcompare(PyObject *v, PyObject *w, int op) else if (PyLong_Check(w)) { int vsign = i == 0.0 ? 0 : i < 0.0 ? -1 : 1; - int wsign = _PyLong_Sign(w); - size_t nbits; + int wsign; int exponent; + (void)PyLong_GetSign(w, &wsign); if (vsign != wsign) { /* Magnitudes are irrelevant -- the signs alone * determine the outcome. @@ -440,20 +442,22 @@ float_richcompare(PyObject *v, PyObject *w, int op) } /* The signs are the same. */ /* Convert w to a double if it fits. In particular, 0 fits. */ - nbits = _PyLong_NumBits(w); - if (nbits == (size_t)-1 && PyErr_Occurred()) { - /* This long is so large that size_t isn't big enough - * to hold the # of bits. Replace with little doubles + int64_t nbits64 = _PyLong_NumBits(w); + assert(nbits64 >= 0); + assert(!PyErr_Occurred()); + if (nbits64 > DBL_MAX_EXP) { + /* This Python integer is larger than any finite C double. + * Replace with little doubles * that give the same outcome -- w is so large that * its magnitude must exceed the magnitude of any * finite float. */ - PyErr_Clear(); i = (double)vsign; assert(wsign != 0); j = wsign * 2.0; goto Compare; } + int nbits = (int)nbits64; if (nbits <= 48) { j = PyLong_AsDouble(w); /* It's impossible that <= 48 bits overflowed. */ @@ -477,12 +481,12 @@ float_richcompare(PyObject *v, PyObject *w, int op) /* exponent is the # of bits in v before the radix point; * we know that nbits (the # of bits in w) > 48 at this point */ - if (exponent < 0 || (size_t)exponent < nbits) { + if (exponent < nbits) { i = 1.0; j = 2.0; goto Compare; } - if ((size_t)exponent > nbits) { + if (exponent > nbits) { i = 2.0; j = 1.0; goto Compare; @@ -618,7 +622,7 @@ float_div(PyObject *v, PyObject *w) CONVERT_TO_DOUBLE(w, b); if (b == 0.0) { PyErr_SetString(PyExc_ZeroDivisionError, - "float division by zero"); + "division by zero"); return NULL; } a = a / b; @@ -634,7 +638,7 @@ float_rem(PyObject *v, PyObject *w) CONVERT_TO_DOUBLE(w, wx); if (wx == 0.0) { PyErr_SetString(PyExc_ZeroDivisionError, - "float modulo by zero"); + "division by zero"); return NULL; } mod = fmod(vx, wx); @@ -699,7 +703,7 @@ float_divmod(PyObject *v, PyObject *w) CONVERT_TO_DOUBLE(v, vx); CONVERT_TO_DOUBLE(w, wx); if (wx == 0.0) { - PyErr_SetString(PyExc_ZeroDivisionError, "float divmod()"); + PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); return NULL; } _float_div_mod(vx, wx, &floordiv, &mod); @@ -714,7 +718,7 @@ float_floor_div(PyObject *v, PyObject *w) CONVERT_TO_DOUBLE(v, vx); CONVERT_TO_DOUBLE(w, wx); if (wx == 0.0) { - PyErr_SetString(PyExc_ZeroDivisionError, "float floor division by zero"); + PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); return NULL; } _float_div_mod(vx, wx, &floordiv, &mod); @@ -744,13 +748,13 @@ float_pow(PyObject *v, PyObject *w, PyObject *z) if (iw == 0) { /* v**0 is 1, even 0**0 */ return PyFloat_FromDouble(1.0); } - if (Py_IS_NAN(iv)) { /* nan**w = nan, unless w == 0 */ + if (isnan(iv)) { /* nan**w = nan, unless w == 0 */ return PyFloat_FromDouble(iv); } - if (Py_IS_NAN(iw)) { /* v**nan = nan, unless v == 1; 1**nan = 1 */ + if (isnan(iw)) { /* v**nan = nan, unless v == 1; 1**nan = 1 */ return PyFloat_FromDouble(iv == 1.0 ? 1.0 : iw); } - if (Py_IS_INFINITY(iw)) { + if (isinf(iw)) { /* v**inf is: 0.0 if abs(v) < 1; 1.0 if abs(v) == 1; inf if * abs(v) > 1 (including case where v infinite) * @@ -765,7 +769,7 @@ float_pow(PyObject *v, PyObject *w, PyObject *z) else return PyFloat_FromDouble(0.0); } - if (Py_IS_INFINITY(iv)) { + if (isinf(iv)) { /* (+-inf)**w is: inf for w positive, 0 for w negative; in * both cases, we need to add the appropriate sign if w is * an odd integer. @@ -783,8 +787,7 @@ float_pow(PyObject *v, PyObject *w, PyObject *z) int iw_is_odd = DOUBLE_IS_ODD_INTEGER(iw); if (iw < 0.0) { PyErr_SetString(PyExc_ZeroDivisionError, - "0.0 cannot be raised to a " - "negative power"); + "zero to a negative power"); return NULL; } /* use correct sign if iw is odd */ @@ -880,7 +883,7 @@ float_is_integer_impl(PyObject *self) if (x == -1.0 && PyErr_Occurred()) return NULL; - if (!Py_IS_FINITE(x)) + if (!isfinite(x)) Py_RETURN_FALSE; errno = 0; o = (floor(x) == x) ? Py_True : Py_False; @@ -1016,7 +1019,7 @@ double_round(double x, int ndigits) { } y = (x*pow1)*pow2; /* if y overflows, then rounded value is exactly x */ - if (!Py_IS_FINITE(y)) + if (!isfinite(y)) return PyFloat_FromDouble(x); } else { @@ -1036,7 +1039,7 @@ double_round(double x, int ndigits) { z *= pow1; /* if computation resulted in overflow, raise OverflowError */ - if (!Py_IS_FINITE(z)) { + if (!isfinite(z)) { PyErr_SetString(PyExc_OverflowError, "overflow occurred during round"); return NULL; @@ -1084,7 +1087,7 @@ float___round___impl(PyObject *self, PyObject *o_ndigits) return NULL; /* nans and infinities round to themselves */ - if (!Py_IS_FINITE(x)) + if (!isfinite(x)) return PyFloat_FromDouble(x); /* Deal with extreme values for ndigits. For ndigits > NDIGITS_MAX, x @@ -1138,69 +1141,39 @@ char_from_hex(int x) return Py_hexdigits[x]; } +/* This table maps characters to their hexadecimal values, only + * works with encodings whose lower half is ASCII (like UTF-8). + * '0' maps to 0, ..., '9' maps to 9. + * 'a' and 'A' map to 10, ..., 'f' and 'F' map to 15. + * All other indices map to -1. + */ +static const int +_CHAR_TO_HEX[256] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, +}; + +/* Convert a character to its hexadecimal value, or -1 if it's not a + * valid hexadecimal character, only works with encodings whose lower + * half is ASCII (like UTF-8). + */ static int -hex_from_char(char c) { - int x; - switch(c) { - case '0': - x = 0; - break; - case '1': - x = 1; - break; - case '2': - x = 2; - break; - case '3': - x = 3; - break; - case '4': - x = 4; - break; - case '5': - x = 5; - break; - case '6': - x = 6; - break; - case '7': - x = 7; - break; - case '8': - x = 8; - break; - case '9': - x = 9; - break; - case 'a': - case 'A': - x = 10; - break; - case 'b': - case 'B': - x = 11; - break; - case 'c': - case 'C': - x = 12; - break; - case 'd': - case 'D': - x = 13; - break; - case 'e': - case 'E': - x = 14; - break; - case 'f': - case 'F': - x = 15; - break; - default: - x = -1; - break; - } - return x; +hex_from_char(unsigned char c) { + return _CHAR_TO_HEX[c]; } /* convert a float to a hexadecimal string */ @@ -1232,7 +1205,7 @@ float_hex_impl(PyObject *self) CONVERT_TO_DOUBLE(self, x); - if (Py_IS_NAN(x) || Py_IS_INFINITY(x)) + if (isnan(x) || isinf(x)) return float_repr((PyFloatObject *)self); if (x == 0.0) { @@ -1565,12 +1538,12 @@ float_as_integer_ratio_impl(PyObject *self) CONVERT_TO_DOUBLE(self, self_double); - if (Py_IS_INFINITY(self_double)) { + if (isinf(self_double)) { PyErr_SetString(PyExc_OverflowError, "cannot convert Infinity to integer ratio"); return NULL; } - if (Py_IS_NAN(self_double)) { + if (isnan(self_double)) { PyErr_SetString(PyExc_ValueError, "cannot convert NaN to integer ratio"); return NULL; @@ -1628,12 +1601,12 @@ float.__new__ as float_new x: object(c_default="NULL") = 0 / -Convert a string or number to a floating point number, if possible. +Convert a string or number to a floating-point number, if possible. [clinic start generated code]*/ static PyObject * float_new_impl(PyTypeObject *type, PyObject *x) -/*[clinic end generated code: output=ccf1e8dc460ba6ba input=f43661b7de03e9d8]*/ +/*[clinic end generated code: output=ccf1e8dc460ba6ba input=55909f888aa0c8a6]*/ { if (type != &PyFloat_Type) { if (x == NULL) { @@ -1695,6 +1668,36 @@ float_vectorcall(PyObject *type, PyObject * const*args, } +/*[clinic input] +@classmethod +float.from_number + + number: object + / + +Convert real number to a floating-point number. +[clinic start generated code]*/ + +static PyObject * +float_from_number(PyTypeObject *type, PyObject *number) +/*[clinic end generated code: output=bbcf05529fe907a3 input=1f8424d9bc11866a]*/ +{ + if (PyFloat_CheckExact(number) && type == &PyFloat_Type) { + Py_INCREF(number); + return number; + } + double x = PyFloat_AsDouble(number); + if (x == -1.0 && PyErr_Occurred()) { + return NULL; + } + PyObject *result = PyFloat_FromDouble(x); + if (type != &PyFloat_Type && result != NULL) { + Py_SETREF(result, PyObject_CallOneArg((PyObject *)type, result)); + } + return result; +} + + /*[clinic input] float.__getnewargs__ [clinic start generated code]*/ @@ -1729,13 +1732,13 @@ You probably don't want to use this function. It exists mainly to be used in Python's test suite. This function returns whichever of 'unknown', 'IEEE, big-endian' or 'IEEE, -little-endian' best describes the format of floating point numbers used by the +little-endian' best describes the format of floating-point numbers used by the C type named by typestr. [clinic start generated code]*/ static PyObject * float___getformat___impl(PyTypeObject *type, const char *typestr) -/*[clinic end generated code: output=2bfb987228cc9628 input=d5a52600f835ad67]*/ +/*[clinic end generated code: output=2bfb987228cc9628 input=90d5e246409a246e]*/ { float_format_type r; @@ -1808,6 +1811,7 @@ float___format___impl(PyObject *self, PyObject *format_spec) } static PyMethodDef float_methods[] = { + FLOAT_FROM_NUMBER_METHODDEF FLOAT_CONJUGATE_METHODDEF FLOAT___TRUNC___METHODDEF FLOAT___FLOOR___METHODDEF @@ -1913,6 +1917,7 @@ PyTypeObject PyFloat_Type = { 0, /* tp_alloc */ float_new, /* tp_new */ .tp_vectorcall = (vectorcallfunc)float_vectorcall, + .tp_version_tag = _Py_TYPE_VERSION_FLOAT, }; static void @@ -1921,7 +1926,7 @@ _init_global_state(void) float_format_type detected_double_format, detected_float_format; /* We attempt to determine if this machine is using IEEE - floating point formats by peering at the bits of some + floating-point formats by peering at the bits of some carefully chosen values. If it looks like we are on an IEEE platform, the float packing/unpacking routines can just copy bits, if not they resort to arithmetic & shifts @@ -1989,27 +1994,6 @@ _PyFloat_InitTypes(PyInterpreterState *interp) return _PyStatus_OK(); } -void -_PyFloat_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) -{ -#ifdef WITH_FREELISTS - struct _Py_float_freelist *state = &freelists->floats; - PyFloatObject *f = state->items; - while (f != NULL) { - PyFloatObject *next = (PyFloatObject*) Py_TYPE(f); - PyObject_Free(f); - f = next; - } - state->items = NULL; - if (is_finalization) { - state->numfree = -1; - } - else { - state->numfree = 0; - } -#endif -} - void _PyFloat_FiniType(PyInterpreterState *interp) { @@ -2020,12 +2004,10 @@ _PyFloat_FiniType(PyInterpreterState *interp) void _PyFloat_DebugMallocStats(FILE *out) { -#ifdef WITH_FREELISTS - struct _Py_float_freelist *float_freelist = get_float_freelist(); _PyDebugAllocatorStats(out, "free PyFloatObject", - float_freelist->numfree, sizeof(PyFloatObject)); -#endif + _Py_FREELIST_SIZE(floats), + sizeof(PyFloatObject)); } @@ -2055,12 +2037,12 @@ PyFloat_Pack2(double x, char *data, int le) e = 0; bits = 0; } - else if (Py_IS_INFINITY(x)) { + else if (isinf(x)) { sign = (x < 0.0); e = 0x1f; bits = 0; } - else if (Py_IS_NAN(x)) { + else if (isnan(x)) { /* There are 2046 distinct half-precision NaNs (1022 signaling and 1024 quiet), but there are only two quiet NaNs that don't arise by quieting a signaling NaN; we get those by setting the topmost bit @@ -2229,7 +2211,7 @@ PyFloat_Pack4(double x, char *data, int le) float y = (float)x; int i, incr = 1; - if (Py_IS_INFINITY(y) && !Py_IS_INFINITY(x)) + if (isinf(y) && !isinf(x)) goto Overflow; unsigned char s[sizeof(float)]; @@ -2410,7 +2392,7 @@ PyFloat_Unpack2(const char *data, int le) if (e == 0x1f) { if (f == 0) { /* Infinity */ - return sign ? -Py_HUGE_VAL : Py_HUGE_VAL; + return sign ? -Py_INFINITY : Py_INFINITY; } else { /* NaN */ diff --git a/Objects/frameobject.c b/Objects/frameobject.c index a914c61aac2fd5..15ec4b78235992 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1,11 +1,14 @@ /* Frame object implementation */ #include "Python.h" -#include "pycore_ceval.h" // _PyEval_BuiltinsFromGlobals() +#include "pycore_ceval.h" // _PyEval_SetOpcodeTrace() #include "pycore_code.h" // CO_FAST_LOCAL, etc. +#include "pycore_dict.h" // _PyDict_LoadBuiltinsFromGlobals() #include "pycore_function.h" // _PyFunction_FromConstructor() #include "pycore_moduleobject.h" // _PyModule_GetDict() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_cell.h" // PyCell_GetRef() PyCell_SetTakeRef() #include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches @@ -16,12 +19,842 @@ #define OFF(x) offsetof(PyFrameObject, x) + +// Returns borrowed reference or NULL +static PyObject * +framelocalsproxy_getval(_PyInterpreterFrame *frame, PyCodeObject *co, int i) +{ + _PyStackRef *fast = _PyFrame_GetLocalsArray(frame); + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + + PyObject *value = PyStackRef_AsPyObjectBorrow(fast[i]); + PyObject *cell = NULL; + + if (value == NULL) { + return NULL; + } + + if (kind == CO_FAST_FREE || kind & CO_FAST_CELL) { + // The cell was set when the frame was created from + // the function's closure. + assert(PyCell_Check(value)); + cell = value; + } + + if (cell != NULL) { + value = PyCell_GET(cell); + } + + if (value == NULL) { + return NULL; + } + + return value; +} + +static int +framelocalsproxy_getkeyindex(PyFrameObject *frame, PyObject* key, bool read) +{ + /* + * Returns -2 (!) if an error occurred; exception will be set. + * Returns the fast locals index of the key on success: + * - if read == true, returns the index if the value is not NULL + * - if read == false, returns the index if the value is not hidden + * Otherwise returns -1. + */ + + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + + // Ensure that the key is hashable. + Py_hash_t key_hash = PyObject_Hash(key); + if (key_hash == -1) { + return -2; + } + bool found = false; + + // We do 2 loops here because it's highly possible the key is interned + // and we can do a pointer comparison. + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + if (name == key) { + if (read) { + if (framelocalsproxy_getval(frame->f_frame, co, i) != NULL) { + return i; + } + } else { + if (!(_PyLocals_GetKind(co->co_localspluskinds, i) & CO_FAST_HIDDEN)) { + return i; + } + } + found = true; + } + } + if (found) { + // This is an attempt to read an unset local variable or + // write to a variable that is hidden from regular write operations + return -1; + } + // This is unlikely, but we need to make sure. This means the key + // is not interned. + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + Py_hash_t name_hash = PyObject_Hash(name); + assert(name_hash != -1); // keys are exact unicode + if (name_hash != key_hash) { + continue; + } + int same = PyObject_RichCompareBool(name, key, Py_EQ); + if (same < 0) { + return -2; + } + if (same) { + if (read) { + if (framelocalsproxy_getval(frame->f_frame, co, i) != NULL) { + return i; + } + } else { + if (!(_PyLocals_GetKind(co->co_localspluskinds, i) & CO_FAST_HIDDEN)) { + return i; + } + } + } + } + + return -1; +} + +static PyObject * +framelocalsproxy_getitem(PyObject *self, PyObject *key) +{ + PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject* co = _PyFrame_GetCode(frame->f_frame); + + int i = framelocalsproxy_getkeyindex(frame, key, true); + if (i == -2) { + return NULL; + } + if (i >= 0) { + PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); + assert(value != NULL); + return Py_NewRef(value); + } + + // Okay not in the fast locals, try extra locals + + PyObject *extra = frame->f_extra_locals; + if (extra != NULL) { + PyObject *value = PyDict_GetItem(extra, key); + if (value != NULL) { + return Py_NewRef(value); + } + } + + PyErr_Format(PyExc_KeyError, "local variable '%R' is not defined", key); + return NULL; +} + +static int +framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) +{ + /* Merge locals into fast locals */ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + _PyStackRef *fast = _PyFrame_GetLocalsArray(frame->f_frame); + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + + int i = framelocalsproxy_getkeyindex(frame, key, false); + if (i == -2) { + return -1; + } + if (i >= 0) { + if (value == NULL) { + PyErr_SetString(PyExc_ValueError, "cannot remove local variables from FrameLocalsProxy"); + return -1; + } + + _Py_Executors_InvalidateDependency(PyInterpreterState_Get(), co, 1); + + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + _PyStackRef oldvalue = fast[i]; + PyObject *cell = NULL; + if (kind == CO_FAST_FREE) { + // The cell was set when the frame was created from + // the function's closure. + assert(!PyStackRef_IsNull(oldvalue) && PyCell_Check(PyStackRef_AsPyObjectBorrow(oldvalue))); + cell = PyStackRef_AsPyObjectBorrow(oldvalue); + } else if (kind & CO_FAST_CELL && !PyStackRef_IsNull(oldvalue)) { + PyObject *as_obj = PyStackRef_AsPyObjectBorrow(oldvalue); + if (PyCell_Check(as_obj)) { + cell = as_obj; + } + } + if (cell != NULL) { + Py_XINCREF(value); + PyCell_SetTakeRef((PyCellObject *)cell, value); + } else if (value != PyStackRef_AsPyObjectBorrow(oldvalue)) { + PyStackRef_XCLOSE(fast[i]); + fast[i] = PyStackRef_FromPyObjectNew(value); + } + return 0; + } + + // Okay not in the fast locals, try extra locals + + PyObject *extra = frame->f_extra_locals; + + if (extra == NULL) { + if (value == NULL) { + _PyErr_SetKeyError(key); + return -1; + } + extra = PyDict_New(); + if (extra == NULL) { + return -1; + } + frame->f_extra_locals = extra; + } + + assert(PyDict_Check(extra)); + + if (value == NULL) { + return PyDict_DelItem(extra, key); + } else { + return PyDict_SetItem(extra, key, value); + } +} + +static int +framelocalsproxy_merge(PyObject* self, PyObject* other) +{ + if (!PyDict_Check(other) && !PyFrameLocalsProxy_Check(other)) { + return -1; + } + + PyObject *keys = PyMapping_Keys(other); + if (keys == NULL) { + return -1; + } + + PyObject *iter = PyObject_GetIter(keys); + Py_DECREF(keys); + if (iter == NULL) { + return -1; + } + + PyObject *key = NULL; + PyObject *value = NULL; + + while ((key = PyIter_Next(iter)) != NULL) { + value = PyObject_GetItem(other, key); + if (value == NULL) { + Py_DECREF(key); + Py_DECREF(iter); + return -1; + } + + if (framelocalsproxy_setitem(self, key, value) < 0) { + Py_DECREF(key); + Py_DECREF(value); + Py_DECREF(iter); + return -1; + } + + Py_DECREF(key); + Py_DECREF(value); + } + + Py_DECREF(iter); + + if (PyErr_Occurred()) { + return -1; + } + + return 0; +} + +static PyObject * +framelocalsproxy_keys(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + PyObject *names = PyList_New(0); + if (names == NULL) { + return NULL; + } + + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *val = framelocalsproxy_getval(frame->f_frame, co, i); + if (val) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + if (PyList_Append(names, name) < 0) { + Py_DECREF(names); + return NULL; + } + } + } + + // Iterate through the extra locals + if (frame->f_extra_locals) { + assert(PyDict_Check(frame->f_extra_locals)); + + Py_ssize_t i = 0; + PyObject *key = NULL; + PyObject *value = NULL; + + while (PyDict_Next(frame->f_extra_locals, &i, &key, &value)) { + if (PyList_Append(names, key) < 0) { + Py_DECREF(names); + return NULL; + } + } + } + + return names; +} + +static void +framelocalsproxy_dealloc(PyObject *self) +{ + PyObject_GC_UnTrack(self); + Py_CLEAR(((PyFrameLocalsProxyObject*)self)->frame); + Py_TYPE(self)->tp_free(self); +} + +static PyObject * +framelocalsproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + if (PyTuple_GET_SIZE(args) != 1) { + PyErr_Format(PyExc_TypeError, + "FrameLocalsProxy expected 1 argument, got %zd", + PyTuple_GET_SIZE(args)); + return NULL; + } + PyObject *item = PyTuple_GET_ITEM(args, 0); + + if (!PyFrame_Check(item)) { + PyErr_Format(PyExc_TypeError, "expect frame, not %T", item); + return NULL; + } + PyFrameObject *frame = (PyFrameObject*)item; + + if (kwds != NULL && PyDict_Size(kwds) != 0) { + PyErr_SetString(PyExc_TypeError, + "FrameLocalsProxy takes no keyword arguments"); + return 0; + } + + PyFrameLocalsProxyObject *self = (PyFrameLocalsProxyObject *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; + } + + ((PyFrameLocalsProxyObject*)self)->frame = (PyFrameObject*)Py_NewRef(frame); + + return (PyObject *)self; +} + +static int +framelocalsproxy_tp_clear(PyObject *self) +{ + Py_CLEAR(((PyFrameLocalsProxyObject*)self)->frame); + return 0; +} + +static int +framelocalsproxy_visit(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(((PyFrameLocalsProxyObject*)self)->frame); + return 0; +} + +static PyObject * +framelocalsproxy_iter(PyObject *self) +{ + PyObject* keys = framelocalsproxy_keys(self, NULL); + if (keys == NULL) { + return NULL; + } + + PyObject* iter = PyObject_GetIter(keys); + Py_XDECREF(keys); + + return iter; +} + +static PyObject * +framelocalsproxy_richcompare(PyObject *self, PyObject *other, int op) +{ + if (PyFrameLocalsProxy_Check(other)) { + bool result = ((PyFrameLocalsProxyObject*)self)->frame == ((PyFrameLocalsProxyObject*)other)->frame; + if (op == Py_EQ) { + return PyBool_FromLong(result); + } else if (op == Py_NE) { + return PyBool_FromLong(!result); + } + } else if (PyDict_Check(other)) { + PyObject *dct = PyDict_New(); + if (dct == NULL) { + return NULL; + } + + if (PyDict_Update(dct, self) < 0) { + Py_DECREF(dct); + return NULL; + } + + PyObject *result = PyObject_RichCompare(dct, other, op); + Py_DECREF(dct); + return result; + } + + Py_RETURN_NOTIMPLEMENTED; +} + +static PyObject * +framelocalsproxy_repr(PyObject *self) +{ + int i = Py_ReprEnter(self); + if (i != 0) { + return i > 0 ? PyUnicode_FromString("{...}") : NULL; + } + + PyObject *dct = PyDict_New(); + if (dct == NULL) { + Py_ReprLeave(self); + return NULL; + } + + if (PyDict_Update(dct, self) < 0) { + Py_DECREF(dct); + Py_ReprLeave(self); + return NULL; + } + + PyObject *repr = PyObject_Repr(dct); + Py_DECREF(dct); + + Py_ReprLeave(self); + + return repr; +} + +static PyObject* +framelocalsproxy_or(PyObject *self, PyObject *other) +{ + if (!PyDict_Check(other) && !PyFrameLocalsProxy_Check(other)) { + Py_RETURN_NOTIMPLEMENTED; + } + + PyObject *result = PyDict_New(); + if (result == NULL) { + return NULL; + } + + if (PyDict_Update(result, self) < 0) { + Py_DECREF(result); + return NULL; + } + + if (PyDict_Update(result, other) < 0) { + Py_DECREF(result); + return NULL; + } + + return result; +} + +static PyObject* +framelocalsproxy_inplace_or(PyObject *self, PyObject *other) +{ + if (!PyDict_Check(other) && !PyFrameLocalsProxy_Check(other)) { + Py_RETURN_NOTIMPLEMENTED; + } + + if (framelocalsproxy_merge(self, other) < 0) { + Py_RETURN_NOTIMPLEMENTED; + } + + return Py_NewRef(self); +} + +static PyObject* +framelocalsproxy_values(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + PyObject *values = PyList_New(0); + if (values == NULL) { + return NULL; + } + + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); + if (value) { + if (PyList_Append(values, value) < 0) { + Py_DECREF(values); + return NULL; + } + } + } + + // Iterate through the extra locals + if (frame->f_extra_locals) { + Py_ssize_t j = 0; + PyObject *key = NULL; + PyObject *value = NULL; + while (PyDict_Next(frame->f_extra_locals, &j, &key, &value)) { + if (PyList_Append(values, value) < 0) { + Py_DECREF(values); + return NULL; + } + } + } + + return values; +} + +static PyObject * +framelocalsproxy_items(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + PyObject *items = PyList_New(0); + if (items == NULL) { + return NULL; + } + + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); + + if (value) { + PyObject *pair = PyTuple_Pack(2, name, value); + if (pair == NULL) { + Py_DECREF(items); + return NULL; + } + + if (PyList_Append(items, pair) < 0) { + Py_DECREF(items); + Py_DECREF(pair); + return NULL; + } + + Py_DECREF(pair); + } + } + + // Iterate through the extra locals + if (frame->f_extra_locals) { + Py_ssize_t j = 0; + PyObject *key = NULL; + PyObject *value = NULL; + while (PyDict_Next(frame->f_extra_locals, &j, &key, &value)) { + PyObject *pair = PyTuple_Pack(2, key, value); + if (pair == NULL) { + Py_DECREF(items); + return NULL; + } + + if (PyList_Append(items, pair) < 0) { + Py_DECREF(items); + Py_DECREF(pair); + return NULL; + } + + Py_DECREF(pair); + } + } + + return items; +} + +static Py_ssize_t +framelocalsproxy_length(PyObject *self) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + Py_ssize_t size = 0; + + if (frame->f_extra_locals != NULL) { + assert(PyDict_Check(frame->f_extra_locals)); + size += PyDict_Size(frame->f_extra_locals); + } + + for (int i = 0; i < co->co_nlocalsplus; i++) { + if (framelocalsproxy_getval(frame->f_frame, co, i) != NULL) { + size++; + } + } + return size; +} + +static int +framelocalsproxy_contains(PyObject *self, PyObject *key) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + + int i = framelocalsproxy_getkeyindex(frame, key, true); + if (i == -2) { + return -1; + } + if (i >= 0) { + return 1; + } + + PyObject *extra = ((PyFrameObject*)frame)->f_extra_locals; + if (extra != NULL) { + return PyDict_Contains(extra, key); + } + + return 0; +} + +static PyObject* framelocalsproxy___contains__(PyObject *self, PyObject *key) +{ + int result = framelocalsproxy_contains(self, key); + if (result < 0) { + return NULL; + } + return PyBool_FromLong(result); +} + +static PyObject* +framelocalsproxy_update(PyObject *self, PyObject *other) +{ + if (framelocalsproxy_merge(self, other) < 0) { + PyErr_SetString(PyExc_TypeError, "update() argument must be dict or another FrameLocalsProxy"); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject* +framelocalsproxy_get(PyObject* self, PyObject *const *args, Py_ssize_t nargs) +{ + if (nargs < 1 || nargs > 2) { + PyErr_SetString(PyExc_TypeError, "get expected 1 or 2 arguments"); + return NULL; + } + + PyObject *key = args[0]; + PyObject *default_value = Py_None; + + if (nargs == 2) { + default_value = args[1]; + } + + PyObject *result = framelocalsproxy_getitem(self, key); + + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + return Py_XNewRef(default_value); + } + return NULL; + } + + return result; +} + +static PyObject* +framelocalsproxy_setdefault(PyObject* self, PyObject *const *args, Py_ssize_t nargs) +{ + if (nargs < 1 || nargs > 2) { + PyErr_SetString(PyExc_TypeError, "setdefault expected 1 or 2 arguments"); + return NULL; + } + + PyObject *key = args[0]; + PyObject *default_value = Py_None; + + if (nargs == 2) { + default_value = args[1]; + } + + PyObject *result = framelocalsproxy_getitem(self, key); + + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + if (framelocalsproxy_setitem(self, key, default_value) < 0) { + return NULL; + } + return Py_XNewRef(default_value); + } + return NULL; + } + + return result; +} + +static PyObject* +framelocalsproxy_pop(PyObject* self, PyObject *const *args, Py_ssize_t nargs) +{ + if (!_PyArg_CheckPositional("pop", nargs, 1, 2)) { + return NULL; + } + + PyObject *key = args[0]; + PyObject *default_value = NULL; + + if (nargs == 2) { + default_value = args[1]; + } + + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + + int i = framelocalsproxy_getkeyindex(frame, key, false); + if (i == -2) { + return NULL; + } + + if (i >= 0) { + PyErr_SetString(PyExc_ValueError, "cannot remove local variables from FrameLocalsProxy"); + return NULL; + } + + PyObject *result = NULL; + + if (frame->f_extra_locals == NULL) { + if (default_value != NULL) { + return Py_XNewRef(default_value); + } else { + _PyErr_SetKeyError(key); + return NULL; + } + } + + if (PyDict_Pop(frame->f_extra_locals, key, &result) < 0) { + return NULL; + } + + if (result == NULL) { + if (default_value != NULL) { + return Py_XNewRef(default_value); + } else { + _PyErr_SetKeyError(key); + return NULL; + } + } + + return result; +} + +static PyObject* +framelocalsproxy_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject* result = PyDict_New(); + + if (result == NULL) { + return NULL; + } + + if (PyDict_Update(result, self) < 0) { + Py_DECREF(result); + return NULL; + } + + return result; +} + +static PyObject* +framelocalsproxy_reversed(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyObject *result = framelocalsproxy_keys(self, NULL); + + if (result == NULL) { + return NULL; + } + + if (PyList_Reverse(result) < 0) { + Py_DECREF(result); + return NULL; + } + return result; +} + +static PyNumberMethods framelocalsproxy_as_number = { + .nb_or = framelocalsproxy_or, + .nb_inplace_or = framelocalsproxy_inplace_or, +}; + +static PySequenceMethods framelocalsproxy_as_sequence = { + .sq_contains = framelocalsproxy_contains, +}; + +static PyMappingMethods framelocalsproxy_as_mapping = { + framelocalsproxy_length, // mp_length + framelocalsproxy_getitem, // mp_subscript + framelocalsproxy_setitem, // mp_ass_subscript +}; + +static PyMethodDef framelocalsproxy_methods[] = { + {"__contains__", framelocalsproxy___contains__, METH_O | METH_COEXIST, + NULL}, + {"__getitem__", framelocalsproxy_getitem, METH_O | METH_COEXIST, + NULL}, + {"update", framelocalsproxy_update, METH_O, + NULL}, + {"__reversed__", _PyCFunction_CAST(framelocalsproxy_reversed), METH_NOARGS, + NULL}, + {"copy", _PyCFunction_CAST(framelocalsproxy_copy), METH_NOARGS, + NULL}, + {"keys", _PyCFunction_CAST(framelocalsproxy_keys), METH_NOARGS, + NULL}, + {"values", _PyCFunction_CAST(framelocalsproxy_values), METH_NOARGS, + NULL}, + {"items", _PyCFunction_CAST(framelocalsproxy_items), METH_NOARGS, + NULL}, + {"get", _PyCFunction_CAST(framelocalsproxy_get), METH_FASTCALL, + NULL}, + {"pop", _PyCFunction_CAST(framelocalsproxy_pop), METH_FASTCALL, + NULL}, + {"setdefault", _PyCFunction_CAST(framelocalsproxy_setdefault), METH_FASTCALL, + NULL}, + {NULL, NULL} /* sentinel */ +}; + +PyTypeObject PyFrameLocalsProxy_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "FrameLocalsProxy", + .tp_basicsize = sizeof(PyFrameLocalsProxyObject), + .tp_dealloc = (destructor)framelocalsproxy_dealloc, + .tp_repr = &framelocalsproxy_repr, + .tp_as_number = &framelocalsproxy_as_number, + .tp_as_sequence = &framelocalsproxy_as_sequence, + .tp_as_mapping = &framelocalsproxy_as_mapping, + .tp_getattro = PyObject_GenericGetAttr, + .tp_setattro = PyObject_GenericSetAttr, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_MAPPING, + .tp_traverse = framelocalsproxy_visit, + .tp_clear = framelocalsproxy_tp_clear, + .tp_richcompare = framelocalsproxy_richcompare, + .tp_iter = framelocalsproxy_iter, + .tp_methods = framelocalsproxy_methods, + .tp_alloc = PyType_GenericAlloc, + .tp_new = framelocalsproxy_new, + .tp_free = PyObject_GC_Del, +}; + +PyObject * +_PyFrameLocalsProxy_New(PyFrameObject *frame) +{ + PyObject* args = PyTuple_Pack(1, frame); + if (args == NULL) { + return NULL; + } + + PyObject* proxy = (PyObject*)framelocalsproxy_new(&PyFrameLocalsProxy_Type, args, NULL); + Py_DECREF(args); + return proxy; +} + static PyMemberDef frame_memberlist[] = { {"f_trace_lines", Py_T_BOOL, OFF(f_trace_lines), 0}, {NULL} /* Sentinel */ }; - static PyObject * frame_getlocals(PyFrameObject *f, void *closure) { @@ -30,23 +863,43 @@ frame_getlocals(PyFrameObject *f, void *closure) return NULL; } assert(!_PyFrame_IsIncomplete(f->f_frame)); - PyObject *locals = _PyFrame_GetLocals(f->f_frame, 1); - if (locals) { - f->f_fast_as_locals = 1; + + PyCodeObject *co = _PyFrame_GetCode(f->f_frame); + + if (!(co->co_flags & CO_OPTIMIZED) && !_PyFrame_HasHiddenLocals(f->f_frame)) { + if (f->f_frame->f_locals == NULL) { + // We found cases when f_locals is NULL for non-optimized code. + // We fill the f_locals with an empty dict to avoid crash until + // we find the root cause. + f->f_frame->f_locals = PyDict_New(); + if (f->f_frame->f_locals == NULL) { + return NULL; + } + } + return Py_NewRef(f->f_frame->f_locals); } - return locals; + + return _PyFrameLocalsProxy_New(f); } int PyFrame_GetLineNumber(PyFrameObject *f) { assert(f != NULL); - if (f->f_lineno != 0) { - return f->f_lineno; + if (f->f_lineno == -1) { + // We should calculate it once. If we can't get the line number, + // set f->f_lineno to 0. + f->f_lineno = PyUnstable_InterpreterFrame_GetLine(f->f_frame); + if (f->f_lineno < 0) { + f->f_lineno = 0; + return -1; + } } - else { - return PyUnstable_InterpreterFrame_GetLine(f->f_frame); + + if (f->f_lineno > 0) { + return f->f_lineno; } + return PyUnstable_InterpreterFrame_GetLine(f->f_frame); } static PyObject * @@ -291,9 +1144,7 @@ mark_stacks(PyCodeObject *code_obj, int len) if (co_code == NULL) { return NULL; } - _Py_CODEUNIT *code = (_Py_CODEUNIT *)PyBytes_AS_STRING(co_code); int64_t *stacks = PyMem_New(int64_t, len+1); - int i, j, opcode; if (stacks == NULL) { PyErr_NoMemory(); @@ -304,31 +1155,29 @@ mark_stacks(PyCodeObject *code_obj, int len) stacks[i] = UNINITIALIZED; } stacks[0] = EMPTY_STACK; - if (code_obj->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) - { - // Generators get sent None while starting: - stacks[0] = push_value(stacks[0], Object); - } int todo = 1; while (todo) { todo = 0; /* Scan instructions */ - for (i = 0; i < len;) { + for (int i = 0; i < len;) { + int j; int64_t next_stack = stacks[i]; - opcode = _Py_GetBaseOpcode(code_obj, i); + _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(code_obj, i); + int opcode = inst.op.code; int oparg = 0; while (opcode == EXTENDED_ARG) { - oparg = (oparg << 8) | code[i].op.arg; + oparg = (oparg << 8) | inst.op.arg; i++; - opcode = _Py_GetBaseOpcode(code_obj, i); + inst = _Py_GetBaseCodeUnit(code_obj, i); + opcode = inst.op.code; stacks[i] = next_stack; } + oparg = (oparg << 8) | inst.op.arg; int next_i = i + _PyOpcode_Caches[opcode] + 1; if (next_stack == UNINITIALIZED) { i = next_i; continue; } - oparg = (oparg << 8) | code[i].op.arg; switch (opcode) { case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: @@ -336,7 +1185,7 @@ mark_stacks(PyCodeObject *code_obj, int len) case POP_JUMP_IF_NOT_NONE: { int64_t target_stack; - int j = next_i + oparg; + j = next_i + oparg; assert(j < len); next_stack = pop_value(next_stack); target_stack = next_stack; @@ -401,8 +1250,6 @@ mark_stacks(PyCodeObject *code_obj, int len) assert(pop_value(next_stack) == EMPTY_STACK); assert(top_of_stack(next_stack) == Object); break; - case RETURN_CONST: - break; case RAISE_VARARGS: break; case RERAISE: @@ -592,25 +1439,11 @@ first_line_not_before(int *lines, int len, int line) return result; } -static bool -frame_is_cleared(PyFrameObject *frame) -{ - assert(!_PyFrame_IsIncomplete(frame->f_frame)); - if (frame->f_frame->stacktop == 0) { - return true; - } - if (frame->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { - PyGenObject *gen = _PyFrame_GetGenerator(frame->f_frame); - return gen->gi_frame_state == FRAME_CLEARED; - } - return false; -} - static bool frame_is_suspended(PyFrameObject *frame) { assert(!_PyFrame_IsIncomplete(frame->f_frame)); if (frame->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { - PyGenObject *gen = _PyFrame_GetGenerator(frame->f_frame); + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame->f_frame); return FRAME_STATE_SUSPENDED(gen->gi_frame_state); } return false; @@ -783,7 +1616,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore for (int i = 0; i < code->co_nlocalsplus; i++) { // Counting every unbound local is overly-cautious, but a full flow // analysis (like we do in the compiler) is probably too expensive: - unbound += f->f_frame->localsplus[i] == NULL; + unbound += PyStackRef_IsNull(f->f_frame->localsplus[i]); } if (unbound) { const char *e = "assigning None to %d unbound local%s"; @@ -794,8 +1627,8 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore // Do this in a second pass to avoid writing a bunch of Nones when // warnings are being treated as errors and the previous bit raises: for (int i = 0; i < code->co_nlocalsplus; i++) { - if (f->f_frame->localsplus[i] == NULL) { - f->f_frame->localsplus[i] = Py_NewRef(Py_None); + if (PyStackRef_IsNull(f->f_frame->localsplus[i])) { + f->f_frame->localsplus[i] = PyStackRef_None; unbound--; } } @@ -808,20 +1641,19 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore while (start_stack > best_stack) { if (top_of_stack(start_stack) == Except) { /* Pop exception stack as well as the evaluation stack */ - PyObject *exc = _PyFrame_StackPop(f->f_frame); + PyObject *exc = PyStackRef_AsPyObjectBorrow(_PyFrame_StackPop(f->f_frame)); assert(PyExceptionInstance_Check(exc) || exc == Py_None); PyThreadState *tstate = _PyThreadState_GET(); Py_XSETREF(tstate->exc_info->exc_value, exc == Py_None ? NULL : exc); } else { - PyObject *v = _PyFrame_StackPop(f->f_frame); - Py_XDECREF(v); + PyStackRef_XCLOSE(_PyFrame_StackPop(f->f_frame)); } start_stack = pop_value(start_stack); } /* Finally set the new lasti and return OK. */ f->f_lineno = 0; - f->f_frame->instr_ptr = _PyCode_CODE(code) + best_addr; + f->f_frame->instr_ptr = _PyFrame_GetBytecode(f->f_frame) + best_addr; return 0; } @@ -849,6 +1681,15 @@ frame_settrace(PyFrameObject *f, PyObject* v, void *closure) return 0; } +static PyObject * +frame_getgenerator(PyFrameObject *f, void *arg) { + if (f->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { + PyObject *gen = (PyObject *)_PyGen_GetGeneratorFromFrame(f->f_frame); + return Py_NewRef(gen); + } + Py_RETURN_NONE; +} + static PyGetSetDef frame_getsetlist[] = { {"f_back", (getter)frame_getback, NULL, NULL}, @@ -861,6 +1702,7 @@ static PyGetSetDef frame_getsetlist[] = { {"f_builtins", (getter)frame_getbuiltins, NULL, NULL}, {"f_code", (getter)frame_getcode, NULL, NULL}, {"f_trace_opcodes", (getter)frame_gettrace_opcodes, (setter)frame_settrace_opcodes, NULL}, + {"f_generator", (getter)frame_getgenerator, NULL, NULL}, {0} }; @@ -875,8 +1717,6 @@ frame_dealloc(PyFrameObject *f) } Py_TRASHCAN_BEGIN(f, frame_dealloc); - PyObject *co = NULL; - /* GH-106092: If f->f_frame was on the stack and we reached the maximum * nesting depth for deallocations, the trashcan may have delayed this * deallocation until after f->f_frame is freed. Avoid dereferencing @@ -885,20 +1725,21 @@ frame_dealloc(PyFrameObject *f) /* Kill all local variables including specials, if we own them */ if (f->f_frame == frame && frame->owner == FRAME_OWNED_BY_FRAME_OBJECT) { - /* Don't clear code object until the end */ - co = frame->f_executable; - frame->f_executable = NULL; - Py_CLEAR(frame->f_funcobj); + PyStackRef_CLEAR(frame->f_executable); + PyStackRef_CLEAR(frame->f_funcobj); Py_CLEAR(frame->f_locals); - PyObject **locals = _PyFrame_GetLocalsArray(frame); - for (int i = 0; i < frame->stacktop; i++) { - Py_CLEAR(locals[i]); + _PyStackRef *locals = _PyFrame_GetLocalsArray(frame); + _PyStackRef *sp = frame->stackpointer; + while (sp > locals) { + sp--; + PyStackRef_CLEAR(*sp); } } Py_CLEAR(f->f_back); Py_CLEAR(f->f_trace); + Py_CLEAR(f->f_extra_locals); + Py_CLEAR(f->f_locals_cache); PyObject_GC_Del(f); - Py_XDECREF(co); Py_TRASHCAN_END; } @@ -907,6 +1748,8 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg) { Py_VISIT(f->f_back); Py_VISIT(f->f_trace); + Py_VISIT(f->f_extra_locals); + Py_VISIT(f->f_locals_cache); if (f->f_frame->owner != FRAME_OWNED_BY_FRAME_OBJECT) { return 0; } @@ -918,14 +1761,18 @@ static int frame_tp_clear(PyFrameObject *f) { Py_CLEAR(f->f_trace); + Py_CLEAR(f->f_extra_locals); + Py_CLEAR(f->f_locals_cache); /* locals and stack */ - PyObject **locals = _PyFrame_GetLocalsArray(f->f_frame); - assert(f->f_frame->stacktop >= 0); - for (int i = 0; i < f->f_frame->stacktop; i++) { - Py_CLEAR(locals[i]); - } - f->f_frame->stacktop = 0; + _PyStackRef *locals = _PyFrame_GetLocalsArray(f->f_frame); + _PyStackRef *sp = f->f_frame->stackpointer; + assert(sp >= locals); + while (sp > locals) { + sp--; + PyStackRef_CLEAR(*sp); + } + f->f_frame->stackpointer = locals; Py_CLEAR(f->f_frame->f_locals); return 0; } @@ -934,7 +1781,7 @@ static PyObject * frame_clear(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) { if (f->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { - PyGenObject *gen = _PyFrame_GetGenerator(f->f_frame); + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(f->f_frame); if (gen->gi_frame_state == FRAME_EXECUTING) { goto running; } @@ -962,7 +1809,7 @@ frame_clear(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(clear__doc__, -"F.clear(): clear most references held by the frame"); +"F.clear(): clear all references held by the frame"); static PyObject * frame_sizeof(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) @@ -1032,12 +1879,12 @@ PyTypeObject PyFrame_Type = { }; static void -init_frame(_PyInterpreterFrame *frame, PyFunctionObject *func, PyObject *locals) +init_frame(PyThreadState *tstate, _PyInterpreterFrame *frame, + PyFunctionObject *func, PyObject *locals) { PyCodeObject *code = (PyCodeObject *)func->func_code; - _PyFrame_Initialize(frame, (PyFunctionObject*)Py_NewRef(func), - Py_XNewRef(locals), code, 0); - frame->previous = NULL; + _PyFrame_Initialize(tstate, frame, PyStackRef_FromPyObjectNew(func), + Py_XNewRef(locals), code, 0, NULL); } PyFrameObject* @@ -1053,8 +1900,9 @@ _PyFrame_New_NoTrack(PyCodeObject *code) f->f_trace = NULL; f->f_trace_lines = 1; f->f_trace_opcodes = 0; - f->f_fast_as_locals = 0; f->f_lineno = 0; + f->f_extra_locals = NULL; + f->f_locals_cache = NULL; return f; } @@ -1063,7 +1911,7 @@ PyFrameObject* PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals, PyObject *locals) { - PyObject *builtins = _PyEval_BuiltinsFromGlobals(tstate, globals); // borrowed ref + PyObject *builtins = _PyDict_LoadBuiltinsFromGlobals(globals); if (builtins == NULL) { return NULL; } @@ -1078,6 +1926,7 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, .fc_closure = NULL }; PyFunctionObject *func = _PyFunction_FromConstructor(&desc); + _Py_DECREF_BUILTINS(builtins); if (func == NULL) { return NULL; } @@ -1086,7 +1935,7 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, Py_DECREF(func); return NULL; } - init_frame((_PyInterpreterFrame *)f->_f_frame_data, func, locals); + init_frame(tstate, (_PyInterpreterFrame *)f->_f_frame_data, func, locals); f->f_frame = (_PyInterpreterFrame *)f->_f_frame_data; f->f_frame->owner = FRAME_OWNED_BY_FRAME_OBJECT; // This frame needs to be "complete", so pretend that the first RESUME ran: @@ -1097,32 +1946,6 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, return f; } -static int -_PyFrame_OpAlreadyRan(_PyInterpreterFrame *frame, int opcode, int oparg) -{ - // This only works when opcode is a non-quickened form: - assert(_PyOpcode_Deopt[opcode] == opcode); - int check_oparg = 0; - for (_Py_CODEUNIT *instruction = _PyCode_CODE(_PyFrame_GetCode(frame)); - instruction < frame->instr_ptr; instruction++) - { - int check_opcode = _PyOpcode_Deopt[instruction->op.code]; - check_oparg |= instruction->op.arg; - if (check_opcode == opcode && check_oparg == oparg) { - return 1; - } - if (check_opcode == EXTENDED_ARG) { - check_oparg <<= 8; - } - else { - check_oparg = 0; - } - instruction += _PyOpcode_Caches[check_opcode]; - } - return 0; -} - - // Initialize frame free variables if needed static void frame_init_get_vars(_PyInterpreterFrame *frame) @@ -1131,22 +1954,24 @@ frame_init_get_vars(_PyInterpreterFrame *frame) // here: PyCodeObject *co = _PyFrame_GetCode(frame); int lasti = _PyInterpreterFrame_LASTI(frame); - if (!(lasti < 0 && _PyCode_CODE(co)->op.code == COPY_FREE_VARS - && PyFunction_Check(frame->f_funcobj))) + if (!(lasti < 0 + && _PyFrame_GetBytecode(frame)->op.code == COPY_FREE_VARS + && PyStackRef_FunctionCheck(frame->f_funcobj))) { /* Free vars are initialized */ return; } /* Free vars have not been initialized -- Do that */ - PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; - int offset = PyCode_GetFirstFree(co); + PyFunctionObject *func = _PyFrame_GetFunction(frame); + PyObject *closure = func->func_closure; + int offset = PyUnstable_Code_GetFirstFree(co); for (int i = 0; i < co->co_nfreevars; ++i) { PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = Py_NewRef(o); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); } // COPY_FREE_VARS doesn't have inline CACHEs, either: - frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)); + frame->instr_ptr = _PyFrame_GetBytecode(frame); } @@ -1168,136 +1993,85 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, return 0; } - PyObject *value = frame->localsplus[i]; - if (frame->stacktop) { + PyObject *value = NULL; + if (frame->stackpointer == NULL || frame->stackpointer > frame->localsplus + i) { + value = PyStackRef_AsPyObjectBorrow(frame->localsplus[i]); if (kind & CO_FAST_FREE) { // The cell was set by COPY_FREE_VARS. assert(value != NULL && PyCell_Check(value)); - value = PyCell_GET(value); + value = PyCell_GetRef((PyCellObject *)value); } else if (kind & CO_FAST_CELL) { - // Note that no *_DEREF ops can happen before MAKE_CELL - // executes. So there's no need to duplicate the work - // that MAKE_CELL would otherwise do later, if it hasn't - // run yet. if (value != NULL) { - if (PyCell_Check(value) && - _PyFrame_OpAlreadyRan(frame, MAKE_CELL, i)) { - // (likely) MAKE_CELL must have executed already. - value = PyCell_GET(value); + if (PyCell_Check(value)) { + assert(!_PyFrame_IsIncomplete(frame)); + value = PyCell_GetRef((PyCellObject *)value); + } + else { + // (likely) Otherwise it is an arg (kind & CO_FAST_LOCAL), + // with the initial value set when the frame was created... + // (unlikely) ...or it was set via the f_locals proxy. + Py_INCREF(value); } - // (likely) Otherwise it is an arg (kind & CO_FAST_LOCAL), - // with the initial value set when the frame was created... - // (unlikely) ...or it was set to some initial value by - // an earlier call to PyFrame_LocalsToFast(). } } - } - else { - assert(value == NULL); + else { + Py_XINCREF(value); + } } *pvalue = value; return 1; } -PyObject * -_PyFrame_GetLocals(_PyInterpreterFrame *frame, int include_hidden) +bool +_PyFrame_HasHiddenLocals(_PyInterpreterFrame *frame) { - /* Merge fast locals into f->f_locals */ - PyObject *locals = frame->f_locals; - if (locals == NULL) { - locals = frame->f_locals = PyDict_New(); - if (locals == NULL) { - return NULL; - } - } - PyObject *hidden = NULL; - - /* If include_hidden, "hidden" fast locals (from inlined comprehensions in - module/class scopes) will be included in the returned dict, but not in - frame->f_locals; the returned dict will be a modified copy. Non-hidden - locals will still be updated in frame->f_locals. */ - if (include_hidden) { - hidden = PyDict_New(); - if (hidden == NULL) { - return NULL; - } - } - - frame_init_get_vars(frame); + /* + * This function returns if there are hidden locals introduced by PEP 709, + * which are the isolated fast locals for inline comprehensions + */ + PyCodeObject* co = _PyFrame_GetCode(frame); - PyCodeObject *co = _PyFrame_GetCode(frame); for (int i = 0; i < co->co_nlocalsplus; i++) { - PyObject *value; // borrowed reference - if (!frame_get_var(frame, co, i, &value)) { - continue; - } - - PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + if (kind & CO_FAST_HIDDEN) { - if (include_hidden && value != NULL) { - if (PyObject_SetItem(hidden, name, value) != 0) { - goto error; - } - } - continue; - } - if (value == NULL) { - if (PyObject_DelItem(locals, name) != 0) { - if (PyErr_ExceptionMatches(PyExc_KeyError)) { - PyErr_Clear(); - } - else { - goto error; - } - } - } - else { - if (PyObject_SetItem(locals, name, value) != 0) { - goto error; - } - } - } + PyObject* value = framelocalsproxy_getval(frame, co, i); - if (include_hidden && PyDict_Size(hidden)) { - PyObject *innerlocals = PyDict_New(); - if (innerlocals == NULL) { - goto error; - } - if (PyDict_Merge(innerlocals, locals, 1) != 0) { - Py_DECREF(innerlocals); - goto error; - } - if (PyDict_Merge(innerlocals, hidden, 1) != 0) { - Py_DECREF(innerlocals); - goto error; + if (value != NULL) { + return true; + } } - locals = innerlocals; - } - else { - Py_INCREF(locals); } - Py_CLEAR(hidden); - return locals; - - error: - Py_XDECREF(hidden); - return NULL; + return false; } -int -_PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) +PyObject * +_PyFrame_GetLocals(_PyInterpreterFrame *frame) { - PyObject *locals = _PyFrame_GetLocals(frame, 0); - if (locals == NULL) { - return -1; + // We should try to avoid creating the FrameObject if possible. + // So we check if the frame is a module or class level scope + PyCodeObject *co = _PyFrame_GetCode(frame); + + if (!(co->co_flags & CO_OPTIMIZED) && !_PyFrame_HasHiddenLocals(frame)) { + if (frame->f_locals == NULL) { + // We found cases when f_locals is NULL for non-optimized code. + // We fill the f_locals with an empty dict to avoid crash until + // we find the root cause. + frame->f_locals = PyDict_New(); + if (frame->f_locals == NULL) { + return NULL; + } + } + return Py_NewRef(frame->f_locals); } - Py_DECREF(locals); - return 0; + + PyFrameObject* f = _PyFrame_GetFrameObject(frame); + + return _PyFrameLocalsProxy_New(f); } @@ -1320,14 +2094,14 @@ PyFrame_GetVar(PyFrameObject *frame_obj, PyObject *name) continue; } - PyObject *value; // borrowed reference + PyObject *value; if (!frame_get_var(frame, co, i, &value)) { break; } if (value == NULL) { break; } - return Py_NewRef(value); + return value; } PyErr_Format(PyExc_NameError, "variable %R does not exist", name); @@ -1351,112 +2125,25 @@ PyFrame_GetVarString(PyFrameObject *frame, const char *name) int PyFrame_FastToLocalsWithError(PyFrameObject *f) { - if (f == NULL) { - PyErr_BadInternalCall(); - return -1; - } - assert(!_PyFrame_IsIncomplete(f->f_frame)); - int err = _PyFrame_FastToLocalsWithError(f->f_frame); - if (err == 0) { - f->f_fast_as_locals = 1; - } - return err; + // Nothing to do here, as f_locals is now a write-through proxy in + // optimized frames. Soft-deprecated, since there's no maintenance hassle. + return 0; } void PyFrame_FastToLocals(PyFrameObject *f) { - int res; - assert(!_PyFrame_IsIncomplete(f->f_frame)); - assert(!PyErr_Occurred()); - - res = PyFrame_FastToLocalsWithError(f); - if (res < 0) - PyErr_Clear(); -} - -void -_PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear) -{ - /* Merge locals into fast locals */ - PyObject *locals; - PyObject **fast; - PyCodeObject *co; - locals = frame->f_locals; - if (locals == NULL) { - return; - } - fast = _PyFrame_GetLocalsArray(frame); - co = _PyFrame_GetCode(frame); - - PyObject *exc = PyErr_GetRaisedException(); - for (int i = 0; i < co->co_nlocalsplus; i++) { - _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); - - /* Same test as in PyFrame_FastToLocals() above. */ - if (kind & CO_FAST_FREE && !(co->co_flags & CO_OPTIMIZED)) { - continue; - } - PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); - PyObject *value = PyObject_GetItem(locals, name); - /* We only care about NULLs if clear is true. */ - if (value == NULL) { - PyErr_Clear(); - if (!clear) { - continue; - } - } - PyObject *oldvalue = fast[i]; - PyObject *cell = NULL; - if (kind == CO_FAST_FREE) { - // The cell was set when the frame was created from - // the function's closure. - assert(oldvalue != NULL && PyCell_Check(oldvalue)); - cell = oldvalue; - } - else if (kind & CO_FAST_CELL && oldvalue != NULL) { - /* Same test as in PyFrame_FastToLocals() above. */ - if (PyCell_Check(oldvalue) && - _PyFrame_OpAlreadyRan(frame, MAKE_CELL, i)) { - // (likely) MAKE_CELL must have executed already. - cell = oldvalue; - } - // (unlikely) Otherwise, it must have been set to some - // initial value by an earlier call to PyFrame_LocalsToFast(). - } - if (cell != NULL) { - oldvalue = PyCell_GET(cell); - if (value != oldvalue) { - PyCell_SET(cell, Py_XNewRef(value)); - Py_XDECREF(oldvalue); - } - } - else if (value != oldvalue) { - if (value == NULL) { - // Probably can't delete this, since the compiler's flow - // analysis may have already "proven" that it exists here: - const char *e = "assigning None to unbound local %R"; - if (PyErr_WarnFormat(PyExc_RuntimeWarning, 0, e, name)) { - // It's okay if frame_obj is NULL, just try anyways: - PyErr_WriteUnraisable((PyObject *)frame->frame_obj); - } - value = Py_NewRef(Py_None); - } - Py_XSETREF(fast[i], Py_NewRef(value)); - } - Py_XDECREF(value); - } - PyErr_SetRaisedException(exc); + // Nothing to do here, as f_locals is now a write-through proxy in + // optimized frames. Soft-deprecated, since there's no maintenance hassle. + return; } void PyFrame_LocalsToFast(PyFrameObject *f, int clear) { - assert(!_PyFrame_IsIncomplete(f->f_frame)); - if (f && f->f_fast_as_locals && !frame_is_cleared(f)) { - _PyFrame_LocalsToFast(f->f_frame, clear); - f->f_fast_as_locals = 0; - } + // Nothing to do here, as f_locals is now a write-through proxy in + // optimized frames. Soft-deprecated, since there's no maintenance hassle. + return; } int @@ -1465,7 +2152,7 @@ _PyFrame_IsEntryFrame(PyFrameObject *frame) assert(frame != NULL); _PyInterpreterFrame *f = frame->f_frame; assert(!_PyFrame_IsIncomplete(f)); - return f->previous && f->previous->owner == FRAME_OWNED_BY_CSTACK; + return f->previous && f->previous->owner == FRAME_OWNED_BY_INTERPRETER; } PyCodeObject * @@ -1534,24 +2221,6 @@ PyFrame_GetGenerator(PyFrameObject *frame) if (frame->f_frame->owner != FRAME_OWNED_BY_GENERATOR) { return NULL; } - PyGenObject *gen = _PyFrame_GetGenerator(frame->f_frame); + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame->f_frame); return Py_NewRef(gen); } - -PyObject* -_PyEval_BuiltinsFromGlobals(PyThreadState *tstate, PyObject *globals) -{ - PyObject *builtins = PyDict_GetItemWithError(globals, &_Py_ID(__builtins__)); - if (builtins) { - if (PyModule_Check(builtins)) { - builtins = _PyModule_GetDict(builtins); - assert(builtins != NULL); - } - return builtins; - } - if (PyErr_Occurred()) { - return NULL; - } - - return _PyEval_GetBuiltins(tstate); -} diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 08b2823d8cf024..7b17a9ba31fac4 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -2,7 +2,8 @@ /* Function object implementation */ #include "Python.h" -#include "pycore_ceval.h" // _PyEval_BuiltinsFromGlobals() +#include "pycore_dict.h" // _Py_INCREF_DICT() +#include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_Occurred() @@ -111,11 +112,14 @@ _PyFunction_FromConstructor(PyFrameConstructor *constr) Py_XDECREF(module); return NULL; } - op->func_globals = Py_NewRef(constr->fc_globals); - op->func_builtins = Py_NewRef(constr->fc_builtins); + _Py_INCREF_DICT(constr->fc_globals); + op->func_globals = constr->fc_globals; + _Py_INCREF_BUILTINS(constr->fc_builtins); + op->func_builtins = constr->fc_builtins; op->func_name = Py_NewRef(constr->fc_name); op->func_qualname = Py_NewRef(constr->fc_qualname); - op->func_code = Py_NewRef(constr->fc_code); + _Py_INCREF_CODE((PyCodeObject *)constr->fc_code); + op->func_code = constr->fc_code; op->func_defaults = Py_XNewRef(constr->fc_defaults); op->func_kwdefaults = Py_XNewRef(constr->fc_kwdefaults); op->func_closure = Py_XNewRef(constr->fc_closure); @@ -124,9 +128,13 @@ _PyFunction_FromConstructor(PyFrameConstructor *constr) op->func_weakreflist = NULL; op->func_module = module; op->func_annotations = NULL; + op->func_annotate = NULL; op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; - op->func_version = 0; + op->func_version = FUNC_VERSION_UNSET; + // NOTE: functions created via FrameConstructor do not use deferred + // reference counting because they are typically not part of cycles + // nor accessed by multiple threads. _PyObject_GC_TRACK(op); handle_func_event(PyFunction_EVENT_CREATE, op, NULL); return op; @@ -137,11 +145,10 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname { assert(globals != NULL); assert(PyDict_Check(globals)); - Py_INCREF(globals); + _Py_INCREF_DICT(globals); - PyThreadState *tstate = _PyThreadState_GET(); - - PyCodeObject *code_obj = (PyCodeObject *)Py_NewRef(code); + PyCodeObject *code_obj = (PyCodeObject *)code; + _Py_INCREF_CODE(code_obj); assert(code_obj->co_name != NULL); PyObject *name = Py_NewRef(code_obj->co_name); @@ -155,7 +162,8 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname PyObject *consts = code_obj->co_consts; assert(PyTuple_Check(consts)); PyObject *doc; - if (PyTuple_Size(consts) >= 1) { + if (code_obj->co_flags & CO_HAS_DOCSTRING) { + assert(PyTuple_Size(consts) >= 1); doc = PyTuple_GetItem(consts, 0); if (!PyUnicode_Check(doc)) { doc = Py_None; @@ -173,11 +181,10 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname goto error; } - builtins = _PyEval_BuiltinsFromGlobals(tstate, globals); // borrowed ref + builtins = _PyDict_LoadBuiltinsFromGlobals(globals); if (builtins == NULL) { goto error; } - Py_INCREF(builtins); PyFunctionObject *op = PyObject_GC_New(PyFunctionObject, &PyFunction_Type); if (op == NULL) { @@ -199,9 +206,20 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname op->func_weakreflist = NULL; op->func_module = module; op->func_annotations = NULL; + op->func_annotate = NULL; op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; - op->func_version = 0; + op->func_version = FUNC_VERSION_UNSET; + if (((code_obj->co_flags & CO_NESTED) == 0) || + (code_obj->co_flags & CO_METHOD)) { + // Use deferred reference counting for top-level functions, but not + // nested functions because they are more likely to capture variables, + // which makes prompt deallocation more important. + // + // Nested methods (functions defined in class scope) are also deferred, + // since they will likely be cleaned up by GC anyway. + _PyObject_SetDeferredRefcount((PyObject *)op); + } _PyObject_GC_TRACK(op); handle_func_event(PyFunction_EVENT_CREATE, op, NULL); return (PyObject *)op; @@ -218,92 +236,167 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname } /* -Function versions ------------------ +(This is purely internal documentation. There are no public APIs here.) + +Function (and code) versions +---------------------------- + +The Tier 1 specializer generates CALL variants that can be invalidated +by changes to critical function attributes: + +- __code__ +- __defaults__ +- __kwdefaults__ +- __closure__ -Function versions are used to detect when a function object has been -updated, invalidating inline cache data used by the `CALL` bytecode -(notably `CALL_PY_EXACT_ARGS` and a few other `CALL` specializations). +For this purpose function objects have a 32-bit func_version member +that the specializer writes to the specialized instruction's inline +cache and which is checked by a guard on the specialized instructions. -They are also used by the Tier 2 superblock creation code to find -the function being called (and from there the code object). +The MAKE_FUNCTION bytecode sets func_version from the code object's +co_version field. The latter is initialized from a counter in the +interpreter state (interp->func_state.next_version) and never changes. +When this counter overflows, it remains zero and the specializer loses +the ability to specialize calls to new functions. -How does a function's `func_version` field get initialized? +The func_version is reset to zero when any of the critical attributes +is modified; after this point the specializer will no longer specialize +calls to this function, and the guard will always fail. -- `PyFunction_New` and friends initialize it to 0. -- The `MAKE_FUNCTION` instruction sets it from the code's `co_version`. -- It is reset to 0 when various attributes like `__code__` are set. -- A new version is allocated by `_PyFunction_GetVersionForCurrentState` - when the specializer needs a version and the version is 0. +The function and code version cache +----------------------------------- -The latter allocates versions using a counter in the interpreter state; -when the counter wraps around to 0, no more versions are allocated. -There is one other special case: functions with a non-standard -`vectorcall` field are not given a version. +The Tier 2 optimizer now has a problem, since it needs to find the +function and code objects given only the version number from the inline +cache. Our solution is to maintain a cache mapping version numbers to +function and code objects. To limit the cache size we could hash +the version number, but for now we simply use it modulo the table size. -When the function version is 0, the `CALL` bytecode is not specialized. +There are some corner cases (e.g. generator expressions) where we will +be unable to find the function object in the cache but we can still +find the code object. For this reason the cache stores both the +function object and the code object. -Code object versions --------------------- +The cache doesn't contain strong references; cache entries are +invalidated whenever the function or code object is deallocated. -So where to code objects get their `co_version`? -There is a per-interpreter counter, `next_func_version`. -This is initialized to 1 when the interpreter is created. +Invariants +---------- + +These should hold at any time except when one of the cache-mutating +functions is running. + +- For any slot s at index i: + - s->func == NULL or s->func->func_version % FUNC_VERSION_CACHE_SIZE == i + - s->code == NULL or s->code->co_version % FUNC_VERSION_CACHE_SIZE == i + if s->func != NULL, then s->func->func_code == s->code -Code objects get a new `co_version` allocated from this counter upon -creation. Since code objects are nominally immutable, `co_version` can -not be invalidated. The only way it can be 0 is when 2**32 or more -code objects have been created during the process's lifetime. -(The counter isn't reset by `fork()`, extending the lifetime.) */ +#ifndef Py_GIL_DISABLED +static inline struct _func_version_cache_item * +get_cache_item(PyInterpreterState *interp, uint32_t version) +{ + return interp->func_state.func_version_cache + + (version % FUNC_VERSION_CACHE_SIZE); +} +#endif + void _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version) { + assert(func->func_version == FUNC_VERSION_UNSET); + assert(version >= FUNC_VERSION_FIRST_VALID); + // This should only be called from MAKE_FUNCTION. No code is specialized + // based on the version, so we do not need to stop the world to set it. + func->func_version = version; +#ifndef Py_GIL_DISABLED PyInterpreterState *interp = _PyInterpreterState_GET(); - if (func->func_version != 0) { - PyFunctionObject **slot = - interp->func_state.func_version_cache - + (func->func_version % FUNC_VERSION_CACHE_SIZE); - if (*slot == func) { - *slot = NULL; - } + struct _func_version_cache_item *slot = get_cache_item(interp, version); + slot->func = func; + slot->code = func->func_code; +#endif +} + +static void +func_clear_version(PyInterpreterState *interp, PyFunctionObject *func) +{ + if (func->func_version < FUNC_VERSION_FIRST_VALID) { + // Version was never set or has already been cleared. + return; } - func->func_version = version; - if (version != 0) { - interp->func_state.func_version_cache[ - version % FUNC_VERSION_CACHE_SIZE] = func; +#ifndef Py_GIL_DISABLED + struct _func_version_cache_item *slot = + get_cache_item(interp, func->func_version); + if (slot->func == func) { + slot->func = NULL; + // Leave slot->code alone, there may be use for it. } +#endif + func->func_version = FUNC_VERSION_CLEARED; +} + +// Called when any of the critical function attributes are changed +static void +_PyFunction_ClearVersion(PyFunctionObject *func) +{ + if (func->func_version < FUNC_VERSION_FIRST_VALID) { + // Version was never set or has already been cleared. + return; + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); + func_clear_version(interp, func); + _PyEval_StartTheWorld(interp); +} + +void +_PyFunction_ClearCodeByVersion(uint32_t version) +{ +#ifndef Py_GIL_DISABLED + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _func_version_cache_item *slot = get_cache_item(interp, version); + if (slot->code) { + assert(PyCode_Check(slot->code)); + PyCodeObject *code = (PyCodeObject *)slot->code; + if (code->co_version == version) { + slot->code = NULL; + slot->func = NULL; + } + } +#endif } PyFunctionObject * -_PyFunction_LookupByVersion(uint32_t version) +_PyFunction_LookupByVersion(uint32_t version, PyObject **p_code) { +#ifdef Py_GIL_DISABLED + return NULL; +#else PyInterpreterState *interp = _PyInterpreterState_GET(); - PyFunctionObject *func = interp->func_state.func_version_cache[ - version % FUNC_VERSION_CACHE_SIZE]; - if (func != NULL && func->func_version == version) { - return func; + struct _func_version_cache_item *slot = get_cache_item(interp, version); + if (slot->code) { + assert(PyCode_Check(slot->code)); + PyCodeObject *code = (PyCodeObject *)slot->code; + if (code->co_version == version) { + *p_code = slot->code; + } + } + else { + *p_code = NULL; + } + if (slot->func && slot->func->func_version == version) { + assert(slot->func->func_code == slot->code); + return slot->func; } return NULL; +#endif } uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func) { - if (func->func_version != 0) { - return func->func_version; - } - if (func->vectorcall != _PyFunction_Vectorcall) { - return 0; - } - PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->func_state.next_version == 0) { - return 0; - } - uint32_t v = interp->func_state.next_version++; - _PyFunction_SetVersion(func, v); - return v; + return func->func_version; } PyObject * @@ -370,7 +463,7 @@ PyFunction_SetDefaults(PyObject *op, PyObject *defaults) } handle_func_event(PyFunction_EVENT_MODIFY_DEFAULTS, (PyFunctionObject *) op, defaults); - _PyFunction_SetVersion((PyFunctionObject *)op, 0); + _PyFunction_ClearVersion((PyFunctionObject *)op); Py_XSETREF(((PyFunctionObject *)op)->func_defaults, defaults); return 0; } @@ -379,7 +472,7 @@ void PyFunction_SetVectorcall(PyFunctionObject *func, vectorcallfunc vectorcall) { assert(func != NULL); - _PyFunction_SetVersion(func, 0); + _PyFunction_ClearVersion(func); func->vectorcall = vectorcall; } @@ -412,7 +505,7 @@ PyFunction_SetKwDefaults(PyObject *op, PyObject *defaults) } handle_func_event(PyFunction_EVENT_MODIFY_KWDEFAULTS, (PyFunctionObject *) op, defaults); - _PyFunction_SetVersion((PyFunctionObject *)op, 0); + _PyFunction_ClearVersion((PyFunctionObject *)op); Py_XSETREF(((PyFunctionObject *)op)->func_kwdefaults, defaults); return 0; } @@ -445,7 +538,7 @@ PyFunction_SetClosure(PyObject *op, PyObject *closure) Py_TYPE(closure)->tp_name); return -1; } - _PyFunction_SetVersion((PyFunctionObject *)op, 0); + _PyFunction_ClearVersion((PyFunctionObject *)op); Py_XSETREF(((PyFunctionObject *)op)->func_closure, closure); return 0; } @@ -454,7 +547,22 @@ static PyObject * func_get_annotation_dict(PyFunctionObject *op) { if (op->func_annotations == NULL) { - return NULL; + if (op->func_annotate == NULL || !PyCallable_Check(op->func_annotate)) { + Py_RETURN_NONE; + } + PyObject *one = _PyLong_GetOne(); + PyObject *ann_dict = _PyObject_CallOneArg(op->func_annotate, one); + if (ann_dict == NULL) { + return NULL; + } + if (!PyDict_Check(ann_dict)) { + PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", + Py_TYPE(ann_dict)->tp_name); + Py_DECREF(ann_dict); + return NULL; + } + Py_XSETREF(op->func_annotations, ann_dict); + return ann_dict; } if (PyTuple_CheckExact(op->func_annotations)) { PyObject *ann_tuple = op->func_annotations; @@ -471,6 +579,7 @@ func_get_annotation_dict(PyFunctionObject *op) PyTuple_GET_ITEM(ann_tuple, i + 1)); if (err < 0) { + Py_DECREF(ann_dict); return NULL; } } @@ -507,8 +616,9 @@ PyFunction_SetAnnotations(PyObject *op, PyObject *annotations) "non-dict annotations"); return -1; } - _PyFunction_SetVersion((PyFunctionObject *)op, 0); - Py_XSETREF(((PyFunctionObject *)op)->func_annotations, annotations); + PyFunctionObject *func = (PyFunctionObject *)op; + Py_XSETREF(func->func_annotations, annotations); + Py_CLEAR(func->func_annotate); return 0; } @@ -526,8 +636,9 @@ static PyMemberDef func_memberlist[] = { }; static PyObject * -func_get_code(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_code(PyObject *self, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); if (PySys_Audit("object.__getattr__", "Os", op, "__code__") < 0) { return NULL; } @@ -536,10 +647,9 @@ func_get_code(PyFunctionObject *op, void *Py_UNUSED(ignored)) } static int -func_set_code(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_code(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { - Py_ssize_t nclosure; - int nfree; + PyFunctionObject *op = _PyFunction_CAST(self); /* Not legal to del f.func_code or to set it to anything * other than a code object. */ @@ -554,9 +664,9 @@ func_set_code(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) return -1; } - nfree = ((PyCodeObject *)value)->co_nfreevars; - nclosure = (op->func_closure == NULL ? 0 : - PyTuple_GET_SIZE(op->func_closure)); + int nfree = ((PyCodeObject *)value)->co_nfreevars; + Py_ssize_t nclosure = (op->func_closure == NULL ? 0 : + PyTuple_GET_SIZE(op->func_closure)); if (nclosure != nfree) { PyErr_Format(PyExc_ValueError, "%U() requires a code object with %zd free vars," @@ -580,20 +690,22 @@ func_set_code(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) } handle_func_event(PyFunction_EVENT_MODIFY_CODE, op, value); - _PyFunction_SetVersion(op, 0); + _PyFunction_ClearVersion(op); Py_XSETREF(op->func_code, Py_NewRef(value)); return 0; } static PyObject * -func_get_name(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_name(PyObject *self, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); return Py_NewRef(op->func_name); } static int -func_set_name(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_name(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); /* Not legal to del f.func_name or to set it to anything * other than a string object. */ if (value == NULL || !PyUnicode_Check(value)) { @@ -606,14 +718,16 @@ func_set_name(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) } static PyObject * -func_get_qualname(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_qualname(PyObject *self, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); return Py_NewRef(op->func_qualname); } static int -func_set_qualname(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_qualname(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); /* Not legal to del f.__qualname__ or to set it to anything * other than a string object. */ if (value == NULL || !PyUnicode_Check(value)) { @@ -626,8 +740,9 @@ func_set_qualname(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored } static PyObject * -func_get_defaults(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_defaults(PyObject *self, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); if (PySys_Audit("object.__getattr__", "Os", op, "__defaults__") < 0) { return NULL; } @@ -638,10 +753,11 @@ func_get_defaults(PyFunctionObject *op, void *Py_UNUSED(ignored)) } static int -func_set_defaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_defaults(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { /* Legal to del f.func_defaults. * Can only set func_defaults to NULL or a tuple. */ + PyFunctionObject *op = _PyFunction_CAST(self); if (value == Py_None) value = NULL; if (value != NULL && !PyTuple_Check(value)) { @@ -660,14 +776,15 @@ func_set_defaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored } handle_func_event(PyFunction_EVENT_MODIFY_DEFAULTS, op, value); - _PyFunction_SetVersion(op, 0); + _PyFunction_ClearVersion(op); Py_XSETREF(op->func_defaults, Py_XNewRef(value)); return 0; } static PyObject * -func_get_kwdefaults(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_kwdefaults(PyObject *self, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); if (PySys_Audit("object.__getattr__", "Os", op, "__kwdefaults__") < 0) { return NULL; @@ -679,8 +796,9 @@ func_get_kwdefaults(PyFunctionObject *op, void *Py_UNUSED(ignored)) } static int -func_set_kwdefaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_kwdefaults(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); if (value == Py_None) value = NULL; /* Legal to del f.func_kwdefaults. @@ -701,15 +819,52 @@ func_set_kwdefaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignor } handle_func_event(PyFunction_EVENT_MODIFY_KWDEFAULTS, op, value); - _PyFunction_SetVersion(op, 0); + _PyFunction_ClearVersion(op); Py_XSETREF(op->func_kwdefaults, Py_XNewRef(value)); return 0; } static PyObject * -func_get_annotations(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_annotate(PyObject *self, void *Py_UNUSED(ignored)) { - if (op->func_annotations == NULL) { + PyFunctionObject *op = _PyFunction_CAST(self); + if (op->func_annotate == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(op->func_annotate); +} + +static int +func_set_annotate(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) +{ + PyFunctionObject *op = _PyFunction_CAST(self); + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "__annotate__ cannot be deleted"); + return -1; + } + if (Py_IsNone(value)) { + Py_XSETREF(op->func_annotate, value); + return 0; + } + else if (PyCallable_Check(value)) { + Py_XSETREF(op->func_annotate, Py_XNewRef(value)); + Py_CLEAR(op->func_annotations); + return 0; + } + else { + PyErr_SetString(PyExc_TypeError, + "__annotate__ must be callable or None"); + return -1; + } +} + +static PyObject * +func_get_annotations(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyFunctionObject *op = _PyFunction_CAST(self); + if (op->func_annotations == NULL && + (op->func_annotate == NULL || !PyCallable_Check(op->func_annotate))) { op->func_annotations = PyDict_New(); if (op->func_annotations == NULL) return NULL; @@ -719,8 +874,9 @@ func_get_annotations(PyFunctionObject *op, void *Py_UNUSED(ignored)) } static int -func_set_annotations(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_annotations(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); if (value == Py_None) value = NULL; /* Legal to del f.func_annotations. @@ -731,14 +887,15 @@ func_set_annotations(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(igno "__annotations__ must be set to a dict object"); return -1; } - _PyFunction_SetVersion(op, 0); Py_XSETREF(op->func_annotations, Py_XNewRef(value)); + Py_CLEAR(op->func_annotate); return 0; } static PyObject * -func_get_type_params(PyFunctionObject *op, void *Py_UNUSED(ignored)) +func_get_type_params(PyObject *self, void *Py_UNUSED(ignored)) { + PyFunctionObject *op = _PyFunction_CAST(self); if (op->func_typeparams == NULL) { return PyTuple_New(0); } @@ -748,10 +905,11 @@ func_get_type_params(PyFunctionObject *op, void *Py_UNUSED(ignored)) } static int -func_set_type_params(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) +func_set_type_params(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { /* Not legal to del f.__type_params__ or to set it to anything * other than a tuple object. */ + PyFunctionObject *op = _PyFunction_CAST(self); if (value == NULL || !PyTuple_Check(value)) { PyErr_SetString(PyExc_TypeError, "__type_params__ must be set to a tuple"); @@ -773,18 +931,15 @@ _Py_set_function_type_params(PyThreadState *Py_UNUSED(ignored), PyObject *func, } static PyGetSetDef func_getsetlist[] = { - {"__code__", (getter)func_get_code, (setter)func_set_code}, - {"__defaults__", (getter)func_get_defaults, - (setter)func_set_defaults}, - {"__kwdefaults__", (getter)func_get_kwdefaults, - (setter)func_set_kwdefaults}, - {"__annotations__", (getter)func_get_annotations, - (setter)func_set_annotations}, + {"__code__", func_get_code, func_set_code}, + {"__defaults__", func_get_defaults, func_set_defaults}, + {"__kwdefaults__", func_get_kwdefaults, func_set_kwdefaults}, + {"__annotations__", func_get_annotations, func_set_annotations}, + {"__annotate__", func_get_annotate, func_set_annotate}, {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict}, - {"__name__", (getter)func_get_name, (setter)func_set_name}, - {"__qualname__", (getter)func_get_qualname, (setter)func_set_qualname}, - {"__type_params__", (getter)func_get_type_params, - (setter)func_set_type_params}, + {"__name__", func_get_name, func_set_name}, + {"__qualname__", func_get_qualname, func_set_qualname}, + {"__type_params__", func_get_type_params, func_set_type_params}, {NULL} /* Sentinel */ }; @@ -904,11 +1059,20 @@ func_new_impl(PyTypeObject *type, PyCodeObject *code, PyObject *globals, } static int -func_clear(PyFunctionObject *op) +func_clear(PyObject *self) { - _PyFunction_SetVersion(op, 0); - Py_CLEAR(op->func_globals); - Py_CLEAR(op->func_builtins); + PyFunctionObject *op = _PyFunction_CAST(self); + func_clear_version(_PyInterpreterState_GET(), op); + PyObject *globals = op->func_globals; + op->func_globals = NULL; + if (globals != NULL) { + _Py_DECREF_DICT(globals); + } + PyObject *builtins = op->func_builtins; + op->func_builtins = NULL; + if (builtins != NULL) { + _Py_DECREF_BUILTINS(builtins); + } Py_CLEAR(op->func_module); Py_CLEAR(op->func_defaults); Py_CLEAR(op->func_kwdefaults); @@ -916,6 +1080,7 @@ func_clear(PyFunctionObject *op) Py_CLEAR(op->func_dict); Py_CLEAR(op->func_closure); Py_CLEAR(op->func_annotations); + Py_CLEAR(op->func_annotate); Py_CLEAR(op->func_typeparams); // Don't Py_CLEAR(op->func_code), since code is always required // to be non-NULL. Similarly, name and qualname shouldn't be NULL. @@ -928,39 +1093,38 @@ func_clear(PyFunctionObject *op) } static void -func_dealloc(PyFunctionObject *op) +func_dealloc(PyObject *self) { - assert(Py_REFCNT(op) == 0); - Py_SET_REFCNT(op, 1); + PyFunctionObject *op = _PyFunction_CAST(self); + _PyObject_ResurrectStart(self); handle_func_event(PyFunction_EVENT_DESTROY, op, NULL); - if (Py_REFCNT(op) > 1) { - Py_SET_REFCNT(op, Py_REFCNT(op) - 1); + if (_PyObject_ResurrectEnd(self)) { return; } - Py_SET_REFCNT(op, 0); _PyObject_GC_UNTRACK(op); if (op->func_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *) op); } - _PyFunction_SetVersion(op, 0); - (void)func_clear(op); + (void)func_clear((PyObject*)op); // These aren't cleared by func_clear(). - Py_DECREF(op->func_code); + _Py_DECREF_CODE((PyCodeObject *)op->func_code); Py_DECREF(op->func_name); Py_DECREF(op->func_qualname); PyObject_GC_Del(op); } static PyObject* -func_repr(PyFunctionObject *op) +func_repr(PyObject *self) { + PyFunctionObject *op = _PyFunction_CAST(self); return PyUnicode_FromFormat("", op->func_qualname, op); } static int -func_traverse(PyFunctionObject *f, visitproc visit, void *arg) +func_traverse(PyObject *self, visitproc visit, void *arg) { + PyFunctionObject *f = _PyFunction_CAST(self); Py_VISIT(f->func_code); Py_VISIT(f->func_globals); Py_VISIT(f->func_builtins); @@ -972,6 +1136,7 @@ func_traverse(PyFunctionObject *f, visitproc visit, void *arg) Py_VISIT(f->func_dict); Py_VISIT(f->func_closure); Py_VISIT(f->func_annotations); + Py_VISIT(f->func_annotate); Py_VISIT(f->func_typeparams); Py_VISIT(f->func_qualname); return 0; @@ -992,12 +1157,12 @@ PyTypeObject PyFunction_Type = { "function", sizeof(PyFunctionObject), 0, - (destructor)func_dealloc, /* tp_dealloc */ + func_dealloc, /* tp_dealloc */ offsetof(PyFunctionObject, vectorcall), /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)func_repr, /* tp_repr */ + func_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -1011,8 +1176,8 @@ PyTypeObject PyFunction_Type = { Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_METHOD_DESCRIPTOR, /* tp_flags */ func_new__doc__, /* tp_doc */ - (traverseproc)func_traverse, /* tp_traverse */ - (inquiry)func_clear, /* tp_clear */ + func_traverse, /* tp_traverse */ + func_clear, /* tp_clear */ 0, /* tp_richcompare */ offsetof(PyFunctionObject, func_weakreflist), /* tp_weaklistoffset */ 0, /* tp_iter */ @@ -1058,12 +1223,75 @@ functools_wraps(PyObject *wrapper, PyObject *wrapped) COPY_ATTR(__name__); COPY_ATTR(__qualname__); COPY_ATTR(__doc__); - COPY_ATTR(__annotations__); return 0; #undef COPY_ATTR } +// Used for wrapping __annotations__ and __annotate__ on classmethod +// and staticmethod objects. +static PyObject * +descriptor_get_wrapped_attribute(PyObject *wrapped, PyObject *obj, PyObject *name) +{ + PyObject *dict = PyObject_GenericGetDict(obj, NULL); + if (dict == NULL) { + return NULL; + } + PyObject *res; + if (PyDict_GetItemRef(dict, name, &res) < 0) { + Py_DECREF(dict); + return NULL; + } + if (res != NULL) { + Py_DECREF(dict); + return res; + } + res = PyObject_GetAttr(wrapped, name); + if (res == NULL) { + Py_DECREF(dict); + return NULL; + } + if (PyDict_SetItem(dict, name, res) < 0) { + Py_DECREF(dict); + Py_DECREF(res); + return NULL; + } + Py_DECREF(dict); + return res; +} + +static int +descriptor_set_wrapped_attribute(PyObject *oobj, PyObject *name, PyObject *value, + char *type_name) +{ + PyObject *dict = PyObject_GenericGetDict(oobj, NULL); + if (dict == NULL) { + return -1; + } + if (value == NULL) { + if (PyDict_DelItem(dict, name) < 0) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + PyErr_Format(PyExc_AttributeError, + "'%.200s' object has no attribute '%U'", + type_name, name); + Py_DECREF(dict); + return -1; + } + else { + Py_DECREF(dict); + return -1; + } + } + Py_DECREF(dict); + return 0; + } + else { + Py_DECREF(dict); + return PyDict_SetItem(dict, name, value); + } +} + /* Class method object */ @@ -1091,9 +1319,14 @@ typedef struct { PyObject *cm_dict; } classmethod; +#define _PyClassMethod_CAST(cm) \ + (assert(PyObject_TypeCheck((cm), &PyClassMethod_Type)), \ + _Py_CAST(classmethod*, cm)) + static void -cm_dealloc(classmethod *cm) +cm_dealloc(PyObject *self) { + classmethod *cm = _PyClassMethod_CAST(self); _PyObject_GC_UNTRACK((PyObject *)cm); Py_XDECREF(cm->cm_callable); Py_XDECREF(cm->cm_dict); @@ -1101,16 +1334,18 @@ cm_dealloc(classmethod *cm) } static int -cm_traverse(classmethod *cm, visitproc visit, void *arg) +cm_traverse(PyObject *self, visitproc visit, void *arg) { + classmethod *cm = _PyClassMethod_CAST(self); Py_VISIT(cm->cm_callable); Py_VISIT(cm->cm_dict); return 0; } static int -cm_clear(classmethod *cm) +cm_clear(PyObject *self) { + classmethod *cm = _PyClassMethod_CAST(self); Py_CLEAR(cm->cm_callable); Py_CLEAR(cm->cm_dict); return 0; @@ -1157,8 +1392,9 @@ static PyMemberDef cm_memberlist[] = { }; static PyObject * -cm_get___isabstractmethod__(classmethod *cm, void *closure) +cm_get___isabstractmethod__(PyObject *self, void *closure) { + classmethod *cm = _PyClassMethod_CAST(self); int res = _PyObject_IsAbstract(cm->cm_callable); if (res == -1) { return NULL; @@ -1169,16 +1405,45 @@ cm_get___isabstractmethod__(classmethod *cm, void *closure) Py_RETURN_FALSE; } +static PyObject * +cm_get___annotations__(PyObject *self, void *closure) +{ + classmethod *cm = _PyClassMethod_CAST(self); + return descriptor_get_wrapped_attribute(cm->cm_callable, self, &_Py_ID(__annotations__)); +} + +static int +cm_set___annotations__(PyObject *self, PyObject *value, void *closure) +{ + return descriptor_set_wrapped_attribute(self, &_Py_ID(__annotations__), value, "classmethod"); +} + +static PyObject * +cm_get___annotate__(PyObject *self, void *closure) +{ + classmethod *cm = _PyClassMethod_CAST(self); + return descriptor_get_wrapped_attribute(cm->cm_callable, self, &_Py_ID(__annotate__)); +} + +static int +cm_set___annotate__(PyObject *self, PyObject *value, void *closure) +{ + return descriptor_set_wrapped_attribute(self, &_Py_ID(__annotate__), value, "classmethod"); +} + + static PyGetSetDef cm_getsetlist[] = { - {"__isabstractmethod__", - (getter)cm_get___isabstractmethod__, NULL, NULL, NULL}, + {"__isabstractmethod__", cm_get___isabstractmethod__, NULL, NULL, NULL}, {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, + {"__annotations__", cm_get___annotations__, cm_set___annotations__, NULL, NULL}, + {"__annotate__", cm_get___annotate__, cm_set___annotate__, NULL, NULL}, {NULL} /* Sentinel */ }; static PyObject* -cm_repr(classmethod *cm) +cm_repr(PyObject *self) { + classmethod *cm = _PyClassMethod_CAST(self); return PyUnicode_FromFormat("", cm->cm_callable); } @@ -1210,12 +1475,12 @@ PyTypeObject PyClassMethod_Type = { "classmethod", sizeof(classmethod), 0, - (destructor)cm_dealloc, /* tp_dealloc */ + cm_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)cm_repr, /* tp_repr */ + cm_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -1227,14 +1492,14 @@ PyTypeObject PyClassMethod_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, classmethod_doc, /* tp_doc */ - (traverseproc)cm_traverse, /* tp_traverse */ - (inquiry)cm_clear, /* tp_clear */ + cm_traverse, /* tp_traverse */ + cm_clear, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ 0, /* tp_methods */ - cm_memberlist, /* tp_members */ + cm_memberlist, /* tp_members */ cm_getsetlist, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ @@ -1283,9 +1548,14 @@ typedef struct { PyObject *sm_dict; } staticmethod; +#define _PyStaticMethod_CAST(cm) \ + (assert(PyObject_TypeCheck((cm), &PyStaticMethod_Type)), \ + _Py_CAST(staticmethod*, cm)) + static void -sm_dealloc(staticmethod *sm) +sm_dealloc(PyObject *self) { + staticmethod *sm = _PyStaticMethod_CAST(self); _PyObject_GC_UNTRACK((PyObject *)sm); Py_XDECREF(sm->sm_callable); Py_XDECREF(sm->sm_dict); @@ -1293,16 +1563,18 @@ sm_dealloc(staticmethod *sm) } static int -sm_traverse(staticmethod *sm, visitproc visit, void *arg) +sm_traverse(PyObject *self, visitproc visit, void *arg) { + staticmethod *sm = _PyStaticMethod_CAST(self); Py_VISIT(sm->sm_callable); Py_VISIT(sm->sm_dict); return 0; } static int -sm_clear(staticmethod *sm) +sm_clear(PyObject *self) { + staticmethod *sm = _PyStaticMethod_CAST(self); Py_CLEAR(sm->sm_callable); Py_CLEAR(sm->sm_dict); return 0; @@ -1353,8 +1625,9 @@ static PyMemberDef sm_memberlist[] = { }; static PyObject * -sm_get___isabstractmethod__(staticmethod *sm, void *closure) +sm_get___isabstractmethod__(PyObject *self, void *closure) { + staticmethod *sm = _PyStaticMethod_CAST(self); int res = _PyObject_IsAbstract(sm->sm_callable); if (res == -1) { return NULL; @@ -1365,16 +1638,44 @@ sm_get___isabstractmethod__(staticmethod *sm, void *closure) Py_RETURN_FALSE; } +static PyObject * +sm_get___annotations__(PyObject *self, void *closure) +{ + staticmethod *sm = _PyStaticMethod_CAST(self); + return descriptor_get_wrapped_attribute(sm->sm_callable, self, &_Py_ID(__annotations__)); +} + +static int +sm_set___annotations__(PyObject *self, PyObject *value, void *closure) +{ + return descriptor_set_wrapped_attribute(self, &_Py_ID(__annotations__), value, "staticmethod"); +} + +static PyObject * +sm_get___annotate__(PyObject *self, void *closure) +{ + staticmethod *sm = _PyStaticMethod_CAST(self); + return descriptor_get_wrapped_attribute(sm->sm_callable, self, &_Py_ID(__annotate__)); +} + +static int +sm_set___annotate__(PyObject *self, PyObject *value, void *closure) +{ + return descriptor_set_wrapped_attribute(self, &_Py_ID(__annotate__), value, "staticmethod"); +} + static PyGetSetDef sm_getsetlist[] = { - {"__isabstractmethod__", - (getter)sm_get___isabstractmethod__, NULL, NULL, NULL}, + {"__isabstractmethod__", sm_get___isabstractmethod__, NULL, NULL, NULL}, {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, + {"__annotations__", sm_get___annotations__, sm_set___annotations__, NULL, NULL}, + {"__annotate__", sm_get___annotate__, sm_set___annotate__, NULL, NULL}, {NULL} /* Sentinel */ }; static PyObject* -sm_repr(staticmethod *sm) +sm_repr(PyObject *self) { + staticmethod *sm = _PyStaticMethod_CAST(self); return PyUnicode_FromFormat("", sm->sm_callable); } @@ -1404,12 +1705,12 @@ PyTypeObject PyStaticMethod_Type = { "staticmethod", sizeof(staticmethod), 0, - (destructor)sm_dealloc, /* tp_dealloc */ + sm_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)sm_repr, /* tp_repr */ + sm_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -1421,14 +1722,14 @@ PyTypeObject PyStaticMethod_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, staticmethod_doc, /* tp_doc */ - (traverseproc)sm_traverse, /* tp_traverse */ - (inquiry)sm_clear, /* tp_clear */ + sm_traverse, /* tp_traverse */ + sm_clear, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ 0, /* tp_methods */ - sm_memberlist, /* tp_members */ + sm_memberlist, /* tp_members */ sm_getsetlist, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index c045d495e85526..64b4e2645cbaee 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -4,6 +4,7 @@ #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" +#include "pycore_typevarobject.h" // _Py_typing_type_repr #include "pycore_unionobject.h" // _Py_union_type_or, _PyGenericAlias_Check @@ -51,96 +52,29 @@ ga_traverse(PyObject *self, visitproc visit, void *arg) } static int -ga_repr_item(_PyUnicodeWriter *writer, PyObject *p) -{ - PyObject *qualname = NULL; - PyObject *module = NULL; - PyObject *r = NULL; - int rc; - - if (p == Py_Ellipsis) { - // The Ellipsis object - r = PyUnicode_FromString("..."); - goto done; - } - - if ((rc = PyObject_HasAttrWithError(p, &_Py_ID(__origin__))) > 0 && - (rc = PyObject_HasAttrWithError(p, &_Py_ID(__args__))) > 0) - { - // It looks like a GenericAlias - goto use_repr; - } - if (rc < 0) { - goto done; - } - - if (PyObject_GetOptionalAttr(p, &_Py_ID(__qualname__), &qualname) < 0) { - goto done; - } - if (qualname == NULL) { - goto use_repr; - } - if (PyObject_GetOptionalAttr(p, &_Py_ID(__module__), &module) < 0) { - goto done; - } - if (module == NULL || module == Py_None) { - goto use_repr; - } - - // Looks like a class - if (PyUnicode_Check(module) && - _PyUnicode_EqualToASCIIString(module, "builtins")) - { - // builtins don't need a module name - r = PyObject_Str(qualname); - goto done; - } - else { - r = PyUnicode_FromFormat("%S.%S", module, qualname); - goto done; - } - -use_repr: - r = PyObject_Repr(p); - -done: - Py_XDECREF(qualname); - Py_XDECREF(module); - if (r == NULL) { - // error if any of the above PyObject_Repr/PyUnicode_From* fail - rc = -1; - } - else { - rc = _PyUnicodeWriter_WriteStr(writer, r); - Py_DECREF(r); - } - return rc; -} - -static int -ga_repr_items_list(_PyUnicodeWriter *writer, PyObject *p) +ga_repr_items_list(PyUnicodeWriter *writer, PyObject *p) { assert(PyList_CheckExact(p)); Py_ssize_t len = PyList_GET_SIZE(p); - if (_PyUnicodeWriter_WriteASCIIString(writer, "[", 1) < 0) { + if (PyUnicodeWriter_WriteChar(writer, '[') < 0) { return -1; } for (Py_ssize_t i = 0; i < len; i++) { if (i > 0) { - if (_PyUnicodeWriter_WriteASCIIString(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { return -1; } } PyObject *item = PyList_GET_ITEM(p, i); - if (ga_repr_item(writer, item) < 0) { + if (_Py_typing_type_repr(writer, item) < 0) { return -1; } } - if (_PyUnicodeWriter_WriteASCIIString(writer, "]", 1) < 0) { + if (PyUnicodeWriter_WriteChar(writer, ']') < 0) { return -1; } @@ -153,49 +87,55 @@ ga_repr(PyObject *self) gaobject *alias = (gaobject *)self; Py_ssize_t len = PyTuple_GET_SIZE(alias->args); - _PyUnicodeWriter writer; - _PyUnicodeWriter_Init(&writer); + // Estimation based on the shortest format: "int[int, int, int]" + Py_ssize_t estimate = (len <= PY_SSIZE_T_MAX / 5) ? len * 5 : len; + estimate = 3 + 1 + estimate + 1; + PyUnicodeWriter *writer = PyUnicodeWriter_Create(estimate); + if (writer == NULL) { + return NULL; + } if (alias->starred) { - if (_PyUnicodeWriter_WriteASCIIString(&writer, "*", 1) < 0) { + if (PyUnicodeWriter_WriteChar(writer, '*') < 0) { goto error; } } - if (ga_repr_item(&writer, alias->origin) < 0) { + if (_Py_typing_type_repr(writer, alias->origin) < 0) { goto error; } - if (_PyUnicodeWriter_WriteASCIIString(&writer, "[", 1) < 0) { + if (PyUnicodeWriter_WriteChar(writer, '[') < 0) { goto error; } for (Py_ssize_t i = 0; i < len; i++) { if (i > 0) { - if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { goto error; } } PyObject *p = PyTuple_GET_ITEM(alias->args, i); if (PyList_CheckExact(p)) { // Looks like we are working with ParamSpec's list of type args: - if (ga_repr_items_list(&writer, p) < 0) { + if (ga_repr_items_list(writer, p) < 0) { goto error; } } - else if (ga_repr_item(&writer, p) < 0) { + else if (_Py_typing_type_repr(writer, p) < 0) { goto error; } } if (len == 0) { // for something like tuple[()] we should print a "()" - if (_PyUnicodeWriter_WriteASCIIString(&writer, "()", 2) < 0) { + if (PyUnicodeWriter_WriteUTF8(writer, "()", 2) < 0) { goto error; } } - if (_PyUnicodeWriter_WriteASCIIString(&writer, "]", 1) < 0) { + if (PyUnicodeWriter_WriteChar(writer, ']') < 0) { goto error; } - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); + error: - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); return NULL; } @@ -537,6 +477,8 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje } PyDoc_STRVAR(genericalias__doc__, +"GenericAlias(origin, args, /)\n" +"--\n\n" "Represent a PEP 585 generic type\n" "\n" "E.g. for t = list[int], t.__origin__ is list and t.__args__ is (int,)."); @@ -559,6 +501,10 @@ ga_getitem(PyObject *self, PyObject *item) } PyObject *res = Py_GenericAlias(alias->origin, newargs); + if (res == NULL) { + Py_DECREF(newargs); + return NULL; + } ((gaobject *)res)->starred = alias->starred; Py_DECREF(newargs); diff --git a/Objects/genobject.c b/Objects/genobject.c index 8d1dbb72ba9ec2..73bbf86588c457 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -6,19 +6,31 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_EvalFrame() #include "pycore_frame.h" // _PyInterpreterFrame +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_gc.h" // _PyGC_CLEAR_FINALIZED() -#include "pycore_genobject.h" // struct _Py_async_gen_freelist #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_opcode_utils.h" // RESUME_AFTER_YIELD_FROM +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pystats.h" -static PyObject *gen_close(PyGenObject *, PyObject *); -static PyObject *async_gen_asend_new(PyAsyncGenObject *, PyObject *); -static PyObject *async_gen_athrow_new(PyAsyncGenObject *, PyObject *); +// Forward declarations +static PyObject* gen_close(PyObject *, PyObject *); +static PyObject* async_gen_asend_new(PyAsyncGenObject *, PyObject *); +static PyObject* async_gen_athrow_new(PyAsyncGenObject *, PyObject *); + + +#define _PyGen_CAST(op) \ + _Py_CAST(PyGenObject*, (op)) +#define _PyCoroObject_CAST(op) \ + (assert(PyCoro_CheckExact(op)), \ + _Py_CAST(PyCoroObject*, (op))) +#define _PyAsyncGenObject_CAST(op) \ + _Py_CAST(PyAsyncGenObject*, (op)) + static const char *NON_INIT_CORO_MSG = "can't send non-None value to a " "just-started coroutine"; @@ -29,8 +41,7 @@ static const char *ASYNC_GEN_IGNORED_EXIT_MSG = /* Returns a borrowed reference */ static inline PyCodeObject * _PyGen_GetCode(PyGenObject *gen) { - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)(gen->gi_iframe); - return _PyFrame_GetCode(frame); + return _PyFrame_GetCode(&gen->gi_iframe); } PyCodeObject * @@ -42,12 +53,13 @@ PyGen_GetCode(PyGenObject *gen) { } static int -gen_traverse(PyGenObject *gen, visitproc visit, void *arg) +gen_traverse(PyObject *self, visitproc visit, void *arg) { + PyGenObject *gen = _PyGen_CAST(self); Py_VISIT(gen->gi_name); Py_VISIT(gen->gi_qualname); if (gen->gi_frame_state != FRAME_CLEARED) { - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)(gen->gi_iframe); + _PyInterpreterFrame *frame = &gen->gi_iframe; assert(frame->frame_obj == NULL || frame->frame_obj->f_frame->owner == FRAME_OWNED_BY_GENERATOR); int err = _PyFrame_Traverse(frame, visit, arg); @@ -55,6 +67,11 @@ gen_traverse(PyGenObject *gen, visitproc visit, void *arg) return err; } } + else { + // We still need to visit the code object when the frame is cleared to + // ensure that it's kept alive if the reference is deferred. + _Py_VISIT_STACKREF(gen->gi_iframe.f_executable); + } /* No need to visit cr_origin, because it's just tuples/str/int, so can't participate in a reference cycle. */ Py_VISIT(gen->gi_exc_state.exc_value); @@ -102,7 +119,7 @@ _PyGen_Finalize(PyObject *self) _PyErr_WarnUnawaitedCoroutine((PyObject *)gen); } else { - PyObject *res = gen_close(gen, NULL); + PyObject *res = gen_close((PyObject*)gen, NULL); if (res == NULL) { if (PyErr_Occurred()) { PyErr_WriteUnraisable(self); @@ -118,9 +135,22 @@ _PyGen_Finalize(PyObject *self) } static void -gen_dealloc(PyGenObject *gen) +gen_clear_frame(PyGenObject *gen) { - PyObject *self = (PyObject *) gen; + if (gen->gi_frame_state == FRAME_CLEARED) + return; + + gen->gi_frame_state = FRAME_CLEARED; + _PyInterpreterFrame *frame = &gen->gi_iframe; + frame->previous = NULL; + _PyFrame_ClearExceptCode(frame); + _PyErr_ClearExcState(&gen->gi_exc_state); +} + +static void +gen_dealloc(PyObject *self) +{ + PyGenObject *gen = _PyGen_CAST(self); _PyObject_GC_UNTRACK(gen); @@ -139,18 +169,12 @@ gen_dealloc(PyGenObject *gen) and GC_Del. */ Py_CLEAR(((PyAsyncGenObject*)gen)->ag_origin_or_finalizer); } - if (gen->gi_frame_state != FRAME_CLEARED) { - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; - gen->gi_frame_state = FRAME_CLEARED; - frame->previous = NULL; - _PyFrame_ClearExceptCode(frame); - _PyErr_ClearExcState(&gen->gi_exc_state); - } - assert(gen->gi_exc_state.exc_value == NULL); - if (_PyGen_GetCode(gen)->co_flags & CO_COROUTINE) { + if (PyCoro_CheckExact(gen)) { Py_CLEAR(((PyCoroObject *)gen)->cr_origin_or_finalizer); } - Py_DECREF(_PyGen_GetCode(gen)); + gen_clear_frame(gen); + assert(gen->gi_exc_state.exc_value == NULL); + PyStackRef_CLEAR(gen->gi_iframe.f_executable); Py_CLEAR(gen->gi_name); Py_CLEAR(gen->gi_qualname); @@ -162,7 +186,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, int exc, int closing) { PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; + _PyInterpreterFrame *frame = &gen->gi_iframe; *presult = NULL; if (gen->gi_frame_state == FRAME_CREATED && arg && arg != Py_None) { @@ -212,7 +236,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, /* Push arg onto the frame's value stack */ PyObject *arg_obj = arg ? arg : Py_None; - _PyFrame_StackPush(frame, Py_NewRef(arg_obj)); + _PyFrame_StackPush(frame, PyStackRef_FromPyObjectNew(arg_obj)); _PyErr_StackItem *prev_exc_info = tstate->exc_info; gen->gi_exc_state.previous_item = prev_exc_info; @@ -257,8 +281,9 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, } static PySendResult -PyGen_am_send(PyGenObject *gen, PyObject *arg, PyObject **result) +PyGen_am_send(PyObject *self, PyObject *arg, PyObject **result) { + PyGenObject *gen = _PyGen_CAST(self); return gen_send_ex2(gen, arg, result, 0, 0); } @@ -287,9 +312,9 @@ PyDoc_STRVAR(send_doc, return next yielded value or raise StopIteration."); static PyObject * -gen_send(PyGenObject *gen, PyObject *arg) +gen_send(PyObject *gen, PyObject *arg) { - return gen_send_ex(gen, arg, 0, 0); + return gen_send_ex((PyGenObject*)gen, arg, 0, 0); } PyDoc_STRVAR(close_doc, @@ -306,7 +331,7 @@ gen_close_iter(PyObject *yf) PyObject *retval = NULL; if (PyGen_CheckExact(yf) || PyCoro_CheckExact(yf)) { - retval = gen_close((PyGenObject *)yf, NULL); + retval = gen_close((PyObject *)yf, NULL); if (retval == NULL) return -1; } @@ -329,10 +354,11 @@ gen_close_iter(PyObject *yf) static inline bool is_resume(_Py_CODEUNIT *instr) { + uint8_t code = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.code); return ( - instr->op.code == RESUME || - instr->op.code == RESUME_CHECK || - instr->op.code == INSTRUMENTED_RESUME + code == RESUME || + code == RESUME_CHECK || + code == INSTRUMENTED_RESUME ); } @@ -340,20 +366,19 @@ PyObject * _PyGen_yf(PyGenObject *gen) { if (gen->gi_frame_state == FRAME_SUSPENDED_YIELD_FROM) { - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; - assert(is_resume(frame->instr_ptr)); - assert((frame->instr_ptr->op.arg & RESUME_OPARG_LOCATION_MASK) >= RESUME_AFTER_YIELD_FROM); - return Py_NewRef(_PyFrame_StackPeek(frame)); + _PyInterpreterFrame *frame = &gen->gi_iframe; + // GH-122390: These asserts are wrong in the presence of ENTER_EXECUTOR! + // assert(is_resume(frame->instr_ptr)); + // assert((frame->instr_ptr->op.arg & RESUME_OPARG_LOCATION_MASK) >= RESUME_AFTER_YIELD_FROM); + return PyStackRef_AsPyObjectNew(_PyFrame_StackPeek(frame)); } return NULL; } static PyObject * -gen_close(PyGenObject *gen, PyObject *args) +gen_close(PyObject *self, PyObject *args) { - PyObject *retval; - int err = 0; - + PyGenObject *gen = _PyGen_CAST(self); if (gen->gi_frame_state == FRAME_CREATED) { gen->gi_frame_state = FRAME_COMPLETED; @@ -362,7 +387,9 @@ gen_close(PyGenObject *gen, PyObject *args) if (FRAME_STATE_FINISHED(gen->gi_frame_state)) { Py_RETURN_NONE; } + PyObject *yf = _PyGen_yf(gen); + int err = 0; if (yf) { PyFrameState state = gen->gi_frame_state; gen->gi_frame_state = FRAME_EXECUTING; @@ -370,7 +397,7 @@ gen_close(PyGenObject *gen, PyObject *args) gen->gi_frame_state = state; Py_DECREF(yf); } - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; + _PyInterpreterFrame *frame = &gen->gi_iframe; if (is_resume(frame->instr_ptr)) { /* We can safely ignore the outermost try block * as it is automatically generated to handle @@ -380,13 +407,15 @@ gen_close(PyGenObject *gen, PyObject *args) // RESUME after YIELD_VALUE and exception depth is 1 assert((oparg & RESUME_OPARG_LOCATION_MASK) != RESUME_AT_FUNC_START); gen->gi_frame_state = FRAME_COMPLETED; + gen_clear_frame(gen); Py_RETURN_NONE; } } if (err == 0) { PyErr_SetNone(PyExc_GeneratorExit); } - retval = gen_send_ex(gen, Py_None, 1, 1); + + PyObject *retval = gen_send_ex(gen, Py_None, 1, 1); if (retval) { const char *msg = "generator ignored GeneratorExit"; if (PyCoro_CheckExact(gen)) { @@ -399,10 +428,12 @@ gen_close(PyGenObject *gen, PyObject *args) return NULL; } assert(PyErr_Occurred()); + if (PyErr_ExceptionMatches(PyExc_GeneratorExit)) { PyErr_Clear(); /* ignore this error */ Py_RETURN_NONE; } + /* if the generator returned a value while closing, StopIteration was * raised in gen_send_ex() above; retrieve and return the value here */ if (_PyGen_FetchStopIterationValue(&retval) == 0) { @@ -428,7 +459,7 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, PyObject *yf = _PyGen_yf(gen); if (yf) { - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; + _PyInterpreterFrame *frame = &gen->gi_iframe; PyObject *ret; int err; if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit) && @@ -447,14 +478,14 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, return gen_send_ex(gen, Py_None, 1, 0); goto throw_here; } + PyThreadState *tstate = _PyThreadState_GET(); + assert(tstate != NULL); if (PyGen_CheckExact(yf) || PyCoro_CheckExact(yf)) { /* `yf` is a generator or a coroutine. */ - PyThreadState *tstate = _PyThreadState_GET(); - /* Since we are fast-tracking things by skipping the eval loop, - we need to update the current frame so the stack trace - will be reported correctly to the user. */ - /* XXX We should probably be updating the current frame - somewhere in ceval.c. */ + + /* Link frame into the stack to enable complete backtraces. */ + /* XXX We should probably be updating the current frame somewhere in + ceval.c. */ _PyInterpreterFrame *prev = tstate->current_frame; frame->previous = prev; tstate->current_frame = frame; @@ -478,10 +509,16 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, Py_DECREF(yf); goto throw_here; } + + _PyInterpreterFrame *prev = tstate->current_frame; + frame->previous = prev; + tstate->current_frame = frame; PyFrameState state = gen->gi_frame_state; gen->gi_frame_state = FRAME_EXECUTING; ret = PyObject_CallFunctionObjArgs(meth, typ, val, tb, NULL); gen->gi_frame_state = state; + tstate->current_frame = prev; + frame->previous = NULL; Py_DECREF(meth); } Py_DECREF(yf); @@ -579,10 +616,12 @@ gen_throw(PyGenObject *gen, PyObject *const *args, Py_ssize_t nargs) static PyObject * -gen_iternext(PyGenObject *gen) +gen_iternext(PyObject *self) { + assert(PyGen_CheckExact(self) || PyCoro_CheckExact(self)); + PyGenObject *gen = _PyGen_CAST(self); + PyObject *result; - assert(PyGen_CheckExact(gen) || PyCoro_CheckExact(gen)); if (gen_send_ex2(gen, NULL, &result, 0, 0) == PYGEN_RETURN) { if (result != Py_None) { _PyGen_SetStopIterationValue(result); @@ -601,30 +640,19 @@ gen_iternext(PyGenObject *gen) int _PyGen_SetStopIterationValue(PyObject *value) { - PyObject *e; - - if (value == NULL || - (!PyTuple_Check(value) && !PyExceptionInstance_Check(value))) - { - /* Delay exception instantiation if we can */ - PyErr_SetObject(PyExc_StopIteration, value); - return 0; - } - /* Construct an exception instance manually with - * PyObject_CallOneArg and pass it to PyErr_SetObject. - * - * We do this to handle a situation when "value" is a tuple, in which - * case PyErr_SetObject would set the value of StopIteration to - * the first element of the tuple. - * - * (See PyErr_SetObject/_PyErr_CreateException code for details.) - */ - e = PyObject_CallOneArg(PyExc_StopIteration, value); - if (e == NULL) { + assert(!PyErr_Occurred()); + // Construct an exception instance manually with PyObject_CallOneArg() + // but use PyErr_SetRaisedException() instead of PyErr_SetObject() as + // PyErr_SetObject(exc_type, value) has a fast path when 'value' + // is a tuple, where the value of the StopIteration exception would be + // set to 'value[0]' instead of 'value'. + PyObject *exc = value == NULL + ? PyObject_CallNoArgs(PyExc_StopIteration) + : PyObject_CallOneArg(PyExc_StopIteration, value); + if (exc == NULL) { return -1; } - PyErr_SetObject(PyExc_StopIteration, e); - Py_DECREF(e); + PyErr_SetRaisedException(exc /* stolen */); return 0; } @@ -656,21 +684,24 @@ _PyGen_FetchStopIterationValue(PyObject **pvalue) } static PyObject * -gen_repr(PyGenObject *gen) +gen_repr(PyObject *self) { + PyGenObject *gen = _PyGen_CAST(self); return PyUnicode_FromFormat("", gen->gi_qualname, gen); } static PyObject * -gen_get_name(PyGenObject *op, void *Py_UNUSED(ignored)) +gen_get_name(PyObject *self, void *Py_UNUSED(ignored)) { + PyGenObject *op = _PyGen_CAST(self); return Py_NewRef(op->gi_name); } static int -gen_set_name(PyGenObject *op, PyObject *value, void *Py_UNUSED(ignored)) +gen_set_name(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { + PyGenObject *op = _PyGen_CAST(self); /* Not legal to del gen.gi_name or to set it to anything * other than a string object. */ if (value == NULL || !PyUnicode_Check(value)) { @@ -683,14 +714,16 @@ gen_set_name(PyGenObject *op, PyObject *value, void *Py_UNUSED(ignored)) } static PyObject * -gen_get_qualname(PyGenObject *op, void *Py_UNUSED(ignored)) +gen_get_qualname(PyObject *self, void *Py_UNUSED(ignored)) { + PyGenObject *op = _PyGen_CAST(self); return Py_NewRef(op->gi_qualname); } static int -gen_set_qualname(PyGenObject *op, PyObject *value, void *Py_UNUSED(ignored)) +gen_set_qualname(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { + PyGenObject *op = _PyGen_CAST(self); /* Not legal to del gen.__qualname__ or to set it to anything * other than a string object. */ if (value == NULL || !PyUnicode_Check(value)) { @@ -703,18 +736,20 @@ gen_set_qualname(PyGenObject *op, PyObject *value, void *Py_UNUSED(ignored)) } static PyObject * -gen_getyieldfrom(PyGenObject *gen, void *Py_UNUSED(ignored)) +gen_getyieldfrom(PyObject *gen, void *Py_UNUSED(ignored)) { - PyObject *yf = _PyGen_yf(gen); - if (yf == NULL) + PyObject *yf = _PyGen_yf(_PyGen_CAST(gen)); + if (yf == NULL) { Py_RETURN_NONE; + } return yf; } static PyObject * -gen_getrunning(PyGenObject *gen, void *Py_UNUSED(ignored)) +gen_getrunning(PyObject *self, void *Py_UNUSED(ignored)) { + PyGenObject *gen = _PyGen_CAST(self); if (gen->gi_frame_state == FRAME_EXECUTING) { Py_RETURN_TRUE; } @@ -722,8 +757,9 @@ gen_getrunning(PyGenObject *gen, void *Py_UNUSED(ignored)) } static PyObject * -gen_getsuspended(PyGenObject *gen, void *Py_UNUSED(ignored)) +gen_getsuspended(PyObject *self, void *Py_UNUSED(ignored)) { + PyGenObject *gen = _PyGen_CAST(self); return PyBool_FromLong(FRAME_STATE_SUSPENDED(gen->gi_frame_state)); } @@ -736,12 +772,13 @@ _gen_getframe(PyGenObject *gen, const char *const name) if (FRAME_STATE_FINISHED(gen->gi_frame_state)) { Py_RETURN_NONE; } - return _Py_XNewRef((PyObject *)_PyFrame_GetFrameObject((_PyInterpreterFrame *)gen->gi_iframe)); + return _Py_XNewRef((PyObject *)_PyFrame_GetFrameObject(&gen->gi_iframe)); } static PyObject * -gen_getframe(PyGenObject *gen, void *Py_UNUSED(ignored)) +gen_getframe(PyObject *self, void *Py_UNUSED(ignored)) { + PyGenObject *gen = _PyGen_CAST(self); return _gen_getframe(gen, "gi_frame"); } @@ -755,22 +792,23 @@ _gen_getcode(PyGenObject *gen, const char *const name) } static PyObject * -gen_getcode(PyGenObject *gen, void *Py_UNUSED(ignored)) +gen_getcode(PyObject *self, void *Py_UNUSED(ignored)) { + PyGenObject *gen = _PyGen_CAST(self); return _gen_getcode(gen, "gi_code"); } static PyGetSetDef gen_getsetlist[] = { - {"__name__", (getter)gen_get_name, (setter)gen_set_name, + {"__name__", gen_get_name, gen_set_name, PyDoc_STR("name of the generator")}, - {"__qualname__", (getter)gen_get_qualname, (setter)gen_set_qualname, + {"__qualname__", gen_get_qualname, gen_set_qualname, PyDoc_STR("qualified name of the generator")}, - {"gi_yieldfrom", (getter)gen_getyieldfrom, NULL, + {"gi_yieldfrom", gen_getyieldfrom, NULL, PyDoc_STR("object being iterated by yield from, or None")}, - {"gi_running", (getter)gen_getrunning, NULL, NULL}, - {"gi_frame", (getter)gen_getframe, NULL, NULL}, - {"gi_suspended", (getter)gen_getsuspended, NULL, NULL}, - {"gi_code", (getter)gen_getcode, NULL, NULL}, + {"gi_running", gen_getrunning, NULL, NULL}, + {"gi_frame", gen_getframe, NULL, NULL}, + {"gi_suspended", gen_getsuspended, NULL, NULL}, + {"gi_code", gen_getcode, NULL, NULL}, {NULL} /* Sentinel */ }; @@ -792,10 +830,11 @@ PyDoc_STRVAR(sizeof__doc__, "gen.__sizeof__() -> size of gen in memory, in bytes"); static PyMethodDef gen_methods[] = { - {"send",(PyCFunction)gen_send, METH_O, send_doc}, - {"throw",_PyCFunction_CAST(gen_throw), METH_FASTCALL, throw_doc}, - {"close",(PyCFunction)gen_close, METH_NOARGS, close_doc}, + {"send", gen_send, METH_O, send_doc}, + {"throw", _PyCFunction_CAST(gen_throw), METH_FASTCALL, throw_doc}, + {"close", gen_close, METH_NOARGS, close_doc}, {"__sizeof__", (PyCFunction)gen_sizeof, METH_NOARGS, sizeof__doc__}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* Sentinel */ }; @@ -803,23 +842,22 @@ static PyAsyncMethods gen_as_async = { 0, /* am_await */ 0, /* am_aiter */ 0, /* am_anext */ - (sendfunc)PyGen_am_send, /* am_send */ + PyGen_am_send, /* am_send */ }; PyTypeObject PyGen_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "generator", /* tp_name */ - offsetof(PyGenObject, gi_iframe) + - offsetof(_PyInterpreterFrame, localsplus), /* tp_basicsize */ + offsetof(PyGenObject, gi_iframe.localsplus), /* tp_basicsize */ sizeof(PyObject *), /* tp_itemsize */ /* methods */ - (destructor)gen_dealloc, /* tp_dealloc */ + gen_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ &gen_as_async, /* tp_as_async */ - (reprfunc)gen_repr, /* tp_repr */ + gen_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -831,12 +869,12 @@ PyTypeObject PyGen_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)gen_traverse, /* tp_traverse */ + gen_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ offsetof(PyGenObject, gi_weakreflist), /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)gen_iternext, /* tp_iternext */ + gen_iternext, /* tp_iternext */ gen_methods, /* tp_methods */ gen_memberlist, /* tp_members */ gen_getsetlist, /* tp_getset */ @@ -895,17 +933,18 @@ _Py_MakeCoro(PyFunctionObject *func) return make_gen(&PyGen_Type, func); } if (coro_flags == CO_ASYNC_GENERATOR) { - PyAsyncGenObject *o; - o = (PyAsyncGenObject *)make_gen(&PyAsyncGen_Type, func); - if (o == NULL) { + PyAsyncGenObject *ag; + ag = (PyAsyncGenObject *)make_gen(&PyAsyncGen_Type, func); + if (ag == NULL) { return NULL; } - o->ag_origin_or_finalizer = NULL; - o->ag_closed = 0; - o->ag_hooks_inited = 0; - o->ag_running_async = 0; - return (PyObject*)o; + ag->ag_origin_or_finalizer = NULL; + ag->ag_closed = 0; + ag->ag_hooks_inited = 0; + ag->ag_running_async = 0; + return (PyObject*)ag; } + assert (coro_flags == CO_COROUTINE); PyObject *coro = make_gen(&PyCoro_Type, func); if (!coro) { @@ -945,7 +984,7 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, /* Copy the frame */ assert(f->f_frame->frame_obj == NULL); assert(f->f_frame->owner == FRAME_OWNED_BY_FRAME_OBJECT); - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; + _PyInterpreterFrame *frame = &gen->gi_iframe; _PyFrame_Copy((_PyInterpreterFrame *)f->_f_frame_data, frame); gen->gi_frame_state = FRAME_CREATED; assert(frame->frame_obj == f); @@ -987,6 +1026,11 @@ typedef struct { PyCoroObject *cw_coroutine; } PyCoroWrapper; +#define _PyCoroWrapper_CAST(op) \ + (assert(Py_IS_TYPE((op), &_PyCoroWrapper_Type)), \ + _Py_CAST(PyCoroWrapper*, (op))) + + static int gen_is_coroutine(PyObject *o) { @@ -1043,20 +1087,21 @@ _PyCoro_GetAwaitableIter(PyObject *o) } PyErr_Format(PyExc_TypeError, - "object %.100s can't be used in 'await' expression", + "'%.100s' object can't be awaited", ot->tp_name); return NULL; } static PyObject * -coro_repr(PyCoroObject *coro) +coro_repr(PyObject *self) { + PyCoroObject *coro = _PyCoroObject_CAST(self); return PyUnicode_FromFormat("", coro->cr_qualname, coro); } static PyObject * -coro_await(PyCoroObject *coro) +coro_await(PyObject *coro) { PyCoroWrapper *cw = PyObject_GC_New(PyCoroWrapper, &_PyCoroWrapper_Type); if (cw == NULL) { @@ -1068,7 +1113,7 @@ coro_await(PyCoroObject *coro) } static PyObject * -coro_get_cr_await(PyCoroObject *coro, void *Py_UNUSED(ignored)) +coro_get_cr_await(PyObject *coro, void *Py_UNUSED(ignored)) { PyObject *yf = _PyGen_yf((PyGenObject *) coro); if (yf == NULL) @@ -1077,8 +1122,9 @@ coro_get_cr_await(PyCoroObject *coro, void *Py_UNUSED(ignored)) } static PyObject * -cr_getsuspended(PyCoroObject *coro, void *Py_UNUSED(ignored)) +cr_getsuspended(PyObject *self, void *Py_UNUSED(ignored)) { + PyCoroObject *coro = _PyCoroObject_CAST(self); if (FRAME_STATE_SUSPENDED(coro->cr_frame_state)) { Py_RETURN_TRUE; } @@ -1086,8 +1132,9 @@ cr_getsuspended(PyCoroObject *coro, void *Py_UNUSED(ignored)) } static PyObject * -cr_getrunning(PyCoroObject *coro, void *Py_UNUSED(ignored)) +cr_getrunning(PyObject *self, void *Py_UNUSED(ignored)) { + PyCoroObject *coro = _PyCoroObject_CAST(self); if (coro->cr_frame_state == FRAME_EXECUTING) { Py_RETURN_TRUE; } @@ -1095,29 +1142,28 @@ cr_getrunning(PyCoroObject *coro, void *Py_UNUSED(ignored)) } static PyObject * -cr_getframe(PyCoroObject *coro, void *Py_UNUSED(ignored)) +cr_getframe(PyObject *coro, void *Py_UNUSED(ignored)) { - return _gen_getframe((PyGenObject *)coro, "cr_frame"); + return _gen_getframe(_PyGen_CAST(coro), "cr_frame"); } static PyObject * -cr_getcode(PyCoroObject *coro, void *Py_UNUSED(ignored)) +cr_getcode(PyObject *coro, void *Py_UNUSED(ignored)) { - return _gen_getcode((PyGenObject *)coro, "cr_code"); + return _gen_getcode(_PyGen_CAST(coro), "cr_code"); } - static PyGetSetDef coro_getsetlist[] = { - {"__name__", (getter)gen_get_name, (setter)gen_set_name, + {"__name__", gen_get_name, gen_set_name, PyDoc_STR("name of the coroutine")}, - {"__qualname__", (getter)gen_get_qualname, (setter)gen_set_qualname, + {"__qualname__", gen_get_qualname, gen_set_qualname, PyDoc_STR("qualified name of the coroutine")}, - {"cr_await", (getter)coro_get_cr_await, NULL, + {"cr_await", coro_get_cr_await, NULL, PyDoc_STR("object being awaited on, or None")}, - {"cr_running", (getter)cr_getrunning, NULL, NULL}, - {"cr_frame", (getter)cr_getframe, NULL, NULL}, - {"cr_code", (getter)cr_getcode, NULL, NULL}, - {"cr_suspended", (getter)cr_getsuspended, NULL, NULL}, + {"cr_running", cr_getrunning, NULL, NULL}, + {"cr_frame", cr_getframe, NULL, NULL}, + {"cr_code", cr_getcode, NULL, NULL}, + {"cr_suspended", cr_getsuspended, NULL, NULL}, {NULL} /* Sentinel */ }; @@ -1144,33 +1190,33 @@ PyDoc_STRVAR(coro_close_doc, "close() -> raise GeneratorExit inside coroutine."); static PyMethodDef coro_methods[] = { - {"send",(PyCFunction)gen_send, METH_O, coro_send_doc}, + {"send", gen_send, METH_O, coro_send_doc}, {"throw",_PyCFunction_CAST(gen_throw), METH_FASTCALL, coro_throw_doc}, - {"close",(PyCFunction)gen_close, METH_NOARGS, coro_close_doc}, + {"close", gen_close, METH_NOARGS, coro_close_doc}, {"__sizeof__", (PyCFunction)gen_sizeof, METH_NOARGS, sizeof__doc__}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* Sentinel */ }; static PyAsyncMethods coro_as_async = { - (unaryfunc)coro_await, /* am_await */ + coro_await, /* am_await */ 0, /* am_aiter */ 0, /* am_anext */ - (sendfunc)PyGen_am_send, /* am_send */ + PyGen_am_send, /* am_send */ }; PyTypeObject PyCoro_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "coroutine", /* tp_name */ - offsetof(PyCoroObject, cr_iframe) + - offsetof(_PyInterpreterFrame, localsplus), /* tp_basicsize */ + offsetof(PyCoroObject, cr_iframe.localsplus),/* tp_basicsize */ sizeof(PyObject *), /* tp_itemsize */ /* methods */ - (destructor)gen_dealloc, /* tp_dealloc */ + gen_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ &coro_as_async, /* tp_as_async */ - (reprfunc)coro_repr, /* tp_repr */ + coro_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -1182,7 +1228,7 @@ PyTypeObject PyCoro_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)gen_traverse, /* tp_traverse */ + gen_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ offsetof(PyCoroObject, cr_weakreflist), /* tp_weaklistoffset */ @@ -1212,49 +1258,55 @@ PyTypeObject PyCoro_Type = { }; static void -coro_wrapper_dealloc(PyCoroWrapper *cw) +coro_wrapper_dealloc(PyObject *self) { + PyCoroWrapper *cw = _PyCoroWrapper_CAST(self); _PyObject_GC_UNTRACK((PyObject *)cw); Py_CLEAR(cw->cw_coroutine); PyObject_GC_Del(cw); } static PyObject * -coro_wrapper_iternext(PyCoroWrapper *cw) +coro_wrapper_iternext(PyObject *self) { - return gen_iternext((PyGenObject *)cw->cw_coroutine); + PyCoroWrapper *cw = _PyCoroWrapper_CAST(self); + return gen_iternext((PyObject *)cw->cw_coroutine); } static PyObject * -coro_wrapper_send(PyCoroWrapper *cw, PyObject *arg) +coro_wrapper_send(PyObject *self, PyObject *arg) { - return gen_send((PyGenObject *)cw->cw_coroutine, arg); + PyCoroWrapper *cw = _PyCoroWrapper_CAST(self); + return gen_send((PyObject *)cw->cw_coroutine, arg); } static PyObject * -coro_wrapper_throw(PyCoroWrapper *cw, PyObject *const *args, Py_ssize_t nargs) +coro_wrapper_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { + PyCoroWrapper *cw = _PyCoroWrapper_CAST(self); return gen_throw((PyGenObject *)cw->cw_coroutine, args, nargs); } static PyObject * -coro_wrapper_close(PyCoroWrapper *cw, PyObject *args) +coro_wrapper_close(PyObject *self, PyObject *args) { - return gen_close((PyGenObject *)cw->cw_coroutine, args); + PyCoroWrapper *cw = _PyCoroWrapper_CAST(self); + return gen_close((PyObject *)cw->cw_coroutine, args); } static int -coro_wrapper_traverse(PyCoroWrapper *cw, visitproc visit, void *arg) +coro_wrapper_traverse(PyObject *self, visitproc visit, void *arg) { + PyCoroWrapper *cw = _PyCoroWrapper_CAST(self); Py_VISIT((PyObject *)cw->cw_coroutine); return 0; } static PyMethodDef coro_wrapper_methods[] = { - {"send",(PyCFunction)coro_wrapper_send, METH_O, coro_send_doc}, - {"throw",_PyCFunction_CAST(coro_wrapper_throw), - METH_FASTCALL, coro_throw_doc}, - {"close",(PyCFunction)coro_wrapper_close, METH_NOARGS, coro_close_doc}, + {"send", coro_wrapper_send, METH_O, coro_send_doc}, + {"throw", _PyCFunction_CAST(coro_wrapper_throw), METH_FASTCALL, + coro_throw_doc}, + {"close", coro_wrapper_close, METH_NOARGS, coro_close_doc}, {NULL, NULL} /* Sentinel */ }; @@ -1263,7 +1315,7 @@ PyTypeObject _PyCoroWrapper_Type = { "coroutine_wrapper", sizeof(PyCoroWrapper), /* tp_basicsize */ 0, /* tp_itemsize */ - (destructor)coro_wrapper_dealloc, /* destructor tp_dealloc */ + coro_wrapper_dealloc, /* destructor tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -1280,12 +1332,12 @@ PyTypeObject _PyCoroWrapper_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ "A wrapper object implementing __await__ for coroutines.", - (traverseproc)coro_wrapper_traverse, /* tp_traverse */ + coro_wrapper_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)coro_wrapper_iternext, /* tp_iternext */ + coro_wrapper_iternext, /* tp_iternext */ coro_wrapper_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ @@ -1379,6 +1431,9 @@ typedef struct PyAsyncGenASend { AwaitableState ags_state; } PyAsyncGenASend; +#define _PyAsyncGenASend_CAST(op) \ + _Py_CAST(PyAsyncGenASend*, (op)) + typedef struct PyAsyncGenAThrow { PyObject_HEAD @@ -1400,19 +1455,24 @@ typedef struct _PyAsyncGenWrappedValue { #define _PyAsyncGenWrappedValue_CheckExact(o) \ Py_IS_TYPE(o, &_PyAsyncGenWrappedValue_Type) +#define _PyAsyncGenWrappedValue_CAST(op) \ + (assert(_PyAsyncGenWrappedValue_CheckExact(op)), \ + _Py_CAST(_PyAsyncGenWrappedValue*, (op))) static int -async_gen_traverse(PyAsyncGenObject *gen, visitproc visit, void *arg) +async_gen_traverse(PyObject *self, visitproc visit, void *arg) { - Py_VISIT(gen->ag_origin_or_finalizer); - return gen_traverse((PyGenObject*)gen, visit, arg); + PyAsyncGenObject *ag = _PyAsyncGenObject_CAST(self); + Py_VISIT(ag->ag_origin_or_finalizer); + return gen_traverse((PyObject*)ag, visit, arg); } static PyObject * -async_gen_repr(PyAsyncGenObject *o) +async_gen_repr(PyObject *self) { + PyAsyncGenObject *o = _PyAsyncGenObject_CAST(self); return PyUnicode_FromFormat("", o->ag_qualname, o); } @@ -1456,12 +1516,13 @@ async_gen_init_hooks(PyAsyncGenObject *o) static PyObject * -async_gen_anext(PyAsyncGenObject *o) +async_gen_anext(PyObject *self) { - if (async_gen_init_hooks(o)) { + PyAsyncGenObject *ag = _PyAsyncGenObject_CAST(self); + if (async_gen_init_hooks(ag)) { return NULL; } - return async_gen_asend_new(o, NULL); + return async_gen_asend_new(ag, NULL); } @@ -1502,20 +1563,21 @@ async_gen_athrow(PyAsyncGenObject *o, PyObject *args) } static PyObject * -ag_getframe(PyAsyncGenObject *ag, void *Py_UNUSED(ignored)) +ag_getframe(PyObject *ag, void *Py_UNUSED(ignored)) { return _gen_getframe((PyGenObject *)ag, "ag_frame"); } static PyObject * -ag_getcode(PyGenObject *gen, void *Py_UNUSED(ignored)) +ag_getcode(PyObject *gen, void *Py_UNUSED(ignored)) { - return _gen_getcode(gen, "ag_code"); + return _gen_getcode((PyGenObject*)gen, "ag_code"); } static PyObject * -ag_getsuspended(PyAsyncGenObject *ag, void *Py_UNUSED(ignored)) +ag_getsuspended(PyObject *self, void *Py_UNUSED(ignored)) { + PyAsyncGenObject *ag = _PyAsyncGenObject_CAST(self); if (FRAME_STATE_SUSPENDED(ag->ag_frame_state)) { Py_RETURN_TRUE; } @@ -1523,15 +1585,15 @@ ag_getsuspended(PyAsyncGenObject *ag, void *Py_UNUSED(ignored)) } static PyGetSetDef async_gen_getsetlist[] = { - {"__name__", (getter)gen_get_name, (setter)gen_set_name, + {"__name__", gen_get_name, gen_set_name, PyDoc_STR("name of the async generator")}, - {"__qualname__", (getter)gen_get_qualname, (setter)gen_set_qualname, + {"__qualname__", gen_get_qualname, gen_set_qualname, PyDoc_STR("qualified name of the async generator")}, - {"ag_await", (getter)coro_get_cr_await, NULL, + {"ag_await", coro_get_cr_await, NULL, PyDoc_STR("object being awaited on, or None")}, - {"ag_frame", (getter)ag_getframe, NULL, NULL}, - {"ag_code", (getter)ag_getcode, NULL, NULL}, - {"ag_suspended", (getter)ag_getsuspended, NULL, NULL}, + {"ag_frame", ag_getframe, NULL, NULL}, + {"ag_code", ag_getcode, NULL, NULL}, + {"ag_suspended", ag_getsuspended, NULL, NULL}, {NULL} /* Sentinel */ }; @@ -1569,24 +1631,23 @@ static PyMethodDef async_gen_methods[] = { static PyAsyncMethods async_gen_as_async = { 0, /* am_await */ PyObject_SelfIter, /* am_aiter */ - (unaryfunc)async_gen_anext, /* am_anext */ - (sendfunc)PyGen_am_send, /* am_send */ + async_gen_anext, /* am_anext */ + PyGen_am_send, /* am_send */ }; PyTypeObject PyAsyncGen_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "async_generator", /* tp_name */ - offsetof(PyAsyncGenObject, ag_iframe) + - offsetof(_PyInterpreterFrame, localsplus), /* tp_basicsize */ + offsetof(PyAsyncGenObject, ag_iframe.localsplus), /* tp_basicsize */ sizeof(PyObject *), /* tp_itemsize */ /* methods */ - (destructor)gen_dealloc, /* tp_dealloc */ + gen_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ &async_gen_as_async, /* tp_as_async */ - (reprfunc)async_gen_repr, /* tp_repr */ + async_gen_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -1598,7 +1659,7 @@ PyTypeObject PyAsyncGen_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)async_gen_traverse, /* tp_traverse */ + async_gen_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ offsetof(PyAsyncGenObject, ag_weakreflist), /* tp_weaklistoffset */ @@ -1628,67 +1689,21 @@ PyTypeObject PyAsyncGen_Type = { }; -#ifdef WITH_FREELISTS -static struct _Py_async_gen_freelist * -get_async_gen_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - return &freelists->async_gens; -} - -static struct _Py_async_gen_asend_freelist * -get_async_gen_asend_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - return &freelists->async_gen_asends; -} -#endif - - PyObject * PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname) { - PyAsyncGenObject *o; - o = (PyAsyncGenObject *)gen_new_with_qualname( - &PyAsyncGen_Type, f, name, qualname); - if (o == NULL) { + PyAsyncGenObject *ag; + ag = (PyAsyncGenObject *)gen_new_with_qualname(&PyAsyncGen_Type, f, + name, qualname); + if (ag == NULL) { return NULL; } - o->ag_origin_or_finalizer = NULL; - o->ag_closed = 0; - o->ag_hooks_inited = 0; - o->ag_running_async = 0; - return (PyObject*)o; -} - - -void -_PyAsyncGen_ClearFreeLists(struct _Py_object_freelists *freelist_state, int is_finalization) -{ -#ifdef WITH_FREELISTS - struct _Py_async_gen_freelist *freelist = &freelist_state->async_gens; - - while (freelist->numfree > 0) { - _PyAsyncGenWrappedValue *o; - o = freelist->items[--freelist->numfree]; - assert(_PyAsyncGenWrappedValue_CheckExact(o)); - PyObject_GC_Del(o); - } - - struct _Py_async_gen_asend_freelist *asend_freelist = &freelist_state->async_gen_asends; - - while (asend_freelist->numfree > 0) { - PyAsyncGenASend *o; - o = asend_freelist->items[--asend_freelist->numfree]; - assert(Py_IS_TYPE(o, &_PyAsyncGenASend_Type)); - PyObject_GC_Del(o); - } - if (is_finalization) { - freelist->numfree = -1; - asend_freelist->numfree = -1; - } -#endif + ag->ag_origin_or_finalizer = NULL; + ag->ag_closed = 0; + ag->ag_hooks_inited = 0; + ag->ag_running_async = 0; + return (PyObject*)ag; } static PyObject * @@ -1725,43 +1740,38 @@ async_gen_unwrap_value(PyAsyncGenObject *gen, PyObject *result) static void -async_gen_asend_dealloc(PyAsyncGenASend *o) +async_gen_asend_dealloc(PyObject *self) { - if (PyObject_CallFinalizerFromDealloc((PyObject *)o)) { + assert(PyAsyncGenASend_CheckExact(self)); + PyAsyncGenASend *ags = _PyAsyncGenASend_CAST(self); + + if (PyObject_CallFinalizerFromDealloc(self)) { return; } - _PyObject_GC_UNTRACK((PyObject *)o); - Py_CLEAR(o->ags_gen); - Py_CLEAR(o->ags_sendval); -#ifdef WITH_FREELISTS - struct _Py_async_gen_asend_freelist *freelist = get_async_gen_asend_freelist(); - if (freelist->numfree >= 0 && freelist->numfree < _PyAsyncGen_MAXFREELIST) { - assert(PyAsyncGenASend_CheckExact(o)); - _PyGC_CLEAR_FINALIZED((PyObject *)o); - freelist->items[freelist->numfree++] = o; - } - else -#endif - { - PyObject_GC_Del(o); - } + _PyObject_GC_UNTRACK(self); + Py_CLEAR(ags->ags_gen); + Py_CLEAR(ags->ags_sendval); + + _PyGC_CLEAR_FINALIZED(self); + + _Py_FREELIST_FREE(async_gen_asends, self, PyObject_GC_Del); } static int -async_gen_asend_traverse(PyAsyncGenASend *o, visitproc visit, void *arg) +async_gen_asend_traverse(PyObject *self, visitproc visit, void *arg) { - Py_VISIT(o->ags_gen); - Py_VISIT(o->ags_sendval); + PyAsyncGenASend *ags = _PyAsyncGenASend_CAST(self); + Py_VISIT(ags->ags_gen); + Py_VISIT(ags->ags_sendval); return 0; } static PyObject * -async_gen_asend_send(PyAsyncGenASend *o, PyObject *arg) +async_gen_asend_send(PyObject *self, PyObject *arg) { - PyObject *result; - + PyAsyncGenASend *o = _PyAsyncGenASend_CAST(self); if (o->ags_state == AWAITABLE_STATE_CLOSED) { PyErr_SetString( PyExc_RuntimeError, @@ -1771,6 +1781,7 @@ async_gen_asend_send(PyAsyncGenASend *o, PyObject *arg) if (o->ags_state == AWAITABLE_STATE_INIT) { if (o->ags_gen->ag_running_async) { + o->ags_state = AWAITABLE_STATE_CLOSED; PyErr_SetString( PyExc_RuntimeError, "anext(): asynchronous generator is already running"); @@ -1784,7 +1795,7 @@ async_gen_asend_send(PyAsyncGenASend *o, PyObject *arg) } o->ags_gen->ag_running_async = 1; - result = gen_send((PyGenObject*)o->ags_gen, arg); + PyObject *result = gen_send((PyObject*)o->ags_gen, arg); result = async_gen_unwrap_value(o->ags_gen, result); if (result == NULL) { @@ -1796,16 +1807,16 @@ async_gen_asend_send(PyAsyncGenASend *o, PyObject *arg) static PyObject * -async_gen_asend_iternext(PyAsyncGenASend *o) +async_gen_asend_iternext(PyObject *ags) { - return async_gen_asend_send(o, NULL); + return async_gen_asend_send(ags, NULL); } static PyObject * -async_gen_asend_throw(PyAsyncGenASend *o, PyObject *const *args, Py_ssize_t nargs) +async_gen_asend_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { - PyObject *result; + PyAsyncGenASend *o = _PyAsyncGenASend_CAST(self); if (o->ags_state == AWAITABLE_STATE_CLOSED) { PyErr_SetString( @@ -1814,10 +1825,24 @@ async_gen_asend_throw(PyAsyncGenASend *o, PyObject *const *args, Py_ssize_t narg return NULL; } - result = gen_throw((PyGenObject*)o->ags_gen, args, nargs); + if (o->ags_state == AWAITABLE_STATE_INIT) { + if (o->ags_gen->ag_running_async) { + o->ags_state = AWAITABLE_STATE_CLOSED; + PyErr_SetString( + PyExc_RuntimeError, + "anext(): asynchronous generator is already running"); + return NULL; + } + + o->ags_state = AWAITABLE_STATE_ITER; + o->ags_gen->ag_running_async = 1; + } + + PyObject *result = gen_throw((PyGenObject*)o->ags_gen, args, nargs); result = async_gen_unwrap_value(o->ags_gen, result); if (result == NULL) { + o->ags_gen->ag_running_async = 0; o->ags_state = AWAITABLE_STATE_CLOSED; } @@ -1826,24 +1851,43 @@ async_gen_asend_throw(PyAsyncGenASend *o, PyObject *const *args, Py_ssize_t narg static PyObject * -async_gen_asend_close(PyAsyncGenASend *o, PyObject *args) +async_gen_asend_close(PyObject *self, PyObject *args) { - o->ags_state = AWAITABLE_STATE_CLOSED; - Py_RETURN_NONE; + PyAsyncGenASend *o = _PyAsyncGenASend_CAST(self); + if (o->ags_state == AWAITABLE_STATE_CLOSED) { + Py_RETURN_NONE; + } + + PyObject *result = async_gen_asend_throw(self, &PyExc_GeneratorExit, 1); + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_StopIteration) || + PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || + PyErr_ExceptionMatches(PyExc_GeneratorExit)) + { + PyErr_Clear(); + Py_RETURN_NONE; + } + return result; + } + + Py_DECREF(result); + PyErr_SetString(PyExc_RuntimeError, "coroutine ignored GeneratorExit"); + return NULL; } static void -async_gen_asend_finalize(PyAsyncGenASend *o) +async_gen_asend_finalize(PyObject *self) { - if (o->ags_state == AWAITABLE_STATE_INIT) { - _PyErr_WarnUnawaitedAgenMethod(o->ags_gen, &_Py_ID(asend)); + PyAsyncGenASend *ags = _PyAsyncGenASend_CAST(self); + if (ags->ags_state == AWAITABLE_STATE_INIT) { + _PyErr_WarnUnawaitedAgenMethod(ags->ags_gen, &_Py_ID(asend)); } } static PyMethodDef async_gen_asend_methods[] = { - {"send", (PyCFunction)async_gen_asend_send, METH_O, send_doc}, + {"send", async_gen_asend_send, METH_O, send_doc}, {"throw", _PyCFunction_CAST(async_gen_asend_throw), METH_FASTCALL, throw_doc}, - {"close", (PyCFunction)async_gen_asend_close, METH_NOARGS, close_doc}, + {"close", async_gen_asend_close, METH_NOARGS, close_doc}, {NULL, NULL} /* Sentinel */ }; @@ -1862,7 +1906,7 @@ PyTypeObject _PyAsyncGenASend_Type = { sizeof(PyAsyncGenASend), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)async_gen_asend_dealloc, /* tp_dealloc */ + async_gen_asend_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -1879,12 +1923,12 @@ PyTypeObject _PyAsyncGenASend_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)async_gen_asend_traverse, /* tp_traverse */ + async_gen_asend_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)async_gen_asend_iternext, /* tp_iternext */ + async_gen_asend_iternext, /* tp_iternext */ async_gen_asend_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ @@ -1896,38 +1940,27 @@ PyTypeObject _PyAsyncGenASend_Type = { 0, /* tp_init */ 0, /* tp_alloc */ 0, /* tp_new */ - .tp_finalize = (destructor)async_gen_asend_finalize, + .tp_finalize = async_gen_asend_finalize, }; static PyObject * async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval) { - PyAsyncGenASend *o; -#ifdef WITH_FREELISTS - struct _Py_async_gen_asend_freelist *freelist = get_async_gen_asend_freelist(); - if (freelist->numfree > 0) { - freelist->numfree--; - o = freelist->items[freelist->numfree]; - _Py_NewReference((PyObject *)o); - } - else -#endif - { - o = PyObject_GC_New(PyAsyncGenASend, &_PyAsyncGenASend_Type); - if (o == NULL) { + PyAsyncGenASend *ags = _Py_FREELIST_POP(PyAsyncGenASend, async_gen_asends); + if (ags == NULL) { + ags = PyObject_GC_New(PyAsyncGenASend, &_PyAsyncGenASend_Type); + if (ags == NULL) { return NULL; } } - o->ags_gen = (PyAsyncGenObject*)Py_NewRef(gen); - - o->ags_sendval = Py_XNewRef(sendval); - - o->ags_state = AWAITABLE_STATE_INIT; + ags->ags_gen = (PyAsyncGenObject*)Py_NewRef(gen); + ags->ags_sendval = Py_XNewRef(sendval); + ags->ags_state = AWAITABLE_STATE_INIT; - _PyObject_GC_TRACK((PyObject*)o); - return (PyObject*)o; + _PyObject_GC_TRACK((PyObject*)ags); + return (PyObject*)ags; } @@ -1935,30 +1968,20 @@ async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval) static void -async_gen_wrapped_val_dealloc(_PyAsyncGenWrappedValue *o) +async_gen_wrapped_val_dealloc(PyObject *self) { - _PyObject_GC_UNTRACK((PyObject *)o); - Py_CLEAR(o->agw_val); -#ifdef WITH_FREELISTS - struct _Py_async_gen_freelist *freelist = get_async_gen_freelist(); - if (freelist->numfree >= 0 && freelist->numfree < _PyAsyncGen_MAXFREELIST) { - assert(_PyAsyncGenWrappedValue_CheckExact(o)); - freelist->items[freelist->numfree++] = o; - OBJECT_STAT_INC(to_freelist); - } - else -#endif - { - PyObject_GC_Del(o); - } + _PyAsyncGenWrappedValue *agw = _PyAsyncGenWrappedValue_CAST(self); + _PyObject_GC_UNTRACK(self); + Py_CLEAR(agw->agw_val); + _Py_FREELIST_FREE(async_gens, self, PyObject_GC_Del); } static int -async_gen_wrapped_val_traverse(_PyAsyncGenWrappedValue *o, - visitproc visit, void *arg) +async_gen_wrapped_val_traverse(PyObject *self, visitproc visit, void *arg) { - Py_VISIT(o->agw_val); + _PyAsyncGenWrappedValue *agw = _PyAsyncGenWrappedValue_CAST(self); + Py_VISIT(agw->agw_val); return 0; } @@ -1969,7 +1992,7 @@ PyTypeObject _PyAsyncGenWrappedValue_Type = { sizeof(_PyAsyncGenWrappedValue), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)async_gen_wrapped_val_dealloc, /* tp_dealloc */ + async_gen_wrapped_val_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -1986,7 +2009,7 @@ PyTypeObject _PyAsyncGenWrappedValue_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)async_gen_wrapped_val_traverse, /* tp_traverse */ + async_gen_wrapped_val_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ @@ -2009,27 +2032,17 @@ PyTypeObject _PyAsyncGenWrappedValue_Type = { PyObject * _PyAsyncGenValueWrapperNew(PyThreadState *tstate, PyObject *val) { - _PyAsyncGenWrappedValue *o; assert(val); -#ifdef WITH_FREELISTS - struct _Py_async_gen_freelist *freelist = get_async_gen_freelist(); - if (freelist->numfree > 0) { - freelist->numfree--; - o = freelist->items[freelist->numfree]; - OBJECT_STAT_INC(from_freelist); - assert(_PyAsyncGenWrappedValue_CheckExact(o)); - _Py_NewReference((PyObject*)o); - } - else -#endif - { + _PyAsyncGenWrappedValue *o = _Py_FREELIST_POP(_PyAsyncGenWrappedValue, async_gens); + if (o == NULL) { o = PyObject_GC_New(_PyAsyncGenWrappedValue, &_PyAsyncGenWrappedValue_Type); if (o == NULL) { return NULL; } } + assert(_PyAsyncGenWrappedValue_CheckExact(o)); o->agw_val = Py_NewRef(val); _PyObject_GC_TRACK((PyObject*)o); return (PyObject*)o; @@ -2038,34 +2051,40 @@ _PyAsyncGenValueWrapperNew(PyThreadState *tstate, PyObject *val) /* ---------- Async Generator AThrow awaitable ------------ */ +#define _PyAsyncGenAThrow_CAST(op) \ + (assert(Py_IS_TYPE((op), &_PyAsyncGenAThrow_Type)), \ + _Py_CAST(PyAsyncGenAThrow*, (op))) static void -async_gen_athrow_dealloc(PyAsyncGenAThrow *o) +async_gen_athrow_dealloc(PyObject *self) { - if (PyObject_CallFinalizerFromDealloc((PyObject *)o)) { + PyAsyncGenAThrow *agt = _PyAsyncGenAThrow_CAST(self); + if (PyObject_CallFinalizerFromDealloc(self)) { return; } - _PyObject_GC_UNTRACK((PyObject *)o); - Py_CLEAR(o->agt_gen); - Py_CLEAR(o->agt_args); - PyObject_GC_Del(o); + _PyObject_GC_UNTRACK(self); + Py_CLEAR(agt->agt_gen); + Py_CLEAR(agt->agt_args); + PyObject_GC_Del(self); } static int -async_gen_athrow_traverse(PyAsyncGenAThrow *o, visitproc visit, void *arg) +async_gen_athrow_traverse(PyObject *self, visitproc visit, void *arg) { - Py_VISIT(o->agt_gen); - Py_VISIT(o->agt_args); + PyAsyncGenAThrow *agt = _PyAsyncGenAThrow_CAST(self); + Py_VISIT(agt->agt_gen); + Py_VISIT(agt->agt_args); return 0; } static PyObject * -async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) +async_gen_athrow_send(PyObject *self, PyObject *arg) { - PyGenObject *gen = (PyGenObject*)o->agt_gen; + PyAsyncGenAThrow *o = _PyAsyncGenAThrow_CAST(self); + PyGenObject *gen = _PyGen_CAST(o->agt_gen); PyObject *retval; if (o->agt_state == AWAITABLE_STATE_CLOSED) { @@ -2148,7 +2167,7 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) assert(o->agt_state == AWAITABLE_STATE_ITER); - retval = gen_send((PyGenObject *)gen, arg); + retval = gen_send((PyObject *)gen, arg); if (o->agt_args) { return async_gen_unwrap_value(o->agt_gen, retval); } else { @@ -2195,9 +2214,9 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) static PyObject * -async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t nargs) +async_gen_athrow_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { - PyObject *retval; + PyAsyncGenAThrow *o = _PyAsyncGenAThrow_CAST(self); if (o->agt_state == AWAITABLE_STATE_CLOSED) { PyErr_SetString( @@ -2206,10 +2225,36 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t na return NULL; } - retval = gen_throw((PyGenObject*)o->agt_gen, args, nargs); + if (o->agt_state == AWAITABLE_STATE_INIT) { + if (o->agt_gen->ag_running_async) { + o->agt_state = AWAITABLE_STATE_CLOSED; + if (o->agt_args == NULL) { + PyErr_SetString( + PyExc_RuntimeError, + "aclose(): asynchronous generator is already running"); + } + else { + PyErr_SetString( + PyExc_RuntimeError, + "athrow(): asynchronous generator is already running"); + } + return NULL; + } + + o->agt_state = AWAITABLE_STATE_ITER; + o->agt_gen->ag_running_async = 1; + } + + PyObject *retval = gen_throw((PyGenObject*)o->agt_gen, args, nargs); if (o->agt_args) { - return async_gen_unwrap_value(o->agt_gen, retval); - } else { + retval = async_gen_unwrap_value(o->agt_gen, retval); + if (retval == NULL) { + o->agt_gen->ag_running_async = 0; + o->agt_state = AWAITABLE_STATE_CLOSED; + } + return retval; + } + else { /* aclose() mode */ if (retval && _PyAsyncGenWrappedValue_CheckExact(retval)) { o->agt_gen->ag_running_async = 0; @@ -2218,6 +2263,10 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t na PyErr_SetString(PyExc_RuntimeError, ASYNC_GEN_IGNORED_EXIT_MSG); return NULL; } + if (retval == NULL) { + o->agt_gen->ag_running_async = 0; + o->agt_state = AWAITABLE_STATE_CLOSED; + } if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || PyErr_ExceptionMatches(PyExc_GeneratorExit)) { @@ -2235,17 +2284,35 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t na static PyObject * -async_gen_athrow_iternext(PyAsyncGenAThrow *o) +async_gen_athrow_iternext(PyObject *agt) { - return async_gen_athrow_send(o, Py_None); + return async_gen_athrow_send(agt, Py_None); } static PyObject * -async_gen_athrow_close(PyAsyncGenAThrow *o, PyObject *args) +async_gen_athrow_close(PyObject *self, PyObject *args) { - o->agt_state = AWAITABLE_STATE_CLOSED; - Py_RETURN_NONE; + PyAsyncGenAThrow *agt = _PyAsyncGenAThrow_CAST(self); + if (agt->agt_state == AWAITABLE_STATE_CLOSED) { + Py_RETURN_NONE; + } + PyObject *result = async_gen_athrow_throw((PyObject*)agt, + &PyExc_GeneratorExit, 1); + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_StopIteration) || + PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || + PyErr_ExceptionMatches(PyExc_GeneratorExit)) + { + PyErr_Clear(); + Py_RETURN_NONE; + } + return result; + } else { + Py_DECREF(result); + PyErr_SetString(PyExc_RuntimeError, "coroutine ignored GeneratorExit"); + return NULL; + } } @@ -2259,10 +2326,10 @@ async_gen_athrow_finalize(PyAsyncGenAThrow *o) } static PyMethodDef async_gen_athrow_methods[] = { - {"send", (PyCFunction)async_gen_athrow_send, METH_O, send_doc}, + {"send", async_gen_athrow_send, METH_O, send_doc}, {"throw", _PyCFunction_CAST(async_gen_athrow_throw), METH_FASTCALL, throw_doc}, - {"close", (PyCFunction)async_gen_athrow_close, METH_NOARGS, close_doc}, + {"close", async_gen_athrow_close, METH_NOARGS, close_doc}, {NULL, NULL} /* Sentinel */ }; @@ -2281,7 +2348,7 @@ PyTypeObject _PyAsyncGenAThrow_Type = { sizeof(PyAsyncGenAThrow), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)async_gen_athrow_dealloc, /* tp_dealloc */ + async_gen_athrow_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -2298,12 +2365,12 @@ PyTypeObject _PyAsyncGenAThrow_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)async_gen_athrow_traverse, /* tp_traverse */ + async_gen_athrow_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)async_gen_athrow_iternext, /* tp_iternext */ + async_gen_athrow_iternext, /* tp_iternext */ async_gen_athrow_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ diff --git a/Objects/interpreteridobject.c b/Objects/interpreteridobject.c deleted file mode 100644 index 16e27b64c0c9c2..00000000000000 --- a/Objects/interpreteridobject.c +++ /dev/null @@ -1,294 +0,0 @@ -/* InterpreterID object */ - -#include "Python.h" -#include "pycore_abstract.h" // _PyIndex_Check() -#include "pycore_interp.h" // _PyInterpreterState_LookUpID() -#include "interpreteridobject.h" - - -typedef struct interpid { - PyObject_HEAD - int64_t id; -} interpid; - -static interpid * -newinterpid(PyTypeObject *cls, int64_t id, int force) -{ - PyInterpreterState *interp = _PyInterpreterState_LookUpID(id); - if (interp == NULL) { - if (force) { - PyErr_Clear(); - } - else { - return NULL; - } - } - - if (interp != NULL) { - if (_PyInterpreterState_IDIncref(interp) < 0) { - return NULL; - } - } - - interpid *self = PyObject_New(interpid, cls); - if (self == NULL) { - if (interp != NULL) { - _PyInterpreterState_IDDecref(interp); - } - return NULL; - } - self->id = id; - - return self; -} - -static int -interp_id_converter(PyObject *arg, void *ptr) -{ - int64_t id; - if (PyObject_TypeCheck(arg, &PyInterpreterID_Type)) { - id = ((interpid *)arg)->id; - } - else if (_PyIndex_Check(arg)) { - id = PyLong_AsLongLong(arg); - if (id == -1 && PyErr_Occurred()) { - return 0; - } - if (id < 0) { - PyErr_Format(PyExc_ValueError, - "interpreter ID must be a non-negative int, got %R", arg); - return 0; - } - } - else { - PyErr_Format(PyExc_TypeError, - "interpreter ID must be an int, got %.100s", - Py_TYPE(arg)->tp_name); - return 0; - } - *(int64_t *)ptr = id; - return 1; -} - -static PyObject * -interpid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"id", "force", NULL}; - int64_t id; - int force = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O&|$p:InterpreterID.__init__", kwlist, - interp_id_converter, &id, &force)) { - return NULL; - } - - return (PyObject *)newinterpid(cls, id, force); -} - -static void -interpid_dealloc(PyObject *v) -{ - int64_t id = ((interpid *)v)->id; - PyInterpreterState *interp = _PyInterpreterState_LookUpID(id); - if (interp != NULL) { - _PyInterpreterState_IDDecref(interp); - } - else { - // already deleted - PyErr_Clear(); - } - Py_TYPE(v)->tp_free(v); -} - -static PyObject * -interpid_repr(PyObject *self) -{ - PyTypeObject *type = Py_TYPE(self); - const char *name = _PyType_Name(type); - interpid *id = (interpid *)self; - return PyUnicode_FromFormat("%s(%" PRId64 ")", name, id->id); -} - -static PyObject * -interpid_str(PyObject *self) -{ - interpid *id = (interpid *)self; - return PyUnicode_FromFormat("%" PRId64 "", id->id); -} - -static PyObject * -interpid_int(PyObject *self) -{ - interpid *id = (interpid *)self; - return PyLong_FromLongLong(id->id); -} - -static PyNumberMethods interpid_as_number = { - 0, /* nb_add */ - 0, /* nb_subtract */ - 0, /* nb_multiply */ - 0, /* nb_remainder */ - 0, /* nb_divmod */ - 0, /* nb_power */ - 0, /* nb_negative */ - 0, /* nb_positive */ - 0, /* nb_absolute */ - 0, /* nb_bool */ - 0, /* nb_invert */ - 0, /* nb_lshift */ - 0, /* nb_rshift */ - 0, /* nb_and */ - 0, /* nb_xor */ - 0, /* nb_or */ - (unaryfunc)interpid_int, /* nb_int */ - 0, /* nb_reserved */ - 0, /* nb_float */ - - 0, /* nb_inplace_add */ - 0, /* nb_inplace_subtract */ - 0, /* nb_inplace_multiply */ - 0, /* nb_inplace_remainder */ - 0, /* nb_inplace_power */ - 0, /* nb_inplace_lshift */ - 0, /* nb_inplace_rshift */ - 0, /* nb_inplace_and */ - 0, /* nb_inplace_xor */ - 0, /* nb_inplace_or */ - - 0, /* nb_floor_divide */ - 0, /* nb_true_divide */ - 0, /* nb_inplace_floor_divide */ - 0, /* nb_inplace_true_divide */ - - (unaryfunc)interpid_int, /* nb_index */ -}; - -static Py_hash_t -interpid_hash(PyObject *self) -{ - interpid *id = (interpid *)self; - PyObject *obj = PyLong_FromLongLong(id->id); - if (obj == NULL) { - return -1; - } - Py_hash_t hash = PyObject_Hash(obj); - Py_DECREF(obj); - return hash; -} - -static PyObject * -interpid_richcompare(PyObject *self, PyObject *other, int op) -{ - if (op != Py_EQ && op != Py_NE) { - Py_RETURN_NOTIMPLEMENTED; - } - - if (!PyObject_TypeCheck(self, &PyInterpreterID_Type)) { - Py_RETURN_NOTIMPLEMENTED; - } - - interpid *id = (interpid *)self; - int equal; - if (PyObject_TypeCheck(other, &PyInterpreterID_Type)) { - interpid *otherid = (interpid *)other; - equal = (id->id == otherid->id); - } - else if (PyLong_CheckExact(other)) { - /* Fast path */ - int overflow; - long long otherid = PyLong_AsLongLongAndOverflow(other, &overflow); - if (otherid == -1 && PyErr_Occurred()) { - return NULL; - } - equal = !overflow && (otherid >= 0) && (id->id == otherid); - } - else if (PyNumber_Check(other)) { - PyObject *pyid = PyLong_FromLongLong(id->id); - if (pyid == NULL) { - return NULL; - } - PyObject *res = PyObject_RichCompare(pyid, other, op); - Py_DECREF(pyid); - return res; - } - else { - Py_RETURN_NOTIMPLEMENTED; - } - - if ((op == Py_EQ && equal) || (op == Py_NE && !equal)) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; -} - -PyDoc_STRVAR(interpid_doc, -"A interpreter ID identifies a interpreter and may be used as an int."); - -PyTypeObject PyInterpreterID_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "InterpreterID", /* tp_name */ - sizeof(interpid), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)interpid_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)interpid_repr, /* tp_repr */ - &interpid_as_number, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - interpid_hash, /* tp_hash */ - 0, /* tp_call */ - (reprfunc)interpid_str, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - interpid_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - interpid_richcompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - interpid_new, /* tp_new */ -}; - -PyObject *PyInterpreterID_New(int64_t id) -{ - return (PyObject *)newinterpid(&PyInterpreterID_Type, id, 0); -} - -PyObject * -PyInterpreterState_GetIDObject(PyInterpreterState *interp) -{ - if (_PyInterpreterState_IDInitref(interp) != 0) { - return NULL; - }; - int64_t id = PyInterpreterState_GetID(interp); - if (id < 0) { - return NULL; - } - return (PyObject *)newinterpid(&PyInterpreterID_Type, id, 0); -} - -PyInterpreterState * -PyInterpreterID_LookUp(PyObject *requested_id) -{ - int64_t id; - if (!interp_id_converter(requested_id, &id)) { - return NULL; - } - return _PyInterpreterState_LookUpID(id); -} diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 135ced9ea1f268..ebb342ff109222 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -384,6 +384,7 @@ anextawaitable_iternext(anextawaitableobject *obj) return result; } if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)) { + PyErr_Clear(); _PyGen_SetStopIterationValue(obj->default_value); } return NULL; @@ -407,6 +408,7 @@ anextawaitable_proxy(anextawaitableobject *obj, char *meth, PyObject *arg) { * exception we replace it with a `StopIteration(default)`, as if * it was the return value of `__anext__()` coroutine. */ + PyErr_Clear(); _PyGen_SetStopIterationValue(obj->default_value); } return NULL; diff --git a/Objects/listobject.c b/Objects/listobject.c index 87effb1b3a65fa..f4a269e4d7b284 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -3,6 +3,9 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_critical_section.h" // _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED() +#include "pycore_dict.h" // _PyDictViewObject +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_pyatomic_ft_wrappers.h" #include "pycore_interp.h" // PyInterpreterState.list #include "pycore_list.h" // struct _Py_list_freelist, _PyListIterObject @@ -10,6 +13,7 @@ #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() #include "pycore_tuple.h" // _PyTuple_FromArray() +#include "pycore_setobject.h" // _PySet_NextEntry() #include /*[clinic input] @@ -21,15 +25,68 @@ class list "PyListObject *" "&PyList_Type" _Py_DECLARE_STR(list_err, "list index out of range"); -#ifdef WITH_FREELISTS -static struct _Py_list_freelist * -get_list_freelist(void) +#ifdef Py_GIL_DISABLED +typedef struct { + Py_ssize_t allocated; + PyObject *ob_item[]; +} _PyListArray; + +static _PyListArray * +list_allocate_array(size_t capacity) +{ + if (capacity > PY_SSIZE_T_MAX/sizeof(PyObject*) - 1) { + return NULL; + } + _PyListArray *array = PyMem_Malloc(sizeof(_PyListArray) + capacity * sizeof(PyObject *)); + if (array == NULL) { + return NULL; + } + array->allocated = capacity; + return array; +} + +static Py_ssize_t +list_capacity(PyObject **items) +{ + _PyListArray *array = _Py_CONTAINER_OF(items, _PyListArray, ob_item); + return array->allocated; +} +#endif + +static void +free_list_items(PyObject** items, bool use_qsbr) { - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - assert(freelists != NULL); - return &freelists->lists; +#ifdef Py_GIL_DISABLED + _PyListArray *array = _Py_CONTAINER_OF(items, _PyListArray, ob_item); + if (use_qsbr) { + _PyMem_FreeDelayed(array); + } + else { + PyMem_Free(array); + } +#else + PyMem_Free(items); +#endif } + +static void +ensure_shared_on_resize(PyListObject *self) +{ +#ifdef Py_GIL_DISABLED + // We can't use _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED here because + // the `CALL_LIST_APPEND` bytecode handler may lock the list without + // a critical section. + assert(Py_REFCNT(self) == 1 || PyMutex_IsLocked(&_PyObject_CAST(self)->ob_mutex)); + + // Ensure that the list array is freed using QSBR if we are not the + // owning thread. + if (!_Py_IsOwnedByCurrentThread((PyObject *)self) && + !_PyObject_GC_IS_SHARED(self)) + { + _PyObject_GC_SET_SHARED(self); + } #endif +} /* Ensure ob_item has room for at least newsize elements, and set * ob_size to newsize. If newsize > ob_size on entry, the content @@ -47,8 +104,7 @@ get_list_freelist(void) static int list_resize(PyListObject *self, Py_ssize_t newsize) { - PyObject **items; - size_t new_allocated, num_allocated_bytes; + size_t new_allocated, target_bytes; Py_ssize_t allocated = self->allocated; /* Bypass realloc() when a previous overallocation is large enough @@ -80,9 +136,39 @@ list_resize(PyListObject *self, Py_ssize_t newsize) if (newsize == 0) new_allocated = 0; + + ensure_shared_on_resize(self); + +#ifdef Py_GIL_DISABLED + _PyListArray *array = list_allocate_array(new_allocated); + if (array == NULL) { + PyErr_NoMemory(); + return -1; + } + PyObject **old_items = self->ob_item; + if (self->ob_item) { + if (new_allocated < (size_t)allocated) { + target_bytes = new_allocated * sizeof(PyObject*); + } + else { + target_bytes = allocated * sizeof(PyObject*); + } + memcpy(array->ob_item, self->ob_item, target_bytes); + } + if (new_allocated > (size_t)allocated) { + memset(array->ob_item + allocated, 0, sizeof(PyObject *) * (new_allocated - allocated)); + } + _Py_atomic_store_ptr_release(&self->ob_item, &array->ob_item); + self->allocated = new_allocated; + Py_SET_SIZE(self, newsize); + if (old_items != NULL) { + free_list_items(old_items, _PyObject_GC_IS_SHARED(self)); + } +#else + PyObject **items; if (new_allocated <= (size_t)PY_SSIZE_T_MAX / sizeof(PyObject *)) { - num_allocated_bytes = new_allocated * sizeof(PyObject *); - items = (PyObject **)PyMem_Realloc(self->ob_item, num_allocated_bytes); + target_bytes = new_allocated * sizeof(PyObject *); + items = (PyObject **)PyMem_Realloc(self->ob_item, target_bytes); } else { // integer overflow @@ -95,12 +181,14 @@ list_resize(PyListObject *self, Py_ssize_t newsize) self->ob_item = items; Py_SET_SIZE(self, newsize); self->allocated = new_allocated; +#endif return 0; } static int list_preallocate_exact(PyListObject *self, Py_ssize_t size) { + PyObject **items; assert(self->ob_item == NULL); assert(size > 0); @@ -110,65 +198,46 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size) * allocated size up to the nearest even number. */ size = (size + 1) & ~(size_t)1; - PyObject **items = PyMem_New(PyObject*, size); +#ifdef Py_GIL_DISABLED + _PyListArray *array = list_allocate_array(size); + if (array == NULL) { + PyErr_NoMemory(); + return -1; + } + items = array->ob_item; + memset(items, 0, size * sizeof(PyObject *)); +#else + items = PyMem_New(PyObject*, size); if (items == NULL) { PyErr_NoMemory(); return -1; } - self->ob_item = items; +#endif + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item, items); self->allocated = size; return 0; } -void -_PyList_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) -{ -#ifdef WITH_FREELISTS - struct _Py_list_freelist *state = &freelists->lists; - while (state->numfree > 0) { - PyListObject *op = state->items[--state->numfree]; - assert(PyList_CheckExact(op)); - PyObject_GC_Del(op); - } - if (is_finalization) { - state->numfree = -1; - } -#endif -} - /* Print summary info about the state of the optimized allocator */ void _PyList_DebugMallocStats(FILE *out) { -#ifdef WITH_FREELISTS - struct _Py_list_freelist *list_freelist = get_list_freelist(); _PyDebugAllocatorStats(out, "free PyListObject", - list_freelist->numfree, sizeof(PyListObject)); -#endif + _Py_FREELIST_SIZE(lists), + sizeof(PyListObject)); } PyObject * PyList_New(Py_ssize_t size) { - PyListObject *op; - if (size < 0) { PyErr_BadInternalCall(); return NULL; } -#ifdef WITH_FREELISTS - struct _Py_list_freelist *list_freelist = get_list_freelist(); - if (PyList_MAXFREELIST && list_freelist->numfree > 0) { - list_freelist->numfree--; - op = list_freelist->items[list_freelist->numfree]; - OBJECT_STAT_INC(from_freelist); - _Py_NewReference((PyObject *)op); - } - else -#endif - { + PyListObject *op = _Py_FREELIST_POP(PyListObject, lists); + if (op == NULL) { op = PyObject_GC_New(PyListObject, &PyList_Type); if (op == NULL) { return NULL; @@ -178,7 +247,17 @@ PyList_New(Py_ssize_t size) op->ob_item = NULL; } else { +#ifdef Py_GIL_DISABLED + _PyListArray *array = list_allocate_array(size); + if (array == NULL) { + Py_DECREF(op); + return PyErr_NoMemory(); + } + memset(&array->ob_item, 0, size * sizeof(PyObject *)); + op->ob_item = array->ob_item; +#else op->ob_item = (PyObject **) PyMem_Calloc(size, sizeof(PyObject *)); +#endif if (op->ob_item == NULL) { Py_DECREF(op); return PyErr_NoMemory(); @@ -199,11 +278,20 @@ list_new_prealloc(Py_ssize_t size) return NULL; } assert(op->ob_item == NULL); +#ifdef Py_GIL_DISABLED + _PyListArray *array = list_allocate_array(size); + if (array == NULL) { + Py_DECREF(op); + return PyErr_NoMemory(); + } + op->ob_item = array->ob_item; +#else op->ob_item = PyMem_New(PyObject *, size); if (op->ob_item == NULL) { Py_DECREF(op); return PyErr_NoMemory(); } +#endif op->allocated = size; return (PyObject *) op; } @@ -233,6 +321,63 @@ valid_index(Py_ssize_t i, Py_ssize_t limit) return (size_t) i < (size_t) limit; } +#ifdef Py_GIL_DISABLED + +static PyObject * +list_item_impl(PyListObject *self, Py_ssize_t idx) +{ + PyObject *item = NULL; + Py_BEGIN_CRITICAL_SECTION(self); + if (!_PyObject_GC_IS_SHARED(self)) { + _PyObject_GC_SET_SHARED(self); + } + Py_ssize_t size = Py_SIZE(self); + if (!valid_index(idx, size)) { + goto exit; + } + item = _Py_NewRefWithLock(self->ob_item[idx]); +exit: + Py_END_CRITICAL_SECTION(); + return item; +} + +static inline PyObject* +list_get_item_ref(PyListObject *op, Py_ssize_t i) +{ + if (!_Py_IsOwnedByCurrentThread((PyObject *)op) && !_PyObject_GC_IS_SHARED(op)) { + return list_item_impl(op, i); + } + // Need atomic operation for the getting size. + Py_ssize_t size = PyList_GET_SIZE(op); + if (!valid_index(i, size)) { + return NULL; + } + PyObject **ob_item = _Py_atomic_load_ptr(&op->ob_item); + if (ob_item == NULL) { + return NULL; + } + Py_ssize_t cap = list_capacity(ob_item); + assert(cap != -1 && cap >= size); + if (!valid_index(i, cap)) { + return NULL; + } + PyObject *item = _Py_TryXGetRef(&ob_item[i]); + if (item == NULL) { + return list_item_impl(op, i); + } + return item; +} +#else +static inline PyObject* +list_get_item_ref(PyListObject *op, Py_ssize_t i) +{ + if (!valid_index(i, Py_SIZE(op))) { + return NULL; + } + return Py_NewRef(PyList_GET_ITEM(op, i)); +} +#endif + PyObject * PyList_GetItem(PyObject *op, Py_ssize_t i) { @@ -255,21 +400,19 @@ PyList_GetItemRef(PyObject *op, Py_ssize_t i) PyErr_SetString(PyExc_TypeError, "expected a list"); return NULL; } - if (!valid_index(i, Py_SIZE(op))) { + PyObject *item = list_get_item_ref((PyListObject *)op, i); + if (item == NULL) { _Py_DECLARE_STR(list_err, "list index out of range"); PyErr_SetObject(PyExc_IndexError, &_Py_STR(list_err)); return NULL; } - return Py_NewRef(PyList_GET_ITEM(op, i)); + return item; } -static inline PyObject* -list_get_item_ref(PyListObject *op, Py_ssize_t i) +PyObject * +_PyList_GetItemRef(PyListObject *list, Py_ssize_t i) { - if (!valid_index(i, Py_SIZE(op))) { - return NULL; - } - return Py_NewRef(PyList_GET_ITEM(op, i)); + return list_get_item_ref(list, i); } int @@ -295,7 +438,7 @@ PyList_SetItem(PyObject *op, Py_ssize_t i, p = self->ob_item + i; Py_XSETREF(*p, newitem); ret = 0; -end: +end:; Py_END_CRITICAL_SECTION(); return ret; } @@ -353,7 +496,7 @@ _PyList_AppendTakeRefListResize(PyListObject *self, PyObject *newitem) Py_DECREF(newitem); return -1; } - PyList_SET_ITEM(self, len, newitem); + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item[len], newitem); return 0; } @@ -389,18 +532,13 @@ list_dealloc(PyObject *self) while (--i >= 0) { Py_XDECREF(op->ob_item[i]); } - PyMem_Free(op->ob_item); + free_list_items(op->ob_item, false); } -#ifdef WITH_FREELISTS - struct _Py_list_freelist *list_freelist = get_list_freelist(); - if (list_freelist->numfree < PyList_MAXFREELIST && list_freelist->numfree >= 0 && PyList_CheckExact(op)) { - list_freelist->items[list_freelist->numfree++] = op; - OBJECT_STAT_INC(to_freelist); + if (PyList_CheckExact(op)) { + _Py_FREELIST_FREE(lists, op, PyObject_GC_Del); } - else -#endif - { - Py_TYPE(op)->tp_free((PyObject *)op); + else { + PyObject_GC_Del(op); } Py_TRASHCAN_END } @@ -408,49 +546,48 @@ list_dealloc(PyObject *self) static PyObject * list_repr_impl(PyListObject *v) { - PyObject *s; - _PyUnicodeWriter writer; - Py_ssize_t i = Py_ReprEnter((PyObject*)v); - if (i != 0) { - return i > 0 ? PyUnicode_FromString("[...]") : NULL; + int res = Py_ReprEnter((PyObject*)v); + if (res != 0) { + return (res > 0 ? PyUnicode_FromString("[...]") : NULL); } - _PyUnicodeWriter_Init(&writer); - writer.overallocate = 1; /* "[" + "1" + ", 2" * (len - 1) + "]" */ - writer.min_length = 1 + 1 + (2 + 1) * (Py_SIZE(v) - 1) + 1; + Py_ssize_t prealloc = 1 + 1 + (2 + 1) * (Py_SIZE(v) - 1) + 1; + PyUnicodeWriter *writer = PyUnicodeWriter_Create(prealloc); + if (writer == NULL) { + goto error; + } - if (_PyUnicodeWriter_WriteChar(&writer, '[') < 0) + if (PyUnicodeWriter_WriteChar(writer, '[') < 0) { goto error; + } /* Do repr() on each element. Note that this may mutate the list, so must refetch the list size on each iteration. */ - for (i = 0; i < Py_SIZE(v); ++i) { + for (Py_ssize_t i = 0; i < Py_SIZE(v); ++i) { if (i > 0) { - if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) + if (PyUnicodeWriter_WriteChar(writer, ',') < 0) { + goto error; + } + if (PyUnicodeWriter_WriteChar(writer, ' ') < 0) { goto error; + } } - s = PyObject_Repr(v->ob_item[i]); - if (s == NULL) - goto error; - - if (_PyUnicodeWriter_WriteStr(&writer, s) < 0) { - Py_DECREF(s); + if (PyUnicodeWriter_WriteRepr(writer, v->ob_item[i]) < 0) { goto error; } - Py_DECREF(s); } - writer.overallocate = 0; - if (_PyUnicodeWriter_WriteChar(&writer, ']') < 0) + if (PyUnicodeWriter_WriteChar(writer, ']') < 0) { goto error; + } Py_ReprLeave((PyObject *)v); - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); error: - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); Py_ReprLeave((PyObject *)v); return NULL; } @@ -503,19 +640,20 @@ list_item(PyObject *aa, Py_ssize_t i) return NULL; } PyObject *item; - Py_BEGIN_CRITICAL_SECTION(a); #ifdef Py_GIL_DISABLED - if (!_Py_IsOwnedByCurrentThread((PyObject *)a) && !_PyObject_GC_IS_SHARED(a)) { - _PyObject_GC_SET_SHARED(a); + item = list_get_item_ref(a, i); + if (item == NULL) { + PyErr_SetObject(PyExc_IndexError, &_Py_STR(list_err)); + return NULL; } -#endif +#else item = Py_NewRef(a->ob_item[i]); - Py_END_CRITICAL_SECTION(); +#endif return item; } static PyObject * -list_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) +list_slice_lock_held(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) { PyListObject *np; PyObject **src, **dest; @@ -559,7 +697,7 @@ PyList_GetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) else if (ihigh > Py_SIZE(a)) { ihigh = Py_SIZE(a); } - ret = list_slice((PyListObject *)a, ilow, ihigh); + ret = list_slice_lock_held((PyListObject *)a, ilow, ihigh); Py_END_CRITICAL_SECTION(); return ret; } @@ -584,13 +722,13 @@ list_concat_lock_held(PyListObject *a, PyListObject *b) dest = np->ob_item; for (i = 0; i < Py_SIZE(a); i++) { PyObject *v = src[i]; - FT_ATOMIC_STORE_PTR_RELAXED(dest[i], Py_NewRef(v)); + dest[i] = Py_NewRef(v); } src = b->ob_item; dest = np->ob_item + Py_SIZE(a); for (i = 0; i < Py_SIZE(b); i++) { PyObject *v = src[i]; - FT_ATOMIC_STORE_PTR_RELAXED(dest[i], Py_NewRef(v)); + dest[i] = Py_NewRef(v); } Py_SET_SIZE(np, size); return (PyObject *)np; @@ -636,7 +774,7 @@ list_repeat_lock_held(PyListObject *a, Py_ssize_t n) _Py_RefcntAdd(elem, n); PyObject **dest_end = dest + output_size; while (dest < dest_end) { - FT_ATOMIC_STORE_PTR_RELAXED(*dest++, elem); + *dest++ = elem; } } else { @@ -644,7 +782,7 @@ list_repeat_lock_held(PyListObject *a, Py_ssize_t n) PyObject **src_end = src + input_size; while (src < src_end) { _Py_RefcntAdd(*src, n); - FT_ATOMIC_STORE_PTR_RELAXED(*dest++, *src++); + *dest++ = *src++; } _Py_memory_repeat((char *)np->ob_item, sizeof(PyObject *)*output_size, @@ -684,16 +822,14 @@ list_clear_impl(PyListObject *a, bool is_resize) Py_XDECREF(items[i]); } #ifdef Py_GIL_DISABLED + if (is_resize) { + ensure_shared_on_resize(a); + } bool use_qsbr = is_resize && _PyObject_GC_IS_SHARED(a); #else bool use_qsbr = false; #endif - if (use_qsbr) { - _PyMem_FreeDelayed(items); - } - else { - PyMem_Free(items); - } + free_list_items(items, use_qsbr); // Note that there is no guarantee that the list is actually empty // at this point, because XDECREF may have populated it indirectly again! } @@ -718,7 +854,7 @@ list_clear_slot(PyObject *self) * guaranteed the call cannot fail. */ static int -list_ass_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) +list_ass_slice_lock_held(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) { /* Because [X]DECREF can recursively invoke list operations on this list, we must postpone all [X]DECREF activity until @@ -741,15 +877,6 @@ list_ass_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) if (v == NULL) n = 0; else { - if (a == b) { - /* Special case "a[i:j] = a" -- copy b first */ - v = list_slice(b, 0, Py_SIZE(b)); - if (v == NULL) - return result; - result = list_ass_slice(a, ilow, ihigh, v); - Py_DECREF(v); - return result; - } v_as_SF = PySequence_Fast(v, "can only assign an iterable"); if(v_as_SF == NULL) goto Error; @@ -823,6 +950,36 @@ list_ass_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) #undef b } +static int +list_ass_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) +{ + int ret; + if (a == (PyListObject *)v) { + Py_BEGIN_CRITICAL_SECTION(a); + Py_ssize_t n = PyList_GET_SIZE(a); + PyObject *copy = list_slice_lock_held(a, 0, n); + if (copy == NULL) { + ret = -1; + } + else { + ret = list_ass_slice_lock_held(a, ilow, ihigh, copy); + Py_DECREF(copy); + } + Py_END_CRITICAL_SECTION(); + } + else if (v != NULL && PyList_CheckExact(v)) { + Py_BEGIN_CRITICAL_SECTION2(a, v); + ret = list_ass_slice_lock_held(a, ilow, ihigh, v); + Py_END_CRITICAL_SECTION2(); + } + else { + Py_BEGIN_CRITICAL_SECTION(a); + ret = list_ass_slice_lock_held(a, ilow, ihigh, v); + Py_END_CRITICAL_SECTION(); + } + return ret; +} + int PyList_SetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) { @@ -833,26 +990,28 @@ PyList_SetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) return list_ass_slice((PyListObject *)a, ilow, ihigh, v); } -static PyObject * +static int list_inplace_repeat_lock_held(PyListObject *self, Py_ssize_t n) { Py_ssize_t input_size = PyList_GET_SIZE(self); if (input_size == 0 || n == 1) { - return Py_NewRef(self); + return 0; } if (n < 1) { list_clear(self); - return Py_NewRef(self); + return 0; } if (input_size > PY_SSIZE_T_MAX / n) { - return PyErr_NoMemory(); + PyErr_NoMemory(); + return -1; } Py_ssize_t output_size = input_size * n; - if (list_resize(self, output_size) < 0) - return NULL; + if (list_resize(self, output_size) < 0) { + return -1; + } PyObject **items = self->ob_item; for (Py_ssize_t j = 0; j < input_size; j++) { @@ -860,8 +1019,7 @@ list_inplace_repeat_lock_held(PyListObject *self, Py_ssize_t n) } _Py_memory_repeat((char *)items, sizeof(PyObject *)*output_size, sizeof(PyObject *)*input_size); - - return Py_NewRef(self); + return 0; } static PyObject * @@ -870,7 +1028,12 @@ list_inplace_repeat(PyObject *_self, Py_ssize_t n) PyObject *ret; PyListObject *self = (PyListObject *) _self; Py_BEGIN_CRITICAL_SECTION(self); - ret = list_inplace_repeat_lock_held(self, n); + if (list_inplace_repeat_lock_held(self, n) < 0) { + ret = NULL; + } + else { + ret = Py_NewRef(self); + } Py_END_CRITICAL_SECTION(); return ret; } @@ -956,7 +1119,7 @@ static PyObject * list_copy_impl(PyListObject *self) /*[clinic end generated code: output=ec6b72d6209d418e input=81c54b0c7bb4f73d]*/ { - return list_slice(self, 0, Py_SIZE(self)); + return list_slice_lock_held(self, 0, Py_SIZE(self)); } /*[clinic input] @@ -1012,13 +1175,13 @@ list_extend_fast(PyListObject *self, PyObject *iterable) PyObject **dest = self->ob_item + m; for (Py_ssize_t i = 0; i < n; i++) { PyObject *o = src[i]; - dest[i] = Py_NewRef(o); + FT_ATOMIC_STORE_PTR_RELEASE(dest[i], Py_NewRef(o)); } return 0; } static int -list_extend_iter(PyListObject *self, PyObject *iterable) +list_extend_iter_lock_held(PyListObject *self, PyObject *iterable) { PyObject *it = PyObject_GetIter(iterable); if (it == NULL) { @@ -1069,7 +1232,7 @@ list_extend_iter(PyListObject *self, PyObject *iterable) if (Py_SIZE(self) < self->allocated) { Py_ssize_t len = Py_SIZE(self); - PyList_SET_ITEM(self, len, item); // steals item ref + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item[len], item); // steals item ref Py_SET_SIZE(self, len + 1); } else { @@ -1092,45 +1255,151 @@ list_extend_iter(PyListObject *self, PyObject *iterable) return -1; } - static int -list_extend(PyListObject *self, PyObject *iterable) +list_extend_lock_held(PyListObject *self, PyObject *iterable) { - // Special cases: - // 1) lists and tuples which can use PySequence_Fast ops - // 2) extending self to self requires making a copy first - if (PyList_CheckExact(iterable) - || PyTuple_CheckExact(iterable) - || (PyObject *)self == iterable) - { - iterable = PySequence_Fast(iterable, "argument must be iterable"); - if (!iterable) { - return -1; - } + PyObject *seq = PySequence_Fast(iterable, "argument must be iterable"); + if (!seq) { + return -1; + } - int res = list_extend_fast(self, iterable); - Py_DECREF(iterable); - return res; + int res = list_extend_fast(self, seq); + Py_DECREF(seq); + return res; +} + +static int +list_extend_set(PyListObject *self, PySetObject *other) +{ + Py_ssize_t m = Py_SIZE(self); + Py_ssize_t n = PySet_GET_SIZE(other); + if (list_resize(self, m + n) < 0) { + return -1; } - else { - return list_extend_iter(self, iterable); + /* populate the end of self with iterable's items */ + Py_ssize_t setpos = 0; + Py_hash_t hash; + PyObject *key; + PyObject **dest = self->ob_item + m; + while (_PySet_NextEntryRef((PyObject *)other, &setpos, &key, &hash)) { + FT_ATOMIC_STORE_PTR_RELEASE(*dest, key); + dest++; } + Py_SET_SIZE(self, m + n); + return 0; } +static int +list_extend_dict(PyListObject *self, PyDictObject *dict, int which_item) +{ + // which_item: 0 for keys and 1 for values + Py_ssize_t m = Py_SIZE(self); + Py_ssize_t n = PyDict_GET_SIZE(dict); + if (list_resize(self, m + n) < 0) { + return -1; + } -PyObject * -_PyList_Extend(PyListObject *self, PyObject *iterable) + PyObject **dest = self->ob_item + m; + Py_ssize_t pos = 0; + PyObject *keyvalue[2]; + while (_PyDict_Next((PyObject *)dict, &pos, &keyvalue[0], &keyvalue[1], NULL)) { + PyObject *obj = keyvalue[which_item]; + Py_INCREF(obj); + FT_ATOMIC_STORE_PTR_RELEASE(*dest, obj); + dest++; + } + + Py_SET_SIZE(self, m + n); + return 0; +} + +static int +list_extend_dictitems(PyListObject *self, PyDictObject *dict) { - if (list_extend(self, iterable) < 0) { - return NULL; + Py_ssize_t m = Py_SIZE(self); + Py_ssize_t n = PyDict_GET_SIZE(dict); + if (list_resize(self, m + n) < 0) { + return -1; } - Py_RETURN_NONE; + + PyObject **dest = self->ob_item + m; + Py_ssize_t pos = 0; + Py_ssize_t i = 0; + PyObject *key, *value; + while (_PyDict_Next((PyObject *)dict, &pos, &key, &value, NULL)) { + PyObject *item = PyTuple_Pack(2, key, value); + if (item == NULL) { + Py_SET_SIZE(self, m + i); + return -1; + } + FT_ATOMIC_STORE_PTR_RELEASE(*dest, item); + dest++; + i++; + } + + Py_SET_SIZE(self, m + n); + return 0; } +static int +_list_extend(PyListObject *self, PyObject *iterable) +{ + // Special case: + // lists and tuples which can use PySequence_Fast ops + int res = -1; + if ((PyObject *)self == iterable) { + Py_BEGIN_CRITICAL_SECTION(self); + res = list_inplace_repeat_lock_held(self, 2); + Py_END_CRITICAL_SECTION(); + } + else if (PyList_CheckExact(iterable)) { + Py_BEGIN_CRITICAL_SECTION2(self, iterable); + res = list_extend_lock_held(self, iterable); + Py_END_CRITICAL_SECTION2(); + } + else if (PyTuple_CheckExact(iterable)) { + Py_BEGIN_CRITICAL_SECTION(self); + res = list_extend_lock_held(self, iterable); + Py_END_CRITICAL_SECTION(); + } + else if (PyAnySet_CheckExact(iterable)) { + Py_BEGIN_CRITICAL_SECTION2(self, iterable); + res = list_extend_set(self, (PySetObject *)iterable); + Py_END_CRITICAL_SECTION2(); + } + else if (PyDict_CheckExact(iterable)) { + Py_BEGIN_CRITICAL_SECTION2(self, iterable); + res = list_extend_dict(self, (PyDictObject *)iterable, 0 /*keys*/); + Py_END_CRITICAL_SECTION2(); + } + else if (Py_IS_TYPE(iterable, &PyDictKeys_Type)) { + PyDictObject *dict = ((_PyDictViewObject *)iterable)->dv_dict; + Py_BEGIN_CRITICAL_SECTION2(self, dict); + res = list_extend_dict(self, dict, 0 /*keys*/); + Py_END_CRITICAL_SECTION2(); + } + else if (Py_IS_TYPE(iterable, &PyDictValues_Type)) { + PyDictObject *dict = ((_PyDictViewObject *)iterable)->dv_dict; + Py_BEGIN_CRITICAL_SECTION2(self, dict); + res = list_extend_dict(self, dict, 1 /*values*/); + Py_END_CRITICAL_SECTION2(); + } + else if (Py_IS_TYPE(iterable, &PyDictItems_Type)) { + PyDictObject *dict = ((_PyDictViewObject *)iterable)->dv_dict; + Py_BEGIN_CRITICAL_SECTION2(self, dict); + res = list_extend_dictitems(self, dict); + Py_END_CRITICAL_SECTION2(); + } + else { + Py_BEGIN_CRITICAL_SECTION(self); + res = list_extend_iter_lock_held(self, iterable); + Py_END_CRITICAL_SECTION(); + } + return res; +} /*[clinic input] -@critical_section self iterable -list.extend as py_list_extend +list.extend as list_extend iterable: object / @@ -1139,12 +1408,20 @@ Extend list by appending elements from the iterable. [clinic start generated code]*/ static PyObject * -py_list_extend_impl(PyListObject *self, PyObject *iterable) -/*[clinic end generated code: output=a2f115ceace2c845 input=1d42175414e1a5f3]*/ +list_extend(PyListObject *self, PyObject *iterable) +/*[clinic end generated code: output=630fb3bca0c8e789 input=979da7597a515791]*/ { - return _PyList_Extend(self, iterable); + if (_list_extend(self, iterable) < 0) { + return NULL; + } + Py_RETURN_NONE; } +PyObject * +_PyList_Extend(PyListObject *self, PyObject *iterable) +{ + return list_extend(self, iterable); +} int PyList_Extend(PyObject *self, PyObject *iterable) @@ -1153,7 +1430,7 @@ PyList_Extend(PyObject *self, PyObject *iterable) PyErr_BadInternalCall(); return -1; } - return list_extend((PyListObject*)self, iterable); + return _list_extend((PyListObject*)self, iterable); } @@ -1164,7 +1441,9 @@ PyList_Clear(PyObject *self) PyErr_BadInternalCall(); return -1; } + Py_BEGIN_CRITICAL_SECTION(self); list_clear((PyListObject*)self); + Py_END_CRITICAL_SECTION(); return 0; } @@ -1173,7 +1452,7 @@ static PyObject * list_inplace_concat(PyObject *_self, PyObject *other) { PyListObject *self = (PyListObject *)_self; - if (list_extend(self, other) < 0) { + if (_list_extend(self, other) < 0) { return NULL; } return Py_NewRef(self); @@ -1349,6 +1628,15 @@ sortslice_advance(sortslice *slice, Py_ssize_t n) /* Avoid malloc for small temp arrays. */ #define MERGESTATE_TEMP_SIZE 256 +/* The largest value of minrun. This must be a power of 2, and >= 1, so that + * the compute_minrun() algorithm guarantees to return a result no larger than + * this, + */ +#define MAX_MINRUN 64 +#if ((MAX_MINRUN) < 1) || ((MAX_MINRUN) & ((MAX_MINRUN) - 1)) +#error "MAX_MINRUN must be a power of 2, and >= 1" +#endif + /* One MergeState exists on the stack per invocation of mergesort. It's just * a convenient way to pass state around among the helper functions. */ @@ -1406,123 +1694,248 @@ struct s_MergeState { int (*tuple_elem_compare)(PyObject *, PyObject *, MergeState *); }; -/* binarysort is the best method for sorting small arrays: it does - few compares, but can do data movement quadratic in the number of - elements. - [lo, hi) is a contiguous slice of a list, and is sorted via - binary insertion. This sort is stable. - On entry, must have lo <= start <= hi, and that [lo, start) is already - sorted (pass start == lo if you don't know!). - If islt() complains return -1, else 0. +/* binarysort is the best method for sorting small arrays: it does few + compares, but can do data movement quadratic in the number of elements. + ss->keys is viewed as an array of n kays, a[:n]. a[:ok] is already sorted. + Pass ok = 0 (or 1) if you don't know. + It's sorted in-place, by a stable binary insertion sort. If ss->values + isn't NULL, it's permuted in lockstap with ss->keys. + On entry, must have n >= 1, and 0 <= ok <= n <= MAX_MINRUN. + Return -1 if comparison raises an exception, else 0. Even in case of error, the output slice will be some permutation of the input (nothing is lost or duplicated). */ static int -binarysort(MergeState *ms, sortslice lo, PyObject **hi, PyObject **start) +binarysort(MergeState *ms, const sortslice *ss, Py_ssize_t n, Py_ssize_t ok) { - Py_ssize_t k; - PyObject **l, **p, **r; + Py_ssize_t k; /* for IFLT macro expansion */ + PyObject ** const a = ss->keys; + PyObject ** const v = ss->values; + const bool has_values = v != NULL; PyObject *pivot; - - assert(lo.keys <= start && start <= hi); - /* assert [lo, start) is sorted */ - if (lo.keys == start) - ++start; - for (; start < hi; ++start) { - /* set l to where *start belongs */ - l = lo.keys; - r = start; - pivot = *r; - /* Invariants: - * pivot >= all in [lo, l). - * pivot < all in [r, start). - * The second is vacuously true at the start. + Py_ssize_t M; + + assert(0 <= ok && ok <= n && 1 <= n && n <= MAX_MINRUN); + /* assert a[:ok] is sorted */ + if (! ok) + ++ok; + /* Regular insertion sort has average- and worst-case O(n**2) cost + for both # of comparisons and number of bytes moved. But its branches + are highly predictable, and it loves sorted input (n-1 compares and no + data movement). This is significant in cases like sortperf.py's %sort, + where an out-of-order element near the start of a run is moved into + place slowly but then the remaining elements up to length minrun are + generally at worst one slot away from their correct position (so only + need 1 or 2 commpares to resolve). If comparisons are very fast (such + as for a list of Python floats), the simple inner loop leaves it + very competitive with binary insertion, despite that it does + significantly more compares overall on random data. + + Binary insertion sort has worst, average, and best case O(n log n) + cost for # of comparisons, but worst and average case O(n**2) cost + for data movement. The more expensive comparisons, the more important + the comparison advantage. But its branches are less predictable the + more "randomish" the data, and that's so significant its worst case + in real life is random input rather than reverse-ordered (which does + about twice the data movement than random input does). + + Note that the number of bytes moved doesn't seem to matter. MAX_MINRUN + of 64 is so small that the key and value pointers all fit in a corner + of L1 cache, and moving things around in that is very fast. */ +#if 0 // ordinary insertion sort. + PyObject * vpivot = NULL; + for (; ok < n; ++ok) { + pivot = a[ok]; + if (has_values) + vpivot = v[ok]; + for (M = ok - 1; M >= 0; --M) { + k = ISLT(pivot, a[M]); + if (k < 0) { + a[M + 1] = pivot; + if (has_values) + v[M + 1] = vpivot; + goto fail; + } + else if (k) { + a[M + 1] = a[M]; + if (has_values) + v[M + 1] = v[M]; + } + else + break; + } + a[M + 1] = pivot; + if (has_values) + v[M + 1] = vpivot; + } +#else // binary insertion sort + Py_ssize_t L, R; + for (; ok < n; ++ok) { + /* set L to where a[ok] belongs */ + L = 0; + R = ok; + pivot = a[ok]; + /* Slice invariants. vacuously true at the start: + * all a[0:L] <= pivot + * all a[L:R] unknown + * all a[R:ok] > pivot */ - assert(l < r); + assert(L < R); do { - p = l + ((r - l) >> 1); - IFLT(pivot, *p) - r = p; + /* don't do silly ;-) things to prevent overflow when finding + the midpoint; L and R are very far from filling a Py_ssize_t */ + M = (L + R) >> 1; +#if 1 // straightforward, but highly unpredictable branch on random data + IFLT(pivot, a[M]) + R = M; else - l = p+1; - } while (l < r); - assert(l == r); - /* The invariants still hold, so pivot >= all in [lo, l) and - pivot < all in [l, start), so pivot belongs at l. Note - that if there are elements equal to pivot, l points to the - first slot after them -- that's why this sort is stable. - Slide over to make room. - Caution: using memmove is much slower under MSVC 5; - we're not usually moving many slots. */ - for (p = start; p > l; --p) - *p = *(p-1); - *l = pivot; - if (lo.values != NULL) { - Py_ssize_t offset = lo.values - lo.keys; - p = start + offset; - pivot = *p; - l += offset; - for (p = start + offset; p > l; --p) - *p = *(p-1); - *l = pivot; + L = M + 1; +#else + /* Try to get compiler to generate conditional move instructions + instead. Works fine, but leaving it disabled for now because + it's not yielding consistently faster sorts. Needs more + investigation. More computation in the inner loop adds its own + costs, which can be significant when compares are fast. */ + k = ISLT(pivot, a[M]); + if (k < 0) + goto fail; + Py_ssize_t Mp1 = M + 1; + R = k ? M : R; + L = k ? L : Mp1; +#endif + } while (L < R); + assert(L == R); + /* a[:L] holds all elements from a[:ok] <= pivot now, so pivot belongs + at index L. Slide a[L:ok] to the right a slot to make room for it. + Caution: using memmove is much slower under MSVC 5; we're not + usually moving many slots. Years later: under Visual Studio 2022, + memmove seems just slightly slower than doing it "by hand". */ + for (M = ok; M > L; --M) + a[M] = a[M - 1]; + a[L] = pivot; + if (has_values) { + pivot = v[ok]; + for (M = ok; M > L; --M) + v[M] = v[M - 1]; + v[L] = pivot; } } +#endif // pick binary or regular insertion sort return 0; fail: return -1; } -/* -Return the length of the run beginning at lo, in the slice [lo, hi). lo < hi -is required on entry. "A run" is the longest ascending sequence, with - - lo[0] <= lo[1] <= lo[2] <= ... - -or the longest descending sequence, with - - lo[0] > lo[1] > lo[2] > ... +static void +sortslice_reverse(sortslice *s, Py_ssize_t n) +{ + reverse_slice(s->keys, &s->keys[n]); + if (s->values != NULL) + reverse_slice(s->values, &s->values[n]); +} -Boolean *descending is set to 0 in the former case, or to 1 in the latter. -For its intended use in a stable mergesort, the strictness of the defn of -"descending" is needed so that the caller can safely reverse a descending -sequence without violating stability (strict > ensures there are no equal -elements to get out of order). +/* +Return the length of the run beginning at slo->keys, spanning no more than +nremaining elements. The run beginning there may be ascending or descending, +but the function permutes it in place, if needed, so that it's always ascending +upon return. Returns -1 in case of error. */ static Py_ssize_t -count_run(MergeState *ms, PyObject **lo, PyObject **hi, int *descending) +count_run(MergeState *ms, sortslice *slo, Py_ssize_t nremaining) { - Py_ssize_t k; + Py_ssize_t k; /* used by IFLT macro expansion */ Py_ssize_t n; + PyObject ** const lo = slo->keys; - assert(lo < hi); - *descending = 0; - ++lo; - if (lo == hi) - return 1; - - n = 2; - IFLT(*lo, *(lo-1)) { - *descending = 1; - for (lo = lo+1; lo < hi; ++lo, ++n) { - IFLT(*lo, *(lo-1)) - ; - else - break; - } + /* In general, as things go on we've established that the slice starts + with a monotone run of n elements, starting at lo. */ + + /* We're n elements into the slice, and the most recent neq+1 elements are + * all equal. This reverses them in-place, and resets neq for reuse. + */ +#define REVERSE_LAST_NEQ \ + if (neq) { \ + sortslice slice = *slo; \ + ++neq; \ + sortslice_advance(&slice, n - neq); \ + sortslice_reverse(&slice, neq); \ + neq = 0; \ + } + + /* Sticking to only __lt__ compares is confusing and error-prone. But in + * this routine, almost all uses of IFLT can be captured by tiny macros + * giving mnemonic names to the intent. Note that inline functions don't + * work for this (IFLT expands to code including `goto fail`). + */ +#define IF_NEXT_LARGER IFLT(lo[n-1], lo[n]) +#define IF_NEXT_SMALLER IFLT(lo[n], lo[n-1]) + + assert(nremaining); + /* try ascending run first */ + for (n = 1; n < nremaining; ++n) { + IF_NEXT_SMALLER + break; } - else { - for (lo = lo+1; lo < hi; ++lo, ++n) { - IFLT(*lo, *(lo-1)) + if (n == nremaining) + return n; + /* lo[n] is strictly less */ + /* If n is 1 now, then the first compare established it's a descending + * run, so fall through to the descending case. But if n > 1, there are + * n elements in an ascending run terminated by the strictly less lo[n]. + * If the first key < lo[n-1], *somewhere* along the way the sequence + * increased, so we're done (there is no descending run). + * Else first key >= lo[n-1], which implies that the entire ascending run + * consists of equal elements. In that case, this is a descending run, + * and we reverse the all-equal prefix in-place. + */ + if (n > 1) { + IFLT(lo[0], lo[n-1]) + return n; + sortslice_reverse(slo, n); + } + ++n; /* in all cases it's been established that lo[n] has been resolved */ + + /* Finish descending run. All-squal subruns are reversed in-place on the + * fly. Their original order will be restored at the end by the whole-slice + * reversal. + */ + Py_ssize_t neq = 0; + for ( ; n < nremaining; ++n) { + IF_NEXT_SMALLER { + /* This ends the most recent run of equal elements, but still in + * the "descending" direction. + */ + REVERSE_LAST_NEQ + } + else { + IF_NEXT_LARGER /* descending run is over */ break; + else /* not x < y and not y < x implies x == y */ + ++neq; } } + REVERSE_LAST_NEQ + sortslice_reverse(slo, n); /* transform to ascending run */ + + /* And after reversing, it's possible this can be extended by a + * naturally increasing suffix; e.g., [3, 2, 3, 4, 1] makes an + * ascending run from the first 4 elements. + */ + for ( ; n < nremaining; ++n) { + IF_NEXT_SMALLER + break; + } return n; fail: return -1; + +#undef REVERSE_LAST_NEQ +#undef IF_NEXT_SMALLER +#undef IF_NEXT_LARGER } /* @@ -2220,10 +2633,10 @@ merge_force_collapse(MergeState *ms) /* Compute a good value for the minimum run length; natural runs shorter * than this are boosted artificially via binary insertion. * - * If n < 64, return n (it's too small to bother with fancy stuff). - * Else if n is an exact power of 2, return 32. - * Else return an int k, 32 <= k <= 64, such that n/k is close to, but - * strictly less than, an exact power of 2. + * If n < MAX_MINRUN return n (it's too small to bother with fancy stuff). + * Else if n is an exact power of 2, return MAX_MINRUN / 2. + * Else return an int k, MAX_MINRUN / 2 <= k <= MAX_MINRUN, such that n/k is + * close to, but strictly less than, an exact power of 2. * * See listsort.txt for more info. */ @@ -2233,21 +2646,13 @@ merge_compute_minrun(Py_ssize_t n) Py_ssize_t r = 0; /* becomes 1 if any 1 bits are shifted off */ assert(n >= 0); - while (n >= 64) { + while (n >= MAX_MINRUN) { r |= n & 1; n >>= 1; } return n + r; } -static void -reverse_sortslice(sortslice *s, Py_ssize_t n) -{ - reverse_slice(s->keys, &s->keys[n]); - if (s->values != NULL) - reverse_slice(s->values, &s->values[n]); -} - /* Here we define custom comparison functions to optimize for the cases one commonly * encounters in practice: homogeneous lists, often of one of the basic types. */ @@ -2416,6 +2821,7 @@ unsafe_tuple_compare(PyObject *v, PyObject *w, MergeState *ms) * duplicated). */ /*[clinic input] +@critical_section list.sort * @@ -2435,7 +2841,7 @@ The reverse flag can be set to sort in descending order. static PyObject * list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) -/*[clinic end generated code: output=57b9f9c5e23fbe42 input=a74c4cd3ec6b5c08]*/ +/*[clinic end generated code: output=57b9f9c5e23fbe42 input=667bf25d0e3a3676]*/ { MergeState ms; Py_ssize_t nremaining; @@ -2462,7 +2868,7 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) saved_ob_item = self->ob_item; saved_allocated = self->allocated; Py_SET_SIZE(self, 0); - self->ob_item = NULL; + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item, NULL); self->allocated = -1; /* any operation will reset it to >= 0 */ if (keyfunc == NULL) { @@ -2614,20 +3020,17 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) */ minrun = merge_compute_minrun(nremaining); do { - int descending; Py_ssize_t n; /* Identify next run. */ - n = count_run(&ms, lo.keys, lo.keys + nremaining, &descending); + n = count_run(&ms, &lo, nremaining); if (n < 0) goto fail; - if (descending) - reverse_sortslice(&lo, n); /* If short, extend to min(minrun, nremaining). */ if (n < minrun) { const Py_ssize_t force = nremaining <= minrun ? nremaining : minrun; - if (binarysort(&ms, lo, lo.keys + force, lo.keys + n) < 0) + if (binarysort(&ms, &lo, force, n) < 0) goto fail; n = force; } @@ -2682,15 +3085,21 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) final_ob_item = self->ob_item; i = Py_SIZE(self); Py_SET_SIZE(self, saved_ob_size); - self->ob_item = saved_ob_item; - self->allocated = saved_allocated; + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item, saved_ob_item); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->allocated, saved_allocated); if (final_ob_item != NULL) { /* we cannot use list_clear() for this because it does not guarantee that the list is really empty when it returns */ while (--i >= 0) { Py_XDECREF(final_ob_item[i]); } - PyMem_Free(final_ob_item); +#ifdef Py_GIL_DISABLED + ensure_shared_on_resize(self); + bool use_qsbr = _PyObject_GC_IS_SHARED(self); +#else + bool use_qsbr = false; +#endif + free_list_items(final_ob_item, use_qsbr); } return Py_XNewRef(result); } @@ -2704,7 +3113,9 @@ PyList_Sort(PyObject *v) PyErr_BadInternalCall(); return -1; } + Py_BEGIN_CRITICAL_SECTION(v); v = list_sort_impl((PyListObject *)v, NULL, 0); + Py_END_CRITICAL_SECTION(); if (v == NULL) return -1; Py_DECREF(v); @@ -2760,7 +3171,26 @@ PyList_AsTuple(PyObject *v) } PyObject * -_PyList_FromArraySteal(PyObject *const *src, Py_ssize_t n) +_PyList_AsTupleAndClear(PyListObject *self) +{ + assert(self != NULL); + PyObject *ret; + if (self->ob_item == NULL) { + return PyTuple_New(0); + } + Py_BEGIN_CRITICAL_SECTION(self); + PyObject **items = self->ob_item; + Py_ssize_t size = Py_SIZE(self); + self->ob_item = NULL; + Py_SET_SIZE(self, 0); + ret = _PyTuple_FromArraySteal(items, size); + free_list_items(items, false); + Py_END_CRITICAL_SECTION(); + return ret; +} + +PyObject * +_PyList_FromStackRefStealOnSuccess(const _PyStackRef *src, Py_ssize_t n) { if (n == 0) { return PyList_New(0); @@ -2768,14 +3198,13 @@ _PyList_FromArraySteal(PyObject *const *src, Py_ssize_t n) PyListObject *list = (PyListObject *)PyList_New(n); if (list == NULL) { - for (Py_ssize_t i = 0; i < n; i++) { - Py_DECREF(src[i]); - } return NULL; } PyObject **dst = list->ob_item; - memcpy(dst, src, n * sizeof(PyObject *)); + for (Py_ssize_t i = 0; i < n; i++) { + dst[i] = PyStackRef_AsPyObjectSteal(src[i]); + } return (PyObject *)list; } @@ -2821,7 +3250,7 @@ list_index_impl(PyListObject *self, PyObject *value, Py_ssize_t start, else if (cmp < 0) return NULL; } - PyErr_Format(PyExc_ValueError, "%R is not in list", value); + PyErr_SetString(PyExc_ValueError, "list.index(x): x not in list"); return NULL; } @@ -2884,15 +3313,14 @@ list_remove_impl(PyListObject *self, PyObject *value) int cmp = PyObject_RichCompareBool(obj, value, Py_EQ); Py_DECREF(obj); if (cmp > 0) { - if (list_ass_slice(self, i, i+1, - (PyObject *)NULL) == 0) + if (list_ass_slice_lock_held(self, i, i+1, NULL) == 0) Py_RETURN_NONE; return NULL; } else if (cmp < 0) return NULL; } - PyErr_Format(PyExc_ValueError, "%R is not in list", value); + PyErr_SetString(PyExc_ValueError, "list.remove(x): x not in list"); return NULL; } @@ -2960,7 +3388,14 @@ list_richcompare_impl(PyObject *v, PyObject *w, int op) } /* Compare the final item again using the proper operator */ - return PyObject_RichCompare(vl->ob_item[i], wl->ob_item[i], op); + PyObject *vitem = vl->ob_item[i]; + PyObject *witem = wl->ob_item[i]; + Py_INCREF(vitem); + Py_INCREF(witem); + PyObject *result = PyObject_RichCompare(vl->ob_item[i], wl->ob_item[i], op); + Py_DECREF(vitem); + Py_DECREF(witem); + return result; } static PyObject * @@ -2997,10 +3432,12 @@ list___init___impl(PyListObject *self, PyObject *iterable) /* Empty previous contents */ if (self->ob_item != NULL) { + Py_BEGIN_CRITICAL_SECTION(self); list_clear(self); + Py_END_CRITICAL_SECTION(); } if (iterable != NULL) { - if (list_extend(self, iterable) < 0) { + if (_list_extend(self, iterable) < 0) { return -1; } } @@ -3061,7 +3498,7 @@ static PyMethodDef list_methods[] = { LIST_COPY_METHODDEF LIST_APPEND_METHODDEF LIST_INSERT_METHODDEF - PY_LIST_EXTEND_METHODDEF + LIST_EXTEND_METHODDEF LIST_POP_METHODDEF LIST_REMOVE_METHODDEF LIST_INDEX_METHODDEF @@ -3085,6 +3522,45 @@ static PySequenceMethods list_as_sequence = { list_inplace_repeat, /* sq_inplace_repeat */ }; +static inline PyObject * +list_slice_step_lock_held(PyListObject *a, Py_ssize_t start, Py_ssize_t step, Py_ssize_t len) +{ + PyListObject *np = (PyListObject *)list_new_prealloc(len); + if (np == NULL) { + return NULL; + } + size_t cur; + Py_ssize_t i; + PyObject **src = a->ob_item; + PyObject **dest = np->ob_item; + for (cur = start, i = 0; i < len; + cur += (size_t)step, i++) { + PyObject *v = src[cur]; + dest[i] = Py_NewRef(v); + } + Py_SET_SIZE(np, len); + return (PyObject *)np; +} + +static PyObject * +list_slice_wrap(PyListObject *aa, Py_ssize_t start, Py_ssize_t stop, Py_ssize_t step) +{ + PyObject *res = NULL; + Py_BEGIN_CRITICAL_SECTION(aa); + Py_ssize_t len = PySlice_AdjustIndices(Py_SIZE(aa), &start, &stop, step); + if (len <= 0) { + res = PyList_New(0); + } + else if (step == 1) { + res = list_slice_lock_held(aa, start, stop); + } + else { + res = list_slice_step_lock_held(aa, start, step, len); + } + Py_END_CRITICAL_SECTION(); + return res; +} + static PyObject * list_subscript(PyObject* _self, PyObject* item) { @@ -3099,38 +3575,11 @@ list_subscript(PyObject* _self, PyObject* item) return list_item((PyObject *)self, i); } else if (PySlice_Check(item)) { - Py_ssize_t start, stop, step, slicelength, i; - size_t cur; - PyObject* result; - PyObject* it; - PyObject **src, **dest; - + Py_ssize_t start, stop, step; if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } - slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, - step); - - if (slicelength <= 0) { - return PyList_New(0); - } - else if (step == 1) { - return list_slice(self, start, stop); - } - else { - result = list_new_prealloc(slicelength); - if (!result) return NULL; - - src = self->ob_item; - dest = ((PyListObject *)result)->ob_item; - for (cur = start, i = 0; i < slicelength; - cur += (size_t)step, i++) { - it = Py_NewRef(src[cur]); - dest[i] = it; - } - Py_SET_SIZE(result, slicelength); - return result; - } + return list_slice_wrap(self, start, stop, step); } else { PyErr_Format(PyExc_TypeError, @@ -3140,9 +3589,28 @@ list_subscript(PyObject* _self, PyObject* item) } } +static Py_ssize_t +adjust_slice_indexes(PyListObject *lst, + Py_ssize_t *start, Py_ssize_t *stop, + Py_ssize_t step) +{ + Py_ssize_t slicelength = PySlice_AdjustIndices(Py_SIZE(lst), start, stop, + step); + + /* Make sure s[5:2] = [..] inserts at the right place: + before 5, not before 2. */ + if ((step < 0 && *start < *stop) || + (step > 0 && *start > *stop)) + *stop = *start; + + return slicelength; +} + static int -list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) +list_ass_subscript_lock_held(PyObject *_self, PyObject *item, PyObject *value) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(_self); + PyListObject *self = (PyListObject *)_self; if (_PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); @@ -3150,25 +3618,14 @@ list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) return -1; if (i < 0) i += PyList_GET_SIZE(self); - return list_ass_item((PyObject *)self, i, value); + return list_ass_item_lock_held(self, i, value); } else if (PySlice_Check(item)) { - Py_ssize_t start, stop, step, slicelength; + Py_ssize_t start, stop, step; if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } - slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, - step); - - if (step == 1) - return list_ass_slice(self, start, stop, value); - - /* Make sure s[5:2] = [..] inserts at the right place: - before 5, not before 2. */ - if ((step < 0 && start < stop) || - (step > 0 && start > stop)) - stop = start; if (value == NULL) { /* delete slice */ @@ -3177,6 +3634,12 @@ list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) Py_ssize_t i; int res; + Py_ssize_t slicelength = adjust_slice_indexes(self, &start, &stop, + step); + + if (step == 1) + return list_ass_slice_lock_held(self, start, stop, value); + if (slicelength <= 0) return 0; @@ -3241,8 +3704,8 @@ list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) /* protect against a[::-1] = a */ if (self == (PyListObject*)value) { - seq = list_slice((PyListObject*)value, 0, - PyList_GET_SIZE(value)); + seq = list_slice_lock_held((PyListObject *)value, 0, + Py_SIZE(value)); } else { seq = PySequence_Fast(value, @@ -3252,6 +3715,15 @@ list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) if (!seq) return -1; + Py_ssize_t slicelength = adjust_slice_indexes(self, &start, &stop, + step); + + if (step == 1) { + int res = list_ass_slice_lock_held(self, start, stop, seq); + Py_DECREF(seq); + return res; + } + if (PySequence_Fast_GET_SIZE(seq) != slicelength) { PyErr_Format(PyExc_ValueError, "attempt to assign sequence of " @@ -3303,6 +3775,24 @@ list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) } } +static int +list_ass_subscript(PyObject *self, PyObject *item, PyObject *value) +{ + int res; +#ifdef Py_GIL_DISABLED + if (PySlice_Check(item) && value != NULL && PyList_CheckExact(value)) { + Py_BEGIN_CRITICAL_SECTION2(self, value); + res = list_ass_subscript_lock_held(self, item, value); + Py_END_CRITICAL_SECTION2(); + return res; + } +#endif + Py_BEGIN_CRITICAL_SECTION(self); + res = list_ass_subscript_lock_held(self, item, value); + Py_END_CRITICAL_SECTION(); + return res; +} + static PyMappingMethods list_as_mapping = { list_length, list_subscript, @@ -3352,6 +3842,7 @@ PyTypeObject PyList_Type = { PyType_GenericNew, /* tp_new */ PyObject_GC_Del, /* tp_free */ .tp_vectorcall = list_vectorcall, + .tp_version_tag = _Py_TYPE_VERSION_LIST, }; /*********************** List Iterator **************************/ @@ -3412,15 +3903,17 @@ PyTypeObject PyListIter_Type = { static PyObject * list_iter(PyObject *seq) { - _PyListIterObject *it; - if (!PyList_Check(seq)) { PyErr_BadInternalCall(); return NULL; } - it = PyObject_GC_New(_PyListIterObject, &PyListIter_Type); - if (it == NULL) - return NULL; + _PyListIterObject *it = _Py_FREELIST_POP(_PyListIterObject, list_iters); + if (it == NULL) { + it = PyObject_GC_New(_PyListIterObject, &PyListIter_Type); + if (it == NULL) { + return NULL; + } + } it->it_index = 0; it->it_seq = (PyListObject *)Py_NewRef(seq); _PyObject_GC_TRACK(it); @@ -3433,7 +3926,8 @@ listiter_dealloc(PyObject *self) _PyListIterObject *it = (_PyListIterObject *)self; _PyObject_GC_UNTRACK(it); Py_XDECREF(it->it_seq); - PyObject_GC_Del(it); + assert(Py_IS_TYPE(self, &PyListIter_Type)); + _Py_FREELIST_FREE(list_iters, it, PyObject_GC_Del); } static int diff --git a/Objects/listsort.txt b/Objects/listsort.txt index 32a59e510f0754..f387d9c116e502 100644 --- a/Objects/listsort.txt +++ b/Objects/listsort.txt @@ -212,24 +212,43 @@ A detailed description of timsort follows. Runs ---- -count_run() returns the # of elements in the next run. A run is either -"ascending", which means non-decreasing: +count_run() returns the # of elements in the next run, and, if it's a +descending run, reverses it in-place. A run is either "ascending", which +means non-decreasing: a0 <= a1 <= a2 <= ... -or "descending", which means strictly decreasing: +or "descending", which means non-increasing: - a0 > a1 > a2 > ... + a0 >= a1 >= a2 >= ... Note that a run is always at least 2 long, unless we start at the array's -last element. - -The definition of descending is strict, because the main routine reverses -a descending run in-place, transforming a descending run into an ascending -run. Reversal is done via the obvious fast "swap elements starting at each -end, and converge at the middle" method, and that can violate stability if -the slice contains any equal elements. Using a strict definition of -descending ensures that a descending run contains distinct elements. +last element. If all elements in the array are equal, it can be viewed as +both ascending and descending. Upon return, the run count_run() identifies +is always ascending. + +Reversal is done via the obvious fast "swap elements starting at each +end, and converge at the middle" method. That can violate stability if +the slice contains any equal elements. For that reason, for a long time +the code used strict inequality (">" rather than ">=") in its definition +of descending. + +Removing that restriction required some complication: when processing a +descending run, all-equal sub-runs of elements are reversed in-place, on the +fly. Their original relative order is restored "by magic" via the final +"reverse the entire run" step. + +This makes processing descending runs a little more costly. We only use +`__lt__` comparisons, so that `x == y` has to be deduced from +`not x < y and not y < x`. But so long as a run remains strictly decreasing, +only one of those compares needs to be done per loop iteration. So the primsry +extra cost is paid only when there are equal elements, and they get some +compensating benefit by not needing to end the descending run. + +There's one more trick added since the original: after reversing a descending +run, it's possible that it can be extended by an adjacent ascending run. For +example, given [3, 2, 1, 3, 4, 5, 0], the 3-element descending prefix is +reversed in-place, and then extended by [3, 4, 5]. If an array is random, it's very unlikely we'll see long runs. If a natural run contains less than minrun elements (see next section), the main loop @@ -251,9 +270,9 @@ result. This has two primary good effects: Computing minrun ---------------- -If N < 64, minrun is N. IOW, binary insertion sort is used for the whole -array then; it's hard to beat that given the overheads of trying something -fancier (see note BINSORT). +If N < MAX_MINRUN, minrun is N. IOW, binary insertion sort is used for the +whole array then; it's hard to beat that given the overheads of trying +something fancier (see note BINSORT). When N is a power of 2, testing on random data showed that minrun values of 16, 32, 64 and 128 worked about equally well. At 256 the data-movement cost @@ -291,12 +310,13 @@ place, and r < minrun is small compared to N), or q a little larger than a power of 2 regardless of r (then we've got a case similar to "2112", again leaving too little work for the last merge to do). -Instead we pick a minrun in range(32, 65) such that N/minrun is exactly a -power of 2, or if that isn't possible, is close to, but strictly less than, -a power of 2. This is easier to do than it may sound: take the first 6 -bits of N, and add 1 if any of the remaining bits are set. In fact, that -rule covers every case in this section, including small N and exact powers -of 2; merge_compute_minrun() is a deceptively simple function. +Instead we pick a minrun in range(MAX_MINRUN / 2, MAX_MINRUN + 1) such that +N/minrun is exactly a power of 2, or if that isn't possible, is close to, but +strictly less than, a power of 2. This is easier to do than it may sound: +take the first log2(MAX_MINRUN) bits of N, and add 1 if any of the remaining +bits are set. In fact, that rule covers every case in this section, including +small N and exact powers of 2; merge_compute_minrun() is a deceptively simple +function. The Merge Pattern diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt index 0f3599340318f0..335e441cfded3d 100644 --- a/Objects/lnotab_notes.txt +++ b/Objects/lnotab_notes.txt @@ -1,7 +1,7 @@ Description of the internal format of the line number table in Python 3.10 and earlier. -(For 3.11 onwards, see Objects/locations.md) +(For 3.11 onwards, see InternalDocs/code_objects.md) Conceptually, the line number table consists of a sequence of triples: start-offset (inclusive), end-offset (exclusive), line-number. diff --git a/Objects/locations.md b/Objects/locations.md deleted file mode 100644 index 18a338a95978a9..00000000000000 --- a/Objects/locations.md +++ /dev/null @@ -1,69 +0,0 @@ -# Locations table - -For versions up to 3.10 see ./lnotab_notes.txt - -In version 3.11 the `co_linetable` bytes object of code objects contains a compact representation of the positions returned by the `co_positions()` iterator. - -The `co_linetable` consists of a sequence of location entries. -Each entry starts with a byte with the most significant bit set, followed by zero or more bytes with most significant bit unset. - -Each entry contains the following information: -* The number of code units covered by this entry (length) -* The start line -* The end line -* The start column -* The end column - -The first byte has the following format: - -Bit 7 | Bits 3-6 | Bits 0-2 - ---- | ---- | ---- - 1 | Code | Length (in code units) - 1 - -The codes are enumerated in the `_PyCodeLocationInfoKind` enum. - -## Variable length integer encodings - -Integers are often encoded using a variable length integer encoding - -### Unsigned integers (varint) - -Unsigned integers are encoded in 6 bit chunks, least significant first. -Each chunk but the last has bit 6 set. -For example: - -* 63 is encoded as `0x3f` -* 200 is encoded as `0x48`, `0x03` - -### Signed integers (svarint) - -Signed integers are encoded by converting them to unsigned integers, using the following function: -```Python -def convert(s): - if s < 0: - return ((-s)<<1) | 1 - else: - return (s<<1) -``` - -## Location entries - -The meaning of the codes and the following bytes are as follows: - -Code | Meaning | Start line | End line | Start column | End column - ---- | ---- | ---- | ---- | ---- | ---- - 0-9 | Short form | Δ 0 | Δ 0 | See below | See below - 10-12 | One line form | Δ (code - 10) | Δ 0 | unsigned byte | unsigned byte - 13 | No column info | Δ svarint | Δ 0 | None | None - 14 | Long form | Δ svarint | Δ varint | varint | varint - 15 | No location | None | None | None | None - -The Δ means the value is encoded as a delta from another value: -* Start line: Delta from the previous start line, or `co_firstlineno` for the first entry. -* End line: Delta from the start line - -### The short forms - -Codes 0-9 are the short forms. The short form consists of two bytes, the second byte holding additional column information. The code is the start column divided by 8 (and rounded down). -* Start column: `(code*8) + ((second_byte>>4)&7)` -* End column: `start_column + (second_byte&15)` diff --git a/Objects/longobject.c b/Objects/longobject.c index 2d1c6ad788e281..370328dcfe8c9a 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6,6 +6,7 @@ #include "pycore_bitutils.h" // _Py_popcount32() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_call.h" // _PyObject_MakeTpCall +#include "pycore_freelist.h" // _Py_FREELIST_FREE, _Py_FREELIST_POP #include "pycore_long.h" // _Py_SmallInts #include "pycore_object.h" // _PyObject_Init() #include "pycore_runtime.h" // _PY_NSMALLPOSINTS @@ -31,11 +32,18 @@ class int "PyObject *" "&PyLong_Type" /* If defined, use algorithms from the _pylong.py module */ #define WITH_PYLONG_MODULE 1 +// Forward declarations +static PyLongObject* long_neg(PyLongObject *v); +static PyLongObject *x_divrem(PyLongObject *, PyLongObject *, PyLongObject **); +static PyObject* long_long(PyObject *v); +static PyObject* long_lshift_int64(PyLongObject *a, int64_t shiftby); + + static inline void _Py_DECREF_INT(PyLongObject *op) { assert(PyLong_CheckExact(op)); - _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED((PyObject *)op, _PyLong_ExactDealloc); } static inline int @@ -133,14 +141,22 @@ long_normalize(PyLongObject *v) /* Allocate a new int object with size digits. Return NULL and set exception if we run out of memory. */ -#define MAX_LONG_DIGITS \ +#if SIZEOF_SIZE_T < 8 +# define MAX_LONG_DIGITS \ ((PY_SSIZE_T_MAX - offsetof(PyLongObject, long_value.ob_digit))/sizeof(digit)) +#else +/* Guarantee that the number of bits fits in int64_t. + This is more than an exbibyte, that is more than many of modern + architectures support in principle. + -1 is added to avoid overflow in _PyLong_Frexp(). */ +# define MAX_LONG_DIGITS ((INT64_MAX-1) / PyLong_SHIFT) +#endif -PyLongObject * -_PyLong_New(Py_ssize_t size) +static PyLongObject * +long_alloc(Py_ssize_t size) { assert(size >= 0); - PyLongObject *result; + PyLongObject *result = NULL; if (size > (Py_ssize_t)MAX_LONG_DIGITS) { PyErr_SetString(PyExc_OverflowError, "too many digits in integer"); @@ -149,25 +165,37 @@ _PyLong_New(Py_ssize_t size) /* Fast operations for single digit integers (including zero) * assume that there is always at least one digit present. */ Py_ssize_t ndigits = size ? size : 1; - /* Number of bytes needed is: offsetof(PyLongObject, ob_digit) + - sizeof(digit)*size. Previous incarnations of this code used - sizeof() instead of the offsetof, but this risks being - incorrect in the presence of padding between the header - and the digits. */ - result = PyObject_Malloc(offsetof(PyLongObject, long_value.ob_digit) + - ndigits*sizeof(digit)); - if (!result) { - PyErr_NoMemory(); - return NULL; + + if (ndigits == 1) { + result = (PyLongObject *)_Py_FREELIST_POP(PyLongObject, ints); + } + if (result == NULL) { + /* Number of bytes needed is: offsetof(PyLongObject, ob_digit) + + sizeof(digit)*size. Previous incarnations of this code used + sizeof() instead of the offsetof, but this risks being + incorrect in the presence of padding between the header + and the digits. */ + result = PyObject_Malloc(offsetof(PyLongObject, long_value.ob_digit) + + ndigits*sizeof(digit)); + if (!result) { + PyErr_NoMemory(); + return NULL; + } + _PyObject_Init((PyObject*)result, &PyLong_Type); } _PyLong_SetSignAndDigitCount(result, size != 0, size); - _PyObject_Init((PyObject*)result, &PyLong_Type); /* The digit has to be initialized explicitly to avoid * use-of-uninitialized-value. */ result->long_value.ob_digit[0] = 0; return result; } +PyLongObject * +_PyLong_New(Py_ssize_t size) +{ + return long_alloc(size); +} + PyLongObject * _PyLong_FromDigits(int negative, Py_ssize_t digit_count, digit *digits) { @@ -175,9 +203,8 @@ _PyLong_FromDigits(int negative, Py_ssize_t digit_count, digit *digits) if (digit_count == 0) { return (PyLongObject *)_PyLong_GetZero(); } - PyLongObject *result = _PyLong_New(digit_count); + PyLongObject *result = long_alloc(digit_count); if (result == NULL) { - PyErr_NoMemory(); return NULL; } _PyLong_SetSignAndDigitCount(result, negative?-1:1, digit_count); @@ -189,15 +216,29 @@ PyObject * _PyLong_Copy(PyLongObject *src) { assert(src != NULL); + int sign; if (_PyLong_IsCompact(src)) { stwodigits ival = medium_value(src); if (IS_SMALL_INT(ival)) { return get_small_int((sdigit)ival); } + sign = _PyLong_CompactSign(src); + } + else { + sign = _PyLong_NonCompactSign(src); } + Py_ssize_t size = _PyLong_DigitCount(src); - return (PyObject *)_PyLong_FromDigits(_PyLong_IsNegative(src), size, src->long_value.ob_digit); + PyLongObject *result = long_alloc(size); + + if (result == NULL) { + return NULL; + } + _PyLong_SetSignAndDigitCount(result, sign, size); + memcpy(result->long_value.ob_digit, src->long_value.ob_digit, + size * sizeof(digit)); + return (PyObject *)result; } static PyObject * @@ -205,15 +246,18 @@ _PyLong_FromMedium(sdigit x) { assert(!IS_SMALL_INT(x)); assert(is_medium_int(x)); - /* We could use a freelist here */ - PyLongObject *v = PyObject_Malloc(sizeof(PyLongObject)); + + PyLongObject *v = (PyLongObject *)_Py_FREELIST_POP(PyLongObject, ints); if (v == NULL) { - PyErr_NoMemory(); - return NULL; + v = PyObject_Malloc(sizeof(PyLongObject)); + if (v == NULL) { + PyErr_NoMemory(); + return NULL; + } + _PyObject_Init((PyObject*)v, &PyLong_Type); } digit abs_x = x < 0 ? -x : x; _PyLong_SetSignAndDigitCount(v, x<0?-1:1, 1); - _PyObject_Init((PyObject*)v, &PyLong_Type); v->long_value.ob_digit[0] = abs_x; return (PyObject*)v; } @@ -243,7 +287,7 @@ _PyLong_FromLarge(stwodigits ival) ++ndigits; t >>= PyLong_SHIFT; } - PyLongObject *v = _PyLong_New(ndigits); + PyLongObject *v = long_alloc(ndigits); if (v != NULL) { digit *p = v->long_value.ob_digit; _PyLong_SetSignAndDigitCount(v, sign, ndigits); @@ -258,17 +302,17 @@ _PyLong_FromLarge(stwodigits ival) } /* Create a new int object from a C word-sized int */ -static inline PyObject * +static inline PyLongObject * _PyLong_FromSTwoDigits(stwodigits x) { if (IS_SMALL_INT(x)) { - return get_small_int((sdigit)x); + return (PyLongObject*)get_small_int((sdigit)x); } assert(x != 0); if (is_medium_int(x)) { - return _PyLong_FromMedium((sdigit)x); + return (PyLongObject*)_PyLong_FromMedium((sdigit)x); } - return _PyLong_FromLarge(x); + return (PyLongObject*)_PyLong_FromLarge(x); } /* If a freshly-allocated int is already shared, it must @@ -284,7 +328,7 @@ _PyLong_Negate(PyLongObject **x_p) return; } - *x_p = (PyLongObject *)_PyLong_FromSTwoDigits(-medium_value(x)); + *x_p = _PyLong_FromSTwoDigits(-medium_value(x)); Py_DECREF(x); } @@ -316,7 +360,7 @@ PyLong_FromLong(long ival) } /* Construct output value. */ - v = _PyLong_New(ndigits); + v = long_alloc(ndigits); if (v != NULL) { digit *p = v->long_value.ob_digit; _PyLong_SetSignAndDigitCount(v, ival < 0 ? -1 : 1, ndigits); @@ -334,14 +378,18 @@ PyLong_FromLong(long ival) if (IS_SMALL_UINT(ival)) { \ return get_small_int((sdigit)(ival)); \ } \ + if ((ival) <= PyLong_MASK) { \ + return _PyLong_FromMedium((sdigit)(ival)); \ + } \ + /* Do shift in two steps to avoid possible undefined behavior. */ \ + INT_TYPE t = (ival) >> PyLong_SHIFT >> PyLong_SHIFT; \ /* Count the number of Python digits. */ \ - Py_ssize_t ndigits = 0; \ - INT_TYPE t = (ival); \ + Py_ssize_t ndigits = 2; \ while (t) { \ ++ndigits; \ t >>= PyLong_SHIFT; \ } \ - PyLongObject *v = _PyLong_New(ndigits); \ + PyLongObject *v = long_alloc(ndigits); \ if (v == NULL) { \ return NULL; \ } \ @@ -401,12 +449,12 @@ PyLong_FromDouble(double dval) double frac; int i, ndig, expo, neg; neg = 0; - if (Py_IS_INFINITY(dval)) { + if (isinf(dval)) { PyErr_SetString(PyExc_OverflowError, "cannot convert float infinity to integer"); return NULL; } - if (Py_IS_NAN(dval)) { + if (isnan(dval)) { PyErr_SetString(PyExc_ValueError, "cannot convert float NaN to integer"); return NULL; @@ -418,7 +466,7 @@ PyLong_FromDouble(double dval) frac = frexp(dval, &expo); /* dval = frac*2**expo; 0.0 <= frac < 1.0 */ assert(expo > 0); ndig = (expo-1) / PyLong_SHIFT + 1; /* Number of 'digits' in result */ - v = _PyLong_New(ndig); + v = long_alloc(ndig); if (v == NULL) return NULL; frac = ldexp(frac, (expo-1) % PyLong_SHIFT + 1); @@ -483,11 +531,18 @@ PyLong_AsLongAndOverflow(PyObject *vv, int *overflow) do_decref = 1; } if (_PyLong_IsCompact(v)) { -#if SIZEOF_LONG < SIZEOF_VOID_P - intptr_t tmp = _PyLong_CompactValue(v); - res = (long)tmp; - if (res != tmp) { - *overflow = tmp < 0 ? -1 : 1; +#if SIZEOF_LONG < SIZEOF_SIZE_T + Py_ssize_t tmp = _PyLong_CompactValue(v); + if (tmp < LONG_MIN) { + *overflow = -1; + res = -1; + } + else if (tmp > LONG_MAX) { + *overflow = 1; + res = -1; + } + else { + res = (long)tmp; } #else res = _PyLong_CompactValue(v); @@ -632,14 +687,15 @@ PyLong_AsUnsignedLong(PyObject *vv) v = (PyLongObject *)vv; if (_PyLong_IsNonNegativeCompact(v)) { -#if SIZEOF_LONG < SIZEOF_VOID_P - intptr_t tmp = _PyLong_CompactValue(v); +#if SIZEOF_LONG < SIZEOF_SIZE_T + size_t tmp = (size_t)_PyLong_CompactValue(v); unsigned long res = (unsigned long)tmp; if (res != tmp) { goto overflow; } + return res; #else - return _PyLong_CompactValue(v); + return (unsigned long)(size_t)_PyLong_CompactValue(v); #endif } if (_PyLong_IsNegative(v)) { @@ -685,7 +741,7 @@ PyLong_AsSize_t(PyObject *vv) v = (PyLongObject *)vv; if (_PyLong_IsNonNegativeCompact(v)) { - return _PyLong_CompactValue(v); + return (size_t)_PyLong_CompactValue(v); } if (_PyLong_IsNegative(v)) { PyErr_SetString(PyExc_OverflowError, @@ -722,7 +778,11 @@ _PyLong_AsUnsignedLongMask(PyObject *vv) } v = (PyLongObject *)vv; if (_PyLong_IsCompact(v)) { - return (unsigned long)_PyLong_CompactValue(v); +#if SIZEOF_LONG < SIZEOF_SIZE_T + return (unsigned long)(size_t)_PyLong_CompactValue(v); +#else + return (unsigned long)(long)_PyLong_CompactValue(v); +#endif } i = _PyLong_DigitCount(v); int sign = _PyLong_NonCompactSign(v); @@ -758,18 +818,69 @@ PyLong_AsUnsignedLongMask(PyObject *op) } int -_PyLong_Sign(PyObject *vv) +PyLong_IsPositive(PyObject *obj) { + assert(obj != NULL); + if (!PyLong_Check(obj)) { + PyErr_Format(PyExc_TypeError, "expected int, got %T", obj); + return -1; + } + return _PyLong_IsPositive((PyLongObject *)obj); +} + +int +PyLong_IsNegative(PyObject *obj) +{ + assert(obj != NULL); + if (!PyLong_Check(obj)) { + PyErr_Format(PyExc_TypeError, "expected int, got %T", obj); + return -1; + } + return _PyLong_IsNegative((PyLongObject *)obj); +} + +int +PyLong_IsZero(PyObject *obj) +{ + assert(obj != NULL); + if (!PyLong_Check(obj)) { + PyErr_Format(PyExc_TypeError, "expected int, got %T", obj); + return -1; + } + return _PyLong_IsZero((PyLongObject *)obj); +} + +static int +long_sign(PyObject *vv) +{ + assert(vv != NULL); + assert(PyLong_Check(vv)); PyLongObject *v = (PyLongObject *)vv; - assert(v != NULL); - assert(PyLong_Check(v)); if (_PyLong_IsCompact(v)) { return _PyLong_CompactSign(v); } return _PyLong_NonCompactSign(v); } +int +_PyLong_Sign(PyObject *vv) +{ + return long_sign(vv); +} + +int +PyLong_GetSign(PyObject *vv, int *sign) +{ + if (!PyLong_Check(vv)) { + PyErr_Format(PyExc_TypeError, "expect int, got %T", vv); + return -1; + } + + *sign = long_sign(vv); + return 0; +} + static int bit_length_digit(digit x) { @@ -780,11 +891,11 @@ bit_length_digit(digit x) return _Py_bit_length((unsigned long)x); } -size_t +int64_t _PyLong_NumBits(PyObject *vv) { PyLongObject *v = (PyLongObject *)vv; - size_t result = 0; + int64_t result = 0; Py_ssize_t ndigits; int msd_bits; @@ -794,20 +905,12 @@ _PyLong_NumBits(PyObject *vv) assert(ndigits == 0 || v->long_value.ob_digit[ndigits - 1] != 0); if (ndigits > 0) { digit msd = v->long_value.ob_digit[ndigits - 1]; - if ((size_t)(ndigits - 1) > SIZE_MAX / (size_t)PyLong_SHIFT) - goto Overflow; - result = (size_t)(ndigits - 1) * (size_t)PyLong_SHIFT; + assert(ndigits <= INT64_MAX / PyLong_SHIFT); + result = (int64_t)(ndigits - 1) * PyLong_SHIFT; msd_bits = bit_length_digit(msd); - if (SIZE_MAX - msd_bits < result) - goto Overflow; result += msd_bits; } return result; - - Overflow: - PyErr_SetString(PyExc_OverflowError, "int has too many bits " - "to express in a platform size_t"); - return (size_t)-1; } PyObject * @@ -872,7 +975,7 @@ _PyLong_FromByteArray(const unsigned char* bytes, size_t n, return NULL; } ndigits = (numsignificantbytes * 8 + PyLong_SHIFT - 1) / PyLong_SHIFT; - v = _PyLong_New(ndigits); + v = long_alloc(ndigits); if (v == NULL) return NULL; @@ -1083,18 +1186,17 @@ _fits_in_n_bits(Py_ssize_t v, Py_ssize_t n) static inline int _resolve_endianness(int *endianness) { - if (*endianness < 0) { + if (*endianness == -1 || (*endianness & 2)) { *endianness = PY_LITTLE_ENDIAN; + } else { + *endianness &= 1; } - if (*endianness != 0 && *endianness != 1) { - PyErr_SetString(PyExc_SystemError, "invalid 'endianness' value"); - return -1; - } + assert(*endianness == 0 || *endianness == 1); return 0; } Py_ssize_t -PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int endianness) +PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int flags) { PyLongObject *v; union { @@ -1109,7 +1211,7 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int endianness) return -1; } - int little_endian = endianness; + int little_endian = flags; if (_resolve_endianness(&little_endian) < 0) { return -1; } @@ -1117,13 +1219,26 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int endianness) if (PyLong_Check(vv)) { v = (PyLongObject *)vv; } - else { + else if (flags != -1 && (flags & Py_ASNATIVEBYTES_ALLOW_INDEX)) { v = (PyLongObject *)_PyNumber_Index(vv); if (v == NULL) { return -1; } do_decref = 1; } + else { + PyErr_Format(PyExc_TypeError, "expect int, got %T", vv); + return -1; + } + + if ((flags != -1 && (flags & Py_ASNATIVEBYTES_REJECT_NEGATIVE)) + && _PyLong_IsNegative(v)) { + PyErr_SetString(PyExc_ValueError, "Cannot convert negative int"); + if (do_decref) { + Py_DECREF(v); + } + return -1; + } if (_PyLong_IsCompact(v)) { res = 0; @@ -1159,6 +1274,15 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int endianness) /* If we fit, return the requested number of bytes */ if (_fits_in_n_bits(cv.v, n * 8)) { res = n; + } else if (cv.v > 0 && _fits_in_n_bits(cv.v, n * 8 + 1)) { + /* Positive values with the MSB set do not require an + * additional bit when the caller's intent is to treat them + * as unsigned. */ + if (flags == -1 || (flags & Py_ASNATIVEBYTES_UNSIGNED_BUFFER)) { + res = n; + } else { + res = n + 1; + } } } else { @@ -1199,17 +1323,52 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int endianness) _PyLong_AsByteArray(v, buffer, (size_t)n, little_endian, 1, 0); } - // More efficient calculation for number of bytes required? - size_t nb = _PyLong_NumBits((PyObject *)v); - /* Normally this would be((nb - 1) / 8) + 1 to avoid rounding up + /* Calculates the number of bits required for the *absolute* value + * of v. This does not take sign into account, only magnitude. */ + int64_t nb = _PyLong_NumBits((PyObject *)v); + assert(nb >= 0); + /* Normally this would be ((nb - 1) / 8) + 1 to avoid rounding up * multiples of 8 to the next byte, but we add an implied bit for * the sign and it cancels out. */ - size_t n_needed = (nb / 8) + 1; - res = (Py_ssize_t)n_needed; - if ((size_t)res != n_needed) { - PyErr_SetString(PyExc_OverflowError, - "value too large to convert"); - res = -1; + res = (Py_ssize_t)(nb / 8) + 1; + + /* Two edge cases exist that are best handled after extracting the + * bits. These may result in us reporting overflow when the value + * actually fits. + */ + if (n > 0 && res == n + 1 && nb % 8 == 0) { + if (_PyLong_IsNegative(v)) { + /* Values of 0x80...00 from negative values that use every + * available bit in the buffer do not require an additional + * bit to store the sign. */ + int is_edge_case = 1; + unsigned char *b = (unsigned char *)buffer; + for (Py_ssize_t i = 0; i < n && is_edge_case; ++i, ++b) { + if (i == 0) { + is_edge_case = (*b == (little_endian ? 0 : 0x80)); + } else if (i < n - 1) { + is_edge_case = (*b == 0); + } else { + is_edge_case = (*b == (little_endian ? 0x80 : 0)); + } + } + if (is_edge_case) { + res = n; + } + } + else { + /* Positive values with the MSB set do not require an + * additional bit when the caller's intent is to treat them + * as unsigned. */ + unsigned char *b = (unsigned char *)buffer; + if (b[little_endian ? n - 1 : 0] & 0x80) { + if (flags == -1 || (flags & Py_ASNATIVEBYTES_UNSIGNED_BUFFER)) { + res = n; + } else { + res = n + 1; + } + } + } } } @@ -1222,38 +1381,41 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int endianness) PyObject * -PyLong_FromNativeBytes(const void* buffer, size_t n, int endianness) +PyLong_FromNativeBytes(const void* buffer, size_t n, int flags) { if (!buffer) { PyErr_BadInternalCall(); return NULL; } - int little_endian = endianness; + int little_endian = flags; if (_resolve_endianness(&little_endian) < 0) { return NULL; } - return _PyLong_FromByteArray((const unsigned char *)buffer, n, - little_endian, 1); + return _PyLong_FromByteArray( + (const unsigned char *)buffer, + n, + little_endian, + (flags == -1 || !(flags & Py_ASNATIVEBYTES_UNSIGNED_BUFFER)) ? 1 : 0 + ); } PyObject * -PyLong_FromUnsignedNativeBytes(const void* buffer, size_t n, int endianness) +PyLong_FromUnsignedNativeBytes(const void* buffer, size_t n, int flags) { if (!buffer) { PyErr_BadInternalCall(); return NULL; } - int little_endian = endianness; + int little_endian = flags; if (_resolve_endianness(&little_endian) < 0) { return NULL; } - return _PyLong_FromByteArray((const unsigned char *)buffer, n, - little_endian, 0); + return _PyLong_FromByteArray((const unsigned char *)buffer, n, little_endian, 0); } @@ -1343,7 +1505,7 @@ PyLong_FromLongLong(long long ival) } /* Construct output value. */ - v = _PyLong_New(ndigits); + v = long_alloc(ndigits); if (v != NULL) { digit *p = v->long_value.ob_digit; _PyLong_SetSignAndDigitCount(v, ival < 0 ? -1 : 1, ndigits); @@ -1386,7 +1548,7 @@ PyLong_FromSsize_t(Py_ssize_t ival) ++ndigits; t >>= PyLong_SHIFT; } - v = _PyLong_New(ndigits); + v = long_alloc(ndigits); if (v != NULL) { digit *p = v->long_value.ob_digit; _PyLong_SetSignAndDigitCount(v, negative ? -1 : 1, ndigits); @@ -1466,7 +1628,18 @@ PyLong_AsUnsignedLongLong(PyObject *vv) v = (PyLongObject*)vv; if (_PyLong_IsNonNegativeCompact(v)) { res = 0; - bytes = _PyLong_CompactValue(v); +#if SIZEOF_LONG_LONG < SIZEOF_SIZE_T + size_t tmp = (size_t)_PyLong_CompactValue(v); + bytes = (unsigned long long)tmp; + if (bytes != tmp) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert " + "to C unsigned long long"); + res = -1; + } +#else + bytes = (unsigned long long)(size_t)_PyLong_CompactValue(v); +#endif } else { res = _PyLong_AsByteArray((PyLongObject *)vv, (unsigned char *)&bytes, @@ -1497,7 +1670,11 @@ _PyLong_AsUnsignedLongLongMask(PyObject *vv) } v = (PyLongObject *)vv; if (_PyLong_IsCompact(v)) { - return (unsigned long long)(signed long long)_PyLong_CompactValue(v); +#if SIZEOF_LONG_LONG < SIZEOF_SIZE_T + return (unsigned long long)(size_t)_PyLong_CompactValue(v); +#else + return (unsigned long long)(long long)_PyLong_CompactValue(v); +#endif } i = _PyLong_DigitCount(v); sign = _PyLong_NonCompactSign(v); @@ -1569,7 +1746,22 @@ PyLong_AsLongLongAndOverflow(PyObject *vv, int *overflow) do_decref = 1; } if (_PyLong_IsCompact(v)) { +#if SIZEOF_LONG_LONG < SIZEOF_SIZE_T + Py_ssize_t tmp = _PyLong_CompactValue(v); + if (tmp < LLONG_MIN) { + *overflow = -1; + res = -1; + } + else if (tmp > LLONG_MAX) { + *overflow = 1; + res = -1; + } + else { + res = (long long)tmp; + } +#else res = _PyLong_CompactValue(v); +#endif } else { i = _PyLong_DigitCount(v); @@ -1846,7 +2038,7 @@ divrem1(PyLongObject *a, digit n, digit *prem) PyLongObject *z; assert(n > 0 && n <= PyLong_MASK); - z = _PyLong_New(size); + z = long_alloc(size); if (z == NULL) return NULL; *prem = inplace_divrem1(z->long_value.ob_digit, a->long_value.ob_digit, size, n); @@ -1965,7 +2157,9 @@ long_to_decimal_string_internal(PyObject *aa, digit *pout, *pin, rem, tenpow; int negative; int d; - int kind; + + // writer or bytes_writer can be used, but not both at the same time. + assert(writer == NULL || bytes_writer == NULL); a = (PyLongObject *)aa; if (a == NULL || !PyLong_Check(a)) { @@ -2021,7 +2215,7 @@ long_to_decimal_string_internal(PyObject *aa, (10 * PyLong_SHIFT - 33 * _PyLong_DECIMAL_SHIFT); assert(size_a < PY_SSIZE_T_MAX/2); size = 1 + size_a + size_a / d; - scratch = _PyLong_New(size); + scratch = long_alloc(size); if (scratch == NULL) return -1; @@ -2078,7 +2272,6 @@ long_to_decimal_string_internal(PyObject *aa, Py_DECREF(scratch); return -1; } - kind = writer->kind; } else if (bytes_writer) { *bytes_str = _PyBytesWriter_Prepare(bytes_writer, *bytes_str, strlen); @@ -2093,7 +2286,6 @@ long_to_decimal_string_internal(PyObject *aa, Py_DECREF(scratch); return -1; } - kind = PyUnicode_KIND(str); } #define WRITE_DIGITS(p) \ @@ -2141,19 +2333,23 @@ long_to_decimal_string_internal(PyObject *aa, WRITE_DIGITS(p); assert(p == *bytes_str); } - else if (kind == PyUnicode_1BYTE_KIND) { - Py_UCS1 *p; - WRITE_UNICODE_DIGITS(Py_UCS1); - } - else if (kind == PyUnicode_2BYTE_KIND) { - Py_UCS2 *p; - WRITE_UNICODE_DIGITS(Py_UCS2); - } else { - Py_UCS4 *p; - assert (kind == PyUnicode_4BYTE_KIND); - WRITE_UNICODE_DIGITS(Py_UCS4); + int kind = writer ? writer->kind : PyUnicode_KIND(str); + if (kind == PyUnicode_1BYTE_KIND) { + Py_UCS1 *p; + WRITE_UNICODE_DIGITS(Py_UCS1); + } + else if (kind == PyUnicode_2BYTE_KIND) { + Py_UCS2 *p; + WRITE_UNICODE_DIGITS(Py_UCS2); + } + else { + assert (kind == PyUnicode_4BYTE_KIND); + Py_UCS4 *p; + WRITE_UNICODE_DIGITS(Py_UCS4); + } } + #undef WRITE_DIGITS #undef WRITE_UNICODE_DIGITS @@ -2194,11 +2390,12 @@ long_format_binary(PyObject *aa, int base, int alternate, PyObject *v = NULL; Py_ssize_t sz; Py_ssize_t size_a; - int kind; int negative; int bits; assert(base == 2 || base == 8 || base == 16); + // writer or bytes_writer can be used, but not both at the same time. + assert(writer == NULL || bytes_writer == NULL); if (a == NULL || !PyLong_Check(a)) { PyErr_BadInternalCall(); return -1; @@ -2246,7 +2443,6 @@ long_format_binary(PyObject *aa, int base, int alternate, if (writer) { if (_PyUnicodeWriter_Prepare(writer, sz, 'x') == -1) return -1; - kind = writer->kind; } else if (bytes_writer) { *bytes_str = _PyBytesWriter_Prepare(bytes_writer, *bytes_str, sz); @@ -2257,7 +2453,6 @@ long_format_binary(PyObject *aa, int base, int alternate, v = PyUnicode_New(sz, 'x'); if (v == NULL) return -1; - kind = PyUnicode_KIND(v); } #define WRITE_DIGITS(p) \ @@ -2318,19 +2513,23 @@ long_format_binary(PyObject *aa, int base, int alternate, WRITE_DIGITS(p); assert(p == *bytes_str); } - else if (kind == PyUnicode_1BYTE_KIND) { - Py_UCS1 *p; - WRITE_UNICODE_DIGITS(Py_UCS1); - } - else if (kind == PyUnicode_2BYTE_KIND) { - Py_UCS2 *p; - WRITE_UNICODE_DIGITS(Py_UCS2); - } else { - Py_UCS4 *p; - assert (kind == PyUnicode_4BYTE_KIND); - WRITE_UNICODE_DIGITS(Py_UCS4); + int kind = writer ? writer->kind : PyUnicode_KIND(v); + if (kind == PyUnicode_1BYTE_KIND) { + Py_UCS1 *p; + WRITE_UNICODE_DIGITS(Py_UCS1); + } + else if (kind == PyUnicode_2BYTE_KIND) { + Py_UCS2 *p; + WRITE_UNICODE_DIGITS(Py_UCS2); + } + else { + assert (kind == PyUnicode_4BYTE_KIND); + Py_UCS4 *p; + WRITE_UNICODE_DIGITS(Py_UCS4); + } } + #undef WRITE_DIGITS #undef WRITE_UNICODE_DIGITS @@ -2459,7 +2658,7 @@ long_from_binary_base(const char *start, const char *end, Py_ssize_t digits, int return 0; } n = (digits * bits_per_char + PyLong_SHIFT - 1) / PyLong_SHIFT; - z = _PyLong_New(n); + z = long_alloc(n); if (z == NULL) { *res = NULL; return 0; @@ -2499,8 +2698,6 @@ long_from_binary_base(const char *start, const char *end, Py_ssize_t digits, int return 0; } -static PyObject *long_neg(PyLongObject *v); - #ifdef WITH_PYLONG_MODULE /* asymptotically faster str-to-long conversion for base 10, using _pylong.py */ static int @@ -2665,7 +2862,7 @@ long_from_non_binary_base(const char *start, const char *end, Py_ssize_t digits, */ double fsize_z = (double)digits * log_base_BASE[base] + 1.0; if (fsize_z > (double)MAX_LONG_DIGITS) { - /* The same exception as in _PyLong_New(). */ + /* The same exception as in long_alloc(). */ PyErr_SetString(PyExc_OverflowError, "too many digits in integer"); *res = NULL; @@ -2675,7 +2872,7 @@ long_from_non_binary_base(const char *start, const char *end, Py_ssize_t digits, /* Uncomment next line to test exceedingly rare copy code */ /* size_z = 1; */ assert(size_z > 0); - z = _PyLong_New(size_z); + z = long_alloc(size_z); if (z == NULL) { *res = NULL; return 0; @@ -2738,7 +2935,7 @@ long_from_non_binary_base(const char *start, const char *end, Py_ssize_t digits, PyLongObject *tmp; /* Extremely rare. Get more space. */ assert(_PyLong_DigitCount(z) == size_z); - tmp = _PyLong_New(size_z + 1); + tmp = long_alloc(size_z + 1); if (tmp == NULL) { Py_DECREF(z); *res = NULL; @@ -3029,11 +3226,6 @@ PyLong_FromUnicodeObject(PyObject *u, int base) return NULL; } -/* forward */ -static PyLongObject *x_divrem - (PyLongObject *, PyLongObject *, PyLongObject **); -static PyObject *long_long(PyObject *v); - /* Int division with remainder, top-level routine */ static int @@ -3044,8 +3236,7 @@ long_divrem(PyLongObject *a, PyLongObject *b, PyLongObject *z; if (size_b == 0) { - PyErr_SetString(PyExc_ZeroDivisionError, - "integer division or modulo by zero"); + PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); return -1; } if (size_a < size_b || @@ -3108,7 +3299,7 @@ long_rem(PyLongObject *a, PyLongObject *b, PyLongObject **prem) if (size_b == 0) { PyErr_SetString(PyExc_ZeroDivisionError, - "integer modulo by zero"); + "division by zero"); return -1; } if (size_a < size_b || @@ -3165,12 +3356,12 @@ x_divrem(PyLongObject *v1, PyLongObject *w1, PyLongObject **prem) size_v = _PyLong_DigitCount(v1); size_w = _PyLong_DigitCount(w1); assert(size_v >= size_w && size_w >= 2); /* Assert checks by div() */ - v = _PyLong_New(size_v+1); + v = long_alloc(size_v+1); if (v == NULL) { *prem = NULL; return NULL; } - w = _PyLong_New(size_w); + w = long_alloc(size_w); if (w == NULL) { Py_DECREF(v); *prem = NULL; @@ -3192,7 +3383,7 @@ x_divrem(PyLongObject *v1, PyLongObject *w1, PyLongObject **prem) at most (and usually exactly) k = size_v - size_w digits. */ k = size_v - size_w; assert(k >= 0); - a = _PyLong_New(k); + a = long_alloc(k); if (a == NULL) { Py_DECREF(w); Py_DECREF(v); @@ -3290,9 +3481,11 @@ x_divrem(PyLongObject *v1, PyLongObject *w1, PyLongObject **prem) #endif double -_PyLong_Frexp(PyLongObject *a, Py_ssize_t *e) +_PyLong_Frexp(PyLongObject *a, int64_t *e) { - Py_ssize_t a_size, a_bits, shift_digits, shift_bits, x_size; + Py_ssize_t a_size, shift_digits, x_size; + int shift_bits; + int64_t a_bits; /* See below for why x_digits is always large enough. */ digit rem; digit x_digits[2 + (DBL_MANT_DIG + 1) / PyLong_SHIFT] = {0,}; @@ -3308,14 +3501,7 @@ _PyLong_Frexp(PyLongObject *a, Py_ssize_t *e) *e = 0; return 0.0; } - a_bits = bit_length_digit(a->long_value.ob_digit[a_size-1]); - /* The following is an overflow-free version of the check - "if ((a_size - 1) * PyLong_SHIFT + a_bits > PY_SSIZE_T_MAX) ..." */ - if (a_size >= (PY_SSIZE_T_MAX - 1) / PyLong_SHIFT + 1 && - (a_size > (PY_SSIZE_T_MAX - 1) / PyLong_SHIFT + 1 || - a_bits > (PY_SSIZE_T_MAX - 1) % PyLong_SHIFT + 1)) - goto overflow; - a_bits = (a_size - 1) * PyLong_SHIFT + a_bits; + a_bits = _PyLong_NumBits((PyObject *)a); /* Shift the first DBL_MANT_DIG + 2 bits of a into x_digits[0:x_size] (shifting left if a_bits <= DBL_MANT_DIG + 2). @@ -3343,19 +3529,19 @@ _PyLong_Frexp(PyLongObject *a, Py_ssize_t *e) in both cases. */ if (a_bits <= DBL_MANT_DIG + 2) { - shift_digits = (DBL_MANT_DIG + 2 - a_bits) / PyLong_SHIFT; - shift_bits = (DBL_MANT_DIG + 2 - a_bits) % PyLong_SHIFT; + shift_digits = (DBL_MANT_DIG + 2 - (Py_ssize_t)a_bits) / PyLong_SHIFT; + shift_bits = (DBL_MANT_DIG + 2 - (int)a_bits) % PyLong_SHIFT; x_size = shift_digits; rem = v_lshift(x_digits + x_size, a->long_value.ob_digit, a_size, - (int)shift_bits); + shift_bits); x_size += a_size; x_digits[x_size++] = rem; } else { - shift_digits = (a_bits - DBL_MANT_DIG - 2) / PyLong_SHIFT; - shift_bits = (a_bits - DBL_MANT_DIG - 2) % PyLong_SHIFT; + shift_digits = (Py_ssize_t)((a_bits - DBL_MANT_DIG - 2) / PyLong_SHIFT); + shift_bits = (int)((a_bits - DBL_MANT_DIG - 2) % PyLong_SHIFT); rem = v_rshift(x_digits, a->long_value.ob_digit + shift_digits, - a_size - shift_digits, (int)shift_bits); + a_size - shift_digits, shift_bits); x_size = a_size - shift_digits; /* For correct rounding below, we need the least significant bit of x to be 'sticky' for this shift: if any of the bits @@ -3381,21 +3567,13 @@ _PyLong_Frexp(PyLongObject *a, Py_ssize_t *e) /* Rescale; make correction if result is 1.0. */ dx /= 4.0 * EXP2_DBL_MANT_DIG; if (dx == 1.0) { - if (a_bits == PY_SSIZE_T_MAX) - goto overflow; + assert(a_bits < INT64_MAX); dx = 0.5; a_bits += 1; } *e = a_bits; return _PyLong_IsNegative(a) ? -dx : dx; - - overflow: - /* exponent > PY_SSIZE_T_MAX */ - PyErr_SetString(PyExc_OverflowError, - "huge integer: number of bits overflows a Py_ssize_t"); - *e = 0; - return -1.0; } /* Get a C double from an int object. Rounds to the nearest double, @@ -3404,7 +3582,7 @@ _PyLong_Frexp(PyLongObject *a, Py_ssize_t *e) double PyLong_AsDouble(PyObject *v) { - Py_ssize_t exponent; + int64_t exponent; double x; if (v == NULL) { @@ -3423,7 +3601,9 @@ PyLong_AsDouble(PyObject *v) return (double)medium_value((PyLongObject *)v); } x = _PyLong_Frexp((PyLongObject *)v, &exponent); - if ((x == -1.0 && PyErr_Occurred()) || exponent > DBL_MAX_EXP) { + assert(exponent >= 0); + assert(!PyErr_Occurred()); + if (exponent > DBL_MAX_EXP) { PyErr_SetString(PyExc_OverflowError, "int too large to convert to float"); return -1.0; @@ -3470,36 +3650,62 @@ long_richcompare(PyObject *self, PyObject *other, int op) Py_RETURN_RICHCOMPARE(result, 0, op); } +static inline int +/// Return 1 if the object is one of the immortal small ints +_long_is_small_int(PyObject *op) +{ + PyLongObject *long_object = (PyLongObject *)op; + int is_small_int = (long_object->long_value.lv_tag & IMMORTALITY_BIT_MASK) != 0; + assert((!is_small_int) || PyLong_CheckExact(op)); + return is_small_int; +} + +void +_PyLong_ExactDealloc(PyObject *self) +{ + assert(PyLong_CheckExact(self)); + if (_long_is_small_int(self)) { + // See PEP 683, section Accidental De-Immortalizing for details + _Py_SetImmortal(self); + return; + } + if (_PyLong_IsCompact((PyLongObject *)self)) { + _Py_FREELIST_FREE(ints, self, PyObject_Free); + return; + } + PyObject_Free(self); +} + static void long_dealloc(PyObject *self) { - /* This should never get called, but we also don't want to SEGV if - * we accidentally decref small Ints out of existence. Instead, - * since small Ints are immortal, re-set the reference count. - */ - PyLongObject *pylong = (PyLongObject*)self; - if (pylong && _PyLong_IsCompact(pylong)) { - stwodigits ival = medium_value(pylong); - if (IS_SMALL_INT(ival)) { - PyLongObject *small_pylong = (PyLongObject *)get_small_int((sdigit)ival); - if (pylong == small_pylong) { - _Py_SetImmortal(self); - return; - } - } + if (_long_is_small_int(self)) { + /* This should never get called, but we also don't want to SEGV if + * we accidentally decref small Ints out of existence. Instead, + * since small Ints are immortal, re-set the reference count. + * + * See PEP 683, section Accidental De-Immortalizing for details + */ + _Py_SetImmortal(self); + return; + } + if (PyLong_CheckExact(self) && _PyLong_IsCompact((PyLongObject *)self)) { + _Py_FREELIST_FREE(ints, self, PyObject_Free); + return; } Py_TYPE(self)->tp_free(self); } static Py_hash_t -long_hash(PyLongObject *v) +long_hash(PyObject *obj) { + PyLongObject *v = (PyLongObject *)obj; Py_uhash_t x; Py_ssize_t i; int sign; if (_PyLong_IsCompact(v)) { - x = _PyLong_CompactValue(v); + x = (Py_uhash_t)_PyLong_CompactValue(v); if (x == (Py_uhash_t)-1) { x = (Py_uhash_t)-2; } @@ -3564,7 +3770,7 @@ x_add(PyLongObject *a, PyLongObject *b) size_a = size_b; size_b = size_temp; } } - z = _PyLong_New(size_a+1); + z = long_alloc(size_a+1); if (z == NULL) return NULL; for (i = 0; i < size_b; ++i) { @@ -3613,7 +3819,7 @@ x_sub(PyLongObject *a, PyLongObject *b) } size_a = size_b = i+1; } - z = _PyLong_New(size_a); + z = long_alloc(size_a); if (z == NULL) return NULL; for (i = 0; i < size_b; ++i) { @@ -3637,11 +3843,12 @@ x_sub(PyLongObject *a, PyLongObject *b) return maybe_small_long(long_normalize(z)); } -PyObject * -_PyLong_Add(PyLongObject *a, PyLongObject *b) +static PyLongObject * +long_add(PyLongObject *a, PyLongObject *b) { if (_PyLong_BothAreCompact(a, b)) { - return _PyLong_FromSTwoDigits(medium_value(a) + medium_value(b)); + stwodigits z = medium_value(a) + medium_value(b); + return _PyLong_FromSTwoDigits(z); } PyLongObject *z; @@ -3666,24 +3873,31 @@ _PyLong_Add(PyLongObject *a, PyLongObject *b) else z = x_add(a, b); } - return (PyObject *)z; + return z; +} + +PyObject * +_PyLong_Add(PyLongObject *a, PyLongObject *b) +{ + return (PyObject*)long_add(a, b); } static PyObject * -long_add(PyLongObject *a, PyLongObject *b) +long_add_method(PyObject *a, PyObject *b) { CHECK_BINOP(a, b); - return _PyLong_Add(a, b); + return (PyObject*)long_add((PyLongObject*)a, (PyLongObject*)b); } -PyObject * -_PyLong_Subtract(PyLongObject *a, PyLongObject *b) -{ - PyLongObject *z; +static PyLongObject * +long_sub(PyLongObject *a, PyLongObject *b) +{ if (_PyLong_BothAreCompact(a, b)) { return _PyLong_FromSTwoDigits(medium_value(a) - medium_value(b)); } + + PyLongObject *z; if (_PyLong_IsNegative(a)) { if (_PyLong_IsNegative(b)) { z = x_sub(b, a); @@ -3702,16 +3916,23 @@ _PyLong_Subtract(PyLongObject *a, PyLongObject *b) else z = x_sub(a, b); } - return (PyObject *)z; + return z; +} + +PyObject * +_PyLong_Subtract(PyLongObject *a, PyLongObject *b) +{ + return (PyObject*)long_sub(a, b); } static PyObject * -long_sub(PyLongObject *a, PyLongObject *b) +long_sub_method(PyObject *a, PyObject *b) { CHECK_BINOP(a, b); - return _PyLong_Subtract(a, b); + return (PyObject*)long_sub((PyLongObject*)a, (PyLongObject*)b); } + /* Grade school multiplication, ignoring the signs. * Returns the absolute value of the product, or NULL if error. */ @@ -3723,14 +3944,14 @@ x_mul(PyLongObject *a, PyLongObject *b) Py_ssize_t size_b = _PyLong_DigitCount(b); Py_ssize_t i; - z = _PyLong_New(size_a + size_b); + z = long_alloc(size_a + size_b); if (z == NULL) return NULL; memset(z->long_value.ob_digit, 0, _PyLong_DigitCount(z) * sizeof(digit)); if (a == b) { /* Efficient squaring per HAC, Algorithm 14.16: - * http://www.cacr.math.uwaterloo.ca/hac/about/chap14.pdf + * https://cacr.uwaterloo.ca/hac/about/chap14.pdf * Gives slightly less than a 2x speedup when a == b, * via exploiting that each entry in the multiplication * pyramid appears twice (except for the size_a squares). @@ -3833,9 +4054,9 @@ kmul_split(PyLongObject *n, size_lo = Py_MIN(size_n, size); size_hi = size_n - size_lo; - if ((hi = _PyLong_New(size_hi)) == NULL) + if ((hi = long_alloc(size_hi)) == NULL) return -1; - if ((lo = _PyLong_New(size_lo)) == NULL) { + if ((lo = long_alloc(size_lo)) == NULL) { Py_DECREF(hi); return -1; } @@ -3935,7 +4156,7 @@ k_mul(PyLongObject *a, PyLongObject *b) */ /* 1. Allocate result space. */ - ret = _PyLong_New(asize + bsize); + ret = long_alloc(asize + bsize); if (ret == NULL) goto fail; #ifdef Py_DEBUG /* Fill with trash, to catch reference to uninitialized digits. */ @@ -4085,13 +4306,13 @@ k_lopsided_mul(PyLongObject *a, PyLongObject *b) assert(2 * asize <= bsize); /* Allocate result space, and zero it out. */ - ret = _PyLong_New(asize + bsize); + ret = long_alloc(asize + bsize); if (ret == NULL) return NULL; memset(ret->long_value.ob_digit, 0, _PyLong_DigitCount(ret) * sizeof(digit)); /* Successive slices of b are copied into bslice. */ - bslice = _PyLong_New(asize); + bslice = long_alloc(asize); if (bslice == NULL) goto fail; @@ -4127,32 +4348,35 @@ k_lopsided_mul(PyLongObject *a, PyLongObject *b) return NULL; } -PyObject * -_PyLong_Multiply(PyLongObject *a, PyLongObject *b) -{ - PyLongObject *z; +static PyLongObject* +long_mul(PyLongObject *a, PyLongObject *b) +{ /* fast path for single-digit multiplication */ if (_PyLong_BothAreCompact(a, b)) { stwodigits v = medium_value(a) * medium_value(b); return _PyLong_FromSTwoDigits(v); } - z = k_mul(a, b); + PyLongObject *z = k_mul(a, b); /* Negate if exactly one of the inputs is negative. */ if (!_PyLong_SameSign(a, b) && z) { _PyLong_Negate(&z); - if (z == NULL) - return NULL; } - return (PyObject *)z; + return z; +} + +PyObject * +_PyLong_Multiply(PyLongObject *a, PyLongObject *b) +{ + return (PyObject*)long_mul(a, b); } static PyObject * -long_mul(PyLongObject *a, PyLongObject *b) +long_mul_method(PyObject *a, PyObject *b) { CHECK_BINOP(a, b); - return _PyLong_Multiply(a, b); + return (PyObject*)long_mul((PyLongObject*)a, (PyLongObject*)b); } /* Fast modulo division for single-digit longs. */ @@ -4307,13 +4531,13 @@ l_divmod(PyLongObject *v, PyLongObject *w, if ((_PyLong_IsNegative(mod) && _PyLong_IsPositive(w)) || (_PyLong_IsPositive(mod) && _PyLong_IsNegative(w))) { PyLongObject *temp; - temp = (PyLongObject *) long_add(mod, w); + temp = long_add(mod, w); Py_SETREF(mod, temp); if (mod == NULL) { Py_DECREF(div); return -1; } - temp = (PyLongObject *) long_sub(div, (PyLongObject *)_PyLong_GetOne()); + temp = long_sub(div, (PyLongObject *)_PyLong_GetOne()); if (temp == NULL) { Py_DECREF(mod); Py_DECREF(div); @@ -4354,7 +4578,7 @@ l_mod(PyLongObject *v, PyLongObject *w, PyLongObject **pmod) if ((_PyLong_IsNegative(mod) && _PyLong_IsPositive(w)) || (_PyLong_IsPositive(mod) && _PyLong_IsNegative(w))) { PyLongObject *temp; - temp = (PyLongObject *) long_add(mod, w); + temp = long_add(mod, w); Py_SETREF(mod, temp); if (mod == NULL) return -1; @@ -4554,7 +4778,7 @@ long_true_divide(PyObject *v, PyObject *w) "intermediate overflow during division"); goto error; } - x = _PyLong_New(a_size + shift_digits + 1); + x = long_alloc(a_size + shift_digits + 1); if (x == NULL) goto error; for (i = 0; i < shift_digits; i++) @@ -4568,7 +4792,7 @@ long_true_divide(PyObject *v, PyObject *w) digit rem; /* x = a >> shift */ assert(a_size >= shift_digits); - x = _PyLong_New(a_size - shift_digits); + x = long_alloc(a_size - shift_digits); if (x == NULL) goto error; rem = v_rshift(x->long_value.ob_digit, a->long_value.ob_digit + shift_digits, @@ -4701,21 +4925,12 @@ long_divmod(PyObject *a, PyObject *b) static PyLongObject * long_invmod(PyLongObject *a, PyLongObject *n) { - PyLongObject *b, *c; - /* Should only ever be called for positive n */ assert(_PyLong_IsPositive(n)); - b = (PyLongObject *)PyLong_FromLong(1L); - if (b == NULL) { - return NULL; - } - c = (PyLongObject *)PyLong_FromLong(0L); - if (c == NULL) { - Py_DECREF(b); - return NULL; - } Py_INCREF(a); + PyLongObject *b = (PyLongObject *)Py_NewRef(_PyLong_GetOne()); + PyLongObject *c = (PyLongObject *)Py_NewRef(_PyLong_GetZero()); Py_INCREF(n); /* references now owned: a, b, c, n */ @@ -4732,7 +4947,7 @@ long_invmod(PyLongObject *a, PyLongObject *n) if (t == NULL) { goto Error; } - s = (PyLongObject *)long_sub(b, t); + s = long_sub(b, t); Py_DECREF(t); if (s == NULL) { goto Error; @@ -4938,7 +5153,7 @@ long_pow(PyObject *v, PyObject *w, PyObject *x) } else if (i <= HUGE_EXP_CUTOFF / PyLong_SHIFT ) { /* Left-to-right binary exponentiation (HAC Algorithm 14.79) */ - /* http://www.cacr.math.uwaterloo.ca/hac/about/chap14.pdf */ + /* https://cacr.uwaterloo.ca/hac/about/chap14.pdf */ /* Find the first significant exponent bit. Search right to left * because we're primarily trying to cut overhead for small powers. @@ -5027,7 +5242,7 @@ long_pow(PyObject *v, PyObject *w, PyObject *x) } if (negativeOutput && !_PyLong_IsZero(z)) { - temp = (PyLongObject *)long_sub(z, c); + temp = long_sub(z, c); if (temp == NULL) goto Error; Py_SETREF(z, temp); @@ -5050,13 +5265,15 @@ long_pow(PyObject *v, PyObject *w, PyObject *x) } static PyObject * -long_invert(PyLongObject *v) +long_invert(PyObject *self) { + PyLongObject *v = _PyLong_CAST(self); + /* Implement ~x as -(x+1) */ - PyLongObject *x; if (_PyLong_IsCompact(v)) - return _PyLong_FromSTwoDigits(~medium_value(v)); - x = (PyLongObject *) long_add(v, (PyLongObject *)_PyLong_GetOne()); + return (PyObject*)_PyLong_FromSTwoDigits(~medium_value(v)); + + PyLongObject *x = long_add(v, (PyLongObject *)_PyLong_GetOne()); if (x == NULL) return NULL; _PyLong_Negate(&x); @@ -5065,64 +5282,45 @@ long_invert(PyLongObject *v) return (PyObject *)x; } -static PyObject * +static PyLongObject * long_neg(PyLongObject *v) { - PyLongObject *z; - if (_PyLong_IsCompact(v)) + if (_PyLong_IsCompact(v)) { return _PyLong_FromSTwoDigits(-medium_value(v)); - z = (PyLongObject *)_PyLong_Copy(v); - if (z != NULL) + } + + PyLongObject *z = (PyLongObject *)_PyLong_Copy(v); + if (z != NULL) { _PyLong_FlipSign(z); - return (PyObject *)z; + } + return z; } static PyObject * +long_neg_method(PyObject *v) +{ + return (PyObject*)long_neg(_PyLong_CAST(v)); +} + +static PyLongObject* long_abs(PyLongObject *v) { if (_PyLong_IsNegative(v)) return long_neg(v); else - return long_long((PyObject *)v); + return (PyLongObject*)long_long((PyObject *)v); } -static int -long_bool(PyLongObject *v) +static PyObject * +long_abs_method(PyObject *v) { - return !_PyLong_IsZero(v); + return (PyObject*)long_abs(_PyLong_CAST(v)); } -/* wordshift, remshift = divmod(shiftby, PyLong_SHIFT) */ static int -divmod_shift(PyObject *shiftby, Py_ssize_t *wordshift, digit *remshift) -{ - assert(PyLong_Check(shiftby)); - assert(!_PyLong_IsNegative((PyLongObject *)shiftby)); - Py_ssize_t lshiftby = PyLong_AsSsize_t((PyObject *)shiftby); - if (lshiftby >= 0) { - *wordshift = lshiftby / PyLong_SHIFT; - *remshift = lshiftby % PyLong_SHIFT; - return 0; - } - /* PyLong_Check(shiftby) is true and shiftby is not negative, so it must - be that PyLong_AsSsize_t raised an OverflowError. */ - assert(PyErr_ExceptionMatches(PyExc_OverflowError)); - PyErr_Clear(); - PyLongObject *wordshift_obj = divrem1((PyLongObject *)shiftby, PyLong_SHIFT, remshift); - if (wordshift_obj == NULL) { - return -1; - } - *wordshift = PyLong_AsSsize_t((PyObject *)wordshift_obj); - Py_DECREF(wordshift_obj); - if (*wordshift >= 0 && *wordshift < PY_SSIZE_T_MAX / (Py_ssize_t)sizeof(digit)) { - return 0; - } - PyErr_Clear(); - /* Clip the value. With such large wordshift the right shift - returns 0 and the left shift raises an error in _PyLong_New(). */ - *wordshift = PY_SSIZE_T_MAX / sizeof(digit); - *remshift = 0; - return 0; +long_bool(PyObject *v) +{ + return !_PyLong_IsZero(_PyLong_CAST(v)); } /* Inner function for both long_rshift and _PyLong_Rshift, shifting an @@ -5148,7 +5346,7 @@ long_rshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) m = medium_value(a); shift = wordshift == 0 ? remshift : PyLong_SHIFT; x = m < 0 ? ~(~m >> shift) : m >> shift; - return _PyLong_FromSTwoDigits(x); + return (PyObject*)_PyLong_FromSTwoDigits(x); } a_negative = _PyLong_IsNegative(a); @@ -5174,7 +5372,7 @@ long_rshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) /* Shifting all the bits of 'a' out gives either -1 or 0. */ return PyLong_FromLong(-a_negative); } - z = _PyLong_New(newsize); + z = long_alloc(newsize); if (z == NULL) { return NULL; } @@ -5218,8 +5416,7 @@ long_rshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) static PyObject * long_rshift(PyObject *a, PyObject *b) { - Py_ssize_t wordshift; - digit remshift; + int64_t shiftby; CHECK_BINOP(a, b); @@ -5230,24 +5427,45 @@ long_rshift(PyObject *a, PyObject *b) if (_PyLong_IsZero((PyLongObject *)a)) { return PyLong_FromLong(0); } - if (divmod_shift(b, &wordshift, &remshift) < 0) - return NULL; - return long_rshift1((PyLongObject *)a, wordshift, remshift); + if (PyLong_AsInt64(b, &shiftby) < 0) { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) { + return NULL; + } + PyErr_Clear(); + if (_PyLong_IsNegative((PyLongObject *)a)) { + return PyLong_FromLong(-1); + } + else { + return PyLong_FromLong(0); + } + } + return _PyLong_Rshift(a, shiftby); } /* Return a >> shiftby. */ PyObject * -_PyLong_Rshift(PyObject *a, size_t shiftby) +_PyLong_Rshift(PyObject *a, int64_t shiftby) { Py_ssize_t wordshift; digit remshift; assert(PyLong_Check(a)); + assert(shiftby >= 0); if (_PyLong_IsZero((PyLongObject *)a)) { return PyLong_FromLong(0); } - wordshift = shiftby / PyLong_SHIFT; - remshift = shiftby % PyLong_SHIFT; +#if PY_SSIZE_T_MAX <= INT64_MAX / PyLong_SHIFT + if (shiftby > (int64_t)PY_SSIZE_T_MAX * PyLong_SHIFT) { + if (_PyLong_IsNegative((PyLongObject *)a)) { + return PyLong_FromLong(-1); + } + else { + return PyLong_FromLong(0); + } + } +#endif + wordshift = (Py_ssize_t)(shiftby / PyLong_SHIFT); + remshift = (digit)(shiftby % PyLong_SHIFT); return long_rshift1((PyLongObject *)a, wordshift, remshift); } @@ -5262,14 +5480,14 @@ long_lshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) stwodigits m = medium_value(a); // bypass undefined shift operator behavior stwodigits x = m < 0 ? -(-m << remshift) : m << remshift; - return _PyLong_FromSTwoDigits(x); + return (PyObject*)_PyLong_FromSTwoDigits(x); } oldsize = _PyLong_DigitCount(a); newsize = oldsize + wordshift; if (remshift) ++newsize; - z = _PyLong_New(newsize); + z = long_alloc(newsize); if (z == NULL) return NULL; if (_PyLong_IsNegative(a)) { @@ -5292,42 +5510,61 @@ long_lshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) return (PyObject *) maybe_small_long(z); } + static PyObject * -long_lshift(PyObject *a, PyObject *b) +long_lshift_method(PyObject *aa, PyObject *bb) { - Py_ssize_t wordshift; - digit remshift; + CHECK_BINOP(aa, bb); + PyLongObject *a = (PyLongObject*)aa; + PyLongObject *b = (PyLongObject*)bb; - CHECK_BINOP(a, b); - - if (_PyLong_IsNegative((PyLongObject *)b)) { + if (_PyLong_IsNegative(b)) { PyErr_SetString(PyExc_ValueError, "negative shift count"); return NULL; } - if (_PyLong_IsZero((PyLongObject *)a)) { + if (_PyLong_IsZero(a)) { return PyLong_FromLong(0); } - if (divmod_shift(b, &wordshift, &remshift) < 0) + + int64_t shiftby; + if (PyLong_AsInt64(bb, &shiftby) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_SetString(PyExc_OverflowError, + "too many digits in integer"); + } return NULL; - return long_lshift1((PyLongObject *)a, wordshift, remshift); + } + return long_lshift_int64(a, shiftby); } /* Return a << shiftby. */ -PyObject * -_PyLong_Lshift(PyObject *a, size_t shiftby) +static PyObject * +long_lshift_int64(PyLongObject *a, int64_t shiftby) { - Py_ssize_t wordshift; - digit remshift; + assert(shiftby >= 0); - assert(PyLong_Check(a)); - if (_PyLong_IsZero((PyLongObject *)a)) { + if (_PyLong_IsZero(a)) { return PyLong_FromLong(0); } - wordshift = shiftby / PyLong_SHIFT; - remshift = shiftby % PyLong_SHIFT; - return long_lshift1((PyLongObject *)a, wordshift, remshift); +#if PY_SSIZE_T_MAX <= INT64_MAX / PyLong_SHIFT + if (shiftby > (int64_t)PY_SSIZE_T_MAX * PyLong_SHIFT) { + PyErr_SetString(PyExc_OverflowError, + "too many digits in integer"); + return NULL; + } +#endif + Py_ssize_t wordshift = (Py_ssize_t)(shiftby / PyLong_SHIFT); + digit remshift = (digit)(shiftby % PyLong_SHIFT); + return long_lshift1(a, wordshift, remshift); +} + +PyObject * +_PyLong_Lshift(PyObject *a, int64_t shiftby) +{ + return long_lshift_int64(_PyLong_CAST(a), shiftby); } + /* Compute two's complement of digit vector a[0:m], writing result to z[0:m]. The digit vector a need not be normalized, but should not be entirely zero. a and z may point to the same digit vector. */ @@ -5365,7 +5602,7 @@ long_bitwise(PyLongObject *a, size_a = _PyLong_DigitCount(a); nega = _PyLong_IsNegative(a); if (nega) { - z = _PyLong_New(size_a); + z = long_alloc(size_a); if (z == NULL) return NULL; v_complement(z->long_value.ob_digit, a->long_value.ob_digit, size_a); @@ -5379,7 +5616,7 @@ long_bitwise(PyLongObject *a, size_b = _PyLong_DigitCount(b); negb = _PyLong_IsNegative(b); if (negb) { - z = _PyLong_New(size_b); + z = long_alloc(size_b); if (z == NULL) { Py_DECREF(a); return NULL; @@ -5423,7 +5660,7 @@ long_bitwise(PyLongObject *a, /* We allow an extra digit if z is negative, to make sure that the final two's complement of z doesn't overflow. */ - z = _PyLong_New(size_z + negz); + z = long_alloc(size_z + negz); if (z == NULL) { Py_DECREF(a); Py_DECREF(b); @@ -5475,7 +5712,7 @@ long_and(PyObject *a, PyObject *b) PyLongObject *x = (PyLongObject*)a; PyLongObject *y = (PyLongObject*)b; if (_PyLong_IsCompact(x) && _PyLong_IsCompact(y)) { - return _PyLong_FromSTwoDigits(medium_value(x) & medium_value(y)); + return (PyObject*)_PyLong_FromSTwoDigits(medium_value(x) & medium_value(y)); } return long_bitwise(x, '&', y); } @@ -5487,7 +5724,7 @@ long_xor(PyObject *a, PyObject *b) PyLongObject *x = (PyLongObject*)a; PyLongObject *y = (PyLongObject*)b; if (_PyLong_IsCompact(x) && _PyLong_IsCompact(y)) { - return _PyLong_FromSTwoDigits(medium_value(x) ^ medium_value(y)); + return (PyObject*)_PyLong_FromSTwoDigits(medium_value(x) ^ medium_value(y)); } return long_bitwise(x, '^', y); } @@ -5499,7 +5736,7 @@ long_or(PyObject *a, PyObject *b) PyLongObject *x = (PyLongObject*)a; PyLongObject *y = (PyLongObject*)b; if (_PyLong_IsCompact(x) && _PyLong_IsCompact(y)) { - return _PyLong_FromSTwoDigits(medium_value(x) | medium_value(y)); + return (PyObject*)_PyLong_FromSTwoDigits(medium_value(x) | medium_value(y)); } return long_bitwise(x, '|', y); } @@ -5533,10 +5770,10 @@ _PyLong_GCD(PyObject *aarg, PyObject *barg) } /* Initial reduction: make sure that 0 <= b <= a. */ - a = (PyLongObject *)long_abs(a); + a = long_abs(a); if (a == NULL) return NULL; - b = (PyLongObject *)long_abs(b); + b = long_abs(b); if (b == NULL) { Py_DECREF(a); return NULL; @@ -5621,7 +5858,7 @@ _PyLong_GCD(PyObject *aarg, PyObject *barg) } else { alloc_a = size_a; - c = _PyLong_New(size_a); + c = long_alloc(size_a); if (c == NULL) goto error; } @@ -5637,7 +5874,7 @@ _PyLong_GCD(PyObject *aarg, PyObject *barg) } else { alloc_b = size_a; - d = _PyLong_New(size_a); + d = long_alloc(size_a); if (d == NULL) goto error; } @@ -5817,7 +6054,7 @@ long_subtype_new(PyTypeObject *type, PyObject *x, PyObject *obase) return NULL; } assert(PyLong_Check(newobj)); - newobj->long_value.lv_tag = tmp->long_value.lv_tag; + newobj->long_value.lv_tag = tmp->long_value.lv_tag & ~IMMORTALITY_BIT_MASK; for (i = 0; i < n; i++) { newobj->long_value.ob_digit[i] = tmp->long_value.ob_digit[i]; } @@ -5918,12 +6155,11 @@ _PyLong_DivmodNear(PyObject *a, PyObject *b) /* compare twice the remainder with the divisor, to see if we need to adjust the quotient and remainder */ - PyObject *one = _PyLong_GetOne(); // borrowed reference - twice_rem = long_lshift((PyObject *)rem, one); + twice_rem = long_lshift_int64(rem, 1); if (twice_rem == NULL) goto error; if (quo_is_neg) { - temp = long_neg((PyLongObject*)twice_rem); + temp = (PyObject*)long_neg((PyLongObject*)twice_rem); Py_SETREF(twice_rem, temp); if (twice_rem == NULL) goto error; @@ -5934,18 +6170,19 @@ _PyLong_DivmodNear(PyObject *a, PyObject *b) quo_is_odd = (quo->long_value.ob_digit[0] & 1) != 0; if ((_PyLong_IsNegative((PyLongObject *)b) ? cmp < 0 : cmp > 0) || (cmp == 0 && quo_is_odd)) { /* fix up quotient */ + PyObject *one = _PyLong_GetOne(); // borrowed reference if (quo_is_neg) - temp = long_sub(quo, (PyLongObject *)one); + temp = (PyObject*)long_sub(quo, (PyLongObject *)one); else - temp = long_add(quo, (PyLongObject *)one); + temp = (PyObject*)long_add(quo, (PyLongObject *)one); Py_SETREF(quo, (PyLongObject *)temp); if (quo == NULL) goto error; /* and remainder */ if (quo_is_neg) - temp = long_add(rem, (PyLongObject *)b); + temp = (PyObject*)long_add(rem, (PyLongObject *)b); else - temp = long_sub(rem, (PyLongObject *)b); + temp = (PyObject*)long_sub(rem, (PyLongObject *)b); Py_SETREF(rem, (PyLongObject *)temp); if (rem == NULL) goto error; @@ -5969,7 +6206,7 @@ _PyLong_DivmodNear(PyObject *a, PyObject *b) /*[clinic input] int.__round__ - ndigits as o_ndigits: object = NULL + ndigits as o_ndigits: object = None / Rounding an Integral returns itself. @@ -5979,10 +6216,8 @@ Rounding with an ndigits argument also returns an integer. static PyObject * int___round___impl(PyObject *self, PyObject *o_ndigits) -/*[clinic end generated code: output=954fda6b18875998 input=1614cf23ec9e18c3]*/ +/*[clinic end generated code: output=954fda6b18875998 input=30c2aec788263144]*/ { - PyObject *temp, *result, *ndigits; - /* To round an integer m to the nearest 10**n (n positive), we make use of * the divmod_near operation, defined by: * @@ -5997,10 +6232,10 @@ int___round___impl(PyObject *self, PyObject *o_ndigits) * * m - divmod_near(m, 10**n)[1]. */ - if (o_ndigits == NULL) + if (o_ndigits == Py_None) return long_long(self); - ndigits = _PyNumber_Index(o_ndigits); + PyObject *ndigits = _PyNumber_Index(o_ndigits); if (ndigits == NULL) return NULL; @@ -6011,12 +6246,12 @@ int___round___impl(PyObject *self, PyObject *o_ndigits) } /* result = self - divmod_near(self, 10 ** -ndigits)[1] */ - temp = long_neg((PyLongObject*)ndigits); + PyObject *temp = (PyObject*)long_neg((PyLongObject*)ndigits); Py_SETREF(ndigits, temp); if (ndigits == NULL) return NULL; - result = PyLong_FromLong(10L); + PyObject *result = PyLong_FromLong(10); if (result == NULL) { Py_DECREF(ndigits); return NULL; @@ -6033,8 +6268,8 @@ int___round___impl(PyObject *self, PyObject *o_ndigits) if (result == NULL) return NULL; - temp = long_sub((PyLongObject *)self, - (PyLongObject *)PyTuple_GET_ITEM(result, 1)); + temp = (PyObject*)long_sub((PyLongObject*)self, + (PyLongObject*)PyTuple_GET_ITEM(result, 1)); Py_SETREF(result, temp); return result; @@ -6071,51 +6306,10 @@ static PyObject * int_bit_length_impl(PyObject *self) /*[clinic end generated code: output=fc1977c9353d6a59 input=e4eb7a587e849a32]*/ { - PyLongObject *result, *x, *y; - Py_ssize_t ndigits; - int msd_bits; - digit msd; - - assert(self != NULL); - assert(PyLong_Check(self)); - - ndigits = _PyLong_DigitCount((PyLongObject *)self); - if (ndigits == 0) - return PyLong_FromLong(0); - - msd = ((PyLongObject *)self)->long_value.ob_digit[ndigits-1]; - msd_bits = bit_length_digit(msd); - - if (ndigits <= PY_SSIZE_T_MAX/PyLong_SHIFT) - return PyLong_FromSsize_t((ndigits-1)*PyLong_SHIFT + msd_bits); - - /* expression above may overflow; use Python integers instead */ - result = (PyLongObject *)PyLong_FromSsize_t(ndigits - 1); - if (result == NULL) - return NULL; - x = (PyLongObject *)PyLong_FromLong(PyLong_SHIFT); - if (x == NULL) - goto error; - y = (PyLongObject *)long_mul(result, x); - Py_DECREF(x); - if (y == NULL) - goto error; - Py_SETREF(result, y); - - x = (PyLongObject *)PyLong_FromLong((long)msd_bits); - if (x == NULL) - goto error; - y = (PyLongObject *)long_add(result, x); - Py_DECREF(x); - if (y == NULL) - goto error; - Py_SETREF(result, y); - - return (PyObject *)result; - - error: - Py_DECREF(result); - return NULL; + int64_t nbits = _PyLong_NumBits(self); + assert(nbits >= 0); + assert(!PyErr_Occurred()); + return PyLong_FromInt64(nbits); } static int @@ -6149,40 +6343,13 @@ int_bit_count_impl(PyObject *self) PyLongObject *z = (PyLongObject *)self; Py_ssize_t ndigits = _PyLong_DigitCount(z); - Py_ssize_t bit_count = 0; + int64_t bit_count = 0; - /* Each digit has up to PyLong_SHIFT ones, so the accumulated bit count - from the first PY_SSIZE_T_MAX/PyLong_SHIFT digits can't overflow a - Py_ssize_t. */ - Py_ssize_t ndigits_fast = Py_MIN(ndigits, PY_SSIZE_T_MAX/PyLong_SHIFT); - for (Py_ssize_t i = 0; i < ndigits_fast; i++) { + for (Py_ssize_t i = 0; i < ndigits; i++) { bit_count += popcount_digit(z->long_value.ob_digit[i]); } - PyObject *result = PyLong_FromSsize_t(bit_count); - if (result == NULL) { - return NULL; - } - - /* Use Python integers if bit_count would overflow. */ - for (Py_ssize_t i = ndigits_fast; i < ndigits; i++) { - PyObject *x = PyLong_FromLong(popcount_digit(z->long_value.ob_digit[i])); - if (x == NULL) { - goto error; - } - PyObject *y = long_add((PyLongObject *)result, (PyLongObject *)x); - Py_DECREF(x); - if (y == NULL) { - goto error; - } - Py_SETREF(result, y); - } - - return result; - - error: - Py_DECREF(result); - return NULL; + return PyLong_FromInt64(bit_count); } /*[clinic input] @@ -6226,7 +6393,7 @@ int.to_bytes the most significant byte is at the beginning of the byte array. If byteorder is 'little', the most significant byte is at the end of the byte array. To request the native byte order of the host system, use - `sys.byteorder' as the byte order value. Default is to use 'big'. + sys.byteorder as the byte order value. Default is to use 'big'. * signed as is_signed: bool = False Determines whether two's complement is used to represent the integer. @@ -6239,7 +6406,7 @@ Return an array of bytes representing an integer. static PyObject * int_to_bytes_impl(PyObject *self, Py_ssize_t length, PyObject *byteorder, int is_signed) -/*[clinic end generated code: output=89c801df114050a3 input=d42ecfb545039d71]*/ +/*[clinic end generated code: output=89c801df114050a3 input=a0103d0e9ad85c2b]*/ { int little_endian; PyObject *bytes; @@ -6290,7 +6457,7 @@ int.from_bytes the most significant byte is at the beginning of the byte array. If byteorder is 'little', the most significant byte is at the end of the byte array. To request the native byte order of the host system, use - `sys.byteorder' as the byte order value. Default is to use 'big'. + sys.byteorder as the byte order value. Default is to use 'big'. * signed as is_signed: bool = False Indicates whether two's complement is used to represent the integer. @@ -6301,7 +6468,7 @@ Return the integer represented by the given array of bytes. static PyObject * int_from_bytes_impl(PyTypeObject *type, PyObject *bytes_obj, PyObject *byteorder, int is_signed) -/*[clinic end generated code: output=efc5d68e31f9314f input=33326dccdd655553]*/ +/*[clinic end generated code: output=efc5d68e31f9314f input=2ff527997fe7b0c5]*/ { int little_endian; PyObject *long_obj, *bytes; @@ -6423,7 +6590,7 @@ PyDoc_STRVAR(long_doc, int(x, base=10) -> integer\n\ \n\ Convert a number or string to an integer, or return 0 if no arguments\n\ -are given. If x is a number, return x.__int__(). For floating point\n\ +are given. If x is a number, return x.__int__(). For floating-point\n\ numbers, this truncates towards zero.\n\ \n\ If x is not a number or if base is given, then x must be a string,\n\ @@ -6435,18 +6602,18 @@ Base 0 means to interpret the base from the string as an integer literal.\n\ 4"); static PyNumberMethods long_as_number = { - (binaryfunc)long_add, /*nb_add*/ - (binaryfunc)long_sub, /*nb_subtract*/ - (binaryfunc)long_mul, /*nb_multiply*/ + long_add_method, /*nb_add*/ + long_sub_method, /*nb_subtract*/ + long_mul_method, /*nb_multiply*/ long_mod, /*nb_remainder*/ long_divmod, /*nb_divmod*/ long_pow, /*nb_power*/ - (unaryfunc)long_neg, /*nb_negative*/ + long_neg_method, /*nb_negative*/ long_long, /*tp_positive*/ - (unaryfunc)long_abs, /*tp_absolute*/ - (inquiry)long_bool, /*tp_bool*/ - (unaryfunc)long_invert, /*nb_invert*/ - long_lshift, /*nb_lshift*/ + long_abs_method, /*tp_absolute*/ + long_bool, /*tp_bool*/ + long_invert, /*nb_invert*/ + long_lshift_method, /*nb_lshift*/ long_rshift, /*nb_rshift*/ long_and, /*nb_and*/ long_xor, /*nb_xor*/ @@ -6485,7 +6652,7 @@ PyTypeObject PyLong_Type = { &long_as_number, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ - (hashfunc)long_hash, /* tp_hash */ + long_hash, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -6514,6 +6681,7 @@ PyTypeObject PyLong_Type = { long_new, /* tp_new */ PyObject_Free, /* tp_free */ .tp_vectorcall = long_vectorcall, + .tp_version_tag = _Py_TYPE_VERSION_INT, }; static PyTypeObject Int_InfoType; @@ -6596,12 +6764,202 @@ _PyLong_FiniTypes(PyInterpreterState *interp) int PyUnstable_Long_IsCompact(const PyLongObject* op) { - return _PyLong_IsCompact(op); + return _PyLong_IsCompact((PyLongObject*)op); } #undef PyUnstable_Long_CompactValue Py_ssize_t PyUnstable_Long_CompactValue(const PyLongObject* op) { - return _PyLong_CompactValue(op); + return _PyLong_CompactValue((PyLongObject*)op); +} + + +PyObject* PyLong_FromInt32(int32_t value) +{ return PyLong_FromNativeBytes(&value, sizeof(value), -1); } + +PyObject* PyLong_FromUInt32(uint32_t value) +{ return PyLong_FromUnsignedNativeBytes(&value, sizeof(value), -1); } + +PyObject* PyLong_FromInt64(int64_t value) +{ return PyLong_FromNativeBytes(&value, sizeof(value), -1); } + +PyObject* PyLong_FromUInt64(uint64_t value) +{ return PyLong_FromUnsignedNativeBytes(&value, sizeof(value), -1); } + +#define LONG_TO_INT(obj, value, type_name) \ + do { \ + int flags = (Py_ASNATIVEBYTES_NATIVE_ENDIAN \ + | Py_ASNATIVEBYTES_ALLOW_INDEX); \ + Py_ssize_t bytes = PyLong_AsNativeBytes(obj, value, sizeof(*value), flags); \ + if (bytes < 0) { \ + return -1; \ + } \ + if ((size_t)bytes > sizeof(*value)) { \ + PyErr_SetString(PyExc_OverflowError, \ + "Python int too large to convert to " type_name); \ + return -1; \ + } \ + return 0; \ + } while (0) + +int PyLong_AsInt32(PyObject *obj, int32_t *value) +{ + LONG_TO_INT(obj, value, "C int32_t"); +} + +int PyLong_AsInt64(PyObject *obj, int64_t *value) +{ + LONG_TO_INT(obj, value, "C int64_t"); +} + +#define LONG_TO_UINT(obj, value, type_name) \ + do { \ + int flags = (Py_ASNATIVEBYTES_NATIVE_ENDIAN \ + | Py_ASNATIVEBYTES_UNSIGNED_BUFFER \ + | Py_ASNATIVEBYTES_REJECT_NEGATIVE \ + | Py_ASNATIVEBYTES_ALLOW_INDEX); \ + Py_ssize_t bytes = PyLong_AsNativeBytes(obj, value, sizeof(*value), flags); \ + if (bytes < 0) { \ + return -1; \ + } \ + if ((size_t)bytes > sizeof(*value)) { \ + PyErr_SetString(PyExc_OverflowError, \ + "Python int too large to convert to " type_name); \ + return -1; \ + } \ + return 0; \ + } while (0) + +int PyLong_AsUInt32(PyObject *obj, uint32_t *value) +{ + LONG_TO_UINT(obj, value, "C uint32_t"); +} + +int PyLong_AsUInt64(PyObject *obj, uint64_t *value) +{ + LONG_TO_UINT(obj, value, "C uint64_t"); +} + + +static const PyLongLayout PyLong_LAYOUT = { + .bits_per_digit = PyLong_SHIFT, + .digits_order = -1, // least significant first + .digit_endianness = PY_LITTLE_ENDIAN ? -1 : 1, + .digit_size = sizeof(digit), +}; + + +const PyLongLayout* +PyLong_GetNativeLayout(void) +{ + return &PyLong_LAYOUT; +} + + +int +PyLong_Export(PyObject *obj, PyLongExport *export_long) +{ + if (!PyLong_Check(obj)) { + memset(export_long, 0, sizeof(*export_long)); + PyErr_Format(PyExc_TypeError, "expect int, got %T", obj); + return -1; + } + + // Fast-path: try to convert to a int64_t + int overflow; +#if SIZEOF_LONG == 8 + long value = PyLong_AsLongAndOverflow(obj, &overflow); +#else + // Windows has 32-bit long, so use 64-bit long long instead + long long value = PyLong_AsLongLongAndOverflow(obj, &overflow); +#endif + Py_BUILD_ASSERT(sizeof(value) == sizeof(int64_t)); + // the function cannot fail since obj is a PyLongObject + assert(!(value == -1 && PyErr_Occurred())); + + if (!overflow) { + export_long->value = value; + export_long->negative = 0; + export_long->ndigits = 0; + export_long->digits = NULL; + export_long->_reserved = 0; + } + else { + PyLongObject *self = (PyLongObject*)obj; + export_long->value = 0; + export_long->negative = _PyLong_IsNegative(self); + export_long->ndigits = _PyLong_DigitCount(self); + if (export_long->ndigits == 0) { + export_long->ndigits = 1; + } + export_long->digits = self->long_value.ob_digit; + export_long->_reserved = (Py_uintptr_t)Py_NewRef(obj); + } + return 0; +} + + +void +PyLong_FreeExport(PyLongExport *export_long) +{ + PyObject *obj = (PyObject*)export_long->_reserved; + if (obj) { + export_long->_reserved = 0; + Py_DECREF(obj); + } +} + + +/* --- PyLongWriter API --------------------------------------------------- */ + +PyLongWriter* +PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits) +{ + if (ndigits <= 0) { + PyErr_SetString(PyExc_ValueError, "ndigits must be positive"); + goto error; + } + assert(digits != NULL); + + PyLongObject *obj = long_alloc(ndigits); + if (obj == NULL) { + goto error; + } + if (negative) { + _PyLong_FlipSign(obj); + } + + *digits = obj->long_value.ob_digit; + return (PyLongWriter*)obj; + +error: + *digits = NULL; + return NULL; +} + + +void +PyLongWriter_Discard(PyLongWriter *writer) +{ + if (writer == NULL) { + return; + } + + PyLongObject *obj = (PyLongObject *)writer; + assert(Py_REFCNT(obj) == 1); + Py_DECREF(obj); +} + + +PyObject* +PyLongWriter_Finish(PyLongWriter *writer) +{ + PyLongObject *obj = (PyLongObject *)writer; + assert(Py_REFCNT(obj) == 1); + + // Normalize and get singleton if possible + obj = maybe_small_long(long_normalize(obj)); + + return (PyObject*)obj; } diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 6a38952fdc1f3b..ea4d24dc690768 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -109,8 +109,6 @@ mbuf_release(_PyManagedBufferObject *self) if (self->flags&_Py_MANAGED_BUFFER_RELEASED) return; - /* NOTE: at this point self->exports can still be > 0 if this function - is called from mbuf_clear() to break up a reference cycle. */ self->flags |= _Py_MANAGED_BUFFER_RELEASED; /* PyBuffer_Release() decrements master->obj and sets it to NULL. */ @@ -268,7 +266,7 @@ PyTypeObject _PyManagedBuffer_Type = { /* Assumptions: ndim >= 1. The macro tests for a corner case that should perhaps be explicitly forbidden in the PEP. */ #define HAVE_SUBOFFSETS_IN_LAST_DIM(view) \ - (view->suboffsets && view->suboffsets[dest->ndim-1] >= 0) + (view->suboffsets && view->suboffsets[view->ndim-1] >= 0) static inline int last_dim_is_contiguous(const Py_buffer *dest, const Py_buffer *src) @@ -1088,6 +1086,16 @@ PyBuffer_ToContiguous(void *buf, const Py_buffer *src, Py_ssize_t len, char orde return ret; } +static inline Py_ssize_t +get_exports(PyMemoryViewObject *buf) +{ +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&buf->exports); +#else + return buf->exports; +#endif +} + /****************************************************************************/ /* Release/GC management */ @@ -1096,32 +1104,19 @@ PyBuffer_ToContiguous(void *buf, const Py_buffer *src, Py_ssize_t len, char orde /* Inform the managed buffer that this particular memoryview will not access the underlying buffer again. If no other memoryviews are registered with the managed buffer, the underlying buffer is released instantly and - marked as inaccessible for both the memoryview and the managed buffer. - - This function fails if the memoryview itself has exported buffers. */ -static int + marked as inaccessible for both the memoryview and the managed buffer. */ +static void _memory_release(PyMemoryViewObject *self) { + assert(get_exports(self) == 0); if (self->flags & _Py_MEMORYVIEW_RELEASED) - return 0; + return; - if (self->exports == 0) { - self->flags |= _Py_MEMORYVIEW_RELEASED; - assert(self->mbuf->exports > 0); - if (--self->mbuf->exports == 0) - mbuf_release(self->mbuf); - return 0; - } - if (self->exports > 0) { - PyErr_Format(PyExc_BufferError, - "memoryview has %zd exported buffer%s", self->exports, - self->exports==1 ? "" : "s"); - return -1; + self->flags |= _Py_MEMORYVIEW_RELEASED; + assert(self->mbuf->exports > 0); + if (--self->mbuf->exports == 0) { + mbuf_release(self->mbuf); } - - PyErr_SetString(PyExc_SystemError, - "_memory_release(): negative export count"); - return -1; } /*[clinic input] @@ -1134,18 +1129,31 @@ static PyObject * memoryview_release_impl(PyMemoryViewObject *self) /*[clinic end generated code: output=d0b7e3ba95b7fcb9 input=bc71d1d51f4a52f0]*/ { - if (_memory_release(self) < 0) + Py_ssize_t exports = get_exports(self); + if (exports == 0) { + _memory_release(self); + Py_RETURN_NONE; + } + + if (exports > 0) { + PyErr_Format(PyExc_BufferError, + "memoryview has %zd exported buffer%s", exports, + exports==1 ? "" : "s"); return NULL; - Py_RETURN_NONE; + } + + PyErr_SetString(PyExc_SystemError, + "memoryview: negative export count"); + return NULL; } static void memory_dealloc(PyObject *_self) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; - assert(self->exports == 0); + assert(get_exports(self) == 0); _PyObject_GC_UNTRACK(self); - (void)_memory_release(self); + _memory_release(self); Py_CLEAR(self->mbuf); if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) self); @@ -1164,8 +1172,10 @@ static int memory_clear(PyObject *_self) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; - (void)_memory_release(self); - Py_CLEAR(self->mbuf); + if (get_exports(self) == 0) { + _memory_release(self); + Py_CLEAR(self->mbuf); + } return 0; } @@ -1590,7 +1600,11 @@ memory_getbuf(PyObject *_self, Py_buffer *view, int flags) view->obj = Py_NewRef(self); +#ifdef Py_GIL_DISABLED + _Py_atomic_add_ssize(&self->exports, 1); +#else self->exports++; +#endif return 0; } @@ -1599,7 +1613,11 @@ static void memory_releasebuf(PyObject *_self, Py_buffer *view) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; +#ifdef Py_GIL_DISABLED + _Py_atomic_add_ssize(&self->exports, -1); +#else self->exports--; +#endif return; /* PyBuffer_Release() decrements view->obj after this function returns. */ } @@ -2749,6 +2767,141 @@ static PySequenceMethods memory_as_sequence = { }; +/****************************************************************************/ +/* Counting */ +/****************************************************************************/ + +/*[clinic input] +memoryview.count + + value: object + / + +Count the number of occurrences of a value. +[clinic start generated code]*/ + +static PyObject * +memoryview_count(PyMemoryViewObject *self, PyObject *value) +/*[clinic end generated code: output=e2c255a8d54eaa12 input=e3036ce1ed7d1823]*/ +{ + PyObject *iter = PyObject_GetIter(_PyObject_CAST(self)); + if (iter == NULL) { + return NULL; + } + + Py_ssize_t count = 0; + PyObject *item = NULL; + while (PyIter_NextItem(iter, &item)) { + if (item == NULL) { + Py_DECREF(iter); + return NULL; + } + if (item == value) { + Py_DECREF(item); + count++; // no overflow since count <= len(mv) <= PY_SSIZE_T_MAX + continue; + } + int contained = PyObject_RichCompareBool(item, value, Py_EQ); + Py_DECREF(item); + if (contained > 0) { // more likely than 'contained < 0' + count++; // no overflow since count <= len(mv) <= PY_SSIZE_T_MAX + } + else if (contained < 0) { + Py_DECREF(iter); + return NULL; + } + } + Py_DECREF(iter); + return PyLong_FromSsize_t(count); +} + + +/**************************************************************************/ +/* Lookup */ +/**************************************************************************/ + +/*[clinic input] +memoryview.index + + value: object + start: slice_index(accept={int}) = 0 + stop: slice_index(accept={int}, c_default="PY_SSIZE_T_MAX") = sys.maxsize + / + +Return the index of the first occurrence of a value. + +Raises ValueError if the value is not present. +[clinic start generated code]*/ + +static PyObject * +memoryview_index_impl(PyMemoryViewObject *self, PyObject *value, + Py_ssize_t start, Py_ssize_t stop) +/*[clinic end generated code: output=e0185e3819e549df input=0697a0165bf90b5a]*/ +{ + const Py_buffer *view = &self->view; + CHECK_RELEASED(self); + + if (view->ndim == 0) { + PyErr_SetString(PyExc_TypeError, "invalid lookup on 0-dim memory"); + return NULL; + } + + if (view->ndim == 1) { + Py_ssize_t n = view->shape[0]; + + if (start < 0) { + start = Py_MAX(start + n, 0); + } + + if (stop < 0) { + stop = Py_MAX(stop + n, 0); + } + + stop = Py_MIN(stop, n); + assert(stop >= 0); + assert(stop <= n); + + start = Py_MIN(start, stop); + assert(0 <= start); + assert(start <= stop); + + PyObject *obj = _PyObject_CAST(self); + for (Py_ssize_t index = start; index < stop; index++) { + // Note: while memoryviews can be mutated during iterations + // when calling the == operator, their shape cannot. As such, + // it is safe to assume that the index remains valid for the + // entire loop. + assert(index < n); + + PyObject *item = memory_item(obj, index); + if (item == NULL) { + return NULL; + } + if (item == value) { + Py_DECREF(item); + return PyLong_FromSsize_t(index); + } + int contained = PyObject_RichCompareBool(item, value, Py_EQ); + Py_DECREF(item); + if (contained > 0) { // more likely than 'contained < 0' + return PyLong_FromSsize_t(index); + } + else if (contained < 0) { + return NULL; + } + } + + PyErr_SetString(PyExc_ValueError, "memoryview.index(x): x not found"); + return NULL; + } + + PyErr_SetString(PyExc_NotImplementedError, + "multi-dimensional lookup is not implemented"); + return NULL; + +} + + /**************************************************************************/ /* Comparisons */ /**************************************************************************/ @@ -3087,7 +3240,7 @@ memory_hash(PyObject *_self) } /* Can't fail */ - self->hash = _Py_HashBytes(mem, view->len); + self->hash = Py_HashBuffer(mem, view->len); if (mem != view->buf) PyMem_Free(mem); @@ -3255,6 +3408,9 @@ PyDoc_STRVAR(memory_f_contiguous_doc, "A bool indicating whether the memory is Fortran contiguous."); PyDoc_STRVAR(memory_contiguous_doc, "A bool indicating whether the memory is contiguous."); +PyDoc_STRVAR(memory_exit_doc, + "__exit__($self, /, *exc_info)\n--\n\n" + "Release the underlying buffer exposed by the memoryview object."); static PyGetSetDef memory_getsetlist[] = { @@ -3282,8 +3438,11 @@ static PyMethodDef memory_methods[] = { MEMORYVIEW_CAST_METHODDEF MEMORYVIEW_TOREADONLY_METHODDEF MEMORYVIEW__FROM_FLAGS_METHODDEF + MEMORYVIEW_COUNT_METHODDEF + MEMORYVIEW_INDEX_METHODDEF {"__enter__", memory_enter, METH_NOARGS, NULL}, - {"__exit__", memory_exit, METH_VARARGS, NULL}, + {"__exit__", memory_exit, METH_VARARGS, memory_exit_doc}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} }; @@ -3353,6 +3512,7 @@ memory_iter(PyObject *seq) PyErr_BadInternalCall(); return NULL; } + CHECK_RELEASED(seq); PyMemoryViewObject *obj = (PyMemoryViewObject *)seq; int ndims = obj->view.ndim; if (ndims == 0) { diff --git a/Objects/methodobject.c b/Objects/methodobject.c index 599fb05cb5874f..ecec0f7205a11d 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -156,8 +156,9 @@ PyCMethod_GetClass(PyObject *op) /* Methods (the standard built-in methods, that is) */ static void -meth_dealloc(PyCFunctionObject *m) +meth_dealloc(PyObject *self) { + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); // The Py_TRASHCAN mechanism requires that we be able to // call PyObject_GC_UnTrack twice on an object. PyObject_GC_UnTrack(m); @@ -175,8 +176,9 @@ meth_dealloc(PyCFunctionObject *m) } static PyObject * -meth_reduce(PyCFunctionObject *m, PyObject *Py_UNUSED(ignored)) +meth_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) { + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); if (m->m_self == NULL || PyModule_Check(m->m_self)) return PyUnicode_FromString(m->m_ml->ml_name); @@ -185,32 +187,35 @@ meth_reduce(PyCFunctionObject *m, PyObject *Py_UNUSED(ignored)) } static PyMethodDef meth_methods[] = { - {"__reduce__", (PyCFunction)meth_reduce, METH_NOARGS, NULL}, + {"__reduce__", meth_reduce, METH_NOARGS, NULL}, {NULL, NULL} }; static PyObject * -meth_get__text_signature__(PyCFunctionObject *m, void *closure) +meth_get__text_signature__(PyObject *self, void *closure) { + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); return _PyType_GetTextSignatureFromInternalDoc(m->m_ml->ml_name, m->m_ml->ml_doc, m->m_ml->ml_flags); } static PyObject * -meth_get__doc__(PyCFunctionObject *m, void *closure) +meth_get__doc__(PyObject *self, void *closure) { + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); return _PyType_GetDocFromInternalDoc(m->m_ml->ml_name, m->m_ml->ml_doc); } static PyObject * -meth_get__name__(PyCFunctionObject *m, void *closure) +meth_get__name__(PyObject *self, void *closure) { + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); return PyUnicode_FromString(m->m_ml->ml_name); } static PyObject * -meth_get__qualname__(PyCFunctionObject *m, void *closure) +meth_get__qualname__(PyObject *self, void *closure) { /* If __self__ is a module or NULL, return m.__name__ (e.g. len.__qualname__ == 'len') @@ -220,14 +225,15 @@ meth_get__qualname__(PyCFunctionObject *m, void *closure) Otherwise return type(m.__self__).__qualname__ + '.' + m.__name__ (e.g. [].append.__qualname__ == 'list.append') */ - PyObject *type, *type_qualname, *res; - if (m->m_self == NULL || PyModule_Check(m->m_self)) + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); + if (m->m_self == NULL || PyModule_Check(m->m_self)) { return PyUnicode_FromString(m->m_ml->ml_name); + } - type = PyType_Check(m->m_self) ? m->m_self : (PyObject*)Py_TYPE(m->m_self); + PyObject *type = PyType_Check(m->m_self) ? m->m_self : (PyObject*)Py_TYPE(m->m_self); - type_qualname = PyObject_GetAttr(type, &_Py_ID(__qualname__)); + PyObject *type_qualname = PyObject_GetAttr(type, &_Py_ID(__qualname__)); if (type_qualname == NULL) return NULL; @@ -238,14 +244,15 @@ meth_get__qualname__(PyCFunctionObject *m, void *closure) return NULL; } - res = PyUnicode_FromFormat("%S.%s", type_qualname, m->m_ml->ml_name); + PyObject *res = PyUnicode_FromFormat("%S.%s", type_qualname, m->m_ml->ml_name); Py_DECREF(type_qualname); return res; } static int -meth_traverse(PyCFunctionObject *m, visitproc visit, void *arg) +meth_traverse(PyObject *self, visitproc visit, void *arg) { + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); Py_VISIT(PyCFunction_GET_CLASS(m)); Py_VISIT(m->m_self); Py_VISIT(m->m_module); @@ -253,22 +260,22 @@ meth_traverse(PyCFunctionObject *m, visitproc visit, void *arg) } static PyObject * -meth_get__self__(PyCFunctionObject *m, void *closure) +meth_get__self__(PyObject *meth, void *closure) { - PyObject *self; - - self = PyCFunction_GET_SELF(m); - if (self == NULL) + PyCFunctionObject *m = _PyCFunctionObject_CAST(meth); + PyObject *self = PyCFunction_GET_SELF(m); + if (self == NULL) { self = Py_None; + } return Py_NewRef(self); } -static PyGetSetDef meth_getsets [] = { - {"__doc__", (getter)meth_get__doc__, NULL, NULL}, - {"__name__", (getter)meth_get__name__, NULL, NULL}, - {"__qualname__", (getter)meth_get__qualname__, NULL, NULL}, - {"__self__", (getter)meth_get__self__, NULL, NULL}, - {"__text_signature__", (getter)meth_get__text_signature__, NULL, NULL}, +static PyGetSetDef meth_getsets[] = { + {"__doc__", meth_get__doc__, NULL, NULL}, + {"__name__", meth_get__name__, NULL, NULL}, + {"__qualname__", meth_get__qualname__, NULL, NULL}, + {"__self__", meth_get__self__, NULL, NULL}, + {"__text_signature__", meth_get__text_signature__, NULL, NULL}, {0} }; @@ -280,15 +287,18 @@ static PyMemberDef meth_members[] = { }; static PyObject * -meth_repr(PyCFunctionObject *m) +meth_repr(PyObject *self) { - if (m->m_self == NULL || PyModule_Check(m->m_self)) + PyCFunctionObject *m = _PyCFunctionObject_CAST(self); + if (m->m_self == NULL || PyModule_Check(m->m_self)) { return PyUnicode_FromFormat("", - m->m_ml->ml_name); + m->m_ml->ml_name); + } + return PyUnicode_FromFormat("", - m->m_ml->ml_name, - Py_TYPE(m->m_self)->tp_name, - m->m_self); + m->m_ml->ml_name, + Py_TYPE(m->m_self)->tp_name, + m->m_self); } static PyObject * @@ -317,14 +327,15 @@ meth_richcompare(PyObject *self, PyObject *other, int op) } static Py_hash_t -meth_hash(PyCFunctionObject *a) +meth_hash(PyObject *self) { - Py_hash_t x, y; - x = _Py_HashPointer(a->m_self); - y = _Py_HashPointer((void*)(a->m_ml->ml_meth)); + PyCFunctionObject *a = _PyCFunctionObject_CAST(self); + Py_hash_t x = PyObject_GenericHash(a->m_self); + Py_hash_t y = Py_HashPointer((void*)(a->m_ml->ml_meth)); x ^= y; - if (x == -1) + if (x == -1) { x = -2; + } return x; } @@ -334,16 +345,16 @@ PyTypeObject PyCFunction_Type = { "builtin_function_or_method", sizeof(PyCFunctionObject), 0, - (destructor)meth_dealloc, /* tp_dealloc */ + meth_dealloc, /* tp_dealloc */ offsetof(PyCFunctionObject, vectorcall), /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)meth_repr, /* tp_repr */ + meth_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ - (hashfunc)meth_hash, /* tp_hash */ + meth_hash, /* tp_hash */ cfunction_call, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -352,7 +363,7 @@ PyTypeObject PyCFunction_Type = { Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_VECTORCALL, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)meth_traverse, /* tp_traverse */ + meth_traverse, /* tp_traverse */ 0, /* tp_clear */ meth_richcompare, /* tp_richcompare */ offsetof(PyCFunctionObject, m_weakreflist), /* tp_weaklistoffset */ diff --git a/Objects/mimalloc/alloc.c b/Objects/mimalloc/alloc.c index e6286b54bedc14..44c84cf1931717 100644 --- a/Objects/mimalloc/alloc.c +++ b/Objects/mimalloc/alloc.c @@ -237,7 +237,7 @@ static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block if (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer? (n==NULL || mi_is_in_same_page(block, n))) // quick check: in same page or NULL? { - // Suspicous: decoded value a in block is in the same page (or NULL) -- maybe a double free? + // Suspicious: decoded value a in block is in the same page (or NULL) -- maybe a double free? // (continue in separate function to improve code generation) is_double_free = mi_check_is_double_freex(page, block); } @@ -609,7 +609,10 @@ bool _mi_free_delayed_block(mi_block_t* block) { // get segment and page const mi_segment_t* const segment = _mi_ptr_segment(block); mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie); +#ifndef Py_GIL_DISABLED + // The GC traverses heaps of other threads, which can trigger this assert. mi_assert_internal(_mi_thread_id() == segment->thread_id); +#endif mi_page_t* const page = _mi_segment_page_of(segment, block); // Clear the no-delayed flag so delayed freeing is used again for this page. diff --git a/Objects/mimalloc/arena.c b/Objects/mimalloc/arena.c index f8883603860dce..5db5d950c43b68 100644 --- a/Objects/mimalloc/arena.c +++ b/Objects/mimalloc/arena.c @@ -269,7 +269,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(mi_arena_t* arena, size_t ar return p; } -// allocate in a speficic arena +// allocate in a specific arena static void* mi_arena_try_alloc_at_id(mi_arena_id_t arena_id, bool match_numa_node, int numa_node, size_t size, size_t alignment, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld ) { @@ -493,7 +493,7 @@ static bool mi_arena_purge_range(mi_arena_t* arena, size_t idx, size_t startidx, size_t bitidx = startidx; bool all_purged = false; while (bitidx < endidx) { - // count consequetive ones in the purge mask + // count consecutive ones in the purge mask size_t count = 0; while (bitidx + count < endidx && (purge & ((size_t)1 << (bitidx + count))) != 0) { count++; @@ -530,7 +530,7 @@ static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force, mi if (purge != 0) { size_t bitidx = 0; while (bitidx < MI_BITMAP_FIELD_BITS) { - // find consequetive range of ones in the purge mask + // find consecutive range of ones in the purge mask size_t bitlen = 0; while (bitidx + bitlen < MI_BITMAP_FIELD_BITS && (purge & ((size_t)1 << (bitidx + bitlen))) != 0) { bitlen++; diff --git a/Objects/mimalloc/bitmap.c b/Objects/mimalloc/bitmap.c index ec3c755822dac1..31830756f58c7c 100644 --- a/Objects/mimalloc/bitmap.c +++ b/Objects/mimalloc/bitmap.c @@ -7,7 +7,7 @@ terms of the MIT license. A copy of the license can be found in the file /* ---------------------------------------------------------------------------- Concurrent bitmap that can set/reset sequences of bits atomically, -represeted as an array of fields where each field is a machine word (`size_t`) +represented as an array of fields where each field is a machine word (`size_t`) There are two api's; the standard one cannot have sequences that cross between the bitmap fields (and a sequence must be <= MI_BITMAP_FIELD_BITS). @@ -108,7 +108,7 @@ bool _mi_bitmap_try_find_from_claim(mi_bitmap_t bitmap, const size_t bitmap_fiel return false; } -// Like _mi_bitmap_try_find_from_claim but with an extra predicate that must be fullfilled +// Like _mi_bitmap_try_find_from_claim but with an extra predicate that must be fulfilled bool _mi_bitmap_try_find_from_claim_pred(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_pred_fun_t pred_fun, void* pred_arg, diff --git a/Objects/mimalloc/bitmap.h b/Objects/mimalloc/bitmap.h index 9ba15d5d6f09ea..d60f6dd3cb8e27 100644 --- a/Objects/mimalloc/bitmap.h +++ b/Objects/mimalloc/bitmap.h @@ -7,7 +7,7 @@ terms of the MIT license. A copy of the license can be found in the file /* ---------------------------------------------------------------------------- Concurrent bitmap that can set/reset sequences of bits atomically, -represeted as an array of fields where each field is a machine word (`size_t`) +represented as an array of fields where each field is a machine word (`size_t`) There are two api's; the standard one cannot have sequences that cross between the bitmap fields (and a sequence must be <= MI_BITMAP_FIELD_BITS). @@ -72,7 +72,7 @@ bool _mi_bitmap_try_find_claim_field(mi_bitmap_t bitmap, size_t idx, const size_ // For now, `count` can be at most MI_BITMAP_FIELD_BITS and will never cross fields. bool _mi_bitmap_try_find_from_claim(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx); -// Like _mi_bitmap_try_find_from_claim but with an extra predicate that must be fullfilled +// Like _mi_bitmap_try_find_from_claim but with an extra predicate that must be fulfilled typedef bool (mi_cdecl *mi_bitmap_pred_fun_t)(mi_bitmap_index_t bitmap_idx, void* pred_arg); bool _mi_bitmap_try_find_from_claim_pred(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_pred_fun_t pred_fun, void* pred_arg, mi_bitmap_index_t* bitmap_idx); diff --git a/Objects/mimalloc/heap.c b/Objects/mimalloc/heap.c index 154dad0b128480..d92dc768e5ec28 100644 --- a/Objects/mimalloc/heap.c +++ b/Objects/mimalloc/heap.c @@ -98,7 +98,10 @@ static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t if (mi_page_all_free(page)) { // no more used blocks, free the page. // note: this will free retired pages as well. - _mi_page_free(page, pq, collect >= MI_FORCE); + bool freed = _PyMem_mi_page_maybe_free(page, pq, collect >= MI_FORCE); + if (!freed && collect == MI_ABANDON) { + _mi_page_abandon(page, pq); + } } else if (collect == MI_ABANDON) { // still used blocks but the thread is done; abandon the page @@ -153,6 +156,9 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) // collect retired pages _mi_heap_collect_retired(heap, force); + // free pages that were delayed with QSBR + _PyMem_mi_heap_collect_qsbr(heap); + // collect all pages owned by this thread mi_heap_visit_pages(heap, &mi_heap_page_collect, &collect, NULL); mi_assert_internal( collect != MI_ABANDON || mi_atomic_load_ptr_acquire(mi_block_t,&heap->thread_delayed_free) == NULL ); @@ -440,7 +446,7 @@ void mi_heap_delete(mi_heap_t* heap) if (heap==NULL || !mi_heap_is_initialized(heap)) return; if (!mi_heap_is_backing(heap)) { - // tranfer still used pages to the backing heap + // transfer still used pages to the backing heap mi_heap_absorb(heap->tld->heap_backing, heap); } else { diff --git a/Objects/mimalloc/init.c b/Objects/mimalloc/init.c index cb0ef6642803cc..81b241063ff40f 100644 --- a/Objects/mimalloc/init.c +++ b/Objects/mimalloc/init.c @@ -560,7 +560,7 @@ void mi_process_init(void) mi_attr_noexcept { _mi_verbose_message("secure level: %d\n", MI_SECURE); _mi_verbose_message("mem tracking: %s\n", MI_TRACK_TOOL); #if MI_TSAN - _mi_verbose_message("thread santizer enabled\n"); + _mi_verbose_message("thread sanitizer enabled\n"); #endif mi_thread_init(); diff --git a/Objects/mimalloc/options.c b/Objects/mimalloc/options.c index 345b560e3e7f4c..2a8f481d569e1e 100644 --- a/Objects/mimalloc/options.c +++ b/Objects/mimalloc/options.c @@ -269,7 +269,7 @@ static _Atomic(size_t) warning_count; // = 0; // when >= max_warning_count stop // (recursively) invoke malloc again to allocate space for the thread local // variables on demand. This is why we use a _mi_preloading test on such // platforms. However, C code generator may move the initial thread local address -// load before the `if` and we therefore split it out in a separate funcion. +// load before the `if` and we therefore split it out in a separate function. static mi_decl_thread bool recurse = false; static mi_decl_noinline bool mi_recurse_enter_prim(void) { diff --git a/Objects/mimalloc/os.c b/Objects/mimalloc/os.c index f3bc7184c41c5b..c9103168c12507 100644 --- a/Objects/mimalloc/os.c +++ b/Objects/mimalloc/os.c @@ -115,8 +115,12 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize uintptr_t init = MI_HINT_BASE; #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode - uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap()); - init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB + mi_heap_t* heap = mi_prim_get_default_heap(); + // gh-123022: default heap may not be initialized in CPython in background threads + if (mi_heap_is_initialized(heap)) { + uintptr_t r = _mi_heap_random_next(heap); + init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB + } #endif uintptr_t expected = hint + size; mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init); @@ -553,8 +557,12 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { // Initialize the start address after the 32TiB area start = ((uintptr_t)32 << 40); // 32TiB virtual start address #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode - uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap()); - start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB + mi_heap_t* heap = mi_prim_get_default_heap(); + // gh-123022: default heap may not be initialized in CPython in background threads + if (mi_heap_is_initialized(heap)) { + uintptr_t r = _mi_heap_random_next(heap); + start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB + } #endif } end = start + size; diff --git a/Objects/mimalloc/page.c b/Objects/mimalloc/page.c index 63db893e49405c..ff7444cce10923 100644 --- a/Objects/mimalloc/page.c +++ b/Objects/mimalloc/page.c @@ -225,6 +225,9 @@ void _mi_page_free_collect(mi_page_t* page, bool force) { // and the local free list if (page->local_free != NULL) { + // any previous QSBR goals are no longer valid because we reused the page + _PyMem_mi_page_clear_qsbr(page); + if mi_likely(page->free == NULL) { // usual case page->free = page->local_free; @@ -267,6 +270,7 @@ void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page) { // TODO: push on full queue immediately if it is full? mi_page_queue_t* pq = mi_page_queue(heap, mi_page_block_size(page)); mi_page_queue_push(heap, pq, page); + _PyMem_mi_page_reclaimed(page); mi_assert_expensive(_mi_page_is_valid(page)); } @@ -383,6 +387,13 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) { mi_heap_t* pheap = mi_page_heap(page); +#ifdef Py_GIL_DISABLED + if (page->qsbr_node.next != NULL) { + // remove from QSBR queue, but keep the goal + llist_remove(&page->qsbr_node); + } +#endif + // remove from our page list mi_segments_tld_t* segments_tld = &pheap->tld->segments; mi_page_queue_remove(pq, page); @@ -417,6 +428,11 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { mi_heap_t* heap = mi_page_heap(page); +#ifdef Py_GIL_DISABLED + mi_assert_internal(page->qsbr_goal == 0); + mi_assert_internal(page->qsbr_node.next == NULL); +#endif + // remove from the page list // (no need to do _mi_heap_delayed_free first as all blocks are already free) mi_segments_tld_t* segments_tld = &heap->tld->segments; @@ -444,6 +460,9 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { mi_page_set_has_aligned(page, false); + // any previous QSBR goals are no longer valid because we reused the page + _PyMem_mi_page_clear_qsbr(page); + // don't retire too often.. // (or we end up retiring and re-allocating most of the time) // NOTE: refine this more: we should not retire if this @@ -462,10 +481,10 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { if (index < heap->page_retired_min) heap->page_retired_min = index; if (index > heap->page_retired_max) heap->page_retired_max = index; mi_assert_internal(mi_page_all_free(page)); - return; // dont't free after all + return; // don't free after all } } - _mi_page_free(page, pq, false); + _PyMem_mi_page_maybe_free(page, pq, false); } // free retired pages: we don't need to look at the entire queues @@ -480,7 +499,10 @@ void _mi_heap_collect_retired(mi_heap_t* heap, bool force) { if (mi_page_all_free(page)) { page->retire_expire--; if (force || page->retire_expire == 0) { - _mi_page_free(pq->first, pq, force); +#ifdef Py_GIL_DISABLED + mi_assert_internal(page->qsbr_goal == 0); +#endif + _PyMem_mi_page_maybe_free(page, pq, force); } else { // keep retired, update min/max @@ -661,6 +683,7 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi // set fields mi_page_set_heap(page, heap); page->tag = heap->tag; + page->use_qsbr = heap->page_use_qsbr; page->debug_offset = heap->debug_offset; page->xblock_size = (block_size < MI_HUGE_BLOCK_SIZE ? (uint32_t)block_size : MI_HUGE_BLOCK_SIZE); // initialize before _mi_segment_page_start size_t page_size; @@ -691,6 +714,10 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi mi_assert_internal(page->xthread_free == 0); mi_assert_internal(page->next == NULL); mi_assert_internal(page->prev == NULL); +#ifdef Py_GIL_DISABLED + mi_assert_internal(page->qsbr_goal == 0); + mi_assert_internal(page->qsbr_node.next == NULL); +#endif mi_assert_internal(page->retire_expire == 0); mi_assert_internal(!mi_page_has_aligned(page)); #if (MI_PADDING || MI_ENCODE_FREELIST) @@ -750,6 +777,7 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p mi_heap_stat_counter_increase(heap, searches, count); if (page == NULL) { + _PyMem_mi_heap_collect_qsbr(heap); // some pages might be safe to free now _mi_heap_collect_retired(heap, false); // perhaps make a page available? page = mi_page_fresh(heap, pq); if (page == NULL && first_try) { @@ -760,6 +788,7 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p else { mi_assert(pq->first == page); page->retire_expire = 0; + _PyMem_mi_page_clear_qsbr(page); } mi_assert_internal(page == NULL || mi_page_immediate_available(page)); return page; @@ -785,6 +814,7 @@ static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) { if (mi_page_immediate_available(page)) { page->retire_expire = 0; + _PyMem_mi_page_clear_qsbr(page); return page; // fast path } } @@ -878,6 +908,7 @@ static mi_page_t* mi_find_page(mi_heap_t* heap, size_t size, size_t huge_alignme return NULL; } else { + _PyMem_mi_heap_collect_qsbr(heap); return mi_large_huge_page_alloc(heap,size,huge_alignment); } } diff --git a/Objects/mimalloc/prim/unix/prim.c b/Objects/mimalloc/prim/unix/prim.c index ec8447ab40d70c..1598ebabf0a4da 100644 --- a/Objects/mimalloc/prim/unix/prim.c +++ b/Objects/mimalloc/prim/unix/prim.c @@ -27,6 +27,7 @@ terms of the MIT license. A copy of the license can be found in the file #include // mmap #include // sysconf +#include // open, close, read, access #if defined(__linux__) #include @@ -50,7 +51,7 @@ terms of the MIT license. A copy of the license can be found in the file #include #endif -#if !defined(__HAIKU__) && !defined(__APPLE__) && !defined(__CYGWIN__) && !defined(_AIX) && !defined(__FreeBSD__) && !defined(__sun) +#if !defined(__HAIKU__) && !defined(__APPLE__) && !defined(__CYGWIN__) && !defined(_AIX) && !defined(__OpenBSD__) && !defined(__FreeBSD__) && !defined(__sun) && !defined(__NetBSD__) #define MI_HAS_SYSCALL_H #include #endif @@ -76,7 +77,7 @@ static int mi_prim_access(const char *fpath, int mode) { return syscall(SYS_access,fpath,mode); } -#elif !defined(__APPLE__) && !defined(_AIX) && !defined(__FreeBSD__) && !defined(__sun) // avoid unused warnings +#elif !defined(__APPLE__) && !defined(_AIX) && !defined(__OpenBSD__) && !defined(__FreeBSD__) && !defined(__sun) && !defined(__NetBSD__) // avoid unused warnings static int mi_prim_open(const char* fpath, int open_flags) { return open(fpath,open_flags); @@ -738,7 +739,7 @@ bool _mi_prim_getenv(const char* name, char* result, size_t result_size) { #endif bool _mi_prim_random_buf(void* buf, size_t buf_len) { #if defined(MAC_OS_X_VERSION_10_15) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_15 - // We prefere CCRandomGenerateBytes as it returns an error code while arc4random_buf + // We prefer CCRandomGenerateBytes as it returns an error code while arc4random_buf // may fail silently on macOS. See PR #390, and return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess); #else diff --git a/Objects/mimalloc/prim/windows/etw.h b/Objects/mimalloc/prim/windows/etw.h index 4e0a092a10f4ba..e9ec35fc9ed7eb 100644 --- a/Objects/mimalloc/prim/windows/etw.h +++ b/Objects/mimalloc/prim/windows/etw.h @@ -136,7 +136,7 @@ extern "C" { // - MCGEN_EVENTSETINFORMATION // - MCGEN_EVENTWRITETRANSFER // -// If the the macro is undefined, the MC implementation will default to the +// If the macro is undefined, the MC implementation will default to the // corresponding ETW APIs. For example, if the MCGEN_EVENTREGISTER macro is // undefined, the EventRegister[MyProviderName] macro will use EventRegister // in user mode and will use EtwRegister in kernel mode. diff --git a/Objects/mimalloc/segment-map.c b/Objects/mimalloc/segment-map.c index 3cd2127e56c1a7..5141f2e1b1dffa 100644 --- a/Objects/mimalloc/segment-map.c +++ b/Objects/mimalloc/segment-map.c @@ -84,7 +84,7 @@ static mi_segment_t* _mi_segment_of(const void* p) { // TODO: maintain max/min allocated range for efficiency for more efficient rejection of invalid pointers? // search downwards for the first segment in case it is an interior pointer - // could be slow but searches in MI_INTPTR_SIZE * MI_SEGMENT_SIZE (512MiB) steps trough + // could be slow but searches in MI_INTPTR_SIZE * MI_SEGMENT_SIZE (512MiB) steps through // valid huge objects // note: we could maintain a lowest index to speed up the path for invalid pointers? size_t lobitidx; diff --git a/Objects/mimalloc/segment.c b/Objects/mimalloc/segment.c index 584233b8b57bb4..9b092b9b734d4c 100644 --- a/Objects/mimalloc/segment.c +++ b/Objects/mimalloc/segment.c @@ -718,7 +718,7 @@ static mi_page_t* mi_segment_span_allocate(mi_segment_t* segment, size_t slice_i // set slice back pointers for the first MI_MAX_SLICE_OFFSET entries size_t extra = slice_count-1; if (extra > MI_MAX_SLICE_OFFSET) extra = MI_MAX_SLICE_OFFSET; - if (slice_index + extra >= segment->slice_entries) extra = segment->slice_entries - slice_index - 1; // huge objects may have more slices than avaiable entries in the segment->slices + if (slice_index + extra >= segment->slice_entries) extra = segment->slice_entries - slice_index - 1; // huge objects may have more slices than available entries in the segment->slices mi_slice_t* slice_next = slice + 1; for (size_t i = 1; i <= extra; i++, slice_next++) { @@ -814,6 +814,9 @@ static mi_segment_t* mi_segment_os_alloc( size_t required, size_t page_alignment const size_t extra = align_offset - info_size; // recalculate due to potential guard pages *psegment_slices = mi_segment_calculate_slices(required + extra, ppre_size, pinfo_slices); + + // mi_page_t.slice_count type is uint32_t + if (*psegment_slices > (size_t)UINT32_MAX) return NULL; } const size_t segment_size = (*psegment_slices) * MI_SEGMENT_SLICE_SIZE; @@ -865,6 +868,9 @@ static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi size_t pre_size; size_t segment_slices = mi_segment_calculate_slices(required, &pre_size, &info_slices); + // mi_page_t.slice_count type is uint32_t + if (segment_slices > (size_t)UINT32_MAX) return NULL; + // Commit eagerly only if not the first N lazy segments (to reduce impact of many threads that allocate just a little) const bool eager_delay = (// !_mi_os_has_overcommit() && // never delay on overcommit systems _mi_current_thread_count() > 1 && // do not delay for the first N threads @@ -982,6 +988,10 @@ static mi_slice_t* mi_segment_page_clear(mi_page_t* page, mi_segments_tld_t* tld mi_assert_internal(mi_page_all_free(page)); mi_segment_t* segment = _mi_ptr_segment(page); mi_assert_internal(segment->used > 0); +#ifdef Py_GIL_DISABLED + mi_assert_internal(page->qsbr_goal == 0); + mi_assert_internal(page->qsbr_node.next == NULL); +#endif size_t inuse = page->capacity * mi_page_block_size(page); _mi_stat_decrease(&tld->stats->page_committed, inuse); @@ -1270,10 +1280,13 @@ static bool mi_segment_check_free(mi_segment_t* segment, size_t slices_needed, s // ensure used count is up to date and collect potential concurrent frees mi_page_t* const page = mi_slice_to_page(slice); _mi_page_free_collect(page, false); - if (mi_page_all_free(page)) { + if (mi_page_all_free(page) && _PyMem_mi_page_is_safe_to_free(page)) { // if this page is all free now, free it without adding to any queues (yet) mi_assert_internal(page->next == NULL && page->prev==NULL); _mi_stat_decrease(&tld->stats->pages_abandoned, 1); +#ifdef Py_GIL_DISABLED + page->qsbr_goal = 0; +#endif segment->abandoned--; slice = mi_segment_page_clear(page, tld); // re-assign slice due to coalesce! mi_assert_internal(!mi_slice_is_used(slice)); @@ -1344,15 +1357,18 @@ static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap, mi_page_set_heap(page, target_heap); _mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, true); // override never (after heap is set) _mi_page_free_collect(page, false); // ensure used count is up to date - if (mi_page_all_free(page)) { + if (mi_page_all_free(page) && _PyMem_mi_page_is_safe_to_free(page)) { // if everything free by now, free the page +#ifdef Py_GIL_DISABLED + page->qsbr_goal = 0; +#endif slice = mi_segment_page_clear(page, tld); // set slice again due to coalesceing } else { // otherwise reclaim it into the heap _mi_page_reclaim(target_heap, page); if (requested_block_size == page->xblock_size && mi_page_has_any_available(page) && - heap == target_heap) { + requested_block_size <= MI_MEDIUM_OBJ_SIZE_MAX && heap == target_heap) { if (right_page_reclaimed != NULL) { *right_page_reclaimed = true; } } } diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 3a1c516658dce7..a8d64c9aefae6b 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -3,13 +3,21 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_dict.h" // _PyDict_EnablePerThreadRefcounting() +#include "pycore_fileutils.h" // _Py_wgetcwd #include "pycore_interp.h" // PyInterpreterState.importlib +#include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyModule_CreateInitialized() #include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_object.h" // _PyType_AllocNoTrack #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "osdefs.h" // MAXPATHLEN + + +#define _PyModule_CAST(op) \ + (assert(PyModule_Check(op)), _Py_CAST(PyModuleObject*, (op))) static PyMemberDef module_members[] = { @@ -88,21 +96,29 @@ new_module_notrack(PyTypeObject *mt) m->md_weaklist = NULL; m->md_name = NULL; m->md_dict = PyDict_New(); - if (m->md_dict != NULL) { - return m; + if (m->md_dict == NULL) { + Py_DECREF(m); + return NULL; } - Py_DECREF(m); - return NULL; + return m; +} + +static void +track_module(PyModuleObject *m) +{ + _PyDict_EnablePerThreadRefcounting(m->md_dict); + _PyObject_SetDeferredRefcount((PyObject *)m); + PyObject_GC_Track(m); } static PyObject * new_module(PyTypeObject *mt, PyObject *args, PyObject *kws) { - PyObject *m = (PyObject *)new_module_notrack(mt); + PyModuleObject *m = new_module_notrack(mt); if (m != NULL) { - PyObject_GC_Track(m); + track_module(m); } - return m; + return (PyObject *)m; } PyObject * @@ -113,7 +129,7 @@ PyModule_NewObject(PyObject *name) return NULL; if (module_init_dict(m, m->md_dict, name, NULL) != 0) goto fail; - PyObject_GC_Track(m); + track_module(m); return (PyObject *)m; fail: @@ -171,6 +187,7 @@ _add_methods_to_object(PyObject *module, PyObject *name, PyMethodDef *functions) if (func == NULL) { return -1; } + _PyObject_SetDeferredRefcount(func); if (PyObject_SetAttrString(module, fdef->ml_name, func) != 0) { Py_DECREF(func); return -1; @@ -211,7 +228,9 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) return NULL; } name = _PyImport_ResolveNameWithPackageContext(name); - if ((m = (PyModuleObject*)PyModule_New(name)) == NULL) + + m = (PyModuleObject*)PyModule_New(name); + if (m == NULL) return NULL; if (module->m_size > 0) { @@ -237,6 +256,9 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) } } m->md_def = module; +#ifdef Py_GIL_DISABLED + m->md_gil = Py_MOD_GIL_USED; +#endif return (PyObject*)m; } @@ -249,6 +271,8 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio PyObject *m = NULL; int has_multiple_interpreters_slot = 0; void *multiple_interpreters = (void *)0; + int has_gil_slot = 0; + void *gil_slot = Py_MOD_GIL_USED; int has_execution_slots = 0; const char *name; int ret; @@ -303,6 +327,17 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio multiple_interpreters = cur_slot->value; has_multiple_interpreters_slot = 1; break; + case Py_mod_gil: + if (has_gil_slot) { + PyErr_Format( + PyExc_SystemError, + "module %s has more than one 'gil' slot", + name); + goto error; + } + gil_slot = cur_slot->value; + has_gil_slot = 1; + break; default: assert(cur_slot->slot < 0 || cur_slot->slot > _Py_mod_LAST_SLOT); PyErr_Format( @@ -362,6 +397,11 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio if (PyModule_Check(m)) { ((PyModuleObject*)m)->md_state = NULL; ((PyModuleObject*)m)->md_def = def; +#ifdef Py_GIL_DISABLED + ((PyModuleObject*)m)->md_gil = gil_slot; +#else + (void)gil_slot; +#endif } else { if (def->m_size > 0 || def->m_traverse || def->m_clear || def->m_free) { PyErr_Format( @@ -403,6 +443,19 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio return NULL; } +#ifdef Py_GIL_DISABLED +int +PyUnstable_Module_SetGIL(PyObject *module, void *gil) +{ + if (!PyModule_Check(module)) { + PyErr_BadInternalCall(); + return -1; + } + ((PyModuleObject *)module)->md_gil = gil; + return 0; +} +#endif + int PyModule_ExecDef(PyObject *module, PyModuleDef *def) { @@ -458,6 +511,7 @@ PyModule_ExecDef(PyObject *module, PyModuleDef *def) } break; case Py_mod_multiple_interpreters: + case Py_mod_gil: /* handled in PyModule_FromDefAndSpec2 */ break; default: @@ -705,35 +759,30 @@ static int module___init___impl(PyModuleObject *self, PyObject *name, PyObject *doc) /*[clinic end generated code: output=e7e721c26ce7aad7 input=57f9e177401e5e1e]*/ { - PyObject *dict = self->md_dict; - if (dict == NULL) { - dict = PyDict_New(); - if (dict == NULL) - return -1; - self->md_dict = dict; - } - if (module_init_dict(self, dict, name, doc) < 0) - return -1; - return 0; + return module_init_dict(self, self->md_dict, name, doc); } static void -module_dealloc(PyModuleObject *m) +module_dealloc(PyObject *self) { - int verbose = _Py_GetConfig()->verbose; + PyModuleObject *m = _PyModule_CAST(self); PyObject_GC_UnTrack(m); + + int verbose = _Py_GetConfig()->verbose; if (verbose && m->md_name) { PySys_FormatStderr("# destroy %U\n", m->md_name); } if (m->md_weaklist != NULL) PyObject_ClearWeakRefs((PyObject *) m); + /* bpo-39824: Don't call m_free() if m_size > 0 and md_state=NULL */ if (m->md_def && m->md_def->m_free && (m->md_def->m_size <= 0 || m->md_state != NULL)) { m->md_def->m_free(m); } + Py_XDECREF(m->md_dict); Py_XDECREF(m->md_name); if (m->md_state != NULL) @@ -742,8 +791,9 @@ module_dealloc(PyModuleObject *m) } static PyObject * -module_repr(PyModuleObject *m) +module_repr(PyObject *self) { + PyModuleObject *m = _PyModule_CAST(self); PyInterpreterState *interp = _PyInterpreterState_GET(); return _PyImport_ImportlibModuleRepr(interp, (PyObject *)m); } @@ -784,11 +834,104 @@ _PyModuleSpec_IsUninitializedSubmodule(PyObject *spec, PyObject *name) return rc; } +int +_PyModuleSpec_GetFileOrigin(PyObject *spec, PyObject **p_origin) +{ + PyObject *has_location = NULL; + int rc = PyObject_GetOptionalAttr(spec, &_Py_ID(has_location), &has_location); + if (rc <= 0) { + return rc; + } + // If origin is not a location, or doesn't exist, or is not a str, we could consider falling + // back to module.__file__. But the cases in which module.__file__ is not __spec__.origin + // are cases in which we probably shouldn't be guessing. + rc = PyObject_IsTrue(has_location); + Py_DECREF(has_location); + if (rc <= 0) { + return rc; + } + // has_location is true, so origin is a location + PyObject *origin = NULL; + rc = PyObject_GetOptionalAttr(spec, &_Py_ID(origin), &origin); + if (rc <= 0) { + return rc; + } + assert(origin != NULL); + if (!PyUnicode_Check(origin)) { + Py_DECREF(origin); + return 0; + } + *p_origin = origin; + return 1; +} + +int +_PyModule_IsPossiblyShadowing(PyObject *origin) +{ + // origin must be a unicode subtype + // Returns 1 if the module at origin could be shadowing a module of the + // same name later in the module search path. The condition we check is basically: + // root = os.path.dirname(origin.removesuffix(os.sep + "__init__.py")) + // return not sys.flags.safe_path and root == (sys.path[0] or os.getcwd()) + // Returns 0 otherwise (or if we aren't sure) + // Returns -1 if an error occurred that should be propagated + if (origin == NULL) { + return 0; + } + + // not sys.flags.safe_path + const PyConfig *config = _Py_GetConfig(); + if (config->safe_path) { + return 0; + } + + // root = os.path.dirname(origin.removesuffix(os.sep + "__init__.py")) + wchar_t root[MAXPATHLEN + 1]; + Py_ssize_t size = PyUnicode_AsWideChar(origin, root, MAXPATHLEN); + if (size < 0) { + return -1; + } + assert(size <= MAXPATHLEN); + root[size] = L'\0'; + + wchar_t *sep = wcsrchr(root, SEP); + if (sep == NULL) { + return 0; + } + // If it's a package then we need to look one directory further up + if (wcscmp(sep + 1, L"__init__.py") == 0) { + *sep = L'\0'; + sep = wcsrchr(root, SEP); + if (sep == NULL) { + return 0; + } + } + *sep = L'\0'; + + // sys.path[0] or os.getcwd() + wchar_t *sys_path_0 = config->sys_path_0; + if (!sys_path_0) { + return 0; + } + + wchar_t sys_path_0_buf[MAXPATHLEN]; + if (sys_path_0[0] == L'\0') { + // if sys.path[0] == "", treat it as if it were the current directory + if (!_Py_wgetcwd(sys_path_0_buf, MAXPATHLEN)) { + return -1; + } + sys_path_0 = sys_path_0_buf; + } + + int result = wcscmp(sys_path_0, root) == 0; + return result; +} + PyObject* _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) { // When suppress=1, this function suppresses AttributeError. - PyObject *attr, *mod_name, *getattr, *origin; + PyObject *attr, *mod_name, *getattr; attr = _PyObject_GenericGetAttrWithDict((PyObject *)m, name, NULL, suppress); if (attr) { return attr; @@ -819,81 +962,131 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) Py_DECREF(getattr); return result; } + + // The attribute was not found. We make a best effort attempt at a useful error message, + // but only if we're not suppressing AttributeError. + if (suppress == 1) { + return NULL; + } if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__name__), &mod_name) < 0) { return NULL; } - if (mod_name && PyUnicode_Check(mod_name)) { - PyObject *spec; - if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__spec__), &spec) < 0) { - Py_DECREF(mod_name); - return NULL; + if (!mod_name || !PyUnicode_Check(mod_name)) { + Py_XDECREF(mod_name); + PyErr_Format(PyExc_AttributeError, + "module has no attribute '%U'", name); + return NULL; + } + PyObject *spec; + if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__spec__), &spec) < 0) { + Py_DECREF(mod_name); + return NULL; + } + if (spec == NULL) { + PyErr_Format(PyExc_AttributeError, + "module '%U' has no attribute '%U'", + mod_name, name); + Py_DECREF(mod_name); + return NULL; + } + + PyObject *origin = NULL; + if (_PyModuleSpec_GetFileOrigin(spec, &origin) < 0) { + goto done; + } + + int is_possibly_shadowing = _PyModule_IsPossiblyShadowing(origin); + if (is_possibly_shadowing < 0) { + goto done; + } + int is_possibly_shadowing_stdlib = 0; + if (is_possibly_shadowing) { + PyObject *stdlib_modules = PySys_GetObject("stdlib_module_names"); + if (stdlib_modules && PyAnySet_Check(stdlib_modules)) { + is_possibly_shadowing_stdlib = PySet_Contains(stdlib_modules, mod_name); + if (is_possibly_shadowing_stdlib < 0) { + goto done; + } + } + } + + if (is_possibly_shadowing_stdlib) { + assert(origin); + PyErr_Format(PyExc_AttributeError, + "module '%U' has no attribute '%U' " + "(consider renaming '%U' since it has the same " + "name as the standard library module named '%U' " + "and prevents importing that standard library module)", + mod_name, name, origin, mod_name); + } + else { + int rc = _PyModuleSpec_IsInitializing(spec); + if (rc < 0) { + goto done; } - if (suppress != 1) { - int rc = _PyModuleSpec_IsInitializing(spec); + else if (rc > 0) { + if (is_possibly_shadowing) { + assert(origin); + // For non-stdlib modules, only mention the possibility of + // shadowing if the module is being initialized. + PyErr_Format(PyExc_AttributeError, + "module '%U' has no attribute '%U' " + "(consider renaming '%U' if it has the same name " + "as a library you intended to import)", + mod_name, name, origin); + } + else if (origin) { + PyErr_Format(PyExc_AttributeError, + "partially initialized " + "module '%U' from '%U' has no attribute '%U' " + "(most likely due to a circular import)", + mod_name, origin, name); + } + else { + PyErr_Format(PyExc_AttributeError, + "partially initialized " + "module '%U' has no attribute '%U' " + "(most likely due to a circular import)", + mod_name, name); + } + } + else { + assert(rc == 0); + rc = _PyModuleSpec_IsUninitializedSubmodule(spec, name); if (rc > 0) { - int valid_spec = PyObject_GetOptionalAttr(spec, &_Py_ID(origin), &origin); - if (valid_spec == -1) { - Py_XDECREF(spec); - Py_DECREF(mod_name); - return NULL; - } - if (valid_spec == 1 && !PyUnicode_Check(origin)) { - valid_spec = 0; - Py_DECREF(origin); - } - if (valid_spec == 1) { - PyErr_Format(PyExc_AttributeError, - "partially initialized " - "module '%U' from '%U' has no attribute '%U' " - "(most likely due to a circular import)", - mod_name, origin, name); - Py_DECREF(origin); - } - else { - PyErr_Format(PyExc_AttributeError, - "partially initialized " - "module '%U' has no attribute '%U' " - "(most likely due to a circular import)", - mod_name, name); - } + PyErr_Format(PyExc_AttributeError, + "cannot access submodule '%U' of module '%U' " + "(most likely due to a circular import)", + name, mod_name); } else if (rc == 0) { - rc = _PyModuleSpec_IsUninitializedSubmodule(spec, name); - if (rc > 0) { - PyErr_Format(PyExc_AttributeError, - "cannot access submodule '%U' of module '%U' " - "(most likely due to a circular import)", - name, mod_name); - } - else if (rc == 0) { - PyErr_Format(PyExc_AttributeError, - "module '%U' has no attribute '%U'", - mod_name, name); - } + PyErr_Format(PyExc_AttributeError, + "module '%U' has no attribute '%U'", + mod_name, name); } } - Py_XDECREF(spec); - Py_DECREF(mod_name); - return NULL; - } - Py_XDECREF(mod_name); - if (suppress != 1) { - PyErr_Format(PyExc_AttributeError, - "module has no attribute '%U'", name); } + +done: + Py_XDECREF(origin); + Py_DECREF(spec); + Py_DECREF(mod_name); return NULL; } PyObject* -_Py_module_getattro(PyModuleObject *m, PyObject *name) +_Py_module_getattro(PyObject *self, PyObject *name) { + PyModuleObject *m = _PyModule_CAST(self); return _Py_module_getattro_impl(m, name, 0); } static int -module_traverse(PyModuleObject *m, visitproc visit, void *arg) +module_traverse(PyObject *self, visitproc visit, void *arg) { + PyModuleObject *m = _PyModule_CAST(self); + /* bpo-39824: Don't call m_traverse() if m_size > 0 and md_state=NULL */ if (m->md_def && m->md_def->m_traverse && (m->md_def->m_size <= 0 || m->md_state != NULL)) @@ -902,13 +1095,16 @@ module_traverse(PyModuleObject *m, visitproc visit, void *arg) if (res) return res; } + Py_VISIT(m->md_dict); return 0; } static int -module_clear(PyModuleObject *m) +module_clear(PyObject *self) { + PyModuleObject *m = _PyModule_CAST(self); + /* bpo-39824: Don't call m_clear() if m_size > 0 and md_state=NULL */ if (m->md_def && m->md_def->m_clear && (m->md_def->m_size <= 0 || m->md_state != NULL)) @@ -958,7 +1154,7 @@ static PyMethodDef module_methods[] = { }; static PyObject * -module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) +module_get_dict(PyModuleObject *m) { PyObject *dict = PyObject_GetAttr((PyObject *)m, &_Py_ID(__dict__)); if (dict == NULL) { @@ -969,10 +1165,103 @@ module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) Py_DECREF(dict); return NULL; } + return dict; +} + +static PyObject * +module_get_annotate(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyModuleObject *m = _PyModule_CAST(self); + + PyObject *dict = module_get_dict(m); + if (dict == NULL) { + return NULL; + } + + PyObject *annotate; + if (PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate) == 0) { + annotate = Py_None; + if (PyDict_SetItem(dict, &_Py_ID(__annotate__), annotate) == -1) { + Py_CLEAR(annotate); + } + } + Py_DECREF(dict); + return annotate; +} + +static int +module_set_annotate(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) +{ + PyModuleObject *m = _PyModule_CAST(self); + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete __annotate__ attribute"); + return -1; + } + + PyObject *dict = module_get_dict(m); + if (dict == NULL) { + return -1; + } + + if (!Py_IsNone(value) && !PyCallable_Check(value)) { + PyErr_SetString(PyExc_TypeError, "__annotate__ must be callable or None"); + Py_DECREF(dict); + return -1; + } + + if (PyDict_SetItem(dict, &_Py_ID(__annotate__), value) == -1) { + Py_DECREF(dict); + return -1; + } + if (!Py_IsNone(value)) { + if (PyDict_Pop(dict, &_Py_ID(__annotations__), NULL) == -1) { + Py_DECREF(dict); + return -1; + } + } + Py_DECREF(dict); + return 0; +} + +static PyObject * +module_get_annotations(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyModuleObject *m = _PyModule_CAST(self); + + PyObject *dict = module_get_dict(m); + if (dict == NULL) { + return NULL; + } PyObject *annotations; if (PyDict_GetItemRef(dict, &_Py_ID(__annotations__), &annotations) == 0) { - annotations = PyDict_New(); + PyObject *annotate; + int annotate_result = PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate); + if (annotate_result < 0) { + Py_DECREF(dict); + return NULL; + } + if (annotate_result == 1 && PyCallable_Check(annotate)) { + PyObject *one = _PyLong_GetOne(); + annotations = _PyObject_CallOneArg(annotate, one); + if (annotations == NULL) { + Py_DECREF(annotate); + Py_DECREF(dict); + return NULL; + } + if (!PyDict_Check(annotations)) { + PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", + Py_TYPE(annotations)->tp_name); + Py_DECREF(annotate); + Py_DECREF(annotations); + Py_DECREF(dict); + return NULL; + } + } + else { + annotations = PyDict_New(); + } + Py_XDECREF(annotate); if (annotations) { int result = PyDict_SetItem( dict, &_Py_ID(__annotations__), annotations); @@ -986,38 +1275,43 @@ module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) } static int -module_set_annotations(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignored)) +module_set_annotations(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) { - int ret = -1; - PyObject *dict = PyObject_GetAttr((PyObject *)m, &_Py_ID(__dict__)); + PyModuleObject *m = _PyModule_CAST(self); + + PyObject *dict = module_get_dict(m); if (dict == NULL) { return -1; } - if (!PyDict_Check(dict)) { - PyErr_Format(PyExc_TypeError, ".__dict__ is not a dictionary"); - goto exit; - } + int ret = -1; if (value != NULL) { /* set */ ret = PyDict_SetItem(dict, &_Py_ID(__annotations__), value); } else { /* delete */ - ret = PyDict_DelItem(dict, &_Py_ID(__annotations__)); - if (ret < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { - PyErr_SetString(PyExc_AttributeError, "__annotations__"); + ret = PyDict_Pop(dict, &_Py_ID(__annotations__), NULL); + if (ret == 0) { + PyErr_SetObject(PyExc_AttributeError, &_Py_ID(__annotations__)); + ret = -1; } + else if (ret > 0) { + ret = 0; + } + } + if (ret == 0 && PyDict_Pop(dict, &_Py_ID(__annotate__), NULL) < 0) { + ret = -1; } -exit: Py_DECREF(dict); return ret; } static PyGetSetDef module_getsets[] = { - {"__annotations__", (getter)module_get_annotations, (setter)module_set_annotations}, + {"__annotations__", module_get_annotations, module_set_annotations}, + {"__annotate__", module_get_annotate, module_set_annotate}, {NULL} }; @@ -1026,26 +1320,26 @@ PyTypeObject PyModule_Type = { "module", /* tp_name */ sizeof(PyModuleObject), /* tp_basicsize */ 0, /* tp_itemsize */ - (destructor)module_dealloc, /* tp_dealloc */ + module_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)module_repr, /* tp_repr */ + module_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ - (getattrofunc)_Py_module_getattro, /* tp_getattro */ + _Py_module_getattro, /* tp_getattro */ PyObject_GenericSetAttr, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /* tp_flags */ module___init____doc__, /* tp_doc */ - (traverseproc)module_traverse, /* tp_traverse */ - (inquiry)module_clear, /* tp_clear */ + module_traverse, /* tp_traverse */ + module_clear, /* tp_clear */ 0, /* tp_richcompare */ offsetof(PyModuleObject, md_weaklist), /* tp_weaklistoffset */ 0, /* tp_iter */ diff --git a/Objects/namespaceobject.c b/Objects/namespaceobject.c index b975bcfeea2cdf..4ef3bd92f5a569 100644 --- a/Objects/namespaceobject.c +++ b/Objects/namespaceobject.c @@ -43,10 +43,28 @@ namespace_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static int namespace_init(_PyNamespaceObject *ns, PyObject *args, PyObject *kwds) { - if (PyTuple_GET_SIZE(args) != 0) { - PyErr_Format(PyExc_TypeError, "no positional arguments expected"); + PyObject *arg = NULL; + if (!PyArg_UnpackTuple(args, _PyType_Name(Py_TYPE(ns)), 0, 1, &arg)) { return -1; } + if (arg != NULL) { + PyObject *dict; + if (PyDict_CheckExact(arg)) { + dict = Py_NewRef(arg); + } + else { + dict = PyObject_CallOneArg((PyObject *)&PyDict_Type, arg); + if (dict == NULL) { + return -1; + } + } + int err = (!PyArg_ValidateKeywordArguments(dict) || + PyDict_Update(ns->ns_dict, dict) < 0); + Py_DECREF(dict); + if (err) { + return -1; + } + } if (kwds == NULL) { return 0; } @@ -123,6 +141,10 @@ namespace_repr(PyObject *ns) goto error; } + if (PyErr_Occurred()) { + goto error; + } + separator = PyUnicode_FromString(", "); if (separator == NULL) goto error; @@ -219,15 +241,17 @@ namespace_replace(PyObject *self, PyObject *args, PyObject *kwargs) static PyMethodDef namespace_methods[] = { {"__reduce__", (PyCFunction)namespace_reduce, METH_NOARGS, namespace_reduce__doc__}, - {"__replace__", _PyCFunction_CAST(namespace_replace), METH_VARARGS|METH_KEYWORDS, NULL}, + {"__replace__", _PyCFunction_CAST(namespace_replace), METH_VARARGS|METH_KEYWORDS, + PyDoc_STR("__replace__($self, /, **changes)\n--\n\n" + "Return a copy of the namespace object with new values for the specified attributes.")}, {NULL, NULL} // sentinel }; PyDoc_STRVAR(namespace_doc, -"A simple attribute-based namespace.\n\ -\n\ -SimpleNamespace(**kwargs)"); +"SimpleNamespace(mapping_or_iterable=(), /, **kwargs)\n\ +--\n\n\ +A simple attribute-based namespace."); PyTypeObject _PyNamespace_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) diff --git a/Objects/object.c b/Objects/object.c index df14fe0c6fbfec..fdff16138201a0 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -6,15 +6,20 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _Py_EnterRecursiveCallTstate() #include "pycore_context.h" // _PyContextTokenMissing_Type +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION, Py_END_CRITICAL_SECTION #include "pycore_descrobject.h" // _PyMethodWrapper_Type #include "pycore_dict.h" // _PyObject_MakeDictFromInstanceAttributes() #include "pycore_floatobject.h" // _PyFloat_DebugMallocStats() +#include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() +#include "pycore_instruction_sequence.h" // _PyInstructionSequence_Type #include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_memoryobject.h" // _PyManagedBuffer_Type #include "pycore_namespace.h" // _PyNamespace_Type #include "pycore_object.h" // PyAPI_DATA() _Py_SwappedOp definition -#include "pycore_optimizer.h" // _PyUOpExecutor_Type, _PyUOpOptimizer_Type, ... +#include "pycore_object_state.h" // struct _reftracer_runtime_state +#include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_optimizer.h" // _PyUOpExecutor_Type, ... #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -23,7 +28,6 @@ #include "pycore_typevarobject.h" // _PyTypeAlias_Type, _Py_initialize_generic #include "pycore_unionobject.h" // _PyUnion_Type -#include "interpreteridobject.h" // _PyInterpreterID_Type #ifdef Py_LIMITED_API // Prevent recursive call _Py_IncRef() <=> Py_INCREF() @@ -74,21 +78,16 @@ get_legacy_reftotal(void) interp->object_state.reftotal static inline void -reftotal_increment(PyInterpreterState *interp) +reftotal_add(PyThreadState *tstate, Py_ssize_t n) { - REFTOTAL(interp)++; -} - -static inline void -reftotal_decrement(PyInterpreterState *interp) -{ - REFTOTAL(interp)--; -} - -static inline void -reftotal_add(PyInterpreterState *interp, Py_ssize_t n) -{ - REFTOTAL(interp) += n; +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + // relaxed store to avoid data race with read in get_reftotal() + Py_ssize_t reftotal = tstate_impl->reftotal + n; + _Py_atomic_store_ssize_relaxed(&tstate_impl->reftotal, reftotal); +#else + REFTOTAL(tstate->interp) += n; +#endif } static inline Py_ssize_t get_global_reftotal(_PyRuntimeState *); @@ -118,7 +117,15 @@ get_reftotal(PyInterpreterState *interp) { /* For a single interpreter, we ignore the legacy _Py_RefTotal, since we can't determine which interpreter updated it. */ - return REFTOTAL(interp); + Py_ssize_t total = REFTOTAL(interp); +#ifdef Py_GIL_DISABLED + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, p) { + /* This may race with other threads modifications to their reftotal */ + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)p; + total += _Py_atomic_load_ssize_relaxed(&tstate_impl->reftotal); + } +#endif + return total; } static inline Py_ssize_t @@ -130,7 +137,7 @@ get_global_reftotal(_PyRuntimeState *runtime) HEAD_LOCK(&_PyRuntime); PyInterpreterState *interp = PyInterpreterState_Head(); for (; interp != NULL; interp = PyInterpreterState_Next(interp)) { - total += REFTOTAL(interp); + total += get_reftotal(interp); } HEAD_UNLOCK(&_PyRuntime); @@ -164,6 +171,48 @@ _PyDebug_PrintTotalRefs(void) { #define REFCHAIN(interp) interp->object_state.refchain #define REFCHAIN_VALUE ((void*)(uintptr_t)1) +static inline int +has_own_refchain(PyInterpreterState *interp) +{ + if (interp->feature_flags & Py_RTFLAGS_USE_MAIN_OBMALLOC) { + return (_Py_IsMainInterpreter(interp) + || _PyInterpreterState_Main() == NULL); + } + return 1; +} + +static int +refchain_init(PyInterpreterState *interp) +{ + if (!has_own_refchain(interp)) { + // Legacy subinterpreters share a refchain with the main interpreter. + REFCHAIN(interp) = REFCHAIN(_PyInterpreterState_Main()); + return 0; + } + _Py_hashtable_allocator_t alloc = { + // Don't use default PyMem_Malloc() and PyMem_Free() which + // require the caller to hold the GIL. + .malloc = PyMem_RawMalloc, + .free = PyMem_RawFree, + }; + REFCHAIN(interp) = _Py_hashtable_new_full( + _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct, + NULL, NULL, &alloc); + if (REFCHAIN(interp) == NULL) { + return -1; + } + return 0; +} + +static void +refchain_fini(PyInterpreterState *interp) +{ + if (has_own_refchain(interp) && REFCHAIN(interp) != NULL) { + _Py_hashtable_destroy(REFCHAIN(interp)); + } + REFCHAIN(interp) = NULL; +} + bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj) { @@ -223,32 +272,32 @@ _Py_NegativeRefcount(const char *filename, int lineno, PyObject *op) void _Py_INCREF_IncRefTotal(void) { - reftotal_increment(_PyInterpreterState_GET()); + reftotal_add(_PyThreadState_GET(), 1); } /* This is used strictly by Py_DECREF(). */ void _Py_DECREF_DecRefTotal(void) { - reftotal_decrement(_PyInterpreterState_GET()); + reftotal_add(_PyThreadState_GET(), -1); } void -_Py_IncRefTotal(PyInterpreterState *interp) +_Py_IncRefTotal(PyThreadState *tstate) { - reftotal_increment(interp); + reftotal_add(tstate, 1); } void -_Py_DecRefTotal(PyInterpreterState *interp) +_Py_DecRefTotal(PyThreadState *tstate) { - reftotal_decrement(interp); + reftotal_add(tstate, -1); } void -_Py_AddRefTotal(PyInterpreterState *interp, Py_ssize_t n) +_Py_AddRefTotal(PyThreadState *tstate, Py_ssize_t n) { - reftotal_add(interp, n); + reftotal_add(tstate, n); } /* This includes the legacy total @@ -268,7 +317,10 @@ _Py_GetLegacyRefTotal(void) Py_ssize_t _PyInterpreterState_GetRefTotal(PyInterpreterState *interp) { - return get_reftotal(interp); + HEAD_LOCK(&_PyRuntime); + Py_ssize_t total = get_reftotal(interp); + HEAD_UNLOCK(&_PyRuntime); + return total; } #endif /* Py_REF_DEBUG */ @@ -310,8 +362,10 @@ is_dead(PyObject *o) } # endif -void -_Py_DecRefSharedDebug(PyObject *o, const char *filename, int lineno) +// Decrement the shared reference count of an object. Return 1 if the object +// is dead and should be deallocated, 0 otherwise. +static int +_Py_DecRefSharedIsDead(PyObject *o, const char *filename, int lineno) { // Should we queue the object for the owning thread to merge? int should_queue; @@ -346,12 +400,21 @@ _Py_DecRefSharedDebug(PyObject *o, const char *filename, int lineno) if (should_queue) { #ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif _Py_brc_queue_object(o); } else if (new_shared == _Py_REF_MERGED) { // refcount is zero AND merged + return 1; + } + return 0; +} + +void +_Py_DecRefSharedDebug(PyObject *o, const char *filename, int lineno) +{ + if (_Py_DecRefSharedIsDead(o, filename, lineno)) { _Py_Dealloc(o); } } @@ -367,14 +430,18 @@ _Py_MergeZeroLocalRefcount(PyObject *op) { assert(op->ob_ref_local == 0); - _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); - Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); + Py_ssize_t shared = _Py_atomic_load_ssize_acquire(&op->ob_ref_shared); if (shared == 0) { // Fast-path: shared refcount is zero (including flags) _Py_Dealloc(op); return; } + // gh-121794: This must be before the store to `ob_ref_shared` (gh-119999), + // but should outside the fast-path to maintain the invariant that + // a zero `ob_tid` implies a merged refcount. + _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); + // Slow-path: atomically set the flags (low two bits) to _Py_REF_MERGED. Py_ssize_t new_shared; do { @@ -393,26 +460,49 @@ Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra) { assert(!_Py_IsImmortal(op)); + +#ifdef Py_REF_DEBUG + _Py_AddRefTotal(_PyThreadState_GET(), extra); +#endif + + // gh-119999: Write to ob_ref_local and ob_tid before merging the refcount. + Py_ssize_t local = (Py_ssize_t)op->ob_ref_local; + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 0); + _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); + Py_ssize_t refcnt; Py_ssize_t new_shared; Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); do { refcnt = Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, shared, _Py_REF_SHARED_SHIFT); - refcnt += (Py_ssize_t)op->ob_ref_local; + refcnt += local; refcnt += extra; new_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED); } while (!_Py_atomic_compare_exchange_ssize(&op->ob_ref_shared, &shared, new_shared)); - -#ifdef Py_REF_DEBUG - _Py_AddRefTotal(_PyInterpreterState_GET(), extra); -#endif - - _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 0); - _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); return refcnt; } + +// The more complicated "slow" path for undoing the resurrection of an object. +int +_PyObject_ResurrectEndSlow(PyObject *op) +{ + if (_Py_IsImmortal(op)) { + return 1; + } + if (_Py_IsOwnedByCurrentThread(op)) { + // If the object is owned by the current thread, give up ownership and + // merge the refcount. This isn't necessary in all cases, but it + // simplifies the implementation. + Py_ssize_t refcount = _Py_ExplicitMergeRefcount(op, -1); + return refcount != 0; + } + int is_dead = _Py_DecRefSharedIsDead(op, NULL, 0); + return !is_dead; +} + + #endif /* Py_GIL_DISABLED */ @@ -491,7 +581,7 @@ PyObject_CallFinalizerFromDealloc(PyObject *self) } /* Temporarily resurrect the object. */ - Py_SET_REFCNT(self, 1); + _PyObject_ResurrectStart(self); PyObject_CallFinalizer(self); @@ -501,8 +591,7 @@ PyObject_CallFinalizerFromDealloc(PyObject *self) /* Undo the temporary resurrection; can't use DECREF here, it would * cause a recursive call. */ - Py_SET_REFCNT(self, Py_REFCNT(self) - 1); - if (Py_REFCNT(self) == 0) { + if (!_PyObject_ResurrectEnd(self)) { return 0; /* this is the normal path out */ } @@ -520,6 +609,7 @@ int PyObject_Print(PyObject *op, FILE *fp, int flags) { int ret = 0; + int write_error = 0; if (PyErr_CheckSignals()) return -1; #ifdef USE_STACKCHECK @@ -558,14 +648,20 @@ PyObject_Print(PyObject *op, FILE *fp, int flags) ret = -1; } else { - fwrite(t, 1, len, fp); + /* Versions of Android and OpenBSD from before 2023 fail to + set the `ferror` indicator when writing to a read-only + stream, so we need to check the return value. + (https://github.com/openbsd/src/commit/fc99cf9338942ecd9adc94ea08bf6188f0428c15) */ + if (fwrite(t, 1, len, fp) != (size_t)len) { + write_error = 1; + } } Py_DECREF(s); } } } if (ret == 0) { - if (ferror(fp)) { + if (write_error || ferror(fp)) { PyErr_SetFromErrno(PyExc_OSError); clearerr(fp); ret = -1; @@ -793,20 +889,57 @@ PyObject_Bytes(PyObject *v) return PyBytes_FromObject(v); } +static void +clear_freelist(struct _Py_freelist *freelist, int is_finalization, + freefunc dofree) +{ + void *ptr; + while ((ptr = _PyFreeList_PopNoStats(freelist)) != NULL) { + dofree(ptr); + } + assert(freelist->size == 0 || freelist->size == -1); + assert(freelist->freelist == NULL); + if (is_finalization) { + freelist->size = -1; + } +} + +static void +free_object(void *obj) +{ + PyObject *op = (PyObject *)obj; + PyTypeObject *tp = Py_TYPE(op); + tp->tp_free(op); + Py_DECREF(tp); +} + void -_PyObject_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization) +_PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization) { // In the free-threaded build, freelists are per-PyThreadState and cleared in PyThreadState_Clear() // In the default build, freelists are per-interpreter and cleared in finalize_interp_types() - _PyFloat_ClearFreeList(freelists, is_finalization); - _PyTuple_ClearFreeList(freelists, is_finalization); - _PyList_ClearFreeList(freelists, is_finalization); - _PyDict_ClearFreeList(freelists, is_finalization); - _PyContext_ClearFreeList(freelists, is_finalization); - _PyAsyncGen_ClearFreeLists(freelists, is_finalization); - // Only be cleared if is_finalization is true. - _PyObjectStackChunk_ClearFreeList(freelists, is_finalization); - _PySlice_ClearFreeList(freelists, is_finalization); + clear_freelist(&freelists->floats, is_finalization, free_object); + for (Py_ssize_t i = 0; i < PyTuple_MAXSAVESIZE; i++) { + clear_freelist(&freelists->tuples[i], is_finalization, free_object); + } + clear_freelist(&freelists->lists, is_finalization, free_object); + clear_freelist(&freelists->list_iters, is_finalization, free_object); + clear_freelist(&freelists->tuple_iters, is_finalization, free_object); + clear_freelist(&freelists->dicts, is_finalization, free_object); + clear_freelist(&freelists->dictkeys, is_finalization, PyMem_Free); + clear_freelist(&freelists->slices, is_finalization, free_object); + clear_freelist(&freelists->contexts, is_finalization, free_object); + clear_freelist(&freelists->async_gens, is_finalization, free_object); + clear_freelist(&freelists->async_gen_asends, is_finalization, free_object); + clear_freelist(&freelists->futureiters, is_finalization, free_object); + if (is_finalization) { + // Only clear object stack chunks during finalization. We use object + // stacks during GC, so emptying the free-list is counterproductive. + clear_freelist(&freelists->object_stack_chunks, 1, PyMem_RawFree); + } + clear_freelist(&freelists->unicode_writers, is_finalization, PyMem_Free); + clear_freelist(&freelists->ints, is_finalization, free_object); + clear_freelist(&freelists->pymethodobjects, is_finalization, free_object); } /* @@ -1114,18 +1247,7 @@ _PyObject_GetAttrId(PyObject *v, _Py_Identifier *name) } int -_PyObject_SetAttrId(PyObject *v, _Py_Identifier *name, PyObject *w) -{ - int result; - PyObject *oname = _PyUnicode_FromId(name); /* borrowed */ - if (!oname) - return -1; - result = PyObject_SetAttr(v, oname, w); - return result; -} - -static inline int -set_attribute_error_context(PyObject* v, PyObject* name) +_PyObject_SetAttributeErrorContext(PyObject* v, PyObject* name) { assert(PyErr_Occurred()); if (!PyErr_ExceptionMatches(PyExc_AttributeError)){ @@ -1180,7 +1302,7 @@ PyObject_GetAttr(PyObject *v, PyObject *name) } if (result == NULL) { - set_attribute_error_context(v, name); + _PyObject_SetAttributeErrorContext(v, name); } return result; } @@ -1209,9 +1331,9 @@ PyObject_GetOptionalAttr(PyObject *v, PyObject *name, PyObject **result) return 0; } if (tp->tp_getattro == _Py_type_getattro) { - int supress_missing_attribute_exception = 0; - *result = _Py_type_getattro_impl((PyTypeObject*)v, name, &supress_missing_attribute_exception); - if (supress_missing_attribute_exception) { + int suppress_missing_attribute_exception = 0; + *result = _Py_type_getattro_impl((PyTypeObject*)v, name, &suppress_missing_attribute_exception); + if (suppress_missing_attribute_exception) { // return 0 without having to clear the exception return 0; } @@ -1315,7 +1437,8 @@ PyObject_SetAttr(PyObject *v, PyObject *name, PyObject *value) } Py_INCREF(name); - PyUnicode_InternInPlace(&name); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternMortal(interp, &name); if (tp->tp_setattro != NULL) { err = (*tp->tp_setattro)(v, name, value); Py_DECREF(name); @@ -1397,16 +1520,15 @@ _PyObject_GetDictPtr(PyObject *obj) if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { return _PyObject_ComputedDictPointer(obj); } - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyObject *dict = _PyObject_MakeDictFromInstanceAttributes(obj, _PyDictOrValues_GetValues(*dorv_ptr)); + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL && Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + dict = _PyObject_MaterializeManagedDict(obj); if (dict == NULL) { PyErr_Clear(); return NULL; } - dorv_ptr->dict = dict; } - return &dorv_ptr->dict; + return (PyObject **)&_PyObject_ManagedDictPointer(obj)->dict; } PyObject * @@ -1459,13 +1581,13 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) return 0; } - PyObject *descr = _PyType_Lookup(tp, name); + PyObject *descr = _PyType_LookupRef(tp, name); descrgetfunc f = NULL; if (descr != NULL) { - Py_INCREF(descr); if (_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)) { meth_found = 1; - } else { + } + else { f = Py_TYPE(descr)->tp_descr_get; if (f != NULL && PyDescr_IsData(descr)) { *method = f(descr, obj, (PyObject *)Py_TYPE(obj)); @@ -1474,22 +1596,18 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) } } } - PyObject *dict; - if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues* dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - PyObject *attr = _PyObject_GetInstanceAttribute(obj, values, name); - if (attr != NULL) { - *method = attr; - Py_XDECREF(descr); - return 0; - } - dict = NULL; - } - else { - dict = dorv_ptr->dict; + PyObject *dict, *attr; + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + _PyObject_TryGetInstanceAttribute(obj, name, &attr)) { + if (attr != NULL) { + *method = attr; + Py_XDECREF(descr); + return 0; } + dict = NULL; + } + else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { + dict = (PyObject *)_PyObject_GetManagedDict(obj); } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); @@ -1532,7 +1650,7 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) "'%.100s' object has no attribute '%U'", tp->tp_name, name); - set_attribute_error_context(obj, name); + _PyObject_SetAttributeErrorContext(obj, name); return 0; } @@ -1566,11 +1684,10 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, goto done; } - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); f = NULL; if (descr != NULL) { - Py_INCREF(descr); f = Py_TYPE(descr)->tp_descr_get; if (f != NULL && PyDescr_IsData(descr)) { res = f(descr, obj, (PyObject *)Py_TYPE(obj)); @@ -1582,33 +1699,32 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, } } if (dict == NULL) { - if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues* dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - if (PyUnicode_CheckExact(name)) { - res = _PyObject_GetInstanceAttribute(obj, values, name); - if (res != NULL) { - goto done; - } - } - else { - dict = _PyObject_MakeDictFromInstanceAttributes(obj, values); - if (dict == NULL) { - res = NULL; - goto done; - } - dorv_ptr->dict = dict; + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES)) { + if (PyUnicode_CheckExact(name) && + _PyObject_TryGetInstanceAttribute(obj, name, &res)) { + if (res != NULL) { + goto done; } } else { - dict = _PyDictOrValues_GetDict(*dorv_ptr); + dict = (PyObject *)_PyObject_MaterializeManagedDict(obj); + if (dict == NULL) { + res = NULL; + goto done; + } } } + else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { + dict = (PyObject *)_PyObject_GetManagedDict(obj); + } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); if (dictptr) { +#ifdef Py_GIL_DISABLED + dict = _Py_atomic_load_ptr_acquire(dictptr); +#else dict = *dictptr; +#endif } } } @@ -1649,7 +1765,7 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, "'%.100s' object has no attribute '%U'", tp->tp_name, name); - set_attribute_error_context(obj, name); + _PyObject_SetAttributeErrorContext(obj, name); } done: Py_XDECREF(descr); @@ -1672,6 +1788,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, descrsetfunc f; int res = -1; + assert(!PyType_IsSubtype(tp, &PyType_Type)); if (!PyUnicode_Check(name)){ PyErr_Format(PyExc_TypeError, "attribute name must be string, not '%.200s'", @@ -1685,10 +1802,9 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, Py_INCREF(name); Py_INCREF(tp); - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); if (descr != NULL) { - Py_INCREF(descr); f = Py_TYPE(descr)->tp_descr_set; if (f != NULL) { res = f(descr, obj, value); @@ -1698,22 +1814,15 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, if (dict == NULL) { PyObject **dictptr; + + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES)) { + res = _PyObject_StoreInstanceAttribute(obj, name, value); + goto error_check; + } + if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - res = _PyObject_StoreInstanceAttribute( - obj, _PyDictOrValues_GetValues(*dorv_ptr), name, value); - goto error_check; - } - dictptr = &dorv_ptr->dict; - if (*dictptr == NULL) { - if (_PyObject_InitInlineValues(obj, tp) < 0) { - goto done; - } - res = _PyObject_StoreInstanceAttribute( - obj, _PyDictOrValues_GetValues(*dorv_ptr), name, value); - goto error_check; - } + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj); + dictptr = (PyObject **)&managed_dict->dict; } else { dictptr = _PyObject_ComputedDictPointer(obj); @@ -1731,7 +1840,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, "'%.100s' object has no attribute '%U'", tp->tp_name, name); } - set_attribute_error_context(obj, name); + _PyObject_SetAttributeErrorContext(obj, name); } else { PyErr_Format(PyExc_AttributeError, @@ -1741,7 +1850,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, goto done; } else { - res = _PyObjectDict_SetItem(tp, dictptr, name, value); + res = _PyObjectDict_SetItem(tp, obj, dictptr, name, value); } } else { @@ -1754,17 +1863,10 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, } error_check: if (res < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { - if (PyType_IsSubtype(tp, &PyType_Type)) { - PyErr_Format(PyExc_AttributeError, - "type object '%.50s' has no attribute '%U'", - ((PyTypeObject*)obj)->tp_name, name); - } - else { - PyErr_Format(PyExc_AttributeError, - "'%.100s' object has no attribute '%U'", - tp->tp_name, name); - } - set_attribute_error_context(obj, name); + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute '%U'", + tp->tp_name, name); + _PyObject_SetAttributeErrorContext(obj, name); } done: Py_XDECREF(descr); @@ -1784,9 +1886,9 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) { PyObject **dictptr = _PyObject_GetDictPtr(obj); if (dictptr == NULL) { - if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_MANAGED_DICT) && - _PyDictOrValues_IsValues(*_PyObject_DictOrValuesPointer(obj))) - { + if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_INLINE_VALUES) && + _PyObject_GetManagedDict(obj) == NULL + ) { /* Was unable to convert to dict */ PyErr_NoMemory(); } @@ -1806,7 +1908,9 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) "not a '%.200s'", Py_TYPE(value)->tp_name); return -1; } + Py_BEGIN_CRITICAL_SECTION(obj); Py_XSETREF(*dictptr, Py_NewRef(value)); + Py_END_CRITICAL_SECTION(); return 0; } @@ -2014,6 +2118,11 @@ static PyNumberMethods none_as_number = { 0, /* nb_index */ }; +PyDoc_STRVAR(none_doc, +"NoneType()\n" +"--\n\n" +"The type of the None singleton."); + PyTypeObject _PyNone_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "NoneType", @@ -2035,7 +2144,7 @@ PyTypeObject _PyNone_Type = { 0, /*tp_setattro */ 0, /*tp_as_buffer */ Py_TPFLAGS_DEFAULT, /*tp_flags */ - 0, /*tp_doc */ + none_doc, /*tp_doc */ 0, /*tp_traverse */ 0, /*tp_clear */ _Py_BaseObject_RichCompare, /*tp_richcompare */ @@ -2100,19 +2209,20 @@ notimplemented_dealloc(PyObject *notimplemented) static int notimplemented_bool(PyObject *v) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "NotImplemented should not be used in a boolean context", - 1) < 0) - { - return -1; - } - return 1; + PyErr_SetString(PyExc_TypeError, + "NotImplemented should not be used in a boolean context"); + return -1; } static PyNumberMethods notimplemented_as_number = { .nb_bool = notimplemented_bool, }; +PyDoc_STRVAR(notimplemented_doc, +"NotImplementedType()\n" +"--\n\n" +"The type of the NotImplemented singleton."); + PyTypeObject _PyNotImplemented_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "NotImplementedType", @@ -2134,7 +2244,7 @@ PyTypeObject _PyNotImplemented_Type = { 0, /*tp_setattro */ 0, /*tp_as_buffer */ Py_TPFLAGS_DEFAULT, /*tp_flags */ - 0, /*tp_doc */ + notimplemented_doc, /*tp_doc */ 0, /*tp_traverse */ 0, /*tp_clear */ 0, /*tp_richcompare */ @@ -2161,16 +2271,7 @@ PyStatus _PyObject_InitState(PyInterpreterState *interp) { #ifdef Py_TRACE_REFS - _Py_hashtable_allocator_t alloc = { - // Don't use default PyMem_Malloc() and PyMem_Free() which - // require the caller to hold the GIL. - .malloc = PyMem_RawMalloc, - .free = PyMem_RawFree, - }; - REFCHAIN(interp) = _Py_hashtable_new_full( - _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct, - NULL, NULL, &alloc); - if (REFCHAIN(interp) == NULL) { + if (refchain_init(interp) < 0) { return _PyStatus_NO_MEMORY(); } #endif @@ -2181,8 +2282,7 @@ void _PyObject_FiniState(PyInterpreterState *interp) { #ifdef Py_TRACE_REFS - _Py_hashtable_destroy(REFCHAIN(interp)); - REFCHAIN(interp) = NULL; + refchain_fini(interp); #endif } @@ -2234,12 +2334,12 @@ static PyTypeObject* static_types[] = { &PyFilter_Type, &PyFloat_Type, &PyFrame_Type, + &PyFrameLocalsProxy_Type, &PyFrozenSet_Type, &PyFunction_Type, &PyGen_Type, &PyGetSetDescr_Type, &PyInstanceMethod_Type, - &PyInterpreterID_Type, &PyListIter_Type, &PyListRevIter_Type, &PyList_Type, @@ -2281,9 +2381,6 @@ static PyTypeObject* static_types[] = { &_PyBufferWrapper_Type, &_PyContextTokenMissing_Type, &_PyCoroWrapper_Type, - &_PyCounterExecutor_Type, - &_PyCounterOptimizer_Type, - &_PyDefaultOptimizer_Type, &_Py_GenericAliasIterType, &_PyHamtItems_Type, &_PyHamtKeys_Type, @@ -2292,6 +2389,7 @@ static PyTypeObject* static_types[] = { &_PyHamt_BitmapNode_Type, &_PyHamt_CollisionNode_Type, &_PyHamt_Type, + &_PyInstructionSequence_Type, &_PyLegacyEventHandler_Type, &_PyLineIterator, &_PyManagedBuffer_Type, @@ -2303,12 +2401,14 @@ static PyTypeObject* static_types[] = { &_PyPositionsIterator, &_PyUnicodeASCIIIter_Type, &_PyUnion_Type, +#ifdef _Py_TIER2 &_PyUOpExecutor_Type, - &_PyUOpOptimizer_Type, +#endif &_PyWeakref_CallableProxyType, &_PyWeakref_ProxyType, &_PyWeakref_RefType, &_PyTypeAlias_Type, + &_PyNoDefault_Type, // subclasses: _PyTypes_FiniTypes() deallocates them before their base // class @@ -2337,6 +2437,14 @@ _PyTypes_InitTypes(PyInterpreterState *interp) } } + // Cache __reduce__ from PyBaseObject_Type object + PyObject *baseobj_dict = _PyType_GetDict(&PyBaseObject_Type); + PyObject *baseobj_reduce = PyDict_GetItemWithError(baseobj_dict, &_Py_ID(__reduce__)); + if (baseobj_reduce == NULL && PyErr_Occurred()) { + return _PyStatus_ERR("Can't get __reduce__ from base object"); + } + _Py_INTERP_CACHED_OBJECT(interp, objreduce) = baseobj_reduce; + // Must be after static types are initialized if (_Py_initialize_generic(interp) < 0) { return _PyStatus_ERR("Can't initialize generic types"); @@ -2359,7 +2467,7 @@ _PyTypes_FiniTypes(PyInterpreterState *interp) // their base classes. for (Py_ssize_t i=Py_ARRAY_LENGTH(static_types)-1; i>=0; i--) { PyTypeObject *type = static_types[i]; - _PyStaticType_Dealloc(interp, type); + _PyStaticType_FiniBuiltin(interp, type); } } @@ -2367,16 +2475,19 @@ _PyTypes_FiniTypes(PyInterpreterState *interp) static inline void new_reference(PyObject *op) { - if (_PyRuntime.tracemalloc.config.tracing) { - _PyTraceMalloc_NewReference(op); - } // Skip the immortal object check in Py_SET_REFCNT; always set refcnt to 1 #if !defined(Py_GIL_DISABLED) +#if SIZEOF_VOID_P > 4 + op->ob_refcnt_full = 1; + assert(op->ob_refcnt == 1); + assert(op->ob_flags == 0); +#else op->ob_refcnt = 1; +#endif #else op->ob_tid = _Py_ThreadId(); - op->_padding = 0; - op->ob_mutex = (struct _PyMutex){ 0 }; + op->ob_flags = 0; + op->ob_mutex = (PyMutex){ 0 }; op->ob_gc_bits = 0; op->ob_ref_local = 1; op->ob_ref_shared = 0; @@ -2384,13 +2495,14 @@ new_reference(PyObject *op) #ifdef Py_TRACE_REFS _Py_AddToAllObjects(op); #endif + _PyReftracerTrack(op, PyRefTracer_CREATE); } void _Py_NewReference(PyObject *op) { #ifdef Py_REF_DEBUG - reftotal_increment(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif new_reference(op); } @@ -2402,11 +2514,95 @@ _Py_NewReferenceNoTotal(PyObject *op) } void -_Py_ResurrectReference(PyObject *op) +_Py_SetImmortalUntracked(PyObject *op) +{ +#ifdef Py_DEBUG + // For strings, use _PyUnicode_InternImmortal instead. + if (PyUnicode_CheckExact(op)) { + assert(PyUnicode_CHECK_INTERNED(op) == SSTATE_INTERNED_IMMORTAL + || PyUnicode_CHECK_INTERNED(op) == SSTATE_INTERNED_IMMORTAL_STATIC); + } +#endif + // Check if already immortal to avoid degrading from static immortal to plain immortal + if (_Py_IsImmortal(op)) { + return; + } +#ifdef Py_GIL_DISABLED + op->ob_tid = _Py_UNOWNED_TID; + op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; + op->ob_ref_shared = 0; +#else + op->ob_refcnt = _Py_IMMORTAL_INITIAL_REFCNT; +#endif +} + +void +_Py_SetImmortal(PyObject *op) +{ + if (PyObject_IS_GC(op) && _PyObject_GC_IS_TRACKED(op)) { + _PyObject_GC_UNTRACK(op); + } + _Py_SetImmortalUntracked(op); +} + +void +_PyObject_SetDeferredRefcount(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + assert(PyType_IS_GC(Py_TYPE(op))); + assert(_Py_IsOwnedByCurrentThread(op)); + assert(op->ob_ref_shared == 0); + _PyObject_SET_GC_BITS(op, _PyGC_BITS_DEFERRED); + op->ob_ref_shared = _Py_REF_SHARED(_Py_REF_DEFERRED, 0); +#endif +} + +int +PyUnstable_Object_EnableDeferredRefcount(PyObject *op) { - if (_PyRuntime.tracemalloc.config.tracing) { - _PyTraceMalloc_NewReference(op); +#ifdef Py_GIL_DISABLED + if (!PyType_IS_GC(Py_TYPE(op))) { + // Deferred reference counting doesn't work + // on untracked types. + return 0; + } + + uint8_t bits = _Py_atomic_load_uint8(&op->ob_gc_bits); + if ((bits & _PyGC_BITS_DEFERRED) != 0) + { + // Nothing to do. + return 0; } + + if (_Py_atomic_compare_exchange_uint8(&op->ob_gc_bits, &bits, bits | _PyGC_BITS_DEFERRED) == 0) + { + // Someone beat us to it! + return 0; + } + _Py_atomic_add_ssize(&op->ob_ref_shared, _Py_REF_SHARED(_Py_REF_DEFERRED, 0)); + return 1; +#else + return 0; +#endif +} + +int +PyUnstable_TryIncRef(PyObject *op) +{ + return _Py_TryIncref(op); +} + +void +PyUnstable_EnableTryIncRef(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + _PyObject_SetMaybeWeakref(op); +#endif +} + +void +_Py_ResurrectReference(PyObject *op) +{ #ifdef Py_TRACE_REFS _Py_AddToAllObjects(op); #endif @@ -2673,33 +2869,30 @@ Py_ReprLeave(PyObject *obj) /* Trashcan support. */ -#define _PyTrash_UNWIND_LEVEL 50 - /* Add op to the gcstate->trash_delete_later list. Called when the current * call-stack depth gets large. op must be a currently untracked gc'ed * object, with refcount 0. Py_DECREF must already have been called on it. */ -static void -_PyTrash_thread_deposit_object(struct _py_trashcan *trash, PyObject *op) +void +_PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op) { _PyObject_ASSERT(op, _PyObject_IS_GC(op)); _PyObject_ASSERT(op, !_PyObject_GC_IS_TRACKED(op)); _PyObject_ASSERT(op, Py_REFCNT(op) == 0); #ifdef Py_GIL_DISABLED - _PyObject_ASSERT(op, op->ob_tid == 0); - op->ob_tid = (uintptr_t)trash->delete_later; + op->ob_tid = (uintptr_t)tstate->delete_later; #else - _PyGCHead_SET_PREV(_Py_AS_GC(op), (PyGC_Head*)trash->delete_later); + _PyGCHead_SET_PREV(_Py_AS_GC(op), (PyGC_Head*)tstate->delete_later); #endif - trash->delete_later = op; + tstate->delete_later = op; } /* Deallocate all the objects in the gcstate->trash_delete_later list. * Called when the call-stack unwinds again. */ -static void -_PyTrash_thread_destroy_chain(struct _py_trashcan *trash) +void +_PyTrash_thread_destroy_chain(PyThreadState *tstate) { - /* We need to increase trash_delete_nesting here, otherwise, + /* We need to increase c_recursion_remaining here, otherwise, _PyTrash_thread_destroy_chain will be called recursively and then possibly crash. An example that may crash without increase: @@ -2710,17 +2903,18 @@ _PyTrash_thread_destroy_chain(struct _py_trashcan *trash) tups = [(tup,) for tup in tups] del tups */ - assert(trash->delete_nesting == 0); - ++trash->delete_nesting; - while (trash->delete_later) { - PyObject *op = trash->delete_later; + assert(tstate->c_recursion_remaining > Py_TRASHCAN_HEADROOM); + tstate->c_recursion_remaining--; + while (tstate->delete_later) { + PyObject *op = tstate->delete_later; destructor dealloc = Py_TYPE(op)->tp_dealloc; #ifdef Py_GIL_DISABLED - trash->delete_later = (PyObject*) op->ob_tid; + tstate->delete_later = (PyObject*) op->ob_tid; op->ob_tid = 0; + _Py_atomic_store_ssize_relaxed(&op->ob_ref_shared, _Py_REF_MERGED); #else - trash->delete_later = (PyObject*) _PyGCHead_PREV(_Py_AS_GC(op)); + tstate->delete_later = (PyObject*) _PyGCHead_PREV(_Py_AS_GC(op)); #endif /* Call the deallocator directly. This used to try to @@ -2731,92 +2925,10 @@ _PyTrash_thread_destroy_chain(struct _py_trashcan *trash) */ _PyObject_ASSERT(op, Py_REFCNT(op) == 0); (*dealloc)(op); - assert(trash->delete_nesting == 1); } - --trash->delete_nesting; + tstate->c_recursion_remaining++; } - -static struct _py_trashcan * -_PyTrash_get_state(PyThreadState *tstate) -{ - if (tstate != NULL) { - return &tstate->trash; - } - // The current thread must be finalizing. - // Fall back to using thread-local state. - // XXX Use thread-local variable syntax? - assert(PyThread_tss_is_created(&_PyRuntime.trashTSSkey)); - struct _py_trashcan *trash = - (struct _py_trashcan *)PyThread_tss_get(&_PyRuntime.trashTSSkey); - if (trash == NULL) { - trash = PyMem_RawMalloc(sizeof(struct _py_trashcan)); - if (trash == NULL) { - Py_FatalError("Out of memory"); - } - PyThread_tss_set(&_PyRuntime.trashTSSkey, (void *)trash); - } - return trash; -} - -static void -_PyTrash_clear_state(PyThreadState *tstate) -{ - if (tstate != NULL) { - assert(tstate->trash.delete_later == NULL); - return; - } - if (PyThread_tss_is_created(&_PyRuntime.trashTSSkey)) { - struct _py_trashcan *trash = - (struct _py_trashcan *)PyThread_tss_get(&_PyRuntime.trashTSSkey); - if (trash != NULL) { - PyThread_tss_set(&_PyRuntime.trashTSSkey, (void *)NULL); - PyMem_RawFree(trash); - } - } -} - - -int -_PyTrash_begin(PyThreadState *tstate, PyObject *op) -{ - // XXX Make sure the GIL is held. - struct _py_trashcan *trash = _PyTrash_get_state(tstate); - if (trash->delete_nesting >= _PyTrash_UNWIND_LEVEL) { - /* Store the object (to be deallocated later) and jump past - * Py_TRASHCAN_END, skipping the body of the deallocator */ - _PyTrash_thread_deposit_object(trash, op); - return 1; - } - ++trash->delete_nesting; - return 0; -} - - -void -_PyTrash_end(PyThreadState *tstate) -{ - // XXX Make sure the GIL is held. - struct _py_trashcan *trash = _PyTrash_get_state(tstate); - --trash->delete_nesting; - if (trash->delete_nesting <= 0) { - if (trash->delete_later != NULL) { - _PyTrash_thread_destroy_chain(trash); - } - _PyTrash_clear_state(tstate); - } -} - - -/* bpo-40170: It's only be used in Py_TRASHCAN_BEGIN macro to hide - implementation details. */ -int -_PyTrash_cond(PyObject *op, destructor dealloc) -{ - return Py_TYPE(op)->tp_dealloc == dealloc; -} - - void _Py_NO_RETURN _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg, const char *file, int line, const char *function) @@ -2886,6 +2998,7 @@ _Py_Dealloc(PyObject *op) #ifdef Py_TRACE_REFS _Py_ForgetReference(op); #endif + _PyReftracerTrack(op, PyRefTracer_DESTROY); (*dealloc)(op); #ifdef Py_DEBUG @@ -2970,3 +3083,96 @@ _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt) { Py_SET_REFCNT(ob, refcnt); } + +int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) { + _Py_AssertHoldsTstate(); + _PyRuntime.ref_tracer.tracer_func = tracer; + _PyRuntime.ref_tracer.tracer_data = data; + return 0; +} + +PyRefTracer PyRefTracer_GetTracer(void** data) { + _Py_AssertHoldsTstate(); + if (data != NULL) { + *data = _PyRuntime.ref_tracer.tracer_data; + } + return _PyRuntime.ref_tracer.tracer_func; +} + + + +static PyObject* constants[] = { + &_Py_NoneStruct, // Py_CONSTANT_NONE + (PyObject*)(&_Py_FalseStruct), // Py_CONSTANT_FALSE + (PyObject*)(&_Py_TrueStruct), // Py_CONSTANT_TRUE + &_Py_EllipsisObject, // Py_CONSTANT_ELLIPSIS + &_Py_NotImplementedStruct, // Py_CONSTANT_NOT_IMPLEMENTED + NULL, // Py_CONSTANT_ZERO + NULL, // Py_CONSTANT_ONE + NULL, // Py_CONSTANT_EMPTY_STR + NULL, // Py_CONSTANT_EMPTY_BYTES + NULL, // Py_CONSTANT_EMPTY_TUPLE +}; + +void +_Py_GetConstant_Init(void) +{ + constants[Py_CONSTANT_ZERO] = _PyLong_GetZero(); + constants[Py_CONSTANT_ONE] = _PyLong_GetOne(); + constants[Py_CONSTANT_EMPTY_STR] = PyUnicode_New(0, 0); + constants[Py_CONSTANT_EMPTY_BYTES] = PyBytes_FromStringAndSize(NULL, 0); + constants[Py_CONSTANT_EMPTY_TUPLE] = PyTuple_New(0); +#ifndef NDEBUG + for (size_t i=0; i < Py_ARRAY_LENGTH(constants); i++) { + assert(constants[i] != NULL); + assert(_Py_IsImmortal(constants[i])); + } +#endif +} + +PyObject* +Py_GetConstant(unsigned int constant_id) +{ + if (constant_id < Py_ARRAY_LENGTH(constants)) { + return constants[constant_id]; + } + else { + PyErr_BadInternalCall(); + return NULL; + } +} + + +PyObject* +Py_GetConstantBorrowed(unsigned int constant_id) +{ + // All constants are immortal + return Py_GetConstant(constant_id); +} + + +// Py_TYPE() implementation for the stable ABI +#undef Py_TYPE +PyTypeObject* +Py_TYPE(PyObject *ob) +{ + return _Py_TYPE(ob); +} + + +// Py_REFCNT() implementation for the stable ABI +#undef Py_REFCNT +Py_ssize_t +Py_REFCNT(PyObject *ob) +{ + return _Py_REFCNT(ob); +} + +int +PyUnstable_IsImmortal(PyObject *op) +{ + /* Checking a reference count requires a thread state */ + _Py_AssertHoldsTstate(); + assert(op != NULL); + return _Py_IsImmortal(op); +} diff --git a/Objects/object_layout.md b/Objects/object_layout.md index 4f379bed8d77e2..4a781668636324 100644 --- a/Objects/object_layout.md +++ b/Objects/object_layout.md @@ -16,7 +16,49 @@ Since the introduction of the cycle GC, there has also been a pre-header. Before 3.11, this pre-header was two words in size. It should be considered opaque to all code except the cycle GC. -## 3.11 pre-header +### 3.13 + +In 3.13, the values array is embedded into the object, so there is no +need for a values pointer (it is just a fixed offset into the object). +So the pre-header is these two fields: + +* weakreflist +* dict_pointer + +If the object has no physical dictionary, then the ``dict_pointer`` +is set to `NULL`. + +In 3.13 only objects with no additional data could have inline values. +That is, instances of classes with `tp_basicsize == sizeof(PyObject)`. +In 3.14, any object whose class has `tp_itemsize == 0` can have inline values. +In both versions, the inline values starts `tp_basicsize` bytes after the object. + +
    + 3.12 + +### 3.12 + +In 3.12, the pointer to the list of weak references is added to the +pre-header. In order to make space for it, the ``dict`` and ``values`` +pointers are combined into a single tagged pointer: + +* weakreflist +* dict_or_values + +If the object has no physical dictionary, then the ``dict_or_values`` +has its low bit set to one, and points to the values array. +If the object has a physical dictionary, then the ``dict_or_values`` +has its low bit set to zero, and points to the dictionary. + +The untagged form is chosen for the dictionary pointer, rather than +the values pointer, to enable the (legacy) C-API function +`_PyObject_GetDictPtr(PyObject *obj)` to work. +
    + +
    + 3.11 + +### 3.11 In 3.11 the pre-header was extended to include pointers to the VM managed ``__dict__``. The reason for moving the ``__dict__`` to the pre-header is that it allows @@ -33,27 +75,49 @@ The values pointer refers to the ``PyDictValues`` array which holds the values of the objects's attributes. Should the dictionary be needed, then ``values`` is set to ``NULL`` and the ``dict`` field points to the dictionary. +
    -## 3.12 pre-header +## Layout of a "normal" Python object -In 3.12, the pointer to the list of weak references is added to the -pre-header. In order to make space for it, the ``dict`` and ``values`` -pointers are combined into a single tagged pointer: +A "normal" Python object is one that doesn't inherit from a builtin +class, doesn't have slots. + +### 3.13 + +In 3.13 the values are embedded into the object, as follows: * weakreflist * dict_or_values +* GC 1 +* GC 2 +* ob_refcnt +* ob_type +* Inlined values: + * Flags + * values 0 + * values 1 + * ... + * Insertion order bytes -If the object has no physical dictionary, then the ``dict_or_values`` -has its low bit set to one, and points to the values array. -If the object has a physical dictionary, then the ``dict_or_values`` -has its low bit set to zero, and points to the dictionary. +This has all the advantages of the layout used in 3.12, plus: +* Access to values is even faster as there is one less load +* Fast access is mostly maintained when the `__dict__` is materialized -The untagged form is chosen for the dictionary pointer, rather than -the values pointer, to enable the (legacy) C-API function -`_PyObject_GetDictPtr(PyObject *obj)` to work. +![Layout of "normal" object in 3.13](./object_layout_313.png) + +For objects with opaque parts defined by a C extension, +the layout is much the same as for 3.12 +![Layout of "full" object in 3.13](./object_layout_full_313.png) -## Layout of a "normal" Python object in 3.12: + +
    + 3.12 + +### 3.12: + +In 3.12, the header and pre-header form the entire object for "normal" +Python objects: * weakreflist * dict_or_values @@ -62,9 +126,6 @@ the values pointer, to enable the (legacy) C-API function * ob_refcnt * ob_type -For a "normal" Python object, one that doesn't inherit from a builtin -class or have slots, the header and pre-header form the entire object. - ![Layout of "normal" object in 3.12](./object_layout_312.png) There are several advantages to this layout: @@ -79,4 +140,6 @@ The full layout object, with an opaque part defined by a C extension, and `__slots__` looks like this: ![Layout of "full" object in 3.12](./object_layout_full_312.png) +
    + diff --git a/Objects/object_layout_312.gv b/Objects/object_layout_312.gv index c0068d78568524..731a25332b3ace 100644 --- a/Objects/object_layout_312.gv +++ b/Objects/object_layout_312.gv @@ -20,6 +20,7 @@ digraph ideal { shape = none label = < + diff --git a/Objects/object_layout_312.png b/Objects/object_layout_312.png index 396dab183b3e9b..a63d095ea0b19e 100644 Binary files a/Objects/object_layout_312.png and b/Objects/object_layout_312.png differ diff --git a/Objects/object_layout_313.gv b/Objects/object_layout_313.gv new file mode 100644 index 00000000000000..d85352876f273b --- /dev/null +++ b/Objects/object_layout_313.gv @@ -0,0 +1,45 @@ +digraph ideal { + + rankdir = "LR" + + + object [ + shape = none + label = <
    values
    Insertion order
    values[0]
    values[1]
    ...
    + + + + + + + + + + + + +
    object
    weakrefs
    dict pointer
    GC info 0
    GC info 1
    refcount
    __class__
    values flags
    values[0]
    values[1]
    ...
    Insertion order
    > + + ] + + class [ + shape = none + label = < + + + + + +
    class
    ...
    dict_offset
    ...
    cached_keys
    > + ] + + keys [label = "dictionary keys"; fillcolor="lightgreen"; style="filled"] + NULL [ label = " NULL"; shape="plain"] + object:w -> NULL + object:h -> class:head + object:dv -> NULL + class:k -> keys + + oop [ label = "pointer"; shape="plain"] + oop -> object:r +} diff --git a/Objects/object_layout_313.png b/Objects/object_layout_313.png new file mode 100644 index 00000000000000..f7059c286c84e6 Binary files /dev/null and b/Objects/object_layout_313.png differ diff --git a/Objects/object_layout_full_313.gv b/Objects/object_layout_full_313.gv new file mode 100644 index 00000000000000..551312d51a7012 --- /dev/null +++ b/Objects/object_layout_full_313.gv @@ -0,0 +1,25 @@ +digraph ideal { + + rankdir = "LR" + + + object [ + shape = none + label = < + + + + + + + + + + + +
    object
    weakrefs
    dict pointer
    GC info 0
    GC info 1
    refcount
    __class__
    opaque (extension) data
    ...
    __slot__ 0
    ...
    > + ] + + oop [ label = "pointer"; shape="plain"] + oop -> object:r +} diff --git a/Objects/object_layout_full_313.png b/Objects/object_layout_full_313.png new file mode 100644 index 00000000000000..7352ec69e5d8db Binary files /dev/null and b/Objects/object_layout_full_313.png differ diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index b2a2286ef22b66..5688049b024696 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -12,6 +12,12 @@ #include // malloc() #include #ifdef WITH_MIMALLOC +// Forward declarations of functions used in our mimalloc modifications +static void _PyMem_mi_page_clear_qsbr(mi_page_t *page); +static bool _PyMem_mi_page_is_safe_to_free(mi_page_t *page); +static bool _PyMem_mi_page_maybe_free(mi_page_t *page, mi_page_queue_t *pq, bool force); +static void _PyMem_mi_page_reclaimed(mi_page_t *page); +static void _PyMem_mi_heap_collect_qsbr(mi_heap_t *heap); # include "pycore_mimalloc.h" # include "mimalloc/static.c" # include "mimalloc/internal.h" // for stats @@ -86,6 +92,113 @@ _PyMem_RawFree(void *Py_UNUSED(ctx), void *ptr) #ifdef WITH_MIMALLOC +static void +_PyMem_mi_page_clear_qsbr(mi_page_t *page) +{ +#ifdef Py_GIL_DISABLED + // Clear the QSBR goal and remove the page from the QSBR linked list. + page->qsbr_goal = 0; + if (page->qsbr_node.next != NULL) { + llist_remove(&page->qsbr_node); + } +#endif +} + +// Check if an empty, newly reclaimed page is safe to free now. +static bool +_PyMem_mi_page_is_safe_to_free(mi_page_t *page) +{ + assert(mi_page_all_free(page)); +#ifdef Py_GIL_DISABLED + assert(page->qsbr_node.next == NULL); + if (page->use_qsbr && page->qsbr_goal != 0) { + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + if (tstate == NULL) { + return false; + } + return _Py_qbsr_goal_reached(tstate->qsbr, page->qsbr_goal); + } +#endif + return true; + +} + +static bool +_PyMem_mi_page_maybe_free(mi_page_t *page, mi_page_queue_t *pq, bool force) +{ +#ifdef Py_GIL_DISABLED + assert(mi_page_all_free(page)); + if (page->use_qsbr) { + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)PyThreadState_GET(); + if (page->qsbr_goal != 0 && _Py_qbsr_goal_reached(tstate->qsbr, page->qsbr_goal)) { + _PyMem_mi_page_clear_qsbr(page); + _mi_page_free(page, pq, force); + return true; + } + + _PyMem_mi_page_clear_qsbr(page); + page->retire_expire = 0; + page->qsbr_goal = _Py_qsbr_deferred_advance(tstate->qsbr); + llist_insert_tail(&tstate->mimalloc.page_list, &page->qsbr_node); + return false; + } +#endif + _mi_page_free(page, pq, force); + return true; +} + +static void +_PyMem_mi_page_reclaimed(mi_page_t *page) +{ +#ifdef Py_GIL_DISABLED + assert(page->qsbr_node.next == NULL); + if (page->qsbr_goal != 0) { + if (mi_page_all_free(page)) { + assert(page->qsbr_node.next == NULL); + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)PyThreadState_GET(); + page->retire_expire = 0; + llist_insert_tail(&tstate->mimalloc.page_list, &page->qsbr_node); + } + else { + page->qsbr_goal = 0; + } + } +#endif +} + +static void +_PyMem_mi_heap_collect_qsbr(mi_heap_t *heap) +{ +#ifdef Py_GIL_DISABLED + if (!heap->page_use_qsbr) { + return; + } + + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + struct llist_node *head = &tstate->mimalloc.page_list; + if (llist_empty(head)) { + return; + } + + struct llist_node *node; + llist_for_each_safe(node, head) { + mi_page_t *page = llist_data(node, mi_page_t, qsbr_node); + if (!mi_page_all_free(page)) { + // We allocated from this page some point after the delayed free + _PyMem_mi_page_clear_qsbr(page); + continue; + } + + if (!_Py_qsbr_poll(tstate->qsbr, page->qsbr_goal)) { + return; + } + + _PyMem_mi_page_clear_qsbr(page); + _mi_page_free(page, mi_page_queue_of(page), false); + } +#endif +} + void * _PyMem_MiMalloc(void *ctx, size_t size) { @@ -273,8 +386,16 @@ _PyMem_ArenaFree(void *Py_UNUSED(ctx), void *ptr, ) { #ifdef MS_WINDOWS + /* Unlike free(), VirtualFree() does not special-case NULL to noop. */ + if (ptr == NULL) { + return; + } VirtualFree(ptr, 0, MEM_RELEASE); #elif defined(ARENAS_USE_MMAP) + /* Unlike free(), munmap() does not special-case NULL to noop. */ + if (ptr == NULL) { + return; + } munmap(ptr, size); #else free(ptr); @@ -957,7 +1078,7 @@ _PyMem_Strdup(const char *str) // A pointer to be freed once the QSBR read sequence reaches qsbr_goal. struct _mem_work_item { - void *ptr; + uintptr_t ptr; // lowest bit tagged 1 for objects freed with PyObject_Free uint64_t qsbr_goal; }; @@ -971,16 +1092,45 @@ struct _mem_work_chunk { struct _mem_work_item array[WORK_ITEMS_PER_CHUNK]; }; -void -_PyMem_FreeDelayed(void *ptr) +static void +free_work_item(uintptr_t ptr, delayed_dealloc_cb cb, void *state) +{ + if (ptr & 0x01) { + PyObject *obj = (PyObject *)(ptr - 1); +#ifdef Py_GIL_DISABLED + if (cb == NULL) { + assert(!_PyInterpreterState_GET()->stoptheworld.world_stopped); + Py_DECREF(obj); + return; + } + + Py_ssize_t refcount = _Py_ExplicitMergeRefcount(obj, -1); + if (refcount == 0) { + cb(obj, state); + } +#else + Py_DECREF(obj); +#endif + } + else { + PyMem_Free((void *)ptr); + } +} + +static void +free_delayed(uintptr_t ptr) { #ifndef Py_GIL_DISABLED - PyMem_Free(ptr); + free_work_item(ptr, NULL, NULL); #else - if (_PyRuntime.stoptheworld.world_stopped) { - // Free immediately if the world is stopped, including during - // interpreter shutdown. - PyMem_Free(ptr); + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyInterpreterState_GetFinalizing(interp) != NULL || + interp->stoptheworld.world_stopped) + { + // Free immediately during interpreter shutdown or if the world is + // stopped. + assert(!interp->stoptheworld.world_stopped || !(ptr & 0x01)); + free_work_item(ptr, NULL, NULL); return; } @@ -1007,7 +1157,8 @@ _PyMem_FreeDelayed(void *ptr) if (buf == NULL) { // failed to allocate a buffer, free immediately _PyEval_StopTheWorld(tstate->base.interp); - PyMem_Free(ptr); + // TODO: Fix me + free_work_item(ptr, NULL, NULL); _PyEval_StartTheWorld(tstate->base.interp); return; } @@ -1024,6 +1175,24 @@ _PyMem_FreeDelayed(void *ptr) #endif } +void +_PyMem_FreeDelayed(void *ptr) +{ + assert(!((uintptr_t)ptr & 0x01)); + free_delayed((uintptr_t)ptr); +} + +#ifdef Py_GIL_DISABLED +void +_PyObject_XDecRefDelayed(PyObject *ptr) +{ + assert(!((uintptr_t)ptr & 0x01)); + if (ptr != NULL) { + free_delayed(((uintptr_t)ptr)|0x01); + } +} +#endif + static struct _mem_work_chunk * work_queue_first(struct llist_node *head) { @@ -1032,7 +1201,7 @@ work_queue_first(struct llist_node *head) static void process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr, - bool keep_empty) + bool keep_empty, delayed_dealloc_cb cb, void *state) { while (!llist_empty(head)) { struct _mem_work_chunk *buf = work_queue_first(head); @@ -1043,7 +1212,7 @@ process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr, return; } - PyMem_Free(item->ptr); + free_work_item(item->ptr, cb, state); buf->rd_idx++; } @@ -1061,7 +1230,8 @@ process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr, static void process_interp_queue(struct _Py_mem_interp_free_queue *queue, - struct _qsbr_thread_state *qsbr) + struct _qsbr_thread_state *qsbr, delayed_dealloc_cb cb, + void *state) { if (!_Py_atomic_load_int_relaxed(&queue->has_work)) { return; @@ -1069,7 +1239,7 @@ process_interp_queue(struct _Py_mem_interp_free_queue *queue, // Try to acquire the lock, but don't block if it's already held. if (_PyMutex_LockTimed(&queue->mutex, 0, 0) == PY_LOCK_ACQUIRED) { - process_queue(&queue->head, qsbr, false); + process_queue(&queue->head, qsbr, false, cb, state); int more_work = !llist_empty(&queue->head); _Py_atomic_store_int_relaxed(&queue->has_work, more_work); @@ -1085,10 +1255,23 @@ _PyMem_ProcessDelayed(PyThreadState *tstate) _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; // Process thread-local work - process_queue(&tstate_impl->mem_free_queue, tstate_impl->qsbr, true); + process_queue(&tstate_impl->mem_free_queue, tstate_impl->qsbr, true, NULL, NULL); + + // Process shared interpreter work + process_interp_queue(&interp->mem_free_queue, tstate_impl->qsbr, NULL, NULL); +} + +void +_PyMem_ProcessDelayedNoDealloc(PyThreadState *tstate, delayed_dealloc_cb cb, void *state) +{ + PyInterpreterState *interp = tstate->interp; + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + + // Process thread-local work + process_queue(&tstate_impl->mem_free_queue, tstate_impl->qsbr, true, cb, state); // Process shared interpreter work - process_interp_queue(&interp->mem_free_queue, tstate_impl->qsbr); + process_interp_queue(&interp->mem_free_queue, tstate_impl->qsbr, cb, state); } void @@ -1130,7 +1313,7 @@ _PyMem_FiniDelayed(PyInterpreterState *interp) // Free the remaining items immediately. There should be no other // threads accessing the memory at this point during shutdown. struct _mem_work_item *item = &buf->array[buf->rd_idx]; - PyMem_Free(item->ptr); + free_work_item(item->ptr, NULL, NULL); buf->rd_idx++; } @@ -1208,7 +1391,7 @@ static int running_on_valgrind = -1; typedef struct _obmalloc_state OMState; /* obmalloc state for main interpreter and shared by all interpreters without - * their own obmalloc state. By not explicitly initalizing this structure, it + * their own obmalloc state. By not explicitly initializing this structure, it * will be allocated in the BSS which is a small performance win. The radix * tree arrays are fairly large but are sparsely used. */ static struct _obmalloc_state obmalloc_state_main; @@ -1256,7 +1439,7 @@ get_mimalloc_allocated_blocks(PyInterpreterState *interp) { size_t allocated_blocks = 0; #ifdef Py_GIL_DISABLED - for (PyThreadState *t = interp->threads.head; t != NULL; t = t->next) { + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, t) { _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)t; for (int i = 0; i < _Py_MIMALLOC_HEAP_COUNT; i++) { mi_heap_t *heap = &tstate->mimalloc.heaps[i]; @@ -1328,6 +1511,8 @@ _PyInterpreterState_FinalizeAllocatedBlocks(PyInterpreterState *interp) { #ifdef WITH_MIMALLOC if (_PyMem_MimallocEnabled()) { + Py_ssize_t leaked = _PyInterpreterState_GetAllocatedBlocks(interp); + interp->runtime->obmalloc.interpreter_leaks += leaked; return; } #endif @@ -2724,10 +2909,18 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) static inline void _PyMem_DebugCheckGIL(const char *func) { - if (!PyGILState_Check()) { + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate == NULL) { +#ifndef Py_GIL_DISABLED _Py_FatalErrorFunc(func, "Python memory allocator called " "without holding the GIL"); +#else + _Py_FatalErrorFunc(func, + "Python memory allocator called " + "without an active thread state. " + "Are you trying to call it inside of a Py_BEGIN_ALLOW_THREADS block?"); +#endif } } diff --git a/Objects/odictobject.c b/Objects/odictobject.c index 421bc52992d735..f2d8da0c567878 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -200,7 +200,7 @@ Here are some ways to address this challenge: Adding the checks to the concrete API would help make any interpreter switch to OrderedDict less painful for extension modules. However, this won't work. The equivalent C API call to `dict.__setitem__(obj, k, v)` -is 'PyDict_SetItem(obj, k, v)`. This illustrates how subclasses in C call +is `PyDict_SetItem(obj, k, v)`. This illustrates how subclasses in C call the base class's methods, since there is no equivalent of super() in the C API. Calling into Python for parent class API would work, but some extension modules already rely on this feature of the concrete API. @@ -260,7 +260,7 @@ mp_length __len__ - dict_length mp_subscript __getitem__ - dict_subscript mp_ass_subscript __setitem__ - dict_ass_sub __delitem__ -tp_hash __hash__ _Py_HashPointer ..._HashNotImpl +tp_hash __hash__ Py_HashPointer ..._HashNotImpl tp_str __str__ object_str - tp_getattro __getattribute__ ..._GenericGetAttr (repeated) __getattr__ @@ -276,7 +276,7 @@ tp_dictoffset (__dict__) - - tp_init __init__ object_init dict_init tp_alloc - PyType_GenericAlloc (repeated) tp_new __new__ object_new dict_new -tp_free - PyObject_Del PyObject_GC_Del +tp_free - PyObject_Free PyObject_GC_Del ================= ================ =================== ================ Relevant Methods @@ -535,8 +535,12 @@ _odict_get_index_raw(PyODictObject *od, PyObject *key, Py_hash_t hash) PyObject *value = NULL; PyDictKeysObject *keys = ((PyDictObject *)od)->ma_keys; Py_ssize_t ix; - +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe((PyDictObject *)od, key, hash, &value); + Py_XDECREF(value); +#else ix = _Py_dict_lookup((PyDictObject *)od, key, hash, &value); +#endif if (ix == DKIX_EMPTY) { return keys->dk_nentries; /* index of new entry */ } @@ -792,6 +796,7 @@ _odict_clear_nodes(PyODictObject *od) _odictnode_DEALLOC(node); node = next; } + od->od_state++; } /* There isn't any memory management of nodes past this point. */ @@ -802,24 +807,40 @@ _odict_keys_equal(PyODictObject *a, PyODictObject *b) { _ODictNode *node_a, *node_b; + // keep operands' state to detect undesired mutations + const size_t state_a = a->od_state; + const size_t state_b = b->od_state; + node_a = _odict_FIRST(a); node_b = _odict_FIRST(b); while (1) { - if (node_a == NULL && node_b == NULL) + if (node_a == NULL && node_b == NULL) { /* success: hit the end of each at the same time */ return 1; - else if (node_a == NULL || node_b == NULL) + } + else if (node_a == NULL || node_b == NULL) { /* unequal length */ return 0; + } else { - int res = PyObject_RichCompareBool( - (PyObject *)_odictnode_KEY(node_a), - (PyObject *)_odictnode_KEY(node_b), - Py_EQ); - if (res < 0) + PyObject *key_a = Py_NewRef(_odictnode_KEY(node_a)); + PyObject *key_b = Py_NewRef(_odictnode_KEY(node_b)); + int res = PyObject_RichCompareBool(key_a, key_b, Py_EQ); + Py_DECREF(key_a); + Py_DECREF(key_b); + if (res < 0) { return res; - else if (res == 0) + } + else if (a->od_state != state_a || b->od_state != state_b) { + PyErr_SetString(PyExc_RuntimeError, + "OrderedDict mutated during iteration"); + return -1; + } + else if (res == 0) { + // This check comes after the check on the state + // in order for the exception to be set correctly. return 0; + } /* otherwise it must match, so move on to the next one */ node_a = _odictnode_NEXT(node_a); diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index ce9eef69ad75a8..2942ab624edf72 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -83,7 +83,7 @@ range_from_array(PyTypeObject *type, PyObject *const *args, Py_ssize_t num_args) switch (num_args) { case 3: step = args[2]; - /* fallthrough */ + _Py_FALLTHROUGH; case 2: /* Convert borrowed refs to owned refs */ start = PyNumber_Index(args[0]); @@ -143,14 +143,14 @@ range_new(PyTypeObject *type, PyObject *args, PyObject *kw) static PyObject * -range_vectorcall(PyTypeObject *type, PyObject *const *args, +range_vectorcall(PyObject *rangetype, PyObject *const *args, size_t nargsf, PyObject *kwnames) { Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); if (!_PyArg_NoKwnames("range", kwnames)) { return NULL; } - return range_from_array(type, args, nargs); + return range_from_array((PyTypeObject *)rangetype, args, nargs); } PyDoc_STRVAR(range_doc, @@ -655,7 +655,7 @@ range_index(rangeobject *r, PyObject *ob) } /* object is not in the range */ - PyErr_Format(PyExc_ValueError, "%R is not in range", ob); + PyErr_SetString(PyExc_ValueError, "range.index(x): x not in range"); return NULL; } @@ -751,7 +751,7 @@ PyDoc_STRVAR(index_doc, static PyMethodDef range_methods[] = { {"__reversed__", range_reverse, METH_NOARGS, reverse_doc}, - {"__reduce__", (PyCFunction)range_reduce, METH_VARARGS}, + {"__reduce__", (PyCFunction)range_reduce, METH_NOARGS}, {"count", (PyCFunction)range_count, METH_O, count_doc}, {"index", (PyCFunction)range_index, METH_O, index_doc}, {NULL, NULL} /* sentinel */ @@ -803,7 +803,7 @@ PyTypeObject PyRange_Type = { 0, /* tp_init */ 0, /* tp_alloc */ range_new, /* tp_new */ - .tp_vectorcall = (vectorcallfunc)range_vectorcall + .tp_vectorcall = range_vectorcall }; /*********************** range Iterator **************************/ @@ -899,7 +899,7 @@ PyTypeObject PyRangeIter_Type = { sizeof(_PyRangeIterObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)PyObject_Del, /* tp_dealloc */ + (destructor)PyObject_Free, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ diff --git a/Objects/setobject.c b/Objects/setobject.c index 6a4c8c45f0836d..26ab352ca6d989 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2,7 +2,7 @@ /* set object implementation Written and maintained by Raymond D. Hettinger - Derived from Lib/sets.py and Objects/dictobject.c. + Derived from Objects/dictobject.c. The basic lookup function used by all operations. This is based on Algorithm D from Knuth Vol. 3, Sec. 6.4. @@ -37,8 +37,11 @@ #include "pycore_dict.h" // _PyDict_Contains_KnownHash() #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_RELAXED() #include "pycore_pyerrors.h" // _PyErr_SetKeyError() #include "pycore_setobject.h" // _PySet_NextEntry() definition + +#include "stringlib/eq.h" // unicode_eq() #include // offsetof() #include "clinic/setobject.c.h" @@ -95,7 +98,7 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) return entry; if (PyUnicode_CheckExact(startkey) && PyUnicode_CheckExact(key) - && _PyUnicode_EQ(startkey, key)) + && unicode_eq(startkey, key)) return entry; table = so->table; Py_INCREF(startkey); @@ -130,6 +133,8 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) int probes; int cmp; + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); + /* Pre-increment is necessary to prevent arbitrary code in the rich comparison from deallocating the key just before the insertion. */ Py_INCREF(key); @@ -154,7 +159,7 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) goto found_active; if (PyUnicode_CheckExact(startkey) && PyUnicode_CheckExact(key) - && _PyUnicode_EQ(startkey, key)) + && unicode_eq(startkey, key)) goto found_active; table = so->table; Py_INCREF(startkey); @@ -181,14 +186,14 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) found_unused_or_dummy: if (freeslot == NULL) goto found_unused; - so->used++; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, so->used + 1); freeslot->key = key; freeslot->hash = hash; return 0; found_unused: so->fill++; - so->used++; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, so->used + 1); entry->key = key; entry->hash = hash; if ((size_t)so->fill*5 < mask*3) @@ -354,7 +359,7 @@ set_discard_entry(PySetObject *so, PyObject *key, Py_hash_t hash) old_key = entry->key; entry->key = dummy; entry->hash = -1; - so->used--; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, so->used - 1); Py_DECREF(old_key); return DISCARD_FOUND; } @@ -362,13 +367,9 @@ set_discard_entry(PySetObject *so, PyObject *key, Py_hash_t hash) static int set_add_key(PySetObject *so, PyObject *key) { - Py_hash_t hash; - - if (!PyUnicode_CheckExact(key) || - (hash = _PyASCIIObject_CAST(key)->hash) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return -1; + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + return -1; } return set_add_entry(so, key, hash); } @@ -376,13 +377,9 @@ set_add_key(PySetObject *so, PyObject *key) static int set_contains_key(PySetObject *so, PyObject *key) { - Py_hash_t hash; - - if (!PyUnicode_CheckExact(key) || - (hash = _PyASCIIObject_CAST(key)->hash) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return -1; + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + return -1; } return set_contains_entry(so, key, hash); } @@ -390,13 +387,9 @@ set_contains_key(PySetObject *so, PyObject *key) static int set_discard_key(PySetObject *so, PyObject *key) { - Py_hash_t hash; - - if (!PyUnicode_CheckExact(key) || - (hash = _PyASCIIObject_CAST(key)->hash) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return -1; + Py_hash_t hash = _PyObject_HashFast(key); + if (hash == -1) { + return -1; } return set_discard_entry(so, key, hash); } @@ -406,15 +399,16 @@ set_empty_to_minsize(PySetObject *so) { memset(so->smalltable, 0, sizeof(so->smalltable)); so->fill = 0; - so->used = 0; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, 0); so->mask = PySet_MINSIZE - 1; so->table = so->smalltable; so->hash = -1; } static int -set_clear_internal(PySetObject *so) +set_clear_internal(PyObject *self) { + PySetObject *so = _PySet_CAST(self); setentry *entry; setentry *table = so->table; Py_ssize_t fill = so->fill; @@ -499,8 +493,9 @@ set_next(PySetObject *so, Py_ssize_t *pos_ptr, setentry **entry_ptr) } static void -set_dealloc(PySetObject *so) +set_dealloc(PyObject *self) { + PySetObject *so = _PySet_CAST(self); setentry *entry; Py_ssize_t used = so->used; @@ -523,7 +518,7 @@ set_dealloc(PySetObject *so) } static PyObject * -set_repr(PySetObject *so) +set_repr_lock_held(PySetObject *so) { PyObject *result=NULL, *keys, *listrepr, *tmp; int status = Py_ReprEnter((PyObject*)so); @@ -567,14 +562,26 @@ set_repr(PySetObject *so) return result; } +static PyObject * +set_repr(PyObject *self) +{ + PySetObject *so = _PySet_CAST(self); + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(so); + result = set_repr_lock_held(so); + Py_END_CRITICAL_SECTION(); + return result; +} + static Py_ssize_t -set_len(PyObject *so) +set_len(PyObject *self) { - return ((PySetObject *)so)->used; + PySetObject *so = _PySet_CAST(self); + return FT_ATOMIC_LOAD_SSIZE_RELAXED(so->used); } static int -set_merge(PySetObject *so, PyObject *otherset) +set_merge_lock_held(PySetObject *so, PyObject *otherset) { PySetObject *other; PyObject *key; @@ -583,9 +590,10 @@ set_merge(PySetObject *so, PyObject *otherset) setentry *other_entry; assert (PyAnySet_Check(so)); - assert (PyAnySet_Check(otherset)); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(otherset); - other = (PySetObject*)otherset; + other = _PySet_CAST(otherset); if (other == so || other->used == 0) /* a.update(a) or a.update(set()); nothing to do */ return 0; @@ -612,7 +620,7 @@ set_merge(PySetObject *so, PyObject *otherset) } } so->fill = other->fill; - so->used = other->used; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, other->used); return 0; } @@ -621,7 +629,7 @@ set_merge(PySetObject *so, PyObject *otherset) setentry *newtable = so->table; size_t newmask = (size_t)so->mask; so->fill = other->used; - so->used = other->used; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, other->used); for (i = other->mask + 1; i > 0 ; i--, other_entry++) { key = other_entry->key; if (key != NULL && key != dummy) { @@ -645,6 +653,7 @@ set_merge(PySetObject *so, PyObject *otherset) } /*[clinic input] +@critical_section set.pop so: setobject @@ -655,7 +664,7 @@ Raises KeyError if the set is empty. static PyObject * set_pop_impl(PySetObject *so) -/*[clinic end generated code: output=4d65180f1271871b input=4a3f5552e660a260]*/ +/*[clinic end generated code: output=4d65180f1271871b input=9296c84921125060]*/ { /* Make sure the search finger is in bounds */ setentry *entry = so->table + (so->finger & so->mask); @@ -674,14 +683,15 @@ set_pop_impl(PySetObject *so) key = entry->key; entry->key = dummy; entry->hash = -1; - so->used--; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, so->used - 1); so->finger = entry - so->table + 1; /* next place to start */ return key; } static int -set_traverse(PySetObject *so, visitproc visit, void *arg) +set_traverse(PyObject *self, visitproc visit, void *arg) { + PySetObject *so = _PySet_CAST(self); Py_ssize_t pos = 0; setentry *entry; @@ -705,18 +715,19 @@ _shuffle_bits(Py_uhash_t h) large primes with "interesting bit patterns" and that passed tests for good collision statistics on a variety of problematic datasets including powersets and graph structures (such as David Eppstein's - graph recipes in Lib/test/test_set.py) */ + graph recipes in Lib/test/test_set.py). + + This hash algorithm can be used on either a frozenset or a set. + When it is used on a set, it computes the hash value of the equivalent + frozenset without creating a new frozenset object. */ static Py_hash_t -frozenset_hash(PyObject *self) +frozenset_hash_impl(PyObject *self) { - PySetObject *so = (PySetObject *)self; + PySetObject *so = _PySet_CAST(self); Py_uhash_t hash = 0; setentry *entry; - if (so->hash != -1) - return so->hash; - /* Xor-in shuffled bits from every entry's hash field because xor is commutative and a frozenset hash should be independent of order. @@ -749,6 +760,20 @@ frozenset_hash(PyObject *self) if (hash == (Py_uhash_t)-1) hash = 590923713UL; + return (Py_hash_t)hash; +} + +static Py_hash_t +frozenset_hash(PyObject *self) +{ + PySetObject *so = _PySet_CAST(self); + Py_uhash_t hash; + + if (so->hash != -1) { + return so->hash; + } + + hash = frozenset_hash_impl(self); so->hash = hash; return hash; } @@ -764,8 +789,9 @@ typedef struct { } setiterobject; static void -setiter_dealloc(setiterobject *si) +setiter_dealloc(PyObject *self) { + setiterobject *si = (setiterobject*)self; /* bpo-31095: UnTrack is needed before calling any callbacks */ _PyObject_GC_UNTRACK(si); Py_XDECREF(si->si_set); @@ -773,8 +799,9 @@ setiter_dealloc(setiterobject *si) } static int -setiter_traverse(setiterobject *si, visitproc visit, void *arg) +setiter_traverse(PyObject *self, visitproc visit, void *arg) { + setiterobject *si = (setiterobject*)self; Py_VISIT(si->si_set); return 0; } @@ -790,8 +817,6 @@ setiter_len(setiterobject *si, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); -static PyObject *setiter_iternext(setiterobject *si); - static PyObject * setiter_reduce(setiterobject *si, PyObject *Py_UNUSED(ignored)) { @@ -816,9 +841,10 @@ static PyMethodDef setiter_methods[] = { {NULL, NULL} /* sentinel */ }; -static PyObject *setiter_iternext(setiterobject *si) +static PyObject *setiter_iternext(PyObject *self) { - PyObject *key; + setiterobject *si = (setiterobject*)self; + PyObject *key = NULL; Py_ssize_t i, mask; setentry *entry; PySetObject *so = si->si_set; @@ -827,30 +853,35 @@ static PyObject *setiter_iternext(setiterobject *si) return NULL; assert (PyAnySet_Check(so)); - if (si->si_used != so->used) { + Py_ssize_t so_used = FT_ATOMIC_LOAD_SSIZE(so->used); + Py_ssize_t si_used = FT_ATOMIC_LOAD_SSIZE(si->si_used); + if (si_used != so_used) { PyErr_SetString(PyExc_RuntimeError, "Set changed size during iteration"); si->si_used = -1; /* Make this state sticky */ return NULL; } + Py_BEGIN_CRITICAL_SECTION(so); i = si->si_pos; assert(i>=0); entry = so->table; mask = so->mask; - while (i <= mask && (entry[i].key == NULL || entry[i].key == dummy)) + while (i <= mask && (entry[i].key == NULL || entry[i].key == dummy)) { i++; + } + if (i <= mask) { + key = Py_NewRef(entry[i].key); + } + Py_END_CRITICAL_SECTION(); si->si_pos = i+1; - if (i > mask) - goto fail; + if (key == NULL) { + si->si_set = NULL; + Py_DECREF(so); + return NULL; + } si->len--; - key = entry[i].key; - return Py_NewRef(key); - -fail: - si->si_set = NULL; - Py_DECREF(so); - return NULL; + return key; } PyTypeObject PySetIter_Type = { @@ -859,7 +890,7 @@ PyTypeObject PySetIter_Type = { sizeof(setiterobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)setiter_dealloc, /* tp_dealloc */ + setiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -876,71 +907,73 @@ PyTypeObject PySetIter_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)setiter_traverse, /* tp_traverse */ + setiter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)setiter_iternext, /* tp_iternext */ + setiter_iternext, /* tp_iternext */ setiter_methods, /* tp_methods */ 0, }; static PyObject * -set_iter(PySetObject *so) +set_iter(PyObject *so) { + Py_ssize_t size = set_len(so); setiterobject *si = PyObject_GC_New(setiterobject, &PySetIter_Type); if (si == NULL) return NULL; si->si_set = (PySetObject*)Py_NewRef(so); - si->si_used = so->used; + si->si_used = size; si->si_pos = 0; - si->len = so->used; + si->len = size; _PyObject_GC_TRACK(si); return (PyObject *)si; } static int -set_update_internal(PySetObject *so, PyObject *other) +set_update_dict_lock_held(PySetObject *so, PyObject *other) { - PyObject *key, *it; - - if (PyAnySet_Check(other)) - return set_merge(so, other); + assert(PyDict_CheckExact(other)); - if (PyDict_CheckExact(other)) { - PyObject *value; - Py_ssize_t pos = 0; - Py_hash_t hash; - Py_ssize_t dictsize = PyDict_GET_SIZE(other); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(other); - /* Do one big resize at the start, rather than - * incrementally resizing as we insert new keys. Expect - * that there will be no (or few) overlapping keys. - */ - if (dictsize < 0) + /* Do one big resize at the start, rather than + * incrementally resizing as we insert new keys. Expect + * that there will be no (or few) overlapping keys. + */ + Py_ssize_t dictsize = PyDict_GET_SIZE(other); + if ((so->fill + dictsize)*5 >= so->mask*3) { + if (set_table_resize(so, (so->used + dictsize)*2) != 0) { return -1; - if ((so->fill + dictsize)*5 >= so->mask*3) { - if (set_table_resize(so, (so->used + dictsize)*2) != 0) - return -1; } - int err = 0; - Py_BEGIN_CRITICAL_SECTION(other); - while (_PyDict_Next(other, &pos, &key, &value, &hash)) { - if (set_add_entry(so, key, hash)) { - err = -1; - goto exit; - } + } + + Py_ssize_t pos = 0; + PyObject *key; + PyObject *value; + Py_hash_t hash; + while (_PyDict_Next(other, &pos, &key, &value, &hash)) { + if (set_add_entry(so, key, hash)) { + return -1; } -exit: - Py_END_CRITICAL_SECTION(); - return err; } + return 0; +} - it = PyObject_GetIter(other); - if (it == NULL) +static int +set_update_iterable_lock_held(PySetObject *so, PyObject *other) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); + + PyObject *it = PyObject_GetIter(other); + if (it == NULL) { return -1; + } + PyObject *key; while ((key = PyIter_Next(it)) != NULL) { if (set_add_key(so, key)) { Py_DECREF(it); @@ -955,23 +988,86 @@ set_update_internal(PySetObject *so, PyObject *other) return 0; } +static int +set_update_lock_held(PySetObject *so, PyObject *other) +{ + if (PyAnySet_Check(other)) { + return set_merge_lock_held(so, other); + } + else if (PyDict_CheckExact(other)) { + return set_update_dict_lock_held(so, other); + } + return set_update_iterable_lock_held(so, other); +} + +// set_update for a `so` that is only visible to the current thread +static int +set_update_local(PySetObject *so, PyObject *other) +{ + assert(Py_REFCNT(so) == 1); + if (PyAnySet_Check(other)) { + int rv; + Py_BEGIN_CRITICAL_SECTION(other); + rv = set_merge_lock_held(so, other); + Py_END_CRITICAL_SECTION(); + return rv; + } + else if (PyDict_CheckExact(other)) { + int rv; + Py_BEGIN_CRITICAL_SECTION(other); + rv = set_update_dict_lock_held(so, other); + Py_END_CRITICAL_SECTION(); + return rv; + } + return set_update_iterable_lock_held(so, other); +} + +static int +set_update_internal(PySetObject *so, PyObject *other) +{ + if (PyAnySet_Check(other)) { + if (Py_Is((PyObject *)so, other)) { + return 0; + } + int rv; + Py_BEGIN_CRITICAL_SECTION2(so, other); + rv = set_merge_lock_held(so, other); + Py_END_CRITICAL_SECTION2(); + return rv; + } + else if (PyDict_CheckExact(other)) { + int rv; + Py_BEGIN_CRITICAL_SECTION2(so, other); + rv = set_update_dict_lock_held(so, other); + Py_END_CRITICAL_SECTION2(); + return rv; + } + else { + int rv; + Py_BEGIN_CRITICAL_SECTION(so); + rv = set_update_iterable_lock_held(so, other); + Py_END_CRITICAL_SECTION(); + return rv; + } +} + /*[clinic input] set.update so: setobject - *others as args: object - / + *others: array Update the set, adding elements from all others. [clinic start generated code]*/ static PyObject * -set_update_impl(PySetObject *so, PyObject *args) -/*[clinic end generated code: output=34f6371704974c8a input=eb47c4fbaeb3286e]*/ +set_update_impl(PySetObject *so, PyObject * const *others, + Py_ssize_t others_length) +/*[clinic end generated code: output=017c781c992d5c23 input=ed5d78885b076636]*/ { Py_ssize_t i; - for (i=0 ; iweakreflist = NULL; if (iterable != NULL) { - if (set_update_internal(so, iterable)) { + if (set_update_local(so, iterable)) { Py_DECREF(so); return NULL; } @@ -1099,7 +1195,9 @@ set_swap_bodies(PySetObject *a, PySetObject *b) Py_hash_t h; t = a->fill; a->fill = b->fill; b->fill = t; - t = a->used; a->used = b->used; b->used = t; + t = a->used; + FT_ATOMIC_STORE_SSIZE_RELAXED(a->used, b->used); + FT_ATOMIC_STORE_SSIZE_RELAXED(b->used, t); t = a->mask; a->mask = b->mask; b->mask = t; u = a->table; @@ -1126,6 +1224,7 @@ set_swap_bodies(PySetObject *a, PySetObject *b) } /*[clinic input] +@critical_section set.copy so: setobject @@ -1134,12 +1233,22 @@ Return a shallow copy of a set. static PyObject * set_copy_impl(PySetObject *so) -/*[clinic end generated code: output=c9223a1e1cc6b041 input=2b80b288d47b8cf1]*/ +/*[clinic end generated code: output=c9223a1e1cc6b041 input=c169a4fbb8209257]*/ { - return make_new_set_basetype(Py_TYPE(so), (PyObject *)so); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); + PyObject *copy = make_new_set_basetype(Py_TYPE(so), NULL); + if (copy == NULL) { + return NULL; + } + if (set_merge_lock_held((PySetObject *)copy, (PyObject *)so) < 0) { + Py_DECREF(copy); + return NULL; + } + return copy; } /*[clinic input] +@critical_section frozenset.copy so: setobject @@ -1148,15 +1257,16 @@ Return a shallow copy of a set. static PyObject * frozenset_copy_impl(PySetObject *so) -/*[clinic end generated code: output=b356263526af9e70 input=3dc65577d344eff7]*/ +/*[clinic end generated code: output=b356263526af9e70 input=fbf5bef131268dd7]*/ { if (PyFrozenSet_CheckExact(so)) { return Py_NewRef(so); } - return set_copy(so, NULL); + return set_copy_impl(so); } /*[clinic input] +@critical_section set.clear so: setobject @@ -1165,38 +1275,38 @@ Remove all elements from this set. static PyObject * set_clear_impl(PySetObject *so) -/*[clinic end generated code: output=4e71d5a83904161a input=74ac19794da81a39]*/ +/*[clinic end generated code: output=4e71d5a83904161a input=c6f831b366111950]*/ { - set_clear_internal(so); + set_clear_internal((PyObject*)so); Py_RETURN_NONE; } /*[clinic input] set.union so: setobject - *others as args: object - / + *others: array Return a new set with elements from the set and all others. [clinic start generated code]*/ static PyObject * -set_union_impl(PySetObject *so, PyObject *args) -/*[clinic end generated code: output=2c83d05a446a1477 input=2e2024fa1e40ac84]*/ +set_union_impl(PySetObject *so, PyObject * const *others, + Py_ssize_t others_length) +/*[clinic end generated code: output=b1bfa3d74065f27e input=55a2e81db6347a4f]*/ { PySetObject *result; PyObject *other; Py_ssize_t i; - result = (PySetObject *)set_copy(so, NULL); + result = (PySetObject *)set_copy((PyObject *)so, NULL); if (result == NULL) return NULL; - for (i=0 ; ikey); + Py_hash_t hash = entry->hash; + int rv = set_discard_entry(so, key, hash); + if (rv < 0) { + Py_DECREF(key); + return -1; + } + if (rv == DISCARD_NOTFOUND) { + if (set_add_entry(so, key, hash)) { + Py_DECREF(key); + return -1; + } + } + Py_DECREF(key); + } + return 0; } /*[clinic input] @@ -1739,58 +1927,41 @@ static PyObject * set_symmetric_difference_update(PySetObject *so, PyObject *other) /*[clinic end generated code: output=fbb049c0806028de input=a50acf0365e1f0a5]*/ { - PySetObject *otherset; - PyObject *key; - Py_ssize_t pos = 0; - Py_hash_t hash; - setentry *entry; - int rv; - - if ((PyObject *)so == other) - return set_clear(so, NULL); + if (Py_Is((PyObject *)so, other)) { + return set_clear((PyObject *)so, NULL); + } + int rv; if (PyDict_CheckExact(other)) { - PyObject *res; - - Py_BEGIN_CRITICAL_SECTION(other); - res = set_symmetric_difference_update_dict(so, other); - Py_END_CRITICAL_SECTION(); - - return res; + Py_BEGIN_CRITICAL_SECTION2(so, other); + rv = set_symmetric_difference_update_dict(so, other); + Py_END_CRITICAL_SECTION2(); } - - if (PyAnySet_Check(other)) { - otherset = (PySetObject *)Py_NewRef(other); - } else { - otherset = (PySetObject *)make_new_set_basetype(Py_TYPE(so), other); - if (otherset == NULL) - return NULL; + else if (PyAnySet_Check(other)) { + Py_BEGIN_CRITICAL_SECTION2(so, other); + rv = set_symmetric_difference_update_set(so, (PySetObject *)other); + Py_END_CRITICAL_SECTION2(); } - - while (set_next(otherset, &pos, &entry)) { - key = entry->key; - hash = entry->hash; - Py_INCREF(key); - rv = set_discard_entry(so, key, hash); - if (rv < 0) { - Py_DECREF(otherset); - Py_DECREF(key); + else { + PySetObject *otherset = (PySetObject *)make_new_set_basetype(Py_TYPE(so), other); + if (otherset == NULL) { return NULL; } - if (rv == DISCARD_NOTFOUND) { - if (set_add_entry(so, key, hash)) { - Py_DECREF(otherset); - Py_DECREF(key); - return NULL; - } - } - Py_DECREF(key); + + Py_BEGIN_CRITICAL_SECTION(so); + rv = set_symmetric_difference_update_set(so, otherset); + Py_END_CRITICAL_SECTION(); + + Py_DECREF(otherset); + } + if (rv < 0) { + return NULL; } - Py_DECREF(otherset); Py_RETURN_NONE; } /*[clinic input] +@critical_section so other set.symmetric_difference so: setobject other: object @@ -1800,39 +1971,42 @@ Return a new set with elements in either the set or other but not both. [clinic start generated code]*/ static PyObject * -set_symmetric_difference(PySetObject *so, PyObject *other) -/*[clinic end generated code: output=f95364211b88775a input=f18af370ad72ebac]*/ +set_symmetric_difference_impl(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=270ee0b5d42b0797 input=624f6e7bbdf70db1]*/ { - PyObject *rv; - PySetObject *otherset; - - otherset = (PySetObject *)make_new_set_basetype(Py_TYPE(so), other); - if (otherset == NULL) + PySetObject *result = (PySetObject *)make_new_set_basetype(Py_TYPE(so), NULL); + if (result == NULL) { return NULL; - rv = set_symmetric_difference_update(otherset, (PyObject *)so); - if (rv == NULL) { - Py_DECREF(otherset); + } + if (set_update_lock_held(result, other) < 0) { + Py_DECREF(result); return NULL; } - Py_DECREF(rv); - return (PyObject *)otherset; + if (set_symmetric_difference_update_set(result, so) < 0) { + Py_DECREF(result); + return NULL; + } + return (PyObject *)result; } static PyObject * -set_xor(PySetObject *so, PyObject *other) +set_xor(PyObject *self, PyObject *other) { - if (!PyAnySet_Check(so) || !PyAnySet_Check(other)) + if (!PyAnySet_Check(self) || !PyAnySet_Check(other)) Py_RETURN_NOTIMPLEMENTED; + PySetObject *so = _PySet_CAST(self); return set_symmetric_difference(so, other); } static PyObject * -set_ixor(PySetObject *so, PyObject *other) +set_ixor(PyObject *self, PyObject *other) { PyObject *result; if (!PyAnySet_Check(other)) Py_RETURN_NOTIMPLEMENTED; + PySetObject *so = _PySet_CAST(self); + result = set_symmetric_difference_update(so, other); if (result == NULL) return NULL; @@ -1841,6 +2015,7 @@ set_ixor(PySetObject *so, PyObject *other) } /*[clinic input] +@critical_section so other set.issubset so: setobject other: object @@ -1850,8 +2025,8 @@ Report whether another set contains this set. [clinic start generated code]*/ static PyObject * -set_issubset(PySetObject *so, PyObject *other) -/*[clinic end generated code: output=78aef1f377aedef1 input=37fbc579b609db0c]*/ +set_issubset_impl(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=b2b59d5f314555ce input=f2a4fd0f2537758b]*/ { setentry *entry; Py_ssize_t pos = 0; @@ -1885,6 +2060,7 @@ set_issubset(PySetObject *so, PyObject *other) } /*[clinic input] +@critical_section so other set.issuperset so: setobject other: object @@ -1894,8 +2070,8 @@ Report whether this set contains another set. [clinic start generated code]*/ static PyObject * -set_issuperset(PySetObject *so, PyObject *other) -/*[clinic end generated code: output=7d2b71dd714a7ec7 input=fd5dab052f2e9bb3]*/ +set_issuperset_impl(PySetObject *so, PyObject *other) +/*[clinic end generated code: output=ecf00ce552c09461 input=5f2e1f262e6e4ccc]*/ { if (PyAnySet_Check(other)) { return set_issubset((PySetObject *)other, (PyObject *)so); @@ -1925,8 +2101,9 @@ set_issuperset(PySetObject *so, PyObject *other) } static PyObject * -set_richcompare(PySetObject *v, PyObject *w, int op) +set_richcompare(PyObject *self, PyObject *w, int op) { + PySetObject *v = _PySet_CAST(self); PyObject *r1; int r2; @@ -1943,7 +2120,7 @@ set_richcompare(PySetObject *v, PyObject *w, int op) Py_RETURN_FALSE; return set_issubset(v, w); case Py_NE: - r1 = set_richcompare(v, w, Py_EQ); + r1 = set_richcompare((PyObject*)v, w, Py_EQ); if (r1 == NULL) return NULL; r2 = PyObject_IsTrue(r1); @@ -1968,6 +2145,7 @@ set_richcompare(PySetObject *v, PyObject *w, int op) } /*[clinic input] +@critical_section set.add so: setobject object as key: object @@ -1979,8 +2157,8 @@ This has no effect if the element is already present. [clinic start generated code]*/ static PyObject * -set_add(PySetObject *so, PyObject *key) -/*[clinic end generated code: output=cd9c2d5c2069c2ba input=96f1efe029e47972]*/ +set_add_impl(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=4cc4a937f1425c96 input=03baf62cb0e66514]*/ { if (set_add_key(so, key)) return NULL; @@ -1988,9 +2166,8 @@ set_add(PySetObject *so, PyObject *key) } static int -set_contains(PySetObject *so, PyObject *key) +set_contains_lock_held(PySetObject *so, PyObject *key) { - PyObject *tmpkey; int rv; rv = set_contains_key(so, key); @@ -1998,16 +2175,34 @@ set_contains(PySetObject *so, PyObject *key) if (!PySet_Check(key) || !PyErr_ExceptionMatches(PyExc_TypeError)) return -1; PyErr_Clear(); - tmpkey = make_new_set(&PyFrozenSet_Type, key); - if (tmpkey == NULL) - return -1; - rv = set_contains_key(so, tmpkey); - Py_DECREF(tmpkey); + Py_hash_t hash; + Py_BEGIN_CRITICAL_SECTION(key); + hash = frozenset_hash_impl(key); + Py_END_CRITICAL_SECTION(); + rv = set_contains_entry(so, key, hash); } return rv; } +int +_PySet_Contains(PySetObject *so, PyObject *key) +{ + int rv; + Py_BEGIN_CRITICAL_SECTION(so); + rv = set_contains_lock_held(so, key); + Py_END_CRITICAL_SECTION(); + return rv; +} + +static int +set_contains(PyObject *self, PyObject *key) +{ + PySetObject *so = _PySet_CAST(self); + return _PySet_Contains(so, key); +} + /*[clinic input] +@critical_section @coexist set.__contains__ so: setobject @@ -2018,18 +2213,19 @@ x.__contains__(y) <==> y in x. [clinic start generated code]*/ static PyObject * -set___contains__(PySetObject *so, PyObject *key) -/*[clinic end generated code: output=b5948bc5c590d3ca input=cf4c72db704e4cf0]*/ +set___contains___impl(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=b44863d034b3c70e input=4a7d568459617f24]*/ { long result; - result = set_contains(so, key); + result = set_contains_lock_held(so, key); if (result < 0) return NULL; return PyBool_FromLong(result); } /*[clinic input] +@critical_section set.remove so: setobject object as key: object @@ -2041,10 +2237,9 @@ If the element is not a member, raise a KeyError. [clinic start generated code]*/ static PyObject * -set_remove(PySetObject *so, PyObject *key) -/*[clinic end generated code: output=08ae496d0cd2b8c1 input=10132515dfe8ebd7]*/ +set_remove_impl(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=0b9134a2a2200363 input=893e1cb1df98227a]*/ { - PyObject *tmpkey; int rv; rv = set_discard_key(so, key); @@ -2052,11 +2247,11 @@ set_remove(PySetObject *so, PyObject *key) if (!PySet_Check(key) || !PyErr_ExceptionMatches(PyExc_TypeError)) return NULL; PyErr_Clear(); - tmpkey = make_new_set(&PyFrozenSet_Type, key); - if (tmpkey == NULL) - return NULL; - rv = set_discard_key(so, tmpkey); - Py_DECREF(tmpkey); + Py_hash_t hash; + Py_BEGIN_CRITICAL_SECTION(key); + hash = frozenset_hash_impl(key); + Py_END_CRITICAL_SECTION(); + rv = set_discard_entry(so, key, hash); if (rv < 0) return NULL; } @@ -2069,6 +2264,7 @@ set_remove(PySetObject *so, PyObject *key) } /*[clinic input] +@critical_section set.discard so: setobject object as key: object @@ -2081,10 +2277,9 @@ an exception when an element is missing from the set. [clinic start generated code]*/ static PyObject * -set_discard(PySetObject *so, PyObject *key) -/*[clinic end generated code: output=9181b60d7bb7d480 input=82a689eba94d5ad9]*/ +set_discard_impl(PySetObject *so, PyObject *key) +/*[clinic end generated code: output=eec3b687bf32759e input=861cb7fb69b4def0]*/ { - PyObject *tmpkey; int rv; rv = set_discard_key(so, key); @@ -2092,11 +2287,11 @@ set_discard(PySetObject *so, PyObject *key) if (!PySet_Check(key) || !PyErr_ExceptionMatches(PyExc_TypeError)) return NULL; PyErr_Clear(); - tmpkey = make_new_set(&PyFrozenSet_Type, key); - if (tmpkey == NULL) - return NULL; - rv = set_discard_key(so, tmpkey); - Py_DECREF(tmpkey); + Py_hash_t hash; + Py_BEGIN_CRITICAL_SECTION(key); + hash = frozenset_hash_impl(key); + Py_END_CRITICAL_SECTION(); + rv = set_discard_entry(so, key, hash); if (rv < 0) return NULL; } @@ -2104,6 +2299,7 @@ set_discard(PySetObject *so, PyObject *key) } /*[clinic input] +@critical_section set.__reduce__ so: setobject @@ -2112,7 +2308,7 @@ Return state information for pickling. static PyObject * set___reduce___impl(PySetObject *so) -/*[clinic end generated code: output=9af7d0e029df87ee input=531375e87a24a449]*/ +/*[clinic end generated code: output=9af7d0e029df87ee input=59405a4249e82f71]*/ { PyObject *keys=NULL, *args=NULL, *result=NULL, *state=NULL; @@ -2134,6 +2330,7 @@ set___reduce___impl(PySetObject *so) } /*[clinic input] +@critical_section set.__sizeof__ so: setobject @@ -2142,7 +2339,7 @@ S.__sizeof__() -> size of S in memory, in bytes. static PyObject * set___sizeof___impl(PySetObject *so) -/*[clinic end generated code: output=4bfa3df7bd38ed88 input=0f214fc2225319fc]*/ +/*[clinic end generated code: output=4bfa3df7bd38ed88 input=09e1a09f168eaa23]*/ { size_t res = _PyObject_SIZE(Py_TYPE(so)); if (so->table != so->smalltable) { @@ -2152,17 +2349,29 @@ set___sizeof___impl(PySetObject *so) } static int -set_init(PySetObject *self, PyObject *args, PyObject *kwds) +set_init(PyObject *so, PyObject *args, PyObject *kwds) { + PySetObject *self = _PySet_CAST(so); PyObject *iterable = NULL; - if (!_PyArg_NoKeywords("set", kwds)) + if (!_PyArg_NoKeywords("set", kwds)) return -1; if (!PyArg_UnpackTuple(args, Py_TYPE(self)->tp_name, 0, 1, &iterable)) return -1; + + if (Py_REFCNT(self) == 1 && self->fill == 0) { + self->hash = -1; + if (iterable == NULL) { + return 0; + } + return set_update_local(self, iterable); + } + Py_BEGIN_CRITICAL_SECTION(self); if (self->fill) - set_clear_internal(self); + set_clear_internal((PyObject*)self); self->hash = -1; + Py_END_CRITICAL_SECTION(); + if (iterable == NULL) return 0; return set_update_internal(self, iterable); @@ -2198,7 +2407,7 @@ static PySequenceMethods set_as_sequence = { 0, /* sq_slice */ 0, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)set_contains, /* sq_contains */ + set_contains, /* sq_contains */ }; /* set object ********************************************************/ @@ -2230,7 +2439,7 @@ static PyMethodDef set_methods[] = { static PyNumberMethods set_as_number = { 0, /*nb_add*/ - (binaryfunc)set_sub, /*nb_subtract*/ + set_sub, /*nb_subtract*/ 0, /*nb_multiply*/ 0, /*nb_remainder*/ 0, /*nb_divmod*/ @@ -2242,22 +2451,22 @@ static PyNumberMethods set_as_number = { 0, /*nb_invert*/ 0, /*nb_lshift*/ 0, /*nb_rshift*/ - (binaryfunc)set_and, /*nb_and*/ - (binaryfunc)set_xor, /*nb_xor*/ - (binaryfunc)set_or, /*nb_or*/ + set_and, /*nb_and*/ + set_xor, /*nb_xor*/ + set_or, /*nb_or*/ 0, /*nb_int*/ 0, /*nb_reserved*/ 0, /*nb_float*/ 0, /*nb_inplace_add*/ - (binaryfunc)set_isub, /*nb_inplace_subtract*/ + set_isub, /*nb_inplace_subtract*/ 0, /*nb_inplace_multiply*/ 0, /*nb_inplace_remainder*/ 0, /*nb_inplace_power*/ 0, /*nb_inplace_lshift*/ 0, /*nb_inplace_rshift*/ - (binaryfunc)set_iand, /*nb_inplace_and*/ - (binaryfunc)set_ixor, /*nb_inplace_xor*/ - (binaryfunc)set_ior, /*nb_inplace_or*/ + set_iand, /*nb_inplace_and*/ + set_ixor, /*nb_inplace_xor*/ + set_ior, /*nb_inplace_or*/ }; PyDoc_STRVAR(set_doc, @@ -2272,12 +2481,12 @@ PyTypeObject PySet_Type = { sizeof(PySetObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)set_dealloc, /* tp_dealloc */ + set_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)set_repr, /* tp_repr */ + set_repr, /* tp_repr */ &set_as_number, /* tp_as_number */ &set_as_sequence, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -2289,13 +2498,13 @@ PyTypeObject PySet_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | - _Py_TPFLAGS_MATCH_SELF, /* tp_flags */ + _Py_TPFLAGS_MATCH_SELF, /* tp_flags */ set_doc, /* tp_doc */ - (traverseproc)set_traverse, /* tp_traverse */ - (inquiry)set_clear_internal, /* tp_clear */ - (richcmpfunc)set_richcompare, /* tp_richcompare */ + set_traverse, /* tp_traverse */ + set_clear_internal, /* tp_clear */ + set_richcompare, /* tp_richcompare */ offsetof(PySetObject, weakreflist), /* tp_weaklistoffset */ - (getiterfunc)set_iter, /* tp_iter */ + set_iter, /* tp_iter */ 0, /* tp_iternext */ set_methods, /* tp_methods */ 0, /* tp_members */ @@ -2305,11 +2514,12 @@ PyTypeObject PySet_Type = { 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ - (initproc)set_init, /* tp_init */ + set_init, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ set_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ .tp_vectorcall = set_vectorcall, + .tp_version_tag = _Py_TYPE_VERSION_SET, }; /* frozenset object ********************************************************/ @@ -2333,7 +2543,7 @@ static PyMethodDef frozenset_methods[] = { static PyNumberMethods frozenset_as_number = { 0, /*nb_add*/ - (binaryfunc)set_sub, /*nb_subtract*/ + set_sub, /*nb_subtract*/ 0, /*nb_multiply*/ 0, /*nb_remainder*/ 0, /*nb_divmod*/ @@ -2345,9 +2555,9 @@ static PyNumberMethods frozenset_as_number = { 0, /*nb_invert*/ 0, /*nb_lshift*/ 0, /*nb_rshift*/ - (binaryfunc)set_and, /*nb_and*/ - (binaryfunc)set_xor, /*nb_xor*/ - (binaryfunc)set_or, /*nb_or*/ + set_and, /*nb_and*/ + set_xor, /*nb_xor*/ + set_or, /*nb_or*/ }; PyDoc_STRVAR(frozenset_doc, @@ -2362,12 +2572,12 @@ PyTypeObject PyFrozenSet_Type = { sizeof(PySetObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)set_dealloc, /* tp_dealloc */ + set_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)set_repr, /* tp_repr */ + set_repr, /* tp_repr */ &frozenset_as_number, /* tp_as_number */ &set_as_sequence, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -2379,13 +2589,13 @@ PyTypeObject PyFrozenSet_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | - _Py_TPFLAGS_MATCH_SELF, /* tp_flags */ + _Py_TPFLAGS_MATCH_SELF, /* tp_flags */ frozenset_doc, /* tp_doc */ - (traverseproc)set_traverse, /* tp_traverse */ - (inquiry)set_clear_internal, /* tp_clear */ - (richcmpfunc)set_richcompare, /* tp_richcompare */ + set_traverse, /* tp_traverse */ + set_clear_internal, /* tp_clear */ + set_richcompare, /* tp_richcompare */ offsetof(PySetObject, weakreflist), /* tp_weaklistoffset */ - (getiterfunc)set_iter, /* tp_iter */ + set_iter, /* tp_iter */ 0, /* tp_iternext */ frozenset_methods, /* tp_methods */ 0, /* tp_members */ @@ -2400,6 +2610,7 @@ PyTypeObject PyFrozenSet_Type = { frozenset_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ .tp_vectorcall = frozenset_vectorcall, + .tp_version_tag = _Py_TYPE_VERSION_FROZEN_SET, }; @@ -2424,7 +2635,7 @@ PySet_Size(PyObject *anyset) PyErr_BadInternalCall(); return -1; } - return PySet_GET_SIZE(anyset); + return set_len(anyset); } int @@ -2434,7 +2645,14 @@ PySet_Clear(PyObject *set) PyErr_BadInternalCall(); return -1; } - return set_clear_internal((PySetObject *)set); + (void)set_clear(set, NULL); + return 0; +} + +void +_PySet_ClearInternal(PySetObject *so) +{ + (void)set_clear_internal((PyObject*)so); } int @@ -2444,7 +2662,12 @@ PySet_Contains(PyObject *anyset, PyObject *key) PyErr_BadInternalCall(); return -1; } - return set_contains_key((PySetObject *)anyset, key); + + int rv; + Py_BEGIN_CRITICAL_SECTION(anyset); + rv = set_contains_key((PySetObject *)anyset, key); + Py_END_CRITICAL_SECTION(); + return rv; } int @@ -2454,7 +2677,12 @@ PySet_Discard(PyObject *set, PyObject *key) PyErr_BadInternalCall(); return -1; } - return set_discard_key((PySetObject *)set, key); + + int rv; + Py_BEGIN_CRITICAL_SECTION(set); + rv = set_discard_key((PySetObject *)set, key); + Py_END_CRITICAL_SECTION(); + return rv; } int @@ -2465,7 +2693,12 @@ PySet_Add(PyObject *anyset, PyObject *key) PyErr_BadInternalCall(); return -1; } - return set_add_key((PySetObject *)anyset, key); + + int rv; + Py_BEGIN_CRITICAL_SECTION(anyset); + rv = set_add_key((PySetObject *)anyset, key); + Py_END_CRITICAL_SECTION(); + return rv; } int @@ -2484,6 +2717,23 @@ _PySet_NextEntry(PyObject *set, Py_ssize_t *pos, PyObject **key, Py_hash_t *hash return 1; } +int +_PySet_NextEntryRef(PyObject *set, Py_ssize_t *pos, PyObject **key, Py_hash_t *hash) +{ + setentry *entry; + + if (!PyAnySet_Check(set)) { + PyErr_BadInternalCall(); + return -1; + } + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(set); + if (set_next((PySetObject *)set, pos, &entry) == 0) + return 0; + *key = Py_NewRef(entry->key); + *hash = entry->hash; + return 1; +} + PyObject * PySet_Pop(PyObject *set) { @@ -2491,7 +2741,7 @@ PySet_Pop(PyObject *set) PyErr_BadInternalCall(); return NULL; } - return set_pop((PySetObject *)set, NULL); + return set_pop(set, NULL); } int diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 7333aea91e5648..1b07c2a2c498b9 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -15,6 +15,7 @@ this type and there is exactly one in existence. #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" // _PyObject_GC_TRACK() @@ -57,6 +58,11 @@ static PyMethodDef ellipsis_methods[] = { {NULL, NULL} }; +PyDoc_STRVAR(ellipsis_doc, +"ellipsis()\n" +"--\n\n" +"The type of the Ellipsis singleton."); + PyTypeObject PyEllipsis_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "ellipsis", /* tp_name */ @@ -78,7 +84,7 @@ PyTypeObject PyEllipsis_Type = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /* tp_doc */ + ellipsis_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ @@ -103,20 +109,6 @@ PyObject _Py_EllipsisObject = _PyObject_HEAD_INIT(&PyEllipsis_Type); /* Slice object implementation */ -void _PySlice_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) -{ - if (!is_finalization) { - return; - } -#ifdef WITH_FREELISTS - PySliceObject *obj = freelists->slices.slice_cache; - if (obj != NULL) { - freelists->slices.slice_cache = NULL; - PyObject_GC_Del(obj); - } -#endif -} - /* start, stop, and step are python objects with None indicating no index is present. */ @@ -125,17 +117,8 @@ static PySliceObject * _PyBuildSlice_Consume2(PyObject *start, PyObject *stop, PyObject *step) { assert(start != NULL && stop != NULL && step != NULL); - PySliceObject *obj; -#ifdef WITH_FREELISTS - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - if (freelists->slices.slice_cache != NULL) { - obj = freelists->slices.slice_cache; - freelists->slices.slice_cache = NULL; - _Py_NewReference((PyObject *)obj); - } - else -#endif - { + PySliceObject *obj = _Py_FREELIST_POP(PySliceObject, slices); + if (obj == NULL) { obj = PyObject_GC_New(PySliceObject, &PySlice_Type); if (obj == NULL) { goto error; @@ -360,20 +343,11 @@ Create a slice object. This is used for extended slicing (e.g. a[0:10:2])."); static void slice_dealloc(PySliceObject *r) { - _PyObject_GC_UNTRACK(r); + PyObject_GC_UnTrack(r); Py_DECREF(r->step); Py_DECREF(r->start); Py_DECREF(r->stop); -#ifdef WITH_FREELISTS - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - if (freelists->slices.slice_cache == NULL) { - freelists->slices.slice_cache = r; - } - else -#endif - { - PyObject_GC_Del(r); - } + _Py_FREELIST_FREE(slices, r, PyObject_GC_Del); } static PyObject * @@ -425,11 +399,14 @@ _PySlice_GetLongIndices(PySliceObject *self, PyObject *length, step_is_negative = 0; } else { - int step_sign; step = evaluate_slice_index(self->step); - if (step == NULL) + if (step == NULL) { goto error; - step_sign = _PyLong_Sign(step); + } + assert(PyLong_Check(step)); + + int step_sign; + (void)PyLong_GetSign(step, &step_sign); if (step_sign == 0) { PyErr_SetString(PyExc_ValueError, "slice step cannot be zero"); diff --git a/Objects/stringlib/clinic/transmogrify.h.h b/Objects/stringlib/clinic/transmogrify.h.h index 3a985ab5c7a9f5..5b82c2de737249 100644 --- a/Objects/stringlib/clinic/transmogrify.h.h +++ b/Objects/stringlib/clinic/transmogrify.h.h @@ -56,7 +56,8 @@ stringlib_expandtabs(PyObject *self, PyObject *const *args, Py_ssize_t nargs, Py Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; int tabsize = 8; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -113,10 +114,24 @@ stringlib_ljust(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + if (PyBytes_Check(args[1])) { + if (PyBytes_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "ljust(): argument 2 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[1])); + goto exit; + } fillchar = PyBytes_AS_STRING(args[1])[0]; } - else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + else if (PyByteArray_Check(args[1])) { + if (PyByteArray_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "ljust(): argument 2 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[1])); + goto exit; + } fillchar = PyByteArray_AS_STRING(args[1])[0]; } else { @@ -169,10 +184,24 @@ stringlib_rjust(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + if (PyBytes_Check(args[1])) { + if (PyBytes_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "rjust(): argument 2 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[1])); + goto exit; + } fillchar = PyBytes_AS_STRING(args[1])[0]; } - else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + else if (PyByteArray_Check(args[1])) { + if (PyByteArray_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "rjust(): argument 2 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[1])); + goto exit; + } fillchar = PyByteArray_AS_STRING(args[1])[0]; } else { @@ -225,10 +254,24 @@ stringlib_center(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - if (PyBytes_Check(args[1]) && PyBytes_GET_SIZE(args[1]) == 1) { + if (PyBytes_Check(args[1])) { + if (PyBytes_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "center(): argument 2 must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(args[1])); + goto exit; + } fillchar = PyBytes_AS_STRING(args[1])[0]; } - else if (PyByteArray_Check(args[1]) && PyByteArray_GET_SIZE(args[1]) == 1) { + else if (PyByteArray_Check(args[1])) { + if (PyByteArray_GET_SIZE(args[1]) != 1) { + PyErr_Format(PyExc_TypeError, + "center(): argument 2 must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(args[1])); + goto exit; + } fillchar = PyByteArray_AS_STRING(args[1])[0]; } else { @@ -279,4 +322,4 @@ stringlib_zfill(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=b409bdf9ab68d5a6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0e24a10bac3ec053 input=a9049054013a1b77]*/ diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index f98e71c3fc6ecf..440410d0aef17d 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -331,7 +331,7 @@ STRINGLIB(utf8_encoder)(_PyBytesWriter *writer, case _Py_ERROR_REPLACE: memset(p, '?', endpos - startpos); p += (endpos - startpos); - /* fall through */ + _Py_FALLTHROUGH; case _Py_ERROR_IGNORE: i += (endpos - startpos - 1); break; @@ -379,7 +379,7 @@ STRINGLIB(utf8_encoder)(_PyBytesWriter *writer, } startpos = k; assert(startpos < endpos); - /* fall through */ + _Py_FALLTHROUGH; default: rep = unicode_encode_call_errorhandler( errors, &error_handler_obj, "utf-8", "surrogates not allowed", diff --git a/Objects/stringlib/eq.h b/Objects/stringlib/eq.h index 2eac4baf5ca9ce..821b692f26b830 100644 --- a/Objects/stringlib/eq.h +++ b/Objects/stringlib/eq.h @@ -4,14 +4,19 @@ * unicode_eq() is called when the hash of two unicode objects is equal. */ Py_LOCAL_INLINE(int) -unicode_eq(PyObject *a, PyObject *b) +unicode_eq(PyObject *str1, PyObject *str2) { - if (PyUnicode_GET_LENGTH(a) != PyUnicode_GET_LENGTH(b)) + Py_ssize_t len = PyUnicode_GET_LENGTH(str1); + if (PyUnicode_GET_LENGTH(str2) != len) { return 0; - if (PyUnicode_GET_LENGTH(a) == 0) - return 1; - if (PyUnicode_KIND(a) != PyUnicode_KIND(b)) + } + + int kind = PyUnicode_KIND(str1); + if (PyUnicode_KIND(str2) != kind) { return 0; - return memcmp(PyUnicode_1BYTE_DATA(a), PyUnicode_1BYTE_DATA(b), - PyUnicode_GET_LENGTH(a) * PyUnicode_KIND(a)) == 0; + } + + const void *data1 = PyUnicode_DATA(str1); + const void *data2 = PyUnicode_DATA(str2); + return (memcmp(data1, data2, len * kind) == 0); } diff --git a/Objects/stringlib/fastsearch.h b/Objects/stringlib/fastsearch.h index 257b7bd6788ad2..05e700b06258f0 100644 --- a/Objects/stringlib/fastsearch.h +++ b/Objects/stringlib/fastsearch.h @@ -256,7 +256,7 @@ STRINGLIB(_factorize)(const STRINGLIB_CHAR *needle, The local period of the cut is the minimal length of a string w such that (left endswith w or w endswith left) - and (right startswith w or w startswith left). + and (right startswith w or w startswith right). The Critical Factorization Theorem says that this maximal local period is the global period of the string. @@ -337,21 +337,20 @@ STRINGLIB(_preprocess)(const STRINGLIB_CHAR *needle, Py_ssize_t len_needle, if (p->is_periodic) { assert(p->cut <= len_needle/2); assert(p->cut < p->period); - p->gap = 0; // unused } else { // A lower bound on the period p->period = Py_MAX(p->cut, len_needle - p->cut) + 1; - // The gap between the last character and the previous - // occurrence of an equivalent character (modulo TABLE_SIZE) - p->gap = len_needle; - STRINGLIB_CHAR last = needle[len_needle - 1] & TABLE_MASK; - for (Py_ssize_t i = len_needle - 2; i >= 0; i--) { - STRINGLIB_CHAR x = needle[i] & TABLE_MASK; - if (x == last) { - p->gap = len_needle - 1 - i; - break; - } + } + // The gap between the last character and the previous + // occurrence of an equivalent character (modulo TABLE_SIZE) + p->gap = len_needle; + STRINGLIB_CHAR last = needle[len_needle - 1] & TABLE_MASK; + for (Py_ssize_t i = len_needle - 2; i >= 0; i--) { + STRINGLIB_CHAR x = needle[i] & TABLE_MASK; + if (x == last) { + p->gap = len_needle - 1 - i; + break; } } // Fill up a compressed Boyer-Moore "Bad Character" table @@ -383,6 +382,8 @@ STRINGLIB(_two_way)(const STRINGLIB_CHAR *haystack, Py_ssize_t len_haystack, const STRINGLIB_CHAR *window; LOG("===== Two-way: \"%s\" in \"%s\". =====\n", needle, haystack); + Py_ssize_t gap = p->gap; + Py_ssize_t gap_jump_end = Py_MIN(len_needle, cut + gap); if (p->is_periodic) { LOG("Needle is periodic.\n"); Py_ssize_t memory = 0; @@ -408,8 +409,16 @@ STRINGLIB(_two_way)(const STRINGLIB_CHAR *haystack, Py_ssize_t len_haystack, Py_ssize_t i = Py_MAX(cut, memory); for (; i < len_needle; i++) { if (needle[i] != window[i]) { - LOG("Right half does not match.\n"); - window_last += i - cut + 1; + if (i < gap_jump_end) { + LOG("Early right half mismatch: jump by gap.\n"); + assert(gap >= i - cut + 1); + window_last += gap; + } + else { + LOG("Late right half mismatch: jump by n (>gap)\n"); + assert(i - cut + 1 > gap); + window_last += i - cut + 1; + } memory = 0; goto periodicwindowloop; } @@ -442,10 +451,8 @@ STRINGLIB(_two_way)(const STRINGLIB_CHAR *haystack, Py_ssize_t len_haystack, } } else { - Py_ssize_t gap = p->gap; period = Py_MAX(gap, period); LOG("Needle is not periodic.\n"); - Py_ssize_t gap_jump_end = Py_MIN(len_needle, cut + gap); windowloop: while (window_last < haystack_end) { for (;;) { @@ -463,19 +470,19 @@ STRINGLIB(_two_way)(const STRINGLIB_CHAR *haystack, Py_ssize_t len_haystack, window = window_last - len_needle + 1; assert((window[len_needle - 1] & TABLE_MASK) == (needle[len_needle - 1] & TABLE_MASK)); - for (Py_ssize_t i = cut; i < gap_jump_end; i++) { - if (needle[i] != window[i]) { - LOG("Early right half mismatch: jump by gap.\n"); - assert(gap >= i - cut + 1); - window_last += gap; - goto windowloop; - } - } - for (Py_ssize_t i = gap_jump_end; i < len_needle; i++) { + Py_ssize_t i = cut; + for (; i < len_needle; i++) { if (needle[i] != window[i]) { - LOG("Late right half mismatch.\n"); - assert(i - cut + 1 > gap); - window_last += i - cut + 1; + if (i < gap_jump_end) { + LOG("Early right half mismatch: jump by gap.\n"); + assert(gap >= i - cut + 1); + window_last += gap; + } + else { + LOG("Late right half mismatch: jump by n (>gap)\n"); + assert(i - cut + 1 > gap); + window_last += i - cut + 1; + } goto windowloop; } } @@ -746,6 +753,22 @@ STRINGLIB(count_char)(const STRINGLIB_CHAR *s, Py_ssize_t n, } +static inline Py_ssize_t +STRINGLIB(count_char_no_maxcount)(const STRINGLIB_CHAR *s, Py_ssize_t n, + const STRINGLIB_CHAR p0) +/* A specialized function of count_char that does not cut off at a maximum. + As a result, the compiler is able to vectorize the loop. */ +{ + Py_ssize_t count = 0; + for (Py_ssize_t i = 0; i < n; i++) { + if (s[i] == p0) { + count++; + } + } + return count; +} + + Py_LOCAL_INLINE(Py_ssize_t) FASTSEARCH(const STRINGLIB_CHAR* s, Py_ssize_t n, const STRINGLIB_CHAR* p, Py_ssize_t m, @@ -766,6 +789,9 @@ FASTSEARCH(const STRINGLIB_CHAR* s, Py_ssize_t n, else if (mode == FAST_RSEARCH) return STRINGLIB(rfind_char)(s, n, p[0]); else { + if (maxcount == PY_SSIZE_T_MAX) { + return STRINGLIB(count_char_no_maxcount)(s, n, p[0]); + } return STRINGLIB(count_char)(s, n, p[0], maxcount); } } diff --git a/Objects/stringlib/find.h b/Objects/stringlib/find.h index 509b9297396be8..c385718a5b2692 100644 --- a/Objects/stringlib/find.h +++ b/Objects/stringlib/find.h @@ -70,50 +70,3 @@ STRINGLIB(contains_obj)(PyObject* str, PyObject* sub) } #endif /* STRINGLIB_WANT_CONTAINS_OBJ */ - -/* -This function is a helper for the "find" family (find, rfind, index, -rindex) and for count, startswith and endswith, because they all have -the same behaviour for the arguments. - -It does not touch the variables received until it knows everything -is ok. -*/ - -#define FORMAT_BUFFER_SIZE 50 - -Py_LOCAL_INLINE(int) -STRINGLIB(parse_args_finds)(const char * function_name, PyObject *args, - PyObject **subobj, - Py_ssize_t *start, Py_ssize_t *end) -{ - PyObject *tmp_subobj; - Py_ssize_t tmp_start = 0; - Py_ssize_t tmp_end = PY_SSIZE_T_MAX; - PyObject *obj_start=Py_None, *obj_end=Py_None; - char format[FORMAT_BUFFER_SIZE] = "O|OO:"; - size_t len = strlen(format); - - strncpy(format + len, function_name, FORMAT_BUFFER_SIZE - len - 1); - format[FORMAT_BUFFER_SIZE - 1] = '\0'; - - if (!PyArg_ParseTuple(args, format, &tmp_subobj, &obj_start, &obj_end)) - return 0; - - /* To support None in "start" and "end" arguments, meaning - the same as if they were not passed. - */ - if (obj_start != Py_None) - if (!_PyEval_SliceIndex(obj_start, &tmp_start)) - return 0; - if (obj_end != Py_None) - if (!_PyEval_SliceIndex(obj_end, &tmp_end)) - return 0; - - *start = tmp_start; - *end = tmp_end; - *subobj = tmp_subobj; - return 1; -} - -#undef FORMAT_BUFFER_SIZE diff --git a/Objects/stringlib/find_max_char.h b/Objects/stringlib/find_max_char.h index b9ffdfc2e352ce..7ab3fc88b331b1 100644 --- a/Objects/stringlib/find_max_char.h +++ b/Objects/stringlib/find_max_char.h @@ -1,6 +1,7 @@ /* Finding the optimal width of unicode characters in a buffer */ -#if !STRINGLIB_IS_UNICODE +/* find_max_char for one-byte will work for bytes objects as well. */ +#if !STRINGLIB_IS_UNICODE && STRINGLIB_SIZEOF_CHAR > 1 # error "find_max_char.h is specific to Unicode" #endif @@ -20,19 +21,20 @@ Py_LOCAL_INLINE(Py_UCS4) STRINGLIB(find_max_char)(const STRINGLIB_CHAR *begin, const STRINGLIB_CHAR *end) { const unsigned char *p = (const unsigned char *) begin; + const unsigned char *_end = (const unsigned char *)end; - while (p < end) { + while (p < _end) { if (_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T)) { /* Help register allocation */ const unsigned char *_p = p; - while (_p + SIZEOF_SIZE_T <= end) { + while (_p + SIZEOF_SIZE_T <= _end) { size_t value = *(const size_t *) _p; if (value & UCS1_ASCII_CHAR_MASK) return 255; _p += SIZEOF_SIZE_T; } p = _p; - if (p == end) + if (p == _end) break; } if (*p++ & 0x80) diff --git a/Objects/stringlib/repr.h b/Objects/stringlib/repr.h new file mode 100644 index 00000000000000..87b1a8ba629dc6 --- /dev/null +++ b/Objects/stringlib/repr.h @@ -0,0 +1,95 @@ +/* stringlib: repr() implementation */ + +#ifndef STRINGLIB_FASTSEARCH_H +#error must include "stringlib/fastsearch.h" before including this module +#endif + + +static void +STRINGLIB(repr)(PyObject *unicode, Py_UCS4 quote, + STRINGLIB_CHAR *odata) +{ + Py_ssize_t isize = PyUnicode_GET_LENGTH(unicode); + const void *idata = PyUnicode_DATA(unicode); + int ikind = PyUnicode_KIND(unicode); + + *odata++ = quote; + for (Py_ssize_t i = 0; i < isize; i++) { + Py_UCS4 ch = PyUnicode_READ(ikind, idata, i); + + /* Escape quotes and backslashes */ + if ((ch == quote) || (ch == '\\')) { + *odata++ = '\\'; + *odata++ = ch; + continue; + } + + /* Map special whitespace to '\t', \n', '\r' */ + if (ch == '\t') { + *odata++ = '\\'; + *odata++ = 't'; + } + else if (ch == '\n') { + *odata++ = '\\'; + *odata++ = 'n'; + } + else if (ch == '\r') { + *odata++ = '\\'; + *odata++ = 'r'; + } + + /* Map non-printable US ASCII to '\xhh' */ + else if (ch < ' ' || ch == 0x7F) { + *odata++ = '\\'; + *odata++ = 'x'; + *odata++ = Py_hexdigits[(ch >> 4) & 0x000F]; + *odata++ = Py_hexdigits[ch & 0x000F]; + } + + /* Copy ASCII characters as-is */ + else if (ch < 0x7F) { + *odata++ = ch; + } + + /* Non-ASCII characters */ + else { + /* Map Unicode whitespace and control characters + (categories Z* and C* except ASCII space) + */ + if (!Py_UNICODE_ISPRINTABLE(ch)) { + *odata++ = '\\'; + /* Map 8-bit characters to '\xhh' */ + if (ch <= 0xff) { + *odata++ = 'x'; + *odata++ = Py_hexdigits[(ch >> 4) & 0x000F]; + *odata++ = Py_hexdigits[ch & 0x000F]; + } + /* Map 16-bit characters to '\uxxxx' */ + else if (ch <= 0xffff) { + *odata++ = 'u'; + *odata++ = Py_hexdigits[(ch >> 12) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 8) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 4) & 0xF]; + *odata++ = Py_hexdigits[ch & 0xF]; + } + /* Map 21-bit characters to '\U00xxxxxx' */ + else { + *odata++ = 'U'; + *odata++ = Py_hexdigits[(ch >> 28) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 24) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 20) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 16) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 12) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 8) & 0xF]; + *odata++ = Py_hexdigits[(ch >> 4) & 0xF]; + *odata++ = Py_hexdigits[ch & 0xF]; + } + } + /* Copy characters as-is */ + else { + *odata++ = ch; + } + } + } + *odata = quote; +} diff --git a/Objects/stringlib/unicode_format.h b/Objects/stringlib/unicode_format.h index 91c71ab5736c25..44b269ba8ceb55 100644 --- a/Objects/stringlib/unicode_format.h +++ b/Objects/stringlib/unicode_format.h @@ -73,7 +73,7 @@ Py_LOCAL_INLINE(PyObject *) SubString_new_object_or_empty(SubString *str) { if (str->str == NULL) { - return PyUnicode_New(0, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } return SubString_new_object(str); } @@ -531,7 +531,7 @@ render_field(PyObject *fieldobj, SubString *format_spec, _PyUnicodeWriter *write format_spec->start, format_spec->end); else - format_spec_object = PyUnicode_New(0, 0); + format_spec_object = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (format_spec_object == NULL) goto done; diff --git a/Objects/structseq.c b/Objects/structseq.c index 581d6ad240885a..56a7851b98788d 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -41,12 +41,20 @@ get_type_attr_as_size(PyTypeObject *tp, PyObject *name) get_type_attr_as_size(tp, &_Py_ID(n_sequence_fields)) #define REAL_SIZE_TP(tp) \ get_type_attr_as_size(tp, &_Py_ID(n_fields)) -#define REAL_SIZE(op) REAL_SIZE_TP(Py_TYPE(op)) +#define REAL_SIZE(op) get_real_size((PyObject *)op) #define UNNAMED_FIELDS_TP(tp) \ get_type_attr_as_size(tp, &_Py_ID(n_unnamed_fields)) #define UNNAMED_FIELDS(op) UNNAMED_FIELDS_TP(Py_TYPE(op)) +static Py_ssize_t +get_real_size(PyObject *op) +{ + // Compute the real size from the visible size (i.e., Py_SIZE()) and the + // number of non-sequence fields accounted for in tp_basicsize. + Py_ssize_t hidden = Py_TYPE(op)->tp_basicsize - offsetof(PyStructSequence, ob_item); + return Py_SIZE(op) + hidden / sizeof(PyObject *); +} PyObject * PyStructSequence_New(PyTypeObject *type) @@ -120,6 +128,9 @@ structseq_dealloc(PyStructSequence *obj) PyObject_GC_UnTrack(obj); PyTypeObject *tp = Py_TYPE(obj); + // gh-122527: We can't use REAL_SIZE_TP() or any macros that access the + // type's dictionary here, because the dictionary may have already been + // cleared by the garbage collector. size = REAL_SIZE(obj); for (i = 0; i < size; ++i) { Py_XDECREF(obj->ob_item[i]); @@ -255,83 +266,66 @@ static PyObject * structseq_repr(PyStructSequence *obj) { PyTypeObject *typ = Py_TYPE(obj); - _PyUnicodeWriter writer; - /* Write "typename(" */ - PyObject *type_name = PyUnicode_DecodeUTF8(typ->tp_name, - strlen(typ->tp_name), - NULL); - if (type_name == NULL) { + // count 5 characters per item: "x=1, " + Py_ssize_t type_name_len = strlen(typ->tp_name); + Py_ssize_t prealloc = (type_name_len + 1 + + VISIBLE_SIZE(obj) * 5 + 1); + PyUnicodeWriter *writer = PyUnicodeWriter_Create(prealloc); + if (writer == NULL) { return NULL; } - _PyUnicodeWriter_Init(&writer); - writer.overallocate = 1; - /* count 5 characters per item: "x=1, " */ - writer.min_length = (PyUnicode_GET_LENGTH(type_name) + 1 - + VISIBLE_SIZE(obj) * 5 + 1); - - if (_PyUnicodeWriter_WriteStr(&writer, type_name) < 0) { - Py_DECREF(type_name); + // Write "typename(" + if (PyUnicodeWriter_WriteUTF8(writer, typ->tp_name, type_name_len) < 0) { goto error; } - Py_DECREF(type_name); - - if (_PyUnicodeWriter_WriteChar(&writer, '(') < 0) { + if (PyUnicodeWriter_WriteChar(writer, '(') < 0) { goto error; } for (Py_ssize_t i=0; i < VISIBLE_SIZE(obj); i++) { if (i > 0) { - /* Write ", " */ - if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) { + // Write ", " + if (PyUnicodeWriter_WriteChar(writer, ',') < 0) { + goto error; + } + if (PyUnicodeWriter_WriteChar(writer, ' ') < 0) { goto error; } } - /* Write "name=repr" */ + // Write name const char *name_utf8 = typ->tp_members[i].name; if (name_utf8 == NULL) { - PyErr_Format(PyExc_SystemError, "In structseq_repr(), member %zd name is NULL" + PyErr_Format(PyExc_SystemError, + "In structseq_repr(), member %zd name is NULL" " for type %.500s", i, typ->tp_name); goto error; } - - PyObject *name = PyUnicode_DecodeUTF8(name_utf8, strlen(name_utf8), NULL); - if (name == NULL) { - goto error; - } - if (_PyUnicodeWriter_WriteStr(&writer, name) < 0) { - Py_DECREF(name); + if (PyUnicodeWriter_WriteUTF8(writer, name_utf8, -1) < 0) { goto error; } - Py_DECREF(name); - if (_PyUnicodeWriter_WriteChar(&writer, '=') < 0) { + // Write "=" + repr(value) + if (PyUnicodeWriter_WriteChar(writer, '=') < 0) { goto error; } - PyObject *value = PyStructSequence_GetItem((PyObject*)obj, i); assert(value != NULL); - PyObject *repr = PyObject_Repr(value); - if (repr == NULL) { + if (PyUnicodeWriter_WriteRepr(writer, value) < 0) { goto error; } - if (_PyUnicodeWriter_WriteStr(&writer, repr) < 0) { - Py_DECREF(repr); - goto error; - } - Py_DECREF(repr); } - if (_PyUnicodeWriter_WriteChar(&writer, ')') < 0) { + if (PyUnicodeWriter_WriteChar(writer, ')') < 0) { goto error; } - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); error: - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); return NULL; } @@ -453,7 +447,9 @@ structseq_replace(PyStructSequence *self, PyObject *args, PyObject *kwargs) static PyMethodDef structseq_methods[] = { {"__reduce__", (PyCFunction)structseq_reduce, METH_NOARGS, NULL}, - {"__replace__", _PyCFunction_CAST(structseq_replace), METH_VARARGS | METH_KEYWORDS, NULL}, + {"__replace__", _PyCFunction_CAST(structseq_replace), METH_VARARGS | METH_KEYWORDS, + PyDoc_STR("__replace__($self, /, **changes)\n--\n\n" + "Return a copy of the structure with new values for the specified fields.")}, {NULL, NULL} // sentinel }; @@ -563,10 +559,14 @@ initialize_members(PyStructSequence_Desc *desc, static void initialize_static_fields(PyTypeObject *type, PyStructSequence_Desc *desc, - PyMemberDef *tp_members, unsigned long tp_flags) + PyMemberDef *tp_members, Py_ssize_t n_members, + unsigned long tp_flags) { type->tp_name = desc->name; - type->tp_basicsize = sizeof(PyStructSequence) - sizeof(PyObject *); + // Account for hidden members in tp_basicsize because they are not + // included in the variable size. + Py_ssize_t n_hidden = n_members - desc->n_in_sequence; + type->tp_basicsize = sizeof(PyStructSequence) + (n_hidden - 1) * sizeof(PyObject *); type->tp_itemsize = sizeof(PyObject *); type->tp_dealloc = (destructor)structseq_dealloc; type->tp_repr = (reprfunc)structseq_repr; @@ -603,6 +603,9 @@ _PyStructSequence_InitBuiltinWithFlags(PyInterpreterState *interp, PyStructSequence_Desc *desc, unsigned long tp_flags) { + if (Py_TYPE(type) == NULL) { + Py_SET_TYPE(type, &PyType_Type); + } Py_ssize_t n_unnamed_members; Py_ssize_t n_members = count_members(desc, &n_unnamed_members); PyMemberDef *members = NULL; @@ -616,9 +619,9 @@ _PyStructSequence_InitBuiltinWithFlags(PyInterpreterState *interp, if (members == NULL) { goto error; } - initialize_static_fields(type, desc, members, tp_flags); + initialize_static_fields(type, desc, members, n_members, tp_flags); - _Py_SetImmortal(type); + _Py_SetImmortal((PyObject *)type); } #ifndef NDEBUG else { @@ -679,7 +682,7 @@ PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) if (members == NULL) { return -1; } - initialize_static_fields(type, desc, members, 0); + initialize_static_fields(type, desc, members, n_members, 0); if (initialize_static_type(type, desc, n_members, n_unnamed_members) < 0) { PyMem_Free(members); return -1; @@ -713,7 +716,7 @@ _PyStructSequence_FiniBuiltin(PyInterpreterState *interp, PyTypeObject *type) return; } - _PyStaticType_Dealloc(interp, type); + _PyStaticType_FiniBuiltin(interp, type); if (_Py_IsMainInterpreter(interp)) { // Undo _PyStructSequence_InitBuiltinWithFlags(). @@ -755,7 +758,8 @@ _PyStructSequence_NewType(PyStructSequence_Desc *desc, unsigned long tp_flags) /* The name in this PyType_Spec is statically allocated so it is */ /* expected that it'll outlive the PyType_Spec */ spec.name = desc->name; - spec.basicsize = sizeof(PyStructSequence) - sizeof(PyObject *); + Py_ssize_t hidden = n_members - desc->n_in_sequence; + spec.basicsize = (int)(sizeof(PyStructSequence) + (hidden - 1) * sizeof(PyObject *)); spec.itemsize = sizeof(PyObject *); spec.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | tp_flags; spec.slots = slots; diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index d9dc00da368a84..60af9e40e3fe83 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_freelist.h" // _Py_FREELIST_PUSH(), _Py_FREELIST_POP() #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_modsupport.h" // _PyArg_NoKwnames() @@ -17,7 +18,6 @@ class tuple "PyTupleObject *" "&PyTuple_Type" #include "clinic/tupleobject.c.h" -static inline PyTupleObject * maybe_freelist_pop(Py_ssize_t); static inline int maybe_freelist_push(PyTupleObject *); @@ -38,22 +38,20 @@ tuple_alloc(Py_ssize_t size) PyErr_BadInternalCall(); return NULL; } -#ifdef Py_DEBUG assert(size != 0); // The empty tuple is statically allocated. -#endif - - PyTupleObject *op = maybe_freelist_pop(size); - if (op == NULL) { - /* Check for overflow */ - if ((size_t)size > ((size_t)PY_SSIZE_T_MAX - (sizeof(PyTupleObject) - - sizeof(PyObject *))) / sizeof(PyObject *)) { - return (PyTupleObject *)PyErr_NoMemory(); + Py_ssize_t index = size - 1; + if (index < PyTuple_MAXSAVESIZE) { + PyTupleObject *op = _Py_FREELIST_POP(PyTupleObject, tuples[index]); + if (op != NULL) { + return op; } - op = PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size); - if (op == NULL) - return NULL; } - return op; + /* Check for overflow */ + if ((size_t)size > ((size_t)PY_SSIZE_T_MAX - (sizeof(PyTupleObject) - + sizeof(PyObject *))) / sizeof(PyObject *)) { + return (PyTupleObject *)PyErr_NoMemory(); + } + return PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size); } // The empty tuple singleton is not tracked by the GC. @@ -183,8 +181,9 @@ PyTuple_Pack(Py_ssize_t n, ...) /* Methods */ static void -tupledealloc(PyTupleObject *op) +tuple_dealloc(PyObject *self) { + PyTupleObject *op = _PyTuple_CAST(self); if (Py_SIZE(op) == 0) { /* The empty tuple is statically allocated. */ if (op == &_Py_SINGLETON(tuple_empty)) { @@ -201,7 +200,7 @@ tupledealloc(PyTupleObject *op) } PyObject_GC_UnTrack(op); - Py_TRASHCAN_BEGIN(op, tupledealloc) + Py_TRASHCAN_BEGIN(op, tuple_dealloc) Py_ssize_t i = Py_SIZE(op); while (--i >= 0) { @@ -216,73 +215,71 @@ tupledealloc(PyTupleObject *op) } static PyObject * -tuplerepr(PyTupleObject *v) +tuple_repr(PyObject *self) { - Py_ssize_t i, n; - _PyUnicodeWriter writer; - - n = Py_SIZE(v); - if (n == 0) + PyTupleObject *v = _PyTuple_CAST(self); + Py_ssize_t n = PyTuple_GET_SIZE(v); + if (n == 0) { return PyUnicode_FromString("()"); + } /* While not mutable, it is still possible to end up with a cycle in a tuple through an object that stores itself within a tuple (and thus infinitely asks for the repr of itself). This should only be possible within a type. */ - i = Py_ReprEnter((PyObject *)v); - if (i != 0) { - return i > 0 ? PyUnicode_FromString("(...)") : NULL; + int res = Py_ReprEnter((PyObject *)v); + if (res != 0) { + return res > 0 ? PyUnicode_FromString("(...)") : NULL; } - _PyUnicodeWriter_Init(&writer); - writer.overallocate = 1; - if (Py_SIZE(v) > 1) { - /* "(" + "1" + ", 2" * (len - 1) + ")" */ - writer.min_length = 1 + 1 + (2 + 1) * (Py_SIZE(v) - 1) + 1; + Py_ssize_t prealloc; + if (n > 1) { + // "(" + "1" + ", 2" * (len - 1) + ")" + prealloc = 1 + 1 + (2 + 1) * (n - 1) + 1; } else { - /* "(1,)" */ - writer.min_length = 4; + // "(1,)" + prealloc = 4; + } + PyUnicodeWriter *writer = PyUnicodeWriter_Create(prealloc); + if (writer == NULL) { + goto error; } - if (_PyUnicodeWriter_WriteChar(&writer, '(') < 0) + if (PyUnicodeWriter_WriteChar(writer, '(') < 0) { goto error; + } /* Do repr() on each element. */ - for (i = 0; i < n; ++i) { - PyObject *s; - + for (Py_ssize_t i = 0; i < n; ++i) { if (i > 0) { - if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) + if (PyUnicodeWriter_WriteChar(writer, ',') < 0) { goto error; + } + if (PyUnicodeWriter_WriteChar(writer, ' ') < 0) { + goto error; + } } - s = PyObject_Repr(v->ob_item[i]); - if (s == NULL) - goto error; - - if (_PyUnicodeWriter_WriteStr(&writer, s) < 0) { - Py_DECREF(s); + if (PyUnicodeWriter_WriteRepr(writer, v->ob_item[i]) < 0) { goto error; } - Py_DECREF(s); } - writer.overallocate = 0; - if (n > 1) { - if (_PyUnicodeWriter_WriteChar(&writer, ')') < 0) + if (n == 1) { + if (PyUnicodeWriter_WriteChar(writer, ',') < 0) { goto error; + } } - else { - if (_PyUnicodeWriter_WriteASCIIString(&writer, ",)", 2) < 0) - goto error; + if (PyUnicodeWriter_WriteChar(writer, ')') < 0) { + goto error; } Py_ReprLeave((PyObject *)v); - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); error: - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); Py_ReprLeave((PyObject *)v); return NULL; } @@ -318,13 +315,14 @@ tuplerepr(PyTupleObject *v) /* Tests have shown that it's not worth to cache the hash value, see https://bugs.python.org/issue9685 */ static Py_hash_t -tuplehash(PyTupleObject *v) +tuple_hash(PyObject *op) { - Py_ssize_t i, len = Py_SIZE(v); + PyTupleObject *v = _PyTuple_CAST(op); + Py_ssize_t len = Py_SIZE(v); PyObject **item = v->ob_item; Py_uhash_t acc = _PyHASH_XXPRIME_5; - for (i = 0; i < len; i++) { + for (Py_ssize_t i = 0; i < len; i++) { Py_uhash_t lane = PyObject_Hash(item[i]); if (lane == (Py_uhash_t)-1) { return -1; @@ -344,25 +342,27 @@ tuplehash(PyTupleObject *v) } static Py_ssize_t -tuplelength(PyTupleObject *a) +tuple_length(PyObject *self) { + PyTupleObject *a = _PyTuple_CAST(self); return Py_SIZE(a); } static int -tuplecontains(PyTupleObject *a, PyObject *el) +tuple_contains(PyObject *self, PyObject *el) { - Py_ssize_t i; - int cmp; - - for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(a); ++i) + PyTupleObject *a = _PyTuple_CAST(self); + int cmp = 0; + for (Py_ssize_t i = 0; cmp == 0 && i < Py_SIZE(a); ++i) { cmp = PyObject_RichCompareBool(PyTuple_GET_ITEM(a, i), el, Py_EQ); + } return cmp; } static PyObject * -tupleitem(PyTupleObject *a, Py_ssize_t i) +tuple_item(PyObject *op, Py_ssize_t i) { + PyTupleObject *a = _PyTuple_CAST(op); if (i < 0 || i >= Py_SIZE(a)) { PyErr_SetString(PyExc_IndexError, "tuple index out of range"); return NULL; @@ -390,6 +390,24 @@ _PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) return (PyObject *)tuple; } +PyObject * +_PyTuple_FromStackRefStealOnSuccess(const _PyStackRef *src, Py_ssize_t n) +{ + if (n == 0) { + return tuple_get_empty(); + } + PyTupleObject *tuple = tuple_alloc(n); + if (tuple == NULL) { + return NULL; + } + PyObject **dst = tuple->ob_item; + for (Py_ssize_t i = 0; i < n; i++) { + dst[i] = PyStackRef_AsPyObjectSteal(src[i]); + } + _PyObject_GC_TRACK(tuple); + return (PyObject *)tuple; +} + PyObject * _PyTuple_FromArraySteal(PyObject *const *src, Py_ssize_t n) { @@ -413,7 +431,7 @@ _PyTuple_FromArraySteal(PyObject *const *src, Py_ssize_t n) } static PyObject * -tupleslice(PyTupleObject *a, Py_ssize_t ilow, +tuple_slice(PyTupleObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) { if (ilow < 0) @@ -435,16 +453,13 @@ PyTuple_GetSlice(PyObject *op, Py_ssize_t i, Py_ssize_t j) PyErr_BadInternalCall(); return NULL; } - return tupleslice((PyTupleObject *)op, i, j); + return tuple_slice((PyTupleObject *)op, i, j); } static PyObject * -tupleconcat(PyTupleObject *a, PyObject *bb) +tuple_concat(PyObject *aa, PyObject *bb) { - Py_ssize_t size; - Py_ssize_t i; - PyObject **src, **dest; - PyTupleObject *np; + PyTupleObject *a = _PyTuple_CAST(aa); if (Py_SIZE(a) == 0 && PyTuple_CheckExact(bb)) { return Py_NewRef(bb); } @@ -460,34 +475,38 @@ tupleconcat(PyTupleObject *a, PyObject *bb) return Py_NewRef(a); } assert((size_t)Py_SIZE(a) + (size_t)Py_SIZE(b) < PY_SSIZE_T_MAX); - size = Py_SIZE(a) + Py_SIZE(b); + Py_ssize_t size = Py_SIZE(a) + Py_SIZE(b); if (size == 0) { return tuple_get_empty(); } - np = tuple_alloc(size); + PyTupleObject *np = tuple_alloc(size); if (np == NULL) { return NULL; } - src = a->ob_item; - dest = np->ob_item; - for (i = 0; i < Py_SIZE(a); i++) { + + PyObject **src = a->ob_item; + PyObject **dest = np->ob_item; + for (Py_ssize_t i = 0; i < Py_SIZE(a); i++) { PyObject *v = src[i]; dest[i] = Py_NewRef(v); } + src = b->ob_item; dest = np->ob_item + Py_SIZE(a); - for (i = 0; i < Py_SIZE(b); i++) { + for (Py_ssize_t i = 0; i < Py_SIZE(b); i++) { PyObject *v = src[i]; dest[i] = Py_NewRef(v); } + _PyObject_GC_TRACK(np); return (PyObject *)np; } static PyObject * -tuplerepeat(PyTupleObject *a, Py_ssize_t n) +tuple_repeat(PyObject *self, Py_ssize_t n) { + PyTupleObject *a = _PyTuple_CAST(self); const Py_ssize_t input_size = Py_SIZE(a); if (input_size == 0 || n == 1) { if (PyTuple_CheckExact(a)) { @@ -602,17 +621,17 @@ tuple_count(PyTupleObject *self, PyObject *value) } static int -tupletraverse(PyTupleObject *o, visitproc visit, void *arg) +tuple_traverse(PyObject *self, visitproc visit, void *arg) { - Py_ssize_t i; - - for (i = Py_SIZE(o); --i >= 0; ) + PyTupleObject *o = _PyTuple_CAST(self); + for (Py_ssize_t i = Py_SIZE(o); --i >= 0; ) { Py_VISIT(o->ob_item[i]); + } return 0; } static PyObject * -tuplerichcompare(PyObject *v, PyObject *w, int op) +tuple_richcompare(PyObject *v, PyObject *w, int op) { PyTupleObject *vt, *wt; Py_ssize_t i; @@ -751,26 +770,27 @@ tuple_subtype_new(PyTypeObject *type, PyObject *iterable) } static PySequenceMethods tuple_as_sequence = { - (lenfunc)tuplelength, /* sq_length */ - (binaryfunc)tupleconcat, /* sq_concat */ - (ssizeargfunc)tuplerepeat, /* sq_repeat */ - (ssizeargfunc)tupleitem, /* sq_item */ + tuple_length, /* sq_length */ + tuple_concat, /* sq_concat */ + tuple_repeat, /* sq_repeat */ + tuple_item, /* sq_item */ 0, /* sq_slice */ 0, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)tuplecontains, /* sq_contains */ + tuple_contains, /* sq_contains */ }; static PyObject* -tuplesubscript(PyTupleObject* self, PyObject* item) +tuple_subscript(PyObject *op, PyObject* item) { + PyTupleObject *self = _PyTuple_CAST(op); if (_PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) return NULL; if (i < 0) i += PyTuple_GET_SIZE(self); - return tupleitem(self, i); + return tuple_item(op, i); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelength, i; @@ -824,7 +844,7 @@ static PyObject * tuple___getnewargs___impl(PyTupleObject *self) /*[clinic end generated code: output=25e06e3ee56027e2 input=1aeb4b286a21639a]*/ { - return Py_BuildValue("(N)", tupleslice(self, 0, Py_SIZE(self))); + return Py_BuildValue("(N)", tuple_slice(self, 0, Py_SIZE(self))); } static PyMethodDef tuple_methods[] = { @@ -836,8 +856,8 @@ static PyMethodDef tuple_methods[] = { }; static PyMappingMethods tuple_as_mapping = { - (lenfunc)tuplelength, - (binaryfunc)tuplesubscript, + tuple_length, + tuple_subscript, 0 }; @@ -848,16 +868,16 @@ PyTypeObject PyTuple_Type = { "tuple", sizeof(PyTupleObject) - sizeof(PyObject *), sizeof(PyObject *), - (destructor)tupledealloc, /* tp_dealloc */ + tuple_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)tuplerepr, /* tp_repr */ + tuple_repr, /* tp_repr */ 0, /* tp_as_number */ &tuple_as_sequence, /* tp_as_sequence */ &tuple_as_mapping, /* tp_as_mapping */ - (hashfunc)tuplehash, /* tp_hash */ + tuple_hash, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -867,9 +887,9 @@ PyTypeObject PyTuple_Type = { Py_TPFLAGS_BASETYPE | Py_TPFLAGS_TUPLE_SUBCLASS | _Py_TPFLAGS_MATCH_SELF | Py_TPFLAGS_SEQUENCE, /* tp_flags */ tuple_new__doc__, /* tp_doc */ - (traverseproc)tupletraverse, /* tp_traverse */ + tuple_traverse, /* tp_traverse */ 0, /* tp_clear */ - tuplerichcompare, /* tp_richcompare */ + tuple_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ tuple_iter, /* tp_iter */ 0, /* tp_iternext */ @@ -886,6 +906,7 @@ PyTypeObject PyTuple_Type = { tuple_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ .tp_vectorcall = tuple_vectorcall, + .tp_version_tag = _Py_TYPE_VERSION_TUPLE, }; /* The following function breaks the notion that tuples are immutable: @@ -942,11 +963,12 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) for (i = newsize; i < oldsize; i++) { Py_CLEAR(v->ob_item[i]); } + _PyReftracerTrack((PyObject *)v, PyRefTracer_DESTROY); sv = PyObject_GC_Resize(PyTupleObject, v, newsize); if (sv == NULL) { *pv = NULL; #ifdef Py_REF_DEBUG - _Py_DecRefTotal(_PyInterpreterState_GET()); + _Py_DecRefTotal(_PyThreadState_GET()); #endif PyObject_GC_Del(v); return -1; @@ -961,37 +983,32 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) return 0; } - -static void maybe_freelist_clear(struct _Py_object_freelists *, int); - - -void -_PyTuple_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) -{ - maybe_freelist_clear(freelists, is_finalization); -} - /*********************** Tuple Iterator **************************/ +#define _PyTupleIterObject_CAST(op) ((_PyTupleIterObject *)(op)) static void -tupleiter_dealloc(_PyTupleIterObject *it) +tupleiter_dealloc(PyObject *self) { + _PyTupleIterObject *it = _PyTupleIterObject_CAST(self); _PyObject_GC_UNTRACK(it); Py_XDECREF(it->it_seq); - PyObject_GC_Del(it); + assert(Py_IS_TYPE(self, &PyTupleIter_Type)); + _Py_FREELIST_FREE(tuple_iters, it, PyObject_GC_Del); } static int -tupleiter_traverse(_PyTupleIterObject *it, visitproc visit, void *arg) +tupleiter_traverse(PyObject *self, visitproc visit, void *arg) { + _PyTupleIterObject *it = _PyTupleIterObject_CAST(self); Py_VISIT(it->it_seq); return 0; } static PyObject * -tupleiter_next(_PyTupleIterObject *it) +tupleiter_next(PyObject *self) { + _PyTupleIterObject *it = _PyTupleIterObject_CAST(self); PyTupleObject *seq; PyObject *item; @@ -1013,8 +1030,9 @@ tupleiter_next(_PyTupleIterObject *it) } static PyObject * -tupleiter_len(_PyTupleIterObject *it, PyObject *Py_UNUSED(ignored)) +tupleiter_len(PyObject *self, PyObject *Py_UNUSED(ignored)) { + _PyTupleIterObject *it = _PyTupleIterObject_CAST(self); Py_ssize_t len = 0; if (it->it_seq) len = PyTuple_GET_SIZE(it->it_seq) - it->it_index; @@ -1024,13 +1042,14 @@ tupleiter_len(_PyTupleIterObject *it, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); static PyObject * -tupleiter_reduce(_PyTupleIterObject *it, PyObject *Py_UNUSED(ignored)) +tupleiter_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter)); /* _PyEval_GetBuiltin can invoke arbitrary code, * call must be before access of iterator pointers. * see issue #101765 */ + _PyTupleIterObject *it = _PyTupleIterObject_CAST(self); if (it->it_seq) return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index); @@ -1039,8 +1058,9 @@ tupleiter_reduce(_PyTupleIterObject *it, PyObject *Py_UNUSED(ignored)) } static PyObject * -tupleiter_setstate(_PyTupleIterObject *it, PyObject *state) +tupleiter_setstate(PyObject *self, PyObject *state) { + _PyTupleIterObject *it = _PyTupleIterObject_CAST(self); Py_ssize_t index = PyLong_AsSsize_t(state); if (index == -1 && PyErr_Occurred()) return NULL; @@ -1058,19 +1078,19 @@ PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); static PyMethodDef tupleiter_methods[] = { - {"__length_hint__", (PyCFunction)tupleiter_len, METH_NOARGS, length_hint_doc}, - {"__reduce__", (PyCFunction)tupleiter_reduce, METH_NOARGS, reduce_doc}, - {"__setstate__", (PyCFunction)tupleiter_setstate, METH_O, setstate_doc}, - {NULL, NULL} /* sentinel */ + {"__length_hint__", tupleiter_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", tupleiter_reduce, METH_NOARGS, reduce_doc}, + {"__setstate__", tupleiter_setstate, METH_O, setstate_doc}, + {NULL, NULL, 0, NULL} /* sentinel */ }; PyTypeObject PyTupleIter_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "tuple_iterator", /* tp_name */ - sizeof(_PyTupleIterObject), /* tp_basicsize */ + sizeof(_PyTupleIterObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)tupleiter_dealloc, /* tp_dealloc */ + tupleiter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -1087,12 +1107,12 @@ PyTypeObject PyTupleIter_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)tupleiter_traverse, /* tp_traverse */ + tupleiter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)tupleiter_next, /* tp_iternext */ + tupleiter_next, /* tp_iternext */ tupleiter_methods, /* tp_methods */ 0, }; @@ -1100,15 +1120,16 @@ PyTypeObject PyTupleIter_Type = { static PyObject * tuple_iter(PyObject *seq) { - _PyTupleIterObject *it; - if (!PyTuple_Check(seq)) { PyErr_BadInternalCall(); return NULL; } - it = PyObject_GC_New(_PyTupleIterObject, &PyTupleIter_Type); - if (it == NULL) - return NULL; + _PyTupleIterObject *it = _Py_FREELIST_POP(_PyTupleIterObject, tuple_iters); + if (it == NULL) { + it = PyObject_GC_New(_PyTupleIterObject, &PyTupleIter_Type); + if (it == NULL) + return NULL; + } it->it_index = 0; it->it_seq = (PyTupleObject *)Py_NewRef(seq); _PyObject_GC_TRACK(it); @@ -1120,102 +1141,29 @@ tuple_iter(PyObject *seq) * freelists * *************/ -#define TUPLE_FREELIST (freelists->tuples) -#define FREELIST_FINALIZED (TUPLE_FREELIST.numfree[0] < 0) - -static inline PyTupleObject * -maybe_freelist_pop(Py_ssize_t size) -{ -#ifdef WITH_FREELISTS - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - if (size == 0) { - return NULL; - } - assert(size > 0); - if (size < PyTuple_MAXSAVESIZE) { - Py_ssize_t index = size - 1; - PyTupleObject *op = TUPLE_FREELIST.items[index]; - if (op != NULL) { - /* op is the head of a linked list, with the first item - pointing to the next node. Here we pop off the old head. */ - TUPLE_FREELIST.items[index] = (PyTupleObject *) op->ob_item[0]; - TUPLE_FREELIST.numfree[index]--; - /* Inlined _PyObject_InitVar() without _PyType_HasFeature() test */ -#ifdef Py_TRACE_REFS - /* maybe_freelist_push() ensures these were already set. */ - // XXX Can we drop these? See commit 68055ce6fe01 (GvR, Dec 1998). - Py_SET_SIZE(op, size); - Py_SET_TYPE(op, &PyTuple_Type); -#endif - _Py_NewReference((PyObject *)op); - /* END inlined _PyObject_InitVar() */ - OBJECT_STAT_INC(from_freelist); - return op; - } - } -#endif - return NULL; -} - static inline int maybe_freelist_push(PyTupleObject *op) { -#ifdef WITH_FREELISTS - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - if (Py_SIZE(op) == 0) { + if (!Py_IS_TYPE(op, &PyTuple_Type)) { return 0; } Py_ssize_t index = Py_SIZE(op) - 1; - if (index < PyTuple_NFREELISTS - && TUPLE_FREELIST.numfree[index] < PyTuple_MAXFREELIST - && TUPLE_FREELIST.numfree[index] >= 0 - && Py_IS_TYPE(op, &PyTuple_Type)) - { - /* op is the head of a linked list, with the first item - pointing to the next node. Here we set op as the new head. */ - op->ob_item[0] = (PyObject *) TUPLE_FREELIST.items[index]; - TUPLE_FREELIST.items[index] = op; - TUPLE_FREELIST.numfree[index]++; - OBJECT_STAT_INC(to_freelist); - return 1; + if (index < PyTuple_MAXSAVESIZE) { + return _Py_FREELIST_PUSH(tuples[index], op, Py_tuple_MAXFREELIST); } -#endif return 0; } -static void -maybe_freelist_clear(struct _Py_object_freelists *freelists, int fini) -{ -#ifdef WITH_FREELISTS - for (Py_ssize_t i = 0; i < PyTuple_NFREELISTS; i++) { - PyTupleObject *p = TUPLE_FREELIST.items[i]; - TUPLE_FREELIST.items[i] = NULL; - TUPLE_FREELIST.numfree[i] = fini ? -1 : 0; - while (p) { - PyTupleObject *q = p; - p = (PyTupleObject *)(p->ob_item[0]); - PyObject_GC_Del(q); - } - } -#endif -} - /* Print summary info about the state of the optimized allocator */ void _PyTuple_DebugMallocStats(FILE *out) { -#ifdef WITH_FREELISTS - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - for (int i = 0; i < PyTuple_NFREELISTS; i++) { + for (int i = 0; i < PyTuple_MAXSAVESIZE; i++) { int len = i + 1; char buf[128]; PyOS_snprintf(buf, sizeof(buf), "free %d-sized PyTupleObject", len); - _PyDebugAllocatorStats(out, buf, TUPLE_FREELIST.numfree[i], + _PyDebugAllocatorStats(out, buf, _Py_FREELIST_SIZE(tuples[i]), _PyObject_VAR_SIZE(&PyTuple_Type, len)); } -#endif } - -#undef STATE -#undef FREELIST_FINALIZED diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 181d0323284ebc..93920341a179e8 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7,7 +7,7 @@ #include "pycore_dict.h" // _PyDict_KeysSize() #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_lock.h" // _PySeqLock_* -#include "pycore_long.h" // _PyLong_IsNegative() +#include "pycore_long.h" // _PyLong_IsNegative(), _PyLong_GetOne() #include "pycore_memoryobject.h" // _PyMemoryView_FromBufferProc() #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_moduleobject.h" // _PyModule_GetDef() @@ -19,6 +19,7 @@ #include "pycore_typeobject.h" // struct type_cache #include "pycore_unionobject.h" // _Py_union_type_or #include "pycore_weakref.h" // _PyWeakref_GET_REF() +#include "pycore_cell.h" // PyCell_GetRef() #include "opcode.h" // MAKE_CELL #include // ptrdiff_t @@ -43,7 +44,8 @@ class object "PyObject *" "&PyBaseObject_Type" & ((1 << MCACHE_SIZE_EXP) - 1)) #define MCACHE_HASH_METHOD(type, name) \ - MCACHE_HASH((type)->tp_version_tag, ((Py_ssize_t)(name)) >> 3) + MCACHE_HASH(FT_ATOMIC_LOAD_UINT32_RELAXED((type)->tp_version_tag), \ + ((Py_ssize_t)(name)) >> 3) #define MCACHE_CACHEABLE_NAME(name) \ PyUnicode_CheckExact(name) && \ PyUnicode_IS_READY(name) && \ @@ -56,19 +58,18 @@ class object "PyObject *" "&PyBaseObject_Type" #ifdef Py_GIL_DISABLED // There's a global lock for mutation of types. This avoids having to take -// additonal locks while doing various subclass processing which may result +// additional locks while doing various subclass processing which may result // in odd behaviors w.r.t. running with the GIL as the outer type lock could // be released and reacquired during a subclass update if there's contention // on the subclass lock. #define TYPE_LOCK &PyInterpreterState_Get()->types.mutex -#define BEGIN_TYPE_LOCK() \ - { \ - _PyCriticalSection _cs; \ - _PyCriticalSection_Begin(&_cs, TYPE_LOCK); \ +#define BEGIN_TYPE_LOCK() Py_BEGIN_CRITICAL_SECTION_MUT(TYPE_LOCK) +#define END_TYPE_LOCK() Py_END_CRITICAL_SECTION() -#define END_TYPE_LOCK() \ - _PyCriticalSection_End(&_cs); \ - } +#define BEGIN_TYPE_DICT_LOCK(d) \ + Py_BEGIN_CRITICAL_SECTION2_MUT(TYPE_LOCK, &_PyObject_CAST(d)->ob_mutex) + +#define END_TYPE_DICT_LOCK() Py_END_CRITICAL_SECTION2() #define ASSERT_TYPE_LOCK_HELD() \ _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(TYPE_LOCK) @@ -77,6 +78,8 @@ class object "PyObject *" "&PyBaseObject_Type" #define BEGIN_TYPE_LOCK() #define END_TYPE_LOCK() +#define BEGIN_TYPE_DICT_LOCK(d) +#define END_TYPE_DICT_LOCK() #define ASSERT_TYPE_LOCK_HELD() #endif @@ -116,88 +119,227 @@ type_from_ref(PyObject *ref) /* helpers for for static builtin types */ +#ifndef NDEBUG static inline int -static_builtin_index_is_set(PyTypeObject *self) +managed_static_type_index_is_set(PyTypeObject *self) { return self->tp_subclasses != NULL; } +#endif static inline size_t -static_builtin_index_get(PyTypeObject *self) +managed_static_type_index_get(PyTypeObject *self) { - assert(static_builtin_index_is_set(self)); + assert(managed_static_type_index_is_set(self)); /* We store a 1-based index so 0 can mean "not initialized". */ return (size_t)self->tp_subclasses - 1; } static inline void -static_builtin_index_set(PyTypeObject *self, size_t index) +managed_static_type_index_set(PyTypeObject *self, size_t index) { - assert(index < _Py_MAX_STATIC_BUILTIN_TYPES); + assert(index < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES); /* We store a 1-based index so 0 can mean "not initialized". */ self->tp_subclasses = (PyObject *)(index + 1); } static inline void -static_builtin_index_clear(PyTypeObject *self) +managed_static_type_index_clear(PyTypeObject *self) { self->tp_subclasses = NULL; } -static inline static_builtin_state * -static_builtin_state_get(PyInterpreterState *interp, PyTypeObject *self) +static PyTypeObject * +static_ext_type_lookup(PyInterpreterState *interp, size_t index, + int64_t *p_interp_count) +{ + assert(interp->runtime == &_PyRuntime); + assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); + + size_t full_index = index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; + int64_t interp_count = + _PyRuntime.types.managed_static.types[full_index].interp_count; + assert((interp_count == 0) == + (_PyRuntime.types.managed_static.types[full_index].type == NULL)); + *p_interp_count = interp_count; + + PyTypeObject *type = interp->types.for_extensions.initialized[index].type; + if (type == NULL) { + return NULL; + } + assert(!interp->types.for_extensions.initialized[index].isbuiltin); + assert(type == _PyRuntime.types.managed_static.types[full_index].type); + assert(managed_static_type_index_is_set(type)); + return type; +} + +static managed_static_type_state * +managed_static_type_state_get(PyInterpreterState *interp, PyTypeObject *self) { - return &(interp->types.builtins[static_builtin_index_get(self)]); + // It's probably a builtin type. + size_t index = managed_static_type_index_get(self); + managed_static_type_state *state = + &(interp->types.builtins.initialized[index]); + if (state->type == self) { + return state; + } + if (index > _Py_MAX_MANAGED_STATIC_EXT_TYPES) { + return state; + } + return &(interp->types.for_extensions.initialized[index]); } /* For static types we store some state in an array on each interpreter. */ -static_builtin_state * +managed_static_type_state * _PyStaticType_GetState(PyInterpreterState *interp, PyTypeObject *self) { assert(self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); - return static_builtin_state_get(interp, self); + return managed_static_type_state_get(interp, self); } /* Set the type's per-interpreter state. */ static void -static_builtin_state_init(PyInterpreterState *interp, PyTypeObject *self) +managed_static_type_state_init(PyInterpreterState *interp, PyTypeObject *self, + int isbuiltin, int initial) { - if (!static_builtin_index_is_set(self)) { - static_builtin_index_set(self, interp->types.num_builtins_initialized); + assert(interp->runtime == &_PyRuntime); + + size_t index; + if (initial) { + assert(!managed_static_type_index_is_set(self)); + if (isbuiltin) { + index = interp->types.builtins.num_initialized; + assert(index < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES); + } + else { + PyMutex_Lock(&interp->types.mutex); + index = interp->types.for_extensions.next_index; + interp->types.for_extensions.next_index++; + PyMutex_Unlock(&interp->types.mutex); + assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); + } + managed_static_type_index_set(self, index); + } + else { + index = managed_static_type_index_get(self); + if (isbuiltin) { + assert(index == interp->types.builtins.num_initialized); + assert(index < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES); + } + else { + assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); + } + } + size_t full_index = isbuiltin + ? index + : index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; + + assert((initial == 1) == + (_PyRuntime.types.managed_static.types[full_index].interp_count == 0)); + (void)_Py_atomic_add_int64( + &_PyRuntime.types.managed_static.types[full_index].interp_count, 1); + + if (initial) { + assert(_PyRuntime.types.managed_static.types[full_index].type == NULL); + _PyRuntime.types.managed_static.types[full_index].type = self; + } + else { + assert(_PyRuntime.types.managed_static.types[full_index].type == self); } - static_builtin_state *state = static_builtin_state_get(interp, self); - /* It should only be called once for each builtin type. */ + managed_static_type_state *state = isbuiltin + ? &(interp->types.builtins.initialized[index]) + : &(interp->types.for_extensions.initialized[index]); + + /* It should only be called once for each builtin type per interpreter. */ assert(state->type == NULL); state->type = self; + state->isbuiltin = isbuiltin; /* state->tp_subclasses is left NULL until init_subclasses() sets it. */ /* state->tp_weaklist is left NULL until insert_head() or insert_after() (in weakrefobject.c) sets it. */ - interp->types.num_builtins_initialized++; + if (isbuiltin) { + interp->types.builtins.num_initialized++; + } + else { + interp->types.for_extensions.num_initialized++; + } } /* Reset the type's per-interpreter state. - This basically undoes what static_builtin_state_init() did. */ + This basically undoes what managed_static_type_state_init() did. */ static void -static_builtin_state_clear(PyInterpreterState *interp, PyTypeObject *self) +managed_static_type_state_clear(PyInterpreterState *interp, PyTypeObject *self, + int isbuiltin, int final) { - static_builtin_state *state = static_builtin_state_get(interp, self); + size_t index = managed_static_type_index_get(self); + size_t full_index = isbuiltin + ? index + : index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; + + managed_static_type_state *state = isbuiltin + ? &(interp->types.builtins.initialized[index]) + : &(interp->types.for_extensions.initialized[index]); + assert(state != NULL); + + assert(_PyRuntime.types.managed_static.types[full_index].interp_count > 0); + assert(_PyRuntime.types.managed_static.types[full_index].type == state->type); assert(state->type != NULL); state->type = NULL; assert(state->tp_weaklist == NULL); // It was already cleared out. - if (_Py_IsMainInterpreter(interp)) { - static_builtin_index_clear(self); + (void)_Py_atomic_add_int64( + &_PyRuntime.types.managed_static.types[full_index].interp_count, -1); + if (final) { + assert(!_PyRuntime.types.managed_static.types[full_index].interp_count); + _PyRuntime.types.managed_static.types[full_index].type = NULL; + + managed_static_type_index_clear(self); + } + + if (isbuiltin) { + assert(interp->types.builtins.num_initialized > 0); + interp->types.builtins.num_initialized--; + } + else { + PyMutex_Lock(&interp->types.mutex); + assert(interp->types.for_extensions.num_initialized > 0); + interp->types.for_extensions.num_initialized--; + if (interp->types.for_extensions.num_initialized == 0) { + interp->types.for_extensions.next_index = 0; + } + PyMutex_Unlock(&interp->types.mutex); + } +} + + + +PyObject * +_PyStaticType_GetBuiltins(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + Py_ssize_t count = (Py_ssize_t)interp->types.builtins.num_initialized; + assert(count <= _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES); + + PyObject *results = PyList_New(count); + if (results == NULL) { + return NULL; + } + for (Py_ssize_t i = 0; i < count; i++) { + PyTypeObject *cls = interp->types.builtins.initialized[i].type; + assert(cls != NULL); + assert(interp->types.builtins.initialized[i].isbuiltin); + PyList_SET_ITEM(results, i, Py_NewRef((PyObject *)cls)); } - assert(interp->types.num_builtins_initialized > 0); - interp->types.num_builtins_initialized--; + return results; } -// Also see _PyStaticType_InitBuiltin() and _PyStaticType_Dealloc(). + +// Also see _PyStaticType_InitBuiltin() and _PyStaticType_FiniBuiltin(). /* end static builtin helpers */ @@ -207,7 +349,7 @@ start_readying(PyTypeObject *type) { if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = static_builtin_state_get(interp, type); + managed_static_type_state *state = managed_static_type_state_get(interp, type); assert(state != NULL); assert(!state->readying); state->readying = 1; @@ -222,7 +364,7 @@ stop_readying(PyTypeObject *type) { if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = static_builtin_state_get(interp, type); + managed_static_type_state *state = managed_static_type_state_get(interp, type); assert(state != NULL); assert(state->readying); state->readying = 0; @@ -237,7 +379,7 @@ is_readying(PyTypeObject *type) { if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = static_builtin_state_get(interp, type); + managed_static_type_state *state = managed_static_type_state_get(interp, type); assert(state != NULL); return state->readying; } @@ -252,7 +394,7 @@ lookup_tp_dict(PyTypeObject *self) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); assert(state != NULL); return state->tp_dict; } @@ -278,7 +420,7 @@ set_tp_dict(PyTypeObject *self, PyObject *dict) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); assert(state != NULL); state->tp_dict = dict; return; @@ -291,7 +433,7 @@ clear_tp_dict(PyTypeObject *self) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); assert(state != NULL); Py_CLEAR(state->tp_dict); return; @@ -314,19 +456,19 @@ _PyType_GetBases(PyTypeObject *self) BEGIN_TYPE_LOCK(); res = lookup_tp_bases(self); Py_INCREF(res); - END_TYPE_LOCK() + END_TYPE_LOCK(); return res; } static inline void -set_tp_bases(PyTypeObject *self, PyObject *bases) +set_tp_bases(PyTypeObject *self, PyObject *bases, int initial) { assert(PyTuple_CheckExact(bases)); if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { // XXX tp_bases can probably be statically allocated for each // static builtin type. - assert(_Py_IsMainInterpreter(_PyInterpreterState_GET())); + assert(initial); assert(self->tp_bases == NULL); if (PyTuple_GET_SIZE(bases) == 0) { assert(self->tp_base == NULL); @@ -343,10 +485,10 @@ set_tp_bases(PyTypeObject *self, PyObject *bases) } static inline void -clear_tp_bases(PyTypeObject *self) +clear_tp_bases(PyTypeObject *self, int final) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - if (_Py_IsMainInterpreter(_PyInterpreterState_GET())) { + if (final) { if (self->tp_bases != NULL) { if (PyTuple_GET_SIZE(self->tp_bases) == 0) { Py_CLEAR(self->tp_bases); @@ -373,22 +515,33 @@ lookup_tp_mro(PyTypeObject *self) PyObject * _PyType_GetMRO(PyTypeObject *self) { - PyObject *mro; +#ifdef Py_GIL_DISABLED + PyObject *mro = _Py_atomic_load_ptr_relaxed(&self->tp_mro); + if (mro == NULL) { + return NULL; + } + if (_Py_TryIncrefCompare(&self->tp_mro, mro)) { + return mro; + } + BEGIN_TYPE_LOCK(); mro = lookup_tp_mro(self); - Py_INCREF(mro); - END_TYPE_LOCK() + Py_XINCREF(mro); + END_TYPE_LOCK(); return mro; +#else + return Py_XNewRef(lookup_tp_mro(self)); +#endif } static inline void -set_tp_mro(PyTypeObject *self, PyObject *mro) +set_tp_mro(PyTypeObject *self, PyObject *mro, int initial) { assert(PyTuple_CheckExact(mro)); if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { // XXX tp_mro can probably be statically allocated for each // static builtin type. - assert(_Py_IsMainInterpreter(_PyInterpreterState_GET())); + assert(initial); assert(self->tp_mro == NULL); /* Other checks are done via set_tp_bases. */ _Py_SetImmortal(mro); @@ -397,10 +550,10 @@ set_tp_mro(PyTypeObject *self, PyObject *mro) } static inline void -clear_tp_mro(PyTypeObject *self) +clear_tp_mro(PyTypeObject *self, int final) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - if (_Py_IsMainInterpreter(_PyInterpreterState_GET())) { + if (final) { if (self->tp_mro != NULL) { if (PyTuple_GET_SIZE(self->tp_mro) == 0) { Py_CLEAR(self->tp_mro); @@ -426,7 +579,7 @@ init_tp_subclasses(PyTypeObject *self) } if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); state->tp_subclasses = subclasses; return subclasses; } @@ -442,7 +595,7 @@ clear_tp_subclasses(PyTypeObject *self) has no subclass. */ if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); Py_CLEAR(state->tp_subclasses); return; } @@ -454,7 +607,7 @@ lookup_tp_subclasses(PyTypeObject *self) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); + managed_static_type_state *state = _PyStaticType_GetState(interp, self); assert(state != NULL); return state->tp_subclasses; } @@ -709,7 +862,7 @@ _PyType_InitCache(PyInterpreterState *interp) assert(entry->name == NULL); entry->version = 0; - // Set to None so _PyType_Lookup() can use Py_SETREF(), + // Set to None so _PyType_LookupRef() can use Py_SETREF(), // rather than using slower Py_XSETREF(). entry->name = Py_None; entry->value = NULL; @@ -721,7 +874,7 @@ static unsigned int _PyType_ClearCache(PyInterpreterState *interp) { struct type_cache *cache = &interp->types.type_cache; - // Set to None, rather than NULL, so _PyType_Lookup() can + // Set to None, rather than NULL, so _PyType_LookupRef() can // use Py_SETREF() rather than using slower Py_XSETREF(). type_cache_clear(cache, Py_None); @@ -743,10 +896,14 @@ _PyTypes_Fini(PyInterpreterState *interp) struct type_cache *cache = &interp->types.type_cache; type_cache_clear(cache, NULL); - assert(interp->types.num_builtins_initialized == 0); - // All the static builtin types should have been finalized already. - for (size_t i = 0; i < _Py_MAX_STATIC_BUILTIN_TYPES; i++) { - assert(interp->types.builtins[i].type == NULL); + // All the managed static types should have been finalized already. + assert(interp->types.for_extensions.num_initialized == 0); + for (size_t i = 0; i < _Py_MAX_MANAGED_STATIC_EXT_TYPES; i++) { + assert(interp->types.for_extensions.initialized[i].type == NULL); + } + assert(interp->types.builtins.num_initialized == 0); + for (size_t i = 0; i < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; i++) { + assert(interp->types.builtins.initialized[i].type == NULL); } } @@ -756,7 +913,8 @@ PyType_AddWatcher(PyType_WatchCallback callback) { PyInterpreterState *interp = _PyInterpreterState_GET(); - for (int i = 0; i < TYPE_MAX_WATCHERS; i++) { + // start at 1, 0 is reserved for cpython optimizer + for (int i = 1; i < TYPE_MAX_WATCHERS; i++) { if (!interp->type_watchers[i]) { interp->type_watchers[i] = callback; return i; @@ -807,10 +965,10 @@ PyType_Watch(int watcher_id, PyObject* obj) return -1; } // ensure we will get a callback on the next modification - BEGIN_TYPE_LOCK() + BEGIN_TYPE_LOCK(); assign_version_tag(interp, type); type->tp_watched |= (1 << watcher_id); - END_TYPE_LOCK() + END_TYPE_LOCK(); return 0; } @@ -830,6 +988,39 @@ PyType_Unwatch(int watcher_id, PyObject* obj) return 0; } +static void +set_version_unlocked(PyTypeObject *tp, unsigned int version) +{ + ASSERT_TYPE_LOCK_HELD(); + assert(version == 0 || (tp->tp_versions_used != _Py_ATTR_CACHE_UNUSED)); +#ifndef Py_GIL_DISABLED + PyInterpreterState *interp = _PyInterpreterState_GET(); + // lookup the old version and set to null + if (tp->tp_version_tag != 0) { + PyTypeObject **slot = + interp->types.type_version_cache + + (tp->tp_version_tag % TYPE_VERSION_CACHE_SIZE); + *slot = NULL; + } + if (version) { + tp->tp_versions_used++; + } +#else + if (version) { + _Py_atomic_add_uint16(&tp->tp_versions_used, 1); + } +#endif + FT_ATOMIC_STORE_UINT32_RELAXED(tp->tp_version_tag, version); +#ifndef Py_GIL_DISABLED + if (version != 0) { + PyTypeObject **slot = + interp->types.type_version_cache + + (version % TYPE_VERSION_CACHE_SIZE); + *slot = tp; + } +#endif +} + static void type_modified_unlocked(PyTypeObject *type) { @@ -839,18 +1030,20 @@ type_modified_unlocked(PyTypeObject *type) Invariants: - - before Py_TPFLAGS_VALID_VERSION_TAG can be set on a type, + - before tp_version_tag can be set on a type, it must first be set on all super types. - This function clears the Py_TPFLAGS_VALID_VERSION_TAG of a + This function clears the tp_version_tag of a type (so it must first clear it on all subclasses). The - tp_version_tag value is meaningless unless this flag is set. + tp_version_tag value is meaningless when equal to zero. We don't assign new version tags eagerly, but only as needed. */ - if (!_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)) { + if (FT_ATOMIC_LOAD_UINT_RELAXED(type->tp_version_tag) == 0) { return; } + // Cannot modify static builtin types. + assert((type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) == 0); PyObject *subclasses = lookup_tp_subclasses(type); if (subclasses != NULL) { @@ -863,7 +1056,7 @@ type_modified_unlocked(PyTypeObject *type) if (subclass == NULL) { continue; } - PyType_Modified(subclass); + type_modified_unlocked(subclass); Py_DECREF(subclass); } } @@ -888,8 +1081,7 @@ type_modified_unlocked(PyTypeObject *type) } } - type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - type->tp_version_tag = 0; /* 0 is not a valid version tag */ + set_version_unlocked(type, 0); /* 0 is not a valid version tag */ if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { // This field *must* be invalidated if the type is modified (see the // comment on struct _specialization_cache): @@ -901,17 +1093,17 @@ void PyType_Modified(PyTypeObject *type) { // Quick check without the lock held - if (!_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)) { + if (type->tp_version_tag == 0) { return; } - BEGIN_TYPE_LOCK() + BEGIN_TYPE_LOCK(); type_modified_unlocked(type); - END_TYPE_LOCK() + END_TYPE_LOCK(); } static int -is_subtype_unlocked(PyTypeObject *a, PyTypeObject *b); +is_subtype_with_mro(PyObject *a_mro, PyTypeObject *a, PyTypeObject *b); static void type_mro_modified(PyTypeObject *type, PyObject *bases) { @@ -957,7 +1149,11 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { PyObject *b = PyTuple_GET_ITEM(bases, i); PyTypeObject *cls = _PyType_CAST(b); - if (!is_subtype_unlocked(type, cls)) { + if (cls->tp_versions_used >= _Py_ATTR_CACHE_UNUSED) { + goto clear; + } + + if (!is_subtype_with_mro(lookup_tp_mro(type), type, cls)) { goto clear; } } @@ -965,8 +1161,8 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { clear: assert(!(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); - type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - type->tp_version_tag = 0; /* 0 is not a valid version tag */ + set_version_unlocked(type, 0); /* 0 is not a valid version tag */ + type->tp_versions_used = _Py_ATTR_CACHE_UNUSED; if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { // This field *must* be invalidated if the type is modified (see the // comment on struct _specialization_cache): @@ -974,35 +1170,89 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { } } +/* +The Tier 2 interpreter requires looking up the type object by the type version, so it can install +watchers to understand when they change. + +So we add a global cache from type version to borrowed references of type objects. + +This is similar to func_version_cache. +*/ + +void +_PyType_SetVersion(PyTypeObject *tp, unsigned int version) +{ + + BEGIN_TYPE_LOCK(); + set_version_unlocked(tp, version); + END_TYPE_LOCK(); +} + +PyTypeObject * +_PyType_LookupByVersion(unsigned int version) +{ +#ifdef Py_GIL_DISABLED + return NULL; +#else + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyTypeObject **slot = + interp->types.type_version_cache + + (version % TYPE_VERSION_CACHE_SIZE); + if (*slot && (*slot)->tp_version_tag == version) { + return *slot; + } + return NULL; +#endif +} + +unsigned int +_PyType_GetVersionForCurrentState(PyTypeObject *tp) +{ + return tp->tp_version_tag; +} + + + #define MAX_VERSIONS_PER_CLASS 1000 +#if _Py_ATTR_CACHE_UNUSED < MAX_VERSIONS_PER_CLASS +#error "_Py_ATTR_CACHE_UNUSED must be bigger than max" +#endif static int assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) { ASSERT_TYPE_LOCK_HELD(); - /* Ensure that the tp_version_tag is valid and set - Py_TPFLAGS_VALID_VERSION_TAG. To respect the invariant, this - must first be done on all super classes. Return 0 if this - cannot be done, 1 if Py_TPFLAGS_VALID_VERSION_TAG. + /* Ensure that the tp_version_tag is valid. + * To respect the invariant, this must first be done on all super classes. + * Return 0 if this cannot be done, 1 if tp_version_tag is set. */ - if (_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)) { + if (type->tp_version_tag != 0) { return 1; } if (!_PyType_HasFeature(type, Py_TPFLAGS_READY)) { return 0; } if (type->tp_versions_used >= MAX_VERSIONS_PER_CLASS) { + /* (this includes `tp_versions_used == _Py_ATTR_CACHE_UNUSED`) */ return 0; } - type->tp_versions_used++; + + PyObject *bases = lookup_tp_bases(type); + Py_ssize_t n = PyTuple_GET_SIZE(bases); + for (Py_ssize_t i = 0; i < n; i++) { + PyObject *b = PyTuple_GET_ITEM(bases, i); + if (!assign_version_tag(interp, _PyType_CAST(b))) { + return 0; + } + } if (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) { /* static types */ if (NEXT_GLOBAL_VERSION_TAG > _Py_MAX_GLOBAL_TYPE_VERSION_TAG) { /* We have run out of version numbers */ return 0; } - type->tp_version_tag = NEXT_GLOBAL_VERSION_TAG++; + set_version_unlocked(type, NEXT_GLOBAL_VERSION_TAG++); assert (type->tp_version_tag <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); } else { @@ -1011,18 +1261,9 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) /* We have run out of version numbers */ return 0; } - type->tp_version_tag = NEXT_VERSION_TAG(interp)++; + set_version_unlocked(type, NEXT_VERSION_TAG(interp)++); assert (type->tp_version_tag != 0); } - - PyObject *bases = lookup_tp_bases(type); - Py_ssize_t n = PyTuple_GET_SIZE(bases); - for (Py_ssize_t i = 0; i < n; i++) { - PyObject *b = PyTuple_GET_ITEM(bases, i); - if (!assign_version_tag(interp, _PyType_CAST(b))) - return 0; - } - type->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; return 1; } @@ -1030,9 +1271,9 @@ int PyUnstable_Type_AssignVersionTag(PyTypeObject *type) { PyInterpreterState *interp = _PyInterpreterState_GET(); int assigned; - BEGIN_TYPE_LOCK() + BEGIN_TYPE_LOCK(); assigned = assign_version_tag(interp, type); - END_TYPE_LOCK() + END_TYPE_LOCK(); return assigned; } @@ -1164,10 +1405,9 @@ type_set_qualname(PyTypeObject *type, PyObject *value, void *context) } static PyObject * -type_module(PyTypeObject *type, void *context) +type_module(PyTypeObject *type) { PyObject *mod; - if (type->tp_flags & Py_TPFLAGS_HEAPTYPE) { PyObject *dict = lookup_tp_dict(type); if (PyDict_GetItemRef(dict, &_Py_ID(__module__), &mod) == 0) { @@ -1179,8 +1419,10 @@ type_module(PyTypeObject *type, void *context) if (s != NULL) { mod = PyUnicode_FromStringAndSize( type->tp_name, (Py_ssize_t)(s - type->tp_name)); - if (mod != NULL) - PyUnicode_InternInPlace(&mod); + if (mod != NULL) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternMortal(interp, &mod); + } } else { mod = &_Py_ID(builtins); @@ -1189,6 +1431,12 @@ type_module(PyTypeObject *type, void *context) return mod; } +static PyObject * +type_get_module(PyTypeObject *type, void *context) +{ + return type_module(type); +} + static int type_set_module(PyTypeObject *type, PyObject *value, void *context) { @@ -1198,9 +1446,53 @@ type_set_module(PyTypeObject *type, PyObject *value, void *context) PyType_Modified(type); PyObject *dict = lookup_tp_dict(type); + if (PyDict_Pop(dict, &_Py_ID(__firstlineno__), NULL) < 0) { + return -1; + } return PyDict_SetItem(dict, &_Py_ID(__module__), value); } + +PyObject * +_PyType_GetFullyQualifiedName(PyTypeObject *type, char sep) +{ + if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + return PyUnicode_FromString(type->tp_name); + } + + PyObject *qualname = type_qualname(type, NULL); + if (qualname == NULL) { + return NULL; + } + + PyObject *module = type_module(type); + if (module == NULL) { + Py_DECREF(qualname); + return NULL; + } + + PyObject *result; + if (PyUnicode_Check(module) + && !_PyUnicode_Equal(module, &_Py_ID(builtins)) + && !_PyUnicode_Equal(module, &_Py_ID(__main__))) + { + result = PyUnicode_FromFormat("%U%c%U", module, sep, qualname); + } + else { + result = Py_NewRef(qualname); + } + Py_DECREF(module); + Py_DECREF(qualname); + return result; +} + +PyObject * +PyType_GetFullyQualifiedName(PyTypeObject *type) +{ + return _PyType_GetFullyQualifiedName(type, '.'); +} + + static PyObject * type_abstractmethods(PyTypeObject *type, void *context) { @@ -1236,20 +1528,22 @@ type_set_abstractmethods(PyTypeObject *type, PyObject *value, void *context) } else { abstract = 0; - res = PyDict_DelItem(dict, &_Py_ID(__abstractmethods__)); - if (res && PyErr_ExceptionMatches(PyExc_KeyError)) { + res = PyDict_Pop(dict, &_Py_ID(__abstractmethods__), NULL); + if (res == 0) { PyErr_SetObject(PyExc_AttributeError, &_Py_ID(__abstractmethods__)); return -1; } } - if (res == 0) { - PyType_Modified(type); - if (abstract) - type->tp_flags |= Py_TPFLAGS_IS_ABSTRACT; - else - type->tp_flags &= ~Py_TPFLAGS_IS_ABSTRACT; + if (res < 0) { + return -1; } - return res; + + PyType_Modified(type); + if (abstract) + type->tp_flags |= Py_TPFLAGS_IS_ABSTRACT; + else + type->tp_flags &= ~Py_TPFLAGS_IS_ABSTRACT; + return 0; } static PyObject * @@ -1267,7 +1561,7 @@ type_get_mro(PyTypeObject *type, void *context) { PyObject *mro; - BEGIN_TYPE_LOCK() + BEGIN_TYPE_LOCK(); mro = lookup_tp_mro(type); if (mro == NULL) { mro = Py_None; @@ -1275,7 +1569,7 @@ type_get_mro(PyTypeObject *type, void *context) Py_INCREF(mro); } - END_TYPE_LOCK() + END_TYPE_LOCK(); return mro; } @@ -1325,7 +1619,7 @@ mro_hierarchy(PyTypeObject *type, PyObject *temp) Py_XDECREF(tuple); if (res < 0) { - set_tp_mro(type, old_mro); + set_tp_mro(type, old_mro, 0); Py_DECREF(new_mro); return -1; } @@ -1394,7 +1688,7 @@ type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases, void *context) } PyTypeObject *base = (PyTypeObject*)ob; - if (is_subtype_unlocked(base, type) || + if (is_subtype_with_mro(lookup_tp_mro(base), base, type) || /* In case of reentering here again through a custom mro() the above check is not enough since it relies on base->tp_mro which would gonna be updated inside @@ -1426,7 +1720,7 @@ type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases, void *context) assert(old_bases != NULL); PyTypeObject *old_base = type->tp_base; - set_tp_bases(type, Py_NewRef(new_bases)); + set_tp_bases(type, Py_NewRef(new_bases), 0); type->tp_base = (PyTypeObject *)Py_NewRef(new_base); PyObject *temp = PyList_New(0); @@ -1474,7 +1768,7 @@ type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases, void *context) "", 2, 3, &cls, &new_mro, &old_mro); /* Do not rollback if cls has a newer version of MRO. */ if (lookup_tp_mro(cls) == new_mro) { - set_tp_mro(cls, Py_XNewRef(old_mro)); + set_tp_mro(cls, Py_XNewRef(old_mro), 0); Py_DECREF(new_mro); } } @@ -1484,7 +1778,7 @@ type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases, void *context) if (lookup_tp_bases(type) == new_bases) { assert(type->tp_base == new_base); - set_tp_bases(type, old_bases); + set_tp_bases(type, old_bases, 0); type->tp_base = old_base; Py_DECREF(new_bases); @@ -1555,6 +1849,76 @@ type_set_doc(PyTypeObject *type, PyObject *value, void *context) return PyDict_SetItem(dict, &_Py_ID(__doc__), value); } +static PyObject * +type_get_annotate(PyTypeObject *type, void *Py_UNUSED(ignored)) +{ + if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + PyErr_Format(PyExc_AttributeError, "type object '%s' has no attribute '__annotate__'", type->tp_name); + return NULL; + } + + PyObject *annotate; + PyObject *dict = PyType_GetDict(type); + if (PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate) < 0) { + Py_DECREF(dict); + return NULL; + } + if (annotate) { + descrgetfunc get = Py_TYPE(annotate)->tp_descr_get; + if (get) { + Py_SETREF(annotate, get(annotate, NULL, (PyObject *)type)); + } + } + else { + annotate = Py_None; + int result = PyDict_SetItem(dict, &_Py_ID(__annotate__), annotate); + if (result < 0) { + Py_DECREF(dict); + return NULL; + } + } + Py_DECREF(dict); + return annotate; +} + +static int +type_set_annotate(PyTypeObject *type, PyObject *value, void *Py_UNUSED(ignored)) +{ + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete __annotate__ attribute"); + return -1; + } + if (_PyType_HasFeature(type, Py_TPFLAGS_IMMUTABLETYPE)) { + PyErr_Format(PyExc_TypeError, + "cannot set '__annotate__' attribute of immutable type '%s'", + type->tp_name); + return -1; + } + + if (!Py_IsNone(value) && !PyCallable_Check(value)) { + PyErr_SetString(PyExc_TypeError, "__annotate__ must be callable or None"); + return -1; + } + + PyObject *dict = PyType_GetDict(type); + assert(PyDict_Check(dict)); + int result = PyDict_SetItem(dict, &_Py_ID(__annotate__), value); + if (result < 0) { + Py_DECREF(dict); + return -1; + } + if (!Py_IsNone(value)) { + if (PyDict_Pop(dict, &_Py_ID(__annotations__), NULL) == -1) { + Py_DECREF(dict); + PyType_Modified(type); + return -1; + } + } + Py_DECREF(dict); + PyType_Modified(type); + return 0; +} + static PyObject * type_get_annotations(PyTypeObject *type, void *context) { @@ -1564,8 +1928,9 @@ type_get_annotations(PyTypeObject *type, void *context) } PyObject *annotations; - PyObject *dict = lookup_tp_dict(type); + PyObject *dict = PyType_GetDict(type); if (PyDict_GetItemRef(dict, &_Py_ID(__annotations__), &annotations) < 0) { + Py_DECREF(dict); return NULL; } if (annotations) { @@ -1575,7 +1940,32 @@ type_get_annotations(PyTypeObject *type, void *context) } } else { - annotations = PyDict_New(); + PyObject *annotate = type_get_annotate(type, NULL); + if (annotate == NULL) { + Py_DECREF(dict); + return NULL; + } + if (PyCallable_Check(annotate)) { + PyObject *one = _PyLong_GetOne(); + annotations = _PyObject_CallOneArg(annotate, one); + if (annotations == NULL) { + Py_DECREF(dict); + Py_DECREF(annotate); + return NULL; + } + if (!PyDict_Check(annotations)) { + PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", + Py_TYPE(annotations)->tp_name); + Py_DECREF(annotations); + Py_DECREF(annotate); + Py_DECREF(dict); + return NULL; + } + } + else { + annotations = PyDict_New(); + } + Py_DECREF(annotate); if (annotations) { int result = PyDict_SetItem( dict, &_Py_ID(__annotations__), annotations); @@ -1586,6 +1976,7 @@ type_get_annotations(PyTypeObject *type, void *context) } } } + Py_DECREF(dict); return annotations; } @@ -1600,27 +1991,42 @@ type_set_annotations(PyTypeObject *type, PyObject *value, void *context) } int result; - PyObject *dict = lookup_tp_dict(type); + PyObject *dict = PyType_GetDict(type); if (value != NULL) { /* set */ result = PyDict_SetItem(dict, &_Py_ID(__annotations__), value); } else { /* delete */ - result = PyDict_DelItem(dict, &_Py_ID(__annotations__)); - if (result < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { + result = PyDict_Pop(dict, &_Py_ID(__annotations__), NULL); + if (result == 0) { PyErr_SetString(PyExc_AttributeError, "__annotations__"); + Py_DECREF(dict); + return -1; } } - - if (result == 0) { - PyType_Modified(type); + if (result < 0) { + Py_DECREF(dict); + return -1; } - return result; + else if (result == 0) { + if (PyDict_Pop(dict, &_Py_ID(__annotate__), NULL) < 0) { + PyType_Modified(type); + Py_DECREF(dict); + return -1; + } + } + PyType_Modified(type); + Py_DECREF(dict); + return 0; } static PyObject * type_get_type_params(PyTypeObject *type, void *context) { + if (type == &PyType_Type) { + return PyTuple_New(0); + } + PyObject *params; if (PyDict_GetItemRef(lookup_tp_dict(type), &_Py_ID(__type_params__), ¶ms) == 0) { return PyTuple_New(0); @@ -1683,13 +2089,14 @@ static PyGetSetDef type_getsets[] = { {"__qualname__", (getter)type_qualname, (setter)type_set_qualname, NULL}, {"__bases__", (getter)type_get_bases, (setter)type_set_bases, NULL}, {"__mro__", (getter)type_get_mro, NULL, NULL}, - {"__module__", (getter)type_module, (setter)type_set_module, NULL}, + {"__module__", (getter)type_get_module, (setter)type_set_module, NULL}, {"__abstractmethods__", (getter)type_abstractmethods, (setter)type_set_abstractmethods, NULL}, {"__dict__", (getter)type_dict, NULL, NULL}, {"__doc__", (getter)type_get_doc, (setter)type_set_doc, NULL}, {"__text_signature__", (getter)type_get_text_signature, NULL, NULL}, {"__annotations__", (getter)type_get_annotations, (setter)type_set_annotations, NULL}, + {"__annotate__", (getter)type_get_annotate, (setter)type_set_annotate, NULL}, {"__type_params__", (getter)type_get_type_params, (setter)type_set_type_params, NULL}, {0} }; @@ -1704,28 +2111,31 @@ type_repr(PyObject *self) return PyUnicode_FromFormat("", type); } - PyObject *mod, *name, *rtn; - - mod = type_module(type, NULL); - if (mod == NULL) + PyObject *mod = type_module(type); + if (mod == NULL) { PyErr_Clear(); + } else if (!PyUnicode_Check(mod)) { - Py_SETREF(mod, NULL); + Py_CLEAR(mod); } - name = type_qualname(type, NULL); + + PyObject *name = type_qualname(type, NULL); if (name == NULL) { Py_XDECREF(mod); return NULL; } - if (mod != NULL && !_PyUnicode_Equal(mod, &_Py_ID(builtins))) - rtn = PyUnicode_FromFormat("", mod, name); - else - rtn = PyUnicode_FromFormat("", type->tp_name); - + PyObject *result; + if (mod != NULL && !_PyUnicode_Equal(mod, &_Py_ID(builtins))) { + result = PyUnicode_FromFormat("", mod, name); + } + else { + result = PyUnicode_FromFormat("", type->tp_name); + } Py_XDECREF(mod); Py_DECREF(name); - return rtn; + + return result; } static PyObject * @@ -1797,7 +2207,7 @@ type_call(PyObject *self, PyObject *args, PyObject *kwds) PyObject * _PyType_NewManagedObject(PyTypeObject *type) { - assert(type->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(type->tp_flags & Py_TPFLAGS_INLINE_VALUES); assert(_PyType_IS_GC(type)); assert(type->tp_new == PyBaseObject_Type.tp_new); assert(type->tp_alloc == PyType_GenericAlloc); @@ -1806,11 +2216,6 @@ _PyType_NewManagedObject(PyTypeObject *type) if (obj == NULL) { return PyErr_NoMemory(); } - _PyObject_DictOrValuesPointer(obj)->dict = NULL; - if (_PyObject_InitInlineValues(obj, type)) { - Py_DECREF(obj); - return NULL; - } return obj; } @@ -1824,9 +2229,13 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems) * flag to indicate when that is safe) it does not seem worth the memory * savings. An example type that doesn't need the +1 is a subclass of * tuple. See GH-100659 and GH-81381. */ - const size_t size = _PyObject_VAR_SIZE(type, nitems+1); + size_t size = _PyObject_VAR_SIZE(type, nitems+1); const size_t presize = _PyType_PreHeaderSize(type); + if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + assert(type->tp_itemsize == 0); + size += _PyInlineValuesSize(type); + } char *alloc = _PyObject_MallocWithType(type, size + presize); if (alloc == NULL) { return PyErr_NoMemory(); @@ -1847,6 +2256,9 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems) else { _PyObject_InitVar((PyVarObject *)obj, type, nitems); } + if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + _PyObject_InitInlineValues(obj, type); + } return obj; } @@ -1996,6 +2408,10 @@ subtype_clear(PyObject *self) if ((base->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { PyObject_ClearManagedDict(self); } + else { + assert((base->tp_flags & Py_TPFLAGS_INLINE_VALUES) == + (type->tp_flags & Py_TPFLAGS_INLINE_VALUES)); + } } else if (type->tp_dictoffset != base->tp_dictoffset) { PyObject **dictptr = _PyObject_ComputedDictPointer(self); @@ -2064,7 +2480,7 @@ subtype_dealloc(PyObject *self) reference counting. Only decref if the base type is not already a heap allocated type. Otherwise, basedealloc should have decref'd it already */ if (type_needs_decref) { - Py_DECREF(type); + _Py_DECREF_TYPE(type); } /* Done */ @@ -2123,15 +2539,7 @@ subtype_dealloc(PyObject *self) finalizers since they might rely on part of the object being finalized that has already been destroyed. */ if (type->tp_weaklistoffset && !base->tp_weaklistoffset) { - /* Modeled after GET_WEAKREFS_LISTPTR(). - - This is never triggered for static types so we can avoid the - (slightly) more costly _PyObject_GET_WEAKREFS_LISTPTR(). */ - PyWeakReference **list = \ - _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(self); - while (*list) { - _PyWeakref_ClearRef(*list); - } + _PyWeakref_ClearWeakRefsNoCallbacks(self); } } @@ -2146,14 +2554,7 @@ subtype_dealloc(PyObject *self) /* If we added a dict, DECREF it, or free inline values. */ if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(self); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - _PyObject_FreeInstanceAttributes(self); - } - else { - Py_XDECREF(_PyDictOrValues_GetDict(*dorv_ptr)); - } - dorv_ptr->values = NULL; + PyObject_ClearManagedDict(self); } else if (type->tp_dictoffset && !base->tp_dictoffset) { PyObject **dictptr = _PyObject_ComputedDictPointer(self); @@ -2189,7 +2590,7 @@ subtype_dealloc(PyObject *self) reference counting. Only decref if the base type is not already a heap allocated type. Otherwise, basedealloc should have decref'd it already */ if (type_needs_decref) { - Py_DECREF(type); + _Py_DECREF_TYPE(type); } endlabel: @@ -2250,37 +2651,34 @@ type_is_subtype_base_chain(PyTypeObject *a, PyTypeObject *b) } static int -is_subtype_unlocked(PyTypeObject *a, PyTypeObject *b) +is_subtype_with_mro(PyObject *a_mro, PyTypeObject *a, PyTypeObject *b) { - PyObject *mro; - - ASSERT_TYPE_LOCK_HELD(); - mro = lookup_tp_mro(a); - if (mro != NULL) { + int res; + if (a_mro != NULL) { /* Deal with multiple inheritance without recursion by walking the MRO tuple */ Py_ssize_t i, n; - assert(PyTuple_Check(mro)); - n = PyTuple_GET_SIZE(mro); + assert(PyTuple_Check(a_mro)); + n = PyTuple_GET_SIZE(a_mro); + res = 0; for (i = 0; i < n; i++) { - if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) - return 1; + if (PyTuple_GET_ITEM(a_mro, i) == (PyObject *)b) { + res = 1; + break; + } } - return 0; } - else + else { /* a is not completely initialized yet; follow tp_base */ - return type_is_subtype_base_chain(a, b); + res = type_is_subtype_base_chain(a, b); + } + return res; } int PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) { - int res; - BEGIN_TYPE_LOCK(); - res = is_subtype_unlocked(a, b); - END_TYPE_LOCK() - return res; + return is_subtype_with_mro(a->tp_mro, a, b); } /* Routines to do a method lookup in the type without looking in the @@ -2290,7 +2688,7 @@ PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) Variants: - _PyObject_LookupSpecial() returns NULL without raising an exception - when the _PyType_Lookup() call fails; + when the _PyType_LookupRef() call fails; - lookup_maybe_method() and lookup_method() are internal routines similar to _PyObject_LookupSpecial(), but can return unbound PyFunction @@ -2303,30 +2701,48 @@ _PyObject_LookupSpecial(PyObject *self, PyObject *attr) { PyObject *res; - res = _PyType_Lookup(Py_TYPE(self), attr); + res = _PyType_LookupRef(Py_TYPE(self), attr); if (res != NULL) { descrgetfunc f; - if ((f = Py_TYPE(res)->tp_descr_get) == NULL) - Py_INCREF(res); - else - res = f(res, self, (PyObject *)(Py_TYPE(self))); + if ((f = Py_TYPE(res)->tp_descr_get) != NULL) { + Py_SETREF(res, f(res, self, (PyObject *)(Py_TYPE(self)))); + } } return res; } +/* Steals a reference to self */ PyObject * -_PyObject_LookupSpecialId(PyObject *self, _Py_Identifier *attrid) +_PyObject_LookupSpecialMethod(PyObject *self, PyObject *attr, PyObject **self_or_null) { - PyObject *attr = _PyUnicode_FromId(attrid); /* borrowed */ - if (attr == NULL) + PyObject *res; + + res = _PyType_LookupRef(Py_TYPE(self), attr); + if (res == NULL) { + Py_DECREF(self); + *self_or_null = NULL; return NULL; - return _PyObject_LookupSpecial(self, attr); + } + + if (_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)) { + /* Avoid temporary PyMethodObject */ + *self_or_null = self; + } + else { + descrgetfunc f = Py_TYPE(res)->tp_descr_get; + if (f != NULL) { + Py_SETREF(res, f(res, self, (PyObject *)(Py_TYPE(self)))); + } + *self_or_null = NULL; + Py_DECREF(self); + } + return res; } static PyObject * lookup_maybe_method(PyObject *self, PyObject *attr, int *unbound) { - PyObject *res = _PyType_Lookup(Py_TYPE(self), attr); + PyObject *res = _PyType_LookupRef(Py_TYPE(self), attr); if (res == NULL) { return NULL; } @@ -2334,16 +2750,12 @@ lookup_maybe_method(PyObject *self, PyObject *attr, int *unbound) if (_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)) { /* Avoid temporary PyMethodObject */ *unbound = 1; - Py_INCREF(res); } else { *unbound = 0; descrgetfunc f = Py_TYPE(res)->tp_descr_get; - if (f == NULL) { - Py_INCREF(res); - } - else { - res = f(res, self, (PyObject *)(Py_TYPE(self))); + if (f != NULL) { + Py_SETREF(res, f(res, self, (PyObject *)(Py_TYPE(self)))); } } return res; @@ -2458,7 +2870,7 @@ vectorcall_maybe(PyThreadState *tstate, PyObject *name, */ static int -tail_contains(PyObject *tuple, int whence, PyObject *o) +tail_contains(PyObject *tuple, Py_ssize_t whence, PyObject *o) { Py_ssize_t j, size; size = PyTuple_GET_SIZE(tuple); @@ -2521,7 +2933,7 @@ check_duplicates(PyObject *tuple) */ static void -set_mro_error(PyObject **to_merge, Py_ssize_t to_merge_size, int *remain) +set_mro_error(PyObject **to_merge, Py_ssize_t to_merge_size, Py_ssize_t *remain) { Py_ssize_t i, n, off; char buf[1000]; @@ -2576,13 +2988,13 @@ pmerge(PyObject *acc, PyObject **to_merge, Py_ssize_t to_merge_size) { int res = 0; Py_ssize_t i, j, empty_cnt; - int *remain; + Py_ssize_t *remain; /* remain stores an index into each sublist of to_merge. remain[i] is the index of the next base in to_merge[i] that is not included in acc. */ - remain = PyMem_New(int, to_merge_size); + remain = PyMem_New(Py_ssize_t, to_merge_size); if (remain == NULL) { PyErr_NoMemory(); return -1; @@ -2729,9 +3141,9 @@ static PyObject * mro_implementation(PyTypeObject *type) { PyObject *mro; - BEGIN_TYPE_LOCK() + BEGIN_TYPE_LOCK(); mro = mro_implementation_unlocked(type); - END_TYPE_LOCK() + END_TYPE_LOCK(); return mro; } @@ -2773,7 +3185,7 @@ mro_check(PyTypeObject *type, PyObject *mro) } PyTypeObject *base = (PyTypeObject*)obj; - if (!is_subtype_unlocked(solid, solid_base(base))) { + if (!is_subtype_with_mro(lookup_tp_mro(solid), solid, solid_base(base))) { PyErr_Format( PyExc_TypeError, "mro() returned base with unsuitable layout ('%.500s')", @@ -2866,7 +3278,7 @@ mro_invoke(PyTypeObject *type) - Returns -1 in case of an error. */ static int -mro_internal_unlocked(PyTypeObject *type, PyObject **p_old_mro) +mro_internal_unlocked(PyTypeObject *type, int initial, PyObject **p_old_mro) { ASSERT_TYPE_LOCK_HELD(); @@ -2889,7 +3301,7 @@ mro_internal_unlocked(PyTypeObject *type, PyObject **p_old_mro) return 0; } - set_tp_mro(type, new_mro); + set_tp_mro(type, new_mro, initial); type_mro_modified(type, new_mro); /* corner case: the super class might have been hidden @@ -2903,7 +3315,7 @@ mro_internal_unlocked(PyTypeObject *type, PyObject **p_old_mro) else { /* For static builtin types, this is only called during init before the method cache has been populated. */ - assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); + assert(type->tp_version_tag); } if (p_old_mro != NULL) @@ -2918,9 +3330,9 @@ static int mro_internal(PyTypeObject *type, PyObject **p_old_mro) { int res; - BEGIN_TYPE_LOCK() - res = mro_internal_unlocked(type, p_old_mro); - END_TYPE_LOCK() + BEGIN_TYPE_LOCK(); + res = mro_internal_unlocked(type, 0, p_old_mro); + END_TYPE_LOCK(); return res; } @@ -3100,19 +3512,26 @@ subtype_setdict(PyObject *obj, PyObject *value, void *context) return func(descr, obj, value); } /* Almost like PyObject_GenericSetDict, but allow __dict__ to be deleted. */ - dictptr = _PyObject_GetDictPtr(obj); - if (dictptr == NULL) { - PyErr_SetString(PyExc_AttributeError, - "This object has no __dict__"); - return -1; - } if (value != NULL && !PyDict_Check(value)) { PyErr_Format(PyExc_TypeError, "__dict__ must be set to a dictionary, " "not a '%.200s'", Py_TYPE(value)->tp_name); return -1; } - Py_XSETREF(*dictptr, Py_XNewRef(value)); + + if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + return _PyObject_SetManagedDict(obj, value); + } + else { + dictptr = _PyObject_ComputedDictPointer(obj); + if (dictptr == NULL) { + PyErr_SetString(PyExc_AttributeError, + "This object has no __dict__"); + return -1; + } + Py_CLEAR(*dictptr); + *dictptr = Py_XNewRef(value); + } return 0; } @@ -3207,7 +3626,7 @@ type_init(PyObject *cls, PyObject *args, PyObject *kwds) unsigned long PyType_GetFlags(PyTypeObject *type) { - return type->tp_flags; + return FT_ATOMIC_LOAD_ULONG_RELAXED(type->tp_flags); } @@ -3507,7 +3926,7 @@ type_new_alloc(type_new_ctx *ctx) type->tp_as_mapping = &et->as_mapping; type->tp_as_buffer = &et->as_buffer; - set_tp_bases(type, Py_NewRef(ctx->bases)); + set_tp_bases(type, Py_NewRef(ctx->bases), 1); type->tp_base = (PyTypeObject *)Py_NewRef(ctx->base); type->tp_dealloc = subtype_dealloc; @@ -3521,6 +3940,11 @@ type_new_alloc(type_new_ctx *ctx) et->ht_name = Py_NewRef(ctx->name); et->ht_module = NULL; et->_ht_tpname = NULL; + et->ht_token = NULL; + +#ifdef Py_GIL_DISABLED + et->unique_id = _PyObject_AssignUniqueId((PyObject *)et); +#endif return type; } @@ -4224,10 +4648,71 @@ check_basicsize_includes_size_and_offsets(PyTypeObject* type) return 1; } -static PyObject * -_PyType_FromMetaclass_impl( +static int +check_immutable_bases(const char *type_name, PyObject *bases, int skip_first) +{ + Py_ssize_t i = 0; + if (skip_first) { + // When testing the MRO, skip the type itself + i = 1; + } + for (; itype != Py_T_PYSSIZET) { + PyErr_Format( + PyExc_SystemError, + "type of %s must be Py_T_PYSSIZET", + memb->name); + return -1; + } + if (memb->flags == Py_READONLY) { + *dest = memb->offset; + return 0; + } + else if (memb->flags == (Py_READONLY | Py_RELATIVE_OFFSET)) { + *dest = memb->offset + type_data_offset; + return 0; + } + PyErr_Format( + PyExc_SystemError, + "flags for %s must be Py_READONLY or (Py_READONLY | Py_RELATIVE_OFFSET)", + memb->name); + return -1; +} + +PyObject * +PyType_FromMetaclass( PyTypeObject *metaclass, PyObject *module, - PyType_Spec *spec, PyObject *bases_in, int _allow_tp_new) + PyType_Spec *spec, PyObject *bases_in) { /* Invariant: A non-NULL value in one of these means this function holds * a strong reference or owns allocated memory. @@ -4249,10 +4734,11 @@ _PyType_FromMetaclass_impl( const PyType_Slot *slot; Py_ssize_t nmembers = 0; - Py_ssize_t weaklistoffset, dictoffset, vectorcalloffset; + const PyMemberDef *weaklistoffset_member = NULL; + const PyMemberDef *dictoffset_member = NULL; + const PyMemberDef *vectorcalloffset_member = NULL; char *res_start; - nmembers = weaklistoffset = dictoffset = vectorcalloffset = 0; for (slot = spec->slots; slot->slot; slot++) { if (slot->slot < 0 || (size_t)slot->slot >= Py_ARRAY_LENGTH(pyslot_offsets)) { @@ -4269,24 +4755,6 @@ _PyType_FromMetaclass_impl( } for (const PyMemberDef *memb = slot->pfunc; memb->name != NULL; memb++) { nmembers++; - if (strcmp(memb->name, "__weaklistoffset__") == 0) { - // The PyMemberDef must be a Py_ssize_t and readonly - assert(memb->type == Py_T_PYSSIZET); - assert(memb->flags == Py_READONLY); - weaklistoffset = memb->offset; - } - if (strcmp(memb->name, "__dictoffset__") == 0) { - // The PyMemberDef must be a Py_ssize_t and readonly - assert(memb->type == Py_T_PYSSIZET); - assert(memb->flags == Py_READONLY); - dictoffset = memb->offset; - } - if (strcmp(memb->name, "__vectorcalloffset__") == 0) { - // The PyMemberDef must be a Py_ssize_t and readonly - assert(memb->type == Py_T_PYSSIZET); - assert(memb->flags == Py_READONLY); - vectorcalloffset = memb->offset; - } if (memb->flags & Py_RELATIVE_OFFSET) { if (spec->basicsize > 0) { PyErr_SetString( @@ -4301,6 +4769,15 @@ _PyType_FromMetaclass_impl( goto finally; } } + if (strcmp(memb->name, "__weaklistoffset__") == 0) { + weaklistoffset_member = memb; + } + else if (strcmp(memb->name, "__dictoffset__") == 0) { + dictoffset_member = memb; + } + else if (strcmp(memb->name, "__vectorcalloffset__") == 0) { + vectorcalloffset_member = memb; + } } break; case Py_tp_doc: @@ -4380,23 +4857,8 @@ _PyType_FromMetaclass_impl( * and only heap types can be mutable.) */ if (spec->flags & Py_TPFLAGS_IMMUTABLETYPE) { - for (int i=0; iname, - b->tp_name)) - { - goto finally; - } - } + if (check_immutable_bases(spec->name, bases, 0) < 0) { + goto finally; } } @@ -4416,21 +4878,10 @@ _PyType_FromMetaclass_impl( goto finally; } if (metaclass->tp_new && metaclass->tp_new != PyType_Type.tp_new) { - if (_allow_tp_new) { - if (PyErr_WarnFormat( - PyExc_DeprecationWarning, 1, - "Type %s uses PyType_Spec with a metaclass that has custom " - "tp_new. This is deprecated and will no longer be allowed in " - "Python 3.14.", spec->name) < 0) { - goto finally; - } - } - else { - PyErr_SetString( - PyExc_TypeError, - "Metaclasses with custom tp_new are not supported."); - goto finally; - } + PyErr_SetString( + PyExc_TypeError, + "Metaclasses with custom tp_new are not supported."); + goto finally; } /* Calculate best base, and check that all bases are type objects */ @@ -4468,6 +4919,24 @@ _PyType_FromMetaclass_impl( Py_ssize_t itemsize = spec->itemsize; + /* Compute special offsets */ + + Py_ssize_t weaklistoffset = 0; + if (special_offset_from_member(weaklistoffset_member, type_data_offset, + &weaklistoffset) < 0) { + goto finally; + } + Py_ssize_t dictoffset = 0; + if (special_offset_from_member(dictoffset_member, type_data_offset, + &dictoffset) < 0) { + goto finally; + } + Py_ssize_t vectorcalloffset = 0; + if (special_offset_from_member(vectorcalloffset_member, type_data_offset, + &vectorcalloffset) < 0) { + goto finally; + } + /* Allocate the new type * * Between here and PyType_Ready, we should limit: @@ -4499,7 +4968,7 @@ _PyType_FromMetaclass_impl( /* Set slots we have prepared */ type->tp_base = (PyTypeObject *)Py_NewRef(base); - set_tp_bases(type, bases); + set_tp_bases(type, bases, 1); bases = NULL; // We give our reference to bases to the type type->tp_doc = tp_doc; @@ -4545,6 +5014,11 @@ _PyType_FromMetaclass_impl( } } break; + case Py_tp_token: + { + res->ht_token = slot->pfunc == Py_TP_USE_SPEC ? spec : slot->pfunc; + } + break; default: { /* Copy other slots directly */ @@ -4576,6 +5050,11 @@ _PyType_FromMetaclass_impl( type->tp_weaklistoffset = weaklistoffset; type->tp_dictoffset = dictoffset; +#ifdef Py_GIL_DISABLED + // Assign a unique id to enable per-thread refcounting + res->unique_id = _PyObject_AssignUniqueId((PyObject *)res); +#endif + /* Ready the type (which includes inheritance). * * After this call we should generally only touch up what's @@ -4654,29 +5133,22 @@ _PyType_FromMetaclass_impl( return (PyObject*)res; } -PyObject * -PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, - PyType_Spec *spec, PyObject *bases_in) -{ - return _PyType_FromMetaclass_impl(metaclass, module, spec, bases_in, 0); -} - PyObject * PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) { - return _PyType_FromMetaclass_impl(NULL, module, spec, bases, 1); + return PyType_FromMetaclass(NULL, module, spec, bases); } PyObject * PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) { - return _PyType_FromMetaclass_impl(NULL, NULL, spec, bases, 1); + return PyType_FromMetaclass(NULL, NULL, spec, bases); } PyObject * PyType_FromSpec(PyType_Spec *spec) { - return _PyType_FromMetaclass_impl(NULL, NULL, spec, NULL, 1); + return PyType_FromMetaclass(NULL, NULL, spec, NULL); } PyObject * @@ -4692,9 +5164,9 @@ PyType_GetQualName(PyTypeObject *type) } PyObject * -_PyType_GetModuleName(PyTypeObject *type) +PyType_GetModuleName(PyTypeObject *type) { - return type_module(type, NULL); + return type_module(type); } void * @@ -4707,8 +5179,15 @@ PyType_GetSlot(PyTypeObject *type, int slot) PyErr_BadInternalCall(); return NULL; } + int slot_offset = pyslot_offsets[slot].slot_offset; - parent_slot = *(void**)((char*)type + pyslot_offsets[slot].slot_offset); + if (slot_offset >= (int)sizeof(PyTypeObject)) { + if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { + return NULL; + } + } + + parent_slot = *(void**)((char*)type + slot_offset); if (parent_slot == NULL) { return NULL; } @@ -4762,21 +5241,36 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) { assert(PyType_Check(type)); + if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { + // type_ready_mro() ensures that no heap type is + // contained in a static type MRO. + return NULL; + } + else { + PyHeapTypeObject *ht = (PyHeapTypeObject*)type; + PyObject *module = ht->ht_module; + if (module && _PyModule_GetDef(module) == def) { + return module; + } + } + PyObject *res = NULL; - BEGIN_TYPE_LOCK() + BEGIN_TYPE_LOCK(); PyObject *mro = lookup_tp_mro(type); // The type must be ready assert(mro != NULL); assert(PyTuple_Check(mro)); - // mro_invoke() ensures that the type MRO cannot be empty, so we don't have - // to check i < PyTuple_GET_SIZE(mro) at the first loop iteration. + // mro_invoke() ensures that the type MRO cannot be empty. assert(PyTuple_GET_SIZE(mro) >= 1); + // Also, the first item in the MRO is the type itself, which + // we already checked above. We skip it in the loop. + assert(PyTuple_GET_ITEM(mro, 0) == (PyObject *)type); Py_ssize_t n = PyTuple_GET_SIZE(mro); - for (Py_ssize_t i = 0; i < n; i++) { + for (Py_ssize_t i = 1; i < n; i++) { PyObject *super = PyTuple_GET_ITEM(mro, i); - if(!_PyType_HasFeature((PyTypeObject *)super, Py_TPFLAGS_HEAPTYPE)) { + if (!_PyType_HasFeature((PyTypeObject *)super, Py_TPFLAGS_HEAPTYPE)) { // Static types in the MRO need to be skipped continue; } @@ -4788,7 +5282,7 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) break; } } - END_TYPE_LOCK() + END_TYPE_LOCK(); if (res == NULL) { PyErr_Format( @@ -4799,6 +5293,92 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) return res; } + +static PyTypeObject * +get_base_by_token_recursive(PyObject *bases, void *token) +{ + assert(bases != NULL); + PyTypeObject *res = NULL; + Py_ssize_t n = PyTuple_GET_SIZE(bases); + for (Py_ssize_t i = 0; i < n; i++) { + PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(bases, i)); + if (!_PyType_HasFeature(base, Py_TPFLAGS_HEAPTYPE)) { + continue; + } + if (((PyHeapTypeObject*)base)->ht_token == token) { + res = base; + break; + } + base = get_base_by_token_recursive(lookup_tp_bases(base), token); + if (base != NULL) { + res = base; + break; + } + } + return res; // Prefer to return recursively from one place +} + +int +PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) +{ + if (result != NULL) { + *result = NULL; + } + + if (token == NULL) { + PyErr_Format(PyExc_SystemError, + "PyType_GetBaseByToken called with token=NULL"); + return -1; + } + if (!PyType_Check(type)) { + PyErr_Format(PyExc_TypeError, + "expected a type, got a '%T' object", type); + return -1; + } + + if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { + // No static type has a heaptype superclass, + // which is ensured by type_ready_mro(). + return 0; + } + if (((PyHeapTypeObject*)type)->ht_token == token) { +found: + if (result != NULL) { + *result = (PyTypeObject *)Py_NewRef(type); + } + return 1; + } + + PyObject *mro = type->tp_mro; // No lookup, following PyType_IsSubtype() + if (mro == NULL) { + PyTypeObject *base; + base = get_base_by_token_recursive(lookup_tp_bases(type), token); + if (base != NULL) { + // Copying the given type can cause a slowdown, + // unlike the overwrite below. + type = base; + goto found; + } + return 0; + } + // mro_invoke() ensures that the type MRO cannot be empty. + assert(PyTuple_GET_SIZE(mro) >= 1); + // Also, the first item in the MRO is the type itself, which + // we already checked above. We skip it in the loop. + assert(PyTuple_GET_ITEM(mro, 0) == (PyObject *)type); + Py_ssize_t n = PyTuple_GET_SIZE(mro); + for (Py_ssize_t i = 1; i < n; i++) { + PyTypeObject *base = (PyTypeObject *)PyTuple_GET_ITEM(mro, i); + if (_PyType_HasFeature(base, Py_TPFLAGS_HEAPTYPE) + && ((PyHeapTypeObject*)base)->ht_token == token) { + type = base; + goto found; + } + } + return 0; +} + + void * PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls) { @@ -4836,15 +5416,10 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) { ASSERT_TYPE_LOCK_HELD(); - Py_hash_t hash; - if (!PyUnicode_CheckExact(name) || - (hash = _PyASCIIObject_CAST(name)->hash) == -1) - { - hash = PyObject_Hash(name); - if (hash == -1) { - *error = -1; - return NULL; - } + Py_hash_t hash = _PyObject_HashFast(name); + if (hash == -1) { + *error = -1; + return NULL; } /* Look in tp_dict of types in MRO */ @@ -4872,14 +5447,13 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) PyObject *base = PyTuple_GET_ITEM(mro, i); PyObject *dict = lookup_tp_dict(_PyType_CAST(base)); assert(dict && PyDict_Check(dict)); - res = _PyDict_GetItem_KnownHash(dict, name, hash); - if (res != NULL) { - break; - } - if (PyErr_Occurred()) { + if (_PyDict_GetItemRef_KnownHash((PyDictObject *)dict, name, hash, &res) < 0) { *error = -1; goto done; } + if (res != NULL) { + break; + } } *error = 0; done: @@ -4905,22 +5479,22 @@ is_dunder_name(PyObject *name) return 0; } -static void +static PyObject * update_cache(struct type_cache_entry *entry, PyObject *name, unsigned int version_tag, PyObject *value) { - entry->version = version_tag; - entry->value = value; /* borrowed */ + _Py_atomic_store_uint32_relaxed(&entry->version, version_tag); + _Py_atomic_store_ptr_relaxed(&entry->value, value); /* borrowed */ assert(_PyASCIIObject_CAST(name)->hash != -1); OBJECT_STAT_INC_COND(type_cache_collisions, entry->name != Py_None && entry->name != name); // We're releasing this under the lock for simplicity sake because it's always a // exact unicode object or Py_None so it's safe to do so. - Py_SETREF(entry->name, Py_NewRef(name)); + PyObject *old_name = entry->name; + _Py_atomic_store_ptr_relaxed(&entry->name, Py_NewRef(name)); + return old_name; } #if Py_GIL_DISABLED -#define TYPE_CACHE_IS_UPDATING(sequence) (sequence & 0x01) - static void update_cache_gil_disabled(struct type_cache_entry *entry, PyObject *name, unsigned int version_tag, PyObject *value) @@ -4936,10 +5510,12 @@ update_cache_gil_disabled(struct type_cache_entry *entry, PyObject *name, return; } - update_cache(entry, name, version_tag, value); + PyObject *old_value = update_cache(entry, name, version_tag, value); // Then update sequence to the next valid value _PySeqLock_UnlockWrite(&entry->sequence); + + Py_DECREF(old_value); } #endif @@ -4962,9 +5538,12 @@ _PyTypes_AfterFork(void) } /* Internal API to look for a name through the MRO. - This returns a borrowed reference, and doesn't set an exception! */ + This returns a strong reference, and doesn't set an exception! + If nonzero, version is set to the value of type->tp_version at the time of + the lookup. +*/ PyObject * -_PyType_Lookup(PyTypeObject *type, PyObject *name) +_PyType_LookupRefAndVersion(PyTypeObject *type, PyObject *name, unsigned int *version) { PyObject *res; int error; @@ -4976,19 +5555,30 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) #ifdef Py_GIL_DISABLED // synchronize-with other writing threads by doing an acquire load on the sequence while (1) { - int sequence = _PySeqLock_BeginRead(&entry->sequence); - if (_Py_atomic_load_uint32_relaxed(&entry->version) == type->tp_version_tag && + uint32_t sequence = _PySeqLock_BeginRead(&entry->sequence); + uint32_t entry_version = _Py_atomic_load_uint32_relaxed(&entry->version); + uint32_t type_version = _Py_atomic_load_uint32_acquire(&type->tp_version_tag); + if (entry_version == type_version && _Py_atomic_load_ptr_relaxed(&entry->name) == name) { - assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); OBJECT_STAT_INC_COND(type_cache_hits, !is_dunder_name(name)); OBJECT_STAT_INC_COND(type_cache_dunder_hits, is_dunder_name(name)); PyObject *value = _Py_atomic_load_ptr_relaxed(&entry->value); - // If the sequence is still valid then we're done - if (_PySeqLock_EndRead(&entry->sequence, sequence)) { - return value; + if (value == NULL || _Py_TryIncref(value)) { + if (_PySeqLock_EndRead(&entry->sequence, sequence)) { + if (version != NULL) { + *version = entry_version; + } + return value; + } + Py_XDECREF(value); } - } else { + else { + // If we can't incref the object we need to fallback to locking + break; + } + } + else { // cache miss break; } @@ -4996,9 +5586,13 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) #else if (entry->version == type->tp_version_tag && entry->name == name) { - assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); + assert(type->tp_version_tag); OBJECT_STAT_INC_COND(type_cache_hits, !is_dunder_name(name)); OBJECT_STAT_INC_COND(type_cache_dunder_hits, is_dunder_name(name)); + Py_XINCREF(entry->value); + if (version != NULL) { + *version = entry->version; + } return entry->value; } #endif @@ -5012,15 +5606,14 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) // anyone else can modify our mro or mutate the type. int has_version = 0; - int version = 0; - BEGIN_TYPE_LOCK() + unsigned int assigned_version = 0; + BEGIN_TYPE_LOCK(); res = find_name_in_mro(type, name, &error); if (MCACHE_CACHEABLE_NAME(name)) { has_version = assign_version_tag(interp, type); - version = type->tp_version_tag; - assert(!has_version || _PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); + assigned_version = type->tp_version_tag; } - END_TYPE_LOCK() + END_TYPE_LOCK(); /* Only put NULL results into cache if there was no error. */ if (error) { @@ -5035,31 +5628,158 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) if (error == -1) { PyErr_Clear(); } + if (version != NULL) { + // 0 is not a valid version + *version = 0; + } return NULL; } if (has_version) { #if Py_GIL_DISABLED - update_cache_gil_disabled(entry, name, version, res); + update_cache_gil_disabled(entry, name, assigned_version, res); #else - update_cache(entry, name, version, res); + PyObject *old_value = update_cache(entry, name, assigned_version, res); + Py_DECREF(old_value); #endif } + if (version != NULL) { + // 0 is not a valid version + *version = has_version ? assigned_version : 0; + } return res; } +/* Internal API to look for a name through the MRO. + This returns a strong reference, and doesn't set an exception! +*/ PyObject * -_PyType_LookupId(PyTypeObject *type, _Py_Identifier *name) +_PyType_LookupRef(PyTypeObject *type, PyObject *name) { - PyObject *oname; - oname = _PyUnicode_FromId(name); /* borrowed */ - if (oname == NULL) - return NULL; - return _PyType_Lookup(type, oname); + return _PyType_LookupRefAndVersion(type, name, NULL); +} + +/* Internal API to look for a name through the MRO. + This returns a borrowed reference, and doesn't set an exception! */ +PyObject * +_PyType_Lookup(PyTypeObject *type, PyObject *name) +{ + PyObject *res = _PyType_LookupRefAndVersion(type, name, NULL); + Py_XDECREF(res); + return res; +} + +int +_PyType_CacheInitForSpecialization(PyHeapTypeObject *type, PyObject *init, + unsigned int tp_version) +{ + if (!init || !tp_version) { + return 0; + } + int can_cache; + BEGIN_TYPE_LOCK(); + can_cache = ((PyTypeObject*)type)->tp_version_tag == tp_version; + #ifdef Py_GIL_DISABLED + can_cache = can_cache && _PyObject_HasDeferredRefcount(init); + #endif + if (can_cache) { + FT_ATOMIC_STORE_PTR_RELEASE(type->_spec_cache.init, init); + } + END_TYPE_LOCK(); + return can_cache; +} + +int +_PyType_CacheGetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor, uint32_t tp_version) +{ + if (!descriptor || !tp_version) { + return 0; + } + int can_cache; + BEGIN_TYPE_LOCK(); + can_cache = ((PyTypeObject*)ht)->tp_version_tag == tp_version; + // This pointer is invalidated by PyType_Modified (see the comment on + // struct _specialization_cache): + PyFunctionObject *func = (PyFunctionObject *)descriptor; + uint32_t version = _PyFunction_GetVersionForCurrentState(func); + can_cache = can_cache && _PyFunction_IsVersionValid(version); +#ifdef Py_GIL_DISABLED + can_cache = can_cache && _PyObject_HasDeferredRefcount(descriptor); +#endif + if (can_cache) { + FT_ATOMIC_STORE_PTR_RELEASE(ht->_spec_cache.getitem, descriptor); + FT_ATOMIC_STORE_UINT32_RELAXED(ht->_spec_cache.getitem_version, version); + } + END_TYPE_LOCK(); + return can_cache; +} + +static void +set_flags(PyTypeObject *self, unsigned long mask, unsigned long flags) +{ + ASSERT_TYPE_LOCK_HELD(); + self->tp_flags = (self->tp_flags & ~mask) | flags; +} + +void +_PyType_SetFlags(PyTypeObject *self, unsigned long mask, unsigned long flags) +{ + BEGIN_TYPE_LOCK(); + set_flags(self, mask, flags); + END_TYPE_LOCK(); +} + +int +_PyType_Validate(PyTypeObject *ty, _py_validate_type validate, unsigned int *tp_version) +{ + int err; + BEGIN_TYPE_LOCK(); + err = validate(ty); + if (!err) { + if(assign_version_tag(_PyInterpreterState_GET(), ty)) { + *tp_version = ty->tp_version_tag; + } + else { + err = -1; + } + } + END_TYPE_LOCK(); + return err; +} + +static void +set_flags_recursive(PyTypeObject *self, unsigned long mask, unsigned long flags) +{ + if (PyType_HasFeature(self, Py_TPFLAGS_IMMUTABLETYPE) || + (self->tp_flags & mask) == flags) + { + return; + } + + set_flags(self, mask, flags); + + PyObject *children = _PyType_GetSubclasses(self); + if (children == NULL) { + return; + } + + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(children); i++) { + PyObject *child = PyList_GET_ITEM(children, i); + set_flags_recursive((PyTypeObject *)child, mask, flags); + } + Py_DECREF(children); +} + +void +_PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask, unsigned long flags) +{ + BEGIN_TYPE_LOCK(); + set_flags_recursive(self, mask, flags); + END_TYPE_LOCK(); } /* This is similar to PyObject_GenericGetAttr(), - but uses _PyType_Lookup() instead of just looking in type->tp_dict. + but uses _PyType_LookupRef() instead of just looking in type->tp_dict. The argument suppress_missing_attribute is used to provide a fast path for hasattr. The possible values are: @@ -5095,10 +5815,9 @@ _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int * suppress_missin meta_get = NULL; /* Look for the attribute in the metatype */ - meta_attribute = _PyType_Lookup(metatype, name); + meta_attribute = _PyType_LookupRef(metatype, name); if (meta_attribute != NULL) { - Py_INCREF(meta_attribute); meta_get = Py_TYPE(meta_attribute)->tp_descr_get; if (meta_get != NULL && PyDescr_IsData(meta_attribute)) { @@ -5115,10 +5834,9 @@ _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int * suppress_missin /* No data descriptor found on metatype. Look in tp_dict of this * type and its bases */ - attribute = _PyType_Lookup(type, name); + attribute = _PyType_LookupRef(type, name); if (attribute != NULL) { /* Implement descriptor functionality, if any */ - Py_INCREF(attribute); descrgetfunc local_get = Py_TYPE(attribute)->tp_descr_get; Py_XDECREF(meta_attribute); @@ -5163,13 +5881,49 @@ _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int * suppress_missin } /* This is similar to PyObject_GenericGetAttr(), - but uses _PyType_Lookup() instead of just looking in type->tp_dict. */ + but uses _PyType_LookupRef() instead of just looking in type->tp_dict. */ PyObject * _Py_type_getattro(PyObject *type, PyObject *name) { return _Py_type_getattro_impl((PyTypeObject *)type, name, NULL); } +static int +type_update_dict(PyTypeObject *type, PyDictObject *dict, PyObject *name, + PyObject *value, PyObject **old_value) +{ + // We don't want any re-entrancy between when we update the dict + // and call type_modified_unlocked, including running the destructor + // of the current value as it can observe the cache in an inconsistent + // state. Because we have an exact unicode and our dict has exact + // unicodes we know that this will all complete without releasing + // the locks. + if (_PyDict_GetItemRef_Unicode_LockHeld(dict, name, old_value) < 0) { + return -1; + } + + /* Clear the VALID_VERSION flag of 'type' and all its + subclasses. This could possibly be unified with the + update_subclasses() recursion in update_slot(), but carefully: + they each have their own conditions on which to stop + recursing into subclasses. */ + type_modified_unlocked(type); + + if (_PyDict_SetItem_LockHeld(dict, name, value) < 0) { + PyErr_Format(PyExc_AttributeError, + "type object '%.50s' has no attribute '%U'", + ((PyTypeObject*)type)->tp_name, name); + _PyObject_SetAttributeErrorContext((PyObject *)type, name); + return -1; + } + + if (is_dunder_name(name)) { + return update_slot(type, name); + } + + return 0; +} + static int type_setattro(PyObject *self, PyObject *name, PyObject *value) { @@ -5182,49 +5936,70 @@ type_setattro(PyObject *self, PyObject *name, PyObject *value) name, type->tp_name); return -1; } - if (PyUnicode_Check(name)) { - if (PyUnicode_CheckExact(name)) { - Py_INCREF(name); - } - else { - name = _PyUnicode_Copy(name); - if (name == NULL) - return -1; - } - /* bpo-40521: Interned strings are shared by all subinterpreters */ + if (!PyUnicode_Check(name)) { + PyErr_Format(PyExc_TypeError, + "attribute name must be string, not '%.200s'", + Py_TYPE(name)->tp_name); + return -1; + } + + if (PyUnicode_CheckExact(name)) { + Py_INCREF(name); + } + else { + name = _PyUnicode_Copy(name); + if (name == NULL) + return -1; + } + if (!PyUnicode_CHECK_INTERNED(name)) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternMortal(interp, &name); if (!PyUnicode_CHECK_INTERNED(name)) { - PyUnicode_InternInPlace(&name); - if (!PyUnicode_CHECK_INTERNED(name)) { - PyErr_SetString(PyExc_MemoryError, - "Out of memory interning an attribute name"); - Py_DECREF(name); - return -1; - } + PyErr_SetString(PyExc_MemoryError, + "Out of memory interning an attribute name"); + Py_DECREF(name); + return -1; } } - else { - /* Will fail in _PyObject_GenericSetAttrWithDict. */ - Py_INCREF(name); - } - BEGIN_TYPE_LOCK() - res = _PyObject_GenericSetAttrWithDict((PyObject *)type, name, value, NULL); - if (res == 0) { - /* Clear the VALID_VERSION flag of 'type' and all its - subclasses. This could possibly be unified with the - update_subclasses() recursion in update_slot(), but carefully: - they each have their own conditions on which to stop - recursing into subclasses. */ - type_modified_unlocked(type); + PyTypeObject *metatype = Py_TYPE(type); + assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_INLINE_VALUES)); + assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_MANAGED_DICT)); + + PyObject *old_value = NULL; + PyObject *descr = _PyType_LookupRef(metatype, name); + if (descr != NULL) { + descrsetfunc f = Py_TYPE(descr)->tp_descr_set; + if (f != NULL) { + res = f(descr, (PyObject *)type, value); + goto done; + } + } - if (is_dunder_name(name)) { - res = update_slot(type, name); + PyObject *dict = type->tp_dict; + if (dict == NULL) { + // We don't just do PyType_Ready because we could already be readying + BEGIN_TYPE_LOCK(); + dict = type->tp_dict; + if (dict == NULL) { + dict = type->tp_dict = PyDict_New(); + } + END_TYPE_LOCK(); + if (dict == NULL) { + res = -1; + goto done; } - assert(_PyType_CheckConsistency(type)); } - END_TYPE_LOCK() + BEGIN_TYPE_DICT_LOCK(dict); + res = type_update_dict(type, (PyDictObject *)dict, name, value, &old_value); + assert(_PyType_CheckConsistency(type)); + END_TYPE_DICT_LOCK(); + +done: Py_DECREF(name); + Py_XDECREF(descr); + Py_XDECREF(old_value); return res; } @@ -5242,7 +6017,7 @@ type_dealloc_common(PyTypeObject *type) static void -clear_static_tp_subclasses(PyTypeObject *type) +clear_static_tp_subclasses(PyTypeObject *type, int isbuiltin) { PyObject *subclasses = lookup_tp_subclasses(type); if (subclasses == NULL) { @@ -5279,47 +6054,74 @@ clear_static_tp_subclasses(PyTypeObject *type) continue; } // All static builtin subtypes should have been finalized already. - assert(!(subclass->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); + assert(!isbuiltin || !(subclass->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); Py_DECREF(subclass); } +#else + (void)isbuiltin; #endif clear_tp_subclasses(type); } static void -clear_static_type_objects(PyInterpreterState *interp, PyTypeObject *type) +clear_static_type_objects(PyInterpreterState *interp, PyTypeObject *type, + int isbuiltin, int final) { - if (_Py_IsMainInterpreter(interp)) { + if (final) { Py_CLEAR(type->tp_cache); } clear_tp_dict(type); - clear_tp_bases(type); - clear_tp_mro(type); - clear_static_tp_subclasses(type); + clear_tp_bases(type, final); + clear_tp_mro(type, final); + clear_static_tp_subclasses(type, isbuiltin); } -void -_PyStaticType_Dealloc(PyInterpreterState *interp, PyTypeObject *type) + +static void +fini_static_type(PyInterpreterState *interp, PyTypeObject *type, + int isbuiltin, int final) { assert(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); assert(_Py_IsImmortal((PyObject *)type)); type_dealloc_common(type); - clear_static_type_objects(interp, type); + clear_static_type_objects(interp, type, isbuiltin, final); - if (_Py_IsMainInterpreter(interp)) { + if (final) { type->tp_flags &= ~Py_TPFLAGS_READY; - type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - type->tp_version_tag = 0; + _PyType_SetVersion(type, 0); } _PyStaticType_ClearWeakRefs(interp, type); - static_builtin_state_clear(interp, type); + managed_static_type_state_clear(interp, type, isbuiltin, final); /* We leave _Py_TPFLAGS_STATIC_BUILTIN set on tp_flags. */ } +void +_PyTypes_FiniExtTypes(PyInterpreterState *interp) +{ + for (size_t i = _Py_MAX_MANAGED_STATIC_EXT_TYPES; i > 0; i--) { + if (interp->types.for_extensions.num_initialized == 0) { + break; + } + int64_t count = 0; + PyTypeObject *type = static_ext_type_lookup(interp, i-1, &count); + if (type == NULL) { + continue; + } + int final = (count == 1); + fini_static_type(interp, type, 0, final); + } +} + +void +_PyStaticType_FiniBuiltin(PyInterpreterState *interp, PyTypeObject *type) +{ + fini_static_type(interp, type, 1, _Py_IsMainInterpreter(interp)); +} + static void type_dealloc(PyObject *self) @@ -5330,7 +6132,6 @@ type_dealloc(PyObject *self) _PyObject_ASSERT((PyObject *)type, type->tp_flags & Py_TPFLAGS_HEAPTYPE); _PyObject_GC_UNTRACK(type); - type_dealloc_common(type); // PyObject_ClearWeakRefs() raises an exception if Py_REFCNT() != 0 @@ -5358,6 +6159,10 @@ type_dealloc(PyObject *self) } Py_XDECREF(et->ht_module); PyMem_Free(et->_ht_tpname); +#ifdef Py_GIL_DISABLED + assert(et->unique_id == _Py_INVALID_UNIQUE_ID); +#endif + et->ht_token = NULL; Py_TYPE(type)->tp_free((PyObject *)type); } @@ -5555,7 +6360,7 @@ type_clear(PyObject *self) the dict, so that other objects caught in a reference cycle don't start calling destroyed methods. - Otherwise, the we need to clear tp_mro, which is + Otherwise, we need to clear tp_mro, which is part of a hard cycle (its first element is the class itself) that won't be broken otherwise (it's a tuple and tuples don't have a tp_clear handler). @@ -5761,15 +6566,19 @@ object_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } comma_w_quotes_sep = PyUnicode_FromString("', '"); + if (!comma_w_quotes_sep) { + Py_DECREF(sorted_methods); + return NULL; + } joined = PyUnicode_Join(comma_w_quotes_sep, sorted_methods); - method_count = PyObject_Length(sorted_methods); - Py_DECREF(sorted_methods); + Py_DECREF(comma_w_quotes_sep); if (joined == NULL) { - Py_DECREF(comma_w_quotes_sep); + Py_DECREF(sorted_methods); return NULL; } + method_count = PyObject_Length(sorted_methods); + Py_DECREF(sorted_methods); if (method_count == -1) { - Py_DECREF(comma_w_quotes_sep); Py_DECREF(joined); return NULL; } @@ -5781,17 +6590,12 @@ object_new(PyTypeObject *type, PyObject *args, PyObject *kwds) method_count > 1 ? "s" : "", joined); Py_DECREF(joined); - Py_DECREF(comma_w_quotes_sep); return NULL; } PyObject *obj = type->tp_alloc(type, 0); if (obj == NULL) { return NULL; } - if (_PyObject_InitializeDict(obj)) { - Py_DECREF(obj); - return NULL; - } return obj; } @@ -5808,7 +6612,7 @@ object_repr(PyObject *self) PyObject *mod, *name, *rtn; type = Py_TYPE(self); - mod = type_module(type, NULL); + mod = type_module(type); if (mod == NULL) PyErr_Clear(); else if (!PyUnicode_Check(mod)) { @@ -5975,6 +6779,11 @@ compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char* !same_slots_added(newbase, oldbase))) { goto differs; } + if ((oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) != + ((newto->tp_flags & Py_TPFLAGS_INLINE_VALUES))) + { + goto differs; + } /* The above does not check for the preheader */ if ((oldto->tp_flags & Py_TPFLAGS_PREHEADER) == ((newto->tp_flags & Py_TPFLAGS_PREHEADER))) @@ -5991,29 +6800,13 @@ compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char* return 0; } + + static int -object_set_class(PyObject *self, PyObject *value, void *closure) +object_set_class_world_stopped(PyObject *self, PyTypeObject *newto) { PyTypeObject *oldto = Py_TYPE(self); - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, - "can't delete __class__ attribute"); - return -1; - } - if (!PyType_Check(value)) { - PyErr_Format(PyExc_TypeError, - "__class__ must be set to a class, not '%s' object", - Py_TYPE(value)->tp_name); - return -1; - } - PyTypeObject *newto = (PyTypeObject *)value; - - if (PySys_Audit("object.__setattr__", "OsO", - self, "__class__", value) < 0) { - return -1; - } - /* In versions of CPython prior to 3.5, the code in compatible_for_assignment was not set up to correctly check for memory layout / slot / etc. compatibility for non-HEAPTYPE classes, so we just @@ -6076,23 +6869,28 @@ object_set_class(PyObject *self, PyObject *value, void *closure) if (compatible_for_assignment(oldto, newto, "__class__")) { /* Changing the class will change the implicit dict keys, * so we must materialize the dictionary first. */ - assert((oldto->tp_flags & Py_TPFLAGS_PREHEADER) == (newto->tp_flags & Py_TPFLAGS_PREHEADER)); - _PyObject_GetDictPtr(self); - if (oldto->tp_flags & Py_TPFLAGS_MANAGED_DICT && - _PyDictOrValues_IsValues(*_PyObject_DictOrValuesPointer(self))) - { - /* Was unable to convert to dict */ - PyErr_NoMemory(); - return -1; + if (oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictObject *dict = _PyObject_GetManagedDict(self); + if (dict == NULL) { + dict = _PyObject_MaterializeManagedDict_LockHeld(self); + if (dict == NULL) { + return -1; + } + } + + assert(_PyObject_GetManagedDict(self) == dict); + + if (_PyDict_DetachFromObject(dict, self) < 0) { + return -1; + } + } if (newto->tp_flags & Py_TPFLAGS_HEAPTYPE) { Py_INCREF(newto); } + Py_SET_TYPE(self, newto); - if (oldto->tp_flags & Py_TPFLAGS_HEAPTYPE) - Py_DECREF(oldto); - RARE_EVENT_INC(set_class); return 0; } else { @@ -6100,6 +6898,48 @@ object_set_class(PyObject *self, PyObject *value, void *closure) } } +static int +object_set_class(PyObject *self, PyObject *value, void *closure) +{ + + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "can't delete __class__ attribute"); + return -1; + } + if (!PyType_Check(value)) { + PyErr_Format(PyExc_TypeError, + "__class__ must be set to a class, not '%s' object", + Py_TYPE(value)->tp_name); + return -1; + } + PyTypeObject *newto = (PyTypeObject *)value; + + if (PySys_Audit("object.__setattr__", "OsO", + self, "__class__", value) < 0) { + return -1; + } + +#ifdef Py_GIL_DISABLED + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); +#endif + PyTypeObject *oldto = Py_TYPE(self); + int res = object_set_class_world_stopped(self, newto); +#ifdef Py_GIL_DISABLED + _PyEval_StartTheWorld(interp); +#endif + if (res == 0) { + if (oldto->tp_flags & Py_TPFLAGS_HEAPTYPE) { + Py_DECREF(oldto); + } + + RARE_EVENT_INC(set_class); + return 0; + } + return res; +} + static PyGetSetDef object_getsets[] = { {"__class__", object_get_class, object_set_class, PyDoc_STR("the object's class")}, @@ -6271,7 +7111,7 @@ object_getstate_default(PyObject *obj, int required) iterate over it */ if (slotnames_size != PyList_GET_SIZE(slotnames)) { PyErr_Format(PyExc_RuntimeError, - "__slotsname__ changed size during iteration"); + "__slotnames__ changed size during iteration"); goto error; } @@ -6636,18 +7476,7 @@ static PyObject * object___reduce_ex___impl(PyObject *self, int protocol) /*[clinic end generated code: output=2e157766f6b50094 input=f326b43fb8a4c5ff]*/ { -#define objreduce \ - (_Py_INTERP_CACHED_OBJECT(_PyInterpreterState_GET(), objreduce)) - PyObject *reduce, *res; - - if (objreduce == NULL) { - PyObject *dict = lookup_tp_dict(&PyBaseObject_Type); - objreduce = PyDict_GetItemWithError(dict, &_Py_ID(__reduce__)); - if (objreduce == NULL && PyErr_Occurred()) { - return NULL; - } - } - + PyObject *reduce; if (PyObject_GetOptionalAttr(self, &_Py_ID(__reduce__), &reduce) < 0) { return NULL; } @@ -6661,10 +7490,12 @@ object___reduce_ex___impl(PyObject *self, int protocol) Py_DECREF(reduce); return NULL; } - override = (clsreduce != objreduce); + + PyInterpreterState *interp = _PyInterpreterState_GET(); + override = (clsreduce != _Py_INTERP_CACHED_OBJECT(interp, objreduce)); Py_DECREF(clsreduce); if (override) { - res = _PyObject_CallNoArgs(reduce); + PyObject *res = _PyObject_CallNoArgs(reduce); Py_DECREF(reduce); return res; } @@ -6673,7 +7504,6 @@ object___reduce_ex___impl(PyObject *self, int protocol) } return _common_reduce(self, protocol); -#undef objreduce } static PyObject * @@ -6742,8 +7572,11 @@ object___sizeof___impl(PyObject *self) res = 0; isize = Py_TYPE(self)->tp_itemsize; - if (isize > 0) - res = Py_SIZE(self) * isize; + if (isize > 0) { + /* This assumes that ob_size is valid if tp_itemsize is not 0, + which isn't true for PyLongObject. */ + res = _PyVarObject_CAST(self)->ob_size * isize; + } res += Py_TYPE(self)->tp_basicsize; return PyLong_FromSsize_t(res); @@ -6807,7 +7640,7 @@ static PyMethodDef object_methods[] = { OBJECT___REDUCE_EX___METHODDEF OBJECT___REDUCE___METHODDEF OBJECT___GETSTATE___METHODDEF - {"__subclasshook__", object_subclasshook, METH_CLASS | METH_VARARGS, + {"__subclasshook__", object_subclasshook, METH_CLASS | METH_O, object_subclasshook_doc}, {"__init_subclass__", object_init_subclass, METH_CLASS | METH_NOARGS, object_init_subclass_doc}, @@ -6823,12 +7656,6 @@ PyDoc_STRVAR(object_doc, "When called, it accepts no arguments and returns a new featureless\n" "instance that has no instance attributes and cannot be given any.\n"); -static Py_hash_t -object_hash(PyObject *obj) -{ - return _Py_HashPointer(obj); -} - PyTypeObject PyBaseObject_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "object", /* tp_name */ @@ -6843,7 +7670,7 @@ PyTypeObject PyBaseObject_Type = { 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ - object_hash, /* tp_hash */ + PyObject_GenericHash, /* tp_hash */ 0, /* tp_call */ object_str, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -6868,7 +7695,7 @@ PyTypeObject PyBaseObject_Type = { object_init, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ object_new, /* tp_new */ - PyObject_Del, /* tp_free */ + PyObject_Free, /* tp_free */ }; @@ -6931,6 +7758,12 @@ type_add_method(PyTypeObject *type, PyMethodDef *meth) return 0; } +int +_PyType_AddMethod(PyTypeObject *type, PyMethodDef *meth) +{ + return type_add_method(type, meth); +} + /* Add the methods from tp_methods to the __dict__ in a type object */ static int @@ -7029,28 +7862,29 @@ inherit_special(PyTypeObject *type, PyTypeObject *base) #undef COPYVAL /* Setup fast subclass flags */ - if (is_subtype_unlocked(base, (PyTypeObject*)PyExc_BaseException)) { + PyObject *mro = lookup_tp_mro(base); + if (is_subtype_with_mro(mro, base, (PyTypeObject*)PyExc_BaseException)) { type->tp_flags |= Py_TPFLAGS_BASE_EXC_SUBCLASS; } - else if (is_subtype_unlocked(base, &PyType_Type)) { + else if (is_subtype_with_mro(mro, base, &PyType_Type)) { type->tp_flags |= Py_TPFLAGS_TYPE_SUBCLASS; } - else if (is_subtype_unlocked(base, &PyLong_Type)) { + else if (is_subtype_with_mro(mro, base, &PyLong_Type)) { type->tp_flags |= Py_TPFLAGS_LONG_SUBCLASS; } - else if (is_subtype_unlocked(base, &PyBytes_Type)) { + else if (is_subtype_with_mro(mro, base, &PyBytes_Type)) { type->tp_flags |= Py_TPFLAGS_BYTES_SUBCLASS; } - else if (is_subtype_unlocked(base, &PyUnicode_Type)) { + else if (is_subtype_with_mro(mro, base, &PyUnicode_Type)) { type->tp_flags |= Py_TPFLAGS_UNICODE_SUBCLASS; } - else if (is_subtype_unlocked(base, &PyTuple_Type)) { + else if (is_subtype_with_mro(mro, base, &PyTuple_Type)) { type->tp_flags |= Py_TPFLAGS_TUPLE_SUBCLASS; } - else if (is_subtype_unlocked(base, &PyList_Type)) { + else if (is_subtype_with_mro(mro, base, &PyList_Type)) { type->tp_flags |= Py_TPFLAGS_LIST_SUBCLASS; } - else if (is_subtype_unlocked(base, &PyDict_Type)) { + else if (is_subtype_with_mro(mro, base, &PyDict_Type)) { type->tp_flags |= Py_TPFLAGS_DICT_SUBCLASS; } @@ -7272,7 +8106,7 @@ inherit_slots(PyTypeObject *type, PyTypeObject *base) return 0; } -static int add_operators(PyTypeObject *); +static int add_operators(PyTypeObject *type); static int add_tp_new_wrapper(PyTypeObject *type); #define COLLECTION_FLAGS (Py_TPFLAGS_SEQUENCE | Py_TPFLAGS_MAPPING) @@ -7355,10 +8189,10 @@ type_ready_set_type(PyTypeObject *type) } static int -type_ready_set_bases(PyTypeObject *type) +type_ready_set_bases(PyTypeObject *type, int initial) { if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - if (!_Py_IsMainInterpreter(_PyInterpreterState_GET())) { + if (!initial) { assert(lookup_tp_bases(type) != NULL); return 0; } @@ -7377,7 +8211,7 @@ type_ready_set_bases(PyTypeObject *type) if (bases == NULL) { return -1; } - set_tp_bases(type, bases); + set_tp_bases(type, bases, 1); } return 0; } @@ -7487,12 +8321,12 @@ type_ready_preheader(PyTypeObject *type) } static int -type_ready_mro(PyTypeObject *type) +type_ready_mro(PyTypeObject *type, int initial) { ASSERT_TYPE_LOCK_HELD(); if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - if (!_Py_IsMainInterpreter(_PyInterpreterState_GET())) { + if (!initial) { assert(lookup_tp_mro(type) != NULL); return 0; } @@ -7500,7 +8334,7 @@ type_ready_mro(PyTypeObject *type) } /* Calculate method resolution order */ - if (mro_internal_unlocked(type, NULL) < 0) { + if (mro_internal_unlocked(type, initial, NULL) < 0) { return -1; } PyObject *mro = lookup_tp_mro(type); @@ -7591,7 +8425,7 @@ type_ready_inherit(PyTypeObject *type) /* Sanity check for tp_free. */ if (_PyType_IS_GC(type) && (type->tp_flags & Py_TPFLAGS_BASETYPE) && - (type->tp_free == NULL || type->tp_free == PyObject_Del)) + (type->tp_free == NULL || type->tp_free == PyObject_Free)) { /* This base class needs to call tp_free, but doesn't have * one, or its tp_free is for non-gc'ed objects. @@ -7655,7 +8489,7 @@ type_ready_add_subclasses(PyTypeObject *type) // Set tp_new and the "__new__" key in the type dictionary. // Use the Py_TPFLAGS_DISALLOW_INSTANTIATION flag. static int -type_ready_set_new(PyTypeObject *type, int rerunbuiltin) +type_ready_set_new(PyTypeObject *type, int initial) { PyTypeObject *base = type->tp_base; /* The condition below could use some explanation. @@ -7677,7 +8511,7 @@ type_ready_set_new(PyTypeObject *type, int rerunbuiltin) if (!(type->tp_flags & Py_TPFLAGS_DISALLOW_INSTANTIATION)) { if (type->tp_new != NULL) { - if (!rerunbuiltin || base == NULL || type->tp_new != base->tp_new) { + if (initial || base == NULL || type->tp_new != base->tp_new) { // If "__new__" key does not exists in the type dictionary, // set it to tp_new_wrapper(). if (add_tp_new_wrapper(type) < 0) { @@ -7712,12 +8546,15 @@ type_ready_managed_dict(PyTypeObject *type) } PyHeapTypeObject* et = (PyHeapTypeObject*)type; if (et->ht_cached_keys == NULL) { - et->ht_cached_keys = _PyDict_NewKeysForClass(); + et->ht_cached_keys = _PyDict_NewKeysForClass(et); if (et->ht_cached_keys == NULL) { PyErr_NoMemory(); return -1; } } + if (type->tp_itemsize == 0) { + type->tp_flags |= Py_TPFLAGS_INLINE_VALUES; + } return 0; } @@ -7756,7 +8593,7 @@ type_ready_post_checks(PyTypeObject *type) static int -type_ready(PyTypeObject *type, int rerunbuiltin) +type_ready(PyTypeObject *type, int initial) { ASSERT_TYPE_LOCK_HELD(); @@ -7786,19 +8623,19 @@ type_ready(PyTypeObject *type, int rerunbuiltin) if (type_ready_set_type(type) < 0) { goto error; } - if (type_ready_set_bases(type) < 0) { + if (type_ready_set_bases(type, initial) < 0) { goto error; } - if (type_ready_mro(type) < 0) { + if (type_ready_mro(type, initial) < 0) { goto error; } - if (type_ready_set_new(type, rerunbuiltin) < 0) { + if (type_ready_set_new(type, initial) < 0) { goto error; } if (type_ready_fill_dict(type) < 0) { goto error; } - if (!rerunbuiltin) { + if (initial) { if (type_ready_inherit(type) < 0) { goto error; } @@ -7812,7 +8649,7 @@ type_ready(PyTypeObject *type, int rerunbuiltin) if (type_ready_add_subclasses(type) < 0) { goto error; } - if (!rerunbuiltin) { + if (initial) { if (type_ready_managed_dict(type) < 0) { goto error; } @@ -7822,7 +8659,7 @@ type_ready(PyTypeObject *type, int rerunbuiltin) } /* All done -- set the ready flag */ - type->tp_flags = type->tp_flags | Py_TPFLAGS_READY; + type->tp_flags |= Py_TPFLAGS_READY; stop_readying(type); assert(_PyType_CheckConsistency(type)); @@ -7845,57 +8682,74 @@ PyType_Ready(PyTypeObject *type) /* Historically, all static types were immutable. See bpo-43908 */ if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { type->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; + /* Static types must be immortal */ + _Py_SetImmortalUntracked((PyObject *)type); } int res; - BEGIN_TYPE_LOCK() + BEGIN_TYPE_LOCK(); if (!(type->tp_flags & Py_TPFLAGS_READY)) { - res = type_ready(type, 0); + res = type_ready(type, 1); } else { res = 0; assert(_PyType_CheckConsistency(type)); } - END_TYPE_LOCK() + END_TYPE_LOCK(); return res; } -int -_PyStaticType_InitBuiltin(PyInterpreterState *interp, PyTypeObject *self) + +static int +init_static_type(PyInterpreterState *interp, PyTypeObject *self, + int isbuiltin, int initial) { assert(_Py_IsImmortal((PyObject *)self)); assert(!(self->tp_flags & Py_TPFLAGS_HEAPTYPE)); assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_DICT)); assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_WEAKREF)); - int ismain = _Py_IsMainInterpreter(interp); if ((self->tp_flags & Py_TPFLAGS_READY) == 0) { - assert(ismain); + assert(initial); self->tp_flags |= _Py_TPFLAGS_STATIC_BUILTIN; self->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; assert(NEXT_GLOBAL_VERSION_TAG <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); - self->tp_version_tag = NEXT_GLOBAL_VERSION_TAG++; - self->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; + if (self->tp_version_tag == 0) { + _PyType_SetVersion(self, NEXT_GLOBAL_VERSION_TAG++); + } } else { - assert(!ismain); + assert(!initial); assert(self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); - assert(self->tp_flags & Py_TPFLAGS_VALID_VERSION_TAG); + assert(self->tp_version_tag != 0); } - static_builtin_state_init(interp, self); + managed_static_type_state_init(interp, self, isbuiltin, initial); int res; BEGIN_TYPE_LOCK(); - res = type_ready(self, !ismain); - END_TYPE_LOCK() + res = type_ready(self, initial); + END_TYPE_LOCK(); if (res < 0) { - static_builtin_state_clear(interp, self); + _PyStaticType_ClearWeakRefs(interp, self); + managed_static_type_state_clear(interp, self, isbuiltin, initial); } return res; } +int +_PyStaticType_InitForExtension(PyInterpreterState *interp, PyTypeObject *self) +{ + return init_static_type(interp, self, 0, ((self->tp_flags & Py_TPFLAGS_READY) == 0)); +} + +int +_PyStaticType_InitBuiltin(PyInterpreterState *interp, PyTypeObject *self) +{ + return init_static_type(interp, self, 1, _Py_IsMainInterpreter(interp)); +} + static int add_subclass(PyTypeObject *base, PyTypeObject *type) @@ -8032,6 +8886,27 @@ check_num_args(PyObject *ob, int n) return 0; } +static Py_ssize_t +check_pow_args(PyObject *ob) +{ + // Returns the argument count on success or `-1` on error. + int min = 1; + int max = 2; + if (!PyTuple_CheckExact(ob)) { + PyErr_SetString(PyExc_SystemError, + "PyArg_UnpackTuple() argument list is not a tuple"); + return -1; + } + Py_ssize_t size = PyTuple_GET_SIZE(ob); + if (size >= min && size <= max) { + return size; + } + PyErr_Format( + PyExc_TypeError, + "expected %d or %d arguments, got %zd", min, max, PyTuple_GET_SIZE(ob)); + return -1; +} + /* Generic wrappers for overloadable 'operators' such as __getitem__ */ /* There's a wrapper *function* for each distinct function typedef used @@ -8113,8 +8988,15 @@ wrap_ternaryfunc(PyObject *self, PyObject *args, void *wrapped) /* Note: This wrapper only works for __pow__() */ - if (!PyArg_UnpackTuple(args, "", 1, 2, &other, &third)) + Py_ssize_t size = check_pow_args(args); + if (size == -1) { return NULL; + } + other = PyTuple_GET_ITEM(args, 0); + if (size == 2) { + third = PyTuple_GET_ITEM(args, 1); + } + return (*func)(self, other, third); } @@ -8125,10 +9007,17 @@ wrap_ternaryfunc_r(PyObject *self, PyObject *args, void *wrapped) PyObject *other; PyObject *third = Py_None; - /* Note: This wrapper only works for __pow__() */ + /* Note: This wrapper only works for __rpow__() */ - if (!PyArg_UnpackTuple(args, "", 1, 2, &other, &third)) + Py_ssize_t size = check_pow_args(args); + if (size == -1) { return NULL; + } + other = PyTuple_GET_ITEM(args, 0); + if (size == 2) { + third = PyTuple_GET_ITEM(args, 1); + } + return (*func)(other, self, third); } @@ -8149,8 +9038,9 @@ wrap_indexargfunc(PyObject *self, PyObject *args, void *wrapped) PyObject* o; Py_ssize_t i; - if (!PyArg_UnpackTuple(args, "", 1, 1, &o)) + if (!check_num_args(args, 1)) return NULL; + o = PyTuple_GET_ITEM(args, 0); i = PyNumber_AsSsize_t(o, PyExc_OverflowError); if (i == -1 && PyErr_Occurred()) return NULL; @@ -8206,7 +9096,7 @@ wrap_sq_setitem(PyObject *self, PyObject *args, void *wrapped) int res; PyObject *arg, *value; - if (!PyArg_UnpackTuple(args, "", 2, 2, &arg, &value)) + if (!PyArg_UnpackTuple(args, "__setitem__", 2, 2, &arg, &value)) return NULL; i = getindex(self, arg); if (i == -1 && PyErr_Occurred()) @@ -8262,7 +9152,7 @@ wrap_objobjargproc(PyObject *self, PyObject *args, void *wrapped) int res; PyObject *key, *value; - if (!PyArg_UnpackTuple(args, "", 2, 2, &key, &value)) + if (!PyArg_UnpackTuple(args, "__setitem__", 2, 2, &key, &value)) return NULL; res = (*func)(self, key, value); if (res == -1 && PyErr_Occurred()) @@ -8348,7 +9238,7 @@ hackcheck(PyObject *self, setattrofunc func, const char *what) int res; BEGIN_TYPE_LOCK(); res = hackcheck_unlocked(self, func, what); - END_TYPE_LOCK() + END_TYPE_LOCK(); return res; } @@ -8359,7 +9249,7 @@ wrap_setattr(PyObject *self, PyObject *args, void *wrapped) int res; PyObject *name, *value; - if (!PyArg_UnpackTuple(args, "", 2, 2, &name, &value)) + if (!PyArg_UnpackTuple(args, "__setattr__", 2, 2, &name, &value)) return NULL; if (!hackcheck(self, func, "__setattr__")) return NULL; @@ -8469,7 +9359,7 @@ wrap_descr_get(PyObject *self, PyObject *args, void *wrapped) PyObject *obj; PyObject *type = NULL; - if (!PyArg_UnpackTuple(args, "", 1, 2, &obj, &type)) + if (!PyArg_UnpackTuple(args, "__get__", 1, 2, &obj, &type)) return NULL; if (obj == Py_None) obj = NULL; @@ -8490,7 +9380,7 @@ wrap_descr_set(PyObject *self, PyObject *args, void *wrapped) PyObject *obj, *value; int ret; - if (!PyArg_UnpackTuple(args, "", 2, 2, &obj, &value)) + if (!PyArg_UnpackTuple(args, "__set__", 2, 2, &obj, &value)) return NULL; ret = (*func)(self, obj, value); if (ret < 0) @@ -8519,20 +9409,20 @@ wrap_buffer(PyObject *self, PyObject *args, void *wrapped) { PyObject *arg = NULL; - if (!PyArg_UnpackTuple(args, "", 1, 1, &arg)) { + if (!PyArg_UnpackTuple(args, "__buffer__", 1, 1, &arg)) { return NULL; } Py_ssize_t flags = PyNumber_AsSsize_t(arg, PyExc_OverflowError); if (flags == -1 && PyErr_Occurred()) { return NULL; } - if (flags > INT_MAX) { + if (flags > INT_MAX || flags < INT_MIN) { PyErr_SetString(PyExc_OverflowError, - "buffer flags too large"); + "buffer flags out of range"); return NULL; } - return _PyMemoryView_FromBufferProc(self, Py_SAFE_DOWNCAST(flags, Py_ssize_t, int), + return _PyMemoryView_FromBufferProc(self, (int)flags, (getbufferproc)wrapped); } @@ -8540,7 +9430,7 @@ static PyObject * wrap_releasebuffer(PyObject *self, PyObject *args, void *wrapped) { PyObject *arg = NULL; - if (!PyArg_UnpackTuple(args, "", 1, 1, &arg)) { + if (!PyArg_UnpackTuple(args, "__release_buffer__", 1, 1, &arg)) { return NULL; } if (!PyMemoryView_Check(arg)) { @@ -9166,33 +10056,33 @@ _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name) /* speed hack: we could use lookup_maybe, but that would resolve the method fully for each attribute lookup for classes with __getattr__, even when the attribute is present. So we use - _PyType_Lookup and create the method only when needed, with + _PyType_LookupRef and create the method only when needed, with call_attribute. */ - getattr = _PyType_Lookup(tp, &_Py_ID(__getattr__)); + getattr = _PyType_LookupRef(tp, &_Py_ID(__getattr__)); if (getattr == NULL) { /* No __getattr__ hook: use a simpler dispatcher */ tp->tp_getattro = _Py_slot_tp_getattro; return _Py_slot_tp_getattro(self, name); } - Py_INCREF(getattr); /* speed hack: we could use lookup_maybe, but that would resolve the method fully for each attribute lookup for classes with __getattr__, even when self has the default __getattribute__ - method. So we use _PyType_Lookup and create the method only when + method. So we use _PyType_LookupRef and create the method only when needed, with call_attribute. */ - getattribute = _PyType_Lookup(tp, &_Py_ID(__getattribute__)); + getattribute = _PyType_LookupRef(tp, &_Py_ID(__getattribute__)); if (getattribute == NULL || (Py_IS_TYPE(getattribute, &PyWrapperDescr_Type) && ((PyWrapperDescrObject *)getattribute)->d_wrapped == (void *)PyObject_GenericGetAttr)) { + Py_XDECREF(getattribute); res = _PyObject_GenericGetAttrWithDict(self, name, NULL, 1); /* if res == NULL with no exception set, then it must be an AttributeError suppressed by us. */ if (res == NULL && !PyErr_Occurred()) { res = call_attribute(self, getattr, name); } - } else { - Py_INCREF(getattribute); + } + else { res = call_attribute(self, getattribute, name); Py_DECREF(getattribute); if (res == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { @@ -9299,7 +10189,7 @@ slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) PyTypeObject *tp = Py_TYPE(self); PyObject *get; - get = _PyType_Lookup(tp, &_Py_ID(__get__)); + get = _PyType_LookupRef(tp, &_Py_ID(__get__)); if (get == NULL) { /* Avoid further slowdowns */ if (tp->tp_descr_get == slot_tp_descr_get) @@ -9311,7 +10201,9 @@ slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) if (type == NULL) type = Py_None; PyObject *stack[3] = {self, obj, type}; - return PyObject_Vectorcall(get, stack, 3, NULL); + PyObject *res = PyObject_Vectorcall(get, stack, 3, NULL); + Py_DECREF(get); + return res; } static int @@ -9782,7 +10674,8 @@ static pytype_slotdef slotdefs[] = { TPSLOT(__getattribute__, tp_getattro, _Py_slot_tp_getattr_hook, wrap_binaryfunc, "__getattribute__($self, name, /)\n--\n\nReturn getattr(self, name)."), - TPSLOT(__getattr__, tp_getattro, _Py_slot_tp_getattr_hook, NULL, ""), + TPSLOT(__getattr__, tp_getattro, _Py_slot_tp_getattr_hook, NULL, + "__getattr__($self, name, /)\n--\n\nImplement getattr(self, name)."), TPSLOT(__setattr__, tp_setattro, slot_tp_setattro, wrap_setattr, "__setattr__($self, name, value, /)\n--\n\nImplement setattr(self, name, value)."), TPSLOT(__delattr__, tp_setattro, slot_tp_setattro, wrap_delattr, @@ -9817,7 +10710,9 @@ static pytype_slotdef slotdefs[] = { TPSLOT(__new__, tp_new, slot_tp_new, NULL, "__new__(type, /, *args, **kwargs)\n--\n\n" "Create and return new object. See help(type) for accurate signature."), - TPSLOT(__del__, tp_finalize, slot_tp_finalize, (wrapperfunc)wrap_del, ""), + TPSLOT(__del__, tp_finalize, slot_tp_finalize, (wrapperfunc)wrap_del, + "__del__($self, /)\n--\n\n" + "Called when the instance is about to be destroyed."), BUFSLOT(__buffer__, bf_getbuffer, slot_bf_getbuffer, wrap_buffer, "__buffer__($self, flags, /)\n--\n\n" @@ -10151,7 +11046,7 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) d = (PyWrapperDescrObject *)descr; if ((specific == NULL || specific == d->d_wrapped) && d->d_base->wrapper == p->wrapper && - is_subtype_unlocked(type, PyDescr_TYPE(d))) + is_subtype_with_mro(lookup_tp_mro(type), type, PyDescr_TYPE(d))) { specific = d->d_wrapped; } @@ -10203,6 +11098,7 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) type->tp_flags &= ~Py_TPFLAGS_HAVE_VECTORCALL; } } + Py_DECREF(descr); } while ((++p)->offset == offset); if (specific && !use_generic) *ptr = specific; @@ -10270,15 +11166,15 @@ fixup_slot_dispatchers(PyTypeObject *type) { // This lock isn't strictly necessary because the type has not been // exposed to anyone else yet, but update_ont_slot calls find_name_in_mro - // where we'd like to assert that the tyep is locked. - BEGIN_TYPE_LOCK() + // where we'd like to assert that the type is locked. + BEGIN_TYPE_LOCK(); assert(!PyErr_Occurred()); for (pytype_slotdef *p = slotdefs; p->name; ) { p = update_one_slot(type, p); } - END_TYPE_LOCK() + END_TYPE_LOCK(); } static void @@ -10298,6 +11194,24 @@ update_all_slots(PyTypeObject* type) } +PyObject * +_PyType_GetSlotWrapperNames(void) +{ + size_t len = Py_ARRAY_LENGTH(slotdefs) - 1; + PyObject *names = PyList_New(len); + if (names == NULL) { + return NULL; + } + assert(slotdefs[len].name == NULL); + for (size_t i = 0; i < len; i++) { + pytype_slotdef *slotdef = &slotdefs[i]; + assert(slotdef->name != NULL); + PyList_SET_ITEM(names, i, Py_NewRef(slotdef->name_strobj)); + } + return names; +} + + /* Call __set_name__ on all attributes (including descriptors) in a newly generated type */ static int @@ -10431,6 +11345,26 @@ recurse_down_subclasses(PyTypeObject *type, PyObject *attr_name, return 0; } +static int +slot_inherited(PyTypeObject *type, pytype_slotdef *slotdef, void **slot) +{ + void **slot_base = slotptr(type->tp_base, slotdef->offset); + if (slot_base == NULL || *slot != *slot_base) { + return 0; + } + + /* Some slots are inherited in pairs. */ + if (slot == (void *)&type->tp_hash) { + return (type->tp_richcompare == type->tp_base->tp_richcompare); + } + else if (slot == (void *)&type->tp_richcompare) { + return (type->tp_hash == type->tp_base->tp_hash); + } + + /* It must be inherited (see type_ready_inherit()). */ + return 1; +} + /* This function is called by PyType_Ready() to populate the type's dictionary with method descriptors for function slots. For each function slot (like tp_repr) that's defined in the type, one or more @@ -10475,6 +11409,13 @@ add_operators(PyTypeObject *type) ptr = slotptr(type, p->offset); if (!ptr || !*ptr) continue; + /* Also ignore when the type slot has been inherited. */ + if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN + && type->tp_base != NULL + && slot_inherited(type, p, ptr)) + { + continue; + } int r = PyDict_Contains(dict, p->name_strobj); if (r > 0) continue; @@ -10503,6 +11444,30 @@ add_operators(PyTypeObject *type) } +int +PyType_Freeze(PyTypeObject *type) +{ + // gh-121654: Check the __mro__ instead of __bases__ + PyObject *mro = type_get_mro(type, NULL); + if (!PyTuple_Check(mro)) { + Py_DECREF(mro); + PyErr_SetString(PyExc_TypeError, "unable to get the type MRO"); + return -1; + } + + int check = check_immutable_bases(type->tp_name, mro, 1); + Py_DECREF(mro); + if (check < 0) { + return -1; + } + + type->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; + PyType_Modified(type); + + return 0; +} + + /* Cooperative 'super' */ typedef struct { @@ -10567,7 +11532,7 @@ _super_lookup_descr(PyTypeObject *su_type, PyTypeObject *su_obj_type, PyObject * another thread can modify it after we end the critical section below */ Py_XINCREF(mro); - END_TYPE_LOCK() + END_TYPE_LOCK(); if (mro == NULL) return NULL; @@ -10783,9 +11748,10 @@ super_descr_get(PyObject *self, PyObject *obj, PyObject *type) } static int -super_init_without_args(_PyInterpreterFrame *cframe, PyCodeObject *co, - PyTypeObject **type_p, PyObject **obj_p) +super_init_without_args(_PyInterpreterFrame *cframe, PyTypeObject **type_p, + PyObject **obj_p) { + PyCodeObject *co = _PyFrame_GetCode(cframe); if (co->co_argcount == 0) { PyErr_SetString(PyExc_RuntimeError, "super(): no arguments"); @@ -10793,50 +11759,59 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyCodeObject *co, } assert(_PyFrame_GetCode(cframe)->co_nlocalsplus > 0); - PyObject *firstarg = _PyFrame_GetLocalsArray(cframe)[0]; + PyObject *firstarg = PyStackRef_AsPyObjectBorrow(_PyFrame_GetLocalsArray(cframe)[0]); + if (firstarg == NULL) { + PyErr_SetString(PyExc_RuntimeError, "super(): arg[0] deleted"); + return -1; + } // The first argument might be a cell. - if (firstarg != NULL && (_PyLocals_GetKind(co->co_localspluskinds, 0) & CO_FAST_CELL)) { - // "firstarg" is a cell here unless (very unlikely) super() - // was called from the C-API before the first MAKE_CELL op. - if (_PyInterpreterFrame_LASTI(cframe) >= 0) { - // MAKE_CELL and COPY_FREE_VARS have no quickened forms, so no need - // to use _PyOpcode_Deopt here: - assert(_PyCode_CODE(co)[0].op.code == MAKE_CELL || - _PyCode_CODE(co)[0].op.code == COPY_FREE_VARS); - assert(PyCell_Check(firstarg)); - firstarg = PyCell_GET(firstarg); + // "firstarg" is a cell here unless (very unlikely) super() + // was called from the C-API before the first MAKE_CELL op. + if ((_PyLocals_GetKind(co->co_localspluskinds, 0) & CO_FAST_CELL) && + (_PyInterpreterFrame_LASTI(cframe) >= 0)) { + // MAKE_CELL and COPY_FREE_VARS have no quickened forms, so no need + // to use _PyOpcode_Deopt here: + assert(_PyCode_CODE(co)[0].op.code == MAKE_CELL || + _PyCode_CODE(co)[0].op.code == COPY_FREE_VARS); + assert(PyCell_Check(firstarg)); + firstarg = PyCell_GetRef((PyCellObject *)firstarg); + if (firstarg == NULL) { + PyErr_SetString(PyExc_RuntimeError, "super(): arg[0] deleted"); + return -1; } } - if (firstarg == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "super(): arg[0] deleted"); - return -1; + else { + Py_INCREF(firstarg); } // Look for __class__ in the free vars. PyTypeObject *type = NULL; - int i = PyCode_GetFirstFree(co); + int i = PyUnstable_Code_GetFirstFree(co); for (; i < co->co_nlocalsplus; i++) { assert((_PyLocals_GetKind(co->co_localspluskinds, i) & CO_FAST_FREE) != 0); PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); assert(PyUnicode_Check(name)); if (_PyUnicode_Equal(name, &_Py_ID(__class__))) { - PyObject *cell = _PyFrame_GetLocalsArray(cframe)[i]; + PyObject *cell = PyStackRef_AsPyObjectBorrow(_PyFrame_GetLocalsArray(cframe)[i]); if (cell == NULL || !PyCell_Check(cell)) { PyErr_SetString(PyExc_RuntimeError, "super(): bad __class__ cell"); + Py_DECREF(firstarg); return -1; } - type = (PyTypeObject *) PyCell_GET(cell); + type = (PyTypeObject *) PyCell_GetRef((PyCellObject *)cell); if (type == NULL) { PyErr_SetString(PyExc_RuntimeError, "super(): empty __class__ cell"); + Py_DECREF(firstarg); return -1; } if (!PyType_Check(type)) { PyErr_Format(PyExc_RuntimeError, "super(): __class__ is not a type (%s)", Py_TYPE(type)->tp_name); + Py_DECREF(type); + Py_DECREF(firstarg); return -1; } break; @@ -10845,6 +11820,7 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyCodeObject *co, if (type == NULL) { PyErr_SetString(PyExc_RuntimeError, "super(): __class__ cell not found"); + Py_DECREF(firstarg); return -1; } @@ -10885,22 +11861,30 @@ super_init_impl(PyObject *self, PyTypeObject *type, PyObject *obj) { "super(): no current frame"); return -1; } - int res = super_init_without_args(frame, _PyFrame_GetCode(frame), &type, &obj); + int res = super_init_without_args(frame, &type, &obj); if (res < 0) { return -1; } } + else { + Py_INCREF(type); + Py_XINCREF(obj); + } - if (obj == Py_None) + if (obj == Py_None) { + Py_DECREF(obj); obj = NULL; + } if (obj != NULL) { obj_type = supercheck(type, obj); - if (obj_type == NULL) + if (obj_type == NULL) { + Py_DECREF(type); + Py_DECREF(obj); return -1; - Py_INCREF(obj); + } } - Py_XSETREF(su->type, (PyTypeObject*)Py_NewRef(type)); + Py_XSETREF(su->type, (PyTypeObject*)type); Py_XSETREF(su->obj, obj); Py_XSETREF(su->obj_type, obj_type); return 0; diff --git a/Objects/typeslots.inc b/Objects/typeslots.inc index 896daa7d8066b7..642160fe0bd8bc 100644 --- a/Objects/typeslots.inc +++ b/Objects/typeslots.inc @@ -80,3 +80,5 @@ {offsetof(PyAsyncMethods, am_anext), offsetof(PyTypeObject, tp_as_async)}, {-1, offsetof(PyTypeObject, tp_finalize)}, {offsetof(PyAsyncMethods, am_send), offsetof(PyTypeObject, tp_as_async)}, +{-1, offsetof(PyTypeObject, tp_vectorcall)}, +{-1, offsetof(PyHeapTypeObject, ht_token)}, diff --git a/Objects/typeslots.py b/Objects/typeslots.py index 8ab05f91be12b0..c7f8a33bb1e74e 100755 --- a/Objects/typeslots.py +++ b/Objects/typeslots.py @@ -13,7 +13,11 @@ def generate_typeslots(out=sys.stdout): continue member = m.group(1) - if member.startswith("tp_"): + if member == "tp_token": + # The heap type structure (ht_*) is an implementation detail; + # the public slot for it has a familiar `tp_` prefix + member = '{-1, offsetof(PyHeapTypeObject, ht_token)}' + elif member.startswith("tp_"): member = f'{{-1, offsetof(PyTypeObject, {member})}}' elif member.startswith("am_"): member = (f'{{offsetof(PyAsyncMethods, {member}),'+ diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 7f80c9c61b8abc..4ed40aa71a595e 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -1,6 +1,6 @@ // TypeVar, TypeVarTuple, and ParamSpec #include "Python.h" -#include "pycore_object.h" // _PyObject_GC_TRACK/UNTRACK +#include "pycore_object.h" // _PyObject_GC_TRACK/UNTRACK, PyAnnotateFormat #include "pycore_typevarobject.h" #include "pycore_unionobject.h" // _Py_union_type_or @@ -23,6 +23,8 @@ typedef struct { PyObject *evaluate_bound; PyObject *constraints; PyObject *evaluate_constraints; + PyObject *default_value; + PyObject *evaluate_default; bool covariant; bool contravariant; bool infer_variance; @@ -31,12 +33,16 @@ typedef struct { typedef struct { PyObject_HEAD PyObject *name; + PyObject *default_value; + PyObject *evaluate_default; } typevartupleobject; typedef struct { PyObject_HEAD PyObject *name; PyObject *bound; + PyObject *default_value; + PyObject *evaluate_default; bool covariant; bool contravariant; bool infer_variance; @@ -53,6 +59,262 @@ typedef struct { #include "clinic/typevarobject.c.h" +/* NoDefault is a marker object to indicate that a parameter has no default. */ + +static PyObject * +NoDefault_repr(PyObject *op) +{ + return PyUnicode_FromString("typing.NoDefault"); +} + +static PyObject * +NoDefault_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("NoDefault"); +} + +static PyMethodDef nodefault_methods[] = { + {"__reduce__", NoDefault_reduce, METH_NOARGS, NULL}, + {NULL, NULL} +}; + +static PyObject * +nodefault_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + if (PyTuple_GET_SIZE(args) || (kwargs && PyDict_GET_SIZE(kwargs))) { + PyErr_SetString(PyExc_TypeError, "NoDefaultType takes no arguments"); + return NULL; + } + return &_Py_NoDefaultStruct; +} + +static void +nodefault_dealloc(PyObject *nodefault) +{ + /* This should never get called, but we also don't want to SEGV if + * we accidentally decref NoDefault out of existence. Instead, + * since NoDefault is an immortal object, re-set the reference count. + */ + _Py_SetImmortal(nodefault); +} + +PyDoc_STRVAR(nodefault_doc, +"NoDefaultType()\n" +"--\n\n" +"The type of the NoDefault singleton."); + +PyTypeObject _PyNoDefault_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "NoDefaultType", + .tp_dealloc = nodefault_dealloc, + .tp_repr = NoDefault_repr, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_doc = nodefault_doc, + .tp_methods = nodefault_methods, + .tp_new = nodefault_new, +}; + +PyObject _Py_NoDefaultStruct = _PyObject_HEAD_INIT(&_PyNoDefault_Type); + +typedef struct { + PyObject_HEAD + PyObject *value; +} constevaluatorobject; + +static void +constevaluator_dealloc(PyObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + constevaluatorobject *ce = (constevaluatorobject *)self; + + _PyObject_GC_UNTRACK(self); + + Py_XDECREF(ce->value); + + Py_TYPE(self)->tp_free(self); + Py_DECREF(tp); +} + +static int +constevaluator_traverse(PyObject *self, visitproc visit, void *arg) +{ + constevaluatorobject *ce = (constevaluatorobject *)self; + Py_VISIT(ce->value); + return 0; +} + +static int +constevaluator_clear(PyObject *self) +{ + Py_CLEAR(((constevaluatorobject *)self)->value); + return 0; +} + +static PyObject * +constevaluator_repr(PyObject *self) +{ + PyObject *value = ((constevaluatorobject *)self)->value; + return PyUnicode_FromFormat("", value); +} + +static PyObject * +constevaluator_call(PyObject *self, PyObject *args, PyObject *kwargs) +{ + if (!_PyArg_NoKeywords("constevaluator.__call__", kwargs)) { + return NULL; + } + int format; + if (!PyArg_ParseTuple(args, "i:constevaluator.__call__", &format)) { + return NULL; + } + PyObject *value = ((constevaluatorobject *)self)->value; + if (format == _Py_ANNOTATE_FORMAT_STRING) { + PyUnicodeWriter *writer = PyUnicodeWriter_Create(5); // cannot be <5 + if (writer == NULL) { + return NULL; + } + if (PyTuple_Check(value)) { + if (PyUnicodeWriter_WriteChar(writer, '(') < 0) { + PyUnicodeWriter_Discard(writer); + return NULL; + } + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(value); i++) { + PyObject *item = PyTuple_GET_ITEM(value, i); + if (i > 0) { + if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + PyUnicodeWriter_Discard(writer); + return NULL; + } + } + if (_Py_typing_type_repr(writer, item) < 0) { + PyUnicodeWriter_Discard(writer); + return NULL; + } + } + if (PyUnicodeWriter_WriteChar(writer, ')') < 0) { + PyUnicodeWriter_Discard(writer); + return NULL; + } + } + else { + if (_Py_typing_type_repr(writer, value) < 0) { + PyUnicodeWriter_Discard(writer); + return NULL; + } + } + return PyUnicodeWriter_Finish(writer); + } + return Py_NewRef(value); +} + +static PyObject * +constevaluator_alloc(PyObject *value) +{ + PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.constevaluator_type; + assert(tp != NULL); + constevaluatorobject *ce = PyObject_GC_New(constevaluatorobject, tp); + if (ce == NULL) { + return NULL; + } + ce->value = Py_NewRef(value); + _PyObject_GC_TRACK(ce); + return (PyObject *)ce; + +} + +PyDoc_STRVAR(constevaluator_doc, +"_ConstEvaluator()\n" +"--\n\n" +"Internal type for implementing evaluation functions."); + +static PyType_Slot constevaluator_slots[] = { + {Py_tp_doc, (void *)constevaluator_doc}, + {Py_tp_dealloc, constevaluator_dealloc}, + {Py_tp_traverse, constevaluator_traverse}, + {Py_tp_clear, constevaluator_clear}, + {Py_tp_repr, constevaluator_repr}, + {Py_tp_call, constevaluator_call}, + {Py_tp_alloc, PyType_GenericAlloc}, + {Py_tp_free, PyObject_GC_Del}, + {0, NULL}, +}; + +PyType_Spec constevaluator_spec = { + .name = "_typing._ConstEvaluator", + .basicsize = sizeof(constevaluatorobject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .slots = constevaluator_slots, +}; + +int +_Py_typing_type_repr(PyUnicodeWriter *writer, PyObject *p) +{ + PyObject *qualname = NULL; + PyObject *module = NULL; + PyObject *r = NULL; + int rc; + + if (p == Py_Ellipsis) { + // The Ellipsis object + r = PyUnicode_FromString("..."); + goto exit; + } + + if (p == (PyObject *)&_PyNone_Type) { + return PyUnicodeWriter_WriteUTF8(writer, "None", 4); + } + + if ((rc = PyObject_HasAttrWithError(p, &_Py_ID(__origin__))) > 0 && + (rc = PyObject_HasAttrWithError(p, &_Py_ID(__args__))) > 0) + { + // It looks like a GenericAlias + goto use_repr; + } + if (rc < 0) { + goto exit; + } + + if (PyObject_GetOptionalAttr(p, &_Py_ID(__qualname__), &qualname) < 0) { + goto exit; + } + if (qualname == NULL) { + goto use_repr; + } + if (PyObject_GetOptionalAttr(p, &_Py_ID(__module__), &module) < 0) { + goto exit; + } + if (module == NULL || module == Py_None) { + goto use_repr; + } + + // Looks like a class + if (PyUnicode_Check(module) && + _PyUnicode_EqualToASCIIString(module, "builtins")) + { + // builtins don't need a module name + r = PyObject_Str(qualname); + goto exit; + } + else { + r = PyUnicode_FromFormat("%S.%S", module, qualname); + goto exit; + } + +use_repr: + r = PyObject_Repr(p); +exit: + Py_XDECREF(qualname); + Py_XDECREF(module); + if (r == NULL) { + return -1; + } + rc = PyUnicodeWriter_WriteStr(writer, r); + Py_DECREF(r); + return rc; +} + + static PyObject * call_typing_func_object(const char *name, PyObject **args, size_t nargs) { @@ -111,10 +373,10 @@ caller(void) if (f == NULL) { Py_RETURN_NONE; } - if (f == NULL || f->f_funcobj == NULL) { + if (f == NULL || PyStackRef_IsNull(f->f_funcobj)) { Py_RETURN_NONE; } - PyObject *r = PyFunction_GetModule(f->f_funcobj); + PyObject *r = PyFunction_GetModule(PyStackRef_AsPyObjectBorrow(f->f_funcobj)); if (!r) { PyErr_Clear(); Py_RETURN_NONE; @@ -200,6 +462,8 @@ typevar_dealloc(PyObject *self) Py_XDECREF(tv->evaluate_bound); Py_XDECREF(tv->constraints); Py_XDECREF(tv->evaluate_constraints); + Py_XDECREF(tv->default_value); + Py_XDECREF(tv->evaluate_default); PyObject_ClearManagedDict(self); PyObject_ClearWeakRefs(self); @@ -216,6 +480,8 @@ typevar_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(tv->evaluate_bound); Py_VISIT(tv->constraints); Py_VISIT(tv->evaluate_constraints); + Py_VISIT(tv->default_value); + Py_VISIT(tv->evaluate_default); PyObject_VisitManagedDict(self, visit, arg); return 0; } @@ -227,6 +493,8 @@ typevar_clear(typevarobject *self) Py_CLEAR(self->evaluate_bound); Py_CLEAR(self->constraints); Py_CLEAR(self->evaluate_constraints); + Py_CLEAR(self->default_value); + Py_CLEAR(self->evaluate_default); PyObject_ClearManagedDict((PyObject *)self); return 0; } @@ -266,6 +534,20 @@ typevar_bound(typevarobject *self, void *Py_UNUSED(ignored)) return bound; } +static PyObject * +typevar_default(typevarobject *self, void *unused) +{ + if (self->default_value != NULL) { + return Py_NewRef(self->default_value); + } + if (self->evaluate_default == NULL) { + return &_Py_NoDefaultStruct; + } + PyObject *default_value = PyObject_CallNoArgs(self->evaluate_default); + self->default_value = Py_XNewRef(default_value); + return default_value; +} + static PyObject * typevar_constraints(typevarobject *self, void *Py_UNUSED(ignored)) { @@ -280,15 +562,56 @@ typevar_constraints(typevarobject *self, void *Py_UNUSED(ignored)) return constraints; } +static PyObject * +typevar_evaluate_bound(typevarobject *self, void *Py_UNUSED(ignored)) +{ + if (self->evaluate_bound != NULL) { + return Py_NewRef(self->evaluate_bound); + } + if (self->bound != NULL) { + return constevaluator_alloc(self->bound); + } + Py_RETURN_NONE; +} + +static PyObject * +typevar_evaluate_constraints(typevarobject *self, void *Py_UNUSED(ignored)) +{ + if (self->evaluate_constraints != NULL) { + return Py_NewRef(self->evaluate_constraints); + } + if (self->constraints != NULL) { + return constevaluator_alloc(self->constraints); + } + Py_RETURN_NONE; +} + +static PyObject * +typevar_evaluate_default(typevarobject *self, void *Py_UNUSED(ignored)) +{ + if (self->evaluate_default != NULL) { + return Py_NewRef(self->evaluate_default); + } + if (self->default_value != NULL) { + return constevaluator_alloc(self->default_value); + } + Py_RETURN_NONE; +} + static PyGetSetDef typevar_getset[] = { {"__bound__", (getter)typevar_bound, NULL, NULL, NULL}, {"__constraints__", (getter)typevar_constraints, NULL, NULL, NULL}, + {"__default__", (getter)typevar_default, NULL, NULL, NULL}, + {"evaluate_bound", (getter)typevar_evaluate_bound, NULL, NULL, NULL}, + {"evaluate_constraints", (getter)typevar_evaluate_constraints, NULL, NULL, NULL}, + {"evaluate_default", (getter)typevar_evaluate_default, NULL, NULL, NULL}, {0} }; static typevarobject * typevar_alloc(PyObject *name, PyObject *bound, PyObject *evaluate_bound, PyObject *constraints, PyObject *evaluate_constraints, + PyObject *default_value, bool covariant, bool contravariant, bool infer_variance, PyObject *module) { @@ -305,6 +628,8 @@ typevar_alloc(PyObject *name, PyObject *bound, PyObject *evaluate_bound, tv->evaluate_bound = Py_XNewRef(evaluate_bound); tv->constraints = Py_XNewRef(constraints); tv->evaluate_constraints = Py_XNewRef(evaluate_constraints); + tv->default_value = Py_XNewRef(default_value); + tv->evaluate_default = NULL; tv->covariant = covariant; tv->contravariant = contravariant; @@ -326,8 +651,9 @@ typevar_alloc(PyObject *name, PyObject *bound, PyObject *evaluate_bound, typevar.__new__ as typevar_new name: object(subclass_of="&PyUnicode_Type") - *constraints: object + *constraints: tuple bound: object = None + default as default_value: object(c_default="&_Py_NoDefaultStruct") = typing.NoDefault covariant: bool = False contravariant: bool = False infer_variance: bool = False @@ -337,9 +663,9 @@ Create a TypeVar. static PyObject * typevar_new_impl(PyTypeObject *type, PyObject *name, PyObject *constraints, - PyObject *bound, int covariant, int contravariant, - int infer_variance) -/*[clinic end generated code: output=1d200450ee99226d input=41ae33a916bfe76f]*/ + PyObject *bound, PyObject *default_value, int covariant, + int contravariant, int infer_variance) +/*[clinic end generated code: output=d2b248ff074eaab6 input=1b5b62e40c92c167]*/ { if (covariant && contravariant) { PyErr_SetString(PyExc_ValueError, @@ -386,6 +712,7 @@ typevar_new_impl(PyTypeObject *type, PyObject *name, PyObject *constraints, PyObject *tv = (PyObject *)typevar_alloc(name, bound, NULL, constraints, NULL, + default_value, covariant, contravariant, infer_variance, module); Py_XDECREF(bound); @@ -410,6 +737,66 @@ typevar_typing_subst(typevarobject *self, PyObject *arg) return result; } +/*[clinic input] +typevar.__typing_prepare_subst__ as typevar_typing_prepare_subst + + alias: object + args: object + / + +[clinic start generated code]*/ + +static PyObject * +typevar_typing_prepare_subst_impl(typevarobject *self, PyObject *alias, + PyObject *args) +/*[clinic end generated code: output=82c3f4691e0ded22 input=201a750415d14ffb]*/ +{ + PyObject *params = PyObject_GetAttrString(alias, "__parameters__"); + if (params == NULL) { + return NULL; + } + Py_ssize_t i = PySequence_Index(params, (PyObject *)self); + if (i == -1) { + Py_DECREF(params); + return NULL; + } + Py_ssize_t args_len = PySequence_Length(args); + if (args_len == -1) { + Py_DECREF(params); + return NULL; + } + if (i < args_len) { + // We already have a value for our TypeVar + Py_DECREF(params); + return Py_NewRef(args); + } + else if (i == args_len) { + // If the TypeVar has a default, use it. + PyObject *dflt = typevar_default(self, NULL); + if (dflt == NULL) { + Py_DECREF(params); + return NULL; + } + if (dflt != &_Py_NoDefaultStruct) { + PyObject *new_args = PyTuple_Pack(1, dflt); + Py_DECREF(dflt); + if (new_args == NULL) { + Py_DECREF(params); + return NULL; + } + PyObject *result = PySequence_Concat(args, new_args); + Py_DECREF(params); + Py_DECREF(new_args); + return result; + } + } + Py_DECREF(params); + PyErr_Format(PyExc_TypeError, + "Too few arguments for %S; actual %d, expected at least %d", + alias, args_len, i + 1); + return NULL; +} + /*[clinic input] typevar.__reduce__ as typevar_reduce @@ -422,6 +809,23 @@ typevar_reduce_impl(typevarobject *self) return Py_NewRef(self->name); } + +/*[clinic input] +typevar.has_default as typevar_has_default + +[clinic start generated code]*/ + +static PyObject * +typevar_has_default_impl(typevarobject *self) +/*[clinic end generated code: output=76bf0b8dc98b97dd input=31024aa030761cf6]*/ +{ + if (self->evaluate_default != NULL || + (self->default_value != &_Py_NoDefaultStruct && self->default_value != NULL)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyObject * typevar_mro_entries(PyObject *self, PyObject *args) { @@ -432,7 +836,9 @@ typevar_mro_entries(PyObject *self, PyObject *args) static PyMethodDef typevar_methods[] = { TYPEVAR_TYPING_SUBST_METHODDEF + TYPEVAR_TYPING_PREPARE_SUBST_METHODDEF TYPEVAR_REDUCE_METHODDEF + TYPEVAR_HAS_DEFAULT_METHODDEF {"__mro_entries__", typevar_mro_entries, METH_O}, {0} }; @@ -457,12 +863,18 @@ variables::\n\ class StrOrBytesSequence[A: (str, bytes)]:\n\ ...\n\ \n\ +Type variables can also have defaults:\n\ +\n\ + class IntDefault[T = int]:\n\ + ...\n\ +\n\ However, if desired, reusable type variables can also be constructed\n\ manually, like so::\n\ \n\ T = TypeVar('T') # Can be anything\n\ S = TypeVar('S', bound=str) # Can be any subtype of str\n\ A = TypeVar('A', str, bytes) # Must be exactly str or bytes\n\ + D = TypeVar('D', default=int) # Defaults to int\n\ \n\ Type variables exist primarily for the benefit of static type\n\ checkers. They serve as the parameters for generic types as well\n\ @@ -739,6 +1151,8 @@ paramspec_dealloc(PyObject *self) Py_DECREF(ps->name); Py_XDECREF(ps->bound); + Py_XDECREF(ps->default_value); + Py_XDECREF(ps->evaluate_default); PyObject_ClearManagedDict(self); PyObject_ClearWeakRefs(self); @@ -752,6 +1166,8 @@ paramspec_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(Py_TYPE(self)); paramspecobject *ps = (paramspecobject *)self; Py_VISIT(ps->bound); + Py_VISIT(ps->default_value); + Py_VISIT(ps->evaluate_default); PyObject_VisitManagedDict(self, visit, arg); return 0; } @@ -760,6 +1176,8 @@ static int paramspec_clear(paramspecobject *self) { Py_CLEAR(self->bound); + Py_CLEAR(self->default_value); + Py_CLEAR(self->evaluate_default); PyObject_ClearManagedDict((PyObject *)self); return 0; } @@ -800,14 +1218,42 @@ paramspec_kwargs(PyObject *self, void *unused) return (PyObject *)paramspecattr_new(tp, self); } +static PyObject * +paramspec_default(paramspecobject *self, void *unused) +{ + if (self->default_value != NULL) { + return Py_NewRef(self->default_value); + } + if (self->evaluate_default == NULL) { + return &_Py_NoDefaultStruct; + } + PyObject *default_value = PyObject_CallNoArgs(self->evaluate_default); + self->default_value = Py_XNewRef(default_value); + return default_value; +} + +static PyObject * +paramspec_evaluate_default(paramspecobject *self, void *unused) +{ + if (self->evaluate_default != NULL) { + return Py_NewRef(self->evaluate_default); + } + if (self->default_value != NULL) { + return constevaluator_alloc(self->default_value); + } + Py_RETURN_NONE; +} + static PyGetSetDef paramspec_getset[] = { {"args", (getter)paramspec_args, NULL, PyDoc_STR("Represents positional arguments."), NULL}, {"kwargs", (getter)paramspec_kwargs, NULL, PyDoc_STR("Represents keyword arguments."), NULL}, + {"__default__", (getter)paramspec_default, NULL, "The default value for this ParamSpec.", NULL}, + {"evaluate_default", (getter)paramspec_evaluate_default, NULL, NULL, NULL}, {0}, }; static paramspecobject * -paramspec_alloc(PyObject *name, PyObject *bound, bool covariant, +paramspec_alloc(PyObject *name, PyObject *bound, PyObject *default_value, bool covariant, bool contravariant, bool infer_variance, PyObject *module) { PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.paramspec_type; @@ -820,6 +1266,8 @@ paramspec_alloc(PyObject *name, PyObject *bound, bool covariant, ps->covariant = covariant; ps->contravariant = contravariant; ps->infer_variance = infer_variance; + ps->default_value = Py_XNewRef(default_value); + ps->evaluate_default = NULL; _PyObject_GC_TRACK(ps); if (module != NULL) { if (PyObject_SetAttrString((PyObject *)ps, "__module__", module) < 0) { @@ -837,6 +1285,7 @@ paramspec.__new__ as paramspec_new name: object(subclass_of="&PyUnicode_Type") * bound: object = None + default as default_value: object(c_default="&_Py_NoDefaultStruct") = typing.NoDefault covariant: bool = False contravariant: bool = False infer_variance: bool = False @@ -846,8 +1295,9 @@ Create a ParamSpec object. static PyObject * paramspec_new_impl(PyTypeObject *type, PyObject *name, PyObject *bound, - int covariant, int contravariant, int infer_variance) -/*[clinic end generated code: output=fd2daab79cba62da input=57c49c581979b952]*/ + PyObject *default_value, int covariant, int contravariant, + int infer_variance) +/*[clinic end generated code: output=47ca9d63fa5a094d input=495e1565bc067ab9]*/ { if (covariant && contravariant) { PyErr_SetString(PyExc_ValueError, "Bivariant types are not supported."); @@ -869,7 +1319,7 @@ paramspec_new_impl(PyTypeObject *type, PyObject *name, PyObject *bound, return NULL; } PyObject *ps = (PyObject *)paramspec_alloc( - name, bound, covariant, contravariant, infer_variance, module); + name, bound, default_value, covariant, contravariant, infer_variance, module); Py_XDECREF(bound); Py_DECREF(module); return ps; @@ -925,6 +1375,22 @@ paramspec_reduce_impl(paramspecobject *self) return Py_NewRef(self->name); } +/*[clinic input] +paramspec.has_default as paramspec_has_default + +[clinic start generated code]*/ + +static PyObject * +paramspec_has_default_impl(paramspecobject *self) +/*[clinic end generated code: output=daaae7467a6a4368 input=2112e97eeb76cd59]*/ +{ + if (self->evaluate_default != NULL || + (self->default_value != &_Py_NoDefaultStruct && self->default_value != NULL)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyObject * paramspec_mro_entries(PyObject *self, PyObject *args) { @@ -936,6 +1402,7 @@ paramspec_mro_entries(PyObject *self, PyObject *args) static PyMethodDef paramspec_methods[] = { PARAMSPEC_TYPING_SUBST_METHODDEF PARAMSPEC_TYPING_PREPARE_SUBST_METHODDEF + PARAMSPEC_HAS_DEFAULT_METHODDEF PARAMSPEC_REDUCE_METHODDEF {"__mro_entries__", paramspec_mro_entries, METH_O}, {0} @@ -950,10 +1417,17 @@ where the use of '**' creates a parameter specification::\n\ \n\ type IntFunc[**P] = Callable[P, int]\n\ \n\ +The following syntax creates a parameter specification that defaults\n\ +to a callable accepting two positional-only arguments of types int\n\ +and str:\n\ +\n\ + type IntFuncDefault[**P = (int, str)] = Callable[P, int]\n\ +\n\ For compatibility with Python 3.11 and earlier, ParamSpec objects\n\ can also be created as follows::\n\ \n\ P = ParamSpec('P')\n\ + DefaultP = ParamSpec('DefaultP', default=(int, str))\n\ \n\ Parameter specification variables exist primarily for the benefit of\n\ static type checkers. They are used to forward the parameter types of\n\ @@ -1021,6 +1495,8 @@ typevartuple_dealloc(PyObject *self) typevartupleobject *tvt = (typevartupleobject *)self; Py_DECREF(tvt->name); + Py_XDECREF(tvt->default_value); + Py_XDECREF(tvt->evaluate_default); PyObject_ClearManagedDict(self); PyObject_ClearWeakRefs(self); @@ -1060,7 +1536,7 @@ static PyMemberDef typevartuple_members[] = { }; static typevartupleobject * -typevartuple_alloc(PyObject *name, PyObject *module) +typevartuple_alloc(PyObject *name, PyObject *module, PyObject *default_value) { PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.typevartuple_type; typevartupleobject *tvt = PyObject_GC_New(typevartupleobject, tp); @@ -1068,6 +1544,8 @@ typevartuple_alloc(PyObject *name, PyObject *module) return NULL; } tvt->name = Py_NewRef(name); + tvt->default_value = Py_XNewRef(default_value); + tvt->evaluate_default = NULL; _PyObject_GC_TRACK(tvt); if (module != NULL) { if (PyObject_SetAttrString((PyObject *)tvt, "__module__", module) < 0) { @@ -1083,19 +1561,22 @@ typevartuple_alloc(PyObject *name, PyObject *module) typevartuple.__new__ name: object(subclass_of="&PyUnicode_Type") + * + default as default_value: object(c_default="&_Py_NoDefaultStruct") = typing.NoDefault Create a new TypeVarTuple with the given name. [clinic start generated code]*/ static PyObject * -typevartuple_impl(PyTypeObject *type, PyObject *name) -/*[clinic end generated code: output=09d417a28f976202 input=00d28abcf1fc96bb]*/ +typevartuple_impl(PyTypeObject *type, PyObject *name, + PyObject *default_value) +/*[clinic end generated code: output=9d6b76dfe95aae51 input=e149739929a866d0]*/ { PyObject *module = caller(); if (module == NULL) { return NULL; } - PyObject *result = (PyObject *)typevartuple_alloc(name, module); + PyObject *result = (PyObject *)typevartuple_alloc(name, module, default_value); Py_DECREF(module); return result; } @@ -1148,6 +1629,23 @@ typevartuple_reduce_impl(typevartupleobject *self) return Py_NewRef(self->name); } + +/*[clinic input] +typevartuple.has_default as typevartuple_has_default + +[clinic start generated code]*/ + +static PyObject * +typevartuple_has_default_impl(typevartupleobject *self) +/*[clinic end generated code: output=4895f602f56a5e29 input=9ef3250ddb2c1851]*/ +{ + if (self->evaluate_default != NULL || + (self->default_value != &_Py_NoDefaultStruct && self->default_value != NULL)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyObject * typevartuple_mro_entries(PyObject *self, PyObject *args) { @@ -1160,6 +1658,8 @@ static int typevartuple_traverse(PyObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); + Py_VISIT(((typevartupleobject *)self)->default_value); + Py_VISIT(((typevartupleobject *)self)->evaluate_default); PyObject_VisitManagedDict(self, visit, arg); return 0; } @@ -1167,14 +1667,49 @@ typevartuple_traverse(PyObject *self, visitproc visit, void *arg) static int typevartuple_clear(PyObject *self) { + Py_CLEAR(((typevartupleobject *)self)->default_value); + Py_CLEAR(((typevartupleobject *)self)->evaluate_default); PyObject_ClearManagedDict(self); return 0; } +static PyObject * +typevartuple_default(typevartupleobject *self, void *unused) +{ + if (self->default_value != NULL) { + return Py_NewRef(self->default_value); + } + if (self->evaluate_default == NULL) { + return &_Py_NoDefaultStruct; + } + PyObject *default_value = PyObject_CallNoArgs(self->evaluate_default); + self->default_value = Py_XNewRef(default_value); + return default_value; +} + +static PyObject * +typevartuple_evaluate_default(typevartupleobject *self, void *unused) +{ + if (self->evaluate_default != NULL) { + return Py_NewRef(self->evaluate_default); + } + if (self->default_value != NULL) { + return constevaluator_alloc(self->default_value); + } + Py_RETURN_NONE; +} + +static PyGetSetDef typevartuple_getset[] = { + {"__default__", (getter)typevartuple_default, NULL, "The default value for this TypeVarTuple.", NULL}, + {"evaluate_default", (getter)typevartuple_evaluate_default, NULL, NULL, NULL}, + {0}, +}; + static PyMethodDef typevartuple_methods[] = { TYPEVARTUPLE_TYPING_SUBST_METHODDEF TYPEVARTUPLE_TYPING_PREPARE_SUBST_METHODDEF TYPEVARTUPLE_REDUCE_METHODDEF + TYPEVARTUPLE_HAS_DEFAULT_METHODDEF {"__mro_entries__", typevartuple_mro_entries, METH_O}, {0} }; @@ -1190,10 +1725,15 @@ where a single '*' indicates a type variable tuple::\n\ def move_first_element_to_last[T, *Ts](tup: tuple[T, *Ts]) -> tuple[*Ts, T]:\n\ return (*tup[1:], tup[0])\n\ \n\ +Type variables tuples can have default values:\n\ +\n\ + type AliasWithDefault[*Ts = (str, int)] = tuple[*Ts]\n\ +\n\ For compatibility with Python 3.11 and earlier, TypeVarTuple objects\n\ can also be created as follows::\n\ \n\ Ts = TypeVarTuple('Ts') # Can be given any name\n\ + DefaultTs = TypeVarTuple('Ts', default=(str, int))\n\ \n\ Just as a TypeVar (type variable) is a placeholder for a single type,\n\ a TypeVarTuple is a placeholder for an *arbitrary* number of types. For\n\ @@ -1218,6 +1758,7 @@ PyType_Slot typevartuple_slots[] = { {Py_tp_doc, (void *)typevartuple_doc}, {Py_tp_members, typevartuple_members}, {Py_tp_methods, typevartuple_methods}, + {Py_tp_getset, typevartuple_getset}, {Py_tp_new, typevartuple}, {Py_tp_iter, typevartuple_iter}, {Py_tp_repr, typevartuple_repr}, @@ -1241,21 +1782,39 @@ PyObject * _Py_make_typevar(PyObject *name, PyObject *evaluate_bound, PyObject *evaluate_constraints) { return (PyObject *)typevar_alloc(name, NULL, evaluate_bound, NULL, evaluate_constraints, - false, false, true, NULL); + NULL, false, false, true, NULL); } PyObject * _Py_make_paramspec(PyThreadState *Py_UNUSED(ignored), PyObject *v) { assert(PyUnicode_Check(v)); - return (PyObject *)paramspec_alloc(v, NULL, false, false, true, NULL); + return (PyObject *)paramspec_alloc(v, NULL, NULL, false, false, true, NULL); } PyObject * _Py_make_typevartuple(PyThreadState *Py_UNUSED(ignored), PyObject *v) { assert(PyUnicode_Check(v)); - return (PyObject *)typevartuple_alloc(v, NULL); + return (PyObject *)typevartuple_alloc(v, NULL, NULL); +} + +static PyObject * +get_type_param_default(PyThreadState *ts, PyObject *typeparam) { + // Does not modify refcount of existing objects. + if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevar_type)) { + return typevar_default((typevarobject *)typeparam, NULL); + } + else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.paramspec_type)) { + return paramspec_default((paramspecobject *)typeparam, NULL); + } + else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevartuple_type)) { + return typevartuple_default((typevartupleobject *)typeparam, NULL); + } + else { + PyErr_Format(PyExc_TypeError, "Expected a type param, got %R", typeparam); + return NULL; + } } static void @@ -1306,6 +1865,17 @@ typealias_value(PyObject *self, void *unused) return typealias_get_value(ta); } +static PyObject * +typealias_evaluate_value(PyObject *self, void *unused) +{ + typealiasobject *ta = (typealiasobject *)self; + if (ta->compute_value != NULL) { + return Py_NewRef(ta->compute_value); + } + assert(ta->value != NULL); + return constevaluator_alloc(ta->value); +} + static PyObject * typealias_parameters(PyObject *self, void *unused) { @@ -1349,10 +1919,70 @@ static PyGetSetDef typealias_getset[] = { {"__parameters__", typealias_parameters, (setter)NULL, NULL, NULL}, {"__type_params__", typealias_type_params, (setter)NULL, NULL, NULL}, {"__value__", typealias_value, (setter)NULL, NULL, NULL}, + {"evaluate_value", typealias_evaluate_value, (setter)NULL, NULL, NULL}, {"__module__", typealias_module, (setter)NULL, NULL, NULL}, {0} }; +static PyObject * +typealias_check_type_params(PyObject *type_params, int *err) { + // Can return type_params or NULL without exception set. + // Does not change the reference count of type_params, + // sets `*err` to 1 when error happens and sets an exception, + // otherwise `*err` is set to 0. + *err = 0; + if (type_params == NULL) { + return NULL; + } + + assert(PyTuple_Check(type_params)); + Py_ssize_t length = PyTuple_GET_SIZE(type_params); + if (!length) { // 0-length tuples are the same as `NULL`. + return NULL; + } + + PyThreadState *ts = _PyThreadState_GET(); + int default_seen = 0; + for (Py_ssize_t index = 0; index < length; index++) { + PyObject *type_param = PyTuple_GET_ITEM(type_params, index); + PyObject *dflt = get_type_param_default(ts, type_param); + if (dflt == NULL) { + *err = 1; + return NULL; + } + if (dflt == &_Py_NoDefaultStruct) { + if (default_seen) { + *err = 1; + PyErr_Format(PyExc_TypeError, + "non-default type parameter '%R' " + "follows default type parameter", + type_param); + return NULL; + } + } else { + default_seen = 1; + Py_DECREF(dflt); + } + } + + return type_params; +} + +static PyObject * +typelias_convert_type_params(PyObject *type_params) +{ + if ( + type_params == NULL + || Py_IsNone(type_params) + || (PyTuple_Check(type_params) && PyTuple_GET_SIZE(type_params) == 0) + ) { + return NULL; + } + else { + return type_params; + } +} + static typealiasobject * typealias_alloc(PyObject *name, PyObject *type_params, PyObject *compute_value, PyObject *value, PyObject *module) @@ -1362,7 +1992,7 @@ typealias_alloc(PyObject *name, PyObject *type_params, PyObject *compute_value, return NULL; } ta->name = Py_NewRef(name); - ta->type_params = Py_IsNone(type_params) ? NULL : Py_XNewRef(type_params); + ta->type_params = Py_XNewRef(type_params); ta->compute_value = Py_XNewRef(compute_value); ta->value = Py_XNewRef(value); ta->module = Py_XNewRef(module); @@ -1440,11 +2070,18 @@ typealias_new_impl(PyTypeObject *type, PyObject *name, PyObject *value, PyErr_SetString(PyExc_TypeError, "type_params must be a tuple"); return NULL; } + + int err = 0; + PyObject *checked_params = typealias_check_type_params(type_params, &err); + if (err) { + return NULL; + } + PyObject *module = caller(); if (module == NULL) { return NULL; } - PyObject *ta = (PyObject *)typealias_alloc(name, type_params, NULL, value, + PyObject *ta = (PyObject *)typealias_alloc(name, checked_params, NULL, value, module); Py_DECREF(module); return ta; @@ -1510,7 +2147,7 @@ _Py_make_typealias(PyThreadState* unused, PyObject *args) assert(PyTuple_GET_SIZE(args) == 3); PyObject *name = PyTuple_GET_ITEM(args, 0); assert(PyUnicode_Check(name)); - PyObject *type_params = PyTuple_GET_ITEM(args, 1); + PyObject *type_params = typelias_convert_type_params(PyTuple_GET_ITEM(args, 1)); PyObject *compute_value = PyTuple_GET_ITEM(args, 2); assert(PyFunction_Check(compute_value)); return (PyObject *)typealias_alloc(name, type_params, compute_value, NULL, NULL); @@ -1674,6 +2311,7 @@ int _Py_initialize_generic(PyInterpreterState *interp) MAKE_TYPE(paramspec); MAKE_TYPE(paramspecargs); MAKE_TYPE(paramspeckwargs); + MAKE_TYPE(constevaluator); #undef MAKE_TYPE return 0; } @@ -1686,4 +2324,26 @@ void _Py_clear_generic_types(PyInterpreterState *interp) Py_CLEAR(interp->cached_objects.paramspec_type); Py_CLEAR(interp->cached_objects.paramspecargs_type); Py_CLEAR(interp->cached_objects.paramspeckwargs_type); + Py_CLEAR(interp->cached_objects.constevaluator_type); +} + +PyObject * +_Py_set_typeparam_default(PyThreadState *ts, PyObject *typeparam, PyObject *evaluate_default) +{ + if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevar_type)) { + Py_XSETREF(((typevarobject *)typeparam)->evaluate_default, Py_NewRef(evaluate_default)); + return Py_NewRef(typeparam); + } + else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.paramspec_type)) { + Py_XSETREF(((paramspecobject *)typeparam)->evaluate_default, Py_NewRef(evaluate_default)); + return Py_NewRef(typeparam); + } + else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevartuple_type)) { + Py_XSETREF(((typevartupleobject *)typeparam)->evaluate_default, Py_NewRef(evaluate_default)); + return Py_NewRef(typeparam); + } + else { + PyErr_Format(PyExc_TypeError, "Expected a type param, got %R", typeparam); + return NULL; + } } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 0a569a950e88e2..d9952f764bb178 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -44,7 +44,9 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_bytesobject.h" // _PyBytes_Repeat() #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_codecs.h" // _PyCodec_Lookup() +#include "pycore_critical_section.h" // Py_*_CRITICAL_SECTION_SEQUENCE_FAST #include "pycore_format.h" // F_LJUST +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // PyInterpreterState.fs_codec #include "pycore_long.h" // _PyLong_FormatWriter() @@ -110,20 +112,42 @@ NOTE: In the interpreter's initialization phase, some globals are currently # define _PyUnicode_CHECK(op) PyUnicode_Check(op) #endif -#define _PyUnicode_UTF8(op) \ - (_PyCompactUnicodeObject_CAST(op)->utf8) -#define PyUnicode_UTF8(op) \ - (assert(_PyUnicode_CHECK(op)), \ - PyUnicode_IS_COMPACT_ASCII(op) ? \ - ((char*)(_PyASCIIObject_CAST(op) + 1)) : \ - _PyUnicode_UTF8(op)) -#define _PyUnicode_UTF8_LENGTH(op) \ - (_PyCompactUnicodeObject_CAST(op)->utf8_length) -#define PyUnicode_UTF8_LENGTH(op) \ - (assert(_PyUnicode_CHECK(op)), \ - PyUnicode_IS_COMPACT_ASCII(op) ? \ - _PyASCIIObject_CAST(op)->length : \ - _PyUnicode_UTF8_LENGTH(op)) +static inline char* _PyUnicode_UTF8(PyObject *op) +{ + return FT_ATOMIC_LOAD_PTR_ACQUIRE(_PyCompactUnicodeObject_CAST(op)->utf8); +} + +static inline char* PyUnicode_UTF8(PyObject *op) +{ + assert(_PyUnicode_CHECK(op)); + if (PyUnicode_IS_COMPACT_ASCII(op)) { + return ((char*)(_PyASCIIObject_CAST(op) + 1)); + } + else { + return _PyUnicode_UTF8(op); + } +} + +static inline void PyUnicode_SET_UTF8(PyObject *op, char *utf8) +{ + FT_ATOMIC_STORE_PTR_RELEASE(_PyCompactUnicodeObject_CAST(op)->utf8, utf8); +} + +static inline Py_ssize_t PyUnicode_UTF8_LENGTH(PyObject *op) +{ + assert(_PyUnicode_CHECK(op)); + if (PyUnicode_IS_COMPACT_ASCII(op)) { + return _PyASCIIObject_CAST(op)->length; + } + else { + return _PyCompactUnicodeObject_CAST(op)->utf8_length; + } +} + +static inline void PyUnicode_SET_UTF8_LENGTH(PyObject *op, Py_ssize_t length) +{ + _PyCompactUnicodeObject_CAST(op)->utf8_length = length; +} #define _PyUnicode_LENGTH(op) \ (_PyASCIIObject_CAST(op)->length) @@ -131,26 +155,37 @@ NOTE: In the interpreter's initialization phase, some globals are currently (_PyASCIIObject_CAST(op)->state) #define _PyUnicode_HASH(op) \ (_PyASCIIObject_CAST(op)->hash) -#define _PyUnicode_KIND(op) \ - (assert(_PyUnicode_CHECK(op)), \ - _PyASCIIObject_CAST(op)->state.kind) -#define _PyUnicode_GET_LENGTH(op) \ - (assert(_PyUnicode_CHECK(op)), \ - _PyASCIIObject_CAST(op)->length) + +static inline Py_hash_t PyUnicode_HASH(PyObject *op) +{ + assert(_PyUnicode_CHECK(op)); + return FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyASCIIObject_CAST(op)->hash); +} + +static inline void PyUnicode_SET_HASH(PyObject *op, Py_hash_t hash) +{ + FT_ATOMIC_STORE_SSIZE_RELAXED(_PyASCIIObject_CAST(op)->hash, hash); +} + #define _PyUnicode_DATA_ANY(op) \ (_PyUnicodeObject_CAST(op)->data.any) -#define _PyUnicode_SHARE_UTF8(op) \ - (assert(_PyUnicode_CHECK(op)), \ - assert(!PyUnicode_IS_COMPACT_ASCII(op)), \ - (_PyUnicode_UTF8(op) == PyUnicode_DATA(op))) +static inline int _PyUnicode_SHARE_UTF8(PyObject *op) +{ + assert(_PyUnicode_CHECK(op)); + assert(!PyUnicode_IS_COMPACT_ASCII(op)); + return (_PyUnicode_UTF8(op) == PyUnicode_DATA(op)); +} /* true if the Unicode object has an allocated UTF-8 memory block (not shared with other data) */ -#define _PyUnicode_HAS_UTF8_MEMORY(op) \ - ((!PyUnicode_IS_COMPACT_ASCII(op) \ - && _PyUnicode_UTF8(op) \ - && _PyUnicode_UTF8(op) != PyUnicode_DATA(op))) +static inline int _PyUnicode_HAS_UTF8_MEMORY(PyObject *op) +{ + return (!PyUnicode_IS_COMPACT_ASCII(op) + && _PyUnicode_UTF8(op) != NULL + && _PyUnicode_UTF8(op) != PyUnicode_DATA(op)); +} + /* Generic helper macro to convert characters of different types. from_type and to_type have to be valid type names, begin and end @@ -176,10 +211,7 @@ NOTE: In the interpreter's initialization phase, some globals are currently *_to++ = (to_type) *_iter++; \ } while (0) -#define LATIN1(ch) \ - (ch < 128 \ - ? (PyObject*)&_Py_SINGLETON(strings).ascii[ch] \ - : (PyObject*)&_Py_SINGLETON(strings).latin1[ch - 128]) +#define LATIN1 _Py_LATIN1_CHR #ifdef MS_WINDOWS /* On Windows, overallocate by 50% is the best factor */ @@ -201,6 +233,11 @@ static PyObject * unicode_decode_utf8(const char *s, Py_ssize_t size, _Py_error_handler error_handler, const char *errors, Py_ssize_t *consumed); +static int +unicode_decode_utf8_writer(_PyUnicodeWriter *writer, + const char *s, Py_ssize_t size, + _Py_error_handler error_handler, const char *errors, + Py_ssize_t *consumed); #ifdef Py_DEBUG static inline int unicode_is_finalizing(void); static int unicode_is_singleton(PyObject *unicode); @@ -214,18 +251,20 @@ static inline PyObject* unicode_get_empty(void) return &_Py_STR(empty); } -/* This dictionary holds all interned unicode strings. Note that references - to strings in this dictionary are *not* counted in the string's ob_refcnt. - When the interned string reaches a refcnt of 0 the string deallocation - function will delete the reference from this dictionary. -*/ +/* This dictionary holds per-interpreter interned strings. + * See InternalDocs/string_interning.md for details. + */ static inline PyObject *get_interned_dict(PyInterpreterState *interp) { return _Py_INTERP_CACHED_OBJECT(interp, interned_strings); } +/* This hashtable holds statically allocated interned strings. + * See InternalDocs/string_interning.md for details. + */ #define INTERNED_STRINGS _PyRuntime.cached_objects.interned_strings +/* Get number of all interned strings for the current interpreter. */ Py_ssize_t _PyUnicode_InternedSize(void) { @@ -233,8 +272,29 @@ _PyUnicode_InternedSize(void) return _Py_hashtable_len(INTERNED_STRINGS) + PyDict_GET_SIZE(dict); } +/* Get number of immortal interned strings for the current interpreter. */ +Py_ssize_t +_PyUnicode_InternedSize_Immortal(void) +{ + PyObject *dict = get_interned_dict(_PyInterpreterState_GET()); + PyObject *key, *value; + Py_ssize_t pos = 0; + Py_ssize_t count = 0; + + // It's tempting to keep a count and avoid a loop here. But, this function + // is intended for refleak tests. It spends extra work to report the true + // value, to help detect bugs in optimizations. + + while (PyDict_Next(dict, &pos, &key, &value)) { + assert(PyUnicode_CHECK_INTERNED(key) != SSTATE_INTERNED_IMMORTAL_STATIC); + if (PyUnicode_CHECK_INTERNED(key) == SSTATE_INTERNED_IMMORTAL) { + count++; + } + } + return _Py_hashtable_len(INTERNED_STRINGS) + count; +} + static Py_hash_t unicode_hash(PyObject *); -static int unicode_compare_eq(PyObject *, PyObject *); static Py_uhash_t hashtable_unicode_hash(const void *key) @@ -248,35 +308,45 @@ hashtable_unicode_compare(const void *key1, const void *key2) PyObject *obj1 = (PyObject *)key1; PyObject *obj2 = (PyObject *)key2; if (obj1 != NULL && obj2 != NULL) { - return unicode_compare_eq(obj1, obj2); + return unicode_eq(obj1, obj2); } else { return obj1 == obj2; } } +/* Return true if this interpreter should share the main interpreter's + intern_dict. That's important for interpreters which load basic + single-phase init extension modules (m_size == -1). There could be interned + immortal strings that are shared between interpreters, due to the + PyDict_Update(mdict, m_copy) call in import_find_extension(). + + It's not safe to deallocate those strings until all interpreters that + potentially use them are freed. By storing them in the main interpreter, we + ensure they get freed after all other interpreters are freed. +*/ +static bool +has_shared_intern_dict(PyInterpreterState *interp) +{ + PyInterpreterState *main_interp = _PyInterpreterState_Main(); + return interp != main_interp && interp->feature_flags & Py_RTFLAGS_USE_MAIN_OBMALLOC; +} + static int init_interned_dict(PyInterpreterState *interp) { - if (_Py_IsMainInterpreter(interp)) { - assert(INTERNED_STRINGS == NULL); - _Py_hashtable_allocator_t hashtable_alloc = {PyMem_RawMalloc, PyMem_RawFree}; - INTERNED_STRINGS = _Py_hashtable_new_full( - hashtable_unicode_hash, - hashtable_unicode_compare, - NULL, - NULL, - &hashtable_alloc - ); - if (INTERNED_STRINGS == NULL) { + assert(get_interned_dict(interp) == NULL); + PyObject *interned; + if (has_shared_intern_dict(interp)) { + interned = get_interned_dict(_PyInterpreterState_Main()); + Py_INCREF(interned); + } + else { + interned = PyDict_New(); + if (interned == NULL) { return -1; } } - assert(get_interned_dict(interp) == NULL); - PyObject *interned = interned = PyDict_New(); - if (interned == NULL) { - return -1; - } _Py_INTERP_CACHED_OBJECT(interp, interned_strings) = interned; return 0; } @@ -286,11 +356,62 @@ clear_interned_dict(PyInterpreterState *interp) { PyObject *interned = get_interned_dict(interp); if (interned != NULL) { - PyDict_Clear(interned); + if (!has_shared_intern_dict(interp)) { + // only clear if the dict belongs to this interpreter + PyDict_Clear(interned); + } Py_DECREF(interned); _Py_INTERP_CACHED_OBJECT(interp, interned_strings) = NULL; } - if (_Py_IsMainInterpreter(interp) && INTERNED_STRINGS != NULL) { +} + +static PyStatus +init_global_interned_strings(PyInterpreterState *interp) +{ + assert(INTERNED_STRINGS == NULL); + _Py_hashtable_allocator_t hashtable_alloc = {PyMem_RawMalloc, PyMem_RawFree}; + + INTERNED_STRINGS = _Py_hashtable_new_full( + hashtable_unicode_hash, + hashtable_unicode_compare, + // Objects stored here are immortal and statically allocated, + // so we don't need key_destroy_func & value_destroy_func: + NULL, + NULL, + &hashtable_alloc + ); + if (INTERNED_STRINGS == NULL) { + PyErr_Clear(); + return _PyStatus_ERR("failed to create global interned dict"); + } + + /* Intern statically allocated string identifiers, deepfreeze strings, + * and one-byte latin-1 strings. + * This must be done before any module initialization so that statically + * allocated string identifiers are used instead of heap allocated strings. + * Deepfreeze uses the interned identifiers if present to save space + * else generates them and they are interned to speed up dict lookups. + */ + _PyUnicode_InitStaticStrings(interp); + + for (int i = 0; i < 256; i++) { + PyObject *s = LATIN1(i); + _PyUnicode_InternStatic(interp, &s); + assert(s == LATIN1(i)); + } +#ifdef Py_DEBUG + assert(_PyUnicode_CheckConsistency(&_Py_STR(empty), 1)); + + for (int i = 0; i < 256; i++) { + assert(_PyUnicode_CheckConsistency(LATIN1(i), 1)); + } +#endif + return _PyStatus_OK(); +} + +static void clear_global_interned_strings(void) +{ + if (INTERNED_STRINGS != NULL) { _Py_hashtable_destroy(INTERNED_STRINGS); INTERNED_STRINGS = NULL; } @@ -505,7 +626,7 @@ unicode_check_encoding_errors(const char *encoding, const char *errors) /* Disable checks during Python finalization. For example, it allows to call _PyObject_Dump() during finalization for debugging purpose. */ - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { return 0; } @@ -567,7 +688,7 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) || kind == PyUnicode_2BYTE_KIND || kind == PyUnicode_4BYTE_KIND); CHECK(ascii->state.ascii == 0); - CHECK(compact->utf8 != data); + CHECK(_PyUnicode_UTF8(op) != data); } else { PyUnicodeObject *unicode = _PyUnicodeObject_CAST(op); @@ -579,16 +700,17 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) CHECK(ascii->state.compact == 0); CHECK(data != NULL); if (ascii->state.ascii) { - CHECK(compact->utf8 == data); + CHECK(_PyUnicode_UTF8(op) == data); CHECK(compact->utf8_length == ascii->length); } else { - CHECK(compact->utf8 != data); + CHECK(_PyUnicode_UTF8(op) != data); } } - - if (compact->utf8 == NULL) +#ifndef Py_GIL_DISABLED + if (_PyUnicode_UTF8(op) == NULL) CHECK(compact->utf8_length == 0); +#endif } /* check that the best kind is used: O(n) operation */ @@ -623,6 +745,41 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) } CHECK(PyUnicode_READ(kind, data, ascii->length) == 0); } + + /* Check interning state */ +#ifdef Py_DEBUG + // Note that we do not check `_Py_IsImmortal(op)`, since stable ABI + // extensions can make immortal strings mortal (but with a high enough + // refcount). + // The other way is extremely unlikely (worth a potential failed assertion + // in a debug build), so we do check `!_Py_IsImmortal(op)`. + switch (PyUnicode_CHECK_INTERNED(op)) { + case SSTATE_NOT_INTERNED: + if (ascii->state.statically_allocated) { + // This state is for two exceptions: + // - strings are currently checked before they're interned + // - the 256 one-latin1-character strings + // are static but use SSTATE_NOT_INTERNED + } + else { + CHECK(!_Py_IsImmortal(op)); + } + break; + case SSTATE_INTERNED_MORTAL: + CHECK(!ascii->state.statically_allocated); + CHECK(!_Py_IsImmortal(op)); + break; + case SSTATE_INTERNED_IMMORTAL: + CHECK(!ascii->state.statically_allocated); + break; + case SSTATE_INTERNED_IMMORTAL_STATIC: + CHECK(ascii->state.statically_allocated); + break; + default: + Py_UNREACHABLE(); + } +#endif + return 1; #undef CHECK @@ -893,6 +1050,7 @@ ensure_unicode(PyObject *obj) #include "stringlib/count.h" #include "stringlib/find.h" #include "stringlib/replace.h" +#include "stringlib/repr.h" #include "stringlib/find_max_char.h" #include "stringlib/undef.h" @@ -903,6 +1061,7 @@ ensure_unicode(PyObject *obj) #include "stringlib/count.h" #include "stringlib/find.h" #include "stringlib/replace.h" +#include "stringlib/repr.h" #include "stringlib/find_max_char.h" #include "stringlib/undef.h" @@ -913,6 +1072,7 @@ ensure_unicode(PyObject *obj) #include "stringlib/count.h" #include "stringlib/find.h" #include "stringlib/replace.h" +#include "stringlib/repr.h" #include "stringlib/find_max_char.h" #include "stringlib/undef.h" @@ -997,12 +1157,13 @@ resize_compact(PyObject *unicode, Py_ssize_t length) if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) { PyMem_Free(_PyUnicode_UTF8(unicode)); - _PyUnicode_UTF8(unicode) = NULL; - _PyUnicode_UTF8_LENGTH(unicode) = 0; + PyUnicode_SET_UTF8_LENGTH(unicode, 0); + PyUnicode_SET_UTF8(unicode, NULL); } #ifdef Py_TRACE_REFS _Py_ForgetReference(unicode); #endif + _PyReftracerTrack(unicode, PyRefTracer_DESTROY); new_unicode = (PyObject *)PyObject_Realloc(unicode, new_size); if (new_unicode == NULL) { @@ -1050,8 +1211,8 @@ resize_inplace(PyObject *unicode, Py_ssize_t length) if (!share_utf8 && _PyUnicode_HAS_UTF8_MEMORY(unicode)) { PyMem_Free(_PyUnicode_UTF8(unicode)); - _PyUnicode_UTF8(unicode) = NULL; - _PyUnicode_UTF8_LENGTH(unicode) = 0; + PyUnicode_SET_UTF8_LENGTH(unicode, 0); + PyUnicode_SET_UTF8(unicode, NULL); } data = (PyObject *)PyObject_Realloc(data, new_size); @@ -1061,8 +1222,8 @@ resize_inplace(PyObject *unicode, Py_ssize_t length) } _PyUnicode_DATA_ANY(unicode) = data; if (share_utf8) { - _PyUnicode_UTF8(unicode) = data; - _PyUnicode_UTF8_LENGTH(unicode) = length; + PyUnicode_SET_UTF8_LENGTH(unicode, length); + PyUnicode_SET_UTF8(unicode, data); } _PyUnicode_LENGTH(unicode) = length; PyUnicode_WRITE(PyUnicode_KIND(unicode), data, length, 0); @@ -1276,46 +1437,6 @@ PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar) return obj; } -#if SIZEOF_WCHAR_T == 2 -/* Helper function to convert a 16-bits wchar_t representation to UCS4, this - will decode surrogate pairs, the other conversions are implemented as macros - for efficiency. - - This function assumes that unicode can hold one more code point than wstr - characters for a terminating null character. */ -static void -unicode_convert_wchar_to_ucs4(const wchar_t *begin, const wchar_t *end, - PyObject *unicode) -{ - const wchar_t *iter; - Py_UCS4 *ucs4_out; - - assert(unicode != NULL); - assert(_PyUnicode_CHECK(unicode)); - assert(_PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND); - ucs4_out = PyUnicode_4BYTE_DATA(unicode); - - for (iter = begin; iter < end; ) { - assert(ucs4_out < (PyUnicode_4BYTE_DATA(unicode) + - _PyUnicode_GET_LENGTH(unicode))); - if (Py_UNICODE_IS_HIGH_SURROGATE(iter[0]) - && (iter+1) < end - && Py_UNICODE_IS_LOW_SURROGATE(iter[1])) - { - *ucs4_out++ = Py_UNICODE_JOIN_SURROGATES(iter[0], iter[1]); - iter += 2; - } - else { - *ucs4_out++ = *iter; - iter++; - } - } - assert(ucs4_out == (PyUnicode_4BYTE_DATA(unicode) + - _PyUnicode_GET_LENGTH(unicode))); - -} -#endif - static int unicode_check_modifiable(PyObject *unicode) { @@ -1342,11 +1463,14 @@ _copy_characters(PyObject *to, Py_ssize_t to_start, assert(PyUnicode_Check(from)); assert(from_start + how_many <= PyUnicode_GET_LENGTH(from)); - assert(PyUnicode_Check(to)); - assert(to_start + how_many <= PyUnicode_GET_LENGTH(to)); + assert(to == NULL || PyUnicode_Check(to)); - if (how_many == 0) + if (how_many == 0) { return 0; + } + + assert(to != NULL); + assert(to_start + how_many <= PyUnicode_GET_LENGTH(to)); from_kind = PyUnicode_KIND(from); from_data = PyUnicode_DATA(from); @@ -1579,16 +1703,74 @@ unicode_dealloc(PyObject *unicode) _Py_FatalRefcountError("deallocating an Unicode singleton"); } #endif - /* This should never get called, but we also don't want to SEGV if - * we accidentally decref an immortal string out of existence. Since - * the string is an immortal object, just re-set the reference count. - */ - if (PyUnicode_CHECK_INTERNED(unicode) - || _PyUnicode_STATE(unicode).statically_allocated) - { + if (_PyUnicode_STATE(unicode).statically_allocated) { + /* This should never get called, but we also don't want to SEGV if + * we accidentally decref an immortal string out of existence. Since + * the string is an immortal object, just re-set the reference count. + */ +#ifdef Py_DEBUG + Py_UNREACHABLE(); +#endif _Py_SetImmortal(unicode); return; } + switch (_PyUnicode_STATE(unicode).interned) { + case SSTATE_NOT_INTERNED: + break; + case SSTATE_INTERNED_MORTAL: + /* Remove the object from the intern dict. + * Before doing so, we set the refcount to 2: the key and value + * in the interned_dict. + */ + assert(Py_REFCNT(unicode) == 0); + Py_SET_REFCNT(unicode, 2); +#ifdef Py_REF_DEBUG + /* let's be pedantic with the ref total */ + _Py_IncRefTotal(_PyThreadState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); +#endif + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *interned = get_interned_dict(interp); + assert(interned != NULL); + PyObject *popped; + int r = PyDict_Pop(interned, unicode, &popped); + if (r == -1) { + PyErr_WriteUnraisable(unicode); + // We don't know what happened to the string. It's probably + // best to leak it: + // - if it was popped, there are no more references to it + // so it can't cause trouble (except wasted memory) + // - if it wasn't popped, it'll remain interned + _Py_SetImmortal(unicode); + _PyUnicode_STATE(unicode).interned = SSTATE_INTERNED_IMMORTAL; + return; + } + if (r == 0) { + // The interned string was not found in the interned_dict. +#ifdef Py_DEBUG + Py_UNREACHABLE(); +#endif + _Py_SetImmortal(unicode); + return; + } + // Successfully popped. + assert(popped == unicode); + // Only our `popped` reference should be left; remove it too. + assert(Py_REFCNT(unicode) == 1); + Py_SET_REFCNT(unicode, 0); +#ifdef Py_REF_DEBUG + /* let's be pedantic with the ref total */ + _Py_DecRefTotal(_PyThreadState_GET()); +#endif + break; + default: + // As with `statically_allocated` above. +#ifdef Py_REF_DEBUG + Py_UNREACHABLE(); +#endif + _Py_SetImmortal(unicode); + return; + } if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) { PyMem_Free(_PyUnicode_UTF8(unicode)); } @@ -1624,7 +1806,7 @@ unicode_modifiable(PyObject *unicode) assert(_PyUnicode_CHECK(unicode)); if (Py_REFCNT(unicode) != 1) return 0; - if (_PyUnicode_HASH(unicode) != -1) + if (PyUnicode_HASH(unicode) != -1) return 0; if (PyUnicode_CHECK_INTERNED(unicode)) return 0; @@ -1751,7 +1933,6 @@ static PyObject* get_latin1_char(Py_UCS1 ch) { PyObject *o = LATIN1(ch); - assert(_Py_IsImmortal(o)); return o; } @@ -1781,6 +1962,62 @@ unicode_char(Py_UCS4 ch) return unicode; } + +static inline void +unicode_write_widechar(int kind, void *data, + const wchar_t *u, Py_ssize_t size, + Py_ssize_t num_surrogates) +{ + switch (kind) { + case PyUnicode_1BYTE_KIND: + _PyUnicode_CONVERT_BYTES(wchar_t, unsigned char, u, u + size, data); + break; + + case PyUnicode_2BYTE_KIND: +#if SIZEOF_WCHAR_T == 2 + memcpy(data, u, size * 2); +#else + _PyUnicode_CONVERT_BYTES(wchar_t, Py_UCS2, u, u + size, data); +#endif + break; + + case PyUnicode_4BYTE_KIND: + { +#if SIZEOF_WCHAR_T == 2 + // Convert a 16-bits wchar_t representation to UCS4, this will decode + // surrogate pairs. + const wchar_t *end = u + size; + Py_UCS4 *ucs4_out = (Py_UCS4*)data; +# ifndef NDEBUG + Py_UCS4 *ucs4_end = (Py_UCS4*)data + (size - num_surrogates); +# endif + for (const wchar_t *iter = u; iter < end; ) { + assert(ucs4_out < ucs4_end); + if (Py_UNICODE_IS_HIGH_SURROGATE(iter[0]) + && (iter+1) < end + && Py_UNICODE_IS_LOW_SURROGATE(iter[1])) + { + *ucs4_out++ = Py_UNICODE_JOIN_SURROGATES(iter[0], iter[1]); + iter += 2; + } + else { + *ucs4_out++ = *iter; + iter++; + } + } + assert(ucs4_out == ucs4_end); +#else + assert(num_surrogates == 0); + memcpy(data, u, size * 4); +#endif + break; + } + default: + Py_UNREACHABLE(); + } +} + + PyObject * PyUnicode_FromWideChar(const wchar_t *u, Py_ssize_t size) { @@ -1833,36 +2070,63 @@ PyUnicode_FromWideChar(const wchar_t *u, Py_ssize_t size) if (!unicode) return NULL; - switch (PyUnicode_KIND(unicode)) { - case PyUnicode_1BYTE_KIND: - _PyUnicode_CONVERT_BYTES(wchar_t, unsigned char, - u, u + size, PyUnicode_1BYTE_DATA(unicode)); - break; - case PyUnicode_2BYTE_KIND: -#if Py_UNICODE_SIZE == 2 - memcpy(PyUnicode_2BYTE_DATA(unicode), u, size * 2); -#else - _PyUnicode_CONVERT_BYTES(wchar_t, Py_UCS2, - u, u + size, PyUnicode_2BYTE_DATA(unicode)); -#endif - break; - case PyUnicode_4BYTE_KIND: -#if SIZEOF_WCHAR_T == 2 - /* This is the only case which has to process surrogates, thus - a simple copy loop is not enough and we need a function. */ - unicode_convert_wchar_to_ucs4(u, u + size, unicode); -#else - assert(num_surrogates == 0); - memcpy(PyUnicode_4BYTE_DATA(unicode), u, size * 4); + unicode_write_widechar(PyUnicode_KIND(unicode), PyUnicode_DATA(unicode), + u, size, num_surrogates); + + return unicode_result(unicode); +} + + +int +PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *pub_writer, + const wchar_t *str, + Py_ssize_t size) +{ + _PyUnicodeWriter *writer = (_PyUnicodeWriter *)pub_writer; + + if (size < 0) { + size = wcslen(str); + } + + if (size == 0) { + return 0; + } + +#ifdef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION + /* Oracle Solaris uses non-Unicode internal wchar_t form for + non-Unicode locales and hence needs conversion to UCS-4 first. */ + if (_Py_LocaleUsesNonUnicodeWchar()) { + wchar_t* converted = _Py_DecodeNonUnicodeWchar(str, size); + if (!converted) { + return -1; + } + + int res = PyUnicodeWriter_WriteUCS4(pub_writer, converted, size); + PyMem_Free(converted); + return res; + } #endif - break; - default: - Py_UNREACHABLE(); + + Py_UCS4 maxchar = 0; + Py_ssize_t num_surrogates; + if (find_maxchar_surrogates(str, str + size, + &maxchar, &num_surrogates) == -1) { + return -1; } - return unicode_result(unicode); + if (_PyUnicodeWriter_Prepare(writer, size - num_surrogates, maxchar) < 0) { + return -1; + } + + int kind = writer->kind; + void *data = (Py_UCS1*)writer->data + writer->pos * kind; + unicode_write_widechar(kind, data, str, size, num_surrogates); + + writer->pos += size - num_surrogates; + return 0; } + PyObject * PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) { @@ -1933,7 +2197,7 @@ _PyUnicode_FromId(_Py_Identifier *id) if (!obj) { goto end; } - PyUnicode_InternInPlace(&obj); + _PyUnicode_InternImmortal(interp, &obj); if (index >= ids->size) { // Overallocate to reduce the number of realloc @@ -2088,6 +2352,51 @@ _PyUnicode_FromUCS4(const Py_UCS4 *u, Py_ssize_t size) return res; } + +int +PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *pub_writer, + Py_UCS4 *str, + Py_ssize_t size) +{ + _PyUnicodeWriter *writer = (_PyUnicodeWriter*)pub_writer; + + if (size < 0) { + PyErr_SetString(PyExc_ValueError, + "size must be positive"); + return -1; + } + + if (size == 0) { + return 0; + } + + Py_UCS4 max_char = ucs4lib_find_max_char(str, str + size); + + if (_PyUnicodeWriter_Prepare(writer, size, max_char) < 0) { + return -1; + } + + int kind = writer->kind; + void *data = (Py_UCS1*)writer->data + writer->pos * kind; + if (kind == PyUnicode_1BYTE_KIND) { + _PyUnicode_CONVERT_BYTES(Py_UCS4, Py_UCS1, + str, str + size, + data); + } + else if (kind == PyUnicode_2BYTE_KIND) { + _PyUnicode_CONVERT_BYTES(Py_UCS4, Py_UCS2, + str, str + size, + data); + } + else { + memcpy(data, str, size * sizeof(Py_UCS4)); + } + writer->pos += size; + + return 0; +} + + PyObject* PyUnicode_FromKindAndData(int kind, const void *buffer, Py_ssize_t size) { @@ -2376,14 +2685,12 @@ unicode_fromformat_write_str(_PyUnicodeWriter *writer, PyObject *str, } static int -unicode_fromformat_write_cstr(_PyUnicodeWriter *writer, const char *str, +unicode_fromformat_write_utf8(_PyUnicodeWriter *writer, const char *str, Py_ssize_t width, Py_ssize_t precision, int flags) { /* UTF-8 */ + Py_ssize_t *pconsumed = NULL; Py_ssize_t length; - PyObject *unicode; - int res; - if (precision == -1) { length = strlen(str); } @@ -2392,12 +2699,28 @@ unicode_fromformat_write_cstr(_PyUnicodeWriter *writer, const char *str, while (length < precision && str[length]) { length++; } + if (length == precision) { + /* The input string is not NUL-terminated. If it ends with an + * incomplete UTF-8 sequence, truncate the string just before it. + * Incomplete sequences in the middle and sequences which cannot + * be valid prefixes are still treated as errors and replaced + * with \xfffd. */ + pconsumed = &length; + } + } + + if (width < 0) { + return unicode_decode_utf8_writer(writer, str, length, + _Py_ERROR_REPLACE, "replace", pconsumed); } - unicode = PyUnicode_DecodeUTF8Stateful(str, length, "replace", NULL); + + PyObject *unicode = PyUnicode_DecodeUTF8Stateful(str, length, + "replace", pconsumed); if (unicode == NULL) return -1; - res = unicode_fromformat_write_str(writer, unicode, width, -1, flags); + int res = unicode_fromformat_write_str(writer, unicode, + width, -1, flags); Py_DECREF(unicode); return res; } @@ -2406,11 +2729,7 @@ static int unicode_fromformat_write_wcstr(_PyUnicodeWriter *writer, const wchar_t *str, Py_ssize_t width, Py_ssize_t precision, int flags) { - /* UTF-8 */ Py_ssize_t length; - PyObject *unicode; - int res; - if (precision == -1) { length = wcslen(str); } @@ -2420,11 +2739,17 @@ unicode_fromformat_write_wcstr(_PyUnicodeWriter *writer, const wchar_t *str, length++; } } - unicode = PyUnicode_FromWideChar(str, length); + + if (width < 0) { + return PyUnicodeWriter_WriteWideChar((PyUnicodeWriter*)writer, + str, length); + } + + PyObject *unicode = PyUnicode_FromWideChar(str, length); if (unicode == NULL) return -1; - res = unicode_fromformat_write_str(writer, unicode, width, -1, flags); + int res = unicode_fromformat_write_str(writer, unicode, width, -1, flags); Py_DECREF(unicode); return res; } @@ -2434,11 +2759,6 @@ unicode_fromformat_write_wcstr(_PyUnicodeWriter *writer, const wchar_t *str, #define F_SIZE 3 #define F_PTRDIFF 4 #define F_INTMAX 5 -static const char * const formats[] = {"%d", "%ld", "%lld", "%zd", "%td", "%jd"}; -static const char * const formats_o[] = {"%o", "%lo", "%llo", "%zo", "%to", "%jo"}; -static const char * const formats_u[] = {"%u", "%lu", "%llu", "%zu", "%tu", "%ju"}; -static const char * const formats_x[] = {"%x", "%lx", "%llx", "%zx", "%tx", "%jx"}; -static const char * const formats_X[] = {"%X", "%lX", "%llX", "%zX", "%tX", "%jX"}; static const char* unicode_fromformat_arg(_PyUnicodeWriter *writer, @@ -2468,6 +2788,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, switch (*f++) { case '-': flags |= F_LJUST; continue; case '0': flags |= F_ZERO; continue; + case '#': flags |= F_ALT; continue; } f--; break; @@ -2579,47 +2900,44 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, case 'd': case 'i': case 'o': case 'u': case 'x': case 'X': { - /* used by sprintf */ char buffer[MAX_INTMAX_CHARS]; - const char *fmt = NULL; - switch (*f) { - case 'o': fmt = formats_o[sizemod]; break; - case 'u': fmt = formats_u[sizemod]; break; - case 'x': fmt = formats_x[sizemod]; break; - case 'X': fmt = formats_X[sizemod]; break; - default: fmt = formats[sizemod]; break; - } - int issigned = (*f == 'd' || *f == 'i'); + + // Fill buffer using sprinf, with one of many possible format + // strings, like "%llX" for `long long` in hexadecimal. + // The type/size is in `sizemod`; the format is in `*f`. + + // Use macros with nested switches to keep the sprintf format strings + // as compile-time literals, avoiding warnings and maybe allowing + // optimizations. + + // `SPRINT` macro does one sprintf + // Example usage: SPRINT("l", "X", unsigned long) expands to + // sprintf(buffer, "%" "l" "X", va_arg(*vargs, unsigned long)) + #define SPRINT(SIZE_SPEC, FMT_CHAR, TYPE) \ + sprintf(buffer, "%" SIZE_SPEC FMT_CHAR, va_arg(*vargs, TYPE)) + + // One inner switch to handle all format variants + #define DO_SPRINTS(SIZE_SPEC, SIGNED_TYPE, UNSIGNED_TYPE) \ + switch (*f) { \ + case 'o': len = SPRINT(SIZE_SPEC, "o", UNSIGNED_TYPE); break; \ + case 'u': len = SPRINT(SIZE_SPEC, "u", UNSIGNED_TYPE); break; \ + case 'x': len = SPRINT(SIZE_SPEC, "x", UNSIGNED_TYPE); break; \ + case 'X': len = SPRINT(SIZE_SPEC, "X", UNSIGNED_TYPE); break; \ + default: len = SPRINT(SIZE_SPEC, "d", SIGNED_TYPE); break; \ + } + + // Outer switch to handle all the sizes/types switch (sizemod) { - case F_LONG: - len = issigned ? - sprintf(buffer, fmt, va_arg(*vargs, long)) : - sprintf(buffer, fmt, va_arg(*vargs, unsigned long)); - break; - case F_LONGLONG: - len = issigned ? - sprintf(buffer, fmt, va_arg(*vargs, long long)) : - sprintf(buffer, fmt, va_arg(*vargs, unsigned long long)); - break; - case F_SIZE: - len = issigned ? - sprintf(buffer, fmt, va_arg(*vargs, Py_ssize_t)) : - sprintf(buffer, fmt, va_arg(*vargs, size_t)); - break; - case F_PTRDIFF: - len = sprintf(buffer, fmt, va_arg(*vargs, ptrdiff_t)); - break; - case F_INTMAX: - len = issigned ? - sprintf(buffer, fmt, va_arg(*vargs, intmax_t)) : - sprintf(buffer, fmt, va_arg(*vargs, uintmax_t)); - break; - default: - len = issigned ? - sprintf(buffer, fmt, va_arg(*vargs, int)) : - sprintf(buffer, fmt, va_arg(*vargs, unsigned int)); - break; + case F_LONG: DO_SPRINTS("l", long, unsigned long); break; + case F_LONGLONG: DO_SPRINTS("ll", long long, unsigned long long); break; + case F_SIZE: DO_SPRINTS("z", Py_ssize_t, size_t); break; + case F_PTRDIFF: DO_SPRINTS("t", ptrdiff_t, ptrdiff_t); break; + case F_INTMAX: DO_SPRINTS("j", intmax_t, uintmax_t); break; + default: DO_SPRINTS("", int, unsigned int); break; } + #undef SPRINT + #undef DO_SPRINTS + assert(len >= 0); int sign = (buffer[0] == '-'); @@ -2698,7 +3016,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, else { /* UTF-8 */ const char *s = va_arg(*vargs, const char*); - if (unicode_fromformat_write_cstr(writer, s, width, precision, flags) < 0) + if (unicode_fromformat_write_utf8(writer, s, width, precision, flags) < 0) return NULL; } break; @@ -2737,7 +3055,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, } else { assert(str != NULL); - if (unicode_fromformat_write_cstr(writer, str, width, precision, flags) < 0) + if (unicode_fromformat_write_utf8(writer, str, width, precision, flags) < 0) return NULL; } break; @@ -2791,6 +3109,62 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, break; } + case 'T': + { + PyObject *obj = va_arg(*vargs, PyObject *); + PyTypeObject *type = (PyTypeObject *)Py_NewRef(Py_TYPE(obj)); + + PyObject *type_name; + if (flags & F_ALT) { + type_name = _PyType_GetFullyQualifiedName(type, ':'); + } + else { + type_name = PyType_GetFullyQualifiedName(type); + } + Py_DECREF(type); + if (!type_name) { + return NULL; + } + + if (unicode_fromformat_write_str(writer, type_name, + width, precision, flags) == -1) { + Py_DECREF(type_name); + return NULL; + } + Py_DECREF(type_name); + break; + } + + case 'N': + { + PyObject *type_raw = va_arg(*vargs, PyObject *); + assert(type_raw != NULL); + + if (!PyType_Check(type_raw)) { + PyErr_SetString(PyExc_TypeError, "%N argument must be a type"); + return NULL; + } + PyTypeObject *type = (PyTypeObject*)type_raw; + + PyObject *type_name; + if (flags & F_ALT) { + type_name = _PyType_GetFullyQualifiedName(type, ':'); + } + else { + type_name = PyType_GetFullyQualifiedName(type); + } + if (!type_name) { + return NULL; + } + if (unicode_fromformat_write_str(writer, type_name, + width, precision, flags) == -1) { + Py_DECREF(type_name); + return NULL; + } + Py_DECREF(type_name); + break; + } + default: invalid_format: PyErr_Format(PyExc_SystemError, "invalid format string: %s", p); @@ -2801,61 +3175,71 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, return f; } -PyObject * -PyUnicode_FromFormatV(const char *format, va_list vargs) +static int +unicode_from_format(_PyUnicodeWriter *writer, const char *format, va_list vargs) { - va_list vargs2; - const char *f; - _PyUnicodeWriter writer; - - _PyUnicodeWriter_Init(&writer); - writer.min_length = strlen(format) + 100; - writer.overallocate = 1; + Py_ssize_t len = strlen(format); + writer->min_length += len + 100; + writer->overallocate = 1; // Copy varags to be able to pass a reference to a subfunction. + va_list vargs2; va_copy(vargs2, vargs); - for (f = format; *f; ) { + // _PyUnicodeWriter_WriteASCIIString() below requires the format string + // to be encoded to ASCII. + int is_ascii = (ucs1lib_find_max_char((Py_UCS1*)format, (Py_UCS1*)format + len) < 128); + if (!is_ascii) { + Py_ssize_t i; + for (i=0; i < len && (unsigned char)format[i] <= 127; i++); + PyErr_Format(PyExc_ValueError, + "PyUnicode_FromFormatV() expects an ASCII-encoded format " + "string, got a non-ASCII byte: 0x%02x", + (unsigned char)format[i]); + goto fail; + } + + for (const char *f = format; *f; ) { if (*f == '%') { - f = unicode_fromformat_arg(&writer, f, &vargs2); + f = unicode_fromformat_arg(writer, f, &vargs2); if (f == NULL) goto fail; } else { - const char *p; - Py_ssize_t len; - - p = f; - do - { - if ((unsigned char)*p > 127) { - PyErr_Format(PyExc_ValueError, - "PyUnicode_FromFormatV() expects an ASCII-encoded format " - "string, got a non-ASCII byte: 0x%02x", - (unsigned char)*p); - goto fail; - } - p++; + const char *p = strchr(f, '%'); + if (p != NULL) { + len = p - f; + } + else { + len = strlen(f); + writer->overallocate = 0; } - while (*p != '\0' && *p != '%'); - len = p - f; - - if (*p == '\0') - writer.overallocate = 0; - if (_PyUnicodeWriter_WriteASCIIString(&writer, f, len) < 0) + if (_PyUnicodeWriter_WriteASCIIString(writer, f, len) < 0) { goto fail; - - f = p; + } + f += len; } } va_end(vargs2); - return _PyUnicodeWriter_Finish(&writer); + return 0; fail: va_end(vargs2); - _PyUnicodeWriter_Dealloc(&writer); - return NULL; + return -1; +} + +PyObject * +PyUnicode_FromFormatV(const char *format, va_list vargs) +{ + _PyUnicodeWriter writer; + _PyUnicodeWriter_Init(&writer); + + if (unicode_from_format(&writer, format, vargs) < 0) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); } PyObject * @@ -2870,6 +3254,23 @@ PyUnicode_FromFormat(const char *format, ...) return ret; } +int +PyUnicodeWriter_Format(PyUnicodeWriter *writer, const char *format, ...) +{ + _PyUnicodeWriter *_writer = (_PyUnicodeWriter*)writer; + Py_ssize_t old_pos = _writer->pos; + + va_list vargs; + va_start(vargs, format); + int res = unicode_from_format(_writer, format, vargs); + va_end(vargs); + + if (res < 0) { + _writer->pos = old_pos; + } + return res; +} + static Py_ssize_t unicode_get_widechar_size(PyObject *unicode) { @@ -3819,6 +4220,21 @@ PyUnicode_FSDecoder(PyObject* arg, void* addr) static int unicode_fill_utf8(PyObject *unicode); + +static int +unicode_ensure_utf8(PyObject *unicode) +{ + int err = 0; + if (PyUnicode_UTF8(unicode) == NULL) { + Py_BEGIN_CRITICAL_SECTION(unicode); + if (PyUnicode_UTF8(unicode) == NULL) { + err = unicode_fill_utf8(unicode); + } + Py_END_CRITICAL_SECTION(); + } + return err; +} + const char * PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize) { @@ -3830,13 +4246,11 @@ PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize) return NULL; } - if (PyUnicode_UTF8(unicode) == NULL) { - if (unicode_fill_utf8(unicode) == -1) { - if (psize) { - *psize = -1; - } - return NULL; + if (unicode_ensure_utf8(unicode) == -1) { + if (psize) { + *psize = -1; } + return NULL; } if (psize) { @@ -4611,42 +5025,238 @@ PyUnicode_DecodeUTF8(const char *s, #include "stringlib/codecs.h" #include "stringlib/undef.h" -#include "stringlib/ucs4lib.h" -#include "stringlib/codecs.h" -#include "stringlib/undef.h" +#include "stringlib/ucs4lib.h" +#include "stringlib/codecs.h" +#include "stringlib/undef.h" + +#if (SIZEOF_SIZE_T == 8) +/* Mask to quickly check whether a C 'size_t' contains a + non-ASCII, UTF8-encoded char. */ +# define ASCII_CHAR_MASK 0x8080808080808080ULL +// used to count codepoints in UTF-8 string. +# define VECTOR_0101 0x0101010101010101ULL +# define VECTOR_00FF 0x00ff00ff00ff00ffULL +#elif (SIZEOF_SIZE_T == 4) +# define ASCII_CHAR_MASK 0x80808080U +# define VECTOR_0101 0x01010101U +# define VECTOR_00FF 0x00ff00ffU +#else +# error C 'size_t' size should be either 4 or 8! +#endif + +#if (defined(__clang__) || defined(__GNUC__)) +#define HAVE_CTZ 1 +static inline unsigned int +ctz(size_t v) +{ + return __builtin_ctzll((unsigned long long)v); +} +#elif defined(_MSC_VER) +#define HAVE_CTZ 1 +static inline unsigned int +ctz(size_t v) +{ + unsigned long pos; +#if SIZEOF_SIZE_T == 4 + _BitScanForward(&pos, v); +#else + _BitScanForward64(&pos, v); +#endif /* SIZEOF_SIZE_T */ + return pos; +} +#else +#define HAVE_CTZ 0 +#endif + +#if HAVE_CTZ && PY_LITTLE_ENDIAN +// load p[0]..p[size-1] as a size_t without unaligned access nor read ahead. +static size_t +load_unaligned(const unsigned char *p, size_t size) +{ + union { + size_t s; + unsigned char b[SIZEOF_SIZE_T]; + } u; + u.s = 0; + // This switch statement assumes little endian because: + // * union is faster than bitwise or and shift. + // * big endian machine is rare and hard to maintain. + switch (size) { + default: +#if SIZEOF_SIZE_T == 8 + case 8: + u.b[7] = p[7]; + _Py_FALLTHROUGH; + case 7: + u.b[6] = p[6]; + _Py_FALLTHROUGH; + case 6: + u.b[5] = p[5]; + _Py_FALLTHROUGH; + case 5: + u.b[4] = p[4]; + _Py_FALLTHROUGH; +#endif + case 4: + u.b[3] = p[3]; + _Py_FALLTHROUGH; + case 3: + u.b[2] = p[2]; + _Py_FALLTHROUGH; + case 2: + u.b[1] = p[1]; + _Py_FALLTHROUGH; + case 1: + u.b[0] = p[0]; + break; + case 0: + break; + } + return u.s; +} +#endif + +/* + * Find the first non-ASCII character in a byte sequence. + * + * This function scans a range of bytes from `start` to `end` and returns the + * index of the first byte that is not an ASCII character (i.e., has the most + * significant bit set). If all characters in the range are ASCII, it returns + * `end - start`. + */ +static Py_ssize_t +find_first_nonascii(const unsigned char *start, const unsigned char *end) +{ + // The search is done in `size_t` chunks. + // The start and end might not be aligned at `size_t` boundaries, + // so they're handled specially. + + const unsigned char *p = start; + + if (end - start >= SIZEOF_SIZE_T) { + // Avoid unaligned read. +#if PY_LITTLE_ENDIAN && HAVE_CTZ + size_t u; + memcpy(&u, p, sizeof(size_t)); + u &= ASCII_CHAR_MASK; + if (u) { + return (ctz(u) - 7) / 8; + } + p = _Py_ALIGN_DOWN(p + SIZEOF_SIZE_T, SIZEOF_SIZE_T); +#else /* PY_LITTLE_ENDIAN && HAVE_CTZ */ + const unsigned char *p2 = _Py_ALIGN_UP(p, SIZEOF_SIZE_T); + while (p < p2) { + if (*p & 0x80) { + return p - start; + } + p++; + } +#endif + + const unsigned char *e = end - SIZEOF_SIZE_T; + while (p <= e) { + size_t u = (*(const size_t *)p) & ASCII_CHAR_MASK; + if (u) { +#if PY_LITTLE_ENDIAN && HAVE_CTZ + return p - start + (ctz(u) - 7) / 8; +#else + // big endian and minor compilers are difficult to test. + // fallback to per byte check. + break; +#endif + } + p += SIZEOF_SIZE_T; + } + } +#if PY_LITTLE_ENDIAN && HAVE_CTZ + assert((end - p) < SIZEOF_SIZE_T); + // we can not use *(const size_t*)p to avoid buffer overrun. + size_t u = load_unaligned(p, end - p) & ASCII_CHAR_MASK; + if (u) { + return p - start + (ctz(u) - 7) / 8; + } + return end - start; +#else + while (p < end) { + if (*p & 0x80) { + break; + } + p++; + } + return p - start; +#endif +} + +static inline int +scalar_utf8_start_char(unsigned int ch) +{ + // 0xxxxxxx or 11xxxxxx are first byte. + return (~ch >> 7 | ch >> 6) & 1; +} + +static inline size_t +vector_utf8_start_chars(size_t v) +{ + return ((~v >> 7) | (v >> 6)) & VECTOR_0101; +} + + +// Count the number of UTF-8 code points in a given byte sequence. +static Py_ssize_t +utf8_count_codepoints(const unsigned char *s, const unsigned char *end) +{ + Py_ssize_t len = 0; + + if (end - s >= SIZEOF_SIZE_T) { + while (!_Py_IS_ALIGNED(s, ALIGNOF_SIZE_T)) { + len += scalar_utf8_start_char(*s++); + } -/* Mask to quickly check whether a C 'size_t' contains a - non-ASCII, UTF8-encoded char. */ -#if (SIZEOF_SIZE_T == 8) -# define ASCII_CHAR_MASK 0x8080808080808080ULL -#elif (SIZEOF_SIZE_T == 4) -# define ASCII_CHAR_MASK 0x80808080U -#else -# error C 'size_t' size should be either 4 or 8! + while (s + SIZEOF_SIZE_T <= end) { + const unsigned char *e = end; + if (e - s > SIZEOF_SIZE_T * 255) { + e = s + SIZEOF_SIZE_T * 255; + } + Py_ssize_t vstart = 0; + while (s + SIZEOF_SIZE_T <= e) { + size_t v = *(size_t*)s; + size_t vs = vector_utf8_start_chars(v); + vstart += vs; + s += SIZEOF_SIZE_T; + } + vstart = (vstart & VECTOR_00FF) + ((vstart >> 8) & VECTOR_00FF); + vstart += vstart >> 16; +#if SIZEOF_SIZE_T == 8 + vstart += vstart >> 32; #endif + len += vstart & 0x7ff; + } + } + while (s < end) { + len += scalar_utf8_start_char(*s++); + } + return len; +} static Py_ssize_t ascii_decode(const char *start, const char *end, Py_UCS1 *dest) { - const char *p = start; - #if SIZEOF_SIZE_T <= SIZEOF_VOID_P - assert(_Py_IS_ALIGNED(dest, ALIGNOF_SIZE_T)); - if (_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T)) { + if (_Py_IS_ALIGNED(start, ALIGNOF_SIZE_T) + && _Py_IS_ALIGNED(dest, ALIGNOF_SIZE_T)) + { /* Fast path, see in STRINGLIB(utf8_decode) for an explanation. */ - /* Help allocation */ - const char *_p = p; - Py_UCS1 * q = dest; - while (_p + SIZEOF_SIZE_T <= end) { - size_t value = *(const size_t *) _p; + const char *p = start; + Py_UCS1 *q = dest; + while (p + SIZEOF_SIZE_T <= end) { + size_t value = *(const size_t *) p; if (value & ASCII_CHAR_MASK) break; *((size_t *)q) = value; - _p += SIZEOF_SIZE_T; + p += SIZEOF_SIZE_T; q += SIZEOF_SIZE_T; } - p = _p; while (p < end) { if ((unsigned char)*p & 0x80) break; @@ -4655,70 +5265,19 @@ ascii_decode(const char *start, const char *end, Py_UCS1 *dest) return p - start; } #endif - while (p < end) { - /* Fast path, see in STRINGLIB(utf8_decode) in stringlib/codecs.h - for an explanation. */ - if (_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T)) { - /* Help allocation */ - const char *_p = p; - while (_p + SIZEOF_SIZE_T <= end) { - size_t value = *(const size_t *) _p; - if (value & ASCII_CHAR_MASK) - break; - _p += SIZEOF_SIZE_T; - } - p = _p; - if (_p == end) - break; - } - if ((unsigned char)*p & 0x80) - break; - ++p; - } - memcpy(dest, start, p - start); - return p - start; + Py_ssize_t pos = find_first_nonascii((const unsigned char*)start, + (const unsigned char*)end); + memcpy(dest, start, pos); + return pos; } -static PyObject * -unicode_decode_utf8(const char *s, Py_ssize_t size, - _Py_error_handler error_handler, const char *errors, - Py_ssize_t *consumed) +static int +unicode_decode_utf8_impl(_PyUnicodeWriter *writer, + const char *starts, const char *s, const char *end, + _Py_error_handler error_handler, + const char *errors, + Py_ssize_t *consumed) { - if (size == 0) { - if (consumed) - *consumed = 0; - _Py_RETURN_UNICODE_EMPTY(); - } - - /* ASCII is equivalent to the first 128 ordinals in Unicode. */ - if (size == 1 && (unsigned char)s[0] < 128) { - if (consumed) { - *consumed = 1; - } - return get_latin1_char((unsigned char)s[0]); - } - - const char *starts = s; - const char *end = s + size; - - // fast path: try ASCII string. - PyObject *u = PyUnicode_New(size, 127); - if (u == NULL) { - return NULL; - } - s += ascii_decode(s, end, PyUnicode_1BYTE_DATA(u)); - if (s == end) { - if (consumed) { - *consumed = size; - } - return u; - } - - // Use _PyUnicodeWriter after fast path is failed. - _PyUnicodeWriter writer; - _PyUnicodeWriter_InitWithBuffer(&writer, u); - writer.pos = s - starts; - Py_ssize_t startinpos, endinpos; const char *errmsg = ""; PyObject *error_handler_obj = NULL; @@ -4726,18 +5285,18 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, while (s < end) { Py_UCS4 ch; - int kind = writer.kind; + int kind = writer->kind; if (kind == PyUnicode_1BYTE_KIND) { - if (PyUnicode_IS_ASCII(writer.buffer)) - ch = asciilib_utf8_decode(&s, end, writer.data, &writer.pos); + if (PyUnicode_IS_ASCII(writer->buffer)) + ch = asciilib_utf8_decode(&s, end, writer->data, &writer->pos); else - ch = ucs1lib_utf8_decode(&s, end, writer.data, &writer.pos); + ch = ucs1lib_utf8_decode(&s, end, writer->data, &writer->pos); } else if (kind == PyUnicode_2BYTE_KIND) { - ch = ucs2lib_utf8_decode(&s, end, writer.data, &writer.pos); + ch = ucs2lib_utf8_decode(&s, end, writer->data, &writer->pos); } else { assert(kind == PyUnicode_4BYTE_KIND); - ch = ucs4lib_utf8_decode(&s, end, writer.data, &writer.pos); + ch = ucs4lib_utf8_decode(&s, end, writer->data, &writer->pos); } switch (ch) { @@ -4760,7 +5319,7 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, /* Truncated surrogate code in range D800-DFFF */ goto End; } - /* fall through */ + _Py_FALLTHROUGH; case 3: case 4: errmsg = "invalid continuation byte"; @@ -4768,7 +5327,9 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, endinpos = startinpos + ch - 1; break; default: - if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0) + // ch doesn't fit into kind, so change the buffer kind to write + // the character + if (_PyUnicodeWriter_WriteCharInline(writer, ch) < 0) goto onError; continue; } @@ -4782,7 +5343,7 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, break; case _Py_ERROR_REPLACE: - if (_PyUnicodeWriter_WriteCharInline(&writer, 0xfffd) < 0) + if (_PyUnicodeWriter_WriteCharInline(writer, 0xfffd) < 0) goto onError; s += (endinpos - startinpos); break; @@ -4791,13 +5352,13 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, { Py_ssize_t i; - if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0) + if (_PyUnicodeWriter_PrepareKind(writer, PyUnicode_2BYTE_KIND) < 0) goto onError; for (i=startinpos; ikind, writer->data, writer->pos, ch + 0xdc00); - writer.pos++; + writer->pos++; } s += (endinpos - startinpos); break; @@ -4808,8 +5369,13 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, errors, &error_handler_obj, "utf-8", errmsg, &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) + writer)) { goto onError; + } + + if (_PyUnicodeWriter_Prepare(writer, end - s, 127) < 0) { + return -1; + } } } @@ -4819,13 +5385,148 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, Py_XDECREF(error_handler_obj); Py_XDECREF(exc); - return _PyUnicodeWriter_Finish(&writer); + return 0; onError: Py_XDECREF(error_handler_obj); Py_XDECREF(exc); - _PyUnicodeWriter_Dealloc(&writer); - return NULL; + return -1; +} + + +static PyObject * +unicode_decode_utf8(const char *s, Py_ssize_t size, + _Py_error_handler error_handler, const char *errors, + Py_ssize_t *consumed) +{ + if (size == 0) { + if (consumed) { + *consumed = 0; + } + _Py_RETURN_UNICODE_EMPTY(); + } + + /* ASCII is equivalent to the first 128 ordinals in Unicode. */ + if (size == 1 && (unsigned char)s[0] < 128) { + if (consumed) { + *consumed = 1; + } + return get_latin1_char((unsigned char)s[0]); + } + + // I don't know this check is necessary or not. But there is a test + // case that requires size=PY_SSIZE_T_MAX cause MemoryError. + if (PY_SSIZE_T_MAX - sizeof(PyCompactUnicodeObject) < (size_t)size) { + PyErr_NoMemory(); + return NULL; + } + + const char *starts = s; + const char *end = s + size; + + Py_ssize_t pos = find_first_nonascii((const unsigned char*)starts, (const unsigned char*)end); + if (pos == size) { // fast path: ASCII string. + PyObject *u = PyUnicode_New(size, 127); + if (u == NULL) { + return NULL; + } + memcpy(PyUnicode_1BYTE_DATA(u), s, size); + if (consumed) { + *consumed = size; + } + return u; + } + + int maxchr = 127; + Py_ssize_t maxsize = size; + + unsigned char ch = (unsigned char)(s[pos]); + // error handler other than strict may remove/replace the invalid byte. + // consumed != NULL allows 1~3 bytes remainings. + // 0x80 <= ch < 0xc2 is invalid start byte that cause UnicodeDecodeError. + // otherwise: check the input and decide the maxchr and maxsize to reduce + // reallocation and copy. + if (error_handler == _Py_ERROR_STRICT && !consumed && ch >= 0xc2) { + // we only calculate the number of codepoints and don't determine the exact maxchr. + // This is because writing fast and portable SIMD code to find maxchr is difficult. + // If reallocation occurs for a larger maxchar, knowing the exact number of codepoints + // means that it is no longer necessary to allocate several times the required amount + // of memory. + maxsize = utf8_count_codepoints((const unsigned char *)s, (const unsigned char *)end); + if (ch < 0xc4) { // latin1 + maxchr = 0xff; + } + else if (ch < 0xf0) { // ucs2 + maxchr = 0xffff; + } + else { // ucs4 + maxchr = 0x10ffff; + } + } + PyObject *u = PyUnicode_New(maxsize, maxchr); + if (!u) { + return NULL; + } + + // Use _PyUnicodeWriter after fast path is failed. + _PyUnicodeWriter writer; + _PyUnicodeWriter_InitWithBuffer(&writer, u); + if (maxchr <= 255) { + memcpy(PyUnicode_1BYTE_DATA(u), s, pos); + s += pos; + size -= pos; + writer.pos = pos; + } + + if (unicode_decode_utf8_impl(&writer, starts, s, end, + error_handler, errors, + consumed) < 0) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); +} + + +// Used by PyUnicodeWriter_WriteUTF8() implementation +static int +unicode_decode_utf8_writer(_PyUnicodeWriter *writer, + const char *s, Py_ssize_t size, + _Py_error_handler error_handler, const char *errors, + Py_ssize_t *consumed) +{ + if (size == 0) { + if (consumed) { + *consumed = 0; + } + return 0; + } + + // fast path: try ASCII string. + if (_PyUnicodeWriter_Prepare(writer, size, 127) < 0) { + return -1; + } + + const char *starts = s; + const char *end = s + size; + Py_ssize_t decoded = 0; + Py_UCS1 *dest = (Py_UCS1*)writer->data + writer->pos * writer->kind; + if (writer->kind == PyUnicode_1BYTE_KIND) { + decoded = ascii_decode(s, end, dest); + writer->pos += decoded; + + if (decoded == size) { + if (consumed) { + *consumed = size; + } + return 0; + } + s += decoded; + size -= decoded; + } + + return unicode_decode_utf8_impl(writer, starts, s, end, + error_handler, errors, consumed); } @@ -4835,7 +5536,9 @@ PyUnicode_DecodeUTF8Stateful(const char *s, const char *errors, Py_ssize_t *consumed) { - return unicode_decode_utf8(s, size, _Py_ERROR_UNKNOWN, errors, consumed); + return unicode_decode_utf8(s, size, + errors ? _Py_ERROR_UNKNOWN : _Py_ERROR_STRICT, + errors, consumed); } @@ -4980,7 +5683,7 @@ _Py_DecodeUTF8_surrogateescape(const char *arg, Py_ssize_t arglen, } -/* UTF-8 encoder using the surrogateescape error handler . +/* UTF-8 encoder. On success, return 0 and write the newly allocated character string (use PyMem_Free() to free the memory) into *str. @@ -5168,6 +5871,7 @@ unicode_encode_utf8(PyObject *unicode, _Py_error_handler error_handler, static int unicode_fill_utf8(PyObject *unicode) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(unicode); /* the string cannot be ASCII, or PyUnicode_UTF8() would be set */ assert(!PyUnicode_IS_ASCII(unicode)); @@ -5209,10 +5913,10 @@ unicode_fill_utf8(PyObject *unicode) PyErr_NoMemory(); return -1; } - _PyUnicode_UTF8(unicode) = cache; - _PyUnicode_UTF8_LENGTH(unicode) = len; memcpy(cache, start, len); cache[len] = '\0'; + PyUnicode_SET_UTF8_LENGTH(unicode, len); + PyUnicode_SET_UTF8(unicode, cache); _PyBytesWriter_Dealloc(&writer); return 0; } @@ -5884,7 +6588,7 @@ _PyUnicode_GetNameCAPI(void) ucnhash_capi = (_PyUnicode_Name_CAPI *)PyCapsule_Import( PyUnicodeData_CAPSULE_NAME, 1); - // It's fine if we overwite the value here. It's always the same value. + // It's fine if we overwrite the value here. It's always the same value. _Py_atomic_store_ptr(&interp->unicode.ucnhash_capi, ucnhash_capi); } return ucnhash_capi; @@ -6149,7 +6853,8 @@ _PyUnicode_DecodeUnicodeEscapeStateful(const char *s, unsigned char c = *first_invalid_escape; if ('4' <= c && c <= '7') { if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "invalid octal escape sequence '\\%.3s'", + "\"\\%.3s\" is an invalid octal escape sequence. " + "Such sequences will not work in the future. ", first_invalid_escape) < 0) { Py_DECREF(result); @@ -6158,7 +6863,8 @@ _PyUnicode_DecodeUnicodeEscapeStateful(const char *s, } else { if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "invalid escape sequence '\\%c'", + "\"\\%c\" is an invalid escape sequence. " + "Such sequences will not work in the future. ", c) < 0) { Py_DECREF(result); @@ -6695,7 +7401,7 @@ unicode_encode_ucs1(PyObject *unicode, case _Py_ERROR_REPLACE: memset(str, '?', collend - collstart); str += (collend - collstart); - /* fall through */ + _Py_FALLTHROUGH; case _Py_ERROR_IGNORE: pos = collend; break; @@ -6734,7 +7440,7 @@ unicode_encode_ucs1(PyObject *unicode, break; collstart = pos; assert(collstart != collend); - /* fall through */ + _Py_FALLTHROUGH; default: rep = unicode_encode_call_errorhandler(errors, &error_handler_obj, @@ -7795,8 +8501,12 @@ charmap_decode_mapping(const char *s, if (key == NULL) goto onError; - item = PyObject_GetItem(mapping, key); + int rc = PyMapping_GetOptionalItem(mapping, key, &item); Py_DECREF(key); + if (rc == 0) { + /* No mapping found means: mapping is undefined. */ + goto Undefined; + } if (item == NULL) { if (PyErr_ExceptionMatches(PyExc_LookupError)) { /* No mapping found means: mapping is undefined. */ @@ -7810,7 +8520,7 @@ charmap_decode_mapping(const char *s, if (item == Py_None) goto Undefined; if (PyLong_Check(item)) { - long value = PyLong_AS_LONG(item); + long value = PyLong_AsLong(item); if (value == 0xFFFE) goto Undefined; if (value < 0 || value > MAX_UNICODE) { @@ -7960,7 +8670,7 @@ PyUnicode_BuildEncodingMap(PyObject* string) int count2 = 0, count3 = 0; int kind; const void *data; - Py_ssize_t length; + int length; Py_UCS4 ch; if (!PyUnicode_Check(string) || !PyUnicode_GET_LENGTH(string)) { @@ -7969,8 +8679,7 @@ PyUnicode_BuildEncodingMap(PyObject* string) } kind = PyUnicode_KIND(string); data = PyUnicode_DATA(string); - length = PyUnicode_GET_LENGTH(string); - length = Py_MIN(length, 256); + length = (int)Py_MIN(PyUnicode_GET_LENGTH(string), 256); memset(level1, 0xFF, sizeof level1); memset(level2, 0xFF, sizeof level2); @@ -8095,19 +8804,25 @@ encoding_map_lookup(Py_UCS4 c, PyObject *mapping) return i; } -/* Lookup the character ch in the mapping. If the character - can't be found, Py_None is returned (or NULL, if another - error occurred). */ +/* Lookup the character in the mapping. + On success, return PyLong, PyBytes or None (if the character can't be found). + If the result is PyLong, put its value in replace. + On error, return NULL. + */ static PyObject * -charmapencode_lookup(Py_UCS4 c, PyObject *mapping) +charmapencode_lookup(Py_UCS4 c, PyObject *mapping, unsigned char *replace) { PyObject *w = PyLong_FromLong((long)c); PyObject *x; if (w == NULL) return NULL; - x = PyObject_GetItem(mapping, w); + int rc = PyMapping_GetOptionalItem(mapping, w, &x); Py_DECREF(w); + if (rc == 0) { + /* No mapping found means: mapping is undefined. */ + Py_RETURN_NONE; + } if (x == NULL) { if (PyErr_ExceptionMatches(PyExc_LookupError)) { /* No mapping found means: mapping is undefined. */ @@ -8119,13 +8834,14 @@ charmapencode_lookup(Py_UCS4 c, PyObject *mapping) else if (x == Py_None) return x; else if (PyLong_Check(x)) { - long value = PyLong_AS_LONG(x); + long value = PyLong_AsLong(x); if (value < 0 || value > 255) { PyErr_SetString(PyExc_TypeError, "character mapping must be in range(256)"); Py_DECREF(x); return NULL; } + *replace = (unsigned char)value; return x; } else if (PyBytes_Check(x)) @@ -8166,6 +8882,7 @@ charmapencode_output(Py_UCS4 c, PyObject *mapping, PyObject **outobj, Py_ssize_t *outpos) { PyObject *rep; + unsigned char replace; char *outstart; Py_ssize_t outsize = PyBytes_GET_SIZE(*outobj); @@ -8182,7 +8899,7 @@ charmapencode_output(Py_UCS4 c, PyObject *mapping, return enc_SUCCESS; } - rep = charmapencode_lookup(c, mapping); + rep = charmapencode_lookup(c, mapping, &replace); if (rep==NULL) return enc_EXCEPTION; else if (rep==Py_None) { @@ -8197,7 +8914,7 @@ charmapencode_output(Py_UCS4 c, PyObject *mapping, return enc_EXCEPTION; } outstart = PyBytes_AS_STRING(*outobj); - outstart[(*outpos)++] = (char)PyLong_AS_LONG(rep); + outstart[(*outpos)++] = (char)replace; } else { const char *repchars = PyBytes_AS_STRING(rep); @@ -8246,6 +8963,7 @@ charmap_encoding_error( /* find all unencodable characters */ while (collendpos < size) { PyObject *rep; + unsigned char replace; if (Py_IS_TYPE(mapping, &EncodingMapType)) { ch = PyUnicode_READ_CHAR(unicode, collendpos); val = encoding_map_lookup(ch, mapping); @@ -8256,7 +8974,7 @@ charmap_encoding_error( } ch = PyUnicode_READ_CHAR(unicode, collendpos); - rep = charmapencode_lookup(ch, mapping); + rep = charmapencode_lookup(ch, mapping, &replace); if (rep==NULL) return -1; else if (rep!=Py_None) { @@ -8287,7 +9005,7 @@ charmap_encoding_error( return -1; } } - /* fall through */ + _Py_FALLTHROUGH; case _Py_ERROR_IGNORE: *inpos = collendpos; break; @@ -8521,17 +9239,24 @@ unicode_translate_call_errorhandler(const char *errors, /* Lookup the character ch in the mapping and put the result in result, which must be decrefed by the caller. + The result can be PyLong, PyUnicode, None or NULL. + If the result is PyLong, put its value in replace. Return 0 on success, -1 on error */ static int -charmaptranslate_lookup(Py_UCS4 c, PyObject *mapping, PyObject **result) +charmaptranslate_lookup(Py_UCS4 c, PyObject *mapping, PyObject **result, Py_UCS4 *replace) { PyObject *w = PyLong_FromLong((long)c); PyObject *x; if (w == NULL) return -1; - x = PyObject_GetItem(mapping, w); + int rc = PyMapping_GetOptionalItem(mapping, w, &x); Py_DECREF(w); + if (rc == 0) { + /* No mapping found means: use 1:1 mapping. */ + *result = NULL; + return 0; + } if (x == NULL) { if (PyErr_ExceptionMatches(PyExc_LookupError)) { /* No mapping found means: use 1:1 mapping. */ @@ -8546,7 +9271,7 @@ charmaptranslate_lookup(Py_UCS4 c, PyObject *mapping, PyObject **result) return 0; } else if (PyLong_Check(x)) { - long value = PyLong_AS_LONG(x); + long value = PyLong_AsLong(x); if (value < 0 || value > MAX_UNICODE) { PyErr_Format(PyExc_ValueError, "character mapping must be in range(0x%x)", @@ -8555,6 +9280,7 @@ charmaptranslate_lookup(Py_UCS4 c, PyObject *mapping, PyObject **result) return -1; } *result = x; + *replace = (Py_UCS4)value; return 0; } else if (PyUnicode_Check(x)) { @@ -8578,8 +9304,9 @@ charmaptranslate_output(Py_UCS4 ch, PyObject *mapping, _PyUnicodeWriter *writer) { PyObject *item; + Py_UCS4 replace; - if (charmaptranslate_lookup(ch, mapping, &item)) + if (charmaptranslate_lookup(ch, mapping, &item, &replace)) return -1; if (item == NULL) { @@ -8596,10 +9323,7 @@ charmaptranslate_output(Py_UCS4 ch, PyObject *mapping, } if (PyLong_Check(item)) { - long ch = (Py_UCS4)PyLong_AS_LONG(item); - /* PyLong_AS_LONG() cannot fail, charmaptranslate_lookup() already - used it */ - if (_PyUnicodeWriter_WriteCharInline(writer, ch) < 0) { + if (_PyUnicodeWriter_WriteCharInline(writer, replace) < 0) { Py_DECREF(item); return -1; } @@ -8626,9 +9350,10 @@ unicode_fast_translate_lookup(PyObject *mapping, Py_UCS1 ch, Py_UCS1 *translate) { PyObject *item = NULL; + Py_UCS4 replace; int ret = 0; - if (charmaptranslate_lookup(ch, mapping, &item)) { + if (charmaptranslate_lookup(ch, mapping, &item, &replace)) { return -1; } @@ -8642,10 +9367,7 @@ unicode_fast_translate_lookup(PyObject *mapping, Py_UCS1 ch, return 1; } else if (PyLong_Check(item)) { - long replace = PyLong_AS_LONG(item); - /* PyLong_AS_LONG() cannot fail, charmaptranslate_lookup() already - used it */ - if (127 < replace) { + if (replace > 127) { /* invalid character or character outside ASCII: skip the fast translate */ goto exit; @@ -8653,8 +9375,6 @@ unicode_fast_translate_lookup(PyObject *mapping, Py_UCS1 ch, translate[ch] = (Py_UCS1)replace; } else if (PyUnicode_Check(item)) { - Py_UCS4 replace; - if (PyUnicode_GET_LENGTH(item) != 1) goto exit; @@ -8807,8 +9527,9 @@ _PyUnicode_TranslateCharmap(PyObject *input, /* find all untranslatable characters */ while (collend < size) { PyObject *x; + Py_UCS4 replace; ch = PyUnicode_READ(kind, data, collend); - if (charmaptranslate_lookup(ch, mapping, &x)) + if (charmaptranslate_lookup(ch, mapping, &x, &replace)) goto onError; Py_XDECREF(x); if (x != Py_None) @@ -8903,19 +9624,24 @@ _PyUnicode_TransformDecimalAndSpaceToASCII(PyObject *unicode) /* --- Helpers ------------------------------------------------------------ */ /* helper macro to fixup start/end slice values */ -#define ADJUST_INDICES(start, end, len) \ - if (end > len) \ - end = len; \ - else if (end < 0) { \ - end += len; \ - if (end < 0) \ - end = 0; \ - } \ - if (start < 0) { \ - start += len; \ - if (start < 0) \ - start = 0; \ - } +#define ADJUST_INDICES(start, end, len) \ + do { \ + if (end > len) { \ + end = len; \ + } \ + else if (end < 0) { \ + end += len; \ + if (end < 0) { \ + end = 0; \ + } \ + } \ + if (start < 0) { \ + start += len; \ + if (start < 0) { \ + start = 0; \ + } \ + } \ + } while (0) static Py_ssize_t any_find_slice(PyObject* s1, PyObject* s2, @@ -9136,75 +9862,6 @@ _PyUnicode_InsertThousandsGrouping( return count; } -static Py_ssize_t -unicode_count_impl(PyObject *str, - PyObject *substr, - Py_ssize_t start, - Py_ssize_t end) -{ - assert(PyUnicode_Check(str)); - assert(PyUnicode_Check(substr)); - - Py_ssize_t result; - int kind1, kind2; - const void *buf1 = NULL, *buf2 = NULL; - Py_ssize_t len1, len2; - - kind1 = PyUnicode_KIND(str); - kind2 = PyUnicode_KIND(substr); - if (kind1 < kind2) - return 0; - - len1 = PyUnicode_GET_LENGTH(str); - len2 = PyUnicode_GET_LENGTH(substr); - ADJUST_INDICES(start, end, len1); - if (end - start < len2) - return 0; - - buf1 = PyUnicode_DATA(str); - buf2 = PyUnicode_DATA(substr); - if (kind2 != kind1) { - buf2 = unicode_askind(kind2, buf2, len2, kind1); - if (!buf2) - goto onError; - } - - // We don't reuse `anylib_count` here because of the explicit casts. - switch (kind1) { - case PyUnicode_1BYTE_KIND: - result = ucs1lib_count( - ((const Py_UCS1*)buf1) + start, end - start, - buf2, len2, PY_SSIZE_T_MAX - ); - break; - case PyUnicode_2BYTE_KIND: - result = ucs2lib_count( - ((const Py_UCS2*)buf1) + start, end - start, - buf2, len2, PY_SSIZE_T_MAX - ); - break; - case PyUnicode_4BYTE_KIND: - result = ucs4lib_count( - ((const Py_UCS4*)buf1) + start, end - start, - buf2, len2, PY_SSIZE_T_MAX - ); - break; - default: - Py_UNREACHABLE(); - } - - assert((kind2 != kind1) == (buf2 != PyUnicode_DATA(substr))); - if (kind2 != kind1) - PyMem_Free((void *)buf2); - - return result; - onError: - assert((kind2 != kind1) == (buf2 != PyUnicode_DATA(substr))); - if (kind2 != kind1) - PyMem_Free((void *)buf2); - return -1; -} - Py_ssize_t PyUnicode_Count(PyObject *str, PyObject *substr, @@ -9571,13 +10228,14 @@ PyUnicode_Join(PyObject *separator, PyObject *seq) return NULL; } - /* NOTE: the following code can't call back into Python code, - * so we are sure that fseq won't be mutated. - */ + Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(seq); items = PySequence_Fast_ITEMS(fseq); seqlen = PySequence_Fast_GET_SIZE(fseq); res = _PyUnicode_JoinArray(separator, items, seqlen); + + Py_END_CRITICAL_SECTION_SEQUENCE_FAST(); + Py_DECREF(fseq); return res; } @@ -10611,26 +11269,6 @@ unicode_compare(PyObject *str1, PyObject *str2) #undef COMPARE } -static int -unicode_compare_eq(PyObject *str1, PyObject *str2) -{ - int kind; - const void *data1, *data2; - Py_ssize_t len; - int cmp; - - len = PyUnicode_GET_LENGTH(str1); - if (PyUnicode_GET_LENGTH(str2) != len) - return 0; - kind = PyUnicode_KIND(str1); - if (PyUnicode_KIND(str2) != kind) - return 0; - data1 = PyUnicode_DATA(str1); - data2 = PyUnicode_DATA(str2); - - cmp = memcmp(data1, data2, len * kind); - return (cmp == 0); -} int _PyUnicode_Equal(PyObject *str1, PyObject *str2) @@ -10640,7 +11278,25 @@ _PyUnicode_Equal(PyObject *str1, PyObject *str2) if (str1 == str2) { return 1; } - return unicode_compare_eq(str1, str2); + return unicode_eq(str1, str2); +} + + +int +PyUnicode_Equal(PyObject *str1, PyObject *str2) +{ + if (!PyUnicode_Check(str1)) { + PyErr_Format(PyExc_TypeError, + "first argument must be str, not %T", str1); + return -1; + } + if (!PyUnicode_Check(str2)) { + PyErr_Format(PyExc_TypeError, + "second argument must be str, not %T", str2); + return -1; + } + + return _PyUnicode_Equal(str1, str2); } @@ -10826,16 +11482,19 @@ _PyUnicode_EqualToASCIIId(PyObject *left, _Py_Identifier *right) if (left == right_uni) return 1; - if (PyUnicode_CHECK_INTERNED(left)) + assert(PyUnicode_CHECK_INTERNED(right_uni)); + if (PyUnicode_CHECK_INTERNED(left)) { return 0; + } - assert(_PyUnicode_HASH(right_uni) != -1); - Py_hash_t hash = _PyUnicode_HASH(left); - if (hash != -1 && hash != _PyUnicode_HASH(right_uni)) { + Py_hash_t right_hash = PyUnicode_HASH(right_uni); + assert(right_hash != -1); + Py_hash_t hash = PyUnicode_HASH(left); + if (hash != -1 && hash != right_hash) { return 0; } - return unicode_compare_eq(left, right_uni); + return unicode_eq(left, right_uni); } PyObject * @@ -10863,7 +11522,7 @@ PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) } } else if (op == Py_EQ || op == Py_NE) { - result = unicode_compare_eq(left, right); + result = unicode_eq(left, right); result ^= (op == Py_NE); return PyBool_FromLong(result); } @@ -10873,12 +11532,6 @@ PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) } } -int -_PyUnicode_EQ(PyObject *aa, PyObject *bb) -{ - return unicode_eq(aa, bb); -} - int PyUnicode_Contains(PyObject *str, PyObject *substr) { @@ -11073,47 +11726,87 @@ PyUnicode_AppendAndDel(PyObject **pleft, PyObject *right) Py_XDECREF(right); } -/* -Wraps asciilib_parse_args_finds() and additionally ensures that the -first argument is a unicode object. -*/ +/*[clinic input] +@text_signature "($self, sub[, start[, end]], /)" +str.count as unicode_count -> Py_ssize_t -static inline int -parse_args_finds_unicode(const char * function_name, PyObject *args, - PyObject **substring, - Py_ssize_t *start, Py_ssize_t *end) -{ - if (asciilib_parse_args_finds(function_name, args, substring, start, end)) { - if (ensure_unicode(*substring) < 0) - return 0; - return 1; - } - return 0; -} + self as str: self + sub as substr: unicode + start: slice_index(accept={int, NoneType}, c_default='0') = None + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + / -PyDoc_STRVAR(count__doc__, - "S.count(sub[, start[, end]]) -> int\n\ -\n\ -Return the number of non-overlapping occurrences of substring sub in\n\ -string S[start:end]. Optional arguments start and end are\n\ -interpreted as in slice notation."); +Return the number of non-overlapping occurrences of substring sub in string S[start:end]. -static PyObject * -unicode_count(PyObject *self, PyObject *args) +Optional arguments start and end are interpreted as in slice notation. +[clinic start generated code]*/ + +static Py_ssize_t +unicode_count_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=8fcc3aef0b18edbf input=6f168ffd94be8785]*/ { - PyObject *substring = NULL; /* initialize to fix a compiler warning */ - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; + assert(PyUnicode_Check(str)); + assert(PyUnicode_Check(substr)); + Py_ssize_t result; + int kind1, kind2; + const void *buf1 = NULL, *buf2 = NULL; + Py_ssize_t len1, len2; - if (!parse_args_finds_unicode("count", args, &substring, &start, &end)) - return NULL; + kind1 = PyUnicode_KIND(str); + kind2 = PyUnicode_KIND(substr); + if (kind1 < kind2) + return 0; - result = unicode_count_impl(self, substring, start, end); - if (result == -1) - return NULL; + len1 = PyUnicode_GET_LENGTH(str); + len2 = PyUnicode_GET_LENGTH(substr); + ADJUST_INDICES(start, end, len1); + if (end - start < len2) + return 0; + + buf1 = PyUnicode_DATA(str); + buf2 = PyUnicode_DATA(substr); + if (kind2 != kind1) { + buf2 = unicode_askind(kind2, buf2, len2, kind1); + if (!buf2) + goto onError; + } + + // We don't reuse `anylib_count` here because of the explicit casts. + switch (kind1) { + case PyUnicode_1BYTE_KIND: + result = ucs1lib_count( + ((const Py_UCS1*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + case PyUnicode_2BYTE_KIND: + result = ucs2lib_count( + ((const Py_UCS2*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + case PyUnicode_4BYTE_KIND: + result = ucs4lib_count( + ((const Py_UCS4*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + default: + Py_UNREACHABLE(); + } + + assert((kind2 != kind1) == (buf2 != PyUnicode_DATA(substr))); + if (kind2 != kind1) + PyMem_Free((void *)buf2); - return PyLong_FromSsize_t(result); + return result; + onError: + assert((kind2 != kind1) == (buf2 != PyUnicode_DATA(substr))); + if (kind2 != kind1) + PyMem_Free((void *)buf2); + return -1; } /*[clinic input] @@ -11224,33 +11917,25 @@ unicode_expandtabs_impl(PyObject *self, int tabsize) return NULL; } -PyDoc_STRVAR(find__doc__, - "S.find(sub[, start[, end]]) -> int\n\ -\n\ -Return the lowest index in S where substring sub is found,\n\ -such that sub is contained within S[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Return -1 on failure."); - -static PyObject * -unicode_find(PyObject *self, PyObject *args) -{ - /* initialize variables to prevent gcc warning */ - PyObject *substring = NULL; - Py_ssize_t start = 0; - Py_ssize_t end = 0; - Py_ssize_t result; - - if (!parse_args_finds_unicode("find", args, &substring, &start, &end)) - return NULL; +/*[clinic input] +str.find as unicode_find = str.count - result = any_find_slice(self, substring, start, end, 1); +Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end]. - if (result == -2) - return NULL; +Optional arguments start and end are interpreted as in slice notation. +Return -1 on failure. +[clinic start generated code]*/ - return PyLong_FromSsize_t(result); +static Py_ssize_t +unicode_find_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=51dbe6255712e278 input=4a89d2d68ef57256]*/ +{ + Py_ssize_t result = any_find_slice(str, substr, start, end, 1); + if (result < 0) { + return -1; + } + return result; } static PyObject * @@ -11284,47 +11969,39 @@ unicode_hash(PyObject *self) #ifdef Py_DEBUG assert(_Py_HashSecret_Initialized); #endif - if (_PyUnicode_HASH(self) != -1) - return _PyUnicode_HASH(self); - - x = _Py_HashBytes(PyUnicode_DATA(self), + Py_hash_t hash = PyUnicode_HASH(self); + if (hash != -1) { + return hash; + } + x = Py_HashBuffer(PyUnicode_DATA(self), PyUnicode_GET_LENGTH(self) * PyUnicode_KIND(self)); - _PyUnicode_HASH(self) = x; + + PyUnicode_SET_HASH(self, x); return x; } -PyDoc_STRVAR(index__doc__, - "S.index(sub[, start[, end]]) -> int\n\ -\n\ -Return the lowest index in S where substring sub is found,\n\ -such that sub is contained within S[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Raises ValueError when the substring is not found."); - -static PyObject * -unicode_index(PyObject *self, PyObject *args) -{ - /* initialize variables to prevent gcc warning */ - Py_ssize_t result; - PyObject *substring = NULL; - Py_ssize_t start = 0; - Py_ssize_t end = 0; - - if (!parse_args_finds_unicode("index", args, &substring, &start, &end)) - return NULL; +/*[clinic input] +str.index as unicode_index = str.count - result = any_find_slice(self, substring, start, end, 1); +Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end]. - if (result == -2) - return NULL; +Optional arguments start and end are interpreted as in slice notation. +Raises ValueError when the substring is not found. +[clinic start generated code]*/ - if (result < 0) { +static Py_ssize_t +unicode_index_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=77558288837cdf40 input=d986aeac0be14a1c]*/ +{ + Py_ssize_t result = any_find_slice(str, substr, start, end, 1); + if (result == -1) { PyErr_SetString(PyExc_ValueError, "substring not found"); - return NULL; } - - return PyLong_FromSsize_t(result); + else if (result < 0) { + return -1; + } + return result; } /*[clinic input] @@ -12245,24 +12922,17 @@ unicode_removesuffix_impl(PyObject *self, PyObject *suffix) static PyObject * unicode_repr(PyObject *unicode) { - PyObject *repr; - Py_ssize_t isize; - Py_ssize_t osize, squote, dquote, i, o; - Py_UCS4 max, quote; - int ikind, okind, unchanged; - const void *idata; - void *odata; - - isize = PyUnicode_GET_LENGTH(unicode); - idata = PyUnicode_DATA(unicode); + Py_ssize_t isize = PyUnicode_GET_LENGTH(unicode); + const void *idata = PyUnicode_DATA(unicode); /* Compute length of output, quote characters, and maximum character */ - osize = 0; - max = 127; - squote = dquote = 0; - ikind = PyUnicode_KIND(unicode); - for (i = 0; i < isize; i++) { + Py_ssize_t osize = 0; + Py_UCS4 maxch = 127; + Py_ssize_t squote = 0; + Py_ssize_t dquote = 0; + int ikind = PyUnicode_KIND(unicode); + for (Py_ssize_t i = 0; i < isize; i++) { Py_UCS4 ch = PyUnicode_READ(ikind, idata, i); Py_ssize_t incr = 1; switch (ch) { @@ -12278,7 +12948,7 @@ unicode_repr(PyObject *unicode) else if (ch < 0x7f) ; else if (Py_UNICODE_ISPRINTABLE(ch)) - max = ch > max ? ch : max; + maxch = (ch > maxch) ? ch : maxch; else if (ch < 0x100) incr = 4; /* \xHH */ else if (ch < 0x10000) @@ -12294,10 +12964,10 @@ unicode_repr(PyObject *unicode) osize += incr; } - quote = '\''; - unchanged = (osize == isize); + Py_UCS4 quote = '\''; + int changed = (osize != isize); if (squote) { - unchanged = 0; + changed = 1; if (dquote) /* Both squote and dquote present. Use squote, and escape them */ @@ -12307,164 +12977,82 @@ unicode_repr(PyObject *unicode) } osize += 2; /* quotes */ - repr = PyUnicode_New(osize, max); + PyObject *repr = PyUnicode_New(osize, maxch); if (repr == NULL) return NULL; - okind = PyUnicode_KIND(repr); - odata = PyUnicode_DATA(repr); + int okind = PyUnicode_KIND(repr); + void *odata = PyUnicode_DATA(repr); + + if (!changed) { + PyUnicode_WRITE(okind, odata, 0, quote); - PyUnicode_WRITE(okind, odata, 0, quote); - PyUnicode_WRITE(okind, odata, osize-1, quote); - if (unchanged) { _PyUnicode_FastCopyCharacters(repr, 1, unicode, 0, isize); + + PyUnicode_WRITE(okind, odata, osize-1, quote); } else { - for (i = 0, o = 1; i < isize; i++) { - Py_UCS4 ch = PyUnicode_READ(ikind, idata, i); - - /* Escape quotes and backslashes */ - if ((ch == quote) || (ch == '\\')) { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, ch); - continue; - } - - /* Map special whitespace to '\t', \n', '\r' */ - if (ch == '\t') { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, 't'); - } - else if (ch == '\n') { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, 'n'); - } - else if (ch == '\r') { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, 'r'); - } - - /* Map non-printable US ASCII to '\xhh' */ - else if (ch < ' ' || ch == 0x7F) { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, 'x'); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0x000F]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0x000F]); - } - - /* Copy ASCII characters as-is */ - else if (ch < 0x7F) { - PyUnicode_WRITE(okind, odata, o++, ch); - } - - /* Non-ASCII characters */ - else { - /* Map Unicode whitespace and control characters - (categories Z* and C* except ASCII space) - */ - if (!Py_UNICODE_ISPRINTABLE(ch)) { - PyUnicode_WRITE(okind, odata, o++, '\\'); - /* Map 8-bit characters to '\xhh' */ - if (ch <= 0xff) { - PyUnicode_WRITE(okind, odata, o++, 'x'); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0x000F]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0x000F]); - } - /* Map 16-bit characters to '\uxxxx' */ - else if (ch <= 0xffff) { - PyUnicode_WRITE(okind, odata, o++, 'u'); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 12) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 8) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0xF]); - } - /* Map 21-bit characters to '\U00xxxxxx' */ - else { - PyUnicode_WRITE(okind, odata, o++, 'U'); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 28) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 24) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 20) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 16) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 12) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 8) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0xF]); - } - } - /* Copy characters as-is */ - else { - PyUnicode_WRITE(okind, odata, o++, ch); - } - } + switch (okind) { + case PyUnicode_1BYTE_KIND: + ucs1lib_repr(unicode, quote, odata); + break; + case PyUnicode_2BYTE_KIND: + ucs2lib_repr(unicode, quote, odata); + break; + default: + assert(okind == PyUnicode_4BYTE_KIND); + ucs4lib_repr(unicode, quote, odata); } } - /* Closing quote already added at the beginning */ + assert(_PyUnicode_CheckConsistency(repr, 1)); return repr; } -PyDoc_STRVAR(rfind__doc__, - "S.rfind(sub[, start[, end]]) -> int\n\ -\n\ -Return the highest index in S where substring sub is found,\n\ -such that sub is contained within S[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Return -1 on failure."); - -static PyObject * -unicode_rfind(PyObject *self, PyObject *args) -{ - /* initialize variables to prevent gcc warning */ - PyObject *substring = NULL; - Py_ssize_t start = 0; - Py_ssize_t end = 0; - Py_ssize_t result; - - if (!parse_args_finds_unicode("rfind", args, &substring, &start, &end)) - return NULL; - - result = any_find_slice(self, substring, start, end, -1); - - if (result == -2) - return NULL; +/*[clinic input] +str.rfind as unicode_rfind = str.count - return PyLong_FromSsize_t(result); -} +Return the highest index in S where substring sub is found, such that sub is contained within S[start:end]. -PyDoc_STRVAR(rindex__doc__, - "S.rindex(sub[, start[, end]]) -> int\n\ -\n\ -Return the highest index in S where substring sub is found,\n\ -such that sub is contained within S[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Raises ValueError when the substring is not found."); +Optional arguments start and end are interpreted as in slice notation. +Return -1 on failure. +[clinic start generated code]*/ -static PyObject * -unicode_rindex(PyObject *self, PyObject *args) +static Py_ssize_t +unicode_rfind_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=880b29f01dd014c8 input=898361fb71f59294]*/ { - /* initialize variables to prevent gcc warning */ - PyObject *substring = NULL; - Py_ssize_t start = 0; - Py_ssize_t end = 0; - Py_ssize_t result; + Py_ssize_t result = any_find_slice(str, substr, start, end, -1); + if (result < 0) { + return -1; + } + return result; +} - if (!parse_args_finds_unicode("rindex", args, &substring, &start, &end)) - return NULL; +/*[clinic input] +str.rindex as unicode_rindex = str.count - result = any_find_slice(self, substring, start, end, -1); +Return the highest index in S where substring sub is found, such that sub is contained within S[start:end]. - if (result == -2) - return NULL; +Optional arguments start and end are interpreted as in slice notation. +Raises ValueError when the substring is not found. +[clinic start generated code]*/ - if (result < 0) { +static Py_ssize_t +unicode_rindex_impl(PyObject *str, PyObject *substr, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=5f3aef124c867fe1 input=35943dead6c1ea9d]*/ +{ + Py_ssize_t result = any_find_slice(str, substr, start, end, -1); + if (result == -1) { PyErr_SetString(PyExc_ValueError, "substring not found"); - return NULL; } - - return PyLong_FromSsize_t(result); + else if (result < 0) { + return -1; + } + return result; } /*[clinic input] @@ -12963,30 +13551,30 @@ unicode_zfill_impl(PyObject *self, Py_ssize_t width) return u; } -PyDoc_STRVAR(startswith__doc__, - "S.startswith(prefix[, start[, end]]) -> bool\n\ -\n\ -Return True if S starts with the specified prefix, False otherwise.\n\ -With optional start, test S beginning at that position.\n\ -With optional end, stop comparing S at that position.\n\ -prefix can also be a tuple of strings to try."); +/*[clinic input] +@text_signature "($self, prefix[, start[, end]], /)" +str.startswith as unicode_startswith + + prefix as subobj: object + A string or a tuple of strings to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the string. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the string. + / + +Return True if the string starts with the specified prefix, False otherwise. +[clinic start generated code]*/ static PyObject * -unicode_startswith(PyObject *self, - PyObject *args) +unicode_startswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=4bd7cfd0803051d4 input=5f918b5f5f89d856]*/ { - PyObject *subobj; - PyObject *substring; - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; - int result; - - if (!asciilib_parse_args_finds("startswith", args, &subobj, &start, &end)) - return NULL; if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - substring = PyTuple_GET_ITEM(subobj, i); + PyObject *substring = PyTuple_GET_ITEM(subobj, i); if (!PyUnicode_Check(substring)) { PyErr_Format(PyExc_TypeError, "tuple for startswith must only contain str, " @@ -12994,9 +13582,10 @@ unicode_startswith(PyObject *self, Py_TYPE(substring)->tp_name); return NULL; } - result = tailmatch(self, substring, start, end, -1); - if (result == -1) + int result = tailmatch(self, substring, start, end, -1); + if (result < 0) { return NULL; + } if (result) { Py_RETURN_TRUE; } @@ -13010,37 +13599,38 @@ unicode_startswith(PyObject *self, "a tuple of str, not %.100s", Py_TYPE(subobj)->tp_name); return NULL; } - result = tailmatch(self, subobj, start, end, -1); - if (result == -1) + int result = tailmatch(self, subobj, start, end, -1); + if (result < 0) { return NULL; + } return PyBool_FromLong(result); } -PyDoc_STRVAR(endswith__doc__, - "S.endswith(suffix[, start[, end]]) -> bool\n\ -\n\ -Return True if S ends with the specified suffix, False otherwise.\n\ -With optional start, test S beginning at that position.\n\ -With optional end, stop comparing S at that position.\n\ -suffix can also be a tuple of strings to try."); +/*[clinic input] +@text_signature "($self, suffix[, start[, end]], /)" +str.endswith as unicode_endswith + + suffix as subobj: object + A string or a tuple of strings to try. + start: slice_index(accept={int, NoneType}, c_default='0') = None + Optional start position. Default: start of the string. + end: slice_index(accept={int, NoneType}, c_default='PY_SSIZE_T_MAX') = None + Optional stop position. Default: end of the string. + / + +Return True if the string ends with the specified suffix, False otherwise. +[clinic start generated code]*/ static PyObject * -unicode_endswith(PyObject *self, - PyObject *args) +unicode_endswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, + Py_ssize_t end) +/*[clinic end generated code: output=cce6f8ceb0102ca9 input=00fbdc774a7d4d71]*/ { - PyObject *subobj; - PyObject *substring; - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; - int result; - - if (!asciilib_parse_args_finds("endswith", args, &subobj, &start, &end)) - return NULL; if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - substring = PyTuple_GET_ITEM(subobj, i); + PyObject *substring = PyTuple_GET_ITEM(subobj, i); if (!PyUnicode_Check(substring)) { PyErr_Format(PyExc_TypeError, "tuple for endswith must only contain str, " @@ -13048,9 +13638,10 @@ unicode_endswith(PyObject *self, Py_TYPE(substring)->tp_name); return NULL; } - result = tailmatch(self, substring, start, end, +1); - if (result == -1) + int result = tailmatch(self, substring, start, end, +1); + if (result < 0) { return NULL; + } if (result) { Py_RETURN_TRUE; } @@ -13063,12 +13654,14 @@ unicode_endswith(PyObject *self, "a tuple of str, not %.100s", Py_TYPE(subobj)->tp_name); return NULL; } - result = tailmatch(self, subobj, start, end, +1); - if (result == -1) + int result = tailmatch(self, subobj, start, end, +1); + if (result < 0) { return NULL; + } return PyBool_FromLong(result); } + static inline void _PyUnicodeWriter_Update(_PyUnicodeWriter *writer) { @@ -13092,6 +13685,7 @@ _PyUnicodeWriter_Update(_PyUnicodeWriter *writer) } } + void _PyUnicodeWriter_Init(_PyUnicodeWriter *writer) { @@ -13100,12 +13694,54 @@ _PyUnicodeWriter_Init(_PyUnicodeWriter *writer) /* ASCII is the bare minimum */ writer->min_char = 127; - /* use a value smaller than PyUnicode_1BYTE_KIND() so + /* use a kind value smaller than PyUnicode_1BYTE_KIND so _PyUnicodeWriter_PrepareKind() will copy the buffer. */ - writer->kind = 0; - assert(writer->kind <= PyUnicode_1BYTE_KIND); + assert(writer->kind == 0); + assert(writer->kind < PyUnicode_1BYTE_KIND); +} + + +PyUnicodeWriter* +PyUnicodeWriter_Create(Py_ssize_t length) +{ + if (length < 0) { + PyErr_SetString(PyExc_ValueError, + "length must be positive"); + return NULL; + } + + const size_t size = sizeof(_PyUnicodeWriter); + PyUnicodeWriter *pub_writer; + pub_writer = _Py_FREELIST_POP_MEM(unicode_writers); + if (pub_writer == NULL) { + pub_writer = (PyUnicodeWriter *)PyMem_Malloc(size); + if (pub_writer == NULL) { + return (PyUnicodeWriter *)PyErr_NoMemory(); + } + } + _PyUnicodeWriter *writer = (_PyUnicodeWriter *)pub_writer; + + _PyUnicodeWriter_Init(writer); + if (_PyUnicodeWriter_Prepare(writer, length, 127) < 0) { + PyUnicodeWriter_Discard(pub_writer); + return NULL; + } + writer->overallocate = 1; + + return pub_writer; +} + + +void PyUnicodeWriter_Discard(PyUnicodeWriter *writer) +{ + if (writer == NULL) { + return; + } + _PyUnicodeWriter_Dealloc((_PyUnicodeWriter*)writer); + _Py_FREELIST_FREE(unicode_writers, writer, PyMem_Free); } + // Initialize _PyUnicodeWriter with initial buffer static inline void _PyUnicodeWriter_InitWithBuffer(_PyUnicodeWriter *writer, PyObject *buffer) @@ -13116,6 +13752,7 @@ _PyUnicodeWriter_InitWithBuffer(_PyUnicodeWriter *writer, PyObject *buffer) writer->min_length = writer->size; } + int _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, Py_ssize_t length, Py_UCS4 maxchar) @@ -13123,6 +13760,7 @@ _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, Py_ssize_t newlen; PyObject *newbuffer; + assert(length >= 0); assert(maxchar <= MAX_UNICODE); /* ensure that the _PyUnicodeWriter_Prepare macro was used */ @@ -13231,9 +13869,23 @@ _PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer, Py_UCS4 ch) return _PyUnicodeWriter_WriteCharInline(writer, ch); } +int +PyUnicodeWriter_WriteChar(PyUnicodeWriter *writer, Py_UCS4 ch) +{ + if (ch > MAX_UNICODE) { + PyErr_SetString(PyExc_ValueError, + "character must be in range(0x110000)"); + return -1; + } + + return _PyUnicodeWriter_WriteChar((_PyUnicodeWriter*)writer, ch); +} + int _PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str) { + assert(PyUnicode_Check(str)); + Py_UCS4 maxchar; Py_ssize_t len; @@ -13259,31 +13911,68 @@ _PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str) return 0; } +int +PyUnicodeWriter_WriteStr(PyUnicodeWriter *writer, PyObject *obj) +{ + if (Py_TYPE(obj) == &PyLong_Type) { + return _PyLong_FormatWriter((_PyUnicodeWriter*)writer, obj, 10, 0); + } + + PyObject *str = PyObject_Str(obj); + if (str == NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str); + Py_DECREF(str); + return res; +} + + +int +PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) +{ + if (Py_TYPE(obj) == &PyLong_Type) { + return _PyLong_FormatWriter((_PyUnicodeWriter*)writer, obj, 10, 0); + } + + PyObject *repr = PyObject_Repr(obj); + if (repr == NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, repr); + Py_DECREF(repr); + return res; +} + + int _PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer, PyObject *str, Py_ssize_t start, Py_ssize_t end) { - Py_UCS4 maxchar; - Py_ssize_t len; - assert(0 <= start); assert(end <= PyUnicode_GET_LENGTH(str)); assert(start <= end); - if (end == 0) - return 0; - if (start == 0 && end == PyUnicode_GET_LENGTH(str)) return _PyUnicodeWriter_WriteStr(writer, str); - if (PyUnicode_MAX_CHAR_VALUE(str) > writer->maxchar) + Py_ssize_t len = end - start; + if (len == 0) { + return 0; + } + + Py_UCS4 maxchar; + if (PyUnicode_MAX_CHAR_VALUE(str) > writer->maxchar) { maxchar = _PyUnicode_FindMaxChar(str, start, end); - else + } + else { maxchar = writer->maxchar; - len = end - start; - - if (_PyUnicodeWriter_Prepare(writer, len, maxchar) < 0) + } + if (_PyUnicodeWriter_Prepare(writer, len, maxchar) < 0) { return -1; + } _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, str, start, len); @@ -13291,6 +13980,29 @@ _PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer, PyObject *str, return 0; } + +int +PyUnicodeWriter_WriteSubstring(PyUnicodeWriter *writer, PyObject *str, + Py_ssize_t start, Py_ssize_t end) +{ + if (!PyUnicode_Check(str)) { + PyErr_Format(PyExc_TypeError, "expect str, not %T", str); + return -1; + } + if (start < 0 || start > end) { + PyErr_Format(PyExc_ValueError, "invalid start argument"); + return -1; + } + if (end > PyUnicode_GET_LENGTH(str)) { + PyErr_Format(PyExc_ValueError, "invalid end argument"); + return -1; + } + + return _PyUnicodeWriter_WriteSubstring((_PyUnicodeWriter*)writer, str, + start, end); +} + + int _PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, const char *ascii, Py_ssize_t len) @@ -13351,6 +14063,51 @@ _PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, return 0; } +int +PyUnicodeWriter_WriteUTF8(PyUnicodeWriter *writer, + const char *str, + Py_ssize_t size) +{ + if (size < 0) { + size = strlen(str); + } + + _PyUnicodeWriter *_writer = (_PyUnicodeWriter*)writer; + Py_ssize_t old_pos = _writer->pos; + int res = unicode_decode_utf8_writer(_writer, str, size, + _Py_ERROR_STRICT, NULL, NULL); + if (res < 0) { + _writer->pos = old_pos; + } + return res; +} + + +int +PyUnicodeWriter_DecodeUTF8Stateful(PyUnicodeWriter *writer, + const char *string, + Py_ssize_t length, + const char *errors, + Py_ssize_t *consumed) +{ + if (length < 0) { + length = strlen(string); + } + + _PyUnicodeWriter *_writer = (_PyUnicodeWriter*)writer; + Py_ssize_t old_pos = _writer->pos; + int res = unicode_decode_utf8_writer(_writer, string, length, + _Py_ERROR_UNKNOWN, errors, consumed); + if (res < 0) { + _writer->pos = old_pos; + if (consumed) { + *consumed = 0; + } + } + return res; +} + + int _PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer, const char *str, Py_ssize_t len) @@ -13397,6 +14154,17 @@ _PyUnicodeWriter_Finish(_PyUnicodeWriter *writer) return unicode_result(str); } + +PyObject* +PyUnicodeWriter_Finish(PyUnicodeWriter *writer) +{ + PyObject *str = _PyUnicodeWriter_Finish((_PyUnicodeWriter*)writer); + assert(((_PyUnicodeWriter*)writer)->buffer == NULL); + _Py_FREELIST_FREE(unicode_writers, writer, PyMem_Free); + return str; +} + + void _PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer) { @@ -13413,7 +14181,7 @@ Return a formatted version of the string, using substitutions from args and kwar The substitutions are identified by braces ('{' and '}')."); PyDoc_STRVAR(format_map__doc__, - "format_map($self, /, mapping)\n\ + "format_map($self, mapping, /)\n\ --\n\ \n\ Return a formatted version of the string, using substitutions from mapping.\n\ @@ -13500,16 +14268,16 @@ static PyMethodDef unicode_methods[] = { UNICODE_CASEFOLD_METHODDEF UNICODE_TITLE_METHODDEF UNICODE_CENTER_METHODDEF - {"count", (PyCFunction) unicode_count, METH_VARARGS, count__doc__}, + UNICODE_COUNT_METHODDEF UNICODE_EXPANDTABS_METHODDEF - {"find", (PyCFunction) unicode_find, METH_VARARGS, find__doc__}, + UNICODE_FIND_METHODDEF UNICODE_PARTITION_METHODDEF - {"index", (PyCFunction) unicode_index, METH_VARARGS, index__doc__}, + UNICODE_INDEX_METHODDEF UNICODE_LJUST_METHODDEF UNICODE_LOWER_METHODDEF UNICODE_LSTRIP_METHODDEF - {"rfind", (PyCFunction) unicode_rfind, METH_VARARGS, rfind__doc__}, - {"rindex", (PyCFunction) unicode_rindex, METH_VARARGS, rindex__doc__}, + UNICODE_RFIND_METHODDEF + UNICODE_RINDEX_METHODDEF UNICODE_RJUST_METHODDEF UNICODE_RSTRIP_METHODDEF UNICODE_RPARTITION_METHODDEF @@ -13518,8 +14286,8 @@ static PyMethodDef unicode_methods[] = { UNICODE_SWAPCASE_METHODDEF UNICODE_TRANSLATE_METHODDEF UNICODE_UPPER_METHODDEF - {"startswith", (PyCFunction) unicode_startswith, METH_VARARGS, startswith__doc__}, - {"endswith", (PyCFunction) unicode_endswith, METH_VARARGS, endswith__doc__}, + UNICODE_STARTSWITH_METHODDEF + UNICODE_ENDSWITH_METHODDEF UNICODE_REMOVEPREFIX_METHODDEF UNICODE_REMOVESUFFIX_METHODDEF UNICODE_ISASCII_METHODDEF @@ -13980,14 +14748,21 @@ formatchar(PyObject *v) if (PyUnicode_GET_LENGTH(v) == 1) { return PyUnicode_READ_CHAR(v, 0); } - goto onError; + PyErr_Format(PyExc_TypeError, + "%%c requires an int or a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(v)); + return (Py_UCS4) -1; } else { int overflow; long x = PyLong_AsLongAndOverflow(v, &overflow); if (x == -1 && PyErr_Occurred()) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { - goto onError; + PyErr_Format(PyExc_TypeError, + "%%c requires an int or a unicode character, not %T", + v); + return (Py_UCS4) -1; } return (Py_UCS4) -1; } @@ -14001,11 +14776,6 @@ formatchar(PyObject *v) return (Py_UCS4) x; } - - onError: - PyErr_SetString(PyExc_TypeError, - "%c requires int or char"); - return (Py_UCS4) -1; } /* Parse options of an argument: flags, width, precision. @@ -14621,6 +15391,65 @@ unicode_new_impl(PyTypeObject *type, PyObject *x, const char *encoding, return unicode; } +static const char * +arg_as_utf8(PyObject *obj, const char *name) +{ + if (!PyUnicode_Check(obj)) { + PyErr_Format(PyExc_TypeError, + "str() argument '%s' must be str, not %T", + name, obj); + return NULL; + } + return _PyUnicode_AsUTF8NoNUL(obj); +} + +static PyObject * +unicode_vectorcall(PyObject *type, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + assert(Py_Is(_PyType_CAST(type), &PyUnicode_Type)); + + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + if (kwnames != NULL && PyTuple_GET_SIZE(kwnames) != 0) { + // Fallback to unicode_new() + PyObject *tuple = _PyTuple_FromArray(args, nargs); + if (tuple == NULL) { + return NULL; + } + PyObject *dict = _PyStack_AsDict(args + nargs, kwnames); + if (dict == NULL) { + Py_DECREF(tuple); + return NULL; + } + PyObject *ret = unicode_new(_PyType_CAST(type), tuple, dict); + Py_DECREF(tuple); + Py_DECREF(dict); + return ret; + } + if (!_PyArg_CheckPositional("str", nargs, 0, 3)) { + return NULL; + } + if (nargs == 0) { + return unicode_get_empty(); + } + PyObject *object = args[0]; + if (nargs == 1) { + return PyObject_Str(object); + } + const char *encoding = arg_as_utf8(args[1], "encoding"); + if (encoding == NULL) { + return NULL; + } + const char *errors = NULL; + if (nargs == 3) { + errors = arg_as_utf8(args[2], "errors"); + if (errors == NULL) { + return NULL; + } + } + return PyUnicode_FromEncodedObject(object, encoding, errors); +} + static PyObject * unicode_subtype_new(PyTypeObject *type, PyObject *unicode) { @@ -14651,8 +15480,8 @@ unicode_subtype_new(PyTypeObject *type, PyObject *unicode) _PyUnicode_STATE(self).compact = 0; _PyUnicode_STATE(self).ascii = _PyUnicode_STATE(unicode).ascii; _PyUnicode_STATE(self).statically_allocated = 0; - _PyUnicode_UTF8_LENGTH(self) = 0; - _PyUnicode_UTF8(self) = NULL; + PyUnicode_SET_UTF8_LENGTH(self, 0); + PyUnicode_SET_UTF8(self, NULL); _PyUnicode_DATA_ANY(self) = NULL; share_utf8 = 0; @@ -14682,8 +15511,8 @@ unicode_subtype_new(PyTypeObject *type, PyObject *unicode) _PyUnicode_DATA_ANY(self) = data; if (share_utf8) { - _PyUnicode_UTF8_LENGTH(self) = length; - _PyUnicode_UTF8(self) = data; + PyUnicode_SET_UTF8_LENGTH(self, length); + PyUnicode_SET_UTF8(self, data); } memcpy(data, PyUnicode_DATA(unicode), kind * (length + 1)); @@ -14761,7 +15590,8 @@ PyTypeObject PyUnicode_Type = { 0, /* tp_init */ 0, /* tp_alloc */ unicode_new, /* tp_new */ - PyObject_Del, /* tp_free */ + PyObject_Free, /* tp_free */ + .tp_vectorcall = unicode_vectorcall, }; /* Initialize the Unicode implementation */ @@ -14804,30 +15634,19 @@ _PyUnicode_InitState(PyInterpreterState *interp) PyStatus _PyUnicode_InitGlobalObjects(PyInterpreterState *interp) { - // Initialize the global interned dict + if (_Py_IsMainInterpreter(interp)) { + PyStatus status = init_global_interned_strings(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } + assert(INTERNED_STRINGS); + if (init_interned_dict(interp)) { PyErr_Clear(); return _PyStatus_ERR("failed to create interned dict"); } - if (_Py_IsMainInterpreter(interp)) { - /* Intern statically allocated string identifiers and deepfreeze strings. - * This must be done before any module initialization so that statically - * allocated string identifiers are used instead of heap allocated strings. - * Deepfreeze uses the interned identifiers if present to save space - * else generates them and they are interned to speed up dict lookups. - */ - _PyUnicode_InitStaticStrings(interp); - -#ifdef Py_DEBUG - assert(_PyUnicode_CheckConsistency(&_Py_STR(empty), 1)); - - for (int i = 0; i < 256; i++) { - assert(_PyUnicode_CheckConsistency(LATIN1(i), 1)); - } -#endif - } - return _PyStatus_OK(); } @@ -14850,106 +15669,250 @@ _PyUnicode_InitTypes(PyInterpreterState *interp) return _PyStatus_ERR("Can't initialize unicode types"); } +static /* non-null */ PyObject* +intern_static(PyInterpreterState *interp, PyObject *s /* stolen */) +{ + // Note that this steals a reference to `s`, but in many cases that + // stolen ref is returned, requiring no decref/incref. + + assert(s != NULL); + assert(_PyUnicode_CHECK(s)); + assert(_PyUnicode_STATE(s).statically_allocated); + assert(!PyUnicode_CHECK_INTERNED(s)); + +#ifdef Py_DEBUG + /* We must not add process-global interned string if there's already a + * per-interpreter interned_dict, which might contain duplicates. + */ + PyObject *interned = get_interned_dict(interp); + assert(interned == NULL); +#endif + + /* Look in the global cache first. */ + PyObject *r = (PyObject *)_Py_hashtable_get(INTERNED_STRINGS, s); + /* We should only init each string once */ + assert(r == NULL); + /* but just in case (for the non-debug build), handle this */ + if (r != NULL && r != s) { + assert(_PyUnicode_STATE(r).interned == SSTATE_INTERNED_IMMORTAL_STATIC); + assert(_PyUnicode_CHECK(r)); + Py_DECREF(s); + return Py_NewRef(r); + } + + if (_Py_hashtable_set(INTERNED_STRINGS, s, s) < -1) { + Py_FatalError("failed to intern static string"); + } + + _PyUnicode_STATE(s).interned = SSTATE_INTERNED_IMMORTAL_STATIC; + return s; +} void -_PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p) +_PyUnicode_InternStatic(PyInterpreterState *interp, PyObject **p) +{ + // This should only be called as part of runtime initialization + assert(!Py_IsInitialized()); + + *p = intern_static(interp, *p); + assert(*p); +} + +static void +immortalize_interned(PyObject *s) +{ + assert(PyUnicode_CHECK_INTERNED(s) == SSTATE_INTERNED_MORTAL); + assert(!_Py_IsImmortal(s)); +#ifdef Py_REF_DEBUG + /* The reference count value should be excluded from the RefTotal. + The decrements to these objects will not be registered so they + need to be accounted for in here. */ + for (Py_ssize_t i = 0; i < Py_REFCNT(s); i++) { + _Py_DecRefTotal(_PyThreadState_GET()); + } +#endif + FT_ATOMIC_STORE_UINT16_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_IMMORTAL); + _Py_SetImmortal(s); +} + +static /* non-null */ PyObject* +intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, + bool immortalize) { - PyObject *s = *p; + // Note that this steals a reference to `s`, but in many cases that + // stolen ref is returned, requiring no decref/incref. + #ifdef Py_DEBUG assert(s != NULL); assert(_PyUnicode_CHECK(s)); #else if (s == NULL || !PyUnicode_Check(s)) { - return; + return s; } #endif /* If it's a subclass, we don't really know what putting it in the interned dict might do. */ if (!PyUnicode_CheckExact(s)) { - return; + return s; } - if (PyUnicode_CHECK_INTERNED(s)) { - return; + /* Is it already interned? */ + switch (PyUnicode_CHECK_INTERNED(s)) { + case SSTATE_NOT_INTERNED: + // no, go on + break; + case SSTATE_INTERNED_MORTAL: + // yes but we might need to make it immortal + if (immortalize) { + immortalize_interned(s); + } + return s; + default: + // all done + return s; } - /* Look in the global cache first. */ - PyObject *r = (PyObject *)_Py_hashtable_get(INTERNED_STRINGS, s); - if (r != NULL && r != s) { - Py_SETREF(*p, Py_NewRef(r)); - return; + /* Statically allocated strings must be already interned. */ + assert(!_PyUnicode_STATE(s).statically_allocated); + +#if Py_GIL_DISABLED + /* In the free-threaded build, all interned strings are immortal */ + immortalize = 1; +#endif + + /* If it's already immortal, intern it as such */ + if (_Py_IsImmortal(s)) { + immortalize = 1; } - /* Handle statically allocated strings. */ - if (_PyUnicode_STATE(s).statically_allocated) { - assert(_Py_IsImmortal(s)); - if (_Py_hashtable_set(INTERNED_STRINGS, s, s) == 0) { - _PyUnicode_STATE(*p).interned = SSTATE_INTERNED_IMMORTAL_STATIC; + /* if it's a short string, get the singleton */ + if (PyUnicode_GET_LENGTH(s) == 1 && + PyUnicode_KIND(s) == PyUnicode_1BYTE_KIND) { + PyObject *r = LATIN1(*(unsigned char*)PyUnicode_DATA(s)); + assert(PyUnicode_CHECK_INTERNED(r)); + Py_DECREF(s); + return r; + } +#ifdef Py_DEBUG + assert(!unicode_is_singleton(s)); +#endif + + /* Look in the global cache now. */ + { + PyObject *r = (PyObject *)_Py_hashtable_get(INTERNED_STRINGS, s); + if (r != NULL) { + assert(_PyUnicode_STATE(r).statically_allocated); + assert(r != s); // r must be statically_allocated; s is not + Py_DECREF(s); + return Py_NewRef(r); } - return; } - /* Look in the per-interpreter cache. */ + /* Do a setdefault on the per-interpreter cache. */ PyObject *interned = get_interned_dict(interp); assert(interned != NULL); PyObject *t; - int res = PyDict_SetDefaultRef(interned, s, s, &t); - if (res < 0) { - PyErr_Clear(); - return; - } - else if (res == 1) { - // value was already present (not inserted) - Py_SETREF(*p, t); - return; + { + int res = PyDict_SetDefaultRef(interned, s, s, &t); + if (res < 0) { + PyErr_Clear(); + return s; + } + else if (res == 1) { + // value was already present (not inserted) + Py_DECREF(s); + if (immortalize && + PyUnicode_CHECK_INTERNED(t) == SSTATE_INTERNED_MORTAL) { + immortalize_interned(t); + } + return t; + } + else { + // value was newly inserted + assert (s == t); + Py_DECREF(t); + } } - Py_DECREF(t); - if (_Py_IsImmortal(s)) { - // XXX Restrict this to the main interpreter? - _PyUnicode_STATE(*p).interned = SSTATE_INTERNED_IMMORTAL_STATIC; - return; - } + /* NOT_INTERNED -> INTERNED_MORTAL */ + + assert(_PyUnicode_STATE(s).interned == SSTATE_NOT_INTERNED); + if (!_Py_IsImmortal(s)) { + /* The two references in interned dict (key and value) are not counted. + unicode_dealloc() and _PyUnicode_ClearInterned() take care of this. */ + Py_SET_REFCNT(s, Py_REFCNT(s) - 2); #ifdef Py_REF_DEBUG - /* The reference count value excluding the 2 references from the - interned dictionary should be excluded from the RefTotal. The - decrements to these objects will not be registered so they - need to be accounted for in here. */ - for (Py_ssize_t i = 0; i < Py_REFCNT(s) - 2; i++) { - _Py_DecRefTotal(_PyInterpreterState_GET()); + /* let's be pedantic with the ref total */ + _Py_DecRefTotal(_PyThreadState_GET()); + _Py_DecRefTotal(_PyThreadState_GET()); +#endif + } + FT_ATOMIC_STORE_UINT16_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_MORTAL); + + /* INTERNED_MORTAL -> INTERNED_IMMORTAL (if needed) */ + +#ifdef Py_DEBUG + if (_Py_IsImmortal(s)) { + assert(immortalize); } #endif - _Py_SetImmortal(s); - _PyUnicode_STATE(*p).interned = SSTATE_INTERNED_IMMORTAL; + if (immortalize) { + immortalize_interned(s); + } + + return s; +} + +void +_PyUnicode_InternImmortal(PyInterpreterState *interp, PyObject **p) +{ + *p = intern_common(interp, *p, 1); + assert(*p); +} + +void +_PyUnicode_InternMortal(PyInterpreterState *interp, PyObject **p) +{ + *p = intern_common(interp, *p, 0); + assert(*p); +} + + +void +_PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p) +{ + _PyUnicode_InternImmortal(interp, p); + return; } void PyUnicode_InternInPlace(PyObject **p) { PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyUnicode_InternInPlace(interp, p); + _PyUnicode_InternMortal(interp, p); } -// Function kept for the stable ABI. +// Public-looking name kept for the stable ABI; user should not call this: PyAPI_FUNC(void) PyUnicode_InternImmortal(PyObject **); void PyUnicode_InternImmortal(PyObject **p) { - PyUnicode_InternInPlace(p); - // Leak a reference on purpose - Py_INCREF(*p); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternImmortal(interp, p); } PyObject * PyUnicode_InternFromString(const char *cp) { PyObject *s = PyUnicode_FromString(cp); - if (s == NULL) + if (s == NULL) { return NULL; - PyUnicode_InternInPlace(&s); + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternMortal(interp, &s); return s; } @@ -14963,20 +15926,13 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) } assert(PyDict_CheckExact(interned)); - /* TODO: - * Currently, the runtime is not able to guarantee that it can exit without - * allocations that carry over to a future initialization of Python within - * the same process. i.e: - * ./python -X showrefcount -c 'import itertools' - * [237 refs, 237 blocks] - * - * Therefore, this should remain disabled for until there is a strict guarantee - * that no memory will be left after `Py_Finalize`. - */ -#ifdef Py_DEBUG - /* For all non-singleton interned strings, restore the two valid references - to that instance from within the intern string dictionary and let the - normal reference counting process clean up these instances. */ + if (has_shared_intern_dict(interp)) { + // the dict doesn't belong to this interpreter, skip the debug + // checks on it and just clear the pointer to it + clear_interned_dict(interp); + return; + } + #ifdef INTERNED_STATS fprintf(stderr, "releasing %zd interned strings\n", PyDict_GET_SIZE(interned)); @@ -14990,10 +15946,16 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) int shared = 0; switch (PyUnicode_CHECK_INTERNED(s)) { case SSTATE_INTERNED_IMMORTAL: + /* Make immortal interned strings mortal again. */ // Skip the Immortal Instance check and restore // the two references (key and value) ignored // by PyUnicode_InternInPlace(). _Py_SetMortal(s, 2); +#ifdef Py_REF_DEBUG + /* let's be pedantic with the ref total */ + _Py_IncRefTotal(_PyThreadState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); +#endif #ifdef INTERNED_STATS total_length += PyUnicode_GET_LENGTH(s); #endif @@ -15009,14 +15971,22 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) } break; case SSTATE_INTERNED_MORTAL: - /* fall through */ + // Restore 2 references held by the interned dict; these will + // be decref'd by clear_interned_dict's PyDict_Clear. + Py_SET_REFCNT(s, Py_REFCNT(s) + 2); +#ifdef Py_REF_DEBUG + /* let's be pedantic with the ref total */ + _Py_IncRefTotal(_PyThreadState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); +#endif + break; case SSTATE_NOT_INTERNED: - /* fall through */ + _Py_FALLTHROUGH; default: Py_UNREACHABLE(); } if (!shared) { - _PyUnicode_STATE(s).interned = SSTATE_NOT_INTERNED; + FT_ATOMIC_STORE_UINT16_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_NOT_INTERNED); } } #ifdef INTERNED_STATS @@ -15030,8 +16000,10 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) for (Py_ssize_t i=0; i < ids->size; i++) { Py_XINCREF(ids->array[i]); } -#endif /* Py_DEBUG */ clear_interned_dict(interp); + if (_Py_IsMainInterpreter(interp)) { + clear_global_interned_strings(); + } } @@ -15241,7 +16213,7 @@ encode_wstr_utf8(wchar_t *wstr, char **str, const char *name) int res; res = _Py_EncodeUTF8Ex(wstr, str, NULL, NULL, 1, _Py_ERROR_STRICT); if (res == -2) { - PyErr_Format(PyExc_RuntimeWarning, "cannot decode %s", name); + PyErr_Format(PyExc_RuntimeError, "cannot encode %s", name); return -1; } if (res < 0) { @@ -15394,7 +16366,11 @@ init_fs_encoding(PyThreadState *tstate) PyStatus _PyUnicode_InitEncodings(PyThreadState *tstate) { - PyStatus status = init_fs_encoding(tstate); + PyStatus status = _PyCodec_InitRegistry(tstate->interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + status = init_fs_encoding(tstate); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -15454,9 +16430,9 @@ unicode_is_finalizing(void) void _PyUnicode_FiniTypes(PyInterpreterState *interp) { - _PyStaticType_Dealloc(interp, &EncodingMapType); - _PyStaticType_Dealloc(interp, &PyFieldNameIter_Type); - _PyStaticType_Dealloc(interp, &PyFormatterIter_Type); + _PyStaticType_FiniBuiltin(interp, &EncodingMapType); + _PyStaticType_FiniBuiltin(interp, &PyFieldNameIter_Type); + _PyStaticType_FiniBuiltin(interp, &PyFormatterIter_Type); } @@ -15465,8 +16441,10 @@ _PyUnicode_Fini(PyInterpreterState *interp) { struct _Py_unicode_state *state = &interp->unicode; - // _PyUnicode_ClearInterned() must be called before _PyUnicode_Fini() - assert(get_interned_dict(interp) == NULL); + if (!has_shared_intern_dict(interp)) { + // _PyUnicode_ClearInterned() must be called before _PyUnicode_Fini() + assert(get_interned_dict(interp) == NULL); + } _PyUnicode_FiniEncodings(&state->fs_codec); @@ -15490,6 +16468,7 @@ static PyMethodDef _string_methods[] = { static PyModuleDef_Slot module_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Objects/unicodetype_db.h b/Objects/unicodetype_db.h index e6dbeffbe2aa3e..5be810dd67426a 100644 --- a/Objects/unicodetype_db.h +++ b/Objects/unicodetype_db.h @@ -1,4 +1,4 @@ -/* this file was generated by ./Tools/unicode/makeunicodedata.py 3.3 */ +/* this file was generated by Tools/unicode/makeunicodedata.py 3.3 */ /* a list of unique character type descriptors */ const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = { @@ -50,6 +50,7 @@ const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = { {0, 211, 0, 0, 0, 10113}, {0, 209, 0, 0, 0, 10113}, {163, 0, 163, 0, 0, 9993}, + {42561, 0, 42561, 0, 0, 9993}, {0, 213, 0, 0, 0, 10113}, {130, 0, 130, 0, 0, 9993}, {0, 214, 0, 0, 0, 10113}, @@ -84,6 +85,7 @@ const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = { {42319, 0, 42319, 0, 0, 9993}, {42315, 0, 42315, 0, 0, 9993}, {-207, 0, -207, 0, 0, 9993}, + {42343, 0, 42343, 0, 0, 9993}, {42280, 0, 42280, 0, 0, 9993}, {42308, 0, 42308, 0, 0, 9993}, {-209, 0, -209, 0, 0, 9993}, @@ -405,6 +407,8 @@ const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = { {0, -48, 0, 0, 0, 10113}, {0, -42307, 0, 0, 0, 10113}, {0, -35384, 0, 0, 0, 10113}, + {0, -42343, 0, 0, 0, 10113}, + {0, -42561, 0, 0, 0, 10113}, {-928, 0, -928, 0, 0, 9993}, {16778124, 17826698, 16778124, 0, 0, 26377}, {16778127, 17826701, 16778127, 0, 0, 26377}, @@ -1751,613 +1755,1218 @@ const Py_UCS4 _PyUnicode_ExtendedCase[] = { }; /* type indexes */ -#define SHIFT 7 +#define SHIFT 6 static const unsigned short index1[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, - 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 34, 35, 36, 37, - 38, 39, 34, 34, 34, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, - 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 64, 64, 64, 65, 66, 64, - 64, 64, 64, 67, 68, 64, 64, 64, 64, 64, 64, 69, 70, 71, 72, 73, 74, 75, - 76, 64, 77, 78, 79, 80, 81, 82, 83, 64, 64, 84, 85, 34, 34, 34, 34, 34, - 34, 86, 34, 34, 34, 34, 34, 87, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 88, 89, 90, 91, 92, 93, 34, 94, 34, 34, - 34, 95, 96, 34, 34, 34, 34, 34, 97, 34, 34, 34, 98, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 99, 100, 101, 34, 34, 34, 34, 34, 34, 102, 103, 34, - 34, 34, 34, 34, 34, 34, 34, 104, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 105, 34, 34, 34, 93, 34, 34, 34, 34, 34, 34, 34, 34, 106, 34, 34, 34, 34, - 107, 108, 34, 34, 34, 34, 34, 109, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 93, 34, 34, 34, 34, 34, 34, 110, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 111, 112, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 113, 34, 34, 34, 34, 114, 34, 34, 115, 116, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 117, 34, 34, 34, - 34, 34, 34, 34, 34, 118, 34, 34, 119, 120, 121, 122, 123, 124, 125, 126, - 127, 128, 129, 130, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 131, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 133, 134, - 135, 136, 137, 138, 139, 34, 140, 141, 142, 143, 144, 145, 146, 147, 148, - 149, 132, 150, 151, 152, 153, 154, 155, 156, 34, 34, 157, 158, 159, 160, - 161, 162, 163, 164, 165, 166, 167, 168, 169, 132, 170, 171, 172, 173, - 174, 175, 176, 177, 178, 179, 180, 132, 181, 182, 132, 183, 184, 185, - 186, 132, 187, 188, 189, 190, 191, 192, 193, 132, 194, 195, 196, 197, - 132, 198, 199, 200, 34, 34, 34, 34, 34, 34, 34, 201, 202, 34, 203, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 204, 34, 34, 34, 34, 34, 34, 34, 34, 205, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 34, 34, 34, 34, 206, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 34, 34, 34, 34, 207, 208, 209, 210, 132, 132, 132, 132, - 211, 212, 213, 214, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 215, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 216, 217, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 218, 34, 34, 219, 34, 34, 220, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 221, 222, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 223, 224, 64, - 225, 226, 227, 228, 229, 230, 132, 231, 232, 233, 234, 235, 236, 237, - 238, 64, 64, 64, 64, 239, 240, 132, 132, 132, 132, 132, 132, 132, 132, - 241, 132, 242, 243, 244, 132, 132, 245, 132, 132, 132, 246, 132, 132, - 132, 132, 132, 247, 34, 248, 249, 132, 132, 132, 132, 132, 250, 251, 252, - 132, 253, 254, 132, 132, 255, 256, 257, 258, 259, 132, 64, 260, 64, 64, - 64, 64, 64, 261, 262, 263, 264, 265, 64, 64, 266, 267, 64, 268, 132, 132, - 132, 132, 132, 132, 132, 132, 269, 270, 271, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 86, 272, 34, 273, 274, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 275, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 276, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 277, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 109, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 278, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 279, 34, 280, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 281, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 282, 34, 34, 34, 34, 283, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 34, 275, 34, 34, 284, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 285, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 286, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 287, 132, 288, 289, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 132, 132, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, + 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 26, 26, 26, 26, 26, 68, 69, + 70, 71, 72, 73, 74, 75, 26, 26, 26, 26, 26, 26, 26, 26, 76, 77, 78, 79, + 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, + 98, 99, 100, 101, 102, 103, 104, 105, 12, 106, 106, 107, 106, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 119, 119, 119, 119, + 119, 119, 119, 119, 120, 121, 122, 123, 119, 119, 119, 119, 119, 119, + 119, 119, 119, 124, 125, 119, 119, 119, 119, 119, 119, 119, 119, 119, + 119, 119, 119, 119, 119, 126, 127, 119, 128, 129, 106, 130, 131, 132, + 133, 134, 135, 136, 137, 138, 119, 119, 119, 139, 140, 141, 142, 143, + 144, 26, 145, 146, 147, 148, 149, 119, 119, 119, 119, 119, 150, 26, 151, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 152, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 153, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 119, 154, 155, 156, 157, 153, + 158, 26, 159, 160, 26, 26, 26, 161, 162, 26, 26, 26, 26, 26, 26, 26, 163, + 26, 164, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 165, 26, 26, 26, 26, + 26, 26, 26, 166, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 167, 26, 168, 169, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 170, 26, 171, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 172, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 173, 26, 26, 26, 26, + 26, 26, 26, 160, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 174, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 175, 176, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 177, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 160, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 178, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 179, 26, 158, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 180, 26, 26, 26, 26, 26, 26, 26, 26, 26, 159, 26, 26, 26, 26, 26, 181, + 182, 26, 183, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 184, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 185, 186, 26, 26, 26, 26, 187, 188, 189, 190, 191, 192, 193, + 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, + 208, 209, 210, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 211, 212, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 26, 214, 215, 216, 26, 217, 26, 218, 219, + 220, 221, 222, 26, 223, 26, 26, 224, 225, 226, 227, 228, 229, 26, 230, + 231, 232, 233, 234, 235, 236, 26, 237, 238, 239, 240, 241, 213, 213, 242, + 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 26, 26, + 26, 26, 256, 257, 258, 213, 259, 260, 261, 262, 263, 213, 264, 265, 266, + 267, 268, 269, 270, 271, 272, 213, 26, 273, 274, 275, 276, 277, 278, 213, + 213, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, + 213, 213, 306, 307, 308, 309, 310, 311, 312, 313, 213, 213, 314, 213, + 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 213, 213, 326, + 327, 328, 329, 213, 330, 331, 332, 213, 213, 213, 213, 333, 334, 335, + 336, 337, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 218, + 213, 338, 339, 26, 26, 26, 340, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 341, 342, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 343, 344, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 237, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 313, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 345, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 26, 26, + 26, 26, 26, 26, 26, 26, 324, 346, 347, 348, 349, 350, 351, 213, 213, 213, + 213, 213, 213, 352, 213, 213, 213, 353, 354, 213, 26, 355, 356, 357, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 358, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 359, 273, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 360, 26, 26, 26, 26, 361, 362, 26, 26, 26, 26, 26, + 363, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 26, 364, 365, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 119, 119, 119, 366, 119, 119, 119, 119, 119, 119, 138, 213, + 367, 368, 119, 369, 119, 119, 119, 370, 371, 372, 373, 374, 119, 375, + 213, 376, 119, 377, 213, 213, 378, 379, 380, 381, 382, 383, 384, 385, + 386, 387, 388, 389, 390, 391, 392, 393, 119, 119, 119, 119, 119, 119, + 119, 119, 394, 395, 396, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 397, 213, 213, 213, 398, 399, + 400, 213, 401, 402, 213, 213, 213, 213, 403, 404, 213, 213, 213, 213, + 213, 213, 213, 405, 213, 213, 213, 406, 213, 213, 213, 213, 213, 213, + 213, 407, 26, 26, 26, 408, 409, 410, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 411, 412, 213, 413, 213, 213, 213, 414, 415, 416, + 417, 213, 213, 213, 213, 418, 119, 419, 420, 421, 422, 423, 424, 425, + 426, 213, 213, 119, 119, 119, 427, 119, 119, 119, 119, 119, 119, 119, + 119, 119, 119, 119, 428, 119, 429, 119, 430, 431, 432, 433, 434, 119, + 119, 119, 119, 119, 435, 436, 437, 119, 119, 438, 366, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 439, + 440, 26, 441, 181, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 152, 26, 442, 26, 26, 26, 26, 443, 444, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 445, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 446, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 165, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 177, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 447, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 448, 26, 26, 26, 449, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 450, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 451, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 452, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 26, 26, 445, 26, + 26, 26, 26, 26, 452, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 453, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 454, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 455, 456, 213, 213, 12, 12, 12, 457, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, }; static const unsigned short index2[] = { @@ -2382,1586 +2991,1316 @@ static const unsigned short index2[] = { 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 34, 29, 30, 29, 30, 29, 30, 35, 36, 37, 29, 30, 29, 30, 38, 29, 30, 39, 39, 29, 30, 19, 40, 41, 42, 29, 30, 39, 43, 44, 45, 46, 29, - 30, 47, 19, 45, 48, 49, 50, 29, 30, 29, 30, 29, 30, 51, 29, 30, 51, 19, - 19, 29, 30, 51, 29, 30, 52, 52, 29, 30, 29, 30, 53, 29, 30, 19, 54, 29, - 30, 19, 55, 54, 54, 54, 54, 56, 57, 58, 56, 57, 58, 56, 57, 58, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 59, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 60, 56, 57, - 58, 29, 30, 61, 62, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 47, 48, 45, 49, 50, 51, 29, 30, 29, 30, 29, 30, 52, 29, 30, 52, 19, + 19, 29, 30, 52, 29, 30, 53, 53, 29, 30, 29, 30, 54, 29, 30, 19, 55, 29, + 30, 19, 56, 55, 55, 55, 55, 57, 58, 59, 57, 58, 59, 57, 58, 59, 29, 30, + 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 60, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 61, 57, 58, + 59, 29, 30, 62, 63, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 63, 19, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 19, 19, 19, 19, 19, 19, 64, - 29, 30, 65, 66, 67, 67, 29, 30, 68, 69, 70, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 71, 72, 73, 74, 75, 19, 76, 76, 19, 77, 19, 78, 79, 19, 19, - 19, 76, 80, 19, 81, 19, 82, 83, 19, 84, 85, 83, 86, 87, 19, 19, 85, 19, - 88, 89, 19, 19, 90, 19, 19, 19, 19, 19, 19, 19, 91, 19, 19, 92, 19, 93, - 92, 19, 19, 19, 94, 92, 95, 96, 96, 97, 19, 19, 19, 19, 19, 98, 19, 54, - 19, 19, 19, 19, 19, 19, 19, 19, 99, 100, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 102, 102, 102, 102, 102, 102, 102, 101, 101, 5, 5, 5, 5, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 101, 101, 101, 101, 101, 5, 5, 5, 5, 5, 5, 5, - 102, 5, 102, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 24, 24, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 64, 19, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 19, 19, 19, 19, 19, 19, 65, + 29, 30, 66, 67, 68, 68, 29, 30, 69, 70, 71, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 72, 73, 74, 75, 76, 19, 77, 77, 19, 78, 19, 79, 80, 19, 19, + 19, 77, 81, 19, 82, 83, 84, 85, 19, 86, 87, 85, 88, 89, 19, 19, 87, 19, + 90, 91, 19, 19, 92, 19, 19, 19, 19, 19, 19, 19, 93, 19, 19, 94, 19, 95, + 94, 19, 19, 19, 96, 94, 97, 98, 98, 99, 19, 19, 19, 19, 19, 100, 19, 55, + 19, 19, 19, 19, 19, 19, 19, 19, 101, 102, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 104, 104, 104, 104, 104, 104, 104, 103, 103, 5, 5, 5, 5, 104, + 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 103, 103, 103, 103, 103, 5, 5, 5, 5, 5, 5, 5, + 104, 5, 104, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 103, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 105, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 29, 30, 29, 30, 102, 5, 29, 30, 0, 0, 104, 49, 49, 49, 4, 105, 0, - 0, 0, 0, 5, 5, 106, 24, 107, 107, 107, 0, 108, 0, 109, 109, 110, 16, 16, + 24, 24, 29, 30, 29, 30, 104, 5, 29, 30, 0, 0, 106, 50, 50, 50, 4, 107, 0, + 0, 0, 0, 5, 5, 108, 24, 109, 109, 109, 0, 110, 0, 111, 111, 112, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 0, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 111, 112, 112, 112, 113, 18, 18, 18, 18, 18, - 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 114, 18, 18, 18, 18, 18, - 18, 18, 18, 18, 115, 116, 116, 117, 118, 119, 120, 120, 120, 121, 122, - 123, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 124, 125, 126, 127, 128, 129, 4, 29, 30, 130, - 29, 30, 19, 63, 63, 63, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 131, 131, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 113, 114, 114, 114, 115, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 116, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 117, 118, 118, 119, 120, 121, 122, 122, 122, 123, 124, + 125, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 126, 127, 128, 129, 130, 131, 4, 29, 30, 132, + 29, 30, 19, 64, 64, 64, 133, 133, 133, 133, 133, 133, 133, 133, 133, 133, + 133, 133, 133, 133, 133, 133, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, - 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 132, - 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, - 132, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 134, + 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, + 134, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 4, 24, 24, 24, 24, 24, 5, 5, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 133, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 134, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, + 29, 30, 29, 30, 29, 30, 135, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 136, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 0, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 0, 0, 102, 4, 4, 4, 4, 4, 5, 19, 136, 136, 136, 136, - 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, - 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, - 136, 136, 136, 136, 136, 136, 137, 19, 4, 4, 0, 0, 4, 4, 4, 0, 24, 24, + 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 0, 137, 137, 137, 137, 137, 137, + 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 137, 137, 137, 137, 0, 0, 104, 4, 4, 4, 4, 4, 5, 19, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 138, 139, 19, 4, 4, 0, 0, 4, 4, 4, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 4, 24, 4, 24, 24, 4, 24, 24, 4, 24, 0, 0, 0, - 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 54, 54, - 54, 54, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 20, 20, 20, 20, 20, 4, + 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 55, 55, + 55, 55, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 20, 20, 20, 20, 20, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 4, - 20, 4, 4, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 102, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, 24, 24, 24, 24, 24, 24, + 20, 4, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 6, 7, 8, 9, 10, 11, - 12, 13, 14, 15, 4, 4, 4, 4, 54, 54, 24, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 4, 54, 24, 24, 24, 24, 24, 24, 24, 20, 4, 24, 24, 24, 24, 24, 24, - 102, 102, 24, 24, 4, 24, 24, 24, 24, 54, 54, 6, 7, 8, 9, 10, 11, 12, 13, - 14, 15, 54, 54, 54, 4, 4, 54, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 0, 20, 54, 24, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, + 12, 13, 14, 15, 4, 4, 4, 4, 55, 55, 24, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 4, 55, 24, 24, 24, 24, 24, 24, 24, 20, 4, 24, 24, 24, 24, 24, 24, + 104, 104, 24, 24, 4, 24, 24, 24, 24, 55, 55, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 55, 55, 55, 4, 4, 55, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 0, 20, 55, 24, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, - 14, 15, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 102, 102, 4, 4, 4, 4, 102, 0, 0, 24, 4, - 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 24, 24, 24, 24, 102, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 102, 24, 24, 24, 102, 24, 24, 24, 24, 24, 0, 0, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, 24, 0, 0, - 4, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 5, 54, 54, 54, 54, 54, 54, 0, 20, 20, 0, 0, 0, 0, 0, 0, - 24, 24, 24, 24, 24, 24, 24, 24, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 102, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 104, 104, 4, 4, 4, 4, 104, 0, 0, 24, 4, + 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 24, 24, 24, 24, 104, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 104, 24, 24, 24, 104, 24, 24, 24, 24, 24, 0, 0, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 0, 0, + 4, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 5, 55, 55, 55, 55, 55, 55, 0, 20, 20, 0, 0, 0, 0, 0, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 20, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 17, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 24, 17, 24, 54, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 17, 17, 17, - 17, 24, 17, 17, 54, 24, 24, 24, 24, 24, 24, 24, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 24, 24, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 102, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 17, 17, - 0, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 54, 54, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, 0, 0, 0, 54, 54, 54, 54, 0, 0, 24, - 54, 17, 17, 17, 24, 24, 24, 24, 0, 0, 17, 17, 0, 0, 17, 17, 24, 54, 0, 0, - 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 54, 54, 0, 54, 54, 54, 24, 24, 0, 0, 6, - 7, 8, 9, 10, 11, 12, 13, 14, 15, 54, 54, 4, 4, 26, 26, 26, 26, 26, 26, 4, - 4, 54, 4, 24, 0, 0, 24, 24, 17, 0, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, - 54, 54, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 0, - 54, 54, 0, 54, 54, 0, 0, 24, 0, 17, 17, 17, 24, 24, 0, 0, 0, 0, 24, 24, - 0, 0, 24, 24, 24, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 0, - 54, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 24, 24, 54, - 54, 54, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 17, 0, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, - 54, 54, 54, 54, 0, 54, 54, 0, 54, 54, 54, 54, 54, 0, 0, 24, 54, 17, 17, - 17, 24, 24, 24, 24, 24, 0, 24, 24, 17, 0, 17, 17, 24, 0, 0, 54, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 24, 24, 0, 0, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 4, 4, 0, 0, 0, 0, 0, 0, 0, 54, 24, 24, 24, 24, 24, - 24, 0, 24, 17, 17, 0, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 54, 54, 0, 0, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 0, 54, 54, 54, - 54, 54, 0, 0, 24, 54, 17, 24, 17, 24, 24, 24, 24, 0, 0, 17, 17, 0, 0, 17, - 17, 24, 0, 0, 0, 0, 0, 0, 0, 24, 24, 17, 0, 0, 0, 0, 54, 54, 0, 54, 54, - 54, 24, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 54, 26, 26, 26, - 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 54, 0, 54, 54, 54, 54, 54, - 54, 0, 0, 0, 54, 54, 54, 0, 54, 54, 54, 54, 0, 0, 0, 54, 54, 0, 54, 0, - 54, 54, 0, 0, 0, 54, 54, 0, 0, 0, 54, 54, 54, 0, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 17, 17, 24, 17, 17, 0, 0, 0, - 17, 17, 17, 0, 17, 17, 17, 24, 0, 0, 54, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 24, 17, 24, 55, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 17, 17, 17, + 17, 24, 17, 17, 55, 24, 24, 24, 24, 24, 24, 24, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 24, 24, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 104, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 17, 17, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 0, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 0, 0, 0, 55, 55, 55, 55, 0, 0, 24, + 55, 17, 17, 17, 24, 24, 24, 24, 0, 0, 17, 17, 0, 0, 17, 17, 24, 55, 0, 0, + 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 55, 55, 0, 55, 55, 55, 24, 24, 0, 0, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 55, 55, 4, 4, 26, 26, 26, 26, 26, 26, 4, + 4, 55, 4, 24, 0, 0, 24, 24, 17, 0, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, + 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, + 55, 55, 0, 55, 55, 0, 0, 24, 0, 17, 17, 17, 24, 24, 0, 0, 0, 0, 24, 24, + 0, 0, 24, 24, 24, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 0, + 55, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 24, 24, 55, + 55, 55, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 17, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 0, 55, 55, 55, 55, 55, 0, 0, 24, 55, 17, 17, + 17, 24, 24, 24, 24, 24, 0, 24, 24, 17, 0, 17, 17, 24, 0, 0, 55, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 24, 24, 0, 0, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 4, 4, 0, 0, 0, 0, 0, 0, 0, 55, 24, 24, 24, 24, 24, + 24, 0, 24, 17, 17, 0, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, 55, 55, 55, + 55, 55, 0, 0, 24, 55, 17, 24, 17, 24, 24, 24, 24, 0, 0, 17, 17, 0, 0, 17, + 17, 24, 0, 0, 0, 0, 0, 0, 0, 24, 24, 17, 0, 0, 0, 0, 55, 55, 0, 55, 55, + 55, 24, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 55, 26, 26, 26, + 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 55, 0, 55, 55, 55, 55, 55, + 55, 0, 0, 0, 55, 55, 55, 0, 55, 55, 55, 55, 0, 0, 0, 55, 55, 0, 55, 0, + 55, 55, 0, 0, 0, 55, 55, 0, 0, 0, 55, 55, 55, 0, 0, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 17, 17, 24, 17, 17, 0, 0, 0, + 17, 17, 17, 0, 17, 17, 17, 24, 0, 0, 55, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 26, - 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 24, 17, 17, 17, 24, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 24, 54, + 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 24, 17, 17, 17, 24, 55, + 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 24, 55, 24, 24, 24, 17, 17, 17, 17, 0, 24, 24, 24, 0, 24, 24, 24, 24, 0, 0, 0, 0, - 0, 0, 0, 24, 24, 0, 54, 54, 54, 0, 0, 54, 0, 0, 54, 54, 24, 24, 0, 0, 6, + 0, 0, 0, 24, 24, 0, 55, 55, 55, 0, 0, 55, 0, 0, 55, 55, 24, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 4, 26, 26, 26, 26, - 26, 26, 26, 4, 54, 24, 17, 17, 4, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, - 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 0, 54, 54, 54, 54, 54, 0, 0, 24, 54, 17, 24, 17, 17, 17, 17, 17, 0, + 26, 26, 26, 4, 55, 24, 17, 17, 4, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, + 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 0, 55, 55, 55, 55, 55, 0, 0, 24, 55, 17, 24, 17, 17, 17, 17, 17, 0, 24, 17, 17, 0, 17, 17, 24, 24, 0, 0, 0, 0, 0, 0, 0, 17, 17, 0, 0, 0, 0, - 0, 0, 54, 54, 0, 54, 54, 24, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 0, 54, 54, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 17, 17, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 0, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 24, 24, 54, 17, 17, 17, 24, 24, 24, 24, 0, 17, 17, 17, 0, 17, 17, 17, 24, - 54, 4, 0, 0, 0, 0, 54, 54, 54, 17, 26, 26, 26, 26, 26, 26, 26, 54, 54, - 54, 24, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 4, 54, 54, 54, 54, 54, 54, 0, 24, 17, 17, 0, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 0, 0, - 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 24, 0, 0, 0, 0, 17, 17, 17, 24, 24, + 0, 0, 55, 55, 0, 55, 55, 24, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 0, 55, 55, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 17, 17, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 24, 24, 55, 17, 17, 17, 24, 24, 24, 24, 0, 17, 17, 17, 0, 17, 17, 17, 24, + 55, 4, 0, 0, 0, 0, 55, 55, 55, 17, 26, 26, 26, 26, 26, 26, 26, 55, 55, + 55, 24, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 4, 55, 55, 55, 55, 55, 55, 0, 24, 17, 17, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 24, 0, 0, 0, 0, 17, 17, 17, 24, 24, 24, 0, 24, 0, 17, 17, 17, 17, 17, 17, 17, 17, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 17, 17, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 54, 138, 24, 24, - 24, 24, 24, 24, 24, 0, 0, 0, 0, 4, 54, 54, 54, 54, 54, 54, 102, 24, 24, + 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 55, 140, 24, 24, + 24, 24, 24, 24, 24, 0, 0, 0, 0, 4, 55, 55, 55, 55, 55, 55, 104, 24, 24, 24, 24, 24, 24, 24, 24, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 0, 54, 0, 54, 54, 54, 54, 54, - 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 24, 54, 138, 24, 24, 24, 24, 24, 24, 24, 24, 24, 54, 0, 0, 54, 54, - 54, 54, 54, 0, 102, 0, 24, 24, 24, 24, 24, 24, 24, 0, 6, 7, 8, 9, 10, 11, - 12, 13, 14, 15, 0, 0, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 4, 4, 4, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 0, 55, 0, 55, 55, 55, 55, 55, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 0, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 24, 55, 140, 24, 24, 24, 24, 24, 24, 24, 24, 24, 55, 0, 0, 55, 55, + 55, 55, 55, 0, 104, 0, 24, 24, 24, 24, 24, 24, 24, 0, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 0, 0, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, 4, 4, 4, 4, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 4, 24, 4, 24, 4, 24, 4, 4, 4, 4, 17, 17, 54, 54, 54, 54, - 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 17, 24, 24, 24, 24, 24, 4, 24, 24, 54, 54, 54, 54, 54, + 26, 26, 26, 26, 4, 24, 4, 24, 4, 24, 4, 4, 4, 4, 17, 17, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 17, 24, 24, 24, 24, 24, 4, 24, 24, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 4, 4, 4, 4, 4, 4, 4, 4, 24, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 17, 17, 24, - 24, 24, 24, 17, 24, 24, 24, 24, 24, 24, 17, 24, 24, 17, 17, 24, 24, 54, - 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 4, 4, 4, 4, 4, 54, 54, 54, 54, 54, - 54, 17, 17, 24, 24, 54, 54, 54, 54, 24, 24, 24, 54, 17, 17, 17, 54, 54, - 17, 17, 17, 17, 17, 17, 17, 54, 54, 54, 24, 24, 24, 24, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 17, 17, 24, 24, 17, 17, 17, 17, - 17, 17, 24, 54, 17, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 17, 17, 24, - 4, 4, 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, - 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, - 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 139, 0, 139, 0, 0, 0, - 0, 0, 139, 0, 0, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, - 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, - 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, - 140, 140, 140, 140, 4, 101, 140, 140, 140, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, - 54, 0, 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, 0, 54, 54, 54, 54, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 0, 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, 54, - 54, 54, 0, 54, 0, 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 24, 24, 24, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 141, 142, 143, 144, 145, 146, 147, 148, 149, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, - 0, 0, 0, 0, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, - 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, - 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, - 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, - 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, - 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, - 232, 233, 234, 235, 0, 0, 236, 237, 238, 239, 240, 241, 0, 0, 4, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 4, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 1, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 4, 4, 0, 0, 0, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 4, 4, 4, - 242, 242, 242, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, - 24, 24, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, 17, 4, 4, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 0, 24, 24, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 24, 24, 17, 24, 24, 24, 24, 24, 24, 24, 17, 17, 17, - 17, 17, 17, 17, 17, 24, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 4, 4, 4, 102, 4, 4, 4, 4, 54, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, - 14, 15, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, - 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, 24, 20, 24, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 102, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, - 54, 54, 243, 243, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 24, 54, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 0, 24, 24, 24, 17, 17, 17, 17, 24, 24, - 17, 17, 17, 0, 0, 0, 0, 17, 17, 24, 17, 17, 17, 17, 17, 17, 24, 24, 24, - 0, 0, 0, 0, 4, 0, 0, 0, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, 54, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, - 13, 14, 15, 141, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 24, 24, 17, 17, 24, 0, 0, 4, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 17, 24, 17, 24, 24, 24, 24, 24, 24, 24, 0, - 24, 17, 24, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 17, 17, 17, 17, 17, - 17, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 24, 6, 7, 8, 9, 10, 11, - 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, - 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 102, 4, 4, 4, 4, 4, 4, 0, 0, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 5, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 17, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 17, 24, 24, 24, 24, 24, 17, 24, - 17, 17, 17, 17, 17, 24, 17, 17, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, - 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 24, 24, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 0, 24, 24, 17, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 17, 24, 24, 24, 24, 17, 17, 24, 24, 17, 24, 24, 24, 54, 54, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 17, 24, - 24, 17, 17, 17, 24, 17, 24, 24, 24, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, - 4, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 17, 17, 17, 17, 17, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 17, - 17, 24, 24, 0, 0, 0, 4, 4, 4, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 0, 0, 54, 54, 54, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 102, 102, 102, 102, 102, 102, 4, 4, 244, - 245, 246, 247, 248, 249, 250, 251, 252, 0, 0, 0, 0, 0, 0, 0, 253, 253, - 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, - 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, - 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 0, 0, - 253, 253, 253, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, - 24, 4, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 17, 24, 24, - 24, 24, 24, 24, 24, 54, 54, 54, 54, 24, 54, 54, 54, 54, 54, 54, 24, 54, - 54, 17, 24, 24, 54, 0, 0, 0, 0, 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 24, + 24, 24, 24, 17, 24, 24, 24, 24, 24, 24, 17, 24, 24, 17, 17, 24, 24, 55, + 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 4, 4, 4, 4, 4, 55, 55, 55, 55, 55, + 55, 17, 17, 24, 24, 55, 55, 55, 55, 24, 24, 24, 55, 17, 17, 17, 55, 55, + 17, 17, 17, 17, 17, 17, 17, 55, 55, 55, 24, 24, 24, 24, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 17, 17, 24, 24, 17, 17, 17, 17, + 17, 17, 24, 55, 17, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 17, 17, 24, + 4, 4, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 0, 141, 0, 0, 0, + 0, 0, 141, 0, 0, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, + 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, + 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, + 142, 142, 142, 142, 4, 103, 142, 142, 142, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 0, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 0, + 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 0, 55, 55, 55, 55, 0, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, + 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 0, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 143, 144, 145, 146, 147, + 148, 149, 150, 151, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 152, 153, 154, 155, 156, 157, 158, + 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, + 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, + 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, + 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, + 229, 230, 231, 232, 233, 234, 235, 236, 237, 0, 0, 238, 239, 240, 241, + 242, 243, 0, 0, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 1, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 4, 4, 4, 244, 244, 244, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, + 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 24, 24, 24, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, + 17, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, + 0, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 17, 24, 24, 24, 24, 24, + 24, 24, 17, 17, 17, 17, 17, 17, 17, 17, 24, 17, 17, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 4, 4, 4, 104, 4, 4, 4, 4, 55, 24, 0, 0, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, + 24, 20, 24, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, + 0, 0, 0, 55, 55, 55, 55, 55, 245, 245, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 24, 55, 0, 0, 0, 0, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 24, 24, 24, + 17, 17, 17, 17, 24, 24, 17, 17, 17, 0, 0, 0, 0, 17, 17, 24, 17, 17, 17, + 17, 17, 17, 24, 24, 24, 0, 0, 0, 0, 4, 0, 0, 0, 4, 4, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, + 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 143, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 24, 24, 17, 17, 24, 0, 0, 4, 4, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 24, 17, 24, 24, + 24, 24, 24, 24, 24, 0, 24, 17, 24, 17, 17, 24, 24, 24, 24, 24, 24, 24, + 24, 17, 17, 17, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, + 24, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 104, 4, 4, 4, + 4, 4, 4, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 5, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, + 24, 24, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 17, 24, 24, + 24, 24, 24, 17, 24, 17, 17, 17, 17, 17, 24, 17, 17, 55, 55, 55, 55, 55, + 55, 55, 55, 0, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, 17, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 17, 24, 24, 24, 24, 17, 17, 24, 24, 17, 24, 24, + 24, 55, 55, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 24, 17, 24, 24, 17, 17, 17, 24, 17, 24, 24, 24, 17, 17, 0, 0, 0, + 0, 0, 0, 0, 0, 4, 4, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, 17, 17, 17, 17, 17, 24, 24, 24, + 24, 24, 24, 24, 24, 17, 17, 24, 24, 0, 0, 0, 4, 4, 4, 4, 4, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 0, 0, 0, 55, 55, 55, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, 104, 104, + 104, 104, 104, 4, 4, 246, 247, 248, 249, 250, 251, 252, 253, 254, 29, 30, + 0, 0, 0, 0, 0, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 0, 0, 255, 255, 255, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 24, 24, 24, 4, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 17, 24, 24, 24, 24, 24, 24, 24, 55, 55, 55, 55, 24, 55, 55, 55, + 55, 55, 55, 24, 55, 55, 17, 24, 24, 55, 0, 0, 0, 0, 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 101, 254, 19, 19, 19, 255, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 256, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 257, 258, 259, 260, 261, 262, 19, - 19, 263, 19, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 103, 256, 19, 19, 19, 257, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 258, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 264, 264, 264, 264, 264, 264, 264, - 264, 265, 265, 265, 265, 265, 265, 265, 265, 264, 264, 264, 264, 264, - 264, 0, 0, 265, 265, 265, 265, 265, 265, 0, 0, 264, 264, 264, 264, 264, - 264, 264, 264, 265, 265, 265, 265, 265, 265, 265, 265, 264, 264, 264, - 264, 264, 264, 264, 264, 265, 265, 265, 265, 265, 265, 265, 265, 264, - 264, 264, 264, 264, 264, 0, 0, 265, 265, 265, 265, 265, 265, 0, 0, 266, - 264, 267, 264, 268, 264, 269, 264, 0, 265, 0, 265, 0, 265, 0, 265, 264, - 264, 264, 264, 264, 264, 264, 264, 265, 265, 265, 265, 265, 265, 265, - 265, 270, 270, 271, 271, 271, 271, 272, 272, 273, 273, 274, 274, 275, - 275, 0, 0, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, - 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, - 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, - 316, 317, 318, 319, 320, 321, 322, 323, 264, 264, 324, 325, 326, 0, 327, - 328, 265, 265, 329, 329, 330, 5, 331, 5, 5, 5, 332, 333, 334, 0, 335, - 336, 337, 337, 337, 337, 338, 5, 5, 5, 264, 264, 339, 340, 0, 0, 341, - 342, 265, 265, 343, 343, 0, 5, 5, 5, 264, 264, 344, 345, 346, 126, 347, - 348, 265, 265, 349, 349, 130, 5, 5, 5, 0, 0, 350, 351, 352, 0, 353, 354, - 355, 355, 356, 356, 357, 5, 5, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 20, - 358, 358, 20, 20, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 5, 4, 4, 5, 2, 2, 20, 20, 20, 20, 20, 1, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 17, 17, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 17, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 1, 20, 20, 20, 20, 20, 0, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 359, 101, 0, 0, 360, 361, 362, 363, - 364, 365, 4, 4, 4, 4, 4, 101, 359, 25, 21, 22, 360, 361, 362, 363, 364, - 365, 4, 4, 4, 4, 4, 0, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 5, 5, 5, 5, 24, 5, 5, 5, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 120, 4, 4, 4, 4, - 120, 4, 4, 19, 120, 120, 120, 19, 19, 120, 120, 120, 19, 4, 120, 4, 4, - 366, 120, 120, 120, 120, 120, 4, 4, 4, 4, 4, 4, 120, 4, 367, 4, 120, 4, - 368, 369, 120, 120, 366, 19, 120, 120, 370, 120, 19, 54, 54, 54, 54, 19, - 4, 4, 19, 19, 120, 120, 4, 4, 4, 4, 4, 120, 19, 19, 19, 19, 4, 4, 4, 4, - 371, 4, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, - 372, 372, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, 373, - 373, 373, 373, 373, 242, 242, 242, 29, 30, 242, 242, 242, 242, 26, 4, 4, - 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 25, 21, 22, 360, 361, 362, 363, 364, 365, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 25, 21, 22, 360, 361, 362, 363, 364, 365, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 25, 21, 22, 360, 361, 362, 363, 364, - 365, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 374, 374, 374, 374, - 374, 374, 374, 374, 374, 374, 374, 374, 374, 374, 374, 374, 374, 374, - 374, 374, 374, 374, 374, 374, 374, 374, 375, 375, 375, 375, 375, 375, - 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, - 375, 375, 375, 375, 375, 375, 359, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 25, 21, 22, 360, 361, 362, 363, 364, 365, 26, 359, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 25, 21, 22, 360, 361, - 362, 363, 364, 365, 26, 25, 21, 22, 360, 361, 362, 363, 364, 365, 26, 25, - 21, 22, 360, 361, 362, 363, 364, 365, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 259, 260, 261, 262, + 263, 264, 19, 19, 265, 19, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 266, 266, 266, 266, 266, 266, 266, 266, 267, 267, 267, 267, + 267, 267, 267, 267, 266, 266, 266, 266, 266, 266, 0, 0, 267, 267, 267, + 267, 267, 267, 0, 0, 266, 266, 266, 266, 266, 266, 266, 266, 267, 267, + 267, 267, 267, 267, 267, 267, 266, 266, 266, 266, 266, 266, 266, 266, + 267, 267, 267, 267, 267, 267, 267, 267, 266, 266, 266, 266, 266, 266, 0, + 0, 267, 267, 267, 267, 267, 267, 0, 0, 268, 266, 269, 266, 270, 266, 271, + 266, 0, 267, 0, 267, 0, 267, 0, 267, 266, 266, 266, 266, 266, 266, 266, + 266, 267, 267, 267, 267, 267, 267, 267, 267, 272, 272, 273, 273, 273, + 273, 274, 274, 275, 275, 276, 276, 277, 277, 0, 0, 278, 279, 280, 281, + 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, + 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, + 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, + 324, 325, 266, 266, 326, 327, 328, 0, 329, 330, 267, 267, 331, 331, 332, + 5, 333, 5, 5, 5, 334, 335, 336, 0, 337, 338, 339, 339, 339, 339, 340, 5, + 5, 5, 266, 266, 341, 342, 0, 0, 343, 344, 267, 267, 345, 345, 0, 5, 5, 5, + 266, 266, 346, 347, 348, 128, 349, 350, 267, 267, 351, 351, 132, 5, 5, 5, + 0, 0, 352, 353, 354, 0, 355, 356, 357, 357, 358, 358, 359, 5, 5, 0, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 20, 360, 360, 20, 20, 4, 4, 4, 4, 4, 4, 4, 4, + 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 4, 4, 5, 2, 2, 20, 20, 20, 20, 20, + 1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 17, 17, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 17, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 1, 20, 20, 20, 20, 20, 0, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 361, + 103, 0, 0, 362, 363, 364, 365, 366, 367, 4, 4, 4, 4, 4, 103, 361, 25, 21, + 22, 362, 363, 364, 365, 366, 367, 4, 4, 4, 4, 4, 0, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 5, 5, 5, 5, 24, 5, 5, 5, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 4, 4, 122, 4, 4, 4, 4, 122, 4, 4, 19, 122, 122, 122, 19, 19, 122, 122, + 122, 19, 4, 122, 4, 4, 368, 122, 122, 122, 122, 122, 4, 4, 4, 4, 4, 4, + 122, 4, 369, 4, 122, 4, 370, 371, 122, 122, 368, 19, 122, 122, 372, 122, + 19, 55, 55, 55, 55, 19, 4, 4, 19, 19, 122, 122, 4, 4, 4, 4, 4, 122, 19, + 19, 19, 19, 4, 4, 4, 4, 373, 4, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 374, 374, 374, 374, 374, 374, 374, 374, 374, 374, + 374, 374, 374, 374, 374, 374, 375, 375, 375, 375, 375, 375, 375, 375, + 375, 375, 375, 375, 375, 375, 375, 375, 244, 244, 244, 29, 30, 244, 244, + 244, 244, 26, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 25, 21, 22, 362, 363, 364, 365, 366, 367, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 25, 21, 22, 362, 363, 364, 365, 366, 367, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 25, 21, 22, 362, 363, 364, 365, 366, 367, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 376, 376, 376, 376, 376, + 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, + 376, 376, 376, 376, 376, 376, 376, 377, 377, 377, 377, 377, 377, 377, + 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, + 377, 377, 377, 377, 377, 361, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 25, + 21, 22, 362, 363, 364, 365, 366, 367, 26, 361, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 25, 21, + 22, 362, 363, 364, 365, 366, 367, 26, 25, 21, 22, 362, 363, 364, 365, + 366, 367, 26, 25, 21, 22, 362, 363, 364, 365, 366, 367, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, - 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, - 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, - 136, 136, 136, 136, 136, 136, 136, 136, 29, 30, 376, 377, 378, 379, 380, - 29, 30, 29, 30, 29, 30, 381, 382, 383, 384, 19, 29, 30, 19, 29, 30, 19, - 19, 19, 19, 19, 101, 101, 385, 385, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 19, 4, 4, 4, 4, 4, 4, 29, 30, 29, 30, 24, 24, 24, 29, 30, 0, 0, 0, 0, - 0, 4, 4, 4, 4, 26, 4, 4, 386, 386, 386, 386, 386, 386, 386, 386, 386, - 386, 386, 386, 386, 386, 386, 386, 386, 386, 386, 386, 386, 386, 386, - 386, 386, 386, 386, 386, 386, 386, 386, 386, 386, 386, 386, 386, 386, - 386, 0, 386, 0, 0, 0, 0, 0, 386, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 102, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, - 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 0, - 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, - 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 0, 24, 24, 24, 24, 24, 24, + 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 137, 137, 137, 137, 137, 137, 138, 138, 138, 138, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 29, 30, 378, + 379, 380, 381, 382, 29, 30, 29, 30, 29, 30, 383, 384, 385, 386, 19, 29, + 30, 19, 29, 30, 19, 19, 19, 19, 19, 103, 103, 387, 387, 29, 30, 29, 30, + 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, + 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 19, 4, 4, 4, 4, + 4, 4, 29, 30, 29, 30, 24, 24, 24, 29, 30, 0, 0, 0, 0, 0, 4, 4, 4, 4, 26, + 4, 4, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, + 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, + 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 0, 388, 0, 0, 0, + 0, 0, 388, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 104, 4, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 24, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, + 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, + 55, 55, 55, 55, 55, 55, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 387, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 389, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 1, 4, 4, 4, 4, 102, 54, 242, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 24, 24, 24, 24, 17, 17, 4, 102, 102, - 102, 102, 102, 4, 4, 242, 242, 242, 102, 54, 4, 4, 4, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 24, 24, 5, 5, 102, 102, 54, - 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 17, 102, 102, 102, 54, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 0, 4, 4, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 26, 26, 26, 26, 26, 26, - 26, 26, 4, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, + 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 54, 54, - 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 1, 4, 4, 4, 4, 104, 55, 244, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 24, 24, 24, 24, 17, 17, 4, 104, + 104, 104, 104, 104, 4, 4, 244, 244, 244, 104, 55, 4, 4, 4, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 24, 24, 5, 5, 104, 104, + 55, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 17, 104, 104, 104, 55, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 4, 4, 26, 26, 26, + 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 388, 54, 54, 388, 54, 54, 54, 388, - 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 388, 54, 388, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 388, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 388, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 388, 54, 388, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, - 54, 388, 388, 388, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 388, 388, 388, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 388, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 388, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 388, 388, - 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, - 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 388, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 102, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 4, 4, 4, + 4, 4, 4, 4, 4, 26, 26, 26, 26, 26, 26, 26, 26, 4, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 55, 55, 55, + 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 390, 55, 55, 390, 55, 55, 55, 390, 55, 390, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, + 55, 55, 55, 55, 55, 55, 390, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 390, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, + 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 390, + 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 390, 55, 390, 390, 390, 55, 55, 55, 55, 55, 55, 390, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 390, 390, + 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, + 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 390, 390, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 390, 390, 390, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, + 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 390, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 104, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 102, 102, 102, - 102, 102, 102, 4, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 102, - 4, 4, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 54, - 24, 5, 5, 5, 4, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 4, 102, 29, 30, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 104, 104, 104, 104, 104, 104, 4, 4, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 104, 4, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 55, + 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 101, 101, 24, 24, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 24, 24, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 5, 5, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 19, 19, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, + 29, 30, 29, 30, 29, 30, 29, 30, 55, 24, 5, 5, 5, 4, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 4, 104, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 103, + 103, 24, 24, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 24, 24, 4, 4, 4, 4, + 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 104, 104, 104, 104, 104, 104, 104, 104, 104, + 5, 5, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 19, 19, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 101, - 19, 19, 19, 19, 19, 19, 19, 19, 29, 30, 29, 30, 389, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 102, 5, 5, 29, 30, 390, 19, 54, 29, 30, 29, 30, 391, - 19, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 392, 393, 394, 395, 392, 19, 396, 397, 398, 399, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 400, 401, 402, 29, - 30, 29, 30, 0, 0, 0, 0, 0, 29, 30, 0, 19, 0, 19, 29, 30, 29, 30, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 101, - 101, 29, 30, 54, 101, 101, 19, 54, 54, 54, 54, 54, 54, 54, 24, 54, 54, - 54, 24, 54, 54, 54, 54, 24, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 17, 17, 24, 24, 17, 4, 4, - 4, 4, 24, 0, 0, 0, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 4, 4, 4, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 17, 17, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 6, 7, 8, 9, 10, 11, 12, - 13, 14, 15, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 54, 54, 54, 54, 54, 54, 4, 4, 4, 54, 4, 54, - 54, 24, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 24, 24, 24, 24, 24, 24, 24, 24, 4, 4, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 24, 24, - 24, 17, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 17, 17, 24, 24, - 24, 24, 17, 17, 24, 24, 17, 17, 17, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 0, 102, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 4, 4, 54, 54, - 54, 54, 54, 24, 102, 54, 54, 54, 54, 54, 54, 54, 54, 54, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, - 24, 24, 24, 24, 17, 17, 24, 24, 17, 17, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 54, 54, 54, 24, 54, 54, 54, 54, 54, 54, 54, 54, 24, 17, 0, 0, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 0, 0, 4, 4, 4, 4, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 102, 54, 54, 54, 54, 54, 54, 4, 4, 4, - 54, 17, 24, 17, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 24, 54, 24, 24, 24, 54, 54, 24, 24, 54, 54, 54, 54, 54, 24, 24, 54, 24, - 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 54, 54, 102, 4, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 17, 24, - 24, 17, 17, 4, 4, 54, 102, 102, 17, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, - 54, 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, - 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, - 54, 54, 54, 54, 54, 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 30, 29, 30, 29, 30, 29, 30, 103, 19, 19, 19, 19, 19, 19, 19, 19, 29, 30, + 29, 30, 391, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 104, 5, 5, 29, 30, + 392, 19, 55, 29, 30, 29, 30, 393, 19, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 394, 395, 396, 397, 394, 19, + 398, 399, 400, 401, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 402, 403, 404, 29, 30, 29, 30, 405, 29, 30, 0, 0, 29, 30, 0, + 19, 0, 19, 29, 30, 29, 30, 29, 30, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 103, 103, 29, 30, 55, 103, 103, 19, + 55, 55, 55, 55, 55, 55, 55, 24, 55, 55, 55, 24, 55, 55, 55, 55, 24, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 17, 17, 24, 24, 17, 4, 4, 4, 4, 24, 0, 0, 0, 26, 26, 26, + 26, 26, 26, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 17, + 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 24, 24, 0, 0, 0, 0, + 0, 0, 0, 0, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 55, 55, 55, 55, 55, 55, 4, 4, 4, 55, 4, 55, 55, 24, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, + 24, 24, 24, 24, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 24, 24, 24, 17, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 24, 17, 17, 24, 24, 24, 24, 17, 17, 24, 24, 17, + 17, 17, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 104, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 0, 0, 0, 0, 4, 4, 55, 55, 55, 55, 55, 24, 104, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 17, 17, + 24, 24, 17, 17, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 24, 55, + 55, 55, 55, 55, 55, 55, 55, 24, 17, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 0, 0, 4, 4, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 104, 55, 55, 55, 55, 55, 55, 4, 4, 4, 55, 17, 24, 17, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 55, 24, 24, 24, 55, + 55, 24, 24, 55, 55, 55, 55, 55, 24, 24, 55, 24, 55, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 104, 4, 4, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 24, 24, 17, 17, 4, 4, 55, + 104, 104, 17, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, + 0, 0, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, + 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 403, 19, 19, 19, 19, 19, 19, 19, 5, 101, 101, 101, - 101, 19, 19, 19, 19, 19, 19, 19, 19, 19, 101, 5, 5, 0, 0, 0, 0, 404, 405, - 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, - 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, - 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, - 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, - 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, - 476, 477, 478, 479, 480, 481, 482, 483, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 17, 17, 24, 17, 17, 24, 17, 17, 4, - 17, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 388, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 388, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 388, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, - 485, 486, 487, 488, 489, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, - 492, 493, 494, 495, 0, 0, 0, 0, 0, 54, 24, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, - 54, 54, 54, 54, 0, 54, 0, 54, 54, 0, 54, 54, 0, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 496, 496, 496, - 496, 496, 496, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 496, 496, 4, 4, 4, - 4, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 4, 4, - 4, 5, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 17, 17, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 17, 17, 17, 4, 4, 5, 0, - 4, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, - 0, 0, 0, 0, 496, 54, 496, 54, 496, 0, 496, 54, 496, 54, 496, 54, 496, 54, - 496, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 0, 0, 20, 0, 4, 4, 4, 4, 4, 4, 5, 4, 4, 4, 4, 4, 4, 5, 4, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, 5, 4, 4, 4, 4, 4, 4, 16, 16, 16, 16, 16, 16, 16, + 407, 19, 19, 19, 19, 19, 19, 19, 5, 103, 103, 103, 103, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 103, 5, 5, 0, 0, 0, 0, 408, 409, 410, 411, 412, 413, + 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, + 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, + 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, + 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, + 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, + 484, 485, 486, 487, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 17, 17, 24, 17, 17, 24, 17, 17, 4, 17, 24, 0, 0, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 390, 55, + 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 390, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 488, 489, 490, 491, 492, 493, 494, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 495, 496, 497, 498, 499, 0, 0, 0, 0, 0, 55, 24, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 0, 55, 0, 55, 55, 0, 55, 55, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 500, 500, 500, 500, 500, 500, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 500, 500, 4, 4, 4, 4, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 5, 4, 4, 4, 4, 4, 4, + 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 4, 4, 4, 17, 17, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 17, 17, 17, 4, 4, 5, 0, 4, 5, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 0, 0, 0, 0, 500, 55, 500, + 55, 500, 0, 500, 55, 500, 55, 500, 55, 500, 55, 500, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 0, 0, 20, 0, 4, 4, 4, 4, 4, 4, 5, 4, 4, 4, 4, 4, 4, 5, 4, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 5, 4, 4, 4, 4, 4, 4, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 16, 4, 4, 4, 5, 17, 5, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, - 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 17, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 102, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 497, 497, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 0, 0, 0, 54, 54, 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, 54, - 54, 0, 0, 54, 54, 54, 54, 54, 54, 0, 0, 54, 54, 54, 0, 0, 0, 4, 4, 4, 5, - 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 20, - 20, 4, 4, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 0, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 4, 4, 4, 0, 0, 0, - 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 16, 16, 16, 4, 4, 4, 5, 17, 5, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 501, 501, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 0, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, + 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 0, 0, 0, 4, + 4, 4, 5, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 20, 20, 20, 4, 4, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 0, 0, 0, 0, 0, 4, 4, 4, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 26, 26, 26, 26, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 26, 26, 4, 4, 4, 0, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 0, 0, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 242, 54, - 54, 54, 54, 54, 54, 54, 54, 242, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, 24, 24, - 24, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 4, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 4, 242, 242, 242, 242, 242, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 498, 498, 498, - 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, - 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, - 498, 498, 498, 498, 498, 498, 498, 498, 499, 499, 499, 499, 499, 499, - 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, - 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, - 499, 499, 499, 499, 499, 499, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 498, 498, 498, 498, 498, 498, 498, - 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, - 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, 498, - 498, 0, 0, 0, 0, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, - 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, - 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 499, 0, 0, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 500, 500, - 500, 500, 500, 500, 500, 500, 500, 500, 500, 0, 500, 500, 500, 500, 500, - 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 0, 500, 500, 500, 500, - 500, 500, 500, 0, 500, 500, 0, 501, 501, 501, 501, 501, 501, 501, 501, - 501, 501, 501, 0, 501, 501, 501, 501, 501, 501, 501, 501, 501, 501, 501, - 501, 501, 501, 501, 0, 501, 501, 501, 501, 501, 501, 501, 0, 501, 501, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, - 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 101, 102, 102, 101, 101, 101, 0, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 0, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 54, 54, 54, 54, 54, 54, 0, 0, 54, 0, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 0, 54, 54, 0, 0, 0, 54, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 4, 26, 26, 26, 26, - 26, 26, 26, 26, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 4, 4, 26, 26, 26, 26, 26, 26, 26, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 26, 26, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 26, 26, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 0, 0, 0, 0, 0, 26, 26, - 26, 26, 26, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 26, 26, 26, 26, 26, 26, 0, 0, 0, 4, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 26, 26, 54, 54, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 24, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 24, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 26, 26, 26, 26, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 244, 55, 55, 55, 55, 55, 55, 55, 55, 244, 0, + 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 4, 244, 244, 244, 244, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, + 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, + 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, + 502, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, + 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, + 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, + 0, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, + 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, + 502, 502, 502, 502, 502, 502, 502, 502, 0, 0, 0, 0, 503, 503, 503, 503, + 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, + 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, + 503, 503, 503, 503, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 4, 504, 504, 504, 504, 504, 504, 504, 504, 504, + 504, 504, 0, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, + 504, 504, 504, 0, 504, 504, 504, 504, 504, 504, 504, 0, 504, 504, 0, 505, + 505, 505, 505, 505, 505, 505, 505, 505, 505, 505, 0, 505, 505, 505, 505, + 505, 505, 505, 505, 505, 505, 505, 505, 505, 505, 505, 0, 505, 505, 505, + 505, 505, 505, 505, 0, 505, 505, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 104, 104, 103, 103, 103, 0, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 0, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, + 0, 0, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, 0, 0, 55, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 4, 26, 26, 26, 26, 26, 26, 26, 26, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 4, 4, 26, 26, 26, 26, 26, 26, 26, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 0, 55, 55, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 26, + 26, 26, 26, 26, 26, 0, 0, 0, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, + 0, 0, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 0, 0, 26, 26, 55, 55, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 54, 24, 24, 24, 0, 24, 24, 0, 0, 0, 0, 0, 24, 24, 24, 24, - 54, 54, 54, 54, 0, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 0, 0, 24, 24, 24, 0, 0, 0, 0, 24, 25, 21, 22, 360, 26, 26, 26, 26, - 26, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 26, 26, 4, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, - 54, 54, 54, 54, 54, 54, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, - 24, 0, 0, 0, 0, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 26, 26, 26, - 26, 26, 26, 26, 26, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, - 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, - 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, - 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, 108, - 108, 108, 108, 108, 108, 108, 108, 108, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, - 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, - 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, - 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 0, 0, 0, 0, 0, 0, - 0, 26, 26, 26, 26, 26, 26, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 25, 21, 22, 360, 361, 362, 363, 364, 365, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 24, 24, 4, 0, 0, 54, 54, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 24, 24, 24, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 54, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 26, 26, 26, 26, 4, 4, 4, 4, 4, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, - 24, 24, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 24, - 17, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 4, - 4, 4, 4, 0, 0, 0, 0, 25, 21, 22, 360, 361, 362, 363, 364, 365, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 24, 54, 54, 24, 24, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 17, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 17, 17, 17, 24, 24, 24, 24, 17, 17, 24, - 24, 4, 4, 20, 4, 4, 4, 4, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, - 14, 15, 0, 0, 0, 0, 0, 0, 24, 24, 24, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, 24, 24, 24, 17, 24, 24, 24, - 24, 24, 24, 24, 24, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 4, 4, 4, - 54, 17, 17, 54, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 4, 4, 54, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 24, 24, 17, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 17, 17, - 17, 24, 24, 24, 24, 24, 24, 24, 24, 24, 17, 17, 54, 54, 54, 54, 4, 4, 4, - 4, 24, 24, 24, 24, 4, 17, 24, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 54, 4, - 54, 4, 4, 4, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 17, 17, 17, 24, 24, 24, 17, 17, 24, 17, 24, 24, 4, 4, 4, - 4, 4, 4, 24, 54, 54, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, - 54, 54, 54, 54, 54, 0, 54, 0, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 4, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 24, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 24, 24, 17, 17, 0, 54, 54, - 54, 54, 54, 54, 54, 54, 0, 0, 54, 54, 0, 0, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, - 54, 54, 54, 54, 54, 0, 54, 54, 0, 54, 54, 54, 54, 54, 0, 24, 24, 54, 17, - 17, 24, 17, 17, 17, 17, 0, 0, 17, 17, 0, 0, 17, 17, 17, 0, 0, 54, 0, 0, - 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 17, 17, 0, 0, 24, 24, - 24, 24, 24, 24, 24, 0, 0, 0, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 17, 17, 24, 24, 24, 17, - 24, 54, 54, 54, 54, 4, 4, 4, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, - 4, 0, 4, 24, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 17, 17, 17, 24, 24, 24, 24, 24, 24, 17, 24, 17, 17, 17, - 17, 24, 24, 17, 24, 24, 54, 54, 4, 54, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 17, 17, 17, 24, 24, 24, 24, 0, 0, - 17, 17, 17, 17, 24, 24, 17, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 54, 54, 54, 54, 24, 24, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 17, 17, - 17, 24, 24, 24, 24, 24, 24, 24, 24, 17, 17, 24, 17, 24, 24, 4, 4, 4, 54, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, - 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, - 17, 24, 17, 17, 24, 24, 24, 24, 24, 24, 17, 24, 54, 4, 0, 0, 0, 0, 0, 0, - 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 0, 0, 24, 24, 24, 17, 17, 24, 24, 24, 24, 17, 24, 24, 24, + 26, 26, 55, 24, 24, 24, 0, 24, 24, 0, 0, 0, 0, 0, 24, 24, 24, 24, 55, 55, + 55, 55, 0, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, + 24, 24, 24, 0, 0, 0, 0, 24, 25, 21, 22, 362, 26, 26, 26, 26, 26, 0, 0, 0, + 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 26, 26, 4, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 0, 0, 0, + 0, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 26, 26, 26, 26, 26, 26, + 26, 26, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, + 0, 0, 0, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, + 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 110, 110, 110, 110, + 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, + 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, + 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, + 110, 110, 110, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, + 26, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 55, 55, 55, + 55, 104, 55, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 0, 0, 0, 24, 24, 24, 24, 24, 4, 104, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 21, 22, + 362, 363, 364, 365, 366, 367, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 24, 24, 4, 0, 0, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 55, 0, 0, + 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 26, 26, 26, 26, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 24, 17, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 25, 21, 22, 362, + 363, 364, 365, 366, 367, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 24, 55, 55, 24, 24, 55, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 24, 24, 24, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, + 17, 24, 24, 24, 24, 17, 17, 24, 24, 4, 4, 20, 4, 4, 4, 4, 24, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 20, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, + 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 24, 24, 24, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, + 24, 24, 24, 24, 17, 24, 24, 24, 24, 24, 24, 24, 24, 0, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 4, 4, 4, 4, 55, 17, 17, 55, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, + 4, 4, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 17, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 17, 17, 55, 55, 55, 55, 4, 4, 4, 4, 24, 24, 24, 24, 4, 17, 24, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 55, 4, 55, 4, 4, 4, 0, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, 24, 24, 24, + 17, 17, 24, 17, 24, 24, 4, 4, 4, 4, 4, 4, 24, 55, 55, 24, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 0, 55, 55, + 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 4, 0, 0, 0, 0, 0, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 24, 17, 17, 17, 24, 24, 24, 24, 24, 24, + 24, 24, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, + 0, 24, 24, 17, 17, 0, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, 55, 55, 55, + 55, 55, 0, 24, 24, 55, 17, 17, 24, 17, 17, 17, 17, 0, 0, 17, 17, 0, 0, + 17, 17, 17, 0, 0, 55, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 55, 55, 55, + 55, 55, 17, 17, 0, 0, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 24, 24, 24, + 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 0, 55, 0, 0, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 17, 17, 17, 24, 24, 24, 24, + 24, 24, 0, 17, 0, 0, 17, 0, 17, 17, 17, 17, 0, 17, 17, 24, 17, 24, 55, + 24, 55, 4, 4, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, + 17, 24, 24, 24, 24, 24, 24, 24, 24, 17, 17, 24, 24, 24, 17, 24, 55, 55, + 55, 55, 4, 4, 4, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 4, 0, 4, + 24, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 17, 17, 17, 24, 24, 24, 24, 24, 24, 17, 24, 17, 17, 17, 17, 24, + 24, 17, 24, 24, 55, 55, 4, 55, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, 24, 24, 24, 24, 0, 0, 17, 17, + 17, 17, 24, 24, 17, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 55, 55, 55, 55, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, 24, 24, + 24, 24, 24, 24, 24, 24, 17, 17, 24, 17, 24, 24, 4, 4, 4, 55, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 17, 24, 17, + 17, 24, 24, 24, 24, 24, 24, 17, 24, 55, 4, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 24, 17, 24, 17, 17, 24, 24, 24, 24, 17, 24, 24, 24, 24, 24, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 26, 26, 4, 4, 4, - 4, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 24, 17, 24, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 24, 17, 24, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, - 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 54, 0, 0, 54, 54, - 54, 54, 54, 54, 54, 54, 0, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 17, 17, 17, - 17, 17, 17, 0, 17, 17, 0, 0, 24, 24, 17, 24, 54, 17, 54, 17, 24, 4, 4, 4, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 17, 17, 17, 24, 24, 24, 24, 0, 0, 24, 24, - 17, 17, 17, 17, 24, 54, 4, 54, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 24, 24, 24, 24, 24, 24, 17, 54, 24, 24, 24, - 24, 4, 4, 4, 4, 4, 4, 4, 4, 24, 0, 0, 0, 0, 0, 0, 0, 0, 54, 24, 24, 24, - 24, 24, 24, 17, 17, 24, 24, 24, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 17, 24, 24, 4, 4, 4, - 54, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, - 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 17, 24, 24, 24, 24, 24, 24, 24, 0, 24, 24, 24, 24, 24, 24, 17, 24, - 54, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, - 13, 14, 15, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 0, 0, 0, 4, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 0, 17, 24, 24, 24, 24, 24, 24, 24, 17, 24, - 24, 17, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 0, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, - 24, 24, 24, 24, 24, 0, 0, 0, 24, 0, 24, 24, 0, 24, 24, 24, 24, 24, 24, - 24, 54, 24, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 0, 54, 54, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 17, 17, 17, 17, 17, 0, 24, 24, 0, - 17, 17, 24, 17, 24, 54, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, - 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, 17, 17, 4, 4, - 0, 0, 0, 0, 0, 0, 0, 24, 24, 54, 17, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 17, 17, 24, 24, 24, 24, 24, 0, 0, 0, 17, 17, 24, 17, 24, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, - 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 242, 0, 4, 4, 4, 4, 4, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 4, 4, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 24, - 54, 54, 54, 54, 54, 54, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, - 0, 0, 4, 4, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 6, 7, 8, 9, 10, 11, 12, 13, - 14, 15, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 0, 0, 24, 24, 24, 24, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 102, 102, 102, 102, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 26, 26, 26, 26, 26, 26, 26, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 18, 18, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, + 55, 0, 0, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 17, 17, 17, 17, 17, 17, 0, 17, 17, 0, 0, 24, 24, 17, 24, + 55, 17, 55, 17, 24, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, + 24, 24, 24, 24, 0, 0, 24, 24, 17, 17, 17, 17, 24, 55, 4, 55, 17, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, + 24, 24, 24, 17, 55, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 24, 0, 0, 0, + 0, 0, 0, 0, 0, 55, 24, 24, 24, 24, 24, 24, 17, 17, 24, 24, 24, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 17, 24, 24, 4, 4, 4, 55, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 17, 24, 24, 24, 24, 24, 24, 24, 0, 24, 24, 24, 24, 24, 24, 17, 24, 55, 4, + 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 0, 0, 0, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 0, 17, 24, 24, 24, 24, 24, 24, 24, 17, 24, 24, 17, + 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 24, 24, 24, 24, 24, 24, 0, 0, 0, 24, 0, 24, 24, 0, 24, 24, + 24, 24, 24, 24, 24, 55, 24, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, 17, + 17, 0, 24, 24, 0, 17, 17, 24, 17, 24, 55, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 17, 17, 4, 4, 0, + 0, 0, 0, 0, 0, 0, 24, 24, 55, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 17, 17, 24, 24, 24, 24, 24, 0, 0, 0, 17, 17, 24, 17, 24, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 24, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 4, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 0, + 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 24, 55, 55, 55, 55, 55, 55, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 17, 17, 17, 24, 24, 24, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 4, 4, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 24, 24, 24, 24, + 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 104, + 104, 104, 104, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 0, 26, 26, 26, 26, 26, 26, 26, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 104, 104, 104, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, 104, 4, 4, 4, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 16, 16, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 16, 16, 16, 16, 16, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, - 18, 18, 18, 18, 18, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, + 18, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 0, 0, 0, 0, 24, 54, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 24, 55, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 102, 4, 102, 24, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 102, 102, 102, 0, 102, - 102, 102, 102, 102, 102, 102, 0, 102, 102, 0, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 0, 0, 4, 24, 24, 4, 20, 20, 20, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 104, 104, 104, 104, 104, 104, 104, + 104, 104, 104, 104, 104, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 104, 104, 4, 104, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 17, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, + 104, 104, 104, 0, 104, 104, 104, 104, 104, 104, 104, 0, 104, 104, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 0, 0, + 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 0, 0, 0, 0, + 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, + 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 4, 24, 24, 4, 20, 20, 20, 20, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 24, 24, 24, 24, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 17, 17, 24, 24, 24, 4, 4, 4, 17, 17, - 17, 17, 17, 17, 20, 20, 20, 20, 20, 20, 20, 20, 24, 24, 24, 24, 24, 24, - 24, 24, 4, 4, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, 24, - 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 17, 17, 24, 24, 24, 4, 4, 4, 17, 17, 17, + 17, 17, 17, 20, 20, 20, 20, 20, 20, 20, 20, 24, 24, 24, 24, 24, 24, 24, + 24, 4, 4, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 24, 24, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 24, 24, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 0, 0, 0, 0, 0, 0, 0, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 19, 19, 19, 19, 19, 19, 19, 0, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 19, 19, 19, 19, 19, 19, + 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, + 0, 0, 0, 0, 0, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 19, 19, 19, 19, 19, 19, 19, 0, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 120, 0, 120, 120, 0, 0, 120, 0, 0, 120, 120, 0, 0, 120, 120, 120, - 120, 0, 120, 120, 120, 120, 120, 120, 120, 120, 19, 19, 19, 19, 0, 19, 0, + 19, 122, 0, 122, 122, 0, 0, 122, 0, 0, 122, 122, 0, 0, 122, 122, 122, + 122, 0, 122, 122, 122, 122, 122, 122, 122, 122, 19, 19, 19, 19, 0, 19, 0, 19, 19, 19, 19, 19, 19, 19, 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 19, 19, 19, + 19, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 120, 120, 0, 120, 120, 120, 120, 0, 0, 120, 120, 120, - 120, 120, 120, 120, 120, 0, 120, 120, 120, 120, 120, 120, 120, 0, 19, 19, + 19, 19, 19, 19, 19, 122, 122, 0, 122, 122, 122, 122, 0, 0, 122, 122, 122, + 122, 122, 122, 122, 122, 0, 122, 122, 122, 122, 122, 122, 122, 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 120, 120, 0, 120, 120, 120, 120, 0, 120, 120, - 120, 120, 120, 0, 120, 0, 0, 0, 120, 120, 120, 120, 120, 120, 120, 0, 19, + 19, 19, 19, 19, 19, 19, 122, 122, 0, 122, 122, 122, 122, 0, 122, 122, + 122, 122, 122, 0, 122, 0, 0, 0, 122, 122, 122, 122, 122, 122, 122, 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 0, 0, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 4, + 19, 19, 19, 0, 0, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 4, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 4, 19, 19, 19, 19, 19, 19, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 4, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 4, 19, 19, 19, 19, 19, 19, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 4, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 4, 19, 19, 19, 19, 19, 19, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 4, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 4, 19, 19, 19, 19, 19, 19, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 4, 19, 19, 19, 19, 19, + 4, 19, 19, 19, 19, 19, 19, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 4, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 4, 19, 19, 19, 19, 19, 19, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 4, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 4, 19, 19, 19, 19, 19, 19, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 4, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 4, 19, 19, 19, 19, 19, 19, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 122, 4, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 4, 19, 19, 19, 19, - 19, 19, 120, 19, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 6, 7, 8, 9, + 19, 19, 122, 19, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, @@ -3973,297 +4312,188 @@ static const unsigned short index2[] = { 4, 4, 4, 4, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 54, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 0, 0, - 0, 0, 0, 0, 19, 19, 19, 19, 19, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 24, 24, 24, 24, 24, 24, 24, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 0, 0, 24, 24, 24, 24, 24, 24, 24, 0, 24, 24, - 0, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 102, - 102, 102, 102, 102, 102, 102, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 0, 0, 0, 54, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 24, 24, 24, 24, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 102, 24, 24, 24, 24, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, - 54, 54, 54, 0, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 24, 24, 24, 24, 24, 24, 24, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 502, 502, 502, - 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, - 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, - 502, 502, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, - 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, - 503, 503, 503, 503, 503, 503, 503, 503, 24, 24, 24, 24, 24, 24, 24, 102, - 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 4, 4, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 55, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 0, 0, 0, 0, 0, 0, 19, 19, 19, 19, 19, 19, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, + 24, 24, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 0, 0, 24, 24, 24, 24, 24, 24, 24, 0, 24, 24, 0, 24, 24, 24, 24, + 24, 0, 0, 0, 0, 0, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 24, 24, 24, 24, 24, 24, + 24, 104, 104, 104, 104, 104, 104, 104, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 0, 0, 0, 0, 55, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, + 24, 24, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 104, 24, 24, 24, 24, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 24, 24, 55, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 0, + 55, 55, 55, 55, 0, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 0, 0, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, + 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, + 506, 506, 506, 506, 506, 506, 506, 506, 506, 507, 507, 507, 507, 507, + 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, + 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, + 507, 24, 24, 24, 24, 24, 24, 24, 104, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 4, 26, 26, 26, 4, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, + 26, 4, 26, 26, 26, 4, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 4, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, - 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 0, 54, 0, - 0, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 0, - 54, 0, 54, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 54, 0, 54, 0, 54, 0, 54, 54, - 54, 0, 54, 54, 0, 54, 0, 0, 54, 0, 54, 0, 54, 0, 54, 0, 54, 0, 54, 54, 0, - 54, 0, 0, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, 54, 0, 54, 54, 54, - 54, 0, 54, 54, 54, 54, 0, 54, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, - 0, 0, 0, 0, 54, 54, 54, 0, 54, 54, 54, 54, 54, 0, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 359, 359, 25, 21, 22, 360, 361, 362, 363, 364, 365, 26, 26, 4, 4, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 0, 0, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 0, 55, 0, 0, 55, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, 55, 0, 55, 0, 0, 0, 0, 0, 0, 55, 0, + 0, 0, 0, 55, 0, 55, 0, 55, 0, 55, 55, 55, 0, 55, 55, 0, 55, 0, 0, 55, 0, + 55, 0, 55, 0, 55, 0, 55, 0, 55, 55, 0, 55, 0, 0, 55, 55, 55, 55, 0, 55, + 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, 55, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 55, 55, 55, 0, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 361, 361, 25, 21, 22, 362, 363, 364, 365, 366, 367, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 504, 504, 504, 504, 504, 504, 504, 504, 504, - 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, - 504, 504, 504, 4, 4, 4, 4, 4, 4, 504, 504, 504, 504, 504, 504, 504, 504, - 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, - 504, 504, 504, 504, 4, 4, 4, 4, 4, 4, 504, 504, 504, 504, 504, 504, 504, - 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, - 504, 504, 504, 504, 504, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, + 4, 4, 4, 4, 4, 4, 4, 4, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, + 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, + 508, 508, 4, 4, 4, 4, 4, 4, 508, 508, 508, 508, 508, 508, 508, 508, 508, + 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, + 508, 508, 508, 4, 4, 4, 4, 4, 4, 508, 508, 508, 508, 508, 508, 508, 508, + 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, + 508, 508, 508, 508, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, + 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, - 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 4, 4, + 4, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, + 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, + 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, + 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, - 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, - 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, + 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 54, 388, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 388, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 388, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 0, 0, 0, 0, 0, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 55, 390, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, @@ -4273,19 +4503,8 @@ static const unsigned short index2[] = { 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; /* Returns the numeric value as double for Unicode characters @@ -4348,6 +4567,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1018A: case 0x104A0: case 0x10D30: + case 0x10D40: case 0x11066: case 0x110F0: case 0x11136: @@ -4357,17 +4577,23 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x114D0: case 0x11650: case 0x116C0: + case 0x116D0: + case 0x116DA: case 0x11730: case 0x118E0: case 0x11950: + case 0x11BF0: case 0x11C50: case 0x11D50: case 0x11DA0: case 0x11F50: + case 0x16130: case 0x16A60: case 0x16AC0: case 0x16B50: + case 0x16D70: case 0x16E80: + case 0x1CCF0: case 0x1D2C0: case 0x1D2E0: case 0x1D7CE: @@ -4378,6 +4604,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1E140: case 0x1E2F0: case 0x1E4F0: + case 0x1E5F1: case 0x1E950: case 0x1F100: case 0x1F101: @@ -4473,6 +4700,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x10BA9: case 0x10CFA: case 0x10D31: + case 0x10D41: case 0x10E60: case 0x10F1D: case 0x10F51: @@ -4488,9 +4716,12 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x114D1: case 0x11651: case 0x116C1: + case 0x116D1: + case 0x116DB: case 0x11731: case 0x118E1: case 0x11951: + case 0x11BF1: case 0x11C51: case 0x11C5A: case 0x11D51: @@ -4502,11 +4733,14 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x12434: case 0x1244F: case 0x12458: + case 0x16131: case 0x16A61: case 0x16AC1: case 0x16B51: + case 0x16D71: case 0x16E81: case 0x16E94: + case 0x1CCF1: case 0x1D2C1: case 0x1D2E1: case 0x1D360: @@ -4520,6 +4754,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1E141: case 0x1E2F1: case 0x1E4F1: + case 0x1E5F2: case 0x1E8C7: case 0x1E951: case 0x1EC71: @@ -4959,6 +5194,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x10B79: case 0x10BAA: case 0x10D32: + case 0x10D42: case 0x10E61: case 0x10F1E: case 0x10FC6: @@ -4973,9 +5209,12 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x114D2: case 0x11652: case 0x116C2: + case 0x116D2: + case 0x116DC: case 0x11732: case 0x118E2: case 0x11952: + case 0x11BF2: case 0x11C52: case 0x11C5B: case 0x11D52: @@ -4991,11 +5230,14 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x12450: case 0x12456: case 0x12459: + case 0x16132: case 0x16A62: case 0x16AC2: case 0x16B52: + case 0x16D72: case 0x16E82: case 0x16E95: + case 0x1CCF2: case 0x1D2C2: case 0x1D2E2: case 0x1D361: @@ -5008,6 +5250,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1E142: case 0x1E2F2: case 0x1E4F2: + case 0x1E5F3: case 0x1E8C8: case 0x1E952: case 0x1EC72: @@ -5189,6 +5432,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x10B7A: case 0x10BAB: case 0x10D33: + case 0x10D43: case 0x10E62: case 0x10F1F: case 0x10FC7: @@ -5203,9 +5447,12 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x114D3: case 0x11653: case 0x116C3: + case 0x116D3: + case 0x116DD: case 0x11733: case 0x118E3: case 0x11953: + case 0x11BF3: case 0x11C53: case 0x11C5C: case 0x11D53: @@ -5226,11 +5473,14 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1244B: case 0x12451: case 0x12457: + case 0x16133: case 0x16A63: case 0x16AC3: case 0x16B53: + case 0x16D73: case 0x16E83: case 0x16E96: + case 0x1CCF3: case 0x1D2C3: case 0x1D2E3: case 0x1D362: @@ -5243,6 +5493,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1E143: case 0x1E2F3: case 0x1E4F3: + case 0x1E5F4: case 0x1E8C9: case 0x1E953: case 0x1EC73: @@ -5415,6 +5666,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x10B7B: case 0x10BAC: case 0x10D34: + case 0x10D44: case 0x10E63: case 0x10F20: case 0x10FC8: @@ -5429,9 +5681,12 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x114D4: case 0x11654: case 0x116C4: + case 0x116D4: + case 0x116DE: case 0x11734: case 0x118E4: case 0x11954: + case 0x11BF4: case 0x11C54: case 0x11C5D: case 0x11D54: @@ -5453,10 +5708,13 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x12452: case 0x12453: case 0x12469: + case 0x16134: case 0x16A64: case 0x16AC4: case 0x16B54: + case 0x16D74: case 0x16E84: + case 0x1CCF4: case 0x1D2C4: case 0x1D2E4: case 0x1D363: @@ -5469,6 +5727,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1E144: case 0x1E2F4: case 0x1E4F4: + case 0x1E5F5: case 0x1E8CA: case 0x1E954: case 0x1EC74: @@ -5618,6 +5877,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x10AEC: case 0x10CFB: case 0x10D35: + case 0x10D45: case 0x10E64: case 0x10F21: case 0x11056: @@ -5631,9 +5891,12 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x114D5: case 0x11655: case 0x116C5: + case 0x116D5: + case 0x116DF: case 0x11735: case 0x118E5: case 0x11955: + case 0x11BF5: case 0x11C55: case 0x11C5E: case 0x11D55: @@ -5651,10 +5914,13 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x12454: case 0x12455: case 0x1246A: + case 0x16135: case 0x16A65: case 0x16AC5: case 0x16B55: + case 0x16D75: case 0x16E85: + case 0x1CCF5: case 0x1D2C5: case 0x1D2E5: case 0x1D364: @@ -5668,6 +5934,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1E145: case 0x1E2F5: case 0x1E4F5: + case 0x1E5F6: case 0x1E8CB: case 0x1E955: case 0x1EC75: @@ -5818,6 +6085,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x104A6: case 0x109C5: case 0x10D36: + case 0x10D46: case 0x10E65: case 0x11057: case 0x1106C: @@ -5830,9 +6098,12 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x114D6: case 0x11656: case 0x116C6: + case 0x116D6: + case 0x116E0: case 0x11736: case 0x118E6: case 0x11956: + case 0x11BF6: case 0x11C56: case 0x11C5F: case 0x11D56: @@ -5846,10 +6117,13 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x12440: case 0x1244E: case 0x1246B: + case 0x16136: case 0x16A66: case 0x16AC6: case 0x16B56: + case 0x16D76: case 0x16E86: + case 0x1CCF6: case 0x1D2C6: case 0x1D2E6: case 0x1D365: @@ -5861,6 +6135,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1E146: case 0x1E2F6: case 0x1E4F6: + case 0x1E5F7: case 0x1E8CC: case 0x1E956: case 0x1EC76: @@ -5971,6 +6246,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x104A7: case 0x109C6: case 0x10D37: + case 0x10D47: case 0x10E66: case 0x11058: case 0x1106D: @@ -5983,9 +6259,12 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x114D7: case 0x11657: case 0x116C7: + case 0x116D7: + case 0x116E1: case 0x11737: case 0x118E7: case 0x11957: + case 0x11BF7: case 0x11C57: case 0x11C60: case 0x11D57: @@ -6000,10 +6279,13 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x12442: case 0x12443: case 0x1246C: + case 0x16137: case 0x16A67: case 0x16AC7: case 0x16B57: + case 0x16D77: case 0x16E87: + case 0x1CCF7: case 0x1D2C7: case 0x1D2E7: case 0x1D366: @@ -6015,6 +6297,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1E147: case 0x1E2F7: case 0x1E4F7: + case 0x1E5F8: case 0x1E8CD: case 0x1E957: case 0x1EC77: @@ -6125,6 +6408,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x104A8: case 0x109C7: case 0x10D38: + case 0x10D48: case 0x10E67: case 0x11059: case 0x1106E: @@ -6137,9 +6421,12 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x114D8: case 0x11658: case 0x116C8: + case 0x116D8: + case 0x116E2: case 0x11738: case 0x118E8: case 0x11958: + case 0x11BF8: case 0x11C58: case 0x11C61: case 0x11D58: @@ -6153,10 +6440,13 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x12444: case 0x12445: case 0x1246D: + case 0x16138: case 0x16A68: case 0x16AC8: case 0x16B58: + case 0x16D78: case 0x16E88: + case 0x1CCF8: case 0x1D2C8: case 0x1D2E8: case 0x1D367: @@ -6168,6 +6458,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1E148: case 0x1E2F8: case 0x1E4F8: + case 0x1E5F9: case 0x1E8CE: case 0x1E958: case 0x1EC78: @@ -6275,6 +6566,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x104A9: case 0x109C8: case 0x10D39: + case 0x10D49: case 0x10E68: case 0x1105A: case 0x1106F: @@ -6287,9 +6579,12 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x114D9: case 0x11659: case 0x116C9: + case 0x116D9: + case 0x116E3: case 0x11739: case 0x118E9: case 0x11959: + case 0x11BF9: case 0x11C59: case 0x11C62: case 0x11D59: @@ -6305,10 +6600,13 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x12448: case 0x12449: case 0x1246E: + case 0x16139: case 0x16A69: case 0x16AC9: case 0x16B59: + case 0x16D79: case 0x16E89: + case 0x1CCF9: case 0x1D2C9: case 0x1D2E9: case 0x1D368: @@ -6320,6 +6618,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1E149: case 0x1E2F9: case 0x1E4F9: + case 0x1E5FA: case 0x1E8CF: case 0x1E959: case 0x1EC79: diff --git a/Objects/unionobject.c b/Objects/unionobject.c index bf5605686f8df7..6e65a653a95c46 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -1,11 +1,10 @@ // types.UnionType -- used to represent e.g. Union[int, str], int | str #include "Python.h" #include "pycore_object.h" // _PyObject_GC_TRACK/UNTRACK -#include "pycore_typevarobject.h" // _PyTypeAlias_Type +#include "pycore_typevarobject.h" // _PyTypeAlias_Type, _Py_typing_type_repr #include "pycore_unionobject.h" - static PyObject *make_union(PyObject *); @@ -81,7 +80,7 @@ is_same(PyObject *left, PyObject *right) static int contains(PyObject **items, Py_ssize_t size, PyObject *obj) { - for (int i = 0; i < size; i++) { + for (Py_ssize_t i = 0; i < size; i++) { int is_duplicate = is_same(items[i], obj); if (is_duplicate) { // -1 or 1 return is_duplicate; @@ -97,7 +96,7 @@ merge(PyObject **items1, Py_ssize_t size1, PyObject *tuple = NULL; Py_ssize_t pos = 0; - for (int i = 0; i < size2; i++) { + for (Py_ssize_t i = 0; i < size2; i++) { PyObject *arg = items2[i]; int is_duplicate = contains(items1, size1, arg); if (is_duplicate < 0) { @@ -181,87 +180,32 @@ _Py_union_type_or(PyObject* self, PyObject* other) return new_union; } -static int -union_repr_item(_PyUnicodeWriter *writer, PyObject *p) -{ - PyObject *qualname = NULL; - PyObject *module = NULL; - PyObject *r = NULL; - int rc; - - if (p == (PyObject *)&_PyNone_Type) { - return _PyUnicodeWriter_WriteASCIIString(writer, "None", 4); - } - - if ((rc = PyObject_HasAttrWithError(p, &_Py_ID(__origin__))) > 0 && - (rc = PyObject_HasAttrWithError(p, &_Py_ID(__args__))) > 0) - { - // It looks like a GenericAlias - goto use_repr; - } - if (rc < 0) { - goto exit; - } - - if (PyObject_GetOptionalAttr(p, &_Py_ID(__qualname__), &qualname) < 0) { - goto exit; - } - if (qualname == NULL) { - goto use_repr; - } - if (PyObject_GetOptionalAttr(p, &_Py_ID(__module__), &module) < 0) { - goto exit; - } - if (module == NULL || module == Py_None) { - goto use_repr; - } - - // Looks like a class - if (PyUnicode_Check(module) && - _PyUnicode_EqualToASCIIString(module, "builtins")) - { - // builtins don't need a module name - r = PyObject_Str(qualname); - goto exit; - } - else { - r = PyUnicode_FromFormat("%S.%S", module, qualname); - goto exit; - } - -use_repr: - r = PyObject_Repr(p); -exit: - Py_XDECREF(qualname); - Py_XDECREF(module); - if (r == NULL) { - return -1; - } - rc = _PyUnicodeWriter_WriteStr(writer, r); - Py_DECREF(r); - return rc; -} - static PyObject * union_repr(PyObject *self) { unionobject *alias = (unionobject *)self; Py_ssize_t len = PyTuple_GET_SIZE(alias->args); - _PyUnicodeWriter writer; - _PyUnicodeWriter_Init(&writer); - for (Py_ssize_t i = 0; i < len; i++) { - if (i > 0 && _PyUnicodeWriter_WriteASCIIString(&writer, " | ", 3) < 0) { + // Shortest type name "int" (3 chars) + " | " (3 chars) separator + Py_ssize_t estimate = (len <= PY_SSIZE_T_MAX / 6) ? len * 6 : len; + PyUnicodeWriter *writer = PyUnicodeWriter_Create(estimate); + if (writer == NULL) { + return NULL; + } + + for (Py_ssize_t i = 0; i < len; i++) { + if (i > 0 && PyUnicodeWriter_WriteUTF8(writer, " | ", 3) < 0) { goto error; } PyObject *p = PyTuple_GET_ITEM(alias->args, i); - if (union_repr_item(&writer, p) < 0) { + if (_Py_typing_type_repr(writer, p) < 0) { goto error; } } - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); + error: - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); return NULL; } diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index df74be6aba7244..0ee64ed70a63cd 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -1,24 +1,58 @@ #include "Python.h" +#include "pycore_critical_section.h" +#include "pycore_lock.h" #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GET_WEAKREFS_LISTPTR() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_pystate.h" #include "pycore_weakref.h" // _PyWeakref_GET_REF() +#ifdef Py_GIL_DISABLED +/* + * Thread-safety for free-threaded builds + * ====================================== + * + * In free-threaded builds we need to protect mutable state of: + * + * - The weakref (wr_object, hash, wr_callback) + * - The referenced object (its head-of-list pointer) + * - The linked list of weakrefs + * + * For now we've chosen to address this in a straightforward way: + * + * - The weakref's hash is protected using the weakref's per-object lock. + * - The other mutable is protected by a striped lock keyed on the referenced + * object's address. + * - The striped lock must be locked using `_Py_LOCK_DONT_DETACH` in order to + * support atomic deletion from WeakValueDictionaries. As a result, we must + * be careful not to perform any operations that could suspend while the + * lock is held. + * + * Since the world is stopped when the GC runs, it is free to clear weakrefs + * without acquiring any locks. + */ +#endif #define GET_WEAKREFS_LISTPTR(o) \ ((PyWeakReference **) _PyObject_GET_WEAKREFS_LISTPTR(o)) Py_ssize_t -_PyWeakref_GetWeakrefCount(PyWeakReference *head) +_PyWeakref_GetWeakrefCount(PyObject *obj) { - Py_ssize_t count = 0; + if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(obj))) { + return 0; + } + LOCK_WEAKREFS(obj); + Py_ssize_t count = 0; + PyWeakReference *head = *GET_WEAKREFS_LISTPTR(obj); while (head != NULL) { ++count; head = head->wr_next; } + UNLOCK_WEAKREFS(obj); return count; } @@ -33,54 +67,57 @@ init_weakref(PyWeakReference *self, PyObject *ob, PyObject *callback) self->wr_next = NULL; self->wr_callback = Py_XNewRef(callback); self->vectorcall = weakref_vectorcall; +#ifdef Py_GIL_DISABLED + self->weakrefs_lock = &WEAKREF_LIST_LOCK(ob); + _PyObject_SetMaybeWeakref(ob); + _PyObject_SetMaybeWeakref((PyObject *)self); +#endif } -static PyWeakReference * -new_weakref(PyObject *ob, PyObject *callback) -{ - PyWeakReference *result; - - result = PyObject_GC_New(PyWeakReference, &_PyWeakref_RefType); - if (result) { - init_weakref(result, ob, callback); - PyObject_GC_Track(result); - } - return result; -} - - -/* This function clears the passed-in reference and removes it from the - * list of weak references for the referent. This is the only code that - * removes an item from the doubly-linked list of weak references for an - * object; it is also responsible for clearing the callback slot. - */ +// Clear the weakref and steal its callback into `callback`, if provided. static void -clear_weakref(PyWeakReference *self) +clear_weakref_lock_held(PyWeakReference *self, PyObject **callback) { - PyObject *callback = self->wr_callback; - if (self->wr_object != Py_None) { PyWeakReference **list = GET_WEAKREFS_LISTPTR(self->wr_object); - - if (*list == self) - /* If 'self' is the end of the list (and thus self->wr_next == NULL) - then the weakref list itself (and thus the value of *list) will - end up being set to NULL. */ - *list = self->wr_next; - self->wr_object = Py_None; - if (self->wr_prev != NULL) + if (*list == self) { + /* If 'self' is the end of the list (and thus self->wr_next == + NULL) then the weakref list itself (and thus the value of *list) + will end up being set to NULL. */ + FT_ATOMIC_STORE_PTR(*list, self->wr_next); + } + FT_ATOMIC_STORE_PTR(self->wr_object, Py_None); + if (self->wr_prev != NULL) { self->wr_prev->wr_next = self->wr_next; - if (self->wr_next != NULL) + } + if (self->wr_next != NULL) { self->wr_next->wr_prev = self->wr_prev; + } self->wr_prev = NULL; self->wr_next = NULL; } if (callback != NULL) { - Py_DECREF(callback); + *callback = self->wr_callback; self->wr_callback = NULL; } } +// Clear the weakref and its callback +static void +clear_weakref(PyObject *op) +{ + PyWeakReference *self = _PyWeakref_CAST(op); + PyObject *callback = NULL; + + // self->wr_object may be Py_None if the GC cleared the weakref, so lock + // using the pointer in the weakref. + LOCK_WEAKREFS_FOR_WR(self); + clear_weakref_lock_held(self, &callback); + UNLOCK_WEAKREFS_FOR_WR(self); + Py_XDECREF(callback); +} + + /* Cyclic gc uses this to *just* clear the passed-in reference, leaving * the callback intact and uncalled. It must be possible to call self's * tp_dealloc() after calling this, so self has to be left in a sane enough @@ -95,38 +132,38 @@ clear_weakref(PyWeakReference *self) void _PyWeakref_ClearRef(PyWeakReference *self) { - PyObject *callback; - assert(self != NULL); assert(PyWeakref_Check(self)); - /* Preserve and restore the callback around clear_weakref. */ - callback = self->wr_callback; - self->wr_callback = NULL; - clear_weakref(self); - self->wr_callback = callback; + clear_weakref_lock_held(self, NULL); } static void weakref_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); - clear_weakref((PyWeakReference *) self); + clear_weakref(self); Py_TYPE(self)->tp_free(self); } static int -gc_traverse(PyWeakReference *self, visitproc visit, void *arg) +gc_traverse(PyObject *op, visitproc visit, void *arg) { + PyWeakReference *self = _PyWeakref_CAST(op); Py_VISIT(self->wr_callback); return 0; } static int -gc_clear(PyWeakReference *self) +gc_clear(PyObject *op) { - clear_weakref(self); + PyWeakReference *self = _PyWeakref_CAST(op); + PyObject *callback; + // The world is stopped during GC in free-threaded builds. It's safe to + // call this without holding the lock. + clear_weakref_lock_held(self, &callback); + Py_XDECREF(callback); return 0; } @@ -150,7 +187,7 @@ weakref_vectorcall(PyObject *self, PyObject *const *args, } static Py_hash_t -weakref_hash(PyWeakReference *self) +weakref_hash_lock_held(PyWeakReference *self) { if (self->hash != -1) return self->hash; @@ -164,6 +201,16 @@ weakref_hash(PyWeakReference *self) return self->hash; } +static Py_hash_t +weakref_hash(PyObject *op) +{ + PyWeakReference *self = _PyWeakref_CAST(op); + Py_hash_t hash; + Py_BEGIN_CRITICAL_SECTION(self); + hash = weakref_hash_lock_held(self); + Py_END_CRITICAL_SECTION(); + return hash; +} static PyObject * weakref_repr(PyObject *self) @@ -177,13 +224,13 @@ weakref_repr(PyObject *self) PyObject *repr; if (name == NULL || !PyUnicode_Check(name)) { repr = PyUnicode_FromFormat( - "", - self, Py_TYPE(obj)->tp_name, obj); + "", + self, obj, obj); } else { repr = PyUnicode_FromFormat( - "", - self, Py_TYPE(obj)->tp_name, obj, name); + "", + self, obj, obj, name); } Py_DECREF(obj); Py_XDECREF(name); @@ -276,6 +323,135 @@ insert_head(PyWeakReference *newref, PyWeakReference **list) *list = newref; } +/* See if we can reuse either the basic ref or proxy in list instead of + * creating a new weakref + */ +static PyWeakReference * +try_reuse_basic_ref(PyWeakReference *list, PyTypeObject *type, + PyObject *callback) +{ + if (callback != NULL) { + return NULL; + } + + PyWeakReference *ref, *proxy; + get_basic_refs(list, &ref, &proxy); + + PyWeakReference *cand = NULL; + if (type == &_PyWeakref_RefType) { + cand = ref; + } + if ((type == &_PyWeakref_ProxyType) || + (type == &_PyWeakref_CallableProxyType)) { + cand = proxy; + } + + if (cand != NULL && _Py_TryIncref((PyObject *) cand)) { + return cand; + } + return NULL; +} + +static int +is_basic_ref(PyWeakReference *ref) +{ + return (ref->wr_callback == NULL) && PyWeakref_CheckRefExact(ref); +} + +static int +is_basic_proxy(PyWeakReference *proxy) +{ + return (proxy->wr_callback == NULL) && PyWeakref_CheckProxy(proxy); +} + +static int +is_basic_ref_or_proxy(PyWeakReference *wr) +{ + return is_basic_ref(wr) || is_basic_proxy(wr); +} + +/* Insert `newref` in the appropriate position in `list` */ +static void +insert_weakref(PyWeakReference *newref, PyWeakReference **list) +{ + PyWeakReference *ref, *proxy; + get_basic_refs(*list, &ref, &proxy); + + PyWeakReference *prev; + if (is_basic_ref(newref)) { + prev = NULL; + } + else if (is_basic_proxy(newref)) { + prev = ref; + } + else { + prev = (proxy == NULL) ? ref : proxy; + } + + if (prev == NULL) { + insert_head(newref, list); + } + else { + insert_after(newref, prev); + } +} + +static PyWeakReference * +allocate_weakref(PyTypeObject *type, PyObject *obj, PyObject *callback) +{ + PyWeakReference *newref = (PyWeakReference *) type->tp_alloc(type, 0); + if (newref == NULL) { + return NULL; + } + init_weakref(newref, obj, callback); + return newref; +} + +static PyWeakReference * +get_or_create_weakref(PyTypeObject *type, PyObject *obj, PyObject *callback) +{ + if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(obj))) { + PyErr_Format(PyExc_TypeError, + "cannot create weak reference to '%s' object", + Py_TYPE(obj)->tp_name); + return NULL; + } + if (callback == Py_None) + callback = NULL; + + PyWeakReference **list = GET_WEAKREFS_LISTPTR(obj); + if ((type == &_PyWeakref_RefType) || + (type == &_PyWeakref_ProxyType) || + (type == &_PyWeakref_CallableProxyType)) + { + LOCK_WEAKREFS(obj); + PyWeakReference *basic_ref = try_reuse_basic_ref(*list, type, callback); + if (basic_ref != NULL) { + UNLOCK_WEAKREFS(obj); + return basic_ref; + } + PyWeakReference *newref = allocate_weakref(type, obj, callback); + if (newref == NULL) { + UNLOCK_WEAKREFS(obj); + return NULL; + } + insert_weakref(newref, list); + UNLOCK_WEAKREFS(obj); + return newref; + } + else { + // We may not be able to safely allocate inside the lock + PyWeakReference *newref = allocate_weakref(type, obj, callback); + if (newref == NULL) { + return NULL; + } + LOCK_WEAKREFS(obj); + insert_weakref(newref, list); + UNLOCK_WEAKREFS(obj); + return newref; + } +} + static int parse_weakref_init_args(const char *funcname, PyObject *args, PyObject *kwargs, PyObject **obp, PyObject **callbackp) @@ -286,54 +462,11 @@ parse_weakref_init_args(const char *funcname, PyObject *args, PyObject *kwargs, static PyObject * weakref___new__(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - PyWeakReference *self = NULL; PyObject *ob, *callback = NULL; - if (parse_weakref_init_args("__new__", args, kwargs, &ob, &callback)) { - PyWeakReference *ref, *proxy; - PyWeakReference **list; - - if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(ob))) { - PyErr_Format(PyExc_TypeError, - "cannot create weak reference to '%s' object", - Py_TYPE(ob)->tp_name); - return NULL; - } - if (callback == Py_None) - callback = NULL; - list = GET_WEAKREFS_LISTPTR(ob); - get_basic_refs(*list, &ref, &proxy); - if (callback == NULL && type == &_PyWeakref_RefType) { - if (ref != NULL) { - /* We can re-use an existing reference. */ - return Py_NewRef(ref); - } - } - /* We have to create a new reference. */ - /* Note: the tp_alloc() can trigger cyclic GC, so the weakref - list on ob can be mutated. This means that the ref and - proxy pointers we got back earlier may have been collected, - so we need to compute these values again before we use - them. */ - self = (PyWeakReference *) (type->tp_alloc(type, 0)); - if (self != NULL) { - init_weakref(self, ob, callback); - if (callback == NULL && type == &_PyWeakref_RefType) { - insert_head(self, list); - } - else { - PyWeakReference *prev; - - get_basic_refs(*list, &ref, &proxy); - prev = (proxy == NULL) ? ref : proxy; - if (prev == NULL) - insert_head(self, list); - else - insert_after(self, prev); - } - } + return (PyObject *)get_or_create_weakref(type, ob, callback); } - return (PyObject *)self; + return NULL; } static int @@ -371,11 +504,11 @@ _PyWeakref_RefType = { .tp_vectorcall_offset = offsetof(PyWeakReference, vectorcall), .tp_call = PyVectorcall_Call, .tp_repr = weakref_repr, - .tp_hash = (hashfunc)weakref_hash, + .tp_hash = weakref_hash, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_BASETYPE, - .tp_traverse = (traverseproc)gc_traverse, - .tp_clear = (inquiry)gc_clear, + .tp_traverse = gc_traverse, + .tp_clear = gc_clear, .tp_richcompare = weakref_richcompare, .tp_methods = weakref_methods, .tp_members = weakref_members, @@ -471,10 +604,18 @@ static PyObject * proxy_repr(PyObject *proxy) { PyObject *obj = _PyWeakref_GET_REF(proxy); - PyObject *repr = PyUnicode_FromFormat( - "", - proxy, Py_TYPE(obj)->tp_name, obj); - Py_DECREF(obj); + PyObject *repr; + if (obj != NULL) { + repr = PyUnicode_FromFormat( + "", + proxy, obj, obj); + Py_DECREF(obj); + } + else { + repr = PyUnicode_FromFormat( + "", + proxy); + } return repr; } @@ -551,11 +692,9 @@ proxy_bool(PyObject *proxy) } static void -proxy_dealloc(PyWeakReference *self) +proxy_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); - if (self->wr_callback != NULL) - PyObject_GC_UnTrack((PyObject *)self); clear_weakref(self); PyObject_GC_Del(self); } @@ -716,7 +855,7 @@ _PyWeakref_ProxyType = { sizeof(PyWeakReference), 0, /* methods */ - (destructor)proxy_dealloc, /* tp_dealloc */ + proxy_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -734,8 +873,8 @@ _PyWeakref_ProxyType = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)gc_traverse, /* tp_traverse */ - (inquiry)gc_clear, /* tp_clear */ + gc_traverse, /* tp_traverse */ + gc_clear, /* tp_clear */ proxy_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ proxy_iter, /* tp_iter */ @@ -751,7 +890,7 @@ _PyWeakref_CallableProxyType = { sizeof(PyWeakReference), 0, /* methods */ - (destructor)proxy_dealloc, /* tp_dealloc */ + proxy_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -768,138 +907,45 @@ _PyWeakref_CallableProxyType = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)gc_traverse, /* tp_traverse */ - (inquiry)gc_clear, /* tp_clear */ + gc_traverse, /* tp_traverse */ + gc_clear, /* tp_clear */ proxy_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ proxy_iter, /* tp_iter */ proxy_iternext, /* tp_iternext */ }; - - PyObject * PyWeakref_NewRef(PyObject *ob, PyObject *callback) { - PyWeakReference *result = NULL; - PyWeakReference **list; - PyWeakReference *ref, *proxy; - - if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(ob))) { - PyErr_Format(PyExc_TypeError, - "cannot create weak reference to '%s' object", - Py_TYPE(ob)->tp_name); - return NULL; - } - list = GET_WEAKREFS_LISTPTR(ob); - get_basic_refs(*list, &ref, &proxy); - if (callback == Py_None) - callback = NULL; - if (callback == NULL) - /* return existing weak reference if it exists */ - result = ref; - if (result != NULL) - Py_INCREF(result); - else { - /* Note: new_weakref() can trigger cyclic GC, so the weakref - list on ob can be mutated. This means that the ref and - proxy pointers we got back earlier may have been collected, - so we need to compute these values again before we use - them. */ - result = new_weakref(ob, callback); - if (result != NULL) { - get_basic_refs(*list, &ref, &proxy); - if (callback == NULL) { - if (ref == NULL) - insert_head(result, list); - else { - /* Someone else added a ref without a callback - during GC. Return that one instead of this one - to avoid violating the invariants of the list - of weakrefs for ob. */ - Py_SETREF(result, (PyWeakReference*)Py_NewRef(ref)); - } - } - else { - PyWeakReference *prev; - - prev = (proxy == NULL) ? ref : proxy; - if (prev == NULL) - insert_head(result, list); - else - insert_after(result, prev); - } - } - } - return (PyObject *) result; + return (PyObject *)get_or_create_weakref(&_PyWeakref_RefType, ob, + callback); } - PyObject * PyWeakref_NewProxy(PyObject *ob, PyObject *callback) { - PyWeakReference *result = NULL; - PyWeakReference **list; - PyWeakReference *ref, *proxy; + PyTypeObject *type = &_PyWeakref_ProxyType; + if (PyCallable_Check(ob)) { + type = &_PyWeakref_CallableProxyType; + } + return (PyObject *)get_or_create_weakref(type, ob, callback); +} - if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(ob))) { - PyErr_Format(PyExc_TypeError, - "cannot create weak reference to '%s' object", - Py_TYPE(ob)->tp_name); - return NULL; +int +PyWeakref_IsDead(PyObject *ref) +{ + if (ref == NULL) { + PyErr_BadInternalCall(); + return -1; } - list = GET_WEAKREFS_LISTPTR(ob); - get_basic_refs(*list, &ref, &proxy); - if (callback == Py_None) - callback = NULL; - if (callback == NULL) - /* attempt to return an existing weak reference if it exists */ - result = proxy; - if (result != NULL) - Py_INCREF(result); - else { - /* Note: new_weakref() can trigger cyclic GC, so the weakref - list on ob can be mutated. This means that the ref and - proxy pointers we got back earlier may have been collected, - so we need to compute these values again before we use - them. */ - result = new_weakref(ob, callback); - if (result != NULL) { - PyWeakReference *prev; - - if (PyCallable_Check(ob)) { - Py_SET_TYPE(result, &_PyWeakref_CallableProxyType); - } - else { - Py_SET_TYPE(result, &_PyWeakref_ProxyType); - } - get_basic_refs(*list, &ref, &proxy); - if (callback == NULL) { - if (proxy != NULL) { - /* Someone else added a proxy without a callback - during GC. Return that one instead of this one - to avoid violating the invariants of the list - of weakrefs for ob. */ - Py_SETREF(result, (PyWeakReference*)Py_NewRef(proxy)); - goto skip_insert; - } - prev = ref; - } - else - prev = (proxy == NULL) ? ref : proxy; - - if (prev == NULL) - insert_head(result, list); - else - insert_after(result, prev); - skip_insert: - ; - } + if (!PyWeakref_Check(ref)) { + PyErr_Format(PyExc_TypeError, "expected a weakref, got %T", ref); + return -1; } - return (PyObject *) result; + return _PyWeakref_IS_DEAD(ref); } - int PyWeakref_GetRef(PyObject *ref, PyObject **pobj) { @@ -965,67 +1011,82 @@ PyObject_ClearWeakRefs(PyObject *object) PyErr_BadInternalCall(); return; } + list = GET_WEAKREFS_LISTPTR(object); - /* Remove the callback-less basic and proxy references */ - if (*list != NULL && (*list)->wr_callback == NULL) { - clear_weakref(*list); - if (*list != NULL && (*list)->wr_callback == NULL) - clear_weakref(*list); + if (FT_ATOMIC_LOAD_PTR(*list) == NULL) { + // Fast path for the common case + return; } - if (*list != NULL) { - PyWeakReference *current = *list; - Py_ssize_t count = _PyWeakref_GetWeakrefCount(current); - PyObject *exc = PyErr_GetRaisedException(); - - if (count == 1) { - PyObject *callback = current->wr_callback; - - current->wr_callback = NULL; - clear_weakref(current); - if (callback != NULL) { - if (Py_REFCNT((PyObject *)current) > 0) { - handle_callback(current, callback); - } - Py_DECREF(callback); - } + + /* Remove the callback-less basic and proxy references, which always appear + at the head of the list. + */ + for (int done = 0; !done;) { + LOCK_WEAKREFS(object); + if (*list != NULL && is_basic_ref_or_proxy(*list)) { + PyObject *callback; + clear_weakref_lock_held(*list, &callback); + assert(callback == NULL); } - else { - PyObject *tuple; - Py_ssize_t i = 0; - - tuple = PyTuple_New(count * 2); - if (tuple == NULL) { - _PyErr_ChainExceptions1(exc); - return; - } + done = (*list == NULL) || !is_basic_ref_or_proxy(*list); + UNLOCK_WEAKREFS(object); + } - for (i = 0; i < count; ++i) { - PyWeakReference *next = current->wr_next; - - if (Py_REFCNT((PyObject *)current) > 0) { - PyTuple_SET_ITEM(tuple, i * 2, Py_NewRef(current)); - PyTuple_SET_ITEM(tuple, i * 2 + 1, current->wr_callback); - } - else { - Py_DECREF(current->wr_callback); - } - current->wr_callback = NULL; - clear_weakref(current); - current = next; - } - for (i = 0; i < count; ++i) { - PyObject *callback = PyTuple_GET_ITEM(tuple, i * 2 + 1); - - /* The tuple may have slots left to NULL */ - if (callback != NULL) { - PyObject *item = PyTuple_GET_ITEM(tuple, i * 2); - handle_callback((PyWeakReference *)item, callback); - } + /* Deal with non-canonical (subtypes or refs with callbacks) references. */ + Py_ssize_t num_weakrefs = _PyWeakref_GetWeakrefCount(object); + if (num_weakrefs == 0) { + return; + } + + PyObject *exc = PyErr_GetRaisedException(); + PyObject *tuple = PyTuple_New(num_weakrefs * 2); + if (tuple == NULL) { + _PyWeakref_ClearWeakRefsNoCallbacks(object); + PyErr_WriteUnraisable(NULL); + PyErr_SetRaisedException(exc); + return; + } + + Py_ssize_t num_items = 0; + for (int done = 0; !done;) { + PyObject *callback = NULL; + LOCK_WEAKREFS(object); + PyWeakReference *cur = *list; + if (cur != NULL) { + clear_weakref_lock_held(cur, &callback); + if (_Py_TryIncref((PyObject *) cur)) { + assert(num_items / 2 < num_weakrefs); + PyTuple_SET_ITEM(tuple, num_items, (PyObject *) cur); + PyTuple_SET_ITEM(tuple, num_items + 1, callback); + num_items += 2; + callback = NULL; } - Py_DECREF(tuple); } - assert(!PyErr_Occurred()); - PyErr_SetRaisedException(exc); + done = (*list == NULL); + UNLOCK_WEAKREFS(object); + + Py_XDECREF(callback); + } + + for (Py_ssize_t i = 0; i < num_items; i += 2) { + PyObject *callback = PyTuple_GET_ITEM(tuple, i + 1); + if (callback != NULL) { + PyObject *weakref = PyTuple_GET_ITEM(tuple, i); + handle_callback((PyWeakReference *)weakref, callback); + } + } + + Py_DECREF(tuple); + + assert(!PyErr_Occurred()); + PyErr_SetRaisedException(exc); +} + +void +PyUnstable_Object_ClearWeakRefsNoCallbacks(PyObject *obj) +{ + if (_PyType_SUPPORTS_WEAKREFS(Py_TYPE(obj))) { + _PyWeakref_ClearWeakRefsNoCallbacks(obj); } } @@ -1038,12 +1099,32 @@ PyObject_ClearWeakRefs(PyObject *object) void _PyStaticType_ClearWeakRefs(PyInterpreterState *interp, PyTypeObject *type) { - static_builtin_state *state = _PyStaticType_GetState(interp, type); + managed_static_type_state *state = _PyStaticType_GetState(interp, type); PyObject **list = _PyStaticType_GET_WEAKREFS_LISTPTR(state); - while (*list != NULL) { - /* Note that clear_weakref() pops the first ref off the type's - weaklist before clearing its wr_object and wr_callback. - That is how we're able to loop over the list. */ - clear_weakref((PyWeakReference *)*list); + // This is safe to do without holding the lock in free-threaded builds; + // there is only one thread running and no new threads can be created. + while (*list) { + _PyWeakref_ClearRef((PyWeakReference *)*list); + } +} + +void +_PyWeakref_ClearWeakRefsNoCallbacks(PyObject *obj) +{ + /* Modeled after GET_WEAKREFS_LISTPTR(). + + This is never triggered for static types so we can avoid the + (slightly) more costly _PyObject_GET_WEAKREFS_LISTPTR(). */ + PyWeakReference **list = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(obj); + LOCK_WEAKREFS(obj); + while (*list) { + _PyWeakref_ClearRef(*list); } + UNLOCK_WEAKREFS(obj); +} + +int +_PyWeakref_IsDead(PyObject *weakref) +{ + return _PyWeakref_IS_DEAD(weakref); } diff --git a/PC/_testconsole.c b/PC/_testconsole.c index 1dc0d230c4d7c3..0dcea866f65d35 100644 --- a/PC/_testconsole.c +++ b/PC/_testconsole.c @@ -1,17 +1,16 @@ /* Testing module for multi-phase initialization of extension modules (PEP 489) */ -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 +// Need limited C API version 3.13 for Py_mod_gil +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" #ifdef MS_WINDOWS -#include "pycore_fileutils.h" // _Py_get_osfhandle() -#include "pycore_runtime.h" // _Py_ID() - #define WIN32_LEAN_AND_MEAN #include #include @@ -32,6 +31,7 @@ static int execfunc(PyObject *m) PyModuleDef_Slot testconsole_slots[] = { {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -57,20 +57,24 @@ module _testconsole _testconsole.write_input file: object - s: PyBytesObject + s: Py_buffer Writes UTF-16-LE encoded bytes to the console as if typed by a user. [clinic start generated code]*/ static PyObject * -_testconsole_write_input_impl(PyObject *module, PyObject *file, - PyBytesObject *s) -/*[clinic end generated code: output=48f9563db34aedb3 input=4c774f2d05770bc6]*/ +_testconsole_write_input_impl(PyObject *module, PyObject *file, Py_buffer *s) +/*[clinic end generated code: output=58631a8985426ad3 input=68062f1bb2e52206]*/ { INPUT_RECORD *rec = NULL; - PyTypeObject *winconsoleio_type = (PyTypeObject *)_PyImport_GetModuleAttr( - &_Py_ID(_io), &_Py_ID(_WindowsConsoleIO)); + PyObject *mod = PyImport_ImportModule("_io"); + if (mod == NULL) { + return NULL; + } + + PyTypeObject *winconsoleio_type = (PyTypeObject *)PyObject_GetAttrString(mod, "_WindowsConsoleIO"); + Py_DECREF(mod); if (winconsoleio_type == NULL) { return NULL; } @@ -81,8 +85,8 @@ _testconsole_write_input_impl(PyObject *module, PyObject *file, return NULL; } - const wchar_t *p = (const wchar_t *)PyBytes_AS_STRING(s); - DWORD size = (DWORD)PyBytes_GET_SIZE(s) / sizeof(wchar_t); + const wchar_t *p = (const wchar_t *)s->buf; + DWORD size = (DWORD)s->len / sizeof(wchar_t); rec = (INPUT_RECORD*)PyMem_Calloc(size, sizeof(INPUT_RECORD)); if (!rec) @@ -96,9 +100,11 @@ _testconsole_write_input_impl(PyObject *module, PyObject *file, prec->Event.KeyEvent.uChar.UnicodeChar = *p; } - HANDLE hInput = _Py_get_osfhandle(((winconsoleio*)file)->fd); - if (hInput == INVALID_HANDLE_VALUE) + HANDLE hInput = (HANDLE)_get_osfhandle(((winconsoleio*)file)->fd); + if (hInput == INVALID_HANDLE_VALUE) { + PyErr_SetFromErrno(PyExc_OSError); goto error; + } DWORD total = 0; while (total < size) { diff --git a/PC/_wmimodule.cpp b/PC/_wmimodule.cpp index 5ab6dcb032550b..3a59e78f171526 100644 --- a/PC/_wmimodule.cpp +++ b/PC/_wmimodule.cpp @@ -55,12 +55,26 @@ _query_thread(LPVOID param) IWbemLocator *locator = NULL; IWbemServices *services = NULL; IEnumWbemClassObject* enumerator = NULL; + HRESULT hr = S_OK; BSTR bstrQuery = NULL; struct _query_data *data = (struct _query_data*)param; - HRESULT hr = CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED); + // gh-125315: Copy the query string first, so that if the main thread gives + // up on waiting we aren't left with a dangling pointer (and a likely crash) + bstrQuery = SysAllocString(data->query); + if (!bstrQuery) { + hr = HRESULT_FROM_WIN32(ERROR_NOT_ENOUGH_MEMORY); + } + + if (SUCCEEDED(hr)) { + hr = CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED); + } + if (FAILED(hr)) { CloseHandle(data->writePipe); + if (bstrQuery) { + SysFreeString(bstrQuery); + } return (DWORD)hr; } @@ -101,12 +115,6 @@ _query_thread(LPVOID param) NULL, EOAC_NONE ); } - if (SUCCEEDED(hr)) { - bstrQuery = SysAllocString(data->query); - if (!bstrQuery) { - hr = HRESULT_FROM_WIN32(ERROR_NOT_ENOUGH_MEMORY); - } - } if (SUCCEEDED(hr)) { hr = services->ExecQuery( bstr_t("WQL"), @@ -279,9 +287,11 @@ _wmi_exec_query_impl(PyObject *module, PyObject *query) // a timeout. The initEvent will be set after COM initialization, it will // take a longer time when first initialized. The connectEvent will be set // after connected to WMI. - err = wait_event(data.initEvent, 1000); if (!err) { - err = wait_event(data.connectEvent, 100); + err = wait_event(data.initEvent, 1000); + if (!err) { + err = wait_event(data.connectEvent, 100); + } } while (!err) { @@ -305,28 +315,33 @@ _wmi_exec_query_impl(PyObject *module, PyObject *query) CloseHandle(data.readPipe); } - // Allow the thread some time to clean up - switch (WaitForSingleObject(hThread, 100)) { - case WAIT_OBJECT_0: - // Thread ended cleanly - if (!GetExitCodeThread(hThread, (LPDWORD)&err)) { - err = GetLastError(); - } - break; - case WAIT_TIMEOUT: - // Probably stuck - there's not much we can do, unfortunately - if (err == 0 || err == ERROR_BROKEN_PIPE) { - err = WAIT_TIMEOUT; + if (hThread) { + // Allow the thread some time to clean up + int thread_err; + switch (WaitForSingleObject(hThread, 100)) { + case WAIT_OBJECT_0: + // Thread ended cleanly + if (!GetExitCodeThread(hThread, (LPDWORD)&thread_err)) { + thread_err = GetLastError(); + } + break; + case WAIT_TIMEOUT: + // Probably stuck - there's not much we can do, unfortunately + thread_err = WAIT_TIMEOUT; + break; + default: + thread_err = GetLastError(); + break; } - break; - default: + // An error on our side is more likely to be relevant than one from + // the thread, but if we don't have one on our side we'll take theirs. if (err == 0 || err == ERROR_BROKEN_PIPE) { - err = GetLastError(); + err = thread_err; } - break; + + CloseHandle(hThread); } - CloseHandle(hThread); CloseHandle(data.initEvent); CloseHandle(data.connectEvent); hThread = NULL; @@ -355,12 +370,18 @@ static PyMethodDef wmi_functions[] = { { NULL, NULL, 0, NULL } }; +static PyModuleDef_Slot wmi_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static PyModuleDef wmi_def = { PyModuleDef_HEAD_INIT, "_wmi", - NULL, // doc - 0, // m_size - wmi_functions + NULL, // doc + 0, // m_size + wmi_functions, // m_methods + wmi_slots, // m_slots }; extern "C" { diff --git a/PC/clinic/_testconsole.c.h b/PC/clinic/_testconsole.c.h index 2c71c11c438b5b..4c11e545499ac5 100644 --- a/PC/clinic/_testconsole.c.h +++ b/PC/clinic/_testconsole.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif -#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() - #if defined(MS_WINDOWS) PyDoc_STRVAR(_testconsole_write_input__doc__, @@ -17,58 +11,30 @@ PyDoc_STRVAR(_testconsole_write_input__doc__, "Writes UTF-16-LE encoded bytes to the console as if typed by a user."); #define _TESTCONSOLE_WRITE_INPUT_METHODDEF \ - {"write_input", _PyCFunction_CAST(_testconsole_write_input), METH_FASTCALL|METH_KEYWORDS, _testconsole_write_input__doc__}, + {"write_input", (PyCFunction)(void(*)(void))_testconsole_write_input, METH_VARARGS|METH_KEYWORDS, _testconsole_write_input__doc__}, static PyObject * -_testconsole_write_input_impl(PyObject *module, PyObject *file, - PyBytesObject *s); +_testconsole_write_input_impl(PyObject *module, PyObject *file, Py_buffer *s); static PyObject * -_testconsole_write_input(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_testconsole_write_input(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(file), &_Py_ID(s), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"file", "s", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "write_input", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; + static char *_keywords[] = {"file", "s", NULL}; PyObject *file; - PyBytesObject *s; + Py_buffer s = {NULL, NULL}; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { - goto exit; - } - file = args[0]; - if (!PyBytes_Check(args[1])) { - _PyArg_BadArgument("write_input", "argument 's'", "bytes", args[1]); + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "Oy*:write_input", _keywords, + &file, &s)) goto exit; - } - s = (PyBytesObject *)args[1]; - return_value = _testconsole_write_input_impl(module, file, s); + return_value = _testconsole_write_input_impl(module, file, &s); exit: + /* Cleanup for s */ + if (s.obj) { + PyBuffer_Release(&s); + } + return return_value; } @@ -83,48 +49,21 @@ PyDoc_STRVAR(_testconsole_read_output__doc__, "Reads a str from the console as written to stdout."); #define _TESTCONSOLE_READ_OUTPUT_METHODDEF \ - {"read_output", _PyCFunction_CAST(_testconsole_read_output), METH_FASTCALL|METH_KEYWORDS, _testconsole_read_output__doc__}, + {"read_output", (PyCFunction)(void(*)(void))_testconsole_read_output, METH_VARARGS|METH_KEYWORDS, _testconsole_read_output__doc__}, static PyObject * _testconsole_read_output_impl(PyObject *module, PyObject *file); static PyObject * -_testconsole_read_output(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_testconsole_read_output(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(file), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"file", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "read_output", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; + static char *_keywords[] = {"file", NULL}; PyObject *file; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O:read_output", _keywords, + &file)) goto exit; - } - file = args[0]; return_value = _testconsole_read_output_impl(module, file); exit: @@ -140,4 +79,4 @@ _testconsole_read_output(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef _TESTCONSOLE_READ_OUTPUT_METHODDEF #define _TESTCONSOLE_READ_OUTPUT_METHODDEF #endif /* !defined(_TESTCONSOLE_READ_OUTPUT_METHODDEF) */ -/*[clinic end generated code: output=08a1c844b3657272 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d60ce07157e3741a input=a9049054013a1b77]*/ diff --git a/PC/clinic/_wmimodule.cpp.h b/PC/clinic/_wmimodule.cpp.h index fccf3d795d5340..f0be691173ad64 100644 --- a/PC/clinic/_wmimodule.cpp.h +++ b/PC/clinic/_wmimodule.cpp.h @@ -55,7 +55,8 @@ _wmi_exec_query(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *argsbuf[1]; PyObject *query; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -69,4 +70,4 @@ _wmi_exec_query(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj exit: return return_value; } -/*[clinic end generated code: output=ba04920d127f3ceb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9ac5fd7faf889e52 input=a9049054013a1b77]*/ diff --git a/PC/clinic/msvcrtmodule.c.h b/PC/clinic/msvcrtmodule.c.h index e3f7ea43f38211..a77d0855af293f 100644 --- a/PC/clinic/msvcrtmodule.c.h +++ b/PC/clinic/msvcrtmodule.c.h @@ -355,10 +355,24 @@ msvcrt_putch(PyObject *module, PyObject *arg) PyObject *return_value = NULL; char char_value; - if (PyBytes_Check(arg) && PyBytes_GET_SIZE(arg) == 1) { + if (PyBytes_Check(arg)) { + if (PyBytes_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "putch(): argument must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(arg)); + goto exit; + } char_value = PyBytes_AS_STRING(arg)[0]; } - else if (PyByteArray_Check(arg) && PyByteArray_GET_SIZE(arg) == 1) { + else if (PyByteArray_Check(arg)) { + if (PyByteArray_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "putch(): argument must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(arg)); + goto exit; + } char_value = PyByteArray_AS_STRING(arg)[0]; } else { @@ -396,7 +410,10 @@ msvcrt_putwch(PyObject *module, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("putwch", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "putwch(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } unicode_char = PyUnicode_READ_CHAR(arg, 0); @@ -430,10 +447,24 @@ msvcrt_ungetch(PyObject *module, PyObject *arg) PyObject *return_value = NULL; char char_value; - if (PyBytes_Check(arg) && PyBytes_GET_SIZE(arg) == 1) { + if (PyBytes_Check(arg)) { + if (PyBytes_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "ungetch(): argument must be a byte string of length 1, " + "not a bytes object of length %zd", + PyBytes_GET_SIZE(arg)); + goto exit; + } char_value = PyBytes_AS_STRING(arg)[0]; } - else if (PyByteArray_Check(arg) && PyByteArray_GET_SIZE(arg) == 1) { + else if (PyByteArray_Check(arg)) { + if (PyByteArray_GET_SIZE(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "ungetch(): argument must be a byte string of length 1, " + "not a bytearray object of length %zd", + PyByteArray_GET_SIZE(arg)); + goto exit; + } char_value = PyByteArray_AS_STRING(arg)[0]; } else { @@ -471,7 +502,10 @@ msvcrt_ungetwch(PyObject *module, PyObject *arg) goto exit; } if (PyUnicode_GET_LENGTH(arg) != 1) { - _PyArg_BadArgument("ungetwch", "argument", "a unicode character", arg); + PyErr_Format(PyExc_TypeError, + "ungetwch(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); goto exit; } unicode_char = PyUnicode_READ_CHAR(arg, 0); @@ -697,4 +731,4 @@ msvcrt_SetErrorMode(PyObject *module, PyObject *arg) #ifndef MSVCRT_GETERRORMODE_METHODDEF #define MSVCRT_GETERRORMODE_METHODDEF #endif /* !defined(MSVCRT_GETERRORMODE_METHODDEF) */ -/*[clinic end generated code: output=de9687b46212c2ed input=a9049054013a1b77]*/ +/*[clinic end generated code: output=692c6f52bb9193ce input=a9049054013a1b77]*/ diff --git a/PC/clinic/winreg.c.h b/PC/clinic/winreg.c.h index 7e5545e113e718..00fa6a75ec113e 100644 --- a/PC/clinic/winreg.c.h +++ b/PC/clinic/winreg.c.h @@ -26,9 +26,9 @@ static PyObject * winreg_HKEYType_Close_impl(PyHKEYObject *self); static PyObject * -winreg_HKEYType_Close(PyHKEYObject *self, PyObject *Py_UNUSED(ignored)) +winreg_HKEYType_Close(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return winreg_HKEYType_Close_impl(self); + return winreg_HKEYType_Close_impl((PyHKEYObject *)self); } #endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ @@ -56,9 +56,9 @@ static PyObject * winreg_HKEYType_Detach_impl(PyHKEYObject *self); static PyObject * -winreg_HKEYType_Detach(PyHKEYObject *self, PyObject *Py_UNUSED(ignored)) +winreg_HKEYType_Detach(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return winreg_HKEYType_Detach_impl(self); + return winreg_HKEYType_Detach_impl((PyHKEYObject *)self); } #endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ @@ -77,12 +77,12 @@ static PyHKEYObject * winreg_HKEYType___enter___impl(PyHKEYObject *self); static PyObject * -winreg_HKEYType___enter__(PyHKEYObject *self, PyObject *Py_UNUSED(ignored)) +winreg_HKEYType___enter__(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; PyHKEYObject *_return_value; - _return_value = winreg_HKEYType___enter___impl(self); + _return_value = winreg_HKEYType___enter___impl((PyHKEYObject *)self); return_value = (PyObject *)_return_value; return return_value; @@ -105,7 +105,7 @@ winreg_HKEYType___exit___impl(PyHKEYObject *self, PyObject *exc_type, PyObject *exc_value, PyObject *traceback); static PyObject * -winreg_HKEYType___exit__(PyHKEYObject *self, PyObject *const *args, Py_ssize_t nargs) +winreg_HKEYType___exit__(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *exc_type; @@ -118,7 +118,7 @@ winreg_HKEYType___exit__(PyHKEYObject *self, PyObject *const *args, Py_ssize_t n exc_type = args[0]; exc_value = args[1]; traceback = args[2]; - return_value = winreg_HKEYType___exit___impl(self, exc_type, exc_value, traceback); + return_value = winreg_HKEYType___exit___impl((PyHKEYObject *)self, exc_type, exc_value, traceback); exit: return return_value; @@ -351,7 +351,8 @@ winreg_CreateKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py REGSAM access = KEY_WRITE; HKEY _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -534,7 +535,8 @@ winreg_DeleteKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py REGSAM access = KEY_WOW64_64KEY; int reserved = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -966,7 +968,8 @@ winreg_OpenKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje REGSAM access = KEY_READ; HKEY _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1083,7 +1086,8 @@ winreg_OpenKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb REGSAM access = KEY_READ; HKEY _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 4, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -1762,4 +1766,4 @@ winreg_QueryReflectionKey(PyObject *module, PyObject *arg) #ifndef WINREG_QUERYREFLECTIONKEY_METHODDEF #define WINREG_QUERYREFLECTIONKEY_METHODDEF #endif /* !defined(WINREG_QUERYREFLECTIONKEY_METHODDEF) */ -/*[clinic end generated code: output=1ee4098b2f143b6a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fbe9b075cd2fa833 input=a9049054013a1b77]*/ diff --git a/PC/config.c b/PC/config.c index 5eff2f5b2310bb..b744f711b0d636 100644 --- a/PC/config.c +++ b/PC/config.c @@ -35,9 +35,9 @@ extern PyObject* PyInit__codecs(void); extern PyObject* PyInit__weakref(void); /* XXX: These two should really be extracted to standalone extensions. */ extern PyObject* PyInit_xxsubtype(void); -extern PyObject* PyInit__xxsubinterpreters(void); -extern PyObject* PyInit__xxinterpchannels(void); -extern PyObject* PyInit__xxinterpqueues(void); +extern PyObject* PyInit__interpreters(void); +extern PyObject* PyInit__interpchannels(void); +extern PyObject* PyInit__interpqueues(void); extern PyObject* PyInit__random(void); extern PyObject* PyInit_itertools(void); extern PyObject* PyInit__collections(void); @@ -139,9 +139,9 @@ struct _inittab _PyImport_Inittab[] = { {"_json", PyInit__json}, {"xxsubtype", PyInit_xxsubtype}, - {"_xxsubinterpreters", PyInit__xxsubinterpreters}, - {"_xxinterpchannels", PyInit__xxinterpchannels}, - {"_xxinterpqueues", PyInit__xxinterpqueues}, + {"_interpreters", PyInit__interpreters}, + {"_interpchannels", PyInit__interpchannels}, + {"_interpqueues", PyInit__interpqueues}, #ifdef _Py_HAVE_ZLIB {"zlib", PyInit_zlib}, #endif diff --git a/PC/launcher.c b/PC/launcher.c index 8e60ab9303cb95..47fafbc3bf6bad 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -1271,6 +1271,7 @@ static PYC_MAGIC magic_values[] = { { 3450, 3499, L"3.11" }, { 3500, 3549, L"3.12" }, { 3550, 3599, L"3.13" }, + { 3600, 3649, L"3.14" }, { 0 } }; diff --git a/PC/launcher2.c b/PC/launcher2.c index 139aa61bbe5cc2..befcbe30600f2c 100644 --- a/PC/launcher2.c +++ b/PC/launcher2.c @@ -853,7 +853,7 @@ searchPath(SearchInfo *search, const wchar_t *shebang, int shebangLength) } wchar_t filename[MAXLEN]; - if (wcsncpy_s(filename, MAXLEN, command, lastDot)) { + if (wcsncpy_s(filename, MAXLEN, command, commandLength)) { return RC_BAD_VIRTUAL_PATH; } @@ -1962,6 +1962,7 @@ struct AppxSearchInfo { struct AppxSearchInfo APPX_SEARCH[] = { // Releases made through the Store + { L"PythonSoftwareFoundation.Python.3.14_qbz5n2kfra8p0", L"3.14", 10 }, { L"PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0", L"3.13", 10 }, { L"PythonSoftwareFoundation.Python.3.12_qbz5n2kfra8p0", L"3.12", 10 }, { L"PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0", L"3.11", 10 }, @@ -1970,8 +1971,9 @@ struct AppxSearchInfo APPX_SEARCH[] = { { L"PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0", L"3.8", 10 }, // Side-loadable releases. Note that the publisher ID changes whenever we - // renew our code-signing certificate, so the newer ID has a higher - // priority (lower sortKey) + // change our code signing certificate subject, so the newer IDs have higher + // priorities (lower sortKey) + { L"PythonSoftwareFoundation.Python.3.14_3847v3x7pw1km", L"3.14", 11 }, { L"PythonSoftwareFoundation.Python.3.13_3847v3x7pw1km", L"3.13", 11 }, { L"PythonSoftwareFoundation.Python.3.12_3847v3x7pw1km", L"3.12", 11 }, { L"PythonSoftwareFoundation.Python.3.11_3847v3x7pw1km", L"3.11", 11 }, @@ -2054,7 +2056,8 @@ struct StoreSearchInfo { struct StoreSearchInfo STORE_SEARCH[] = { - { L"3", /* 3.12 */ L"9NCVDN91XZQP" }, + { L"3", /* 3.13 */ L"9PNRBTZXMB4Z" }, + { L"3.14", L"9NTRHQCBBPR8" }, { L"3.13", L"9PNRBTZXMB4Z" }, { L"3.12", L"9NCVDN91XZQP" }, { L"3.11", L"9NRWMJP3717K" }, @@ -2707,6 +2710,11 @@ process(int argc, wchar_t ** argv) DWORD len = GetEnvironmentVariableW(L"PYLAUNCHER_LIMIT_TO_COMPANY", NULL, 0); if (len > 1) { wchar_t *limitToCompany = allocSearchInfoBuffer(&search, len); + if (!limitToCompany) { + exitCode = RC_NO_MEMORY; + winerror(0, L"Failed to allocate internal buffer"); + goto abort; + } search.limitToCompany = limitToCompany; if (0 == GetEnvironmentVariableW(L"PYLAUNCHER_LIMIT_TO_COMPANY", limitToCompany, len)) { exitCode = RC_INTERNAL_ERROR; diff --git a/PC/layout/__main__.py b/PC/layout/__main__.py index f7aa1e6d261f4a..05a059eee7c1d7 100644 --- a/PC/layout/__main__.py +++ b/PC/layout/__main__.py @@ -1,7 +1,7 @@ import sys try: - import layout + import layout # noqa: F401 except ImportError: # Failed to import our package, which likely means we were started directly # Add the additional search path needed to locate our module. diff --git a/PC/layout/main.py b/PC/layout/main.py index d176b272f1c19d..8bd435456c635a 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -31,11 +31,13 @@ from .support.nuspec import * TEST_PYDS_ONLY = FileStemSet("xxlimited", "xxlimited_35", "_ctypes_test", "_test*") +TEST_DLLS_ONLY = set() TEST_DIRS_ONLY = FileNameSet("test", "tests") IDLE_DIRS_ONLY = FileNameSet("idlelib") -TCLTK_PYDS_ONLY = FileStemSet("tcl*", "tk*", "_tkinter", "zlib1") +TCLTK_PYDS_ONLY = FileStemSet("_tkinter") +TCLTK_DLLS_ONLY = FileStemSet("tcl*", "tk*", "zlib1") TCLTK_DIRS_ONLY = FileNameSet("tkinter", "turtledemo") TCLTK_FILES_ONLY = FileNameSet("turtle.py") @@ -121,7 +123,7 @@ def get_tcltk_lib(ns): def get_layout(ns): - def in_build(f, dest="", new_name=None): + def in_build(f, dest="", new_name=None, no_lib=False): n, _, x = f.rpartition(".") n = new_name or n src = ns.build / f @@ -136,7 +138,7 @@ def in_build(f, dest="", new_name=None): pdb = src.with_suffix(".pdb") if pdb.is_file(): yield dest + n + ".pdb", pdb - if ns.include_dev: + if ns.include_dev and not no_lib: lib = src.with_suffix(".lib") if lib.is_file(): yield "libs/" + n + ".lib", lib @@ -202,7 +204,9 @@ def in_build(f, dest="", new_name=None): yield "LICENSE.txt", ns.build / "LICENSE.txt" - for dest, src in rglob(ns.build, "*.pyd"): + dest = "" if ns.flat_dlls else "DLLs/" + + for _, src in rglob(ns.build, "*.pyd"): if ns.include_freethreaded: if not src.match("*.cp*t-win*.pyd"): continue @@ -217,14 +221,18 @@ def in_build(f, dest="", new_name=None): continue if src in TCLTK_PYDS_ONLY and not ns.include_tcltk: continue - yield from in_build(src.name, dest="" if ns.flat_dlls else "DLLs/") + yield from in_build(src.name, dest=dest, no_lib=True) - for dest, src in rglob(ns.build, "*.dll"): + for _, src in rglob(ns.build, "*.dll"): if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS: continue if src in EXCLUDE_FROM_DLLS: continue - yield from in_build(src.name, dest="" if ns.flat_dlls else "DLLs/") + if src in TEST_DLLS_ONLY and not ns.include_tests: + continue + if src in TCLTK_DLLS_ONLY and not ns.include_tcltk: + continue + yield from in_build(src.name, dest=dest, no_lib=True) if ns.zip_lib: zip_name = PYTHON_ZIP_NAME @@ -599,6 +607,15 @@ def main(): ns.source = ns.source or (Path(__file__).resolve().parent.parent.parent) ns.build = ns.build or Path(sys.executable).parent ns.doc_build = ns.doc_build or (ns.source / "Doc" / "build") + if ns.copy and not ns.copy.is_absolute(): + ns.copy = (Path.cwd() / ns.copy).resolve() + if not ns.temp: + # Put temp on a Dev Drive for speed if we're copying to one. + # If not, the regular temp dir will have to do. + if ns.copy and getattr(os.path, "isdevdrive", lambda d: False)(ns.copy): + ns.temp = ns.copy.with_name(ns.copy.name + "_temp") + else: + ns.temp = Path(tempfile.mkdtemp()) if not ns.source.is_absolute(): ns.source = (Path.cwd() / ns.source).resolve() if not ns.build.is_absolute(): @@ -617,21 +634,11 @@ def main(): else: ns.arch = "amd64" - if ns.copy and not ns.copy.is_absolute(): - ns.copy = (Path.cwd() / ns.copy).resolve() if ns.zip and not ns.zip.is_absolute(): ns.zip = (Path.cwd() / ns.zip).resolve() if ns.catalog and not ns.catalog.is_absolute(): ns.catalog = (Path.cwd() / ns.catalog).resolve() - if not ns.temp: - # Put temp on a Dev Drive for speed if we're copying to one. - # If not, the regular temp dir will have to do. - if ns.copy and getattr(os.path, "isdevdrive", lambda d: False)(ns.copy): - ns.temp = ns.copy.with_name(ns.copy.name + "_temp") - else: - ns.temp = Path(tempfile.mkdtemp()) - configure_logger(ns) log_info( diff --git a/PC/layout/support/appxmanifest.py b/PC/layout/support/appxmanifest.py index 1fb03380278f43..53977beb8af834 100644 --- a/PC/layout/support/appxmanifest.py +++ b/PC/layout/support/appxmanifest.py @@ -209,7 +209,7 @@ class PACKAGE_ID(ctypes.Structure): result = ctypes.create_unicode_buffer(256) result_len = ctypes.c_uint32(256) r = ctypes.windll.kernel32.PackageFamilyNameFromId( - pid, ctypes.byref(result_len), result + ctypes.byref(pid), ctypes.byref(result_len), result ) if r: raise OSError(r, "failed to get package family name") diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index 5ff703217b421f..b170e06b47dd59 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -656,6 +656,7 @@ exec_module(PyObject* m) static PyModuleDef_Slot msvcrt_slots[] = { {Py_mod_exec, exec_module}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/PC/pyconfig.h.in b/PC/pyconfig.h.in index d72d6282c2806f..f4f57c5d270028 100644 --- a/PC/pyconfig.h.in +++ b/PC/pyconfig.h.in @@ -169,9 +169,9 @@ WIN32 is still required for the locale module. #endif /* MS_WIN64 */ /* set the version macros for the windows headers */ -/* Python 3.9+ requires Windows 8 or greater */ -#define Py_WINVER 0x0602 /* _WIN32_WINNT_WIN8 */ -#define Py_NTDDI NTDDI_WIN8 +/* Python 3.13+ requires Windows 10 or greater */ +#define Py_WINVER 0x0A00 /* _WIN32_WINNT_WIN10 */ +#define Py_NTDDI NTDDI_WIN10 /* We only set these values when building Python - we don't want to force these values on extensions, as that will affect the prototypes and @@ -316,19 +316,19 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ file in their Makefile */ # if defined(Py_GIL_DISABLED) # if defined(_DEBUG) -# pragma comment(lib,"python313t_d.lib") +# pragma comment(lib,"python314t_d.lib") # elif defined(Py_LIMITED_API) # pragma comment(lib,"python3t.lib") # else -# pragma comment(lib,"python313t.lib") +# pragma comment(lib,"python314t.lib") # endif /* _DEBUG */ # else /* Py_GIL_DISABLED */ # if defined(_DEBUG) -# pragma comment(lib,"python313_d.lib") +# pragma comment(lib,"python314_d.lib") # elif defined(Py_LIMITED_API) # pragma comment(lib,"python3.lib") # else -# pragma comment(lib,"python313.lib") +# pragma comment(lib,"python314.lib") # endif /* _DEBUG */ # endif /* Py_GIL_DISABLED */ # endif /* _MSC_VER */ @@ -531,9 +531,6 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ /* Define if you want to compile in mimalloc memory allocator. */ #define WITH_MIMALLOC 1 -/* Define if you want to compile in object freelists optimization */ -#define WITH_FREELISTS 1 - /* Define if you have clock. */ /* #define HAVE_CLOCK */ @@ -756,4 +753,7 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ /* Define if libssl has X509_VERIFY_PARAM_set1_host and related function */ #define HAVE_X509_VERIFY_PARAM_SET1_HOST 1 +// Truncate the thread name to 32766 characters. +#define _PYTHREAD_NAME_MAXLEN 32766 + #endif /* !Py_CONFIG_H */ diff --git a/PC/python.manifest b/PC/python.manifest index 8e1bc022adfb4f..19c9fc1b80a3ec 100644 --- a/PC/python.manifest +++ b/PC/python.manifest @@ -9,10 +9,15 @@ + + + + + diff --git a/PC/python3dll.c b/PC/python3dll.c index aa6bfe2c4022db..84b3c735240b73 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -55,6 +55,8 @@ EXPORT_FUNC(Py_GenericAlias) EXPORT_FUNC(Py_GetArgcArgv) EXPORT_FUNC(Py_GetBuildInfo) EXPORT_FUNC(Py_GetCompiler) +EXPORT_FUNC(Py_GetConstant) +EXPORT_FUNC(Py_GetConstantBorrowed) EXPORT_FUNC(Py_GetCopyright) EXPORT_FUNC(Py_GetExecPrefix) EXPORT_FUNC(Py_GetPath) @@ -79,12 +81,16 @@ EXPORT_FUNC(Py_Main) EXPORT_FUNC(Py_MakePendingCalls) EXPORT_FUNC(Py_NewInterpreter) EXPORT_FUNC(Py_NewRef) +EXPORT_FUNC(Py_PACK_FULL_VERSION) +EXPORT_FUNC(Py_PACK_VERSION) +EXPORT_FUNC(Py_REFCNT) EXPORT_FUNC(Py_ReprEnter) EXPORT_FUNC(Py_ReprLeave) EXPORT_FUNC(Py_SetPath) EXPORT_FUNC(Py_SetProgramName) EXPORT_FUNC(Py_SetPythonHome) EXPORT_FUNC(Py_SetRecursionLimit) +EXPORT_FUNC(Py_TYPE) EXPORT_FUNC(Py_VaBuildValue) EXPORT_FUNC(Py_XNewRef) EXPORT_FUNC(PyAIter_Check) @@ -251,6 +257,9 @@ EXPORT_FUNC(PyEval_EvalFrame) EXPORT_FUNC(PyEval_EvalFrameEx) EXPORT_FUNC(PyEval_GetBuiltins) EXPORT_FUNC(PyEval_GetFrame) +EXPORT_FUNC(PyEval_GetFrameBuiltins) +EXPORT_FUNC(PyEval_GetFrameGlobals) +EXPORT_FUNC(PyEval_GetFrameLocals) EXPORT_FUNC(PyEval_GetFuncDesc) EXPORT_FUNC(PyEval_GetFuncName) EXPORT_FUNC(PyEval_GetGlobals) @@ -320,6 +329,7 @@ EXPORT_FUNC(PyInterpreterState_GetID) EXPORT_FUNC(PyInterpreterState_New) EXPORT_FUNC(PyIter_Check) EXPORT_FUNC(PyIter_Next) +EXPORT_FUNC(PyIter_NextItem) EXPORT_FUNC(PyIter_Send) EXPORT_FUNC(PyList_Append) EXPORT_FUNC(PyList_AsTuple) @@ -335,23 +345,31 @@ EXPORT_FUNC(PyList_Size) EXPORT_FUNC(PyList_Sort) EXPORT_FUNC(PyLong_AsDouble) EXPORT_FUNC(PyLong_AsInt) +EXPORT_FUNC(PyLong_AsInt32) +EXPORT_FUNC(PyLong_AsInt64) EXPORT_FUNC(PyLong_AsLong) EXPORT_FUNC(PyLong_AsLongAndOverflow) EXPORT_FUNC(PyLong_AsLongLong) EXPORT_FUNC(PyLong_AsLongLongAndOverflow) EXPORT_FUNC(PyLong_AsSize_t) EXPORT_FUNC(PyLong_AsSsize_t) +EXPORT_FUNC(PyLong_AsUInt32) +EXPORT_FUNC(PyLong_AsUInt64) EXPORT_FUNC(PyLong_AsUnsignedLong) EXPORT_FUNC(PyLong_AsUnsignedLongLong) EXPORT_FUNC(PyLong_AsUnsignedLongLongMask) EXPORT_FUNC(PyLong_AsUnsignedLongMask) EXPORT_FUNC(PyLong_AsVoidPtr) EXPORT_FUNC(PyLong_FromDouble) +EXPORT_FUNC(PyLong_FromInt32) +EXPORT_FUNC(PyLong_FromInt64) EXPORT_FUNC(PyLong_FromLong) EXPORT_FUNC(PyLong_FromLongLong) EXPORT_FUNC(PyLong_FromSize_t) EXPORT_FUNC(PyLong_FromSsize_t) EXPORT_FUNC(PyLong_FromString) +EXPORT_FUNC(PyLong_FromUInt32) +EXPORT_FUNC(PyLong_FromUInt64) EXPORT_FUNC(PyLong_FromUnsignedLong) EXPORT_FUNC(PyLong_FromUnsignedLongLong) EXPORT_FUNC(PyLong_FromVoidPtr) @@ -630,14 +648,19 @@ EXPORT_FUNC(PyTuple_Pack) EXPORT_FUNC(PyTuple_SetItem) EXPORT_FUNC(PyTuple_Size) EXPORT_FUNC(PyType_ClearCache) +EXPORT_FUNC(PyType_Freeze) EXPORT_FUNC(PyType_FromMetaclass) EXPORT_FUNC(PyType_FromModuleAndSpec) EXPORT_FUNC(PyType_FromSpec) EXPORT_FUNC(PyType_FromSpecWithBases) EXPORT_FUNC(PyType_GenericAlloc) EXPORT_FUNC(PyType_GenericNew) +EXPORT_FUNC(PyType_GetBaseByToken) EXPORT_FUNC(PyType_GetFlags) +EXPORT_FUNC(PyType_GetFullyQualifiedName) EXPORT_FUNC(PyType_GetModule) +EXPORT_FUNC(PyType_GetModuleByDef) +EXPORT_FUNC(PyType_GetModuleName) EXPORT_FUNC(PyType_GetModuleState) EXPORT_FUNC(PyType_GetName) EXPORT_FUNC(PyType_GetQualName) @@ -697,6 +720,7 @@ EXPORT_FUNC(PyUnicode_DecodeUTF8Stateful) EXPORT_FUNC(PyUnicode_EncodeCodePage) EXPORT_FUNC(PyUnicode_EncodeFSDefault) EXPORT_FUNC(PyUnicode_EncodeLocale) +EXPORT_FUNC(PyUnicode_Equal) EXPORT_FUNC(PyUnicode_EqualToUTF8) EXPORT_FUNC(PyUnicode_EqualToUTF8AndSize) EXPORT_FUNC(PyUnicode_Find) @@ -831,7 +855,6 @@ EXPORT_DATA(PyExc_FutureWarning) EXPORT_DATA(PyExc_GeneratorExit) EXPORT_DATA(PyExc_ImportError) EXPORT_DATA(PyExc_ImportWarning) -EXPORT_DATA(PyExc_IncompleteInputError) EXPORT_DATA(PyExc_IndentationError) EXPORT_DATA(PyExc_IndexError) EXPORT_DATA(PyExc_InterruptedError) diff --git a/PC/python_ver_rc.h b/PC/python_ver_rc.h index 08509f96ed1db8..ee867fe41224c3 100644 --- a/PC/python_ver_rc.h +++ b/PC/python_ver_rc.h @@ -5,7 +5,7 @@ #include "winver.h" #define PYTHON_COMPANY "Python Software Foundation" -#define PYTHON_COPYRIGHT "Copyright \xA9 2001-2024 Python Software Foundation. Copyright \xA9 2000 BeOpen.com. Copyright \xA9 1995-2001 CNRI. Copyright \xA9 1991-1995 SMC." +#define PYTHON_COPYRIGHT "Copyright \xA9 2001 Python Software Foundation. Copyright \xA9 2000 BeOpen.com. Copyright \xA9 1995-2001 CNRI. Copyright \xA9 1991-1995 SMC." #define MS_WINDOWS #include "modsupport.h" diff --git a/PC/store_info.txt b/PC/store_info.txt index f6a85cb8ebec1f..d150ba17cbe62d 100644 --- a/PC/store_info.txt +++ b/PC/store_info.txt @@ -109,7 +109,7 @@ PSF LICENSE AGREEMENT FOR PYTHON 3.9 analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python 3.9 alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of - copyright, i.e., "Copyright © 2001-2018 Python Software Foundation; All Rights + copyright, i.e., "Copyright © 2001 Python Software Foundation; All Rights Reserved" are retained in Python 3.9 alone or in any derivative version prepared by Licensee. diff --git a/PC/venvlauncher.c b/PC/venvlauncher.c index fe97d32e93b5f6..b6bb0218236ae9 100644 --- a/PC/venvlauncher.c +++ b/PC/venvlauncher.c @@ -223,7 +223,7 @@ find_home_value(const char *buffer, DWORD maxlen, const char **start, DWORD *len return 0; } for (const char *s = strstr(buffer, "home"); - s && ((ptrdiff_t)s - (ptrdiff_t)buffer) < maxlen; + s && (size_t)((ptrdiff_t)s - (ptrdiff_t)buffer) < maxlen; s = strstr(s + 1, "\nhome") ) { if (*s == '\n') { @@ -484,8 +484,8 @@ process(int argc, wchar_t ** argv) // We do not update argv[0] to point at the target runtime, and so we do not // pass through our original argv[0] in an environment variable. - //exitCode = smuggle_path(); - //if (exitCode) return exitCode; + exitCode = smuggle_path(); + if (exitCode) return exitCode; exitCode = launch(home_path, GetCommandLineW()); return exitCode; diff --git a/PC/winreg.c b/PC/winreg.c index 77b80217ac0ab1..efdf8addc06186 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -200,7 +200,7 @@ PyHKEY_hashFunc(PyObject *ob) /* Just use the address. XXX - should we use the handle value? */ - return _Py_HashPointer(ob); + return PyObject_GenericHash(ob); } @@ -2179,6 +2179,7 @@ exec_module(PyObject *m) static PyModuleDef_Slot winreg_slots[] = { {Py_mod_exec, exec_module}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/PC/winsound.c b/PC/winsound.c index 7e4ebd90f50c2e..094c77ae34d678 100644 --- a/PC/winsound.c +++ b/PC/winsound.c @@ -35,11 +35,10 @@ winsound.PlaySound(None, 0) */ +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED - #ifndef Py_GIL_DISABLED -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include @@ -247,6 +246,7 @@ exec_module(PyObject *module) static PyModuleDef_Slot sound_slots[] = { {Py_mod_exec, exec_module}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/PCbuild/_ctypes_test.vcxproj b/PCbuild/_ctypes_test.vcxproj index 97354739c09834..f15b80852e362a 100644 --- a/PCbuild/_ctypes_test.vcxproj +++ b/PCbuild/_ctypes_test.vcxproj @@ -87,6 +87,7 @@ + @@ -94,6 +95,7 @@ + @@ -109,4 +111,4 @@ - \ No newline at end of file + diff --git a/PCbuild/_ctypes_test.vcxproj.filters b/PCbuild/_ctypes_test.vcxproj.filters index 5174196c52e4d0..618cfb32115e99 100644 --- a/PCbuild/_ctypes_test.vcxproj.filters +++ b/PCbuild/_ctypes_test.vcxproj.filters @@ -15,6 +15,9 @@ Header Files + + Header Files + @@ -26,4 +29,4 @@ Resource Files - \ No newline at end of file + diff --git a/PCbuild/_decimal.vcxproj b/PCbuild/_decimal.vcxproj index 490d7df87eb1c6..ee7421484b5312 100644 --- a/PCbuild/_decimal.vcxproj +++ b/PCbuild/_decimal.vcxproj @@ -93,51 +93,55 @@ - _CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + BUILD_LIBMPDEC;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) CONFIG_32;PPRO;MASM;%(PreprocessorDefinitions) CONFIG_32;ANSI;%(PreprocessorDefinitions) CONFIG_64;ANSI;%(PreprocessorDefinitions) CONFIG_64;MASM;%(PreprocessorDefinitions) - ..\Modules\_decimal;..\Modules\_decimal\libmpdec;%(AdditionalIncludeDirectories) + ..\Modules\_decimal;..\Modules\_decimal\windows;$(mpdecimalDir)\libmpdec;%(AdditionalIncludeDirectories) - - - - - - + + + + + + - - - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + true true true diff --git a/PCbuild/_decimal.vcxproj.filters b/PCbuild/_decimal.vcxproj.filters index 0cbd3d0736c241..e4bdb64ec1fb9f 100644 --- a/PCbuild/_decimal.vcxproj.filters +++ b/PCbuild/_decimal.vcxproj.filters @@ -21,49 +21,55 @@ Header Files - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + + Header Files\libmpdec + + + Header Files\libmpdec + + Header Files\libmpdec @@ -71,46 +77,46 @@ Source Files - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec @@ -120,8 +126,8 @@ - + Source Files\libmpdec - \ No newline at end of file + diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj index 8c9c0e42f7fe3e..3eb9c89bcb6740 100644 --- a/PCbuild/_elementtree.vcxproj +++ b/PCbuild/_elementtree.vcxproj @@ -106,7 +106,6 @@ - diff --git a/PCbuild/_elementtree.vcxproj.filters b/PCbuild/_elementtree.vcxproj.filters index bc14e31f32b95c..a5368024ccebfd 100644 --- a/PCbuild/_elementtree.vcxproj.filters +++ b/PCbuild/_elementtree.vcxproj.filters @@ -42,9 +42,6 @@ Header Files\expat - - Header Files\expat - Header Files\expat diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 3a8a417a6bf47a..51b493f8a84c6f 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -142,7 +142,6 @@ - @@ -195,6 +194,7 @@ + @@ -222,7 +222,10 @@ + + + @@ -239,6 +242,7 @@ + @@ -265,6 +269,7 @@ + @@ -476,25 +481,6 @@
    - - - - -$(IntDir)\deepfreeze_mappings.txt - - - - - - - - diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 5b34440af9322b..3842f52e514bb4 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -94,6 +94,12 @@ Source Files + + Source Files + + + Source Files + Source Files @@ -226,9 +232,18 @@ Source Files + + Source Files + Source Files + + Source Files + + + Source Files + Source Files @@ -241,9 +256,6 @@ Source Files - - Source Files - Source Files @@ -460,6 +472,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 66df0a61b5b5a6..c41235eac356af 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -96,11 +96,8 @@ - - - @@ -120,14 +117,16 @@ - - + + + + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 651eb1d6ba0b7f..0a00df655deefc 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -30,9 +30,6 @@ Source Files - - Source Files - Source Files @@ -90,27 +87,33 @@ Source Files - - Source Files - Source Files Source Files - - Source Files - Source Files Source Files + + Source Files + Source Files + + Source Files + + + Source Files + + + Source Files + diff --git a/PCbuild/_testinternalcapi.vcxproj b/PCbuild/_testinternalcapi.vcxproj index a825cac9138674..87db569423de2a 100644 --- a/PCbuild/_testinternalcapi.vcxproj +++ b/PCbuild/_testinternalcapi.vcxproj @@ -108,6 +108,12 @@ false + + + _Py_JIT;%(PreprocessorDefinitions) + _Py_TIER2=$(UseTIER2);%(PreprocessorDefinitions) + + diff --git a/PCbuild/_testinternalcapi.vcxproj.filters b/PCbuild/_testinternalcapi.vcxproj.filters index abfeeb39630daf..eb0a83ba17f815 100644 --- a/PCbuild/_testinternalcapi.vcxproj.filters +++ b/PCbuild/_testinternalcapi.vcxproj.filters @@ -21,10 +21,13 @@ Source Files + + Source Files + Resource Files - \ No newline at end of file + diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj new file mode 100644 index 00000000000000..87abff52493098 --- /dev/null +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -0,0 +1,134 @@ + + + + + Debug + ARM + + + Debug + ARM64 + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + ARM + + + PGInstrument + ARM64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + ARM + + + PGUpdate + ARM64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + ARM + + + Release + ARM64 + + + Release + Win32 + + + Release + x64 + + + + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933} + _testlimitedcapi + Win32Proj + false + + + + + DynamicLibrary + NotSet + + + + $(PyStdlibPydExt) + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + {885d4898-d08d-4091-9c40-c700cfe3fc5a} + false + + + + + + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters new file mode 100644 index 00000000000000..a975a508506905 --- /dev/null +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -0,0 +1,40 @@ + + + + + {a76a90d8-8e8b-4c36-8f58-8bd46abe9f5e} + + + {071b2ff4-e5a1-4e79-b0c5-cf46b0094a80} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Resource Files + + + diff --git a/PCbuild/_tkinter.vcxproj b/PCbuild/_tkinter.vcxproj index 117488a01621cc..87f6005fffc7c5 100644 --- a/PCbuild/_tkinter.vcxproj +++ b/PCbuild/_tkinter.vcxproj @@ -94,6 +94,7 @@ $(tcltkDir)include;%(AdditionalIncludeDirectories) + TCL_WITH_EXTERNAL_TOMMATH;%(PreprocessorDefinitions) WITH_APPINIT;%(PreprocessorDefinitions) Py_TCLTK_DIR="$(tcltkDir.TrimEnd('\').Replace('\', '\\'))";%(PreprocessorDefinitions) @@ -109,9 +110,10 @@ - <_TclTkDLL Include="$(tcltkdir)\bin\$(tclDllName)" /> - <_TclTkDLL Include="$(tcltkdir)\bin\$(tkDllName)" /> - <_TclTkDLL Include="$(tcltkdir)\bin\$(tclZlibDllName)" /> + <_TclTkDLL Include="$(tcltkdir)\bin\$(tclDLLName)" /> + <_TclTkDLL Include="$(tcltkdir)\bin\$(tkDLLName)" /> + <_TclTkDLL Include="$(tcltkdir)\bin\$(tclZlibDLLName)" /> + <_TclTkDLL Include="$(tcltkdir)\bin\$(tommathDLLName)" Condition="$(tommathDLLName) != ''"/> @@ -134,4 +136,4 @@ - \ No newline at end of file + diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 83b50db4467033..d3e3894c203a65 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -8,10 +8,10 @@ echo.version(s) of Microsoft Visual Studio to be installed (see readme.txt). echo. echo.After the flags recognized by this script, up to 9 arguments to be passed echo.directly to MSBuild may be passed. If the argument contains an '=', the -echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v100"`). +echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v141"`). echo.Alternatively you can put extra flags for MSBuild in a file named echo.`msbuild.rsp` in the `PCbuild` directory, one flag per line. This file -echo.will be picked automatically by MSBuild. Flags put in this file does not +echo.will be picked automatically by MSBuild. Flags put in this file do not echo.need to be quoted. You can still use environment variables inside the echo.response file. echo. @@ -36,7 +36,10 @@ echo. overrides -c and -d echo. --disable-gil Enable experimental support for running without the GIL. echo. --test-marker Enable the test marker within the build. echo. --regen Regenerate all opcodes, grammar and tokens. -echo. --experimental-jit Enable the experimental just-in-time compiler. +echo. --experimental-jit Enable the experimental just-in-time compiler. +echo. --experimental-jit-off Ditto but off by default (PYTHON_JIT=1 enables). +echo. --experimental-jit-interpreter Enable the experimental Tier 2 interpreter. +echo. --pystats Enable PyStats collection. echo. echo.Available flags to avoid building certain modules. echo.These flags have no effect if '-e' is not given: @@ -66,6 +69,7 @@ set verbose=/nologo /v:m /clp:summary set kill= set do_pgo= set pgo_job=-m test --pgo +set UseTIER2= :CheckOpts if "%~1"=="-h" goto Usage @@ -86,7 +90,11 @@ if "%~1"=="--disable-gil" (set UseDisableGil=true) & shift & goto CheckOpts if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts if "%~1"=="-V" shift & goto Version if "%~1"=="--regen" (set Regen=true) & shift & goto CheckOpts -if "%~1"=="--experimental-jit" (set UseJIT=true) & shift & goto CheckOpts +if "%~1"=="--experimental-jit" (set UseJIT=true) & (set UseTIER2=1) & shift & goto CheckOpts +if "%~1"=="--experimental-jit-off" (set UseJIT=true) & (set UseTIER2=3) & shift & goto CheckOpts +if "%~1"=="--experimental-jit-interpreter" (set UseTIER2=4) & shift & goto CheckOpts +if "%~1"=="--experimental-jit-interpreter-off" (set UseTIER2=6) & shift & goto CheckOpts +if "%~1"=="--pystats" (set PyStats=1) & shift & goto CheckOpts rem These use the actual property names used by MSBuild. We could just let rem them in through the environment, but we specify them on the command line rem anyway for visibility so set defaults after this @@ -179,6 +187,8 @@ echo on /p:DisableGil=%UseDisableGil%^ /p:UseTestMarker=%UseTestMarker% %GITProperty%^ /p:UseJIT=%UseJIT%^ + /p:UseTIER2=%UseTIER2%^ + /p:PyStats=%PyStats%^ %1 %2 %3 %4 %5 %6 %7 %8 %9 @echo off @@ -189,4 +199,4 @@ rem Display the current build version information call "%dir%find_msbuild.bat" %MSBUILD% if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) %MSBUILD% "%dir%pythoncore.vcxproj" /t:ShowVersionInfo /v:m /nologo %1 %2 %3 %4 %5 %6 %7 %8 %9 -if ERRORLEVEL 1 exit /b 3 \ No newline at end of file +if ERRORLEVEL 1 exit /b 3 diff --git a/PCbuild/env.bat b/PCbuild/env.bat index 2820e304582cff..cf4638b7aa63a7 100644 --- a/PCbuild/env.bat +++ b/PCbuild/env.bat @@ -4,8 +4,8 @@ rem command window. However, most builds of Python will ignore the version rem of the tools on PATH and use PlatformToolset instead. Ideally, both sets of rem tools should be the same version to avoid potential conflicts. rem -rem To build Python with an earlier toolset, pass "/p:PlatformToolset=v100" (or -rem 'v110', 'v120' or 'v140') to the build script. +rem To build Python with an earlier toolset, pass "/p:PlatformToolset=v141" (or +rem 'v142', 'v143') to the build script. echo Build environments: x86, amd64, x86_amd64 echo. @@ -20,8 +20,7 @@ call "%VSTOOLS%" %_ARGS% exit /B 0 :skip_vswhere -if not defined VSTOOLS set VSTOOLS=%VS140COMNTOOLS% -if not defined VSTOOLS set VSTOOLS=%VS120COMNTOOLS% -if not defined VSTOOLS set VSTOOLS=%VS110COMNTOOLS% -if not defined VSTOOLS set VSTOOLS=%VS100COMNTOOLS% +if not defined VSTOOLS set VSTOOLS=%VS143COMNTOOLS% +if not defined VSTOOLS set VSTOOLS=%VS142COMNTOOLS% +if not defined VSTOOLS set VSTOOLS=%VS141COMNTOOLS% call "%VSTOOLS%..\..\VC\vcvarsall.bat" %_ARGS% diff --git a/PCbuild/find_msbuild.bat b/PCbuild/find_msbuild.bat index ce7e71efa31f6c..82dd34beede6ee 100644 --- a/PCbuild/find_msbuild.bat +++ b/PCbuild/find_msbuild.bat @@ -39,16 +39,6 @@ @if defined MSBUILD @if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio installation) & goto :found :skip_vswhere -@rem VS 2015 and earlier register MSBuild separately, so we can find it. -@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul -@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @( - @if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe") -) -@if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found - - -@exit /b 1 - :found @pushd %MSBUILD% >nul 2>nul @if not ERRORLEVEL 1 @( diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat index af85f6d362466e..d65d080ca71a90 100644 --- a/PCbuild/find_python.bat +++ b/PCbuild/find_python.bat @@ -39,15 +39,15 @@ @if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%_Py_D%\..\externals) @rem If we have Python in externals, use that one -@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" ("%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" -Ec "import sys; assert sys.version_info[:2] >= (3, 8)" >nul 2>nul) && (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") && (set _Py_Python_Source=found in externals directory) && goto :found || rmdir /Q /S "%_Py_EXTERNALS_DIR%\pythonx86" +@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" ("%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" -Ec "import sys; assert sys.version_info[:2] >= (3, 10)" >nul 2>nul) && (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") && (set _Py_Python_Source=found in externals directory) && goto :found || rmdir /Q /S "%_Py_EXTERNALS_DIR%\pythonx86" @rem If HOST_PYTHON is recent enough, use that -@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 9)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found +@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 10)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found @rem If py.exe finds a recent enough version, use that one @rem It is fine to add new versions to this list when they have released, @rem but we do not use prerelease builds here. -@for %%p in (3.12 3.11 3.10 3.9) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found +@for %%p in (3.13 3.12 3.11 3.10) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found @if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%" @set _Py_NUGET=%NUGET% diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 60ce12b725e233..dfacd1d1e788d4 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -45,7 +45,7 @@ if "%ORG%"=="" (set ORG=python) call "%PCBUILD%\find_python.bat" "%PYTHON%" if NOT DEFINED PYTHON ( - where /Q git || echo Python 3.6 could not be found or installed, and git.exe is not on your PATH && exit /B 1 + where /Q git || echo Python 3.10 or later could not be found or installed, and git.exe is not on your PATH && exit /B 1 ) echo.Fetching external libraries... @@ -53,10 +53,11 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 -if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.13 -set libraries=%libraries% sqlite-3.45.1.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.1 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.1 +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.15 +set libraries=%libraries% mpdecimal-4.0.0 +set libraries=%libraries% sqlite-3.45.3.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.15.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.15.0 set libraries=%libraries% xz-5.2.5 set libraries=%libraries% zlib-1.3.1 @@ -76,8 +77,8 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.13 -if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.13.1 +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.15 +if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 for %%b in (%binaries%) do ( diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index b7b78be768d7ec..a2a637a3044373 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -77,7 +77,7 @@ - + diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index a0b5fbd1304b41..f2a48a7cb63666 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -1,6 +1,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.30028.174 +# Visual Studio Version 17 +VisualStudioVersion = 17.8.34525.116 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{553EC33E-9816-4996-A660-5D6186A0B0B3}" ProjectSection(SolutionItems) = preProject @@ -159,6 +159,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_wmi", "_wmi.vcxproj", "{54 EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testclinic_limited", "_testclinic_limited.vcxproj", "{01FDF29A-40A1-46DF-84F5-85EBBD2A2410}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testlimitedcapi", "_testlimitedcapi.vcxproj", "{7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|ARM = Debug|ARM @@ -1097,30 +1099,30 @@ Global {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|Win32.Build.0 = Release|Win32 {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|x64.ActiveCfg = Release|x64 {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|x64.Build.0 = Release|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|ARM.ActiveCfg = Debug|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|ARM.Build.0 = Debug|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|ARM64.ActiveCfg = Debug|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|ARM64.Build.0 = Debug|x64 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|ARM.ActiveCfg = Debug|ARM + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|ARM.Build.0 = Debug|ARM + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|ARM64.Build.0 = Debug|ARM64 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|Win32.ActiveCfg = Debug|Win32 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|Win32.Build.0 = Debug|Win32 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|x64.ActiveCfg = Debug|x64 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|x64.Build.0 = Debug|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|ARM.ActiveCfg = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|ARM.Build.0 = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|ARM64.ActiveCfg = Release|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|ARM64.Build.0 = Release|x64 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|ARM.ActiveCfg = Release|ARM + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|ARM.Build.0 = Release|ARM + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|ARM64.ActiveCfg = Release|ARM64 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|ARM64.Build.0 = Release|ARM64 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|Win32.ActiveCfg = Release|Win32 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|Win32.Build.0 = Release|Win32 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|x64.ActiveCfg = Release|x64 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|x64.Build.0 = Release|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|ARM.ActiveCfg = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|ARM64.ActiveCfg = Release|x64 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|ARM.ActiveCfg = Release|ARM + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|ARM64.ActiveCfg = Release|ARM64 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|Win32.ActiveCfg = Release|Win32 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|x64.ActiveCfg = Release|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|ARM.ActiveCfg = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|ARM.Build.0 = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|ARM64.ActiveCfg = Release|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|ARM64.Build.0 = Release|x64 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|ARM.ActiveCfg = Release|ARM + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|ARM.Build.0 = Release|ARM + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|ARM64.ActiveCfg = Release|ARM64 + {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|ARM64.Build.0 = Release|ARM64 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|Win32.ActiveCfg = Release|Win32 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|Win32.Build.0 = Release|Win32 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|x64.ActiveCfg = Release|x64 @@ -1439,26 +1441,26 @@ Global {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|Win32.Build.0 = Release|Win32 {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|x64.ActiveCfg = Release|x64 {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|x64.Build.0 = Release|x64 - {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Debug|ARM.ActiveCfg = Debug|Win32 - {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Debug|ARM64.ActiveCfg = Debug|Win32 + {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Debug|ARM.ActiveCfg = Debug|ARM + {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Debug|ARM64.ActiveCfg = Debug|ARM64 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Debug|Win32.ActiveCfg = Debug|Win32 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Debug|Win32.Build.0 = Debug|Win32 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Debug|x64.ActiveCfg = Debug|x64 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Debug|x64.Build.0 = Debug|x64 - {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGInstrument|ARM.ActiveCfg = PGInstrument|Win32 - {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGInstrument|ARM64.ActiveCfg = PGInstrument|Win32 + {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGInstrument|ARM.ActiveCfg = PGInstrument|ARM + {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGInstrument|ARM64.ActiveCfg = PGInstrument|ARM64 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGUpdate|ARM.ActiveCfg = PGUpdate|Win32 - {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGUpdate|ARM64.ActiveCfg = PGUpdate|Win32 + {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGUpdate|ARM.ActiveCfg = PGUpdate|ARM + {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGUpdate|ARM64.ActiveCfg = PGUpdate|ARM64 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Release|ARM.ActiveCfg = Release|Win32 - {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Release|ARM64.ActiveCfg = Release|Win32 + {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Release|ARM.ActiveCfg = Release|ARM + {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Release|ARM64.ActiveCfg = Release|ARM64 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Release|Win32.ActiveCfg = Release|Win32 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Release|Win32.Build.0 = Release|Win32 {9DE9E23D-C8D4-4817-92A9-920A8B1FE5FF}.Release|x64.ActiveCfg = Release|x64 @@ -1527,27 +1529,27 @@ Global {FDB84CBB-2FB6-47C8-A2D6-091E0833239D}.Release|Win32.Build.0 = Release|Win32 {FDB84CBB-2FB6-47C8-A2D6-091E0833239D}.Release|x64.ActiveCfg = Release|x64 {FDB84CBB-2FB6-47C8-A2D6-091E0833239D}.Release|x64.Build.0 = Release|x64 - {AB603547-1E2A-45B3-9E09-B04596006393}.Debug|ARM.ActiveCfg = Debug|Win32 - {AB603547-1E2A-45B3-9E09-B04596006393}.Debug|ARM.Build.0 = Debug|Win32 - {AB603547-1E2A-45B3-9E09-B04596006393}.Debug|ARM64.ActiveCfg = Debug|Win32 + {AB603547-1E2A-45B3-9E09-B04596006393}.Debug|ARM.ActiveCfg = Debug|ARM + {AB603547-1E2A-45B3-9E09-B04596006393}.Debug|ARM.Build.0 = Debug|ARM + {AB603547-1E2A-45B3-9E09-B04596006393}.Debug|ARM64.ActiveCfg = Debug|ARM64 {AB603547-1E2A-45B3-9E09-B04596006393}.Debug|Win32.ActiveCfg = Debug|Win32 {AB603547-1E2A-45B3-9E09-B04596006393}.Debug|Win32.Build.0 = Debug|Win32 {AB603547-1E2A-45B3-9E09-B04596006393}.Debug|x64.ActiveCfg = Debug|x64 {AB603547-1E2A-45B3-9E09-B04596006393}.Debug|x64.Build.0 = Debug|x64 - {AB603547-1E2A-45B3-9E09-B04596006393}.PGInstrument|ARM.ActiveCfg = PGInstrument|Win32 - {AB603547-1E2A-45B3-9E09-B04596006393}.PGInstrument|ARM64.ActiveCfg = PGInstrument|Win32 + {AB603547-1E2A-45B3-9E09-B04596006393}.PGInstrument|ARM.ActiveCfg = PGInstrument|ARM + {AB603547-1E2A-45B3-9E09-B04596006393}.PGInstrument|ARM64.ActiveCfg = PGInstrument|ARM64 {AB603547-1E2A-45B3-9E09-B04596006393}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 {AB603547-1E2A-45B3-9E09-B04596006393}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 {AB603547-1E2A-45B3-9E09-B04596006393}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 {AB603547-1E2A-45B3-9E09-B04596006393}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {AB603547-1E2A-45B3-9E09-B04596006393}.PGUpdate|ARM.ActiveCfg = PGUpdate|Win32 - {AB603547-1E2A-45B3-9E09-B04596006393}.PGUpdate|ARM64.ActiveCfg = PGUpdate|Win32 + {AB603547-1E2A-45B3-9E09-B04596006393}.PGUpdate|ARM.ActiveCfg = PGUpdate|ARM + {AB603547-1E2A-45B3-9E09-B04596006393}.PGUpdate|ARM64.ActiveCfg = PGUpdate|ARM64 {AB603547-1E2A-45B3-9E09-B04596006393}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 {AB603547-1E2A-45B3-9E09-B04596006393}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 {AB603547-1E2A-45B3-9E09-B04596006393}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 {AB603547-1E2A-45B3-9E09-B04596006393}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {AB603547-1E2A-45B3-9E09-B04596006393}.Release|ARM.ActiveCfg = Release|Win32 - {AB603547-1E2A-45B3-9E09-B04596006393}.Release|ARM64.ActiveCfg = Release|Win32 + {AB603547-1E2A-45B3-9E09-B04596006393}.Release|ARM.ActiveCfg = Release|ARM + {AB603547-1E2A-45B3-9E09-B04596006393}.Release|ARM64.ActiveCfg = Release|ARM64 {AB603547-1E2A-45B3-9E09-B04596006393}.Release|Win32.ActiveCfg = Release|Win32 {AB603547-1E2A-45B3-9E09-B04596006393}.Release|Win32.Build.0 = Release|Win32 {AB603547-1E2A-45B3-9E09-B04596006393}.Release|x64.ActiveCfg = Release|x64 @@ -1648,6 +1650,38 @@ Global {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Release|Win32.Build.0 = Release|Win32 {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Release|x64.ActiveCfg = Release|x64 {01FDF29A-40A1-46DF-84F5-85EBBD2A2410}.Release|x64.Build.0 = Release|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|ARM.ActiveCfg = Debug|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|ARM.Build.0 = Debug|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|ARM64.Build.0 = Debug|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|Win32.ActiveCfg = Debug|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|Win32.Build.0 = Debug|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|x64.ActiveCfg = Debug|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Debug|x64.Build.0 = Debug|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|ARM.ActiveCfg = PGInstrument|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|ARM.Build.0 = PGInstrument|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|ARM64.ActiveCfg = PGInstrument|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|ARM64.Build.0 = PGInstrument|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|ARM.ActiveCfg = PGUpdate|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|ARM.Build.0 = PGUpdate|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|ARM64.ActiveCfg = PGUpdate|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|ARM64.Build.0 = PGUpdate|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|ARM.ActiveCfg = Release|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|ARM.Build.0 = Release|ARM + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|ARM64.ActiveCfg = Release|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|ARM64.Build.0 = Release|ARM64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|Win32.ActiveCfg = Release|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|Win32.Build.0 = Release|Win32 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|x64.ActiveCfg = Release|x64 + {7467D86C-1CEB-4CB9-B65E-E9A54ABDC933}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/PCbuild/pylauncher.vcxproj b/PCbuild/pylauncher.vcxproj index 35f2f7e505bf92..ca422ef4d02e93 100644 --- a/PCbuild/pylauncher.vcxproj +++ b/PCbuild/pylauncher.vcxproj @@ -103,7 +103,7 @@ - + diff --git a/PCbuild/pylauncher.vcxproj.filters b/PCbuild/pylauncher.vcxproj.filters index 17d0389ca50f1c..f10f6d5669e283 100644 --- a/PCbuild/pylauncher.vcxproj.filters +++ b/PCbuild/pylauncher.vcxproj.filters @@ -16,7 +16,7 @@ - + Resource Files diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 9c85e5efa4af4a..dbdb6b743bea37 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -42,6 +42,8 @@ <_DebugPreprocessorDefinition>NDEBUG; <_DebugPreprocessorDefinition Condition="$(Configuration) == 'Debug'">_DEBUG; + <_PyStatsPreprocessorDefinition>PyStats; + <_PyStatsPreprocessorDefinition Condition="$(PySTATS) != ''">Py_STATS; <_PlatformPreprocessorDefinition>_WIN32; <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64'">_WIN64; <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64' and $(PlatformToolset) != 'ClangCL'">_M_X64;$(_PlatformPreprocessorDefinition) @@ -50,8 +52,7 @@ $(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)Include\internal\mimalloc;$(GeneratedPyConfigDir);$(PySourcePath)PC;%(AdditionalIncludeDirectories) - WIN32;$(_Py3NamePreprocessorDefinition);$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions) - _Py_USING_PGO=1;%(PreprocessorDefinitions) + WIN32;$(_Py3NamePreprocessorDefinition);$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PyStatsPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions) MaxSpeed true @@ -217,11 +218,6 @@ public override bool Execute() { - - - $(VCInstallDir)\redist\ - - <_RedistFiles Include="$(VCInstallDir)\Redist\MSVC\*\*.*" /> diff --git a/PCbuild/python.props b/PCbuild/python.props index e21f1f60464bc8..6e90178f4ea8ab 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -6,7 +6,7 @@ Release - - + @@ -716,7 +739,7 @@ - + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 27bd1121663398..6c76a6ab592a84 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -48,6 +48,9 @@ Include + + Include + Include @@ -72,6 +75,9 @@ Include + + Include + Include @@ -114,6 +120,9 @@ Include + + Include + Include @@ -207,6 +216,9 @@ Include + + Include + Include @@ -219,9 +231,6 @@ Include - - Include - Include @@ -330,12 +339,6 @@ Include - - Include - - - Modules - Parser @@ -348,6 +351,9 @@ Include\cpython + + Include\cpython + Include\cpython @@ -375,6 +381,9 @@ Include\cpython + + Include\cpython + Include\cpython @@ -396,6 +405,9 @@ Include\cpython + + Include + Include @@ -408,9 +420,6 @@ Include - - Include - Include\cpython @@ -423,21 +432,18 @@ Include\cpython + + Include\cpython + Include\cpython Include\cpython - - Include - Include - - Include - Include @@ -492,9 +498,6 @@ Include - - Include\cpython - Include\cpython @@ -504,9 +507,6 @@ Include\cpython - - Include\cpython - Include\cpython @@ -543,6 +543,9 @@ Include\internal + + Include\internal + Include\internal @@ -561,6 +564,9 @@ Include\internal + + Include\internal + Include\internal @@ -591,6 +597,12 @@ Include\internal + + Include\internal + + + Include\internal + Include\internal @@ -627,6 +639,9 @@ Include\internal + + Include\internal + Include\internal @@ -663,9 +678,15 @@ Include\internal + + Include\internal + Include\internal + + Include\internal + Include\internal @@ -702,6 +723,9 @@ Include\internal + + Include\internal + Include\internal @@ -789,6 +813,9 @@ Include\internal + + Include\internal + Include\internal @@ -819,6 +846,9 @@ Include\internal + + Include\internal + Include\internal\mimalloc @@ -861,7 +891,7 @@ Modules\zlib - + Modules\zlib @@ -881,15 +911,6 @@ Modules - - Modules - - - Modules - - - Modules - Modules @@ -1271,6 +1292,9 @@ Python + + Python + Python @@ -1343,12 +1367,21 @@ Python + + Python + Python + + Python + Source Files + + Source Files + Source Files @@ -1394,6 +1427,9 @@ Python + + Python + Python @@ -1460,6 +1496,9 @@ Python + + Python + Python @@ -1475,9 +1514,6 @@ Objects - - Objects - Modules @@ -1544,13 +1580,13 @@ Parser - + Modules - + Modules - + Modules diff --git a/PCbuild/pythonw.vcxproj b/PCbuild/pythonw.vcxproj index d08c210ef8a1dc..c6a5b8ce90a0d9 100644 --- a/PCbuild/pythonw.vcxproj +++ b/PCbuild/pythonw.vcxproj @@ -97,6 +97,9 @@ 3000000 + + + diff --git a/PCbuild/pythonw.vcxproj.filters b/PCbuild/pythonw.vcxproj.filters index 20d87051e2511d..9f92485d1a74d2 100644 --- a/PCbuild/pythonw.vcxproj.filters +++ b/PCbuild/pythonw.vcxproj.filters @@ -18,4 +18,9 @@ Resource Files + + + Resource Files + + \ No newline at end of file diff --git a/PCbuild/pythonw_uwp.vcxproj b/PCbuild/pythonw_uwp.vcxproj index e21e46a1b722ed..f548d022642890 100644 --- a/PCbuild/pythonw_uwp.vcxproj +++ b/PCbuild/pythonw_uwp.vcxproj @@ -108,7 +108,7 @@ - + diff --git a/PCbuild/pythonw_uwp.vcxproj.filters b/PCbuild/pythonw_uwp.vcxproj.filters index 2f39bdea9e910d..72756e14cdc4a9 100644 --- a/PCbuild/pythonw_uwp.vcxproj.filters +++ b/PCbuild/pythonw_uwp.vcxproj.filters @@ -9,7 +9,7 @@ - + Resource Files diff --git a/PCbuild/pywlauncher.vcxproj b/PCbuild/pywlauncher.vcxproj index e50b69aefe2b9c..1694548935a165 100644 --- a/PCbuild/pywlauncher.vcxproj +++ b/PCbuild/pywlauncher.vcxproj @@ -103,7 +103,7 @@ - + diff --git a/PCbuild/pywlauncher.vcxproj.filters b/PCbuild/pywlauncher.vcxproj.filters index 17d0389ca50f1c..f10f6d5669e283 100644 --- a/PCbuild/pywlauncher.vcxproj.filters +++ b/PCbuild/pywlauncher.vcxproj.filters @@ -16,7 +16,7 @@ - + Resource Files diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index a7c2a68928ca84..693fcee5f90ce2 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -3,7 +3,7 @@ Quick Start Guide 1. Install Microsoft Visual Studio 2017 or later with Python workload and Python native development component. -1a. Optionally install Python 3.6 or later. If not installed, +1a. Optionally install Python 3.10 or later. If not installed, get_externals.bat (via build.bat) will download and use Python via NuGet. 2. Run "build.bat" to build Python in 32-bit Release configuration. @@ -57,7 +57,7 @@ Building Python using the build.bat script In this directory you can find build.bat, a script designed to make building Python on Windows simpler. This script will use the env.bat -script to detect either Visual Studio 2017 or 2015, either of +script to detect either Visual Studio 2017 or later, either of which may be used to build Python. Currently Visual Studio 2017 is officially supported. @@ -131,29 +131,31 @@ xxlimited_35 The following sub-projects are for individual modules of the standard library which are implemented in C; each one builds a DLL (renamed to .pyd) of the same name as the project: -_asyncio -_ctypes -_ctypes_test -_zoneinfo -_decimal -_elementtree -_hashlib -_multiprocessing -_overlapped -_socket -_testbuffer -_testcapi -_testclinic -_testclinic_limited -_testconsole -_testimportmultiple -_testmultiphase -_testsinglephase -_tkinter -pyexpat -select -unicodedata -winsound + * _asyncio + * _ctypes + * _ctypes_test + * _zoneinfo + * _decimal + * _elementtree + * _hashlib + * _multiprocessing + * _overlapped + * _socket + * _testbuffer + * _testcapi + * _testlimitedcapi + * _testinternalcapi + * _testclinic + * _testclinic_limited + * _testconsole + * _testimportmultiple + * _testmultiphase + * _testsinglephase + * _tkinter + * pyexpat + * select + * unicodedata + * winsound The following Python-controlled sub-projects wrap external projects. Note that these external libraries are not necessary for a working @@ -170,7 +172,7 @@ _lzma Homepage: https://tukaani.org/xz/ _ssl - Python wrapper for version 3.0 of the OpenSSL secure sockets + Python wrapper for version 3.0.15 of the OpenSSL secure sockets library, which is downloaded from our binaries repository at https://github.com/python/cpython-bin-deps. @@ -189,11 +191,11 @@ _ssl again when building. _sqlite3 - Wraps SQLite 3.45.1, which is itself built by sqlite3.vcxproj + Wraps SQLite 3.45.3, which is itself built by sqlite3.vcxproj Homepage: https://www.sqlite.org/ _tkinter - Wraps version 8.6.6 of the Tk windowing system, which is downloaded + Wraps version 8.6.15 of the Tk windowing system, which is downloaded from our binaries repository at https://github.com/python/cpython-bin-deps. @@ -309,6 +311,8 @@ _testclinic_limited extension, the file Modules/_testclinic_limited.c: * In PCbuild/, copy _testclinic.vcxproj to _testclinic_limited.vcxproj, replace RootNamespace value with `_testclinic_limited`, replace `_asyncio.c` with `_testclinic_limited.c`. +* In PCbuild/, copy _testclinic.vcxproj.filters to + _testclinic_limited.vcxproj.filters, edit the list of files in the new file. * Open Visual Studio, open PCbuild\pcbuild.sln solution, add the PCbuild\_testclinic_limited.vcxproj project to the solution ("add existing project). diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index f36387231a0d3a..416241d9d0df10 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -33,11 +33,15 @@ <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/> <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/> <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/> + <_SbomSources Include="$(PySourcePath)PCbuild\get_externals.bat" /> + <_SbomOutputs Include="$(PySourcePath)Misc\externals.spdx.json;$(PySourcePath)Misc\sbom.spdx.json"> + json + - @@ -86,23 +90,23 @@ Inputs="@(_CasesSources)" Outputs="@(_CasesOutputs)" DependsOnTargets="FindPythonForBuild"> - - - - - - - - - @@ -126,7 +130,14 @@ DependsOnTargets="_TouchRegenSources;_RegenPegen;_RegenAST_H;_RegenTokens;_RegenKeywords;_RegenGlobalObjects">
    - + + + + + @@ -150,9 +161,7 @@ Condition="($(Platform) == 'Win32' or $(Platform) == 'x64') and $(Configuration) != 'PGInstrument' and $(Configuration) != 'PGUpdate'"> - diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index ac530a5206271f..c436215780fda0 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -42,7 +42,7 @@ if "%~1"=="-O" (set dashO=-O) & shift & goto CheckOpts if "%~1"=="-q" (set qmode=yes) & shift & goto CheckOpts if "%~1"=="-d" (set suffix=_d) & shift & goto CheckOpts rem HACK: Need some way to infer the version number in this script -if "%~1"=="--disable-gil" (set pyname=python3.13t) & shift & goto CheckOpts +if "%~1"=="--disable-gil" (set pyname=python3.14t) & shift & goto CheckOpts if "%~1"=="-win32" (set prefix=%pcbuild%win32) & shift & goto CheckOpts if "%~1"=="-x64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts if "%~1"=="-amd64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts diff --git a/PCbuild/tcltk.props b/PCbuild/tcltk.props index 8ddf01d5dd1dca..d26b36ba98e493 100644 --- a/PCbuild/tcltk.props +++ b/PCbuild/tcltk.props @@ -2,7 +2,7 @@ - 8.6.13.1 + 8.6.15.0 $(TclVersion) $([System.Version]::Parse($(TclVersion)).Major) $([System.Version]::Parse($(TclVersion)).Minor) @@ -17,15 +17,21 @@ $(ExternalsDir)tcltk-$(TclVersion)\$(ArchName)\ $(tcltkDir)\bin\tclsh$(TclMajorVersion)$(TclMinorVersion)t.exe $(tcltkDir)\..\win32\bin\tclsh$(TclMajorVersion)$(TclMinorVersion)t.exe + TCL_WITH_EXTERNAL_TOMMATH; - tcl$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).dll - tcl$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).lib - tclsh$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).exe - tk$(TkMajorVersion)$(TkMinorVersion)t$(TclDebugExt).dll - tk$(TkMajorVersion)$(TkMinorVersion)t$(TclDebugExt).lib + t + tcl9 + tcl$(TclMajorVersion)$(TclMinorVersion)$(tcltkSuffix)$(TclDebugExt).dll + tcl$(TclMajorVersion)$(TclMinorVersion)$(tcltkSuffix)$(TclDebugExt).lib + tclsh$(TclMajorVersion)$(TclMinorVersion)$(tcltkSuffix)$(TclDebugExt).exe + $(tkPrefix)tk$(TkMajorVersion)$(TkMinorVersion)$(tcltkSuffix)$(TclDebugExt).dll + $(tkPrefix)tk$(TclMajorVersion)$(TclMinorVersion)$(tcltkSuffix)$(TclDebugExt).lib zlib1.dll - $(tcltkDir)lib\tcl$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).lib;$(tcltkDir)lib\tk$(TkMajorVersion)$(TkMinorVersion)t$(TclDebugExt).lib + libtommath.dll + tommath.lib + $(tcltkDir)lib\$(TclLibName);$(tcltkDir)lib\$(TkLibName); + $(tcltkLib);$(tcltkDir)lib\$(tommathLibName) IX86 AMD64 ARM64 @@ -37,9 +43,6 @@ Debug $(BuildDirTop)_$(TclMachine) $(BuildDirTop)_VC13 - $(BuildDirTop)_VC12 - $(BuildDirTop)_VC11 - $(BuildDirTop)_VC10 0) { - *outp++ = '0' + ch/base; - ch %= base; - base /= 10; - } - *outp++ = ';'; + else if (ch < 1000) { + ressize += 2 + 3 + 1; + } + else if (ch < 10000) { + ressize += 2 + 4 + 1; + } + else if (ch < 100000) { + ressize += 2 + 5 + 1; + } + else if (ch < 1000000) { + ressize += 2 + 6 + 1; + } + else { + assert(ch < 10000000); + ressize += 2 + 7 + 1; } - assert(_PyUnicode_CheckConsistency(res, 1)); - restuple = Py_BuildValue("(Nn)", res, end); - Py_DECREF(object); - return restuple; } - else { - wrong_exception_type(exc); + + /* allocate replacement */ + PyObject *res = PyUnicode_New(ressize, 127); + if (res == NULL) { + Py_DECREF(obj); return NULL; } + Py_UCS1 *outp = PyUnicode_1BYTE_DATA(res); + /* generate replacement */ + for (Py_ssize_t i = start; i < end; ++i) { + int digits, base; + Py_UCS4 ch = PyUnicode_READ_CHAR(obj, i); + if (ch < 10) { + digits = 1; + base = 1; + } + else if (ch < 100) { + digits = 2; + base = 10; + } + else if (ch < 1000) { + digits = 3; + base = 100; + } + else if (ch < 10000) { + digits = 4; + base = 1000; + } + else if (ch < 100000) { + digits = 5; + base = 10000; + } + else if (ch < 1000000) { + digits = 6; + base = 100000; + } + else { + assert(ch < 10000000); + digits = 7; + base = 1000000; + } + *outp++ = '&'; + *outp++ = '#'; + while (digits-- > 0) { + assert(base >= 1); + *outp++ = '0' + ch / base; + ch %= base; + base /= 10; + } + *outp++ = ';'; + } + assert(_PyUnicode_CheckConsistency(res, 1)); + PyObject *restuple = Py_BuildValue("(Nn)", res, end); + Py_DECREF(obj); + return restuple; } PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc) { - PyObject *object; - Py_ssize_t i; - Py_ssize_t start; - Py_ssize_t end; - PyObject *res; - Py_UCS1 *outp; - int ressize; - Py_UCS4 c; - + PyObject *obj; + Py_ssize_t objlen, start, end, slen; if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeDecodeError)) { - const unsigned char *p; - if (PyUnicodeDecodeError_GetStart(exc, &start)) - return NULL; - if (PyUnicodeDecodeError_GetEnd(exc, &end)) - return NULL; - if (!(object = PyUnicodeDecodeError_GetObject(exc))) + if (_PyUnicodeError_GetParams(exc, + &obj, &objlen, + &start, &end, &slen, true) < 0) + { return NULL; - p = (const unsigned char*)PyBytes_AS_STRING(object); - res = PyUnicode_New(4 * (end - start), 127); + } + PyObject *res = PyUnicode_New(4 * slen, 127); if (res == NULL) { - Py_DECREF(object); + Py_DECREF(obj); return NULL; } - outp = PyUnicode_1BYTE_DATA(res); - for (i = start; i < end; i++, outp += 4) { - unsigned char c = p[i]; + Py_UCS1 *outp = PyUnicode_1BYTE_DATA(res); + const unsigned char *p = (const unsigned char *)PyBytes_AS_STRING(obj); + for (Py_ssize_t i = start; i < end; i++, outp += 4) { + const unsigned char ch = p[i]; outp[0] = '\\'; outp[1] = 'x'; - outp[2] = Py_hexdigits[(c>>4)&0xf]; - outp[3] = Py_hexdigits[c&0xf]; + outp[2] = Py_hexdigits[(ch >> 4) & 0xf]; + outp[3] = Py_hexdigits[ch & 0xf]; } - assert(_PyUnicode_CheckConsistency(res, 1)); - Py_DECREF(object); + Py_DECREF(obj); return Py_BuildValue("(Nn)", res, end); } - if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) { - if (PyUnicodeEncodeError_GetStart(exc, &start)) - return NULL; - if (PyUnicodeEncodeError_GetEnd(exc, &end)) - return NULL; - if (!(object = PyUnicodeEncodeError_GetObject(exc))) - return NULL; - } - else if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeTranslateError)) { - if (PyUnicodeTranslateError_GetStart(exc, &start)) - return NULL; - if (PyUnicodeTranslateError_GetEnd(exc, &end)) - return NULL; - if (!(object = PyUnicodeTranslateError_GetObject(exc))) + + if ( + PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError) + || PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeTranslateError) + ) { + if (_PyUnicodeError_GetParams(exc, + &obj, &objlen, + &start, &end, &slen, false) < 0) + { return NULL; + } } else { wrong_exception_type(exc); return NULL; } - if (end - start > PY_SSIZE_T_MAX / (1+1+8)) - end = start + PY_SSIZE_T_MAX / (1+1+8); - for (i = start, ressize = 0; i < end; ++i) { + // The number of characters that each character 'ch' contributes + // in the result is 1 + 1 + k, where k >= min{t >= 1 | 16^t > ch} + // and will be formatted as "\\" + ('U'|'u'|'x') + HEXDIGITS, + // where the number of hexdigits is either 2, 4, or 8 (not 6). + // Since the Unicode range is below 10^7, we choose k = 8 whence + // each "block" requires at most 1 + 1 + 8 characters. + if (slen > PY_SSIZE_T_MAX / (1 + 1 + 8)) { + end = start + PY_SSIZE_T_MAX / (1 + 1 + 8); + end = Py_MIN(end, objlen); + slen = Py_MAX(0, end - start); + } + + Py_ssize_t ressize = 0; + for (Py_ssize_t i = start; i < end; ++i) { /* object is guaranteed to be "ready" */ - c = PyUnicode_READ_CHAR(object, i); + Py_UCS4 c = PyUnicode_READ_CHAR(obj, i); if (c >= 0x10000) { - ressize += 1+1+8; + ressize += 1 + 1 + 8; } else if (c >= 0x100) { - ressize += 1+1+4; + ressize += 1 + 1 + 4; + } + else { + ressize += 1 + 1 + 2; } - else - ressize += 1+1+2; } - res = PyUnicode_New(ressize, 127); + PyObject *res = PyUnicode_New(ressize, 127); if (res == NULL) { - Py_DECREF(object); + Py_DECREF(obj); return NULL; } - outp = PyUnicode_1BYTE_DATA(res); - for (i = start; i < end; ++i) { - c = PyUnicode_READ_CHAR(object, i); + Py_UCS1 *outp = PyUnicode_1BYTE_DATA(res); + for (Py_ssize_t i = start; i < end; ++i) { + Py_UCS4 c = PyUnicode_READ_CHAR(obj, i); *outp++ = '\\'; if (c >= 0x00010000) { *outp++ = 'U'; - *outp++ = Py_hexdigits[(c>>28)&0xf]; - *outp++ = Py_hexdigits[(c>>24)&0xf]; - *outp++ = Py_hexdigits[(c>>20)&0xf]; - *outp++ = Py_hexdigits[(c>>16)&0xf]; - *outp++ = Py_hexdigits[(c>>12)&0xf]; - *outp++ = Py_hexdigits[(c>>8)&0xf]; + *outp++ = Py_hexdigits[(c >> 28) & 0xf]; + *outp++ = Py_hexdigits[(c >> 24) & 0xf]; + *outp++ = Py_hexdigits[(c >> 20) & 0xf]; + *outp++ = Py_hexdigits[(c >> 16) & 0xf]; + *outp++ = Py_hexdigits[(c >> 12) & 0xf]; + *outp++ = Py_hexdigits[(c >> 8) & 0xf]; } else if (c >= 0x100) { *outp++ = 'u'; - *outp++ = Py_hexdigits[(c>>12)&0xf]; - *outp++ = Py_hexdigits[(c>>8)&0xf]; + *outp++ = Py_hexdigits[(c >> 12) & 0xf]; + *outp++ = Py_hexdigits[(c >> 8) & 0xf]; } - else + else { *outp++ = 'x'; - *outp++ = Py_hexdigits[(c>>4)&0xf]; - *outp++ = Py_hexdigits[c&0xf]; + } + *outp++ = Py_hexdigits[(c >> 4) & 0xf]; + *outp++ = Py_hexdigits[c & 0xf]; } - assert(_PyUnicode_CheckConsistency(res, 1)); - Py_DECREF(object); + Py_DECREF(obj); return Py_BuildValue("(Nn)", res, end); } @@ -1331,13 +1388,17 @@ PyCodec_SurrogateEscapeErrors(PyObject *exc) } -static PyObject *strict_errors(PyObject *self, PyObject *exc) +// --- Codecs registry handlers ----------------------------------------------- + +static inline PyObject * +strict_errors(PyObject *Py_UNUSED(self), PyObject *exc) { return PyCodec_StrictErrors(exc); } -static PyObject *ignore_errors(PyObject *self, PyObject *exc) +static inline PyObject * +ignore_errors(PyObject *Py_UNUSED(self), PyObject *exc) { return PyCodec_IgnoreErrors(exc); } @@ -1375,7 +1436,8 @@ static PyObject *surrogateescape_errors(PyObject *self, PyObject *exc) return PyCodec_SurrogateEscapeErrors(exc); } -static int _PyCodecRegistry_Init(void) +PyStatus +_PyCodec_InitRegistry(PyInterpreterState *interp) { static struct { const char *name; @@ -1462,46 +1524,54 @@ static int _PyCodecRegistry_Init(void) } } }; + // ensure that the built-in error handlers' names are kept in sync + assert(Py_ARRAY_LENGTH(methods) == Py_ARRAY_LENGTH(codecs_builtin_error_handlers)); - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *mod; - - if (interp->codec_search_path != NULL) - return 0; - - interp->codec_search_path = PyList_New(0); - if (interp->codec_search_path == NULL) { - return -1; + assert(interp->codecs.initialized == 0); + interp->codecs.search_path = PyList_New(0); + if (interp->codecs.search_path == NULL) { + return PyStatus_NoMemory(); } - - interp->codec_search_cache = PyDict_New(); - if (interp->codec_search_cache == NULL) { - return -1; + interp->codecs.search_cache = PyDict_New(); + if (interp->codecs.search_cache == NULL) { + return PyStatus_NoMemory(); } - - interp->codec_error_registry = PyDict_New(); - if (interp->codec_error_registry == NULL) { - return -1; + interp->codecs.error_registry = PyDict_New(); + if (interp->codecs.error_registry == NULL) { + return PyStatus_NoMemory(); } - for (size_t i = 0; i < Py_ARRAY_LENGTH(methods); ++i) { PyObject *func = PyCFunction_NewEx(&methods[i].def, NULL, NULL); - if (!func) { - return -1; + if (func == NULL) { + return PyStatus_NoMemory(); } - int res = PyCodec_RegisterError(methods[i].name, func); + int res = PyDict_SetItemString(interp->codecs.error_registry, + methods[i].name, func); Py_DECREF(func); - if (res) { - return -1; + if (res < 0) { + return PyStatus_Error("Failed to insert into codec error registry"); } } - mod = PyImport_ImportModule("encodings"); + interp->codecs.initialized = 1; + + // Importing `encodings' will call back into this module to register codec + // search functions, so this is done after everything else is initialized. + PyObject *mod = PyImport_ImportModule("encodings"); if (mod == NULL) { - return -1; + return PyStatus_Error("Failed to import encodings module"); } Py_DECREF(mod); - interp->codecs_initialized = 1; - return 0; + + return PyStatus_Ok(); +} + +void +_PyCodec_Fini(PyInterpreterState *interp) +{ + Py_CLEAR(interp->codecs.search_path); + Py_CLEAR(interp->codecs.search_cache); + Py_CLEAR(interp->codecs.error_registry); + interp->codecs.initialized = 0; } diff --git a/Python/codegen.c b/Python/codegen.c new file mode 100644 index 00000000000000..df3b5aaac1d0d9 --- /dev/null +++ b/Python/codegen.c @@ -0,0 +1,6215 @@ +/* + * This file implements the compiler's code generation stage, which + * produces a sequence of pseudo-instructions from an AST. + * + * The primary entry point is _PyCodegen_Body() for modules, and + * _PyCodegen_Expression() for expressions. + * + * CAUTION: The VISIT_* macros abort the current function when they + * encounter a problem. So don't invoke them when there is memory + * which needs to be released. Code blocks are OK, as the compiler + * structure takes care of releasing those. Use the arena to manage + * objects. + */ + +#include + +#include "Python.h" +#include "opcode.h" +#include "pycore_ast.h" // _PyAST_GetDocString() +#define NEED_OPCODE_TABLES +#include "pycore_opcode_utils.h" +#undef NEED_OPCODE_TABLES +#include "pycore_compile.h" +#include "pycore_instruction_sequence.h" // _PyInstructionSequence_NewLabel() +#include "pycore_intrinsics.h" +#include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_object.h" // _Py_ANNOTATE_FORMAT_VALUE_WITH_FAKE_GLOBALS +#include "pycore_pystate.h" // _Py_GetConfig() +#include "pycore_symtable.h" // PySTEntryObject + +#define NEED_OPCODE_METADATA +#include "pycore_opcode_metadata.h" // _PyOpcode_opcode_metadata, _PyOpcode_num_popped/pushed +#undef NEED_OPCODE_METADATA + +#define COMP_GENEXP 0 +#define COMP_LISTCOMP 1 +#define COMP_SETCOMP 2 +#define COMP_DICTCOMP 3 + +/* A soft limit for stack use, to avoid excessive + * memory use for large constants, etc. + * + * The value 30 is plucked out of thin air. + * Code that could use more stack than this is + * rare, so the exact value is unimportant. + */ +#define STACK_USE_GUIDELINE 30 + +#undef SUCCESS +#undef ERROR +#define SUCCESS 0 +#define ERROR -1 + +#define RETURN_IF_ERROR(X) \ + do { \ + if ((X) == -1) { \ + return ERROR; \ + } \ + } while (0) + +#define RETURN_IF_ERROR_IN_SCOPE(C, CALL) \ + do { \ + if ((CALL) < 0) { \ + _PyCompile_ExitScope((C)); \ + return ERROR; \ + } \ + } while (0) + +struct _PyCompiler; +typedef struct _PyCompiler compiler; + +#define INSTR_SEQUENCE(C) _PyCompile_InstrSequence(C) +#define FUTURE_FEATURES(C) _PyCompile_FutureFeatures(C) +#define SYMTABLE(C) _PyCompile_Symtable(C) +#define SYMTABLE_ENTRY(C) _PyCompile_SymtableEntry(C) +#define OPTIMIZATION_LEVEL(C) _PyCompile_OptimizationLevel(C) +#define IS_INTERACTIVE_TOP_LEVEL(C) _PyCompile_IsInteractiveTopLevel(C) +#define SCOPE_TYPE(C) _PyCompile_ScopeType(C) +#define QUALNAME(C) _PyCompile_Qualname(C) +#define METADATA(C) _PyCompile_Metadata(C) + +typedef _PyInstruction instruction; +typedef _PyInstructionSequence instr_sequence; +typedef _Py_SourceLocation location; +typedef _PyJumpTargetLabel jump_target_label; + +typedef _PyCompile_FBlockInfo fblockinfo; + +#define LOCATION(LNO, END_LNO, COL, END_COL) \ + ((const _Py_SourceLocation){(LNO), (END_LNO), (COL), (END_COL)}) + +#define LOC(x) SRC_LOCATION_FROM_AST(x) + +#define NEW_JUMP_TARGET_LABEL(C, NAME) \ + jump_target_label NAME = _PyInstructionSequence_NewLabel(INSTR_SEQUENCE(C)); \ + if (!IS_JUMP_TARGET_LABEL(NAME)) { \ + return ERROR; \ + } + +#define USE_LABEL(C, LBL) \ + RETURN_IF_ERROR(_PyInstructionSequence_UseLabel(INSTR_SEQUENCE(C), (LBL).id)) + +static const int compare_masks[] = { + [Py_LT] = COMPARISON_LESS_THAN, + [Py_LE] = COMPARISON_LESS_THAN | COMPARISON_EQUALS, + [Py_EQ] = COMPARISON_EQUALS, + [Py_NE] = COMPARISON_NOT_EQUALS, + [Py_GT] = COMPARISON_GREATER_THAN, + [Py_GE] = COMPARISON_GREATER_THAN | COMPARISON_EQUALS, +}; + +/* + * Resize the array if index is out of range. + * + * idx: the index we want to access + * arr: pointer to the array + * alloc: pointer to the capacity of the array + * default_alloc: initial number of items + * item_size: size of each item + * + */ +int +_PyCompile_EnsureArrayLargeEnough(int idx, void **array, int *alloc, + int default_alloc, size_t item_size) +{ + void *arr = *array; + if (arr == NULL) { + int new_alloc = default_alloc; + if (idx >= new_alloc) { + new_alloc = idx + default_alloc; + } + arr = PyMem_Calloc(new_alloc, item_size); + if (arr == NULL) { + PyErr_NoMemory(); + return ERROR; + } + *alloc = new_alloc; + } + else if (idx >= *alloc) { + size_t oldsize = *alloc * item_size; + int new_alloc = *alloc << 1; + if (idx >= new_alloc) { + new_alloc = idx + default_alloc; + } + size_t newsize = new_alloc * item_size; + + if (oldsize > (SIZE_MAX >> 1)) { + PyErr_NoMemory(); + return ERROR; + } + + assert(newsize > 0); + void *tmp = PyMem_Realloc(arr, newsize); + if (tmp == NULL) { + PyErr_NoMemory(); + return ERROR; + } + *alloc = new_alloc; + arr = tmp; + memset((char *)arr + oldsize, 0, newsize - oldsize); + } + + *array = arr; + return SUCCESS; +} + + +typedef struct { + // A list of strings corresponding to name captures. It is used to track: + // - Repeated name assignments in the same pattern. + // - Different name assignments in alternatives. + // - The order of name assignments in alternatives. + PyObject *stores; + // If 0, any name captures against our subject will raise. + int allow_irrefutable; + // An array of blocks to jump to on failure. Jumping to fail_pop[i] will pop + // i items off of the stack. The end result looks like this (with each block + // falling through to the next): + // fail_pop[4]: POP_TOP + // fail_pop[3]: POP_TOP + // fail_pop[2]: POP_TOP + // fail_pop[1]: POP_TOP + // fail_pop[0]: NOP + jump_target_label *fail_pop; + // The current length of fail_pop. + Py_ssize_t fail_pop_size; + // The number of items on top of the stack that need to *stay* on top of the + // stack. Variable captures go beneath these. All of them will be popped on + // failure. + Py_ssize_t on_top; +} pattern_context; + +static int codegen_nameop(compiler *, location, identifier, expr_context_ty); + +static int codegen_visit_stmt(compiler *, stmt_ty); +static int codegen_visit_keyword(compiler *, keyword_ty); +static int codegen_visit_expr(compiler *, expr_ty); +static int codegen_augassign(compiler *, stmt_ty); +static int codegen_annassign(compiler *, stmt_ty); +static int codegen_subscript(compiler *, expr_ty); +static int codegen_slice_two_parts(compiler *, expr_ty); +static int codegen_slice(compiler *, expr_ty); + +static bool are_all_items_const(asdl_expr_seq *, Py_ssize_t, Py_ssize_t); + + +static int codegen_with(compiler *, stmt_ty, int); +static int codegen_async_with(compiler *, stmt_ty, int); +static int codegen_async_for(compiler *, stmt_ty); +static int codegen_call_simple_kw_helper(compiler *c, + location loc, + asdl_keyword_seq *keywords, + Py_ssize_t nkwelts); +static int codegen_call_helper_impl(compiler *c, location loc, + int n, /* Args already pushed */ + asdl_expr_seq *args, + PyObject *injected_arg, + asdl_keyword_seq *keywords); +static int codegen_call_helper(compiler *c, location loc, + int n, asdl_expr_seq *args, + asdl_keyword_seq *keywords); +static int codegen_try_except(compiler *, stmt_ty); +static int codegen_try_star_except(compiler *, stmt_ty); + +static int codegen_sync_comprehension_generator( + compiler *c, location loc, + asdl_comprehension_seq *generators, int gen_index, + int depth, + expr_ty elt, expr_ty val, int type, + int iter_on_stack); + +static int codegen_async_comprehension_generator( + compiler *c, location loc, + asdl_comprehension_seq *generators, int gen_index, + int depth, + expr_ty elt, expr_ty val, int type, + int iter_on_stack); + +static int codegen_pattern(compiler *, pattern_ty, pattern_context *); +static int codegen_match(compiler *, stmt_ty); +static int codegen_pattern_subpattern(compiler *, + pattern_ty, pattern_context *); +static int codegen_make_closure(compiler *c, location loc, + PyCodeObject *co, Py_ssize_t flags); + + +/* Add an opcode with an integer argument */ +static int +codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc) +{ + /* oparg value is unsigned, but a signed C int is usually used to store + it in the C code (like Python/ceval.c). + + Limit to 32-bit signed C int (rather than INT_MAX) for portability. + + The argument of a concrete bytecode instruction is limited to 8-bit. + EXTENDED_ARG is used for 16, 24, and 32-bit arguments. */ + + int oparg_ = Py_SAFE_DOWNCAST(oparg, Py_ssize_t, int); + assert(!IS_ASSEMBLER_OPCODE(opcode)); + return _PyInstructionSequence_Addop(seq, opcode, oparg_, loc); +} + +#define ADDOP_I(C, LOC, OP, O) \ + RETURN_IF_ERROR(codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))) + +#define ADDOP_I_IN_SCOPE(C, LOC, OP, O) \ + RETURN_IF_ERROR_IN_SCOPE(C, codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))) + +static int +codegen_addop_noarg(instr_sequence *seq, int opcode, location loc) +{ + assert(!OPCODE_HAS_ARG(opcode)); + assert(!IS_ASSEMBLER_OPCODE(opcode)); + return _PyInstructionSequence_Addop(seq, opcode, 0, loc); +} + +#define ADDOP(C, LOC, OP) \ + RETURN_IF_ERROR(codegen_addop_noarg(INSTR_SEQUENCE(C), (OP), (LOC))) + +#define ADDOP_IN_SCOPE(C, LOC, OP) \ + RETURN_IF_ERROR_IN_SCOPE((C), codegen_addop_noarg(INSTR_SEQUENCE(C), (OP), (LOC))) + +static int +codegen_addop_load_const(compiler *c, location loc, PyObject *o) +{ + if (PyLong_CheckExact(o)) { + int overflow; + long val = PyLong_AsLongAndOverflow(o, &overflow); + if (!overflow && val >= 0 && val < 256 && val < _PY_NSMALLPOSINTS) { + ADDOP_I(c, loc, LOAD_SMALL_INT, val); + return SUCCESS; + } + } + Py_ssize_t arg = _PyCompile_AddConst(c, o); + if (arg < 0) { + return ERROR; + } + ADDOP_I(c, loc, LOAD_CONST, arg); + return SUCCESS; +} + +#define ADDOP_LOAD_CONST(C, LOC, O) \ + RETURN_IF_ERROR(codegen_addop_load_const((C), (LOC), (O))) + +#define ADDOP_LOAD_CONST_IN_SCOPE(C, LOC, O) \ + RETURN_IF_ERROR_IN_SCOPE((C), codegen_addop_load_const((C), (LOC), (O))) + +/* Same as ADDOP_LOAD_CONST, but steals a reference. */ +#define ADDOP_LOAD_CONST_NEW(C, LOC, O) \ + do { \ + PyObject *__new_const = (O); \ + if (__new_const == NULL) { \ + return ERROR; \ + } \ + if (codegen_addop_load_const((C), (LOC), __new_const) < 0) { \ + Py_DECREF(__new_const); \ + return ERROR; \ + } \ + Py_DECREF(__new_const); \ + } while (0) + +static int +codegen_addop_o(compiler *c, location loc, + int opcode, PyObject *dict, PyObject *o) +{ + Py_ssize_t arg = _PyCompile_DictAddObj(dict, o); + RETURN_IF_ERROR(arg); + ADDOP_I(c, loc, opcode, arg); + return SUCCESS; +} + +#define ADDOP_N(C, LOC, OP, O, TYPE) \ + do { \ + assert(!OPCODE_HAS_CONST(OP)); /* use ADDOP_LOAD_CONST_NEW */ \ + int ret = codegen_addop_o((C), (LOC), (OP), \ + METADATA(C)->u_ ## TYPE, (O)); \ + Py_DECREF((O)); \ + RETURN_IF_ERROR(ret); \ + } while (0) + +#define ADDOP_N_IN_SCOPE(C, LOC, OP, O, TYPE) \ + do { \ + assert(!OPCODE_HAS_CONST(OP)); /* use ADDOP_LOAD_CONST_NEW */ \ + int ret = codegen_addop_o((C), (LOC), (OP), \ + METADATA(C)->u_ ## TYPE, (O)); \ + Py_DECREF((O)); \ + RETURN_IF_ERROR_IN_SCOPE((C), ret); \ + } while (0) + +#define LOAD_METHOD -1 +#define LOAD_SUPER_METHOD -2 +#define LOAD_ZERO_SUPER_ATTR -3 +#define LOAD_ZERO_SUPER_METHOD -4 + +static int +codegen_addop_name(compiler *c, location loc, + int opcode, PyObject *dict, PyObject *o) +{ + PyObject *mangled = _PyCompile_MaybeMangle(c, o); + if (!mangled) { + return ERROR; + } + Py_ssize_t arg = _PyCompile_DictAddObj(dict, mangled); + Py_DECREF(mangled); + if (arg < 0) { + return ERROR; + } + if (opcode == LOAD_ATTR) { + arg <<= 1; + } + if (opcode == LOAD_METHOD) { + opcode = LOAD_ATTR; + arg <<= 1; + arg |= 1; + } + if (opcode == LOAD_SUPER_ATTR) { + arg <<= 2; + arg |= 2; + } + if (opcode == LOAD_SUPER_METHOD) { + opcode = LOAD_SUPER_ATTR; + arg <<= 2; + arg |= 3; + } + if (opcode == LOAD_ZERO_SUPER_ATTR) { + opcode = LOAD_SUPER_ATTR; + arg <<= 2; + } + if (opcode == LOAD_ZERO_SUPER_METHOD) { + opcode = LOAD_SUPER_ATTR; + arg <<= 2; + arg |= 1; + } + ADDOP_I(c, loc, opcode, arg); + return SUCCESS; +} + +#define ADDOP_NAME(C, LOC, OP, O, TYPE) \ + RETURN_IF_ERROR(codegen_addop_name((C), (LOC), (OP), METADATA(C)->u_ ## TYPE, (O))) + +static int +codegen_addop_j(instr_sequence *seq, location loc, + int opcode, jump_target_label target) +{ + assert(IS_JUMP_TARGET_LABEL(target)); + assert(OPCODE_HAS_JUMP(opcode) || IS_BLOCK_PUSH_OPCODE(opcode)); + assert(!IS_ASSEMBLER_OPCODE(opcode)); + return _PyInstructionSequence_Addop(seq, opcode, target.id, loc); +} + +#define ADDOP_JUMP(C, LOC, OP, O) \ + RETURN_IF_ERROR(codegen_addop_j(INSTR_SEQUENCE(C), (LOC), (OP), (O))) + +#define ADDOP_COMPARE(C, LOC, CMP) \ + RETURN_IF_ERROR(codegen_addcompare((C), (LOC), (cmpop_ty)(CMP))) + +#define ADDOP_BINARY(C, LOC, BINOP) \ + RETURN_IF_ERROR(addop_binary((C), (LOC), (BINOP), false)) + +#define ADDOP_INPLACE(C, LOC, BINOP) \ + RETURN_IF_ERROR(addop_binary((C), (LOC), (BINOP), true)) + +#define ADD_YIELD_FROM(C, LOC, await) \ + RETURN_IF_ERROR(codegen_add_yield_from((C), (LOC), (await))) + +#define POP_EXCEPT_AND_RERAISE(C, LOC) \ + RETURN_IF_ERROR(codegen_pop_except_and_reraise((C), (LOC))) + +#define ADDOP_YIELD(C, LOC) \ + RETURN_IF_ERROR(codegen_addop_yield((C), (LOC))) + +/* VISIT and VISIT_SEQ takes an ASDL type as their second argument. They use + the ASDL name to synthesize the name of the C type and the visit function. +*/ + +#define VISIT(C, TYPE, V) \ + RETURN_IF_ERROR(codegen_visit_ ## TYPE((C), (V))) + +#define VISIT_IN_SCOPE(C, TYPE, V) \ + RETURN_IF_ERROR_IN_SCOPE((C), codegen_visit_ ## TYPE((C), (V))) + +#define VISIT_SEQ(C, TYPE, SEQ) \ + do { \ + asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ + for (int _i = 0; _i < asdl_seq_LEN(seq); _i++) { \ + TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i); \ + RETURN_IF_ERROR(codegen_visit_ ## TYPE((C), elt)); \ + } \ + } while (0) + +#define VISIT_SEQ_IN_SCOPE(C, TYPE, SEQ) \ + do { \ + asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ + for (int _i = 0; _i < asdl_seq_LEN(seq); _i++) { \ + TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i); \ + if (codegen_visit_ ## TYPE((C), elt) < 0) { \ + _PyCompile_ExitScope(C); \ + return ERROR; \ + } \ + } \ + } while (0) + +static int +codegen_call_exit_with_nones(compiler *c, location loc) +{ + ADDOP_LOAD_CONST(c, loc, Py_None); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADDOP_I(c, loc, CALL, 3); + return SUCCESS; +} + +static int +codegen_add_yield_from(compiler *c, location loc, int await) +{ + NEW_JUMP_TARGET_LABEL(c, send); + NEW_JUMP_TARGET_LABEL(c, fail); + NEW_JUMP_TARGET_LABEL(c, exit); + + USE_LABEL(c, send); + ADDOP_JUMP(c, loc, SEND, exit); + // Set up a virtual try/except to handle when StopIteration is raised during + // a close or throw call. The only way YIELD_VALUE raises if they do! + ADDOP_JUMP(c, loc, SETUP_FINALLY, fail); + ADDOP_I(c, loc, YIELD_VALUE, 1); + ADDOP(c, NO_LOCATION, POP_BLOCK); + ADDOP_I(c, loc, RESUME, await ? RESUME_AFTER_AWAIT : RESUME_AFTER_YIELD_FROM); + ADDOP_JUMP(c, loc, JUMP_NO_INTERRUPT, send); + + USE_LABEL(c, fail); + ADDOP(c, loc, CLEANUP_THROW); + + USE_LABEL(c, exit); + ADDOP(c, loc, END_SEND); + return SUCCESS; +} + +static int +codegen_pop_except_and_reraise(compiler *c, location loc) +{ + /* Stack contents + * [exc_info, lasti, exc] COPY 3 + * [exc_info, lasti, exc, exc_info] POP_EXCEPT + * [exc_info, lasti, exc] RERAISE 1 + * (exception_unwind clears the stack) + */ + + ADDOP_I(c, loc, COPY, 3); + ADDOP(c, loc, POP_EXCEPT); + ADDOP_I(c, loc, RERAISE, 1); + return SUCCESS; +} + +/* Unwind a frame block. If preserve_tos is true, the TOS before + * popping the blocks will be restored afterwards, unless another + * return, break or continue is found. In which case, the TOS will + * be popped. + */ +static int +codegen_unwind_fblock(compiler *c, location *ploc, + fblockinfo *info, int preserve_tos) +{ + switch (info->fb_type) { + case COMPILE_FBLOCK_WHILE_LOOP: + case COMPILE_FBLOCK_EXCEPTION_HANDLER: + case COMPILE_FBLOCK_EXCEPTION_GROUP_HANDLER: + case COMPILE_FBLOCK_ASYNC_COMPREHENSION_GENERATOR: + case COMPILE_FBLOCK_STOP_ITERATION: + return SUCCESS; + + case COMPILE_FBLOCK_FOR_LOOP: + /* Pop the iterator */ + if (preserve_tos) { + ADDOP_I(c, *ploc, SWAP, 2); + } + ADDOP(c, *ploc, POP_TOP); + return SUCCESS; + + case COMPILE_FBLOCK_TRY_EXCEPT: + ADDOP(c, *ploc, POP_BLOCK); + return SUCCESS; + + case COMPILE_FBLOCK_FINALLY_TRY: + /* This POP_BLOCK gets the line number of the unwinding statement */ + ADDOP(c, *ploc, POP_BLOCK); + if (preserve_tos) { + RETURN_IF_ERROR( + _PyCompile_PushFBlock(c, *ploc, COMPILE_FBLOCK_POP_VALUE, + NO_LABEL, NO_LABEL, NULL)); + } + /* Emit the finally block */ + VISIT_SEQ(c, stmt, info->fb_datum); + if (preserve_tos) { + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_POP_VALUE, NO_LABEL); + } + /* The finally block should appear to execute after the + * statement causing the unwinding, so make the unwinding + * instruction artificial */ + *ploc = NO_LOCATION; + return SUCCESS; + + case COMPILE_FBLOCK_FINALLY_END: + if (preserve_tos) { + ADDOP_I(c, *ploc, SWAP, 2); + } + ADDOP(c, *ploc, POP_TOP); /* exc_value */ + if (preserve_tos) { + ADDOP_I(c, *ploc, SWAP, 2); + } + ADDOP(c, *ploc, POP_BLOCK); + ADDOP(c, *ploc, POP_EXCEPT); + return SUCCESS; + + case COMPILE_FBLOCK_WITH: + case COMPILE_FBLOCK_ASYNC_WITH: + *ploc = info->fb_loc; + ADDOP(c, *ploc, POP_BLOCK); + if (preserve_tos) { + ADDOP_I(c, *ploc, SWAP, 3); + ADDOP_I(c, *ploc, SWAP, 2); + } + RETURN_IF_ERROR(codegen_call_exit_with_nones(c, *ploc)); + if (info->fb_type == COMPILE_FBLOCK_ASYNC_WITH) { + ADDOP_I(c, *ploc, GET_AWAITABLE, 2); + ADDOP_LOAD_CONST(c, *ploc, Py_None); + ADD_YIELD_FROM(c, *ploc, 1); + } + ADDOP(c, *ploc, POP_TOP); + /* The exit block should appear to execute after the + * statement causing the unwinding, so make the unwinding + * instruction artificial */ + *ploc = NO_LOCATION; + return SUCCESS; + + case COMPILE_FBLOCK_HANDLER_CLEANUP: { + if (info->fb_datum) { + ADDOP(c, *ploc, POP_BLOCK); + } + if (preserve_tos) { + ADDOP_I(c, *ploc, SWAP, 2); + } + ADDOP(c, *ploc, POP_BLOCK); + ADDOP(c, *ploc, POP_EXCEPT); + if (info->fb_datum) { + ADDOP_LOAD_CONST(c, *ploc, Py_None); + RETURN_IF_ERROR(codegen_nameop(c, *ploc, info->fb_datum, Store)); + RETURN_IF_ERROR(codegen_nameop(c, *ploc, info->fb_datum, Del)); + } + return SUCCESS; + } + case COMPILE_FBLOCK_POP_VALUE: { + if (preserve_tos) { + ADDOP_I(c, *ploc, SWAP, 2); + } + ADDOP(c, *ploc, POP_TOP); + return SUCCESS; + } + } + Py_UNREACHABLE(); +} + +/** Unwind block stack. If loop is not NULL, then stop when the first loop is encountered. */ +static int +codegen_unwind_fblock_stack(compiler *c, location *ploc, + int preserve_tos, fblockinfo **loop) +{ + fblockinfo *top = _PyCompile_TopFBlock(c); + if (top == NULL) { + return SUCCESS; + } + if (top->fb_type == COMPILE_FBLOCK_EXCEPTION_GROUP_HANDLER) { + return _PyCompile_Error( + c, *ploc, "'break', 'continue' and 'return' cannot appear in an except* block"); + } + if (loop != NULL && (top->fb_type == COMPILE_FBLOCK_WHILE_LOOP || + top->fb_type == COMPILE_FBLOCK_FOR_LOOP)) { + *loop = top; + return SUCCESS; + } + fblockinfo copy = *top; + _PyCompile_PopFBlock(c, top->fb_type, top->fb_block); + RETURN_IF_ERROR(codegen_unwind_fblock(c, ploc, ©, preserve_tos)); + RETURN_IF_ERROR(codegen_unwind_fblock_stack(c, ploc, preserve_tos, loop)); + _PyCompile_PushFBlock(c, copy.fb_loc, copy.fb_type, copy.fb_block, + copy.fb_exit, copy.fb_datum); + return SUCCESS; +} + +static int +codegen_enter_scope(compiler *c, identifier name, int scope_type, + void *key, int lineno, PyObject *private, + _PyCompile_CodeUnitMetadata *umd) +{ + RETURN_IF_ERROR( + _PyCompile_EnterScope(c, name, scope_type, key, lineno, private, umd)); + location loc = LOCATION(lineno, lineno, 0, 0); + if (scope_type == COMPILE_SCOPE_MODULE) { + loc.lineno = 0; + } + ADDOP_I(c, loc, RESUME, RESUME_AT_FUNC_START); + return SUCCESS; +} + +static int +codegen_setup_annotations_scope(compiler *c, location loc, + void *key, PyObject *name) +{ + _PyCompile_CodeUnitMetadata umd = { + .u_posonlyargcount = 1, + }; + RETURN_IF_ERROR( + codegen_enter_scope(c, name, COMPILE_SCOPE_ANNOTATIONS, + key, loc.lineno, NULL, &umd)); + + // if .format > VALUE_WITH_FAKE_GLOBALS: raise NotImplementedError + PyObject *value_with_fake_globals = PyLong_FromLong(_Py_ANNOTATE_FORMAT_VALUE_WITH_FAKE_GLOBALS); + assert(!SYMTABLE_ENTRY(c)->ste_has_docstring); + _Py_DECLARE_STR(format, ".format"); + ADDOP_I(c, loc, LOAD_FAST, 0); + ADDOP_LOAD_CONST(c, loc, value_with_fake_globals); + ADDOP_I(c, loc, COMPARE_OP, (Py_GT << 5) | compare_masks[Py_GT]); + NEW_JUMP_TARGET_LABEL(c, body); + ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, body); + ADDOP_I(c, loc, LOAD_COMMON_CONSTANT, CONSTANT_NOTIMPLEMENTEDERROR); + ADDOP_I(c, loc, RAISE_VARARGS, 1); + USE_LABEL(c, body); + return SUCCESS; +} + +static int +codegen_leave_annotations_scope(compiler *c, location loc, + Py_ssize_t annotations_len) +{ + ADDOP_I(c, loc, BUILD_MAP, annotations_len); + ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); + PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 1); + + // We want the parameter to __annotate__ to be named "format" in the + // signature shown by inspect.signature(), but we need to use a + // different name (.format) in the symtable; if the name + // "format" appears in the annotations, it doesn't get clobbered + // by this name. This code is essentially: + // co->co_localsplusnames = ("format", *co->co_localsplusnames[1:]) + const Py_ssize_t size = PyObject_Size(co->co_localsplusnames); + if (size == -1) { + return ERROR; + } + PyObject *new_names = PyTuple_New(size); + if (new_names == NULL) { + return ERROR; + } + PyTuple_SET_ITEM(new_names, 0, Py_NewRef(&_Py_ID(format))); + for (int i = 1; i < size; i++) { + PyObject *item = PyTuple_GetItem(co->co_localsplusnames, i); + if (item == NULL) { + Py_DECREF(new_names); + return ERROR; + } + Py_INCREF(item); + PyTuple_SET_ITEM(new_names, i, item); + } + Py_SETREF(co->co_localsplusnames, new_names); + + _PyCompile_ExitScope(c); + if (co == NULL) { + return ERROR; + } + int ret = codegen_make_closure(c, loc, co, 0); + Py_DECREF(co); + RETURN_IF_ERROR(ret); + return SUCCESS; +} + +static int +codegen_process_deferred_annotations(compiler *c, location loc) +{ + PyObject *deferred_anno = _PyCompile_DeferredAnnotations(c); + if (deferred_anno == NULL) { + return SUCCESS; + } + Py_INCREF(deferred_anno); + + // It's possible that ste_annotations_block is set but + // u_deferred_annotations is not, because the former is still + // set if there are only non-simple annotations (i.e., annotations + // for attributes, subscripts, or parenthesized names). However, the + // reverse should not be possible. + PySTEntryObject *ste = SYMTABLE_ENTRY(c); + assert(ste->ste_annotation_block != NULL); + void *key = (void *)((uintptr_t)ste->ste_id + 1); + if (codegen_setup_annotations_scope(c, loc, key, + ste->ste_annotation_block->ste_name) < 0) { + Py_DECREF(deferred_anno); + return ERROR; + } + Py_ssize_t annotations_len = PyList_Size(deferred_anno); + for (Py_ssize_t i = 0; i < annotations_len; i++) { + PyObject *ptr = PyList_GET_ITEM(deferred_anno, i); + stmt_ty st = (stmt_ty)PyLong_AsVoidPtr(ptr); + if (st == NULL) { + _PyCompile_ExitScope(c); + Py_DECREF(deferred_anno); + return ERROR; + } + PyObject *mangled = _PyCompile_Mangle(c, st->v.AnnAssign.target->v.Name.id); + if (!mangled) { + _PyCompile_ExitScope(c); + Py_DECREF(deferred_anno); + return ERROR; + } + ADDOP_LOAD_CONST_NEW(c, LOC(st), mangled); + VISIT(c, expr, st->v.AnnAssign.annotation); + } + Py_DECREF(deferred_anno); + + RETURN_IF_ERROR(codegen_leave_annotations_scope(c, loc, annotations_len)); + RETURN_IF_ERROR(codegen_nameop(c, loc, &_Py_ID(__annotate__), Store)); + + return SUCCESS; +} + +/* Compile an expression */ +int +_PyCodegen_Expression(compiler *c, expr_ty e) +{ + VISIT(c, expr, e); + return SUCCESS; +} + +/* Compile a sequence of statements, checking for a docstring + and for annotations. */ + +int +_PyCodegen_Body(compiler *c, location loc, asdl_stmt_seq *stmts, bool is_interactive) +{ + /* If from __future__ import annotations is active, + * every annotated class and module should have __annotations__. + * Else __annotate__ is created when necessary. */ + PySTEntryObject *ste = SYMTABLE_ENTRY(c); + if ((FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS) && ste->ste_annotations_used) { + ADDOP(c, loc, SETUP_ANNOTATIONS); + } + if (!asdl_seq_LEN(stmts)) { + return SUCCESS; + } + Py_ssize_t first_instr = 0; + if (!is_interactive) { /* A string literal on REPL prompt is not a docstring */ + if (ste->ste_has_docstring) { + PyObject *docstring = _PyAST_GetDocString(stmts); + assert(docstring); + first_instr = 1; + /* set docstring */ + assert(OPTIMIZATION_LEVEL(c) < 2); + PyObject *cleandoc = _PyCompile_CleanDoc(docstring); + if (cleandoc == NULL) { + return ERROR; + } + stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); + assert(st->kind == Expr_kind); + location loc = LOC(st->v.Expr.value); + ADDOP_LOAD_CONST(c, loc, cleandoc); + Py_DECREF(cleandoc); + RETURN_IF_ERROR(codegen_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)); + } + } + for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(stmts); i++) { + VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i)); + } + // If there are annotations and the future import is not on, we + // collect the annotations in a separate pass and generate an + // __annotate__ function. See PEP 649. + if (!(FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS)) { + RETURN_IF_ERROR(codegen_process_deferred_annotations(c, loc)); + } + return SUCCESS; +} + +int +_PyCodegen_EnterAnonymousScope(compiler* c, mod_ty mod) +{ + _Py_DECLARE_STR(anon_module, ""); + RETURN_IF_ERROR( + codegen_enter_scope(c, &_Py_STR(anon_module), COMPILE_SCOPE_MODULE, + mod, 1, NULL, NULL)); + return SUCCESS; +} + +static int +codegen_make_closure(compiler *c, location loc, + PyCodeObject *co, Py_ssize_t flags) +{ + if (co->co_nfreevars) { + int i = PyUnstable_Code_GetFirstFree(co); + for (; i < co->co_nlocalsplus; ++i) { + /* Bypass com_addop_varname because it will generate + LOAD_DEREF but LOAD_CLOSURE is needed. + */ + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + int arg = _PyCompile_LookupArg(c, co, name); + RETURN_IF_ERROR(arg); + ADDOP_I(c, loc, LOAD_CLOSURE, arg); + } + flags |= MAKE_FUNCTION_CLOSURE; + ADDOP_I(c, loc, BUILD_TUPLE, co->co_nfreevars); + } + ADDOP_LOAD_CONST(c, loc, (PyObject*)co); + + ADDOP(c, loc, MAKE_FUNCTION); + + if (flags & MAKE_FUNCTION_CLOSURE) { + ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_CLOSURE); + } + if (flags & MAKE_FUNCTION_ANNOTATIONS) { + ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATIONS); + } + if (flags & MAKE_FUNCTION_ANNOTATE) { + ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATE); + } + if (flags & MAKE_FUNCTION_KWDEFAULTS) { + ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_KWDEFAULTS); + } + if (flags & MAKE_FUNCTION_DEFAULTS) { + ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_DEFAULTS); + } + return SUCCESS; +} + +static int +codegen_decorators(compiler *c, asdl_expr_seq* decos) +{ + if (!decos) { + return SUCCESS; + } + + for (Py_ssize_t i = 0; i < asdl_seq_LEN(decos); i++) { + VISIT(c, expr, (expr_ty)asdl_seq_GET(decos, i)); + } + return SUCCESS; +} + +static int +codegen_apply_decorators(compiler *c, asdl_expr_seq* decos) +{ + if (!decos) { + return SUCCESS; + } + + for (Py_ssize_t i = asdl_seq_LEN(decos) - 1; i > -1; i--) { + location loc = LOC((expr_ty)asdl_seq_GET(decos, i)); + ADDOP_I(c, loc, CALL, 0); + } + return SUCCESS; +} + +static int +codegen_kwonlydefaults(compiler *c, location loc, + asdl_arg_seq *kwonlyargs, asdl_expr_seq *kw_defaults) +{ + /* Push a dict of keyword-only default values. + + Return -1 on error, 0 if no dict pushed, 1 if a dict is pushed. + */ + int default_count = 0; + for (int i = 0; i < asdl_seq_LEN(kwonlyargs); i++) { + arg_ty arg = asdl_seq_GET(kwonlyargs, i); + expr_ty default_ = asdl_seq_GET(kw_defaults, i); + if (default_) { + default_count++; + PyObject *mangled = _PyCompile_MaybeMangle(c, arg->arg); + if (!mangled) { + return ERROR; + } + ADDOP_LOAD_CONST_NEW(c, loc, mangled); + VISIT(c, expr, default_); + } + } + if (default_count) { + ADDOP_I(c, loc, BUILD_MAP, default_count); + return 1; + } + else { + return 0; + } +} + +static int +codegen_visit_annexpr(compiler *c, expr_ty annotation) +{ + location loc = LOC(annotation); + ADDOP_LOAD_CONST_NEW(c, loc, _PyAST_ExprAsUnicode(annotation)); + return SUCCESS; +} + +static int +codegen_argannotation(compiler *c, identifier id, + expr_ty annotation, Py_ssize_t *annotations_len, location loc) +{ + if (!annotation) { + return SUCCESS; + } + PyObject *mangled = _PyCompile_MaybeMangle(c, id); + if (!mangled) { + return ERROR; + } + ADDOP_LOAD_CONST(c, loc, mangled); + Py_DECREF(mangled); + + if (FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS) { + VISIT(c, annexpr, annotation); + } + else { + if (annotation->kind == Starred_kind) { + // *args: *Ts (where Ts is a TypeVarTuple). + // Do [annotation_value] = [*Ts]. + // (Note that in theory we could end up here even for an argument + // other than *args, but in practice the grammar doesn't allow it.) + VISIT(c, expr, annotation->v.Starred.value); + ADDOP_I(c, loc, UNPACK_SEQUENCE, (Py_ssize_t) 1); + } + else { + VISIT(c, expr, annotation); + } + } + *annotations_len += 1; + return SUCCESS; +} + +static int +codegen_argannotations(compiler *c, asdl_arg_seq* args, + Py_ssize_t *annotations_len, location loc) +{ + int i; + for (i = 0; i < asdl_seq_LEN(args); i++) { + arg_ty arg = (arg_ty)asdl_seq_GET(args, i); + RETURN_IF_ERROR( + codegen_argannotation( + c, + arg->arg, + arg->annotation, + annotations_len, + loc)); + } + return SUCCESS; +} + +static int +codegen_annotations_in_scope(compiler *c, location loc, + arguments_ty args, expr_ty returns, + Py_ssize_t *annotations_len) +{ + RETURN_IF_ERROR( + codegen_argannotations(c, args->args, annotations_len, loc)); + + RETURN_IF_ERROR( + codegen_argannotations(c, args->posonlyargs, annotations_len, loc)); + + if (args->vararg && args->vararg->annotation) { + RETURN_IF_ERROR( + codegen_argannotation(c, args->vararg->arg, + args->vararg->annotation, annotations_len, loc)); + } + + RETURN_IF_ERROR( + codegen_argannotations(c, args->kwonlyargs, annotations_len, loc)); + + if (args->kwarg && args->kwarg->annotation) { + RETURN_IF_ERROR( + codegen_argannotation(c, args->kwarg->arg, + args->kwarg->annotation, annotations_len, loc)); + } + + RETURN_IF_ERROR( + codegen_argannotation(c, &_Py_ID(return), returns, annotations_len, loc)); + + return 0; +} + +static int +codegen_annotations(compiler *c, location loc, + arguments_ty args, expr_ty returns) +{ + /* Push arg annotation names and values. + The expressions are evaluated separately from the rest of the source code. + + Return -1 on error, or a combination of flags to add to the function. + */ + Py_ssize_t annotations_len = 0; + + PySTEntryObject *ste; + RETURN_IF_ERROR(_PySymtable_LookupOptional(SYMTABLE(c), args, &ste)); + assert(ste != NULL); + + if (ste->ste_annotations_used) { + int err = codegen_setup_annotations_scope(c, loc, (void *)args, ste->ste_name); + Py_DECREF(ste); + RETURN_IF_ERROR(err); + RETURN_IF_ERROR_IN_SCOPE( + c, codegen_annotations_in_scope(c, loc, args, returns, &annotations_len) + ); + RETURN_IF_ERROR( + codegen_leave_annotations_scope(c, loc, annotations_len) + ); + return MAKE_FUNCTION_ANNOTATE; + } + else { + Py_DECREF(ste); + } + + return 0; +} + +static int +codegen_defaults(compiler *c, arguments_ty args, + location loc) +{ + VISIT_SEQ(c, expr, args->defaults); + ADDOP_I(c, loc, BUILD_TUPLE, asdl_seq_LEN(args->defaults)); + return SUCCESS; +} + +static Py_ssize_t +codegen_default_arguments(compiler *c, location loc, + arguments_ty args) +{ + Py_ssize_t funcflags = 0; + if (args->defaults && asdl_seq_LEN(args->defaults) > 0) { + RETURN_IF_ERROR(codegen_defaults(c, args, loc)); + funcflags |= MAKE_FUNCTION_DEFAULTS; + } + if (args->kwonlyargs) { + int res = codegen_kwonlydefaults(c, loc, + args->kwonlyargs, + args->kw_defaults); + RETURN_IF_ERROR(res); + if (res > 0) { + funcflags |= MAKE_FUNCTION_KWDEFAULTS; + } + } + return funcflags; +} + +static int +codegen_wrap_in_stopiteration_handler(compiler *c) +{ + NEW_JUMP_TARGET_LABEL(c, handler); + + /* Insert SETUP_CLEANUP at start */ + RETURN_IF_ERROR( + _PyInstructionSequence_InsertInstruction( + INSTR_SEQUENCE(c), 0, + SETUP_CLEANUP, handler.id, NO_LOCATION)); + + ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); + ADDOP(c, NO_LOCATION, RETURN_VALUE); + USE_LABEL(c, handler); + ADDOP_I(c, NO_LOCATION, CALL_INTRINSIC_1, INTRINSIC_STOPITERATION_ERROR); + ADDOP_I(c, NO_LOCATION, RERAISE, 1); + return SUCCESS; +} + +static int +codegen_type_param_bound_or_default(compiler *c, expr_ty e, + identifier name, void *key, + bool allow_starred) +{ + PyObject *defaults = PyTuple_Pack(1, _PyLong_GetOne()); + ADDOP_LOAD_CONST_NEW(c, LOC(e), defaults); + RETURN_IF_ERROR(codegen_setup_annotations_scope(c, LOC(e), key, name)); + if (allow_starred && e->kind == Starred_kind) { + VISIT(c, expr, e->v.Starred.value); + ADDOP_I(c, LOC(e), UNPACK_SEQUENCE, (Py_ssize_t)1); + } + else { + VISIT(c, expr, e); + } + ADDOP_IN_SCOPE(c, LOC(e), RETURN_VALUE); + PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 1); + _PyCompile_ExitScope(c); + if (co == NULL) { + return ERROR; + } + int ret = codegen_make_closure(c, LOC(e), co, MAKE_FUNCTION_DEFAULTS); + Py_DECREF(co); + RETURN_IF_ERROR(ret); + return SUCCESS; +} + +static int +codegen_type_params(compiler *c, asdl_type_param_seq *type_params) +{ + if (!type_params) { + return SUCCESS; + } + Py_ssize_t n = asdl_seq_LEN(type_params); + bool seen_default = false; + + for (Py_ssize_t i = 0; i < n; i++) { + type_param_ty typeparam = asdl_seq_GET(type_params, i); + location loc = LOC(typeparam); + switch(typeparam->kind) { + case TypeVar_kind: + ADDOP_LOAD_CONST(c, loc, typeparam->v.TypeVar.name); + if (typeparam->v.TypeVar.bound) { + expr_ty bound = typeparam->v.TypeVar.bound; + RETURN_IF_ERROR( + codegen_type_param_bound_or_default(c, bound, typeparam->v.TypeVar.name, + (void *)typeparam, false)); + + int intrinsic = bound->kind == Tuple_kind + ? INTRINSIC_TYPEVAR_WITH_CONSTRAINTS + : INTRINSIC_TYPEVAR_WITH_BOUND; + ADDOP_I(c, loc, CALL_INTRINSIC_2, intrinsic); + } + else { + ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_TYPEVAR); + } + if (typeparam->v.TypeVar.default_value) { + seen_default = true; + expr_ty default_ = typeparam->v.TypeVar.default_value; + RETURN_IF_ERROR( + codegen_type_param_bound_or_default(c, default_, typeparam->v.TypeVar.name, + (void *)((uintptr_t)typeparam + 1), false)); + ADDOP_I(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_TYPEPARAM_DEFAULT); + } + else if (seen_default) { + return _PyCompile_Error(c, loc, "non-default type parameter '%U' " + "follows default type parameter", + typeparam->v.TypeVar.name); + } + ADDOP_I(c, loc, COPY, 1); + RETURN_IF_ERROR(codegen_nameop(c, loc, typeparam->v.TypeVar.name, Store)); + break; + case TypeVarTuple_kind: + ADDOP_LOAD_CONST(c, loc, typeparam->v.TypeVarTuple.name); + ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_TYPEVARTUPLE); + if (typeparam->v.TypeVarTuple.default_value) { + expr_ty default_ = typeparam->v.TypeVarTuple.default_value; + RETURN_IF_ERROR( + codegen_type_param_bound_or_default(c, default_, typeparam->v.TypeVarTuple.name, + (void *)typeparam, true)); + ADDOP_I(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_TYPEPARAM_DEFAULT); + seen_default = true; + } + else if (seen_default) { + return _PyCompile_Error(c, loc, "non-default type parameter '%U' " + "follows default type parameter", + typeparam->v.TypeVarTuple.name); + } + ADDOP_I(c, loc, COPY, 1); + RETURN_IF_ERROR(codegen_nameop(c, loc, typeparam->v.TypeVarTuple.name, Store)); + break; + case ParamSpec_kind: + ADDOP_LOAD_CONST(c, loc, typeparam->v.ParamSpec.name); + ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_PARAMSPEC); + if (typeparam->v.ParamSpec.default_value) { + expr_ty default_ = typeparam->v.ParamSpec.default_value; + RETURN_IF_ERROR( + codegen_type_param_bound_or_default(c, default_, typeparam->v.ParamSpec.name, + (void *)typeparam, false)); + ADDOP_I(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_TYPEPARAM_DEFAULT); + seen_default = true; + } + else if (seen_default) { + return _PyCompile_Error(c, loc, "non-default type parameter '%U' " + "follows default type parameter", + typeparam->v.ParamSpec.name); + } + ADDOP_I(c, loc, COPY, 1); + RETURN_IF_ERROR(codegen_nameop(c, loc, typeparam->v.ParamSpec.name, Store)); + break; + } + } + ADDOP_I(c, LOC(asdl_seq_GET(type_params, 0)), BUILD_TUPLE, n); + return SUCCESS; +} + +static int +codegen_function_body(compiler *c, stmt_ty s, int is_async, Py_ssize_t funcflags, + int firstlineno) +{ + arguments_ty args; + identifier name; + asdl_stmt_seq *body; + int scope_type; + + if (is_async) { + assert(s->kind == AsyncFunctionDef_kind); + + args = s->v.AsyncFunctionDef.args; + name = s->v.AsyncFunctionDef.name; + body = s->v.AsyncFunctionDef.body; + + scope_type = COMPILE_SCOPE_ASYNC_FUNCTION; + } else { + assert(s->kind == FunctionDef_kind); + + args = s->v.FunctionDef.args; + name = s->v.FunctionDef.name; + body = s->v.FunctionDef.body; + + scope_type = COMPILE_SCOPE_FUNCTION; + } + + _PyCompile_CodeUnitMetadata umd = { + .u_argcount = asdl_seq_LEN(args->args), + .u_posonlyargcount = asdl_seq_LEN(args->posonlyargs), + .u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs), + }; + RETURN_IF_ERROR( + codegen_enter_scope(c, name, scope_type, (void *)s, firstlineno, NULL, &umd)); + + PySTEntryObject *ste = SYMTABLE_ENTRY(c); + Py_ssize_t first_instr = 0; + if (ste->ste_has_docstring) { + PyObject *docstring = _PyAST_GetDocString(body); + assert(docstring); + first_instr = 1; + docstring = _PyCompile_CleanDoc(docstring); + if (docstring == NULL) { + _PyCompile_ExitScope(c); + return ERROR; + } + Py_ssize_t idx = _PyCompile_AddConst(c, docstring); + Py_DECREF(docstring); + RETURN_IF_ERROR_IN_SCOPE(c, idx < 0 ? ERROR : SUCCESS); + } + + NEW_JUMP_TARGET_LABEL(c, start); + USE_LABEL(c, start); + bool add_stopiteration_handler = ste->ste_coroutine || ste->ste_generator; + if (add_stopiteration_handler) { + /* codegen_wrap_in_stopiteration_handler will push a block, so we need to account for that */ + RETURN_IF_ERROR( + _PyCompile_PushFBlock(c, NO_LOCATION, COMPILE_FBLOCK_STOP_ITERATION, + start, NO_LABEL, NULL)); + } + + for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(body); i++) { + VISIT_IN_SCOPE(c, stmt, (stmt_ty)asdl_seq_GET(body, i)); + } + if (add_stopiteration_handler) { + RETURN_IF_ERROR_IN_SCOPE(c, codegen_wrap_in_stopiteration_handler(c)); + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_STOP_ITERATION, start); + } + PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 1); + _PyCompile_ExitScope(c); + if (co == NULL) { + Py_XDECREF(co); + return ERROR; + } + int ret = codegen_make_closure(c, LOC(s), co, funcflags); + Py_DECREF(co); + return ret; +} + +static int +codegen_function(compiler *c, stmt_ty s, int is_async) +{ + arguments_ty args; + expr_ty returns; + identifier name; + asdl_expr_seq *decos; + asdl_type_param_seq *type_params; + Py_ssize_t funcflags; + int firstlineno; + + if (is_async) { + assert(s->kind == AsyncFunctionDef_kind); + + args = s->v.AsyncFunctionDef.args; + returns = s->v.AsyncFunctionDef.returns; + decos = s->v.AsyncFunctionDef.decorator_list; + name = s->v.AsyncFunctionDef.name; + type_params = s->v.AsyncFunctionDef.type_params; + } else { + assert(s->kind == FunctionDef_kind); + + args = s->v.FunctionDef.args; + returns = s->v.FunctionDef.returns; + decos = s->v.FunctionDef.decorator_list; + name = s->v.FunctionDef.name; + type_params = s->v.FunctionDef.type_params; + } + + RETURN_IF_ERROR(codegen_decorators(c, decos)); + + firstlineno = s->lineno; + if (asdl_seq_LEN(decos)) { + firstlineno = ((expr_ty)asdl_seq_GET(decos, 0))->lineno; + } + + location loc = LOC(s); + + int is_generic = asdl_seq_LEN(type_params) > 0; + + funcflags = codegen_default_arguments(c, loc, args); + RETURN_IF_ERROR(funcflags); + + int num_typeparam_args = 0; + + if (is_generic) { + if (funcflags & MAKE_FUNCTION_DEFAULTS) { + num_typeparam_args += 1; + } + if (funcflags & MAKE_FUNCTION_KWDEFAULTS) { + num_typeparam_args += 1; + } + if (num_typeparam_args == 2) { + ADDOP_I(c, loc, SWAP, 2); + } + PyObject *type_params_name = PyUnicode_FromFormat("", name); + if (!type_params_name) { + return ERROR; + } + _PyCompile_CodeUnitMetadata umd = { + .u_argcount = num_typeparam_args, + }; + int ret = codegen_enter_scope(c, type_params_name, COMPILE_SCOPE_ANNOTATIONS, + (void *)type_params, firstlineno, NULL, &umd); + Py_DECREF(type_params_name); + RETURN_IF_ERROR(ret); + RETURN_IF_ERROR_IN_SCOPE(c, codegen_type_params(c, type_params)); + for (int i = 0; i < num_typeparam_args; i++) { + ADDOP_I_IN_SCOPE(c, loc, LOAD_FAST, i); + } + } + + int annotations_flag = codegen_annotations(c, loc, args, returns); + if (annotations_flag < 0) { + if (is_generic) { + _PyCompile_ExitScope(c); + } + return ERROR; + } + funcflags |= annotations_flag; + + int ret = codegen_function_body(c, s, is_async, funcflags, firstlineno); + if (is_generic) { + RETURN_IF_ERROR_IN_SCOPE(c, ret); + } + else { + RETURN_IF_ERROR(ret); + } + + if (is_generic) { + ADDOP_I_IN_SCOPE(c, loc, SWAP, 2); + ADDOP_I_IN_SCOPE(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_FUNCTION_TYPE_PARAMS); + + PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 0); + _PyCompile_ExitScope(c); + if (co == NULL) { + return ERROR; + } + int ret = codegen_make_closure(c, loc, co, 0); + Py_DECREF(co); + RETURN_IF_ERROR(ret); + if (num_typeparam_args > 0) { + ADDOP_I(c, loc, SWAP, num_typeparam_args + 1); + ADDOP_I(c, loc, CALL, num_typeparam_args - 1); + } + else { + ADDOP(c, loc, PUSH_NULL); + ADDOP_I(c, loc, CALL, 0); + } + } + + RETURN_IF_ERROR(codegen_apply_decorators(c, decos)); + return codegen_nameop(c, loc, name, Store); +} + +static int +codegen_set_type_params_in_class(compiler *c, location loc) +{ + _Py_DECLARE_STR(type_params, ".type_params"); + RETURN_IF_ERROR(codegen_nameop(c, loc, &_Py_STR(type_params), Load)); + RETURN_IF_ERROR(codegen_nameop(c, loc, &_Py_ID(__type_params__), Store)); + return SUCCESS; +} + + +static int +codegen_class_body(compiler *c, stmt_ty s, int firstlineno) +{ + /* ultimately generate code for: + = __build_class__(, , *, **) + where: + is a zero arg function/closure created from the class body. + It mutates its locals to build the class namespace. + is the class name + is the positional arguments and *varargs argument + is the keyword arguments and **kwds argument + This borrows from codegen_call. + */ + + /* 1. compile the class body into a code object */ + RETURN_IF_ERROR( + codegen_enter_scope(c, s->v.ClassDef.name, COMPILE_SCOPE_CLASS, + (void *)s, firstlineno, s->v.ClassDef.name, NULL)); + + location loc = LOCATION(firstlineno, firstlineno, 0, 0); + /* load (global) __name__ ... */ + RETURN_IF_ERROR_IN_SCOPE(c, codegen_nameop(c, loc, &_Py_ID(__name__), Load)); + /* ... and store it as __module__ */ + RETURN_IF_ERROR_IN_SCOPE(c, codegen_nameop(c, loc, &_Py_ID(__module__), Store)); + ADDOP_LOAD_CONST(c, loc, QUALNAME(c)); + RETURN_IF_ERROR_IN_SCOPE(c, codegen_nameop(c, loc, &_Py_ID(__qualname__), Store)); + ADDOP_LOAD_CONST_NEW(c, loc, PyLong_FromLong(METADATA(c)->u_firstlineno)); + RETURN_IF_ERROR_IN_SCOPE(c, codegen_nameop(c, loc, &_Py_ID(__firstlineno__), Store)); + asdl_type_param_seq *type_params = s->v.ClassDef.type_params; + if (asdl_seq_LEN(type_params) > 0) { + RETURN_IF_ERROR_IN_SCOPE(c, codegen_set_type_params_in_class(c, loc)); + } + if (SYMTABLE_ENTRY(c)->ste_needs_classdict) { + ADDOP(c, loc, LOAD_LOCALS); + + // We can't use codegen_nameop here because we need to generate a + // STORE_DEREF in a class namespace, and codegen_nameop() won't do + // that by default. + ADDOP_N_IN_SCOPE(c, loc, STORE_DEREF, &_Py_ID(__classdict__), cellvars); + } + /* compile the body proper */ + RETURN_IF_ERROR_IN_SCOPE(c, _PyCodegen_Body(c, loc, s->v.ClassDef.body, false)); + PyObject *static_attributes = _PyCompile_StaticAttributesAsTuple(c); + if (static_attributes == NULL) { + _PyCompile_ExitScope(c); + return ERROR; + } + ADDOP_LOAD_CONST(c, NO_LOCATION, static_attributes); + Py_CLEAR(static_attributes); + RETURN_IF_ERROR_IN_SCOPE( + c, codegen_nameop(c, NO_LOCATION, &_Py_ID(__static_attributes__), Store)); + /* The following code is artificial */ + /* Set __classdictcell__ if necessary */ + if (SYMTABLE_ENTRY(c)->ste_needs_classdict) { + /* Store __classdictcell__ into class namespace */ + int i = _PyCompile_LookupCellvar(c, &_Py_ID(__classdict__)); + RETURN_IF_ERROR_IN_SCOPE(c, i); + ADDOP_I(c, NO_LOCATION, LOAD_CLOSURE, i); + RETURN_IF_ERROR_IN_SCOPE( + c, codegen_nameop(c, NO_LOCATION, &_Py_ID(__classdictcell__), Store)); + } + /* Return __classcell__ if it is referenced, otherwise return None */ + if (SYMTABLE_ENTRY(c)->ste_needs_class_closure) { + /* Store __classcell__ into class namespace & return it */ + int i = _PyCompile_LookupCellvar(c, &_Py_ID(__class__)); + RETURN_IF_ERROR_IN_SCOPE(c, i); + ADDOP_I(c, NO_LOCATION, LOAD_CLOSURE, i); + ADDOP_I(c, NO_LOCATION, COPY, 1); + RETURN_IF_ERROR_IN_SCOPE( + c, codegen_nameop(c, NO_LOCATION, &_Py_ID(__classcell__), Store)); + } + else { + /* No methods referenced __class__, so just return None */ + ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); + } + ADDOP_IN_SCOPE(c, NO_LOCATION, RETURN_VALUE); + /* create the code object */ + PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 1); + + /* leave the new scope */ + _PyCompile_ExitScope(c); + if (co == NULL) { + return ERROR; + } + + /* 2. load the 'build_class' function */ + + // these instructions should be attributed to the class line, + // not a decorator line + loc = LOC(s); + ADDOP(c, loc, LOAD_BUILD_CLASS); + ADDOP(c, loc, PUSH_NULL); + + /* 3. load a function (or closure) made from the code object */ + int ret = codegen_make_closure(c, loc, co, 0); + Py_DECREF(co); + RETURN_IF_ERROR(ret); + + /* 4. load class name */ + ADDOP_LOAD_CONST(c, loc, s->v.ClassDef.name); + + return SUCCESS; +} + +static int +codegen_class(compiler *c, stmt_ty s) +{ + asdl_expr_seq *decos = s->v.ClassDef.decorator_list; + + RETURN_IF_ERROR(codegen_decorators(c, decos)); + + int firstlineno = s->lineno; + if (asdl_seq_LEN(decos)) { + firstlineno = ((expr_ty)asdl_seq_GET(decos, 0))->lineno; + } + location loc = LOC(s); + + asdl_type_param_seq *type_params = s->v.ClassDef.type_params; + int is_generic = asdl_seq_LEN(type_params) > 0; + if (is_generic) { + PyObject *type_params_name = PyUnicode_FromFormat("", + s->v.ClassDef.name); + if (!type_params_name) { + return ERROR; + } + int ret = codegen_enter_scope(c, type_params_name, COMPILE_SCOPE_ANNOTATIONS, + (void *)type_params, firstlineno, s->v.ClassDef.name, NULL); + Py_DECREF(type_params_name); + RETURN_IF_ERROR(ret); + RETURN_IF_ERROR_IN_SCOPE(c, codegen_type_params(c, type_params)); + _Py_DECLARE_STR(type_params, ".type_params"); + RETURN_IF_ERROR_IN_SCOPE(c, codegen_nameop(c, loc, &_Py_STR(type_params), Store)); + } + + int ret = codegen_class_body(c, s, firstlineno); + if (is_generic) { + RETURN_IF_ERROR_IN_SCOPE(c, ret); + } + else { + RETURN_IF_ERROR(ret); + } + + /* generate the rest of the code for the call */ + + if (is_generic) { + _Py_DECLARE_STR(type_params, ".type_params"); + _Py_DECLARE_STR(generic_base, ".generic_base"); + RETURN_IF_ERROR_IN_SCOPE(c, codegen_nameop(c, loc, &_Py_STR(type_params), Load)); + ADDOP_I_IN_SCOPE(c, loc, CALL_INTRINSIC_1, INTRINSIC_SUBSCRIPT_GENERIC); + RETURN_IF_ERROR_IN_SCOPE(c, codegen_nameop(c, loc, &_Py_STR(generic_base), Store)); + + RETURN_IF_ERROR_IN_SCOPE(c, codegen_call_helper_impl(c, loc, 2, + s->v.ClassDef.bases, + &_Py_STR(generic_base), + s->v.ClassDef.keywords)); + + PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 0); + + _PyCompile_ExitScope(c); + if (co == NULL) { + return ERROR; + } + int ret = codegen_make_closure(c, loc, co, 0); + Py_DECREF(co); + RETURN_IF_ERROR(ret); + ADDOP(c, loc, PUSH_NULL); + ADDOP_I(c, loc, CALL, 0); + } else { + RETURN_IF_ERROR(codegen_call_helper(c, loc, 2, + s->v.ClassDef.bases, + s->v.ClassDef.keywords)); + } + + /* 6. apply decorators */ + RETURN_IF_ERROR(codegen_apply_decorators(c, decos)); + + /* 7. store into */ + RETURN_IF_ERROR(codegen_nameop(c, loc, s->v.ClassDef.name, Store)); + return SUCCESS; +} + +static int +codegen_typealias_body(compiler *c, stmt_ty s) +{ + location loc = LOC(s); + PyObject *name = s->v.TypeAlias.name->v.Name.id; + PyObject *defaults = PyTuple_Pack(1, _PyLong_GetOne()); + ADDOP_LOAD_CONST_NEW(c, loc, defaults); + RETURN_IF_ERROR( + codegen_setup_annotations_scope(c, LOC(s), s, name)); + + assert(!SYMTABLE_ENTRY(c)->ste_has_docstring); + VISIT_IN_SCOPE(c, expr, s->v.TypeAlias.value); + ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); + PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 0); + _PyCompile_ExitScope(c); + if (co == NULL) { + return ERROR; + } + int ret = codegen_make_closure(c, loc, co, MAKE_FUNCTION_DEFAULTS); + Py_DECREF(co); + RETURN_IF_ERROR(ret); + + ADDOP_I(c, loc, BUILD_TUPLE, 3); + ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_TYPEALIAS); + return SUCCESS; +} + +static int +codegen_typealias(compiler *c, stmt_ty s) +{ + location loc = LOC(s); + asdl_type_param_seq *type_params = s->v.TypeAlias.type_params; + int is_generic = asdl_seq_LEN(type_params) > 0; + PyObject *name = s->v.TypeAlias.name->v.Name.id; + if (is_generic) { + PyObject *type_params_name = PyUnicode_FromFormat("", + name); + if (!type_params_name) { + return ERROR; + } + int ret = codegen_enter_scope(c, type_params_name, COMPILE_SCOPE_ANNOTATIONS, + (void *)type_params, loc.lineno, NULL, NULL); + Py_DECREF(type_params_name); + RETURN_IF_ERROR(ret); + ADDOP_LOAD_CONST_IN_SCOPE(c, loc, name); + RETURN_IF_ERROR_IN_SCOPE(c, codegen_type_params(c, type_params)); + } + else { + ADDOP_LOAD_CONST(c, loc, name); + ADDOP_LOAD_CONST(c, loc, Py_None); + } + + int ret = codegen_typealias_body(c, s); + if (is_generic) { + RETURN_IF_ERROR_IN_SCOPE(c, ret); + } + else { + RETURN_IF_ERROR(ret); + } + + if (is_generic) { + PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 0); + _PyCompile_ExitScope(c); + if (co == NULL) { + return ERROR; + } + int ret = codegen_make_closure(c, loc, co, 0); + Py_DECREF(co); + RETURN_IF_ERROR(ret); + ADDOP(c, loc, PUSH_NULL); + ADDOP_I(c, loc, CALL, 0); + } + RETURN_IF_ERROR(codegen_nameop(c, loc, name, Store)); + return SUCCESS; +} + +/* Return false if the expression is a constant value except named singletons. + Return true otherwise. */ +static bool +check_is_arg(expr_ty e) +{ + if (e->kind != Constant_kind) { + return true; + } + PyObject *value = e->v.Constant.value; + return (value == Py_None + || value == Py_False + || value == Py_True + || value == Py_Ellipsis); +} + +static PyTypeObject * infer_type(expr_ty e); + +/* Check operands of identity checks ("is" and "is not"). + Emit a warning if any operand is a constant except named singletons. + */ +static int +codegen_check_compare(compiler *c, expr_ty e) +{ + Py_ssize_t i, n; + bool left = check_is_arg(e->v.Compare.left); + expr_ty left_expr = e->v.Compare.left; + n = asdl_seq_LEN(e->v.Compare.ops); + for (i = 0; i < n; i++) { + cmpop_ty op = (cmpop_ty)asdl_seq_GET(e->v.Compare.ops, i); + expr_ty right_expr = (expr_ty)asdl_seq_GET(e->v.Compare.comparators, i); + bool right = check_is_arg(right_expr); + if (op == Is || op == IsNot) { + if (!right || !left) { + const char *msg = (op == Is) + ? "\"is\" with '%.200s' literal. Did you mean \"==\"?" + : "\"is not\" with '%.200s' literal. Did you mean \"!=\"?"; + expr_ty literal = !left ? left_expr : right_expr; + return _PyCompile_Warn( + c, LOC(e), msg, infer_type(literal)->tp_name + ); + } + } + left = right; + left_expr = right_expr; + } + return SUCCESS; +} + +static int +codegen_addcompare(compiler *c, location loc, cmpop_ty op) +{ + int cmp; + switch (op) { + case Eq: + cmp = Py_EQ; + break; + case NotEq: + cmp = Py_NE; + break; + case Lt: + cmp = Py_LT; + break; + case LtE: + cmp = Py_LE; + break; + case Gt: + cmp = Py_GT; + break; + case GtE: + cmp = Py_GE; + break; + case Is: + ADDOP_I(c, loc, IS_OP, 0); + return SUCCESS; + case IsNot: + ADDOP_I(c, loc, IS_OP, 1); + return SUCCESS; + case In: + ADDOP_I(c, loc, CONTAINS_OP, 0); + return SUCCESS; + case NotIn: + ADDOP_I(c, loc, CONTAINS_OP, 1); + return SUCCESS; + default: + Py_UNREACHABLE(); + } + // cmp goes in top three bits of the oparg, while the low four bits are used + // by quickened versions of this opcode to store the comparison mask. The + // fifth-lowest bit indicates whether the result should be converted to bool + // and is set later): + ADDOP_I(c, loc, COMPARE_OP, (cmp << 5) | compare_masks[cmp]); + return SUCCESS; +} + +static int +codegen_jump_if(compiler *c, location loc, + expr_ty e, jump_target_label next, int cond) +{ + switch (e->kind) { + case UnaryOp_kind: + if (e->v.UnaryOp.op == Not) { + return codegen_jump_if(c, loc, e->v.UnaryOp.operand, next, !cond); + } + /* fallback to general implementation */ + break; + case BoolOp_kind: { + asdl_expr_seq *s = e->v.BoolOp.values; + Py_ssize_t i, n = asdl_seq_LEN(s) - 1; + assert(n >= 0); + int cond2 = e->v.BoolOp.op == Or; + jump_target_label next2 = next; + if (!cond2 != !cond) { + NEW_JUMP_TARGET_LABEL(c, new_next2); + next2 = new_next2; + } + for (i = 0; i < n; ++i) { + RETURN_IF_ERROR( + codegen_jump_if(c, loc, (expr_ty)asdl_seq_GET(s, i), next2, cond2)); + } + RETURN_IF_ERROR( + codegen_jump_if(c, loc, (expr_ty)asdl_seq_GET(s, n), next, cond)); + if (!SAME_JUMP_TARGET_LABEL(next2, next)) { + USE_LABEL(c, next2); + } + return SUCCESS; + } + case IfExp_kind: { + NEW_JUMP_TARGET_LABEL(c, end); + NEW_JUMP_TARGET_LABEL(c, next2); + RETURN_IF_ERROR( + codegen_jump_if(c, loc, e->v.IfExp.test, next2, 0)); + RETURN_IF_ERROR( + codegen_jump_if(c, loc, e->v.IfExp.body, next, cond)); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); + + USE_LABEL(c, next2); + RETURN_IF_ERROR( + codegen_jump_if(c, loc, e->v.IfExp.orelse, next, cond)); + + USE_LABEL(c, end); + return SUCCESS; + } + case Compare_kind: { + Py_ssize_t n = asdl_seq_LEN(e->v.Compare.ops) - 1; + if (n > 0) { + RETURN_IF_ERROR(codegen_check_compare(c, e)); + NEW_JUMP_TARGET_LABEL(c, cleanup); + VISIT(c, expr, e->v.Compare.left); + for (Py_ssize_t i = 0; i < n; i++) { + VISIT(c, expr, + (expr_ty)asdl_seq_GET(e->v.Compare.comparators, i)); + ADDOP_I(c, LOC(e), SWAP, 2); + ADDOP_I(c, LOC(e), COPY, 2); + ADDOP_COMPARE(c, LOC(e), asdl_seq_GET(e->v.Compare.ops, i)); + ADDOP(c, LOC(e), TO_BOOL); + ADDOP_JUMP(c, LOC(e), POP_JUMP_IF_FALSE, cleanup); + } + VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, n)); + ADDOP_COMPARE(c, LOC(e), asdl_seq_GET(e->v.Compare.ops, n)); + ADDOP(c, LOC(e), TO_BOOL); + ADDOP_JUMP(c, LOC(e), cond ? POP_JUMP_IF_TRUE : POP_JUMP_IF_FALSE, next); + NEW_JUMP_TARGET_LABEL(c, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); + + USE_LABEL(c, cleanup); + ADDOP(c, LOC(e), POP_TOP); + if (!cond) { + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, next); + } + + USE_LABEL(c, end); + return SUCCESS; + } + /* fallback to general implementation */ + break; + } + default: + /* fallback to general implementation */ + break; + } + + /* general implementation */ + VISIT(c, expr, e); + ADDOP(c, LOC(e), TO_BOOL); + ADDOP_JUMP(c, LOC(e), cond ? POP_JUMP_IF_TRUE : POP_JUMP_IF_FALSE, next); + return SUCCESS; +} + +static int +codegen_ifexp(compiler *c, expr_ty e) +{ + assert(e->kind == IfExp_kind); + NEW_JUMP_TARGET_LABEL(c, end); + NEW_JUMP_TARGET_LABEL(c, next); + + RETURN_IF_ERROR( + codegen_jump_if(c, LOC(e), e->v.IfExp.test, next, 0)); + + VISIT(c, expr, e->v.IfExp.body); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); + + USE_LABEL(c, next); + VISIT(c, expr, e->v.IfExp.orelse); + + USE_LABEL(c, end); + return SUCCESS; +} + +static int +codegen_lambda(compiler *c, expr_ty e) +{ + PyCodeObject *co; + Py_ssize_t funcflags; + arguments_ty args = e->v.Lambda.args; + assert(e->kind == Lambda_kind); + + location loc = LOC(e); + funcflags = codegen_default_arguments(c, loc, args); + RETURN_IF_ERROR(funcflags); + + _PyCompile_CodeUnitMetadata umd = { + .u_argcount = asdl_seq_LEN(args->args), + .u_posonlyargcount = asdl_seq_LEN(args->posonlyargs), + .u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs), + }; + _Py_DECLARE_STR(anon_lambda, ""); + RETURN_IF_ERROR( + codegen_enter_scope(c, &_Py_STR(anon_lambda), COMPILE_SCOPE_LAMBDA, + (void *)e, e->lineno, NULL, &umd)); + + assert(!SYMTABLE_ENTRY(c)->ste_has_docstring); + + VISIT_IN_SCOPE(c, expr, e->v.Lambda.body); + if (SYMTABLE_ENTRY(c)->ste_generator) { + co = _PyCompile_OptimizeAndAssemble(c, 0); + } + else { + location loc = LOC(e->v.Lambda.body); + ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); + co = _PyCompile_OptimizeAndAssemble(c, 1); + } + _PyCompile_ExitScope(c); + if (co == NULL) { + return ERROR; + } + + int ret = codegen_make_closure(c, loc, co, funcflags); + Py_DECREF(co); + RETURN_IF_ERROR(ret); + return SUCCESS; +} + +static int +codegen_if(compiler *c, stmt_ty s) +{ + jump_target_label next; + assert(s->kind == If_kind); + NEW_JUMP_TARGET_LABEL(c, end); + if (asdl_seq_LEN(s->v.If.orelse)) { + NEW_JUMP_TARGET_LABEL(c, orelse); + next = orelse; + } + else { + next = end; + } + RETURN_IF_ERROR( + codegen_jump_if(c, LOC(s), s->v.If.test, next, 0)); + + VISIT_SEQ(c, stmt, s->v.If.body); + if (asdl_seq_LEN(s->v.If.orelse)) { + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); + + USE_LABEL(c, next); + VISIT_SEQ(c, stmt, s->v.If.orelse); + } + + USE_LABEL(c, end); + return SUCCESS; +} + +static int +codegen_for(compiler *c, stmt_ty s) +{ + location loc = LOC(s); + NEW_JUMP_TARGET_LABEL(c, start); + NEW_JUMP_TARGET_LABEL(c, body); + NEW_JUMP_TARGET_LABEL(c, cleanup); + NEW_JUMP_TARGET_LABEL(c, end); + + RETURN_IF_ERROR(_PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_FOR_LOOP, start, end, NULL)); + + VISIT(c, expr, s->v.For.iter); + + loc = LOC(s->v.For.iter); + ADDOP(c, loc, GET_ITER); + + USE_LABEL(c, start); + ADDOP_JUMP(c, loc, FOR_ITER, cleanup); + + /* Add NOP to ensure correct line tracing of multiline for statements. + * It will be removed later if redundant. + */ + ADDOP(c, LOC(s->v.For.target), NOP); + + USE_LABEL(c, body); + VISIT(c, expr, s->v.For.target); + VISIT_SEQ(c, stmt, s->v.For.body); + /* Mark jump as artificial */ + ADDOP_JUMP(c, NO_LOCATION, JUMP, start); + + USE_LABEL(c, cleanup); + /* It is important for instrumentation that the `END_FOR` comes first. + * Iteration over a generator will jump to the first of these instructions, + * but a non-generator will jump to a later instruction. + */ + ADDOP(c, NO_LOCATION, END_FOR); + ADDOP(c, NO_LOCATION, POP_ITER); + + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_FOR_LOOP, start); + + VISIT_SEQ(c, stmt, s->v.For.orelse); + + USE_LABEL(c, end); + return SUCCESS; +} + + +static int +codegen_async_for(compiler *c, stmt_ty s) +{ + location loc = LOC(s); + + NEW_JUMP_TARGET_LABEL(c, start); + NEW_JUMP_TARGET_LABEL(c, except); + NEW_JUMP_TARGET_LABEL(c, end); + + VISIT(c, expr, s->v.AsyncFor.iter); + ADDOP(c, LOC(s->v.AsyncFor.iter), GET_AITER); + + USE_LABEL(c, start); + RETURN_IF_ERROR(_PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_FOR_LOOP, start, end, NULL)); + + /* SETUP_FINALLY to guard the __anext__ call */ + ADDOP_JUMP(c, loc, SETUP_FINALLY, except); + ADDOP(c, loc, GET_ANEXT); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADD_YIELD_FROM(c, loc, 1); + ADDOP(c, loc, POP_BLOCK); /* for SETUP_FINALLY */ + + /* Success block for __anext__ */ + VISIT(c, expr, s->v.AsyncFor.target); + VISIT_SEQ(c, stmt, s->v.AsyncFor.body); + /* Mark jump as artificial */ + ADDOP_JUMP(c, NO_LOCATION, JUMP, start); + + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_FOR_LOOP, start); + + /* Except block for __anext__ */ + USE_LABEL(c, except); + + /* Use same line number as the iterator, + * as the END_ASYNC_FOR succeeds the `for`, not the body. */ + loc = LOC(s->v.AsyncFor.iter); + ADDOP(c, loc, END_ASYNC_FOR); + + /* `else` block */ + VISIT_SEQ(c, stmt, s->v.AsyncFor.orelse); + + USE_LABEL(c, end); + return SUCCESS; +} + +static int +codegen_while(compiler *c, stmt_ty s) +{ + NEW_JUMP_TARGET_LABEL(c, loop); + NEW_JUMP_TARGET_LABEL(c, end); + NEW_JUMP_TARGET_LABEL(c, anchor); + + USE_LABEL(c, loop); + + RETURN_IF_ERROR(_PyCompile_PushFBlock(c, LOC(s), COMPILE_FBLOCK_WHILE_LOOP, loop, end, NULL)); + RETURN_IF_ERROR(codegen_jump_if(c, LOC(s), s->v.While.test, anchor, 0)); + + VISIT_SEQ(c, stmt, s->v.While.body); + ADDOP_JUMP(c, NO_LOCATION, JUMP, loop); + + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_WHILE_LOOP, loop); + + USE_LABEL(c, anchor); + if (s->v.While.orelse) { + VISIT_SEQ(c, stmt, s->v.While.orelse); + } + + USE_LABEL(c, end); + return SUCCESS; +} + +static int +codegen_return(compiler *c, stmt_ty s) +{ + location loc = LOC(s); + int preserve_tos = ((s->v.Return.value != NULL) && + (s->v.Return.value->kind != Constant_kind)); + + PySTEntryObject *ste = SYMTABLE_ENTRY(c); + if (!_PyST_IsFunctionLike(ste)) { + return _PyCompile_Error(c, loc, "'return' outside function"); + } + if (s->v.Return.value != NULL && ste->ste_coroutine && ste->ste_generator) { + return _PyCompile_Error(c, loc, "'return' with value in async generator"); + } + + if (preserve_tos) { + VISIT(c, expr, s->v.Return.value); + } else { + /* Emit instruction with line number for return value */ + if (s->v.Return.value != NULL) { + loc = LOC(s->v.Return.value); + ADDOP(c, loc, NOP); + } + } + if (s->v.Return.value == NULL || s->v.Return.value->lineno != s->lineno) { + loc = LOC(s); + ADDOP(c, loc, NOP); + } + + RETURN_IF_ERROR(codegen_unwind_fblock_stack(c, &loc, preserve_tos, NULL)); + if (s->v.Return.value == NULL) { + ADDOP_LOAD_CONST(c, loc, Py_None); + } + else if (!preserve_tos) { + ADDOP_LOAD_CONST(c, loc, s->v.Return.value->v.Constant.value); + } + ADDOP(c, loc, RETURN_VALUE); + + return SUCCESS; +} + +static int +codegen_break(compiler *c, location loc) +{ + fblockinfo *loop = NULL; + location origin_loc = loc; + /* Emit instruction with line number */ + ADDOP(c, loc, NOP); + RETURN_IF_ERROR(codegen_unwind_fblock_stack(c, &loc, 0, &loop)); + if (loop == NULL) { + return _PyCompile_Error(c, origin_loc, "'break' outside loop"); + } + RETURN_IF_ERROR(codegen_unwind_fblock(c, &loc, loop, 0)); + ADDOP_JUMP(c, loc, JUMP, loop->fb_exit); + return SUCCESS; +} + +static int +codegen_continue(compiler *c, location loc) +{ + fblockinfo *loop = NULL; + location origin_loc = loc; + /* Emit instruction with line number */ + ADDOP(c, loc, NOP); + RETURN_IF_ERROR(codegen_unwind_fblock_stack(c, &loc, 0, &loop)); + if (loop == NULL) { + return _PyCompile_Error(c, origin_loc, "'continue' not properly in loop"); + } + ADDOP_JUMP(c, loc, JUMP, loop->fb_block); + return SUCCESS; +} + + +/* Code generated for "try: finally: " is as follows: + + SETUP_FINALLY L + + POP_BLOCK + + JUMP E + L: + + E: + + The special instructions use the block stack. Each block + stack entry contains the instruction that created it (here + SETUP_FINALLY), the level of the value stack at the time the + block stack entry was created, and a label (here L). + + SETUP_FINALLY: + Pushes the current value stack level and the label + onto the block stack. + POP_BLOCK: + Pops en entry from the block stack. + + The block stack is unwound when an exception is raised: + when a SETUP_FINALLY entry is found, the raised and the caught + exceptions are pushed onto the value stack (and the exception + condition is cleared), and the interpreter jumps to the label + gotten from the block stack. +*/ + +static int +codegen_try_finally(compiler *c, stmt_ty s) +{ + location loc = LOC(s); + + NEW_JUMP_TARGET_LABEL(c, body); + NEW_JUMP_TARGET_LABEL(c, end); + NEW_JUMP_TARGET_LABEL(c, exit); + NEW_JUMP_TARGET_LABEL(c, cleanup); + + /* `try` block */ + ADDOP_JUMP(c, loc, SETUP_FINALLY, end); + + USE_LABEL(c, body); + RETURN_IF_ERROR( + _PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_FINALLY_TRY, body, end, + s->v.Try.finalbody)); + + if (s->v.Try.handlers && asdl_seq_LEN(s->v.Try.handlers)) { + RETURN_IF_ERROR(codegen_try_except(c, s)); + } + else { + VISIT_SEQ(c, stmt, s->v.Try.body); + } + ADDOP(c, NO_LOCATION, POP_BLOCK); + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_FINALLY_TRY, body); + VISIT_SEQ(c, stmt, s->v.Try.finalbody); + + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, exit); + /* `finally` block */ + + USE_LABEL(c, end); + + loc = NO_LOCATION; + ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); + ADDOP(c, loc, PUSH_EXC_INFO); + RETURN_IF_ERROR( + _PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_FINALLY_END, end, NO_LABEL, NULL)); + VISIT_SEQ(c, stmt, s->v.Try.finalbody); + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_FINALLY_END, end); + + loc = NO_LOCATION; + ADDOP_I(c, loc, RERAISE, 0); + + USE_LABEL(c, cleanup); + POP_EXCEPT_AND_RERAISE(c, loc); + + USE_LABEL(c, exit); + return SUCCESS; +} + +static int +codegen_try_star_finally(compiler *c, stmt_ty s) +{ + location loc = LOC(s); + + NEW_JUMP_TARGET_LABEL(c, body); + NEW_JUMP_TARGET_LABEL(c, end); + NEW_JUMP_TARGET_LABEL(c, exit); + NEW_JUMP_TARGET_LABEL(c, cleanup); + /* `try` block */ + ADDOP_JUMP(c, loc, SETUP_FINALLY, end); + + USE_LABEL(c, body); + RETURN_IF_ERROR( + _PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_FINALLY_TRY, body, end, + s->v.TryStar.finalbody)); + + if (s->v.TryStar.handlers && asdl_seq_LEN(s->v.TryStar.handlers)) { + RETURN_IF_ERROR(codegen_try_star_except(c, s)); + } + else { + VISIT_SEQ(c, stmt, s->v.TryStar.body); + } + ADDOP(c, NO_LOCATION, POP_BLOCK); + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_FINALLY_TRY, body); + VISIT_SEQ(c, stmt, s->v.TryStar.finalbody); + + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, exit); + + /* `finally` block */ + USE_LABEL(c, end); + + loc = NO_LOCATION; + ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); + ADDOP(c, loc, PUSH_EXC_INFO); + RETURN_IF_ERROR( + _PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_FINALLY_END, end, NO_LABEL, NULL)); + + VISIT_SEQ(c, stmt, s->v.TryStar.finalbody); + + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_FINALLY_END, end); + loc = NO_LOCATION; + ADDOP_I(c, loc, RERAISE, 0); + + USE_LABEL(c, cleanup); + POP_EXCEPT_AND_RERAISE(c, loc); + + USE_LABEL(c, exit); + return SUCCESS; +} + + +/* + Code generated for "try: S except E1 as V1: S1 except E2 as V2: S2 ...": + (The contents of the value stack is shown in [], with the top + at the right; 'tb' is trace-back info, 'val' the exception's + associated value, and 'exc' the exception.) + + Value stack Label Instruction Argument + [] SETUP_FINALLY L1 + [] + [] POP_BLOCK + [] JUMP L0 + + [exc] L1: ) + [exc, E1] CHECK_EXC_MATCH ) + [exc, bool] POP_JUMP_IF_FALSE L2 ) only if E1 + [exc] (or POP if no V1) + [] + JUMP L0 + + [exc] L2: + .............................etc....................... + + [exc] Ln+1: RERAISE # re-raise exception + + [] L0: + + Of course, parts are not generated if Vi or Ei is not present. +*/ +static int +codegen_try_except(compiler *c, stmt_ty s) +{ + location loc = LOC(s); + Py_ssize_t i, n; + + NEW_JUMP_TARGET_LABEL(c, body); + NEW_JUMP_TARGET_LABEL(c, except); + NEW_JUMP_TARGET_LABEL(c, end); + NEW_JUMP_TARGET_LABEL(c, cleanup); + + ADDOP_JUMP(c, loc, SETUP_FINALLY, except); + + USE_LABEL(c, body); + RETURN_IF_ERROR( + _PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_TRY_EXCEPT, body, NO_LABEL, NULL)); + VISIT_SEQ(c, stmt, s->v.Try.body); + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_TRY_EXCEPT, body); + ADDOP(c, NO_LOCATION, POP_BLOCK); + if (s->v.Try.orelse && asdl_seq_LEN(s->v.Try.orelse)) { + VISIT_SEQ(c, stmt, s->v.Try.orelse); + } + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); + n = asdl_seq_LEN(s->v.Try.handlers); + + USE_LABEL(c, except); + + ADDOP_JUMP(c, NO_LOCATION, SETUP_CLEANUP, cleanup); + ADDOP(c, NO_LOCATION, PUSH_EXC_INFO); + + /* Runtime will push a block here, so we need to account for that */ + RETURN_IF_ERROR( + _PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_EXCEPTION_HANDLER, + NO_LABEL, NO_LABEL, NULL)); + + for (i = 0; i < n; i++) { + excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( + s->v.Try.handlers, i); + location loc = LOC(handler); + if (!handler->v.ExceptHandler.type && i < n-1) { + return _PyCompile_Error(c, loc, "default 'except:' must be last"); + } + NEW_JUMP_TARGET_LABEL(c, next_except); + except = next_except; + if (handler->v.ExceptHandler.type) { + VISIT(c, expr, handler->v.ExceptHandler.type); + ADDOP(c, loc, CHECK_EXC_MATCH); + ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, except); + } + if (handler->v.ExceptHandler.name) { + NEW_JUMP_TARGET_LABEL(c, cleanup_end); + NEW_JUMP_TARGET_LABEL(c, cleanup_body); + + RETURN_IF_ERROR( + codegen_nameop(c, loc, handler->v.ExceptHandler.name, Store)); + + /* + try: + # body + except type as name: + try: + # body + finally: + name = None # in case body contains "del name" + del name + */ + + /* second try: */ + ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup_end); + + USE_LABEL(c, cleanup_body); + RETURN_IF_ERROR( + _PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_HANDLER_CLEANUP, cleanup_body, + NO_LABEL, handler->v.ExceptHandler.name)); + + /* second # body */ + VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_HANDLER_CLEANUP, cleanup_body); + /* name = None; del name; # Mark as artificial */ + ADDOP(c, NO_LOCATION, POP_BLOCK); + ADDOP(c, NO_LOCATION, POP_BLOCK); + ADDOP(c, NO_LOCATION, POP_EXCEPT); + ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); + RETURN_IF_ERROR( + codegen_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store)); + RETURN_IF_ERROR( + codegen_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del)); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); + + /* except: */ + USE_LABEL(c, cleanup_end); + + /* name = None; del name; # artificial */ + ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); + RETURN_IF_ERROR( + codegen_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store)); + RETURN_IF_ERROR( + codegen_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del)); + + ADDOP_I(c, NO_LOCATION, RERAISE, 1); + } + else { + NEW_JUMP_TARGET_LABEL(c, cleanup_body); + + ADDOP(c, loc, POP_TOP); /* exc_value */ + + USE_LABEL(c, cleanup_body); + RETURN_IF_ERROR( + _PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_HANDLER_CLEANUP, cleanup_body, + NO_LABEL, NULL)); + + VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_HANDLER_CLEANUP, cleanup_body); + ADDOP(c, NO_LOCATION, POP_BLOCK); + ADDOP(c, NO_LOCATION, POP_EXCEPT); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); + } + + USE_LABEL(c, except); + } + /* artificial */ + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_EXCEPTION_HANDLER, NO_LABEL); + ADDOP_I(c, NO_LOCATION, RERAISE, 0); + + USE_LABEL(c, cleanup); + POP_EXCEPT_AND_RERAISE(c, NO_LOCATION); + + USE_LABEL(c, end); + return SUCCESS; +} + +/* + Code generated for "try: S except* E1 as V1: S1 except* E2 as V2: S2 ...": + (The contents of the value stack is shown in [], with the top + at the right; 'tb' is trace-back info, 'val' the exception instance, + and 'typ' the exception's type.) + + Value stack Label Instruction Argument + [] SETUP_FINALLY L1 + [] + [] POP_BLOCK + [] JUMP L0 + + [exc] L1: BUILD_LIST ) list for raised/reraised excs ("result") + [orig, res] COPY 2 ) make a copy of the original EG + + [orig, res, exc] + [orig, res, exc, E1] CHECK_EG_MATCH + [orig, res, rest/exc, match?] COPY 1 + [orig, res, rest/exc, match?, match?] POP_JUMP_IF_NONE C1 + + [orig, res, rest, match] (or POP if no V1) + + [orig, res, rest] SETUP_FINALLY R1 + [orig, res, rest] + [orig, res, rest] JUMP L2 + + [orig, res, rest, i, v] R1: LIST_APPEND 3 ) exc raised in except* body - add to res + [orig, res, rest, i] POP + [orig, res, rest] JUMP LE2 + + [orig, res, rest] L2: NOP ) for lineno + [orig, res, rest] JUMP LE2 + + [orig, res, rest/exc, None] C1: POP + + [orig, res, rest] LE2: + .............................etc....................... + + [orig, res, rest] Ln+1: LIST_APPEND 1 ) add unhandled exc to res (could be None) + + [orig, res] CALL_INTRINSIC_2 PREP_RERAISE_STAR + [exc] COPY 1 + [exc, exc] POP_JUMP_IF_NOT_NONE RER + [exc] POP_TOP + [] JUMP L0 + + [exc] RER: SWAP 2 + [exc, prev_exc_info] POP_EXCEPT + [exc] RERAISE 0 + + [] L0: +*/ +static int +codegen_try_star_except(compiler *c, stmt_ty s) +{ + location loc = LOC(s); + + NEW_JUMP_TARGET_LABEL(c, body); + NEW_JUMP_TARGET_LABEL(c, except); + NEW_JUMP_TARGET_LABEL(c, orelse); + NEW_JUMP_TARGET_LABEL(c, end); + NEW_JUMP_TARGET_LABEL(c, cleanup); + NEW_JUMP_TARGET_LABEL(c, reraise_star); + + ADDOP_JUMP(c, loc, SETUP_FINALLY, except); + + USE_LABEL(c, body); + RETURN_IF_ERROR( + _PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_TRY_EXCEPT, body, NO_LABEL, NULL)); + VISIT_SEQ(c, stmt, s->v.TryStar.body); + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_TRY_EXCEPT, body); + ADDOP(c, NO_LOCATION, POP_BLOCK); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, orelse); + Py_ssize_t n = asdl_seq_LEN(s->v.TryStar.handlers); + + USE_LABEL(c, except); + + ADDOP_JUMP(c, NO_LOCATION, SETUP_CLEANUP, cleanup); + ADDOP(c, NO_LOCATION, PUSH_EXC_INFO); + + /* Runtime will push a block here, so we need to account for that */ + RETURN_IF_ERROR( + _PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_EXCEPTION_GROUP_HANDLER, + NO_LABEL, NO_LABEL, "except handler")); + + for (Py_ssize_t i = 0; i < n; i++) { + excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( + s->v.TryStar.handlers, i); + location loc = LOC(handler); + NEW_JUMP_TARGET_LABEL(c, next_except); + except = next_except; + NEW_JUMP_TARGET_LABEL(c, except_with_error); + NEW_JUMP_TARGET_LABEL(c, no_match); + if (i == 0) { + /* create empty list for exceptions raised/reraise in the except* blocks */ + /* + [orig] BUILD_LIST + */ + /* Create a copy of the original EG */ + /* + [orig, []] COPY 2 + [orig, [], exc] + */ + ADDOP_I(c, loc, BUILD_LIST, 0); + ADDOP_I(c, loc, COPY, 2); + } + if (handler->v.ExceptHandler.type) { + VISIT(c, expr, handler->v.ExceptHandler.type); + ADDOP(c, loc, CHECK_EG_MATCH); + ADDOP_I(c, loc, COPY, 1); + ADDOP_JUMP(c, loc, POP_JUMP_IF_NONE, no_match); + } + + NEW_JUMP_TARGET_LABEL(c, cleanup_end); + NEW_JUMP_TARGET_LABEL(c, cleanup_body); + + if (handler->v.ExceptHandler.name) { + RETURN_IF_ERROR( + codegen_nameop(c, loc, handler->v.ExceptHandler.name, Store)); + } + else { + ADDOP(c, loc, POP_TOP); // match + } + + /* + try: + # body + except type as name: + try: + # body + finally: + name = None # in case body contains "del name" + del name + */ + /* second try: */ + ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup_end); + + USE_LABEL(c, cleanup_body); + RETURN_IF_ERROR( + _PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_HANDLER_CLEANUP, cleanup_body, + NO_LABEL, handler->v.ExceptHandler.name)); + + /* second # body */ + VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_HANDLER_CLEANUP, cleanup_body); + /* name = None; del name; # artificial */ + ADDOP(c, NO_LOCATION, POP_BLOCK); + if (handler->v.ExceptHandler.name) { + ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); + RETURN_IF_ERROR( + codegen_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store)); + RETURN_IF_ERROR( + codegen_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del)); + } + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, except); + + /* except: */ + USE_LABEL(c, cleanup_end); + + /* name = None; del name; # artificial */ + if (handler->v.ExceptHandler.name) { + ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); + RETURN_IF_ERROR( + codegen_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store)); + RETURN_IF_ERROR( + codegen_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del)); + } + + /* add exception raised to the res list */ + ADDOP_I(c, NO_LOCATION, LIST_APPEND, 3); // exc + ADDOP(c, NO_LOCATION, POP_TOP); // lasti + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, except_with_error); + + USE_LABEL(c, except); + ADDOP(c, NO_LOCATION, NOP); // to hold a propagated location info + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, except_with_error); + + USE_LABEL(c, no_match); + ADDOP(c, loc, POP_TOP); // match (None) + + USE_LABEL(c, except_with_error); + + if (i == n - 1) { + /* Add exc to the list (if not None it's the unhandled part of the EG) */ + ADDOP_I(c, NO_LOCATION, LIST_APPEND, 1); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, reraise_star); + } + } + /* artificial */ + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_EXCEPTION_GROUP_HANDLER, NO_LABEL); + NEW_JUMP_TARGET_LABEL(c, reraise); + + USE_LABEL(c, reraise_star); + ADDOP_I(c, NO_LOCATION, CALL_INTRINSIC_2, INTRINSIC_PREP_RERAISE_STAR); + ADDOP_I(c, NO_LOCATION, COPY, 1); + ADDOP_JUMP(c, NO_LOCATION, POP_JUMP_IF_NOT_NONE, reraise); + + /* Nothing to reraise */ + ADDOP(c, NO_LOCATION, POP_TOP); + ADDOP(c, NO_LOCATION, POP_BLOCK); + ADDOP(c, NO_LOCATION, POP_EXCEPT); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); + + USE_LABEL(c, reraise); + ADDOP(c, NO_LOCATION, POP_BLOCK); + ADDOP_I(c, NO_LOCATION, SWAP, 2); + ADDOP(c, NO_LOCATION, POP_EXCEPT); + ADDOP_I(c, NO_LOCATION, RERAISE, 0); + + USE_LABEL(c, cleanup); + POP_EXCEPT_AND_RERAISE(c, NO_LOCATION); + + USE_LABEL(c, orelse); + VISIT_SEQ(c, stmt, s->v.TryStar.orelse); + + USE_LABEL(c, end); + return SUCCESS; +} + +static int +codegen_try(compiler *c, stmt_ty s) { + if (s->v.Try.finalbody && asdl_seq_LEN(s->v.Try.finalbody)) + return codegen_try_finally(c, s); + else + return codegen_try_except(c, s); +} + +static int +codegen_try_star(compiler *c, stmt_ty s) +{ + if (s->v.TryStar.finalbody && asdl_seq_LEN(s->v.TryStar.finalbody)) { + return codegen_try_star_finally(c, s); + } + else { + return codegen_try_star_except(c, s); + } +} + +static int +codegen_import_as(compiler *c, location loc, + identifier name, identifier asname) +{ + /* The IMPORT_NAME opcode was already generated. This function + merely needs to bind the result to a name. + + If there is a dot in name, we need to split it and emit a + IMPORT_FROM for each name. + */ + Py_ssize_t len = PyUnicode_GET_LENGTH(name); + Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, len, 1); + if (dot == -2) { + return ERROR; + } + if (dot != -1) { + /* Consume the base module name to get the first attribute */ + while (1) { + Py_ssize_t pos = dot + 1; + PyObject *attr; + dot = PyUnicode_FindChar(name, '.', pos, len, 1); + if (dot == -2) { + return ERROR; + } + attr = PyUnicode_Substring(name, pos, (dot != -1) ? dot : len); + if (!attr) { + return ERROR; + } + ADDOP_N(c, loc, IMPORT_FROM, attr, names); + if (dot == -1) { + break; + } + ADDOP_I(c, loc, SWAP, 2); + ADDOP(c, loc, POP_TOP); + } + RETURN_IF_ERROR(codegen_nameop(c, loc, asname, Store)); + ADDOP(c, loc, POP_TOP); + return SUCCESS; + } + return codegen_nameop(c, loc, asname, Store); +} + +static int +codegen_import(compiler *c, stmt_ty s) +{ + location loc = LOC(s); + /* The Import node stores a module name like a.b.c as a single + string. This is convenient for all cases except + import a.b.c as d + where we need to parse that string to extract the individual + module names. + XXX Perhaps change the representation to make this case simpler? + */ + Py_ssize_t i, n = asdl_seq_LEN(s->v.Import.names); + + PyObject *zero = _PyLong_GetZero(); // borrowed reference + for (i = 0; i < n; i++) { + alias_ty alias = (alias_ty)asdl_seq_GET(s->v.Import.names, i); + int r; + + ADDOP_LOAD_CONST(c, loc, zero); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADDOP_NAME(c, loc, IMPORT_NAME, alias->name, names); + + if (alias->asname) { + r = codegen_import_as(c, loc, alias->name, alias->asname); + RETURN_IF_ERROR(r); + } + else { + identifier tmp = alias->name; + Py_ssize_t dot = PyUnicode_FindChar( + alias->name, '.', 0, PyUnicode_GET_LENGTH(alias->name), 1); + if (dot != -1) { + tmp = PyUnicode_Substring(alias->name, 0, dot); + if (tmp == NULL) { + return ERROR; + } + } + r = codegen_nameop(c, loc, tmp, Store); + if (dot != -1) { + Py_DECREF(tmp); + } + RETURN_IF_ERROR(r); + } + } + return SUCCESS; +} + +static int +codegen_from_import(compiler *c, stmt_ty s) +{ + Py_ssize_t n = asdl_seq_LEN(s->v.ImportFrom.names); + + ADDOP_LOAD_CONST_NEW(c, LOC(s), PyLong_FromLong(s->v.ImportFrom.level)); + + PyObject *names = PyTuple_New(n); + if (!names) { + return ERROR; + } + + /* build up the names */ + for (Py_ssize_t i = 0; i < n; i++) { + alias_ty alias = (alias_ty)asdl_seq_GET(s->v.ImportFrom.names, i); + PyTuple_SET_ITEM(names, i, Py_NewRef(alias->name)); + } + + ADDOP_LOAD_CONST_NEW(c, LOC(s), names); + + if (s->v.ImportFrom.module) { + ADDOP_NAME(c, LOC(s), IMPORT_NAME, s->v.ImportFrom.module, names); + } + else { + _Py_DECLARE_STR(empty, ""); + ADDOP_NAME(c, LOC(s), IMPORT_NAME, &_Py_STR(empty), names); + } + for (Py_ssize_t i = 0; i < n; i++) { + alias_ty alias = (alias_ty)asdl_seq_GET(s->v.ImportFrom.names, i); + identifier store_name; + + if (i == 0 && PyUnicode_READ_CHAR(alias->name, 0) == '*') { + assert(n == 1); + ADDOP_I(c, LOC(s), CALL_INTRINSIC_1, INTRINSIC_IMPORT_STAR); + ADDOP(c, NO_LOCATION, POP_TOP); + return SUCCESS; + } + + ADDOP_NAME(c, LOC(s), IMPORT_FROM, alias->name, names); + store_name = alias->name; + if (alias->asname) { + store_name = alias->asname; + } + + RETURN_IF_ERROR(codegen_nameop(c, LOC(s), store_name, Store)); + } + /* remove imported module */ + ADDOP(c, LOC(s), POP_TOP); + return SUCCESS; +} + +static int +codegen_assert(compiler *c, stmt_ty s) +{ + /* Always emit a warning if the test is a non-zero length tuple */ + if ((s->v.Assert.test->kind == Tuple_kind && + asdl_seq_LEN(s->v.Assert.test->v.Tuple.elts) > 0) || + (s->v.Assert.test->kind == Constant_kind && + PyTuple_Check(s->v.Assert.test->v.Constant.value) && + PyTuple_Size(s->v.Assert.test->v.Constant.value) > 0)) + { + RETURN_IF_ERROR( + _PyCompile_Warn(c, LOC(s), "assertion is always true, " + "perhaps remove parentheses?")); + } + if (OPTIMIZATION_LEVEL(c)) { + return SUCCESS; + } + NEW_JUMP_TARGET_LABEL(c, end); + RETURN_IF_ERROR(codegen_jump_if(c, LOC(s), s->v.Assert.test, end, 1)); + ADDOP_I(c, LOC(s), LOAD_COMMON_CONSTANT, CONSTANT_ASSERTIONERROR); + if (s->v.Assert.msg) { + VISIT(c, expr, s->v.Assert.msg); + ADDOP_I(c, LOC(s), CALL, 0); + } + ADDOP_I(c, LOC(s->v.Assert.test), RAISE_VARARGS, 1); + + USE_LABEL(c, end); + return SUCCESS; +} + +static int +codegen_stmt_expr(compiler *c, location loc, expr_ty value) +{ + if (IS_INTERACTIVE_TOP_LEVEL(c)) { + VISIT(c, expr, value); + ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_PRINT); + ADDOP(c, NO_LOCATION, POP_TOP); + return SUCCESS; + } + + if (value->kind == Constant_kind) { + /* ignore constant statement */ + ADDOP(c, loc, NOP); + return SUCCESS; + } + + VISIT(c, expr, value); + ADDOP(c, NO_LOCATION, POP_TOP); /* artificial */ + return SUCCESS; +} + +static int +codegen_visit_stmt(compiler *c, stmt_ty s) +{ + + switch (s->kind) { + case FunctionDef_kind: + return codegen_function(c, s, 0); + case ClassDef_kind: + return codegen_class(c, s); + case TypeAlias_kind: + return codegen_typealias(c, s); + case Return_kind: + return codegen_return(c, s); + case Delete_kind: + VISIT_SEQ(c, expr, s->v.Delete.targets); + break; + case Assign_kind: + { + Py_ssize_t n = asdl_seq_LEN(s->v.Assign.targets); + VISIT(c, expr, s->v.Assign.value); + for (Py_ssize_t i = 0; i < n; i++) { + if (i < n - 1) { + ADDOP_I(c, LOC(s), COPY, 1); + } + VISIT(c, expr, + (expr_ty)asdl_seq_GET(s->v.Assign.targets, i)); + } + break; + } + case AugAssign_kind: + return codegen_augassign(c, s); + case AnnAssign_kind: + return codegen_annassign(c, s); + case For_kind: + return codegen_for(c, s); + case While_kind: + return codegen_while(c, s); + case If_kind: + return codegen_if(c, s); + case Match_kind: + return codegen_match(c, s); + case Raise_kind: + { + Py_ssize_t n = 0; + if (s->v.Raise.exc) { + VISIT(c, expr, s->v.Raise.exc); + n++; + if (s->v.Raise.cause) { + VISIT(c, expr, s->v.Raise.cause); + n++; + } + } + ADDOP_I(c, LOC(s), RAISE_VARARGS, (int)n); + break; + } + case Try_kind: + return codegen_try(c, s); + case TryStar_kind: + return codegen_try_star(c, s); + case Assert_kind: + return codegen_assert(c, s); + case Import_kind: + return codegen_import(c, s); + case ImportFrom_kind: + return codegen_from_import(c, s); + case Global_kind: + case Nonlocal_kind: + break; + case Expr_kind: + { + return codegen_stmt_expr(c, LOC(s), s->v.Expr.value); + } + case Pass_kind: + { + ADDOP(c, LOC(s), NOP); + break; + } + case Break_kind: + { + return codegen_break(c, LOC(s)); + } + case Continue_kind: + { + return codegen_continue(c, LOC(s)); + } + case With_kind: + return codegen_with(c, s, 0); + case AsyncFunctionDef_kind: + return codegen_function(c, s, 1); + case AsyncWith_kind: + return codegen_async_with(c, s, 0); + case AsyncFor_kind: + return codegen_async_for(c, s); + } + + return SUCCESS; +} + +static int +unaryop(unaryop_ty op) +{ + switch (op) { + case Invert: + return UNARY_INVERT; + case USub: + return UNARY_NEGATIVE; + default: + PyErr_Format(PyExc_SystemError, + "unary op %d should not be possible", op); + return 0; + } +} + +static int +addop_binary(compiler *c, location loc, operator_ty binop, + bool inplace) +{ + int oparg; + switch (binop) { + case Add: + oparg = inplace ? NB_INPLACE_ADD : NB_ADD; + break; + case Sub: + oparg = inplace ? NB_INPLACE_SUBTRACT : NB_SUBTRACT; + break; + case Mult: + oparg = inplace ? NB_INPLACE_MULTIPLY : NB_MULTIPLY; + break; + case MatMult: + oparg = inplace ? NB_INPLACE_MATRIX_MULTIPLY : NB_MATRIX_MULTIPLY; + break; + case Div: + oparg = inplace ? NB_INPLACE_TRUE_DIVIDE : NB_TRUE_DIVIDE; + break; + case Mod: + oparg = inplace ? NB_INPLACE_REMAINDER : NB_REMAINDER; + break; + case Pow: + oparg = inplace ? NB_INPLACE_POWER : NB_POWER; + break; + case LShift: + oparg = inplace ? NB_INPLACE_LSHIFT : NB_LSHIFT; + break; + case RShift: + oparg = inplace ? NB_INPLACE_RSHIFT : NB_RSHIFT; + break; + case BitOr: + oparg = inplace ? NB_INPLACE_OR : NB_OR; + break; + case BitXor: + oparg = inplace ? NB_INPLACE_XOR : NB_XOR; + break; + case BitAnd: + oparg = inplace ? NB_INPLACE_AND : NB_AND; + break; + case FloorDiv: + oparg = inplace ? NB_INPLACE_FLOOR_DIVIDE : NB_FLOOR_DIVIDE; + break; + default: + PyErr_Format(PyExc_SystemError, "%s op %d should not be possible", + inplace ? "inplace" : "binary", binop); + return ERROR; + } + ADDOP_I(c, loc, BINARY_OP, oparg); + return SUCCESS; +} + + +static int +codegen_addop_yield(compiler *c, location loc) { + PySTEntryObject *ste = SYMTABLE_ENTRY(c); + if (ste->ste_generator && ste->ste_coroutine) { + ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_ASYNC_GEN_WRAP); + } + ADDOP_I(c, loc, YIELD_VALUE, 0); + ADDOP_I(c, loc, RESUME, RESUME_AFTER_YIELD); + return SUCCESS; +} + +static int +codegen_load_classdict_freevar(compiler *c, location loc) +{ + ADDOP_N(c, loc, LOAD_DEREF, &_Py_ID(__classdict__), freevars); + return SUCCESS; +} + +static int +codegen_nameop(compiler *c, location loc, + identifier name, expr_context_ty ctx) +{ + assert(!_PyUnicode_EqualToASCIIString(name, "None") && + !_PyUnicode_EqualToASCIIString(name, "True") && + !_PyUnicode_EqualToASCIIString(name, "False")); + + PyObject *mangled = _PyCompile_MaybeMangle(c, name); + if (!mangled) { + return ERROR; + } + + int scope = _PyST_GetScope(SYMTABLE_ENTRY(c), mangled); + RETURN_IF_ERROR(scope); + _PyCompile_optype optype; + Py_ssize_t arg = 0; + if (_PyCompile_ResolveNameop(c, mangled, scope, &optype, &arg) < 0) { + Py_DECREF(mangled); + return ERROR; + } + + /* XXX Leave assert here, but handle __doc__ and the like better */ + assert(scope || PyUnicode_READ_CHAR(name, 0) == '_'); + + int op = 0; + switch (optype) { + case COMPILE_OP_DEREF: + switch (ctx) { + case Load: + if (SYMTABLE_ENTRY(c)->ste_type == ClassBlock && !_PyCompile_IsInInlinedComp(c)) { + op = LOAD_FROM_DICT_OR_DEREF; + // First load the locals + if (codegen_addop_noarg(INSTR_SEQUENCE(c), LOAD_LOCALS, loc) < 0) { + goto error; + } + } + else if (SYMTABLE_ENTRY(c)->ste_can_see_class_scope) { + op = LOAD_FROM_DICT_OR_DEREF; + // First load the classdict + if (codegen_load_classdict_freevar(c, loc) < 0) { + goto error; + } + } + else { + op = LOAD_DEREF; + } + break; + case Store: op = STORE_DEREF; break; + case Del: op = DELETE_DEREF; break; + } + break; + case COMPILE_OP_FAST: + switch (ctx) { + case Load: op = LOAD_FAST; break; + case Store: op = STORE_FAST; break; + case Del: op = DELETE_FAST; break; + } + ADDOP_N(c, loc, op, mangled, varnames); + return SUCCESS; + case COMPILE_OP_GLOBAL: + switch (ctx) { + case Load: + if (SYMTABLE_ENTRY(c)->ste_can_see_class_scope && scope == GLOBAL_IMPLICIT) { + op = LOAD_FROM_DICT_OR_GLOBALS; + // First load the classdict + if (codegen_load_classdict_freevar(c, loc) < 0) { + goto error; + } + } else { + op = LOAD_GLOBAL; + } + break; + case Store: op = STORE_GLOBAL; break; + case Del: op = DELETE_GLOBAL; break; + } + break; + case COMPILE_OP_NAME: + switch (ctx) { + case Load: + op = (SYMTABLE_ENTRY(c)->ste_type == ClassBlock + && _PyCompile_IsInInlinedComp(c)) + ? LOAD_GLOBAL + : LOAD_NAME; + break; + case Store: op = STORE_NAME; break; + case Del: op = DELETE_NAME; break; + } + break; + } + + assert(op); + Py_DECREF(mangled); + if (op == LOAD_GLOBAL) { + arg <<= 1; + } + ADDOP_I(c, loc, op, arg); + return SUCCESS; + +error: + Py_DECREF(mangled); + return ERROR; +} + +static int +codegen_boolop(compiler *c, expr_ty e) +{ + int jumpi; + Py_ssize_t i, n; + asdl_expr_seq *s; + + location loc = LOC(e); + assert(e->kind == BoolOp_kind); + if (e->v.BoolOp.op == And) + jumpi = JUMP_IF_FALSE; + else + jumpi = JUMP_IF_TRUE; + NEW_JUMP_TARGET_LABEL(c, end); + s = e->v.BoolOp.values; + n = asdl_seq_LEN(s) - 1; + assert(n >= 0); + for (i = 0; i < n; ++i) { + VISIT(c, expr, (expr_ty)asdl_seq_GET(s, i)); + ADDOP_JUMP(c, loc, jumpi, end); + ADDOP(c, loc, POP_TOP); + } + VISIT(c, expr, (expr_ty)asdl_seq_GET(s, n)); + + USE_LABEL(c, end); + return SUCCESS; +} + +static int +starunpack_helper_impl(compiler *c, location loc, + asdl_expr_seq *elts, PyObject *injected_arg, int pushed, + int build, int add, int extend, int tuple) +{ + Py_ssize_t n = asdl_seq_LEN(elts); + if (!injected_arg && n > 2 && are_all_items_const(elts, 0, n)) { + PyObject *folded = PyTuple_New(n); + if (folded == NULL) { + return ERROR; + } + for (Py_ssize_t i = 0; i < n; i++) { + PyObject *val = ((expr_ty)asdl_seq_GET(elts, i))->v.Constant.value; + PyTuple_SET_ITEM(folded, i, Py_NewRef(val)); + } + if (tuple && !pushed) { + ADDOP_LOAD_CONST_NEW(c, loc, folded); + } else { + if (add == SET_ADD) { + Py_SETREF(folded, PyFrozenSet_New(folded)); + if (folded == NULL) { + return ERROR; + } + } + ADDOP_I(c, loc, build, pushed); + ADDOP_LOAD_CONST_NEW(c, loc, folded); + ADDOP_I(c, loc, extend, 1); + if (tuple) { + ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_LIST_TO_TUPLE); + } + } + return SUCCESS; + } + + int big = n + pushed + (injected_arg ? 1 : 0) > STACK_USE_GUIDELINE; + int seen_star = 0; + for (Py_ssize_t i = 0; i < n; i++) { + expr_ty elt = asdl_seq_GET(elts, i); + if (elt->kind == Starred_kind) { + seen_star = 1; + break; + } + } + if (!seen_star && !big) { + for (Py_ssize_t i = 0; i < n; i++) { + expr_ty elt = asdl_seq_GET(elts, i); + VISIT(c, expr, elt); + } + if (injected_arg) { + RETURN_IF_ERROR(codegen_nameop(c, loc, injected_arg, Load)); + n++; + } + if (tuple) { + ADDOP_I(c, loc, BUILD_TUPLE, n+pushed); + } else { + ADDOP_I(c, loc, build, n+pushed); + } + return SUCCESS; + } + int sequence_built = 0; + if (big) { + ADDOP_I(c, loc, build, pushed); + sequence_built = 1; + } + for (Py_ssize_t i = 0; i < n; i++) { + expr_ty elt = asdl_seq_GET(elts, i); + if (elt->kind == Starred_kind) { + if (sequence_built == 0) { + ADDOP_I(c, loc, build, i+pushed); + sequence_built = 1; + } + VISIT(c, expr, elt->v.Starred.value); + ADDOP_I(c, loc, extend, 1); + } + else { + VISIT(c, expr, elt); + if (sequence_built) { + ADDOP_I(c, loc, add, 1); + } + } + } + assert(sequence_built); + if (injected_arg) { + RETURN_IF_ERROR(codegen_nameop(c, loc, injected_arg, Load)); + ADDOP_I(c, loc, add, 1); + } + if (tuple) { + ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_LIST_TO_TUPLE); + } + return SUCCESS; +} + +static int +starunpack_helper(compiler *c, location loc, + asdl_expr_seq *elts, int pushed, + int build, int add, int extend, int tuple) +{ + return starunpack_helper_impl(c, loc, elts, NULL, pushed, + build, add, extend, tuple); +} + +static int +unpack_helper(compiler *c, location loc, asdl_expr_seq *elts) +{ + Py_ssize_t n = asdl_seq_LEN(elts); + int seen_star = 0; + for (Py_ssize_t i = 0; i < n; i++) { + expr_ty elt = asdl_seq_GET(elts, i); + if (elt->kind == Starred_kind && !seen_star) { + if ((i >= (1 << 8)) || + (n-i-1 >= (INT_MAX >> 8))) { + return _PyCompile_Error(c, loc, + "too many expressions in " + "star-unpacking assignment"); + } + ADDOP_I(c, loc, UNPACK_EX, (i + ((n-i-1) << 8))); + seen_star = 1; + } + else if (elt->kind == Starred_kind) { + return _PyCompile_Error(c, loc, + "multiple starred expressions in assignment"); + } + } + if (!seen_star) { + ADDOP_I(c, loc, UNPACK_SEQUENCE, n); + } + return SUCCESS; +} + +static int +assignment_helper(compiler *c, location loc, asdl_expr_seq *elts) +{ + Py_ssize_t n = asdl_seq_LEN(elts); + RETURN_IF_ERROR(unpack_helper(c, loc, elts)); + for (Py_ssize_t i = 0; i < n; i++) { + expr_ty elt = asdl_seq_GET(elts, i); + VISIT(c, expr, elt->kind != Starred_kind ? elt : elt->v.Starred.value); + } + return SUCCESS; +} + +static int +codegen_list(compiler *c, expr_ty e) +{ + location loc = LOC(e); + asdl_expr_seq *elts = e->v.List.elts; + if (e->v.List.ctx == Store) { + return assignment_helper(c, loc, elts); + } + else if (e->v.List.ctx == Load) { + return starunpack_helper(c, loc, elts, 0, + BUILD_LIST, LIST_APPEND, LIST_EXTEND, 0); + } + else { + VISIT_SEQ(c, expr, elts); + } + return SUCCESS; +} + +static int +codegen_tuple(compiler *c, expr_ty e) +{ + location loc = LOC(e); + asdl_expr_seq *elts = e->v.Tuple.elts; + if (e->v.Tuple.ctx == Store) { + return assignment_helper(c, loc, elts); + } + else if (e->v.Tuple.ctx == Load) { + return starunpack_helper(c, loc, elts, 0, + BUILD_LIST, LIST_APPEND, LIST_EXTEND, 1); + } + else { + VISIT_SEQ(c, expr, elts); + } + return SUCCESS; +} + +static int +codegen_set(compiler *c, expr_ty e) +{ + location loc = LOC(e); + return starunpack_helper(c, loc, e->v.Set.elts, 0, + BUILD_SET, SET_ADD, SET_UPDATE, 0); +} + +static bool +are_all_items_const(asdl_expr_seq *seq, Py_ssize_t begin, Py_ssize_t end) +{ + for (Py_ssize_t i = begin; i < end; i++) { + expr_ty key = (expr_ty)asdl_seq_GET(seq, i); + if (key == NULL || key->kind != Constant_kind) { + return false; + } + } + return true; +} + +static int +codegen_subdict(compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end) +{ + Py_ssize_t i, n = end - begin; + int big = n*2 > STACK_USE_GUIDELINE; + location loc = LOC(e); + if (big) { + ADDOP_I(c, loc, BUILD_MAP, 0); + } + for (i = begin; i < end; i++) { + VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.keys, i)); + VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i)); + if (big) { + ADDOP_I(c, loc, MAP_ADD, 1); + } + } + if (!big) { + ADDOP_I(c, loc, BUILD_MAP, n); + } + return SUCCESS; +} + +static int +codegen_dict(compiler *c, expr_ty e) +{ + location loc = LOC(e); + Py_ssize_t i, n, elements; + int have_dict; + int is_unpacking = 0; + n = asdl_seq_LEN(e->v.Dict.values); + have_dict = 0; + elements = 0; + for (i = 0; i < n; i++) { + is_unpacking = (expr_ty)asdl_seq_GET(e->v.Dict.keys, i) == NULL; + if (is_unpacking) { + if (elements) { + RETURN_IF_ERROR(codegen_subdict(c, e, i - elements, i)); + if (have_dict) { + ADDOP_I(c, loc, DICT_UPDATE, 1); + } + have_dict = 1; + elements = 0; + } + if (have_dict == 0) { + ADDOP_I(c, loc, BUILD_MAP, 0); + have_dict = 1; + } + VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i)); + ADDOP_I(c, loc, DICT_UPDATE, 1); + } + else { + if (elements*2 > STACK_USE_GUIDELINE) { + RETURN_IF_ERROR(codegen_subdict(c, e, i - elements, i + 1)); + if (have_dict) { + ADDOP_I(c, loc, DICT_UPDATE, 1); + } + have_dict = 1; + elements = 0; + } + else { + elements++; + } + } + } + if (elements) { + RETURN_IF_ERROR(codegen_subdict(c, e, n - elements, n)); + if (have_dict) { + ADDOP_I(c, loc, DICT_UPDATE, 1); + } + have_dict = 1; + } + if (!have_dict) { + ADDOP_I(c, loc, BUILD_MAP, 0); + } + return SUCCESS; +} + +static int +codegen_compare(compiler *c, expr_ty e) +{ + location loc = LOC(e); + Py_ssize_t i, n; + + RETURN_IF_ERROR(codegen_check_compare(c, e)); + VISIT(c, expr, e->v.Compare.left); + assert(asdl_seq_LEN(e->v.Compare.ops) > 0); + n = asdl_seq_LEN(e->v.Compare.ops) - 1; + if (n == 0) { + VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, 0)); + ADDOP_COMPARE(c, loc, asdl_seq_GET(e->v.Compare.ops, 0)); + } + else { + NEW_JUMP_TARGET_LABEL(c, cleanup); + for (i = 0; i < n; i++) { + VISIT(c, expr, + (expr_ty)asdl_seq_GET(e->v.Compare.comparators, i)); + ADDOP_I(c, loc, SWAP, 2); + ADDOP_I(c, loc, COPY, 2); + ADDOP_COMPARE(c, loc, asdl_seq_GET(e->v.Compare.ops, i)); + ADDOP_I(c, loc, COPY, 1); + ADDOP(c, loc, TO_BOOL); + ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, cleanup); + ADDOP(c, loc, POP_TOP); + } + VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, n)); + ADDOP_COMPARE(c, loc, asdl_seq_GET(e->v.Compare.ops, n)); + NEW_JUMP_TARGET_LABEL(c, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); + + USE_LABEL(c, cleanup); + ADDOP_I(c, loc, SWAP, 2); + ADDOP(c, loc, POP_TOP); + + USE_LABEL(c, end); + } + return SUCCESS; +} + +static PyTypeObject * +infer_type(expr_ty e) +{ + switch (e->kind) { + case Tuple_kind: + return &PyTuple_Type; + case List_kind: + case ListComp_kind: + return &PyList_Type; + case Dict_kind: + case DictComp_kind: + return &PyDict_Type; + case Set_kind: + case SetComp_kind: + return &PySet_Type; + case GeneratorExp_kind: + return &PyGen_Type; + case Lambda_kind: + return &PyFunction_Type; + case JoinedStr_kind: + case FormattedValue_kind: + return &PyUnicode_Type; + case Constant_kind: + return Py_TYPE(e->v.Constant.value); + default: + return NULL; + } +} + +static int +check_caller(compiler *c, expr_ty e) +{ + switch (e->kind) { + case Constant_kind: + case Tuple_kind: + case List_kind: + case ListComp_kind: + case Dict_kind: + case DictComp_kind: + case Set_kind: + case SetComp_kind: + case GeneratorExp_kind: + case JoinedStr_kind: + case FormattedValue_kind: { + location loc = LOC(e); + return _PyCompile_Warn(c, loc, "'%.200s' object is not callable; " + "perhaps you missed a comma?", + infer_type(e)->tp_name); + } + default: + return SUCCESS; + } +} + +static int +check_subscripter(compiler *c, expr_ty e) +{ + PyObject *v; + + switch (e->kind) { + case Constant_kind: + v = e->v.Constant.value; + if (!(v == Py_None || v == Py_Ellipsis || + PyLong_Check(v) || PyFloat_Check(v) || PyComplex_Check(v) || + PyAnySet_Check(v))) + { + return SUCCESS; + } + _Py_FALLTHROUGH; + case Set_kind: + case SetComp_kind: + case GeneratorExp_kind: + case Lambda_kind: { + location loc = LOC(e); + return _PyCompile_Warn(c, loc, "'%.200s' object is not subscriptable; " + "perhaps you missed a comma?", + infer_type(e)->tp_name); + } + default: + return SUCCESS; + } +} + +static int +check_index(compiler *c, expr_ty e, expr_ty s) +{ + PyObject *v; + + PyTypeObject *index_type = infer_type(s); + if (index_type == NULL + || PyType_FastSubclass(index_type, Py_TPFLAGS_LONG_SUBCLASS) + || index_type == &PySlice_Type) { + return SUCCESS; + } + + switch (e->kind) { + case Constant_kind: + v = e->v.Constant.value; + if (!(PyUnicode_Check(v) || PyBytes_Check(v) || PyTuple_Check(v))) { + return SUCCESS; + } + _Py_FALLTHROUGH; + case Tuple_kind: + case List_kind: + case ListComp_kind: + case JoinedStr_kind: + case FormattedValue_kind: { + location loc = LOC(e); + return _PyCompile_Warn(c, loc, "%.200s indices must be integers " + "or slices, not %.200s; " + "perhaps you missed a comma?", + infer_type(e)->tp_name, + index_type->tp_name); + } + default: + return SUCCESS; + } +} + +static int +is_import_originated(compiler *c, expr_ty e) +{ + /* Check whether the global scope has an import named + e, if it is a Name object. For not traversing all the + scope stack every time this function is called, it will + only check the global scope to determine whether something + is imported or not. */ + + if (e->kind != Name_kind) { + return 0; + } + + long flags = _PyST_GetSymbol(SYMTABLE(c)->st_top, e->v.Name.id); + RETURN_IF_ERROR(flags); + return flags & DEF_IMPORT; +} + +static int +can_optimize_super_call(compiler *c, expr_ty attr) +{ + expr_ty e = attr->v.Attribute.value; + if (e->kind != Call_kind || + e->v.Call.func->kind != Name_kind || + !_PyUnicode_EqualToASCIIString(e->v.Call.func->v.Name.id, "super") || + _PyUnicode_EqualToASCIIString(attr->v.Attribute.attr, "__class__") || + asdl_seq_LEN(e->v.Call.keywords) != 0) { + return 0; + } + Py_ssize_t num_args = asdl_seq_LEN(e->v.Call.args); + + PyObject *super_name = e->v.Call.func->v.Name.id; + // detect statically-visible shadowing of 'super' name + int scope = _PyST_GetScope(SYMTABLE_ENTRY(c), super_name); + RETURN_IF_ERROR(scope); + if (scope != GLOBAL_IMPLICIT) { + return 0; + } + scope = _PyST_GetScope(SYMTABLE(c)->st_top, super_name); + RETURN_IF_ERROR(scope); + if (scope != 0) { + return 0; + } + + if (num_args == 2) { + for (Py_ssize_t i = 0; i < num_args; i++) { + expr_ty elt = asdl_seq_GET(e->v.Call.args, i); + if (elt->kind == Starred_kind) { + return 0; + } + } + // exactly two non-starred args; we can just load + // the provided args + return 1; + } + + if (num_args != 0) { + return 0; + } + // we need the following for zero-arg super(): + + // enclosing function should have at least one argument + if (METADATA(c)->u_argcount == 0 && + METADATA(c)->u_posonlyargcount == 0) { + return 0; + } + // __class__ cell should be available + if (_PyCompile_GetRefType(c, &_Py_ID(__class__)) == FREE) { + return 1; + } + return 0; +} + +static int +load_args_for_super(compiler *c, expr_ty e) { + location loc = LOC(e); + + // load super() global + PyObject *super_name = e->v.Call.func->v.Name.id; + RETURN_IF_ERROR(codegen_nameop(c, LOC(e->v.Call.func), super_name, Load)); + + if (asdl_seq_LEN(e->v.Call.args) == 2) { + VISIT(c, expr, asdl_seq_GET(e->v.Call.args, 0)); + VISIT(c, expr, asdl_seq_GET(e->v.Call.args, 1)); + return SUCCESS; + } + + // load __class__ cell + PyObject *name = &_Py_ID(__class__); + assert(_PyCompile_GetRefType(c, name) == FREE); + RETURN_IF_ERROR(codegen_nameop(c, loc, name, Load)); + + // load self (first argument) + Py_ssize_t i = 0; + PyObject *key, *value; + if (!PyDict_Next(METADATA(c)->u_varnames, &i, &key, &value)) { + return ERROR; + } + RETURN_IF_ERROR(codegen_nameop(c, loc, key, Load)); + + return SUCCESS; +} + +// If an attribute access spans multiple lines, update the current start +// location to point to the attribute name. +static location +update_start_location_to_match_attr(compiler *c, location loc, + expr_ty attr) +{ + assert(attr->kind == Attribute_kind); + if (loc.lineno != attr->end_lineno) { + loc.lineno = attr->end_lineno; + int len = (int)PyUnicode_GET_LENGTH(attr->v.Attribute.attr); + if (len <= attr->end_col_offset) { + loc.col_offset = attr->end_col_offset - len; + } + else { + // GH-94694: Somebody's compiling weird ASTs. Just drop the columns: + loc.col_offset = -1; + loc.end_col_offset = -1; + } + // Make sure the end position still follows the start position, even for + // weird ASTs: + loc.end_lineno = Py_MAX(loc.lineno, loc.end_lineno); + if (loc.lineno == loc.end_lineno) { + loc.end_col_offset = Py_MAX(loc.col_offset, loc.end_col_offset); + } + } + return loc; +} + +// Return 1 if the method call was optimized, 0 if not, and -1 on error. +static int +maybe_optimize_method_call(compiler *c, expr_ty e) +{ + Py_ssize_t argsl, i, kwdsl; + expr_ty meth = e->v.Call.func; + asdl_expr_seq *args = e->v.Call.args; + asdl_keyword_seq *kwds = e->v.Call.keywords; + + /* Check that the call node is an attribute access */ + if (meth->kind != Attribute_kind || meth->v.Attribute.ctx != Load) { + return 0; + } + + /* Check that the base object is not something that is imported */ + int ret = is_import_originated(c, meth->v.Attribute.value); + RETURN_IF_ERROR(ret); + if (ret) { + return 0; + } + + /* Check that there aren't too many arguments */ + argsl = asdl_seq_LEN(args); + kwdsl = asdl_seq_LEN(kwds); + if (argsl + kwdsl + (kwdsl != 0) >= STACK_USE_GUIDELINE) { + return 0; + } + /* Check that there are no *varargs types of arguments. */ + for (i = 0; i < argsl; i++) { + expr_ty elt = asdl_seq_GET(args, i); + if (elt->kind == Starred_kind) { + return 0; + } + } + + for (i = 0; i < kwdsl; i++) { + keyword_ty kw = asdl_seq_GET(kwds, i); + if (kw->arg == NULL) { + return 0; + } + } + + /* Alright, we can optimize the code. */ + location loc = LOC(meth); + + ret = can_optimize_super_call(c, meth); + RETURN_IF_ERROR(ret); + if (ret) { + RETURN_IF_ERROR(load_args_for_super(c, meth->v.Attribute.value)); + int opcode = asdl_seq_LEN(meth->v.Attribute.value->v.Call.args) ? + LOAD_SUPER_METHOD : LOAD_ZERO_SUPER_METHOD; + ADDOP_NAME(c, loc, opcode, meth->v.Attribute.attr, names); + loc = update_start_location_to_match_attr(c, loc, meth); + ADDOP(c, loc, NOP); + } else { + VISIT(c, expr, meth->v.Attribute.value); + loc = update_start_location_to_match_attr(c, loc, meth); + ADDOP_NAME(c, loc, LOAD_METHOD, meth->v.Attribute.attr, names); + } + + VISIT_SEQ(c, expr, e->v.Call.args); + + if (kwdsl) { + VISIT_SEQ(c, keyword, kwds); + RETURN_IF_ERROR( + codegen_call_simple_kw_helper(c, loc, kwds, kwdsl)); + loc = update_start_location_to_match_attr(c, LOC(e), meth); + ADDOP_I(c, loc, CALL_KW, argsl + kwdsl); + } + else { + loc = update_start_location_to_match_attr(c, LOC(e), meth); + ADDOP_I(c, loc, CALL, argsl); + } + return 1; +} + +static int +codegen_validate_keywords(compiler *c, asdl_keyword_seq *keywords) +{ + Py_ssize_t nkeywords = asdl_seq_LEN(keywords); + for (Py_ssize_t i = 0; i < nkeywords; i++) { + keyword_ty key = ((keyword_ty)asdl_seq_GET(keywords, i)); + if (key->arg == NULL) { + continue; + } + for (Py_ssize_t j = i + 1; j < nkeywords; j++) { + keyword_ty other = ((keyword_ty)asdl_seq_GET(keywords, j)); + if (other->arg && !PyUnicode_Compare(key->arg, other->arg)) { + return _PyCompile_Error(c, LOC(other), "keyword argument repeated: %U", key->arg); + } + } + } + return SUCCESS; +} + +static int +codegen_call(compiler *c, expr_ty e) +{ + RETURN_IF_ERROR(codegen_validate_keywords(c, e->v.Call.keywords)); + int ret = maybe_optimize_method_call(c, e); + if (ret < 0) { + return ERROR; + } + if (ret == 1) { + return SUCCESS; + } + RETURN_IF_ERROR(check_caller(c, e->v.Call.func)); + VISIT(c, expr, e->v.Call.func); + location loc = LOC(e->v.Call.func); + ADDOP(c, loc, PUSH_NULL); + loc = LOC(e); + return codegen_call_helper(c, loc, 0, + e->v.Call.args, + e->v.Call.keywords); +} + +static int +codegen_joined_str(compiler *c, expr_ty e) +{ + location loc = LOC(e); + Py_ssize_t value_count = asdl_seq_LEN(e->v.JoinedStr.values); + if (value_count > STACK_USE_GUIDELINE) { + _Py_DECLARE_STR(empty, ""); + ADDOP_LOAD_CONST_NEW(c, loc, Py_NewRef(&_Py_STR(empty))); + ADDOP_NAME(c, loc, LOAD_METHOD, &_Py_ID(join), names); + ADDOP_I(c, loc, BUILD_LIST, 0); + for (Py_ssize_t i = 0; i < asdl_seq_LEN(e->v.JoinedStr.values); i++) { + VISIT(c, expr, asdl_seq_GET(e->v.JoinedStr.values, i)); + ADDOP_I(c, loc, LIST_APPEND, 1); + } + ADDOP_I(c, loc, CALL, 1); + } + else { + VISIT_SEQ(c, expr, e->v.JoinedStr.values); + if (value_count > 1) { + ADDOP_I(c, loc, BUILD_STRING, value_count); + } + else if (value_count == 0) { + _Py_DECLARE_STR(empty, ""); + ADDOP_LOAD_CONST_NEW(c, loc, Py_NewRef(&_Py_STR(empty))); + } + } + return SUCCESS; +} + +/* Used to implement f-strings. Format a single value. */ +static int +codegen_formatted_value(compiler *c, expr_ty e) +{ + /* Our oparg encodes 2 pieces of information: the conversion + character, and whether or not a format_spec was provided. + + Convert the conversion char to 3 bits: + : 000 0x0 FVC_NONE The default if nothing specified. + !s : 001 0x1 FVC_STR + !r : 010 0x2 FVC_REPR + !a : 011 0x3 FVC_ASCII + + next bit is whether or not we have a format spec: + yes : 100 0x4 + no : 000 0x0 + */ + + int conversion = e->v.FormattedValue.conversion; + int oparg; + + /* The expression to be formatted. */ + VISIT(c, expr, e->v.FormattedValue.value); + + location loc = LOC(e); + if (conversion != -1) { + switch (conversion) { + case 's': oparg = FVC_STR; break; + case 'r': oparg = FVC_REPR; break; + case 'a': oparg = FVC_ASCII; break; + default: + PyErr_Format(PyExc_SystemError, + "Unrecognized conversion character %d", conversion); + return ERROR; + } + ADDOP_I(c, loc, CONVERT_VALUE, oparg); + } + if (e->v.FormattedValue.format_spec) { + /* Evaluate the format spec, and update our opcode arg. */ + VISIT(c, expr, e->v.FormattedValue.format_spec); + ADDOP(c, loc, FORMAT_WITH_SPEC); + } else { + ADDOP(c, loc, FORMAT_SIMPLE); + } + return SUCCESS; +} + +static int +codegen_subkwargs(compiler *c, location loc, + asdl_keyword_seq *keywords, + Py_ssize_t begin, Py_ssize_t end) +{ + Py_ssize_t i, n = end - begin; + keyword_ty kw; + assert(n > 0); + int big = n*2 > STACK_USE_GUIDELINE; + if (big) { + ADDOP_I(c, NO_LOCATION, BUILD_MAP, 0); + } + for (i = begin; i < end; i++) { + kw = asdl_seq_GET(keywords, i); + ADDOP_LOAD_CONST(c, loc, kw->arg); + VISIT(c, expr, kw->value); + if (big) { + ADDOP_I(c, NO_LOCATION, MAP_ADD, 1); + } + } + if (!big) { + ADDOP_I(c, loc, BUILD_MAP, n); + } + return SUCCESS; +} + +/* Used by codegen_call_helper and maybe_optimize_method_call to emit + * a tuple of keyword names before CALL. + */ +static int +codegen_call_simple_kw_helper(compiler *c, location loc, + asdl_keyword_seq *keywords, Py_ssize_t nkwelts) +{ + PyObject *names; + names = PyTuple_New(nkwelts); + if (names == NULL) { + return ERROR; + } + for (Py_ssize_t i = 0; i < nkwelts; i++) { + keyword_ty kw = asdl_seq_GET(keywords, i); + PyTuple_SET_ITEM(names, i, Py_NewRef(kw->arg)); + } + ADDOP_LOAD_CONST_NEW(c, loc, names); + return SUCCESS; +} + +/* shared code between codegen_call and codegen_class */ +static int +codegen_call_helper_impl(compiler *c, location loc, + int n, /* Args already pushed */ + asdl_expr_seq *args, + PyObject *injected_arg, + asdl_keyword_seq *keywords) +{ + Py_ssize_t i, nseen, nelts, nkwelts; + + RETURN_IF_ERROR(codegen_validate_keywords(c, keywords)); + + nelts = asdl_seq_LEN(args); + nkwelts = asdl_seq_LEN(keywords); + + if (nelts + nkwelts*2 > STACK_USE_GUIDELINE) { + goto ex_call; + } + for (i = 0; i < nelts; i++) { + expr_ty elt = asdl_seq_GET(args, i); + if (elt->kind == Starred_kind) { + goto ex_call; + } + } + for (i = 0; i < nkwelts; i++) { + keyword_ty kw = asdl_seq_GET(keywords, i); + if (kw->arg == NULL) { + goto ex_call; + } + } + + /* No * or ** args, so can use faster calling sequence */ + for (i = 0; i < nelts; i++) { + expr_ty elt = asdl_seq_GET(args, i); + assert(elt->kind != Starred_kind); + VISIT(c, expr, elt); + } + if (injected_arg) { + RETURN_IF_ERROR(codegen_nameop(c, loc, injected_arg, Load)); + nelts++; + } + if (nkwelts) { + VISIT_SEQ(c, keyword, keywords); + RETURN_IF_ERROR( + codegen_call_simple_kw_helper(c, loc, keywords, nkwelts)); + ADDOP_I(c, loc, CALL_KW, n + nelts + nkwelts); + } + else { + ADDOP_I(c, loc, CALL, n + nelts); + } + return SUCCESS; + +ex_call: + + /* Do positional arguments. */ + if (n == 0 && nelts == 1 && ((expr_ty)asdl_seq_GET(args, 0))->kind == Starred_kind) { + VISIT(c, expr, ((expr_ty)asdl_seq_GET(args, 0))->v.Starred.value); + } + else { + RETURN_IF_ERROR(starunpack_helper_impl(c, loc, args, injected_arg, n, + BUILD_LIST, LIST_APPEND, LIST_EXTEND, 1)); + } + /* Then keyword arguments */ + if (nkwelts) { + /* Has a new dict been pushed */ + int have_dict = 0; + + nseen = 0; /* the number of keyword arguments on the stack following */ + for (i = 0; i < nkwelts; i++) { + keyword_ty kw = asdl_seq_GET(keywords, i); + if (kw->arg == NULL) { + /* A keyword argument unpacking. */ + if (nseen) { + RETURN_IF_ERROR(codegen_subkwargs(c, loc, keywords, i - nseen, i)); + if (have_dict) { + ADDOP_I(c, loc, DICT_MERGE, 1); + } + have_dict = 1; + nseen = 0; + } + if (!have_dict) { + ADDOP_I(c, loc, BUILD_MAP, 0); + have_dict = 1; + } + VISIT(c, expr, kw->value); + ADDOP_I(c, loc, DICT_MERGE, 1); + } + else { + nseen++; + } + } + if (nseen) { + /* Pack up any trailing keyword arguments. */ + RETURN_IF_ERROR(codegen_subkwargs(c, loc, keywords, nkwelts - nseen, nkwelts)); + if (have_dict) { + ADDOP_I(c, loc, DICT_MERGE, 1); + } + have_dict = 1; + } + assert(have_dict); + } + if (nkwelts == 0) { + ADDOP(c, loc, PUSH_NULL); + } + ADDOP(c, loc, CALL_FUNCTION_EX); + return SUCCESS; +} + +static int +codegen_call_helper(compiler *c, location loc, + int n, /* Args already pushed */ + asdl_expr_seq *args, + asdl_keyword_seq *keywords) +{ + return codegen_call_helper_impl(c, loc, n, args, NULL, keywords); +} + +/* List and set comprehensions work by being inlined at the location where + they are defined. The isolation of iteration variables is provided by + pushing/popping clashing locals on the stack. Generator expressions work + by creating a nested function to perform the actual iteration. + This means that the iteration variables don't leak into the current scope. + See https://peps.python.org/pep-0709/ for additional information. + The defined function is called immediately following its definition, with the + result of that call being the result of the expression. + The LC/SC version returns the populated container, while the GE version is + flagged in symtable.c as a generator, so it returns the generator object + when the function is called. + + Possible cleanups: + - iterate over the generator sequence instead of using recursion +*/ + + +static int +codegen_comprehension_generator(compiler *c, location loc, + asdl_comprehension_seq *generators, int gen_index, + int depth, + expr_ty elt, expr_ty val, int type, + int iter_on_stack) +{ + comprehension_ty gen; + gen = (comprehension_ty)asdl_seq_GET(generators, gen_index); + if (gen->is_async) { + return codegen_async_comprehension_generator( + c, loc, generators, gen_index, depth, elt, val, type, + iter_on_stack); + } else { + return codegen_sync_comprehension_generator( + c, loc, generators, gen_index, depth, elt, val, type, + iter_on_stack); + } +} + +static int +codegen_sync_comprehension_generator(compiler *c, location loc, + asdl_comprehension_seq *generators, + int gen_index, int depth, + expr_ty elt, expr_ty val, int type, + int iter_on_stack) +{ + /* generate code for the iterator, then each of the ifs, + and then write to the element */ + + NEW_JUMP_TARGET_LABEL(c, start); + NEW_JUMP_TARGET_LABEL(c, if_cleanup); + NEW_JUMP_TARGET_LABEL(c, anchor); + + comprehension_ty gen = (comprehension_ty)asdl_seq_GET(generators, + gen_index); + + if (!iter_on_stack) { + if (gen_index == 0) { + assert(METADATA(c)->u_argcount == 1); + ADDOP_I(c, loc, LOAD_FAST, 0); + } + else { + /* Sub-iter - calculate on the fly */ + /* Fast path for the temporary variable assignment idiom: + for y in [f(x)] + */ + asdl_expr_seq *elts; + switch (gen->iter->kind) { + case List_kind: + elts = gen->iter->v.List.elts; + break; + case Tuple_kind: + elts = gen->iter->v.Tuple.elts; + break; + default: + elts = NULL; + } + if (asdl_seq_LEN(elts) == 1) { + expr_ty elt = asdl_seq_GET(elts, 0); + if (elt->kind != Starred_kind) { + VISIT(c, expr, elt); + start = NO_LABEL; + } + } + if (IS_JUMP_TARGET_LABEL(start)) { + VISIT(c, expr, gen->iter); + ADDOP(c, LOC(gen->iter), GET_ITER); + } + } + } + + if (IS_JUMP_TARGET_LABEL(start)) { + depth++; + ADDOP(c, LOC(gen->iter), GET_ITER); + USE_LABEL(c, start); + ADDOP_JUMP(c, LOC(gen->iter), FOR_ITER, anchor); + } + VISIT(c, expr, gen->target); + + /* XXX this needs to be cleaned up...a lot! */ + Py_ssize_t n = asdl_seq_LEN(gen->ifs); + for (Py_ssize_t i = 0; i < n; i++) { + expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i); + RETURN_IF_ERROR(codegen_jump_if(c, loc, e, if_cleanup, 0)); + } + + if (++gen_index < asdl_seq_LEN(generators)) { + RETURN_IF_ERROR( + codegen_comprehension_generator(c, loc, + generators, gen_index, depth, + elt, val, type, 0)); + } + + location elt_loc = LOC(elt); + + /* only append after the last for generator */ + if (gen_index >= asdl_seq_LEN(generators)) { + /* comprehension specific code */ + switch (type) { + case COMP_GENEXP: + VISIT(c, expr, elt); + ADDOP_YIELD(c, elt_loc); + ADDOP(c, elt_loc, POP_TOP); + break; + case COMP_LISTCOMP: + VISIT(c, expr, elt); + ADDOP_I(c, elt_loc, LIST_APPEND, depth + 1); + break; + case COMP_SETCOMP: + VISIT(c, expr, elt); + ADDOP_I(c, elt_loc, SET_ADD, depth + 1); + break; + case COMP_DICTCOMP: + /* With '{k: v}', k is evaluated before v, so we do + the same. */ + VISIT(c, expr, elt); + VISIT(c, expr, val); + elt_loc = LOCATION(elt->lineno, + val->end_lineno, + elt->col_offset, + val->end_col_offset); + ADDOP_I(c, elt_loc, MAP_ADD, depth + 1); + break; + default: + return ERROR; + } + } + + USE_LABEL(c, if_cleanup); + if (IS_JUMP_TARGET_LABEL(start)) { + ADDOP_JUMP(c, elt_loc, JUMP, start); + + USE_LABEL(c, anchor); + /* It is important for instrumentation that the `END_FOR` comes first. + * Iteration over a generator will jump to the first of these instructions, + * but a non-generator will jump to a later instruction. + */ + ADDOP(c, NO_LOCATION, END_FOR); + ADDOP(c, NO_LOCATION, POP_ITER); + } + + return SUCCESS; +} + +static int +codegen_async_comprehension_generator(compiler *c, location loc, + asdl_comprehension_seq *generators, + int gen_index, int depth, + expr_ty elt, expr_ty val, int type, + int iter_on_stack) +{ + NEW_JUMP_TARGET_LABEL(c, start); + NEW_JUMP_TARGET_LABEL(c, except); + NEW_JUMP_TARGET_LABEL(c, if_cleanup); + + comprehension_ty gen = (comprehension_ty)asdl_seq_GET(generators, + gen_index); + + if (!iter_on_stack) { + if (gen_index == 0) { + assert(METADATA(c)->u_argcount == 1); + ADDOP_I(c, loc, LOAD_FAST, 0); + } + else { + /* Sub-iter - calculate on the fly */ + VISIT(c, expr, gen->iter); + ADDOP(c, LOC(gen->iter), GET_AITER); + } + } + + USE_LABEL(c, start); + /* Runtime will push a block here, so we need to account for that */ + RETURN_IF_ERROR( + _PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_ASYNC_COMPREHENSION_GENERATOR, + start, NO_LABEL, NULL)); + + ADDOP_JUMP(c, loc, SETUP_FINALLY, except); + ADDOP(c, loc, GET_ANEXT); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADD_YIELD_FROM(c, loc, 1); + ADDOP(c, loc, POP_BLOCK); + VISIT(c, expr, gen->target); + + Py_ssize_t n = asdl_seq_LEN(gen->ifs); + for (Py_ssize_t i = 0; i < n; i++) { + expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i); + RETURN_IF_ERROR(codegen_jump_if(c, loc, e, if_cleanup, 0)); + } + + depth++; + if (++gen_index < asdl_seq_LEN(generators)) { + RETURN_IF_ERROR( + codegen_comprehension_generator(c, loc, + generators, gen_index, depth, + elt, val, type, 0)); + } + + location elt_loc = LOC(elt); + /* only append after the last for generator */ + if (gen_index >= asdl_seq_LEN(generators)) { + /* comprehension specific code */ + switch (type) { + case COMP_GENEXP: + VISIT(c, expr, elt); + ADDOP_YIELD(c, elt_loc); + ADDOP(c, elt_loc, POP_TOP); + break; + case COMP_LISTCOMP: + VISIT(c, expr, elt); + ADDOP_I(c, elt_loc, LIST_APPEND, depth + 1); + break; + case COMP_SETCOMP: + VISIT(c, expr, elt); + ADDOP_I(c, elt_loc, SET_ADD, depth + 1); + break; + case COMP_DICTCOMP: + /* With '{k: v}', k is evaluated before v, so we do + the same. */ + VISIT(c, expr, elt); + VISIT(c, expr, val); + elt_loc = LOCATION(elt->lineno, + val->end_lineno, + elt->col_offset, + val->end_col_offset); + ADDOP_I(c, elt_loc, MAP_ADD, depth + 1); + break; + default: + return ERROR; + } + } + + USE_LABEL(c, if_cleanup); + ADDOP_JUMP(c, elt_loc, JUMP, start); + + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_ASYNC_COMPREHENSION_GENERATOR, start); + + USE_LABEL(c, except); + + ADDOP(c, loc, END_ASYNC_FOR); + + return SUCCESS; +} + +static int +codegen_push_inlined_comprehension_locals(compiler *c, location loc, + PySTEntryObject *comp, + _PyCompile_InlinedComprehensionState *state) +{ + int in_class_block = (SYMTABLE_ENTRY(c)->ste_type == ClassBlock) && + !_PyCompile_IsInInlinedComp(c); + PySTEntryObject *outer = SYMTABLE_ENTRY(c); + // iterate over names bound in the comprehension and ensure we isolate + // them from the outer scope as needed + PyObject *k, *v; + Py_ssize_t pos = 0; + while (PyDict_Next(comp->ste_symbols, &pos, &k, &v)) { + long symbol = PyLong_AsLong(v); + assert(symbol >= 0 || PyErr_Occurred()); + RETURN_IF_ERROR(symbol); + long scope = SYMBOL_TO_SCOPE(symbol); + + long outsymbol = _PyST_GetSymbol(outer, k); + RETURN_IF_ERROR(outsymbol); + long outsc = SYMBOL_TO_SCOPE(outsymbol); + + if ((symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) || in_class_block) { + // local names bound in comprehension must be isolated from + // outer scope; push existing value (which may be NULL if + // not defined) on stack + if (state->pushed_locals == NULL) { + state->pushed_locals = PyList_New(0); + if (state->pushed_locals == NULL) { + return ERROR; + } + } + // in the case of a cell, this will actually push the cell + // itself to the stack, then we'll create a new one for the + // comprehension and restore the original one after + ADDOP_NAME(c, loc, LOAD_FAST_AND_CLEAR, k, varnames); + if (scope == CELL) { + if (outsc == FREE) { + ADDOP_NAME(c, loc, MAKE_CELL, k, freevars); + } else { + ADDOP_NAME(c, loc, MAKE_CELL, k, cellvars); + } + } + if (PyList_Append(state->pushed_locals, k) < 0) { + return ERROR; + } + } + } + if (state->pushed_locals) { + // Outermost iterable expression was already evaluated and is on the + // stack, we need to swap it back to TOS. This also rotates the order of + // `pushed_locals` on the stack, but this will be reversed when we swap + // out the comprehension result in pop_inlined_comprehension_state + ADDOP_I(c, loc, SWAP, PyList_GET_SIZE(state->pushed_locals) + 1); + + // Add our own cleanup handler to restore comprehension locals in case + // of exception, so they have the correct values inside an exception + // handler or finally block. + NEW_JUMP_TARGET_LABEL(c, cleanup); + state->cleanup = cleanup; + + // no need to push an fblock for this "virtual" try/finally; there can't + // be return/continue/break inside a comprehension + ADDOP_JUMP(c, loc, SETUP_FINALLY, cleanup); + } + return SUCCESS; +} + +static int +push_inlined_comprehension_state(compiler *c, location loc, + PySTEntryObject *comp, + _PyCompile_InlinedComprehensionState *state) +{ + RETURN_IF_ERROR( + _PyCompile_TweakInlinedComprehensionScopes(c, loc, comp, state)); + RETURN_IF_ERROR( + codegen_push_inlined_comprehension_locals(c, loc, comp, state)); + return SUCCESS; +} + +static int +restore_inlined_comprehension_locals(compiler *c, location loc, + _PyCompile_InlinedComprehensionState *state) +{ + PyObject *k; + // pop names we pushed to stack earlier + Py_ssize_t npops = PyList_GET_SIZE(state->pushed_locals); + // Preserve the comprehension result (or exception) as TOS. This + // reverses the SWAP we did in push_inlined_comprehension_state + // to get the outermost iterable to TOS, so we can still just iterate + // pushed_locals in simple reverse order + ADDOP_I(c, loc, SWAP, npops + 1); + for (Py_ssize_t i = npops - 1; i >= 0; --i) { + k = PyList_GetItem(state->pushed_locals, i); + if (k == NULL) { + return ERROR; + } + ADDOP_NAME(c, loc, STORE_FAST_MAYBE_NULL, k, varnames); + } + return SUCCESS; +} + +static int +codegen_pop_inlined_comprehension_locals(compiler *c, location loc, + _PyCompile_InlinedComprehensionState *state) +{ + if (state->pushed_locals) { + ADDOP(c, NO_LOCATION, POP_BLOCK); + + NEW_JUMP_TARGET_LABEL(c, end); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); + + // cleanup from an exception inside the comprehension + USE_LABEL(c, state->cleanup); + // discard incomplete comprehension result (beneath exc on stack) + ADDOP_I(c, NO_LOCATION, SWAP, 2); + ADDOP(c, NO_LOCATION, POP_TOP); + RETURN_IF_ERROR(restore_inlined_comprehension_locals(c, loc, state)); + ADDOP_I(c, NO_LOCATION, RERAISE, 0); + + USE_LABEL(c, end); + RETURN_IF_ERROR(restore_inlined_comprehension_locals(c, loc, state)); + Py_CLEAR(state->pushed_locals); + } + return SUCCESS; +} + +static int +pop_inlined_comprehension_state(compiler *c, location loc, + _PyCompile_InlinedComprehensionState *state) +{ + RETURN_IF_ERROR(codegen_pop_inlined_comprehension_locals(c, loc, state)); + RETURN_IF_ERROR(_PyCompile_RevertInlinedComprehensionScopes(c, loc, state)); + return SUCCESS; +} + +static inline int +codegen_comprehension_iter(compiler *c, comprehension_ty comp) +{ + VISIT(c, expr, comp->iter); + if (comp->is_async) { + ADDOP(c, LOC(comp->iter), GET_AITER); + } + else { + ADDOP(c, LOC(comp->iter), GET_ITER); + } + return SUCCESS; +} + +static int +codegen_comprehension(compiler *c, expr_ty e, int type, + identifier name, asdl_comprehension_seq *generators, expr_ty elt, + expr_ty val) +{ + PyCodeObject *co = NULL; + _PyCompile_InlinedComprehensionState inline_state = {NULL, NULL, NULL, NO_LABEL}; + comprehension_ty outermost; + PySTEntryObject *entry = _PySymtable_Lookup(SYMTABLE(c), (void *)e); + if (entry == NULL) { + goto error; + } + int is_inlined = entry->ste_comp_inlined; + int is_async_comprehension = entry->ste_coroutine; + + location loc = LOC(e); + + outermost = (comprehension_ty) asdl_seq_GET(generators, 0); + if (is_inlined) { + if (codegen_comprehension_iter(c, outermost)) { + goto error; + } + if (push_inlined_comprehension_state(c, loc, entry, &inline_state)) { + goto error; + } + } + else { + /* Receive outermost iter as an implicit argument */ + _PyCompile_CodeUnitMetadata umd = { + .u_argcount = 1, + }; + if (codegen_enter_scope(c, name, COMPILE_SCOPE_COMPREHENSION, + (void *)e, e->lineno, NULL, &umd) < 0) { + goto error; + } + } + Py_CLEAR(entry); + + if (type != COMP_GENEXP) { + int op; + switch (type) { + case COMP_LISTCOMP: + op = BUILD_LIST; + break; + case COMP_SETCOMP: + op = BUILD_SET; + break; + case COMP_DICTCOMP: + op = BUILD_MAP; + break; + default: + PyErr_Format(PyExc_SystemError, + "unknown comprehension type %d", type); + goto error_in_scope; + } + + ADDOP_I(c, loc, op, 0); + if (is_inlined) { + ADDOP_I(c, loc, SWAP, 2); + } + } + + if (codegen_comprehension_generator(c, loc, generators, 0, 0, + elt, val, type, is_inlined) < 0) { + goto error_in_scope; + } + + if (is_inlined) { + if (pop_inlined_comprehension_state(c, loc, &inline_state)) { + goto error; + } + return SUCCESS; + } + + if (type != COMP_GENEXP) { + ADDOP(c, LOC(e), RETURN_VALUE); + } + if (type == COMP_GENEXP) { + if (codegen_wrap_in_stopiteration_handler(c) < 0) { + goto error_in_scope; + } + } + + co = _PyCompile_OptimizeAndAssemble(c, 1); + _PyCompile_ExitScope(c); + if (co == NULL) { + goto error; + } + + loc = LOC(e); + if (codegen_make_closure(c, loc, co, 0) < 0) { + goto error; + } + Py_CLEAR(co); + + if (codegen_comprehension_iter(c, outermost)) { + goto error; + } + + ADDOP_I(c, loc, CALL, 0); + + if (is_async_comprehension && type != COMP_GENEXP) { + ADDOP_I(c, loc, GET_AWAITABLE, 0); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADD_YIELD_FROM(c, loc, 1); + } + + return SUCCESS; +error_in_scope: + if (!is_inlined) { + _PyCompile_ExitScope(c); + } +error: + Py_XDECREF(co); + Py_XDECREF(entry); + Py_XDECREF(inline_state.pushed_locals); + Py_XDECREF(inline_state.temp_symbols); + Py_XDECREF(inline_state.fast_hidden); + return ERROR; +} + +static int +codegen_genexp(compiler *c, expr_ty e) +{ + assert(e->kind == GeneratorExp_kind); + _Py_DECLARE_STR(anon_genexpr, ""); + return codegen_comprehension(c, e, COMP_GENEXP, &_Py_STR(anon_genexpr), + e->v.GeneratorExp.generators, + e->v.GeneratorExp.elt, NULL); +} + +static int +codegen_listcomp(compiler *c, expr_ty e) +{ + assert(e->kind == ListComp_kind); + _Py_DECLARE_STR(anon_listcomp, ""); + return codegen_comprehension(c, e, COMP_LISTCOMP, &_Py_STR(anon_listcomp), + e->v.ListComp.generators, + e->v.ListComp.elt, NULL); +} + +static int +codegen_setcomp(compiler *c, expr_ty e) +{ + assert(e->kind == SetComp_kind); + _Py_DECLARE_STR(anon_setcomp, ""); + return codegen_comprehension(c, e, COMP_SETCOMP, &_Py_STR(anon_setcomp), + e->v.SetComp.generators, + e->v.SetComp.elt, NULL); +} + + +static int +codegen_dictcomp(compiler *c, expr_ty e) +{ + assert(e->kind == DictComp_kind); + _Py_DECLARE_STR(anon_dictcomp, ""); + return codegen_comprehension(c, e, COMP_DICTCOMP, &_Py_STR(anon_dictcomp), + e->v.DictComp.generators, + e->v.DictComp.key, e->v.DictComp.value); +} + + +static int +codegen_visit_keyword(compiler *c, keyword_ty k) +{ + VISIT(c, expr, k->value); + return SUCCESS; +} + + +static int +codegen_with_except_finish(compiler *c, jump_target_label cleanup) { + NEW_JUMP_TARGET_LABEL(c, suppress); + ADDOP(c, NO_LOCATION, TO_BOOL); + ADDOP_JUMP(c, NO_LOCATION, POP_JUMP_IF_TRUE, suppress); + ADDOP_I(c, NO_LOCATION, RERAISE, 2); + + USE_LABEL(c, suppress); + ADDOP(c, NO_LOCATION, POP_TOP); /* exc_value */ + ADDOP(c, NO_LOCATION, POP_BLOCK); + ADDOP(c, NO_LOCATION, POP_EXCEPT); + ADDOP(c, NO_LOCATION, POP_TOP); + ADDOP(c, NO_LOCATION, POP_TOP); + ADDOP(c, NO_LOCATION, POP_TOP); + NEW_JUMP_TARGET_LABEL(c, exit); + ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, exit); + + USE_LABEL(c, cleanup); + POP_EXCEPT_AND_RERAISE(c, NO_LOCATION); + + USE_LABEL(c, exit); + return SUCCESS; +} + +/* + Implements the async with statement. + + The semantics outlined in that PEP are as follows: + + async with EXPR as VAR: + BLOCK + + It is implemented roughly as: + + context = EXPR + exit = context.__aexit__ # not calling it + value = await context.__aenter__() + try: + VAR = value # if VAR present in the syntax + BLOCK + finally: + if an exception was raised: + exc = copy of (exception, instance, traceback) + else: + exc = (None, None, None) + if not (await exit(*exc)): + raise + */ +static int +codegen_async_with(compiler *c, stmt_ty s, int pos) +{ + location loc = LOC(s); + withitem_ty item = asdl_seq_GET(s->v.AsyncWith.items, pos); + + assert(s->kind == AsyncWith_kind); + + NEW_JUMP_TARGET_LABEL(c, block); + NEW_JUMP_TARGET_LABEL(c, final); + NEW_JUMP_TARGET_LABEL(c, exit); + NEW_JUMP_TARGET_LABEL(c, cleanup); + + /* Evaluate EXPR */ + VISIT(c, expr, item->context_expr); + loc = LOC(item->context_expr); + ADDOP_I(c, loc, COPY, 1); + ADDOP_I(c, loc, LOAD_SPECIAL, SPECIAL___AEXIT__); + ADDOP_I(c, loc, SWAP, 2); + ADDOP_I(c, loc, SWAP, 3); + ADDOP_I(c, loc, LOAD_SPECIAL, SPECIAL___AENTER__); + ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, GET_AWAITABLE, 1); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADD_YIELD_FROM(c, loc, 1); + + ADDOP_JUMP(c, loc, SETUP_WITH, final); + + /* SETUP_WITH pushes a finally block. */ + USE_LABEL(c, block); + RETURN_IF_ERROR(_PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_ASYNC_WITH, block, final, s)); + + if (item->optional_vars) { + VISIT(c, expr, item->optional_vars); + } + else { + /* Discard result from context.__aenter__() */ + ADDOP(c, loc, POP_TOP); + } + + pos++; + if (pos == asdl_seq_LEN(s->v.AsyncWith.items)) { + /* BLOCK code */ + VISIT_SEQ(c, stmt, s->v.AsyncWith.body); + } + else { + RETURN_IF_ERROR(codegen_async_with(c, s, pos)); + } + + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_ASYNC_WITH, block); + + ADDOP(c, loc, POP_BLOCK); + /* End of body; start the cleanup */ + + /* For successful outcome: + * call __exit__(None, None, None) + */ + RETURN_IF_ERROR(codegen_call_exit_with_nones(c, loc)); + ADDOP_I(c, loc, GET_AWAITABLE, 2); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADD_YIELD_FROM(c, loc, 1); + + ADDOP(c, loc, POP_TOP); + + ADDOP_JUMP(c, loc, JUMP, exit); + + /* For exceptional outcome: */ + USE_LABEL(c, final); + + ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); + ADDOP(c, loc, PUSH_EXC_INFO); + ADDOP(c, loc, WITH_EXCEPT_START); + ADDOP_I(c, loc, GET_AWAITABLE, 2); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADD_YIELD_FROM(c, loc, 1); + RETURN_IF_ERROR(codegen_with_except_finish(c, cleanup)); + + USE_LABEL(c, exit); + return SUCCESS; +} + + +/* + Implements the with statement from PEP 343. + with EXPR as VAR: + BLOCK + is implemented as: + + SETUP_WITH E + or POP_TOP + + LOAD_CONST (None, None, None) + CALL_FUNCTION_EX 0 + JUMP EXIT + E: WITH_EXCEPT_START (calls EXPR.__exit__) + POP_JUMP_IF_TRUE T: + RERAISE + T: POP_TOP (remove exception from stack) + POP_EXCEPT + POP_TOP + EXIT: + */ + +static int +codegen_with(compiler *c, stmt_ty s, int pos) +{ + withitem_ty item = asdl_seq_GET(s->v.With.items, pos); + + assert(s->kind == With_kind); + + NEW_JUMP_TARGET_LABEL(c, block); + NEW_JUMP_TARGET_LABEL(c, final); + NEW_JUMP_TARGET_LABEL(c, exit); + NEW_JUMP_TARGET_LABEL(c, cleanup); + + /* Evaluate EXPR */ + VISIT(c, expr, item->context_expr); + /* Will push bound __exit__ */ + location loc = LOC(item->context_expr); + ADDOP_I(c, loc, COPY, 1); + ADDOP_I(c, loc, LOAD_SPECIAL, SPECIAL___EXIT__); + ADDOP_I(c, loc, SWAP, 2); + ADDOP_I(c, loc, SWAP, 3); + ADDOP_I(c, loc, LOAD_SPECIAL, SPECIAL___ENTER__); + ADDOP_I(c, loc, CALL, 0); + ADDOP_JUMP(c, loc, SETUP_WITH, final); + + /* SETUP_WITH pushes a finally block. */ + USE_LABEL(c, block); + RETURN_IF_ERROR(_PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_WITH, block, final, s)); + + if (item->optional_vars) { + VISIT(c, expr, item->optional_vars); + } + else { + /* Discard result from context.__enter__() */ + ADDOP(c, loc, POP_TOP); + } + + pos++; + if (pos == asdl_seq_LEN(s->v.With.items)) { + /* BLOCK code */ + VISIT_SEQ(c, stmt, s->v.With.body); + } + else { + RETURN_IF_ERROR(codegen_with(c, s, pos)); + } + + ADDOP(c, NO_LOCATION, POP_BLOCK); + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_WITH, block); + + /* End of body; start the cleanup. */ + + /* For successful outcome: + * call __exit__(None, None, None) + */ + RETURN_IF_ERROR(codegen_call_exit_with_nones(c, loc)); + ADDOP(c, loc, POP_TOP); + ADDOP_JUMP(c, loc, JUMP, exit); + + /* For exceptional outcome: */ + USE_LABEL(c, final); + + ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); + ADDOP(c, loc, PUSH_EXC_INFO); + ADDOP(c, loc, WITH_EXCEPT_START); + RETURN_IF_ERROR(codegen_with_except_finish(c, cleanup)); + + USE_LABEL(c, exit); + return SUCCESS; +} + +static int +codegen_visit_expr(compiler *c, expr_ty e) +{ + location loc = LOC(e); + switch (e->kind) { + case NamedExpr_kind: + VISIT(c, expr, e->v.NamedExpr.value); + ADDOP_I(c, loc, COPY, 1); + VISIT(c, expr, e->v.NamedExpr.target); + break; + case BoolOp_kind: + return codegen_boolop(c, e); + case BinOp_kind: + VISIT(c, expr, e->v.BinOp.left); + VISIT(c, expr, e->v.BinOp.right); + ADDOP_BINARY(c, loc, e->v.BinOp.op); + break; + case UnaryOp_kind: + VISIT(c, expr, e->v.UnaryOp.operand); + if (e->v.UnaryOp.op == UAdd) { + ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_UNARY_POSITIVE); + } + else if (e->v.UnaryOp.op == Not) { + ADDOP(c, loc, TO_BOOL); + ADDOP(c, loc, UNARY_NOT); + } + else { + ADDOP(c, loc, unaryop(e->v.UnaryOp.op)); + } + break; + case Lambda_kind: + return codegen_lambda(c, e); + case IfExp_kind: + return codegen_ifexp(c, e); + case Dict_kind: + return codegen_dict(c, e); + case Set_kind: + return codegen_set(c, e); + case GeneratorExp_kind: + return codegen_genexp(c, e); + case ListComp_kind: + return codegen_listcomp(c, e); + case SetComp_kind: + return codegen_setcomp(c, e); + case DictComp_kind: + return codegen_dictcomp(c, e); + case Yield_kind: + if (!_PyST_IsFunctionLike(SYMTABLE_ENTRY(c))) { + return _PyCompile_Error(c, loc, "'yield' outside function"); + } + if (e->v.Yield.value) { + VISIT(c, expr, e->v.Yield.value); + } + else { + ADDOP_LOAD_CONST(c, loc, Py_None); + } + ADDOP_YIELD(c, loc); + break; + case YieldFrom_kind: + if (!_PyST_IsFunctionLike(SYMTABLE_ENTRY(c))) { + return _PyCompile_Error(c, loc, "'yield from' outside function"); + } + if (SCOPE_TYPE(c) == COMPILE_SCOPE_ASYNC_FUNCTION) { + return _PyCompile_Error(c, loc, "'yield from' inside async function"); + } + VISIT(c, expr, e->v.YieldFrom.value); + ADDOP(c, loc, GET_YIELD_FROM_ITER); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADD_YIELD_FROM(c, loc, 0); + break; + case Await_kind: + VISIT(c, expr, e->v.Await.value); + ADDOP_I(c, loc, GET_AWAITABLE, 0); + ADDOP_LOAD_CONST(c, loc, Py_None); + ADD_YIELD_FROM(c, loc, 1); + break; + case Compare_kind: + return codegen_compare(c, e); + case Call_kind: + return codegen_call(c, e); + case Constant_kind: + ADDOP_LOAD_CONST(c, loc, e->v.Constant.value); + break; + case JoinedStr_kind: + return codegen_joined_str(c, e); + case FormattedValue_kind: + return codegen_formatted_value(c, e); + /* The following exprs can be assignment targets. */ + case Attribute_kind: + if (e->v.Attribute.ctx == Load) { + int ret = can_optimize_super_call(c, e); + RETURN_IF_ERROR(ret); + if (ret) { + RETURN_IF_ERROR(load_args_for_super(c, e->v.Attribute.value)); + int opcode = asdl_seq_LEN(e->v.Attribute.value->v.Call.args) ? + LOAD_SUPER_ATTR : LOAD_ZERO_SUPER_ATTR; + ADDOP_NAME(c, loc, opcode, e->v.Attribute.attr, names); + loc = update_start_location_to_match_attr(c, loc, e); + ADDOP(c, loc, NOP); + return SUCCESS; + } + } + RETURN_IF_ERROR(_PyCompile_MaybeAddStaticAttributeToClass(c, e)); + VISIT(c, expr, e->v.Attribute.value); + loc = LOC(e); + loc = update_start_location_to_match_attr(c, loc, e); + switch (e->v.Attribute.ctx) { + case Load: + ADDOP_NAME(c, loc, LOAD_ATTR, e->v.Attribute.attr, names); + break; + case Store: + ADDOP_NAME(c, loc, STORE_ATTR, e->v.Attribute.attr, names); + break; + case Del: + ADDOP_NAME(c, loc, DELETE_ATTR, e->v.Attribute.attr, names); + break; + } + break; + case Subscript_kind: + return codegen_subscript(c, e); + case Starred_kind: + switch (e->v.Starred.ctx) { + case Store: + /* In all legitimate cases, the Starred node was already replaced + * by codegen_list/codegen_tuple. XXX: is that okay? */ + return _PyCompile_Error(c, loc, + "starred assignment target must be in a list or tuple"); + default: + return _PyCompile_Error(c, loc, + "can't use starred expression here"); + } + break; + case Slice_kind: + RETURN_IF_ERROR(codegen_slice(c, e)); + break; + case Name_kind: + return codegen_nameop(c, loc, e->v.Name.id, e->v.Name.ctx); + /* child nodes of List and Tuple will have expr_context set */ + case List_kind: + return codegen_list(c, e); + case Tuple_kind: + return codegen_tuple(c, e); + } + return SUCCESS; +} + +static bool +is_constant_slice(expr_ty s) +{ + return s->kind == Slice_kind && + (s->v.Slice.lower == NULL || + s->v.Slice.lower->kind == Constant_kind) && + (s->v.Slice.upper == NULL || + s->v.Slice.upper->kind == Constant_kind) && + (s->v.Slice.step == NULL || + s->v.Slice.step->kind == Constant_kind); +} + +static bool +should_apply_two_element_slice_optimization(expr_ty s) +{ + return !is_constant_slice(s) && + s->kind == Slice_kind && + s->v.Slice.step == NULL; +} + +static int +codegen_augassign(compiler *c, stmt_ty s) +{ + assert(s->kind == AugAssign_kind); + expr_ty e = s->v.AugAssign.target; + + location loc = LOC(e); + + switch (e->kind) { + case Attribute_kind: + VISIT(c, expr, e->v.Attribute.value); + ADDOP_I(c, loc, COPY, 1); + loc = update_start_location_to_match_attr(c, loc, e); + ADDOP_NAME(c, loc, LOAD_ATTR, e->v.Attribute.attr, names); + break; + case Subscript_kind: + VISIT(c, expr, e->v.Subscript.value); + if (should_apply_two_element_slice_optimization(e->v.Subscript.slice)) { + RETURN_IF_ERROR(codegen_slice_two_parts(c, e->v.Subscript.slice)); + ADDOP_I(c, loc, COPY, 3); + ADDOP_I(c, loc, COPY, 3); + ADDOP_I(c, loc, COPY, 3); + ADDOP(c, loc, BINARY_SLICE); + } + else { + VISIT(c, expr, e->v.Subscript.slice); + ADDOP_I(c, loc, COPY, 2); + ADDOP_I(c, loc, COPY, 2); + ADDOP(c, loc, BINARY_SUBSCR); + } + break; + case Name_kind: + RETURN_IF_ERROR(codegen_nameop(c, loc, e->v.Name.id, Load)); + break; + default: + PyErr_Format(PyExc_SystemError, + "invalid node type (%d) for augmented assignment", + e->kind); + return ERROR; + } + + loc = LOC(s); + + VISIT(c, expr, s->v.AugAssign.value); + ADDOP_INPLACE(c, loc, s->v.AugAssign.op); + + loc = LOC(e); + + switch (e->kind) { + case Attribute_kind: + loc = update_start_location_to_match_attr(c, loc, e); + ADDOP_I(c, loc, SWAP, 2); + ADDOP_NAME(c, loc, STORE_ATTR, e->v.Attribute.attr, names); + break; + case Subscript_kind: + if (should_apply_two_element_slice_optimization(e->v.Subscript.slice)) { + ADDOP_I(c, loc, SWAP, 4); + ADDOP_I(c, loc, SWAP, 3); + ADDOP_I(c, loc, SWAP, 2); + ADDOP(c, loc, STORE_SLICE); + } + else { + ADDOP_I(c, loc, SWAP, 3); + ADDOP_I(c, loc, SWAP, 2); + ADDOP(c, loc, STORE_SUBSCR); + } + break; + case Name_kind: + return codegen_nameop(c, loc, e->v.Name.id, Store); + default: + Py_UNREACHABLE(); + } + return SUCCESS; +} + +static int +codegen_check_ann_expr(compiler *c, expr_ty e) +{ + VISIT(c, expr, e); + ADDOP(c, LOC(e), POP_TOP); + return SUCCESS; +} + +static int +codegen_check_annotation(compiler *c, stmt_ty s) +{ + /* Annotations of complex targets does not produce anything + under annotations future */ + if (FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS) { + return SUCCESS; + } + + /* Annotations are only evaluated in a module or class. */ + if (SCOPE_TYPE(c) == COMPILE_SCOPE_MODULE || + SCOPE_TYPE(c) == COMPILE_SCOPE_CLASS) { + return codegen_check_ann_expr(c, s->v.AnnAssign.annotation); + } + return SUCCESS; +} + +static int +codegen_check_ann_subscr(compiler *c, expr_ty e) +{ + /* We check that everything in a subscript is defined at runtime. */ + switch (e->kind) { + case Slice_kind: + if (e->v.Slice.lower && codegen_check_ann_expr(c, e->v.Slice.lower) < 0) { + return ERROR; + } + if (e->v.Slice.upper && codegen_check_ann_expr(c, e->v.Slice.upper) < 0) { + return ERROR; + } + if (e->v.Slice.step && codegen_check_ann_expr(c, e->v.Slice.step) < 0) { + return ERROR; + } + return SUCCESS; + case Tuple_kind: { + /* extended slice */ + asdl_expr_seq *elts = e->v.Tuple.elts; + Py_ssize_t i, n = asdl_seq_LEN(elts); + for (i = 0; i < n; i++) { + RETURN_IF_ERROR(codegen_check_ann_subscr(c, asdl_seq_GET(elts, i))); + } + return SUCCESS; + } + default: + return codegen_check_ann_expr(c, e); + } +} + +static int +codegen_annassign(compiler *c, stmt_ty s) +{ + location loc = LOC(s); + expr_ty targ = s->v.AnnAssign.target; + bool future_annotations = FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS; + PyObject *mangled; + + assert(s->kind == AnnAssign_kind); + + /* We perform the actual assignment first. */ + if (s->v.AnnAssign.value) { + VISIT(c, expr, s->v.AnnAssign.value); + VISIT(c, expr, targ); + } + switch (targ->kind) { + case Name_kind: + /* If we have a simple name in a module or class, store annotation. */ + if (s->v.AnnAssign.simple && + (SCOPE_TYPE(c) == COMPILE_SCOPE_MODULE || + SCOPE_TYPE(c) == COMPILE_SCOPE_CLASS)) { + if (future_annotations) { + VISIT(c, annexpr, s->v.AnnAssign.annotation); + ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names); + mangled = _PyCompile_MaybeMangle(c, targ->v.Name.id); + ADDOP_LOAD_CONST_NEW(c, loc, mangled); + ADDOP(c, loc, STORE_SUBSCR); + } + else { + RETURN_IF_ERROR(_PyCompile_AddDeferredAnnotaion(c, s)); + } + } + break; + case Attribute_kind: + if (!s->v.AnnAssign.value && + codegen_check_ann_expr(c, targ->v.Attribute.value) < 0) { + return ERROR; + } + break; + case Subscript_kind: + if (!s->v.AnnAssign.value && + (codegen_check_ann_expr(c, targ->v.Subscript.value) < 0 || + codegen_check_ann_subscr(c, targ->v.Subscript.slice) < 0)) { + return ERROR; + } + break; + default: + PyErr_Format(PyExc_SystemError, + "invalid node type (%d) for annotated assignment", + targ->kind); + return ERROR; + } + /* Annotation is evaluated last. */ + if (future_annotations && !s->v.AnnAssign.simple && codegen_check_annotation(c, s) < 0) { + return ERROR; + } + return SUCCESS; +} + +static int +codegen_subscript(compiler *c, expr_ty e) +{ + location loc = LOC(e); + expr_context_ty ctx = e->v.Subscript.ctx; + int op = 0; + + if (ctx == Load) { + RETURN_IF_ERROR(check_subscripter(c, e->v.Subscript.value)); + RETURN_IF_ERROR(check_index(c, e->v.Subscript.value, e->v.Subscript.slice)); + } + + VISIT(c, expr, e->v.Subscript.value); + if (should_apply_two_element_slice_optimization(e->v.Subscript.slice) && + ctx != Del + ) { + RETURN_IF_ERROR(codegen_slice_two_parts(c, e->v.Subscript.slice)); + if (ctx == Load) { + ADDOP(c, loc, BINARY_SLICE); + } + else { + assert(ctx == Store); + ADDOP(c, loc, STORE_SLICE); + } + } + else { + VISIT(c, expr, e->v.Subscript.slice); + switch (ctx) { + case Load: op = BINARY_SUBSCR; break; + case Store: op = STORE_SUBSCR; break; + case Del: op = DELETE_SUBSCR; break; + } + assert(op); + ADDOP(c, loc, op); + } + return SUCCESS; +} + +static int +codegen_slice_two_parts(compiler *c, expr_ty s) +{ + if (s->v.Slice.lower) { + VISIT(c, expr, s->v.Slice.lower); + } + else { + ADDOP_LOAD_CONST(c, LOC(s), Py_None); + } + + if (s->v.Slice.upper) { + VISIT(c, expr, s->v.Slice.upper); + } + else { + ADDOP_LOAD_CONST(c, LOC(s), Py_None); + } + + return 0; +} + +static int +codegen_slice(compiler *c, expr_ty s) +{ + int n = 2; + assert(s->kind == Slice_kind); + + if (is_constant_slice(s)) { + PyObject *start = NULL; + if (s->v.Slice.lower) { + start = s->v.Slice.lower->v.Constant.value; + } + PyObject *stop = NULL; + if (s->v.Slice.upper) { + stop = s->v.Slice.upper->v.Constant.value; + } + PyObject *step = NULL; + if (s->v.Slice.step) { + step = s->v.Slice.step->v.Constant.value; + } + PyObject *slice = PySlice_New(start, stop, step); + if (slice == NULL) { + return ERROR; + } + ADDOP_LOAD_CONST_NEW(c, LOC(s), slice); + return SUCCESS; + } + + RETURN_IF_ERROR(codegen_slice_two_parts(c, s)); + + if (s->v.Slice.step) { + n++; + VISIT(c, expr, s->v.Slice.step); + } + + ADDOP_I(c, LOC(s), BUILD_SLICE, n); + return SUCCESS; +} + + +// PEP 634: Structural Pattern Matching + +// To keep things simple, all codegen_pattern_* routines follow the convention +// of consuming TOS (the subject for the given pattern) and calling +// jump_to_fail_pop on failure (no match). + +// When calling into these routines, it's important that pc->on_top be kept +// updated to reflect the current number of items that we are using on the top +// of the stack: they will be popped on failure, and any name captures will be +// stored *underneath* them on success. This lets us defer all names stores +// until the *entire* pattern matches. + +#define WILDCARD_CHECK(N) \ + ((N)->kind == MatchAs_kind && !(N)->v.MatchAs.name) + +#define WILDCARD_STAR_CHECK(N) \ + ((N)->kind == MatchStar_kind && !(N)->v.MatchStar.name) + +// Limit permitted subexpressions, even if the parser & AST validator let them through +#define MATCH_VALUE_EXPR(N) \ + ((N)->kind == Constant_kind || (N)->kind == Attribute_kind) + +// Allocate or resize pc->fail_pop to allow for n items to be popped on failure. +static int +ensure_fail_pop(compiler *c, pattern_context *pc, Py_ssize_t n) +{ + Py_ssize_t size = n + 1; + if (size <= pc->fail_pop_size) { + return SUCCESS; + } + Py_ssize_t needed = sizeof(jump_target_label) * size; + jump_target_label *resized = PyMem_Realloc(pc->fail_pop, needed); + if (resized == NULL) { + PyErr_NoMemory(); + return ERROR; + } + pc->fail_pop = resized; + while (pc->fail_pop_size < size) { + NEW_JUMP_TARGET_LABEL(c, new_block); + pc->fail_pop[pc->fail_pop_size++] = new_block; + } + return SUCCESS; +} + +// Use op to jump to the correct fail_pop block. +static int +jump_to_fail_pop(compiler *c, location loc, + pattern_context *pc, int op) +{ + // Pop any items on the top of the stack, plus any objects we were going to + // capture on success: + Py_ssize_t pops = pc->on_top + PyList_GET_SIZE(pc->stores); + RETURN_IF_ERROR(ensure_fail_pop(c, pc, pops)); + ADDOP_JUMP(c, loc, op, pc->fail_pop[pops]); + return SUCCESS; +} + +// Build all of the fail_pop blocks and reset fail_pop. +static int +emit_and_reset_fail_pop(compiler *c, location loc, + pattern_context *pc) +{ + if (!pc->fail_pop_size) { + assert(pc->fail_pop == NULL); + return SUCCESS; + } + while (--pc->fail_pop_size) { + USE_LABEL(c, pc->fail_pop[pc->fail_pop_size]); + if (codegen_addop_noarg(INSTR_SEQUENCE(c), POP_TOP, loc) < 0) { + pc->fail_pop_size = 0; + PyMem_Free(pc->fail_pop); + pc->fail_pop = NULL; + return ERROR; + } + } + USE_LABEL(c, pc->fail_pop[0]); + PyMem_Free(pc->fail_pop); + pc->fail_pop = NULL; + return SUCCESS; +} + +static int +codegen_error_duplicate_store(compiler *c, location loc, identifier n) +{ + return _PyCompile_Error(c, loc, + "multiple assignments to name %R in pattern", n); +} + +// Duplicate the effect of 3.10's ROT_* instructions using SWAPs. +static int +codegen_pattern_helper_rotate(compiler *c, location loc, Py_ssize_t count) +{ + while (1 < count) { + ADDOP_I(c, loc, SWAP, count--); + } + return SUCCESS; +} + +static int +codegen_pattern_helper_store_name(compiler *c, location loc, + identifier n, pattern_context *pc) +{ + if (n == NULL) { + ADDOP(c, loc, POP_TOP); + return SUCCESS; + } + // Can't assign to the same name twice: + int duplicate = PySequence_Contains(pc->stores, n); + RETURN_IF_ERROR(duplicate); + if (duplicate) { + return codegen_error_duplicate_store(c, loc, n); + } + // Rotate this object underneath any items we need to preserve: + Py_ssize_t rotations = pc->on_top + PyList_GET_SIZE(pc->stores) + 1; + RETURN_IF_ERROR(codegen_pattern_helper_rotate(c, loc, rotations)); + RETURN_IF_ERROR(PyList_Append(pc->stores, n)); + return SUCCESS; +} + + +static int +codegen_pattern_unpack_helper(compiler *c, location loc, + asdl_pattern_seq *elts) +{ + Py_ssize_t n = asdl_seq_LEN(elts); + int seen_star = 0; + for (Py_ssize_t i = 0; i < n; i++) { + pattern_ty elt = asdl_seq_GET(elts, i); + if (elt->kind == MatchStar_kind && !seen_star) { + if ((i >= (1 << 8)) || + (n-i-1 >= (INT_MAX >> 8))) { + return _PyCompile_Error(c, loc, + "too many expressions in " + "star-unpacking sequence pattern"); + } + ADDOP_I(c, loc, UNPACK_EX, (i + ((n-i-1) << 8))); + seen_star = 1; + } + else if (elt->kind == MatchStar_kind) { + return _PyCompile_Error(c, loc, + "multiple starred expressions in sequence pattern"); + } + } + if (!seen_star) { + ADDOP_I(c, loc, UNPACK_SEQUENCE, n); + } + return SUCCESS; +} + +static int +pattern_helper_sequence_unpack(compiler *c, location loc, + asdl_pattern_seq *patterns, Py_ssize_t star, + pattern_context *pc) +{ + RETURN_IF_ERROR(codegen_pattern_unpack_helper(c, loc, patterns)); + Py_ssize_t size = asdl_seq_LEN(patterns); + // We've now got a bunch of new subjects on the stack. They need to remain + // there after each subpattern match: + pc->on_top += size; + for (Py_ssize_t i = 0; i < size; i++) { + // One less item to keep track of each time we loop through: + pc->on_top--; + pattern_ty pattern = asdl_seq_GET(patterns, i); + RETURN_IF_ERROR(codegen_pattern_subpattern(c, pattern, pc)); + } + return SUCCESS; +} + +// Like pattern_helper_sequence_unpack, but uses BINARY_SUBSCR instead of +// UNPACK_SEQUENCE / UNPACK_EX. This is more efficient for patterns with a +// starred wildcard like [first, *_] / [first, *_, last] / [*_, last] / etc. +static int +pattern_helper_sequence_subscr(compiler *c, location loc, + asdl_pattern_seq *patterns, Py_ssize_t star, + pattern_context *pc) +{ + // We need to keep the subject around for extracting elements: + pc->on_top++; + Py_ssize_t size = asdl_seq_LEN(patterns); + for (Py_ssize_t i = 0; i < size; i++) { + pattern_ty pattern = asdl_seq_GET(patterns, i); + if (WILDCARD_CHECK(pattern)) { + continue; + } + if (i == star) { + assert(WILDCARD_STAR_CHECK(pattern)); + continue; + } + ADDOP_I(c, loc, COPY, 1); + if (i < star) { + ADDOP_LOAD_CONST_NEW(c, loc, PyLong_FromSsize_t(i)); + } + else { + // The subject may not support negative indexing! Compute a + // nonnegative index: + ADDOP(c, loc, GET_LEN); + ADDOP_LOAD_CONST_NEW(c, loc, PyLong_FromSsize_t(size - i)); + ADDOP_BINARY(c, loc, Sub); + } + ADDOP(c, loc, BINARY_SUBSCR); + RETURN_IF_ERROR(codegen_pattern_subpattern(c, pattern, pc)); + } + // Pop the subject, we're done with it: + pc->on_top--; + ADDOP(c, loc, POP_TOP); + return SUCCESS; +} + +// Like codegen_pattern, but turn off checks for irrefutability. +static int +codegen_pattern_subpattern(compiler *c, + pattern_ty p, pattern_context *pc) +{ + int allow_irrefutable = pc->allow_irrefutable; + pc->allow_irrefutable = 1; + RETURN_IF_ERROR(codegen_pattern(c, p, pc)); + pc->allow_irrefutable = allow_irrefutable; + return SUCCESS; +} + +static int +codegen_pattern_as(compiler *c, pattern_ty p, pattern_context *pc) +{ + assert(p->kind == MatchAs_kind); + if (p->v.MatchAs.pattern == NULL) { + // An irrefutable match: + if (!pc->allow_irrefutable) { + if (p->v.MatchAs.name) { + const char *e = "name capture %R makes remaining patterns unreachable"; + return _PyCompile_Error(c, LOC(p), e, p->v.MatchAs.name); + } + const char *e = "wildcard makes remaining patterns unreachable"; + return _PyCompile_Error(c, LOC(p), e); + } + return codegen_pattern_helper_store_name(c, LOC(p), p->v.MatchAs.name, pc); + } + // Need to make a copy for (possibly) storing later: + pc->on_top++; + ADDOP_I(c, LOC(p), COPY, 1); + RETURN_IF_ERROR(codegen_pattern(c, p->v.MatchAs.pattern, pc)); + // Success! Store it: + pc->on_top--; + RETURN_IF_ERROR(codegen_pattern_helper_store_name(c, LOC(p), p->v.MatchAs.name, pc)); + return SUCCESS; +} + +static int +codegen_pattern_star(compiler *c, pattern_ty p, pattern_context *pc) +{ + assert(p->kind == MatchStar_kind); + RETURN_IF_ERROR( + codegen_pattern_helper_store_name(c, LOC(p), p->v.MatchStar.name, pc)); + return SUCCESS; +} + +static int +validate_kwd_attrs(compiler *c, asdl_identifier_seq *attrs, asdl_pattern_seq* patterns) +{ + // Any errors will point to the pattern rather than the arg name as the + // parser is only supplying identifiers rather than Name or keyword nodes + Py_ssize_t nattrs = asdl_seq_LEN(attrs); + for (Py_ssize_t i = 0; i < nattrs; i++) { + identifier attr = ((identifier)asdl_seq_GET(attrs, i)); + for (Py_ssize_t j = i + 1; j < nattrs; j++) { + identifier other = ((identifier)asdl_seq_GET(attrs, j)); + if (!PyUnicode_Compare(attr, other)) { + location loc = LOC((pattern_ty) asdl_seq_GET(patterns, j)); + return _PyCompile_Error(c, loc, "attribute name repeated " + "in class pattern: %U", attr); + } + } + } + return SUCCESS; +} + +static int +codegen_pattern_class(compiler *c, pattern_ty p, pattern_context *pc) +{ + assert(p->kind == MatchClass_kind); + asdl_pattern_seq *patterns = p->v.MatchClass.patterns; + asdl_identifier_seq *kwd_attrs = p->v.MatchClass.kwd_attrs; + asdl_pattern_seq *kwd_patterns = p->v.MatchClass.kwd_patterns; + Py_ssize_t nargs = asdl_seq_LEN(patterns); + Py_ssize_t nattrs = asdl_seq_LEN(kwd_attrs); + Py_ssize_t nkwd_patterns = asdl_seq_LEN(kwd_patterns); + if (nattrs != nkwd_patterns) { + // AST validator shouldn't let this happen, but if it does, + // just fail, don't crash out of the interpreter + const char * e = "kwd_attrs (%d) / kwd_patterns (%d) length mismatch in class pattern"; + return _PyCompile_Error(c, LOC(p), e, nattrs, nkwd_patterns); + } + if (INT_MAX < nargs || INT_MAX < nargs + nattrs - 1) { + const char *e = "too many sub-patterns in class pattern %R"; + return _PyCompile_Error(c, LOC(p), e, p->v.MatchClass.cls); + } + if (nattrs) { + RETURN_IF_ERROR(validate_kwd_attrs(c, kwd_attrs, kwd_patterns)); + } + VISIT(c, expr, p->v.MatchClass.cls); + PyObject *attr_names = PyTuple_New(nattrs); + if (attr_names == NULL) { + return ERROR; + } + Py_ssize_t i; + for (i = 0; i < nattrs; i++) { + PyObject *name = asdl_seq_GET(kwd_attrs, i); + PyTuple_SET_ITEM(attr_names, i, Py_NewRef(name)); + } + ADDOP_LOAD_CONST_NEW(c, LOC(p), attr_names); + ADDOP_I(c, LOC(p), MATCH_CLASS, nargs); + ADDOP_I(c, LOC(p), COPY, 1); + ADDOP_LOAD_CONST(c, LOC(p), Py_None); + ADDOP_I(c, LOC(p), IS_OP, 1); + // TOS is now a tuple of (nargs + nattrs) attributes (or None): + pc->on_top++; + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + ADDOP_I(c, LOC(p), UNPACK_SEQUENCE, nargs + nattrs); + pc->on_top += nargs + nattrs - 1; + for (i = 0; i < nargs + nattrs; i++) { + pc->on_top--; + pattern_ty pattern; + if (i < nargs) { + // Positional: + pattern = asdl_seq_GET(patterns, i); + } + else { + // Keyword: + pattern = asdl_seq_GET(kwd_patterns, i - nargs); + } + if (WILDCARD_CHECK(pattern)) { + ADDOP(c, LOC(p), POP_TOP); + continue; + } + RETURN_IF_ERROR(codegen_pattern_subpattern(c, pattern, pc)); + } + // Success! Pop the tuple of attributes: + return SUCCESS; +} + +static int +codegen_pattern_mapping_key(compiler *c, PyObject *seen, pattern_ty p, Py_ssize_t i) +{ + asdl_expr_seq *keys = p->v.MatchMapping.keys; + asdl_pattern_seq *patterns = p->v.MatchMapping.patterns; + expr_ty key = asdl_seq_GET(keys, i); + if (key == NULL) { + const char *e = "can't use NULL keys in MatchMapping " + "(set 'rest' parameter instead)"; + location loc = LOC((pattern_ty) asdl_seq_GET(patterns, i)); + return _PyCompile_Error(c, loc, e); + } + + if (key->kind == Constant_kind) { + int in_seen = PySet_Contains(seen, key->v.Constant.value); + RETURN_IF_ERROR(in_seen); + if (in_seen) { + const char *e = "mapping pattern checks duplicate key (%R)"; + return _PyCompile_Error(c, LOC(p), e, key->v.Constant.value); + } + RETURN_IF_ERROR(PySet_Add(seen, key->v.Constant.value)); + } + else if (key->kind != Attribute_kind) { + const char *e = "mapping pattern keys may only match literals and attribute lookups"; + return _PyCompile_Error(c, LOC(p), e); + } + VISIT(c, expr, key); + return SUCCESS; +} + +static int +codegen_pattern_mapping(compiler *c, pattern_ty p, + pattern_context *pc) +{ + assert(p->kind == MatchMapping_kind); + asdl_expr_seq *keys = p->v.MatchMapping.keys; + asdl_pattern_seq *patterns = p->v.MatchMapping.patterns; + Py_ssize_t size = asdl_seq_LEN(keys); + Py_ssize_t npatterns = asdl_seq_LEN(patterns); + if (size != npatterns) { + // AST validator shouldn't let this happen, but if it does, + // just fail, don't crash out of the interpreter + const char * e = "keys (%d) / patterns (%d) length mismatch in mapping pattern"; + return _PyCompile_Error(c, LOC(p), e, size, npatterns); + } + // We have a double-star target if "rest" is set + PyObject *star_target = p->v.MatchMapping.rest; + // We need to keep the subject on top during the mapping and length checks: + pc->on_top++; + ADDOP(c, LOC(p), MATCH_MAPPING); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + if (!size && !star_target) { + // If the pattern is just "{}", we're done! Pop the subject: + pc->on_top--; + ADDOP(c, LOC(p), POP_TOP); + return SUCCESS; + } + if (size) { + // If the pattern has any keys in it, perform a length check: + ADDOP(c, LOC(p), GET_LEN); + ADDOP_LOAD_CONST_NEW(c, LOC(p), PyLong_FromSsize_t(size)); + ADDOP_COMPARE(c, LOC(p), GtE); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + } + if (INT_MAX < size - 1) { + return _PyCompile_Error(c, LOC(p), "too many sub-patterns in mapping pattern"); + } + // Collect all of the keys into a tuple for MATCH_KEYS and + // **rest. They can either be dotted names or literals: + + // Maintaining a set of Constant_kind kind keys allows us to raise a + // SyntaxError in the case of duplicates. + PyObject *seen = PySet_New(NULL); + if (seen == NULL) { + return ERROR; + } + for (Py_ssize_t i = 0; i < size; i++) { + if (codegen_pattern_mapping_key(c, seen, p, i) < 0) { + Py_DECREF(seen); + return ERROR; + } + } + Py_DECREF(seen); + + // all keys have been checked; there are no duplicates + + ADDOP_I(c, LOC(p), BUILD_TUPLE, size); + ADDOP(c, LOC(p), MATCH_KEYS); + // There's now a tuple of keys and a tuple of values on top of the subject: + pc->on_top += 2; + ADDOP_I(c, LOC(p), COPY, 1); + ADDOP_LOAD_CONST(c, LOC(p), Py_None); + ADDOP_I(c, LOC(p), IS_OP, 1); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + // So far so good. Use that tuple of values on the stack to match + // sub-patterns against: + ADDOP_I(c, LOC(p), UNPACK_SEQUENCE, size); + pc->on_top += size - 1; + for (Py_ssize_t i = 0; i < size; i++) { + pc->on_top--; + pattern_ty pattern = asdl_seq_GET(patterns, i); + RETURN_IF_ERROR(codegen_pattern_subpattern(c, pattern, pc)); + } + // If we get this far, it's a match! Whatever happens next should consume + // the tuple of keys and the subject: + pc->on_top -= 2; + if (star_target) { + // If we have a starred name, bind a dict of remaining items to it (this may + // seem a bit inefficient, but keys is rarely big enough to actually impact + // runtime): + // rest = dict(TOS1) + // for key in TOS: + // del rest[key] + ADDOP_I(c, LOC(p), BUILD_MAP, 0); // [subject, keys, empty] + ADDOP_I(c, LOC(p), SWAP, 3); // [empty, keys, subject] + ADDOP_I(c, LOC(p), DICT_UPDATE, 2); // [copy, keys] + ADDOP_I(c, LOC(p), UNPACK_SEQUENCE, size); // [copy, keys...] + while (size) { + ADDOP_I(c, LOC(p), COPY, 1 + size--); // [copy, keys..., copy] + ADDOP_I(c, LOC(p), SWAP, 2); // [copy, keys..., copy, key] + ADDOP(c, LOC(p), DELETE_SUBSCR); // [copy, keys...] + } + RETURN_IF_ERROR(codegen_pattern_helper_store_name(c, LOC(p), star_target, pc)); + } + else { + ADDOP(c, LOC(p), POP_TOP); // Tuple of keys. + ADDOP(c, LOC(p), POP_TOP); // Subject. + } + return SUCCESS; +} + +static int +codegen_pattern_or(compiler *c, pattern_ty p, pattern_context *pc) +{ + assert(p->kind == MatchOr_kind); + NEW_JUMP_TARGET_LABEL(c, end); + Py_ssize_t size = asdl_seq_LEN(p->v.MatchOr.patterns); + assert(size > 1); + // We're going to be messing with pc. Keep the original info handy: + pattern_context old_pc = *pc; + Py_INCREF(pc->stores); + // control is the list of names bound by the first alternative. It is used + // for checking different name bindings in alternatives, and for correcting + // the order in which extracted elements are placed on the stack. + PyObject *control = NULL; + // NOTE: We can't use returning macros anymore! goto error on error. + for (Py_ssize_t i = 0; i < size; i++) { + pattern_ty alt = asdl_seq_GET(p->v.MatchOr.patterns, i); + PyObject *pc_stores = PyList_New(0); + if (pc_stores == NULL) { + goto error; + } + Py_SETREF(pc->stores, pc_stores); + // An irrefutable sub-pattern must be last, if it is allowed at all: + pc->allow_irrefutable = (i == size - 1) && old_pc.allow_irrefutable; + pc->fail_pop = NULL; + pc->fail_pop_size = 0; + pc->on_top = 0; + if (codegen_addop_i(INSTR_SEQUENCE(c), COPY, 1, LOC(alt)) < 0 || + codegen_pattern(c, alt, pc) < 0) { + goto error; + } + // Success! + Py_ssize_t nstores = PyList_GET_SIZE(pc->stores); + if (!i) { + // This is the first alternative, so save its stores as a "control" + // for the others (they can't bind a different set of names, and + // might need to be reordered): + assert(control == NULL); + control = Py_NewRef(pc->stores); + } + else if (nstores != PyList_GET_SIZE(control)) { + goto diff; + } + else if (nstores) { + // There were captures. Check to see if we differ from control: + Py_ssize_t icontrol = nstores; + while (icontrol--) { + PyObject *name = PyList_GET_ITEM(control, icontrol); + Py_ssize_t istores = PySequence_Index(pc->stores, name); + if (istores < 0) { + PyErr_Clear(); + goto diff; + } + if (icontrol != istores) { + // Reorder the names on the stack to match the order of the + // names in control. There's probably a better way of doing + // this; the current solution is potentially very + // inefficient when each alternative subpattern binds lots + // of names in different orders. It's fine for reasonable + // cases, though, and the peephole optimizer will ensure + // that the final code is as efficient as possible. + assert(istores < icontrol); + Py_ssize_t rotations = istores + 1; + // Perform the same rotation on pc->stores: + PyObject *rotated = PyList_GetSlice(pc->stores, 0, + rotations); + if (rotated == NULL || + PyList_SetSlice(pc->stores, 0, rotations, NULL) || + PyList_SetSlice(pc->stores, icontrol - istores, + icontrol - istores, rotated)) + { + Py_XDECREF(rotated); + goto error; + } + Py_DECREF(rotated); + // That just did: + // rotated = pc_stores[:rotations] + // del pc_stores[:rotations] + // pc_stores[icontrol-istores:icontrol-istores] = rotated + // Do the same thing to the stack, using several + // rotations: + while (rotations--) { + if (codegen_pattern_helper_rotate(c, LOC(alt), icontrol + 1) < 0) { + goto error; + } + } + } + } + } + assert(control); + if (codegen_addop_j(INSTR_SEQUENCE(c), LOC(alt), JUMP, end) < 0 || + emit_and_reset_fail_pop(c, LOC(alt), pc) < 0) + { + goto error; + } + } + Py_DECREF(pc->stores); + *pc = old_pc; + Py_INCREF(pc->stores); + // Need to NULL this for the PyMem_Free call in the error block. + old_pc.fail_pop = NULL; + // No match. Pop the remaining copy of the subject and fail: + if (codegen_addop_noarg(INSTR_SEQUENCE(c), POP_TOP, LOC(p)) < 0 || + jump_to_fail_pop(c, LOC(p), pc, JUMP) < 0) { + goto error; + } + + USE_LABEL(c, end); + Py_ssize_t nstores = PyList_GET_SIZE(control); + // There's a bunch of stuff on the stack between where the new stores + // are and where they need to be: + // - The other stores. + // - A copy of the subject. + // - Anything else that may be on top of the stack. + // - Any previous stores we've already stashed away on the stack. + Py_ssize_t nrots = nstores + 1 + pc->on_top + PyList_GET_SIZE(pc->stores); + for (Py_ssize_t i = 0; i < nstores; i++) { + // Rotate this capture to its proper place on the stack: + if (codegen_pattern_helper_rotate(c, LOC(p), nrots) < 0) { + goto error; + } + // Update the list of previous stores with this new name, checking for + // duplicates: + PyObject *name = PyList_GET_ITEM(control, i); + int dupe = PySequence_Contains(pc->stores, name); + if (dupe < 0) { + goto error; + } + if (dupe) { + codegen_error_duplicate_store(c, LOC(p), name); + goto error; + } + if (PyList_Append(pc->stores, name)) { + goto error; + } + } + Py_DECREF(old_pc.stores); + Py_DECREF(control); + // NOTE: Returning macros are safe again. + // Pop the copy of the subject: + ADDOP(c, LOC(p), POP_TOP); + return SUCCESS; +diff: + _PyCompile_Error(c, LOC(p), "alternative patterns bind different names"); +error: + PyMem_Free(old_pc.fail_pop); + Py_DECREF(old_pc.stores); + Py_XDECREF(control); + return ERROR; +} + + +static int +codegen_pattern_sequence(compiler *c, pattern_ty p, + pattern_context *pc) +{ + assert(p->kind == MatchSequence_kind); + asdl_pattern_seq *patterns = p->v.MatchSequence.patterns; + Py_ssize_t size = asdl_seq_LEN(patterns); + Py_ssize_t star = -1; + int only_wildcard = 1; + int star_wildcard = 0; + // Find a starred name, if it exists. There may be at most one: + for (Py_ssize_t i = 0; i < size; i++) { + pattern_ty pattern = asdl_seq_GET(patterns, i); + if (pattern->kind == MatchStar_kind) { + if (star >= 0) { + const char *e = "multiple starred names in sequence pattern"; + return _PyCompile_Error(c, LOC(p), e); + } + star_wildcard = WILDCARD_STAR_CHECK(pattern); + only_wildcard &= star_wildcard; + star = i; + continue; + } + only_wildcard &= WILDCARD_CHECK(pattern); + } + // We need to keep the subject on top during the sequence and length checks: + pc->on_top++; + ADDOP(c, LOC(p), MATCH_SEQUENCE); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + if (star < 0) { + // No star: len(subject) == size + ADDOP(c, LOC(p), GET_LEN); + ADDOP_LOAD_CONST_NEW(c, LOC(p), PyLong_FromSsize_t(size)); + ADDOP_COMPARE(c, LOC(p), Eq); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + } + else if (size > 1) { + // Star: len(subject) >= size - 1 + ADDOP(c, LOC(p), GET_LEN); + ADDOP_LOAD_CONST_NEW(c, LOC(p), PyLong_FromSsize_t(size - 1)); + ADDOP_COMPARE(c, LOC(p), GtE); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + } + // Whatever comes next should consume the subject: + pc->on_top--; + if (only_wildcard) { + // Patterns like: [] / [_] / [_, _] / [*_] / [_, *_] / [_, _, *_] / etc. + ADDOP(c, LOC(p), POP_TOP); + } + else if (star_wildcard) { + RETURN_IF_ERROR(pattern_helper_sequence_subscr(c, LOC(p), patterns, star, pc)); + } + else { + RETURN_IF_ERROR(pattern_helper_sequence_unpack(c, LOC(p), patterns, star, pc)); + } + return SUCCESS; +} + +static int +codegen_pattern_value(compiler *c, pattern_ty p, pattern_context *pc) +{ + assert(p->kind == MatchValue_kind); + expr_ty value = p->v.MatchValue.value; + if (!MATCH_VALUE_EXPR(value)) { + const char *e = "patterns may only match literals and attribute lookups"; + return _PyCompile_Error(c, LOC(p), e); + } + VISIT(c, expr, value); + ADDOP_COMPARE(c, LOC(p), Eq); + ADDOP(c, LOC(p), TO_BOOL); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + return SUCCESS; +} + +static int +codegen_pattern_singleton(compiler *c, pattern_ty p, pattern_context *pc) +{ + assert(p->kind == MatchSingleton_kind); + ADDOP_LOAD_CONST(c, LOC(p), p->v.MatchSingleton.value); + ADDOP_COMPARE(c, LOC(p), Is); + RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); + return SUCCESS; +} + +static int +codegen_pattern(compiler *c, pattern_ty p, pattern_context *pc) +{ + switch (p->kind) { + case MatchValue_kind: + return codegen_pattern_value(c, p, pc); + case MatchSingleton_kind: + return codegen_pattern_singleton(c, p, pc); + case MatchSequence_kind: + return codegen_pattern_sequence(c, p, pc); + case MatchMapping_kind: + return codegen_pattern_mapping(c, p, pc); + case MatchClass_kind: + return codegen_pattern_class(c, p, pc); + case MatchStar_kind: + return codegen_pattern_star(c, p, pc); + case MatchAs_kind: + return codegen_pattern_as(c, p, pc); + case MatchOr_kind: + return codegen_pattern_or(c, p, pc); + } + // AST validator shouldn't let this happen, but if it does, + // just fail, don't crash out of the interpreter + const char *e = "invalid match pattern node in AST (kind=%d)"; + return _PyCompile_Error(c, LOC(p), e, p->kind); +} + +static int +codegen_match_inner(compiler *c, stmt_ty s, pattern_context *pc) +{ + VISIT(c, expr, s->v.Match.subject); + NEW_JUMP_TARGET_LABEL(c, end); + Py_ssize_t cases = asdl_seq_LEN(s->v.Match.cases); + assert(cases > 0); + match_case_ty m = asdl_seq_GET(s->v.Match.cases, cases - 1); + int has_default = WILDCARD_CHECK(m->pattern) && 1 < cases; + for (Py_ssize_t i = 0; i < cases - has_default; i++) { + m = asdl_seq_GET(s->v.Match.cases, i); + // Only copy the subject if we're *not* on the last case: + if (i != cases - has_default - 1) { + ADDOP_I(c, LOC(m->pattern), COPY, 1); + } + pc->stores = PyList_New(0); + if (pc->stores == NULL) { + return ERROR; + } + // Irrefutable cases must be either guarded, last, or both: + pc->allow_irrefutable = m->guard != NULL || i == cases - 1; + pc->fail_pop = NULL; + pc->fail_pop_size = 0; + pc->on_top = 0; + // NOTE: Can't use returning macros here (they'll leak pc->stores)! + if (codegen_pattern(c, m->pattern, pc) < 0) { + Py_DECREF(pc->stores); + return ERROR; + } + assert(!pc->on_top); + // It's a match! Store all of the captured names (they're on the stack). + Py_ssize_t nstores = PyList_GET_SIZE(pc->stores); + for (Py_ssize_t n = 0; n < nstores; n++) { + PyObject *name = PyList_GET_ITEM(pc->stores, n); + if (codegen_nameop(c, LOC(m->pattern), name, Store) < 0) { + Py_DECREF(pc->stores); + return ERROR; + } + } + Py_DECREF(pc->stores); + // NOTE: Returning macros are safe again. + if (m->guard) { + RETURN_IF_ERROR(ensure_fail_pop(c, pc, 0)); + RETURN_IF_ERROR(codegen_jump_if(c, LOC(m->pattern), m->guard, pc->fail_pop[0], 0)); + } + // Success! Pop the subject off, we're done with it: + if (i != cases - has_default - 1) { + ADDOP(c, LOC(m->pattern), POP_TOP); + } + VISIT_SEQ(c, stmt, m->body); + ADDOP_JUMP(c, NO_LOCATION, JUMP, end); + // If the pattern fails to match, we want the line number of the + // cleanup to be associated with the failed pattern, not the last line + // of the body + RETURN_IF_ERROR(emit_and_reset_fail_pop(c, LOC(m->pattern), pc)); + } + if (has_default) { + // A trailing "case _" is common, and lets us save a bit of redundant + // pushing and popping in the loop above: + m = asdl_seq_GET(s->v.Match.cases, cases - 1); + if (cases == 1) { + // No matches. Done with the subject: + ADDOP(c, LOC(m->pattern), POP_TOP); + } + else { + // Show line coverage for default case (it doesn't create bytecode) + ADDOP(c, LOC(m->pattern), NOP); + } + if (m->guard) { + RETURN_IF_ERROR(codegen_jump_if(c, LOC(m->pattern), m->guard, end, 0)); + } + VISIT_SEQ(c, stmt, m->body); + } + USE_LABEL(c, end); + return SUCCESS; +} + +static int +codegen_match(compiler *c, stmt_ty s) +{ + pattern_context pc; + pc.fail_pop = NULL; + int result = codegen_match_inner(c, s, &pc); + PyMem_Free(pc.fail_pop); + return result; +} + +#undef WILDCARD_CHECK +#undef WILDCARD_STAR_CHECK + + +int +_PyCodegen_AddReturnAtEnd(compiler *c, int addNone) +{ + /* Make sure every instruction stream that falls off the end returns None. + * This also ensures that no jump target offsets are out of bounds. + */ + if (addNone) { + ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); + } + ADDOP(c, NO_LOCATION, RETURN_VALUE); + return SUCCESS; +} diff --git a/Python/compile.c b/Python/compile.c index 6b17f3bcaf2264..ef470830336dde 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -6,55 +6,24 @@ * object: * 1. Checks for future statements. See future.c * 2. Builds a symbol table. See symtable.c. - * 3. Generate an instruction sequence. See compiler_mod() in this file. + * 3. Generate an instruction sequence. See compiler_mod() in this file, which + * calls functions from codegen.c. * 4. Generate a control flow graph and run optimizations on it. See flowgraph.c. * 5. Assemble the basic blocks into final code. See optimize_and_assemble() in * this file, and assembler.c. * - * Note that compiler_mod() suggests module, but the module ast type - * (mod_ty) has cases for expressions and interactive statements. - * - * CAUTION: The VISIT_* macros abort the current function when they - * encounter a problem. So don't invoke them when there is memory - * which needs to be released. Code blocks are OK, as the compiler - * structure takes care of releasing those. Use the arena to manage - * objects. */ #include #include "Python.h" -#include "opcode.h" -#include "pycore_ast.h" // _PyAST_GetDocString() -#define NEED_OPCODE_TABLES -#include "pycore_opcode_utils.h" -#undef NEED_OPCODE_TABLES -#include "pycore_code.h" // _PyCode_New() +#include "pycore_ast.h" // PyAST_Check, _PyAST_GetDocString() #include "pycore_compile.h" #include "pycore_flowgraph.h" -#include "pycore_intrinsics.h" -#include "pycore_long.h" // _PyLong_GetZero() #include "pycore_pystate.h" // _Py_GetConfig() #include "pycore_setobject.h" // _PySet_NextEntry() -#include "pycore_symtable.h" // PySTEntryObject, _PyFuture_FromAST() - -#define NEED_OPCODE_METADATA -#include "pycore_opcode_metadata.h" // _PyOpcode_opcode_metadata, _PyOpcode_num_popped/pushed -#undef NEED_OPCODE_METADATA -#define COMP_GENEXP 0 -#define COMP_LISTCOMP 1 -#define COMP_SETCOMP 2 -#define COMP_DICTCOMP 3 - -/* A soft limit for stack use, to avoid excessive - * memory use for large constants, etc. - * - * The value 30 is plucked out of thin air. - * Code that could use more stack than this is - * rare, so the exact value is unimportant. - */ -#define STACK_USE_GUIDELINE 30 +#include "cpython/code.h" #undef SUCCESS #undef ERROR @@ -62,293 +31,18 @@ #define ERROR -1 #define RETURN_IF_ERROR(X) \ - if ((X) == -1) { \ - return ERROR; \ - } - -#define IS_TOP_LEVEL_AWAIT(C) ( \ - ((C)->c_flags.cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT) \ - && ((C)->u->u_ste->ste_type == ModuleBlock)) - -typedef _PyCompilerSrcLocation location; + do { \ + if ((X) == -1) { \ + return ERROR; \ + } \ + } while (0) + +typedef _Py_SourceLocation location; +typedef _PyJumpTargetLabel jump_target_label; +typedef _PyInstructionSequence instr_sequence; typedef struct _PyCfgBuilder cfg_builder; - -#define LOCATION(LNO, END_LNO, COL, END_COL) \ - ((const _PyCompilerSrcLocation){(LNO), (END_LNO), (COL), (END_COL)}) - -/* Return true if loc1 starts after loc2 ends. */ -static inline bool -location_is_after(location loc1, location loc2) { - return (loc1.lineno > loc2.end_lineno) || - ((loc1.lineno == loc2.end_lineno) && - (loc1.col_offset > loc2.end_col_offset)); -} - -#define LOC(x) SRC_LOCATION_FROM_AST(x) - -typedef _PyCfgJumpTargetLabel jump_target_label; - -static jump_target_label NO_LABEL = {-1}; - -#define SAME_LABEL(L1, L2) ((L1).id == (L2).id) -#define IS_LABEL(L) (!SAME_LABEL((L), (NO_LABEL))) - -#define NEW_JUMP_TARGET_LABEL(C, NAME) \ - jump_target_label NAME = instr_sequence_new_label(INSTR_SEQUENCE(C)); \ - if (!IS_LABEL(NAME)) { \ - return ERROR; \ - } - -#define USE_LABEL(C, LBL) \ - RETURN_IF_ERROR(_PyCompile_InstructionSequence_UseLabel(INSTR_SEQUENCE(C), (LBL).id)) - - -/* fblockinfo tracks the current frame block. - -A frame block is used to handle loops, try/except, and try/finally. -It's called a frame block to distinguish it from a basic block in the -compiler IR. -*/ - -enum fblocktype { WHILE_LOOP, FOR_LOOP, TRY_EXCEPT, FINALLY_TRY, FINALLY_END, - WITH, ASYNC_WITH, HANDLER_CLEANUP, POP_VALUE, EXCEPTION_HANDLER, - EXCEPTION_GROUP_HANDLER, ASYNC_COMPREHENSION_GENERATOR }; - -struct fblockinfo { - enum fblocktype fb_type; - jump_target_label fb_block; - /* (optional) type-specific exit or cleanup block */ - jump_target_label fb_exit; - /* (optional) additional information required for unwinding */ - void *fb_datum; -}; - -enum { - COMPILER_SCOPE_MODULE, - COMPILER_SCOPE_CLASS, - COMPILER_SCOPE_FUNCTION, - COMPILER_SCOPE_ASYNC_FUNCTION, - COMPILER_SCOPE_LAMBDA, - COMPILER_SCOPE_COMPREHENSION, - COMPILER_SCOPE_TYPEPARAMS, -}; - - -typedef _PyCompile_Instruction instruction; -typedef _PyCompile_InstructionSequence instr_sequence; - -#define INITIAL_INSTR_SEQUENCE_SIZE 100 -#define INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE 10 - -/* - * Resize the array if index is out of range. - * - * idx: the index we want to access - * arr: pointer to the array - * alloc: pointer to the capacity of the array - * default_alloc: initial number of items - * item_size: size of each item - * - */ -int -_PyCompile_EnsureArrayLargeEnough(int idx, void **array, int *alloc, - int default_alloc, size_t item_size) -{ - void *arr = *array; - if (arr == NULL) { - int new_alloc = default_alloc; - if (idx >= new_alloc) { - new_alloc = idx + default_alloc; - } - arr = PyMem_Calloc(new_alloc, item_size); - if (arr == NULL) { - PyErr_NoMemory(); - return ERROR; - } - *alloc = new_alloc; - } - else if (idx >= *alloc) { - size_t oldsize = *alloc * item_size; - int new_alloc = *alloc << 1; - if (idx >= new_alloc) { - new_alloc = idx + default_alloc; - } - size_t newsize = new_alloc * item_size; - - if (oldsize > (SIZE_MAX >> 1)) { - PyErr_NoMemory(); - return ERROR; - } - - assert(newsize > 0); - void *tmp = PyMem_Realloc(arr, newsize); - if (tmp == NULL) { - PyErr_NoMemory(); - return ERROR; - } - *alloc = new_alloc; - arr = tmp; - memset((char *)arr + oldsize, 0, newsize - oldsize); - } - - *array = arr; - return SUCCESS; -} - -static int -instr_sequence_next_inst(instr_sequence *seq) { - assert(seq->s_instrs != NULL || seq->s_used == 0); - - RETURN_IF_ERROR( - _PyCompile_EnsureArrayLargeEnough(seq->s_used + 1, - (void**)&seq->s_instrs, - &seq->s_allocated, - INITIAL_INSTR_SEQUENCE_SIZE, - sizeof(instruction))); - assert(seq->s_allocated >= 0); - assert(seq->s_used < seq->s_allocated); - return seq->s_used++; -} - -static jump_target_label -instr_sequence_new_label(instr_sequence *seq) -{ - jump_target_label lbl = {++seq->s_next_free_label}; - return lbl; -} - -int -_PyCompile_InstructionSequence_UseLabel(instr_sequence *seq, int lbl) -{ - int old_size = seq->s_labelmap_size; - RETURN_IF_ERROR( - _PyCompile_EnsureArrayLargeEnough(lbl, - (void**)&seq->s_labelmap, - &seq->s_labelmap_size, - INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE, - sizeof(int))); - - for(int i = old_size; i < seq->s_labelmap_size; i++) { - seq->s_labelmap[i] = -111; /* something weird, for debugging */ - } - seq->s_labelmap[lbl] = seq->s_used; /* label refers to the next instruction */ - return SUCCESS; -} - - -#define MAX_OPCODE 511 - -int -_PyCompile_InstructionSequence_Addop(instr_sequence *seq, int opcode, int oparg, - location loc) -{ - assert(0 <= opcode && opcode <= MAX_OPCODE); - assert(IS_WITHIN_OPCODE_RANGE(opcode)); - assert(OPCODE_HAS_ARG(opcode) || HAS_TARGET(opcode) || oparg == 0); - assert(0 <= oparg && oparg < (1 << 30)); - - int idx = instr_sequence_next_inst(seq); - RETURN_IF_ERROR(idx); - instruction *ci = &seq->s_instrs[idx]; - ci->i_opcode = opcode; - ci->i_oparg = oparg; - ci->i_loc = loc; - return SUCCESS; -} - -static int -instr_sequence_insert_instruction(instr_sequence *seq, int pos, - int opcode, int oparg, location loc) -{ - assert(pos >= 0 && pos <= seq->s_used); - int last_idx = instr_sequence_next_inst(seq); - RETURN_IF_ERROR(last_idx); - for (int i=last_idx-1; i >= pos; i--) { - seq->s_instrs[i+1] = seq->s_instrs[i]; - } - instruction *ci = &seq->s_instrs[pos]; - ci->i_opcode = opcode; - ci->i_oparg = oparg; - ci->i_loc = loc; - - /* fix the labels map */ - for(int lbl=0; lbl < seq->s_labelmap_size; lbl++) { - if (seq->s_labelmap[lbl] >= pos) { - seq->s_labelmap[lbl]++; - } - } - return SUCCESS; -} - -static void -instr_sequence_fini(instr_sequence *seq) { - PyMem_Free(seq->s_labelmap); - seq->s_labelmap = NULL; - - PyMem_Free(seq->s_instrs); - seq->s_instrs = NULL; -} - -static cfg_builder* -instr_sequence_to_cfg(instr_sequence *seq) { - cfg_builder *g = _PyCfgBuilder_New(); - if (g == NULL) { - return NULL; - } - - /* There can be more than one label for the same offset. The - * offset2lbl maping selects one of them which we use consistently. - */ - - int *offset2lbl = PyMem_Malloc(seq->s_used * sizeof(int)); - if (offset2lbl == NULL) { - PyErr_NoMemory(); - goto error; - } - for (int i = 0; i < seq->s_used; i++) { - offset2lbl[i] = -1; - } - for (int lbl=0; lbl < seq->s_labelmap_size; lbl++) { - int offset = seq->s_labelmap[lbl]; - if (offset >= 0) { - assert(offset < seq->s_used); - offset2lbl[offset] = lbl; - } - } - - for (int i = 0; i < seq->s_used; i++) { - int lbl = offset2lbl[i]; - if (lbl >= 0) { - assert (lbl < seq->s_labelmap_size); - jump_target_label lbl_ = {lbl}; - if (_PyCfgBuilder_UseLabel(g, lbl_) < 0) { - goto error; - } - } - instruction *instr = &seq->s_instrs[i]; - int opcode = instr->i_opcode; - int oparg = instr->i_oparg; - if (HAS_TARGET(opcode)) { - int offset = seq->s_labelmap[oparg]; - assert(offset >= 0 && offset < seq->s_used); - int lbl = offset2lbl[offset]; - assert(lbl >= 0 && lbl < seq->s_labelmap_size); - oparg = lbl; - } - if (_PyCfgBuilder_Addop(g, opcode, oparg, instr->i_loc) < 0) { - goto error; - } - } - if (_PyCfgBuilder_CheckSize(g) < 0) { - goto error; - } - PyMem_Free(offset2lbl); - return g; -error: - _PyCfgBuilder_Free(g); - PyMem_Free(offset2lbl); - return NULL; -} +typedef _PyCompile_FBlockInfo fblockinfo; +typedef enum _PyCompile_FBlockType fblocktype; /* The following items change on entry and exit of code blocks. They must be saved and restored when returning to a block. @@ -358,14 +52,16 @@ struct compiler_unit { int u_scope_type; - PyObject *u_private; /* for private name mangling */ + PyObject *u_private; /* for private name mangling */ + PyObject *u_static_attributes; /* for class: attributes accessed via self.X */ + PyObject *u_deferred_annotations; /* AnnAssign nodes deferred to the end of compilation */ - instr_sequence u_instr_sequence; /* codegen output */ + instr_sequence *u_instr_sequence; /* codegen output */ int u_nfblocks; int u_in_inlined_comp; - struct fblockinfo u_fblock[CO_MAXBLOCKS]; + _PyCompile_FBlockInfo u_fblock[CO_MAXBLOCKS]; _PyCompile_CodeUnitMetadata u_metadata; }; @@ -374,7 +70,7 @@ struct compiler_unit { The u pointer points to the current compilation unit, while units for enclosing blocks are stored in c_stack. The u and c_stack are -managed by compiler_enter_scope() and compiler_exit_scope(). +managed by _PyCompile_EnterScope() and _PyCompile_ExitScope(). Note that we don't track recursion levels during compilation - the task of detecting and rejecting excessive levels of nesting is @@ -382,109 +78,26 @@ handled by the symbol analysis pass. */ -struct compiler { +typedef struct _PyCompiler { PyObject *c_filename; struct symtable *c_st; - PyFutureFeatures c_future; /* module's __future__ */ + _PyFutureFeatures c_future; /* module's __future__ */ PyCompilerFlags c_flags; int c_optimize; /* optimization level */ int c_interactive; /* true if in interactive mode */ - int c_nestlevel; PyObject *c_const_cache; /* Python dict holding all constants, including names tuple */ - struct compiler_unit *u; /* compiler state for current block */ + struct compiler_unit *u; /* compiler state for current block */ PyObject *c_stack; /* Python list holding compiler_unit ptrs */ - PyArena *c_arena; /* pointer to memory allocation arena */ -}; - -#define INSTR_SEQUENCE(C) (&((C)->u->u_instr_sequence)) - - -typedef struct { - // A list of strings corresponding to name captures. It is used to track: - // - Repeated name assignments in the same pattern. - // - Different name assignments in alternatives. - // - The order of name assignments in alternatives. - PyObject *stores; - // If 0, any name captures against our subject will raise. - int allow_irrefutable; - // An array of blocks to jump to on failure. Jumping to fail_pop[i] will pop - // i items off of the stack. The end result looks like this (with each block - // falling through to the next): - // fail_pop[4]: POP_TOP - // fail_pop[3]: POP_TOP - // fail_pop[2]: POP_TOP - // fail_pop[1]: POP_TOP - // fail_pop[0]: NOP - jump_target_label *fail_pop; - // The current length of fail_pop. - Py_ssize_t fail_pop_size; - // The number of items on top of the stack that need to *stay* on top of the - // stack. Variable captures go beneath these. All of them will be popped on - // failure. - Py_ssize_t on_top; -} pattern_context; - -static int codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc); - -static void compiler_free(struct compiler *); -static int compiler_error(struct compiler *, location loc, const char *, ...); -static int compiler_warn(struct compiler *, location loc, const char *, ...); -static int compiler_nameop(struct compiler *, location, identifier, expr_context_ty); - -static PyCodeObject *compiler_mod(struct compiler *, mod_ty); -static int compiler_visit_stmt(struct compiler *, stmt_ty); -static int compiler_visit_keyword(struct compiler *, keyword_ty); -static int compiler_visit_expr(struct compiler *, expr_ty); -static int compiler_augassign(struct compiler *, stmt_ty); -static int compiler_annassign(struct compiler *, stmt_ty); -static int compiler_subscript(struct compiler *, expr_ty); -static int compiler_slice(struct compiler *, expr_ty); - -static bool are_all_items_const(asdl_expr_seq *, Py_ssize_t, Py_ssize_t); - - -static int compiler_with(struct compiler *, stmt_ty, int); -static int compiler_async_with(struct compiler *, stmt_ty, int); -static int compiler_async_for(struct compiler *, stmt_ty); -static int compiler_call_simple_kw_helper(struct compiler *c, - location loc, - asdl_keyword_seq *keywords, - Py_ssize_t nkwelts); -static int compiler_call_helper(struct compiler *c, location loc, - int n, asdl_expr_seq *args, - asdl_keyword_seq *keywords); -static int compiler_try_except(struct compiler *, stmt_ty); -static int compiler_try_star_except(struct compiler *, stmt_ty); -static int compiler_set_qualname(struct compiler *); - -static int compiler_sync_comprehension_generator( - struct compiler *c, location loc, - asdl_comprehension_seq *generators, int gen_index, - int depth, - expr_ty elt, expr_ty val, int type, - int iter_on_stack); - -static int compiler_async_comprehension_generator( - struct compiler *c, location loc, - asdl_comprehension_seq *generators, int gen_index, - int depth, - expr_ty elt, expr_ty val, int type, - int iter_on_stack); - -static int compiler_pattern(struct compiler *, pattern_ty, pattern_context *); -static int compiler_match(struct compiler *, stmt_ty); -static int compiler_pattern_subpattern(struct compiler *, - pattern_ty, pattern_context *); - -static PyCodeObject *optimize_and_assemble(struct compiler *, int addNone); - -#define CAPSULE_NAME "compile.c compiler unit" + bool c_save_nested_seqs; /* if true, construct recursive instruction sequences + * (including instructions for nested code objects) + */ +} compiler; static int -compiler_setup(struct compiler *c, mod_ty mod, PyObject *filename, +compiler_setup(compiler *c, mod_ty mod, PyObject *filename, PyCompilerFlags *flags, int optimize, PyArena *arena) { PyCompilerFlags local_flags = _PyCompilerFlags_INIT; @@ -500,7 +113,6 @@ compiler_setup(struct compiler *c, mod_ty mod, PyObject *filename, } c->c_filename = Py_NewRef(filename); - c->c_arena = arena; if (!_PyFuture_FromAST(mod, filename, &c->c_future)) { return ERROR; } @@ -512,7 +124,7 @@ compiler_setup(struct compiler *c, mod_ty mod, PyObject *filename, flags->cf_flags = merged; c->c_flags = *flags; c->c_optimize = (optimize == -1) ? _Py_GetConfig()->optimization_level : optimize; - c->c_nestlevel = 0; + c->c_save_nested_seqs = false; if (!_PyAST_Optimize(mod, arena, c->c_optimize, merged)) { return ERROR; @@ -527,159 +139,37 @@ compiler_setup(struct compiler *c, mod_ty mod, PyObject *filename, return SUCCESS; } -static struct compiler* -new_compiler(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, - int optimize, PyArena *arena) -{ - struct compiler *c = PyMem_Calloc(1, sizeof(struct compiler)); - if (c == NULL) { - return NULL; - } - if (compiler_setup(c, mod, filename, pflags, optimize, arena) < 0) { - compiler_free(c); - return NULL; - } - return c; -} - -PyCodeObject * -_PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, - int optimize, PyArena *arena) -{ - assert(!PyErr_Occurred()); - struct compiler *c = new_compiler(mod, filename, pflags, optimize, arena); - if (c == NULL) { - return NULL; - } - - PyCodeObject *co = compiler_mod(c, mod); - compiler_free(c); - assert(co || PyErr_Occurred()); - return co; -} - -int -_PyCompile_AstOptimize(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, - int optimize, PyArena *arena) -{ - PyFutureFeatures future; - if (!_PyFuture_FromAST(mod, filename, &future)) { - return -1; - } - int flags = future.ff_features | cf->cf_flags; - if (optimize == -1) { - optimize = _Py_GetConfig()->optimization_level; - } - if (!_PyAST_Optimize(mod, arena, optimize, flags)) { - return -1; - } - return 0; -} - static void -compiler_free(struct compiler *c) +compiler_free(compiler *c) { - if (c->c_st) + if (c->c_st) { _PySymtable_Free(c->c_st); + } Py_XDECREF(c->c_filename); Py_XDECREF(c->c_const_cache); Py_XDECREF(c->c_stack); PyMem_Free(c); } -static PyObject * -list2dict(PyObject *list) -{ - Py_ssize_t i, n; - PyObject *v, *k; - PyObject *dict = PyDict_New(); - if (!dict) return NULL; - - n = PyList_Size(list); - for (i = 0; i < n; i++) { - v = PyLong_FromSsize_t(i); - if (!v) { - Py_DECREF(dict); - return NULL; - } - k = PyList_GET_ITEM(list, i); - if (PyDict_SetItem(dict, k, v) < 0) { - Py_DECREF(v); - Py_DECREF(dict); - return NULL; - } - Py_DECREF(v); - } - return dict; -} - -/* Return new dict containing names from src that match scope(s). - -src is a symbol table dictionary. If the scope of a name matches -either scope_type or flag is set, insert it into the new dict. The -values are integers, starting at offset and increasing by one for -each key. -*/ - -static PyObject * -dictbytype(PyObject *src, int scope_type, int flag, Py_ssize_t offset) +static compiler* +new_compiler(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, + int optimize, PyArena *arena) { - Py_ssize_t i = offset, scope, num_keys, key_i; - PyObject *k, *v, *dest = PyDict_New(); - PyObject *sorted_keys; - - assert(offset >= 0); - if (dest == NULL) - return NULL; - - /* Sort the keys so that we have a deterministic order on the indexes - saved in the returned dictionary. These indexes are used as indexes - into the free and cell var storage. Therefore if they aren't - deterministic, then the generated bytecode is not deterministic. - */ - sorted_keys = PyDict_Keys(src); - if (sorted_keys == NULL) - return NULL; - if (PyList_Sort(sorted_keys) != 0) { - Py_DECREF(sorted_keys); + compiler *c = PyMem_Calloc(1, sizeof(compiler)); + if (c == NULL) { return NULL; } - num_keys = PyList_GET_SIZE(sorted_keys); - - for (key_i = 0; key_i < num_keys; key_i++) { - /* XXX this should probably be a macro in symtable.h */ - long vi; - k = PyList_GET_ITEM(sorted_keys, key_i); - v = PyDict_GetItemWithError(src, k); - assert(v && PyLong_Check(v)); - vi = PyLong_AS_LONG(v); - scope = (vi >> SCOPE_OFFSET) & SCOPE_MASK; - - if (scope == scope_type || vi & flag) { - PyObject *item = PyLong_FromSsize_t(i); - if (item == NULL) { - Py_DECREF(sorted_keys); - Py_DECREF(dest); - return NULL; - } - i++; - if (PyDict_SetItem(dest, k, item) < 0) { - Py_DECREF(sorted_keys); - Py_DECREF(item); - Py_DECREF(dest); - return NULL; - } - Py_DECREF(item); - } + if (compiler_setup(c, mod, filename, pflags, optimize, arena) < 0) { + compiler_free(c); + return NULL; } - Py_DECREF(sorted_keys); - return dest; + return c; } static void compiler_unit_free(struct compiler_unit *u) { - instr_sequence_fini(&u->u_instr_sequence); + Py_CLEAR(u->u_instr_sequence); Py_CLEAR(u->u_ste); Py_CLEAR(u->u_metadata.u_name); Py_CLEAR(u->u_metadata.u_qualname); @@ -690,11 +180,41 @@ compiler_unit_free(struct compiler_unit *u) Py_CLEAR(u->u_metadata.u_cellvars); Py_CLEAR(u->u_metadata.u_fasthidden); Py_CLEAR(u->u_private); + Py_CLEAR(u->u_static_attributes); + Py_CLEAR(u->u_deferred_annotations); PyMem_Free(u); } +#define CAPSULE_NAME "compile.c compiler unit" + +int +_PyCompile_MaybeAddStaticAttributeToClass(compiler *c, expr_ty e) +{ + assert(e->kind == Attribute_kind); + expr_ty attr_value = e->v.Attribute.value; + if (attr_value->kind != Name_kind || + e->v.Attribute.ctx != Store || + !_PyUnicode_EqualToASCIIString(attr_value->v.Name.id, "self")) + { + return SUCCESS; + } + Py_ssize_t stack_size = PyList_GET_SIZE(c->c_stack); + for (Py_ssize_t i = stack_size - 1; i >= 0; i--) { + PyObject *capsule = PyList_GET_ITEM(c->c_stack, i); + struct compiler_unit *u = (struct compiler_unit *)PyCapsule_GetPointer( + capsule, CAPSULE_NAME); + assert(u); + if (u->u_scope_type == COMPILE_SCOPE_CLASS) { + assert(u->u_static_attributes); + RETURN_IF_ERROR(PySet_Add(u->u_static_attributes, e->v.Attribute.attr)); + break; + } + } + return SUCCESS; +} + static int -compiler_set_qualname(struct compiler *c) +compiler_set_qualname(compiler *c) { Py_ssize_t stack_size; struct compiler_unit *u = c->u; @@ -711,8 +231,8 @@ compiler_set_qualname(struct compiler *c) capsule = PyList_GET_ITEM(c->c_stack, stack_size - 1); parent = (struct compiler_unit *)PyCapsule_GetPointer(capsule, CAPSULE_NAME); assert(parent); - if (parent->u_scope_type == COMPILER_SCOPE_TYPEPARAMS) { - /* The parent is a type parameter scope, so we need to + if (parent->u_scope_type == COMPILE_SCOPE_ANNOTATIONS) { + /* The parent is an annotation scope, so we need to look at the grandparent. */ if (stack_size == 2) { // If we're immediately within the module, we can skip @@ -725,9 +245,9 @@ compiler_set_qualname(struct compiler *c) assert(parent); } - if (u->u_scope_type == COMPILER_SCOPE_FUNCTION - || u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION - || u->u_scope_type == COMPILER_SCOPE_CLASS) { + if (u->u_scope_type == COMPILE_SCOPE_FUNCTION + || u->u_scope_type == COMPILE_SCOPE_ASYNC_FUNCTION + || u->u_scope_type == COMPILE_SCOPE_CLASS) { assert(u->u_metadata.u_name); mangled = _Py_Mangle(parent->u_private, u->u_metadata.u_name); if (!mangled) { @@ -736,15 +256,16 @@ compiler_set_qualname(struct compiler *c) scope = _PyST_GetScope(parent->u_ste, mangled); Py_DECREF(mangled); + RETURN_IF_ERROR(scope); assert(scope != GLOBAL_IMPLICIT); if (scope == GLOBAL_EXPLICIT) force_global = 1; } if (!force_global) { - if (parent->u_scope_type == COMPILER_SCOPE_FUNCTION - || parent->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION - || parent->u_scope_type == COMPILER_SCOPE_LAMBDA) + if (parent->u_scope_type == COMPILE_SCOPE_FUNCTION + || parent->u_scope_type == COMPILE_SCOPE_ASYNC_FUNCTION + || parent->u_scope_type == COMPILE_SCOPE_LAMBDA) { _Py_DECLARE_STR(dot_locals, "."); base = PyUnicode_Concat(parent->u_metadata.u_qualname, @@ -760,8 +281,7 @@ compiler_set_qualname(struct compiler *c) } if (base != NULL) { - _Py_DECLARE_STR(dot, "."); - name = PyUnicode_Concat(base, &_Py_STR(dot)); + name = PyUnicode_Concat(base, _Py_LATIN1_CHR('.')); Py_DECREF(base); if (name == NULL) { return ERROR; @@ -779,171 +299,11 @@ compiler_set_qualname(struct compiler *c) return SUCCESS; } -/* Return the stack effect of opcode with argument oparg. - - Some opcodes have different stack effect when jump to the target and - when not jump. The 'jump' parameter specifies the case: - - * 0 -- when not jump - * 1 -- when jump - * -1 -- maximal +/* Merge const *o* and return constant key object. + * If recursive, insert all elements if o is a tuple or frozen set. */ -static int -stack_effect(int opcode, int oparg, int jump) -{ - if (0 <= opcode && opcode <= MAX_REAL_OPCODE) { - if (_PyOpcode_Deopt[opcode] != opcode) { - // Specialized instructions are not supported. - return PY_INVALID_STACK_EFFECT; - } - int popped = _PyOpcode_num_popped(opcode, oparg); - int pushed = _PyOpcode_num_pushed(opcode, oparg); - if (popped < 0 || pushed < 0) { - return PY_INVALID_STACK_EFFECT; - } - return pushed - popped; - } - - // Pseudo ops - switch (opcode) { - case POP_BLOCK: - case JUMP: - case JUMP_NO_INTERRUPT: - return 0; - - case EXIT_INIT_CHECK: - return -1; - - /* Exception handling pseudo-instructions */ - case SETUP_FINALLY: - /* 0 in the normal flow. - * Restore the stack position and push 1 value before jumping to - * the handler if an exception be raised. */ - return jump ? 1 : 0; - case SETUP_CLEANUP: - /* As SETUP_FINALLY, but pushes lasti as well */ - return jump ? 2 : 0; - case SETUP_WITH: - /* 0 in the normal flow. - * Restore the stack position to the position before the result - * of __(a)enter__ and push 2 values before jumping to the handler - * if an exception be raised. */ - return jump ? 1 : 0; - - case STORE_FAST_MAYBE_NULL: - return -1; - case LOAD_CLOSURE: - return 1; - case LOAD_METHOD: - return 1; - case LOAD_SUPER_METHOD: - case LOAD_ZERO_SUPER_METHOD: - case LOAD_ZERO_SUPER_ATTR: - return -1; - default: - return PY_INVALID_STACK_EFFECT; - } - - return PY_INVALID_STACK_EFFECT; /* not reachable */ -} - -int -PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump) -{ - return stack_effect(opcode, oparg, jump); -} - -int -PyCompile_OpcodeStackEffect(int opcode, int oparg) -{ - return stack_effect(opcode, oparg, -1); -} - -int -_PyCompile_OpcodeIsValid(int opcode) -{ - return IS_VALID_OPCODE(opcode); -} - -int -_PyCompile_OpcodeHasArg(int opcode) -{ - return OPCODE_HAS_ARG(opcode); -} - -int -_PyCompile_OpcodeHasConst(int opcode) -{ - return OPCODE_HAS_CONST(opcode); -} - -int -_PyCompile_OpcodeHasName(int opcode) -{ - return OPCODE_HAS_NAME(opcode); -} - -int -_PyCompile_OpcodeHasJump(int opcode) -{ - return OPCODE_HAS_JUMP(opcode); -} - -int -_PyCompile_OpcodeHasFree(int opcode) -{ - return OPCODE_HAS_FREE(opcode); -} - -int -_PyCompile_OpcodeHasLocal(int opcode) -{ - return OPCODE_HAS_LOCAL(opcode); -} - -int -_PyCompile_OpcodeHasExc(int opcode) -{ - return IS_BLOCK_PUSH_OPCODE(opcode); -} - -static int -codegen_addop_noarg(instr_sequence *seq, int opcode, location loc) -{ - assert(!OPCODE_HAS_ARG(opcode)); - assert(!IS_ASSEMBLER_OPCODE(opcode)); - return _PyCompile_InstructionSequence_Addop(seq, opcode, 0, loc); -} - -static Py_ssize_t -dict_add_o(PyObject *dict, PyObject *o) -{ - PyObject *v; - Py_ssize_t arg; - - if (PyDict_GetItemRef(dict, o, &v) < 0) { - return ERROR; - } - if (!v) { - arg = PyDict_GET_SIZE(dict); - v = PyLong_FromSsize_t(arg); - if (!v) { - return ERROR; - } - if (PyDict_SetItem(dict, o, v) < 0) { - Py_DECREF(v); - return ERROR; - } - } - else - arg = PyLong_AsLong(v); - Py_DECREF(v); - return arg; -} - -// Merge const *o* recursively and return constant key object. -static PyObject* -merge_consts_recursive(PyObject *const_cache, PyObject *o) +static PyObject* +const_cache_insert(PyObject *const_cache, PyObject *o, bool recursive) { assert(PyDict_CheckExact(const_cache)); // None and Ellipsis are immortal objects, and key is the singleton. @@ -967,6 +327,10 @@ merge_consts_recursive(PyObject *const_cache, PyObject *o) } Py_DECREF(t); + if (!recursive) { + return key; + } + // We registered o in const_cache. // When o is a tuple or frozenset, we want to merge its // items too. @@ -974,7 +338,7 @@ merge_consts_recursive(PyObject *const_cache, PyObject *o) Py_ssize_t len = PyTuple_GET_SIZE(o); for (Py_ssize_t i = 0; i < len; i++) { PyObject *item = PyTuple_GET_ITEM(o, i); - PyObject *u = merge_consts_recursive(const_cache, item); + PyObject *u = const_cache_insert(const_cache, item, recursive); if (u == NULL) { Py_DECREF(key); return NULL; @@ -1016,7 +380,7 @@ merge_consts_recursive(PyObject *const_cache, PyObject *o) PyObject *item; Py_hash_t hash; while (_PySet_NextEntry(o, &pos, &item, &hash)) { - PyObject *k = merge_consts_recursive(const_cache, item); + PyObject *k = const_cache_insert(const_cache, item, recursive); if (k == NULL) { Py_DECREF(tuple); Py_DECREF(key); @@ -1050,227 +414,171 @@ merge_consts_recursive(PyObject *const_cache, PyObject *o) return key; } -static Py_ssize_t -compiler_add_const(PyObject *const_cache, struct compiler_unit *u, PyObject *o) +static PyObject* +merge_consts_recursive(PyObject *const_cache, PyObject *o) { - assert(PyDict_CheckExact(const_cache)); - PyObject *key = merge_consts_recursive(const_cache, o); - if (key == NULL) { - return ERROR; - } - - Py_ssize_t arg = dict_add_o(u->u_metadata.u_consts, key); - Py_DECREF(key); - return arg; + return const_cache_insert(const_cache, o, true); } -static int -compiler_addop_load_const(PyObject *const_cache, struct compiler_unit *u, location loc, PyObject *o) +Py_ssize_t +_PyCompile_DictAddObj(PyObject *dict, PyObject *o) { - Py_ssize_t arg = compiler_add_const(const_cache, u, o); - if (arg < 0) { - return ERROR; - } - return codegen_addop_i(&u->u_instr_sequence, LOAD_CONST, arg, loc); -} + PyObject *v; + Py_ssize_t arg; -static int -compiler_addop_o(struct compiler_unit *u, location loc, - int opcode, PyObject *dict, PyObject *o) -{ - Py_ssize_t arg = dict_add_o(dict, o); - if (arg < 0) { + if (PyDict_GetItemRef(dict, o, &v) < 0) { return ERROR; } - return codegen_addop_i(&u->u_instr_sequence, opcode, arg, loc); + if (!v) { + arg = PyDict_GET_SIZE(dict); + v = PyLong_FromSsize_t(arg); + if (!v) { + return ERROR; + } + if (PyDict_SetItem(dict, o, v) < 0) { + Py_DECREF(v); + return ERROR; + } + } + else + arg = PyLong_AsLong(v); + Py_DECREF(v); + return arg; } -static int -compiler_addop_name(struct compiler_unit *u, location loc, - int opcode, PyObject *dict, PyObject *o) +Py_ssize_t +_PyCompile_AddConst(compiler *c, PyObject *o) { - PyObject *mangled = _Py_Mangle(u->u_private, o); - if (!mangled) { - return ERROR; - } - Py_ssize_t arg = dict_add_o(dict, mangled); - Py_DECREF(mangled); - if (arg < 0) { + PyObject *key = merge_consts_recursive(c->c_const_cache, o); + if (key == NULL) { return ERROR; } - if (opcode == LOAD_ATTR) { - arg <<= 1; - } - if (opcode == LOAD_METHOD) { - assert(is_pseudo_target(LOAD_METHOD, LOAD_ATTR)); - opcode = LOAD_ATTR; - arg <<= 1; - arg |= 1; - } - if (opcode == LOAD_SUPER_ATTR) { - arg <<= 2; - arg |= 2; - } - if (opcode == LOAD_SUPER_METHOD) { - assert(is_pseudo_target(LOAD_SUPER_METHOD, LOAD_SUPER_ATTR)); - opcode = LOAD_SUPER_ATTR; - arg <<= 2; - arg |= 3; - } - if (opcode == LOAD_ZERO_SUPER_ATTR) { - assert(is_pseudo_target(LOAD_ZERO_SUPER_ATTR, LOAD_SUPER_ATTR)); - opcode = LOAD_SUPER_ATTR; - arg <<= 2; - } - if (opcode == LOAD_ZERO_SUPER_METHOD) { - assert(is_pseudo_target(LOAD_ZERO_SUPER_METHOD, LOAD_SUPER_ATTR)); - opcode = LOAD_SUPER_ATTR; - arg <<= 2; - arg |= 1; - } - return codegen_addop_i(&u->u_instr_sequence, opcode, arg, loc); -} - -/* Add an opcode with an integer argument */ -static int -codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc) -{ - /* oparg value is unsigned, but a signed C int is usually used to store - it in the C code (like Python/ceval.c). - - Limit to 32-bit signed C int (rather than INT_MAX) for portability. - The argument of a concrete bytecode instruction is limited to 8-bit. - EXTENDED_ARG is used for 16, 24, and 32-bit arguments. */ - - int oparg_ = Py_SAFE_DOWNCAST(oparg, Py_ssize_t, int); - assert(!IS_ASSEMBLER_OPCODE(opcode)); - return _PyCompile_InstructionSequence_Addop(seq, opcode, oparg_, loc); + Py_ssize_t arg = _PyCompile_DictAddObj(c->u->u_metadata.u_consts, key); + Py_DECREF(key); + return arg; } -static int -codegen_addop_j(instr_sequence *seq, location loc, - int opcode, jump_target_label target) +static PyObject * +list2dict(PyObject *list) { - assert(IS_LABEL(target)); - assert(OPCODE_HAS_JUMP(opcode) || IS_BLOCK_PUSH_OPCODE(opcode)); - assert(!IS_ASSEMBLER_OPCODE(opcode)); - return _PyCompile_InstructionSequence_Addop(seq, opcode, target.id, loc); -} - -#define RETURN_IF_ERROR_IN_SCOPE(C, CALL) { \ - if ((CALL) < 0) { \ - compiler_exit_scope((C)); \ - return ERROR; \ - } \ -} - -#define ADDOP(C, LOC, OP) \ - RETURN_IF_ERROR(codegen_addop_noarg(INSTR_SEQUENCE(C), (OP), (LOC))) - -#define ADDOP_IN_SCOPE(C, LOC, OP) RETURN_IF_ERROR_IN_SCOPE((C), codegen_addop_noarg(INSTR_SEQUENCE(C), (OP), (LOC))) - -#define ADDOP_LOAD_CONST(C, LOC, O) \ - RETURN_IF_ERROR(compiler_addop_load_const((C)->c_const_cache, (C)->u, (LOC), (O))) - -/* Same as ADDOP_LOAD_CONST, but steals a reference. */ -#define ADDOP_LOAD_CONST_NEW(C, LOC, O) { \ - PyObject *__new_const = (O); \ - if (__new_const == NULL) { \ - return ERROR; \ - } \ - if (compiler_addop_load_const((C)->c_const_cache, (C)->u, (LOC), __new_const) < 0) { \ - Py_DECREF(__new_const); \ - return ERROR; \ - } \ - Py_DECREF(__new_const); \ -} + Py_ssize_t i, n; + PyObject *v, *k; + PyObject *dict = PyDict_New(); + if (!dict) return NULL; -#define ADDOP_N(C, LOC, OP, O, TYPE) { \ - assert(!OPCODE_HAS_CONST(OP)); /* use ADDOP_LOAD_CONST_NEW */ \ - if (compiler_addop_o((C)->u, (LOC), (OP), (C)->u->u_metadata.u_ ## TYPE, (O)) < 0) { \ - Py_DECREF((O)); \ - return ERROR; \ - } \ - Py_DECREF((O)); \ + n = PyList_Size(list); + for (i = 0; i < n; i++) { + v = PyLong_FromSsize_t(i); + if (!v) { + Py_DECREF(dict); + return NULL; + } + k = PyList_GET_ITEM(list, i); + if (PyDict_SetItem(dict, k, v) < 0) { + Py_DECREF(v); + Py_DECREF(dict); + return NULL; + } + Py_DECREF(v); + } + return dict; } -#define ADDOP_NAME(C, LOC, OP, O, TYPE) \ - RETURN_IF_ERROR(compiler_addop_name((C)->u, (LOC), (OP), (C)->u->u_metadata.u_ ## TYPE, (O))) - -#define ADDOP_I(C, LOC, OP, O) \ - RETURN_IF_ERROR(codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))) - -#define ADDOP_JUMP(C, LOC, OP, O) \ - RETURN_IF_ERROR(codegen_addop_j(INSTR_SEQUENCE(C), (LOC), (OP), (O))) - -#define ADDOP_COMPARE(C, LOC, CMP) \ - RETURN_IF_ERROR(compiler_addcompare((C), (LOC), (cmpop_ty)(CMP))) - -#define ADDOP_BINARY(C, LOC, BINOP) \ - RETURN_IF_ERROR(addop_binary((C), (LOC), (BINOP), false)) - -#define ADDOP_INPLACE(C, LOC, BINOP) \ - RETURN_IF_ERROR(addop_binary((C), (LOC), (BINOP), true)) - -#define ADD_YIELD_FROM(C, LOC, await) \ - RETURN_IF_ERROR(compiler_add_yield_from((C), (LOC), (await))) - -#define POP_EXCEPT_AND_RERAISE(C, LOC) \ - RETURN_IF_ERROR(compiler_pop_except_and_reraise((C), (LOC))) - -#define ADDOP_YIELD(C, LOC) \ - RETURN_IF_ERROR(addop_yield((C), (LOC))) +/* Return new dict containing names from src that match scope(s). -/* VISIT and VISIT_SEQ takes an ASDL type as their second argument. They use - the ASDL name to synthesize the name of the C type and the visit function. +src is a symbol table dictionary. If the scope of a name matches +either scope_type or flag is set, insert it into the new dict. The +values are integers, starting at offset and increasing by one for +each key. */ -#define VISIT(C, TYPE, V) \ - RETURN_IF_ERROR(compiler_visit_ ## TYPE((C), (V))); +static PyObject * +dictbytype(PyObject *src, int scope_type, int flag, Py_ssize_t offset) +{ + Py_ssize_t i = offset, num_keys, key_i; + PyObject *k, *v, *dest = PyDict_New(); + PyObject *sorted_keys; -#define VISIT_IN_SCOPE(C, TYPE, V) \ - RETURN_IF_ERROR_IN_SCOPE((C), compiler_visit_ ## TYPE((C), (V))) + assert(offset >= 0); + if (dest == NULL) + return NULL; -#define VISIT_SEQ(C, TYPE, SEQ) { \ - int _i; \ - asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ - for (_i = 0; _i < asdl_seq_LEN(seq); _i++) { \ - TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i); \ - RETURN_IF_ERROR(compiler_visit_ ## TYPE((C), elt)); \ - } \ -} + /* Sort the keys so that we have a deterministic order on the indexes + saved in the returned dictionary. These indexes are used as indexes + into the free and cell var storage. Therefore if they aren't + deterministic, then the generated bytecode is not deterministic. + */ + sorted_keys = PyDict_Keys(src); + if (sorted_keys == NULL) { + Py_DECREF(dest); + return NULL; + } + if (PyList_Sort(sorted_keys) != 0) { + Py_DECREF(sorted_keys); + Py_DECREF(dest); + return NULL; + } + num_keys = PyList_GET_SIZE(sorted_keys); -#define VISIT_SEQ_IN_SCOPE(C, TYPE, SEQ) { \ - int _i; \ - asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ - for (_i = 0; _i < asdl_seq_LEN(seq); _i++) { \ - TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i); \ - if (compiler_visit_ ## TYPE((C), elt) < 0) { \ - compiler_exit_scope(C); \ - return ERROR; \ - } \ - } \ + for (key_i = 0; key_i < num_keys; key_i++) { + k = PyList_GET_ITEM(sorted_keys, key_i); + v = PyDict_GetItemWithError(src, k); + if (!v) { + if (!PyErr_Occurred()) { + PyErr_SetObject(PyExc_KeyError, k); + } + Py_DECREF(sorted_keys); + Py_DECREF(dest); + return NULL; + } + long vi = PyLong_AsLong(v); + if (vi == -1 && PyErr_Occurred()) { + Py_DECREF(sorted_keys); + Py_DECREF(dest); + return NULL; + } + if (SYMBOL_TO_SCOPE(vi) == scope_type || vi & flag) { + PyObject *item = PyLong_FromSsize_t(i); + if (item == NULL) { + Py_DECREF(sorted_keys); + Py_DECREF(dest); + return NULL; + } + i++; + if (PyDict_SetItem(dest, k, item) < 0) { + Py_DECREF(sorted_keys); + Py_DECREF(item); + Py_DECREF(dest); + return NULL; + } + Py_DECREF(item); + } + } + Py_DECREF(sorted_keys); + return dest; } - -static int -compiler_enter_scope(struct compiler *c, identifier name, - int scope_type, void *key, int lineno) +int +_PyCompile_EnterScope(compiler *c, identifier name, int scope_type, + void *key, int lineno, PyObject *private, + _PyCompile_CodeUnitMetadata *umd) { - location loc = LOCATION(lineno, lineno, 0, 0); - struct compiler_unit *u; - u = (struct compiler_unit *)PyMem_Calloc(1, sizeof(struct compiler_unit)); if (!u) { PyErr_NoMemory(); return ERROR; } u->u_scope_type = scope_type; - u->u_metadata.u_argcount = 0; - u->u_metadata.u_posonlyargcount = 0; - u->u_metadata.u_kwonlyargcount = 0; + if (umd != NULL) { + u->u_metadata = *umd; + } + else { + u->u_metadata.u_argcount = 0; + u->u_metadata.u_posonlyargcount = 0; + u->u_metadata.u_kwonlyargcount = 0; + } u->u_ste = _PySymtable_Lookup(c->c_st, key); if (!u->u_ste) { compiler_unit_free(u); @@ -1290,8 +598,8 @@ compiler_enter_scope(struct compiler *c, identifier name, if (u->u_ste->ste_needs_class_closure) { /* Cook up an implicit __class__ cell. */ Py_ssize_t res; - assert(u->u_scope_type == COMPILER_SCOPE_CLASS); - res = dict_add_o(u->u_metadata.u_cellvars, &_Py_ID(__class__)); + assert(u->u_scope_type == COMPILE_SCOPE_CLASS); + res = _PyCompile_DictAddObj(u->u_metadata.u_cellvars, &_Py_ID(__class__)); if (res < 0) { compiler_unit_free(u); return ERROR; @@ -1300,8 +608,8 @@ compiler_enter_scope(struct compiler *c, identifier name, if (u->u_ste->ste_needs_classdict) { /* Cook up an implicit __classdict__ cell. */ Py_ssize_t res; - assert(u->u_scope_type == COMPILER_SCOPE_CLASS); - res = dict_add_o(u->u_metadata.u_cellvars, &_Py_ID(__classdict__)); + assert(u->u_scope_type == COMPILE_SCOPE_CLASS); + res = _PyCompile_DictAddObj(u->u_metadata.u_cellvars, &_Py_ID(__classdict__)); if (res < 0) { compiler_unit_free(u); return ERROR; @@ -1335,7 +643,23 @@ compiler_enter_scope(struct compiler *c, identifier name, return ERROR; } - u->u_private = NULL; + u->u_deferred_annotations = NULL; + if (scope_type == COMPILE_SCOPE_CLASS) { + u->u_static_attributes = PySet_New(0); + if (!u->u_static_attributes) { + compiler_unit_free(u); + return ERROR; + } + } + else { + u->u_static_attributes = NULL; + } + + u->u_instr_sequence = (instr_sequence*)_PyInstructionSequence_New(); + if (!u->u_instr_sequence) { + compiler_unit_free(u); + return ERROR; + } /* Push the old compiler_unit on the stack. */ if (c->u) { @@ -1346,33 +670,31 @@ compiler_enter_scope(struct compiler *c, identifier name, return ERROR; } Py_DECREF(capsule); - u->u_private = Py_XNewRef(c->u->u_private); + if (private == NULL) { + private = c->u->u_private; + } } - c->u = u; - c->c_nestlevel++; + u->u_private = Py_XNewRef(private); - if (u->u_scope_type == COMPILER_SCOPE_MODULE) { - loc.lineno = 0; - } - else { + c->u = u; + if (scope_type != COMPILE_SCOPE_MODULE) { RETURN_IF_ERROR(compiler_set_qualname(c)); } - ADDOP_I(c, loc, RESUME, RESUME_AT_FUNC_START); - - if (u->u_scope_type == COMPILER_SCOPE_MODULE) { - loc.lineno = -1; - } return SUCCESS; } -static void -compiler_exit_scope(struct compiler *c) +void +_PyCompile_ExitScope(compiler *c) { // Don't call PySequence_DelItem() with an exception raised PyObject *exc = PyErr_GetRaisedException(); - c->c_nestlevel--; + instr_sequence *nested_seq = NULL; + if (c->c_save_nested_seqs) { + nested_seq = c->u->u_instr_sequence; + Py_INCREF(nested_seq); + } compiler_unit_free(c->u); /* Restore c->u to the parent unit. */ Py_ssize_t n = PyList_GET_SIZE(c->c_stack) - 1; @@ -1385,5132 +707,415 @@ compiler_exit_scope(struct compiler *c) PyErr_FormatUnraisable("Exception ignored on removing " "the last compiler stack item"); } + if (nested_seq != NULL) { + if (_PyInstructionSequence_AddNested(c->u->u_instr_sequence, nested_seq) < 0) { + PyErr_FormatUnraisable("Exception ignored on appending " + "nested instruction sequence"); + } + } } else { c->u = NULL; } + Py_XDECREF(nested_seq); PyErr_SetRaisedException(exc); } -/* Search if variable annotations are present statically in a block. */ +/* + * Frame block handling functions + */ -static bool -find_ann(asdl_stmt_seq *stmts) +int +_PyCompile_PushFBlock(compiler *c, location loc, + fblocktype t, jump_target_label block_label, + jump_target_label exit, void *datum) { - int i, j, res = 0; - stmt_ty st; - - for (i = 0; i < asdl_seq_LEN(stmts); i++) { - st = (stmt_ty)asdl_seq_GET(stmts, i); - switch (st->kind) { - case AnnAssign_kind: - return true; - case For_kind: - res = find_ann(st->v.For.body) || - find_ann(st->v.For.orelse); - break; - case AsyncFor_kind: - res = find_ann(st->v.AsyncFor.body) || - find_ann(st->v.AsyncFor.orelse); - break; - case While_kind: - res = find_ann(st->v.While.body) || - find_ann(st->v.While.orelse); - break; - case If_kind: - res = find_ann(st->v.If.body) || - find_ann(st->v.If.orelse); - break; - case With_kind: - res = find_ann(st->v.With.body); - break; - case AsyncWith_kind: - res = find_ann(st->v.AsyncWith.body); - break; - case Try_kind: - for (j = 0; j < asdl_seq_LEN(st->v.Try.handlers); j++) { - excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( - st->v.Try.handlers, j); - if (find_ann(handler->v.ExceptHandler.body)) { - return true; - } - } - res = find_ann(st->v.Try.body) || - find_ann(st->v.Try.finalbody) || - find_ann(st->v.Try.orelse); - break; - case TryStar_kind: - for (j = 0; j < asdl_seq_LEN(st->v.TryStar.handlers); j++) { - excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( - st->v.TryStar.handlers, j); - if (find_ann(handler->v.ExceptHandler.body)) { - return true; - } - } - res = find_ann(st->v.TryStar.body) || - find_ann(st->v.TryStar.finalbody) || - find_ann(st->v.TryStar.orelse); - break; - case Match_kind: - for (j = 0; j < asdl_seq_LEN(st->v.Match.cases); j++) { - match_case_ty match_case = (match_case_ty)asdl_seq_GET( - st->v.Match.cases, j); - if (find_ann(match_case->body)) { - return true; - } - } - break; - default: - res = false; - break; - } - if (res) { - break; - } - } - return res; -} - -/* - * Frame block handling functions - */ - -static int -compiler_push_fblock(struct compiler *c, location loc, - enum fblocktype t, jump_target_label block_label, - jump_target_label exit, void *datum) -{ - struct fblockinfo *f; + fblockinfo *f; if (c->u->u_nfblocks >= CO_MAXBLOCKS) { - return compiler_error(c, loc, "too many statically nested blocks"); + return _PyCompile_Error(c, loc, "too many statically nested blocks"); } f = &c->u->u_fblock[c->u->u_nfblocks++]; f->fb_type = t; f->fb_block = block_label; + f->fb_loc = loc; f->fb_exit = exit; f->fb_datum = datum; return SUCCESS; } -static void -compiler_pop_fblock(struct compiler *c, enum fblocktype t, jump_target_label block_label) +void +_PyCompile_PopFBlock(compiler *c, fblocktype t, jump_target_label block_label) { struct compiler_unit *u = c->u; assert(u->u_nfblocks > 0); u->u_nfblocks--; assert(u->u_fblock[u->u_nfblocks].fb_type == t); - assert(SAME_LABEL(u->u_fblock[u->u_nfblocks].fb_block, block_label)); -} - -static int -compiler_call_exit_with_nones(struct compiler *c, location loc) -{ - ADDOP_LOAD_CONST(c, loc, Py_None); - ADDOP_LOAD_CONST(c, loc, Py_None); - ADDOP_LOAD_CONST(c, loc, Py_None); - ADDOP_I(c, loc, CALL, 2); - return SUCCESS; -} - -static int -compiler_add_yield_from(struct compiler *c, location loc, int await) -{ - NEW_JUMP_TARGET_LABEL(c, send); - NEW_JUMP_TARGET_LABEL(c, fail); - NEW_JUMP_TARGET_LABEL(c, exit); - - USE_LABEL(c, send); - ADDOP_JUMP(c, loc, SEND, exit); - // Set up a virtual try/except to handle when StopIteration is raised during - // a close or throw call. The only way YIELD_VALUE raises if they do! - ADDOP_JUMP(c, loc, SETUP_FINALLY, fail); - ADDOP_I(c, loc, YIELD_VALUE, 1); - ADDOP(c, NO_LOCATION, POP_BLOCK); - ADDOP_I(c, loc, RESUME, await ? RESUME_AFTER_AWAIT : RESUME_AFTER_YIELD_FROM); - ADDOP_JUMP(c, loc, JUMP_NO_INTERRUPT, send); - - USE_LABEL(c, fail); - ADDOP(c, loc, CLEANUP_THROW); - - USE_LABEL(c, exit); - ADDOP(c, loc, END_SEND); - return SUCCESS; -} - -static int -compiler_pop_except_and_reraise(struct compiler *c, location loc) -{ - /* Stack contents - * [exc_info, lasti, exc] COPY 3 - * [exc_info, lasti, exc, exc_info] POP_EXCEPT - * [exc_info, lasti, exc] RERAISE 1 - * (exception_unwind clears the stack) - */ - - ADDOP_I(c, loc, COPY, 3); - ADDOP(c, loc, POP_EXCEPT); - ADDOP_I(c, loc, RERAISE, 1); - return SUCCESS; -} - -/* Unwind a frame block. If preserve_tos is true, the TOS before - * popping the blocks will be restored afterwards, unless another - * return, break or continue is found. In which case, the TOS will - * be popped. - */ -static int -compiler_unwind_fblock(struct compiler *c, location *ploc, - struct fblockinfo *info, int preserve_tos) -{ - switch (info->fb_type) { - case WHILE_LOOP: - case EXCEPTION_HANDLER: - case EXCEPTION_GROUP_HANDLER: - case ASYNC_COMPREHENSION_GENERATOR: - return SUCCESS; - - case FOR_LOOP: - /* Pop the iterator */ - if (preserve_tos) { - ADDOP_I(c, *ploc, SWAP, 2); - } - ADDOP(c, *ploc, POP_TOP); - return SUCCESS; - - case TRY_EXCEPT: - ADDOP(c, *ploc, POP_BLOCK); - return SUCCESS; - - case FINALLY_TRY: - /* This POP_BLOCK gets the line number of the unwinding statement */ - ADDOP(c, *ploc, POP_BLOCK); - if (preserve_tos) { - RETURN_IF_ERROR( - compiler_push_fblock(c, *ploc, POP_VALUE, NO_LABEL, NO_LABEL, NULL)); - } - /* Emit the finally block */ - VISIT_SEQ(c, stmt, info->fb_datum); - if (preserve_tos) { - compiler_pop_fblock(c, POP_VALUE, NO_LABEL); - } - /* The finally block should appear to execute after the - * statement causing the unwinding, so make the unwinding - * instruction artificial */ - *ploc = NO_LOCATION; - return SUCCESS; - - case FINALLY_END: - if (preserve_tos) { - ADDOP_I(c, *ploc, SWAP, 2); - } - ADDOP(c, *ploc, POP_TOP); /* exc_value */ - if (preserve_tos) { - ADDOP_I(c, *ploc, SWAP, 2); - } - ADDOP(c, *ploc, POP_BLOCK); - ADDOP(c, *ploc, POP_EXCEPT); - return SUCCESS; - - case WITH: - case ASYNC_WITH: - *ploc = LOC((stmt_ty)info->fb_datum); - ADDOP(c, *ploc, POP_BLOCK); - if (preserve_tos) { - ADDOP_I(c, *ploc, SWAP, 2); - } - RETURN_IF_ERROR(compiler_call_exit_with_nones(c, *ploc)); - if (info->fb_type == ASYNC_WITH) { - ADDOP_I(c, *ploc, GET_AWAITABLE, 2); - ADDOP_LOAD_CONST(c, *ploc, Py_None); - ADD_YIELD_FROM(c, *ploc, 1); - } - ADDOP(c, *ploc, POP_TOP); - /* The exit block should appear to execute after the - * statement causing the unwinding, so make the unwinding - * instruction artificial */ - *ploc = NO_LOCATION; - return SUCCESS; - - case HANDLER_CLEANUP: { - if (info->fb_datum) { - ADDOP(c, *ploc, POP_BLOCK); - } - if (preserve_tos) { - ADDOP_I(c, *ploc, SWAP, 2); - } - ADDOP(c, *ploc, POP_BLOCK); - ADDOP(c, *ploc, POP_EXCEPT); - if (info->fb_datum) { - ADDOP_LOAD_CONST(c, *ploc, Py_None); - RETURN_IF_ERROR(compiler_nameop(c, *ploc, info->fb_datum, Store)); - RETURN_IF_ERROR(compiler_nameop(c, *ploc, info->fb_datum, Del)); - } - return SUCCESS; - } - case POP_VALUE: { - if (preserve_tos) { - ADDOP_I(c, *ploc, SWAP, 2); - } - ADDOP(c, *ploc, POP_TOP); - return SUCCESS; - } - } - Py_UNREACHABLE(); + assert(SAME_JUMP_TARGET_LABEL(u->u_fblock[u->u_nfblocks].fb_block, block_label)); } -/** Unwind block stack. If loop is not NULL, then stop when the first loop is encountered. */ -static int -compiler_unwind_fblock_stack(struct compiler *c, location *ploc, - int preserve_tos, struct fblockinfo **loop) +fblockinfo * +_PyCompile_TopFBlock(compiler *c) { if (c->u->u_nfblocks == 0) { - return SUCCESS; - } - struct fblockinfo *top = &c->u->u_fblock[c->u->u_nfblocks-1]; - if (top->fb_type == EXCEPTION_GROUP_HANDLER) { - return compiler_error( - c, *ploc, "'break', 'continue' and 'return' cannot appear in an except* block"); - } - if (loop != NULL && (top->fb_type == WHILE_LOOP || top->fb_type == FOR_LOOP)) { - *loop = top; - return SUCCESS; + return NULL; } - struct fblockinfo copy = *top; - c->u->u_nfblocks--; - RETURN_IF_ERROR(compiler_unwind_fblock(c, ploc, ©, preserve_tos)); - RETURN_IF_ERROR(compiler_unwind_fblock_stack(c, ploc, preserve_tos, loop)); - c->u->u_fblock[c->u->u_nfblocks] = copy; - c->u->u_nfblocks++; - return SUCCESS; + return &c->u->u_fblock[c->u->u_nfblocks - 1]; } -/* Compile a sequence of statements, checking for a docstring - and for annotations. */ - -static int -compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) +PyObject * +_PyCompile_DeferredAnnotations(compiler *c) { + return c->u->u_deferred_annotations; +} - /* Set current line number to the line number of first statement. - This way line number for SETUP_ANNOTATIONS will always - coincide with the line number of first "real" statement in module. - If body is empty, then lineno will be set later in optimize_and_assemble. */ - if (c->u->u_scope_type == COMPILER_SCOPE_MODULE && asdl_seq_LEN(stmts)) { +static location +start_location(asdl_stmt_seq *stmts) +{ + if (asdl_seq_LEN(stmts) > 0) { + /* Set current line number to the line number of first statement. + * This way line number for SETUP_ANNOTATIONS will always + * coincide with the line number of first "real" statement in module. + * If body is empty, then lineno will be set later in the assembly stage. + */ stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); - loc = LOC(st); - } - /* Every annotated class and module should have __annotations__. */ - if (find_ann(stmts)) { - ADDOP(c, loc, SETUP_ANNOTATIONS); - } - if (!asdl_seq_LEN(stmts)) { - return SUCCESS; - } - Py_ssize_t first_instr = 0; - PyObject *docstring = _PyAST_GetDocString(stmts); - if (docstring) { - first_instr = 1; - /* if not -OO mode, set docstring */ - if (c->c_optimize < 2) { - PyObject *cleandoc = _PyCompile_CleanDoc(docstring); - if (cleandoc == NULL) { - return ERROR; - } - stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); - assert(st->kind == Expr_kind); - location loc = LOC(st->v.Expr.value); - ADDOP_LOAD_CONST(c, loc, cleandoc); - Py_DECREF(cleandoc); - RETURN_IF_ERROR(compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)); - } - } - for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(stmts); i++) { - VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i)); + return SRC_LOCATION_FROM_AST(st); } - return SUCCESS; + return (const _Py_SourceLocation){1, 1, 0, 0}; } static int -compiler_codegen(struct compiler *c, mod_ty mod) +compiler_codegen(compiler *c, mod_ty mod) { - location loc = LOCATION(1, 1, 0, 0); + RETURN_IF_ERROR(_PyCodegen_EnterAnonymousScope(c, mod)); + assert(c->u->u_scope_type == COMPILE_SCOPE_MODULE); switch (mod->kind) { - case Module_kind: - if (compiler_body(c, loc, mod->v.Module.body) < 0) { - return ERROR; - } + case Module_kind: { + asdl_stmt_seq *stmts = mod->v.Module.body; + RETURN_IF_ERROR(_PyCodegen_Body(c, start_location(stmts), stmts, false)); break; - case Interactive_kind: - if (find_ann(mod->v.Interactive.body)) { - ADDOP(c, loc, SETUP_ANNOTATIONS); - } + } + case Interactive_kind: { c->c_interactive = 1; - VISIT_SEQ(c, stmt, mod->v.Interactive.body); + asdl_stmt_seq *stmts = mod->v.Interactive.body; + RETURN_IF_ERROR(_PyCodegen_Body(c, start_location(stmts), stmts, true)); break; - case Expression_kind: - VISIT(c, expr, mod->v.Expression.body); + } + case Expression_kind: { + RETURN_IF_ERROR(_PyCodegen_Expression(c, mod->v.Expression.body)); break; - default: + } + default: { PyErr_Format(PyExc_SystemError, "module kind %d should not be possible", mod->kind); return ERROR; - } - return SUCCESS; -} - -static int -compiler_enter_anonymous_scope(struct compiler* c, mod_ty mod) -{ - _Py_DECLARE_STR(anon_module, ""); - RETURN_IF_ERROR( - compiler_enter_scope(c, &_Py_STR(anon_module), COMPILER_SCOPE_MODULE, - mod, 1)); + }} return SUCCESS; } static PyCodeObject * -compiler_mod(struct compiler *c, mod_ty mod) +compiler_mod(compiler *c, mod_ty mod) { PyCodeObject *co = NULL; int addNone = mod->kind != Expression_kind; - if (compiler_enter_anonymous_scope(c, mod) < 0) { - return NULL; - } if (compiler_codegen(c, mod) < 0) { goto finally; } - co = optimize_and_assemble(c, addNone); + co = _PyCompile_OptimizeAndAssemble(c, addNone); finally: - compiler_exit_scope(c); + _PyCompile_ExitScope(c); return co; } -/* The test for LOCAL must come before the test for FREE in order to - handle classes where name is both local and free. The local var is - a method and the free var is a free var referenced within a method. -*/ - -static int -get_ref_type(struct compiler *c, PyObject *name) +int +_PyCompile_GetRefType(compiler *c, PyObject *name) { - int scope; - if (c->u->u_scope_type == COMPILER_SCOPE_CLASS && + if (c->u->u_scope_type == COMPILE_SCOPE_CLASS && (_PyUnicode_EqualToASCIIString(name, "__class__") || _PyUnicode_EqualToASCIIString(name, "__classdict__"))) { return CELL; } - scope = _PyST_GetScope(c->u->u_ste, name); + PySTEntryObject *ste = c->u->u_ste; + int scope = _PyST_GetScope(ste, name); if (scope == 0) { PyErr_Format(PyExc_SystemError, "_PyST_GetScope(name=%R) failed: " "unknown scope in unit %S (%R); " - "symbols: %R; locals: %R; globals: %R", + "symbols: %R; locals: %R; " + "globals: %R", name, - c->u->u_metadata.u_name, c->u->u_ste->ste_id, - c->u->u_ste->ste_symbols, c->u->u_metadata.u_varnames, c->u->u_metadata.u_names); + c->u->u_metadata.u_name, ste->ste_id, + ste->ste_symbols, c->u->u_metadata.u_varnames, + c->u->u_metadata.u_names); return ERROR; } return scope; } static int -compiler_lookup_arg(PyObject *dict, PyObject *name) +dict_lookup_arg(PyObject *dict, PyObject *name) { PyObject *v = PyDict_GetItemWithError(dict, name); if (v == NULL) { return ERROR; } - return PyLong_AS_LONG(v); + return PyLong_AsLong(v); } -static int -compiler_make_closure(struct compiler *c, location loc, - PyCodeObject *co, Py_ssize_t flags) +int +_PyCompile_LookupCellvar(compiler *c, PyObject *name) { - if (co->co_nfreevars) { - int i = PyCode_GetFirstFree(co); - for (; i < co->co_nlocalsplus; ++i) { - /* Bypass com_addop_varname because it will generate - LOAD_DEREF but LOAD_CLOSURE is needed. - */ - PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); - - /* Special case: If a class contains a method with a - free variable that has the same name as a method, - the name will be considered free *and* local in the - class. It should be handled by the closure, as - well as by the normal name lookup logic. - */ - int reftype = get_ref_type(c, name); - if (reftype == -1) { - return ERROR; - } - int arg; - if (reftype == CELL) { - arg = compiler_lookup_arg(c->u->u_metadata.u_cellvars, name); - } - else { - arg = compiler_lookup_arg(c->u->u_metadata.u_freevars, name); - } - if (arg == -1) { - PyObject *freevars = _PyCode_GetFreevars(co); - if (freevars == NULL) { - PyErr_Clear(); - } - PyErr_Format(PyExc_SystemError, - "compiler_lookup_arg(name=%R) with reftype=%d failed in %S; " - "freevars of code %S: %R", - name, - reftype, - c->u->u_metadata.u_name, - co->co_name, - freevars); - Py_DECREF(freevars); - return ERROR; - } - ADDOP_I(c, loc, LOAD_CLOSURE, arg); - } - flags |= MAKE_FUNCTION_CLOSURE; - ADDOP_I(c, loc, BUILD_TUPLE, co->co_nfreevars); - } - ADDOP_LOAD_CONST(c, loc, (PyObject*)co); - - ADDOP(c, loc, MAKE_FUNCTION); + assert(c->u->u_metadata.u_cellvars); + return dict_lookup_arg(c->u->u_metadata.u_cellvars, name); +} - if (flags & MAKE_FUNCTION_CLOSURE) { - ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_CLOSURE); +int +_PyCompile_LookupArg(compiler *c, PyCodeObject *co, PyObject *name) +{ + /* Special case: If a class contains a method with a + * free variable that has the same name as a method, + * the name will be considered free *and* local in the + * class. It should be handled by the closure, as + * well as by the normal name lookup logic. + */ + int reftype = _PyCompile_GetRefType(c, name); + if (reftype == -1) { + return ERROR; } - if (flags & MAKE_FUNCTION_ANNOTATIONS) { - ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATIONS); + int arg; + if (reftype == CELL) { + arg = dict_lookup_arg(c->u->u_metadata.u_cellvars, name); } - if (flags & MAKE_FUNCTION_KWDEFAULTS) { - ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_KWDEFAULTS); + else { + arg = dict_lookup_arg(c->u->u_metadata.u_freevars, name); } - if (flags & MAKE_FUNCTION_DEFAULTS) { - ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_DEFAULTS); + if (arg == -1 && !PyErr_Occurred()) { + PyObject *freevars = _PyCode_GetFreevars(co); + if (freevars == NULL) { + PyErr_Clear(); + } + PyErr_Format(PyExc_SystemError, + "compiler_lookup_arg(name=%R) with reftype=%d failed in %S; " + "freevars of code %S: %R", + name, + reftype, + c->u->u_metadata.u_name, + co->co_name, + freevars); + Py_XDECREF(freevars); + return ERROR; } - return SUCCESS; + return arg; } -static int -compiler_decorators(struct compiler *c, asdl_expr_seq* decos) +PyObject * +_PyCompile_StaticAttributesAsTuple(compiler *c) { - if (!decos) { - return SUCCESS; + assert(c->u->u_static_attributes); + PyObject *static_attributes_unsorted = PySequence_List(c->u->u_static_attributes); + if (static_attributes_unsorted == NULL) { + return NULL; } - - for (Py_ssize_t i = 0; i < asdl_seq_LEN(decos); i++) { - VISIT(c, expr, (expr_ty)asdl_seq_GET(decos, i)); + if (PyList_Sort(static_attributes_unsorted) != 0) { + Py_DECREF(static_attributes_unsorted); + return NULL; } - return SUCCESS; + PyObject *static_attributes = PySequence_Tuple(static_attributes_unsorted); + Py_DECREF(static_attributes_unsorted); + return static_attributes; } -static int -compiler_apply_decorators(struct compiler *c, asdl_expr_seq* decos) +int +_PyCompile_ResolveNameop(compiler *c, PyObject *mangled, int scope, + _PyCompile_optype *optype, Py_ssize_t *arg) { - if (!decos) { - return SUCCESS; - } + PyObject *dict = c->u->u_metadata.u_names; + *optype = COMPILE_OP_NAME; - for (Py_ssize_t i = asdl_seq_LEN(decos) - 1; i > -1; i--) { - location loc = LOC((expr_ty)asdl_seq_GET(decos, i)); - ADDOP_I(c, loc, CALL, 0); + assert(scope >= 0); + switch (scope) { + case FREE: + dict = c->u->u_metadata.u_freevars; + *optype = COMPILE_OP_DEREF; + break; + case CELL: + dict = c->u->u_metadata.u_cellvars; + *optype = COMPILE_OP_DEREF; + break; + case LOCAL: + if (_PyST_IsFunctionLike(c->u->u_ste)) { + *optype = COMPILE_OP_FAST; + } + else { + PyObject *item; + RETURN_IF_ERROR(PyDict_GetItemRef(c->u->u_metadata.u_fasthidden, mangled, + &item)); + if (item == Py_True) { + *optype = COMPILE_OP_FAST; + } + Py_XDECREF(item); + } + break; + case GLOBAL_IMPLICIT: + if (_PyST_IsFunctionLike(c->u->u_ste)) { + *optype = COMPILE_OP_GLOBAL; + } + break; + case GLOBAL_EXPLICIT: + *optype = COMPILE_OP_GLOBAL; + break; + default: + /* scope can be 0 */ + break; + } + if (*optype != COMPILE_OP_FAST) { + *arg = _PyCompile_DictAddObj(dict, mangled); + RETURN_IF_ERROR(*arg); } return SUCCESS; } -static int -compiler_visit_kwonlydefaults(struct compiler *c, location loc, - asdl_arg_seq *kwonlyargs, asdl_expr_seq *kw_defaults) +int +_PyCompile_TweakInlinedComprehensionScopes(compiler *c, location loc, + PySTEntryObject *entry, + _PyCompile_InlinedComprehensionState *state) { - /* Push a dict of keyword-only default values. - - Return -1 on error, 0 if no dict pushed, 1 if a dict is pushed. - */ - int i; - PyObject *keys = NULL; - - for (i = 0; i < asdl_seq_LEN(kwonlyargs); i++) { - arg_ty arg = asdl_seq_GET(kwonlyargs, i); - expr_ty default_ = asdl_seq_GET(kw_defaults, i); - if (default_) { - PyObject *mangled = _Py_Mangle(c->u->u_private, arg->arg); - if (!mangled) { - goto error; - } - if (keys == NULL) { - keys = PyList_New(1); - if (keys == NULL) { - Py_DECREF(mangled); + int in_class_block = (c->u->u_ste->ste_type == ClassBlock) && !c->u->u_in_inlined_comp; + c->u->u_in_inlined_comp++; + + PyObject *k, *v; + Py_ssize_t pos = 0; + while (PyDict_Next(entry->ste_symbols, &pos, &k, &v)) { + long symbol = PyLong_AsLong(v); + assert(symbol >= 0 || PyErr_Occurred()); + RETURN_IF_ERROR(symbol); + long scope = SYMBOL_TO_SCOPE(symbol); + + long outsymbol = _PyST_GetSymbol(c->u->u_ste, k); + RETURN_IF_ERROR(outsymbol); + long outsc = SYMBOL_TO_SCOPE(outsymbol); + + // If a name has different scope inside than outside the comprehension, + // we need to temporarily handle it with the right scope while + // compiling the comprehension. If it's free in the comprehension + // scope, no special handling; it should be handled the same as the + // enclosing scope. (If it's free in outer scope and cell in inner + // scope, we can't treat it as both cell and free in the same function, + // but treating it as free throughout is fine; it's *_DEREF + // either way.) + if ((scope != outsc && scope != FREE && !(scope == CELL && outsc == FREE)) + || in_class_block) { + if (state->temp_symbols == NULL) { + state->temp_symbols = PyDict_New(); + if (state->temp_symbols == NULL) { return ERROR; } - PyList_SET_ITEM(keys, 0, mangled); } - else { - int res = PyList_Append(keys, mangled); - Py_DECREF(mangled); - if (res == -1) { - goto error; - } + // update the symbol to the in-comprehension version and save + // the outer version; we'll restore it after running the + // comprehension + if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v) < 0) { + return ERROR; } - if (compiler_visit_expr(c, default_) < 0) { - goto error; + PyObject *outv = PyLong_FromLong(outsymbol); + if (outv == NULL) { + return ERROR; + } + int res = PyDict_SetItem(state->temp_symbols, k, outv); + Py_DECREF(outv); + RETURN_IF_ERROR(res); + } + // locals handling for names bound in comprehension (DEF_LOCAL | + // DEF_NONLOCAL occurs in assignment expression to nonlocal) + if ((symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) || in_class_block) { + if (!_PyST_IsFunctionLike(c->u->u_ste)) { + // non-function scope: override this name to use fast locals + PyObject *orig; + if (PyDict_GetItemRef(c->u->u_metadata.u_fasthidden, k, &orig) < 0) { + return ERROR; + } + assert(orig == NULL || orig == Py_True || orig == Py_False); + if (orig != Py_True) { + if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_True) < 0) { + return ERROR; + } + if (state->fast_hidden == NULL) { + state->fast_hidden = PySet_New(NULL); + if (state->fast_hidden == NULL) { + return ERROR; + } + } + if (PySet_Add(state->fast_hidden, k) < 0) { + return ERROR; + } + } } } } - if (keys != NULL) { - Py_ssize_t default_count = PyList_GET_SIZE(keys); - PyObject *keys_tuple = PyList_AsTuple(keys); - Py_DECREF(keys); - ADDOP_LOAD_CONST_NEW(c, loc, keys_tuple); - ADDOP_I(c, loc, BUILD_CONST_KEY_MAP, default_count); - assert(default_count > 0); - return 1; - } - else { - return 0; - } - -error: - Py_XDECREF(keys); - return ERROR; -} - -static int -compiler_visit_annexpr(struct compiler *c, expr_ty annotation) -{ - location loc = LOC(annotation); - ADDOP_LOAD_CONST_NEW(c, loc, _PyAST_ExprAsUnicode(annotation)); return SUCCESS; } -static int -compiler_visit_argannotation(struct compiler *c, identifier id, - expr_ty annotation, Py_ssize_t *annotations_len, location loc) +int +_PyCompile_RevertInlinedComprehensionScopes(compiler *c, location loc, + _PyCompile_InlinedComprehensionState *state) { - if (!annotation) { - return SUCCESS; - } - PyObject *mangled = _Py_Mangle(c->u->u_private, id); - if (!mangled) { - return ERROR; - } - ADDOP_LOAD_CONST(c, loc, mangled); - Py_DECREF(mangled); - - if (c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) { - VISIT(c, annexpr, annotation); - } - else { - if (annotation->kind == Starred_kind) { - // *args: *Ts (where Ts is a TypeVarTuple). - // Do [annotation_value] = [*Ts]. - // (Note that in theory we could end up here even for an argument - // other than *args, but in practice the grammar doesn't allow it.) - VISIT(c, expr, annotation->v.Starred.value); - ADDOP_I(c, loc, UNPACK_SEQUENCE, (Py_ssize_t) 1); + c->u->u_in_inlined_comp--; + if (state->temp_symbols) { + PyObject *k, *v; + Py_ssize_t pos = 0; + while (PyDict_Next(state->temp_symbols, &pos, &k, &v)) { + if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v)) { + return ERROR; + } } - else { - VISIT(c, expr, annotation); + Py_CLEAR(state->temp_symbols); + } + if (state->fast_hidden) { + while (PySet_Size(state->fast_hidden) > 0) { + PyObject *k = PySet_Pop(state->fast_hidden); + if (k == NULL) { + return ERROR; + } + // we set to False instead of clearing, so we can track which names + // were temporarily fast-locals and should use CO_FAST_HIDDEN + if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_False)) { + Py_DECREF(k); + return ERROR; + } + Py_DECREF(k); } + Py_CLEAR(state->fast_hidden); } - *annotations_len += 2; return SUCCESS; } -static int -compiler_visit_argannotations(struct compiler *c, asdl_arg_seq* args, - Py_ssize_t *annotations_len, location loc) +int +_PyCompile_AddDeferredAnnotaion(compiler *c, stmt_ty s) { - int i; - for (i = 0; i < asdl_seq_LEN(args); i++) { - arg_ty arg = (arg_ty)asdl_seq_GET(args, i); - RETURN_IF_ERROR( - compiler_visit_argannotation( - c, - arg->arg, - arg->annotation, - annotations_len, - loc)); + if (c->u->u_deferred_annotations == NULL) { + c->u->u_deferred_annotations = PyList_New(0); + if (c->u->u_deferred_annotations == NULL) { + return ERROR; + } } + PyObject *ptr = PyLong_FromVoidPtr((void *)s); + if (ptr == NULL) { + return ERROR; + } + if (PyList_Append(c->u->u_deferred_annotations, ptr) < 0) { + Py_DECREF(ptr); + return ERROR; + } + Py_DECREF(ptr); return SUCCESS; } -static int -compiler_visit_annotations(struct compiler *c, location loc, - arguments_ty args, expr_ty returns) -{ - /* Push arg annotation names and values. - The expressions are evaluated out-of-order wrt the source code. - - Return -1 on error, 0 if no annotations pushed, 1 if a annotations is pushed. - */ - Py_ssize_t annotations_len = 0; - - RETURN_IF_ERROR( - compiler_visit_argannotations(c, args->args, &annotations_len, loc)); - - RETURN_IF_ERROR( - compiler_visit_argannotations(c, args->posonlyargs, &annotations_len, loc)); - - if (args->vararg && args->vararg->annotation) { - RETURN_IF_ERROR( - compiler_visit_argannotation(c, args->vararg->arg, - args->vararg->annotation, &annotations_len, loc)); - } - - RETURN_IF_ERROR( - compiler_visit_argannotations(c, args->kwonlyargs, &annotations_len, loc)); - - if (args->kwarg && args->kwarg->annotation) { - RETURN_IF_ERROR( - compiler_visit_argannotation(c, args->kwarg->arg, - args->kwarg->annotation, &annotations_len, loc)); - } - - RETURN_IF_ERROR( - compiler_visit_argannotation(c, &_Py_ID(return), returns, &annotations_len, loc)); - - if (annotations_len) { - ADDOP_I(c, loc, BUILD_TUPLE, annotations_len); - return 1; - } - - return 0; -} - -static int -compiler_visit_defaults(struct compiler *c, arguments_ty args, - location loc) -{ - VISIT_SEQ(c, expr, args->defaults); - ADDOP_I(c, loc, BUILD_TUPLE, asdl_seq_LEN(args->defaults)); - return SUCCESS; -} - -static Py_ssize_t -compiler_default_arguments(struct compiler *c, location loc, - arguments_ty args) -{ - Py_ssize_t funcflags = 0; - if (args->defaults && asdl_seq_LEN(args->defaults) > 0) { - RETURN_IF_ERROR(compiler_visit_defaults(c, args, loc)); - funcflags |= MAKE_FUNCTION_DEFAULTS; - } - if (args->kwonlyargs) { - int res = compiler_visit_kwonlydefaults(c, loc, - args->kwonlyargs, - args->kw_defaults); - RETURN_IF_ERROR(res); - if (res > 0) { - funcflags |= MAKE_FUNCTION_KWDEFAULTS; - } - } - return funcflags; -} - -static bool -forbidden_name(struct compiler *c, location loc, identifier name, - expr_context_ty ctx) -{ - if (ctx == Store && _PyUnicode_EqualToASCIIString(name, "__debug__")) { - compiler_error(c, loc, "cannot assign to __debug__"); - return true; - } - if (ctx == Del && _PyUnicode_EqualToASCIIString(name, "__debug__")) { - compiler_error(c, loc, "cannot delete __debug__"); - return true; - } - return false; -} - -static int -compiler_check_debug_one_arg(struct compiler *c, arg_ty arg) -{ - if (arg != NULL) { - if (forbidden_name(c, LOC(arg), arg->arg, Store)) { - return ERROR; - } - } - return SUCCESS; -} - -static int -compiler_check_debug_args_seq(struct compiler *c, asdl_arg_seq *args) -{ - if (args != NULL) { - for (Py_ssize_t i = 0, n = asdl_seq_LEN(args); i < n; i++) { - RETURN_IF_ERROR( - compiler_check_debug_one_arg(c, asdl_seq_GET(args, i))); - } - } - return SUCCESS; -} - -static int -compiler_check_debug_args(struct compiler *c, arguments_ty args) -{ - RETURN_IF_ERROR(compiler_check_debug_args_seq(c, args->posonlyargs)); - RETURN_IF_ERROR(compiler_check_debug_args_seq(c, args->args)); - RETURN_IF_ERROR(compiler_check_debug_one_arg(c, args->vararg)); - RETURN_IF_ERROR(compiler_check_debug_args_seq(c, args->kwonlyargs)); - RETURN_IF_ERROR(compiler_check_debug_one_arg(c, args->kwarg)); - return SUCCESS; -} - -static int -wrap_in_stopiteration_handler(struct compiler *c) -{ - NEW_JUMP_TARGET_LABEL(c, handler); - - /* Insert SETUP_CLEANUP at start */ - RETURN_IF_ERROR( - instr_sequence_insert_instruction( - INSTR_SEQUENCE(c), 0, - SETUP_CLEANUP, handler.id, NO_LOCATION)); - - ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); - ADDOP(c, NO_LOCATION, RETURN_VALUE); - USE_LABEL(c, handler); - ADDOP_I(c, NO_LOCATION, CALL_INTRINSIC_1, INTRINSIC_STOPITERATION_ERROR); - ADDOP_I(c, NO_LOCATION, RERAISE, 1); - return SUCCESS; -} - -static int -compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) -{ - if (!type_params) { - return SUCCESS; - } - Py_ssize_t n = asdl_seq_LEN(type_params); - - for (Py_ssize_t i = 0; i < n; i++) { - type_param_ty typeparam = asdl_seq_GET(type_params, i); - location loc = LOC(typeparam); - switch(typeparam->kind) { - case TypeVar_kind: - ADDOP_LOAD_CONST(c, loc, typeparam->v.TypeVar.name); - if (typeparam->v.TypeVar.bound) { - expr_ty bound = typeparam->v.TypeVar.bound; - if (compiler_enter_scope(c, typeparam->v.TypeVar.name, COMPILER_SCOPE_TYPEPARAMS, - (void *)typeparam, bound->lineno) == -1) { - return ERROR; - } - VISIT_IN_SCOPE(c, expr, bound); - ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); - PyCodeObject *co = optimize_and_assemble(c, 1); - compiler_exit_scope(c); - if (co == NULL) { - return ERROR; - } - if (compiler_make_closure(c, loc, co, 0) < 0) { - Py_DECREF(co); - return ERROR; - } - Py_DECREF(co); - - int intrinsic = bound->kind == Tuple_kind - ? INTRINSIC_TYPEVAR_WITH_CONSTRAINTS - : INTRINSIC_TYPEVAR_WITH_BOUND; - ADDOP_I(c, loc, CALL_INTRINSIC_2, intrinsic); - } - else { - ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_TYPEVAR); - } - ADDOP_I(c, loc, COPY, 1); - RETURN_IF_ERROR(compiler_nameop(c, loc, typeparam->v.TypeVar.name, Store)); - break; - case TypeVarTuple_kind: - ADDOP_LOAD_CONST(c, loc, typeparam->v.TypeVarTuple.name); - ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_TYPEVARTUPLE); - ADDOP_I(c, loc, COPY, 1); - RETURN_IF_ERROR(compiler_nameop(c, loc, typeparam->v.TypeVarTuple.name, Store)); - break; - case ParamSpec_kind: - ADDOP_LOAD_CONST(c, loc, typeparam->v.ParamSpec.name); - ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_PARAMSPEC); - ADDOP_I(c, loc, COPY, 1); - RETURN_IF_ERROR(compiler_nameop(c, loc, typeparam->v.ParamSpec.name, Store)); - break; - } - } - ADDOP_I(c, LOC(asdl_seq_GET(type_params, 0)), BUILD_TUPLE, n); - return SUCCESS; -} - -static int -compiler_function_body(struct compiler *c, stmt_ty s, int is_async, Py_ssize_t funcflags, - int firstlineno) -{ - arguments_ty args; - identifier name; - asdl_stmt_seq *body; - int scope_type; - - if (is_async) { - assert(s->kind == AsyncFunctionDef_kind); - - args = s->v.AsyncFunctionDef.args; - name = s->v.AsyncFunctionDef.name; - body = s->v.AsyncFunctionDef.body; - - scope_type = COMPILER_SCOPE_ASYNC_FUNCTION; - } else { - assert(s->kind == FunctionDef_kind); - - args = s->v.FunctionDef.args; - name = s->v.FunctionDef.name; - body = s->v.FunctionDef.body; - - scope_type = COMPILER_SCOPE_FUNCTION; - } - - RETURN_IF_ERROR( - compiler_enter_scope(c, name, scope_type, (void *)s, firstlineno)); - - Py_ssize_t first_instr = 0; - PyObject *docstring = _PyAST_GetDocString(body); - if (docstring) { - first_instr = 1; - /* if not -OO mode, add docstring */ - if (c->c_optimize < 2) { - docstring = _PyCompile_CleanDoc(docstring); - if (docstring == NULL) { - compiler_exit_scope(c); - return ERROR; - } - } - else { - docstring = NULL; - } - } - if (compiler_add_const(c->c_const_cache, c->u, docstring ? docstring : Py_None) < 0) { - Py_XDECREF(docstring); - compiler_exit_scope(c); - return ERROR; - } - Py_CLEAR(docstring); - - c->u->u_metadata.u_argcount = asdl_seq_LEN(args->args); - c->u->u_metadata.u_posonlyargcount = asdl_seq_LEN(args->posonlyargs); - c->u->u_metadata.u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs); - for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(body); i++) { - VISIT_IN_SCOPE(c, stmt, (stmt_ty)asdl_seq_GET(body, i)); - } - if (c->u->u_ste->ste_coroutine || c->u->u_ste->ste_generator) { - if (wrap_in_stopiteration_handler(c) < 0) { - compiler_exit_scope(c); - return ERROR; - } - } - PyCodeObject *co = optimize_and_assemble(c, 1); - compiler_exit_scope(c); - if (co == NULL) { - Py_XDECREF(co); - return ERROR; - } - location loc = LOC(s); - if (compiler_make_closure(c, loc, co, funcflags) < 0) { - Py_DECREF(co); - return ERROR; - } - Py_DECREF(co); - return SUCCESS; -} - -static int -compiler_function(struct compiler *c, stmt_ty s, int is_async) -{ - arguments_ty args; - expr_ty returns; - identifier name; - asdl_expr_seq *decos; - asdl_type_param_seq *type_params; - Py_ssize_t funcflags; - int annotations; - int firstlineno; - - if (is_async) { - assert(s->kind == AsyncFunctionDef_kind); - - args = s->v.AsyncFunctionDef.args; - returns = s->v.AsyncFunctionDef.returns; - decos = s->v.AsyncFunctionDef.decorator_list; - name = s->v.AsyncFunctionDef.name; - type_params = s->v.AsyncFunctionDef.type_params; - } else { - assert(s->kind == FunctionDef_kind); - - args = s->v.FunctionDef.args; - returns = s->v.FunctionDef.returns; - decos = s->v.FunctionDef.decorator_list; - name = s->v.FunctionDef.name; - type_params = s->v.FunctionDef.type_params; - } - - RETURN_IF_ERROR(compiler_check_debug_args(c, args)); - RETURN_IF_ERROR(compiler_decorators(c, decos)); - - firstlineno = s->lineno; - if (asdl_seq_LEN(decos)) { - firstlineno = ((expr_ty)asdl_seq_GET(decos, 0))->lineno; - } - - location loc = LOC(s); - - int is_generic = asdl_seq_LEN(type_params) > 0; - - funcflags = compiler_default_arguments(c, loc, args); - if (funcflags == -1) { - return ERROR; - } - - int num_typeparam_args = 0; - - if (is_generic) { - if (funcflags & MAKE_FUNCTION_DEFAULTS) { - num_typeparam_args += 1; - } - if (funcflags & MAKE_FUNCTION_KWDEFAULTS) { - num_typeparam_args += 1; - } - if (num_typeparam_args == 2) { - ADDOP_I(c, loc, SWAP, 2); - } - PyObject *type_params_name = PyUnicode_FromFormat("", name); - if (!type_params_name) { - return ERROR; - } - if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS, - (void *)type_params, firstlineno) == -1) { - Py_DECREF(type_params_name); - return ERROR; - } - Py_DECREF(type_params_name); - RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params)); - for (int i = 0; i < num_typeparam_args; i++) { - RETURN_IF_ERROR_IN_SCOPE(c, codegen_addop_i(INSTR_SEQUENCE(c), LOAD_FAST, i, loc)); - } - } - - annotations = compiler_visit_annotations(c, loc, args, returns); - if (annotations < 0) { - if (is_generic) { - compiler_exit_scope(c); - } - return ERROR; - } - if (annotations > 0) { - funcflags |= MAKE_FUNCTION_ANNOTATIONS; - } - - if (compiler_function_body(c, s, is_async, funcflags, firstlineno) < 0) { - if (is_generic) { - compiler_exit_scope(c); - } - return ERROR; - } - - if (is_generic) { - RETURN_IF_ERROR_IN_SCOPE(c, codegen_addop_i( - INSTR_SEQUENCE(c), SWAP, 2, loc)); - RETURN_IF_ERROR_IN_SCOPE(c, codegen_addop_i( - INSTR_SEQUENCE(c), CALL_INTRINSIC_2, INTRINSIC_SET_FUNCTION_TYPE_PARAMS, loc)); - - c->u->u_metadata.u_argcount = num_typeparam_args; - PyCodeObject *co = optimize_and_assemble(c, 0); - compiler_exit_scope(c); - if (co == NULL) { - return ERROR; - } - if (compiler_make_closure(c, loc, co, 0) < 0) { - Py_DECREF(co); - return ERROR; - } - Py_DECREF(co); - if (num_typeparam_args > 0) { - ADDOP_I(c, loc, SWAP, num_typeparam_args + 1); - ADDOP_I(c, loc, CALL, num_typeparam_args - 1); - } - else { - ADDOP(c, loc, PUSH_NULL); - ADDOP_I(c, loc, CALL, 0); - } - } - - RETURN_IF_ERROR(compiler_apply_decorators(c, decos)); - return compiler_nameop(c, loc, name, Store); -} - -static int -compiler_set_type_params_in_class(struct compiler *c, location loc) -{ - _Py_DECLARE_STR(type_params, ".type_params"); - RETURN_IF_ERROR(compiler_nameop(c, loc, &_Py_STR(type_params), Load)); - RETURN_IF_ERROR(compiler_nameop(c, loc, &_Py_ID(__type_params__), Store)); - return 1; -} - -static int -compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) -{ - /* ultimately generate code for: - = __build_class__(, , *, **) - where: - is a zero arg function/closure created from the class body. - It mutates its locals to build the class namespace. - is the class name - is the positional arguments and *varargs argument - is the keyword arguments and **kwds argument - This borrows from compiler_call. - */ - - /* 1. compile the class body into a code object */ - RETURN_IF_ERROR( - compiler_enter_scope(c, s->v.ClassDef.name, - COMPILER_SCOPE_CLASS, (void *)s, firstlineno)); - - location loc = LOCATION(firstlineno, firstlineno, 0, 0); - /* use the class name for name mangling */ - Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name)); - /* load (global) __name__ ... */ - if (compiler_nameop(c, loc, &_Py_ID(__name__), Load) < 0) { - compiler_exit_scope(c); - return ERROR; - } - /* ... and store it as __module__ */ - if (compiler_nameop(c, loc, &_Py_ID(__module__), Store) < 0) { - compiler_exit_scope(c); - return ERROR; - } - assert(c->u->u_metadata.u_qualname); - ADDOP_LOAD_CONST(c, loc, c->u->u_metadata.u_qualname); - if (compiler_nameop(c, loc, &_Py_ID(__qualname__), Store) < 0) { - compiler_exit_scope(c); - return ERROR; - } - asdl_type_param_seq *type_params = s->v.ClassDef.type_params; - if (asdl_seq_LEN(type_params) > 0) { - if (!compiler_set_type_params_in_class(c, loc)) { - compiler_exit_scope(c); - return ERROR; - } - } - if (c->u->u_ste->ste_needs_classdict) { - ADDOP(c, loc, LOAD_LOCALS); - - // We can't use compiler_nameop here because we need to generate a - // STORE_DEREF in a class namespace, and compiler_nameop() won't do - // that by default. - PyObject *cellvars = c->u->u_metadata.u_cellvars; - if (compiler_addop_o(c->u, loc, STORE_DEREF, cellvars, - &_Py_ID(__classdict__)) < 0) { - compiler_exit_scope(c); - return ERROR; - } - } - /* compile the body proper */ - if (compiler_body(c, loc, s->v.ClassDef.body) < 0) { - compiler_exit_scope(c); - return ERROR; - } - /* The following code is artificial */ - /* Set __classdictcell__ if necessary */ - if (c->u->u_ste->ste_needs_classdict) { - /* Store __classdictcell__ into class namespace */ - int i = compiler_lookup_arg(c->u->u_metadata.u_cellvars, &_Py_ID(__classdict__)); - if (i < 0) { - compiler_exit_scope(c); - return ERROR; - } - ADDOP_I(c, NO_LOCATION, LOAD_CLOSURE, i); - if (compiler_nameop(c, NO_LOCATION, &_Py_ID(__classdictcell__), Store) < 0) { - compiler_exit_scope(c); - return ERROR; - } - } - /* Return __classcell__ if it is referenced, otherwise return None */ - if (c->u->u_ste->ste_needs_class_closure) { - /* Store __classcell__ into class namespace & return it */ - int i = compiler_lookup_arg(c->u->u_metadata.u_cellvars, &_Py_ID(__class__)); - if (i < 0) { - compiler_exit_scope(c); - return ERROR; - } - ADDOP_I(c, NO_LOCATION, LOAD_CLOSURE, i); - ADDOP_I(c, NO_LOCATION, COPY, 1); - if (compiler_nameop(c, NO_LOCATION, &_Py_ID(__classcell__), Store) < 0) { - compiler_exit_scope(c); - return ERROR; - } - } - else { - /* No methods referenced __class__, so just return None */ - ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); - } - ADDOP_IN_SCOPE(c, NO_LOCATION, RETURN_VALUE); - /* create the code object */ - PyCodeObject *co = optimize_and_assemble(c, 1); - - /* leave the new scope */ - compiler_exit_scope(c); - if (co == NULL) { - return ERROR; - } - - /* 2. load the 'build_class' function */ - - // these instructions should be attributed to the class line, - // not a decorator line - loc = LOC(s); - ADDOP(c, loc, LOAD_BUILD_CLASS); - ADDOP(c, loc, PUSH_NULL); - - /* 3. load a function (or closure) made from the code object */ - if (compiler_make_closure(c, loc, co, 0) < 0) { - Py_DECREF(co); - return ERROR; - } - Py_DECREF(co); - - /* 4. load class name */ - ADDOP_LOAD_CONST(c, loc, s->v.ClassDef.name); - - return SUCCESS; -} - -static int -compiler_class(struct compiler *c, stmt_ty s) -{ - asdl_expr_seq *decos = s->v.ClassDef.decorator_list; - - RETURN_IF_ERROR(compiler_decorators(c, decos)); - - int firstlineno = s->lineno; - if (asdl_seq_LEN(decos)) { - firstlineno = ((expr_ty)asdl_seq_GET(decos, 0))->lineno; - } - location loc = LOC(s); - - asdl_type_param_seq *type_params = s->v.ClassDef.type_params; - int is_generic = asdl_seq_LEN(type_params) > 0; - if (is_generic) { - Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name)); - PyObject *type_params_name = PyUnicode_FromFormat("", - s->v.ClassDef.name); - if (!type_params_name) { - return ERROR; - } - if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS, - (void *)type_params, firstlineno) == -1) { - Py_DECREF(type_params_name); - return ERROR; - } - Py_DECREF(type_params_name); - RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params)); - _Py_DECLARE_STR(type_params, ".type_params"); - RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_STR(type_params), Store)); - } - - if (compiler_class_body(c, s, firstlineno) < 0) { - if (is_generic) { - compiler_exit_scope(c); - } - return ERROR; - } - - /* generate the rest of the code for the call */ - - if (is_generic) { - _Py_DECLARE_STR(type_params, ".type_params"); - _Py_DECLARE_STR(generic_base, ".generic_base"); - RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_STR(type_params), Load)); - RETURN_IF_ERROR_IN_SCOPE( - c, codegen_addop_i(INSTR_SEQUENCE(c), CALL_INTRINSIC_1, INTRINSIC_SUBSCRIPT_GENERIC, loc) - ) - RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_STR(generic_base), Store)); - - Py_ssize_t original_len = asdl_seq_LEN(s->v.ClassDef.bases); - asdl_expr_seq *bases = _Py_asdl_expr_seq_new( - original_len + 1, c->c_arena); - if (bases == NULL) { - compiler_exit_scope(c); - return ERROR; - } - for (Py_ssize_t i = 0; i < original_len; i++) { - asdl_seq_SET(bases, i, asdl_seq_GET(s->v.ClassDef.bases, i)); - } - expr_ty name_node = _PyAST_Name( - &_Py_STR(generic_base), Load, - loc.lineno, loc.col_offset, loc.end_lineno, loc.end_col_offset, c->c_arena - ); - if (name_node == NULL) { - compiler_exit_scope(c); - return ERROR; - } - asdl_seq_SET(bases, original_len, name_node); - RETURN_IF_ERROR_IN_SCOPE(c, compiler_call_helper(c, loc, 2, - bases, - s->v.ClassDef.keywords)); - - PyCodeObject *co = optimize_and_assemble(c, 0); - compiler_exit_scope(c); - if (co == NULL) { - return ERROR; - } - if (compiler_make_closure(c, loc, co, 0) < 0) { - Py_DECREF(co); - return ERROR; - } - Py_DECREF(co); - ADDOP(c, loc, PUSH_NULL); - ADDOP_I(c, loc, CALL, 0); - } else { - RETURN_IF_ERROR(compiler_call_helper(c, loc, 2, - s->v.ClassDef.bases, - s->v.ClassDef.keywords)); - } - - /* 6. apply decorators */ - RETURN_IF_ERROR(compiler_apply_decorators(c, decos)); - - /* 7. store into */ - RETURN_IF_ERROR(compiler_nameop(c, loc, s->v.ClassDef.name, Store)); - return SUCCESS; -} - -static int -compiler_typealias_body(struct compiler *c, stmt_ty s) -{ - location loc = LOC(s); - PyObject *name = s->v.TypeAlias.name->v.Name.id; - RETURN_IF_ERROR( - compiler_enter_scope(c, name, COMPILER_SCOPE_FUNCTION, s, loc.lineno)); - /* Make None the first constant, so the evaluate function can't have a - docstring. */ - RETURN_IF_ERROR(compiler_add_const(c->c_const_cache, c->u, Py_None)); - VISIT_IN_SCOPE(c, expr, s->v.TypeAlias.value); - ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); - PyCodeObject *co = optimize_and_assemble(c, 0); - compiler_exit_scope(c); - if (co == NULL) { - return ERROR; - } - if (compiler_make_closure(c, loc, co, 0) < 0) { - Py_DECREF(co); - return ERROR; - } - Py_DECREF(co); - ADDOP_I(c, loc, BUILD_TUPLE, 3); - ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_TYPEALIAS); - return SUCCESS; -} - -static int -compiler_typealias(struct compiler *c, stmt_ty s) -{ - location loc = LOC(s); - asdl_type_param_seq *type_params = s->v.TypeAlias.type_params; - int is_generic = asdl_seq_LEN(type_params) > 0; - PyObject *name = s->v.TypeAlias.name->v.Name.id; - if (is_generic) { - PyObject *type_params_name = PyUnicode_FromFormat("", - name); - if (!type_params_name) { - return ERROR; - } - if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS, - (void *)type_params, loc.lineno) == -1) { - Py_DECREF(type_params_name); - return ERROR; - } - Py_DECREF(type_params_name); - RETURN_IF_ERROR_IN_SCOPE( - c, compiler_addop_load_const(c->c_const_cache, c->u, loc, name) - ); - RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params)); - } - else { - ADDOP_LOAD_CONST(c, loc, name); - ADDOP_LOAD_CONST(c, loc, Py_None); - } - - if (compiler_typealias_body(c, s) < 0) { - if (is_generic) { - compiler_exit_scope(c); - } - return ERROR; - } - - if (is_generic) { - PyCodeObject *co = optimize_and_assemble(c, 0); - compiler_exit_scope(c); - if (co == NULL) { - return ERROR; - } - if (compiler_make_closure(c, loc, co, 0) < 0) { - Py_DECREF(co); - return ERROR; - } - Py_DECREF(co); - ADDOP(c, loc, PUSH_NULL); - ADDOP_I(c, loc, CALL, 0); - } - RETURN_IF_ERROR(compiler_nameop(c, loc, name, Store)); - return SUCCESS; -} - -/* Return false if the expression is a constant value except named singletons. - Return true otherwise. */ -static bool -check_is_arg(expr_ty e) -{ - if (e->kind != Constant_kind) { - return true; - } - PyObject *value = e->v.Constant.value; - return (value == Py_None - || value == Py_False - || value == Py_True - || value == Py_Ellipsis); -} - -static PyTypeObject * infer_type(expr_ty e); - -/* Check operands of identity checks ("is" and "is not"). - Emit a warning if any operand is a constant except named singletons. - */ -static int -check_compare(struct compiler *c, expr_ty e) -{ - Py_ssize_t i, n; - bool left = check_is_arg(e->v.Compare.left); - expr_ty left_expr = e->v.Compare.left; - n = asdl_seq_LEN(e->v.Compare.ops); - for (i = 0; i < n; i++) { - cmpop_ty op = (cmpop_ty)asdl_seq_GET(e->v.Compare.ops, i); - expr_ty right_expr = (expr_ty)asdl_seq_GET(e->v.Compare.comparators, i); - bool right = check_is_arg(right_expr); - if (op == Is || op == IsNot) { - if (!right || !left) { - const char *msg = (op == Is) - ? "\"is\" with '%.200s' literal. Did you mean \"==\"?" - : "\"is not\" with '%.200s' literal. Did you mean \"!=\"?"; - expr_ty literal = !left ? left_expr : right_expr; - return compiler_warn( - c, LOC(e), msg, infer_type(literal)->tp_name - ); - } - } - left = right; - left_expr = right_expr; - } - return SUCCESS; -} - -static const int compare_masks[] = { - [Py_LT] = COMPARISON_LESS_THAN, - [Py_LE] = COMPARISON_LESS_THAN | COMPARISON_EQUALS, - [Py_EQ] = COMPARISON_EQUALS, - [Py_NE] = COMPARISON_NOT_EQUALS, - [Py_GT] = COMPARISON_GREATER_THAN, - [Py_GE] = COMPARISON_GREATER_THAN | COMPARISON_EQUALS, -}; - -static int compiler_addcompare(struct compiler *c, location loc, - cmpop_ty op) -{ - int cmp; - switch (op) { - case Eq: - cmp = Py_EQ; - break; - case NotEq: - cmp = Py_NE; - break; - case Lt: - cmp = Py_LT; - break; - case LtE: - cmp = Py_LE; - break; - case Gt: - cmp = Py_GT; - break; - case GtE: - cmp = Py_GE; - break; - case Is: - ADDOP_I(c, loc, IS_OP, 0); - return SUCCESS; - case IsNot: - ADDOP_I(c, loc, IS_OP, 1); - return SUCCESS; - case In: - ADDOP_I(c, loc, CONTAINS_OP, 0); - return SUCCESS; - case NotIn: - ADDOP_I(c, loc, CONTAINS_OP, 1); - return SUCCESS; - default: - Py_UNREACHABLE(); - } - // cmp goes in top three bits of the oparg, while the low four bits are used - // by quickened versions of this opcode to store the comparison mask. The - // fifth-lowest bit indicates whether the result should be converted to bool - // and is set later): - ADDOP_I(c, loc, COMPARE_OP, (cmp << 5) | compare_masks[cmp]); - return SUCCESS; -} - - - -static int -compiler_jump_if(struct compiler *c, location loc, - expr_ty e, jump_target_label next, int cond) -{ - switch (e->kind) { - case UnaryOp_kind: - if (e->v.UnaryOp.op == Not) { - return compiler_jump_if(c, loc, e->v.UnaryOp.operand, next, !cond); - } - /* fallback to general implementation */ - break; - case BoolOp_kind: { - asdl_expr_seq *s = e->v.BoolOp.values; - Py_ssize_t i, n = asdl_seq_LEN(s) - 1; - assert(n >= 0); - int cond2 = e->v.BoolOp.op == Or; - jump_target_label next2 = next; - if (!cond2 != !cond) { - NEW_JUMP_TARGET_LABEL(c, new_next2); - next2 = new_next2; - } - for (i = 0; i < n; ++i) { - RETURN_IF_ERROR( - compiler_jump_if(c, loc, (expr_ty)asdl_seq_GET(s, i), next2, cond2)); - } - RETURN_IF_ERROR( - compiler_jump_if(c, loc, (expr_ty)asdl_seq_GET(s, n), next, cond)); - if (!SAME_LABEL(next2, next)) { - USE_LABEL(c, next2); - } - return SUCCESS; - } - case IfExp_kind: { - NEW_JUMP_TARGET_LABEL(c, end); - NEW_JUMP_TARGET_LABEL(c, next2); - RETURN_IF_ERROR( - compiler_jump_if(c, loc, e->v.IfExp.test, next2, 0)); - RETURN_IF_ERROR( - compiler_jump_if(c, loc, e->v.IfExp.body, next, cond)); - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); - - USE_LABEL(c, next2); - RETURN_IF_ERROR( - compiler_jump_if(c, loc, e->v.IfExp.orelse, next, cond)); - - USE_LABEL(c, end); - return SUCCESS; - } - case Compare_kind: { - Py_ssize_t n = asdl_seq_LEN(e->v.Compare.ops) - 1; - if (n > 0) { - RETURN_IF_ERROR(check_compare(c, e)); - NEW_JUMP_TARGET_LABEL(c, cleanup); - VISIT(c, expr, e->v.Compare.left); - for (Py_ssize_t i = 0; i < n; i++) { - VISIT(c, expr, - (expr_ty)asdl_seq_GET(e->v.Compare.comparators, i)); - ADDOP_I(c, LOC(e), SWAP, 2); - ADDOP_I(c, LOC(e), COPY, 2); - ADDOP_COMPARE(c, LOC(e), asdl_seq_GET(e->v.Compare.ops, i)); - ADDOP(c, LOC(e), TO_BOOL); - ADDOP_JUMP(c, LOC(e), POP_JUMP_IF_FALSE, cleanup); - } - VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, n)); - ADDOP_COMPARE(c, LOC(e), asdl_seq_GET(e->v.Compare.ops, n)); - ADDOP(c, LOC(e), TO_BOOL); - ADDOP_JUMP(c, LOC(e), cond ? POP_JUMP_IF_TRUE : POP_JUMP_IF_FALSE, next); - NEW_JUMP_TARGET_LABEL(c, end); - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); - - USE_LABEL(c, cleanup); - ADDOP(c, LOC(e), POP_TOP); - if (!cond) { - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, next); - } - - USE_LABEL(c, end); - return SUCCESS; - } - /* fallback to general implementation */ - break; - } - default: - /* fallback to general implementation */ - break; - } - - /* general implementation */ - VISIT(c, expr, e); - ADDOP(c, LOC(e), TO_BOOL); - ADDOP_JUMP(c, LOC(e), cond ? POP_JUMP_IF_TRUE : POP_JUMP_IF_FALSE, next); - return SUCCESS; -} - -static int -compiler_ifexp(struct compiler *c, expr_ty e) -{ - assert(e->kind == IfExp_kind); - NEW_JUMP_TARGET_LABEL(c, end); - NEW_JUMP_TARGET_LABEL(c, next); - - RETURN_IF_ERROR( - compiler_jump_if(c, LOC(e), e->v.IfExp.test, next, 0)); - - VISIT(c, expr, e->v.IfExp.body); - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); - - USE_LABEL(c, next); - VISIT(c, expr, e->v.IfExp.orelse); - - USE_LABEL(c, end); - return SUCCESS; -} - -static int -compiler_lambda(struct compiler *c, expr_ty e) -{ - PyCodeObject *co; - Py_ssize_t funcflags; - arguments_ty args = e->v.Lambda.args; - assert(e->kind == Lambda_kind); - - RETURN_IF_ERROR(compiler_check_debug_args(c, args)); - - location loc = LOC(e); - funcflags = compiler_default_arguments(c, loc, args); - if (funcflags == -1) { - return ERROR; - } - - _Py_DECLARE_STR(anon_lambda, ""); - RETURN_IF_ERROR( - compiler_enter_scope(c, &_Py_STR(anon_lambda), COMPILER_SCOPE_LAMBDA, - (void *)e, e->lineno)); - - /* Make None the first constant, so the lambda can't have a - docstring. */ - RETURN_IF_ERROR(compiler_add_const(c->c_const_cache, c->u, Py_None)); - - c->u->u_metadata.u_argcount = asdl_seq_LEN(args->args); - c->u->u_metadata.u_posonlyargcount = asdl_seq_LEN(args->posonlyargs); - c->u->u_metadata.u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs); - VISIT_IN_SCOPE(c, expr, e->v.Lambda.body); - if (c->u->u_ste->ste_generator) { - co = optimize_and_assemble(c, 0); - } - else { - location loc = LOCATION(e->lineno, e->lineno, 0, 0); - ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); - co = optimize_and_assemble(c, 1); - } - compiler_exit_scope(c); - if (co == NULL) { - return ERROR; - } - - if (compiler_make_closure(c, loc, co, funcflags) < 0) { - Py_DECREF(co); - return ERROR; - } - Py_DECREF(co); - - return SUCCESS; -} - -static int -compiler_if(struct compiler *c, stmt_ty s) -{ - jump_target_label next; - assert(s->kind == If_kind); - NEW_JUMP_TARGET_LABEL(c, end); - if (asdl_seq_LEN(s->v.If.orelse)) { - NEW_JUMP_TARGET_LABEL(c, orelse); - next = orelse; - } - else { - next = end; - } - RETURN_IF_ERROR( - compiler_jump_if(c, LOC(s), s->v.If.test, next, 0)); - - VISIT_SEQ(c, stmt, s->v.If.body); - if (asdl_seq_LEN(s->v.If.orelse)) { - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); - - USE_LABEL(c, next); - VISIT_SEQ(c, stmt, s->v.If.orelse); - } - - USE_LABEL(c, end); - return SUCCESS; -} - -static int -compiler_for(struct compiler *c, stmt_ty s) -{ - location loc = LOC(s); - NEW_JUMP_TARGET_LABEL(c, start); - NEW_JUMP_TARGET_LABEL(c, body); - NEW_JUMP_TARGET_LABEL(c, cleanup); - NEW_JUMP_TARGET_LABEL(c, end); - - RETURN_IF_ERROR(compiler_push_fblock(c, loc, FOR_LOOP, start, end, NULL)); - - VISIT(c, expr, s->v.For.iter); - ADDOP(c, loc, GET_ITER); - - USE_LABEL(c, start); - ADDOP_JUMP(c, loc, FOR_ITER, cleanup); - - USE_LABEL(c, body); - VISIT(c, expr, s->v.For.target); - VISIT_SEQ(c, stmt, s->v.For.body); - /* Mark jump as artificial */ - ADDOP_JUMP(c, NO_LOCATION, JUMP, start); - - USE_LABEL(c, cleanup); - /* It is important for instrumentation that the `END_FOR` comes first. - * Iteration over a generator will jump to the first of these instructions, - * but a non-generator will jump to a later instruction. - */ - ADDOP(c, NO_LOCATION, END_FOR); - ADDOP(c, NO_LOCATION, POP_TOP); - - compiler_pop_fblock(c, FOR_LOOP, start); - - VISIT_SEQ(c, stmt, s->v.For.orelse); - - USE_LABEL(c, end); - return SUCCESS; -} - - -static int -compiler_async_for(struct compiler *c, stmt_ty s) -{ - location loc = LOC(s); - if (IS_TOP_LEVEL_AWAIT(c)){ - c->u->u_ste->ste_coroutine = 1; - } else if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) { - return compiler_error(c, loc, "'async for' outside async function"); - } - - NEW_JUMP_TARGET_LABEL(c, start); - NEW_JUMP_TARGET_LABEL(c, except); - NEW_JUMP_TARGET_LABEL(c, end); - - VISIT(c, expr, s->v.AsyncFor.iter); - ADDOP(c, loc, GET_AITER); - - USE_LABEL(c, start); - RETURN_IF_ERROR(compiler_push_fblock(c, loc, FOR_LOOP, start, end, NULL)); - - /* SETUP_FINALLY to guard the __anext__ call */ - ADDOP_JUMP(c, loc, SETUP_FINALLY, except); - ADDOP(c, loc, GET_ANEXT); - ADDOP_LOAD_CONST(c, loc, Py_None); - ADD_YIELD_FROM(c, loc, 1); - ADDOP(c, loc, POP_BLOCK); /* for SETUP_FINALLY */ - - /* Success block for __anext__ */ - VISIT(c, expr, s->v.AsyncFor.target); - VISIT_SEQ(c, stmt, s->v.AsyncFor.body); - /* Mark jump as artificial */ - ADDOP_JUMP(c, NO_LOCATION, JUMP, start); - - compiler_pop_fblock(c, FOR_LOOP, start); - - /* Except block for __anext__ */ - USE_LABEL(c, except); - - /* Use same line number as the iterator, - * as the END_ASYNC_FOR succeeds the `for`, not the body. */ - loc = LOC(s->v.AsyncFor.iter); - ADDOP(c, loc, END_ASYNC_FOR); - - /* `else` block */ - VISIT_SEQ(c, stmt, s->v.For.orelse); - - USE_LABEL(c, end); - return SUCCESS; -} - -static int -compiler_while(struct compiler *c, stmt_ty s) -{ - NEW_JUMP_TARGET_LABEL(c, loop); - NEW_JUMP_TARGET_LABEL(c, body); - NEW_JUMP_TARGET_LABEL(c, end); - NEW_JUMP_TARGET_LABEL(c, anchor); - - USE_LABEL(c, loop); - - RETURN_IF_ERROR(compiler_push_fblock(c, LOC(s), WHILE_LOOP, loop, end, NULL)); - RETURN_IF_ERROR(compiler_jump_if(c, LOC(s), s->v.While.test, anchor, 0)); - - USE_LABEL(c, body); - VISIT_SEQ(c, stmt, s->v.While.body); - RETURN_IF_ERROR(compiler_jump_if(c, LOC(s), s->v.While.test, body, 1)); - - compiler_pop_fblock(c, WHILE_LOOP, loop); - - USE_LABEL(c, anchor); - if (s->v.While.orelse) { - VISIT_SEQ(c, stmt, s->v.While.orelse); - } - - USE_LABEL(c, end); - return SUCCESS; -} - -static int -compiler_return(struct compiler *c, stmt_ty s) -{ - location loc = LOC(s); - int preserve_tos = ((s->v.Return.value != NULL) && - (s->v.Return.value->kind != Constant_kind)); - if (!_PyST_IsFunctionLike(c->u->u_ste)) { - return compiler_error(c, loc, "'return' outside function"); - } - if (s->v.Return.value != NULL && - c->u->u_ste->ste_coroutine && c->u->u_ste->ste_generator) - { - return compiler_error(c, loc, "'return' with value in async generator"); - } - - if (preserve_tos) { - VISIT(c, expr, s->v.Return.value); - } else { - /* Emit instruction with line number for return value */ - if (s->v.Return.value != NULL) { - loc = LOC(s->v.Return.value); - ADDOP(c, loc, NOP); - } - } - if (s->v.Return.value == NULL || s->v.Return.value->lineno != s->lineno) { - loc = LOC(s); - ADDOP(c, loc, NOP); - } - - RETURN_IF_ERROR(compiler_unwind_fblock_stack(c, &loc, preserve_tos, NULL)); - if (s->v.Return.value == NULL) { - ADDOP_LOAD_CONST(c, loc, Py_None); - } - else if (!preserve_tos) { - ADDOP_LOAD_CONST(c, loc, s->v.Return.value->v.Constant.value); - } - ADDOP(c, loc, RETURN_VALUE); - - return SUCCESS; -} - -static int -compiler_break(struct compiler *c, location loc) -{ - struct fblockinfo *loop = NULL; - location origin_loc = loc; - /* Emit instruction with line number */ - ADDOP(c, loc, NOP); - RETURN_IF_ERROR(compiler_unwind_fblock_stack(c, &loc, 0, &loop)); - if (loop == NULL) { - return compiler_error(c, origin_loc, "'break' outside loop"); - } - RETURN_IF_ERROR(compiler_unwind_fblock(c, &loc, loop, 0)); - ADDOP_JUMP(c, loc, JUMP, loop->fb_exit); - return SUCCESS; -} - -static int -compiler_continue(struct compiler *c, location loc) -{ - struct fblockinfo *loop = NULL; - location origin_loc = loc; - /* Emit instruction with line number */ - ADDOP(c, loc, NOP); - RETURN_IF_ERROR(compiler_unwind_fblock_stack(c, &loc, 0, &loop)); - if (loop == NULL) { - return compiler_error(c, origin_loc, "'continue' not properly in loop"); - } - ADDOP_JUMP(c, loc, JUMP, loop->fb_block); - return SUCCESS; -} - - -/* Code generated for "try: finally: " is as follows: - - SETUP_FINALLY L - - POP_BLOCK - - JUMP E - L: - - E: - - The special instructions use the block stack. Each block - stack entry contains the instruction that created it (here - SETUP_FINALLY), the level of the value stack at the time the - block stack entry was created, and a label (here L). - - SETUP_FINALLY: - Pushes the current value stack level and the label - onto the block stack. - POP_BLOCK: - Pops en entry from the block stack. - - The block stack is unwound when an exception is raised: - when a SETUP_FINALLY entry is found, the raised and the caught - exceptions are pushed onto the value stack (and the exception - condition is cleared), and the interpreter jumps to the label - gotten from the block stack. -*/ - -static int -compiler_try_finally(struct compiler *c, stmt_ty s) -{ - location loc = LOC(s); - - NEW_JUMP_TARGET_LABEL(c, body); - NEW_JUMP_TARGET_LABEL(c, end); - NEW_JUMP_TARGET_LABEL(c, exit); - NEW_JUMP_TARGET_LABEL(c, cleanup); - - /* `try` block */ - ADDOP_JUMP(c, loc, SETUP_FINALLY, end); - - USE_LABEL(c, body); - RETURN_IF_ERROR( - compiler_push_fblock(c, loc, FINALLY_TRY, body, end, - s->v.Try.finalbody)); - - if (s->v.Try.handlers && asdl_seq_LEN(s->v.Try.handlers)) { - RETURN_IF_ERROR(compiler_try_except(c, s)); - } - else { - VISIT_SEQ(c, stmt, s->v.Try.body); - } - ADDOP(c, NO_LOCATION, POP_BLOCK); - compiler_pop_fblock(c, FINALLY_TRY, body); - VISIT_SEQ(c, stmt, s->v.Try.finalbody); - - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, exit); - /* `finally` block */ - - USE_LABEL(c, end); - - loc = NO_LOCATION; - ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); - ADDOP(c, loc, PUSH_EXC_INFO); - RETURN_IF_ERROR( - compiler_push_fblock(c, loc, FINALLY_END, end, NO_LABEL, NULL)); - VISIT_SEQ(c, stmt, s->v.Try.finalbody); - compiler_pop_fblock(c, FINALLY_END, end); - - loc = NO_LOCATION; - ADDOP_I(c, loc, RERAISE, 0); - - USE_LABEL(c, cleanup); - POP_EXCEPT_AND_RERAISE(c, loc); - - USE_LABEL(c, exit); - return SUCCESS; -} - -static int -compiler_try_star_finally(struct compiler *c, stmt_ty s) -{ - location loc = LOC(s); - - NEW_JUMP_TARGET_LABEL(c, body); - NEW_JUMP_TARGET_LABEL(c, end); - NEW_JUMP_TARGET_LABEL(c, exit); - NEW_JUMP_TARGET_LABEL(c, cleanup); - /* `try` block */ - ADDOP_JUMP(c, loc, SETUP_FINALLY, end); - - USE_LABEL(c, body); - RETURN_IF_ERROR( - compiler_push_fblock(c, loc, FINALLY_TRY, body, end, - s->v.TryStar.finalbody)); - - if (s->v.TryStar.handlers && asdl_seq_LEN(s->v.TryStar.handlers)) { - RETURN_IF_ERROR(compiler_try_star_except(c, s)); - } - else { - VISIT_SEQ(c, stmt, s->v.TryStar.body); - } - ADDOP(c, NO_LOCATION, POP_BLOCK); - compiler_pop_fblock(c, FINALLY_TRY, body); - VISIT_SEQ(c, stmt, s->v.TryStar.finalbody); - - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, exit); - - /* `finally` block */ - USE_LABEL(c, end); - - loc = NO_LOCATION; - ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); - ADDOP(c, loc, PUSH_EXC_INFO); - RETURN_IF_ERROR( - compiler_push_fblock(c, loc, FINALLY_END, end, NO_LABEL, NULL)); - - VISIT_SEQ(c, stmt, s->v.TryStar.finalbody); - - compiler_pop_fblock(c, FINALLY_END, end); - loc = NO_LOCATION; - ADDOP_I(c, loc, RERAISE, 0); - - USE_LABEL(c, cleanup); - POP_EXCEPT_AND_RERAISE(c, loc); - - USE_LABEL(c, exit); - return SUCCESS; -} - - -/* - Code generated for "try: S except E1 as V1: S1 except E2 as V2: S2 ...": - (The contents of the value stack is shown in [], with the top - at the right; 'tb' is trace-back info, 'val' the exception's - associated value, and 'exc' the exception.) - - Value stack Label Instruction Argument - [] SETUP_FINALLY L1 - [] - [] POP_BLOCK - [] JUMP L0 - - [exc] L1: ) - [exc, E1] CHECK_EXC_MATCH ) - [exc, bool] POP_JUMP_IF_FALSE L2 ) only if E1 - [exc] (or POP if no V1) - [] - JUMP L0 - - [exc] L2: - .............................etc....................... - - [exc] Ln+1: RERAISE # re-raise exception - - [] L0: - - Of course, parts are not generated if Vi or Ei is not present. -*/ -static int -compiler_try_except(struct compiler *c, stmt_ty s) -{ - location loc = LOC(s); - Py_ssize_t i, n; - - NEW_JUMP_TARGET_LABEL(c, body); - NEW_JUMP_TARGET_LABEL(c, except); - NEW_JUMP_TARGET_LABEL(c, end); - NEW_JUMP_TARGET_LABEL(c, cleanup); - - ADDOP_JUMP(c, loc, SETUP_FINALLY, except); - - USE_LABEL(c, body); - RETURN_IF_ERROR( - compiler_push_fblock(c, loc, TRY_EXCEPT, body, NO_LABEL, NULL)); - VISIT_SEQ(c, stmt, s->v.Try.body); - compiler_pop_fblock(c, TRY_EXCEPT, body); - ADDOP(c, NO_LOCATION, POP_BLOCK); - if (s->v.Try.orelse && asdl_seq_LEN(s->v.Try.orelse)) { - VISIT_SEQ(c, stmt, s->v.Try.orelse); - } - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); - n = asdl_seq_LEN(s->v.Try.handlers); - - USE_LABEL(c, except); - - ADDOP_JUMP(c, NO_LOCATION, SETUP_CLEANUP, cleanup); - ADDOP(c, NO_LOCATION, PUSH_EXC_INFO); - - /* Runtime will push a block here, so we need to account for that */ - RETURN_IF_ERROR( - compiler_push_fblock(c, loc, EXCEPTION_HANDLER, NO_LABEL, NO_LABEL, NULL)); - - for (i = 0; i < n; i++) { - excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( - s->v.Try.handlers, i); - location loc = LOC(handler); - if (!handler->v.ExceptHandler.type && i < n-1) { - return compiler_error(c, loc, "default 'except:' must be last"); - } - NEW_JUMP_TARGET_LABEL(c, next_except); - except = next_except; - if (handler->v.ExceptHandler.type) { - VISIT(c, expr, handler->v.ExceptHandler.type); - ADDOP(c, loc, CHECK_EXC_MATCH); - ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, except); - } - if (handler->v.ExceptHandler.name) { - NEW_JUMP_TARGET_LABEL(c, cleanup_end); - NEW_JUMP_TARGET_LABEL(c, cleanup_body); - - RETURN_IF_ERROR( - compiler_nameop(c, loc, handler->v.ExceptHandler.name, Store)); - - /* - try: - # body - except type as name: - try: - # body - finally: - name = None # in case body contains "del name" - del name - */ - - /* second try: */ - ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup_end); - - USE_LABEL(c, cleanup_body); - RETURN_IF_ERROR( - compiler_push_fblock(c, loc, HANDLER_CLEANUP, cleanup_body, - NO_LABEL, handler->v.ExceptHandler.name)); - - /* second # body */ - VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); - compiler_pop_fblock(c, HANDLER_CLEANUP, cleanup_body); - /* name = None; del name; # Mark as artificial */ - ADDOP(c, NO_LOCATION, POP_BLOCK); - ADDOP(c, NO_LOCATION, POP_BLOCK); - ADDOP(c, NO_LOCATION, POP_EXCEPT); - ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); - RETURN_IF_ERROR( - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store)); - RETURN_IF_ERROR( - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del)); - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); - - /* except: */ - USE_LABEL(c, cleanup_end); - - /* name = None; del name; # artificial */ - ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); - RETURN_IF_ERROR( - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store)); - RETURN_IF_ERROR( - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del)); - - ADDOP_I(c, NO_LOCATION, RERAISE, 1); - } - else { - NEW_JUMP_TARGET_LABEL(c, cleanup_body); - - ADDOP(c, loc, POP_TOP); /* exc_value */ - - USE_LABEL(c, cleanup_body); - RETURN_IF_ERROR( - compiler_push_fblock(c, loc, HANDLER_CLEANUP, cleanup_body, - NO_LABEL, NULL)); - - VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); - compiler_pop_fblock(c, HANDLER_CLEANUP, cleanup_body); - ADDOP(c, NO_LOCATION, POP_BLOCK); - ADDOP(c, NO_LOCATION, POP_EXCEPT); - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); - } - - USE_LABEL(c, except); - } - /* artificial */ - compiler_pop_fblock(c, EXCEPTION_HANDLER, NO_LABEL); - ADDOP_I(c, NO_LOCATION, RERAISE, 0); - - USE_LABEL(c, cleanup); - POP_EXCEPT_AND_RERAISE(c, NO_LOCATION); - - USE_LABEL(c, end); - return SUCCESS; -} - -/* - Code generated for "try: S except* E1 as V1: S1 except* E2 as V2: S2 ...": - (The contents of the value stack is shown in [], with the top - at the right; 'tb' is trace-back info, 'val' the exception instance, - and 'typ' the exception's type.) - - Value stack Label Instruction Argument - [] SETUP_FINALLY L1 - [] - [] POP_BLOCK - [] JUMP L0 - - [exc] L1: BUILD_LIST ) list for raised/reraised excs ("result") - [orig, res] COPY 2 ) make a copy of the original EG - - [orig, res, exc] - [orig, res, exc, E1] CHECK_EG_MATCH - [orig, res, rest/exc, match?] COPY 1 - [orig, res, rest/exc, match?, match?] POP_JUMP_IF_NONE C1 - - [orig, res, rest, match] (or POP if no V1) - - [orig, res, rest] SETUP_FINALLY R1 - [orig, res, rest] - [orig, res, rest] JUMP L2 - - [orig, res, rest, i, v] R1: LIST_APPEND 3 ) exc raised in except* body - add to res - [orig, res, rest, i] POP - [orig, res, rest] JUMP LE2 - - [orig, res, rest] L2: NOP ) for lineno - [orig, res, rest] JUMP LE2 - - [orig, res, rest/exc, None] C1: POP - - [orig, res, rest] LE2: - .............................etc....................... - - [orig, res, rest] Ln+1: LIST_APPEND 1 ) add unhandled exc to res (could be None) - - [orig, res] CALL_INTRINSIC_2 PREP_RERAISE_STAR - [exc] COPY 1 - [exc, exc] POP_JUMP_IF_NOT_NONE RER - [exc] POP_TOP - [] JUMP L0 - - [exc] RER: SWAP 2 - [exc, prev_exc_info] POP_EXCEPT - [exc] RERAISE 0 - - [] L0: -*/ -static int -compiler_try_star_except(struct compiler *c, stmt_ty s) -{ - location loc = LOC(s); - - NEW_JUMP_TARGET_LABEL(c, body); - NEW_JUMP_TARGET_LABEL(c, except); - NEW_JUMP_TARGET_LABEL(c, orelse); - NEW_JUMP_TARGET_LABEL(c, end); - NEW_JUMP_TARGET_LABEL(c, cleanup); - NEW_JUMP_TARGET_LABEL(c, reraise_star); - - ADDOP_JUMP(c, loc, SETUP_FINALLY, except); - - USE_LABEL(c, body); - RETURN_IF_ERROR( - compiler_push_fblock(c, loc, TRY_EXCEPT, body, NO_LABEL, NULL)); - VISIT_SEQ(c, stmt, s->v.TryStar.body); - compiler_pop_fblock(c, TRY_EXCEPT, body); - ADDOP(c, NO_LOCATION, POP_BLOCK); - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, orelse); - Py_ssize_t n = asdl_seq_LEN(s->v.TryStar.handlers); - - USE_LABEL(c, except); - - ADDOP_JUMP(c, NO_LOCATION, SETUP_CLEANUP, cleanup); - ADDOP(c, NO_LOCATION, PUSH_EXC_INFO); - - /* Runtime will push a block here, so we need to account for that */ - RETURN_IF_ERROR( - compiler_push_fblock(c, loc, EXCEPTION_GROUP_HANDLER, - NO_LABEL, NO_LABEL, "except handler")); - - for (Py_ssize_t i = 0; i < n; i++) { - excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( - s->v.TryStar.handlers, i); - location loc = LOC(handler); - NEW_JUMP_TARGET_LABEL(c, next_except); - except = next_except; - NEW_JUMP_TARGET_LABEL(c, except_with_error); - NEW_JUMP_TARGET_LABEL(c, no_match); - if (i == 0) { - /* create empty list for exceptions raised/reraise in the except* blocks */ - /* - [orig] BUILD_LIST - */ - /* Create a copy of the original EG */ - /* - [orig, []] COPY 2 - [orig, [], exc] - */ - ADDOP_I(c, loc, BUILD_LIST, 0); - ADDOP_I(c, loc, COPY, 2); - } - if (handler->v.ExceptHandler.type) { - VISIT(c, expr, handler->v.ExceptHandler.type); - ADDOP(c, loc, CHECK_EG_MATCH); - ADDOP_I(c, loc, COPY, 1); - ADDOP_JUMP(c, loc, POP_JUMP_IF_NONE, no_match); - } - - NEW_JUMP_TARGET_LABEL(c, cleanup_end); - NEW_JUMP_TARGET_LABEL(c, cleanup_body); - - if (handler->v.ExceptHandler.name) { - RETURN_IF_ERROR( - compiler_nameop(c, loc, handler->v.ExceptHandler.name, Store)); - } - else { - ADDOP(c, loc, POP_TOP); // match - } - - /* - try: - # body - except type as name: - try: - # body - finally: - name = None # in case body contains "del name" - del name - */ - /* second try: */ - ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup_end); - - USE_LABEL(c, cleanup_body); - RETURN_IF_ERROR( - compiler_push_fblock(c, loc, HANDLER_CLEANUP, cleanup_body, - NO_LABEL, handler->v.ExceptHandler.name)); - - /* second # body */ - VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); - compiler_pop_fblock(c, HANDLER_CLEANUP, cleanup_body); - /* name = None; del name; # artificial */ - ADDOP(c, NO_LOCATION, POP_BLOCK); - if (handler->v.ExceptHandler.name) { - ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); - RETURN_IF_ERROR( - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store)); - RETURN_IF_ERROR( - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del)); - } - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, except); - - /* except: */ - USE_LABEL(c, cleanup_end); - - /* name = None; del name; # artificial */ - if (handler->v.ExceptHandler.name) { - ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); - RETURN_IF_ERROR( - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Store)); - RETURN_IF_ERROR( - compiler_nameop(c, NO_LOCATION, handler->v.ExceptHandler.name, Del)); - } - - /* add exception raised to the res list */ - ADDOP_I(c, NO_LOCATION, LIST_APPEND, 3); // exc - ADDOP(c, NO_LOCATION, POP_TOP); // lasti - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, except_with_error); - - USE_LABEL(c, except); - ADDOP(c, NO_LOCATION, NOP); // to hold a propagated location info - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, except_with_error); - - USE_LABEL(c, no_match); - ADDOP(c, loc, POP_TOP); // match (None) - - USE_LABEL(c, except_with_error); - - if (i == n - 1) { - /* Add exc to the list (if not None it's the unhandled part of the EG) */ - ADDOP_I(c, NO_LOCATION, LIST_APPEND, 1); - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, reraise_star); - } - } - /* artificial */ - compiler_pop_fblock(c, EXCEPTION_GROUP_HANDLER, NO_LABEL); - NEW_JUMP_TARGET_LABEL(c, reraise); - - USE_LABEL(c, reraise_star); - ADDOP_I(c, NO_LOCATION, CALL_INTRINSIC_2, INTRINSIC_PREP_RERAISE_STAR); - ADDOP_I(c, NO_LOCATION, COPY, 1); - ADDOP_JUMP(c, NO_LOCATION, POP_JUMP_IF_NOT_NONE, reraise); - - /* Nothing to reraise */ - ADDOP(c, NO_LOCATION, POP_TOP); - ADDOP(c, NO_LOCATION, POP_BLOCK); - ADDOP(c, NO_LOCATION, POP_EXCEPT); - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); - - USE_LABEL(c, reraise); - ADDOP(c, NO_LOCATION, POP_BLOCK); - ADDOP_I(c, NO_LOCATION, SWAP, 2); - ADDOP(c, NO_LOCATION, POP_EXCEPT); - ADDOP_I(c, NO_LOCATION, RERAISE, 0); - - USE_LABEL(c, cleanup); - POP_EXCEPT_AND_RERAISE(c, NO_LOCATION); - - USE_LABEL(c, orelse); - VISIT_SEQ(c, stmt, s->v.TryStar.orelse); - - USE_LABEL(c, end); - return SUCCESS; -} - -static int -compiler_try(struct compiler *c, stmt_ty s) { - if (s->v.Try.finalbody && asdl_seq_LEN(s->v.Try.finalbody)) - return compiler_try_finally(c, s); - else - return compiler_try_except(c, s); -} - -static int -compiler_try_star(struct compiler *c, stmt_ty s) -{ - if (s->v.TryStar.finalbody && asdl_seq_LEN(s->v.TryStar.finalbody)) { - return compiler_try_star_finally(c, s); - } - else { - return compiler_try_star_except(c, s); - } -} - -static int -compiler_import_as(struct compiler *c, location loc, - identifier name, identifier asname) -{ - /* The IMPORT_NAME opcode was already generated. This function - merely needs to bind the result to a name. - - If there is a dot in name, we need to split it and emit a - IMPORT_FROM for each name. - */ - Py_ssize_t len = PyUnicode_GET_LENGTH(name); - Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, len, 1); - if (dot == -2) { - return ERROR; - } - if (dot != -1) { - /* Consume the base module name to get the first attribute */ - while (1) { - Py_ssize_t pos = dot + 1; - PyObject *attr; - dot = PyUnicode_FindChar(name, '.', pos, len, 1); - if (dot == -2) { - return ERROR; - } - attr = PyUnicode_Substring(name, pos, (dot != -1) ? dot : len); - if (!attr) { - return ERROR; - } - ADDOP_N(c, loc, IMPORT_FROM, attr, names); - if (dot == -1) { - break; - } - ADDOP_I(c, loc, SWAP, 2); - ADDOP(c, loc, POP_TOP); - } - RETURN_IF_ERROR(compiler_nameop(c, loc, asname, Store)); - ADDOP(c, loc, POP_TOP); - return SUCCESS; - } - return compiler_nameop(c, loc, asname, Store); -} - -static int -compiler_import(struct compiler *c, stmt_ty s) -{ - location loc = LOC(s); - /* The Import node stores a module name like a.b.c as a single - string. This is convenient for all cases except - import a.b.c as d - where we need to parse that string to extract the individual - module names. - XXX Perhaps change the representation to make this case simpler? - */ - Py_ssize_t i, n = asdl_seq_LEN(s->v.Import.names); - - PyObject *zero = _PyLong_GetZero(); // borrowed reference - for (i = 0; i < n; i++) { - alias_ty alias = (alias_ty)asdl_seq_GET(s->v.Import.names, i); - int r; - - ADDOP_LOAD_CONST(c, loc, zero); - ADDOP_LOAD_CONST(c, loc, Py_None); - ADDOP_NAME(c, loc, IMPORT_NAME, alias->name, names); - - if (alias->asname) { - r = compiler_import_as(c, loc, alias->name, alias->asname); - RETURN_IF_ERROR(r); - } - else { - identifier tmp = alias->name; - Py_ssize_t dot = PyUnicode_FindChar( - alias->name, '.', 0, PyUnicode_GET_LENGTH(alias->name), 1); - if (dot != -1) { - tmp = PyUnicode_Substring(alias->name, 0, dot); - if (tmp == NULL) { - return ERROR; - } - } - r = compiler_nameop(c, loc, tmp, Store); - if (dot != -1) { - Py_DECREF(tmp); - } - RETURN_IF_ERROR(r); - } - } - return SUCCESS; -} - -static int -compiler_from_import(struct compiler *c, stmt_ty s) -{ - Py_ssize_t n = asdl_seq_LEN(s->v.ImportFrom.names); - - ADDOP_LOAD_CONST_NEW(c, LOC(s), PyLong_FromLong(s->v.ImportFrom.level)); - - PyObject *names = PyTuple_New(n); - if (!names) { - return ERROR; - } - - /* build up the names */ - for (Py_ssize_t i = 0; i < n; i++) { - alias_ty alias = (alias_ty)asdl_seq_GET(s->v.ImportFrom.names, i); - PyTuple_SET_ITEM(names, i, Py_NewRef(alias->name)); - } - - if (location_is_after(LOC(s), c->c_future.ff_location) && - s->v.ImportFrom.module && - _PyUnicode_EqualToASCIIString(s->v.ImportFrom.module, "__future__")) - { - Py_DECREF(names); - return compiler_error(c, LOC(s), "from __future__ imports must occur " - "at the beginning of the file"); - } - ADDOP_LOAD_CONST_NEW(c, LOC(s), names); - - if (s->v.ImportFrom.module) { - ADDOP_NAME(c, LOC(s), IMPORT_NAME, s->v.ImportFrom.module, names); - } - else { - _Py_DECLARE_STR(empty, ""); - ADDOP_NAME(c, LOC(s), IMPORT_NAME, &_Py_STR(empty), names); - } - for (Py_ssize_t i = 0; i < n; i++) { - alias_ty alias = (alias_ty)asdl_seq_GET(s->v.ImportFrom.names, i); - identifier store_name; - - if (i == 0 && PyUnicode_READ_CHAR(alias->name, 0) == '*') { - assert(n == 1); - ADDOP_I(c, LOC(s), CALL_INTRINSIC_1, INTRINSIC_IMPORT_STAR); - ADDOP(c, NO_LOCATION, POP_TOP); - return SUCCESS; - } - - ADDOP_NAME(c, LOC(s), IMPORT_FROM, alias->name, names); - store_name = alias->name; - if (alias->asname) { - store_name = alias->asname; - } - - RETURN_IF_ERROR(compiler_nameop(c, LOC(s), store_name, Store)); - } - /* remove imported module */ - ADDOP(c, LOC(s), POP_TOP); - return SUCCESS; -} - -static int -compiler_assert(struct compiler *c, stmt_ty s) -{ - /* Always emit a warning if the test is a non-zero length tuple */ - if ((s->v.Assert.test->kind == Tuple_kind && - asdl_seq_LEN(s->v.Assert.test->v.Tuple.elts) > 0) || - (s->v.Assert.test->kind == Constant_kind && - PyTuple_Check(s->v.Assert.test->v.Constant.value) && - PyTuple_Size(s->v.Assert.test->v.Constant.value) > 0)) - { - RETURN_IF_ERROR( - compiler_warn(c, LOC(s), "assertion is always true, " - "perhaps remove parentheses?")); - } - if (c->c_optimize) { - return SUCCESS; - } - NEW_JUMP_TARGET_LABEL(c, end); - RETURN_IF_ERROR(compiler_jump_if(c, LOC(s), s->v.Assert.test, end, 1)); - ADDOP(c, LOC(s), LOAD_ASSERTION_ERROR); - if (s->v.Assert.msg) { - VISIT(c, expr, s->v.Assert.msg); - ADDOP_I(c, LOC(s), CALL, 0); - } - ADDOP_I(c, LOC(s->v.Assert.test), RAISE_VARARGS, 1); - - USE_LABEL(c, end); - return SUCCESS; -} - -static int -compiler_stmt_expr(struct compiler *c, location loc, expr_ty value) -{ - if (c->c_interactive && c->c_nestlevel <= 1) { - VISIT(c, expr, value); - ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_PRINT); - ADDOP(c, NO_LOCATION, POP_TOP); - return SUCCESS; - } - - if (value->kind == Constant_kind) { - /* ignore constant statement */ - ADDOP(c, loc, NOP); - return SUCCESS; - } - - VISIT(c, expr, value); - ADDOP(c, NO_LOCATION, POP_TOP); /* artificial */ - return SUCCESS; -} - -static int -compiler_visit_stmt(struct compiler *c, stmt_ty s) -{ - - switch (s->kind) { - case FunctionDef_kind: - return compiler_function(c, s, 0); - case ClassDef_kind: - return compiler_class(c, s); - case TypeAlias_kind: - return compiler_typealias(c, s); - case Return_kind: - return compiler_return(c, s); - case Delete_kind: - VISIT_SEQ(c, expr, s->v.Delete.targets) - break; - case Assign_kind: - { - Py_ssize_t n = asdl_seq_LEN(s->v.Assign.targets); - VISIT(c, expr, s->v.Assign.value); - for (Py_ssize_t i = 0; i < n; i++) { - if (i < n - 1) { - ADDOP_I(c, LOC(s), COPY, 1); - } - VISIT(c, expr, - (expr_ty)asdl_seq_GET(s->v.Assign.targets, i)); - } - break; - } - case AugAssign_kind: - return compiler_augassign(c, s); - case AnnAssign_kind: - return compiler_annassign(c, s); - case For_kind: - return compiler_for(c, s); - case While_kind: - return compiler_while(c, s); - case If_kind: - return compiler_if(c, s); - case Match_kind: - return compiler_match(c, s); - case Raise_kind: - { - Py_ssize_t n = 0; - if (s->v.Raise.exc) { - VISIT(c, expr, s->v.Raise.exc); - n++; - if (s->v.Raise.cause) { - VISIT(c, expr, s->v.Raise.cause); - n++; - } - } - ADDOP_I(c, LOC(s), RAISE_VARARGS, (int)n); - break; - } - case Try_kind: - return compiler_try(c, s); - case TryStar_kind: - return compiler_try_star(c, s); - case Assert_kind: - return compiler_assert(c, s); - case Import_kind: - return compiler_import(c, s); - case ImportFrom_kind: - return compiler_from_import(c, s); - case Global_kind: - case Nonlocal_kind: - break; - case Expr_kind: - { - return compiler_stmt_expr(c, LOC(s), s->v.Expr.value); - } - case Pass_kind: - { - ADDOP(c, LOC(s), NOP); - break; - } - case Break_kind: - { - return compiler_break(c, LOC(s)); - } - case Continue_kind: - { - return compiler_continue(c, LOC(s)); - } - case With_kind: - return compiler_with(c, s, 0); - case AsyncFunctionDef_kind: - return compiler_function(c, s, 1); - case AsyncWith_kind: - return compiler_async_with(c, s, 0); - case AsyncFor_kind: - return compiler_async_for(c, s); - } - - return SUCCESS; -} - -static int -unaryop(unaryop_ty op) -{ - switch (op) { - case Invert: - return UNARY_INVERT; - case USub: - return UNARY_NEGATIVE; - default: - PyErr_Format(PyExc_SystemError, - "unary op %d should not be possible", op); - return 0; - } -} - -static int -addop_binary(struct compiler *c, location loc, operator_ty binop, - bool inplace) -{ - int oparg; - switch (binop) { - case Add: - oparg = inplace ? NB_INPLACE_ADD : NB_ADD; - break; - case Sub: - oparg = inplace ? NB_INPLACE_SUBTRACT : NB_SUBTRACT; - break; - case Mult: - oparg = inplace ? NB_INPLACE_MULTIPLY : NB_MULTIPLY; - break; - case MatMult: - oparg = inplace ? NB_INPLACE_MATRIX_MULTIPLY : NB_MATRIX_MULTIPLY; - break; - case Div: - oparg = inplace ? NB_INPLACE_TRUE_DIVIDE : NB_TRUE_DIVIDE; - break; - case Mod: - oparg = inplace ? NB_INPLACE_REMAINDER : NB_REMAINDER; - break; - case Pow: - oparg = inplace ? NB_INPLACE_POWER : NB_POWER; - break; - case LShift: - oparg = inplace ? NB_INPLACE_LSHIFT : NB_LSHIFT; - break; - case RShift: - oparg = inplace ? NB_INPLACE_RSHIFT : NB_RSHIFT; - break; - case BitOr: - oparg = inplace ? NB_INPLACE_OR : NB_OR; - break; - case BitXor: - oparg = inplace ? NB_INPLACE_XOR : NB_XOR; - break; - case BitAnd: - oparg = inplace ? NB_INPLACE_AND : NB_AND; - break; - case FloorDiv: - oparg = inplace ? NB_INPLACE_FLOOR_DIVIDE : NB_FLOOR_DIVIDE; - break; - default: - PyErr_Format(PyExc_SystemError, "%s op %d should not be possible", - inplace ? "inplace" : "binary", binop); - return ERROR; - } - ADDOP_I(c, loc, BINARY_OP, oparg); - return SUCCESS; -} - - -static int -addop_yield(struct compiler *c, location loc) { - if (c->u->u_ste->ste_generator && c->u->u_ste->ste_coroutine) { - ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_ASYNC_GEN_WRAP); - } - ADDOP_I(c, loc, YIELD_VALUE, 0); - ADDOP_I(c, loc, RESUME, RESUME_AFTER_YIELD); - return SUCCESS; -} - -static int -compiler_nameop(struct compiler *c, location loc, - identifier name, expr_context_ty ctx) -{ - int op, scope; - Py_ssize_t arg; - enum { OP_FAST, OP_GLOBAL, OP_DEREF, OP_NAME } optype; - - PyObject *dict = c->u->u_metadata.u_names; - PyObject *mangled; - - assert(!_PyUnicode_EqualToASCIIString(name, "None") && - !_PyUnicode_EqualToASCIIString(name, "True") && - !_PyUnicode_EqualToASCIIString(name, "False")); - - if (forbidden_name(c, loc, name, ctx)) { - return ERROR; - } - - mangled = _Py_Mangle(c->u->u_private, name); - if (!mangled) { - return ERROR; - } - - op = 0; - optype = OP_NAME; - scope = _PyST_GetScope(c->u->u_ste, mangled); - switch (scope) { - case FREE: - dict = c->u->u_metadata.u_freevars; - optype = OP_DEREF; - break; - case CELL: - dict = c->u->u_metadata.u_cellvars; - optype = OP_DEREF; - break; - case LOCAL: - if (_PyST_IsFunctionLike(c->u->u_ste)) { - optype = OP_FAST; - } - else { - PyObject *item; - if (PyDict_GetItemRef(c->u->u_metadata.u_fasthidden, mangled, - &item) < 0) { - goto error; - } - if (item == Py_True) { - optype = OP_FAST; - } - Py_XDECREF(item); - } - break; - case GLOBAL_IMPLICIT: - if (_PyST_IsFunctionLike(c->u->u_ste)) - optype = OP_GLOBAL; - break; - case GLOBAL_EXPLICIT: - optype = OP_GLOBAL; - break; - default: - /* scope can be 0 */ - break; - } - - /* XXX Leave assert here, but handle __doc__ and the like better */ - assert(scope || PyUnicode_READ_CHAR(name, 0) == '_'); - - switch (optype) { - case OP_DEREF: - switch (ctx) { - case Load: - if (c->u->u_ste->ste_type == ClassBlock && !c->u->u_in_inlined_comp) { - op = LOAD_FROM_DICT_OR_DEREF; - // First load the locals - if (codegen_addop_noarg(INSTR_SEQUENCE(c), LOAD_LOCALS, loc) < 0) { - goto error; - } - } - else if (c->u->u_ste->ste_can_see_class_scope) { - op = LOAD_FROM_DICT_OR_DEREF; - // First load the classdict - if (compiler_addop_o(c->u, loc, LOAD_DEREF, - c->u->u_metadata.u_freevars, &_Py_ID(__classdict__)) < 0) { - goto error; - } - } - else { - op = LOAD_DEREF; - } - break; - case Store: op = STORE_DEREF; break; - case Del: op = DELETE_DEREF; break; - } - break; - case OP_FAST: - switch (ctx) { - case Load: op = LOAD_FAST; break; - case Store: op = STORE_FAST; break; - case Del: op = DELETE_FAST; break; - } - ADDOP_N(c, loc, op, mangled, varnames); - return SUCCESS; - case OP_GLOBAL: - switch (ctx) { - case Load: - if (c->u->u_ste->ste_can_see_class_scope && scope == GLOBAL_IMPLICIT) { - op = LOAD_FROM_DICT_OR_GLOBALS; - // First load the classdict - if (compiler_addop_o(c->u, loc, LOAD_DEREF, - c->u->u_metadata.u_freevars, &_Py_ID(__classdict__)) < 0) { - goto error; - } - } else { - op = LOAD_GLOBAL; - } - break; - case Store: op = STORE_GLOBAL; break; - case Del: op = DELETE_GLOBAL; break; - } - break; - case OP_NAME: - switch (ctx) { - case Load: - op = (c->u->u_ste->ste_type == ClassBlock - && c->u->u_in_inlined_comp) - ? LOAD_GLOBAL - : LOAD_NAME; - break; - case Store: op = STORE_NAME; break; - case Del: op = DELETE_NAME; break; - } - break; - } - - assert(op); - arg = dict_add_o(dict, mangled); - Py_DECREF(mangled); - if (arg < 0) { - return ERROR; - } - if (op == LOAD_GLOBAL) { - arg <<= 1; - } - return codegen_addop_i(INSTR_SEQUENCE(c), op, arg, loc); - -error: - Py_DECREF(mangled); - return ERROR; -} - -static int -compiler_boolop(struct compiler *c, expr_ty e) -{ - int jumpi; - Py_ssize_t i, n; - asdl_expr_seq *s; - - location loc = LOC(e); - assert(e->kind == BoolOp_kind); - if (e->v.BoolOp.op == And) - jumpi = POP_JUMP_IF_FALSE; - else - jumpi = POP_JUMP_IF_TRUE; - NEW_JUMP_TARGET_LABEL(c, end); - s = e->v.BoolOp.values; - n = asdl_seq_LEN(s) - 1; - assert(n >= 0); - for (i = 0; i < n; ++i) { - VISIT(c, expr, (expr_ty)asdl_seq_GET(s, i)); - ADDOP_I(c, loc, COPY, 1); - ADDOP(c, loc, TO_BOOL); - ADDOP_JUMP(c, loc, jumpi, end); - ADDOP(c, loc, POP_TOP); - } - VISIT(c, expr, (expr_ty)asdl_seq_GET(s, n)); - - USE_LABEL(c, end); - return SUCCESS; -} - -static int -starunpack_helper(struct compiler *c, location loc, - asdl_expr_seq *elts, int pushed, - int build, int add, int extend, int tuple) -{ - Py_ssize_t n = asdl_seq_LEN(elts); - if (n > 2 && are_all_items_const(elts, 0, n)) { - PyObject *folded = PyTuple_New(n); - if (folded == NULL) { - return ERROR; - } - PyObject *val; - for (Py_ssize_t i = 0; i < n; i++) { - val = ((expr_ty)asdl_seq_GET(elts, i))->v.Constant.value; - PyTuple_SET_ITEM(folded, i, Py_NewRef(val)); - } - if (tuple && !pushed) { - ADDOP_LOAD_CONST_NEW(c, loc, folded); - } else { - if (add == SET_ADD) { - Py_SETREF(folded, PyFrozenSet_New(folded)); - if (folded == NULL) { - return ERROR; - } - } - ADDOP_I(c, loc, build, pushed); - ADDOP_LOAD_CONST_NEW(c, loc, folded); - ADDOP_I(c, loc, extend, 1); - if (tuple) { - ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_LIST_TO_TUPLE); - } - } - return SUCCESS; - } - - int big = n+pushed > STACK_USE_GUIDELINE; - int seen_star = 0; - for (Py_ssize_t i = 0; i < n; i++) { - expr_ty elt = asdl_seq_GET(elts, i); - if (elt->kind == Starred_kind) { - seen_star = 1; - break; - } - } - if (!seen_star && !big) { - for (Py_ssize_t i = 0; i < n; i++) { - expr_ty elt = asdl_seq_GET(elts, i); - VISIT(c, expr, elt); - } - if (tuple) { - ADDOP_I(c, loc, BUILD_TUPLE, n+pushed); - } else { - ADDOP_I(c, loc, build, n+pushed); - } - return SUCCESS; - } - int sequence_built = 0; - if (big) { - ADDOP_I(c, loc, build, pushed); - sequence_built = 1; - } - for (Py_ssize_t i = 0; i < n; i++) { - expr_ty elt = asdl_seq_GET(elts, i); - if (elt->kind == Starred_kind) { - if (sequence_built == 0) { - ADDOP_I(c, loc, build, i+pushed); - sequence_built = 1; - } - VISIT(c, expr, elt->v.Starred.value); - ADDOP_I(c, loc, extend, 1); - } - else { - VISIT(c, expr, elt); - if (sequence_built) { - ADDOP_I(c, loc, add, 1); - } - } - } - assert(sequence_built); - if (tuple) { - ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_LIST_TO_TUPLE); - } - return SUCCESS; -} - -static int -unpack_helper(struct compiler *c, location loc, asdl_expr_seq *elts) -{ - Py_ssize_t n = asdl_seq_LEN(elts); - int seen_star = 0; - for (Py_ssize_t i = 0; i < n; i++) { - expr_ty elt = asdl_seq_GET(elts, i); - if (elt->kind == Starred_kind && !seen_star) { - if ((i >= (1 << 8)) || - (n-i-1 >= (INT_MAX >> 8))) { - return compiler_error(c, loc, - "too many expressions in " - "star-unpacking assignment"); - } - ADDOP_I(c, loc, UNPACK_EX, (i + ((n-i-1) << 8))); - seen_star = 1; - } - else if (elt->kind == Starred_kind) { - return compiler_error(c, loc, - "multiple starred expressions in assignment"); - } - } - if (!seen_star) { - ADDOP_I(c, loc, UNPACK_SEQUENCE, n); - } - return SUCCESS; -} - -static int -assignment_helper(struct compiler *c, location loc, asdl_expr_seq *elts) -{ - Py_ssize_t n = asdl_seq_LEN(elts); - RETURN_IF_ERROR(unpack_helper(c, loc, elts)); - for (Py_ssize_t i = 0; i < n; i++) { - expr_ty elt = asdl_seq_GET(elts, i); - VISIT(c, expr, elt->kind != Starred_kind ? elt : elt->v.Starred.value); - } - return SUCCESS; -} - -static int -compiler_list(struct compiler *c, expr_ty e) -{ - location loc = LOC(e); - asdl_expr_seq *elts = e->v.List.elts; - if (e->v.List.ctx == Store) { - return assignment_helper(c, loc, elts); - } - else if (e->v.List.ctx == Load) { - return starunpack_helper(c, loc, elts, 0, - BUILD_LIST, LIST_APPEND, LIST_EXTEND, 0); - } - else { - VISIT_SEQ(c, expr, elts); - } - return SUCCESS; -} - -static int -compiler_tuple(struct compiler *c, expr_ty e) -{ - location loc = LOC(e); - asdl_expr_seq *elts = e->v.Tuple.elts; - if (e->v.Tuple.ctx == Store) { - return assignment_helper(c, loc, elts); - } - else if (e->v.Tuple.ctx == Load) { - return starunpack_helper(c, loc, elts, 0, - BUILD_LIST, LIST_APPEND, LIST_EXTEND, 1); - } - else { - VISIT_SEQ(c, expr, elts); - } - return SUCCESS; -} - -static int -compiler_set(struct compiler *c, expr_ty e) -{ - location loc = LOC(e); - return starunpack_helper(c, loc, e->v.Set.elts, 0, - BUILD_SET, SET_ADD, SET_UPDATE, 0); -} - -static bool -are_all_items_const(asdl_expr_seq *seq, Py_ssize_t begin, Py_ssize_t end) -{ - for (Py_ssize_t i = begin; i < end; i++) { - expr_ty key = (expr_ty)asdl_seq_GET(seq, i); - if (key == NULL || key->kind != Constant_kind) { - return false; - } - } - return true; -} - -static int -compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end) -{ - Py_ssize_t i, n = end - begin; - PyObject *keys, *key; - int big = n*2 > STACK_USE_GUIDELINE; - location loc = LOC(e); - if (n > 1 && !big && are_all_items_const(e->v.Dict.keys, begin, end)) { - for (i = begin; i < end; i++) { - VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i)); - } - keys = PyTuple_New(n); - if (keys == NULL) { - return SUCCESS; - } - for (i = begin; i < end; i++) { - key = ((expr_ty)asdl_seq_GET(e->v.Dict.keys, i))->v.Constant.value; - PyTuple_SET_ITEM(keys, i - begin, Py_NewRef(key)); - } - ADDOP_LOAD_CONST_NEW(c, loc, keys); - ADDOP_I(c, loc, BUILD_CONST_KEY_MAP, n); - return SUCCESS; - } - if (big) { - ADDOP_I(c, loc, BUILD_MAP, 0); - } - for (i = begin; i < end; i++) { - VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.keys, i)); - VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i)); - if (big) { - ADDOP_I(c, loc, MAP_ADD, 1); - } - } - if (!big) { - ADDOP_I(c, loc, BUILD_MAP, n); - } - return SUCCESS; -} - -static int -compiler_dict(struct compiler *c, expr_ty e) -{ - location loc = LOC(e); - Py_ssize_t i, n, elements; - int have_dict; - int is_unpacking = 0; - n = asdl_seq_LEN(e->v.Dict.values); - have_dict = 0; - elements = 0; - for (i = 0; i < n; i++) { - is_unpacking = (expr_ty)asdl_seq_GET(e->v.Dict.keys, i) == NULL; - if (is_unpacking) { - if (elements) { - RETURN_IF_ERROR(compiler_subdict(c, e, i - elements, i)); - if (have_dict) { - ADDOP_I(c, loc, DICT_UPDATE, 1); - } - have_dict = 1; - elements = 0; - } - if (have_dict == 0) { - ADDOP_I(c, loc, BUILD_MAP, 0); - have_dict = 1; - } - VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i)); - ADDOP_I(c, loc, DICT_UPDATE, 1); - } - else { - if (elements*2 > STACK_USE_GUIDELINE) { - RETURN_IF_ERROR(compiler_subdict(c, e, i - elements, i + 1)); - if (have_dict) { - ADDOP_I(c, loc, DICT_UPDATE, 1); - } - have_dict = 1; - elements = 0; - } - else { - elements++; - } - } - } - if (elements) { - RETURN_IF_ERROR(compiler_subdict(c, e, n - elements, n)); - if (have_dict) { - ADDOP_I(c, loc, DICT_UPDATE, 1); - } - have_dict = 1; - } - if (!have_dict) { - ADDOP_I(c, loc, BUILD_MAP, 0); - } - return SUCCESS; -} - -static int -compiler_compare(struct compiler *c, expr_ty e) -{ - location loc = LOC(e); - Py_ssize_t i, n; - - RETURN_IF_ERROR(check_compare(c, e)); - VISIT(c, expr, e->v.Compare.left); - assert(asdl_seq_LEN(e->v.Compare.ops) > 0); - n = asdl_seq_LEN(e->v.Compare.ops) - 1; - if (n == 0) { - VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, 0)); - ADDOP_COMPARE(c, loc, asdl_seq_GET(e->v.Compare.ops, 0)); - } - else { - NEW_JUMP_TARGET_LABEL(c, cleanup); - for (i = 0; i < n; i++) { - VISIT(c, expr, - (expr_ty)asdl_seq_GET(e->v.Compare.comparators, i)); - ADDOP_I(c, loc, SWAP, 2); - ADDOP_I(c, loc, COPY, 2); - ADDOP_COMPARE(c, loc, asdl_seq_GET(e->v.Compare.ops, i)); - ADDOP_I(c, loc, COPY, 1); - ADDOP(c, loc, TO_BOOL); - ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, cleanup); - ADDOP(c, loc, POP_TOP); - } - VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, n)); - ADDOP_COMPARE(c, loc, asdl_seq_GET(e->v.Compare.ops, n)); - NEW_JUMP_TARGET_LABEL(c, end); - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); - - USE_LABEL(c, cleanup); - ADDOP_I(c, loc, SWAP, 2); - ADDOP(c, loc, POP_TOP); - - USE_LABEL(c, end); - } - return SUCCESS; -} - -static PyTypeObject * -infer_type(expr_ty e) -{ - switch (e->kind) { - case Tuple_kind: - return &PyTuple_Type; - case List_kind: - case ListComp_kind: - return &PyList_Type; - case Dict_kind: - case DictComp_kind: - return &PyDict_Type; - case Set_kind: - case SetComp_kind: - return &PySet_Type; - case GeneratorExp_kind: - return &PyGen_Type; - case Lambda_kind: - return &PyFunction_Type; - case JoinedStr_kind: - case FormattedValue_kind: - return &PyUnicode_Type; - case Constant_kind: - return Py_TYPE(e->v.Constant.value); - default: - return NULL; - } -} - -static int -check_caller(struct compiler *c, expr_ty e) -{ - switch (e->kind) { - case Constant_kind: - case Tuple_kind: - case List_kind: - case ListComp_kind: - case Dict_kind: - case DictComp_kind: - case Set_kind: - case SetComp_kind: - case GeneratorExp_kind: - case JoinedStr_kind: - case FormattedValue_kind: { - location loc = LOC(e); - return compiler_warn(c, loc, "'%.200s' object is not callable; " - "perhaps you missed a comma?", - infer_type(e)->tp_name); - } - default: - return SUCCESS; - } -} - -static int -check_subscripter(struct compiler *c, expr_ty e) -{ - PyObject *v; - - switch (e->kind) { - case Constant_kind: - v = e->v.Constant.value; - if (!(v == Py_None || v == Py_Ellipsis || - PyLong_Check(v) || PyFloat_Check(v) || PyComplex_Check(v) || - PyAnySet_Check(v))) - { - return SUCCESS; - } - /* fall through */ - case Set_kind: - case SetComp_kind: - case GeneratorExp_kind: - case Lambda_kind: { - location loc = LOC(e); - return compiler_warn(c, loc, "'%.200s' object is not subscriptable; " - "perhaps you missed a comma?", - infer_type(e)->tp_name); - } - default: - return SUCCESS; - } -} - -static int -check_index(struct compiler *c, expr_ty e, expr_ty s) -{ - PyObject *v; - - PyTypeObject *index_type = infer_type(s); - if (index_type == NULL - || PyType_FastSubclass(index_type, Py_TPFLAGS_LONG_SUBCLASS) - || index_type == &PySlice_Type) { - return SUCCESS; - } - - switch (e->kind) { - case Constant_kind: - v = e->v.Constant.value; - if (!(PyUnicode_Check(v) || PyBytes_Check(v) || PyTuple_Check(v))) { - return SUCCESS; - } - /* fall through */ - case Tuple_kind: - case List_kind: - case ListComp_kind: - case JoinedStr_kind: - case FormattedValue_kind: { - location loc = LOC(e); - return compiler_warn(c, loc, "%.200s indices must be integers " - "or slices, not %.200s; " - "perhaps you missed a comma?", - infer_type(e)->tp_name, - index_type->tp_name); - } - default: - return SUCCESS; - } -} - -static int -is_import_originated(struct compiler *c, expr_ty e) -{ - /* Check whether the global scope has an import named - e, if it is a Name object. For not traversing all the - scope stack every time this function is called, it will - only check the global scope to determine whether something - is imported or not. */ - - if (e->kind != Name_kind) { - return 0; - } - - long flags = _PyST_GetSymbol(c->c_st->st_top, e->v.Name.id); - return flags & DEF_IMPORT; -} - -static int -can_optimize_super_call(struct compiler *c, expr_ty attr) -{ - expr_ty e = attr->v.Attribute.value; - if (e->kind != Call_kind || - e->v.Call.func->kind != Name_kind || - !_PyUnicode_EqualToASCIIString(e->v.Call.func->v.Name.id, "super") || - _PyUnicode_EqualToASCIIString(attr->v.Attribute.attr, "__class__") || - asdl_seq_LEN(e->v.Call.keywords) != 0) { - return 0; - } - Py_ssize_t num_args = asdl_seq_LEN(e->v.Call.args); - - PyObject *super_name = e->v.Call.func->v.Name.id; - // detect statically-visible shadowing of 'super' name - int scope = _PyST_GetScope(c->u->u_ste, super_name); - if (scope != GLOBAL_IMPLICIT) { - return 0; - } - scope = _PyST_GetScope(c->c_st->st_top, super_name); - if (scope != 0) { - return 0; - } - - if (num_args == 2) { - for (Py_ssize_t i = 0; i < num_args; i++) { - expr_ty elt = asdl_seq_GET(e->v.Call.args, i); - if (elt->kind == Starred_kind) { - return 0; - } - } - // exactly two non-starred args; we can just load - // the provided args - return 1; - } - - if (num_args != 0) { - return 0; - } - // we need the following for zero-arg super(): - - // enclosing function should have at least one argument - if (c->u->u_metadata.u_argcount == 0 && - c->u->u_metadata.u_posonlyargcount == 0) { - return 0; - } - // __class__ cell should be available - if (get_ref_type(c, &_Py_ID(__class__)) == FREE) { - return 1; - } - return 0; -} - -static int -load_args_for_super(struct compiler *c, expr_ty e) { - location loc = LOC(e); - - // load super() global - PyObject *super_name = e->v.Call.func->v.Name.id; - RETURN_IF_ERROR(compiler_nameop(c, LOC(e->v.Call.func), super_name, Load)); - - if (asdl_seq_LEN(e->v.Call.args) == 2) { - VISIT(c, expr, asdl_seq_GET(e->v.Call.args, 0)); - VISIT(c, expr, asdl_seq_GET(e->v.Call.args, 1)); - return SUCCESS; - } - - // load __class__ cell - PyObject *name = &_Py_ID(__class__); - assert(get_ref_type(c, name) == FREE); - RETURN_IF_ERROR(compiler_nameop(c, loc, name, Load)); - - // load self (first argument) - Py_ssize_t i = 0; - PyObject *key, *value; - if (!PyDict_Next(c->u->u_metadata.u_varnames, &i, &key, &value)) { - return ERROR; - } - RETURN_IF_ERROR(compiler_nameop(c, loc, key, Load)); - - return SUCCESS; -} - -// If an attribute access spans multiple lines, update the current start -// location to point to the attribute name. -static location -update_start_location_to_match_attr(struct compiler *c, location loc, - expr_ty attr) -{ - assert(attr->kind == Attribute_kind); - if (loc.lineno != attr->end_lineno) { - loc.lineno = attr->end_lineno; - int len = (int)PyUnicode_GET_LENGTH(attr->v.Attribute.attr); - if (len <= attr->end_col_offset) { - loc.col_offset = attr->end_col_offset - len; - } - else { - // GH-94694: Somebody's compiling weird ASTs. Just drop the columns: - loc.col_offset = -1; - loc.end_col_offset = -1; - } - // Make sure the end position still follows the start position, even for - // weird ASTs: - loc.end_lineno = Py_MAX(loc.lineno, loc.end_lineno); - if (loc.lineno == loc.end_lineno) { - loc.end_col_offset = Py_MAX(loc.col_offset, loc.end_col_offset); - } - } - return loc; -} - -// Return 1 if the method call was optimized, 0 if not, and -1 on error. -static int -maybe_optimize_method_call(struct compiler *c, expr_ty e) -{ - Py_ssize_t argsl, i, kwdsl; - expr_ty meth = e->v.Call.func; - asdl_expr_seq *args = e->v.Call.args; - asdl_keyword_seq *kwds = e->v.Call.keywords; - - /* Check that the call node is an attribute access */ - if (meth->kind != Attribute_kind || meth->v.Attribute.ctx != Load) { - return 0; - } - - /* Check that the base object is not something that is imported */ - if (is_import_originated(c, meth->v.Attribute.value)) { - return 0; - } - - /* Check that there aren't too many arguments */ - argsl = asdl_seq_LEN(args); - kwdsl = asdl_seq_LEN(kwds); - if (argsl + kwdsl + (kwdsl != 0) >= STACK_USE_GUIDELINE) { - return 0; - } - /* Check that there are no *varargs types of arguments. */ - for (i = 0; i < argsl; i++) { - expr_ty elt = asdl_seq_GET(args, i); - if (elt->kind == Starred_kind) { - return 0; - } - } - - for (i = 0; i < kwdsl; i++) { - keyword_ty kw = asdl_seq_GET(kwds, i); - if (kw->arg == NULL) { - return 0; - } - } - - /* Alright, we can optimize the code. */ - location loc = LOC(meth); - - if (can_optimize_super_call(c, meth)) { - RETURN_IF_ERROR(load_args_for_super(c, meth->v.Attribute.value)); - int opcode = asdl_seq_LEN(meth->v.Attribute.value->v.Call.args) ? - LOAD_SUPER_METHOD : LOAD_ZERO_SUPER_METHOD; - ADDOP_NAME(c, loc, opcode, meth->v.Attribute.attr, names); - loc = update_start_location_to_match_attr(c, loc, meth); - ADDOP(c, loc, NOP); - } else { - VISIT(c, expr, meth->v.Attribute.value); - loc = update_start_location_to_match_attr(c, loc, meth); - ADDOP_NAME(c, loc, LOAD_METHOD, meth->v.Attribute.attr, names); - } - - VISIT_SEQ(c, expr, e->v.Call.args); - - if (kwdsl) { - VISIT_SEQ(c, keyword, kwds); - RETURN_IF_ERROR( - compiler_call_simple_kw_helper(c, loc, kwds, kwdsl)); - loc = update_start_location_to_match_attr(c, LOC(e), meth); - ADDOP_I(c, loc, CALL_KW, argsl + kwdsl); - } - else { - loc = update_start_location_to_match_attr(c, LOC(e), meth); - ADDOP_I(c, loc, CALL, argsl); - } - return 1; -} - -static int -validate_keywords(struct compiler *c, asdl_keyword_seq *keywords) -{ - Py_ssize_t nkeywords = asdl_seq_LEN(keywords); - for (Py_ssize_t i = 0; i < nkeywords; i++) { - keyword_ty key = ((keyword_ty)asdl_seq_GET(keywords, i)); - if (key->arg == NULL) { - continue; - } - location loc = LOC(key); - if (forbidden_name(c, loc, key->arg, Store)) { - return ERROR; - } - for (Py_ssize_t j = i + 1; j < nkeywords; j++) { - keyword_ty other = ((keyword_ty)asdl_seq_GET(keywords, j)); - if (other->arg && !PyUnicode_Compare(key->arg, other->arg)) { - compiler_error(c, LOC(other), "keyword argument repeated: %U", key->arg); - return ERROR; - } - } - } - return SUCCESS; -} - -static int -compiler_call(struct compiler *c, expr_ty e) -{ - RETURN_IF_ERROR(validate_keywords(c, e->v.Call.keywords)); - int ret = maybe_optimize_method_call(c, e); - if (ret < 0) { - return ERROR; - } - if (ret == 1) { - return SUCCESS; - } - RETURN_IF_ERROR(check_caller(c, e->v.Call.func)); - VISIT(c, expr, e->v.Call.func); - location loc = LOC(e->v.Call.func); - ADDOP(c, loc, PUSH_NULL); - loc = LOC(e); - return compiler_call_helper(c, loc, 0, - e->v.Call.args, - e->v.Call.keywords); -} - -static int -compiler_joined_str(struct compiler *c, expr_ty e) -{ - location loc = LOC(e); - Py_ssize_t value_count = asdl_seq_LEN(e->v.JoinedStr.values); - if (value_count > STACK_USE_GUIDELINE) { - _Py_DECLARE_STR(empty, ""); - ADDOP_LOAD_CONST_NEW(c, loc, Py_NewRef(&_Py_STR(empty))); - ADDOP_NAME(c, loc, LOAD_METHOD, &_Py_ID(join), names); - ADDOP_I(c, loc, BUILD_LIST, 0); - for (Py_ssize_t i = 0; i < asdl_seq_LEN(e->v.JoinedStr.values); i++) { - VISIT(c, expr, asdl_seq_GET(e->v.JoinedStr.values, i)); - ADDOP_I(c, loc, LIST_APPEND, 1); - } - ADDOP_I(c, loc, CALL, 1); - } - else { - VISIT_SEQ(c, expr, e->v.JoinedStr.values); - if (value_count > 1) { - ADDOP_I(c, loc, BUILD_STRING, value_count); - } - else if (value_count == 0) { - _Py_DECLARE_STR(empty, ""); - ADDOP_LOAD_CONST_NEW(c, loc, Py_NewRef(&_Py_STR(empty))); - } - } - return SUCCESS; -} - -/* Used to implement f-strings. Format a single value. */ -static int -compiler_formatted_value(struct compiler *c, expr_ty e) -{ - /* Our oparg encodes 2 pieces of information: the conversion - character, and whether or not a format_spec was provided. - - Convert the conversion char to 3 bits: - : 000 0x0 FVC_NONE The default if nothing specified. - !s : 001 0x1 FVC_STR - !r : 010 0x2 FVC_REPR - !a : 011 0x3 FVC_ASCII - - next bit is whether or not we have a format spec: - yes : 100 0x4 - no : 000 0x0 - */ - - int conversion = e->v.FormattedValue.conversion; - int oparg; - - /* The expression to be formatted. */ - VISIT(c, expr, e->v.FormattedValue.value); - - location loc = LOC(e); - if (conversion != -1) { - switch (conversion) { - case 's': oparg = FVC_STR; break; - case 'r': oparg = FVC_REPR; break; - case 'a': oparg = FVC_ASCII; break; - default: - PyErr_Format(PyExc_SystemError, - "Unrecognized conversion character %d", conversion); - return ERROR; - } - ADDOP_I(c, loc, CONVERT_VALUE, oparg); - } - if (e->v.FormattedValue.format_spec) { - /* Evaluate the format spec, and update our opcode arg. */ - VISIT(c, expr, e->v.FormattedValue.format_spec); - ADDOP(c, loc, FORMAT_WITH_SPEC); - } else { - ADDOP(c, loc, FORMAT_SIMPLE); - } - return SUCCESS; -} - -static int -compiler_subkwargs(struct compiler *c, location loc, - asdl_keyword_seq *keywords, - Py_ssize_t begin, Py_ssize_t end) -{ - Py_ssize_t i, n = end - begin; - keyword_ty kw; - PyObject *keys, *key; - assert(n > 0); - int big = n*2 > STACK_USE_GUIDELINE; - if (n > 1 && !big) { - for (i = begin; i < end; i++) { - kw = asdl_seq_GET(keywords, i); - VISIT(c, expr, kw->value); - } - keys = PyTuple_New(n); - if (keys == NULL) { - return ERROR; - } - for (i = begin; i < end; i++) { - key = ((keyword_ty) asdl_seq_GET(keywords, i))->arg; - PyTuple_SET_ITEM(keys, i - begin, Py_NewRef(key)); - } - ADDOP_LOAD_CONST_NEW(c, loc, keys); - ADDOP_I(c, loc, BUILD_CONST_KEY_MAP, n); - return SUCCESS; - } - if (big) { - ADDOP_I(c, NO_LOCATION, BUILD_MAP, 0); - } - for (i = begin; i < end; i++) { - kw = asdl_seq_GET(keywords, i); - ADDOP_LOAD_CONST(c, loc, kw->arg); - VISIT(c, expr, kw->value); - if (big) { - ADDOP_I(c, NO_LOCATION, MAP_ADD, 1); - } - } - if (!big) { - ADDOP_I(c, loc, BUILD_MAP, n); - } - return SUCCESS; -} - -/* Used by compiler_call_helper and maybe_optimize_method_call to emit - * a tuple of keyword names before CALL. - */ -static int -compiler_call_simple_kw_helper(struct compiler *c, location loc, - asdl_keyword_seq *keywords, Py_ssize_t nkwelts) -{ - PyObject *names; - names = PyTuple_New(nkwelts); - if (names == NULL) { - return ERROR; - } - for (int i = 0; i < nkwelts; i++) { - keyword_ty kw = asdl_seq_GET(keywords, i); - PyTuple_SET_ITEM(names, i, Py_NewRef(kw->arg)); - } - ADDOP_LOAD_CONST_NEW(c, loc, names); - return SUCCESS; -} - - -/* shared code between compiler_call and compiler_class */ -static int -compiler_call_helper(struct compiler *c, location loc, - int n, /* Args already pushed */ - asdl_expr_seq *args, - asdl_keyword_seq *keywords) -{ - Py_ssize_t i, nseen, nelts, nkwelts; - - RETURN_IF_ERROR(validate_keywords(c, keywords)); - - nelts = asdl_seq_LEN(args); - nkwelts = asdl_seq_LEN(keywords); - - if (nelts + nkwelts*2 > STACK_USE_GUIDELINE) { - goto ex_call; - } - for (i = 0; i < nelts; i++) { - expr_ty elt = asdl_seq_GET(args, i); - if (elt->kind == Starred_kind) { - goto ex_call; - } - } - for (i = 0; i < nkwelts; i++) { - keyword_ty kw = asdl_seq_GET(keywords, i); - if (kw->arg == NULL) { - goto ex_call; - } - } - - /* No * or ** args, so can use faster calling sequence */ - for (i = 0; i < nelts; i++) { - expr_ty elt = asdl_seq_GET(args, i); - assert(elt->kind != Starred_kind); - VISIT(c, expr, elt); - } - if (nkwelts) { - VISIT_SEQ(c, keyword, keywords); - RETURN_IF_ERROR( - compiler_call_simple_kw_helper(c, loc, keywords, nkwelts)); - ADDOP_I(c, loc, CALL_KW, n + nelts + nkwelts); - } - else { - ADDOP_I(c, loc, CALL, n + nelts); - } - return SUCCESS; - -ex_call: - - /* Do positional arguments. */ - if (n ==0 && nelts == 1 && ((expr_ty)asdl_seq_GET(args, 0))->kind == Starred_kind) { - VISIT(c, expr, ((expr_ty)asdl_seq_GET(args, 0))->v.Starred.value); - } - else { - RETURN_IF_ERROR(starunpack_helper(c, loc, args, n, BUILD_LIST, - LIST_APPEND, LIST_EXTEND, 1)); - } - /* Then keyword arguments */ - if (nkwelts) { - /* Has a new dict been pushed */ - int have_dict = 0; - - nseen = 0; /* the number of keyword arguments on the stack following */ - for (i = 0; i < nkwelts; i++) { - keyword_ty kw = asdl_seq_GET(keywords, i); - if (kw->arg == NULL) { - /* A keyword argument unpacking. */ - if (nseen) { - RETURN_IF_ERROR(compiler_subkwargs(c, loc, keywords, i - nseen, i)); - if (have_dict) { - ADDOP_I(c, loc, DICT_MERGE, 1); - } - have_dict = 1; - nseen = 0; - } - if (!have_dict) { - ADDOP_I(c, loc, BUILD_MAP, 0); - have_dict = 1; - } - VISIT(c, expr, kw->value); - ADDOP_I(c, loc, DICT_MERGE, 1); - } - else { - nseen++; - } - } - if (nseen) { - /* Pack up any trailing keyword arguments. */ - RETURN_IF_ERROR(compiler_subkwargs(c, loc, keywords, nkwelts - nseen, nkwelts)); - if (have_dict) { - ADDOP_I(c, loc, DICT_MERGE, 1); - } - have_dict = 1; - } - assert(have_dict); - } - ADDOP_I(c, loc, CALL_FUNCTION_EX, nkwelts > 0); - return SUCCESS; -} - - -/* List and set comprehensions and generator expressions work by creating a - nested function to perform the actual iteration. This means that the - iteration variables don't leak into the current scope. - The defined function is called immediately following its definition, with the - result of that call being the result of the expression. - The LC/SC version returns the populated container, while the GE version is - flagged in symtable.c as a generator, so it returns the generator object - when the function is called. - - Possible cleanups: - - iterate over the generator sequence instead of using recursion -*/ - - -static int -compiler_comprehension_generator(struct compiler *c, location loc, - asdl_comprehension_seq *generators, int gen_index, - int depth, - expr_ty elt, expr_ty val, int type, - int iter_on_stack) -{ - comprehension_ty gen; - gen = (comprehension_ty)asdl_seq_GET(generators, gen_index); - if (gen->is_async) { - return compiler_async_comprehension_generator( - c, loc, generators, gen_index, depth, elt, val, type, - iter_on_stack); - } else { - return compiler_sync_comprehension_generator( - c, loc, generators, gen_index, depth, elt, val, type, - iter_on_stack); - } -} - -static int -compiler_sync_comprehension_generator(struct compiler *c, location loc, - asdl_comprehension_seq *generators, - int gen_index, int depth, - expr_ty elt, expr_ty val, int type, - int iter_on_stack) -{ - /* generate code for the iterator, then each of the ifs, - and then write to the element */ - - NEW_JUMP_TARGET_LABEL(c, start); - NEW_JUMP_TARGET_LABEL(c, if_cleanup); - NEW_JUMP_TARGET_LABEL(c, anchor); - - comprehension_ty gen = (comprehension_ty)asdl_seq_GET(generators, - gen_index); - - if (!iter_on_stack) { - if (gen_index == 0) { - /* Receive outermost iter as an implicit argument */ - c->u->u_metadata.u_argcount = 1; - ADDOP_I(c, loc, LOAD_FAST, 0); - } - else { - /* Sub-iter - calculate on the fly */ - /* Fast path for the temporary variable assignment idiom: - for y in [f(x)] - */ - asdl_expr_seq *elts; - switch (gen->iter->kind) { - case List_kind: - elts = gen->iter->v.List.elts; - break; - case Tuple_kind: - elts = gen->iter->v.Tuple.elts; - break; - default: - elts = NULL; - } - if (asdl_seq_LEN(elts) == 1) { - expr_ty elt = asdl_seq_GET(elts, 0); - if (elt->kind != Starred_kind) { - VISIT(c, expr, elt); - start = NO_LABEL; - } - } - if (IS_LABEL(start)) { - VISIT(c, expr, gen->iter); - ADDOP(c, loc, GET_ITER); - } - } - } - if (IS_LABEL(start)) { - depth++; - USE_LABEL(c, start); - ADDOP_JUMP(c, loc, FOR_ITER, anchor); - } - VISIT(c, expr, gen->target); - - /* XXX this needs to be cleaned up...a lot! */ - Py_ssize_t n = asdl_seq_LEN(gen->ifs); - for (Py_ssize_t i = 0; i < n; i++) { - expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i); - RETURN_IF_ERROR(compiler_jump_if(c, loc, e, if_cleanup, 0)); - } - - if (++gen_index < asdl_seq_LEN(generators)) { - RETURN_IF_ERROR( - compiler_comprehension_generator(c, loc, - generators, gen_index, depth, - elt, val, type, 0)); - } - - location elt_loc = LOC(elt); - - /* only append after the last for generator */ - if (gen_index >= asdl_seq_LEN(generators)) { - /* comprehension specific code */ - switch (type) { - case COMP_GENEXP: - VISIT(c, expr, elt); - ADDOP_YIELD(c, elt_loc); - ADDOP(c, elt_loc, POP_TOP); - break; - case COMP_LISTCOMP: - VISIT(c, expr, elt); - ADDOP_I(c, elt_loc, LIST_APPEND, depth + 1); - break; - case COMP_SETCOMP: - VISIT(c, expr, elt); - ADDOP_I(c, elt_loc, SET_ADD, depth + 1); - break; - case COMP_DICTCOMP: - /* With '{k: v}', k is evaluated before v, so we do - the same. */ - VISIT(c, expr, elt); - VISIT(c, expr, val); - elt_loc = LOCATION(elt->lineno, - val->end_lineno, - elt->col_offset, - val->end_col_offset); - ADDOP_I(c, elt_loc, MAP_ADD, depth + 1); - break; - default: - return ERROR; - } - } - - USE_LABEL(c, if_cleanup); - if (IS_LABEL(start)) { - ADDOP_JUMP(c, elt_loc, JUMP, start); - - USE_LABEL(c, anchor); - /* It is important for instrumentation that the `END_FOR` comes first. - * Iteration over a generator will jump to the first of these instructions, - * but a non-generator will jump to a later instruction. - */ - ADDOP(c, NO_LOCATION, END_FOR); - ADDOP(c, NO_LOCATION, POP_TOP); - } - - return SUCCESS; -} - -static int -compiler_async_comprehension_generator(struct compiler *c, location loc, - asdl_comprehension_seq *generators, - int gen_index, int depth, - expr_ty elt, expr_ty val, int type, - int iter_on_stack) -{ - NEW_JUMP_TARGET_LABEL(c, start); - NEW_JUMP_TARGET_LABEL(c, except); - NEW_JUMP_TARGET_LABEL(c, if_cleanup); - - comprehension_ty gen = (comprehension_ty)asdl_seq_GET(generators, - gen_index); - - if (!iter_on_stack) { - if (gen_index == 0) { - /* Receive outermost iter as an implicit argument */ - c->u->u_metadata.u_argcount = 1; - ADDOP_I(c, loc, LOAD_FAST, 0); - } - else { - /* Sub-iter - calculate on the fly */ - VISIT(c, expr, gen->iter); - ADDOP(c, loc, GET_AITER); - } - } - - USE_LABEL(c, start); - /* Runtime will push a block here, so we need to account for that */ - RETURN_IF_ERROR( - compiler_push_fblock(c, loc, ASYNC_COMPREHENSION_GENERATOR, - start, NO_LABEL, NULL)); - - ADDOP_JUMP(c, loc, SETUP_FINALLY, except); - ADDOP(c, loc, GET_ANEXT); - ADDOP_LOAD_CONST(c, loc, Py_None); - ADD_YIELD_FROM(c, loc, 1); - ADDOP(c, loc, POP_BLOCK); - VISIT(c, expr, gen->target); - - Py_ssize_t n = asdl_seq_LEN(gen->ifs); - for (Py_ssize_t i = 0; i < n; i++) { - expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i); - RETURN_IF_ERROR(compiler_jump_if(c, loc, e, if_cleanup, 0)); - } - - depth++; - if (++gen_index < asdl_seq_LEN(generators)) { - RETURN_IF_ERROR( - compiler_comprehension_generator(c, loc, - generators, gen_index, depth, - elt, val, type, 0)); - } - - location elt_loc = LOC(elt); - /* only append after the last for generator */ - if (gen_index >= asdl_seq_LEN(generators)) { - /* comprehension specific code */ - switch (type) { - case COMP_GENEXP: - VISIT(c, expr, elt); - ADDOP_YIELD(c, elt_loc); - ADDOP(c, elt_loc, POP_TOP); - break; - case COMP_LISTCOMP: - VISIT(c, expr, elt); - ADDOP_I(c, elt_loc, LIST_APPEND, depth + 1); - break; - case COMP_SETCOMP: - VISIT(c, expr, elt); - ADDOP_I(c, elt_loc, SET_ADD, depth + 1); - break; - case COMP_DICTCOMP: - /* With '{k: v}', k is evaluated before v, so we do - the same. */ - VISIT(c, expr, elt); - VISIT(c, expr, val); - elt_loc = LOCATION(elt->lineno, - val->end_lineno, - elt->col_offset, - val->end_col_offset); - ADDOP_I(c, elt_loc, MAP_ADD, depth + 1); - break; - default: - return ERROR; - } - } - - USE_LABEL(c, if_cleanup); - ADDOP_JUMP(c, elt_loc, JUMP, start); - - compiler_pop_fblock(c, ASYNC_COMPREHENSION_GENERATOR, start); - - USE_LABEL(c, except); - - ADDOP(c, loc, END_ASYNC_FOR); - - return SUCCESS; -} - -typedef struct { - PyObject *pushed_locals; - PyObject *temp_symbols; - PyObject *fast_hidden; - jump_target_label cleanup; - jump_target_label end; -} inlined_comprehension_state; - -static int -push_inlined_comprehension_state(struct compiler *c, location loc, - PySTEntryObject *entry, - inlined_comprehension_state *state) -{ - int in_class_block = (c->u->u_ste->ste_type == ClassBlock) && !c->u->u_in_inlined_comp; - c->u->u_in_inlined_comp++; - // iterate over names bound in the comprehension and ensure we isolate - // them from the outer scope as needed - PyObject *k, *v; - Py_ssize_t pos = 0; - while (PyDict_Next(entry->ste_symbols, &pos, &k, &v)) { - assert(PyLong_Check(v)); - long symbol = PyLong_AS_LONG(v); - // only values bound in the comprehension (DEF_LOCAL) need to be handled - // at all; DEF_LOCAL | DEF_NONLOCAL can occur in the case of an - // assignment expression to a nonlocal in the comprehension, these don't - // need handling here since they shouldn't be isolated - if ((symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) || in_class_block) { - if (!_PyST_IsFunctionLike(c->u->u_ste)) { - // non-function scope: override this name to use fast locals - PyObject *orig; - if (PyDict_GetItemRef(c->u->u_metadata.u_fasthidden, k, &orig) < 0) { - return ERROR; - } - int orig_is_true = (orig == Py_True); - Py_XDECREF(orig); - if (!orig_is_true) { - if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_True) < 0) { - return ERROR; - } - if (state->fast_hidden == NULL) { - state->fast_hidden = PySet_New(NULL); - if (state->fast_hidden == NULL) { - return ERROR; - } - } - if (PySet_Add(state->fast_hidden, k) < 0) { - return ERROR; - } - } - } - long scope = (symbol >> SCOPE_OFFSET) & SCOPE_MASK; - PyObject *outv = PyDict_GetItemWithError(c->u->u_ste->ste_symbols, k); - if (outv == NULL) { - outv = _PyLong_GetZero(); - } - assert(PyLong_Check(outv)); - long outsc = (PyLong_AS_LONG(outv) >> SCOPE_OFFSET) & SCOPE_MASK; - if (scope != outsc && !(scope == CELL && outsc == FREE)) { - // If a name has different scope inside than outside the - // comprehension, we need to temporarily handle it with the - // right scope while compiling the comprehension. (If it's free - // in outer scope and cell in inner scope, we can't treat it as - // both cell and free in the same function, but treating it as - // free throughout is fine; it's *_DEREF either way.) - - if (state->temp_symbols == NULL) { - state->temp_symbols = PyDict_New(); - if (state->temp_symbols == NULL) { - return ERROR; - } - } - // update the symbol to the in-comprehension version and save - // the outer version; we'll restore it after running the - // comprehension - Py_INCREF(outv); - if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v) < 0) { - Py_DECREF(outv); - return ERROR; - } - if (PyDict_SetItem(state->temp_symbols, k, outv) < 0) { - Py_DECREF(outv); - return ERROR; - } - Py_DECREF(outv); - } - // local names bound in comprehension must be isolated from - // outer scope; push existing value (which may be NULL if - // not defined) on stack - if (state->pushed_locals == NULL) { - state->pushed_locals = PyList_New(0); - if (state->pushed_locals == NULL) { - return ERROR; - } - } - // in the case of a cell, this will actually push the cell - // itself to the stack, then we'll create a new one for the - // comprehension and restore the original one after - ADDOP_NAME(c, loc, LOAD_FAST_AND_CLEAR, k, varnames); - if (scope == CELL) { - if (outsc == FREE) { - ADDOP_NAME(c, loc, MAKE_CELL, k, freevars); - } else { - ADDOP_NAME(c, loc, MAKE_CELL, k, cellvars); - } - } - if (PyList_Append(state->pushed_locals, k) < 0) { - return ERROR; - } - } - } - if (state->pushed_locals) { - // Outermost iterable expression was already evaluated and is on the - // stack, we need to swap it back to TOS. This also rotates the order of - // `pushed_locals` on the stack, but this will be reversed when we swap - // out the comprehension result in pop_inlined_comprehension_state - ADDOP_I(c, loc, SWAP, PyList_GET_SIZE(state->pushed_locals) + 1); - - // Add our own cleanup handler to restore comprehension locals in case - // of exception, so they have the correct values inside an exception - // handler or finally block. - NEW_JUMP_TARGET_LABEL(c, cleanup); - state->cleanup = cleanup; - NEW_JUMP_TARGET_LABEL(c, end); - state->end = end; - - // no need to push an fblock for this "virtual" try/finally; there can't - // be return/continue/break inside a comprehension - ADDOP_JUMP(c, loc, SETUP_FINALLY, cleanup); - } - - return SUCCESS; -} - -static int -restore_inlined_comprehension_locals(struct compiler *c, location loc, - inlined_comprehension_state state) -{ - PyObject *k; - // pop names we pushed to stack earlier - Py_ssize_t npops = PyList_GET_SIZE(state.pushed_locals); - // Preserve the comprehension result (or exception) as TOS. This - // reverses the SWAP we did in push_inlined_comprehension_state to get - // the outermost iterable to TOS, so we can still just iterate - // pushed_locals in simple reverse order - ADDOP_I(c, loc, SWAP, npops + 1); - for (Py_ssize_t i = npops - 1; i >= 0; --i) { - k = PyList_GetItem(state.pushed_locals, i); - if (k == NULL) { - return ERROR; - } - ADDOP_NAME(c, loc, STORE_FAST_MAYBE_NULL, k, varnames); - } - return SUCCESS; -} - -static int -pop_inlined_comprehension_state(struct compiler *c, location loc, - inlined_comprehension_state state) -{ - c->u->u_in_inlined_comp--; - PyObject *k, *v; - Py_ssize_t pos = 0; - if (state.temp_symbols) { - while (PyDict_Next(state.temp_symbols, &pos, &k, &v)) { - if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v)) { - return ERROR; - } - } - Py_CLEAR(state.temp_symbols); - } - if (state.pushed_locals) { - ADDOP(c, NO_LOCATION, POP_BLOCK); - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, state.end); - - // cleanup from an exception inside the comprehension - USE_LABEL(c, state.cleanup); - // discard incomplete comprehension result (beneath exc on stack) - ADDOP_I(c, NO_LOCATION, SWAP, 2); - ADDOP(c, NO_LOCATION, POP_TOP); - if (restore_inlined_comprehension_locals(c, loc, state) < 0) { - return ERROR; - } - ADDOP_I(c, NO_LOCATION, RERAISE, 0); - - USE_LABEL(c, state.end); - if (restore_inlined_comprehension_locals(c, loc, state) < 0) { - return ERROR; - } - Py_CLEAR(state.pushed_locals); - } - if (state.fast_hidden) { - while (PySet_Size(state.fast_hidden) > 0) { - PyObject *k = PySet_Pop(state.fast_hidden); - if (k == NULL) { - return ERROR; - } - // we set to False instead of clearing, so we can track which names - // were temporarily fast-locals and should use CO_FAST_HIDDEN - if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_False)) { - Py_DECREF(k); - return ERROR; - } - Py_DECREF(k); - } - Py_CLEAR(state.fast_hidden); - } - return SUCCESS; -} - -static inline int -compiler_comprehension_iter(struct compiler *c, location loc, - comprehension_ty comp) -{ - VISIT(c, expr, comp->iter); - if (comp->is_async) { - ADDOP(c, loc, GET_AITER); - } - else { - ADDOP(c, loc, GET_ITER); - } - return SUCCESS; -} - -static int -compiler_comprehension(struct compiler *c, expr_ty e, int type, - identifier name, asdl_comprehension_seq *generators, expr_ty elt, - expr_ty val) -{ - PyCodeObject *co = NULL; - inlined_comprehension_state inline_state = {NULL, NULL, NULL, NO_LABEL, NO_LABEL}; - comprehension_ty outermost; - int scope_type = c->u->u_scope_type; - int is_top_level_await = IS_TOP_LEVEL_AWAIT(c); - PySTEntryObject *entry = _PySymtable_Lookup(c->c_st, (void *)e); - if (entry == NULL) { - goto error; - } - int is_inlined = entry->ste_comp_inlined; - int is_async_generator = entry->ste_coroutine; - - location loc = LOC(e); - - outermost = (comprehension_ty) asdl_seq_GET(generators, 0); - if (is_inlined) { - if (compiler_comprehension_iter(c, loc, outermost)) { - goto error; - } - if (push_inlined_comprehension_state(c, loc, entry, &inline_state)) { - goto error; - } - } - else { - if (compiler_enter_scope(c, name, COMPILER_SCOPE_COMPREHENSION, - (void *)e, e->lineno) < 0) - { - goto error; - } - } - Py_CLEAR(entry); - - if (is_async_generator && type != COMP_GENEXP && - scope_type != COMPILER_SCOPE_ASYNC_FUNCTION && - scope_type != COMPILER_SCOPE_COMPREHENSION && - !is_top_level_await) - { - compiler_error(c, loc, "asynchronous comprehension outside of " - "an asynchronous function"); - goto error_in_scope; - } - - if (type != COMP_GENEXP) { - int op; - switch (type) { - case COMP_LISTCOMP: - op = BUILD_LIST; - break; - case COMP_SETCOMP: - op = BUILD_SET; - break; - case COMP_DICTCOMP: - op = BUILD_MAP; - break; - default: - PyErr_Format(PyExc_SystemError, - "unknown comprehension type %d", type); - goto error_in_scope; - } - - ADDOP_I(c, loc, op, 0); - if (is_inlined) { - ADDOP_I(c, loc, SWAP, 2); - } - } - - if (compiler_comprehension_generator(c, loc, generators, 0, 0, - elt, val, type, is_inlined) < 0) { - goto error_in_scope; - } - - if (is_inlined) { - if (pop_inlined_comprehension_state(c, loc, inline_state)) { - goto error; - } - return SUCCESS; - } - - if (type != COMP_GENEXP) { - ADDOP(c, LOC(e), RETURN_VALUE); - } - if (type == COMP_GENEXP) { - if (wrap_in_stopiteration_handler(c) < 0) { - goto error_in_scope; - } - } - - co = optimize_and_assemble(c, 1); - compiler_exit_scope(c); - if (is_top_level_await && is_async_generator){ - c->u->u_ste->ste_coroutine = 1; - } - if (co == NULL) { - goto error; - } - - loc = LOC(e); - if (compiler_make_closure(c, loc, co, 0) < 0) { - goto error; - } - Py_CLEAR(co); - - if (compiler_comprehension_iter(c, loc, outermost)) { - goto error; - } - - ADDOP_I(c, loc, CALL, 0); - - if (is_async_generator && type != COMP_GENEXP) { - ADDOP_I(c, loc, GET_AWAITABLE, 0); - ADDOP_LOAD_CONST(c, loc, Py_None); - ADD_YIELD_FROM(c, loc, 1); - } - - return SUCCESS; -error_in_scope: - if (!is_inlined) { - compiler_exit_scope(c); - } -error: - Py_XDECREF(co); - Py_XDECREF(entry); - Py_XDECREF(inline_state.pushed_locals); - Py_XDECREF(inline_state.temp_symbols); - Py_XDECREF(inline_state.fast_hidden); - return ERROR; -} - -static int -compiler_genexp(struct compiler *c, expr_ty e) -{ - assert(e->kind == GeneratorExp_kind); - _Py_DECLARE_STR(anon_genexpr, ""); - return compiler_comprehension(c, e, COMP_GENEXP, &_Py_STR(anon_genexpr), - e->v.GeneratorExp.generators, - e->v.GeneratorExp.elt, NULL); -} - -static int -compiler_listcomp(struct compiler *c, expr_ty e) -{ - assert(e->kind == ListComp_kind); - _Py_DECLARE_STR(anon_listcomp, ""); - return compiler_comprehension(c, e, COMP_LISTCOMP, &_Py_STR(anon_listcomp), - e->v.ListComp.generators, - e->v.ListComp.elt, NULL); -} - -static int -compiler_setcomp(struct compiler *c, expr_ty e) -{ - assert(e->kind == SetComp_kind); - _Py_DECLARE_STR(anon_setcomp, ""); - return compiler_comprehension(c, e, COMP_SETCOMP, &_Py_STR(anon_setcomp), - e->v.SetComp.generators, - e->v.SetComp.elt, NULL); -} - - -static int -compiler_dictcomp(struct compiler *c, expr_ty e) -{ - assert(e->kind == DictComp_kind); - _Py_DECLARE_STR(anon_dictcomp, ""); - return compiler_comprehension(c, e, COMP_DICTCOMP, &_Py_STR(anon_dictcomp), - e->v.DictComp.generators, - e->v.DictComp.key, e->v.DictComp.value); -} - - -static int -compiler_visit_keyword(struct compiler *c, keyword_ty k) -{ - VISIT(c, expr, k->value); - return SUCCESS; -} - - -static int -compiler_with_except_finish(struct compiler *c, jump_target_label cleanup) { - NEW_JUMP_TARGET_LABEL(c, suppress); - ADDOP(c, NO_LOCATION, TO_BOOL); - ADDOP_JUMP(c, NO_LOCATION, POP_JUMP_IF_TRUE, suppress); - ADDOP_I(c, NO_LOCATION, RERAISE, 2); - - USE_LABEL(c, suppress); - ADDOP(c, NO_LOCATION, POP_TOP); /* exc_value */ - ADDOP(c, NO_LOCATION, POP_BLOCK); - ADDOP(c, NO_LOCATION, POP_EXCEPT); - ADDOP(c, NO_LOCATION, POP_TOP); - ADDOP(c, NO_LOCATION, POP_TOP); - NEW_JUMP_TARGET_LABEL(c, exit); - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, exit); - - USE_LABEL(c, cleanup); - POP_EXCEPT_AND_RERAISE(c, NO_LOCATION); - - USE_LABEL(c, exit); - return SUCCESS; -} - -/* - Implements the async with statement. - - The semantics outlined in that PEP are as follows: - - async with EXPR as VAR: - BLOCK - - It is implemented roughly as: - - context = EXPR - exit = context.__aexit__ # not calling it - value = await context.__aenter__() - try: - VAR = value # if VAR present in the syntax - BLOCK - finally: - if an exception was raised: - exc = copy of (exception, instance, traceback) - else: - exc = (None, None, None) - if not (await exit(*exc)): - raise - */ -static int -compiler_async_with(struct compiler *c, stmt_ty s, int pos) -{ - location loc = LOC(s); - withitem_ty item = asdl_seq_GET(s->v.AsyncWith.items, pos); - - assert(s->kind == AsyncWith_kind); - if (IS_TOP_LEVEL_AWAIT(c)){ - c->u->u_ste->ste_coroutine = 1; - } else if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION){ - return compiler_error(c, loc, "'async with' outside async function"); - } - - NEW_JUMP_TARGET_LABEL(c, block); - NEW_JUMP_TARGET_LABEL(c, final); - NEW_JUMP_TARGET_LABEL(c, exit); - NEW_JUMP_TARGET_LABEL(c, cleanup); - - /* Evaluate EXPR */ - VISIT(c, expr, item->context_expr); - - ADDOP(c, loc, BEFORE_ASYNC_WITH); - ADDOP_I(c, loc, GET_AWAITABLE, 1); - ADDOP_LOAD_CONST(c, loc, Py_None); - ADD_YIELD_FROM(c, loc, 1); - - ADDOP_JUMP(c, loc, SETUP_WITH, final); - - /* SETUP_WITH pushes a finally block. */ - USE_LABEL(c, block); - RETURN_IF_ERROR(compiler_push_fblock(c, loc, ASYNC_WITH, block, final, s)); - - if (item->optional_vars) { - VISIT(c, expr, item->optional_vars); - } - else { - /* Discard result from context.__aenter__() */ - ADDOP(c, loc, POP_TOP); - } - - pos++; - if (pos == asdl_seq_LEN(s->v.AsyncWith.items)) { - /* BLOCK code */ - VISIT_SEQ(c, stmt, s->v.AsyncWith.body) - } - else { - RETURN_IF_ERROR(compiler_async_with(c, s, pos)); - } - - compiler_pop_fblock(c, ASYNC_WITH, block); - - ADDOP(c, loc, POP_BLOCK); - /* End of body; start the cleanup */ - - /* For successful outcome: - * call __exit__(None, None, None) - */ - RETURN_IF_ERROR(compiler_call_exit_with_nones(c, loc)); - ADDOP_I(c, loc, GET_AWAITABLE, 2); - ADDOP_LOAD_CONST(c, loc, Py_None); - ADD_YIELD_FROM(c, loc, 1); - - ADDOP(c, loc, POP_TOP); - - ADDOP_JUMP(c, loc, JUMP, exit); - - /* For exceptional outcome: */ - USE_LABEL(c, final); - - ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); - ADDOP(c, loc, PUSH_EXC_INFO); - ADDOP(c, loc, WITH_EXCEPT_START); - ADDOP_I(c, loc, GET_AWAITABLE, 2); - ADDOP_LOAD_CONST(c, loc, Py_None); - ADD_YIELD_FROM(c, loc, 1); - RETURN_IF_ERROR(compiler_with_except_finish(c, cleanup)); - - USE_LABEL(c, exit); - return SUCCESS; -} - - -/* - Implements the with statement from PEP 343. - with EXPR as VAR: - BLOCK - is implemented as: - - SETUP_WITH E - or POP_TOP - - LOAD_CONST (None, None, None) - CALL_FUNCTION_EX 0 - JUMP EXIT - E: WITH_EXCEPT_START (calls EXPR.__exit__) - POP_JUMP_IF_TRUE T: - RERAISE - T: POP_TOP (remove exception from stack) - POP_EXCEPT - POP_TOP - EXIT: - */ - -static int -compiler_with(struct compiler *c, stmt_ty s, int pos) -{ - withitem_ty item = asdl_seq_GET(s->v.With.items, pos); - - assert(s->kind == With_kind); - - NEW_JUMP_TARGET_LABEL(c, block); - NEW_JUMP_TARGET_LABEL(c, final); - NEW_JUMP_TARGET_LABEL(c, exit); - NEW_JUMP_TARGET_LABEL(c, cleanup); - - /* Evaluate EXPR */ - VISIT(c, expr, item->context_expr); - /* Will push bound __exit__ */ - location loc = LOC(s); - ADDOP(c, loc, BEFORE_WITH); - ADDOP_JUMP(c, loc, SETUP_WITH, final); - - /* SETUP_WITH pushes a finally block. */ - USE_LABEL(c, block); - RETURN_IF_ERROR(compiler_push_fblock(c, loc, WITH, block, final, s)); - - if (item->optional_vars) { - VISIT(c, expr, item->optional_vars); - } - else { - /* Discard result from context.__enter__() */ - ADDOP(c, loc, POP_TOP); - } - - pos++; - if (pos == asdl_seq_LEN(s->v.With.items)) { - /* BLOCK code */ - VISIT_SEQ(c, stmt, s->v.With.body) - } - else { - RETURN_IF_ERROR(compiler_with(c, s, pos)); - } - - ADDOP(c, NO_LOCATION, POP_BLOCK); - compiler_pop_fblock(c, WITH, block); - - /* End of body; start the cleanup. */ - - /* For successful outcome: - * call __exit__(None, None, None) - */ - loc = LOC(s); - RETURN_IF_ERROR(compiler_call_exit_with_nones(c, loc)); - ADDOP(c, loc, POP_TOP); - ADDOP_JUMP(c, loc, JUMP, exit); - - /* For exceptional outcome: */ - USE_LABEL(c, final); - - ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); - ADDOP(c, loc, PUSH_EXC_INFO); - ADDOP(c, loc, WITH_EXCEPT_START); - RETURN_IF_ERROR(compiler_with_except_finish(c, cleanup)); - - USE_LABEL(c, exit); - return SUCCESS; -} - -static int -compiler_visit_expr1(struct compiler *c, expr_ty e) -{ - location loc = LOC(e); - switch (e->kind) { - case NamedExpr_kind: - VISIT(c, expr, e->v.NamedExpr.value); - ADDOP_I(c, loc, COPY, 1); - VISIT(c, expr, e->v.NamedExpr.target); - break; - case BoolOp_kind: - return compiler_boolop(c, e); - case BinOp_kind: - VISIT(c, expr, e->v.BinOp.left); - VISIT(c, expr, e->v.BinOp.right); - ADDOP_BINARY(c, loc, e->v.BinOp.op); - break; - case UnaryOp_kind: - VISIT(c, expr, e->v.UnaryOp.operand); - if (e->v.UnaryOp.op == UAdd) { - ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_UNARY_POSITIVE); - } - else if (e->v.UnaryOp.op == Not) { - ADDOP(c, loc, TO_BOOL); - ADDOP(c, loc, UNARY_NOT); - } - else { - ADDOP(c, loc, unaryop(e->v.UnaryOp.op)); - } - break; - case Lambda_kind: - return compiler_lambda(c, e); - case IfExp_kind: - return compiler_ifexp(c, e); - case Dict_kind: - return compiler_dict(c, e); - case Set_kind: - return compiler_set(c, e); - case GeneratorExp_kind: - return compiler_genexp(c, e); - case ListComp_kind: - return compiler_listcomp(c, e); - case SetComp_kind: - return compiler_setcomp(c, e); - case DictComp_kind: - return compiler_dictcomp(c, e); - case Yield_kind: - if (!_PyST_IsFunctionLike(c->u->u_ste)) { - return compiler_error(c, loc, "'yield' outside function"); - } - if (e->v.Yield.value) { - VISIT(c, expr, e->v.Yield.value); - } - else { - ADDOP_LOAD_CONST(c, loc, Py_None); - } - ADDOP_YIELD(c, loc); - break; - case YieldFrom_kind: - if (!_PyST_IsFunctionLike(c->u->u_ste)) { - return compiler_error(c, loc, "'yield' outside function"); - } - if (c->u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION) { - return compiler_error(c, loc, "'yield from' inside async function"); - } - VISIT(c, expr, e->v.YieldFrom.value); - ADDOP(c, loc, GET_YIELD_FROM_ITER); - ADDOP_LOAD_CONST(c, loc, Py_None); - ADD_YIELD_FROM(c, loc, 0); - break; - case Await_kind: - if (!IS_TOP_LEVEL_AWAIT(c)){ - if (!_PyST_IsFunctionLike(c->u->u_ste)) { - return compiler_error(c, loc, "'await' outside function"); - } - - if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION && - c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION) { - return compiler_error(c, loc, "'await' outside async function"); - } - } - - VISIT(c, expr, e->v.Await.value); - ADDOP_I(c, loc, GET_AWAITABLE, 0); - ADDOP_LOAD_CONST(c, loc, Py_None); - ADD_YIELD_FROM(c, loc, 1); - break; - case Compare_kind: - return compiler_compare(c, e); - case Call_kind: - return compiler_call(c, e); - case Constant_kind: - ADDOP_LOAD_CONST(c, loc, e->v.Constant.value); - break; - case JoinedStr_kind: - return compiler_joined_str(c, e); - case FormattedValue_kind: - return compiler_formatted_value(c, e); - /* The following exprs can be assignment targets. */ - case Attribute_kind: - if (e->v.Attribute.ctx == Load && can_optimize_super_call(c, e)) { - RETURN_IF_ERROR(load_args_for_super(c, e->v.Attribute.value)); - int opcode = asdl_seq_LEN(e->v.Attribute.value->v.Call.args) ? - LOAD_SUPER_ATTR : LOAD_ZERO_SUPER_ATTR; - ADDOP_NAME(c, loc, opcode, e->v.Attribute.attr, names); - loc = update_start_location_to_match_attr(c, loc, e); - ADDOP(c, loc, NOP); - return SUCCESS; - } - VISIT(c, expr, e->v.Attribute.value); - loc = LOC(e); - loc = update_start_location_to_match_attr(c, loc, e); - switch (e->v.Attribute.ctx) { - case Load: - ADDOP_NAME(c, loc, LOAD_ATTR, e->v.Attribute.attr, names); - break; - case Store: - if (forbidden_name(c, loc, e->v.Attribute.attr, e->v.Attribute.ctx)) { - return ERROR; - } - ADDOP_NAME(c, loc, STORE_ATTR, e->v.Attribute.attr, names); - break; - case Del: - ADDOP_NAME(c, loc, DELETE_ATTR, e->v.Attribute.attr, names); - break; - } - break; - case Subscript_kind: - return compiler_subscript(c, e); - case Starred_kind: - switch (e->v.Starred.ctx) { - case Store: - /* In all legitimate cases, the Starred node was already replaced - * by compiler_list/compiler_tuple. XXX: is that okay? */ - return compiler_error(c, loc, - "starred assignment target must be in a list or tuple"); - default: - return compiler_error(c, loc, - "can't use starred expression here"); - } - break; - case Slice_kind: - { - int n = compiler_slice(c, e); - RETURN_IF_ERROR(n); - ADDOP_I(c, loc, BUILD_SLICE, n); - break; - } - case Name_kind: - return compiler_nameop(c, loc, e->v.Name.id, e->v.Name.ctx); - /* child nodes of List and Tuple will have expr_context set */ - case List_kind: - return compiler_list(c, e); - case Tuple_kind: - return compiler_tuple(c, e); - } - return SUCCESS; -} - -static int -compiler_visit_expr(struct compiler *c, expr_ty e) -{ - int res = compiler_visit_expr1(c, e); - return res; -} - -static bool -is_two_element_slice(expr_ty s) -{ - return s->kind == Slice_kind && - s->v.Slice.step == NULL; -} - -static int -compiler_augassign(struct compiler *c, stmt_ty s) -{ - assert(s->kind == AugAssign_kind); - expr_ty e = s->v.AugAssign.target; - - location loc = LOC(e); - - switch (e->kind) { - case Attribute_kind: - VISIT(c, expr, e->v.Attribute.value); - ADDOP_I(c, loc, COPY, 1); - loc = update_start_location_to_match_attr(c, loc, e); - ADDOP_NAME(c, loc, LOAD_ATTR, e->v.Attribute.attr, names); - break; - case Subscript_kind: - VISIT(c, expr, e->v.Subscript.value); - if (is_two_element_slice(e->v.Subscript.slice)) { - RETURN_IF_ERROR(compiler_slice(c, e->v.Subscript.slice)); - ADDOP_I(c, loc, COPY, 3); - ADDOP_I(c, loc, COPY, 3); - ADDOP_I(c, loc, COPY, 3); - ADDOP(c, loc, BINARY_SLICE); - } - else { - VISIT(c, expr, e->v.Subscript.slice); - ADDOP_I(c, loc, COPY, 2); - ADDOP_I(c, loc, COPY, 2); - ADDOP(c, loc, BINARY_SUBSCR); - } - break; - case Name_kind: - RETURN_IF_ERROR(compiler_nameop(c, loc, e->v.Name.id, Load)); - break; - default: - PyErr_Format(PyExc_SystemError, - "invalid node type (%d) for augmented assignment", - e->kind); - return ERROR; - } - - loc = LOC(s); - - VISIT(c, expr, s->v.AugAssign.value); - ADDOP_INPLACE(c, loc, s->v.AugAssign.op); - - loc = LOC(e); - - switch (e->kind) { - case Attribute_kind: - loc = update_start_location_to_match_attr(c, loc, e); - ADDOP_I(c, loc, SWAP, 2); - ADDOP_NAME(c, loc, STORE_ATTR, e->v.Attribute.attr, names); - break; - case Subscript_kind: - if (is_two_element_slice(e->v.Subscript.slice)) { - ADDOP_I(c, loc, SWAP, 4); - ADDOP_I(c, loc, SWAP, 3); - ADDOP_I(c, loc, SWAP, 2); - ADDOP(c, loc, STORE_SLICE); - } - else { - ADDOP_I(c, loc, SWAP, 3); - ADDOP_I(c, loc, SWAP, 2); - ADDOP(c, loc, STORE_SUBSCR); - } - break; - case Name_kind: - return compiler_nameop(c, loc, e->v.Name.id, Store); - default: - Py_UNREACHABLE(); - } - return SUCCESS; -} - -static int -check_ann_expr(struct compiler *c, expr_ty e) -{ - VISIT(c, expr, e); - ADDOP(c, LOC(e), POP_TOP); - return SUCCESS; -} - -static int -check_annotation(struct compiler *c, stmt_ty s) -{ - /* Annotations of complex targets does not produce anything - under annotations future */ - if (c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) { - return SUCCESS; - } - - /* Annotations are only evaluated in a module or class. */ - if (c->u->u_scope_type == COMPILER_SCOPE_MODULE || - c->u->u_scope_type == COMPILER_SCOPE_CLASS) { - return check_ann_expr(c, s->v.AnnAssign.annotation); - } - return SUCCESS; -} - -static int -check_ann_subscr(struct compiler *c, expr_ty e) -{ - /* We check that everything in a subscript is defined at runtime. */ - switch (e->kind) { - case Slice_kind: - if (e->v.Slice.lower && check_ann_expr(c, e->v.Slice.lower) < 0) { - return ERROR; - } - if (e->v.Slice.upper && check_ann_expr(c, e->v.Slice.upper) < 0) { - return ERROR; - } - if (e->v.Slice.step && check_ann_expr(c, e->v.Slice.step) < 0) { - return ERROR; - } - return SUCCESS; - case Tuple_kind: { - /* extended slice */ - asdl_expr_seq *elts = e->v.Tuple.elts; - Py_ssize_t i, n = asdl_seq_LEN(elts); - for (i = 0; i < n; i++) { - RETURN_IF_ERROR(check_ann_subscr(c, asdl_seq_GET(elts, i))); - } - return SUCCESS; - } - default: - return check_ann_expr(c, e); - } -} - -static int -compiler_annassign(struct compiler *c, stmt_ty s) -{ - location loc = LOC(s); - expr_ty targ = s->v.AnnAssign.target; - PyObject* mangled; - - assert(s->kind == AnnAssign_kind); - - /* We perform the actual assignment first. */ - if (s->v.AnnAssign.value) { - VISIT(c, expr, s->v.AnnAssign.value); - VISIT(c, expr, targ); - } - switch (targ->kind) { - case Name_kind: - if (forbidden_name(c, loc, targ->v.Name.id, Store)) { - return ERROR; - } - /* If we have a simple name in a module or class, store annotation. */ - if (s->v.AnnAssign.simple && - (c->u->u_scope_type == COMPILER_SCOPE_MODULE || - c->u->u_scope_type == COMPILER_SCOPE_CLASS)) { - if (c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) { - VISIT(c, annexpr, s->v.AnnAssign.annotation) - } - else { - VISIT(c, expr, s->v.AnnAssign.annotation); - } - ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names); - mangled = _Py_Mangle(c->u->u_private, targ->v.Name.id); - ADDOP_LOAD_CONST_NEW(c, loc, mangled); - ADDOP(c, loc, STORE_SUBSCR); - } - break; - case Attribute_kind: - if (forbidden_name(c, loc, targ->v.Attribute.attr, Store)) { - return ERROR; - } - if (!s->v.AnnAssign.value && - check_ann_expr(c, targ->v.Attribute.value) < 0) { - return ERROR; - } - break; - case Subscript_kind: - if (!s->v.AnnAssign.value && - (check_ann_expr(c, targ->v.Subscript.value) < 0 || - check_ann_subscr(c, targ->v.Subscript.slice) < 0)) { - return ERROR; - } - break; - default: - PyErr_Format(PyExc_SystemError, - "invalid node type (%d) for annotated assignment", - targ->kind); - return ERROR; - } - /* Annotation is evaluated last. */ - if (!s->v.AnnAssign.simple && check_annotation(c, s) < 0) { - return ERROR; - } - return SUCCESS; -} - -/* Raises a SyntaxError and returns 0. - If something goes wrong, a different exception may be raised. -*/ - -static int -compiler_error(struct compiler *c, location loc, - const char *format, ...) +/* Raises a SyntaxError and returns ERROR. + * If something goes wrong, a different exception may be raised. + */ +int +_PyCompile_Error(compiler *c, location loc, const char *format, ...) { va_list vargs; va_start(vargs, format); @@ -6519,956 +1124,126 @@ compiler_error(struct compiler *c, location loc, if (msg == NULL) { return ERROR; } - PyObject *loc_obj = PyErr_ProgramTextObject(c->c_filename, loc.lineno); - if (loc_obj == NULL) { - loc_obj = Py_None; - } - PyObject *args = Py_BuildValue("O(OiiOii)", msg, c->c_filename, - loc.lineno, loc.col_offset + 1, loc_obj, - loc.end_lineno, loc.end_col_offset + 1); + _PyErr_RaiseSyntaxError(msg, c->c_filename, loc.lineno, loc.col_offset + 1, + loc.end_lineno, loc.end_col_offset + 1); Py_DECREF(msg); - if (args == NULL) { - goto exit; - } - PyErr_SetObject(PyExc_SyntaxError, args); - exit: - Py_DECREF(loc_obj); - Py_XDECREF(args); return ERROR; } -/* Emits a SyntaxWarning and returns 1 on success. +/* Emits a SyntaxWarning and returns 0 on success. If a SyntaxWarning raised as error, replaces it with a SyntaxError - and returns 0. + and returns -1. */ -static int -compiler_warn(struct compiler *c, location loc, - const char *format, ...) +int +_PyCompile_Warn(compiler *c, location loc, const char *format, ...) { va_list vargs; va_start(vargs, format); PyObject *msg = PyUnicode_FromFormatV(format, vargs); va_end(vargs); if (msg == NULL) { - return ERROR; - } - if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, c->c_filename, - loc.lineno, NULL, NULL) < 0) - { - if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { - /* Replace the SyntaxWarning exception with a SyntaxError - to get a more accurate error report */ - PyErr_Clear(); - assert(PyUnicode_AsUTF8(msg) != NULL); - compiler_error(c, loc, PyUnicode_AsUTF8(msg)); - } - Py_DECREF(msg); - return ERROR; - } - Py_DECREF(msg); - return SUCCESS; -} - -static int -compiler_subscript(struct compiler *c, expr_ty e) -{ - location loc = LOC(e); - expr_context_ty ctx = e->v.Subscript.ctx; - int op = 0; - - if (ctx == Load) { - RETURN_IF_ERROR(check_subscripter(c, e->v.Subscript.value)); - RETURN_IF_ERROR(check_index(c, e->v.Subscript.value, e->v.Subscript.slice)); - } - - VISIT(c, expr, e->v.Subscript.value); - if (is_two_element_slice(e->v.Subscript.slice) && ctx != Del) { - RETURN_IF_ERROR(compiler_slice(c, e->v.Subscript.slice)); - if (ctx == Load) { - ADDOP(c, loc, BINARY_SLICE); - } - else { - assert(ctx == Store); - ADDOP(c, loc, STORE_SLICE); - } - } - else { - VISIT(c, expr, e->v.Subscript.slice); - switch (ctx) { - case Load: op = BINARY_SUBSCR; break; - case Store: op = STORE_SUBSCR; break; - case Del: op = DELETE_SUBSCR; break; - } - assert(op); - ADDOP(c, loc, op); - } - return SUCCESS; -} - -/* Returns the number of the values emitted, - * thus are needed to build the slice, or -1 if there is an error. */ -static int -compiler_slice(struct compiler *c, expr_ty s) -{ - int n = 2; - assert(s->kind == Slice_kind); - - /* only handles the cases where BUILD_SLICE is emitted */ - if (s->v.Slice.lower) { - VISIT(c, expr, s->v.Slice.lower); - } - else { - ADDOP_LOAD_CONST(c, LOC(s), Py_None); - } - - if (s->v.Slice.upper) { - VISIT(c, expr, s->v.Slice.upper); - } - else { - ADDOP_LOAD_CONST(c, LOC(s), Py_None); - } - - if (s->v.Slice.step) { - n++; - VISIT(c, expr, s->v.Slice.step); - } - return n; -} - - -// PEP 634: Structural Pattern Matching - -// To keep things simple, all compiler_pattern_* and pattern_helper_* routines -// follow the convention of consuming TOS (the subject for the given pattern) -// and calling jump_to_fail_pop on failure (no match). - -// When calling into these routines, it's important that pc->on_top be kept -// updated to reflect the current number of items that we are using on the top -// of the stack: they will be popped on failure, and any name captures will be -// stored *underneath* them on success. This lets us defer all names stores -// until the *entire* pattern matches. - -#define WILDCARD_CHECK(N) \ - ((N)->kind == MatchAs_kind && !(N)->v.MatchAs.name) - -#define WILDCARD_STAR_CHECK(N) \ - ((N)->kind == MatchStar_kind && !(N)->v.MatchStar.name) - -// Limit permitted subexpressions, even if the parser & AST validator let them through -#define MATCH_VALUE_EXPR(N) \ - ((N)->kind == Constant_kind || (N)->kind == Attribute_kind) - -// Allocate or resize pc->fail_pop to allow for n items to be popped on failure. -static int -ensure_fail_pop(struct compiler *c, pattern_context *pc, Py_ssize_t n) -{ - Py_ssize_t size = n + 1; - if (size <= pc->fail_pop_size) { - return SUCCESS; - } - Py_ssize_t needed = sizeof(jump_target_label) * size; - jump_target_label *resized = PyMem_Realloc(pc->fail_pop, needed); - if (resized == NULL) { - PyErr_NoMemory(); - return ERROR; - } - pc->fail_pop = resized; - while (pc->fail_pop_size < size) { - NEW_JUMP_TARGET_LABEL(c, new_block); - pc->fail_pop[pc->fail_pop_size++] = new_block; - } - return SUCCESS; -} - -// Use op to jump to the correct fail_pop block. -static int -jump_to_fail_pop(struct compiler *c, location loc, - pattern_context *pc, int op) -{ - // Pop any items on the top of the stack, plus any objects we were going to - // capture on success: - Py_ssize_t pops = pc->on_top + PyList_GET_SIZE(pc->stores); - RETURN_IF_ERROR(ensure_fail_pop(c, pc, pops)); - ADDOP_JUMP(c, loc, op, pc->fail_pop[pops]); - return SUCCESS; -} - -// Build all of the fail_pop blocks and reset fail_pop. -static int -emit_and_reset_fail_pop(struct compiler *c, location loc, - pattern_context *pc) -{ - if (!pc->fail_pop_size) { - assert(pc->fail_pop == NULL); - return SUCCESS; - } - while (--pc->fail_pop_size) { - USE_LABEL(c, pc->fail_pop[pc->fail_pop_size]); - if (codegen_addop_noarg(INSTR_SEQUENCE(c), POP_TOP, loc) < 0) { - pc->fail_pop_size = 0; - PyMem_Free(pc->fail_pop); - pc->fail_pop = NULL; - return ERROR; - } - } - USE_LABEL(c, pc->fail_pop[0]); - PyMem_Free(pc->fail_pop); - pc->fail_pop = NULL; - return SUCCESS; -} - -static int -compiler_error_duplicate_store(struct compiler *c, location loc, identifier n) -{ - return compiler_error(c, loc, - "multiple assignments to name %R in pattern", n); -} - -// Duplicate the effect of 3.10's ROT_* instructions using SWAPs. -static int -pattern_helper_rotate(struct compiler *c, location loc, Py_ssize_t count) -{ - while (1 < count) { - ADDOP_I(c, loc, SWAP, count--); - } - return SUCCESS; -} - -static int -pattern_helper_store_name(struct compiler *c, location loc, - identifier n, pattern_context *pc) -{ - if (n == NULL) { - ADDOP(c, loc, POP_TOP); - return SUCCESS; - } - if (forbidden_name(c, loc, n, Store)) { - return ERROR; - } - // Can't assign to the same name twice: - int duplicate = PySequence_Contains(pc->stores, n); - RETURN_IF_ERROR(duplicate); - if (duplicate) { - return compiler_error_duplicate_store(c, loc, n); - } - // Rotate this object underneath any items we need to preserve: - Py_ssize_t rotations = pc->on_top + PyList_GET_SIZE(pc->stores) + 1; - RETURN_IF_ERROR(pattern_helper_rotate(c, loc, rotations)); - RETURN_IF_ERROR(PyList_Append(pc->stores, n)); - return SUCCESS; -} - - -static int -pattern_unpack_helper(struct compiler *c, location loc, - asdl_pattern_seq *elts) -{ - Py_ssize_t n = asdl_seq_LEN(elts); - int seen_star = 0; - for (Py_ssize_t i = 0; i < n; i++) { - pattern_ty elt = asdl_seq_GET(elts, i); - if (elt->kind == MatchStar_kind && !seen_star) { - if ((i >= (1 << 8)) || - (n-i-1 >= (INT_MAX >> 8))) { - return compiler_error(c, loc, - "too many expressions in " - "star-unpacking sequence pattern"); - } - ADDOP_I(c, loc, UNPACK_EX, (i + ((n-i-1) << 8))); - seen_star = 1; - } - else if (elt->kind == MatchStar_kind) { - return compiler_error(c, loc, - "multiple starred expressions in sequence pattern"); - } - } - if (!seen_star) { - ADDOP_I(c, loc, UNPACK_SEQUENCE, n); - } - return SUCCESS; -} - -static int -pattern_helper_sequence_unpack(struct compiler *c, location loc, - asdl_pattern_seq *patterns, Py_ssize_t star, - pattern_context *pc) -{ - RETURN_IF_ERROR(pattern_unpack_helper(c, loc, patterns)); - Py_ssize_t size = asdl_seq_LEN(patterns); - // We've now got a bunch of new subjects on the stack. They need to remain - // there after each subpattern match: - pc->on_top += size; - for (Py_ssize_t i = 0; i < size; i++) { - // One less item to keep track of each time we loop through: - pc->on_top--; - pattern_ty pattern = asdl_seq_GET(patterns, i); - RETURN_IF_ERROR(compiler_pattern_subpattern(c, pattern, pc)); - } - return SUCCESS; -} - -// Like pattern_helper_sequence_unpack, but uses BINARY_SUBSCR instead of -// UNPACK_SEQUENCE / UNPACK_EX. This is more efficient for patterns with a -// starred wildcard like [first, *_] / [first, *_, last] / [*_, last] / etc. -static int -pattern_helper_sequence_subscr(struct compiler *c, location loc, - asdl_pattern_seq *patterns, Py_ssize_t star, - pattern_context *pc) -{ - // We need to keep the subject around for extracting elements: - pc->on_top++; - Py_ssize_t size = asdl_seq_LEN(patterns); - for (Py_ssize_t i = 0; i < size; i++) { - pattern_ty pattern = asdl_seq_GET(patterns, i); - if (WILDCARD_CHECK(pattern)) { - continue; - } - if (i == star) { - assert(WILDCARD_STAR_CHECK(pattern)); - continue; - } - ADDOP_I(c, loc, COPY, 1); - if (i < star) { - ADDOP_LOAD_CONST_NEW(c, loc, PyLong_FromSsize_t(i)); - } - else { - // The subject may not support negative indexing! Compute a - // nonnegative index: - ADDOP(c, loc, GET_LEN); - ADDOP_LOAD_CONST_NEW(c, loc, PyLong_FromSsize_t(size - i)); - ADDOP_BINARY(c, loc, Sub); - } - ADDOP(c, loc, BINARY_SUBSCR); - RETURN_IF_ERROR(compiler_pattern_subpattern(c, pattern, pc)); - } - // Pop the subject, we're done with it: - pc->on_top--; - ADDOP(c, loc, POP_TOP); - return SUCCESS; -} - -// Like compiler_pattern, but turn off checks for irrefutability. -static int -compiler_pattern_subpattern(struct compiler *c, - pattern_ty p, pattern_context *pc) -{ - int allow_irrefutable = pc->allow_irrefutable; - pc->allow_irrefutable = 1; - RETURN_IF_ERROR(compiler_pattern(c, p, pc)); - pc->allow_irrefutable = allow_irrefutable; - return SUCCESS; -} - -static int -compiler_pattern_as(struct compiler *c, pattern_ty p, pattern_context *pc) -{ - assert(p->kind == MatchAs_kind); - if (p->v.MatchAs.pattern == NULL) { - // An irrefutable match: - if (!pc->allow_irrefutable) { - if (p->v.MatchAs.name) { - const char *e = "name capture %R makes remaining patterns unreachable"; - return compiler_error(c, LOC(p), e, p->v.MatchAs.name); - } - const char *e = "wildcard makes remaining patterns unreachable"; - return compiler_error(c, LOC(p), e); - } - return pattern_helper_store_name(c, LOC(p), p->v.MatchAs.name, pc); - } - // Need to make a copy for (possibly) storing later: - pc->on_top++; - ADDOP_I(c, LOC(p), COPY, 1); - RETURN_IF_ERROR(compiler_pattern(c, p->v.MatchAs.pattern, pc)); - // Success! Store it: - pc->on_top--; - RETURN_IF_ERROR(pattern_helper_store_name(c, LOC(p), p->v.MatchAs.name, pc)); - return SUCCESS; -} - -static int -compiler_pattern_star(struct compiler *c, pattern_ty p, pattern_context *pc) -{ - assert(p->kind == MatchStar_kind); - RETURN_IF_ERROR( - pattern_helper_store_name(c, LOC(p), p->v.MatchStar.name, pc)); - return SUCCESS; -} - -static int -validate_kwd_attrs(struct compiler *c, asdl_identifier_seq *attrs, asdl_pattern_seq* patterns) -{ - // Any errors will point to the pattern rather than the arg name as the - // parser is only supplying identifiers rather than Name or keyword nodes - Py_ssize_t nattrs = asdl_seq_LEN(attrs); - for (Py_ssize_t i = 0; i < nattrs; i++) { - identifier attr = ((identifier)asdl_seq_GET(attrs, i)); - location loc = LOC((pattern_ty) asdl_seq_GET(patterns, i)); - if (forbidden_name(c, loc, attr, Store)) { - return ERROR; - } - for (Py_ssize_t j = i + 1; j < nattrs; j++) { - identifier other = ((identifier)asdl_seq_GET(attrs, j)); - if (!PyUnicode_Compare(attr, other)) { - location loc = LOC((pattern_ty) asdl_seq_GET(patterns, j)); - compiler_error(c, loc, "attribute name repeated in class pattern: %U", attr); - return ERROR; - } - } - } - return SUCCESS; -} - -static int -compiler_pattern_class(struct compiler *c, pattern_ty p, pattern_context *pc) -{ - assert(p->kind == MatchClass_kind); - asdl_pattern_seq *patterns = p->v.MatchClass.patterns; - asdl_identifier_seq *kwd_attrs = p->v.MatchClass.kwd_attrs; - asdl_pattern_seq *kwd_patterns = p->v.MatchClass.kwd_patterns; - Py_ssize_t nargs = asdl_seq_LEN(patterns); - Py_ssize_t nattrs = asdl_seq_LEN(kwd_attrs); - Py_ssize_t nkwd_patterns = asdl_seq_LEN(kwd_patterns); - if (nattrs != nkwd_patterns) { - // AST validator shouldn't let this happen, but if it does, - // just fail, don't crash out of the interpreter - const char * e = "kwd_attrs (%d) / kwd_patterns (%d) length mismatch in class pattern"; - return compiler_error(c, LOC(p), e, nattrs, nkwd_patterns); - } - if (INT_MAX < nargs || INT_MAX < nargs + nattrs - 1) { - const char *e = "too many sub-patterns in class pattern %R"; - return compiler_error(c, LOC(p), e, p->v.MatchClass.cls); - } - if (nattrs) { - RETURN_IF_ERROR(validate_kwd_attrs(c, kwd_attrs, kwd_patterns)); - } - VISIT(c, expr, p->v.MatchClass.cls); - PyObject *attr_names = PyTuple_New(nattrs); - if (attr_names == NULL) { - return ERROR; - } - Py_ssize_t i; - for (i = 0; i < nattrs; i++) { - PyObject *name = asdl_seq_GET(kwd_attrs, i); - PyTuple_SET_ITEM(attr_names, i, Py_NewRef(name)); - } - ADDOP_LOAD_CONST_NEW(c, LOC(p), attr_names); - ADDOP_I(c, LOC(p), MATCH_CLASS, nargs); - ADDOP_I(c, LOC(p), COPY, 1); - ADDOP_LOAD_CONST(c, LOC(p), Py_None); - ADDOP_I(c, LOC(p), IS_OP, 1); - // TOS is now a tuple of (nargs + nattrs) attributes (or None): - pc->on_top++; - RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); - ADDOP_I(c, LOC(p), UNPACK_SEQUENCE, nargs + nattrs); - pc->on_top += nargs + nattrs - 1; - for (i = 0; i < nargs + nattrs; i++) { - pc->on_top--; - pattern_ty pattern; - if (i < nargs) { - // Positional: - pattern = asdl_seq_GET(patterns, i); - } - else { - // Keyword: - pattern = asdl_seq_GET(kwd_patterns, i - nargs); - } - if (WILDCARD_CHECK(pattern)) { - ADDOP(c, LOC(p), POP_TOP); - continue; - } - RETURN_IF_ERROR(compiler_pattern_subpattern(c, pattern, pc)); - } - // Success! Pop the tuple of attributes: - return SUCCESS; -} - -static int -compiler_pattern_mapping(struct compiler *c, pattern_ty p, - pattern_context *pc) -{ - assert(p->kind == MatchMapping_kind); - asdl_expr_seq *keys = p->v.MatchMapping.keys; - asdl_pattern_seq *patterns = p->v.MatchMapping.patterns; - Py_ssize_t size = asdl_seq_LEN(keys); - Py_ssize_t npatterns = asdl_seq_LEN(patterns); - if (size != npatterns) { - // AST validator shouldn't let this happen, but if it does, - // just fail, don't crash out of the interpreter - const char * e = "keys (%d) / patterns (%d) length mismatch in mapping pattern"; - return compiler_error(c, LOC(p), e, size, npatterns); - } - // We have a double-star target if "rest" is set - PyObject *star_target = p->v.MatchMapping.rest; - // We need to keep the subject on top during the mapping and length checks: - pc->on_top++; - ADDOP(c, LOC(p), MATCH_MAPPING); - RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); - if (!size && !star_target) { - // If the pattern is just "{}", we're done! Pop the subject: - pc->on_top--; - ADDOP(c, LOC(p), POP_TOP); - return SUCCESS; - } - if (size) { - // If the pattern has any keys in it, perform a length check: - ADDOP(c, LOC(p), GET_LEN); - ADDOP_LOAD_CONST_NEW(c, LOC(p), PyLong_FromSsize_t(size)); - ADDOP_COMPARE(c, LOC(p), GtE); - RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); - } - if (INT_MAX < size - 1) { - return compiler_error(c, LOC(p), "too many sub-patterns in mapping pattern"); - } - // Collect all of the keys into a tuple for MATCH_KEYS and - // **rest. They can either be dotted names or literals: - - // Maintaining a set of Constant_kind kind keys allows us to raise a - // SyntaxError in the case of duplicates. - PyObject *seen = PySet_New(NULL); - if (seen == NULL) { - return ERROR; - } - - // NOTE: goto error on failure in the loop below to avoid leaking `seen` - for (Py_ssize_t i = 0; i < size; i++) { - expr_ty key = asdl_seq_GET(keys, i); - if (key == NULL) { - const char *e = "can't use NULL keys in MatchMapping " - "(set 'rest' parameter instead)"; - location loc = LOC((pattern_ty) asdl_seq_GET(patterns, i)); - compiler_error(c, loc, e); - goto error; - } - - if (key->kind == Constant_kind) { - int in_seen = PySet_Contains(seen, key->v.Constant.value); - if (in_seen < 0) { - goto error; - } - if (in_seen) { - const char *e = "mapping pattern checks duplicate key (%R)"; - compiler_error(c, LOC(p), e, key->v.Constant.value); - goto error; - } - if (PySet_Add(seen, key->v.Constant.value)) { - goto error; - } - } - - else if (key->kind != Attribute_kind) { - const char *e = "mapping pattern keys may only match literals and attribute lookups"; - compiler_error(c, LOC(p), e); - goto error; - } - if (compiler_visit_expr(c, key) < 0) { - goto error; - } - } - - // all keys have been checked; there are no duplicates - Py_DECREF(seen); - - ADDOP_I(c, LOC(p), BUILD_TUPLE, size); - ADDOP(c, LOC(p), MATCH_KEYS); - // There's now a tuple of keys and a tuple of values on top of the subject: - pc->on_top += 2; - ADDOP_I(c, LOC(p), COPY, 1); - ADDOP_LOAD_CONST(c, LOC(p), Py_None); - ADDOP_I(c, LOC(p), IS_OP, 1); - RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); - // So far so good. Use that tuple of values on the stack to match - // sub-patterns against: - ADDOP_I(c, LOC(p), UNPACK_SEQUENCE, size); - pc->on_top += size - 1; - for (Py_ssize_t i = 0; i < size; i++) { - pc->on_top--; - pattern_ty pattern = asdl_seq_GET(patterns, i); - RETURN_IF_ERROR(compiler_pattern_subpattern(c, pattern, pc)); - } - // If we get this far, it's a match! Whatever happens next should consume - // the tuple of keys and the subject: - pc->on_top -= 2; - if (star_target) { - // If we have a starred name, bind a dict of remaining items to it (this may - // seem a bit inefficient, but keys is rarely big enough to actually impact - // runtime): - // rest = dict(TOS1) - // for key in TOS: - // del rest[key] - ADDOP_I(c, LOC(p), BUILD_MAP, 0); // [subject, keys, empty] - ADDOP_I(c, LOC(p), SWAP, 3); // [empty, keys, subject] - ADDOP_I(c, LOC(p), DICT_UPDATE, 2); // [copy, keys] - ADDOP_I(c, LOC(p), UNPACK_SEQUENCE, size); // [copy, keys...] - while (size) { - ADDOP_I(c, LOC(p), COPY, 1 + size--); // [copy, keys..., copy] - ADDOP_I(c, LOC(p), SWAP, 2); // [copy, keys..., copy, key] - ADDOP(c, LOC(p), DELETE_SUBSCR); // [copy, keys...] - } - RETURN_IF_ERROR(pattern_helper_store_name(c, LOC(p), star_target, pc)); - } - else { - ADDOP(c, LOC(p), POP_TOP); // Tuple of keys. - ADDOP(c, LOC(p), POP_TOP); // Subject. + return ERROR; } - return SUCCESS; + int ret = _PyErr_EmitSyntaxWarning(msg, c->c_filename, loc.lineno, loc.col_offset + 1, + loc.end_lineno, loc.end_col_offset + 1); + Py_DECREF(msg); + return ret; +} -error: - Py_DECREF(seen); - return ERROR; +PyObject * +_PyCompile_Mangle(compiler *c, PyObject *name) +{ + return _Py_Mangle(c->u->u_private, name); } -static int -compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) +PyObject * +_PyCompile_MaybeMangle(compiler *c, PyObject *name) { - assert(p->kind == MatchOr_kind); - NEW_JUMP_TARGET_LABEL(c, end); - Py_ssize_t size = asdl_seq_LEN(p->v.MatchOr.patterns); - assert(size > 1); - // We're going to be messing with pc. Keep the original info handy: - pattern_context old_pc = *pc; - Py_INCREF(pc->stores); - // control is the list of names bound by the first alternative. It is used - // for checking different name bindings in alternatives, and for correcting - // the order in which extracted elements are placed on the stack. - PyObject *control = NULL; - // NOTE: We can't use returning macros anymore! goto error on error. - for (Py_ssize_t i = 0; i < size; i++) { - pattern_ty alt = asdl_seq_GET(p->v.MatchOr.patterns, i); - PyObject *pc_stores = PyList_New(0); - if (pc_stores == NULL) { - goto error; - } - Py_SETREF(pc->stores, pc_stores); - // An irrefutable sub-pattern must be last, if it is allowed at all: - pc->allow_irrefutable = (i == size - 1) && old_pc.allow_irrefutable; - pc->fail_pop = NULL; - pc->fail_pop_size = 0; - pc->on_top = 0; - if (codegen_addop_i(INSTR_SEQUENCE(c), COPY, 1, LOC(alt)) < 0 || - compiler_pattern(c, alt, pc) < 0) { - goto error; - } - // Success! - Py_ssize_t nstores = PyList_GET_SIZE(pc->stores); - if (!i) { - // This is the first alternative, so save its stores as a "control" - // for the others (they can't bind a different set of names, and - // might need to be reordered): - assert(control == NULL); - control = Py_NewRef(pc->stores); - } - else if (nstores != PyList_GET_SIZE(control)) { - goto diff; - } - else if (nstores) { - // There were captures. Check to see if we differ from control: - Py_ssize_t icontrol = nstores; - while (icontrol--) { - PyObject *name = PyList_GET_ITEM(control, icontrol); - Py_ssize_t istores = PySequence_Index(pc->stores, name); - if (istores < 0) { - PyErr_Clear(); - goto diff; - } - if (icontrol != istores) { - // Reorder the names on the stack to match the order of the - // names in control. There's probably a better way of doing - // this; the current solution is potentially very - // inefficient when each alternative subpattern binds lots - // of names in different orders. It's fine for reasonable - // cases, though, and the peephole optimizer will ensure - // that the final code is as efficient as possible. - assert(istores < icontrol); - Py_ssize_t rotations = istores + 1; - // Perform the same rotation on pc->stores: - PyObject *rotated = PyList_GetSlice(pc->stores, 0, - rotations); - if (rotated == NULL || - PyList_SetSlice(pc->stores, 0, rotations, NULL) || - PyList_SetSlice(pc->stores, icontrol - istores, - icontrol - istores, rotated)) - { - Py_XDECREF(rotated); - goto error; - } - Py_DECREF(rotated); - // That just did: - // rotated = pc_stores[:rotations] - // del pc_stores[:rotations] - // pc_stores[icontrol-istores:icontrol-istores] = rotated - // Do the same thing to the stack, using several - // rotations: - while (rotations--) { - if (pattern_helper_rotate(c, LOC(alt), icontrol + 1) < 0) { - goto error; - } - } - } - } - } - assert(control); - if (codegen_addop_j(INSTR_SEQUENCE(c), LOC(alt), JUMP, end) < 0 || - emit_and_reset_fail_pop(c, LOC(alt), pc) < 0) - { - goto error; - } - } - Py_DECREF(pc->stores); - *pc = old_pc; - Py_INCREF(pc->stores); - // Need to NULL this for the PyMem_Free call in the error block. - old_pc.fail_pop = NULL; - // No match. Pop the remaining copy of the subject and fail: - if (codegen_addop_noarg(INSTR_SEQUENCE(c), POP_TOP, LOC(p)) < 0 || - jump_to_fail_pop(c, LOC(p), pc, JUMP) < 0) { - goto error; - } + return _Py_MaybeMangle(c->u->u_private, c->u->u_ste, name); +} - USE_LABEL(c, end); - Py_ssize_t nstores = PyList_GET_SIZE(control); - // There's a bunch of stuff on the stack between where the new stores - // are and where they need to be: - // - The other stores. - // - A copy of the subject. - // - Anything else that may be on top of the stack. - // - Any previous stores we've already stashed away on the stack. - Py_ssize_t nrots = nstores + 1 + pc->on_top + PyList_GET_SIZE(pc->stores); - for (Py_ssize_t i = 0; i < nstores; i++) { - // Rotate this capture to its proper place on the stack: - if (pattern_helper_rotate(c, LOC(p), nrots) < 0) { - goto error; - } - // Update the list of previous stores with this new name, checking for - // duplicates: - PyObject *name = PyList_GET_ITEM(control, i); - int dupe = PySequence_Contains(pc->stores, name); - if (dupe < 0) { - goto error; - } - if (dupe) { - compiler_error_duplicate_store(c, LOC(p), name); - goto error; - } - if (PyList_Append(pc->stores, name)) { - goto error; - } - } - Py_DECREF(old_pc.stores); - Py_DECREF(control); - // NOTE: Returning macros are safe again. - // Pop the copy of the subject: - ADDOP(c, LOC(p), POP_TOP); - return SUCCESS; -diff: - compiler_error(c, LOC(p), "alternative patterns bind different names"); -error: - PyMem_Free(old_pc.fail_pop); - Py_DECREF(old_pc.stores); - Py_XDECREF(control); - return ERROR; +instr_sequence * +_PyCompile_InstrSequence(compiler *c) +{ + return c->u->u_instr_sequence; } +int +_PyCompile_FutureFeatures(compiler *c) +{ + return c->c_future.ff_features; +} -static int -compiler_pattern_sequence(struct compiler *c, pattern_ty p, - pattern_context *pc) +struct symtable * +_PyCompile_Symtable(compiler *c) { - assert(p->kind == MatchSequence_kind); - asdl_pattern_seq *patterns = p->v.MatchSequence.patterns; - Py_ssize_t size = asdl_seq_LEN(patterns); - Py_ssize_t star = -1; - int only_wildcard = 1; - int star_wildcard = 0; - // Find a starred name, if it exists. There may be at most one: - for (Py_ssize_t i = 0; i < size; i++) { - pattern_ty pattern = asdl_seq_GET(patterns, i); - if (pattern->kind == MatchStar_kind) { - if (star >= 0) { - const char *e = "multiple starred names in sequence pattern"; - return compiler_error(c, LOC(p), e); - } - star_wildcard = WILDCARD_STAR_CHECK(pattern); - only_wildcard &= star_wildcard; - star = i; - continue; - } - only_wildcard &= WILDCARD_CHECK(pattern); - } - // We need to keep the subject on top during the sequence and length checks: - pc->on_top++; - ADDOP(c, LOC(p), MATCH_SEQUENCE); - RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); - if (star < 0) { - // No star: len(subject) == size - ADDOP(c, LOC(p), GET_LEN); - ADDOP_LOAD_CONST_NEW(c, LOC(p), PyLong_FromSsize_t(size)); - ADDOP_COMPARE(c, LOC(p), Eq); - RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); - } - else if (size > 1) { - // Star: len(subject) >= size - 1 - ADDOP(c, LOC(p), GET_LEN); - ADDOP_LOAD_CONST_NEW(c, LOC(p), PyLong_FromSsize_t(size - 1)); - ADDOP_COMPARE(c, LOC(p), GtE); - RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); - } - // Whatever comes next should consume the subject: - pc->on_top--; - if (only_wildcard) { - // Patterns like: [] / [_] / [_, _] / [*_] / [_, *_] / [_, _, *_] / etc. - ADDOP(c, LOC(p), POP_TOP); - } - else if (star_wildcard) { - RETURN_IF_ERROR(pattern_helper_sequence_subscr(c, LOC(p), patterns, star, pc)); - } - else { - RETURN_IF_ERROR(pattern_helper_sequence_unpack(c, LOC(p), patterns, star, pc)); - } - return SUCCESS; + return c->c_st; } -static int -compiler_pattern_value(struct compiler *c, pattern_ty p, pattern_context *pc) +PySTEntryObject * +_PyCompile_SymtableEntry(compiler *c) { - assert(p->kind == MatchValue_kind); - expr_ty value = p->v.MatchValue.value; - if (!MATCH_VALUE_EXPR(value)) { - const char *e = "patterns may only match literals and attribute lookups"; - return compiler_error(c, LOC(p), e); - } - VISIT(c, expr, value); - ADDOP_COMPARE(c, LOC(p), Eq); - ADDOP(c, LOC(p), TO_BOOL); - RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); - return SUCCESS; + return c->u->u_ste; } -static int -compiler_pattern_singleton(struct compiler *c, pattern_ty p, pattern_context *pc) +int +_PyCompile_OptimizationLevel(compiler *c) { - assert(p->kind == MatchSingleton_kind); - ADDOP_LOAD_CONST(c, LOC(p), p->v.MatchSingleton.value); - ADDOP_COMPARE(c, LOC(p), Is); - RETURN_IF_ERROR(jump_to_fail_pop(c, LOC(p), pc, POP_JUMP_IF_FALSE)); - return SUCCESS; + return c->c_optimize; } -static int -compiler_pattern(struct compiler *c, pattern_ty p, pattern_context *pc) +int +_PyCompile_IsInteractiveTopLevel(compiler *c) { - switch (p->kind) { - case MatchValue_kind: - return compiler_pattern_value(c, p, pc); - case MatchSingleton_kind: - return compiler_pattern_singleton(c, p, pc); - case MatchSequence_kind: - return compiler_pattern_sequence(c, p, pc); - case MatchMapping_kind: - return compiler_pattern_mapping(c, p, pc); - case MatchClass_kind: - return compiler_pattern_class(c, p, pc); - case MatchStar_kind: - return compiler_pattern_star(c, p, pc); - case MatchAs_kind: - return compiler_pattern_as(c, p, pc); - case MatchOr_kind: - return compiler_pattern_or(c, p, pc); - } - // AST validator shouldn't let this happen, but if it does, - // just fail, don't crash out of the interpreter - const char *e = "invalid match pattern node in AST (kind=%d)"; - return compiler_error(c, LOC(p), e, p->kind); + assert(c->c_stack != NULL); + assert(PyList_CheckExact(c->c_stack)); + bool is_nested_scope = PyList_GET_SIZE(c->c_stack) > 0; + return c->c_interactive && !is_nested_scope; } -static int -compiler_match_inner(struct compiler *c, stmt_ty s, pattern_context *pc) +int +_PyCompile_ScopeType(compiler *c) { - VISIT(c, expr, s->v.Match.subject); - NEW_JUMP_TARGET_LABEL(c, end); - Py_ssize_t cases = asdl_seq_LEN(s->v.Match.cases); - assert(cases > 0); - match_case_ty m = asdl_seq_GET(s->v.Match.cases, cases - 1); - int has_default = WILDCARD_CHECK(m->pattern) && 1 < cases; - for (Py_ssize_t i = 0; i < cases - has_default; i++) { - m = asdl_seq_GET(s->v.Match.cases, i); - // Only copy the subject if we're *not* on the last case: - if (i != cases - has_default - 1) { - ADDOP_I(c, LOC(m->pattern), COPY, 1); - } - pc->stores = PyList_New(0); - if (pc->stores == NULL) { - return ERROR; - } - // Irrefutable cases must be either guarded, last, or both: - pc->allow_irrefutable = m->guard != NULL || i == cases - 1; - pc->fail_pop = NULL; - pc->fail_pop_size = 0; - pc->on_top = 0; - // NOTE: Can't use returning macros here (they'll leak pc->stores)! - if (compiler_pattern(c, m->pattern, pc) < 0) { - Py_DECREF(pc->stores); - return ERROR; - } - assert(!pc->on_top); - // It's a match! Store all of the captured names (they're on the stack). - Py_ssize_t nstores = PyList_GET_SIZE(pc->stores); - for (Py_ssize_t n = 0; n < nstores; n++) { - PyObject *name = PyList_GET_ITEM(pc->stores, n); - if (compiler_nameop(c, LOC(m->pattern), name, Store) < 0) { - Py_DECREF(pc->stores); - return ERROR; - } - } - Py_DECREF(pc->stores); - // NOTE: Returning macros are safe again. - if (m->guard) { - RETURN_IF_ERROR(ensure_fail_pop(c, pc, 0)); - RETURN_IF_ERROR(compiler_jump_if(c, LOC(m->pattern), m->guard, pc->fail_pop[0], 0)); - } - // Success! Pop the subject off, we're done with it: - if (i != cases - has_default - 1) { - ADDOP(c, LOC(m->pattern), POP_TOP); - } - VISIT_SEQ(c, stmt, m->body); - ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end); - // If the pattern fails to match, we want the line number of the - // cleanup to be associated with the failed pattern, not the last line - // of the body - RETURN_IF_ERROR(emit_and_reset_fail_pop(c, LOC(m->pattern), pc)); - } - if (has_default) { - // A trailing "case _" is common, and lets us save a bit of redundant - // pushing and popping in the loop above: - m = asdl_seq_GET(s->v.Match.cases, cases - 1); - if (cases == 1) { - // No matches. Done with the subject: - ADDOP(c, LOC(m->pattern), POP_TOP); - } - else { - // Show line coverage for default case (it doesn't create bytecode) - ADDOP(c, LOC(m->pattern), NOP); - } - if (m->guard) { - RETURN_IF_ERROR(compiler_jump_if(c, LOC(m->pattern), m->guard, end, 0)); - } - VISIT_SEQ(c, stmt, m->body); - } - USE_LABEL(c, end); - return SUCCESS; + return c->u->u_scope_type; } -static int -compiler_match(struct compiler *c, stmt_ty s) +int +_PyCompile_IsInInlinedComp(compiler *c) +{ + return c->u->u_in_inlined_comp; +} + +PyObject * +_PyCompile_Qualname(compiler *c) +{ + assert(c->u->u_metadata.u_qualname); + return c->u->u_metadata.u_qualname; +} + +_PyCompile_CodeUnitMetadata * +_PyCompile_Metadata(compiler *c) { - pattern_context pc; - pc.fail_pop = NULL; - int result = compiler_match_inner(c, s, &pc); - PyMem_Free(pc.fail_pop); - return result; + return &c->u->u_metadata; } -#undef WILDCARD_CHECK -#undef WILDCARD_STAR_CHECK +// Merge *obj* with constant cache, without recursion. +int +_PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj) +{ + PyObject *key = const_cache_insert(const_cache, *obj, false); + if (key == NULL) { + return ERROR; + } + if (PyTuple_CheckExact(key)) { + PyObject *item = PyTuple_GET_ITEM(key, 1); + Py_SETREF(*obj, Py_NewRef(item)); + Py_DECREF(key); + } + else { + Py_SETREF(*obj, key); + } + return SUCCESS; +} static PyObject * consts_dict_keys_inorder(PyObject *dict) @@ -7480,9 +1255,10 @@ consts_dict_keys_inorder(PyObject *dict) if (consts == NULL) return NULL; while (PyDict_Next(dict, &pos, &k, &v)) { - i = PyLong_AS_LONG(v); + assert(PyLong_CheckExact(v)); + i = PyLong_AsLong(v); /* The keys of the dictionary can be tuples wrapping a constant. - * (see dict_add_o and _PyCode_ConstantKey). In that case + * (see _PyCompile_DictAddObj and _PyCode_ConstantKey). In that case * the object we want is always second. */ if (PyTuple_CheckExact(k)) { k = PyTuple_GET_ITEM(k, 1); @@ -7495,88 +1271,41 @@ consts_dict_keys_inorder(PyObject *dict) } static int -compute_code_flags(struct compiler *c) +compute_code_flags(compiler *c) { PySTEntryObject *ste = c->u->u_ste; int flags = 0; - if (_PyST_IsFunctionLike(c->u->u_ste)) { + if (_PyST_IsFunctionLike(ste)) { flags |= CO_NEWLOCALS | CO_OPTIMIZED; if (ste->ste_nested) flags |= CO_NESTED; if (ste->ste_generator && !ste->ste_coroutine) flags |= CO_GENERATOR; - if (!ste->ste_generator && ste->ste_coroutine) - flags |= CO_COROUTINE; if (ste->ste_generator && ste->ste_coroutine) flags |= CO_ASYNC_GENERATOR; if (ste->ste_varargs) flags |= CO_VARARGS; if (ste->ste_varkeywords) flags |= CO_VARKEYWORDS; + if (ste->ste_has_docstring) + flags |= CO_HAS_DOCSTRING; + if (ste->ste_method) + flags |= CO_METHOD; } - /* (Only) inherit compilerflags in PyCF_MASK */ - flags |= (c->c_flags.cf_flags & PyCF_MASK); - - if ((IS_TOP_LEVEL_AWAIT(c)) && - ste->ste_coroutine && - !ste->ste_generator) { + if (ste->ste_coroutine && !ste->ste_generator) { flags |= CO_COROUTINE; } - return flags; -} - -// Merge *obj* with constant cache. -// Unlike merge_consts_recursive(), this function doesn't work recursively. -int -_PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj) -{ - assert(PyDict_CheckExact(const_cache)); - PyObject *key = _PyCode_ConstantKey(*obj); - if (key == NULL) { - return ERROR; - } - - PyObject *t; - int res = PyDict_SetDefaultRef(const_cache, key, key, &t); - Py_DECREF(key); - if (res < 0) { - return ERROR; - } - if (res == 0) { // inserted: obj is new constant. - Py_DECREF(t); - return SUCCESS; - } - - if (PyTuple_CheckExact(t)) { - PyObject *item = PyTuple_GET_ITEM(t, 1); - Py_SETREF(*obj, Py_NewRef(item)); - Py_DECREF(t); - } - else { - Py_SETREF(*obj, t); - } - - return SUCCESS; -} + /* (Only) inherit compilerflags in PyCF_MASK */ + flags |= (c->c_flags.cf_flags & PyCF_MASK); -static int -add_return_at_end(struct compiler *c, int addNone) -{ - /* Make sure every instruction stream that falls off the end returns None. - * This also ensures that no jump target offsets are out of bounds. - */ - if (addNone) { - ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); - } - ADDOP(c, NO_LOCATION, RETURN_VALUE); - return SUCCESS; + return flags; } static PyCodeObject * optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, - int code_flags, PyObject *filename) + int code_flags, PyObject *filename) { cfg_builder *g = NULL; instr_sequence optimized_instrs; @@ -7587,7 +1316,7 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, if (consts == NULL) { goto error; } - g = instr_sequence_to_cfg(&u->u_instr_sequence); + g = _PyCfg_FromInstructionSequence(u->u_instr_sequence); if (g == NULL) { goto error; } @@ -7609,20 +1338,20 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, } /** Assembly **/ - co = _PyAssemble_MakeCodeObject(&u->u_metadata, const_cache, consts, stackdepth, &optimized_instrs, nlocalsplus, code_flags, filename); error: Py_XDECREF(consts); - instr_sequence_fini(&optimized_instrs); + PyInstructionSequence_Fini(&optimized_instrs); _PyCfgBuilder_Free(g); return co; } -static PyCodeObject * -optimize_and_assemble(struct compiler *c, int addNone) + +PyCodeObject * +_PyCompile_OptimizeAndAssemble(compiler *c, int addNone) { struct compiler_unit *u = c->u; PyObject *const_cache = c->c_const_cache; @@ -7633,179 +1362,45 @@ optimize_and_assemble(struct compiler *c, int addNone) return NULL; } - if (add_return_at_end(c, addNone) < 0) { + if (_PyCodegen_AddReturnAtEnd(c, addNone) < 0) { return NULL; } return optimize_and_assemble_code_unit(u, const_cache, code_flags, filename); } -/* Access to compiler optimizations for unit tests. - * - * _PyCompile_CodeGen takes and AST, applies code-gen and - * returns the unoptimized CFG as an instruction list. - * - * _PyCompile_OptimizeCfg takes an instruction list, constructs - * a CFG, optimizes it and converts back to an instruction list. - * - * An instruction list is a PyList where each item is either - * a tuple describing a single instruction: - * (opcode, oparg, lineno, end_lineno, col, end_col), or - * a jump target label marking the beginning of a basic block. - */ - -static int -instructions_to_instr_sequence(PyObject *instructions, instr_sequence *seq) +PyCodeObject * +_PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, + int optimize, PyArena *arena) { - assert(PyList_Check(instructions)); - - Py_ssize_t num_insts = PyList_GET_SIZE(instructions); - bool *is_target = PyMem_Calloc(num_insts, sizeof(bool)); - if (is_target == NULL) { - return ERROR; - } - for (Py_ssize_t i = 0; i < num_insts; i++) { - PyObject *item = PyList_GET_ITEM(instructions, i); - if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 6) { - PyErr_SetString(PyExc_ValueError, "expected a 6-tuple"); - goto error; - } - int opcode = PyLong_AsLong(PyTuple_GET_ITEM(item, 0)); - if (PyErr_Occurred()) { - goto error; - } - if (HAS_TARGET(opcode)) { - int oparg = PyLong_AsLong(PyTuple_GET_ITEM(item, 1)); - if (PyErr_Occurred()) { - goto error; - } - if (oparg < 0 || oparg >= num_insts) { - PyErr_SetString(PyExc_ValueError, "label out of range"); - goto error; - } - is_target[oparg] = true; - } - } - - for (int i = 0; i < num_insts; i++) { - if (is_target[i]) { - if (_PyCompile_InstructionSequence_UseLabel(seq, i) < 0) { - goto error; - } - } - PyObject *item = PyList_GET_ITEM(instructions, i); - if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 6) { - PyErr_SetString(PyExc_ValueError, "expected a 6-tuple"); - goto error; - } - int opcode = PyLong_AsLong(PyTuple_GET_ITEM(item, 0)); - if (PyErr_Occurred()) { - goto error; - } - int oparg; - if (OPCODE_HAS_ARG(opcode)) { - oparg = PyLong_AsLong(PyTuple_GET_ITEM(item, 1)); - if (PyErr_Occurred()) { - goto error; - } - } - else { - oparg = 0; - } - location loc; - loc.lineno = PyLong_AsLong(PyTuple_GET_ITEM(item, 2)); - if (PyErr_Occurred()) { - goto error; - } - loc.end_lineno = PyLong_AsLong(PyTuple_GET_ITEM(item, 3)); - if (PyErr_Occurred()) { - goto error; - } - loc.col_offset = PyLong_AsLong(PyTuple_GET_ITEM(item, 4)); - if (PyErr_Occurred()) { - goto error; - } - loc.end_col_offset = PyLong_AsLong(PyTuple_GET_ITEM(item, 5)); - if (PyErr_Occurred()) { - goto error; - } - if (_PyCompile_InstructionSequence_Addop(seq, opcode, oparg, loc) < 0) { - goto error; - } + assert(!PyErr_Occurred()); + compiler *c = new_compiler(mod, filename, pflags, optimize, arena); + if (c == NULL) { + return NULL; } - PyMem_Free(is_target); - return SUCCESS; -error: - PyMem_Free(is_target); - return ERROR; -} - -static cfg_builder* -instructions_to_cfg(PyObject *instructions) -{ - cfg_builder *g = NULL; - instr_sequence seq; - memset(&seq, 0, sizeof(instr_sequence)); - if (instructions_to_instr_sequence(instructions, &seq) < 0) { - goto error; - } - g = instr_sequence_to_cfg(&seq); - if (g == NULL) { - goto error; - } - instr_sequence_fini(&seq); - return g; -error: - _PyCfgBuilder_Free(g); - instr_sequence_fini(&seq); - return NULL; + PyCodeObject *co = compiler_mod(c, mod); + compiler_free(c); + assert(co || PyErr_Occurred()); + return co; } -static PyObject * -instr_sequence_to_instructions(instr_sequence *seq) +int +_PyCompile_AstOptimize(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, + int optimize, PyArena *arena) { - PyObject *instructions = PyList_New(0); - if (instructions == NULL) { - return NULL; + _PyFutureFeatures future; + if (!_PyFuture_FromAST(mod, filename, &future)) { + return -1; } - for (int i = 0; i < seq->s_used; i++) { - instruction *instr = &seq->s_instrs[i]; - location loc = instr->i_loc; - int arg = HAS_TARGET(instr->i_opcode) ? - seq->s_labelmap[instr->i_oparg] : instr->i_oparg; - - PyObject *inst_tuple = Py_BuildValue( - "(iiiiii)", instr->i_opcode, arg, - loc.lineno, loc.end_lineno, - loc.col_offset, loc.end_col_offset); - if (inst_tuple == NULL) { - goto error; - } - - int res = PyList_Append(instructions, inst_tuple); - Py_DECREF(inst_tuple); - if (res != 0) { - goto error; - } + int flags = future.ff_features | cf->cf_flags; + if (optimize == -1) { + optimize = _Py_GetConfig()->optimization_level; } - return instructions; -error: - Py_XDECREF(instructions); - return NULL; -} - -static PyObject * -cfg_to_instructions(cfg_builder *g) -{ - instr_sequence seq; - memset(&seq, 0, sizeof(seq)); - if (_PyCfg_ToInstructionSequence(g, &seq) < 0) { - return NULL; + if (!_PyAST_Optimize(mod, arena, optimize, flags)) { + return -1; } - PyObject *res = instr_sequence_to_instructions(&seq); - instr_sequence_fini(&seq); - return res; + return 0; } // C implementation of inspect.cleandoc() @@ -7898,7 +1493,12 @@ _PyCompile_CleanDoc(PyObject *doc) return res; } - +/* Access to compiler optimizations for unit tests. + * + * _PyCompile_CodeGen takes an AST, applies code-gen and + * returns the unoptimized CFG as an instruction list. + * + */ PyObject * _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, int optimize, int compile_mode) @@ -7922,41 +1522,25 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, return NULL; } - struct compiler *c = new_compiler(mod, filename, pflags, optimize, arena); + compiler *c = new_compiler(mod, filename, pflags, optimize, arena); if (c == NULL) { _PyArena_Free(arena); return NULL; } + c->c_save_nested_seqs = true; metadata = PyDict_New(); if (metadata == NULL) { return NULL; } - if (compiler_enter_anonymous_scope(c, mod) < 0) { - return NULL; - } if (compiler_codegen(c, mod) < 0) { goto finally; } _PyCompile_CodeUnitMetadata *umd = &c->u->u_metadata; -#define SET_MATADATA_ITEM(key, value) \ - if (value != NULL) { \ - if (PyDict_SetItemString(metadata, key, value) < 0) goto finally; \ - } - - SET_MATADATA_ITEM("name", umd->u_name); - SET_MATADATA_ITEM("qualname", umd->u_qualname); - SET_MATADATA_ITEM("consts", umd->u_consts); - SET_MATADATA_ITEM("names", umd->u_names); - SET_MATADATA_ITEM("varnames", umd->u_varnames); - SET_MATADATA_ITEM("cellvars", umd->u_cellvars); - SET_MATADATA_ITEM("freevars", umd->u_freevars); -#undef SET_MATADATA_ITEM - -#define SET_MATADATA_INT(key, value) do { \ +#define SET_METADATA_INT(key, value) do { \ PyObject *v = PyLong_FromLong((long)value); \ if (v == NULL) goto finally; \ int res = PyDict_SetItemString(metadata, key, v); \ @@ -7964,63 +1548,40 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, if (res < 0) goto finally; \ } while (0); - SET_MATADATA_INT("argcount", umd->u_argcount); - SET_MATADATA_INT("posonlyargcount", umd->u_posonlyargcount); - SET_MATADATA_INT("kwonlyargcount", umd->u_kwonlyargcount); -#undef SET_MATADATA_INT + SET_METADATA_INT("argcount", umd->u_argcount); + SET_METADATA_INT("posonlyargcount", umd->u_posonlyargcount); + SET_METADATA_INT("kwonlyargcount", umd->u_kwonlyargcount); +#undef SET_METADATA_INT int addNone = mod->kind != Expression_kind; - if (add_return_at_end(c, addNone) < 0) { + if (_PyCodegen_AddReturnAtEnd(c, addNone) < 0) { goto finally; } - PyObject *insts = instr_sequence_to_instructions(INSTR_SEQUENCE(c)); - if (insts == NULL) { - goto finally; + if (_PyInstructionSequence_ApplyLabelMap(_PyCompile_InstrSequence(c)) < 0) { + return NULL; } - res = PyTuple_Pack(2, insts, metadata); - Py_DECREF(insts); + /* Allocate a copy of the instruction sequence on the heap */ + res = PyTuple_Pack(2, _PyCompile_InstrSequence(c), metadata); finally: Py_XDECREF(metadata); - compiler_exit_scope(c); + _PyCompile_ExitScope(c); compiler_free(c); _PyArena_Free(arena); return res; } -PyObject * -_PyCompile_OptimizeCfg(PyObject *instructions, PyObject *consts, int nlocals) -{ - cfg_builder *g = NULL; - PyObject *res = NULL; - PyObject *const_cache = PyDict_New(); - if (const_cache == NULL) { - return NULL; - } - - g = instructions_to_cfg(instructions); - if (g == NULL) { - goto error; - } - int nparams = 0, firstlineno = 1; - if (_PyCfg_OptimizeCodeUnit(g, consts, const_cache, nlocals, - nparams, firstlineno) < 0) { - goto error; - } - res = cfg_to_instructions(g); -error: - Py_DECREF(const_cache); - _PyCfgBuilder_Free(g); - return res; -} - int _PyCfg_JumpLabelsToTargets(cfg_builder *g); PyCodeObject * _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, - PyObject *instructions) + PyObject *seq) { + if (!_PyInstructionSequence_Check(seq)) { + PyErr_SetString(PyExc_TypeError, "expected an instruction sequence"); + return NULL; + } cfg_builder *g = NULL; PyCodeObject *co = NULL; instr_sequence optimized_instrs; @@ -8031,7 +1592,7 @@ _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, return NULL; } - g = instructions_to_cfg(instructions); + g = _PyCfg_FromInstructionSequence((instr_sequence*)seq); if (g == NULL) { goto error; } @@ -8060,11 +1621,10 @@ _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, error: Py_DECREF(const_cache); _PyCfgBuilder_Free(g); - instr_sequence_fini(&optimized_instrs); + PyInstructionSequence_Fini(&optimized_instrs); return co; } - /* Retained for API compatibility. * Optimization is now done in _PyCfg_OptimizeCodeUnit */ diff --git a/Python/config_common.h b/Python/config_common.h new file mode 100644 index 00000000000000..e749bd4bf0dc68 --- /dev/null +++ b/Python/config_common.h @@ -0,0 +1,36 @@ + +static inline int +_config_dict_get(PyObject *dict, const char *name, PyObject **p_item) +{ + PyObject *item; + if (PyDict_GetItemStringRef(dict, name, &item) < 0) { + return -1; + } + if (item == NULL) { + // We do not set an exception. + return -1; + } + *p_item = item; + return 0; +} + + +static PyObject* +config_dict_get(PyObject *dict, const char *name) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_ValueError, "missing config key: %s", name); + } + return NULL; + } + return item; +} + + +static void +config_dict_invalid_type(const char *name) +{ + PyErr_Format(PyExc_TypeError, "invalid config type: %s", name); +} diff --git a/Python/context.c b/Python/context.c index 3937819b3c386c..bb1aa42b9c5e4f 100644 --- a/Python/context.c +++ b/Python/context.c @@ -1,6 +1,7 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_VectorcallTstate() #include "pycore_context.h" +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_gc.h" // _PyObject_GC_MAY_BE_TRACKED() #include "pycore_hamt.h" #include "pycore_initconfig.h" // _PyStatus_OK() @@ -64,16 +65,6 @@ static int contextvar_del(PyContextVar *var); -#ifdef WITH_FREELISTS -static struct _Py_context_freelist * -get_context_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - return &freelists->contexts; -} -#endif - - PyObject * _PyContext_NewHamtForTests(void) { @@ -108,6 +99,96 @@ PyContext_CopyCurrent(void) return (PyObject *)context_new_from_vars(ctx->ctx_vars); } +static const char * +context_event_name(PyContextEvent event) { + switch (event) { + case Py_CONTEXT_SWITCHED: + return "Py_CONTEXT_SWITCHED"; + default: + return "?"; + } + Py_UNREACHABLE(); +} + +static void +notify_context_watchers(PyThreadState *ts, PyContextEvent event, PyObject *ctx) +{ + if (ctx == NULL) { + // This will happen after exiting the last context in the stack, which + // can occur if context_get was never called before entering a context + // (e.g., called `contextvars.Context().run()` on a fresh thread, as + // PyContext_Enter doesn't call context_get). + ctx = Py_None; + } + assert(Py_REFCNT(ctx) > 0); + PyInterpreterState *interp = ts->interp; + assert(interp->_initialized); + uint8_t bits = interp->active_context_watchers; + int i = 0; + while (bits) { + assert(i < CONTEXT_MAX_WATCHERS); + if (bits & 1) { + PyContext_WatchCallback cb = interp->context_watchers[i]; + assert(cb != NULL); + if (cb(event, ctx) < 0) { + PyErr_FormatUnraisable( + "Exception ignored in %s watcher callback for %R", + context_event_name(event), ctx); + } + } + i++; + bits >>= 1; + } +} + + +int +PyContext_AddWatcher(PyContext_WatchCallback callback) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp->_initialized); + + for (int i = 0; i < CONTEXT_MAX_WATCHERS; i++) { + if (!interp->context_watchers[i]) { + interp->context_watchers[i] = callback; + interp->active_context_watchers |= (1 << i); + return i; + } + } + + PyErr_SetString(PyExc_RuntimeError, "no more context watcher IDs available"); + return -1; +} + + +int +PyContext_ClearWatcher(int watcher_id) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp->_initialized); + if (watcher_id < 0 || watcher_id >= CONTEXT_MAX_WATCHERS) { + PyErr_Format(PyExc_ValueError, "Invalid context watcher ID %d", watcher_id); + return -1; + } + if (!interp->context_watchers[watcher_id]) { + PyErr_Format(PyExc_ValueError, "No context watcher set for ID %d", watcher_id); + return -1; + } + interp->context_watchers[watcher_id] = NULL; + interp->active_context_watchers &= ~(1 << watcher_id); + return 0; +} + + +static inline void +context_switched(PyThreadState *ts) +{ + ts->context_ver++; + // ts->context is used instead of context_get() because context_get() might + // throw if ts->context is NULL. + notify_context_watchers(ts, Py_CONTEXT_SWITCHED, ts->context); +} + static int _PyContext_Enter(PyThreadState *ts, PyObject *octx) @@ -125,8 +206,7 @@ _PyContext_Enter(PyThreadState *ts, PyObject *octx) ctx->ctx_entered = 1; ts->context = Py_NewRef(ctx); - ts->context_ver++; - + context_switched(ts); return 0; } @@ -161,11 +241,10 @@ _PyContext_Exit(PyThreadState *ts, PyObject *octx) } Py_SETREF(ts->context, (PyObject *)ctx->ctx_prev); - ts->context_ver++; ctx->ctx_prev = NULL; ctx->ctx_entered = 0; - + context_switched(ts); return 0; } @@ -203,6 +282,7 @@ PyContextVar_Get(PyObject *ovar, PyObject *def, PyObject **val) goto not_found; } +#ifndef Py_GIL_DISABLED if (var->var_cached != NULL && var->var_cached_tsid == ts->id && var->var_cached_tsver == ts->context_ver) @@ -210,6 +290,7 @@ PyContextVar_Get(PyObject *ovar, PyObject *def, PyObject **val) *val = var->var_cached; goto found; } +#endif assert(PyContext_CheckExact(ts->context)); PyHamtObject *vars = ((PyContext *)ts->context)->ctx_vars; @@ -221,9 +302,11 @@ PyContextVar_Get(PyObject *ovar, PyObject *def, PyObject **val) } if (res == 1) { assert(found != NULL); +#ifndef Py_GIL_DISABLED var->var_cached = found; /* borrow */ var->var_cached_tsid = ts->id; var->var_cached_tsver = ts->context_ver; +#endif *val = found; goto found; @@ -336,23 +419,14 @@ class _contextvars.Context "PyContext *" "&PyContext_Type" /*[clinic end generated code: output=da39a3ee5e6b4b0d input=bdf87f8e0cb580e8]*/ +#define _PyContext_CAST(op) ((PyContext *)(op)) + + static inline PyContext * _context_alloc(void) { - PyContext *ctx; -#ifdef WITH_FREELISTS - struct _Py_context_freelist *context_freelist = get_context_freelist(); - if (context_freelist->numfree > 0) { - context_freelist->numfree--; - ctx = context_freelist->items; - context_freelist->items = (PyContext *)ctx->ctx_weakreflist; - OBJECT_STAT_INC(from_freelist); - ctx->ctx_weakreflist = NULL; - _Py_NewReference((PyObject *)ctx); - } - else -#endif - { + PyContext *ctx = _Py_FREELIST_POP(PyContext, contexts); + if (ctx == NULL) { ctx = PyObject_GC_New(PyContext, &PyContext_Type); if (ctx == NULL) { return NULL; @@ -442,49 +516,40 @@ context_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } static int -context_tp_clear(PyContext *self) +context_tp_clear(PyObject *op) { + PyContext *self = _PyContext_CAST(op); Py_CLEAR(self->ctx_prev); Py_CLEAR(self->ctx_vars); return 0; } static int -context_tp_traverse(PyContext *self, visitproc visit, void *arg) +context_tp_traverse(PyObject *op, visitproc visit, void *arg) { + PyContext *self = _PyContext_CAST(op); Py_VISIT(self->ctx_prev); Py_VISIT(self->ctx_vars); return 0; } static void -context_tp_dealloc(PyContext *self) +context_tp_dealloc(PyObject *self) { _PyObject_GC_UNTRACK(self); - - if (self->ctx_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject*)self); + PyContext *ctx = _PyContext_CAST(self); + if (ctx->ctx_weakreflist != NULL) { + PyObject_ClearWeakRefs(self); } (void)context_tp_clear(self); -#ifdef WITH_FREELISTS - struct _Py_context_freelist *context_freelist = get_context_freelist(); - if (context_freelist->numfree >= 0 && context_freelist->numfree < PyContext_MAXFREELIST) { - context_freelist->numfree++; - self->ctx_weakreflist = (PyObject *)context_freelist->items; - context_freelist->items = self; - OBJECT_STAT_INC(to_freelist); - } - else -#endif - { - Py_TYPE(self)->tp_free(self); - } + _Py_FREELIST_FREE(contexts, self, Py_TYPE(self)->tp_free); } static PyObject * -context_tp_iter(PyContext *self) +context_tp_iter(PyObject *op) { + PyContext *self = _PyContext_CAST(op); return _PyHamt_NewIterKeys(self->ctx_vars); } @@ -516,18 +581,20 @@ context_tp_richcompare(PyObject *v, PyObject *w, int op) } static Py_ssize_t -context_tp_len(PyContext *self) +context_tp_len(PyObject *op) { + PyContext *self = _PyContext_CAST(op); return _PyHamt_Len(self->ctx_vars); } static PyObject * -context_tp_subscript(PyContext *self, PyObject *key) +context_tp_subscript(PyObject *op, PyObject *key) { if (context_check_key_type(key)) { return NULL; } PyObject *val = NULL; + PyContext *self = _PyContext_CAST(op); int found = _PyHamt_Find(self->ctx_vars, key, &val); if (found < 0) { return NULL; @@ -540,12 +607,13 @@ context_tp_subscript(PyContext *self, PyObject *key) } static int -context_tp_contains(PyContext *self, PyObject *key) +context_tp_contains(PyObject *op, PyObject *key) { if (context_check_key_type(key)) { return -1; } PyObject *val = NULL; + PyContext *self = _PyContext_CAST(op); return _PyHamt_Find(self->ctx_vars, key, &val); } @@ -642,7 +710,7 @@ _contextvars_Context_copy_impl(PyContext *self) static PyObject * -context_run(PyContext *self, PyObject *const *args, +context_run(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyThreadState *ts = _PyThreadState_GET(); @@ -653,14 +721,15 @@ context_run(PyContext *self, PyObject *const *args, return NULL; } - if (_PyContext_Enter(ts, (PyObject *)self)) { + if (_PyContext_Enter(ts, self)) { return NULL; } PyObject *call_result = _PyObject_VectorcallTstate( ts, args[0], args + 1, nargs - 1, kwnames); - if (_PyContext_Exit(ts, (PyObject *)self)) { + if (_PyContext_Exit(ts, self)) { + Py_XDECREF(call_result); return NULL; } @@ -679,21 +748,12 @@ static PyMethodDef PyContext_methods[] = { }; static PySequenceMethods PyContext_as_sequence = { - 0, /* sq_length */ - 0, /* sq_concat */ - 0, /* sq_repeat */ - 0, /* sq_item */ - 0, /* sq_slice */ - 0, /* sq_ass_item */ - 0, /* sq_ass_slice */ - (objobjproc)context_tp_contains, /* sq_contains */ - 0, /* sq_inplace_concat */ - 0, /* sq_inplace_repeat */ + .sq_contains = context_tp_contains }; static PyMappingMethods PyContext_as_mapping = { - (lenfunc)context_tp_len, /* mp_length */ - (binaryfunc)context_tp_subscript, /* mp_subscript */ + .mp_length = context_tp_len, + .mp_subscript = context_tp_subscript }; PyTypeObject PyContext_Type = { @@ -703,13 +763,13 @@ PyTypeObject PyContext_Type = { .tp_methods = PyContext_methods, .tp_as_mapping = &PyContext_as_mapping, .tp_as_sequence = &PyContext_as_sequence, - .tp_iter = (getiterfunc)context_tp_iter, - .tp_dealloc = (destructor)context_tp_dealloc, + .tp_iter = context_tp_iter, + .tp_dealloc = context_tp_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .tp_richcompare = context_tp_richcompare, - .tp_traverse = (traverseproc)context_tp_traverse, - .tp_clear = (inquiry)context_tp_clear, + .tp_traverse = context_tp_traverse, + .tp_clear = context_tp_clear, .tp_new = context_tp_new, .tp_weaklistoffset = offsetof(PyContext, ctx_weakreflist), .tp_hash = PyObject_HashNotImplemented, @@ -722,8 +782,10 @@ PyTypeObject PyContext_Type = { static int contextvar_set(PyContextVar *var, PyObject *val) { +#ifndef Py_GIL_DISABLED var->var_cached = NULL; PyThreadState *ts = _PyThreadState_GET(); +#endif PyContext *ctx = context_get(); if (ctx == NULL) { @@ -738,16 +800,20 @@ contextvar_set(PyContextVar *var, PyObject *val) Py_SETREF(ctx->ctx_vars, new_vars); +#ifndef Py_GIL_DISABLED var->var_cached = val; /* borrow */ var->var_cached_tsid = ts->id; var->var_cached_tsver = ts->context_ver; +#endif return 0; } static int contextvar_del(PyContextVar *var) { +#ifndef Py_GIL_DISABLED var->var_cached = NULL; +#endif PyContext *ctx = context_get(); if (ctx == NULL) { @@ -794,7 +860,7 @@ contextvar_generate_hash(void *addr, PyObject *name) return -1; } - Py_hash_t res = _Py_HashPointer(addr) ^ name_hash; + Py_hash_t res = Py_HashPointer(addr) ^ name_hash; return res == -1 ? -2 : res; } @@ -822,9 +888,11 @@ contextvar_new(PyObject *name, PyObject *def) var->var_default = Py_XNewRef(def); +#ifndef Py_GIL_DISABLED var->var_cached = NULL; var->var_cached_tsid = 0; var->var_cached_tsver = 0; +#endif if (_PyObject_GC_MAY_BE_TRACKED(name) || (def != NULL && _PyObject_GC_MAY_BE_TRACKED(def))) @@ -841,6 +909,9 @@ class _contextvars.ContextVar "PyContextVar *" "&PyContextVar_Type" /*[clinic end generated code: output=da39a3ee5e6b4b0d input=445da935fa8883c3]*/ +#define _PyContextVar_CAST(op) ((PyContextVar *)(op)) + + static PyObject * contextvar_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -858,26 +929,30 @@ contextvar_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } static int -contextvar_tp_clear(PyContextVar *self) +contextvar_tp_clear(PyObject *op) { + PyContextVar *self = _PyContextVar_CAST(op); Py_CLEAR(self->var_name); Py_CLEAR(self->var_default); +#ifndef Py_GIL_DISABLED self->var_cached = NULL; self->var_cached_tsid = 0; self->var_cached_tsver = 0; +#endif return 0; } static int -contextvar_tp_traverse(PyContextVar *self, visitproc visit, void *arg) +contextvar_tp_traverse(PyObject *op, visitproc visit, void *arg) { + PyContextVar *self = _PyContextVar_CAST(op); Py_VISIT(self->var_name); Py_VISIT(self->var_default); return 0; } static void -contextvar_tp_dealloc(PyContextVar *self) +contextvar_tp_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); (void)contextvar_tp_clear(self); @@ -885,64 +960,49 @@ contextvar_tp_dealloc(PyContextVar *self) } static Py_hash_t -contextvar_tp_hash(PyContextVar *self) +contextvar_tp_hash(PyObject *op) { + PyContextVar *self = _PyContextVar_CAST(op); return self->var_hash; } static PyObject * -contextvar_tp_repr(PyContextVar *self) +contextvar_tp_repr(PyObject *op) { - _PyUnicodeWriter writer; - - _PyUnicodeWriter_Init(&writer); - - if (_PyUnicodeWriter_WriteASCIIString( - &writer, "" + // "" + Py_ssize_t estimate = self->var_default ? 53 : 43; + PyUnicodeWriter *writer = PyUnicodeWriter_Create(estimate); + if (writer == NULL) { + return NULL; } - PyObject *name = PyObject_Repr(self->var_name); - if (name == NULL) { + if (PyUnicodeWriter_WriteUTF8(writer, "", self); - if (addr == NULL) { + if (PyUnicodeWriter_Format(writer, " at %p>", self) < 0) { goto error; } - if (_PyUnicodeWriter_WriteStr(&writer, addr) < 0) { - Py_DECREF(addr); - goto error; - } - Py_DECREF(addr); - - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); error: - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); return NULL; } @@ -1053,15 +1113,15 @@ PyTypeObject PyContextVar_Type = { sizeof(PyContextVar), .tp_methods = PyContextVar_methods, .tp_members = PyContextVar_members, - .tp_dealloc = (destructor)contextvar_tp_dealloc, + .tp_dealloc = contextvar_tp_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)contextvar_tp_traverse, - .tp_clear = (inquiry)contextvar_tp_clear, + .tp_traverse = contextvar_tp_traverse, + .tp_clear = contextvar_tp_clear, .tp_new = contextvar_tp_new, .tp_free = PyObject_GC_Del, - .tp_hash = (hashfunc)contextvar_tp_hash, - .tp_repr = (reprfunc)contextvar_tp_repr, + .tp_hash = contextvar_tp_hash, + .tp_repr = contextvar_tp_repr, }; @@ -1076,6 +1136,9 @@ class _contextvars.Token "PyContextToken *" "&PyContextToken_Type" /*[clinic end generated code: output=da39a3ee5e6b4b0d input=338a5e2db13d3f5b]*/ +#define _PyContextToken_CAST(op) ((PyContextToken *)(op)) + + static PyObject * token_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -1085,8 +1148,9 @@ token_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } static int -token_tp_clear(PyContextToken *self) +token_tp_clear(PyObject *op) { + PyContextToken *self = _PyContextToken_CAST(op); Py_CLEAR(self->tok_ctx); Py_CLEAR(self->tok_var); Py_CLEAR(self->tok_oldval); @@ -1094,8 +1158,9 @@ token_tp_clear(PyContextToken *self) } static int -token_tp_traverse(PyContextToken *self, visitproc visit, void *arg) +token_tp_traverse(PyObject *op, visitproc visit, void *arg) { + PyContextToken *self = _PyContextToken_CAST(op); Py_VISIT(self->tok_ctx); Py_VISIT(self->tok_var); Py_VISIT(self->tok_oldval); @@ -1103,7 +1168,7 @@ token_tp_traverse(PyContextToken *self, visitproc visit, void *arg) } static void -token_tp_dealloc(PyContextToken *self) +token_tp_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); (void)token_tp_clear(self); @@ -1111,62 +1176,48 @@ token_tp_dealloc(PyContextToken *self) } static PyObject * -token_tp_repr(PyContextToken *self) +token_tp_repr(PyObject *op) { - _PyUnicodeWriter writer; - - _PyUnicodeWriter_Init(&writer); - - if (_PyUnicodeWriter_WriteASCIIString(&writer, "tok_used) { - if (_PyUnicodeWriter_WriteASCIIString(&writer, " used", 5) < 0) { + if (PyUnicodeWriter_WriteUTF8(writer, " used", 5) < 0) { goto error; } } - - if (_PyUnicodeWriter_WriteASCIIString(&writer, " var=", 5) < 0) { - goto error; - } - - PyObject *var = PyObject_Repr((PyObject *)self->tok_var); - if (var == NULL) { - goto error; - } - if (_PyUnicodeWriter_WriteStr(&writer, var) < 0) { - Py_DECREF(var); + if (PyUnicodeWriter_WriteUTF8(writer, " var=", 5) < 0) { goto error; } - Py_DECREF(var); - - PyObject *addr = PyUnicode_FromFormat(" at %p>", self); - if (addr == NULL) { + if (PyUnicodeWriter_WriteRepr(writer, (PyObject *)self->tok_var) < 0) { goto error; } - if (_PyUnicodeWriter_WriteStr(&writer, addr) < 0) { - Py_DECREF(addr); + if (PyUnicodeWriter_Format(writer, " at %p>", self) < 0) { goto error; } - Py_DECREF(addr); - - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); error: - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); return NULL; } static PyObject * -token_get_var(PyContextToken *self, void *Py_UNUSED(ignored)) +token_get_var(PyObject *op, void *Py_UNUSED(ignored)) { + PyContextToken *self = _PyContextToken_CAST(op); return Py_NewRef(self->tok_var);; } static PyObject * -token_get_old_value(PyContextToken *self, void *Py_UNUSED(ignored)) +token_get_old_value(PyObject *op, void *Py_UNUSED(ignored)) { + PyContextToken *self = _PyContextToken_CAST(op); if (self->tok_oldval == NULL) { return get_token_missing(); } @@ -1175,8 +1226,8 @@ token_get_old_value(PyContextToken *self, void *Py_UNUSED(ignored)) } static PyGetSetDef PyContextTokenType_getsetlist[] = { - {"var", (getter)token_get_var, NULL, NULL}, - {"old_value", (getter)token_get_old_value, NULL, NULL}, + {"var", token_get_var, NULL, NULL}, + {"old_value", token_get_old_value, NULL, NULL}, {NULL} }; @@ -1192,15 +1243,15 @@ PyTypeObject PyContextToken_Type = { sizeof(PyContextToken), .tp_methods = PyContextTokenType_methods, .tp_getset = PyContextTokenType_getsetlist, - .tp_dealloc = (destructor)token_tp_dealloc, + .tp_dealloc = token_tp_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)token_tp_traverse, - .tp_clear = (inquiry)token_tp_clear, + .tp_traverse = token_tp_traverse, + .tp_clear = token_tp_clear, .tp_new = token_tp_new, .tp_free = PyObject_GC_Del, .tp_hash = PyObject_HashNotImplemented, - .tp_repr = (reprfunc)token_tp_repr, + .tp_repr = token_tp_repr, }; static PyContextToken * @@ -1234,7 +1285,7 @@ context_token_missing_tp_repr(PyObject *self) } static void -context_token_missing_tp_dealloc(_PyContextTokenMissing *Py_UNUSED(self)) +context_token_missing_tp_dealloc(PyObject *Py_UNUSED(self)) { #ifdef Py_DEBUG /* The singleton is statically allocated. */ @@ -1249,7 +1300,7 @@ PyTypeObject _PyContextTokenMissing_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "Token.MISSING", sizeof(_PyContextTokenMissing), - .tp_dealloc = (destructor)context_token_missing_tp_dealloc, + .tp_dealloc = context_token_missing_tp_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_repr = context_token_missing_tp_repr, @@ -1266,24 +1317,6 @@ get_token_missing(void) /////////////////////////// -void -_PyContext_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) -{ -#ifdef WITH_FREELISTS - struct _Py_context_freelist *state = &freelists->contexts; - for (; state->numfree > 0; state->numfree--) { - PyContext *ctx = state->items; - state->items = (PyContext *)ctx->ctx_weakreflist; - ctx->ctx_weakreflist = NULL; - PyObject_GC_Del(ctx); - } - if (is_finalization) { - state->numfree = -1; - } -#endif -} - - PyStatus _PyContext_Init(PyInterpreterState *interp) { diff --git a/Python/critical_section.c b/Python/critical_section.c index 2214d80eeb297b..73857b85496316 100644 --- a/Python/critical_section.c +++ b/Python/critical_section.c @@ -3,85 +3,106 @@ #include "pycore_lock.h" #include "pycore_critical_section.h" -static_assert(_Alignof(_PyCriticalSection) >= 4, +#ifdef Py_GIL_DISABLED +static_assert(_Alignof(PyCriticalSection) >= 4, "critical section must be aligned to at least 4 bytes"); +#endif + +#ifdef Py_GIL_DISABLED +static PyCriticalSection * +untag_critical_section(uintptr_t tag) +{ + return (PyCriticalSection *)(tag & ~_Py_CRITICAL_SECTION_MASK); +} +#endif void -_PyCriticalSection_BeginSlow(_PyCriticalSection *c, PyMutex *m) +_PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m) { +#ifdef Py_GIL_DISABLED PyThreadState *tstate = _PyThreadState_GET(); - c->mutex = NULL; - c->prev = (uintptr_t)tstate->critical_section; + // As an optimisation for locking the same object recursively, skip + // locking if the mutex is currently locked by the top-most critical + // section. + if (tstate->critical_section && + untag_critical_section(tstate->critical_section)->_cs_mutex == m) { + c->_cs_mutex = NULL; + c->_cs_prev = 0; + return; + } + c->_cs_mutex = NULL; + c->_cs_prev = (uintptr_t)tstate->critical_section; tstate->critical_section = (uintptr_t)c; - _PyMutex_LockSlow(m); - c->mutex = m; + PyMutex_Lock(m); + c->_cs_mutex = m; +#endif } void -_PyCriticalSection2_BeginSlow(_PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, +_PyCriticalSection2_BeginSlow(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, int is_m1_locked) { +#ifdef Py_GIL_DISABLED PyThreadState *tstate = _PyThreadState_GET(); - c->base.mutex = NULL; - c->mutex2 = NULL; - c->base.prev = tstate->critical_section; + c->_cs_base._cs_mutex = NULL; + c->_cs_mutex2 = NULL; + c->_cs_base._cs_prev = tstate->critical_section; tstate->critical_section = (uintptr_t)c | _Py_CRITICAL_SECTION_TWO_MUTEXES; if (!is_m1_locked) { PyMutex_Lock(m1); } PyMutex_Lock(m2); - c->base.mutex = m1; - c->mutex2 = m2; + c->_cs_base._cs_mutex = m1; + c->_cs_mutex2 = m2; +#endif } -static _PyCriticalSection * -untag_critical_section(uintptr_t tag) -{ - return (_PyCriticalSection *)(tag & ~_Py_CRITICAL_SECTION_MASK); -} // Release all locks held by critical sections. This is called by // _PyThreadState_Detach. void _PyCriticalSection_SuspendAll(PyThreadState *tstate) { +#ifdef Py_GIL_DISABLED uintptr_t *tagptr = &tstate->critical_section; while (_PyCriticalSection_IsActive(*tagptr)) { - _PyCriticalSection *c = untag_critical_section(*tagptr); + PyCriticalSection *c = untag_critical_section(*tagptr); - if (c->mutex) { - PyMutex_Unlock(c->mutex); + if (c->_cs_mutex) { + PyMutex_Unlock(c->_cs_mutex); if ((*tagptr & _Py_CRITICAL_SECTION_TWO_MUTEXES)) { - _PyCriticalSection2 *c2 = (_PyCriticalSection2 *)c; - if (c2->mutex2) { - PyMutex_Unlock(c2->mutex2); + PyCriticalSection2 *c2 = (PyCriticalSection2 *)c; + if (c2->_cs_mutex2) { + PyMutex_Unlock(c2->_cs_mutex2); } } } *tagptr |= _Py_CRITICAL_SECTION_INACTIVE; - tagptr = &c->prev; + tagptr = &c->_cs_prev; } +#endif } void _PyCriticalSection_Resume(PyThreadState *tstate) { +#ifdef Py_GIL_DISABLED uintptr_t p = tstate->critical_section; - _PyCriticalSection *c = untag_critical_section(p); + PyCriticalSection *c = untag_critical_section(p); assert(!_PyCriticalSection_IsActive(p)); - PyMutex *m1 = c->mutex; - c->mutex = NULL; + PyMutex *m1 = c->_cs_mutex; + c->_cs_mutex = NULL; PyMutex *m2 = NULL; - _PyCriticalSection2 *c2 = NULL; + PyCriticalSection2 *c2 = NULL; if ((p & _Py_CRITICAL_SECTION_TWO_MUTEXES)) { - c2 = (_PyCriticalSection2 *)c; - m2 = c2->mutex2; - c2->mutex2 = NULL; + c2 = (PyCriticalSection2 *)c; + m2 = c2->_cs_mutex2; + c2->_cs_mutex2 = NULL; } if (m1) { @@ -91,10 +112,47 @@ _PyCriticalSection_Resume(PyThreadState *tstate) PyMutex_Lock(m2); } - c->mutex = m1; + c->_cs_mutex = m1; if (m2) { - c2->mutex2 = m2; + c2->_cs_mutex2 = m2; } tstate->critical_section &= ~_Py_CRITICAL_SECTION_INACTIVE; +#endif +} + +#undef PyCriticalSection_Begin +void +PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op) +{ +#ifdef Py_GIL_DISABLED + _PyCriticalSection_Begin(c, op); +#endif +} + +#undef PyCriticalSection_End +void +PyCriticalSection_End(PyCriticalSection *c) +{ +#ifdef Py_GIL_DISABLED + _PyCriticalSection_End(c); +#endif +} + +#undef PyCriticalSection2_Begin +void +PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) +{ +#ifdef Py_GIL_DISABLED + _PyCriticalSection2_Begin(c, a, b); +#endif +} + +#undef PyCriticalSection2_End +void +PyCriticalSection2_End(PyCriticalSection2 *c) +{ +#ifdef Py_GIL_DISABLED + _PyCriticalSection2_End(c); +#endif } diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 143b261f9a5396..0a106ad636bfe8 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -3,23 +3,21 @@ #include "Python.h" #include "pycore_ceval.h" // _Py_simple_func -#include "pycore_crossinterp.h" // struct _xid +#include "pycore_crossinterp.h" // _PyXIData_t #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_namespace.h" //_PyNamespace_New() #include "pycore_pyerrors.h" // _PyErr_Clear() -#include "pycore_typeobject.h" // _PyType_GetModuleName() -#include "pycore_weakref.h" // _PyWeakref_GET_REF() /**************/ /* exceptions */ /**************/ -static int init_exceptions(PyInterpreterState *); -static void fini_exceptions(PyInterpreterState *); -static int _init_not_shareable_error_type(PyInterpreterState *); -static void _fini_not_shareable_error_type(PyInterpreterState *); -static PyObject * _get_not_shareable_error_type(PyInterpreterState *); +typedef struct xi_exceptions exceptions_t; +static int init_static_exctypes(exceptions_t *, PyInterpreterState *); +static void fini_static_exctypes(exceptions_t *, PyInterpreterState *); +static int init_heap_exctypes(exceptions_t *); +static void fini_heap_exctypes(exceptions_t *); #include "crossinterp_exceptions.h" @@ -58,25 +56,24 @@ _Py_CallInInterpreterAndRawFree(PyInterpreterState *interp, /* cross-interpreter data */ /**************************/ -/* registry of {type -> crossinterpdatafunc} */ +/* registry of {type -> xidatafunc} */ /* For now we use a global registry of shareable classes. An alternative would be to add a tp_* slot for a class's - crossinterpdatafunc. It would be simpler and more efficient. */ + xidatafunc. It would be simpler and more efficient. */ -static void xid_lookup_init(PyInterpreterState *); -static void xid_lookup_fini(PyInterpreterState *); -static crossinterpdatafunc lookup_getdata(PyInterpreterState *, PyObject *); +static void xid_lookup_init(_PyXIData_lookup_t *); +static void xid_lookup_fini(_PyXIData_lookup_t *); +static xidatafunc lookup_getdata(_PyXIData_lookup_context_t *, PyObject *); #include "crossinterp_data_lookup.h" /* lifecycle */ -_PyCrossInterpreterData * -_PyCrossInterpreterData_New(void) +_PyXIData_t * +_PyXIData_New(void) { - _PyCrossInterpreterData *xid = PyMem_RawMalloc( - sizeof(_PyCrossInterpreterData)); + _PyXIData_t *xid = PyMem_RawMalloc(sizeof(_PyXIData_t)); if (xid == NULL) { PyErr_NoMemory(); } @@ -84,10 +81,10 @@ _PyCrossInterpreterData_New(void) } void -_PyCrossInterpreterData_Free(_PyCrossInterpreterData *xid) +_PyXIData_Free(_PyXIData_t *xid) { PyInterpreterState *interp = PyInterpreterState_Get(); - _PyCrossInterpreterData_Clear(interp, xid); + _PyXIData_Clear(interp, xid); PyMem_RawFree(xid); } @@ -95,20 +92,20 @@ _PyCrossInterpreterData_Free(_PyCrossInterpreterData *xid) /* defining cross-interpreter data */ static inline void -_xidata_init(_PyCrossInterpreterData *data) +_xidata_init(_PyXIData_t *data) { // If the value is being reused // then _xidata_clear() should have been called already. assert(data->data == NULL); assert(data->obj == NULL); - *data = (_PyCrossInterpreterData){0}; - _PyCrossInterpreterData_INTERPID(data) = -1; + *data = (_PyXIData_t){0}; + _PyXIData_INTERPID(data) = -1; } static inline void -_xidata_clear(_PyCrossInterpreterData *data) +_xidata_clear(_PyXIData_t *data) { - // _PyCrossInterpreterData only has two members that need to be + // _PyXIData_t only has two members that need to be // cleaned up, if set: "data" must be freed and "obj" must be decref'ed. // In both cases the original (owning) interpreter must be used, // which is the caller's responsibility to ensure. @@ -122,10 +119,10 @@ _xidata_clear(_PyCrossInterpreterData *data) } void -_PyCrossInterpreterData_Init(_PyCrossInterpreterData *data, - PyInterpreterState *interp, - void *shared, PyObject *obj, - xid_newobjectfunc new_object) +_PyXIData_Init(_PyXIData_t *data, + PyInterpreterState *interp, + void *shared, PyObject *obj, + xid_newobjfunc new_object) { assert(data != NULL); assert(new_object != NULL); @@ -133,29 +130,29 @@ _PyCrossInterpreterData_Init(_PyCrossInterpreterData *data, data->data = shared; if (obj != NULL) { assert(interp != NULL); - // released in _PyCrossInterpreterData_Clear() + // released in _PyXIData_Clear() data->obj = Py_NewRef(obj); } // Ideally every object would know its owning interpreter. // Until then, we have to rely on the caller to identify it // (but we don't need it in all cases). - _PyCrossInterpreterData_INTERPID(data) = (interp != NULL) + _PyXIData_INTERPID(data) = (interp != NULL) ? PyInterpreterState_GetID(interp) : -1; data->new_object = new_object; } int -_PyCrossInterpreterData_InitWithSize(_PyCrossInterpreterData *data, - PyInterpreterState *interp, - const size_t size, PyObject *obj, - xid_newobjectfunc new_object) +_PyXIData_InitWithSize(_PyXIData_t *data, + PyInterpreterState *interp, + const size_t size, PyObject *obj, + xid_newobjfunc new_object) { assert(size > 0); // For now we always free the shared data in the same interpreter // where it was allocated, so the interpreter is required. assert(interp != NULL); - _PyCrossInterpreterData_Init(data, interp, NULL, obj, new_object); + _PyXIData_Init(data, interp, NULL, obj, new_object); data->data = PyMem_RawMalloc(size); if (data->data == NULL) { return -1; @@ -165,14 +162,13 @@ _PyCrossInterpreterData_InitWithSize(_PyCrossInterpreterData *data, } void -_PyCrossInterpreterData_Clear(PyInterpreterState *interp, - _PyCrossInterpreterData *data) +_PyXIData_Clear(PyInterpreterState *interp, _PyXIData_t *data) { assert(data != NULL); // This must be called in the owning interpreter. assert(interp == NULL - || _PyCrossInterpreterData_INTERPID(data) == -1 - || _PyCrossInterpreterData_INTERPID(data) == PyInterpreterState_GetID(interp)); + || _PyXIData_INTERPID(data) == -1 + || _PyXIData_INTERPID(data) == PyInterpreterState_GetID(interp)); _xidata_clear(data); } @@ -180,13 +176,13 @@ _PyCrossInterpreterData_Clear(PyInterpreterState *interp, /* using cross-interpreter data */ static int -_check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data) +_check_xidata(PyThreadState *tstate, _PyXIData_t *data) { // data->data can be anything, including NULL, so we don't check it. // data->obj may be NULL, so we don't check it. - if (_PyCrossInterpreterData_INTERPID(data) < 0) { + if (_PyXIData_INTERPID(data) < 0) { PyErr_SetString(PyExc_SystemError, "missing interp"); return -1; } @@ -202,10 +198,9 @@ _check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data) } static inline void -_set_xid_lookup_failure(PyInterpreterState *interp, - PyObject *obj, const char *msg) +_set_xid_lookup_failure(dlcontext_t *ctx, PyObject *obj, const char *msg) { - PyObject *exctype = _get_not_shareable_error_type(interp); + PyObject *exctype = ctx->PyExc_NotShareableError; assert(exctype != NULL); if (msg != NULL) { assert(obj == NULL); @@ -222,13 +217,12 @@ _set_xid_lookup_failure(PyInterpreterState *interp, } int -_PyObject_CheckCrossInterpreterData(PyObject *obj) +_PyObject_CheckXIData(_PyXIData_lookup_context_t *ctx, PyObject *obj) { - PyInterpreterState *interp = PyInterpreterState_Get(); - crossinterpdatafunc getdata = lookup_getdata(interp, obj); + xidatafunc getdata = lookup_getdata(ctx, obj); if (getdata == NULL) { if (!PyErr_Occurred()) { - _set_xid_lookup_failure(interp, obj, NULL); + _set_xid_lookup_failure(ctx, obj, NULL); } return -1; } @@ -236,22 +230,23 @@ _PyObject_CheckCrossInterpreterData(PyObject *obj) } int -_PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) +_PyObject_GetXIData(_PyXIData_lookup_context_t *ctx, + PyObject *obj, _PyXIData_t *data) { PyThreadState *tstate = PyThreadState_Get(); PyInterpreterState *interp = tstate->interp; // Reset data before re-populating. - *data = (_PyCrossInterpreterData){0}; - _PyCrossInterpreterData_INTERPID(data) = -1; + *data = (_PyXIData_t){0}; + _PyXIData_INTERPID(data) = -1; // Call the "getdata" func for the object. Py_INCREF(obj); - crossinterpdatafunc getdata = lookup_getdata(interp, obj); + xidatafunc getdata = lookup_getdata(ctx, obj); if (getdata == NULL) { Py_DECREF(obj); if (!PyErr_Occurred()) { - _set_xid_lookup_failure(interp, obj, NULL); + _set_xid_lookup_failure(ctx, obj, NULL); } return -1; } @@ -262,9 +257,9 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) } // Fill in the blanks and validate the result. - _PyCrossInterpreterData_INTERPID(data) = PyInterpreterState_GetID(interp); + _PyXIData_INTERPID(data) = PyInterpreterState_GetID(interp); if (_check_xidata(tstate, data) != 0) { - (void)_PyCrossInterpreterData_Release(data); + (void)_PyXIData_Release(data); return -1; } @@ -272,7 +267,7 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) } PyObject * -_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) +_PyXIData_NewObject(_PyXIData_t *data) { return data->new_object(data); } @@ -280,12 +275,12 @@ _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) static int _call_clear_xidata(void *data) { - _xidata_clear((_PyCrossInterpreterData *)data); + _xidata_clear((_PyXIData_t *)data); return 0; } static int -_xidata_release(_PyCrossInterpreterData *data, int rawfree) +_xidata_release(_PyXIData_t *data, int rawfree) { if ((data->data == NULL || data->free == NULL) && data->obj == NULL) { // Nothing to release! @@ -300,7 +295,7 @@ _xidata_release(_PyCrossInterpreterData *data, int rawfree) // Switch to the original interpreter. PyInterpreterState *interp = _PyInterpreterState_LookUpID( - _PyCrossInterpreterData_INTERPID(data)); + _PyXIData_INTERPID(data)); if (interp == NULL) { // The interpreter was already destroyed. // This function shouldn't have been called. @@ -322,13 +317,13 @@ _xidata_release(_PyCrossInterpreterData *data, int rawfree) } int -_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) +_PyXIData_Release(_PyXIData_t *data) { return _xidata_release(data, 0); } int -_PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *data) +_PyXIData_ReleaseAndRawFree(_PyXIData_t *data) { return _xidata_release(data, 1); } @@ -347,6 +342,11 @@ _copy_string_obj_raw(PyObject *strobj, Py_ssize_t *p_size) return NULL; } + if (size != (Py_ssize_t)strlen(str)) { + PyErr_SetString(PyExc_ValueError, "found embedded NULL character"); + return NULL; + } + char *copied = PyMem_RawMalloc(size+1); if (copied == NULL) { PyErr_NoMemory(); @@ -447,15 +447,15 @@ _format_TracebackException(PyObject *tbexc) static int -_release_xid_data(_PyCrossInterpreterData *data, int rawfree) +_release_xid_data(_PyXIData_t *data, int rawfree) { PyObject *exc = PyErr_GetRaisedException(); int res = rawfree - ? _PyCrossInterpreterData_Release(data) - : _PyCrossInterpreterData_ReleaseAndRawFree(data); + ? _PyXIData_Release(data) + : _PyXIData_ReleaseAndRawFree(data); if (res < 0) { /* The owning interpreter is already destroyed. */ - _PyCrossInterpreterData_Clear(NULL, data); + _PyXIData_Clear(NULL, data); // XXX Emit a warning? PyErr_Clear(); } @@ -469,7 +469,7 @@ _release_xid_data(_PyCrossInterpreterData *data, int rawfree) /***********************/ static int -_excinfo_init_type(struct _excinfo_type *info, PyObject *exc) +_excinfo_init_type_from_exception(struct _excinfo_type *info, PyObject *exc) { /* Note that this copies directly rather than into an intermediate struct and does not clear on error. If we need that then we @@ -505,21 +505,62 @@ _excinfo_init_type(struct _excinfo_type *info, PyObject *exc) } info->qualname = _copy_string_obj_raw(strobj, NULL); Py_DECREF(strobj); - if (info->name == NULL) { + if (info->qualname == NULL) { return -1; } // __module__ - strobj = _PyType_GetModuleName(type); + strobj = PyType_GetModuleName(type); if (strobj == NULL) { return -1; } info->module = _copy_string_obj_raw(strobj, NULL); Py_DECREF(strobj); + if (info->module == NULL) { + return -1; + } + + return 0; +} + +static int +_excinfo_init_type_from_object(struct _excinfo_type *info, PyObject *exctype) +{ + PyObject *strobj = NULL; + + // __name__ + strobj = PyObject_GetAttrString(exctype, "__name__"); + if (strobj == NULL) { + return -1; + } + info->name = _copy_string_obj_raw(strobj, NULL); + Py_DECREF(strobj); if (info->name == NULL) { return -1; } + // __qualname__ + strobj = PyObject_GetAttrString(exctype, "__qualname__"); + if (strobj == NULL) { + return -1; + } + info->qualname = _copy_string_obj_raw(strobj, NULL); + Py_DECREF(strobj); + if (info->qualname == NULL) { + return -1; + } + + // __module__ + strobj = PyObject_GetAttrString(exctype, "__module__"); + if (strobj == NULL) { + return -1; + } + info->module = _copy_string_obj_raw(strobj, NULL); + Py_DECREF(strobj); + if (info->module == NULL) { + return -1; + } + return 0; } @@ -585,7 +626,7 @@ _PyXI_excinfo_Clear(_PyXI_excinfo *info) *info = (_PyXI_excinfo){{NULL}}; } -static PyObject * +PyObject * _PyXI_excinfo_format(_PyXI_excinfo *info) { const char *module, *qualname; @@ -628,7 +669,7 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) } const char *failure = NULL; - if (_excinfo_init_type(&info->type, exc) < 0) { + if (_excinfo_init_type_from_exception(&info->type, exc) < 0) { failure = "error while initializing exception type snapshot"; goto error; } @@ -659,7 +700,7 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) Py_DECREF(tbexc); if (info->errdisplay == NULL) { #ifdef Py_DEBUG - PyErr_FormatUnraisable("Exception ignored while formating TracebackException"); + PyErr_FormatUnraisable("Exception ignored while formatting TracebackException"); #endif PyErr_Clear(); } @@ -673,6 +714,57 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) return failure; } +static const char * +_PyXI_excinfo_InitFromObject(_PyXI_excinfo *info, PyObject *obj) +{ + const char *failure = NULL; + + PyObject *exctype = PyObject_GetAttrString(obj, "type"); + if (exctype == NULL) { + failure = "exception snapshot missing 'type' attribute"; + goto error; + } + int res = _excinfo_init_type_from_object(&info->type, exctype); + Py_DECREF(exctype); + if (res < 0) { + failure = "error while initializing exception type snapshot"; + goto error; + } + + // Extract the exception message. + PyObject *msgobj = PyObject_GetAttrString(obj, "msg"); + if (msgobj == NULL) { + failure = "exception snapshot missing 'msg' attribute"; + goto error; + } + info->msg = _copy_string_obj_raw(msgobj, NULL); + Py_DECREF(msgobj); + if (info->msg == NULL) { + failure = "error while copying exception message"; + goto error; + } + + // Pickle a traceback.TracebackException. + PyObject *errdisplay = PyObject_GetAttrString(obj, "errdisplay"); + if (errdisplay == NULL) { + failure = "exception snapshot missing 'errdisplay' attribute"; + goto error; + } + info->errdisplay = _copy_string_obj_raw(errdisplay, NULL); + Py_DECREF(errdisplay); + if (info->errdisplay == NULL) { + failure = "error while copying exception error display"; + goto error; + } + + return NULL; + +error: + assert(failure != NULL); + _PyXI_excinfo_Clear(info); + return failure; +} + static void _PyXI_excinfo_Apply(_PyXI_excinfo *info, PyObject *exctype) { @@ -826,6 +918,47 @@ _PyXI_excinfo_AsObject(_PyXI_excinfo *info) } +int +_PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc) +{ + assert(!PyErr_Occurred()); + if (exc == NULL || exc == Py_None) { + PyErr_SetString(PyExc_ValueError, "missing exc"); + return -1; + } + const char *failure; + if (PyExceptionInstance_Check(exc) || PyExceptionClass_Check(exc)) { + failure = _PyXI_excinfo_InitFromException(info, exc); + } + else { + failure = _PyXI_excinfo_InitFromObject(info, exc); + } + if (failure != NULL) { + PyErr_SetString(PyExc_Exception, failure); + return -1; + } + return 0; +} + +PyObject * +_PyXI_FormatExcInfo(_PyXI_excinfo *info) +{ + return _PyXI_excinfo_format(info); +} + +PyObject * +_PyXI_ExcInfoAsObject(_PyXI_excinfo *info) +{ + return _PyXI_excinfo_AsObject(info); +} + +void +_PyXI_ClearExcInfo(_PyXI_excinfo *info) +{ + _PyXI_excinfo_Clear(info); +} + + /***************************/ /* short-term data sharing */ /***************************/ @@ -835,9 +968,11 @@ _PyXI_excinfo_AsObject(_PyXI_excinfo *info) static int _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp) { + dlcontext_t ctx; + assert(!PyErr_Occurred()); switch (code) { - case _PyXI_ERR_NO_ERROR: // fall through + case _PyXI_ERR_NO_ERROR: _Py_FALLTHROUGH; case _PyXI_ERR_UNCAUGHT_EXCEPTION: // There is nothing to apply. #ifdef Py_DEBUG @@ -846,26 +981,28 @@ _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp) return 0; case _PyXI_ERR_OTHER: // XXX msg? - PyErr_SetNone(PyExc_RuntimeError); + PyErr_SetNone(PyExc_InterpreterError); break; case _PyXI_ERR_NO_MEMORY: PyErr_NoMemory(); break; case _PyXI_ERR_ALREADY_RUNNING: assert(interp != NULL); - assert(_PyInterpreterState_IsRunningMain(interp)); - _PyInterpreterState_FailIfRunningMain(interp); + _PyErr_SetInterpreterAlreadyRunning(); break; case _PyXI_ERR_MAIN_NS_FAILURE: - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_InterpreterError, "failed to get __main__ namespace"); break; case _PyXI_ERR_APPLY_NS_FAILURE: - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_InterpreterError, "failed to apply namespace to __main__"); break; case _PyXI_ERR_NOT_SHAREABLE: - _set_xid_lookup_failure(interp, NULL, NULL); + if (_PyXIData_GetLookupContext(interp, &ctx) < 0) { + return -1; + } + _set_xid_lookup_failure(&ctx, NULL, NULL); break; default: #ifdef Py_DEBUG @@ -927,7 +1064,11 @@ _PyXI_ApplyError(_PyXI_error *error) } else if (error->code == _PyXI_ERR_NOT_SHAREABLE) { // Propagate the exception directly. - _set_xid_lookup_failure(error->interp, NULL, error->uncaught.msg); + dlcontext_t ctx; + if (_PyXIData_GetLookupContext(error->interp, &ctx) < 0) { + return NULL; + } + _set_xid_lookup_failure(&ctx, NULL, error->uncaught.msg); } else { // Raise an exception corresponding to the code. @@ -936,7 +1077,7 @@ _PyXI_ApplyError(_PyXI_error *error) if (error->uncaught.type.name != NULL || error->uncaught.msg != NULL) { // __context__ will be set to a proxy of the propagated exception. PyObject *exc = PyErr_GetRaisedException(); - _PyXI_excinfo_Apply(&error->uncaught, PyExc_RuntimeError); + _PyXI_excinfo_Apply(&error->uncaught, PyExc_InterpreterError); PyObject *exc2 = PyErr_GetRaisedException(); PyException_SetContext(exc, exc2); PyErr_SetRaisedException(exc); @@ -962,8 +1103,8 @@ _PyXI_ApplyError(_PyXI_error *error) typedef struct _sharednsitem { const char *name; - _PyCrossInterpreterData *data; - // We could have a "PyCrossInterpreterData _data" field, so it would + _PyXIData_t *data; + // We could have a "PyXIData _data" field, so it would // be allocated as part of the item and avoid an extra allocation. // However, doing so adds a bunch of complexity because we must // ensure the item isn't freed before a pending call might happen @@ -999,7 +1140,7 @@ _sharednsitem_has_value(_PyXI_namespace_item *item, int64_t *p_interpid) return 0; } if (p_interpid != NULL) { - *p_interpid = _PyCrossInterpreterData_INTERPID(item->data); + *p_interpid = _PyXIData_INTERPID(item->data); } return 1; } @@ -1009,12 +1150,17 @@ _sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value) { assert(_sharednsitem_is_initialized(item)); assert(item->data == NULL); - item->data = PyMem_RawMalloc(sizeof(_PyCrossInterpreterData)); + item->data = PyMem_RawMalloc(sizeof(_PyXIData_t)); if (item->data == NULL) { PyErr_NoMemory(); return -1; } - if (_PyObject_GetCrossInterpreterData(value, item->data) != 0) { + PyInterpreterState *interp = PyInterpreterState_Get(); + dlcontext_t ctx; + if (_PyXIData_GetLookupContext(interp, &ctx) < 0) { + return -1; + } + if (_PyObject_GetXIData(&ctx, value, item->data) != 0) { PyMem_RawFree(item->data); item->data = NULL; // The caller may want to propagate PyExc_NotShareableError @@ -1027,7 +1173,7 @@ _sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value) static void _sharednsitem_clear_value(_PyXI_namespace_item *item) { - _PyCrossInterpreterData *data = item->data; + _PyXIData_t *data = item->data; if (data != NULL) { item->data = NULL; int rawfree = 1; @@ -1073,7 +1219,7 @@ _sharednsitem_apply(_PyXI_namespace_item *item, PyObject *ns, PyObject *dflt) } PyObject *value; if (item->data != NULL) { - value = _PyCrossInterpreterData_NewObject(item->data); + value = _PyXIData_NewObject(item->data); if (value == NULL) { Py_DECREF(name); return -1; @@ -1412,8 +1558,7 @@ _enter_session(_PyXI_session *session, PyInterpreterState *interp) PyThreadState *tstate = PyThreadState_Get(); PyThreadState *prev = tstate; if (interp != tstate->interp) { - tstate = PyThreadState_New(interp); - _PyThreadState_SetWhence(tstate, _PyThreadState_WHENCE_EXEC); + tstate = _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_EXEC); // XXX Possible GILState issues? session->prev_tstate = PyThreadState_Swap(tstate); assert(session->prev_tstate == prev); @@ -1473,7 +1618,13 @@ _propagate_not_shareable_error(_PyXI_session *session) return; } PyInterpreterState *interp = PyInterpreterState_Get(); - if (PyErr_ExceptionMatches(_get_not_shareable_error_type(interp))) { + dlcontext_t ctx; + if (_PyXIData_GetLookupContext(interp, &ctx) < 0) { + PyErr_FormatUnraisable( + "Exception ignored while propagating not shareable error"); + return; + } + if (PyErr_ExceptionMatches(ctx.PyExc_NotShareableError)) { // We want to propagate the exception directly. session->_error_override = _PyXI_ERR_NOT_SHAREABLE; session->error_override = &session->_error_override; @@ -1641,15 +1792,87 @@ _PyXI_Exit(_PyXI_session *session) /* runtime lifecycle */ /*********************/ +int +_Py_xi_global_state_init(_PyXI_global_state_t *state) +{ + assert(state != NULL); + xid_lookup_init(&state->data_lookup); + return 0; +} + +void +_Py_xi_global_state_fini(_PyXI_global_state_t *state) +{ + assert(state != NULL); + xid_lookup_fini(&state->data_lookup); +} + +int +_Py_xi_state_init(_PyXI_state_t *state, PyInterpreterState *interp) +{ + assert(state != NULL); + assert(interp == NULL || state == _PyXI_GET_STATE(interp)); + + xid_lookup_init(&state->data_lookup); + + // Initialize exceptions. + if (interp != NULL) { + if (init_static_exctypes(&state->exceptions, interp) < 0) { + fini_heap_exctypes(&state->exceptions); + return -1; + } + } + if (init_heap_exctypes(&state->exceptions) < 0) { + return -1; + } + + return 0; +} + +void +_Py_xi_state_fini(_PyXI_state_t *state, PyInterpreterState *interp) +{ + assert(state != NULL); + assert(interp == NULL || state == _PyXI_GET_STATE(interp)); + + fini_heap_exctypes(&state->exceptions); + if (interp != NULL) { + fini_static_exctypes(&state->exceptions, interp); + } + + xid_lookup_fini(&state->data_lookup); +} + + PyStatus _PyXI_Init(PyInterpreterState *interp) { - // Initialize the XID lookup state (e.g. registry). - xid_lookup_init(interp); + if (_Py_IsMainInterpreter(interp)) { + _PyXI_global_state_t *global_state = _PyXI_GET_GLOBAL_STATE(interp); + if (global_state == NULL) { + PyErr_PrintEx(0); + return _PyStatus_ERR( + "failed to get global cross-interpreter state"); + } + if (_Py_xi_global_state_init(global_state) < 0) { + PyErr_PrintEx(0); + return _PyStatus_ERR( + "failed to initialize global cross-interpreter state"); + } + } - // Initialize exceptions (heap types). - if (_init_not_shareable_error_type(interp) < 0) { - return _PyStatus_ERR("failed to initialize NotShareableError"); + _PyXI_state_t *state = _PyXI_GET_STATE(interp); + if (state == NULL) { + PyErr_PrintEx(0); + return _PyStatus_ERR( + "failed to get interpreter's cross-interpreter state"); + } + // The static types were already initialized in _PyXI_InitTypes(), + // so we pass in NULL here to avoid initializing them again. + if (_Py_xi_state_init(state, NULL) < 0) { + PyErr_PrintEx(0); + return _PyStatus_ERR( + "failed to initialize interpreter's cross-interpreter state"); } return _PyStatus_OK(); @@ -1661,24 +1884,140 @@ _PyXI_Init(PyInterpreterState *interp) void _PyXI_Fini(PyInterpreterState *interp) { - // Finalize exceptions (heap types). - _fini_not_shareable_error_type(interp); + _PyXI_state_t *state = _PyXI_GET_STATE(interp); +#ifndef NDEBUG + if (state == NULL) { + PyErr_PrintEx(0); + return; + } +#endif + // The static types will be finalized soon in _PyXI_FiniTypes(), + // so we pass in NULL here to avoid finalizing them right now. + _Py_xi_state_fini(state, NULL); - // Finalize the XID lookup state (e.g. registry). - xid_lookup_fini(interp); + if (_Py_IsMainInterpreter(interp)) { + _PyXI_global_state_t *global_state = _PyXI_GET_GLOBAL_STATE(interp); + _Py_xi_global_state_fini(global_state); + } } PyStatus _PyXI_InitTypes(PyInterpreterState *interp) { - if (init_exceptions(interp) < 0) { - return _PyStatus_ERR("failed to initialize an exception type"); + if (init_static_exctypes(&_PyXI_GET_STATE(interp)->exceptions, interp) < 0) { + PyErr_PrintEx(0); + return _PyStatus_ERR( + "failed to initialize the cross-interpreter exception types"); } + // We would initialize heap types here too but that leads to ref leaks. + // Instead, we intialize them in _PyXI_Init(). return _PyStatus_OK(); } void _PyXI_FiniTypes(PyInterpreterState *interp) { - fini_exceptions(interp); + // We would finalize heap types here too but that leads to ref leaks. + // Instead, we finalize them in _PyXI_Fini(). + fini_static_exctypes(&_PyXI_GET_STATE(interp)->exceptions, interp); +} + + +/*************/ +/* other API */ +/*************/ + +PyInterpreterState * +_PyXI_NewInterpreter(PyInterpreterConfig *config, long *maybe_whence, + PyThreadState **p_tstate, PyThreadState **p_save_tstate) +{ + PyThreadState *save_tstate = PyThreadState_Swap(NULL); + assert(save_tstate != NULL); + + PyThreadState *tstate; + PyStatus status = Py_NewInterpreterFromConfig(&tstate, config); + if (PyStatus_Exception(status)) { + // Since no new thread state was created, there is no exception + // to propagate; raise a fresh one after swapping back in the + // old thread state. + PyThreadState_Swap(save_tstate); + _PyErr_SetFromPyStatus(status); + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_InterpreterError, + "sub-interpreter creation failed"); + _PyErr_ChainExceptions1(exc); + return NULL; + } + assert(tstate != NULL); + PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); + + long whence = _PyInterpreterState_WHENCE_XI; + if (maybe_whence != NULL) { + whence = *maybe_whence; + } + _PyInterpreterState_SetWhence(interp, whence); + + if (p_tstate != NULL) { + // We leave the new thread state as the current one. + *p_tstate = tstate; + } + else { + // Throw away the initial tstate. + PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); + save_tstate = NULL; + } + if (p_save_tstate != NULL) { + *p_save_tstate = save_tstate; + } + return interp; +} + +void +_PyXI_EndInterpreter(PyInterpreterState *interp, + PyThreadState *tstate, PyThreadState **p_save_tstate) +{ +#ifndef NDEBUG + long whence = _PyInterpreterState_GetWhence(interp); +#endif + assert(whence != _PyInterpreterState_WHENCE_RUNTIME); + + if (!_PyInterpreterState_IsReady(interp)) { + assert(whence == _PyInterpreterState_WHENCE_UNKNOWN); + // PyInterpreterState_Clear() requires the GIL, + // which a not-ready does not have, so we don't clear it. + // That means there may be leaks here until clearing the + // interpreter is fixed. + PyInterpreterState_Delete(interp); + return; + } + assert(whence != _PyInterpreterState_WHENCE_UNKNOWN); + + PyThreadState *save_tstate = NULL; + PyThreadState *cur_tstate = PyThreadState_GET(); + if (tstate == NULL) { + if (PyThreadState_GetInterpreter(cur_tstate) == interp) { + tstate = cur_tstate; + } + else { + tstate = _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_FINI); + assert(tstate != NULL); + save_tstate = PyThreadState_Swap(tstate); + } + } + else { + assert(PyThreadState_GetInterpreter(tstate) == interp); + if (tstate != cur_tstate) { + assert(PyThreadState_GetInterpreter(cur_tstate) != interp); + save_tstate = PyThreadState_Swap(tstate); + } + } + + Py_EndInterpreter(tstate); + + if (p_save_tstate != NULL) { + save_tstate = *p_save_tstate; + } + PyThreadState_Swap(save_tstate); } diff --git a/Python/crossinterp_data_lookup.h b/Python/crossinterp_data_lookup.h index 863919ad42fb97..48e5d9762cd697 100644 --- a/Python/crossinterp_data_lookup.h +++ b/Python/crossinterp_data_lookup.h @@ -1,39 +1,87 @@ +#include "pycore_weakref.h" // _PyWeakref_GET_REF() -static crossinterpdatafunc _lookup_getdata_from_registry( - PyInterpreterState *, PyObject *); -static crossinterpdatafunc -lookup_getdata(PyInterpreterState *interp, PyObject *obj) +typedef _PyXIData_lookup_context_t dlcontext_t; +typedef _PyXIData_registry_t dlregistry_t; +typedef _PyXIData_regitem_t dlregitem_t; + + +// forward +static void _xidregistry_init(dlregistry_t *); +static void _xidregistry_fini(dlregistry_t *); +static xidatafunc _lookup_getdata_from_registry(dlcontext_t *, PyObject *); + + +/* used in crossinterp.c */ + +static void +xid_lookup_init(_PyXIData_lookup_t *state) +{ + _xidregistry_init(&state->registry); +} + +static void +xid_lookup_fini(_PyXIData_lookup_t *state) +{ + _xidregistry_fini(&state->registry); +} + +static xidatafunc +lookup_getdata(dlcontext_t *ctx, PyObject *obj) { /* Cross-interpreter objects are looked up by exact match on the class. We can reassess this policy when we move from a global registry to a tp_* slot. */ - return _lookup_getdata_from_registry(interp, obj); + return _lookup_getdata_from_registry(ctx, obj); } -crossinterpdatafunc -_PyCrossInterpreterData_Lookup(PyObject *obj) + +/* exported API */ + +int +_PyXIData_GetLookupContext(PyInterpreterState *interp, + _PyXIData_lookup_context_t *res) { - PyInterpreterState *interp = PyInterpreterState_Get(); - return lookup_getdata(interp, obj); + _PyXI_global_state_t *global = _PyXI_GET_GLOBAL_STATE(interp); + if (global == NULL) { + assert(PyErr_Occurred()); + return -1; + } + _PyXI_state_t *local = _PyXI_GET_STATE(interp); + if (local == NULL) { + assert(PyErr_Occurred()); + return -1; + } + *res = (dlcontext_t){ + .global = &global->data_lookup, + .local = &local->data_lookup, + .PyExc_NotShareableError = local->exceptions.PyExc_NotShareableError, + }; + return 0; +} + +xidatafunc +_PyXIData_Lookup(_PyXIData_lookup_context_t *ctx, PyObject *obj) +{ + return lookup_getdata(ctx, obj); } /***********************************************/ -/* a registry of {type -> crossinterpdatafunc} */ +/* a registry of {type -> xidatafunc} */ /***********************************************/ /* For now we use a global registry of shareable classes. An alternative would be to add a tp_* slot for a class's - crossinterpdatafunc. It would be simpler and more efficient. */ + xidatafunc. It would be simpler and more efficient. */ /* registry lifecycle */ -static void _register_builtins_for_crossinterpreter_data(struct _xidregistry *); +static void _register_builtins_for_crossinterpreter_data(dlregistry_t *); static void -_xidregistry_init(struct _xidregistry *registry) +_xidregistry_init(dlregistry_t *registry) { if (registry->initialized) { return; @@ -47,10 +95,10 @@ _xidregistry_init(struct _xidregistry *registry) } } -static void _xidregistry_clear(struct _xidregistry *); +static void _xidregistry_clear(dlregistry_t *); static void -_xidregistry_fini(struct _xidregistry *registry) +_xidregistry_fini(dlregistry_t *registry) { if (!registry->initialized) { return; @@ -60,32 +108,11 @@ _xidregistry_fini(struct _xidregistry *registry) _xidregistry_clear(registry); } -static inline struct _xidregistry * _get_global_xidregistry(_PyRuntimeState *); -static inline struct _xidregistry * _get_xidregistry(PyInterpreterState *); - -static void -xid_lookup_init(PyInterpreterState *interp) -{ - if (_Py_IsMainInterpreter(interp)) { - _xidregistry_init(_get_global_xidregistry(interp->runtime)); - } - _xidregistry_init(_get_xidregistry(interp)); -} - -static void -xid_lookup_fini(PyInterpreterState *interp) -{ - _xidregistry_fini(_get_xidregistry(interp)); - if (_Py_IsMainInterpreter(interp)) { - _xidregistry_fini(_get_global_xidregistry(interp->runtime)); - } -} - /* registry thread safety */ static void -_xidregistry_lock(struct _xidregistry *registry) +_xidregistry_lock(dlregistry_t *registry) { if (registry->global) { PyMutex_Lock(®istry->mutex); @@ -94,7 +121,7 @@ _xidregistry_lock(struct _xidregistry *registry) } static void -_xidregistry_unlock(struct _xidregistry *registry) +_xidregistry_unlock(dlregistry_t *registry) { if (registry->global) { PyMutex_Unlock(®istry->mutex); @@ -104,35 +131,21 @@ _xidregistry_unlock(struct _xidregistry *registry) /* accessing the registry */ -static inline struct _xidregistry * -_get_global_xidregistry(_PyRuntimeState *runtime) -{ - return &runtime->xi.registry; -} - -static inline struct _xidregistry * -_get_xidregistry(PyInterpreterState *interp) +static inline dlregistry_t * +_get_xidregistry_for_type(dlcontext_t *ctx, PyTypeObject *cls) { - return &interp->xi.registry; -} - -static inline struct _xidregistry * -_get_xidregistry_for_type(PyInterpreterState *interp, PyTypeObject *cls) -{ - struct _xidregistry *registry = _get_global_xidregistry(interp->runtime); if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { - registry = _get_xidregistry(interp); + return &ctx->local->registry; } - return registry; + return &ctx->global->registry; } -static struct _xidregitem * _xidregistry_remove_entry( - struct _xidregistry *, struct _xidregitem *); +static dlregitem_t* _xidregistry_remove_entry(dlregistry_t *, dlregitem_t *); -static struct _xidregitem * -_xidregistry_find_type(struct _xidregistry *xidregistry, PyTypeObject *cls) +static dlregitem_t * +_xidregistry_find_type(dlregistry_t *xidregistry, PyTypeObject *cls) { - struct _xidregitem *cur = xidregistry->head; + dlregitem_t *cur = xidregistry->head; while (cur != NULL) { if (cur->weakref != NULL) { // cur is/was a heap type. @@ -155,16 +168,16 @@ _xidregistry_find_type(struct _xidregistry *xidregistry, PyTypeObject *cls) return NULL; } -static crossinterpdatafunc -_lookup_getdata_from_registry(PyInterpreterState *interp, PyObject *obj) +static xidatafunc +_lookup_getdata_from_registry(dlcontext_t *ctx, PyObject *obj) { PyTypeObject *cls = Py_TYPE(obj); - struct _xidregistry *xidregistry = _get_xidregistry_for_type(interp, cls); + dlregistry_t *xidregistry = _get_xidregistry_for_type(ctx, cls); _xidregistry_lock(xidregistry); - struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); - crossinterpdatafunc func = matched != NULL ? matched->getdata : NULL; + dlregitem_t *matched = _xidregistry_find_type(xidregistry, cls); + xidatafunc func = matched != NULL ? matched->getdata : NULL; _xidregistry_unlock(xidregistry); return func; @@ -174,14 +187,14 @@ _lookup_getdata_from_registry(PyInterpreterState *interp, PyObject *obj) /* updating the registry */ static int -_xidregistry_add_type(struct _xidregistry *xidregistry, - PyTypeObject *cls, crossinterpdatafunc getdata) +_xidregistry_add_type(dlregistry_t *xidregistry, + PyTypeObject *cls, xidatafunc getdata) { - struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem)); + dlregitem_t *newhead = PyMem_RawMalloc(sizeof(dlregitem_t)); if (newhead == NULL) { return -1; } - *newhead = (struct _xidregitem){ + *newhead = (dlregitem_t){ // We do not keep a reference, to avoid keeping the class alive. .cls = cls, .refcount = 1, @@ -203,11 +216,10 @@ _xidregistry_add_type(struct _xidregistry *xidregistry, return 0; } -static struct _xidregitem * -_xidregistry_remove_entry(struct _xidregistry *xidregistry, - struct _xidregitem *entry) +static dlregitem_t * +_xidregistry_remove_entry(dlregistry_t *xidregistry, dlregitem_t *entry) { - struct _xidregitem *next = entry->next; + dlregitem_t *next = entry->next; if (entry->prev != NULL) { assert(entry->prev->next == entry); entry->prev->next = next; @@ -225,12 +237,12 @@ _xidregistry_remove_entry(struct _xidregistry *xidregistry, } static void -_xidregistry_clear(struct _xidregistry *xidregistry) +_xidregistry_clear(dlregistry_t *xidregistry) { - struct _xidregitem *cur = xidregistry->head; + dlregitem_t *cur = xidregistry->head; xidregistry->head = NULL; while (cur != NULL) { - struct _xidregitem *next = cur->next; + dlregitem_t *next = cur->next; Py_XDECREF(cur->weakref); PyMem_RawFree(cur); cur = next; @@ -238,8 +250,8 @@ _xidregistry_clear(struct _xidregistry *xidregistry) } int -_PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, - crossinterpdatafunc getdata) +_PyXIData_RegisterClass(_PyXIData_lookup_context_t *ctx, + PyTypeObject *cls, xidatafunc getdata) { if (!PyType_Check(cls)) { PyErr_Format(PyExc_ValueError, "only classes may be registered"); @@ -251,11 +263,10 @@ _PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, } int res = 0; - PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _xidregistry *xidregistry = _get_xidregistry_for_type(interp, cls); + dlregistry_t *xidregistry = _get_xidregistry_for_type(ctx, cls); _xidregistry_lock(xidregistry); - struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + dlregitem_t *matched = _xidregistry_find_type(xidregistry, cls); if (matched != NULL) { assert(matched->getdata == getdata); matched->refcount += 1; @@ -270,14 +281,13 @@ _PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, } int -_PyCrossInterpreterData_UnregisterClass(PyTypeObject *cls) +_PyXIData_UnregisterClass(_PyXIData_lookup_context_t *ctx, PyTypeObject *cls) { int res = 0; - PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _xidregistry *xidregistry = _get_xidregistry_for_type(interp, cls); + dlregistry_t *xidregistry = _get_xidregistry_for_type(ctx, cls); _xidregistry_lock(xidregistry); - struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + dlregitem_t *matched = _xidregistry_find_type(xidregistry, cls); if (matched != NULL) { assert(matched->refcount > 0); matched->refcount -= 1; @@ -304,17 +314,16 @@ struct _shared_bytes_data { }; static PyObject * -_new_bytes_object(_PyCrossInterpreterData *data) +_new_bytes_object(_PyXIData_t *data) { struct _shared_bytes_data *shared = (struct _shared_bytes_data *)(data->data); return PyBytes_FromStringAndSize(shared->bytes, shared->len); } static int -_bytes_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) +_bytes_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { - if (_PyCrossInterpreterData_InitWithSize( + if (_PyXIData_InitWithSize( data, tstate->interp, sizeof(struct _shared_bytes_data), obj, _new_bytes_object ) < 0) @@ -323,7 +332,7 @@ _bytes_shared(PyThreadState *tstate, PyObject *obj, } struct _shared_bytes_data *shared = (struct _shared_bytes_data *)data->data; if (PyBytes_AsStringAndSize(obj, &shared->bytes, &shared->len) < 0) { - _PyCrossInterpreterData_Clear(tstate->interp, data); + _PyXIData_Clear(tstate->interp, data); return -1; } return 0; @@ -338,17 +347,16 @@ struct _shared_str_data { }; static PyObject * -_new_str_object(_PyCrossInterpreterData *data) +_new_str_object(_PyXIData_t *data) { struct _shared_str_data *shared = (struct _shared_str_data *)(data->data); return PyUnicode_FromKindAndData(shared->kind, shared->buffer, shared->len); } static int -_str_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) +_str_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { - if (_PyCrossInterpreterData_InitWithSize( + if (_PyXIData_InitWithSize( data, tstate->interp, sizeof(struct _shared_str_data), obj, _new_str_object ) < 0) @@ -365,14 +373,13 @@ _str_shared(PyThreadState *tstate, PyObject *obj, // int static PyObject * -_new_long_object(_PyCrossInterpreterData *data) +_new_long_object(_PyXIData_t *data) { return PyLong_FromSsize_t((Py_ssize_t)(data->data)); } static int -_long_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) +_long_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { /* Note that this means the size of shareable ints is bounded by * sys.maxsize. Hence on 32-bit architectures that is half the @@ -385,8 +392,7 @@ _long_shared(PyThreadState *tstate, PyObject *obj, } return -1; } - _PyCrossInterpreterData_Init(data, tstate->interp, (void *)value, NULL, - _new_long_object); + _PyXIData_Init(data, tstate->interp, (void *)value, NULL, _new_long_object); // data->obj and data->free remain NULL return 0; } @@ -394,17 +400,16 @@ _long_shared(PyThreadState *tstate, PyObject *obj, // float static PyObject * -_new_float_object(_PyCrossInterpreterData *data) +_new_float_object(_PyXIData_t *data) { double * value_ptr = data->data; return PyFloat_FromDouble(*value_ptr); } static int -_float_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) +_float_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { - if (_PyCrossInterpreterData_InitWithSize( + if (_PyXIData_InitWithSize( data, tstate->interp, sizeof(double), NULL, _new_float_object ) < 0) @@ -419,18 +424,16 @@ _float_shared(PyThreadState *tstate, PyObject *obj, // None static PyObject * -_new_none_object(_PyCrossInterpreterData *data) +_new_none_object(_PyXIData_t *data) { // XXX Singleton refcounts are problematic across interpreters... return Py_NewRef(Py_None); } static int -_none_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) +_none_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { - _PyCrossInterpreterData_Init(data, tstate->interp, NULL, NULL, - _new_none_object); + _PyXIData_Init(data, tstate->interp, NULL, NULL, _new_none_object); // data->data, data->obj and data->free remain NULL return 0; } @@ -438,7 +441,7 @@ _none_shared(PyThreadState *tstate, PyObject *obj, // bool static PyObject * -_new_bool_object(_PyCrossInterpreterData *data) +_new_bool_object(_PyXIData_t *data) { if (data->data){ Py_RETURN_TRUE; @@ -447,10 +450,9 @@ _new_bool_object(_PyCrossInterpreterData *data) } static int -_bool_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) +_bool_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { - _PyCrossInterpreterData_Init(data, tstate->interp, + _PyXIData_Init(data, tstate->interp, (void *) (Py_IsTrue(obj) ? (uintptr_t) 1 : (uintptr_t) 0), NULL, _new_bool_object); // data->obj and data->free remain NULL @@ -461,11 +463,11 @@ _bool_shared(PyThreadState *tstate, PyObject *obj, struct _shared_tuple_data { Py_ssize_t len; - _PyCrossInterpreterData **data; + _PyXIData_t **data; }; static PyObject * -_new_tuple_object(_PyCrossInterpreterData *data) +_new_tuple_object(_PyXIData_t *data) { struct _shared_tuple_data *shared = (struct _shared_tuple_data *)(data->data); PyObject *tuple = PyTuple_New(shared->len); @@ -474,7 +476,7 @@ _new_tuple_object(_PyCrossInterpreterData *data) } for (Py_ssize_t i = 0; i < shared->len; i++) { - PyObject *item = _PyCrossInterpreterData_NewObject(shared->data[i]); + PyObject *item = _PyXIData_NewObject(shared->data[i]); if (item == NULL){ Py_DECREF(tuple); return NULL; @@ -493,8 +495,8 @@ _tuple_shared_free(void* data) #endif for (Py_ssize_t i = 0; i < shared->len; i++) { if (shared->data[i] != NULL) { - assert(_PyCrossInterpreterData_INTERPID(shared->data[i]) == interpid); - _PyCrossInterpreterData_Release(shared->data[i]); + assert(_PyXIData_INTERPID(shared->data[i]) == interpid); + _PyXIData_Release(shared->data[i]); PyMem_RawFree(shared->data[i]); shared->data[i] = NULL; } @@ -504,9 +506,13 @@ _tuple_shared_free(void* data) } static int -_tuple_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) +_tuple_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { + dlcontext_t ctx; + if (_PyXIData_GetLookupContext(tstate->interp, &ctx) < 0) { + return -1; + } + Py_ssize_t len = PyTuple_GET_SIZE(obj); if (len < 0) { return -1; @@ -518,14 +524,14 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, } shared->len = len; - shared->data = (_PyCrossInterpreterData **) PyMem_Calloc(shared->len, sizeof(_PyCrossInterpreterData *)); + shared->data = (_PyXIData_t **) PyMem_Calloc(shared->len, sizeof(_PyXIData_t *)); if (shared->data == NULL) { PyErr_NoMemory(); return -1; } for (Py_ssize_t i = 0; i < shared->len; i++) { - _PyCrossInterpreterData *data = _PyCrossInterpreterData_New(); + _PyXIData_t *data = _PyXIData_New(); if (data == NULL) { goto error; // PyErr_NoMemory already set } @@ -533,7 +539,7 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, int res = -1; if (!_Py_EnterRecursiveCallTstate(tstate, " while sharing a tuple")) { - res = _PyObject_GetCrossInterpreterData(item, data); + res = _PyObject_GetXIData(&ctx, item, data); _Py_LeaveRecursiveCallTstate(tstate); } if (res < 0) { @@ -542,8 +548,7 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, } shared->data[i] = data; } - _PyCrossInterpreterData_Init( - data, tstate->interp, shared, obj, _new_tuple_object); + _PyXIData_Init(data, tstate->interp, shared, obj, _new_tuple_object); data->free = _tuple_shared_free; return 0; @@ -555,7 +560,7 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, // registration static void -_register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry) +_register_builtins_for_crossinterpreter_data(dlregistry_t *xidregistry) { // None if (_xidregistry_add_type(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { diff --git a/Python/crossinterp_exceptions.h b/Python/crossinterp_exceptions.h index e418cf91d4a7af..3cb45d2067710b 100644 --- a/Python/crossinterp_exceptions.h +++ b/Python/crossinterp_exceptions.h @@ -5,7 +5,10 @@ static PyTypeObject _PyExc_InterpreterError = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "interpreters.InterpreterError", .tp_doc = PyDoc_STR("A cross-interpreter operation failed"), - //.tp_base = (PyTypeObject *)PyExc_BaseException, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + //.tp_traverse = ((PyTypeObject *)PyExc_Exception)->tp_traverse, + //.tp_clear = ((PyTypeObject *)PyExc_Exception)->tp_clear, + //.tp_base = (PyTypeObject *)PyExc_Exception, }; PyObject *PyExc_InterpreterError = (PyObject *)&_PyExc_InterpreterError; @@ -15,66 +18,85 @@ static PyTypeObject _PyExc_InterpreterNotFoundError = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "interpreters.InterpreterNotFoundError", .tp_doc = PyDoc_STR("An interpreter was not found"), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + //.tp_traverse = ((PyTypeObject *)PyExc_Exception)->tp_traverse, + //.tp_clear = ((PyTypeObject *)PyExc_Exception)->tp_clear, .tp_base = &_PyExc_InterpreterError, }; PyObject *PyExc_InterpreterNotFoundError = (PyObject *)&_PyExc_InterpreterNotFoundError; -/* NotShareableError extends ValueError */ + +/* lifecycle */ static int -_init_not_shareable_error_type(PyInterpreterState *interp) +init_static_exctypes(exceptions_t *state, PyInterpreterState *interp) { - const char *name = "interpreters.NotShareableError"; - PyObject *base = PyExc_ValueError; - PyObject *ns = NULL; - PyObject *exctype = PyErr_NewException(name, base, ns); - if (exctype == NULL) { - return -1; + assert(state == &_PyXI_GET_STATE(interp)->exceptions); + PyTypeObject *base = (PyTypeObject *)PyExc_Exception; + + // PyExc_InterpreterError + _PyExc_InterpreterError.tp_base = base; + _PyExc_InterpreterError.tp_traverse = base->tp_traverse; + _PyExc_InterpreterError.tp_clear = base->tp_clear; + if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterError) < 0) { + goto error; } + state->PyExc_InterpreterError = (PyObject *)&_PyExc_InterpreterError; + + // PyExc_InterpreterNotFoundError + _PyExc_InterpreterNotFoundError.tp_traverse = base->tp_traverse; + _PyExc_InterpreterNotFoundError.tp_clear = base->tp_clear; + if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterNotFoundError) < 0) { + goto error; + } + state->PyExc_InterpreterNotFoundError = + (PyObject *)&_PyExc_InterpreterNotFoundError; - _PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError = exctype; return 0; -} -static void -_fini_not_shareable_error_type(PyInterpreterState *interp) -{ - Py_CLEAR(_PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError); +error: + fini_static_exctypes(state, interp); + return -1; } -static PyObject * -_get_not_shareable_error_type(PyInterpreterState *interp) +static void +fini_static_exctypes(exceptions_t *state, PyInterpreterState *interp) { - assert(_PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError != NULL); - return _PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError; + assert(state == &_PyXI_GET_STATE(interp)->exceptions); + if (state->PyExc_InterpreterNotFoundError != NULL) { + state->PyExc_InterpreterNotFoundError = NULL; + _PyStaticType_FiniBuiltin(interp, &_PyExc_InterpreterNotFoundError); + } + if (state->PyExc_InterpreterError != NULL) { + state->PyExc_InterpreterError = NULL; + _PyStaticType_FiniBuiltin(interp, &_PyExc_InterpreterError); + } } - -/* lifecycle */ - static int -init_exceptions(PyInterpreterState *interp) +init_heap_exctypes(exceptions_t *state) { - // builtin static types - _PyExc_InterpreterError.tp_base = (PyTypeObject *)PyExc_BaseException; - if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterError) < 0) { - return -1; - } - if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterNotFoundError) < 0) { - return -1; - } + PyObject *exctype; - // heap types - // We would call _init_not_shareable_error_type() here too, - // but that leads to ref leaks + /* NotShareableError extends ValueError */ + const char *name = "interpreters.NotShareableError"; + PyObject *base = PyExc_ValueError; + PyObject *ns = NULL; + exctype = PyErr_NewException(name, base, ns); + if (exctype == NULL) { + goto error; + } + state->PyExc_NotShareableError = exctype; return 0; + +error: + fini_heap_exctypes(state); + return -1; } static void -fini_exceptions(PyInterpreterState *interp) +fini_heap_exctypes(exceptions_t *state) { - // Likewise with _fini_not_shareable_error_type(). - _PyStaticType_Dealloc(interp, &_PyExc_InterpreterNotFoundError); - _PyStaticType_Dealloc(interp, &_PyExc_InterpreterError); + Py_CLEAR(state->PyExc_NotShareableError); } diff --git a/Python/deepfreeze/README.txt b/Python/deepfreeze/README.txt deleted file mode 100644 index 276ab51143ab33..00000000000000 --- a/Python/deepfreeze/README.txt +++ /dev/null @@ -1,6 +0,0 @@ -This directory contains the generated .c files for all the deep-frozen -modules. Python/frozen.c depends on these files. - -None of these files are committed into the repo. - -See Tools/build/freeze_modules.py for more info. diff --git a/Python/dtoa.c b/Python/dtoa.c index 6e3162f80bdae1..d0c89b2b468f75 100644 --- a/Python/dtoa.c +++ b/Python/dtoa.c @@ -72,7 +72,7 @@ /* Please send bug reports for the original dtoa.c code to David M. Gay (dmg * at acm dot org, with " at " changed at "@" and " dot " changed to "."). * Please report bugs for this modified version using the Python issue tracker - * (http://bugs.python.org). */ + * as detailed at (https://devguide.python.org/triage/issue-tracker/). */ /* On a machine with IEEE extended-precision registers, it is * necessary to specify double-precision (53-bit) rounding precision @@ -1405,7 +1405,7 @@ _Py_dg_strtod(const char *s00, char **se) switch (c) { case '-': sign = 1; - /* fall through */ + _Py_FALLTHROUGH; case '+': c = *++s; } @@ -1474,7 +1474,7 @@ _Py_dg_strtod(const char *s00, char **se) switch (c) { case '-': esign = 1; - /* fall through */ + _Py_FALLTHROUGH; case '+': c = *++s; } @@ -2362,7 +2362,7 @@ _Py_dg_dtoa(double dd, int mode, int ndigits, break; case 2: leftright = 0; - /* fall through */ + _Py_FALLTHROUGH; case 4: if (ndigits <= 0) ndigits = 1; @@ -2370,7 +2370,7 @@ _Py_dg_dtoa(double dd, int mode, int ndigits, break; case 3: leftright = 0; - /* fall through */ + _Py_FALLTHROUGH; case 5: i = ndigits + k + 1; ilim = i; diff --git a/Python/emscripten_trampoline.c b/Python/emscripten_trampoline.c index 2a80ec4f18d757..0293c7ea0f37ad 100644 --- a/Python/emscripten_trampoline.c +++ b/Python/emscripten_trampoline.c @@ -1,69 +1,203 @@ #if defined(PY_CALL_TRAMPOLINE) -#include // EM_JS +#include // EM_JS, EM_JS_DEPS #include #include "pycore_runtime.h" // _PyRuntime +typedef int (*CountArgsFunc)(PyCFunctionWithKeywords func); -/** - * This is the GoogleChromeLabs approved way to feature detect type-reflection: - * https://github.com/GoogleChromeLabs/wasm-feature-detect/blob/main/src/detectors/type-reflection/index.js - */ -EM_JS(int, _PyEM_detect_type_reflection, (), { - return "Function" in WebAssembly; +// Offset of emscripten_count_args_function in _PyRuntimeState. There's a couple +// of alternatives: +// 1. Just make emscripten_count_args_function a real C global variable instead +// of a field of _PyRuntimeState. This would violate our rule against mutable +// globals. +// 2. #define a preprocessor constant equal to a hard coded number and make a +// _Static_assert(offsetof(_PyRuntimeState, emscripten_count_args_function) +// == OURCONSTANT) This has the disadvantage that we have to update the hard +// coded constant when _PyRuntimeState changes +// +// So putting the mutable constant in _PyRuntime and using a immutable global to +// record the offset so we can access it from JS is probably the best way. +EMSCRIPTEN_KEEPALIVE const int _PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET = offsetof(_PyRuntimeState, emscripten_count_args_function); + +EM_JS(CountArgsFunc, _PyEM_GetCountArgsPtr, (), { + return Module._PyEM_CountArgsPtr; // initialized below +} +// Binary module for the checks. It has to be done in web assembly because +// clang/llvm have no support yet for the reference types yet. In fact, the wasm +// binary toolkit doesn't yet support the ref.test instruction either. To +// convert the following textual wasm to a binary, you can build wabt from this +// branch: https://github.com/WebAssembly/wabt/pull/2529 and then use that +// wat2wasm binary. +// +// (module +// (type $type0 (func (param) (result i32))) +// (type $type1 (func (param i32) (result i32))) +// (type $type2 (func (param i32 i32) (result i32))) +// (type $type3 (func (param i32 i32 i32) (result i32))) +// (type $blocktype (func (param i32) (result))) +// (table $funcs (import "e" "t") 0 funcref) +// (export "f" (func $f)) +// (func $f (param $fptr i32) (result i32) +// (local $fref funcref) +// local.get $fptr +// table.get $funcs +// local.tee $fref +// ref.test $type3 +// (block $b (type $blocktype) +// i32.eqz +// br_if $b +// i32.const 3 +// return +// ) +// local.get $fref +// ref.test $type2 +// (block $b (type $blocktype) +// i32.eqz +// br_if $b +// i32.const 2 +// return +// ) +// local.get $fref +// ref.test $type1 +// (block $b (type $blocktype) +// i32.eqz +// br_if $b +// i32.const 1 +// return +// ) +// local.get $fref +// ref.test $type0 +// (block $b (type $blocktype) +// i32.eqz +// br_if $b +// i32.const 0 +// return +// ) +// i32.const -1 +// ) +// ) +addOnPreRun(() => { + // Try to initialize countArgsFunc + const code = new Uint8Array([ + 0x00, 0x61, 0x73, 0x6d, // \0asm magic number + 0x01, 0x00, 0x00, 0x00, // version 1 + 0x01, 0x1b, // Type section, body is 0x1b bytes + 0x05, // 6 entries + 0x60, 0x00, 0x01, 0x7f, // (type $type0 (func (param) (result i32))) + 0x60, 0x01, 0x7f, 0x01, 0x7f, // (type $type1 (func (param i32) (result i32))) + 0x60, 0x02, 0x7f, 0x7f, 0x01, 0x7f, // (type $type2 (func (param i32 i32) (result i32))) + 0x60, 0x03, 0x7f, 0x7f, 0x7f, 0x01, 0x7f, // (type $type3 (func (param i32 i32 i32) (result i32))) + 0x60, 0x01, 0x7f, 0x00, // (type $blocktype (func (param i32) (result))) + 0x02, 0x09, // Import section, 0x9 byte body + 0x01, // 1 import (table $funcs (import "e" "t") 0 funcref) + 0x01, 0x65, // "e" + 0x01, 0x74, // "t" + 0x01, // importing a table + 0x70, // of entry type funcref + 0x00, 0x00, // table limits: no max, min of 0 + 0x03, 0x02, // Function section + 0x01, 0x01, // We're going to define one function of type 1 (func (param i32) (result i32)) + 0x07, 0x05, // export section + 0x01, // 1 export + 0x01, 0x66, // called "f" + 0x00, // a function + 0x00, // at index 0 + + 0x0a, 0x44, // Code section, + 0x01, 0x42, // one entry of length 50 + 0x01, 0x01, 0x70, // one local of type funcref + // Body of the function + 0x20, 0x00, // local.get $fptr + 0x25, 0x00, // table.get $funcs + 0x22, 0x01, // local.tee $fref + 0xfb, 0x14, 0x03, // ref.test $type3 + 0x02, 0x04, // block $b (type $blocktype) + 0x45, // i32.eqz + 0x0d, 0x00, // br_if $b + 0x41, 0x03, // i32.const 3 + 0x0f, // return + 0x0b, // end block + + 0x20, 0x01, // local.get $fref + 0xfb, 0x14, 0x02, // ref.test $type2 + 0x02, 0x04, // block $b (type $blocktype) + 0x45, // i32.eqz + 0x0d, 0x00, // br_if $b + 0x41, 0x02, // i32.const 2 + 0x0f, // return + 0x0b, // end block + + 0x20, 0x01, // local.get $fref + 0xfb, 0x14, 0x01, // ref.test $type1 + 0x02, 0x04, // block $b (type $blocktype) + 0x45, // i32.eqz + 0x0d, 0x00, // br_if $b + 0x41, 0x01, // i32.const 1 + 0x0f, // return + 0x0b, // end block + + 0x20, 0x01, // local.get $fref + 0xfb, 0x14, 0x00, // ref.test $type0 + 0x02, 0x04, // block $b (type $blocktype) + 0x45, // i32.eqz + 0x0d, 0x00, // br_if $b + 0x41, 0x00, // i32.const 0 + 0x0f, // return + 0x0b, // end block + + 0x41, 0x7f, // i32.const -1 + 0x0b // end function + ]); + let ptr = 0; + try { + const mod = new WebAssembly.Module(code); + const inst = new WebAssembly.Instance(mod, { e: { t: wasmTable } }); + ptr = addFunction(inst.exports.f); + } catch (e) { + // If something goes wrong, we'll null out _PyEM_CountFuncParams and fall + // back to the JS trampoline. + } + Module._PyEM_CountArgsPtr = ptr; + const offset = HEAP32[__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET / 4]; + HEAP32[(__PyRuntime + offset) / 4] = ptr; }); +); void _Py_EmscriptenTrampoline_Init(_PyRuntimeState *runtime) { - runtime->wasm_type_reflection_available = _PyEM_detect_type_reflection(); + runtime->emscripten_count_args_function = _PyEM_GetCountArgsPtr(); } +// We have to be careful to work correctly with memory snapshots. Even if we are +// loading a memory snapshot, we need to perform the JS initialization work. +// That means we can't call the initialization code from C. Instead, we export +// this function pointer to JS and then fill it in a preRun function which runs +// unconditionally. /** * Backwards compatible trampoline works with all JS runtimes */ -EM_JS(PyObject*, -_PyEM_TrampolineCall_JavaScript, (PyCFunctionWithKeywords func, - PyObject *arg1, - PyObject *arg2, - PyObject *arg3), -{ +EM_JS(PyObject*, _PyEM_TrampolineCall_JS, (PyCFunctionWithKeywords func, PyObject *arg1, PyObject *arg2, PyObject *arg3), { return wasmTable.get(func)(arg1, arg2, arg3); -} -); - -/** - * In runtimes with WebAssembly type reflection, count the number of parameters - * and cast to the appropriate signature - */ -EM_JS(int, _PyEM_CountFuncParams, (PyCFunctionWithKeywords func), -{ - let n = _PyEM_CountFuncParams.cache.get(func); - - if (n !== undefined) { - return n; - } - n = WebAssembly.Function.type(wasmTable.get(func)).parameters.length; - _PyEM_CountFuncParams.cache.set(func, n); - return n; -} -_PyEM_CountFuncParams.cache = new Map(); -) - +}); typedef PyObject* (*zero_arg)(void); typedef PyObject* (*one_arg)(PyObject*); typedef PyObject* (*two_arg)(PyObject*, PyObject*); typedef PyObject* (*three_arg)(PyObject*, PyObject*, PyObject*); - PyObject* -_PyEM_TrampolineCall_Reflection(PyCFunctionWithKeywords func, - PyObject* self, - PyObject* args, - PyObject* kw) +_PyEM_TrampolineCall(PyCFunctionWithKeywords func, + PyObject* self, + PyObject* args, + PyObject* kw) { - switch (_PyEM_CountFuncParams(func)) { + CountArgsFunc count_args = _PyRuntime.emscripten_count_args_function; + if (count_args == 0) { + return _PyEM_TrampolineCall_JS(func, self, args, kw); + } + switch (count_args(func)) { case 0: return ((zero_arg)func)(); case 1: @@ -73,8 +207,7 @@ _PyEM_TrampolineCall_Reflection(PyCFunctionWithKeywords func, case 3: return ((three_arg)func)(self, args, kw); default: - PyErr_SetString(PyExc_SystemError, - "Handler takes too many arguments"); + PyErr_SetString(PyExc_SystemError, "Handler takes too many arguments"); return NULL; } } diff --git a/Python/errors.c b/Python/errors.c index e5f176a5dd208e..9c7b771133dcf4 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -2,12 +2,13 @@ /* Error handling */ #include "Python.h" +#include "pycore_audit.h" // _PySys_Audit() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_initconfig.h" // _PyStatus_ERR() #include "pycore_pyerrors.h" // _PyErr_Format() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() -#include "pycore_sysmodule.h" // _PySys_Audit() +#include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_traceback.h" // _PyTraceBack_FromFrame() #ifdef MS_WINDOWS @@ -257,13 +258,14 @@ void _PyErr_SetKeyError(PyObject *arg) { PyThreadState *tstate = _PyThreadState_GET(); - PyObject *tup = PyTuple_Pack(1, arg); - if (!tup) { + PyObject *exc = PyObject_CallOneArg(PyExc_KeyError, arg); + if (!exc) { /* caller will expect error to be set anyway */ return; } - _PyErr_SetObject(tstate, PyExc_KeyError, tup); - Py_DECREF(tup); + + _PyErr_SetObject(tstate, (PyObject*)Py_TYPE(exc), exc); + Py_DECREF(exc); } void @@ -299,12 +301,21 @@ PyErr_SetString(PyObject *exception, const char *string) _PyErr_SetString(tstate, exception, string); } +void +_PyErr_SetLocaleString(PyObject *exception, const char *string) +{ + PyObject *value = PyUnicode_DecodeLocale(string, "surrogateescape"); + if (value != NULL) { + PyErr_SetObject(exception, value); + Py_DECREF(value); + } +} PyObject* _Py_HOT_FUNCTION PyErr_Occurred(void) { - /* The caller must hold the GIL. */ - assert(PyGILState_Check()); + /* The caller must hold a thread state. */ + _Py_AssertHoldsTstate(); PyThreadState *tstate = _PyThreadState_GET(); return _PyErr_Occurred(tstate); @@ -632,8 +643,8 @@ _PyErr_StackItemToExcInfoTuple(_PyErr_StackItem *err_info) PyObject *exc_type = get_exc_type(exc_value); PyObject *exc_traceback = get_exc_traceback(exc_value); - return Py_BuildValue( - "(OOO)", + return PyTuple_Pack( + 3, exc_type ? exc_type : Py_None, exc_value ? exc_value : Py_None, exc_traceback ? exc_traceback : Py_None); @@ -1849,6 +1860,52 @@ PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) Py_XDECREF(fileobj); } +/* Raises a SyntaxError. + * If something goes wrong, a different exception may be raised. + */ +void +_PyErr_RaiseSyntaxError(PyObject *msg, PyObject *filename, int lineno, int col_offset, + int end_lineno, int end_col_offset) +{ + PyObject *text = PyErr_ProgramTextObject(filename, lineno); + if (text == NULL) { + text = Py_NewRef(Py_None); + } + PyObject *args = Py_BuildValue("O(OiiOii)", msg, filename, + lineno, col_offset, text, + end_lineno, end_col_offset); + if (args == NULL) { + goto exit; + } + PyErr_SetObject(PyExc_SyntaxError, args); + exit: + Py_DECREF(text); + Py_XDECREF(args); +} + +/* Emits a SyntaxWarning and returns 0 on success. + If a SyntaxWarning is raised as error, replaces it with a SyntaxError + and returns -1. +*/ +int +_PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset, + int end_lineno, int end_col_offset) +{ + if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, filename, + lineno, NULL, NULL) < 0) + { + if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { + /* Replace the SyntaxWarning exception with a SyntaxError + to get a more accurate error report */ + PyErr_Clear(); + _PyErr_RaiseSyntaxError(msg, filename, lineno, col_offset, + end_lineno, end_col_offset); + } + return -1; + } + return 0; +} + /* Attempt to load the line of text that the exception refers to. If it fails, it will return NULL but will not set an exception. @@ -1856,44 +1913,44 @@ PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) functionality in tb_displayline() in traceback.c. */ static PyObject * -err_programtext(PyThreadState *tstate, FILE *fp, int lineno, const char* encoding) +err_programtext(FILE *fp, int lineno, const char* encoding) { - int i; char linebuf[1000]; - if (fp == NULL) { - return NULL; + size_t line_size = 0; + + for (int i = 0; i < lineno; ) { + line_size = 0; + if (_Py_UniversalNewlineFgetsWithSize(linebuf, sizeof(linebuf), + fp, NULL, &line_size) == NULL) + { + /* Error or EOF. */ + return NULL; + } + /* fgets read *something*; if it didn't fill the + whole buffer, it must have found a newline + or hit the end of the file; if the last character is \n, + it obviously found a newline; else we haven't + yet seen a newline, so must continue */ + if (i + 1 < lineno + && line_size == sizeof(linebuf) - 1 + && linebuf[sizeof(linebuf) - 2] != '\n') + { + continue; + } + i++; } - for (i = 0; i < lineno; i++) { - char *pLastChar = &linebuf[sizeof(linebuf) - 2]; - do { - *pLastChar = '\0'; - if (Py_UniversalNewlineFgets(linebuf, sizeof linebuf, - fp, NULL) == NULL) { - goto after_loop; - } - /* fgets read *something*; if it didn't get as - far as pLastChar, it must have found a newline - or hit the end of the file; if pLastChar is \n, - it obviously found a newline; else we haven't - yet seen a newline, so must continue */ - } while (*pLastChar != '\0' && *pLastChar != '\n'); + const char *line = linebuf; + /* Skip BOM. */ + if (lineno == 1 && line_size >= 3 && memcmp(line, "\xef\xbb\xbf", 3) == 0) { + line += 3; + line_size -= 3; } - -after_loop: - fclose(fp); - if (i == lineno) { - PyObject *res; - if (encoding != NULL) { - res = PyUnicode_Decode(linebuf, strlen(linebuf), encoding, "replace"); - } else { - res = PyUnicode_FromString(linebuf); - } - if (res == NULL) - _PyErr_Clear(tstate); - return res; + PyObject *res = PyUnicode_Decode(line, line_size, encoding, "replace"); + if (res == NULL) { + PyErr_Clear(); } - return NULL; + return res; } PyObject * @@ -1913,20 +1970,41 @@ PyErr_ProgramText(const char *filename, int lineno) return res; } +/* Function from Parser/tokenizer/file_tokenizer.c */ +extern char* _PyTokenizer_FindEncodingFilename(int, PyObject *); + PyObject * _PyErr_ProgramDecodedTextObject(PyObject *filename, int lineno, const char* encoding) { + char *found_encoding = NULL; if (filename == NULL || lineno <= 0) { return NULL; } - PyThreadState *tstate = _PyThreadState_GET(); - FILE *fp = _Py_fopen_obj(filename, "r" PY_STDIOTEXTMODE); + FILE *fp = Py_fopen(filename, "r" PY_STDIOTEXTMODE); if (fp == NULL) { - _PyErr_Clear(tstate); + PyErr_Clear(); return NULL; } - return err_programtext(tstate, fp, lineno, encoding); + if (encoding == NULL) { + int fd = fileno(fp); + found_encoding = _PyTokenizer_FindEncodingFilename(fd, filename); + encoding = found_encoding; + if (encoding == NULL) { + PyErr_Clear(); + encoding = "utf-8"; + } + /* Reset position */ + if (lseek(fd, 0, SEEK_SET) == (off_t)-1) { + fclose(fp); + PyMem_Free(found_encoding); + return NULL; + } + } + PyObject *res = err_programtext(fp, lineno, encoding); + fclose(fp); + PyMem_Free(found_encoding); + return res; } PyObject * diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 56ee93862743d5..4addfbcf6d6419 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -12,835 +12,1439 @@ break; } + case _CHECK_PERIODIC: { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) JUMP_TO_ERROR(); + } + break; + } + + case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: { + oparg = CURRENT_OPARG(); + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) JUMP_TO_ERROR(); + } + } + break; + } + + /* _QUICKEN_RESUME is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + /* _LOAD_BYTECODE is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + case _RESUME_CHECK: { #if defined(__EMSCRIPTEN__) - if (_Py_emscripten_signal_clock == 0) goto deoptimize; + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); - if (eval_breaker != version) goto deoptimize; + if (eval_breaker != version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #endif break; } - /* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 */ + /* _MONITOR_RESUME is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _LOAD_FAST_CHECK: { - PyObject *value; + _PyStackRef value; oparg = CURRENT_OPARG(); - value = GETLOCAL(oparg); - if (value == NULL) goto unbound_local_error_tier_two; - Py_INCREF(value); + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (1) JUMP_TO_ERROR(); + } + value = PyStackRef_DUP(value_s); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_FAST_0: { - PyObject *value; + _PyStackRef value; oparg = 0; assert(oparg == CURRENT_OPARG()); - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_FAST_1: { - PyObject *value; + _PyStackRef value; oparg = 1; assert(oparg == CURRENT_OPARG()); - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_FAST_2: { - PyObject *value; + _PyStackRef value; oparg = 2; assert(oparg == CURRENT_OPARG()); - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_FAST_3: { - PyObject *value; + _PyStackRef value; oparg = 3; assert(oparg == CURRENT_OPARG()); - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_FAST_4: { - PyObject *value; + _PyStackRef value; oparg = 4; assert(oparg == CURRENT_OPARG()); - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_FAST_5: { - PyObject *value; + _PyStackRef value; oparg = 5; assert(oparg == CURRENT_OPARG()); - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_FAST_6: { - PyObject *value; + _PyStackRef value; oparg = 6; assert(oparg == CURRENT_OPARG()); - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_FAST_7: { - PyObject *value; + _PyStackRef value; oparg = 7; assert(oparg == CURRENT_OPARG()); - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_FAST: { - PyObject *value; + _PyStackRef value; oparg = CURRENT_OPARG(); - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_FAST_AND_CLEAR: { - PyObject *value; + _PyStackRef value; oparg = CURRENT_OPARG(); value = GETLOCAL(oparg); // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = NULL; + GETLOCAL(oparg) = PyStackRef_NULL; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _LOAD_CONST is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + case _LOAD_CONST_MORTAL: { + _PyStackRef value; + oparg = CURRENT_OPARG(); + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectNew(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_CONST_IMMORTAL: { + _PyStackRef value; + oparg = CURRENT_OPARG(); + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + assert(_Py_IsImmortal(obj)); + value = PyStackRef_FromPyObjectImmortal(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_SMALL_INT_0: { + _PyStackRef value; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectImmortal(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_SMALL_INT_1: { + _PyStackRef value; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectImmortal(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_SMALL_INT_2: { + _PyStackRef value; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectImmortal(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_SMALL_INT_3: { + _PyStackRef value; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectImmortal(obj); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_CONST: { - PyObject *value; + case _LOAD_SMALL_INT: { + _PyStackRef value; oparg = CURRENT_OPARG(); - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectImmortal(obj); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_FAST_0: { - PyObject *value; + _PyStackRef value; oparg = 0; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; SETLOCAL(oparg, value); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_FAST_1: { - PyObject *value; + _PyStackRef value; oparg = 1; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; SETLOCAL(oparg, value); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_FAST_2: { - PyObject *value; + _PyStackRef value; oparg = 2; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; SETLOCAL(oparg, value); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_FAST_3: { - PyObject *value; + _PyStackRef value; oparg = 3; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; SETLOCAL(oparg, value); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_FAST_4: { - PyObject *value; + _PyStackRef value; oparg = 4; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; SETLOCAL(oparg, value); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_FAST_5: { - PyObject *value; + _PyStackRef value; oparg = 5; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; SETLOCAL(oparg, value); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_FAST_6: { - PyObject *value; + _PyStackRef value; oparg = 6; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; SETLOCAL(oparg, value); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_FAST_7: { - PyObject *value; + _PyStackRef value; oparg = 7; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; SETLOCAL(oparg, value); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_FAST: { - PyObject *value; + _PyStackRef value; oparg = CURRENT_OPARG(); value = stack_pointer[-1]; SETLOCAL(oparg, value); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _POP_TOP: { - PyObject *value; + _PyStackRef value; value = stack_pointer[-1]; - Py_DECREF(value); + PyStackRef_CLOSE(value); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _PUSH_NULL: { - PyObject *res; - res = NULL; + _PyStackRef res; + res = PyStackRef_NULL; stack_pointer[0] = res; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _END_FOR: { + _PyStackRef value; + value = stack_pointer[-1]; + /* Don't update instr_ptr, so that POP_ITER sees + * the FOR_ITER as the previous instruction. + * This has the benign side effect that if value is + * finalized it will see the location as the FOR_ITER's. + */ + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _END_SEND: { - PyObject *value; - PyObject *receiver; + _PyStackRef value; + _PyStackRef receiver; + _PyStackRef val; value = stack_pointer[-1]; receiver = stack_pointer[-2]; - Py_DECREF(receiver); - stack_pointer[-2] = value; + (void)receiver; + val = value; + PyStackRef_CLOSE(receiver); + stack_pointer[-2] = val; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _UNARY_NEGATIVE: { - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - res = PyNumber_Negative(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; break; } case _UNARY_NOT: { - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - assert(PyBool_Check(value)); - res = Py_IsFalse(value) ? Py_True : Py_False; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; stack_pointer[-1] = res; break; } case _TO_BOOL: { - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - int err = PyObject_IsTrue(value); - Py_DECREF(value); - if (err < 0) goto pop_1_error_tier_two; - res = err ? Py_True : Py_False; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (err < 0) JUMP_TO_ERROR(); + res = err ? PyStackRef_True : PyStackRef_False; stack_pointer[-1] = res; break; } case _TO_BOOL_BOOL: { - PyObject *value; + _PyStackRef value; value = stack_pointer[-1]; - if (!PyBool_Check(value)) goto side_exit; + if (!PyStackRef_BoolCheck(value)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(TO_BOOL, hit); break; } case _TO_BOOL_INT: { - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - if (!PyLong_CheckExact(value)) goto side_exit; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value)) { - assert(_Py_IsImmortal(value)); - res = Py_False; + if (_PyLong_IsZero((PyLongObject *)value_o)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; } else { - Py_DECREF(value); - res = Py_True; + PyStackRef_CLOSE(value); + res = PyStackRef_True; } stack_pointer[-1] = res; break; } case _TO_BOOL_LIST: { - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - if (!PyList_CheckExact(value)) goto side_exit; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyList_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(TO_BOOL, hit); - res = Py_SIZE(value) ? Py_True : Py_False; - Py_DECREF(value); + res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; + PyStackRef_CLOSE(value); stack_pointer[-1] = res; break; } case _TO_BOOL_NONE: { - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - if (!Py_IsNone(value)) goto side_exit; + if (!PyStackRef_IsNone(value)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(TO_BOOL, hit); - res = Py_False; + res = PyStackRef_False; stack_pointer[-1] = res; break; } case _TO_BOOL_STR: { - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - if (!PyUnicode_CheckExact(value)) goto side_exit; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(TO_BOOL, hit); - if (value == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value)); - res = Py_False; + if (value_o == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; } else { - assert(Py_SIZE(value)); - Py_DECREF(value); - res = Py_True; + assert(Py_SIZE(value_o)); + PyStackRef_CLOSE(value); + res = PyStackRef_True; } stack_pointer[-1] = res; break; } - case _TO_BOOL_ALWAYS_TRUE: { - PyObject *value; - PyObject *res; + case _REPLACE_WITH_TRUE: { + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - uint32_t version = (uint32_t)CURRENT_OPERAND(); - // This one is a bit weird, because we expect *some* failures: - assert(version); - if (Py_TYPE(value)->tp_version_tag != version) goto side_exit; - STAT_INC(TO_BOOL, hit); - Py_DECREF(value); - res = Py_True; + PyStackRef_CLOSE(value); + res = PyStackRef_True; stack_pointer[-1] = res; break; } case _UNARY_INVERT: { - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - res = PyNumber_Invert(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; break; } case _GUARD_BOTH_INT: { - PyObject *right; - PyObject *left; + _PyStackRef right; + _PyStackRef left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyLong_CheckExact(left)) goto side_exit; - if (!PyLong_CheckExact(right)) goto side_exit; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyLong_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyLong_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _GUARD_NOS_INT: { + _PyStackRef left; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyLong_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _GUARD_TOS_INT: { + _PyStackRef value; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _BINARY_OP_MULTIPLY_INT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); - res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error_tier_two; + PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_OP_ADD_INT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); - res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error_tier_two; + PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_OP_SUBTRACT_INT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); - res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error_tier_two; + PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_BOTH_FLOAT: { - PyObject *right; - PyObject *left; + _PyStackRef right; + _PyStackRef left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyFloat_CheckExact(left)) goto side_exit; - if (!PyFloat_CheckExact(right)) goto side_exit; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyFloat_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _GUARD_NOS_FLOAT: { + _PyStackRef left; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _GUARD_TOS_FLOAT: { + _PyStackRef value; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _BINARY_OP_MULTIPLY_FLOAT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_OP_ADD_FLOAT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_OP_SUBTRACT_FLOAT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval - - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_BOTH_UNICODE: { - PyObject *right; - PyObject *left; + _PyStackRef right; + _PyStackRef left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyUnicode_CheckExact(left)) goto side_exit; - if (!PyUnicode_CheckExact(right)) goto side_exit; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyUnicode_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyUnicode_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _BINARY_OP_ADD_UNICODE: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_INPLACE_ADD_UNICODE: { + _PyStackRef right; + _PyStackRef left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + int next_oparg; + #if TIER_ONE + assert(next_instr->op.code == STORE_FAST); + next_oparg = next_instr->op.arg; + #else + next_oparg = CURRENT_OPERAND0(); + #endif + _PyStackRef *target_local = &GETLOCAL(next_oparg); + if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(BINARY_OP, hit); + /* Handle `left = left + right` or `left += right` for str. + * + * When possible, extend `left` in place rather than + * allocating a new PyUnicodeObject. This attempts to avoid + * quadratic behavior when one neglects to use str.join(). + * + * If `left` has only two references remaining (one from + * the stack, one in the locals), DECREFing `left` leaves + * only the locals reference, so PyUnicode_Append knows + * that the string is safe to mutate. + */ + assert(Py_REFCNT(left_o) >= 2); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); + PyUnicode_Append(&temp, right_o); + *target_local = PyStackRef_FromPyObjectSteal(temp); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (PyStackRef_IsNull(*target_local)) JUMP_TO_ERROR(); + #if TIER_ONE + // The STORE_FAST is already done. This is done here in tier one, + // and during trace projection in tier two: + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); + #endif + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_BINARY_OP_EXTEND: { + _PyStackRef right; + _PyStackRef left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + assert(d && d->guard); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = d->guard(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!res) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _BINARY_OP_EXTEND: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; STAT_INC(BINARY_OP, hit); - res = PyUnicode_Concat(left, right); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (res == NULL) goto pop_2_error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = d->action(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_SUBSCR: { - PyObject *sub; - PyObject *container; - PyObject *res; + _PyStackRef sub; + _PyStackRef container; + _PyStackRef res; sub = stack_pointer[-1]; container = stack_pointer[-2]; - res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (res == NULL) goto pop_2_error_tier_two; + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_GetItem(container_o, sub_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_SLICE: { - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *res; + _PyStackRef stop; + _PyStackRef start; + _PyStackRef container; + _PyStackRef res; stop = stack_pointer[-1]; start = stack_pointer[-2]; container = stack_pointer[-3]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyObject *res_o; // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. if (slice == NULL) { - res = NULL; + res_o = NULL; } else { - res = PyObject_GetItem(container, slice); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } - Py_DECREF(container); - if (res == NULL) goto pop_3_error_tier_two; + PyStackRef_CLOSE(container); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-3] = res; stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_SLICE: { - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *v; + _PyStackRef stop; + _PyStackRef start; + _PyStackRef container; + _PyStackRef v; stop = stack_pointer[-1]; start = stack_pointer[-2]; container = stack_pointer[-3]; v = stack_pointer[-4]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); int err; if (slice == NULL) { err = 1; } else { - err = PyObject_SetItem(container, slice, v); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } - Py_DECREF(v); - Py_DECREF(container); - if (err) goto pop_4_error_tier_two; + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + if (err) JUMP_TO_ERROR(); stack_pointer += -4; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_SUBSCR_LIST_INT: { - PyObject *sub; - PyObject *list; - PyObject *res; - sub = stack_pointer[-1]; - list = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) goto deoptimize; - if (!PyList_CheckExact(list)) goto deoptimize; + _PyStackRef sub_st; + _PyStackRef list_st; + _PyStackRef res; + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + if (!PyLong_CheckExact(sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyList_CheckExact(list)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // Deopt unless 0 <= sub < PyList_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize; + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyList_GET_SIZE(list)) goto deoptimize; + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(BINARY_SUBSCR, hit); - res = PyList_GET_ITEM(list, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + #else + if (index >= PyList_GET_SIZE(list)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + Py_INCREF(res_o); + #endif + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(list_st); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_SUBSCR_STR_INT: { - PyObject *sub; - PyObject *str; - PyObject *res; - sub = stack_pointer[-1]; - str = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) goto deoptimize; - if (!PyUnicode_CheckExact(str)) goto deoptimize; - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize; + _PyStackRef sub_st; + _PyStackRef str_st; + _PyStackRef res; + sub_st = stack_pointer[-1]; + str_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + if (!PyLong_CheckExact(sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyUnicode_CheckExact(str)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) goto deoptimize; + if (PyUnicode_GET_LENGTH(str) <= index) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) goto deoptimize; + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(BINARY_SUBSCR, hit); - res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(str); + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(str_st); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_SUBSCR_TUPLE_INT: { - PyObject *sub; - PyObject *tuple; - PyObject *res; - sub = stack_pointer[-1]; - tuple = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) goto deoptimize; - if (!PyTuple_CheckExact(tuple)) goto deoptimize; + _PyStackRef sub_st; + _PyStackRef tuple_st; + _PyStackRef res; + sub_st = stack_pointer[-1]; + tuple_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + if (!PyLong_CheckExact(sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyTuple_CheckExact(tuple)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // Deopt unless 0 <= sub < PyTuple_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize; + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) goto deoptimize; + if (index >= PyTuple_GET_SIZE(tuple)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(BINARY_SUBSCR, hit); - res = PyTuple_GET_ITEM(tuple, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(tuple); + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + Py_INCREF(res_o); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(tuple_st); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_SUBSCR_DICT: { - PyObject *sub; - PyObject *dict; - PyObject *res; - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; - if (!PyDict_CheckExact(dict)) goto deoptimize; + _PyStackRef sub_st; + _PyStackRef dict_st; + _PyStackRef res; + sub_st = stack_pointer[-1]; + dict_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(BINARY_SUBSCR, hit); - int rc = PyDict_GetItemRef(dict, sub, &res); + PyObject *res_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int rc = PyDict_GetItemRef(dict, sub, &res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (rc == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetKeyError(sub); + stack_pointer = _PyFrame_GetStackPointer(frame); } - Py_DECREF(dict); - Py_DECREF(sub); - if (rc <= 0) goto pop_2_error_tier_two; + PyStackRef_CLOSE(dict_st); + PyStackRef_CLOSE(sub_st); + if (rc <= 0) JUMP_TO_ERROR(); // not found or error + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR_CHECK_FUNC: { + _PyStackRef container; + _PyStackRef getitem; + container = stack_pointer[-2]; + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); + if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); + if (getitem_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + assert(PyFunction_Check(getitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); + if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); + assert(code->co_argcount == 2); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + getitem = PyStackRef_FromPyObjectNew(getitem_o); + STAT_INC(BINARY_SUBSCR, hit); + stack_pointer[0] = getitem; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - /* _BINARY_SUBSCR_GETITEM is not a viable micro-op for tier 2 */ + case _BINARY_SUBSCR_INIT_CALL: { + _PyStackRef getitem; + _PyStackRef sub; + _PyStackRef container; + _PyInterpreterFrame *new_frame; + getitem = stack_pointer[-1]; + sub = stack_pointer[-2]; + container = stack_pointer[-3]; + new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + new_frame->localsplus[0] = container; + new_frame->localsplus[1] = sub; + frame->return_offset = 2 ; + stack_pointer[-3].bits = (uintptr_t)new_frame; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _LIST_APPEND: { - PyObject *v; - PyObject *list; + _PyStackRef v; + _PyStackRef list; oparg = CURRENT_OPARG(); v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error_tier_two; + int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)); + if (err < 0) JUMP_TO_ERROR(); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _SET_ADD: { - PyObject *v; - PyObject *set; + _PyStackRef v; + _PyStackRef set; oparg = CURRENT_OPARG(); v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; - int err = PySet_Add(set, v); - Py_DECREF(v); - if (err) goto pop_1_error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + if (err) JUMP_TO_ERROR(); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_SUBSCR: { - PyObject *sub; - PyObject *container; - PyObject *v; + _PyStackRef sub; + _PyStackRef container; + _PyStackRef v; sub = stack_pointer[-1]; container = stack_pointer[-2]; v = stack_pointer[-3]; /* container[sub] = v */ - int err = PyObject_SetItem(container, sub, v); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); - if (err) goto pop_3_error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (err) JUMP_TO_ERROR(); stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_SUBSCR_LIST_INT: { - PyObject *sub; - PyObject *list; - PyObject *value; - sub = stack_pointer[-1]; - list = stack_pointer[-2]; + _PyStackRef sub_st; + _PyStackRef list_st; + _PyStackRef value; + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; value = stack_pointer[-3]; - if (!PyLong_CheckExact(sub)) goto deoptimize; - if (!PyList_CheckExact(list)) goto deoptimize; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + if (!PyLong_CheckExact(sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyList_CheckExact(list)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // Ensure nonnegative, zero-or-one-digit ints. - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize; + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (!LOCK_OBJECT(list)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // Ensure index < len(list) - if (index >= PyList_GET_SIZE(list)) goto deoptimize; + if (index >= PyList_GET_SIZE(list)) { + UNLOCK_OBJECT(list); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, value); + PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); - Py_DECREF(old_value); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + UNLOCK_OBJECT(list); // unlock before decrefs! + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(list_st); stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } case _STORE_SUBSCR_DICT: { - PyObject *sub; - PyObject *dict; - PyObject *value; + _PyStackRef sub; + _PyStackRef dict_st; + _PyStackRef value; sub = stack_pointer[-1]; - dict = stack_pointer[-2]; + dict_st = stack_pointer[-2]; value = stack_pointer[-3]; - if (!PyDict_CheckExact(dict)) goto deoptimize; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); - Py_DECREF(dict); - if (err) goto pop_3_error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(dict_st); + if (err) JUMP_TO_ERROR(); stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); break; } case _DELETE_SUBSCR: { - PyObject *sub; - PyObject *container; + _PyStackRef sub; + _PyStackRef container; sub = stack_pointer[-1]; container = stack_pointer[-2]; /* del container[sub] */ - int err = PyObject_DelItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (err) goto pop_2_error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + PyStackRef_AsPyObjectBorrow(sub)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (err) JUMP_TO_ERROR(); stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_INTRINSIC_1: { - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; oparg = CURRENT_OPARG(); value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); - res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; break; } case _CALL_INTRINSIC_2: { - PyObject *value1; - PyObject *value2; - PyObject *res; + _PyStackRef value1_st; + _PyStackRef value2_st; + _PyStackRef res; oparg = CURRENT_OPARG(); - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; assert(oparg <= MAX_INTRINSIC_2); - res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - Py_DECREF(value2); - Py_DECREF(value1); - if (res == NULL) goto pop_2_error_tier_two; + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value2_st); + PyStackRef_CLOSE(value1_st); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _POP_FRAME: { - PyObject *retval; + case _RETURN_VALUE: { + _PyStackRef retval; + _PyStackRef res; retval = stack_pointer[-1]; - #if TIER_ONE - assert(frame != &entry_frame); - #endif + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = retval; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); @@ -848,186 +1452,251 @@ _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif + res = temp; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - /* _INSTRUMENTED_RETURN_VALUE is not a viable micro-op for tier 2 */ - - /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 */ - case _GET_AITER: { - PyObject *obj; - PyObject *iter; + _PyStackRef obj; + _PyStackRef iter; obj = stack_pointer[-1]; unaryfunc getter = NULL; - PyTypeObject *type = Py_TYPE(obj); + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; } if (getter == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - Py_DECREF(obj); - if (true) goto pop_1_error_tier_two; - } - iter = (*getter)(obj); - Py_DECREF(obj); - if (iter == NULL) goto pop_1_error_tier_two; - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(obj); + if (true) JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + iter_o = (*getter)(obj_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(obj); + if (iter_o == NULL) JUMP_TO_ERROR(); + if (Py_TYPE(iter_o)->tp_as_async == NULL || + Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", - Py_TYPE(iter)->tp_name); - Py_DECREF(iter); - if (true) goto pop_1_error_tier_two; + Py_TYPE(iter_o)->tp_name); + Py_DECREF(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (true) JUMP_TO_ERROR(); } + iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; break; } case _GET_ANEXT: { - PyObject *aiter; - PyObject *awaitable; + _PyStackRef aiter; + _PyStackRef awaitable; aiter = stack_pointer[-1]; - unaryfunc getter = NULL; - PyObject *next_iter = NULL; - PyTypeObject *type = Py_TYPE(aiter); - if (PyAsyncGen_CheckExact(aiter)) { - awaitable = type->tp_as_async->am_anext(aiter); - if (awaitable == NULL) { - GOTO_ERROR(error); - } - } else { - if (type->tp_as_async != NULL){ - getter = type->tp_as_async->am_anext; - } - if (getter != NULL) { - next_iter = (*getter)(aiter); - if (next_iter == NULL) { - GOTO_ERROR(error); - } - } - else { - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an iterator with " - "__anext__ method, got %.100s", - type->tp_name); - GOTO_ERROR(error); - } - awaitable = _PyCoro_GetAwaitableIter(next_iter); - if (awaitable == NULL) { - _PyErr_FormatFromCause( - PyExc_TypeError, - "'async for' received an invalid object " - "from __anext__: %.100s", - Py_TYPE(next_iter)->tp_name); - Py_DECREF(next_iter); - GOTO_ERROR(error); - } else { - Py_DECREF(next_iter); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (awaitable_o == NULL) { + JUMP_TO_ERROR(); } + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); stack_pointer[0] = awaitable; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _GET_AWAITABLE: { - PyObject *iterable; - PyObject *iter; + _PyStackRef iterable; + _PyStackRef iter; oparg = CURRENT_OPARG(); iterable = stack_pointer[-1]; - iter = _PyCoro_GetAwaitableIter(iterable); - if (iter == NULL) { - _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); - } - Py_DECREF(iterable); - if (iter != NULL && PyCoro_CheckExact(iter)) { - PyObject *yf = _PyGen_yf((PyGenObject*)iter); - if (yf != NULL) { - /* `iter` is a coroutine object that is being - awaited, `yf` is a pointer to the current awaitable - being awaited on. */ - Py_DECREF(yf); - Py_CLEAR(iter); - _PyErr_SetString(tstate, PyExc_RuntimeError, - "coroutine is being awaited already"); - /* The code below jumps to `error` if `iter` is NULL. */ - } - } - if (iter == NULL) goto pop_1_error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(iterable); + if (iter_o == NULL) JUMP_TO_ERROR(); + iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; break; } - /* _SEND is not a viable micro-op for tier 2 */ + /* _SEND is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - /* _SEND_GEN is not a viable micro-op for tier 2 */ - - /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 */ + case _SEND_GEN_FRAME: { + _PyStackRef v; + _PyStackRef receiver; + _PyInterpreterFrame *gen_frame; + oparg = CURRENT_OPARG(); + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(SEND, hit); + gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2 + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2 + oparg); + gen_frame->previous = frame; + stack_pointer[-1].bits = (uintptr_t)gen_frame; + break; + } + + case _YIELD_VALUE: { + _PyStackRef retval; + _PyStackRef value; + oparg = CURRENT_OPARG(); + retval = stack_pointer[-1]; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = temp; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _POP_EXCEPT: { - PyObject *exc_value; + _PyStackRef exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XSETREF(exc_info->exc_value, + PyStackRef_IsNone(exc_value) + ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_ASSERTION_ERROR: { - PyObject *value; - value = Py_NewRef(PyExc_AssertionError); + case _LOAD_COMMON_CONSTANT: { + _PyStackRef value; + oparg = CURRENT_OPARG(); + // Keep in sync with _common_constants in opcode.py + // If we ever have more than two constants, use a lookup table + PyObject *val; + if (oparg == CONSTANT_ASSERTIONERROR) { + val = PyExc_AssertionError; + } + else { + assert(oparg == CONSTANT_NOTIMPLEMENTEDERROR); + val = PyExc_NotImplementedError; + } + value = PyStackRef_FromPyObjectImmortal(val); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_BUILD_CLASS: { - PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error_tier_two; - if (bc == NULL) { + _PyStackRef bc; + PyObject *bc_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) JUMP_TO_ERROR(); + if (bc_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); - if (true) goto error_tier_two; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (true) JUMP_TO_ERROR(); } + bc = PyStackRef_FromPyObjectSteal(bc_o); stack_pointer[0] = bc; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_NAME: { - PyObject *v; + _PyStackRef v; oparg = CURRENT_OPARG(); v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - Py_DECREF(v); - if (true) goto pop_1_error_tier_two; - } - if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); - else - err = PyObject_SetItem(ns, name, v); - Py_DECREF(v); - if (err) goto pop_1_error_tier_two; + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + if (true) JUMP_TO_ERROR(); + } + if (PyDict_CheckExact(ns)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + PyStackRef_CLOSE(v); + if (err) JUMP_TO_ERROR(); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -1037,416 +1706,501 @@ PyObject *ns = LOCALS(); int err; if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - GOTO_ERROR(error); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_ERROR(); } + _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_DelItem(ns, name); + stack_pointer = _PyFrame_GetStackPointer(frame); // Can't use ERROR_IF here. if (err != 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_ERROR(); } break; } case _UNPACK_SEQUENCE: { - PyObject *seq; + _PyStackRef seq; + _PyStackRef *output; oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; - PyObject **top = stack_pointer + oparg - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); - Py_DECREF(seq); - if (res == 0) goto pop_1_error_tier_two; + output = &stack_pointer[-1]; + _PyStackRef *top = output + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(seq); + if (res == 0) JUMP_TO_ERROR(); stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _UNPACK_SEQUENCE_TWO_TUPLE: { - PyObject *seq; - PyObject **values; + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - if (!PyTuple_CheckExact(seq)) goto deoptimize; - if (PyTuple_GET_SIZE(seq) != 2) goto deoptimize; assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + if (!PyTuple_CheckExact(seq_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyTuple_GET_SIZE(seq_o) != 2) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(UNPACK_SEQUENCE, hit); - values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - Py_DECREF(seq); - stack_pointer += -1 + oparg; + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + PyStackRef_CLOSE(seq); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _UNPACK_SEQUENCE_TUPLE: { - PyObject *seq; - PyObject **values; + _PyStackRef seq; + _PyStackRef *values; oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; values = &stack_pointer[-1]; - if (!PyTuple_CheckExact(seq)) goto deoptimize; - if (PyTuple_GET_SIZE(seq) != oparg) goto deoptimize; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + if (!PyTuple_CheckExact(seq_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyTuple_GET_SIZE(seq_o) != oparg) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq); + PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + *values++ = PyStackRef_FromPyObjectNew(items[i]); } - Py_DECREF(seq); + PyStackRef_CLOSE(seq); stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _UNPACK_SEQUENCE_LIST: { - PyObject *seq; - PyObject **values; + _PyStackRef seq; + _PyStackRef *values; oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; values = &stack_pointer[-1]; - if (!PyList_CheckExact(seq)) goto deoptimize; - if (PyList_GET_SIZE(seq) != oparg) goto deoptimize; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + if (!PyList_CheckExact(seq_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!LOCK_OBJECT(seq_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyList_GET_SIZE(seq_o) != oparg) { + UNLOCK_OBJECT(seq_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq); + PyObject **items = _PyList_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + *values++ = PyStackRef_FromPyObjectNew(items[i]); } - Py_DECREF(seq); + UNLOCK_OBJECT(seq_o); + PyStackRef_CLOSE(seq); stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _UNPACK_EX: { - PyObject *seq; + _PyStackRef seq; + _PyStackRef *right; oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; - int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); - PyObject **top = stack_pointer + totalargs - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq); - if (res == 0) goto pop_1_error_tier_two; - stack_pointer += (oparg >> 8) + (oparg & 0xFF); + right = &stack_pointer[(oparg & 0xFF)]; + _PyStackRef *top = right + (oparg >> 8); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(seq); + if (res == 0) JUMP_TO_ERROR(); + stack_pointer += (oparg & 0xFF) + (oparg >> 8); + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_ATTR: { - PyObject *owner; - PyObject *v; + _PyStackRef owner; + _PyStackRef v; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; v = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); - if (err) goto pop_2_error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(owner); + if (err) JUMP_TO_ERROR(); stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } case _DELETE_ATTR: { - PyObject *owner; + _PyStackRef owner; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_DelAttr(owner, name); - Py_DECREF(owner); - if (err) goto pop_1_error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(owner); + if (err) JUMP_TO_ERROR(); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_GLOBAL: { - PyObject *v; + _PyStackRef v; oparg = CURRENT_OPARG(); v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyDict_SetItem(GLOBALS(), name, v); - Py_DECREF(v); - if (err) goto pop_1_error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + if (err) JUMP_TO_ERROR(); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _DELETE_GLOBAL: { oparg = CURRENT_OPARG(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err; - err = PyDict_DelItem(GLOBALS(), name); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Pop(GLOBALS(), name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); // Can't use ERROR_IF here. - if (err != 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - GOTO_ERROR(error); + if (err < 0) { + JUMP_TO_ERROR(); + } + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_ERROR(); } break; } case _LOAD_LOCALS: { - PyObject *locals; - locals = LOCALS(); - if (locals == NULL) { + _PyStackRef locals; + PyObject *l = LOCALS(); + if (l == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - if (true) goto error_tier_two; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (true) JUMP_TO_ERROR(); } - Py_INCREF(locals); + locals = PyStackRef_FromPyObjectNew(l); stack_pointer[0] = locals; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_FROM_DICT_OR_GLOBALS: { - PyObject *mod_or_class_dict; - PyObject *v; - oparg = CURRENT_OPARG(); - mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - GOTO_ERROR(error); - } - } - } - Py_DECREF(mod_or_class_dict); - stack_pointer[-1] = v; - break; - } + /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ case _LOAD_NAME: { - PyObject *v; + _PyStackRef v; oparg = CURRENT_OPARG(); - PyObject *mod_or_class_dict = LOCALS(); - if (mod_or_class_dict == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error_tier_two; - } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - GOTO_ERROR(error); - } - } - } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) JUMP_TO_ERROR(); + v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_GLOBAL: { - PyObject *res; - PyObject *null = NULL; + _PyStackRef *res; oparg = CURRENT_OPARG(); + res = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - if (res == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - if (true) goto error_tier_two; - } - Py_INCREF(res); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*res)) JUMP_TO_ERROR(); + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _PUSH_NULL_CONDITIONAL: { + _PyStackRef null = PyStackRef_NULL; + oparg = CURRENT_OPARG(); + null = PyStackRef_NULL; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_GLOBALS_VERSION: { + uint16_t version = (uint16_t)CURRENT_OPERAND0(); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); } - else { - /* Slow-path if globals or builtins is not a dict */ - /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error_tier_two; - if (res == NULL) { - /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error_tier_two; - if (res == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - if (true) goto error_tier_two; - } - } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); } - null = NULL; - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + assert(DK_IS_UNICODE(keys)); break; } - case _GUARD_GLOBALS_VERSION: { - uint16_t version = (uint16_t)CURRENT_OPERAND(); + case _GUARD_GLOBALS_VERSION_PUSH_KEYS: { + PyDictKeysObject *globals_keys; + uint16_t version = (uint16_t)CURRENT_OPERAND0(); PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) goto deoptimize; - if (dict->ma_keys->dk_version != version) goto deoptimize; - assert(DK_IS_UNICODE(dict->ma_keys)); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + globals_keys = keys; + assert(DK_IS_UNICODE(globals_keys)); + stack_pointer[0].bits = (uintptr_t)globals_keys; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _GUARD_BUILTINS_VERSION: { - uint16_t version = (uint16_t)CURRENT_OPERAND(); + case _GUARD_BUILTINS_VERSION_PUSH_KEYS: { + PyDictKeysObject *builtins_keys; + uint16_t version = (uint16_t)CURRENT_OPERAND0(); PyDictObject *dict = (PyDictObject *)BUILTINS(); - if (!PyDict_CheckExact(dict)) goto deoptimize; - if (dict->ma_keys->dk_version != version) goto deoptimize; - assert(DK_IS_UNICODE(dict->ma_keys)); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + builtins_keys = keys; + assert(DK_IS_UNICODE(builtins_keys)); + stack_pointer[0].bits = (uintptr_t)builtins_keys; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_GLOBAL_MODULE: { - PyObject *res; - PyObject *null = NULL; - oparg = CURRENT_OPARG(); - uint16_t index = (uint16_t)CURRENT_OPERAND(); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - res = entries[index].me_value; - if (res == NULL) goto deoptimize; - Py_INCREF(res); + case _LOAD_GLOBAL_MODULE_FROM_KEYS: { + PyDictKeysObject *globals_keys; + _PyStackRef res; + globals_keys = (PyDictKeysObject *)stack_pointer[-1].bits; + uint16_t index = (uint16_t)CURRENT_OPERAND0(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #else + Py_INCREF(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + #endif STAT_INC(LOAD_GLOBAL, hit); - null = NULL; stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_GLOBAL_BUILTINS: { - PyObject *res; - PyObject *null = NULL; - oparg = CURRENT_OPARG(); - uint16_t index = (uint16_t)CURRENT_OPERAND(); - PyDictObject *bdict = (PyDictObject *)BUILTINS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); - res = entries[index].me_value; - if (res == NULL) goto deoptimize; - Py_INCREF(res); + case _LOAD_GLOBAL_BUILTINS_FROM_KEYS: { + PyDictKeysObject *builtins_keys; + _PyStackRef res; + builtins_keys = (PyDictKeysObject *)stack_pointer[-1].bits; + uint16_t index = (uint16_t)CURRENT_OPERAND0(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #else + Py_INCREF(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + #endif STAT_INC(LOAD_GLOBAL, hit); - null = NULL; stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _DELETE_FAST: { oparg = CURRENT_OPARG(); - PyObject *v = GETLOCAL(oparg); - if (v == NULL) goto unbound_local_error_tier_two; - SETLOCAL(oparg, NULL); + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (1) JUMP_TO_ERROR(); + } + SETLOCAL(oparg, PyStackRef_NULL); break; } case _MAKE_CELL: { oparg = CURRENT_OPARG(); // "initial" is probably NULL but not if it's an arg (or set - // via PyFrame_LocalsToFast() before MAKE_CELL has run). - PyObject *initial = GETLOCAL(oparg); + // via the f_locals proxy before MAKE_CELL has run). + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *cell = PyCell_New(initial); if (cell == NULL) { - GOTO_ERROR(error); + JUMP_TO_ERROR(); } - SETLOCAL(oparg, cell); + SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); break; } case _DELETE_DEREF: { oparg = CURRENT_OPARG(); - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); // Can't use ERROR_IF here. // Fortunately we don't need its superpower. + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_ERROR(); } - PyCell_SET(cell, NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(oldobj); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } case _LOAD_FROM_DICT_OR_DEREF: { - PyObject *class_dict; - PyObject *value; + _PyStackRef class_dict_st; + _PyStackRef value; oparg = CURRENT_OPARG(); - class_dict = stack_pointer[-1]; + class_dict_st = stack_pointer[-1]; + PyObject *value_o; PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { - GOTO_ERROR(error); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_ERROR(); } - if (!value) { - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_ERROR(); } - Py_INCREF(value); } - Py_DECREF(class_dict); + PyStackRef_CLOSE(class_dict_st); + value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[-1] = value; break; } case _LOAD_DEREF: { - PyObject *value; + _PyStackRef value; oparg = CURRENT_OPARG(); - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - if (true) goto error_tier_two; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (true) JUMP_TO_ERROR(); } - Py_INCREF(value); + value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_DEREF: { - PyObject *v; + _PyStackRef v; oparg = CURRENT_OPARG(); v = stack_pointer[-1]; - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - PyCell_SET(cell, v); - Py_XDECREF(oldobj); + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -1454,320 +2208,424 @@ oparg = CURRENT_OPARG(); /* Copy closure variables to free variables */ PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyFunction_Check(frame->f_funcobj)); - PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; assert(oparg == co->co_nfreevars); int offset = co->co_nlocalsplus - oparg; for (int i = 0; i < oparg; ++i) { PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = Py_NewRef(o); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); } break; } case _BUILD_STRING: { - PyObject **pieces; - PyObject *str; + _PyStackRef *pieces; + _PyStackRef str; oparg = CURRENT_OPARG(); pieces = &stack_pointer[-oparg]; - str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); + STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + if (CONVERSION_FAILED(pieces_o)) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(pieces[_i]); + } + if (true) JUMP_TO_ERROR(); + } + PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); for (int _i = oparg; --_i >= 0;) { - Py_DECREF(pieces[_i]); + PyStackRef_CLOSE(pieces[_i]); } - if (str == NULL) { stack_pointer += -oparg; goto error_tier_two; } + if (str_o == NULL) JUMP_TO_ERROR(); + str = PyStackRef_FromPyObjectSteal(str_o); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _BUILD_TUPLE: { - PyObject **values; - PyObject *tup; + _PyStackRef *values; + _PyStackRef tup; oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg]; - tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) { stack_pointer += -oparg; goto error_tier_two; } + PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); + if (tup_o == NULL) { + JUMP_TO_ERROR(); + } + tup = PyStackRef_FromPyObjectSteal(tup_o); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _BUILD_LIST: { - PyObject **values; - PyObject *list; + _PyStackRef *values; + _PyStackRef list; oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg]; - list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) { stack_pointer += -oparg; goto error_tier_two; } + PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); + if (list_o == NULL) { + JUMP_TO_ERROR(); + } + list = PyStackRef_FromPyObjectSteal(list_o); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _LIST_EXTEND: { - PyObject *iterable; - PyObject *list; + _PyStackRef iterable_st; + _PyStackRef list_st; oparg = CURRENT_OPARG(); - iterable = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; + iterable_st = stack_pointer[-1]; + list_st = stack_pointer[-2 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); if (none_val == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Clear(tstate); _PyErr_Format(tstate, PyExc_TypeError, "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); } - Py_DECREF(iterable); - if (true) goto pop_1_error_tier_two; + PyStackRef_CLOSE(iterable_st); + if (true) JUMP_TO_ERROR(); } assert(Py_IsNone(none_val)); - Py_DECREF(iterable); + PyStackRef_CLOSE(iterable_st); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _SET_UPDATE: { - PyObject *iterable; - PyObject *set; + _PyStackRef iterable; + _PyStackRef set; oparg = CURRENT_OPARG(); iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; - int err = _PySet_Update(set, iterable); - Py_DECREF(iterable); - if (err < 0) goto pop_1_error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(iterable); + if (err < 0) JUMP_TO_ERROR(); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BUILD_SET: { - PyObject **values; - PyObject *set; + _PyStackRef *values; + _PyStackRef set; oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg]; - set = PySet_New(NULL); - if (set == NULL) - GOTO_ERROR(error); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *set_o = PySet_New(NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (set_o == NULL) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + if (true) JUMP_TO_ERROR(); + } int err = 0; for (int i = 0; i < oparg; i++) { - PyObject *item = values[i]; - if (err == 0) - err = PySet_Add(set, item); - Py_DECREF(item); + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PySet_Add(set_o, PyStackRef_AsPyObjectBorrow(values[i])); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); } if (err != 0) { - Py_DECREF(set); - if (true) { stack_pointer += -oparg; goto error_tier_two; } + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(set_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (true) JUMP_TO_ERROR(); } + set = PyStackRef_FromPyObjectSteal(set_o); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _BUILD_MAP: { - PyObject **values; - PyObject *map; + _PyStackRef *values; + _PyStackRef map; oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg*2]; - map = _PyDict_FromItems( - values, 2, - values+1, 2, - oparg); + STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + if (CONVERSION_FAILED(values_o)) { + for (int _i = oparg*2; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + if (true) JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *map_o = _PyDict_FromItems( + values_o, 2, + values_o+1, 2, + oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); for (int _i = oparg*2; --_i >= 0;) { - Py_DECREF(values[_i]); + PyStackRef_CLOSE(values[_i]); } - if (map == NULL) { stack_pointer += -oparg*2; goto error_tier_two; } + if (map_o == NULL) JUMP_TO_ERROR(); + map = PyStackRef_FromPyObjectSteal(map_o); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; + assert(WITHIN_STACK_BOUNDS()); break; } case _SETUP_ANNOTATIONS: { - int err; PyObject *ann_dict; if (LOCALS() == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); - if (true) goto error_tier_two; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (true) JUMP_TO_ERROR(); } /* check if __annotations__ in locals()... */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) JUMP_TO_ERROR(); if (ann_dict == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); ann_dict = PyDict_New(); - if (ann_dict == NULL) goto error_tier_two; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (ann_dict == NULL) JUMP_TO_ERROR(); + _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); Py_DECREF(ann_dict); - if (err) goto error_tier_two; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) JUMP_TO_ERROR(); } else { + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); } break; } - case _BUILD_CONST_KEY_MAP: { - PyObject *keys; - PyObject **values; - PyObject *map; - oparg = CURRENT_OPARG(); - keys = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - if (!PyTuple_CheckExact(keys) || - PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { - _PyErr_SetString(tstate, PyExc_SystemError, - "bad BUILD_CONST_KEY_MAP keys argument"); - GOTO_ERROR(error); // Pop the keys and values. - } - map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); - for (int _i = oparg; --_i >= 0;) { - Py_DECREF(values[_i]); - } - Py_DECREF(keys); - if (map == NULL) { stack_pointer += -1 - oparg; goto error_tier_two; } - stack_pointer[-1 - oparg] = map; - stack_pointer += -oparg; - break; - } - case _DICT_UPDATE: { - PyObject *update; - PyObject *dict; + _PyStackRef update; + _PyStackRef dict; oparg = CURRENT_OPARG(); update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; - if (PyDict_Update(dict, update) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Update(dict_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); + Py_TYPE(update_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); } - Py_DECREF(update); - if (true) goto pop_1_error_tier_two; + PyStackRef_CLOSE(update); + if (true) JUMP_TO_ERROR(); } - Py_DECREF(update); + PyStackRef_CLOSE(update); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _DICT_MERGE: { - PyObject *update; - PyObject *dict; - PyObject *callable; + _PyStackRef update; + _PyStackRef dict; + _PyStackRef callable; oparg = CURRENT_OPARG(); update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; callable = stack_pointer[-5 - (oparg - 1)]; - if (_PyDict_MergeEx(dict, update, 2) < 0) { - _PyEval_FormatKwargsError(tstate, callable, update); - Py_DECREF(update); - if (true) goto pop_1_error_tier_two; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_MergeEx(dict_o, update_o, 2); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatKwargsError(tstate, callable_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(update); + if (true) JUMP_TO_ERROR(); } - Py_DECREF(update); + PyStackRef_CLOSE(update); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _MAP_ADD: { - PyObject *value; - PyObject *key; - PyObject *dict; + _PyStackRef value; + _PyStackRef key; + _PyStackRef dict_st; oparg = CURRENT_OPARG(); value = stack_pointer[-1]; key = stack_pointer[-2]; - dict = stack_pointer[-3 - (oparg - 1)]; + dict_st = stack_pointer[-3 - (oparg - 1)]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2( + (PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(key), + PyStackRef_AsPyObjectSteal(value) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) JUMP_TO_ERROR(); stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } - /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 because it is instrumented */ case _LOAD_SUPER_ATTR_ATTR: { - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - oparg = CURRENT_OPARG(); - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; + _PyStackRef self_st; + _PyStackRef class_st; + _PyStackRef global_super_st; + _PyStackRef attr_st; + oparg = CURRENT_OPARG(); + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(!(oparg & 1)); - if (global_super != (PyObject *)&PySuper_Type) goto deoptimize; - if (!PyType_Check(class)) goto deoptimize; + if (global_super != (PyObject *)&PySuper_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyType_Check(class)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); - if (attr == NULL) goto pop_3_error_tier_two; - stack_pointer[-3] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (attr == NULL) JUMP_TO_ERROR(); + attr_st = PyStackRef_FromPyObjectSteal(attr); + stack_pointer[-3] = attr_st; stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_SUPER_ATTR_METHOD: { - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - PyObject *self_or_null; - oparg = CURRENT_OPARG(); - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; + _PyStackRef self_st; + _PyStackRef class_st; + _PyStackRef global_super_st; + _PyStackRef attr; + _PyStackRef self_or_null; + oparg = CURRENT_OPARG(); + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); - if (global_super != (PyObject *)&PySuper_Type) goto deoptimize; - if (!PyType_Check(class)) goto deoptimize; + if (global_super != (PyObject *)&PySuper_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyType_Check(class)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; - attr = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - Py_DECREF(global_super); - Py_DECREF(class); - if (attr == NULL) { - Py_DECREF(self); - if (true) goto pop_3_error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + JUMP_TO_ERROR(); } if (method_found) { - self_or_null = self; // transfer ownership + self_or_null = self_st; // transfer ownership } else { - Py_DECREF(self); - self_or_null = NULL; + PyStackRef_CLOSE(self_st); + self_or_null = PyStackRef_NULL; } + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(global_super_st); + attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_ATTR: { - PyObject *owner; - PyObject *attr; - PyObject *self_or_null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *self_or_null; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; + self_or_null = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *attr_o; if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ - attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { + attr_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_meth) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN */ - assert(attr != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership + assert(attr_o != NULL); // No errors on this branch + self_or_null[0] = owner; // Transfer ownership } else { /* meth is not an unbound method (but a regular attr, or @@ -1776,1820 +2634,3062 @@ CALL that it's not a method call. meth | NULL | arg1 | ... | argN */ - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error_tier_two; - self_or_null = NULL; + PyStackRef_CLOSE(owner); + if (attr_o == NULL) JUMP_TO_ERROR(); + self_or_null[0] = PyStackRef_NULL; } } else { /* Classic, pushes one value. */ - attr = PyObject_GetAttr(owner, name); - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(owner); + if (attr_o == NULL) JUMP_TO_ERROR(); } + attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = self_or_null; - stack_pointer += (oparg & 1); + stack_pointer += (oparg&1); + assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_TYPE_VERSION: { - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND(); - PyTypeObject *tp = Py_TYPE(owner); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - if (tp->tp_version_tag != type_version) goto side_exit; + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } - case _CHECK_MANAGED_OBJECT_HAS_VALUES: { - PyObject *owner; + case _GUARD_TYPE_VERSION_AND_LOCK: { + _PyStackRef owner; owner = stack_pointer[-1]; - assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) goto deoptimize; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + if (!LOCK_OBJECT(owner_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } break; } - case _LOAD_ATTR_INSTANCE_VALUE_0: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - (void)null; + case _CHECK_MANAGED_OBJECT_HAS_VALUES: { + _PyStackRef owner; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)CURRENT_OPERAND(); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - attr = _PyDictOrValues_GetValues(dorv)->values[index]; - if (attr == NULL) goto deoptimize; - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - stack_pointer[-1] = attr; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } - case _LOAD_ATTR_INSTANCE_VALUE_1: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - (void)null; + case _LOAD_ATTR_INSTANCE_VALUE: { + _PyStackRef owner; + _PyStackRef attr; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)CURRENT_OPERAND(); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - attr = _PyDictOrValues_GetValues(dorv)->values[index]; - if (attr == NULL) goto deoptimize; + uint16_t offset = (uint16_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; - stack_pointer[0] = null; - stack_pointer += 1; break; } - /* _LOAD_ATTR_INSTANCE_VALUE is split on (oparg & 1) */ - - case _CHECK_ATTR_MODULE: { - PyObject *owner; + case _CHECK_ATTR_MODULE_PUSH_KEYS: { + _PyStackRef owner; + PyDictKeysObject *mod_keys; owner = stack_pointer[-1]; - uint32_t dict_version = (uint32_t)CURRENT_OPERAND(); - if (!PyModule_CheckExact(owner)) goto deoptimize; - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + uint32_t dict_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); - if (dict->ma_keys->dk_version != dict_version) goto deoptimize; - break; - } - - case _LOAD_ATTR_MODULE: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; - uint16_t index = (uint16_t)CURRENT_OPERAND(); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; - assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < dict->ma_keys->dk_nentries); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - attr = ep->me_value; - if (attr == NULL) goto deoptimize; + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + mod_keys = keys; + stack_pointer[0].bits = (uintptr_t)mod_keys; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_ATTR_MODULE_FROM_KEYS: { + PyDictKeysObject *mod_keys; + _PyStackRef owner; + _PyStackRef attr; + mod_keys = (PyDictKeysObject *)stack_pointer[-1].bits; + owner = stack_pointer[-2]; + uint16_t index = (uint16_t)CURRENT_OPERAND0(); + assert(mod_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + // Clear mod_keys from stack in case we need to deopt + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #else + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); break; } case _CHECK_ATTR_WITH_HINT: { - PyObject *owner; + _PyStackRef owner; + PyDictObject *dict; owner = stack_pointer[-1]; - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - if (_PyDictOrValues_IsValues(dorv)) goto deoptimize; - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - if (dict == NULL) goto deoptimize; - assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); + if (dict_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + assert(PyDict_CheckExact((PyObject *)dict_o)); + dict = dict_o; + stack_pointer[0].bits = (uintptr_t)dict; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_ATTR_WITH_HINT: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + PyDictObject *dict; + _PyStackRef owner; + _PyStackRef attr; oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; - uint16_t hint = (uint16_t)CURRENT_OPERAND(); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - if (hint >= (size_t)dict->ma_keys->dk_nentries) goto deoptimize; + dict = (PyDictObject *)stack_pointer[-1].bits; + owner = stack_pointer[-2]; + uint16_t hint = (uint16_t)CURRENT_OPERAND0(); + PyObject *attr_o; + if (!LOCK_OBJECT(dict)) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + if (hint >= (size_t)dict->ma_keys->dk_nentries) { + UNLOCK_OBJECT(dict); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (DK_IS_UNICODE(dict->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) goto deoptimize; - attr = ep->me_value; + if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { + UNLOCK_OBJECT(dict); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } - else { - PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) goto deoptimize; - attr = ep->me_value; + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + attr_o = ep->me_value; + if (attr_o == NULL) { + UNLOCK_OBJECT(dict); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } - if (attr == NULL) goto deoptimize; - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - break; - } - - case _LOAD_ATTR_SLOT_0: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - (void)null; - owner = stack_pointer[-1]; - uint16_t index = (uint16_t)CURRENT_OPERAND(); - char *addr = (char *)owner + index; - attr = *(PyObject **)addr; - if (attr == NULL) goto deoptimize; STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - stack_pointer[-1] = attr; + attr = PyStackRef_FromPyObjectNew(attr_o); + UNLOCK_OBJECT(dict); + PyStackRef_CLOSE(owner); + stack_pointer[-2] = attr; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_ATTR_SLOT_1: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - (void)null; + case _LOAD_ATTR_SLOT: { + _PyStackRef owner; + _PyStackRef attr; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)CURRENT_OPERAND(); - char *addr = (char *)owner + index; - attr = *(PyObject **)addr; - if (attr == NULL) goto deoptimize; + uint16_t index = (uint16_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + if (!increfed) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; - stack_pointer[0] = null; - stack_pointer += 1; break; } - /* _LOAD_ATTR_SLOT is split on (oparg & 1) */ - case _CHECK_ATTR_CLASS: { - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND(); - if (!PyType_Check(owner)) goto deoptimize; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } assert(type_version != 0); - if (((PyTypeObject *)owner)->tp_version_tag != type_version) goto deoptimize; + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } - case _LOAD_ATTR_CLASS_0: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - (void)null; + case _LOAD_ATTR_CLASS: { + _PyStackRef owner; + _PyStackRef attr; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - attr = Py_NewRef(descr); - null = NULL; - Py_DECREF(owner); + attr = PyStackRef_FromPyObjectNew(descr); + PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; break; } - case _LOAD_ATTR_CLASS_1: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - (void)null; + case _LOAD_ATTR_PROPERTY_FRAME: { + _PyStackRef owner; + _PyInterpreterFrame *new_frame; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject *fget = (PyObject *)CURRENT_OPERAND0(); + assert((oparg & 1) == 0); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + PyCodeObject *code = (PyCodeObject *)f->func_code; + if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (code->co_kwonlyargcount) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (code->co_argcount != 1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = Py_NewRef(descr); - null = NULL; - Py_DECREF(owner); - stack_pointer[-1] = attr; - stack_pointer[0] = null; - stack_pointer += 1; + new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); + new_frame->localsplus[0] = owner; + stack_pointer[-1].bits = (uintptr_t)new_frame; break; } - /* _LOAD_ATTR_CLASS is split on (oparg & 1) */ - - /* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 */ + /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it has unused cache entries */ - /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */ - - case _GUARD_DORV_VALUES: { - PyObject *owner; + case _GUARD_DORV_NO_DICT: { + _PyStackRef owner; owner = stack_pointer[-1]; - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - if (!_PyDictOrValues_IsValues(dorv)) goto deoptimize; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } break; } case _STORE_ATTR_INSTANCE_VALUE: { - PyObject *owner; - PyObject *value; + _PyStackRef owner; + _PyStackRef value; owner = stack_pointer[-1]; value = stack_pointer[-2]; - uint16_t index = (uint16_t)CURRENT_OPERAND(); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + uint16_t offset = (uint16_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); STAT_INC(STORE_ATTR, hit); - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyObject *old_value = values->values[index]; - values->values[index] = value; + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *old_value = *value_ptr; + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); if (old_value == NULL) { + PyDictValues *values = _PyObject_InlineValues(owner_o); + Py_ssize_t index = value_ptr - values->values; _PyDictValues_AddToInsertionOrder(values, index); } - else { - Py_DECREF(old_value); - } - Py_DECREF(owner); + UNLOCK_OBJECT(owner_o); + PyStackRef_CLOSE(owner); stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } - /* _STORE_ATTR_WITH_HINT is not a viable micro-op for tier 2 */ + case _STORE_ATTR_WITH_HINT: { + _PyStackRef owner; + _PyStackRef value; + oparg = CURRENT_OPARG(); + owner = stack_pointer[-1]; + value = stack_pointer[-2]; + uint16_t hint = (uint16_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!LOCK_OBJECT(dict)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (dict != _PyObject_GetManagedDict(owner_o)) { + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #endif + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + !DK_IS_UNICODE(dict->ma_keys)) { + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + PyObject *old_value = ep->me_value; + if (old_value == NULL) { + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); + // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, + // when dict only holds the strong reference to value in ep->me_value. + STAT_INC(STORE_ATTR, hit); + PyStackRef_CLOSE(owner); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + break; + } case _STORE_ATTR_SLOT: { - PyObject *owner; - PyObject *value; + _PyStackRef owner; + _PyStackRef value; owner = stack_pointer[-1]; value = stack_pointer[-2]; - uint16_t index = (uint16_t)CURRENT_OPERAND(); - char *addr = (char *)owner + index; + uint16_t index = (uint16_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!LOCK_OBJECT(owner_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = value; - Py_XDECREF(old_value); - Py_DECREF(owner); + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); + PyStackRef_CLOSE(owner); stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } case _COMPARE_OP: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert((oparg >> 5) <= Py_GE); - res = PyObject_RichCompare(left, right, oparg >> 5); - Py_DECREF(left); - Py_DECREF(right); - if (res == NULL) goto pop_2_error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res_o == NULL) JUMP_TO_ERROR(); if (oparg & 16) { - int res_bool = PyObject_IsTrue(res); - Py_DECREF(res); - if (res_bool < 0) goto pop_2_error_tier_two; - res = res_bool ? Py_True : Py_False; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_bool < 0) JUMP_TO_ERROR(); + res = res_bool ? PyStackRef_True : PyStackRef_False; } - stack_pointer[-2] = res; - stack_pointer += -1; + else { + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _COMPARE_OP_FLOAT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? Py_True : Py_False; + PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _COMPARE_OP_INT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!_PyLong_IsCompact((PyLongObject *)left)) goto deoptimize; - if (!_PyLong_IsCompact((PyLongObject *)right)) goto deoptimize; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!_PyLong_IsCompact((PyLongObject *)left_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!_PyLong_IsCompact((PyLongObject *)right_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && - _PyLong_DigitCount((PyLongObject *)right) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + _PyLong_DigitCount((PyLongObject *)right_o) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - res = (sign_ish & oparg) ? Py_True : Py_False; + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _COMPARE_OP_STR: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left, right); + int eq = _PyUnicode_Equal(left_o, right_o); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); assert(eq == 0 || eq == 1); assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _IS_OP: { - PyObject *right; - PyObject *left; - PyObject *b; + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - int res = Py_Is(left, right) ^ oparg; - Py_DECREF(left); - Py_DECREF(right); - b = res ? Py_True : Py_False; + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _CONTAINS_OP: { - PyObject *right; - PyObject *left; - PyObject *b; + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - int res = PySequence_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); - if (res < 0) goto pop_2_error_tier_two; - b = (res ^ oparg) ? Py_True : Py_False; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PySequence_Contains(right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) JUMP_TO_ERROR(); + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _CHECK_EG_MATCH: { - PyObject *match_type; - PyObject *exc_value; - PyObject *rest; - PyObject *match; - match_type = stack_pointer[-1]; - exc_value = stack_pointer[-2]; - if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { - Py_DECREF(exc_value); - Py_DECREF(match_type); - if (true) goto pop_2_error_tier_two; - } - match = NULL; - rest = NULL; - int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); - Py_DECREF(exc_value); - Py_DECREF(match_type); - if (res < 0) goto pop_2_error_tier_two; - assert((match == NULL) == (rest == NULL)); - if (match == NULL) goto pop_2_error_tier_two; - if (!Py_IsNone(match)) { - PyErr_SetHandledException(match); + case _CONTAINS_OP_SET: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + oparg = CURRENT_OPARG(); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(CONTAINS_OP, hit); + // Note: both set and frozenset use the same seq_contains method! + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PySet_Contains((PySetObject *)right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) JUMP_TO_ERROR(); + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CONTAINS_OP_DICT: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + oparg = CURRENT_OPARG(); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyDict_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); } + STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyDict_Contains(right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) JUMP_TO_ERROR(); + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_EG_MATCH: { + _PyStackRef match_type_st; + _PyStackRef exc_value_st; + _PyStackRef rest; + _PyStackRef match; + match_type_st = stack_pointer[-1]; + exc_value_st = stack_pointer[-2]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); + if (true) JUMP_TO_ERROR(); + } + PyObject *match_o = NULL; + PyObject *rest_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, + &match_o, &rest_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); + if (res < 0) JUMP_TO_ERROR(); + assert((match_o == NULL) == (rest_o == NULL)); + if (match_o == NULL) JUMP_TO_ERROR(); + if (!Py_IsNone(match_o)) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_SetHandledException(match_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); stack_pointer[-2] = rest; stack_pointer[-1] = match; break; } case _CHECK_EXC_MATCH: { - PyObject *right; - PyObject *left; - PyObject *b; + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; right = stack_pointer[-1]; left = stack_pointer[-2]; - assert(PyExceptionInstance_Check(left)); - if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); - if (true) goto pop_1_error_tier_two; - } - int res = PyErr_GivenExceptionMatches(left, right); - Py_DECREF(right); - b = res ? Py_True : Py_False; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyExceptionInstance_Check(left_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptTypeValid(tstate, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + PyStackRef_CLOSE(right); + if (true) JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyErr_GivenExceptionMatches(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(right); + b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[-1] = b; break; } - /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ + case _IMPORT_NAME: { + _PyStackRef fromlist; + _PyStackRef level; + _PyStackRef res; + oparg = CURRENT_OPARG(); + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportName(tstate, frame, name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(level); + PyStackRef_CLOSE(fromlist); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _IMPORT_FROM: { + _PyStackRef from; + _PyStackRef res; + oparg = CURRENT_OPARG(); + from = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is replaced */ - /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ + /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is replaced */ case _IS_NONE: { - PyObject *value; - PyObject *b; + _PyStackRef value; + _PyStackRef b; value = stack_pointer[-1]; - if (Py_IsNone(value)) { - b = Py_True; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; } else { - b = Py_False; - Py_DECREF(value); + b = PyStackRef_False; + PyStackRef_CLOSE(value); } stack_pointer[-1] = b; break; } case _GET_LEN: { - PyObject *obj; - PyObject *len_o; + _PyStackRef obj; + _PyStackRef len; obj = stack_pointer[-1]; // PUSH(len(TOS)) - Py_ssize_t len_i = PyObject_Length(obj); - if (len_i < 0) goto error_tier_two; - len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) goto error_tier_two; - stack_pointer[0] = len_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) JUMP_TO_ERROR(); + PyObject *len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) JUMP_TO_ERROR(); + len = PyStackRef_FromPyObjectSteal(len_o); + stack_pointer[0] = len; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _MATCH_CLASS: { - PyObject *names; - PyObject *type; - PyObject *subject; - PyObject *attrs; + _PyStackRef names; + _PyStackRef type; + _PyStackRef subject; + _PyStackRef attrs; oparg = CURRENT_OPARG(); names = stack_pointer[-1]; type = stack_pointer[-2]; subject = stack_pointer[-3]; // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. - assert(PyTuple_CheckExact(names)); - attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); - Py_DECREF(subject); - Py_DECREF(type); - Py_DECREF(names); - if (attrs) { - assert(PyTuple_CheckExact(attrs)); // Success! + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(subject); + PyStackRef_CLOSE(type); + PyStackRef_CLOSE(names); + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); // Success! + attrs = PyStackRef_FromPyObjectSteal(attrs_o); } else { - if (_PyErr_Occurred(tstate)) goto pop_3_error_tier_two; + if (_PyErr_Occurred(tstate)) JUMP_TO_ERROR(); // Error! - attrs = Py_None; // Failure! + attrs = PyStackRef_None; // Failure! } stack_pointer[-3] = attrs; stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } case _MATCH_MAPPING: { - PyObject *subject; - PyObject *res; + _PyStackRef subject; + _PyStackRef res; subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? Py_True : Py_False; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _MATCH_SEQUENCE: { - PyObject *subject; - PyObject *res; + _PyStackRef subject; + _PyStackRef res; subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? Py_True : Py_False; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _MATCH_KEYS: { - PyObject *keys; - PyObject *subject; - PyObject *values_or_none; + _PyStackRef keys; + _PyStackRef subject; + _PyStackRef values_or_none; keys = stack_pointer[-1]; subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = _PyEval_MatchKeys(tstate, subject, keys); - if (values_or_none == NULL) goto error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (values_or_none_o == NULL) JUMP_TO_ERROR(); + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _GET_ITER: { - PyObject *iterable; - PyObject *iter; + _PyStackRef iterable; + _PyStackRef iter; iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ - iter = PyObject_GetIter(iterable); - Py_DECREF(iterable); - if (iter == NULL) goto pop_1_error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(iterable); + if (iter_o == NULL) JUMP_TO_ERROR(); + iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; break; } case _GET_YIELD_FROM_ITER: { - PyObject *iterable; - PyObject *iter; + _PyStackRef iterable; + _PyStackRef iter; iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ - if (PyCoro_CheckExact(iterable)) { + PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); + if (PyCoro_CheckExact(iterable_o)) { /* `iterable` is a coroutine */ if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { /* and it is used in a 'yield from' expression of a regular generator. */ + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); - GOTO_ERROR(error); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_ERROR(); } iter = iterable; } - else if (PyGen_CheckExact(iterable)) { - iter = iterable; - } else { - /* `iterable` is not a generator. */ - iter = PyObject_GetIter(iterable); - if (iter == NULL) { - GOTO_ERROR(error); + if (PyGen_CheckExact(iterable_o)) { + iter = iterable; + } + else { + /* `iterable` is not a generator. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(iterable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + JUMP_TO_ERROR(); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + PyStackRef_CLOSE(iterable); } - Py_DECREF(iterable); } stack_pointer[-1] = iter; break; } - /* _FOR_ITER is not a viable micro-op for tier 2 */ + /* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */ case _FOR_ITER_TIER_TWO: { - PyObject *iter; - PyObject *next; + _PyStackRef iter; + _PyStackRef next; iter = stack_pointer[-1]; /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next == NULL) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_o == NULL) { if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!matches) { + JUMP_TO_ERROR(); } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr); _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); } /* iterator ended normally */ - Py_DECREF(iter); - STACK_SHRINK(1); - /* The translator sets the deopt target just past END_FOR */ - if (true) goto deoptimize; + /* The translator sets the deopt target just past the matching END_FOR */ + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } + next = PyStackRef_FromPyObjectSteal(next_o); // Common case: no jump, leave it to the code generator stack_pointer[0] = next; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 because it is instrumented */ case _ITER_CHECK_LIST: { - PyObject *iter; + _PyStackRef iter; iter = stack_pointer[-1]; - if (Py_TYPE(iter) != &PyListIter_Type) goto deoptimize; + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } - /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 */ + /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 because it is replaced */ case _GUARD_NOT_EXHAUSTED_LIST: { - PyObject *iter; + _PyStackRef iter; iter = stack_pointer[-1]; - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); PyListObject *seq = it->it_seq; - if (seq == NULL) goto deoptimize; - if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) goto deoptimize; + if (seq == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { + it->it_index = -1; + if (1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } break; } case _ITER_NEXT_LIST: { - PyObject *iter; - PyObject *next; + _PyStackRef iter; + _PyStackRef next; iter = stack_pointer[-1]; - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); PyListObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyList_GET_SIZE(seq)); - next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); stack_pointer[0] = next; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _ITER_CHECK_TUPLE: { - PyObject *iter; + _PyStackRef iter; iter = stack_pointer[-1]; - if (Py_TYPE(iter) != &PyTupleIter_Type) goto deoptimize; + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } - /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 */ + /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 because it is replaced */ case _GUARD_NOT_EXHAUSTED_TUPLE: { - PyObject *iter; + _PyStackRef iter; iter = stack_pointer[-1]; - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; - if (seq == NULL) goto deoptimize; - if (it->it_index >= PyTuple_GET_SIZE(seq)) goto deoptimize; + if (seq == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (it->it_index >= PyTuple_GET_SIZE(seq)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _ITER_NEXT_TUPLE: { - PyObject *iter; - PyObject *next; + _PyStackRef iter; + _PyStackRef next; iter = stack_pointer[-1]; - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); stack_pointer[0] = next; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _ITER_CHECK_RANGE: { - PyObject *iter; + _PyStackRef iter; iter = stack_pointer[-1]; - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - if (Py_TYPE(r) != &PyRangeIter_Type) goto deoptimize; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } - /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 */ + /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 because it is replaced */ case _GUARD_NOT_EXHAUSTED_RANGE: { - PyObject *iter; + _PyStackRef iter; iter = stack_pointer[-1]; - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); - if (r->len <= 0) goto deoptimize; + if (r->len <= 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _ITER_NEXT_RANGE: { - PyObject *iter; - PyObject *next; + _PyStackRef iter; + _PyStackRef next; iter = stack_pointer[-1]; - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); assert(r->len > 0); long value = r->start; r->start = value + r->step; r->len--; - next = PyLong_FromLong(value); - if (next == NULL) goto error_tier_two; + PyObject *res = PyLong_FromLong(value); + if (res == NULL) JUMP_TO_ERROR(); + next = PyStackRef_FromPyObjectSteal(res); stack_pointer[0] = next; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - /* _FOR_ITER_GEN is not a viable micro-op for tier 2 */ - - case _BEFORE_ASYNC_WITH: { - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol", - Py_TYPE(mgr)->tp_name); - } - GOTO_ERROR(error); - } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol " - "(missed __aexit__ method)", - Py_TYPE(mgr)->tp_name); - } - Py_DECREF(enter); - GOTO_ERROR(error); + case _FOR_ITER_GEN_FRAME: { + _PyStackRef iter; + _PyInterpreterFrame *gen_frame; + oparg = CURRENT_OPARG(); + iter = stack_pointer[-1]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(gen) != &PyGen_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); } - Py_DECREF(mgr); - res = PyObject_CallNoArgs(enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error_tier_two; + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); } - stack_pointer[-1] = exit; - stack_pointer[0] = res; + STAT_INC(FOR_ITER, hit); + gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, PyStackRef_None); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + gen_frame->previous = frame; + // oparg is the return offset from the next instruction. + frame->return_offset = (uint16_t)( 2 + oparg); + stack_pointer[0].bits = (uintptr_t)gen_frame; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _BEFORE_WITH: { - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - /* pop the context manager, push its __exit__ and the - * value returned from calling its __enter__ - */ - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol", - Py_TYPE(mgr)->tp_name); - } - GOTO_ERROR(error); - } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); - if (exit == NULL) { + case _LOAD_SPECIAL: { + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null; + oparg = CURRENT_OPARG(); + owner = stack_pointer[-1]; + assert(oparg <= SPECIAL_MAX); + PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); + PyObject *name = _Py_SpecialMethods[oparg].name; + PyObject *self_or_null_o; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = _PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { if (!_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol " - "(missed __exit__ method)", - Py_TYPE(mgr)->tp_name); + _Py_SpecialMethods[oparg].error, + Py_TYPE(owner_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); } - Py_DECREF(enter); - GOTO_ERROR(error); - } - Py_DECREF(mgr); - res = PyObject_CallNoArgs(enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error_tier_two; + if (true) JUMP_TO_ERROR(); } - stack_pointer[-1] = exit; - stack_pointer[0] = res; - stack_pointer += 1; + attr = PyStackRef_FromPyObjectSteal(attr_o); + self_or_null = self_or_null_o == NULL ? + PyStackRef_NULL : PyStackRef_FromPyObjectSteal(self_or_null_o); + stack_pointer[0] = attr; + stack_pointer[1] = self_or_null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); break; } case _WITH_EXCEPT_START: { - PyObject *val; - PyObject *lasti; - PyObject *exit_func; - PyObject *res; + _PyStackRef val; + _PyStackRef lasti; + _PyStackRef exit_self; + _PyStackRef exit_func; + _PyStackRef res; val = stack_pointer[-1]; lasti = stack_pointer[-3]; - exit_func = stack_pointer[-4]; + exit_self = stack_pointer[-4]; + exit_func = stack_pointer[-5]; /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception - lasti: THIRD = lasti of exception in exc_info() - - exit_func: FOURTH = the context.__exit__ bound method + - exit_self: FOURTH = the context or NULL + - exit_func: FIFTH = the context.__exit__ function or context.__exit__ bound method We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). Then we push the __exit__ return value. */ PyObject *exc, *tb; - assert(val && PyExceptionInstance_Check(val)); - exc = PyExceptionInstance_Class(val); - tb = PyException_GetTraceback(val); + PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); + PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); + assert(val_o && PyExceptionInstance_Check(val_o)); + exc = PyExceptionInstance_Class(val_o); + tb = PyException_GetTraceback(val_o); if (tb == NULL) { tb = Py_None; } else { + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(tb); + stack_pointer = _PyFrame_GetStackPointer(frame); } - assert(PyLong_Check(lasti)); + assert(PyStackRef_LongCheck(lasti)); (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[4] = {NULL, exc, val, tb}; - res = PyObject_Vectorcall(exit_func, stack + 1, - 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - if (res == NULL) goto error_tier_two; + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + int has_self = !PyStackRef_IsNull(exit_self); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _PUSH_EXC_INFO: { - PyObject *new_exc; - PyObject *prev_exc; - new_exc = stack_pointer[-1]; + _PyStackRef exc; + _PyStackRef prev_exc; + _PyStackRef new_exc; + exc = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { - prev_exc = exc_info->exc_value; + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); } else { - prev_exc = Py_None; + prev_exc = PyStackRef_None; } - assert(PyExceptionInstance_Check(new_exc)); - exc_info->exc_value = Py_NewRef(new_exc); + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: { - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) goto deoptimize; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _GUARD_KEYS_VERSION: { - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t keys_version = (uint32_t)CURRENT_OPERAND(); - PyTypeObject *owner_cls = Py_TYPE(owner); + uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - if (owner_heap_type->ht_cached_keys->dk_version != keys_version) goto deoptimize; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _LOAD_ATTR_METHOD_WITH_VALUES: { - PyObject *owner; - PyObject *attr; - PyObject *self = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); assert(oparg & 1); /* Cached method object */ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - attr = Py_NewRef(descr); - assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_ATTR_METHOD_NO_DICT: { - PyObject *owner; - PyObject *attr; - PyObject *self = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); assert(oparg & 1); - assert(Py_TYPE(owner)->tp_dictoffset == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - PyObject *owner; - PyObject *attr; + _PyStackRef owner; + _PyStackRef attr; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); stack_pointer[-1] = attr; break; } case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - PyObject *owner; - PyObject *attr; + _PyStackRef owner; + _PyStackRef attr; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); assert((oparg & 1) == 0); - assert(Py_TYPE(owner)->tp_dictoffset == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); stack_pointer[-1] = attr; break; } case _CHECK_ATTR_METHOD_LAZY_DICT: { - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; - Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; - assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)owner + dictoffset); + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ - if (dict != NULL) goto deoptimize; + if (dict != NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _LOAD_ATTR_METHOD_LAZY_DICT: { - PyObject *owner; - PyObject *attr; - PyObject *self = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _MAYBE_EXPAND_METHOD: { + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; + _PyStackRef *func; + _PyStackRef *maybe_self; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + func = &stack_pointer[-2 - oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); + } + break; + } + + /* _DO_CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + /* _MONITOR_CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + case _PY_FRAME_GENERAL: { + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; + _PyInterpreterFrame *new_frame; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + // The frame has stolen all the arguments from the stack. + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + JUMP_TO_ERROR(); + } + new_frame = temp; + stack_pointer[0].bits = (uintptr_t)new_frame; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_FUNCTION_VERSION: { + _PyStackRef *callable; + oparg = CURRENT_OPARG(); + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + if (!PyFunction_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _CHECK_FUNCTION_VERSION_INLINE: { + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND1(); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } - /* _INSTRUMENTED_CALL is not a viable micro-op for tier 2 */ + case _CHECK_METHOD_VERSION: { + _PyStackRef *null; + _PyStackRef *callable; + oparg = CURRENT_OPARG(); + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyStackRef_IsNull(null[0])) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } - /* _CALL is not a viable micro-op for tier 2 */ + case _EXPAND_METHOD: { + _PyStackRef *null; + _PyStackRef *callable; + _PyStackRef *method; + _PyStackRef *self; + oparg = CURRENT_OPARG(); + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + method = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + assert(PyStackRef_IsNull(null[0])); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable[0]; + method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method[0])); + PyStackRef_CLOSE(temp); + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE: { + _PyStackRef *callable; + oparg = CURRENT_OPARG(); + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + if (PyFunction_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _CALL_NON_PY_GENERAL: { + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - PyObject *null; - PyObject *callable; + _PyStackRef *null; + _PyStackRef *callable; oparg = CURRENT_OPARG(); - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - if (null != NULL) goto deoptimize; - if (Py_TYPE(callable) != &PyMethod_Type) goto deoptimize; + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + if (!PyStackRef_IsNull(null[0])) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - PyObject *callable; - PyObject *func; - PyObject *self; + _PyStackRef *null; + _PyStackRef *callable; + _PyStackRef *func; + _PyStackRef *self; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + func = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); STAT_INC(CALL, hit); - self = Py_NewRef(((PyMethodObject *)callable)->im_self); - stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS - func = Py_NewRef(((PyMethodObject *)callable)->im_func); - stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); - stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = self; + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + PyStackRef_CLOSE(temp); break; } case _CHECK_PEP_523: { - if (tstate->interp->eval_frame) goto deoptimize; + if (tstate->interp->eval_frame) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _CHECK_FUNCTION_EXACT_ARGS: { - PyObject *self_or_null; - PyObject *callable; - oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND(); - if (!PyFunction_Check(callable)) goto deoptimize; - PyFunctionObject *func = (PyFunctionObject *)callable; - if (func->func_version != func_version) goto deoptimize; + _PyStackRef *self_or_null; + _PyStackRef *callable; + oparg = CURRENT_OPARG(); + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (self_or_null != NULL)) goto deoptimize; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _CHECK_STACK_SPACE: { - PyObject *callable; + _PyStackRef *callable; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; - PyFunctionObject *func = (PyFunctionObject *)callable; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) goto deoptimize; - if (tstate->py_recursion_remaining <= 1) goto deoptimize; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _INIT_CALL_PY_EXACT_ARGS_0: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; _PyInterpreterFrame *new_frame; oparg = 0; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null[0]; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _INIT_CALL_PY_EXACT_ARGS_1: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; _PyInterpreterFrame *new_frame; oparg = 1; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null[0]; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _INIT_CALL_PY_EXACT_ARGS_2: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; _PyInterpreterFrame *new_frame; oparg = 2; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null[0]; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _INIT_CALL_PY_EXACT_ARGS_3: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; _PyInterpreterFrame *new_frame; oparg = 3; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null[0]; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _INIT_CALL_PY_EXACT_ARGS_4: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; _PyInterpreterFrame *new_frame; oparg = 4; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null[0]; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _INIT_CALL_PY_EXACT_ARGS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; _PyInterpreterFrame *new_frame; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null[0]; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _PUSH_FRAME: { _PyInterpreterFrame *new_frame; - new_frame = (_PyInterpreterFrame *)stack_pointer[-1]; + new_frame = (_PyInterpreterFrame *)stack_pointer[-1].bits; // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif + LLTRACE_RESUME_FRAME(); break; } - /* _CALL_PY_WITH_DEFAULTS is not a viable micro-op for tier 2 */ - case _CALL_TYPE_1: { - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; + _PyStackRef arg; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef res; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - if (null != NULL) goto deoptimize; - PyObject *obj = args[0]; - if (callable != (PyObject *)&PyType_Type) goto deoptimize; + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (callable_o != (PyObject *)&PyType_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(obj); - Py_DECREF(&PyType_Type); // I.e., callable - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); + PyStackRef_CLOSE(arg); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_STR_1: { - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; + _PyStackRef arg; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef res; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - if (null != NULL) goto deoptimize; - if (callable != (PyObject *)&PyUnicode_Type) goto deoptimize; + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (callable_o != (PyObject *)&PyUnicode_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PyObject_Str(arg); - Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Str(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(arg); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_TUPLE_1: { - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; + _PyStackRef arg; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef res; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - if (null != NULL) goto deoptimize; - if (callable != (PyObject *)&PyTuple_Type) goto deoptimize; + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (callable_o != (PyObject *)&PyTuple_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PySequence_Tuple(arg); - Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PySequence_Tuple(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(arg); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } - /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 */ + case _CHECK_AND_ALLOCATE_OBJECT: { + _PyStackRef *args; + _PyStackRef *null; + _PyStackRef *callable; + _PyStackRef *init; + _PyStackRef *self; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + init = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + if (!PyStackRef_IsNull(null[0])) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyType_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + assert(tp->tp_new == PyBaseObject_Type.tp_new); + assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_alloc == PyType_GenericAlloc); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; + PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); + PyCodeObject *code = (PyCodeObject *)init_func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *self_o = PyType_GenericAlloc(tp, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (self_o == NULL) { + JUMP_TO_ERROR(); + } + self[0] = PyStackRef_FromPyObjectSteal(self_o); + _PyStackRef temp = callable[0]; + init[0] = PyStackRef_FromPyObjectNew(init_func); + PyStackRef_CLOSE(temp); + break; + } + + case _CREATE_INIT_FRAME: { + _PyStackRef *args; + _PyStackRef *self; + _PyStackRef *init; + _PyInterpreterFrame *init_frame; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self = &stack_pointer[-1 - oparg]; + init = &stack_pointer[-2 - oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); + stack_pointer = _PyFrame_GetStackPointer(frame); + /* Push self onto stack of shim */ + shim->localsplus[0] = PyStackRef_DUP(self[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, init[0], NULL, args-1, oparg+1, NULL, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + _PyEval_FrameClearAndPop(tstate, shim); + JUMP_TO_ERROR(); + } + init_frame = temp; + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + /* Account for pushing the extra frame. + * We don't check recursion depth here, + * as it will be checked after start_frame */ + tstate->py_recursion_remaining--; + stack_pointer[0].bits = (uintptr_t)init_frame; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _EXIT_INIT_CHECK: { - PyObject *should_be_none; + _PyStackRef should_be_none; should_be_none = stack_pointer[-1]; assert(STACK_LEVEL() == 2); - if (should_be_none != Py_None) { + if (!PyStackRef_IsNone(should_be_none)) { + _PyFrame_SetStackPointer(frame, stack_pointer); PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", - Py_TYPE(should_be_none)->tp_name); - GOTO_ERROR(error); + Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_ERROR(); } stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_BUILTIN_CLASS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; + _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + if (!PyType_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; int total_args = oparg; - if (self_or_null != NULL) { - args--; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; total_args++; } - if (!PyType_Check(callable)) goto deoptimize; - PyTypeObject *tp = (PyTypeObject *)callable; - if (tp->tp_vectorcall == NULL) goto deoptimize; + if (tp->tp_vectorcall == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); - res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) JUMP_TO_ERROR(); } - Py_DECREF(tp); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_BUILTIN_O: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; + _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } - if (total_args != 1) goto deoptimize; - if (!PyCFunction_CheckExact(callable)) goto deoptimize; - if (PyCFunction_GET_FLAGS(callable) != METH_O) goto deoptimize; - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); + if (total_args != 1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyCFunction_CheckExact(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + // CPython promises to check all non-vectorcall function calls. + if (tstate->c_recursion_remaining <= 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); } - PyObject *arg = args[0]; - res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + _PyStackRef arg = args[0]; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); + stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(arg); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_BUILTIN_FAST: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; + _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; - if (self_or_null != NULL) { - args--; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; total_args++; } - if (!PyCFunction_CheckExact(callable)) goto deoptimize; - if (PyCFunction_GET_FLAGS(callable) != METH_FASTCALL) goto deoptimize; + if (!PyCFunction_CheckExact(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ - res = ((PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable), - args, + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable_o), + args_o, total_args); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; + _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; - if (self_or_null != NULL) { - args--; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; total_args++; } - if (!PyCFunction_CheckExact(callable)) goto deoptimize; - if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) goto deoptimize; + if (!PyCFunction_CheckExact(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); - /* res = func(self, args, nargs, kwnames) */ + /* res = func(self, arguments, nargs, kwnames) */ + _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); - res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + PyCFunction_GET_FUNCTION(callable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_LEN: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; + _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* len(o) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } - if (total_args != 1) goto deoptimize; + if (total_args != 1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyInterpreterState *interp = tstate->interp; - if (callable != interp->callable_cache.len) goto deoptimize; + if (callable_o != interp->callable_cache.len) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); - PyObject *arg = args[0]; + _PyStackRef arg_stackref = args[0]; + PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); + _PyFrame_SetStackPointer(frame, stack_pointer); Py_ssize_t len_i = PyObject_Length(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) { + JUMP_TO_ERROR(); + } + PyObject *res_o = PyLong_FromSsize_t(len_i); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { GOTO_ERROR(error); } - res = PyLong_FromSsize_t(len_i); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - Py_DECREF(arg); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + PyStackRef_CLOSE(arg_stackref); + PyStackRef_CLOSE(callable[0]); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_ISINSTANCE: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; + _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* isinstance(o, o2) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; - if (self_or_null != NULL) { - args--; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; total_args++; } - if (total_args != 2) goto deoptimize; + if (total_args != 2) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyInterpreterState *interp = tstate->interp; - if (callable != interp->callable_cache.isinstance) goto deoptimize; + if (callable_o != interp->callable_cache.isinstance) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); - PyObject *cls = args[1]; - PyObject *inst = args[0]; - int retval = PyObject_IsInstance(inst, cls); + _PyStackRef cls_stackref = arguments[1]; + _PyStackRef inst_stackref = arguments[0]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (retval < 0) { - GOTO_ERROR(error); + JUMP_TO_ERROR(); + } + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); } - res = PyBool_FromLong(retval); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(inst); - Py_DECREF(cls); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_LIST_APPEND: { + _PyStackRef arg; + _PyStackRef self; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + arg = stack_pointer[-1]; + self = stack_pointer[-2]; + callable = stack_pointer[-3]; + assert(oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.list_append) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + assert(self_o != NULL); + if (!PyList_Check(self_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!LOCK_OBJECT(self_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(CALL, hit); + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); + PyStackRef_CLOSE(self); + PyStackRef_CLOSE(callable); + if (err) JUMP_TO_ERROR(); + #if TIER_ONE + // Skip the following POP_TOP. This is done here in tier one, and + // during trace projection in tier two: + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); + #endif + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_METHOD_DESCRIPTOR_O: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; + _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (total_args != 2) goto deoptimize; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (total_args != 2) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_O) goto deoptimize; - PyObject *arg = args[1]; - PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize; + if (meth->ml_flags != METH_O) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + // CPython promises to check all non-vectorcall function calls. + if (tstate->c_recursion_remaining <= 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + _PyStackRef arg_stackref = args[1]; + _PyStackRef self_stackref = args[0]; + if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } - res = _PyCFunction_TrampolineCall(cfunc, self, arg); + _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, + PyStackRef_AsPyObjectBorrow(self_stackref), + PyStackRef_AsPyObjectBorrow(arg_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(arg_stackref); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; + _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; - if (self_or_null != NULL) { - args--; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; total_args++; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyMethodDef *meth = method->d_method; - if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) goto deoptimize; + if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = args[0]; - if (!Py_IS_TYPE(self, d_type)) goto deoptimize; + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + if (!Py_IS_TYPE(self, d_type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, args + 1, nargs, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_METHOD_DESCRIPTOR_NOARGS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; + _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; assert(oparg == 0 || oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } - if (total_args != 1) goto deoptimize; - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; + if (total_args != 1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyMethodDef *meth = method->d_method; - PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize; - if (meth->ml_flags != METH_NOARGS) goto deoptimize; + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (meth->ml_flags != METH_NOARGS) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + // CPython promises to check all non-vectorcall function calls. + if (tstate->c_recursion_remaining <= 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } - res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_METHOD_DESCRIPTOR_FAST: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; + _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; - if (self_or_null != NULL) { - args--; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; total_args++; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_FASTCALL) goto deoptimize; - PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize; + if (meth->ml_flags != METH_FASTCALL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFast cfunc = (PyCFunctionFast)(void(*)(void))meth->ml_meth; - int nargs = total_args - 1; - res = cfunc(self, args + 1, nargs); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + PyObject *res_o = cfunc(self, (args_o + 1), nargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + assert(WITHIN_STACK_BOUNDS()); break; } - /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 because it is instrumented */ - /* _CALL_KW is not a viable micro-op for tier 2 */ + case _MAYBE_EXPAND_METHOD_KW: { + _PyStackRef kwnames_in; + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef kwnames_out; + oparg = CURRENT_OPARG(); + kwnames_in = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + func = &stack_pointer[-3 - oparg]; + maybe_self = &stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); + } + kwnames_out = kwnames_in; + stack_pointer[-1] = kwnames_out; + break; + } + + /* _DO_CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + case _PY_FRAME_KW: { + _PyStackRef kwnames; + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; + _PyInterpreterFrame *new_frame; + oparg = CURRENT_OPARG(); + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(kwnames); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + JUMP_TO_ERROR(); + } + new_frame = temp; + stack_pointer[0].bits = (uintptr_t)new_frame; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } - /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + case _CHECK_FUNCTION_VERSION_KW: { + _PyStackRef *callable; + oparg = CURRENT_OPARG(); + callable = &stack_pointer[-3 - oparg]; + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + if (!PyFunction_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _CHECK_METHOD_VERSION_KW: { + _PyStackRef *null; + _PyStackRef *callable; + oparg = CURRENT_OPARG(); + null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyStackRef_IsNull(null[0])) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _EXPAND_METHOD_KW: { + _PyStackRef *null; + _PyStackRef *callable; + _PyStackRef *method; + _PyStackRef *self; + oparg = CURRENT_OPARG(); + null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + method = &stack_pointer[-3 - oparg]; + self = &stack_pointer[-2 - oparg]; + _PyStackRef callable_s = callable[0]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable_s); + assert(PyStackRef_IsNull(null[0])); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method[0])); + PyStackRef_CLOSE(callable_s); + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE_KW: { + _PyStackRef *callable; + oparg = CURRENT_OPARG(); + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + if (PyFunction_Check(callable_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _CALL_KW_NON_PY: { + _PyStackRef kwnames; + _PyStackRef *args; + _PyStackRef *self_or_null; + _PyStackRef *callable; + _PyStackRef res; + oparg = CURRENT_OPARG(); + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + if (true) JUMP_TO_ERROR(); + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(kwnames); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it is instrumented */ + + case _MAKE_CALLARGS_A_TUPLE: { + _PyStackRef kwargs_in; + _PyStackRef callargs; + _PyStackRef func; + _PyStackRef tuple; + _PyStackRef kwargs_out; + kwargs_in = stack_pointer[-1]; + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (PyTuple_CheckExact(callargs_o)) { + tuple = callargs; + kwargs_out = kwargs_in; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + JUMP_TO_ERROR(); + } + kwargs_out = kwargs_in; + PyStackRef_CLOSE(callargs); + tuple = PyStackRef_FromPyObjectSteal(tuple_o); + } + stack_pointer[-2] = tuple; + stack_pointer[-1] = kwargs_out; + break; + } - /* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + /* _DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _MAKE_FUNCTION: { - PyObject *codeobj; - PyObject *func; - codeobj = stack_pointer[-1]; + _PyStackRef codeobj_st; + _PyStackRef func; + codeobj_st = stack_pointer[-1]; + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); + _PyFrame_SetStackPointer(frame, stack_pointer); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); - Py_DECREF(codeobj); - if (func_obj == NULL) { - GOTO_ERROR(error); - } + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(codeobj_st); + if (func_obj == NULL) JUMP_TO_ERROR(); _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); - func = (PyObject *)func_obj; + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); stack_pointer[-1] = func; break; } case _SET_FUNCTION_ATTRIBUTE: { - PyObject *func; - PyObject *attr; + _PyStackRef func_in; + _PyStackRef attr_st; + _PyStackRef func_out; oparg = CURRENT_OPARG(); - func = stack_pointer[-1]; - attr = stack_pointer[-2]; + func_in = stack_pointer[-1]; + attr_st = stack_pointer[-2]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); + PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); + func_out = func_in; assert(PyFunction_Check(func)); - PyFunctionObject *func_obj = (PyFunctionObject *)func; - switch(oparg) { - case MAKE_FUNCTION_CLOSURE: - assert(func_obj->func_closure == NULL); - func_obj->func_closure = attr; - break; - case MAKE_FUNCTION_ANNOTATIONS: - assert(func_obj->func_annotations == NULL); - func_obj->func_annotations = attr; - break; - case MAKE_FUNCTION_KWDEFAULTS: - assert(PyDict_CheckExact(attr)); - assert(func_obj->func_kwdefaults == NULL); - func_obj->func_kwdefaults = attr; - break; - case MAKE_FUNCTION_DEFAULTS: - assert(PyTuple_CheckExact(attr)); - assert(func_obj->func_defaults == NULL); - func_obj->func_defaults = attr; - break; - default: - Py_UNREACHABLE(); - } - stack_pointer[-2] = func; + size_t offset = _Py_FunctionAttributeOffsets[oparg]; + assert(offset != 0); + PyObject **ptr = (PyObject **)(((char *)func) + offset); + assert(*ptr == NULL); + *ptr = attr; + stack_pointer[-2] = func_out; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _RETURN_GENERATOR: { + _PyStackRef res; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (gen == NULL) JUMP_TO_ERROR(); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectSteal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BUILD_SLICE: { - PyObject *step = NULL; - PyObject *stop; - PyObject *start; - PyObject *slice; + _PyStackRef *args; + _PyStackRef slice; oparg = CURRENT_OPARG(); - if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } - stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; - start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; - slice = PySlice_New(start, stop, step); - Py_DECREF(start); - Py_DECREF(stop); - Py_XDECREF(step); - if (slice == NULL) { stack_pointer += -2 - ((oparg == 3) ? 1 : 0); goto error_tier_two; } - stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; - stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + args = &stack_pointer[-oparg]; + PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); + PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (slice_o == NULL) JUMP_TO_ERROR(); + slice = PyStackRef_FromPyObjectSteal(slice_o); + stack_pointer[-oparg] = slice; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _CONVERT_VALUE: { - PyObject *value; - PyObject *result; + _PyStackRef value; + _PyStackRef result; oparg = CURRENT_OPARG(); value = stack_pointer[-1]; conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = _PyEval_ConversionFuncs[oparg]; - result = conv_fn(value); - Py_DECREF(value); - if (result == NULL) goto pop_1_error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (result_o == NULL) JUMP_TO_ERROR(); + result = PyStackRef_FromPyObjectSteal(result_o); stack_pointer[-1] = result; break; } case _FORMAT_SIMPLE: { - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); /* If value is a unicode object, then we know the result * of format(value) is value itself. */ - if (!PyUnicode_CheckExact(value)) { - res = PyObject_Format(value, NULL); - Py_DECREF(value); - if (res == NULL) goto pop_1_error_tier_two; + if (!PyUnicode_CheckExact(value_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(value_o, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); } else { res = value; @@ -3599,127 +5699,177 @@ } case _FORMAT_WITH_SPEC: { - PyObject *fmt_spec; - PyObject *value; - PyObject *res; + _PyStackRef fmt_spec; + _PyStackRef value; + _PyStackRef res; fmt_spec = stack_pointer[-1]; value = stack_pointer[-2]; - res = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_DECREF(fmt_spec); - if (res == NULL) goto pop_2_error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + PyStackRef_CLOSE(fmt_spec); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _COPY: { - PyObject *bottom; - PyObject *top; + _PyStackRef bottom; + _PyStackRef top; oparg = CURRENT_OPARG(); bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); - top = Py_NewRef(bottom); + top = PyStackRef_DUP(bottom); stack_pointer[0] = top; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_OP: { - PyObject *rhs; - PyObject *lhs; - PyObject *res; + _PyStackRef rhs; + _PyStackRef lhs; + _PyStackRef res; oparg = CURRENT_OPARG(); rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); assert(_PyEval_BinaryOps[oparg]); - res = _PyEval_BinaryOps[oparg](lhs, rhs); - Py_DECREF(lhs); - Py_DECREF(rhs); - if (res == NULL) goto pop_2_error_tier_two; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(lhs); + PyStackRef_CLOSE(rhs); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _SWAP: { - PyObject *top; - PyObject *bottom; + _PyStackRef top_in; + _PyStackRef bottom_in; + _PyStackRef top_out; + _PyStackRef bottom_out; oparg = CURRENT_OPARG(); - top = stack_pointer[-1]; - bottom = stack_pointer[-2 - (oparg-2)]; + top_in = stack_pointer[-1]; + bottom_in = stack_pointer[-2 - (oparg-2)]; + bottom_out = bottom_in; + top_out = top_in; assert(oparg >= 2); - stack_pointer[-2 - (oparg-2)] = top; - stack_pointer[-1] = bottom; + stack_pointer[-2 - (oparg-2)] = top_out; + stack_pointer[-1] = bottom_out; break; } - /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_LINE is not a viable micro-op for tier 2 because it is instrumented */ + + /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 because it is instrumented */ + + /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 because it is instrumented */ - /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 */ + /* _MONITOR_JUMP_BACKWARD is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - /* _INSTRUMENTED_JUMP_BACKWARD is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_NOT_TAKEN is not a viable micro-op for tier 2 because it is instrumented */ - /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is instrumented */ - /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is instrumented */ - /* _INSTRUMENTED_POP_JUMP_IF_NONE is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_POP_JUMP_IF_NONE is not a viable micro-op for tier 2 because it is instrumented */ - /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 because it is instrumented */ case _GUARD_IS_TRUE_POP: { - PyObject *flag; + _PyStackRef flag; flag = stack_pointer[-1]; + int is_true = PyStackRef_IsTrue(flag); stack_pointer += -1; - if (!Py_IsTrue(flag)) goto side_exit; - assert(Py_IsTrue(flag)); + assert(WITHIN_STACK_BOUNDS()); + if (!is_true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _GUARD_IS_FALSE_POP: { - PyObject *flag; + _PyStackRef flag; flag = stack_pointer[-1]; + int is_false = PyStackRef_IsFalse(flag); stack_pointer += -1; - if (!Py_IsFalse(flag)) goto side_exit; - assert(Py_IsFalse(flag)); + assert(WITHIN_STACK_BOUNDS()); + if (!is_false) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _GUARD_IS_NONE_POP: { - PyObject *val; + _PyStackRef val; val = stack_pointer[-1]; - stack_pointer += -1; - if (!Py_IsNone(val)) { - Py_DECREF(val); - if (1) goto side_exit; + int is_none = PyStackRef_IsNone(val); + if (!is_none) { + PyStackRef_CLOSE(val); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_IS_NOT_NONE_POP: { - PyObject *val; + _PyStackRef val; val = stack_pointer[-1]; + int is_none = PyStackRef_IsNone(val); + PyStackRef_CLOSE(val); stack_pointer += -1; - if (Py_IsNone(val)) goto side_exit; - Py_DECREF(val); + assert(WITHIN_STACK_BOUNDS()); + if (is_none) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _JUMP_TO_TOP: { - #ifndef _Py_JIT - next_uop = ¤t_executor->trace[1]; - #endif - CHECK_EVAL_BREAKER(); + JUMP_TO_JUMP_TARGET(); break; } case _SET_IP: { - PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND(); + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; break; } + case _CHECK_STACK_SPACE_OPERAND: { + uint32_t framesize = (uint32_t)CURRENT_OPERAND0(); + assert(framesize <= INT_MAX); + if (!_PyThreadState_HasStackSpace(tstate, framesize)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + case _SAVE_RETURN_OFFSET: { oparg = CURRENT_OPARG(); #if TIER_ONE @@ -3732,131 +5882,241 @@ } case _EXIT_TRACE: { - if (1) goto side_exit; + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + _PyExitData *exit = (_PyExitData *)exit_p; + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + stack_pointer = _PyFrame_GetStackPointer(frame); + #if defined(Py_DEBUG) && !defined(_Py_JIT) + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 2) { + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("SIDE EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %u, temp %d, target %d -> %s]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + #endif + if (exit->executor && !exit->executor->vm_data.valid) { + exit->temperature = initial_temperature_backoff_counter(); + Py_CLEAR(exit->executor); + } + if (exit->executor == NULL) { + _Py_BackoffCounter temperature = exit->temperature; + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_ONE(target); + } + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + } + else { + int chain_depth = current_executor->vm_data.chain_depth + 1; + _PyFrame_SetStackPointer(frame, stack_pointer); + int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor, chain_depth); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (optimized <= 0) { + exit->temperature = restart_backoff_counter(temperature); + if (optimized < 0) { + GOTO_UNWIND(); + } + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_ONE(target); + } + else { + exit->temperature = initial_temperature_backoff_counter(); + } + } + exit->executor = executor; + } + Py_INCREF(exit->executor); + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_TWO(exit->executor); break; } case _CHECK_VALIDITY: { - if (!current_executor->vm_data.valid) goto deoptimize; + if (!current_executor->vm_data.valid) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _LOAD_CONST_INLINE: { - PyObject *value; - PyObject *ptr = (PyObject *)CURRENT_OPERAND(); - value = Py_NewRef(ptr); + _PyStackRef value; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + value = PyStackRef_FromPyObjectNew(ptr); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_CONST_INLINE_BORROW: { - PyObject *value; - PyObject *ptr = (PyObject *)CURRENT_OPERAND(); - value = ptr; + _PyStackRef value; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + value = PyStackRef_FromPyObjectImmortal(ptr); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _POP_TOP_LOAD_CONST_INLINE_BORROW: { - PyObject *pop; - PyObject *value; + _PyStackRef pop; + _PyStackRef value; pop = stack_pointer[-1]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND(); - Py_DECREF(pop); - value = ptr; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyStackRef_CLOSE(pop); + value = PyStackRef_FromPyObjectImmortal(ptr); stack_pointer[-1] = value; break; } - case _LOAD_CONST_INLINE_WITH_NULL: { - PyObject *value; - PyObject *null; - PyObject *ptr = (PyObject *)CURRENT_OPERAND(); - value = Py_NewRef(ptr); - null = NULL; - stack_pointer[0] = value; - stack_pointer[1] = null; - stack_pointer += 2; - break; - } - - case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { - PyObject *value; - PyObject *null; - PyObject *ptr = (PyObject *)CURRENT_OPERAND(); - value = ptr; - null = NULL; - stack_pointer[0] = value; - stack_pointer[1] = null; - stack_pointer += 2; + case _CHECK_FUNCTION: { + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } - case _CHECK_GLOBALS: { - PyObject *dict = (PyObject *)CURRENT_OPERAND(); - if (GLOBALS() != dict) goto deoptimize; + case _LOAD_GLOBAL_MODULE: { + _PyStackRef res; + uint16_t index = (uint16_t)CURRENT_OPERAND0(); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + PyObject *res_o = entries[index].me_value; + if (res_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + Py_INCREF(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _CHECK_BUILTINS: { - PyObject *dict = (PyObject *)CURRENT_OPERAND(); - if (BUILTINS() != dict) goto deoptimize; + case _LOAD_GLOBAL_BUILTINS: { + _PyStackRef res; + uint16_t index = (uint16_t)CURRENT_OPERAND0(); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + PyObject *res_o = entries[index].me_value; + if (res_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + Py_INCREF(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _INTERNAL_INCREMENT_OPT_COUNTER: { - PyObject *opt; - opt = stack_pointer[-1]; - _PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)opt; - exe->count++; - stack_pointer += -1; + case _LOAD_ATTR_MODULE: { + _PyStackRef owner; + _PyStackRef attr; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; + assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < dict->ma_keys->dk_nentries); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; + PyObject *attr_o = ep->me_value; + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + PyStackRef_CLOSE(owner); + stack_pointer[-1] = attr; break; } - case _COLD_EXIT: { - oparg = CURRENT_OPARG(); - _PyExecutorObject *previous = (_PyExecutorObject *)tstate->previous_executor; - _PyExitData *exit = &previous->exits[oparg]; - exit->temperature++; - PyCodeObject *code = _PyFrame_GetCode(frame); - _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; - if (exit->temperature < (int32_t)tstate->interp->optimizer_side_threshold) { - GOTO_TIER_ONE(target); + case _DYNAMIC_EXIT: { + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + tstate->previous_executor = (PyObject *)current_executor; + _PyExitData *exit = (_PyExitData *)exit_p; + _Py_CODEUNIT *target = frame->instr_ptr; + #if defined(Py_DEBUG) && !defined(_Py_JIT) + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 2) { + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("DYNAMIC EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %u, temp %d, target %d -> %s]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + stack_pointer = _PyFrame_GetStackPointer(frame); } + #endif _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { + PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); - } else { - int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); + } + else { + if (!backoff_counter_triggers(exit->temperature)) { + exit->temperature = advance_backoff_counter(exit->temperature); + GOTO_TIER_ONE(target); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); if (optimized <= 0) { - int32_t new_temp = -1 * tstate->interp->optimizer_side_threshold; - exit->temperature = (new_temp < INT16_MIN) ? INT16_MIN : new_temp; + exit->temperature = restart_backoff_counter(exit->temperature); if (optimized < 0) { - Py_DECREF(previous); - tstate->previous_executor = Py_None; - if (1) goto error_tier_two; + GOTO_UNWIND(); } GOTO_TIER_ONE(target); } + else { + exit->temperature = initial_temperature_backoff_counter(); + } } - /* We need two references. One to store in exit->executor and - * one to keep the executor alive when executing. */ - Py_INCREF(executor); - exit->executor = executor; GOTO_TIER_TWO(executor); break; } case _START_EXECUTOR: { - PyObject *executor = (PyObject *)CURRENT_OPERAND(); + PyObject *executor = (PyObject *)CURRENT_OPERAND0(); + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(tstate->previous_executor); + stack_pointer = _PyFrame_GetStackPointer(frame); tstate->previous_executor = NULL; #ifndef _Py_JIT current_executor = (_PyExecutorObject*)executor; #endif + assert(((_PyExecutorObject *)executor)->vm_data.valid); + break; + } + + case _MAKE_WARM: { + current_executor->vm_data.warm = true; + // It's okay if this ends up going negative. + if (--tstate->interp->trace_run_counter == 0) { + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + } break; } @@ -3867,10 +6127,47 @@ } case _CHECK_VALIDITY_AND_SET_IP: { - PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND(); - if (!current_executor->vm_data.valid) goto deoptimize; + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); + if (!current_executor->vm_data.valid) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; break; } + case _DEOPT: { + EXIT_TO_TIER1(); + break; + } + + case _ERROR_POP_N: { + oparg = CURRENT_OPARG(); + uint32_t target = (uint32_t)CURRENT_OPERAND0(); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; + stack_pointer = _PyFrame_GetStackPointer(frame); + GOTO_UNWIND(); + break; + } + + case _TIER2_RESUME_CHECK: { + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + if (eval_breaker & _PY_EVAL_EVENTS_MASK) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + break; + } + #undef TIER_TWO diff --git a/Python/fileutils.c b/Python/fileutils.c index 882d3299575cf3..68d24bc6b93465 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1,6 +1,7 @@ #include "Python.h" #include "pycore_fileutils.h" // fileutils definitions #include "pycore_runtime.h" // _PyRuntime +#include "pycore_pystate.h" // _Py_AssertHoldsTstate() #include "osdefs.h" // SEP #include // mbstowcs() @@ -1311,7 +1312,7 @@ _Py_fstat(int fd, struct _Py_stat_struct *status) { int res; - assert(PyGILState_Check()); + _Py_AssertHoldsTstate(); Py_BEGIN_ALLOW_THREADS res = _Py_fstat_noraise(fd, status); @@ -1468,14 +1469,14 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works) assert(!(atomic_flag_works != NULL && inheritable)); if (atomic_flag_works != NULL && !inheritable) { - if (*atomic_flag_works == -1) { + if (_Py_atomic_load_int_relaxed(atomic_flag_works) == -1) { int isInheritable = get_inheritable(fd, raise); if (isInheritable == -1) return -1; - *atomic_flag_works = !isInheritable; + _Py_atomic_store_int_relaxed(atomic_flag_works, !isInheritable); } - if (*atomic_flag_works) + if (_Py_atomic_load_int_relaxed(atomic_flag_works)) return 0; } @@ -1502,7 +1503,7 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works) #else #if defined(HAVE_SYS_IOCTL_H) && defined(FIOCLEX) && defined(FIONCLEX) - if (ioctl_works != 0 && raise != 0) { + if (raise != 0 && _Py_atomic_load_int_relaxed(&ioctl_works) != 0) { /* fast-path: ioctl() only requires one syscall */ /* caveat: raise=0 is an indicator that we must be async-signal-safe * thus avoid using ioctl() so we skip the fast-path. */ @@ -1512,7 +1513,9 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works) request = FIOCLEX; err = ioctl(fd, request, NULL); if (!err) { - ioctl_works = 1; + if (_Py_atomic_load_int_relaxed(&ioctl_works) == -1) { + _Py_atomic_store_int_relaxed(&ioctl_works, 1); + } return 0; } @@ -1539,7 +1542,7 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works) with EACCES. While FIOCLEX is safe operation it may be unavailable because ioctl was denied altogether. This can be the case on Android. */ - ioctl_works = 0; + _Py_atomic_store_int_relaxed(&ioctl_works, 0); } /* fallback to fcntl() if ioctl() does not work */ } @@ -1689,7 +1692,7 @@ int _Py_open(const char *pathname, int flags) { /* _Py_open() must be called with the GIL held. */ - assert(PyGILState_Check()); + _Py_AssertHoldsTstate(); return _Py_open_impl(pathname, flags, 1); } @@ -1746,8 +1749,10 @@ _Py_wfopen(const wchar_t *path, const wchar_t *mode) } -/* Open a file. Call _wfopen() on Windows, or encode the path to the filesystem - encoding and call fopen() otherwise. +/* Open a file. + + On Windows, if 'path' is a Unicode string, call _wfopen(). Otherwise, encode + the path to the filesystem encoding and call fopen(). Return the new file object on success. Raise an exception and return NULL on error. @@ -1760,32 +1765,32 @@ _Py_wfopen(const wchar_t *path, const wchar_t *mode) Release the GIL to call _wfopen() or fopen(). The caller must hold the GIL. */ FILE* -_Py_fopen_obj(PyObject *path, const char *mode) +Py_fopen(PyObject *path, const char *mode) { - FILE *f; - int async_err = 0; -#ifdef MS_WINDOWS - wchar_t wmode[10]; - int usize; - - assert(PyGILState_Check()); + _Py_AssertHoldsTstate(); if (PySys_Audit("open", "Osi", path, mode, 0) < 0) { return NULL; } - if (!PyUnicode_Check(path)) { - PyErr_Format(PyExc_TypeError, - "str file path expected under Windows, got %R", - Py_TYPE(path)); + + FILE *f; + int async_err = 0; + int saved_errno; +#ifdef MS_WINDOWS + PyObject *unicode; + if (!PyUnicode_FSDecoder(path, &unicode)) { return NULL; } - wchar_t *wpath = PyUnicode_AsWideCharString(path, NULL); - if (wpath == NULL) + wchar_t *wpath = PyUnicode_AsWideCharString(unicode, NULL); + Py_DECREF(unicode); + if (wpath == NULL) { return NULL; + } - usize = MultiByteToWideChar(CP_ACP, 0, mode, -1, - wmode, Py_ARRAY_LENGTH(wmode)); + wchar_t wmode[10]; + int usize = MultiByteToWideChar(CP_ACP, 0, mode, -1, + wmode, Py_ARRAY_LENGTH(wmode)); if (usize == 0) { PyErr_SetFromWindowsErr(0); PyMem_Free(wpath); @@ -1794,26 +1799,20 @@ _Py_fopen_obj(PyObject *path, const char *mode) do { Py_BEGIN_ALLOW_THREADS + _Py_BEGIN_SUPPRESS_IPH f = _wfopen(wpath, wmode); + _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS } while (f == NULL && errno == EINTR && !(async_err = PyErr_CheckSignals())); - int saved_errno = errno; + saved_errno = errno; PyMem_Free(wpath); #else PyObject *bytes; - const char *path_bytes; - - assert(PyGILState_Check()); - - if (!PyUnicode_FSConverter(path, &bytes)) - return NULL; - path_bytes = PyBytes_AS_STRING(bytes); - - if (PySys_Audit("open", "Osi", path, mode, 0) < 0) { - Py_DECREF(bytes); + if (!PyUnicode_FSConverter(path, &bytes)) { return NULL; } + const char *path_bytes = PyBytes_AS_STRING(bytes); do { Py_BEGIN_ALLOW_THREADS @@ -1821,11 +1820,13 @@ _Py_fopen_obj(PyObject *path, const char *mode) Py_END_ALLOW_THREADS } while (f == NULL && errno == EINTR && !(async_err = PyErr_CheckSignals())); - int saved_errno = errno; + saved_errno = errno; Py_DECREF(bytes); #endif - if (async_err) + + if (async_err) { return NULL; + } if (f == NULL) { errno = saved_errno; @@ -1840,6 +1841,19 @@ _Py_fopen_obj(PyObject *path, const char *mode) return f; } + +// Call fclose(). +// +// On Windows, files opened by Py_fopen() in the Python DLL must be closed by +// the Python DLL to use the same C runtime version. Otherwise, calling +// fclose() directly can cause undefined behavior. +int +Py_fclose(FILE *file) +{ + return fclose(file); +} + + /* Read count bytes from fd into buf. On success, return the number of read bytes, it can be lower than count. @@ -1860,7 +1874,7 @@ _Py_read(int fd, void *buf, size_t count) int err; int async_err = 0; - assert(PyGILState_Check()); + _Py_AssertHoldsTstate(); /* _Py_read() must not be called with an exception set, otherwise the * caller may think that read() was interrupted by a signal and the signal @@ -2026,7 +2040,7 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held) Py_ssize_t _Py_write(int fd, const void *buf, size_t count) { - assert(PyGILState_Check()); + _Py_AssertHoldsTstate(); /* _Py_write() must not be called with an exception set, otherwise the * caller may think that write() was interrupted by a signal and the signal @@ -2295,6 +2309,99 @@ PathCchCombineEx(wchar_t *buffer, size_t bufsize, const wchar_t *dirname, #endif /* defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) */ +void +_Py_skiproot(const wchar_t *path, Py_ssize_t size, Py_ssize_t *drvsize, + Py_ssize_t *rootsize) +{ + assert(drvsize); + assert(rootsize); +#ifndef MS_WINDOWS +#define IS_SEP(x) (*(x) == SEP) + *drvsize = 0; + if (!IS_SEP(&path[0])) { + // Relative path, e.g.: 'foo' + *rootsize = 0; + } + else if (!IS_SEP(&path[1]) || IS_SEP(&path[2])) { + // Absolute path, e.g.: '/foo', '///foo', '////foo', etc. + *rootsize = 1; + } + else { + // Precisely two leading slashes, e.g.: '//foo'. Implementation defined per POSIX, see + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 + *rootsize = 2; + } +#undef IS_SEP +#else + const wchar_t *pEnd = size >= 0 ? &path[size] : NULL; +#define IS_END(x) (pEnd ? (x) == pEnd : !*(x)) +#define IS_SEP(x) (*(x) == SEP || *(x) == ALTSEP) +#define SEP_OR_END(x) (IS_SEP(x) || IS_END(x)) + if (IS_SEP(&path[0])) { + if (IS_SEP(&path[1])) { + // Device drives, e.g. \\.\device or \\?\device + // UNC drives, e.g. \\server\share or \\?\UNC\server\share + Py_ssize_t idx; + if (path[2] == L'?' && IS_SEP(&path[3]) && + (path[4] == L'U' || path[4] == L'u') && + (path[5] == L'N' || path[5] == L'n') && + (path[6] == L'C' || path[6] == L'c') && + IS_SEP(&path[7])) + { + idx = 8; + } + else { + idx = 2; + } + while (!SEP_OR_END(&path[idx])) { + idx++; + } + if (IS_END(&path[idx])) { + *drvsize = idx; + *rootsize = 0; + } + else { + idx++; + while (!SEP_OR_END(&path[idx])) { + idx++; + } + *drvsize = idx; + if (IS_END(&path[idx])) { + *rootsize = 0; + } + else { + *rootsize = 1; + } + } + } + else { + // Relative path with root, e.g. \Windows + *drvsize = 0; + *rootsize = 1; + } + } + else if (!IS_END(&path[0]) && path[1] == L':') { + *drvsize = 2; + if (IS_SEP(&path[2])) { + // Absolute drive-letter path, e.g. X:\Windows + *rootsize = 1; + } + else { + // Relative path with drive, e.g. X:Windows + *rootsize = 0; + } + } + else { + // Relative path, e.g. Windows + *drvsize = 0; + *rootsize = 0; + } +#undef SEP_OR_END +#undef IS_SEP +#undef IS_END +#endif +} + // The caller must ensure "buffer" is big enough. static int join_relfile(wchar_t *buffer, size_t bufsize, @@ -2411,49 +2518,40 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) #endif #define SEP_OR_END(x) (IS_SEP(x) || IS_END(x)) - // Skip leading '.\' - if (p1[0] == L'.' && IS_SEP(&p1[1])) { - path = &path[2]; - while (IS_SEP(path) && !IS_END(path)) { - path++; - } - p1 = p2 = minP2 = path; - lastC = SEP; - } -#ifdef MS_WINDOWS - // Skip past drive segment and update minP2 - else if (p1[0] && p1[1] == L':') { - *p2++ = *p1++; - *p2++ = *p1++; - minP2 = p2; - lastC = L':'; - } - // Skip past all \\-prefixed paths, including \\?\, \\.\, - // and network paths, including the first segment. - else if (IS_SEP(&p1[0]) && IS_SEP(&p1[1])) { - int sepCount = 2; - *p2++ = SEP; - *p2++ = SEP; - p1 += 2; - for (; !IS_END(p1) && sepCount; ++p1) { - if (IS_SEP(p1)) { - --sepCount; - *p2++ = lastC = SEP; - } else { - *p2++ = lastC = *p1; + Py_ssize_t drvsize, rootsize; + _Py_skiproot(path, size, &drvsize, &rootsize); + if (drvsize || rootsize) { + // Skip past root and update minP2 + p1 = &path[drvsize + rootsize]; +#ifndef ALTSEP + p2 = p1; +#else + for (; p2 < p1; ++p2) { + if (*p2 == ALTSEP) { + *p2 = SEP; } } +#endif minP2 = p2 - 1; + lastC = *minP2; +#ifdef MS_WINDOWS + if (lastC != SEP) { + minP2++; + } +#endif } -#else - // Skip past two leading SEPs - else if (IS_SEP(&p1[0]) && IS_SEP(&p1[1]) && !IS_SEP(&p1[2])) { - *p2++ = *p1++; - *p2++ = *p1++; - minP2 = p2 - 1; // Absolute path has SEP at minP2 - lastC = SEP; + if (p1[0] == L'.' && SEP_OR_END(&p1[1])) { + // Skip leading '.\' + lastC = *++p1; +#ifdef ALTSEP + if (lastC == ALTSEP) { + lastC = SEP; + } +#endif + while (IS_SEP(p1)) { + p1++; + } } -#endif /* MS_WINDOWS */ /* if pEnd is specified, check that. Else, check for null terminator */ for (; !IS_END(p1); ++p1) { @@ -2570,7 +2668,7 @@ _Py_dup(int fd) HANDLE handle; #endif - assert(PyGILState_Check()); + _Py_AssertHoldsTstate(); #ifdef MS_WINDOWS handle = _Py_get_osfhandle(fd); @@ -2967,3 +3065,52 @@ _Py_GetTicksPerSecond(long *ticks_per_second) return 0; } #endif + + +/* Check if a file descriptor is valid or not. + Return 0 if the file descriptor is invalid, return non-zero otherwise. */ +int +_Py_IsValidFD(int fd) +{ +/* dup() is faster than fstat(): fstat() can require input/output operations, + whereas dup() doesn't. There is a low risk of EMFILE/ENFILE at Python + startup. Problem: dup() doesn't check if the file descriptor is valid on + some platforms. + + fcntl(fd, F_GETFD) is even faster, because it only checks the process table. + It is preferred over dup() when available, since it cannot fail with the + "too many open files" error (EMFILE). + + bpo-30225: On macOS Tiger, when stdout is redirected to a pipe and the other + side of the pipe is closed, dup(1) succeed, whereas fstat(1, &st) fails with + EBADF. FreeBSD has similar issue (bpo-32849). + + Only use dup() on Linux where dup() is enough to detect invalid FD + (bpo-32849). +*/ + if (fd < 0) { + return 0; + } +#if defined(F_GETFD) && ( \ + defined(__linux__) || \ + defined(__APPLE__) || \ + (defined(__wasm__) && !defined(__wasi__))) + return fcntl(fd, F_GETFD) >= 0; +#elif defined(__linux__) + int fd2 = dup(fd); + if (fd2 >= 0) { + close(fd2); + } + return (fd2 >= 0); +#elif defined(MS_WINDOWS) + HANDLE hfile; + _Py_BEGIN_SUPPRESS_IPH + hfile = (HANDLE)_get_osfhandle(fd); + _Py_END_SUPPRESS_IPH + return (hfile != INVALID_HANDLE_VALUE + && GetFileType(hfile) != FILE_TYPE_UNKNOWN); +#else + struct stat st; + return (fstat(fd, &st) == 0); +#endif +} diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 2f47e47bf9d29d..24561c1ee04db9 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_flowgraph.h" #include "pycore_compile.h" +#include "pycore_intrinsics.h" #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_opcode_utils.h" @@ -22,13 +23,13 @@ #define DEFAULT_BLOCK_SIZE 16 -typedef _PyCompilerSrcLocation location; -typedef _PyCfgJumpTargetLabel jump_target_label; +typedef _Py_SourceLocation location; +typedef _PyJumpTargetLabel jump_target_label; typedef struct _PyCfgInstruction { int i_opcode; int i_oparg; - _PyCompilerSrcLocation i_loc; + _Py_SourceLocation i_loc; struct _PyCfgBasicblock *i_target; /* target block (if jump instruction) */ struct _PyCfgBasicblock *i_except; /* target block when exception is raised */ } cfg_instr; @@ -40,7 +41,7 @@ typedef struct _PyCfgBasicblock { control flow. */ struct _PyCfgBasicblock *b_list; /* The label of this block if it is a jump target, -1 otherwise */ - _PyCfgJumpTargetLabel b_label; + _PyJumpTargetLabel b_label; /* Exception stack at start of block, used by assembler to create the exception handling table */ struct _PyCfgExceptStack *b_exceptstack; /* pointer to an array of instructions, initially NULL */ @@ -81,18 +82,16 @@ struct _PyCfgBuilder { /* pointer to the block currently being constructed */ struct _PyCfgBasicblock *g_curblock; /* label for the next instruction to be placed */ - _PyCfgJumpTargetLabel g_current_label; + _PyJumpTargetLabel g_current_label; }; typedef struct _PyCfgBuilder cfg_builder; -static const jump_target_label NO_LABEL = {-1}; - #define SAME_LABEL(L1, L2) ((L1).id == (L2).id) #define IS_LABEL(L) (!SAME_LABEL((L), (NO_LABEL))) #define LOCATION(LNO, END_LNO, COL, END_COL) \ - ((const _PyCompilerSrcLocation){(LNO), (END_LNO), (COL), (END_COL)}) + ((const _Py_SourceLocation){(LNO), (END_LNO), (COL), (END_COL)}) static inline int is_block_push(cfg_instr *i) @@ -285,7 +284,7 @@ dump_instr(cfg_instr *i) static inline int basicblock_returns(const basicblock *b) { cfg_instr *last = basicblock_last_instr(b); - return last && (last->i_opcode == RETURN_VALUE || last->i_opcode == RETURN_CONST); + return last && last->i_opcode == RETURN_VALUE; } static void @@ -517,22 +516,6 @@ no_redundant_jumps(cfg_builder *g) { } return true; } - -static bool -all_exits_have_lineno(basicblock *entryblock) { - for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - for (int i = 0; i < b->b_iused; i++) { - cfg_instr *instr = &b->b_instr[i]; - if (instr->i_opcode == RETURN_VALUE) { - if (instr->i_loc.lineno < 0) { - assert(0); - return false; - } - } - } - } - return true; -} #endif /***** CFG preprocessing (jump targets and exceptions) *****/ @@ -540,14 +523,15 @@ all_exits_have_lineno(basicblock *entryblock) { static int normalize_jumps_in_block(cfg_builder *g, basicblock *b) { cfg_instr *last = basicblock_last_instr(b); - if (last == NULL || !is_jump(last) || - IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode)) { + if (last == NULL || !IS_CONDITIONAL_JUMP_OPCODE(last->i_opcode)) { return SUCCESS; } assert(!IS_ASSEMBLER_OPCODE(last->i_opcode)); bool is_forward = last->i_target->b_visited == 0; if (is_forward) { + RETURN_IF_ERROR( + basicblock_addop(b, NOT_TAKEN, 0, last->i_loc)); return SUCCESS; } @@ -575,6 +559,8 @@ normalize_jumps_in_block(cfg_builder *g, basicblock *b) { if (backwards_jump == NULL) { return ERROR; } + RETURN_IF_ERROR( + basicblock_addop(backwards_jump, NOT_TAKEN, 0, last->i_loc)); RETURN_IF_ERROR( basicblock_add_jump(backwards_jump, JUMP, target, last->i_loc)); last->i_opcode = reversed_opcode; @@ -751,6 +737,53 @@ make_cfg_traversal_stack(basicblock *entryblock) { return stack; } +/* Compute the stack effects of opcode with argument oparg. + + Some opcodes have different stack effect when jump to the target and + when not jump. The 'jump' parameter specifies the case: + + * 0 -- when not jump + * 1 -- when jump + * -1 -- maximal + */ +typedef struct { + /* The stack effect of the instruction. */ + int net; + + /* The maximum stack usage of the instruction. Some instructions may + * temporarily push extra values to the stack while they are executing. + */ + int max; +} stack_effects; + +Py_LOCAL(int) +get_stack_effects(int opcode, int oparg, int jump, stack_effects *effects) +{ + if (opcode < 0) { + return -1; + } + if ((opcode <= MAX_REAL_OPCODE) && (_PyOpcode_Deopt[opcode] != opcode)) { + // Specialized instructions are not supported. + return -1; + } + int popped = _PyOpcode_num_popped(opcode, oparg); + int pushed = _PyOpcode_num_pushed(opcode, oparg); + if (popped < 0 || pushed < 0) { + return -1; + } + if (IS_BLOCK_PUSH_OPCODE(opcode) && !jump) { + effects->net = 0; + effects->max = 0; + return 0; + } + if (_PyOpcode_max_stack_effect(opcode, oparg, &effects->max) < 0) { + return -1; + } + effects->net = pushed - popped; + assert(effects->max >= effects->net); + return 0; +} + Py_LOCAL_INLINE(int) stackdepth_push(basicblock ***sp, basicblock *b, int depth) { @@ -795,37 +828,30 @@ calculate_stackdepth(cfg_builder *g) basicblock *next = b->b_next; for (int i = 0; i < b->b_iused; i++) { cfg_instr *instr = &b->b_instr[i]; - int effect = PyCompile_OpcodeStackEffectWithJump( - instr->i_opcode, instr->i_oparg, 0); - if (effect == PY_INVALID_STACK_EFFECT) { + stack_effects effects; + if (get_stack_effects(instr->i_opcode, instr->i_oparg, 0, &effects) < 0) { PyErr_Format(PyExc_SystemError, "Invalid stack effect for opcode=%d, arg=%i", instr->i_opcode, instr->i_oparg); goto error; } - int new_depth = depth + effect; + int new_depth = depth + effects.net; if (new_depth < 0) { - PyErr_Format(PyExc_ValueError, - "Invalid CFG, stack underflow"); - goto error; - } - if (new_depth > maxdepth) { - maxdepth = new_depth; + PyErr_Format(PyExc_ValueError, + "Invalid CFG, stack underflow"); + goto error; } + maxdepth = Py_MAX(maxdepth, depth + effects.max); if (HAS_TARGET(instr->i_opcode)) { - effect = PyCompile_OpcodeStackEffectWithJump( - instr->i_opcode, instr->i_oparg, 1); - if (effect == PY_INVALID_STACK_EFFECT) { + if (get_stack_effects(instr->i_opcode, instr->i_oparg, 1, &effects) < 0) { PyErr_Format(PyExc_SystemError, "Invalid stack effect for opcode=%d, arg=%i", instr->i_opcode, instr->i_oparg); goto error; } - int target_depth = depth + effect; + int target_depth = depth + effects.net; assert(target_depth >= 0); /* invalid code or bug in stackdepth() */ - if (target_depth > maxdepth) { - maxdepth = target_depth; - } + maxdepth = Py_MAX(maxdepth, depth + effects.max); if (stackdepth_push(&sp, instr->i_target, target_depth) < 0) { goto error; } @@ -975,13 +1001,14 @@ remove_unreachable(basicblock *entryblock) { basicblock **sp = stack; entryblock->b_predecessors = 1; *sp++ = entryblock; + entryblock->b_visited = 1; while (sp > stack) { basicblock *b = *(--sp); - b->b_visited = 1; if (b->b_next && BB_HAS_FALLTHROUGH(b)) { if (!b->b_next->b_visited) { assert(b->b_next->b_predecessors == 0); *sp++ = b->b_next; + b->b_next->b_visited = 1; } b->b_next->b_predecessors++; } @@ -991,8 +1018,8 @@ remove_unreachable(basicblock *entryblock) { if (is_jump(instr) || is_block_push(instr)) { target = instr->i_target; if (!target->b_visited) { - assert(target->b_predecessors == 0 || target == b->b_next); *sp++ = target; + target->b_visited = 1; } target->b_predecessors++; } @@ -1104,7 +1131,7 @@ remove_redundant_nops_and_pairs(basicblock *entryblock) int opcode = instr->i_opcode; bool is_redundant_pair = false; if (opcode == POP_TOP) { - if (prev_opcode == LOAD_CONST) { + if (prev_opcode == LOAD_CONST || prev_opcode == LOAD_SMALL_INT) { is_redundant_pair = true; } else if (prev_opcode == COPY && prev_oparg == 1) { @@ -1253,14 +1280,23 @@ jump_thread(basicblock *bb, cfg_instr *inst, cfg_instr *target, int opcode) return false; } +static int +loads_const(int opcode) +{ + return OPCODE_HAS_CONST(opcode) || opcode == LOAD_SMALL_INT; +} + static PyObject* get_const_value(int opcode, int oparg, PyObject *co_consts) { PyObject *constant = NULL; - assert(OPCODE_HAS_CONST(opcode)); + assert(loads_const(opcode)); if (opcode == LOAD_CONST) { constant = PyList_GET_ITEM(co_consts, oparg); } + if (opcode == LOAD_SMALL_INT) { + return PyLong_FromLong(oparg); + } if (constant == NULL) { PyErr_SetString(PyExc_SystemError, @@ -1318,7 +1354,7 @@ fold_tuple_on_constants(PyObject *const_cache, assert(inst[n].i_oparg == n); for (int i = 0; i < n; i++) { - if (!OPCODE_HAS_CONST(inst[i].i_opcode)) { + if (!loads_const(inst[i].i_opcode)) { return SUCCESS; } } @@ -1556,13 +1592,15 @@ basicblock_optimize_load_const(PyObject *const_cache, basicblock *bb, PyObject * oparg = inst->i_oparg; } assert(!IS_ASSEMBLER_OPCODE(opcode)); - if (opcode != LOAD_CONST) { + if (opcode != LOAD_CONST && opcode != LOAD_SMALL_INT) { continue; } int nextop = i+1 < bb->b_iused ? bb->b_instr[i+1].i_opcode : 0; switch(nextop) { case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: + case JUMP_IF_FALSE: + case JUMP_IF_TRUE: { /* Remove LOAD_CONST const; conditional jump */ PyObject* cnt = get_const_value(opcode, oparg, consts); @@ -1574,8 +1612,11 @@ basicblock_optimize_load_const(PyObject *const_cache, basicblock *bb, PyObject * if (is_true == -1) { return ERROR; } - INSTR_SET_OP0(inst, NOP); - int jump_if_true = nextop == POP_JUMP_IF_TRUE; + if (PyCompile_OpcodeStackEffect(nextop, 0) == -1) { + /* POP_JUMP_IF_FALSE or POP_JUMP_IF_TRUE */ + INSTR_SET_OP0(inst, NOP); + } + int jump_if_true = (nextop == POP_JUMP_IF_TRUE || nextop == JUMP_IF_TRUE); if (is_true == jump_if_true) { bb->b_instr[i+1].i_opcode = JUMP; } @@ -1630,12 +1671,6 @@ basicblock_optimize_load_const(PyObject *const_cache, basicblock *bb, PyObject * : POP_JUMP_IF_NONE; break; } - case RETURN_VALUE: - { - INSTR_SET_OP0(inst, NOP); - INSTR_SET_OP1(&bb->b_instr[++i], RETURN_CONST, oparg); - break; - } case TO_BOOL: { PyObject *cnt = get_const_value(opcode, oparg, consts); @@ -1735,6 +1770,36 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) i -= jump_thread(bb, inst, target, POP_JUMP_IF_TRUE); } break; + case JUMP_IF_FALSE: + switch (target->i_opcode) { + case JUMP: + case JUMP_IF_FALSE: + i -= jump_thread(bb, inst, target, JUMP_IF_FALSE); + continue; + case JUMP_IF_TRUE: + // No need to check for loops here, a block's b_next + // cannot point to itself. + assert(inst->i_target != inst->i_target->b_next); + inst->i_target = inst->i_target->b_next; + i--; + continue; + } + break; + case JUMP_IF_TRUE: + switch (target->i_opcode) { + case JUMP: + case JUMP_IF_TRUE: + i -= jump_thread(bb, inst, target, JUMP_IF_TRUE); + continue; + case JUMP_IF_FALSE: + // No need to check for loops here, a block's b_next + // cannot point to itself. + assert(inst->i_target != inst->i_target->b_next); + inst->i_target = inst->i_target->b_next; + i--; + continue; + } + break; case JUMP: case JUMP_NO_INTERRUPT: switch (target->i_opcode) { @@ -1810,6 +1875,12 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) continue; } break; + case CALL_INTRINSIC_1: + // for _ in (*foo, *bar) -> for _ in [*foo, *bar] + if (oparg == INTRINSIC_LIST_TO_TUPLE && nextop == GET_ITER) { + INSTR_SET_OP0(inst, NOP); + } + break; } } @@ -1829,6 +1900,22 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) static int resolve_line_numbers(cfg_builder *g, int firstlineno); +static int +remove_redundant_nops_and_jumps(cfg_builder *g) +{ + int removed_nops, removed_jumps; + do { + /* Convergence is guaranteed because the number of + * redundant jumps and nops only decreases. + */ + removed_nops = remove_redundant_nops(g); + RETURN_IF_ERROR(removed_nops); + removed_jumps = remove_redundant_jumps(g); + RETURN_IF_ERROR(removed_jumps); + } while(removed_nops + removed_jumps > 0); + return SUCCESS; +} + /* Perform optimizations on a control flow graph. The consts object should still be in list form to allow new constants to be appended. @@ -1850,17 +1937,7 @@ optimize_cfg(cfg_builder *g, PyObject *consts, PyObject *const_cache, int firstl } RETURN_IF_ERROR(remove_redundant_nops_and_pairs(g->g_entryblock)); RETURN_IF_ERROR(remove_unreachable(g->g_entryblock)); - - int removed_nops, removed_jumps; - do { - /* Convergence is guaranteed because the number of - * redundant jumps and nops only decreases. - */ - removed_nops = remove_redundant_nops(g); - RETURN_IF_ERROR(removed_nops); - removed_jumps = remove_redundant_jumps(g); - RETURN_IF_ERROR(removed_jumps); - } while(removed_nops + removed_jumps > 0); + RETURN_IF_ERROR(remove_redundant_nops_and_jumps(g)); assert(no_redundant_jumps(g)); return SUCCESS; } @@ -2052,7 +2129,8 @@ remove_unused_consts(basicblock *entryblock, PyObject *consts) /* mark used consts */ for (basicblock *b = entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { - if (OPCODE_HAS_CONST(b->b_instr[i].i_opcode)) { + int opcode = b->b_instr[i].i_opcode; + if (OPCODE_HAS_CONST(opcode)) { int index = b->b_instr[i].i_oparg; index_map[index] = index; } @@ -2061,7 +2139,7 @@ remove_unused_consts(basicblock *entryblock, PyObject *consts) /* now index_map[i] == i if consts[i] is used, -1 otherwise */ /* condense consts */ Py_ssize_t n_used_consts = 0; - for (int i = 0; i < nconsts; i++) { + for (Py_ssize_t i = 0; i < nconsts; i++) { if (index_map[i] != -1) { assert(index_map[i] == i); index_map[n_used_consts++] = index_map[i]; @@ -2105,7 +2183,8 @@ remove_unused_consts(basicblock *entryblock, PyObject *consts) for (basicblock *b = entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { - if (OPCODE_HAS_CONST(b->b_instr[i].i_opcode)) { + int opcode = b->b_instr[i].i_opcode; + if (OPCODE_HAS_CONST(opcode)) { int index = b->b_instr[i].i_oparg; assert(reverse_index_map[index] >= 0); assert(reverse_index_map[index] < n_used_consts); @@ -2276,15 +2355,11 @@ push_cold_blocks_to_end(cfg_builder *g) { if (!IS_LABEL(b->b_next->b_label)) { b->b_next->b_label.id = next_lbl++; } - cfg_instr *prev_instr = basicblock_last_instr(b); - // b cannot be empty because at the end of an exception handler - // there is always a POP_EXCEPT + RERAISE/RETURN - assert(prev_instr); - basicblock_addop(explicit_jump, JUMP_NO_INTERRUPT, b->b_next->b_label.id, - prev_instr->i_loc); + NO_LOCATION); explicit_jump->b_cold = 1; explicit_jump->b_next = b->b_next; + explicit_jump->b_predecessors = 1; b->b_next = explicit_jump; /* set target */ @@ -2334,7 +2409,39 @@ push_cold_blocks_to_end(cfg_builder *g) { b->b_next = cold_blocks; if (cold_blocks != NULL) { - RETURN_IF_ERROR(remove_redundant_jumps(g)); + RETURN_IF_ERROR(remove_redundant_nops_and_jumps(g)); + } + return SUCCESS; +} + +static int +convert_pseudo_conditional_jumps(cfg_builder *g) +{ + basicblock *entryblock = g->g_entryblock; + for (basicblock *b = entryblock; b != NULL; b = b->b_next) { + for (int i = 0; i < b->b_iused; i++) { + cfg_instr *instr = &b->b_instr[i]; + if (instr->i_opcode == JUMP_IF_FALSE || instr->i_opcode == JUMP_IF_TRUE) { + assert(i == b->b_iused - 1); + instr->i_opcode = instr->i_opcode == JUMP_IF_FALSE ? + POP_JUMP_IF_FALSE : POP_JUMP_IF_TRUE; + location loc = instr->i_loc; + cfg_instr copy = { + .i_opcode = COPY, + .i_oparg = 1, + .i_loc = loc, + .i_target = NULL, + }; + RETURN_IF_ERROR(basicblock_insert_instruction(b, i++, ©)); + cfg_instr to_bool = { + .i_opcode = TO_BOOL, + .i_oparg = 0, + .i_loc = loc, + .i_target = NULL, + }; + RETURN_IF_ERROR(basicblock_insert_instruction(b, i++, &to_bool)); + } + } } return SUCCESS; } @@ -2359,7 +2466,7 @@ convert_pseudo_ops(cfg_builder *g) } } } - return remove_redundant_nops(g); + return remove_redundant_nops_and_jumps(g); } static inline bool @@ -2498,8 +2605,9 @@ _PyCfg_OptimizeCodeUnit(cfg_builder *g, PyObject *consts, PyObject *const_cache, RETURN_IF_ERROR(insert_superinstructions(g)); RETURN_IF_ERROR(push_cold_blocks_to_end(g)); - assert(all_exits_have_lineno(g->g_entryblock)); RETURN_IF_ERROR(resolve_line_numbers(g, firstlineno)); + // temporarily remove assert. See https://github.com/python/cpython/issues/125845 + // assert(all_exits_have_lineno(g->g_entryblock)); return SUCCESS; } @@ -2711,26 +2819,70 @@ prepare_localsplus(_PyCompile_CodeUnitMetadata *umd, cfg_builder *g, int code_fl return nlocalsplus; } +cfg_builder * +_PyCfg_FromInstructionSequence(_PyInstructionSequence *seq) +{ + if (_PyInstructionSequence_ApplyLabelMap(seq) < 0) { + return NULL; + } + cfg_builder *g = _PyCfgBuilder_New(); + if (g == NULL) { + return NULL; + } + for (int i = 0; i < seq->s_used; i++) { + seq->s_instrs[i].i_target = 0; + } + for (int i = 0; i < seq->s_used; i++) { + _PyInstruction *instr = &seq->s_instrs[i]; + if (HAS_TARGET(instr->i_opcode)) { + assert(instr->i_oparg >= 0 && instr->i_oparg < seq->s_used); + seq->s_instrs[instr->i_oparg].i_target = 1; + } + } + for (int i = 0; i < seq->s_used; i++) { + _PyInstruction *instr = &seq->s_instrs[i]; + if (instr->i_target) { + jump_target_label lbl_ = {i}; + if (_PyCfgBuilder_UseLabel(g, lbl_) < 0) { + goto error; + } + } + int opcode = instr->i_opcode; + int oparg = instr->i_oparg; + if (_PyCfgBuilder_Addop(g, opcode, oparg, instr->i_loc) < 0) { + goto error; + } + } + if (_PyCfgBuilder_CheckSize(g) < 0) { + goto error; + } + return g; +error: + _PyCfgBuilder_Free(g); + return NULL; +} + int -_PyCfg_ToInstructionSequence(cfg_builder *g, _PyCompile_InstructionSequence *seq) +_PyCfg_ToInstructionSequence(cfg_builder *g, _PyInstructionSequence *seq) { int lbl = 0; for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { b->b_label = (jump_target_label){lbl}; - lbl += b->b_iused; + lbl += 1; } for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - RETURN_IF_ERROR(_PyCompile_InstructionSequence_UseLabel(seq, b->b_label.id)); + RETURN_IF_ERROR(_PyInstructionSequence_UseLabel(seq, b->b_label.id)); for (int i = 0; i < b->b_iused; i++) { cfg_instr *instr = &b->b_instr[i]; - if (OPCODE_HAS_JUMP(instr->i_opcode) || is_block_push(instr)) { + if (HAS_TARGET(instr->i_opcode)) { + /* Set oparg to the label id (it will later be mapped to an offset) */ instr->i_oparg = instr->i_target->b_label.id; } RETURN_IF_ERROR( - _PyCompile_InstructionSequence_Addop( + _PyInstructionSequence_Addop( seq, instr->i_opcode, instr->i_oparg, instr->i_loc)); - _PyCompile_ExceptHandlerInfo *hi = &seq->s_instrs[seq->s_used-1].i_except_handler_info; + _PyExceptHandlerInfo *hi = &seq->s_instrs[seq->s_used-1].i_except_handler_info; if (instr->i_except != NULL) { hi->h_label = instr->i_except->b_label.id; hi->h_startdepth = instr->i_except->b_startdepth; @@ -2741,6 +2893,9 @@ _PyCfg_ToInstructionSequence(cfg_builder *g, _PyCompile_InstructionSequence *seq } } } + if (_PyInstructionSequence_ApplyLabelMap(seq) < 0) { + return ERROR; + } return SUCCESS; } @@ -2749,8 +2904,10 @@ int _PyCfg_OptimizedCfgToInstructionSequence(cfg_builder *g, _PyCompile_CodeUnitMetadata *umd, int code_flags, int *stackdepth, int *nlocalsplus, - _PyCompile_InstructionSequence *seq) + _PyInstructionSequence *seq) { + RETURN_IF_ERROR(convert_pseudo_conditional_jumps(g)); + *stackdepth = calculate_stackdepth(g); if (*stackdepth < 0) { return ERROR; @@ -2786,7 +2943,7 @@ _PyCfg_OptimizedCfgToInstructionSequence(cfg_builder *g, } /* This is used by _PyCompile_Assemble to fill in the jump and exception - * targets in a synthetic CFG (which is not the ouptut of the builtin compiler). + * targets in a synthetic CFG (which is not the output of the builtin compiler). */ int _PyCfg_JumpLabelsToTargets(cfg_builder *g) @@ -2795,3 +2952,74 @@ _PyCfg_JumpLabelsToTargets(cfg_builder *g) RETURN_IF_ERROR(label_exception_targets(g->g_entryblock)); return SUCCESS; } + +/* Exported API functions */ + +int +PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump) +{ + stack_effects effs; + if (get_stack_effects(opcode, oparg, jump, &effs) < 0) { + return PY_INVALID_STACK_EFFECT; + } + return effs.net; +} + +int +PyCompile_OpcodeStackEffect(int opcode, int oparg) +{ + stack_effects effs; + if (get_stack_effects(opcode, oparg, -1, &effs) < 0) { + return PY_INVALID_STACK_EFFECT; + } + return effs.net; +} + +/* Access to compiler optimizations for unit tests. + + * _PyCompile_OptimizeCfg takes an instruction list, constructs + * a CFG, optimizes it and converts back to an instruction list. + */ + +static PyObject * +cfg_to_instruction_sequence(cfg_builder *g) +{ + _PyInstructionSequence *seq = (_PyInstructionSequence *)_PyInstructionSequence_New(); + if (seq == NULL) { + return NULL; + } + if (_PyCfg_ToInstructionSequence(g, seq) < 0) { + PyInstructionSequence_Fini(seq); + return NULL; + } + return (PyObject*)seq; +} + +PyObject * +_PyCompile_OptimizeCfg(PyObject *seq, PyObject *consts, int nlocals) +{ + if (!_PyInstructionSequence_Check(seq)) { + PyErr_SetString(PyExc_ValueError, "expected an instruction sequence"); + return NULL; + } + PyObject *const_cache = PyDict_New(); + if (const_cache == NULL) { + return NULL; + } + + PyObject *res = NULL; + cfg_builder *g = _PyCfg_FromInstructionSequence((_PyInstructionSequence*)seq); + if (g == NULL) { + goto error; + } + int nparams = 0, firstlineno = 1; + if (_PyCfg_OptimizeCodeUnit(g, consts, const_cache, nlocals, + nparams, firstlineno) < 0) { + goto error; + } + res = cfg_to_instruction_sequence(g); +error: + Py_DECREF(const_cache); + _PyCfgBuilder_Free(g); + return res; +} diff --git a/Python/formatter_unicode.c b/Python/formatter_unicode.c index 6af589f966a502..16f711184990ac 100644 --- a/Python/formatter_unicode.c +++ b/Python/formatter_unicode.c @@ -320,7 +320,7 @@ parse_internal_render_format_spec(PyObject *obj, format->thousands_separators = LT_UNDER_FOUR_LOCALE; break; } - /* fall through */ + _Py_FALLTHROUGH; default: invalid_thousands_separator_type(format->thousands_separators, format->type); return 0; @@ -740,7 +740,7 @@ get_locale_info(enum LocaleType type, LocaleInfo *locale_info) break; case LT_NO_LOCALE: locale_info->decimal_point = PyUnicode_FromOrdinal('.'); - locale_info->thousands_sep = PyUnicode_New(0, 0); + locale_info->thousands_sep = Py_GetConstant(Py_CONSTANT_EMPTY_STR); if (!locale_info->decimal_point || !locale_info->thousands_sep) return -1; locale_info->grouping = no_grouping; diff --git a/Python/frame.c b/Python/frame.c index ddf6ef6ba5465c..68ac2acbaee342 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -13,16 +13,9 @@ _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg) { Py_VISIT(frame->frame_obj); Py_VISIT(frame->f_locals); - Py_VISIT(frame->f_funcobj); - Py_VISIT(_PyFrame_GetCode(frame)); - /* locals */ - PyObject **locals = _PyFrame_GetLocalsArray(frame); - int i = 0; - /* locals and stack */ - for (; i stacktop; i++) { - Py_VISIT(locals[i]); - } - return 0; + _Py_VISIT_STACKREF(frame->f_funcobj); + _Py_VISIT_STACKREF(frame->f_executable); + return _PyGC_VisitFrameStack(frame, visit, arg); } PyFrameObject * @@ -37,67 +30,46 @@ _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame) return NULL; } PyErr_SetRaisedException(exc); - if (frame->frame_obj) { - // GH-97002: How did we get into this horrible situation? Most likely, - // allocating f triggered a GC collection, which ran some code that - // *also* created the same frame... while we were in the middle of - // creating it! See test_sneaky_frame_object in test_frame.py for a - // concrete example. - // - // Regardless, just throw f away and use that frame instead, since it's - // already been exposed to user code. It's actually a bit tricky to do - // this, since we aren't backed by a real _PyInterpreterFrame anymore. - // Just pretend that we have an owned, cleared frame so frame_dealloc - // doesn't make the situation worse: - f->f_frame = (_PyInterpreterFrame *)f->_f_frame_data; - f->f_frame->owner = FRAME_CLEARED; - f->f_frame->frame_obj = f; - Py_DECREF(f); - return frame->frame_obj; - } + + // GH-97002: There was a time when a frame object could be created when we + // are allocating the new frame object f above, so frame->frame_obj would + // be assigned already. That path does not exist anymore. We won't call any + // Python code in this function and garbage collection will not run. + // Notice that _PyFrame_New_NoTrack() can potentially raise a MemoryError, + // but it won't allocate a traceback until the frame unwinds, so we are safe + // here. + assert(frame->frame_obj == NULL); assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT); - assert(frame->owner != FRAME_CLEARED); f->f_frame = frame; frame->frame_obj = f; return f; } -void -_PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest) -{ - assert(src->stacktop >= _PyFrame_GetCode(src)->co_nlocalsplus); - Py_ssize_t size = ((char*)&src->localsplus[src->stacktop]) - (char *)src; - memcpy(dest, src, size); - // Don't leave a dangling pointer to the old frame when creating generators - // and coroutines: - dest->previous = NULL; -} - - static void take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) { - assert(frame->owner != FRAME_OWNED_BY_CSTACK); + assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT); - assert(frame->owner != FRAME_CLEARED); - Py_ssize_t size = ((char*)&frame->localsplus[frame->stacktop]) - (char *)frame; - Py_INCREF(_PyFrame_GetCode(frame)); + Py_ssize_t size = ((char*)frame->stackpointer) - (char *)frame; memcpy((_PyInterpreterFrame *)f->_f_frame_data, frame, size); frame = (_PyInterpreterFrame *)f->_f_frame_data; + frame->stackpointer = (_PyStackRef *)(((char *)frame) + size); + frame->f_executable = PyStackRef_DUP(frame->f_executable); f->f_frame = frame; frame->owner = FRAME_OWNED_BY_FRAME_OBJECT; if (_PyFrame_IsIncomplete(frame)) { // This may be a newly-created generator or coroutine frame. Since it's // dead anyways, just pretend that the first RESUME ran: PyCodeObject *code = _PyFrame_GetCode(frame); - frame->instr_ptr = _PyCode_CODE(code) + code->_co_firsttraceable + 1; + frame->instr_ptr = + _PyFrame_GetBytecode(frame) + code->_co_firsttraceable + 1; } assert(!_PyFrame_IsIncomplete(frame)); assert(f->f_back == NULL); _PyInterpreterFrame *prev = _PyFrame_GetFirstComplete(frame->previous); frame->previous = NULL; if (prev) { - assert(prev->owner != FRAME_OWNED_BY_CSTACK); + assert(prev->owner < FRAME_OWNED_BY_INTERPRETER); /* Link PyFrameObjects.f_back and remove link through _PyInterpreterFrame.previous */ PyFrameObject *back = _PyFrame_GetFrameObject(prev); if (back == NULL) { @@ -115,13 +87,27 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) } } +void +_PyFrame_ClearLocals(_PyInterpreterFrame *frame) +{ + assert(frame->stackpointer != NULL); + _PyStackRef *sp = frame->stackpointer; + _PyStackRef *locals = frame->localsplus; + frame->stackpointer = locals; + while (sp > locals) { + sp--; + PyStackRef_XCLOSE(*sp); + } + Py_CLEAR(frame->f_locals); +} + void _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) { /* It is the responsibility of the owning generator/coroutine * to have cleared the enclosing generator, if any. */ assert(frame->owner != FRAME_OWNED_BY_GENERATOR || - _PyFrame_GetGenerator(frame)->gi_frame_state == FRAME_CLEARED); + _PyGen_GetGeneratorFromFrame(frame)->gi_frame_state == FRAME_CLEARED); // GH-99729: Clearing this frame can expose the stack (via finalizers). It's // crucial that this frame has been unlinked, and is no longer visible: assert(_PyThreadState_GET()->current_frame != frame); @@ -135,12 +121,8 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) } Py_DECREF(f); } - assert(frame->stacktop >= 0); - for (int i = 0; i < frame->stacktop; i++) { - Py_XDECREF(frame->localsplus[i]); - } - Py_XDECREF(frame->f_locals); - Py_DECREF(frame->f_funcobj); + _PyFrame_ClearLocals(frame); + PyStackRef_CLEAR(frame->f_funcobj); } /* Unstable API functions */ @@ -148,9 +130,7 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) PyObject * PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame) { - PyObject *code = frame->f_executable; - Py_INCREF(code); - return code; + return PyStackRef_AsPyObjectNew(frame->f_executable); } int diff --git a/Python/frozen.c b/Python/frozen.c index 77f51a7f750965..15d256b6743e0a 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -66,34 +66,6 @@ #include "frozen_modules/frozen_only.h" /* End includes */ -#define GET_CODE(name) _Py_get_##name##_toplevel - -/* Start extern declarations */ -extern PyObject *_Py_get_importlib__bootstrap_toplevel(void); -extern PyObject *_Py_get_importlib__bootstrap_external_toplevel(void); -extern PyObject *_Py_get_zipimport_toplevel(void); -extern PyObject *_Py_get_abc_toplevel(void); -extern PyObject *_Py_get_codecs_toplevel(void); -extern PyObject *_Py_get_io_toplevel(void); -extern PyObject *_Py_get__collections_abc_toplevel(void); -extern PyObject *_Py_get__sitebuiltins_toplevel(void); -extern PyObject *_Py_get_genericpath_toplevel(void); -extern PyObject *_Py_get_ntpath_toplevel(void); -extern PyObject *_Py_get_posixpath_toplevel(void); -extern PyObject *_Py_get_os_toplevel(void); -extern PyObject *_Py_get_site_toplevel(void); -extern PyObject *_Py_get_stat_toplevel(void); -extern PyObject *_Py_get_importlib_util_toplevel(void); -extern PyObject *_Py_get_importlib_machinery_toplevel(void); -extern PyObject *_Py_get_runpy_toplevel(void); -extern PyObject *_Py_get___hello___toplevel(void); -extern PyObject *_Py_get___phello___toplevel(void); -extern PyObject *_Py_get___phello___ham_toplevel(void); -extern PyObject *_Py_get___phello___ham_eggs_toplevel(void); -extern PyObject *_Py_get___phello___spam_toplevel(void); -extern PyObject *_Py_get_frozen_only_toplevel(void); -/* End extern declarations */ - static const struct _frozen bootstrap_modules[] = { {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap), false}, {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external), false}, @@ -112,7 +84,6 @@ static const struct _frozen stdlib_modules[] = { {"genericpath", _Py_M__genericpath, (int)sizeof(_Py_M__genericpath), false}, {"ntpath", _Py_M__ntpath, (int)sizeof(_Py_M__ntpath), false}, {"posixpath", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), false}, - {"os.path", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), false}, {"os", _Py_M__os, (int)sizeof(_Py_M__os), false}, {"site", _Py_M__site, (int)sizeof(_Py_M__site), false}, {"stat", _Py_M__stat, (int)sizeof(_Py_M__stat), false}, @@ -144,7 +115,6 @@ const struct _frozen *_PyImport_FrozenTest = test_modules; static const struct _module_alias aliases[] = { {"_frozen_importlib", "importlib._bootstrap"}, {"_frozen_importlib_external", "importlib._bootstrap_external"}, - {"os.path", "posixpath"}, {"__hello_alias__", "__hello__"}, {"__phello_alias__", "__hello__"}, {"__phello_alias__.spam", "__hello__"}, diff --git a/Python/frozenmain.c b/Python/frozenmain.c index 3ce9476c9ad46c..ec4566bd4f84bc 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -54,6 +54,12 @@ Py_FrozenMain(int argc, char **argv) Py_ExitStatusException(status); } + PyInterpreterState *interp = PyInterpreterState_Get(); + if (_PyInterpreterState_SetRunningMain(interp) < 0) { + PyErr_Print(); + exit(1); + } + #ifdef MS_WINDOWS PyWinFreeze_ExeInit(); #endif @@ -83,6 +89,9 @@ Py_FrozenMain(int argc, char **argv) #ifdef MS_WINDOWS PyWinFreeze_ExeTerm(); #endif + + _PyInterpreterState_SetNotRunningMain(interp); + if (Py_FinalizeEx() < 0) { sts = 120; } diff --git a/Python/future.c b/Python/future.c index 0dbc7ede20f324..79b6c0c503bace 100644 --- a/Python/future.c +++ b/Python/future.c @@ -1,13 +1,14 @@ #include "Python.h" #include "pycore_ast.h" // _PyAST_GetDocString() +#include "pycore_symtable.h" // _PyFutureFeatures #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() #define UNDEFINED_FUTURE_FEATURE "future feature %.100s is not defined" static int -future_check_features(PyFutureFeatures *ff, stmt_ty s, PyObject *filename) +future_check_features(_PyFutureFeatures *ff, stmt_ty s, PyObject *filename) { - int i; + Py_ssize_t i; assert(s->kind == ImportFrom_kind); @@ -40,12 +41,20 @@ future_check_features(PyFutureFeatures *ff, stmt_ty s, PyObject *filename) } else if (strcmp(feature, "braces") == 0) { PyErr_SetString(PyExc_SyntaxError, "not a chance"); - PyErr_SyntaxLocationObject(filename, s->lineno, s->col_offset + 1); + PyErr_RangedSyntaxLocationObject(filename, + name->lineno, + name->col_offset + 1, + name->end_lineno, + name->end_col_offset + 1); return 0; } else { PyErr_Format(PyExc_SyntaxError, UNDEFINED_FUTURE_FEATURE, feature); - PyErr_SyntaxLocationObject(filename, s->lineno, s->col_offset + 1); + PyErr_RangedSyntaxLocationObject(filename, + name->lineno, + name->col_offset + 1, + name->end_lineno, + name->end_col_offset + 1); return 0; } } @@ -53,7 +62,7 @@ future_check_features(PyFutureFeatures *ff, stmt_ty s, PyObject *filename) } static int -future_parse(PyFutureFeatures *ff, mod_ty mod, PyObject *filename) +future_parse(_PyFutureFeatures *ff, mod_ty mod, PyObject *filename) { if (!(mod->kind == Module_kind || mod->kind == Interactive_kind)) { return 1; @@ -76,7 +85,7 @@ future_parse(PyFutureFeatures *ff, mod_ty mod, PyObject *filename) * are another future statement and a doc string. */ - if (s->kind == ImportFrom_kind) { + if (s->kind == ImportFrom_kind && s->v.ImportFrom.level == 0) { identifier modname = s->v.ImportFrom.module; if (modname && _PyUnicode_EqualToASCIIString(modname, "__future__")) { @@ -98,10 +107,10 @@ future_parse(PyFutureFeatures *ff, mod_ty mod, PyObject *filename) int -_PyFuture_FromAST(mod_ty mod, PyObject *filename, PyFutureFeatures *ff) +_PyFuture_FromAST(mod_ty mod, PyObject *filename, _PyFutureFeatures *ff) { ff->ff_features = 0; - ff->ff_location = (_PyCompilerSrcLocation){-1, -1, -1, -1}; + ff->ff_location = (_Py_SourceLocation){-1, -1, -1, -1}; if (!future_parse(ff, mod, filename)) { return 0; diff --git a/Python/gc.c b/Python/gc.c index ea3b596d1713df..3fe0b7f814544d 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1,18 +1,17 @@ // This implements the reference cycle garbage collector. -// The Python module inteface to the collector is in gcmodule.c. +// The Python module interface to the collector is in gcmodule.c. // See https://devguide.python.org/internals/garbage-collector/ #include "Python.h" #include "pycore_ceval.h" // _Py_set_eval_breaker_bit() #include "pycore_context.h" -#include "pycore_dict.h" // _PyDict_MaybeUntrack() +#include "pycore_dict.h" // _PyInlineValuesSize() #include "pycore_initconfig.h" #include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_object.h" #include "pycore_object_alloc.h" // _PyObject_MallocWithType() #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_time.h" // _PyTime_PerfCounterUnchecked() #include "pycore_weakref.h" // _PyWeakref_ClearRef() #include "pydtrace.h" @@ -24,6 +23,10 @@ typedef struct _gc_runtime_state GCState; # define GC_DEBUG #endif +// Define this when debugging the GC +// #define GC_EXTRA_DEBUG + + #define GC_NEXT _PyGCHead_NEXT #define GC_PREV _PyGCHead_PREV @@ -46,7 +49,7 @@ typedef struct _gc_runtime_state GCState; // move_legacy_finalizers() removes this flag instead. // Between them, unreachable list is not normal list and we can not use // most gc_list_* functions for it. -#define NEXT_MASK_UNREACHABLE (1) +#define NEXT_MASK_UNREACHABLE 2 #define AS_GC(op) _Py_AS_GC(op) #define FROM_GC(gc) _Py_FROM_GC(gc) @@ -96,9 +99,48 @@ gc_decref(PyGC_Head *g) g->_gc_prev -= 1 << _PyGC_PREV_SHIFT; } +static inline int +gc_old_space(PyGC_Head *g) +{ + return g->_gc_next & _PyGC_NEXT_MASK_OLD_SPACE_1; +} -#define GEN_HEAD(gcstate, n) (&(gcstate)->generations[n].head) +static inline int +other_space(int space) +{ + assert(space == 0 || space == 1); + return space ^ _PyGC_NEXT_MASK_OLD_SPACE_1; +} +static inline void +gc_flip_old_space(PyGC_Head *g) +{ + g->_gc_next ^= _PyGC_NEXT_MASK_OLD_SPACE_1; +} + +static inline void +gc_set_old_space(PyGC_Head *g, int space) +{ + assert(space == 0 || space == _PyGC_NEXT_MASK_OLD_SPACE_1); + g->_gc_next &= ~_PyGC_NEXT_MASK_OLD_SPACE_1; + g->_gc_next |= space; +} + +static PyGC_Head * +GEN_HEAD(GCState *gcstate, int n) +{ + assert((gcstate->visited_space & (~1)) == 0); + switch(n) { + case 0: + return &gcstate->young.head; + case 1: + return &gcstate->old[gcstate->visited_space].head; + case 2: + return &gcstate->old[gcstate->visited_space^1].head; + default: + Py_UNREACHABLE(); + } +} static GCState * get_gc_state(void) @@ -117,11 +159,12 @@ _PyGC_InitState(GCState *gcstate) GEN.head._gc_prev = (uintptr_t)&GEN.head; \ } while (0) - for (int i = 0; i < NUM_GENERATIONS; i++) { - assert(gcstate->generations[i].count == 0); - INIT_HEAD(gcstate->generations[i]); - }; - gcstate->generation0 = GEN_HEAD(gcstate, 0); + assert(gcstate->young.count == 0); + assert(gcstate->old[0].count == 0); + assert(gcstate->old[1].count == 0); + INIT_HEAD(gcstate->young); + INIT_HEAD(gcstate->old[0]); + INIT_HEAD(gcstate->old[1]); INIT_HEAD(gcstate->permanent_generation); #undef INIT_HEAD @@ -142,6 +185,7 @@ _PyGC_Init(PyInterpreterState *interp) if (gcstate->callbacks == NULL) { return _PyStatus_NO_MEMORY(); } + gcstate->heap_size = 0; return _PyStatus_OK(); } @@ -219,6 +263,7 @@ gc_list_is_empty(PyGC_Head *list) static inline void gc_list_append(PyGC_Head *node, PyGC_Head *list) { + assert((list->_gc_prev & ~_PyGC_PREV_MASK) == 0); PyGC_Head *last = (PyGC_Head *)list->_gc_prev; // last <-> node @@ -276,6 +321,8 @@ gc_list_merge(PyGC_Head *from, PyGC_Head *to) PyGC_Head *from_tail = GC_PREV(from); assert(from_head != from); assert(from_tail != from); + assert(gc_list_is_empty(to) || + gc_old_space(to_tail) == gc_old_space(from_tail)); _PyGCHead_SET_NEXT(to_tail, from_head); _PyGCHead_SET_PREV(from_head, to_tail); @@ -344,8 +391,8 @@ enum flagstates {collecting_clear_unreachable_clear, static void validate_list(PyGC_Head *head, enum flagstates flags) { - assert((head->_gc_prev & PREV_MASK_COLLECTING) == 0); - assert((head->_gc_next & NEXT_MASK_UNREACHABLE) == 0); + assert((head->_gc_prev & ~_PyGC_PREV_MASK) == 0); + assert((head->_gc_next & ~_PyGC_PREV_MASK) == 0); uintptr_t prev_value = 0, next_value = 0; switch (flags) { case collecting_clear_unreachable_clear: @@ -367,7 +414,7 @@ validate_list(PyGC_Head *head, enum flagstates flags) PyGC_Head *gc = GC_NEXT(head); while (gc != head) { PyGC_Head *trueprev = GC_PREV(gc); - PyGC_Head *truenext = (PyGC_Head *)(gc->_gc_next & ~NEXT_MASK_UNREACHABLE); + PyGC_Head *truenext = GC_NEXT(gc); assert(truenext != NULL); assert(trueprev == prev); assert((gc->_gc_prev & PREV_MASK_COLLECTING) == prev_value); @@ -377,10 +424,58 @@ validate_list(PyGC_Head *head, enum flagstates flags) } assert(prev == GC_PREV(head)); } + #else #define validate_list(x, y) do{}while(0) #endif +#ifdef GC_EXTRA_DEBUG + + +static void +gc_list_validate_space(PyGC_Head *head, int space) { + PyGC_Head *gc = GC_NEXT(head); + while (gc != head) { + assert(gc_old_space(gc) == space); + gc = GC_NEXT(gc); + } +} + +static void +validate_spaces(GCState *gcstate) +{ + int visited = gcstate->visited_space; + int not_visited = other_space(visited); + gc_list_validate_space(&gcstate->young.head, not_visited); + for (int space = 0; space < 2; space++) { + gc_list_validate_space(&gcstate->old[space].head, space); + } + gc_list_validate_space(&gcstate->permanent_generation.head, visited); +} + +static void +validate_consistent_old_space(PyGC_Head *head) +{ + PyGC_Head *gc = GC_NEXT(head); + if (gc == head) { + return; + } + int old_space = gc_old_space(gc); + while (gc != head) { + PyGC_Head *truenext = GC_NEXT(gc); + assert(truenext != NULL); + assert(gc_old_space(gc) == old_space); + gc = truenext; + } +} + + +#else +#define validate_spaces(g) do{}while(0) +#define validate_consistent_old_space(l) do{}while(0) +#define gc_list_validate_space(l, s) do{}while(0) +#endif + /*** end of list stuff ***/ @@ -396,12 +491,8 @@ update_refs(PyGC_Head *containers) while (gc != containers) { next = GC_NEXT(gc); PyObject *op = FROM_GC(gc); - /* Move any object that might have become immortal to the - * permanent generation as the reference count is not accurately - * reflecting the actual number of live references to this object - */ if (_Py_IsImmortal(op)) { - gc_list_move(gc, &get_gc_state()->permanent_generation.head); + _PyObject_GC_UNTRACK(op); gc = next; continue; } @@ -449,6 +540,24 @@ visit_decref(PyObject *op, void *parent) return 0; } +int +_PyGC_VisitStackRef(_PyStackRef *ref, visitproc visit, void *arg) +{ + Py_VISIT(PyStackRef_AsPyObjectBorrow(*ref)); + return 0; +} + +int +_PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg) +{ + _PyStackRef *ref = _PyFrame_GetLocalsArray(frame); + /* locals and stack */ + for (; ref < frame->stackpointer; ref++) { + Py_VISIT(PyStackRef_AsPyObjectBorrow(*ref)); + } + return 0; +} + /* Subtract internal references from gc_refs. After this, gc_refs is >= 0 * for all objects in containers, and is GC_REACHABLE for all tracked gc * objects not in containers. The ones with gc_refs > 0 are directly @@ -502,12 +611,13 @@ visit_reachable(PyObject *op, void *arg) // Manually unlink gc from unreachable list because the list functions // don't work right in the presence of NEXT_MASK_UNREACHABLE flags. PyGC_Head *prev = GC_PREV(gc); - PyGC_Head *next = (PyGC_Head*)(gc->_gc_next & ~NEXT_MASK_UNREACHABLE); + PyGC_Head *next = GC_NEXT(gc); _PyObject_ASSERT(FROM_GC(prev), prev->_gc_next & NEXT_MASK_UNREACHABLE); _PyObject_ASSERT(FROM_GC(next), next->_gc_next & NEXT_MASK_UNREACHABLE); - prev->_gc_next = gc->_gc_next; // copy NEXT_MASK_UNREACHABLE + prev->_gc_next = gc->_gc_next; // copy flag bits + gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; _PyGCHead_SET_PREV(next, prev); gc_list_append(gc, reachable); @@ -559,6 +669,9 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) * or to the right have been scanned yet. */ + validate_consistent_old_space(young); + /* Record which old space we are in, and set NEXT_MASK_UNREACHABLE bit for convenience */ + uintptr_t flags = NEXT_MASK_UNREACHABLE | (gc->_gc_next & _PyGC_NEXT_MASK_OLD_SPACE_1); while (gc != young) { if (gc_get_refs(gc)) { /* gc is definitely reachable from outside the @@ -604,26 +717,39 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) // But this may pollute the unreachable list head's 'next' pointer // too. That's semantically senseless but expedient here - the // damage is repaired when this function ends. - last->_gc_next = (NEXT_MASK_UNREACHABLE | (uintptr_t)gc); + last->_gc_next = flags | (uintptr_t)gc; _PyGCHead_SET_PREV(gc, last); - gc->_gc_next = (NEXT_MASK_UNREACHABLE | (uintptr_t)unreachable); + gc->_gc_next = flags | (uintptr_t)unreachable; unreachable->_gc_prev = (uintptr_t)gc; } - gc = (PyGC_Head*)prev->_gc_next; + gc = _PyGCHead_NEXT(prev); } // young->_gc_prev must be last element remained in the list. young->_gc_prev = (uintptr_t)prev; + young->_gc_next &= _PyGC_PREV_MASK; // don't let the pollution of the list head's next pointer leak - unreachable->_gc_next &= ~NEXT_MASK_UNREACHABLE; -} - + unreachable->_gc_next &= _PyGC_PREV_MASK; +} + +/* In theory, all tuples should be younger than the +* objects they refer to, as tuples are immortal. +* Therefore, untracking tuples in oldest-first order in the +* young generation before promoting them should have tracked +* all the tuples that can be untracked. +* +* Unfortunately, the C API allows tuples to be created +* and then filled in. So this won't untrack all tuples +* that can be untracked. It should untrack most of them +* and is much faster than a more complex approach that +* would untrack all relevant tuples. +*/ static void untrack_tuples(PyGC_Head *head) { - PyGC_Head *next, *gc = GC_NEXT(head); + PyGC_Head *gc = GC_NEXT(head); while (gc != head) { PyObject *op = FROM_GC(gc); - next = GC_NEXT(gc); + PyGC_Head *next = GC_NEXT(gc); if (PyTuple_CheckExact(op)) { _PyTuple_MaybeUntrack(op); } @@ -631,21 +757,6 @@ untrack_tuples(PyGC_Head *head) } } -/* Try to untrack all currently tracked dictionaries */ -static void -untrack_dicts(PyGC_Head *head) -{ - PyGC_Head *next, *gc = GC_NEXT(head); - while (gc != head) { - PyObject *op = FROM_GC(gc); - next = GC_NEXT(gc); - if (PyDict_CheckExact(op)) { - _PyDict_MaybeUntrack(op); - } - gc = next; - } -} - /* Return true if object has a pre-PEP 442 finalization method. */ static int has_legacy_finalizer(PyObject *op) @@ -673,8 +784,8 @@ move_legacy_finalizers(PyGC_Head *unreachable, PyGC_Head *finalizers) PyObject *op = FROM_GC(gc); _PyObject_ASSERT(op, gc->_gc_next & NEXT_MASK_UNREACHABLE); + next = GC_NEXT(gc); gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; - next = (PyGC_Head*)gc->_gc_next; if (has_legacy_finalizer(op)) { gc_clear_collecting(gc); @@ -697,8 +808,8 @@ clear_unreachable_mask(PyGC_Head *unreachable) PyGC_Head *gc, *next; for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { _PyObject_ASSERT((PyObject*)FROM_GC(gc), gc->_gc_next & NEXT_MASK_UNREACHABLE); + next = GC_NEXT(gc); gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; - next = (PyGC_Head*)gc->_gc_next; } validate_list(unreachable, collecting_set_unreachable_clear); } @@ -868,6 +979,7 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) /* Invoke the callbacks we decided to honor. It's safe to invoke them * because they can't reference unreachable objects. */ + int visited_space = get_gc_state()->visited_space; while (! gc_list_is_empty(&wrcb_to_call)) { PyObject *temp; PyObject *callback; @@ -902,6 +1014,7 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) Py_DECREF(op); if (wrcb_to_call._gc_next == (uintptr_t)gc) { /* object is still alive -- move it */ + gc_set_old_space(gc, visited_space); gc_list_move(gc, old); } else { @@ -1030,25 +1143,6 @@ delete_garbage(PyThreadState *tstate, GCState *gcstate, } -// Show stats for objects in each generations -static void -show_stats_each_generations(GCState *gcstate) -{ - char buf[100]; - size_t pos = 0; - - for (int i = 0; i < NUM_GENERATIONS && pos < sizeof(buf); i++) { - pos += PyOS_snprintf(buf+pos, sizeof(buf)-pos, - " %zd", - gc_list_size(GEN_HEAD(gcstate, i))); - } - - PySys_FormatStderr( - "gc: objects in each generation:%s\n" - "gc: objects in permanent generation: %zd\n", - buf, gc_list_size(&gcstate->permanent_generation.head)); -} - /* Deduce which objects among "base" are unreachable from outside the list and move them to 'unreachable'. The process consist in the following steps: @@ -1122,7 +1216,6 @@ deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable) { * the reachable objects instead. But this is a one-time cost, probably not * worth complicating the code to speed just a little. */ - gc_list_init(unreachable); move_unreachable(base, unreachable); // gc_prev is pointer again validate_list(base, collecting_clear_unreachable_clear); validate_list(unreachable, collecting_set_unreachable_set); @@ -1160,220 +1253,452 @@ handle_resurrected_objects(PyGC_Head *unreachable, PyGC_Head* still_unreachable, gc_list_merge(resurrected, old_generation); } +static void +gc_collect_region(PyThreadState *tstate, + PyGC_Head *from, + PyGC_Head *to, + struct gc_collection_stats *stats); -/* Invoke progress callbacks to notify clients that garbage collection - * is starting or stopping +static inline Py_ssize_t +gc_list_set_space(PyGC_Head *list, int space) +{ + Py_ssize_t size = 0; + PyGC_Head *gc; + for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc)) { + gc_set_old_space(gc, space); + size++; + } + return size; +} + +/* Making progress in the incremental collector + * In order to eventually collect all cycles + * the incremental collector must progress through the old + * space faster than objects are added to the old space. + * + * Each young or incremental collection adds a number of + * objects, S (for survivors) to the old space, and + * incremental collectors scan I objects from the old space. + * I > S must be true. We also want I > S * N to be where + * N > 1. Higher values of N mean that the old space is + * scanned more rapidly. + * The default incremental threshold of 10 translates to + * N == 1.4 (1 + 4/threshold) */ + +/* Divide by 10, so that the default incremental threshold of 10 + * scans objects at 1% of the heap size */ +#define SCAN_RATE_DIVISOR 10 + static void -invoke_gc_callback(PyThreadState *tstate, const char *phase, - int generation, Py_ssize_t collected, - Py_ssize_t uncollectable) +add_stats(GCState *gcstate, int gen, struct gc_collection_stats *stats) { - assert(!_PyErr_Occurred(tstate)); + gcstate->generation_stats[gen].collected += stats->collected; + gcstate->generation_stats[gen].uncollectable += stats->uncollectable; + gcstate->generation_stats[gen].collections += 1; +} - /* we may get called very early */ +static void +gc_collect_young(PyThreadState *tstate, + struct gc_collection_stats *stats) +{ GCState *gcstate = &tstate->interp->gc; - if (gcstate->callbacks == NULL) { - return; - } - - /* The local variable cannot be rebound, check it for sanity */ - assert(PyList_CheckExact(gcstate->callbacks)); - PyObject *info = NULL; - if (PyList_GET_SIZE(gcstate->callbacks) != 0) { - info = Py_BuildValue("{sisnsn}", - "generation", generation, - "collected", collected, - "uncollectable", uncollectable); - if (info == NULL) { - PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); - return; + validate_spaces(gcstate); + PyGC_Head *young = &gcstate->young.head; + PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; + untrack_tuples(young); + GC_STAT_ADD(0, collections, 1); +#ifdef Py_STATS + { + Py_ssize_t count = 0; + PyGC_Head *gc; + for (gc = GC_NEXT(young); gc != young; gc = GC_NEXT(gc)) { + count++; } } +#endif - PyObject *phase_obj = PyUnicode_FromString(phase); - if (phase_obj == NULL) { - Py_XDECREF(info); - PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); - return; - } + PyGC_Head survivors; + gc_list_init(&survivors); + gc_list_set_space(young, gcstate->visited_space); + gc_collect_region(tstate, young, &survivors, stats); + gc_list_merge(&survivors, visited); + validate_spaces(gcstate); + gcstate->young.count = 0; + gcstate->old[gcstate->visited_space].count++; + add_stats(gcstate, 0, stats); + validate_spaces(gcstate); +} - PyObject *stack[] = {phase_obj, info}; - for (Py_ssize_t i=0; icallbacks); i++) { - PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); - Py_INCREF(cb); /* make sure cb doesn't go away */ - r = PyObject_Vectorcall(cb, stack, 2, NULL); - if (r == NULL) { - PyErr_WriteUnraisable(cb); - } - else { - Py_DECREF(r); - } - Py_DECREF(cb); - } - Py_DECREF(phase_obj); - Py_XDECREF(info); - assert(!_PyErr_Occurred(tstate)); +#ifndef NDEBUG +static inline int +IS_IN_VISITED(PyGC_Head *gc, int visited_space) +{ + assert(visited_space == 0 || other_space(visited_space) == 0); + return gc_old_space(gc) == visited_space; } +#endif +struct container_and_flag { + PyGC_Head *container; + int visited_space; + intptr_t size; +}; -/* Find the oldest generation (highest numbered) where the count - * exceeds the threshold. Objects in the that generation and - * generations younger than it will be collected. */ +/* A traversal callback for adding to container) */ static int -gc_select_generation(GCState *gcstate) -{ - for (int i = NUM_GENERATIONS-1; i >= 0; i--) { - if (gcstate->generations[i].count > gcstate->generations[i].threshold) { - /* Avoid quadratic performance degradation in number - of tracked objects (see also issue #4074): - - To limit the cost of garbage collection, there are two strategies; - - make each collection faster, e.g. by scanning fewer objects - - do less collections - This heuristic is about the latter strategy. - - In addition to the various configurable thresholds, we only trigger a - full collection if the ratio - - long_lived_pending / long_lived_total - - is above a given value (hardwired to 25%). - - The reason is that, while "non-full" collections (i.e., collections of - the young and middle generations) will always examine roughly the same - number of objects -- determined by the aforementioned thresholds --, - the cost of a full collection is proportional to the total number of - long-lived objects, which is virtually unbounded. - - Indeed, it has been remarked that doing a full collection every - of object creations entails a dramatic performance - degradation in workloads which consist in creating and storing lots of - long-lived objects (e.g. building a large list of GC-tracked objects would - show quadratic performance, instead of linear as expected: see issue #4074). - - Using the above ratio, instead, yields amortized linear performance in - the total number of objects (the effect of which can be summarized - thusly: "each full garbage collection is more and more costly as the - number of objects grows, but we do fewer and fewer of them"). - - This heuristic was suggested by Martin von Löwis on python-dev in - June 2008. His original analysis and proposal can be found at: - http://mail.python.org/pipermail/python-dev/2008-June/080579.html - */ - if (i == NUM_GENERATIONS - 1 - && gcstate->long_lived_pending < gcstate->long_lived_total / 4) - { - continue; - } - return i; +visit_add_to_container(PyObject *op, void *arg) +{ + OBJECT_STAT_INC(object_visits); + struct container_and_flag *cf = (struct container_and_flag *)arg; + int visited = cf->visited_space; + assert(visited == get_gc_state()->visited_space); + if (!_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { + PyGC_Head *gc = AS_GC(op); + if (_PyObject_GC_IS_TRACKED(op) && + gc_old_space(gc) != visited) { + gc_flip_old_space(gc); + gc_list_move(gc, cf->container); + cf->size++; } } - return -1; + return 0; } - -/* This is the main function. Read this to understand how the - * collection process works. */ -static Py_ssize_t -gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) +static intptr_t +expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCState *gcstate) { - int i; - Py_ssize_t m = 0; /* # objects collected */ - Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */ - PyGC_Head *young; /* the generation we are examining */ - PyGC_Head *old; /* next older generation */ - PyGC_Head unreachable; /* non-problematic unreachable trash */ - PyGC_Head finalizers; /* objects with, & reachable from, __del__ */ - PyGC_Head *gc; - PyTime_t t1 = 0; /* initialize to prevent a compiler warning */ - GCState *gcstate = &tstate->interp->gc; - - // gc_collect_main() must not be called before _PyGC_Init - // or after _PyGC_Fini() - assert(gcstate->garbage != NULL); - assert(!_PyErr_Occurred(tstate)); - - int expected = 0; - if (!_Py_atomic_compare_exchange_int(&gcstate->collecting, &expected, 1)) { - // Don't start a garbage collection if one is already in progress. - return 0; - } - - if (generation == GENERATION_AUTO) { - // Select the oldest generation that needs collecting. We will collect - // objects from that generation and all generations younger than it. - generation = gc_select_generation(gcstate); - if (generation < 0) { - // No generation needs to be collected. - _Py_atomic_store_int(&gcstate->collecting, 0); - return 0; + struct container_and_flag arg = { + .container = container, + .visited_space = gcstate->visited_space, + .size = 0 + }; + assert(GC_NEXT(gc) == container); + while (gc != container) { + /* Survivors will be moved to visited space, so they should + * have been marked as visited */ + assert(IS_IN_VISITED(gc, gcstate->visited_space)); + PyObject *op = FROM_GC(gc); + assert(_PyObject_GC_IS_TRACKED(op)); + if (_Py_IsImmortal(op)) { + PyGC_Head *next = GC_NEXT(gc); + gc_list_move(gc, &get_gc_state()->permanent_generation.head); + gc = next; + continue; } + traverseproc traverse = Py_TYPE(op)->tp_traverse; + (void) traverse(op, + visit_add_to_container, + &arg); + gc = GC_NEXT(gc); } + return arg.size; +} - assert(generation >= 0 && generation < NUM_GENERATIONS); - -#ifdef Py_STATS - if (_Py_stats) { - _Py_stats->object_stats.object_visits = 0; +/* Do bookkeeping for a completed GC cycle */ +static void +completed_scavenge(GCState *gcstate) +{ + /* We must observe two invariants: + * 1. Members of the permanent generation must be marked visited. + * 2. We cannot touch members of the permanent generation. */ + int visited; + if (gc_list_is_empty(&gcstate->permanent_generation.head)) { + /* Permanent generation is empty so we can flip spaces bit */ + int not_visited = gcstate->visited_space; + visited = other_space(not_visited); + gcstate->visited_space = visited; + /* Make sure all objects have visited bit set correctly */ + gc_list_set_space(&gcstate->young.head, not_visited); } -#endif - GC_STAT_ADD(generation, collections, 1); - - if (reason != _Py_GC_REASON_SHUTDOWN) { - invoke_gc_callback(tstate, "start", generation, 0, 0); + else { + /* We must move the objects from visited to pending space. */ + visited = gcstate->visited_space; + int not_visited = other_space(visited); + assert(gc_list_is_empty(&gcstate->old[not_visited].head)); + gc_list_merge(&gcstate->old[visited].head, &gcstate->old[not_visited].head); + gc_list_set_space(&gcstate->old[not_visited].head, not_visited); } + assert(gc_list_is_empty(&gcstate->old[visited].head)); + gcstate->work_to_do = 0; + gcstate->phase = GC_PHASE_MARK; +} - if (gcstate->debug & _PyGC_DEBUG_STATS) { - PySys_WriteStderr("gc: collecting generation %d...\n", generation); - show_stats_each_generations(gcstate); - t1 = _PyTime_PerfCounterUnchecked(); +static intptr_t +move_to_reachable(PyObject *op, PyGC_Head *reachable, int visited_space) +{ + if (op != NULL && !_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { + PyGC_Head *gc = AS_GC(op); + if (_PyObject_GC_IS_TRACKED(op) && + gc_old_space(gc) != visited_space) { + gc_flip_old_space(gc); + gc_list_move(gc, reachable); + return 1; + } } + return 0; +} - if (PyDTrace_GC_START_ENABLED()) { - PyDTrace_GC_START(generation); +static intptr_t +mark_all_reachable(PyGC_Head *reachable, PyGC_Head *visited, int visited_space) +{ + // Transitively traverse all objects from reachable, until empty + struct container_and_flag arg = { + .container = reachable, + .visited_space = visited_space, + .size = 0 + }; + while (!gc_list_is_empty(reachable)) { + PyGC_Head *gc = _PyGCHead_NEXT(reachable); + assert(gc_old_space(gc) == visited_space); + gc_list_move(gc, visited); + PyObject *op = FROM_GC(gc); + traverseproc traverse = Py_TYPE(op)->tp_traverse; + (void) traverse(op, + visit_add_to_container, + &arg); + } + gc_list_validate_space(visited, visited_space); + return arg.size; +} + +static intptr_t +mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, bool start) +{ + PyGC_Head reachable; + gc_list_init(&reachable); + Py_ssize_t objects_marked = 0; + // Move all objects on stacks to reachable + _PyRuntimeState *runtime = &_PyRuntime; + HEAD_LOCK(runtime); + PyThreadState* ts = PyInterpreterState_ThreadHead(interp); + HEAD_UNLOCK(runtime); + while (ts) { + _PyInterpreterFrame *frame = ts->current_frame; + while (frame) { + if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { + frame = frame->previous; + continue; + } + _PyStackRef *locals = frame->localsplus; + _PyStackRef *sp = frame->stackpointer; + objects_marked += move_to_reachable(frame->f_locals, &reachable, visited_space); + PyObject *func = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + objects_marked += move_to_reachable(func, &reachable, visited_space); + while (sp > locals) { + sp--; + if (PyStackRef_IsNull(*sp)) { + continue; + } + PyObject *op = PyStackRef_AsPyObjectBorrow(*sp); + if (!_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { + PyGC_Head *gc = AS_GC(op); + if (_PyObject_GC_IS_TRACKED(op) && + gc_old_space(gc) != visited_space) { + gc_flip_old_space(gc); + objects_marked++; + gc_list_move(gc, &reachable); + } + } + } + if (!start && frame->visited) { + // If this frame has already been visited, then the lower frames + // will have already been visited and will not have changed + break; + } + frame->visited = 1; + frame = frame->previous; + } + HEAD_LOCK(runtime); + ts = PyThreadState_Next(ts); + HEAD_UNLOCK(runtime); } + objects_marked += mark_all_reachable(&reachable, visited, visited_space); + assert(gc_list_is_empty(&reachable)); + return objects_marked; +} - /* update collection and allocation counters */ - if (generation+1 < NUM_GENERATIONS) { - gcstate->generations[generation+1].count += 1; - } - for (i = 0; i <= generation; i++) { - gcstate->generations[i].count = 0; +static intptr_t +mark_global_roots(PyInterpreterState *interp, PyGC_Head *visited, int visited_space) +{ + PyGC_Head reachable; + gc_list_init(&reachable); + Py_ssize_t objects_marked = 0; + objects_marked += move_to_reachable(interp->sysdict, &reachable, visited_space); + objects_marked += move_to_reachable(interp->builtins, &reachable, visited_space); + objects_marked += move_to_reachable(interp->dict, &reachable, visited_space); + struct types_state *types = &interp->types; + for (int i = 0; i < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; i++) { + objects_marked += move_to_reachable(types->builtins.initialized[i].tp_dict, &reachable, visited_space); + objects_marked += move_to_reachable(types->builtins.initialized[i].tp_subclasses, &reachable, visited_space); } - - /* merge younger generations with one we are currently collecting */ - for (i = 0; i < generation; i++) { - gc_list_merge(GEN_HEAD(gcstate, i), GEN_HEAD(gcstate, generation)); + for (int i = 0; i < _Py_MAX_MANAGED_STATIC_EXT_TYPES; i++) { + objects_marked += move_to_reachable(types->for_extensions.initialized[i].tp_dict, &reachable, visited_space); + objects_marked += move_to_reachable(types->for_extensions.initialized[i].tp_subclasses, &reachable, visited_space); } + objects_marked += mark_all_reachable(&reachable, visited, visited_space); + assert(gc_list_is_empty(&reachable)); + return objects_marked; +} - /* handy references */ - young = GEN_HEAD(gcstate, generation); - if (generation < NUM_GENERATIONS-1) { - old = GEN_HEAD(gcstate, generation+1); +static intptr_t +mark_at_start(PyThreadState *tstate) +{ + // TO DO -- Make this incremental + GCState *gcstate = &tstate->interp->gc; + PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; + Py_ssize_t objects_marked = mark_global_roots(tstate->interp, visited, gcstate->visited_space); + objects_marked += mark_stacks(tstate->interp, visited, gcstate->visited_space, true); + gcstate->work_to_do -= objects_marked; + gcstate->phase = GC_PHASE_COLLECT; + validate_spaces(gcstate); + return objects_marked; +} + +static intptr_t +assess_work_to_do(GCState *gcstate) +{ + /* The amount of work we want to do depends on three things. + * 1. The number of new objects created + * 2. The growth in heap size since the last collection + * 3. The heap size (up to the number of new objects, to avoid quadratic effects) + * + * For a steady state heap, the amount of work to do is three times the number + * of new objects added to the heap. This ensures that we stay ahead in the + * worst case of all new objects being garbage. + * + * This could be improved by tracking survival rates, but it is still a + * large improvement on the non-marking approach. + */ + intptr_t scale_factor = gcstate->old[0].threshold; + if (scale_factor < 2) { + scale_factor = 2; } - else { - old = young; + intptr_t new_objects = gcstate->young.count; + intptr_t max_heap_fraction = new_objects*3/2; + intptr_t heap_fraction = gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; + if (heap_fraction > max_heap_fraction) { + heap_fraction = max_heap_fraction; } - validate_list(old, collecting_clear_unreachable_clear); + gcstate->young.count = 0; + return new_objects + heap_fraction; +} - deduce_unreachable(young, &unreachable); +static void +gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) +{ + GC_STAT_ADD(1, collections, 1); + GCState *gcstate = &tstate->interp->gc; + gcstate->work_to_do += assess_work_to_do(gcstate); + untrack_tuples(&gcstate->young.head); + if (gcstate->phase == GC_PHASE_MARK) { + Py_ssize_t objects_marked = mark_at_start(tstate); + GC_STAT_ADD(1, objects_transitively_reachable, objects_marked); + gcstate->work_to_do -= objects_marked; + validate_spaces(gcstate); + return; + } + PyGC_Head *not_visited = &gcstate->old[gcstate->visited_space^1].head; + PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; + PyGC_Head increment; + gc_list_init(&increment); + int scale_factor = gcstate->old[0].threshold; + if (scale_factor < 2) { + scale_factor = 2; + } + intptr_t objects_marked = mark_stacks(tstate->interp, visited, gcstate->visited_space, false); + GC_STAT_ADD(1, objects_transitively_reachable, objects_marked); + gcstate->work_to_do -= objects_marked; + gc_list_set_space(&gcstate->young.head, gcstate->visited_space); + gc_list_merge(&gcstate->young.head, &increment); + gc_list_validate_space(&increment, gcstate->visited_space); + Py_ssize_t increment_size = gc_list_size(&increment); + while (increment_size < gcstate->work_to_do) { + if (gc_list_is_empty(not_visited)) { + break; + } + PyGC_Head *gc = _PyGCHead_NEXT(not_visited); + gc_list_move(gc, &increment); + increment_size++; + assert(!_Py_IsImmortal(FROM_GC(gc))); + gc_set_old_space(gc, gcstate->visited_space); + increment_size += expand_region_transitively_reachable(&increment, gc, gcstate); + } + GC_STAT_ADD(1, objects_not_transitively_reachable, increment_size); + validate_list(&increment, collecting_clear_unreachable_clear); + gc_list_validate_space(&increment, gcstate->visited_space); + PyGC_Head survivors; + gc_list_init(&survivors); + gc_collect_region(tstate, &increment, &survivors, stats); + gc_list_merge(&survivors, visited); + assert(gc_list_is_empty(&increment)); + gcstate->work_to_do += gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; + gcstate->work_to_do -= increment_size; + + add_stats(gcstate, 1, stats); + if (gc_list_is_empty(not_visited)) { + completed_scavenge(gcstate); + } + validate_spaces(gcstate); +} +static void +gc_collect_full(PyThreadState *tstate, + struct gc_collection_stats *stats) +{ + GC_STAT_ADD(2, collections, 1); + GCState *gcstate = &tstate->interp->gc; + validate_spaces(gcstate); + PyGC_Head *young = &gcstate->young.head; + PyGC_Head *pending = &gcstate->old[gcstate->visited_space^1].head; + PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; untrack_tuples(young); - /* Move reachable objects to next generation. */ - if (young != old) { - if (generation == NUM_GENERATIONS - 2) { - gcstate->long_lived_pending += gc_list_size(young); - } - gc_list_merge(young, old); - } - else { - /* We only un-track dicts in full collections, to avoid quadratic - dict build-up. See issue #14775. */ - untrack_dicts(young); - gcstate->long_lived_pending = 0; - gcstate->long_lived_total = gc_list_size(young); + /* merge all generations into visited */ + gc_list_merge(young, pending); + gc_list_validate_space(pending, 1-gcstate->visited_space); + gc_list_set_space(pending, gcstate->visited_space); + gcstate->young.count = 0; + gc_list_merge(pending, visited); + validate_spaces(gcstate); + + gc_collect_region(tstate, visited, visited, + stats); + validate_spaces(gcstate); + gcstate->young.count = 0; + gcstate->old[0].count = 0; + gcstate->old[1].count = 0; + completed_scavenge(gcstate); + _PyGC_ClearAllFreeLists(tstate->interp); + validate_spaces(gcstate); + add_stats(gcstate, 2, stats); +} + +/* This is the main function. Read this to understand how the + * collection process works. */ +static void +gc_collect_region(PyThreadState *tstate, + PyGC_Head *from, + PyGC_Head *to, + struct gc_collection_stats *stats) +{ + PyGC_Head unreachable; /* non-problematic unreachable trash */ + PyGC_Head finalizers; /* objects with, & reachable from, __del__ */ + PyGC_Head *gc; /* initialize to prevent a compiler warning */ + GCState *gcstate = &tstate->interp->gc; + + assert(gcstate->garbage != NULL); + assert(!_PyErr_Occurred(tstate)); + + gc_list_init(&unreachable); + deduce_unreachable(from, &unreachable); + validate_consistent_old_space(from); + untrack_tuples(from); + validate_consistent_old_space(to); + if (from != to) { + gc_list_merge(from, to); } + validate_consistent_old_space(to); + /* Move reachable objects to next generation. */ /* All objects in unreachable are trash, but objects reachable from * legacy finalizers (e.g. tp_del) can't safely be deleted. @@ -1387,10 +1712,8 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) * and we move those into the finalizers list too. */ move_legacy_finalizer_reachable(&finalizers); - validate_list(&finalizers, collecting_clear_unreachable_clear); validate_list(&unreachable, collecting_set_unreachable_clear); - /* Print debugging information. */ if (gcstate->debug & _PyGC_DEBUG_COLLECTABLE) { for (gc = GC_NEXT(&unreachable); gc != &unreachable; gc = GC_NEXT(gc)) { @@ -1399,89 +1722,101 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) } /* Clear weakrefs and invoke callbacks as necessary. */ - m += handle_weakrefs(&unreachable, old); - - validate_list(old, collecting_clear_unreachable_clear); + stats->collected += handle_weakrefs(&unreachable, to); + gc_list_validate_space(to, gcstate->visited_space); + validate_list(to, collecting_clear_unreachable_clear); validate_list(&unreachable, collecting_set_unreachable_clear); /* Call tp_finalize on objects which have one. */ finalize_garbage(tstate, &unreachable); - /* Handle any objects that may have resurrected after the call * to 'finalize_garbage' and continue the collection with the * objects that are still unreachable */ PyGC_Head final_unreachable; - handle_resurrected_objects(&unreachable, &final_unreachable, old); + gc_list_init(&final_unreachable); + handle_resurrected_objects(&unreachable, &final_unreachable, to); /* Call tp_clear on objects in the final_unreachable set. This will cause * the reference cycles to be broken. It may also cause some objects * in finalizers to be freed. */ - m += gc_list_size(&final_unreachable); - delete_garbage(tstate, gcstate, &final_unreachable, old); + stats->collected += gc_list_size(&final_unreachable); + delete_garbage(tstate, gcstate, &final_unreachable, to); /* Collect statistics on uncollectable objects found and print * debugging information. */ + Py_ssize_t n = 0; for (gc = GC_NEXT(&finalizers); gc != &finalizers; gc = GC_NEXT(gc)) { n++; - if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) + if (gcstate->debug & _PyGC_DEBUG_COLLECTABLE) debug_cycle("uncollectable", FROM_GC(gc)); } - if (gcstate->debug & _PyGC_DEBUG_STATS) { - double d = PyTime_AsSecondsDouble(_PyTime_PerfCounterUnchecked() - t1); - PySys_WriteStderr( - "gc: done, %zd unreachable, %zd uncollectable, %.4fs elapsed\n", - n+m, n, d); - } - + stats->uncollectable = n; /* Append instances in the uncollectable set to a Python * reachable list of garbage. The programmer has to deal with * this if they insist on creating this type of structure. */ - handle_legacy_finalizers(tstate, gcstate, &finalizers, old); - validate_list(old, collecting_clear_unreachable_clear); + handle_legacy_finalizers(tstate, gcstate, &finalizers, to); + gc_list_validate_space(to, gcstate->visited_space); + validate_list(to, collecting_clear_unreachable_clear); +} - /* Clear free list only during the collection of the highest - * generation */ - if (generation == NUM_GENERATIONS-1) { - _PyGC_ClearAllFreeLists(tstate->interp); - } +/* Invoke progress callbacks to notify clients that garbage collection + * is starting or stopping + */ +static void +do_gc_callback(GCState *gcstate, const char *phase, + int generation, struct gc_collection_stats *stats) +{ + assert(!PyErr_Occurred()); - if (_PyErr_Occurred(tstate)) { - if (reason == _Py_GC_REASON_SHUTDOWN) { - _PyErr_Clear(tstate); - } - else { - PyErr_FormatUnraisable("Exception ignored in garbage collection"); + /* The local variable cannot be rebound, check it for sanity */ + assert(PyList_CheckExact(gcstate->callbacks)); + PyObject *info = NULL; + if (PyList_GET_SIZE(gcstate->callbacks) != 0) { + info = Py_BuildValue("{sisnsn}", + "generation", generation, + "collected", stats->collected, + "uncollectable", stats->uncollectable); + if (info == NULL) { + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; } } - /* Update stats */ - struct gc_generation_stats *stats = &gcstate->generation_stats[generation]; - stats->collections++; - stats->collected += m; - stats->uncollectable += n; - - GC_STAT_ADD(generation, objects_collected, m); -#ifdef Py_STATS - if (_Py_stats) { - GC_STAT_ADD(generation, object_visits, - _Py_stats->object_stats.object_visits); - _Py_stats->object_stats.object_visits = 0; + PyObject *phase_obj = PyUnicode_FromString(phase); + if (phase_obj == NULL) { + Py_XDECREF(info); + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; } -#endif - if (PyDTrace_GC_DONE_ENABLED()) { - PyDTrace_GC_DONE(n + m); + PyObject *stack[] = {phase_obj, info}; + for (Py_ssize_t i=0; icallbacks); i++) { + PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); + Py_INCREF(cb); /* make sure cb doesn't go away */ + r = PyObject_Vectorcall(cb, stack, 2, NULL); + if (r == NULL) { + PyErr_WriteUnraisable(cb); + } + else { + Py_DECREF(r); + } + Py_DECREF(cb); } + Py_DECREF(phase_obj); + Py_XDECREF(info); + assert(!PyErr_Occurred()); +} - if (reason != _Py_GC_REASON_SHUTDOWN) { - invoke_gc_callback(tstate, "stop", generation, m, n); +static void +invoke_gc_callback(GCState *gcstate, const char *phase, + int generation, struct gc_collection_stats *stats) +{ + if (gcstate->callbacks == NULL) { + return; } - - assert(!_PyErr_Occurred(tstate)); - _Py_atomic_store_int(&gcstate->collecting, 0); - return n + m; + do_gc_callback(gcstate, phase, generation, stats); } static int @@ -1537,26 +1872,31 @@ _PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs) } PyObject * -_PyGC_GetObjects(PyInterpreterState *interp, Py_ssize_t generation) +_PyGC_GetObjects(PyInterpreterState *interp, int generation) { assert(generation >= -1 && generation < NUM_GENERATIONS); GCState *gcstate = &interp->gc; PyObject *result = PyList_New(0); - if (result == NULL) { - return NULL; + /* Generation: + * -1: Return all objects + * 0: All young objects + * 1: No objects + * 2: All old objects + */ + if (result == NULL || generation == 1) { + return result; } - - if (generation == -1) { - /* If generation is -1, get all objects from all generations */ - for (int i = 0; i < NUM_GENERATIONS; i++) { - if (append_objects(result, GEN_HEAD(gcstate, i))) { - goto error; - } + if (generation <= 0) { + if (append_objects(result, &gcstate->young.head)) { + goto error; } } - else { - if (append_objects(result, GEN_HEAD(gcstate, generation))) { + if (generation != 0) { + if (append_objects(result, &gcstate->old[0].head)) { + goto error; + } + if (append_objects(result, &gcstate->old[1].head)) { goto error; } } @@ -1571,10 +1911,23 @@ void _PyGC_Freeze(PyInterpreterState *interp) { GCState *gcstate = &interp->gc; - for (int i = 0; i < NUM_GENERATIONS; ++i) { - gc_list_merge(GEN_HEAD(gcstate, i), &gcstate->permanent_generation.head); - gcstate->generations[i].count = 0; + /* The permanent_generation must be visited */ + gc_list_set_space(&gcstate->young.head, gcstate->visited_space); + gc_list_merge(&gcstate->young.head, &gcstate->permanent_generation.head); + gcstate->young.count = 0; + PyGC_Head*old0 = &gcstate->old[0].head; + PyGC_Head*old1 = &gcstate->old[1].head; + if (gcstate->visited_space) { + gc_list_set_space(old0, 1); } + else { + gc_list_set_space(old1, 0); + } + gc_list_merge(old0, &gcstate->permanent_generation.head); + gcstate->old[0].count = 0; + gc_list_merge(old1, &gcstate->permanent_generation.head); + gcstate->old[1].count = 0; + validate_spaces(gcstate); } void @@ -1582,7 +1935,8 @@ _PyGC_Unfreeze(PyInterpreterState *interp) { GCState *gcstate = &interp->gc; gc_list_merge(&gcstate->permanent_generation.head, - GEN_HEAD(gcstate, NUM_GENERATIONS-1)); + &gcstate->old[gcstate->visited_space].head); + validate_spaces(gcstate); } Py_ssize_t @@ -1618,32 +1972,88 @@ PyGC_IsEnabled(void) return gcstate->enabled; } -/* Public API to invoke gc.collect() from C */ +// Show stats for objects in each generations +static void +show_stats_each_generations(GCState *gcstate) +{ + char buf[100]; + size_t pos = 0; + + for (int i = 0; i < NUM_GENERATIONS && pos < sizeof(buf); i++) { + pos += PyOS_snprintf(buf+pos, sizeof(buf)-pos, + " %zd", + gc_list_size(GEN_HEAD(gcstate, i))); + } + PySys_FormatStderr( + "gc: objects in each generation:%s\n" + "gc: objects in permanent generation: %zd\n", + buf, gc_list_size(&gcstate->permanent_generation.head)); +} + Py_ssize_t -PyGC_Collect(void) +_PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) { - PyThreadState *tstate = _PyThreadState_GET(); GCState *gcstate = &tstate->interp->gc; - if (!gcstate->enabled) { + int expected = 0; + if (!_Py_atomic_compare_exchange_int(&gcstate->collecting, &expected, 1)) { + // Don't start a garbage collection if one is already in progress. return 0; } - Py_ssize_t n; + struct gc_collection_stats stats = { 0 }; + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(gcstate, "start", generation, &stats); + } + if (gcstate->debug & _PyGC_DEBUG_STATS) { + PySys_WriteStderr("gc: collecting generation %d...\n", generation); + show_stats_each_generations(gcstate); + } + if (PyDTrace_GC_START_ENABLED()) { + PyDTrace_GC_START(generation); + } PyObject *exc = _PyErr_GetRaisedException(tstate); - n = gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_MANUAL); + switch(generation) { + case 0: + gc_collect_young(tstate, &stats); + break; + case 1: + gc_collect_increment(tstate, &stats); + break; + case 2: + gc_collect_full(tstate, &stats); + break; + default: + Py_UNREACHABLE(); + } + if (PyDTrace_GC_DONE_ENABLED()) { + PyDTrace_GC_DONE(stats.uncollectable + stats.collected); + } + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(gcstate, "stop", generation, &stats); + } _PyErr_SetRaisedException(tstate, exc); - - return n; + GC_STAT_ADD(generation, objects_collected, stats.collected); +#ifdef Py_STATS + if (_Py_stats) { + GC_STAT_ADD(generation, object_visits, + _Py_stats->object_stats.object_visits); + _Py_stats->object_stats.object_visits = 0; + } +#endif + validate_spaces(gcstate); + _Py_atomic_store_int(&gcstate->collecting, 0); + return stats.uncollectable + stats.collected; } +/* Public API to invoke gc.collect() from C */ Py_ssize_t -_PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) +PyGC_Collect(void) { - return gc_collect_main(tstate, generation, reason); + return _PyGC_Collect(_PyThreadState_GET(), 2, _Py_GC_REASON_MANUAL); } -Py_ssize_t +void _PyGC_CollectNoFail(PyThreadState *tstate) { /* Ideally, this function is only called on interpreter shutdown, @@ -1652,7 +2062,7 @@ _PyGC_CollectNoFail(PyThreadState *tstate) during interpreter shutdown (and then never finish it). See http://bugs.python.org/issue8713#msg195178 for an example. */ - return gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_SHUTDOWN); + _PyGC_Collect(_PyThreadState_GET(), 2, _Py_GC_REASON_SHUTDOWN); } void @@ -1696,6 +2106,13 @@ _PyGC_DumpShutdownStats(PyInterpreterState *interp) } } +static void +finalize_unlink_gc_head(PyGC_Head *gc) { + PyGC_Head *prev = GC_PREV(gc); + PyGC_Head *next = GC_NEXT(gc); + _PyGCHead_SET_NEXT(prev, next); + _PyGCHead_SET_PREV(next, prev); +} void _PyGC_Fini(PyInterpreterState *interp) @@ -1704,9 +2121,25 @@ _PyGC_Fini(PyInterpreterState *interp) Py_CLEAR(gcstate->garbage); Py_CLEAR(gcstate->callbacks); - /* We expect that none of this interpreters objects are shared - with other interpreters. - See https://github.com/python/cpython/issues/90228. */ + /* Prevent a subtle bug that affects sub-interpreters that use basic + * single-phase init extensions (m_size == -1). Those extensions cause objects + * to be shared between interpreters, via the PyDict_Update(mdict, m_copy) call + * in import_find_extension(). + * + * If they are GC objects, their GC head next or prev links could refer to + * the interpreter _gc_runtime_state PyGC_Head nodes. Those nodes go away + * when the interpreter structure is freed and so pointers to them become + * invalid. If those objects are still used by another interpreter and + * UNTRACK is called on them, a crash will happen. We untrack the nodes + * here to avoid that. + * + * This bug was originally fixed when reported as gh-90228. The bug was + * re-introduced in gh-94673. + */ + finalize_unlink_gc_head(&gcstate->young.head); + finalize_unlink_gc_head(&gcstate->old[0].head); + finalize_unlink_gc_head(&gcstate->old[1].head); + finalize_unlink_gc_head(&gcstate->permanent_generation.head); } /* for debugging */ @@ -1791,10 +2224,11 @@ _PyObject_GC_Link(PyObject *op) GCState *gcstate = &tstate->interp->gc; gc->_gc_next = 0; gc->_gc_prev = 0; - gcstate->generations[0].count++; /* number of allocated GC objects */ - if (gcstate->generations[0].count > gcstate->generations[0].threshold && + gcstate->young.count++; /* number of allocated GC objects */ + gcstate->heap_size++; + if (gcstate->young.count > gcstate->young.threshold && gcstate->enabled && - gcstate->generations[0].threshold && + gcstate->young.threshold && !_Py_atomic_load_int_relaxed(&gcstate->collecting) && !_PyErr_Occurred(tstate)) { @@ -1805,7 +2239,9 @@ _PyObject_GC_Link(PyObject *op) void _Py_RunGC(PyThreadState *tstate) { - gc_collect_main(tstate, GENERATION_AUTO, _Py_GC_REASON_HEAP); + if (tstate->interp->gc.enabled) { + _PyGC_Collect(tstate, 1, _Py_GC_REASON_HEAP); + } } static PyObject * @@ -1827,15 +2263,23 @@ gc_alloc(PyTypeObject *tp, size_t basicsize, size_t presize) return op; } + PyObject * _PyObject_GC_New(PyTypeObject *tp) { size_t presize = _PyType_PreHeaderSize(tp); - PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp), presize); + size_t size = _PyObject_SIZE(tp); + if (_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)) { + size += _PyInlineValuesSize(tp); + } + PyObject *op = gc_alloc(tp, size, presize); if (op == NULL) { return NULL; } _PyObject_Init(op, tp); + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + _PyObject_InitInlineValues(op, tp); + } return op; } @@ -1875,7 +2319,7 @@ PyVarObject * _PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems) { const size_t basicsize = _PyObject_VAR_SIZE(Py_TYPE(op), nitems); - const size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type); + const size_t presize = _PyType_PreHeaderSize(Py_TYPE(op)); _PyObject_ASSERT((PyObject *)op, !_PyObject_GC_IS_TRACKED(op)); if (basicsize > (size_t)PY_SSIZE_T_MAX - presize) { return (PyVarObject *)PyErr_NoMemory(); @@ -1893,7 +2337,7 @@ _PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems) void PyObject_GC_Del(void *op) { - size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type); + size_t presize = _PyType_PreHeaderSize(Py_TYPE(op)); PyGC_Head *g = AS_GC(op); if (_PyObject_GC_IS_TRACKED(op)) { gc_list_remove(g); @@ -1901,16 +2345,17 @@ PyObject_GC_Del(void *op) PyObject *exc = PyErr_GetRaisedException(); if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, "gc", NULL, "Object of type %s is not untracked before destruction", - ((PyObject*)op)->ob_type->tp_name)) { + Py_TYPE(op)->tp_name)) { PyErr_WriteUnraisable(NULL); } PyErr_SetRaisedException(exc); #endif } GCState *gcstate = get_gc_state(); - if (gcstate->generations[0].count > 0) { - gcstate->generations[0].count--; + if (gcstate->young.count > 0) { + gcstate->young.count--; } + gcstate->heap_size--; PyObject_Free(((char *)op)-presize); } @@ -1932,26 +2377,36 @@ PyObject_GC_IsFinalized(PyObject *obj) return 0; } +static int +visit_generation(gcvisitobjects_t callback, void *arg, struct gc_generation *gen) +{ + PyGC_Head *gc_list, *gc; + gc_list = &gen->head; + for (gc = GC_NEXT(gc_list); gc != gc_list; gc = GC_NEXT(gc)) { + PyObject *op = FROM_GC(gc); + Py_INCREF(op); + int res = callback(op, arg); + Py_DECREF(op); + if (!res) { + return -1; + } + } + return 0; +} + void PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) { - size_t i; GCState *gcstate = get_gc_state(); int origenstate = gcstate->enabled; gcstate->enabled = 0; - for (i = 0; i < NUM_GENERATIONS; i++) { - PyGC_Head *gc_list, *gc; - gc_list = GEN_HEAD(gcstate, i); - for (gc = GC_NEXT(gc_list); gc != gc_list; gc = GC_NEXT(gc)) { - PyObject *op = FROM_GC(gc); - Py_INCREF(op); - int res = callback(op, arg); - Py_DECREF(op); - if (!res) { - goto done; - } - } + if (visit_generation(callback, arg, &gcstate->young)) { + goto done; + } + if (visit_generation(callback, arg, &gcstate->old[0])) { + goto done; } + visit_generation(callback, arg, &gcstate->old[1]); done: gcstate->enabled = origenstate; } diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 18893c6c391fff..211d52d6cc7164 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -3,7 +3,8 @@ #include "pycore_brc.h" // struct _brc_thread_state #include "pycore_ceval.h" // _Py_set_eval_breaker_bit() #include "pycore_context.h" -#include "pycore_dict.h" // _PyDict_MaybeUntrack() +#include "pycore_dict.h" // _PyInlineValuesSize() +#include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_initconfig.h" #include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_object.h" @@ -11,10 +12,21 @@ #include "pycore_object_stack.h" #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_time.h" // _PyTime_GetPerfCounter() #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_weakref.h" // _PyWeakref_ClearRef() #include "pydtrace.h" +#include "pycore_uniqueid.h" // _PyObject_MergeThreadLocalRefcounts() + + +// enable the "mark alive" pass of GC +#define GC_ENABLE_MARK_ALIVE 1 + +// include additional roots in "mark alive" pass +#define GC_MARK_ALIVE_EXTRA_ROOTS 1 + +// include Python stacks as set of known roots +#define GC_MARK_ALIVE_STACKS 1 + #ifdef Py_GIL_DISABLED @@ -54,6 +66,11 @@ struct collection_state { struct visitor_args base; PyInterpreterState *interp; GCState *gcstate; + _PyGC_Reason reason; + // GH-129236: If we see an active frame without a valid stack pointer, + // we can't collect objects with deferred references because we may not + // see all references. + int skip_deferred_objects; Py_ssize_t collected; Py_ssize_t uncollectable; Py_ssize_t long_lived_total; @@ -87,7 +104,7 @@ worklist_pop(struct worklist *worklist) PyObject *op = (PyObject *)worklist->head; if (op != NULL) { worklist->head = op->ob_tid; - op->ob_tid = 0; + _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); } return op; } @@ -111,22 +128,66 @@ worklist_remove(struct worklist_iter *iter) iter->next = iter->ptr; } +static inline int +gc_has_bit(PyObject *op, uint8_t bit) +{ + return (op->ob_gc_bits & bit) != 0; +} + +static inline void +gc_set_bit(PyObject *op, uint8_t bit) +{ + op->ob_gc_bits |= bit; +} + +static inline void +gc_clear_bit(PyObject *op, uint8_t bit) +{ + op->ob_gc_bits &= ~bit; +} + +static inline int +gc_is_frozen(PyObject *op) +{ + return gc_has_bit(op, _PyGC_BITS_FROZEN); +} + static inline int gc_is_unreachable(PyObject *op) { - return (op->ob_gc_bits & _PyGC_BITS_UNREACHABLE) != 0; + return gc_has_bit(op, _PyGC_BITS_UNREACHABLE); } -static void +static inline void gc_set_unreachable(PyObject *op) { - op->ob_gc_bits |= _PyGC_BITS_UNREACHABLE; + gc_set_bit(op, _PyGC_BITS_UNREACHABLE); } -static void +static inline void gc_clear_unreachable(PyObject *op) { - op->ob_gc_bits &= ~_PyGC_BITS_UNREACHABLE; + gc_clear_bit(op, _PyGC_BITS_UNREACHABLE); +} + +static inline int +gc_is_alive(PyObject *op) +{ + return gc_has_bit(op, _PyGC_BITS_ALIVE); +} + +#ifdef GC_ENABLE_MARK_ALIVE +static inline void +gc_set_alive(PyObject *op) +{ + gc_set_bit(op, _PyGC_BITS_ALIVE); +} +#endif + +static inline void +gc_clear_alive(PyObject *op) +{ + gc_clear_bit(op, _PyGC_BITS_ALIVE); } // Initialize the `ob_tid` field to zero if the object is not already @@ -135,6 +196,7 @@ static void gc_maybe_init_refs(PyObject *op) { if (!gc_is_unreachable(op)) { + assert(!gc_is_alive(op)); gc_set_unreachable(op); op->ob_tid = 0; } @@ -168,7 +230,7 @@ merge_refcount(PyObject *op, Py_ssize_t extra) refcount += extra; #ifdef Py_REF_DEBUG - _Py_AddRefTotal(_PyInterpreterState_GET(), extra); + _Py_AddRefTotal(_PyThreadState_GET(), extra); #endif // No atomics necessary; all other threads in this interpreter are paused. @@ -178,9 +240,64 @@ merge_refcount(PyObject *op, Py_ssize_t extra) return refcount; } +static void +frame_disable_deferred_refcounting(_PyInterpreterFrame *frame) +{ + // Convert locals, variables, and the executable object to strong + // references from (possibly) deferred references. + assert(frame->stackpointer != NULL); + assert(frame->owner == FRAME_OWNED_BY_FRAME_OBJECT || + frame->owner == FRAME_OWNED_BY_GENERATOR); + + frame->f_executable = PyStackRef_AsStrongReference(frame->f_executable); + + if (frame->owner == FRAME_OWNED_BY_GENERATOR) { + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + if (gen->gi_frame_state == FRAME_CLEARED) { + // gh-124068: if the generator is cleared, then most fields other + // than f_executable are not valid. + return; + } + } + + frame->f_funcobj = PyStackRef_AsStrongReference(frame->f_funcobj); + for (_PyStackRef *ref = frame->localsplus; ref < frame->stackpointer; ref++) { + if (!PyStackRef_IsNull(*ref) && PyStackRef_IsDeferred(*ref)) { + *ref = PyStackRef_AsStrongReference(*ref); + } + } +} + +static void +disable_deferred_refcounting(PyObject *op) +{ + if (_PyObject_HasDeferredRefcount(op)) { + op->ob_gc_bits &= ~_PyGC_BITS_DEFERRED; + op->ob_ref_shared -= _Py_REF_SHARED(_Py_REF_DEFERRED, 0); + merge_refcount(op, 0); + + // Heap types and code objects also use per-thread refcounting, which + // should also be disabled when we turn off deferred refcounting. + _PyObject_DisablePerThreadRefcounting(op); + } + + // Generators and frame objects may contain deferred references to other + // objects. If the pointed-to objects are part of cyclic trash, we may + // have disabled deferred refcounting on them and need to ensure that we + // use strong references, in case the generator or frame object is + // resurrected by a finalizer. + if (PyGen_CheckExact(op) || PyCoro_CheckExact(op) || PyAsyncGen_CheckExact(op)) { + frame_disable_deferred_refcounting(&((PyGenObject *)op)->gi_iframe); + } + else if (PyFrame_Check(op)) { + frame_disable_deferred_refcounting(((PyFrameObject *)op)->f_frame); + } +} + static void gc_restore_tid(PyObject *op) { + assert(_PyInterpreterState_GET()->stoptheworld.world_stopped); mi_segment_t *segment = _mi_ptr_segment(op); if (_Py_REF_IS_MERGED(op->ob_ref_shared)) { op->ob_tid = 0; @@ -201,9 +318,13 @@ static void gc_restore_refs(PyObject *op) { if (gc_is_unreachable(op)) { + assert(!gc_is_alive(op)); gc_restore_tid(op); gc_clear_unreachable(op); } + else { + gc_clear_alive(op); + } } // Given a mimalloc memory block return the PyObject stored in it or NULL if @@ -220,7 +341,7 @@ op_from_block(void *block, void *arg, bool include_frozen) if (!_PyObject_GC_IS_TRACKED(op)) { return NULL; } - if (!include_frozen && (op->ob_gc_bits & _PyGC_BITS_FROZEN) != 0) { + if (!include_frozen && gc_is_frozen(op)) { return NULL; } return op; @@ -241,8 +362,12 @@ gc_visit_heaps_lock_held(PyInterpreterState *interp, mi_block_visit_fun *visitor Py_ssize_t offset_pre = offset_base + 2 * sizeof(PyObject*); // visit each thread's heaps for GC objects - for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, p) { struct _mimalloc_thread_state *m = &((_PyThreadStateImpl *)p)->mimalloc; + if (!_Py_atomic_load_int(&m->initialized)) { + // The thread may not have called tstate_mimalloc_bind() yet. + continue; + } arg->offset = offset_base; if (!mi_heap_visit_blocks(&m->heaps[_Py_MIMALLOC_HEAP_GC], true, @@ -289,6 +414,183 @@ gc_visit_heaps(PyInterpreterState *interp, mi_block_visit_fun *visitor, return err; } +static inline void +gc_visit_stackref(_PyStackRef stackref) +{ + if (PyStackRef_IsDeferred(stackref) && !PyStackRef_IsNull(stackref)) { + PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref); + if (_PyObject_GC_IS_TRACKED(obj) && !gc_is_frozen(obj)) { + gc_add_refs(obj, 1); + } + } +} + +// Add 1 to the gc_refs for every deferred reference on each thread's stack. +static void +gc_visit_thread_stacks(PyInterpreterState *interp, struct collection_state *state) +{ + _Py_FOR_EACH_TSTATE_BEGIN(interp, p) { + for (_PyInterpreterFrame *f = p->current_frame; f != NULL; f = f->previous) { + if (f->owner >= FRAME_OWNED_BY_INTERPRETER) { + continue; + } + + _PyStackRef *top = f->stackpointer; + if (top == NULL) { + // GH-129236: The stackpointer may be NULL in cases where + // the GC is run during a PyStackRef_CLOSE() call. Skip this + // frame and don't collect objects with deferred references. + state->skip_deferred_objects = 1; + continue; + } + + gc_visit_stackref(f->f_executable); + while (top != f->localsplus) { + --top; + gc_visit_stackref(*top); + } + } + } + _Py_FOR_EACH_TSTATE_END(interp); +} + +// Untrack objects that can never create reference cycles. +// Return true if the object was untracked. +static bool +gc_maybe_untrack(PyObject *op) +{ + // Currently we only check for tuples containing only non-GC objects. In + // theory we could check other immutable objects that contain references + // to non-GC objects. + if (PyTuple_CheckExact(op)) { + _PyTuple_MaybeUntrack(op); + if (!_PyObject_GC_IS_TRACKED(op)) { + return true; + } + } + return false; +} + +#ifdef GC_ENABLE_MARK_ALIVE +static int +mark_alive_stack_push(PyObject *op, _PyObjectStack *stack) +{ + if (op == NULL) { + return 0; + } + if (!_PyObject_GC_IS_TRACKED(op)) { + return 0; + } + if (gc_is_alive(op)) { + return 0; // already visited this object + } + if (gc_maybe_untrack(op)) { + return 0; // was untracked, don't visit it + } + + // Need to call tp_traverse on this object. Add to stack and mark it + // alive so we don't traverse it a second time. + gc_set_alive(op); + if (_PyObjectStack_Push(stack, op) < 0) { + return -1; + } + return 0; +} + +static bool +gc_clear_alive_bits(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + if (gc_is_alive(op)) { + gc_clear_alive(op); + } + return true; +} + +static void +gc_abort_mark_alive(PyInterpreterState *interp, + struct collection_state *state, + _PyObjectStack *stack) +{ + // We failed to allocate memory for "stack" while doing the "mark + // alive" phase. In that case, free the object stack and make sure + // that no objects have the alive bit set. + _PyObjectStack_Clear(stack); + gc_visit_heaps(interp, &gc_clear_alive_bits, &state->base); +} + +#ifdef GC_MARK_ALIVE_STACKS +static int +gc_visit_stackref_mark_alive(_PyObjectStack *stack, _PyStackRef stackref) +{ + if (!PyStackRef_IsNull(stackref)) { + PyObject *op = PyStackRef_AsPyObjectBorrow(stackref); + if (mark_alive_stack_push(op, stack) < 0) { + return -1; + } + } + return 0; +} + +static int +gc_visit_thread_stacks_mark_alive(PyInterpreterState *interp, _PyObjectStack *stack) +{ + int err = 0; + _Py_FOR_EACH_TSTATE_BEGIN(interp, p) { + for (_PyInterpreterFrame *f = p->current_frame; f != NULL; f = f->previous) { + if (f->owner >= FRAME_OWNED_BY_INTERPRETER) { + continue; + } + + if (f->stackpointer == NULL) { + // GH-129236: The stackpointer may be NULL in cases where + // the GC is run during a PyStackRef_CLOSE() call. Skip this + // frame for now. + continue; + } + + _PyStackRef *top = f->stackpointer; + if (gc_visit_stackref_mark_alive(stack, f->f_executable) < 0) { + err = -1; + goto exit; + } + while (top != f->localsplus) { + --top; + if (gc_visit_stackref_mark_alive(stack, *top) < 0) { + err = -1; + goto exit; + } + } + } + } +exit: + _Py_FOR_EACH_TSTATE_END(interp); + return err; +} +#endif // GC_MARK_ALIVE_STACKS +#endif // GC_ENABLE_MARK_ALIVE + +static void +queue_untracked_obj_decref(PyObject *op, struct collection_state *state) +{ + if (!_PyObject_GC_IS_TRACKED(op)) { + // GC objects with zero refcount are handled subsequently by the + // GC as if they were cyclic trash, but we have to handle dead + // non-GC objects here. Add one to the refcount so that we can + // decref and deallocate the object once we start the world again. + op->ob_ref_shared += (1 << _Py_REF_SHARED_SHIFT); +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyThreadState_GET()); +#endif + worklist_push(&state->objs_to_decref, op); + } + +} + static void merge_queued_objects(_PyThreadStateImpl *tstate, struct collection_state *state) { @@ -300,52 +602,49 @@ merge_queued_objects(_PyThreadStateImpl *tstate, struct collection_state *state) // Subtract one when merging because the queue had a reference. Py_ssize_t refcount = merge_refcount(op, -1); - if (!_PyObject_GC_IS_TRACKED(op) && refcount == 0) { - // GC objects with zero refcount are handled subsequently by the - // GC as if they were cyclic trash, but we have to handle dead - // non-GC objects here. Add one to the refcount so that we can - // decref and deallocate the object once we start the world again. - op->ob_ref_shared += (1 << _Py_REF_SHARED_SHIFT); -#ifdef Py_REF_DEBUG - _Py_IncRefTotal(_PyInterpreterState_GET()); -#endif - worklist_push(&state->objs_to_decref, op); + if (refcount == 0) { + queue_untracked_obj_decref(op, state); } } } static void -merge_all_queued_objects(PyInterpreterState *interp, struct collection_state *state) +queue_freed_object(PyObject *obj, void *arg) { - HEAD_LOCK(&_PyRuntime); - for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { - merge_queued_objects((_PyThreadStateImpl *)p, state); - } - HEAD_UNLOCK(&_PyRuntime); + queue_untracked_obj_decref(obj, arg); } static void -process_delayed_frees(PyInterpreterState *interp) +process_delayed_frees(PyInterpreterState *interp, struct collection_state *state) { - // In STW status, we can observe the latest write sequence by - // advancing the write sequence immediately. + // While we are in a "stop the world" pause, we can observe the latest + // write sequence by advancing the write sequence immediately. _Py_qsbr_advance(&interp->qsbr); _PyThreadStateImpl *current_tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); _Py_qsbr_quiescent_state(current_tstate->qsbr); - HEAD_LOCK(&_PyRuntime); - PyThreadState *tstate = interp->threads.head; - while (tstate != NULL) { - _PyMem_ProcessDelayed(tstate); - tstate = (PyThreadState *)tstate->next; + + // Merge the queues from other threads into our own queue so that we can + // process all of the pending delayed free requests at once. + _Py_FOR_EACH_TSTATE_BEGIN(interp, p) { + _PyThreadStateImpl *other = (_PyThreadStateImpl *)p; + if (other != current_tstate) { + llist_concat(¤t_tstate->mem_free_queue, &other->mem_free_queue); + } } - HEAD_UNLOCK(&_PyRuntime); + _Py_FOR_EACH_TSTATE_END(interp); + + _PyMem_ProcessDelayedNoDealloc((PyThreadState *)current_tstate, queue_freed_object, state); } // Subtract an incoming reference from the computed "gc_refs" refcount. static int visit_decref(PyObject *op, void *arg) { - if (_PyObject_GC_IS_TRACKED(op) && !_Py_IsImmortal(op)) { + if (_PyObject_GC_IS_TRACKED(op) + && !_Py_IsImmortal(op) + && !gc_is_frozen(op) + && !gc_is_alive(op)) + { // If update_refs hasn't reached this object yet, mark it // as (tentatively) unreachable and initialize ob_tid to zero. gc_maybe_init_refs(op); @@ -366,6 +665,10 @@ update_refs(const mi_heap_t *heap, const mi_heap_area_t *area, return true; } + if (gc_is_alive(op)) { + return true; + } + // Exclude immortal objects from garbage collection if (_Py_IsImmortal(op)) { op->ob_tid = 0; @@ -374,25 +677,19 @@ update_refs(const mi_heap_t *heap, const mi_heap_area_t *area, return true; } - // Untrack tuples and dicts as necessary in this pass. - if (PyTuple_CheckExact(op)) { - _PyTuple_MaybeUntrack(op); - if (!_PyObject_GC_IS_TRACKED(op)) { - gc_restore_refs(op); - return true; - } + Py_ssize_t refcount = Py_REFCNT(op); + if (_PyObject_HasDeferredRefcount(op)) { + refcount -= _Py_REF_DEFERRED; } - else if (PyDict_CheckExact(op)) { - _PyDict_MaybeUntrack(op); - if (!_PyObject_GC_IS_TRACKED(op)) { + _PyObject_ASSERT(op, refcount >= 0); + + if (refcount > 0 && !_PyObject_HasDeferredRefcount(op)) { + if (gc_maybe_untrack(op)) { gc_restore_refs(op); return true; } } - Py_ssize_t refcount = Py_REFCNT(op); - _PyObject_ASSERT(op, refcount >= 0); - // We repurpose ob_tid to compute "gc_refs", the number of external // references to the object (i.e., from outside the GC heaps). This means // that ob_tid is no longer a valid thread id until it is restored by @@ -438,6 +735,45 @@ mark_reachable(PyObject *op) } #ifdef GC_DEBUG +static bool +validate_alive_bits(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + + _PyObject_ASSERT_WITH_MSG(op, !gc_is_alive(op), + "object should not be marked as alive yet"); + + return true; +} + +static bool +validate_refcounts(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + + _PyObject_ASSERT_WITH_MSG(op, !gc_is_unreachable(op), + "object should not be marked as unreachable yet"); + + if (_Py_REF_IS_MERGED(op->ob_ref_shared)) { + _PyObject_ASSERT_WITH_MSG(op, op->ob_tid == 0, + "merged objects should have ob_tid == 0"); + } + else if (!_Py_IsImmortal(op)) { + _PyObject_ASSERT_WITH_MSG(op, op->ob_tid != 0, + "unmerged objects should have ob_tid != 0"); + } + + return true; +} + static bool validate_gc_objects(const mi_heap_t *heap, const mi_heap_area_t *area, void *block, size_t block_size, void *args) @@ -447,6 +783,11 @@ validate_gc_objects(const mi_heap_t *heap, const mi_heap_area_t *area, return true; } + if (gc_is_alive(op)) { + _PyObject_ASSERT(op, !gc_is_unreachable(op)); + return true; + } + _PyObject_ASSERT(op, gc_is_unreachable(op)); _PyObject_ASSERT_WITH_MSG(op, gc_get_refs(op) >= 0, "refcount is too small"); @@ -463,7 +804,23 @@ mark_heap_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, return true; } - if (gc_is_unreachable(op) && gc_get_refs(op) != 0) { + _PyObject_ASSERT_WITH_MSG(op, gc_get_refs(op) >= 0, + "refcount is too small"); + + if (gc_is_alive(op) || !gc_is_unreachable(op)) { + // Object was already marked as reachable. + return true; + } + + // GH-129236: If we've seen an active frame without a valid stack pointer, + // then we can't collect objects with deferred references because we may + // have missed some reference to the object on the stack. In that case, + // treat the object as reachable even if gc_refs is zero. + struct collection_state *state = (struct collection_state *)args; + int keep_alive = (state->skip_deferred_objects && + _PyObject_HasDeferredRefcount(op)); + + if (gc_get_refs(op) != 0 || keep_alive) { // Object is reachable but currently marked as unreachable. // Mark it as reachable and traverse its pointers to find // any other object that may be directly reachable from it. @@ -478,6 +835,20 @@ mark_heap_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, return true; } +static bool +restore_refs(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + gc_restore_tid(op); + gc_clear_unreachable(op); + gc_clear_alive(op); + return true; +} + /* Return true if object has a pre-PEP 442 finalization method. */ static int has_legacy_finalizer(PyObject *op) @@ -496,6 +867,10 @@ scan_heap_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, struct collection_state *state = (struct collection_state *)args; if (gc_is_unreachable(op)) { + // Disable deferred refcounting for unreachable objects so that they + // are collected immediately after finalization. + disable_deferred_refcounting(op); + // Merge and add one to the refcount to prevent deallocation while we // are holding on to it in a worklist. merge_refcount(op, 1); @@ -508,23 +883,120 @@ scan_heap_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, else { worklist_push(&state->unreachable, op); } + return true; } - else { - // object is reachable, restore `ob_tid`; we're done with these objects - gc_restore_tid(op); - state->long_lived_total++; + + if (state->reason == _Py_GC_REASON_SHUTDOWN) { + // Disable deferred refcounting for reachable objects as well during + // interpreter shutdown. This ensures that these objects are collected + // immediately when their last reference is removed. + disable_deferred_refcounting(op); } + // object is reachable, restore `ob_tid`; we're done with these objects + gc_restore_tid(op); + gc_clear_alive(op); + state->long_lived_total++; return true; } static int move_legacy_finalizer_reachable(struct collection_state *state); +#ifdef GC_ENABLE_MARK_ALIVE +static int +propagate_alive_bits(_PyObjectStack *stack) +{ + for (;;) { + PyObject *op = _PyObjectStack_Pop(stack); + if (op == NULL) { + break; + } + assert(_PyObject_GC_IS_TRACKED(op)); + assert(gc_is_alive(op)); + traverseproc traverse = Py_TYPE(op)->tp_traverse; + if (traverse(op, (visitproc)&mark_alive_stack_push, stack) < 0) { + return -1; + } + } + return 0; +} + +// Using tp_traverse, mark everything reachable from known root objects +// (which must be non-garbage) as alive (_PyGC_BITS_ALIVE is set). In +// most programs, this marks nearly all objects that are not actually +// unreachable. +// +// Actually alive objects can be missed in this pass if they are alive +// due to being referenced from an unknown root (e.g. an extension +// module global), some tp_traverse methods are either missing or not +// accurate, or objects that have been untracked. Objects that are only +// reachable from the aforementioned are also missed. +// +// If gc.freeze() is used, this pass is disabled since it is unlikely to +// help much. The next stages of cyclic GC will ignore objects with the +// alive bit set. +// +// Returns -1 on failure (out of memory). +static int +mark_alive_from_roots(PyInterpreterState *interp, + struct collection_state *state) +{ +#ifdef GC_DEBUG + // Check that all objects don't have alive bit set + gc_visit_heaps(interp, &validate_alive_bits, &state->base); +#endif + _PyObjectStack stack = { NULL }; + + #define STACK_PUSH(op) \ + if (mark_alive_stack_push(op, &stack) < 0) { \ + gc_abort_mark_alive(interp, state, &stack); \ + return -1; \ + } + STACK_PUSH(interp->sysdict); +#ifdef GC_MARK_ALIVE_EXTRA_ROOTS + STACK_PUSH(interp->builtins); + STACK_PUSH(interp->dict); + struct types_state *types = &interp->types; + for (int i = 0; i < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; i++) { + STACK_PUSH(types->builtins.initialized[i].tp_dict); + STACK_PUSH(types->builtins.initialized[i].tp_subclasses); + } + for (int i = 0; i < _Py_MAX_MANAGED_STATIC_EXT_TYPES; i++) { + STACK_PUSH(types->for_extensions.initialized[i].tp_dict); + STACK_PUSH(types->for_extensions.initialized[i].tp_subclasses); + } +#endif +#ifdef GC_MARK_ALIVE_STACKS + if (gc_visit_thread_stacks_mark_alive(interp, &stack) < 0) { + gc_abort_mark_alive(interp, state, &stack); + return -1; + } +#endif + #undef STACK_PUSH + + // Use tp_traverse to find everything reachable from roots. + if (propagate_alive_bits(&stack) < 0) { + gc_abort_mark_alive(interp, state, &stack); + return -1; + } + + return 0; +} +#endif // GC_ENABLE_MARK_ALIVE + + static int deduce_unreachable_heap(PyInterpreterState *interp, struct collection_state *state) { + +#ifdef GC_DEBUG + // Check that all objects are marked as unreachable and that the computed + // reference count difference (stored in `ob_tid`) is non-negative. + gc_visit_heaps(interp, &validate_refcounts, &state->base); +#endif + // Identify objects that are directly reachable from outside the GC heap // by computing the difference between the refcount and the number of // incoming references. @@ -536,9 +1008,14 @@ deduce_unreachable_heap(PyInterpreterState *interp, gc_visit_heaps(interp, &validate_gc_objects, &state->base); #endif + // Visit the thread stacks to account for any deferred references. + gc_visit_thread_stacks(interp, state); + // Transitively mark reachable objects by clearing the // _PyGC_BITS_UNREACHABLE flag. if (gc_visit_heaps(interp, &mark_heap_visitor, &state->base) < 0) { + // On out-of-memory, restore the refcounts and bail out. + gc_visit_heaps(interp, &restore_refs, &state->base); return -1; } @@ -657,7 +1134,6 @@ call_weakref_callbacks(struct collection_state *state) Py_DECREF(temp); } - gc_restore_tid(op); Py_DECREF(op); // drop worklist reference } } @@ -675,7 +1151,7 @@ void _PyGC_InitState(GCState *gcstate) { // TODO: move to pycore_runtime_init.h once the incremental GC lands. - gcstate->generations[0].threshold = 2000; + gcstate->young.threshold = 2000; } @@ -815,6 +1291,31 @@ visit_decref_unreachable(PyObject *op, void *data) return 0; } +int +_PyGC_VisitStackRef(_PyStackRef *ref, visitproc visit, void *arg) +{ + // This is a bit tricky! We want to ignore deferred references when + // computing the incoming references, but otherwise treat them like + // regular references. + if (!PyStackRef_IsDeferred(*ref) || + (visit != visit_decref && visit != visit_decref_unreachable)) + { + Py_VISIT(PyStackRef_AsPyObjectBorrow(*ref)); + } + return 0; +} + +int +_PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg) +{ + _PyStackRef *ref = _PyFrame_GetLocalsArray(frame); + /* locals and stack */ + for (; ref < frame->stackpointer; ref++) { + _Py_VISIT_STACKREF(*ref); + } + return 0; +} + // Handle objects that may have resurrected after a call to 'finalize_garbage'. static int handle_resurrected_objects(struct collection_state *state) @@ -961,7 +1462,6 @@ cleanup_worklist(struct worklist *worklist) { PyObject *op; while ((op = worklist_pop(worklist)) != NULL) { - gc_restore_tid(op); gc_clear_unreachable(op); Py_DECREF(op); } @@ -970,8 +1470,8 @@ cleanup_worklist(struct worklist *worklist) static bool gc_should_collect(GCState *gcstate) { - int count = _Py_atomic_load_int_relaxed(&gcstate->generations[0].count); - int threshold = gcstate->generations[0].threshold; + int count = _Py_atomic_load_int_relaxed(&gcstate->young.count); + int threshold = gcstate->young.threshold; if (count <= threshold || threshold == 0 || !gcstate->enabled) { return false; } @@ -979,7 +1479,7 @@ gc_should_collect(GCState *gcstate) // objects. A few tests rely on immediate scheduling of the GC so we ignore // the scaled threshold if generations[1].threshold is set to zero. return (count > gcstate->long_lived_total / 4 || - gcstate->generations[1].threshold == 0); + gcstate->old[0].threshold == 0); } static void @@ -993,7 +1493,7 @@ record_allocation(PyThreadState *tstate) if (gc->alloc_count >= LOCAL_ALLOC_COUNT_THRESHOLD) { // TODO: Use Py_ssize_t for the generation count. GCState *gcstate = &tstate->interp->gc; - _Py_atomic_add_int(&gcstate->generations[0].count, (int)gc->alloc_count); + _Py_atomic_add_int(&gcstate->young.count, (int)gc->alloc_count); gc->alloc_count = 0; if (gc_should_collect(gcstate) && @@ -1012,26 +1512,71 @@ record_deallocation(PyThreadState *tstate) gc->alloc_count--; if (gc->alloc_count <= -LOCAL_ALLOC_COUNT_THRESHOLD) { GCState *gcstate = &tstate->interp->gc; - _Py_atomic_add_int(&gcstate->generations[0].count, (int)gc->alloc_count); + _Py_atomic_add_int(&gcstate->young.count, (int)gc->alloc_count); gc->alloc_count = 0; } } static void -gc_collect_internal(PyInterpreterState *interp, struct collection_state *state) +gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, int generation) { _PyEval_StopTheWorld(interp); - // merge refcounts for all queued objects - merge_all_queued_objects(interp, state); - process_delayed_frees(interp); + + // update collection and allocation counters + if (generation+1 < NUM_GENERATIONS) { + state->gcstate->old[generation].count += 1; + } + + state->gcstate->young.count = 0; + for (int i = 1; i <= generation; ++i) { + state->gcstate->old[i-1].count = 0; + } + + _Py_FOR_EACH_TSTATE_BEGIN(interp, p) { + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)p; + + // merge per-thread refcount for types into the type's actual refcount + _PyObject_MergePerThreadRefcounts(tstate); + + // merge refcounts for all queued objects + merge_queued_objects(tstate, state); + } + _Py_FOR_EACH_TSTATE_END(interp); + + process_delayed_frees(interp, state); + + #ifdef GC_ENABLE_MARK_ALIVE + // If gc.freeze() was used, it seems likely that doing this "mark alive" + // pass will not be a performance win. Typically the majority of alive + // objects will be marked as frozen and will be skipped anyhow, without + // doing this extra work. Doing this pass also defeats one of the + // purposes of using freeze: avoiding writes to objects that are frozen. + // So, we just skip this if gc.freeze() was used. + if (!state->gcstate->freeze_active) { + // Mark objects reachable from known roots as "alive". These will + // be ignored for rest of the GC pass. + int err = mark_alive_from_roots(interp, state); + if (err < 0) { + _PyEval_StartTheWorld(interp); + PyErr_NoMemory(); + return; + } + } + #endif // Find unreachable objects int err = deduce_unreachable_heap(interp, state); if (err < 0) { _PyEval_StartTheWorld(interp); - goto error; + PyErr_NoMemory(); + return; } +#ifdef GC_DEBUG + // At this point, no object should have the alive bit set + gc_visit_heaps(interp, &validate_alive_bits, &state->base); +#endif + // Print debugging information. if (interp->gc.debug & _PyGC_DEBUG_COLLECTABLE) { PyObject *op; @@ -1058,10 +1603,17 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state) // Handle any objects that may have resurrected after the finalization. _PyEval_StopTheWorld(interp); err = handle_resurrected_objects(state); + // Clear free lists in all threads + _PyGC_ClearAllFreeLists(interp); _PyEval_StartTheWorld(interp); if (err < 0) { - goto error; + cleanup_worklist(&state->unreachable); + cleanup_worklist(&state->legacy_finalizers); + cleanup_worklist(&state->wrcb_to_call); + cleanup_worklist(&state->objs_to_decref); + PyErr_NoMemory(); + return; } // Call tp_clear on objects in the unreachable set. This will cause @@ -1071,15 +1623,6 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state) // Append objects with legacy finalizers to the "gc.garbage" list. handle_legacy_finalizers(state); - return; - -error: - cleanup_worklist(&state->unreachable); - cleanup_worklist(&state->legacy_finalizers); - cleanup_worklist(&state->wrcb_to_call); - cleanup_worklist(&state->objs_to_decref); - PyErr_NoMemory(); - PyErr_FormatUnraisable("Out of memory during garbage collection"); } /* This is the main function. Read this to understand how the @@ -1087,7 +1630,6 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state) static Py_ssize_t gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) { - int i; Py_ssize_t m = 0; /* # objects collected */ Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */ PyTime_t t1 = 0; /* initialize to prevent a compiler warning */ @@ -1126,42 +1668,39 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) if (gcstate->debug & _PyGC_DEBUG_STATS) { PySys_WriteStderr("gc: collecting generation %d...\n", generation); show_stats_each_generations(gcstate); - t1 = _PyTime_PerfCounterUnchecked(); + // ignore error: don't interrupt the GC if reading the clock fails + (void)PyTime_PerfCounterRaw(&t1); } if (PyDTrace_GC_START_ENABLED()) { PyDTrace_GC_START(generation); } - /* update collection and allocation counters */ - if (generation+1 < NUM_GENERATIONS) { - gcstate->generations[generation+1].count += 1; - } - for (i = 0; i <= generation; i++) { - gcstate->generations[i].count = 0; - } - PyInterpreterState *interp = tstate->interp; struct collection_state state = { .interp = interp, .gcstate = gcstate, + .reason = reason, }; - gc_collect_internal(interp, &state); + gc_collect_internal(interp, &state, generation); m = state.collected; n = state.uncollectable; if (gcstate->debug & _PyGC_DEBUG_STATS) { - double d = PyTime_AsSecondsDouble(_PyTime_PerfCounterUnchecked() - t1); + PyTime_t t2; + (void)PyTime_PerfCounterRaw(&t2); + double d = PyTime_AsSecondsDouble(t2 - t1); PySys_WriteStderr( "gc: done, %zd unreachable, %zd uncollectable, %.4fs elapsed\n", n+m, n, d); } - // Clear free lists in all threads - _PyGC_ClearAllFreeLists(interp); + // Clear the current thread's free-list again. + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + _PyObject_ClearFreeLists(&tstate_impl->freelists, 0); if (_PyErr_Occurred(tstate)) { if (reason == _Py_GC_REASON_SHUTDOWN) { @@ -1200,10 +1739,32 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) return n + m; } +static PyObject * +list_from_object_stack(_PyObjectStack *stack) +{ + PyObject *list = PyList_New(_PyObjectStack_Size(stack)); + if (list == NULL) { + PyObject *op; + while ((op = _PyObjectStack_Pop(stack)) != NULL) { + Py_DECREF(op); + } + return NULL; + } + + PyObject *op; + Py_ssize_t idx = 0; + while ((op = _PyObjectStack_Pop(stack)) != NULL) { + assert(idx < PyList_GET_SIZE(list)); + PyList_SET_ITEM(list, idx++, op); + } + assert(idx == PyList_GET_SIZE(list)); + return list; +} + struct get_referrers_args { struct visitor_args base; PyObject *objs; - struct worklist results; + _PyObjectStack results; }; static int @@ -1227,11 +1788,21 @@ visit_get_referrers(const mi_heap_t *heap, const mi_heap_area_t *area, if (op == NULL) { return true; } + if (op->ob_gc_bits & (_PyGC_BITS_UNREACHABLE | _PyGC_BITS_FROZEN)) { + // Exclude unreachable objects (in-progress GC) and frozen + // objects from gc.get_objects() to match the default build. + return true; + } struct get_referrers_args *arg = (struct get_referrers_args *)args; + if (op == arg->objs) { + // Don't include the tuple itself in the referrers list. + return true; + } if (Py_TYPE(op)->tp_traverse(op, referrersvisit, arg->objs)) { - op->ob_tid = 0; // we will restore the refcount later - worklist_push(&arg->results, op); + if (_PyObjectStack_Push(&arg->results, Py_NewRef(op)) < 0) { + return false; + } } return true; @@ -1240,48 +1811,25 @@ visit_get_referrers(const mi_heap_t *heap, const mi_heap_area_t *area, PyObject * _PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs) { - PyObject *result = PyList_New(0); - if (!result) { - return NULL; - } - - _PyEval_StopTheWorld(interp); - - // Append all objects to a worklist. This abuses ob_tid. We will restore - // it later. NOTE: We can't append to the PyListObject during - // gc_visit_heaps() because PyList_Append() may reclaim an abandoned - // mimalloc segments while we are traversing them. + // NOTE: We can't append to the PyListObject during gc_visit_heaps() + // because PyList_Append() may reclaim an abandoned mimalloc segments + // while we are traversing them. struct get_referrers_args args = { .objs = objs }; - gc_visit_heaps(interp, &visit_get_referrers, &args.base); - - bool error = false; - PyObject *op; - while ((op = worklist_pop(&args.results)) != NULL) { - gc_restore_tid(op); - if (op != objs && PyList_Append(result, op) < 0) { - error = true; - break; - } - } - - // In case of error, clear the remaining worklist - while ((op = worklist_pop(&args.results)) != NULL) { - gc_restore_tid(op); - } - + _PyEval_StopTheWorld(interp); + int err = gc_visit_heaps(interp, &visit_get_referrers, &args.base); _PyEval_StartTheWorld(interp); - if (error) { - Py_DECREF(result); - return NULL; + PyObject *list = list_from_object_stack(&args.results); + if (err < 0) { + PyErr_NoMemory(); + Py_CLEAR(list); } - - return result; + return list; } struct get_objects_args { struct visitor_args base; - struct worklist objects; + _PyObjectStack objects; }; static bool @@ -1292,54 +1840,36 @@ visit_get_objects(const mi_heap_t *heap, const mi_heap_area_t *area, if (op == NULL) { return true; } + if (op->ob_gc_bits & (_PyGC_BITS_UNREACHABLE | _PyGC_BITS_FROZEN)) { + // Exclude unreachable objects (in-progress GC) and frozen + // objects from gc.get_objects() to match the default build. + return true; + } struct get_objects_args *arg = (struct get_objects_args *)args; - op->ob_tid = 0; // we will restore the refcount later - worklist_push(&arg->objects, op); - + if (_PyObjectStack_Push(&arg->objects, Py_NewRef(op)) < 0) { + return false; + } return true; } PyObject * -_PyGC_GetObjects(PyInterpreterState *interp, Py_ssize_t generation) +_PyGC_GetObjects(PyInterpreterState *interp, int generation) { - PyObject *result = PyList_New(0); - if (!result) { - return NULL; - } - - _PyEval_StopTheWorld(interp); - - // Append all objects to a worklist. This abuses ob_tid. We will restore - // it later. NOTE: We can't append to the list during gc_visit_heaps() - // because PyList_Append() may reclaim an abandoned mimalloc segment - // while we are traversing it. + // NOTE: We can't append to the PyListObject during gc_visit_heaps() + // because PyList_Append() may reclaim an abandoned mimalloc segments + // while we are traversing them. struct get_objects_args args = { 0 }; - gc_visit_heaps(interp, &visit_get_objects, &args.base); - - bool error = false; - PyObject *op; - while ((op = worklist_pop(&args.objects)) != NULL) { - gc_restore_tid(op); - if (op != result && PyList_Append(result, op) < 0) { - error = true; - break; - } - } - - // In case of error, clear the remaining worklist - while ((op = worklist_pop(&args.objects)) != NULL) { - gc_restore_tid(op); - } - + _PyEval_StopTheWorld(interp); + int err = gc_visit_heaps(interp, &visit_get_objects, &args.base); _PyEval_StartTheWorld(interp); - if (error) { - Py_DECREF(result); - return NULL; + PyObject *list = list_from_object_stack(&args.objects); + if (err < 0) { + PyErr_NoMemory(); + Py_CLEAR(list); } - - return result; + return list; } static bool @@ -1347,7 +1877,7 @@ visit_freeze(const mi_heap_t *heap, const mi_heap_area_t *area, void *block, size_t block_size, void *args) { PyObject *op = op_from_block(block, args, true); - if (op != NULL) { + if (op != NULL && !gc_is_unreachable(op)) { op->ob_gc_bits |= _PyGC_BITS_FROZEN; } return true; @@ -1358,6 +1888,8 @@ _PyGC_Freeze(PyInterpreterState *interp) { struct visitor_args args; _PyEval_StopTheWorld(interp); + GCState *gcstate = get_gc_state(); + gcstate->freeze_active = true; gc_visit_heaps(interp, &visit_freeze, &args); _PyEval_StartTheWorld(interp); } @@ -1368,7 +1900,7 @@ visit_unfreeze(const mi_heap_t *heap, const mi_heap_area_t *area, { PyObject *op = op_from_block(block, args, true); if (op != NULL) { - op->ob_gc_bits &= ~_PyGC_BITS_FROZEN; + gc_clear_bit(op, _PyGC_BITS_FROZEN); } return true; } @@ -1378,6 +1910,8 @@ _PyGC_Unfreeze(PyInterpreterState *interp) { struct visitor_args args; _PyEval_StopTheWorld(interp); + GCState *gcstate = get_gc_state(); + gcstate->freeze_active = false; gc_visit_heaps(interp, &visit_unfreeze, &args); _PyEval_StartTheWorld(interp); } @@ -1392,7 +1926,7 @@ visit_count_frozen(const mi_heap_t *heap, const mi_heap_area_t *area, void *block, size_t block_size, void *args) { PyObject *op = op_from_block(block, args, true); - if (op != NULL && (op->ob_gc_bits & _PyGC_BITS_FROZEN) != 0) { + if (op != NULL && gc_is_frozen(op)) { struct count_frozen_args *arg = (struct count_frozen_args *)args; arg->count++; } @@ -1460,7 +1994,7 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) return gc_collect_main(tstate, generation, reason); } -Py_ssize_t +void _PyGC_CollectNoFail(PyThreadState *tstate) { /* Ideally, this function is only called on interpreter shutdown, @@ -1469,7 +2003,7 @@ _PyGC_CollectNoFail(PyThreadState *tstate) during interpreter shutdown (and then never finish it). See http://bugs.python.org/issue8713#msg195178 for an example. */ - return gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_SHUTDOWN); + gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_SHUTDOWN); } void @@ -1600,6 +2134,10 @@ _PyObject_GC_Link(PyObject *op) void _Py_RunGC(PyThreadState *tstate) { + GCState *gcstate = get_gc_state(); + if (!gcstate->enabled) { + return; + } gc_collect_main(tstate, 0, _Py_GC_REASON_HEAP); } @@ -1628,11 +2166,18 @@ PyObject * _PyObject_GC_New(PyTypeObject *tp) { size_t presize = _PyType_PreHeaderSize(tp); - PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp), presize); + size_t size = _PyObject_SIZE(tp); + if (_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)) { + size += _PyInlineValuesSize(tp); + } + PyObject *op = gc_alloc(tp, size, presize); if (op == NULL) { return NULL; } _PyObject_Init(op, tp); + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + _PyObject_InitInlineValues(op, tp); + } return op; } @@ -1692,6 +2237,7 @@ PyObject_GC_Del(void *op) { size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type); if (_PyObject_GC_IS_TRACKED(op)) { + _PyObject_GC_UNTRACK(op); #ifdef Py_DEBUG PyObject *exc = PyErr_GetRaisedException(); if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, @@ -1704,7 +2250,6 @@ PyObject_GC_Del(void *op) } record_deallocation(_PyThreadState_GET()); - PyObject_Free(((char *)op)-presize); } @@ -1744,16 +2289,22 @@ custom_visitor_wrapper(const mi_heap_t *heap, const mi_heap_area_t *area, } void -PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) +_PyGC_VisitObjectsWorldStopped(PyInterpreterState *interp, + gcvisitobjects_t callback, void *arg) { - PyInterpreterState *interp = _PyInterpreterState_GET(); struct custom_visitor_args wrapper = { .callback = callback, .arg = arg, }; + gc_visit_heaps(interp, &custom_visitor_wrapper, &wrapper.base); +} +void +PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); _PyEval_StopTheWorld(interp); - gc_visit_heaps(interp, &custom_visitor_wrapper, &wrapper.base); + _PyGC_VisitObjectsWorldStopped(interp, callback, arg); _PyEval_StartTheWorld(interp); } @@ -1767,13 +2318,11 @@ PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) void _PyGC_ClearAllFreeLists(PyInterpreterState *interp) { - HEAD_LOCK(&_PyRuntime); - _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)interp->threads.head; - while (tstate != NULL) { + _Py_FOR_EACH_TSTATE_BEGIN(interp, p) { + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)p; _PyObject_ClearFreeLists(&tstate->freelists, 0); - tstate = (_PyThreadStateImpl *)tstate->base.next; } - HEAD_UNLOCK(&_PyRuntime); + _Py_FOR_EACH_TSTATE_END(interp); } #endif // Py_GIL_DISABLED diff --git a/Python/gc_gil.c b/Python/gc_gil.c index 48646c7af86b7f..7d62a9ed0b1ca8 100644 --- a/Python/gc_gil.c +++ b/Python/gc_gil.c @@ -1,5 +1,5 @@ #include "Python.h" -#include "pycore_pystate.h" // _Py_ClearFreeLists() +#include "pycore_freelist.h" // _PyObject_ClearFreeLists() #ifndef Py_GIL_DISABLED diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e612c9e4c37632..ffdad70815caef 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,244 +8,255 @@ #endif #define TIER_ONE 1 +#if !USE_COMPUTED_GOTOS + dispatch_opcode: + switch (opcode) +#endif + { + /* BEGIN INSTRUCTIONS */ - TARGET(BEFORE_ASYNC_WITH) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BEFORE_ASYNC_WITH); - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol", - Py_TYPE(mgr)->tp_name); - } - GOTO_ERROR(error); - } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol " - "(missed __aexit__ method)", - Py_TYPE(mgr)->tp_name); - } - Py_DECREF(enter); - GOTO_ERROR(error); - } - Py_DECREF(mgr); - res = PyObject_CallNoArgs(enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error; - } - stack_pointer[-1] = exit; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(BEFORE_WITH) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BEFORE_WITH); - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - /* pop the context manager, push its __exit__ and the - * value returned from calling its __enter__ - */ - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol", - Py_TYPE(mgr)->tp_name); - } - GOTO_ERROR(error); - } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol " - "(missed __exit__ method)", - Py_TYPE(mgr)->tp_name); - } - Py_DECREF(enter); - GOTO_ERROR(error); - } - Py_DECREF(mgr); - res = PyObject_CallNoArgs(enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error; - } - stack_pointer[-1] = exit; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } TARGET(BINARY_OP) { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP); - PREDICTED(BINARY_OP); - _Py_CODEUNIT *this_instr = next_instr - 2; + PREDICTED_BINARY_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 6; (void)this_instr; - PyObject *rhs; - PyObject *lhs; - PyObject *res; + _PyStackRef lhs; + _PyStackRef rhs; + _PyStackRef res; // _SPECIALIZE_BINARY_OP - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; { + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } - STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ + OPCODE_DEFERRED_INC(BINARY_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ assert(NB_ADD <= oparg); assert(oparg <= NB_INPLACE_XOR); } + /* Skip 4 cache entries */ // _BINARY_OP { + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); assert(_PyEval_BinaryOps[oparg]); - res = _PyEval_BinaryOps[oparg](lhs, rhs); - Py_DECREF(lhs); - Py_DECREF(rhs); - if (res == NULL) goto pop_2_error; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(lhs); + PyStackRef_CLOSE(rhs); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(BINARY_OP_ADD_FLOAT) { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_ADD_FLOAT { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(BINARY_OP_ADD_INT) { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_ADD_INT { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); - res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; + PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(BINARY_OP_ADD_UNICODE) { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; // _GUARD_BOTH_UNICODE - right = stack_pointer[-1]; - left = stack_pointer[-2]; { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_ADD_UNICODE { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(BINARY_OP_EXTEND) { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_EXTEND); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_BINARY_OP_EXTEND + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *descr = read_obj(&this_instr[2].cache); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + assert(d && d->guard); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = d->guard(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + DEOPT_IF(!res, BINARY_OP); + } + /* Skip -4 cache entry */ + // _BINARY_OP_EXTEND + { + PyObject *descr = read_obj(&this_instr[2].cache); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; STAT_INC(BINARY_OP, hit); - res = PyUnicode_Concat(left, right); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (res == NULL) goto pop_2_error; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = d->action(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; // _GUARD_BOTH_UNICODE - right = stack_pointer[-1]; - left = stack_pointer[-2]; { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_INPLACE_ADD_UNICODE { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + int next_oparg; + #if TIER_ONE assert(next_instr->op.code == STORE_FAST); - PyObject **target_local = &GETLOCAL(next_instr->op.arg); - DEOPT_IF(*target_local != left, BINARY_OP); + next_oparg = next_instr->op.arg; + #else + next_oparg = CURRENT_OPERAND0(); + #endif + _PyStackRef *target_local = &GETLOCAL(next_oparg); + DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -258,132 +269,172 @@ * only the locals reference, so PyUnicode_Append knows * that the string is safe to mutate. */ - assert(Py_REFCNT(left) >= 2); - _Py_DECREF_NO_DEALLOC(left); - PyUnicode_Append(target_local, right); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (*target_local == NULL) goto pop_2_error; - // The STORE_FAST is already done. + assert(Py_REFCNT(left_o) >= 2); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); + PyUnicode_Append(&temp, right_o); + *target_local = PyStackRef_FromPyObjectSteal(temp); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (PyStackRef_IsNull(*target_local)) goto pop_2_error; + #if TIER_ONE + // The STORE_FAST is already done. This is done here in tier one, + // and during trace projection in tier two: assert(next_instr->op.code == STORE_FAST); SKIP_OVER(1); + #endif } stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(BINARY_OP_MULTIPLY_FLOAT) { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_FLOAT { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(BINARY_OP_MULTIPLY_INT) { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_INT { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); - res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; + PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(BINARY_OP_SUBTRACT_FLOAT) { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_FLOAT { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval - - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(BINARY_OP_SUBTRACT_INT) { frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } - /* Skip 1 cache entry */ + /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_INT { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); - res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; + PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -391,27 +442,49 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BINARY_SLICE); - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *res; - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); - // Can't use ERROR_IF() here, because we haven't - // DECREF'ed container yet, and we still own slice. - if (slice == NULL) { - res = NULL; + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + _PyStackRef res; + // _SPECIALIZE_BINARY_SLICE + { + // Placeholder until we implement BINARY_SLICE specialization + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(BINARY_SLICE); + #endif /* ENABLE_SPECIALIZATION */ } - else { - res = PyObject_GetItem(container, slice); - Py_DECREF(slice); + // _BINARY_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyObject *res_o; + // Can't use ERROR_IF() here, because we haven't + // DECREF'ed container yet, and we still own slice. + if (slice == NULL) { + res_o = NULL; + } + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); + Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } + PyStackRef_CLOSE(container); + if (res_o == NULL) goto pop_3_error; + res = PyStackRef_FromPyObjectSteal(res_o); } - Py_DECREF(container); - if (res == NULL) goto pop_3_error; stack_pointer[-3] = res; stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -419,37 +492,46 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR); - PREDICTED(BINARY_SUBSCR); - _Py_CODEUNIT *this_instr = next_instr - 2; + PREDICTED_BINARY_SUBSCR:; + _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyObject *sub; - PyObject *container; - PyObject *res; + _PyStackRef container; + _PyStackRef sub; + _PyStackRef res; // _SPECIALIZE_BINARY_SUBSCR - sub = stack_pointer[-1]; - container = stack_pointer[-2]; { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + #if ENABLE_SPECIALIZATION_FT + assert(frame->stackpointer == NULL); + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_BinarySubscr(container, sub, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } - STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ + OPCODE_DEFERRED_INC(BINARY_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } // _BINARY_SUBSCR { - res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (res == NULL) goto pop_2_error; + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_GetItem(container_o, sub_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -458,58 +540,93 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *dict; - PyObject *res; + _PyStackRef dict_st; + _PyStackRef sub_st; + _PyStackRef res; /* Skip 1 cache entry */ - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; + sub_st = stack_pointer[-1]; + dict_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); - int rc = PyDict_GetItemRef(dict, sub, &res); + PyObject *res_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int rc = PyDict_GetItemRef(dict, sub, &res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (rc == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetKeyError(sub); + stack_pointer = _PyFrame_GetStackPointer(frame); } - Py_DECREF(dict); - Py_DECREF(sub); + PyStackRef_CLOSE(dict_st); + PyStackRef_CLOSE(sub_st); if (rc <= 0) goto pop_2_error; // not found or error + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(BINARY_SUBSCR_GETITEM) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *container; + _PyStackRef container; + _PyStackRef getitem; + _PyStackRef sub; + _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); - PyTypeObject *tp = Py_TYPE(container); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *cached = ht->_spec_cache.getitem; - DEOPT_IF(cached == NULL, BINARY_SUBSCR); - assert(PyFunction_Check(cached)); - PyFunctionObject *getitem = (PyFunctionObject *)cached; - uint32_t cached_version = ht->_spec_cache.getitem_version; - DEOPT_IF(getitem->func_version != cached_version, BINARY_SUBSCR); - PyCodeObject *code = (PyCodeObject *)getitem->func_code; - assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - Py_INCREF(getitem); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2); - STACK_SHRINK(2); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + } + // _BINARY_SUBSCR_CHECK_FUNC + { + container = stack_pointer[-2]; + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); + DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR); + assert(PyFunction_Check(getitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); + DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR); + PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); + assert(code->co_argcount == 2); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + getitem = PyStackRef_FromPyObjectNew(getitem_o); + STAT_INC(BINARY_SUBSCR, hit); + } + // _BINARY_SUBSCR_INIT_CALL + { + sub = stack_pointer[-1]; + new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + new_frame->localsplus[0] = container; + new_frame->localsplus[1] = sub; + frame->return_offset = 2 ; + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); } TARGET(BINARY_SUBSCR_LIST_INT) { @@ -517,26 +634,38 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *list; - PyObject *res; + _PyStackRef list_st; + _PyStackRef sub_st; + _PyStackRef res; /* Skip 1 cache entry */ - sub = stack_pointer[-1]; - list = stack_pointer[-2]; + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); // Deopt unless 0 <= sub < PyList_Size(list) DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + DEOPT_IF(res_o == NULL, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + #else DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); - res = PyList_GET_ITEM(list, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + Py_INCREF(res_o); + #endif + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(list_st); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -545,12 +674,14 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *str; - PyObject *res; + _PyStackRef str_st; + _PyStackRef sub_st; + _PyStackRef res; /* Skip 1 cache entry */ - sub = stack_pointer[-1]; - str = stack_pointer[-2]; + sub_st = stack_pointer[-1]; + str_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); @@ -560,11 +691,13 @@ Py_UCS4 c = PyUnicode_READ_CHAR(str, index); DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); - res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(str); + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(str_st); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -573,12 +706,14 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *tuple; - PyObject *res; + _PyStackRef tuple_st; + _PyStackRef sub_st; + _PyStackRef res; /* Skip 1 cache entry */ - sub = stack_pointer[-1]; - tuple = stack_pointer[-2]; + sub_st = stack_pointer[-1]; + tuple_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); // Deopt unless 0 <= sub < PyTuple_Size(list) @@ -586,41 +721,15 @@ Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); - res = PyTuple_GET_ITEM(tuple, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(tuple); + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + Py_INCREF(res_o); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(tuple_st); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; - DISPATCH(); - } - - TARGET(BUILD_CONST_KEY_MAP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_CONST_KEY_MAP); - PyObject *keys; - PyObject **values; - PyObject *map; - keys = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - if (!PyTuple_CheckExact(keys) || - PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { - _PyErr_SetString(tstate, PyExc_SystemError, - "bad BUILD_CONST_KEY_MAP keys argument"); - GOTO_ERROR(error); // Pop the keys and values. - } - map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); - for (int _i = oparg; --_i >= 0;) { - Py_DECREF(values[_i]); - } - Py_DECREF(keys); - if (map == NULL) { stack_pointer += -1 - oparg; goto error; } - stack_pointer[-1 - oparg] = map; - stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -628,13 +737,17 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_LIST); - PyObject **values; - PyObject *list; + _PyStackRef *values; + _PyStackRef list; values = &stack_pointer[-oparg]; - list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) { stack_pointer += -oparg; goto error; } + PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); + if (list_o == NULL) { + goto error; + } + list = PyStackRef_FromPyObjectSteal(list_o); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -642,19 +755,39 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_MAP); - PyObject **values; - PyObject *map; + _PyStackRef *values; + _PyStackRef map; values = &stack_pointer[-oparg*2]; - map = _PyDict_FromItems( - values, 2, - values+1, 2, - oparg); + STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + if (CONVERSION_FAILED(values_o)) { + for (int _i = oparg*2; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + { + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *map_o = _PyDict_FromItems( + values_o, 2, + values_o+1, 2, + oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); for (int _i = oparg*2; --_i >= 0;) { - Py_DECREF(values[_i]); + PyStackRef_CLOSE(values[_i]); + } + if (map_o == NULL) { + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + goto error; } - if (map == NULL) { stack_pointer += -oparg*2; goto error; } + map = PyStackRef_FromPyObjectSteal(map_o); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -662,25 +795,45 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SET); - PyObject **values; - PyObject *set; + _PyStackRef *values; + _PyStackRef set; values = &stack_pointer[-oparg]; - set = PySet_New(NULL); - if (set == NULL) - GOTO_ERROR(error); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *set_o = PySet_New(NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (set_o == NULL) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } int err = 0; for (int i = 0; i < oparg; i++) { - PyObject *item = values[i]; - if (err == 0) - err = PySet_Add(set, item); - Py_DECREF(item); + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PySet_Add(set_o, PyStackRef_AsPyObjectBorrow(values[i])); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); } if (err != 0) { - Py_DECREF(set); - if (true) { stack_pointer += -oparg; goto error; } + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(set_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; } + set = PyStackRef_FromPyObjectSteal(set_o); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -688,20 +841,25 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SLICE); - PyObject *step = NULL; - PyObject *stop; - PyObject *start; - PyObject *slice; - if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } - stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; - start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; - slice = PySlice_New(start, stop, step); - Py_DECREF(start); - Py_DECREF(stop); - Py_XDECREF(step); - if (slice == NULL) { stack_pointer += -2 - ((oparg == 3) ? 1 : 0); goto error; } - stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; - stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + _PyStackRef *args; + _PyStackRef slice; + args = &stack_pointer[-oparg]; + PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); + PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (slice_o == NULL) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + slice = PyStackRef_FromPyObjectSteal(slice_o); + stack_pointer[-oparg] = slice; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -709,16 +867,34 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_STRING); - PyObject **pieces; - PyObject *str; + _PyStackRef *pieces; + _PyStackRef str; pieces = &stack_pointer[-oparg]; - str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); + STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + if (CONVERSION_FAILED(pieces_o)) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(pieces[_i]); + } + { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); for (int _i = oparg; --_i >= 0;) { - Py_DECREF(pieces[_i]); + PyStackRef_CLOSE(pieces[_i]); + } + if (str_o == NULL) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } - if (str == NULL) { stack_pointer += -oparg; goto error; } + str = PyStackRef_FromPyObjectSteal(str_o); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -726,13 +902,17 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_TUPLE); - PyObject **values; - PyObject *tup; + _PyStackRef *values; + _PyStackRef tup; values = &stack_pointer[-oparg]; - tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) { stack_pointer += -oparg; goto error; } + PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); + if (tup_o == NULL) { + goto error; + } + tup = PyStackRef_FromPyObjectSteal(tup_o); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -749,178 +929,274 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL); - PREDICTED(CALL); - _Py_CODEUNIT *this_instr = next_instr - 4; + PREDICTED_CALL:; + _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef res; // _SPECIALIZE_CALL - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } - STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ + OPCODE_DEFERRED_INC(CALL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } /* Skip 2 cache entries */ - // _CALL + // _MAYBE_EXPAND_METHOD { + args = &stack_pointer[-oparg]; + func = &stack_pointer[-2 - oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); + } + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null != NULL) { - args--; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; total_args++; } - else if (Py_TYPE(callable) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; - } // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && + if (Py_TYPE(callable_o) == &PyFunction_Type && tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, - args, total_args, NULL + tstate, callable[0], locals, + arguments, total_args, NULL, frame ); + stack_pointer = _PyFrame_GetStackPointer(frame); // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - GOTO_ERROR(error); + goto error; } - frame->return_offset = (uint16_t)(next_instr - this_instr); + frame->return_offset = 4 ; DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable, arg); + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); } else { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable, arg); + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - Py_CLEAR(res); + Py_CLEAR(res_o); } } } - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(CALL_ALLOC_AND_ENTER_INIT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *null; - PyObject *callable; + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef *args; + _PyStackRef *init; + _PyStackRef *self; + _PyInterpreterFrame *init_frame; + _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* This instruction does the following: - * 1. Creates the object (by calling ``object.__new__``) - * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) - * 3. Pushes the frame for ``__init__`` to the frame stack - * */ - _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; - PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; - PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(code->co_argcount != oparg+1, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); - STAT_INC(CALL, hit); - PyObject *self = _PyType_NewManagedObject(tp); - if (self == NULL) { - GOTO_ERROR(error); + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); } - Py_DECREF(tp); - _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1); - assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); - /* Push self onto stack of shim */ - Py_INCREF(self); - shim->localsplus[0] = self; - Py_INCREF(init); - _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); - /* Copy self followed by args to __init__ frame */ - init_frame->localsplus[0] = self; - for (int i = 0; i < oparg; i++) { - init_frame->localsplus[i+1] = args[i]; + // _CHECK_AND_ALLOCATE_OBJECT + { + args = &stack_pointer[-oparg]; + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + init = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL); + PyTypeObject *tp = (PyTypeObject *)callable_o; + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL); + assert(tp->tp_new == PyBaseObject_Type.tp_new); + assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_alloc == PyType_GenericAlloc); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; + PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); + PyCodeObject *code = (PyCodeObject *)init_func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *self_o = PyType_GenericAlloc(tp, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (self_o == NULL) { + goto error; + } + self[0] = PyStackRef_FromPyObjectSteal(self_o); + _PyStackRef temp = callable[0]; + init[0] = PyStackRef_FromPyObjectNew(init_func); + PyStackRef_CLOSE(temp); } - frame->return_offset = (uint16_t)(next_instr - this_instr); - STACK_SHRINK(oparg+2); - _PyFrame_SetStackPointer(frame, stack_pointer); - /* Link frames */ - init_frame->previous = shim; - shim->previous = frame; - frame = tstate->current_frame = init_frame; - CALL_STAT_INC(inlined_py_calls); - /* Account for pushing the extra frame. - * We don't check recursion depth here, - * as it will be checked after start_frame */ - tstate->py_recursion_remaining--; - goto start_frame; + // _CREATE_INIT_FRAME + { + args = &stack_pointer[-oparg]; + self = &stack_pointer[-1 - oparg]; + init = &stack_pointer[-2 - oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); + stack_pointer = _PyFrame_GetStackPointer(frame); + /* Push self onto stack of shim */ + shim->localsplus[0] = PyStackRef_DUP(self[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, init[0], NULL, args-1, oparg+1, NULL, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + _PyEval_FrameClearAndPop(tstate, shim); + goto error; + } + init_frame = temp; + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + /* Account for pushing the extra frame. + * We don't check recursion depth here, + * as it will be checked after start_frame */ + tstate->py_recursion_remaining--; + } + // _PUSH_FRAME + { + new_frame = init_frame; + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject *null; - PyObject *callable; - PyObject *func; - PyObject *self; - PyObject *self_or_null; - PyObject **args; + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef *func; + _PyStackRef *self; + _PyStackRef *self_or_null; + _PyStackRef *args; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 @@ -928,53 +1204,60 @@ DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; { - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL); } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { + func = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); STAT_INC(CALL, hit); - self = Py_NewRef(((PyMethodObject *)callable)->im_self); - stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS - func = Py_NewRef(((PyMethodObject *)callable)->im_func); - stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + PyStackRef_CLOSE(temp); } - // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = self; - callable = func; + // flush + // _CHECK_FUNCTION_VERSION { + callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable_o; DEOPT_IF(func->func_version != func_version, CALL); + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); } // _CHECK_STACK_SPACE { - PyFunctionObject *func = (PyFunctionObject *)callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; { - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null[0]; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } } // _SAVE_RETURN_OFFSET @@ -991,21 +1274,118 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_BOUND_METHOD_GENERAL) { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef *method; + _PyStackRef *self; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_METHOD_VERSION + { + null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + DEOPT_IF(!PyFunction_Check(func), CALL); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + } + // _EXPAND_METHOD + { + method = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + assert(PyStackRef_IsNull(null[0])); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable[0]; + method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method[0])); + PyStackRef_CLOSE(temp); + } + // flush + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + // The frame has stolen all the arguments from the stack. + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + goto error; + } + new_frame = temp; + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; #endif } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } DISPATCH(); } @@ -1014,34 +1394,76 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + // _CALL_BUILTIN_CLASS + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyType_Check(callable_o), CALL); + PyTypeObject *tp = (PyTypeObject *)callable_o; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + STAT_INC(CALL, hit); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); } - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); - STAT_INC(CALL, hit); - res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } - Py_DECREF(tp); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1050,45 +1472,82 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL functions, without keywords */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + // _CALL_BUILTIN_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL functions, without keywords */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + /* res = func(self, args, nargs) */ + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable_o), + args_o, + total_args); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - /* res = func(self, args, nargs) */ - res = ((PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable), - args, - total_args); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1097,39 +1556,83 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + // _CALL_BUILTIN_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); + STAT_INC(CALL, hit); + /* res = func(self, arguments, nargs, kwnames) */ + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); - STAT_INC(CALL, hit); - /* res = func(self, args, nargs, kwnames) */ - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); - res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1138,41 +1641,66 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_O functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + // _CALL_BUILTIN_O + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + /* Builtin METH_O functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + _PyStackRef arg = args[0]; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(arg); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } - PyObject *arg = args[0]; - res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1180,85 +1708,162 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_FUNCTION_EX); - PREDICTED(CALL_FUNCTION_EX); - _Py_CODEUNIT *this_instr = next_instr - 1; + PREDICTED_CALL_FUNCTION_EX:; + _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; - PyObject *kwargs = NULL; - PyObject *callargs; - PyObject *func; - PyObject *result; - if (oparg & 1) { kwargs = stack_pointer[-(oparg & 1)]; } - callargs = stack_pointer[-1 - (oparg & 1)]; - func = stack_pointer[-3 - (oparg & 1)]; - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - if (!PyTuple_CheckExact(callargs)) { - if (check_args_iterable(tstate, func, callargs) < 0) { - GOTO_ERROR(error); - } - PyObject *tuple = PySequence_Tuple(callargs); - if (tuple == NULL) { - GOTO_ERROR(error); - } - Py_SETREF(callargs, tuple); - } - assert(PyTuple_CheckExact(callargs)); - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && - !PyFunction_Check(func) && !PyMethod_Check(func) - ) { - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : Py_None; - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - if (err) GOTO_ERROR(error); - result = PyObject_Call(func, callargs, kwargs); - if (result == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef kwargs_in; + _PyStackRef tuple; + _PyStackRef kwargs_out; + _PyStackRef func_st; + _PyStackRef callargs_st; + _PyStackRef kwargs_st; + _PyStackRef result; + // _MAKE_CALLARGS_A_TUPLE + { + kwargs_in = stack_pointer[-1]; + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (PyTuple_CheckExact(callargs_o)) { + tuple = callargs; + kwargs_out = kwargs_in; } else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - Py_CLEAR(result); + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + goto error; } + kwargs_out = kwargs_in; + PyStackRef_CLOSE(callargs); + tuple = PyStackRef_FromPyObjectSteal(tuple_o); } } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + // _DO_CALL_FUNCTION_EX + { + kwargs_st = kwargs_out; + callargs_st = tuple; + func_st = func; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); assert(PyTuple_CheckExact(callargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)func, locals, - nargs, callargs, kwargs); - // Need to manually shrink the stack since we exit with DISPATCH_INLINED. - STACK_SHRINK(oparg + 3); - if (new_frame == NULL) { - GOTO_ERROR(error); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-2] = callargs_st; + stack_pointer[-1] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(result_o); + } + } } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); } - result = PyObject_Call(func, callargs, kwargs); + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Need to sync the stack since we exit with DISPATCH_INLINED. + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + goto error; + } + assert( 1 == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-2] = callargs_st; + stack_pointer[-1] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(kwargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(callargs_st); + PyStackRef_CLOSE(func_st); + if (result_o == NULL) goto pop_4_error; + result = PyStackRef_FromPyObjectSteal(result_o); } - Py_DECREF(func); - Py_DECREF(callargs); - Py_XDECREF(kwargs); - assert(PEEK(2 + (oparg & 1)) == NULL); - if (result == NULL) { stack_pointer += -3 - (oparg & 1); goto error; } - stack_pointer[-3 - (oparg & 1)] = result; - stack_pointer += -2 - (oparg & 1); - CHECK_EVAL_BREAKER(); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-4] = result; + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-4] = result; + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1266,13 +1871,16 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_1); - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); - res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) goto pop_1_error; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; DISPATCH(); } @@ -1281,18 +1889,24 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_2); - PyObject *value1; - PyObject *value2; - PyObject *res; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; + _PyStackRef value2_st; + _PyStackRef value1_st; + _PyStackRef res; + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; assert(oparg <= MAX_INTRINSIC_2); - res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - Py_DECREF(value2); - Py_DECREF(value1); - if (res == NULL) goto pop_2_error; + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value2_st); + PyStackRef_CLOSE(value1_st); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1301,130 +1915,490 @@ next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* isinstance(o, o2) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; - if (self_or_null != NULL) { - args--; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; total_args++; } DEOPT_IF(total_args != 2, CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); + DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); STAT_INC(CALL, hit); - PyObject *cls = args[1]; - PyObject *inst = args[0]; - int retval = PyObject_IsInstance(inst, cls); + _PyStackRef cls_stackref = arguments[1]; + _PyStackRef inst_stackref = arguments[0]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (retval < 0) { - GOTO_ERROR(error); + goto error; + } + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); } - res = PyBool_FromLong(retval); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(inst); - Py_DECREF(cls); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(CALL_KW) { frame->instr_ptr = next_instr; - next_instr += 1; + next_instr += 4; INSTRUCTION_STATS(CALL_KW); - PREDICTED(CALL_KW); - _Py_CODEUNIT *this_instr = next_instr - 1; + PREDICTED_CALL_KW:; + _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; - PyObject *kwnames; - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef kwnames_in; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef kwnames_out; + _PyStackRef res; + // _SPECIALIZE_CALL_KW + { + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CALL_KW); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } - if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); - // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, - args, positional_args, kwnames + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + kwnames_in = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + func = &stack_pointer[-3 - oparg]; + maybe_self = &stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); + } + kwnames_out = kwnames_in; + } + // _DO_CALL_KW + { + kwnames = kwnames_out; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(kwnames); + // Sync stack explicitly since we leave using DISPATCH_INLINED(). + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; + } + assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(res_o); + } + } + } + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + if (res_o == NULL) { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(CALL_KW_BOUND_METHOD) { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *null; + _PyStackRef kwnames; + _PyStackRef *method; + _PyStackRef *self; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + } + // _CHECK_METHOD_VERSION_KW + { + null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + DEOPT_IF(!PyFunction_Check(func), CALL_KW); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); + DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW); + } + // _EXPAND_METHOD_KW + { + method = &stack_pointer[-3 - oparg]; + self = &stack_pointer[-2 - oparg]; + _PyStackRef callable_s = callable[0]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable_s); + assert(PyStackRef_IsNull(null[0])); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + method[0] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(method[0])); + PyStackRef_CLOSE(callable_s); + } + // flush + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + arguments, positional_args, kwnames_o, frame ); - Py_DECREF(kwnames); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 3); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. - if (new_frame == NULL) { - GOTO_ERROR(error); - } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable, arg); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + goto error; } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable, arg); - if (err < 0) { - Py_CLEAR(res); + new_frame = temp; + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_KW_NON_PY) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_NON_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef kwnames; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE_KW + { + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); + } + // _CALL_KW_NON_PY + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; } } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(kwnames); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (res_o == NULL) { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); } - Py_DECREF(kwnames); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 2 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } - if (res == NULL) { stack_pointer += -3 - oparg; goto error; } stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; - CHECK_EVAL_BREAKER(); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(CALL_KW_PY) { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef kwnames; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + } + // _CHECK_FUNCTION_VERSION_KW + { + callable = &stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL_KW); + } + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(kwnames); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + goto error; + } + new_frame = temp; + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } DISPATCH(); } @@ -1433,37 +2407,45 @@ next_instr += 4; INSTRUCTION_STATS(CALL_LEN); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; /* len(o) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null[0])) { args--; total_args++; } DEOPT_IF(total_args != 1, CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.len, CALL); + DEOPT_IF(callable_o != interp->callable_cache.len, CALL); STAT_INC(CALL, hit); - PyObject *arg = args[0]; + _PyStackRef arg_stackref = args[0]; + PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); + _PyFrame_SetStackPointer(frame, stack_pointer); Py_ssize_t len_i = PyObject_Length(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) { + goto error; + } + PyObject *res_o = PyLong_FromSsize_t(len_i); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { GOTO_ERROR(error); } - res = PyLong_FromSsize_t(len_i); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - Py_DECREF(arg); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + PyStackRef_CLOSE(arg_stackref); + PyStackRef_CLOSE(callable[0]); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1472,29 +2454,36 @@ next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self; - PyObject *callable; + _PyStackRef callable; + _PyStackRef self; + _PyStackRef arg; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - self = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + arg = stack_pointer[-1]; + self = stack_pointer[-2]; + callable = stack_pointer[-3]; assert(oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.list_append, CALL); - assert(self != NULL); - DEOPT_IF(!PyList_Check(self), CALL); + DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); + assert(self_o != NULL); + DEOPT_IF(!PyList_Check(self_o), CALL); + DEOPT_IF(!LOCK_OBJECT(self_o), CALL); STAT_INC(CALL, hit); - if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { - goto pop_1_error; // Since arg is DECREF'ed already - } - Py_DECREF(self); - Py_DECREF(callable); - STACK_SHRINK(3); - // Skip POP_TOP + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); + PyStackRef_CLOSE(self); + PyStackRef_CLOSE(callable); + if (err) goto pop_3_error; + #if TIER_ONE + // Skip the following POP_TOP. This is done here in tier one, and + // during trace projection in tier two: assert(next_instr->op.code == POP_TOP); SKIP_OVER(1); + #endif + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1503,42 +2492,84 @@ next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + // _CALL_METHOD_DESCRIPTOR_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + /* Builtin METH_FASTCALL methods, without keywords */ + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFast cfunc = + (PyCFunctionFast)(void(*)(void))meth->ml_meth; + PyObject *res_o = cfunc(self, (args_o + 1), nargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - PyCFunctionFast cfunc = - (PyCFunctionFast)(void(*)(void))meth->ml_meth; - int nargs = total_args - 1; - res = cfunc(self, args + 1, nargs); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1547,42 +2578,84 @@ next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + STAT_INC(CALL, hit); + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); - PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); - STAT_INC(CALL, hit); - int nargs = total_args - 1; - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, args + 1, nargs, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1591,44 +2664,70 @@ next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + // _CALL_METHOD_DESCRIPTOR_NOARGS + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + assert(oparg == 0 || oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); } - DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } - res = _PyCFunction_TrampolineCall(cfunc, self, NULL); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1637,94 +2736,215 @@ next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + // _CALL_METHOD_DESCRIPTOR_O + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_O, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + _PyStackRef arg_stackref = args[1]; + _PyStackRef self_stackref = args[0]; + DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, + PyStackRef_AsPyObjectBorrow(self_stackref), + PyStackRef_AsPyObjectBorrow(arg_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(arg_stackref); + PyStackRef_CLOSE(callable[0]); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); - PyObject *arg = args[1]; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(CALL_NON_PY_GENERAL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_NON_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE + { + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(PyFunction_Check(callable_o), CALL); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); + } + // _CALL_NON_PY_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } - res = _PyCFunction_TrampolineCall(cfunc, self, arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(CALL_PY_EXACT_ARGS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject *self_or_null; - PyObject *callable; - PyObject **args; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { DEOPT_IF(tstate->interp->eval_frame, CALL); } - // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + // _CHECK_FUNCTION_VERSION { + callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable_o; DEOPT_IF(func->func_version != func_version, CALL); + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); } // _CHECK_STACK_SPACE { - PyFunctionObject *func = (PyFunctionObject *)callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; { - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; + new_frame->localsplus[0] = self_or_null[0]; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; } } // _SAVE_RETURN_OFFSET @@ -1741,68 +2961,98 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; + assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; + frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif + LLTRACE_RESUME_FRAME(); } DISPATCH(); } - TARGET(CALL_PY_WITH_DEFAULTS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + TARGET(CALL_PY_GENERAL) { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; - INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS); + INSTRUCTION_STATS(CALL_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - DEOPT_IF(tstate->interp->eval_frame, CALL); - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - assert(func->func_defaults); - assert(PyTuple_CheckExact(func->func_defaults)); - int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); - assert(defcount <= code->co_argcount); - int min_args = code->co_argcount - defcount; - DEOPT_IF(argcount > code->co_argcount, CALL); - DEOPT_IF(argcount < min_args, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - STAT_INC(CALL, hit); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); } - for (int i = argcount; i < code->co_argcount; i++) { - PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); - new_frame->localsplus[i] = Py_NewRef(def); + // _CHECK_FUNCTION_VERSION + { + callable = &stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + DEOPT_IF(func->func_version != func_version, CALL); } - // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null[0])) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + // The frame has stolen all the arguments from the stack. + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + goto error; + } + new_frame = temp; + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); } TARGET(CALL_STR_1) { @@ -1810,27 +3060,49 @@ next_instr += 4; INSTRUCTION_STATS(CALL_STR_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; + _PyStackRef callable; + _PyStackRef null; + _PyStackRef arg; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PyObject_Str(arg); - Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + // _CALL_STR_1 + { + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Str(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(arg); + if (res_o == NULL) goto pop_3_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1839,27 +3111,49 @@ next_instr += 4; INSTRUCTION_STATS(CALL_TUPLE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; + _PyStackRef callable; + _PyStackRef null; + _PyStackRef arg; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PySequence_Tuple(arg); - Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + // _CALL_TUPLE_1 + { + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PySequence_Tuple(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(arg); + if (res_o == NULL) goto pop_3_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1868,25 +3162,26 @@ next_instr += 4; INSTRUCTION_STATS(CALL_TYPE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; + _PyStackRef callable; + _PyStackRef null; + _PyStackRef arg; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + arg = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - PyObject *obj = args[0]; - DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL); STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(obj); - Py_DECREF(&PyType_Type); // I.e., callable - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); + PyStackRef_CLOSE(arg); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1894,29 +3189,44 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EG_MATCH); - PyObject *match_type; - PyObject *exc_value; - PyObject *rest; - PyObject *match; - match_type = stack_pointer[-1]; - exc_value = stack_pointer[-2]; - if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { - Py_DECREF(exc_value); - Py_DECREF(match_type); - if (true) goto pop_2_error; - } - match = NULL; - rest = NULL; - int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); - Py_DECREF(exc_value); - Py_DECREF(match_type); + _PyStackRef exc_value_st; + _PyStackRef match_type_st; + _PyStackRef rest; + _PyStackRef match; + match_type_st = stack_pointer[-1]; + exc_value_st = stack_pointer[-2]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); + goto pop_2_error; + } + PyObject *match_o = NULL; + PyObject *rest_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, + &match_o, &rest_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); if (res < 0) goto pop_2_error; - assert((match == NULL) == (rest == NULL)); - if (match == NULL) goto pop_2_error; - if (!Py_IsNone(match)) { - PyErr_SetHandledException(match); + assert((match_o == NULL) == (rest_o == NULL)); + if (match_o == NULL) goto pop_2_error; + if (!Py_IsNone(match_o)) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_SetHandledException(match_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); } + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); stack_pointer[-2] = rest; stack_pointer[-1] = match; DISPATCH(); @@ -1926,53 +3236,67 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EXC_MATCH); - PyObject *right; - PyObject *left; - PyObject *b; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; right = stack_pointer[-1]; left = stack_pointer[-2]; - assert(PyExceptionInstance_Check(left)); - if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); - if (true) goto pop_1_error; - } - int res = PyErr_GivenExceptionMatches(left, right); - Py_DECREF(right); - b = res ? Py_True : Py_False; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyExceptionInstance_Check(left_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptTypeValid(tstate, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + PyStackRef_CLOSE(right); + goto pop_1_error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyErr_GivenExceptionMatches(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(right); + b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[-1] = b; DISPATCH(); } TARGET(CLEANUP_THROW) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(CLEANUP_THROW); - PyObject *exc_value; - PyObject *last_sent_val; - PyObject *sub_iter; - PyObject *none; - PyObject *value; - exc_value = stack_pointer[-1]; - last_sent_val = stack_pointer[-2]; - sub_iter = stack_pointer[-3]; + _PyStackRef sub_iter_st; + _PyStackRef last_sent_val_st; + _PyStackRef exc_value_st; + _PyStackRef none; + _PyStackRef value; + exc_value_st = stack_pointer[-1]; + last_sent_val_st = stack_pointer[-2]; + sub_iter_st = stack_pointer[-3]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); - if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { - value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); - Py_DECREF(sub_iter); - Py_DECREF(last_sent_val); - Py_DECREF(exc_value); - none = Py_None; + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + none = PyStackRef_None; + value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); + PyStackRef_CLOSE(sub_iter_st); + PyStackRef_CLOSE(last_sent_val_st); + PyStackRef_CLOSE(exc_value_st); } else { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); goto exception_unwind; } stack_pointer[-3] = none; stack_pointer[-2] = value; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1980,44 +3304,60 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); - PREDICTED(COMPARE_OP); - _Py_CODEUNIT *this_instr = next_instr - 2; + PREDICTED_COMPARE_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; // _SPECIALIZE_COMPARE_OP - right = stack_pointer[-1]; - left = stack_pointer[-2]; { + right = stack_pointer[-1]; + left = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_CompareOp(left, right, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } - STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ + OPCODE_DEFERRED_INC(COMPARE_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } // _COMPARE_OP { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert((oparg >> 5) <= Py_GE); - res = PyObject_RichCompare(left, right, oparg >> 5); - Py_DECREF(left); - Py_DECREF(right); - if (res == NULL) goto pop_2_error; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res_o == NULL) goto pop_2_error; if (oparg & 16) { - int res_bool = PyObject_IsTrue(res); - Py_DECREF(res); - if (res_bool < 0) goto pop_2_error; - res = res_bool ? Py_True : Py_False; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_bool < 0) goto error; + res = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2026,31 +3366,36 @@ next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_FLOAT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_FLOAT { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? Py_True : Py_False; + PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2059,35 +3404,40 @@ next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_INT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_INT { - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP); STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && - _PyLong_DigitCount((PyLongObject *)right) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + _PyLong_DigitCount((PyLongObject *)right_o) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - res = (sign_ish & oparg) ? Py_True : Py_False; + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2096,51 +3446,140 @@ next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_STR); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; // _GUARD_BOTH_UNICODE - right = stack_pointer[-1]; - left = stack_pointer[-2]; { - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_STR { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left, right); + int eq = _PyUnicode_Equal(left_o, right_o); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); assert(eq == 0 || eq == 1); assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(CONTAINS_OP) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP); + PREDICTED_CONTAINS_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + // _SPECIALIZE_CONTAINS_OP + { + right = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ContainsOp(right, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CONTAINS_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CONTAINS_OP + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PySequence_Contains(right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(CONTAINS_OP_DICT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_DICT); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + /* Skip 1 cache entry */ + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); + STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyDict_Contains(right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - TARGET(CONTAINS_OP) { + TARGET(CONTAINS_OP_SET) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CONTAINS_OP); - PyObject *right; - PyObject *left; - PyObject *b; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_SET); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + /* Skip 1 cache entry */ right = stack_pointer[-1]; left = stack_pointer[-2]; - int res = PySequence_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); + STAT_INC(CONTAINS_OP, hit); + // Note: both set and frozenset use the same seq_contains method! + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PySet_Contains((PySetObject *)right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); if (res < 0) goto pop_2_error; - b = (res ^ oparg) ? Py_True : Py_False; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2148,15 +3587,18 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CONVERT_VALUE); - PyObject *value; - PyObject *result; + _PyStackRef value; + _PyStackRef result; value = stack_pointer[-1]; conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = _PyEval_ConversionFuncs[oparg]; - result = conv_fn(value); - Py_DECREF(value); - if (result == NULL) goto pop_1_error; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (result_o == NULL) goto pop_1_error; + result = PyStackRef_FromPyObjectSteal(result_o); stack_pointer[-1] = result; DISPATCH(); } @@ -2165,13 +3607,14 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY); - PyObject *bottom; - PyObject *top; + _PyStackRef bottom; + _PyStackRef top; bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); - top = Py_NewRef(bottom); + top = PyStackRef_DUP(bottom); stack_pointer[0] = top; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2181,13 +3624,14 @@ INSTRUCTION_STATS(COPY_FREE_VARS); /* Copy closure variables to free variables */ PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyFunction_Check(frame->f_funcobj)); - PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; assert(oparg == co->co_nfreevars); int offset = co->co_nlocalsplus - oparg; for (int i = 0; i < oparg; ++i) { PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = Py_NewRef(o); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); } DISPATCH(); } @@ -2196,13 +3640,16 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_ATTR); - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_DelAttr(owner, name); - Py_DECREF(owner); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(owner); if (err) goto pop_1_error; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2210,16 +3657,19 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_DEREF); - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); // Can't use ERROR_IF here. // Fortunately we don't need its superpower. + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; } - PyCell_SET(cell, NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(oldobj); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -2227,9 +3677,17 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_FAST); - PyObject *v = GETLOCAL(oparg); - if (v == NULL) goto unbound_local_error; - SETLOCAL(oparg, NULL); + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + SETLOCAL(oparg, PyStackRef_NULL); DISPATCH(); } @@ -2238,15 +3696,19 @@ next_instr += 1; INSTRUCTION_STATS(DELETE_GLOBAL); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err; - err = PyDict_DelItem(GLOBALS(), name); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Pop(GLOBALS(), name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); // Can't use ERROR_IF here. - if (err != 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - GOTO_ERROR(error); + if (err < 0) { + goto error; + } + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; } DISPATCH(); } @@ -2259,17 +3721,23 @@ PyObject *ns = LOCALS(); int err; if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - GOTO_ERROR(error); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; } + _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_DelItem(ns, name); + stack_pointer = _PyFrame_GetStackPointer(frame); // Can't use ERROR_IF here. if (err != 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; } DISPATCH(); } @@ -2278,16 +3746,20 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_SUBSCR); - PyObject *sub; - PyObject *container; + _PyStackRef container; + _PyStackRef sub; sub = stack_pointer[-1]; container = stack_pointer[-2]; /* del container[sub] */ - int err = PyObject_DelItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + PyStackRef_AsPyObjectBorrow(sub)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); if (err) goto pop_2_error; stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2295,19 +3767,28 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_MERGE); - PyObject *update; - PyObject *dict; - PyObject *callable; + _PyStackRef callable; + _PyStackRef dict; + _PyStackRef update; update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; callable = stack_pointer[-5 - (oparg - 1)]; - if (_PyDict_MergeEx(dict, update, 2) < 0) { - _PyEval_FormatKwargsError(tstate, callable, update); - Py_DECREF(update); - if (true) goto pop_1_error; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_MergeEx(dict_o, update_o, 2); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatKwargsError(tstate, callable_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(update); + goto pop_1_error; } - Py_DECREF(update); + PyStackRef_CLOSE(update); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2315,56 +3796,79 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_UPDATE); - PyObject *update; - PyObject *dict; + _PyStackRef dict; + _PyStackRef update; update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; - if (PyDict_Update(dict, update) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Update(dict_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); + Py_TYPE(update_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); } - Py_DECREF(update); - if (true) goto pop_1_error; + PyStackRef_CLOSE(update); + goto pop_1_error; } - Py_DECREF(update); + PyStackRef_CLOSE(update); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(END_ASYNC_FOR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(END_ASYNC_FOR); - PyObject *exc; - PyObject *awaitable; - exc = stack_pointer[-1]; - awaitable = stack_pointer[-2]; + _PyStackRef awaitable_st; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + awaitable_st = stack_pointer[-2]; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); assert(exc && PyExceptionInstance_Check(exc)); - if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - Py_DECREF(awaitable); - Py_DECREF(exc); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + PyStackRef_CLOSE(awaitable_st); + PyStackRef_CLOSE(exc_st); } else { Py_INCREF(exc); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); goto exception_unwind; } stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(END_FOR) { - frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_FOR); - PyObject *value; + _PyStackRef value; value = stack_pointer[-1]; - Py_DECREF(value); + /* Don't update instr_ptr, so that POP_ITER sees + * the FOR_ITER as the previous instruction. + * This has the benign side effect that if value is + * finalized it will see the location as the FOR_ITER's. + */ + PyStackRef_CLOSE(value); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2372,30 +3876,50 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_SEND); - PyObject *value; - PyObject *receiver; + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; value = stack_pointer[-1]; receiver = stack_pointer[-2]; - Py_DECREF(receiver); - stack_pointer[-2] = value; + (void)receiver; + val = value; + PyStackRef_CLOSE(receiver); + stack_pointer[-2] = val; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(ENTER_EXECUTOR) { - frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); - CHECK_EVAL_BREAKER(); + #ifdef _Py_TIER2 PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); assert(executor->vm_data.code == code); assert(executor->vm_data.valid); assert(tstate->previous_executor == NULL); + /* If the eval breaker is set then stay in tier 1. + * This avoids any potentially infinite loops + * involving _RESUME_CHECK */ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + opcode = executor->vm_data.opcode; + oparg = (oparg & ~255) | executor->vm_data.oparg; + next_instr = this_instr; + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + DISPATCH_GOTO(); + } tstate->previous_executor = Py_None; Py_INCREF(executor); GOTO_TIER_TWO(executor); + #else + Py_FatalError("ENTER_EXECUTOR is not supported in this build"); + #endif /* _Py_TIER2 */ DISPATCH(); } @@ -2403,16 +3927,19 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXIT_INIT_CHECK); - PyObject *should_be_none; + _PyStackRef should_be_none; should_be_none = stack_pointer[-1]; assert(STACK_LEVEL() == 2); - if (should_be_none != Py_None) { + if (!PyStackRef_IsNone(should_be_none)) { + _PyFrame_SetStackPointer(frame, stack_pointer); PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", - Py_TYPE(should_be_none)->tp_name); - GOTO_ERROR(error); + Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; } stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2431,15 +3958,19 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_SIMPLE); - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); /* If value is a unicode object, then we know the result * of format(value) is value itself. */ - if (!PyUnicode_CheckExact(value)) { - res = PyObject_Format(value, NULL); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; + if (!PyUnicode_CheckExact(value_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(value_o, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) goto pop_1_error; + res = PyStackRef_FromPyObjectSteal(res_o); } else { res = value; @@ -2452,17 +3983,21 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_WITH_SPEC); - PyObject *fmt_spec; - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef fmt_spec; + _PyStackRef res; fmt_spec = stack_pointer[-1]; value = stack_pointer[-2]; - res = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_DECREF(fmt_spec); - if (res == NULL) goto pop_2_error; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + PyStackRef_CLOSE(fmt_spec); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2470,77 +4005,110 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER); - PREDICTED(FOR_ITER); - _Py_CODEUNIT *this_instr = next_instr - 2; + PREDICTED_FOR_ITER:; + _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyObject *iter; - PyObject *next; + _PyStackRef iter; + _PyStackRef next; // _SPECIALIZE_FOR_ITER - iter = stack_pointer[-1]; { + iter = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ForIter(iter, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } - STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + OPCODE_DEFERRED_INC(FOR_ITER); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } // _FOR_ITER { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next == NULL) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_o == NULL) { if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!matches) { + goto error; } - monitor_raise(tstate, frame, this_instr); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); } /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ - JUMPBY(oparg + 2); + /* Jump forward oparg, then skip following END_FOR */ + JUMPBY(oparg + 1); DISPATCH(); } + next = PyStackRef_FromPyObjectSteal(next_o); // Common case: no jump, leave it to the code generator } stack_pointer[0] = next; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(FOR_ITER_GEN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter; + _PyStackRef iter; + _PyInterpreterFrame *gen_frame; + _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ - iter = stack_pointer[-1]; - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); - PyGenObject *gen = (PyGenObject *)iter; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); - STAT_INC(FOR_ITER, hit); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - _PyFrame_StackPush(gen_frame, Py_None); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - assert(next_instr - this_instr + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); - DISPATCH_INLINED(gen_frame); + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + } + // _FOR_ITER_GEN_FRAME + { + iter = stack_pointer[-1]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + STAT_INC(FOR_ITER, hit); + gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, PyStackRef_None); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + gen_frame->previous = frame; + // oparg is the return offset from the next instruction. + frame->return_offset = (uint16_t)( 2 + oparg); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); } TARGET(FOR_ITER_LIST) { @@ -2548,18 +4116,19 @@ next_instr += 2; INSTRUCTION_STATS(FOR_ITER_LIST); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter; - PyObject *next; + _PyStackRef iter; + _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_LIST - iter = stack_pointer[-1]; { - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + iter = stack_pointer[-1]; + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); } // _ITER_JUMP_LIST { - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); STAT_INC(FOR_ITER, hit); PyListObject *seq = it->it_seq; if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { @@ -2567,27 +4136,29 @@ #ifndef Py_GIL_DISABLED if (seq != NULL) { it->it_seq = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); } #endif - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ - JUMPBY(oparg + 2); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); DISPATCH(); } } // _ITER_NEXT_LIST { - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); PyListObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyList_GET_SIZE(seq)); - next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); } stack_pointer[0] = next; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2596,41 +4167,41 @@ next_instr += 2; INSTRUCTION_STATS(FOR_ITER_RANGE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter; - PyObject *next; + _PyStackRef iter; + _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_RANGE - iter = stack_pointer[-1]; { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + iter = stack_pointer[-1]; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); } // _ITER_JUMP_RANGE { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); STAT_INC(FOR_ITER, hit); if (r->len <= 0) { - STACK_SHRINK(1); - Py_DECREF(r); - // Jump over END_FOR and POP_TOP instructions. - JUMPBY(oparg + 2); + // Jump over END_FOR instruction. + JUMPBY(oparg + 1); DISPATCH(); } } // _ITER_NEXT_RANGE { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); assert(r->len > 0); long value = r->start; r->start = value + r->step; r->len--; - next = PyLong_FromLong(value); - if (next == NULL) goto error; + PyObject *res = PyLong_FromLong(value); + if (res == NULL) goto error; + next = PyStackRef_FromPyObjectSteal(res); } stack_pointer[0] = next; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2639,43 +4210,46 @@ next_instr += 2; INSTRUCTION_STATS(FOR_ITER_TUPLE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter; - PyObject *next; + _PyStackRef iter; + _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_TUPLE - iter = stack_pointer[-1]; { - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + iter = stack_pointer[-1]; + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); } // _ITER_JUMP_TUPLE { - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); STAT_INC(FOR_ITER, hit); PyTupleObject *seq = it->it_seq; if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { if (seq != NULL) { it->it_seq = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); } - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ - JUMPBY(oparg + 2); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); DISPATCH(); } } // _ITER_NEXT_TUPLE { - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); } stack_pointer[0] = next; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2683,34 +4257,45 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AITER); - PyObject *obj; - PyObject *iter; + _PyStackRef obj; + _PyStackRef iter; obj = stack_pointer[-1]; unaryfunc getter = NULL; - PyTypeObject *type = Py_TYPE(obj); + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; } if (getter == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - Py_DECREF(obj); - if (true) goto pop_1_error; - } - iter = (*getter)(obj); - Py_DECREF(obj); - if (iter == NULL) goto pop_1_error; - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(obj); + goto pop_1_error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + iter_o = (*getter)(obj_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(obj); + if (iter_o == NULL) goto pop_1_error; + if (Py_TYPE(iter_o)->tp_as_async == NULL || + Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", - Py_TYPE(iter)->tp_name); - Py_DECREF(iter); - if (true) goto pop_1_error; + Py_TYPE(iter_o)->tp_name); + Py_DECREF(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; } + iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; DISPATCH(); } @@ -2719,49 +4304,19 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ANEXT); - PyObject *aiter; - PyObject *awaitable; + _PyStackRef aiter; + _PyStackRef awaitable; aiter = stack_pointer[-1]; - unaryfunc getter = NULL; - PyObject *next_iter = NULL; - PyTypeObject *type = Py_TYPE(aiter); - if (PyAsyncGen_CheckExact(aiter)) { - awaitable = type->tp_as_async->am_anext(aiter); - if (awaitable == NULL) { - GOTO_ERROR(error); - } - } else { - if (type->tp_as_async != NULL){ - getter = type->tp_as_async->am_anext; - } - if (getter != NULL) { - next_iter = (*getter)(aiter); - if (next_iter == NULL) { - GOTO_ERROR(error); - } - } - else { - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an iterator with " - "__anext__ method, got %.100s", - type->tp_name); - GOTO_ERROR(error); - } - awaitable = _PyCoro_GetAwaitableIter(next_iter); - if (awaitable == NULL) { - _PyErr_FormatFromCause( - PyExc_TypeError, - "'async for' received an invalid object " - "from __anext__: %.100s", - Py_TYPE(next_iter)->tp_name); - Py_DECREF(next_iter); - GOTO_ERROR(error); - } else { - Py_DECREF(next_iter); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (awaitable_o == NULL) { + goto error; } + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); stack_pointer[0] = awaitable; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2769,28 +4324,15 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AWAITABLE); - PyObject *iterable; - PyObject *iter; + _PyStackRef iterable; + _PyStackRef iter; iterable = stack_pointer[-1]; - iter = _PyCoro_GetAwaitableIter(iterable); - if (iter == NULL) { - _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); - } - Py_DECREF(iterable); - if (iter != NULL && PyCoro_CheckExact(iter)) { - PyObject *yf = _PyGen_yf((PyGenObject*)iter); - if (yf != NULL) { - /* `iter` is a coroutine object that is being - awaited, `yf` is a pointer to the current awaitable - being awaited on. */ - Py_DECREF(yf); - Py_CLEAR(iter); - _PyErr_SetString(tstate, PyExc_RuntimeError, - "coroutine is being awaited already"); - /* The code below jumps to `error` if `iter` is NULL. */ - } - } - if (iter == NULL) goto pop_1_error; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(iterable); + if (iter_o == NULL) goto pop_1_error; + iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; DISPATCH(); } @@ -2799,13 +4341,16 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ITER); - PyObject *iterable; - PyObject *iter; + _PyStackRef iterable; + _PyStackRef iter; iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ - iter = PyObject_GetIter(iterable); - Py_DECREF(iterable); - if (iter == NULL) goto pop_1_error; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(iterable); + if (iter_o == NULL) goto pop_1_error; + iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; DISPATCH(); } @@ -2814,16 +4359,20 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_LEN); - PyObject *obj; - PyObject *len_o; + _PyStackRef obj; + _PyStackRef len; obj = stack_pointer[-1]; // PUSH(len(TOS)) - Py_ssize_t len_i = PyObject_Length(obj); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) goto error; - len_o = PyLong_FromSsize_t(len_i); + PyObject *len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - stack_pointer[0] = len_o; + len = PyStackRef_FromPyObjectSteal(len_o); + stack_pointer[0] = len; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2831,32 +4380,40 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_YIELD_FROM_ITER); - PyObject *iterable; - PyObject *iter; + _PyStackRef iterable; + _PyStackRef iter; iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ - if (PyCoro_CheckExact(iterable)) { + PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); + if (PyCoro_CheckExact(iterable_o)) { /* `iterable` is a coroutine */ if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { /* and it is used in a 'yield from' expression of a regular generator. */ + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); - GOTO_ERROR(error); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; } iter = iterable; } - else if (PyGen_CheckExact(iterable)) { - iter = iterable; - } else { - /* `iterable` is not a generator. */ - iter = PyObject_GetIter(iterable); - if (iter == NULL) { - GOTO_ERROR(error); + if (PyGen_CheckExact(iterable_o)) { + iter = iterable; + } + else { + /* `iterable` is not a generator. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(iterable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + goto error; + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + PyStackRef_CLOSE(iterable); } - Py_DECREF(iterable); } stack_pointer[-1] = iter; DISPATCH(); @@ -2866,14 +4423,18 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_FROM); - PyObject *from; - PyObject *res; + _PyStackRef from; + _PyStackRef res; from = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_from(tstate, from, name); - if (res == NULL) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) goto error; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2881,153 +4442,330 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_NAME); - PyObject *fromlist; - PyObject *level; - PyObject *res; + _PyStackRef level; + _PyStackRef fromlist; + _PyStackRef res; fromlist = stack_pointer[-1]; level = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_name(tstate, frame, name, fromlist, level); - Py_DECREF(level); - Py_DECREF(fromlist); - if (res == NULL) goto pop_2_error; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportName(tstate, frame, name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(level); + PyStackRef_CLOSE(fromlist); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(INSTRUMENTED_CALL) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef res; /* Skip 3 cache entries */ - int is_meth = PEEK(oparg + 1) != NULL; - int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 2); - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PEEK(total_args); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - if (err) goto error; - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - GO_TO_INSTRUCTION(CALL); + // _MAYBE_EXPAND_METHOD + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + func = &stack_pointer[-2 - oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); + } + } + // _MONITOR_CALL + { + args = &stack_pointer[-oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + func = &stack_pointer[-2 - oparg]; + int is_meth = !PyStackRef_IsNull(maybe_self[0]); + PyObject *function = PyStackRef_AsPyObjectBorrow(func[0]); + PyObject *arg0; + if (is_meth) { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self[0]); + } + else { + if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg0 = &_PyInstrumentation_MISSING; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg0 + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) goto error; + } + // _DO_CALL + { + self_or_null = maybe_self; + callable = func; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + arguments, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; + } + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(res_o); + } + } + } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + + goto PREDICTED_CALL_FUNCTION_EX; } TARGET(INSTRUMENTED_CALL_KW) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; - next_instr += 1; + next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - int is_meth = PEEK(oparg + 2) != NULL; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + uint32_t version = read_u32(&this_instr[2].cache); + (void)version; + int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 3); + PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING - : PEEK(total_args + 1); + : PyStackRef_AsPyObjectBorrow(PEEK(total_args + 1)); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; - GO_TO_INSTRUCTION(CALL_KW); + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + goto PREDICTED_CALL_KW; } TARGET(INSTRUMENTED_END_FOR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_FOR); - PyObject *value; - PyObject *receiver; + _PyStackRef receiver; + _PyStackRef value; value = stack_pointer[-1]; receiver = stack_pointer[-2]; /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ - if (PyGen_Check(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, this_instr)) { - GOTO_ERROR(error); + if (PyStackRef_GenCheck(receiver)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; } - PyErr_SetRaisedException(NULL); } - Py_DECREF(value); + PyStackRef_CLOSE(value); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(INSTRUMENTED_END_SEND) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_SEND); - PyObject *value; - PyObject *receiver; + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; value = stack_pointer[-1]; receiver = stack_pointer[-2]; - if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, this_instr)) { - GOTO_ERROR(error); + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; } - PyErr_SetRaisedException(NULL); } - Py_DECREF(receiver); - stack_pointer[-2] = value; + val = value; + PyStackRef_CLOSE(receiver); + stack_pointer[-2] = val; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(INSTRUMENTED_FOR_ITER) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); /* Skip 1 cache entry */ - _Py_CODEUNIT *target; - PyObject *iter = TOP(); + _PyStackRef iter_stackref = TOP(); + PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); if (next != NULL) { - PUSH(next); - target = next_instr; + PUSH(PyStackRef_FromPyObjectSteal(next)); + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); } else { if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!matches) { + goto error; } - monitor_raise(tstate, frame, this_instr); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); } /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - STACK_SHRINK(1); - Py_DECREF(iter); - /* Skip END_FOR and POP_TOP */ - target = next_instr + oparg + 2; + /* Skip END_FOR */ + JUMPBY(oparg + 1); } - INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } TARGET(INSTRUMENTED_INSTRUCTION) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + _PyFrame_SetStackPointer(frame, stack_pointer); int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); if (next_opcode < 0) goto error; next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); } assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; @@ -3035,18 +4773,31 @@ } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); /* Skip 1 cache entry */ - CHECK_EVAL_BREAKER(); - INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + } + } + // _MONITOR_JUMP_BACKWARD + { + INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); + } DISPATCH(); } TARGET(INSTRUMENTED_JUMP_FORWARD) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); @@ -3054,115 +4805,207 @@ DISPATCH(); } + TARGET(INSTRUMENTED_LINE) { + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_LINE); + int original_opcode = 0; + if (tstate->tracing) { + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + int index = (int)(this_instr - _PyFrame_GetBytecode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + original_opcode = code->_co_monitoring->lines->data[index*code->_co_monitoring->lines->bytes_per_entry]; + next_instr = this_instr; + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = _Py_call_instrumentation_line( + tstate, frame, this_instr, prev_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = this_instr+1; + goto error; + } + next_instr = frame->instr_ptr; + if (next_instr != this_instr) { + DISPATCH(); + } + } + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + /* Prevent the underlying instruction from specializing + * and overwriting the instrumentation. */ + PAUSE_ADAPTIVE_COUNTER(cache->counter); + } + opcode = original_opcode; + DISPATCH_GOTO(); + } + TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); /* Skip 1 cache entry */ // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + goto PREDICTED_LOAD_SUPER_ATTR; + } + + TARGET(INSTRUMENTED_NOT_TAKEN) { + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); + (void)this_instr; // INSTRUMENTED_JUMP requires this_instr + INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_ITER) { + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); + _PyStackRef iter; + iter = stack_pointer[-1]; + INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); + PyStackRef_CLOSE(iter); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); /* Skip 1 cache entry */ - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); - int offset = flag * oparg; - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + _PyStackRef cond = POP(); + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); /* Skip 1 cache entry */ - PyObject *value = POP(); - int flag = Py_IsNone(value); - int offset; - if (flag) { - offset = oparg; + _PyStackRef value_stackref = POP(); + int jump = PyStackRef_IsNone(value_stackref); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } else { - Py_DECREF(value); - offset = 0; + PyStackRef_CLOSE(value_stackref); } - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); /* Skip 1 cache entry */ - PyObject *value = POP(); - int offset; - int nflag = Py_IsNone(value); - if (nflag) { - offset = 0; - } - else { - Py_DECREF(value); - offset = oparg; + _PyStackRef value_stackref = POP(); + int jump = !PyStackRef_IsNone(value_stackref); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + PyStackRef_CLOSE(value_stackref); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); /* Skip 1 cache entry */ - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); - int offset = flag * oparg; - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + _PyStackRef cond = POP(); + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } DISPATCH(); } TARGET(INSTRUMENTED_RESUME) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RESUME); - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - if (code_version != global_version) { - if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { - GOTO_ERROR(error); + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (bytecode == NULL) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + // Make sure this_instr gets reset correctley for any uops that + // follow + next_instr = frame->instr_ptr; + DISPATCH(); } - next_instr = this_instr; + #endif } - else { + // _MAYBE_INSTRUMENT + { + if (tstate->tracing == 0) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } + next_instr = this_instr; + DISPATCH(); + } + } + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + } } + } + // _MONITOR_RESUME + { _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( tstate, oparg > 0, frame, this_instr); @@ -3171,173 +5014,263 @@ if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ next_instr = frame->instr_ptr; - DISPATCH(); } } DISPATCH(); } - TARGET(INSTRUMENTED_RETURN_CONST) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); - PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); - if (err) GOTO_ERROR(error); - Py_INCREF(retval); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_IP(frame->return_offset); - goto resume_frame; - } - TARGET(INSTRUMENTED_RETURN_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); - if (err) GOTO_ERROR(error); - STACK_SHRINK(1); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_IP(frame->return_offset); - goto resume_frame; + _PyStackRef val; + _PyStackRef retval; + _PyStackRef res; + // _RETURN_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) goto error; + } + // _RETURN_VALUE + { + retval = val; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(frame->return_offset); + res = temp; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } TARGET(INSTRUMENTED_YIELD_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; - assert(frame != &entry_frame); - frame->instr_ptr = next_instr; - PyGenObject *gen = _PyFrame_GetGenerator(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, retval); - if (err) GOTO_ERROR(error); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; + _PyStackRef val; + _PyStackRef retval; + _PyStackRef value; + // _YIELD_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } + if (frame->instr_ptr != this_instr) { + next_instr = frame->instr_ptr; + DISPATCH(); + } + } + // _YIELD_VALUE + { + retval = val; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = temp; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } TARGET(INTERPRETER_EXIT) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INTERPRETER_EXIT); - PyObject *retval; + _PyStackRef retval; retval = stack_pointer[-1]; - assert(frame == &entry_frame); + assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous frame and return. */ tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return retval; + PyObject *result = PyStackRef_AsPyObjectSteal(retval); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + /* Not strictly necessary, but prevents warnings */ + return result; } TARGET(IS_OP) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IS_OP); - PyObject *right; - PyObject *left; - PyObject *b; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; right = stack_pointer[-1]; left = stack_pointer[-2]; - int res = Py_Is(left, right) ^ oparg; - Py_DECREF(left); - Py_DECREF(right); - b = res ? Py_True : Py_False; + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(JUMP_BACKWARD) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD); + PREDICTED_JUMP_BACKWARD:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; /* Skip 1 cache entry */ - CHECK_EVAL_BREAKER(); - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - #if ENABLE_SPECIALIZATION - uint16_t counter = this_instr[1].cache; - this_instr[1].cache = counter + (1 << OPTIMIZER_BITS_IN_COUNTER); - /* We are using unsigned values, but we really want signed values, so - * do the 2s complement adjustment manually */ - uint32_t offset_counter = counter ^ (1 << 15); - uint32_t threshold = tstate->interp->optimizer_backedge_threshold; - assert((threshold & OPTIMIZER_BITS_MASK) == 0); - // Use '>=' not '>' so that the optimizer/backoff bits do not effect the result. - // Double-check that the opcode isn't instrumented or something: - if (offset_counter >= threshold && this_instr->op.code == JUMP_BACKWARD) { - OPT_STAT_INC(attempts); - _Py_CODEUNIT *start = this_instr; - /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ - while (oparg > 255) { - oparg >>= 8; - start--; - } - _PyExecutorObject *executor; - int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor); - if (optimized < 0) goto error; - if (optimized) { - assert(tstate->previous_executor == NULL); - tstate->previous_executor = Py_None; - GOTO_TIER_TWO(executor); + // _SPECIALIZE_JUMP_BACKWARD + { + #if ENABLE_SPECIALIZATION + if (this_instr->op.code == JUMP_BACKWARD) { + this_instr->op.code = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; + // Need to re-dispatch so the warmup counter isn't off by one: + next_instr = this_instr; + DISPATCH_SAME_OPARG(); } - else { - int backoff = this_instr[1].cache & OPTIMIZER_BITS_MASK; - backoff++; - if (backoff < MIN_TIER2_BACKOFF) { - backoff = MIN_TIER2_BACKOFF; + #endif + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } + DISPATCH(); + } + + TARGET(JUMP_BACKWARD_JIT) { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_JIT); + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } + // _JIT + { + #ifdef _Py_TIER2 + _Py_BackoffCounter counter = this_instr[1].counter; + if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { + _Py_CODEUNIT *start = this_instr; + /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ + while (oparg > 255) { + oparg >>= 8; + start--; } - else if (backoff > MAX_TIER2_BACKOFF) { - backoff = MAX_TIER2_BACKOFF; + _PyExecutorObject *executor; + _PyFrame_SetStackPointer(frame, stack_pointer); + int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (optimized <= 0) { + this_instr[1].counter = restart_backoff_counter(counter); + if (optimized < 0) goto error; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + this_instr[1].counter = initial_jump_backoff_counter(); + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; + GOTO_TIER_TWO(executor); } - this_instr[1].cache = ((UINT16_MAX << OPTIMIZER_BITS_IN_COUNTER) << backoff) | backoff; } + else { + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + #endif } - #endif /* ENABLE_SPECIALIZATION */ DISPATCH(); } @@ -3350,10 +5283,41 @@ * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ + assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); DISPATCH(); } + TARGET(JUMP_BACKWARD_NO_JIT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } + DISPATCH(); + } + TARGET(JUMP_FORWARD) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3366,12 +5330,15 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_APPEND); - PyObject *v; - PyObject *list; + _PyStackRef list; + _PyStackRef v; v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; + int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)); + if (err < 0) goto pop_1_error; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3379,37 +5346,36 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_EXTEND); - PyObject *iterable; - PyObject *list; - iterable = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; + _PyStackRef list_st; + _PyStackRef iterable_st; + iterable_st = stack_pointer[-1]; + list_st = stack_pointer[-2 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); if (none_val == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Clear(tstate); _PyErr_Format(tstate, PyExc_TypeError, "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); } - Py_DECREF(iterable); - if (true) goto pop_1_error; + PyStackRef_CLOSE(iterable_st); + goto pop_1_error; } assert(Py_IsNone(none_val)); - Py_DECREF(iterable); + PyStackRef_CLOSE(iterable_st); stack_pointer += -1; - DISPATCH(); - } - - TARGET(LOAD_ASSERTION_ERROR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); - PyObject *value; - value = Py_NewRef(PyExc_AssertionError); - stack_pointer[0] = value; - stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3417,42 +5383,49 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); - PREDICTED(LOAD_ATTR); - _Py_CODEUNIT *this_instr = next_instr - 10; + PREDICTED_LOAD_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 10; (void)this_instr; - PyObject *owner; - PyObject *attr; - PyObject *self_or_null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *self_or_null; // _SPECIALIZE_LOAD_ATTR - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } - STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ + OPCODE_DEFERRED_INC(LOAD_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } /* Skip 8 cache entries */ // _LOAD_ATTR { + self_or_null = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *attr_o; if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ - attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { + attr_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_meth) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN */ - assert(attr != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership + assert(attr_o != NULL); // No errors on this branch + self_or_null[0] = owner; // Transfer ownership } else { /* meth is not an unbound method (but a regular attr, or @@ -3461,40 +5434,45 @@ CALL that it's not a method call. meth | NULL | arg1 | ... | argN */ - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error; - self_or_null = NULL; + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; + self_or_null[0] = PyStackRef_NULL; } } else { /* Classic, pushes one value. */ - attr = PyObject_GetAttr(owner, name); - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error; + _PyFrame_SetStackPointer(frame, stack_pointer); + attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; } + attr = PyStackRef_FromPyObjectSteal(attr_o); } stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = self_or_null; - stack_pointer += (oparg & 1); + stack_pointer += (oparg&1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(LOAD_ATTR_CLASS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS @@ -3502,32 +5480,83 @@ PyObject *descr = read_obj(&this_instr[6].cache); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - attr = Py_NewRef(descr); - null = NULL; - Py_DECREF(owner); + attr = PyStackRef_FromPyObjectNew(descr); + PyStackRef_CLOSE(owner); + } + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); + } + // _GUARD_TYPE_VERSION + { + uint32_t type_version = read_u32(&this_instr[4].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + PyStackRef_CLOSE(owner); + } + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; + _PyStackRef owner; /* Skip 1 cache entry */ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); uint32_t func_version = read_u32(&this_instr[4].cache); PyObject *getattribute = read_obj(&this_instr[6].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner); + PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); @@ -3537,84 +5566,97 @@ DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - Py_INCREF(f); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( + tstate, PyStackRef_FromPyObjectNew(f), 2, frame); // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; - new_frame->localsplus[1] = Py_NewRef(name); - frame->return_offset = (uint16_t)(next_instr - this_instr); + new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); + frame->return_offset = 10 ; DISPATCH_INLINED(new_frame); } TARGET(LOAD_ATTR_INSTANCE_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _CHECK_MANAGED_OBJECT_HAS_VALUES { - assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, LOAD_ATTR); + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + #ifdef Py_GIL_DISABLED + if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { + DEOPT_IF(true, LOAD_ATTR); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *self = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _CHECK_ATTR_METHOD_LAZY_DICT { - Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; - assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)owner + dictoffset); + uint16_t dictoffset = read_u16(&this_instr[4].cache); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL, LOAD_ATTR); } - /* Skip 2 cache entries */ + /* Skip 1 cache entry */ // _LOAD_ATTR_METHOD_LAZY_DICT { PyObject *descr = read_obj(&this_instr[6].cache); @@ -3622,79 +5664,85 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; } stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(LOAD_ATTR_METHOD_NO_DICT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *self = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT { PyObject *descr = read_obj(&this_instr[6].cache); assert(oparg & 1); - assert(Py_TYPE(owner)->tp_dictoffset == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; } stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *self = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(owner); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); } // _LOAD_ATTR_METHOD_WITH_VALUES { @@ -3703,114 +5751,136 @@ /* Cached method object */ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - attr = Py_NewRef(descr); - assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; } stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(LOAD_ATTR_MODULE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef owner; + PyDictKeysObject *mod_keys; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ - // _CHECK_ATTR_MODULE - owner = stack_pointer[-1]; + // _CHECK_ATTR_MODULE_PUSH_KEYS { + owner = stack_pointer[-1]; uint32_t dict_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != dict_version, LOAD_ATTR); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR); + mod_keys = keys; } - // _LOAD_ATTR_MODULE + // _LOAD_ATTR_MODULE_FROM_KEYS { uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; - assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < dict->ma_keys->dk_nentries); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - attr = ep->me_value; - DEOPT_IF(attr == NULL, LOAD_ATTR); + assert(mod_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + // Clear mod_keys from stack in case we need to deopt + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + DEOPT_IF(true, LOAD_ATTR); + } + #else + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; + _PyStackRef owner; + _PyStackRef attr; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT { PyObject *descr = read_obj(&this_instr[6].cache); assert((oparg & 1) == 0); - assert(Py_TYPE(owner)->tp_dictoffset == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); } stack_pointer[-1] = attr; DISPATCH(); } TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; + _PyStackRef owner; + _PyStackRef attr; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(owner); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { @@ -3818,134 +5888,193 @@ assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); } stack_pointer[-1] = attr; DISPATCH(); } TARGET(LOAD_ATTR_PROPERTY) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; + _PyStackRef owner; + _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *fget = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); - assert(Py_IS_TYPE(fget, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)fget; - assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 1); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(fget); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); - new_frame->localsplus[0] = owner; - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + } + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_PROPERTY_FRAME + { + PyObject *fget = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + PyCodeObject *code = (PyCodeObject *)f->func_code; + DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR); + DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR); + DEOPT_IF(code->co_argcount != 1, LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); + new_frame->localsplus[0] = owner; + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); } TARGET(LOAD_ATTR_SLOT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _LOAD_ATTR_SLOT { uint16_t index = read_u16(&this_instr[4].cache); - char *addr = (char *)owner + index; - attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, LOAD_ATTR); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); + DEOPT_IF(attr_o == NULL, LOAD_ATTR); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + DEOPT_IF(!increfed, LOAD_ATTR); + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(LOAD_ATTR_WITH_HINT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef owner; + PyDictObject *dict; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _CHECK_ATTR_WITH_HINT { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, LOAD_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(dict_o == NULL, LOAD_ATTR); + assert(PyDict_CheckExact((PyObject *)dict_o)); + dict = dict_o; } // _LOAD_ATTR_WITH_HINT { uint16_t hint = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + PyObject *attr_o; + if (!LOCK_OBJECT(dict)) { + DEOPT_IF(true, LOAD_ATTR); + } + if (hint >= (size_t)dict->ma_keys->dk_nentries) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, LOAD_ATTR); + } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (DK_IS_UNICODE(dict->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); - attr = ep->me_value; + if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, LOAD_ATTR); } - else { - PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); - attr = ep->me_value; + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, LOAD_ATTR); + } + attr_o = ep->me_value; + if (attr_o == NULL) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, LOAD_ATTR); } - DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + attr = PyStackRef_FromPyObjectNew(attr_o); + UNLOCK_OBJECT(dict); + PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3953,15 +6082,45 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_BUILD_CLASS); - PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; - if (bc == NULL) { + _PyStackRef bc; + PyObject *bc_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) goto error; + if (bc_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); - if (true) goto error; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; } + bc = PyStackRef_FromPyObjectSteal(bc_o); stack_pointer[0] = bc; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(LOAD_COMMON_CONSTANT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); + _PyStackRef value; + // Keep in sync with _common_constants in opcode.py + // If we ever have more than two constants, use a lookup table + PyObject *val; + if (oparg == CONSTANT_ASSERTIONERROR) { + val = PyExc_AssertionError; + } + else { + assert(oparg == CONSTANT_NOTIMPLEMENTEDERROR); + val = PyExc_NotImplementedError; + } + value = PyStackRef_FromPyObjectImmortal(val); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3969,11 +6128,61 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); - PyObject *value; - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); + PREDICTED_LOAD_CONST:; + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + _PyStackRef value; + /* We can't do this in the bytecode compiler as + * marshalling can intern strings and make them immortal. */ + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectNew(obj); + #if ENABLE_SPECIALIZATION_FT + #ifdef Py_GIL_DISABLED + uint8_t expected = LOAD_CONST; + if (!_Py_atomic_compare_exchange_uint8( + &this_instr->op.code, &expected, + _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL)) { + // We might lose a race with instrumentation, which we don't care about. + assert(expected >= MIN_INSTRUMENTED_OPCODE); + } + #else + if (this_instr->op.code == LOAD_CONST) { + this_instr->op.code = _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL; + } + #endif + #endif + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(LOAD_CONST_IMMORTAL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); + static_assert(0 == 0, "incorrect cache size"); + _PyStackRef value; + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + assert(_Py_IsImmortal(obj)); + value = PyStackRef_FromPyObjectImmortal(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(LOAD_CONST_MORTAL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST_MORTAL); + static_assert(0 == 0, "incorrect cache size"); + _PyStackRef value; + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectNew(obj); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3981,16 +6190,19 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_DEREF); - PyObject *value; - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { + _PyStackRef value; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - if (true) goto error; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; } - Py_INCREF(value); + value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3998,12 +6210,12 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST); - PyObject *value; - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + _PyStackRef value; + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4011,12 +6223,13 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); - PyObject *value; + _PyStackRef value; value = GETLOCAL(oparg); // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = NULL; + GETLOCAL(oparg) = PyStackRef_NULL; stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4024,12 +6237,21 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_CHECK); - PyObject *value; - value = GETLOCAL(oparg); - if (value == NULL) goto unbound_local_error; - Py_INCREF(value); + _PyStackRef value; + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + value = PyStackRef_DUP(value_s); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4037,17 +6259,16 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); - PyObject *value1; - PyObject *value2; + _PyStackRef value1; + _PyStackRef value2; uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; - value1 = GETLOCAL(oparg1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value1); - Py_INCREF(value2); + value1 = PyStackRef_DUP(GETLOCAL(oparg1)); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); stack_pointer[0] = value1; stack_pointer[1] = value2; stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4055,26 +6276,33 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); - PyObject *class_dict; - PyObject *value; - class_dict = stack_pointer[-1]; + _PyStackRef class_dict_st; + _PyStackRef value; + class_dict_st = stack_pointer[-1]; + PyObject *value_o; PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { - GOTO_ERROR(error); - } - if (!value) { - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + goto error; + } + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; } - Py_INCREF(value); } - Py_DECREF(class_dict); + PyStackRef_CLOSE(class_dict_st); + value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[-1] = value; DISPATCH(); } @@ -4083,30 +6311,68 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); - PyObject *mod_or_class_dict; - PyObject *v; + _PyStackRef mod_or_class_dict; + _PyStackRef v; mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); + PyObject *v_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(mod_or_class_dict); + if (err < 0) goto pop_1_error; + if (v_o == NULL) { + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + goto error; } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - GOTO_ERROR(error); + } + else { + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) goto error; + if (v_o == NULL) { + /* namespace 2: builtins */ + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) goto error; + if (v_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } } } + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - Py_DECREF(mod_or_class_dict); + v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[-1] = v; DISPATCH(); } @@ -4115,144 +6381,149 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); - PREDICTED(LOAD_GLOBAL); - _Py_CODEUNIT *this_instr = next_instr - 5; + PREDICTED_LOAD_GLOBAL:; + _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; - PyObject *res; - PyObject *null = NULL; + _PyStackRef *res; + _PyStackRef null = PyStackRef_NULL; // _SPECIALIZE_LOAD_GLOBAL { uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } - STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ + OPCODE_DEFERRED_INC(LOAD_GLOBAL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } /* Skip 1 cache entry */ /* Skip 1 cache entry */ /* Skip 1 cache entry */ // _LOAD_GLOBAL { + res = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - if (res == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - if (true) goto error; - } - Py_INCREF(res); - } - else { - /* Slow-path if globals or builtins is not a dict */ - /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; - if (res == NULL) { - /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; - if (res == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - if (true) goto error; - } - } - } - null = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*res)) goto error; + } + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; } - stack_pointer[0] = res; if (oparg & 1) stack_pointer[1] = null; stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(LOAD_GLOBAL_BUILTIN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyObject *res; - PyObject *null = NULL; + PyDictKeysObject *builtins_keys; + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(keys)); } - // _GUARD_BUILTINS_VERSION + // _GUARD_BUILTINS_VERSION_PUSH_KEYS { uint16_t version = read_u16(&this_instr[3].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + builtins_keys = keys; + assert(DK_IS_UNICODE(builtins_keys)); } - // _LOAD_GLOBAL_BUILTINS + // _LOAD_GLOBAL_BUILTINS_FROM_KEYS { uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *bdict = (PyDictObject *)BUILTINS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); - Py_INCREF(res); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + DEOPT_IF(!increfed, LOAD_GLOBAL); + #else + Py_INCREF(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + #endif STAT_INC(LOAD_GLOBAL, hit); - null = NULL; + } + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; } stack_pointer[0] = res; if (oparg & 1) stack_pointer[1] = null; stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(LOAD_GLOBAL_MODULE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyObject *res; - PyObject *null = NULL; + PyDictKeysObject *globals_keys; + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION + // _GUARD_GLOBALS_VERSION_PUSH_KEYS { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + globals_keys = keys; + assert(DK_IS_UNICODE(globals_keys)); } /* Skip 1 cache entry */ - // _LOAD_GLOBAL_MODULE + // _LOAD_GLOBAL_MODULE_FROM_KEYS { uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); - Py_INCREF(res); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + DEOPT_IF(!increfed, LOAD_GLOBAL); + #else + Py_INCREF(res_o); + res = PyStackRef_FromPyObjectSteal(res_o); + #endif STAT_INC(LOAD_GLOBAL, hit); - null = NULL; + } + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; } stack_pointer[0] = res; if (oparg & 1) stack_pointer[1] = null; stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4260,16 +6531,19 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_LOCALS); - PyObject *locals; - locals = LOCALS(); - if (locals == NULL) { + _PyStackRef locals; + PyObject *l = LOCALS(); + if (l == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - if (true) goto error; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; } - Py_INCREF(locals); + locals = PyStackRef_FromPyObjectNew(l); stack_pointer[0] = locals; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4277,35 +6551,67 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_NAME); - PyObject *v; - PyObject *mod_or_class_dict = LOCALS(); - if (mod_or_class_dict == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error; - } + _PyStackRef v; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - GOTO_ERROR(error); - } - } - } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) goto error; + v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(LOAD_SMALL_INT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SMALL_INT); + _PyStackRef value; + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectImmortal(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(LOAD_SPECIAL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SPECIAL); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null; + owner = stack_pointer[-1]; + assert(oparg <= SPECIAL_MAX); + PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); + PyObject *name = _Py_SpecialMethods[oparg].name; + PyObject *self_or_null_o; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = _PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + _Py_SpecialMethods[oparg].error, + Py_TYPE(owner_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + goto error; + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + self_or_null = self_or_null_o == NULL ? + PyStackRef_NULL : PyStackRef_FromPyObjectSteal(self_or_null_o); + stack_pointer[0] = attr; + stack_pointer[1] = self_or_null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4313,74 +6619,101 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); - PREDICTED(LOAD_SUPER_ATTR); - _Py_CODEUNIT *this_instr = next_instr - 2; + PREDICTED_LOAD_SUPER_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyObject *class; - PyObject *global_super; - PyObject *self; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; // _SPECIALIZE_LOAD_SUPER_ATTR - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; { + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT int load_method = oparg & 1; - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } - STAT_INC(LOAD_SUPER_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ + OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } // _LOAD_SUPER_ATTR - self = stack_pointer[-1]; { + self_st = stack_pointer[-1]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); - if (err) goto pop_3_error; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + goto pop_3_error; + } } // we make no attempt to optimize here; specializations should // handle any case whose performance we care about PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); } else { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { Py_CLEAR(super); } } } - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); if (super == NULL) goto pop_3_error; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = PyObject_GetAttr(super, name); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); Py_DECREF(super); - if (attr == NULL) goto pop_3_error; - null = NULL; + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) goto error; + attr = PyStackRef_FromPyObjectSteal(attr_o); } - stack_pointer[-3] = attr; - if (oparg & 1) stack_pointer[-2] = null; - stack_pointer += -2 + (oparg & 1); + // _PUSH_NULL_CONDITIONAL + { + null = PyStackRef_NULL; + } + stack_pointer[0] = attr; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4389,26 +6722,33 @@ next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr_st; /* Skip 1 cache entry */ - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(!(oparg & 1)); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); if (attr == NULL) goto pop_3_error; - stack_pointer[-3] = attr; + attr_st = PyStackRef_FromPyObjectSteal(attr); + stack_pointer[-3] = attr_st; stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4417,15 +6757,18 @@ next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - PyObject *self_or_null; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef self_or_null; /* Skip 1 cache entry */ - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); @@ -4433,23 +6776,26 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; - attr = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - Py_DECREF(global_super); - Py_DECREF(class); - if (attr == NULL) { - Py_DECREF(self); - if (true) goto pop_3_error; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + goto error; } if (method_found) { - self_or_null = self; // transfer ownership + self_or_null = self_st; // transfer ownership } else { - Py_DECREF(self); - self_or_null = NULL; + PyStackRef_CLOSE(self_st); + self_or_null = PyStackRef_NULL; } + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(global_super_st); + attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4458,13 +6804,13 @@ next_instr += 1; INSTRUCTION_STATS(MAKE_CELL); // "initial" is probably NULL but not if it's an arg (or set - // via PyFrame_LocalsToFast() before MAKE_CELL has run). - PyObject *initial = GETLOCAL(oparg); + // via the f_locals proxy before MAKE_CELL has run). + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *cell = PyCell_New(initial); if (cell == NULL) { - GOTO_ERROR(error); + goto error; } - SETLOCAL(oparg, cell); + SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); DISPATCH(); } @@ -4472,18 +6818,19 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_FUNCTION); - PyObject *codeobj; - PyObject *func; - codeobj = stack_pointer[-1]; + _PyStackRef codeobj_st; + _PyStackRef func; + codeobj_st = stack_pointer[-1]; + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); + _PyFrame_SetStackPointer(frame, stack_pointer); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); - Py_DECREF(codeobj); - if (func_obj == NULL) { - GOTO_ERROR(error); - } + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(codeobj_st); + if (func_obj == NULL) goto pop_1_error; _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); - func = (PyObject *)func_obj; + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); stack_pointer[-1] = func; DISPATCH(); } @@ -4492,17 +6839,26 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAP_ADD); - PyObject *value; - PyObject *key; - PyObject *dict; + _PyStackRef dict_st; + _PyStackRef key; + _PyStackRef value; value = stack_pointer[-1]; key = stack_pointer[-2]; - dict = stack_pointer[-3 - (oparg - 1)]; + dict_st = stack_pointer[-3 - (oparg - 1)]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2( + (PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(key), + PyStackRef_AsPyObjectSteal(value) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto pop_2_error; stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4510,30 +6866,37 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_CLASS); - PyObject *names; - PyObject *type; - PyObject *subject; - PyObject *attrs; + _PyStackRef subject; + _PyStackRef type; + _PyStackRef names; + _PyStackRef attrs; names = stack_pointer[-1]; type = stack_pointer[-2]; subject = stack_pointer[-3]; // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. - assert(PyTuple_CheckExact(names)); - attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); - Py_DECREF(subject); - Py_DECREF(type); - Py_DECREF(names); - if (attrs) { - assert(PyTuple_CheckExact(attrs)); // Success! + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(subject); + PyStackRef_CLOSE(type); + PyStackRef_CLOSE(names); + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); // Success! + attrs = PyStackRef_FromPyObjectSteal(attrs_o); } else { if (_PyErr_Occurred(tstate)) goto pop_3_error; // Error! - attrs = Py_None; // Failure! + attrs = PyStackRef_None; // Failure! } stack_pointer[-3] = attrs; stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4541,16 +6904,21 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_KEYS); - PyObject *keys; - PyObject *subject; - PyObject *values_or_none; + _PyStackRef subject; + _PyStackRef keys; + _PyStackRef values_or_none; keys = stack_pointer[-1]; subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = _PyEval_MatchKeys(tstate, subject, keys); - if (values_or_none == NULL) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (values_or_none_o == NULL) goto error; + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4558,13 +6926,14 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_MAPPING); - PyObject *subject; - PyObject *res; + _PyStackRef subject; + _PyStackRef res; subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? Py_True : Py_False; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4572,13 +6941,14 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_SEQUENCE); - PyObject *subject; - PyObject *res; + _PyStackRef subject; + _PyStackRef res; subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? Py_True : Py_False; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4589,119 +6959,139 @@ DISPATCH(); } + TARGET(NOT_TAKEN) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOT_TAKEN); + DISPATCH(); + } + TARGET(POP_EXCEPT) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_EXCEPT); - PyObject *exc_value; + _PyStackRef exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XSETREF(exc_info->exc_value, + PyStackRef_IsNone(exc_value) + ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(POP_ITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_ITER); + _PyStackRef value; + value = stack_pointer[-1]; + PyStackRef_CLOSE(value); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(POP_JUMP_IF_FALSE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_FALSE); - PyObject *cond; + _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - JUMPBY(oparg * flag); + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NONE); - PyObject *value; - PyObject *b; - PyObject *cond; + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; /* Skip 1 cache entry */ // _IS_NONE - value = stack_pointer[-1]; { - if (Py_IsNone(value)) { - b = Py_True; + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; } else { - b = Py_False; - Py_DECREF(value); + b = PyStackRef_False; + PyStackRef_CLOSE(value); } } // _POP_JUMP_IF_TRUE - cond = b; { - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - JUMPBY(oparg * flag); + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); - PyObject *value; - PyObject *b; - PyObject *cond; + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; /* Skip 1 cache entry */ // _IS_NONE - value = stack_pointer[-1]; { - if (Py_IsNone(value)) { - b = Py_True; + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; } else { - b = Py_False; - Py_DECREF(value); + b = PyStackRef_False; + PyStackRef_CLOSE(value); } } // _POP_JUMP_IF_FALSE - cond = b; { - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - JUMPBY(oparg * flag); + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_TRUE); - PyObject *cond; + _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - JUMPBY(oparg * flag); + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4709,10 +7099,11 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_TOP); - PyObject *value; + _PyStackRef value; value = stack_pointer[-1]; - Py_DECREF(value); + PyStackRef_CLOSE(value); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4720,21 +7111,24 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_EXC_INFO); - PyObject *new_exc; - PyObject *prev_exc; - new_exc = stack_pointer[-1]; + _PyStackRef exc; + _PyStackRef prev_exc; + _PyStackRef new_exc; + exc = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { - prev_exc = exc_info->exc_value; + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); } else { - prev_exc = Py_None; + prev_exc = PyStackRef_None; } - assert(PyExceptionInstance_Check(new_exc)); - exc_info->exc_value = Py_NewRef(new_exc); + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4742,69 +7136,79 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_NULL); - PyObject *res; - res = NULL; + _PyStackRef res; + res = PyStackRef_NULL; stack_pointer[0] = res; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(RAISE_VARARGS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(RAISE_VARARGS); - PyObject **args; + _PyStackRef *args; args = &stack_pointer[-oparg]; - PyObject *cause = NULL, *exc = NULL; - switch (oparg) { - case 2: - cause = args[1]; - /* fall through */ - case 1: - exc = args[0]; - /* fall through */ - case 0: - if (do_raise(tstate, exc, cause)) { - assert(oparg == 0); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; - } - break; - default: - _PyErr_SetString(tstate, PyExc_SystemError, - "bad RAISE_VARARGS oparg"); - break; + assert(oparg < 3); + PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; + PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = do_raise(tstate, exc, cause); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + assert(oparg == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto exception_unwind; } - if (true) { stack_pointer += -oparg; goto error; } + goto error; } TARGET(RERAISE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(RERAISE); - PyObject *exc; - PyObject **values; - exc = stack_pointer[-1]; + _PyStackRef *values; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; values = &stack_pointer[-1 - oparg]; + PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); assert(oparg >= 0 && oparg <= 2); if (oparg) { - PyObject *lasti = values[0]; + PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); if (PyLong_Check(lasti)) { - frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyLong_AsLong(lasti); + stack_pointer = _PyFrame_GetStackPointer(frame); assert(!_PyErr_Occurred(tstate)); } else { - assert(PyLong_Check(lasti)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - GOTO_ERROR(error); + Py_DECREF(exc); + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; } + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } assert(exc && PyExceptionInstance_Check(exc)); - Py_INCREF(exc); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); goto exception_unwind; } @@ -4821,120 +7225,136 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME); - PREDICTED(RESUME); - _Py_CODEUNIT *this_instr = next_instr - 1; + PREDICTED_RESUME:; + _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; - assert(frame == tstate->current_frame); - uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((code_version & 255) == 0); - if (code_version != global_version) { - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - if (err) goto error; - next_instr = this_instr; - } - else { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (bytecode == NULL) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + // Make sure this_instr gets reset correctley for any uops that + // follow + next_instr = frame->instr_ptr; + DISPATCH(); } - this_instr->op.code = RESUME_CHECK; + #endif } - DISPATCH(); - } - - TARGET(RESUME_CHECK) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME_CHECK); - static_assert(0 == 0, "incorrect cache size"); - #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); - DISPATCH(); - } - - TARGET(RETURN_CONST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_CONST); - PyObject *value; - PyObject *retval; - // _LOAD_CONST + // _MAYBE_INSTRUMENT { - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); + if (tstate->tracing == 0) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } + next_instr = this_instr; + DISPATCH(); + } + } } - // _POP_FRAME - retval = value; + // _QUICKEN_RESUME { - #if TIER_ONE - assert(frame != &entry_frame); - #endif - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(EMPTY()); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_SP(); - LOAD_IP(frame->return_offset); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; + #if ENABLE_SPECIALIZATION_FT + if (tstate->tracing == 0 && this_instr->op.code == RESUME) { + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + } + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + } } - #endif } DISPATCH(); } + TARGET(RESUME_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME_CHECK); + static_assert(0 == 0, "incorrect cache size"); + #if defined(__EMSCRIPTEN__) + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + DEOPT_IF(eval_breaker != version, RESUME); + #ifdef Py_GIL_DISABLED + DEOPT_IF(frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME); + #endif + DISPATCH(); + } + TARGET(RETURN_GENERATOR) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); - assert(PyFunction_Check(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + _PyStackRef res; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - if (gen == NULL) { - GOTO_ERROR(error); - } + stack_pointer = _PyFrame_GetStackPointer(frame); + if (gen == NULL) goto error; assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->instr_ptr = next_instr; + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; _PyFrame_Copy(frame, gen_frame); assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; - _PyFrame_StackPush(frame, (PyObject *)gen); LOAD_IP(frame->return_offset); - goto resume_frame; + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectSteal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } TARGET(RETURN_VALUE) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_VALUE); - PyObject *retval; + _PyStackRef retval; + _PyStackRef res; retval = stack_pointer[-1]; - #if TIER_ONE - assert(frame != &entry_frame); - #endif + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = retval; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); @@ -4942,15 +7362,13 @@ _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif + res = temp; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4958,120 +7376,177 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND); - PREDICTED(SEND); - _Py_CODEUNIT *this_instr = next_instr - 2; + PREDICTED_SEND:; + _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyObject *receiver; - PyObject *v; - PyObject *retval; + _PyStackRef receiver; + _PyStackRef v; + _PyStackRef retval; // _SPECIALIZE_SEND - receiver = stack_pointer[-2]; { + receiver = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_Send(receiver, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } - STAT_INC(SEND, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ + OPCODE_DEFERRED_INC(SEND); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } // _SEND - v = stack_pointer[-1]; { - assert(frame != &entry_frame); + v = stack_pointer[-1]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + PyObject *retval_o; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && - ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) + (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && + ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) { - PyGenObject *gen = (PyGenObject *)receiver; - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + PyGenObject *gen = (PyGenObject *)receiver_o; + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; STACK_SHRINK(1); _PyFrame_StackPush(gen_frame, v); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - assert(next_instr - this_instr + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + assert( 2 + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2 + oparg); + assert(gen_frame->previous == NULL); + gen_frame->previous = frame; DISPATCH_INLINED(gen_frame); } - if (Py_IsNone(v) && PyIter_Check(receiver)) { - retval = Py_TYPE(receiver)->tp_iternext(receiver); + if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); + stack_pointer = _PyFrame_GetStackPointer(frame); } else { - retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = PyObject_CallMethodOneArg(receiver_o, + &_Py_ID(send), + PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); } - if (retval == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) - ) { - monitor_raise(tstate, frame, this_instr); + if (retval_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); } - if (_PyGen_FetchStopIterationValue(&retval) == 0) { - assert(retval != NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyGen_FetchStopIterationValue(&retval_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err == 0) { + assert(retval_o != NULL); JUMPBY(oparg); } else { - GOTO_ERROR(error); + PyStackRef_CLOSE(v); + goto pop_1_error; } } - Py_DECREF(v); + PyStackRef_CLOSE(v); + retval = PyStackRef_FromPyObjectSteal(retval_o); } stack_pointer[-1] = retval; DISPATCH(); } TARGET(SEND_GEN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - PyObject *v; - PyObject *receiver; + _PyStackRef receiver; + _PyStackRef v; + _PyInterpreterFrame *gen_frame; + _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ - v = stack_pointer[-1]; - receiver = stack_pointer[-2]; - DEOPT_IF(tstate->interp->eval_frame, SEND); - PyGenObject *gen = (PyGenObject *)receiver; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); - STAT_INC(SEND, hit); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert(next_instr - this_instr + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); - DISPATCH_INLINED(gen_frame); + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, SEND); + } + // _SEND_GEN_FRAME + { + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + STAT_INC(SEND, hit); + gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2 + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2 + oparg); + gen_frame->previous = frame; + } + // _PUSH_FRAME + { + new_frame = gen_frame; + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = new_frame; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(new_frame->previous == frame || new_frame->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); } TARGET(SETUP_ANNOTATIONS) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SETUP_ANNOTATIONS); - int err; PyObject *ann_dict; if (LOCALS() == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); - if (true) goto error; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; } /* check if __annotations__ in locals()... */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) goto error; if (ann_dict == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); ann_dict = PyDict_New(); + stack_pointer = _PyFrame_GetStackPointer(frame); if (ann_dict == NULL) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; } else { + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); } DISPATCH(); } @@ -5080,14 +7555,18 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_ADD); - PyObject *v; - PyObject *set; + _PyStackRef set; + _PyStackRef v; v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; - int err = PySet_Add(set, v); - Py_DECREF(v); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); if (err) goto pop_1_error; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5095,36 +7574,23 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); - PyObject *func; - PyObject *attr; - func = stack_pointer[-1]; - attr = stack_pointer[-2]; + _PyStackRef attr_st; + _PyStackRef func_in; + _PyStackRef func_out; + func_in = stack_pointer[-1]; + attr_st = stack_pointer[-2]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); + PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); + func_out = func_in; assert(PyFunction_Check(func)); - PyFunctionObject *func_obj = (PyFunctionObject *)func; - switch(oparg) { - case MAKE_FUNCTION_CLOSURE: - assert(func_obj->func_closure == NULL); - func_obj->func_closure = attr; - break; - case MAKE_FUNCTION_ANNOTATIONS: - assert(func_obj->func_annotations == NULL); - func_obj->func_annotations = attr; - break; - case MAKE_FUNCTION_KWDEFAULTS: - assert(PyDict_CheckExact(attr)); - assert(func_obj->func_kwdefaults == NULL); - func_obj->func_kwdefaults = attr; - break; - case MAKE_FUNCTION_DEFAULTS: - assert(PyTuple_CheckExact(attr)); - assert(func_obj->func_defaults == NULL); - func_obj->func_defaults = attr; - break; - default: - Py_UNREACHABLE(); - } - stack_pointer[-2] = func; + size_t offset = _Py_FunctionAttributeOffsets[oparg]; + assert(offset != 0); + PyObject **ptr = (PyObject **)(((char *)func) + offset); + assert(*ptr == NULL); + *ptr = attr; + stack_pointer[-2] = func_out; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5132,14 +7598,18 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_UPDATE); - PyObject *iterable; - PyObject *set; + _PyStackRef set; + _PyStackRef iterable; iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; - int err = _PySet_Update(set, iterable); - Py_DECREF(iterable); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(iterable); if (err < 0) goto pop_1_error; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5147,166 +7617,208 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); - PREDICTED(STORE_ATTR); - _Py_CODEUNIT *this_instr = next_instr - 5; + PREDICTED_STORE_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; - PyObject *owner; - PyObject *v; + _PyStackRef owner; + _PyStackRef v; // _SPECIALIZE_STORE_ATTR - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_StoreAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } - STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ + OPCODE_DEFERRED_INC(STORE_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } /* Skip 3 cache entries */ // _STORE_ATTR - v = stack_pointer[-2]; { + v = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(owner); if (err) goto pop_2_error; } stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(STORE_ATTR_INSTANCE_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner; - PyObject *value; + _PyStackRef owner; + _PyStackRef value; /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; + // _GUARD_TYPE_VERSION_AND_LOCK { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + DEOPT_IF(true, STORE_ATTR); + } } - // _GUARD_DORV_VALUES + // _GUARD_DORV_NO_DICT { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + DEOPT_IF(true, STORE_ATTR); + } } // _STORE_ATTR_INSTANCE_VALUE - value = stack_pointer[-2]; { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + value = stack_pointer[-2]; + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); STAT_INC(STORE_ATTR, hit); - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyObject *old_value = values->values[index]; - values->values[index] = value; + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *old_value = *value_ptr; + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); if (old_value == NULL) { + PyDictValues *values = _PyObject_InlineValues(owner_o); + Py_ssize_t index = value_ptr - values->values; _PyDictValues_AddToInsertionOrder(values, index); } - else { - Py_DECREF(old_value); - } - Py_DECREF(owner); + UNLOCK_OBJECT(owner_o); + PyStackRef_CLOSE(owner); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); } - stack_pointer += -2; DISPATCH(); } TARGET(STORE_ATTR_SLOT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner; - PyObject *value; + _PyStackRef owner; + _PyStackRef value; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; { + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); } // _STORE_ATTR_SLOT - value = stack_pointer[-2]; { + value = stack_pointer[-2]; uint16_t index = read_u16(&this_instr[4].cache); - char *addr = (char *)owner + index; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = value; + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); + PyStackRef_CLOSE(owner); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); Py_XDECREF(old_value); - Py_DECREF(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); } - stack_pointer += -2; DISPATCH(); } TARGET(STORE_ATTR_WITH_HINT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner; - PyObject *value; + _PyStackRef owner; + _PyStackRef value; /* Skip 1 cache entry */ - owner = stack_pointer[-1]; - value = stack_pointer[-2]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint16_t hint = read_u16(&this_instr[4].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, STORE_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); - PyObject *old_value; - uint64_t new_version; - if (DK_IS_UNICODE(dict->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); } - else { - PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + // _STORE_ATTR_WITH_HINT + { + value = stack_pointer[-2]; + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(dict == NULL, STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR); + #ifdef Py_GIL_DISABLED + if (dict != _PyObject_GetManagedDict(owner_o)) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } + #endif + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + !DK_IS_UNICODE(dict->ma_keys)) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } + PyObject *old_value = ep->me_value; + if (old_value == NULL) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); + // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, + // when dict only holds the strong reference to value in ep->me_value. + STAT_INC(STORE_ATTR, hit); + PyStackRef_CLOSE(owner); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); } - Py_DECREF(old_value); - STAT_INC(STORE_ATTR, hit); - /* Ensure dict is GC tracked if it needs to be */ - if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { - _PyObject_GC_TRACK(dict); - } - /* PEP 509 */ - dict->ma_version_tag = new_version; - Py_DECREF(owner); - stack_pointer += -2; DISPATCH(); } @@ -5314,13 +7826,14 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_DEREF); - PyObject *v; + _PyStackRef v; v = stack_pointer[-1]; - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - PyCell_SET(cell, v); - Py_XDECREF(oldobj); + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5328,10 +7841,11 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST); - PyObject *value; + _PyStackRef value; value = stack_pointer[-1]; SETLOCAL(oparg, value); stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5339,14 +7853,13 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); - PyObject *value1; - PyObject *value2; + _PyStackRef value1; + _PyStackRef value2; value1 = stack_pointer[-1]; uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; SETLOCAL(oparg1, value1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value2); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); stack_pointer[-1] = value2; DISPATCH(); } @@ -5355,8 +7868,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_STORE_FAST); - PyObject *value1; - PyObject *value2; + _PyStackRef value2; + _PyStackRef value1; value1 = stack_pointer[-1]; value2 = stack_pointer[-2]; uint32_t oparg1 = oparg >> 4; @@ -5364,6 +7877,7 @@ SETLOCAL(oparg1, value1); SETLOCAL(oparg2, value2); stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5371,13 +7885,16 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_GLOBAL); - PyObject *v; + _PyStackRef v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyDict_SetItem(GLOBALS(), name, v); - Py_DECREF(v); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); if (err) goto pop_1_error; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5385,24 +7902,33 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_NAME); - PyObject *v; + _PyStackRef v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - Py_DECREF(v); - if (true) goto pop_1_error; - } - if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); - else - err = PyObject_SetItem(ns, name, v); - Py_DECREF(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + goto pop_1_error; + } + if (PyDict_CheckExact(ns)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + PyStackRef_CLOSE(v); if (err) goto pop_1_error; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5410,27 +7936,47 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_SLICE); - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *v; - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - v = stack_pointer[-4]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); - int err; - if (slice == NULL) { - err = 1; + _PyStackRef v; + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + // _SPECIALIZE_STORE_SLICE + { + // Placeholder until we implement STORE_SLICE specialization + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(STORE_SLICE); + #endif /* ENABLE_SPECIALIZATION */ } - else { - err = PyObject_SetItem(container, slice, v); - Py_DECREF(slice); + // _STORE_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); + int err; + if (slice == NULL) { + err = 1; + } + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + if (err) goto pop_4_error; } - Py_DECREF(v); - Py_DECREF(container); - if (err) goto pop_4_error; stack_pointer += -4; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5438,39 +7984,44 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); - PREDICTED(STORE_SUBSCR); - _Py_CODEUNIT *this_instr = next_instr - 2; + PREDICTED_STORE_SUBSCR:; + _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyObject *sub; - PyObject *container; - PyObject *v; + _PyStackRef container; + _PyStackRef sub; + _PyStackRef v; // _SPECIALIZE_STORE_SUBSCR - sub = stack_pointer[-1]; - container = stack_pointer[-2]; { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_StoreSubscr(container, sub, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } - STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ + OPCODE_DEFERRED_INC(STORE_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } // _STORE_SUBSCR - v = stack_pointer[-3]; { + v = stack_pointer[-3]; /* container[sub] = v */ - int err = PyObject_SetItem(container, sub, v); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); if (err) goto pop_3_error; } stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5479,19 +8030,25 @@ next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *dict; - PyObject *value; + _PyStackRef value; + _PyStackRef dict_st; + _PyStackRef sub; /* Skip 1 cache entry */ sub = stack_pointer[-1]; - dict = stack_pointer[-2]; + dict_st = stack_pointer[-2]; value = stack_pointer[-3]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); - Py_DECREF(dict); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(dict_st); if (err) goto pop_3_error; stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5500,28 +8057,38 @@ next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *list; - PyObject *value; + _PyStackRef value; + _PyStackRef list_st; + _PyStackRef sub_st; /* Skip 1 cache entry */ - sub = stack_pointer[-1]; - list = stack_pointer[-2]; + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; value = stack_pointer[-3]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); // Ensure nonnegative, zero-or-one-digit ints. DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR); // Ensure index < len(list) - DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); + if (index >= PyList_GET_SIZE(list)) { + UNLOCK_OBJECT(list); + DEOPT_IF(true, STORE_SUBSCR); + } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, value); + PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); - Py_DECREF(old_value); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + UNLOCK_OBJECT(list); // unlock before decrefs! + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + PyStackRef_CLOSE(list_st); stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -5529,13 +8096,17 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SWAP); - PyObject *top; - PyObject *bottom; - top = stack_pointer[-1]; - bottom = stack_pointer[-2 - (oparg-2)]; + _PyStackRef bottom_in; + _PyStackRef top_in; + _PyStackRef top_out; + _PyStackRef bottom_out; + top_in = stack_pointer[-1]; + bottom_in = stack_pointer[-2 - (oparg-2)]; + bottom_out = bottom_in; + top_out = top_in; assert(oparg >= 2); - stack_pointer[-2 - (oparg-2)] = top; - stack_pointer[-1] = bottom; + stack_pointer[-2 - (oparg-2)] = top_out; + stack_pointer[-1] = bottom_out; DISPATCH(); } @@ -5543,54 +8114,66 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL); - PREDICTED(TO_BOOL); - _Py_CODEUNIT *this_instr = next_instr - 4; + PREDICTED_TO_BOOL:; + _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; // _SPECIALIZE_TO_BOOL - value = stack_pointer[-1]; { + value = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ToBool(value, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } - STAT_INC(TO_BOOL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ + OPCODE_DEFERRED_INC(TO_BOOL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } /* Skip 2 cache entries */ // _TO_BOOL { - int err = PyObject_IsTrue(value); - Py_DECREF(value); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); if (err < 0) goto pop_1_error; - res = err ? Py_True : Py_False; + res = err ? PyStackRef_True : PyStackRef_False; } stack_pointer[-1] = res; DISPATCH(); } TARGET(TO_BOOL_ALWAYS_TRUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; + _PyStackRef owner; + _PyStackRef value; + _PyStackRef res; /* Skip 1 cache entry */ - value = stack_pointer[-1]; - uint32_t version = read_u32(&this_instr[2].cache); - // This one is a bit weird, because we expect *some* failures: - assert(version); - DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); - STAT_INC(TO_BOOL, hit); - Py_DECREF(value); - res = Py_True; + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL); + } + // _REPLACE_WITH_TRUE + { + value = owner; + PyStackRef_CLOSE(value); + res = PyStackRef_True; + } stack_pointer[-1] = res; DISPATCH(); } @@ -5600,11 +8183,11 @@ next_instr += 4; INSTRUCTION_STATS(TO_BOOL_BOOL); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; + _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - DEOPT_IF(!PyBool_Check(value), TO_BOOL); + DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL); STAT_INC(TO_BOOL, hit); DISPATCH(); } @@ -5614,20 +8197,21 @@ next_instr += 4; INSTRUCTION_STATS(TO_BOOL_INT); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL); STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value)) { - assert(_Py_IsImmortal(value)); - res = Py_False; + if (_PyLong_IsZero((PyLongObject *)value_o)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; } else { - Py_DECREF(value); - res = Py_True; + PyStackRef_CLOSE(value); + res = PyStackRef_True; } stack_pointer[-1] = res; DISPATCH(); @@ -5638,15 +8222,16 @@ next_instr += 4; INSTRUCTION_STATS(TO_BOOL_LIST); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL); STAT_INC(TO_BOOL, hit); - res = Py_SIZE(value) ? Py_True : Py_False; - Py_DECREF(value); + res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; + PyStackRef_CLOSE(value); stack_pointer[-1] = res; DISPATCH(); } @@ -5656,15 +8241,15 @@ next_instr += 4; INSTRUCTION_STATS(TO_BOOL_NONE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value), TO_BOOL); + DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL); STAT_INC(TO_BOOL, hit); - res = Py_False; + res = PyStackRef_False; stack_pointer[-1] = res; DISPATCH(); } @@ -5674,21 +8259,22 @@ next_instr += 4; INSTRUCTION_STATS(TO_BOOL_STR); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL); STAT_INC(TO_BOOL, hit); - if (value == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value)); - res = Py_False; + if (value_o == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; } else { - assert(Py_SIZE(value)); - Py_DECREF(value); - res = Py_True; + assert(Py_SIZE(value_o)); + PyStackRef_CLOSE(value); + res = PyStackRef_True; } stack_pointer[-1] = res; DISPATCH(); @@ -5698,12 +8284,15 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_INVERT); - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - res = PyNumber_Invert(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) goto pop_1_error; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; DISPATCH(); } @@ -5712,12 +8301,15 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NEGATIVE); - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - res = PyNumber_Negative(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (res_o == NULL) goto pop_1_error; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; DISPATCH(); } @@ -5726,11 +8318,12 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NOT); - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - assert(PyBool_Check(value)); - res = Py_IsFalse(value) ? Py_True : Py_False; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; stack_pointer[-1] = res; DISPATCH(); } @@ -5739,14 +8332,18 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNPACK_EX); - PyObject *seq; + _PyStackRef seq; + _PyStackRef *right; seq = stack_pointer[-1]; - int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); - PyObject **top = stack_pointer + totalargs - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq); + right = &stack_pointer[(oparg & 0xFF)]; + _PyStackRef *top = right + (oparg >> 8); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(seq); if (res == 0) goto pop_1_error; - stack_pointer += (oparg >> 8) + (oparg & 0xFF); + stack_pointer += (oparg & 0xFF) + (oparg >> 8); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5754,35 +8351,42 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); - PREDICTED(UNPACK_SEQUENCE); - _Py_CODEUNIT *this_instr = next_instr - 2; + PREDICTED_UNPACK_SEQUENCE:; + _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyObject *seq; + _PyStackRef seq; + _PyStackRef *output; // _SPECIALIZE_UNPACK_SEQUENCE - seq = stack_pointer[-1]; { + seq = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } - STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ + OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ (void)seq; (void)counter; } // _UNPACK_SEQUENCE { - PyObject **top = stack_pointer + oparg - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); - Py_DECREF(seq); + output = &stack_pointer[-1]; + _PyStackRef *top = output + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(seq); if (res == 0) goto pop_1_error; } stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5791,20 +8395,27 @@ next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq; - PyObject **values; + _PyStackRef seq; + _PyStackRef *values; /* Skip 1 cache entry */ seq = stack_pointer[-1]; values = &stack_pointer[-1]; - DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE); + if (PyList_GET_SIZE(seq_o) != oparg) { + UNLOCK_OBJECT(seq_o); + DEOPT_IF(true, UNPACK_SEQUENCE); + } STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq); + PyObject **items = _PyList_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + *values++ = PyStackRef_FromPyObjectNew(items[i]); } - Py_DECREF(seq); + UNLOCK_OBJECT(seq_o); + PyStackRef_CLOSE(seq); stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5813,20 +8424,22 @@ next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq; - PyObject **values; + _PyStackRef seq; + _PyStackRef *values; /* Skip 1 cache entry */ seq = stack_pointer[-1]; values = &stack_pointer[-1]; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq); + PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + *values++ = PyStackRef_FromPyObjectNew(items[i]); } - Py_DECREF(seq); + PyStackRef_CLOSE(seq); stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5835,19 +8448,23 @@ next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq; - PyObject **values; + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; /* Skip 1 cache entry */ seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); - values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - Py_DECREF(seq); - stack_pointer += -1 + oparg; + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + PyStackRef_CLOSE(seq); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5855,39 +8472,51 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(WITH_EXCEPT_START); - PyObject *val; - PyObject *lasti; - PyObject *exit_func; - PyObject *res; + _PyStackRef exit_func; + _PyStackRef exit_self; + _PyStackRef lasti; + _PyStackRef val; + _PyStackRef res; val = stack_pointer[-1]; lasti = stack_pointer[-3]; - exit_func = stack_pointer[-4]; + exit_self = stack_pointer[-4]; + exit_func = stack_pointer[-5]; /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception - lasti: THIRD = lasti of exception in exc_info() - - exit_func: FOURTH = the context.__exit__ bound method + - exit_self: FOURTH = the context or NULL + - exit_func: FIFTH = the context.__exit__ function or context.__exit__ bound method We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). Then we push the __exit__ return value. */ PyObject *exc, *tb; - assert(val && PyExceptionInstance_Check(val)); - exc = PyExceptionInstance_Class(val); - tb = PyException_GetTraceback(val); + PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); + PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); + assert(val_o && PyExceptionInstance_Check(val_o)); + exc = PyExceptionInstance_Class(val_o); + tb = PyException_GetTraceback(val_o); if (tb == NULL) { tb = Py_None; } else { + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(tb); + stack_pointer = _PyFrame_GetStackPointer(frame); } - assert(PyLong_Check(lasti)); + assert(PyStackRef_LongCheck(lasti)); (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[4] = {NULL, exc, val, tb}; - res = PyObject_Vectorcall(exit_func, stack + 1, - 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - if (res == NULL) goto error; + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + int has_self = !PyStackRef_IsNull(exit_self); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) goto error; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5895,28 +8524,194 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); - PyObject *retval; + _PyStackRef retval; + _PyStackRef value; retval = stack_pointer[-1]; // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. - assert(frame != &entry_frame); - frame->instr_ptr = next_instr; - PyGenObject *gen = _PyFrame_GetGenerator(frame); + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); /* We don't know which of these is relevant here, so keep them equal */ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; + value = temp; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + /* END INSTRUCTIONS */ +#if USE_COMPUTED_GOTOS + _unknown_opcode: +#else + EXTRA_CASES // From pycore_opcode_metadata.h, a 'case' for each unused opcode +#endif + /* Tell C compilers not to hold the opcode variable in the loop. + next_instr points the current instruction without TARGET(). */ + opcode = next_instr->op.code; + _PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); + goto error; + + } + + /* This should never be reached. Every opcode should end with DISPATCH() + or goto error. */ + Py_UNREACHABLE(); + /* BEGIN LABELS */ + + pop_4_error: + { + STACK_SHRINK(1); + goto pop_3_error; + } + + pop_3_error: + { + STACK_SHRINK(1); + goto pop_2_error; + } + + pop_2_error: + { + STACK_SHRINK(1); + goto pop_1_error; + } + + pop_1_error: + { + STACK_SHRINK(1); + goto error; + } + + error: + { + /* Double-check exception status. */ + #ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } + #else + assert(_PyErr_Occurred(tstate)); + #endif + + /* Log traceback info. */ + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); + goto exception_unwind; + } + + exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ + #ifdef LLTRACE + if (frame->lltrace >= 5) { + lltrace_resume_frame(frame); + } + #endif + DISPATCH(); + } + + exit_unwind: + { + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + goto resume_with_error; + } + + resume_with_error: + { + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; } + +/* END LABELS */ #undef TIER_ONE diff --git a/Python/getargs.c b/Python/getargs.c index 08e97ee3e627b5..6485a714a4e3fe 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -7,6 +7,7 @@ #include "pycore_dict.h" // _PyDict_HasOnlyStringKeys() #include "pycore_modsupport.h" // export _PyArg_NoKeywords() #include "pycore_pylifecycle.h" // _PyArg_Fini +#include "pycore_pystate.h" // _Py_IsMainInterpreter() #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_pyerrors.h" // _Py_CalculateSuggestions() @@ -588,6 +589,17 @@ converterr(const char *expected, PyObject *arg, char *msgbuf, size_t bufsize) return msgbuf; } +static const char * +convertcharerr(const char *expected, const char *what, Py_ssize_t size, + char *msgbuf, size_t bufsize) +{ + assert(expected != NULL); + PyOS_snprintf(msgbuf, bufsize, + "must be %.50s, not %.50s of length %zd", + expected, what, size); + return msgbuf; +} + #define CONV_UNICODE "(unicode conversion error)" /* Convert a non-tuple argument. Return NULL if conversion went OK, @@ -612,7 +624,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, switch (c) { case 'b': { /* unsigned byte -- very short int */ - char *p = va_arg(*p_va, char *); + unsigned char *p = va_arg(*p_va, unsigned char *); long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -633,7 +645,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'B': {/* byte sized bitfield - both signed and unsigned values allowed */ - char *p = va_arg(*p_va, char *); + unsigned char *p = va_arg(*p_va, unsigned char *); unsigned long ival = PyLong_AsUnsignedLongMask(arg); if (ival == (unsigned long)-1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -794,10 +806,22 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'c': {/* char */ char *p = va_arg(*p_va, char *); - if (PyBytes_Check(arg) && PyBytes_Size(arg) == 1) + if (PyBytes_Check(arg)) { + if (PyBytes_GET_SIZE(arg) != 1) { + return convertcharerr("a byte string of length 1", + "a bytes object", PyBytes_GET_SIZE(arg), + msgbuf, bufsize); + } *p = PyBytes_AS_STRING(arg)[0]; - else if (PyByteArray_Check(arg) && PyByteArray_Size(arg) == 1) + } + else if (PyByteArray_Check(arg)) { + if (PyByteArray_GET_SIZE(arg) != 1) { + return convertcharerr("a byte string of length 1", + "a bytearray object", PyByteArray_GET_SIZE(arg), + msgbuf, bufsize); + } *p = PyByteArray_AS_STRING(arg)[0]; + } else return converterr("a byte string of length 1", arg, msgbuf, bufsize); break; @@ -811,8 +835,11 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, if (!PyUnicode_Check(arg)) return converterr("a unicode character", arg, msgbuf, bufsize); - if (PyUnicode_GET_LENGTH(arg) != 1) - return converterr("a unicode character", arg, msgbuf, bufsize); + if (PyUnicode_GET_LENGTH(arg) != 1) { + return convertcharerr("a unicode character", + "a string", PyUnicode_GET_LENGTH(arg), + msgbuf, bufsize); + } kind = PyUnicode_KIND(arg); data = PyUnicode_DATA(arg); @@ -1907,7 +1934,8 @@ new_kwtuple(const char * const *keywords, int total, int pos) Py_DECREF(kwtuple); return NULL; } - PyUnicode_InternInPlace(&str); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternImmortal(interp, &str); PyTuple_SET_ITEM(kwtuple, i, str); } return kwtuple; @@ -1947,7 +1975,23 @@ _parser_init(void *arg) int owned; PyObject *kwtuple = parser->kwtuple; if (kwtuple == NULL) { + /* We may temporarily switch to the main interpreter to avoid + * creating a tuple that could outlive its owning interpreter. */ + PyThreadState *save_tstate = NULL; + PyThreadState *temp_tstate = NULL; + if (!_Py_IsMainInterpreter(PyInterpreterState_Get())) { + temp_tstate = PyThreadState_New(_PyInterpreterState_Main()); + if (temp_tstate == NULL) { + return -1; + } + save_tstate = PyThreadState_Swap(temp_tstate); + } kwtuple = new_kwtuple(keywords, len, pos); + if (temp_tstate != NULL) { + PyThreadState_Clear(temp_tstate); + (void)PyThreadState_Swap(save_tstate); + PyThreadState_Delete(temp_tstate); + } if (kwtuple == NULL) { return -1; } @@ -1969,8 +2013,8 @@ _parser_init(void *arg) parser->next = _Py_atomic_load_ptr(&_PyRuntime.getargs.static_parsers); do { // compare-exchange updates parser->next on failure - } while (_Py_atomic_compare_exchange_ptr(&_PyRuntime.getargs.static_parsers, - &parser->next, parser)); + } while (!_Py_atomic_compare_exchange_ptr(&_PyRuntime.getargs.static_parsers, + &parser->next, parser)); return 0; } @@ -1986,6 +2030,19 @@ parser_clear(struct _PyArg_Parser *parser) if (parser->is_kwtuple_owned) { Py_CLEAR(parser->kwtuple); } + + if (parser->format) { + parser->fname = NULL; + } + else { + assert(parser->fname != NULL); + } + parser->custom_msg = NULL; + parser->pos = 0; + parser->min = 0; + parser->max = 0; + parser->is_kwtuple_owned = 0; + parser->once.v = 0; } static PyObject* @@ -2007,7 +2064,7 @@ find_keyword(PyObject *kwnames, PyObject *const *kwstack, PyObject *key) for (i = 0; i < nkwargs; i++) { PyObject *kwname = PyTuple_GET_ITEM(kwnames, i); assert(PyUnicode_Check(kwname)); - if (_PyUnicode_EQ(kwname, key)) { + if (_PyUnicode_Equal(kwname, key)) { return Py_NewRef(kwstack[i]); } } @@ -2026,7 +2083,8 @@ vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, const char *format; const char *msg; PyObject *keyword; - int i, pos, len; + Py_ssize_t i; + int pos, len; Py_ssize_t nkwargs; freelistentry_t static_entries[STATIC_FREELIST_ENTRIES]; freelist_t freelist; @@ -2256,7 +2314,7 @@ PyObject * const * _PyArg_UnpackKeywords(PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs, PyObject *kwnames, struct _PyArg_Parser *parser, - int minpos, int maxpos, int minkw, + int minpos, int maxpos, int minkw, int varpos, PyObject **buf) { PyObject *kwtuple; @@ -2302,11 +2360,11 @@ _PyArg_UnpackKeywords(PyObject *const *args, Py_ssize_t nargs, else { nkwargs = 0; } - if (nkwargs == 0 && minkw == 0 && minpos <= nargs && nargs <= maxpos) { + if (nkwargs == 0 && minkw == 0 && minpos <= nargs && (varpos || nargs <= maxpos)) { /* Fast path. */ return args; } - if (nargs + nkwargs > maxargs) { + if (!varpos && nargs + nkwargs > maxargs) { /* Adding "keyword" (when nargs == 0) prevents producing wrong error messages in some special cases (see bpo-31229). */ PyErr_Format(PyExc_TypeError, @@ -2319,7 +2377,7 @@ _PyArg_UnpackKeywords(PyObject *const *args, Py_ssize_t nargs, nargs + nkwargs); return NULL; } - if (nargs > maxpos) { + if (!varpos && nargs > maxpos) { if (maxpos == 0) { PyErr_Format(PyExc_TypeError, "%.200s%s takes no positional arguments", @@ -2344,13 +2402,16 @@ _PyArg_UnpackKeywords(PyObject *const *args, Py_ssize_t nargs, " (%zd given)", (parser->fname == NULL) ? "function" : parser->fname, (parser->fname == NULL) ? "" : "()", - minposonly < maxpos ? "at least" : "exactly", + (varpos || minposonly < maxpos) ? "at least" : "exactly", minposonly, minposonly == 1 ? "" : "s", nargs); return NULL; } + if (varpos) { + nargs = Py_MIN(maxpos, nargs); + } /* copy tuple args */ for (i = 0; i < nargs; i++) { buf[i] = args[i]; @@ -2428,157 +2489,6 @@ _PyArg_UnpackKeywords(PyObject *const *args, Py_ssize_t nargs, return buf; } -PyObject * const * -_PyArg_UnpackKeywordsWithVararg(PyObject *const *args, Py_ssize_t nargs, - PyObject *kwargs, PyObject *kwnames, - struct _PyArg_Parser *parser, - int minpos, int maxpos, int minkw, - int vararg, PyObject **buf) -{ - PyObject *kwtuple; - PyObject *keyword; - Py_ssize_t varargssize = 0; - int i, posonly, minposonly, maxargs; - int reqlimit = minkw ? maxpos + minkw : minpos; - Py_ssize_t nkwargs; - PyObject * const *kwstack = NULL; - - assert(kwargs == NULL || PyDict_Check(kwargs)); - assert(kwargs == NULL || kwnames == NULL); - - if (parser == NULL) { - PyErr_BadInternalCall(); - return NULL; - } - - if (kwnames != NULL && !PyTuple_Check(kwnames)) { - PyErr_BadInternalCall(); - return NULL; - } - - if (args == NULL && nargs == 0) { - args = buf; - } - - if (parser_init(parser) < 0) { - return NULL; - } - - kwtuple = parser->kwtuple; - posonly = parser->pos; - minposonly = Py_MIN(posonly, minpos); - maxargs = posonly + (int)PyTuple_GET_SIZE(kwtuple); - if (kwargs != NULL) { - nkwargs = PyDict_GET_SIZE(kwargs); - } - else if (kwnames != NULL) { - nkwargs = PyTuple_GET_SIZE(kwnames); - kwstack = args + nargs; - } - else { - nkwargs = 0; - } - if (nargs < minposonly) { - PyErr_Format(PyExc_TypeError, - "%.200s%s takes %s %d positional argument%s" - " (%zd given)", - (parser->fname == NULL) ? "function" : parser->fname, - (parser->fname == NULL) ? "" : "()", - minposonly < maxpos ? "at least" : "exactly", - minposonly, - minposonly == 1 ? "" : "s", - nargs); - return NULL; - } - - /* create varargs tuple */ - varargssize = nargs - maxpos; - if (varargssize < 0) { - varargssize = 0; - } - buf[vararg] = PyTuple_New(varargssize); - if (!buf[vararg]) { - return NULL; - } - - /* copy tuple args */ - for (i = 0; i < nargs; i++) { - if (i >= vararg) { - PyTuple_SET_ITEM(buf[vararg], i - vararg, Py_NewRef(args[i])); - continue; - } - else { - buf[i] = args[i]; - } - } - - /* copy keyword args using kwtuple to drive process */ - for (i = Py_MAX((int)nargs, posonly) - Py_SAFE_DOWNCAST(varargssize, Py_ssize_t, int); i < maxargs; i++) { - PyObject *current_arg; - if (nkwargs) { - keyword = PyTuple_GET_ITEM(kwtuple, i - posonly); - if (kwargs != NULL) { - if (PyDict_GetItemRef(kwargs, keyword, ¤t_arg) < 0) { - goto exit; - } - } - else { - current_arg = find_keyword(kwnames, kwstack, keyword); - } - } - else { - current_arg = NULL; - } - - /* If an arguments is passed in as a keyword argument, - * it should be placed before `buf[vararg]`. - * - * For example: - * def f(a, /, b, *args): - * pass - * f(1, b=2) - * - * This `buf` array should be: [1, 2, NULL]. - * In this case, nargs < vararg. - * - * Otherwise, we leave a place at `buf[vararg]` for vararg tuple - * so the index is `i + 1`. */ - if (nargs < vararg && i != vararg) { - buf[i] = current_arg; - } - else { - buf[i + 1] = current_arg; - } - - if (current_arg) { - Py_DECREF(current_arg); - --nkwargs; - } - else if (i < minpos || (maxpos <= i && i < reqlimit)) { - /* Less arguments than required */ - keyword = PyTuple_GET_ITEM(kwtuple, i - posonly); - PyErr_Format(PyExc_TypeError, "%.200s%s missing required " - "argument '%U' (pos %d)", - (parser->fname == NULL) ? "function" : parser->fname, - (parser->fname == NULL) ? "" : "()", - keyword, i+1); - goto exit; - } - } - - if (nkwargs > 0) { - error_unexpected_keyword_arg(kwargs, kwnames, kwtuple, parser->fname); - goto exit; - } - - return buf; - -exit: - Py_XDECREF(buf[vararg]); - return NULL; -} - - static const char * skipitem(const char **p_format, va_list *p_va, int flags) { @@ -2631,7 +2541,7 @@ skipitem(const char **p_format, va_list *p_va, int flags) goto err; format++; } - /* fall through */ + _Py_FALLTHROUGH; case 's': /* string */ case 'z': /* string or None */ @@ -2641,6 +2551,11 @@ skipitem(const char **p_format, va_list *p_va, int flags) if (p_va != NULL) { (void) va_arg(*p_va, char **); } + if (c == 'w' && *format != '*') + { + /* after 'w', only '*' is allowed */ + goto err; + } if (*format == '#') { if (p_va != NULL) { (void) va_arg(*p_va, Py_ssize_t *); diff --git a/Python/getcopyright.c b/Python/getcopyright.c index 066c2ed66acddf..964584ddf7998e 100644 --- a/Python/getcopyright.c +++ b/Python/getcopyright.c @@ -4,7 +4,7 @@ static const char cprt[] = "\ -Copyright (c) 2001-2024 Python Software Foundation.\n\ +Copyright (c) 2001 Python Software Foundation.\n\ All Rights Reserved.\n\ \n\ Copyright (c) 2000 BeOpen.com.\n\ diff --git a/Python/getopt.c b/Python/getopt.c index f64c89fa22734a..39a6938dec7663 100644 --- a/Python/getopt.c +++ b/Python/getopt.c @@ -102,7 +102,7 @@ int _PyOS_GetOpt(Py_ssize_t argc, wchar_t * const *argv, int *longindex) // Parse long option. if (*opt_ptr == L'\0') { if (_PyOS_opterr) { - fprintf(stderr, "expected long option\n"); + fprintf(stderr, "Expected long option\n"); } return -1; } @@ -114,7 +114,7 @@ int _PyOS_GetOpt(Py_ssize_t argc, wchar_t * const *argv, int *longindex) } if (!opt->name) { if (_PyOS_opterr) { - fprintf(stderr, "unknown option %ls\n", argv[_PyOS_optind - 1]); + fprintf(stderr, "Unknown option: %ls\n", argv[_PyOS_optind - 1]); } return '_'; } diff --git a/Python/getversion.c b/Python/getversion.c index 5db836ab4bfd6d..226b2f999a6bfd 100644 --- a/Python/getversion.c +++ b/Python/getversion.c @@ -6,7 +6,7 @@ #include "patchlevel.h" static int initialized = 0; -static char version[250]; +static char version[300]; void _Py_InitVersion(void) { @@ -14,7 +14,12 @@ void _Py_InitVersion(void) return; } initialized = 1; - PyOS_snprintf(version, sizeof(version), "%.80s (%.80s) %.80s", +#ifdef Py_GIL_DISABLED + const char *buildinfo_format = "%.80s experimental free-threading build (%.80s) %.80s"; +#else + const char *buildinfo_format = "%.80s (%.80s) %.80s"; +#endif + PyOS_snprintf(version, sizeof(version), buildinfo_format, PY_VERSION, Py_GetBuildInfo(), Py_GetCompiler()); } diff --git a/Python/hamt.c b/Python/hamt.c index a8fbb00b807934..ed43a0449d7a01 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -319,6 +319,8 @@ typedef struct { Py_ssize_t a_count; } PyHamtNode_Array; +#define _PyHamtNode_Array_CAST(op) ((PyHamtNode_Array *)(op)) + typedef struct { PyObject_VAR_HEAD @@ -326,6 +328,8 @@ typedef struct { PyObject *c_array[1]; } PyHamtNode_Collision; +#define _PyHamtNode_Collision_CAST(op) ((PyHamtNode_Collision *)(op)) + static PyHamtObject * hamt_alloc(void); @@ -349,7 +353,7 @@ hamt_node_find(PyHamtNode *node, #ifdef Py_DEBUG static int hamt_node_dump(PyHamtNode *node, - _PyUnicodeWriter *writer, int level); + PyUnicodeWriter *writer, int level); #endif static PyHamtNode * @@ -444,7 +448,7 @@ hamt_bitindex(uint32_t bitmap, uint32_t bit) #ifdef Py_DEBUG static int -_hamt_dump_ident(_PyUnicodeWriter *writer, int level) +_hamt_dump_ident(PyUnicodeWriter *writer, int level) { /* Write `' ' * level` to the `writer` */ PyObject *str = NULL; @@ -467,7 +471,7 @@ _hamt_dump_ident(_PyUnicodeWriter *writer, int level) goto error; } - ret = _PyUnicodeWriter_WriteStr(writer, res); + ret = PyUnicodeWriter_WriteStr(writer, res); error: Py_XDECREF(res); @@ -476,32 +480,11 @@ _hamt_dump_ident(_PyUnicodeWriter *writer, int level) return ret; } -static int -_hamt_dump_format(_PyUnicodeWriter *writer, const char *format, ...) -{ - /* A convenient helper combining _PyUnicodeWriter_WriteStr and - PyUnicode_FromFormatV. - */ - PyObject* msg; - int ret; - - va_list vargs; - va_start(vargs, format); - msg = PyUnicode_FromFormatV(format, vargs); - va_end(vargs); - - if (msg == NULL) { - return -1; - } - - ret = _PyUnicodeWriter_WriteStr(writer, msg); - Py_DECREF(msg); - return ret; -} - #endif /* Py_DEBUG */ /////////////////////////////////// Bitmap Node +#define _PyHamtNode_Bitmap_CAST(op) ((PyHamtNode_Bitmap *)(op)) + static PyHamtNode * hamt_node_bitmap_new(Py_ssize_t size) @@ -1106,30 +1089,27 @@ hamt_node_bitmap_find(PyHamtNode_Bitmap *self, } static int -hamt_node_bitmap_traverse(PyHamtNode_Bitmap *self, visitproc visit, void *arg) +hamt_node_bitmap_traverse(PyObject *op, visitproc visit, void *arg) { /* Bitmap's tp_traverse */ - - Py_ssize_t i; - - for (i = Py_SIZE(self); --i >= 0; ) { + PyHamtNode_Bitmap *self = _PyHamtNode_Bitmap_CAST(op); + for (Py_ssize_t i = Py_SIZE(self); --i >= 0;) { Py_VISIT(self->b_array[i]); } - return 0; } static void -hamt_node_bitmap_dealloc(PyHamtNode_Bitmap *self) +hamt_node_bitmap_dealloc(PyObject *self) { /* Bitmap's tp_dealloc */ - Py_ssize_t len = Py_SIZE(self); - Py_ssize_t i; + PyHamtNode_Bitmap *node = _PyHamtNode_Bitmap_CAST(self); + Py_ssize_t i, len = Py_SIZE(self); - if (Py_SIZE(self) == 0) { + if (len == 0) { /* The empty node is statically allocated. */ - assert(self == &_Py_SINGLETON(hamt_bitmap_node_empty)); + assert(node == &_Py_SINGLETON(hamt_bitmap_node_empty)); #ifdef Py_DEBUG _Py_FatalRefcountError("deallocating the empty hamt node bitmap singleton"); #else @@ -1143,18 +1123,18 @@ hamt_node_bitmap_dealloc(PyHamtNode_Bitmap *self) if (len > 0) { i = len; while (--i >= 0) { - Py_XDECREF(self->b_array[i]); + Py_XDECREF(node->b_array[i]); } } - Py_TYPE(self)->tp_free((PyObject *)self); + Py_TYPE(self)->tp_free(self); Py_TRASHCAN_END } #ifdef Py_DEBUG static int hamt_node_bitmap_dump(PyHamtNode_Bitmap *node, - _PyUnicodeWriter *writer, int level) + PyUnicodeWriter *writer, int level) { /* Debug build: __dump__() method implementation for Bitmap nodes. */ @@ -1166,8 +1146,8 @@ hamt_node_bitmap_dump(PyHamtNode_Bitmap *node, goto error; } - if (_hamt_dump_format(writer, "BitmapNode(size=%zd count=%zd ", - Py_SIZE(node), Py_SIZE(node) / 2)) + if (PyUnicodeWriter_Format(writer, "BitmapNode(size=%zd count=%zd ", + Py_SIZE(node), Py_SIZE(node) / 2) < 0) { goto error; } @@ -1181,7 +1161,9 @@ hamt_node_bitmap_dump(PyHamtNode_Bitmap *node, if (tmp2 == NULL) { goto error; } - if (_hamt_dump_format(writer, "bitmap=%S id=%p):\n", tmp2, node)) { + if (PyUnicodeWriter_Format(writer, "bitmap=%S id=%p):\n", + tmp2, node) < 0) + { Py_DECREF(tmp2); goto error; } @@ -1196,7 +1178,7 @@ hamt_node_bitmap_dump(PyHamtNode_Bitmap *node, } if (key_or_null == NULL) { - if (_hamt_dump_format(writer, "NULL:\n")) { + if (PyUnicodeWriter_WriteUTF8(writer, "NULL:\n", -1) < 0) { goto error; } @@ -1207,14 +1189,14 @@ hamt_node_bitmap_dump(PyHamtNode_Bitmap *node, } } else { - if (_hamt_dump_format(writer, "%R: %R", key_or_null, - val_or_node)) + if (PyUnicodeWriter_Format(writer, "%R: %R", + key_or_null, val_or_node) < 0) { goto error; } } - if (_hamt_dump_format(writer, "\n")) { + if (PyUnicodeWriter_WriteUTF8(writer, "\n", 1) < 0) { goto error; } } @@ -1510,45 +1492,37 @@ hamt_node_collision_find(PyHamtNode_Collision *self, static int -hamt_node_collision_traverse(PyHamtNode_Collision *self, - visitproc visit, void *arg) +hamt_node_collision_traverse(PyObject *op, visitproc visit, void *arg) { /* Collision's tp_traverse */ - - Py_ssize_t i; - - for (i = Py_SIZE(self); --i >= 0; ) { + PyHamtNode_Collision *self = _PyHamtNode_Collision_CAST(op); + for (Py_ssize_t i = Py_SIZE(self); --i >= 0; ) { Py_VISIT(self->c_array[i]); } - return 0; } static void -hamt_node_collision_dealloc(PyHamtNode_Collision *self) +hamt_node_collision_dealloc(PyObject *self) { /* Collision's tp_dealloc */ - Py_ssize_t len = Py_SIZE(self); - PyObject_GC_UnTrack(self); Py_TRASHCAN_BEGIN(self, hamt_node_collision_dealloc) - if (len > 0) { - + PyHamtNode_Collision *node = _PyHamtNode_Collision_CAST(self); while (--len >= 0) { - Py_XDECREF(self->c_array[len]); + Py_XDECREF(node->c_array[len]); } } - - Py_TYPE(self)->tp_free((PyObject *)self); + Py_TYPE(self)->tp_free(self); Py_TRASHCAN_END } #ifdef Py_DEBUG static int hamt_node_collision_dump(PyHamtNode_Collision *node, - _PyUnicodeWriter *writer, int level) + PyUnicodeWriter *writer, int level) { /* Debug build: __dump__() method implementation for Collision nodes. */ @@ -1558,8 +1532,8 @@ hamt_node_collision_dump(PyHamtNode_Collision *node, goto error; } - if (_hamt_dump_format(writer, "CollisionNode(size=%zd id=%p):\n", - Py_SIZE(node), node)) + if (PyUnicodeWriter_Format(writer, "CollisionNode(size=%zd id=%p):\n", + Py_SIZE(node), node) < 0) { goto error; } @@ -1572,7 +1546,7 @@ hamt_node_collision_dump(PyHamtNode_Collision *node, goto error; } - if (_hamt_dump_format(writer, "%R: %R\n", key, val)) { + if (PyUnicodeWriter_Format(writer, "%R: %R\n", key, val) < 0) { goto error; } } @@ -1889,42 +1863,34 @@ hamt_node_array_find(PyHamtNode_Array *self, } static int -hamt_node_array_traverse(PyHamtNode_Array *self, - visitproc visit, void *arg) +hamt_node_array_traverse(PyObject *op, visitproc visit, void *arg) { /* Array's tp_traverse */ - - Py_ssize_t i; - - for (i = 0; i < HAMT_ARRAY_NODE_SIZE; i++) { + PyHamtNode_Array *self = _PyHamtNode_Array_CAST(op); + for (Py_ssize_t i = 0; i < HAMT_ARRAY_NODE_SIZE; i++) { Py_VISIT(self->a_array[i]); } - return 0; } static void -hamt_node_array_dealloc(PyHamtNode_Array *self) +hamt_node_array_dealloc(PyObject *self) { /* Array's tp_dealloc */ - - Py_ssize_t i; - PyObject_GC_UnTrack(self); Py_TRASHCAN_BEGIN(self, hamt_node_array_dealloc) - - for (i = 0; i < HAMT_ARRAY_NODE_SIZE; i++) { - Py_XDECREF(self->a_array[i]); + PyHamtNode_Array *obj = _PyHamtNode_Array_CAST(self); + for (Py_ssize_t i = 0; i < HAMT_ARRAY_NODE_SIZE; i++) { + Py_XDECREF(obj->a_array[i]); } - - Py_TYPE(self)->tp_free((PyObject *)self); + Py_TYPE(self)->tp_free(self); Py_TRASHCAN_END } #ifdef Py_DEBUG static int hamt_node_array_dump(PyHamtNode_Array *node, - _PyUnicodeWriter *writer, int level) + PyUnicodeWriter *writer, int level) { /* Debug build: __dump__() method implementation for Array nodes. */ @@ -1934,7 +1900,7 @@ hamt_node_array_dump(PyHamtNode_Array *node, goto error; } - if (_hamt_dump_format(writer, "ArrayNode(id=%p):\n", node)) { + if (PyUnicodeWriter_Format(writer, "ArrayNode(id=%p):\n", node) < 0) { goto error; } @@ -1947,7 +1913,7 @@ hamt_node_array_dump(PyHamtNode_Array *node, goto error; } - if (_hamt_dump_format(writer, "%zd::\n", i)) { + if (PyUnicodeWriter_Format(writer, "%zd::\n", i) < 0) { goto error; } @@ -1955,7 +1921,7 @@ hamt_node_array_dump(PyHamtNode_Array *node, goto error; } - if (_hamt_dump_format(writer, "\n")) { + if (PyUnicodeWriter_WriteUTF8(writer, "\n", 1) < 0) { goto error; } } @@ -2071,7 +2037,7 @@ hamt_node_find(PyHamtNode *node, #ifdef Py_DEBUG static int hamt_node_dump(PyHamtNode *node, - _PyUnicodeWriter *writer, int level) + PyUnicodeWriter *writer, int level) { /* Debug build: __dump__() method implementation for a node. @@ -2440,22 +2406,24 @@ _PyHamt_New(void) static PyObject * hamt_dump(PyHamtObject *self) { - _PyUnicodeWriter writer; - - _PyUnicodeWriter_Init(&writer); + PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); + if (writer == NULL) { + return NULL; + } - if (_hamt_dump_format(&writer, "HAMT(len=%zd):\n", self->h_count)) { + if (PyUnicodeWriter_Format(writer, "HAMT(len=%zd):\n", + self->h_count) < 0) { goto error; } - if (hamt_node_dump(self->h_root, &writer, 0)) { + if (hamt_node_dump(self->h_root, writer, 0)) { goto error; } - return _PyUnicodeWriter_Finish(&writer); + return PyUnicodeWriter_Finish(writer); error: - _PyUnicodeWriter_Dealloc(&writer); + PyUnicodeWriter_Discard(writer); return NULL; } #endif /* Py_DEBUG */ @@ -2624,6 +2592,8 @@ static PyObject * hamt_dump(PyHamtObject *self); #endif +#define _PyHamtObject_CAST(op) ((PyHamtObject *)(op)) + static PyObject * hamt_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) @@ -2632,24 +2602,27 @@ hamt_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } static int -hamt_tp_clear(PyHamtObject *self) +hamt_tp_clear(PyObject *op) { + PyHamtObject *self = _PyHamtObject_CAST(op); Py_CLEAR(self->h_root); return 0; } static int -hamt_tp_traverse(PyHamtObject *self, visitproc visit, void *arg) +hamt_tp_traverse(PyObject *op, visitproc visit, void *arg) { + PyHamtObject *self = _PyHamtObject_CAST(op); Py_VISIT(self->h_root); return 0; } static void -hamt_tp_dealloc(PyHamtObject *self) +hamt_tp_dealloc(PyObject *self) { - if (self == _empty_hamt) { + PyHamtObject *obj = _PyHamtObject_CAST(self); + if (obj == _empty_hamt) { /* The empty one is statically allocated. */ #ifdef Py_DEBUG _Py_FatalRefcountError("deallocating the empty hamt singleton"); @@ -2659,8 +2632,8 @@ hamt_tp_dealloc(PyHamtObject *self) } PyObject_GC_UnTrack(self); - if (self->h_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject*)self); + if (obj->h_weakreflist != NULL) { + PyObject_ClearWeakRefs(self); } (void)hamt_tp_clear(self); Py_TYPE(self)->tp_free(self); @@ -2692,16 +2665,18 @@ hamt_tp_richcompare(PyObject *v, PyObject *w, int op) } static int -hamt_tp_contains(PyHamtObject *self, PyObject *key) +hamt_tp_contains(PyObject *op, PyObject *key) { PyObject *val; + PyHamtObject *self = _PyHamtObject_CAST(op); return _PyHamt_Find(self, key, &val); } static PyObject * -hamt_tp_subscript(PyHamtObject *self, PyObject *key) +hamt_tp_subscript(PyObject *op, PyObject *key) { PyObject *val; + PyHamtObject *self = _PyHamtObject_CAST(op); hamt_find_t res = hamt_find(self, key, &val); switch (res) { case F_ERROR: @@ -2717,19 +2692,21 @@ hamt_tp_subscript(PyHamtObject *self, PyObject *key) } static Py_ssize_t -hamt_tp_len(PyHamtObject *self) +hamt_tp_len(PyObject *op) { + PyHamtObject *self = _PyHamtObject_CAST(op); return _PyHamt_Len(self); } static PyObject * -hamt_tp_iter(PyHamtObject *self) +hamt_tp_iter(PyObject *op) { + PyHamtObject *self = _PyHamtObject_CAST(op); return _PyHamt_NewIterKeys(self); } static PyObject * -hamt_py_set(PyHamtObject *self, PyObject *args) +hamt_py_set(PyObject *op, PyObject *args) { PyObject *key; PyObject *val; @@ -2738,11 +2715,12 @@ hamt_py_set(PyHamtObject *self, PyObject *args) return NULL; } + PyHamtObject *self = _PyHamtObject_CAST(op); return (PyObject *)_PyHamt_Assoc(self, key, val); } static PyObject * -hamt_py_get(PyHamtObject *self, PyObject *args) +hamt_py_get(PyObject *op, PyObject *args) { PyObject *key; PyObject *def = NULL; @@ -2752,6 +2730,7 @@ hamt_py_get(PyHamtObject *self, PyObject *args) } PyObject *val = NULL; + PyHamtObject *self = _PyHamtObject_CAST(op); hamt_find_t res = hamt_find(self, key, &val); switch (res) { case F_ERROR: @@ -2769,67 +2748,63 @@ hamt_py_get(PyHamtObject *self, PyObject *args) } static PyObject * -hamt_py_delete(PyHamtObject *self, PyObject *key) +hamt_py_delete(PyObject *op, PyObject *key) { + PyHamtObject *self = _PyHamtObject_CAST(op); return (PyObject *)_PyHamt_Without(self, key); } static PyObject * -hamt_py_items(PyHamtObject *self, PyObject *args) +hamt_py_items(PyObject *op, PyObject *args) { + PyHamtObject *self = _PyHamtObject_CAST(op); return _PyHamt_NewIterItems(self); } static PyObject * -hamt_py_values(PyHamtObject *self, PyObject *args) +hamt_py_values(PyObject *op, PyObject *args) { + PyHamtObject *self = _PyHamtObject_CAST(op); return _PyHamt_NewIterValues(self); } static PyObject * -hamt_py_keys(PyHamtObject *self, PyObject *Py_UNUSED(args)) +hamt_py_keys(PyObject *op, PyObject *Py_UNUSED(args)) { + PyHamtObject *self = _PyHamtObject_CAST(op); return _PyHamt_NewIterKeys(self); } #ifdef Py_DEBUG static PyObject * -hamt_py_dump(PyHamtObject *self, PyObject *Py_UNUSED(args)) +hamt_py_dump(PyObject *op, PyObject *Py_UNUSED(args)) { + PyHamtObject *self = _PyHamtObject_CAST(op); return hamt_dump(self); } #endif static PyMethodDef PyHamt_methods[] = { - {"set", _PyCFunction_CAST(hamt_py_set), METH_VARARGS, NULL}, - {"get", _PyCFunction_CAST(hamt_py_get), METH_VARARGS, NULL}, - {"delete", _PyCFunction_CAST(hamt_py_delete), METH_O, NULL}, - {"items", _PyCFunction_CAST(hamt_py_items), METH_NOARGS, NULL}, - {"keys", _PyCFunction_CAST(hamt_py_keys), METH_NOARGS, NULL}, - {"values", _PyCFunction_CAST(hamt_py_values), METH_NOARGS, NULL}, + {"set", hamt_py_set, METH_VARARGS, NULL}, + {"get", hamt_py_get, METH_VARARGS, NULL}, + {"delete", hamt_py_delete, METH_O, NULL}, + {"items", hamt_py_items, METH_NOARGS, NULL}, + {"keys", hamt_py_keys, METH_NOARGS, NULL}, + {"values", hamt_py_values, METH_NOARGS, NULL}, #ifdef Py_DEBUG - {"__dump__", _PyCFunction_CAST(hamt_py_dump), METH_NOARGS, NULL}, + {"__dump__", hamt_py_dump, METH_NOARGS, NULL}, #endif {NULL, NULL} }; static PySequenceMethods PyHamt_as_sequence = { - 0, /* sq_length */ - 0, /* sq_concat */ - 0, /* sq_repeat */ - 0, /* sq_item */ - 0, /* sq_slice */ - 0, /* sq_ass_item */ - 0, /* sq_ass_slice */ - (objobjproc)hamt_tp_contains, /* sq_contains */ - 0, /* sq_inplace_concat */ - 0, /* sq_inplace_repeat */ + .sq_contains = hamt_tp_contains, }; static PyMappingMethods PyHamt_as_mapping = { - (lenfunc)hamt_tp_len, /* mp_length */ - (binaryfunc)hamt_tp_subscript, /* mp_subscript */ + .mp_length = hamt_tp_len, + .mp_subscript = hamt_tp_subscript, }; PyTypeObject _PyHamt_Type = { @@ -2839,13 +2814,13 @@ PyTypeObject _PyHamt_Type = { .tp_methods = PyHamt_methods, .tp_as_mapping = &PyHamt_as_mapping, .tp_as_sequence = &PyHamt_as_sequence, - .tp_iter = (getiterfunc)hamt_tp_iter, - .tp_dealloc = (destructor)hamt_tp_dealloc, + .tp_iter = hamt_tp_iter, + .tp_dealloc = hamt_tp_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .tp_richcompare = hamt_tp_richcompare, - .tp_traverse = (traverseproc)hamt_tp_traverse, - .tp_clear = (inquiry)hamt_tp_clear, + .tp_traverse = hamt_tp_traverse, + .tp_clear = hamt_tp_clear, .tp_new = hamt_tp_new, .tp_weaklistoffset = offsetof(PyHamtObject, h_weakreflist), .tp_hash = PyObject_HashNotImplemented, @@ -2860,10 +2835,10 @@ PyTypeObject _PyHamt_ArrayNode_Type = { "hamt_array_node", sizeof(PyHamtNode_Array), 0, - .tp_dealloc = (destructor)hamt_node_array_dealloc, + .tp_dealloc = hamt_node_array_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)hamt_node_array_traverse, + .tp_traverse = hamt_node_array_traverse, .tp_free = PyObject_GC_Del, .tp_hash = PyObject_HashNotImplemented, }; @@ -2873,10 +2848,10 @@ PyTypeObject _PyHamt_BitmapNode_Type = { "hamt_bitmap_node", sizeof(PyHamtNode_Bitmap) - sizeof(PyObject *), sizeof(PyObject *), - .tp_dealloc = (destructor)hamt_node_bitmap_dealloc, + .tp_dealloc = hamt_node_bitmap_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)hamt_node_bitmap_traverse, + .tp_traverse = hamt_node_bitmap_traverse, .tp_free = PyObject_GC_Del, .tp_hash = PyObject_HashNotImplemented, }; @@ -2886,10 +2861,10 @@ PyTypeObject _PyHamt_CollisionNode_Type = { "hamt_collision_node", sizeof(PyHamtNode_Collision) - sizeof(PyObject *), sizeof(PyObject *), - .tp_dealloc = (destructor)hamt_node_collision_dealloc, + .tp_dealloc = hamt_node_collision_dealloc, .tp_getattro = PyObject_GenericGetAttr, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)hamt_node_collision_traverse, + .tp_traverse = hamt_node_collision_traverse, .tp_free = PyObject_GC_Del, .tp_hash = PyObject_HashNotImplemented, }; diff --git a/Python/import.c b/Python/import.c index dc92708c8b6ea0..b3648e24d0e064 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1,10 +1,13 @@ /* Module definition and import implementation */ #include "Python.h" +#include "pycore_audit.h" // _PySys_Audit() +#include "pycore_ceval.h" #include "pycore_hashtable.h" // _Py_hashtable_new_full() #include "pycore_import.h" // _PyImport_BootstrapImp() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // struct _import_runtime_state +#include "pycore_magic_number.h" // PYC_MAGIC_NUMBER_TOKEN #include "pycore_namespace.h" // _PyNamespace_Type #include "pycore_object.h" // _Py_SetImmortal() #include "pycore_pyerrors.h" // _PyErr_SetString() @@ -12,8 +15,8 @@ #include "pycore_pylifecycle.h" #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_sysmodule.h" // _PySys_Audit() -#include "pycore_time.h" // _PyTime_PerfCounterUnchecked() +#include "pycore_sysmodule.h" // _PySys_ClearAttrString() +#include "pycore_time.h" // _PyTime_AsMicroseconds() #include "pycore_weakref.h" // _PyWeakref_GET_REF() #include "marshal.h" // PyMarshal_ReadObjectFromString() @@ -34,6 +37,17 @@ module _imp #include "clinic/import.c.h" +#ifndef NDEBUG +static bool +is_interpreter_isolated(PyInterpreterState *interp) +{ + return !_Py_IsMainInterpreter(interp) + && !(interp->feature_flags & Py_RTFLAGS_USE_MAIN_OBMALLOC) + && interp->ceval.own_gil; +} +#endif + + /*******************************/ /* process-global import state */ /*******************************/ @@ -82,11 +96,7 @@ static struct _inittab *inittab_copy = NULL; (interp)->imports.import_func #define IMPORT_LOCK(interp) \ - (interp)->imports.lock.mutex -#define IMPORT_LOCK_THREAD(interp) \ - (interp)->imports.lock.thread -#define IMPORT_LOCK_LEVEL(interp) \ - (interp)->imports.lock.level + (interp)->imports.lock #define FIND_AND_LOAD(interp) \ (interp)->imports.find_and_load @@ -103,74 +113,21 @@ static struct _inittab *inittab_copy = NULL; void _PyImport_AcquireLock(PyInterpreterState *interp) { - unsigned long me = PyThread_get_thread_ident(); - if (me == PYTHREAD_INVALID_THREAD_ID) - return; /* Too bad */ - if (IMPORT_LOCK(interp) == NULL) { - IMPORT_LOCK(interp) = PyThread_allocate_lock(); - if (IMPORT_LOCK(interp) == NULL) - return; /* Nothing much we can do. */ - } - if (IMPORT_LOCK_THREAD(interp) == me) { - IMPORT_LOCK_LEVEL(interp)++; - return; - } - if (IMPORT_LOCK_THREAD(interp) != PYTHREAD_INVALID_THREAD_ID || - !PyThread_acquire_lock(IMPORT_LOCK(interp), 0)) - { - PyThreadState *tstate = PyEval_SaveThread(); - PyThread_acquire_lock(IMPORT_LOCK(interp), WAIT_LOCK); - PyEval_RestoreThread(tstate); - } - assert(IMPORT_LOCK_LEVEL(interp) == 0); - IMPORT_LOCK_THREAD(interp) = me; - IMPORT_LOCK_LEVEL(interp) = 1; + _PyRecursiveMutex_Lock(&IMPORT_LOCK(interp)); } -int +void _PyImport_ReleaseLock(PyInterpreterState *interp) { - unsigned long me = PyThread_get_thread_ident(); - if (me == PYTHREAD_INVALID_THREAD_ID || IMPORT_LOCK(interp) == NULL) - return 0; /* Too bad */ - if (IMPORT_LOCK_THREAD(interp) != me) - return -1; - IMPORT_LOCK_LEVEL(interp)--; - assert(IMPORT_LOCK_LEVEL(interp) >= 0); - if (IMPORT_LOCK_LEVEL(interp) == 0) { - IMPORT_LOCK_THREAD(interp) = PYTHREAD_INVALID_THREAD_ID; - PyThread_release_lock(IMPORT_LOCK(interp)); - } - return 1; + _PyRecursiveMutex_Unlock(&IMPORT_LOCK(interp)); } -#ifdef HAVE_FORK -/* This function is called from PyOS_AfterFork_Child() to ensure that newly - created child processes do not share locks with the parent. - We now acquire the import lock around fork() calls but on some platforms - (Solaris 9 and earlier? see isue7242) that still left us with problems. */ -PyStatus +void _PyImport_ReInitLock(PyInterpreterState *interp) { - if (IMPORT_LOCK(interp) != NULL) { - if (_PyThread_at_fork_reinit(&IMPORT_LOCK(interp)) < 0) { - return _PyStatus_ERR("failed to create a new lock"); - } - } - - if (IMPORT_LOCK_LEVEL(interp) > 1) { - /* Forked as a side effect of import */ - unsigned long me = PyThread_get_thread_ident(); - PyThread_acquire_lock(IMPORT_LOCK(interp), WAIT_LOCK); - IMPORT_LOCK_THREAD(interp) = me; - IMPORT_LOCK_LEVEL(interp)--; - } else { - IMPORT_LOCK_THREAD(interp) = PYTHREAD_INVALID_THREAD_ID; - IMPORT_LOCK_LEVEL(interp) = 0; - } - return _PyStatus_OK(); + // gh-126688: Thread id may change after fork() on some operating systems. + IMPORT_LOCK(interp).thread = PyThread_get_thread_ident_ex(); } -#endif /***************/ @@ -200,39 +157,54 @@ _PyImport_ClearModules(PyInterpreterState *interp) Py_SETREF(MODULES(interp), NULL); } +static inline PyObject * +get_modules_dict(PyThreadState *tstate, bool fatal) +{ + /* Technically, it would make sense to incref the dict, + * since sys.modules could be swapped out and decref'ed to 0 + * before the caller is done using it. However, that is highly + * unlikely, especially since we can rely on a global lock + * (i.e. the GIL) for thread-safety. */ + PyObject *modules = MODULES(tstate->interp); + if (modules == NULL) { + if (fatal) { + Py_FatalError("interpreter has no modules dictionary"); + } + _PyErr_SetString(tstate, PyExc_RuntimeError, + "unable to get sys.modules"); + return NULL; + } + return modules; +} + PyObject * PyImport_GetModuleDict(void) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - if (MODULES(interp) == NULL) { - Py_FatalError("interpreter has no modules dictionary"); - } - return MODULES(interp); + PyThreadState *tstate = _PyThreadState_GET(); + return get_modules_dict(tstate, true); } int _PyImport_SetModule(PyObject *name, PyObject *m) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *modules = MODULES(interp); + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *modules = get_modules_dict(tstate, true); return PyObject_SetItem(modules, name, m); } int _PyImport_SetModuleString(const char *name, PyObject *m) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *modules = MODULES(interp); + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *modules = get_modules_dict(tstate, true); return PyMapping_SetItemString(modules, name, m); } static PyObject * import_get_module(PyThreadState *tstate, PyObject *name) { - PyObject *modules = MODULES(tstate->interp); + PyObject *modules = get_modules_dict(tstate, false); if (modules == NULL) { - _PyErr_SetString(tstate, PyExc_RuntimeError, - "unable to get sys.modules"); return NULL; } @@ -297,10 +269,8 @@ PyImport_GetModule(PyObject *name) static PyObject * import_add_module(PyThreadState *tstate, PyObject *name) { - PyObject *modules = MODULES(tstate->interp); + PyObject *modules = get_modules_dict(tstate, false); if (modules == NULL) { - _PyErr_SetString(tstate, PyExc_RuntimeError, - "no import module dictionary"); return NULL; } @@ -397,7 +367,7 @@ remove_module(PyThreadState *tstate, PyObject *name) { PyObject *exc = _PyErr_GetRaisedException(tstate); - PyObject *modules = MODULES(tstate->interp); + PyObject *modules = get_modules_dict(tstate, true); if (PyDict_CheckExact(modules)) { // Error is reported to the caller (void)PyDict_Pop(modules, name, NULL); @@ -422,25 +392,62 @@ _PyImport_GetNextModuleIndex(void) return _Py_atomic_add_ssize(&LAST_MODULE_INDEX, 1) + 1; } +#ifndef NDEBUG +struct extensions_cache_value; +static struct extensions_cache_value * _find_cached_def(PyModuleDef *); +static Py_ssize_t _get_cached_module_index(struct extensions_cache_value *); +#endif + +static Py_ssize_t +_get_module_index_from_def(PyModuleDef *def) +{ + Py_ssize_t index = def->m_base.m_index; +#ifndef NDEBUG + struct extensions_cache_value *cached = _find_cached_def(def); + assert(cached == NULL || index == _get_cached_module_index(cached)); +#endif + return index; +} + +static void +_set_module_index(PyModuleDef *def, Py_ssize_t index) +{ + assert(index > 0); + if (index == def->m_base.m_index) { + /* There's nothing to do. */ + } + else if (def->m_base.m_index == 0) { + /* It should have been initialized by PyModuleDef_Init(). + * We assert here to catch this in dev, but keep going otherwise. */ + assert(def->m_base.m_index != 0); + def->m_base.m_index = index; + } + else { + /* It was already set for a different module. + * We replace the old value. */ + assert(def->m_base.m_index > 0); + def->m_base.m_index = index; + } +} + static const char * _modules_by_index_check(PyInterpreterState *interp, Py_ssize_t index) { - if (index == 0) { + if (index <= 0) { return "invalid module index"; } if (MODULES_BY_INDEX(interp) == NULL) { return "Interpreters module-list not accessible."; } - if (index > PyList_GET_SIZE(MODULES_BY_INDEX(interp))) { + if (index >= PyList_GET_SIZE(MODULES_BY_INDEX(interp))) { return "Module index out of bounds."; } return NULL; } static PyObject * -_modules_by_index_get(PyInterpreterState *interp, PyModuleDef *def) +_modules_by_index_get(PyInterpreterState *interp, Py_ssize_t index) { - Py_ssize_t index = def->m_base.m_index; if (_modules_by_index_check(interp, index) != NULL) { return NULL; } @@ -450,11 +457,9 @@ _modules_by_index_get(PyInterpreterState *interp, PyModuleDef *def) static int _modules_by_index_set(PyInterpreterState *interp, - PyModuleDef *def, PyObject *module) + Py_ssize_t index, PyObject *module) { - assert(def != NULL); - assert(def->m_slots == NULL); - assert(def->m_base.m_index > 0); + assert(index > 0); if (MODULES_BY_INDEX(interp) == NULL) { MODULES_BY_INDEX(interp) = PyList_New(0); @@ -463,7 +468,6 @@ _modules_by_index_set(PyInterpreterState *interp, } } - Py_ssize_t index = def->m_base.m_index; while (PyList_GET_SIZE(MODULES_BY_INDEX(interp)) <= index) { if (PyList_Append(MODULES_BY_INDEX(interp), Py_None) < 0) { return -1; @@ -474,9 +478,8 @@ _modules_by_index_set(PyInterpreterState *interp, } static int -_modules_by_index_clear_one(PyInterpreterState *interp, PyModuleDef *def) +_modules_by_index_clear_one(PyInterpreterState *interp, Py_ssize_t index) { - Py_ssize_t index = def->m_base.m_index; const char *err = _modules_by_index_check(interp, index); if (err != NULL) { Py_FatalError(err); @@ -493,7 +496,8 @@ PyState_FindModule(PyModuleDef* module) if (module->m_slots) { return NULL; } - return _modules_by_index_get(interp, module); + Py_ssize_t index = _get_module_index_from_def(module); + return _modules_by_index_get(interp, index); } /* _PyState_AddModule() has been completely removed from the C-API @@ -513,7 +517,9 @@ _PyState_AddModule(PyThreadState *tstate, PyObject* module, PyModuleDef* def) "PyState_AddModule called on module with slots"); return -1; } - return _modules_by_index_set(tstate->interp, def, module); + assert(def->m_slots == NULL); + Py_ssize_t index = _get_module_index_from_def(def); + return _modules_by_index_set(tstate->interp, index, module); } int @@ -533,7 +539,7 @@ PyState_AddModule(PyObject* module, PyModuleDef* def) } PyInterpreterState *interp = tstate->interp; - Py_ssize_t index = def->m_base.m_index; + Py_ssize_t index = _get_module_index_from_def(def); if (MODULES_BY_INDEX(interp) && index < PyList_GET_SIZE(MODULES_BY_INDEX(interp)) && module == PyList_GET_ITEM(MODULES_BY_INDEX(interp), index)) @@ -542,7 +548,8 @@ PyState_AddModule(PyObject* module, PyModuleDef* def) return -1; } - return _modules_by_index_set(interp, def, module); + assert(def->m_slots == NULL); + return _modules_by_index_set(interp, index, module); } int @@ -555,7 +562,8 @@ PyState_RemoveModule(PyModuleDef* def) "PyState_RemoveModule called on module with slots"); return -1; } - return _modules_by_index_clear_one(tstate->interp, def); + Py_ssize_t index = _get_module_index_from_def(def); + return _modules_by_index_clear_one(tstate->interp, index); } @@ -574,6 +582,8 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) /* cleanup the saved copy of module dicts */ PyModuleDef *md = PyModule_GetDef(m); if (md) { + // XXX Do this more carefully. The dict might be owned + // by another interpreter. Py_CLEAR(md->m_base.m_copy); } } @@ -618,77 +628,100 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) ...for single-phase init modules, where m_size == -1: (6). first time (not found in _PyRuntime.imports.extensions): - 1. _imp_create_dynamic_impl() -> import_find_extension() - 2. _imp_create_dynamic_impl() -> _PyImport_LoadDynamicModuleWithSpec() - 3. _PyImport_LoadDynamicModuleWithSpec(): load - 4. _PyImport_LoadDynamicModuleWithSpec(): call - 5. -> PyModule_Create() -> PyModule_Create2() -> PyModule_CreateInitialized() - 6. PyModule_CreateInitialized() -> PyModule_New() - 7. PyModule_CreateInitialized(): allocate mod->md_state - 8. PyModule_CreateInitialized() -> PyModule_AddFunctions() - 9. PyModule_CreateInitialized() -> PyModule_SetDocString() - 10. PyModule_CreateInitialized(): set mod->md_def - 11. : initialize the module - 12. _PyImport_LoadDynamicModuleWithSpec() -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() - 13. _PyImport_LoadDynamicModuleWithSpec(): set def->m_base.m_init - 14. _PyImport_LoadDynamicModuleWithSpec(): set __file__ - 15. _PyImport_LoadDynamicModuleWithSpec() -> _PyImport_FixupExtensionObject() - 16. _PyImport_FixupExtensionObject(): add it to interp->imports.modules_by_index - 17. _PyImport_FixupExtensionObject(): copy __dict__ into def->m_base.m_copy - 18. _PyImport_FixupExtensionObject(): add it to _PyRuntime.imports.extensions + A. _imp_create_dynamic_impl() -> import_find_extension() + B. _imp_create_dynamic_impl() -> _PyImport_GetModInitFunc() + C. _PyImport_GetModInitFunc(): load + D. _imp_create_dynamic_impl() -> import_run_extension() + E. import_run_extension() -> _PyImport_RunModInitFunc() + F. _PyImport_RunModInitFunc(): call + G. -> PyModule_Create() -> PyModule_Create2() + -> PyModule_CreateInitialized() + H. PyModule_CreateInitialized() -> PyModule_New() + I. PyModule_CreateInitialized(): allocate mod->md_state + J. PyModule_CreateInitialized() -> PyModule_AddFunctions() + K. PyModule_CreateInitialized() -> PyModule_SetDocString() + L. PyModule_CreateInitialized(): set mod->md_def + M. : initialize the module, etc. + N. import_run_extension() + -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() + O. import_run_extension(): set __file__ + P. import_run_extension() -> update_global_state_for_extension() + Q. update_global_state_for_extension(): + copy __dict__ into def->m_base.m_copy + R. update_global_state_for_extension(): + add it to _PyRuntime.imports.extensions + S. import_run_extension() -> finish_singlephase_extension() + T. finish_singlephase_extension(): + add it to interp->imports.modules_by_index + U. finish_singlephase_extension(): add it to sys.modules + + Step (Q) is skipped for core modules (sys/builtins). (6). subsequent times (found in _PyRuntime.imports.extensions): - 1. _imp_create_dynamic_impl() -> import_find_extension() - 2. import_find_extension() -> import_add_module() - 3. if name in sys.modules: use that module - 4. else: - 1. import_add_module() -> PyModule_NewObject() - 2. import_add_module(): set it on sys.modules - 5. import_find_extension(): copy the "m_copy" dict into __dict__ - 6. _imp_create_dynamic_impl() -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() + A. _imp_create_dynamic_impl() -> import_find_extension() + B. import_find_extension() -> reload_singlephase_extension() + C. reload_singlephase_extension() + -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() + D. reload_singlephase_extension() -> import_add_module() + E. if name in sys.modules: use that module + F. else: + 1. import_add_module() -> PyModule_NewObject() + 2. import_add_module(): set it on sys.modules + G. reload_singlephase_extension(): copy the "m_copy" dict into __dict__ + H. reload_singlephase_extension(): add to modules_by_index (10). (every time): - 1. noop + A. noop ...for single-phase init modules, where m_size >= 0: (6). not main interpreter and never loaded there - every time (not found in _PyRuntime.imports.extensions): - 1-16. (same as for m_size == -1) + A-P. (same as for m_size == -1) + Q. _PyImport_RunModInitFunc(): set def->m_base.m_init + R. (skipped) + S-U. (same as for m_size == -1) (6). main interpreter - first time (not found in _PyRuntime.imports.extensions): - 1-16. (same as for m_size == -1) - 17. _PyImport_FixupExtensionObject(): add it to _PyRuntime.imports.extensions + A-P. (same as for m_size == -1) + Q. _PyImport_RunModInitFunc(): set def->m_base.m_init + R-U. (same as for m_size == -1) - (6). previously loaded in main interpreter (found in _PyRuntime.imports.extensions): - 1. _imp_create_dynamic_impl() -> import_find_extension() - 2. import_find_extension(): call def->m_base.m_init - 3. import_find_extension(): add the module to sys.modules + (6). subsequent times (found in _PyRuntime.imports.extensions): + A. _imp_create_dynamic_impl() -> import_find_extension() + B. import_find_extension() -> reload_singlephase_extension() + C. reload_singlephase_extension() + -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() + D. reload_singlephase_extension(): call def->m_base.m_init (see above) + E. reload_singlephase_extension(): add the module to sys.modules + F. reload_singlephase_extension(): add to modules_by_index (10). every time: - 1. noop + A. noop ...for multi-phase init modules: (6). every time: - 1. _imp_create_dynamic_impl() -> import_find_extension() (not found) - 2. _imp_create_dynamic_impl() -> _PyImport_LoadDynamicModuleWithSpec() - 3. _PyImport_LoadDynamicModuleWithSpec(): load module init func - 4. _PyImport_LoadDynamicModuleWithSpec(): call module init func - 5. _PyImport_LoadDynamicModuleWithSpec() -> PyModule_FromDefAndSpec() - 6. PyModule_FromDefAndSpec(): gather/check moduledef slots - 7. if there's a Py_mod_create slot: + A. _imp_create_dynamic_impl() -> import_find_extension() (not found) + B. _imp_create_dynamic_impl() -> _PyImport_GetModInitFunc() + C. _PyImport_GetModInitFunc(): load + D. _imp_create_dynamic_impl() -> import_run_extension() + E. import_run_extension() -> _PyImport_RunModInitFunc() + F. _PyImport_RunModInitFunc(): call + G. import_run_extension() -> PyModule_FromDefAndSpec() + H. PyModule_FromDefAndSpec(): gather/check moduledef slots + I. if there's a Py_mod_create slot: 1. PyModule_FromDefAndSpec(): call its function - 8. else: + J. else: 1. PyModule_FromDefAndSpec() -> PyModule_NewObject() - 9: PyModule_FromDefAndSpec(): set mod->md_def - 10. PyModule_FromDefAndSpec() -> _add_methods_to_object() - 11. PyModule_FromDefAndSpec() -> PyModule_SetDocString() + K: PyModule_FromDefAndSpec(): set mod->md_def + L. PyModule_FromDefAndSpec() -> _add_methods_to_object() + M. PyModule_FromDefAndSpec() -> PyModule_SetDocString() (10). every time: - 1. _imp_exec_dynamic_impl() -> exec_builtin_or_dynamic() - 2. if mod->md_state == NULL (including if m_size == 0): + A. _imp_exec_dynamic_impl() -> exec_builtin_or_dynamic() + B. if mod->md_state == NULL (including if m_size == 0): 1. exec_builtin_or_dynamic() -> PyModule_ExecDef() 2. PyModule_ExecDef(): allocate mod->md_state 3. if there's a Py_mod_exec slot: @@ -716,7 +749,7 @@ const char * _PyImport_ResolveNameWithPackageContext(const char *name) { #ifndef HAVE_THREAD_LOCAL - PyThread_acquire_lock(EXTENSIONS.mutex, WAIT_LOCK); + PyMutex_Lock(&EXTENSIONS.mutex); #endif if (PKGCONTEXT != NULL) { const char *p = strrchr(PKGCONTEXT, '.'); @@ -726,7 +759,7 @@ _PyImport_ResolveNameWithPackageContext(const char *name) } } #ifndef HAVE_THREAD_LOCAL - PyThread_release_lock(EXTENSIONS.mutex); + PyMutex_Unlock(&EXTENSIONS.mutex); #endif return name; } @@ -735,12 +768,12 @@ const char * _PyImport_SwapPackageContext(const char *newcontext) { #ifndef HAVE_THREAD_LOCAL - PyThread_acquire_lock(EXTENSIONS.mutex, WAIT_LOCK); + PyMutex_Lock(&EXTENSIONS.mutex); #endif const char *oldcontext = PKGCONTEXT; PKGCONTEXT = newcontext; #ifndef HAVE_THREAD_LOCAL - PyThread_release_lock(EXTENSIONS.mutex); + PyMutex_Unlock(&EXTENSIONS.mutex); #endif return oldcontext; } @@ -790,6 +823,8 @@ static int clear_singlephase_extension(PyInterpreterState *interp, // Currently, this is only used for testing. // (See _testinternalcapi.clear_extension().) +// If adding another use, be careful about modules that import themselves +// recursively (see gh-123880). int _PyImport_ClearExtension(PyObject *name, PyObject *filename) { @@ -888,13 +923,14 @@ extensions_lock_release(void) PyMutex_Unlock(&_PyRuntime.imports.extensions.mutex); } + /* Magic for extension modules (built-in as well as dynamically loaded). To prevent initializing an extension module more than once, we keep a static dictionary 'extensions' keyed by the tuple (module name, module name) (for built-in modules) or by (filename, module name) (for dynamically loaded modules), containing these modules. A copy of the module's dictionary is stored by calling - _PyImport_FixupExtensionObject() immediately after the module initialization + fix_up_extension() immediately after the module initialization function succeeds. A copy can be retrieved from there by calling import_find_extension(). @@ -904,28 +940,246 @@ extensions_lock_release(void) dictionary, to avoid loading shared libraries twice. */ +typedef struct cached_m_dict { + /* A shallow copy of the original module's __dict__. */ + PyObject *copied; + /* The interpreter that owns the copy. */ + int64_t interpid; +} *cached_m_dict_t; + +struct extensions_cache_value { + PyModuleDef *def; + + /* The function used to re-initialize the module. + This is only set for legacy (single-phase init) extension modules + and only used for those that support multiple initializations + (m_size >= 0). + It is set by update_global_state_for_extension(). */ + PyModInitFunction m_init; + + /* The module's index into its interpreter's modules_by_index cache. + This is set for all extension modules but only used for legacy ones. + (See PyInterpreterState.modules_by_index for more info.) */ + Py_ssize_t m_index; + + /* A copy of the module's __dict__ after the first time it was loaded. + This is only set/used for legacy modules that do not support + multiple initializations. + It is set exclusively by fixup_cached_def(). */ + cached_m_dict_t m_dict; + struct cached_m_dict _m_dict; + + _Py_ext_module_origin origin; + +#ifdef Py_GIL_DISABLED + /* The module's md_gil slot, for legacy modules that are reinitialized from + m_dict rather than calling their initialization function again. */ + void *md_gil; +#endif +}; + +static struct extensions_cache_value * +alloc_extensions_cache_value(void) +{ + struct extensions_cache_value *value + = PyMem_RawMalloc(sizeof(struct extensions_cache_value)); + if (value == NULL) { + PyErr_NoMemory(); + return NULL; + } + *value = (struct extensions_cache_value){0}; + return value; +} + +static void +free_extensions_cache_value(struct extensions_cache_value *value) +{ + PyMem_RawFree(value); +} + +static Py_ssize_t +_get_cached_module_index(struct extensions_cache_value *cached) +{ + assert(cached->m_index > 0); + return cached->m_index; +} + +static void +fixup_cached_def(struct extensions_cache_value *value) +{ + /* For the moment, the values in the def's m_base may belong + * to another module, and we're replacing them here. This can + * cause problems later if the old module is reloaded. + * + * Also, we don't decref any old cached values first when we + * replace them here, in case we need to restore them in the + * near future. Instead, the caller is responsible for wrapping + * this up by calling cleanup_old_cached_def() or + * restore_old_cached_def() if there was an error. */ + PyModuleDef *def = value->def; + assert(def != NULL); + + /* We assume that all module defs are statically allocated + and will never be freed. Otherwise, we would incref here. */ + _Py_SetImmortalUntracked((PyObject *)def); + + def->m_base.m_init = value->m_init; + + assert(value->m_index > 0); + _set_module_index(def, value->m_index); + + /* Different modules can share the same def, so we can't just + * expect m_copy to be NULL. */ + assert(def->m_base.m_copy == NULL + || def->m_base.m_init == NULL + || value->m_dict != NULL); + if (value->m_dict != NULL) { + assert(value->m_dict->copied != NULL); + /* As noted above, we don't first decref the old value, if any. */ + def->m_base.m_copy = Py_NewRef(value->m_dict->copied); + } +} + +static void +restore_old_cached_def(PyModuleDef *def, PyModuleDef_Base *oldbase) +{ + def->m_base = *oldbase; +} + +static void +cleanup_old_cached_def(PyModuleDef_Base *oldbase) +{ + Py_XDECREF(oldbase->m_copy); +} + +static void +del_cached_def(struct extensions_cache_value *value) +{ + /* If we hadn't made the stored defs immortal, we would decref here. + However, this decref would be problematic if the module def were + dynamically allocated, it were the last ref, and this function + were called with an interpreter other than the def's owner. */ + assert(value->def == NULL || _Py_IsImmortal(value->def)); + + Py_XDECREF(value->def->m_base.m_copy); + value->def->m_base.m_copy = NULL; +} + +static int +init_cached_m_dict(struct extensions_cache_value *value, PyObject *m_dict) +{ + assert(value != NULL); + /* This should only have been called without an m_dict already set. */ + assert(value->m_dict == NULL); + if (m_dict == NULL) { + return 0; + } + assert(PyDict_Check(m_dict)); + assert(value->origin != _Py_ext_module_origin_CORE); + + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(!is_interpreter_isolated(interp)); + + /* XXX gh-88216: The copied dict is owned by the current + * interpreter. That's a problem if the interpreter has + * its own obmalloc state or if the module is successfully + * imported into such an interpreter. If the interpreter + * has its own GIL then there may be data races and + * PyImport_ClearModulesByIndex() can crash. Normally, + * a single-phase init module cannot be imported in an + * isolated interpreter, but there are ways around that. + * Hence, heere be dragons! Ideally we would instead do + * something like make a read-only, immortal copy of the + * dict using PyMem_RawMalloc() and store *that* in m_copy. + * Then we'd need to make sure to clear that when the + * runtime is finalized, rather than in + * PyImport_ClearModulesByIndex(). */ + PyObject *copied = PyDict_Copy(m_dict); + if (copied == NULL) { + /* We expect this can only be "out of memory". */ + return -1; + } + // XXX We may want to make the copy immortal. + + value->_m_dict = (struct cached_m_dict){ + .copied=copied, + .interpid=PyInterpreterState_GetID(interp), + }; + + value->m_dict = &value->_m_dict; + return 0; +} + +static void +del_cached_m_dict(struct extensions_cache_value *value) +{ + if (value->m_dict != NULL) { + assert(value->m_dict == &value->_m_dict); + assert(value->m_dict->copied != NULL); + /* In the future we can take advantage of m_dict->interpid + * to decref the dict using the owning interpreter. */ + Py_XDECREF(value->m_dict->copied); + value->m_dict = NULL; + } +} + +static PyObject * get_core_module_dict( + PyInterpreterState *interp, PyObject *name, PyObject *path); + +static PyObject * +get_cached_m_dict(struct extensions_cache_value *value, + PyObject *name, PyObject *path) +{ + assert(value != NULL); + PyInterpreterState *interp = _PyInterpreterState_GET(); + /* It might be a core module (e.g. sys & builtins), + for which we don't cache m_dict. */ + if (value->origin == _Py_ext_module_origin_CORE) { + return get_core_module_dict(interp, name, path); + } + assert(value->def != NULL); + // XXX Switch to value->m_dict. + PyObject *m_dict = value->def->m_base.m_copy; + Py_XINCREF(m_dict); + return m_dict; +} + +static void +del_extensions_cache_value(struct extensions_cache_value *value) +{ + if (value != NULL) { + del_cached_m_dict(value); + del_cached_def(value); + free_extensions_cache_value(value); + } +} + static void * hashtable_key_from_2_strings(PyObject *str1, PyObject *str2, const char sep) { - Py_ssize_t str1_len, str2_len; - const char *str1_data = PyUnicode_AsUTF8AndSize(str1, &str1_len); - const char *str2_data = PyUnicode_AsUTF8AndSize(str2, &str2_len); + const char *str1_data = _PyUnicode_AsUTF8NoNUL(str1); + const char *str2_data = _PyUnicode_AsUTF8NoNUL(str2); if (str1_data == NULL || str2_data == NULL) { return NULL; } + Py_ssize_t str1_len = strlen(str1_data); + Py_ssize_t str2_len = strlen(str2_data); + /* Make sure sep and the NULL byte won't cause an overflow. */ assert(SIZE_MAX - str1_len - str2_len > 2); size_t size = str1_len + 1 + str2_len + 1; + // XXX Use a buffer if it's a temp value (every case but "set"). char *key = PyMem_RawMalloc(size); if (key == NULL) { PyErr_NoMemory(); return NULL; } - strncpy(key, str1_data, str1_len); + memcpy(key, str1_data, str1_len); key[str1_len] = sep; - strncpy(key + str1_len + 1, str2_data, str2_len + 1); + memcpy(key + str1_len + 1, str2_data, str2_len); + key[size - 1] = '\0'; assert(strlen(key) == size - 1); return key; } @@ -933,7 +1187,7 @@ hashtable_key_from_2_strings(PyObject *str1, PyObject *str2, const char sep) static Py_uhash_t hashtable_hash_str(const void *key) { - return _Py_HashBytes(key, strlen((const char *)key)); + return Py_HashBuffer(key, strlen((const char *)key)); } static int @@ -948,102 +1202,225 @@ hashtable_destroy_str(void *ptr) PyMem_RawFree(ptr); } -#define HTSEP ':' +#ifndef NDEBUG +struct hashtable_next_match_def_data { + PyModuleDef *def; + struct extensions_cache_value *matched; +}; -static PyModuleDef * -_extensions_cache_get(PyObject *filename, PyObject *name) +static int +hashtable_next_match_def(_Py_hashtable_t *ht, + const void *key, const void *value, void *user_data) { - PyModuleDef *def = NULL; - void *key = NULL; - extensions_lock_acquire(); + if (value == NULL) { + /* It was previously deleted. */ + return 0; + } + struct hashtable_next_match_def_data *data + = (struct hashtable_next_match_def_data *)user_data; + struct extensions_cache_value *cur + = (struct extensions_cache_value *)value; + if (cur->def == data->def) { + data->matched = cur; + return 1; + } + return 0; +} + +static struct extensions_cache_value * +_find_cached_def(PyModuleDef *def) +{ + struct hashtable_next_match_def_data data = {0}; + (void)_Py_hashtable_foreach( + EXTENSIONS.hashtable, hashtable_next_match_def, &data); + return data.matched; +} +#endif + +#define HTSEP ':' +static int +_extensions_cache_init(void) +{ + _Py_hashtable_allocator_t alloc = {PyMem_RawMalloc, PyMem_RawFree}; + EXTENSIONS.hashtable = _Py_hashtable_new_full( + hashtable_hash_str, + hashtable_compare_str, + hashtable_destroy_str, // key + (_Py_hashtable_destroy_func)del_extensions_cache_value, // value + &alloc + ); if (EXTENSIONS.hashtable == NULL) { - goto finally; + PyErr_NoMemory(); + return -1; } + return 0; +} - key = hashtable_key_from_2_strings(filename, name, HTSEP); +static _Py_hashtable_entry_t * +_extensions_cache_find_unlocked(PyObject *path, PyObject *name, + void **p_key) +{ + if (EXTENSIONS.hashtable == NULL) { + return NULL; + } + void *key = hashtable_key_from_2_strings(path, name, HTSEP); if (key == NULL) { - goto finally; + return NULL; + } + _Py_hashtable_entry_t *entry = + _Py_hashtable_get_entry(EXTENSIONS.hashtable, key); + if (p_key != NULL) { + *p_key = key; + } + else { + hashtable_destroy_str(key); } - _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry( - EXTENSIONS.hashtable, key); + return entry; +} + +/* This can only fail with "out of memory". */ +static struct extensions_cache_value * +_extensions_cache_get(PyObject *path, PyObject *name) +{ + struct extensions_cache_value *value = NULL; + extensions_lock_acquire(); + + _Py_hashtable_entry_t *entry = + _extensions_cache_find_unlocked(path, name, NULL); if (entry == NULL) { + /* It was never added. */ goto finally; } - def = (PyModuleDef *)entry->value; + value = (struct extensions_cache_value *)entry->value; finally: extensions_lock_release(); - if (key != NULL) { - PyMem_RawFree(key); - } - return def; + return value; } -static int -_extensions_cache_set(PyObject *filename, PyObject *name, PyModuleDef *def) +/* This can only fail with "out of memory". */ +static struct extensions_cache_value * +_extensions_cache_set(PyObject *path, PyObject *name, + PyModuleDef *def, PyModInitFunction m_init, + Py_ssize_t m_index, PyObject *m_dict, + _Py_ext_module_origin origin, void *md_gil) { - int res = -1; + struct extensions_cache_value *value = NULL; + void *key = NULL; + struct extensions_cache_value *newvalue = NULL; + PyModuleDef_Base olddefbase = def->m_base; + + assert(def != NULL); + assert(m_init == NULL || m_dict == NULL); + /* We expect the same symbol to be used and the shared object file + * to have remained loaded, so it must be the same pointer. */ + assert(def->m_base.m_init == NULL || def->m_base.m_init == m_init); + /* For now we don't worry about comparing value->m_copy. */ + assert(def->m_base.m_copy == NULL || m_dict != NULL); + assert((origin == _Py_ext_module_origin_DYNAMIC) == (name != path)); + assert(origin != _Py_ext_module_origin_CORE || m_dict == NULL); + extensions_lock_acquire(); if (EXTENSIONS.hashtable == NULL) { - _Py_hashtable_allocator_t alloc = {PyMem_RawMalloc, PyMem_RawFree}; - EXTENSIONS.hashtable = _Py_hashtable_new_full( - hashtable_hash_str, - hashtable_compare_str, - hashtable_destroy_str, // key - /* There's no need to decref the def since it's immortal. */ - NULL, // value - &alloc - ); - if (EXTENSIONS.hashtable == NULL) { - PyErr_NoMemory(); + if (_extensions_cache_init() < 0) { goto finally; } } - void *key = hashtable_key_from_2_strings(filename, name, HTSEP); - if (key == NULL) { + /* Create a cached value to populate for the module. */ + _Py_hashtable_entry_t *entry = + _extensions_cache_find_unlocked(path, name, &key); + value = entry == NULL + ? NULL + : (struct extensions_cache_value *)entry->value; + if (value != NULL) { + /* gh-123880: If there's an existing cache value, it means a module is + * being imported recursively from its PyInit_* or Py_mod_* function. + * (That function presumably handles returning a partially + * constructed module in such a case.) + * We can reuse the existing cache value; it is owned by the cache. + * (Entries get removed from it in exceptional circumstances, + * after interpreter shutdown, and in runtime shutdown.) + */ + goto finally_oldvalue; + } + newvalue = alloc_extensions_cache_value(); + if (newvalue == NULL) { + goto finally; + } + + /* Populate the new cache value data. */ + *newvalue = (struct extensions_cache_value){ + .def=def, + .m_init=m_init, + .m_index=m_index, + /* m_dict is set by set_cached_m_dict(). */ + .origin=origin, +#ifdef Py_GIL_DISABLED + .md_gil=md_gil, +#endif + }; +#ifndef Py_GIL_DISABLED + (void)md_gil; +#endif + if (init_cached_m_dict(newvalue, m_dict) < 0) { goto finally; } + fixup_cached_def(newvalue); - int already_set = 0; - _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry( - EXTENSIONS.hashtable, key); if (entry == NULL) { - if (_Py_hashtable_set(EXTENSIONS.hashtable, key, def) < 0) { - PyMem_RawFree(key); + /* It was never added. */ + if (_Py_hashtable_set(EXTENSIONS.hashtable, key, newvalue) < 0) { PyErr_NoMemory(); goto finally; } + /* The hashtable owns the key now. */ + key = NULL; + } + else if (value == NULL) { + /* It was previously deleted. */ + entry->value = newvalue; } else { - if (entry->value == NULL) { - entry->value = def; - } - else { - /* We expect it to be static, so it must be the same pointer. */ - assert((PyModuleDef *)entry->value == def); - already_set = 1; + /* We are updating the entry for an existing module. */ + /* We expect def to be static, so it must be the same pointer. */ + assert(value->def == def); + /* We expect the same symbol to be used and the shared object file + * to have remained loaded, so it must be the same pointer. */ + assert(value->m_init == m_init); + /* The same module can't switch between caching __dict__ and not. */ + assert((value->m_dict == NULL) == (m_dict == NULL)); + /* This shouldn't ever happen. */ + Py_UNREACHABLE(); + } + + value = newvalue; + +finally: + if (value == NULL) { + restore_old_cached_def(def, &olddefbase); + if (newvalue != NULL) { + del_extensions_cache_value(newvalue); } - PyMem_RawFree(key); } - if (!already_set) { - /* We assume that all module defs are statically allocated - and will never be freed. Otherwise, we would incref here. */ - _Py_SetImmortal(def); + else { + cleanup_old_cached_def(&olddefbase); } - res = 0; -finally: +finally_oldvalue: extensions_lock_release(); - return res; + if (key != NULL) { + hashtable_destroy_str(key); + } + + return value; } static void -_extensions_cache_delete(PyObject *filename, PyObject *name) +_extensions_cache_delete(PyObject *path, PyObject *name) { - void *key = NULL; extensions_lock_acquire(); if (EXTENSIONS.hashtable == NULL) { @@ -1051,13 +1428,8 @@ _extensions_cache_delete(PyObject *filename, PyObject *name) goto finally; } - key = hashtable_key_from_2_strings(filename, name, HTSEP); - if (key == NULL) { - goto finally; - } - - _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry( - EXTENSIONS.hashtable, key); + _Py_hashtable_entry_t *entry = + _extensions_cache_find_unlocked(path, name, NULL); if (entry == NULL) { /* It was never added. */ goto finally; @@ -1066,18 +1438,13 @@ _extensions_cache_delete(PyObject *filename, PyObject *name) /* It was already removed. */ goto finally; } - /* If we hadn't made the stored defs immortal, we would decref here. - However, this decref would be problematic if the module def were - dynamically allocated, it were the last ref, and this function - were called with an interpreter other than the def's owner. */ - assert(_Py_IsImmortal(entry->value)); + struct extensions_cache_value *value = entry->value; entry->value = NULL; + del_extensions_cache_value(value); + finally: extensions_lock_release(); - if (key != NULL) { - PyMem_RawFree(key); - } } static void @@ -1123,31 +1490,122 @@ _PyImport_CheckSubinterpIncompatibleExtensionAllowed(const char *name) return 0; } +#ifdef Py_GIL_DISABLED +int +_PyImport_CheckGILForModule(PyObject* module, PyObject *module_name) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (module == NULL) { + _PyEval_DisableGIL(tstate); + return 0; + } + + if (!PyModule_Check(module) || + ((PyModuleObject *)module)->md_gil == Py_MOD_GIL_USED) { + if (_PyEval_EnableGILPermanent(tstate)) { + int warn_result = PyErr_WarnFormat( + PyExc_RuntimeWarning, + 1, + "The global interpreter lock (GIL) has been enabled to load " + "module '%U', which has not declared that it can run safely " + "without the GIL. To override this behavior and keep the GIL " + "disabled (at your own risk), run with PYTHON_GIL=0 or -Xgil=0.", + module_name + ); + if (warn_result < 0) { + return warn_result; + } + } + + const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); + if (config->enable_gil == _PyConfig_GIL_DEFAULT && config->verbose) { + PySys_FormatStderr("# loading module '%U', which requires the GIL\n", + module_name); + } + } + else { + _PyEval_DisableGIL(tstate); + } + + return 0; +} +#endif + +static PyThreadState * +switch_to_main_interpreter(PyThreadState *tstate) +{ + if (_Py_IsMainInterpreter(tstate->interp)) { + return tstate; + } + PyThreadState *main_tstate = _PyThreadState_NewBound( + _PyInterpreterState_Main(), _PyThreadState_WHENCE_EXEC); + if (main_tstate == NULL) { + return NULL; + } +#ifndef NDEBUG + PyThreadState *old_tstate = PyThreadState_Swap(main_tstate); + assert(old_tstate == tstate); +#else + (void)PyThreadState_Swap(main_tstate); +#endif + return main_tstate; +} + +static void +switch_back_from_main_interpreter(PyThreadState *tstate, + PyThreadState *main_tstate, + PyObject *tempobj) +{ + assert(main_tstate == PyThreadState_GET()); + assert(_Py_IsMainInterpreter(main_tstate->interp)); + assert(tstate->interp != main_tstate->interp); + + /* Handle any exceptions, which we cannot propagate directly + * to the subinterpreter. */ + if (PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_MemoryError)) { + /* We trust it will be caught again soon. */ + PyErr_Clear(); + } + else { + /* Printing the exception should be sufficient. */ + PyErr_PrintEx(0); + } + } + + Py_XDECREF(tempobj); + + PyThreadState_Clear(main_tstate); + (void)PyThreadState_Swap(tstate); + PyThreadState_Delete(main_tstate); +} + static PyObject * get_core_module_dict(PyInterpreterState *interp, - PyObject *name, PyObject *filename) + PyObject *name, PyObject *path) { /* Only builtin modules are core. */ - if (filename == name) { + if (path == name) { assert(!PyErr_Occurred()); if (PyUnicode_CompareWithASCIIString(name, "sys") == 0) { - return interp->sysdict_copy; + return Py_NewRef(interp->sysdict_copy); } assert(!PyErr_Occurred()); if (PyUnicode_CompareWithASCIIString(name, "builtins") == 0) { - return interp->builtins_copy; + return Py_NewRef(interp->builtins_copy); } assert(!PyErr_Occurred()); } return NULL; } +#ifndef NDEBUG static inline int -is_core_module(PyInterpreterState *interp, PyObject *name, PyObject *filename) +is_core_module(PyInterpreterState *interp, PyObject *name, PyObject *path) { /* This might be called before the core dict copies are in place, so we can't rely on get_core_module_dict() here. */ - if (filename == name) { + if (path == name) { if (PyUnicode_CompareWithASCIIString(name, "sys") == 0) { return 1; } @@ -1158,171 +1616,577 @@ is_core_module(PyInterpreterState *interp, PyObject *name, PyObject *filename) return 0; } -static int -fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename) + +static _Py_ext_module_kind +_get_extension_kind(PyModuleDef *def, bool check_size) { - if (mod == NULL || !PyModule_Check(mod)) { - PyErr_BadInternalCall(); - return -1; + _Py_ext_module_kind kind; + if (def == NULL) { + /* It must be a module created by reload_singlephase_extension() + * from m_copy. Ideally we'd do away with this case. */ + kind = _Py_ext_module_kind_SINGLEPHASE; } - - struct PyModuleDef *def = PyModule_GetDef(mod); - if (!def) { - PyErr_BadInternalCall(); - return -1; + else if (def->m_slots != NULL) { + kind = _Py_ext_module_kind_MULTIPHASE; } - - PyThreadState *tstate = _PyThreadState_GET(); - if (_modules_by_index_set(tstate->interp, def, mod) < 0) { - return -1; + else if (check_size && def->m_size == -1) { + kind = _Py_ext_module_kind_SINGLEPHASE; } + else if (def->m_base.m_init != NULL) { + kind = _Py_ext_module_kind_SINGLEPHASE; + } + else { + // This is probably single-phase init, but a multi-phase + // module *can* have NULL m_slots. + kind = _Py_ext_module_kind_UNKNOWN; + } + return kind; +} + +/* The module might not be fully initialized yet + * and PyModule_FromDefAndSpec() checks m_size + * so we skip m_size. */ +#define assert_multiphase_def(def) \ + do { \ + _Py_ext_module_kind kind = _get_extension_kind(def, false); \ + assert(kind == _Py_ext_module_kind_MULTIPHASE \ + /* m_slots can be NULL. */ \ + || kind == _Py_ext_module_kind_UNKNOWN); \ + } while (0) + +#define assert_singlephase_def(def) \ + do { \ + _Py_ext_module_kind kind = _get_extension_kind(def, true); \ + assert(kind == _Py_ext_module_kind_SINGLEPHASE \ + || kind == _Py_ext_module_kind_UNKNOWN); \ + } while (0) + +#define assert_singlephase(cached) \ + do { \ + _Py_ext_module_kind kind = _get_extension_kind(cached->def, true); \ + assert(kind == _Py_ext_module_kind_SINGLEPHASE); \ + } while (0) + +#else /* defined(NDEBUG) */ +#define assert_multiphase_def(def) +#define assert_singlephase_def(def) +#define assert_singlephase(cached) +#endif - // bpo-44050: Extensions and def->m_base.m_copy can be updated - // when the extension module doesn't support sub-interpreters. - if (def->m_size == -1) { - if (!is_core_module(tstate->interp, name, filename)) { + +struct singlephase_global_update { + PyModInitFunction m_init; + Py_ssize_t m_index; + PyObject *m_dict; + _Py_ext_module_origin origin; + void *md_gil; +}; + +static struct extensions_cache_value * +update_global_state_for_extension(PyThreadState *tstate, + PyObject *path, PyObject *name, + PyModuleDef *def, + struct singlephase_global_update *singlephase) +{ + struct extensions_cache_value *cached = NULL; + PyModInitFunction m_init = NULL; + PyObject *m_dict = NULL; + + /* Set up for _extensions_cache_set(). */ + if (singlephase == NULL) { + assert(def->m_base.m_init == NULL); + assert(def->m_base.m_copy == NULL); + } + else { + if (singlephase->m_init != NULL) { + assert(singlephase->m_dict == NULL); + assert(def->m_base.m_copy == NULL); + assert(def->m_size >= 0); + /* Remember pointer to module init function. */ + // XXX If two modules share a def then def->m_base will + // reflect the last one added (here) to the global cache. + // We should prevent this somehow. The simplest solution + // is probably to store m_copy/m_init in the cache along + // with the def, rather than within the def. + m_init = singlephase->m_init; + } + else if (singlephase->m_dict == NULL) { + /* It must be a core builtin module. */ + assert(is_core_module(tstate->interp, name, path)); + assert(def->m_size == -1); + assert(def->m_base.m_copy == NULL); + assert(def->m_base.m_init == NULL); + } + else { + assert(PyDict_Check(singlephase->m_dict)); + // gh-88216: Extensions and def->m_base.m_copy can be updated + // when the extension module doesn't support sub-interpreters. + assert(def->m_size == -1); + assert(!is_core_module(tstate->interp, name, path)); assert(PyUnicode_CompareWithASCIIString(name, "sys") != 0); assert(PyUnicode_CompareWithASCIIString(name, "builtins") != 0); - if (def->m_base.m_copy) { - /* Somebody already imported the module, - likely under a different name. - XXX this should really not happen. */ - Py_CLEAR(def->m_base.m_copy); - } - PyObject *dict = PyModule_GetDict(mod); - if (dict == NULL) { - return -1; - } - def->m_base.m_copy = PyDict_Copy(dict); - if (def->m_base.m_copy == NULL) { - return -1; - } + m_dict = singlephase->m_dict; } } + /* Add the module's def to the global cache. */ // XXX Why special-case the main interpreter? if (_Py_IsMainInterpreter(tstate->interp) || def->m_size == -1) { - if (_extensions_cache_set(filename, name, def) < 0) { - return -1; +#ifndef NDEBUG + cached = _extensions_cache_get(path, name); + assert(cached == NULL || cached->def == def); +#endif + cached = _extensions_cache_set( + path, name, def, m_init, singlephase->m_index, m_dict, + singlephase->origin, singlephase->md_gil); + if (cached == NULL) { + // XXX Ignore this error? Doing so would effectively + // mark the module as not loadable. + return NULL; } } - return 0; + return cached; } -int -_PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, - PyObject *filename, PyObject *modules) +/* For multi-phase init modules, the module is finished + * by PyModule_FromDefAndSpec(). */ +static int +finish_singlephase_extension(PyThreadState *tstate, PyObject *mod, + struct extensions_cache_value *cached, + PyObject *name, PyObject *modules) { - if (PyObject_SetItem(modules, name, mod) < 0) { + assert(mod != NULL && PyModule_Check(mod)); + assert(cached->def == _PyModule_GetDef(mod)); + + Py_ssize_t index = _get_cached_module_index(cached); + if (_modules_by_index_set(tstate->interp, index, mod) < 0) { return -1; } - if (fix_up_extension(mod, name, filename) < 0) { - PyMapping_DelItem(modules, name); - return -1; + + if (modules != NULL) { + if (PyObject_SetItem(modules, name, mod) < 0) { + return -1; + } } + return 0; } static PyObject * -import_find_extension(PyThreadState *tstate, PyObject *name, - PyObject *filename) +reload_singlephase_extension(PyThreadState *tstate, + struct extensions_cache_value *cached, + struct _Py_ext_module_loader_info *info) { - /* Only single-phase init modules will be in the cache. */ - PyModuleDef *def = _extensions_cache_get(filename, name); - if (def == NULL) { - return NULL; - } + PyModuleDef *def = cached->def; + assert(def != NULL); + assert_singlephase(cached); + PyObject *mod = NULL; /* It may have been successfully imported previously in an interpreter that allows legacy modules but is not allowed in the current interpreter. */ - const char *name_buf = PyUnicode_AsUTF8(name); + const char *name_buf = PyUnicode_AsUTF8(info->name); assert(name_buf != NULL); if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) { return NULL; } - PyObject *mod, *mdict; - PyObject *modules = MODULES(tstate->interp); - + PyObject *modules = get_modules_dict(tstate, true); if (def->m_size == -1) { - PyObject *m_copy = def->m_base.m_copy; /* Module does not support repeated initialization */ + assert(cached->m_init == NULL); + assert(def->m_base.m_init == NULL); + // XXX Copying the cached dict may break interpreter isolation. + // We could solve this by temporarily acquiring the original + // interpreter's GIL. + PyObject *m_copy = get_cached_m_dict(cached, info->name, info->path); if (m_copy == NULL) { - /* It might be a core module (e.g. sys & builtins), - for which we don't set m_copy. */ - m_copy = get_core_module_dict(tstate->interp, name, filename); - if (m_copy == NULL) { - return NULL; - } + assert(!PyErr_Occurred()); + return NULL; } - mod = import_add_module(tstate, name); + mod = import_add_module(tstate, info->name); if (mod == NULL) { + Py_DECREF(m_copy); return NULL; } - mdict = PyModule_GetDict(mod); + PyObject *mdict = PyModule_GetDict(mod); if (mdict == NULL) { + Py_DECREF(m_copy); Py_DECREF(mod); return NULL; } - if (PyDict_Update(mdict, m_copy)) { + int rc = PyDict_Update(mdict, m_copy); + Py_DECREF(m_copy); + if (rc < 0) { Py_DECREF(mod); return NULL; } +#ifdef Py_GIL_DISABLED + if (def->m_base.m_copy != NULL) { + // For non-core modules, fetch the GIL slot that was stored by + // import_run_extension(). + ((PyModuleObject *)mod)->md_gil = cached->md_gil; + } +#endif + /* We can't set mod->md_def if it's missing, + * because _PyImport_ClearModulesByIndex() might break + * due to violating interpreter isolation. + * See the note in set_cached_m_dict(). + * Until that is solved, we leave md_def set to NULL. */ + assert(_PyModule_GetDef(mod) == NULL + || _PyModule_GetDef(mod) == def); } else { - if (def->m_base.m_init == NULL) + assert(cached->m_dict == NULL); + assert(def->m_base.m_copy == NULL); + // XXX Use cached->m_init. + PyModInitFunction p0 = def->m_base.m_init; + if (p0 == NULL) { + assert(!PyErr_Occurred()); return NULL; - mod = def->m_base.m_init(); - if (mod == NULL) + } + struct _Py_ext_module_loader_result res; + if (_PyImport_RunModInitFunc(p0, info, &res) < 0) { + _Py_ext_module_loader_result_apply_error(&res, name_buf); return NULL; - if (PyObject_SetItem(modules, name, mod) == -1) { + } + assert(!PyErr_Occurred()); + assert(res.err == NULL); + assert(res.kind == _Py_ext_module_kind_SINGLEPHASE); + mod = res.module; + /* Tchnically, the init function could return a different module def. + * Then we would probably need to update the global cache. + * However, we don't expect anyone to change the def. */ + assert(res.def == def); + _Py_ext_module_loader_result_clear(&res); + + /* Remember the filename as the __file__ attribute */ + if (info->filename != NULL) { + if (PyModule_AddObjectRef(mod, "__file__", info->filename) < 0) { + PyErr_Clear(); /* Not important enough to report */ + } + } + + if (PyObject_SetItem(modules, info->name, mod) == -1) { Py_DECREF(mod); return NULL; } } - if (_modules_by_index_set(tstate->interp, def, mod) < 0) { - PyMapping_DelItem(modules, name); + + Py_ssize_t index = _get_cached_module_index(cached); + if (_modules_by_index_set(tstate->interp, index, mod) < 0) { + PyMapping_DelItem(modules, info->name); Py_DECREF(mod); return NULL; } + return mod; +} + +static PyObject * +import_find_extension(PyThreadState *tstate, + struct _Py_ext_module_loader_info *info, + struct extensions_cache_value **p_cached) +{ + /* Only single-phase init modules will be in the cache. */ + struct extensions_cache_value *cached + = _extensions_cache_get(info->path, info->name); + if (cached == NULL) { + return NULL; + } + assert(cached->def != NULL); + assert_singlephase(cached); + *p_cached = cached; + + /* It may have been successfully imported previously + in an interpreter that allows legacy modules + but is not allowed in the current interpreter. */ + const char *name_buf = PyUnicode_AsUTF8(info->name); + assert(name_buf != NULL); + if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) { + return NULL; + } + + PyObject *mod = reload_singlephase_extension(tstate, cached, info); + if (mod == NULL) { + return NULL; + } + int verbose = _PyInterpreterState_GetConfig(tstate->interp)->verbose; if (verbose) { PySys_FormatStderr("import %U # previously loaded (%R)\n", - name, filename); + info->name, info->path); } + return mod; } +static PyObject * +import_run_extension(PyThreadState *tstate, PyModInitFunction p0, + struct _Py_ext_module_loader_info *info, + PyObject *spec, PyObject *modules) +{ + /* Core modules go through _PyImport_FixupBuiltin(). */ + assert(!is_core_module(tstate->interp, info->name, info->path)); + + PyObject *mod = NULL; + PyModuleDef *def = NULL; + struct extensions_cache_value *cached = NULL; + const char *name_buf = PyBytes_AS_STRING(info->name_encoded); + + /* We cannot know if the module is single-phase init or + * multi-phase init until after we call its init function. Even + * in isolated interpreters (that do not support single-phase init), + * the init function will run without restriction. For multi-phase + * init modules that isn't a problem because the init function only + * runs PyModuleDef_Init() on the module's def and then returns it. + * + * However, for single-phase init the module's init function will + * create the module, create other objects (and allocate other + * memory), populate it and its module state, and initialize static + * types. Some modules store other objects and data in global C + * variables and register callbacks with the runtime/stdlib or + * even external libraries (which is part of why we can't just + * dlclose() the module in the error case). That's a problem + * for isolated interpreters since all of the above happens + * and only then * will the import fail. Memory will leak, + * callbacks will still get used, and sometimes there + * will be crashes (memory access violations + * and use-after-free). + * + * To put it another way, if the module is single-phase init + * then the import will probably break interpreter isolation + * and should fail ASAP. However, the module's init function + * will still get run. That means it may still store state + * in the shared-object/DLL address space (which never gets + * closed/cleared), including objects (e.g. static types). + * This is a problem for isolated subinterpreters since each + * has its own object allocator. If the loaded shared-object + * still holds a reference to an object after the corresponding + * interpreter has finalized then either we must let it leak + * or else any later use of that object by another interpreter + * (or across multiple init-fini cycles) will crash the process. + * + * To avoid all of that, we make sure the module's init function + * is always run first with the main interpreter active. If it was + * already the main interpreter then we can continue loading the + * module like normal. Otherwise, right after the init function, + * we take care of some import state bookkeeping, switch back + * to the subinterpreter, check for single-phase init, + * and then continue loading like normal. */ + + bool switched = false; + /* We *could* leave in place a legacy interpreter here + * (one that shares obmalloc/GIL with main interp), + * but there isn't a big advantage, we anticipate + * such interpreters will be increasingly uncommon, + * and the code is a bit simpler if we always switch + * to the main interpreter. */ + PyThreadState *main_tstate = switch_to_main_interpreter(tstate); + if (main_tstate == NULL) { + return NULL; + } + else if (main_tstate != tstate) { + switched = true; + /* In the switched case, we could play it safe + * by getting the main interpreter's import lock here. + * It's unlikely to matter though. */ + } + + struct _Py_ext_module_loader_result res; + int rc = _PyImport_RunModInitFunc(p0, info, &res); + if (rc < 0) { + /* We discard res.def. */ + assert(res.module == NULL); + } + else { + assert(!PyErr_Occurred()); + assert(res.err == NULL); + + mod = res.module; + res.module = NULL; + def = res.def; + assert(def != NULL); + + /* Do anything else that should be done + * while still using the main interpreter. */ + if (res.kind == _Py_ext_module_kind_SINGLEPHASE) { + /* Remember the filename as the __file__ attribute */ + if (info->filename != NULL) { + PyObject *filename = NULL; + if (switched) { + // The original filename may be allocated by subinterpreter's + // obmalloc, so we create a copy here. + filename = _PyUnicode_Copy(info->filename); + if (filename == NULL) { + return NULL; + } + } else { + filename = Py_NewRef(info->filename); + } + // XXX There's a refleak somewhere with the filename. + // Until we can track it down, we immortalize it. + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternImmortal(interp, &filename); + + if (PyModule_AddObjectRef(mod, "__file__", filename) < 0) { + PyErr_Clear(); /* Not important enough to report */ + } + } + + /* Update global import state. */ + assert(def->m_base.m_index != 0); + struct singlephase_global_update singlephase = { + // XXX Modules that share a def should each get their own index, + // whereas currently they share (which means the per-interpreter + // cache is less reliable than it should be). + .m_index=def->m_base.m_index, + .origin=info->origin, +#ifdef Py_GIL_DISABLED + .md_gil=((PyModuleObject *)mod)->md_gil, +#endif + }; + // gh-88216: Extensions and def->m_base.m_copy can be updated + // when the extension module doesn't support sub-interpreters. + if (def->m_size == -1) { + /* We will reload from m_copy. */ + assert(def->m_base.m_init == NULL); + singlephase.m_dict = PyModule_GetDict(mod); + assert(singlephase.m_dict != NULL); + } + else { + /* We will reload via the init function. */ + assert(def->m_size >= 0); + assert(def->m_base.m_copy == NULL); + singlephase.m_init = p0; + } + cached = update_global_state_for_extension( + main_tstate, info->path, info->name, def, &singlephase); + if (cached == NULL) { + assert(PyErr_Occurred()); + goto main_finally; + } + } + } + +main_finally: + /* Switch back to the subinterpreter. */ + if (switched) { + assert(main_tstate != tstate); + switch_back_from_main_interpreter(tstate, main_tstate, mod); + /* Any module we got from the init function will have to be + * reloaded in the subinterpreter. */ + mod = NULL; + } + + /*****************************************************************/ + /* At this point we are back to the interpreter we started with. */ + /*****************************************************************/ + + /* Finally we handle the error return from _PyImport_RunModInitFunc(). */ + if (rc < 0) { + _Py_ext_module_loader_result_apply_error(&res, name_buf); + goto error; + } + + if (res.kind == _Py_ext_module_kind_MULTIPHASE) { + assert_multiphase_def(def); + assert(mod == NULL); + /* Note that we cheat a little by not repeating the calls + * to _PyImport_GetModInitFunc() and _PyImport_RunModInitFunc(). */ + mod = PyModule_FromDefAndSpec(def, spec); + if (mod == NULL) { + goto error; + } + } + else { + assert(res.kind == _Py_ext_module_kind_SINGLEPHASE); + assert_singlephase_def(def); + + if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) { + goto error; + } + assert(!PyErr_Occurred()); + + if (switched) { + /* We switched to the main interpreter to run the init + * function, so now we will "reload" the module from the + * cached data using the original subinterpreter. */ + assert(mod == NULL); + mod = reload_singlephase_extension(tstate, cached, info); + if (mod == NULL) { + goto error; + } + assert(!PyErr_Occurred()); + assert(PyModule_Check(mod)); + } + else { + assert(mod != NULL); + assert(PyModule_Check(mod)); + + /* Update per-interpreter import state. */ + PyObject *modules = get_modules_dict(tstate, true); + if (finish_singlephase_extension( + tstate, mod, cached, info->name, modules) < 0) + { + goto error; + } + } + } + + _Py_ext_module_loader_result_clear(&res); + return mod; + +error: + Py_XDECREF(mod); + _Py_ext_module_loader_result_clear(&res); + return NULL; +} + + +// Used in _PyImport_ClearExtension; see notes there. static int clear_singlephase_extension(PyInterpreterState *interp, - PyObject *name, PyObject *filename) + PyObject *name, PyObject *path) { - PyModuleDef *def = _extensions_cache_get(filename, name); - if (def == NULL) { + struct extensions_cache_value *cached = _extensions_cache_get(path, name); + if (cached == NULL) { if (PyErr_Occurred()) { return -1; } return 0; } + PyModuleDef *def = cached->def; /* Clear data set when the module was initially loaded. */ def->m_base.m_init = NULL; Py_CLEAR(def->m_base.m_copy); - // We leave m_index alone since there's no reason to reset it. + def->m_base.m_index = 0; /* Clear the PyState_*Module() cache entry. */ - if (_modules_by_index_check(interp, def->m_base.m_index) == NULL) { - if (_modules_by_index_clear_one(interp, def) < 0) { + Py_ssize_t index = _get_cached_module_index(cached); + if (_modules_by_index_check(interp, index) == NULL) { + if (_modules_by_index_clear_one(interp, index) < 0) { return -1; } } + /* We must use the main interpreter to clean up the cache. + * See the note in import_run_extension(). */ + PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *main_tstate = switch_to_main_interpreter(tstate); + if (main_tstate == NULL) { + return -1; + } + /* Clear the cached module def. */ - _extensions_cache_delete(filename, name); + _extensions_cache_delete(path, name); + + if (main_tstate != tstate) { + switch_back_from_main_interpreter(tstate, main_tstate, NULL); + } return 0; } @@ -1333,21 +2197,62 @@ clear_singlephase_extension(PyInterpreterState *interp, /*******************/ int -_PyImport_FixupBuiltin(PyObject *mod, const char *name, PyObject *modules) +_PyImport_FixupBuiltin(PyThreadState *tstate, PyObject *mod, const char *name, + PyObject *modules) { int res = -1; + assert(mod != NULL && PyModule_Check(mod)); + PyObject *nameobj; nameobj = PyUnicode_InternFromString(name); if (nameobj == NULL) { return -1; } - if (PyObject_SetItem(modules, nameobj, mod) < 0) { + + PyModuleDef *def = PyModule_GetDef(mod); + if (def == NULL) { + PyErr_BadInternalCall(); goto finally; } - if (fix_up_extension(mod, nameobj, nameobj) < 0) { - PyMapping_DelItem(modules, nameobj); + + /* We only use _PyImport_FixupBuiltin() for the core builtin modules + * (sys and builtins). These modules are single-phase init with no + * module state, but we also don't populate def->m_base.m_copy + * for them. */ + assert(is_core_module(tstate->interp, nameobj, nameobj)); + assert_singlephase_def(def); + assert(def->m_size == -1); + assert(def->m_base.m_copy == NULL); + assert(def->m_base.m_index >= 0); + + /* We aren't using import_find_extension() for core modules, + * so we have to do the extra check to make sure the module + * isn't already in the global cache before calling + * update_global_state_for_extension(). */ + struct extensions_cache_value *cached + = _extensions_cache_get(nameobj, nameobj); + if (cached == NULL) { + struct singlephase_global_update singlephase = { + .m_index=def->m_base.m_index, + /* We don't want def->m_base.m_copy populated. */ + .m_dict=NULL, + .origin=_Py_ext_module_origin_CORE, +#ifdef Py_GIL_DISABLED + /* Unused when m_dict == NULL. */ + .md_gil=NULL, +#endif + }; + cached = update_global_state_for_extension( + tstate, nameobj, nameobj, def, &singlephase); + if (cached == NULL) { + goto finally; + } + } + + if (finish_singlephase_extension(tstate, mod, cached, nameobj, modules) < 0) { goto finally; } + res = 0; finally: @@ -1376,45 +2281,76 @@ is_builtin(PyObject *name) static PyObject* create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) { - PyObject *mod = import_find_extension(tstate, name, name); - if (mod || _PyErr_Occurred(tstate)) { - return mod; + struct _Py_ext_module_loader_info info; + if (_Py_ext_module_loader_info_init_for_builtin(&info, name) < 0) { + return NULL; } - PyObject *modules = MODULES(tstate->interp); - for (struct _inittab *p = INITTAB; p->name != NULL; p++) { - if (_PyUnicode_EqualToASCIIString(name, p->name)) { - if (p->initfunc == NULL) { - /* Cannot re-init internal module ("sys" or "builtins") */ - return import_add_module(tstate, name); - } - mod = (*p->initfunc)(); - if (mod == NULL) { - return NULL; - } + struct extensions_cache_value *cached = NULL; + PyObject *mod = import_find_extension(tstate, &info, &cached); + if (mod != NULL) { + assert(!_PyErr_Occurred(tstate)); + assert(cached != NULL); + /* The module might not have md_def set in certain reload cases. */ + assert(_PyModule_GetDef(mod) == NULL + || cached->def == _PyModule_GetDef(mod)); + assert_singlephase(cached); + goto finally; + } + else if (_PyErr_Occurred(tstate)) { + goto finally; + } - if (PyObject_TypeCheck(mod, &PyModuleDef_Type)) { - return PyModule_FromDefAndSpec((PyModuleDef*)mod, spec); - } - else { - /* Remember pointer to module init function. */ - PyModuleDef *def = PyModule_GetDef(mod); - if (def == NULL) { - return NULL; - } + /* If the module was added to the global cache + * but def->m_base.m_copy was cleared (e.g. subinterp fini) + * then we have to do a little dance here. */ + if (cached != NULL) { + assert(cached->def->m_base.m_copy == NULL); + /* For now we clear the cache and move on. */ + _extensions_cache_delete(info.path, info.name); + } - def->m_base.m_init = p->initfunc; - if (_PyImport_FixupExtensionObject(mod, name, name, - modules) < 0) { - return NULL; - } - return mod; - } + struct _inittab *found = NULL; + for (struct _inittab *p = INITTAB; p->name != NULL; p++) { + if (_PyUnicode_EqualToASCIIString(info.name, p->name)) { + found = p; } } + if (found == NULL) { + // not found + mod = Py_NewRef(Py_None); + goto finally; + } - // not found - Py_RETURN_NONE; + PyModInitFunction p0 = (PyModInitFunction)found->initfunc; + if (p0 == NULL) { + /* Cannot re-init internal module ("sys" or "builtins") */ + assert(is_core_module(tstate->interp, info.name, info.path)); + mod = import_add_module(tstate, info.name); + goto finally; + } + +#ifdef Py_GIL_DISABLED + // This call (and the corresponding call to _PyImport_CheckGILForModule()) + // would ideally be inside import_run_extension(). They are kept in the + // callers for now because that would complicate the control flow inside + // import_run_extension(). It should be possible to restructure + // import_run_extension() to address this. + _PyEval_EnableGILTransient(tstate); +#endif + /* Now load it. */ + mod = import_run_extension( + tstate, p0, &info, spec, get_modules_dict(tstate, true)); +#ifdef Py_GIL_DISABLED + if (_PyImport_CheckGILForModule(mod, info.name) < 0) { + Py_CLEAR(mod); + goto finally; + } +#endif + +finally: + _Py_ext_module_loader_info_clear(&info); + return mod; } @@ -1559,23 +2495,9 @@ _PyImport_GetBuiltinModuleNames(void) long PyImport_GetMagicNumber(void) { - long res; - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *external, *pyc_magic; - - external = PyObject_GetAttrString(IMPORTLIB(interp), "_bootstrap_external"); - if (external == NULL) - return -1; - pyc_magic = PyObject_GetAttrString(external, "_RAW_MAGIC_NUMBER"); - Py_DECREF(external); - if (pyc_magic == NULL) - return -1; - res = PyLong_AsLong(pyc_magic); - Py_DECREF(pyc_magic); - return res; + return PYC_MAGIC_NUMBER_TOKEN; } - extern const char * _PySys_ImplCacheTag; const char * @@ -2748,7 +3670,8 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) #undef header import_level++; - t1 = _PyTime_PerfCounterUnchecked(); + // ignore error: don't block import if reading the clock fails + (void)PyTime_PerfCounterRaw(&t1); accumulated = 0; } @@ -2763,7 +3686,9 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) mod != NULL); if (import_time) { - PyTime_t cum = _PyTime_PerfCounterUnchecked() - t1; + PyTime_t t2; + (void)PyTime_PerfCounterRaw(&t2); + PyTime_t cum = t2 - t1; import_level--; fprintf(stderr, "import time: %9ld | %10ld | %*s%s\n", @@ -3164,11 +4089,6 @@ _PyImport_FiniCore(PyInterpreterState *interp) PyErr_FormatUnraisable("Exception ignored on clearing sys.modules"); } - if (IMPORT_LOCK(interp) != NULL) { - PyThread_free_lock(IMPORT_LOCK(interp)); - IMPORT_LOCK(interp) = NULL; - } - _PyImport_ClearCore(interp); } @@ -3301,8 +4221,7 @@ _imp_lock_held_impl(PyObject *module) /*[clinic end generated code: output=8b89384b5e1963fc input=9b088f9b217d9bdf]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); - return PyBool_FromLong( - IMPORT_LOCK_THREAD(interp) != PYTHREAD_INVALID_THREAD_ID); + return PyBool_FromLong(PyMutex_IsLocked(&IMPORT_LOCK(interp).mutex)); } /*[clinic input] @@ -3336,11 +4255,12 @@ _imp_release_lock_impl(PyObject *module) /*[clinic end generated code: output=7faab6d0be178b0a input=934fb11516dd778b]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (_PyImport_ReleaseLock(interp) < 0) { + if (!_PyRecursiveMutex_IsLockedByCurrentThread(&IMPORT_LOCK(interp))) { PyErr_SetString(PyExc_RuntimeError, "not holding the import lock"); return NULL; } + _PyImport_ReleaseLock(interp); Py_RETURN_NONE; } @@ -3514,7 +4434,7 @@ _imp_find_frozen_impl(PyObject *module, PyObject *name, int withdata) if (info.origname != NULL && info.origname[0] != '\0') { origname = PyUnicode_FromString(info.origname); if (origname == NULL) { - Py_DECREF(data); + Py_XDECREF(data); return NULL; } } @@ -3696,9 +4616,16 @@ _imp__override_multi_interp_extensions_check_impl(PyObject *module, "cannot be used in the main interpreter"); return NULL; } +#ifdef Py_GIL_DISABLED + PyErr_SetString(PyExc_RuntimeError, + "_imp._override_multi_interp_extensions_check() " + "cannot be used in the free-threaded build"); + return NULL; +#else int oldvalue = OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp); OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp) = override; return PyLong_FromLong(oldvalue); +#endif } #ifdef HAVE_DYNAMIC_LOADING @@ -3717,44 +4644,88 @@ static PyObject * _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) /*[clinic end generated code: output=83249b827a4fde77 input=c31b954f4cf4e09d]*/ { - PyObject *mod, *name, *path; - FILE *fp; + PyObject *mod = NULL; + PyThreadState *tstate = _PyThreadState_GET(); - name = PyObject_GetAttrString(spec, "name"); - if (name == NULL) { + struct _Py_ext_module_loader_info info; + if (_Py_ext_module_loader_info_init_from_spec(&info, spec) < 0) { return NULL; } - path = PyObject_GetAttrString(spec, "origin"); - if (path == NULL) { - Py_DECREF(name); - return NULL; + struct extensions_cache_value *cached = NULL; + mod = import_find_extension(tstate, &info, &cached); + if (mod != NULL) { + assert(!_PyErr_Occurred(tstate)); + assert(cached != NULL); + /* The module might not have md_def set in certain reload cases. */ + assert(_PyModule_GetDef(mod) == NULL + || cached->def == _PyModule_GetDef(mod)); + assert_singlephase(cached); + goto finally; } + else if (_PyErr_Occurred(tstate)) { + goto finally; + } + /* Otherwise it must be multi-phase init or the first time it's loaded. */ - PyThreadState *tstate = _PyThreadState_GET(); - mod = import_find_extension(tstate, name, path); - if (mod != NULL || _PyErr_Occurred(tstate)) { - assert(mod == NULL || !_PyErr_Occurred(tstate)); + /* If the module was added to the global cache + * but def->m_base.m_copy was cleared (e.g. subinterp fini) + * then we have to do a little dance here. */ + if (cached != NULL) { + assert(cached->def->m_base.m_copy == NULL); + /* For now we clear the cache and move on. */ + _extensions_cache_delete(info.path, info.name); + } + + if (PySys_Audit("import", "OOOOO", info.name, info.filename, + Py_None, Py_None, Py_None) < 0) + { goto finally; } + /* We would move this (and the fclose() below) into + * _PyImport_GetModInitFunc(), but it isn't clear if the intervening + * code relies on fp still being open. */ + FILE *fp; if (file != NULL) { - fp = _Py_fopen_obj(path, "r"); + fp = Py_fopen(info.filename, "r"); if (fp == NULL) { goto finally; } } - else + else { fp = NULL; + } - mod = _PyImport_LoadDynamicModuleWithSpec(spec, fp); + PyModInitFunction p0 = _PyImport_GetModInitFunc(&info, fp); + if (p0 == NULL) { + goto finally; + } - if (fp) +#ifdef Py_GIL_DISABLED + // This call (and the corresponding call to _PyImport_CheckGILForModule()) + // would ideally be inside import_run_extension(). They are kept in the + // callers for now because that would complicate the control flow inside + // import_run_extension(). It should be possible to restructure + // import_run_extension() to address this. + _PyEval_EnableGILTransient(tstate); +#endif + mod = import_run_extension( + tstate, p0, &info, spec, get_modules_dict(tstate, true)); +#ifdef Py_GIL_DISABLED + if (_PyImport_CheckGILForModule(mod, info.name) < 0) { + Py_CLEAR(mod); + goto finally; + } +#endif + + // XXX Shouldn't this happen in the error cases too (i.e. in "finally")? + if (fp) { fclose(fp); + } finally: - Py_DECREF(name); - Py_DECREF(path); + _Py_ext_module_loader_info_clear(&info); return mod; } @@ -3858,6 +4829,12 @@ imp_module_exec(PyObject *module) return -1; } + if (PyModule_AddIntConstant( + module, "pyc_magic_number_token", PYC_MAGIC_NUMBER_TOKEN) < 0) + { + return -1; + } + return 0; } @@ -3865,6 +4842,7 @@ imp_module_exec(PyObject *module) static PyModuleDef_Slot imp_slots[] = { {Py_mod_exec, imp_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Python/importdl.c b/Python/importdl.c index 7dfd301d77efb4..996ca7ed2a3108 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -8,6 +8,8 @@ #include "pycore_pystate.h" #include "pycore_runtime.h" +#include "pycore_importdl.h" + /* ./configure sets HAVE_DYNAMIC_LOADING if dynamic loading of modules is supported on this platform. configure will then compile and link in one of the dynload_*.c files, as appropriate. We will call a function in @@ -15,8 +17,6 @@ */ #ifdef HAVE_DYNAMIC_LOADING -#include "pycore_importdl.h" - #ifdef MS_WINDOWS extern dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, const char *shortname, @@ -28,6 +28,13 @@ extern dl_funcptr _PyImport_FindSharedFuncptr(const char *prefix, const char *pathname, FILE *fp); #endif +#endif /* HAVE_DYNAMIC_LOADING */ + + +/***********************************/ +/* module info to use when loading */ +/***********************************/ + static const char * const ascii_only_prefix = "PyInit"; static const char * const nonascii_prefix = "PyInitU"; @@ -93,59 +100,298 @@ get_encoded_name(PyObject *name, const char **hook_prefix) { return NULL; } -PyObject * -_PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) +void +_Py_ext_module_loader_info_clear(struct _Py_ext_module_loader_info *info) { + Py_CLEAR(info->filename); #ifndef MS_WINDOWS - PyObject *pathbytes = NULL; + Py_CLEAR(info->filename_encoded); #endif - PyObject *name_unicode = NULL, *name = NULL, *path = NULL, *m = NULL; - const char *name_buf, *hook_prefix; - const char *oldcontext, *newcontext; - dl_funcptr exportfunc; - PyModuleDef *def; - PyModInitFunction p0; + Py_CLEAR(info->name); + Py_CLEAR(info->name_encoded); +} - name_unicode = PyObject_GetAttrString(spec, "name"); - if (name_unicode == NULL) { - return NULL; - } - if (!PyUnicode_Check(name_unicode)) { +int +_Py_ext_module_loader_info_init(struct _Py_ext_module_loader_info *p_info, + PyObject *name, PyObject *filename, + _Py_ext_module_origin origin) +{ + struct _Py_ext_module_loader_info info = { + .origin=origin, + }; + + assert(name != NULL); + if (!PyUnicode_Check(name)) { PyErr_SetString(PyExc_TypeError, - "spec.name must be a string"); - goto error; + "module name must be a string"); + _Py_ext_module_loader_info_clear(&info); + return -1; } - newcontext = PyUnicode_AsUTF8(name_unicode); - if (newcontext == NULL) { - goto error; + assert(PyUnicode_GetLength(name) > 0); + info.name = Py_NewRef(name); + + info.name_encoded = get_encoded_name(info.name, &info.hook_prefix); + if (info.name_encoded == NULL) { + _Py_ext_module_loader_info_clear(&info); + return -1; + } + + info.newcontext = PyUnicode_AsUTF8(info.name); + if (info.newcontext == NULL) { + _Py_ext_module_loader_info_clear(&info); + return -1; } - name = get_encoded_name(name_unicode, &hook_prefix); + if (filename != NULL) { + if (!PyUnicode_Check(filename)) { + PyErr_SetString(PyExc_TypeError, + "module filename must be a string"); + _Py_ext_module_loader_info_clear(&info); + return -1; + } + info.filename = Py_NewRef(filename); + +#ifndef MS_WINDOWS + info.filename_encoded = PyUnicode_EncodeFSDefault(info.filename); + if (info.filename_encoded == NULL) { + _Py_ext_module_loader_info_clear(&info); + return -1; + } +#endif + + info.path = info.filename; + } + else { + info.path = info.name; + } + + *p_info = info; + return 0; +} + +int +_Py_ext_module_loader_info_init_for_builtin( + struct _Py_ext_module_loader_info *info, + PyObject *name) +{ + assert(PyUnicode_Check(name)); + assert(PyUnicode_FindChar(name, '.', 0, PyUnicode_GetLength(name), -1) == -1); + assert(PyUnicode_GetLength(name) > 0); + + PyObject *name_encoded = PyUnicode_AsEncodedString(name, "ascii", NULL); + if (name_encoded == NULL) { + return -1; + } + + *info = (struct _Py_ext_module_loader_info){ + .name=Py_NewRef(name), + .name_encoded=name_encoded, + /* We won't need filename. */ + .path=name, + .origin=_Py_ext_module_origin_BUILTIN, + .hook_prefix=ascii_only_prefix, + .newcontext=NULL, + }; + return 0; +} + +int +_Py_ext_module_loader_info_init_for_core( + struct _Py_ext_module_loader_info *info, + PyObject *name) +{ + if (_Py_ext_module_loader_info_init_for_builtin(info, name) < 0) { + return -1; + } + info->origin = _Py_ext_module_origin_CORE; + return 0; +} + +#ifdef HAVE_DYNAMIC_LOADING +int +_Py_ext_module_loader_info_init_from_spec( + struct _Py_ext_module_loader_info *p_info, + PyObject *spec) +{ + PyObject *name = PyObject_GetAttrString(spec, "name"); if (name == NULL) { - goto error; + return -1; + } + PyObject *filename = PyObject_GetAttrString(spec, "origin"); + if (filename == NULL) { + Py_DECREF(name); + return -1; } - name_buf = PyBytes_AS_STRING(name); + /* We could also accommodate builtin modules here without much trouble. */ + _Py_ext_module_origin origin = _Py_ext_module_origin_DYNAMIC; + int err = _Py_ext_module_loader_info_init(p_info, name, filename, origin); + Py_DECREF(name); + Py_DECREF(filename); + return err; +} +#endif /* HAVE_DYNAMIC_LOADING */ - path = PyObject_GetAttrString(spec, "origin"); - if (path == NULL) - goto error; - if (PySys_Audit("import", "OOOOO", name_unicode, path, - Py_None, Py_None, Py_None) < 0) { - goto error; +/********************************/ +/* module init function results */ +/********************************/ + +void +_Py_ext_module_loader_result_clear(struct _Py_ext_module_loader_result *res) +{ + /* Instead, the caller should have called + * _Py_ext_module_loader_result_apply_error(). */ + assert(res->err == NULL); + *res = (struct _Py_ext_module_loader_result){0}; +} + +static void +_Py_ext_module_loader_result_set_error( + struct _Py_ext_module_loader_result *res, + enum _Py_ext_module_loader_result_error_kind kind) +{ +#ifndef NDEBUG + switch (kind) { + case _Py_ext_module_loader_result_EXCEPTION: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_UNREPORTED_EXC: + assert(PyErr_Occurred()); + break; + case _Py_ext_module_loader_result_ERR_MISSING: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_UNINITIALIZED: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_NOT_MODULE: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_MISSING_DEF: + assert(!PyErr_Occurred()); + break; + default: + /* We added a new error kind but forgot to add it to this switch. */ + assert(0); } +#endif + assert(res->err == NULL && res->_err.exc == NULL); + res->err = &res->_err; + *res->err = (struct _Py_ext_module_loader_result_error){ + .kind=kind, + .exc=PyErr_GetRaisedException(), + }; + + /* For some kinds, we also set/check res->kind. */ + switch (kind) { + case _Py_ext_module_loader_result_ERR_UNINITIALIZED: + assert(res->kind == _Py_ext_module_kind_UNKNOWN); + res->kind = _Py_ext_module_kind_INVALID; + break; + /* None of the rest affect the result kind. */ + case _Py_ext_module_loader_result_EXCEPTION: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_MISSING: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_UNREPORTED_EXC: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_NOT_MODULE: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_MISSING_DEF: + break; + default: + /* We added a new error kind but forgot to add it to this switch. */ + assert(0); + } +} + +void +_Py_ext_module_loader_result_apply_error( + struct _Py_ext_module_loader_result *res, + const char *name) +{ + assert(!PyErr_Occurred()); + assert(res->err != NULL && res->err == &res->_err); + struct _Py_ext_module_loader_result_error err = *res->err; + res->err = NULL; + + /* We're otherwise done with the result at this point. */ + _Py_ext_module_loader_result_clear(res); + +#ifndef NDEBUG + switch (err.kind) { + case _Py_ext_module_loader_result_EXCEPTION: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_UNREPORTED_EXC: + assert(err.exc != NULL); + break; + case _Py_ext_module_loader_result_ERR_MISSING: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_UNINITIALIZED: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_NOT_MODULE: _Py_FALLTHROUGH; + case _Py_ext_module_loader_result_ERR_MISSING_DEF: + assert(err.exc == NULL); + break; + default: + /* We added a new error kind but forgot to add it to this switch. */ + assert(0); + } +#endif + + const char *msg = NULL; + switch (err.kind) { + case _Py_ext_module_loader_result_EXCEPTION: + break; + case _Py_ext_module_loader_result_ERR_MISSING: + msg = "initialization of %s failed without raising an exception"; + break; + case _Py_ext_module_loader_result_ERR_UNREPORTED_EXC: + msg = "initialization of %s raised unreported exception"; + break; + case _Py_ext_module_loader_result_ERR_UNINITIALIZED: + msg = "init function of %s returned uninitialized object"; + break; + case _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE: + msg = "initialization of %s did not return PyModuleDef"; + break; + case _Py_ext_module_loader_result_ERR_NOT_MODULE: + msg = "initialization of %s did not return an extension module"; + break; + case _Py_ext_module_loader_result_ERR_MISSING_DEF: + msg = "initialization of %s did not return a valid extension module"; + break; + default: + /* We added a new error kind but forgot to add it to this switch. */ + assert(0); + PyErr_Format(PyExc_SystemError, + "loading %s failed due to init function", name); + return; + } + + if (err.exc != NULL) { + PyErr_SetRaisedException(err.exc); + err.exc = NULL; /* PyErr_SetRaisedException() stole our reference. */ + if (msg != NULL) { + _PyErr_FormatFromCause(PyExc_SystemError, msg, name); + } + } + else { + assert(msg != NULL); + PyErr_Format(PyExc_SystemError, msg, name); + } +} + + +/********************************************/ +/* getting/running the module init function */ +/********************************************/ + +#ifdef HAVE_DYNAMIC_LOADING +PyModInitFunction +_PyImport_GetModInitFunc(struct _Py_ext_module_loader_info *info, + FILE *fp) +{ + const char *name_buf = PyBytes_AS_STRING(info->name_encoded); + dl_funcptr exportfunc; #ifdef MS_WINDOWS - exportfunc = _PyImport_FindSharedFuncptrWindows(hook_prefix, name_buf, - path, fp); + exportfunc = _PyImport_FindSharedFuncptrWindows( + info->hook_prefix, name_buf, info->filename, fp); #else - pathbytes = PyUnicode_EncodeFSDefault(path); - if (pathbytes == NULL) - goto error; - exportfunc = _PyImport_FindSharedFuncptr(hook_prefix, name_buf, - PyBytes_AS_STRING(pathbytes), - fp); - Py_DECREF(pathbytes); + { + const char *path_buf = PyBytes_AS_STRING(info->filename_encoded); + exportfunc = _PyImport_FindSharedFuncptr( + info->hook_prefix, name_buf, path_buf, fp); + } #endif if (exportfunc == NULL) { @@ -154,101 +400,122 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) msg = PyUnicode_FromFormat( "dynamic module does not define " "module export function (%s_%s)", - hook_prefix, name_buf); - if (msg == NULL) - goto error; - PyErr_SetImportError(msg, name_unicode, path); - Py_DECREF(msg); + info->hook_prefix, name_buf); + if (msg != NULL) { + PyErr_SetImportError(msg, info->name, info->filename); + Py_DECREF(msg); + } } - goto error; + return NULL; } - p0 = (PyModInitFunction)exportfunc; + return (PyModInitFunction)exportfunc; +} +#endif /* HAVE_DYNAMIC_LOADING */ + +int +_PyImport_RunModInitFunc(PyModInitFunction p0, + struct _Py_ext_module_loader_info *info, + struct _Py_ext_module_loader_result *p_res) +{ + struct _Py_ext_module_loader_result res = { + .kind=_Py_ext_module_kind_UNKNOWN, + }; + + /* Call the module init function. */ /* Package context is needed for single-phase init */ - oldcontext = _PyImport_SwapPackageContext(newcontext); - m = p0(); + const char *oldcontext = _PyImport_SwapPackageContext(info->newcontext); + PyObject *m = p0(); _PyImport_SwapPackageContext(oldcontext); + /* Validate the result (and populate "res". */ + if (m == NULL) { - if (!PyErr_Occurred()) { - PyErr_Format( - PyExc_SystemError, - "initialization of %s failed without raising an exception", - name_buf); + /* The init func for multi-phase init modules is expected + * to return a PyModuleDef after calling PyModuleDef_Init(). + * That function never raises an exception nor returns NULL, + * so at this point it must be a single-phase init modules. */ + res.kind = _Py_ext_module_kind_SINGLEPHASE; + if (PyErr_Occurred()) { + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_EXCEPTION); + } + else { + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_MISSING); } goto error; } else if (PyErr_Occurred()) { - _PyErr_FormatFromCause( - PyExc_SystemError, - "initialization of %s raised unreported exception", - name_buf); + /* Likewise, we infer that this is a single-phase init module. */ + res.kind = _Py_ext_module_kind_SINGLEPHASE; + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_UNREPORTED_EXC); + /* We would probably be correct to decref m here, + * but we weren't doing so before, + * so we stick with doing nothing. */ m = NULL; goto error; } + if (Py_IS_TYPE(m, NULL)) { /* This can happen when a PyModuleDef is returned without calling * PyModuleDef_Init on it */ - PyErr_Format(PyExc_SystemError, - "init function of %s returned uninitialized object", - name_buf); + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_UNINITIALIZED); + /* Likewise, decref'ing here makes sense. However, the original + * code has a note about "prevent segfault in DECREF", + * so we play it safe and leave it alone. */ m = NULL; /* prevent segfault in DECREF */ goto error; } - if (PyObject_TypeCheck(m, &PyModuleDef_Type)) { - Py_DECREF(name_unicode); - Py_DECREF(name); - Py_DECREF(path); - return PyModule_FromDefAndSpec((PyModuleDef*)m, spec); - } - /* Fall back to single-phase init mechanism */ - - if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) { - goto error; + if (PyObject_TypeCheck(m, &PyModuleDef_Type)) { + /* multi-phase init */ + res.kind = _Py_ext_module_kind_MULTIPHASE; + res.def = (PyModuleDef *)m; + /* Run PyModule_FromDefAndSpec() to finish loading the module. */ } - - if (hook_prefix == nonascii_prefix) { - /* don't allow legacy init for non-ASCII module names */ - PyErr_Format( - PyExc_SystemError, - "initialization of %s did not return PyModuleDef", - name_buf); + else if (info->hook_prefix == nonascii_prefix) { + /* Non-ASCII is only supported for multi-phase init. */ + res.kind = _Py_ext_module_kind_MULTIPHASE; + /* Don't allow legacy init for non-ASCII module names. */ + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE); goto error; } + else { + /* single-phase init (legacy) */ + res.kind = _Py_ext_module_kind_SINGLEPHASE; + res.module = m; - /* Remember pointer to module init function. */ - def = PyModule_GetDef(m); - if (def == NULL) { - PyErr_Format(PyExc_SystemError, - "initialization of %s did not return an extension " - "module", name_buf); - goto error; - } - def->m_base.m_init = p0; + if (!PyModule_Check(m)) { + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_NOT_MODULE); + goto error; + } - /* Remember the filename as the __file__ attribute */ - if (PyModule_AddObjectRef(m, "__file__", path) < 0) { - PyErr_Clear(); /* Not important enough to report */ + res.def = _PyModule_GetDef(m); + if (res.def == NULL) { + PyErr_Clear(); + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_MISSING_DEF); + goto error; + } } - PyObject *modules = PyImport_GetModuleDict(); - if (_PyImport_FixupExtensionObject(m, name_unicode, path, modules) < 0) - goto error; - - Py_DECREF(name_unicode); - Py_DECREF(name); - Py_DECREF(path); - - return m; + assert(!PyErr_Occurred()); + assert(res.err == NULL); + *p_res = res; + return 0; error: - Py_DECREF(name_unicode); - Py_XDECREF(name); - Py_XDECREF(path); - Py_XDECREF(m); - return NULL; + assert(!PyErr_Occurred()); + assert(res.err != NULL); + Py_CLEAR(res.module); + res.def = NULL; + *p_res = res; + p_res->err = &p_res->_err; + return -1; } - -#endif /* HAVE_DYNAMIC_LOADING */ diff --git a/Python/index_pool.c b/Python/index_pool.c new file mode 100644 index 00000000000000..526eccff74af00 --- /dev/null +++ b/Python/index_pool.c @@ -0,0 +1,193 @@ +#include + +#include "Python.h" + +#include "pycore_index_pool.h" +#include "pycore_lock.h" + +#ifdef Py_GIL_DISABLED + +static inline void +swap(int32_t *values, Py_ssize_t i, Py_ssize_t j) +{ + int32_t tmp = values[i]; + values[i] = values[j]; + values[j] = tmp; +} + +static bool +heap_try_swap(_PyIndexHeap *heap, Py_ssize_t i, Py_ssize_t j) +{ + if (i < 0 || i >= heap->size) { + return 0; + } + if (j < 0 || j >= heap->size) { + return 0; + } + if (i <= j) { + if (heap->values[i] <= heap->values[j]) { + return 0; + } + } + else if (heap->values[j] <= heap->values[i]) { + return 0; + } + swap(heap->values, i, j); + return 1; +} + +static inline Py_ssize_t +parent(Py_ssize_t i) +{ + return (i - 1) / 2; +} + +static inline Py_ssize_t +left_child(Py_ssize_t i) +{ + return 2 * i + 1; +} + +static inline Py_ssize_t +right_child(Py_ssize_t i) +{ + return 2 * i + 2; +} + +static void +heap_add(_PyIndexHeap *heap, int32_t val) +{ + assert(heap->size < heap->capacity); + // Add val to end + heap->values[heap->size] = val; + heap->size++; + // Sift up + for (Py_ssize_t cur = heap->size - 1; cur > 0; cur = parent(cur)) { + if (!heap_try_swap(heap, cur, parent(cur))) { + break; + } + } +} + +static Py_ssize_t +heap_min_child(_PyIndexHeap *heap, Py_ssize_t i) +{ + if (left_child(i) < heap->size) { + if (right_child(i) < heap->size) { + Py_ssize_t lval = heap->values[left_child(i)]; + Py_ssize_t rval = heap->values[right_child(i)]; + return lval < rval ? left_child(i) : right_child(i); + } + return left_child(i); + } + else if (right_child(i) < heap->size) { + return right_child(i); + } + return -1; +} + +static int32_t +heap_pop(_PyIndexHeap *heap) +{ + assert(heap->size > 0); + // Pop smallest and replace with the last element + int32_t result = heap->values[0]; + heap->values[0] = heap->values[heap->size - 1]; + heap->size--; + // Sift down + for (Py_ssize_t cur = 0; cur < heap->size;) { + Py_ssize_t min_child = heap_min_child(heap, cur); + if (min_child > -1 && heap_try_swap(heap, cur, min_child)) { + cur = min_child; + } + else { + break; + } + } + return result; +} + +static int +heap_ensure_capacity(_PyIndexHeap *heap, Py_ssize_t limit) +{ + assert(limit > 0); + if (heap->capacity > limit) { + return 0; + } + Py_ssize_t new_capacity = heap->capacity ? heap->capacity : 1024; + while (new_capacity && new_capacity < limit) { + new_capacity <<= 1; + } + if (!new_capacity) { + return -1; + } + int32_t *new_values = PyMem_RawCalloc(new_capacity, sizeof(int32_t)); + if (new_values == NULL) { + return -1; + } + if (heap->values != NULL) { + memcpy(new_values, heap->values, heap->capacity); + PyMem_RawFree(heap->values); + } + heap->values = new_values; + heap->capacity = new_capacity; + return 0; +} + +static void +heap_fini(_PyIndexHeap *heap) +{ + if (heap->values != NULL) { + PyMem_RawFree(heap->values); + heap->values = NULL; + } + heap->size = -1; + heap->capacity = -1; +} + +#define LOCK_POOL(pool) PyMutex_LockFlags(&pool->mutex, _Py_LOCK_DONT_DETACH) +#define UNLOCK_POOL(pool) PyMutex_Unlock(&pool->mutex) + +int32_t +_PyIndexPool_AllocIndex(_PyIndexPool *pool) +{ + LOCK_POOL(pool); + int32_t index; + _PyIndexHeap *free_indices = &pool->free_indices; + if (free_indices->size == 0) { + // No free indices. Make sure the heap can always store all of the + // indices that have been allocated to avoid having to allocate memory + // (which can fail) when freeing an index. Freeing indices happens when + // threads are being destroyed, which makes error handling awkward / + // impossible. This arrangement shifts handling of allocation failures + // to when indices are allocated, which happens at thread creation, + // where we are better equipped to deal with failure. + if (heap_ensure_capacity(free_indices, pool->next_index + 1) < 0) { + UNLOCK_POOL(pool); + PyErr_NoMemory(); + return -1; + } + index = pool->next_index++; + } + else { + index = heap_pop(free_indices); + } + UNLOCK_POOL(pool); + return index; +} + +void +_PyIndexPool_FreeIndex(_PyIndexPool *pool, int32_t index) +{ + LOCK_POOL(pool); + heap_add(&pool->free_indices, index); + UNLOCK_POOL(pool); +} + +void +_PyIndexPool_Fini(_PyIndexPool *pool) +{ + heap_fini(&pool->free_indices); +} + +#endif // Py_GIL_DISABLED diff --git a/Python/initconfig.c b/Python/initconfig.c index 74f28f3b39175b..4db77ef47d2362 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -10,6 +10,7 @@ #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_pystats.h" // _Py_StatsOn() +#include "pycore_sysmodule.h" // _PySys_SetIntMaxStrDigits() #include "osdefs.h" // DELIM @@ -24,107 +25,210 @@ # endif #endif +#include "config_common.h" + +/* --- PyConfig setters ------------------------------------------- */ + +typedef PyObject* (*config_sys_flag_setter) (int value); + +static PyObject* +config_sys_flag_long(int value) +{ + return PyLong_FromLong(value); +} + +static PyObject* +config_sys_flag_not(int value) +{ + value = (!value); + return config_sys_flag_long(value); +} + /* --- PyConfig spec ---------------------------------------------- */ typedef enum { PyConfig_MEMBER_INT = 0, PyConfig_MEMBER_UINT = 1, PyConfig_MEMBER_ULONG = 2, + PyConfig_MEMBER_BOOL = 3, PyConfig_MEMBER_WSTR = 10, PyConfig_MEMBER_WSTR_OPT = 11, PyConfig_MEMBER_WSTR_LIST = 12, } PyConfigMemberType; +typedef enum { + // Option which cannot be get or set by PyConfig_Get() and PyConfig_Set() + PyConfig_MEMBER_INIT_ONLY = 0, + + // Option which cannot be set by PyConfig_Set() + PyConfig_MEMBER_READ_ONLY = 1, + + // Public option: can be get and set by PyConfig_Get() and PyConfig_Set() + PyConfig_MEMBER_PUBLIC = 2, +} PyConfigMemberVisibility; + +typedef struct { + const char *attr; + int flag_index; + config_sys_flag_setter flag_setter; +} PyConfigSysSpec; + typedef struct { const char *name; size_t offset; PyConfigMemberType type; + PyConfigMemberVisibility visibility; + PyConfigSysSpec sys; } PyConfigSpec; -#define SPEC(MEMBER, TYPE) \ - {#MEMBER, offsetof(PyConfig, MEMBER), PyConfig_MEMBER_##TYPE} +#define SPEC(MEMBER, TYPE, VISIBILITY, sys) \ + {#MEMBER, offsetof(PyConfig, MEMBER), \ + PyConfig_MEMBER_##TYPE, PyConfig_MEMBER_##VISIBILITY, sys} + +#define SYS_ATTR(name) {name, -1, NULL} +#define SYS_FLAG_SETTER(index, setter) {NULL, index, setter} +#define SYS_FLAG(index) SYS_FLAG_SETTER(index, NULL) +#define NO_SYS SYS_ATTR(NULL) +// Update _test_embed_set_config when adding new members static const PyConfigSpec PYCONFIG_SPEC[] = { - SPEC(_config_init, UINT), - SPEC(isolated, UINT), - SPEC(use_environment, UINT), - SPEC(dev_mode, UINT), - SPEC(install_signal_handlers, UINT), - SPEC(use_hash_seed, UINT), - SPEC(hash_seed, ULONG), - SPEC(faulthandler, UINT), - SPEC(tracemalloc, UINT), - SPEC(perf_profiling, UINT), - SPEC(import_time, UINT), - SPEC(code_debug_ranges, UINT), - SPEC(show_ref_count, UINT), - SPEC(dump_refs, UINT), - SPEC(dump_refs_file, WSTR_OPT), - SPEC(malloc_stats, UINT), - SPEC(filesystem_encoding, WSTR), - SPEC(filesystem_errors, WSTR), - SPEC(pycache_prefix, WSTR_OPT), - SPEC(parse_argv, UINT), - SPEC(orig_argv, WSTR_LIST), - SPEC(argv, WSTR_LIST), - SPEC(xoptions, WSTR_LIST), - SPEC(warnoptions, WSTR_LIST), - SPEC(site_import, UINT), - SPEC(bytes_warning, UINT), - SPEC(warn_default_encoding, UINT), - SPEC(inspect, UINT), - SPEC(interactive, UINT), - SPEC(optimization_level, UINT), - SPEC(parser_debug, UINT), - SPEC(write_bytecode, UINT), - SPEC(verbose, UINT), - SPEC(quiet, UINT), - SPEC(user_site_directory, UINT), - SPEC(configure_c_stdio, UINT), - SPEC(buffered_stdio, UINT), - SPEC(stdio_encoding, WSTR), - SPEC(stdio_errors, WSTR), -#ifdef MS_WINDOWS - SPEC(legacy_windows_stdio, UINT), -#endif - SPEC(check_hash_pycs_mode, WSTR), - SPEC(use_frozen_modules, UINT), - SPEC(safe_path, UINT), - SPEC(int_max_str_digits, INT), - SPEC(cpu_count, INT), - SPEC(pathconfig_warnings, UINT), - SPEC(program_name, WSTR), - SPEC(pythonpath_env, WSTR_OPT), - SPEC(home, WSTR_OPT), - SPEC(platlibdir, WSTR), - SPEC(sys_path_0, WSTR_OPT), - SPEC(module_search_paths_set, UINT), - SPEC(module_search_paths, WSTR_LIST), - SPEC(stdlib_dir, WSTR_OPT), - SPEC(executable, WSTR_OPT), - SPEC(base_executable, WSTR_OPT), - SPEC(prefix, WSTR_OPT), - SPEC(base_prefix, WSTR_OPT), - SPEC(exec_prefix, WSTR_OPT), - SPEC(base_exec_prefix, WSTR_OPT), - SPEC(skip_source_first_line, UINT), - SPEC(run_command, WSTR_OPT), - SPEC(run_module, WSTR_OPT), - SPEC(run_filename, WSTR_OPT), - SPEC(_install_importlib, UINT), - SPEC(_init_main, UINT), - SPEC(_is_python_build, UINT), + // --- Public options ----------- + + SPEC(argv, WSTR_LIST, PUBLIC, SYS_ATTR("argv")), + SPEC(base_exec_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("base_exec_prefix")), + SPEC(base_executable, WSTR_OPT, PUBLIC, SYS_ATTR("_base_executable")), + SPEC(base_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("base_prefix")), + SPEC(bytes_warning, UINT, PUBLIC, SYS_FLAG(9)), + SPEC(exec_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("exec_prefix")), + SPEC(executable, WSTR_OPT, PUBLIC, SYS_ATTR("executable")), + SPEC(inspect, BOOL, PUBLIC, SYS_FLAG(1)), + SPEC(int_max_str_digits, UINT, PUBLIC, NO_SYS), + SPEC(interactive, BOOL, PUBLIC, SYS_FLAG(2)), + SPEC(module_search_paths, WSTR_LIST, PUBLIC, SYS_ATTR("path")), + SPEC(optimization_level, UINT, PUBLIC, SYS_FLAG(3)), + SPEC(parser_debug, BOOL, PUBLIC, SYS_FLAG(0)), + SPEC(platlibdir, WSTR, PUBLIC, SYS_ATTR("platlibdir")), + SPEC(prefix, WSTR_OPT, PUBLIC, SYS_ATTR("prefix")), + SPEC(pycache_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("pycache_prefix")), + SPEC(quiet, BOOL, PUBLIC, SYS_FLAG(10)), + SPEC(stdlib_dir, WSTR_OPT, PUBLIC, SYS_ATTR("_stdlib_dir")), + SPEC(use_environment, BOOL, PUBLIC, SYS_FLAG_SETTER(7, config_sys_flag_not)), + SPEC(verbose, UINT, PUBLIC, SYS_FLAG(8)), + SPEC(warnoptions, WSTR_LIST, PUBLIC, SYS_ATTR("warnoptions")), + SPEC(write_bytecode, BOOL, PUBLIC, SYS_FLAG_SETTER(4, config_sys_flag_not)), + SPEC(xoptions, WSTR_LIST, PUBLIC, SYS_ATTR("_xoptions")), + + // --- Read-only options ----------- + #ifdef Py_STATS - SPEC(_pystats, UINT), + SPEC(_pystats, BOOL, READ_ONLY, NO_SYS), +#endif + SPEC(buffered_stdio, BOOL, READ_ONLY, NO_SYS), + SPEC(check_hash_pycs_mode, WSTR, READ_ONLY, NO_SYS), + SPEC(code_debug_ranges, BOOL, READ_ONLY, NO_SYS), + SPEC(configure_c_stdio, BOOL, READ_ONLY, NO_SYS), + SPEC(cpu_count, INT, READ_ONLY, NO_SYS), + SPEC(dev_mode, BOOL, READ_ONLY, NO_SYS), // sys.flags.dev_mode + SPEC(dump_refs, BOOL, READ_ONLY, NO_SYS), + SPEC(dump_refs_file, WSTR_OPT, READ_ONLY, NO_SYS), +#ifdef Py_GIL_DISABLED + SPEC(enable_gil, INT, READ_ONLY, NO_SYS), + SPEC(tlbc_enabled, INT, READ_ONLY, NO_SYS), +#endif + SPEC(faulthandler, BOOL, READ_ONLY, NO_SYS), + SPEC(filesystem_encoding, WSTR, READ_ONLY, NO_SYS), + SPEC(filesystem_errors, WSTR, READ_ONLY, NO_SYS), + SPEC(hash_seed, ULONG, READ_ONLY, NO_SYS), + SPEC(home, WSTR_OPT, READ_ONLY, NO_SYS), + SPEC(import_time, BOOL, READ_ONLY, NO_SYS), + SPEC(install_signal_handlers, BOOL, READ_ONLY, NO_SYS), + SPEC(isolated, BOOL, READ_ONLY, NO_SYS), // sys.flags.isolated +#ifdef MS_WINDOWS + SPEC(legacy_windows_stdio, BOOL, READ_ONLY, NO_SYS), #endif + SPEC(malloc_stats, BOOL, READ_ONLY, NO_SYS), + SPEC(orig_argv, WSTR_LIST, READ_ONLY, SYS_ATTR("orig_argv")), + SPEC(parse_argv, BOOL, READ_ONLY, NO_SYS), + SPEC(pathconfig_warnings, BOOL, READ_ONLY, NO_SYS), + SPEC(perf_profiling, UINT, READ_ONLY, NO_SYS), + SPEC(program_name, WSTR, READ_ONLY, NO_SYS), + SPEC(run_command, WSTR_OPT, READ_ONLY, NO_SYS), + SPEC(run_filename, WSTR_OPT, READ_ONLY, NO_SYS), + SPEC(run_module, WSTR_OPT, READ_ONLY, NO_SYS), #ifdef Py_DEBUG - SPEC(run_presite, WSTR_OPT), + SPEC(run_presite, WSTR_OPT, READ_ONLY, NO_SYS), +#endif + SPEC(safe_path, BOOL, READ_ONLY, NO_SYS), + SPEC(show_ref_count, BOOL, READ_ONLY, NO_SYS), + SPEC(site_import, BOOL, READ_ONLY, NO_SYS), // sys.flags.no_site + SPEC(skip_source_first_line, BOOL, READ_ONLY, NO_SYS), + SPEC(stdio_encoding, WSTR, READ_ONLY, NO_SYS), + SPEC(stdio_errors, WSTR, READ_ONLY, NO_SYS), + SPEC(tracemalloc, UINT, READ_ONLY, NO_SYS), + SPEC(use_frozen_modules, BOOL, READ_ONLY, NO_SYS), + SPEC(use_hash_seed, BOOL, READ_ONLY, NO_SYS), +#ifdef __APPLE__ + SPEC(use_system_logger, BOOL, READ_ONLY, NO_SYS), +#endif + SPEC(user_site_directory, BOOL, READ_ONLY, NO_SYS), // sys.flags.no_user_site + SPEC(warn_default_encoding, BOOL, READ_ONLY, NO_SYS), + + // --- Init-only options ----------- + + SPEC(_config_init, UINT, INIT_ONLY, NO_SYS), + SPEC(_init_main, BOOL, INIT_ONLY, NO_SYS), + SPEC(_install_importlib, BOOL, INIT_ONLY, NO_SYS), + SPEC(_is_python_build, BOOL, INIT_ONLY, NO_SYS), + SPEC(module_search_paths_set, BOOL, INIT_ONLY, NO_SYS), + SPEC(pythonpath_env, WSTR_OPT, INIT_ONLY, NO_SYS), + SPEC(sys_path_0, WSTR_OPT, INIT_ONLY, NO_SYS), + + // Array terminator + {NULL, 0, 0, 0, NO_SYS}, +}; + +#undef SPEC +#define SPEC(MEMBER, TYPE, VISIBILITY) \ + {#MEMBER, offsetof(PyPreConfig, MEMBER), PyConfig_MEMBER_##TYPE, \ + PyConfig_MEMBER_##VISIBILITY, NO_SYS} + +static const PyConfigSpec PYPRECONFIG_SPEC[] = { + // --- Read-only options ----------- + + SPEC(allocator, INT, READ_ONLY), + SPEC(coerce_c_locale, BOOL, READ_ONLY), + SPEC(coerce_c_locale_warn, BOOL, READ_ONLY), + SPEC(configure_locale, BOOL, READ_ONLY), +#ifdef MS_WINDOWS + SPEC(legacy_windows_fs_encoding, BOOL, READ_ONLY), #endif - {NULL, 0, 0}, + SPEC(utf8_mode, BOOL, READ_ONLY), + + // --- Init-only options ----------- + // Members already present in PYCONFIG_SPEC + + SPEC(_config_init, INT, INIT_ONLY), + SPEC(dev_mode, BOOL, INIT_ONLY), + SPEC(isolated, BOOL, INIT_ONLY), + SPEC(parse_argv, BOOL, INIT_ONLY), + SPEC(use_environment, BOOL, INIT_ONLY), + + // Array terminator + {NULL, 0, 0, 0, NO_SYS}, }; #undef SPEC +#undef SYS_ATTR +#undef SYS_FLAG_SETTER +#undef SYS_FLAG +#undef NO_SYS + + +// Forward declarations +static PyObject* +config_get(const PyConfig *config, const PyConfigSpec *spec, + int use_sys); /* --- Command line options --------------------------------------- */ @@ -137,8 +241,8 @@ static const char usage_line[] = /* Lines sorted by option name; keep in sync with usage_envvars* below */ static const char usage_help[] = "\ Options (and corresponding environment variables):\n\ --b : issue warnings about str(bytes_instance), str(bytearray_instance)\n\ - and comparing bytes/bytearray with str. (-bb: issue errors)\n\ +-b : issue warnings about converting bytes/bytearray to str and comparing\n\ + bytes/bytearray with str or bytes with int. (-bb: issue errors)\n\ -B : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x\n\ -c cmd : program passed in as string (terminates option list)\n\ -d : turn on parser debugging output (for experts only, only works on\n\ @@ -156,7 +260,7 @@ Options (and corresponding environment variables):\n\ -P : don't prepend a potentially unsafe path to sys.path; also\n\ PYTHONSAFEPATH\n\ -q : don't print version and copyright messages on interactive startup\n\ --s : don't add user site directory to sys.path; also PYTHONNOUSERSITE\n\ +-s : don't add user site directory to sys.path; also PYTHONNOUSERSITE=x\n\ -S : don't imply 'import site' on initialization\n\ -u : force the stdout and stderr streams to be unbuffered;\n\ this option has no effect on stdin; also PYTHONUNBUFFERED=x\n\ @@ -182,64 +286,51 @@ arg ...: arguments passed to program in sys.argv[1:]\n\ static const char usage_xoptions[] = "\ The following implementation-specific options are available:\n\ --X faulthandler: enable faulthandler\n\ --X showrefcount: output the total reference count and number of used\n\ - memory blocks when the program finishes or after each statement in\n\ - the interactive interpreter. This only works on debug builds\n\ --X tracemalloc: start tracing Python memory allocations using the\n\ - tracemalloc module. By default, only the most recent frame is stored\n\ - in a traceback of a trace. Use -X tracemalloc=NFRAME to start\n\ - tracing with a traceback limit of NFRAME frames\n\ --X importtime: show how long each import takes. It shows module name,\n\ - cumulative time (including nested imports) and self time (excluding\n\ - nested imports). Note that its output may be broken in\n\ - multi-threaded application.\n\ - Typical usage is python3 -X importtime -c 'import asyncio'\n\ --X dev : enable CPython's \"development mode\", introducing additional runtime\n\ - checks which are too expensive to be enabled by default. Effect of\n\ - the developer mode:\n\ - * Add default warning filter, as -W default\n\ - * Install debug hooks on memory allocators: see the\n\ - PyMem_SetupDebugHooks() C function\n\ - * Enable the faulthandler module to dump the Python traceback on\n\ - a crash\n\ - * Enable asyncio debug mode\n\ - * Set the dev_mode attribute of sys.flags to True\n\ - * io.IOBase destructor logs close() exceptions\n\ --X utf8: enable UTF-8 mode for operating system interfaces, overriding the\n\ - default locale-aware mode. -X utf8=0 explicitly disables UTF-8 mode\n\ - (even when it would otherwise activate automatically)\n\ --X pycache_prefix=PATH: enable writing .pyc files to a parallel tree rooted\n\ - at the given directory instead of to the code tree\n\ --X warn_default_encoding: enable opt-in EncodingWarning for 'encoding=None'\n\ --X no_debug_ranges: disable the inclusion of the tables mapping extra location\n\ - information (end line, start column offset and end column offset) to\n\ - every instruction in code objects. This is useful when smaller code\n\ - objects and pyc files are desired as well as suppressing the extra\n\ - visual location indicators when the interpreter displays tracebacks.\n\ --X perf: activate support for the Linux \"perf\" profiler by activating the\n\ - \"perf\" trampoline. When this option is activated, the Linux \"perf\"\n\ - profiler will be able to report Python calls. This option is only\n\ - available on some platforms and will do nothing if is not supported\n\ - on the current system. The default value is \"off\".\n\ --X frozen_modules=[on|off]: whether or not frozen modules should be used.\n\ - The default is \"on\" (or \"off\" if you are running a local build).\n\ --X int_max_str_digits=number: limit the size of int<->str conversions.\n\ - This helps avoid denial of service attacks when parsing untrusted\n\ - data. The default is sys.int_info.default_max_str_digits.\n\ - 0 disables.\n\ --X cpu_count=[n|default]: Override the return value of os.cpu_count(),\n\ - os.process_cpu_count(), and multiprocessing.cpu_count(). This can\n\ - help users who need to limit resources in a container." -#ifdef Py_STATS -"\n\ --X pystats: Enable pystats collection at startup." +-X cpu_count=N: override the return value of os.cpu_count();\n\ + -X cpu_count=default cancels overriding; also PYTHON_CPU_COUNT\n\ +-X dev : enable Python Development Mode; also PYTHONDEVMODE\n\ +-X faulthandler: dump the Python traceback on fatal errors;\n\ + also PYTHONFAULTHANDLER\n\ +-X frozen_modules=[on|off]: whether to use frozen modules; the default is \"on\"\n\ + for installed Python and \"off\" for a local build;\n\ + also PYTHON_FROZEN_MODULES\n\ +" +#ifdef Py_GIL_DISABLED +"-X gil=[0|1]: enable (1) or disable (0) the GIL; also PYTHON_GIL\n" #endif +"\ +-X importtime: show how long each import takes; also PYTHONPROFILEIMPORTTIME\n\ +-X int_max_str_digits=N: limit the size of int<->str conversions;\n\ + 0 disables the limit; also PYTHONINTMAXSTRDIGITS\n\ +-X no_debug_ranges: don't include extra location information in code objects;\n\ + also PYTHONNODEBUGRANGES\n\ +-X perf: support the Linux \"perf\" profiler; also PYTHONPERFSUPPORT=1\n\ +" #ifdef Py_DEBUG -"\n\ --X presite=package.module: import this module before site.py is run." +"-X presite=MOD: import this module before site; also PYTHON_PRESITE\n" #endif -; +"\ +-X pycache_prefix=PATH: write .pyc files to a parallel tree instead of to the\n\ + code tree; also PYTHONPYCACHEPREFIX\n\ +" +#ifdef Py_STATS +"-X pystats: enable pystats collection at startup; also PYTHONSTATS\n" +#endif +"\ +-X showrefcount: output the total reference count and number of used\n\ + memory blocks when the program finishes or after each statement in\n\ + the interactive interpreter; only works on debug builds\n" +#ifdef Py_GIL_DISABLED +"-X tlbc=[0|1]: enable (1) or disable (0) thread-local bytecode. Also\n\ + PYTHON_TLBC\n" +#endif +"\ +-X tracemalloc[=N]: trace Python memory allocations; N sets a traceback limit\n \ + of N frames (default: 1); also PYTHONTRACEMALLOC=N\n\ +-X utf8[=0|1]: enable (1) or disable (0) UTF-8 mode; also PYTHONUTF8\n\ +-X warn_default_encoding: enable opt-in EncodingWarning for 'encoding=None';\n\ + also PYTHONWARNDEFAULTENCODING\ +"; /* Envvars that don't have equivalent command-line options are listed first */ static const char usage_envvars[] = @@ -249,23 +340,17 @@ static const char usage_envvars[] = " default module search path. The result is sys.path.\n" "PYTHONHOME : alternate directory (or %lc).\n" " The default module search path uses %s.\n" -"PYTHONPLATLIBDIR: override sys.platlibdir.\n" -"PYTHONCASEOK : ignore case in 'import' statements (Windows).\n" -"PYTHONUTF8 : if set to 1, enable the UTF-8 mode.\n" -"PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n" -"PYTHONFAULTHANDLER: dump the Python traceback on fatal errors.\n" +"PYTHONPLATLIBDIR: override sys.platlibdir\n" +"PYTHONCASEOK : ignore case in 'import' statements (Windows)\n" +"PYTHONIOENCODING: encoding[:errors] used for stdin/stdout/stderr\n" "PYTHONHASHSEED : if this variable is set to 'random', a random value is used\n" " to seed the hashes of str and bytes objects. It can also be\n" " set to an integer in the range [0,4294967295] to get hash\n" " values with a predictable seed.\n" -"PYTHONINTMAXSTRDIGITS: limits the maximum digit characters in an int value\n" -" when converting from a string and when converting an int\n" -" back to a str. A value of 0 disables the limit.\n" -" Conversions to or from bases 2, 4, 8, 16, and 32 are never\n" -" limited.\n" "PYTHONMALLOC : set the Python memory allocators and/or install debug hooks\n" " on Python memory allocators. Use PYTHONMALLOC=debug to\n" " install debug hooks.\n" +"PYTHONMALLOCSTATS: print memory allocator statistics\n" "PYTHONCOERCECLOCALE: if this variable is set to 0, it disables the locale\n" " coercion behavior. Use PYTHONCOERCECLOCALE=warn to request\n" " display of locale coercion and locale compatibility warnings\n" @@ -273,45 +358,67 @@ static const char usage_envvars[] = "PYTHONBREAKPOINT: if this variable is set to 0, it disables the default\n" " debugger. It can be set to the callable of your debugger of\n" " choice.\n" -"PYTHON_CPU_COUNT: Overrides the return value of os.process_cpu_count(),\n" -" os.cpu_count(), and multiprocessing.cpu_count() if set to\n" -" a positive integer.\n" -"PYTHONDEVMODE : enable the development mode.\n" -"PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files.\n" -"PYTHONWARNDEFAULTENCODING: enable opt-in EncodingWarning for 'encoding=None'.\n" -"PYTHONNODEBUGRANGES: if this variable is set, it disables the inclusion of\n" -" the tables mapping extra location information (end line,\n" -" start column offset and end column offset) to every\n" -" instruction in code objects. This is useful when smaller\n" -" code objects and pyc files are desired as well as\n" -" suppressing the extra visual location indicators when the\n" -" interpreter displays tracebacks.\n" -"PYTHON_FROZEN_MODULES: if this variable is set, it determines whether or not\n" -" frozen modules should be used. The default is \"on\" (or\n" -" \"off\" if you are running a local build).\n" "PYTHON_COLORS : if this variable is set to 1, the interpreter will colorize\n" " various kinds of output. Setting it to 0 deactivates\n" " this behavior.\n" "PYTHON_HISTORY : the location of a .python_history file.\n" +"PYTHONASYNCIODEBUG: enable asyncio debug mode\n" +#ifdef Py_TRACE_REFS +"PYTHONDUMPREFS : dump objects and reference counts still alive after shutdown\n" +"PYTHONDUMPREFSFILE: dump objects and reference counts to the specified file\n" +#endif +#ifdef __APPLE__ +"PYTHONEXECUTABLE: set sys.argv[0] to this value (macOS only)\n" +#endif +#ifdef MS_WINDOWS +"PYTHONLEGACYWINDOWSFSENCODING: use legacy \"mbcs\" encoding for file system\n" +"PYTHONLEGACYWINDOWSSTDIO: use legacy Windows stdio\n" +#endif +"PYTHONUSERBASE : defines the user base directory (site.USER_BASE)\n" +"PYTHON_BASIC_REPL: use the traditional parser-based REPL\n" "\n" "These variables have equivalent command-line options (see --help for details):\n" +"PYTHON_CPU_COUNT: override the return value of os.cpu_count() (-X cpu_count)\n" "PYTHONDEBUG : enable parser debug mode (-d)\n" +"PYTHONDEVMODE : enable Python Development Mode (-X dev)\n" "PYTHONDONTWRITEBYTECODE: don't write .pyc files (-B)\n" +"PYTHONFAULTHANDLER: dump the Python traceback on fatal errors (-X faulthandler)\n" +"PYTHON_FROZEN_MODULES: whether to use frozen modules; the default is \"on\"\n" +" for installed Python and \"off\" for a local build\n" +" (-X frozen_modules)\n" +#ifdef Py_GIL_DISABLED +"PYTHON_GIL : when set to 0, disables the GIL (-X gil)\n" +#endif "PYTHONINSPECT : inspect interactively after running script (-i)\n" -"PYTHONINTMAXSTRDIGITS: limit max digit characters in an int value\n" -" (-X int_max_str_digits=number)\n" +"PYTHONINTMAXSTRDIGITS: limit the size of int<->str conversions;\n" +" 0 disables the limit (-X int_max_str_digits=N)\n" +"PYTHONNODEBUGRANGES: don't include extra location information in code objects\n" +" (-X no_debug_ranges)\n" "PYTHONNOUSERSITE: disable user site directory (-s)\n" "PYTHONOPTIMIZE : enable level 1 optimizations (-O)\n" +"PYTHONPERFSUPPORT: support the Linux \"perf\" profiler (-X perf)\n" +"PYTHON_PERF_JIT_SUPPORT: enable Linux \"perf\" profiler support with JIT\n" +" (-X perf_jit)\n" +#ifdef Py_DEBUG +"PYTHON_PRESITE: import this module before site (-X presite)\n" +#endif +"PYTHONPROFILEIMPORTTIME: show how long each import takes (-X importtime)\n" +"PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files\n" +" (-X pycache_prefix)\n" "PYTHONSAFEPATH : don't prepend a potentially unsafe path to sys.path.\n" -"PYTHONUNBUFFERED: disable stdout/stderr buffering (-u)\n" -"PYTHONVERBOSE : trace import statements (-v)\n" -"PYTHONWARNINGS=arg: warning control (-W arg)\n" #ifdef Py_STATS -"PYTHONSTATS : turns on statistics gathering\n" +"PYTHONSTATS : turns on statistics gathering (-X pystats)\n" #endif -#ifdef Py_DEBUG -"PYTHON_PRESITE=pkg.mod: import this module before site.py is run\n" +#ifdef Py_GIL_DISABLED +"PYTHON_TLBC : when set to 0, disables thread-local bytecode (-X tlbc)\n" #endif +"PYTHONTRACEMALLOC: trace Python memory allocations (-X tracemalloc)\n" +"PYTHONUNBUFFERED: disable stdout/stderr buffering (-u)\n" +"PYTHONUTF8 : control the UTF-8 mode (-X utf8)\n" +"PYTHONVERBOSE : trace import statements (-v)\n" +"PYTHONWARNDEFAULTENCODING: enable opt-in EncodingWarning for 'encoding=None'\n" +" (-X warn_default_encoding)\n" +"PYTHONWARNINGS : warning control (-W)\n" ; #if defined(MS_WINDOWS) @@ -653,6 +760,28 @@ _PyWideStringList_AsList(const PyWideStringList *list) } +static PyObject* +_PyWideStringList_AsTuple(const PyWideStringList *list) +{ + assert(_PyWideStringList_CheckConsistency(list)); + + PyObject *tuple = PyTuple_New(list->length); + if (tuple == NULL) { + return NULL; + } + + for (Py_ssize_t i = 0; i < list->length; i++) { + PyObject *item = PyUnicode_FromWideChar(list->items[i], -1); + if (item == NULL) { + Py_DECREF(tuple); + return NULL; + } + PyTuple_SET_ITEM(tuple, i, item); + } + return tuple; +} + + /* --- Py_GetArgcArgv() ------------------------------------------- */ void @@ -758,6 +887,9 @@ config_check_consistency(const PyConfig *config) assert(config->cpu_count != 0); // config->use_frozen_modules is initialized later // by _PyConfig_InitImportConfig(). +#ifdef __APPLE__ + assert(config->use_system_logger >= 0); +#endif #ifdef Py_STATS assert(config->_pystats >= 0); #endif @@ -860,6 +992,13 @@ _PyConfig_InitCompatConfig(PyConfig *config) config->_is_python_build = 0; config->code_debug_ranges = 1; config->cpu_count = -1; +#ifdef __APPLE__ + config->use_system_logger = 0; +#endif +#ifdef Py_GIL_DISABLED + config->enable_gil = _PyConfig_GIL_DEFAULT; + config->tlbc_enabled = 1; +#endif } @@ -885,6 +1024,9 @@ config_init_defaults(PyConfig *config) #ifdef MS_WINDOWS config->legacy_windows_stdio = 0; #endif +#ifdef __APPLE__ + config->use_system_logger = 0; +#endif } @@ -911,7 +1053,6 @@ PyConfig_InitIsolatedConfig(PyConfig *config) config->dev_mode = 0; config->install_signal_handlers = 0; config->use_hash_seed = 0; - config->faulthandler = 0; config->tracemalloc = 0; config->perf_profiling = 0; config->int_max_str_digits = _PY_LONG_DEFAULT_MAX_STR_DIGITS; @@ -920,6 +1061,9 @@ PyConfig_InitIsolatedConfig(PyConfig *config) #ifdef MS_WINDOWS config->legacy_windows_stdio = 0; #endif +#ifdef __APPLE__ + config->use_system_logger = 0; +#endif } @@ -994,6 +1138,20 @@ PyConfig_SetBytesString(PyConfig *config, wchar_t **config_str, } +static inline void* +config_get_spec_member(const PyConfig *config, const PyConfigSpec *spec) +{ + return (char *)config + spec->offset; +} + + +static inline void* +preconfig_get_spec_member(const PyPreConfig *preconfig, const PyConfigSpec *spec) +{ + return (char *)preconfig + spec->offset; +} + + PyStatus _PyConfig_Copy(PyConfig *config, const PyConfig *config2) { @@ -1002,11 +1160,12 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2) PyStatus status; const PyConfigSpec *spec = PYCONFIG_SPEC; for (; spec->name != NULL; spec++) { - char *member = (char *)config + spec->offset; - char *member2 = (char *)config2 + spec->offset; + void *member = config_get_spec_member(config, spec); + const void *member2 = config_get_spec_member((PyConfig*)config2, spec); switch (spec->type) { case PyConfig_MEMBER_INT: case PyConfig_MEMBER_UINT: + case PyConfig_MEMBER_BOOL: { *(int*)member = *(int*)member2; break; @@ -1052,48 +1211,12 @@ _PyConfig_AsDict(const PyConfig *config) const PyConfigSpec *spec = PYCONFIG_SPEC; for (; spec->name != NULL; spec++) { - char *member = (char *)config + spec->offset; - PyObject *obj; - switch (spec->type) { - case PyConfig_MEMBER_INT: - case PyConfig_MEMBER_UINT: - { - int value = *(int*)member; - obj = PyLong_FromLong(value); - break; - } - case PyConfig_MEMBER_ULONG: - { - unsigned long value = *(unsigned long*)member; - obj = PyLong_FromUnsignedLong(value); - break; - } - case PyConfig_MEMBER_WSTR: - case PyConfig_MEMBER_WSTR_OPT: - { - const wchar_t *wstr = *(const wchar_t**)member; - if (wstr != NULL) { - obj = PyUnicode_FromWideChar(wstr, -1); - } - else { - obj = Py_NewRef(Py_None); - } - break; - } - case PyConfig_MEMBER_WSTR_LIST: - { - const PyWideStringList *list = (const PyWideStringList*)member; - obj = _PyWideStringList_AsList(list); - break; - } - default: - Py_UNREACHABLE(); - } - + PyObject *obj = config_get(config, spec, 0); if (obj == NULL) { Py_DECREF(dict); return NULL; } + int res = PyDict_SetItemString(dict, spec->name, obj); Py_DECREF(obj); if (res < 0) { @@ -1105,21 +1228,6 @@ _PyConfig_AsDict(const PyConfig *config) } -static PyObject* -config_dict_get(PyObject *dict, const char *name) -{ - PyObject *item; - if (PyDict_GetItemStringRef(dict, name, &item) < 0) { - return NULL; - } - if (item == NULL) { - PyErr_Format(PyExc_ValueError, "missing config key: %s", name); - return NULL; - } - return item; -} - - static void config_dict_invalid_value(const char *name) { @@ -1127,13 +1235,6 @@ config_dict_invalid_value(const char *name) } -static void -config_dict_invalid_type(const char *name) -{ - PyErr_Format(PyExc_TypeError, "invalid config type: %s", name); -} - - static int config_dict_get_int(PyObject *dict, const char *name, int *result) { @@ -1227,15 +1328,17 @@ config_dict_get_wstrlist(PyObject *dict, const char *name, PyConfig *config, return -1; } - if (!PyList_CheckExact(list)) { + int is_list = PyList_CheckExact(list); + if (!is_list && !PyTuple_CheckExact(list)) { Py_DECREF(list); config_dict_invalid_type(name); return -1; } PyWideStringList wstrlist = _PyWideStringList_INIT; - for (Py_ssize_t i=0; i < PyList_GET_SIZE(list); i++) { - PyObject *item = PyList_GET_ITEM(list, i); + Py_ssize_t len = is_list ? PyList_GET_SIZE(list) : PyTuple_GET_SIZE(list); + for (Py_ssize_t i=0; i < len; i++) { + PyObject *item = is_list ? PyList_GET_ITEM(list, i) : PyTuple_GET_ITEM(list, i); if (item == Py_None) { config_dict_invalid_value(name); @@ -1272,6 +1375,66 @@ config_dict_get_wstrlist(PyObject *dict, const char *name, PyConfig *config, } +static int +config_dict_get_xoptions(PyObject *dict, const char *name, PyConfig *config, + PyWideStringList *result) +{ + PyObject *xoptions = config_dict_get(dict, name); + if (xoptions == NULL) { + return -1; + } + + if (!PyDict_CheckExact(xoptions)) { + Py_DECREF(xoptions); + config_dict_invalid_type(name); + return -1; + } + + Py_ssize_t pos = 0; + PyObject *key, *value; + PyWideStringList wstrlist = _PyWideStringList_INIT; + while (PyDict_Next(xoptions, &pos, &key, &value)) { + PyObject *item; + + if (value != Py_True) { + item = PyUnicode_FromFormat("%S=%S", key, value); + if (item == NULL) { + goto error; + } + } + else { + item = Py_NewRef(key); + } + + wchar_t *wstr = PyUnicode_AsWideCharString(item, NULL); + Py_DECREF(item); + if (wstr == NULL) { + goto error; + } + + PyStatus status = PyWideStringList_Append(&wstrlist, wstr); + PyMem_Free(wstr); + if (_PyStatus_EXCEPTION(status)) { + PyErr_NoMemory(); + goto error; + } + } + + if (_PyWideStringList_Copy(result, &wstrlist) < 0) { + PyErr_NoMemory(); + goto error; + } + _PyWideStringList_Clear(&wstrlist); + Py_DECREF(xoptions); + return 0; + +error: + _PyWideStringList_Clear(&wstrlist); + Py_DECREF(xoptions); + return -1; +} + + int _PyConfig_FromDict(PyConfig *config, PyObject *dict) { @@ -1285,19 +1448,20 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) char *member = (char *)config + spec->offset; switch (spec->type) { case PyConfig_MEMBER_INT: - if (config_dict_get_int(dict, spec->name, (int*)member) < 0) { - return -1; - } - break; case PyConfig_MEMBER_UINT: + case PyConfig_MEMBER_BOOL: { int value; if (config_dict_get_int(dict, spec->name, &value) < 0) { return -1; } - if (value < 0) { - config_dict_invalid_value(spec->name); - return -1; + if (spec->type == PyConfig_MEMBER_BOOL + || spec->type == PyConfig_MEMBER_UINT) + { + if (value < 0) { + config_dict_invalid_value(spec->name); + return -1; + } } *(int*)member = value; break; @@ -1332,9 +1496,17 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) } case PyConfig_MEMBER_WSTR_LIST: { - if (config_dict_get_wstrlist(dict, spec->name, config, - (PyWideStringList*)member) < 0) { - return -1; + if (strcmp(spec->name, "xoptions") == 0) { + if (config_dict_get_xoptions(dict, spec->name, config, + (PyWideStringList*)member) < 0) { + return -1; + } + } + else { + if (config_dict_get_wstrlist(dict, spec->name, config, + (PyWideStringList*)member) < 0) { + return -1; + } } break; } @@ -1564,6 +1736,28 @@ config_wstr_to_int(const wchar_t *wstr, int *result) return 0; } +static PyStatus +config_read_gil(PyConfig *config, size_t len, wchar_t first_char) +{ + if (len == 1 && first_char == L'0') { +#ifdef Py_GIL_DISABLED + config->enable_gil = _PyConfig_GIL_DISABLE; +#else + return _PyStatus_ERR("Disabling the GIL is not supported by this build"); +#endif + } + else if (len == 1 && first_char == L'1') { +#ifdef Py_GIL_DISABLED + config->enable_gil = _PyConfig_GIL_ENABLE; +#else + return _PyStatus_OK(); +#endif + } + else { + return _PyStatus_ERR("PYTHON_GIL / -X gil must be \"0\" or \"1\""); + } + return _PyStatus_OK(); +} static PyStatus config_read_env_vars(PyConfig *config) @@ -1642,6 +1836,15 @@ config_read_env_vars(PyConfig *config) config->safe_path = 1; } + const char *gil = config_get_env(config, "PYTHON_GIL"); + if (gil != NULL) { + size_t len = strlen(gil); + status = config_read_gil(config, len, gil[0]); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } + return _PyStatus_OK(); } @@ -1684,6 +1887,36 @@ config_init_cpu_count(PyConfig *config) "n must be greater than 0"); } +static PyStatus +config_init_tlbc(PyConfig *config) +{ +#ifdef Py_GIL_DISABLED + const char *env = config_get_env(config, "PYTHON_TLBC"); + if (env) { + int enabled; + if (_Py_str_to_int(env, &enabled) < 0 || (enabled < 0) || (enabled > 1)) { + return _PyStatus_ERR( + "PYTHON_TLBC=N: N is missing or invalid"); + } + config->tlbc_enabled = enabled; + } + + const wchar_t *xoption = config_get_xoption(config, L"tlbc"); + if (xoption) { + int enabled; + const wchar_t *sep = wcschr(xoption, L'='); + if (!sep || (config_wstr_to_int(sep + 1, &enabled) < 0) || (enabled < 0) || (enabled > 1)) { + return _PyStatus_ERR( + "-X tlbc=n: n is missing or invalid"); + } + config->tlbc_enabled = enabled; + } + return _PyStatus_OK(); +#else + return _PyStatus_OK(); +#endif +} + static PyStatus config_init_perf_profiling(PyConfig *config) { @@ -1701,6 +1934,20 @@ config_init_perf_profiling(PyConfig *config) if (xoption) { config->perf_profiling = 1; } + env = config_get_env(config, "PYTHON_PERF_JIT_SUPPORT"); + if (env) { + if (_Py_str_to_int(env, &active) != 0) { + active = 0; + } + if (active) { + config->perf_profiling = 2; + } + } + xoption = config_get_xoption(config, L"perf_jit"); + if (xoption) { + config->perf_profiling = 2; + } + return _PyStatus_OK(); } @@ -1919,6 +2166,11 @@ config_read_complex_options(PyConfig *config) } #endif + status = config_init_tlbc(config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + return _PyStatus_OK(); } @@ -2197,6 +2449,15 @@ config_read(PyConfig *config, int compute_path_config) config->show_ref_count = 1; } + const wchar_t *x_gil = config_get_xoption_value(config, L"gil"); + if (x_gil != NULL) { + size_t len = wcslen(x_gil); + status = config_read_gil(config, len, x_gil[0]); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } + #ifdef Py_STATS if (config_get_xoption(config, L"pystats")) { config->_pystats = 1; @@ -3219,3 +3480,1013 @@ _Py_DumpPathConfig(PyThreadState *tstate) _PyErr_SetRaisedException(tstate, exc); } + + +// --- PyInitConfig API --------------------------------------------------- + +struct PyInitConfig { + PyPreConfig preconfig; + PyConfig config; + struct _inittab *inittab; + Py_ssize_t inittab_size; + PyStatus status; + char *err_msg; +}; + +static PyInitConfig* +initconfig_alloc(void) +{ + return calloc(1, sizeof(PyInitConfig)); +} + + +PyInitConfig* +PyInitConfig_Create(void) +{ + PyInitConfig *config = initconfig_alloc(); + if (config == NULL) { + return NULL; + } + PyPreConfig_InitIsolatedConfig(&config->preconfig); + PyConfig_InitIsolatedConfig(&config->config); + config->status = _PyStatus_OK(); + return config; +} + + +void +PyInitConfig_Free(PyInitConfig *config) +{ + if (config == NULL) { + return; + } + free(config->err_msg); + free(config); +} + + +int +PyInitConfig_GetError(PyInitConfig* config, const char **perr_msg) +{ + if (_PyStatus_IS_EXIT(config->status)) { + char buffer[22]; // len("exit code -2147483648\0") + PyOS_snprintf(buffer, sizeof(buffer), + "exit code %i", + config->status.exitcode); + + if (config->err_msg != NULL) { + free(config->err_msg); + } + config->err_msg = strdup(buffer); + if (config->err_msg != NULL) { + *perr_msg = config->err_msg; + return 1; + } + config->status = _PyStatus_NO_MEMORY(); + } + + if (_PyStatus_IS_ERROR(config->status) && config->status.err_msg != NULL) { + *perr_msg = config->status.err_msg; + return 1; + } + else { + *perr_msg = NULL; + return 0; + } +} + + +int +PyInitConfig_GetExitCode(PyInitConfig* config, int *exitcode) +{ + if (_PyStatus_IS_EXIT(config->status)) { + *exitcode = config->status.exitcode; + return 1; + } + else { + return 0; + } +} + + +static void +initconfig_set_error(PyInitConfig *config, const char *err_msg) +{ + config->status = _PyStatus_ERR(err_msg); +} + + +static const PyConfigSpec* +initconfig_find_spec(const PyConfigSpec *spec, const char *name) +{ + for (; spec->name != NULL; spec++) { + if (strcmp(name, spec->name) == 0) { + return spec; + } + } + return NULL; +} + + +int +PyInitConfig_HasOption(PyInitConfig *config, const char *name) +{ + const PyConfigSpec *spec = initconfig_find_spec(PYCONFIG_SPEC, name); + if (spec == NULL) { + spec = initconfig_find_spec(PYPRECONFIG_SPEC, name); + } + return (spec != NULL); +} + + +static const PyConfigSpec* +initconfig_prepare(PyInitConfig *config, const char *name, void **raw_member) +{ + const PyConfigSpec *spec = initconfig_find_spec(PYCONFIG_SPEC, name); + if (spec != NULL) { + *raw_member = config_get_spec_member(&config->config, spec); + return spec; + } + + spec = initconfig_find_spec(PYPRECONFIG_SPEC, name); + if (spec != NULL) { + *raw_member = preconfig_get_spec_member(&config->preconfig, spec); + return spec; + } + + initconfig_set_error(config, "unknown config option name"); + return NULL; +} + + +int +PyInitConfig_GetInt(PyInitConfig *config, const char *name, int64_t *value) +{ + void *raw_member; + const PyConfigSpec *spec = initconfig_prepare(config, name, &raw_member); + if (spec == NULL) { + return -1; + } + + switch (spec->type) { + case PyConfig_MEMBER_INT: + case PyConfig_MEMBER_UINT: + case PyConfig_MEMBER_BOOL: + { + int *member = raw_member; + *value = *member; + break; + } + + case PyConfig_MEMBER_ULONG: + { + unsigned long *member = raw_member; +#if SIZEOF_LONG >= 8 + if ((unsigned long)INT64_MAX < *member) { + initconfig_set_error(config, + "config option value doesn't fit into int64_t"); + return -1; + } +#endif + *value = *member; + break; + } + + default: + initconfig_set_error(config, "config option type is not int"); + return -1; + } + return 0; +} + + +static char* +wstr_to_utf8(PyInitConfig *config, wchar_t *wstr) +{ + char *utf8; + int res = _Py_EncodeUTF8Ex(wstr, &utf8, NULL, NULL, 1, _Py_ERROR_STRICT); + if (res == -2) { + initconfig_set_error(config, "encoding error"); + return NULL; + } + if (res < 0) { + config->status = _PyStatus_NO_MEMORY(); + return NULL; + } + + // Copy to use the malloc() memory allocator + size_t size = strlen(utf8) + 1; + char *str = malloc(size); + if (str == NULL) { + PyMem_RawFree(utf8); + config->status = _PyStatus_NO_MEMORY(); + return NULL; + } + + memcpy(str, utf8, size); + PyMem_RawFree(utf8); + return str; +} + + +int +PyInitConfig_GetStr(PyInitConfig *config, const char *name, char **value) +{ + void *raw_member; + const PyConfigSpec *spec = initconfig_prepare(config, name, &raw_member); + if (spec == NULL) { + return -1; + } + + if (spec->type != PyConfig_MEMBER_WSTR + && spec->type != PyConfig_MEMBER_WSTR_OPT) + { + initconfig_set_error(config, "config option type is not string"); + return -1; + } + + wchar_t **member = raw_member; + if (*member == NULL) { + *value = NULL; + return 0; + } + + *value = wstr_to_utf8(config, *member); + if (*value == NULL) { + return -1; + } + return 0; +} + + +int +PyInitConfig_GetStrList(PyInitConfig *config, const char *name, size_t *length, char ***items) +{ + void *raw_member; + const PyConfigSpec *spec = initconfig_prepare(config, name, &raw_member); + if (spec == NULL) { + return -1; + } + + if (spec->type != PyConfig_MEMBER_WSTR_LIST) { + initconfig_set_error(config, "config option type is not string list"); + return -1; + } + + PyWideStringList *list = raw_member; + *length = list->length; + + *items = malloc(list->length * sizeof(char*)); + if (*items == NULL) { + config->status = _PyStatus_NO_MEMORY(); + return -1; + } + + for (Py_ssize_t i=0; i < list->length; i++) { + (*items)[i] = wstr_to_utf8(config, list->items[i]); + if ((*items)[i] == NULL) { + PyInitConfig_FreeStrList(i, *items); + return -1; + } + } + return 0; +} + + +void +PyInitConfig_FreeStrList(size_t length, char **items) +{ + for (size_t i=0; i < length; i++) { + free(items[i]); + } + free(items); +} + + +int +PyInitConfig_SetInt(PyInitConfig *config, const char *name, int64_t value) +{ + void *raw_member; + const PyConfigSpec *spec = initconfig_prepare(config, name, &raw_member); + if (spec == NULL) { + return -1; + } + + switch (spec->type) { + case PyConfig_MEMBER_INT: + { + if (value < (int64_t)INT_MIN || (int64_t)INT_MAX < value) { + initconfig_set_error(config, + "config option value is out of int range"); + return -1; + } + int int_value = (int)value; + + int *member = raw_member; + *member = int_value; + break; + } + + case PyConfig_MEMBER_UINT: + case PyConfig_MEMBER_BOOL: + { + if (value < 0 || (uint64_t)UINT_MAX < (uint64_t)value) { + initconfig_set_error(config, + "config option value is out of unsigned int range"); + return -1; + } + int int_value = (int)value; + + int *member = raw_member; + *member = int_value; + break; + } + + case PyConfig_MEMBER_ULONG: + { + if (value < 0 || (uint64_t)ULONG_MAX < (uint64_t)value) { + initconfig_set_error(config, + "config option value is out of unsigned long range"); + return -1; + } + unsigned long ulong_value = (unsigned long)value; + + unsigned long *member = raw_member; + *member = ulong_value; + break; + } + + default: + initconfig_set_error(config, "config option type is not int"); + return -1; + } + + if (strcmp(name, "hash_seed") == 0) { + config->config.use_hash_seed = 1; + } + + return 0; +} + + +static wchar_t* +utf8_to_wstr(PyInitConfig *config, const char *str) +{ + wchar_t *wstr; + size_t wlen; + int res = _Py_DecodeUTF8Ex(str, strlen(str), &wstr, &wlen, NULL, _Py_ERROR_STRICT); + if (res == -2) { + initconfig_set_error(config, "decoding error"); + return NULL; + } + if (res < 0) { + config->status = _PyStatus_NO_MEMORY(); + return NULL; + } + + // Copy to use the malloc() memory allocator + size_t size = (wlen + 1) * sizeof(wchar_t); + wchar_t *wstr2 = malloc(size); + if (wstr2 == NULL) { + PyMem_RawFree(wstr); + config->status = _PyStatus_NO_MEMORY(); + return NULL; + } + + memcpy(wstr2, wstr, size); + PyMem_RawFree(wstr); + return wstr2; +} + + +int +PyInitConfig_SetStr(PyInitConfig *config, const char *name, const char* value) +{ + void *raw_member; + const PyConfigSpec *spec = initconfig_prepare(config, name, &raw_member); + if (spec == NULL) { + return -1; + } + + if (spec->type != PyConfig_MEMBER_WSTR + && spec->type != PyConfig_MEMBER_WSTR_OPT) { + initconfig_set_error(config, "config option type is not string"); + return -1; + } + + if (value == NULL && spec->type != PyConfig_MEMBER_WSTR_OPT) { + initconfig_set_error(config, "config option string cannot be NULL"); + } + + wchar_t **member = raw_member; + + *member = utf8_to_wstr(config, value); + if (*member == NULL) { + return -1; + } + return 0; +} + + +static int +_PyWideStringList_FromUTF8(PyInitConfig *config, PyWideStringList *list, + Py_ssize_t length, char * const *items) +{ + PyWideStringList wlist = _PyWideStringList_INIT; + size_t size = sizeof(wchar_t*) * length; + wlist.items = (wchar_t **)PyMem_RawMalloc(size); + if (wlist.items == NULL) { + config->status = _PyStatus_NO_MEMORY(); + return -1; + } + + for (Py_ssize_t i = 0; i < length; i++) { + wchar_t *arg = utf8_to_wstr(config, items[i]); + if (arg == NULL) { + _PyWideStringList_Clear(&wlist); + return -1; + } + wlist.items[i] = arg; + wlist.length++; + } + + _PyWideStringList_Clear(list); + *list = wlist; + return 0; +} + + +int +PyInitConfig_SetStrList(PyInitConfig *config, const char *name, + size_t length, char * const *items) +{ + void *raw_member; + const PyConfigSpec *spec = initconfig_prepare(config, name, &raw_member); + if (spec == NULL) { + return -1; + } + + if (spec->type != PyConfig_MEMBER_WSTR_LIST) { + initconfig_set_error(config, "config option type is not strings list"); + return -1; + } + PyWideStringList *list = raw_member; + if (_PyWideStringList_FromUTF8(config, list, length, items) < 0) { + return -1; + } + + if (strcmp(name, "module_search_paths") == 0) { + config->config.module_search_paths_set = 1; + } + return 0; +} + + +int +PyInitConfig_AddModule(PyInitConfig *config, const char *name, + PyObject* (*initfunc)(void)) +{ + size_t size = sizeof(struct _inittab) * (config->inittab_size + 2); + struct _inittab *new_inittab = PyMem_RawRealloc(config->inittab, size); + if (new_inittab == NULL) { + config->status = _PyStatus_NO_MEMORY(); + return -1; + } + config->inittab = new_inittab; + + struct _inittab *entry = &config->inittab[config->inittab_size]; + entry->name = name; + entry->initfunc = initfunc; + + // Terminator entry + entry = &config->inittab[config->inittab_size + 1]; + entry->name = NULL; + entry->initfunc = NULL; + + config->inittab_size++; + return 0; +} + + +int +Py_InitializeFromInitConfig(PyInitConfig *config) +{ + if (config->inittab_size >= 1) { + if (PyImport_ExtendInittab(config->inittab) < 0) { + config->status = _PyStatus_NO_MEMORY(); + return -1; + } + } + + _PyPreConfig_GetConfig(&config->preconfig, &config->config); + + config->status = Py_PreInitializeFromArgs( + &config->preconfig, + config->config.argv.length, + config->config.argv.items); + if (_PyStatus_EXCEPTION(config->status)) { + return -1; + } + + config->status = Py_InitializeFromConfig(&config->config); + if (_PyStatus_EXCEPTION(config->status)) { + return -1; + } + + return 0; +} + + +// --- PyConfig_Get() ------------------------------------------------------- + +static const PyConfigSpec* +config_generic_find_spec(const PyConfigSpec *spec, const char *name) +{ + for (; spec->name != NULL; spec++) { + if (spec->visibility == PyConfig_MEMBER_INIT_ONLY) { + continue; + } + if (strcmp(name, spec->name) == 0) { + return spec; + } + } + return NULL; +} + + +static const PyConfigSpec* +config_find_spec(const char *name) +{ + return config_generic_find_spec(PYCONFIG_SPEC, name); +} + + +static const PyConfigSpec* +preconfig_find_spec(const char *name) +{ + return config_generic_find_spec(PYPRECONFIG_SPEC, name); +} + + +static int +config_add_xoption(PyObject *dict, const wchar_t *str) +{ + PyObject *name = NULL, *value = NULL; + + const wchar_t *name_end = wcschr(str, L'='); + if (!name_end) { + name = PyUnicode_FromWideChar(str, -1); + if (name == NULL) { + goto error; + } + value = Py_NewRef(Py_True); + } + else { + name = PyUnicode_FromWideChar(str, name_end - str); + if (name == NULL) { + goto error; + } + value = PyUnicode_FromWideChar(name_end + 1, -1); + if (value == NULL) { + goto error; + } + } + if (PyDict_SetItem(dict, name, value) < 0) { + goto error; + } + Py_DECREF(name); + Py_DECREF(value); + return 0; + +error: + Py_XDECREF(name); + Py_XDECREF(value); + return -1; +} + + +PyObject* +_PyConfig_CreateXOptionsDict(const PyConfig *config) +{ + PyObject *dict = PyDict_New(); + if (dict == NULL) { + return NULL; + } + + Py_ssize_t nxoption = config->xoptions.length; + wchar_t **xoptions = config->xoptions.items; + for (Py_ssize_t i=0; i < nxoption; i++) { + const wchar_t *option = xoptions[i]; + if (config_add_xoption(dict, option) < 0) { + Py_DECREF(dict); + return NULL; + } + } + return dict; +} + + +static PyObject* +config_get_sys(const char *name) +{ + PyObject *value = PySys_GetObject(name); + if (value == NULL) { + PyErr_Format(PyExc_RuntimeError, "lost sys.%s", name); + return NULL; + } + return Py_NewRef(value); +} + + +static int +config_get_sys_write_bytecode(const PyConfig *config, int *value) +{ + PyObject *attr = config_get_sys("dont_write_bytecode"); + if (attr == NULL) { + return -1; + } + + int is_true = PyObject_IsTrue(attr); + Py_DECREF(attr); + if (is_true < 0) { + return -1; + } + *value = (!is_true); + return 0; +} + + +static PyObject* +config_get(const PyConfig *config, const PyConfigSpec *spec, + int use_sys) +{ + if (use_sys) { + if (spec->sys.attr != NULL) { + return config_get_sys(spec->sys.attr); + } + + if (strcmp(spec->name, "write_bytecode") == 0) { + int value; + if (config_get_sys_write_bytecode(config, &value) < 0) { + return NULL; + } + return PyBool_FromLong(value); + } + + if (strcmp(spec->name, "int_max_str_digits") == 0) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + return PyLong_FromLong(interp->long_state.max_str_digits); + } + } + + void *member = config_get_spec_member(config, spec); + switch (spec->type) { + case PyConfig_MEMBER_INT: + case PyConfig_MEMBER_UINT: + { + int value = *(int *)member; + return PyLong_FromLong(value); + } + + case PyConfig_MEMBER_BOOL: + { + int value = *(int *)member; + return PyBool_FromLong(value != 0); + } + + case PyConfig_MEMBER_ULONG: + { + unsigned long value = *(unsigned long *)member; + return PyLong_FromUnsignedLong(value); + } + + case PyConfig_MEMBER_WSTR: + case PyConfig_MEMBER_WSTR_OPT: + { + wchar_t *wstr = *(wchar_t **)member; + if (wstr != NULL) { + return PyUnicode_FromWideChar(wstr, -1); + } + else { + return Py_NewRef(Py_None); + } + } + + case PyConfig_MEMBER_WSTR_LIST: + { + if (strcmp(spec->name, "xoptions") == 0) { + return _PyConfig_CreateXOptionsDict(config); + } + else { + const PyWideStringList *list = (const PyWideStringList *)member; + return _PyWideStringList_AsTuple(list); + } + } + + default: + Py_UNREACHABLE(); + } +} + + +static PyObject* +preconfig_get(const PyPreConfig *preconfig, const PyConfigSpec *spec) +{ + // The type of all PYPRECONFIG_SPEC members is INT or BOOL. + assert(spec->type == PyConfig_MEMBER_INT + || spec->type == PyConfig_MEMBER_BOOL); + + char *member = (char *)preconfig + spec->offset; + int value = *(int *)member; + + if (spec->type == PyConfig_MEMBER_BOOL) { + return PyBool_FromLong(value != 0); + } + else { + return PyLong_FromLong(value); + } +} + + +static void +config_unknown_name_error(const char *name) +{ + PyErr_Format(PyExc_ValueError, "unknown config option name: %s", name); +} + + +PyObject* +PyConfig_Get(const char *name) +{ + const PyConfigSpec *spec = config_find_spec(name); + if (spec != NULL) { + const PyConfig *config = _Py_GetConfig(); + return config_get(config, spec, 1); + } + + spec = preconfig_find_spec(name); + if (spec != NULL) { + const PyPreConfig *preconfig = &_PyRuntime.preconfig; + return preconfig_get(preconfig, spec); + } + + config_unknown_name_error(name); + return NULL; +} + + +int +PyConfig_GetInt(const char *name, int *value) +{ + assert(!PyErr_Occurred()); + + PyObject *obj = PyConfig_Get(name); + if (obj == NULL) { + return -1; + } + + if (!PyLong_Check(obj)) { + Py_DECREF(obj); + PyErr_Format(PyExc_TypeError, "config option %s is not an int", name); + return -1; + } + + int as_int = PyLong_AsInt(obj); + Py_DECREF(obj); + if (as_int == -1 && PyErr_Occurred()) { + PyErr_Format(PyExc_OverflowError, + "config option %s value does not fit into a C int", name); + return -1; + } + + *value = as_int; + return 0; +} + + +static int +config_names_add(PyObject *names, const PyConfigSpec *spec) +{ + for (; spec->name != NULL; spec++) { + if (spec->visibility == PyConfig_MEMBER_INIT_ONLY) { + continue; + } + PyObject *name = PyUnicode_FromString(spec->name); + if (name == NULL) { + return -1; + } + int res = PyList_Append(names, name); + Py_DECREF(name); + if (res < 0) { + return -1; + } + } + return 0; +} + + +PyObject* +PyConfig_Names(void) +{ + PyObject *names = PyList_New(0); + if (names == NULL) { + goto error; + } + + if (config_names_add(names, PYCONFIG_SPEC) < 0) { + goto error; + } + if (config_names_add(names, PYPRECONFIG_SPEC) < 0) { + goto error; + } + + PyObject *frozen = PyFrozenSet_New(names); + Py_DECREF(names); + return frozen; + +error: + Py_XDECREF(names); + return NULL; +} + + +// --- PyConfig_Set() ------------------------------------------------------- + +static int +config_set_sys_flag(const PyConfigSpec *spec, int int_value) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyConfig *config = &interp->config; + + if (spec->type == PyConfig_MEMBER_BOOL) { + if (int_value != 0) { + // convert values < 0 and values > 1 to 1 + int_value = 1; + } + } + + PyObject *value; + if (spec->sys.flag_setter) { + value = spec->sys.flag_setter(int_value); + } + else { + value = config_sys_flag_long(int_value); + } + if (value == NULL) { + return -1; + } + + // Set sys.flags.FLAG + Py_ssize_t pos = spec->sys.flag_index; + if (_PySys_SetFlagObj(pos, value) < 0) { + goto error; + } + + // Set PyConfig.ATTR + assert(spec->type == PyConfig_MEMBER_INT + || spec->type == PyConfig_MEMBER_UINT + || spec->type == PyConfig_MEMBER_BOOL); + int *member = config_get_spec_member(config, spec); + *member = int_value; + + // Set sys.dont_write_bytecode attribute + if (strcmp(spec->name, "write_bytecode") == 0) { + if (PySys_SetObject("dont_write_bytecode", value) < 0) { + goto error; + } + } + + Py_DECREF(value); + return 0; + +error: + Py_DECREF(value); + return -1; +} + + +int +PyConfig_Set(const char *name, PyObject *value) +{ + const PyConfigSpec *spec = config_find_spec(name); + if (spec == NULL) { + spec = preconfig_find_spec(name); + if (spec == NULL) { + config_unknown_name_error(name); + return -1; + } + assert(spec->visibility != PyConfig_MEMBER_PUBLIC); + } + + if (spec->visibility != PyConfig_MEMBER_PUBLIC) { + PyErr_Format(PyExc_ValueError, "cannot set read-only option %s", + name); + return -1; + } + + int int_value = 0; + int has_int_value = 0; + + switch (spec->type) { + case PyConfig_MEMBER_INT: + case PyConfig_MEMBER_UINT: + case PyConfig_MEMBER_BOOL: + if (!PyLong_Check(value)) { + PyErr_Format(PyExc_TypeError, "expected int or bool, got %T", value); + return -1; + } + int_value = PyLong_AsInt(value); + if (int_value == -1 && PyErr_Occurred()) { + return -1; + } + if (int_value < 0 && spec->type != PyConfig_MEMBER_INT) { + PyErr_Format(PyExc_ValueError, "value must be >= 0"); + return -1; + } + has_int_value = 1; + break; + + case PyConfig_MEMBER_ULONG: + // not implemented: only hash_seed uses this type, and it's read-only + goto cannot_set; + + case PyConfig_MEMBER_WSTR: + if (!PyUnicode_CheckExact(value)) { + PyErr_Format(PyExc_TypeError, "expected str, got %T", value); + return -1; + } + break; + + case PyConfig_MEMBER_WSTR_OPT: + if (value != Py_None && !PyUnicode_CheckExact(value)) { + PyErr_Format(PyExc_TypeError, "expected str or None, got %T", value); + return -1; + } + break; + + case PyConfig_MEMBER_WSTR_LIST: + if (strcmp(spec->name, "xoptions") != 0) { + if (!PyList_Check(value)) { + PyErr_Format(PyExc_TypeError, "expected list[str], got %T", + value); + return -1; + } + for (Py_ssize_t i=0; i < PyList_GET_SIZE(value); i++) { + PyObject *item = PyList_GET_ITEM(value, i); + if (!PyUnicode_Check(item)) { + PyErr_Format(PyExc_TypeError, + "expected str, list item %zd has type %T", + i, item); + return -1; + } + } + } + else { + // xoptions type is dict[str, str] + if (!PyDict_Check(value)) { + PyErr_Format(PyExc_TypeError, + "expected dict[str, str | bool], got %T", + value); + return -1; + } + + Py_ssize_t pos = 0; + PyObject *key, *item; + while (PyDict_Next(value, &pos, &key, &item)) { + if (!PyUnicode_Check(key)) { + PyErr_Format(PyExc_TypeError, + "expected str, " + "got dict key type %T", key); + return -1; + } + if (!PyUnicode_Check(item) && !PyBool_Check(item)) { + PyErr_Format(PyExc_TypeError, + "expected str or bool, " + "got dict value type %T", key); + return -1; + } + } + } + break; + + default: + Py_UNREACHABLE(); + } + + + if (spec->sys.attr != NULL) { + // Set the sys attribute, but don't set PyInterpreterState.config + // to keep the code simple. + return PySys_SetObject(spec->sys.attr, value); + } + else if (spec->sys.flag_index >= 0 && has_int_value) { + return config_set_sys_flag(spec, int_value); + } + else if (strcmp(spec->name, "int_max_str_digits") == 0 && has_int_value) { + return _PySys_SetIntMaxStrDigits(int_value); + } + +cannot_set: + PyErr_Format(PyExc_ValueError, "cannot set option %s", name); + return -1; +} diff --git a/Python/instruction_sequence.c b/Python/instruction_sequence.c new file mode 100644 index 00000000000000..a3f85f754d71bb --- /dev/null +++ b/Python/instruction_sequence.c @@ -0,0 +1,453 @@ +/* + * This file implements a data structure representing a sequence of + * instructions, which is used by different parts of the compilation + * pipeline. + */ + + +#include + +#include "Python.h" + +#include "pycore_compile.h" // _PyCompile_EnsureArrayLargeEnough +#include "pycore_opcode_utils.h" +#include "pycore_opcode_metadata.h" // OPCODE_HAS_ARG, etc + +typedef _PyInstruction instruction; +typedef _PyInstructionSequence instr_sequence; +typedef _Py_SourceLocation location; + +#define INITIAL_INSTR_SEQUENCE_SIZE 100 +#define INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE 10 + +#include "clinic/instruction_sequence.c.h" + +#undef SUCCESS +#undef ERROR +#define SUCCESS 0 +#define ERROR -1 + +#define RETURN_IF_ERROR(X) \ + if ((X) == -1) { \ + return ERROR; \ + } + +static int +instr_sequence_next_inst(instr_sequence *seq) { + assert(seq->s_instrs != NULL || seq->s_used == 0); + + RETURN_IF_ERROR( + _PyCompile_EnsureArrayLargeEnough(seq->s_used + 1, + (void**)&seq->s_instrs, + &seq->s_allocated, + INITIAL_INSTR_SEQUENCE_SIZE, + sizeof(instruction))); + assert(seq->s_allocated >= 0); + assert(seq->s_used < seq->s_allocated); + return seq->s_used++; +} + +_PyJumpTargetLabel +_PyInstructionSequence_NewLabel(instr_sequence *seq) +{ + _PyJumpTargetLabel lbl = {++seq->s_next_free_label}; + return lbl; +} + +int +_PyInstructionSequence_UseLabel(instr_sequence *seq, int lbl) +{ + int old_size = seq->s_labelmap_size; + RETURN_IF_ERROR( + _PyCompile_EnsureArrayLargeEnough(lbl, + (void**)&seq->s_labelmap, + &seq->s_labelmap_size, + INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE, + sizeof(int))); + + for(int i = old_size; i < seq->s_labelmap_size; i++) { + seq->s_labelmap[i] = -111; /* something weird, for debugging */ + } + seq->s_labelmap[lbl] = seq->s_used; /* label refers to the next instruction */ + return SUCCESS; +} + +int +_PyInstructionSequence_ApplyLabelMap(instr_sequence *instrs) +{ + if (instrs->s_labelmap == NULL) { + /* Already applied - nothing to do */ + return SUCCESS; + } + /* Replace labels by offsets in the code */ + for (int i=0; i < instrs->s_used; i++) { + instruction *instr = &instrs->s_instrs[i]; + if (HAS_TARGET(instr->i_opcode)) { + assert(instr->i_oparg < instrs->s_labelmap_size); + instr->i_oparg = instrs->s_labelmap[instr->i_oparg]; + } + _PyExceptHandlerInfo *hi = &instr->i_except_handler_info; + if (hi->h_label >= 0) { + assert(hi->h_label < instrs->s_labelmap_size); + hi->h_label = instrs->s_labelmap[hi->h_label]; + } + } + /* Clear label map so it's never used again */ + PyMem_Free(instrs->s_labelmap); + instrs->s_labelmap = NULL; + instrs->s_labelmap_size = 0; + return SUCCESS; +} + +#define MAX_OPCODE 511 + +int +_PyInstructionSequence_Addop(instr_sequence *seq, int opcode, int oparg, + location loc) +{ + assert(0 <= opcode && opcode <= MAX_OPCODE); + assert(IS_WITHIN_OPCODE_RANGE(opcode)); + assert(OPCODE_HAS_ARG(opcode) || HAS_TARGET(opcode) || oparg == 0); + assert(0 <= oparg && oparg < (1 << 30)); + + int idx = instr_sequence_next_inst(seq); + RETURN_IF_ERROR(idx); + instruction *ci = &seq->s_instrs[idx]; + ci->i_opcode = opcode; + ci->i_oparg = oparg; + ci->i_loc = loc; + return SUCCESS; +} + +int +_PyInstructionSequence_InsertInstruction(instr_sequence *seq, int pos, + int opcode, int oparg, location loc) +{ + assert(pos >= 0 && pos <= seq->s_used); + int last_idx = instr_sequence_next_inst(seq); + RETURN_IF_ERROR(last_idx); + for (int i=last_idx-1; i >= pos; i--) { + seq->s_instrs[i+1] = seq->s_instrs[i]; + } + instruction *ci = &seq->s_instrs[pos]; + ci->i_opcode = opcode; + ci->i_oparg = oparg; + ci->i_loc = loc; + + /* fix the labels map */ + for(int lbl=0; lbl < seq->s_labelmap_size; lbl++) { + if (seq->s_labelmap[lbl] >= pos) { + seq->s_labelmap[lbl]++; + } + } + return SUCCESS; +} + +int +_PyInstructionSequence_AddNested(instr_sequence *seq, instr_sequence *nested) +{ + if (seq->s_nested == NULL) { + seq->s_nested = PyList_New(0); + if (seq->s_nested == NULL) { + return ERROR; + } + } + if (PyList_Append(seq->s_nested, (PyObject*)nested) < 0) { + return ERROR; + } + return SUCCESS; +} + +void +PyInstructionSequence_Fini(instr_sequence *seq) { + Py_XDECREF(seq->s_nested); + + PyMem_Free(seq->s_labelmap); + seq->s_labelmap = NULL; + + PyMem_Free(seq->s_instrs); + seq->s_instrs = NULL; +} + +/*[clinic input] +class InstructionSequenceType "_PyInstructionSequence *" "&_PyInstructionSequence_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=589963e07480390f]*/ + +static _PyInstructionSequence* +inst_seq_create(void) +{ + _PyInstructionSequence *seq; + seq = PyObject_GC_New(_PyInstructionSequence, &_PyInstructionSequence_Type); + if (seq == NULL) { + return NULL; + } + seq->s_instrs = NULL; + seq->s_allocated = 0; + seq->s_used = 0; + seq->s_next_free_label = 0; + seq->s_labelmap = NULL; + seq->s_labelmap_size = 0; + seq->s_nested = NULL; + + PyObject_GC_Track(seq); + return seq; +} + +PyObject* +_PyInstructionSequence_New(void) +{ + _PyInstructionSequence *seq = inst_seq_create(); + if (seq == NULL) { + return NULL; + } + return (PyObject*)seq; +} + +/*[clinic input] +@classmethod +InstructionSequenceType.__new__ as inst_seq_new + +Create a new InstructionSequence object. +[clinic start generated code]*/ + +static PyObject * +inst_seq_new_impl(PyTypeObject *type) +/*[clinic end generated code: output=98881de92c8876f6 input=b393150146849c74]*/ +{ + return (PyObject*)inst_seq_create(); +} + +/*[clinic input] +InstructionSequenceType.use_label + + label: int + +Place label at current location. +[clinic start generated code]*/ + +static PyObject * +InstructionSequenceType_use_label_impl(_PyInstructionSequence *self, + int label) +/*[clinic end generated code: output=4c06bbacb2854755 input=da55f49bb91841f3]*/ + +{ + if (_PyInstructionSequence_UseLabel(self, label) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/*[clinic input] +InstructionSequenceType.addop + + opcode: int + oparg: int + lineno: int + col_offset: int + end_lineno: int + end_col_offset: int + +Append an instruction. +[clinic start generated code]*/ + +static PyObject * +InstructionSequenceType_addop_impl(_PyInstructionSequence *self, int opcode, + int oparg, int lineno, int col_offset, + int end_lineno, int end_col_offset) +/*[clinic end generated code: output=af0cc22c048dfbf3 input=012762ac88198713]*/ +{ + _Py_SourceLocation loc = {lineno, col_offset, end_lineno, end_col_offset}; + if (_PyInstructionSequence_Addop(self, opcode, oparg, loc) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/*[clinic input] +InstructionSequenceType.new_label -> int + +Return a new label. +[clinic start generated code]*/ + +static int +InstructionSequenceType_new_label_impl(_PyInstructionSequence *self) +/*[clinic end generated code: output=dcb0589e4f5bf4bd input=c66040b9897bc327]*/ +{ + _PyJumpTargetLabel lbl = _PyInstructionSequence_NewLabel(self); + return lbl.id; +} + +/*[clinic input] +InstructionSequenceType.add_nested + + nested: object + +Add a nested sequence. +[clinic start generated code]*/ + +static PyObject * +InstructionSequenceType_add_nested_impl(_PyInstructionSequence *self, + PyObject *nested) +/*[clinic end generated code: output=14540fad459f7971 input=f2c482568b3b3c0f]*/ +{ + if (!_PyInstructionSequence_Check(nested)) { + PyErr_Format(PyExc_TypeError, + "expected an instruction sequence, not %T", + Py_TYPE(nested)); + return NULL; + } + if (_PyInstructionSequence_AddNested(self, (_PyInstructionSequence*)nested) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/*[clinic input] +InstructionSequenceType.get_nested + +Add a nested sequence. +[clinic start generated code]*/ + +static PyObject * +InstructionSequenceType_get_nested_impl(_PyInstructionSequence *self) +/*[clinic end generated code: output=f415112c292630cb input=e429e474c57b95b4]*/ +{ + if (self->s_nested == NULL) { + return PyList_New(0); + } + return Py_NewRef(self->s_nested); +} + +/*[clinic input] +InstructionSequenceType.get_instructions + +Return the instructions as a list of tuples or labels. +[clinic start generated code]*/ + +static PyObject * +InstructionSequenceType_get_instructions_impl(_PyInstructionSequence *self) +/*[clinic end generated code: output=23f4f3f894c301b3 input=fbadb5dadb611291]*/ +{ + if (_PyInstructionSequence_ApplyLabelMap(self) < 0) { + return NULL; + } + PyObject *instructions = PyList_New(0); + if (instructions == NULL) { + return NULL; + } + for (int i = 0; i < self->s_used; i++) { + instruction *instr = &self->s_instrs[i]; + location loc = instr->i_loc; + PyObject *inst_tuple; + + if (OPCODE_HAS_ARG(instr->i_opcode)) { + inst_tuple = Py_BuildValue( + "(iiiiii)", instr->i_opcode, instr->i_oparg, + loc.lineno, loc.end_lineno, + loc.col_offset, loc.end_col_offset); + } + else { + inst_tuple = Py_BuildValue( + "(iOiiii)", instr->i_opcode, Py_None, + loc.lineno, loc.end_lineno, + loc.col_offset, loc.end_col_offset); + } + if (inst_tuple == NULL) { + goto error; + } + + int res = PyList_Append(instructions, inst_tuple); + Py_DECREF(inst_tuple); + if (res != 0) { + goto error; + } + } + return instructions; +error: + Py_XDECREF(instructions); + return NULL; +} + +static PyMethodDef inst_seq_methods[] = { + INSTRUCTIONSEQUENCETYPE_ADDOP_METHODDEF + INSTRUCTIONSEQUENCETYPE_NEW_LABEL_METHODDEF + INSTRUCTIONSEQUENCETYPE_USE_LABEL_METHODDEF + INSTRUCTIONSEQUENCETYPE_ADD_NESTED_METHODDEF + INSTRUCTIONSEQUENCETYPE_GET_NESTED_METHODDEF + INSTRUCTIONSEQUENCETYPE_GET_INSTRUCTIONS_METHODDEF + {NULL, NULL, 0, NULL}, +}; + +static PyMemberDef inst_seq_memberlist[] = { + {NULL} /* Sentinel */ +}; + +static PyGetSetDef inst_seq_getsetters[] = { + {NULL} /* Sentinel */ +}; + +static void +inst_seq_dealloc(_PyInstructionSequence *seq) +{ + PyObject_GC_UnTrack(seq); + Py_TRASHCAN_BEGIN(seq, inst_seq_dealloc) + PyInstructionSequence_Fini(seq); + PyObject_GC_Del(seq); + Py_TRASHCAN_END +} + +static int +inst_seq_traverse(_PyInstructionSequence *seq, visitproc visit, void *arg) +{ + Py_VISIT(seq->s_nested); + return 0; +} + +static int +inst_seq_clear(_PyInstructionSequence *seq) +{ + Py_CLEAR(seq->s_nested); + return 0; +} + +PyTypeObject _PyInstructionSequence_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "InstructionSequence", + sizeof(_PyInstructionSequence), + 0, + (destructor)inst_seq_dealloc, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ + inst_seq_new__doc__, /* tp_doc */ + (traverseproc)inst_seq_traverse, /* tp_traverse */ + (inquiry)inst_seq_clear, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + inst_seq_methods, /* tp_methods */ + inst_seq_memberlist, /* tp_members */ + inst_seq_getsetters, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + inst_seq_new, /* tp_new */ +}; diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 018cd662b1561a..0e7b4810726434 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -6,6 +6,7 @@ #include "pycore_call.h" #include "pycore_ceval.h" // _PY_EVAL_EVENTS_BITS #include "pycore_code.h" // _PyCode_Clear_Executors() +#include "pycore_critical_section.h" #include "pycore_frame.h" #include "pycore_interp.h" #include "pycore_long.h" @@ -13,19 +14,63 @@ #include "pycore_namespace.h" #include "pycore_object.h" #include "pycore_opcode_metadata.h" // IS_VALID_OPCODE, _PyOpcode_Caches +#include "pycore_opcode_utils.h" // IS_CONDITIONAL_JUMP_OPCODE +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_UINTPTR_RELEASE #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() /* Uncomment this to dump debugging output when assertions fail */ // #define INSTRUMENT_DEBUG 1 +#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED) + +#define ASSERT_WORLD_STOPPED_OR_LOCKED(obj) \ + if (!_PyInterpreterState_GET()->stoptheworld.world_stopped) { \ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); \ + } +#define ASSERT_WORLD_STOPPED() assert(_PyInterpreterState_GET()->stoptheworld.world_stopped); + +#else + +#define ASSERT_WORLD_STOPPED_OR_LOCKED(obj) +#define ASSERT_WORLD_STOPPED() + +#endif + +#ifdef Py_GIL_DISABLED + +#define LOCK_CODE(code) \ + assert(!_PyInterpreterState_GET()->stoptheworld.world_stopped); \ + Py_BEGIN_CRITICAL_SECTION(code) + +#define UNLOCK_CODE() Py_END_CRITICAL_SECTION() + +#define MODIFY_BYTECODE(code, func, ...) \ + do { \ + PyCodeObject *co = (code); \ + for (Py_ssize_t i = 0; i < code->co_tlbc->size; i++) { \ + char *bc = co->co_tlbc->entries[i]; \ + if (bc == NULL) { \ + continue; \ + } \ + (func)(code, (_Py_CODEUNIT *)bc, __VA_ARGS__); \ + } \ + } while (0) + +#else + +#define LOCK_CODE(code) +#define UNLOCK_CODE() +#define MODIFY_BYTECODE(code, func, ...) \ + (func)(code, _PyCode_CODE(code), __VA_ARGS__) + +#endif + PyObject _PyInstrumentation_DISABLE = _PyObject_HEAD_INIT(&PyBaseObject_Type); PyObject _PyInstrumentation_MISSING = _PyObject_HEAD_INIT(&PyBaseObject_Type); static const int8_t EVENT_FOR_OPCODE[256] = { - [RETURN_CONST] = PY_MONITORING_EVENT_PY_RETURN, - [INSTRUMENTED_RETURN_CONST] = PY_MONITORING_EVENT_PY_RETURN, [RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, [INSTRUMENTED_RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, [CALL] = PY_MONITORING_EVENT_CALL, @@ -41,28 +86,31 @@ static const int8_t EVENT_FOR_OPCODE[256] = { [INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, [JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, [JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, - [POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT, [INSTRUMENTED_JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, [INSTRUMENTED_JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, - [INSTRUMENTED_POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, - [INSTRUMENTED_POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, - [INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, - [FOR_ITER] = PY_MONITORING_EVENT_BRANCH, - [INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [FOR_ITER] = PY_MONITORING_EVENT_BRANCH_LEFT, + [INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH_LEFT, + [POP_ITER] = PY_MONITORING_EVENT_BRANCH_RIGHT, + [INSTRUMENTED_POP_ITER] = PY_MONITORING_EVENT_BRANCH_RIGHT, [END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION, [INSTRUMENTED_END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION, [END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION, [INSTRUMENTED_END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION, + [NOT_TAKEN] = PY_MONITORING_EVENT_BRANCH_LEFT, + [INSTRUMENTED_NOT_TAKEN] = PY_MONITORING_EVENT_BRANCH_LEFT, }; static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_RESUME] = RESUME, [INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE, - [INSTRUMENTED_RETURN_CONST] = RETURN_CONST, [INSTRUMENTED_CALL] = CALL, [INSTRUMENTED_CALL_KW] = CALL_KW, [INSTRUMENTED_CALL_FUNCTION_EX] = CALL_FUNCTION_EX, @@ -74,14 +122,14 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE, [INSTRUMENTED_FOR_ITER] = FOR_ITER, + [INSTRUMENTED_POP_ITER] = POP_ITER, [INSTRUMENTED_END_FOR] = END_FOR, [INSTRUMENTED_END_SEND] = END_SEND, [INSTRUMENTED_LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR, + [INSTRUMENTED_NOT_TAKEN] = NOT_TAKEN, }; static const uint8_t INSTRUMENTED_OPCODES[256] = { - [RETURN_CONST] = INSTRUMENTED_RETURN_CONST, - [INSTRUMENTED_RETURN_CONST] = INSTRUMENTED_RETURN_CONST, [RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, [CALL] = INSTRUMENTED_CALL, @@ -112,8 +160,12 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND, [FOR_ITER] = INSTRUMENTED_FOR_ITER, [INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER, + [POP_ITER] = INSTRUMENTED_POP_ITER, + [INSTRUMENTED_POP_ITER] = INSTRUMENTED_POP_ITER, [LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR, [INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR, + [NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN, + [INSTRUMENTED_NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN, [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, @@ -133,6 +185,7 @@ is_instrumented(int opcode) { assert(opcode != 0); assert(opcode != RESERVED); + assert(opcode != ENTER_EXECUTOR); return opcode >= MIN_INSTRUMENTED_OPCODE; } @@ -232,79 +285,96 @@ get_events(_Py_GlobalMonitors *m, int tool_id) return result; } -/* Line delta. - * 8 bit value. - * if line_delta == -128: - * line = None # represented as -1 - * elif line_delta == -127: - * line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); +/* Module code can have line 0, even though modules start at line 1, + * so -1 is a legal delta. */ +#define NO_LINE (-2) + +/* Returns the line delta. Defined as: + * if line is None: + * line_delta = NO_LINE * else: - * line = first_line + (offset >> OFFSET_SHIFT) + line_delta; + * line_delta = line - first_line */ - -#define NO_LINE -128 -#define COMPUTED_LINE -127 - -#define OFFSET_SHIFT 4 - -static int8_t -compute_line_delta(PyCodeObject *code, int offset, int line) +static int +compute_line_delta(PyCodeObject *code, int line) { if (line < 0) { + assert(line == -1); return NO_LINE; } - int delta = line - code->co_firstlineno - (offset >> OFFSET_SHIFT); - if (delta <= INT8_MAX && delta > COMPUTED_LINE) { - return delta; - } - return COMPUTED_LINE; + int delta = line - code->co_firstlineno; + assert(delta > NO_LINE); + return delta; } static int -compute_line(PyCodeObject *code, int offset, int8_t line_delta) +compute_line(PyCodeObject *code, int line_delta) { - if (line_delta > COMPUTED_LINE) { - return code->co_firstlineno + (offset >> OFFSET_SHIFT) + line_delta; - } if (line_delta == NO_LINE) { - return -1; } - assert(line_delta == COMPUTED_LINE); - /* Look it up */ - return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); + assert(line_delta > NO_LINE); + return code->co_firstlineno + line_delta; } int _PyInstruction_GetLength(PyCodeObject *code, int offset) { - int opcode = _PyCode_CODE(code)[offset].op.code; - assert(opcode != 0); - assert(opcode != RESERVED); - if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_monitoring->lines[offset].original_opcode; - } - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_monitoring->per_instruction_opcodes[offset]; - } - int deinstrumented = DE_INSTRUMENT[opcode]; - if (deinstrumented) { - opcode = deinstrumented; - } - else { - opcode = _PyOpcode_Deopt[opcode]; + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + + _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(code, offset); + return 1 + _PyOpcode_Caches[inst.op.code]; +} + +static inline uint8_t +get_original_opcode(_PyCoLineInstrumentationData *line_data, int index) +{ + return line_data->data[index*line_data->bytes_per_entry]; +} + +static inline uint8_t * +get_original_opcode_ptr(_PyCoLineInstrumentationData *line_data, int index) +{ + return &line_data->data[index*line_data->bytes_per_entry]; +} + +static inline void +set_original_opcode(_PyCoLineInstrumentationData *line_data, int index, uint8_t opcode) +{ + line_data->data[index*line_data->bytes_per_entry] = opcode; +} + +static inline int +get_line_delta(_PyCoLineInstrumentationData *line_data, int index) +{ + uint8_t *ptr = &line_data->data[index*line_data->bytes_per_entry+1]; + assert(line_data->bytes_per_entry >= 2); + uint32_t value = *ptr; + for (int idx = 2; idx < line_data->bytes_per_entry; idx++) { + ptr++; + int shift = (idx-1)*8; + value |= ((uint32_t)(*ptr)) << shift; } - assert(opcode != 0); - assert(!is_instrumented(opcode)); - if (opcode == ENTER_EXECUTOR) { - int exec_index = _PyCode_CODE(code)[offset].op.arg; - _PyExecutorObject *exec = code->co_executors->executors[exec_index]; - opcode = _PyOpcode_Deopt[exec->vm_data.opcode]; + assert(value < INT_MAX); + /* NO_LINE is stored as zero. */ + return ((int)value) + NO_LINE; +} +static inline void +set_line_delta(_PyCoLineInstrumentationData *line_data, int index, int line_delta) +{ + /* Store line_delta + 2 as we need -2 to represent no line number */ + assert(line_delta >= NO_LINE); + uint32_t adjusted = line_delta - NO_LINE; + uint8_t *ptr = &line_data->data[index*line_data->bytes_per_entry+1]; + assert(adjusted < (1ULL << ((line_data->bytes_per_entry-1)*8))); + assert(line_data->bytes_per_entry >= 2); + *ptr = adjusted & 0xff; + for (int idx = 2; idx < line_data->bytes_per_entry; idx++) { + ptr++; + adjusted >>= 8; + *ptr = adjusted & 0xff; } - assert(opcode != ENTER_EXECUTOR); - assert(opcode == _PyOpcode_Deopt[opcode]); - return 1 + _PyOpcode_Caches[opcode]; } #ifdef INSTRUMENT_DEBUG @@ -326,11 +396,15 @@ dump_instrumentation_data_lines(PyCodeObject *code, _PyCoLineInstrumentationData if (lines == NULL) { fprintf(out, ", lines = NULL"); } - else if (lines[i].original_opcode == 0) { - fprintf(out, ", lines = {original_opcode = No LINE (0), line_delta = %d)", lines[i].line_delta); - } else { - fprintf(out, ", lines = {original_opcode = %s, line_delta = %d)", _PyOpcode_OpName[lines[i].original_opcode], lines[i].line_delta); + int opcode = get_original_opcode(lines, i); + int line_delta = get_line_delta(lines, i); + if (opcode == 0) { + fprintf(out, ", lines = {original_opcode = No LINE (0), line_delta = %d)", line_delta); + } + else { + fprintf(out, ", lines = {original_opcode = %s, line_delta = %d)", _PyOpcode_OpName[opcode], line_delta); + } } } @@ -380,6 +454,12 @@ dump_local_monitors(const char *prefix, _Py_LocalMonitors monitors, FILE*out) } } +/** NOTE: + * Do not use PyCode_Addr2Line to determine the line number in instrumentation, + * as `PyCode_Addr2Line` uses the monitoring data if it is available. + */ + + /* No error checking -- Don't use this for anything but experimental debugging */ static void dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) @@ -397,6 +477,8 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) dump_local_monitors("Active", data->active_monitors, out); int code_len = (int)Py_SIZE(code); bool starred = false; + PyCodeAddressRange range; + _PyCode_InitAddressRange(code, &range); for (int i = 0; i < code_len; i += _PyInstruction_GetLength(code, i)) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = instr->op.code; @@ -404,7 +486,7 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) fprintf(out, "** "); starred = true; } - fprintf(out, "Offset: %d, line: %d %s: ", i, PyCode_Addr2Line(code, i*2), _PyOpcode_OpName[opcode]); + fprintf(out, "Offset: %d, line: %d %s: ", i, _PyCode_CheckLineNumber(i*2, &range), _PyOpcode_OpName[opcode]); dump_instrumentation_data_tools(code, data->tools, i, out); dump_instrumentation_data_lines(code, data->lines, i, out); dump_instrumentation_data_line_tools(code, data->line_tools, i, out); @@ -449,6 +531,8 @@ valid_opcode(int opcode) static void sanity_check_instrumentation(PyCodeObject *code) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + _PyCoMonitoringData *data = code->_co_monitoring; if (data == NULL) { return; @@ -467,10 +551,12 @@ sanity_check_instrumentation(PyCodeObject *code) code->_co_monitoring->active_monitors, active_monitors)); int code_len = (int)Py_SIZE(code); + PyCodeAddressRange range; + _PyCode_InitAddressRange(co, &range); for (int i = 0; i < code_len;) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = instr->op.code; - int base_opcode = _Py_GetBaseOpcode(code, i); + int base_opcode = _Py_GetBaseCodeUnit(code, i).op.code; CHECK(valid_opcode(opcode)); CHECK(valid_opcode(base_opcode)); if (opcode == INSTRUMENTED_INSTRUCTION) { @@ -478,15 +564,11 @@ sanity_check_instrumentation(PyCodeObject *code) if (!is_instrumented(opcode)) { CHECK(_PyOpcode_Deopt[opcode] == opcode); } - if (data->per_instruction_tools) { - uint8_t tools = active_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION]; - CHECK((tools & data->per_instruction_tools[i]) == data->per_instruction_tools[i]); - } } if (opcode == INSTRUMENTED_LINE) { CHECK(data->lines); - CHECK(valid_opcode(data->lines[i].original_opcode)); - opcode = data->lines[i].original_opcode; + opcode = get_original_opcode(data->lines, i); + CHECK(valid_opcode(opcode)); CHECK(opcode != END_FOR); CHECK(opcode != RESUME); CHECK(opcode != RESUME_CHECK); @@ -501,7 +583,7 @@ sanity_check_instrumentation(PyCodeObject *code) * *and* we are executing a INSTRUMENTED_LINE instruction * that has de-instrumented itself, then we will execute * an invalid INSTRUMENTED_INSTRUCTION */ - CHECK(data->lines[i].original_opcode != INSTRUMENTED_INSTRUCTION); + CHECK(get_original_opcode(data->lines, i) != INSTRUMENTED_INSTRUCTION); } if (opcode == INSTRUMENTED_INSTRUCTION) { CHECK(data->per_instruction_opcodes[i] != 0); @@ -516,9 +598,9 @@ sanity_check_instrumentation(PyCodeObject *code) } CHECK(active_monitors.tools[event] != 0); } - if (data->lines && base_opcode != END_FOR) { - int line1 = compute_line(code, i, data->lines[i].line_delta); - int line2 = PyCode_Addr2Line(code, i*sizeof(_Py_CODEUNIT)); + if (data->lines && get_original_opcode(data->lines, i)) { + int line1 = compute_line(code, get_line_delta(data->lines, i)); + int line2 = _PyCode_CheckLineNumber(i*sizeof(_Py_CODEUNIT), &range); CHECK(line1 == line2); } CHECK(valid_opcode(opcode)); @@ -546,12 +628,29 @@ sanity_check_instrumentation(PyCodeObject *code) #endif -/* Get the underlying opcode, stripping instrumentation */ -int _Py_GetBaseOpcode(PyCodeObject *code, int i) +/* Get the underlying code unit, stripping instrumentation and ENTER_EXECUTOR */ +_Py_CODEUNIT +_Py_GetBaseCodeUnit(PyCodeObject *code, int i) { - int opcode = _PyCode_CODE(code)[i].op.code; + _Py_CODEUNIT *src_instr = _PyCode_CODE(code) + i; + _Py_CODEUNIT inst = { + .cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t *)src_instr)}; + int opcode = inst.op.code; + if (opcode < MIN_INSTRUMENTED_OPCODE) { + inst.op.code = _PyOpcode_Deopt[opcode]; + assert(inst.op.code < MIN_SPECIALIZED_OPCODE); + return inst; + } + if (opcode == ENTER_EXECUTOR) { + _PyExecutorObject *exec = code->co_executors->executors[inst.op.arg]; + opcode = _PyOpcode_Deopt[exec->vm_data.opcode]; + inst.op.code = opcode; + inst.op.arg = exec->vm_data.oparg; + assert(inst.op.code < MIN_SPECIALIZED_OPCODE); + return inst; + } if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_monitoring->lines[i].original_opcode; + opcode = get_original_opcode(code->_co_monitoring->lines, i); } if (opcode == INSTRUMENTED_INSTRUCTION) { opcode = code->_co_monitoring->per_instruction_opcodes[i]; @@ -560,27 +659,32 @@ int _Py_GetBaseOpcode(PyCodeObject *code, int i) CHECK(opcode != INSTRUMENTED_LINE); int deinstrumented = DE_INSTRUMENT[opcode]; if (deinstrumented) { - return deinstrumented; + inst.op.code = deinstrumented; + } + else { + inst.op.code = _PyOpcode_Deopt[opcode]; } - return _PyOpcode_Deopt[opcode]; + assert(inst.op.code < MIN_SPECIALIZED_OPCODE); + return inst; } static void -de_instrument(PyCodeObject *code, int i, int event) +de_instrument(PyCodeObject *code, _Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring, int i, + int event) { assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(event != PY_MONITORING_EVENT_LINE); - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + _Py_CODEUNIT *instr = &bytecode[i]; uint8_t *opcode_ptr = &instr->op.code; int opcode = *opcode_ptr; assert(opcode != ENTER_EXECUTOR); if (opcode == INSTRUMENTED_LINE) { - opcode_ptr = &code->_co_monitoring->lines[i].original_opcode; + opcode_ptr = get_original_opcode_ptr(monitoring->lines, i); opcode = *opcode_ptr; } if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; + opcode_ptr = &monitoring->per_instruction_opcodes[i]; opcode = *opcode_ptr; } int deinstrumented = DE_INSTRUMENT[opcode]; @@ -588,74 +692,75 @@ de_instrument(PyCodeObject *code, int i, int event) return; } CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented); - *opcode_ptr = deinstrumented; + FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, deinstrumented); if (_PyOpcode_Caches[deinstrumented]) { - instr[1].cache = adaptive_counter_warmup(); + FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff, + adaptive_counter_warmup().value_and_backoff); } } static void -de_instrument_line(PyCodeObject *code, int i) +de_instrument_line(PyCodeObject *code, _Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring, + int i) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + _Py_CODEUNIT *instr = &bytecode[i]; int opcode = instr->op.code; if (opcode != INSTRUMENTED_LINE) { return; } - _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; - int original_opcode = lines->original_opcode; + _PyCoLineInstrumentationData *lines = monitoring->lines; + int original_opcode = get_original_opcode(lines, i); if (original_opcode == INSTRUMENTED_INSTRUCTION) { - lines->original_opcode = code->_co_monitoring->per_instruction_opcodes[i]; + set_original_opcode(lines, i, monitoring->per_instruction_opcodes[i]); } CHECK(original_opcode != 0); CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); - instr->op.code = original_opcode; + FT_ATOMIC_STORE_UINT8(instr->op.code, original_opcode); if (_PyOpcode_Caches[original_opcode]) { - instr[1].cache = adaptive_counter_warmup(); + FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff, + adaptive_counter_warmup().value_and_backoff); } assert(instr->op.code != INSTRUMENTED_LINE); } static void -de_instrument_per_instruction(PyCodeObject *code, int i) +de_instrument_per_instruction(PyCodeObject *code, _Py_CODEUNIT *bytecode, + _PyCoMonitoringData *monitoring, int i) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + _Py_CODEUNIT *instr = &bytecode[i]; uint8_t *opcode_ptr = &instr->op.code; int opcode = *opcode_ptr; if (opcode == INSTRUMENTED_LINE) { - opcode_ptr = &code->_co_monitoring->lines[i].original_opcode; + opcode_ptr = get_original_opcode_ptr(monitoring->lines, i); opcode = *opcode_ptr; } if (opcode != INSTRUMENTED_INSTRUCTION) { return; } - int original_opcode = code->_co_monitoring->per_instruction_opcodes[i]; + int original_opcode = monitoring->per_instruction_opcodes[i]; CHECK(original_opcode != 0); CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); - *opcode_ptr = original_opcode; + FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, original_opcode); if (_PyOpcode_Caches[original_opcode]) { - instr[1].cache = adaptive_counter_warmup(); + FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff, + adaptive_counter_warmup().value_and_backoff); } assert(*opcode_ptr != INSTRUMENTED_INSTRUCTION); assert(instr->op.code != INSTRUMENTED_INSTRUCTION); - /* Keep things clean for sanity check */ - code->_co_monitoring->per_instruction_opcodes[i] = 0; } - static void -instrument(PyCodeObject *code, int i) +instrument(PyCodeObject *code, _Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring, int i) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + _Py_CODEUNIT *instr = &bytecode[i]; uint8_t *opcode_ptr = &instr->op.code; int opcode =*opcode_ptr; if (opcode == INSTRUMENTED_LINE) { - _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; - opcode_ptr = &lines->original_opcode; + opcode_ptr = get_original_opcode_ptr(monitoring->lines, i); opcode = *opcode_ptr; } if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; + opcode_ptr = &monitoring->per_instruction_opcodes[i]; opcode = *opcode_ptr; CHECK(opcode != INSTRUMENTED_INSTRUCTION && opcode != INSTRUMENTED_LINE); CHECK(opcode == _PyOpcode_Deopt[opcode]); @@ -665,77 +770,78 @@ instrument(PyCodeObject *code, int i) int deopt = _PyOpcode_Deopt[opcode]; int instrumented = INSTRUMENTED_OPCODES[deopt]; assert(instrumented); - *opcode_ptr = instrumented; + FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, instrumented); if (_PyOpcode_Caches[deopt]) { - instr[1].cache = adaptive_counter_warmup(); + FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff, + adaptive_counter_warmup().value_and_backoff); } } } static void -instrument_line(PyCodeObject *code, int i) +instrument_line(PyCodeObject *code, _Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring, int i) { - uint8_t *opcode_ptr = &_PyCode_CODE(code)[i].op.code; + uint8_t *opcode_ptr = &bytecode[i].op.code; int opcode = *opcode_ptr; if (opcode == INSTRUMENTED_LINE) { return; } - _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; - lines->original_opcode = _PyOpcode_Deopt[opcode]; - CHECK(lines->original_opcode > 0); - *opcode_ptr = INSTRUMENTED_LINE; + set_original_opcode(monitoring->lines, i, _PyOpcode_Deopt[opcode]); + CHECK(get_line_delta(monitoring->lines, i) > NO_LINE); + FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, INSTRUMENTED_LINE); } static void -instrument_per_instruction(PyCodeObject *code, int i) +instrument_per_instruction(PyCodeObject *code, _Py_CODEUNIT *bytecode, + _PyCoMonitoringData *monitoring, int i) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + _Py_CODEUNIT *instr = &bytecode[i]; uint8_t *opcode_ptr = &instr->op.code; int opcode = *opcode_ptr; if (opcode == INSTRUMENTED_LINE) { - _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; - opcode_ptr = &lines->original_opcode; + opcode_ptr = get_original_opcode_ptr(monitoring->lines, i); opcode = *opcode_ptr; } if (opcode == INSTRUMENTED_INSTRUCTION) { - assert(code->_co_monitoring->per_instruction_opcodes[i] > 0); + assert(monitoring->per_instruction_opcodes[i] > 0); return; } CHECK(opcode != 0); if (is_instrumented(opcode)) { - code->_co_monitoring->per_instruction_opcodes[i] = opcode; + monitoring->per_instruction_opcodes[i] = opcode; } else { assert(opcode != 0); assert(_PyOpcode_Deopt[opcode] != 0); assert(_PyOpcode_Deopt[opcode] != RESUME); - code->_co_monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode]; + monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode]; } - assert(code->_co_monitoring->per_instruction_opcodes[i] > 0); - *opcode_ptr = INSTRUMENTED_INSTRUCTION; + assert(monitoring->per_instruction_opcodes[i] > 0); + FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, INSTRUMENTED_INSTRUCTION); } static void remove_tools(PyCodeObject * code, int offset, int event, int tools) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event)); - assert(opcode_has_event(_Py_GetBaseOpcode(code, offset))); + assert(opcode_has_event(_Py_GetBaseCodeUnit(code, offset).op.code)); _PyCoMonitoringData *monitoring = code->_co_monitoring; + bool should_de_instrument; if (monitoring && monitoring->tools) { monitoring->tools[offset] &= ~tools; - if (monitoring->tools[offset] == 0) { - de_instrument(code, offset, event); - } + should_de_instrument = (monitoring->tools[offset] == 0); } else { /* Single tool */ uint8_t single_tool = code->_co_monitoring->active_monitors.tools[event]; assert(_Py_popcount32(single_tool) <= 1); - if (((single_tool & tools) == single_tool)) { - de_instrument(code, offset, event); - } + should_de_instrument = ((single_tool & tools) == single_tool); + } + if (should_de_instrument) { + MODIFY_BYTECODE(code, de_instrument, monitoring, offset, event); } } @@ -752,28 +858,32 @@ tools_is_subset_for_event(PyCodeObject * code, int event, int tools) static void remove_line_tools(PyCodeObject * code, int offset, int tools) { - assert(code->_co_monitoring); - if (code->_co_monitoring->line_tools) + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + + _PyCoMonitoringData *monitoring = code->_co_monitoring; + assert(monitoring); + bool should_de_instrument; + if (monitoring->line_tools) { - uint8_t *toolsptr = &code->_co_monitoring->line_tools[offset]; + uint8_t *toolsptr = &monitoring->line_tools[offset]; *toolsptr &= ~tools; - if (*toolsptr == 0 ) { - de_instrument_line(code, offset); - } + should_de_instrument = (*toolsptr == 0); } else { /* Single tool */ - uint8_t single_tool = code->_co_monitoring->active_monitors.tools[PY_MONITORING_EVENT_LINE]; + uint8_t single_tool = monitoring->active_monitors.tools[PY_MONITORING_EVENT_LINE]; assert(_Py_popcount32(single_tool) <= 1); - if (((single_tool & tools) == single_tool)) { - de_instrument_line(code, offset); - } + should_de_instrument = ((single_tool & tools) == single_tool); + } + if (should_de_instrument) { + MODIFY_BYTECODE(code, de_instrument_line, monitoring, offset); } } static void add_tools(PyCodeObject * code, int offset, int event, int tools) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event)); @@ -788,12 +898,14 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) assert(_Py_popcount32(tools) == 1); assert(tools_is_subset_for_event(code, event, tools)); } - instrument(code, offset); + MODIFY_BYTECODE(code, instrument, code->_co_monitoring, offset); } static void add_line_tools(PyCodeObject * code, int offset, int tools) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_LINE, tools)); assert(code->_co_monitoring); if (code->_co_monitoring->line_tools) { @@ -803,13 +915,15 @@ add_line_tools(PyCodeObject * code, int offset, int tools) /* Single tool */ assert(_Py_popcount32(tools) == 1); } - instrument_line(code, offset); + MODIFY_BYTECODE(code, instrument_line, code->_co_monitoring, offset); } static void add_per_instruction_tools(PyCodeObject * code, int offset, int tools) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_INSTRUCTION, tools)); assert(code->_co_monitoring); if (code->_co_monitoring->per_instruction_tools) { @@ -819,28 +933,31 @@ add_per_instruction_tools(PyCodeObject * code, int offset, int tools) /* Single tool */ assert(_Py_popcount32(tools) == 1); } - instrument_per_instruction(code, offset); + MODIFY_BYTECODE(code, instrument_per_instruction, code->_co_monitoring, offset); } static void remove_per_instruction_tools(PyCodeObject * code, int offset, int tools) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + + _PyCoMonitoringData *monitoring = code->_co_monitoring; assert(code->_co_monitoring); + bool should_de_instrument; if (code->_co_monitoring->per_instruction_tools) { uint8_t *toolsptr = &code->_co_monitoring->per_instruction_tools[offset]; *toolsptr &= ~tools; - if (*toolsptr == 0) { - de_instrument_per_instruction(code, offset); - } + should_de_instrument = (*toolsptr == 0); } else { /* Single tool */ uint8_t single_tool = code->_co_monitoring->active_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION]; assert(_Py_popcount32(single_tool) <= 1); - if (((single_tool & tools) == single_tool)) { - de_instrument_per_instruction(code, offset); - } + should_de_instrument = ((single_tool & tools) == single_tool); + } + if (should_de_instrument) { + MODIFY_BYTECODE(code, de_instrument_per_instruction, monitoring, offset); } } @@ -849,7 +966,7 @@ remove_per_instruction_tools(PyCodeObject * code, int offset, int tools) static int call_one_instrument( PyInterpreterState *interp, PyThreadState *tstate, PyObject **args, - Py_ssize_t nargsf, int8_t tool, int event) + size_t nargsf, int8_t tool, int event) { assert(0 <= tool && tool < 8); assert(tstate->tracing == 0); @@ -924,13 +1041,10 @@ set_global_version(PyThreadState *tstate, uint32_t version) #ifdef Py_GIL_DISABLED // Set the version on all threads in free-threaded builds. - _PyRuntimeState *runtime = &_PyRuntime; - HEAD_LOCK(runtime); - for (tstate = interp->threads.head; tstate; - tstate = PyThreadState_Next(tstate)) { + _Py_FOR_EACH_TSTATE_BEGIN(interp, tstate) { set_version_raw(&tstate->eval_breaker, version); }; - HEAD_UNLOCK(runtime); + _Py_FOR_EACH_TSTATE_END(interp); #else // Normal builds take the current version from instrumentation_version when // attaching a thread, so we only have to set the current thread's version. @@ -941,6 +1055,7 @@ set_global_version(PyThreadState *tstate, uint32_t version) static bool is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); return global_version(interp) == code->_co_instrumentation_version; } @@ -948,11 +1063,24 @@ is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp) static bool instrumentation_cross_checks(PyInterpreterState *interp, PyCodeObject *code) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); _Py_LocalMonitors expected = local_union( interp->monitors, code->_co_monitoring->local_monitors); return monitors_equals(code->_co_monitoring->active_monitors, expected); } + +static int +debug_check_sanity(PyInterpreterState *interp, PyCodeObject *code) +{ + int res; + LOCK_CODE(code); + res = is_version_up_to_date(code, interp) && + instrumentation_cross_checks(interp, code); + UNLOCK_CODE(); + return res; +} + #endif static inline uint8_t @@ -967,8 +1095,7 @@ get_tools_for_instruction(PyCodeObject *code, PyInterpreterState *interp, int i, event = PY_MONITORING_EVENT_CALL; } if (PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) { - CHECK(is_version_up_to_date(code, interp)); - CHECK(instrumentation_cross_checks(interp, code)); + CHECK(debug_check_sanity(interp, code)); if (code->_co_monitoring->tools) { tools = code->_co_monitoring->tools[i]; } @@ -992,6 +1119,8 @@ static const char *const event_names [] = { [PY_MONITORING_EVENT_INSTRUCTION] = "INSTRUCTION", [PY_MONITORING_EVENT_JUMP] = "JUMP", [PY_MONITORING_EVENT_BRANCH] = "BRANCH", + [PY_MONITORING_EVENT_BRANCH_LEFT] = "BRANCH_LEFT", + [PY_MONITORING_EVENT_BRANCH_RIGHT] = "BRANCH_RIGHT", [PY_MONITORING_EVENT_C_RETURN] = "C_RETURN", [PY_MONITORING_EVENT_PY_THROW] = "PY_THROW", [PY_MONITORING_EVENT_RAISE] = "RAISE", @@ -1004,8 +1133,8 @@ static const char *const event_names [] = { static int call_instrumentation_vector( - PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargs, PyObject *args[]) + _Py_CODEUNIT *instr, PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *arg2, Py_ssize_t nargs, PyObject *args[]) { if (tstate->tracing) { return 0; @@ -1015,19 +1144,19 @@ call_instrumentation_vector( PyCodeObject *code = _PyFrame_GetCode(frame); assert(args[1] == NULL); args[1] = (PyObject *)code; - int offset = (int)(instr - _PyCode_CODE(code)); + int offset = (int)(instr - _PyFrame_GetBytecode(frame)); /* Offset visible to user should be the offset in bytes, as that is the * convention for APIs involving code offsets. */ - int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT); - PyObject *offset_obj = PyLong_FromLong(bytes_offset); - if (offset_obj == NULL) { + int bytes_arg2 = (int)(arg2 - _PyFrame_GetBytecode(frame)) * (int)sizeof(_Py_CODEUNIT); + PyObject *arg2_obj = PyLong_FromLong(bytes_arg2); + if (arg2_obj == NULL) { return -1; } assert(args[2] == NULL); - args[2] = offset_obj; + args[2] = arg2_obj; PyInterpreterState *interp = tstate->interp; uint8_t tools = get_tools_for_instruction(code, interp, offset, event); - Py_ssize_t nargsf = nargs | PY_VECTORCALL_ARGUMENTS_OFFSET; + size_t nargsf = (size_t) nargs | PY_VECTORCALL_ARGUMENTS_OFFSET; PyObject **callargs = &args[1]; int err = 0; while (tools) { @@ -1056,11 +1185,13 @@ call_instrumentation_vector( break; } else { + LOCK_CODE(code); remove_tools(code, offset, event, 1 << tool); + UNLOCK_CODE(); } } } - Py_DECREF(offset_obj); + Py_DECREF(arg2_obj); return err; } @@ -1070,7 +1201,7 @@ _Py_call_instrumentation( _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { PyObject *args[3] = { NULL, NULL, NULL }; - return call_instrumentation_vector(tstate, event, frame, instr, 2, args); + return call_instrumentation_vector(instr, tstate, event, frame, instr, 2, args); } int @@ -1079,7 +1210,7 @@ _Py_call_instrumentation_arg( _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) { PyObject *args[4] = { NULL, NULL, NULL, arg }; - return call_instrumentation_vector(tstate, event, frame, instr, 3, args); + return call_instrumentation_vector(instr, tstate, event, frame, instr, 3, args); } int @@ -1088,34 +1219,34 @@ _Py_call_instrumentation_2args( _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1) { PyObject *args[5] = { NULL, NULL, NULL, arg0, arg1 }; - return call_instrumentation_vector(tstate, event, frame, instr, 4, args); + return call_instrumentation_vector(instr, tstate, event, frame, instr, 4, args); } _Py_CODEUNIT * _Py_call_instrumentation_jump( - PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target) + _Py_CODEUNIT *instr, PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *dest) { assert(event == PY_MONITORING_EVENT_JUMP || - event == PY_MONITORING_EVENT_BRANCH); - assert(frame->instr_ptr == instr); - PyCodeObject *code = _PyFrame_GetCode(frame); - int to = (int)(target - _PyCode_CODE(code)); + event == PY_MONITORING_EVENT_BRANCH_RIGHT || + event == PY_MONITORING_EVENT_BRANCH_LEFT); + int to = (int)(dest - _PyFrame_GetBytecode(frame)); PyObject *to_obj = PyLong_FromLong(to * (int)sizeof(_Py_CODEUNIT)); if (to_obj == NULL) { return NULL; } PyObject *args[4] = { NULL, NULL, NULL, to_obj }; - int err = call_instrumentation_vector(tstate, event, frame, instr, 3, args); + _Py_CODEUNIT *instr_ptr = frame->instr_ptr; + int err = call_instrumentation_vector(instr, tstate, event, frame, src, 3, args); Py_DECREF(to_obj); if (err) { return NULL; } - if (frame->instr_ptr != instr) { + if (frame->instr_ptr != instr_ptr) { /* The callback has caused a jump (by setting the line number) */ return frame->instr_ptr; } - return target; + return dest; } static void @@ -1125,7 +1256,7 @@ call_instrumentation_vector_protected( { assert(_PyErr_Occurred(tstate)); PyObject *exc = _PyErr_GetRaisedException(tstate); - int err = call_instrumentation_vector(tstate, event, frame, instr, nargs, args); + int err = call_instrumentation_vector(instr, tstate, event, frame, instr, nargs, args); if (err) { Py_XDECREF(exc); } @@ -1145,18 +1276,16 @@ _Py_call_instrumentation_exc2( call_instrumentation_vector_protected(tstate, event, frame, instr, 4, args); } - int _Py_Instrumentation_GetLine(PyCodeObject *code, int index) { _PyCoMonitoringData *monitoring = code->_co_monitoring; assert(monitoring != NULL); assert(monitoring->lines != NULL); - assert(index >= code->_co_firsttraceable); assert(index < Py_SIZE(code)); - _PyCoLineInstrumentationData *line_data = &monitoring->lines[index]; - int8_t line_delta = line_data->line_delta; - int line = compute_line(code, index, line_delta); + _PyCoLineInstrumentationData *line_data = monitoring->lines; + int line_delta = get_line_delta(line_data, index); + int line = compute_line(code, line_delta); return line; } @@ -1164,31 +1293,29 @@ int _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev) { PyCodeObject *code = _PyFrame_GetCode(frame); - assert(is_version_up_to_date(code, tstate->interp)); - assert(instrumentation_cross_checks(tstate->interp, code)); - int i = (int)(instr - _PyCode_CODE(code)); + assert(tstate->tracing == 0); + assert(debug_check_sanity(tstate->interp, code)); + _Py_CODEUNIT *bytecode = _PyFrame_GetBytecode(frame); + int i = (int)(instr - bytecode); _PyCoMonitoringData *monitoring = code->_co_monitoring; - _PyCoLineInstrumentationData *line_data = &monitoring->lines[i]; - if (tstate->tracing) { - goto done; - } + _PyCoLineInstrumentationData *line_data = monitoring->lines; PyInterpreterState *interp = tstate->interp; - int8_t line_delta = line_data->line_delta; - int line = compute_line(code, i, line_delta); + int line = _Py_Instrumentation_GetLine(code, i); assert(line >= 0); assert(prev != NULL); - int prev_index = (int)(prev - _PyCode_CODE(code)); + int prev_index = (int)(prev - bytecode); int prev_line = _Py_Instrumentation_GetLine(code, prev_index); if (prev_line == line) { - int prev_opcode = _PyCode_CODE(code)[prev_index].op.code; + int prev_opcode = bytecode[prev_index].op.code; /* RESUME and INSTRUMENTED_RESUME are needed for the operation of - * instrumentation, so must never be hidden by an INSTRUMENTED_LINE. - */ + * instrumentation, so must never be hidden by an INSTRUMENTED_LINE. + */ if (prev_opcode != RESUME && prev_opcode != INSTRUMENTED_RESUME) { goto done; } } + uint8_t tools = code->_co_monitoring->line_tools != NULL ? code->_co_monitoring->line_tools[i] : (interp->monitors.tools[PY_MONITORING_EVENT_LINE] | @@ -1197,7 +1324,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, /* Special case sys.settrace to avoid boxing the line number, * only to immediately unbox it. */ if (tools & (1 << PY_MONITORING_SYS_TRACE_ID)) { - if (tstate->c_tracefunc != NULL && line >= 0) { + if (tstate->c_tracefunc != NULL) { PyFrameObject *frame_obj = _PyFrame_GetFrameObject(frame); if (frame_obj == NULL) { return -1; @@ -1249,13 +1376,15 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, } else { /* DISABLE */ + LOCK_CODE(code); remove_line_tools(code, i, 1 << tool); + UNLOCK_CODE(); } } while (tools); Py_DECREF(line_obj); uint8_t original_opcode; done: - original_opcode = line_data->original_opcode; + original_opcode = get_original_opcode(line_data, i); assert(original_opcode != 0); assert(original_opcode != INSTRUMENTED_LINE); assert(_PyOpcode_Deopt[original_opcode] == original_opcode); @@ -1266,15 +1395,14 @@ int _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr) { PyCodeObject *code = _PyFrame_GetCode(frame); - assert(is_version_up_to_date(code, tstate->interp)); - assert(instrumentation_cross_checks(tstate->interp, code)); - int offset = (int)(instr - _PyCode_CODE(code)); + int offset = (int)(instr - _PyFrame_GetBytecode(frame)); _PyCoMonitoringData *instrumentation_data = code->_co_monitoring; assert(instrumentation_data->per_instruction_opcodes); int next_opcode = instrumentation_data->per_instruction_opcodes[offset]; if (tstate->tracing) { return next_opcode; } + assert(debug_check_sanity(tstate->interp, code)); PyInterpreterState *interp = tstate->interp; uint8_t tools = instrumentation_data->per_instruction_tools != NULL ? instrumentation_data->per_instruction_tools[offset] : @@ -1305,7 +1433,9 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* } else { /* DISABLE */ + LOCK_CODE(code); remove_per_instruction_tools(code, offset, 1 << tool); + UNLOCK_CODE(); } } Py_DECREF(offset_obj); @@ -1313,22 +1443,12 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* return next_opcode; } - -PyObject * -_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) -{ - PyInterpreterState *is = _PyInterpreterState_GET(); - assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); - assert(0 <= event_id && event_id < _PY_MONITORING_EVENTS); - PyObject *callback = is->monitoring_callables[tool_id][event_id]; - is->monitoring_callables[tool_id][event_id] = Py_XNewRef(obj); - return callback; -} - static void initialize_tools(PyCodeObject *code) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); uint8_t* tools = code->_co_monitoring->tools; + assert(tools != NULL); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { @@ -1336,7 +1456,7 @@ initialize_tools(PyCodeObject *code) int opcode = instr->op.code; assert(opcode != ENTER_EXECUTOR); if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_monitoring->lines[i].original_opcode; + opcode = get_original_opcode(code->_co_monitoring->lines, i); } if (opcode == INSTRUMENTED_INSTRUCTION) { opcode = code->_co_monitoring->per_instruction_opcodes[i]; @@ -1379,66 +1499,72 @@ initialize_tools(PyCodeObject *code) } } -#define NO_LINE -128 - static void -initialize_lines(PyCodeObject *code) +initialize_lines(PyCodeObject *code, int bytes_per_entry) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; + assert(line_data != NULL); + line_data->bytes_per_entry = bytes_per_entry; int code_len = (int)Py_SIZE(code); PyCodeAddressRange range; _PyCode_InitAddressRange(code, &range); - for (int i = 0; i < code->_co_firsttraceable && i < code_len; i++) { - line_data[i].original_opcode = 0; - line_data[i].line_delta = -127; - } int current_line = -1; - for (int i = code->_co_firsttraceable; i < code_len; ) { - int opcode = _Py_GetBaseOpcode(code, i); + for (int i = 0; i < code_len; ) { + int opcode = _Py_GetBaseCodeUnit(code, i).op.code; int line = _PyCode_CheckLineNumber(i*(int)sizeof(_Py_CODEUNIT), &range); - line_data[i].line_delta = compute_line_delta(code, i, line); + set_line_delta(line_data, i, compute_line_delta(code, line)); int length = _PyInstruction_GetLength(code, i); - switch (opcode) { - case END_ASYNC_FOR: - case END_FOR: - case END_SEND: - case RESUME: - /* END_FOR cannot start a line, as it is skipped by FOR_ITER - * END_SEND cannot start a line, as it is skipped by SEND - * RESUME must not be instrumented with INSTRUMENT_LINE */ - line_data[i].original_opcode = 0; - break; - default: - /* Set original_opcode to the opcode iff the instruction - * starts a line, and thus should be instrumented. - * This saves having to perform this check every time the - * we turn instrumentation on or off, and serves as a sanity - * check when debugging. - */ - if (line != current_line && line >= 0) { - line_data[i].original_opcode = opcode; - } - else { - line_data[i].original_opcode = 0; - } - current_line = line; + if (i < code->_co_firsttraceable) { + set_original_opcode(line_data, i, 0); + } + else { + switch (opcode) { + case END_ASYNC_FOR: + case END_FOR: + case END_SEND: + case RESUME: + case POP_ITER: + /* END_FOR cannot start a line, as it is skipped by FOR_ITER + * END_SEND cannot start a line, as it is skipped by SEND + * RESUME and POP_ITER must not be instrumented with INSTRUMENTED_LINE */ + set_original_opcode(line_data, i, 0); + break; + default: + /* Set original_opcode to the opcode iff the instruction + * starts a line, and thus should be instrumented. + * This saves having to perform this check every time the + * we turn instrumentation on or off, and serves as a sanity + * check when debugging. + */ + if (line != current_line && line >= 0) { + set_original_opcode(line_data, i, opcode); + CHECK(get_line_delta(line_data, i) != NO_LINE); + } + else { + set_original_opcode(line_data, i, 0); + } + current_line = line; + } } for (int j = 1; j < length; j++) { - line_data[i+j].original_opcode = 0; - line_data[i+j].line_delta = NO_LINE; + set_original_opcode(line_data, i+j, 0); + set_line_delta(line_data, i+j, NO_LINE); } i += length; } for (int i = code->_co_firsttraceable; i < code_len; ) { - int opcode = _Py_GetBaseOpcode(code, i); + _Py_CODEUNIT inst =_Py_GetBaseCodeUnit(code, i); + int opcode = inst.op.code; int oparg = 0; while (opcode == EXTENDED_ARG) { - oparg = (oparg << 8) | _PyCode_CODE(code)[i].op.arg; + oparg = (oparg << 8) | inst.op.arg; i++; - opcode = _Py_GetBaseOpcode(code, i); + inst =_Py_GetBaseCodeUnit(code, i); + opcode = inst.op.code; } - oparg = (oparg << 8) | _PyCode_CODE(code)[i].op.arg; + oparg = (oparg << 8) | inst.op.arg; i += _PyInstruction_GetLength(code, i); int target = -1; switch (opcode) { @@ -1468,8 +1594,11 @@ initialize_lines(PyCodeObject *code) continue; } assert(target >= 0); - if (line_data[target].line_delta != NO_LINE) { - line_data[target].original_opcode = _Py_GetBaseOpcode(code, target); + if (get_line_delta(line_data, target) != NO_LINE) { + int opcode = _Py_GetBaseCodeUnit(code, target).op.code; + if (opcode != POP_ITER) { + set_original_opcode(line_data, target, opcode); + } } } /* Scan exception table */ @@ -1486,14 +1615,13 @@ initialize_lines(PyCodeObject *code) assert(handler >= 0 && handler < code_len); int depth_and_lasti; scan = parse_varint(scan, &depth_and_lasti); - int original_opcode = _Py_GetBaseOpcode(code, handler); + int original_opcode = _Py_GetBaseCodeUnit(code, handler).op.code; /* Skip if not the start of a line. * END_ASYNC_FOR is a bit special as it marks the end of * an `async for` loop, which should not generate its own * line event. */ - if (line_data[handler].line_delta != NO_LINE && - original_opcode != END_ASYNC_FOR) { - line_data[handler].original_opcode = original_opcode; + if (get_line_delta(line_data, handler) != NO_LINE && original_opcode != END_ASYNC_FOR) { + set_original_opcode(line_data, handler, original_opcode); } } } @@ -1501,7 +1629,9 @@ initialize_lines(PyCodeObject *code) static void initialize_line_tools(PyCodeObject *code, _Py_LocalMonitors *all_events) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); uint8_t *line_tools = code->_co_monitoring->line_tools; + assert(line_tools != NULL); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { @@ -1512,6 +1642,7 @@ initialize_line_tools(PyCodeObject *code, _Py_LocalMonitors *all_events) static int allocate_instrumentation_data(PyCodeObject *code) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); if (code->_co_monitoring == NULL) { code->_co_monitoring = PyMem_Malloc(sizeof(_PyCoMonitoringData)); @@ -1533,10 +1664,22 @@ allocate_instrumentation_data(PyCodeObject *code) static int update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + int code_len = (int)Py_SIZE(code); if (allocate_instrumentation_data(code)) { return -1; } + // If the local monitors are out of date, clear them up + _Py_LocalMonitors *local_monitors = &code->_co_monitoring->local_monitors; + for (int i = 0; i < PY_MONITORING_TOOL_IDS; i++) { + if (code->_co_monitoring->tool_versions[i] != interp->monitoring_tool_versions[i]) { + for (int j = 0; j < _PY_MONITORING_LOCAL_EVENTS; j++) { + local_monitors->tools[j] &= ~(1 << i); + } + } + } + _Py_LocalMonitors all_events = local_union( interp->monitors, code->_co_monitoring->local_monitors); @@ -1551,12 +1694,39 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) } if (all_events.tools[PY_MONITORING_EVENT_LINE]) { if (code->_co_monitoring->lines == NULL) { - code->_co_monitoring->lines = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); + PyCodeAddressRange range; + _PyCode_InitAddressRange(code, &range); + int max_line = code->co_firstlineno + 1; + _PyCode_InitAddressRange(code, &range); + for (int i = code->_co_firsttraceable; i < code_len; ) { + int line = _PyCode_CheckLineNumber(i*(int)sizeof(_Py_CODEUNIT), &range); + if (line > max_line) { + max_line = line; + } + int length = _PyInstruction_GetLength(code, i); + i += length; + } + int bytes_per_entry; + int max_delta = max_line - code->co_firstlineno; + /* We store delta+2 in the table, so 253 is max for one byte */ + if (max_delta < 256+NO_LINE) { + bytes_per_entry = 2; + } + else if (max_delta < (1 << 16)+NO_LINE) { + bytes_per_entry = 3; + } + else if (max_delta < (1 << 24)+NO_LINE) { + bytes_per_entry = 4; + } + else { + bytes_per_entry = 5; + } + code->_co_monitoring->lines = PyMem_Malloc(1 + code_len * bytes_per_entry); if (code->_co_monitoring->lines == NULL) { PyErr_NoMemory(); return -1; } - initialize_lines(code); + initialize_lines(code, bytes_per_entry); } if (multitools && code->_co_monitoring->line_tools == NULL) { code->_co_monitoring->line_tools = PyMem_Malloc(code_len); @@ -1574,9 +1744,11 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) PyErr_NoMemory(); return -1; } - /* This may not be necessary, as we can initialize this memory lazily, but it helps catch errors. */ + // Initialize all of the instructions so if local events change while another thread is executing + // we know what the original opcode was. for (int i = 0; i < code_len; i++) { - code->_co_monitoring->per_instruction_opcodes[i] = 0; + int opcode = _PyCode_CODE(code)[i].op.code; + code->_co_monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode]; } } if (multitools && code->_co_monitoring->per_instruction_tools == NULL) { @@ -1585,7 +1757,6 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) PyErr_NoMemory(); return -1; } - /* This may not be necessary, as we can initialize this memory lazily, but it helps catch errors. */ for (int i = 0; i < code_len; i++) { code->_co_monitoring->per_instruction_tools[i] = 0; } @@ -1594,20 +1765,17 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) return 0; } -int -_Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) +static int +force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) { - if (is_version_up_to_date(code, interp)) { - assert( - interp->ceval.instrumentation_version == 0 || - instrumentation_cross_checks(interp, code) - ); - return 0; - } + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + +#ifdef _Py_TIER2 if (code->co_executors != NULL) { _PyCode_Clear_Executors(code); } _Py_Executors_InvalidateDependency(interp, code, 1); +#endif int code_len = (int)Py_SIZE(code); /* Exit early to avoid creating instrumentation * data for potential statically allocated code @@ -1636,24 +1804,19 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) assert(monitors_are_empty(monitors_and(new_events, removed_events))); } code->_co_monitoring->active_monitors = active_events; - code->_co_instrumentation_version = global_version(interp); if (monitors_are_empty(new_events) && monitors_are_empty(removed_events)) { -#ifdef INSTRUMENT_DEBUG - sanity_check_instrumentation(code); -#endif - return 0; + goto done; } /* Insert instrumentation */ for (int i = code->_co_firsttraceable; i < code_len; i+= _PyInstruction_GetLength(code, i)) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - CHECK(instr->op.code != 0); - assert(instr->op.code != ENTER_EXECUTOR); - int base_opcode = _Py_GetBaseOpcode(code, i); - assert(base_opcode != ENTER_EXECUTOR); + assert(_PyCode_CODE(code)[i].op.code != ENTER_EXECUTOR); + _Py_CODEUNIT instr = _Py_GetBaseCodeUnit(code, i); + CHECK(instr.op.code != 0); + int base_opcode = instr.op.code; if (opcode_has_event(base_opcode)) { int8_t event; if (base_opcode == RESUME) { - event = instr->op.arg > 0; + event = instr.op.arg > 0; } else { event = EVENT_FOR_OPCODE[base_opcode]; @@ -1672,70 +1835,93 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) // GH-103845: We need to remove both the line and instruction instrumentation before // adding new ones, otherwise we may remove the newly added instrumentation. - uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE]; uint8_t removed_per_instruction_tools = removed_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; if (removed_line_tools) { _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; for (int i = code->_co_firsttraceable; i < code_len;) { - if (line_data[i].original_opcode) { - if (removed_line_tools) { - remove_line_tools(code, i, removed_line_tools); - } + if (get_original_opcode(line_data, i)) { + remove_line_tools(code, i, removed_line_tools); } i += _PyInstruction_GetLength(code, i); } } if (removed_per_instruction_tools) { for (int i = code->_co_firsttraceable; i < code_len;) { - int opcode = _Py_GetBaseOpcode(code, i); + int opcode = _Py_GetBaseCodeUnit(code, i).op.code; if (opcode == RESUME || opcode == END_FOR) { i += _PyInstruction_GetLength(code, i); continue; } - if (removed_per_instruction_tools) { - remove_per_instruction_tools(code, i, removed_per_instruction_tools); - } + remove_per_instruction_tools(code, i, removed_per_instruction_tools); i += _PyInstruction_GetLength(code, i); } } #ifdef INSTRUMENT_DEBUG sanity_check_instrumentation(code); #endif + uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE]; uint8_t new_per_instruction_tools = new_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; if (new_line_tools) { _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; for (int i = code->_co_firsttraceable; i < code_len;) { - if (line_data[i].original_opcode) { - if (new_line_tools) { - add_line_tools(code, i, new_line_tools); - } + if (get_original_opcode(line_data, i)) { + add_line_tools(code, i, new_line_tools); } i += _PyInstruction_GetLength(code, i); } } if (new_per_instruction_tools) { for (int i = code->_co_firsttraceable; i < code_len;) { - int opcode = _Py_GetBaseOpcode(code, i); + int opcode = _Py_GetBaseCodeUnit(code, i).op.code; if (opcode == RESUME || opcode == END_FOR) { i += _PyInstruction_GetLength(code, i); continue; } - if (new_per_instruction_tools) { - add_per_instruction_tools(code, i, new_per_instruction_tools); - } + add_per_instruction_tools(code, i, new_per_instruction_tools); i += _PyInstruction_GetLength(code, i); } } + +done: + FT_ATOMIC_STORE_UINTPTR_RELEASE(code->_co_instrumentation_version, + global_version(interp)); + #ifdef INSTRUMENT_DEBUG sanity_check_instrumentation(code); #endif return 0; } +static int +instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) +{ + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + + if (is_version_up_to_date(code, interp)) { + assert( + interp->ceval.instrumentation_version == 0 || + instrumentation_cross_checks(interp, code) + ); + return 0; + } + + return force_instrument_lock_held(code, interp); +} + +int +_Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) +{ + int res; + LOCK_CODE(code); + res = instrument_lock_held(code, interp); + UNLOCK_CODE(); + return res; +} + #define C_RETURN_EVENTS \ ((1 << PY_MONITORING_EVENT_C_RETURN) | \ (1 << PY_MONITORING_EVENT_C_RAISE)) @@ -1746,6 +1932,8 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) static int instrument_all_executing_code_objects(PyInterpreterState *interp) { + ASSERT_WORLD_STOPPED(); + _PyRuntimeState *runtime = &_PyRuntime; HEAD_LOCK(runtime); PyThreadState* ts = PyInterpreterState_ThreadHead(interp); @@ -1753,8 +1941,8 @@ instrument_all_executing_code_objects(PyInterpreterState *interp) { while (ts) { _PyInterpreterFrame *frame = ts->current_frame; while (frame) { - if (frame->owner != FRAME_OWNED_BY_CSTACK) { - if (_Py_Instrument(_PyFrame_GetCode(frame), interp)) { + if (frame->owner < FRAME_OWNED_BY_INTERPRETER) { + if (instrument_lock_held(_PyFrame_GetCode(frame), interp)) { return -1; } } @@ -1817,19 +2005,29 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) if (check_tool(interp, tool_id)) { return -1; } + + int res; + _PyEval_StopTheWorld(interp); uint32_t existing_events = get_events(&interp->monitors, tool_id); if (existing_events == events) { - return 0; + res = 0; + goto done; } set_events(&interp->monitors, tool_id, events); uint32_t new_version = global_version(interp) + MONITORING_VERSION_INCREMENT; if (new_version == 0) { PyErr_Format(PyExc_OverflowError, "events set too many times"); - return -1; + res = -1; + goto done; } set_global_version(tstate, new_version); +#ifdef _Py_TIER2 _Py_Executors_InvalidateAll(interp, 1); - return instrument_all_executing_code_objects(interp); +#endif + res = instrument_all_executing_code_objects(interp); +done: + _PyEval_StartTheWorld(interp); + return res; } int @@ -1845,24 +2043,29 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent if (check_tool(interp, tool_id)) { return -1; } + + int res; + _PyEval_StopTheWorld(interp); if (allocate_instrumentation_data(code)) { - return -1; + res = -1; + goto done; } + + code->_co_monitoring->tool_versions[tool_id] = interp->monitoring_tool_versions[tool_id]; + _Py_LocalMonitors *local = &code->_co_monitoring->local_monitors; uint32_t existing_events = get_local_events(local, tool_id); if (existing_events == events) { - return 0; + res = 0; + goto done; } set_local_events(local, tool_id, events); - if (is_version_up_to_date(code, interp)) { - /* Force instrumentation update */ - code->_co_instrumentation_version -= MONITORING_VERSION_INCREMENT; - } - _Py_Executors_InvalidateDependency(interp, code, 1); - if (_Py_Instrument(code, interp)) { - return -1; - } - return 0; + + res = force_instrument_lock_held(code, interp); + +done: + _PyEval_StartTheWorld(interp); + return res; } int @@ -1882,6 +2085,43 @@ _PyMonitoring_GetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent return 0; } +int _PyMonitoring_ClearToolId(int tool_id) +{ + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); + PyInterpreterState *interp = _PyInterpreterState_GET(); + + for (int i = 0; i < _PY_MONITORING_EVENTS; i++) { + PyObject *func = _PyMonitoring_RegisterCallback(tool_id, i, NULL); + if (func != NULL) { + Py_DECREF(func); + } + } + + if (_PyMonitoring_SetEvents(tool_id, 0) < 0) { + return -1; + } + + _PyEval_StopTheWorld(interp); + uint32_t version = global_version(interp) + MONITORING_VERSION_INCREMENT; + if (version == 0) { + PyErr_Format(PyExc_OverflowError, "events set too many times"); + _PyEval_StartTheWorld(interp); + return -1; + } + + // monitoring_tool_versions[tool_id] is set to latest global version here to + // 1. invalidate local events on all existing code objects + // 2. be ready for the next call to set local events + interp->monitoring_tool_versions[tool_id] = version; + + // Set the new global version so all the code objects can refresh the + // instrumentation. + set_global_version(_PyThreadState_GET(), version); + int res = instrument_all_executing_code_objects(interp); + _PyEval_StartTheWorld(interp); + return res; +} + /*[clinic input] module monitoring [clinic start generated code]*/ @@ -1930,7 +2170,7 @@ monitoring_use_tool_id_impl(PyObject *module, int tool_id, PyObject *name) } /*[clinic input] -monitoring.free_tool_id +monitoring.clear_tool_id tool_id: int / @@ -1938,19 +2178,53 @@ monitoring.free_tool_id [clinic start generated code]*/ static PyObject * -monitoring_free_tool_id_impl(PyObject *module, int tool_id) -/*[clinic end generated code: output=86c2d2a1219a8591 input=a23fb6be3a8618e9]*/ +monitoring_clear_tool_id_impl(PyObject *module, int tool_id) +/*[clinic end generated code: output=04defc23470b1be7 input=af643d6648a66163]*/ { if (check_valid_tool(tool_id)) { return NULL; } + PyInterpreterState *interp = _PyInterpreterState_GET(); - Py_CLEAR(interp->monitoring_tool_names[tool_id]); + + if (interp->monitoring_tool_names[tool_id] != NULL) { + if (_PyMonitoring_ClearToolId(tool_id) < 0) { + return NULL; + } + } + Py_RETURN_NONE; } /*[clinic input] -monitoring.get_tool +monitoring.free_tool_id + + tool_id: int + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_free_tool_id_impl(PyObject *module, int tool_id) +/*[clinic end generated code: output=86c2d2a1219a8591 input=a23fb6be3a8618e9]*/ +{ + if (check_valid_tool(tool_id)) { + return NULL; + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + + if (interp->monitoring_tool_names[tool_id] != NULL) { + if (_PyMonitoring_ClearToolId(tool_id) < 0) { + return NULL; + } + } + + Py_CLEAR(interp->monitoring_tool_names[tool_id]); + Py_RETURN_NONE; +} + +/*[clinic input] +monitoring.get_tool tool_id: int / @@ -2060,6 +2334,10 @@ monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) return NULL; } event_set &= ~C_RETURN_EVENTS; + if (event_set & (1 << PY_MONITORING_EVENT_BRANCH)) { + event_set &= ~(1 << PY_MONITORING_EVENT_BRANCH); + event_set |= (1 << PY_MONITORING_EVENT_BRANCH_RIGHT) | (1 << PY_MONITORING_EVENT_BRANCH_LEFT); + } if (_PyMonitoring_SetEvents(tool_id, event_set)) { return NULL; } @@ -2132,6 +2410,10 @@ monitoring_set_local_events_impl(PyObject *module, int tool_id, return NULL; } event_set &= ~C_RETURN_EVENTS; + if (event_set & (1 << PY_MONITORING_EVENT_BRANCH)) { + event_set &= ~(1 << PY_MONITORING_EVENT_BRANCH); + event_set |= (1 << PY_MONITORING_EVENT_BRANCH_RIGHT) | (1 << PY_MONITORING_EVENT_BRANCH_LEFT); + } if (event_set < 0 || event_set >= (1 << _PY_MONITORING_LOCAL_EVENTS)) { PyErr_Format(PyExc_ValueError, "invalid local event set 0x%x", event_set); return NULL; @@ -2158,15 +2440,21 @@ monitoring_restart_events_impl(PyObject *module) */ PyThreadState *tstate = _PyThreadState_GET(); PyInterpreterState *interp = tstate->interp; + + _PyEval_StopTheWorld(interp); uint32_t restart_version = global_version(interp) + MONITORING_VERSION_INCREMENT; uint32_t new_version = restart_version + MONITORING_VERSION_INCREMENT; if (new_version <= MONITORING_VERSION_INCREMENT) { + _PyEval_StartTheWorld(interp); PyErr_Format(PyExc_OverflowError, "events set too many times"); return NULL; } interp->last_restart_version = restart_version; set_global_version(tstate, new_version); - if (instrument_all_executing_code_objects(interp)) { + int res = instrument_all_executing_code_objects(interp); + _PyEval_StartTheWorld(interp); + + if (res) { return NULL; } Py_RETURN_NONE; @@ -2216,6 +2504,7 @@ monitoring__all_events_impl(PyObject *module) static PyMethodDef methods[] = { MONITORING_USE_TOOL_ID_METHODDEF + MONITORING_CLEAR_TOOL_ID_METHODDEF MONITORING_FREE_TOOL_ID_METHODDEF MONITORING_GET_TOOL_METHODDEF MONITORING_REGISTER_CALLBACK_METHODDEF @@ -2284,3 +2573,576 @@ PyObject *_Py_CreateMonitoringObject(void) Py_DECREF(mod); return NULL; } + + +static int +capi_call_instrumentation(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject **args, Py_ssize_t nargs, int event) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyInterpreterState *interp = tstate->interp; + + uint8_t tools = state->active; + assert(args[1] == NULL); + args[1] = codelike; + if (offset < 0) { + PyErr_SetString(PyExc_ValueError, "offset must be non-negative"); + return -1; + } + if (event != PY_MONITORING_EVENT_LINE) { + PyObject *offset_obj = PyLong_FromLong(offset); + if (offset_obj == NULL) { + return -1; + } + assert(args[2] == NULL); + args[2] = offset_obj; + } + size_t nargsf = (size_t) nargs | PY_VECTORCALL_ARGUMENTS_OFFSET; + PyObject **callargs = &args[1]; + int err = 0; + + while (tools) { + int tool = most_significant_bit(tools); + assert(tool >= 0 && tool < 8); + assert(tools & (1 << tool)); + tools ^= (1 << tool); + int res = call_one_instrument(interp, tstate, callargs, nargsf, tool, event); + if (res == 0) { + /* Nothing to do */ + } + else if (res < 0) { + /* error */ + err = -1; + break; + } + else { + /* DISABLE */ + if (!PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) { + PyErr_Format(PyExc_ValueError, + "Cannot disable %s events. Callback removed.", + event_names[event]); + /* Clear tool to prevent infinite loop */ + Py_CLEAR(interp->monitoring_callables[tool][event]); + err = -1; + break; + } + else { + state->active &= ~(1 << tool); + } + } + } + return err; +} + +int +PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, + const uint8_t *event_types, Py_ssize_t length) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (global_version(interp) == *version) { + return 0; + } + + _Py_GlobalMonitors *m = &interp->monitors; + for (Py_ssize_t i = 0; i < length; i++) { + int event = event_types[i]; + state_array[i].active = m->tools[event]; + } + *version = global_version(interp); + return 0; +} + +int +PyMonitoring_ExitScope(void) +{ + return 0; +} + +int +_PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + assert(state->active); + PyObject *args[3] = { NULL, NULL, NULL }; + return capi_call_instrumentation(state, codelike, offset, args, 2, + PY_MONITORING_EVENT_PY_START); +} + +int +_PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + assert(state->active); + PyObject *args[3] = { NULL, NULL, NULL }; + return capi_call_instrumentation(state, codelike, offset, args, 2, + PY_MONITORING_EVENT_PY_RESUME); +} + + + +int +_PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* retval) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, retval }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_PY_RETURN); +} + +int +_PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* retval) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, retval }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_PY_YIELD); +} + +int +_PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* callable, PyObject *arg0) +{ + assert(state->active); + PyObject *args[5] = { NULL, NULL, NULL, callable, arg0 }; + return capi_call_instrumentation(state, codelike, offset, args, 4, + PY_MONITORING_EVENT_CALL); +} + +int +_PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + int lineno) +{ + assert(state->active); + PyObject *lno = PyLong_FromLong(lineno); + if (lno == NULL) { + return -1; + } + PyObject *args[3] = { NULL, NULL, lno }; + int res= capi_call_instrumentation(state, codelike, offset, args, 2, + PY_MONITORING_EVENT_LINE); + Py_DECREF(lno); + return res; +} + +int +_PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, target_offset }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_JUMP); +} + +int +_PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, target_offset }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_BRANCH_RIGHT); +} + +int +_PyMonitoring_FireBranchRightEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, target_offset }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_BRANCH_RIGHT); +} + +int +_PyMonitoring_FireBranchLeftEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, target_offset }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_BRANCH_LEFT); +} + +int +_PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, retval }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_C_RETURN); +} + +static inline int +exception_event_setup(PyObject **exc, int event) { + *exc = PyErr_GetRaisedException(); + if (*exc == NULL) { + PyErr_Format(PyExc_ValueError, + "Firing event %d with no exception set", + event); + return -1; + } + return 0; +} + + +static inline int +exception_event_teardown(int err, PyObject *exc) { + if (err == 0) { + PyErr_SetRaisedException(exc); + } + else { + assert(PyErr_Occurred()); + Py_XDECREF(exc); + } + return err; +} + +int +_PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_PY_THROW; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_RAISE; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_C_RAISE; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_RERAISE; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_EXCEPTION_HANDLED; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_PY_UNWIND; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value) +{ + int event = PY_MONITORING_EVENT_STOP_ITERATION; + assert(state->active); + assert(!PyErr_Occurred()); + PyErr_SetObject(PyExc_StopIteration, value); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + Py_DECREF(exc); + return exception_event_teardown(err, NULL); +} + + + +/* Handle legacy BRANCH event */ + +typedef struct _PyLegacyBranchEventHandler { + PyObject_HEAD + vectorcallfunc vectorcall; + PyObject *handler; + bool right; + int tool_id; +} _PyLegacyBranchEventHandler; + +static void +dealloc_branch_handler(_PyLegacyBranchEventHandler *self) +{ + Py_CLEAR(self->handler); + PyObject_Free((PyObject *)self); +} + +static PyTypeObject _PyLegacyBranchEventHandler_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "sys.monitoring.branch_event_handler", + sizeof(_PyLegacyBranchEventHandler), + .tp_dealloc = (destructor)dealloc_branch_handler, + .tp_vectorcall_offset = offsetof(_PyLegacyBranchEventHandler, vectorcall), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .tp_call = PyVectorcall_Call, +}; + + +static PyObject * +branch_handler( + _PyLegacyBranchEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + // Find the other instrumented instruction and remove tool + // The spec (PEP 669) allows spurious events after a DISABLE, + // so a best effort is good enough. + assert(PyVectorcall_NARGS(nargsf) >= 3); + PyCodeObject *code = (PyCodeObject *)args[0]; + int src_offset = PyLong_AsLong(args[1]); + if (PyErr_Occurred()) { + return NULL; + } + _Py_CODEUNIT instr = _PyCode_CODE(code)[src_offset/2]; + if (!is_instrumented(instr.op.code)) { + /* Already disabled */ + return &_PyInstrumentation_DISABLE; + } + PyObject *res = PyObject_Vectorcall(self->handler, args, nargsf, kwnames); + if (res == &_PyInstrumentation_DISABLE) { + /* We need FOR_ITER and POP_JUMP_ to be the same size */ + assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1); + int offset; + int other_event; + if (instr.op.code == FOR_ITER) { + if (self->right) { + offset = src_offset/2; + other_event = PY_MONITORING_EVENT_BRANCH_LEFT; + } + else { + // We don't know where the POP_ITER is, so + // we cannot de-instrument it. + return res; + } + } + else if (IS_CONDITIONAL_JUMP_OPCODE(instr.op.code)) { + if (self->right) { + offset = src_offset/2 + 2; + other_event = PY_MONITORING_EVENT_BRANCH_LEFT; + assert(_Py_GetBaseCodeUnit(code, offset).op.code == NOT_TAKEN); + } + else { + offset = src_offset/2; + other_event = PY_MONITORING_EVENT_BRANCH_RIGHT; + } + } + else { + // Orphaned NOT_TAKEN -- Jump removed by the compiler + return res; + } + LOCK_CODE(code); + remove_tools(code, offset, other_event, 1 << self->tool_id); + UNLOCK_CODE(); + } + return res; +} + +static PyObject *make_branch_handler(int tool_id, PyObject *handler, bool right) +{ + _PyLegacyBranchEventHandler *callback = + PyObject_NEW(_PyLegacyBranchEventHandler, &_PyLegacyBranchEventHandler_Type); + if (callback == NULL) { + return NULL; + } + callback->vectorcall = (vectorcallfunc)branch_handler; + callback->handler = Py_NewRef(handler); + callback->right = right; + callback->tool_id = tool_id; + return (PyObject *)callback; +} + +/* Consumes a reference to obj */ +static PyObject *exchange_callables(int tool_id, int event_id, PyObject *obj) +{ + PyInterpreterState *is = _PyInterpreterState_GET(); + return _Py_atomic_exchange_ptr(&is->monitoring_callables[tool_id][event_id], obj); +} + +PyObject * +_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) +{ + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); + assert(0 <= event_id && event_id < _PY_MONITORING_EVENTS); + PyObject *res; + if (event_id == PY_MONITORING_EVENT_BRANCH) { + PyObject *left, *right; + if (obj == NULL) { + left = NULL; + right = NULL; + } + else { + right = make_branch_handler(tool_id, obj, true); + if (right == NULL) { + return NULL; + } + left = make_branch_handler(tool_id, obj, false); + if (left == NULL) { + Py_DECREF(right); + return NULL; + } + } + Py_XDECREF(exchange_callables(tool_id, PY_MONITORING_EVENT_BRANCH_RIGHT, right)); + res = exchange_callables(tool_id, PY_MONITORING_EVENT_BRANCH_LEFT, left); + } + else { + res = exchange_callables(tool_id, event_id, Py_XNewRef(obj)); + } + if (res != NULL && Py_TYPE(res) == &_PyLegacyBranchEventHandler_Type) { + _PyLegacyBranchEventHandler *wrapper = (_PyLegacyBranchEventHandler *)res; + res = Py_NewRef(wrapper->handler); + Py_DECREF(wrapper); + } + return res; +} + +/* Branch Iterator */ + +typedef struct { + PyObject_HEAD + PyCodeObject *bi_code; + int bi_offset; +} branchesiterator; + +static PyObject * +int_triple(int a, int b, int c) { + PyObject *obja = PyLong_FromLong(a); + PyObject *objb = NULL; + PyObject *objc = NULL; + if (obja == NULL) { + goto error; + } + objb = PyLong_FromLong(b); + if (objb == NULL) { + goto error; + } + objc = PyLong_FromLong(c); + if (objc == NULL) { + goto error; + } + PyObject *array[3] = { obja, objb, objc }; + return _PyTuple_FromArraySteal(array, 3); +error: + Py_XDECREF(obja); + Py_XDECREF(objb); + Py_XDECREF(objc); + return NULL; +} + +static PyObject * +branchesiter_next(branchesiterator *bi) +{ + int offset = bi->bi_offset; + int oparg = 0; + while (offset < Py_SIZE(bi->bi_code)) { + _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(bi->bi_code, offset); + int next_offset = offset + 1 + _PyOpcode_Caches[inst.op.code]; + switch(inst.op.code) { + case EXTENDED_ARG: + oparg = (oparg << 8) | inst.op.arg; + break; + case FOR_ITER: + oparg = (oparg << 8) | inst.op.arg; + bi->bi_offset = next_offset; + int target = next_offset + oparg+2; // Skips END_FOR and POP_ITER + return int_triple(offset*2, next_offset*2, target*2); + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: + case POP_JUMP_IF_NONE: + case POP_JUMP_IF_NOT_NONE: + oparg = (oparg << 8) | inst.op.arg; + /* Skip NOT_TAKEN */ + int not_taken = next_offset + 1; + bi->bi_offset = not_taken; + return int_triple(offset*2, not_taken*2, (next_offset + oparg)*2); + default: + oparg = 0; + } + offset = next_offset; + } + return NULL; +} + +static void +branchesiter_dealloc(branchesiterator *bi) +{ + Py_DECREF(bi->bi_code); + PyObject_Free(bi); +} + +static PyTypeObject _PyBranchesIterator = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "line_iterator", /* tp_name */ + sizeof(branchesiterator), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + .tp_dealloc = (destructor)branchesiter_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_iter = PyObject_SelfIter, + .tp_iternext = (iternextfunc)branchesiter_next, + .tp_free = PyObject_Del, +}; + +PyObject * +_PyInstrumentation_BranchesIterator(PyCodeObject *code) +{ + + branchesiterator *bi = (branchesiterator *)PyType_GenericAlloc(&_PyBranchesIterator, 0); + if (bi == NULL) { + return NULL; + } + bi->bi_code = (PyCodeObject*)Py_NewRef(code); + bi->bi_offset = 0; + return (PyObject *)bi; +} diff --git a/Python/interpconfig.c b/Python/interpconfig.c new file mode 100644 index 00000000000000..54e5dca284c215 --- /dev/null +++ b/Python/interpconfig.c @@ -0,0 +1,268 @@ +/* PyInterpreterConfig API */ + +#include "Python.h" +#include "pycore_pylifecycle.h" + +#include + +#include "config_common.h" + + +static const char * +gil_flag_to_str(int flag) +{ + switch (flag) { + case PyInterpreterConfig_DEFAULT_GIL: + return "default"; + case PyInterpreterConfig_SHARED_GIL: + return "shared"; + case PyInterpreterConfig_OWN_GIL: + return "own"; + default: + PyErr_SetString(PyExc_SystemError, + "invalid interpreter config 'gil' value"); + return NULL; + } +} + +static int +gil_flag_from_str(const char *str, int *p_flag) +{ + int flag; + if (str == NULL) { + flag = PyInterpreterConfig_DEFAULT_GIL; + } + else if (strcmp(str, "default") == 0) { + flag = PyInterpreterConfig_DEFAULT_GIL; + } + else if (strcmp(str, "shared") == 0) { + flag = PyInterpreterConfig_SHARED_GIL; + } + else if (strcmp(str, "own") == 0) { + flag = PyInterpreterConfig_OWN_GIL; + } + else { + PyErr_Format(PyExc_ValueError, + "unsupported interpreter config .gil value '%s'", str); + return -1; + } + *p_flag = flag; + return 0; +} + +PyObject * +_PyInterpreterConfig_AsDict(PyInterpreterConfig *config) +{ + PyObject *dict = PyDict_New(); + if (dict == NULL) { + return NULL; + } + +#define ADD(NAME, OBJ) \ + do { \ + int res = PyDict_SetItemString(dict, NAME, (OBJ)); \ + Py_DECREF(OBJ); \ + if (res < 0) { \ + goto error; \ + } \ + } while (0) +#define ADD_BOOL(FIELD) \ + ADD(#FIELD, Py_NewRef(config->FIELD ? Py_True : Py_False)) +#define ADD_STR(FIELD, STR) \ + do { \ + if (STR == NULL) { \ + goto error; \ + } \ + PyObject *obj = PyUnicode_FromString(STR); \ + if (obj == NULL) { \ + goto error; \ + } \ + ADD(#FIELD, obj); \ + } while (0) + + ADD_BOOL(use_main_obmalloc); + ADD_BOOL(allow_fork); + ADD_BOOL(allow_exec); + ADD_BOOL(allow_threads); + ADD_BOOL(allow_daemon_threads); + ADD_BOOL(check_multi_interp_extensions); + + ADD_STR(gil, gil_flag_to_str(config->gil)); + +#undef ADD_STR +#undef ADD_BOOL +#undef ADD + + return dict; + +error: + Py_DECREF(dict); + return NULL; +} + +static int +_config_dict_get_bool(PyObject *dict, const char *name, int *p_flag) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + return -1; + } + // For now we keep things strict, rather than using PyObject_IsTrue(). + int flag = item == Py_True; + if (!flag && item != Py_False) { + Py_DECREF(item); + config_dict_invalid_type(name); + return -1; + } + Py_DECREF(item); + *p_flag = flag; + return 0; +} + +static int +_config_dict_copy_str(PyObject *dict, const char *name, + char *buf, size_t bufsize) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + return -1; + } + if (!PyUnicode_Check(item)) { + Py_DECREF(item); + config_dict_invalid_type(name); + return -1; + } + strncpy(buf, PyUnicode_AsUTF8(item), bufsize-1); + buf[bufsize-1] = '\0'; + Py_DECREF(item); + return 0; +} + +static int +interp_config_from_dict(PyObject *origdict, PyInterpreterConfig *config, + bool missing_allowed) +{ + PyObject *dict = PyDict_New(); + if (dict == NULL) { + return -1; + } + if (PyDict_Update(dict, origdict) < 0) { + goto error; + } + +#define CHECK(NAME) \ + do { \ + if (PyErr_Occurred()) { \ + goto error; \ + } \ + else { \ + if (!missing_allowed) { \ + (void)config_dict_get(dict, NAME); \ + assert(PyErr_Occurred()); \ + goto error; \ + } \ + } \ + } while (0) +#define COPY_BOOL(FIELD) \ + do { \ + int flag; \ + if (_config_dict_get_bool(dict, #FIELD, &flag) < 0) { \ + CHECK(#FIELD); \ + } \ + else { \ + config->FIELD = flag; \ + (void)PyDict_PopString(dict, #FIELD, NULL); \ + } \ + } while (0) + + COPY_BOOL(use_main_obmalloc); + COPY_BOOL(allow_fork); + COPY_BOOL(allow_exec); + COPY_BOOL(allow_threads); + COPY_BOOL(allow_daemon_threads); + COPY_BOOL(check_multi_interp_extensions); + + // PyInterpreterConfig.gil + char buf[20]; + if (_config_dict_copy_str(dict, "gil", buf, 20) < 0) { + CHECK("gil"); + } + else { + int flag; + if (gil_flag_from_str(buf, &flag) < 0) { + goto error; + } + config->gil = flag; + (void)PyDict_PopString(dict, "gil", NULL); + } + +#undef COPY_BOOL +#undef CHECK + + Py_ssize_t unused = PyDict_GET_SIZE(dict); + if (unused == 1) { + PyErr_Format(PyExc_ValueError, + "config dict has 1 extra item (%R)", dict); + goto error; + } + else if (unused > 0) { + PyErr_Format(PyExc_ValueError, + "config dict has %d extra items (%R)", unused, dict); + goto error; + } + + Py_DECREF(dict); + return 0; + +error: + Py_DECREF(dict); + return -1; +} + +int +_PyInterpreterConfig_InitFromDict(PyInterpreterConfig *config, PyObject *dict) +{ + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_TypeError, "dict expected"); + return -1; + } + if (interp_config_from_dict(dict, config, false) < 0) { + return -1; + } + return 0; +} + +int +_PyInterpreterConfig_UpdateFromDict(PyInterpreterConfig *config, PyObject *dict) +{ + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_TypeError, "dict expected"); + return -1; + } + if (interp_config_from_dict(dict, config, true) < 0) { + return -1; + } + return 0; +} + +int +_PyInterpreterConfig_InitFromState(PyInterpreterConfig *config, + PyInterpreterState *interp) +{ + // Populate the config by re-constructing the values from the interpreter. + *config = (PyInterpreterConfig){ +#define FLAG(flag) \ + (interp->feature_flags & Py_RTFLAGS_ ## flag) + .use_main_obmalloc = FLAG(USE_MAIN_OBMALLOC), + .allow_fork = FLAG(FORK), + .allow_exec = FLAG(EXEC), + .allow_threads = FLAG(THREADS), + .allow_daemon_threads = FLAG(DAEMON_THREADS), + .check_multi_interp_extensions = FLAG(MULTI_INTERP_EXTENSIONS), +#undef FLAG + .gil = interp->ceval.own_gil + ? PyInterpreterConfig_OWN_GIL + : PyInterpreterConfig_SHARED_GIL, + }; + return 0; +} diff --git a/Python/intrinsics.c b/Python/intrinsics.c index d3146973b75178..a6b2c108b67175 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -123,18 +123,15 @@ static PyObject * import_star(PyThreadState* tstate, PyObject *from) { _PyInterpreterFrame *frame = tstate->current_frame; - if (_PyFrame_FastToLocalsWithError(frame) < 0) { - return NULL; - } - PyObject *locals = frame->f_locals; + PyObject *locals = _PyFrame_GetLocals(frame); if (locals == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found during 'import *'"); return NULL; } int err = import_all_from(tstate, locals, from); - _PyFrame_LocalsToFast(frame, 0); + Py_DECREF(locals); if (err < 0) { return NULL; } @@ -265,6 +262,7 @@ _PyIntrinsics_BinaryFunctions[] = { INTRINSIC_FUNC_ENTRY(INTRINSIC_TYPEVAR_WITH_BOUND, make_typevar_with_bound) INTRINSIC_FUNC_ENTRY(INTRINSIC_TYPEVAR_WITH_CONSTRAINTS, make_typevar_with_constraints) INTRINSIC_FUNC_ENTRY(INTRINSIC_SET_FUNCTION_TYPE_PARAMS, _Py_set_function_type_params) + INTRINSIC_FUNC_ENTRY(INTRINSIC_SET_TYPEPARAM_DEFAULT, _Py_set_typeparam_default) }; #undef INTRINSIC_FUNC_ENTRY diff --git a/Python/jit.c b/Python/jit.c index dae25166b1f106..7dd0da7a45055a 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -3,8 +3,10 @@ #include "Python.h" #include "pycore_abstract.h" +#include "pycore_bitutils.h" #include "pycore_call.h" #include "pycore_ceval.h" +#include "pycore_critical_section.h" #include "pycore_dict.h" #include "pycore_intrinsics.h" #include "pycore_long.h" @@ -16,8 +18,6 @@ #include "pycore_sliceobject.h" #include "pycore_jit.h" -#include "jit_stencils.h" - // Memory management stuff: //////////////////////////////////////////////////// #ifndef MS_WINDOWS @@ -58,7 +58,12 @@ jit_alloc(size_t size) int failed = memory == NULL; #else int flags = MAP_ANONYMOUS | MAP_PRIVATE; - unsigned char *memory = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, -1, 0); + int prot = PROT_READ | PROT_WRITE; +# ifdef MAP_JIT + flags |= MAP_JIT; + prot |= PROT_EXEC; +# endif + unsigned char *memory = mmap(NULL, size, prot, flags, -1, 0); int failed = memory == MAP_FAILED; #endif if (failed) { @@ -102,31 +107,14 @@ mark_executable(unsigned char *memory, size_t size) int old; int failed = !VirtualProtect(memory, size, PAGE_EXECUTE_READ, &old); #else + int failed = 0; __builtin___clear_cache((char *)memory, (char *)memory + size); - int failed = mprotect(memory, size, PROT_EXEC | PROT_READ); +#ifndef MAP_JIT + failed = mprotect(memory, size, PROT_EXEC | PROT_READ); #endif - if (failed) { - jit_error("unable to protect executable memory"); - return -1; - } - return 0; -} - -static int -mark_readable(unsigned char *memory, size_t size) -{ - if (size == 0) { - return 0; - } - assert(size % get_page_size() == 0); -#ifdef MS_WINDOWS - DWORD old; - int failed = !VirtualProtect(memory, size, PAGE_READONLY, &old); -#else - int failed = mprotect(memory, size, PROT_READ); #endif if (failed) { - jit_error("unable to protect readable memory"); + jit_error("unable to protect executable memory"); return -1; } return 0; @@ -134,6 +122,21 @@ mark_readable(unsigned char *memory, size_t size) // JIT compiler stuff: ///////////////////////////////////////////////////////// +#define SYMBOL_MASK_WORDS 4 + +typedef uint32_t symbol_mask[SYMBOL_MASK_WORDS]; + +typedef struct { + unsigned char *mem; + symbol_mask mask; + size_t size; +} trampoline_state; + +typedef struct { + trampoline_state trampolines; + uintptr_t instruction_starts[UOP_MAX_TRACE_LENGTH]; +} jit_state; + // Warning! AArch64 requires you to get your hands dirty. These are your gloves: // value[value_start : value_start + len] @@ -166,292 +169,387 @@ set_bits(uint32_t *loc, uint8_t loc_start, uint64_t value, uint8_t value_start, #define IS_AARCH64_LDR_OR_STR(I) (((I) & 0x3B000000) == 0x39000000) #define IS_AARCH64_MOV(I) (((I) & 0x9F800000) == 0x92800000) -// Fill all of stencil's holes in the memory pointed to by base, using the -// values in patches. -static void -patch(unsigned char *base, const Stencil *stencil, uint64_t *patches) +// LLD is a great reference for performing relocations... just keep in +// mind that Tools/jit/build.py does filtering and preprocessing for us! +// Here's a good place to start for each platform: +// - aarch64-apple-darwin: +// - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64.cpp +// - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.cpp +// - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.h +// - aarch64-pc-windows-msvc: +// - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp +// - aarch64-unknown-linux-gnu: +// - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/AArch64.cpp +// - i686-pc-windows-msvc: +// - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp +// - x86_64-apple-darwin: +// - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/X86_64.cpp +// - x86_64-pc-windows-msvc: +// - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp +// - x86_64-unknown-linux-gnu: +// - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/X86_64.cpp + +// Many of these patches are "relaxing", meaning that they can rewrite the +// code they're patching to be more efficient (like turning a 64-bit memory +// load into a 32-bit immediate load). These patches have an "x" in their name. +// Relative patches have an "r" in their name. + +// 32-bit absolute address. +void +patch_32(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + // Check that we're not out of range of 32 unsigned bits: + assert(value < (1ULL << 32)); + *loc32 = (uint32_t)value; +} + +// 32-bit relative address. +void +patch_32r(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + value -= (uintptr_t)location; + // Check that we're not out of range of 32 signed bits: + assert((int64_t)value >= -(1LL << 31)); + assert((int64_t)value < (1LL << 31)); + *loc32 = (uint32_t)value; +} + +// 64-bit absolute address. +void +patch_64(unsigned char *location, uint64_t value) +{ + uint64_t *loc64 = (uint64_t *)location; + *loc64 = value; +} + +// 12-bit low part of an absolute address. Pairs nicely with patch_aarch64_21r +// (below). +void +patch_aarch64_12(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_LDR_OR_STR(*loc32) || IS_AARCH64_ADD_OR_SUB(*loc32)); + // There might be an implicit shift encoded in the instruction: + uint8_t shift = 0; + if (IS_AARCH64_LDR_OR_STR(*loc32)) { + shift = (uint8_t)get_bits(*loc32, 30, 2); + // If both of these are set, the shift is supposed to be 4. + // That's pretty weird, and it's never actually been observed... + assert(get_bits(*loc32, 23, 1) == 0 || get_bits(*loc32, 26, 1) == 0); + } + value = get_bits(value, 0, 12); + assert(get_bits(value, 0, shift) == 0); + set_bits(loc32, 10, value, shift, 12); +} + +// Relaxable 12-bit low part of an absolute address. Pairs nicely with +// patch_aarch64_21rx (below). +void +patch_aarch64_12x(unsigned char *location, uint64_t value) +{ + // This can *only* be relaxed if it occurs immediately before a matching + // patch_aarch64_21rx. If that happens, the JIT build step will replace both + // calls with a single call to patch_aarch64_33rx. Otherwise, we end up + // here, and the instruction is patched normally: + patch_aarch64_12(location, value); +} + +// 16-bit low part of an absolute address. +void +patch_aarch64_16a(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 0 of 3"): + assert(get_bits(*loc32, 21, 2) == 0); + set_bits(loc32, 5, value, 0, 16); +} + +// 16-bit middle-low part of an absolute address. +void +patch_aarch64_16b(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 1 of 3"): + assert(get_bits(*loc32, 21, 2) == 1); + set_bits(loc32, 5, value, 16, 16); +} + +// 16-bit middle-high part of an absolute address. +void +patch_aarch64_16c(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 2 of 3"): + assert(get_bits(*loc32, 21, 2) == 2); + set_bits(loc32, 5, value, 32, 16); +} + +// 16-bit high part of an absolute address. +void +patch_aarch64_16d(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 3 of 3"): + assert(get_bits(*loc32, 21, 2) == 3); + set_bits(loc32, 5, value, 48, 16); +} + +// 21-bit count of pages between this page and an absolute address's page... I +// know, I know, it's weird. Pairs nicely with patch_aarch64_12 (above). +void +patch_aarch64_21r(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + value = (value >> 12) - ((uintptr_t)location >> 12); + // Check that we're not out of range of 21 signed bits: + assert((int64_t)value >= -(1 << 20)); + assert((int64_t)value < (1 << 20)); + // value[0:2] goes in loc[29:31]: + set_bits(loc32, 29, value, 0, 2); + // value[2:21] goes in loc[5:26]: + set_bits(loc32, 5, value, 2, 19); +} + +// Relaxable 21-bit count of pages between this page and an absolute address's +// page. Pairs nicely with patch_aarch64_12x (above). +void +patch_aarch64_21rx(unsigned char *location, uint64_t value) +{ + // This can *only* be relaxed if it occurs immediately before a matching + // patch_aarch64_12x. If that happens, the JIT build step will replace both + // calls with a single call to patch_aarch64_33rx. Otherwise, we end up + // here, and the instruction is patched normally: + patch_aarch64_21r(location, value); +} + +// 28-bit relative branch. +void +patch_aarch64_26r(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_BRANCH(*loc32)); + value -= (uintptr_t)location; + // Check that we're not out of range of 28 signed bits: + assert((int64_t)value >= -(1 << 27)); + assert((int64_t)value < (1 << 27)); + // Since instructions are 4-byte aligned, only use 26 bits: + assert(get_bits(value, 0, 2) == 0); + set_bits(loc32, 0, value, 2, 26); +} + +// A pair of patch_aarch64_21rx and patch_aarch64_12x. +void +patch_aarch64_33rx(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + // Try to relax the pair of GOT loads into an immediate value: + assert(IS_AARCH64_ADRP(*loc32)); + unsigned char reg = get_bits(loc32[0], 0, 5); + assert(IS_AARCH64_LDR_OR_STR(loc32[1])); + // There should be only one register involved: + assert(reg == get_bits(loc32[1], 0, 5)); // ldr's output register. + assert(reg == get_bits(loc32[1], 5, 5)); // ldr's input register. + uint64_t relaxed = *(uint64_t *)value; + if (relaxed < (1UL << 16)) { + // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; nop + loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | reg; + loc32[1] = 0xD503201F; + return; + } + if (relaxed < (1ULL << 32)) { + // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; movk reg, YYY + loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | reg; + loc32[1] = 0xF2A00000 | (get_bits(relaxed, 16, 16) << 5) | reg; + return; + } + relaxed = value - (uintptr_t)location; + if ((relaxed & 0x3) == 0 && + (int64_t)relaxed >= -(1L << 19) && + (int64_t)relaxed < (1L << 19)) + { + // adrp reg, AAA; ldr reg, [reg + BBB] -> ldr reg, XXX; nop + loc32[0] = 0x58000000 | (get_bits(relaxed, 2, 19) << 5) | reg; + loc32[1] = 0xD503201F; + return; + } + // Couldn't do it. Just patch the two instructions normally: + patch_aarch64_21rx(location, value); + patch_aarch64_12x(location + 4, value); +} + +// Relaxable 32-bit relative address. +void +patch_x86_64_32rx(unsigned char *location, uint64_t value) { - for (uint64_t i = 0; i < stencil->holes_size; i++) { - const Hole *hole = &stencil->holes[i]; - unsigned char *location = base + hole->offset; - uint64_t value = patches[hole->value] + (uint64_t)hole->symbol + hole->addend; - uint8_t *loc8 = (uint8_t *)location; - uint32_t *loc32 = (uint32_t *)location; - uint64_t *loc64 = (uint64_t *)location; - // LLD is a great reference for performing relocations... just keep in - // mind that Tools/jit/build.py does filtering and preprocessing for us! - // Here's a good place to start for each platform: - // - aarch64-apple-darwin: - // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64.cpp - // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.cpp - // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.h - // - aarch64-pc-windows-msvc: - // - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp - // - aarch64-unknown-linux-gnu: - // - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/AArch64.cpp - // - i686-pc-windows-msvc: - // - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp - // - x86_64-apple-darwin: - // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/X86_64.cpp - // - x86_64-pc-windows-msvc: - // - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp - // - x86_64-unknown-linux-gnu: - // - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/X86_64.cpp - switch (hole->kind) { - case HoleKind_IMAGE_REL_I386_DIR32: - // 32-bit absolute address. - // Check that we're not out of range of 32 unsigned bits: - assert(value < (1ULL << 32)); - *loc32 = (uint32_t)value; - continue; - case HoleKind_ARM64_RELOC_UNSIGNED: - case HoleKind_R_AARCH64_ABS64: - case HoleKind_X86_64_RELOC_UNSIGNED: - case HoleKind_R_X86_64_64: - // 64-bit absolute address. - *loc64 = value; - continue; - case HoleKind_IMAGE_REL_AMD64_REL32: - case HoleKind_IMAGE_REL_I386_REL32: - case HoleKind_R_X86_64_GOTPCRELX: - case HoleKind_R_X86_64_REX_GOTPCRELX: - case HoleKind_X86_64_RELOC_GOT: - case HoleKind_X86_64_RELOC_GOT_LOAD: { - // 32-bit relative address. - // Try to relax the GOT load into an immediate value: - uint64_t relaxed = *(uint64_t *)(value + 4) - 4; - if ((int64_t)relaxed - (int64_t)location >= -(1LL << 31) && - (int64_t)relaxed - (int64_t)location + 1 < (1LL << 31)) - { - if (loc8[-2] == 0x8B) { - // mov reg, dword ptr [rip + AAA] -> lea reg, [rip + XXX] - loc8[-2] = 0x8D; - value = relaxed; - } - else if (loc8[-2] == 0xFF && loc8[-1] == 0x15) { - // call qword ptr [rip + AAA] -> nop; call XXX - loc8[-2] = 0x90; - loc8[-1] = 0xE8; - value = relaxed; - } - else if (loc8[-2] == 0xFF && loc8[-1] == 0x25) { - // jmp qword ptr [rip + AAA] -> nop; jmp XXX - loc8[-2] = 0x90; - loc8[-1] = 0xE9; - value = relaxed; - } - } - } - // Fall through... - case HoleKind_R_X86_64_GOTPCREL: - case HoleKind_R_X86_64_PC32: - case HoleKind_X86_64_RELOC_SIGNED: - case HoleKind_X86_64_RELOC_BRANCH: - // 32-bit relative address. - value -= (uint64_t)location; - // Check that we're not out of range of 32 signed bits: - assert((int64_t)value >= -(1LL << 31)); - assert((int64_t)value < (1LL << 31)); - *loc32 = (uint32_t)value; - continue; - case HoleKind_IMAGE_REL_ARM64_BRANCH26: - case HoleKind_R_AARCH64_CALL26: - case HoleKind_R_AARCH64_JUMP26: - // 28-bit relative branch. - assert(IS_AARCH64_BRANCH(*loc32)); - value -= (uint64_t)location; - // Check that we're not out of range of 28 signed bits: - assert((int64_t)value >= -(1 << 27)); - assert((int64_t)value < (1 << 27)); - // Since instructions are 4-byte aligned, only use 26 bits: - assert(get_bits(value, 0, 2) == 0); - set_bits(loc32, 0, value, 2, 26); - continue; - case HoleKind_R_AARCH64_MOVW_UABS_G0_NC: - // 16-bit low part of an absolute address. - assert(IS_AARCH64_MOV(*loc32)); - // Check the implicit shift (this is "part 0 of 3"): - assert(get_bits(*loc32, 21, 2) == 0); - set_bits(loc32, 5, value, 0, 16); - continue; - case HoleKind_R_AARCH64_MOVW_UABS_G1_NC: - // 16-bit middle-low part of an absolute address. - assert(IS_AARCH64_MOV(*loc32)); - // Check the implicit shift (this is "part 1 of 3"): - assert(get_bits(*loc32, 21, 2) == 1); - set_bits(loc32, 5, value, 16, 16); - continue; - case HoleKind_R_AARCH64_MOVW_UABS_G2_NC: - // 16-bit middle-high part of an absolute address. - assert(IS_AARCH64_MOV(*loc32)); - // Check the implicit shift (this is "part 2 of 3"): - assert(get_bits(*loc32, 21, 2) == 2); - set_bits(loc32, 5, value, 32, 16); - continue; - case HoleKind_R_AARCH64_MOVW_UABS_G3: - // 16-bit high part of an absolute address. - assert(IS_AARCH64_MOV(*loc32)); - // Check the implicit shift (this is "part 3 of 3"): - assert(get_bits(*loc32, 21, 2) == 3); - set_bits(loc32, 5, value, 48, 16); - continue; - case HoleKind_ARM64_RELOC_GOT_LOAD_PAGE21: - case HoleKind_IMAGE_REL_ARM64_PAGEBASE_REL21: - case HoleKind_R_AARCH64_ADR_GOT_PAGE: - // 21-bit count of pages between this page and an absolute address's - // page... I know, I know, it's weird. Pairs nicely with - // ARM64_RELOC_GOT_LOAD_PAGEOFF12 (below). - assert(IS_AARCH64_ADRP(*loc32)); - // Try to relax the pair of GOT loads into an immediate value: - const Hole *next_hole = &stencil->holes[i + 1]; - if (i + 1 < stencil->holes_size && - (next_hole->kind == HoleKind_ARM64_RELOC_GOT_LOAD_PAGEOFF12 || - next_hole->kind == HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12L || - next_hole->kind == HoleKind_R_AARCH64_LD64_GOT_LO12_NC) && - next_hole->offset == hole->offset + 4 && - next_hole->symbol == hole->symbol && - next_hole->addend == hole->addend && - next_hole->value == hole->value) - { - unsigned char reg = get_bits(loc32[0], 0, 5); - assert(IS_AARCH64_LDR_OR_STR(loc32[1])); - // There should be only one register involved: - assert(reg == get_bits(loc32[1], 0, 5)); // ldr's output register. - assert(reg == get_bits(loc32[1], 5, 5)); // ldr's input register. - uint64_t relaxed = *(uint64_t *)value; - if (relaxed < (1UL << 16)) { - // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; nop - loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | reg; - loc32[1] = 0xD503201F; - i++; - continue; - } - if (relaxed < (1ULL << 32)) { - // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; movk reg, YYY - loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | reg; - loc32[1] = 0xF2A00000 | (get_bits(relaxed, 16, 16) << 5) | reg; - i++; - continue; - } - relaxed = (uint64_t)value - (uint64_t)location; - if ((relaxed & 0x3) == 0 && - (int64_t)relaxed >= -(1L << 19) && - (int64_t)relaxed < (1L << 19)) - { - // adrp reg, AAA; ldr reg, [reg + BBB] -> ldr reg, XXX; nop - loc32[0] = 0x58000000 | (get_bits(relaxed, 2, 19) << 5) | reg; - loc32[1] = 0xD503201F; - i++; - continue; - } - } - // Fall through... - case HoleKind_ARM64_RELOC_PAGE21: - // Number of pages between this page and the value's page: - value = (value >> 12) - ((uint64_t)location >> 12); - // Check that we're not out of range of 21 signed bits: - assert((int64_t)value >= -(1 << 20)); - assert((int64_t)value < (1 << 20)); - // value[0:2] goes in loc[29:31]: - set_bits(loc32, 29, value, 0, 2); - // value[2:21] goes in loc[5:26]: - set_bits(loc32, 5, value, 2, 19); - continue; - case HoleKind_ARM64_RELOC_GOT_LOAD_PAGEOFF12: - case HoleKind_ARM64_RELOC_PAGEOFF12: - case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12A: - case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12L: - case HoleKind_R_AARCH64_LD64_GOT_LO12_NC: - // 12-bit low part of an absolute address. Pairs nicely with - // ARM64_RELOC_GOT_LOAD_PAGE21 (above). - assert(IS_AARCH64_LDR_OR_STR(*loc32) || IS_AARCH64_ADD_OR_SUB(*loc32)); - // There might be an implicit shift encoded in the instruction: - uint8_t shift = 0; - if (IS_AARCH64_LDR_OR_STR(*loc32)) { - shift = (uint8_t)get_bits(*loc32, 30, 2); - // If both of these are set, the shift is supposed to be 4. - // That's pretty weird, and it's never actually been observed... - assert(get_bits(*loc32, 23, 1) == 0 || get_bits(*loc32, 26, 1) == 0); - } - value = get_bits(value, 0, 12); - assert(get_bits(value, 0, shift) == 0); - set_bits(loc32, 10, value, shift, 12); - continue; + uint8_t *loc8 = (uint8_t *)location; + // Try to relax the GOT load into an immediate value: + uint64_t relaxed = *(uint64_t *)(value + 4) - 4; + if ((int64_t)relaxed - (int64_t)location >= -(1LL << 31) && + (int64_t)relaxed - (int64_t)location + 1 < (1LL << 31)) + { + if (loc8[-2] == 0x8B) { + // mov reg, dword ptr [rip + AAA] -> lea reg, [rip + XXX] + loc8[-2] = 0x8D; + value = relaxed; + } + else if (loc8[-2] == 0xFF && loc8[-1] == 0x15) { + // call qword ptr [rip + AAA] -> nop; call XXX + loc8[-2] = 0x90; + loc8[-1] = 0xE8; + value = relaxed; + } + else if (loc8[-2] == 0xFF && loc8[-1] == 0x25) { + // jmp qword ptr [rip + AAA] -> nop; jmp XXX + loc8[-2] = 0x90; + loc8[-1] = 0xE9; + value = relaxed; } - Py_UNREACHABLE(); } + patch_32r(location, value); } -static void -copy_and_patch(unsigned char *base, const Stencil *stencil, uint64_t *patches) +void patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state); + +#include "jit_stencils.h" + +#if defined(__aarch64__) || defined(_M_ARM64) + #define TRAMPOLINE_SIZE 16 +#else + #define TRAMPOLINE_SIZE 0 +#endif + +// Generate and patch AArch64 trampolines. The symbols to jump to are stored +// in the jit_stencils.h in the symbols_map. +void +patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state) { - memcpy(base, stencil->body, stencil->body_size); - patch(base, stencil, patches); + // Masking is done modulo 32 as the mask is stored as an array of uint32_t + const uint32_t symbol_mask = 1 << (ordinal % 32); + const uint32_t trampoline_mask = state->trampolines.mask[ordinal / 32]; + assert(symbol_mask & trampoline_mask); + + // Count the number of set bits in the trampoline mask lower than ordinal, + // this gives the index into the array of trampolines. + int index = _Py_popcount32(trampoline_mask & (symbol_mask - 1)); + for (int i = 0; i < ordinal / 32; i++) { + index += _Py_popcount32(state->trampolines.mask[i]); + } + + uint32_t *p = (uint32_t*)(state->trampolines.mem + index * TRAMPOLINE_SIZE); + assert((size_t)(index + 1) * TRAMPOLINE_SIZE <= state->trampolines.size); + + uint64_t value = (uintptr_t)symbols_map[ordinal]; + + /* Generate the trampoline + 0: 58000048 ldr x8, 8 + 4: d61f0100 br x8 + 8: 00000000 // The next two words contain the 64-bit address to jump to. + c: 00000000 + */ + p[0] = 0x58000048; + p[1] = 0xD61F0100; + p[2] = value & 0xffffffff; + p[3] = value >> 32; + + patch_aarch64_26r(location, (uintptr_t)p); } static void -emit(const StencilGroup *group, uint64_t patches[]) +combine_symbol_mask(const symbol_mask src, symbol_mask dest) { - copy_and_patch((unsigned char *)patches[HoleValue_DATA], &group->data, patches); - copy_and_patch((unsigned char *)patches[HoleValue_CODE], &group->code, patches); + // Calculate the union of the trampolines required by each StencilGroup + for (size_t i = 0; i < SYMBOL_MASK_WORDS; i++) { + dest[i] |= src[i]; + } } // Compiles executor in-place. Don't forget to call _PyJIT_Free later! int -_PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length) +_PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], size_t length) { + const StencilGroup *group; // Loop once to find the total compiled size: size_t code_size = 0; size_t data_size = 0; + jit_state state = {0}; + group = &shim; + code_size += group->code_size; + data_size += group->data_size; + combine_symbol_mask(group->trampoline_mask, state.trampolines.mask); for (size_t i = 0; i < length; i++) { - _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i]; - const StencilGroup *group = &stencil_groups[instruction->opcode]; - code_size += group->code.body_size; - data_size += group->data.body_size; + const _PyUOpInstruction *instruction = &trace[i]; + group = &stencil_groups[instruction->opcode]; + state.instruction_starts[i] = code_size; + code_size += group->code_size; + data_size += group->data_size; + combine_symbol_mask(group->trampoline_mask, state.trampolines.mask); + } + group = &stencil_groups[_FATAL_ERROR]; + code_size += group->code_size; + data_size += group->data_size; + combine_symbol_mask(group->trampoline_mask, state.trampolines.mask); + // Calculate the size of the trampolines required by the whole trace + for (size_t i = 0; i < Py_ARRAY_LENGTH(state.trampolines.mask); i++) { + state.trampolines.size += _Py_popcount32(state.trampolines.mask[i]) * TRAMPOLINE_SIZE; } - // Round up to the nearest page (code and data need separate pages): + // Round up to the nearest page: size_t page_size = get_page_size(); assert((page_size & (page_size - 1)) == 0); - code_size += page_size - (code_size & (page_size - 1)); - data_size += page_size - (data_size & (page_size - 1)); - unsigned char *memory = jit_alloc(code_size + data_size); + size_t padding = page_size - ((code_size + data_size + state.trampolines.size) & (page_size - 1)); + size_t total_size = code_size + data_size + state.trampolines.size + padding; + unsigned char *memory = jit_alloc(total_size); if (memory == NULL) { return -1; } +#ifdef MAP_JIT + pthread_jit_write_protect_np(0); +#endif + // Update the offsets of each instruction: + for (size_t i = 0; i < length; i++) { + state.instruction_starts[i] += (uintptr_t)memory; + } // Loop again to emit the code: unsigned char *code = memory; unsigned char *data = memory + code_size; - unsigned char *top = code; - if (trace[0].opcode == _START_EXECUTOR) { - // Don't want to execute this more than once: - top += stencil_groups[_START_EXECUTOR].code.body_size; - } + state.trampolines.mem = memory + code_size + data_size; + // Compile the shim, which handles converting between the native + // calling convention and the calling convention used by jitted code + // (which may be different for efficiency reasons). + group = &shim; + group->emit(code, data, executor, NULL, &state); + code += group->code_size; + data += group->data_size; + assert(trace[0].opcode == _START_EXECUTOR); for (size_t i = 0; i < length; i++) { - _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i]; - const StencilGroup *group = &stencil_groups[instruction->opcode]; - // Think of patches as a dictionary mapping HoleValue to uint64_t: - uint64_t patches[] = GET_PATCHES(); - patches[HoleValue_CODE] = (uint64_t)code; - patches[HoleValue_CONTINUE] = (uint64_t)code + group->code.body_size; - patches[HoleValue_DATA] = (uint64_t)data; - patches[HoleValue_EXECUTOR] = (uint64_t)executor; - patches[HoleValue_OPARG] = instruction->oparg; - patches[HoleValue_OPERAND] = instruction->operand; - patches[HoleValue_TARGET] = instruction->target; - patches[HoleValue_TOP] = (uint64_t)top; - patches[HoleValue_ZERO] = 0; - emit(group, patches); - code += group->code.body_size; - data += group->data.body_size; + const _PyUOpInstruction *instruction = &trace[i]; + group = &stencil_groups[instruction->opcode]; + group->emit(code, data, executor, instruction, &state); + code += group->code_size; + data += group->data_size; } - if (mark_executable(memory, code_size) || - mark_readable(memory + code_size, data_size)) - { - jit_free(memory, code_size + data_size); + // Protect against accidental buffer overrun into data: + group = &stencil_groups[_FATAL_ERROR]; + group->emit(code, data, executor, NULL, &state); + code += group->code_size; + data += group->data_size; + assert(code == memory + code_size); + assert(data == memory + code_size + data_size); +#ifdef MAP_JIT + pthread_jit_write_protect_np(1); +#endif + if (mark_executable(memory, total_size)) { + jit_free(memory, total_size); return -1; } executor->jit_code = memory; - executor->jit_size = code_size + data_size; + executor->jit_side_entry = memory + shim.code_size; + executor->jit_size = total_size; return 0; } @@ -462,6 +560,7 @@ _PyJIT_Free(_PyExecutorObject *executor) size_t size = executor->jit_size; if (memory) { executor->jit_code = NULL; + executor->jit_side_entry = NULL; executor->jit_size = 0; if (jit_free(memory, size)) { PyErr_WriteUnraisable(NULL); diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index ccbb3eb3f7c82a..97634f9183c7d5 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -3,9 +3,9 @@ */ #include "Python.h" +#include "pycore_audit.h" // _PySys_Audit() #include "pycore_ceval.h" // export _PyEval_SetProfile() #include "pycore_object.h" -#include "pycore_sysmodule.h" // _PySys_Audit() #include "opcode.h" #include @@ -16,6 +16,13 @@ typedef struct _PyLegacyEventHandler { int event; } _PyLegacyEventHandler; +#ifdef Py_GIL_DISABLED +#define LOCK_SETUP() PyMutex_Lock(&_PyRuntime.ceval.sys_trace_profile_mutex); +#define UNLOCK_SETUP() PyMutex_Unlock(&_PyRuntime.ceval.sys_trace_profile_mutex); +#else +#define LOCK_SETUP() +#define UNLOCK_SETUP() +#endif /* The Py_tracefunc function expects the following arguments: * obj: the trace object (PyObject *) * frame: the current frame (PyFrameObject *) @@ -114,6 +121,19 @@ sys_profile_call_or_return( Py_DECREF(meth); return res; } + else if (Py_TYPE(callable) == &PyMethod_Type) { + // CALL instruction will grab the function from the method, + // so if the function is a C function, the return event will + // be emitted. However, CALL event happens before CALL + // instruction, so we need to handle this case here. + PyObject* func = PyMethod_GET_FUNCTION(callable); + if (func == NULL) { + return NULL; + } + if (PyCFunction_Check(func)) { + return call_profile_func(self, func); + } + } Py_RETURN_NONE; } @@ -123,9 +143,8 @@ _PyEval_SetOpcodeTrace( bool enable ) { assert(frame != NULL); - assert(PyCode_Check(frame->f_frame->f_executable)); - PyCodeObject *code = (PyCodeObject *)frame->f_frame->f_executable; + PyCodeObject *code = _PyFrame_GetCode(frame->f_frame); _PyMonitoringEventSet events = 0; if (_PyMonitoring_GetLocalEvents(code, PY_MONITORING_SYS_TRACE_ID, &events) < 0) { @@ -167,6 +186,7 @@ call_trace_func(_PyLegacyEventHandler *self, PyObject *arg) Py_INCREF(frame); int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, arg); + frame->f_lineno = 0; Py_DECREF(frame); if (err) { return NULL; @@ -414,19 +434,10 @@ is_tstate_valid(PyThreadState *tstate) } #endif -int -_PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +static Py_ssize_t +setup_profile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg, PyObject **old_profileobj) { - assert(is_tstate_valid(tstate)); - /* The caller must hold the GIL */ - assert(PyGILState_Check()); - - /* Call _PySys_Audit() in the context of the current thread state, - even if tstate is not the current thread state. */ - PyThreadState *current_tstate = _PyThreadState_GET(); - if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) { - return -1; - } + *old_profileobj = NULL; /* Setup PEP 669 monitoring callbacks and events. */ if (!tstate->interp->sys_profile_initialized) { tstate->interp->sys_profile_initialized = true; @@ -469,38 +480,49 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) int delta = (func != NULL) - (tstate->c_profilefunc != NULL); tstate->c_profilefunc = func; - PyObject *old_profileobj = tstate->c_profileobj; + *old_profileobj = tstate->c_profileobj; tstate->c_profileobj = Py_XNewRef(arg); - Py_XDECREF(old_profileobj); tstate->interp->sys_profiling_threads += delta; assert(tstate->interp->sys_profiling_threads >= 0); - - uint32_t events = 0; - if (tstate->interp->sys_profiling_threads) { - events = - (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | - (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | - (1 << PY_MONITORING_EVENT_CALL) | (1 << PY_MONITORING_EVENT_PY_UNWIND) | - (1 << PY_MONITORING_EVENT_PY_THROW); - } - return _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, events); + return tstate->interp->sys_profiling_threads; } int -_PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +_PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) { assert(is_tstate_valid(tstate)); - /* The caller must hold the GIL */ - assert(PyGILState_Check()); + /* The caller must hold a thread state */ + _Py_AssertHoldsTstate(); /* Call _PySys_Audit() in the context of the current thread state, even if tstate is not the current thread state. */ PyThreadState *current_tstate = _PyThreadState_GET(); - if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) { + if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) { return -1; } - assert(tstate->interp->sys_tracing_threads >= 0); + // needs to be decref'd outside of the lock + PyObject *old_profileobj; + LOCK_SETUP(); + Py_ssize_t profiling_threads = setup_profile(tstate, func, arg, &old_profileobj); + UNLOCK_SETUP(); + Py_XDECREF(old_profileobj); + + uint32_t events = 0; + if (profiling_threads) { + events = + (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | + (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | + (1 << PY_MONITORING_EVENT_CALL) | (1 << PY_MONITORING_EVENT_PY_UNWIND) | + (1 << PY_MONITORING_EVENT_PY_THROW); + } + return _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, events); +} + +static Py_ssize_t +setup_tracing(PyThreadState *tstate, Py_tracefunc func, PyObject *arg, PyObject **old_traceobj) +{ + *old_traceobj = NULL; /* Setup PEP 669 monitoring callbacks and events. */ if (!tstate->interp->sys_trace_initialized) { tstate->interp->sys_trace_initialized = true; @@ -553,25 +575,49 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) int delta = (func != NULL) - (tstate->c_tracefunc != NULL); tstate->c_tracefunc = func; - PyObject *old_traceobj = tstate->c_traceobj; + *old_traceobj = tstate->c_traceobj; tstate->c_traceobj = Py_XNewRef(arg); - Py_XDECREF(old_traceobj); tstate->interp->sys_tracing_threads += delta; assert(tstate->interp->sys_tracing_threads >= 0); + return tstate->interp->sys_tracing_threads; +} + +int +_PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +{ + assert(is_tstate_valid(tstate)); + /* The caller must hold a thread state */ + _Py_AssertHoldsTstate(); + + /* Call _PySys_Audit() in the context of the current thread state, + even if tstate is not the current thread state. */ + PyThreadState *current_tstate = _PyThreadState_GET(); + if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) { + return -1; + } + assert(tstate->interp->sys_tracing_threads >= 0); + // needs to be decref'd outside of the lock + PyObject *old_traceobj; + LOCK_SETUP(); + Py_ssize_t tracing_threads = setup_tracing(tstate, func, arg, &old_traceobj); + UNLOCK_SETUP(); + Py_XDECREF(old_traceobj); + if (tracing_threads < 0) { + return -1; + } uint32_t events = 0; - if (tstate->interp->sys_tracing_threads) { + if (tracing_threads) { events = (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | - (1 << PY_MONITORING_EVENT_JUMP) | (1 << PY_MONITORING_EVENT_BRANCH) | + (1 << PY_MONITORING_EVENT_JUMP) | (1 << PY_MONITORING_EVENT_PY_UNWIND) | (1 << PY_MONITORING_EVENT_PY_THROW) | - (1 << PY_MONITORING_EVENT_STOP_ITERATION) | - (1 << PY_MONITORING_EVENT_EXCEPTION_HANDLED); + (1 << PY_MONITORING_EVENT_STOP_ITERATION); PyFrameObject* frame = PyEval_GetFrame(); - if (frame->f_trace_opcodes) { + if (frame && frame->f_trace_opcodes) { int ret = _PyEval_SetOpcodeTrace(frame, true); if (ret != 0) { return ret; diff --git a/Python/lock.c b/Python/lock.c index de25adce385105..554c51d7780322 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -5,13 +5,13 @@ #include "pycore_lock.h" #include "pycore_parking_lot.h" #include "pycore_semaphore.h" -#include "pycore_time.h" // _PyTime_MonotonicUnchecked() +#include "pycore_time.h" // _PyTime_Add() #ifdef MS_WINDOWS # define WIN32_LEAN_AND_MEAN -# include // SwitchToThread() +# include // SwitchToThread() #elif defined(HAVE_SCHED_H) -# include // sched_yield() +# include // sched_yield() #endif // If a thread waits on a lock for longer than TIME_TO_BE_FAIR_NS (1 ms), then @@ -47,18 +47,12 @@ _Py_yield(void) #endif } -void -_PyMutex_LockSlow(PyMutex *m) -{ - _PyMutex_LockTimed(m, -1, _PY_LOCK_DETACH); -} - PyLockStatus _PyMutex_LockTimed(PyMutex *m, PyTime_t timeout, _PyLockFlags flags) { - uint8_t v = _Py_atomic_load_uint8_relaxed(&m->v); + uint8_t v = _Py_atomic_load_uint8_relaxed(&m->_bits); if ((v & _Py_LOCKED) == 0) { - if (_Py_atomic_compare_exchange_uint8(&m->v, &v, v|_Py_LOCKED)) { + if (_Py_atomic_compare_exchange_uint8(&m->_bits, &v, v|_Py_LOCKED)) { return PY_LOCK_ACQUIRED; } } @@ -66,7 +60,9 @@ _PyMutex_LockTimed(PyMutex *m, PyTime_t timeout, _PyLockFlags flags) return PY_LOCK_FAILURE; } - PyTime_t now = _PyTime_MonotonicUnchecked(); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); PyTime_t endtime = 0; if (timeout > 0) { endtime = _PyTime_Add(now, timeout); @@ -81,7 +77,7 @@ _PyMutex_LockTimed(PyMutex *m, PyTime_t timeout, _PyLockFlags flags) for (;;) { if ((v & _Py_LOCKED) == 0) { // The lock is unlocked. Try to grab it. - if (_Py_atomic_compare_exchange_uint8(&m->v, &v, v|_Py_LOCKED)) { + if (_Py_atomic_compare_exchange_uint8(&m->_bits, &v, v|_Py_LOCKED)) { return PY_LOCK_ACQUIRED; } continue; @@ -102,17 +98,17 @@ _PyMutex_LockTimed(PyMutex *m, PyTime_t timeout, _PyLockFlags flags) if (!(v & _Py_HAS_PARKED)) { // We are the first waiter. Set the _Py_HAS_PARKED flag. newv = v | _Py_HAS_PARKED; - if (!_Py_atomic_compare_exchange_uint8(&m->v, &v, newv)) { + if (!_Py_atomic_compare_exchange_uint8(&m->_bits, &v, newv)) { continue; } } - int ret = _PyParkingLot_Park(&m->v, &newv, sizeof(newv), timeout, + int ret = _PyParkingLot_Park(&m->_bits, &newv, sizeof(newv), timeout, &entry, (flags & _PY_LOCK_DETACH) != 0); if (ret == Py_PARK_OK) { if (entry.handed_off) { // We own the lock now. - assert(_Py_atomic_load_uint8_relaxed(&m->v) & _Py_LOCKED); + assert(_Py_atomic_load_uint8_relaxed(&m->_bits) & _Py_LOCKED); return PY_LOCK_ACQUIRED; } } @@ -134,7 +130,7 @@ _PyMutex_LockTimed(PyMutex *m, PyTime_t timeout, _PyLockFlags flags) } } - v = _Py_atomic_load_uint8_relaxed(&m->v); + v = _Py_atomic_load_uint8_relaxed(&m->_bits); } } @@ -143,7 +139,9 @@ mutex_unpark(PyMutex *m, struct mutex_entry *entry, int has_more_waiters) { uint8_t v = 0; if (entry) { - PyTime_t now = _PyTime_MonotonicUnchecked(); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); int should_be_fair = now > entry->time_to_be_fair; entry->handed_off = should_be_fair; @@ -154,13 +152,13 @@ mutex_unpark(PyMutex *m, struct mutex_entry *entry, int has_more_waiters) v |= _Py_HAS_PARKED; } } - _Py_atomic_store_uint8(&m->v, v); + _Py_atomic_store_uint8(&m->_bits, v); } int _PyMutex_TryUnlock(PyMutex *m) { - uint8_t v = _Py_atomic_load_uint8(&m->v); + uint8_t v = _Py_atomic_load_uint8(&m->_bits); for (;;) { if ((v & _Py_LOCKED) == 0) { // error: the mutex is not locked @@ -168,24 +166,16 @@ _PyMutex_TryUnlock(PyMutex *m) } else if ((v & _Py_HAS_PARKED)) { // wake up a single thread - _PyParkingLot_Unpark(&m->v, (_Py_unpark_fn_t *)mutex_unpark, m); + _PyParkingLot_Unpark(&m->_bits, (_Py_unpark_fn_t *)mutex_unpark, m); return 0; } - else if (_Py_atomic_compare_exchange_uint8(&m->v, &v, _Py_UNLOCKED)) { + else if (_Py_atomic_compare_exchange_uint8(&m->_bits, &v, _Py_UNLOCKED)) { // fast-path: no waiters return 0; } } } -void -_PyMutex_UnlockSlow(PyMutex *m) -{ - if (_PyMutex_TryUnlock(m) < 0) { - Py_FatalError("unlocking mutex that is not locked"); - } -} - // _PyRawMutex stores a linked list of `struct raw_mutex_entry`, one for each // thread waiting on the mutex, directly in the mutex itself. struct raw_mutex_entry { @@ -277,12 +267,12 @@ _PyEvent_Notify(PyEvent *evt) void PyEvent_Wait(PyEvent *evt) { - while (!PyEvent_WaitTimed(evt, -1)) + while (!PyEvent_WaitTimed(evt, -1, /*detach=*/1)) ; } int -PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns) +PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns, int detach) { for (;;) { uint8_t v = _Py_atomic_load_uint8(&evt->v); @@ -298,36 +288,12 @@ PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns) uint8_t expected = _Py_HAS_PARKED; (void) _PyParkingLot_Park(&evt->v, &expected, sizeof(evt->v), - timeout_ns, NULL, 1); + timeout_ns, NULL, detach); return _Py_atomic_load_uint8(&evt->v) == _Py_LOCKED; } } -_PyEventRc * -_PyEventRc_New(void) -{ - _PyEventRc *erc = (_PyEventRc *)PyMem_RawCalloc(1, sizeof(_PyEventRc)); - if (erc != NULL) { - erc->refcount = 1; - } - return erc; -} - -void -_PyEventRc_Incref(_PyEventRc *erc) -{ - _Py_atomic_add_ssize(&erc->refcount, 1); -} - -void -_PyEventRc_Decref(_PyEventRc *erc) -{ - if (_Py_atomic_add_ssize(&erc->refcount, -1) == 1) { - PyMem_RawFree(erc); - } -} - static int unlock_once(_PyOnceFlag *o, int res) { @@ -386,6 +352,73 @@ _PyOnceFlag_CallOnceSlow(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg) } } +static int +recursive_mutex_is_owned_by(_PyRecursiveMutex *m, PyThread_ident_t tid) +{ + return _Py_atomic_load_ullong_relaxed(&m->thread) == tid; +} + +int +_PyRecursiveMutex_IsLockedByCurrentThread(_PyRecursiveMutex *m) +{ + return recursive_mutex_is_owned_by(m, PyThread_get_thread_ident_ex()); +} + +void +_PyRecursiveMutex_Lock(_PyRecursiveMutex *m) +{ + PyThread_ident_t thread = PyThread_get_thread_ident_ex(); + if (recursive_mutex_is_owned_by(m, thread)) { + m->level++; + return; + } + PyMutex_Lock(&m->mutex); + _Py_atomic_store_ullong_relaxed(&m->thread, thread); + assert(m->level == 0); +} + +PyLockStatus +_PyRecursiveMutex_LockTimed(_PyRecursiveMutex *m, PyTime_t timeout, _PyLockFlags flags) +{ + PyThread_ident_t thread = PyThread_get_thread_ident_ex(); + if (recursive_mutex_is_owned_by(m, thread)) { + m->level++; + return PY_LOCK_ACQUIRED; + } + PyLockStatus s = _PyMutex_LockTimed(&m->mutex, timeout, flags); + if (s == PY_LOCK_ACQUIRED) { + _Py_atomic_store_ullong_relaxed(&m->thread, thread); + assert(m->level == 0); + } + return s; +} + +void +_PyRecursiveMutex_Unlock(_PyRecursiveMutex *m) +{ + if (_PyRecursiveMutex_TryUnlock(m) < 0) { + Py_FatalError("unlocking a recursive mutex that is not " + "owned by the current thread"); + } +} + +int +_PyRecursiveMutex_TryUnlock(_PyRecursiveMutex *m) +{ + PyThread_ident_t thread = PyThread_get_thread_ident_ex(); + if (!recursive_mutex_is_owned_by(m, thread)) { + return -1; + } + if (m->level > 0) { + m->level--; + return 0; + } + assert(m->level == 0); + _Py_atomic_store_ullong_relaxed(&m->thread, 0); + PyMutex_Unlock(&m->mutex); + return 0; +} + #define _Py_WRITE_LOCKED 1 #define _PyRWMutex_READER_SHIFT 2 #define _Py_RWMUTEX_MAX_READERS (UINTPTR_MAX >> _PyRWMutex_READER_SHIFT) @@ -496,7 +529,7 @@ _PyRWMutex_Unlock(_PyRWMutex *rwmutex) void _PySeqLock_LockWrite(_PySeqLock *seqlock) { - // lock the entry by setting by moving to an odd sequence number + // lock by moving to an odd sequence number uint32_t prev = _Py_atomic_load_uint32_relaxed(&seqlock->sequence); while (1) { if (SEQLOCK_IS_UPDATING(prev)) { @@ -506,6 +539,7 @@ void _PySeqLock_LockWrite(_PySeqLock *seqlock) } else if (_Py_atomic_compare_exchange_uint32(&seqlock->sequence, &prev, prev + 1)) { // We've locked the cache + _Py_atomic_fence_release(); break; } else { @@ -516,14 +550,14 @@ void _PySeqLock_LockWrite(_PySeqLock *seqlock) void _PySeqLock_AbandonWrite(_PySeqLock *seqlock) { - uint32_t new_seq = seqlock->sequence - 1; + uint32_t new_seq = _Py_atomic_load_uint32_relaxed(&seqlock->sequence) - 1; assert(!SEQLOCK_IS_UPDATING(new_seq)); _Py_atomic_store_uint32(&seqlock->sequence, new_seq); } void _PySeqLock_UnlockWrite(_PySeqLock *seqlock) { - uint32_t new_seq = seqlock->sequence + 1; + uint32_t new_seq = _Py_atomic_load_uint32_relaxed(&seqlock->sequence) + 1; assert(!SEQLOCK_IS_UPDATING(new_seq)); _Py_atomic_store_uint32(&seqlock->sequence, new_seq); } @@ -539,26 +573,45 @@ uint32_t _PySeqLock_BeginRead(_PySeqLock *seqlock) return sequence; } -uint32_t _PySeqLock_EndRead(_PySeqLock *seqlock, uint32_t previous) +int _PySeqLock_EndRead(_PySeqLock *seqlock, uint32_t previous) { - // Synchronize again and validate that the entry hasn't been updated - // while we were readying the values. - if (_Py_atomic_load_uint32_acquire(&seqlock->sequence) == previous) { + // gh-121368: We need an explicit acquire fence here to ensure that + // this load of the sequence number is not reordered before any loads + // within the read lock. + _Py_atomic_fence_acquire(); + + if (_Py_atomic_load_uint32_relaxed(&seqlock->sequence) == previous) { return 1; - } + } - _Py_yield(); - return 0; + _Py_yield(); + return 0; } -uint32_t _PySeqLock_AfterFork(_PySeqLock *seqlock) +int _PySeqLock_AfterFork(_PySeqLock *seqlock) { // Synchronize again and validate that the entry hasn't been updated // while we were readying the values. - if (SEQLOCK_IS_UPDATING(seqlock->sequence)) { + if (SEQLOCK_IS_UPDATING(seqlock->sequence)) { seqlock->sequence = 0; return 1; - } + } + + return 0; +} + +#undef PyMutex_Lock +void +PyMutex_Lock(PyMutex *m) +{ + _PyMutex_LockTimed(m, -1, _PY_LOCK_DETACH); +} - return 0; +#undef PyMutex_Unlock +void +PyMutex_Unlock(PyMutex *m) +{ + if (_PyMutex_TryUnlock(m) < 0) { + Py_FatalError("unlocking mutex that is not locked"); + } } diff --git a/Python/marshal.c b/Python/marshal.c index daec7415b3fc7e..cf7011652513ae 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -7,12 +7,18 @@ and sharing. */ #include "Python.h" -#include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_code.h" // _PyCode_New() -#include "pycore_hashtable.h" // _Py_hashtable_t -#include "pycore_long.h" // _PyLong_DigitCount -#include "pycore_setobject.h" // _PySet_NextEntry() -#include "marshal.h" // Py_MARSHAL_VERSION +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_code.h" // _PyCode_New() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() +#include "pycore_hashtable.h" // _Py_hashtable_t +#include "pycore_long.h" // _PyLong_DigitCount +#include "pycore_setobject.h" // _PySet_NextEntry() +#include "marshal.h" // Py_MARSHAL_VERSION +#include "pycore_pystate.h" // _PyInterpreterState_GET() + +#ifdef __APPLE__ +# include "TargetConditionals.h" +#endif /* __APPLE__ */ /*[clinic input] module marshal @@ -33,47 +39,63 @@ module marshal * #if defined(MS_WINDOWS) && defined(_DEBUG) */ #if defined(MS_WINDOWS) -#define MAX_MARSHAL_STACK_DEPTH 1000 +# define MAX_MARSHAL_STACK_DEPTH 1000 #elif defined(__wasi__) -#define MAX_MARSHAL_STACK_DEPTH 1500 +# define MAX_MARSHAL_STACK_DEPTH 1500 +// TARGET_OS_IPHONE covers any non-macOS Apple platform. +// It won't be defined on older macOS SDKs +#elif defined(__APPLE__) && defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE +# define MAX_MARSHAL_STACK_DEPTH 1500 #else -#define MAX_MARSHAL_STACK_DEPTH 2000 +# define MAX_MARSHAL_STACK_DEPTH 2000 #endif +/* Supported types */ #define TYPE_NULL '0' #define TYPE_NONE 'N' #define TYPE_FALSE 'F' #define TYPE_TRUE 'T' #define TYPE_STOPITER 'S' #define TYPE_ELLIPSIS '.' -#define TYPE_INT 'i' -/* TYPE_INT64 is not generated anymore. - Supported for backward compatibility only. */ -#define TYPE_INT64 'I' -#define TYPE_FLOAT 'f' -#define TYPE_BINARY_FLOAT 'g' -#define TYPE_COMPLEX 'x' -#define TYPE_BINARY_COMPLEX 'y' -#define TYPE_LONG 'l' -#define TYPE_STRING 's' -#define TYPE_INTERNED 't' -#define TYPE_REF 'r' -#define TYPE_TUPLE '(' +#define TYPE_BINARY_FLOAT 'g' // Version 0 uses TYPE_FLOAT instead. +#define TYPE_BINARY_COMPLEX 'y' // Version 0 uses TYPE_COMPLEX instead. +#define TYPE_LONG 'l' // See also TYPE_INT. +#define TYPE_STRING 's' // Bytes. (Name comes from Python 2.) +#define TYPE_TUPLE '(' // See also TYPE_SMALL_TUPLE. #define TYPE_LIST '[' #define TYPE_DICT '{' #define TYPE_CODE 'c' #define TYPE_UNICODE 'u' #define TYPE_UNKNOWN '?' +// added in version 2: #define TYPE_SET '<' #define TYPE_FROZENSET '>' -#define FLAG_REF '\x80' /* with a type, add obj to index */ +// added in version 5: +#define TYPE_SLICE ':' +// Remember to update the version and documentation when adding new types. +/* Special cases for unicode strings (added in version 4) */ +#define TYPE_INTERNED 't' // Version 1+ #define TYPE_ASCII 'a' #define TYPE_ASCII_INTERNED 'A' -#define TYPE_SMALL_TUPLE ')' #define TYPE_SHORT_ASCII 'z' #define TYPE_SHORT_ASCII_INTERNED 'Z' +/* Special cases for small objects */ +#define TYPE_INT 'i' // All versions. 32-bit encoding. +#define TYPE_SMALL_TUPLE ')' // Version 4+ + +/* Supported for backwards compatibility */ +#define TYPE_COMPLEX 'x' // Generated for version 0 only. +#define TYPE_FLOAT 'f' // Generated for version 0 only. +#define TYPE_INT64 'I' // Not generated any more. + +/* References (added in version 3) */ +#define TYPE_REF 'r' +#define FLAG_REF '\x80' /* with a type, add obj to index */ + + +// Error codes: #define WFERR_OK 0 #define WFERR_UNMARSHALLABLE 1 #define WFERR_NESTEDTOODEEP 2 @@ -218,10 +240,6 @@ w_short_pstring(const void *s, Py_ssize_t n, WFILE *p) #define PyLong_MARSHAL_SHIFT 15 #define PyLong_MARSHAL_BASE ((short)1 << PyLong_MARSHAL_SHIFT) #define PyLong_MARSHAL_MASK (PyLong_MARSHAL_BASE - 1) -#if PyLong_SHIFT % PyLong_MARSHAL_SHIFT != 0 -#error "PyLong_SHIFT must be a multiple of PyLong_MARSHAL_SHIFT" -#endif -#define PyLong_MARSHAL_RATIO (PyLong_SHIFT / PyLong_MARSHAL_SHIFT) #define W_TYPE(t, p) do { \ w_byte((t) | flag, (p)); \ @@ -230,47 +248,106 @@ w_short_pstring(const void *s, Py_ssize_t n, WFILE *p) static PyObject * _PyMarshal_WriteObjectToString(PyObject *x, int version, int allow_code); +#define _r_digits(bitsize) \ +static void \ +_r_digits##bitsize(const uint ## bitsize ## _t *digits, Py_ssize_t n, \ + uint8_t negative, Py_ssize_t marshal_ratio, WFILE *p) \ +{ \ + /* set l to number of base PyLong_MARSHAL_BASE digits */ \ + Py_ssize_t l = (n - 1)*marshal_ratio; \ + uint ## bitsize ## _t d = digits[n - 1]; \ + \ + assert(marshal_ratio > 0); \ + assert(n >= 1); \ + assert(d != 0); /* a PyLong is always normalized */ \ + do { \ + d >>= PyLong_MARSHAL_SHIFT; \ + l++; \ + } while (d != 0); \ + if (l > SIZE32_MAX) { \ + p->depth--; \ + p->error = WFERR_UNMARSHALLABLE; \ + return; \ + } \ + w_long((long)(negative ? -l : l), p); \ + \ + for (Py_ssize_t i = 0; i < n - 1; i++) { \ + d = digits[i]; \ + for (Py_ssize_t j = 0; j < marshal_ratio; j++) { \ + w_short(d & PyLong_MARSHAL_MASK, p); \ + d >>= PyLong_MARSHAL_SHIFT; \ + } \ + assert(d == 0); \ + } \ + d = digits[n - 1]; \ + do { \ + w_short(d & PyLong_MARSHAL_MASK, p); \ + d >>= PyLong_MARSHAL_SHIFT; \ + } while (d != 0); \ +} +_r_digits(16) +_r_digits(32) +#undef _r_digits + static void w_PyLong(const PyLongObject *ob, char flag, WFILE *p) { - Py_ssize_t i, j, n, l; - digit d; - W_TYPE(TYPE_LONG, p); if (_PyLong_IsZero(ob)) { w_long((long)0, p); return; } - /* set l to number of base PyLong_MARSHAL_BASE digits */ - n = _PyLong_DigitCount(ob); - l = (n-1) * PyLong_MARSHAL_RATIO; - d = ob->long_value.ob_digit[n-1]; - assert(d != 0); /* a PyLong is always normalized */ - do { - d >>= PyLong_MARSHAL_SHIFT; - l++; - } while (d != 0); - if (l > SIZE32_MAX) { + PyLongExport long_export; + + if (PyLong_Export((PyObject *)ob, &long_export) < 0) { p->depth--; p->error = WFERR_UNMARSHALLABLE; return; } - w_long((long)(_PyLong_IsNegative(ob) ? -l : l), p); + if (!long_export.digits) { + int8_t sign = long_export.value < 0 ? -1 : 1; + uint64_t abs_value = Py_ABS(long_export.value); + uint64_t d = abs_value; + long l = 0; + + /* set l to number of base PyLong_MARSHAL_BASE digits */ + do { + d >>= PyLong_MARSHAL_SHIFT; + l += sign; + } while (d); + w_long(l, p); - for (i=0; i < n-1; i++) { - d = ob->long_value.ob_digit[i]; - for (j=0; j < PyLong_MARSHAL_RATIO; j++) { + d = abs_value; + do { w_short(d & PyLong_MARSHAL_MASK, p); d >>= PyLong_MARSHAL_SHIFT; - } - assert (d == 0); + } while (d); + return; + } + + const PyLongLayout *layout = PyLong_GetNativeLayout(); + Py_ssize_t marshal_ratio = layout->bits_per_digit/PyLong_MARSHAL_SHIFT; + + /* must be a multiple of PyLong_MARSHAL_SHIFT */ + assert(layout->bits_per_digit % PyLong_MARSHAL_SHIFT == 0); + assert(layout->bits_per_digit >= PyLong_MARSHAL_SHIFT); + + /* other assumptions on PyLongObject internals */ + assert(layout->bits_per_digit <= 32); + assert(layout->digits_order == -1); + assert(layout->digit_endianness == (PY_LITTLE_ENDIAN ? -1 : 1)); + assert(layout->digit_size == 2 || layout->digit_size == 4); + + if (layout->digit_size == 4) { + _r_digits32(long_export.digits, long_export.ndigits, + long_export.negative, marshal_ratio, p); } - d = ob->long_value.ob_digit[n-1]; - do { - w_short(d & PyLong_MARSHAL_MASK, p); - d >>= PyLong_MARSHAL_SHIFT; - } while (d != 0); + else { + _r_digits16(long_export.digits, long_export.ndigits, + long_export.negative, marshal_ratio, p); + } + PyLong_FreeExport(&long_export); } static void @@ -524,23 +601,29 @@ w_complex_object(PyObject *v, char flag, WFILE *p) return; } Py_ssize_t i = 0; - while (_PySet_NextEntry(v, &pos, &value, &hash)) { + Py_BEGIN_CRITICAL_SECTION(v); + while (_PySet_NextEntryRef(v, &pos, &value, &hash)) { PyObject *dump = _PyMarshal_WriteObjectToString(value, p->version, p->allow_code); if (dump == NULL) { p->error = WFERR_UNMARSHALLABLE; - Py_DECREF(pairs); - return; + Py_DECREF(value); + break; } PyObject *pair = PyTuple_Pack(2, dump, value); Py_DECREF(dump); + Py_DECREF(value); if (pair == NULL) { p->error = WFERR_NOMEMORY; - Py_DECREF(pairs); - return; + break; } PyList_SET_ITEM(pairs, i++, pair); } + Py_END_CRITICAL_SECTION(); + if (p->error == WFERR_UNMARSHALLABLE || p->error == WFERR_NOMEMORY) { + Py_DECREF(pairs); + return; + } assert(i == n); if (PyList_Sort(pairs)) { p->error = WFERR_NOMEMORY; @@ -597,6 +680,18 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_pstring(view.buf, view.len, p); PyBuffer_Release(&view); } + else if (PySlice_Check(v)) { + if (p->version < 5) { + w_byte(TYPE_UNKNOWN, p); + p->error = WFERR_UNMARSHALLABLE; + return; + } + PySliceObject *slice = (PySliceObject *)v; + W_TYPE(TYPE_SLICE, p); + w_object(slice->start, p); + w_object(slice->stop, p); + w_object(slice->step, p); + } else { W_TYPE(TYPE_UNKNOWN, p); p->error = WFERR_UNMARSHALLABLE; @@ -835,17 +930,62 @@ r_long64(RFILE *p) 1 /* signed */); } +#define _w_digits(bitsize) \ +static int \ +_w_digits##bitsize(uint ## bitsize ## _t *digits, Py_ssize_t size, \ + Py_ssize_t marshal_ratio, \ + int shorts_in_top_digit, RFILE *p) \ +{ \ + uint ## bitsize ## _t d; \ + \ + assert(size >= 1); \ + for (Py_ssize_t i = 0; i < size - 1; i++) { \ + d = 0; \ + for (Py_ssize_t j = 0; j < marshal_ratio; j++) { \ + int md = r_short(p); \ + if (md < 0 || md > PyLong_MARSHAL_BASE) { \ + goto bad_digit; \ + } \ + d += (uint ## bitsize ## _t)md << j*PyLong_MARSHAL_SHIFT; \ + } \ + digits[i] = d; \ + } \ + \ + d = 0; \ + for (Py_ssize_t j = 0; j < shorts_in_top_digit; j++) { \ + int md = r_short(p); \ + if (md < 0 || md > PyLong_MARSHAL_BASE) { \ + goto bad_digit; \ + } \ + /* topmost marshal digit should be nonzero */ \ + if (md == 0 && j == shorts_in_top_digit - 1) { \ + PyErr_SetString(PyExc_ValueError, \ + "bad marshal data (unnormalized long data)"); \ + return -1; \ + } \ + d += (uint ## bitsize ## _t)md << j*PyLong_MARSHAL_SHIFT; \ + } \ + assert(!PyErr_Occurred()); \ + /* top digit should be nonzero, else the resulting PyLong won't be \ + normalized */ \ + digits[size - 1] = d; \ + return 0; \ + \ +bad_digit: \ + if (!PyErr_Occurred()) { \ + PyErr_SetString(PyExc_ValueError, \ + "bad marshal data (digit out of range in long)"); \ + } \ + return -1; \ +} +_w_digits(32) +_w_digits(16) +#undef _w_digits + static PyObject * r_PyLong(RFILE *p) { - PyLongObject *ob; - long n, size, i; - int j, md, shorts_in_top_digit; - digit d; - - n = r_long(p); - if (n == 0) - return (PyObject *)_PyLong_New(0); + long n = r_long(p); if (n == -1 && PyErr_Occurred()) { return NULL; } @@ -855,51 +995,44 @@ r_PyLong(RFILE *p) return NULL; } - size = 1 + (Py_ABS(n) - 1) / PyLong_MARSHAL_RATIO; - shorts_in_top_digit = 1 + (Py_ABS(n) - 1) % PyLong_MARSHAL_RATIO; - ob = _PyLong_New(size); - if (ob == NULL) - return NULL; + const PyLongLayout *layout = PyLong_GetNativeLayout(); + Py_ssize_t marshal_ratio = layout->bits_per_digit/PyLong_MARSHAL_SHIFT; - _PyLong_SetSignAndDigitCount(ob, n < 0 ? -1 : 1, size); + /* must be a multiple of PyLong_MARSHAL_SHIFT */ + assert(layout->bits_per_digit % PyLong_MARSHAL_SHIFT == 0); + assert(layout->bits_per_digit >= PyLong_MARSHAL_SHIFT); - for (i = 0; i < size-1; i++) { - d = 0; - for (j=0; j < PyLong_MARSHAL_RATIO; j++) { - md = r_short(p); - if (md < 0 || md > PyLong_MARSHAL_BASE) - goto bad_digit; - d += (digit)md << j*PyLong_MARSHAL_SHIFT; - } - ob->long_value.ob_digit[i] = d; + /* other assumptions on PyLongObject internals */ + assert(layout->bits_per_digit <= 32); + assert(layout->digits_order == -1); + assert(layout->digit_endianness == (PY_LITTLE_ENDIAN ? -1 : 1)); + assert(layout->digit_size == 2 || layout->digit_size == 4); + + Py_ssize_t size = 1 + (Py_ABS(n) - 1) / marshal_ratio; + + assert(size >= 1); + + int shorts_in_top_digit = 1 + (Py_ABS(n) - 1) % marshal_ratio; + void *digits; + PyLongWriter *writer = PyLongWriter_Create(n < 0, size, &digits); + + if (writer == NULL) { + return NULL; } - d = 0; - for (j=0; j < shorts_in_top_digit; j++) { - md = r_short(p); - if (md < 0 || md > PyLong_MARSHAL_BASE) - goto bad_digit; - /* topmost marshal digit should be nonzero */ - if (md == 0 && j == shorts_in_top_digit - 1) { - Py_DECREF(ob); - PyErr_SetString(PyExc_ValueError, - "bad marshal data (unnormalized long data)"); - return NULL; - } - d += (digit)md << j*PyLong_MARSHAL_SHIFT; + int ret; + + if (layout->digit_size == 4) { + ret = _w_digits32(digits, size, marshal_ratio, shorts_in_top_digit, p); } - assert(!PyErr_Occurred()); - /* top digit should be nonzero, else the resulting PyLong won't be - normalized */ - ob->long_value.ob_digit[size-1] = d; - return (PyObject *)ob; - bad_digit: - Py_DECREF(ob); - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_ValueError, - "bad marshal data (digit out of range in long)"); + else { + ret = _w_digits16(digits, size, marshal_ratio, shorts_in_top_digit, p); } - return NULL; + if (ret < 0) { + PyLongWriter_Discard(writer); + return NULL; + } + return PyLongWriter_Finish(writer); } static double @@ -1140,7 +1273,7 @@ r_object(RFILE *p) case TYPE_ASCII_INTERNED: is_interned = 1; - /* fall through */ + _Py_FALLTHROUGH; case TYPE_ASCII: n = r_long(p); if (n < 0 || n > SIZE32_MAX) { @@ -1154,7 +1287,7 @@ r_object(RFILE *p) case TYPE_SHORT_ASCII_INTERNED: is_interned = 1; - /* fall through */ + _Py_FALLTHROUGH; case TYPE_SHORT_ASCII: n = r_byte(p); if (n == EOF) { @@ -1169,8 +1302,12 @@ r_object(RFILE *p) v = PyUnicode_FromKindAndData(PyUnicode_1BYTE_KIND, ptr, n); if (v == NULL) break; - if (is_interned) - PyUnicode_InternInPlace(&v); + if (is_interned) { + // marshal is meant to serialize .pyc files with code + // objects, and code-related strings are currently immortal. + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternImmortal(interp, &v); + } retval = v; R_REF(retval); break; @@ -1178,7 +1315,7 @@ r_object(RFILE *p) case TYPE_INTERNED: is_interned = 1; - /* fall through */ + _Py_FALLTHROUGH; case TYPE_UNICODE: { const char *buffer; @@ -1198,12 +1335,16 @@ r_object(RFILE *p) v = PyUnicode_DecodeUTF8(buffer, n, "surrogatepass"); } else { - v = PyUnicode_New(0, 0); + v = Py_GetConstant(Py_CONSTANT_EMPTY_STR); } if (v == NULL) break; - if (is_interned) - PyUnicode_InternInPlace(&v); + if (is_interned) { + // marshal is meant to serialize .pyc files with code + // objects, and code-related strings are currently immortal. + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_InternImmortal(interp, &v); + } retval = v; R_REF(retval); break; @@ -1510,6 +1651,32 @@ r_object(RFILE *p) retval = Py_NewRef(v); break; + case TYPE_SLICE: + { + Py_ssize_t idx = r_ref_reserve(flag, p); + PyObject *stop = NULL; + PyObject *step = NULL; + PyObject *start = r_object(p); + if (start == NULL) { + goto cleanup; + } + stop = r_object(p); + if (stop == NULL) { + goto cleanup; + } + step = r_object(p); + if (step == NULL) { + goto cleanup; + } + retval = PySlice_New(start, stop, step); + r_ref_insert(retval, idx, flag, p); + cleanup: + Py_XDECREF(start); + Py_XDECREF(stop); + Py_XDECREF(step); + break; + } + default: /* Bogus data got written, which isn't ideal. This will let you keep working and recover. */ @@ -1903,7 +2070,7 @@ machine architecture issues.\n\ Not all Python object types are supported; in general, only objects\n\ whose value is independent from a particular invocation of Python can be\n\ written and read by this module. The following types are supported:\n\ -None, integers, floating point numbers, strings, bytes, bytearrays,\n\ +None, integers, floating-point numbers, strings, bytes, bytearrays,\n\ tuples, lists, sets, dictionaries, and code objects, where it\n\ should be understood that tuples, lists and dictionaries are only\n\ supported as long as the values contained therein are themselves\n\ @@ -1914,7 +2081,7 @@ Variables:\n\ \n\ version -- indicates the format that the module uses. Version 0 is the\n\ historical format, version 1 shares interned strings and version 2\n\ - uses a binary format for floating point numbers.\n\ + uses a binary format for floating-point numbers.\n\ Version 3 shares common object references (New in version 3.4).\n\ \n\ Functions:\n\ @@ -1937,6 +2104,7 @@ marshal_module_exec(PyObject *mod) static PyModuleDef_Slot marshalmodule_slots[] = { {Py_mod_exec, marshal_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Python/modsupport.c b/Python/modsupport.c index e9abf304e6502c..517dc971f88c87 100644 --- a/Python/modsupport.c +++ b/Python/modsupport.c @@ -306,6 +306,7 @@ do_mkvalue(const char **p_format, va_list *p_va) return PyLong_FromSsize_t(va_arg(*p_va, Py_ssize_t)); #endif /* Fall through from 'n' to 'l' if Py_ssize_t is long */ + _Py_FALLTHROUGH; case 'l': return PyLong_FromLong(va_arg(*p_va, long)); @@ -647,3 +648,20 @@ PyModule_AddType(PyObject *module, PyTypeObject *type) return PyModule_AddObjectRef(module, name, (PyObject *)type); } + + +/* Exported functions for version helper macros */ + +#undef Py_PACK_FULL_VERSION +uint32_t +Py_PACK_FULL_VERSION(int x, int y, int z, int level, int serial) +{ + return _Py_PACK_FULL_VERSION(x, y, z, level, serial); +} + +#undef Py_PACK_VERSION +uint32_t +Py_PACK_VERSION(int x, int y) +{ + return Py_PACK_FULL_VERSION(x, y, 0, 0, 0); +} diff --git a/Python/object_stack.c b/Python/object_stack.c index bd9696822c8a9d..bda3190f45e0ff 100644 --- a/Python/object_stack.c +++ b/Python/object_stack.c @@ -8,24 +8,11 @@ extern _PyObjectStackChunk *_PyObjectStackChunk_New(void); extern void _PyObjectStackChunk_Free(_PyObjectStackChunk *); -static struct _Py_object_stack_freelist * -get_object_stack_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - return &freelists->object_stacks; -} - _PyObjectStackChunk * _PyObjectStackChunk_New(void) { - _PyObjectStackChunk *buf; - struct _Py_object_stack_freelist *obj_stack_freelist = get_object_stack_freelist(); - if (obj_stack_freelist->numfree > 0) { - buf = obj_stack_freelist->items; - obj_stack_freelist->items = buf->prev; - obj_stack_freelist->numfree--; - } - else { + _PyObjectStackChunk *buf = _Py_FREELIST_POP_MEM(object_stack_chunks); + if (buf == NULL) { // NOTE: we use PyMem_RawMalloc() here because this is used by the GC // during mimalloc heap traversal. In that context, it is not safe to // allocate mimalloc memory, such as via PyMem_Malloc(). @@ -43,17 +30,7 @@ void _PyObjectStackChunk_Free(_PyObjectStackChunk *buf) { assert(buf->n == 0); - struct _Py_object_stack_freelist *obj_stack_freelist = get_object_stack_freelist(); - if (obj_stack_freelist->numfree >= 0 && - obj_stack_freelist->numfree < _PyObjectStackChunk_MAXFREELIST) - { - buf->prev = obj_stack_freelist->items; - obj_stack_freelist->items = buf; - obj_stack_freelist->numfree++; - } - else { - PyMem_RawFree(buf); - } + _Py_FREELIST_FREE(object_stack_chunks, buf, PyMem_RawFree); } void @@ -87,22 +64,3 @@ _PyObjectStack_Merge(_PyObjectStack *dst, _PyObjectStack *src) dst->head = src->head; src->head = NULL; } - -void -_PyObjectStackChunk_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) -{ - if (!is_finalization) { - // Ignore requests to clear the free list during GC. We use object - // stacks during GC, so emptying the free-list is counterproductive. - return; - } - - struct _Py_object_stack_freelist *freelist = &freelists->object_stacks; - while (freelist->numfree > 0) { - _PyObjectStackChunk *buf = freelist->items; - freelist->items = buf->prev; - freelist->numfree--; - PyMem_RawFree(buf); - } - freelist->numfree = -1; -} diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index e664e638bdb749..09a834bb38fa67 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -1,10 +1,9 @@ static void *opcode_targets[256] = { &&TARGET_CACHE, - &&TARGET_BEFORE_ASYNC_WITH, - &&TARGET_BEFORE_WITH, - &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, &&TARGET_BINARY_SLICE, &&TARGET_BINARY_SUBSCR, + &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, + &&TARGET_CALL_FUNCTION_EX, &&TARGET_CHECK_EG_MATCH, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CLEANUP_THROW, @@ -16,13 +15,12 @@ static void *opcode_targets[256] = { &&TARGET_FORMAT_SIMPLE, &&TARGET_FORMAT_WITH_SPEC, &&TARGET_GET_AITER, - &&TARGET_RESERVED, &&TARGET_GET_ANEXT, + &&TARGET_RESERVED, &&TARGET_GET_ITER, &&TARGET_GET_LEN, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_INTERPRETER_EXIT, - &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_LOAD_BUILD_CLASS, &&TARGET_LOAD_LOCALS, &&TARGET_MAKE_FUNCTION, @@ -30,7 +28,9 @@ static void *opcode_targets[256] = { &&TARGET_MATCH_MAPPING, &&TARGET_MATCH_SEQUENCE, &&TARGET_NOP, + &&TARGET_NOT_TAKEN, &&TARGET_POP_EXCEPT, + &&TARGET_POP_ITER, &&TARGET_POP_TOP, &&TARGET_PUSH_EXC_INFO, &&TARGET_PUSH_NULL, @@ -45,7 +45,6 @@ static void *opcode_targets[256] = { &&TARGET_UNARY_NOT, &&TARGET_WITH_EXCEPT_START, &&TARGET_BINARY_OP, - &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_LIST, &&TARGET_BUILD_MAP, &&TARGET_BUILD_SET, @@ -53,7 +52,6 @@ static void *opcode_targets[256] = { &&TARGET_BUILD_STRING, &&TARGET_BUILD_TUPLE, &&TARGET_CALL, - &&TARGET_CALL_FUNCTION_EX, &&TARGET_CALL_INTRINSIC_1, &&TARGET_CALL_INTRINSIC_2, &&TARGET_CALL_KW, @@ -69,7 +67,6 @@ static void *opcode_targets[256] = { &&TARGET_DELETE_NAME, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, - &&TARGET_ENTER_EXECUTOR, &&TARGET_EXTENDED_ARG, &&TARGET_FOR_ITER, &&TARGET_GET_AWAITABLE, @@ -82,6 +79,7 @@ static void *opcode_targets[256] = { &&TARGET_LIST_APPEND, &&TARGET_LIST_EXTEND, &&TARGET_LOAD_ATTR, + &&TARGET_LOAD_COMMON_CONSTANT, &&TARGET_LOAD_CONST, &&TARGET_LOAD_DEREF, &&TARGET_LOAD_FAST, @@ -92,6 +90,8 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_FROM_DICT_OR_GLOBALS, &&TARGET_LOAD_GLOBAL, &&TARGET_LOAD_NAME, + &&TARGET_LOAD_SMALL_INT, + &&TARGET_LOAD_SPECIAL, &&TARGET_LOAD_SUPER_ATTR, &&TARGET_MAKE_CELL, &&TARGET_MAP_ADD, @@ -102,7 +102,6 @@ static void *opcode_targets[256] = { &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_RAISE_VARARGS, &&TARGET_RERAISE, - &&TARGET_RETURN_CONST, &&TARGET_SEND, &&TARGET_SET_ADD, &&TARGET_SET_FUNCTION_ATTRIBUTE, @@ -148,10 +147,12 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_UNICODE, + &&TARGET_BINARY_OP_EXTEND, &&TARGET_BINARY_OP_MULTIPLY_FLOAT, &&TARGET_BINARY_OP_MULTIPLY_INT, &&TARGET_BINARY_OP_SUBTRACT_FLOAT, @@ -163,30 +164,40 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_SUBSCR_TUPLE_INT, &&TARGET_CALL_ALLOC_AND_ENTER_INIT, &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_CALL_BOUND_METHOD_GENERAL, &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_CALL_BUILTIN_FAST, &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_CALL_BUILTIN_O, &&TARGET_CALL_ISINSTANCE, + &&TARGET_CALL_KW_BOUND_METHOD, + &&TARGET_CALL_KW_NON_PY, + &&TARGET_CALL_KW_PY, &&TARGET_CALL_LEN, &&TARGET_CALL_LIST_APPEND, &&TARGET_CALL_METHOD_DESCRIPTOR_FAST, &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, &&TARGET_CALL_METHOD_DESCRIPTOR_NOARGS, &&TARGET_CALL_METHOD_DESCRIPTOR_O, + &&TARGET_CALL_NON_PY_GENERAL, &&TARGET_CALL_PY_EXACT_ARGS, - &&TARGET_CALL_PY_WITH_DEFAULTS, + &&TARGET_CALL_PY_GENERAL, &&TARGET_CALL_STR_1, &&TARGET_CALL_TUPLE_1, &&TARGET_CALL_TYPE_1, &&TARGET_COMPARE_OP_FLOAT, &&TARGET_COMPARE_OP_INT, &&TARGET_COMPARE_OP_STR, + &&TARGET_CONTAINS_OP_DICT, + &&TARGET_CONTAINS_OP_SET, &&TARGET_FOR_ITER_GEN, &&TARGET_FOR_ITER_LIST, &&TARGET_FOR_ITER_RANGE, &&TARGET_FOR_ITER_TUPLE, + &&TARGET_JUMP_BACKWARD_JIT, + &&TARGET_JUMP_BACKWARD_NO_JIT, &&TARGET_LOAD_ATTR_CLASS, + &&TARGET_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, @@ -198,6 +209,8 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ATTR_WITH_HINT, + &&TARGET_LOAD_CONST_IMMORTAL, + &&TARGET_LOAD_CONST_MORTAL, &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_LOAD_SUPER_ATTR_ATTR, @@ -221,38 +234,25 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&TARGET_INSTRUMENTED_RESUME, &&TARGET_INSTRUMENTED_END_FOR, + &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, - &&TARGET_INSTRUMENTED_RETURN_VALUE, - &&TARGET_INSTRUMENTED_RETURN_CONST, - &&TARGET_INSTRUMENTED_YIELD_VALUE, &&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR, &&TARGET_INSTRUMENTED_FOR_ITER, - &&TARGET_INSTRUMENTED_CALL, &&TARGET_INSTRUMENTED_CALL_KW, &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, &&TARGET_INSTRUMENTED_INSTRUCTION, &&TARGET_INSTRUMENTED_JUMP_FORWARD, - &&TARGET_INSTRUMENTED_JUMP_BACKWARD, + &&TARGET_INSTRUMENTED_NOT_TAKEN, &&TARGET_INSTRUMENTED_POP_JUMP_IF_TRUE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_NONE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + &&TARGET_INSTRUMENTED_RESUME, + &&TARGET_INSTRUMENTED_RETURN_VALUE, + &&TARGET_INSTRUMENTED_YIELD_VALUE, + &&TARGET_INSTRUMENTED_CALL, + &&TARGET_INSTRUMENTED_JUMP_BACKWARD, &&TARGET_INSTRUMENTED_LINE, - &&_unknown_opcode, + &&TARGET_ENTER_EXECUTOR, }; diff --git a/Python/optimizer.c b/Python/optimizer.c index acd6d52c4a885f..b16695a3c3d33e 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1,6 +1,10 @@ #include "Python.h" + +#ifdef _Py_TIER2 + #include "opcode.h" #include "pycore_interp.h" +#include "pycore_backoff.h" #include "pycore_bitutils.h" // _Py_popcount32() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_opcode_metadata.h" // _PyOpcode_OpName[] @@ -9,7 +13,6 @@ #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_uop_ids.h" #include "pycore_jit.h" -#include "cpython/optimizer.h" #include #include #include @@ -20,7 +23,6 @@ #define MAX_EXECUTORS_SIZE 256 - static bool has_space_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr) { @@ -72,7 +74,7 @@ insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorO Py_INCREF(executor); if (instr->op.code == ENTER_EXECUTOR) { assert(index == instr->op.arg); - _Py_ExecutorClear(code->co_executors->executors[index]); + _Py_ExecutorDetach(code->co_executors->executors[index]); } else { assert(code->co_executors->size == index); @@ -89,119 +91,13 @@ insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorO instr->op.arg = index; } -int -PyUnstable_Replace_Executor(PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutorObject *new) -{ - if (instr->op.code != ENTER_EXECUTOR) { - PyErr_Format(PyExc_ValueError, "No executor to replace"); - return -1; - } - int index = instr->op.arg; - assert(index >= 0); - insert_executor(code, instr, index, new); - return 0; -} - -static int -never_optimize( - _PyOptimizerObject* self, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, - _PyExecutorObject **exec, - int Py_UNUSED(stack_entries)) -{ - /* Although it should be benign for this to be called, - * it shouldn't happen, so fail in debug builds. */ - assert(0 && "never optimize should never be called"); - return 0; -} - -PyTypeObject _PyDefaultOptimizer_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - .tp_name = "noop_optimizer", - .tp_basicsize = sizeof(_PyOptimizerObject), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, -}; - -static _PyOptimizerObject _PyOptimizer_Default = { - PyObject_HEAD_INIT(&_PyDefaultOptimizer_Type) - .optimize = never_optimize, - .resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD, - .backedge_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD, - .side_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD, -}; - -static uint32_t -shift_and_offset_threshold(uint32_t threshold) -{ - return (threshold << OPTIMIZER_BITS_IN_COUNTER) + (1 << 15); -} - -_PyOptimizerObject * -PyUnstable_GetOptimizer(void) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - assert(interp->optimizer_backedge_threshold == - shift_and_offset_threshold(interp->optimizer->backedge_threshold)); - assert(interp->optimizer_resume_threshold == - shift_and_offset_threshold(interp->optimizer->resume_threshold)); - if (interp->optimizer == &_PyOptimizer_Default) { - return NULL; - } - Py_INCREF(interp->optimizer); - return interp->optimizer; -} - static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *dependencies); +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies); static int -init_cold_exit_executor(_PyExecutorObject *executor, int oparg); - -static int cold_exits_initialized = 0; -static _PyExecutorObject COLD_EXITS[UOP_MAX_TRACE_LENGTH] = { 0 }; - -static const _PyBloomFilter EMPTY_FILTER = { 0 }; - -_PyOptimizerObject * -_Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject *optimizer) -{ - if (optimizer == NULL) { - optimizer = &_PyOptimizer_Default; - } - else if (cold_exits_initialized == 0) { - cold_exits_initialized = 1; - for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) { - if (init_cold_exit_executor(&COLD_EXITS[i], i)) { - return NULL; - } - } - } - _PyOptimizerObject *old = interp->optimizer; - if (old == NULL) { - old = &_PyOptimizer_Default; - } - Py_INCREF(optimizer); - interp->optimizer = optimizer; - interp->optimizer_backedge_threshold = shift_and_offset_threshold(optimizer->backedge_threshold); - interp->optimizer_resume_threshold = shift_and_offset_threshold(optimizer->resume_threshold); - interp->optimizer_side_threshold = optimizer->side_threshold; - if (optimizer == &_PyOptimizer_Default) { - assert(interp->optimizer_backedge_threshold > (1 << 16)); - assert(interp->optimizer_resume_threshold > (1 << 16)); - } - return old; -} - -int -PyUnstable_SetOptimizer(_PyOptimizerObject *optimizer) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyOptimizerObject *old = _Py_SetOptimizer(interp, optimizer); - Py_XDECREF(old); - return old == NULL ? -1 : 0; -} +uop_optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, + _PyExecutorObject **exec_ptr, int curr_stackentries, + bool progress_needed); /* Returns 1 if optimized, 0 if not optimized, and -1 for an error. * If optimized, *executor_ptr contains a new reference to the executor @@ -209,38 +105,49 @@ PyUnstable_SetOptimizer(_PyOptimizerObject *optimizer) int _PyOptimizer_Optimize( _PyInterpreterFrame *frame, _Py_CODEUNIT *start, - PyObject **stack_pointer, _PyExecutorObject **executor_ptr) + _PyStackRef *stack_pointer, _PyExecutorObject **executor_ptr, int chain_depth) { - PyCodeObject *code = (PyCodeObject *)frame->f_executable; + assert(_PyInterpreterState_GET()->jit); + // The first executor in a chain and the MAX_CHAIN_DEPTH'th executor *must* + // make progress in order to avoid infinite loops or excessively-long + // side-exit chains. We can only insert the executor into the bytecode if + // this is true, since a deopt won't infinitely re-enter the executor: + chain_depth %= MAX_CHAIN_DEPTH; + bool progress_needed = chain_depth == 0; + PyCodeObject *code = _PyFrame_GetCode(frame); assert(PyCode_Check(code)); - PyInterpreterState *interp = _PyInterpreterState_GET(); - if (!has_space_for_executor(code, start)) { + if (progress_needed && !has_space_for_executor(code, start)) { return 0; } - _PyOptimizerObject *opt = interp->optimizer; - int err = opt->optimize(opt, frame, start, executor_ptr, (int)(stack_pointer - _PyFrame_Stackbase(frame))); + int err = uop_optimize(frame, start, executor_ptr, (int)(stack_pointer - _PyFrame_Stackbase(frame)), progress_needed); if (err <= 0) { return err; } assert(*executor_ptr != NULL); - int index = get_index_for_executor(code, start); - if (index < 0) { - /* Out of memory. Don't raise and assume that the - * error will show up elsewhere. - * - * If an optimizer has already produced an executor, - * it might get confused by the executor disappearing, - * but there is not much we can do about that here. */ - Py_DECREF(*executor_ptr); - return 0; + if (progress_needed) { + int index = get_index_for_executor(code, start); + if (index < 0) { + /* Out of memory. Don't raise and assume that the + * error will show up elsewhere. + * + * If an optimizer has already produced an executor, + * it might get confused by the executor disappearing, + * but there is not much we can do about that here. */ + Py_DECREF(*executor_ptr); + return 0; + } + insert_executor(code, start, index, *executor_ptr); + } + else { + (*executor_ptr)->vm_data.code = NULL; } - insert_executor(code, start, index, *executor_ptr); + (*executor_ptr)->vm_data.chain_depth = chain_depth; assert((*executor_ptr)->vm_data.valid); return 1; } -_PyExecutorObject * -PyUnstable_GetExecutor(PyCodeObject *code, int offset) +static _PyExecutorObject * +get_executor_lock_held(PyCodeObject *code, int offset) { int code_len = (int)Py_SIZE(code); for (int i = 0 ; i < code_len;) { @@ -256,6 +163,16 @@ PyUnstable_GetExecutor(PyCodeObject *code, int offset) return NULL; } +_PyExecutorObject * +_Py_GetExecutor(PyCodeObject *code, int offset) +{ + _PyExecutorObject *executor; + Py_BEGIN_CRITICAL_SECTION(code); + executor = get_executor_lock_held(code, offset); + Py_END_CRITICAL_SECTION(); + return executor; +} + static PyObject * is_valid(PyObject *self, PyObject *Py_UNUSED(ignored)) { @@ -274,19 +191,16 @@ get_oparg(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyLong_FromUnsignedLong(((_PyExecutorObject *)self)->vm_data.oparg); } -static PyMethodDef executor_methods[] = { - { "is_valid", is_valid, METH_NOARGS, NULL }, - { "get_opcode", get_opcode, METH_NOARGS, NULL }, - { "get_oparg", get_oparg, METH_NOARGS, NULL }, - { NULL, NULL }, -}; - ///////////////////// Experimental UOp Optimizer ///////////////////// +static int executor_clear(_PyExecutorObject *executor); +static void unlink_executor(_PyExecutorObject *executor); + static void uop_dealloc(_PyExecutorObject *self) { _PyObject_GC_UNTRACK(self); - _Py_ExecutorClear(self); + assert(self->vm_data.code == NULL); + unlink_executor(self); #ifdef _Py_JIT _PyJIT_Free(self); #endif @@ -313,10 +227,27 @@ _PyUOpPrint(const _PyUOpInstruction *uop) else { printf("%s", name); } - printf(" (%d, target=%d, operand=%" PRIx64 ")", - uop->oparg, - uop->target, - (uint64_t)uop->operand); + switch(uop->format) { + case UOP_FORMAT_TARGET: + printf(" (%d, target=%d, operand=%#" PRIx64, + uop->oparg, + uop->target, + (uint64_t)uop->operand0); + break; + case UOP_FORMAT_JUMP: + printf(" (%d, jump_target=%d, operand=%#" PRIx64, + uop->oparg, + uop->jump_target, + (uint64_t)uop->operand0); + break; + default: + printf(" (%d, Unknown format)", uop->oparg); + } + if (_PyUop_Flags[uop->opcode] & HAS_ERROR_FLAG) { + printf(", error_target=%d", uop->error_target); + } + + printf(")"); } #endif @@ -353,7 +284,7 @@ uop_item(_PyExecutorObject *self, Py_ssize_t index) Py_DECREF(oname); return NULL; } - PyObject *operand = PyLong_FromUnsignedLongLong(self->trace[index].operand); + PyObject *operand = PyLong_FromUnsignedLongLong(self->trace[index].operand0); if (operand == NULL) { Py_DECREF(target); Py_DECREF(oparg); @@ -369,13 +300,6 @@ PySequenceMethods uop_as_sequence = { .sq_item = (ssizeargfunc)uop_item, }; -static int -executor_clear(PyObject *o) -{ - _Py_ExecutorClear((_PyExecutorObject *)o); - return 0; -} - static int executor_traverse(PyObject *o, visitproc visit, void *arg) { @@ -386,6 +310,35 @@ executor_traverse(PyObject *o, visitproc visit, void *arg) return 0; } +static PyObject * +get_jit_code(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ +#ifndef _Py_JIT + PyErr_SetString(PyExc_RuntimeError, "JIT support not enabled."); + return NULL; +#else + _PyExecutorObject *executor = (_PyExecutorObject *)self; + if (executor->jit_code == NULL || executor->jit_size == 0) { + Py_RETURN_NONE; + } + return PyBytes_FromStringAndSize(executor->jit_code, executor->jit_size); +#endif +} + +static PyMethodDef uop_executor_methods[] = { + { "is_valid", is_valid, METH_NOARGS, NULL }, + { "get_jit_code", get_jit_code, METH_NOARGS, NULL}, + { "get_opcode", get_opcode, METH_NOARGS, NULL }, + { "get_oparg", get_oparg, METH_NOARGS, NULL }, + { NULL, NULL }, +}; + +static int +executor_is_gc(PyObject *o) +{ + return !_Py_IsImmortal(o); +} + PyTypeObject _PyUOpExecutor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "uop_executor", @@ -394,9 +347,10 @@ PyTypeObject _PyUOpExecutor_Type = { .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_HAVE_GC, .tp_dealloc = (destructor)uop_dealloc, .tp_as_sequence = &uop_as_sequence, - .tp_methods = executor_methods, + .tp_methods = uop_executor_methods, .tp_traverse = executor_traverse, - .tp_clear = executor_clear, + .tp_clear = (inquiry)executor_clear, + .tp_is_gc = executor_is_gc, }; /* TO DO -- Generate these tables */ @@ -408,6 +362,14 @@ _PyUOp_Replacements[MAX_UOP_ID + 1] = { [_FOR_ITER] = _FOR_ITER_TIER_TWO, }; +static const uint8_t +is_for_iter_test[MAX_UOP_ID + 1] = { + [_GUARD_NOT_EXHAUSTED_RANGE] = 1, + [_GUARD_NOT_EXHAUSTED_LIST] = 1, + [_GUARD_NOT_EXHAUSTED_TUPLE] = 1, + [_FOR_ITER_TIER_TWO] = 1, +}; + static const uint16_t BRANCH_TO_GUARD[4][2] = { [POP_JUMP_IF_FALSE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_TRUE_POP, @@ -432,28 +394,39 @@ BRANCH_TO_GUARD[4][2] = { #endif -// Beware: Macro arg order differs from struct member order +static inline int +add_to_trace( + _PyUOpInstruction *trace, + int trace_length, + uint16_t opcode, + uint16_t oparg, + uint64_t operand, + uint32_t target) +{ + trace[trace_length].opcode = opcode; + trace[trace_length].format = UOP_FORMAT_TARGET; + trace[trace_length].target = target; + trace[trace_length].oparg = oparg; + trace[trace_length].operand0 = operand; +#ifdef Py_STATS + trace[trace_length].execution_count = 0; +#endif + return trace_length + 1; +} + #ifdef Py_DEBUG #define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \ assert(trace_length < max_length); \ - trace[trace_length].opcode = (OPCODE); \ - trace[trace_length].oparg = (OPARG); \ - trace[trace_length].target = (TARGET); \ - trace[trace_length].operand = (OPERAND); \ + trace_length = add_to_trace(trace, trace_length, (OPCODE), (OPARG), (OPERAND), (TARGET)); \ if (lltrace >= 2) { \ printf("%4d ADD_TO_TRACE: ", trace_length); \ - _PyUOpPrint(&trace[trace_length]); \ + _PyUOpPrint(&trace[trace_length-1]); \ printf("\n"); \ - } \ - trace_length++; + } #else #define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \ assert(trace_length < max_length); \ - trace[trace_length].opcode = (OPCODE); \ - trace[trace_length].oparg = (OPARG); \ - trace[trace_length].target = (TARGET); \ - trace[trace_length].operand = (OPERAND); \ - trace_length++; + trace_length = add_to_trace(trace, trace_length, (OPCODE), (OPARG), (OPERAND), (TARGET)); #endif #define INSTR_IP(INSTR, CODE) \ @@ -471,16 +444,16 @@ BRANCH_TO_GUARD[4][2] = { // Reserve space for N uops, plus 3 for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE #define RESERVE(needed) RESERVE_RAW((needed) + 3, _PyUOpName(opcode)) -// Trace stack operations (used by _PUSH_FRAME, _POP_FRAME) +// Trace stack operations (used by _PUSH_FRAME, _RETURN_VALUE) #define TRACE_STACK_PUSH() \ if (trace_stack_depth >= TRACE_STACK_SIZE) { \ DPRINTF(2, "Trace stack overflow\n"); \ OPT_STAT_INC(trace_stack_overflow); \ - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); \ - goto done; \ + return 0; \ } \ - assert(func->func_code == (PyObject *)code); \ + assert(func == NULL || func->func_code == (PyObject *)code); \ trace_stack[trace_stack_depth].func = func; \ + trace_stack[trace_stack_depth].code = code; \ trace_stack[trace_stack_depth].instr = instr; \ trace_stack_depth++; #define TRACE_STACK_POP() \ @@ -489,10 +462,11 @@ BRANCH_TO_GUARD[4][2] = { } \ trace_stack_depth--; \ func = trace_stack[trace_stack_depth].func; \ - code = (PyCodeObject *)trace_stack[trace_stack_depth].func->func_code; \ + code = trace_stack[trace_stack_depth].code; \ + assert(func == NULL || func->func_code == (PyObject *)code); \ instr = trace_stack[trace_stack_depth].instr; -/* Returns 1 on success, +/* Returns the length of the trace on success, * 0 if it failed to produce a worthwhile trace, * and -1 on an error. */ @@ -502,23 +476,26 @@ translate_bytecode_to_trace( _Py_CODEUNIT *instr, _PyUOpInstruction *trace, int buffer_size, - _PyBloomFilter *dependencies) + _PyBloomFilter *dependencies, bool progress_needed) { - bool progress_needed = true; - PyCodeObject *code = (PyCodeObject *)frame->f_executable; - PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + bool first = true; + PyCodeObject *code = _PyFrame_GetCode(frame); + PyFunctionObject *func = _PyFrame_GetFunction(frame); assert(PyFunction_Check(func)); PyCodeObject *initial_code = code; _Py_BloomFilter_Add(dependencies, initial_code); _Py_CODEUNIT *initial_instr = instr; int trace_length = 0; - int max_length = buffer_size; + // Leave space for possible trailing _EXIT_TRACE + int max_length = buffer_size-2; struct { PyFunctionObject *func; + PyCodeObject *code; _Py_CODEUNIT *instr; } trace_stack[TRACE_STACK_SIZE]; int trace_stack_depth = 0; int confidence = CONFIDENCE_RANGE; // Adjusted by branch instructions + bool jump_seen = false; #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -528,37 +505,35 @@ translate_bytecode_to_trace( } #endif - DPRINTF(4, + DPRINTF(2, "Optimizing %s (%s:%d) at byte offset %d\n", PyUnicode_AsUTF8(code->co_qualname), PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, 2 * INSTR_IP(initial_instr, code)); + ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)instr, INSTR_IP(instr, code)); + ADD_TO_TRACE(_MAKE_WARM, 0, 0, 0); uint32_t target = 0; -top: // Jump here after _PUSH_FRAME or likely branches for (;;) { target = INSTR_IP(instr, code); - RESERVE_RAW(2, "epilogue"); // Always need space for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE - ADD_TO_TRACE(_CHECK_VALIDITY_AND_SET_IP, 0, (uintptr_t)instr, target); + // Need space for _DEOPT + max_length--; uint32_t opcode = instr->op.code; uint32_t oparg = instr->op.arg; - uint32_t extended = 0; - - DPRINTF(3, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); - if (opcode == ENTER_EXECUTOR) { - assert(oparg < 256); - _PyExecutorObject *executor = code->co_executors->executors[oparg]; - opcode = executor->vm_data.opcode; - DPRINTF(2, " * ENTER_EXECUTOR -> %s\n", _PyOpcode_OpName[opcode]); - oparg = executor->vm_data.oparg; + if (!first && instr == initial_instr) { + // We have looped around to the start: + RESERVE(1); + ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); + goto done; } + DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); + if (opcode == EXTENDED_ARG) { instr++; - extended = 1; opcode = instr->op.code; oparg = (oparg << 8) | instr->op.arg; if (opcode == EXTENDED_ARG) { @@ -566,25 +541,50 @@ translate_bytecode_to_trace( goto done; } } + if (opcode == ENTER_EXECUTOR) { + // We have a couple of options here. We *could* peek "underneath" + // this executor and continue tracing, which could give us a longer, + // more optimizeable trace (at the expense of lots of duplicated + // tier two code). Instead, we choose to just end here and stitch to + // the other trace, which allows a side-exit traces to rejoin the + // "main" trace periodically (and also helps protect us against + // pathological behavior where the amount of tier two code explodes + // for a medium-length, branchy code path). This seems to work + // better in practice, but in the future we could be smarter about + // what we do here: + goto done; + } assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); + if (OPCODE_HAS_NO_SAVE_IP(opcode)) { + RESERVE_RAW(2, "_CHECK_VALIDITY"); + ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); + } + else { + RESERVE_RAW(2, "_CHECK_VALIDITY_AND_SET_IP"); + ADD_TO_TRACE(_CHECK_VALIDITY_AND_SET_IP, 0, (uintptr_t)instr, target); + } /* Special case the first instruction, * so that we can guarantee forward progress */ - if (progress_needed) { - progress_needed = false; - if (opcode == JUMP_BACKWARD || opcode == JUMP_BACKWARD_NO_INTERRUPT) { - instr += 1 + _PyOpcode_Caches[opcode] - (int32_t)oparg; - initial_instr = instr; - continue; - } - else { - if (OPCODE_HAS_DEOPT(opcode)) { - opcode = _PyOpcode_Deopt[opcode]; - } - assert(!OPCODE_HAS_DEOPT(opcode)); + if (first && progress_needed) { + assert(first); + if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { + opcode = _PyOpcode_Deopt[opcode]; } + assert(!OPCODE_HAS_EXIT(opcode)); + assert(!OPCODE_HAS_DEOPT(opcode)); } + if (OPCODE_HAS_EXIT(opcode)) { + // Make space for side exit and final _EXIT_TRACE: + RESERVE_RAW(2, "_EXIT_TRACE"); + max_length--; + } + if (OPCODE_HAS_ERROR(opcode)) { + // Make space for error stub and final _EXIT_TRACE: + RESERVE_RAW(2, "_ERROR_POP_N"); + max_length--; + } switch (opcode) { case POP_JUMP_IF_NONE: case POP_JUMP_IF_NOT_NONE: @@ -596,17 +596,15 @@ translate_bytecode_to_trace( int bitcount = _Py_popcount32(counter); int jump_likely = bitcount > 8; /* If bitcount is 8 (half the jumps were taken), adjust confidence by 50%. - If it's 16 or 0 (all or none were taken), adjust by 10% - (since the future is still somewhat uncertain). For values in between, adjust proportionally. */ if (jump_likely) { - confidence = confidence * (bitcount + 2) / 20; + confidence = confidence * bitcount / 16; } else { - confidence = confidence * (18 - bitcount) / 20; + confidence = confidence * (16 - bitcount) / 16; } uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; - DPRINTF(2, "%d: %s(%d): counter=%x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n", + DPRINTF(2, "%d: %s(%d): counter=%04x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n", target, _PyOpcode_OpName[opcode], oparg, counter, bitcount, jump_likely, confidence, _PyUOpName(uopcode)); if (confidence < CONFIDENCE_CUTOFF) { @@ -617,30 +615,30 @@ translate_bytecode_to_trace( _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; _Py_CODEUNIT *target_instr = next_instr + oparg; if (jump_likely) { - DPRINTF(2, "Jump likely (%x = %d bits), continue at byte offset %d\n", + DPRINTF(2, "Jump likely (%04x = %d bits), continue at byte offset %d\n", instr[1].cache, bitcount, 2 * INSTR_IP(target_instr, code)); instr = target_instr; - ADD_TO_TRACE(uopcode, max_length, 0, INSTR_IP(next_instr, code)); + ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(next_instr, code)); goto top; } - ADD_TO_TRACE(uopcode, max_length, 0, INSTR_IP(target_instr, code)); + ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(target_instr, code)); break; } case JUMP_BACKWARD: + case JUMP_BACKWARD_JIT: + ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); + _Py_FALLTHROUGH; case JUMP_BACKWARD_NO_INTERRUPT: { - _Py_CODEUNIT *target = instr + 1 + _PyOpcode_Caches[opcode] - (int)oparg; - if (target == initial_instr) { - /* We have looped round to the start */ - RESERVE(1); - ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); - } - else { + instr += 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] - (int)oparg; + if (jump_seen) { OPT_STAT_INC(inner_loop); DPRINTF(2, "JUMP_BACKWARD not to top ends trace\n"); + goto done; } - goto done; + jump_seen = true; + goto top; } case JUMP_FORWARD: @@ -651,17 +649,24 @@ translate_bytecode_to_trace( break; } + case RESUME: + /* Use a special tier 2 version of RESUME_CHECK to allow traces to + * start with RESUME_CHECK */ + ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); + break; + default: { const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; if (expansion->nuops > 0) { // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) int nuops = expansion->nuops; - RESERVE(nuops); - if (expansion->uops[nuops-1].uop == _POP_FRAME) { + RESERVE(nuops + 1); /* One extra for exit */ + int16_t last_op = expansion->uops[nuops-1].uop; + if (last_op == _RETURN_VALUE || last_op == _RETURN_GENERATOR || last_op == _YIELD_VALUE) { // Check for trace stack underflow now: // We can't bail e.g. in the middle of - // LOAD_CONST + _POP_FRAME. + // LOAD_CONST + _RETURN_VALUE. if (trace_stack_depth == 0) { DPRINTF(2, "Trace stack underflow\n"); OPT_STAT_INC(trace_stack_underflow); @@ -701,12 +706,14 @@ translate_bytecode_to_trace( case OPARG_REPLACED: uop = _PyUOp_Replacements[uop]; assert(uop != 0); - if (uop == _FOR_ITER_TIER_TWO) { - target += 1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 2 + extended; - assert(_PyCode_CODE(code)[target-2].op.code == END_FOR || - _PyCode_CODE(code)[target-2].op.code == INSTRUMENTED_END_FOR); - assert(_PyCode_CODE(code)[target-1].op.code == POP_TOP); +#ifdef Py_DEBUG + { + uint32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + (oparg > 255); + uint32_t jump_target = next_inst + oparg; + assert(_Py_GetBaseCodeUnit(code, jump_target).op.code == END_FOR); + assert(_Py_GetBaseCodeUnit(code, jump_target+1).op.code == POP_ITER); } +#endif break; default: fprintf(stderr, @@ -716,12 +723,22 @@ translate_bytecode_to_trace( expansion->uops[i].offset); Py_FatalError("garbled expansion"); } - ADD_TO_TRACE(uop, oparg, operand, target); - if (uop == _POP_FRAME) { + + if (uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { TRACE_STACK_POP(); - /* Set the operand to the function object returned to, - * to assist optimization passes */ - trace[trace_length-1].operand = (uintptr_t)func; + /* Set the operand to the function or code object returned to, + * to assist optimization passes. (See _PUSH_FRAME below.) + */ + if (func != NULL) { + operand = (uintptr_t)func; + } + else if (code != NULL) { + operand = (uintptr_t)code | 1; + } + else { + operand = 0; + } + ADD_TO_TRACE(uop, oparg, operand, target); DPRINTF(2, "Returning to %s (%s:%d) at byte offset %d\n", PyUnicode_AsUTF8(code->co_qualname), @@ -730,17 +747,31 @@ translate_bytecode_to_trace( 2 * INSTR_IP(instr, code)); goto top; } + if (uop == _PUSH_FRAME) { assert(i + 1 == nuops); + if (opcode == FOR_ITER_GEN || + opcode == LOAD_ATTR_PROPERTY || + opcode == BINARY_SUBSCR_GETITEM || + opcode == SEND_GEN) + { + DPRINTF(2, "Bailing due to dynamic target\n"); + ADD_TO_TRACE(uop, oparg, 0, target); + ADD_TO_TRACE(_DYNAMIC_EXIT, 0, 0, 0); + goto done; + } + assert(_PyOpcode_Deopt[opcode] == CALL || _PyOpcode_Deopt[opcode] == CALL_KW); int func_version_offset = offsetof(_PyCallCache, func_version)/sizeof(_Py_CODEUNIT) // Add one to account for the actual opcode/oparg pair: + 1; uint32_t func_version = read_u32(&instr[func_version_offset].cache); - PyFunctionObject *new_func = _PyFunction_LookupByVersion(func_version); - DPRINTF(3, "Function object: %p\n", func); - if (new_func != NULL) { - PyCodeObject *new_code = (PyCodeObject *)PyFunction_GET_CODE(new_func); + PyCodeObject *new_code = NULL; + PyFunctionObject *new_func = + _PyFunction_LookupByVersion(func_version, (PyObject **) &new_code); + DPRINTF(2, "Function: version=%#x; new_func=%p, new_code=%p\n", + (int)func_version, new_func, new_code); + if (new_code != NULL) { if (new_code == code) { // Recursive call, bail (we could be here forever). DPRINTF(2, "Bailing on recursive call to %s (%s:%d)\n", @@ -748,6 +779,7 @@ translate_bytecode_to_trace( PyUnicode_AsUTF8(new_code->co_filename), new_code->co_firstlineno); OPT_STAT_INC(recursive_call); + ADD_TO_TRACE(uop, oparg, 0, target); ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); goto done; } @@ -756,6 +788,7 @@ translate_bytecode_to_trace( // Perhaps it may happen again, so don't bother tracing. // TODO: Reason about this -- is it better to bail or not? DPRINTF(2, "Bailing because co_version != func_version\n"); + ADD_TO_TRACE(uop, oparg, 0, target); ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); goto done; } @@ -763,9 +796,22 @@ translate_bytecode_to_trace( instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1; TRACE_STACK_PUSH(); _Py_BloomFilter_Add(dependencies, new_code); - /* Set the operand to the callee's code object, - * to assist optimization passes */ - trace[trace_length-1].operand = (uintptr_t)new_func; + /* Set the operand to the callee's function or code object, + * to assist optimization passes. + * We prefer setting it to the function (for remove_globals()) + * but if that's not available but the code is available, + * use the code, setting the low bit so the optimizer knows. + */ + if (new_func != NULL) { + operand = (uintptr_t)new_func; + } + else if (new_code != NULL) { + operand = (uintptr_t)new_code | 1; + } + else { + operand = 0; + } + ADD_TO_TRACE(uop, oparg, operand, target); code = new_code; func = new_func; instr = _PyCode_CODE(code); @@ -777,9 +823,23 @@ translate_bytecode_to_trace( 2 * INSTR_IP(instr, code)); goto top; } - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); + DPRINTF(2, "Bail, new_code == NULL\n"); + ADD_TO_TRACE(uop, oparg, 0, target); + ADD_TO_TRACE(_DYNAMIC_EXIT, 0, 0, 0); goto done; } + + if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) { + assert(i + 1 == nuops); + _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + assert(next_instr->op.code == STORE_FAST); + operand = next_instr->op.arg; + // Skip the STORE_FAST: + instr++; + } + + // All other instructions + ADD_TO_TRACE(uop, oparg, operand, target); } break; } @@ -793,6 +853,14 @@ translate_bytecode_to_trace( instr++; // Add cache size for opcode instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + + if (opcode == CALL_LIST_APPEND) { + assert(instr->op.code == POP_TOP); + instr++; + } + top: + // Jump here after _PUSH_FRAME or likely branches. + first = false; } // End for (;;) done: @@ -800,27 +868,31 @@ translate_bytecode_to_trace( TRACE_STACK_POP(); } assert(code == initial_code); - // Skip short traces like _SET_IP, LOAD_FAST, _SET_IP, _EXIT_TRACE - if (progress_needed || trace_length < 5) { + // Skip short traces where we can't even translate a single instruction: + if (first) { OPT_STAT_INC(trace_too_short); - DPRINTF(4, - "No trace for %s (%s:%d) at byte offset %d\n", + DPRINTF(2, + "No trace for %s (%s:%d) at byte offset %d (no progress)\n", PyUnicode_AsUTF8(code->co_qualname), PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, 2 * INSTR_IP(initial_instr, code)); return 0; } - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + if (!is_terminator(&trace[trace_length-1])) { + /* Allow space for _EXIT_TRACE */ + max_length += 2; + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + } DPRINTF(1, - "Created a trace for %s (%s:%d) at byte offset %d -- length %d\n", + "Created a proto-trace for %s (%s:%d) at byte offset %d -- length %d\n", PyUnicode_AsUTF8(code->co_qualname), PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, 2 * INSTR_IP(initial_instr, code), trace_length); - OPT_HIST(trace_length + buffer_size - max_length, trace_length_hist); - return 1; + OPT_HIST(trace_length, trace_length_hist); + return trace_length; } #undef RESERVE @@ -833,43 +905,107 @@ translate_bytecode_to_trace( #define SET_BIT(array, bit) (array[(bit)>>5] |= (1<<((bit)&31))) #define BIT_IS_SET(array, bit) (array[(bit)>>5] & (1<<((bit)&31))) -/* Count the number of used uops, and mark them in the bit vector `used`. - * This can be done in a single pass using simple reachability analysis, - * as there are no backward jumps. - * NOPs are excluded from the count. +/* Count the number of unused uops and exits */ static int -compute_used(_PyUOpInstruction *buffer, uint32_t *used, int *exit_count_ptr) +count_exits(_PyUOpInstruction *buffer, int length) { - int count = 0; int exit_count = 0; - SET_BIT(used, 0); - for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) { - if (!BIT_IS_SET(used, i)) { - continue; - } - count++; + for (int i = 0; i < length; i++) { int opcode = buffer[i].opcode; - if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { + if (opcode == _EXIT_TRACE || opcode == _DYNAMIC_EXIT) { exit_count++; } - if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE) { - continue; + } + return exit_count; +} + +static void make_exit(_PyUOpInstruction *inst, int opcode, int target) +{ + inst->opcode = opcode; + inst->oparg = 0; + inst->operand0 = 0; + inst->format = UOP_FORMAT_TARGET; + inst->target = target; +#ifdef Py_STATS + inst->execution_count = 0; +#endif +} + +/* Convert implicit exits, errors and deopts + * into explicit ones. */ +static int +prepare_for_execution(_PyUOpInstruction *buffer, int length) +{ + int32_t current_jump = -1; + int32_t current_jump_target = -1; + int32_t current_error = -1; + int32_t current_error_target = -1; + int32_t current_popped = -1; + int32_t current_exit_op = -1; + /* Leaving in NOPs slows down the interpreter and messes up the stats */ + _PyUOpInstruction *copy_to = &buffer[0]; + for (int i = 0; i < length; i++) { + _PyUOpInstruction *inst = &buffer[i]; + if (inst->opcode != _NOP) { + if (copy_to != inst) { + *copy_to = *inst; + } + copy_to++; } - /* All other micro-ops fall through, so i+1 is reachable */ - SET_BIT(used, i+1); - assert(opcode <= MAX_UOP_ID); - if (_PyUop_Flags[opcode] & HAS_JUMP_FLAG) { - /* Mark target as reachable */ - SET_BIT(used, buffer[i].oparg); + } + length = (int)(copy_to - buffer); + int next_spare = length; + for (int i = 0; i < length; i++) { + _PyUOpInstruction *inst = &buffer[i]; + int opcode = inst->opcode; + int32_t target = (int32_t)uop_get_target(inst); + if (_PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) { + uint16_t exit_op = (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) ? + _EXIT_TRACE : _DEOPT; + int32_t jump_target = target; + if (is_for_iter_test[opcode]) { + /* Target the POP_TOP immediately after the END_FOR, + * leaving only the iterator on the stack. */ + int extended_arg = inst->oparg > 255; + int32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + extended_arg; + jump_target = next_inst + inst->oparg + 1; + } + if (jump_target != current_jump_target || current_exit_op != exit_op) { + make_exit(&buffer[next_spare], exit_op, jump_target); + current_exit_op = exit_op; + current_jump_target = jump_target; + current_jump = next_spare; + next_spare++; + } + buffer[i].jump_target = current_jump; + buffer[i].format = UOP_FORMAT_JUMP; } - if (opcode == NOP) { - count--; - UNSET_BIT(used, i); + if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) { + int popped = (_PyUop_Flags[opcode] & HAS_ERROR_NO_POP_FLAG) ? + 0 : _PyUop_num_popped(opcode, inst->oparg); + if (target != current_error_target || popped != current_popped) { + current_popped = popped; + current_error = next_spare; + current_error_target = target; + make_exit(&buffer[next_spare], _ERROR_POP_N, 0); + buffer[next_spare].oparg = popped; + buffer[next_spare].operand0 = target; + next_spare++; + } + buffer[i].error_target = current_error; + if (buffer[i].format == UOP_FORMAT_TARGET) { + buffer[i].format = UOP_FORMAT_JUMP; + buffer[i].jump_target = 0; + } + } + if (opcode == _JUMP_TO_TOP) { + assert(buffer[0].opcode == _START_EXECUTOR); + buffer[i].format = UOP_FORMAT_JUMP; + buffer[i].jump_target = 1; } } - *exit_count_ptr = exit_count; - return count; + return next_spare; } /* Executor side exits */ @@ -888,56 +1024,111 @@ allocate_executor(int exit_count, int length) return res; } +#ifdef Py_DEBUG + +#define CHECK(PRED) \ +if (!(PRED)) { \ + printf(#PRED " at %d\n", i); \ + assert(0); \ +} + +static int +target_unused(int opcode) +{ + return (_PyUop_Flags[opcode] & (HAS_ERROR_FLAG | HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) == 0; +} + +static void +sanity_check(_PyExecutorObject *executor) +{ + for (uint32_t i = 0; i < executor->exit_count; i++) { + _PyExitData *exit = &executor->exits[i]; + CHECK(exit->target < (1 << 25)); + } + bool ended = false; + uint32_t i = 0; + CHECK(executor->trace[0].opcode == _START_EXECUTOR); + for (; i < executor->code_size; i++) { + const _PyUOpInstruction *inst = &executor->trace[i]; + uint16_t opcode = inst->opcode; + CHECK(opcode <= MAX_UOP_ID); + CHECK(_PyOpcode_uop_name[opcode] != NULL); + switch(inst->format) { + case UOP_FORMAT_TARGET: + CHECK(target_unused(opcode)); + break; + case UOP_FORMAT_JUMP: + CHECK(inst->jump_target < executor->code_size); + break; + } + if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) { + CHECK(inst->format == UOP_FORMAT_JUMP); + CHECK(inst->error_target < executor->code_size); + } + if (is_terminator(inst)) { + ended = true; + i++; + break; + } + } + CHECK(ended); + for (; i < executor->code_size; i++) { + const _PyUOpInstruction *inst = &executor->trace[i]; + uint16_t opcode = inst->opcode; + CHECK( + opcode == _DEOPT || + opcode == _EXIT_TRACE || + opcode == _ERROR_POP_N); + } +} + +#undef CHECK +#endif + /* Makes an executor from a buffer of uops. * Account for the buffer having gaps and NOPs by computing a "used" * bit vector and only copying the used uops. Here "used" means reachable * and not a NOP. */ static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *dependencies) +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies) { - uint32_t used[(UOP_MAX_TRACE_LENGTH + 31)/32] = { 0 }; - int exit_count; - int length = compute_used(buffer, used, &exit_count); - _PyExecutorObject *executor = allocate_executor(exit_count, length+1); + int exit_count = count_exits(buffer, length); + _PyExecutorObject *executor = allocate_executor(exit_count, length); if (executor == NULL) { return NULL; } + /* Initialize exits */ for (int i = 0; i < exit_count; i++) { - executor->exits[i].executor = &COLD_EXITS[i]; - executor->exits[i].temperature = 0; + executor->exits[i].executor = NULL; + executor->exits[i].temperature = initial_temperature_backoff_counter(); } int next_exit = exit_count-1; _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length]; - /* Scan backwards, so that we see the destinations of jumps before the jumps themselves. */ - for (int i = UOP_MAX_TRACE_LENGTH-1; i >= 0; i--) { - if (!BIT_IS_SET(used, i)) { - continue; - } - *dest = buffer[i]; + assert(buffer[0].opcode == _START_EXECUTOR); + buffer[0].operand0 = (uint64_t)executor; + for (int i = length-1; i >= 0; i--) { int opcode = buffer[i].opcode; - if (opcode == _POP_JUMP_IF_FALSE || - opcode == _POP_JUMP_IF_TRUE) - { - /* The oparg of the target will already have been set to its new offset */ - int oparg = dest->oparg; - dest->oparg = buffer[oparg].oparg; + dest--; + *dest = buffer[i]; + assert(opcode != _POP_JUMP_IF_FALSE && opcode != _POP_JUMP_IF_TRUE); + if (opcode == _EXIT_TRACE) { + _PyExitData *exit = &executor->exits[next_exit]; + exit->target = buffer[i].target; + dest->operand0 = (uint64_t)exit; + next_exit--; } - if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { - executor->exits[next_exit].target = buffer[i].target; - dest->exit_index = next_exit; + if (opcode == _DYNAMIC_EXIT) { + _PyExitData *exit = &executor->exits[next_exit]; + exit->target = 0; + dest->operand0 = (uint64_t)exit; next_exit--; } - /* Set the oparg to be the destination offset, - * so that we can set the oparg of earlier jumps correctly. */ - buffer[i].oparg = (uint16_t)(dest - executor->trace); - dest--; } assert(next_exit == -1); assert(dest == executor->trace); - dest->opcode = _START_EXECUTOR; - dest->operand = (uintptr_t)executor; + assert(dest->opcode == _START_EXECUTOR); _Py_ExecutorInit(executor, dependencies); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -946,18 +1137,23 @@ make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *depende lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } if (lltrace >= 2) { - printf("Optimized executor (length %d):\n", length); + printf("Optimized trace (length %d):\n", length); for (int i = 0; i < length; i++) { printf("%4d OPTIMIZED: ", i); _PyUOpPrint(&executor->trace[i]); printf("\n"); } } + sanity_check(executor); #endif #ifdef _Py_JIT executor->jit_code = NULL; + executor->jit_side_entry = NULL; executor->jit_size = 0; - if (_PyJIT_Compile(executor, executor->trace, length+1)) { + // This is initialized to true so we can prevent the executor + // from being immediately detected as cold and invalidated. + executor->vm_data.warm = true; + if (_PyJIT_Compile(executor, executor->trace, length)) { Py_DECREF(executor); return NULL; } @@ -966,203 +1162,85 @@ make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *depende return executor; } -static int -init_cold_exit_executor(_PyExecutorObject *executor, int oparg) +#ifdef Py_STATS +/* Returns the effective trace length. + * Ignores NOPs and trailing exit and error handling.*/ +int effective_trace_length(_PyUOpInstruction *buffer, int length) { - _Py_SetImmortal(executor); - Py_SET_TYPE(executor, &_PyUOpExecutor_Type); - executor->trace = (_PyUOpInstruction *)executor->exits; - executor->code_size = 1; - executor->exit_count = 0; - _PyUOpInstruction *inst = (_PyUOpInstruction *)&executor->trace[0]; - inst->opcode = _COLD_EXIT; - inst->oparg = oparg; - executor->vm_data.valid = true; - for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { - assert(executor->vm_data.bloom.bits[i] == 0); - } -#ifdef _Py_JIT - executor->jit_code = NULL; - executor->jit_size = 0; - if (_PyJIT_Compile(executor, executor->trace, 1)) { - return -1; + int nop_count = 0; + for (int i = 0; i < length; i++) { + int opcode = buffer[i].opcode; + if (opcode == _NOP) { + nop_count++; + } + if (is_terminator(&buffer[i])) { + return i+1-nop_count; + } } -#endif - return 0; + Py_FatalError("No terminating instruction"); + Py_UNREACHABLE(); } +#endif static int uop_optimize( - _PyOptimizerObject *self, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _PyExecutorObject **exec_ptr, - int curr_stackentries) + int curr_stackentries, + bool progress_needed) { _PyBloomFilter dependencies; _Py_BloomFilter_Init(&dependencies); _PyUOpInstruction buffer[UOP_MAX_TRACE_LENGTH]; - int err = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies); - if (err <= 0) { + OPT_STAT_INC(attempts); + int length = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies, progress_needed); + if (length <= 0) { // Error or nothing translated - return err; + return length; } + assert(length < UOP_MAX_TRACE_LENGTH); OPT_STAT_INC(traces_created); char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); if (env_var == NULL || *env_var == '\0' || *env_var > '0') { - err = _Py_uop_analyze_and_optimize(frame, buffer, - UOP_MAX_TRACE_LENGTH, + length = _Py_uop_analyze_and_optimize(frame, buffer, + length, curr_stackentries, &dependencies); - if (err <= 0) { - return err; + if (length <= 0) { + return length; } } - assert(err == 1); + assert(length < UOP_MAX_TRACE_LENGTH); + assert(length >= 1); /* Fix up */ - for (int pc = 0; pc < UOP_MAX_TRACE_LENGTH; pc++) { + for (int pc = 0; pc < length; pc++) { int opcode = buffer[pc].opcode; int oparg = buffer[pc].oparg; if (_PyUop_Flags[opcode] & HAS_OPARG_AND_1_FLAG) { buffer[pc].opcode = opcode + 1 + (oparg & 1); + assert(strncmp(_PyOpcode_uop_name[buffer[pc].opcode], _PyOpcode_uop_name[opcode], strlen(_PyOpcode_uop_name[opcode])) == 0); } else if (oparg < _PyUop_Replication[opcode]) { buffer[pc].opcode = opcode + oparg + 1; + assert(strncmp(_PyOpcode_uop_name[buffer[pc].opcode], _PyOpcode_uop_name[opcode], strlen(_PyOpcode_uop_name[opcode])) == 0); } - else if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE) { + else if (is_terminator(&buffer[pc])) { break; } assert(_PyOpcode_uop_name[buffer[pc].opcode]); } - _PyExecutorObject *executor = make_executor_from_uops(buffer, &dependencies); - if (executor == NULL) { - return -1; - } - OPT_HIST(Py_SIZE(executor), optimized_trace_length_hist); - *exec_ptr = executor; - return 1; -} - -static void -uop_opt_dealloc(PyObject *self) { - PyObject_Free(self); -} - -PyTypeObject _PyUOpOptimizer_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - .tp_name = "uop_optimizer", - .tp_basicsize = sizeof(_PyOptimizerObject), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, - .tp_dealloc = uop_opt_dealloc, -}; - -PyObject * -PyUnstable_Optimizer_NewUOpOptimizer(void) -{ - _PyOptimizerObject *opt = PyObject_New(_PyOptimizerObject, &_PyUOpOptimizer_Type); - if (opt == NULL) { - return NULL; - } - opt->optimize = uop_optimize; - opt->resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD; - // Need a few iterations to settle specializations, - // and to ammortize the cost of optimization. - opt->side_threshold = 16; - opt->backedge_threshold = 16; - return (PyObject *)opt; -} - -static void -counter_dealloc(_PyExecutorObject *self) { - /* The optimizer is the operand of the second uop. */ - PyObject *opt = (PyObject *)self->trace[1].operand; - Py_DECREF(opt); - uop_dealloc(self); -} - -PyTypeObject _PyCounterExecutor_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - .tp_name = "counting_executor", - .tp_basicsize = offsetof(_PyExecutorObject, exits), - .tp_itemsize = 1, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_HAVE_GC, - .tp_dealloc = (destructor)counter_dealloc, - .tp_methods = executor_methods, - .tp_traverse = executor_traverse, - .tp_clear = executor_clear, -}; - -static int -counter_optimize( - _PyOptimizerObject* self, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, - _PyExecutorObject **exec_ptr, - int Py_UNUSED(curr_stackentries) -) -{ - PyCodeObject *code = (PyCodeObject *)frame->f_executable; - int oparg = instr->op.arg; - while (instr->op.code == EXTENDED_ARG) { - instr++; - oparg = (oparg << 8) | instr->op.arg; - } - if (instr->op.code != JUMP_BACKWARD) { - /* Counter optimizer can only handle backward edges */ - return 0; - } - _Py_CODEUNIT *target = instr + 1 + _PyOpcode_Caches[JUMP_BACKWARD] - oparg; - _PyUOpInstruction buffer[3] = { - { .opcode = _LOAD_CONST_INLINE_BORROW, .operand = (uintptr_t)self }, - { .opcode = _INTERNAL_INCREMENT_OPT_COUNTER }, - { .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)) } - }; - _PyExecutorObject *executor = make_executor_from_uops(buffer, &EMPTY_FILTER); + OPT_HIST(effective_trace_length(buffer, length), optimized_trace_length_hist); + length = prepare_for_execution(buffer, length); + assert(length <= UOP_MAX_TRACE_LENGTH); + _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &dependencies); if (executor == NULL) { return -1; } - Py_INCREF(self); - Py_SET_TYPE(executor, &_PyCounterExecutor_Type); + assert(length <= UOP_MAX_TRACE_LENGTH); *exec_ptr = executor; return 1; } -static PyObject * -counter_get_counter(PyObject *self, PyObject *args) -{ - return PyLong_FromLongLong(((_PyCounterOptimizerObject *)self)->count); -} - -static PyMethodDef counter_optimizer_methods[] = { - { "get_count", counter_get_counter, METH_NOARGS, NULL }, - { NULL, NULL }, -}; - -PyTypeObject _PyCounterOptimizer_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - .tp_name = "Counter optimizer", - .tp_basicsize = sizeof(_PyCounterOptimizerObject), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, - .tp_methods = counter_optimizer_methods, - .tp_dealloc = (destructor)PyObject_Del, -}; - -PyObject * -PyUnstable_Optimizer_NewCounter(void) -{ - _PyCounterOptimizerObject *opt = (_PyCounterOptimizerObject *)_PyObject_New(&_PyCounterOptimizer_Type); - if (opt == NULL) { - return NULL; - } - opt->base.optimize = counter_optimize; - opt->base.resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD; - opt->base.side_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD; - opt->base.backedge_threshold = 0; - opt->count = 0; - return (PyObject *)opt; -} - /***************************************** * Executor management @@ -1170,7 +1248,7 @@ PyUnstable_Optimizer_NewCounter(void) /* We use a bloomfilter with k = 6, m = 256 * The choice of k and the following constants - * could do with a more rigourous analysis, + * could do with a more rigorous analysis, * but here is a simple analysis: * * We want to keep the false positive rate low. @@ -1211,7 +1289,7 @@ address_to_hash(void *ptr) { uintptr_t addr = (uintptr_t)ptr; for (int i = 0; i < SIZEOF_VOID_P; i++) { uhash ^= addr & 255; - uhash *= (uint64_t)_PyHASH_MULTIPLIER; + uhash *= (uint64_t)PyHASH_MULTIPLIER; addr >>= 8; } return uhash; @@ -1220,7 +1298,7 @@ address_to_hash(void *ptr) { void _Py_BloomFilter_Init(_PyBloomFilter *bloom) { - for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { + for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) { bloom->bits[i] = 0; } } @@ -1245,7 +1323,7 @@ _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *ptr) static bool bloom_filter_may_contain(_PyBloomFilter *bloom, _PyBloomFilter *hashes) { - for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { + for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) { if ((bloom->bits[i] & hashes->bits[i]) != hashes->bits[i]) { return false; } @@ -1265,15 +1343,13 @@ link_executor(_PyExecutorObject *executor) links->next = NULL; } else { - _PyExecutorObject *next = head->vm_data.links.next; - links->previous = head; - links->next = next; - if (next != NULL) { - next->vm_data.links.previous = executor; - } - head->vm_data.links.next = executor; + assert(head->vm_data.links.previous == NULL); + links->previous = NULL; + links->next = head; + head->vm_data.links.previous = executor; + interp->executor_list_head = executor; } - executor->vm_data.valid = true; + executor->vm_data.linked = true; /* executor_list_head must be first in list */ assert(interp->executor_list_head->vm_data.links.previous == NULL); } @@ -1281,7 +1357,11 @@ link_executor(_PyExecutorObject *executor) static void unlink_executor(_PyExecutorObject *executor) { + if (!executor->vm_data.linked) { + return; + } _PyExecutorLinkListNode *links = &executor->vm_data.links; + assert(executor->vm_data.valid); _PyExecutorObject *next = links->next; _PyExecutorObject *prev = links->previous; if (next != NULL) { @@ -1296,7 +1376,7 @@ unlink_executor(_PyExecutorObject *executor) assert(interp->executor_list_head == executor); interp->executor_list_head = next; } - executor->vm_data.valid = false; + executor->vm_data.linked = false; } /* This must be called by optimizers before using the executor */ @@ -1304,29 +1384,21 @@ void _Py_ExecutorInit(_PyExecutorObject *executor, const _PyBloomFilter *dependency_set) { executor->vm_data.valid = true; - for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { + for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) { executor->vm_data.bloom.bits[i] = dependency_set->bits[i]; } link_executor(executor); } -/* This must be called by executors during dealloc */ +/* Detaches the executor from the code object (if any) that + * holds a reference to it */ void -_Py_ExecutorClear(_PyExecutorObject *executor) +_Py_ExecutorDetach(_PyExecutorObject *executor) { - if (!executor->vm_data.valid) { - return; - } - unlink_executor(executor); PyCodeObject *code = executor->vm_data.code; if (code == NULL) { return; } - for (uint32_t i = 0; i < executor->exit_count; i++) { - Py_DECREF(executor->exits[i].executor); - executor->exits[i].executor = &COLD_EXITS[i]; - executor->exits[i].temperature = INT16_MIN; - } _Py_CODEUNIT *instruction = &_PyCode_CODE(code)[executor->vm_data.index]; assert(instruction->op.code == ENTER_EXECUTOR); int index = instruction->op.arg; @@ -1334,7 +1406,31 @@ _Py_ExecutorClear(_PyExecutorObject *executor) instruction->op.code = executor->vm_data.opcode; instruction->op.arg = executor->vm_data.oparg; executor->vm_data.code = NULL; - Py_CLEAR(code->co_executors->executors[index]); + code->co_executors->executors[index] = NULL; + Py_DECREF(executor); +} + +static int +executor_clear(_PyExecutorObject *executor) +{ + if (!executor->vm_data.valid) { + return 0; + } + assert(executor->vm_data.valid == 1); + unlink_executor(executor); + executor->vm_data.valid = 0; + /* It is possible for an executor to form a reference + * cycle with itself, so decref'ing a side exit could + * free the executor unless we hold a strong reference to it + */ + Py_INCREF(executor); + for (uint32_t i = 0; i < executor->exit_count; i++) { + executor->exits[i].temperature = initial_unreachable_backoff_counter(); + Py_CLEAR(executor->exits[i].executor); + } + _Py_ExecutorDetach(executor); + Py_DECREF(executor); + return 0; } void @@ -1355,17 +1451,36 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is _Py_BloomFilter_Add(&obj_filter, obj); /* Walk the list of executors */ /* TO DO -- Use a tree to avoid traversing as many objects */ + PyObject *invalidate = PyList_New(0); + if (invalidate == NULL) { + goto error; + } + /* Clearing an executor can deallocate others, so we need to make a list of + * executors to invalidate first */ for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) { assert(exec->vm_data.valid); _PyExecutorObject *next = exec->vm_data.links.next; - if (bloom_filter_may_contain(&exec->vm_data.bloom, &obj_filter)) { - _Py_ExecutorClear(exec); - if (is_invalidation) { - OPT_STAT_INC(executors_invalidated); - } + if (bloom_filter_may_contain(&exec->vm_data.bloom, &obj_filter) && + PyList_Append(invalidate, (PyObject *)exec)) + { + goto error; } exec = next; } + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(invalidate); i++) { + _PyExecutorObject *exec = (_PyExecutorObject *)PyList_GET_ITEM(invalidate, i); + executor_clear(exec); + if (is_invalidation) { + OPT_STAT_INC(executors_invalidated); + } + } + Py_DECREF(invalidate); + return; +error: + PyErr_Clear(); + Py_XDECREF(invalidate); + // If we're truly out of memory, wiping out everything is a fine fallback: + _Py_Executors_InvalidateAll(interp, is_invalidation); } /* Invalidate all executors */ @@ -1374,15 +1489,183 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation) { while (interp->executor_list_head) { _PyExecutorObject *executor = interp->executor_list_head; + assert(executor->vm_data.valid == 1 && executor->vm_data.linked == 1); if (executor->vm_data.code) { // Clear the entire code object so its co_executors array be freed: _PyCode_Clear_Executors(executor->vm_data.code); } else { - _Py_ExecutorClear(executor); + executor_clear(executor); } if (is_invalidation) { OPT_STAT_INC(executors_invalidated); } } } + +void +_Py_Executors_InvalidateCold(PyInterpreterState *interp) +{ + /* Walk the list of executors */ + /* TO DO -- Use a tree to avoid traversing as many objects */ + PyObject *invalidate = PyList_New(0); + if (invalidate == NULL) { + goto error; + } + + /* Clearing an executor can deallocate others, so we need to make a list of + * executors to invalidate first */ + for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) { + assert(exec->vm_data.valid); + _PyExecutorObject *next = exec->vm_data.links.next; + + if (!exec->vm_data.warm && PyList_Append(invalidate, (PyObject *)exec) < 0) { + goto error; + } + else { + exec->vm_data.warm = false; + } + + exec = next; + } + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(invalidate); i++) { + _PyExecutorObject *exec = (_PyExecutorObject *)PyList_GET_ITEM(invalidate, i); + executor_clear(exec); + } + Py_DECREF(invalidate); + return; +error: + PyErr_Clear(); + Py_XDECREF(invalidate); + // If we're truly out of memory, wiping out everything is a fine fallback + _Py_Executors_InvalidateAll(interp, 0); +} + +static void +write_str(PyObject *str, FILE *out) +{ + // Encode the Unicode object to the specified encoding + PyObject *encoded_obj = PyUnicode_AsEncodedString(str, "utf8", "strict"); + if (encoded_obj == NULL) { + PyErr_Clear(); + return; + } + const char *encoded_str = PyBytes_AsString(encoded_obj); + Py_ssize_t encoded_size = PyBytes_Size(encoded_obj); + fwrite(encoded_str, 1, encoded_size, out); + Py_DECREF(encoded_obj); +} + +static int +find_line_number(PyCodeObject *code, _PyExecutorObject *executor) +{ + int code_len = (int)Py_SIZE(code); + for (int i = 0; i < code_len; i++) { + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + int opcode = instr->op.code; + if (opcode == ENTER_EXECUTOR) { + _PyExecutorObject *exec = code->co_executors->executors[instr->op.arg]; + if (exec == executor) { + return PyCode_Addr2Line(code, i*2); + } + } + i += _PyOpcode_Caches[_Py_GetBaseCodeUnit(code, i).op.code]; + } + return -1; +} + +/* Writes the node and outgoing edges for a single tracelet in graphviz format. + * Each tracelet is presented as a table of the uops it contains. + * If Py_STATS is enabled, execution counts are included. + * + * https://graphviz.readthedocs.io/en/stable/manual.html + * https://graphviz.org/gallery/ + */ +static void +executor_to_gv(_PyExecutorObject *executor, FILE *out) +{ + PyCodeObject *code = executor->vm_data.code; + fprintf(out, "executor_%p [\n", executor); + fprintf(out, " shape = none\n"); + + /* Write the HTML table for the uops */ + fprintf(out, " label = <\n"); + fprintf(out, " \n"); + if (code == NULL) { + fprintf(out, " \n"); + } + else { + fprintf(out, " \n", line); + } + for (uint32_t i = 0; i < executor->code_size; i++) { + /* Write row for uop. + * The `port` is a marker so that outgoing edges can + * be placed correctly. If a row is marked `port=17`, + * then the outgoing edge is `{EXEC_NAME}:17 -> {TARGET}` + * https://graphviz.readthedocs.io/en/stable/manual.html#node-ports-compass + */ + _PyUOpInstruction const *inst = &executor->trace[i]; + const char *opname = _PyOpcode_uop_name[inst->opcode]; +#ifdef Py_STATS + fprintf(out, " \n", i, opname, inst->execution_count); +#else + fprintf(out, " \n", i, opname); +#endif + if (inst->opcode == _EXIT_TRACE || inst->opcode == _JUMP_TO_TOP) { + break; + } + } + fprintf(out, "
    Executor
    No code object
    "); + write_str(code->co_qualname, out); + int line = find_line_number(code, executor); + fprintf(out, ": %d
    %s -- %" PRIu64 "
    %s
    >\n"); + fprintf(out, "]\n\n"); + + /* Write all the outgoing edges */ + for (uint32_t i = 0; i < executor->code_size; i++) { + _PyUOpInstruction const *inst = &executor->trace[i]; + uint16_t flags = _PyUop_Flags[inst->opcode]; + _PyExitData *exit = NULL; + if (inst->opcode == _EXIT_TRACE) { + exit = (_PyExitData *)inst->operand0; + } + else if (flags & HAS_EXIT_FLAG) { + assert(inst->format == UOP_FORMAT_JUMP); + _PyUOpInstruction const *exit_inst = &executor->trace[inst->jump_target]; + assert(exit_inst->opcode == _EXIT_TRACE); + exit = (_PyExitData *)exit_inst->operand0; + } + if (exit != NULL && exit->executor != NULL) { + fprintf(out, "executor_%p:i%d -> executor_%p:start\n", executor, i, exit->executor); + } + if (inst->opcode == _EXIT_TRACE || inst->opcode == _JUMP_TO_TOP) { + break; + } + } +} + +/* Write the graph of all the live tracelets in graphviz format. */ +int +_PyDumpExecutors(FILE *out) +{ + fprintf(out, "digraph ideal {\n\n"); + fprintf(out, " rankdir = \"LR\"\n\n"); + PyInterpreterState *interp = PyInterpreterState_Get(); + for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) { + executor_to_gv(exec, out); + exec = exec->vm_data.links.next; + } + fprintf(out, "}\n\n"); + return 0; +} + +#else + +int +_PyDumpExecutors(FILE *out) +{ + PyErr_SetString(PyExc_NotImplementedError, "No JIT available"); + return -1; +} + +#endif /* _Py_TIER2 */ diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index a326e2249bb4de..5dd7725f398cd1 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -1,3 +1,5 @@ +#ifdef _Py_TIER2 + /* * This file contains the support code for CPython's uops optimizer. * It also performs some simple optimizations. @@ -19,7 +21,6 @@ #include "pycore_uop_metadata.h" #include "pycore_dict.h" #include "pycore_long.h" -#include "cpython/optimizer.h" #include "pycore_optimizer.h" #include "pycore_object.h" #include "pycore_dict.h" @@ -35,6 +36,7 @@ #ifdef Py_DEBUG extern const char *_PyUOpName(int index); + extern void _PyUOpPrint(const _PyUOpInstruction *uop); static const char *const DEBUG_ENV = "PYTHON_OPT_DEBUG"; static inline int get_lltrace(void) { char *uop_debug = Py_GETENV(DEBUG_ENV); @@ -50,32 +52,25 @@ #define DPRINTF(level, ...) #endif - - -static inline bool -op_is_end(uint32_t opcode) -{ - return opcode == _EXIT_TRACE || opcode == _JUMP_TO_TOP; -} - static int get_mutations(PyObject* dict) { assert(PyDict_CheckExact(dict)); PyDictObject *d = (PyDictObject *)dict; - return (d->ma_version_tag >> DICT_MAX_WATCHERS) & ((1 << DICT_WATCHED_MUTATION_BITS)-1); + return (d->_ma_watcher_tag >> DICT_MAX_WATCHERS) & ((1 << DICT_WATCHED_MUTATION_BITS)-1); } static void increment_mutations(PyObject* dict) { assert(PyDict_CheckExact(dict)); PyDictObject *d = (PyDictObject *)dict; - d->ma_version_tag += (1 << DICT_MAX_WATCHERS); + d->_ma_watcher_tag += (1 << DICT_MAX_WATCHERS); } /* The first two dict watcher IDs are reserved for CPython, * so we don't need to check that they haven't been used */ #define BUILTINS_WATCHER_ID 0 #define GLOBALS_WATCHER_ID 1 +#define TYPE_WATCHER_ID 0 static int globals_watcher_callback(PyDict_WatchEvent event, PyObject* dict, @@ -89,29 +84,41 @@ globals_watcher_callback(PyDict_WatchEvent event, PyObject* dict, return 0; } +static int +type_watcher_callback(PyTypeObject* type) +{ + _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), type, 1); + PyType_Unwatch(TYPE_WATCHER_ID, (PyObject *)type); + return 0; +} + static PyObject * convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj) { - assert(inst->opcode == _LOAD_GLOBAL_MODULE || inst->opcode == _LOAD_GLOBAL_BUILTINS || inst->opcode == _LOAD_ATTR_MODULE); + assert(inst->opcode == _LOAD_GLOBAL_MODULE || inst->opcode == _LOAD_GLOBAL_BUILTINS || inst->opcode == _LOAD_ATTR_MODULE_FROM_KEYS); assert(PyDict_CheckExact(obj)); PyDictObject *dict = (PyDictObject *)obj; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - assert(inst->operand <= UINT16_MAX); - if ((int)inst->operand >= dict->ma_keys->dk_nentries) { + assert(inst->operand0 <= UINT16_MAX); + if ((int)inst->operand0 >= dict->ma_keys->dk_nentries) { return NULL; } - PyObject *res = entries[inst->operand].me_value; + PyObject *res = entries[inst->operand0].me_value; if (res == NULL) { return NULL; } if (_Py_IsImmortal(res)) { - inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_BORROW_WITH_NULL : _LOAD_CONST_INLINE_BORROW; + inst->opcode = _LOAD_CONST_INLINE_BORROW; } else { - inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_WITH_NULL : _LOAD_CONST_INLINE; + inst->opcode = _LOAD_CONST_INLINE; } - inst->operand = (uint64_t)res; + if (inst->oparg & 1) { + assert(inst[1].opcode == _PUSH_NULL_CONDITIONAL); + assert(inst[1].oparg & 1); + } + inst->operand0 = (uint64_t)res; return res; } @@ -122,12 +129,32 @@ incorrect_keys(_PyUOpInstruction *inst, PyObject *obj) return 1; } PyDictObject *dict = (PyDictObject *)obj; - if (dict->ma_keys->dk_version != inst->operand) { + if (dict->ma_keys->dk_version != inst->operand0) { return 1; } return 0; } +static int +check_next_uop(_PyUOpInstruction *buffer, int size, int pc, uint16_t expected) +{ + if (pc + 1 >= size) { + DPRINTF(1, "Cannot rewrite %s at pc %d: buffer too small\n", + _PyOpcode_uop_name[buffer[pc].opcode], pc); + return 0; + } + uint16_t next_opcode = buffer[pc + 1].opcode; + if (next_opcode != expected) { + DPRINTF(1, + "Cannot rewrite %s at pc %d: unexpected next opcode %s, " + "expected %s\n", + _PyOpcode_uop_name[buffer[pc].opcode], pc, + _PyOpcode_uop_name[next_opcode], _PyOpcode_uop_name[expected]); + return 0; + } + return 1; +} + /* Returns 1 if successfully optimized * 0 if the trace is not suitable for optimization (yet) * -1 if there was an error. */ @@ -138,12 +165,15 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, PyInterpreterState *interp = _PyInterpreterState_GET(); PyObject *builtins = frame->f_builtins; if (builtins != interp->builtins) { + OPT_STAT_INC(remove_globals_builtins_changed); return 1; } PyObject *globals = frame->f_globals; - assert(PyFunction_Check(((PyFunctionObject *)frame->f_funcobj))); - assert(((PyFunctionObject *)frame->f_funcobj)->func_builtins == builtins); - assert(((PyFunctionObject *)frame->f_funcobj)->func_globals == globals); + PyFunctionObject *function = _PyFrame_GetFunction(frame); + assert(PyFunction_Check(function)); + assert(function->func_builtins == builtins); + assert(function->func_globals == globals); + uint32_t function_version = _PyFunction_GetVersionForCurrentState(function); /* In order to treat globals as constants, we need to * know that the globals dict is the one we expected, and * that it hasn't changed @@ -154,101 +184,147 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, /* These values represent stacks of booleans (one bool per bit). * Pushing a frame shifts left, popping a frame shifts right. */ - uint32_t builtins_checked = 0; + uint32_t function_checked = 0; uint32_t builtins_watched = 0; - uint32_t globals_checked = 0; uint32_t globals_watched = 0; + uint32_t prechecked_function_version = 0; if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) { interp->dict_state.watchers[GLOBALS_WATCHER_ID] = globals_watcher_callback; } + if (interp->type_watchers[TYPE_WATCHER_ID] == NULL) { + interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback; + } for (int pc = 0; pc < buffer_size; pc++) { _PyUOpInstruction *inst = &buffer[pc]; int opcode = inst->opcode; switch(opcode) { - case _GUARD_BUILTINS_VERSION: + case _GUARD_BUILTINS_VERSION_PUSH_KEYS: if (incorrect_keys(inst, builtins)) { + OPT_STAT_INC(remove_globals_incorrect_keys); return 0; } if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { continue; } + if (!check_next_uop(buffer, buffer_size, pc, + _LOAD_GLOBAL_BUILTINS_FROM_KEYS)) { + continue; + } if ((builtins_watched & 1) == 0) { PyDict_Watch(BUILTINS_WATCHER_ID, builtins); builtins_watched |= 1; } - if (builtins_checked & 1) { + if (function_checked & 1) { buffer[pc].opcode = NOP; } else { - buffer[pc].opcode = _CHECK_BUILTINS; - buffer[pc].operand = (uintptr_t)builtins; - builtins_checked |= 1; + buffer[pc].opcode = _CHECK_FUNCTION; + buffer[pc].operand0 = function_version; + function_checked |= 1; } + // We're no longer pushing the builtins keys; rewrite the + // instruction that consumed the keys to load them from the + // frame. + buffer[pc + 1].opcode = _LOAD_GLOBAL_BUILTINS; break; case _GUARD_GLOBALS_VERSION: + case _GUARD_GLOBALS_VERSION_PUSH_KEYS: if (incorrect_keys(inst, globals)) { + OPT_STAT_INC(remove_globals_incorrect_keys); return 0; } uint64_t watched_mutations = get_mutations(globals); if (watched_mutations >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { continue; } + if (opcode == _GUARD_GLOBALS_VERSION_PUSH_KEYS && + !check_next_uop(buffer, buffer_size, pc, + _LOAD_GLOBAL_MODULE_FROM_KEYS)) { + continue; + } if ((globals_watched & 1) == 0) { PyDict_Watch(GLOBALS_WATCHER_ID, globals); _Py_BloomFilter_Add(dependencies, globals); globals_watched |= 1; } - if (globals_checked & 1) { + if (function_checked & 1) { buffer[pc].opcode = NOP; } else { - buffer[pc].opcode = _CHECK_GLOBALS; - buffer[pc].operand = (uintptr_t)globals; - globals_checked |= 1; + buffer[pc].opcode = _CHECK_FUNCTION; + buffer[pc].operand0 = function_version; + function_checked |= 1; + } + if (opcode == _GUARD_GLOBALS_VERSION_PUSH_KEYS) { + // We're no longer pushing the globals keys; rewrite the + // instruction that consumed the keys to load them from the + // frame. + buffer[pc + 1].opcode = _LOAD_GLOBAL_MODULE; } break; case _LOAD_GLOBAL_BUILTINS: - if (globals_checked & builtins_checked & globals_watched & builtins_watched & 1) { + if (function_checked & globals_watched & builtins_watched & 1) { convert_global_to_const(inst, builtins); } break; case _LOAD_GLOBAL_MODULE: - if (globals_checked & globals_watched & 1) { + if (function_checked & globals_watched & 1) { convert_global_to_const(inst, globals); } break; case _PUSH_FRAME: { - globals_checked <<= 1; - globals_watched <<= 1; - builtins_checked <<= 1; builtins_watched <<= 1; - PyFunctionObject *func = (PyFunctionObject *)buffer[pc].operand; + globals_watched <<= 1; + function_checked <<= 1; + uint64_t operand = buffer[pc].operand0; + if (operand == 0 || (operand & 1)) { + // It's either a code object or NULL, so bail + return 1; + } + PyFunctionObject *func = (PyFunctionObject *)operand; if (func == NULL) { return 1; } assert(PyFunction_Check(func)); + function_version = func->func_version; + if (prechecked_function_version == function_version) { + function_checked |= 1; + } + prechecked_function_version = 0; globals = func->func_globals; builtins = func->func_builtins; if (builtins != interp->builtins) { + OPT_STAT_INC(remove_globals_builtins_changed); return 1; } break; } - case _POP_FRAME: + case _RETURN_VALUE: { - globals_checked >>= 1; - globals_watched >>= 1; - builtins_checked >>= 1; builtins_watched >>= 1; - PyFunctionObject *func = (PyFunctionObject *)buffer[pc].operand; + globals_watched >>= 1; + function_checked >>= 1; + uint64_t operand = buffer[pc].operand0; + if (operand == 0 || (operand & 1)) { + // It's either a code object or NULL, so bail + return 1; + } + PyFunctionObject *func = (PyFunctionObject *)operand; + if (func == NULL) { + return 1; + } assert(PyFunction_Check(func)); + function_version = func->func_version; globals = func->func_globals; builtins = func->func_builtins; break; } + case _CHECK_FUNCTION_EXACT_ARGS: + prechecked_function_version = (uint32_t)buffer[pc].operand0; + break; default: - if (op_is_end(opcode)) { + if (is_terminator(inst)) { return 1; } break; @@ -260,27 +336,18 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define STACK_LEVEL() ((int)(stack_pointer - ctx->frame->stack)) +#define STACK_SIZE() ((int)(ctx->frame->stack_len)) + +#define WITHIN_STACK_BOUNDS() \ + (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE()) + #define GETLOCAL(idx) ((ctx->frame->locals[idx])) #define REPLACE_OP(INST, OP, ARG, OPERAND) \ INST->opcode = OP; \ INST->oparg = ARG; \ - INST->operand = OPERAND; - -#define OUT_OF_SPACE_IF_NULL(EXPR) \ - do { \ - if ((EXPR) == NULL) { \ - goto out_of_space; \ - } \ - } while (0); - -#define _LOAD_ATTR_NOT_NULL \ - do { \ - OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_not_null(ctx)); \ - OUT_OF_SPACE_IF_NULL(null = _Py_uop_sym_new_null(ctx)); \ - } while (0); - + INST->operand0 = OPERAND; /* Shortened forms for convenience, used in optimizer_bytecodes.c */ #define sym_is_not_null _Py_uop_sym_is_not_null @@ -292,15 +359,103 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_is_null _Py_uop_sym_is_null #define sym_new_const _Py_uop_sym_new_const #define sym_new_null _Py_uop_sym_new_null +#define sym_has_type _Py_uop_sym_has_type +#define sym_get_type _Py_uop_sym_get_type #define sym_matches_type _Py_uop_sym_matches_type -#define sym_set_null _Py_uop_sym_set_null -#define sym_set_non_null _Py_uop_sym_set_non_null -#define sym_set_type _Py_uop_sym_set_type -#define sym_set_const _Py_uop_sym_set_const +#define sym_matches_type_version _Py_uop_sym_matches_type_version +#define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) +#define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) +#define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) +#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) +#define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) #define sym_is_bottom _Py_uop_sym_is_bottom +#define sym_truthiness _Py_uop_sym_truthiness #define frame_new _Py_uop_frame_new #define frame_pop _Py_uop_frame_pop +#define sym_new_tuple _Py_uop_sym_new_tuple +#define sym_tuple_getitem _Py_uop_sym_tuple_getitem +#define sym_tuple_length _Py_uop_sym_tuple_length +#define sym_is_immortal _Py_uop_sym_is_immortal +static int +optimize_to_bool( + _PyUOpInstruction *this_instr, + JitOptContext *ctx, + JitOptSymbol *value, + JitOptSymbol **result_ptr) +{ + if (sym_matches_type(value, &PyBool_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + *result_ptr = value; + return 1; + } + int truthiness = sym_truthiness(value); + if (truthiness >= 0) { + PyObject *load = truthiness ? Py_True : Py_False; + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); + *result_ptr = sym_new_const(ctx, load); + return 1; + } + return 0; +} + +static void +eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit) +{ + REPLACE_OP(this_instr, _POP_TOP, 0, 0); + if (exit) { + REPLACE_OP((this_instr+1), _EXIT_TRACE, 0, 0); + this_instr[1].target = this_instr->target; + } +} + +/* _PUSH_FRAME/_RETURN_VALUE's operand can be 0, a PyFunctionObject *, or a + * PyCodeObject *. Retrieve the code object if possible. + */ +static PyCodeObject * +get_code(_PyUOpInstruction *op) +{ + assert(op->opcode == _PUSH_FRAME || op->opcode == _RETURN_VALUE || op->opcode == _RETURN_GENERATOR); + PyCodeObject *co = NULL; + uint64_t operand = op->operand0; + if (operand == 0) { + return NULL; + } + if (operand & 1) { + co = (PyCodeObject *)(operand & ~1); + } + else { + PyFunctionObject *func = (PyFunctionObject *)operand; + assert(PyFunction_Check(func)); + co = (PyCodeObject *)func->func_code; + } + assert(PyCode_Check(co)); + return co; +} + +static PyCodeObject * +get_code_with_logging(_PyUOpInstruction *op) +{ + PyCodeObject *co = NULL; + uint64_t push_operand = op->operand0; + if (push_operand & 1) { + co = (PyCodeObject *)(push_operand & ~1); + DPRINTF(3, "code=%p ", co); + assert(PyCode_Check(co)); + } + else { + PyFunctionObject *func = (PyFunctionObject *)push_operand; + DPRINTF(3, "func=%p ", func); + if (func == NULL) { + DPRINTF(3, "\n"); + DPRINTF(1, "Missing function\n"); + return NULL; + } + co = (PyCodeObject *)func->func_code; + DPRINTF(3, "code=%p ", co); + } + return co; +} /* 1 for success, 0 for not ready, cannot error at the moment. */ static int @@ -313,36 +468,48 @@ optimize_uops( ) { - _Py_UOpsContext context; - _Py_UOpsContext *ctx = &context; + JitOptContext context; + JitOptContext *ctx = &context; + uint32_t opcode = UINT16_MAX; + int curr_space = 0; + int max_space = 0; + _PyUOpInstruction *first_valid_check_stack = NULL; + _PyUOpInstruction *corresponding_check_stack = NULL; - if (_Py_uop_abstractcontext_init(ctx) < 0) { - goto out_of_space; - } - _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, co, ctx->n_consumed, 0, curr_stacklen); + _Py_uop_abstractcontext_init(ctx); + _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, co, curr_stacklen, NULL, 0); if (frame == NULL) { return -1; } ctx->curr_frame_depth++; ctx->frame = frame; + ctx->done = false; + ctx->out_of_space = false; + ctx->contradiction = false; - for (_PyUOpInstruction *this_instr = trace; - this_instr < trace + trace_len && !op_is_end(this_instr->opcode); - this_instr++) { + _PyUOpInstruction *this_instr = NULL; + for (int i = 0; !ctx->done; i++) { + assert(i < trace_len); + this_instr = &trace[i]; int oparg = this_instr->oparg; - uint32_t opcode = this_instr->opcode; + opcode = this_instr->opcode; + JitOptSymbol **stack_pointer = ctx->frame->stack_pointer; - _Py_UopsSymbol **stack_pointer = ctx->frame->stack_pointer; +#ifdef Py_DEBUG + if (get_lltrace() >= 3) { + printf("%4d abs: ", (int)(this_instr - trace)); + _PyUOpPrint(this_instr); + printf(" "); + } +#endif - DPRINTF(3, "Abstract interpreting %s:%d ", - _PyUOpName(opcode), - oparg); switch (opcode) { + #include "optimizer_cases.c.h" default: - DPRINTF(1, "Unknown opcode in abstract interpreter\n"); + DPRINTF(1, "\nUnknown opcode in abstract interpreter\n"); Py_UNREACHABLE(); } assert(ctx->frame != NULL); @@ -350,32 +517,48 @@ optimize_uops( ctx->frame->stack_pointer = stack_pointer; assert(STACK_LEVEL() >= 0); } + if (ctx->out_of_space) { + DPRINTF(3, "\n"); + DPRINTF(1, "Out of space in abstract interpreter\n"); + } + if (ctx->contradiction) { + // Attempted to push a "bottom" (contradiction) symbol onto the stack. + // This means that the abstract interpreter has hit unreachable code. + // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but hitting + // bottom indicates type instability, so we are probably better off + // retrying later. + DPRINTF(3, "\n"); + DPRINTF(1, "Hit bottom in abstract interpreter\n"); + _Py_uop_abstractcontext_fini(ctx); + return 0; + } + /* Either reached the end or cannot optimize further, but there + * would be no benefit in retrying later */ _Py_uop_abstractcontext_fini(ctx); - return 1; - -out_of_space: - DPRINTF(1, "Out of space in abstract interpreter\n"); - _Py_uop_abstractcontext_fini(ctx); - return 0; + if (first_valid_check_stack != NULL) { + assert(first_valid_check_stack->opcode == _CHECK_STACK_SPACE); + assert(max_space > 0); + assert(max_space <= INT_MAX); + assert(max_space <= INT32_MAX); + first_valid_check_stack->opcode = _CHECK_STACK_SPACE_OPERAND; + first_valid_check_stack->operand0 = max_space; + } + return trace_len; error: + DPRINTF(3, "\n"); DPRINTF(1, "Encountered error in abstract interpreter\n"); + if (opcode <= MAX_UOP_ID) { + OPT_ERROR_IN_OPCODE(opcode); + } _Py_uop_abstractcontext_fini(ctx); - return 0; + return -1; -hit_bottom: - // Attempted to push a "bottom" (contradition) symbol onto the stack. - // This means that the abstract interpreter has hit unreachable code. - // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but it's - // simpler to just admit failure and not create the executor. - DPRINTF(1, "Hit bottom in abstract interpreter\n"); - _Py_uop_abstractcontext_fini(ctx); - return 0; } -static void +static int remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) { /* Remove _SET_IP and _CHECK_VALIDITY where possible. @@ -387,8 +570,11 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) for (int pc = 0; pc < buffer_size; pc++) { int opcode = buffer[pc].opcode; switch (opcode) { + case _START_EXECUTOR: + may_have_escaped = false; + break; case _SET_IP: - buffer[pc].opcode = NOP; + buffer[pc].opcode = _NOP; last_set_ip = pc; break; case _CHECK_VALIDITY: @@ -396,7 +582,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) may_have_escaped = false; } else { - buffer[pc].opcode = NOP; + buffer[pc].opcode = _NOP; } break; case _CHECK_VALIDITY_AND_SET_IP: @@ -405,7 +591,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) buffer[pc].opcode = _CHECK_VALIDITY; } else { - buffer[pc].opcode = NOP; + buffer[pc].opcode = _NOP; } last_set_ip = pc; break; @@ -421,23 +607,26 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) last->opcode == _COPY ) { last->opcode = _NOP; - buffer[pc].opcode = NOP; + buffer[pc].opcode = _NOP; + } + if (last->opcode == _REPLACE_WITH_TRUE) { + last->opcode = _NOP; } break; } case _JUMP_TO_TOP: case _EXIT_TRACE: - return; + case _DYNAMIC_EXIT: + return pc + 1; default: { - bool needs_ip = false; + /* _PUSH_FRAME doesn't escape or error, but it + * does need the IP for the return address */ + bool needs_ip = opcode == _PUSH_FRAME; if (_PyUop_Flags[opcode] & HAS_ESCAPES_FLAG) { needs_ip = true; may_have_escaped = true; } - if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) { - needs_ip = true; - } if (needs_ip && last_set_ip >= 0) { if (buffer[last_set_ip].opcode == _CHECK_VALIDITY) { buffer[last_set_ip].opcode = _CHECK_VALIDITY_AND_SET_IP; @@ -451,90 +640,41 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } } } -} - -static void -peephole_opt(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, int buffer_size) -{ - PyCodeObject *co = (PyCodeObject *)frame->f_executable; - for (int pc = 0; pc < buffer_size; pc++) { - int opcode = buffer[pc].opcode; - switch(opcode) { - case _LOAD_CONST: { - assert(co != NULL); - PyObject *val = PyTuple_GET_ITEM(co->co_consts, buffer[pc].oparg); - buffer[pc].opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; - buffer[pc].operand = (uintptr_t)val; - break; - } - case _CHECK_PEP_523: - { - /* Setting the eval frame function invalidates - * all executors, so no need to check dynamically */ - if (_PyInterpreterState_GET()->eval_frame == NULL) { - buffer[pc].opcode = _NOP; - } - break; - } - case _PUSH_FRAME: - case _POP_FRAME: - { - PyFunctionObject *func = (PyFunctionObject *)buffer[pc].operand; - if (func == NULL) { - co = NULL; - } - else { - assert(PyFunction_Check(func)); - co = (PyCodeObject *)func->func_code; - } - break; - } - case _JUMP_TO_TOP: - case _EXIT_TRACE: - return; - } - } + Py_UNREACHABLE(); } // 0 - failure, no error raised, just fall back to Tier 1 // -1 - failure, and raise error -// 1 - optimizer success +// > 0 - length of optimized trace int _Py_uop_analyze_and_optimize( _PyInterpreterFrame *frame, _PyUOpInstruction *buffer, - int buffer_size, + int length, int curr_stacklen, _PyBloomFilter *dependencies ) { OPT_STAT_INC(optimizer_attempts); - int err = remove_globals(frame, buffer, buffer_size, dependencies); - if (err == 0) { - goto not_ready; - } - if (err < 0) { - goto error; + int err = remove_globals(frame, buffer, length, dependencies); + if (err <= 0) { + return err; } - peephole_opt(frame, buffer, buffer_size); + length = optimize_uops( + _PyFrame_GetCode(frame), buffer, + length, curr_stacklen, dependencies); - err = optimize_uops( - (PyCodeObject *)frame->f_executable, buffer, - buffer_size, curr_stacklen, dependencies); - - if (err == 0) { - goto not_ready; + if (length <= 0) { + return length; } - assert(err == 1); - remove_unneeded_uops(buffer, buffer_size); + length = remove_unneeded_uops(buffer, length); + assert(length > 0); OPT_STAT_INC(optimizer_successes); - return 1; -not_ready: - return 0; -error: - return -1; + return length; } + +#endif /* _Py_TIER2 */ diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 786d884fc5a1a8..fb14c4b7c8645b 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -6,8 +6,6 @@ #define op(name, ...) /* NAME is ignored */ -typedef struct _Py_UopsSymbol _Py_UopsSymbol; -typedef struct _Py_UOpsContext _Py_UOpsContext; typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; /* Shortened forms for convenience */ @@ -21,32 +19,57 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; #define sym_new_const _Py_uop_sym_new_const #define sym_new_null _Py_uop_sym_new_null #define sym_matches_type _Py_uop_sym_matches_type -#define sym_set_null _Py_uop_sym_set_null -#define sym_set_non_null _Py_uop_sym_set_non_null -#define sym_set_type _Py_uop_sym_set_type -#define sym_set_const _Py_uop_sym_set_const +#define sym_matches_type_version _Py_uop_sym_matches_type_version +#define sym_get_type _Py_uop_sym_get_type +#define sym_has_type _Py_uop_sym_has_type +#define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) +#define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) +#define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) +#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) +#define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) #define sym_is_bottom _Py_uop_sym_is_bottom #define frame_new _Py_uop_frame_new #define frame_pop _Py_uop_frame_pop +#define sym_new_tuple _Py_uop_sym_new_tuple +#define sym_tuple_getitem _Py_uop_sym_tuple_getitem +#define sym_tuple_length _Py_uop_sym_tuple_length +#define sym_is_immortal _Py_uop_sym_is_immortal + +extern int +optimize_to_bool( + _PyUOpInstruction *this_instr, + JitOptContext *ctx, + JitOptSymbol *value, + JitOptSymbol **result_ptr); + +extern void +eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit); + +extern PyCodeObject *get_code(_PyUOpInstruction *op); static int dummy_func(void) { - PyCodeObject *code; + PyCodeObject *co; int oparg; - _Py_UopsSymbol *flag; - _Py_UopsSymbol *left; - _Py_UopsSymbol *right; - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; - _Py_UopsSymbol *iter; - _Py_UopsSymbol *top; - _Py_UopsSymbol *bottom; + JitOptSymbol *flag; + JitOptSymbol *left; + JitOptSymbol *right; + JitOptSymbol *value; + JitOptSymbol *res; + JitOptSymbol *iter; + JitOptSymbol *top; + JitOptSymbol *bottom; _Py_UOpsAbstractFrame *frame; - _Py_UOpsContext *ctx; + _Py_UOpsAbstractFrame *new_frame; + JitOptContext *ctx; _PyUOpInstruction *this_instr; _PyBloomFilter *dependencies; int modified; + int curr_space; + int max_space; + _PyUOpInstruction *first_valid_check_stack; + _PyUOpInstruction *corresponding_check_stack; // BEGIN BYTECODES // @@ -54,7 +77,7 @@ dummy_func(void) { value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. if (sym_is_null(value)) { - goto out_of_space; + ctx->done = true; } } @@ -64,8 +87,7 @@ dummy_func(void) { op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); - _Py_UopsSymbol *temp; - OUT_OF_SPACE_IF_NULL(temp = sym_new_null(ctx)); + JitOptSymbol *temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; } @@ -75,35 +97,66 @@ dummy_func(void) { op(_PUSH_NULL, (-- res)) { res = sym_new_null(ctx); - if (res == NULL) { - goto out_of_space; - }; } op(_GUARD_BOTH_INT, (left, right -- left, right)) { - if (sym_matches_type(left, &PyLong_Type) && - sym_matches_type(right, &PyLong_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + if (sym_matches_type(left, &PyLong_Type)) { + if (sym_matches_type(right, &PyLong_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + REPLACE_OP(this_instr, _GUARD_TOS_INT, 0, 0); + } } - if (!sym_set_type(left, &PyLong_Type)) { - goto hit_bottom; + else { + if (sym_matches_type(right, &PyLong_Type)) { + REPLACE_OP(this_instr, _GUARD_NOS_INT, 0, 0); + } } - if (!sym_set_type(right, &PyLong_Type)) { - goto hit_bottom; + sym_set_type(left, &PyLong_Type); + sym_set_type(right, &PyLong_Type); + } + + op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { + assert(type_version); + if (sym_matches_type_version(owner, type_version)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } else { + // add watcher so that whenever the type changes we invalidate this + PyTypeObject *type = _PyType_LookupByVersion(type_version); + // if the type is null, it was not found in the cache (there was a conflict) + // with the key, in which case we can't trust the version + if (type) { + // if the type version was set properly, then add a watcher + // if it wasn't this means that the type version was previously set to something else + // and we set the owner to bottom, so we don't need to add a watcher because we must have + // already added one earlier. + if (sym_set_type_version(owner, type_version)) { + PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); + _Py_BloomFilter_Add(dependencies, type); + } + } + } } op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { - if (sym_matches_type(left, &PyFloat_Type) && - sym_matches_type(right, &PyFloat_Type)) { - REPLACE_OP(this_instr, _NOP, 0 ,0); - } - if (!sym_set_type(left, &PyFloat_Type)) { - goto hit_bottom; + if (sym_matches_type(left, &PyFloat_Type)) { + if (sym_matches_type(right, &PyFloat_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + REPLACE_OP(this_instr, _GUARD_TOS_FLOAT, 0, 0); + } } - if (!sym_set_type(right, &PyFloat_Type)) { - goto hit_bottom; + else { + if (sym_matches_type(right, &PyFloat_Type)) { + REPLACE_OP(this_instr, _GUARD_NOS_FLOAT, 0, 0); + } } + + sym_set_type(left, &PyFloat_Type); + sym_set_type(right, &PyFloat_Type); } op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) { @@ -111,11 +164,61 @@ dummy_func(void) { sym_matches_type(right, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } - if (!sym_set_type(left, &PyUnicode_Type)) { - goto hit_bottom; + sym_set_type(left, &PyUnicode_Type); + sym_set_type(left, &PyUnicode_Type); + } + + op(_BINARY_OP, (left, right -- res)) { + bool lhs_int = sym_matches_type(left, &PyLong_Type); + bool rhs_int = sym_matches_type(right, &PyLong_Type); + bool lhs_float = sym_matches_type(left, &PyFloat_Type); + bool rhs_float = sym_matches_type(right, &PyFloat_Type); + if (!((lhs_int || lhs_float) && (rhs_int || rhs_float))) { + // There's something other than an int or float involved: + res = sym_new_unknown(ctx); + } + else if (oparg == NB_POWER || oparg == NB_INPLACE_POWER) { + // This one's fun... the *type* of the result depends on the + // *values* being exponentiated. However, exponents with one + // constant part are reasonably common, so it's probably worth + // trying to infer some simple cases: + // - A: 1 ** 1 -> 1 (int ** int -> int) + // - B: 1 ** -1 -> 1.0 (int ** int -> float) + // - C: 1.0 ** 1 -> 1.0 (float ** int -> float) + // - D: 1 ** 1.0 -> 1.0 (int ** float -> float) + // - E: -1 ** 0.5 ~> 1j (int ** float -> complex) + // - F: 1.0 ** 1.0 -> 1.0 (float ** float -> float) + // - G: -1.0 ** 0.5 ~> 1j (float ** float -> complex) + if (rhs_float) { + // Case D, E, F, or G... can't know without the sign of the LHS + // or whether the RHS is whole, which isn't worth the effort: + res = sym_new_unknown(ctx); + } + else if (lhs_float) { + // Case C: + res = sym_new_type(ctx, &PyFloat_Type); + } + else if (!sym_is_const(right)) { + // Case A or B... can't know without the sign of the RHS: + res = sym_new_unknown(ctx); + } + else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(right))) { + // Case B: + res = sym_new_type(ctx, &PyFloat_Type); + } + else { + // Case A: + res = sym_new_type(ctx, &PyLong_Type); + } + } + else if (oparg == NB_TRUE_DIVIDE || oparg == NB_INPLACE_TRUE_DIVIDE) { + res = sym_new_type(ctx, &PyFloat_Type); + } + else if (lhs_int && rhs_int) { + res = sym_new_type(ctx, &PyLong_Type); } - if (!sym_set_type(right, &PyUnicode_Type)) { - goto hit_bottom; + else { + res = sym_new_type(ctx, &PyFloat_Type); } } @@ -132,12 +235,11 @@ dummy_func(void) { } res = sym_new_const(ctx, temp); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + res = sym_new_type(ctx, &PyLong_Type); } } @@ -154,12 +256,11 @@ dummy_func(void) { } res = sym_new_const(ctx, temp); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + res = sym_new_type(ctx, &PyLong_Type); } } @@ -176,12 +277,11 @@ dummy_func(void) { } res = sym_new_const(ctx, temp); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + res = sym_new_type(ctx, &PyLong_Type); } } @@ -199,12 +299,11 @@ dummy_func(void) { } res = sym_new_const(ctx, temp); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + res = sym_new_type(ctx, &PyFloat_Type); } } @@ -222,12 +321,11 @@ dummy_func(void) { } res = sym_new_const(ctx, temp); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + res = sym_new_type(ctx, &PyFloat_Type); } } @@ -245,12 +343,11 @@ dummy_func(void) { } res = sym_new_const(ctx, temp); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + res = sym_new_type(ctx, &PyFloat_Type); } } @@ -263,116 +360,173 @@ dummy_func(void) { } res = sym_new_const(ctx, temp); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyUnicode_Type)); + res = sym_new_type(ctx, &PyUnicode_Type); } } - op(_TO_BOOL, (value -- res)) { - (void)value; - res = sym_new_type(ctx, &PyBool_Type); - OUT_OF_SPACE_IF_NULL(res); - } - - op(_TO_BOOL_BOOL, (value -- value)) { - if (sym_matches_type(value, &PyBool_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right -- )) { + JitOptSymbol *res; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyUnicode_Type) && sym_matches_type(right, &PyUnicode_Type)) { + PyObject *temp = PyUnicode_Concat(sym_get_const(left), sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); } else { - if(!sym_set_type(value, &PyBool_Type)) { - goto hit_bottom; - } + res = sym_new_type(ctx, &PyUnicode_Type); } + // _STORE_FAST: + GETLOCAL(this_instr->operand0) = res; } - op(_TO_BOOL_INT, (value -- res)) { - if (sym_is_const(value) && sym_matches_type(value, &PyLong_Type)) { - PyObject *load = _PyLong_IsZero((PyLongObject *)sym_get_const(value)) - ? Py_False : Py_True; - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, load)); + op(_BINARY_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame: _Py_UOpsAbstractFrame *)) { + (void)container; + (void)sub; + (void)getitem; + new_frame = NULL; + ctx->done = true; + } + + op(_TO_BOOL, (value -- res)) { + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + res = sym_new_type(ctx, &PyBool_Type); } - else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + } + + op(_TO_BOOL_BOOL, (value -- res)) { + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyBool_Type); + res = value; } - if(!sym_set_type(value, &PyLong_Type)) { - goto hit_bottom; + } + + op(_TO_BOOL_INT, (value -- res)) { + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyLong_Type); + res = sym_new_type(ctx, &PyBool_Type); } } op(_TO_BOOL_LIST, (value -- res)) { - if(!sym_set_type(value, &PyList_Type)) { - goto hit_bottom; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyList_Type); + res = sym_new_type(ctx, &PyBool_Type); } - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); } op(_TO_BOOL_NONE, (value -- res)) { - if (sym_get_const(value) == Py_None) { - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_False); + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_const(value, Py_None); + res = sym_new_const(ctx, Py_False); } - sym_set_const(value, Py_None); - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, Py_False)); } op(_TO_BOOL_STR, (value -- res)) { - if (sym_is_const(value) && sym_matches_type(value, &PyUnicode_Type)) { - PyObject *load = sym_get_const(value) == &_Py_STR(empty) ? Py_False : Py_True; - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, load)); + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + res = sym_new_type(ctx, &PyBool_Type); + sym_set_type(value, &PyUnicode_Type); } - else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + } + + op(_COMPARE_OP, (left, right -- res)) { + (void)left; + (void)right; + if (oparg & 16) { + res = sym_new_type(ctx, &PyBool_Type); } - if(!sym_set_type(value, &PyUnicode_Type)) { - goto hit_bottom; + else { + res = _Py_uop_sym_new_not_null(ctx); } } + op(_COMPARE_OP_INT, (left, right -- res)) { + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + } + + op(_COMPARE_OP_FLOAT, (left, right -- res)) { + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + } + + op(_COMPARE_OP_STR, (left, right -- res)) { + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + } + + op(_IS_OP, (left, right -- res)) { + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + } + + op(_CONTAINS_OP, (left, right -- res)) { + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + } + op(_LOAD_CONST, (-- value)) { - // There should be no LOAD_CONST. It should be all - // replaced by peephole_opt. - Py_UNREACHABLE(); + PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); + int opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; + REPLACE_OP(this_instr, opcode, 0, (uintptr_t)val); + value = sym_new_const(ctx, val); } - op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + op(_LOAD_CONST_MORTAL, (-- value)) { + PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); + int opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; + REPLACE_OP(this_instr, opcode, 0, (uintptr_t)val); + value = sym_new_const(ctx, val); } - op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + op(_LOAD_CONST_IMMORTAL, (-- value)) { + PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); + REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + value = sym_new_const(ctx, val); } - op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); - OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + op(_LOAD_SMALL_INT, (-- value)) { + PyObject *val = PyLong_FromLong(this_instr->oparg); + value = sym_new_const(ctx, val); } - op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); - OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + value = sym_new_const(ctx, ptr); } + op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { + value = sym_new_const(ctx, ptr); + } op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { assert(oparg > 0); top = bottom; } - op(_SWAP, (bottom, unused[oparg-2], top -- - top, unused[oparg-2], bottom)) { + op(_SWAP, (bottom_in, unused[oparg-2], top_in -- + top_out, unused[oparg-2], bottom_out)) { + bottom_out = bottom_in; + top_out = top_in; } - op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { - _LOAD_ATTR_NOT_NULL - (void)index; + op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr)) { + attr = sym_new_not_null(ctx); + (void)offset; (void)owner; } - op(_CHECK_ATTR_MODULE, (dict_version/2, owner -- owner)) { + op(_CHECK_ATTR_MODULE_PUSH_KEYS, (dict_version/2, owner -- owner, mod_keys)) { (void)dict_version; + mod_keys = sym_new_not_null(ctx); if (sym_is_const(owner)) { PyObject *cnst = sym_get_const(owner); if (PyModule_CheckExact(cnst)) { @@ -388,12 +542,25 @@ dummy_func(void) { } } - op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { + op (_PUSH_NULL_CONDITIONAL, ( -- null if (oparg & 1))) { + int opcode = (oparg & 1) ? _PUSH_NULL : _NOP; + REPLACE_OP(this_instr, opcode, 0, 0); + null = sym_new_null(ctx); + } + + op(_LOAD_ATTR, (owner -- attr, self_or_null[oparg&1])) { + (void)owner; + attr = sym_new_not_null(ctx); + if (oparg &1) { + self_or_null[0] = sym_new_unknown(ctx); + } + } + + op(_LOAD_ATTR_MODULE_FROM_KEYS, (index/1, owner, mod_keys -- attr)) { (void)index; - OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); attr = NULL; if (this_instr[-1].opcode == _NOP) { - // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. + // Preceding _CHECK_ATTR_MODULE_PUSH_KEYS was removed: mod is const and dict is watched. assert(sym_is_const(owner)); PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); assert(PyModule_CheckExact(mod)); @@ -401,85 +568,114 @@ dummy_func(void) { PyObject *res = convert_global_to_const(this_instr, dict); if (res != NULL) { this_instr[-1].opcode = _POP_TOP; - OUT_OF_SPACE_IF_NULL(attr = sym_new_const(ctx, res)); + attr = sym_new_const(ctx, res); + } + else { + this_instr->opcode = _LOAD_ATTR_MODULE; } } if (attr == NULL) { /* No conversion made. We don't know what `attr` is. */ - OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + attr = sym_new_not_null(ctx); } } - op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) { - _LOAD_ATTR_NOT_NULL + op(_CHECK_ATTR_WITH_HINT, (owner -- owner, dict)) { + dict = sym_new_not_null(ctx); + (void)owner; + } + + op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict -- attr)) { + attr = sym_new_not_null(ctx); (void)hint; (void)owner; + (void)dict; } - op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { - _LOAD_ATTR_NOT_NULL + op(_LOAD_ATTR_SLOT, (index/1, owner -- attr)) { + attr = sym_new_not_null(ctx); (void)index; (void)owner; } - op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { - _LOAD_ATTR_NOT_NULL + op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr)) { + attr = sym_new_not_null(ctx); (void)descr; (void)owner; } - op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { + op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self)) { (void)descr; - OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + attr = sym_new_not_null(ctx); self = owner; } - op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self if (1))) { + op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self)) { (void)descr; - OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + attr = sym_new_not_null(ctx); self = owner; } - op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self if (1))) { + op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self)) { (void)descr; - OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + attr = sym_new_not_null(ctx); self = owner; } + op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame: _Py_UOpsAbstractFrame *)) { + (void)fget; + (void)owner; + new_frame = NULL; + ctx->done = true; + } + op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, unused, unused[oparg] -- func, self, unused[oparg])) { (void)callable; - OUT_OF_SPACE_IF_NULL(func = sym_new_not_null(ctx)); - OUT_OF_SPACE_IF_NULL(self = sym_new_not_null(ctx)); + func = sym_new_not_null(ctx); + self = sym_new_not_null(ctx); } - - op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - if (!sym_set_type(callable, &PyFunction_Type)) { - goto hit_bottom; + op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + (void)self_or_null; + if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { + assert(PyFunction_Check(sym_get_const(callable))); + REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + this_instr->operand1 = (uintptr_t)sym_get_const(callable); + } + sym_set_type(callable, &PyFunction_Type); + } + + op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + assert(sym_matches_type(callable, &PyFunction_Type)); + if (sym_is_const(callable)) { + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + PyFunctionObject *func = (PyFunctionObject *)sym_get_const(callable); + PyCodeObject *co = (PyCodeObject *)func->func_code; + if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { + REPLACE_OP(this_instr, _NOP, 0 ,0); + } + } } (void)self_or_null; - (void)func_version; } op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { - if (!sym_set_null(null)) { - goto hit_bottom; - } - if (!sym_set_type(callable, &PyMethod_Type)) { - goto hit_bottom; - } + sym_set_null(null); + sym_set_type(callable, &PyMethod_Type); } op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { int argcount = oparg; - (void)callable; - PyFunctionObject *func = (PyFunctionObject *)(this_instr + 2)->operand; - if (func == NULL) { - goto error; + PyCodeObject *co = NULL; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; } - PyCodeObject *co = (PyCodeObject *)func->func_code; + assert(self_or_null != NULL); assert(args != NULL); @@ -489,40 +685,171 @@ dummy_func(void) { argcount++; } - _Py_UopsSymbol **localsplus_start = ctx->n_consumed; - int n_locals_already_filled = 0; - // Can determine statically, so we interleave the new locals - // and make the current stack the new locals. - // This also sets up for true call inlining. if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - localsplus_start = args; - n_locals_already_filled = argcount; + new_frame = frame_new(ctx, co, 0, args, argcount); + } else { + new_frame = frame_new(ctx, co, 0, NULL, 0); + } - OUT_OF_SPACE_IF_NULL(new_frame = - frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0)); } - op(_POP_FRAME, (retval -- res)) { - SYNC_SP(); + op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- func, maybe_self, args[oparg])) { + (void)callable; + (void)self_or_null; + (void)args; + func = sym_new_not_null(ctx); + maybe_self = sym_new_not_null(ctx); + } + + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { + (void)(self_or_null); + (void)(callable); + PyCodeObject *co = NULL; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; + } + + new_frame = frame_new(ctx, co, 0, NULL, 0); + } + + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _Py_UOpsAbstractFrame *)) { + (void)callable; + (void)self_or_null; + (void)args; + (void)kwnames; + new_frame = NULL; + ctx->done = true; + } + + op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, null, args[oparg] -- self, init, args[oparg])) { + (void)type_version; + (void)callable; + (void)null; + (void)args; + self = sym_new_not_null(ctx); + init = sym_new_not_null(ctx); + } + + op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame: _Py_UOpsAbstractFrame *)) { + (void)self; + (void)init; + (void)args; + init_frame = NULL; + ctx->done = true; + } + + op(_RETURN_VALUE, (retval -- res)) { + SAVE_STACK(); ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; + + /* Stack space handling */ + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; + + co = get_code(this_instr); + if (co == NULL) { + // might be impossible, but bailing is still safe + ctx->done = true; + } + RELOAD_STACK(); res = retval; } + op(_RETURN_GENERATOR, ( -- res)) { + SYNC_SP(); + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); + stack_pointer = ctx->frame->stack_pointer; + res = sym_new_unknown(ctx); + + /* Stack space handling */ + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; + + co = get_code(this_instr); + if (co == NULL) { + // might be impossible, but bailing is still safe + ctx->done = true; + } + } + + op(_YIELD_VALUE, (unused -- res)) { + res = sym_new_unknown(ctx); + } + + op(_FOR_ITER_GEN_FRAME, ( -- )) { + /* We are about to hit the end of the trace */ + ctx->done = true; + } + + op(_SEND_GEN_FRAME, ( -- )) { + // We are about to hit the end of the trace: + ctx->done = true; + } + + op(_CHECK_STACK_SPACE, ( --)) { + assert(corresponding_check_stack == NULL); + corresponding_check_stack = this_instr; + } + + op (_CHECK_STACK_SPACE_OPERAND, ( -- )) { + (void)framesize; + /* We should never see _CHECK_STACK_SPACE_OPERANDs. + * They are only created at the end of this pass. */ + Py_UNREACHABLE(); + } + op(_PUSH_FRAME, (new_frame: _Py_UOpsAbstractFrame * -- unused if (0))) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; ctx->frame = new_frame; ctx->curr_frame_depth++; stack_pointer = new_frame->stack_pointer; + co = get_code(this_instr); + if (co == NULL) { + // should be about to _EXIT_TRACE anyway + ctx->done = true; + break; + } + + /* Stack space handling */ + int framesize = co->co_framesize; + assert(framesize > 0); + curr_space += framesize; + if (curr_space < 0 || curr_space > INT32_MAX) { + // won't fit in signed 32-bit int + ctx->done = true; + break; + } + max_space = curr_space > max_space ? curr_space : max_space; + if (first_valid_check_stack == NULL) { + first_valid_check_stack = corresponding_check_stack; + } + else if (corresponding_check_stack) { + // delete all but the first valid _CHECK_STACK_SPACE + corresponding_check_stack->opcode = _NOP; + } + corresponding_check_stack = NULL; } op(_UNPACK_SEQUENCE, (seq -- values[oparg])) { /* This has to be done manually */ (void)seq; for (int i = 0; i < oparg; i++) { - OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); + values[i] = sym_new_unknown(ctx); } } @@ -531,16 +858,106 @@ dummy_func(void) { (void)seq; int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { - OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); + values[i] = sym_new_unknown(ctx); } } op(_ITER_NEXT_RANGE, (iter -- iter, next)) { - OUT_OF_SPACE_IF_NULL(next = sym_new_type(ctx, &PyLong_Type)); + next = sym_new_type(ctx, &PyLong_Type); (void)iter; } + op(_GUARD_IS_TRUE_POP, (flag -- )) { + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, value != Py_True); + } + } + + op(_GUARD_IS_FALSE_POP, (flag -- )) { + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, value != Py_False); + } + } + + op(_GUARD_IS_NONE_POP, (flag -- )) { + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, !Py_IsNone(value)); + } + else if (sym_has_type(flag)) { + assert(!sym_matches_type(flag, &_PyNone_Type)); + eliminate_pop_guard(this_instr, true); + } + } + + op(_GUARD_IS_NOT_NONE_POP, (flag -- )) { + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, Py_IsNone(value)); + } + else if (sym_has_type(flag)) { + assert(!sym_matches_type(flag, &_PyNone_Type)); + eliminate_pop_guard(this_instr, false); + } + } + + op(_CHECK_PEP_523, (--)) { + /* Setting the eval frame function invalidates + * all executors, so no need to check dynamically */ + if (_PyInterpreterState_GET()->eval_frame == NULL) { + REPLACE_OP(this_instr, _NOP, 0 ,0); + } + } + + op(_LOAD_SPECIAL, (owner -- attr, self_or_null)) { + (void)owner; + attr = sym_new_not_null(ctx); + self_or_null = sym_new_unknown(ctx); + } + + op(_JUMP_TO_TOP, (--)) { + ctx->done = true; + } + + op(_EXIT_TRACE, (exit_p/4 --)) { + (void)exit_p; + ctx->done = true; + } + + op(_GUARD_GLOBALS_VERSION_PUSH_KEYS, (version/1 -- globals_keys)) { + globals_keys = sym_new_unknown(ctx); + (void)version; + } + op(_GUARD_BUILTINS_VERSION_PUSH_KEYS, (version/1 -- builtins_keys)) { + builtins_keys = sym_new_unknown(ctx); + (void)version; + } + + op(_REPLACE_WITH_TRUE, (value -- res)) { + res = sym_new_const(ctx, Py_True); + } + + op(_BUILD_TUPLE, (values[oparg] -- tup)) { + tup = sym_new_tuple(ctx, oparg, values); + } + + op(_UNPACK_SEQUENCE_TWO_TUPLE, (seq -- val1, val0)) { + val0 = sym_tuple_getitem(ctx, seq, 0); + val1 = sym_tuple_getitem(ctx, seq, 1); + } + + op(_UNPACK_SEQUENCE_TUPLE, (seq -- values[oparg])) { + for (int i = 0; i < oparg; i++) { + values[i] = sym_tuple_getitem(ctx, seq, i); + } + } // END BYTECODES // diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 6d3488f2118589..2497754745c28b 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -7,229 +7,265 @@ break; } + case _CHECK_PERIODIC: { + break; + } + + case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: { + break; + } + + /* _QUICKEN_RESUME is not a viable micro-op for tier 2 */ + + /* _LOAD_BYTECODE is not a viable micro-op for tier 2 */ + case _RESUME_CHECK: { break; } - /* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 */ + /* _MONITOR_RESUME is not a viable micro-op for tier 2 */ case _LOAD_FAST_CHECK: { - _Py_UopsSymbol *value; + JitOptSymbol *value; value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. if (sym_is_null(value)) { - goto out_of_space; + ctx->done = true; } stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_FAST: { - _Py_UopsSymbol *value; + JitOptSymbol *value; value = GETLOCAL(oparg); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_FAST_AND_CLEAR: { - _Py_UopsSymbol *value; + JitOptSymbol *value; value = GETLOCAL(oparg); - _Py_UopsSymbol *temp; - OUT_OF_SPACE_IF_NULL(temp = sym_new_null(ctx)); + JitOptSymbol *temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_CONST: { - _Py_UopsSymbol *value; - // There should be no LOAD_CONST. It should be all - // replaced by peephole_opt. - Py_UNREACHABLE(); + /* _LOAD_CONST is not a viable micro-op for tier 2 */ + + case _LOAD_CONST_MORTAL: { + JitOptSymbol *value; + PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); + int opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; + REPLACE_OP(this_instr, opcode, 0, (uintptr_t)val); + value = sym_new_const(ctx, val); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_CONST_IMMORTAL: { + JitOptSymbol *value; + PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); + REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + value = sym_new_const(ctx, val); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_SMALL_INT: { + JitOptSymbol *value; + PyObject *val = PyLong_FromLong(this_instr->oparg); + value = sym_new_const(ctx, val); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_FAST: { - _Py_UopsSymbol *value; + JitOptSymbol *value; value = stack_pointer[-1]; GETLOCAL(oparg) = value; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _POP_TOP: { stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _PUSH_NULL: { - _Py_UopsSymbol *res; + JitOptSymbol *res; res = sym_new_null(ctx); - if (res == NULL) { - goto out_of_space; - }; stack_pointer[0] = res; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _END_FOR: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _END_SEND: { - _Py_UopsSymbol *value; - value = sym_new_unknown(ctx); - if (value == NULL) goto out_of_space; - stack_pointer[-2] = value; + JitOptSymbol *val; + val = sym_new_not_null(ctx); + stack_pointer[-2] = val; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _UNARY_NEGATIVE: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _UNARY_NOT: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _TO_BOOL: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + JitOptSymbol *value; + JitOptSymbol *res; value = stack_pointer[-1]; - (void)value; - res = sym_new_type(ctx, &PyBool_Type); - OUT_OF_SPACE_IF_NULL(res); + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + res = sym_new_type(ctx, &PyBool_Type); + } stack_pointer[-1] = res; break; } case _TO_BOOL_BOOL: { - _Py_UopsSymbol *value; + JitOptSymbol *value; + JitOptSymbol *res; value = stack_pointer[-1]; - if (sym_matches_type(value, &PyBool_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); - } - else { - if(!sym_set_type(value, &PyBool_Type)) { - goto hit_bottom; - } + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyBool_Type); + res = value; } + stack_pointer[-1] = res; break; } case _TO_BOOL_INT: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + JitOptSymbol *value; + JitOptSymbol *res; value = stack_pointer[-1]; - if (sym_is_const(value) && sym_matches_type(value, &PyLong_Type)) { - PyObject *load = _PyLong_IsZero((PyLongObject *)sym_get_const(value)) - ? Py_False : Py_True; - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, load)); - } - else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); - } - if(!sym_set_type(value, &PyLong_Type)) { - goto hit_bottom; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyLong_Type); + res = sym_new_type(ctx, &PyBool_Type); } stack_pointer[-1] = res; break; } case _TO_BOOL_LIST: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + JitOptSymbol *value; + JitOptSymbol *res; value = stack_pointer[-1]; - if(!sym_set_type(value, &PyList_Type)) { - goto hit_bottom; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyList_Type); + res = sym_new_type(ctx, &PyBool_Type); } - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); stack_pointer[-1] = res; break; } case _TO_BOOL_NONE: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + JitOptSymbol *value; + JitOptSymbol *res; value = stack_pointer[-1]; - if (sym_get_const(value) == Py_None) { - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_False); + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_const(value, Py_None); + res = sym_new_const(ctx, Py_False); } - sym_set_const(value, Py_None); - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, Py_False)); stack_pointer[-1] = res; break; } case _TO_BOOL_STR: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + JitOptSymbol *value; + JitOptSymbol *res; value = stack_pointer[-1]; - if (sym_is_const(value) && sym_matches_type(value, &PyUnicode_Type)) { - PyObject *load = sym_get_const(value) == &_Py_STR(empty) ? Py_False : Py_True; - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, load)); - } - else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); - } - if(!sym_set_type(value, &PyUnicode_Type)) { - goto hit_bottom; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + res = sym_new_type(ctx, &PyBool_Type); + sym_set_type(value, &PyUnicode_Type); } stack_pointer[-1] = res; break; } - case _TO_BOOL_ALWAYS_TRUE: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + case _REPLACE_WITH_TRUE: { + JitOptSymbol *res; + res = sym_new_const(ctx, Py_True); stack_pointer[-1] = res; break; } case _UNARY_INVERT: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _GUARD_BOTH_INT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; + JitOptSymbol *right; + JitOptSymbol *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_matches_type(left, &PyLong_Type) && - sym_matches_type(right, &PyLong_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); - } - if (!sym_set_type(left, &PyLong_Type)) { - goto hit_bottom; + if (sym_matches_type(left, &PyLong_Type)) { + if (sym_matches_type(right, &PyLong_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + REPLACE_OP(this_instr, _GUARD_TOS_INT, 0, 0); + } } - if (!sym_set_type(right, &PyLong_Type)) { - goto hit_bottom; + else { + if (sym_matches_type(right, &PyLong_Type)) { + REPLACE_OP(this_instr, _GUARD_NOS_INT, 0, 0); + } } + sym_set_type(left, &PyLong_Type); + sym_set_type(right, &PyLong_Type); + break; + } + + case _GUARD_NOS_INT: { + break; + } + + case _GUARD_TOS_INT: { break; } case _BINARY_OP_MULTIPLY_INT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + JitOptSymbol *right; + JitOptSymbol *left; + JitOptSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -243,23 +279,26 @@ goto error; } res = sym_new_const(ctx, temp); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + res = sym_new_type(ctx, &PyLong_Type); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[-1] = res; break; } case _BINARY_OP_ADD_INT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + JitOptSymbol *right; + JitOptSymbol *left; + JitOptSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -273,23 +312,26 @@ goto error; } res = sym_new_const(ctx, temp); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + res = sym_new_type(ctx, &PyLong_Type); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[-1] = res; break; } case _BINARY_OP_SUBTRACT_INT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + JitOptSymbol *right; + JitOptSymbol *left; + JitOptSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -303,41 +345,57 @@ goto error; } res = sym_new_const(ctx, temp); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and add tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + res = sym_new_type(ctx, &PyLong_Type); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[-1] = res; break; } case _GUARD_BOTH_FLOAT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; + JitOptSymbol *right; + JitOptSymbol *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_matches_type(left, &PyFloat_Type) && - sym_matches_type(right, &PyFloat_Type)) { - REPLACE_OP(this_instr, _NOP, 0 ,0); - } - if (!sym_set_type(left, &PyFloat_Type)) { - goto hit_bottom; + if (sym_matches_type(left, &PyFloat_Type)) { + if (sym_matches_type(right, &PyFloat_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + REPLACE_OP(this_instr, _GUARD_TOS_FLOAT, 0, 0); + } } - if (!sym_set_type(right, &PyFloat_Type)) { - goto hit_bottom; + else { + if (sym_matches_type(right, &PyFloat_Type)) { + REPLACE_OP(this_instr, _GUARD_NOS_FLOAT, 0, 0); + } } + sym_set_type(left, &PyFloat_Type); + sym_set_type(right, &PyFloat_Type); + break; + } + + case _GUARD_NOS_FLOAT: { + break; + } + + case _GUARD_TOS_FLOAT: { break; } case _BINARY_OP_MULTIPLY_FLOAT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + JitOptSymbol *right; + JitOptSymbol *left; + JitOptSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -352,23 +410,26 @@ goto error; } res = sym_new_const(ctx, temp); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + res = sym_new_type(ctx, &PyFloat_Type); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[-1] = res; break; } case _BINARY_OP_ADD_FLOAT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + JitOptSymbol *right; + JitOptSymbol *left; + JitOptSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -383,23 +444,26 @@ goto error; } res = sym_new_const(ctx, temp); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + res = sym_new_type(ctx, &PyFloat_Type); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[-1] = res; break; } case _BINARY_OP_SUBTRACT_FLOAT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + JitOptSymbol *right; + JitOptSymbol *left; + JitOptSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -414,43 +478,69 @@ goto error; } res = sym_new_const(ctx, temp); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); // TODO gh-115506: // replace opcode with constant propagated one and update tests! } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + res = sym_new_type(ctx, &PyFloat_Type); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[-1] = res; break; } case _GUARD_BOTH_UNICODE: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; + JitOptSymbol *right; + JitOptSymbol *left; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_matches_type(left, &PyUnicode_Type) && sym_matches_type(right, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } - if (!sym_set_type(left, &PyUnicode_Type)) { - goto hit_bottom; + sym_set_type(left, &PyUnicode_Type); + sym_set_type(left, &PyUnicode_Type); + break; + } + + case _BINARY_OP_ADD_UNICODE: { + JitOptSymbol *right; + JitOptSymbol *left; + JitOptSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyUnicode_Type) && sym_matches_type(right, &PyUnicode_Type)) { + PyObject *temp = PyUnicode_Concat(sym_get_const(left), sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + Py_DECREF(temp); } - if (!sym_set_type(right, &PyUnicode_Type)) { - goto hit_bottom; + else { + res = sym_new_type(ctx, &PyUnicode_Type); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } + stack_pointer[-1] = res; break; } - case _BINARY_OP_ADD_UNICODE: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + case _BINARY_OP_INPLACE_ADD_UNICODE: { + JitOptSymbol *right; + JitOptSymbol *left; right = stack_pointer[-1]; left = stack_pointer[-2]; + JitOptSymbol *res; if (sym_is_const(left) && sym_is_const(right) && sym_matches_type(left, &PyUnicode_Type) && sym_matches_type(right, &PyUnicode_Type)) { PyObject *temp = PyUnicode_Concat(sym_get_const(left), sym_get_const(right)); @@ -458,199 +548,266 @@ goto error; } res = sym_new_const(ctx, temp); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); - OUT_OF_SPACE_IF_NULL(res); } else { - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyUnicode_Type)); + res = sym_new_type(ctx, &PyUnicode_Type); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } + // _STORE_FAST: + GETLOCAL(this_instr->operand0) = res; + break; + } + + case _GUARD_BINARY_OP_EXTEND: { + break; + } + + case _BINARY_OP_EXTEND: { + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_SUBSCR: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_SLICE: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_SLICE: { stack_pointer += -4; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_SUBSCR_LIST_INT: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_SUBSCR_STR_INT: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_SUBSCR_TUPLE_INT: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_SUBSCR_DICT: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - /* _BINARY_SUBSCR_GETITEM is not a viable micro-op for tier 2 */ + case _BINARY_SUBSCR_CHECK_FUNC: { + JitOptSymbol *getitem; + getitem = sym_new_not_null(ctx); + stack_pointer[0] = getitem; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR_INIT_CALL: { + JitOptSymbol *getitem; + JitOptSymbol *sub; + JitOptSymbol *container; + _Py_UOpsAbstractFrame *new_frame; + getitem = stack_pointer[-1]; + sub = stack_pointer[-2]; + container = stack_pointer[-3]; + (void)container; + (void)sub; + (void)getitem; + new_frame = NULL; + ctx->done = true; + stack_pointer[-3] = (JitOptSymbol *)new_frame; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _LIST_APPEND: { stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _SET_ADD: { stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_SUBSCR: { stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_SUBSCR_LIST_INT: { stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_SUBSCR_DICT: { stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); break; } case _DELETE_SUBSCR: { stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_INTRINSIC_1: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _CALL_INTRINSIC_2: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _POP_FRAME: { - _Py_UopsSymbol *retval; - _Py_UopsSymbol *res; + case _RETURN_VALUE: { + JitOptSymbol *retval; + JitOptSymbol *res; retval = stack_pointer[-1]; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; + /* Stack space handling */ + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; + co = get_code(this_instr); + if (co == NULL) { + // might be impossible, but bailing is still safe + ctx->done = true; + } res = retval; stack_pointer[0] = res; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - /* _INSTRUMENTED_RETURN_VALUE is not a viable micro-op for tier 2 */ - - /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 */ - case _GET_AITER: { - _Py_UopsSymbol *iter; - iter = sym_new_unknown(ctx); - if (iter == NULL) goto out_of_space; + JitOptSymbol *iter; + iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_ANEXT: { - _Py_UopsSymbol *awaitable; - awaitable = sym_new_unknown(ctx); - if (awaitable == NULL) goto out_of_space; + JitOptSymbol *awaitable; + awaitable = sym_new_not_null(ctx); stack_pointer[0] = awaitable; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _GET_AWAITABLE: { - _Py_UopsSymbol *iter; - iter = sym_new_unknown(ctx); - if (iter == NULL) goto out_of_space; + JitOptSymbol *iter; + iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } /* _SEND is not a viable micro-op for tier 2 */ - /* _SEND_GEN is not a viable micro-op for tier 2 */ + case _SEND_GEN_FRAME: { + // We are about to hit the end of the trace: + ctx->done = true; + break; + } - /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 */ + case _YIELD_VALUE: { + JitOptSymbol *res; + res = sym_new_unknown(ctx); + stack_pointer[-1] = res; + break; + } case _POP_EXCEPT: { stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_ASSERTION_ERROR: { - _Py_UopsSymbol *value; - value = sym_new_unknown(ctx); - if (value == NULL) goto out_of_space; + case _LOAD_COMMON_CONSTANT: { + JitOptSymbol *value; + value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_BUILD_CLASS: { - _Py_UopsSymbol *bc; - bc = sym_new_unknown(ctx); - if (bc == NULL) goto out_of_space; + JitOptSymbol *bc; + bc = sym_new_not_null(ctx); stack_pointer[0] = bc; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_NAME: { stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -659,79 +816,89 @@ } case _UNPACK_SEQUENCE: { - _Py_UopsSymbol *seq; - _Py_UopsSymbol **values; + JitOptSymbol *seq; + JitOptSymbol **values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; /* This has to be done manually */ (void)seq; for (int i = 0; i < oparg; i++) { - OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); + values[i] = sym_new_unknown(ctx); } stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _UNPACK_SEQUENCE_TWO_TUPLE: { - _Py_UopsSymbol **values; - values = &stack_pointer[-1]; - for (int _i = oparg; --_i >= 0;) { - values[_i] = sym_new_unknown(ctx); - if (values[_i] == NULL) goto out_of_space; - } - stack_pointer += -1 + oparg; + JitOptSymbol *seq; + JitOptSymbol *val1; + JitOptSymbol *val0; + seq = stack_pointer[-1]; + val0 = sym_tuple_getitem(ctx, seq, 0); + val1 = sym_tuple_getitem(ctx, seq, 1); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _UNPACK_SEQUENCE_TUPLE: { - _Py_UopsSymbol **values; + JitOptSymbol *seq; + JitOptSymbol **values; + seq = stack_pointer[-1]; values = &stack_pointer[-1]; - for (int _i = oparg; --_i >= 0;) { - values[_i] = sym_new_unknown(ctx); - if (values[_i] == NULL) goto out_of_space; + for (int i = 0; i < oparg; i++) { + values[i] = sym_tuple_getitem(ctx, seq, i); } stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _UNPACK_SEQUENCE_LIST: { - _Py_UopsSymbol **values; + JitOptSymbol **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { - values[_i] = sym_new_unknown(ctx); - if (values[_i] == NULL) goto out_of_space; + values[_i] = sym_new_not_null(ctx); } stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _UNPACK_EX: { - _Py_UopsSymbol *seq; - _Py_UopsSymbol **values; + JitOptSymbol *seq; + JitOptSymbol **values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; /* This has to be done manually */ (void)seq; int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { - OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); + values[i] = sym_new_unknown(ctx); } - stack_pointer += (oparg >> 8) + (oparg & 0xFF); + stack_pointer += (oparg & 0xFF) + (oparg >> 8); + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_ATTR: { stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } case _DELETE_ATTR: { stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_GLOBAL: { stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -740,41 +907,42 @@ } case _LOAD_LOCALS: { - _Py_UopsSymbol *locals; - locals = sym_new_unknown(ctx); - if (locals == NULL) goto out_of_space; + JitOptSymbol *locals; + locals = sym_new_not_null(ctx); stack_pointer[0] = locals; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_FROM_DICT_OR_GLOBALS: { - _Py_UopsSymbol *v; - v = sym_new_unknown(ctx); - if (v == NULL) goto out_of_space; - stack_pointer[-1] = v; - break; - } + /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ case _LOAD_NAME: { - _Py_UopsSymbol *v; - v = sym_new_unknown(ctx); - if (v == NULL) goto out_of_space; + JitOptSymbol *v; + v = sym_new_not_null(ctx); stack_pointer[0] = v; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_GLOBAL: { - _Py_UopsSymbol *res; - _Py_UopsSymbol *null = NULL; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol **res; + res = &stack_pointer[0]; + res[0] = sym_new_not_null(ctx); + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _PUSH_NULL_CONDITIONAL: { + JitOptSymbol *null = NULL; + int opcode = (oparg & 1) ? _PUSH_NULL : _NOP; + REPLACE_OP(this_instr, opcode, 0, 0); null = sym_new_null(ctx); - if (null == NULL) goto out_of_space; - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); break; } @@ -782,33 +950,39 @@ break; } - case _GUARD_BUILTINS_VERSION: { + case _GUARD_GLOBALS_VERSION_PUSH_KEYS: { + JitOptSymbol *globals_keys; + uint16_t version = (uint16_t)this_instr->operand0; + globals_keys = sym_new_unknown(ctx); + (void)version; + stack_pointer[0] = globals_keys; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_GLOBAL_MODULE: { - _Py_UopsSymbol *res; - _Py_UopsSymbol *null = NULL; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; - null = sym_new_null(ctx); - if (null == NULL) goto out_of_space; - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + case _GUARD_BUILTINS_VERSION_PUSH_KEYS: { + JitOptSymbol *builtins_keys; + uint16_t version = (uint16_t)this_instr->operand0; + builtins_keys = sym_new_unknown(ctx); + (void)version; + stack_pointer[0] = builtins_keys; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_GLOBAL_BUILTINS: { - _Py_UopsSymbol *res; - _Py_UopsSymbol *null = NULL; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; - null = sym_new_null(ctx); - if (null == NULL) goto out_of_space; - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + case _LOAD_GLOBAL_MODULE_FROM_KEYS: { + JitOptSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _LOAD_GLOBAL_BUILTINS_FROM_KEYS: { + JitOptSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; break; } @@ -825,24 +999,24 @@ } case _LOAD_FROM_DICT_OR_DEREF: { - _Py_UopsSymbol *value; - value = sym_new_unknown(ctx); - if (value == NULL) goto out_of_space; + JitOptSymbol *value; + value = sym_new_not_null(ctx); stack_pointer[-1] = value; break; } case _LOAD_DEREF: { - _Py_UopsSymbol *value; - value = sym_new_unknown(ctx); - if (value == NULL) goto out_of_space; + JitOptSymbol *value; + value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_DEREF: { stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -851,57 +1025,61 @@ } case _BUILD_STRING: { - _Py_UopsSymbol *str; - str = sym_new_unknown(ctx); - if (str == NULL) goto out_of_space; + JitOptSymbol *str; + str = sym_new_not_null(ctx); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _BUILD_TUPLE: { - _Py_UopsSymbol *tup; - tup = sym_new_unknown(ctx); - if (tup == NULL) goto out_of_space; + JitOptSymbol **values; + JitOptSymbol *tup; + values = &stack_pointer[-oparg]; + tup = sym_new_tuple(ctx, oparg, values); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _BUILD_LIST: { - _Py_UopsSymbol *list; - list = sym_new_unknown(ctx); - if (list == NULL) goto out_of_space; + JitOptSymbol *list; + list = sym_new_not_null(ctx); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _LIST_EXTEND: { stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _SET_UPDATE: { stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BUILD_SET: { - _Py_UopsSymbol *set; - set = sym_new_unknown(ctx); - if (set == NULL) goto out_of_space; + JitOptSymbol *set; + set = sym_new_not_null(ctx); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _BUILD_MAP: { - _Py_UopsSymbol *map; - map = sym_new_unknown(ctx); - if (map == NULL) goto out_of_space; + JitOptSymbol *map; + map = sym_new_not_null(ctx); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -909,68 +1087,91 @@ break; } - case _BUILD_CONST_KEY_MAP: { - _Py_UopsSymbol *map; - map = sym_new_unknown(ctx); - if (map == NULL) goto out_of_space; - stack_pointer[-1 - oparg] = map; - stack_pointer += -oparg; - break; - } - case _DICT_UPDATE: { stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _DICT_MERGE: { stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _MAP_ADD: { stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ case _LOAD_SUPER_ATTR_ATTR: { - _Py_UopsSymbol *attr; - attr = sym_new_unknown(ctx); - if (attr == NULL) goto out_of_space; - stack_pointer[-3] = attr; + JitOptSymbol *attr_st; + attr_st = sym_new_not_null(ctx); + stack_pointer[-3] = attr_st; stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_SUPER_ATTR_METHOD: { - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self_or_null; - attr = sym_new_unknown(ctx); - if (attr == NULL) goto out_of_space; - self_or_null = sym_new_unknown(ctx); - if (self_or_null == NULL) goto out_of_space; + JitOptSymbol *attr; + JitOptSymbol *self_or_null; + attr = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_ATTR: { - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self_or_null = NULL; - attr = sym_new_unknown(ctx); - if (attr == NULL) goto out_of_space; - self_or_null = sym_new_unknown(ctx); - if (self_or_null == NULL) goto out_of_space; + JitOptSymbol *owner; + JitOptSymbol *attr; + JitOptSymbol **self_or_null; + owner = stack_pointer[-1]; + self_or_null = &stack_pointer[0]; + (void)owner; + attr = sym_new_not_null(ctx); + if (oparg &1) { + self_or_null[0] = sym_new_unknown(ctx); + } stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = self_or_null; - stack_pointer += (oparg & 1); + stack_pointer += (oparg&1); + assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_TYPE_VERSION: { + JitOptSymbol *owner; + owner = stack_pointer[-1]; + uint32_t type_version = (uint32_t)this_instr->operand0; + assert(type_version); + if (sym_matches_type_version(owner, type_version)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } else { + // add watcher so that whenever the type changes we invalidate this + PyTypeObject *type = _PyType_LookupByVersion(type_version); + // if the type is null, it was not found in the cache (there was a conflict) + // with the key, in which case we can't trust the version + if (type) { + // if the type version was set properly, then add a watcher + // if it wasn't this means that the type version was previously set to something else + // and we set the owner to bottom, so we don't need to add a watcher because we must have + // already added one earlier. + if (sym_set_type_version(owner, type_version)) { + PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); + _Py_BloomFilter_Add(dependencies, type); + } + } + } + break; + } + + case _GUARD_TYPE_VERSION_AND_LOCK: { break; } @@ -979,103 +1180,123 @@ } case _LOAD_ATTR_INSTANCE_VALUE: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *null = NULL; + JitOptSymbol *owner; + JitOptSymbol *attr; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)this_instr->operand; - _LOAD_ATTR_NOT_NULL - (void)index; + uint16_t offset = (uint16_t)this_instr->operand0; + attr = sym_new_not_null(ctx); + (void)offset; (void)owner; stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); break; } - case _CHECK_ATTR_MODULE: { - _Py_UopsSymbol *owner; + case _CHECK_ATTR_MODULE_PUSH_KEYS: { + JitOptSymbol *owner; + JitOptSymbol *mod_keys; owner = stack_pointer[-1]; - uint32_t dict_version = (uint32_t)this_instr->operand; + uint32_t dict_version = (uint32_t)this_instr->operand0; (void)dict_version; + mod_keys = sym_new_not_null(ctx); if (sym_is_const(owner)) { PyObject *cnst = sym_get_const(owner); if (PyModule_CheckExact(cnst)) { PyModuleObject *mod = (PyModuleObject *)cnst; PyObject *dict = mod->md_dict; + stack_pointer[0] = mod_keys; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); uint64_t watched_mutations = get_mutations(dict); if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { PyDict_Watch(GLOBALS_WATCHER_ID, dict); _Py_BloomFilter_Add(dependencies, dict); this_instr->opcode = _NOP; } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } } + stack_pointer[0] = mod_keys; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_ATTR_MODULE: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *null = NULL; - owner = stack_pointer[-1]; - uint16_t index = (uint16_t)this_instr->operand; + case _LOAD_ATTR_MODULE_FROM_KEYS: { + JitOptSymbol *owner; + JitOptSymbol *attr; + owner = stack_pointer[-2]; + uint16_t index = (uint16_t)this_instr->operand0; (void)index; - OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); attr = NULL; if (this_instr[-1].opcode == _NOP) { - // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. + // Preceding _CHECK_ATTR_MODULE_PUSH_KEYS was removed: mod is const and dict is watched. assert(sym_is_const(owner)); PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); assert(PyModule_CheckExact(mod)); PyObject *dict = mod->md_dict; + stack_pointer[-2] = attr; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); PyObject *res = convert_global_to_const(this_instr, dict); if (res != NULL) { this_instr[-1].opcode = _POP_TOP; - OUT_OF_SPACE_IF_NULL(attr = sym_new_const(ctx, res)); + attr = sym_new_const(ctx, res); + } + else { + this_instr->opcode = _LOAD_ATTR_MODULE; } + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } if (attr == NULL) { /* No conversion made. We don't know what `attr` is. */ - OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + attr = sym_new_not_null(ctx); } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); + stack_pointer[-2] = attr; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _CHECK_ATTR_WITH_HINT: { + JitOptSymbol *owner; + JitOptSymbol *dict; + owner = stack_pointer[-1]; + dict = sym_new_not_null(ctx); + (void)owner; + stack_pointer[0] = dict; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_ATTR_WITH_HINT: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *null = NULL; - owner = stack_pointer[-1]; - uint16_t hint = (uint16_t)this_instr->operand; - _LOAD_ATTR_NOT_NULL + JitOptSymbol *dict; + JitOptSymbol *owner; + JitOptSymbol *attr; + dict = stack_pointer[-1]; + owner = stack_pointer[-2]; + uint16_t hint = (uint16_t)this_instr->operand0; + attr = sym_new_not_null(ctx); (void)hint; (void)owner; - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); + (void)dict; + stack_pointer[-2] = attr; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_ATTR_SLOT: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *null = NULL; + JitOptSymbol *owner; + JitOptSymbol *attr; owner = stack_pointer[-1]; - uint16_t index = (uint16_t)this_instr->operand; - _LOAD_ATTR_NOT_NULL + uint16_t index = (uint16_t)this_instr->operand0; + attr = sym_new_not_null(ctx); (void)index; (void)owner; stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); break; } @@ -1084,183 +1305,272 @@ } case _LOAD_ATTR_CLASS: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *null = NULL; + JitOptSymbol *owner; + JitOptSymbol *attr; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)this_instr->operand; - _LOAD_ATTR_NOT_NULL + PyObject *descr = (PyObject *)this_instr->operand0; + attr = sym_new_not_null(ctx); (void)descr; (void)owner; stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); break; } - /* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 */ + case _LOAD_ATTR_PROPERTY_FRAME: { + JitOptSymbol *owner; + _Py_UOpsAbstractFrame *new_frame; + owner = stack_pointer[-1]; + PyObject *fget = (PyObject *)this_instr->operand0; + (void)fget; + (void)owner; + new_frame = NULL; + ctx->done = true; + stack_pointer[-1] = (JitOptSymbol *)new_frame; + break; + } /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */ - case _GUARD_DORV_VALUES: { + case _GUARD_DORV_NO_DICT: { break; } case _STORE_ATTR_INSTANCE_VALUE: { stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } - /* _STORE_ATTR_WITH_HINT is not a viable micro-op for tier 2 */ + case _STORE_ATTR_WITH_HINT: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _STORE_ATTR_SLOT: { stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } case _COMPARE_OP: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *right; + JitOptSymbol *left; + JitOptSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + if (oparg & 16) { + res = sym_new_type(ctx, &PyBool_Type); + } + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + res = _Py_uop_sym_new_not_null(ctx); + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + } stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _COMPARE_OP_FLOAT: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *right; + JitOptSymbol *left; + JitOptSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _COMPARE_OP_INT: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *right; + JitOptSymbol *left; + JitOptSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _COMPARE_OP_STR: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *right; + JitOptSymbol *left; + JitOptSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _IS_OP: { - _Py_UopsSymbol *b; - b = sym_new_unknown(ctx); - if (b == NULL) goto out_of_space; - stack_pointer[-2] = b; + JitOptSymbol *right; + JitOptSymbol *left; + JitOptSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _CONTAINS_OP: { - _Py_UopsSymbol *b; - b = sym_new_unknown(ctx); - if (b == NULL) goto out_of_space; + JitOptSymbol *right; + JitOptSymbol *left; + JitOptSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CONTAINS_OP_SET: { + JitOptSymbol *b; + b = sym_new_not_null(ctx); + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CONTAINS_OP_DICT: { + JitOptSymbol *b; + b = sym_new_not_null(ctx); stack_pointer[-2] = b; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _CHECK_EG_MATCH: { - _Py_UopsSymbol *rest; - _Py_UopsSymbol *match; - rest = sym_new_unknown(ctx); - if (rest == NULL) goto out_of_space; - match = sym_new_unknown(ctx); - if (match == NULL) goto out_of_space; + JitOptSymbol *rest; + JitOptSymbol *match; + rest = sym_new_not_null(ctx); + match = sym_new_not_null(ctx); stack_pointer[-2] = rest; stack_pointer[-1] = match; break; } case _CHECK_EXC_MATCH: { - _Py_UopsSymbol *b; - b = sym_new_unknown(ctx); - if (b == NULL) goto out_of_space; + JitOptSymbol *b; + b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } + case _IMPORT_NAME: { + JitOptSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _IMPORT_FROM: { + JitOptSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ case _IS_NONE: { - _Py_UopsSymbol *b; - b = sym_new_unknown(ctx); - if (b == NULL) goto out_of_space; + JitOptSymbol *b; + b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _GET_LEN: { - _Py_UopsSymbol *len_o; - len_o = sym_new_unknown(ctx); - if (len_o == NULL) goto out_of_space; - stack_pointer[0] = len_o; + JitOptSymbol *len; + len = sym_new_not_null(ctx); + stack_pointer[0] = len; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _MATCH_CLASS: { - _Py_UopsSymbol *attrs; - attrs = sym_new_unknown(ctx); - if (attrs == NULL) goto out_of_space; + JitOptSymbol *attrs; + attrs = sym_new_not_null(ctx); stack_pointer[-3] = attrs; stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } case _MATCH_MAPPING: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _MATCH_SEQUENCE: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _MATCH_KEYS: { - _Py_UopsSymbol *values_or_none; - values_or_none = sym_new_unknown(ctx); - if (values_or_none == NULL) goto out_of_space; + JitOptSymbol *values_or_none; + values_or_none = sym_new_not_null(ctx); stack_pointer[0] = values_or_none; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _GET_ITER: { - _Py_UopsSymbol *iter; - iter = sym_new_unknown(ctx); - if (iter == NULL) goto out_of_space; + JitOptSymbol *iter; + iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_YIELD_FROM_ITER: { - _Py_UopsSymbol *iter; - iter = sym_new_unknown(ctx); - if (iter == NULL) goto out_of_space; + JitOptSymbol *iter; + iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } @@ -1268,11 +1578,11 @@ /* _FOR_ITER is not a viable micro-op for tier 2 */ case _FOR_ITER_TIER_TWO: { - _Py_UopsSymbol *next; - next = sym_new_unknown(ctx); - if (next == NULL) goto out_of_space; + JitOptSymbol *next; + next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -1289,11 +1599,11 @@ } case _ITER_NEXT_LIST: { - _Py_UopsSymbol *next; - next = sym_new_unknown(ctx); - if (next == NULL) goto out_of_space; + JitOptSymbol *next; + next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -1308,11 +1618,11 @@ } case _ITER_NEXT_TUPLE: { - _Py_UopsSymbol *next; - next = sym_new_unknown(ctx); - if (next == NULL) goto out_of_space; + JitOptSymbol *next; + next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -1327,63 +1637,56 @@ } case _ITER_NEXT_RANGE: { - _Py_UopsSymbol *iter; - _Py_UopsSymbol *next; + JitOptSymbol *iter; + JitOptSymbol *next; iter = stack_pointer[-1]; - OUT_OF_SPACE_IF_NULL(next = sym_new_type(ctx, &PyLong_Type)); + next = sym_new_type(ctx, &PyLong_Type); (void)iter; stack_pointer[0] = next; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - /* _FOR_ITER_GEN is not a viable micro-op for tier 2 */ - - case _BEFORE_ASYNC_WITH: { - _Py_UopsSymbol *exit; - _Py_UopsSymbol *res; - exit = sym_new_unknown(ctx); - if (exit == NULL) goto out_of_space; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; - stack_pointer[-1] = exit; - stack_pointer[0] = res; - stack_pointer += 1; + case _FOR_ITER_GEN_FRAME: { + /* We are about to hit the end of the trace */ + ctx->done = true; break; } - case _BEFORE_WITH: { - _Py_UopsSymbol *exit; - _Py_UopsSymbol *res; - exit = sym_new_unknown(ctx); - if (exit == NULL) goto out_of_space; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; - stack_pointer[-1] = exit; - stack_pointer[0] = res; + case _LOAD_SPECIAL: { + JitOptSymbol *owner; + JitOptSymbol *attr; + JitOptSymbol *self_or_null; + owner = stack_pointer[-1]; + (void)owner; + attr = sym_new_not_null(ctx); + self_or_null = sym_new_unknown(ctx); + stack_pointer[-1] = attr; + stack_pointer[0] = self_or_null; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _WITH_EXCEPT_START: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _PUSH_EXC_INFO: { - _Py_UopsSymbol *prev_exc; - _Py_UopsSymbol *new_exc; - prev_exc = sym_new_unknown(ctx); - if (prev_exc == NULL) goto out_of_space; - new_exc = sym_new_unknown(ctx); - if (new_exc == NULL) goto out_of_space; + JitOptSymbol *prev_exc; + JitOptSymbol *new_exc; + prev_exc = sym_new_not_null(ctx); + new_exc = sym_new_not_null(ctx); stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -1396,47 +1699,47 @@ } case _LOAD_ATTR_METHOD_WITH_VALUES: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self = NULL; + JitOptSymbol *owner; + JitOptSymbol *attr; + JitOptSymbol *self; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)this_instr->operand; + PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + attr = sym_new_not_null(ctx); self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_ATTR_METHOD_NO_DICT: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self = NULL; + JitOptSymbol *owner; + JitOptSymbol *attr; + JitOptSymbol *self; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)this_instr->operand; + PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + attr = sym_new_not_null(ctx); self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - _Py_UopsSymbol *attr; - attr = sym_new_unknown(ctx); - if (attr == NULL) goto out_of_space; + JitOptSymbol *attr; + attr = sym_new_not_null(ctx); stack_pointer[-1] = attr; break; } case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - _Py_UopsSymbol *attr; - attr = sym_new_unknown(ctx); - if (attr == NULL) goto out_of_space; + JitOptSymbol *attr; + attr = sym_new_not_null(ctx); stack_pointer[-1] = attr; break; } @@ -1446,88 +1749,192 @@ } case _LOAD_ATTR_METHOD_LAZY_DICT: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self = NULL; + JitOptSymbol *owner; + JitOptSymbol *attr; + JitOptSymbol *self; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)this_instr->operand; + PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + attr = sym_new_not_null(ctx); self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - /* _INSTRUMENTED_CALL is not a viable micro-op for tier 2 */ + case _MAYBE_EXPAND_METHOD: { + JitOptSymbol **args; + JitOptSymbol *self_or_null; + JitOptSymbol *callable; + JitOptSymbol *func; + JitOptSymbol *maybe_self; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-oparg]; + (void)callable; + (void)self_or_null; + (void)args; + func = sym_new_not_null(ctx); + maybe_self = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = maybe_self; + break; + } - /* _CALL is not a viable micro-op for tier 2 */ + /* _DO_CALL is not a viable micro-op for tier 2 */ - case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - _Py_UopsSymbol *null; - _Py_UopsSymbol *callable; - null = stack_pointer[-1 - oparg]; + /* _MONITOR_CALL is not a viable micro-op for tier 2 */ + + case _PY_FRAME_GENERAL: { + JitOptSymbol *self_or_null; + JitOptSymbol *callable; + _Py_UOpsAbstractFrame *new_frame; + self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - if (!sym_set_null(null)) { - goto hit_bottom; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + (void)(self_or_null); + (void)(callable); + PyCodeObject *co = NULL; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; } - if (!sym_set_type(callable, &PyMethod_Type)) { - goto hit_bottom; + new_frame = frame_new(ctx, co, 0, NULL, 0); + stack_pointer[0] = (JitOptSymbol *)new_frame; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_FUNCTION_VERSION: { + JitOptSymbol *self_or_null; + JitOptSymbol *callable; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = (uint32_t)this_instr->operand0; + (void)self_or_null; + if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { + assert(PyFunction_Check(sym_get_const(callable))); + REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + this_instr->operand1 = (uintptr_t)sym_get_const(callable); } + sym_set_type(callable, &PyFunction_Type); + break; + } + + case _CHECK_FUNCTION_VERSION_INLINE: { + break; + } + + case _CHECK_METHOD_VERSION: { + break; + } + + case _EXPAND_METHOD: { + JitOptSymbol **method; + JitOptSymbol **self; + method = &stack_pointer[-2 - oparg]; + self = &stack_pointer[-1 - oparg]; + method[0] = sym_new_not_null(ctx); + self[0] = sym_new_not_null(ctx); + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE: { + break; + } + + case _CALL_NON_PY_GENERAL: { + JitOptSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { + JitOptSymbol *null; + JitOptSymbol *callable; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + sym_set_null(null); + sym_set_type(callable, &PyMethod_Type); break; } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - _Py_UopsSymbol *callable; - _Py_UopsSymbol *func; - _Py_UopsSymbol *self; + JitOptSymbol *callable; + JitOptSymbol *func; + JitOptSymbol *self; callable = stack_pointer[-2 - oparg]; (void)callable; - OUT_OF_SPACE_IF_NULL(func = sym_new_not_null(ctx)); - OUT_OF_SPACE_IF_NULL(self = sym_new_not_null(ctx)); + func = sym_new_not_null(ctx); + self = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = func; stack_pointer[-1 - oparg] = self; break; } case _CHECK_PEP_523: { + /* Setting the eval frame function invalidates + * all executors, so no need to check dynamically */ + if (_PyInterpreterState_GET()->eval_frame == NULL) { + REPLACE_OP(this_instr, _NOP, 0 ,0); + } break; } case _CHECK_FUNCTION_EXACT_ARGS: { - _Py_UopsSymbol *self_or_null; - _Py_UopsSymbol *callable; + JitOptSymbol *self_or_null; + JitOptSymbol *callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)this_instr->operand; - if (!sym_set_type(callable, &PyFunction_Type)) { - goto hit_bottom; + assert(sym_matches_type(callable, &PyFunction_Type)); + if (sym_is_const(callable)) { + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + PyFunctionObject *func = (PyFunctionObject *)sym_get_const(callable); + PyCodeObject *co = (PyCodeObject *)func->func_code; + if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { + REPLACE_OP(this_instr, _NOP, 0 ,0); + } + } } (void)self_or_null; - (void)func_version; break; } case _CHECK_STACK_SPACE: { + assert(corresponding_check_stack == NULL); + corresponding_check_stack = this_instr; break; } case _INIT_CALL_PY_EXACT_ARGS: { - _Py_UopsSymbol **args; - _Py_UopsSymbol *self_or_null; - _Py_UopsSymbol *callable; + JitOptSymbol **args; + JitOptSymbol *self_or_null; + JitOptSymbol *callable; _Py_UOpsAbstractFrame *new_frame; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int argcount = oparg; (void)callable; - PyFunctionObject *func = (PyFunctionObject *)(this_instr + 2)->operand; - if (func == NULL) { - goto error; + PyCodeObject *co = NULL; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; } - PyCodeObject *co = (PyCodeObject *)func->func_code; assert(self_or_null != NULL); assert(args != NULL); if (sym_is_not_null(self_or_null)) { @@ -1535,19 +1942,14 @@ args--; argcount++; } - _Py_UopsSymbol **localsplus_start = ctx->n_consumed; - int n_locals_already_filled = 0; - // Can determine statically, so we interleave the new locals - // and make the current stack the new locals. - // This also sets up for true call inlining. if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - localsplus_start = args; - n_locals_already_filled = argcount; + new_frame = frame_new(ctx, co, 0, args, argcount); + } else { + new_frame = frame_new(ctx, co, 0, NULL, 0); } - OUT_OF_SPACE_IF_NULL(new_frame = - frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0)); - stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; - stack_pointer += -1 - oparg; + stack_pointer[0] = (JitOptSymbol *)new_frame; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -1555,233 +1957,481 @@ _Py_UOpsAbstractFrame *new_frame; new_frame = (_Py_UOpsAbstractFrame *)stack_pointer[-1]; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; ctx->frame = new_frame; ctx->curr_frame_depth++; stack_pointer = new_frame->stack_pointer; + co = get_code(this_instr); + if (co == NULL) { + // should be about to _EXIT_TRACE anyway + ctx->done = true; + break; + } + /* Stack space handling */ + int framesize = co->co_framesize; + assert(framesize > 0); + curr_space += framesize; + if (curr_space < 0 || curr_space > INT32_MAX) { + // won't fit in signed 32-bit int + ctx->done = true; + break; + } + max_space = curr_space > max_space ? curr_space : max_space; + if (first_valid_check_stack == NULL) { + first_valid_check_stack = corresponding_check_stack; + } + else { + if (corresponding_check_stack) { + // delete all but the first valid _CHECK_STACK_SPACE + corresponding_check_stack->opcode = _NOP; + } + } + corresponding_check_stack = NULL; break; } - /* _CALL_PY_WITH_DEFAULTS is not a viable micro-op for tier 2 */ - case _CALL_TYPE_1: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + JitOptSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_STR_1: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + JitOptSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_TUPLE_1: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + JitOptSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } - /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 */ + case _CHECK_AND_ALLOCATE_OBJECT: { + JitOptSymbol **args; + JitOptSymbol *null; + JitOptSymbol *callable; + JitOptSymbol *self; + JitOptSymbol *init; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-oparg]; + uint32_t type_version = (uint32_t)this_instr->operand0; + (void)type_version; + (void)callable; + (void)null; + (void)args; + self = sym_new_not_null(ctx); + init = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = self; + stack_pointer[-1 - oparg] = init; + break; + } + + case _CREATE_INIT_FRAME: { + JitOptSymbol **args; + JitOptSymbol *init; + JitOptSymbol *self; + _Py_UOpsAbstractFrame *init_frame; + args = &stack_pointer[-oparg]; + init = stack_pointer[-1 - oparg]; + self = stack_pointer[-2 - oparg]; + (void)self; + (void)init; + (void)args; + init_frame = NULL; + ctx->done = true; + stack_pointer[-2 - oparg] = (JitOptSymbol *)init_frame; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _EXIT_INIT_CHECK: { stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_BUILTIN_CLASS: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_BUILTIN_O: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_BUILTIN_FAST: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_LEN: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_ISINSTANCE: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_LIST_APPEND: { + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_METHOD_DESCRIPTOR_O: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_METHOD_DESCRIPTOR_NOARGS: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _CALL_METHOD_DESCRIPTOR_FAST: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 */ - /* _CALL_KW is not a viable micro-op for tier 2 */ + case _MAYBE_EXPAND_METHOD_KW: { + JitOptSymbol **func; + JitOptSymbol **maybe_self; + JitOptSymbol **args; + JitOptSymbol *kwnames_out; + func = &stack_pointer[-3 - oparg]; + maybe_self = &stack_pointer[-2 - oparg]; + args = &stack_pointer[-1 - oparg]; + func[0] = sym_new_not_null(ctx); + maybe_self[0] = sym_new_not_null(ctx); + for (int _i = oparg; --_i >= 0;) { + args[_i] = sym_new_not_null(ctx); + } + kwnames_out = sym_new_not_null(ctx); + stack_pointer[-1] = kwnames_out; + break; + } + + /* _DO_CALL_KW is not a viable micro-op for tier 2 */ + + case _PY_FRAME_KW: { + JitOptSymbol *kwnames; + JitOptSymbol **args; + JitOptSymbol *self_or_null; + JitOptSymbol *callable; + _Py_UOpsAbstractFrame *new_frame; + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + (void)callable; + (void)self_or_null; + (void)args; + (void)kwnames; + new_frame = NULL; + ctx->done = true; + stack_pointer[-3 - oparg] = (JitOptSymbol *)new_frame; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_FUNCTION_VERSION_KW: { + break; + } + + case _CHECK_METHOD_VERSION_KW: { + break; + } + + case _EXPAND_METHOD_KW: { + JitOptSymbol **method; + JitOptSymbol **self; + method = &stack_pointer[-3 - oparg]; + self = &stack_pointer[-2 - oparg]; + method[0] = sym_new_not_null(ctx); + self[0] = sym_new_not_null(ctx); + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE_KW: { + break; + } + + case _CALL_KW_NON_PY: { + JitOptSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ - /* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + case _MAKE_CALLARGS_A_TUPLE: { + JitOptSymbol *tuple; + JitOptSymbol *kwargs_out; + tuple = sym_new_not_null(ctx); + kwargs_out = sym_new_not_null(ctx); + stack_pointer[-2] = tuple; + stack_pointer[-1] = kwargs_out; + break; + } + + /* _DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ case _MAKE_FUNCTION: { - _Py_UopsSymbol *func; - func = sym_new_unknown(ctx); - if (func == NULL) goto out_of_space; + JitOptSymbol *func; + func = sym_new_not_null(ctx); stack_pointer[-1] = func; break; } case _SET_FUNCTION_ATTRIBUTE: { - _Py_UopsSymbol *func; - func = sym_new_unknown(ctx); - if (func == NULL) goto out_of_space; - stack_pointer[-2] = func; + JitOptSymbol *func_out; + func_out = sym_new_not_null(ctx); + stack_pointer[-2] = func_out; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _RETURN_GENERATOR: { + JitOptSymbol *res; + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); + stack_pointer = ctx->frame->stack_pointer; + res = sym_new_unknown(ctx); + /* Stack space handling */ + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + co = get_code(this_instr); + if (co == NULL) { + // might be impossible, but bailing is still safe + ctx->done = true; + } break; } case _BUILD_SLICE: { - _Py_UopsSymbol *slice; - slice = sym_new_unknown(ctx); - if (slice == NULL) goto out_of_space; - stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; - stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + JitOptSymbol *slice; + slice = sym_new_not_null(ctx); + stack_pointer[-oparg] = slice; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); break; } case _CONVERT_VALUE: { - _Py_UopsSymbol *result; - result = sym_new_unknown(ctx); - if (result == NULL) goto out_of_space; + JitOptSymbol *result; + result = sym_new_not_null(ctx); stack_pointer[-1] = result; break; } case _FORMAT_SIMPLE: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _FORMAT_WITH_SPEC: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *res; + res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _COPY: { - _Py_UopsSymbol *bottom; - _Py_UopsSymbol *top; + JitOptSymbol *bottom; + JitOptSymbol *top; bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = bottom; stack_pointer[0] = top; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _BINARY_OP: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + JitOptSymbol *right; + JitOptSymbol *left; + JitOptSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + bool lhs_int = sym_matches_type(left, &PyLong_Type); + bool rhs_int = sym_matches_type(right, &PyLong_Type); + bool lhs_float = sym_matches_type(left, &PyFloat_Type); + bool rhs_float = sym_matches_type(right, &PyFloat_Type); + if (!((lhs_int || lhs_float) && (rhs_int || rhs_float))) { + // There's something other than an int or float involved: + res = sym_new_unknown(ctx); + } + else { + if (oparg == NB_POWER || oparg == NB_INPLACE_POWER) { + // This one's fun... the *type* of the result depends on the + // *values* being exponentiated. However, exponents with one + // constant part are reasonably common, so it's probably worth + // trying to infer some simple cases: + // - A: 1 ** 1 -> 1 (int ** int -> int) + // - B: 1 ** -1 -> 1.0 (int ** int -> float) + // - C: 1.0 ** 1 -> 1.0 (float ** int -> float) + // - D: 1 ** 1.0 -> 1.0 (int ** float -> float) + // - E: -1 ** 0.5 ~> 1j (int ** float -> complex) + // - F: 1.0 ** 1.0 -> 1.0 (float ** float -> float) + // - G: -1.0 ** 0.5 ~> 1j (float ** float -> complex) + if (rhs_float) { + // Case D, E, F, or G... can't know without the sign of the LHS + // or whether the RHS is whole, which isn't worth the effort: + res = sym_new_unknown(ctx); + } + else { + if (lhs_float) { + // Case C: + res = sym_new_type(ctx, &PyFloat_Type); + } + else { + if (!sym_is_const(right)) { + // Case A or B... can't know without the sign of the RHS: + res = sym_new_unknown(ctx); + } + else { + if (_PyLong_IsNegative((PyLongObject *)sym_get_const(right))) { + // Case B: + res = sym_new_type(ctx, &PyFloat_Type); + } + else { + // Case A: + res = sym_new_type(ctx, &PyLong_Type); + } + } + } + } + } + else { + if (oparg == NB_TRUE_DIVIDE || oparg == NB_INPLACE_TRUE_DIVIDE) { + res = sym_new_type(ctx, &PyFloat_Type); + } + else { + if (lhs_int && rhs_int) { + res = sym_new_type(ctx, &PyLong_Type); + } + else { + res = sym_new_type(ctx, &PyFloat_Type); + } + } + } + } stack_pointer[-2] = res; stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _SWAP: { - _Py_UopsSymbol *top; - _Py_UopsSymbol *bottom; - top = stack_pointer[-1]; - bottom = stack_pointer[-2 - (oparg-2)]; - stack_pointer[-2 - (oparg-2)] = top; - stack_pointer[-1] = bottom; + JitOptSymbol *top_in; + JitOptSymbol *bottom_in; + JitOptSymbol *top_out; + JitOptSymbol *bottom_out; + top_in = stack_pointer[-1]; + bottom_in = stack_pointer[-2 - (oparg-2)]; + bottom_out = bottom_in; + top_out = top_in; + stack_pointer[-2 - (oparg-2)] = top_out; + stack_pointer[-1] = bottom_out; break; } + /* _INSTRUMENTED_LINE is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 */ /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 */ - /* _INSTRUMENTED_JUMP_BACKWARD is not a viable micro-op for tier 2 */ + /* _MONITOR_JUMP_BACKWARD is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_NOT_TAKEN is not a viable micro-op for tier 2 */ /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ @@ -1792,26 +2442,91 @@ /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ case _GUARD_IS_TRUE_POP: { + JitOptSymbol *flag; + flag = stack_pointer[-1]; + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + eliminate_pop_guard(this_instr, value != Py_True); + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_IS_FALSE_POP: { + JitOptSymbol *flag; + flag = stack_pointer[-1]; + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + eliminate_pop_guard(this_instr, value != Py_False); + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } case _GUARD_IS_NONE_POP: { - stack_pointer += -1; + JitOptSymbol *flag; + flag = stack_pointer[-1]; + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + eliminate_pop_guard(this_instr, !Py_IsNone(value)); + } + else { + if (sym_has_type(flag)) { + assert(!sym_matches_type(flag, &_PyNone_Type)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + eliminate_pop_guard(this_instr, true); + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } break; } case _GUARD_IS_NOT_NONE_POP: { - stack_pointer += -1; + JitOptSymbol *flag; + flag = stack_pointer[-1]; + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + eliminate_pop_guard(this_instr, Py_IsNone(value)); + } + else { + if (sym_has_type(flag)) { + assert(!sym_matches_type(flag, &_PyNone_Type)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + eliminate_pop_guard(this_instr, false); + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } break; } case _JUMP_TO_TOP: { + ctx->done = true; break; } @@ -1819,11 +2534,23 @@ break; } + case _CHECK_STACK_SPACE_OPERAND: { + uint32_t framesize = (uint32_t)this_instr->operand0; + (void)framesize; + /* We should never see _CHECK_STACK_SPACE_OPERANDs. + * They are only created at the end of this pass. */ + Py_UNREACHABLE(); + break; + } + case _SAVE_RETURN_OFFSET: { break; } case _EXIT_TRACE: { + PyObject *exit_p = (PyObject *)this_instr->operand0; + (void)exit_p; + ctx->done = true; break; } @@ -1832,73 +2559,70 @@ } case _LOAD_CONST_INLINE: { - _Py_UopsSymbol *value; - PyObject *ptr = (PyObject *)this_instr->operand; - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + JitOptSymbol *value; + PyObject *ptr = (PyObject *)this_instr->operand0; + value = sym_new_const(ctx, ptr); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_CONST_INLINE_BORROW: { - _Py_UopsSymbol *value; - PyObject *ptr = (PyObject *)this_instr->operand; - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + JitOptSymbol *value; + PyObject *ptr = (PyObject *)this_instr->operand0; + value = sym_new_const(ctx, ptr); stack_pointer[0] = value; stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _POP_TOP_LOAD_CONST_INLINE_BORROW: { - _Py_UopsSymbol *value; - value = sym_new_unknown(ctx); - if (value == NULL) goto out_of_space; + JitOptSymbol *value; + value = sym_new_not_null(ctx); stack_pointer[-1] = value; break; } - case _LOAD_CONST_INLINE_WITH_NULL: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *null; - PyObject *ptr = (PyObject *)this_instr->operand; - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); - OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); - stack_pointer[0] = value; - stack_pointer[1] = null; - stack_pointer += 2; + case _CHECK_FUNCTION: { break; } - case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *null; - PyObject *ptr = (PyObject *)this_instr->operand; - OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); - OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); - stack_pointer[0] = value; - stack_pointer[1] = null; - stack_pointer += 2; + case _LOAD_GLOBAL_MODULE: { + JitOptSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _CHECK_GLOBALS: { + case _LOAD_GLOBAL_BUILTINS: { + JitOptSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } - case _CHECK_BUILTINS: { + case _LOAD_ATTR_MODULE: { + JitOptSymbol *attr; + attr = sym_new_not_null(ctx); + stack_pointer[-1] = attr; break; } - case _INTERNAL_INCREMENT_OPT_COUNTER: { - stack_pointer += -1; + case _DYNAMIC_EXIT: { break; } - case _COLD_EXIT: { + case _START_EXECUTOR: { break; } - case _START_EXECUTOR: { + case _MAKE_WARM: { break; } @@ -1910,3 +2634,17 @@ break; } + case _DEOPT: { + break; + } + + case _ERROR_POP_N: { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _TIER2_RESUME_CHECK: { + break; + } + diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 5c3ec2b5ed1a4c..dcde8e7ce81577 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -1,9 +1,10 @@ +#ifdef _Py_TIER2 #include "Python.h" -#include "cpython/optimizer.h" #include "pycore_code.h" #include "pycore_frame.h" +#include "pycore_long.h" #include "pycore_optimizer.h" #include @@ -27,10 +28,6 @@ - Bottom: IS_NULL and NOT_NULL flags set, type and const_val NULL. */ -// Flags for below. -#define IS_NULL 1 << 0 -#define NOT_NULL 1 << 1 - #ifdef Py_DEBUG static inline int get_lltrace(void) { char *uop_debug = Py_GETENV("PYTHON_OPT_DEBUG"); @@ -46,208 +43,477 @@ static inline int get_lltrace(void) { #define DPRINTF(level, ...) #endif -static _Py_UopsSymbol * -sym_new(_Py_UOpsContext *ctx) + +static JitOptSymbol NO_SPACE_SYMBOL = { + .tag = JIT_SYM_BOTTOM_TAG +}; + +JitOptSymbol * +out_of_space(JitOptContext *ctx) { - _Py_UopsSymbol *self = &ctx->t_arena.arena[ctx->t_arena.ty_curr_number]; + ctx->done = true; + ctx->out_of_space = true; + return &NO_SPACE_SYMBOL; +} + +static JitOptSymbol * +sym_new(JitOptContext *ctx) +{ + JitOptSymbol *self = &ctx->t_arena.arena[ctx->t_arena.ty_curr_number]; if (ctx->t_arena.ty_curr_number >= ctx->t_arena.ty_max_number) { OPT_STAT_INC(optimizer_failure_reason_no_memory); DPRINTF(1, "out of space for symbolic expression type\n"); return NULL; } ctx->t_arena.ty_curr_number++; - self->flags = 0; - self->typ = NULL; - self->const_val = NULL; - + self->tag = JIT_SYM_UNKNOWN_TAG; return self; } static inline void -sym_set_flag(_Py_UopsSymbol *sym, int flag) +sym_set_bottom(JitOptContext *ctx, JitOptSymbol *sym) { - sym->flags |= flag; -} - -static inline void -sym_set_bottom(_Py_UopsSymbol *sym) -{ - sym_set_flag(sym, IS_NULL | NOT_NULL); - sym->typ = NULL; - Py_CLEAR(sym->const_val); + sym->tag = JIT_SYM_BOTTOM_TAG; + ctx->done = true; + ctx->contradiction = true; } bool -_Py_uop_sym_is_bottom(_Py_UopsSymbol *sym) +_Py_uop_sym_is_bottom(JitOptSymbol *sym) { - if ((sym->flags & IS_NULL) && (sym->flags & NOT_NULL)) { - assert(sym->flags == (IS_NULL | NOT_NULL)); - assert(sym->typ == NULL); - assert(sym->const_val == NULL); - return true; - } - return false; + return sym->tag == JIT_SYM_BOTTOM_TAG; } bool -_Py_uop_sym_is_not_null(_Py_UopsSymbol *sym) -{ - return sym->flags == NOT_NULL; +_Py_uop_sym_is_not_null(JitOptSymbol *sym) { + return sym->tag == JIT_SYM_NON_NULL_TAG || sym->tag > JIT_SYM_BOTTOM_TAG; } bool -_Py_uop_sym_is_null(_Py_UopsSymbol *sym) +_Py_uop_sym_is_const(JitOptSymbol *sym) { - return sym->flags == IS_NULL; + return sym->tag == JIT_SYM_KNOWN_VALUE_TAG; } bool -_Py_uop_sym_is_const(_Py_UopsSymbol *sym) +_Py_uop_sym_is_null(JitOptSymbol *sym) { - return sym->const_val != NULL; + return sym->tag == JIT_SYM_NULL_TAG; } + PyObject * -_Py_uop_sym_get_const(_Py_UopsSymbol *sym) +_Py_uop_sym_get_const(JitOptSymbol *sym) { - return sym->const_val; + if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { + return sym->value.value; + } + return NULL; } -bool -_Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *typ) +void +_Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeObject *typ) { - assert(typ != NULL && PyType_Check(typ)); - if (sym->flags & IS_NULL) { - sym_set_bottom(sym); - return false; - } - if (sym->typ != NULL) { - if (sym->typ != typ) { - sym_set_bottom(sym); - return false; - } - } - else { - sym_set_flag(sym, NOT_NULL); - sym->typ = typ; + JitSymType tag = sym->tag; + switch(tag) { + case JIT_SYM_NULL_TAG: + sym_set_bottom(ctx, sym); + return; + case JIT_SYM_KNOWN_CLASS_TAG: + if (sym->cls.type != typ) { + sym_set_bottom(ctx, sym); + } + return; + case JIT_SYM_TYPE_VERSION_TAG: + if (sym->version.version == typ->tp_version_tag) { + sym->tag = JIT_SYM_KNOWN_CLASS_TAG; + sym->cls.type = typ; + sym->cls.version = typ->tp_version_tag; + } + else { + sym_set_bottom(ctx, sym); + } + return; + case JIT_SYM_KNOWN_VALUE_TAG: + if (Py_TYPE(sym->value.value) != typ) { + Py_CLEAR(sym->value.value); + sym_set_bottom(ctx, sym); + } + return; + case JIT_SYM_TUPLE_TAG: + if (typ != &PyTuple_Type) { + sym_set_bottom(ctx, sym); + } + return; + case JIT_SYM_BOTTOM_TAG: + return; + case JIT_SYM_NON_NULL_TAG: + case JIT_SYM_UNKNOWN_TAG: + sym->tag = JIT_SYM_KNOWN_CLASS_TAG; + sym->cls.version = 0; + sym->cls.type = typ; + return; } - return true; } bool -_Py_uop_sym_set_const(_Py_UopsSymbol *sym, PyObject *const_val) +_Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int version) { - assert(const_val != NULL); - if (sym->flags & IS_NULL) { - sym_set_bottom(sym); - return false; - } - PyTypeObject *typ = Py_TYPE(const_val); - if (sym->typ != NULL && sym->typ != typ) { - sym_set_bottom(sym); - return false; - } - if (sym->const_val != NULL) { - if (sym->const_val != const_val) { - // TODO: What if they're equal? - sym_set_bottom(sym); + JitSymType tag = sym->tag; + switch(tag) { + case JIT_SYM_NULL_TAG: + sym_set_bottom(ctx, sym); return false; - } - } - else { - sym_set_flag(sym, NOT_NULL); - sym->typ = typ; - sym->const_val = Py_NewRef(const_val); + case JIT_SYM_KNOWN_CLASS_TAG: + if (sym->cls.type->tp_version_tag != version) { + sym_set_bottom(ctx, sym); + return false; + } + else { + sym->cls.version = version; + return true; + } + case JIT_SYM_KNOWN_VALUE_TAG: + Py_CLEAR(sym->value.value); + sym_set_bottom(ctx, sym); + return false; + case JIT_SYM_TUPLE_TAG: + sym_set_bottom(ctx, sym); + return false; + case JIT_SYM_TYPE_VERSION_TAG: + if (sym->version.version == version) { + return true; + } + sym_set_bottom(ctx, sym); + return false; + case JIT_SYM_BOTTOM_TAG: + return false; + case JIT_SYM_NON_NULL_TAG: + case JIT_SYM_UNKNOWN_TAG: + sym->tag = JIT_SYM_TYPE_VERSION_TAG; + sym->version.version = version; + return true; } - return true; + Py_UNREACHABLE(); } +static void make_const(JitOptSymbol *sym, PyObject *val) +{ + sym->tag = JIT_SYM_KNOWN_VALUE_TAG; + sym->value.value = Py_NewRef(val); +} -bool -_Py_uop_sym_set_null(_Py_UopsSymbol *sym) +void +_Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val) { - sym_set_flag(sym, IS_NULL); - return !_Py_uop_sym_is_bottom(sym); + JitSymType tag = sym->tag; + switch(tag) { + case JIT_SYM_NULL_TAG: + sym_set_bottom(ctx, sym); + return; + case JIT_SYM_KNOWN_CLASS_TAG: + if (sym->cls.type != Py_TYPE(const_val)) { + sym_set_bottom(ctx, sym); + return; + } + make_const(sym, const_val); + return; + case JIT_SYM_KNOWN_VALUE_TAG: + if (sym->value.value != const_val) { + Py_CLEAR(sym->value.value); + sym_set_bottom(ctx, sym); + } + return; + case JIT_SYM_TUPLE_TAG: + sym_set_bottom(ctx, sym); + return; + case JIT_SYM_TYPE_VERSION_TAG: + if (sym->version.version != Py_TYPE(const_val)->tp_version_tag) { + sym_set_bottom(ctx, sym); + return; + } + make_const(sym, const_val); + return; + case JIT_SYM_BOTTOM_TAG: + return; + case JIT_SYM_NON_NULL_TAG: + case JIT_SYM_UNKNOWN_TAG: + make_const(sym, const_val); + return; + } } -bool -_Py_uop_sym_set_non_null(_Py_UopsSymbol *sym) +void +_Py_uop_sym_set_null(JitOptContext *ctx, JitOptSymbol *sym) { - sym_set_flag(sym, NOT_NULL); - return !_Py_uop_sym_is_bottom(sym); + if (sym->tag == JIT_SYM_UNKNOWN_TAG) { + sym->tag = JIT_SYM_NULL_TAG; + } + else if (sym->tag > JIT_SYM_NULL_TAG) { + sym_set_bottom(ctx, sym); + } +} + +void +_Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym) +{ + if (sym->tag == JIT_SYM_UNKNOWN_TAG) { + sym->tag = JIT_SYM_NON_NULL_TAG; + } + else if (sym->tag == JIT_SYM_NULL_TAG) { + sym_set_bottom(ctx, sym); + } } -_Py_UopsSymbol * -_Py_uop_sym_new_unknown(_Py_UOpsContext *ctx) +JitOptSymbol * +_Py_uop_sym_new_unknown(JitOptContext *ctx) { - return sym_new(ctx); + JitOptSymbol *res = sym_new(ctx); + if (res == NULL) { + return out_of_space(ctx); + } + return res; } -_Py_UopsSymbol * -_Py_uop_sym_new_not_null(_Py_UOpsContext *ctx) +JitOptSymbol * +_Py_uop_sym_new_not_null(JitOptContext *ctx) { - _Py_UopsSymbol *res = _Py_uop_sym_new_unknown(ctx); + JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return NULL; + return out_of_space(ctx); } - sym_set_flag(res, NOT_NULL); + res->tag = JIT_SYM_NON_NULL_TAG; return res; } -_Py_UopsSymbol * -_Py_uop_sym_new_type(_Py_UOpsContext *ctx, PyTypeObject *typ) +JitOptSymbol * +_Py_uop_sym_new_type(JitOptContext *ctx, PyTypeObject *typ) { - _Py_UopsSymbol *res = sym_new(ctx); + JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return NULL; + return out_of_space(ctx); } - _Py_uop_sym_set_type(res, typ); + _Py_uop_sym_set_type(ctx, res, typ); return res; } // Adds a new reference to const_val, owned by the symbol. -_Py_UopsSymbol * -_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val) +JitOptSymbol * +_Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val) { assert(const_val != NULL); - _Py_UopsSymbol *res = sym_new(ctx); + JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return NULL; + return out_of_space(ctx); } - _Py_uop_sym_set_const(res, const_val); + _Py_uop_sym_set_const(ctx, res, const_val); return res; } -_Py_UopsSymbol * -_Py_uop_sym_new_null(_Py_UOpsContext *ctx) +JitOptSymbol * +_Py_uop_sym_new_null(JitOptContext *ctx) { - _Py_UopsSymbol *null_sym = _Py_uop_sym_new_unknown(ctx); + JitOptSymbol *null_sym = sym_new(ctx); if (null_sym == NULL) { - return NULL; + return out_of_space(ctx); } - _Py_uop_sym_set_null(null_sym); + _Py_uop_sym_set_null(ctx, null_sym); return null_sym; } +PyTypeObject * +_Py_uop_sym_get_type(JitOptSymbol *sym) +{ + JitSymType tag = sym->tag; + switch(tag) { + case JIT_SYM_NULL_TAG: + case JIT_SYM_TYPE_VERSION_TAG: + case JIT_SYM_BOTTOM_TAG: + case JIT_SYM_NON_NULL_TAG: + case JIT_SYM_UNKNOWN_TAG: + return NULL; + case JIT_SYM_KNOWN_CLASS_TAG: + return sym->cls.type; + case JIT_SYM_KNOWN_VALUE_TAG: + return Py_TYPE(sym->value.value); + case JIT_SYM_TUPLE_TAG: + return &PyTuple_Type; + } + Py_UNREACHABLE(); +} + +unsigned int +_Py_uop_sym_get_type_version(JitOptSymbol *sym) +{ + JitSymType tag = sym->tag; + switch(tag) { + case JIT_SYM_NULL_TAG: + case JIT_SYM_BOTTOM_TAG: + case JIT_SYM_NON_NULL_TAG: + case JIT_SYM_UNKNOWN_TAG: + return 0; + case JIT_SYM_TYPE_VERSION_TAG: + return sym->version.version; + case JIT_SYM_KNOWN_CLASS_TAG: + return sym->cls.version; + case JIT_SYM_KNOWN_VALUE_TAG: + return Py_TYPE(sym->value.value)->tp_version_tag; + case JIT_SYM_TUPLE_TAG: + return PyTuple_Type.tp_version_tag; + } + Py_UNREACHABLE(); +} + bool -_Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ) +_Py_uop_sym_has_type(JitOptSymbol *sym) +{ + JitSymType tag = sym->tag; + switch(tag) { + case JIT_SYM_NULL_TAG: + case JIT_SYM_TYPE_VERSION_TAG: + case JIT_SYM_BOTTOM_TAG: + case JIT_SYM_NON_NULL_TAG: + case JIT_SYM_UNKNOWN_TAG: + return false; + case JIT_SYM_KNOWN_CLASS_TAG: + case JIT_SYM_KNOWN_VALUE_TAG: + case JIT_SYM_TUPLE_TAG: + return true; + } + Py_UNREACHABLE(); +} + +bool +_Py_uop_sym_matches_type(JitOptSymbol *sym, PyTypeObject *typ) { assert(typ != NULL && PyType_Check(typ)); - if (_Py_uop_sym_is_bottom(sym)) { - return false; + return _Py_uop_sym_get_type(sym) == typ; +} + +bool +_Py_uop_sym_matches_type_version(JitOptSymbol *sym, unsigned int version) +{ + return _Py_uop_sym_get_type_version(sym) == version; +} + +int +_Py_uop_sym_truthiness(JitOptSymbol *sym) +{ + switch(sym->tag) { + case JIT_SYM_NULL_TAG: + case JIT_SYM_TYPE_VERSION_TAG: + case JIT_SYM_BOTTOM_TAG: + case JIT_SYM_NON_NULL_TAG: + case JIT_SYM_UNKNOWN_TAG: + return -1; + case JIT_SYM_KNOWN_CLASS_TAG: + /* TODO : + * Instances of some classes are always + * true. We should return 1 in those cases */ + return -1; + case JIT_SYM_KNOWN_VALUE_TAG: + break; + case JIT_SYM_TUPLE_TAG: + return sym->tuple.length != 0; + } + PyObject *value = sym->value.value; + /* Only handle a few known safe types */ + if (value == Py_None) { + return 0; } - return sym->typ == typ; + PyTypeObject *tp = Py_TYPE(value); + if (tp == &PyLong_Type) { + return !_PyLong_IsZero((PyLongObject *)value); + } + if (tp == &PyUnicode_Type) { + return value != &_Py_STR(empty); + } + if (tp == &PyBool_Type) { + return value == Py_True; + } + return -1; +} + +static JitOptSymbol * +allocation_base(JitOptContext *ctx) +{ + return ctx->t_arena.arena; +} + +JitOptSymbol * +_Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptSymbol **args) +{ + JitOptSymbol *res = sym_new(ctx); + if (res == NULL) { + return out_of_space(ctx); + } + if (size > MAX_SYMBOLIC_TUPLE_SIZE) { + res->tag = JIT_SYM_KNOWN_CLASS_TAG; + res->cls.type = &PyTuple_Type; + } + else { + res->tag = JIT_SYM_TUPLE_TAG; + res->tuple.length = size; + for (int i = 0; i < size; i++) { + res->tuple.items[i] = (uint16_t)(args[i] - allocation_base(ctx)); + } + } + return res; +} + +JitOptSymbol * +_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol *sym, int item) +{ + assert(item >= 0); + if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { + PyObject *tuple = sym->value.value; + if (PyTuple_CheckExact(tuple) && item < PyTuple_GET_SIZE(tuple)) { + return _Py_uop_sym_new_const(ctx, PyTuple_GET_ITEM(tuple, item)); + } + } + else if (sym->tag == JIT_SYM_TUPLE_TAG && item < sym->tuple.length) { + return allocation_base(ctx) + sym->tuple.items[item]; + } + return _Py_uop_sym_new_unknown(ctx); +} + +int +_Py_uop_sym_tuple_length(JitOptSymbol *sym) +{ + if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { + PyObject *tuple = sym->value.value; + if (PyTuple_CheckExact(tuple)) { + return PyTuple_GET_SIZE(tuple); + } + } + else if (sym->tag == JIT_SYM_TUPLE_TAG) { + return sym->tuple.length; + } + return -1; +} + +// Return true if known to be immortal. +bool +_Py_uop_sym_is_immortal(JitOptSymbol *sym) +{ + if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { + return _Py_IsImmortal(sym->value.value); + } + if (sym->tag == JIT_SYM_KNOWN_CLASS_TAG) { + return sym->cls.type == &PyBool_Type; + } + return false; } // 0 on success, -1 on error. _Py_UOpsAbstractFrame * _Py_uop_frame_new( - _Py_UOpsContext *ctx, + JitOptContext *ctx, PyCodeObject *co, - _Py_UopsSymbol **localsplus_start, - int n_locals_already_filled, - int curr_stackentries) + int curr_stackentries, + JitOptSymbol **args, + int arg_len) { assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; @@ -255,31 +521,30 @@ _Py_uop_frame_new( frame->stack_len = co->co_stacksize; frame->locals_len = co->co_nlocalsplus; - frame->locals = localsplus_start; + frame->locals = ctx->n_consumed; frame->stack = frame->locals + co->co_nlocalsplus; frame->stack_pointer = frame->stack + curr_stackentries; - ctx->n_consumed = localsplus_start + (co->co_nlocalsplus + co->co_stacksize); + ctx->n_consumed = ctx->n_consumed + (co->co_nlocalsplus + co->co_stacksize); if (ctx->n_consumed >= ctx->limit) { + ctx->done = true; + ctx->out_of_space = true; return NULL; } - // Initialize with the initial state of all local variables - for (int i = n_locals_already_filled; i < co->co_nlocalsplus; i++) { - _Py_UopsSymbol *local = _Py_uop_sym_new_unknown(ctx); - if (local == NULL) { - return NULL; - } + for (int i = 0; i < arg_len; i++) { + frame->locals[i] = args[i]; + } + + for (int i = arg_len; i < co->co_nlocalsplus; i++) { + JitOptSymbol *local = _Py_uop_sym_new_unknown(ctx); frame->locals[i] = local; } // Initialize the stack as well for (int i = 0; i < curr_stackentries; i++) { - _Py_UopsSymbol *stackvar = _Py_uop_sym_new_unknown(ctx); - if (stackvar == NULL) { - return NULL; - } + JitOptSymbol *stackvar = _Py_uop_sym_new_unknown(ctx); frame->stack[i] = stackvar; } @@ -287,7 +552,7 @@ _Py_uop_frame_new( } void -_Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx) +_Py_uop_abstractcontext_fini(JitOptContext *ctx) { if (ctx == NULL) { return; @@ -295,13 +560,17 @@ _Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx) ctx->curr_frame_depth = 0; int tys = ctx->t_arena.ty_curr_number; for (int i = 0; i < tys; i++) { - Py_CLEAR(ctx->t_arena.arena[i].const_val); + JitOptSymbol *sym = &ctx->t_arena.arena[i]; + if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { + Py_CLEAR(sym->value.value); + } } } -int -_Py_uop_abstractcontext_init(_Py_UOpsContext *ctx) +void +_Py_uop_abstractcontext_init(JitOptContext *ctx) { + static_assert(sizeof(JitOptSymbol) <= 2*sizeof(uint64_t)); ctx->limit = ctx->locals_and_stack + MAX_ABSTRACT_INTERP_SIZE; ctx->n_consumed = ctx->locals_and_stack; #ifdef Py_DEBUG // Aids debugging a little. There should never be NULL in the abstract interpreter. @@ -316,12 +585,10 @@ _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx) // Frame setup ctx->curr_frame_depth = 0; - - return 0; } int -_Py_uop_frame_pop(_Py_UOpsContext *ctx) +_Py_uop_frame_pop(JitOptContext *ctx) { _Py_UOpsAbstractFrame *frame = ctx->frame; ctx->n_consumed = frame->locals; @@ -342,26 +609,25 @@ do { \ } \ } while (0) -static _Py_UopsSymbol * -make_bottom(_Py_UOpsContext *ctx) +static JitOptSymbol * +make_bottom(JitOptContext *ctx) { - _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); - _Py_uop_sym_set_null(sym); - _Py_uop_sym_set_non_null(sym); + JitOptSymbol *sym = sym_new(ctx); + sym->tag = JIT_SYM_BOTTOM_TAG; return sym; } PyObject * _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) { - _Py_UOpsContext context; - _Py_UOpsContext *ctx = &context; + JitOptContext context; + JitOptContext *ctx = &context; _Py_uop_abstractcontext_init(ctx); PyObject *val_42 = NULL; PyObject *val_43 = NULL; // Use a single 'sym' variable so copy-pasting tests is easier. - _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); + JitOptSymbol *sym = _Py_uop_sym_new_unknown(ctx); if (sym == NULL) { goto fail; } @@ -394,10 +660,10 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "int is a constant"); TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "int as constant is not NULL"); - _Py_uop_sym_set_type(sym, &PyLong_Type); // Should be a no-op + _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(int and int) isn't int"); - _Py_uop_sym_set_type(sym, &PyFloat_Type); // Should make it bottom + _Py_uop_sym_set_type(ctx, sym, &PyFloat_Type); // Should make it bottom TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(int and float) isn't bottom"); val_42 = PyLong_FromLong(42); @@ -412,7 +678,8 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) if (sym == NULL) { goto fail; } - _Py_uop_sym_set_const(sym, val_42); + _Py_uop_sym_set_const(ctx, sym, val_42); + TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 1, "bool(42) is not True"); TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "42 is NULL"); TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "42 isn't not NULL"); TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "42 isn't an int"); @@ -420,30 +687,66 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) TEST_PREDICATE(_Py_uop_sym_is_const(sym), "42 is not a constant"); TEST_PREDICATE(_Py_uop_sym_get_const(sym) != NULL, "42 as constant is NULL"); TEST_PREDICATE(_Py_uop_sym_get_const(sym) == val_42, "42 as constant isn't 42"); + TEST_PREDICATE(_Py_uop_sym_is_immortal(sym), "42 is not immortal"); - _Py_uop_sym_set_type(sym, &PyLong_Type); // Should be a no-op + _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(42 and 42) isn't an int"); TEST_PREDICATE(_Py_uop_sym_get_const(sym) == val_42, "(42 and 42) as constant isn't 42"); - _Py_uop_sym_set_type(sym, &PyFloat_Type); // Should make it bottom + _Py_uop_sym_set_type(ctx, sym, &PyFloat_Type); // Should make it bottom TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and float) isn't bottom"); + sym = _Py_uop_sym_new_type(ctx, &PyBool_Type); + TEST_PREDICATE(_Py_uop_sym_is_immortal(sym), "a bool is not immortal"); + sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); if (sym == NULL) { goto fail; } - _Py_uop_sym_set_const(sym, val_42); - _Py_uop_sym_set_const(sym, val_43); // Should make it bottom + _Py_uop_sym_set_const(ctx, sym, val_42); + _Py_uop_sym_set_const(ctx, sym, val_43); // Should make it bottom TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and 43) isn't bottom"); + + sym = _Py_uop_sym_new_const(ctx, Py_None); + TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 0, "bool(None) is not False"); + sym = _Py_uop_sym_new_const(ctx, Py_False); + TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 0, "bool(False) is not False"); + sym = _Py_uop_sym_new_const(ctx, PyLong_FromLong(0)); + TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 0, "bool(0) is not False"); + + JitOptSymbol *i1 = _Py_uop_sym_new_type(ctx, &PyFloat_Type); + JitOptSymbol *i2 = _Py_uop_sym_new_const(ctx, val_43); + JitOptSymbol *array[2] = { i1, i2 }; + sym = _Py_uop_sym_new_tuple(ctx, 2, array); + TEST_PREDICATE( + _Py_uop_sym_matches_type(_Py_uop_sym_tuple_getitem(ctx, sym, 0), &PyFloat_Type), + "tuple item does not match value used to create tuple" + ); + TEST_PREDICATE( + _Py_uop_sym_get_const(_Py_uop_sym_tuple_getitem(ctx, sym, 1)) == val_43, + "tuple item does not match value used to create tuple" + ); + PyObject *pair[2] = { val_42, val_43 }; + PyObject *tuple = _PyTuple_FromArray(pair, 2); + sym = _Py_uop_sym_new_const(ctx, tuple); + TEST_PREDICATE( + _Py_uop_sym_get_const(_Py_uop_sym_tuple_getitem(ctx, sym, 1)) == val_43, + "tuple item does not match value used to create tuple" + ); + _Py_uop_abstractcontext_fini(ctx); Py_DECREF(val_42); Py_DECREF(val_43); + Py_DECREF(tuple); Py_RETURN_NONE; fail: _Py_uop_abstractcontext_fini(ctx); Py_XDECREF(val_42); Py_XDECREF(val_43); + Py_DECREF(tuple); return NULL; } + +#endif /* _Py_TIER2 */ diff --git a/Python/parking_lot.c b/Python/parking_lot.c index 0a897f9952f648..8edf43235942ab 100644 --- a/Python/parking_lot.c +++ b/Python/parking_lot.c @@ -6,7 +6,7 @@ #include "pycore_pyerrors.h" // _Py_FatalErrorFormat #include "pycore_pystate.h" // _PyThreadState_GET #include "pycore_semaphore.h" // _PySemaphore -#include "pycore_time.h" //_PyTime_MonotonicUnchecked() +#include "pycore_time.h" // _PyTime_Add() #include @@ -102,31 +102,56 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, PyTime_t timeout) millis = INFINITE; } else { - millis = (DWORD) (timeout / 1000000); + PyTime_t div = _PyTime_AsMilliseconds(timeout, _PyTime_ROUND_TIMEOUT); + // Prevent overflow with clamping the result + if ((PyTime_t)PY_DWORD_MAX < div) { + millis = PY_DWORD_MAX; + } + else { + millis = (DWORD) div; + } } - wait = WaitForSingleObjectEx(sema->platform_sem, millis, FALSE); + + // NOTE: we wait on the sigint event even in non-main threads to match the + // behavior of the other platforms. Non-main threads will ignore the + // Py_PARK_INTR result. + HANDLE sigint_event = _PyOS_SigintEvent(); + HANDLE handles[2] = { sema->platform_sem, sigint_event }; + DWORD count = sigint_event != NULL ? 2 : 1; + wait = WaitForMultipleObjects(count, handles, FALSE, millis); if (wait == WAIT_OBJECT_0) { res = Py_PARK_OK; } + else if (wait == WAIT_OBJECT_0 + 1) { + ResetEvent(sigint_event); + res = Py_PARK_INTR; + } else if (wait == WAIT_TIMEOUT) { res = Py_PARK_TIMEOUT; } else { - res = Py_PARK_INTR; + _Py_FatalErrorFormat(__func__, + "unexpected error from semaphore: %u (error: %u)", + wait, GetLastError()); } #elif defined(_Py_USE_SEMAPHORES) int err; if (timeout >= 0) { struct timespec ts; -#if defined(CLOCK_MONOTONIC) && defined(HAVE_SEM_CLOCKWAIT) - PyTime_t deadline = _PyTime_Add(_PyTime_MonotonicUnchecked(), timeout); - +#if defined(CLOCK_MONOTONIC) && defined(HAVE_SEM_CLOCKWAIT) && !defined(_Py_THREAD_SANITIZER) + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); + PyTime_t deadline = _PyTime_Add(now, timeout); _PyTime_AsTimespec_clamp(deadline, &ts); err = sem_clockwait(&sema->platform_sem, CLOCK_MONOTONIC, &ts); #else - PyTime_t deadline = _PyTime_Add(_PyTime_TimeUnchecked(), timeout); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_TimeRaw(&now); + PyTime_t deadline = _PyTime_Add(now, timeout); _PyTime_AsTimespec_clamp(deadline, &ts); @@ -163,7 +188,9 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, PyTime_t timeout) _PyTime_AsTimespec_clamp(timeout, &ts); err = pthread_cond_timedwait_relative_np(&sema->cond, &sema->mutex, &ts); #else - PyTime_t deadline = _PyTime_Add(_PyTime_TimeUnchecked(), timeout); + PyTime_t now; + (void)PyTime_TimeRaw(&now); + PyTime_t deadline = _PyTime_Add(now, timeout); _PyTime_AsTimespec_clamp(deadline, &ts); err = pthread_cond_timedwait(&sema->cond, &sema->mutex, &ts); @@ -194,14 +221,16 @@ _PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout, int detach) PyThreadState *tstate = NULL; if (detach) { tstate = _PyThreadState_GET(); - if (tstate) { + if (tstate && _PyThreadState_IsAttached(tstate)) { + // Only detach if we are attached PyEval_ReleaseThread(tstate); } + else { + tstate = NULL; + } } - int res = _PySemaphore_PlatformWait(sema, timeout); - - if (detach && tstate) { + if (tstate) { PyEval_AcquireThread(tstate); } return res; diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 50c60093cd4e32..33abaddc1b5df4 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -251,8 +251,7 @@ Py_SetPath(const wchar_t *path) } -// Removed in Python 3.13 API, but kept for the stable ABI -PyAPI_FUNC(void) +void Py_SetPythonHome(const wchar_t *home) { int has_value = home && home[0]; @@ -275,8 +274,7 @@ Py_SetPythonHome(const wchar_t *home) } -// Removed in Python 3.13 API, but kept for the stable ABI -PyAPI_FUNC(void) +void Py_SetProgramName(const wchar_t *program_name) { int has_value = program_name && program_name[0]; diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c new file mode 100644 index 00000000000000..0a8945958b4b3c --- /dev/null +++ b/Python/perf_jit_trampoline.c @@ -0,0 +1,616 @@ +#include "Python.h" +#include "pycore_ceval.h" // _PyPerf_Callbacks +#include "pycore_frame.h" +#include "pycore_interp.h" + + +#ifdef PY_HAVE_PERF_TRAMPOLINE + +#include +#include +#include +#include // mmap() +#include +#include // sysconf() +#include // gettimeofday() +#include + +// ---------------------------------- +// Perf jitdump API +// ---------------------------------- + +typedef struct { + FILE* perf_map; + PyThread_type_lock map_lock; + void* mapped_buffer; + size_t mapped_size; + int code_id; +} PerfMapJitState; + +static PerfMapJitState perf_jit_map_state; + +/* +Usually the binary and libraries are mapped in separate region like below: + + address -> + --+---------------------+--//--+---------------------+-- + | .text | .data | ... | | .text | .data | ... | + --+---------------------+--//--+---------------------+-- + myprog libc.so + +So it'd be easy and straight-forward to find a mapped binary or library from an +address. + +But for JIT code, the code arena only cares about the code section. But the +resulting DSOs (which is generated by perf inject -j) contain ELF headers and +unwind info too. Then it'd generate following address space with synthesized +MMAP events. Let's say it has a sample between address B and C. + + sample + | + address -> A B v C + --------------------------------------------------------------------------------------------------- + /tmp/jitted-PID-0.so | (headers) | .text | unwind info | + /tmp/jitted-PID-1.so | (headers) | .text | unwind info | + /tmp/jitted-PID-2.so | (headers) | .text | unwind info | + ... + --------------------------------------------------------------------------------------------------- + +If it only maps the .text section, it'd find the jitted-PID-1.so but cannot see +the unwind info. If it maps both .text section and unwind sections, the sample +could be mapped to either jitted-PID-0.so or jitted-PID-1.so and it's confusing +which one is right. So to make perf happy we have non-overlapping ranges for each +DSO: + + address -> + ------------------------------------------------------------------------------------------------------- + /tmp/jitted-PID-0.so | (headers) | .text | unwind info | + /tmp/jitted-PID-1.so | (headers) | .text | unwind info | + /tmp/jitted-PID-2.so | (headers) | .text | unwind info | + ... + ------------------------------------------------------------------------------------------------------- + +As the trampolines are constant, we add a constant padding but in general the padding needs to have the +size of the unwind info rounded to 16 bytes. In general, for our trampolines this is 0x50 + */ + +#define PERF_JIT_CODE_PADDING 0x100 +#define trampoline_api _PyRuntime.ceval.perf.trampoline_api + +typedef uint64_t uword; +typedef const char* CodeComments; + +#define Pd "d" +#define MB (1024 * 1024) + +#define EM_386 3 +#define EM_X86_64 62 +#define EM_ARM 40 +#define EM_AARCH64 183 +#define EM_RISCV 243 + +#define TARGET_ARCH_IA32 0 +#define TARGET_ARCH_X64 0 +#define TARGET_ARCH_ARM 0 +#define TARGET_ARCH_ARM64 0 +#define TARGET_ARCH_RISCV32 0 +#define TARGET_ARCH_RISCV64 0 + +#define FLAG_generate_perf_jitdump 0 +#define FLAG_write_protect_code 0 +#define FLAG_write_protect_vm_isolate 0 +#define FLAG_code_comments 0 + +#define UNREACHABLE() + +static uword GetElfMachineArchitecture(void) { +#if TARGET_ARCH_IA32 + return EM_386; +#elif TARGET_ARCH_X64 + return EM_X86_64; +#elif TARGET_ARCH_ARM + return EM_ARM; +#elif TARGET_ARCH_ARM64 + return EM_AARCH64; +#elif TARGET_ARCH_RISCV32 || TARGET_ARCH_RISCV64 + return EM_RISCV; +#else + UNREACHABLE(); + return 0; +#endif +} + +typedef struct { + uint32_t magic; + uint32_t version; + uint32_t size; + uint32_t elf_mach_target; + uint32_t reserved; + uint32_t process_id; + uint64_t time_stamp; + uint64_t flags; +} Header; + + enum PerfEvent { + PerfLoad = 0, + PerfMove = 1, + PerfDebugInfo = 2, + PerfClose = 3, + PerfUnwindingInfo = 4 +}; + +struct BaseEvent { + uint32_t event; + uint32_t size; + uint64_t time_stamp; + }; + +typedef struct { + struct BaseEvent base; + uint32_t process_id; + uint32_t thread_id; + uint64_t vma; + uint64_t code_address; + uint64_t code_size; + uint64_t code_id; +} CodeLoadEvent; + +typedef struct { + struct BaseEvent base; + uint64_t unwind_data_size; + uint64_t eh_frame_hdr_size; + uint64_t mapped_size; +} CodeUnwindingInfoEvent; + +static const intptr_t nanoseconds_per_second = 1000000000; + +// Dwarf encoding constants + +static const uint8_t DwarfUData4 = 0x03; +static const uint8_t DwarfSData4 = 0x0b; +static const uint8_t DwarfPcRel = 0x10; +static const uint8_t DwarfDataRel = 0x30; +// static uint8_t DwarfOmit = 0xff; +typedef struct { + unsigned char version; + unsigned char eh_frame_ptr_enc; + unsigned char fde_count_enc; + unsigned char table_enc; + int32_t eh_frame_ptr; + int32_t eh_fde_count; + int32_t from; + int32_t to; +} EhFrameHeader; + +static int64_t get_current_monotonic_ticks(void) { + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { + UNREACHABLE(); + return 0; + } + // Convert to nanoseconds. + int64_t result = ts.tv_sec; + result *= nanoseconds_per_second; + result += ts.tv_nsec; + return result; +} + +static int64_t get_current_time_microseconds(void) { + // gettimeofday has microsecond resolution. + struct timeval tv; + if (gettimeofday(&tv, NULL) < 0) { + UNREACHABLE(); + return 0; + } + return ((int64_t)(tv.tv_sec) * 1000000) + tv.tv_usec; +} + + +static size_t round_up(int64_t value, int64_t multiple) { + if (multiple == 0) { + // Avoid division by zero + return value; + } + + int64_t remainder = value % multiple; + if (remainder == 0) { + // Value is already a multiple of 'multiple' + return value; + } + + // Calculate the difference to the next multiple + int64_t difference = multiple - remainder; + + // Add the difference to the value + int64_t rounded_up_value = value + difference; + + return rounded_up_value; +} + + +static void perf_map_jit_write_fully(const void* buffer, size_t size) { + FILE* out_file = perf_jit_map_state.perf_map; + const char* ptr = (const char*)(buffer); + while (size > 0) { + const size_t written = fwrite(ptr, 1, size, out_file); + if (written == 0) { + UNREACHABLE(); + break; + } + size -= written; + ptr += written; + } +} + +static void perf_map_jit_write_header(int pid, FILE* out_file) { + Header header; + header.magic = 0x4A695444; + header.version = 1; + header.size = sizeof(Header); + header.elf_mach_target = GetElfMachineArchitecture(); + header.process_id = pid; + header.time_stamp = get_current_time_microseconds(); + header.flags = 0; + perf_map_jit_write_fully(&header, sizeof(header)); +} + +static void* perf_map_jit_init(void) { + char filename[100]; + int pid = getpid(); + snprintf(filename, sizeof(filename) - 1, "/tmp/jit-%d.dump", pid); + const int fd = open(filename, O_CREAT | O_TRUNC | O_RDWR, 0666); + if (fd == -1) { + return NULL; + } + + const long page_size = sysconf(_SC_PAGESIZE); // NOLINT(runtime/int) + if (page_size == -1) { + close(fd); + return NULL; + } + + // The perf jit interface forces us to map the first page of the file + // to signal that we are using the interface. + perf_jit_map_state.mapped_buffer = mmap(NULL, page_size, PROT_READ | PROT_EXEC, MAP_PRIVATE, fd, 0); + if (perf_jit_map_state.mapped_buffer == NULL) { + close(fd); + return NULL; + } + perf_jit_map_state.mapped_size = page_size; + perf_jit_map_state.perf_map = fdopen(fd, "w+"); + if (perf_jit_map_state.perf_map == NULL) { + close(fd); + return NULL; + } + setvbuf(perf_jit_map_state.perf_map, NULL, _IOFBF, 2 * MB); + perf_map_jit_write_header(pid, perf_jit_map_state.perf_map); + + perf_jit_map_state.map_lock = PyThread_allocate_lock(); + if (perf_jit_map_state.map_lock == NULL) { + fclose(perf_jit_map_state.perf_map); + return NULL; + } + perf_jit_map_state.code_id = 0; + + trampoline_api.code_padding = PERF_JIT_CODE_PADDING; + return &perf_jit_map_state; +} + +/* DWARF definitions. */ + +#define DWRF_CIE_VERSION 1 + +enum { + DWRF_CFA_nop = 0x0, + DWRF_CFA_offset_extended = 0x5, + DWRF_CFA_def_cfa = 0xc, + DWRF_CFA_def_cfa_offset = 0xe, + DWRF_CFA_offset_extended_sf = 0x11, + DWRF_CFA_advance_loc = 0x40, + DWRF_CFA_offset = 0x80 +}; + +enum + { + DWRF_EH_PE_absptr = 0x00, + DWRF_EH_PE_omit = 0xff, + + /* FDE data encoding. */ + DWRF_EH_PE_uleb128 = 0x01, + DWRF_EH_PE_udata2 = 0x02, + DWRF_EH_PE_udata4 = 0x03, + DWRF_EH_PE_udata8 = 0x04, + DWRF_EH_PE_sleb128 = 0x09, + DWRF_EH_PE_sdata2 = 0x0a, + DWRF_EH_PE_sdata4 = 0x0b, + DWRF_EH_PE_sdata8 = 0x0c, + DWRF_EH_PE_signed = 0x08, + + /* FDE flags. */ + DWRF_EH_PE_pcrel = 0x10, + DWRF_EH_PE_textrel = 0x20, + DWRF_EH_PE_datarel = 0x30, + DWRF_EH_PE_funcrel = 0x40, + DWRF_EH_PE_aligned = 0x50, + + DWRF_EH_PE_indirect = 0x80 + }; + +enum { DWRF_TAG_compile_unit = 0x11 }; + +enum { DWRF_children_no = 0, DWRF_children_yes = 1 }; + +enum { DWRF_AT_name = 0x03, DWRF_AT_stmt_list = 0x10, DWRF_AT_low_pc = 0x11, DWRF_AT_high_pc = 0x12 }; + +enum { DWRF_FORM_addr = 0x01, DWRF_FORM_data4 = 0x06, DWRF_FORM_string = 0x08 }; + +enum { DWRF_LNS_extended_op = 0, DWRF_LNS_copy = 1, DWRF_LNS_advance_pc = 2, DWRF_LNS_advance_line = 3 }; + +enum { DWRF_LNE_end_sequence = 1, DWRF_LNE_set_address = 2 }; + +enum { +#ifdef __x86_64__ + /* Yes, the order is strange, but correct. */ + DWRF_REG_AX, + DWRF_REG_DX, + DWRF_REG_CX, + DWRF_REG_BX, + DWRF_REG_SI, + DWRF_REG_DI, + DWRF_REG_BP, + DWRF_REG_SP, + DWRF_REG_8, + DWRF_REG_9, + DWRF_REG_10, + DWRF_REG_11, + DWRF_REG_12, + DWRF_REG_13, + DWRF_REG_14, + DWRF_REG_15, + DWRF_REG_RA, +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + DWRF_REG_SP = 31, + DWRF_REG_RA = 30, +#else +# error "Unsupported target architecture" +#endif +}; + +typedef struct ELFObjectContext +{ + uint8_t* p; /* Pointer to next address in obj.space. */ + uint8_t* startp; /* Pointer to start address in obj.space. */ + uint8_t* eh_frame_p; /* Pointer to start address in obj.space. */ + uint32_t code_size; /* Size of machine code. */ +} ELFObjectContext; + +/* Append a null-terminated string. */ +static uint32_t +elfctx_append_string(ELFObjectContext* ctx, const char* str) +{ + uint8_t* p = ctx->p; + uint32_t ofs = (uint32_t)(p - ctx->startp); + do { + *p++ = (uint8_t)*str; + } while (*str++); + ctx->p = p; + return ofs; +} + +/* Append a SLEB128 value. */ +static void +elfctx_append_sleb128(ELFObjectContext* ctx, int32_t v) +{ + uint8_t* p = ctx->p; + for (; (uint32_t)(v + 0x40) >= 0x80; v >>= 7) { + *p++ = (uint8_t)((v & 0x7f) | 0x80); + } + *p++ = (uint8_t)(v & 0x7f); + ctx->p = p; +} + +/* Append a ULEB128 to buffer. */ +static void +elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) +{ + uint8_t* p = ctx->p; + for (; v >= 0x80; v >>= 7) { + *p++ = (char)((v & 0x7f) | 0x80); + } + *p++ = (char)v; + ctx->p = p; +} + +/* Shortcuts to generate DWARF structures. */ +#define DWRF_U8(x) (*p++ = (x)) +#define DWRF_I8(x) (*(int8_t*)p = (x), p++) +#define DWRF_U16(x) (*(uint16_t*)p = (x), p += 2) +#define DWRF_U32(x) (*(uint32_t*)p = (x), p += 4) +#define DWRF_ADDR(x) (*(uintptr_t*)p = (x), p += sizeof(uintptr_t)) +#define DWRF_UV(x) (ctx->p = p, elfctx_append_uleb128(ctx, (x)), p = ctx->p) +#define DWRF_SV(x) (ctx->p = p, elfctx_append_sleb128(ctx, (x)), p = ctx->p) +#define DWRF_STR(str) (ctx->p = p, elfctx_append_string(ctx, (str)), p = ctx->p) +#define DWRF_ALIGNNOP(s) \ + while ((uintptr_t)p & ((s)-1)) { \ + *p++ = DWRF_CFA_nop; \ + } +#define DWRF_SECTION(name, stmt) \ + { \ + uint32_t* szp_##name = (uint32_t*)p; \ + p += 4; \ + stmt; \ + *szp_##name = (uint32_t)((p - (uint8_t*)szp_##name) - 4); \ + } + +/* Initialize .eh_frame section. */ +static void +elf_init_ehframe(ELFObjectContext* ctx) +{ + uint8_t* p = ctx->p; + uint8_t* framep = p; + + /* Emit DWARF EH CIE. */ + DWRF_SECTION(CIE, DWRF_U32(0); /* Offset to CIE itself. */ + DWRF_U8(DWRF_CIE_VERSION); + DWRF_STR("zR"); /* Augmentation. */ + DWRF_UV(1); /* Code alignment factor. */ + DWRF_SV(-(int64_t)sizeof(uintptr_t)); /* Data alignment factor. */ + DWRF_U8(DWRF_REG_RA); /* Return address register. */ + DWRF_UV(1); + DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); /* Augmentation data. */ + DWRF_U8(DWRF_CFA_def_cfa); DWRF_UV(DWRF_REG_SP); DWRF_UV(sizeof(uintptr_t)); + DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); DWRF_UV(1); + DWRF_ALIGNNOP(sizeof(uintptr_t)); + ) + + ctx->eh_frame_p = p; + + /* Emit DWARF EH FDE. */ + DWRF_SECTION(FDE, DWRF_U32((uint32_t)(p - framep)); /* Offset to CIE. */ + DWRF_U32(-0x30); /* Machine code offset relative to .text. */ + DWRF_U32(ctx->code_size); /* Machine code length. */ + DWRF_U8(0); /* Augmentation data. */ + /* Registers saved in CFRAME. */ +#ifdef __x86_64__ + DWRF_U8(DWRF_CFA_advance_loc | 4); + DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); + DWRF_U8(DWRF_CFA_advance_loc | 6); + DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(8); + /* Extra registers saved for JIT-compiled code. */ +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + DWRF_U8(DWRF_CFA_advance_loc | 1); + DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); + DWRF_U8(DWRF_CFA_offset | 29); DWRF_UV(2); + DWRF_U8(DWRF_CFA_offset | 30); DWRF_UV(1); + DWRF_U8(DWRF_CFA_advance_loc | 3); + DWRF_U8(DWRF_CFA_offset | -(64 - 29)); + DWRF_U8(DWRF_CFA_offset | -(64 - 30)); + DWRF_U8(DWRF_CFA_def_cfa_offset); + DWRF_UV(0); +#else +# error "Unsupported target architecture" +#endif + DWRF_ALIGNNOP(sizeof(uintptr_t));) + + ctx->p = p; +} + +static void perf_map_jit_write_entry(void *state, const void *code_addr, + unsigned int code_size, PyCodeObject *co) +{ + + if (perf_jit_map_state.perf_map == NULL) { + void* ret = perf_map_jit_init(); + if(ret == NULL){ + return; + } + } + + const char *entry = ""; + if (co->co_qualname != NULL) { + entry = PyUnicode_AsUTF8(co->co_qualname); + } + const char *filename = ""; + if (co->co_filename != NULL) { + filename = PyUnicode_AsUTF8(co->co_filename); + } + + + size_t perf_map_entry_size = snprintf(NULL, 0, "py::%s:%s", entry, filename) + 1; + char* perf_map_entry = (char*) PyMem_RawMalloc(perf_map_entry_size); + if (perf_map_entry == NULL) { + return; + } + snprintf(perf_map_entry, perf_map_entry_size, "py::%s:%s", entry, filename); + + const size_t name_length = strlen(perf_map_entry); + uword base = (uword)code_addr; + uword size = code_size; + + // Write the code unwinding info event. + + // Create unwinding information (eh frame) + ELFObjectContext ctx; + char buffer[1024]; + ctx.code_size = code_size; + ctx.startp = ctx.p = (uint8_t*)buffer; + elf_init_ehframe(&ctx); + int eh_frame_size = ctx.p - ctx.startp; + + // Populate the unwind info event for perf + CodeUnwindingInfoEvent ev2; + ev2.base.event = PerfUnwindingInfo; + ev2.base.time_stamp = get_current_monotonic_ticks(); + ev2.unwind_data_size = sizeof(EhFrameHeader) + eh_frame_size; + // Ensure we have enough space between DSOs when perf maps them + assert(ev2.unwind_data_size <= PERF_JIT_CODE_PADDING); + ev2.eh_frame_hdr_size = sizeof(EhFrameHeader); + ev2.mapped_size = round_up(ev2.unwind_data_size, 16); + int content_size = sizeof(ev2) + sizeof(EhFrameHeader) + eh_frame_size; + int padding_size = round_up(content_size, 8) - content_size; + ev2.base.size = content_size + padding_size; + perf_map_jit_write_fully(&ev2, sizeof(ev2)); + + + // Populate the eh Frame header + EhFrameHeader f; + f.version = 1; + f.eh_frame_ptr_enc = DwarfSData4 | DwarfPcRel; + f.fde_count_enc = DwarfUData4; + f.table_enc = DwarfSData4 | DwarfDataRel; + f.eh_frame_ptr = -(eh_frame_size + 4 * sizeof(unsigned char)); + f.eh_fde_count = 1; + f.from = -(round_up(code_size, 8) + eh_frame_size); + int cie_size = ctx.eh_frame_p - ctx.startp; + f.to = -(eh_frame_size - cie_size); + + perf_map_jit_write_fully(ctx.startp, eh_frame_size); + perf_map_jit_write_fully(&f, sizeof(f)); + + char padding_bytes[] = "\0\0\0\0\0\0\0\0"; + perf_map_jit_write_fully(&padding_bytes, padding_size); + + // Write the code load event. + CodeLoadEvent ev; + ev.base.event = PerfLoad; + ev.base.size = sizeof(ev) + (name_length+1) + size; + ev.base.time_stamp = get_current_monotonic_ticks(); + ev.process_id = getpid(); + ev.thread_id = syscall(SYS_gettid); + ev.vma = base; + ev.code_address = base; + ev.code_size = size; + perf_jit_map_state.code_id += 1; + ev.code_id = perf_jit_map_state.code_id; + + perf_map_jit_write_fully(&ev, sizeof(ev)); + perf_map_jit_write_fully(perf_map_entry, name_length+1); + perf_map_jit_write_fully((void*)(base), size); + return; +} + +static int perf_map_jit_fini(void* state) { + if (perf_jit_map_state.perf_map != NULL) { + // close the file + PyThread_acquire_lock(perf_jit_map_state.map_lock, 1); + fclose(perf_jit_map_state.perf_map); + PyThread_release_lock(perf_jit_map_state.map_lock); + + // clean up the lock and state + PyThread_free_lock(perf_jit_map_state.map_lock); + perf_jit_map_state.perf_map = NULL; + } + if (perf_jit_map_state.mapped_buffer != NULL) { + munmap(perf_jit_map_state.mapped_buffer, perf_jit_map_state.mapped_size); + } + trampoline_api.state = NULL; + return 0; +} + +_PyPerf_Callbacks _Py_perfmap_jit_callbacks = { + &perf_map_jit_init, + &perf_map_jit_write_entry, + &perf_map_jit_fini, +}; + +#endif diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 750ba18d3510ed..ad077dc861b0a7 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -143,6 +143,8 @@ any DWARF information available for them). #include // mmap() #include #include // sysconf() +#include // gettimeofday() + #if defined(__arm__) || defined(__arm64__) || defined(__aarch64__) #define PY_HAVE_INVALIDATE_ICACHE @@ -187,12 +189,19 @@ struct code_arena_st { typedef struct code_arena_st code_arena_t; typedef struct trampoline_api_st trampoline_api_t; +enum perf_trampoline_type { + PERF_TRAMPOLINE_UNSET = 0, + PERF_TRAMPOLINE_TYPE_MAP = 1, + PERF_TRAMPOLINE_TYPE_JITDUMP = 2, +}; + #define perf_status _PyRuntime.ceval.perf.status #define extra_code_index _PyRuntime.ceval.perf.extra_code_index #define perf_code_arena _PyRuntime.ceval.perf.code_arena #define trampoline_api _PyRuntime.ceval.perf.trampoline_api #define perf_map_file _PyRuntime.ceval.perf.map_file #define persist_after_fork _PyRuntime.ceval.perf.persist_after_fork +#define perf_trampoline_type _PyRuntime.ceval.perf.perf_trampoline_type static void perf_map_write_entry(void *state, const void *code_addr, @@ -220,6 +229,8 @@ static void* perf_map_init_state(void) { PyUnstable_PerfMapState_Init(); + trampoline_api.code_padding = 0; + perf_trampoline_type = PERF_TRAMPOLINE_TYPE_MAP; return NULL; } @@ -236,6 +247,30 @@ _PyPerf_Callbacks _Py_perfmap_callbacks = { &perf_map_free_state, }; + +static size_t round_up(int64_t value, int64_t multiple) { + if (multiple == 0) { + // Avoid division by zero + return value; + } + + int64_t remainder = value % multiple; + if (remainder == 0) { + // Value is already a multiple of 'multiple' + return value; + } + + // Calculate the difference to the next multiple + int64_t difference = multiple - remainder; + + // Add the difference to the value + int64_t rounded_up_value = value + difference; + + return rounded_up_value; +} + +// TRAMPOLINE MANAGEMENT API + static int new_code_arena(void) { @@ -256,6 +291,7 @@ new_code_arena(void) void *start = &_Py_trampoline_func_start; void *end = &_Py_trampoline_func_end; size_t code_size = end - start; + size_t chunk_size = round_up(code_size + trampoline_api.code_padding, 16); // TODO: Check the effect of alignment of the code chunks. Initial investigation // showed that this has no effect on performance in x86-64 or aarch64 and the current // version has the advantage that the unwinder in GDB can unwind across JIT-ed code. @@ -264,9 +300,9 @@ new_code_arena(void) // measurable performance improvement by rounding trampolines up to 32-bit // or 64-bit alignment. - size_t n_copies = mem_size / code_size; + size_t n_copies = mem_size / chunk_size; for (size_t i = 0; i < n_copies; i++) { - memcpy(memory + i * code_size, start, code_size * sizeof(char)); + memcpy(memory + i * chunk_size, start, code_size * sizeof(char)); } // Some systems may prevent us from creating executable code on the fly. int res = mprotect(memory, mem_size, PROT_READ | PROT_EXEC); @@ -320,16 +356,18 @@ static inline py_trampoline code_arena_new_code(code_arena_t *code_arena) { py_trampoline trampoline = (py_trampoline)code_arena->current_addr; - code_arena->size_left -= code_arena->code_size; - code_arena->current_addr += code_arena->code_size; + size_t total_code_size = round_up(code_arena->code_size + trampoline_api.code_padding, 16); + code_arena->size_left -= total_code_size; + code_arena->current_addr += total_code_size; return trampoline; } static inline py_trampoline compile_trampoline(void) { + size_t total_code_size = round_up(perf_code_arena->code_size + trampoline_api.code_padding, 16); if ((perf_code_arena == NULL) || - (perf_code_arena->size_left <= perf_code_arena->code_size)) { + (perf_code_arena->size_left <= total_code_size)) { if (new_code_arena() < 0) { return NULL; } @@ -446,11 +484,11 @@ _PyPerfTrampoline_Init(int activate) return -1; } if (!activate) { - tstate->interp->eval_frame = NULL; + _PyInterpreterState_SetEvalFrameFunc(tstate->interp, NULL); perf_status = PERF_STATUS_NO_INIT; } else { - tstate->interp->eval_frame = py_trampoline_evaluator; + _PyInterpreterState_SetEvalFrameFunc(tstate->interp, py_trampoline_evaluator); if (new_code_arena() < 0) { return -1; } @@ -476,10 +514,11 @@ _PyPerfTrampoline_Fini(void) } PyThreadState *tstate = _PyThreadState_GET(); if (tstate->interp->eval_frame == py_trampoline_evaluator) { - tstate->interp->eval_frame = NULL; + _PyInterpreterState_SetEvalFrameFunc(tstate->interp, NULL); } if (perf_status == PERF_STATUS_OK) { trampoline_api.free_state(trampoline_api.state); + perf_trampoline_type = PERF_TRAMPOLINE_UNSET; } extra_code_index = -1; perf_status = PERF_STATUS_NO_INIT; @@ -508,6 +547,9 @@ _PyPerfTrampoline_AfterFork_Child(void) { #ifdef PY_HAVE_PERF_TRAMPOLINE if (persist_after_fork) { + if (perf_trampoline_type != PERF_TRAMPOLINE_TYPE_MAP) { + return PyStatus_Error("Failed to copy perf map file as perf trampoline type is not type map."); + } _PyPerfTrampoline_Fini(); char filename[256]; pid_t parent_pid = getppid(); diff --git a/Python/pyarena.c b/Python/pyarena.c index ead03370d153c3..28970f9d0670f4 100644 --- a/Python/pyarena.c +++ b/Python/pyarena.c @@ -4,11 +4,8 @@ /* A simple arena block structure. Measurements with standard library modules suggest the average - allocation is about 20 bytes and that most compiles use a single + allocation is about 20 bytes and that most compilers use a single block. - - TODO(jhylton): Think about a realloc API, maybe just for the last - allocation? */ #define DEFAULT_BLOCK_SIZE 8192 @@ -108,7 +105,6 @@ block_alloc(block *b, size_t size) /* If we need to allocate more memory than will fit in the default block, allocate a one-off block that is exactly the right size. */ - /* TODO(jhylton): Think about space waste at end of block */ block *newbl = block_new( size < DEFAULT_BLOCK_SIZE ? DEFAULT_BLOCK_SIZE : size); diff --git a/Python/pyhash.c b/Python/pyhash.c index 141407c265677a..216f437dd9a2d4 100644 --- a/Python/pyhash.c +++ b/Python/pyhash.c @@ -22,7 +22,7 @@ extern PyHash_FuncDef PyHash_Func; static PyHash_FuncDef PyHash_Func; #endif -/* Count _Py_HashBytes() calls */ +/* Count Py_HashBuffer() calls */ #ifdef Py_HASH_STATS #define Py_HASH_STATS_MAX 32 static Py_ssize_t hashstats[Py_HASH_STATS_MAX + 1] = {0}; @@ -90,11 +90,11 @@ _Py_HashDouble(PyObject *inst, double v) double m; Py_uhash_t x, y; - if (!Py_IS_FINITE(v)) { - if (Py_IS_INFINITY(v)) + if (!isfinite(v)) { + if (isinf(v)) return v > 0 ? _PyHASH_INF : -_PyHASH_INF; else - return _Py_HashPointer(inst); + return PyObject_GenericHash(inst); } m = frexp(v, &e); @@ -140,9 +140,14 @@ Py_HashPointer(const void *ptr) } Py_hash_t -_Py_HashBytes(const void *src, Py_ssize_t len) +PyObject_GenericHash(PyObject *obj) +{ + return Py_HashPointer(obj); +} + +Py_hash_t +Py_HashBuffer(const void *ptr, Py_ssize_t len) { - Py_hash_t x; /* We make the hash of the empty string be 0, rather than using (prefix ^ suffix), since this slightly obfuscates the hash secret @@ -155,21 +160,22 @@ _Py_HashBytes(const void *src, Py_ssize_t len) hashstats[(len <= Py_HASH_STATS_MAX) ? len : 0]++; #endif + Py_hash_t x; #if Py_HASH_CUTOFF > 0 if (len < Py_HASH_CUTOFF) { /* Optimize hashing of very small strings with inline DJBX33A. */ Py_uhash_t hash; - const unsigned char *p = src; + const unsigned char *p = ptr; hash = 5381; /* DJBX33A starts with 5381 */ switch(len) { /* ((hash << 5) + hash) + *p == hash * 33 + *p */ - case 7: hash = ((hash << 5) + hash) + *p++; /* fallthrough */ - case 6: hash = ((hash << 5) + hash) + *p++; /* fallthrough */ - case 5: hash = ((hash << 5) + hash) + *p++; /* fallthrough */ - case 4: hash = ((hash << 5) + hash) + *p++; /* fallthrough */ - case 3: hash = ((hash << 5) + hash) + *p++; /* fallthrough */ - case 2: hash = ((hash << 5) + hash) + *p++; /* fallthrough */ + case 7: hash = ((hash << 5) + hash) + *p++; _Py_FALLTHROUGH; + case 6: hash = ((hash << 5) + hash) + *p++; _Py_FALLTHROUGH; + case 5: hash = ((hash << 5) + hash) + *p++; _Py_FALLTHROUGH; + case 4: hash = ((hash << 5) + hash) + *p++; _Py_FALLTHROUGH; + case 3: hash = ((hash << 5) + hash) + *p++; _Py_FALLTHROUGH; + case 2: hash = ((hash << 5) + hash) + *p++; _Py_FALLTHROUGH; case 1: hash = ((hash << 5) + hash) + *p++; break; default: Py_UNREACHABLE(); @@ -180,10 +186,13 @@ _Py_HashBytes(const void *src, Py_ssize_t len) } else #endif /* Py_HASH_CUTOFF */ - x = PyHash_Func.hash(src, len); + { + x = PyHash_Func.hash(ptr, len); + } - if (x == -1) + if (x == -1) { return -2; + } return x; } @@ -257,12 +266,12 @@ fnv(const void *src, Py_ssize_t len) x ^= (Py_uhash_t) *p << 7; while (blocks--) { PY_UHASH_CPY(block.bytes, p); - x = (_PyHASH_MULTIPLIER * x) ^ block.value; + x = (PyHASH_MULTIPLIER * x) ^ block.value; p += SIZEOF_PY_UHASH_T; } /* add remainder */ for (; remainder > 0; remainder--) - x = (_PyHASH_MULTIPLIER * x) ^ (Py_uhash_t) *p++; + x = (PyHASH_MULTIPLIER * x) ^ (Py_uhash_t) *p++; x ^= (Py_uhash_t) len; x ^= (Py_uhash_t) _Py_HashSecret.fnv.suffix; if (x == (Py_uhash_t) -1) { @@ -385,13 +394,13 @@ siphash13(uint64_t k0, uint64_t k1, const void *src, Py_ssize_t src_sz) { t = 0; pt = (uint8_t *)&t; switch (src_sz) { - case 7: pt[6] = in[6]; /* fall through */ - case 6: pt[5] = in[5]; /* fall through */ - case 5: pt[4] = in[4]; /* fall through */ + case 7: pt[6] = in[6]; _Py_FALLTHROUGH; + case 6: pt[5] = in[5]; _Py_FALLTHROUGH; + case 5: pt[4] = in[4]; _Py_FALLTHROUGH; case 4: memcpy(pt, in, sizeof(uint32_t)); break; - case 3: pt[2] = in[2]; /* fall through */ - case 2: pt[1] = in[1]; /* fall through */ - case 1: pt[0] = in[0]; /* fall through */ + case 3: pt[2] = in[2]; _Py_FALLTHROUGH; + case 2: pt[1] = in[1]; _Py_FALLTHROUGH; + case 1: pt[0] = in[0]; break; } b |= _le64toh(t); @@ -436,13 +445,13 @@ siphash24(uint64_t k0, uint64_t k1, const void *src, Py_ssize_t src_sz) { t = 0; pt = (uint8_t *)&t; switch (src_sz) { - case 7: pt[6] = in[6]; /* fall through */ - case 6: pt[5] = in[5]; /* fall through */ - case 5: pt[4] = in[4]; /* fall through */ + case 7: pt[6] = in[6]; _Py_FALLTHROUGH; + case 6: pt[5] = in[5]; _Py_FALLTHROUGH; + case 5: pt[4] = in[4]; _Py_FALLTHROUGH; case 4: memcpy(pt, in, sizeof(uint32_t)); break; - case 3: pt[2] = in[2]; /* fall through */ - case 2: pt[1] = in[1]; /* fall through */ - case 1: pt[0] = in[0]; /* fall through */ + case 3: pt[2] = in[2]; _Py_FALLTHROUGH; + case 2: pt[1] = in[1]; _Py_FALLTHROUGH; + case 1: pt[0] = in[0]; break; } b |= _le64toh(t); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 3a2c0a450ac9d9..00a98af998cfce 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2,6 +2,7 @@ #include "Python.h" +#include "pycore_audit.h" // _PySys_ClearAuditHooks() #include "pycore_call.h" // _PyObject_CallMethod() #include "pycore_ceval.h" // _PyEval_FiniGIL() #include "pycore_codecs.h" // _PyCodec_Lookup() @@ -9,8 +10,8 @@ #include "pycore_dict.h" // _PyDict_Fini() #include "pycore_exceptions.h" // _PyExc_InitTypes() #include "pycore_fileutils.h" // _Py_ResetForceASCII() +#include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_floatobject.h" // _PyFloat_InitTypes() -#include "pycore_genobject.h" // _PyAsyncGen_Fini() #include "pycore_global_objects_fini_generated.h" // "_PyStaticObjects_CheckRefcnt() #include "pycore_import.h" // _PyImport_BootstrapImp() #include "pycore_initconfig.h" // _PyStatus_OK() @@ -26,13 +27,13 @@ #include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_sliceobject.h" // _PySlice_Fini() -#include "pycore_sysmodule.h" // _PySys_ClearAuditHooks() +#include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_traceback.h" // _Py_DumpTracebackThreads() +#include "pycore_uniqueid.h" // _PyObject_FinalizeUniqueIdPool() #include "pycore_typeobject.h" // _PyTypes_InitTypes() #include "pycore_typevarobject.h" // _Py_clear_generic_types() #include "pycore_unicodeobject.h" // _PyUnicode_InitTypes() #include "pycore_weakref.h" // _PyWeakref_GET_REF() -#include "cpython/optimizer.h" // _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS #include "pycore_obmalloc.h" // _PyMem_init_obmalloc() #include "opcode.h" @@ -44,7 +45,26 @@ #endif #if defined(__APPLE__) +# include +# include # include +// The os_log unified logging APIs were introduced in macOS 10.12, iOS 10.0, +// tvOS 10.0, and watchOS 3.0; +# if defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE +# define HAS_APPLE_SYSTEM_LOG 1 +# elif defined(TARGET_OS_OSX) && TARGET_OS_OSX +# if defined(MAC_OS_X_VERSION_10_12) && MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12 +# define HAS_APPLE_SYSTEM_LOG 1 +# else +# define HAS_APPLE_SYSTEM_LOG 0 +# endif +# else +# define HAS_APPLE_SYSTEM_LOG 0 +# endif + +# if HAS_APPLE_SYSTEM_LOG +# include +# endif #endif #ifdef HAVE_SIGNAL_H @@ -71,9 +91,17 @@ static PyStatus add_main_module(PyInterpreterState *interp); static PyStatus init_import_site(void); static PyStatus init_set_builtins_open(void); static PyStatus init_sys_streams(PyThreadState *tstate); +#ifdef __ANDROID__ +static PyStatus init_android_streams(PyThreadState *tstate); +#endif +#if defined(__APPLE__) && HAS_APPLE_SYSTEM_LOG +static PyStatus init_apple_streams(PyThreadState *tstate); +#endif static void wait_for_thread_shutdown(PyThreadState *tstate); +static void finalize_subinterpreters(void); static void call_ll_exitfuncs(_PyRuntimeState *runtime); + /* The following places the `_PyRuntime` structure in a location that can be * found without any external information. This is meant to ease access to the * interpreter state for various runtime debugging tools, but is *not* an @@ -83,26 +111,11 @@ static void call_ll_exitfuncs(_PyRuntimeState *runtime); _Py_COMP_DIAG_PUSH _Py_COMP_DIAG_IGNORE_DEPR_DECLS -#if defined(MS_WINDOWS) - -#pragma section("PyRuntime", read, write) -__declspec(allocate("PyRuntime")) - -#elif defined(__APPLE__) - -__attribute__(( - section(SEG_DATA ",PyRuntime") -)) - -#endif - -_PyRuntimeState _PyRuntime -#if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) -__attribute__ ((section (".PyRuntime"))) -#endif -= _PyRuntimeState_INIT(_PyRuntime); +GENERATE_DEBUG_SECTION(PyRuntime, _PyRuntimeState _PyRuntime) += _PyRuntimeState_INIT(_PyRuntime, _Py_Debug_Cookie); _Py_COMP_DIAG_POP + static int runtime_initialized = 0; PyStatus @@ -415,40 +428,6 @@ interpreter_update_config(PyThreadState *tstate, int only_update_path_config) } -int -_PyInterpreterState_SetConfig(const PyConfig *src_config) -{ - PyThreadState *tstate = _PyThreadState_GET(); - int res = -1; - - PyConfig config; - PyConfig_InitPythonConfig(&config); - PyStatus status = _PyConfig_Copy(&config, src_config); - if (_PyStatus_EXCEPTION(status)) { - _PyErr_SetFromPyStatus(status); - goto done; - } - - status = _PyConfig_Read(&config, 1); - if (_PyStatus_EXCEPTION(status)) { - _PyErr_SetFromPyStatus(status); - goto done; - } - - status = _PyConfig_Copy(&tstate->interp->config, &config); - if (_PyStatus_EXCEPTION(status)) { - _PyErr_SetFromPyStatus(status); - goto done; - } - - res = interpreter_update_config(tstate, 0); - -done: - PyConfig_Clear(&config); - return res; -} - - /* Global initializations. Can be undone by Py_Finalize(). Don't call this twice without an intervening Py_Finalize() call. @@ -477,6 +456,7 @@ pyinit_core_reconfigure(_PyRuntimeState *runtime, if (interp == NULL) { return _PyStatus_ERR("can't make main interpreter"); } + assert(interp->_ready); status = _PyConfig_Write(config, runtime); if (_PyStatus_EXCEPTION(status)) { @@ -558,6 +538,15 @@ init_interp_settings(PyInterpreterState *interp, return _PyStatus_ERR("per-interpreter obmalloc does not support " "single-phase init extension modules"); } +#ifdef Py_GIL_DISABLED + if (!_Py_IsMainInterpreter(interp) && + !config->check_multi_interp_extensions) + { + return _PyStatus_ERR("The free-threaded build does not support " + "single-phase init extension modules in " + "subinterpreters"); + } +#endif if (config->allow_fork) { interp->feature_flags |= Py_RTFLAGS_FORK; @@ -611,9 +600,11 @@ static int builtins_dict_watcher(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *new_value) { PyInterpreterState *interp = _PyInterpreterState_GET(); +#ifdef _Py_TIER2 if (interp->rare_events.builtin_dict < _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { _Py_Executors_InvalidateAll(interp, 1); } +#endif RARE_EVENT_INTERP_INC(interp, builtin_dict); return 0; } @@ -631,6 +622,8 @@ pycore_create_interpreter(_PyRuntimeState *runtime, } assert(interp != NULL); assert(_Py_IsMainInterpreter(interp)); + _PyInterpreterState_SetWhence(interp, _PyInterpreterState_WHENCE_RUNTIME); + interp->_ready = 1; status = _PyConfig_Copy(&interp->config, src_config); if (_PyStatus_EXCEPTION(status)) { @@ -644,22 +637,36 @@ pycore_create_interpreter(_PyRuntimeState *runtime, } PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; - // The main interpreter always has its own GIL. + // The main interpreter always has its own GIL and supports single-phase + // init extensions. config.gil = PyInterpreterConfig_OWN_GIL; + config.check_multi_interp_extensions = 0; status = init_interp_settings(interp, &config); if (_PyStatus_EXCEPTION(status)) { return status; } + // This could be done in init_interpreter() (in pystate.c) if it + // didn't depend on interp->feature_flags being set already. + status = _PyObject_InitState(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + // initialize the interp->obmalloc state. This must be done after // the settings are loaded (so that feature_flags are set) but before // any calls are made to obmalloc functions. if (_PyMem_init_obmalloc(interp) < 0) { - return _PyStatus_NO_MEMORY(); + return _PyStatus_NO_MEMORY(); + } + + status = _PyTraceMalloc_Init(); + if (_PyStatus_EXCEPTION(status)) { + return status; } PyThreadState *tstate = _PyThreadState_New(interp, - _PyThreadState_WHENCE_INTERP); + _PyThreadState_WHENCE_INIT); if (tstate == NULL) { return _PyStatus_ERR("can't make first thread"); } @@ -687,6 +694,10 @@ pycore_init_global_objects(PyInterpreterState *interp) _PyUnicode_InitState(interp); + if (_Py_IsMainInterpreter(interp)) { + _Py_GetConstant_Init(); + } + return _PyStatus_OK(); } @@ -759,7 +770,7 @@ pycore_init_builtins(PyThreadState *tstate) } PyObject *modules = _PyImport_GetModules(interp); - if (_PyImport_FixupBuiltin(bimod, "builtins", modules) < 0) { + if (_PyImport_FixupBuiltin(tstate, bimod, "builtins", modules) < 0) { goto error; } @@ -831,6 +842,11 @@ pycore_interp_init(PyThreadState *tstate) return status; } + status = _PyCode_Init(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + status = _PyDtoa_Init(interp); if (_PyStatus_EXCEPTION(status)) { return status; @@ -1187,7 +1203,14 @@ init_interp_main(PyThreadState *tstate) #ifdef PY_HAVE_PERF_TRAMPOLINE if (config->perf_profiling) { - if (_PyPerfTrampoline_SetCallbacks(&_Py_perfmap_callbacks) < 0 || + _PyPerf_Callbacks *cur_cb; + if (config->perf_profiling == 1) { + cur_cb = &_Py_perfmap_callbacks; + } + else { + cur_cb = &_Py_perfmap_jit_callbacks; + } + if (_PyPerfTrampoline_SetCallbacks(cur_cb) < 0 || _PyPerfTrampoline_Init(config->perf_profiling) < 0) { return _PyStatus_ERR("can't initialize the perf trampoline"); } @@ -1205,6 +1228,21 @@ init_interp_main(PyThreadState *tstate) return status; } +#ifdef __ANDROID__ + status = init_android_streams(tstate); + if (_PyStatus_EXCEPTION(status)) { + return status; + } +#endif +#if defined(__APPLE__) && HAS_APPLE_SYSTEM_LOG + if (config->use_system_logger) { + status = init_apple_streams(tstate); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } +#endif + #ifdef Py_DEBUG run_presite(tstate); #endif @@ -1244,30 +1282,35 @@ init_interp_main(PyThreadState *tstate) } // Turn on experimental tier 2 (uops-based) optimizer + // This is also needed when the JIT is enabled +#ifdef _Py_TIER2 if (is_main_interp) { -#ifndef _Py_JIT - // No JIT, maybe use the tier two interpreter: - char *envvar = Py_GETENV("PYTHON_UOPS"); - int enabled = envvar != NULL && *envvar > '0'; - if (_Py_get_xoption(&config->xoptions, L"uops") != NULL) { - enabled = 1; + int enabled = 1; +#if _Py_TIER2 & 2 + enabled = 0; +#endif + char *env = Py_GETENV("PYTHON_JIT"); + if (env && *env != '\0') { + // PYTHON_JIT=0|1 overrides the default + enabled = *env != '0'; } if (enabled) { -#else - // Always enable tier two for JIT builds (ignoring the environment - // variable and command-line option above): - if (true) { +#ifdef _Py_JIT + // perf profiler works fine with tier 2 interpreter, so + // only checking for a "real JIT". + if (config->perf_profiling > 0) { + (void)PyErr_WarnEx( + PyExc_RuntimeWarning, + "JIT deactivated as perf profiling support is active", + 0); + } else #endif - PyObject *opt = PyUnstable_Optimizer_NewUOpOptimizer(); - if (opt == NULL) { - return _PyStatus_ERR("can't initialize optimizer"); - } - if (PyUnstable_SetOptimizer((_PyOptimizerObject *)opt)) { - return _PyStatus_ERR("can't initialize optimizer"); + { + interp->jit = true; } - Py_DECREF(opt); } } +#endif if (!is_main_interp) { // The main interpreter is handled in Py_Main(), for now. @@ -1405,18 +1448,6 @@ Py_Initialize(void) } -PyStatus -_Py_InitializeMain(void) -{ - PyStatus status = _PyRuntime_Initialize(); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - PyThreadState *tstate = _PyThreadState_GET(); - return pyinit_main(tstate); -} - - static void finalize_modules_delete_special(PyThreadState *tstate, int verbose) { @@ -1627,10 +1658,11 @@ finalize_modules(PyThreadState *tstate) { PyInterpreterState *interp = tstate->interp; - // Invalidate all executors and turn off tier 2 optimizer + // Invalidate all executors and turn off JIT: + interp->jit = false; +#ifdef _Py_TIER2 _Py_Executors_InvalidateAll(interp, 0); - _PyOptimizerObject *old = _Py_SetOptimizer(interp, NULL); - Py_XDECREF(old); +#endif // Stop watching __builtin__ modifications PyDict_Unwatch(0, interp->builtins); @@ -1774,6 +1806,7 @@ flush_std_files(void) static void finalize_interp_types(PyInterpreterState *interp) { + _PyTypes_FiniExtTypes(interp); _PyUnicode_FiniTypes(interp); _PySys_FiniTypes(interp); _PyXI_FiniTypes(interp); @@ -1787,6 +1820,11 @@ finalize_interp_types(PyInterpreterState *interp) _PyTypes_FiniTypes(interp); _PyTypes_Fini(interp); +#ifdef Py_GIL_DISABLED + _PyObject_FinalizeUniqueIdPool(interp); +#endif + + _PyCode_Fini(interp); // Call _PyUnicode_ClearInterned() before _PyDict_Fini() since it uses // a dict internally. @@ -1797,7 +1835,7 @@ finalize_interp_types(PyInterpreterState *interp) #ifndef Py_GIL_DISABLED // With Py_GIL_DISABLED: // the freelists for the current thread state have already been cleared. - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + struct _Py_freelists *freelists = _Py_freelists_GET(); _PyObject_ClearFreeLists(freelists, 1); #endif @@ -1815,7 +1853,6 @@ finalize_interp_clear(PyThreadState *tstate) _PyXI_Fini(tstate->interp); _PyExc_ClearExceptionGroupType(tstate->interp); _Py_clear_generic_types(tstate->interp); - _PyDtoa_Fini(tstate->interp); /* Clear interpreter state and all thread states */ _PyInterpreterState_Clear(tstate); @@ -1837,6 +1874,9 @@ finalize_interp_clear(PyThreadState *tstate) finalize_interp_types(tstate->interp); + /* Finalize dtoa at last so that finalizers calling repr of float doesn't crash */ + _PyDtoa_Fini(tstate->interp); + /* Free any delayed free requests immediately */ _PyMem_FiniDelayed(tstate->interp); @@ -1862,20 +1902,73 @@ finalize_interp_delete(PyInterpreterState *interp) } -int -Py_FinalizeEx(void) +/* Conceptually, there isn't a good reason for Py_Finalize() + to be called in any other thread than the one where Py_Initialize() + was called. Consequently, it would make sense to fail if the thread + or thread state (or interpreter) don't match. However, such + constraints have never been enforced, and, as unlikely as it may be, + there may be users relying on the unconstrained behavior. Thus, + we do our best here to accommodate that possibility. */ + +static PyThreadState * +resolve_final_tstate(_PyRuntimeState *runtime) +{ + PyThreadState *main_tstate = runtime->main_tstate; + assert(main_tstate != NULL); + assert(main_tstate->thread_id == runtime->main_thread); + PyInterpreterState *main_interp = _PyInterpreterState_Main(); + assert(main_tstate->interp == main_interp); + + PyThreadState *tstate = _PyThreadState_GET(); + if (_Py_IsMainThread()) { + if (tstate != main_tstate) { + /* This implies that Py_Finalize() was called while + a non-main interpreter was active or while the main + tstate was temporarily swapped out with another. + Neither case should be allowed, but, until we get around + to fixing that (and Py_Exit()), we're letting it go. */ + (void)PyThreadState_Swap(main_tstate); + } + } + else { + /* This is another unfortunate case where Py_Finalize() was + called when it shouldn't have been. We can't simply switch + over to the main thread. At the least, however, we can make + sure the main interpreter is active. */ + if (!_Py_IsMainInterpreter(tstate->interp)) { + /* We don't go to the trouble of updating runtime->main_tstate + since it will be dead soon anyway. */ + main_tstate = + _PyThreadState_New(main_interp, _PyThreadState_WHENCE_FINI); + if (main_tstate != NULL) { + _PyThreadState_Bind(main_tstate); + (void)PyThreadState_Swap(main_tstate); + } + else { + /* Fall back to the current tstate. It's better than nothing. */ + main_tstate = tstate; + } + } + } + assert(main_tstate != NULL); + + /* We might want to warn if main_tstate->current_frame != NULL. */ + + return main_tstate; +} + +static int +_Py_Finalize(_PyRuntimeState *runtime) { int status = 0; - _PyRuntimeState *runtime = &_PyRuntime; + /* Bail out early if already finalized (or never initialized). */ if (!runtime->initialized) { return status; } - /* Get current thread state and interpreter pointer */ - PyThreadState *tstate = _PyThreadState_GET(); - // XXX assert(_Py_IsMainInterpreter(tstate->interp)); - // XXX assert(_Py_IsMainThread()); + /* Get final thread state pointer. */ + PyThreadState *tstate = resolve_final_tstate(runtime); // Block some operations. tstate->interp->finalizing = 1; @@ -1898,6 +1991,8 @@ Py_FinalizeEx(void) _PyAtExit_Call(tstate->interp); + assert(_PyThreadState_GET() == tstate); + /* Copy the core config, PyInterpreterState_Delete() free the core config memory */ #ifdef Py_REF_DEBUG @@ -1911,7 +2006,10 @@ Py_FinalizeEx(void) int malloc_stats = tstate->interp->config.malloc_stats; #endif - /* Remaining daemon threads will automatically exit + /* Ensure that remaining threads are detached */ + _PyEval_StopTheWorldAll(runtime); + + /* Remaining daemon threads will be trapped in PyThread_hang_thread when they attempt to take the GIL (ex: PyEval_RestoreThread()). */ _PyInterpreterState_SetFinalizing(tstate->interp, tstate); _PyRuntimeState_SetFinalizing(runtime, tstate); @@ -1927,8 +2025,11 @@ Py_FinalizeEx(void) will be called in the current Python thread. Since _PyRuntimeState_SetFinalizing() has been called, no other Python thread can take the GIL at this point: if they try, they will exit - immediately. */ - _PyThreadState_DeleteExcept(tstate); + immediately. We start the world once we are the only thread state left, + before we call destructors. */ + PyThreadState *list = _PyThreadState_RemoveExcept(tstate); + _PyEval_StartTheWorldAll(runtime); + _PyThreadState_DeleteList(list); /* At this point no Python code should be running at all. The only thread state left should be the main thread of the main @@ -1972,6 +2073,9 @@ Py_FinalizeEx(void) _PyImport_FiniExternal(tstate->interp); finalize_modules(tstate); + /* Clean up any lingering subinterpreters. */ + finalize_subinterpreters(); + /* Print debug stats if any */ _PyEval_Fini(); @@ -2054,6 +2158,7 @@ Py_FinalizeEx(void) finalize_interp_clear(tstate); + #ifdef Py_TRACE_REFS /* Display addresses (& refcnts) of all objects still alive. * An address can be used to find the repr of the object, printed @@ -2067,6 +2172,12 @@ Py_FinalizeEx(void) } #endif /* Py_TRACE_REFS */ +#ifdef WITH_PYMALLOC + if (malloc_stats) { + _PyObject_DebugMallocStats(stderr); + } +#endif + finalize_interp_delete(tstate->interp); #ifdef Py_REF_DEBUG @@ -2077,22 +2188,22 @@ Py_FinalizeEx(void) #endif _Py_FinalizeAllocatedBlocks(runtime); -#ifdef WITH_PYMALLOC - if (malloc_stats) { - _PyObject_DebugMallocStats(stderr); - } -#endif - call_ll_exitfuncs(runtime); _PyRuntime_Finalize(); return status; } +int +Py_FinalizeEx(void) +{ + return _Py_Finalize(&_PyRuntime); +} + void Py_Finalize(void) { - Py_FinalizeEx(); + (void)_Py_Finalize(&_PyRuntime); } @@ -2110,7 +2221,8 @@ Py_Finalize(void) */ static PyStatus -new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) +new_interpreter(PyThreadState **tstate_p, + const PyInterpreterConfig *config, long whence) { PyStatus status; @@ -2133,6 +2245,8 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) *tstate_p = NULL; return _PyStatus_OK(); } + _PyInterpreterState_SetWhence(interp, whence); + interp->_ready = 1; // XXX Might new_interpreter() have been called without the GIL held? PyThreadState *save_tstate = _PyThreadState_GET(); @@ -2170,6 +2284,13 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) goto error; } + // This could be done in init_interpreter() (in pystate.c) if it + // didn't depend on interp->feature_flags being set already. + status = _PyObject_InitState(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + // initialize the interp->obmalloc state. This must be done after // the settings are loaded (so that feature_flags are set) but before // any calls are made to obmalloc functions. @@ -2178,7 +2299,7 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) goto error; } - tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_INTERP); + tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_INIT); if (tstate == NULL) { status = _PyStatus_NO_MEMORY(); goto error; @@ -2221,15 +2342,17 @@ PyStatus Py_NewInterpreterFromConfig(PyThreadState **tstate_p, const PyInterpreterConfig *config) { - return new_interpreter(tstate_p, config); + long whence = _PyInterpreterState_WHENCE_CAPI; + return new_interpreter(tstate_p, config, whence); } PyThreadState * Py_NewInterpreter(void) { PyThreadState *tstate = NULL; + long whence = _PyInterpreterState_WHENCE_LEGACY_CAPI; const PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; - PyStatus status = new_interpreter(&tstate, &config); + PyStatus status = new_interpreter(&tstate, &config, whence); if (_PyStatus_EXCEPTION(status)) { Py_ExitStatusException(status); } @@ -2299,23 +2422,90 @@ _Py_IsInterpreterFinalizing(PyInterpreterState *interp) return finalizing != NULL; } +static void +finalize_subinterpreters(void) +{ + PyThreadState *final_tstate = _PyThreadState_GET(); + PyInterpreterState *main_interp = _PyInterpreterState_Main(); + assert(final_tstate->interp == main_interp); + _PyRuntimeState *runtime = main_interp->runtime; + struct pyinterpreters *interpreters = &runtime->interpreters; + + /* Get the first interpreter in the list. */ + HEAD_LOCK(runtime); + PyInterpreterState *interp = interpreters->head; + if (interp == main_interp) { + interp = interp->next; + } + HEAD_UNLOCK(runtime); + + /* Bail out if there are no subinterpreters left. */ + if (interp == NULL) { + return; + } + + /* Warn the user if they forgot to clean up subinterpreters. */ + (void)PyErr_WarnEx( + PyExc_RuntimeWarning, + "remaining subinterpreters; " + "destroy them with _interpreters.destroy()", + 0); + + /* Swap out the current tstate, which we know must belong + to the main interpreter. */ + _PyThreadState_Detach(final_tstate); + + /* Clean up all remaining subinterpreters. */ + while (interp != NULL) { + assert(!_PyInterpreterState_IsRunningMain(interp)); + + /* Find the tstate to use for fini. We assume the interpreter + will have at most one tstate at this point. */ + PyThreadState *tstate = interp->threads.head; + if (tstate != NULL) { + /* Ideally we would be able to use tstate as-is, and rely + on it being in a ready state: no exception set, not + running anything (tstate->current_frame), matching the + current thread ID (tstate->thread_id). To play it safe, + we always delete it and use a fresh tstate instead. */ + assert(tstate != final_tstate); + _PyThreadState_Attach(tstate); + PyThreadState_Clear(tstate); + _PyThreadState_Detach(tstate); + PyThreadState_Delete(tstate); + } + tstate = _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_FINI); + + /* Destroy the subinterpreter. */ + _PyThreadState_Attach(tstate); + Py_EndInterpreter(tstate); + assert(_PyThreadState_GET() == NULL); + + /* Advance to the next interpreter. */ + HEAD_LOCK(runtime); + interp = interpreters->head; + if (interp == main_interp) { + interp = interp->next; + } + HEAD_UNLOCK(runtime); + } + + /* Switch back to the main interpreter. */ + _PyThreadState_Attach(final_tstate); +} + + /* Add the __main__ module */ static PyStatus add_main_module(PyInterpreterState *interp) { - PyObject *m, *d, *ann_dict; - m = PyImport_AddModule("__main__"); + PyObject *m, *d; + m = PyImport_AddModuleObject(&_Py_ID(__main__)); if (m == NULL) return _PyStatus_ERR("can't create __main__ module"); d = PyModule_GetDict(m); - ann_dict = PyDict_New(); - if ((ann_dict == NULL) || - (PyDict_SetItemString(d, "__annotations__", ann_dict) < 0)) { - return _PyStatus_ERR("Failed to initialize __main__.__annotations__"); - } - Py_DECREF(ann_dict); int has_builtins = PyDict_ContainsString(d, "__builtins__"); if (has_builtins < 0) { @@ -2371,54 +2561,6 @@ init_import_site(void) return _PyStatus_OK(); } -/* Check if a file descriptor is valid or not. - Return 0 if the file descriptor is invalid, return non-zero otherwise. */ -static int -is_valid_fd(int fd) -{ -/* dup() is faster than fstat(): fstat() can require input/output operations, - whereas dup() doesn't. There is a low risk of EMFILE/ENFILE at Python - startup. Problem: dup() doesn't check if the file descriptor is valid on - some platforms. - - fcntl(fd, F_GETFD) is even faster, because it only checks the process table. - It is preferred over dup() when available, since it cannot fail with the - "too many open files" error (EMFILE). - - bpo-30225: On macOS Tiger, when stdout is redirected to a pipe and the other - side of the pipe is closed, dup(1) succeed, whereas fstat(1, &st) fails with - EBADF. FreeBSD has similar issue (bpo-32849). - - Only use dup() on Linux where dup() is enough to detect invalid FD - (bpo-32849). -*/ - if (fd < 0) { - return 0; - } -#if defined(F_GETFD) && ( \ - defined(__linux__) || \ - defined(__APPLE__) || \ - defined(__wasm__)) - return fcntl(fd, F_GETFD) >= 0; -#elif defined(__linux__) - int fd2 = dup(fd); - if (fd2 >= 0) { - close(fd2); - } - return (fd2 >= 0); -#elif defined(MS_WINDOWS) - HANDLE hfile; - _Py_BEGIN_SUPPRESS_IPH - hfile = (HANDLE)_get_osfhandle(fd); - _Py_END_SUPPRESS_IPH - return (hfile != INVALID_HANDLE_VALUE - && GetFileType(hfile) != FILE_TYPE_UNKNOWN); -#else - struct stat st; - return (fstat(fd, &st) == 0); -#endif -} - /* returns Py_None if the fd is not valid */ static PyObject* create_stdio(const PyConfig *config, PyObject* io, @@ -2432,8 +2574,9 @@ create_stdio(const PyConfig *config, PyObject* io, int buffering, isatty; const int buffered_stdio = config->buffered_stdio; - if (!is_valid_fd(fd)) + if (!_Py_IsValidFD(fd)) { Py_RETURN_NONE; + } /* stdin is always opened in buffered mode, first because it shouldn't make a difference in common use cases, second because TextIOWrapper @@ -2549,9 +2692,9 @@ create_stdio(const PyConfig *config, PyObject* io, Py_XDECREF(text); Py_XDECREF(raw); - if (PyErr_ExceptionMatches(PyExc_OSError) && !is_valid_fd(fd)) { + if (PyErr_ExceptionMatches(PyExc_OSError) && !_Py_IsValidFD(fd)) { /* Issue #24891: the file descriptor was closed after the first - is_valid_fd() check was called. Ignore the OSError and set the + _Py_IsValidFD() check was called. Ignore the OSError and set the stream to None. */ PyErr_Clear(); Py_RETURN_NONE; @@ -2690,6 +2833,136 @@ init_sys_streams(PyThreadState *tstate) } +#ifdef __ANDROID__ +#include + +static PyObject * +android_log_write_impl(PyObject *self, PyObject *args) +{ + int prio = 0; + const char *tag = NULL; + const char *text = NULL; + if (!PyArg_ParseTuple(args, "isy", &prio, &tag, &text)) { + return NULL; + } + + // Despite its name, this function is part of the public API + // (https://developer.android.com/ndk/reference/group/logging). + __android_log_write(prio, tag, text); + Py_RETURN_NONE; +} + + +static PyMethodDef android_log_write_method = { + "android_log_write", android_log_write_impl, METH_VARARGS +}; + + +static PyStatus +init_android_streams(PyThreadState *tstate) +{ + PyStatus status = _PyStatus_OK(); + PyObject *_android_support = NULL; + PyObject *android_log_write = NULL; + PyObject *result = NULL; + + _android_support = PyImport_ImportModule("_android_support"); + if (_android_support == NULL) { + goto error; + } + + android_log_write = PyCFunction_New(&android_log_write_method, NULL); + if (android_log_write == NULL) { + goto error; + } + + // These log priorities match those used by Java's System.out and System.err. + result = PyObject_CallMethod( + _android_support, "init_streams", "Oii", + android_log_write, ANDROID_LOG_INFO, ANDROID_LOG_WARN); + if (result == NULL) { + goto error; + } + + goto done; + +error: + _PyErr_Print(tstate); + status = _PyStatus_ERR("failed to initialize Android streams"); + +done: + Py_XDECREF(result); + Py_XDECREF(android_log_write); + Py_XDECREF(_android_support); + return status; +} + +#endif // __ANDROID__ + +#if defined(__APPLE__) && HAS_APPLE_SYSTEM_LOG + +static PyObject * +apple_log_write_impl(PyObject *self, PyObject *args) +{ + int logtype = 0; + const char *text = NULL; + if (!PyArg_ParseTuple(args, "iy", &logtype, &text)) { + return NULL; + } + + // Pass the user-provided text through explicit %s formatting + // to avoid % literals being interpreted as a formatting directive. + os_log_with_type(OS_LOG_DEFAULT, logtype, "%s", text); + Py_RETURN_NONE; +} + + +static PyMethodDef apple_log_write_method = { + "apple_log_write", apple_log_write_impl, METH_VARARGS +}; + + +static PyStatus +init_apple_streams(PyThreadState *tstate) +{ + PyStatus status = _PyStatus_OK(); + PyObject *_apple_support = NULL; + PyObject *apple_log_write = NULL; + PyObject *result = NULL; + + _apple_support = PyImport_ImportModule("_apple_support"); + if (_apple_support == NULL) { + goto error; + } + + apple_log_write = PyCFunction_New(&apple_log_write_method, NULL); + if (apple_log_write == NULL) { + goto error; + } + + // Initialize the logging streams, sending stdout -> Default; stderr -> Error + result = PyObject_CallMethod( + _apple_support, "init_streams", "Oii", + apple_log_write, OS_LOG_TYPE_DEFAULT, OS_LOG_TYPE_ERROR); + if (result == NULL) { + goto error; + } + goto done; + +error: + _PyErr_Print(tstate); + status = _PyStatus_ERR("failed to initialize Apple log streams"); + +done: + Py_XDECREF(result); + Py_XDECREF(apple_log_write); + Py_XDECREF(_apple_support); + return status; +} + +#endif // __APPLE__ && HAS_APPLE_SYSTEM_LOG + + static void _Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp, PyThreadState *tstate) @@ -2697,7 +2970,11 @@ _Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp, PUTS(fd, "\n"); /* display the current Python stack */ +#ifndef Py_GIL_DISABLED _Py_DumpTracebackThreads(fd, interp, tstate); +#else + _Py_DumpTraceback(fd, tstate); +#endif } /* Print the current exception (if an exception is set) with its traceback, @@ -2821,6 +3098,30 @@ fatal_error_exit(int status) } } +static inline int +acquire_dict_lock_for_dump(PyObject *obj) +{ +#ifdef Py_GIL_DISABLED + PyMutex *mutex = &obj->ob_mutex; + if (_PyMutex_LockTimed(mutex, 0, 0) == PY_LOCK_ACQUIRED) { + return 1; + } + return 0; +#else + return 1; +#endif +} + +static inline void +release_dict_lock_for_dump(PyObject *obj) +{ +#ifdef Py_GIL_DISABLED + PyMutex *mutex = &obj->ob_mutex; + // We can not call PyMutex_Unlock because it's not async-signal-safe. + // So not to wake up other threads, we just use a simple atomic store in here. + _Py_atomic_store_uint8(&mutex->_bits, _Py_UNLOCKED); +#endif +} // Dump the list of extension modules of sys.modules, excluding stdlib modules // (sys.stdlib_module_names), into fd file descriptor. @@ -2848,13 +3149,18 @@ _Py_DumpExtensionModules(int fd, PyInterpreterState *interp) PyObject *stdlib_module_names = NULL; if (interp->sysdict != NULL) { pos = 0; - while (PyDict_Next(interp->sysdict, &pos, &key, &value)) { + if (!acquire_dict_lock_for_dump(interp->sysdict)) { + // If we cannot acquire the lock, just don't dump the list of extension modules. + return; + } + while (_PyDict_Next(interp->sysdict, &pos, &key, &value, NULL)) { if (PyUnicode_Check(key) && PyUnicode_CompareWithASCIIString(key, "stdlib_module_names") == 0) { stdlib_module_names = value; break; } } + release_dict_lock_for_dump(interp->sysdict); } // If we failed to get sys.stdlib_module_names or it's not a frozenset, // don't exclude stdlib modules. @@ -2866,7 +3172,11 @@ _Py_DumpExtensionModules(int fd, PyInterpreterState *interp) int header = 1; Py_ssize_t count = 0; pos = 0; - while (PyDict_Next(modules, &pos, &key, &value)) { + if (!acquire_dict_lock_for_dump(modules)) { + // If we cannot acquire the lock, just don't dump the list of extension modules. + return; + } + while (_PyDict_Next(modules, &pos, &key, &value, NULL)) { if (!PyUnicode_Check(key)) { continue; } @@ -2881,6 +3191,7 @@ _Py_DumpExtensionModules(int fd, PyInterpreterState *interp) Py_ssize_t i = 0; PyObject *item; Py_hash_t hash; + // if stdlib_module_names is not NULL, it is always a frozenset. while (_PySet_NextEntry(stdlib_module_names, &i, &item, &hash)) { if (PyUnicode_Check(item) && PyUnicode_Compare(key, item) == 0) @@ -2906,6 +3217,7 @@ _Py_DumpExtensionModules(int fd, PyInterpreterState *interp) _Py_DumpASCII(fd, key); count++; } + release_dict_lock_for_dump(modules); if (count) { PUTS(fd, " (total: "); @@ -3135,7 +3447,11 @@ call_ll_exitfuncs(_PyRuntimeState *runtime) void _Py_NO_RETURN Py_Exit(int sts) { - if (Py_FinalizeEx() < 0) { + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate != NULL && _PyThreadState_IsRunningMain(tstate)) { + _PyInterpreterState_SetNotRunningMain(tstate->interp); + } + if (_Py_Finalize(&_PyRuntime) < 0) { sts = 120; } diff --git a/Python/pystate.c b/Python/pystate.c index 3d6394f81da816..e6770ef40df740 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2,23 +2,26 @@ /* Thread and interpreter state structures and their interfaces */ #include "Python.h" +#include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_audit.h" // _Py_AuditHookEntry #include "pycore_ceval.h" #include "pycore_code.h" // stats #include "pycore_critical_section.h" // _PyCriticalSection_Resume() #include "pycore_dtoa.h" // _dtoa_state_INIT() #include "pycore_emscripten_trampoline.h" // _Py_EmscriptenTrampoline_Init() #include "pycore_frame.h" +#include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_object.h" // _PyType_InitCache() -#include "pycore_object_stack.h" // _PyObjectStackChunk_ClearFreeList() #include "pycore_parking_lot.h" // _PyParkingLot_AfterFork() #include "pycore_pyerrors.h" // _PyErr_Clear() #include "pycore_pylifecycle.h" // _PyAST_Fini() #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" #include "pycore_runtime_init.h" // _PyRuntimeState_INIT -#include "pycore_sysmodule.h" // _PySys_Audit() +#include "pycore_stackref.h" // Py_STACKREF_DEBUG #include "pycore_obmalloc.h" // _PyMem_obmalloc_state_on_heap() +#include "pycore_uniqueid.h" // _PyObject_FinalizePerThreadRefcounts() /* -------------------------------------------------------------------------- CAUTION @@ -260,6 +263,12 @@ bind_tstate(PyThreadState *tstate) tstate->native_thread_id = PyThread_get_thread_native_id(); #endif +#ifdef Py_GIL_DISABLED + // Initialize biased reference counting inter-thread queue. Note that this + // needs to be initialized from the active thread. + _Py_brc_init_thread(tstate); +#endif + // mimalloc state needs to be initialized from the active thread. tstate_mimalloc_bind(tstate); @@ -382,20 +391,22 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS Note that we initialize "initial" relative to _PyRuntime, to ensure pre-initialized pointers point to the active runtime state (and not "initial"). */ -static const _PyRuntimeState initial = _PyRuntimeState_INIT(_PyRuntime); +static const _PyRuntimeState initial = _PyRuntimeState_INIT(_PyRuntime, ""); _Py_COMP_DIAG_POP #define LOCKS_INIT(runtime) \ { \ &(runtime)->interpreters.mutex, \ - &(runtime)->xi.registry.mutex, \ + &(runtime)->xi.data_lookup.registry.mutex, \ &(runtime)->unicode_state.ids.mutex, \ &(runtime)->imports.extensions.mutex, \ &(runtime)->ceval.pending_mainthread.mutex, \ + &(runtime)->ceval.sys_trace_profile_mutex, \ &(runtime)->atexit.mutex, \ &(runtime)->audit_hooks.mutex, \ &(runtime)->allocators.mutex, \ &(runtime)->_main_interpreter.types.mutex, \ + &(runtime)->_main_interpreter.code_state.mutex, \ } static void @@ -445,6 +456,8 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) // Py_Initialize() must be running again. // Reset to _PyRuntimeState_INIT. memcpy(runtime, &initial, sizeof(*runtime)); + // Preserve the cookie from the original runtime. + memcpy(runtime->debug_offsets.cookie, _Py_Debug_Cookie, 8); assert(!runtime->_initialized); } @@ -499,15 +512,18 @@ _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime) for (size_t i = 0; i < Py_ARRAY_LENGTH(locks); i++) { _PyMutex_at_fork_reinit(locks[i]); } +#ifdef Py_GIL_DISABLED + for (PyInterpreterState *interp = runtime->interpreters.head; + interp != NULL; interp = interp->next) + { + for (int i = 0; i < NUM_WEAKREF_LIST_LOCKS; i++) { + _PyMutex_at_fork_reinit(&interp->weakref_locks[i]); + } + } +#endif _PyTypes_AfterFork(); - /* bpo-42540: id_mutex is freed by _PyInterpreterState_Delete, which does - * not force the default allocator. */ - if (_PyThread_at_fork_reinit(&runtime->interpreters.main->id_mutex) < 0) { - return _PyStatus_ERR("Failed to reinitialize runtime locks"); - } - PyStatus status = gilstate_tss_reinit(runtime); if (_PyStatus_EXCEPTION(status)) { return status; @@ -566,7 +582,9 @@ free_interpreter(PyInterpreterState *interp) PyMem_RawFree(interp); } } - +#ifndef NDEBUG +static inline int check_interpreter_whence(long); +#endif /* Get the interpreter state to a minimal consistent state. Further init happens in pylifecycle.c before it can be used. All fields not initialized here are expected to be zeroed out, @@ -589,26 +607,33 @@ free_interpreter(PyInterpreterState *interp) static PyStatus init_interpreter(PyInterpreterState *interp, _PyRuntimeState *runtime, int64_t id, - PyInterpreterState *next) + PyInterpreterState *next, + long whence) { if (interp->_initialized) { return _PyStatus_ERR("interpreter already initialized"); } + assert(interp->_whence == _PyInterpreterState_WHENCE_NOTSET); + assert(check_interpreter_whence(whence) == 0); + interp->_whence = whence; + assert(runtime != NULL); interp->runtime = runtime; assert(id > 0 || (id == 0 && interp == runtime->interpreters.main)); interp->id = id; + interp->id_refcount = 0; + assert(runtime->interpreters.head == interp); assert(next != NULL || (interp == runtime->interpreters.main)); interp->next = next; - PyStatus status = _PyObject_InitState(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + interp->threads.preallocated = &interp->_initial_thread; + + // We would call _PyObject_InitState() at this point + // if interp->feature_flags were alredy set. _PyEval_InitState(interp); _PyGC_InitState(&interp->gc); @@ -626,16 +651,34 @@ init_interpreter(PyInterpreterState *interp, interp->monitoring_callables[t][e] = NULL; } + interp->monitoring_tool_versions[t] = 0; } interp->sys_profile_initialized = false; interp->sys_trace_initialized = false; - (void)_Py_SetOptimizer(interp, NULL); - interp->next_func_version = 1; + interp->jit = false; interp->executor_list_head = NULL; + interp->trace_run_counter = JIT_CLEANUP_THRESHOLD; if (interp != &runtime->_main_interpreter) { /* Fix the self-referential, statically initialized fields. */ interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp); } +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) + interp->next_stackref = 1; + _Py_hashtable_allocator_t alloc = { + .malloc = malloc, + .free = free, + }; + interp->stackref_debug_table = _Py_hashtable_new_full( + _Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct, + NULL, + NULL, + &alloc + ); + _Py_stackref_associate(interp, Py_None, PyStackRef_None); + _Py_stackref_associate(interp, Py_False, PyStackRef_False); + _Py_stackref_associate(interp, Py_True, PyStackRef_True); +#endif interp->_initialized = 1; return _PyStatus_OK(); @@ -703,8 +746,9 @@ _PyInterpreterState_New(PyThreadState *tstate, PyInterpreterState **pinterp) } interpreters->head = interp; + long whence = _PyInterpreterState_WHENCE_UNKNOWN; status = init_interpreter(interp, runtime, - id, old_head); + id, old_head, whence); if (_PyStatus_EXCEPTION(status)) { goto error; } @@ -740,6 +784,10 @@ PyInterpreterState_New(void) return interp; } +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) +extern void +_Py_stackref_report_leaks(PyInterpreterState *interp); +#endif static void interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) @@ -763,18 +811,15 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) } // Clear the current/main thread state last. - HEAD_LOCK(runtime); - PyThreadState *p = interp->threads.head; - HEAD_UNLOCK(runtime); - while (p != NULL) { + _Py_FOR_EACH_TSTATE_BEGIN(interp, p) { // See https://github.com/python/cpython/issues/102126 // Must be called without HEAD_LOCK held as it can deadlock // if any finalizer tries to acquire that lock. + HEAD_UNLOCK(runtime); PyThreadState_Clear(p); HEAD_LOCK(runtime); - p = p->next; - HEAD_UNLOCK(runtime); } + _Py_FOR_EACH_TSTATE_END(interp); if (tstate->interp == interp) { /* We fix tstate->_status below when we for sure aren't using it (e.g. no longer need the GIL). */ @@ -782,10 +827,6 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) tstate->_status.cleared = 0; } - _PyOptimizerObject *old = _Py_SetOptimizer(interp, NULL); - assert(old != NULL); - Py_DECREF(old); - /* It is possible that any of the objects below have a finalizer that runs Python code or otherwise relies on a thread state or even the interpreter state. For now we trust that isn't @@ -815,9 +856,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) } PyConfig_Clear(&interp->config); - Py_CLEAR(interp->codec_search_path); - Py_CLEAR(interp->codec_search_cache); - Py_CLEAR(interp->codec_error_registry); + _PyCodec_Fini(interp); assert(interp->imports.modules == NULL); assert(interp->imports.modules_by_index == NULL); @@ -853,6 +892,12 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->sysdict); Py_CLEAR(interp->builtins); +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) + _Py_stackref_report_leaks(interp); + _Py_hashtable_destroy(interp->stackref_debug_table); + interp->stackref_debug_table = NULL; +#endif + if (tstate->interp == interp) { /* We are now safe to fix tstate->_status.cleared. */ // XXX Do this (much) earlier? @@ -876,9 +921,17 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) interp->code_watchers[i] = NULL; } interp->active_code_watchers = 0; + + for (int i=0; i < CONTEXT_MAX_WATCHERS; i++) { + interp->context_watchers[i] = NULL; + } + interp->active_context_watchers = 0; // XXX Once we have one allocator per interpreter (i.e. // per-interpreter GC) we must ensure that all of the interpreter's // objects have been cleaned up at the point. + + // We could clear interp->threads.freelist here + // if it held more than just the initial thread state. } @@ -903,7 +956,7 @@ _PyInterpreterState_Clear(PyThreadState *tstate) static inline void tstate_deactivate(PyThreadState *tstate); -static void tstate_set_detached(PyThreadState *tstate); +static void tstate_set_detached(PyThreadState *tstate, int detached_state); static void zapthreads(PyInterpreterState *interp); void @@ -952,10 +1005,6 @@ PyInterpreterState_Delete(PyInterpreterState *interp) } HEAD_UNLOCK(runtime); - if (interp->id_mutex != NULL) { - PyThread_free_lock(interp->id_mutex); - } - _Py_qsbr_fini(interp); _PyObject_FiniState(interp); @@ -994,9 +1043,6 @@ _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime) // the "current" tstate to be set? PyInterpreterState_Clear(interp); // XXX must activate? zapthreads(interp); - if (interp->id_mutex != NULL) { - PyThread_free_lock(interp->id_mutex); - } PyInterpreterState *prev_interp = interp; interp = interp->next; free_interpreter(prev_interp); @@ -1011,11 +1057,29 @@ _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime) } #endif +static inline void +set_main_thread(PyInterpreterState *interp, PyThreadState *tstate) +{ + _Py_atomic_store_ptr_relaxed(&interp->threads.main, tstate); +} + +static inline PyThreadState * +get_main_thread(PyInterpreterState *interp) +{ + return _Py_atomic_load_ptr_relaxed(&interp->threads.main); +} + +void +_PyErr_SetInterpreterAlreadyRunning(void) +{ + PyErr_SetString(PyExc_InterpreterError, "interpreter already running"); +} int _PyInterpreterState_SetRunningMain(PyInterpreterState *interp) { - if (_PyInterpreterState_FailIfRunningMain(interp) < 0) { + if (get_main_thread(interp) != NULL) { + _PyErr_SetInterpreterAlreadyRunning(); return -1; } PyThreadState *tstate = current_fast_get(); @@ -1025,59 +1089,156 @@ _PyInterpreterState_SetRunningMain(PyInterpreterState *interp) "current tstate has wrong interpreter"); return -1; } - interp->threads.main = tstate; + set_main_thread(interp, tstate); + return 0; } void _PyInterpreterState_SetNotRunningMain(PyInterpreterState *interp) { - PyThreadState *tstate = interp->threads.main; - assert(tstate == current_fast_get()); - - if (tstate->on_delete != NULL) { - // The threading module was imported for the first time in this - // thread, so it was set as threading._main_thread. (See gh-75698.) - // The thread has finished running the Python program so we mark - // the thread object as finished. - assert(tstate->_whence != _PyThreadState_WHENCE_THREADING); - tstate->on_delete(tstate->on_delete_data); - tstate->on_delete = NULL; - tstate->on_delete_data = NULL; - } - - interp->threads.main = NULL; + assert(get_main_thread(interp) == current_fast_get()); + set_main_thread(interp, NULL); } int _PyInterpreterState_IsRunningMain(PyInterpreterState *interp) { - if (interp->threads.main != NULL) { - return 1; - } - // For now, we assume the main interpreter is always running. - if (_Py_IsMainInterpreter(interp)) { + if (get_main_thread(interp) != NULL) { return 1; } + // Embedders might not know to call _PyInterpreterState_SetRunningMain(), + // so their main thread wouldn't show it is running the main interpreter's + // program. (Py_Main() doesn't have this problem.) For now this isn't + // critical. If it were, we would need to infer "running main" from other + // information, like if it's the main interpreter. We used to do that + // but the naive approach led to some inconsistencies that caused problems. return 0; } int -_PyInterpreterState_FailIfRunningMain(PyInterpreterState *interp) +_PyThreadState_IsRunningMain(PyThreadState *tstate) { - if (interp->threads.main != NULL) { - PyErr_SetString(PyExc_RuntimeError, - "interpreter already running"); + PyInterpreterState *interp = tstate->interp; + // See the note in _PyInterpreterState_IsRunningMain() about + // possible false negatives here for embedders. + return get_main_thread(interp) == tstate; +} + +void +_PyInterpreterState_ReinitRunningMain(PyThreadState *tstate) +{ + PyInterpreterState *interp = tstate->interp; + if (get_main_thread(interp) != tstate) { + set_main_thread(interp, NULL); + } +} + + +//---------- +// accessors +//---------- + +int +_PyInterpreterState_IsReady(PyInterpreterState *interp) +{ + return interp->_ready; +} + +#ifndef NDEBUG +static inline int +check_interpreter_whence(long whence) +{ + if(whence < 0) { + return -1; + } + if (whence > _PyInterpreterState_WHENCE_MAX) { return -1; } return 0; } +#endif + +long +_PyInterpreterState_GetWhence(PyInterpreterState *interp) +{ + assert(check_interpreter_whence(interp->_whence) == 0); + return interp->_whence; +} + +void +_PyInterpreterState_SetWhence(PyInterpreterState *interp, long whence) +{ + assert(interp->_whence != _PyInterpreterState_WHENCE_NOTSET); + assert(check_interpreter_whence(whence) == 0); + interp->_whence = whence; +} + + +PyObject * +PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *interp) +{ + PyObject *modules = _PyImport_GetModules(interp); + if (modules == NULL) { + PyErr_SetString(PyExc_RuntimeError, "interpreter not initialized"); + return NULL; + } + return PyMapping_GetItemString(modules, "__main__"); +} + + +PyObject * +PyInterpreterState_GetDict(PyInterpreterState *interp) +{ + if (interp->dict == NULL) { + interp->dict = PyDict_New(); + if (interp->dict == NULL) { + PyErr_Clear(); + } + } + /* Returning NULL means no per-interpreter dict is available. */ + return interp->dict; +} //---------- -// accessors +// interp ID //---------- +int64_t +_PyInterpreterState_ObjectToID(PyObject *idobj) +{ + if (!_PyIndex_Check(idobj)) { + PyErr_Format(PyExc_TypeError, + "interpreter ID must be an int, got %.100s", + Py_TYPE(idobj)->tp_name); + return -1; + } + + // This may raise OverflowError. + // For now, we don't worry about if LLONG_MAX < INT64_MAX. + long long id = PyLong_AsLongLong(idobj); + if (id == -1 && PyErr_Occurred()) { + return -1; + } + + if (id < 0) { + PyErr_Format(PyExc_ValueError, + "interpreter ID must be a non-negative int, got %R", + idobj); + return -1; + } +#if LLONG_MAX > INT64_MAX + else if (id > INT64_MAX) { + PyErr_SetString(PyExc_OverflowError, "int too big to convert"); + return -1; + } +#endif + else { + return (int64_t)id; + } +} + int64_t PyInterpreterState_GetID(PyInterpreterState *interp) { @@ -1088,54 +1249,36 @@ PyInterpreterState_GetID(PyInterpreterState *interp) return interp->id; } - -int -_PyInterpreterState_IDInitref(PyInterpreterState *interp) +PyObject * +_PyInterpreterState_GetIDObject(PyInterpreterState *interp) { - if (interp->id_mutex != NULL) { - return 0; - } - interp->id_mutex = PyThread_allocate_lock(); - if (interp->id_mutex == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "failed to create init interpreter ID mutex"); - return -1; + int64_t interpid = interp->id; + if (interpid < 0) { + return NULL; } - interp->id_refcount = 0; - return 0; + assert(interpid < LLONG_MAX); + return PyLong_FromLongLong(interpid); } -int + +void _PyInterpreterState_IDIncref(PyInterpreterState *interp) { - if (_PyInterpreterState_IDInitref(interp) < 0) { - return -1; - } - - PyThread_acquire_lock(interp->id_mutex, WAIT_LOCK); - interp->id_refcount += 1; - PyThread_release_lock(interp->id_mutex); - return 0; + _Py_atomic_add_ssize(&interp->id_refcount, 1); } void _PyInterpreterState_IDDecref(PyInterpreterState *interp) { - assert(interp->id_mutex != NULL); _PyRuntimeState *runtime = interp->runtime; - PyThread_acquire_lock(interp->id_mutex, WAIT_LOCK); - assert(interp->id_refcount != 0); - interp->id_refcount -= 1; - int64_t refcount = interp->id_refcount; - PyThread_release_lock(interp->id_mutex); + Py_ssize_t refcount = _Py_atomic_add_ssize(&interp->id_refcount, -1); - if (refcount == 0 && interp->requires_idref) { - PyThreadState *tstate = _PyThreadState_New(interp, - _PyThreadState_WHENCE_INTERP); - _PyThreadState_Bind(tstate); + if (refcount == 1 && interp->requires_idref) { + PyThreadState *tstate = + _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_FINI); // XXX Possible GILState issues? PyThreadState *save_tstate = _PyThreadState_Swap(runtime, tstate); @@ -1156,30 +1299,6 @@ _PyInterpreterState_RequireIDRef(PyInterpreterState *interp, int required) interp->requires_idref = required ? 1 : 0; } -PyObject * -PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *interp) -{ - PyObject *modules = _PyImport_GetModules(interp); - if (modules == NULL) { - PyErr_SetString(PyExc_RuntimeError, "interpreter not initialized"); - return NULL; - } - return PyMapping_GetItemString(modules, "__main__"); -} - -PyObject * -PyInterpreterState_GetDict(PyInterpreterState *interp) -{ - if (interp->dict == NULL) { - interp->dict = PyDict_New(); - if (interp->dict == NULL) { - PyErr_Clear(); - } - } - /* Returning NULL means no per-interpreter dict is available. */ - return interp->dict; -} - //----------------------------- // look up an interpreter state @@ -1241,6 +1360,16 @@ _PyInterpreterState_LookUpID(int64_t requested_id) return interp; } +PyInterpreterState * +_PyInterpreterState_LookUpIDObject(PyObject *requested_id) +{ + int64_t id = _PyInterpreterState_ObjectToID(requested_id); + if (id < 0) { + return NULL; + } + return _PyInterpreterState_LookUpID(id); +} + /********************************/ /* the per-thread runtime state */ @@ -1279,22 +1408,45 @@ allocate_chunk(int size_in_bytes, _PyStackChunk* previous) return res; } +static void +reset_threadstate(_PyThreadStateImpl *tstate) +{ + // Set to _PyThreadState_INIT directly? + memcpy(tstate, + &initial._main_interpreter._initial_thread, + sizeof(*tstate)); +} + static _PyThreadStateImpl * -alloc_threadstate(void) +alloc_threadstate(PyInterpreterState *interp) { - return PyMem_RawCalloc(1, sizeof(_PyThreadStateImpl)); + _PyThreadStateImpl *tstate; + + // Try the preallocated tstate first. + tstate = _Py_atomic_exchange_ptr(&interp->threads.preallocated, NULL); + + // Fall back to the allocator. + if (tstate == NULL) { + tstate = PyMem_RawCalloc(1, sizeof(_PyThreadStateImpl)); + if (tstate == NULL) { + return NULL; + } + reset_threadstate(tstate); + } + return tstate; } static void free_threadstate(_PyThreadStateImpl *tstate) { + PyInterpreterState *interp = tstate->base.interp; // The initial thread state of the interpreter is allocated // as part of the interpreter state so should not be freed. - if (tstate == &tstate->base.interp->_initial_thread) { - // Restore to _PyThreadState_INIT. - memcpy(tstate, - &initial._main_interpreter._initial_thread, - sizeof(*tstate)); + if (tstate == &interp->_initial_thread) { + // Make it available again. + reset_threadstate(tstate); + assert(interp->threads.preallocated == NULL); + _Py_atomic_store_ptr(&interp->threads.preallocated, tstate); } else { PyMem_RawFree(tstate); @@ -1352,11 +1504,13 @@ init_threadstate(_PyThreadStateImpl *_tstate, tstate->datastack_limit = NULL; tstate->what_event = -1; tstate->previous_executor = NULL; + tstate->dict_global_version = 0; + + _tstate->asyncio_running_loop = NULL; + _tstate->asyncio_running_task = NULL; + + tstate->delete_later = NULL; -#ifdef Py_GIL_DISABLED - // Initialize biased reference counting inter-thread queue - _Py_brc_init_thread(tstate); -#endif llist_init(&_tstate->mem_free_queue); if (interp->stoptheworld.requested || _PyRuntime.stoptheworld.requested) { @@ -1384,63 +1538,43 @@ add_threadstate(PyInterpreterState *interp, PyThreadState *tstate, static PyThreadState * new_threadstate(PyInterpreterState *interp, int whence) { - _PyThreadStateImpl *tstate; - _PyRuntimeState *runtime = interp->runtime; - // We don't need to allocate a thread state for the main interpreter - // (the common case), but doing it later for the other case revealed a - // reentrancy problem (deadlock). So for now we always allocate before - // taking the interpreters lock. See GH-96071. - _PyThreadStateImpl *new_tstate = alloc_threadstate(); - int used_newtstate; - if (new_tstate == NULL) { + // Allocate the thread state. + _PyThreadStateImpl *tstate = alloc_threadstate(interp); + if (tstate == NULL) { return NULL; } + #ifdef Py_GIL_DISABLED Py_ssize_t qsbr_idx = _Py_qsbr_reserve(interp); if (qsbr_idx < 0) { - PyMem_RawFree(new_tstate); + free_threadstate(tstate); + return NULL; + } + int32_t tlbc_idx = _Py_ReserveTLBCIndex(interp); + if (tlbc_idx < 0) { + free_threadstate(tstate); return NULL; } #endif /* We serialize concurrent creation to protect global state. */ - HEAD_LOCK(runtime); + HEAD_LOCK(interp->runtime); + // Initialize the new thread state. interp->threads.next_unique_id += 1; uint64_t id = interp->threads.next_unique_id; + init_threadstate(tstate, interp, id, whence); - // Allocate the thread state and add it to the interpreter. + // Add the new thread state to the interpreter. PyThreadState *old_head = interp->threads.head; - if (old_head == NULL) { - // It's the interpreter's initial thread state. - used_newtstate = 0; - tstate = &interp->_initial_thread; - } - // XXX Re-use interp->_initial_thread if not in use? - else { - // Every valid interpreter must have at least one thread. - assert(id > 1); - assert(old_head->prev == NULL); - used_newtstate = 1; - tstate = new_tstate; - // Set to _PyThreadState_INIT. - memcpy(tstate, - &initial._main_interpreter._initial_thread, - sizeof(*tstate)); - } - - init_threadstate(tstate, interp, id, whence); add_threadstate(interp, (PyThreadState *)tstate, old_head); - HEAD_UNLOCK(runtime); - if (!used_newtstate) { - // Must be called with lock unlocked to avoid re-entrancy deadlock. - PyMem_RawFree(new_tstate); - } + HEAD_UNLOCK(interp->runtime); #ifdef Py_GIL_DISABLED // Must be called with lock unlocked to avoid lock ordering deadlocks. _Py_qsbr_register(tstate, interp, qsbr_idx); + tstate->tlbc_index = tlbc_idx; #endif return (PyThreadState *)tstate; @@ -1449,8 +1583,13 @@ new_threadstate(PyInterpreterState *interp, int whence) PyThreadState * PyThreadState_New(PyInterpreterState *interp) { - PyThreadState *tstate = new_threadstate(interp, - _PyThreadState_WHENCE_UNKNOWN); + return _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_UNKNOWN); +} + +PyThreadState * +_PyThreadState_NewBound(PyInterpreterState *interp, int whence) +{ + PyThreadState *tstate = new_threadstate(interp, whence); if (tstate) { bind_tstate(tstate); // This makes sure there's a gilstate tstate bound @@ -1502,6 +1641,7 @@ PyThreadState_Clear(PyThreadState *tstate) { assert(tstate->_status.initialized && !tstate->_status.cleared); assert(current_fast_get()->interp == tstate->interp); + assert(!_PyThreadState_IsRunningMain(tstate)); // XXX assert(!tstate->_status.bound || tstate->_status.unbound); tstate->_status.finalizing = 1; // just in case @@ -1528,6 +1668,11 @@ PyThreadState_Clear(PyThreadState *tstate) "PyThreadState_Clear: warning: thread still has a frame\n"); } + if (verbose && tstate->current_exception != NULL) { + fprintf(stderr, "PyThreadState_Clear: warning: thread has an exception set\n"); + _PyErr_Print(tstate); + } + /* At this point tstate shouldn't be used any more, neither to run Python code nor for other uses. @@ -1541,6 +1686,12 @@ PyThreadState_Clear(PyThreadState *tstate) /* Don't clear tstate->pyframe: it is a borrowed reference */ + Py_CLEAR(tstate->threading_local_key); + Py_CLEAR(tstate->threading_local_sentinel); + + Py_CLEAR(((_PyThreadStateImpl *)tstate)->asyncio_running_loop); + Py_CLEAR(((_PyThreadStateImpl *)tstate)->asyncio_running_task); + Py_CLEAR(tstate->dict); Py_CLEAR(tstate->async_exc); @@ -1570,23 +1721,21 @@ PyThreadState_Clear(PyThreadState *tstate) Py_CLEAR(tstate->context); - if (tstate->on_delete != NULL) { - // For the "main" thread of each interpreter, this is meant - // to be done in _PyInterpreterState_SetNotRunningMain(). - // That leaves threads created by the threading module, - // and any threads killed by forking. - // However, we also accommodate "main" threads that still - // don't call _PyInterpreterState_SetNotRunningMain() yet. - tstate->on_delete(tstate->on_delete_data); - } - #ifdef Py_GIL_DISABLED // Each thread should clear own freelists in free-threading builds. - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + struct _Py_freelists *freelists = _Py_freelists_GET(); _PyObject_ClearFreeLists(freelists, 1); + // Merge our thread-local refcounts into the type's own refcount and + // free our local refcount array. + _PyObject_FinalizePerThreadRefcounts((_PyThreadStateImpl *)tstate); + // Remove ourself from the biased reference counting table of threads. _Py_brc_remove_thread(tstate); + + // Release our thread-local copies of the bytecode for reuse by another + // thread + _Py_ClearTLBCIndex((_PyThreadStateImpl *)tstate); #endif // Merge our queue of pointers to be freed into the interpreter queue. @@ -1605,10 +1754,11 @@ decrement_stoptheworld_countdown(struct _stoptheworld_state *stw); /* Common code for PyThreadState_Delete() and PyThreadState_DeleteCurrent() */ static void -tstate_delete_common(PyThreadState *tstate) +tstate_delete_common(PyThreadState *tstate, int release_gil) { assert(tstate->_status.cleared && !tstate->_status.finalized); - assert(tstate->state != _Py_THREAD_ATTACHED); + tstate_verify_not_active(tstate); + assert(!_PyThreadState_IsRunningMain(tstate)); PyInterpreterState *interp = tstate->interp; if (interp == NULL) { @@ -1636,34 +1786,48 @@ tstate_delete_common(PyThreadState *tstate) decrement_stoptheworld_countdown(&runtime->stoptheworld); } } - HEAD_UNLOCK(runtime); -#ifdef Py_GIL_DISABLED - _Py_qsbr_unregister((_PyThreadStateImpl *)tstate); +#if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) + // Add our portion of the total refcount to the interpreter's total. + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + tstate->interp->object_state.reftotal += tstate_impl->reftotal; + tstate_impl->reftotal = 0; + assert(tstate_impl->refcounts.values == NULL); #endif + HEAD_UNLOCK(runtime); + // XXX Unbind in PyThreadState_Clear(), or earlier // (and assert not-equal here)? if (tstate->_status.bound_gilstate) { unbind_gilstate_tstate(tstate); } - unbind_tstate(tstate); + if (tstate->_status.bound) { + unbind_tstate(tstate); + } // XXX Move to PyThreadState_Clear()? clear_datastack(tstate); + if (release_gil) { + _PyEval_ReleaseLock(tstate->interp, tstate, 1); + } + +#ifdef Py_GIL_DISABLED + _Py_qsbr_unregister(tstate); +#endif + tstate->_status.finalized = 1; } static void zapthreads(PyInterpreterState *interp) { - PyThreadState *tstate; /* No need to lock the mutex here because this should only happen when the threads are all really dead (XXX famous last words). */ - while ((tstate = interp->threads.head) != NULL) { + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { tstate_verify_not_active(tstate); - tstate_delete_common(tstate); + tstate_delete_common(tstate, 0); free_threadstate((_PyThreadStateImpl *)tstate); } } @@ -1674,7 +1838,7 @@ PyThreadState_Delete(PyThreadState *tstate) { _Py_EnsureTstateNotNULL(tstate); tstate_verify_not_active(tstate); - tstate_delete_common(tstate); + tstate_delete_common(tstate, 0); free_threadstate((_PyThreadStateImpl *)tstate); } @@ -1686,10 +1850,8 @@ _PyThreadState_DeleteCurrent(PyThreadState *tstate) #ifdef Py_GIL_DISABLED _Py_qsbr_detach(((_PyThreadStateImpl *)tstate)->qsbr); #endif - tstate_set_detached(tstate); - tstate_delete_common(tstate); current_fast_clear(tstate->interp->runtime); - _PyEval_ReleaseLock(tstate->interp, NULL); + tstate_delete_common(tstate, 1); // release GIL as part of call free_threadstate((_PyThreadStateImpl *)tstate); } @@ -1701,24 +1863,29 @@ PyThreadState_DeleteCurrent(void) } -/* - * Delete all thread states except the one passed as argument. - * Note that, if there is a current thread state, it *must* be the one - * passed as argument. Also, this won't touch any other interpreters - * than the current one, since we don't know which thread state should - * be kept in those other interpreters. - */ -void -_PyThreadState_DeleteExcept(PyThreadState *tstate) +// Unlinks and removes all thread states from `tstate->interp`, with the +// exception of the one passed as an argument. However, it does not delete +// these thread states. Instead, it returns the removed thread states as a +// linked list. +// +// Note that if there is a current thread state, it *must* be the one +// passed as argument. Also, this won't touch any interpreters other +// than the current one, since we don't know which thread state should +// be kept in those other interpreters. +PyThreadState * +_PyThreadState_RemoveExcept(PyThreadState *tstate) { assert(tstate != NULL); PyInterpreterState *interp = tstate->interp; _PyRuntimeState *runtime = interp->runtime; +#ifdef Py_GIL_DISABLED + assert(runtime->stoptheworld.world_stopped); +#endif + HEAD_LOCK(runtime); /* Remove all thread states, except tstate, from the linked list of - thread states. This will allow calling PyThreadState_Clear() - without holding the lock. */ + thread states. */ PyThreadState *list = interp->threads.head; if (list == tstate) { list = tstate->next; @@ -1733,9 +1900,19 @@ _PyThreadState_DeleteExcept(PyThreadState *tstate) interp->threads.head = tstate; HEAD_UNLOCK(runtime); - /* Clear and deallocate all stale thread states. Even if this - executes Python code, we should be safe since it executes - in the current thread, not one of the stale threads. */ + return list; +} + +// Deletes the thread states in the linked list `list`. +// +// This is intended to be used in conjunction with _PyThreadState_RemoveExcept. +void +_PyThreadState_DeleteList(PyThreadState *list) +{ + // The world can't be stopped because we PyThreadState_Clear() can + // call destructors. + assert(!_PyRuntime.stoptheworld.world_stopped); + PyThreadState *p, *next; for (p = list; p; p = next) { next = p->next; @@ -1859,13 +2036,13 @@ tstate_try_attach(PyThreadState *tstate) } static void -tstate_set_detached(PyThreadState *tstate) +tstate_set_detached(PyThreadState *tstate, int detached_state) { - assert(tstate->state == _Py_THREAD_ATTACHED); + assert(_Py_atomic_load_int_relaxed(&tstate->state) == _Py_THREAD_ATTACHED); #ifdef Py_GIL_DISABLED - _Py_atomic_store_int(&tstate->state, _Py_THREAD_DETACHED); + _Py_atomic_store_int(&tstate->state, detached_state); #else - tstate->state = _Py_THREAD_DETACHED; + tstate->state = detached_state; #endif } @@ -1897,19 +2074,33 @@ _PyThreadState_Attach(PyThreadState *tstate) Py_FatalError("non-NULL old thread state"); } - _PyEval_AcquireLock(tstate); - // XXX assert(tstate_is_alive(tstate)); - current_fast_set(&_PyRuntime, tstate); - tstate_activate(tstate); + while (1) { + _PyEval_AcquireLock(tstate); - if (!tstate_try_attach(tstate)) { - tstate_wait_attach(tstate); - } + // XXX assert(tstate_is_alive(tstate)); + current_fast_set(&_PyRuntime, tstate); + tstate_activate(tstate); + + if (!tstate_try_attach(tstate)) { + tstate_wait_attach(tstate); + } #ifdef Py_GIL_DISABLED - _Py_qsbr_attach(((_PyThreadStateImpl *)tstate)->qsbr); + if (_PyEval_IsGILEnabled(tstate) && !tstate->_status.holds_gil) { + // The GIL was enabled between our call to _PyEval_AcquireLock() + // and when we attached (the GIL can't go from enabled to disabled + // here because only a thread holding the GIL can disable + // it). Detach and try again. + tstate_set_detached(tstate, _Py_THREAD_DETACHED); + tstate_deactivate(tstate); + current_fast_clear(&_PyRuntime); + continue; + } + _Py_qsbr_attach(((_PyThreadStateImpl *)tstate)->qsbr); #endif + break; + } // Resume previous critical section. This acquires the lock(s) from the // top-most critical section. @@ -1926,7 +2117,7 @@ static void detach_thread(PyThreadState *tstate, int detached_state) { // XXX assert(tstate_is_alive(tstate) && tstate_is_bound(tstate)); - assert(tstate->state == _Py_THREAD_ATTACHED); + assert(_Py_atomic_load_int_relaxed(&tstate->state) == _Py_THREAD_ATTACHED); assert(tstate == current_fast_get()); if (tstate->critical_section != 0) { _PyCriticalSection_SuspendAll(tstate); @@ -1935,9 +2126,9 @@ detach_thread(PyThreadState *tstate, int detached_state) _Py_qsbr_detach(((_PyThreadStateImpl *)tstate)->qsbr); #endif tstate_deactivate(tstate); - tstate_set_detached(tstate); + tstate_set_detached(tstate, detached_state); current_fast_clear(&_PyRuntime); - _PyEval_ReleaseLock(tstate->interp, tstate); + _PyEval_ReleaseLock(tstate->interp, tstate, 0); } void @@ -1951,7 +2142,7 @@ _PyThreadState_Suspend(PyThreadState *tstate) { _PyRuntimeState *runtime = &_PyRuntime; - assert(tstate->state == _Py_THREAD_ATTACHED); + assert(_Py_atomic_load_int_relaxed(&tstate->state) == _Py_THREAD_ATTACHED); struct _stoptheworld_state *stw = NULL; HEAD_LOCK(runtime); @@ -1991,7 +2182,7 @@ decrement_stoptheworld_countdown(struct _stoptheworld_state *stw) } #ifdef Py_GIL_DISABLED -// Interpreter for _Py_FOR_EACH_THREAD(). For global stop-the-world events, +// Interpreter for _Py_FOR_EACH_STW_INTERP(). For global stop-the-world events, // we start with the first interpreter and then iterate over all interpreters. // For per-interpreter stop-the-world events, we only operate on the one // interpreter. @@ -2006,10 +2197,9 @@ interp_for_stop_the_world(struct _stoptheworld_state *stw) // Loops over threads for a stop-the-world event. // For global: all threads in all interpreters // For per-interpreter: all threads in the interpreter -#define _Py_FOR_EACH_THREAD(stw, i, t) \ - for (i = interp_for_stop_the_world((stw)); \ - i != NULL; i = ((stw->is_global) ? i->next : NULL)) \ - for (t = i->threads.head; t; t = t->next) +#define _Py_FOR_EACH_STW_INTERP(stw, i) \ + for (PyInterpreterState *i = interp_for_stop_the_world((stw)); \ + i != NULL; i = ((stw->is_global) ? i->next : NULL)) // Try to transition threads atomically from the "detached" state to the @@ -2018,19 +2208,19 @@ static bool park_detached_threads(struct _stoptheworld_state *stw) { int num_parked = 0; - PyInterpreterState *i; - PyThreadState *t; - _Py_FOR_EACH_THREAD(stw, i, t) { - int state = _Py_atomic_load_int_relaxed(&t->state); - if (state == _Py_THREAD_DETACHED) { - // Atomically transition to "suspended" if in "detached" state. - if (_Py_atomic_compare_exchange_int(&t->state, - &state, _Py_THREAD_SUSPENDED)) { - num_parked++; + _Py_FOR_EACH_STW_INTERP(stw, i) { + _Py_FOR_EACH_TSTATE_UNLOCKED(i, t) { + int state = _Py_atomic_load_int_relaxed(&t->state); + if (state == _Py_THREAD_DETACHED) { + // Atomically transition to "suspended" if in "detached" state. + if (_Py_atomic_compare_exchange_int( + &t->state, &state, _Py_THREAD_SUSPENDED)) { + num_parked++; + } + } + else if (state == _Py_THREAD_ATTACHED && t != stw->requester) { + _Py_set_eval_breaker_bit(t, _PY_EVAL_PLEASE_STOP_BIT); } - } - else if (state == _Py_THREAD_ATTACHED && t != stw->requester) { - _Py_set_eval_breaker_bit(t, _PY_EVAL_PLEASE_STOP_BIT); } } stw->thread_countdown -= num_parked; @@ -2057,12 +2247,12 @@ stop_the_world(struct _stoptheworld_state *stw) stw->stop_event = (PyEvent){0}; // zero-initialize (unset) stw->requester = _PyThreadState_GET(); // may be NULL - PyInterpreterState *i; - PyThreadState *t; - _Py_FOR_EACH_THREAD(stw, i, t) { - if (t != stw->requester) { - // Count all the other threads (we don't wait on ourself). - stw->thread_countdown++; + _Py_FOR_EACH_STW_INTERP(stw, i) { + _Py_FOR_EACH_TSTATE_UNLOCKED(i, t) { + if (t != stw->requester) { + // Count all the other threads (we don't wait on ourself). + stw->thread_countdown++; + } } } @@ -2082,7 +2272,8 @@ stop_the_world(struct _stoptheworld_state *stw) } PyTime_t wait_ns = 1000*1000; // 1ms (arbitrary, may need tuning) - if (PyEvent_WaitTimed(&stw->stop_event, wait_ns)) { + int detach = 0; + if (PyEvent_WaitTimed(&stw->stop_event, wait_ns, detach)) { assert(stw->thread_countdown == 0); break; } @@ -2102,13 +2293,14 @@ start_the_world(struct _stoptheworld_state *stw) stw->requested = 0; stw->world_stopped = 0; // Switch threads back to the detached state. - PyInterpreterState *i; - PyThreadState *t; - _Py_FOR_EACH_THREAD(stw, i, t) { - if (t != stw->requester) { - assert(t->state == _Py_THREAD_SUSPENDED); - _Py_atomic_store_int(&t->state, _Py_THREAD_DETACHED); - _PyParkingLot_UnparkAll(&t->state); + _Py_FOR_EACH_STW_INTERP(stw, i) { + _Py_FOR_EACH_TSTATE_UNLOCKED(i, t) { + if (t != stw->requester) { + assert(_Py_atomic_load_int_relaxed(&t->state) == + _Py_THREAD_SUSPENDED); + _Py_atomic_store_int(&t->state, _Py_THREAD_DETACHED); + _PyParkingLot_UnparkAll(&t->state); + } } } stw->requester = NULL; @@ -2172,7 +2364,6 @@ _PyEval_StartTheWorld(PyInterpreterState *interp) int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) { - _PyRuntimeState *runtime = &_PyRuntime; PyInterpreterState *interp = _PyInterpreterState_GET(); /* Although the GIL is held, a few C API functions can be called @@ -2181,12 +2372,16 @@ PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) * list of thread states we're traversing, so to prevent that we lock * head_mutex for the duration. */ - HEAD_LOCK(runtime); - for (PyThreadState *tstate = interp->threads.head; tstate != NULL; tstate = tstate->next) { - if (tstate->thread_id != id) { - continue; + PyThreadState *tstate = NULL; + _Py_FOR_EACH_TSTATE_BEGIN(interp, t) { + if (t->thread_id == id) { + tstate = t; + break; } + } + _Py_FOR_EACH_TSTATE_END(interp); + if (tstate != NULL) { /* Tricky: we need to decref the current value * (if any) in tstate->async_exc, but that can in turn * allow arbitrary Python code to run, including @@ -2196,14 +2391,12 @@ PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) */ Py_XINCREF(exc); PyObject *old_exc = _Py_atomic_exchange_ptr(&tstate->async_exc, exc); - HEAD_UNLOCK(runtime); Py_XDECREF(old_exc); _Py_set_eval_breaker_bit(tstate, _PY_ASYNC_EXCEPTION_BIT); - return 1; } - HEAD_UNLOCK(runtime); - return 0; + + return tstate != NULL; } //--------------------------------- @@ -2339,11 +2532,11 @@ _PyThread_CurrentFrames(void) * Because these lists can mutate even when the GIL is held, we * need to grab head_mutex for the duration. */ + _PyEval_StopTheWorldAll(runtime); HEAD_LOCK(runtime); PyInterpreterState *i; for (i = runtime->interpreters.head; i != NULL; i = i->next) { - PyThreadState *t; - for (t = i->threads.head; t != NULL; t = t->next) { + _Py_FOR_EACH_TSTATE_UNLOCKED(i, t) { _PyInterpreterFrame *frame = t->current_frame; frame = _PyFrame_GetFirstComplete(frame); if (frame == NULL) { @@ -2372,6 +2565,7 @@ _PyThread_CurrentFrames(void) done: HEAD_UNLOCK(runtime); + _PyEval_StartTheWorldAll(runtime); return result; } @@ -2403,11 +2597,11 @@ _PyThread_CurrentExceptions(void) * Because these lists can mutate even when the GIL is held, we * need to grab head_mutex for the duration. */ + _PyEval_StopTheWorldAll(runtime); HEAD_LOCK(runtime); PyInterpreterState *i; for (i = runtime->interpreters.head; i != NULL; i = i->next) { - PyThreadState *t; - for (t = i->threads.head; t != NULL; t = t->next) { + _Py_FOR_EACH_TSTATE_UNLOCKED(i, t) { _PyErr_StackItem *err_info = _PyErr_GetTopmostException(t); if (err_info == NULL) { continue; @@ -2435,6 +2629,7 @@ _PyThread_CurrentExceptions(void) done: HEAD_UNLOCK(runtime); + _PyEval_StartTheWorldAll(runtime); return result; } @@ -2599,16 +2794,11 @@ PyGILState_Release(PyGILState_STATE oldstate) } /* We must hold the GIL and have our thread state current */ - /* XXX - remove the check - the assert should be fine, - but while this is very new (April 2003), the extra check - by release-only users can't hurt. - */ if (!holds_gil(tstate)) { _Py_FatalErrorFormat(__func__, "thread state %p must be current when releasing", tstate); } - assert(holds_gil(tstate)); --tstate->gilstate_counter; assert(tstate->gilstate_counter >= 0); /* illegal counter value */ @@ -2619,12 +2809,18 @@ PyGILState_Release(PyGILState_STATE oldstate) /* can't have been locked when we created it */ assert(oldstate == PyGILState_UNLOCKED); // XXX Unbind tstate here. + // gh-119585: `PyThreadState_Clear()` may call destructors that + // themselves use PyGILState_Ensure and PyGILState_Release, so make + // sure that gilstate_counter is not zero when calling it. + ++tstate->gilstate_counter; PyThreadState_Clear(tstate); + --tstate->gilstate_counter; /* Delete the thread-state. Note this releases the GIL too! * It's vital that the GIL be held here, to avoid shutdown * races; see bugs 225673 and 1061968 (that nasty bug has a * habit of coming back). */ + assert(tstate->gilstate_counter == 0); assert(current_fast_get() == tstate); _PyThreadState_DeleteCurrent(tstate); } @@ -2659,11 +2855,15 @@ _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, if (eval_frame == interp->eval_frame) { return; } +#ifdef _Py_TIER2 if (eval_frame != NULL) { _Py_Executors_InvalidateAll(interp, 1); } +#endif RARE_EVENT_INC(set_eval_frame_func); + _PyEval_StopTheWorld(interp); interp->eval_frame = eval_frame; + _PyEval_StartTheWorld(interp); } @@ -2674,24 +2874,9 @@ _PyInterpreterState_GetConfig(PyInterpreterState *interp) } -int -_PyInterpreterState_GetConfigCopy(PyConfig *config) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - - PyStatus status = _PyConfig_Copy(config, &interp->config); - if (PyStatus_Exception(status)) { - _PyErr_SetFromPyStatus(status); - return -1; - } - return 0; -} - - const PyConfig* _Py_GetConfig(void) { - assert(PyGILState_Check()); PyThreadState *tstate = current_fast_get(); _Py_EnsureTstateNotNULL(tstate); return _PyInterpreterState_GetConfig(tstate->interp); @@ -2839,6 +3024,7 @@ tstate_mimalloc_bind(PyThreadState *tstate) // the "backing" heap. mi_tld_t *tld = &mts->tld; _mi_tld_init(tld, &mts->heaps[_Py_MIMALLOC_HEAP_MEM]); + llist_init(&mts->page_list); // Exiting threads push any remaining in-use segments to the abandoned // pool to be re-claimed later by other threads. We use per-interpreter @@ -2865,10 +3051,18 @@ tstate_mimalloc_bind(PyThreadState *tstate) mts->heaps[i].debug_offset = (uint8_t)debug_offsets[i]; } + // Heaps that store Python objects should use QSBR to delay freeing + // mimalloc pages while there may be concurrent lock-free readers. + mts->heaps[_Py_MIMALLOC_HEAP_OBJECT].page_use_qsbr = true; + mts->heaps[_Py_MIMALLOC_HEAP_GC].page_use_qsbr = true; + mts->heaps[_Py_MIMALLOC_HEAP_GC_PRE].page_use_qsbr = true; + // By default, object allocations use _Py_MIMALLOC_HEAP_OBJECT. // _PyObject_GC_New() and similar functions temporarily override this to // use one of the GC heaps. mts->current_object_heap = &mts->heaps[_Py_MIMALLOC_HEAP_OBJECT]; + + _Py_atomic_store_int(&mts->initialized, 1); #endif } diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 16bf06f0e6cca2..7b74f613ed563b 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -43,7 +43,7 @@ _Py_parse_inf_or_nan(const char *p, char **endptr) s += 3; if (case_insensitive_match(s, "inity")) s += 5; - retval = negate ? -Py_HUGE_VAL : Py_HUGE_VAL; + retval = negate ? -Py_INFINITY : Py_INFINITY; } else if (case_insensitive_match(s, "nan")) { s += 3; @@ -286,7 +286,7 @@ _PyOS_ascii_strtod(const char *nptr, char **endptr) string, -1.0 is returned and again ValueError is raised. On overflow (e.g., when trying to convert '1e500' on an IEEE 754 machine), - if overflow_exception is NULL then +-Py_HUGE_VAL is returned, and no Python + if overflow_exception is NULL then +-Py_INFINITY is returned, and no Python exception is raised. Otherwise, overflow_exception should point to a Python exception, this exception will be raised, -1.0 will be returned, and *endptr will point just past the end of the converted value. @@ -842,7 +842,7 @@ char * PyOS_double_to_string(double val, */ - if (Py_IS_NAN(val) || Py_IS_INFINITY(val)) + if (isnan(val) || isinf(val)) /* 3 for 'inf'/'nan', 1 for sign, 1 for '\0' */ bufsize = 5; else { @@ -860,10 +860,10 @@ char * PyOS_double_to_string(double val, } /* Handle nan and inf. */ - if (Py_IS_NAN(val)) { + if (isnan(val)) { strcpy(buf, "nan"); t = Py_DTST_NAN; - } else if (Py_IS_INFINITY(val)) { + } else if (isinf(val)) { if (copysign(1., val) == 1.) strcpy(buf, "inf"); else @@ -1234,7 +1234,7 @@ char * PyOS_double_to_string(double val, case 'E': float_strings = uc_float_strings; format_code = 'e'; - /* Fall through. */ + _Py_FALLTHROUGH; case 'e': mode = 2; precision++; @@ -1244,7 +1244,7 @@ char * PyOS_double_to_string(double val, case 'F': float_strings = uc_float_strings; format_code = 'f'; - /* Fall through. */ + _Py_FALLTHROUGH; case 'f': mode = 3; break; @@ -1253,7 +1253,7 @@ char * PyOS_double_to_string(double val, case 'G': float_strings = uc_float_strings; format_code = 'g'; - /* Fall through. */ + _Py_FALLTHROUGH; case 'g': mode = 2; /* precision 0 makes no sense for 'g' format; interpret as 1 */ diff --git a/Python/pythonrun.c b/Python/pythonrun.c index f87c53fb28fbea..0da26ad3f9b4bd 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -13,6 +13,7 @@ #include "Python.h" #include "pycore_ast.h" // PyAST_mod2obj() +#include "pycore_audit.h" // _PySys_Audit() #include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_compile.h" // _PyAST_Compile() #include "pycore_interp.h" // PyInterpreterState.importlib @@ -22,7 +23,7 @@ #include "pycore_pylifecycle.h" // _Py_FdIsInteractive() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_pythonrun.h" // export _PyRun_InteractiveLoopObject() -#include "pycore_sysmodule.h" // _PySys_Audit() +#include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_traceback.h" // _PyTraceBack_Print() #include "errcode.h" // E_EOF @@ -83,8 +84,6 @@ _PyRun_AnyFileObject(FILE *fp, PyObject *filename, int closeit, return res; } - -/* Parse input from a file and execute it */ int PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags) @@ -282,11 +281,42 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename, PyObject *main_dict = PyModule_GetDict(main_module); // borrowed ref PyObject *res = run_mod(mod, filename, main_dict, main_dict, flags, arena, interactive_src, 1); + Py_INCREF(interactive_src); _PyArena_Free(arena); Py_DECREF(main_module); if (res == NULL) { + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *exc = _PyErr_GetRaisedException(tstate); + if (PyType_IsSubtype(Py_TYPE(exc), + (PyTypeObject *) PyExc_SyntaxError)) + { + /* fix "text" attribute */ + assert(interactive_src != NULL); + PyObject *xs = PyUnicode_Splitlines(interactive_src, 1); + if (xs == NULL) { + goto error; + } + PyObject *exc_lineno = PyObject_GetAttr(exc, &_Py_ID(lineno)); + if (exc_lineno == NULL) { + Py_DECREF(xs); + goto error; + } + int n = PyLong_AsInt(exc_lineno); + Py_DECREF(exc_lineno); + if (n <= 0 || n > PyList_GET_SIZE(xs)) { + Py_DECREF(xs); + goto error; + } + PyObject *line = PyList_GET_ITEM(xs, n - 1); + PyObject_SetAttr(exc, &_Py_ID(text), line); + Py_DECREF(xs); + } +error: + Py_DECREF(interactive_src); + _PyErr_SetRaisedException(tstate, exc); return -1; } + Py_DECREF(interactive_src); Py_DECREF(res); flush_io(); @@ -437,7 +467,7 @@ _PyRun_SimpleFileObject(FILE *fp, PyObject *filename, int closeit, fclose(fp); } - pyc_fp = _Py_fopen_obj(filename, "rb"); + pyc_fp = Py_fopen(filename, "rb"); if (pyc_fp == NULL) { fprintf(stderr, "python: Can't reopen .pyc file\n"); goto done; @@ -452,7 +482,7 @@ _PyRun_SimpleFileObject(FILE *fp, PyObject *filename, int closeit, v = run_pyc_file(pyc_fp, dict, dict, flags); } else { /* When running from stdin, leave __main__.__loader__ alone */ - if (PyUnicode_CompareWithASCIIString(filename, "") != 0 && + if ((!PyUnicode_Check(filename) || !PyUnicode_EqualToUTF8(filename, "")) && set_main_loader(dict, filename, "SourceFileLoader") < 0) { fprintf(stderr, "python: failed to set __main__.__loader__\n"); ret = -1; @@ -472,11 +502,11 @@ _PyRun_SimpleFileObject(FILE *fp, PyObject *filename, int closeit, done: if (set_file_name) { - if (PyDict_DelItemString(dict, "__file__")) { - PyErr_Clear(); + if (PyDict_PopString(dict, "__file__", NULL) < 0) { + PyErr_Print(); } - if (PyDict_DelItemString(dict, "__cached__")) { - PyErr_Clear(); + if (PyDict_PopString(dict, "__cached__", NULL) < 0) { + PyErr_Print(); } } Py_XDECREF(main_module); @@ -534,6 +564,30 @@ PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags) return _PyRun_SimpleStringFlagsWithName(command, NULL, flags); } +static int +parse_exit_code(PyObject *code, int *exitcode_p) +{ + if (PyLong_Check(code)) { + // gh-125842: Use a long long to avoid an overflow error when `long` + // is 32-bit. We still truncate the result to an int. + int exitcode = (int)PyLong_AsLongLong(code); + if (exitcode == -1 && PyErr_Occurred()) { + // On overflow or other error, clear the exception and use -1 + // as the exit code to match historical Python behavior. + PyErr_Clear(); + *exitcode_p = -1; + return 1; + } + *exitcode_p = exitcode; + return 1; + } + else if (code == Py_None) { + *exitcode_p = 0; + return 1; + } + return 0; +} + int _Py_HandleSystemExit(int *exitcode_p) { @@ -550,50 +604,40 @@ _Py_HandleSystemExit(int *exitcode_p) fflush(stdout); - int exitcode = 0; - PyObject *exc = PyErr_GetRaisedException(); - if (exc == NULL) { - goto done; - } - assert(PyExceptionInstance_Check(exc)); + assert(exc != NULL && PyExceptionInstance_Check(exc)); - /* The error code should be in the `code' attribute. */ PyObject *code = PyObject_GetAttr(exc, &_Py_ID(code)); - if (code) { + if (code == NULL) { + // If the exception has no 'code' attribute, print the exception below + PyErr_Clear(); + } + else if (parse_exit_code(code, exitcode_p)) { + Py_DECREF(code); + Py_CLEAR(exc); + return 1; + } + else { + // If code is not an int or None, print it below Py_SETREF(exc, code); - if (exc == Py_None) { - goto done; - } } - /* If we failed to dig out the 'code' attribute, - * just let the else clause below print the error. - */ - if (PyLong_Check(exc)) { - exitcode = (int)PyLong_AsLong(exc); + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *sys_stderr = _PySys_GetAttr(tstate, &_Py_ID(stderr)); + if (sys_stderr != NULL && sys_stderr != Py_None) { + if (PyFile_WriteObject(exc, sys_stderr, Py_PRINT_RAW) < 0) { + PyErr_Clear(); + } } else { - PyThreadState *tstate = _PyThreadState_GET(); - PyObject *sys_stderr = _PySys_GetAttr(tstate, &_Py_ID(stderr)); - /* We clear the exception here to avoid triggering the assertion - * in PyObject_Str that ensures it won't silently lose exception - * details. - */ - PyErr_Clear(); - if (sys_stderr != NULL && sys_stderr != Py_None) { - PyFile_WriteObject(exc, sys_stderr, Py_PRINT_RAW); - } else { - PyObject_Print(exc, stderr, Py_PRINT_RAW); - fflush(stderr); + if (PyObject_Print(exc, stderr, Py_PRINT_RAW) < 0) { + PyErr_Clear(); } - PySys_WriteStderr("\n"); - exitcode = 1; + fflush(stderr); } - -done: + PySys_WriteStderr("\n"); Py_CLEAR(exc); - *exitcode_p = exitcode; + *exitcode_p = 1; return 1; } @@ -1275,17 +1319,20 @@ run_eval_code_obj(PyThreadState *tstate, PyCodeObject *co, PyObject *globals, Py _PyRuntime.signals.unhandled_keyboard_interrupt = 0; /* Set globals['__builtins__'] if it doesn't exist */ - if (globals != NULL) { - int has_builtins = PyDict_ContainsString(globals, "__builtins__"); - if (has_builtins < 0) { + if (!globals || !PyDict_Check(globals)) { + PyErr_SetString(PyExc_SystemError, "globals must be a real dict"); + return NULL; + } + int has_builtins = PyDict_ContainsString(globals, "__builtins__"); + if (has_builtins < 0) { + return NULL; + } + if (!has_builtins) { + if (PyDict_SetItemString(globals, "__builtins__", + tstate->interp->builtins) < 0) + { return NULL; } - if (!has_builtins) { - if (PyDict_SetItemString(globals, "__builtins__", - tstate->interp->builtins) < 0) { - return NULL; - } - } } v = PyEval_EvalCode((PyObject*)co, globals, locals); @@ -1439,6 +1486,7 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, if (flags && (flags->cf_flags & PyCF_ONLY_AST)) { if ((flags->cf_flags & PyCF_OPTIMIZED_AST) == PyCF_OPTIMIZED_AST) { if (_PyCompile_AstOptimize(mod, filename, flags, optimize, arena) < 0) { + _PyArena_Free(arena); return NULL; } } diff --git a/Python/pytime.c b/Python/pytime.c index 90ef2eeb546f7f..c039fc98ce4bde 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -1,5 +1,6 @@ #include "Python.h" #include "pycore_time.h" // PyTime_t +#include "pycore_pystate.h" // _Py_AssertHoldsTstate() #include // gmtime_r() #ifdef HAVE_SYS_TIME_H @@ -55,6 +56,14 @@ #endif +#ifdef MS_WINDOWS +static _PyTimeFraction py_qpc_base = {0, 0}; + +// Forward declaration +static int py_win_perf_counter_frequency(_PyTimeFraction *base, int raise_exc); +#endif + + static PyTime_t _PyTime_GCD(PyTime_t x, PyTime_t y) { @@ -367,7 +376,7 @@ pytime_object_to_denominator(PyObject *obj, time_t *sec, long *numerator, if (PyFloat_Check(obj)) { double d = PyFloat_AsDouble(obj); - if (Py_IS_NAN(d)) { + if (isnan(d)) { *numerator = 0; PyErr_SetString(PyExc_ValueError, "Invalid value NaN (not a number)"); return -1; @@ -379,6 +388,10 @@ pytime_object_to_denominator(PyObject *obj, time_t *sec, long *numerator, *sec = _PyLong_AsTime_t(obj); *numerator = 0; if (*sec == (time_t)-1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_Format(PyExc_TypeError, + "argument must be int or float, not %T", obj); + } return -1; } return 0; @@ -395,7 +408,7 @@ _PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) volatile double d; d = PyFloat_AsDouble(obj); - if (Py_IS_NAN(d)) { + if (isnan(d)) { PyErr_SetString(PyExc_ValueError, "Invalid value NaN (not a number)"); return -1; } @@ -565,6 +578,7 @@ pytime_from_double(PyTime_t *tp, double value, _PyTime_round_t round, /* See comments in pytime_double_to_denominator */ if (!((double)PyTime_MIN <= d && d < -(double)PyTime_MIN)) { pytime_time_t_overflow(); + *tp = 0; return -1; } PyTime_t ns = (PyTime_t)d; @@ -581,7 +595,7 @@ pytime_from_object(PyTime_t *tp, PyObject *obj, _PyTime_round_t round, if (PyFloat_Check(obj)) { double d; d = PyFloat_AsDouble(obj); - if (Py_IS_NAN(d)) { + if (isnan(d)) { PyErr_SetString(PyExc_ValueError, "Invalid value NaN (not a number)"); return -1; } @@ -652,14 +666,10 @@ _PyTime_AsLong(PyTime_t ns) return PyLong_FromLongLong((long long)ns); } -PyTime_t -_PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round) +int +_PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round, PyTime_t *result) { - PyTime_t tp; - if(pytime_from_double(&tp, seconds, round, SEC_TO_NS) < 0) { - return -1; - } - return tp; + return pytime_from_double(result, seconds, round, SEC_TO_NS); } @@ -888,17 +898,21 @@ _PyTime_AsTimespec(PyTime_t t, struct timespec *ts) #endif -// N.B. If raise_exc=0, this may be called without the GIL. +// N.B. If raise_exc=0, this may be called without a thread state. static int py_get_system_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) { assert(info == NULL || raise_exc); + if (raise_exc) { + // raise_exc requires to hold a thread state + _Py_AssertHoldsTstate(); + } #ifdef MS_WINDOWS FILETIME system_time; ULARGE_INTEGER large; - GetSystemTimeAsFileTime(&system_time); + GetSystemTimePreciseAsFileTime(&system_time); large.u.LowPart = system_time.dwLowDateTime; large.u.HighPart = system_time.dwHighDateTime; /* 11,644,473,600,000,000,000: number of nanoseconds between @@ -907,18 +921,17 @@ py_get_system_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) PyTime_t ns = large.QuadPart * 100 - 11644473600000000000; *tp = ns; if (info) { - DWORD timeAdjustment, timeIncrement; - BOOL isTimeAdjustmentDisabled, ok; + // GetSystemTimePreciseAsFileTime() is implemented using + // QueryPerformanceCounter() internally. + if (py_qpc_base.denom == 0) { + if (py_win_perf_counter_frequency(&py_qpc_base, raise_exc) < 0) { + return -1; + } + } - info->implementation = "GetSystemTimeAsFileTime()"; + info->implementation = "GetSystemTimePreciseAsFileTime()"; info->monotonic = 0; - ok = GetSystemTimeAdjustment(&timeAdjustment, &timeIncrement, - &isTimeAdjustmentDisabled); - if (!ok) { - PyErr_SetFromWindowsErr(0); - return -1; - } - info->resolution = timeIncrement * 1e-7; + info->resolution = _PyTimeFraction_Resolution(&py_qpc_base); info->adjustable = 1; } @@ -1000,29 +1013,28 @@ py_get_system_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) } -PyTime_t -_PyTime_TimeUnchecked(void) +int +PyTime_Time(PyTime_t *result) { - PyTime_t t; - if (py_get_system_clock(&t, NULL, 0) < 0) { - // If clock_gettime(CLOCK_REALTIME) or gettimeofday() fails: - // silently ignore the failure and return 0. - t = 0; + if (py_get_system_clock(result, NULL, 1) < 0) { + *result = 0; + return -1; } - return t; + return 0; } int -PyTime_Time(PyTime_t *result) +PyTime_TimeRaw(PyTime_t *result) { - if (py_get_system_clock(result, NULL, 1) < 0) { + if (py_get_system_clock(result, NULL, 0) < 0) { *result = 0; return -1; } return 0; } + int _PyTime_TimeWithInfo(PyTime_t *t, _Py_clock_info_t *info) { @@ -1030,9 +1042,75 @@ _PyTime_TimeWithInfo(PyTime_t *t, _Py_clock_info_t *info) } +#ifdef MS_WINDOWS +static int +py_win_perf_counter_frequency(_PyTimeFraction *base, int raise_exc) +{ + LARGE_INTEGER freq; + // Since Windows XP, the function cannot fail. + (void)QueryPerformanceFrequency(&freq); + LONGLONG frequency = freq.QuadPart; + + // Since Windows XP, frequency cannot be zero. + assert(frequency >= 1); + + Py_BUILD_ASSERT(sizeof(PyTime_t) == sizeof(frequency)); + PyTime_t denom = (PyTime_t)frequency; + + // Known QueryPerformanceFrequency() values: + // + // * 10,000,000 (10 MHz): 100 ns resolution + // * 3,579,545 Hz (3.6 MHz): 279 ns resolution + if (_PyTimeFraction_Set(base, SEC_TO_NS, denom) < 0) { + if (raise_exc) { + PyErr_SetString(PyExc_RuntimeError, + "invalid QueryPerformanceFrequency"); + } + return -1; + } + return 0; +} + + +// N.B. If raise_exc=0, this may be called without the GIL. +static int +py_get_win_perf_counter(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) +{ + assert(info == NULL || raise_exc); + + if (py_qpc_base.denom == 0) { + if (py_win_perf_counter_frequency(&py_qpc_base, raise_exc) < 0) { + return -1; + } + } + + if (info) { + info->implementation = "QueryPerformanceCounter()"; + info->resolution = _PyTimeFraction_Resolution(&py_qpc_base); + info->monotonic = 1; + info->adjustable = 0; + } + + LARGE_INTEGER now; + QueryPerformanceCounter(&now); + LONGLONG ticksll = now.QuadPart; + + /* Make sure that casting LONGLONG to PyTime_t cannot overflow, + both types are signed */ + PyTime_t ticks; + static_assert(sizeof(ticksll) <= sizeof(ticks), + "LONGLONG is larger than PyTime_t"); + ticks = (PyTime_t)ticksll; + + *tp = _PyTimeFraction_Mul(ticks, &py_qpc_base); + return 0; +} +#endif // MS_WINDOWS + + #ifdef __APPLE__ static int -py_mach_timebase_info(_PyTimeFraction *base, int raise) +py_mach_timebase_info(_PyTimeFraction *base, int raise_exc) { mach_timebase_info_data_t timebase; // According to the Technical Q&A QA1398, mach_timebase_info() cannot @@ -1054,7 +1132,7 @@ py_mach_timebase_info(_PyTimeFraction *base, int raise) // * (1000000000, 33333335) on PowerPC: ~30 ns // * (1000000000, 25000000) on PowerPC: 40 ns if (_PyTimeFraction_Set(base, numer, denom) < 0) { - if (raise) { + if (raise_exc) { PyErr_SetString(PyExc_RuntimeError, "invalid mach_timebase_info"); } @@ -1065,49 +1143,20 @@ py_mach_timebase_info(_PyTimeFraction *base, int raise) #endif -// N.B. If raise_exc=0, this may be called without the GIL. +// N.B. If raise_exc=0, this may be called without a thread state. static int py_get_monotonic_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) { assert(info == NULL || raise_exc); - -#if defined(MS_WINDOWS) - ULONGLONG ticks = GetTickCount64(); - static_assert(sizeof(ticks) <= sizeof(PyTime_t), - "ULONGLONG is larger than PyTime_t"); - PyTime_t t; - if (ticks <= (ULONGLONG)PyTime_MAX) { - t = (PyTime_t)ticks; - } - else { - // GetTickCount64() maximum is larger than PyTime_t maximum: - // ULONGLONG is unsigned, whereas PyTime_t is signed. - t = PyTime_MAX; + if (raise_exc) { + // raise_exc requires to hold a thread state + _Py_AssertHoldsTstate(); } - int res = pytime_mul(&t, MS_TO_NS); - *tp = t; - - if (raise_exc && res < 0) { - pytime_overflow(); +#if defined(MS_WINDOWS) + if (py_get_win_perf_counter(tp, info, raise_exc) < 0) { return -1; } - - if (info) { - DWORD timeAdjustment, timeIncrement; - BOOL isTimeAdjustmentDisabled, ok; - info->implementation = "GetTickCount64()"; - info->monotonic = 1; - ok = GetSystemTimeAdjustment(&timeAdjustment, &timeIncrement, - &isTimeAdjustmentDisabled); - if (!ok) { - PyErr_SetFromWindowsErr(0); - return -1; - } - info->resolution = timeIncrement * 1e-7; - info->adjustable = 0; - } - #elif defined(__APPLE__) static _PyTimeFraction base = {0, 0}; if (base.denom == 0) { @@ -1188,19 +1237,6 @@ py_get_monotonic_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) } -PyTime_t -_PyTime_MonotonicUnchecked(void) -{ - PyTime_t t; - if (py_get_monotonic_clock(&t, NULL, 0) < 0) { - // If mach_timebase_info(), clock_gettime() or gethrtime() fails: - // silently ignore the failure and return 0. - t = 0; - } - return t; -} - - int PyTime_Monotonic(PyTime_t *result) { @@ -1213,128 +1249,41 @@ PyTime_Monotonic(PyTime_t *result) int -_PyTime_MonotonicWithInfo(PyTime_t *tp, _Py_clock_info_t *info) -{ - return py_get_monotonic_clock(tp, info, 1); -} - - -#ifdef MS_WINDOWS -static int -py_win_perf_counter_frequency(_PyTimeFraction *base, int raise) +PyTime_MonotonicRaw(PyTime_t *result) { - LONGLONG frequency; - - LARGE_INTEGER freq; - // Since Windows XP, the function cannot fail. - (void)QueryPerformanceFrequency(&freq); - frequency = freq.QuadPart; - - // Since Windows XP, frequency cannot be zero. - assert(frequency >= 1); - - Py_BUILD_ASSERT(sizeof(PyTime_t) == sizeof(frequency)); - PyTime_t denom = (PyTime_t)frequency; - - // Known QueryPerformanceFrequency() values: - // - // * 10,000,000 (10 MHz): 100 ns resolution - // * 3,579,545 Hz (3.6 MHz): 279 ns resolution - if (_PyTimeFraction_Set(base, SEC_TO_NS, denom) < 0) { - if (raise) { - PyErr_SetString(PyExc_RuntimeError, - "invalid QueryPerformanceFrequency"); - } + if (py_get_monotonic_clock(result, NULL, 0) < 0) { + *result = 0; return -1; } return 0; } -// N.B. If raise_exc=0, this may be called without the GIL. -static int -py_get_win_perf_counter(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) +int +_PyTime_MonotonicWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { - assert(info == NULL || raise_exc); - - static _PyTimeFraction base = {0, 0}; - if (base.denom == 0) { - if (py_win_perf_counter_frequency(&base, raise_exc) < 0) { - return -1; - } - } - - if (info) { - info->implementation = "QueryPerformanceCounter()"; - info->resolution = _PyTimeFraction_Resolution(&base); - info->monotonic = 1; - info->adjustable = 0; - } - - LARGE_INTEGER now; - QueryPerformanceCounter(&now); - LONGLONG ticksll = now.QuadPart; - - /* Make sure that casting LONGLONG to PyTime_t cannot overflow, - both types are signed */ - PyTime_t ticks; - static_assert(sizeof(ticksll) <= sizeof(ticks), - "LONGLONG is larger than PyTime_t"); - ticks = (PyTime_t)ticksll; - - PyTime_t ns = _PyTimeFraction_Mul(ticks, &base); - *tp = ns; - return 0; + return py_get_monotonic_clock(tp, info, 1); } -#endif // MS_WINDOWS int _PyTime_PerfCounterWithInfo(PyTime_t *t, _Py_clock_info_t *info) { -#ifdef MS_WINDOWS - return py_get_win_perf_counter(t, info, 1); -#else return _PyTime_MonotonicWithInfo(t, info); -#endif } -PyTime_t -_PyTime_PerfCounterUnchecked(void) +int +PyTime_PerfCounter(PyTime_t *result) { - PyTime_t t; - int res; -#ifdef MS_WINDOWS - res = py_get_win_perf_counter(&t, NULL, 0); -#else - res = py_get_monotonic_clock(&t, NULL, 0); -#endif - if (res < 0) { - // If py_win_perf_counter_frequency() or py_get_monotonic_clock() - // fails: silently ignore the failure and return 0. - t = 0; - } - return t; + return PyTime_Monotonic(result); } int -PyTime_PerfCounter(PyTime_t *result) +PyTime_PerfCounterRaw(PyTime_t *result) { - int res; -#ifdef MS_WINDOWS - res = py_get_win_perf_counter(result, NULL, 1); -#else - res = py_get_monotonic_clock(result, NULL, 1); -#endif - if (res < 0) { - // If py_win_perf_counter_frequency() or py_get_monotonic_clock() - // fails: silently ignore the failure and return 0. - *result = 0; - return -1; - } - return 0; + return PyTime_MonotonicRaw(result); } @@ -1408,7 +1357,9 @@ _PyTime_gmtime(time_t t, struct tm *tm) PyTime_t _PyDeadline_Init(PyTime_t timeout) { - PyTime_t now = _PyTime_MonotonicUnchecked(); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); return _PyTime_Add(now, timeout); } @@ -1416,6 +1367,8 @@ _PyDeadline_Init(PyTime_t timeout) PyTime_t _PyDeadline_Get(PyTime_t deadline) { - PyTime_t now = _PyTime_MonotonicUnchecked(); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); return deadline - now; } diff --git a/Python/qsbr.c b/Python/qsbr.c index 7f7ae03cf60d22..a40219acfe2c29 100644 --- a/Python/qsbr.c +++ b/Python/qsbr.c @@ -2,7 +2,7 @@ * Implementation of safe memory reclamation scheme using * quiescent states. * - * This is dervied from the "GUS" safe memory reclamation technique + * This is derived from the "GUS" safe memory reclamation technique * in FreeBSD written by Jeffrey Roberson. It is heavily modified. Any bugs * in this code are likely due to the modifications. * @@ -38,12 +38,6 @@ #include "pycore_pystate.h" // _PyThreadState_GET() -// Wrap-around safe comparison. This is a holdover from the FreeBSD -// implementation, which uses 32-bit sequence numbers. We currently use 64-bit -// sequence numbers, so wrap-around is unlikely. -#define QSBR_LT(a, b) ((int64_t)((a)-(b)) < 0) -#define QSBR_LEQ(a, b) ((int64_t)((a)-(b)) <= 0) - // Starting size of the array of qsbr thread states #define MIN_ARRAY_SIZE 8 @@ -166,14 +160,13 @@ qsbr_poll_scan(struct _qsbr_shared *shared) bool _Py_qsbr_poll(struct _qsbr_thread_state *qsbr, uint64_t goal) { - assert(_PyThreadState_GET()->state == _Py_THREAD_ATTACHED); + assert(_Py_atomic_load_int_relaxed(&_PyThreadState_GET()->state) == _Py_THREAD_ATTACHED); - uint64_t rd_seq = _Py_atomic_load_uint64(&qsbr->shared->rd_seq); - if (QSBR_LEQ(goal, rd_seq)) { + if (_Py_qbsr_goal_reached(qsbr, goal)) { return true; } - rd_seq = qsbr_poll_scan(qsbr->shared); + uint64_t rd_seq = qsbr_poll_scan(qsbr->shared); return QSBR_LEQ(goal, rd_seq); } @@ -239,16 +232,26 @@ _Py_qsbr_register(_PyThreadStateImpl *tstate, PyInterpreterState *interp, } void -_Py_qsbr_unregister(_PyThreadStateImpl *tstate) +_Py_qsbr_unregister(PyThreadState *tstate) { - struct _qsbr_thread_state *qsbr = tstate->qsbr; - struct _qsbr_shared *shared = qsbr->shared; + struct _qsbr_shared *shared = &tstate->interp->qsbr; + struct _PyThreadStateImpl *tstate_imp = (_PyThreadStateImpl*) tstate; - assert(qsbr->seq == 0 && "thread state must be detached"); + // gh-119369: GIL must be released (if held) to prevent deadlocks, because + // we might not have an active tstate, which means that blocking on PyMutex + // locks will not implicitly release the GIL. + assert(!tstate->_status.holds_gil); PyMutex_Lock(&shared->mutex); - assert(qsbr->allocated && qsbr->tstate == (PyThreadState *)tstate); - tstate->qsbr = NULL; + // NOTE: we must load (or reload) the thread state's qbsr inside the mutex + // because the array may have been resized (changing tstate->qsbr) while + // we waited to acquire the mutex. + struct _qsbr_thread_state *qsbr = tstate_imp->qsbr; + + assert(qsbr->seq == 0 && "thread state must be detached"); + assert(qsbr->allocated && qsbr->tstate == tstate); + + tstate_imp->qsbr = NULL; qsbr->tstate = NULL; qsbr->allocated = false; qsbr->freelist_next = shared->freelist; diff --git a/Python/specialize.c b/Python/specialize.c index f83d8a9ceb0282..abb130d73eeebd 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -3,6 +3,7 @@ #include "opcode.h" #include "pycore_code.h" +#include "pycore_critical_section.h" #include "pycore_descrobject.h" // _PyMethodWrapper_Type #include "pycore_dict.h" // DICT_KEYS_UNICODE #include "pycore_function.h" // _PyFunction_GetVersionForCurrentState() @@ -11,6 +12,7 @@ #include "pycore_object.h" #include "pycore_opcode_metadata.h" // _PyOpcode_Caches #include "pycore_uop_metadata.h" // _PyOpcode_uop_name +#include "pycore_uop_ids.h" // MAX_UOP_ID #include "pycore_opcode_utils.h" // RESUME_AT_FUNC_START #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() #include "pycore_runtime.h" // _Py_ID() @@ -20,7 +22,7 @@ extern const char *_PyUOpName(int index); /* For guidance on adding or extending families of instructions see - * ./adaptive.md + * InternalDocs/interpreter.md `Specialization` section. */ #ifdef Py_STATS @@ -28,6 +30,10 @@ GCStats _py_gc_stats[NUM_GENERATIONS] = { 0 }; static PyStats _Py_stats_struct = { .gc_stats = _py_gc_stats }; PyStats *_Py_stats = NULL; +#if PYSTATS_MAX_UOP_ID < MAX_UOP_ID +#error "Not enough space allocated for pystats. Increase PYSTATS_MAX_UOP_ID to at least MAX_UOP_ID" +#endif + #define ADD_STAT_TO_DICT(res, field) \ do { \ PyObject *val = PyLong_FromUnsignedLongLong(stats->field); \ @@ -103,6 +109,7 @@ _Py_GetSpecializationStats(void) { return NULL; } int err = 0; + err += add_stat_dict(stats, CONTAINS_OP, "contains_op"); err += add_stat_dict(stats, LOAD_SUPER_ATTR, "load_super_attr"); err += add_stat_dict(stats, LOAD_ATTR, "load_attr"); err += add_stat_dict(stats, LOAD_GLOBAL, "load_global"); @@ -110,6 +117,7 @@ _Py_GetSpecializationStats(void) { err += add_stat_dict(stats, STORE_SUBSCR, "store_subscr"); err += add_stat_dict(stats, STORE_ATTR, "store_attr"); err += add_stat_dict(stats, CALL, "call"); + err += add_stat_dict(stats, CALL_KW, "call_kw"); err += add_stat_dict(stats, BINARY_OP, "binary_op"); err += add_stat_dict(stats, COMPARE_OP, "compare_op"); err += add_stat_dict(stats, UNPACK_SEQUENCE, "unpack_sequence"); @@ -137,8 +145,18 @@ print_spec_stats(FILE *out, OpcodeStats *stats) fprintf(out, "opcode[BINARY_SLICE].specializable : 1\n"); fprintf(out, "opcode[STORE_SLICE].specializable : 1\n"); for (int i = 0; i < 256; i++) { - if (_PyOpcode_Caches[i] && i != JUMP_BACKWARD) { - fprintf(out, "opcode[%s].specializable : 1\n", _PyOpcode_OpName[i]); + if (_PyOpcode_Caches[i]) { + /* Ignore jumps as they cannot be specialized */ + switch (i) { + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: + case POP_JUMP_IF_NONE: + case POP_JUMP_IF_NOT_NONE: + case JUMP_BACKWARD: + break; + default: + fprintf(out, "opcode[%s].specializable : 1\n", _PyOpcode_OpName[i]); + } } PRINT_STAT(i, specialization.success); PRINT_STAT(i, specialization.failure); @@ -187,16 +205,19 @@ print_object_stats(FILE *out, ObjectStats *stats) fprintf(out, "Object allocations to 4 kbytes: %" PRIu64 "\n", stats->allocations4k); fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big); fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees); - fprintf(out, "Object new values: %" PRIu64 "\n", stats->new_values); - fprintf(out, "Object interpreter increfs: %" PRIu64 "\n", stats->interpreter_increfs); - fprintf(out, "Object interpreter decrefs: %" PRIu64 "\n", stats->interpreter_decrefs); - fprintf(out, "Object increfs: %" PRIu64 "\n", stats->increfs); - fprintf(out, "Object decrefs: %" PRIu64 "\n", stats->decrefs); + fprintf(out, "Object inline values: %" PRIu64 "\n", stats->inline_values); + fprintf(out, "Object interpreter mortal increfs: %" PRIu64 "\n", stats->interpreter_increfs); + fprintf(out, "Object interpreter mortal decrefs: %" PRIu64 "\n", stats->interpreter_decrefs); + fprintf(out, "Object mortal increfs: %" PRIu64 "\n", stats->increfs); + fprintf(out, "Object mortal decrefs: %" PRIu64 "\n", stats->decrefs); + fprintf(out, "Object interpreter immortal increfs: %" PRIu64 "\n", stats->interpreter_immortal_increfs); + fprintf(out, "Object interpreter immortal decrefs: %" PRIu64 "\n", stats->interpreter_immortal_decrefs); + fprintf(out, "Object immortal increfs: %" PRIu64 "\n", stats->immortal_increfs); + fprintf(out, "Object immortal decrefs: %" PRIu64 "\n", stats->immortal_decrefs); fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request); fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key); fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big); fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass); - fprintf(out, "Object dematerialize dict: %" PRIu64 "\n", stats->dict_dematerialized); fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits); fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses); fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions); @@ -211,9 +232,12 @@ print_gc_stats(FILE *out, GCStats *stats) fprintf(out, "GC[%d] collections: %" PRIu64 "\n", i, stats[i].collections); fprintf(out, "GC[%d] object visits: %" PRIu64 "\n", i, stats[i].object_visits); fprintf(out, "GC[%d] objects collected: %" PRIu64 "\n", i, stats[i].objects_collected); + fprintf(out, "GC[%d] objects reachable from roots: %" PRIu64 "\n", i, stats[i].objects_transitively_reachable); + fprintf(out, "GC[%d] objects not reachable from roots: %" PRIu64 "\n", i, stats[i].objects_not_transitively_reachable); } } +#ifdef _Py_TIER2 static void print_histogram(FILE *out, const char *name, uint64_t hist[_Py_UOP_HIST_SIZE]) { @@ -246,8 +270,8 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) fprintf(out, "Optimization optimizer successes: %" PRIu64 "\n", stats->optimizer_successes); fprintf(out, "Optimization optimizer failure no memory: %" PRIu64 "\n", stats->optimizer_failure_reason_no_memory); - - const char* const* names; + fprintf(out, "Optimizer remove globals builtins changed: %" PRIu64 "\n", stats->remove_globals_builtins_changed); + fprintf(out, "Optimizer remove globals incorrect keys: %" PRIu64 "\n", stats->remove_globals_incorrect_keys); for (int i = 0; i <= MAX_UOP_ID; i++) { if (stats->opcode[i].execution_count) { fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].execution_count); @@ -256,7 +280,6 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) fprintf(out, "uops[%s].specialization.miss : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].miss); } } - for (int i = 0; i < 256; i++) { if (stats->unsupported_opcode[i]) { fprintf( @@ -267,7 +290,27 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) ); } } + + for (int i = 1; i <= MAX_UOP_ID; i++){ + for (int j = 1; j <= MAX_UOP_ID; j++) { + if (stats->opcode[i].pair_count[j]) { + fprintf(out, "uop[%s].pair_count[%s] : %" PRIu64 "\n", + _PyOpcode_uop_name[i], _PyOpcode_uop_name[j], stats->opcode[i].pair_count[j]); + } + } + } + for (int i = 0; i < MAX_UOP_ID; i++) { + if (stats->error_in_opcode[i]) { + fprintf( + out, + "error_in_opcode[%s].count : %" PRIu64 "\n", + _PyUOpName(i), + stats->error_in_opcode[i] + ); + } + } } +#endif static void print_rare_event_stats(FILE *out, RareEventStats *stats) @@ -288,7 +331,9 @@ print_stats(FILE *out, PyStats *stats) print_call_stats(out, &stats->call_stats); print_object_stats(out, &stats->object_stats); print_gc_stats(out, stats->gc_stats); +#ifdef _Py_TIER2 print_optimization_stats(out, &stats->optimization_stats); +#endif print_rare_event_stats(out, &stats->rare_event_stats); } @@ -394,39 +439,51 @@ do { \ # define SPECIALIZATION_FAIL(opcode, kind) ((void)0) #endif -// Initialize warmup counters and insert superinstructions. This cannot fail. +// Initialize warmup counters and optimize instructions. This cannot fail. void -_PyCode_Quicken(PyCodeObject *code) +_PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, PyObject *consts, + int enable_counters) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT + _Py_BackoffCounter jump_counter, adaptive_counter; + if (enable_counters) { + jump_counter = initial_jump_backoff_counter(); + adaptive_counter = adaptive_counter_warmup(); + } + else { + jump_counter = initial_unreachable_backoff_counter(); + adaptive_counter = initial_unreachable_backoff_counter(); + } int opcode = 0; - _Py_CODEUNIT *instructions = _PyCode_CODE(code); - for (int i = 0; i < Py_SIZE(code); i++) { - opcode = _Py_GetBaseOpcode(code, i); - assert(opcode < MIN_INSTRUMENTED_OPCODE); + int oparg = 0; + /* The last code unit cannot have a cache, so we don't need to check it */ + for (Py_ssize_t i = 0; i < size-1; i++) { + opcode = instructions[i].op.code; int caches = _PyOpcode_Caches[opcode]; + oparg = (oparg << 8) | instructions[i].op.arg; if (caches) { // The initial value depends on the opcode - int initial_value; switch (opcode) { case JUMP_BACKWARD: - initial_value = 0; + instructions[i + 1].counter = jump_counter; break; case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: case POP_JUMP_IF_NONE: case POP_JUMP_IF_NOT_NONE: - initial_value = 0x5555; // Alternating 0, 1 bits + instructions[i + 1].cache = 0x5555; // Alternating 0, 1 bits break; default: - initial_value = adaptive_counter_warmup(); + instructions[i + 1].counter = adaptive_counter; break; } - instructions[i + 1].cache = initial_value; i += caches; } + if (opcode != EXTENDED_ARG) { + oparg = 0; + } } - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } #define SIMPLE_FUNCTION 0 @@ -464,14 +521,13 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_ATTR_READ_ONLY 16 #define SPEC_FAIL_ATTR_AUDITED_SLOT 17 #define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 18 -#define SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT 19 +#define SPEC_FAIL_ATTR_NON_STRING 19 #define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 20 - #define SPEC_FAIL_ATTR_SHADOWED 21 #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22 #define SPEC_FAIL_ATTR_CLASS_METHOD_OBJ 23 #define SPEC_FAIL_ATTR_OBJECT_SLOT 24 -#define SPEC_FAIL_ATTR_HAS_MANAGED_DICT 25 + #define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26 #define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27 #define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION 28 @@ -480,6 +536,9 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE 31 #define SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR 32 #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ 33 +#define SPEC_FAIL_ATTR_METACLASS_OVERRIDDEN 34 +#define SPEC_FAIL_ATTR_SPLIT_DICT 35 +#define SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED 36 /* Binary subscr and store subscr */ @@ -522,6 +581,10 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT 26 #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER 27 #define SPEC_FAIL_BINARY_OP_XOR 28 +#define SPEC_FAIL_BINARY_OP_OR_INT 29 +#define SPEC_FAIL_BINARY_OP_OR_DIFFERENT_TYPES 30 +#define SPEC_FAIL_BINARY_OP_XOR_INT 31 +#define SPEC_FAIL_BINARY_OP_XOR_DIFFERENT_TYPES 32 /* Calls */ @@ -538,13 +601,12 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_CALL_INIT_NOT_PYTHON 21 #define SPEC_FAIL_CALL_PEP_523 22 #define SPEC_FAIL_CALL_BOUND_METHOD 23 -#define SPEC_FAIL_CALL_STR 24 -#define SPEC_FAIL_CALL_CLASS_NO_VECTORCALL 25 #define SPEC_FAIL_CALL_CLASS_MUTABLE 26 #define SPEC_FAIL_CALL_METHOD_WRAPPER 28 #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29 #define SPEC_FAIL_CALL_INIT_NOT_SIMPLE 30 #define SPEC_FAIL_CALL_METACLASS 31 +#define SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES 32 /* COMPARE_OP */ #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12 @@ -599,25 +661,89 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_TO_BOOL_SET 17 #define SPEC_FAIL_TO_BOOL_TUPLE 18 +// CONTAINS_OP +#define SPEC_FAIL_CONTAINS_OP_STR 9 +#define SPEC_FAIL_CONTAINS_OP_TUPLE 10 +#define SPEC_FAIL_CONTAINS_OP_LIST 11 +#define SPEC_FAIL_CONTAINS_OP_USER_CLASS 12 + +static inline int +set_opcode(_Py_CODEUNIT *instr, uint8_t opcode) +{ +#ifdef Py_GIL_DISABLED + uint8_t old_op = _Py_atomic_load_uint8_relaxed(&instr->op.code); + if (old_op >= MIN_INSTRUMENTED_OPCODE) { + /* Lost race with instrumentation */ + return 0; + } + if (!_Py_atomic_compare_exchange_uint8(&instr->op.code, &old_op, opcode)) { + /* Lost race with instrumentation */ + assert(old_op >= MIN_INSTRUMENTED_OPCODE); + return 0; + } + return 1; +#else + instr->op.code = opcode; + return 1; +#endif +} + +static inline void +set_counter(_Py_BackoffCounter *counter, _Py_BackoffCounter value) +{ + FT_ATOMIC_STORE_UINT16_RELAXED(counter->value_and_backoff, + value.value_and_backoff); +} + +static inline _Py_BackoffCounter +load_counter(_Py_BackoffCounter *counter) +{ + _Py_BackoffCounter result = { + .value_and_backoff = + FT_ATOMIC_LOAD_UINT16_RELAXED(counter->value_and_backoff)}; + return result; +} + +static inline void +specialize(_Py_CODEUNIT *instr, uint8_t specialized_opcode) +{ + assert(!PyErr_Occurred()); + if (!set_opcode(instr, specialized_opcode)) { + STAT_INC(_PyOpcode_Deopt[specialized_opcode], failure); + SPECIALIZATION_FAIL(_PyOpcode_Deopt[specialized_opcode], + SPEC_FAIL_OTHER); + return; + } + STAT_INC(_PyOpcode_Deopt[specialized_opcode], success); + set_counter((_Py_BackoffCounter *)instr + 1, adaptive_counter_cooldown()); +} + +static inline void +unspecialize(_Py_CODEUNIT *instr) +{ + assert(!PyErr_Occurred()); + uint8_t opcode = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.code); + uint8_t generic_opcode = _PyOpcode_Deopt[opcode]; + STAT_INC(generic_opcode, failure); + if (!set_opcode(instr, generic_opcode)) { + SPECIALIZATION_FAIL(generic_opcode, SPEC_FAIL_OTHER); + return; + } + _Py_BackoffCounter *counter = (_Py_BackoffCounter *)instr + 1; + _Py_BackoffCounter cur = load_counter(counter); + set_counter(counter, adaptive_counter_backoff(cur)); +} + static int function_kind(PyCodeObject *code); static bool function_check_args(PyObject *o, int expected_argcount, int opcode); static uint32_t function_get_version(PyObject *o, int opcode); -static uint32_t type_get_version(PyTypeObject *t, int opcode); static int -specialize_module_load_attr( - PyObject *owner, _Py_CODEUNIT *instr, PyObject *name -) { +specialize_module_load_attr_lock_held(PyDictObject *dict, _Py_CODEUNIT *instr, PyObject *name) +{ _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); - PyModuleObject *m = (PyModuleObject *)owner; - assert((owner->ob_type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0); - PyDictObject *dict = (PyDictObject *)m->md_dict; - if (dict == NULL) { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_NO_DICT); - return -1; - } if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_STRING); return -1; } Py_ssize_t index = _PyDict_LookupIndex(dict, &_Py_ID(__getattr__)); @@ -635,27 +761,45 @@ specialize_module_load_attr( SPEC_FAIL_OUT_OF_RANGE); return -1; } - uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState( - _PyInterpreterState_GET(), dict->ma_keys); + uint32_t keys_version = _PyDict_GetKeysVersionForCurrentState( + _PyInterpreterState_GET(), dict); if (keys_version == 0) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return -1; } write_u32(cache->version, keys_version); cache->index = (uint16_t)index; - instr->op.code = LOAD_ATTR_MODULE; + specialize(instr, LOAD_ATTR_MODULE); return 0; } - +static int +specialize_module_load_attr( + PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) +{ + PyModuleObject *m = (PyModuleObject *)owner; + assert((Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0); + PyDictObject *dict = (PyDictObject *)m->md_dict; + if (dict == NULL) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_NO_DICT); + return -1; + } + int result; + Py_BEGIN_CRITICAL_SECTION(dict); + result = specialize_module_load_attr_lock_held(dict, instr, name); + Py_END_CRITICAL_SECTION(); + return result; +} /* Attribute specialization */ void -_Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, _Py_CODEUNIT *instr, int load_method) { - assert(ENABLE_SPECIALIZATION); +_Py_Specialize_LoadSuperAttr(_PyStackRef global_super_st, _PyStackRef cls_st, _Py_CODEUNIT *instr, int load_method) { + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *cls = PyStackRef_AsPyObjectBorrow(cls_st); + + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[LOAD_SUPER_ATTR] == INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); - _PySuperAttrCache *cache = (_PySuperAttrCache *)(instr + 1); if (global_super != (PyObject *)&PySuper_Type) { SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_SHADOWED); goto fail; @@ -664,19 +808,11 @@ _Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, _Py_CODEUNIT SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_BAD_CLASS); goto fail; } - instr->op.code = load_method ? LOAD_SUPER_ATTR_METHOD : LOAD_SUPER_ATTR_ATTR; - goto success; - -fail: - STAT_INC(LOAD_SUPER_ATTR, failure); - assert(!PyErr_Occurred()); - instr->op.code = LOAD_SUPER_ATTR; - cache->counter = adaptive_counter_backoff(cache->counter); + uint8_t load_code = load_method ? LOAD_SUPER_ATTR_METHOD : LOAD_SUPER_ATTR_ATTR; + specialize(instr, load_code); return; -success: - STAT_INC(LOAD_SUPER_ATTR, success); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_cooldown(); +fail: + unspecialize(instr); } typedef enum { @@ -698,58 +834,8 @@ typedef enum { static DescriptorClassification -analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int store) +classify_descriptor(PyObject *descriptor, bool has_getattr) { - bool has_getattr = false; - if (store) { - if (type->tp_setattro != PyObject_GenericSetAttr) { - *descr = NULL; - return GETSET_OVERRIDDEN; - } - } - else { - getattrofunc getattro_slot = type->tp_getattro; - if (getattro_slot == PyObject_GenericGetAttr) { - /* Normal attribute lookup; */ - has_getattr = false; - } - else if (getattro_slot == _Py_slot_tp_getattr_hook || - getattro_slot == _Py_slot_tp_getattro) { - /* One or both of __getattribute__ or __getattr__ may have been - overridden See typeobject.c for why these functions are special. */ - PyObject *getattribute = _PyType_Lookup(type, - &_Py_ID(__getattribute__)); - PyInterpreterState *interp = _PyInterpreterState_GET(); - bool has_custom_getattribute = getattribute != NULL && - getattribute != interp->callable_cache.object__getattribute__; - has_getattr = _PyType_Lookup(type, &_Py_ID(__getattr__)) != NULL; - if (has_custom_getattribute) { - if (getattro_slot == _Py_slot_tp_getattro && - !has_getattr && - Py_IS_TYPE(getattribute, &PyFunction_Type)) { - *descr = getattribute; - return GETATTRIBUTE_IS_PYTHON_FUNCTION; - } - /* Potentially both __getattr__ and __getattribute__ are set. - Too complicated */ - *descr = NULL; - return GETSET_OVERRIDDEN; - } - /* Potentially has __getattr__ but no custom __getattribute__. - Fall through to usual descriptor analysis. - Usual attribute lookup should only be allowed at runtime - if we can guarantee that there is no way an exception can be - raised. This means some specializations, e.g. specializing - for property() isn't safe. - */ - } - else { - *descr = NULL; - return GETSET_OVERRIDDEN; - } - } - PyObject *descriptor = _PyType_Lookup(type, name); - *descr = descriptor; if (descriptor == NULL) { return ABSENT; } @@ -772,14 +858,7 @@ analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int sto __getattr__. */ return has_getattr ? GETSET_OVERRIDDEN : PROPERTY; } - if (PyUnicode_CompareWithASCIIString(name, "__class__") == 0) { - if (descriptor == _PyType_Lookup(&PyBaseObject_Type, name)) { - return DUNDER_CLASS; - } - } - if (store) { - return OVERRIDING; - } + return OVERRIDING; } if (desc_cls->tp_descr_get) { if (desc_cls->tp_flags & Py_TPFLAGS_METHOD_DESCRIPTOR) { @@ -796,120 +875,274 @@ analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int sto return NON_DESCRIPTOR; } +static bool +descriptor_is_class(PyObject *descriptor, PyObject *name) +{ + return ((PyUnicode_CompareWithASCIIString(name, "__class__") == 0) && + (descriptor == _PyType_Lookup(&PyBaseObject_Type, name))); +} + +static DescriptorClassification +analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version) { + bool has_getattr = false; + bool have_ga_version = false; + unsigned int ga_version; + getattrofunc getattro_slot = type->tp_getattro; + if (getattro_slot == PyObject_GenericGetAttr) { + /* Normal attribute lookup; */ + has_getattr = false; + } + else if (getattro_slot == _Py_slot_tp_getattr_hook || + getattro_slot == _Py_slot_tp_getattro) { + /* One or both of __getattribute__ or __getattr__ may have been + overridden See typeobject.c for why these functions are special. */ + PyObject *getattribute = _PyType_LookupRefAndVersion(type, + &_Py_ID(__getattribute__), &ga_version); + have_ga_version = true; + PyInterpreterState *interp = _PyInterpreterState_GET(); + bool has_custom_getattribute = getattribute != NULL && + getattribute != interp->callable_cache.object__getattribute__; + PyObject *getattr = _PyType_Lookup(type, &_Py_ID(__getattr__)); + has_getattr = getattr != NULL; + if (has_custom_getattribute) { + if (getattro_slot == _Py_slot_tp_getattro && + !has_getattr && + Py_IS_TYPE(getattribute, &PyFunction_Type)) { + *descr = getattribute; + *tp_version = ga_version; + return GETATTRIBUTE_IS_PYTHON_FUNCTION; + } + /* Potentially both __getattr__ and __getattribute__ are set. + Too complicated */ + Py_DECREF(getattribute); + *descr = NULL; + *tp_version = ga_version; + return GETSET_OVERRIDDEN; + } + /* Potentially has __getattr__ but no custom __getattribute__. + Fall through to usual descriptor analysis. + Usual attribute lookup should only be allowed at runtime + if we can guarantee that there is no way an exception can be + raised. This means some specializations, e.g. specializing + for property() isn't safe. + */ + Py_XDECREF(getattribute); + } + else { + *descr = NULL; + *tp_version = FT_ATOMIC_LOAD_UINT_RELAXED(type->tp_version_tag); + return GETSET_OVERRIDDEN; + } + unsigned int descr_version; + PyObject *descriptor = _PyType_LookupRefAndVersion(type, name, &descr_version); + *descr = descriptor; + *tp_version = have_ga_version ? ga_version : descr_version; + if (descriptor_is_class(descriptor, name)) { + return DUNDER_CLASS; + } + return classify_descriptor(descriptor, has_getattr); +} + +static DescriptorClassification +analyze_descriptor_store(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version) +{ + if (type->tp_setattro != PyObject_GenericSetAttr) { + *descr = NULL; + return GETSET_OVERRIDDEN; + } + PyObject *descriptor = _PyType_LookupRefAndVersion(type, name, tp_version); + *descr = descriptor; + if (descriptor_is_class(descriptor, name)) { + return DUNDER_CLASS; + } + return classify_descriptor(descriptor, false); +} + +static int +specialize_dict_access_inline( + PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type, + PyObject *name, unsigned int tp_version, + int base_op, int values_op) +{ + _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); + PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; + assert(PyUnicode_CheckExact(name)); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(owner); + Py_ssize_t index = _PyDictKeys_StringLookupSplit(keys, name); + assert (index != DKIX_ERROR); + if (index == DKIX_EMPTY) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_IN_KEYS); + return 0; + } + assert(index >= 0); + assert(_PyObject_InlineValues(owner)->valid); + char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index]; + Py_ssize_t offset = value_addr - (char *)owner; + if (offset != (uint16_t)offset) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); + return 0; + } + cache->index = (uint16_t)offset; + write_u32(cache->version, tp_version); + specialize(instr, values_op); + return 1; +} + +static int +specialize_dict_access_hint( + PyDictObject *dict, _Py_CODEUNIT *instr, PyTypeObject *type, + PyObject *name, unsigned int tp_version, + int base_op, int hint_op) +{ + _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); + + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(dict); + + // We found an instance with a __dict__. + if (_PyDict_HasSplitTable(dict)) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT); + return 0; + } + Py_ssize_t index = _PyDict_LookupIndex(dict, name); + if (index != (uint16_t)index) { + SPECIALIZATION_FAIL(base_op, + index == DKIX_EMPTY ? + SPEC_FAIL_ATTR_NOT_IN_DICT : + SPEC_FAIL_OUT_OF_RANGE); + return 0; + } + cache->index = (uint16_t)index; + write_u32(cache->version, tp_version); + specialize(instr, hint_op); + return 1; +} + + static int specialize_dict_access( PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type, - DescriptorClassification kind, PyObject *name, + DescriptorClassification kind, PyObject *name, unsigned int tp_version, int base_op, int values_op, int hint_op) { assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT || - kind == BUILTIN_CLASSMETHOD || kind == PYTHON_CLASSMETHOD); + kind == BUILTIN_CLASSMETHOD || kind == PYTHON_CLASSMETHOD || + kind == METHOD); // No descriptor, or non overriding. if ((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_MANAGED_DICT); return 0; } - _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - if (_PyDictOrValues_IsValues(*dorv) || - _PyObject_MakeInstanceAttributesFromDict(owner, dorv)) + if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES && + FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner)->valid) && + !(base_op == STORE_ATTR && _PyObject_GetManagedDict(owner) != NULL)) { - // Virtual dictionary - PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; - assert(PyUnicode_CheckExact(name)); - Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); - assert (index != DKIX_ERROR); - if (index != (uint16_t)index) { - SPECIALIZATION_FAIL(base_op, - index == DKIX_EMPTY ? - SPEC_FAIL_ATTR_NOT_IN_KEYS : - SPEC_FAIL_OUT_OF_RANGE); - return 0; + int res; + Py_BEGIN_CRITICAL_SECTION(owner); + PyDictObject *dict = _PyObject_GetManagedDict(owner); + if (dict == NULL) { + // managed dict, not materialized, inline values valid + res = specialize_dict_access_inline(owner, instr, type, name, + tp_version, base_op, values_op); } - write_u32(cache->version, type->tp_version_tag); - cache->index = (uint16_t)index; - instr->op.code = values_op; + else { + // lost race and dict was created, fail specialization + SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OTHER); + res = 0; + } + Py_END_CRITICAL_SECTION(); + return res; } else { - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv); + PyDictObject *dict = _PyObject_GetManagedDict(owner); if (dict == NULL || !PyDict_CheckExact(dict)) { SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT); return 0; } - // We found an instance with a __dict__. - if (dict->ma_values) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT); - return 0; - } - Py_ssize_t index = - _PyDict_LookupIndex(dict, name); - if (index != (uint16_t)index) { - SPECIALIZATION_FAIL(base_op, - index == DKIX_EMPTY ? - SPEC_FAIL_ATTR_NOT_IN_DICT : - SPEC_FAIL_OUT_OF_RANGE); - return 0; - } - cache->index = (uint16_t)index; - write_u32(cache->version, type->tp_version_tag); - instr->op.code = hint_op; + int res; + Py_BEGIN_CRITICAL_SECTION(dict); + // materialized managed dict + res = specialize_dict_access_hint(dict, instr, type, name, + tp_version, base_op, hint_op); + Py_END_CRITICAL_SECTION(); + return res; } - return 1; } -static int specialize_attr_loadclassattr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name, - PyObject* descr, DescriptorClassification kind, bool is_method); +static int +specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, + PyObject *name, PyObject *descr, + unsigned int tp_version, + DescriptorClassification kind, bool is_method, + uint32_t shared_keys_version); static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name); -void -_Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) +/* Returns true if instances of obj's class are + * likely to have `name` in their __dict__. + * For objects with inline values, we check in the shared keys. + * For other objects, we check their actual dictionary. + */ +static bool +instance_has_key(PyObject *obj, PyObject *name, uint32_t *shared_keys_version) { - assert(ENABLE_SPECIALIZATION); - assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR); - _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); - PyTypeObject *type = Py_TYPE(owner); - if (!_PyType_IsReady(type)) { - // We *might* not really need this check, but we inherited it from - // PyObject_GenericGetAttr and friends... and this way we still do the - // right thing if someone forgets to call PyType_Ready(type): - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); - goto fail; + PyTypeObject *cls = Py_TYPE(obj); + if ((cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { + return false; } - if (PyModule_CheckExact(owner)) { - if (specialize_module_load_attr(owner, instr, name)) - { - goto fail; - } - goto success; + if (cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictKeysObject *keys = ((PyHeapTypeObject *)cls)->ht_cached_keys; + Py_ssize_t index = + _PyDictKeys_StringLookupAndVersion(keys, name, shared_keys_version); + return index >= 0; } - if (PyType_Check(owner)) { - if (specialize_class_load_attr(owner, instr, name)) { - goto fail; - } - goto success; + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL || !PyDict_CheckExact(dict)) { + return false; } - PyObject *descr = NULL; - DescriptorClassification kind = analyze_descriptor(type, name, &descr, 0); - assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN); - if (type_get_version(type, LOAD_ATTR) == 0) { - goto fail; + bool result; + Py_BEGIN_CRITICAL_SECTION(dict); + if (dict->ma_values) { + result = false; + } + else { + result = (_PyDict_LookupIndex(dict, name) >= 0); + } + Py_END_CRITICAL_SECTION(); + return result; +} + +static int +do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name, + bool shadow, uint32_t shared_keys_version, + DescriptorClassification kind, PyObject *descr, unsigned int tp_version) +{ + _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); + PyTypeObject *type = Py_TYPE(owner); + if (tp_version == 0) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); + return -1; } + uint8_t oparg = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.arg); switch(kind) { case OVERRIDING: SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR); - goto fail; + return -1; case METHOD: { - int oparg = instr->op.arg; + if (shadow) { + goto try_instance; + } if (oparg & 1) { - if (specialize_attr_loadclassattr(owner, instr, name, descr, kind, true)) { - goto success; + if (specialize_attr_loadclassattr(owner, instr, name, descr, + tp_version, kind, true, + shared_keys_version)) { + return 0; + } + else { + return -1; } } - else { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); - } - goto fail; + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); + return -1; } case PROPERTY: { @@ -918,34 +1151,36 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) PyObject *fget = ((_PyPropertyObject *)descr)->prop_get; if (fget == NULL) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); - goto fail; + return -1; } if (!Py_IS_TYPE(fget, &PyFunction_Type)) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION); - goto fail; + return -1; } if (!function_check_args(fget, 1, LOAD_ATTR)) { - goto fail; + return -1; } - if (instr->op.arg & 1) { + if (oparg & 1) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); - goto fail; - } - uint32_t version = function_get_version(fget, LOAD_ATTR); - if (version == 0) { - goto fail; + return -1; } + /* Don't specialize if PEP 523 is active */ if (_PyInterpreterState_GET()->eval_frame) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); - goto fail; + return -1; } - write_u32(lm_cache->keys_version, version); - assert(type->tp_version_tag != 0); - write_u32(lm_cache->type_version, type->tp_version_tag); + #ifdef Py_GIL_DISABLED + if (!_PyObject_HasDeferredRefcount(fget)) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); + return -1; + } + #endif + assert(tp_version != 0); + write_u32(lm_cache->type_version, tp_version); /* borrowed */ - write_obj(lm_cache->descr, fget); - instr->op.code = LOAD_ATTR_PROPERTY; - goto success; + write_ptr(lm_cache->descr, fget); + specialize(instr, LOAD_ATTR_PROPERTY); + return 0; } case OBJECT_SLOT: { @@ -954,114 +1189,170 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) Py_ssize_t offset = dmem->offset; if (!PyObject_TypeCheck(owner, member->d_common.d_type)) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); - goto fail; + return -1; } if (dmem->flags & Py_AUDIT_READ) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_AUDITED_SLOT); - goto fail; + return -1; } if (offset != (uint16_t)offset) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); - goto fail; + return -1; } assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT); assert(offset > 0); cache->index = (uint16_t)offset; - write_u32(cache->version, type->tp_version_tag); - instr->op.code = LOAD_ATTR_SLOT; - goto success; + write_u32(cache->version, tp_version); + specialize(instr, LOAD_ATTR_SLOT); + return 0; } case DUNDER_CLASS: { Py_ssize_t offset = offsetof(PyObject, ob_type); assert(offset == (uint16_t)offset); cache->index = (uint16_t)offset; - write_u32(cache->version, type->tp_version_tag); - instr->op.code = LOAD_ATTR_SLOT; - goto success; + write_u32(cache->version, tp_version); + specialize(instr, LOAD_ATTR_SLOT); + return 0; } case OTHER_SLOT: SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT); - goto fail; + return -1; case MUTABLE: SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS); - goto fail; + return -1; case GETSET_OVERRIDDEN: SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OVERRIDDEN); - goto fail; + return -1; case GETATTRIBUTE_IS_PYTHON_FUNCTION: { + #ifndef Py_GIL_DISABLED + // In free-threaded builds it's possible for tp_getattro to change + // after the call to analyze_descriptor. That is fine: the version + // guard will fail. assert(type->tp_getattro == _Py_slot_tp_getattro); + #endif assert(Py_IS_TYPE(descr, &PyFunction_Type)); _PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1); if (!function_check_args(descr, 2, LOAD_ATTR)) { - goto fail; + return -1; } - if (instr->op.arg & 1) { + if (oparg & 1) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); - goto fail; + return -1; } uint32_t version = function_get_version(descr, LOAD_ATTR); if (version == 0) { - goto fail; + return -1; } + /* Don't specialize if PEP 523 is active */ if (_PyInterpreterState_GET()->eval_frame) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); - goto fail; + return -1; + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_HasDeferredRefcount(descr)) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); + return -1; } + #endif write_u32(lm_cache->keys_version, version); /* borrowed */ - write_obj(lm_cache->descr, descr); - write_u32(lm_cache->type_version, type->tp_version_tag); - instr->op.code = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; - goto success; + write_ptr(lm_cache->descr, descr); + write_u32(lm_cache->type_version, tp_version); + specialize(instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + return 0; } case BUILTIN_CLASSMETHOD: - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ); - goto fail; case PYTHON_CLASSMETHOD: - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_METHOD_OBJ); - goto fail; case NON_OVERRIDING: - SPECIALIZATION_FAIL(LOAD_ATTR, - (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) ? - SPEC_FAIL_ATTR_CLASS_ATTR_DESCRIPTOR : - SPEC_FAIL_ATTR_NOT_MANAGED_DICT); - goto fail; + if (shadow) { + goto try_instance; + } + return -1; case NON_DESCRIPTOR: - if ((instr->op.arg & 1) == 0) { - if (specialize_attr_loadclassattr(owner, instr, name, descr, kind, false)) { - goto success; - } + if (shadow) { + goto try_instance; } - else { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE); + if ((oparg & 1) == 0) { + if (specialize_attr_loadclassattr(owner, instr, name, descr, + tp_version, kind, false, + shared_keys_version)) { + return 0; + } } - goto fail; + return -1; case ABSENT: - if (specialize_dict_access(owner, instr, type, kind, name, LOAD_ATTR, - LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT)) - { - goto success; + if (shadow) { + goto try_instance; } + set_counter((_Py_BackoffCounter*)instr + 1, adaptive_counter_cooldown()); + return 0; } -fail: - STAT_INC(LOAD_ATTR, failure); - assert(!PyErr_Occurred()); - instr->op.code = LOAD_ATTR; - cache->counter = adaptive_counter_backoff(cache->counter); - return; -success: - STAT_INC(LOAD_ATTR, success); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_cooldown(); + Py_UNREACHABLE(); +try_instance: + if (specialize_dict_access(owner, instr, type, kind, name, tp_version, + LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT)) + { + return 0; + } + return -1; +} + +static int +specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name) +{ + // 0 is not a valid version + uint32_t shared_keys_version = 0; + bool shadow = instance_has_key(owner, name, &shared_keys_version); + PyObject *descr = NULL; + unsigned int tp_version = 0; + PyTypeObject *type = Py_TYPE(owner); + DescriptorClassification kind = analyze_descriptor_load(type, name, &descr, &tp_version); + int result = do_specialize_instance_load_attr(owner, instr, name, shadow, shared_keys_version, kind, descr, tp_version); + Py_XDECREF(descr); + return result; } void -_Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) +_Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name) { - assert(ENABLE_SPECIALIZATION); + PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st); + + assert(ENABLE_SPECIALIZATION_FT); + assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR); + PyTypeObject *type = Py_TYPE(owner); + bool fail; + if (!_PyType_IsReady(type)) { + // We *might* not really need this check, but we inherited it from + // PyObject_GenericGetAttr and friends... and this way we still do the + // right thing if someone forgets to call PyType_Ready(type): + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); + fail = true; + } + else if (Py_TYPE(owner)->tp_getattro == PyModule_Type.tp_getattro) { + fail = specialize_module_load_attr(owner, instr, name); + } + else if (PyType_Check(owner)) { + fail = specialize_class_load_attr(owner, instr, name); + } + else { + fail = specialize_instance_load_attr(owner, instr, name); + } + + if (fail) { + unspecialize(instr); + } +} + +void +_Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name) +{ + PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st); + + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR); + PyObject *descr = NULL; _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyTypeObject *type = Py_TYPE(owner); if (!_PyType_IsReady(type)) { @@ -1075,11 +1366,12 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN); goto fail; } - PyObject *descr; - DescriptorClassification kind = analyze_descriptor(type, name, &descr, 1); - if (type_get_version(type, STORE_ATTR) == 0) { + unsigned int tp_version = 0; + DescriptorClassification kind = analyze_descriptor_store(type, name, &descr, &tp_version); + if (tp_version == 0) { goto fail; } + assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN); switch(kind) { case OVERRIDING: SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR); @@ -1110,8 +1402,8 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT); assert(offset > 0); cache->index = (uint16_t)offset; - write_u32(cache->version, type->tp_version_tag); - instr->op.code = STORE_ATTR_SLOT; + write_u32(cache->version, tp_version); + specialize(instr, STORE_ATTR_SLOT); goto success; } case DUNDER_CLASS: @@ -1138,25 +1430,21 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE); goto fail; case ABSENT: - if (specialize_dict_access(owner, instr, type, kind, name, STORE_ATTR, - STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT)) - { + if (specialize_dict_access(owner, instr, type, kind, name, tp_version, + STORE_ATTR, STORE_ATTR_INSTANCE_VALUE, + STORE_ATTR_WITH_HINT)) { goto success; } } fail: - STAT_INC(STORE_ATTR, failure); - assert(!PyErr_Occurred()); - instr->op.code = STORE_ATTR; - cache->counter = adaptive_counter_backoff(cache->counter); + Py_XDECREF(descr); + unspecialize(instr); return; success: - STAT_INC(STORE_ATTR, success); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_cooldown(); + Py_XDECREF(descr); + return; } - #ifdef Py_STATS static int load_attr_fail_kind(DescriptorClassification kind) @@ -1198,31 +1486,77 @@ static int specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) { + assert(PyType_Check(owner)); + PyTypeObject *cls = (PyTypeObject *)owner; _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1); - if (!PyType_CheckExact(owner) || _PyType_Lookup(Py_TYPE(owner), name)) { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE); + if (Py_TYPE(cls)->tp_getattro != _Py_type_getattro) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METACLASS_OVERRIDDEN); return -1; } + unsigned int meta_version = 0; + PyObject *metadescriptor = _PyType_LookupRefAndVersion(Py_TYPE(cls), name, &meta_version); + DescriptorClassification metakind = classify_descriptor(metadescriptor, false); + Py_XDECREF(metadescriptor); + switch (metakind) { + case METHOD: + case NON_DESCRIPTOR: + case NON_OVERRIDING: + case BUILTIN_CLASSMETHOD: + case PYTHON_CLASSMETHOD: + case ABSENT: + break; + default: + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE); + return -1; + } PyObject *descr = NULL; DescriptorClassification kind = 0; - kind = analyze_descriptor((PyTypeObject *)owner, name, &descr, 0); - if (type_get_version((PyTypeObject *)owner, LOAD_ATTR) == 0) { + unsigned int tp_version = 0; + kind = analyze_descriptor_load(cls, name, &descr, &tp_version); + if (tp_version == 0) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); + Py_XDECREF(descr); return -1; } + bool metaclass_check = false; + if ((Py_TYPE(cls)->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) == 0) { + metaclass_check = true; + if (meta_version == 0) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); + Py_XDECREF(descr); + return -1; + } + } switch (kind) { case METHOD: case NON_DESCRIPTOR: - write_u32(cache->type_version, ((PyTypeObject *)owner)->tp_version_tag); - write_obj(cache->descr, descr); - instr->op.code = LOAD_ATTR_CLASS; + #ifdef Py_GIL_DISABLED + if (!_PyObject_HasDeferredRefcount(descr)) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); + Py_XDECREF(descr); + return -1; + } + #endif + write_u32(cache->type_version, tp_version); + write_ptr(cache->descr, descr); + if (metaclass_check) { + write_u32(cache->keys_version, meta_version); + specialize(instr, LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); + } + else { + specialize(instr, LOAD_ATTR_CLASS); + } + Py_XDECREF(descr); return 0; #ifdef Py_STATS case ABSENT: SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); + Py_XDECREF(descr); return -1; #endif default: SPECIALIZATION_FAIL(LOAD_ATTR, load_attr_fail_kind(kind)); + Py_XDECREF(descr); return -1; } } @@ -1231,45 +1565,52 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, // can cause a significant drop in cache hits. A possible test is // python.exe -m test_typing test_re test_dis test_zlib. static int -specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, -PyObject *descr, DescriptorClassification kind, bool is_method) +specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, + PyObject *name, PyObject *descr, + unsigned int tp_version, + DescriptorClassification kind, bool is_method, + uint32_t shared_keys_version) { _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1); PyTypeObject *owner_cls = Py_TYPE(owner); assert(descr != NULL); assert((is_method && kind == METHOD) || (!is_method && kind == NON_DESCRIPTOR)); - if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - PyDictKeysObject *keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys; - if (!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) - { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_HAS_MANAGED_DICT); - return 0; - } - Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); - if (index != DKIX_EMPTY) { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SHADOWED); - return 0; - } - uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState( - _PyInterpreterState_GET(), keys); - if (keys_version == 0) { + + #ifdef Py_GIL_DISABLED + if (!_PyObject_HasDeferredRefcount(descr)) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); + return 0; + } + #endif + + unsigned long tp_flags = PyType_GetFlags(owner_cls); + if (tp_flags & Py_TPFLAGS_INLINE_VALUES) { + #ifndef Py_GIL_DISABLED + assert(_PyDictKeys_StringLookup( + ((PyHeapTypeObject *)owner_cls)->ht_cached_keys, name) < 0); + #endif + if (shared_keys_version == 0) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return 0; } - write_u32(cache->keys_version, keys_version); - instr->op.code = is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; + write_u32(cache->keys_version, shared_keys_version); + specialize(instr, is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); } else { - Py_ssize_t dictoffset = owner_cls->tp_dictoffset; - if (dictoffset < 0 || dictoffset > INT16_MAX) { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); - return 0; + Py_ssize_t dictoffset; + if (tp_flags & Py_TPFLAGS_MANAGED_DICT) { + dictoffset = MANAGED_DICT_OFFSET; + } + else { + dictoffset = owner_cls->tp_dictoffset; + if (dictoffset < 0 || dictoffset > INT16_MAX + MANAGED_DICT_OFFSET) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); + return 0; + } } if (dictoffset == 0) { - instr->op.code = is_method ? LOAD_ATTR_METHOD_NO_DICT : LOAD_ATTR_NONDESCRIPTOR_NO_DICT; + specialize(instr, is_method ? LOAD_ATTR_METHOD_NO_DICT : LOAD_ATTR_NONDESCRIPTOR_NO_DICT); } else if (is_method) { PyObject *dict = *(PyObject **) ((char *)owner + dictoffset); @@ -1277,9 +1618,13 @@ PyObject *descr, DescriptorClassification kind, bool is_method) SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NOT_MANAGED_DICT); return 0; } - assert(owner_cls->tp_dictoffset > 0); - assert(owner_cls->tp_dictoffset <= INT16_MAX); - instr->op.code = LOAD_ATTR_METHOD_LAZY_DICT; + /* Cache entries must be unsigned values, so we offset the + * dictoffset by MANAGED_DICT_OFFSET. + * We do the reverse offset in LOAD_ATTR_METHOD_LAZY_DICT */ + dictoffset -= MANAGED_DICT_OFFSET; + assert(((uint16_t)dictoffset) == dictoffset); + cache->dict_offset = (uint16_t)dictoffset; + specialize(instr, LOAD_ATTR_METHOD_LAZY_DICT); } else { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE); @@ -1300,17 +1645,18 @@ PyObject *descr, DescriptorClassification kind, bool is_method) * PyType_Modified usages in typeobject.c). The MCACHE has been * working since Python 2.6 and it's battle-tested. */ - write_u32(cache->type_version, owner_cls->tp_version_tag); - write_obj(cache->descr, descr); + write_u32(cache->type_version, tp_version); + write_ptr(cache->descr, descr); return 1; } -void -_Py_Specialize_LoadGlobal( + +static void +specialize_load_global_lock_held( PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name) { - assert(ENABLE_SPECIALIZATION); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[LOAD_GLOBAL] == INLINE_CACHE_ENTRIES_LOAD_GLOBAL); /* Use inline cache */ _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)(instr + 1); @@ -1335,8 +1681,8 @@ _Py_Specialize_LoadGlobal( SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); goto fail; } - uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState( - interp, globals_keys); + uint32_t keys_version = _PyDict_GetKeysVersionForCurrentState( + interp, (PyDictObject*) globals); if (keys_version == 0) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; @@ -1347,8 +1693,8 @@ _Py_Specialize_LoadGlobal( } cache->index = (uint16_t)index; cache->module_keys_version = (uint16_t)keys_version; - instr->op.code = LOAD_GLOBAL_MODULE; - goto success; + specialize(instr, LOAD_GLOBAL_MODULE); + return; } if (!PyDict_CheckExact(builtins)) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_DICT); @@ -1368,8 +1714,8 @@ _Py_Specialize_LoadGlobal( SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); goto fail; } - uint32_t globals_version = _PyDictKeys_GetVersionForCurrentState( - interp, globals_keys); + uint32_t globals_version = _PyDict_GetKeysVersionForCurrentState( + interp, (PyDictObject*) globals); if (globals_version == 0) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; @@ -1378,8 +1724,8 @@ _Py_Specialize_LoadGlobal( SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); goto fail; } - uint32_t builtins_version = _PyDictKeys_GetVersionForCurrentState( - interp, builtin_keys); + uint32_t builtins_version = _PyDict_GetKeysVersionForCurrentState( + interp, (PyDictObject*) builtins); if (builtins_version == 0) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; @@ -1391,18 +1737,20 @@ _Py_Specialize_LoadGlobal( cache->index = (uint16_t)index; cache->module_keys_version = (uint16_t)globals_version; cache->builtin_keys_version = (uint16_t)builtins_version; - instr->op.code = LOAD_GLOBAL_BUILTIN; - goto success; -fail: - STAT_INC(LOAD_GLOBAL, failure); - assert(!PyErr_Occurred()); - instr->op.code = LOAD_GLOBAL; - cache->counter = adaptive_counter_backoff(cache->counter); + specialize(instr, LOAD_GLOBAL_BUILTIN); return; -success: - STAT_INC(LOAD_GLOBAL, success); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_cooldown(); +fail: + unspecialize(instr); +} + +void +_Py_Specialize_LoadGlobal( + PyObject *globals, PyObject *builtins, + _Py_CODEUNIT *instr, PyObject *name) +{ + Py_BEGIN_CRITICAL_SECTION2(globals, builtins); + specialize_load_global_lock_held(globals, builtins, instr, name); + Py_END_CRITICAL_SECTION2(); } #ifdef Py_STATS @@ -1474,19 +1822,7 @@ function_get_version(PyObject *o, int opcode) assert(Py_IS_TYPE(o, &PyFunction_Type)); PyFunctionObject *func = (PyFunctionObject *)o; uint32_t version = _PyFunction_GetVersionForCurrentState(func); - if (version == 0) { - SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS); - return 0; - } - return version; -} - -/* Returning 0 indicates a failure. */ -static uint32_t -type_get_version(PyTypeObject *t, int opcode) -{ - uint32_t version = t->tp_version_tag; - if (version == 0) { + if (!_PyFunction_IsVersionValid(version)) { SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS); return 0; } @@ -1495,17 +1831,20 @@ type_get_version(PyTypeObject *t, int opcode) void _Py_Specialize_BinarySubscr( - PyObject *container, PyObject *sub, _Py_CODEUNIT *instr) + _PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr) { - assert(ENABLE_SPECIALIZATION); + PyObject *container = PyStackRef_AsPyObjectBorrow(container_st); + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[BINARY_SUBSCR] == INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)(instr + 1); PyTypeObject *container_type = Py_TYPE(container); + uint8_t specialized_op; if (container_type == &PyList_Type) { if (PyLong_CheckExact(sub)) { if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - instr->op.code = BINARY_SUBSCR_LIST_INT; + specialized_op = BINARY_SUBSCR_LIST_INT; goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); @@ -1518,7 +1857,7 @@ _Py_Specialize_BinarySubscr( if (container_type == &PyTuple_Type) { if (PyLong_CheckExact(sub)) { if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - instr->op.code = BINARY_SUBSCR_TUPLE_INT; + specialized_op = BINARY_SUBSCR_TUPLE_INT; goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); @@ -1531,7 +1870,7 @@ _Py_Specialize_BinarySubscr( if (container_type == &PyUnicode_Type) { if (PyLong_CheckExact(sub)) { if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - instr->op.code = BINARY_SUBSCR_STR_INT; + specialized_op = BINARY_SUBSCR_STR_INT; goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); @@ -1542,14 +1881,15 @@ _Py_Specialize_BinarySubscr( goto fail; } if (container_type == &PyDict_Type) { - instr->op.code = BINARY_SUBSCR_DICT; + specialized_op = BINARY_SUBSCR_DICT; goto success; } - PyTypeObject *cls = Py_TYPE(container); - PyObject *descriptor = _PyType_Lookup(cls, &_Py_ID(__getitem__)); + unsigned int tp_version; + PyObject *descriptor = _PyType_LookupRefAndVersion(container_type, &_Py_ID(__getitem__), &tp_version); if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type) { if (!(container_type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_SUBSCR_NOT_HEAP_TYPE); + Py_DECREF(descriptor); goto fail; } PyFunctionObject *func = (PyFunctionObject *)descriptor; @@ -1557,120 +1897,87 @@ _Py_Specialize_BinarySubscr( int kind = function_kind(fcode); if (kind != SIMPLE_FUNCTION) { SPECIALIZATION_FAIL(BINARY_SUBSCR, kind); + Py_DECREF(descriptor); goto fail; } if (fcode->co_argcount != 2) { SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); + Py_DECREF(descriptor); goto fail; } - uint32_t version = _PyFunction_GetVersionForCurrentState(func); - if (version == 0) { - SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_VERSIONS); - goto fail; - } + + PyHeapTypeObject *ht = (PyHeapTypeObject *)container_type; + /* Don't specialize if PEP 523 is active */ if (_PyInterpreterState_GET()->eval_frame) { SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OTHER); + Py_DECREF(descriptor); goto fail; } - PyHeapTypeObject *ht = (PyHeapTypeObject *)container_type; - // This pointer is invalidated by PyType_Modified (see the comment on - // struct _specialization_cache): - ht->_spec_cache.getitem = descriptor; - ht->_spec_cache.getitem_version = version; - instr->op.code = BINARY_SUBSCR_GETITEM; - goto success; + if (_PyType_CacheGetItemForSpecialization(ht, descriptor, (uint32_t)tp_version)) { + specialized_op = BINARY_SUBSCR_GETITEM; + Py_DECREF(descriptor); + goto success; + } } + Py_XDECREF(descriptor); SPECIALIZATION_FAIL(BINARY_SUBSCR, binary_subscr_fail_kind(container_type, sub)); fail: - STAT_INC(BINARY_SUBSCR, failure); - assert(!PyErr_Occurred()); - instr->op.code = BINARY_SUBSCR; - cache->counter = adaptive_counter_backoff(cache->counter); + unspecialize(instr); return; success: - STAT_INC(BINARY_SUBSCR, success); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_cooldown(); + specialize(instr, specialized_op); } -void -_Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr) + +#ifdef Py_STATS +static int +store_subscr_fail_kind(PyObject *container, PyObject *sub) { - assert(ENABLE_SPECIALIZATION); - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)(instr + 1); PyTypeObject *container_type = Py_TYPE(container); - if (container_type == &PyList_Type) { - if (PyLong_CheckExact(sub)) { - if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub) - && ((PyLongObject *)sub)->long_value.ob_digit[0] < (size_t)PyList_GET_SIZE(container)) - { - instr->op.code = STORE_SUBSCR_LIST_INT; - goto success; - } - else { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); - goto fail; - } - } - else if (PySlice_Check(sub)) { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_LIST_SLICE); - goto fail; - } - else { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER); - goto fail; - } - } - if (container_type == &PyDict_Type) { - instr->op.code = STORE_SUBSCR_DICT; - goto success; - } -#ifdef Py_STATS PyMappingMethods *as_mapping = container_type->tp_as_mapping; if (as_mapping && (as_mapping->mp_ass_subscript == PyDict_Type.tp_as_mapping->mp_ass_subscript)) { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE); - goto fail; + return SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE; } if (PyObject_CheckBuffer(container)) { if (PyLong_CheckExact(sub) && (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub))) { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); + return SPEC_FAIL_OUT_OF_RANGE; } else if (strcmp(container_type->tp_name, "array.array") == 0) { if (PyLong_CheckExact(sub)) { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_ARRAY_INT); + return SPEC_FAIL_SUBSCR_ARRAY_INT; } else if (PySlice_Check(sub)) { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_ARRAY_SLICE); + return SPEC_FAIL_SUBSCR_ARRAY_SLICE; } else { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER); + return SPEC_FAIL_OTHER; } } else if (PyByteArray_CheckExact(container)) { if (PyLong_CheckExact(sub)) { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BYTEARRAY_INT); + return SPEC_FAIL_SUBSCR_BYTEARRAY_INT; } else if (PySlice_Check(sub)) { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE); + return SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE; } else { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER); + return SPEC_FAIL_OTHER; } } else { if (PyLong_CheckExact(sub)) { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BUFFER_INT); + return SPEC_FAIL_SUBSCR_BUFFER_INT; } else if (PySlice_Check(sub)) { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BUFFER_SLICE); + return SPEC_FAIL_SUBSCR_BUFFER_SLICE; } else { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER); + return SPEC_FAIL_OTHER; } } - goto fail; + return SPEC_FAIL_OTHER; } PyObject *descriptor = _PyType_Lookup(container_type, &_Py_ID(__setitem__)); if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type) { @@ -1678,59 +1985,85 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins PyCodeObject *code = (PyCodeObject *)func->func_code; int kind = function_kind(code); if (kind == SIMPLE_FUNCTION) { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_PY_SIMPLE); + return SPEC_FAIL_SUBSCR_PY_SIMPLE; } else { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_PY_OTHER); + return SPEC_FAIL_SUBSCR_PY_OTHER; } - goto fail; } -#endif // Py_STATS - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER); -fail: - STAT_INC(STORE_SUBSCR, failure); - assert(!PyErr_Occurred()); - instr->op.code = STORE_SUBSCR; - cache->counter = adaptive_counter_backoff(cache->counter); - return; -success: - STAT_INC(STORE_SUBSCR, success); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_cooldown(); + return SPEC_FAIL_OTHER; } +#endif -/* Returns a borrowed reference. - * The reference is only valid if guarded by a type version check. - */ -static PyFunctionObject * -get_init_for_simple_managed_python_class(PyTypeObject *tp) +void +_Py_Specialize_StoreSubscr(_PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr) +{ + PyObject *container = PyStackRef_AsPyObjectBorrow(container_st); + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + + assert(ENABLE_SPECIALIZATION_FT); + PyTypeObject *container_type = Py_TYPE(container); + if (container_type == &PyList_Type) { + if (PyLong_CheckExact(sub)) { + if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub) + && ((PyLongObject *)sub)->long_value.ob_digit[0] < (size_t)PyList_GET_SIZE(container)) + { + specialize(instr, STORE_SUBSCR_LIST_INT); + return; + } + else { + SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); + unspecialize(instr); + return; + } + } + else if (PySlice_Check(sub)) { + SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_LIST_SLICE); + unspecialize(instr); + return; + } + else { + SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER); + unspecialize(instr); + return; + } + } + if (container_type == &PyDict_Type) { + specialize(instr, STORE_SUBSCR_DICT); + return; + } + SPECIALIZATION_FAIL(STORE_SUBSCR, store_subscr_fail_kind(container, sub)); + unspecialize(instr); +} + +/* Returns a strong reference. */ +static PyObject * +get_init_for_simple_managed_python_class(PyTypeObject *tp, unsigned int *tp_version) { assert(tp->tp_new == PyBaseObject_Type.tp_new); if (tp->tp_alloc != PyType_GenericAlloc) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OVERRIDDEN); return NULL; } - if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_NO_DICT); - return NULL; - } - if (!(tp->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + unsigned long tp_flags = PyType_GetFlags(tp); + if (!(tp_flags & Py_TPFLAGS_HEAPTYPE)) { /* Is this possible? */ SPECIALIZATION_FAIL(CALL, SPEC_FAIL_EXPECTED_ERROR); return NULL; } - PyObject *init = _PyType_Lookup(tp, &_Py_ID(__init__)); + PyObject *init = _PyType_LookupRefAndVersion(tp, &_Py_ID(__init__), tp_version); if (init == NULL || !PyFunction_Check(init)) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_PYTHON); + Py_XDECREF(init); return NULL; } int kind = function_kind((PyCodeObject *)PyFunction_GET_CODE(init)); if (kind != SIMPLE_FUNCTION) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_SIMPLE); + Py_DECREF(init); return NULL; } - ((PyHeapTypeObject *)tp)->_spec_cache.init = init; - return (PyFunctionObject *)init; + return init; } static int @@ -1742,97 +2075,49 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) int oparg = instr->op.arg; if (nargs == 1 && oparg == 1) { if (tp == &PyUnicode_Type) { - instr->op.code = CALL_STR_1; + specialize(instr, CALL_STR_1); return 0; } else if (tp == &PyType_Type) { - instr->op.code = CALL_TYPE_1; + specialize(instr, CALL_TYPE_1); return 0; } else if (tp == &PyTuple_Type) { - instr->op.code = CALL_TUPLE_1; + specialize(instr, CALL_TUPLE_1); return 0; } } if (tp->tp_vectorcall != NULL) { - instr->op.code = CALL_BUILTIN_CLASS; + specialize(instr, CALL_BUILTIN_CLASS); return 0; } - SPECIALIZATION_FAIL(CALL, tp == &PyUnicode_Type ? - SPEC_FAIL_CALL_STR : SPEC_FAIL_CALL_CLASS_NO_VECTORCALL); - return -1; + goto generic; } if (Py_TYPE(tp) != &PyType_Type) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_METACLASS); - return -1; + goto generic; } if (tp->tp_new == PyBaseObject_Type.tp_new) { - PyFunctionObject *init = get_init_for_simple_managed_python_class(tp); - if (type_get_version(tp, CALL) == 0) { + unsigned int tp_version = 0; + PyObject *init = get_init_for_simple_managed_python_class(tp, &tp_version); + if (!tp_version) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS); + Py_XDECREF(init); return -1; } - if (init != NULL) { - if (((PyCodeObject *)init->func_code)->co_argcount != nargs+1) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); - return -1; - } + if (init != NULL && _PyType_CacheInitForSpecialization( + (PyHeapTypeObject *)tp, init, tp_version)) { _PyCallCache *cache = (_PyCallCache *)(instr + 1); - write_u32(cache->func_version, tp->tp_version_tag); - _Py_SET_OPCODE(*instr, CALL_ALLOC_AND_ENTER_INIT); + write_u32(cache->func_version, tp_version); + specialize(instr, CALL_ALLOC_AND_ENTER_INIT); + Py_DECREF(init); return 0; } - return -1; - } - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_CLASS_MUTABLE); - return -1; -} - -#ifdef Py_STATS -static int -builtin_call_fail_kind(int ml_flags) -{ - switch (ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | - METH_KEYWORDS | METH_METHOD)) { - case METH_VARARGS: - return SPEC_FAIL_CALL_CFUNC_VARARGS; - case METH_VARARGS | METH_KEYWORDS: - return SPEC_FAIL_CALL_CFUNC_VARARGS_KEYWORDS; - case METH_NOARGS: - return SPEC_FAIL_CALL_CFUNC_NOARGS; - case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: - return SPEC_FAIL_CALL_CFUNC_METHOD_FASTCALL_KEYWORDS; - /* These cases should be optimized, but return "other" just in case */ - case METH_O: - case METH_FASTCALL: - case METH_FASTCALL | METH_KEYWORDS: - return SPEC_FAIL_OTHER; - default: - return SPEC_FAIL_CALL_BAD_CALL_FLAGS; - } -} - -static int -meth_descr_call_fail_kind(int ml_flags) -{ - switch (ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | - METH_KEYWORDS | METH_METHOD)) { - case METH_VARARGS: - return SPEC_FAIL_CALL_METH_DESCR_VARARGS; - case METH_VARARGS | METH_KEYWORDS: - return SPEC_FAIL_CALL_METH_DESCR_VARARGS_KEYWORDS; - case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: - return SPEC_FAIL_CALL_METH_DESCR_METHOD_FASTCALL_KEYWORDS; - /* These cases should be optimized, but return "other" just in case */ - case METH_NOARGS: - case METH_O: - case METH_FASTCALL: - case METH_FASTCALL | METH_KEYWORDS: - return SPEC_FAIL_OTHER; - default: - return SPEC_FAIL_CALL_BAD_CALL_FLAGS; + Py_XDECREF(init); } +generic: + specialize(instr, CALL_NON_PY_GENERAL); + return 0; } -#endif // Py_STATS static int specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, @@ -1846,7 +2131,7 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); return -1; } - instr->op.code = CALL_METHOD_DESCRIPTOR_NOARGS; + specialize(instr, CALL_METHOD_DESCRIPTOR_NOARGS); return 0; } case METH_O: { @@ -1860,23 +2145,23 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, bool pop = (next.op.code == POP_TOP); int oparg = instr->op.arg; if ((PyObject *)descr == list_append && oparg == 1 && pop) { - instr->op.code = CALL_LIST_APPEND; + specialize(instr, CALL_LIST_APPEND); return 0; } - instr->op.code = CALL_METHOD_DESCRIPTOR_O; + specialize(instr, CALL_METHOD_DESCRIPTOR_O); return 0; } case METH_FASTCALL: { - instr->op.code = CALL_METHOD_DESCRIPTOR_FAST; + specialize(instr, CALL_METHOD_DESCRIPTOR_FAST); return 0; } case METH_FASTCALL | METH_KEYWORDS: { - instr->op.code = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; + specialize(instr, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); return 0; } } - SPECIALIZATION_FAIL(CALL, meth_descr_call_fail_kind(descr->d_method->ml_flags)); - return -1; + specialize(instr, CALL_NON_PY_GENERAL); + return 0; } static int @@ -1891,37 +2176,56 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523); return -1; } - if (kind != SIMPLE_FUNCTION) { - SPECIALIZATION_FAIL(CALL, kind); + int argcount = -1; + if (kind == SPEC_FAIL_CODE_NOT_OPTIMIZED) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CODE_NOT_OPTIMIZED); return -1; } - int argcount = code->co_argcount; - int defcount = func->func_defaults == NULL ? 0 : (int)PyTuple_GET_SIZE(func->func_defaults); - int min_args = argcount-defcount; - // GH-105840: min_args is negative when somebody sets too many __defaults__! - if (min_args < 0 || nargs > argcount || nargs < min_args) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); - return -1; + if (kind == SIMPLE_FUNCTION) { + argcount = code->co_argcount; } - assert(nargs <= argcount && nargs >= min_args); - assert(min_args >= 0 && defcount >= 0); - assert(defcount == 0 || func->func_defaults != NULL); int version = _PyFunction_GetVersionForCurrentState(func); - if (version == 0) { + if (!_PyFunction_IsVersionValid(version)) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS); return -1; } write_u32(cache->func_version, version); - if (argcount == nargs) { - instr->op.code = bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS; + uint8_t opcode; + if (argcount == nargs + bound_method) { + opcode = + bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS; + } + else { + opcode = bound_method ? CALL_BOUND_METHOD_GENERAL : CALL_PY_GENERAL; } - else if (bound_method) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_BOUND_METHOD); + specialize(instr, opcode); + return 0; +} + + +static int +specialize_py_call_kw(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, + bool bound_method) +{ + _PyCallCache *cache = (_PyCallCache *)(instr + 1); + PyCodeObject *code = (PyCodeObject *)func->func_code; + int kind = function_kind(code); + /* Don't specialize if PEP 523 is active */ + if (_PyInterpreterState_GET()->eval_frame) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523); return -1; } - else { - instr->op.code = CALL_PY_WITH_DEFAULTS; + if (kind == SPEC_FAIL_CODE_NOT_OPTIMIZED) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CODE_NOT_OPTIMIZED); + return -1; + } + int version = _PyFunction_GetVersionForCurrentState(func); + if (!_PyFunction_IsVersionValid(version)) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS); + return -1; } + write_u32(cache->func_version, version); + specialize(instr, bound_method ? CALL_KW_BOUND_METHOD : CALL_KW_PY); return 0; } @@ -1929,6 +2233,7 @@ static int specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) { if (PyCFunction_GET_FUNCTION(callable) == NULL) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OTHER); return 1; } switch (PyCFunction_GET_FLAGS(callable) & @@ -1942,10 +2247,10 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) /* len(o) */ PyInterpreterState *interp = _PyInterpreterState_GET(); if (callable == interp->callable_cache.len) { - instr->op.code = CALL_LEN; + specialize(instr, CALL_LEN); return 0; } - instr->op.code = CALL_BUILTIN_O; + specialize(instr, CALL_BUILTIN_O); return 0; } case METH_FASTCALL: { @@ -1953,58 +2258,31 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) /* isinstance(o1, o2) */ PyInterpreterState *interp = _PyInterpreterState_GET(); if (callable == interp->callable_cache.isinstance) { - instr->op.code = CALL_ISINSTANCE; + specialize(instr, CALL_ISINSTANCE); return 0; } } - instr->op.code = CALL_BUILTIN_FAST; + specialize(instr, CALL_BUILTIN_FAST); return 0; } case METH_FASTCALL | METH_KEYWORDS: { - instr->op.code = CALL_BUILTIN_FAST_WITH_KEYWORDS; + specialize(instr, CALL_BUILTIN_FAST_WITH_KEYWORDS); return 0; } default: - SPECIALIZATION_FAIL(CALL, - builtin_call_fail_kind(PyCFunction_GET_FLAGS(callable))); - return 1; - } -} - -#ifdef Py_STATS -static int -call_fail_kind(PyObject *callable) -{ - assert(!PyCFunction_CheckExact(callable)); - assert(!PyFunction_Check(callable)); - assert(!PyType_Check(callable)); - assert(!Py_IS_TYPE(callable, &PyMethodDescr_Type)); - assert(!PyMethod_Check(callable)); - if (PyInstanceMethod_Check(callable)) { - return SPEC_FAIL_CALL_INSTANCE_METHOD; - } - // builtin method - else if (PyCMethod_Check(callable)) { - return SPEC_FAIL_CALL_CMETHOD; - } - else if (Py_TYPE(callable) == &PyWrapperDescr_Type) { - return SPEC_FAIL_CALL_OPERATOR_WRAPPER; - } - else if (Py_TYPE(callable) == &_PyMethodWrapper_Type) { - return SPEC_FAIL_CALL_METHOD_WRAPPER; + specialize(instr, CALL_NON_PY_GENERAL); + return 0; } - return SPEC_FAIL_OTHER; } -#endif // Py_STATS - void -_Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) +_Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) { - assert(ENABLE_SPECIALIZATION); + PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st); + + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL); assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL); - _PyCallCache *cache = (_PyCallCache *)(instr + 1); int fail; if (PyCFunction_CheckExact(callable)) { fail = specialize_c_call(callable, instr, nargs); @@ -2021,7 +2299,7 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) else if (PyMethod_Check(callable)) { PyObject *func = ((PyMethodObject *)callable)->im_func; if (PyFunction_Check(func)) { - fail = specialize_py_call((PyFunctionObject *)func, instr, nargs+1, true); + fail = specialize_py_call((PyFunctionObject *)func, instr, nargs, true); } else { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_BOUND_METHOD); @@ -2029,19 +2307,42 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) } } else { - SPECIALIZATION_FAIL(CALL, call_fail_kind(callable)); - fail = -1; + specialize(instr, CALL_NON_PY_GENERAL); + fail = 0; } if (fail) { - STAT_INC(CALL, failure); - assert(!PyErr_Occurred()); - instr->op.code = CALL; - cache->counter = adaptive_counter_backoff(cache->counter); + unspecialize(instr); + } +} + +void +_Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) +{ + PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st); + + assert(ENABLE_SPECIALIZATION_FT); + assert(_PyOpcode_Caches[CALL_KW] == INLINE_CACHE_ENTRIES_CALL_KW); + assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL_KW); + int fail; + if (PyFunction_Check(callable)) { + fail = specialize_py_call_kw((PyFunctionObject *)callable, instr, nargs, false); + } + else if (PyMethod_Check(callable)) { + PyObject *func = ((PyMethodObject *)callable)->im_func; + if (PyFunction_Check(func)) { + fail = specialize_py_call_kw((PyFunctionObject *)func, instr, nargs, true); + } + else { + SPECIALIZATION_FAIL(CALL_KW, SPEC_FAIL_CALL_BOUND_METHOD); + fail = -1; + } } else { - STAT_INC(CALL, success); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_cooldown(); + specialize(instr, CALL_KW_NON_PY); + fail = 0; + } + if (fail) { + unspecialize(instr); } } @@ -2082,6 +2383,12 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) return SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER; case NB_OR: case NB_INPLACE_OR: + if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { + return SPEC_FAIL_BINARY_OP_OR_DIFFERENT_TYPES; + } + if (PyLong_CheckExact(lhs)) { + return SPEC_FAIL_BINARY_OP_OR_INT; + } return SPEC_FAIL_BINARY_OP_OR; case NB_POWER: case NB_INPLACE_POWER: @@ -2109,19 +2416,174 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) return SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER; case NB_XOR: case NB_INPLACE_XOR: + if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { + return SPEC_FAIL_BINARY_OP_XOR_DIFFERENT_TYPES; + } + if (PyLong_CheckExact(lhs)) { + return SPEC_FAIL_BINARY_OP_XOR_INT; + } return SPEC_FAIL_BINARY_OP_XOR; } Py_UNREACHABLE(); } -#endif // Py_STATS +#endif + +/** Binary Op Specialization Extensions */ + +/* long-long */ + +static inline int +is_compactlong(PyObject *v) +{ + return PyLong_CheckExact(v) && + _PyLong_IsCompact((PyLongObject *)v); +} + +static int +compactlongs_guard(PyObject *lhs, PyObject *rhs) +{ + return (is_compactlong(lhs) && is_compactlong(rhs)); +} + +#define BITWISE_LONGS_ACTION(NAME, OP) \ + static PyObject * \ + (NAME)(PyObject *lhs, PyObject *rhs) \ + { \ + Py_ssize_t rhs_val = _PyLong_CompactValue((PyLongObject *)rhs); \ + Py_ssize_t lhs_val = _PyLong_CompactValue((PyLongObject *)lhs); \ + return PyLong_FromSsize_t(lhs_val OP rhs_val); \ + } +BITWISE_LONGS_ACTION(compactlongs_or, |) +BITWISE_LONGS_ACTION(compactlongs_and, &) +BITWISE_LONGS_ACTION(compactlongs_xor, ^) +#undef BITWISE_LONGS_ACTION + +/* float-long */ + +static inline int +float_compactlong_guard(PyObject *lhs, PyObject *rhs) +{ + return ( + PyFloat_CheckExact(lhs) && + !isnan(PyFloat_AsDouble(lhs)) && + PyLong_CheckExact(rhs) && + _PyLong_IsCompact((PyLongObject *)rhs) + ); +} + +static inline int +nonzero_float_compactlong_guard(PyObject *lhs, PyObject *rhs) +{ + return ( + float_compactlong_guard(lhs, rhs) && !PyLong_IsZero(rhs) + ); +} + +#define FLOAT_LONG_ACTION(NAME, OP) \ + static PyObject * \ + (NAME)(PyObject *lhs, PyObject *rhs) \ + { \ + double lhs_val = PyFloat_AsDouble(lhs); \ + Py_ssize_t rhs_val = _PyLong_CompactValue((PyLongObject *)rhs); \ + return PyFloat_FromDouble(lhs_val OP rhs_val); \ + } +FLOAT_LONG_ACTION(float_compactlong_add, +) +FLOAT_LONG_ACTION(float_compactlong_subtract, -) +FLOAT_LONG_ACTION(float_compactlong_multiply, *) +FLOAT_LONG_ACTION(float_compactlong_true_div, /) +#undef FLOAT_LONG_ACTION + +/* long-float */ + +static inline int +compactlong_float_guard(PyObject *lhs, PyObject *rhs) +{ + return ( + PyLong_CheckExact(lhs) && + _PyLong_IsCompact((PyLongObject *)lhs) && + PyFloat_CheckExact(rhs) && + !isnan(PyFloat_AsDouble(rhs)) + ); +} + +static inline int +nonzero_compactlong_float_guard(PyObject *lhs, PyObject *rhs) +{ + return ( + compactlong_float_guard(lhs, rhs) && PyFloat_AsDouble(rhs) != 0.0 + ); +} + +#define LONG_FLOAT_ACTION(NAME, OP) \ + static PyObject * \ + (NAME)(PyObject *lhs, PyObject *rhs) \ + { \ + double rhs_val = PyFloat_AsDouble(rhs); \ + Py_ssize_t lhs_val = _PyLong_CompactValue((PyLongObject *)lhs); \ + return PyFloat_FromDouble(lhs_val OP rhs_val); \ + } +LONG_FLOAT_ACTION(compactlong_float_add, +) +LONG_FLOAT_ACTION(compactlong_float_subtract, -) +LONG_FLOAT_ACTION(compactlong_float_multiply, *) +LONG_FLOAT_ACTION(compactlong_float_true_div, /) +#undef LONG_FLOAT_ACTION + +static _PyBinaryOpSpecializationDescr compactlongs_specs[NB_OPARG_LAST+1] = { + [NB_OR] = {compactlongs_guard, compactlongs_or}, + [NB_AND] = {compactlongs_guard, compactlongs_and}, + [NB_XOR] = {compactlongs_guard, compactlongs_xor}, + [NB_INPLACE_OR] = {compactlongs_guard, compactlongs_or}, + [NB_INPLACE_AND] = {compactlongs_guard, compactlongs_and}, + [NB_INPLACE_XOR] = {compactlongs_guard, compactlongs_xor}, +}; + +static _PyBinaryOpSpecializationDescr float_compactlong_specs[NB_OPARG_LAST+1] = { + [NB_ADD] = {float_compactlong_guard, float_compactlong_add}, + [NB_SUBTRACT] = {float_compactlong_guard, float_compactlong_subtract}, + [NB_TRUE_DIVIDE] = {nonzero_float_compactlong_guard, float_compactlong_true_div}, + [NB_MULTIPLY] = {float_compactlong_guard, float_compactlong_multiply}, +}; + +static _PyBinaryOpSpecializationDescr compactlong_float_specs[NB_OPARG_LAST+1] = { + [NB_ADD] = {compactlong_float_guard, compactlong_float_add}, + [NB_SUBTRACT] = {compactlong_float_guard, compactlong_float_subtract}, + [NB_TRUE_DIVIDE] = {nonzero_compactlong_float_guard, compactlong_float_true_div}, + [NB_MULTIPLY] = {compactlong_float_guard, compactlong_float_multiply}, +}; + +static int +binary_op_extended_specialization(PyObject *lhs, PyObject *rhs, int oparg, + _PyBinaryOpSpecializationDescr **descr) +{ +#define LOOKUP_SPEC(TABLE, OPARG) \ + if ((TABLE)[(OPARG)].action) { \ + if ((TABLE)[(OPARG)].guard(lhs, rhs)) { \ + *descr = &((TABLE)[OPARG]); \ + return 1; \ + } \ + } + + LOOKUP_SPEC(compactlong_float_specs, oparg); + LOOKUP_SPEC(float_compactlong_specs, oparg); + LOOKUP_SPEC(compactlongs_specs, oparg); +#undef LOOKUP_SPEC + return 0; +} void -_Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, - int oparg, PyObject **locals) +_Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *instr, + int oparg, _PyStackRef *locals) { - assert(ENABLE_SPECIALIZATION); + PyObject *lhs = PyStackRef_AsPyObjectBorrow(lhs_st); + PyObject *rhs = PyStackRef_AsPyObjectBorrow(rhs_st); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[BINARY_OP] == INLINE_CACHE_ENTRIES_BINARY_OP); + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(instr + 1); + if (instr->op.code == BINARY_OP_EXTEND) { + write_ptr(cache->external_cache, NULL); + } + switch (oparg) { case NB_ADD: case NB_INPLACE_ADD: @@ -2131,20 +2593,20 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, if (PyUnicode_CheckExact(lhs)) { _Py_CODEUNIT next = instr[INLINE_CACHE_ENTRIES_BINARY_OP + 1]; bool to_store = (next.op.code == STORE_FAST); - if (to_store && locals[next.op.arg] == lhs) { - instr->op.code = BINARY_OP_INPLACE_ADD_UNICODE; - goto success; + if (to_store && PyStackRef_AsPyObjectBorrow(locals[next.op.arg]) == lhs) { + specialize(instr, BINARY_OP_INPLACE_ADD_UNICODE); + return; } - instr->op.code = BINARY_OP_ADD_UNICODE; - goto success; + specialize(instr, BINARY_OP_ADD_UNICODE); + return; } if (PyLong_CheckExact(lhs)) { - instr->op.code = BINARY_OP_ADD_INT; - goto success; + specialize(instr, BINARY_OP_ADD_INT); + return; } if (PyFloat_CheckExact(lhs)) { - instr->op.code = BINARY_OP_ADD_FLOAT; - goto success; + specialize(instr, BINARY_OP_ADD_FLOAT); + return; } break; case NB_MULTIPLY: @@ -2153,12 +2615,12 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, break; } if (PyLong_CheckExact(lhs)) { - instr->op.code = BINARY_OP_MULTIPLY_INT; - goto success; + specialize(instr, BINARY_OP_MULTIPLY_INT); + return; } if (PyFloat_CheckExact(lhs)) { - instr->op.code = BINARY_OP_MULTIPLY_FLOAT; - goto success; + specialize(instr, BINARY_OP_MULTIPLY_FLOAT); + return; } break; case NB_SUBTRACT: @@ -2167,23 +2629,26 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, break; } if (PyLong_CheckExact(lhs)) { - instr->op.code = BINARY_OP_SUBTRACT_INT; - goto success; + specialize(instr, BINARY_OP_SUBTRACT_INT); + return; } if (PyFloat_CheckExact(lhs)) { - instr->op.code = BINARY_OP_SUBTRACT_FLOAT; - goto success; + specialize(instr, BINARY_OP_SUBTRACT_FLOAT); + return; } break; } + + _PyBinaryOpSpecializationDescr *descr; + if (binary_op_extended_specialization(lhs, rhs, oparg, &descr)) { + specialize(instr, BINARY_OP_EXTEND); + write_ptr(cache->external_cache, (void*)descr); + return; + } + SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs)); - STAT_INC(BINARY_OP, failure); - instr->op.code = BINARY_OP; - cache->counter = adaptive_counter_backoff(cache->counter); + unspecialize(instr); return; -success: - STAT_INC(BINARY_OP, success); - cache->counter = adaptive_counter_cooldown(); } @@ -2223,25 +2688,28 @@ compare_op_fail_kind(PyObject *lhs, PyObject *rhs) #endif // Py_STATS void -_Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, +_Py_Specialize_CompareOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *instr, int oparg) { - assert(ENABLE_SPECIALIZATION); + PyObject *lhs = PyStackRef_AsPyObjectBorrow(lhs_st); + PyObject *rhs = PyStackRef_AsPyObjectBorrow(rhs_st); + uint8_t specialized_op; + + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP); // All of these specializations compute boolean values, so they're all valid // regardless of the fifth-lowest oparg bit. - _PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1); if (Py_TYPE(lhs) != Py_TYPE(rhs)) { SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs)); goto failure; } if (PyFloat_CheckExact(lhs)) { - instr->op.code = COMPARE_OP_FLOAT; + specialized_op = COMPARE_OP_FLOAT; goto success; } if (PyLong_CheckExact(lhs)) { if (_PyLong_IsCompact((PyLongObject *)lhs) && _PyLong_IsCompact((PyLongObject *)rhs)) { - instr->op.code = COMPARE_OP_INT; + specialized_op = COMPARE_OP_INT; goto success; } else { @@ -2256,19 +2724,16 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, goto failure; } else { - instr->op.code = COMPARE_OP_STR; + specialized_op = COMPARE_OP_STR; goto success; } } SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs)); failure: - STAT_INC(COMPARE_OP, failure); - instr->op.code = COMPARE_OP; - cache->counter = adaptive_counter_backoff(cache->counter); + unspecialize(instr); return; success: - STAT_INC(COMPARE_OP, success); - cache->counter = adaptive_counter_cooldown(); + specialize(instr, specialized_op); } #ifdef Py_STATS @@ -2286,41 +2751,37 @@ unpack_sequence_fail_kind(PyObject *seq) #endif // Py_STATS void -_Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) +_Py_Specialize_UnpackSequence(_PyStackRef seq_st, _Py_CODEUNIT *instr, int oparg) { - assert(ENABLE_SPECIALIZATION); + PyObject *seq = PyStackRef_AsPyObjectBorrow(seq_st); + + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[UNPACK_SEQUENCE] == INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); - _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)(instr + 1); if (PyTuple_CheckExact(seq)) { if (PyTuple_GET_SIZE(seq) != oparg) { SPECIALIZATION_FAIL(UNPACK_SEQUENCE, SPEC_FAIL_EXPECTED_ERROR); - goto failure; + unspecialize(instr); + return; } if (PyTuple_GET_SIZE(seq) == 2) { - instr->op.code = UNPACK_SEQUENCE_TWO_TUPLE; - goto success; + specialize(instr, UNPACK_SEQUENCE_TWO_TUPLE); + return; } - instr->op.code = UNPACK_SEQUENCE_TUPLE; - goto success; + specialize(instr, UNPACK_SEQUENCE_TUPLE); + return; } if (PyList_CheckExact(seq)) { if (PyList_GET_SIZE(seq) != oparg) { SPECIALIZATION_FAIL(UNPACK_SEQUENCE, SPEC_FAIL_EXPECTED_ERROR); - goto failure; + unspecialize(instr); + return; } - instr->op.code = UNPACK_SEQUENCE_LIST; - goto success; + specialize(instr, UNPACK_SEQUENCE_LIST); + return; } SPECIALIZATION_FAIL(UNPACK_SEQUENCE, unpack_sequence_fail_kind(seq)); -failure: - STAT_INC(UNPACK_SEQUENCE, failure); - instr->op.code = UNPACK_SEQUENCE; - cache->counter = adaptive_counter_backoff(cache->counter); - return; -success: - STAT_INC(UNPACK_SEQUENCE, success); - cache->counter = adaptive_counter_cooldown(); + unspecialize(instr); } #ifdef Py_STATS @@ -2397,12 +2858,13 @@ int #endif // Py_STATS void -_Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) +_Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg) { assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[FOR_ITER] == INLINE_CACHE_ENTRIES_FOR_ITER); _PyForIterCache *cache = (_PyForIterCache *)(instr + 1); - PyTypeObject *tp = Py_TYPE(iter); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + PyTypeObject *tp = Py_TYPE(iter_o); if (tp == &PyListIter_Type) { instr->op.code = FOR_ITER_LIST; goto success; @@ -2419,6 +2881,7 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) assert(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == END_FOR || instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == INSTRUMENTED_END_FOR ); + /* Don't specialize if PEP 523 is active */ if (_PyInterpreterState_GET()->eval_frame) { SPECIALIZATION_FAIL(FOR_ITER, SPEC_FAIL_OTHER); goto failure; @@ -2427,7 +2890,7 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) goto success; } SPECIALIZATION_FAIL(FOR_ITER, - _PySpecialization_ClassifyIterator(iter)); + _PySpecialization_ClassifyIterator(iter_o)); failure: STAT_INC(FOR_ITER, failure); instr->op.code = FOR_ITER; @@ -2439,126 +2902,168 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) } void -_Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr) +_Py_Specialize_Send(_PyStackRef receiver_st, _Py_CODEUNIT *instr) { - assert(ENABLE_SPECIALIZATION); + PyObject *receiver = PyStackRef_AsPyObjectBorrow(receiver_st); + + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[SEND] == INLINE_CACHE_ENTRIES_SEND); - _PySendCache *cache = (_PySendCache *)(instr + 1); PyTypeObject *tp = Py_TYPE(receiver); if (tp == &PyGen_Type || tp == &PyCoro_Type) { + /* Don't specialize if PEP 523 is active */ if (_PyInterpreterState_GET()->eval_frame) { SPECIALIZATION_FAIL(SEND, SPEC_FAIL_OTHER); goto failure; } - instr->op.code = SEND_GEN; - goto success; + specialize(instr, SEND_GEN); + return; } SPECIALIZATION_FAIL(SEND, _PySpecialization_ClassifyIterator(receiver)); failure: - STAT_INC(SEND, failure); - instr->op.code = SEND; - cache->counter = adaptive_counter_backoff(cache->counter); - return; -success: - STAT_INC(SEND, success); - cache->counter = adaptive_counter_cooldown(); + unspecialize(instr); +} + +#ifdef Py_STATS +static int +to_bool_fail_kind(PyObject *value) +{ + if (PyByteArray_CheckExact(value)) { + return SPEC_FAIL_TO_BOOL_BYTEARRAY; + } + if (PyBytes_CheckExact(value)) { + return SPEC_FAIL_TO_BOOL_BYTES; + } + if (PyDict_CheckExact(value)) { + return SPEC_FAIL_TO_BOOL_DICT; + } + if (PyFloat_CheckExact(value)) { + return SPEC_FAIL_TO_BOOL_FLOAT; + } + if (PyMemoryView_Check(value)) { + return SPEC_FAIL_TO_BOOL_MEMORY_VIEW; + } + if (PyAnySet_CheckExact(value)) { + return SPEC_FAIL_TO_BOOL_SET; + } + if (PyTuple_CheckExact(value)) { + return SPEC_FAIL_TO_BOOL_TUPLE; + } + return SPEC_FAIL_OTHER; +} +#endif // Py_STATS + +static int +check_type_always_true(PyTypeObject *ty) +{ + PyNumberMethods *nb = ty->tp_as_number; + if (nb && nb->nb_bool) { + return SPEC_FAIL_TO_BOOL_NUMBER; + } + PyMappingMethods *mp = ty->tp_as_mapping; + if (mp && mp->mp_length) { + return SPEC_FAIL_TO_BOOL_MAPPING; + } + PySequenceMethods *sq = ty->tp_as_sequence; + if (sq && sq->sq_length) { + return SPEC_FAIL_TO_BOOL_SEQUENCE; + } + return 0; } void -_Py_Specialize_ToBool(PyObject *value, _Py_CODEUNIT *instr) +_Py_Specialize_ToBool(_PyStackRef value_o, _Py_CODEUNIT *instr) { - assert(ENABLE_SPECIALIZATION); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[TO_BOOL] == INLINE_CACHE_ENTRIES_TO_BOOL); _PyToBoolCache *cache = (_PyToBoolCache *)(instr + 1); + PyObject *value = PyStackRef_AsPyObjectBorrow(value_o); + uint8_t specialized_op; if (PyBool_Check(value)) { - instr->op.code = TO_BOOL_BOOL; + specialized_op = TO_BOOL_BOOL; goto success; } if (PyLong_CheckExact(value)) { - instr->op.code = TO_BOOL_INT; + specialized_op = TO_BOOL_INT; goto success; } if (PyList_CheckExact(value)) { - instr->op.code = TO_BOOL_LIST; + specialized_op = TO_BOOL_LIST; goto success; } if (Py_IsNone(value)) { - instr->op.code = TO_BOOL_NONE; + specialized_op = TO_BOOL_NONE; goto success; } if (PyUnicode_CheckExact(value)) { - instr->op.code = TO_BOOL_STR; + specialized_op = TO_BOOL_STR; goto success; } if (PyType_HasFeature(Py_TYPE(value), Py_TPFLAGS_HEAPTYPE)) { - PyNumberMethods *nb = Py_TYPE(value)->tp_as_number; - if (nb && nb->nb_bool) { - SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_TO_BOOL_NUMBER); - goto failure; - } - PyMappingMethods *mp = Py_TYPE(value)->tp_as_mapping; - if (mp && mp->mp_length) { - SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_TO_BOOL_MAPPING); - goto failure; - } - PySequenceMethods *sq = Py_TYPE(value)->tp_as_sequence; - if (sq && sq->sq_length) { - SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_TO_BOOL_SEQUENCE); - goto failure; - } - if (!PyUnstable_Type_AssignVersionTag(Py_TYPE(value))) { + unsigned int version = 0; + int err = _PyType_Validate(Py_TYPE(value), check_type_always_true, &version); + if (err < 0) { SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_OUT_OF_VERSIONS); goto failure; } - uint32_t version = type_get_version(Py_TYPE(value), TO_BOOL); - if (version == 0) { + else if (err > 0) { + SPECIALIZATION_FAIL(TO_BOOL, err); goto failure; } - instr->op.code = TO_BOOL_ALWAYS_TRUE; - write_u32(cache->version, version); + + assert(err == 0); assert(version); + write_u32(cache->version, version); + specialized_op = TO_BOOL_ALWAYS_TRUE; goto success; } + + SPECIALIZATION_FAIL(TO_BOOL, to_bool_fail_kind(value)); +failure: + unspecialize(instr); + return; +success: + specialize(instr, specialized_op); +} + #ifdef Py_STATS - if (PyByteArray_CheckExact(value)) { - SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_TO_BOOL_BYTEARRAY); - goto failure; - } - if (PyBytes_CheckExact(value)) { - SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_TO_BOOL_BYTES); - goto failure; +static int +containsop_fail_kind(PyObject *value) { + if (PyUnicode_CheckExact(value)) { + return SPEC_FAIL_CONTAINS_OP_STR; } - if (PyDict_CheckExact(value)) { - SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_TO_BOOL_DICT); - goto failure; + if (PyList_CheckExact(value)) { + return SPEC_FAIL_CONTAINS_OP_LIST; } - if (PyFloat_CheckExact(value)) { - SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_TO_BOOL_FLOAT); - goto failure; + if (PyTuple_CheckExact(value)) { + return SPEC_FAIL_CONTAINS_OP_TUPLE; } - if (PyMemoryView_Check(value)) { - SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_TO_BOOL_MEMORY_VIEW); - goto failure; + if (PyType_Check(value)) { + return SPEC_FAIL_CONTAINS_OP_USER_CLASS; } - if (PyAnySet_CheckExact(value)) { - SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_TO_BOOL_SET); - goto failure; + return SPEC_FAIL_OTHER; +} +#endif + +void +_Py_Specialize_ContainsOp(_PyStackRef value_st, _Py_CODEUNIT *instr) +{ + PyObject *value = PyStackRef_AsPyObjectBorrow(value_st); + + assert(ENABLE_SPECIALIZATION_FT); + assert(_PyOpcode_Caches[CONTAINS_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP); + if (PyDict_CheckExact(value)) { + specialize(instr, CONTAINS_OP_DICT); + return; } - if (PyTuple_CheckExact(value)) { - SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_TO_BOOL_TUPLE); - goto failure; + if (PySet_CheckExact(value) || PyFrozenSet_CheckExact(value)) { + specialize(instr, CONTAINS_OP_SET); + return; } - SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_OTHER); -#endif // Py_STATS -failure: - STAT_INC(TO_BOOL, failure); - instr->op.code = TO_BOOL; - cache->counter = adaptive_counter_backoff(cache->counter); + + SPECIALIZATION_FAIL(CONTAINS_OP, containsop_fail_kind(value)); + unspecialize(instr); return; -success: - STAT_INC(TO_BOOL, success); - cache->counter = adaptive_counter_cooldown(); } /* Code init cleanup. @@ -2568,6 +3073,16 @@ _Py_Specialize_ToBool(PyObject *value, _Py_CODEUNIT *instr) * Ends with a RESUME so that it is not traced. * This is used as a plain code object, not a function, * so must not access globals or builtins. + * There are a few other constraints imposed on the code + * by the free-threaded build: + * + * 1. The RESUME instruction must not be executed. Otherwise we may attempt to + * free the statically allocated TLBC array. + * 2. It must contain no specializable instructions. Specializing multiple + * copies of the same bytecode is not thread-safe in free-threaded builds. + * + * This should be dynamically allocated if either of those restrictions need to + * be lifted. */ #define NO_LOC_4 (128 | (PY_CODE_LOCATION_INFO_NONE << 3) | 3) @@ -2577,7 +3092,14 @@ static const PyBytesObject no_location = { .ob_sval = { NO_LOC_4 } }; -const struct _PyCode_DEF(8) _Py_InitCleanup = { +#ifdef Py_GIL_DISABLED +static _PyCodeArray init_cleanup_tlbc = { + .size = 1, + .entries = {(char*) &_Py_InitCleanup.co_code_adaptive}, +}; +#endif + +const struct _PyCode8 _Py_InitCleanup = { _PyVarObject_HEAD_INIT(&PyCode_Type, 3), .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty), .co_names = (PyObject *)&_Py_SINGLETON(tuple_empty), @@ -2592,6 +3114,9 @@ const struct _PyCode_DEF(8) _Py_InitCleanup = { ._co_firsttraceable = 4, .co_stacksize = 2, .co_framesize = 2 + FRAME_SPECIALS_SIZE, +#ifdef Py_GIL_DISABLED + .co_tlbc = &init_cleanup_tlbc, +#endif .co_code_adaptive = { EXIT_INIT_CHECK, 0, RETURN_VALUE, 0, diff --git a/Python/stackrefs.c b/Python/stackrefs.c new file mode 100644 index 00000000000000..9bb46897685570 --- /dev/null +++ b/Python/stackrefs.c @@ -0,0 +1,156 @@ + +#include "Python.h" + +#include "pycore_stackref.h" + +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) + +#if SIZEOF_VOID_P < 8 +#error "Py_STACKREF_DEBUG requires 64 bit machine" +#endif + +#include "pycore_interp.h" +#include "pycore_hashtable.h" + +typedef struct _table_entry { + PyObject *obj; + const char *classname; + const char *filename; + int linenumber; + const char *filename_borrow; + int linenumber_borrow; +} TableEntry; + +TableEntry * +make_table_entry(PyObject *obj, const char *filename, int linenumber) +{ + TableEntry *result = malloc(sizeof(TableEntry)); + if (result == NULL) { + return NULL; + } + result->obj = obj; + result->classname = Py_TYPE(obj)->tp_name; + result->filename = filename; + result->linenumber = linenumber; + result->filename_borrow = NULL; + return result; +} + +PyObject * +_Py_stackref_get_object(_PyStackRef ref) +{ + if (ref.index == 0) { + return NULL; + } + PyInterpreterState *interp = PyInterpreterState_Get(); + assert(interp != NULL); + if (ref.index >= interp->next_stackref) { + _Py_FatalErrorFormat(__func__, "Garbled stack ref with ID %" PRIu64 "\n", ref.index); + } + TableEntry *entry = _Py_hashtable_get(interp->stackref_debug_table, (void *)ref.index); + if (entry == NULL) { + _Py_FatalErrorFormat(__func__, "Accessing closed stack ref with ID %" PRIu64 "\n", ref.index); + } + return entry->obj; +} + +PyObject * +_Py_stackref_close(_PyStackRef ref) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + if (ref.index >= interp->next_stackref) { + _Py_FatalErrorFormat(__func__, "Garbled stack ref with ID %" PRIu64 "\n", ref.index); + } + PyObject *obj; + if (ref.index <= LAST_PREDEFINED_STACKREF_INDEX) { + // Pre-allocated reference to None, False or True -- Do not clear + TableEntry *entry = _Py_hashtable_get(interp->stackref_debug_table, (void *)ref.index); + obj = entry->obj; + } + else { + TableEntry *entry = _Py_hashtable_steal(interp->stackref_debug_table, (void *)ref.index); + if (entry == NULL) { + _Py_FatalErrorFormat(__func__, "Invalid StackRef with ID %" PRIu64 "\n", (void *)ref.index); + } + obj = entry->obj; + free(entry); + } + return obj; +} + +_PyStackRef +_Py_stackref_create(PyObject *obj, const char *filename, int linenumber) +{ + if (obj == NULL) { + Py_FatalError("Cannot create a stackref for NULL"); + } + PyInterpreterState *interp = PyInterpreterState_Get(); + uint64_t new_id = interp->next_stackref++; + TableEntry *entry = make_table_entry(obj, filename, linenumber); + if (entry == NULL) { + Py_FatalError("No memory left for stackref debug table"); + } + if (_Py_hashtable_set(interp->stackref_debug_table, (void *)new_id, entry) < 0) { + Py_FatalError("No memory left for stackref debug table"); + } + return (_PyStackRef){ .index = new_id }; +} + +void +_Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber) +{ + if (ref.index <= LAST_PREDEFINED_STACKREF_INDEX) { + return; + } + PyInterpreterState *interp = PyInterpreterState_Get(); + TableEntry *entry = _Py_hashtable_get(interp->stackref_debug_table, (void *)ref.index); + if (entry == NULL) { + _Py_FatalErrorFormat(__func__, "Invalid StackRef with ID %" PRIu64 "\n", (void *)ref.index); + } + entry->filename_borrow = filename; + entry->linenumber_borrow = linenumber; +} + + +void +_Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref) +{ + assert(interp->next_stackref >= ref.index); + interp->next_stackref = ref.index+1; + TableEntry *entry = make_table_entry(obj, "builtin-object", 0); + if (entry == NULL) { + Py_FatalError("No memory left for stackref debug table"); + } + if (_Py_hashtable_set(interp->stackref_debug_table, (void *)ref.index, (void *)entry) < 0) { + Py_FatalError("No memory left for stackref debug table"); + } +} + + +static int +report_leak(_Py_hashtable_t *ht, const void *key, const void *value, void *leak) +{ + TableEntry *entry = (TableEntry *)value; + if (!_Py_IsStaticImmortal(entry->obj)) { + *(int *)leak = 1; + printf("Stackref leak. Refers to instance of %s at %p. Created at %s:%d", + entry->classname, entry->obj, entry->filename, entry->linenumber); + if (entry->filename_borrow != NULL) { + printf(". Last borrow at %s:%d",entry->filename_borrow, entry->linenumber_borrow); + } + printf("\n"); + } + return 0; +} + +void +_Py_stackref_report_leaks(PyInterpreterState *interp) +{ + int leak = 0; + _Py_hashtable_foreach(interp->stackref_debug_table, report_leak, &leak); + if (leak) { + Py_FatalError("Stackrefs leaked."); + } +} + +#endif diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 2445a5c838a7d7..584b050fc4bb6e 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -5,6 +5,8 @@ static const char* _Py_stdlib_module_names[] = { "__future__", "_abc", "_aix_support", +"_android_support", +"_apple_support", "_ast", "_asyncio", "_bisect", @@ -19,6 +21,7 @@ static const char* _Py_stdlib_module_names[] = { "_codecs_tw", "_collections", "_collections_abc", +"_colorize", "_compat_pickle", "_compression", "_contextvars", @@ -37,7 +40,11 @@ static const char* _Py_stdlib_module_names[] = { "_hashlib", "_heapq", "_imp", +"_interpchannels", +"_interpqueues", +"_interpreters", "_io", +"_ios_support", "_json", "_locale", "_lsprof", @@ -59,6 +66,7 @@ static const char* _Py_stdlib_module_names[] = { "_pydecimal", "_pyio", "_pylong", +"_pyrepl", "_queue", "_random", "_scproxy", @@ -90,8 +98,10 @@ static const char* _Py_stdlib_module_names[] = { "_weakref", "_weakrefset", "_winapi", +"_wmi", "_zoneinfo", "abc", +"annotationlib", "antigravity", "argparse", "array", diff --git a/Python/structmember.c b/Python/structmember.c index ba881d18a0973d..d36e049d6b5d20 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -4,8 +4,22 @@ #include "Python.h" #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_long.h" // _PyLong_IsNegative() +#include "pycore_object.h" // _Py_TryIncrefCompare(), FT_ATOMIC_*() +#include "pycore_critical_section.h" +static inline PyObject * +member_get_object(const char *addr, const char *obj_addr, PyMemberDef *l) +{ + PyObject *v = FT_ATOMIC_LOAD_PTR(*(PyObject **) addr); + if (v == NULL) { + PyErr_Format(PyExc_AttributeError, + "'%T' object has no attribute '%s'", + (PyObject *)obj_addr, l->name); + } + return v; +} + PyObject * PyMember_GetOne(const char *obj_addr, PyMemberDef *l) { @@ -20,40 +34,40 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) const char* addr = obj_addr + l->offset; switch (l->type) { case Py_T_BOOL: - v = PyBool_FromLong(*(char*)addr); + v = PyBool_FromLong(FT_ATOMIC_LOAD_CHAR_RELAXED(*(char*)addr)); break; case Py_T_BYTE: - v = PyLong_FromLong(*(char*)addr); + v = PyLong_FromLong(FT_ATOMIC_LOAD_CHAR_RELAXED(*(char*)addr)); break; case Py_T_UBYTE: - v = PyLong_FromUnsignedLong(*(unsigned char*)addr); + v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_UCHAR_RELAXED(*(unsigned char*)addr)); break; case Py_T_SHORT: - v = PyLong_FromLong(*(short*)addr); + v = PyLong_FromLong(FT_ATOMIC_LOAD_SHORT_RELAXED(*(short*)addr)); break; case Py_T_USHORT: - v = PyLong_FromUnsignedLong(*(unsigned short*)addr); + v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_USHORT_RELAXED(*(unsigned short*)addr)); break; case Py_T_INT: - v = PyLong_FromLong(*(int*)addr); + v = PyLong_FromLong(FT_ATOMIC_LOAD_INT_RELAXED(*(int*)addr)); break; case Py_T_UINT: - v = PyLong_FromUnsignedLong(*(unsigned int*)addr); + v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_UINT_RELAXED(*(unsigned int*)addr)); break; case Py_T_LONG: - v = PyLong_FromLong(*(long*)addr); + v = PyLong_FromLong(FT_ATOMIC_LOAD_LONG_RELAXED(*(long*)addr)); break; case Py_T_ULONG: - v = PyLong_FromUnsignedLong(*(unsigned long*)addr); + v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_ULONG_RELAXED(*(unsigned long*)addr)); break; case Py_T_PYSSIZET: - v = PyLong_FromSsize_t(*(Py_ssize_t*)addr); + v = PyLong_FromSsize_t(FT_ATOMIC_LOAD_SSIZE_RELAXED(*(Py_ssize_t*)addr)); break; case Py_T_FLOAT: - v = PyFloat_FromDouble((double)*(float*)addr); + v = PyFloat_FromDouble((double)FT_ATOMIC_LOAD_FLOAT_RELAXED(*(float*)addr)); break; case Py_T_DOUBLE: - v = PyFloat_FromDouble(*(double*)addr); + v = PyFloat_FromDouble(FT_ATOMIC_LOAD_DOUBLE_RELAXED(*(double*)addr)); break; case Py_T_STRING: if (*(char**)addr == NULL) { @@ -65,9 +79,11 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) case Py_T_STRING_INPLACE: v = PyUnicode_FromString((char*)addr); break; - case Py_T_CHAR: - v = PyUnicode_FromStringAndSize((char*)addr, 1); + case Py_T_CHAR: { + char char_val = FT_ATOMIC_LOAD_CHAR_RELAXED(*addr); + v = PyUnicode_FromStringAndSize(&char_val, 1); break; + } case _Py_T_OBJECT: v = *(PyObject **)addr; if (v == NULL) @@ -75,23 +91,28 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) Py_INCREF(v); break; case Py_T_OBJECT_EX: - v = *(PyObject **)addr; - if (v == NULL) { - PyObject *obj = (PyObject *)obj_addr; - PyTypeObject *tp = Py_TYPE(obj); - PyErr_Format(PyExc_AttributeError, - "'%.200s' object has no attribute '%s'", - tp->tp_name, l->name); - } + v = member_get_object(addr, obj_addr, l); +#ifndef Py_GIL_DISABLED Py_XINCREF(v); +#else + if (v != NULL) { + if (!_Py_TryIncrefCompare((PyObject **) addr, v)) { + Py_BEGIN_CRITICAL_SECTION((PyObject *) obj_addr); + v = member_get_object(addr, obj_addr, l); + Py_XINCREF(v); + Py_END_CRITICAL_SECTION(); + } + } +#endif break; case Py_T_LONGLONG: - v = PyLong_FromLongLong(*(long long *)addr); + v = PyLong_FromLongLong(FT_ATOMIC_LOAD_LLONG_RELAXED(*(long long *)addr)); break; case Py_T_ULONGLONG: - v = PyLong_FromUnsignedLongLong(*(unsigned long long *)addr); + v = PyLong_FromUnsignedLongLong(FT_ATOMIC_LOAD_ULLONG_RELAXED(*(unsigned long long *)addr)); break; case _Py_T_NONE: + // doesn't require free-threading code path v = Py_NewRef(Py_None); break; default: @@ -118,6 +139,9 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) return -1; } +#ifdef Py_GIL_DISABLED + PyObject *obj = (PyObject *) addr; +#endif addr += l->offset; if ((l->flags & Py_READONLY)) @@ -147,16 +171,16 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) return -1; } if (v == Py_True) - *(char*)addr = (char) 1; + FT_ATOMIC_STORE_CHAR_RELAXED(*(char*)addr, 1); else - *(char*)addr = (char) 0; + FT_ATOMIC_STORE_CHAR_RELAXED(*(char*)addr, 0); break; } case Py_T_BYTE:{ long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(char*)addr = (char)long_val; + FT_ATOMIC_STORE_CHAR_RELAXED(*(char*)addr, (char)long_val); /* XXX: For compatibility, only warn about truncations for now. */ if ((long_val > CHAR_MAX) || (long_val < CHAR_MIN)) @@ -167,7 +191,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(unsigned char*)addr = (unsigned char)long_val; + FT_ATOMIC_STORE_UCHAR_RELAXED(*(unsigned char*)addr, (unsigned char)long_val); if ((long_val > UCHAR_MAX) || (long_val < 0)) WARN("Truncation of value to unsigned char"); break; @@ -176,7 +200,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(short*)addr = (short)long_val; + FT_ATOMIC_STORE_SHORT_RELAXED(*(short*)addr, (short)long_val); if ((long_val > SHRT_MAX) || (long_val < SHRT_MIN)) WARN("Truncation of value to short"); break; @@ -185,7 +209,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(unsigned short*)addr = (unsigned short)long_val; + FT_ATOMIC_STORE_USHORT_RELAXED(*(unsigned short*)addr, (unsigned short)long_val); if ((long_val > USHRT_MAX) || (long_val < 0)) WARN("Truncation of value to unsigned short"); break; @@ -194,7 +218,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(int *)addr = (int)long_val; + FT_ATOMIC_STORE_INT_RELAXED(*(int *)addr, (int)long_val); if ((long_val > INT_MAX) || (long_val < INT_MIN)) WARN("Truncation of value to int"); break; @@ -212,7 +236,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (long_val == -1 && PyErr_Occurred()) { return -1; } - *(unsigned int *)addr = (unsigned int)(unsigned long)long_val; + FT_ATOMIC_STORE_UINT_RELAXED(*(unsigned int *)addr, (unsigned int)(unsigned long)long_val); WARN("Writing negative value into unsigned field"); } else { @@ -221,19 +245,20 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { return -1; } - *(unsigned int*)addr = (unsigned int)ulong_val; + FT_ATOMIC_STORE_UINT_RELAXED(*(unsigned int *)addr, (unsigned int)ulong_val); if (ulong_val > UINT_MAX) { WARN("Truncation of value to unsigned int"); } } break; } - case Py_T_LONG:{ - *(long*)addr = PyLong_AsLong(v); - if ((*(long*)addr == -1) && PyErr_Occurred()) + case Py_T_LONG: { + const long long_val = PyLong_AsLong(v); + if ((long_val == -1) && PyErr_Occurred()) return -1; + FT_ATOMIC_STORE_LONG_RELAXED(*(long*)addr, long_val); break; - } + } case Py_T_ULONG: { /* XXX: For compatibility, accept negative int values as well. */ @@ -247,7 +272,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (long_val == -1 && PyErr_Occurred()) { return -1; } - *(unsigned long *)addr = (unsigned long)long_val; + FT_ATOMIC_STORE_ULONG_RELAXED(*(unsigned long *)addr, (unsigned long)long_val); WARN("Writing negative value into unsigned field"); } else { @@ -256,33 +281,37 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { return -1; } - *(unsigned long*)addr = ulong_val; + FT_ATOMIC_STORE_ULONG_RELAXED(*(unsigned long *)addr, ulong_val); } break; } - case Py_T_PYSSIZET:{ - *(Py_ssize_t*)addr = PyLong_AsSsize_t(v); - if ((*(Py_ssize_t*)addr == (Py_ssize_t)-1) - && PyErr_Occurred()) - return -1; + case Py_T_PYSSIZET: { + const Py_ssize_t ssize_val = PyLong_AsSsize_t(v); + if ((ssize_val == (Py_ssize_t)-1) && PyErr_Occurred()) + return -1; + FT_ATOMIC_STORE_SSIZE_RELAXED(*(Py_ssize_t*)addr, ssize_val); break; - } + } case Py_T_FLOAT:{ double double_val = PyFloat_AsDouble(v); if ((double_val == -1) && PyErr_Occurred()) return -1; - *(float*)addr = (float)double_val; + FT_ATOMIC_STORE_FLOAT_RELAXED(*(float*)addr, (float)double_val); break; } - case Py_T_DOUBLE: - *(double*)addr = PyFloat_AsDouble(v); - if ((*(double*)addr == -1) && PyErr_Occurred()) + case Py_T_DOUBLE: { + const double double_val = PyFloat_AsDouble(v); + if ((double_val == -1) && PyErr_Occurred()) return -1; + FT_ATOMIC_STORE_DOUBLE_RELAXED(*(double *) addr, double_val); break; + } case _Py_T_OBJECT: case Py_T_OBJECT_EX: + Py_BEGIN_CRITICAL_SECTION(obj); oldv = *(PyObject **)addr; - *(PyObject **)addr = Py_XNewRef(v); + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, Py_XNewRef(v)); + Py_END_CRITICAL_SECTION(); Py_XDECREF(oldv); break; case Py_T_CHAR: { @@ -294,7 +323,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) PyErr_BadArgument(); return -1; } - *(char*)addr = string[0]; + FT_ATOMIC_STORE_CHAR_RELAXED(*(char*)addr, string[0]); break; } case Py_T_STRING: @@ -302,10 +331,10 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) PyErr_SetString(PyExc_TypeError, "readonly attribute"); return -1; case Py_T_LONGLONG:{ - long long value; - *(long long*)addr = value = PyLong_AsLongLong(v); + long long value = PyLong_AsLongLong(v); if ((value == -1) && PyErr_Occurred()) return -1; + FT_ATOMIC_STORE_LLONG_RELAXED(*(long long*)addr, value); break; } case Py_T_ULONGLONG: { @@ -319,7 +348,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (long_val == -1 && PyErr_Occurred()) { return -1; } - *(unsigned long long *)addr = (unsigned long long)(long long)long_val; + FT_ATOMIC_STORE_ULLONG_RELAXED(*(unsigned long long *)addr, (unsigned long long)(long long)long_val); WARN("Writing negative value into unsigned field"); } else { @@ -328,7 +357,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (ulonglong_val == (unsigned long long)-1 && PyErr_Occurred()) { return -1; } - *(unsigned long long*)addr = ulonglong_val; + FT_ATOMIC_STORE_ULLONG_RELAXED(*(unsigned long long *)addr, ulonglong_val); } break; } diff --git a/Python/suggestions.c b/Python/suggestions.c index a09b3ce6d9dab2..2ce0bcfd54e23f 100644 --- a/Python/suggestions.c +++ b/Python/suggestions.c @@ -146,7 +146,7 @@ _Py_CalculateSuggestions(PyObject *dir, if (buffer == NULL) { return PyErr_NoMemory(); } - for (int i = 0; i < dir_size; ++i) { + for (Py_ssize_t i = 0; i < dir_size; ++i) { PyObject *item = PyList_GET_ITEM(dir, i); if (_PyUnicode_Equal(name, item)) { continue; diff --git a/Python/symtable.c b/Python/symtable.c index b69452bf77c517..49bd01ba68ac9e 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -58,29 +58,35 @@ #define ANNOTATION_NOT_ALLOWED \ "%s cannot be used within an annotation" -#define TYPEVAR_BOUND_NOT_ALLOWED \ -"%s cannot be used within a TypeVar bound" +#define EXPR_NOT_ALLOWED_IN_TYPE_VARIABLE \ +"%s cannot be used within %s" -#define TYPEALIAS_NOT_ALLOWED \ +#define EXPR_NOT_ALLOWED_IN_TYPE_ALIAS \ "%s cannot be used within a type alias" -#define TYPEPARAM_NOT_ALLOWED \ +#define EXPR_NOT_ALLOWED_IN_TYPE_PARAMETERS \ "%s cannot be used within the definition of a generic" #define DUPLICATE_TYPE_PARAM \ "duplicate type parameter '%U'" +#define ASYNC_WITH_OUTSIDE_ASYNC_FUNC \ +"'async with' outside async function" -#define LOCATION(x) \ - (x)->lineno, (x)->col_offset, (x)->end_lineno, (x)->end_col_offset +#define ASYNC_FOR_OUTSIDE_ASYNC_FUNC \ +"'async for' outside async function" -#define ST_LOCATION(x) \ - (x)->ste_lineno, (x)->ste_col_offset, (x)->ste_end_lineno, (x)->ste_end_col_offset +#define LOCATION(x) SRC_LOCATION_FROM_AST(x) + +#define SET_ERROR_LOCATION(FNAME, L) \ + PyErr_RangedSyntaxLocationObject((FNAME), \ + (L).lineno, (L).col_offset + 1, (L).end_lineno, (L).end_col_offset + 1) + +#define IS_ASYNC_DEF(st) ((st)->st_cur->ste_type == FunctionBlock && (st)->st_cur->ste_coroutine) static PySTEntryObject * ste_new(struct symtable *st, identifier name, _Py_block_ty block, - void *key, int lineno, int col_offset, - int end_lineno, int end_col_offset) + void *key, _Py_SourceLocation loc) { PySTEntryObject *ste = NULL; PyObject *k = NULL; @@ -103,24 +109,21 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_children = NULL; ste->ste_directives = NULL; + ste->ste_mangled_names = NULL; ste->ste_type = block; + ste->ste_scope_info = NULL; + ste->ste_nested = 0; - ste->ste_free = 0; ste->ste_varargs = 0; ste->ste_varkeywords = 0; - ste->ste_opt_lineno = 0; - ste->ste_opt_col_offset = 0; - ste->ste_lineno = lineno; - ste->ste_col_offset = col_offset; - ste->ste_end_lineno = end_lineno; - ste->ste_end_col_offset = end_col_offset; + ste->ste_annotations_used = 0; + ste->ste_loc = loc; if (st->st_cur != NULL && (st->st_cur->ste_nested || _PyST_IsFunctionLike(st->st_cur))) ste->ste_nested = 1; - ste->ste_child_free = 0; ste->ste_generator = 0; ste->ste_coroutine = 0; ste->ste_comprehension = NoComprehension; @@ -131,6 +134,16 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_can_see_class_scope = 0; ste->ste_comp_iter_expr = 0; ste->ste_needs_classdict = 0; + ste->ste_annotation_block = NULL; + + ste->ste_has_docstring = 0; + + ste->ste_method = 0; + if (st->st_cur != NULL && + st->st_cur->ste_type == ClassBlock && + block == FunctionBlock) { + ste->ste_method = 1; + } ste->ste_symbols = PyDict_New(); ste->ste_varnames = PyList_New(0); @@ -153,7 +166,7 @@ static PyObject * ste_repr(PySTEntryObject *ste) { return PyUnicode_FromFormat("", - ste->ste_name, ste->ste_id, ste->ste_lineno); + ste->ste_name, ste->ste_id, ste->ste_loc.lineno); } static void @@ -166,6 +179,8 @@ ste_dealloc(PySTEntryObject *ste) Py_XDECREF(ste->ste_varnames); Py_XDECREF(ste->ste_children); Py_XDECREF(ste->ste_directives); + Py_XDECREF(ste->ste_annotation_block); + Py_XDECREF(ste->ste_mangled_names); PyObject_Free(ste); } @@ -179,7 +194,7 @@ static PyMemberDef ste_memberlist[] = { {"children", _Py_T_OBJECT, OFF(ste_children), Py_READONLY}, {"nested", Py_T_INT, OFF(ste_nested), Py_READONLY}, {"type", Py_T_INT, OFF(ste_type), Py_READONLY}, - {"lineno", Py_T_INT, OFF(ste_lineno), Py_READONLY}, + {"lineno", Py_T_INT, OFF(ste_loc.lineno), Py_READONLY}, {NULL} }; @@ -226,9 +241,7 @@ PyTypeObject PySTEntry_Type = { static int symtable_analyze(struct symtable *st); static int symtable_enter_block(struct symtable *st, identifier name, - _Py_block_ty block, void *ast, - int lineno, int col_offset, - int end_lineno, int end_col_offset); + _Py_block_ty block, void *ast, _Py_SourceLocation loc); static int symtable_exit_block(struct symtable *st); static int symtable_visit_stmt(struct symtable *st, stmt_ty s); static int symtable_visit_expr(struct symtable *st, expr_ty s); @@ -243,15 +256,18 @@ static int symtable_visit_alias(struct symtable *st, alias_ty); static int symtable_visit_comprehension(struct symtable *st, comprehension_ty); static int symtable_visit_keyword(struct symtable *st, keyword_ty); static int symtable_visit_params(struct symtable *st, asdl_arg_seq *args); -static int symtable_visit_annotation(struct symtable *st, expr_ty annotation); +static int symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key); static int symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args); static int symtable_implicit_arg(struct symtable *st, int pos); -static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty); +static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty, + struct _symtable_entry *parent_ste); static int symtable_visit_withitem(struct symtable *st, withitem_ty item); static int symtable_visit_match_case(struct symtable *st, match_case_ty m); static int symtable_visit_pattern(struct symtable *st, pattern_ty s); static int symtable_raise_if_annotation_block(struct symtable *st, const char *, expr_ty); +static int symtable_raise_if_not_coroutine(struct symtable *st, const char *msg, _Py_SourceLocation loc); static int symtable_raise_if_comprehension_block(struct symtable *st, expr_ty); +static int symtable_add_def(struct symtable *st, PyObject *name, int flag, _Py_SourceLocation loc); /* For debugging purposes only */ #if _PY_DUMP_SYMTABLE @@ -263,9 +279,9 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) case ClassBlock: blocktype = "ClassBlock"; break; case ModuleBlock: blocktype = "ModuleBlock"; break; case AnnotationBlock: blocktype = "AnnotationBlock"; break; - case TypeVarBoundBlock: blocktype = "TypeVarBoundBlock"; break; + case TypeVariableBlock: blocktype = "TypeVariableBlock"; break; case TypeAliasBlock: blocktype = "TypeAliasBlock"; break; - case TypeParamBlock: blocktype = "TypeParamBlock"; break; + case TypeParametersBlock: blocktype = "TypeParametersBlock"; break; } const char *comptype = ""; switch (ste->ste_comprehension) { @@ -279,7 +295,7 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) ( "%U=== Symtable for %U ===\n" "%U%s%s\n" - "%U%s%s%s%s%s%s%s%s%s%s%s%s%s\n" + "%U%s%s%s%s%s%s%s%s%s%s%s\n" "%Ulineno: %d col_offset: %d\n" "%U--- Symbols ---\n" ), @@ -290,8 +306,6 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) comptype, prefix, ste->ste_nested ? " nested" : "", - ste->ste_free ? " free" : "", - ste->ste_child_free ? " child_free" : "", ste->ste_generator ? " generator" : "", ste->ste_coroutine ? " coroutine" : "", ste->ste_varargs ? " varargs" : "", @@ -303,8 +317,8 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) ste->ste_comp_iter_target ? " comp_iter_target" : "", ste->ste_can_see_class_scope ? " can_see_class_scope" : "", prefix, - ste->ste_lineno, - ste->ste_col_offset, + ste->ste_loc.lineno, + ste->ste_loc.col_offset, prefix ); assert(msg != NULL); @@ -321,7 +335,6 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) if (flags & DEF_PARAM) printf(" DEF_PARAM"); if (flags & DEF_NONLOCAL) printf(" DEF_NONLOCAL"); if (flags & USE) printf(" USE"); - if (flags & DEF_FREE) printf(" DEF_FREE"); if (flags & DEF_FREE_CLASS) printf(" DEF_FREE_CLASS"); if (flags & DEF_IMPORT) printf(" DEF_IMPORT"); if (flags & DEF_ANNOT) printf(" DEF_ANNOT"); @@ -350,7 +363,7 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) static void dump_symtable(PySTEntryObject* ste) { - PyObject *empty = PyUnicode_FromString(""); + PyObject *empty = Py_GetConstant(Py_CONSTANT_EMPTY_STR); assert(empty != NULL); _dump_symtable(ste, empty); Py_DECREF(empty); @@ -387,11 +400,11 @@ symtable_new(void) } struct symtable * -_PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) +_PySymtable_Build(mod_ty mod, PyObject *filename, _PyFutureFeatures *future) { struct symtable *st = symtable_new(); asdl_stmt_seq *seq; - int i; + Py_ssize_t i; PyThreadState *tstate; int starting_recursion_depth; @@ -417,7 +430,9 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) st->recursion_limit = Py_C_RECURSION_LIMIT; /* Make the initial symbol information gathering pass */ - if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, 0, 0, 0, 0)) { + + _Py_SourceLocation loc0 = {0, 0, 0, 0}; + if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, loc0)) { _PySymtable_Free(st); return NULL; } @@ -426,6 +441,9 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) switch (mod->kind) { case Module_kind: seq = mod->v.Module.body; + if (_PyAST_GetDocString(seq)) { + st->st_cur->ste_has_docstring = 1; + } for (i = 0; i < asdl_seq_LEN(seq); i++) if (!symtable_visit_stmt(st, (stmt_ty)asdl_seq_GET(seq, i))) @@ -502,30 +520,60 @@ _PySymtable_Lookup(struct symtable *st, void *key) return (PySTEntryObject *)v; } +int +_PySymtable_LookupOptional(struct symtable *st, void *key, + PySTEntryObject **out) +{ + PyObject *k = PyLong_FromVoidPtr(key); + if (k == NULL) { + *out = NULL; + return -1; + } + int result = PyDict_GetItemRef(st->st_blocks, k, (PyObject **)out); + Py_DECREF(k); + assert(*out == NULL || PySTEntry_Check(*out)); + return result; +} + long _PyST_GetSymbol(PySTEntryObject *ste, PyObject *name) { - PyObject *v = PyDict_GetItemWithError(ste->ste_symbols, name); - if (!v) + PyObject *v; + if (PyDict_GetItemRef(ste->ste_symbols, name, &v) < 0) { + return -1; + } + if (!v) { return 0; - assert(PyLong_Check(v)); - return PyLong_AS_LONG(v); + } + long symbol = PyLong_AsLong(v); + Py_DECREF(v); + if (symbol < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_SystemError, "invalid symbol"); + } + return -1; + } + return symbol; } int _PyST_GetScope(PySTEntryObject *ste, PyObject *name) { long symbol = _PyST_GetSymbol(ste, name); - return (symbol >> SCOPE_OFFSET) & SCOPE_MASK; + if (symbol < 0) { + return -1; + } + return SYMBOL_TO_SCOPE(symbol); } int _PyST_IsFunctionLike(PySTEntryObject *ste) { return ste->ste_type == FunctionBlock - || ste->ste_type == TypeVarBoundBlock + || ste->ste_type == AnnotationBlock + || ste->ste_type == TypeVariableBlock || ste->ste_type == TypeAliasBlock - || ste->ste_type == TypeParamBlock; + || ste->ste_type == TypeParametersBlock; } static int @@ -599,16 +647,17 @@ error_at_directive(PySTEntryObject *ste, PyObject *name) global: set of all symbol names explicitly declared as global */ -#define SET_SCOPE(DICT, NAME, I) { \ - PyObject *o = PyLong_FromLong(I); \ - if (!o) \ - return 0; \ - if (PyDict_SetItem((DICT), (NAME), o) < 0) { \ +#define SET_SCOPE(DICT, NAME, I) \ + do { \ + PyObject *o = PyLong_FromLong(I); \ + if (!o) \ + return 0; \ + if (PyDict_SetItem((DICT), (NAME), o) < 0) { \ + Py_DECREF(o); \ + return 0; \ + } \ Py_DECREF(o); \ - return 0; \ - } \ - Py_DECREF(o); \ -} + } while(0) /* Decide on scope of name, given flags. @@ -665,7 +714,6 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, return error_at_directive(ste, name); } SET_SCOPE(scopes, name, FREE); - ste->ste_free = 1; return PySet_Add(free, name) >= 0; } if (flags & DEF_BOUND) { @@ -693,11 +741,14 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, // global statement), we want to also treat it as a global in this scope. if (class_entry != NULL) { long class_flags = _PyST_GetSymbol(class_entry, name); + if (class_flags < 0) { + return 0; + } if (class_flags & DEF_GLOBAL) { SET_SCOPE(scopes, name, GLOBAL_EXPLICIT); return 1; } - else if (class_flags & DEF_BOUND && !(class_flags & DEF_NONLOCAL)) { + else if ((class_flags & DEF_BOUND) && !(class_flags & DEF_NONLOCAL)) { SET_SCOPE(scopes, name, GLOBAL_IMPLICIT); return 1; } @@ -714,7 +765,6 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, } if (contains) { SET_SCOPE(scopes, name, FREE); - ste->ste_free = 1; return PySet_Add(free, name) >= 0; } } @@ -731,8 +781,6 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, return 1; } } - if (ste->ste_nested) - ste->ste_free = 1; SET_SCOPE(scopes, name, GLOBAL_IMPLICIT); return 1; } @@ -744,6 +792,9 @@ is_free_in_any_child(PySTEntryObject *entry, PyObject *key) PySTEntryObject *child_ste = (PySTEntryObject *)PyList_GET_ITEM( entry->ste_children, i); long scope = _PyST_GetScope(child_ste, key); + if (scope < 0) { + return -1; + } if (scope == FREE) { return 1; } @@ -762,12 +813,15 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, while (PyDict_Next(comp->ste_symbols, &pos, &k, &v)) { // skip comprehension parameter - long comp_flags = PyLong_AS_LONG(v); + long comp_flags = PyLong_AsLong(v); + if (comp_flags == -1 && PyErr_Occurred()) { + return 0; + } if (comp_flags & DEF_PARAM) { assert(_PyUnicode_EqualToASCIIString(k, ".0")); continue; } - int scope = (comp_flags >> SCOPE_OFFSET) & SCOPE_MASK; + int scope = SYMBOL_TO_SCOPE(comp_flags); int only_flags = comp_flags & ((1 << SCOPE_OFFSET) - 1); if (scope == CELL || only_flags & DEF_COMP_CELL) { if (PySet_Add(inlined_cells, k) < 0) { @@ -778,22 +832,19 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, if (existing == NULL && PyErr_Occurred()) { return 0; } + // __class__ is never allowed to be free through a class scope (see + // drop_class_free) + if (scope == FREE && ste->ste_type == ClassBlock && + _PyUnicode_EqualToASCIIString(k, "__class__")) { + scope = GLOBAL_IMPLICIT; + if (PySet_Discard(comp_free, k) < 0) { + return 0; + } + remove_dunder_class = 1; + } if (!existing) { // name does not exist in scope, copy from comprehension assert(scope != FREE || PySet_Contains(comp_free, k) == 1); - if (scope == FREE && ste->ste_type == ClassBlock && - _PyUnicode_EqualToASCIIString(k, "__class__")) { - // if __class__ is unbound in the enclosing class scope and free - // in the comprehension scope, it needs special handling; just - // letting it be marked as free in class scope will break due to - // drop_class_free - scope = GLOBAL_IMPLICIT; - only_flags &= ~DEF_FREE; - if (PySet_Discard(comp_free, k) < 0) { - return 0; - } - remove_dunder_class = 1; - } PyObject *v_flags = PyLong_FromLong(only_flags); if (v_flags == NULL) { return 0; @@ -806,11 +857,19 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, SET_SCOPE(scopes, k, scope); } else { - if (PyLong_AsLong(existing) & DEF_BOUND) { + long flags = PyLong_AsLong(existing); + if (flags == -1 && PyErr_Occurred()) { + return 0; + } + if ((flags & DEF_BOUND) && ste->ste_type != ClassBlock) { // free vars in comprehension that are locals in outer scope can // now simply be locals, unless they are free in comp children, // or if the outer scope is a class block - if (!is_free_in_any_child(comp, k) && ste->ste_type != ClassBlock) { + int ok = is_free_in_any_child(comp, k); + if (ok < 0) { + return 0; + } + if (!ok) { if (PySet_Discard(comp_free, k) < 0) { return 0; } @@ -818,7 +877,6 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, } } } - comp->ste_free = PySet_Size(comp_free) > 0; if (remove_dunder_class && PyDict_DelItemString(comp->ste_symbols, "__class__") < 0) { return 0; } @@ -846,9 +904,10 @@ analyze_cells(PyObject *scopes, PyObject *free, PyObject *inlined_cells) if (!v_cell) return 0; while (PyDict_Next(scopes, &pos, &name, &v)) { - long scope; - assert(PyLong_Check(v)); - scope = PyLong_AS_LONG(v); + long scope = PyLong_AsLong(v); + if (scope == -1 && PyErr_Occurred()) { + goto error; + } if (scope != LOCAL) continue; int contains = PySet_Contains(free, name); @@ -911,9 +970,10 @@ update_symbols(PyObject *symbols, PyObject *scopes, /* Update scope information for all symbols in this scope */ while (PyDict_Next(symbols, &pos, &name, &v)) { - long scope, flags; - assert(PyLong_Check(v)); - flags = PyLong_AS_LONG(v); + long flags = PyLong_AsLong(v); + if (flags == -1 && PyErr_Occurred()) { + return 0; + } int contains = PySet_Contains(inlined_cells, name); if (contains < 0) { return 0; @@ -921,9 +981,18 @@ update_symbols(PyObject *symbols, PyObject *scopes, if (contains) { flags |= DEF_COMP_CELL; } - v_scope = PyDict_GetItemWithError(scopes, name); - assert(v_scope && PyLong_Check(v_scope)); - scope = PyLong_AS_LONG(v_scope); + if (PyDict_GetItemRef(scopes, name, &v_scope) < 0) { + return 0; + } + if (!v_scope) { + PyErr_SetObject(PyExc_KeyError, name); + return 0; + } + long scope = PyLong_AsLong(v_scope); + Py_DECREF(v_scope); + if (scope == -1 && PyErr_Occurred()) { + return 0; + } flags |= (scope << SCOPE_OFFSET); v_new = PyLong_FromLong(flags); if (!v_new) @@ -956,7 +1025,11 @@ update_symbols(PyObject *symbols, PyObject *scopes, or global in the class scope. */ if (classflag) { - long flags = PyLong_AS_LONG(v) | DEF_FREE_CLASS; + long flags = PyLong_AsLong(v); + if (flags == -1 && PyErr_Occurred()) { + goto error; + } + flags |= DEF_FREE_CLASS; v_new = PyLong_FromLong(flags); if (!v_new) { goto error; @@ -1095,7 +1168,10 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, } while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) { - long flags = PyLong_AS_LONG(v); + long flags = PyLong_AsLong(v); + if (flags == -1 && PyErr_Occurred()) { + goto error; + } if (!analyze_name(ste, scopes, name, flags, bound, local, free, global, type_params, class_entry)) goto error; @@ -1154,10 +1230,12 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, } } - // we inline all non-generator-expression comprehensions + // we inline all non-generator-expression comprehensions, + // except those in annotation scopes that are nested in classes int inline_comp = entry->ste_comprehension && - !entry->ste_generator; + !entry->ste_generator && + !ste->ste_can_see_class_scope; if (!analyze_child_block(entry, newbound, newfree, newglobal, type_params, new_class_entry, &child_free)) @@ -1176,9 +1254,6 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, if (!temp) goto error; Py_DECREF(temp); - /* Check if any children have free variables */ - if (entry->ste_free || entry->ste_child_free) - ste->ste_child_free = 1; } /* Splice children of inlined comprehensions into our children list */ @@ -1315,20 +1390,12 @@ symtable_exit_block(struct symtable *st) } static int -symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, - void *ast, int lineno, int col_offset, - int end_lineno, int end_col_offset) +symtable_enter_existing_block(struct symtable *st, PySTEntryObject* ste) { - PySTEntryObject *prev = NULL, *ste; - - ste = ste_new(st, name, block, ast, lineno, col_offset, end_lineno, end_col_offset); - if (ste == NULL) - return 0; if (PyList_Append(st->st_stack, (PyObject *)ste) < 0) { - Py_DECREF(ste); return 0; } - prev = st->st_cur; + PySTEntryObject *prev = st->st_cur; /* bpo-37757: For now, disallow *all* assignment expressions in the * outermost iterator expression of a comprehension, even those inside * a nested comprehension or a lambda expression. @@ -1336,18 +1403,22 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, if (prev) { ste->ste_comp_iter_expr = prev->ste_comp_iter_expr; } + /* No need to inherit ste_mangled_names in classes, where all names + * are mangled. */ + if (prev && prev->ste_mangled_names != NULL && ste->ste_type != ClassBlock) { + ste->ste_mangled_names = Py_NewRef(prev->ste_mangled_names); + } /* The entry is owned by the stack. Borrow it for st_cur. */ - Py_DECREF(ste); st->st_cur = ste; - /* Annotation blocks shouldn't have any affect on the symbol table since in - * the compilation stage, they will all be transformed to strings. They are - * only created if future 'annotations' feature is activated. */ - if (block == AnnotationBlock) { + /* If "from __future__ import annotations" is active, + * annotation blocks shouldn't have any affect on the symbol table since in + * the compilation stage, they will all be transformed to strings. */ + if (st->st_future->ff_features & CO_FUTURE_ANNOTATIONS && ste->ste_type == AnnotationBlock) { return 1; } - if (block == ModuleBlock) + if (ste->ste_type == ModuleBlock) st->st_global = st->st_cur->ste_symbols; if (prev) { @@ -1358,14 +1429,39 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, return 1; } +static int +symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, + void *ast, _Py_SourceLocation loc) +{ + PySTEntryObject *ste = ste_new(st, name, block, ast, loc); + if (ste == NULL) + return 0; + int result = symtable_enter_existing_block(st, ste); + Py_DECREF(ste); + if (block == AnnotationBlock || block == TypeVariableBlock || block == TypeAliasBlock) { + _Py_DECLARE_STR(format, ".format"); + // We need to insert code that reads this "parameter" to the function. + if (!symtable_add_def(st, &_Py_STR(format), DEF_PARAM, loc)) { + return 0; + } + if (!symtable_add_def(st, &_Py_STR(format), USE, loc)) { + return 0; + } + } + return result; +} + static long symtable_lookup_entry(struct symtable *st, PySTEntryObject *ste, PyObject *name) { - PyObject *mangled = _Py_Mangle(st->st_private, name); + PyObject *mangled = _Py_MaybeMangle(st->st_private, ste, name); if (!mangled) - return 0; + return -1; long ret = _PyST_GetSymbol(ste, mangled); Py_DECREF(mangled); + if (ret < 0) { + return -1; + } return ret; } @@ -1377,32 +1473,30 @@ symtable_lookup(struct symtable *st, PyObject *name) static int symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _symtable_entry *ste, - int lineno, int col_offset, int end_lineno, int end_col_offset) + _Py_SourceLocation loc) { PyObject *o; PyObject *dict; long val; - PyObject *mangled = _Py_Mangle(st->st_private, name); - + PyObject *mangled = _Py_MaybeMangle(st->st_private, st->st_cur, name); if (!mangled) return 0; dict = ste->ste_symbols; if ((o = PyDict_GetItemWithError(dict, mangled))) { - val = PyLong_AS_LONG(o); + val = PyLong_AsLong(o); + if (val == -1 && PyErr_Occurred()) { + goto error; + } if ((flag & DEF_PARAM) && (val & DEF_PARAM)) { /* Is it better to use 'mangled' or 'name' here? */ PyErr_Format(PyExc_SyntaxError, DUPLICATE_ARGUMENT, name); - PyErr_RangedSyntaxLocationObject(st->st_filename, - lineno, col_offset + 1, - end_lineno, end_col_offset + 1); + SET_ERROR_LOCATION(st->st_filename, loc); goto error; } if ((flag & DEF_TYPE_PARAM) && (val & DEF_TYPE_PARAM)) { PyErr_Format(PyExc_SyntaxError, DUPLICATE_TYPE_PARAM, name); - PyErr_RangedSyntaxLocationObject(st->st_filename, - lineno, col_offset + 1, - end_lineno, end_col_offset + 1); + SET_ERROR_LOCATION(st->st_filename, loc); goto error; } val |= flag; @@ -1422,9 +1516,7 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s if (val & (DEF_GLOBAL | DEF_NONLOCAL)) { PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_INNER_LOOP_CONFLICT, name); - PyErr_RangedSyntaxLocationObject(st->st_filename, - lineno, col_offset + 1, - end_lineno, end_col_offset + 1); + SET_ERROR_LOCATION(st->st_filename, loc); goto error; } val |= DEF_COMP_ITER; @@ -1441,16 +1533,20 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s if (flag & DEF_PARAM) { if (PyList_Append(ste->ste_varnames, mangled) < 0) goto error; - } else if (flag & DEF_GLOBAL) { + } else if (flag & DEF_GLOBAL) { /* XXX need to update DEF_GLOBAL for other flags too; perhaps only DEF_FREE_GLOBAL */ - val = flag; + val = 0; if ((o = PyDict_GetItemWithError(st->st_global, mangled))) { - val |= PyLong_AS_LONG(o); + val = PyLong_AsLong(o); + if (val == -1 && PyErr_Occurred()) { + goto error; + } } else if (PyErr_Occurred()) { goto error; } + val |= flag; o = PyLong_FromLong(val); if (o == NULL) goto error; @@ -1468,29 +1564,86 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s return 0; } +static int +check_name(struct symtable *st, PyObject *name, _Py_SourceLocation loc, + expr_context_ty ctx) +{ + if (ctx == Store && _PyUnicode_EqualToASCIIString(name, "__debug__")) { + PyErr_SetString(PyExc_SyntaxError, "cannot assign to __debug__"); + SET_ERROR_LOCATION(st->st_filename, loc); + return 0; + } + if (ctx == Del && _PyUnicode_EqualToASCIIString(name, "__debug__")) { + PyErr_SetString(PyExc_SyntaxError, "cannot delete __debug__"); + SET_ERROR_LOCATION(st->st_filename, loc); + return 0; + } + return 1; +} + +static int +check_keywords(struct symtable *st, asdl_keyword_seq *keywords) +{ + for (Py_ssize_t i = 0; i < asdl_seq_LEN(keywords); i++) { + keyword_ty key = ((keyword_ty)asdl_seq_GET(keywords, i)); + if (key->arg && !check_name(st, key->arg, LOCATION(key), Store)) { + return 0; + } + } + return 1; +} + +static int +check_kwd_patterns(struct symtable *st, pattern_ty p) +{ + assert(p->kind == MatchClass_kind); + asdl_identifier_seq *kwd_attrs = p->v.MatchClass.kwd_attrs; + asdl_pattern_seq *kwd_patterns = p->v.MatchClass.kwd_patterns; + for (Py_ssize_t i = 0; i < asdl_seq_LEN(kwd_attrs); i++) { + _Py_SourceLocation loc = LOCATION(asdl_seq_GET(kwd_patterns, i)); + if (!check_name(st, asdl_seq_GET(kwd_attrs, i), loc, Store)) { + return 0; + } + } + return 1; +} + +static int +symtable_add_def_ctx(struct symtable *st, PyObject *name, int flag, + _Py_SourceLocation loc, expr_context_ty ctx) +{ + int write_mask = DEF_PARAM | DEF_LOCAL | DEF_IMPORT; + if ((flag & write_mask) && !check_name(st, name, loc, ctx)) { + return 0; + } + if ((flag & DEF_TYPE_PARAM) && st->st_cur->ste_mangled_names != NULL) { + if(PySet_Add(st->st_cur->ste_mangled_names, name) < 0) { + return 0; + } + } + return symtable_add_def_helper(st, name, flag, st->st_cur, loc); +} + static int symtable_add_def(struct symtable *st, PyObject *name, int flag, - int lineno, int col_offset, int end_lineno, int end_col_offset) + _Py_SourceLocation loc) { - return symtable_add_def_helper(st, name, flag, st->st_cur, - lineno, col_offset, end_lineno, end_col_offset); + return symtable_add_def_ctx(st, name, flag, loc, + flag == USE ? Load : Store); } static int symtable_enter_type_param_block(struct symtable *st, identifier name, void *ast, int has_defaults, int has_kwdefaults, - enum _stmt_kind kind, - int lineno, int col_offset, - int end_lineno, int end_col_offset) + enum _stmt_kind kind, _Py_SourceLocation loc) { _Py_block_ty current_type = st->st_cur->ste_type; - if(!symtable_enter_block(st, name, TypeParamBlock, ast, lineno, - col_offset, end_lineno, end_col_offset)) { + if(!symtable_enter_block(st, name, TypeParametersBlock, ast, loc)) { return 0; } if (current_type == ClassBlock) { st->st_cur->ste_can_see_class_scope = 1; - if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, lineno, col_offset, end_lineno, end_col_offset)) { + if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, loc)) { return 0; } } @@ -1498,37 +1651,30 @@ symtable_enter_type_param_block(struct symtable *st, identifier name, _Py_DECLARE_STR(type_params, ".type_params"); // It gets "set" when we create the type params tuple and // "used" when we build up the bases. - if (!symtable_add_def(st, &_Py_STR(type_params), DEF_LOCAL, - lineno, col_offset, end_lineno, end_col_offset)) { + if (!symtable_add_def(st, &_Py_STR(type_params), DEF_LOCAL, loc)) { return 0; } - if (!symtable_add_def(st, &_Py_STR(type_params), USE, - lineno, col_offset, end_lineno, end_col_offset)) { + if (!symtable_add_def(st, &_Py_STR(type_params), USE, loc)) { return 0; } - st->st_private = name; // This is used for setting the generic base _Py_DECLARE_STR(generic_base, ".generic_base"); - if (!symtable_add_def(st, &_Py_STR(generic_base), DEF_LOCAL, - lineno, col_offset, end_lineno, end_col_offset)) { + if (!symtable_add_def(st, &_Py_STR(generic_base), DEF_LOCAL, loc)) { return 0; } - if (!symtable_add_def(st, &_Py_STR(generic_base), USE, - lineno, col_offset, end_lineno, end_col_offset)) { + if (!symtable_add_def(st, &_Py_STR(generic_base), USE, loc)) { return 0; } } if (has_defaults) { _Py_DECLARE_STR(defaults, ".defaults"); - if (!symtable_add_def(st, &_Py_STR(defaults), DEF_PARAM, - lineno, col_offset, end_lineno, end_col_offset)) { + if (!symtable_add_def(st, &_Py_STR(defaults), DEF_PARAM, loc)) { return 0; } } if (has_kwdefaults) { _Py_DECLARE_STR(kwdefaults, ".kwdefaults"); - if (!symtable_add_def(st, &_Py_STR(kwdefaults), DEF_PARAM, - lineno, col_offset, end_lineno, end_col_offset)) { + if (!symtable_add_def(st, &_Py_STR(kwdefaults), DEF_PARAM, loc)) { return 0; } } @@ -1542,51 +1688,71 @@ symtable_enter_type_param_block(struct symtable *st, identifier name, VISIT_SEQ_TAIL permits the start of an ASDL sequence to be skipped, which is useful if the first node in the sequence requires special treatment. - VISIT_QUIT macro returns the specified value exiting from the function but - first adjusts current recursion counter depth. -*/ + ENTER_RECURSIVE macro increments the current recursion depth counter. + It should be used at the beginning of the recursive function. -#define VISIT_QUIT(ST, X) \ - return --(ST)->recursion_depth,(X) + LEAVE_RECURSIVE macro decrements the current recursion depth counter. + It should be used at the end of the recursive function. +*/ #define VISIT(ST, TYPE, V) \ - if (!symtable_visit_ ## TYPE((ST), (V))) \ - VISIT_QUIT((ST), 0); - -#define VISIT_SEQ(ST, TYPE, SEQ) { \ - int i; \ - asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ - for (i = 0; i < asdl_seq_LEN(seq); i++) { \ - TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ - if (!symtable_visit_ ## TYPE((ST), elt)) \ - VISIT_QUIT((ST), 0); \ - } \ -} - -#define VISIT_SEQ_TAIL(ST, TYPE, SEQ, START) { \ - int i; \ - asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ - for (i = (START); i < asdl_seq_LEN(seq); i++) { \ - TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ - if (!symtable_visit_ ## TYPE((ST), elt)) \ - VISIT_QUIT((ST), 0); \ - } \ -} - -#define VISIT_SEQ_WITH_NULL(ST, TYPE, SEQ) { \ - int i = 0; \ - asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ - for (i = 0; i < asdl_seq_LEN(seq); i++) { \ - TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ - if (!elt) continue; /* can be NULL */ \ - if (!symtable_visit_ ## TYPE((ST), elt)) \ - VISIT_QUIT((ST), 0); \ - } \ -} + do { \ + if (!symtable_visit_ ## TYPE((ST), (V))) { \ + return 0; \ + } \ + } while(0) + +#define VISIT_SEQ(ST, TYPE, SEQ) \ + do { \ + Py_ssize_t i; \ + asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ + for (i = 0; i < asdl_seq_LEN(seq); i++) { \ + TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ + if (!symtable_visit_ ## TYPE((ST), elt)) \ + return 0; \ + } \ + } while(0) + +#define VISIT_SEQ_TAIL(ST, TYPE, SEQ, START) \ + do { \ + Py_ssize_t i; \ + asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ + for (i = (START); i < asdl_seq_LEN(seq); i++) { \ + TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ + if (!symtable_visit_ ## TYPE((ST), elt)) \ + return 0; \ + } \ + } while(0) + +#define VISIT_SEQ_WITH_NULL(ST, TYPE, SEQ) \ + do { \ + int i = 0; \ + asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \ + for (i = 0; i < asdl_seq_LEN(seq); i++) { \ + TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ + if (!elt) continue; /* can be NULL */ \ + if (!symtable_visit_ ## TYPE((ST), elt)) \ + return 0; \ + } \ + } while(0) + +#define ENTER_RECURSIVE(ST) \ + do { \ + if (++(ST)->recursion_depth > (ST)->recursion_limit) { \ + PyErr_SetString(PyExc_RecursionError, \ + "maximum recursion depth exceeded during compilation"); \ + return 0; \ + } \ + } while(0) + +#define LEAVE_RECURSIVE(ST) \ + do { \ + --(ST)->recursion_depth; \ + } while(0) + static int -symtable_record_directive(struct symtable *st, identifier name, int lineno, - int col_offset, int end_lineno, int end_col_offset) +symtable_record_directive(struct symtable *st, identifier name, _Py_SourceLocation loc) { PyObject *data, *mangled; int res; @@ -1595,10 +1761,11 @@ symtable_record_directive(struct symtable *st, identifier name, int lineno, if (!st->st_cur->ste_directives) return 0; } - mangled = _Py_Mangle(st->st_private, name); + mangled = _Py_MaybeMangle(st->st_private, st->st_cur, name); if (!mangled) return 0; - data = Py_BuildValue("(Niiii)", mangled, lineno, col_offset, end_lineno, end_col_offset); + data = Py_BuildValue("(Niiii)", mangled, loc.lineno, loc.col_offset, + loc.end_lineno, loc.end_col_offset); if (!data) return 0; res = PyList_Append(st->st_cur->ste_directives, data); @@ -1619,17 +1786,48 @@ has_kwonlydefaults(asdl_arg_seq *kwonlyargs, asdl_expr_seq *kw_defaults) } static int -symtable_visit_stmt(struct symtable *st, stmt_ty s) +check_import_from(struct symtable *st, stmt_ty s) { - if (++st->recursion_depth > st->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); - VISIT_QUIT(st, 0); + assert(s->kind == ImportFrom_kind); + _Py_SourceLocation fut = st->st_future->ff_location; + if (s->v.ImportFrom.module && s->v.ImportFrom.level == 0 && + _PyUnicode_EqualToASCIIString(s->v.ImportFrom.module, "__future__") && + ((s->lineno > fut.lineno) || + ((s->lineno == fut.end_lineno) && (s->col_offset > fut.end_col_offset)))) + { + PyErr_SetString(PyExc_SyntaxError, + "from __future__ imports must occur " + "at the beginning of the file"); + SET_ERROR_LOCATION(st->st_filename, LOCATION(s)); + return 0; } + return 1; +} + +static bool +allows_top_level_await(struct symtable *st) +{ + return (st->st_future->ff_features & PyCF_ALLOW_TOP_LEVEL_AWAIT) && + st->st_cur->ste_type == ModuleBlock; +} + + +static void +maybe_set_ste_coroutine_for_module(struct symtable *st, stmt_ty s) +{ + if (allows_top_level_await(st)) { + st->st_cur->ste_coroutine = 1; + } +} + +static int +symtable_visit_stmt(struct symtable *st, stmt_ty s) +{ + ENTER_RECURSIVE(st); switch (s->kind) { - case FunctionDef_kind: + case FunctionDef_kind: { if (!symtable_add_def(st, s->v.FunctionDef.name, DEF_LOCAL, LOCATION(s))) - VISIT_QUIT(st, 0); + return 0; if (s->v.FunctionDef.args->defaults) VISIT_SEQ(st, expr, s->v.FunctionDef.args->defaults); if (s->v.FunctionDef.args->kw_defaults) @@ -1645,68 +1843,95 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) s->v.FunctionDef.args->kw_defaults), s->kind, LOCATION(s))) { - VISIT_QUIT(st, 0); + return 0; } VISIT_SEQ(st, type_param, s->v.FunctionDef.type_params); } + PySTEntryObject *new_ste = ste_new(st, s->v.FunctionDef.name, FunctionBlock, (void *)s, + LOCATION(s)); + if (!new_ste) { + return 0; + } + + if (_PyAST_GetDocString(s->v.FunctionDef.body)) { + new_ste->ste_has_docstring = 1; + } + if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args, - s->v.FunctionDef.returns)) - VISIT_QUIT(st, 0); - if (!symtable_enter_block(st, s->v.FunctionDef.name, - FunctionBlock, (void *)s, - LOCATION(s))) - VISIT_QUIT(st, 0); + s->v.FunctionDef.returns, new_ste)) { + Py_DECREF(new_ste); + return 0; + } + if (!symtable_enter_existing_block(st, new_ste)) { + Py_DECREF(new_ste); + return 0; + } + Py_DECREF(new_ste); VISIT(st, arguments, s->v.FunctionDef.args); VISIT_SEQ(st, stmt, s->v.FunctionDef.body); if (!symtable_exit_block(st)) - VISIT_QUIT(st, 0); + return 0; if (asdl_seq_LEN(s->v.FunctionDef.type_params) > 0) { if (!symtable_exit_block(st)) - VISIT_QUIT(st, 0); + return 0; } break; + } case ClassDef_kind: { PyObject *tmp; if (!symtable_add_def(st, s->v.ClassDef.name, DEF_LOCAL, LOCATION(s))) - VISIT_QUIT(st, 0); + return 0; if (s->v.ClassDef.decorator_list) VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list); + tmp = st->st_private; if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) { if (!symtable_enter_type_param_block(st, s->v.ClassDef.name, (void *)s->v.ClassDef.type_params, false, false, s->kind, LOCATION(s))) { - VISIT_QUIT(st, 0); + return 0; + } + st->st_private = s->v.ClassDef.name; + st->st_cur->ste_mangled_names = PySet_New(NULL); + if (!st->st_cur->ste_mangled_names) { + return 0; } VISIT_SEQ(st, type_param, s->v.ClassDef.type_params); } VISIT_SEQ(st, expr, s->v.ClassDef.bases); + if (!check_keywords(st, s->v.ClassDef.keywords)) { + return 0; + } VISIT_SEQ(st, keyword, s->v.ClassDef.keywords); if (!symtable_enter_block(st, s->v.ClassDef.name, ClassBlock, - (void *)s, s->lineno, s->col_offset, - s->end_lineno, s->end_col_offset)) - VISIT_QUIT(st, 0); - tmp = st->st_private; + (void *)s, LOCATION(s))) { + return 0; + } st->st_private = s->v.ClassDef.name; if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) { if (!symtable_add_def(st, &_Py_ID(__type_params__), DEF_LOCAL, LOCATION(s))) { - VISIT_QUIT(st, 0); + return 0; } _Py_DECLARE_STR(type_params, ".type_params"); if (!symtable_add_def(st, &_Py_STR(type_params), USE, LOCATION(s))) { - VISIT_QUIT(st, 0); + return 0; } } + + if (_PyAST_GetDocString(s->v.ClassDef.body)) { + st->st_cur->ste_has_docstring = 1; + } + VISIT_SEQ(st, stmt, s->v.ClassDef.body); - st->st_private = tmp; if (!symtable_exit_block(st)) - VISIT_QUIT(st, 0); + return 0; if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) { if (!symtable_exit_block(st)) - VISIT_QUIT(st, 0); + return 0; } + st->st_private = tmp; break; } case TypeAlias_kind: { @@ -1721,23 +1946,24 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) (void *)s->v.TypeAlias.type_params, false, false, s->kind, LOCATION(s))) { - VISIT_QUIT(st, 0); + return 0; } VISIT_SEQ(st, type_param, s->v.TypeAlias.type_params); } if (!symtable_enter_block(st, name, TypeAliasBlock, - (void *)s, LOCATION(s))) - VISIT_QUIT(st, 0); + (void *)s, LOCATION(s))) { + return 0; + } st->st_cur->ste_can_see_class_scope = is_in_class; if (is_in_class && !symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(s->v.TypeAlias.value))) { - VISIT_QUIT(st, 0); + return 0; } VISIT(st, expr, s->v.TypeAlias.value); if (!symtable_exit_block(st)) - VISIT_QUIT(st, 0); + return 0; if (is_generic) { if (!symtable_exit_block(st)) - VISIT_QUIT(st, 0); + return 0; } break; } @@ -1755,11 +1981,12 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT(st, expr, s->v.Assign.value); break; case AnnAssign_kind: + st->st_cur->ste_annotations_used = 1; if (s->v.AnnAssign.target->kind == Name_kind) { expr_ty e_name = s->v.AnnAssign.target; long cur = symtable_lookup(st, e_name->v.Name.id); if (cur < 0) { - VISIT_QUIT(st, 0); + return 0; } if ((cur & (DEF_GLOBAL | DEF_NONLOCAL)) && (st->st_cur->ste_symbols != st->st_global) @@ -1767,40 +1994,38 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) PyErr_Format(PyExc_SyntaxError, cur & DEF_GLOBAL ? GLOBAL_ANNOT : NONLOCAL_ANNOT, e_name->v.Name.id); - PyErr_RangedSyntaxLocationObject(st->st_filename, - s->lineno, - s->col_offset + 1, - s->end_lineno, - s->end_col_offset + 1); - VISIT_QUIT(st, 0); + SET_ERROR_LOCATION(st->st_filename, LOCATION(s)); + return 0; } if (s->v.AnnAssign.simple && !symtable_add_def(st, e_name->v.Name.id, DEF_ANNOT | DEF_LOCAL, LOCATION(e_name))) { - VISIT_QUIT(st, 0); + return 0; } else { if (s->v.AnnAssign.value && !symtable_add_def(st, e_name->v.Name.id, DEF_LOCAL, LOCATION(e_name))) { - VISIT_QUIT(st, 0); + return 0; } } } else { VISIT(st, expr, s->v.AnnAssign.target); } - if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation)) { - VISIT_QUIT(st, 0); + if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation, + (void *)((uintptr_t)st->st_cur->ste_id + 1))) { + return 0; } if (s->v.AnnAssign.value) { VISIT(st, expr, s->v.AnnAssign.value); } break; - case AugAssign_kind: + case AugAssign_kind: { VISIT(st, expr, s->v.AugAssign.target); VISIT(st, expr, s->v.AugAssign.value); break; + } case For_kind: VISIT(st, expr, s->v.For.target); VISIT(st, expr, s->v.For.iter); @@ -1855,15 +2080,18 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) break; case ImportFrom_kind: VISIT_SEQ(st, alias, s->v.ImportFrom.names); + if (!check_import_from(st, s)) { + return 0; + } break; case Global_kind: { - int i; + Py_ssize_t i; asdl_identifier_seq *seq = s->v.Global.names; for (i = 0; i < asdl_seq_LEN(seq); i++) { identifier name = (identifier)asdl_seq_GET(seq, i); long cur = symtable_lookup(st, name); if (cur < 0) - VISIT_QUIT(st, 0); + return 0; if (cur & (DEF_PARAM | DEF_LOCAL | USE | DEF_ANNOT)) { const char* msg; if (cur & DEF_PARAM) { @@ -1877,29 +2105,26 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } PyErr_Format(PyExc_SyntaxError, msg, name); - PyErr_RangedSyntaxLocationObject(st->st_filename, - s->lineno, - s->col_offset + 1, - s->end_lineno, - s->end_col_offset + 1); - VISIT_QUIT(st, 0); + SET_ERROR_LOCATION(st->st_filename, LOCATION(s)); + return 0; + } + if (!symtable_add_def(st, name, DEF_GLOBAL, LOCATION(s))) { + return 0; + } + if (!symtable_record_directive(st, name, LOCATION(s))) { + return 0; } - if (!symtable_add_def(st, name, DEF_GLOBAL, LOCATION(s))) - VISIT_QUIT(st, 0); - if (!symtable_record_directive(st, name, s->lineno, s->col_offset, - s->end_lineno, s->end_col_offset)) - VISIT_QUIT(st, 0); } break; } case Nonlocal_kind: { - int i; + Py_ssize_t i; asdl_identifier_seq *seq = s->v.Nonlocal.names; for (i = 0; i < asdl_seq_LEN(seq); i++) { identifier name = (identifier)asdl_seq_GET(seq, i); long cur = symtable_lookup(st, name); if (cur < 0) - VISIT_QUIT(st, 0); + return 0; if (cur & (DEF_PARAM | DEF_LOCAL | USE | DEF_ANNOT)) { const char* msg; if (cur & DEF_PARAM) { @@ -1912,18 +2137,14 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) msg = NONLOCAL_AFTER_ASSIGN; } PyErr_Format(PyExc_SyntaxError, msg, name); - PyErr_RangedSyntaxLocationObject(st->st_filename, - s->lineno, - s->col_offset + 1, - s->end_lineno, - s->end_col_offset + 1); - VISIT_QUIT(st, 0); + SET_ERROR_LOCATION(st->st_filename, LOCATION(s)); + return 0; } if (!symtable_add_def(st, name, DEF_NONLOCAL, LOCATION(s))) - VISIT_QUIT(st, 0); - if (!symtable_record_directive(st, name, s->lineno, s->col_offset, - s->end_lineno, s->end_col_offset)) - VISIT_QUIT(st, 0); + return 0; + if (!symtable_record_directive(st, name, LOCATION(s))) { + return 0; + } } break; } @@ -1939,9 +2160,9 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ(st, withitem, s->v.With.items); VISIT_SEQ(st, stmt, s->v.With.body); break; - case AsyncFunctionDef_kind: + case AsyncFunctionDef_kind: { if (!symtable_add_def(st, s->v.AsyncFunctionDef.name, DEF_LOCAL, LOCATION(s))) - VISIT_QUIT(st, 0); + return 0; if (s->v.AsyncFunctionDef.args->defaults) VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.args->defaults); if (s->v.AsyncFunctionDef.args->kw_defaults) @@ -1958,33 +2179,55 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) s->v.AsyncFunctionDef.args->kw_defaults), s->kind, LOCATION(s))) { - VISIT_QUIT(st, 0); + return 0; } VISIT_SEQ(st, type_param, s->v.AsyncFunctionDef.type_params); } + PySTEntryObject *new_ste = ste_new(st, s->v.FunctionDef.name, FunctionBlock, (void *)s, + LOCATION(s)); + if (!new_ste) { + return 0; + } + + if (_PyAST_GetDocString(s->v.AsyncFunctionDef.body)) { + new_ste->ste_has_docstring = 1; + } + if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args, - s->v.AsyncFunctionDef.returns)) - VISIT_QUIT(st, 0); - if (!symtable_enter_block(st, s->v.AsyncFunctionDef.name, - FunctionBlock, (void *)s, - s->lineno, s->col_offset, - s->end_lineno, s->end_col_offset)) - VISIT_QUIT(st, 0); + s->v.AsyncFunctionDef.returns, new_ste)) { + Py_DECREF(new_ste); + return 0; + } + if (!symtable_enter_existing_block(st, new_ste)) { + Py_DECREF(new_ste); + return 0; + } + Py_DECREF(new_ste); + st->st_cur->ste_coroutine = 1; VISIT(st, arguments, s->v.AsyncFunctionDef.args); VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body); if (!symtable_exit_block(st)) - VISIT_QUIT(st, 0); + return 0; if (asdl_seq_LEN(s->v.AsyncFunctionDef.type_params) > 0) { if (!symtable_exit_block(st)) - VISIT_QUIT(st, 0); + return 0; } break; + } case AsyncWith_kind: + maybe_set_ste_coroutine_for_module(st, s); + if (!symtable_raise_if_not_coroutine(st, ASYNC_WITH_OUTSIDE_ASYNC_FUNC, LOCATION(s))) { + return 0; + } VISIT_SEQ(st, withitem, s->v.AsyncWith.items); VISIT_SEQ(st, stmt, s->v.AsyncWith.body); break; case AsyncFor_kind: + maybe_set_ste_coroutine_for_module(st, s); + if (!symtable_raise_if_not_coroutine(st, ASYNC_FOR_OUTSIDE_ASYNC_FUNC, LOCATION(s))) { + return 0; + } VISIT(st, expr, s->v.AsyncFor.target); VISIT(st, expr, s->v.AsyncFor.iter); VISIT_SEQ(st, stmt, s->v.AsyncFor.body); @@ -1992,7 +2235,8 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ(st, stmt, s->v.AsyncFor.orelse); break; } - VISIT_QUIT(st, 1); + LEAVE_RECURSIVE(st); + return 1; } static int @@ -2016,15 +2260,14 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) */ if (ste->ste_comprehension) { long target_in_scope = symtable_lookup_entry(st, ste, target_name); + if (target_in_scope < 0) { + return 0; + } if ((target_in_scope & DEF_COMP_ITER) && (target_in_scope & DEF_LOCAL)) { PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_CONFLICT, target_name); - PyErr_RangedSyntaxLocationObject(st->st_filename, - e->lineno, - e->col_offset + 1, - e->end_lineno, - e->end_col_offset + 1); - VISIT_QUIT(st, 0); + SET_ERROR_LOCATION(st->st_filename, LOCATION(e)); + return 0; } continue; } @@ -2032,54 +2275,57 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) /* If we find a FunctionBlock entry, add as GLOBAL/LOCAL or NONLOCAL/LOCAL */ if (ste->ste_type == FunctionBlock) { long target_in_scope = symtable_lookup_entry(st, ste, target_name); + if (target_in_scope < 0) { + return 0; + } if (target_in_scope & DEF_GLOBAL) { if (!symtable_add_def(st, target_name, DEF_GLOBAL, LOCATION(e))) - VISIT_QUIT(st, 0); + return 0; } else { - if (!symtable_add_def(st, target_name, DEF_NONLOCAL, LOCATION(e))) - VISIT_QUIT(st, 0); + if (!symtable_add_def(st, target_name, DEF_NONLOCAL, LOCATION(e))) { + return 0; + } + } + if (!symtable_record_directive(st, target_name, LOCATION(e))) { + return 0; } - if (!symtable_record_directive(st, target_name, LOCATION(e))) - VISIT_QUIT(st, 0); return symtable_add_def_helper(st, target_name, DEF_LOCAL, ste, LOCATION(e)); } /* If we find a ModuleBlock entry, add as GLOBAL */ if (ste->ste_type == ModuleBlock) { - if (!symtable_add_def(st, target_name, DEF_GLOBAL, LOCATION(e))) - VISIT_QUIT(st, 0); - if (!symtable_record_directive(st, target_name, LOCATION(e))) - VISIT_QUIT(st, 0); + if (!symtable_add_def(st, target_name, DEF_GLOBAL, LOCATION(e))) { + return 0; + } + if (!symtable_record_directive(st, target_name, LOCATION(e))) { + return 0; + } return symtable_add_def_helper(st, target_name, DEF_GLOBAL, ste, LOCATION(e)); } /* Disallow usage in ClassBlock and type scopes */ if (ste->ste_type == ClassBlock || - ste->ste_type == TypeParamBlock || + ste->ste_type == TypeParametersBlock || ste->ste_type == TypeAliasBlock || - ste->ste_type == TypeVarBoundBlock) { + ste->ste_type == TypeVariableBlock) { switch (ste->ste_type) { case ClassBlock: PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_CLASS); break; - case TypeParamBlock: + case TypeParametersBlock: PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_TYPEPARAM); break; case TypeAliasBlock: PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_TYPEALIAS); break; - case TypeVarBoundBlock: + case TypeVariableBlock: PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_TYPEVAR_BOUND); break; default: Py_UNREACHABLE(); } - PyErr_RangedSyntaxLocationObject(st->st_filename, - e->lineno, - e->col_offset + 1, - e->end_lineno, - e->end_col_offset + 1); - VISIT_QUIT(st, 0); + SET_ERROR_LOCATION(st->st_filename, LOCATION(e)); + return 0; } } @@ -2096,11 +2342,7 @@ symtable_handle_namedexpr(struct symtable *st, expr_ty e) if (st->st_cur->ste_comp_iter_expr > 0) { /* Assignment isn't allowed in a comprehension iterable expression */ PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_ITER_EXPR); - PyErr_RangedSyntaxLocationObject(st->st_filename, - e->lineno, - e->col_offset + 1, - e->end_lineno, - e->end_col_offset + 1); + SET_ERROR_LOCATION(st->st_filename, LOCATION(e)); return 0; } if (st->st_cur->ste_comprehension) { @@ -2116,18 +2358,14 @@ symtable_handle_namedexpr(struct symtable *st, expr_ty e) static int symtable_visit_expr(struct symtable *st, expr_ty e) { - if (++st->recursion_depth > st->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); - VISIT_QUIT(st, 0); - } + ENTER_RECURSIVE(st); switch (e->kind) { case NamedExpr_kind: if (!symtable_raise_if_annotation_block(st, "named expression", e)) { - VISIT_QUIT(st, 0); + return 0; } if(!symtable_handle_namedexpr(st, e)) - VISIT_QUIT(st, 0); + return 0; break; case BoolOp_kind: VISIT_SEQ(st, expr, e->v.BoolOp.values); @@ -2140,30 +2378,18 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT(st, expr, e->v.UnaryOp.operand); break; case Lambda_kind: { - if (st->st_cur->ste_can_see_class_scope) { - // gh-109118 - PyErr_Format(PyExc_SyntaxError, - "Cannot use lambda in annotation scope within class scope"); - PyErr_RangedSyntaxLocationObject(st->st_filename, - e->lineno, - e->col_offset + 1, - e->end_lineno, - e->end_col_offset + 1); - VISIT_QUIT(st, 0); - } if (e->v.Lambda.args->defaults) VISIT_SEQ(st, expr, e->v.Lambda.args->defaults); if (e->v.Lambda.args->kw_defaults) VISIT_SEQ_WITH_NULL(st, expr, e->v.Lambda.args->kw_defaults); if (!symtable_enter_block(st, &_Py_ID(lambda), - FunctionBlock, (void *)e, - e->lineno, e->col_offset, - e->end_lineno, e->end_col_offset)) - VISIT_QUIT(st, 0); + FunctionBlock, (void *)e, LOCATION(e))) { + return 0; + } VISIT(st, arguments, e->v.Lambda.args); VISIT(st, expr, e->v.Lambda.body); if (!symtable_exit_block(st)) - VISIT_QUIT(st, 0); + return 0; break; } case IfExp_kind: @@ -2180,23 +2406,23 @@ symtable_visit_expr(struct symtable *st, expr_ty e) break; case GeneratorExp_kind: if (!symtable_visit_genexp(st, e)) - VISIT_QUIT(st, 0); + return 0; break; case ListComp_kind: if (!symtable_visit_listcomp(st, e)) - VISIT_QUIT(st, 0); + return 0; break; case SetComp_kind: if (!symtable_visit_setcomp(st, e)) - VISIT_QUIT(st, 0); + return 0; break; case DictComp_kind: if (!symtable_visit_dictcomp(st, e)) - VISIT_QUIT(st, 0); + return 0; break; case Yield_kind: if (!symtable_raise_if_annotation_block(st, "yield expression", e)) { - VISIT_QUIT(st, 0); + return 0; } if (e->v.Yield.value) VISIT(st, expr, e->v.Yield.value); @@ -2207,7 +2433,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) break; case YieldFrom_kind: if (!symtable_raise_if_annotation_block(st, "yield expression", e)) { - VISIT_QUIT(st, 0); + return 0; } VISIT(st, expr, e->v.YieldFrom.value); st->st_cur->ste_generator = 1; @@ -2217,7 +2443,21 @@ symtable_visit_expr(struct symtable *st, expr_ty e) break; case Await_kind: if (!symtable_raise_if_annotation_block(st, "await expression", e)) { - VISIT_QUIT(st, 0); + return 0; + } + if (!allows_top_level_await(st)) { + if (!_PyST_IsFunctionLike(st->st_cur)) { + PyErr_SetString(PyExc_SyntaxError, + "'await' outside function"); + SET_ERROR_LOCATION(st->st_filename, LOCATION(e)); + return 0; + } + if (!IS_ASYNC_DEF(st) && st->st_cur->ste_comprehension == NoComprehension) { + PyErr_SetString(PyExc_SyntaxError, + "'await' outside async function"); + SET_ERROR_LOCATION(st->st_filename, LOCATION(e)); + return 0; + } } VISIT(st, expr, e->v.Await.value); st->st_cur->ste_coroutine = 1; @@ -2229,6 +2469,9 @@ symtable_visit_expr(struct symtable *st, expr_ty e) case Call_kind: VISIT(st, expr, e->v.Call.func); VISIT_SEQ(st, expr, e->v.Call.args); + if (!check_keywords(st, e->v.Call.keywords)) { + return 0; + } VISIT_SEQ_WITH_NULL(st, keyword, e->v.Call.keywords); break; case FormattedValue_kind: @@ -2244,6 +2487,9 @@ symtable_visit_expr(struct symtable *st, expr_ty e) break; /* The following exprs can be assignment targets. */ case Attribute_kind: + if (!check_name(st, e->v.Attribute.attr, LOCATION(e), e->v.Attribute.ctx)) { + return 0; + } VISIT(st, expr, e->v.Attribute.value); break; case Subscript_kind: @@ -2255,22 +2501,24 @@ symtable_visit_expr(struct symtable *st, expr_ty e) break; case Slice_kind: if (e->v.Slice.lower) - VISIT(st, expr, e->v.Slice.lower) + VISIT(st, expr, e->v.Slice.lower); if (e->v.Slice.upper) - VISIT(st, expr, e->v.Slice.upper) + VISIT(st, expr, e->v.Slice.upper); if (e->v.Slice.step) - VISIT(st, expr, e->v.Slice.step) + VISIT(st, expr, e->v.Slice.step); break; case Name_kind: - if (!symtable_add_def(st, e->v.Name.id, - e->v.Name.ctx == Load ? USE : DEF_LOCAL, LOCATION(e))) - VISIT_QUIT(st, 0); + if (!symtable_add_def_ctx(st, e->v.Name.id, + e->v.Name.ctx == Load ? USE : DEF_LOCAL, + LOCATION(e), e->v.Name.ctx)) { + return 0; + } /* Special-case super: it counts as a use of __class__ */ if (e->v.Name.ctx == Load && _PyST_IsFunctionLike(st->st_cur) && _PyUnicode_EqualToASCIIString(e->v.Name.id, "super")) { if (!symtable_add_def(st, &_Py_ID(__class__), USE, LOCATION(e))) - VISIT_QUIT(st, 0); + return 0; } break; /* child nodes of List and Tuple will have expr_context set */ @@ -2281,56 +2529,97 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT_SEQ(st, expr, e->v.Tuple.elts); break; } - VISIT_QUIT(st, 1); + LEAVE_RECURSIVE(st); + return 1; } static int -symtable_visit_type_param(struct symtable *st, type_param_ty tp) +symtable_visit_type_param_bound_or_default( + struct symtable *st, expr_ty e, identifier name, + void *key, const char *ste_scope_info) { - if (++st->recursion_depth > st->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); - VISIT_QUIT(st, 0); + if (e) { + int is_in_class = st->st_cur->ste_can_see_class_scope; + if (!symtable_enter_block(st, name, TypeVariableBlock, key, LOCATION(e))) { + return 0; + } + + st->st_cur->ste_can_see_class_scope = is_in_class; + if (is_in_class && !symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(e))) { + return 0; + } + + assert(ste_scope_info != NULL); + st->st_cur->ste_scope_info = ste_scope_info; + VISIT(st, expr, e); + + if (!symtable_exit_block(st)) { + return 0; + } } + return 1; +} + +static int +symtable_visit_type_param(struct symtable *st, type_param_ty tp) +{ + ENTER_RECURSIVE(st); switch(tp->kind) { case TypeVar_kind: if (!symtable_add_def(st, tp->v.TypeVar.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) - VISIT_QUIT(st, 0); - if (tp->v.TypeVar.bound) { - int is_in_class = st->st_cur->ste_can_see_class_scope; - if (!symtable_enter_block(st, tp->v.TypeVar.name, - TypeVarBoundBlock, (void *)tp, - LOCATION(tp))) - VISIT_QUIT(st, 0); - st->st_cur->ste_can_see_class_scope = is_in_class; - if (is_in_class && !symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(tp->v.TypeVar.bound))) { - VISIT_QUIT(st, 0); - } - VISIT(st, expr, tp->v.TypeVar.bound); - if (!symtable_exit_block(st)) - VISIT_QUIT(st, 0); + return 0; + + const char *ste_scope_info = NULL; + const expr_ty bound = tp->v.TypeVar.bound; + if (bound != NULL) { + ste_scope_info = bound->kind == Tuple_kind ? "a TypeVar constraint" : "a TypeVar bound"; + } + + // We must use a different key for the bound and default. The obvious choice would be to + // use the .bound and .default_value pointers, but that fails when the expression immediately + // inside the bound or default is a comprehension: we would reuse the same key for + // the comprehension scope. Therefore, use the address + 1 as the second key. + // The only requirement for the key is that it is unique and it matches the logic in + // compile.c where the scope is retrieved. + if (!symtable_visit_type_param_bound_or_default(st, tp->v.TypeVar.bound, tp->v.TypeVar.name, + (void *)tp, ste_scope_info)) { + return 0; + } + + if (!symtable_visit_type_param_bound_or_default(st, tp->v.TypeVar.default_value, tp->v.TypeVar.name, + (void *)((uintptr_t)tp + 1), "a TypeVar default")) { + return 0; } break; case TypeVarTuple_kind: - if (!symtable_add_def(st, tp->v.TypeVarTuple.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) - VISIT_QUIT(st, 0); + if (!symtable_add_def(st, tp->v.TypeVarTuple.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) { + return 0; + } + + if (!symtable_visit_type_param_bound_or_default(st, tp->v.TypeVarTuple.default_value, tp->v.TypeVarTuple.name, + (void *)tp, "a TypeVarTuple default")) { + return 0; + } break; case ParamSpec_kind: - if (!symtable_add_def(st, tp->v.ParamSpec.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) - VISIT_QUIT(st, 0); + if (!symtable_add_def(st, tp->v.ParamSpec.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) { + return 0; + } + + if (!symtable_visit_type_param_bound_or_default(st, tp->v.ParamSpec.default_value, tp->v.ParamSpec.name, + (void *)tp, "a ParamSpec default")) { + return 0; + } break; } - VISIT_QUIT(st, 1); + LEAVE_RECURSIVE(st); + return 1; } static int symtable_visit_pattern(struct symtable *st, pattern_ty p) { - if (++st->recursion_depth > st->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); - VISIT_QUIT(st, 0); - } + ENTER_RECURSIVE(st); switch (p->kind) { case MatchValue_kind: VISIT(st, expr, p->v.MatchValue.value); @@ -2343,19 +2632,26 @@ symtable_visit_pattern(struct symtable *st, pattern_ty p) break; case MatchStar_kind: if (p->v.MatchStar.name) { - symtable_add_def(st, p->v.MatchStar.name, DEF_LOCAL, LOCATION(p)); + if (!symtable_add_def(st, p->v.MatchStar.name, DEF_LOCAL, LOCATION(p))) { + return 0; + } } break; case MatchMapping_kind: VISIT_SEQ(st, expr, p->v.MatchMapping.keys); VISIT_SEQ(st, pattern, p->v.MatchMapping.patterns); if (p->v.MatchMapping.rest) { - symtable_add_def(st, p->v.MatchMapping.rest, DEF_LOCAL, LOCATION(p)); + if (!symtable_add_def(st, p->v.MatchMapping.rest, DEF_LOCAL, LOCATION(p))) { + return 0; + } } break; case MatchClass_kind: VISIT(st, expr, p->v.MatchClass.cls); VISIT_SEQ(st, pattern, p->v.MatchClass.patterns); + if (!check_kwd_patterns(st, p)) { + return 0; + } VISIT_SEQ(st, pattern, p->v.MatchClass.kwd_patterns); break; case MatchAs_kind: @@ -2363,14 +2659,17 @@ symtable_visit_pattern(struct symtable *st, pattern_ty p) VISIT(st, pattern, p->v.MatchAs.pattern); } if (p->v.MatchAs.name) { - symtable_add_def(st, p->v.MatchAs.name, DEF_LOCAL, LOCATION(p)); + if (!symtable_add_def(st, p->v.MatchAs.name, DEF_LOCAL, LOCATION(p))) { + return 0; + } } break; case MatchOr_kind: VISIT_SEQ(st, pattern, p->v.MatchOr.patterns); break; } - VISIT_QUIT(st, 1); + LEAVE_RECURSIVE(st); + return 1; } static int @@ -2379,7 +2678,7 @@ symtable_implicit_arg(struct symtable *st, int pos) PyObject *id = PyUnicode_FromFormat(".%d", pos); if (id == NULL) return 0; - if (!symtable_add_def(st, id, DEF_PARAM, ST_LOCATION(st->st_cur))) { + if (!symtable_add_def(st, id, DEF_PARAM, st->st_cur->ste_loc)) { Py_DECREF(id); return 0; } @@ -2390,10 +2689,7 @@ symtable_implicit_arg(struct symtable *st, int pos) static int symtable_visit_params(struct symtable *st, asdl_arg_seq *args) { - int i; - - if (!args) - return -1; + Py_ssize_t i; for (i = 0; i < asdl_seq_LEN(args); i++) { arg_ty arg = (arg_ty)asdl_seq_GET(args, i); @@ -2405,19 +2701,33 @@ symtable_visit_params(struct symtable *st, asdl_arg_seq *args) } static int -symtable_visit_annotation(struct symtable *st, expr_ty annotation) +symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key) { - int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS; - if (future_annotations && - !symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock, - (void *)annotation, annotation->lineno, - annotation->col_offset, annotation->end_lineno, - annotation->end_col_offset)) { - VISIT_QUIT(st, 0); + struct _symtable_entry *parent_ste = st->st_cur; + if (parent_ste->ste_annotation_block == NULL) { + _Py_block_ty current_type = parent_ste->ste_type; + if (!symtable_enter_block(st, &_Py_ID(__annotate__), AnnotationBlock, + key, LOCATION(annotation))) { + return 0; + } + parent_ste->ste_annotation_block = + (struct _symtable_entry *)Py_NewRef(st->st_cur); + int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS; + if (current_type == ClassBlock && !future_annotations) { + st->st_cur->ste_can_see_class_scope = 1; + if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(annotation))) { + return 0; + } + } + } + else { + if (!symtable_enter_existing_block(st, parent_ste->ste_annotation_block)) { + return 0; + } } VISIT(st, expr, annotation); - if (future_annotations && !symtable_exit_block(st)) { - VISIT_QUIT(st, 0); + if (!symtable_exit_block(st)) { + return 0; } return 1; } @@ -2425,45 +2735,55 @@ symtable_visit_annotation(struct symtable *st, expr_ty annotation) static int symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args) { - int i; - - if (!args) - return -1; + Py_ssize_t i; for (i = 0; i < asdl_seq_LEN(args); i++) { arg_ty arg = (arg_ty)asdl_seq_GET(args, i); - if (arg->annotation) + if (arg->annotation) { + st->st_cur->ste_annotations_used = 1; VISIT(st, expr, arg->annotation); + } } return 1; } static int -symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns) +symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns, + struct _symtable_entry *function_ste) { - int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS; - if (future_annotations && - !symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock, - (void *)o, o->lineno, o->col_offset, o->end_lineno, - o->end_col_offset)) { - VISIT_QUIT(st, 0); + int is_in_class = st->st_cur->ste_can_see_class_scope; + _Py_block_ty current_type = st->st_cur->ste_type; + if (!symtable_enter_block(st, &_Py_ID(__annotate__), AnnotationBlock, + (void *)a, LOCATION(o))) { + return 0; + } + if (is_in_class || current_type == ClassBlock) { + st->st_cur->ste_can_see_class_scope = 1; + if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(o))) { + return 0; + } } if (a->posonlyargs && !symtable_visit_argannotations(st, a->posonlyargs)) return 0; if (a->args && !symtable_visit_argannotations(st, a->args)) return 0; - if (a->vararg && a->vararg->annotation) + if (a->vararg && a->vararg->annotation) { + st->st_cur->ste_annotations_used = 1; VISIT(st, expr, a->vararg->annotation); - if (a->kwarg && a->kwarg->annotation) + } + if (a->kwarg && a->kwarg->annotation) { + st->st_cur->ste_annotations_used = 1; VISIT(st, expr, a->kwarg->annotation); + } if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs)) return 0; - if (future_annotations && !symtable_exit_block(st)) { - VISIT_QUIT(st, 0); + if (returns) { + st->st_cur->ste_annotations_used = 1; + VISIT(st, expr, returns); } - if (returns && !symtable_visit_annotation(st, returns)) { - VISIT_QUIT(st, 0); + if (!symtable_exit_block(st)) { + return 0; } return 1; } @@ -2553,14 +2873,8 @@ symtable_visit_alias(struct symtable *st, alias_ty a) } else { if (st->st_cur->ste_type != ModuleBlock) { - int lineno = a->lineno; - int col_offset = a->col_offset; - int end_lineno = a->end_lineno; - int end_col_offset = a->end_col_offset; PyErr_SetString(PyExc_SyntaxError, IMPORT_STAR_WARNING); - PyErr_RangedSyntaxLocationObject(st->st_filename, - lineno, col_offset + 1, - end_lineno, end_col_offset + 1); + SET_ERROR_LOCATION(st->st_filename, LOCATION(a)); Py_DECREF(store_name); return 0; } @@ -2600,18 +2914,6 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, identifier scope_name, asdl_comprehension_seq *generators, expr_ty elt, expr_ty value) { - if (st->st_cur->ste_can_see_class_scope) { - // gh-109118 - PyErr_Format(PyExc_SyntaxError, - "Cannot use comprehension in annotation scope within class scope"); - PyErr_RangedSyntaxLocationObject(st->st_filename, - e->lineno, - e->col_offset + 1, - e->end_lineno, - e->end_col_offset + 1); - VISIT_QUIT(st, 0); - } - int is_generator = (e->kind == GeneratorExp_kind); comprehension_ty outermost = ((comprehension_ty) asdl_seq_GET(generators, 0)); @@ -2621,9 +2923,7 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, st->st_cur->ste_comp_iter_expr--; /* Create comprehension scope for the rest */ if (!scope_name || - !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, - e->lineno, e->col_offset, - e->end_lineno, e->end_col_offset)) { + !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, LOCATION(e))) { return 0; } switch(e->kind) { @@ -2664,6 +2964,16 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, if (!symtable_exit_block(st)) { return 0; } + if (is_async && + !IS_ASYNC_DEF(st) && + st->st_cur->ste_comprehension == NoComprehension && + !allows_top_level_await(st)) + { + PyErr_SetString(PyExc_SyntaxError, "asynchronous comprehension outside of " + "an asynchronous function"); + SET_ERROR_LOCATION(st->st_filename, LOCATION(e)); + return 0; + } if (is_async) { st->st_cur->ste_coroutine = 1; } @@ -2706,23 +3016,28 @@ symtable_visit_dictcomp(struct symtable *st, expr_ty e) static int symtable_raise_if_annotation_block(struct symtable *st, const char *name, expr_ty e) { - enum _block_type type = st->st_cur->ste_type; + _Py_block_ty type = st->st_cur->ste_type; if (type == AnnotationBlock) PyErr_Format(PyExc_SyntaxError, ANNOTATION_NOT_ALLOWED, name); - else if (type == TypeVarBoundBlock) - PyErr_Format(PyExc_SyntaxError, TYPEVAR_BOUND_NOT_ALLOWED, name); - else if (type == TypeAliasBlock) - PyErr_Format(PyExc_SyntaxError, TYPEALIAS_NOT_ALLOWED, name); - else if (type == TypeParamBlock) - PyErr_Format(PyExc_SyntaxError, TYPEPARAM_NOT_ALLOWED, name); + else if (type == TypeVariableBlock) { + const char *info = st->st_cur->ste_scope_info; + assert(info != NULL); // e.g., info == "a ParamSpec default" + PyErr_Format(PyExc_SyntaxError, EXPR_NOT_ALLOWED_IN_TYPE_VARIABLE, name, info); + } + else if (type == TypeAliasBlock) { + // for now, we do not have any extra information + assert(st->st_cur->ste_scope_info == NULL); + PyErr_Format(PyExc_SyntaxError, EXPR_NOT_ALLOWED_IN_TYPE_ALIAS, name); + } + else if (type == TypeParametersBlock) { + // for now, we do not have any extra information + assert(st->st_cur->ste_scope_info == NULL); + PyErr_Format(PyExc_SyntaxError, EXPR_NOT_ALLOWED_IN_TYPE_PARAMETERS, name); + } else return 1; - PyErr_RangedSyntaxLocationObject(st->st_filename, - e->lineno, - e->col_offset + 1, - e->end_lineno, - e->end_col_offset + 1); + SET_ERROR_LOCATION(st->st_filename, LOCATION(e)); return 0; } @@ -2734,10 +3049,18 @@ symtable_raise_if_comprehension_block(struct symtable *st, expr_ty e) { (type == SetComprehension) ? "'yield' inside set comprehension" : (type == DictComprehension) ? "'yield' inside dict comprehension" : "'yield' inside generator expression"); - PyErr_RangedSyntaxLocationObject(st->st_filename, - e->lineno, e->col_offset + 1, - e->end_lineno, e->end_col_offset + 1); - VISIT_QUIT(st, 0); + SET_ERROR_LOCATION(st->st_filename, LOCATION(e)); + return 0; +} + +static int +symtable_raise_if_not_coroutine(struct symtable *st, const char *msg, _Py_SourceLocation loc) { + if (!st->st_cur->ste_coroutine) { + PyErr_SetString(PyExc_SyntaxError, msg); + SET_ERROR_LOCATION(st->st_filename, loc); + return 0; + } + return 1; } struct symtable * @@ -2757,7 +3080,7 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, _PyArena_Free(arena); return NULL; } - PyFutureFeatures future; + _PyFutureFeatures future; if (!_PyFuture_FromAST(mod, filename, &future)) { _PyArena_Free(arena); return NULL; @@ -2768,6 +3091,26 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, return st; } +PyObject * +_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name) +{ + /* Special case for type parameter blocks around generic classes: + * we want to mangle type parameter names (so a type param with a private + * name can be used inside the class body), but we don't want to mangle + * any other names that appear within the type parameter scope. + */ + if (ste->ste_mangled_names != NULL) { + int result = PySet_Contains(ste->ste_mangled_names, name); + if (result < 0) { + return NULL; + } + if (result == 0) { + return Py_NewRef(name); + } + } + return _Py_Mangle(privateobj, name); +} + PyObject * _Py_Mangle(PyObject *privateobj, PyObject *ident) { @@ -2802,33 +3145,30 @@ _Py_Mangle(PyObject *privateobj, PyObject *ident) if (ipriv == plen) { return Py_NewRef(ident); /* Don't mangle if class is just underscores */ } - plen -= ipriv; - if (plen + nlen >= PY_SSIZE_T_MAX - 1) { + if (nlen + (plen - ipriv) >= PY_SSIZE_T_MAX - 1) { PyErr_SetString(PyExc_OverflowError, "private identifier too large to be mangled"); return NULL; } - Py_UCS4 maxchar = PyUnicode_MAX_CHAR_VALUE(ident); - if (PyUnicode_MAX_CHAR_VALUE(privateobj) > maxchar) { - maxchar = PyUnicode_MAX_CHAR_VALUE(privateobj); - } - - PyObject *result = PyUnicode_New(1 + nlen + plen, maxchar); - if (!result) { + PyUnicodeWriter *writer = PyUnicodeWriter_Create(1 + nlen + (plen - ipriv)); + if (!writer) { return NULL; } - /* ident = "_" + priv[ipriv:] + ident # i.e. 1+plen+nlen bytes */ - PyUnicode_WRITE(PyUnicode_KIND(result), PyUnicode_DATA(result), 0, '_'); - if (PyUnicode_CopyCharacters(result, 1, privateobj, ipriv, plen) < 0) { - Py_DECREF(result); - return NULL; + // ident = "_" + priv[ipriv:] + ident + if (PyUnicodeWriter_WriteChar(writer, '_') < 0) { + goto error; } - if (PyUnicode_CopyCharacters(result, plen+1, ident, 0, nlen) < 0) { - Py_DECREF(result); - return NULL; + if (PyUnicodeWriter_WriteSubstring(writer, privateobj, ipriv, plen) < 0) { + goto error; } - assert(_PyUnicode_CheckConsistency(result, 1)); - return result; + if (PyUnicodeWriter_WriteStr(writer, ident) < 0) { + goto error; + } + return PyUnicodeWriter_Finish(writer); + +error: + PyUnicodeWriter_Discard(writer); + return NULL; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index a6e171601447c0..7e4cb45af05672 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -15,6 +15,7 @@ Data members: */ #include "Python.h" +#include "pycore_audit.h" // _Py_AuditHookEntry #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_SetAsyncGenFinalizer() #include "pycore_dict.h" // _PyDict_GetItemWithError() @@ -35,7 +36,6 @@ Data members: #include "pycore_sysmodule.h" // export _PySys_GetSizeOf() #include "pycore_tuple.h" // _PyTuple_FromArray() -#include "frameobject.h" // PyFrame_FastToLocalsWithError() #include "pydtrace.h" // PyDTrace_AUDIT() #include "osdefs.h" // DELIM #include "stdlib_module_names.h" // _Py_stdlib_module_names @@ -499,56 +499,28 @@ sys_addaudithook_impl(PyObject *module, PyObject *hook) Py_RETURN_NONE; } -PyDoc_STRVAR(audit_doc, -"audit(event, *args)\n\ -\n\ -Passes the event to any audit hooks that are attached."); +/*[clinic input] +sys.audit + + event: str + / + *args: tuple + +Passes the event to any audit hooks that are attached. +[clinic start generated code]*/ static PyObject * -sys_audit(PyObject *self, PyObject *const *args, Py_ssize_t argc) +sys_audit_impl(PyObject *module, const char *event, PyObject *args) +/*[clinic end generated code: output=1d0fc82da768f49d input=ec3b688527945109]*/ { PyThreadState *tstate = _PyThreadState_GET(); _Py_EnsureTstateNotNULL(tstate); - if (argc == 0) { - _PyErr_SetString(tstate, PyExc_TypeError, - "audit() missing 1 required positional argument: " - "'event'"); - return NULL; - } - - assert(args[0] != NULL); - assert(PyUnicode_Check(args[0])); - if (!should_audit(tstate->interp)) { Py_RETURN_NONE; } - PyObject *auditEvent = args[0]; - if (!auditEvent) { - _PyErr_SetString(tstate, PyExc_TypeError, - "expected str for argument 'event'"); - return NULL; - } - if (!PyUnicode_Check(auditEvent)) { - _PyErr_Format(tstate, PyExc_TypeError, - "expected str for argument 'event', not %.200s", - Py_TYPE(auditEvent)->tp_name); - return NULL; - } - const char *event = PyUnicode_AsUTF8(auditEvent); - if (!event) { - return NULL; - } - - PyObject *auditArgs = _PyTuple_FromArray(args + 1, argc - 1); - if (!auditArgs) { - return NULL; - } - - int res = _PySys_Audit(tstate, event, "O", auditArgs); - Py_DECREF(auditArgs); - + int res = _PySys_Audit(tstate, event, "O", args); if (res < 0) { return NULL; } @@ -644,7 +616,8 @@ sys_breakpointhook(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyOb } PyDoc_STRVAR(breakpointhook_doc, -"breakpointhook(*args, **kws)\n" +"breakpointhook($module, /, *args, **kwargs)\n" +"--\n" "\n" "This hook function is called by built-in breakpoint().\n" ); @@ -746,7 +719,7 @@ sys_displayhook(PyObject *module, PyObject *o) if (o == Py_None) { Py_RETURN_NONE; } - if (PyObject_SetAttr(builtins, &_Py_ID(_), Py_None) != 0) + if (PyObject_SetAttr(builtins, _Py_LATIN1_CHR('_'), Py_None) != 0) return NULL; outf = _PySys_GetAttr(tstate, &_Py_ID(stdout)); if (outf == NULL || outf == Py_None) { @@ -768,10 +741,9 @@ sys_displayhook(PyObject *module, PyObject *o) return NULL; } } - _Py_DECLARE_STR(newline, "\n"); - if (PyFile_WriteObject(&_Py_STR(newline), outf, Py_PRINT_RAW) != 0) + if (PyFile_WriteObject(_Py_LATIN1_CHR('\n'), outf, Py_PRINT_RAW) != 0) return NULL; - if (PyObject_SetAttr(builtins, &_Py_ID(_), o) != 0) + if (PyObject_SetAttr(builtins, _Py_LATIN1_CHR('_'), o) != 0) return NULL; Py_RETURN_NONE; } @@ -929,7 +901,7 @@ sys_getfilesystemencoding_impl(PyObject *module) if (u == NULL) { return NULL; } - _PyUnicode_InternInPlace(interp, &u); + _PyUnicode_InternImmortal(interp, &u); return u; } @@ -949,7 +921,7 @@ sys_getfilesystemencodeerrors_impl(PyObject *module) if (u == NULL) { return NULL; } - _PyUnicode_InternInPlace(interp, &u); + _PyUnicode_InternImmortal(interp, &u); return u; } @@ -971,8 +943,9 @@ sys_intern_impl(PyObject *module, PyObject *s) /*[clinic end generated code: output=be680c24f5c9e5d6 input=849483c006924e2f]*/ { if (PyUnicode_CheckExact(s)) { + PyInterpreterState *interp = _PyInterpreterState_GET(); Py_INCREF(s); - PyUnicode_InternInPlace(&s); + _PyUnicode_InternMortal(interp, &s); return s; } else { @@ -1020,13 +993,6 @@ static PyObject * call_trampoline(PyThreadState *tstate, PyObject* callback, PyFrameObject *frame, int what, PyObject *arg) { - /* Discard any previous modifications the frame's fast locals */ - if (frame->f_fast_as_locals) { - if (PyFrame_FastToLocalsWithError(frame) < 0) { - return NULL; - } - } - /* call the Python-level function */ if (arg == NULL) { arg = Py_None; @@ -1034,7 +1000,6 @@ call_trampoline(PyThreadState *tstate, PyObject* callback, PyObject *args[3] = {(PyObject *)frame, whatstrings[what], arg}; PyObject *result = _PyObject_VectorcallTstate(tstate, callback, args, 3, NULL); - PyFrame_LocalsToFast(frame, 1); return result; } @@ -1085,34 +1050,40 @@ trace_trampoline(PyObject *self, PyFrameObject *frame, return 0; } +/*[clinic input] +sys.settrace + + function: object + / + +Set the global debug tracing function. + +It will be called on each function call. See the debugger chapter +in the library manual. +[clinic start generated code]*/ + static PyObject * -sys_settrace(PyObject *self, PyObject *args) +sys_settrace(PyObject *module, PyObject *function) +/*[clinic end generated code: output=999d12e9d6ec4678 input=8107feb01c5f1c4e]*/ { PyThreadState *tstate = _PyThreadState_GET(); - if (args == Py_None) { + if (function == Py_None) { if (_PyEval_SetTrace(tstate, NULL, NULL) < 0) { return NULL; } } else { - if (_PyEval_SetTrace(tstate, trace_trampoline, args) < 0) { + if (_PyEval_SetTrace(tstate, trace_trampoline, function) < 0) { return NULL; } } Py_RETURN_NONE; } -PyDoc_STRVAR(settrace_doc, -"settrace(function)\n\ -\n\ -Set the global debug tracing function. It will be called on each\n\ -function call. See the debugger chapter in the library manual." -); - /*[clinic input] sys._settraceallthreads - arg: object + function as arg: object / Set the global debug tracing function in all running threads belonging to the current interpreter. @@ -1123,7 +1094,7 @@ in the library manual. static PyObject * sys__settraceallthreads(PyObject *module, PyObject *arg) -/*[clinic end generated code: output=161cca30207bf3ca input=5906aa1485a50289]*/ +/*[clinic end generated code: output=161cca30207bf3ca input=d4bde1f810d73675]*/ { PyObject* argument = NULL; Py_tracefunc func = NULL; @@ -1159,45 +1130,51 @@ sys_gettrace_impl(PyObject *module) return Py_NewRef(temp); } +/*[clinic input] +sys.setprofile + + function: object + / + +Set the profiling function. + +It will be called on each function call and return. See the profiler +chapter in the library manual. +[clinic start generated code]*/ + static PyObject * -sys_setprofile(PyObject *self, PyObject *args) +sys_setprofile(PyObject *module, PyObject *function) +/*[clinic end generated code: output=1c3503105939db9c input=055d0d7961413a62]*/ { PyThreadState *tstate = _PyThreadState_GET(); - if (args == Py_None) { + if (function == Py_None) { if (_PyEval_SetProfile(tstate, NULL, NULL) < 0) { return NULL; } } else { - if (_PyEval_SetProfile(tstate, profile_trampoline, args) < 0) { + if (_PyEval_SetProfile(tstate, profile_trampoline, function) < 0) { return NULL; } } Py_RETURN_NONE; } -PyDoc_STRVAR(setprofile_doc, -"setprofile(function)\n\ -\n\ -Set the profiling function. It will be called on each function call\n\ -and return. See the profiler chapter in the library manual." -); - /*[clinic input] sys._setprofileallthreads - arg: object + function as arg: object / Set the profiling function in all running threads belonging to the current interpreter. -It will be called on each function call and return. See the profiler chapter -in the library manual. +It will be called on each function call and return. See the profiler +chapter in the library manual. [clinic start generated code]*/ static PyObject * sys__setprofileallthreads(PyObject *module, PyObject *arg) -/*[clinic end generated code: output=2d61319e27b309fe input=d1a356d3f4f9060a]*/ +/*[clinic end generated code: output=2d61319e27b309fe input=a10589439ba20cee]*/ { PyObject* argument = NULL; Py_tracefunc func = NULL; @@ -1393,12 +1370,6 @@ sys_set_asyncgen_hooks(PyObject *self, PyObject *args, PyObject *kw) Py_TYPE(finalizer)->tp_name); return NULL; } - if (_PyEval_SetAsyncGenFinalizer(finalizer) < 0) { - return NULL; - } - } - else if (finalizer == Py_None && _PyEval_SetAsyncGenFinalizer(NULL) < 0) { - return NULL; } if (firstiter && firstiter != Py_None) { @@ -1408,15 +1379,33 @@ sys_set_asyncgen_hooks(PyObject *self, PyObject *args, PyObject *kw) Py_TYPE(firstiter)->tp_name); return NULL; } - if (_PyEval_SetAsyncGenFirstiter(firstiter) < 0) { + } + + PyObject *cur_finalizer = _PyEval_GetAsyncGenFinalizer(); + + if (finalizer && finalizer != Py_None) { + if (_PyEval_SetAsyncGenFinalizer(finalizer) < 0) { return NULL; } } - else if (firstiter == Py_None && _PyEval_SetAsyncGenFirstiter(NULL) < 0) { + else if (finalizer == Py_None && _PyEval_SetAsyncGenFinalizer(NULL) < 0) { return NULL; } + if (firstiter && firstiter != Py_None) { + if (_PyEval_SetAsyncGenFirstiter(firstiter) < 0) { + goto error; + } + } + else if (firstiter == Py_None && _PyEval_SetAsyncGenFirstiter(NULL) < 0) { + goto error; + } + Py_RETURN_NONE; + +error: + _PyEval_SetAsyncGenFinalizer(cur_finalizer); + return NULL; } PyDoc_STRVAR(set_asyncgen_hooks_doc, @@ -1643,12 +1632,13 @@ sys_getwindowsversion_impl(PyObject *module) int pos = 0; OSVERSIONINFOEXW ver; - version = PyObject_GetAttrString(module, "_cached_windows_version"); + if (PyObject_GetOptionalAttrString(module, "_cached_windows_version", &version) < 0) { + return NULL; + }; if (version && PyObject_TypeCheck(version, &WindowsVersionType)) { return version; } Py_XDECREF(version); - PyErr_Clear(); ver.dwOSVersionInfoSize = sizeof(ver); if (!GetVersionExW((OSVERSIONINFOW*) &ver)) @@ -1658,15 +1648,24 @@ sys_getwindowsversion_impl(PyObject *module) if (version == NULL) return NULL; - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.dwMajorVersion)); - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.dwMinorVersion)); - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.dwBuildNumber)); - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.dwPlatformId)); - PyStructSequence_SET_ITEM(version, pos++, PyUnicode_FromWideChar(ver.szCSDVersion, -1)); - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.wServicePackMajor)); - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.wServicePackMinor)); - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.wSuiteMask)); - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.wProductType)); +#define SET_VERSION_INFO(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + goto error; \ + } \ + PyStructSequence_SET_ITEM(version, pos++, item); \ + } while(0) + + SET_VERSION_INFO(PyLong_FromLong(ver.dwMajorVersion)); + SET_VERSION_INFO(PyLong_FromLong(ver.dwMinorVersion)); + SET_VERSION_INFO(PyLong_FromLong(ver.dwBuildNumber)); + SET_VERSION_INFO(PyLong_FromLong(ver.dwPlatformId)); + SET_VERSION_INFO(PyUnicode_FromWideChar(ver.szCSDVersion, -1)); + SET_VERSION_INFO(PyLong_FromLong(ver.wServicePackMajor)); + SET_VERSION_INFO(PyLong_FromLong(ver.wServicePackMinor)); + SET_VERSION_INFO(PyLong_FromLong(ver.wSuiteMask)); + SET_VERSION_INFO(PyLong_FromLong(ver.wProductType)); // GetVersion will lie if we are running in a compatibility mode. // We need to read the version info from a system file resource @@ -1674,6 +1673,10 @@ sys_getwindowsversion_impl(PyObject *module) // just return whatever GetVersion said. PyObject *realVersion = _sys_getwindowsversion_from_kernel32(); if (!realVersion) { + if (!PyErr_ExceptionMatches(PyExc_WindowsError)) { + return NULL; + } + PyErr_Clear(); realVersion = Py_BuildValue("(kkk)", ver.dwMajorVersion, @@ -1682,21 +1685,19 @@ sys_getwindowsversion_impl(PyObject *module) ); } - if (realVersion) { - PyStructSequence_SET_ITEM(version, pos++, realVersion); - } + SET_VERSION_INFO(realVersion); - if (PyErr_Occurred()) { - Py_DECREF(version); - return NULL; - } +#undef SET_VERSION_INFO if (PyObject_SetAttrString(module, "_cached_windows_version", version) < 0) { - Py_DECREF(version); - return NULL; + goto error; } return version; + +error: + Py_DECREF(version); + return NULL; } #pragma warning(pop) @@ -1816,6 +1817,7 @@ sys_get_int_max_str_digits_impl(PyObject *module) return PyLong_FromLong(interp->long_state.max_str_digits); } + /*[clinic input] sys.set_int_max_str_digits @@ -1828,16 +1830,10 @@ static PyObject * sys_set_int_max_str_digits_impl(PyObject *module, int maxdigits) /*[clinic end generated code: output=734d4c2511f2a56d input=d7e3f325db6910c5]*/ { - PyThreadState *tstate = _PyThreadState_GET(); - if ((!maxdigits) || (maxdigits >= _PY_LONG_MAX_STR_DIGITS_THRESHOLD)) { - tstate->interp->long_state.max_str_digits = maxdigits; - Py_RETURN_NONE; - } else { - PyErr_Format( - PyExc_ValueError, "maxdigits must be 0 or larger than %d", - _PY_LONG_MAX_STR_DIGITS_THRESHOLD); + if (_PySys_SetIntMaxStrDigits(maxdigits) < 0) { return NULL; } + Py_RETURN_NONE; } size_t @@ -1978,14 +1974,22 @@ sys_getallocatedblocks_impl(PyObject *module) /*[clinic input] sys.getunicodeinternedsize -> Py_ssize_t + * + _only_immortal: bool = False + Return the number of elements of the unicode interned dictionary [clinic start generated code]*/ static Py_ssize_t -sys_getunicodeinternedsize_impl(PyObject *module) -/*[clinic end generated code: output=ad0e4c9738ed4129 input=726298eaa063347a]*/ +sys_getunicodeinternedsize_impl(PyObject *module, int _only_immortal) +/*[clinic end generated code: output=29a6377a94a14f70 input=0330b3408dd5bcc6]*/ { - return _PyUnicode_InternedSize(); + if (_only_immortal) { + return _PyUnicode_InternedSize_Immortal(); + } + else { + return _PyUnicode_InternedSize(); + } } /*[clinic input] @@ -2139,8 +2143,15 @@ static PyObject * sys__clear_internal_caches_impl(PyObject *module) /*[clinic end generated code: output=0ee128670a4966d6 input=253e741ca744f6e8]*/ { +#ifdef _Py_TIER2 PyInterpreterState *interp = _PyInterpreterState_GET(); _Py_Executors_InvalidateAll(interp, 0); +#endif +#ifdef Py_GIL_DISABLED + if (_Py_ClearUnusedTLBC(_PyInterpreterState_GET()) < 0) { + return NULL; + } +#endif PyType_ClearCache(); Py_RETURN_NONE; } @@ -2253,6 +2264,13 @@ sys_activate_stack_trampoline_impl(PyObject *module, const char *backend) /*[clinic end generated code: output=5783cdeb51874b43 input=a12df928758a82b4]*/ { #ifdef PY_HAVE_PERF_TRAMPOLINE +#ifdef _Py_JIT + if (_PyInterpreterState_GET()->jit) { + PyErr_SetString(PyExc_ValueError, "Cannot activate the perf trampoline if the JIT is active"); + return NULL; + } +#endif + if (strcmp(backend, "perf") == 0) { _PyPerf_Callbacks cur_cb; _PyPerfTrampoline_GetCallbacks(&cur_cb); @@ -2262,6 +2280,16 @@ sys_activate_stack_trampoline_impl(PyObject *module, const char *backend) return NULL; } } + else if (strcmp(backend, "perf_jit") == 0) { + _PyPerf_Callbacks cur_cb; + _PyPerfTrampoline_GetCallbacks(&cur_cb); + if (cur_cb.write_state != _Py_perfmap_jit_callbacks.write_state) { + if (_PyPerfTrampoline_SetCallbacks(&_Py_perfmap_jit_callbacks) < 0 ) { + PyErr_SetString(PyExc_ValueError, "can't activate perf jit trampoline"); + return NULL; + } + } + } } else { PyErr_Format(PyExc_ValueError, "invalid backend: %s", backend); @@ -2314,6 +2342,30 @@ sys_is_stack_trampoline_active_impl(PyObject *module) Py_RETURN_FALSE; } +/*[clinic input] +sys._dump_tracelets + + outpath: object + +Dump the graph of tracelets in graphviz format +[clinic start generated code]*/ + +static PyObject * +sys__dump_tracelets_impl(PyObject *module, PyObject *outpath) +/*[clinic end generated code: output=a7fe265e2bc3b674 input=5bff6880cd28ffd1]*/ +{ + FILE *out = Py_fopen(outpath, "wb"); + if (out == NULL) { + return NULL; + } + int err = _PyDumpExecutors(out); + fclose(out); + if (err) { + return NULL; + } + Py_RETURN_NONE; +} + /*[clinic input] sys._getframemodulename @@ -2340,10 +2392,11 @@ sys__getframemodulename_impl(PyObject *module, int depth) while (f && (_PyFrame_IsIncomplete(f) || depth-- > 0)) { f = f->previous; } - if (f == NULL || f->f_funcobj == NULL) { + if (f == NULL || PyStackRef_IsNull(f->f_funcobj)) { Py_RETURN_NONE; } - PyObject *r = PyFunction_GetModule(f->f_funcobj); + PyObject *func = PyStackRef_AsPyObjectBorrow(f->f_funcobj); + PyObject *r = PyFunction_GetModule(func); if (!r) { PyErr_Clear(); r = Py_None; @@ -2365,6 +2418,39 @@ sys__get_cpu_count_config_impl(PyObject *module) return config->cpu_count; } +/*[clinic input] +sys._baserepl + +Private function for getting the base REPL +[clinic start generated code]*/ + +static PyObject * +sys__baserepl_impl(PyObject *module) +/*[clinic end generated code: output=f19a36375ebe0a45 input=ade0ebb9fab56f3c]*/ +{ + PyCompilerFlags cf = _PyCompilerFlags_INIT; + PyRun_AnyFileExFlags(stdin, "", 0, &cf); + Py_RETURN_NONE; +} + +/*[clinic input] +sys._is_gil_enabled -> bool + +Return True if the GIL is currently enabled and False otherwise. +[clinic start generated code]*/ + +static int +sys__is_gil_enabled_impl(PyObject *module) +/*[clinic end generated code: output=57732cf53f5b9120 input=7e9c47f15a00e809]*/ +{ +#ifdef Py_GIL_DISABLED + return _PyEval_IsGILEnabled(_PyThreadState_GET()); +#else + return 1; +#endif +} + + static PerfMapState perf_map_state; PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void) { @@ -2435,16 +2521,16 @@ PyAPI_FUNC(void) PyUnstable_PerfMapState_Fini(void) { PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename) { #ifndef MS_WINDOWS - FILE* from = fopen(parent_filename, "r"); - if (!from) { - return -1; - } if (perf_map_state.perf_map == NULL) { int ret = PyUnstable_PerfMapState_Init(); if (ret != 0) { return ret; } } + FILE* from = fopen(parent_filename, "r"); + if (!from) { + return -1; + } char buf[4096]; PyThread_acquire_lock(perf_map_state.map_lock, 1); int fflush_result = 0, result = 0; @@ -2468,15 +2554,11 @@ PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename) { return 0; } -#ifdef __cplusplus -} -#endif - static PyMethodDef sys_methods[] = { /* Might as well keep this in alphabetic order */ SYS_ADDAUDITHOOK_METHODDEF - {"audit", _PyCFunction_CAST(sys_audit), METH_FASTCALL, audit_doc }, + SYS_AUDIT_METHODDEF {"breakpointhook", _PyCFunction_CAST(sys_breakpointhook), METH_FASTCALL | METH_KEYWORDS, breakpointhook_doc}, SYS__CLEAR_INTERNAL_CACHES_METHODDEF @@ -2513,11 +2595,11 @@ static PyMethodDef sys_methods[] = { SYS_SETSWITCHINTERVAL_METHODDEF SYS_GETSWITCHINTERVAL_METHODDEF SYS_SETDLOPENFLAGS_METHODDEF - {"setprofile", sys_setprofile, METH_O, setprofile_doc}, + SYS_SETPROFILE_METHODDEF SYS__SETPROFILEALLTHREADS_METHODDEF SYS_GETPROFILE_METHODDEF SYS_SETRECURSIONLIMIT_METHODDEF - {"settrace", sys_settrace, METH_O, settrace_doc}, + SYS_SETTRACE_METHODDEF SYS__SETTRACEALLTHREADS_METHODDEF SYS_GETTRACE_METHODDEF SYS_CALL_TRACING_METHODDEF @@ -2534,6 +2616,7 @@ static PyMethodDef sys_methods[] = { SYS_UNRAISABLEHOOK_METHODDEF SYS_GET_INT_MAX_STR_DIGITS_METHODDEF SYS_SET_INT_MAX_STR_DIGITS_METHODDEF + SYS__BASEREPL_METHODDEF #ifdef Py_STATS SYS__STATS_ON_METHODDEF SYS__STATS_OFF_METHODDEF @@ -2541,6 +2624,8 @@ static PyMethodDef sys_methods[] = { SYS__STATS_DUMP_METHODDEF #endif SYS__GET_CPU_COUNT_CONFIG_METHODDEF + SYS__IS_GIL_ENABLED_METHODDEF + SYS__DUMP_TRACELETS_METHODDEF {NULL, NULL} // sentinel }; @@ -3036,9 +3121,12 @@ static PyStructSequence_Field flags_fields[] = { {"warn_default_encoding", "-X warn_default_encoding"}, {"safe_path", "-P"}, {"int_max_str_digits", "-X int_max_str_digits"}, + {"gil", "-X gil"}, {0} }; +#define SYS_FLAGS_INT_MAX_STR_DIGITS 17 + static PyStructSequence_Desc flags_desc = { "sys.flags", /* name */ flags__doc__, /* doc */ @@ -3046,6 +3134,48 @@ static PyStructSequence_Desc flags_desc = { 18 }; +static void +sys_set_flag(PyObject *flags, Py_ssize_t pos, PyObject *value) +{ + assert(pos >= 0 && pos < (Py_ssize_t)(Py_ARRAY_LENGTH(flags_fields) - 1)); + + PyObject *old_value = PyStructSequence_GET_ITEM(flags, pos); + PyStructSequence_SET_ITEM(flags, pos, Py_NewRef(value)); + Py_XDECREF(old_value); +} + + +int +_PySys_SetFlagObj(Py_ssize_t pos, PyObject *value) +{ + PyObject *flags = Py_XNewRef(PySys_GetObject("flags")); + if (flags == NULL) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, "lost sys.flags"); + } + return -1; + } + + sys_set_flag(flags, pos, value); + Py_DECREF(flags); + return 0; +} + + +static int +_PySys_SetFlagInt(Py_ssize_t pos, int value) +{ + PyObject *obj = PyLong_FromLong(value); + if (obj == NULL) { + return -1; + } + + int res = _PySys_SetFlagObj(pos, obj); + Py_DECREF(obj); + return res; +} + + static int set_flags_from_config(PyInterpreterState *interp, PyObject *flags) { @@ -3061,8 +3191,8 @@ set_flags_from_config(PyInterpreterState *interp, PyObject *flags) if (value == NULL) { \ return -1; \ } \ - Py_XDECREF(PyStructSequence_GET_ITEM(flags, pos)); \ - PyStructSequence_SET_ITEM(flags, pos, value); \ + sys_set_flag(flags, pos, value); \ + Py_DECREF(value); \ pos++; \ } while (0) #define SetFlag(expr) SetFlagObj(PyLong_FromLong(expr)) @@ -3085,6 +3215,16 @@ set_flags_from_config(PyInterpreterState *interp, PyObject *flags) SetFlag(config->warn_default_encoding); SetFlagObj(PyBool_FromLong(config->safe_path)); SetFlag(config->int_max_str_digits); +#ifdef Py_GIL_DISABLED + if (config->enable_gil == _PyConfig_GIL_DEFAULT) { + SetFlagObj(Py_NewRef(Py_None)); + } + else { + SetFlag(config->enable_gil); + } +#else + SetFlagObj(PyLong_FromLong(1)); +#endif #undef SetFlagObj #undef SetFlag return 0; @@ -3508,64 +3648,6 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) return _PyStatus_ERR("can't initialize sys module"); } -static int -sys_add_xoption(PyObject *opts, const wchar_t *s) -{ - PyObject *name, *value = NULL; - - const wchar_t *name_end = wcschr(s, L'='); - if (!name_end) { - name = PyUnicode_FromWideChar(s, -1); - if (name == NULL) { - goto error; - } - value = Py_NewRef(Py_True); - } - else { - name = PyUnicode_FromWideChar(s, name_end - s); - if (name == NULL) { - goto error; - } - value = PyUnicode_FromWideChar(name_end + 1, -1); - if (value == NULL) { - goto error; - } - } - if (PyDict_SetItem(opts, name, value) < 0) { - goto error; - } - Py_DECREF(name); - Py_DECREF(value); - return 0; - -error: - Py_XDECREF(name); - Py_XDECREF(value); - return -1; -} - - -static PyObject* -sys_create_xoptions_dict(const PyConfig *config) -{ - Py_ssize_t nxoption = config->xoptions.length; - wchar_t * const * xoptions = config->xoptions.items; - PyObject *dict = PyDict_New(); - if (dict == NULL) { - return NULL; - } - - for (Py_ssize_t i=0; i < nxoption; i++) { - const wchar_t *option = xoptions[i]; - if (sys_add_xoption(dict, option) < 0) { - Py_DECREF(dict); - return NULL; - } - } - - return dict; -} - // Update sys attributes for a new PyConfig configuration. // This function also adds attributes that _PySys_InitCore() didn't add. @@ -3612,7 +3694,7 @@ _PySys_UpdateConfig(PyThreadState *tstate) COPY_LIST("orig_argv", config->orig_argv); COPY_LIST("warnoptions", config->warnoptions); - SET_SYS("_xoptions", sys_create_xoptions_dict(config)); + SET_SYS("_xoptions", _PyConfig_CreateXOptionsDict(config)); const wchar_t *stdlibdir = _Py_GetStdlibDir(); if (stdlibdir != NULL) { @@ -3701,6 +3783,9 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) if (sysmod == NULL) { return _PyStatus_ERR("failed to create a module object"); } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(sysmod, Py_MOD_GIL_NOT_USED); +#endif PyObject *sysdict = PyModule_GetDict(sysmod); if (sysdict == NULL) { @@ -3727,7 +3812,7 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) return status; } - if (_PyImport_FixupBuiltin(sysmod, "sys", modules) < 0) { + if (_PyImport_FixupBuiltin(tstate, sysmod, "sys", modules) < 0) { goto error; } @@ -3835,8 +3920,7 @@ make_sys_argv(int argc, wchar_t * const * argv) return list; } -// Removed in Python 3.13 API, but kept for the stable ABI -PyAPI_FUNC(void) +void PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) { wchar_t* empty_argv[1] = {L""}; @@ -3880,8 +3964,7 @@ PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) } } -// Removed in Python 3.13 API, but kept for the stable ABI -PyAPI_FUNC(void) +void PySys_SetArgv(int argc, wchar_t **argv) { _Py_COMP_DIAG_PUSH @@ -4037,3 +4120,28 @@ PySys_FormatStderr(const char *format, ...) sys_format(&_Py_ID(stderr), stderr, format, va); va_end(va); } + + +int +_PySys_SetIntMaxStrDigits(int maxdigits) +{ + if (maxdigits != 0 && maxdigits < _PY_LONG_MAX_STR_DIGITS_THRESHOLD) { + PyErr_Format( + PyExc_ValueError, "maxdigits must be >= %d or 0 for unlimited", + _PY_LONG_MAX_STR_DIGITS_THRESHOLD); + return -1; + } + + // Set sys.flags.int_max_str_digits + const Py_ssize_t pos = SYS_FLAGS_INT_MAX_STR_DIGITS; + if (_PySys_SetFlagInt(pos, maxdigits) < 0) { + return -1; + } + + // Set PyInterpreterState.long_state.max_str_digits + // and PyInterpreterState.config.int_max_str_digits. + PyInterpreterState *interp = _PyInterpreterState_GET(); + interp->long_state.max_str_digits = maxdigits; + interp->config.int_max_str_digits = maxdigits; + return 0; +} diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 9dca833ff203ca..3a01a7fe327c8f 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -77,17 +77,18 @@ EnterNonRecursiveMutex(PNRMUTEX mutex, DWORD milliseconds) } } else if (milliseconds != 0) { /* wait at least until the deadline */ - PyTime_t nanoseconds = (PyTime_t)milliseconds * (1000 * 1000); - PyTime_t deadline = _PyTime_Add(_PyTime_PerfCounterUnchecked(), nanoseconds); + PyTime_t timeout = (PyTime_t)milliseconds * (1000 * 1000); + PyTime_t deadline = _PyDeadline_Init(timeout); while (mutex->locked) { - PyTime_t microseconds = _PyTime_AsMicroseconds(nanoseconds, - _PyTime_ROUND_TIMEOUT); + PyTime_t microseconds = _PyTime_AsMicroseconds(timeout, + _PyTime_ROUND_TIMEOUT); if (PyCOND_TIMEDWAIT(&mutex->cv, &mutex->cs, microseconds) < 0) { result = WAIT_FAILED; break; } - nanoseconds = deadline - _PyTime_PerfCounterUnchecked(); - if (nanoseconds <= 0) { + + timeout = _PyDeadline_Get(deadline); + if (timeout <= 0) { break; } } @@ -290,6 +291,14 @@ PyThread_exit_thread(void) _endthreadex(0); } +void _Py_NO_RETURN +PyThread_hang_thread(void) +{ + while (1) { + SleepEx(INFINITE, TRUE); + } +} + /* * Lock support. It has to be implemented as semaphores. * I [Dag] tried to implement it with mutex but I could find a way to diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 64cc60053e6cf7..c010b3a827757f 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -16,6 +16,7 @@ #undef destructor #endif #include +#include /* pause(), also getthrid() on OpenBSD */ #if defined(__linux__) # include /* syscall(SYS_gettid) */ @@ -23,8 +24,6 @@ # include /* pthread_getthreadid_np() */ #elif defined(__FreeBSD_kernel__) # include /* syscall(SYS_thr_self) */ -#elif defined(__OpenBSD__) -# include /* getthrid() */ #elif defined(_AIX) # include /* thread_self() */ #elif defined(__NetBSD__) @@ -95,6 +94,10 @@ #endif #endif +/* Thread sanitizer doesn't currently support sem_clockwait */ +#ifdef _Py_THREAD_SANITIZER +#undef HAVE_SEM_CLOCKWAIT +#endif /* Whether or not to use semaphores directly rather than emulating them with * mutexes and condition variables: @@ -154,12 +157,14 @@ _PyThread_cond_after(long long us, struct timespec *abs) PyTime_t t; #ifdef CONDATTR_MONOTONIC if (condattr_monotonic) { - t = _PyTime_MonotonicUnchecked(); + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&t); } else #endif { - t = _PyTime_TimeUnchecked(); + // silently ignore error: cannot report error to the caller + (void)PyTime_TimeRaw(&t); } t = _PyTime_Add(t, timeout); _PyTime_AsTimespec_clamp(t, abs); @@ -413,6 +418,18 @@ PyThread_exit_thread(void) #endif } +void _Py_NO_RETURN +PyThread_hang_thread(void) +{ + while (1) { +#if defined(__wasi__) + sleep(9999999); // WASI doesn't have pause() ?! +#else + pause(); +#endif + } +} + #ifdef USE_SEMAPHORES /* @@ -502,7 +519,10 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, struct timespec abs_timeout; // Local scope for deadline { - PyTime_t deadline = _PyTime_Add(_PyTime_MonotonicUnchecked(), timeout); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); + PyTime_t deadline = _PyTime_Add(now, timeout); _PyTime_AsTimespec_clamp(deadline, &abs_timeout); } #else @@ -518,8 +538,11 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, status = fix_status(sem_clockwait(thelock, CLOCK_MONOTONIC, &abs_timeout)); #else - PyTime_t abs_time = _PyTime_Add(_PyTime_TimeUnchecked(), - timeout); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_TimeRaw(&now); + PyTime_t abs_time = _PyTime_Add(now, timeout); + struct timespec ts; _PyTime_AsTimespec_clamp(abs_time, &ts); status = fix_status(sem_timedwait(thelock, &ts)); diff --git a/Python/thread_pthread_stubs.h b/Python/thread_pthread_stubs.h index 4741e594e52e65..458f8fc5951720 100644 --- a/Python/thread_pthread_stubs.h +++ b/Python/thread_pthread_stubs.h @@ -1,5 +1,9 @@ #include "cpython/pthread_stubs.h" +typedef struct py_stub_tls_entry py_tls_entry; + +#define py_tls_entries (_PyRuntime.threads.stubs.tls_entries) + // mutex int pthread_mutex_init(pthread_mutex_t *restrict mutex, @@ -105,7 +109,7 @@ pthread_join(pthread_t thread, void** value_ptr) PyAPI_FUNC(pthread_t) pthread_self(void) { - return 0; + return (pthread_t)(uintptr_t)&py_tls_entries; } int @@ -134,10 +138,6 @@ pthread_attr_destroy(pthread_attr_t *attr) } -typedef struct py_stub_tls_entry py_tls_entry; - -#define py_tls_entries (_PyRuntime.threads.stubs.tls_entries) - int pthread_key_create(pthread_key_t *key, void (*destr_function)(void *)) { diff --git a/Python/tier2_engine.md b/Python/tier2_engine.md index df9f6c124509bd..5ceda8e806045d 100644 --- a/Python/tier2_engine.md +++ b/Python/tier2_engine.md @@ -52,7 +52,7 @@ The JIT compiler converts superblocks into machine code executors. These have identical behavior to interpreted executors, except that they consume more memory for the generated machine code and are a lot faster. -## Transfering control +## Transferring control There are three types of control transfer that we need to consider: * Tier 1 to tier 2 @@ -104,7 +104,7 @@ reference to the previous executor in the thread state's #### The interpreter The tier 2 interpreter has a variable `current_executor` which -points to the currently live executor. When transfering from executor +points to the currently live executor. When transferring from executor `A` to executor `B` we do the following: (Initially `current_executor` points to `A`, and the refcount of `A` is elevated by one) @@ -122,7 +122,7 @@ increment the refcount of `B` and set `current_executor` to point to `B`. #### In the JIT -Transfering control from one executor to another is done via tailcalls. +Transferring control from one executor to another is done via tailcalls. The compiled executor should do the same, except that there is no local variable `current_executor`. diff --git a/Python/traceback.c b/Python/traceback.c index 7a188e56c939c0..870ae5bcefe9eb 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -5,7 +5,6 @@ #include "pycore_ast.h" // asdl_seq_GET() #include "pycore_call.h" // _PyObject_CallMethodFormat() -#include "pycore_compile.h" // _PyAST_Optimize() #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_frame.h" // _PyFrame_GetCode() #include "pycore_interp.h" // PyInterpreterState.gc @@ -35,9 +34,11 @@ extern char* _PyTokenizer_FindEncodingFilename(int, PyObject *); /*[clinic input] -class TracebackType "PyTracebackObject *" "&PyTraceback_Type" +class traceback "PyTracebackObject *" "&PyTraceback_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=928fa06c10151120]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=cf96294b2bebc811]*/ + +#define _PyTracebackObject_CAST(op) ((PyTracebackObject *)(op)) #include "clinic/traceback.c.h" @@ -64,7 +65,7 @@ tb_create_raw(PyTracebackObject *next, PyFrameObject *frame, int lasti, /*[clinic input] @classmethod -TracebackType.__new__ as tb_new +traceback.__new__ as tb_new tb_next: object tb_frame: object(type='PyFrameObject *', subclass_of='&PyFrame_Type') @@ -77,7 +78,7 @@ Create a new traceback object. static PyObject * tb_new_impl(PyTypeObject *type, PyObject *tb_next, PyFrameObject *tb_frame, int tb_lasti, int tb_lineno) -/*[clinic end generated code: output=fa077debd72d861a input=01cbe8ec8783fca7]*/ +/*[clinic end generated code: output=fa077debd72d861a input=b88143145454cb59]*/ { if (tb_next == Py_None) { tb_next = NULL; @@ -92,15 +93,16 @@ tb_new_impl(PyTypeObject *type, PyObject *tb_next, PyFrameObject *tb_frame, } static PyObject * -tb_dir(PyTracebackObject *self, PyObject *Py_UNUSED(ignored)) +tb_dir(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) { return Py_BuildValue("[ssss]", "tb_frame", "tb_next", "tb_lasti", "tb_lineno"); } static PyObject * -tb_next_get(PyTracebackObject *self, void *Py_UNUSED(_)) +tb_next_get(PyObject *op, void *Py_UNUSED(_)) { + PyTracebackObject *self = _PyTracebackObject_CAST(op); PyObject* ret = (PyObject*)self->tb_next; if (!ret) { ret = Py_None; @@ -109,18 +111,21 @@ tb_next_get(PyTracebackObject *self, void *Py_UNUSED(_)) } static int -tb_get_lineno(PyTracebackObject* tb) { +tb_get_lineno(PyObject *op) +{ + PyTracebackObject *tb = _PyTracebackObject_CAST(op); _PyInterpreterFrame* frame = tb->tb_frame->f_frame; assert(frame != NULL); return PyCode_Addr2Line(_PyFrame_GetCode(frame), tb->tb_lasti); } static PyObject * -tb_lineno_get(PyTracebackObject *self, void *Py_UNUSED(_)) +tb_lineno_get(PyObject *op, void *Py_UNUSED(_)) { + PyTracebackObject *self = _PyTracebackObject_CAST(op); int lineno = self->tb_lineno; if (lineno == -1) { - lineno = tb_get_lineno(self); + lineno = tb_get_lineno(op); if (lineno < 0) { Py_RETURN_NONE; } @@ -129,7 +134,7 @@ tb_lineno_get(PyTracebackObject *self, void *Py_UNUSED(_)) } static int -tb_next_set(PyTracebackObject *self, PyObject *new_next, void *Py_UNUSED(_)) +tb_next_set(PyObject *op, PyObject *new_next, void *Py_UNUSED(_)) { if (!new_next) { PyErr_Format(PyExc_TypeError, "can't delete tb_next attribute"); @@ -148,6 +153,7 @@ tb_next_set(PyTracebackObject *self, PyObject *new_next, void *Py_UNUSED(_)) } /* Check for loops */ + PyTracebackObject *self = _PyTracebackObject_CAST(op); PyTracebackObject *cursor = (PyTracebackObject *)new_next; while (cursor) { if (cursor == self) { @@ -164,7 +170,7 @@ tb_next_set(PyTracebackObject *self, PyObject *new_next, void *Py_UNUSED(_)) static PyMethodDef tb_methods[] = { - {"__dir__", _PyCFunction_CAST(tb_dir), METH_NOARGS}, + {"__dir__", tb_dir, METH_NOARGS, NULL}, {NULL, NULL, 0, NULL}, }; @@ -175,14 +181,15 @@ static PyMemberDef tb_memberlist[] = { }; static PyGetSetDef tb_getsetters[] = { - {"tb_next", (getter)tb_next_get, (setter)tb_next_set, NULL, NULL}, - {"tb_lineno", (getter)tb_lineno_get, NULL, NULL, NULL}, + {"tb_next", tb_next_get, tb_next_set, NULL, NULL}, + {"tb_lineno", tb_lineno_get, NULL, NULL, NULL}, {NULL} /* Sentinel */ }; static void -tb_dealloc(PyTracebackObject *tb) +tb_dealloc(PyObject *op) { + PyTracebackObject *tb = _PyTracebackObject_CAST(op); PyObject_GC_UnTrack(tb); Py_TRASHCAN_BEGIN(tb, tb_dealloc) Py_XDECREF(tb->tb_next); @@ -192,16 +199,18 @@ tb_dealloc(PyTracebackObject *tb) } static int -tb_traverse(PyTracebackObject *tb, visitproc visit, void *arg) +tb_traverse(PyObject *op, visitproc visit, void *arg) { + PyTracebackObject *tb = _PyTracebackObject_CAST(op); Py_VISIT(tb->tb_next); Py_VISIT(tb->tb_frame); return 0; } static int -tb_clear(PyTracebackObject *tb) +tb_clear(PyObject *op) { + PyTracebackObject *tb = _PyTracebackObject_CAST(op); Py_CLEAR(tb->tb_next); Py_CLEAR(tb->tb_frame); return 0; @@ -212,7 +221,7 @@ PyTypeObject PyTraceBack_Type = { "traceback", sizeof(PyTracebackObject), 0, - (destructor)tb_dealloc, /*tp_dealloc*/ + tb_dealloc, /*tp_dealloc*/ 0, /*tp_vectorcall_offset*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ @@ -229,8 +238,8 @@ PyTypeObject PyTraceBack_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ tb_new__doc__, /* tp_doc */ - (traverseproc)tb_traverse, /* tp_traverse */ - (inquiry)tb_clear, /* tp_clear */ + tb_traverse, /* tp_traverse */ + tb_clear, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ @@ -664,7 +673,7 @@ tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit) code = PyFrame_GetCode(tb->tb_frame); int tb_lineno = tb->tb_lineno; if (tb_lineno == -1) { - tb_lineno = tb_get_lineno(tb); + tb_lineno = tb_get_lineno((PyObject *)tb); } if (last_file == NULL || code->co_filename != last_file || @@ -891,6 +900,8 @@ _Py_DumpASCII(int fd, PyObject *text) static void dump_frame(int fd, _PyInterpreterFrame *frame) { + assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); + PyCodeObject *code =_PyFrame_GetCode(frame); PUTS(fd, " File "); if (code->co_filename != NULL @@ -964,6 +975,17 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) unsigned int depth = 0; while (1) { + if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { + /* Trampoline frame */ + frame = frame->previous; + if (frame == NULL) { + break; + } + + /* Can't have more than one shim frame in a row */ + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + } + if (MAX_FRAME_DEPTH <= depth) { if (MAX_FRAME_DEPTH < depth) { PUTS(fd, "plus "); @@ -972,20 +994,12 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) } break; } + dump_frame(fd, frame); frame = frame->previous; if (frame == NULL) { break; } - if (frame->owner == FRAME_OWNED_BY_CSTACK) { - /* Trampoline frame */ - frame = frame->previous; - } - if (frame == NULL) { - break; - } - /* Can't have more than one shim frame in a row */ - assert(frame->owner != FRAME_OWNED_BY_CSTACK); depth++; } } diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index 19b64c619feb6a..d69b0ebd585a7f 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -2,6 +2,8 @@ #include "pycore_fileutils.h" // _Py_write_noraise() #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_t +#include "pycore_initconfig.h" // _PyStatus_NO_MEMORY() +#include "pycore_lock.h" // PyMutex_LockFlags() #include "pycore_object.h" // _PyType_PreHeaderSize() #include "pycore_pymem.h" // _Py_tracemalloc_config #include "pycore_runtime.h" // _Py_ID() @@ -19,6 +21,8 @@ _Py_DECLARE_STR(anon_unknown, ""); /* Forward declaration */ static void* raw_malloc(size_t size); static void raw_free(void *ptr); +static int _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, + void* Py_UNUSED(ignore)); #ifdef Py_DEBUG # define TRACE_DEBUG @@ -30,18 +34,12 @@ static void raw_free(void *ptr); #define allocators _PyRuntime.tracemalloc.allocators -#if defined(TRACE_RAW_MALLOC) /* This lock is needed because tracemalloc_free() is called without the GIL held from PyMem_RawFree(). It cannot acquire the lock because it would introduce a deadlock in _PyThreadState_DeleteCurrent(). */ -# define tables_lock _PyRuntime.tracemalloc.tables_lock -# define TABLES_LOCK() PyThread_acquire_lock(tables_lock, 1) -# define TABLES_UNLOCK() PyThread_release_lock(tables_lock) -#else - /* variables are protected by the GIL */ -# define TABLES_LOCK() -# define TABLES_UNLOCK() -#endif +#define tables_lock _PyRuntime.tracemalloc.tables_lock +#define TABLES_LOCK() PyMutex_LockFlags(&tables_lock, _Py_LOCK_DONT_DETACH) +#define TABLES_UNLOCK() PyMutex_Unlock(&tables_lock) #define DEFAULT_DOMAIN 0 @@ -95,9 +93,6 @@ tracemalloc_error(const char *format, ...) #endif -#if defined(TRACE_RAW_MALLOC) -#define REENTRANT_THREADLOCAL - #define tracemalloc_reentrant_key _PyRuntime.tracemalloc.reentrant_key /* Any non-NULL pointer can be used */ @@ -106,16 +101,16 @@ tracemalloc_error(const char *format, ...) static int get_reentrant(void) { - void *ptr; - assert(PyThread_tss_is_created(&tracemalloc_reentrant_key)); - ptr = PyThread_tss_get(&tracemalloc_reentrant_key); + + void *ptr = PyThread_tss_get(&tracemalloc_reentrant_key); if (ptr != NULL) { assert(ptr == REENTRANT); return 1; } - else + else { return 0; + } } static void @@ -134,25 +129,6 @@ set_reentrant(int reentrant) } } -#else - -/* TRACE_RAW_MALLOC not defined: variable protected by the GIL */ -static int tracemalloc_reentrant = 0; - -static int -get_reentrant(void) -{ - return tracemalloc_reentrant; -} - -static void -set_reentrant(int reentrant) -{ - assert(reentrant != tracemalloc_reentrant); - tracemalloc_reentrant = reentrant; -} -#endif - static Py_uhash_t hashtable_hash_pyobject(const void *key) @@ -250,16 +226,16 @@ hashtable_compare_traceback(const void *key1, const void *key2) static void tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) { - assert(PyCode_Check(pyframe->f_executable)); + assert(PyStackRef_CodeCheck(pyframe->f_executable)); frame->filename = &_Py_STR(anon_unknown); + int lineno = PyUnstable_InterpreterFrame_GetLine(pyframe); if (lineno < 0) { lineno = 0; } frame->lineno = (unsigned int)lineno; - PyObject *filename = filename = ((PyCodeObject *)pyframe->f_executable)->co_filename; - + PyObject *filename = filename = _PyFrame_GetCode(pyframe)->co_filename; if (filename == NULL) { #ifdef TRACE_DEBUG tracemalloc_error("failed to get the filename of the code object"); @@ -275,7 +251,7 @@ tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) } if (!PyUnicode_IS_READY(filename)) { /* Don't make a Unicode string ready to avoid reentrant calls - to tracemalloc_malloc() or tracemalloc_realloc() */ + to tracemalloc_alloc() or tracemalloc_realloc() */ #ifdef TRACE_DEBUG tracemalloc_error("filename is not a ready unicode string"); #endif @@ -309,10 +285,10 @@ tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) static Py_uhash_t traceback_hash(traceback_t *traceback) { - /* code based on tuplehash() of Objects/tupleobject.c */ + /* code based on tuple_hash() of Objects/tupleobject.c */ Py_uhash_t x, y; /* Unsigned for defined overflow behavior. */ int len = traceback->nframe; - Py_uhash_t mult = _PyHASH_MULTIPLIER; + Py_uhash_t mult = PyHASH_MULTIPLIER; frame_t *frame; x = 0x345678UL; @@ -335,13 +311,8 @@ traceback_hash(traceback_t *traceback) static void traceback_get_frames(traceback_t *traceback) { - PyThreadState *tstate = PyGILState_GetThisThreadState(); - if (tstate == NULL) { -#ifdef TRACE_DEBUG - tracemalloc_error("failed to get the current thread state"); -#endif - return; - } + PyThreadState *tstate = _PyThreadState_GET(); + assert(tstate != NULL); _PyInterpreterFrame *pyframe = _PyThreadState_GetFrame(tstate); while (pyframe) { @@ -364,7 +335,7 @@ traceback_new(void) traceback_t *traceback; _Py_hashtable_entry_t *entry; - assert(PyGILState_Check()); + _Py_AssertHoldsTstate(); /* get frames */ traceback = tracemalloc_traceback; @@ -440,7 +411,7 @@ tracemalloc_get_traces_table(unsigned int domain) static void -tracemalloc_remove_trace(unsigned int domain, uintptr_t ptr) +tracemalloc_remove_trace_unlocked(unsigned int domain, uintptr_t ptr) { assert(tracemalloc_config.tracing); @@ -459,12 +430,12 @@ tracemalloc_remove_trace(unsigned int domain, uintptr_t ptr) } #define REMOVE_TRACE(ptr) \ - tracemalloc_remove_trace(DEFAULT_DOMAIN, (uintptr_t)(ptr)) + tracemalloc_remove_trace_unlocked(DEFAULT_DOMAIN, (uintptr_t)(ptr)) static int -tracemalloc_add_trace(unsigned int domain, uintptr_t ptr, - size_t size) +tracemalloc_add_trace_unlocked(unsigned int domain, uintptr_t ptr, + size_t size) { assert(tracemalloc_config.tracing); @@ -519,82 +490,147 @@ tracemalloc_add_trace(unsigned int domain, uintptr_t ptr, } #define ADD_TRACE(ptr, size) \ - tracemalloc_add_trace(DEFAULT_DOMAIN, (uintptr_t)(ptr), size) + tracemalloc_add_trace_unlocked(DEFAULT_DOMAIN, (uintptr_t)(ptr), size) static void* -tracemalloc_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) +tracemalloc_alloc(int need_gil, int use_calloc, + void *ctx, size_t nelem, size_t elsize) { - PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; - void *ptr; - assert(elsize == 0 || nelem <= SIZE_MAX / elsize); - if (use_calloc) + int reentrant = get_reentrant(); + + // Ignore reentrant call. + // + // For example, PyObjet_Malloc() calls + // PyMem_Malloc() for allocations larger than 512 bytes: don't trace the + // same memory allocation twice. + // + // If reentrant calls are not ignored, PyGILState_Ensure() can call + // PyMem_RawMalloc() which would call PyGILState_Ensure() again in a loop. + if (!reentrant) { + set_reentrant(1); + } + + PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; + void *ptr; + if (use_calloc) { ptr = alloc->calloc(alloc->ctx, nelem, elsize); - else + } + else { ptr = alloc->malloc(alloc->ctx, nelem * elsize); - if (ptr == NULL) - return NULL; + } + if (ptr == NULL) { + goto done; + } + if (reentrant) { + goto done; + } + + PyGILState_STATE gil_state; + if (need_gil) { + gil_state = PyGILState_Ensure(); + } TABLES_LOCK(); - if (ADD_TRACE(ptr, nelem * elsize) < 0) { - /* Failed to allocate a trace for the new memory block */ - TABLES_UNLOCK(); - alloc->free(alloc->ctx, ptr); - return NULL; + + if (tracemalloc_config.tracing) { + if (ADD_TRACE(ptr, nelem * elsize) < 0) { + // Failed to allocate a trace for the new memory block + alloc->free(alloc->ctx, ptr); + ptr = NULL; + } } + // else: gh-128679: tracemalloc.stop() was called by another thread + TABLES_UNLOCK(); + if (need_gil) { + PyGILState_Release(gil_state); + } + +done: + if (!reentrant) { + set_reentrant(0); + } return ptr; } static void* -tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) +tracemalloc_realloc(int need_gil, void *ctx, void *ptr, size_t new_size) { + int reentrant = get_reentrant(); + + // Ignore reentrant call. PyObjet_Realloc() calls PyMem_Realloc() for + // allocations larger than 512 bytes: don't trace the same memory block + // twice. + if (!reentrant) { + set_reentrant(1); + } + PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; - void *ptr2; + void *ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); - ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); - if (ptr2 == NULL) - return NULL; + if (ptr2 == NULL) { + goto done; + } + if (reentrant) { + goto done; + } - if (ptr != NULL) { - /* an existing memory block has been resized */ + PyGILState_STATE gil_state; + if (need_gil) { + gil_state = PyGILState_Ensure(); + } + TABLES_LOCK(); + + if (!tracemalloc_config.tracing) { + // gh-128679: tracemalloc.stop() was called by another thread + goto unlock; + } - TABLES_LOCK(); + if (ptr != NULL) { + // An existing memory block has been resized - /* tracemalloc_add_trace() updates the trace if there is already - a trace at address ptr2 */ + // tracemalloc_add_trace_unlocked() updates the trace if there is + // already a trace at address ptr2. if (ptr2 != ptr) { REMOVE_TRACE(ptr); } if (ADD_TRACE(ptr2, new_size) < 0) { - /* Memory allocation failed. The error cannot be reported to - the caller, because realloc() may already have shrunk the - memory block and so removed bytes. - - This case is very unlikely: a hash entry has just been - released, so the hash table should have at least one free entry. - - The GIL and the table lock ensures that only one thread is - allocating memory. */ + // Memory allocation failed. The error cannot be reported to the + // caller, because realloc() already have shrunk the memory block + // and so removed bytes. + // + // This case is very unlikely: a hash entry has just been released, + // so the hash table should have at least one free entry. + // + // The GIL and the table lock ensures that only one thread is + // allocating memory. Py_FatalError("tracemalloc_realloc() failed to allocate a trace"); } - TABLES_UNLOCK(); } else { - /* new allocation */ + // New allocation - TABLES_LOCK(); if (ADD_TRACE(ptr2, new_size) < 0) { - /* Failed to allocate a trace for the new memory block */ - TABLES_UNLOCK(); + // Failed to allocate a trace for the new memory block alloc->free(alloc->ctx, ptr2); - return NULL; + ptr2 = NULL; } - TABLES_UNLOCK(); + } + +unlock: + TABLES_UNLOCK(); + if (need_gil) { + PyGILState_Release(gil_state); + } + +done: + if (!reentrant) { + set_reentrant(0); } return ptr2; } @@ -603,170 +639,68 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) static void tracemalloc_free(void *ctx, void *ptr) { - PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; - - if (ptr == NULL) + if (ptr == NULL) { return; + } - /* GIL cannot be locked in PyMem_RawFree() because it would introduce - a deadlock in _PyThreadState_DeleteCurrent(). */ - + PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; alloc->free(alloc->ctx, ptr); - TABLES_LOCK(); - REMOVE_TRACE(ptr); - TABLES_UNLOCK(); -} - - -static void* -tracemalloc_alloc_gil(int use_calloc, void *ctx, size_t nelem, size_t elsize) -{ - void *ptr; - if (get_reentrant()) { - PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; - if (use_calloc) - return alloc->calloc(alloc->ctx, nelem, elsize); - else - return alloc->malloc(alloc->ctx, nelem * elsize); + return; } - /* Ignore reentrant call. PyObjet_Malloc() calls PyMem_Malloc() for - allocations larger than 512 bytes, don't trace the same memory - allocation twice. */ - set_reentrant(1); + TABLES_LOCK(); - ptr = tracemalloc_alloc(use_calloc, ctx, nelem, elsize); + if (tracemalloc_config.tracing) { + REMOVE_TRACE(ptr); + } + // else: gh-128679: tracemalloc.stop() was called by another thread - set_reentrant(0); - return ptr; + TABLES_UNLOCK(); } static void* tracemalloc_malloc_gil(void *ctx, size_t size) { - return tracemalloc_alloc_gil(0, ctx, 1, size); + return tracemalloc_alloc(0, 0, ctx, 1, size); } static void* tracemalloc_calloc_gil(void *ctx, size_t nelem, size_t elsize) { - return tracemalloc_alloc_gil(1, ctx, nelem, elsize); + return tracemalloc_alloc(0, 1, ctx, nelem, elsize); } static void* tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size) { - void *ptr2; - - if (get_reentrant()) { - /* Reentrant call to PyMem_Realloc() and PyMem_RawRealloc(). - Example: PyMem_RawRealloc() is called internally by pymalloc - (_PyObject_Malloc() and _PyObject_Realloc()) to allocate a new - arena (new_arena()). */ - PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; - - ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); - if (ptr2 != NULL && ptr != NULL) { - TABLES_LOCK(); - REMOVE_TRACE(ptr); - TABLES_UNLOCK(); - } - return ptr2; - } - - /* Ignore reentrant call. PyObjet_Realloc() calls PyMem_Realloc() for - allocations larger than 512 bytes. Don't trace the same memory - allocation twice. */ - set_reentrant(1); - - ptr2 = tracemalloc_realloc(ctx, ptr, new_size); - - set_reentrant(0); - return ptr2; -} - - -#ifdef TRACE_RAW_MALLOC -static void* -tracemalloc_raw_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) -{ - PyGILState_STATE gil_state; - void *ptr; - - if (get_reentrant()) { - PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; - if (use_calloc) - return alloc->calloc(alloc->ctx, nelem, elsize); - else - return alloc->malloc(alloc->ctx, nelem * elsize); - } - - /* Ignore reentrant call. PyGILState_Ensure() may call PyMem_RawMalloc() - indirectly which would call PyGILState_Ensure() if reentrant are not - disabled. */ - set_reentrant(1); - - gil_state = PyGILState_Ensure(); - ptr = tracemalloc_alloc(use_calloc, ctx, nelem, elsize); - PyGILState_Release(gil_state); - - set_reentrant(0); - return ptr; + return tracemalloc_realloc(0, ctx, ptr, new_size); } static void* tracemalloc_raw_malloc(void *ctx, size_t size) { - return tracemalloc_raw_alloc(0, ctx, 1, size); + return tracemalloc_alloc(1, 0, ctx, 1, size); } static void* tracemalloc_raw_calloc(void *ctx, size_t nelem, size_t elsize) { - return tracemalloc_raw_alloc(1, ctx, nelem, elsize); + return tracemalloc_alloc(1, 1, ctx, nelem, elsize); } static void* tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) { - PyGILState_STATE gil_state; - void *ptr2; - - if (get_reentrant()) { - /* Reentrant call to PyMem_RawRealloc(). */ - PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; - - ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); - - if (ptr2 != NULL && ptr != NULL) { - TABLES_LOCK(); - REMOVE_TRACE(ptr); - TABLES_UNLOCK(); - } - return ptr2; - } - - /* Ignore reentrant call. PyGILState_Ensure() may call PyMem_RawMalloc() - indirectly which would call PyGILState_Ensure() if reentrant calls are - not disabled. */ - set_reentrant(1); - - gil_state = PyGILState_Ensure(); - ptr2 = tracemalloc_realloc(ctx, ptr, new_size); - PyGILState_Release(gil_state); - - set_reentrant(0); - return ptr2; + return tracemalloc_realloc(1, ctx, ptr, new_size); } -#endif /* TRACE_RAW_MALLOC */ static void @@ -777,60 +711,36 @@ tracemalloc_clear_filename(void *value) } -/* reentrant flag must be set to call this function and GIL must be held */ static void -tracemalloc_clear_traces(void) +tracemalloc_clear_traces_unlocked(void) { - /* The GIL protects variables against concurrent access */ - assert(PyGILState_Check()); + // Clearing tracemalloc_filenames requires the GIL to call Py_DECREF() + _Py_AssertHoldsTstate(); + + set_reentrant(1); - TABLES_LOCK(); _Py_hashtable_clear(tracemalloc_traces); _Py_hashtable_clear(tracemalloc_domains); + _Py_hashtable_clear(tracemalloc_tracebacks); + _Py_hashtable_clear(tracemalloc_filenames); + tracemalloc_traced_memory = 0; tracemalloc_peak_traced_memory = 0; - TABLES_UNLOCK(); - _Py_hashtable_clear(tracemalloc_tracebacks); - - _Py_hashtable_clear(tracemalloc_filenames); + set_reentrant(0); } -int +PyStatus _PyTraceMalloc_Init(void) { - if (tracemalloc_config.initialized == TRACEMALLOC_FINALIZED) { - PyErr_SetString(PyExc_RuntimeError, - "the tracemalloc module has been unloaded"); - return -1; - } - - if (tracemalloc_config.initialized == TRACEMALLOC_INITIALIZED) - return 0; + assert(tracemalloc_config.initialized == TRACEMALLOC_NOT_INITIALIZED); PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); -#ifdef REENTRANT_THREADLOCAL if (PyThread_tss_create(&tracemalloc_reentrant_key) != 0) { -#ifdef MS_WINDOWS - PyErr_SetFromWindowsErr(0); -#else - PyErr_SetFromErrno(PyExc_OSError); -#endif - return -1; - } -#endif - -#if defined(TRACE_RAW_MALLOC) - if (tables_lock == NULL) { - tables_lock = PyThread_allocate_lock(); - if (tables_lock == NULL) { - PyErr_SetString(PyExc_RuntimeError, "cannot allocate lock"); - return -1; - } + return _PyStatus_NO_MEMORY(); } -#endif tracemalloc_filenames = hashtable_new(hashtable_hash_pyobject, hashtable_compare_unicode, @@ -838,15 +748,15 @@ _PyTraceMalloc_Init(void) tracemalloc_tracebacks = hashtable_new(hashtable_hash_traceback, hashtable_compare_traceback, - NULL, raw_free); + raw_free, NULL); tracemalloc_traces = tracemalloc_create_traces_table(); tracemalloc_domains = tracemalloc_create_domains_table(); if (tracemalloc_filenames == NULL || tracemalloc_tracebacks == NULL - || tracemalloc_traces == NULL || tracemalloc_domains == NULL) { - PyErr_NoMemory(); - return -1; + || tracemalloc_traces == NULL || tracemalloc_domains == NULL) + { + return _PyStatus_NO_MEMORY(); } tracemalloc_empty_traceback.nframe = 1; @@ -857,7 +767,7 @@ _PyTraceMalloc_Init(void) tracemalloc_empty_traceback.hash = traceback_hash(&tracemalloc_empty_traceback); tracemalloc_config.initialized = TRACEMALLOC_INITIALIZED; - return 0; + return _PyStatus_OK(); } @@ -876,25 +786,13 @@ tracemalloc_deinit(void) _Py_hashtable_destroy(tracemalloc_tracebacks); _Py_hashtable_destroy(tracemalloc_filenames); -#if defined(TRACE_RAW_MALLOC) - if (tables_lock != NULL) { - PyThread_free_lock(tables_lock); - tables_lock = NULL; - } -#endif - -#ifdef REENTRANT_THREADLOCAL PyThread_tss_delete(&tracemalloc_reentrant_key); -#endif } int _PyTraceMalloc_Start(int max_nframe) { - PyMemAllocatorEx alloc; - size_t size; - if (max_nframe < 1 || (unsigned long) max_nframe > MAX_NFRAME) { PyErr_Format(PyExc_ValueError, "the number of frames must be in range [1; %lu]", @@ -902,19 +800,15 @@ _PyTraceMalloc_Start(int max_nframe) return -1; } - if (_PyTraceMalloc_Init() < 0) { - return -1; - } - - if (tracemalloc_config.tracing) { - /* hook already installed: do nothing */ + if (_PyTraceMalloc_IsTracing()) { + /* hooks already installed: do nothing */ return 0; } tracemalloc_config.max_nframe = max_nframe; /* allocate a buffer to store a new traceback */ - size = TRACEBACK_SIZE(max_nframe); + size_t size = TRACEBACK_SIZE(max_nframe); assert(tracemalloc_traceback == NULL); tracemalloc_traceback = raw_malloc(size); if (tracemalloc_traceback == NULL) { @@ -922,7 +816,7 @@ _PyTraceMalloc_Start(int max_nframe) return -1; } -#ifdef TRACE_RAW_MALLOC + PyMemAllocatorEx alloc; alloc.malloc = tracemalloc_raw_malloc; alloc.calloc = tracemalloc_raw_calloc; alloc.realloc = tracemalloc_raw_realloc; @@ -931,7 +825,6 @@ _PyTraceMalloc_Start(int max_nframe) alloc.ctx = &allocators.raw; PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc); -#endif alloc.malloc = tracemalloc_malloc_gil; alloc.calloc = tracemalloc_calloc_gil; @@ -946,8 +839,14 @@ _PyTraceMalloc_Start(int max_nframe) PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &allocators.obj); PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc); + if (PyRefTracer_SetTracer(_PyTraceMalloc_TraceRef, NULL) < 0) { + return -1; + } + /* everything is ready: start tracing Python memory allocations */ + TABLES_LOCK(); tracemalloc_config.tracing = 1; + TABLES_UNLOCK(); return 0; } @@ -956,24 +855,30 @@ _PyTraceMalloc_Start(int max_nframe) void _PyTraceMalloc_Stop(void) { - if (!tracemalloc_config.tracing) - return; + TABLES_LOCK(); + + if (!tracemalloc_config.tracing) { + goto done; + } /* stop tracing Python memory allocations */ tracemalloc_config.tracing = 0; /* unregister the hook on memory allocators */ -#ifdef TRACE_RAW_MALLOC PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); -#endif PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &allocators.mem); PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &allocators.obj); - tracemalloc_clear_traces(); + tracemalloc_clear_traces_unlocked(); /* release memory */ raw_free(tracemalloc_traceback); tracemalloc_traceback = NULL; + + (void)PyRefTracer_SetTracer(NULL, NULL); + +done: + TABLES_UNLOCK(); } @@ -981,15 +886,16 @@ _PyTraceMalloc_Stop(void) static PyObject* frame_to_pyobject(frame_t *frame) { - PyObject *frame_obj, *lineno_obj; + assert(get_reentrant()); - frame_obj = PyTuple_New(2); - if (frame_obj == NULL) + PyObject *frame_obj = PyTuple_New(2); + if (frame_obj == NULL) { return NULL; + } PyTuple_SET_ITEM(frame_obj, 0, Py_NewRef(frame->filename)); - lineno_obj = PyLong_FromUnsignedLong(frame->lineno); + PyObject *lineno_obj = PyLong_FromUnsignedLong(frame->lineno); if (lineno_obj == NULL) { Py_DECREF(frame_obj); return NULL; @@ -1004,7 +910,6 @@ static PyObject* traceback_to_pyobject(traceback_t *traceback, _Py_hashtable_t *intern_table) { PyObject *frames; - if (intern_table != NULL) { frames = _Py_hashtable_get(intern_table, (const void *)traceback); if (frames) { @@ -1013,8 +918,9 @@ traceback_to_pyobject(traceback_t *traceback, _Py_hashtable_t *intern_table) } frames = PyTuple_New(traceback->nframe); - if (frames == NULL) + if (frames == NULL) { return NULL; + } for (int i=0; i < traceback->nframe; i++) { PyObject *frame = frame_to_pyobject(&traceback->frames[i]); @@ -1042,14 +948,14 @@ static PyObject* trace_to_pyobject(unsigned int domain, const trace_t *trace, _Py_hashtable_t *intern_tracebacks) { - PyObject *trace_obj = NULL; - PyObject *obj; + assert(get_reentrant()); - trace_obj = PyTuple_New(4); - if (trace_obj == NULL) + PyObject *trace_obj = PyTuple_New(4); + if (trace_obj == NULL) { return NULL; + } - obj = PyLong_FromSize_t(domain); + PyObject *obj = PyLong_FromSize_t(domain); if (obj == NULL) { Py_DECREF(trace_obj); return NULL; @@ -1096,7 +1002,6 @@ tracemalloc_copy_trace(_Py_hashtable_t *traces, void *user_data) { _Py_hashtable_t *traces2 = (_Py_hashtable_t *)user_data; - trace_t *trace = (trace_t *)value; trace_t *trace2 = raw_malloc(sizeof(trace_t)); @@ -1137,7 +1042,6 @@ tracemalloc_copy_domain(_Py_hashtable_t *domains, void *user_data) { _Py_hashtable_t *domains2 = (_Py_hashtable_t *)user_data; - unsigned int domain = (unsigned int)FROM_PTR(key); _Py_hashtable_t *traces = (_Py_hashtable_t *)value; @@ -1178,7 +1082,6 @@ tracemalloc_get_traces_fill(_Py_hashtable_t *traces, void *user_data) { get_traces_t *get_traces = user_data; - const trace_t *trace = (const trace_t *)value; PyObject *tuple = trace_to_pyobject(get_traces->domain, trace, @@ -1192,7 +1095,6 @@ tracemalloc_get_traces_fill(_Py_hashtable_t *traces, if (res < 0) { return 1; } - return 0; } @@ -1203,7 +1105,6 @@ tracemalloc_get_traces_domain(_Py_hashtable_t *domains, void *user_data) { get_traces_t *get_traces = user_data; - unsigned int domain = (unsigned int)FROM_PTR(key); _Py_hashtable_t *traces = (_Py_hashtable_t *)value; @@ -1223,27 +1124,21 @@ tracemalloc_pyobject_decref(void *value) static traceback_t* -tracemalloc_get_traceback(unsigned int domain, uintptr_t ptr) +tracemalloc_get_traceback_unlocked(unsigned int domain, uintptr_t ptr) { - - if (!tracemalloc_config.tracing) + if (!tracemalloc_config.tracing) { return NULL; + } - trace_t *trace; - TABLES_LOCK(); _Py_hashtable_t *traces = tracemalloc_get_traces_table(domain); - if (traces) { - trace = _Py_hashtable_get(traces, TO_PTR(ptr)); - } - else { - trace = NULL; + if (!traces) { + return NULL; } - TABLES_UNLOCK(); + trace_t *trace = _Py_hashtable_get(traces, TO_PTR(ptr)); if (!trace) { return NULL; } - return trace->traceback; } @@ -1265,24 +1160,28 @@ _PyMem_DumpFrame(int fd, frame_t * frame) void _PyMem_DumpTraceback(int fd, const void *ptr) { - traceback_t *traceback; - int i; - + TABLES_LOCK(); if (!tracemalloc_config.tracing) { PUTS(fd, "Enable tracemalloc to get the memory block " "allocation traceback\n\n"); - return; + goto done; } - traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, (uintptr_t)ptr); - if (traceback == NULL) - return; + traceback_t *traceback; + traceback = tracemalloc_get_traceback_unlocked(DEFAULT_DOMAIN, + (uintptr_t)ptr); + if (traceback == NULL) { + goto done; + } PUTS(fd, "Memory block allocated at (most recent call first):\n"); - for (i=0; i < traceback->nframe; i++) { + for (int i=0; i < traceback->nframe; i++) { _PyMem_DumpFrame(fd, &traceback->frames[i]); } PUTS(fd, "\n"); + +done: + TABLES_UNLOCK(); } #undef PUTS @@ -1303,45 +1202,48 @@ int PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, size_t size) { - int res; - PyGILState_STATE gil_state; + PyGILState_STATE gil_state = PyGILState_Ensure(); + TABLES_LOCK(); - if (!tracemalloc_config.tracing) { + int result; + if (tracemalloc_config.tracing) { + result = tracemalloc_add_trace_unlocked(domain, ptr, size); + } + else { /* tracemalloc is not tracing: do nothing */ - return -2; + result = -2; } - gil_state = PyGILState_Ensure(); - - TABLES_LOCK(); - res = tracemalloc_add_trace(domain, ptr, size); TABLES_UNLOCK(); - PyGILState_Release(gil_state); - return res; + return result; } int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) { - if (!tracemalloc_config.tracing) { + TABLES_LOCK(); + + int result; + if (tracemalloc_config.tracing) { + tracemalloc_remove_trace_unlocked(domain, ptr); + result = 0; + } + else { /* tracemalloc is not tracing: do nothing */ - return -2; + result = -2; } - TABLES_LOCK(); - tracemalloc_remove_trace(domain, ptr); TABLES_UNLOCK(); - - return 0; + return result; } void _PyTraceMalloc_Fini(void) { - assert(PyGILState_Check()); + _Py_AssertHoldsTstate(); tracemalloc_deinit(); } @@ -1351,83 +1253,102 @@ _PyTraceMalloc_Fini(void) Do nothing if tracemalloc is not tracing memory allocations or if the object memory block is not already traced. */ -int -_PyTraceMalloc_NewReference(PyObject *op) +static int +_PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, + void* Py_UNUSED(ignore)) { - assert(PyGILState_Check()); + if (event != PyRefTracer_CREATE) { + return 0; + } + if (get_reentrant()) { + return 0; + } + + _Py_AssertHoldsTstate(); + TABLES_LOCK(); if (!tracemalloc_config.tracing) { - /* tracemalloc is not tracing: do nothing */ - return -1; + goto done; } PyTypeObject *type = Py_TYPE(op); const size_t presize = _PyType_PreHeaderSize(type); uintptr_t ptr = (uintptr_t)((char *)op - presize); - int res = -1; - - TABLES_LOCK(); trace_t *trace = _Py_hashtable_get(tracemalloc_traces, TO_PTR(ptr)); if (trace != NULL) { /* update the traceback of the memory block */ traceback_t *traceback = traceback_new(); if (traceback != NULL) { trace->traceback = traceback; - res = 0; } } /* else: cannot track the object, its memory block size is unknown */ - TABLES_UNLOCK(); - return res; +done: + TABLES_UNLOCK(); + return 0; } PyObject* _PyTraceMalloc_GetTraceback(unsigned int domain, uintptr_t ptr) { - traceback_t *traceback; + TABLES_LOCK(); - traceback = tracemalloc_get_traceback(domain, ptr); - if (traceback == NULL) - Py_RETURN_NONE; + traceback_t *traceback = tracemalloc_get_traceback_unlocked(domain, ptr); + PyObject *result; + if (traceback) { + set_reentrant(1); + result = traceback_to_pyobject(traceback, NULL); + set_reentrant(0); + } + else { + result = Py_NewRef(Py_None); + } - return traceback_to_pyobject(traceback, NULL); + TABLES_UNLOCK(); + return result; } int _PyTraceMalloc_IsTracing(void) { - return tracemalloc_config.tracing; + TABLES_LOCK(); + int tracing = tracemalloc_config.tracing; + TABLES_UNLOCK(); + return tracing; } void _PyTraceMalloc_ClearTraces(void) { - - if (!tracemalloc_config.tracing) { - return; + TABLES_LOCK(); + if (tracemalloc_config.tracing) { + tracemalloc_clear_traces_unlocked(); } - set_reentrant(1); - tracemalloc_clear_traces(); - set_reentrant(0); + TABLES_UNLOCK(); } PyObject * _PyTraceMalloc_GetTraces(void) { + TABLES_LOCK(); + set_reentrant(1); + get_traces_t get_traces; get_traces.domain = DEFAULT_DOMAIN; get_traces.traces = NULL; get_traces.domains = NULL; get_traces.tracebacks = NULL; get_traces.list = PyList_New(0); - if (get_traces.list == NULL) - goto error; + if (get_traces.list == NULL) { + goto finally; + } - if (!tracemalloc_config.tracing) - return get_traces.list; + if (!tracemalloc_config.tracing) { + goto finally; + } /* the traceback hash table is used temporarily to intern traceback tuple of (filename, lineno) tuples */ @@ -1441,24 +1362,17 @@ _PyTraceMalloc_GetTraces(void) // Copy all traces so tracemalloc_get_traces_fill() doesn't have to disable // temporarily tracemalloc which would impact other threads and so would // miss allocations while get_traces() is called. - TABLES_LOCK(); get_traces.traces = tracemalloc_copy_traces(tracemalloc_traces); - TABLES_UNLOCK(); - if (get_traces.traces == NULL) { goto no_memory; } - TABLES_LOCK(); get_traces.domains = tracemalloc_copy_domains(tracemalloc_domains); - TABLES_UNLOCK(); - if (get_traces.domains == NULL) { goto no_memory; } // Convert traces to a list of tuples - set_reentrant(1); int err = _Py_hashtable_foreach(get_traces.traces, tracemalloc_get_traces_fill, &get_traces); @@ -1467,20 +1381,22 @@ _PyTraceMalloc_GetTraces(void) tracemalloc_get_traces_domain, &get_traces); } - set_reentrant(0); + if (err) { - goto error; + Py_CLEAR(get_traces.list); + goto finally; } - goto finally; no_memory: PyErr_NoMemory(); - -error: Py_CLEAR(get_traces.list); + goto finally; finally: + set_reentrant(0); + TABLES_UNLOCK(); + if (get_traces.tracebacks != NULL) { _Py_hashtable_destroy(get_traces.tracebacks); } @@ -1498,37 +1414,33 @@ PyObject * _PyTraceMalloc_GetObjectTraceback(PyObject *obj) /*[clinic end generated code: output=41ee0553a658b0aa input=29495f1b21c53212]*/ { - PyTypeObject *type; - traceback_t *traceback; - - type = Py_TYPE(obj); + PyTypeObject *type = Py_TYPE(obj); const size_t presize = _PyType_PreHeaderSize(type); uintptr_t ptr = (uintptr_t)((char *)obj - presize); - - traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, ptr); - if (traceback == NULL) { - Py_RETURN_NONE; - } - - return traceback_to_pyobject(traceback, NULL); + return _PyTraceMalloc_GetTraceback(DEFAULT_DOMAIN, ptr); } -int _PyTraceMalloc_GetTracebackLimit(void) { +int _PyTraceMalloc_GetTracebackLimit(void) +{ return tracemalloc_config.max_nframe; } size_t -_PyTraceMalloc_GetMemory(void) { - +_PyTraceMalloc_GetMemory(void) +{ + TABLES_LOCK(); size_t size; + if (tracemalloc_config.tracing) { + size = _Py_hashtable_size(tracemalloc_tracebacks); + size += _Py_hashtable_size(tracemalloc_filenames); - size = _Py_hashtable_size(tracemalloc_tracebacks); - size += _Py_hashtable_size(tracemalloc_filenames); - - TABLES_LOCK(); - size += _Py_hashtable_size(tracemalloc_traces); - _Py_hashtable_foreach(tracemalloc_domains, - tracemalloc_get_tracemalloc_memory_cb, &size); + size += _Py_hashtable_size(tracemalloc_traces); + _Py_hashtable_foreach(tracemalloc_domains, + tracemalloc_get_tracemalloc_memory_cb, &size); + } + else { + size = 0; + } TABLES_UNLOCK(); return size; } @@ -1537,26 +1449,27 @@ _PyTraceMalloc_GetMemory(void) { PyObject * _PyTraceMalloc_GetTracedMemory(void) { - Py_ssize_t size, peak_size; - - if (!tracemalloc_config.tracing) - return Py_BuildValue("ii", 0, 0); - TABLES_LOCK(); - size = tracemalloc_traced_memory; - peak_size = tracemalloc_peak_traced_memory; + Py_ssize_t traced, peak; + if (tracemalloc_config.tracing) { + traced = tracemalloc_traced_memory; + peak = tracemalloc_peak_traced_memory; + } + else { + traced = 0; + peak = 0; + } TABLES_UNLOCK(); - return Py_BuildValue("nn", size, peak_size); + return Py_BuildValue("nn", traced, peak); } void _PyTraceMalloc_ResetPeak(void) { - if (!tracemalloc_config.tracing) { - return; - } TABLES_LOCK(); - tracemalloc_peak_traced_memory = tracemalloc_traced_memory; + if (tracemalloc_config.tracing) { + tracemalloc_peak_traced_memory = tracemalloc_traced_memory; + } TABLES_UNLOCK(); } diff --git a/Python/uniqueid.c b/Python/uniqueid.c new file mode 100644 index 00000000000000..64c3e6cfbbe825 --- /dev/null +++ b/Python/uniqueid.c @@ -0,0 +1,232 @@ +#include "Python.h" + +#include "pycore_dict.h" // _PyDict_UniqueId() +#include "pycore_lock.h" // PyMutex_LockFlags() +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_object.h" // _Py_IncRefTotal +#include "pycore_uniqueid.h" + +// This contains code for allocating unique ids for per-thread reference +// counting and re-using those ids when an object is deallocated. +// +// Currently, per-thread reference counting is only used for heap types. +// +// See Include/internal/pycore_uniqueid.h for more details. + +#ifdef Py_GIL_DISABLED + +#define POOL_MIN_SIZE 8 + +#define LOCK_POOL(pool) PyMutex_LockFlags(&pool->mutex, _Py_LOCK_DONT_DETACH) +#define UNLOCK_POOL(pool) PyMutex_Unlock(&pool->mutex) + +static int +resize_interp_type_id_pool(struct _Py_unique_id_pool *pool) +{ + if ((size_t)pool->size > PY_SSIZE_T_MAX / (2 * sizeof(*pool->table))) { + return -1; + } + + Py_ssize_t new_size = pool->size * 2; + if (new_size < POOL_MIN_SIZE) { + new_size = POOL_MIN_SIZE; + } + + _Py_unique_id_entry *table = PyMem_Realloc(pool->table, + new_size * sizeof(*pool->table)); + if (table == NULL) { + return -1; + } + + Py_ssize_t start = pool->size; + for (Py_ssize_t i = start; i < new_size - 1; i++) { + table[i].next = &table[i + 1]; + } + table[new_size - 1].next = NULL; + + pool->table = table; + pool->freelist = &table[start]; + _Py_atomic_store_ssize(&pool->size, new_size); + return 0; +} + +static int +resize_local_refcounts(_PyThreadStateImpl *tstate) +{ + if (tstate->refcounts.is_finalized) { + return -1; + } + + struct _Py_unique_id_pool *pool = &tstate->base.interp->unique_ids; + Py_ssize_t size = _Py_atomic_load_ssize(&pool->size); + + Py_ssize_t *refcnts = PyMem_Realloc(tstate->refcounts.values, + size * sizeof(Py_ssize_t)); + if (refcnts == NULL) { + return -1; + } + + Py_ssize_t old_size = tstate->refcounts.size; + if (old_size < size) { + memset(refcnts + old_size, 0, (size - old_size) * sizeof(Py_ssize_t)); + } + + tstate->refcounts.values = refcnts; + tstate->refcounts.size = size; + return 0; +} + +Py_ssize_t +_PyObject_AssignUniqueId(PyObject *obj) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _Py_unique_id_pool *pool = &interp->unique_ids; + + LOCK_POOL(pool); + if (pool->freelist == NULL) { + if (resize_interp_type_id_pool(pool) < 0) { + UNLOCK_POOL(pool); + return _Py_INVALID_UNIQUE_ID; + } + } + + _Py_unique_id_entry *entry = pool->freelist; + pool->freelist = entry->next; + entry->obj = obj; + _PyObject_SetDeferredRefcount(obj); + // The unique id is one plus the index of the entry in the table. + Py_ssize_t unique_id = (entry - pool->table) + 1; + assert(unique_id > 0); + UNLOCK_POOL(pool); + return unique_id; +} + +void +_PyObject_ReleaseUniqueId(Py_ssize_t unique_id) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _Py_unique_id_pool *pool = &interp->unique_ids; + + LOCK_POOL(pool); + assert(unique_id > 0 && unique_id <= pool->size); + Py_ssize_t idx = unique_id - 1; + _Py_unique_id_entry *entry = &pool->table[idx]; + entry->next = pool->freelist; + pool->freelist = entry; + UNLOCK_POOL(pool); +} + +static Py_ssize_t +clear_unique_id(PyObject *obj) +{ + Py_ssize_t id = _Py_INVALID_UNIQUE_ID; + if (PyType_Check(obj)) { + if (PyType_HasFeature((PyTypeObject *)obj, Py_TPFLAGS_HEAPTYPE)) { + PyHeapTypeObject *ht = (PyHeapTypeObject *)obj; + id = ht->unique_id; + ht->unique_id = _Py_INVALID_UNIQUE_ID; + } + } + else if (PyCode_Check(obj)) { + PyCodeObject *co = (PyCodeObject *)obj; + id = co->_co_unique_id; + co->_co_unique_id = _Py_INVALID_UNIQUE_ID; + } + else if (PyDict_Check(obj)) { + PyDictObject *mp = (PyDictObject *)obj; + id = _PyDict_UniqueId(mp); + mp->_ma_watcher_tag &= ~(UINT64_MAX << DICT_UNIQUE_ID_SHIFT); + } + return id; +} + +void +_PyObject_DisablePerThreadRefcounting(PyObject *obj) +{ + Py_ssize_t id = clear_unique_id(obj); + if (id != _Py_INVALID_UNIQUE_ID) { + _PyObject_ReleaseUniqueId(id); + } +} + +void +_PyObject_ThreadIncrefSlow(PyObject *obj, size_t idx) +{ + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + if (((Py_ssize_t)idx) < 0 || resize_local_refcounts(tstate) < 0) { + // just incref the object directly. + Py_INCREF(obj); + return; + } + + assert(idx < (size_t)tstate->refcounts.size); + tstate->refcounts.values[idx]++; +#ifdef Py_REF_DEBUG + _Py_IncRefTotal((PyThreadState *)tstate); +#endif + _Py_INCREF_STAT_INC(); +} + +void +_PyObject_MergePerThreadRefcounts(_PyThreadStateImpl *tstate) +{ + if (tstate->refcounts.values == NULL) { + return; + } + + struct _Py_unique_id_pool *pool = &tstate->base.interp->unique_ids; + + LOCK_POOL(pool); + for (Py_ssize_t i = 0, n = tstate->refcounts.size; i < n; i++) { + Py_ssize_t refcnt = tstate->refcounts.values[i]; + if (refcnt != 0) { + PyObject *obj = pool->table[i].obj; + _Py_atomic_add_ssize(&obj->ob_ref_shared, + refcnt << _Py_REF_SHARED_SHIFT); + tstate->refcounts.values[i] = 0; + } + } + UNLOCK_POOL(pool); +} + +void +_PyObject_FinalizePerThreadRefcounts(_PyThreadStateImpl *tstate) +{ + _PyObject_MergePerThreadRefcounts(tstate); + + PyMem_Free(tstate->refcounts.values); + tstate->refcounts.values = NULL; + tstate->refcounts.size = 0; + tstate->refcounts.is_finalized = 1; +} + +void +_PyObject_FinalizeUniqueIdPool(PyInterpreterState *interp) +{ + struct _Py_unique_id_pool *pool = &interp->unique_ids; + + // First, set the free-list to NULL values + while (pool->freelist) { + _Py_unique_id_entry *next = pool->freelist->next; + pool->freelist->obj = NULL; + pool->freelist = next; + } + + // Now everything non-NULL is a object. Clear their unique ids as the + // object outlives the interpreter. + for (Py_ssize_t i = 0; i < pool->size; i++) { + PyObject *obj = pool->table[i].obj; + pool->table[i].obj = NULL; + if (obj != NULL) { + Py_ssize_t id = clear_unique_id(obj); + (void)id; + assert(id == i + 1); + } + } + PyMem_Free(pool->table); + pool->table = NULL; + pool->freelist = NULL; + pool->size = 0; +} + +#endif /* Py_GIL_DISABLED */ diff --git a/Python/vm-state.md b/Python/vm-state.md index 4c68ba3b575cc8..b3246557dbeea3 100644 --- a/Python/vm-state.md +++ b/Python/vm-state.md @@ -87,4 +87,4 @@ Tier 2 IR entries are all the same size; there is no equivalent to `EXTENDED_ARG - **opcode**: Sometimes the same as a Tier 1 opcode, sometimes a separate micro opcode. Tier 2 opcodes are 9 bits (as opposed to Tier 1 opcodes, which fit in 8 bits). By convention, Tier 2 opcode names start with `_`. - **oparg**: The argument. Usually the same as the Tier 1 oparg after expansion of `EXTENDED_ARG` prefixes. Up to 32 bits. -- **operand**: An aditional argument, Typically the value of *one* cache item from the Tier 1 inline cache, up to 64 bits. +- **operand**: An additional argument, Typically the value of *one* cache item from the Tier 1 inline cache, up to 64 bits. diff --git a/README.rst b/README.rst index fd3e6dd1771ae1..6f74bc3f9ed7ef 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ -This is Python version 3.13.0 alpha 4 +This is Python version 3.14.0 alpha 4 ===================================== -.. image:: https://github.com/python/cpython/workflows/Tests/badge.svg +.. image:: https://github.com/python/cpython/actions/workflows/build.yml/badge.svg?branch=main&event=push :alt: CPython build status on GitHub Actions :target: https://github.com/python/cpython/actions @@ -14,7 +14,7 @@ This is Python version 3.13.0 alpha 4 :target: https://discuss.python.org/ -Copyright © 2001-2024 Python Software Foundation. All rights reserved. +Copyright © 2001 Python Software Foundation. All rights reserved. See the end of this file for further copyright and license information. @@ -64,7 +64,7 @@ the executable is called ``python.exe``; elsewhere it's just ``python``. Building a complete Python installation requires the use of various additional third-party libraries, depending on your build platform and configure options. Not all standard library modules are buildable or -useable on all platforms. Refer to the +usable on all platforms. Refer to the `Install dependencies `_ section of the `Developer Guide`_ for current detailed information on dependencies for various Linux distributions and macOS. @@ -136,7 +136,7 @@ What's New ---------- We have a comprehensive overview of the changes in the `What's New in Python -3.13 `_ document. For a more +3.14 `_ document. For a more detailed change log, read `Misc/NEWS `_, but a full accounting of changes can only be gleaned from the `commit history @@ -149,7 +149,7 @@ entitled "Installing multiple versions". Documentation ------------- -`Documentation for Python 3.13 `_ is online, +`Documentation for Python 3.14 `_ is online, updated daily. It can also be downloaded in many formats for faster access. The documentation @@ -200,22 +200,22 @@ intend to install multiple versions using the same prefix you must decide which version (if any) is your "primary" version. Install that version using ``make install``. Install all other versions using ``make altinstall``. -For example, if you want to install Python 2.7, 3.6, and 3.13 with 3.13 being the -primary version, you would execute ``make install`` in your 3.13 build directory +For example, if you want to install Python 2.7, 3.6, and 3.14 with 3.14 being the +primary version, you would execute ``make install`` in your 3.14 build directory and ``make altinstall`` in the others. Release Schedule ---------------- -See :pep:`719` for Python 3.13 release details. +See `PEP 745 `__ for Python 3.14 release details. Copyright and License Information --------------------------------- -Copyright © 2001-2024 Python Software Foundation. All rights reserved. +Copyright © 2001 Python Software Foundation. All rights reserved. Copyright © 2000 BeOpen.com. All rights reserved. diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml new file mode 100644 index 00000000000000..53bfe18f8de1b8 --- /dev/null +++ b/Tools/build/.ruff.toml @@ -0,0 +1,20 @@ +target-version = "py310" +fix = true +line-length = 79 + +[lint] +select = [ + "C4", # flake8-comprehensions + "E", # pycodestyle + "F", # pyflakes + "I", # isort + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "PYI", # flake8-pyi + "RUF100", # Ban unused `# noqa` comments + "UP", # pyupgrade + "W", # pycodestyle + "YTT", # flake8-2020 +] diff --git a/Tools/build/.warningignore_macos b/Tools/build/.warningignore_macos new file mode 100644 index 00000000000000..e72309229cc60f --- /dev/null +++ b/Tools/build/.warningignore_macos @@ -0,0 +1,9 @@ +# Files listed will be ignored by the compiler warning checker +# for the macOS/build and test job. +# Keep lines sorted lexicographically to help avoid merge conflicts. +# Format example: +# /path/to/file (number of warnings in file) +Modules/expat/siphash.h 7 +Modules/expat/xmlparse.c 8 +Modules/expat/xmltok.c 3 +Modules/expat/xmltok_impl.c 26 diff --git a/Tools/build/.warningignore_ubuntu b/Tools/build/.warningignore_ubuntu new file mode 100644 index 00000000000000..469c727abfb11c --- /dev/null +++ b/Tools/build/.warningignore_ubuntu @@ -0,0 +1,5 @@ +# Files listed will be ignored by the compiler warning checker +# for the Ubuntu/build and test job. +# Keep lines sorted lexicographically to help avoid merge conflicts. +# Format example: +# /path/to/file (number of warnings in file) diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index a9fee4981ea915..66b2a262e11c57 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -27,6 +27,7 @@ import sys import sysconfig import warnings +import _imp from importlib._bootstrap import _load as bootstrap_load from importlib.machinery import BuiltinImporter, ExtensionFileLoader, ModuleSpec @@ -53,6 +54,7 @@ "_overlapped", "_testconsole", "_winapi", + "_wmi", "msvcrt", "nt", "winreg", @@ -153,6 +155,11 @@ def __init__(self, cross_compiling: bool = False, strict: bool = False): self.notavailable = [] def check(self): + if not hasattr(_imp, 'create_dynamic'): + logger.warning( + ('Dynamic extensions not supported ' + '(HAVE_DYNAMIC_LOADING not defined)'), + ) for modinfo in self.modules: logger.debug("Checking '%s' (%s)", modinfo.name, self.get_location(modinfo)) if modinfo.state == ModuleState.DISABLED: @@ -414,6 +421,9 @@ def check_module_import(self, modinfo: ModuleInfo): logger.error("%s failed to import: %s", modinfo.name, e) raise except Exception as e: + if not hasattr(_imp, 'create_dynamic'): + logger.warning("Dynamic extension '%s' ignored", modinfo.name) + return logger.exception("Importing extension '%s' failed!", modinfo.name) raise diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py new file mode 100644 index 00000000000000..7a8721087b6c66 --- /dev/null +++ b/Tools/build/check_warnings.py @@ -0,0 +1,323 @@ +""" +Parses compiler output from Clang or GCC and checks that warnings +exist only in files that are expected to have warnings. +""" + +import argparse +import re +import sys +from collections import defaultdict +from pathlib import Path +from typing import NamedTuple + + +class IgnoreRule(NamedTuple): + file_path: str + count: int + ignore_all: bool = False + is_directory: bool = False + + +def parse_warning_ignore_file(file_path: str) -> set[IgnoreRule]: + """ + Parses the warning ignore file and returns a set of IgnoreRules + """ + files_with_expected_warnings = set() + with Path(file_path).open(encoding="UTF-8") as ignore_rules_file: + files_with_expected_warnings = set() + for i, line in enumerate(ignore_rules_file): + line = line.strip() + if line and not line.startswith("#"): + line_parts = line.split() + if len(line_parts) >= 2: + file_name = line_parts[0] + count = line_parts[1] + ignore_all = count == "*" + is_directory = file_name.endswith("/") + + # Directories must have a wildcard count + if is_directory and count != "*": + print( + f"Error parsing ignore file: {file_path} " + f"at line: {i}" + ) + print( + f"Directory {file_name} must have count set to *" + ) + sys.exit(1) + if ignore_all: + count = 0 + + files_with_expected_warnings.add( + IgnoreRule( + file_name, int(count), ignore_all, is_directory + ) + ) + + return files_with_expected_warnings + + +def extract_warnings_from_compiler_output( + compiler_output: str, + compiler_output_type: str, + path_prefix: str = "", +) -> list[dict]: + """ + Extracts warnings from the compiler output based on compiler + output type. Removes path prefix from file paths if provided. + Compatible with GCC and Clang compiler output. + """ + # Choose pattern and compile regex for particular compiler output + if compiler_output_type == "gcc": + regex_pattern = ( + r"(?P.*):(?P\d+):(?P\d+): warning: " + r"(?P.*?)(?: (?P
    Installs the Python documentation files. &pip Installs pip, which can download and install other Python packages. - tcl/tk and &IDLE - Installs tkinter and the IDLE development environment. + Tcl/Tk, turtle and &IDLE + Installs tkinter, turtle and the IDLE development environment. Python &test suite Installs the standard library test suite. py &launcher @@ -123,7 +123,7 @@ Feel free to post at <a href="https://discuss.python.org/c/users/7">discus You must restart your computer to complete the rollback of the software. &Restart Unable to install [WixBundleName] due to an existing install. Use Programs and Features to modify, repair or remove [WixBundleName]. - At least Windows 8.1 or Windows Server 2012 are required to install [WixBundleName] + At least Windows 10 or Windows Server 2016 are required to install [WixBundleName] Visit <a href="https://www.python.org/downloads/">python.org</a> to download an earlier version of Python. Disable path length limit diff --git a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp index e0e179e3aede6d..1e0df5084ff079 100644 --- a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp +++ b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp @@ -213,6 +213,7 @@ static struct { LPCWSTR regName; LPCWSTR variableName; } OPTIONAL_FEATURES[] = { { L"Shortcuts", L"Shortcuts" }, // Include_launcher and AssociateFiles are handled separately and so do // not need to be included in this list. + { L"freethreaded", L"Include_freethreaded" }, { nullptr, nullptr } }; @@ -464,11 +465,11 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { LOC_STRING *pLocString = nullptr; LPCWSTR locKey = L"#(loc.Include_launcherHelp)"; - LONGLONG detectedLauncher; + LONGLONG blockedLauncher; - if (SUCCEEDED(BalGetNumericVariable(L"DetectedLauncher", &detectedLauncher)) && detectedLauncher) { + if (SUCCEEDED(BalGetNumericVariable(L"BlockedLauncher", &blockedLauncher)) && blockedLauncher) { locKey = L"#(loc.Include_launcherRemove)"; - } else if (SUCCEEDED(BalGetNumericVariable(L"DetectedOldLauncher", &detectedLauncher)) && detectedLauncher) { + } else if (SUCCEEDED(BalGetNumericVariable(L"DetectedOldLauncher", &blockedLauncher)) && blockedLauncher) { locKey = L"#(loc.Include_launcherUpgrade)"; } @@ -2671,7 +2672,7 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { /*Elevate when installing for all users*/ L"InstallAllUsers or " /*Elevate when installing the launcher for all users and it was not detected*/ - L"(Include_launcher and InstallLauncherAllUsers and not DetectedLauncher)" + L"(Include_launcher and InstallLauncherAllUsers and not BlockedLauncher)" L")", L"" }; @@ -3085,11 +3086,13 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { LOC_STRING *pLocString = nullptr; if (IsWindowsServer()) { - if (IsWindowsVersionOrGreater(6, 2, 0)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Target OS is Windows Server 2012 or later"); + if (IsWindowsVersionOrGreater(10, 0, 0)) { + BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Target OS is Windows Server 2016 or later"); return; + } else if (IsWindowsVersionOrGreater(6, 2, 0)) { + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Server 2012"); } else if (IsWindowsVersionOrGreater(6, 1, 1)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Detected Windows Server 2008 R2"); + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Server 2008 R2"); } else if (IsWindowsVersionOrGreater(6, 1, 0)) { BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Server 2008 R2"); } else if (IsWindowsVersionOrGreater(6, 0, 0)) { @@ -3097,14 +3100,13 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { } else { BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Server 2003 or earlier"); } - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Windows Server 2012 or later is required to continue installation"); + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Windows Server 2016 or later is required to continue installation"); } else { if (IsWindows10OrGreater()) { BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Target OS is Windows 10 or later"); return; } else if (IsWindows8Point1OrGreater()) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Target OS is Windows 8.1"); - return; + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows 8.1"); } else if (IsWindows8OrGreater()) { BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows 8"); } else if (IsWindows7OrGreater()) { @@ -3114,7 +3116,7 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { } else { BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows XP or earlier"); } - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Windows 8.1 or later is required to continue installation"); + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Windows 10 or later is required to continue installation"); } LocGetString(_wixLoc, L"#(loc.FailureOldOS)", &pLocString); diff --git a/Tools/msi/bundle/bootstrap/pythonba.vcxproj b/Tools/msi/bundle/bootstrap/pythonba.vcxproj index bb383bfcaa240e..3970e857894ba5 100644 --- a/Tools/msi/bundle/bootstrap/pythonba.vcxproj +++ b/Tools/msi/bundle/bootstrap/pythonba.vcxproj @@ -24,7 +24,6 @@ v143 v142 v141 - v140 {7A09B132-B3EE-499B-A700-A4B2157FEA3D} PythonBA @@ -51,8 +50,6 @@ comctl32.lib;gdiplus.lib;msimg32.lib;shlwapi.lib;wininet.lib;dutil.lib;balutil.lib;version.lib;uxtheme.lib;%(AdditionalDependencies) $(WixInstallPath)sdk\vs2017\lib\x86 - $(WixInstallPath)sdk\vs2015\lib\x86 - $(WixInstallPath)sdk\vs2013\lib\x86 pythonba.def true diff --git a/Tools/msi/bundle/packagegroups/launcher.wxs b/Tools/msi/bundle/packagegroups/launcher.wxs index a6922758f31f14..080598a0a486ef 100644 --- a/Tools/msi/bundle/packagegroups/launcher.wxs +++ b/Tools/msi/bundle/packagegroups/launcher.wxs @@ -11,7 +11,7 @@ EnableFeatureSelection="yes" Permanent="yes" Visible="yes" - InstallCondition="(InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not DetectedLauncher"> + InstallCondition="(InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not BlockedLauncher"> @@ -25,7 +25,7 @@ EnableFeatureSelection="yes" Permanent="yes" Visible="yes" - InstallCondition="not (InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not DetectedLauncher"> + InstallCondition="not (InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not BlockedLauncher"> diff --git a/Tools/msi/freethreaded/freethreaded_files.wxs b/Tools/msi/freethreaded/freethreaded_files.wxs index adaf63c69d5ade..b3ce28e7aedc84 100644 --- a/Tools/msi/freethreaded/freethreaded_files.wxs +++ b/Tools/msi/freethreaded/freethreaded_files.wxs @@ -48,6 +48,12 @@ + + + + + + @@ -69,8 +75,14 @@ - - + + + + + + + + @@ -91,7 +103,7 @@ - + @@ -147,17 +159,13 @@ - - - - - - - - + + + - - + + + diff --git a/Tools/msi/lib/lib.wixproj b/Tools/msi/lib/lib.wixproj index 26311ea32724d1..02078e503d74a4 100644 --- a/Tools/msi/lib/lib.wixproj +++ b/Tools/msi/lib/lib.wixproj @@ -19,6 +19,7 @@ @@ -32,4 +33,4 @@ - \ No newline at end of file + diff --git a/Tools/msi/pip/pip.wxs b/Tools/msi/pip/pip.wxs index 1d8083cad91a56..627c4710a9fdfa 100644 --- a/Tools/msi/pip/pip.wxs +++ b/Tools/msi/pip/pip.wxs @@ -25,8 +25,8 @@ - - + + (&DefaultFeature=3) AND NOT (!DefaultFeature=3) diff --git a/Tools/msi/purge.py b/Tools/msi/purge.py index e25219a6caf9d4..4a13d368d7c744 100644 --- a/Tools/msi/purge.py +++ b/Tools/msi/purge.py @@ -51,14 +51,6 @@ "test_pdb.msi", "tools.msi", "ucrt.msi", - "Windows6.0-KB2999226-x64.msu", - "Windows6.0-KB2999226-x86.msu", - "Windows6.1-KB2999226-x64.msu", - "Windows6.1-KB2999226-x86.msu", - "Windows8.1-KB2999226-x64.msu", - "Windows8.1-KB2999226-x86.msu", - "Windows8-RT-KB2999226-x64.msu", - "Windows8-RT-KB2999226-x86.msu", ] PATHS = [ "python-{}.exe".format(m.group(0)), diff --git a/Tools/msi/tcltk/tcltk.wixproj b/Tools/msi/tcltk/tcltk.wixproj index 218f3d15ec88fc..c8b7ab77c4dc6b 100644 --- a/Tools/msi/tcltk/tcltk.wixproj +++ b/Tools/msi/tcltk/tcltk.wixproj @@ -28,7 +28,7 @@ tcltk_lib - $(PySourcePath) !(bindpath.src) @@ -39,4 +39,4 @@ - \ No newline at end of file + diff --git a/Tools/msi/test/test_files.wxs b/Tools/msi/test/test_files.wxs index bb9b258692a62f..6862a5899db2ba 100644 --- a/Tools/msi/test/test_files.wxs +++ b/Tools/msi/test/test_files.wxs @@ -1,6 +1,6 @@ - + diff --git a/Tools/msi/wix.props b/Tools/msi/wix.props index d8ced317d0ce81..707c8aeacb648d 100644 --- a/Tools/msi/wix.props +++ b/Tools/msi/wix.props @@ -5,8 +5,6 @@ $(MSBuildThisFileDirectory)\Wix\ $(ExternalsDir)\windows-installer\wix-314\ - $(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows Installer XML\3.10@InstallRoot) - $(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\Windows Installer XML\3.10@InstallRoot) $(WixInstallPath)\Wix.targets \ No newline at end of file diff --git a/Tools/patchcheck/patchcheck.py b/Tools/patchcheck/patchcheck.py index af1f0584bb5403..0dcf6ef844a048 100755 --- a/Tools/patchcheck/patchcheck.py +++ b/Tools/patchcheck/patchcheck.py @@ -1,15 +1,10 @@ #!/usr/bin/env python3 """Check proposed changes for common issues.""" -import re import sys -import shutil import os.path import subprocess import sysconfig -import reindent -import untabify - def get_python_source_dir(): src_dir = sysconfig.get_config_var('abs_srcdir') @@ -18,13 +13,6 @@ def get_python_source_dir(): return os.path.abspath(src_dir) -# Excluded directories which are copies of external libraries: -# don't check their coding style -EXCLUDE_DIRS = [ - os.path.join('Modules', '_decimal', 'libmpdec'), - os.path.join('Modules', 'expat'), - os.path.join('Modules', 'zlib'), - ] SRCDIR = get_python_source_dir() @@ -155,62 +143,7 @@ def changed_files(base_branch=None): else: sys.exit('need a git checkout to get modified files') - filenames2 = [] - for filename in filenames: - # Normalize the path to be able to match using .startswith() - filename = os.path.normpath(filename) - if any(filename.startswith(path) for path in EXCLUDE_DIRS): - # Exclude the file - continue - filenames2.append(filename) - - return filenames2 - - -def report_modified_files(file_paths): - count = len(file_paths) - if count == 0: - return n_files_str(count) - else: - lines = [f"{n_files_str(count)}:"] - for path in file_paths: - lines.append(f" {path}") - return "\n".join(lines) - - -#: Python files that have tabs by design: -_PYTHON_FILES_WITH_TABS = frozenset({ - 'Tools/c-analyzer/cpython/_parser.py', -}) - - -@status("Fixing Python file whitespace", info=report_modified_files) -def normalize_whitespace(file_paths): - """Make sure that the whitespace for .py files have been normalized.""" - reindent.makebackup = False # No need to create backups. - fixed = [ - path for path in file_paths - if ( - path.endswith('.py') - and path not in _PYTHON_FILES_WITH_TABS - and reindent.check(os.path.join(SRCDIR, path)) - ) - ] - return fixed - - -@status("Fixing C file whitespace", info=report_modified_files) -def normalize_c_whitespace(file_paths): - """Report if any C files """ - fixed = [] - for path in file_paths: - abspath = os.path.join(SRCDIR, path) - with open(abspath, 'r') as f: - if '\t' not in f.read(): - continue - untabify.process(abspath, 8, verbose=False) - fixed.append(path) - return fixed + return list(map(os.path.normpath, filenames)) @status("Docs modified", modal=True) @@ -250,40 +183,14 @@ def regenerated_pyconfig_h_in(file_paths): return "not needed" -def ci(pull_request): - if pull_request == 'false': - print('Not a pull request; skipping') - return - base_branch = get_base_branch() - file_paths = changed_files(base_branch) - python_files = [fn for fn in file_paths if fn.endswith('.py')] - c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))] - fixed = [] - fixed.extend(normalize_whitespace(python_files)) - fixed.extend(normalize_c_whitespace(c_files)) - if not fixed: - print('No whitespace issues found') - else: - count = len(fixed) - print(f'Please fix the {n_files_str(count)} with whitespace issues') - print('(on Unix you can run `make patchcheck` to make the fixes)') - sys.exit(1) - - def main(): base_branch = get_base_branch() file_paths = changed_files(base_branch) - python_files = [fn for fn in file_paths if fn.endswith('.py')] - c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))] - doc_files = [fn for fn in file_paths if fn.startswith('Doc') and - fn.endswith(('.rst', '.inc'))] + has_doc_files = any(fn for fn in file_paths if fn.startswith('Doc') and + fn.endswith(('.rst', '.inc'))) misc_files = {p for p in file_paths if p.startswith('Misc')} - # PEP 8 whitespace rules enforcement. - normalize_whitespace(python_files) - # C rules enforcement. - normalize_c_whitespace(c_files) # Docs updated. - docs_modified(doc_files) + docs_modified(has_doc_files) # Misc/ACKS changed. credit_given(misc_files) # Misc/NEWS changed. @@ -294,19 +201,14 @@ def main(): regenerated_pyconfig_h_in(file_paths) # Test suite run and passed. - if python_files or c_files: - end = " and check for refleaks?" if c_files else "?" - print() - print("Did you run the test suite" + end) + has_c_files = any(fn for fn in file_paths if fn.endswith(('.c', '.h'))) + has_python_files = any(fn for fn in file_paths if fn.endswith('.py')) + print() + if has_c_files: + print("Did you run the test suite and check for refleaks?") + elif has_python_files: + print("Did you run the test suite?") if __name__ == '__main__': - import argparse - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--ci', - help='Perform pass/fail checks') - args = parser.parse_args() - if args.ci: - ci(args.ci) - else: - main() + main() diff --git a/Tools/peg_generator/peg_extension/peg_extension.c b/Tools/peg_generator/peg_extension/peg_extension.c index b081240ffff017..1587d53d59472e 100644 --- a/Tools/peg_generator/peg_extension/peg_extension.c +++ b/Tools/peg_generator/peg_extension/peg_extension.c @@ -108,7 +108,7 @@ parse_string(PyObject *self, PyObject *args, PyObject *kwds) static PyObject * clear_memo_stats(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) { -#if defined(PY_DEBUG) +#if defined(Py_DEBUG) _PyPegen_clear_memo_statistics(); #endif Py_RETURN_NONE; @@ -117,7 +117,7 @@ clear_memo_stats(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) static PyObject * get_memo_stats(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) { -#if defined(PY_DEBUG) +#if defined(Py_DEBUG) return _PyPegen_get_memo_statistics(); #else Py_RETURN_NONE; @@ -128,7 +128,7 @@ get_memo_stats(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) static PyObject * dump_memo_stats(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) { -#if defined(PY_DEBUG) +#if defined(Py_DEBUG) PyObject *list = _PyPegen_get_memo_statistics(); if (list == NULL) { return NULL; diff --git a/Tools/peg_generator/pegen/__main__.py b/Tools/peg_generator/pegen/__main__.py index 262c8a6db68f6e..0b0b4b291c2b0e 100755 --- a/Tools/peg_generator/pegen/__main__.py +++ b/Tools/peg_generator/pegen/__main__.py @@ -107,7 +107,10 @@ def generate_python_code( help="Suppress code emission for rule actions", ) -python_parser = subparsers.add_parser("python", help="Generate Python code") +python_parser = subparsers.add_parser( + "python", + help="Generate Python code, needs grammar definition with Python actions", +) python_parser.set_defaults(func=generate_python_code) python_parser.add_argument("grammar_filename", help="Grammar description") python_parser.add_argument( diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index 7cdd5debe9a225..a4951d05e80ebd 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -3,7 +3,7 @@ import re from dataclasses import dataclass, field from enum import Enum -from typing import IO, Any, Dict, List, Optional, Set, Text, Tuple +from typing import IO, Any, Callable, Dict, List, Optional, Set, Text, Tuple from pegen import grammar from pegen.grammar import ( @@ -130,7 +130,7 @@ def __init__( self.gen = parser_generator self.exact_tokens = exact_tokens self.non_exact_tokens = non_exact_tokens - self.cache: Dict[Any, FunctionCall] = {} + self.cache: Dict[str, str] = {} self.cleanup_statements: List[str] = [] def keyword_helper(self, keyword: str) -> FunctionCall: @@ -206,21 +206,6 @@ def visit_StringLeaf(self, node: StringLeaf) -> FunctionCall: comment=f"token='{val}'", ) - def visit_Rhs(self, node: Rhs) -> FunctionCall: - if node in self.cache: - return self.cache[node] - if node.can_be_inlined: - self.cache[node] = self.generate_call(node.alts[0].items[0]) - else: - name = self.gen.artifical_rule_from_rhs(node) - self.cache[node] = FunctionCall( - assigned_variable=f"{name}_var", - function=f"{name}_rule", - arguments=["p"], - comment=f"{node}", - ) - return self.cache[node] - def visit_NamedItem(self, node: NamedItem) -> FunctionCall: call = self.generate_call(node.item) if node.name: @@ -253,7 +238,7 @@ def lookahead_call_helper(self, node: Lookahead, positive: int) -> FunctionCall: else: return FunctionCall( function=f"_PyPegen_lookahead", - arguments=[positive, call.function, *call.arguments], + arguments=[positive, f"(void *(*)(Parser *)) {call.function}", *call.arguments], return_type="int", ) @@ -302,44 +287,62 @@ def visit_Opt(self, node: Opt) -> FunctionCall: comment=f"{node}", ) - def visit_Repeat0(self, node: Repeat0) -> FunctionCall: - if node in self.cache: - return self.cache[node] - name = self.gen.artificial_rule_from_repeat(node.node, False) - self.cache[node] = FunctionCall( + def _generate_artificial_rule_call( + self, + node: Any, + prefix: str, + rule_generation_func: Callable[[], str], + return_type: Optional[str] = None, + ) -> FunctionCall: + node_str = f"{node}" + key = f"{prefix}_{node_str}" + if key in self.cache: + name = self.cache[key] + else: + name = rule_generation_func() + self.cache[key] = name + + return FunctionCall( assigned_variable=f"{name}_var", function=f"{name}_rule", arguments=["p"], - return_type="asdl_seq *", - comment=f"{node}", + return_type=return_type, + comment=node_str, + ) + + def visit_Rhs(self, node: Rhs) -> FunctionCall: + if node.can_be_inlined: + return self.generate_call(node.alts[0].items[0]) + + return self._generate_artificial_rule_call( + node, + "rhs", + lambda: self.gen.artificial_rule_from_rhs(node), + ) + + def visit_Repeat0(self, node: Repeat0) -> FunctionCall: + return self._generate_artificial_rule_call( + node, + "repeat0", + lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=False), + "asdl_seq *", ) - return self.cache[node] def visit_Repeat1(self, node: Repeat1) -> FunctionCall: - if node in self.cache: - return self.cache[node] - name = self.gen.artificial_rule_from_repeat(node.node, True) - self.cache[node] = FunctionCall( - assigned_variable=f"{name}_var", - function=f"{name}_rule", - arguments=["p"], - return_type="asdl_seq *", - comment=f"{node}", + return self._generate_artificial_rule_call( + node, + "repeat1", + lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=True), + "asdl_seq *", ) - return self.cache[node] def visit_Gather(self, node: Gather) -> FunctionCall: - if node in self.cache: - return self.cache[node] - name = self.gen.artifical_rule_from_gather(node) - self.cache[node] = FunctionCall( - assigned_variable=f"{name}_var", - function=f"{name}_rule", - arguments=["p"], - return_type="asdl_seq *", - comment=f"{node}", + return self._generate_artificial_rule_call( + node, + "gather", + lambda: self.gen.artificial_rule_from_gather(node), + "asdl_seq *", ) - return self.cache[node] def visit_Group(self, node: Group) -> FunctionCall: return self.generate_call(node.rhs) @@ -645,7 +648,7 @@ def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None: self.print("}") self.print("asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena);") self.out_of_memory_return(f"!_seq", cleanup_code="PyMem_Free(_children);") - self.print("for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]);") + self.print("for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]);") self.print("PyMem_Free(_children);") if memoize and node.name: self.print(f"_PyPegen_insert_memo(p, _start_mark, {node.name}_type, _seq);") diff --git a/Tools/peg_generator/pegen/grammar_parser.py b/Tools/peg_generator/pegen/grammar_parser.py index bf31fe53263645..2e3a607f7209b0 100644 --- a/Tools/peg_generator/pegen/grammar_parser.py +++ b/Tools/peg_generator/pegen/grammar_parser.py @@ -575,7 +575,7 @@ def target_atoms(self) -> Optional[str]: @memoize def target_atom(self) -> Optional[str]: - # target_atom: "{" ~ target_atoms? "}" | "[" ~ target_atoms? "]" | NAME "*" | NAME | NUMBER | STRING | "?" | ":" | !"}" !"]" OP + # target_atom: "{" ~ target_atoms? "}" | "[" ~ target_atoms? "]" | NAME "*" | NAME | NUMBER | STRING | FSTRING_START | FSTRING_MIDDLE | FSTRING_END | "?" | ":" | !"}" !"]" OP mark = self._mark() cut = False if ( @@ -625,6 +625,21 @@ def target_atom(self) -> Optional[str]: ): return string . string self._reset(mark) + if ( + (fstring_start := self.fstring_start()) + ): + return fstring_start . string + self._reset(mark) + if ( + (fstring_middle := self.fstring_middle()) + ): + return fstring_middle . string + self._reset(mark) + if ( + (fstring_end := self.fstring_end()) + ): + return fstring_end . string + self._reset(mark) if ( (literal := self.expect("?")) ): diff --git a/Tools/peg_generator/pegen/metagrammar.gram b/Tools/peg_generator/pegen/metagrammar.gram index f22c334ca371b2..f484c4781823bc 100644 --- a/Tools/peg_generator/pegen/metagrammar.gram +++ b/Tools/peg_generator/pegen/metagrammar.gram @@ -126,6 +126,9 @@ target_atom[str]: | NAME { name.string } | NUMBER { number.string } | STRING { string.string } + | FSTRING_START { fstring_start.string } + | FSTRING_MIDDLE { fstring_middle.string } + | FSTRING_END { fstring_end.string } | "?" { "?" } | ":" { ":" } | !"}" !"]" OP { op.string } diff --git a/Tools/peg_generator/pegen/parser.py b/Tools/peg_generator/pegen/parser.py index ed0aec9db2443f..a987d30a9d6438 100644 --- a/Tools/peg_generator/pegen/parser.py +++ b/Tools/peg_generator/pegen/parser.py @@ -205,6 +205,36 @@ def string(self) -> Optional[tokenize.TokenInfo]: return self._tokenizer.getnext() return None + @memoize + def fstring_start(self) -> Optional[tokenize.TokenInfo]: + FSTRING_START = getattr(token, "FSTRING_START", None) + if not FSTRING_START: + return None + tok = self._tokenizer.peek() + if tok.type == FSTRING_START: + return self._tokenizer.getnext() + return None + + @memoize + def fstring_middle(self) -> Optional[tokenize.TokenInfo]: + FSTRING_MIDDLE = getattr(token, "FSTRING_MIDDLE", None) + if not FSTRING_MIDDLE: + return None + tok = self._tokenizer.peek() + if tok.type == FSTRING_MIDDLE: + return self._tokenizer.getnext() + return None + + @memoize + def fstring_end(self) -> Optional[tokenize.TokenInfo]: + FSTRING_END = getattr(token, "FSTRING_END", None) + if not FSTRING_END: + return None + tok = self._tokenizer.peek() + if tok.type == FSTRING_END: + return self._tokenizer.getnext() + return None + @memoize def op(self) -> Optional[tokenize.TokenInfo]: tok = self._tokenizer.peek() diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py index ad28f6c27dcb37..6ce0649aefe7ff 100644 --- a/Tools/peg_generator/pegen/parser_generator.py +++ b/Tools/peg_generator/pegen/parser_generator.py @@ -1,3 +1,4 @@ +import sys import ast import contextlib import re @@ -41,7 +42,7 @@ class RuleCollectorVisitor(GrammarVisitor): - """Visitor that invokes a provieded callmaker visitor with just the NamedItem nodes""" + """Visitor that invokes a provided callmaker visitor with just the NamedItem nodes""" def __init__(self, rules: Dict[str, Rule], callmakervisitor: GrammarVisitor) -> None: self.rulses = rules @@ -75,6 +76,11 @@ class RuleCheckingVisitor(GrammarVisitor): def __init__(self, rules: Dict[str, Rule], tokens: Set[str]): self.rules = rules self.tokens = tokens + # If python < 3.12 add the virtual fstring tokens + if sys.version_info < (3, 12): + self.tokens.add("FSTRING_START") + self.tokens.add("FSTRING_END") + self.tokens.add("FSTRING_MIDDLE") def visit_NameLeaf(self, node: NameLeaf) -> None: if node.value not in self.rules and node.value not in self.tokens: @@ -167,7 +173,7 @@ def keyword_type(self) -> int: self.keyword_counter += 1 return self.keyword_counter - def artifical_rule_from_rhs(self, rhs: Rhs) -> str: + def artificial_rule_from_rhs(self, rhs: Rhs) -> str: self.counter += 1 name = f"_tmp_{self.counter}" # TODO: Pick a nicer name. self.all_rules[name] = Rule(name, None, rhs) @@ -183,9 +189,7 @@ def artificial_rule_from_repeat(self, node: Plain, is_repeat1: bool) -> str: self.all_rules[name] = Rule(name, None, Rhs([Alt([NamedItem(None, node)])])) return name - def artifical_rule_from_gather(self, node: Gather) -> str: - self.counter += 1 - name = f"_gather_{self.counter}" + def artificial_rule_from_gather(self, node: Gather) -> str: self.counter += 1 extra_function_name = f"_loop0_{self.counter}" extra_function_alt = Alt( @@ -197,6 +201,8 @@ def artifical_rule_from_gather(self, node: Gather) -> str: None, Rhs([extra_function_alt]), ) + self.counter += 1 + name = f"_gather_{self.counter}" alt = Alt( [NamedItem("elem", node.node), NamedItem("seq", NameLeaf(extra_function_name))], ) diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py index 4a2883eb4ee202..4bb26480ebc0af 100644 --- a/Tools/peg_generator/pegen/python_generator.py +++ b/Tools/peg_generator/pegen/python_generator.py @@ -1,6 +1,6 @@ import os.path import token -from typing import IO, Any, Dict, Optional, Sequence, Set, Text, Tuple +from typing import IO, Any, Callable, Dict, Optional, Sequence, Set, Text, Tuple from pegen import grammar from pegen.grammar import ( @@ -93,13 +93,14 @@ def visit_Forced(self, node: Forced) -> bool: class PythonCallMakerVisitor(GrammarVisitor): def __init__(self, parser_generator: ParserGenerator): self.gen = parser_generator - self.cache: Dict[Any, Any] = {} + self.cache: Dict[str, Tuple[str, str]] = {} def visit_NameLeaf(self, node: NameLeaf) -> Tuple[Optional[str], str]: name = node.value if name == "SOFT_KEYWORD": return "soft_keyword", "self.soft_keyword()" - if name in ("NAME", "NUMBER", "STRING", "OP", "TYPE_COMMENT"): + if name in ("NAME", "NUMBER", "STRING", "OP", "TYPE_COMMENT", + "FSTRING_END", "FSTRING_MIDDLE", "FSTRING_START"): name = name.lower() return name, f"self.{name}()" if name in ("NEWLINE", "DEDENT", "INDENT", "ENDMARKER"): @@ -110,16 +111,6 @@ def visit_NameLeaf(self, node: NameLeaf) -> Tuple[Optional[str], str]: def visit_StringLeaf(self, node: StringLeaf) -> Tuple[str, str]: return "literal", f"self.expect({node.value})" - def visit_Rhs(self, node: Rhs) -> Tuple[Optional[str], str]: - if node in self.cache: - return self.cache[node] - if len(node.alts) == 1 and len(node.alts[0].items) == 1: - self.cache[node] = self.visit(node.alts[0].items[0]) - else: - name = self.gen.artifical_rule_from_rhs(node) - self.cache[node] = name, f"self.{name}()" - return self.cache[node] - def visit_NamedItem(self, node: NamedItem) -> Tuple[Optional[str], str]: name, call = self.visit(node.item) if node.name: @@ -151,26 +142,57 @@ def visit_Opt(self, node: Opt) -> Tuple[str, str]: else: return "opt", f"{call}," + def _generate_artificial_rule_call( + self, + node: Any, + prefix: str, + call_by_name_func: Callable[[str], str], + rule_generation_func: Callable[[], str], + ) -> Tuple[str, str]: + node_str = f"{node}" + key = f"{prefix}_{node_str}" + if key in self.cache: + return self.cache[key] + + name = rule_generation_func() + call = call_by_name_func(name) + self.cache[key] = name, call + return self.cache[key] + + def visit_Rhs(self, node: Rhs) -> Tuple[str, str]: + if len(node.alts) == 1 and len(node.alts[0].items) == 1: + return self.visit(node.alts[0].items[0]) + + return self._generate_artificial_rule_call( + node, + "rhs", + lambda name: f"self.{name}()", + lambda: self.gen.artificial_rule_from_rhs(node), + ) + def visit_Repeat0(self, node: Repeat0) -> Tuple[str, str]: - if node in self.cache: - return self.cache[node] - name = self.gen.artificial_rule_from_repeat(node.node, False) - self.cache[node] = name, f"self.{name}()," # Also a trailing comma! - return self.cache[node] + return self._generate_artificial_rule_call( + node, + "repeat0", + lambda name: f"self.{name}(),", # Also a trailing comma! + lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=False), + ) def visit_Repeat1(self, node: Repeat1) -> Tuple[str, str]: - if node in self.cache: - return self.cache[node] - name = self.gen.artificial_rule_from_repeat(node.node, True) - self.cache[node] = name, f"self.{name}()" # But no trailing comma here! - return self.cache[node] + return self._generate_artificial_rule_call( + node, + "repeat1", + lambda name: f"self.{name}()", # But no trailing comma here! + lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=True), + ) def visit_Gather(self, node: Gather) -> Tuple[str, str]: - if node in self.cache: - return self.cache[node] - name = self.gen.artifical_rule_from_gather(node) - self.cache[node] = name, f"self.{name}()" # No trailing comma here either! - return self.cache[node] + return self._generate_artificial_rule_call( + node, + "gather", + lambda name: f"self.{name}()", # No trailing comma here either! + lambda: self.gen.artificial_rule_from_gather(node), + ) def visit_Group(self, node: Group) -> Tuple[Optional[str], str]: return self.visit(node.rhs) diff --git a/Tools/peg_generator/pegen/sccutils.py b/Tools/peg_generator/pegen/sccutils.py index 1f0586bb2f7d6d..da4c9331625dd9 100644 --- a/Tools/peg_generator/pegen/sccutils.py +++ b/Tools/peg_generator/pegen/sccutils.py @@ -18,7 +18,7 @@ def strongly_connected_components( exactly once; vertices not part of a SCC are returned as singleton sets. - From http://code.activestate.com/recipes/578507/. + From https://code.activestate.com/recipes/578507-strongly-connected-components-of-a-directed-graph/. """ identified: Set[str] = set() stack: List[str] = [] @@ -81,7 +81,7 @@ def topsort( {B, C} {A} - From http://code.activestate.com/recipes/577413/. + From https://code.activestate.com/recipes/577413-topological-sort/history/1/. """ # TODO: Use a faster algorithm? for k, v in data.items(): diff --git a/Tools/peg_generator/pegen/validator.py b/Tools/peg_generator/pegen/validator.py index c48a01eedf5d5c..4699d5712d9522 100644 --- a/Tools/peg_generator/pegen/validator.py +++ b/Tools/peg_generator/pegen/validator.py @@ -34,6 +34,18 @@ def check_intersection(self, first_alt: Alt, second_alt: Alt) -> None: ) +class RaiseRuleValidator(GrammarValidator): + def visit_Alt(self, node: Alt) -> None: + if self.rulename and self.rulename.startswith('invalid'): + # raising is allowed in invalid rules + return + if node.action and 'RAISE_SYNTAX_ERROR' in node.action: + raise ValidationError( + f"In {self.rulename!r} there is an alternative that contains " + f"RAISE_SYNTAX_ERROR; this is only allowed in invalid_ rules" + ) + + def validate_grammar(the_grammar: grammar.Grammar) -> None: for validator_cls in GrammarValidator.__subclasses__(): validator = validator_cls(the_grammar) diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index 2fb616e894a734..e5badaccadda40 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -1,7 +1,7 @@ # Requirements file for external linters and checks we run on # Tools/clinic, Tools/cases_generator/, and Tools/peg_generator/ in CI -mypy==1.8.0 +mypy==1.13 # needed for peg_generator: -types-psutil==5.9.5.20240205 -types-setuptools==69.1.0.20240301 +types-psutil==6.0.0.20240901 +types-setuptools==74.0.0.20240831 diff --git a/Tools/requirements-hypothesis.txt b/Tools/requirements-hypothesis.txt index 1a45d1c431dd11..66898885c0a412 100644 --- a/Tools/requirements-hypothesis.txt +++ b/Tools/requirements-hypothesis.txt @@ -1,4 +1,4 @@ # Requirements file for hypothesis that # we use to run our property-based tests in CI. -hypothesis==6.98.15 +hypothesis==6.111.2 diff --git a/Tools/scripts/sortperf.py b/Tools/scripts/sortperf.py index b54681524ac173..1978a6c83dbe2b 100644 --- a/Tools/scripts/sortperf.py +++ b/Tools/scripts/sortperf.py @@ -130,7 +130,8 @@ def run(self, loops: int) -> float: def _prepare_data(self, loops: int) -> list[float]: bench = BENCHMARKS[self._name] - return [bench(self._size, self._random)] * loops + data = bench(self._size, self._random) + return [data.copy() for _ in range(loops)] def add_cmdline_args(cmd: list[str], args) -> None: diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 2925e096f4d95e..bc7ccfe33e777d 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -114,7 +114,7 @@ def load_raw_data(input: Path) -> RawData: return data else: - raise ValueError(f"{input:r} is not a file or directory path") + raise ValueError(f"{input} is not a file or directory path") def save_raw_data(data: RawData, json_output: TextIO): @@ -394,16 +394,22 @@ def get_call_stats(self) -> dict[str, int]: return result def get_object_stats(self) -> dict[str, tuple[int, int]]: - total_materializations = self._data.get("Object new values", 0) + total_materializations = self._data.get("Object inline values", 0) total_allocations = self._data.get("Object allocations", 0) + self._data.get( "Object allocations from freelist", 0 ) - total_increfs = self._data.get( - "Object interpreter increfs", 0 - ) + self._data.get("Object increfs", 0) - total_decrefs = self._data.get( - "Object interpreter decrefs", 0 - ) + self._data.get("Object decrefs", 0) + total_increfs = ( + self._data.get("Object interpreter mortal increfs", 0) + + self._data.get("Object mortal increfs", 0) + + self._data.get("Object interpreter immortal increfs", 0) + + self._data.get("Object immortal increfs", 0) + ) + total_decrefs = ( + self._data.get("Object interpreter mortal decrefs", 0) + + self._data.get("Object mortal decrefs", 0) + + self._data.get("Object interpreter immortal decrefs", 0) + + self._data.get("Object immortal decrefs", 0) + ) result = {} for key, value in self._data.items(): @@ -459,10 +465,7 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: "The number of times a potential trace is identified. Specifically, this " "occurs in the JUMP BACKWARD instruction when the counter reaches a " "threshold.", - ): ( - attempts, - None, - ), + ): (attempts, None), Doc( "Traces created", "The number of traces that were successfully created." ): (created, attempts), @@ -480,7 +483,7 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: ): (trace_too_long, attempts), Doc( "Trace too short", - "A potential trace is abandoced because it it too short.", + "A potential trace is abandoned because it it too short.", ): (trace_too_short, attempts), Doc( "Inner loop found", "A trace is truncated because it has an inner loop" @@ -512,6 +515,36 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: ), } + def get_optimizer_stats(self) -> dict[str, tuple[int, int | None]]: + attempts = self._data["Optimization optimizer attempts"] + successes = self._data["Optimization optimizer successes"] + no_memory = self._data["Optimization optimizer failure no memory"] + builtins_changed = self._data["Optimizer remove globals builtins changed"] + incorrect_keys = self._data["Optimizer remove globals incorrect keys"] + + return { + Doc( + "Optimizer attempts", + "The number of times the trace optimizer (_Py_uop_analyze_and_optimize) was run.", + ): (attempts, None), + Doc( + "Optimizer successes", + "The number of traces that were successfully optimized.", + ): (successes, attempts), + Doc( + "Optimizer no memory", + "The number of optimizations that failed due to no memory.", + ): (no_memory, attempts), + Doc( + "Remove globals builtins changed", + "The builtins changed during optimization", + ): (builtins_changed, attempts), + Doc( + "Remove globals incorrect keys", + "The keys in the globals dictionary aren't what was expected", + ): (incorrect_keys, attempts), + } + def get_histogram(self, prefix: str) -> list[tuple[int, int]]: rows = [] for k, v in self._data.items(): @@ -709,9 +742,9 @@ def execution_count_section() -> Section: ) -def pair_count_section() -> Section: +def pair_count_section(prefix: str, title=None) -> Section: def calc_pair_count_table(stats: Stats) -> Rows: - opcode_stats = stats.get_opcode_stats("opcode") + opcode_stats = stats.get_opcode_stats(prefix) pair_counts = opcode_stats.get_pair_counts() total = opcode_stats.get_total_execution_count() @@ -733,7 +766,7 @@ def calc_pair_count_table(stats: Stats) -> Rows: return Section( "Pair counts", - "Pair counts for top 100 Tier 1 instructions", + f"Pair counts for top 100 {title if title else prefix} pairs", [ Table( ("Pair", "Count:", "Self:", "Cumulative:"), @@ -1067,8 +1100,7 @@ def calc_object_stats_table(stats: Stats) -> Rows: Below, "allocations" means "allocations that are not from a freelist". Total allocations = "Allocations from freelist" + "Allocations". - "New values" is the number of values arrays created for objects with - managed dicts. + "Inline values" is the number of values arrays inlined into objects. The cache hit/miss numbers are for the MRO cache, split into dunder and other names. @@ -1086,6 +1118,8 @@ def calc_gc_stats(stats: Stats) -> Rows: Count(gen["collections"]), Count(gen["objects collected"]), Count(gen["object visits"]), + Count(gen["objects reachable from roots"]), + Count(gen["objects not reachable from roots"]), ) for (i, gen) in enumerate(gc_stats) ] @@ -1095,7 +1129,8 @@ def calc_gc_stats(stats: Stats) -> Rows: "GC collections and effectiveness", [ Table( - ("Generation:", "Collections:", "Objects collected:", "Object visits:"), + ("Generation:", "Collections:", "Objects collected:", "Object visits:", + "Reachable from roots:", "Not reachable from roots:"), calc_gc_stats, ) ], @@ -1118,6 +1153,14 @@ def calc_optimization_table(stats: Stats) -> Rows: for label, (value, den) in optimization_stats.items() ] + def calc_optimizer_table(stats: Stats) -> Rows: + optimizer_stats = stats.get_optimizer_stats() + + return [ + (label, Count(value), Ratio(value, den)) + for label, (value, den) in optimizer_stats.items() + ] + def calc_histogram_table(key: str, den: str) -> RowCalculator: def calc(stats: Stats) -> Rows: histogram = stats.get_histogram(key) @@ -1152,6 +1195,17 @@ def calc_unsupported_opcodes_table(stats: Stats) -> Rows: reverse=True, ) + def calc_error_in_opcodes_table(stats: Stats) -> Rows: + error_in_opcodes = stats.get_opcode_stats("error_in_opcode") + return sorted( + [ + (opcode, Count(count)) + for opcode, count in error_in_opcodes.get_opcode_counts().items() + ], + key=itemgetter(1), + reverse=True, + ) + def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None): if not base_stats.get_optimization_stats() or ( head_stats is not None and not head_stats.get_optimization_stats() @@ -1159,6 +1213,7 @@ def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None) return yield Table(("", "Count:", "Ratio:"), calc_optimization_table, JoinMode.CHANGE) + yield Table(("", "Count:", "Ratio:"), calc_optimizer_table, JoinMode.CHANGE) for name, den in [ ("Trace length", "Optimization traces created"), ("Optimized trace length", "Optimization traces created"), @@ -1186,6 +1241,7 @@ def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None) ) ], ) + yield pair_count_section(prefix="uop", title="Non-JIT uop") yield Section( "Unsupported opcodes", "", @@ -1197,6 +1253,11 @@ def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None) ) ], ) + yield Section( + "Optimizer errored out with opcode", + "Optimization stopped after encountering this opcode", + [Table(("Opcode", "Count:"), calc_error_in_opcodes_table, JoinMode.CHANGE)], + ) return Section( "Optimization (Tier 2) stats", @@ -1241,7 +1302,7 @@ def calc_rows(stats: Stats) -> Rows: LAYOUT = [ execution_count_section(), - pair_count_section(), + pair_count_section("opcode"), pre_succ_pairs_section(), specialization_section(), specialization_effectiveness_section(), diff --git a/Tools/ssl/make_ssl_data.py b/Tools/ssl/make_ssl_data.py index 98608716576792..da05d2bc8b9752 100755 --- a/Tools/ssl/make_ssl_data.py +++ b/Tools/ssl/make_ssl_data.py @@ -5,9 +5,28 @@ `library` and `reason` mnemonics to a more recent OpenSSL version. It takes two arguments: -- the path to the OpenSSL source tree (e.g. git checkout) +- the path to the OpenSSL git checkout - the path to the header file to be generated Modules/_ssl_data_{version}.h - error codes are version specific + +The OpenSSL git checkout should be at a specific tag, using commands like: + git tag --list 'openssl-*' + git switch --detach openssl-3.4.0 + + +After generating the definitions, compare the result with newest pre-existing file. +You can use a command like: + + git diff --no-index Modules/_ssl_data_31.h Modules/_ssl_data_34.h + +- If the new version *only* adds new definitions, remove the pre-existing file + and adjust the #include in _ssl.c to point to the new version. +- If the new version removes or renumbers some definitions, keep both files and + add a new #include in _ssl.c. + +A newly supported OpenSSL version should also be added to: +- Tools/ssl/multissltests.py +- .github/workflows/build.yml """ import argparse @@ -15,7 +34,7 @@ import operator import os import re -import sys +import subprocess parser = argparse.ArgumentParser( @@ -118,9 +137,17 @@ def main(): # sort by libname, numeric error code args.reasons = sorted(reasons, key=operator.itemgetter(0, 3)) + git_describe = subprocess.run( + ['git', 'describe', '--long', '--dirty'], + cwd=args.srcdir, + capture_output=True, + encoding='utf-8', + check=True, + ) lines = [ - "/* File generated by Tools/ssl/make_ssl_data.py */" - f"/* Generated on {datetime.datetime.utcnow().isoformat()} */" + "/* File generated by Tools/ssl/make_ssl_data.py */", + f"/* Generated on {datetime.datetime.now(datetime.UTC).isoformat()} */", + f"/* Generated from Git commit {git_describe.stdout.strip()} */", ] lines.extend(gen_library_codes(args)) lines.append("") diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index baa16102068aa0..2cd0c39b5a6477 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -43,13 +43,16 @@ log = logging.getLogger("multissl") OPENSSL_OLD_VERSIONS = [ + "1.1.1w", ] OPENSSL_RECENT_VERSIONS = [ - "1.1.1w", - "3.0.13", - "3.1.5", - "3.2.1", + "3.0.15", + "3.1.7", + "3.2.3", + "3.3.2", + "3.4.0", + # See make_ssl_data.py for notes on adding a new version. ] LIBRESSL_OLD_VERSIONS = [ @@ -397,6 +400,7 @@ def run_python_tests(self, tests, network=True): class BuildOpenSSL(AbstractBuilder): library = "OpenSSL" url_templates = ( + "https://github.com/openssl/openssl/releases/download/openssl-{v}/openssl-{v}.tar.gz", "https://www.openssl.org/source/openssl-{v}.tar.gz", "https://www.openssl.org/source/old/{s}/openssl-{v}.tar.gz" ) @@ -439,6 +443,7 @@ def short_version(self): parsed = parsed[:2] return ".".join(str(i) for i in parsed) + class BuildLibreSSL(AbstractBuilder): library = "LibreSSL" url_templates = ( diff --git a/Tools/tsan/suppressions.txt b/Tools/tsan/suppressions.txt new file mode 100644 index 00000000000000..22ba9d6ba2ab4d --- /dev/null +++ b/Tools/tsan/suppressions.txt @@ -0,0 +1,7 @@ +# This file contains suppressions for the default (with GIL) build. +# reference: https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions +race:get_allocator_unlocked +race:set_allocator_unlocked + +# https://gist.github.com/mpage/daaf32b39180c1989572957b943eb665 +thread:pthread_create diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt new file mode 100644 index 00000000000000..e5eb665ae212de --- /dev/null +++ b/Tools/tsan/suppressions_free_threading.txt @@ -0,0 +1,48 @@ +# This file contains suppressions for the free-threaded build. It contains the +# suppressions for the default build and additional suppressions needed only in +# the free-threaded build. +# +# reference: https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions + +## Default build suppresssions + +race:get_allocator_unlocked +race:set_allocator_unlocked + +## Free-threaded suppressions + + +# These entries are for warnings that trigger in a library function, as called +# by a CPython function. + +# https://gist.github.com/swtaarrs/08dfe7883b4c975c31ecb39388987a67 +race:free_threadstate + + +# These warnings trigger directly in a CPython function. + +race_top:assign_version_tag +race_top:new_reference +race_top:_multiprocessing_SemLock_acquire_impl +race_top:list_get_item_ref +race_top:_Py_slot_tp_getattr_hook +race_top:add_threadstate +race_top:dump_traceback +race_top:fatal_error +race_top:_multiprocessing_SemLock_release_impl +race_top:_PyFrame_GetCode +race_top:_PyFrame_Initialize +race_top:PyInterpreterState_ThreadHead +race_top:_PyObject_TryGetInstanceAttribute +race_top:PyThreadState_Next +race_top:PyUnstable_InterpreterFrame_GetLine +race_top:tstate_delete_common +race_top:tstate_is_freed +race_top:type_modified_unlocked +race_top:write_thread_id +race_top:PyThreadState_Clear +# Only seen on macOS, sample: https://gist.github.com/aisk/dda53f5d494a4556c35dde1fce03259c +race_top:set_default_allocator_unlocked + +# https://gist.github.com/mpage/6962e8870606cfc960e159b407a0cb40 +thread:pthread_create diff --git a/Tools/unicode/makeunicodedata.py b/Tools/unicode/makeunicodedata.py index a5ac09eb96f19a..889ae8fc869b8a 100644 --- a/Tools/unicode/makeunicodedata.py +++ b/Tools/unicode/makeunicodedata.py @@ -35,7 +35,7 @@ from textwrap import dedent from typing import Iterator, List, Optional, Set, Tuple -SCRIPT = sys.argv[0] +SCRIPT = os.path.normpath(sys.argv[0]) VERSION = "3.3" # The Unicode Database @@ -44,7 +44,7 @@ # * Doc/library/stdtypes.rst, and # * Doc/library/unicodedata.rst # * Doc/reference/lexical_analysis.rst (two occurrences) -UNIDATA_VERSION = "15.1.0" +UNIDATA_VERSION = "16.0.0" UNICODE_DATA = "UnicodeData%s.txt" COMPOSITION_EXCLUSIONS = "CompositionExclusions%s.txt" EASTASIAN_WIDTH = "EastAsianWidth%s.txt" diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 23b38c8e93638a..2e0fa2fb533d67 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -1,6 +1,7 @@ # Python WebAssembly (WASM) build -**WARNING: WASM support is work-in-progress! Lots of features are not working yet.** +**WASI support is [tier 2](https://peps.python.org/pep-0011/#tier-2).** +**Emscripten support is [tier 3](https://peps.python.org/pep-0011/#tier-3).** This directory contains configuration and helpers to facilitate cross compilation of CPython to WebAssembly (WASM). Python supports Emscripten @@ -20,160 +21,140 @@ https://github.com/psf/webassembly for more information. ### Build -For now the build system has two target flavors. The ``Emscripten/browser`` -target (``--with-emscripten-target=browser``) is optimized for browsers. -It comes with a reduced and preloaded stdlib without tests and threading -support. The ``Emscripten/node`` target has threading enabled and can -access the file system directly. +To cross compile to the ``wasm32-emscripten`` platform you need +[the Emscripten compiler toolchain](https://emscripten.org/), +a Python interpreter, and an installation of Node version 18 or newer. +Emscripten version 3.1.73 or newer is recommended. All commands below are +relative to a checkout of the Python repository. -Cross compiling to the wasm32-emscripten platform needs the -[Emscripten](https://emscripten.org/) SDK and a build Python interpreter. -Emscripten 3.1.19 or newer are recommended. All commands below are relative -to a repository checkout. +#### Install [the Emscripten compiler toolchain](https://emscripten.org/docs/getting_started/downloads.html) -#### Toolchain - -##### Container image - -Christian Heimes maintains a container image with Emscripten SDK, Python -build dependencies, WASI-SDK, wasmtime, and several additional tools. - -From within your local CPython repo clone, run one of the following commands: - -``` -# Fedora, RHEL, CentOS -podman run --rm -ti -v $(pwd):/python-wasm/cpython:Z -w /python-wasm/cpython quay.io/tiran/cpythonbuild:emsdk3 - -# other -docker run --rm -ti -v $(pwd):/python-wasm/cpython -w /python-wasm/cpython quay.io/tiran/cpythonbuild:emsdk3 +You can install the Emscripten toolchain as follows: +```shell +git clone https://github.com/emscripten-core/emsdk.git --depth 1 +./emsdk/emsdk install latest +./emsdk/emsdk activate latest ``` - -##### Manually - -###### Install [Emscripten SDK](https://emscripten.org/docs/getting_started/downloads.html) - -**NOTE**: Follow the on-screen instructions how to add the SDK to ``PATH``. - +To add the Emscripten compiler to your path: ```shell -git clone https://github.com/emscripten-core/emsdk.git /opt/emsdk -/opt/emsdk/emsdk install latest -/opt/emsdk/emsdk activate latest +source ./emsdk/emsdk_env.sh ``` +This adds `emcc` and `emconfigure` to your path. -###### Optionally: enable ccache for EMSDK +##### Optionally: enable ccache for EMSDK The ``EM_COMPILER_WRAPPER`` must be set after the EMSDK environment is sourced. Otherwise the source script removes the environment variable. -``` -. /opt/emsdk/emsdk_env.sh -EM_COMPILER_WRAPPER=ccache -``` - -###### Optionally: pre-build and cache static libraries - -Emscripten SDK provides static builds of core libraries without PIC -(position-independent code). Python builds with ``dlopen`` support require -PIC. To populate the build cache, run: - -```shell -. /opt/emsdk/emsdk_env.sh -embuilder build zlib bzip2 MINIMAL_PIC -embuilder --pic build zlib bzip2 MINIMAL_PIC -``` - - -### Compile and build Python interpreter - -From within the container, run the following command: - ```shell -./Tools/wasm/wasm_build.py build +export EM_COMPILER_WRAPPER=ccache ``` -The command is roughly equivalent to: +#### Compile and build Python interpreter +You can use `python Tools/wasm/emscripten` to compile and build targetting +Emscripten. You can do everything at once with: ```shell -mkdir -p builddir/build -pushd builddir/build -../../configure -C -make -j$(nproc) -popd +python Tools/wasm/emscripten build ``` - -#### Cross-compile to wasm32-emscripten for browser - +or you can break it out into four separate steps: ```shell -./Tools/wasm/wasm_build.py emscripten-browser +python Tools/wasm/emscripten configure-build-python +python Tools/wasm/emscripten make-build-python +python Tools/wasm/emscripten make-libffi +python Tools/wasm/emscripten configure-host +python Tools/wasm/emscripten make-host ``` - -The command is roughly equivalent to: - +Extra arguments to the configure steps are passed along to configure. For +instance, to do a debug build, you can use: ```shell -mkdir -p builddir/emscripten-browser -pushd builddir/emscripten-browser - -CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \ - emconfigure ../../configure -C \ - --host=wasm32-unknown-emscripten \ - --build=$(../../config.guess) \ - --with-emscripten-target=browser \ - --with-build-python=$(pwd)/../build/python - -emmake make -j$(nproc) -popd +python Tools/wasm/emscripten build --with-py-debug ``` -Serve `python.html` with a local webserver and open the file in a browser. -Python comes with a minimal web server script that sets necessary HTTP -headers like COOP, COEP, and mimetypes. Run the script outside the container -and from the root of the CPython checkout. - -```shell -./Tools/wasm/wasm_webserver.py +### Running from node + +If you want to run the normal Python CLI, you can use `python.sh`. It takes the +same options as the normal Python CLI entrypoint, though the REPL does not +function and will crash. + +`python.sh` invokes `node_entry.mjs` which imports the Emscripten module for the +Python process and starts it up with the appropriate settings. If you wish to +make a node application that "embeds" the interpreter instead of acting like the +CLI you will need to write your own alternative to `node_entry.mjs`. + + +### The Web Example + +When building for Emscripten, the web example will be built automatically. It is +in the ``web_example`` directory. To run the web example, ``cd`` into the +``web_example`` directory, then run ``python server.py``. This will start a web +server; you can then visit ``http://localhost:8000/python.html`` in a browser to +see a simple REPL example. + +The web example relies on a bug fix in Emscripten version 3.1.73 so if you build +with earlier versions of Emscripten it may not work. The web example uses +``SharedArrayBuffer``. For security reasons browsers only provide +``SharedArrayBuffer`` in secure environments with cross-origin isolation. The +webserver must send cross-origin headers and correct MIME types for the +JavaScript and WebAssembly files. Otherwise the terminal will fail to load with +an error message like ``ReferenceError: SharedArrayBuffer is not defined``. See +more information here: +https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer#security_requirements + +Note that ``SharedArrayBuffer`` is _not required_ to use Python itself, only the +web example. If cross-origin isolation is not appropriate for your use case you +may make your own application embedding `python.mjs` which does not use +``SharedArrayBuffer`` and serve it without the cross-origin isolation headers. + +### Embedding Python in a custom JavaScript application + +You can look at `python.worker.mjs` and `node_entry.mjs` for inspiration. At a +minimum you must import ``createEmscriptenModule`` and you need to call +``createEmscriptenModule`` with an appropriate settings object. This settings +object will need a prerun hook that installs the Python standard library into +the Emscripten file system. + +#### NodeJs + +In Node, you can use the NodeFS to mount the standard library in your native +file system into the Emscripten file system: +```js +import createEmscriptenModule from "./python.mjs"; + +await createEmscriptenModule({ + preRun(Module) { + Module.FS.mount( + Module.FS.filesystems.NODEFS, + { root: "/path/to/python/stdlib" }, + "/lib/", + ); + }, +}); ``` -and open http://localhost:8000/builddir/emscripten-browser/python.html . This -directory structure enables the *C/C++ DevTools Support (DWARF)* to load C -and header files with debug builds. - - -#### Cross compile to wasm32-emscripten for node - -```shell -./Tools/wasm/wasm_build.py emscripten-node-dl +#### Browser + +In the browser, the simplest approach is to put the standard library in a zip +file it and install it. With Python 3.14 this could look like: +```js +import createEmscriptenModule from "./python.mjs"; + +await createEmscriptenModule({ + async preRun(Module) { + Module.FS.mkdirTree("/lib/python3.14/lib-dynload/"); + Module.addRunDependency("install-stdlib"); + const resp = await fetch("python3.14.zip"); + const stdlibBuffer = await resp.arrayBuffer(); + Module.FS.writeFile(`/lib/python314.zip`, new Uint8Array(stdlibBuffer), { + canOwn: true, + }); + Module.removeRunDependency("install-stdlib"); + }, +}); ``` -The command is roughly equivalent to: - -```shell -mkdir -p builddir/emscripten-node-dl -pushd builddir/emscripten-node-dl - -CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \ - emconfigure ../../configure -C \ - --host=wasm32-unknown-emscripten \ - --build=$(../../config.guess) \ - --with-emscripten-target=node \ - --enable-wasm-dynamic-linking \ - --with-build-python=$(pwd)/../build/python - -emmake make -j$(nproc) -popd -``` - -```shell -node --experimental-wasm-threads --experimental-wasm-bulk-memory --experimental-wasm-bigint builddir/emscripten-node-dl/python.js -``` - -(``--experimental-wasm-bigint`` is not needed with recent NodeJS versions) - ### Limitations and issues -Emscripten before 3.1.8 has known bugs that can cause memory corruption and -resource leaks. 3.1.8 contains several fixes for bugs in date and time -functions. - #### Network stack - Python's socket module does not work with Emscripten's emulated POSIX @@ -240,8 +221,6 @@ functions. [gh-90548](https://github.com/python/cpython/issues/90548). - Python's object allocator ``obmalloc`` is disabled by default. - ``ensurepip`` is not available. -- Some ``ctypes`` features like ``c_longlong`` and ``c_longdouble`` may need - NodeJS option ``--experimental-wasm-bigint``. #### In the browser @@ -255,109 +234,9 @@ functions. - Test modules are disabled by default. Use ``--enable-test-modules`` build test modules like ``_testcapi``. -### wasm32-emscripten in node - -Node builds use ``NODERAWFS``. - -- Node RawFS allows direct access to the host file system without need to - perform ``FS.mount()`` call. - -### wasm64-emscripten - -- wasm64 requires recent NodeJS and ``--experimental-wasm-memory64``. -- ``EM_JS`` functions must return ``BigInt()``. -- ``Py_BuildValue()`` format strings must match size of types. Confusing 32 - and 64 bits types leads to memory corruption, see - [gh-95876](https://github.com/python/cpython/issues/95876) and - [gh-95878](https://github.com/python/cpython/issues/95878). - -### Hosting Python WASM builds - -The simple REPL terminal uses SharedArrayBuffer. For security reasons -browsers only provide the feature in secure environents with cross-origin -isolation. The webserver must send cross-origin headers and correct MIME types -for the JavaScript and WASM files. Otherwise the terminal will fail to load -with an error message like ``Browsers disable shared array buffer``. - -#### Apache HTTP .htaccess - -Place a ``.htaccess`` file in the same directory as ``python.wasm``. - -``` -# .htaccess -Header set Cross-Origin-Opener-Policy same-origin -Header set Cross-Origin-Embedder-Policy require-corp - -AddType application/javascript js -AddType application/wasm wasm - - - AddOutputFilterByType DEFLATE text/html application/javascript application/wasm - -``` - ## WASI (wasm32-wasi) -**NOTE**: The instructions below assume a Unix-based OS due to cross-compilation for CPython being set up for `./configure`. - -### Prerequisites - -Developing for WASI requires two additional tools to be installed beyond the typical tools required to build CPython: - -1. The [WASI SDK](https://github.com/WebAssembly/wasi-sdk) 16.0+ -2. A WASI host/runtime ([wasmtime](https://wasmtime.dev) 14+ is recommended and what the instructions below assume) - -All of this is provided in the [devcontainer](https://devguide.python.org/getting-started/setup-building/#contribute-using-github-codespaces) if you don't want to install these tools locally. - -### Building - -Building for WASI requires doing a cross-build where you have a "build" Python to help produce a WASI build of CPython (technically it's a "host x host" cross-build because the build Python is also the target Python while the host build is the WASI build; yes, it's confusing terminology). In the end you should have a build Python in `cross-build/build` and a WASI build in `cross-build/wasm32-wasi`. - -The easiest way to do a build is to use the `wasi.py` script. You can either have it perform the entire build process from start to finish in one step, or you can do it in discrete steps that mirror running `configure` and `make` for each of the two builds of Python you end up producing (which are beneficial when you only need to do a specific step after getting a complete build, e.g. editing some code and you just need to run `make` for the WASI build). The script is designed to self-document what actions it is performing on your behalf, both as a way to check its work but also for educaitonal purposes. - -The discrete steps for building via `wasi.py` are: -```shell -python Tools/wasm/wasi.py configure-build-python -python Tools/wasm/wasi.py make-build-python -python Tools/wasm/wasi.py configure-host -python Tools/wasm/wasi.py make-host -``` - -To do it all in a single command, run: -```shell -python Tools/wasm/wasi.py build -``` - -That will: - -1. Run `configure` for the build Python (same as `wasi.py configure-build-python`) -2. Run `make` for the build Python (`wasi.py make-build-python`) -3. Run `configure` for the WASI build (`wasi.py configure-host`) -4. Run `make` for the WASI build (`wasi.py make-host`) - -See the `--help` for the various options available for each of the subcommands which controls things like the location of the WASI SDK, the command to use with the WASI host/runtime, etc. Also note that you can use `--` as a separator for any of the `configure`-related commands -- including `build` itself -- to pass arguments to the underlying `configure` call. For example, if you want a pydebug build that also caches the results from `configure`, you can do: -```shell -python Tools/wasm/wasi.py build -- -C --with-pydebug -``` - -The `wasi.py` script is able to infer details from the build Python, and so you only technically need to specify `--with-pydebug` once via `configure-build-python` as this will lead to `configure-host` detecting its use if you use the discrete steps: -```shell -python Tools/wasm/wasi.py configure-build-python -- -C --with-pydebug -python Tools/wasm/wasi.py make-build-python -python Tools/wasm/wasi.py configure-host -- -C -python Tools/wasm/wasi.py make-host -``` - - -### Running - -If you used `wasi.py` to do your build then there will be a `cross-build/wasm32-wasi/python.sh` file which you can use to run the `python.wasm` file (see the output from the `configure-host` subcommand): -```shell -cross-build/wasm32-wasi/python.sh --version -``` - -While you _can_ run `python.wasm` directly, Python will fail to start up without certain things being set (e.g. `PYTHONPATH` for `sysconfig` data). As such, the `python.sh` file records these details for you. - +See [the devguide on how to build and run for WASI](https://devguide.python.org/getting-started/setup-building/#wasi). ## Detecting WebAssembly builds diff --git a/Tools/wasm/Setup.local.example b/Tools/wasm/Setup.local.example deleted file mode 100644 index 7b2fb13f6ceef2..00000000000000 --- a/Tools/wasm/Setup.local.example +++ /dev/null @@ -1,13 +0,0 @@ -# Module/Setup.local with reduced stdlib -*disabled* -_asyncio -_bz2 -_decimal -_pickle -pyexpat _elementtree -_sha3 _blake2 -_zoneinfo -xxsubtype - -# cjk codecs -#_multibytecodec _codecs_cn _codecs_hk _codecs_iso2022 _codecs_jp _codecs_kr _codecs_tw diff --git a/Tools/wasm/build_wasi.sh b/Tools/wasm/build_wasi.sh deleted file mode 100755 index 436306222ce1d0..00000000000000 --- a/Tools/wasm/build_wasi.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/bash - -set -e -x - -# Quick check to avoid running configure just to fail in the end. -if [ -f Programs/python.o ]; then - echo "Can't do an out-of-tree build w/ an in-place build pre-existing (i.e., found Programs/python.o)" >&2 - exit 1 -fi - -if [ ! -f Modules/Setup.local ]; then - touch Modules/Setup.local -fi - -# TODO: check if `make` and `pkgconfig` are installed -# TODO: detect if wasmtime is installed - -# Create the "build" Python. -mkdir -p builddir/build -pushd builddir/build -../../configure -C -make -s -j 4 all -export PYTHON_VERSION=`./python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")'` -popd - -# Create the "host"/WASI Python. -export CONFIG_SITE="$(pwd)/Tools/wasm/config.site-wasm32-wasi" -export HOSTRUNNER="wasmtime run --mapdir /::$(pwd) --env PYTHONPATH=/builddir/wasi/build/lib.wasi-wasm32-$PYTHON_VERSION $(pwd)/builddir/wasi/python.wasm --" - -mkdir -p builddir/wasi -pushd builddir/wasi -../../Tools/wasm/wasi-env \ - ../../configure \ - -C \ - --host=wasm32-unknown-wasi \ - --build=$(../../config.guess) \ - --with-build-python=../build/python -make -s -j 4 all -# Create a helper script for executing the host/WASI Python. -printf "#!/bin/sh\nexec $HOSTRUNNER \"\$@\"\n" > run_wasi.sh -chmod 755 run_wasi.sh -./run_wasi.sh --version -popd - diff --git a/Tools/wasm/config.site-wasm32-wasi b/Tools/wasm/config.site-wasm32-wasi index 4a1a466a4ab3f1..c5d8b3e205db26 100644 --- a/Tools/wasm/config.site-wasm32-wasi +++ b/Tools/wasm/config.site-wasm32-wasi @@ -49,3 +49,11 @@ ac_cv_func_preadv=no ac_cv_func_readv=no ac_cv_func_pwritev=no ac_cv_func_writev=no + +# WASI SDK 22 added multiple stubs which we don't implement. +# https://github.com/python/cpython/issues/120371 +ac_cv_func_chmod=no +ac_cv_func_fchmod=no +ac_cv_func_fchmodat=no +ac_cv_func_statvfs=no +ac_cv_func_fstatvfs=no diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py new file mode 100644 index 00000000000000..4a53e0bd1bee1b --- /dev/null +++ b/Tools/wasm/emscripten/__main__.py @@ -0,0 +1,398 @@ +#!/usr/bin/env python3 + +import argparse +import contextlib +import functools +import os +import shutil +import subprocess +import sys +import sysconfig +import tempfile +from urllib.request import urlopen +from pathlib import Path +from textwrap import dedent + +try: + from os import process_cpu_count as cpu_count +except ImportError: + from os import cpu_count + + +EMSCRIPTEN_DIR = Path(__file__).parent +CHECKOUT = EMSCRIPTEN_DIR.parent.parent.parent + +CROSS_BUILD_DIR = CHECKOUT / "cross-build" +NATIVE_BUILD_DIR = CROSS_BUILD_DIR / "build" +HOST_TRIPLE = "wasm32-emscripten" + +DOWNLOAD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build" +HOST_BUILD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build" +HOST_DIR = HOST_BUILD_DIR / "python" +PREFIX_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "prefix" + +LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" +LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/emscripten.py\n".encode("utf-8") + + +def updated_env(updates={}): + """Create a new dict representing the environment to use. + + The changes made to the execution environment are printed out. + """ + env_defaults = {} + # https://reproducible-builds.org/docs/source-date-epoch/ + git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] + try: + epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() + env_defaults["SOURCE_DATE_EPOCH"] = epoch + except subprocess.CalledProcessError: + pass # Might be building from a tarball. + # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. + environment = env_defaults | os.environ | updates + + env_diff = {} + for key, value in environment.items(): + if os.environ.get(key) != value: + env_diff[key] = value + + print("🌎 Environment changes:") + for key in sorted(env_diff.keys()): + print(f" {key}={env_diff[key]}") + + return environment + + +def subdir(working_dir, *, clean_ok=False): + """Decorator to change to a working directory.""" + + def decorator(func): + @functools.wraps(func) + def wrapper(context): + try: + tput_output = subprocess.check_output( + ["tput", "cols"], encoding="utf-8" + ) + terminal_width = int(tput_output.strip()) + except subprocess.CalledProcessError: + terminal_width = 80 + print("⎯" * terminal_width) + print("📁", working_dir) + if clean_ok and getattr(context, "clean", False) and working_dir.exists(): + print("🚮 Deleting directory (--clean)...") + shutil.rmtree(working_dir) + + working_dir.mkdir(parents=True, exist_ok=True) + + with contextlib.chdir(working_dir): + return func(context, working_dir) + + return wrapper + + return decorator + + +def call(command, *, quiet, **kwargs): + """Execute a command. + + If 'quiet' is true, then redirect stdout and stderr to a temporary file. + """ + print("❯", " ".join(map(str, command))) + if not quiet: + stdout = None + stderr = None + else: + stdout = tempfile.NamedTemporaryFile( + "w", + encoding="utf-8", + delete=False, + prefix="cpython-emscripten-", + suffix=".log", + ) + stderr = subprocess.STDOUT + print(f"📝 Logging output to {stdout.name} (--quiet)...") + + subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) + + +def build_platform(): + """The name of the build/host platform.""" + # Can also be found via `config.guess`.` + return sysconfig.get_config_var("BUILD_GNU_TYPE") + + +def build_python_path(): + """The path to the build Python binary.""" + binary = NATIVE_BUILD_DIR / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError("Unable to find `python(.exe)` in " f"{NATIVE_BUILD_DIR}") + + return binary + + +@subdir(NATIVE_BUILD_DIR, clean_ok=True) +def configure_build_python(context, working_dir): + """Configure the build/host Python.""" + if LOCAL_SETUP.exists(): + print(f"👍 {LOCAL_SETUP} exists ...") + else: + print(f"📝 Touching {LOCAL_SETUP} ...") + LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) + + configure = [os.path.relpath(CHECKOUT / "configure", working_dir)] + if context.args: + configure.extend(context.args) + + call(configure, quiet=context.quiet) + + +@subdir(NATIVE_BUILD_DIR) +def make_build_python(context, working_dir): + """Make/build the build Python.""" + call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet) + + binary = build_python_path() + cmd = [ + binary, + "-c", + "import sys; " "print(f'{sys.version_info.major}.{sys.version_info.minor}')", + ] + version = subprocess.check_output(cmd, encoding="utf-8").strip() + + print(f"🎉 {binary} {version}") + + +@subdir(HOST_BUILD_DIR, clean_ok=True) +def make_emscripten_libffi(context, working_dir): + shutil.rmtree(working_dir / "libffi-3.4.6", ignore_errors=True) + with tempfile.NamedTemporaryFile(suffix=".tar.gz") as tmp_file: + with urlopen( + "https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz" + ) as response: + shutil.copyfileobj(response, tmp_file) + shutil.unpack_archive(tmp_file.name, working_dir) + call( + [EMSCRIPTEN_DIR / "make_libffi.sh"], + env=updated_env({"PREFIX": PREFIX_DIR}), + cwd=working_dir / "libffi-3.4.6", + quiet=context.quiet, + ) + + +@subdir(HOST_DIR, clean_ok=True) +def configure_emscripten_python(context, working_dir): + """Configure the emscripten/host build.""" + config_site = os.fsdecode( + CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-emscripten" + ) + + emscripten_build_dir = working_dir.relative_to(CHECKOUT) + + python_build_dir = NATIVE_BUILD_DIR / "build" + lib_dirs = list(python_build_dir.glob("lib.*")) + assert ( + len(lib_dirs) == 1 + ), f"Expected a single lib.* directory in {python_build_dir}" + lib_dir = os.fsdecode(lib_dirs[0]) + pydebug = lib_dir.endswith("-pydebug") + python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] + sysconfig_data = ( + f"{emscripten_build_dir}/build/lib.emscripten-wasm32-{python_version}" + ) + if pydebug: + sysconfig_data += "-pydebug" + + host_runner = context.host_runner + pkg_config_path_dir = (PREFIX_DIR / "lib/pkgconfig/").resolve() + env_additions = { + "CONFIG_SITE": config_site, + "HOSTRUNNER": host_runner, + "EM_PKG_CONFIG_PATH": str(pkg_config_path_dir), + } + build_python = os.fsdecode(build_python_path()) + configure = [ + "emconfigure", + os.path.relpath(CHECKOUT / "configure", working_dir), + "CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2", + "PKG_CONFIG=pkg-config", + f"--host={HOST_TRIPLE}", + f"--build={build_platform()}", + f"--with-build-python={build_python}", + "--without-pymalloc", + "--disable-shared", + "--disable-ipv6", + "--enable-big-digits=30", + "--enable-wasm-dynamic-linking", + f"--prefix={PREFIX_DIR}", + ] + if pydebug: + configure.append("--with-pydebug") + if context.args: + configure.extend(context.args) + call( + configure, + env=updated_env(env_additions), + quiet=context.quiet, + ) + + shutil.copy(EMSCRIPTEN_DIR / "node_entry.mjs", working_dir / "node_entry.mjs") + + node_entry = working_dir / "node_entry.mjs" + exec_script = working_dir / "python.sh" + exec_script.write_text( + dedent( + f"""\ + #!/bin/sh + + # Macs come with FreeBSD coreutils which doesn't have the -s option + # so feature detect and work around it. + if which grealpath > /dev/null; then + # It has brew installed gnu core utils, use that + REALPATH="grealpath -s" + elif which realpath > /dev/null && realpath --version > /dev/null 2> /dev/null && realpath --version | grep GNU > /dev/null; then + # realpath points to GNU realpath so use it. + REALPATH="realpath -s" + else + # Shim for macs without GNU coreutils + abs_path () {{ + echo "$(cd "$(dirname "$1")" || exit; pwd)/$(basename "$1")" + }} + REALPATH=abs_path + fi + + # We compute our own path, not following symlinks and pass it in so that + # node_entry.mjs can set sys.executable correctly. + # Intentionally allow word splitting on NODEFLAGS. + exec {host_runner} $NODEFLAGS {node_entry} --this-program="$($REALPATH "$0")" "$@" + """ + ) + ) + exec_script.chmod(0o755) + print(f"🏃‍♀️ Created {exec_script} ... ") + sys.stdout.flush() + + +@subdir(HOST_DIR) +def make_emscripten_python(context, working_dir): + """Run `make` for the emscripten/host build.""" + call( + ["make", "--jobs", str(cpu_count()), "all"], + env=updated_env(), + quiet=context.quiet, + ) + + exec_script = working_dir / "python.sh" + subprocess.check_call([exec_script, "--version"]) + + +def build_all(context): + """Build everything.""" + steps = [ + configure_build_python, + make_build_python, + make_emscripten_libffi, + configure_emscripten_python, + make_emscripten_python, + ] + for step in steps: + step(context) + + +def clean_contents(context): + """Delete all files created by this script.""" + if CROSS_BUILD_DIR.exists(): + print(f"🧹 Deleting {CROSS_BUILD_DIR} ...") + shutil.rmtree(CROSS_BUILD_DIR) + + if LOCAL_SETUP.exists(): + with LOCAL_SETUP.open("rb") as file: + if file.read(len(LOCAL_SETUP_MARKER)) == LOCAL_SETUP_MARKER: + print(f"🧹 Deleting generated {LOCAL_SETUP} ...") + + +def main(): + default_host_runner = "node" + + parser = argparse.ArgumentParser() + subcommands = parser.add_subparsers(dest="subcommand") + build = subcommands.add_parser("build", help="Build everything") + configure_build = subcommands.add_parser( + "configure-build-python", help="Run `configure` for the " "build Python" + ) + make_libffi_cmd = subcommands.add_parser( + "make-libffi", help="Clone libffi repo, configure and build it for emscripten" + ) + make_build = subcommands.add_parser( + "make-build-python", help="Run `make` for the build Python" + ) + configure_host = subcommands.add_parser( + "configure-host", + help="Run `configure` for the host/emscripten (pydebug builds are inferred from the build Python)", + ) + make_host = subcommands.add_parser( + "make-host", help="Run `make` for the host/emscripten" + ) + clean = subcommands.add_parser( + "clean", help="Delete files and directories created by this script" + ) + for subcommand in ( + build, + configure_build, + make_libffi_cmd, + make_build, + configure_host, + make_host, + ): + subcommand.add_argument( + "--quiet", + action="store_true", + default=False, + dest="quiet", + help="Redirect output from subprocesses to a log file", + ) + for subcommand in configure_build, configure_host: + subcommand.add_argument( + "--clean", + action="store_true", + default=False, + dest="clean", + help="Delete any relevant directories before building", + ) + for subcommand in build, configure_build, configure_host: + subcommand.add_argument( + "args", nargs="*", help="Extra arguments to pass to `configure`" + ) + for subcommand in build, configure_host: + subcommand.add_argument( + "--host-runner", + action="store", + default=default_host_runner, + dest="host_runner", + help="Command template for running the emscripten host" + f"`{default_host_runner}`)", + ) + + context = parser.parse_args() + + dispatch = { + "make-libffi": make_emscripten_libffi, + "configure-build-python": configure_build_python, + "make-build-python": make_build_python, + "configure-host": configure_emscripten_python, + "make-host": make_emscripten_python, + "build": build_all, + "clean": clean_contents, + } + + if not context.subcommand: + # No command provided, display help and exit + print("Expected one of", ", ".join(sorted(dispatch.keys())), file=sys.stderr) + parser.print_help(sys.stderr) + sys.exit(1) + dispatch[context.subcommand](context) + + +if __name__ == "__main__": + main() diff --git a/Tools/wasm/emscripten/make_libffi.sh b/Tools/wasm/emscripten/make_libffi.sh new file mode 100755 index 00000000000000..3c75c4d5127102 --- /dev/null +++ b/Tools/wasm/emscripten/make_libffi.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set +e + +export CFLAGS="-O2 -fPIC -DWASM_BIGINT" +export CXXFLAGS="$CFLAGS" + +# Build paths +export CPATH="$PREFIX/include" +export PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig" +export EM_PKG_CONFIG_PATH="$PKG_CONFIG_PATH" + +# Specific variables for cross-compilation +export CHOST="wasm32-unknown-linux" # wasm32-unknown-emscripten + +emconfigure ./configure --host=$CHOST --prefix="$PREFIX" --enable-static --disable-shared --disable-dependency-tracking \ + --disable-builddir --disable-multi-os-directory --disable-raw-api --disable-docs + +make install +# Some forgotten headers? +cp fficonfig.h $PREFIX/include/ +cp include/ffi_common.h $PREFIX/include/ diff --git a/Tools/wasm/emscripten/node_entry.mjs b/Tools/wasm/emscripten/node_entry.mjs new file mode 100644 index 00000000000000..98b8f572a7e762 --- /dev/null +++ b/Tools/wasm/emscripten/node_entry.mjs @@ -0,0 +1,48 @@ +import EmscriptenModule from "./python.mjs"; +import fs from "node:fs"; + +if (process?.versions?.node) { + const nodeVersion = Number(process.versions.node.split(".", 1)[0]); + if (nodeVersion < 18) { + process.stderr.write( + `Node version must be >= 18, got version ${process.version}\n`, + ); + process.exit(1); + } +} + +function rootDirsToMount(Module) { + return fs + .readdirSync("/") + .filter((dir) => !["dev", "lib", "proc"].includes(dir)) + .map((dir) => "/" + dir); +} + +function mountDirectories(Module) { + for (const dir of rootDirsToMount(Module)) { + Module.FS.mkdirTree(dir); + Module.FS.mount(Module.FS.filesystems.NODEFS, { root: dir }, dir); + } +} + +const thisProgram = "--this-program="; +const thisProgramIndex = process.argv.findIndex((x) => + x.startsWith(thisProgram), +); + +const settings = { + preRun(Module) { + mountDirectories(Module); + Module.FS.chdir(process.cwd()); + Object.assign(Module.ENV, process.env); + delete Module.ENV.PATH; + }, + // Ensure that sys.executable, sys._base_executable, etc point to python.sh + // not to this file. To properly handle symlinks, python.sh needs to compute + // its own path. + thisProgram: process.argv[thisProgramIndex].slice(thisProgram.length), + // After python.sh come the arguments thatthe user passed to python.sh. + arguments: process.argv.slice(thisProgramIndex + 1), +}; + +await EmscriptenModule(settings); diff --git a/Tools/wasm/emscripten/web_example/python.html b/Tools/wasm/emscripten/web_example/python.html new file mode 100644 index 00000000000000..078f86eb764419 --- /dev/null +++ b/Tools/wasm/emscripten/web_example/python.html @@ -0,0 +1,433 @@ + + + + + + + + + wasm-python terminal + + + + + +

    Simple REPL for Python WASM

    + +
    + + + + +
    +
    +
    + The simple REPL provides a limited Python experience in the browser. + + Tools/wasm/README.md + + contains a list of known limitations and issues. Networking, + subprocesses, and threading are not available. +
    + + diff --git a/Tools/wasm/emscripten/web_example/python.worker.mjs b/Tools/wasm/emscripten/web_example/python.worker.mjs new file mode 100644 index 00000000000000..8043e419966743 --- /dev/null +++ b/Tools/wasm/emscripten/web_example/python.worker.mjs @@ -0,0 +1,109 @@ +import createEmscriptenModule from "./python.mjs"; + +class StdinBuffer { + constructor() { + this.sab = new SharedArrayBuffer(128 * Int32Array.BYTES_PER_ELEMENT); + this.buffer = new Int32Array(this.sab); + this.readIndex = 1; + this.numberOfCharacters = 0; + this.sentNull = true; + } + + prompt() { + this.readIndex = 1; + Atomics.store(this.buffer, 0, -1); + postMessage({ + type: "stdin", + buffer: this.sab, + }); + Atomics.wait(this.buffer, 0, -1); + this.numberOfCharacters = this.buffer[0]; + } + + stdin = () => { + while (this.numberOfCharacters + 1 === this.readIndex) { + if (!this.sentNull) { + // Must return null once to indicate we're done for now. + this.sentNull = true; + return null; + } + this.sentNull = false; + // Prompt will reset this.readIndex to 1 + this.prompt(); + } + const char = this.buffer[this.readIndex]; + this.readIndex += 1; + return char; + }; +} + +const stdout = (charCode) => { + if (charCode) { + postMessage({ + type: "stdout", + stdout: charCode, + }); + } else { + console.log(typeof charCode, charCode); + } +}; + +const stderr = (charCode) => { + if (charCode) { + postMessage({ + type: "stderr", + stderr: charCode, + }); + } else { + console.log(typeof charCode, charCode); + } +}; + +const stdinBuffer = new StdinBuffer(); + +const emscriptenSettings = { + noInitialRun: true, + stdin: stdinBuffer.stdin, + stdout: stdout, + stderr: stderr, + onRuntimeInitialized: () => { + postMessage({ type: "ready", stdinBuffer: stdinBuffer.sab }); + }, + async preRun(Module) { + const versionHex = Module.HEAPU32[Module._Py_Version / 4].toString(16); + const versionTuple = versionHex + .padStart(8, "0") + .match(/.{1,2}/g) + .map((x) => parseInt(x, 16)); + const [major, minor, ..._] = versionTuple; + // Prevent complaints about not finding exec-prefix by making a lib-dynload directory + Module.FS.mkdirTree(`/lib/python${major}.${minor}/lib-dynload/`); + Module.addRunDependency("install-stdlib"); + const resp = await fetch(`python${major}.${minor}.zip`); + const stdlibBuffer = await resp.arrayBuffer(); + Module.FS.writeFile( + `/lib/python${major}${minor}.zip`, + new Uint8Array(stdlibBuffer), + { canOwn: true }, + ); + Module.removeRunDependency("install-stdlib"); + }, +}; + +const modulePromise = createEmscriptenModule(emscriptenSettings); + +onmessage = async (event) => { + if (event.data.type === "run") { + const Module = await modulePromise; + if (event.data.files) { + for (const [filename, contents] of Object.entries(event.data.files)) { + Module.FS.writeFile(filename, contents); + } + } + const ret = Module.callMain(event.data.args); + postMessage({ + type: "finished", + returnCode: ret, + }); + } +}; diff --git a/Tools/wasm/wasm_webserver.py b/Tools/wasm/emscripten/web_example/server.py similarity index 85% rename from Tools/wasm/wasm_webserver.py rename to Tools/wasm/emscripten/web_example/server.py index 3d1d5d42a1e8c4..768e6f84e07798 100755 --- a/Tools/wasm/wasm_webserver.py +++ b/Tools/wasm/emscripten/web_example/server.py @@ -14,13 +14,6 @@ class MyHTTPRequestHandler(server.SimpleHTTPRequestHandler): - extensions_map = server.SimpleHTTPRequestHandler.extensions_map.copy() - extensions_map.update( - { - ".wasm": "application/wasm", - } - ) - def end_headers(self) -> None: self.send_my_headers() super().end_headers() @@ -42,5 +35,6 @@ def main() -> None: bind=args.bind, ) + if __name__ == "__main__": main() diff --git a/Tools/wasm/wasm_assets.py b/Tools/wasm/emscripten/web_example/wasm_assets.py similarity index 91% rename from Tools/wasm/wasm_assets.py rename to Tools/wasm/emscripten/web_example/wasm_assets.py index ffa5e303412c46..7f0fa7ae7c10ec 100755 --- a/Tools/wasm/wasm_assets.py +++ b/Tools/wasm/emscripten/web_example/wasm_assets.py @@ -19,7 +19,7 @@ from typing import Dict # source directory -SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute() +SRCDIR = pathlib.Path(__file__).parents[4].absolute() SRCDIR_LIB = SRCDIR / "Lib" @@ -28,9 +28,7 @@ WASM_STDLIB_ZIP = ( WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip" ) -WASM_STDLIB = ( - WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}" -) +WASM_STDLIB = WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}" WASM_DYNLOAD = WASM_STDLIB / "lib-dynload" @@ -114,9 +112,7 @@ def get_sysconfigdata(args: argparse.Namespace) -> pathlib.Path: assert isinstance(args.builddir, pathlib.Path) data_name: str = sysconfig._get_sysconfigdata_name() # type: ignore[attr-defined] if not data_name.startswith(SYSCONFIG_NAMES): - raise ValueError( - f"Invalid sysconfig data name '{data_name}'.", SYSCONFIG_NAMES - ) + raise ValueError(f"Invalid sysconfig data name '{data_name}'.", SYSCONFIG_NAMES) filename = data_name + ".py" return args.builddir / filename @@ -131,7 +127,7 @@ def filterfunc(filename: str) -> bool: return pathname not in args.omit_files_absolute with zipfile.PyZipFile( - args.wasm_stdlib_zip, + args.output, mode="w", compression=args.compression, optimize=optimize, @@ -195,6 +191,12 @@ def path(val: str) -> pathlib.Path: default=pathlib.Path("/usr/local"), type=path, ) +parser.add_argument( + "-o", + "--output", + help="output file", + type=path, +) def main() -> None: @@ -204,7 +206,6 @@ def main() -> None: args.srcdir = SRCDIR args.srcdir_lib = SRCDIR_LIB args.wasm_root = args.buildroot / relative_prefix - args.wasm_stdlib_zip = args.wasm_root / WASM_STDLIB_ZIP args.wasm_stdlib = args.wasm_root / WASM_STDLIB args.wasm_dynload = args.wasm_root / WASM_DYNLOAD @@ -234,12 +235,10 @@ def main() -> None: args.wasm_dynload.mkdir(parents=True, exist_ok=True) marker = args.wasm_dynload / ".empty" marker.touch() - # os.py is a marker for finding the correct lib directory. - shutil.copy(args.srcdir_lib / "os.py", args.wasm_stdlib) # The rest of stdlib that's useful in a WASM context. create_stdlib_zip(args) - size = round(args.wasm_stdlib_zip.stat().st_size / 1024**2, 2) - parser.exit(0, f"Created {args.wasm_stdlib_zip} ({size} MiB)\n") + size = round(args.output.stat().st_size / 1024**2, 2) + parser.exit(0, f"Created {args.output} ({size} MiB)\n") if __name__ == "__main__": diff --git a/Tools/wasm/python.html b/Tools/wasm/python.html deleted file mode 100644 index 17ffa0ea8bfeff..00000000000000 --- a/Tools/wasm/python.html +++ /dev/null @@ -1,316 +0,0 @@ - - - - - - - - - wasm-python terminal - - - - - - -

    Simple REPL for Python WASM

    -
    -
    - - -
    -
    - The simple REPL provides a limited Python experience in the browser. - - Tools/wasm/README.md contains a list of known limitations and - issues. Networking, subprocesses, and threading are not available. -
    - - diff --git a/Tools/wasm/python.worker.js b/Tools/wasm/python.worker.js deleted file mode 100644 index 1b794608fffe7b..00000000000000 --- a/Tools/wasm/python.worker.js +++ /dev/null @@ -1,83 +0,0 @@ -class StdinBuffer { - constructor() { - this.sab = new SharedArrayBuffer(128 * Int32Array.BYTES_PER_ELEMENT) - this.buffer = new Int32Array(this.sab) - this.readIndex = 1; - this.numberOfCharacters = 0; - this.sentNull = true - } - - prompt() { - this.readIndex = 1 - Atomics.store(this.buffer, 0, -1) - postMessage({ - type: 'stdin', - buffer: this.sab - }) - Atomics.wait(this.buffer, 0, -1) - this.numberOfCharacters = this.buffer[0] - } - - stdin = () => { - if (this.numberOfCharacters + 1 === this.readIndex) { - if (!this.sentNull) { - // Must return null once to indicate we're done for now. - this.sentNull = true - return null - } - this.sentNull = false - this.prompt() - } - const char = this.buffer[this.readIndex] - this.readIndex += 1 - // How do I send an EOF?? - return char - } -} - -const stdout = (charCode) => { - if (charCode) { - postMessage({ - type: 'stdout', - stdout: charCode, - }) - } else { - console.log(typeof charCode, charCode) - } -} - -const stderr = (charCode) => { - if (charCode) { - postMessage({ - type: 'stderr', - stderr: charCode, - }) - } else { - console.log(typeof charCode, charCode) - } -} - -const stdinBuffer = new StdinBuffer() - -var Module = { - noInitialRun: true, - stdin: stdinBuffer.stdin, - stdout: stdout, - stderr: stderr, - onRuntimeInitialized: () => { - postMessage({type: 'ready', stdinBuffer: stdinBuffer.sab}) - } -} - -onmessage = (event) => { - if (event.data.type === 'run') { - // TODO: Set up files from event.data.files - const ret = callMain(event.data.args) - postMessage({ - type: 'finished', - returnCode: ret - }) - } -} - -importScripts('python.js') diff --git a/Tools/wasm/wasi-env b/Tools/wasm/wasi-env index e6c6fb2d8e47e7..4c5078a1f675e2 100755 --- a/Tools/wasm/wasi-env +++ b/Tools/wasm/wasi-env @@ -1,6 +1,8 @@ #!/bin/sh set -e +# NOTE: to be removed once no longer used in https://github.com/python/buildmaster-config/blob/main/master/custom/factories.py . + # function usage() { echo "wasi-env - Run command with WASI-SDK" @@ -71,5 +73,5 @@ export CFLAGS LDFLAGS export PKG_CONFIG_PATH PKG_CONFIG_LIBDIR PKG_CONFIG_SYSROOT_DIR export PATH -# no exec, it makes arvg[0] path absolute. +# no exec, it makes argv[0] path absolute. "$@" diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 1e75db5c7b8329..da847c4ff86215 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -20,12 +20,13 @@ CROSS_BUILD_DIR = CHECKOUT / "cross-build" BUILD_DIR = CROSS_BUILD_DIR / "build" -HOST_TRIPLE = "wasm32-wasi" -HOST_DIR = CROSS_BUILD_DIR / HOST_TRIPLE LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/wasi.py\n".encode("utf-8") +WASMTIME_VAR_NAME = "WASMTIME" +WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}" + def updated_env(updates={}): """Create a new dict representing the environment to use. @@ -60,12 +61,17 @@ def subdir(working_dir, *, clean_ok=False): def decorator(func): @functools.wraps(func) def wrapper(context): + nonlocal working_dir + + if callable(working_dir): + working_dir = working_dir(context) try: tput_output = subprocess.check_output(["tput", "cols"], encoding="utf-8") - terminal_width = int(tput_output.strip()) except subprocess.CalledProcessError: terminal_width = 80 + else: + terminal_width = int(tput_output.strip()) print("⎯" * terminal_width) print("📁", working_dir) if (clean_ok and getattr(context, "clean", False) and @@ -190,7 +196,7 @@ def wasi_sdk_env(context): return env -@subdir(HOST_DIR, clean_ok=True) +@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True) def configure_wasi_python(context, working_dir): """Configure the WASI/host build.""" if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): @@ -215,18 +221,27 @@ def configure_wasi_python(context, working_dir): # Use PYTHONPATH to include sysconfig data which must be anchored to the # WASI guest's `/` directory. - host_runner = context.host_runner.format(GUEST_DIR="/", - HOST_DIR=CHECKOUT, - ENV_VAR_NAME="PYTHONPATH", - ENV_VAR_VALUE=f"/{sysconfig_data}", - PYTHON_WASM=working_dir / "python.wasm") + args = {"GUEST_DIR": "/", + "HOST_DIR": CHECKOUT, + "ENV_VAR_NAME": "PYTHONPATH", + "ENV_VAR_VALUE": f"/{sysconfig_data}", + "PYTHON_WASM": working_dir / "python.wasm"} + # Check dynamically for wasmtime in case it was specified manually via + # `--host-runner`. + if WASMTIME_HOST_RUNNER_VAR in context.host_runner: + if wasmtime := shutil.which("wasmtime"): + args[WASMTIME_VAR_NAME] = wasmtime + else: + raise FileNotFoundError("wasmtime not found; download from " + "https://github.com/bytecodealliance/wasmtime") + host_runner = context.host_runner.format_map(args) env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} build_python = os.fsdecode(build_python_path()) # The path to `configure` MUST be relative, else `python.wasm` is unable # to find the stdlib due to Python not recognizing that it's being # executed from within a checkout. configure = [os.path.relpath(CHECKOUT / 'configure', working_dir), - f"--host={HOST_TRIPLE}", + f"--host={context.host_triple}", f"--build={build_platform()}", f"--with-build-python={build_python}"] if pydebug: @@ -246,7 +261,7 @@ def configure_wasi_python(context, working_dir): sys.stdout.flush() -@subdir(HOST_DIR) +@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple) def make_wasi_python(context, working_dir): """Run `make` for the WASI/host build.""" call(["make", "--jobs", str(cpu_count()), "all"], @@ -277,14 +292,13 @@ def clean_contents(context): def main(): - default_host_runner = (f"{shutil.which('wasmtime')} run " + default_host_runner = (f"{WASMTIME_HOST_RUNNER_VAR} run " # Make sure the stack size will work for a pydebug # build. - # The 8388608 value comes from `ulimit -s` under Linux - # which equates to 8291 KiB. - "--wasm max-wasm-stack=8388608 " - # Enable thread support. - "--wasm threads=y --wasi threads=y " + # Use 16 MiB stack. + "--wasm max-wasm-stack=16777216 " + # Enable thread support; causes use of preview1. + #"--wasm threads=y --wasi threads=y " # Map the checkout to / to load the stdlib from /Lib. "--dir {HOST_DIR}::{GUEST_DIR} " # Set PYTHONPATH to the sysconfig data. @@ -329,6 +343,9 @@ def main(): help="Command template for running the WASI host " "(default designed for wasmtime 14 or newer: " f"`{default_host_runner}`)") + for subcommand in build, configure_host, make_host: + subcommand.add_argument("--host-triple", action="store", default="wasm32-wasip1", + help="The target triple for the WASI host build") context = parser.parse_args() diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index c0b9999a5dad03..bcb80212362b71 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -329,8 +329,10 @@ def _check_wasi() -> None: # workaround for https://github.com/python/cpython/issues/95952 "HOSTRUNNER": ( "wasmtime run " - "--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-{version}:/Lib " - "--mapdir /::{srcdir} --" + "--wasm max-wasm-stack=16777216 " + "--wasi preview2 " + "--dir {srcdir}::/ " + "--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-{version}:/Lib" ), "PATH": [WASI_SDK_PATH / "bin", os.environ["PATH"]], }, diff --git a/aclocal.m4 b/aclocal.m4 index 832aec19f48f17..920c2b38560faa 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -41,32 +41,81 @@ m4_ifndef([AC_CONFIG_MACRO_DIRS], [m4_defun([_AM_CONFIG_MACRO_DIRS], [])m4_defun # If neither value is found, the user is instructed to specify the # ordering. # +# Early versions of this macro (i.e., before serial 12) would not work +# when interprocedural optimization (via link-time optimization) was +# enabled. This would happen when, say, the GCC/clang "-flto" flag, or the +# ICC "-ipo" flag was used, for example. The problem was that under +# these conditions, the compiler did not allocate for and write the special +# float value in the data segment of the object file, since doing so might +# not prove optimal once more context was available. Thus, the special value +# (in platform-dependent binary form) could not be found in the object file, +# and the macro would fail. +# +# The solution to the above problem was to: +# +# 1) Compile and link a whole test program rather than just compile an +# object file. This ensures that we reach the point where even an +# interprocedural optimizing compiler writes values to the data segment. +# +# 2) Add code that requires the compiler to write the special value to +# the data segment, as opposed to "optimizing away" the variable's +# allocation. This could be done via compiler keywords or options, but +# it's tricky to make this work for all versions of all compilers with +# all optimization settings. The chosen solution was to make the exit +# code of the test program depend on the storing of the special value +# in memory (in the data segment). Because the exit code can be +# verified, any compiler that aspires to be correct will produce a +# program binary that contains the value, which the macro can then find. +# +# How does the exit code depend on the special value residing in memory? +# Memory, unlike variables and registers, can be addressed indirectly at run +# time. The exit code of this test program is a result of indirectly reading +# and writing to the memory region where the special value is supposed to +# reside. The actual memory addresses used and the values to be written are +# derived from the the program input ("argv") and are therefore not known at +# compile or link time. The compiler has no choice but to defer the +# computation to run time, and to prepare by allocating and populating the +# data segment with the special value. For further details, refer to the +# source code of the test program. +# +# Note that the test program is never meant to be run. It only exists to host +# a double float value in a given platform's binary format. Thus, error +# handling is not included. +# # LICENSE # -# Copyright (c) 2008 Daniel Amelang +# Copyright (c) 2008, 2023 Daniel Amelang # # Copying and distribution of this file, with or without modification, are # permitted in any medium without royalty provided the copyright notice # and this notice are preserved. This file is offered as-is, without any # warranty. -#serial 11 +#serial 14 AC_DEFUN([AX_C_FLOAT_WORDS_BIGENDIAN], [AC_CACHE_CHECK(whether float word ordering is bigendian, ax_cv_c_float_words_bigendian, [ ax_cv_c_float_words_bigendian=unknown -AC_COMPILE_IFELSE([AC_LANG_SOURCE([[ +AC_LINK_IFELSE([AC_LANG_SOURCE([[ + +#include + +static double m[] = {9.090423496703681e+223, 0.0}; -double d = 90904234967036810337470478905505011476211692735615632014797120844053488865816695273723469097858056257517020191247487429516932130503560650002327564517570778480236724525140520121371739201496540132640109977779420565776568942592.0; +int main (int argc, char *argv[]) +{ + m[atoi (argv[1])] += atof (argv[2]); + return m[atoi (argv[3])] > 0.0; +} ]])], [ -if grep noonsees conftest.$ac_objext >/dev/null ; then +if grep noonsees conftest* > /dev/null ; then ax_cv_c_float_words_bigendian=yes fi -if grep seesnoon conftest.$ac_objext >/dev/null ; then +if grep seesnoon conftest* >/dev/null ; then if test "$ax_cv_c_float_words_bigendian" = unknown; then ax_cv_c_float_words_bigendian=no else @@ -349,7 +398,7 @@ AC_DEFUN([AX_CHECK_OPENSSL], [ AC_SUBST([OPENSSL_LDFLAGS]) ]) -# pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*- +# pkg.m4 - Macros to locate and use pkg-config. -*- Autoconf -*- # serial 12 (pkg-config-0.29.2) dnl Copyright © 2004 Scott James Remnant . @@ -437,7 +486,7 @@ dnl Check to see whether a particular set of modules exists. Similar to dnl PKG_CHECK_MODULES(), but does not set variables or print errors. dnl dnl Please remember that m4 expands AC_REQUIRE([PKG_PROG_PKG_CONFIG]) -dnl only at the first occurence in configure.ac, so if the first place +dnl only at the first occurrence in configure.ac, so if the first place dnl it's called might be skipped (such as if it is within an "if", you dnl have to call PKG_CHECK_EXISTS manually AC_DEFUN([PKG_CHECK_EXISTS], @@ -506,14 +555,14 @@ if test $pkg_failed = yes; then AC_MSG_RESULT([no]) _PKG_SHORT_ERRORS_SUPPORTED if test $_pkg_short_errors_supported = yes; then - $1[]_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "$2" 2>&1` + $1[]_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "$2" 2>&1` else - $1[]_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "$2" 2>&1` + $1[]_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "$2" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$$1[]_PKG_ERRORS" >&AS_MESSAGE_LOG_FD + # Put the nasty error message in config.log where it belongs + echo "$$1[]_PKG_ERRORS" >&AS_MESSAGE_LOG_FD - m4_default([$4], [AC_MSG_ERROR( + m4_default([$4], [AC_MSG_ERROR( [Package requirements ($2) were not met: $$1_PKG_ERRORS @@ -525,7 +574,7 @@ _PKG_TEXT])[]dnl ]) elif test $pkg_failed = untried; then AC_MSG_RESULT([no]) - m4_default([$4], [AC_MSG_FAILURE( + m4_default([$4], [AC_MSG_FAILURE( [The pkg-config script could not be found or is too old. Make sure it is in your PATH or set the PKG_CONFIG environment variable to the full path to pkg-config. @@ -535,10 +584,10 @@ _PKG_TEXT To get pkg-config, see .])[]dnl ]) else - $1[]_CFLAGS=$pkg_cv_[]$1[]_CFLAGS - $1[]_LIBS=$pkg_cv_[]$1[]_LIBS + $1[]_CFLAGS=$pkg_cv_[]$1[]_CFLAGS + $1[]_LIBS=$pkg_cv_[]$1[]_LIBS AC_MSG_RESULT([yes]) - $3 + $3 fi[]dnl ])dnl PKG_CHECK_MODULES @@ -625,6 +674,74 @@ AS_VAR_COPY([$1], [pkg_cv_][$1]) AS_VAR_IF([$1], [""], [$5], [$4])dnl ])dnl PKG_CHECK_VAR +dnl PKG_WITH_MODULES(VARIABLE-PREFIX, MODULES, +dnl [ACTION-IF-FOUND],[ACTION-IF-NOT-FOUND], +dnl [DESCRIPTION], [DEFAULT]) +dnl ------------------------------------------ +dnl +dnl Prepare a "--with-" configure option using the lowercase +dnl [VARIABLE-PREFIX] name, merging the behaviour of AC_ARG_WITH and +dnl PKG_CHECK_MODULES in a single macro. +AC_DEFUN([PKG_WITH_MODULES], +[ +m4_pushdef([with_arg], m4_tolower([$1])) + +m4_pushdef([description], + [m4_default([$5], [build with ]with_arg[ support])]) + +m4_pushdef([def_arg], [m4_default([$6], [auto])]) +m4_pushdef([def_action_if_found], [AS_TR_SH([with_]with_arg)=yes]) +m4_pushdef([def_action_if_not_found], [AS_TR_SH([with_]with_arg)=no]) + +m4_case(def_arg, + [yes],[m4_pushdef([with_without], [--without-]with_arg)], + [m4_pushdef([with_without],[--with-]with_arg)]) + +AC_ARG_WITH(with_arg, + AS_HELP_STRING(with_without, description[ @<:@default=]def_arg[@:>@]),, + [AS_TR_SH([with_]with_arg)=def_arg]) + +AS_CASE([$AS_TR_SH([with_]with_arg)], + [yes],[PKG_CHECK_MODULES([$1],[$2],$3,$4)], + [auto],[PKG_CHECK_MODULES([$1],[$2], + [m4_n([def_action_if_found]) $3], + [m4_n([def_action_if_not_found]) $4])]) + +m4_popdef([with_arg]) +m4_popdef([description]) +m4_popdef([def_arg]) + +])dnl PKG_WITH_MODULES + +dnl PKG_HAVE_WITH_MODULES(VARIABLE-PREFIX, MODULES, +dnl [DESCRIPTION], [DEFAULT]) +dnl ----------------------------------------------- +dnl +dnl Convenience macro to trigger AM_CONDITIONAL after PKG_WITH_MODULES +dnl check._[VARIABLE-PREFIX] is exported as make variable. +AC_DEFUN([PKG_HAVE_WITH_MODULES], +[ +PKG_WITH_MODULES([$1],[$2],,,[$3],[$4]) + +AM_CONDITIONAL([HAVE_][$1], + [test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"]) +])dnl PKG_HAVE_WITH_MODULES + +dnl PKG_HAVE_DEFINE_WITH_MODULES(VARIABLE-PREFIX, MODULES, +dnl [DESCRIPTION], [DEFAULT]) +dnl ------------------------------------------------------ +dnl +dnl Convenience macro to run AM_CONDITIONAL and AC_DEFINE after +dnl PKG_WITH_MODULES check. HAVE_[VARIABLE-PREFIX] is exported as make +dnl and preprocessor variable. +AC_DEFUN([PKG_HAVE_DEFINE_WITH_MODULES], +[ +PKG_HAVE_WITH_MODULES([$1],[$2],[$3],[$4]) + +AS_IF([test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"], + [AC_DEFINE([HAVE_][$1], 1, [Enable ]m4_tolower([$1])[ support])]) +])dnl PKG_HAVE_DEFINE_WITH_MODULES + # AM_CONDITIONAL -*- Autoconf -*- # Copyright (C) 1997-2021 Free Software Foundation, Inc. diff --git a/config.guess b/config.guess index e81d3ae7c210ba..cdfc4392047ce3 100755 --- a/config.guess +++ b/config.guess @@ -1,14 +1,14 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright 1992-2021 Free Software Foundation, Inc. +# Copyright 1992-2023 Free Software Foundation, Inc. # shellcheck disable=SC2006,SC2268 # see below for rationale -timestamp='2021-06-03' +timestamp='2023-08-22' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or +# the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, but @@ -47,7 +47,7 @@ me=`echo "$0" | sed -e 's,.*/,,'` usage="\ Usage: $0 [OPTION] -Output the configuration name of the system \`$me' is run on. +Output the configuration name of the system '$me' is run on. Options: -h, --help print this help, then exit @@ -60,13 +60,13 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright 1992-2021 Free Software Foundation, Inc. +Copyright 1992-2023 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." help=" -Try \`$me --help' for more information." +Try '$me --help' for more information." # Parse command line while test $# -gt 0 ; do @@ -102,8 +102,8 @@ GUESS= # temporary files to be created and, as you can see below, it is a # headache to deal with in a portable fashion. -# Historically, `CC_FOR_BUILD' used to be named `HOST_CC'. We still -# use `HOST_CC' if defined, but it is deprecated. +# Historically, 'CC_FOR_BUILD' used to be named 'HOST_CC'. We still +# use 'HOST_CC' if defined, but it is deprecated. # Portable tmp directory creation inspired by the Autoconf team. @@ -155,6 +155,9 @@ Linux|GNU|GNU/*) set_cc_for_build cat <<-EOF > "$dummy.c" + #if defined(__ANDROID__) + LIBC=android + #else #include #if defined(__UCLIBC__) LIBC=uclibc @@ -169,6 +172,7 @@ Linux|GNU|GNU/*) LIBC=musl #endif #endif + #endif EOF cc_set_libc=`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g'` eval "$cc_set_libc" @@ -437,7 +441,7 @@ case $UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION in # This test works for both compilers. if test "$CC_FOR_BUILD" != no_compiler_found; then if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + (CCOPTS="" $CC_FOR_BUILD -m64 -E - 2>/dev/null) | \ grep IS_64BIT_ARCH >/dev/null then SUN_ARCH=x86_64 @@ -459,7 +463,7 @@ case $UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION in UNAME_RELEASE=`uname -v` ;; esac - # Japanese Language versions have a version number like `4.1.3-JL'. + # Japanese Language versions have a version number like '4.1.3-JL'. SUN_REL=`echo "$UNAME_RELEASE" | sed -e 's/-/_/'` GUESS=sparc-sun-sunos$SUN_REL ;; @@ -904,7 +908,7 @@ EOF fi ;; *:FreeBSD:*:*) - UNAME_PROCESSOR=`/usr/bin/uname -p` + UNAME_PROCESSOR=`uname -p` case $UNAME_PROCESSOR in amd64) UNAME_PROCESSOR=x86_64 ;; @@ -929,6 +933,9 @@ EOF i*:PW*:*) GUESS=$UNAME_MACHINE-pc-pw32 ;; + *:SerenityOS:*:*) + GUESS=$UNAME_MACHINE-pc-serenity + ;; *:Interix*:*) case $UNAME_MACHINE in x86) @@ -963,11 +970,37 @@ EOF GNU_REL=`echo "$UNAME_RELEASE" | sed -e 's/[-(].*//'` GUESS=$UNAME_MACHINE-unknown-$GNU_SYS$GNU_REL-$LIBC ;; + x86_64:[Mm]anagarm:*:*|i?86:[Mm]anagarm:*:*) + GUESS="$UNAME_MACHINE-pc-managarm-mlibc" + ;; + *:[Mm]anagarm:*:*) + GUESS="$UNAME_MACHINE-unknown-managarm-mlibc" + ;; *:Minix:*:*) GUESS=$UNAME_MACHINE-unknown-minix ;; aarch64:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + set_cc_for_build + CPU=$UNAME_MACHINE + LIBCABI=$LIBC + if test "$CC_FOR_BUILD" != no_compiler_found; then + ABI=64 + sed 's/^ //' << EOF > "$dummy.c" + #ifdef __ARM_EABI__ + #ifdef __ARM_PCS_VFP + ABI=eabihf + #else + ABI=eabi + #endif + #endif +EOF + cc_set_abi=`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^ABI' | sed 's, ,,g'` + eval "$cc_set_abi" + case $ABI in + eabi | eabihf) CPU=armv8l; LIBCABI=$LIBC$ABI ;; + esac + fi + GUESS=$CPU-unknown-linux-$LIBCABI ;; aarch64_be:Linux:*:*) UNAME_MACHINE=aarch64_be @@ -1033,7 +1066,16 @@ EOF k1om:Linux:*:*) GUESS=$UNAME_MACHINE-unknown-linux-$LIBC ;; - loongarch32:Linux:*:* | loongarch64:Linux:*:* | loongarchx32:Linux:*:*) + kvx:Linux:*:*) + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; + kvx:cos:*:*) + GUESS=$UNAME_MACHINE-unknown-cos + ;; + kvx:mbr:*:*) + GUESS=$UNAME_MACHINE-unknown-mbr + ;; + loongarch32:Linux:*:* | loongarch64:Linux:*:*) GUESS=$UNAME_MACHINE-unknown-linux-$LIBC ;; m32r*:Linux:*:*) @@ -1148,16 +1190,27 @@ EOF ;; x86_64:Linux:*:*) set_cc_for_build + CPU=$UNAME_MACHINE LIBCABI=$LIBC if test "$CC_FOR_BUILD" != no_compiler_found; then - if (echo '#ifdef __ILP32__'; echo IS_X32; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_X32 >/dev/null - then - LIBCABI=${LIBC}x32 - fi + ABI=64 + sed 's/^ //' << EOF > "$dummy.c" + #ifdef __i386__ + ABI=x86 + #else + #ifdef __ILP32__ + ABI=x32 + #endif + #endif +EOF + cc_set_abi=`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^ABI' | sed 's, ,,g'` + eval "$cc_set_abi" + case $ABI in + x86) CPU=i686 ;; + x32) LIBCABI=${LIBC}x32 ;; + esac fi - GUESS=$UNAME_MACHINE-pc-linux-$LIBCABI + GUESS=$CPU-pc-linux-$LIBCABI ;; xtensa*:Linux:*:*) GUESS=$UNAME_MACHINE-unknown-linux-$LIBC @@ -1177,7 +1230,7 @@ EOF GUESS=$UNAME_MACHINE-pc-sysv4.2uw$UNAME_VERSION ;; i*86:OS/2:*:*) - # If we were able to find `uname', then EMX Unix compatibility + # If we were able to find 'uname', then EMX Unix compatibility # is probably installed. GUESS=$UNAME_MACHINE-pc-os2-emx ;; @@ -1318,7 +1371,7 @@ EOF GUESS=ns32k-sni-sysv fi ;; - PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort + PENTIUM:*:4.0*:*) # Unisys 'ClearPath HMP IX 4000' SVR4/MP effort # says GUESS=i586-unisys-sysv4 ;; @@ -1364,8 +1417,11 @@ EOF BePC:Haiku:*:*) # Haiku running on Intel PC compatible. GUESS=i586-pc-haiku ;; - x86_64:Haiku:*:*) - GUESS=x86_64-unknown-haiku + ppc:Haiku:*:*) # Haiku running on Apple PowerPC + GUESS=powerpc-apple-haiku + ;; + *:Haiku:*:*) # Haiku modern gcc (not bound by BeOS compat) + GUESS=$UNAME_MACHINE-unknown-haiku ;; SX-4:SUPER-UX:*:*) GUESS=sx4-nec-superux$UNAME_RELEASE @@ -1522,6 +1578,9 @@ EOF i*86:rdos:*:*) GUESS=$UNAME_MACHINE-pc-rdos ;; + i*86:Fiwix:*:*) + GUESS=$UNAME_MACHINE-pc-fiwix + ;; *:AROS:*:*) GUESS=$UNAME_MACHINE-unknown-aros ;; diff --git a/configure b/configure index 4a980fea453697..885c2cf7828d6c 100755 --- a/configure +++ b/configure @@ -1,11 +1,11 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.71 for python 3.13. +# Generated by GNU Autoconf 2.72 for python 3.14. # # Report bugs to . # # -# Copyright (C) 1992-1996, 1998-2017, 2020-2021 Free Software Foundation, +# Copyright (C) 1992-1996, 1998-2017, 2020-2023 Free Software Foundation, # Inc. # # @@ -17,7 +17,6 @@ # Be more Bourne compatible DUALCASE=1; export DUALCASE # for MKS sh -as_nop=: if test ${ZSH_VERSION+y} && (emulate sh) >/dev/null 2>&1 then : emulate sh @@ -26,12 +25,13 @@ then : # is contrary to our usage. Disable this feature. alias -g '${1+"$@"}'='"$@"' setopt NO_GLOB_SUBST -else $as_nop - case `(set -o) 2>/dev/null` in #( +else case e in #( + e) case `(set -o) 2>/dev/null` in #( *posix*) : set -o posix ;; #( *) : ;; +esac ;; esac fi @@ -103,7 +103,7 @@ IFS=$as_save_IFS ;; esac -# We did not find ourselves, most probably we were run as `sh COMMAND' +# We did not find ourselves, most probably we were run as 'sh COMMAND' # in which case we are not to be found in the path. if test "x$as_myself" = x; then as_myself=$0 @@ -133,15 +133,14 @@ case $- in # (((( esac exec $CONFIG_SHELL $as_opts "$as_myself" ${1+"$@"} # Admittedly, this is quite paranoid, since all the known shells bail -# out after a failed `exec'. +# out after a failed 'exec'. printf "%s\n" "$0: could not re-execute with $CONFIG_SHELL" >&2 exit 255 fi # We don't want this to propagate to other subprocesses. { _as_can_reexec=; unset _as_can_reexec;} if test "x$CONFIG_SHELL" = x; then - as_bourne_compatible="as_nop=: -if test \${ZSH_VERSION+y} && (emulate sh) >/dev/null 2>&1 + as_bourne_compatible="if test \${ZSH_VERSION+y} && (emulate sh) >/dev/null 2>&1 then : emulate sh NULLCMD=: @@ -149,12 +148,13 @@ then : # is contrary to our usage. Disable this feature. alias -g '\${1+\"\$@\"}'='\"\$@\"' setopt NO_GLOB_SUBST -else \$as_nop - case \`(set -o) 2>/dev/null\` in #( +else case e in #( + e) case \`(set -o) 2>/dev/null\` in #( *posix*) : set -o posix ;; #( *) : ;; +esac ;; esac fi " @@ -172,8 +172,9 @@ as_fn_ret_failure && { exitcode=1; echo as_fn_ret_failure succeeded.; } if ( set x; as_fn_ret_success y && test x = \"\$1\" ) then : -else \$as_nop - exitcode=1; echo positional parameters were not saved. +else case e in #( + e) exitcode=1; echo positional parameters were not saved. ;; +esac fi test x\$exitcode = x0 || exit 1 blah=\$(echo \$(echo blah)) @@ -187,14 +188,15 @@ test \$(( 1 + 1 )) = 2 || exit 1" if (eval "$as_required") 2>/dev/null then : as_have_required=yes -else $as_nop - as_have_required=no +else case e in #( + e) as_have_required=no ;; +esac fi if test x$as_have_required = xyes && (eval "$as_suggested") 2>/dev/null then : -else $as_nop - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +else case e in #( + e) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR as_found=false for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH do @@ -227,12 +229,13 @@ IFS=$as_save_IFS if $as_found then : -else $as_nop - if { test -f "$SHELL" || test -f "$SHELL.exe"; } && +else case e in #( + e) if { test -f "$SHELL" || test -f "$SHELL.exe"; } && as_run=a "$SHELL" -c "$as_bourne_compatible""$as_required" 2>/dev/null then : CONFIG_SHELL=$SHELL as_have_required=yes -fi +fi ;; +esac fi @@ -254,7 +257,7 @@ case $- in # (((( esac exec $CONFIG_SHELL $as_opts "$as_myself" ${1+"$@"} # Admittedly, this is quite paranoid, since all the known shells bail -# out after a failed `exec'. +# out after a failed 'exec'. printf "%s\n" "$0: could not re-execute with $CONFIG_SHELL" >&2 exit 255 fi @@ -274,7 +277,8 @@ $0: message. Then install a modern shell, or manually run $0: the script under such a shell if you do have one." fi exit 1 -fi +fi ;; +esac fi fi SHELL=${CONFIG_SHELL-/bin/sh} @@ -313,14 +317,6 @@ as_fn_exit () as_fn_set_status $1 exit $1 } # as_fn_exit -# as_fn_nop -# --------- -# Do nothing but, unlike ":", preserve the value of $?. -as_fn_nop () -{ - return $? -} -as_nop=as_fn_nop # as_fn_mkdir_p # ------------- @@ -389,11 +385,12 @@ then : { eval $1+=\$2 }' -else $as_nop - as_fn_append () +else case e in #( + e) as_fn_append () { eval $1=\$$1\$2 - } + } ;; +esac fi # as_fn_append # as_fn_arith ARG... @@ -407,21 +404,14 @@ then : { as_val=$(( $* )) }' -else $as_nop - as_fn_arith () +else case e in #( + e) as_fn_arith () { as_val=`expr "$@" || test $? -eq 1` - } + } ;; +esac fi # as_fn_arith -# as_fn_nop -# --------- -# Do nothing but, unlike ":", preserve the value of $?. -as_fn_nop () -{ - return $? -} -as_nop=as_fn_nop # as_fn_error STATUS ERROR [LINENO LOG_FD] # ---------------------------------------- @@ -495,6 +485,8 @@ as_cr_alnum=$as_cr_Letters$as_cr_digits /[$]LINENO/= ' <$as_myself | sed ' + t clear + :clear s/[$]LINENO.*/&-/ t lineno b @@ -543,7 +535,6 @@ esac as_echo='printf %s\n' as_echo_n='printf %s' - rm -f conf$$ conf$$.exe conf$$.file if test -d conf$$.dir; then rm -f conf$$.dir/conf$$.file @@ -555,9 +546,9 @@ if (echo >conf$$.file) 2>/dev/null; then if ln -s conf$$.file conf$$ 2>/dev/null; then as_ln_s='ln -s' # ... but there are two gotchas: - # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail. - # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable. - # In both cases, we have to default to `cp -pR'. + # 1) On MSYS, both 'ln -s file dir' and 'ln file dir' fail. + # 2) DJGPP < 2.04 has no symlinks; 'ln -s' creates a wrapper executable. + # In both cases, we have to default to 'cp -pR'. ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe || as_ln_s='cp -pR' elif ln conf$$.file conf$$ 2>/dev/null; then @@ -582,10 +573,12 @@ as_test_x='test -x' as_executable_p=as_fn_executable_p # Sed expression to map a string onto a valid CPP name. -as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" +as_sed_cpp="y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g" +as_tr_cpp="eval sed '$as_sed_cpp'" # deprecated # Sed expression to map a string onto a valid variable name. -as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'" +as_sed_sh="y%*+%pp%;s%[^_$as_cr_alnum]%_%g" +as_tr_sh="eval sed '$as_sed_sh'" # deprecated test -n "$DJDIR" || exec 7<&0 /dev/null && - as_fn_error $? "invalid feature name: \`$ac_useropt'" + as_fn_error $? "invalid feature name: '$ac_useropt'" ac_useropt_orig=$ac_useropt ac_useropt=`printf "%s\n" "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in @@ -1302,7 +1308,7 @@ do ac_useropt=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error $? "invalid feature name: \`$ac_useropt'" + as_fn_error $? "invalid feature name: '$ac_useropt'" ac_useropt_orig=$ac_useropt ac_useropt=`printf "%s\n" "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in @@ -1515,7 +1521,7 @@ do ac_useropt=`expr "x$ac_option" : 'x-*with-\([^=]*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error $? "invalid package name: \`$ac_useropt'" + as_fn_error $? "invalid package name: '$ac_useropt'" ac_useropt_orig=$ac_useropt ac_useropt=`printf "%s\n" "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in @@ -1531,7 +1537,7 @@ do ac_useropt=`expr "x$ac_option" : 'x-*without-\(.*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error $? "invalid package name: \`$ac_useropt'" + as_fn_error $? "invalid package name: '$ac_useropt'" ac_useropt_orig=$ac_useropt ac_useropt=`printf "%s\n" "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in @@ -1561,8 +1567,8 @@ do | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*) x_libraries=$ac_optarg ;; - -*) as_fn_error $? "unrecognized option: \`$ac_option' -Try \`$0 --help' for more information" + -*) as_fn_error $? "unrecognized option: '$ac_option' +Try '$0 --help' for more information" ;; *=*) @@ -1570,7 +1576,7 @@ Try \`$0 --help' for more information" # Reject names that are not valid shell variable names. case $ac_envvar in #( '' | [0-9]* | *[!_$as_cr_alnum]* ) - as_fn_error $? "invalid variable name: \`$ac_envvar'" ;; + as_fn_error $? "invalid variable name: '$ac_envvar'" ;; esac eval $ac_envvar=\$ac_optarg export $ac_envvar ;; @@ -1620,7 +1626,7 @@ do as_fn_error $? "expected an absolute directory name for --$ac_var: $ac_val" done -# There might be people who depend on the old broken behavior: `$host' +# There might be people who depend on the old broken behavior: '$host' # used to hold the argument of --host etc. # FIXME: To remove some day. build=$build_alias @@ -1688,7 +1694,7 @@ if test ! -r "$srcdir/$ac_unique_file"; then test "$ac_srcdir_defaulted" = yes && srcdir="$ac_confdir or .." as_fn_error $? "cannot find sources ($ac_unique_file) in $srcdir" fi -ac_msg="sources are in $srcdir, but \`cd $srcdir' does not work" +ac_msg="sources are in $srcdir, but 'cd $srcdir' does not work" ac_abs_confdir=`( cd "$srcdir" && test -r "./$ac_unique_file" || as_fn_error $? "$ac_msg" pwd)` @@ -1716,7 +1722,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures python 3.13 to adapt to many kinds of systems. +'configure' configures python 3.14 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1730,11 +1736,11 @@ Configuration: --help=short display options specific to this package --help=recursive display the short help of all the included packages -V, --version display version information and exit - -q, --quiet, --silent do not print \`checking ...' messages + -q, --quiet, --silent do not print 'checking ...' messages --cache-file=FILE cache test results in FILE [disabled] - -C, --config-cache alias for \`--cache-file=config.cache' + -C, --config-cache alias for '--cache-file=config.cache' -n, --no-create do not create output files - --srcdir=DIR find the sources in DIR [configure dir or \`..'] + --srcdir=DIR find the sources in DIR [configure dir or '..'] Installation directories: --prefix=PREFIX install architecture-independent files in PREFIX @@ -1742,10 +1748,10 @@ Installation directories: --exec-prefix=EPREFIX install architecture-dependent files in EPREFIX [PREFIX] -By default, \`make install' will install all the files in -\`$ac_default_prefix/bin', \`$ac_default_prefix/lib' etc. You can specify -an installation prefix other than \`$ac_default_prefix' using \`--prefix', -for instance \`--prefix=\$HOME'. +By default, 'make install' will install all the files in +'$ac_default_prefix/bin', '$ac_default_prefix/lib' etc. You can specify +an installation prefix other than '$ac_default_prefix' using '--prefix', +for instance '--prefix=\$HOME'. For better control, use the options below. @@ -1782,7 +1788,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of python 3.13:";; + short | recursive ) echo "Configuration of python 3.14:";; esac cat <<\_ACEOF @@ -1811,13 +1817,17 @@ Optional Features: --disable-gil enable experimental support for running without the GIL (default is no) --enable-pystats enable internal statistics gathering (default is no) - --enable-experimental-jit + --enable-experimental-jit[=no|yes|yes-off|interpreter] build the experimental just-in-time compiler (default is no) --enable-optimizations enable expensive, stable optimizations (PGO, etc.) (default is no) --enable-bolt enable usage of the llvm-bolt post-link optimizer (default is no) + --enable-safety enable usage of the security compiler options with + no performance overhead + --enable-slower-safety enable usage of the security compiler options with + performance overhead --enable-loadable-sqlite-extensions support loadable extensions in the sqlite3 module, see Doc/library/sqlite3.rst (default is no) @@ -1831,9 +1841,9 @@ Optional Features: Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) - --with-build-python=python3.13 + --with-build-python=python3.14 path to build python binary for cross compiling - (default: _bootstrap_python or python3.13) + (default: _bootstrap_python or python3.14) --with-pkg-config=[yes|no|check] use pkg-config to detect build options (default is check) @@ -1848,8 +1858,10 @@ Optional Packages: specify the name for the python framework on macOS only valid when --enable-framework is set. see Mac/README.rst (default is 'Python') - --with-emscripten-target=[browser|node] - Emscripten platform + --with-app-store-compliance=[PATCH-FILE] + Enable any patches required for compiliance with app + stores. Optional PATCH-FILE specifies the custom + patch to apply. --with-suffix=SUFFIX set executable suffix to SUFFIX (default is empty, yes is mapped to '.exe') --without-static-libpython @@ -1885,8 +1897,9 @@ Optional Packages: --with-libs='lib1 ...' link against additional libs (default is no) --with-system-expat build pyexpat module using an installed expat library, see Doc/library/pyexpat.rst (default is no) - --with-system-libmpdec build _decimal module using an installed libmpdec - library, see Doc/library/decimal.rst (default is no) + --with-system-libmpdec build _decimal module using an installed mpdecimal + library, see Doc/library/decimal.rst (default is + yes) --with-decimal-contextvar build _decimal module using a coroutine-local rather than a thread-local context (default is yes) @@ -1898,7 +1911,6 @@ Optional Packages: --with-mimalloc build with mimalloc memory allocator (default is yes if C11 stdatomic.h is available.) --with-pymalloc enable specialized mallocs (default is yes) - --with-freelists enable object freelists (default is yes) --with-c-locale-coercion enable C locale coercion to a UTF-8 based locale (default is yes) @@ -1951,9 +1963,10 @@ Some influential environment variables: CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I if you have headers in a nonstandard directory CPP C preprocessor - HOSTRUNNER Program to run CPython for the host platform PROFILE_TASK Python args for PGO generation task + BOLT_COMMON_FLAGS + Common arguments to llvm-bolt when instrumenting and applying BOLT_INSTRUMENT_FLAGS Arguments to llvm-bolt when instrumenting binaries BOLT_APPLY_FLAGS @@ -1965,6 +1978,10 @@ Some influential environment variables: LIBFFI_CFLAGS C compiler flags for LIBFFI, overriding pkg-config LIBFFI_LIBS linker flags for LIBFFI, overriding pkg-config + LIBMPDEC_CFLAGS + C compiler flags for LIBMPDEC, overriding pkg-config + LIBMPDEC_LIBS + linker flags for LIBMPDEC, overriding pkg-config LIBSQLITE3_CFLAGS C compiler flags for LIBSQLITE3, overriding pkg-config LIBSQLITE3_LIBS @@ -1999,11 +2016,8 @@ Some influential environment variables: PANEL_CFLAGS C compiler flags for PANEL, overriding pkg-config PANEL_LIBS linker flags for PANEL, overriding pkg-config - LIBB2_CFLAGS - C compiler flags for LIBB2, overriding pkg-config - LIBB2_LIBS linker flags for LIBB2, overriding pkg-config -Use these variables to override the choices made by `configure' or to help +Use these variables to override the choices made by 'configure' or to help it to find libraries and programs with nonstandard names/locations. Report bugs to . @@ -2070,10 +2084,10 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -python configure 3.13 -generated by GNU Autoconf 2.71 +python configure 3.14 +generated by GNU Autoconf 2.72 -Copyright (C) 2021 Free Software Foundation, Inc. +Copyright (C) 2023 Free Software Foundation, Inc. This configure script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it. _ACEOF @@ -2112,11 +2126,12 @@ printf "%s\n" "$ac_try_echo"; } >&5 } && test -s conftest.$ac_objext then : ac_retval=0 -else $as_nop - printf "%s\n" "$as_me: failed program was:" >&5 +else case e in #( + e) printf "%s\n" "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 - ac_retval=1 + ac_retval=1 ;; +esac fi eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno as_fn_set_status $ac_retval @@ -2150,11 +2165,12 @@ printf "%s\n" "$ac_try_echo"; } >&5 } then : ac_retval=0 -else $as_nop - printf "%s\n" "$as_me: failed program was:" >&5 +else case e in #( + e) printf "%s\n" "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 - ac_retval=1 + ac_retval=1 ;; +esac fi eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno as_fn_set_status $ac_retval @@ -2173,8 +2189,8 @@ printf %s "checking for $2... " >&6; } if eval test \${$3+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 #include <$2> @@ -2182,10 +2198,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : eval "$3=yes" -else $as_nop - eval "$3=no" +else case e in #( + e) eval "$3=no" ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi eval ac_res=\$$3 { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 @@ -2225,11 +2243,12 @@ printf "%s\n" "$ac_try_echo"; } >&5 } then : ac_retval=0 -else $as_nop - printf "%s\n" "$as_me: failed program was:" >&5 +else case e in #( + e) printf "%s\n" "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 - ac_retval=1 + ac_retval=1 ;; +esac fi # Delete the IPA/IPO (Inter Procedural Analysis/Optimization) information # created by the PGI compiler (conftest_ipa8_conftest.oo), as it would @@ -2271,12 +2290,13 @@ printf "%s\n" "$ac_try_echo"; } >&5 test $ac_status = 0; }; } then : ac_retval=0 -else $as_nop - printf "%s\n" "$as_me: program exited with status $ac_status" >&5 +else case e in #( + e) printf "%s\n" "$as_me: program exited with status $ac_status" >&5 printf "%s\n" "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 - ac_retval=$ac_status + ac_retval=$ac_status ;; +esac fi rm -rf conftest.dSYM conftest_ipa8_conftest.oo eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno @@ -2296,8 +2316,8 @@ printf %s "checking for $2... " >&6; } if eval test \${$3+y} then : printf %s "(cached) " >&6 -else $as_nop - eval "$3=no" +else case e in #( + e) eval "$3=no" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 @@ -2327,12 +2347,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : -else $as_nop - eval "$3=yes" +else case e in #( + e) eval "$3=yes" ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi eval ac_res=\$$3 { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 @@ -2386,18 +2408,19 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_hi=$ac_mid; break -else $as_nop - as_fn_arith $ac_mid + 1 && ac_lo=$as_val +else case e in #( + e) as_fn_arith $ac_mid + 1 && ac_lo=$as_val if test $ac_lo -le $ac_mid; then ac_lo= ac_hi= break fi - as_fn_arith 2 '*' $ac_mid + 1 && ac_mid=$as_val + as_fn_arith 2 '*' $ac_mid + 1 && ac_mid=$as_val ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext done -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 int @@ -2432,20 +2455,23 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_lo=$ac_mid; break -else $as_nop - as_fn_arith '(' $ac_mid ')' - 1 && ac_hi=$as_val +else case e in #( + e) as_fn_arith '(' $ac_mid ')' - 1 && ac_hi=$as_val if test $ac_mid -le $ac_hi; then ac_lo= ac_hi= break fi - as_fn_arith 2 '*' $ac_mid && ac_mid=$as_val + as_fn_arith 2 '*' $ac_mid && ac_mid=$as_val ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext done -else $as_nop - ac_lo= ac_hi= +else case e in #( + e) ac_lo= ac_hi= ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext # Binary search between lo and hi bounds. @@ -2468,8 +2494,9 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_hi=$ac_mid -else $as_nop - as_fn_arith '(' $ac_mid ')' + 1 && ac_lo=$as_val +else case e in #( + e) as_fn_arith '(' $ac_mid ')' + 1 && ac_lo=$as_val ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext done @@ -2517,8 +2544,9 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : echo >>conftest.val; read $3 &6; } if eval test \${$3+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Define $2 to an innocuous variant, in case declares $2. For example, HP-UX 11i declares gettimeofday. */ #define $2 innocuous_$2 /* System header to define __stub macros and hopefully few prototypes, - which can conflict with char $2 (); below. */ + which can conflict with char $2 (void); below. */ #include #undef $2 @@ -2560,7 +2588,7 @@ else $as_nop #ifdef __cplusplus extern "C" #endif -char $2 (); +char $2 (void); /* The GNU C library defines this for functions which it implements to always fail with ENOSYS. Some functions are actually named something starting with __ and the normal name is an alias. */ @@ -2579,11 +2607,13 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : eval "$3=yes" -else $as_nop - eval "$3=no" +else case e in #( + e) eval "$3=no" ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext + conftest$ac_exeext conftest.$ac_ext ;; +esac fi eval ac_res=\$$3 { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 @@ -2605,8 +2635,8 @@ printf %s "checking whether $as_decl_name is declared... " >&6; } if eval test \${$3+y} then : printf %s "(cached) " >&6 -else $as_nop - as_decl_use=`echo $2|sed -e 's/(/((/' -e 's/)/) 0&/' -e 's/,/) 0& (/g'` +else case e in #( + e) as_decl_use=`echo $2|sed -e 's/(/((/' -e 's/)/) 0&/' -e 's/,/) 0& (/g'` eval ac_save_FLAGS=\$$6 as_fn_append $6 " $5" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -2630,12 +2660,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : eval "$3=yes" -else $as_nop - eval "$3=no" +else case e in #( + e) eval "$3=no" ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext eval $6=\$ac_save_FLAGS - + ;; +esac fi eval ac_res=\$$3 { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 @@ -2656,8 +2688,8 @@ printf %s "checking for $2.$3... " >&6; } if eval test \${$4+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $5 int @@ -2673,8 +2705,8 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : eval "$4=yes" -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $5 int @@ -2690,12 +2722,15 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : eval "$4=yes" -else $as_nop - eval "$4=no" +else case e in #( + e) eval "$4=no" ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi eval ac_res=\$$4 { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 @@ -2727,8 +2762,8 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by python $as_me 3.13, which was -generated by GNU Autoconf 2.71. Invocation command line was +It was created by python $as_me 3.14, which was +generated by GNU Autoconf 2.72. Invocation command line was $ $0$ac_configure_args_raw @@ -2974,10 +3009,10 @@ esac printf "%s\n" "$as_me: loading site script $ac_site_file" >&6;} sed 's/^/| /' "$ac_site_file" >&5 . "$ac_site_file" \ - || { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} + || { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error $? "failed to load site script $ac_site_file -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } fi done @@ -3013,9 +3048,7 @@ struct stat; /* Most of the following tests are stolen from RCS 5.7 src/conf.sh. */ struct buf { int x; }; struct buf * (*rcsopen) (struct buf *, struct stat *, int); -static char *e (p, i) - char **p; - int i; +static char *e (char **p, int i) { return p[i]; } @@ -3029,6 +3062,21 @@ static char *f (char * (*g) (char **, int), char **p, ...) return s; } +/* C89 style stringification. */ +#define noexpand_stringify(a) #a +const char *stringified = noexpand_stringify(arbitrary+token=sequence); + +/* C89 style token pasting. Exercises some of the corner cases that + e.g. old MSVC gets wrong, but not very hard. */ +#define noexpand_concat(a,b) a##b +#define expand_concat(a,b) noexpand_concat(a,b) +extern int vA; +extern int vbee; +#define aye A +#define bee B +int *pvA = &expand_concat(v,aye); +int *pvbee = &noexpand_concat(v,bee); + /* OSF 4.0 Compaq cc is some sort of almost-ANSI by default. It has function prototypes and stuff, but not \xHH hex character constants. These do not provoke an error unfortunately, instead are silently treated @@ -3056,16 +3104,19 @@ ok |= (argc == 0 || f (e, argv, 0) != argv[0] || f (e, argv, 1) != argv[1]); # Test code for whether the C compiler supports C99 (global declarations) ac_c_conftest_c99_globals=' -// Does the compiler advertise C99 conformance? +/* Does the compiler advertise C99 conformance? */ #if !defined __STDC_VERSION__ || __STDC_VERSION__ < 199901L # error "Compiler does not advertise C99 conformance" #endif +// See if C++-style comments work. + #include extern int puts (const char *); extern int printf (const char *, ...); extern int dprintf (int, const char *, ...); extern void *malloc (size_t); +extern void free (void *); // Check varargs macros. These examples are taken from C99 6.10.3.5. // dprintf is used instead of fprintf to avoid needing to declare @@ -3115,7 +3166,6 @@ typedef const char *ccp; static inline int test_restrict (ccp restrict text) { - // See if C++-style comments work. // Iterate through items via the restricted pointer. // Also check for declarations in for loops. for (unsigned int i = 0; *(text+i) != '\''\0'\''; ++i) @@ -3181,6 +3231,8 @@ ac_c_conftest_c99_main=' ia->datasize = 10; for (int i = 0; i < ia->datasize; ++i) ia->data[i] = i * 1.234; + // Work around memory leak warnings. + free (ia); // Check named initializers. struct named_init ni = { @@ -3202,7 +3254,7 @@ ac_c_conftest_c99_main=' # Test code for whether the C compiler supports C11 (global declarations) ac_c_conftest_c11_globals=' -// Does the compiler advertise C11 conformance? +/* Does the compiler advertise C11 conformance? */ #if !defined __STDC_VERSION__ || __STDC_VERSION__ < 201112L # error "Compiler does not advertise C11 conformance" #endif @@ -3396,8 +3448,9 @@ IFS=$as_save_IFS if $as_found then : -else $as_nop - as_fn_error $? "cannot find required auxiliary files:$ac_missing_aux_files" "$LINENO" 5 +else case e in #( + e) as_fn_error $? "cannot find required auxiliary files:$ac_missing_aux_files" "$LINENO" 5 ;; +esac fi @@ -3425,12 +3478,12 @@ for ac_var in $ac_precious_vars; do eval ac_new_val=\$ac_env_${ac_var}_value case $ac_old_set,$ac_new_set in set,) - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&5 -printf "%s\n" "$as_me: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&2;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: '$ac_var' was set to '$ac_old_val' in the previous run" >&5 +printf "%s\n" "$as_me: error: '$ac_var' was set to '$ac_old_val' in the previous run" >&2;} ac_cache_corrupted=: ;; ,set) - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' was not set in the previous run" >&5 -printf "%s\n" "$as_me: error: \`$ac_var' was not set in the previous run" >&2;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: '$ac_var' was not set in the previous run" >&5 +printf "%s\n" "$as_me: error: '$ac_var' was not set in the previous run" >&2;} ac_cache_corrupted=: ;; ,);; *) @@ -3439,18 +3492,18 @@ printf "%s\n" "$as_me: error: \`$ac_var' was not set in the previous run" >&2;} ac_old_val_w=`echo x $ac_old_val` ac_new_val_w=`echo x $ac_new_val` if test "$ac_old_val_w" != "$ac_new_val_w"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' has changed since the previous run:" >&5 -printf "%s\n" "$as_me: error: \`$ac_var' has changed since the previous run:" >&2;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: '$ac_var' has changed since the previous run:" >&5 +printf "%s\n" "$as_me: error: '$ac_var' has changed since the previous run:" >&2;} ac_cache_corrupted=: else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: warning: ignoring whitespace changes in \`$ac_var' since the previous run:" >&5 -printf "%s\n" "$as_me: warning: ignoring whitespace changes in \`$ac_var' since the previous run:" >&2;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: warning: ignoring whitespace changes in '$ac_var' since the previous run:" >&5 +printf "%s\n" "$as_me: warning: ignoring whitespace changes in '$ac_var' since the previous run:" >&2;} eval $ac_var=\$ac_old_val fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: former value: \`$ac_old_val'" >&5 -printf "%s\n" "$as_me: former value: \`$ac_old_val'" >&2;} - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: current value: \`$ac_new_val'" >&5 -printf "%s\n" "$as_me: current value: \`$ac_new_val'" >&2;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: former value: '$ac_old_val'" >&5 +printf "%s\n" "$as_me: former value: '$ac_old_val'" >&2;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: current value: '$ac_new_val'" >&5 +printf "%s\n" "$as_me: current value: '$ac_new_val'" >&2;} fi;; esac # Pass precious variables to config.status. @@ -3466,11 +3519,11 @@ printf "%s\n" "$as_me: current value: \`$ac_new_val'" >&2;} fi done if $ac_cache_corrupted; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: changes in the environment can compromise the build" >&5 printf "%s\n" "$as_me: error: changes in the environment can compromise the build" >&2;} - as_fn_error $? "run \`${MAKE-make} distclean' and/or \`rm $cache_file' + as_fn_error $? "run '${MAKE-make} distclean' and/or 'rm $cache_file' and start over" "$LINENO" 5 fi ## -------------------- ## @@ -3521,8 +3574,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_HAS_GIT+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$HAS_GIT"; then +else case e in #( + e) if test -n "$HAS_GIT"; then ac_cv_prog_HAS_GIT="$HAS_GIT" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -3545,7 +3598,8 @@ done IFS=$as_save_IFS test -z "$ac_cv_prog_HAS_GIT" && ac_cv_prog_HAS_GIT="not-found" -fi +fi ;; +esac fi HAS_GIT=$ac_cv_prog_HAS_GIT if test -n "$HAS_GIT"; then @@ -3587,15 +3641,16 @@ printf %s "checking build system type... " >&6; } if test ${ac_cv_build+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_build_alias=$build_alias +else case e in #( + e) ac_build_alias=$build_alias test "x$ac_build_alias" = x && ac_build_alias=`$SHELL "${ac_aux_dir}config.guess"` test "x$ac_build_alias" = x && as_fn_error $? "cannot guess build type; you must specify one" "$LINENO" 5 ac_cv_build=`$SHELL "${ac_aux_dir}config.sub" $ac_build_alias` || as_fn_error $? "$SHELL ${ac_aux_dir}config.sub $ac_build_alias failed" "$LINENO" 5 - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_build" >&5 printf "%s\n" "$ac_cv_build" >&6; } @@ -3622,14 +3677,15 @@ printf %s "checking host system type... " >&6; } if test ${ac_cv_host+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "x$host_alias" = x; then +else case e in #( + e) if test "x$host_alias" = x; then ac_cv_host=$ac_cv_build else ac_cv_host=`$SHELL "${ac_aux_dir}config.sub" $host_alias` || as_fn_error $? "$SHELL ${ac_aux_dir}config.sub $host_alias failed" "$LINENO" 5 fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_host" >&5 printf "%s\n" "$ac_cv_host" >&6; } @@ -3689,12 +3745,12 @@ fi fi ac_cv_prog_PYTHON_FOR_REGEN=$with_build_python PYTHON_FOR_FREEZE="$with_build_python" - PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$with_build_python + PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) _PYTHON_SYSCONFIGDATA_PATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`) '$with_build_python { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_build_python" >&5 printf "%s\n" "$with_build_python" >&6; } -else $as_nop - +else case e in #( + e) if test "x$cross_compiling" = xyes then : as_fn_error $? "Cross compiling requires --with-build-python" "$LINENO" 5 @@ -3703,7 +3759,8 @@ fi PYTHON_FOR_BUILD='./$(BUILDPYTHON) -E' PYTHON_FOR_FREEZE="./_bootstrap_python" - + ;; +esac fi @@ -3723,15 +3780,16 @@ then : FREEZE_MODULE_DEPS='$(FREEZE_MODULE_BOOTSTRAP_DEPS)' PYTHON_FOR_BUILD_DEPS='' -else $as_nop - +else case e in #( + e) FREEZE_MODULE_BOOTSTRAP='./Programs/_freeze_module' FREEZE_MODULE_BOOTSTRAP_DEPS="Programs/_freeze_module" FREEZE_MODULE='$(PYTHON_FOR_FREEZE) $(srcdir)/Programs/_freeze_module.py' FREEZE_MODULE_DEPS="_bootstrap_python \$(srcdir)/Programs/_freeze_module.py" PYTHON_FOR_BUILD_DEPS='$(BUILDPYTHON)' - + ;; +esac fi @@ -3748,8 +3806,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_PYTHON_FOR_REGEN+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$PYTHON_FOR_REGEN"; then +else case e in #( + e) if test -n "$PYTHON_FOR_REGEN"; then ac_cv_prog_PYTHON_FOR_REGEN="$PYTHON_FOR_REGEN" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -3771,7 +3829,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi PYTHON_FOR_REGEN=$ac_cv_prog_PYTHON_FOR_REGEN if test -n "$PYTHON_FOR_REGEN"; then @@ -3814,7 +3873,7 @@ rm confdefs.h mv confdefs.h.new confdefs.h -VERSION=3.13 +VERSION=3.14 # Version number of Python's own shared library file. @@ -3853,9 +3912,10 @@ CONFIG_ARGS="$ac_configure_args" if test ${with_pkg_config+y} then : withval=$with_pkg_config; -else $as_nop - with_pkg_config=check - +else case e in #( + e) with_pkg_config=check + ;; +esac fi case $with_pkg_config in #( @@ -3882,8 +3942,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_PKG_CONFIG+y} then : printf %s "(cached) " >&6 -else $as_nop - case $PKG_CONFIG in +else case e in #( + e) case $PKG_CONFIG in [\\/]* | ?:[\\/]*) ac_cv_path_PKG_CONFIG="$PKG_CONFIG" # Let the user override the test with a path. ;; @@ -3908,6 +3968,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi PKG_CONFIG=$ac_cv_path_PKG_CONFIG @@ -3930,8 +3991,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_ac_pt_PKG_CONFIG+y} then : printf %s "(cached) " >&6 -else $as_nop - case $ac_pt_PKG_CONFIG in +else case e in #( + e) case $ac_pt_PKG_CONFIG in [\\/]* | ?:[\\/]*) ac_cv_path_ac_pt_PKG_CONFIG="$ac_pt_PKG_CONFIG" # Let the user override the test with a path. ;; @@ -3956,6 +4017,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi ac_pt_PKG_CONFIG=$ac_cv_path_ac_pt_PKG_CONFIG @@ -4035,13 +4097,16 @@ then *-apple-ios*) ac_sys_system=iOS ;; + *-*-darwin*) + ac_sys_system=Darwin + ;; *-*-vxworks*) ac_sys_system=VxWorks ;; *-*-emscripten) ac_sys_system=Emscripten ;; - *-*-wasi) + *-*-wasi*) ac_sys_system=WASI ;; *) @@ -4067,6 +4132,7 @@ then case $MACHDEP in aix*) MACHDEP="aix";; + linux-android*) MACHDEP="android";; linux*) MACHDEP="linux";; cygwin*) MACHDEP="cygwin";; darwin*) MACHDEP="darwin";; @@ -4085,6 +4151,75 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: \"$MACHDEP\"" >&5 printf "%s\n" "\"$MACHDEP\"" >&6; } + +if test -z "$host_prefix"; then + case $ac_sys_system in #( + Emscripten) : + host_prefix=/ ;; #( + *) : + host_prefix='${prefix}' + ;; +esac +fi + + +if test -z "$host_exec_prefix"; then + case $ac_sys_system in #( + Emscripten) : + host_exec_prefix=$host_prefix ;; #( + *) : + host_exec_prefix='${exec_prefix}' + ;; +esac +fi + + +# On cross-compile builds, configure will look for a host-specific compiler by +# prepending the user-provided host triple to the required binary name. +# +# On iOS, this results in binaries like "arm64-apple-ios13.0-simulator-gcc", +# which isn't a binary that exists, and isn't very convenient, as it contains the +# iOS version. As the default cross-compiler name won't exist, configure falls +# back to gcc, which *definitely* won't work. We're providing wrapper scripts for +# these tools; the binary names of these scripts are better defaults than "gcc". +# This only requires that the user put the platform scripts folder (e.g., +# "iOS/Resources/bin") in their path, rather than defining platform-specific +# names/paths for AR, CC, CPP, and CXX explicitly; and if the user forgets to +# either put the platform scripts folder in the path, or specify CC etc, +# configure will fail. +if test -z "$AR"; then + case "$host" in + aarch64-apple-ios*-simulator) AR=arm64-apple-ios-simulator-ar ;; + aarch64-apple-ios*) AR=arm64-apple-ios-ar ;; + x86_64-apple-ios*-simulator) AR=x86_64-apple-ios-simulator-ar ;; + *) + esac +fi +if test -z "$CC"; then + case "$host" in + aarch64-apple-ios*-simulator) CC=arm64-apple-ios-simulator-clang ;; + aarch64-apple-ios*) CC=arm64-apple-ios-clang ;; + x86_64-apple-ios*-simulator) CC=x86_64-apple-ios-simulator-clang ;; + *) + esac +fi +if test -z "$CPP"; then + case "$host" in + aarch64-apple-ios*-simulator) CPP=arm64-apple-ios-simulator-cpp ;; + aarch64-apple-ios*) CPP=arm64-apple-ios-cpp ;; + x86_64-apple-ios*-simulator) CPP=x86_64-apple-ios-simulator-cpp ;; + *) + esac +fi +if test -z "$CXX"; then + case "$host" in + aarch64-apple-ios*-simulator) CXX=arm64-apple-ios-simulator-clang++ ;; + aarch64-apple-ios*) CXX=arm64-apple-ios-clang++ ;; + x86_64-apple-ios*-simulator) CXX=x86_64-apple-ios-simulator-clang++ ;; + *) + esac +fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-universalsdk" >&5 printf %s "checking for --enable-universalsdk... " >&6; } # Check whether --enable-universalsdk was given. @@ -4121,11 +4256,12 @@ then : esac -else $as_nop - +else case e in #( + e) UNIVERSALSDK= enable_universalsdk= - + ;; +esac fi if test -n "${UNIVERSALSDK}" @@ -4186,12 +4322,13 @@ then : PYTHONFRAMEWORKDIR=${withval}.framework PYTHONFRAMEWORKIDENTIFIER=org.python.`echo $withval | tr 'A-Z' 'a-z'` -else $as_nop - +else case e in #( + e) PYTHONFRAMEWORK=Python PYTHONFRAMEWORKDIR=Python.framework PYTHONFRAMEWORKIDENTIFIER=org.python.python - + ;; +esac fi # Check whether --enable-framework was given. @@ -4200,37 +4337,38 @@ then : enableval=$enable_framework; case $enableval in yes) - if test "$ac_sys_system" = "iOS"; then - as_fn_error $? "iOS builds must provide an explicit path for --enable-framework" "$LINENO" 5 - fi - - enableval=/Library/Frameworks + case $ac_sys_system in + Darwin) enableval=/Library/Frameworks ;; + iOS) enableval=iOS/Frameworks/\$\(MULTIARCH\) ;; + *) as_fn_error $? "Unknown platform for framework build" "$LINENO" 5 + esac esac + case $enableval in no) - if test "$ac_sys_system" = "iOS"; then - as_fn_error $? "iOS builds must use --enable-framework=" "$LINENO" 5 - fi + case $ac_sys_system in + iOS) as_fn_error $? "iOS builds must use --enable-framework" "$LINENO" 5 ;; + *) + PYTHONFRAMEWORK= + PYTHONFRAMEWORKDIR=no-framework + PYTHONFRAMEWORKPREFIX= + PYTHONFRAMEWORKINSTALLDIR= + PYTHONFRAMEWORKINSTALLNAMEPREFIX= + RESSRCDIR= + FRAMEWORKINSTALLFIRST= + FRAMEWORKINSTALLLAST= + FRAMEWORKALTINSTALLFIRST= + FRAMEWORKALTINSTALLLAST= + FRAMEWORKPYTHONW= + INSTALLTARGETS="commoninstall bininstall maninstall" - PYTHONFRAMEWORK= - PYTHONFRAMEWORKDIR=no-framework - PYTHONFRAMEWORKPREFIX= - PYTHONFRAMEWORKINSTALLDIR= - PYTHONFRAMEWORKINSTALLNAMEPREFIX= - RESSRCDIR= - FRAMEWORKINSTALLFIRST= - FRAMEWORKINSTALLLAST= - FRAMEWORKALTINSTALLFIRST= - FRAMEWORKALTINSTALLLAST= - FRAMEWORKPYTHONW= - INSTALLTARGETS="commoninstall bininstall maninstall" - - if test "x${prefix}" = "xNONE"; then - FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" - else - FRAMEWORKUNIXTOOLSPREFIX="${prefix}" - fi - enable_framework= + if test "x${prefix}" = "xNONE"; then + FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" + else + FRAMEWORKUNIXTOOLSPREFIX="${prefix}" + fi + enable_framework= + esac ;; *) PYTHONFRAMEWORKPREFIX="${enableval}" @@ -4323,33 +4461,38 @@ then : esac esac -else $as_nop +else case e in #( + e) + case $ac_sys_system in + iOS) as_fn_error $? "iOS builds must use --enable-framework" "$LINENO" 5 ;; + *) + PYTHONFRAMEWORK= + PYTHONFRAMEWORKDIR=no-framework + PYTHONFRAMEWORKPREFIX= + PYTHONFRAMEWORKINSTALLDIR= + PYTHONFRAMEWORKINSTALLNAMEPREFIX= + RESSRCDIR= + FRAMEWORKINSTALLFIRST= + FRAMEWORKINSTALLLAST= + FRAMEWORKALTINSTALLFIRST= + FRAMEWORKALTINSTALLLAST= + FRAMEWORKPYTHONW= + INSTALLTARGETS="commoninstall bininstall maninstall" + if test "x${prefix}" = "xNONE" ; then + FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" + else + FRAMEWORKUNIXTOOLSPREFIX="${prefix}" + fi + enable_framework= + esac + ;; +esac +fi + - if test "$ac_sys_system" = "iOS"; then - as_fn_error $? "iOS builds must use --enable-framework=" "$LINENO" 5 - fi - PYTHONFRAMEWORK= - PYTHONFRAMEWORKDIR=no-framework - PYTHONFRAMEWORKPREFIX= - PYTHONFRAMEWORKINSTALLDIR= - PYTHONFRAMEWORKINSTALLNAMEPREFIX= - RESSRCDIR= - FRAMEWORKINSTALLFIRST= - FRAMEWORKINSTALLLAST= - FRAMEWORKALTINSTALLFIRST= - FRAMEWORKALTINSTALLLAST= - FRAMEWORKPYTHONW= - INSTALLTARGETS="commoninstall bininstall maninstall" - if test "x${prefix}" = "xNONE" ; then - FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" - else - FRAMEWORKUNIXTOOLSPREFIX="${prefix}" - fi - enable_framework= -fi @@ -4363,12 +4506,55 @@ fi +printf "%s\n" "#define _PYTHONFRAMEWORK \"${PYTHONFRAMEWORK}\"" >>confdefs.h +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-app-store-compliance" >&5 +printf %s "checking for --with-app-store-compliance... " >&6; } +# Check whether --with-app_store_compliance was given. +if test ${with_app_store_compliance+y} +then : + withval=$with_app_store_compliance; + case "$withval" in + yes) + case $ac_sys_system in + Darwin|iOS) + # iOS is able to share the macOS patch + APP_STORE_COMPLIANCE_PATCH="Mac/Resources/app-store-compliance.patch" + ;; + *) as_fn_error $? "no default app store compliance patch available for $ac_sys_system" "$LINENO" 5 ;; + esac + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: applying default app store compliance patch" >&5 +printf "%s\n" "applying default app store compliance patch" >&6; } + ;; + *) + APP_STORE_COMPLIANCE_PATCH="${withval}" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: applying custom app store compliance patch" >&5 +printf "%s\n" "applying custom app store compliance patch" >&6; } + ;; + esac +else case e in #( + e) + case $ac_sys_system in + iOS) + # Always apply the compliance patch on iOS; we can use the macOS patch + APP_STORE_COMPLIANCE_PATCH="Mac/Resources/app-store-compliance.patch" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: applying default app store compliance patch" >&5 +printf "%s\n" "applying default app store compliance patch" >&6; } + ;; + *) + # No default app compliance patching on any other platform + APP_STORE_COMPLIANCE_PATCH= + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: not patching for app store compliance" >&5 +printf "%s\n" "not patching for app store compliance" >&6; } + ;; + esac + ;; +esac +fi -printf "%s\n" "#define _PYTHONFRAMEWORK \"${PYTHONFRAMEWORK}\"" >>confdefs.h @@ -4391,21 +4577,38 @@ if test "$cross_compiling" = yes; then _host_device=`echo $host | cut -d '-' -f4` _host_device=${_host_device:=os} - IOS_DEPLOYMENT_TARGET=${_host_os:3} - IOS_DEPLOYMENT_TARGET=${IOS_DEPLOYMENT_TARGET:=12.0} + # IPHONEOS_DEPLOYMENT_TARGET is the minimum supported iOS version + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking iOS deployment target" >&5 +printf %s "checking iOS deployment target... " >&6; } + IPHONEOS_DEPLOYMENT_TARGET=$(echo ${_host_os} | cut -c4-) + IPHONEOS_DEPLOYMENT_TARGET=${IPHONEOS_DEPLOYMENT_TARGET:=13.0} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $IPHONEOS_DEPLOYMENT_TARGET" >&5 +printf "%s\n" "$IPHONEOS_DEPLOYMENT_TARGET" >&6; } case "$host_cpu" in aarch64) - _host_ident=${IOS_DEPLOYMENT_TARGET}-arm64-iphone${_host_device} + _host_ident=${IPHONEOS_DEPLOYMENT_TARGET}-arm64-iphone${_host_device} ;; *) - _host_ident=${IOS_DEPLOYMENT_TARGET}-$host_cpu-iphone${_host_device} + _host_ident=${IPHONEOS_DEPLOYMENT_TARGET}-$host_cpu-iphone${_host_device} ;; esac ;; + *-*-darwin*) + case "$host_cpu" in + arm*) + _host_ident=arm + ;; + *) + _host_ident=$host_cpu + esac + ;; *-*-vxworks*) _host_ident=$host_cpu ;; + *-*-emscripten) + _host_ident=$(emcc -dumpversion | cut -f1 -d-)-$host_cpu + ;; wasm32-*-* | wasm64-*-*) _host_ident=$host_cpu ;; @@ -4546,6 +4749,9 @@ fi CONFIGURE_MACOSX_DEPLOYMENT_TARGET= EXPORT_MACOSX_DEPLOYMENT_TARGET='#' +# Record the value of IPHONEOS_DEPLOYMENT_TARGET enforced by the selected host triple. + + # checks for alternative programs # compiler flags are generated in two sets, BASECFLAGS and OPT. OPT is just @@ -4581,9 +4787,8 @@ esac case $ac_sys_system in #( iOS) : - as_fn_append CFLAGS " -mios-version-min=${IOS_DEPLOYMENT_TARGET}" - as_fn_append LDFLAGS " -mios-version-min=${IOS_DEPLOYMENT_TARGET}" - + as_fn_append CFLAGS " -mios-version-min=${IPHONEOS_DEPLOYMENT_TARGET}" + as_fn_append LDFLAGS " -mios-version-min=${IPHONEOS_DEPLOYMENT_TARGET}" ;; #( *) : ;; @@ -4598,8 +4803,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_HAS_XCRUN+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$HAS_XCRUN"; then +else case e in #( + e) if test -n "$HAS_XCRUN"; then ac_cv_prog_HAS_XCRUN="$HAS_XCRUN" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -4622,7 +4827,8 @@ done IFS=$as_save_IFS test -z "$ac_cv_prog_HAS_XCRUN" && ac_cv_prog_HAS_XCRUN="missing" -fi +fi ;; +esac fi HAS_XCRUN=$ac_cv_prog_HAS_XCRUN if test -n "$HAS_XCRUN"; then @@ -4725,8 +4931,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_CC+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$CC"; then +else case e in #( + e) if test -n "$CC"; then ac_cv_prog_CC="$CC" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -4748,7 +4954,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi CC=$ac_cv_prog_CC if test -n "$CC"; then @@ -4770,8 +4977,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_ac_ct_CC+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$ac_ct_CC"; then +else case e in #( + e) if test -n "$ac_ct_CC"; then ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -4793,7 +5000,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi ac_ct_CC=$ac_cv_prog_ac_ct_CC if test -n "$ac_ct_CC"; then @@ -4828,8 +5036,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_CC+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$CC"; then +else case e in #( + e) if test -n "$CC"; then ac_cv_prog_CC="$CC" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -4851,7 +5059,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi CC=$ac_cv_prog_CC if test -n "$CC"; then @@ -4873,8 +5082,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_CC+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$CC"; then +else case e in #( + e) if test -n "$CC"; then ac_cv_prog_CC="$CC" # Let the user override the test. else ac_prog_rejected=no @@ -4913,7 +5122,8 @@ if test $ac_prog_rejected = yes; then ac_cv_prog_CC="$as_dir$ac_word${1+' '}$@" fi fi -fi +fi ;; +esac fi CC=$ac_cv_prog_CC if test -n "$CC"; then @@ -4937,8 +5147,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_CC+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$CC"; then +else case e in #( + e) if test -n "$CC"; then ac_cv_prog_CC="$CC" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -4960,7 +5170,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi CC=$ac_cv_prog_CC if test -n "$CC"; then @@ -4986,8 +5197,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_ac_ct_CC+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$ac_ct_CC"; then +else case e in #( + e) if test -n "$ac_ct_CC"; then ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -5009,7 +5220,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi ac_ct_CC=$ac_cv_prog_ac_ct_CC if test -n "$ac_ct_CC"; then @@ -5047,8 +5259,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_CC+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$CC"; then +else case e in #( + e) if test -n "$CC"; then ac_cv_prog_CC="$CC" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -5070,7 +5282,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi CC=$ac_cv_prog_CC if test -n "$CC"; then @@ -5092,8 +5305,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_ac_ct_CC+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$ac_ct_CC"; then +else case e in #( + e) if test -n "$ac_ct_CC"; then ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -5115,7 +5328,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi ac_ct_CC=$ac_cv_prog_ac_ct_CC if test -n "$ac_ct_CC"; then @@ -5144,10 +5358,10 @@ fi fi -test -z "$CC" && { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +test -z "$CC" && { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error $? "no acceptable C compiler found in \$PATH -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } # Provide some information about the compiler. printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for C compiler version" >&5 @@ -5219,8 +5433,8 @@ printf "%s\n" "$ac_try_echo"; } >&5 printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } then : - # Autoconf-2.13 could set the ac_cv_exeext variable to `no'. -# So ignore a value of `no', otherwise this would lead to `EXEEXT = no' + # Autoconf-2.13 could set the ac_cv_exeext variable to 'no'. +# So ignore a value of 'no', otherwise this would lead to 'EXEEXT = no' # in a Makefile. We should not override ac_cv_exeext if it was cached, # so that the user can short-circuit this test for compilers unknown to # Autoconf. @@ -5240,7 +5454,7 @@ do ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'` fi # We set ac_cv_exeext here because the later test for it is not - # safe: cross compilers may not add the suffix if given an `-o' + # safe: cross compilers may not add the suffix if given an '-o' # argument, so we may need to know it at that point already. # Even if this section looks crufty: it has the advantage of # actually working. @@ -5251,8 +5465,9 @@ do done test "$ac_cv_exeext" = no && ac_cv_exeext= -else $as_nop - ac_file='' +else case e in #( + e) ac_file='' ;; +esac fi if test -z "$ac_file" then : @@ -5261,13 +5476,14 @@ printf "%s\n" "no" >&6; } printf "%s\n" "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 -{ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +{ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "C compiler cannot create executables -See \`config.log' for more details" "$LINENO" 5; } -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } +See 'config.log' for more details" "$LINENO" 5; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for C compiler default output file name" >&5 printf %s "checking for C compiler default output file name... " >&6; } @@ -5291,10 +5507,10 @@ printf "%s\n" "$ac_try_echo"; } >&5 printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } then : - # If both `conftest.exe' and `conftest' are `present' (well, observable) -# catch `conftest.exe'. For instance with Cygwin, `ls conftest' will -# work properly (i.e., refer to `conftest.exe'), while it won't with -# `rm'. + # If both 'conftest.exe' and 'conftest' are 'present' (well, observable) +# catch 'conftest.exe'. For instance with Cygwin, 'ls conftest' will +# work properly (i.e., refer to 'conftest.exe'), while it won't with +# 'rm'. for ac_file in conftest.exe conftest conftest.*; do test -f "$ac_file" || continue case $ac_file in @@ -5304,11 +5520,12 @@ for ac_file in conftest.exe conftest conftest.*; do * ) break;; esac done -else $as_nop - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error $? "cannot compute suffix of executables: cannot compile and link -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } ;; +esac fi rm -f conftest conftest$ac_cv_exeext { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_exeext" >&5 @@ -5324,6 +5541,8 @@ int main (void) { FILE *f = fopen ("conftest.out", "w"); + if (!f) + return 1; return ferror (f) || fclose (f) != 0; ; @@ -5363,26 +5582,27 @@ printf "%s\n" "$ac_try_echo"; } >&5 if test "$cross_compiling" = maybe; then cross_compiling=yes else - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot run C compiled programs. -If you meant to cross compile, use \`--host'. -See \`config.log' for more details" "$LINENO" 5; } +If you meant to cross compile, use '--host'. +See 'config.log' for more details" "$LINENO" 5; } fi fi fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $cross_compiling" >&5 printf "%s\n" "$cross_compiling" >&6; } -rm -f conftest.$ac_ext conftest$ac_cv_exeext conftest.out +rm -f conftest.$ac_ext conftest$ac_cv_exeext \ + conftest.o conftest.obj conftest.out ac_clean_files=$ac_clean_files_save { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for suffix of object files" >&5 printf %s "checking for suffix of object files... " >&6; } if test ${ac_cv_objext+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int @@ -5414,16 +5634,18 @@ then : break;; esac done -else $as_nop - printf "%s\n" "$as_me: failed program was:" >&5 +else case e in #( + e) printf "%s\n" "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 -{ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +{ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error $? "cannot compute suffix of object files: cannot compile -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } ;; +esac fi -rm -f conftest.$ac_cv_objext conftest.$ac_ext +rm -f conftest.$ac_cv_objext conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_objext" >&5 printf "%s\n" "$ac_cv_objext" >&6; } @@ -5434,8 +5656,8 @@ printf %s "checking whether the compiler supports GNU C... " >&6; } if test ${ac_cv_c_compiler_gnu+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int @@ -5452,12 +5674,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_compiler_gnu=yes -else $as_nop - ac_compiler_gnu=no +else case e in #( + e) ac_compiler_gnu=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ac_cv_c_compiler_gnu=$ac_compiler_gnu - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_compiler_gnu" >&5 printf "%s\n" "$ac_cv_c_compiler_gnu" >&6; } @@ -5475,8 +5699,8 @@ printf %s "checking whether $CC accepts -g... " >&6; } if test ${ac_cv_prog_cc_g+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_save_c_werror_flag=$ac_c_werror_flag +else case e in #( + e) ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes ac_cv_prog_cc_g=no CFLAGS="-g" @@ -5494,8 +5718,8 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_prog_cc_g=yes -else $as_nop - CFLAGS="" +else case e in #( + e) CFLAGS="" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -5510,8 +5734,8 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : -else $as_nop - ac_c_werror_flag=$ac_save_c_werror_flag +else case e in #( + e) ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="-g" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -5528,12 +5752,15 @@ if ac_fn_c_try_compile "$LINENO" then : ac_cv_prog_cc_g=yes fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - ac_c_werror_flag=$ac_save_c_werror_flag + ac_c_werror_flag=$ac_save_c_werror_flag ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_g" >&5 printf "%s\n" "$ac_cv_prog_cc_g" >&6; } @@ -5560,8 +5787,8 @@ printf %s "checking for $CC option to enable C11 features... " >&6; } if test ${ac_cv_prog_cc_c11+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_cv_prog_cc_c11=no +else case e in #( + e) ac_cv_prog_cc_c11=no ac_save_CC=$CC cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -5578,25 +5805,28 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam test "x$ac_cv_prog_cc_c11" != "xno" && break done rm -f conftest.$ac_ext -CC=$ac_save_CC +CC=$ac_save_CC ;; +esac fi if test "x$ac_cv_prog_cc_c11" = xno then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 printf "%s\n" "unsupported" >&6; } -else $as_nop - if test "x$ac_cv_prog_cc_c11" = x +else case e in #( + e) if test "x$ac_cv_prog_cc_c11" = x then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 printf "%s\n" "none needed" >&6; } -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c11" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c11" >&5 printf "%s\n" "$ac_cv_prog_cc_c11" >&6; } - CC="$CC $ac_cv_prog_cc_c11" + CC="$CC $ac_cv_prog_cc_c11" ;; +esac fi ac_cv_prog_cc_stdc=$ac_cv_prog_cc_c11 - ac_prog_cc_stdc=c11 + ac_prog_cc_stdc=c11 ;; +esac fi fi if test x$ac_prog_cc_stdc = xno @@ -5606,8 +5836,8 @@ printf %s "checking for $CC option to enable C99 features... " >&6; } if test ${ac_cv_prog_cc_c99+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_cv_prog_cc_c99=no +else case e in #( + e) ac_cv_prog_cc_c99=no ac_save_CC=$CC cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -5624,25 +5854,28 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam test "x$ac_cv_prog_cc_c99" != "xno" && break done rm -f conftest.$ac_ext -CC=$ac_save_CC +CC=$ac_save_CC ;; +esac fi if test "x$ac_cv_prog_cc_c99" = xno then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 printf "%s\n" "unsupported" >&6; } -else $as_nop - if test "x$ac_cv_prog_cc_c99" = x +else case e in #( + e) if test "x$ac_cv_prog_cc_c99" = x then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 printf "%s\n" "none needed" >&6; } -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c99" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c99" >&5 printf "%s\n" "$ac_cv_prog_cc_c99" >&6; } - CC="$CC $ac_cv_prog_cc_c99" + CC="$CC $ac_cv_prog_cc_c99" ;; +esac fi ac_cv_prog_cc_stdc=$ac_cv_prog_cc_c99 - ac_prog_cc_stdc=c99 + ac_prog_cc_stdc=c99 ;; +esac fi fi if test x$ac_prog_cc_stdc = xno @@ -5652,8 +5885,8 @@ printf %s "checking for $CC option to enable C89 features... " >&6; } if test ${ac_cv_prog_cc_c89+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_cv_prog_cc_c89=no +else case e in #( + e) ac_cv_prog_cc_c89=no ac_save_CC=$CC cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -5670,25 +5903,28 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam test "x$ac_cv_prog_cc_c89" != "xno" && break done rm -f conftest.$ac_ext -CC=$ac_save_CC +CC=$ac_save_CC ;; +esac fi if test "x$ac_cv_prog_cc_c89" = xno then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 printf "%s\n" "unsupported" >&6; } -else $as_nop - if test "x$ac_cv_prog_cc_c89" = x +else case e in #( + e) if test "x$ac_cv_prog_cc_c89" = x then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 printf "%s\n" "none needed" >&6; } -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c89" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c89" >&5 printf "%s\n" "$ac_cv_prog_cc_c89" >&6; } - CC="$CC $ac_cv_prog_cc_c89" + CC="$CC $ac_cv_prog_cc_c89" ;; +esac fi ac_cv_prog_cc_stdc=$ac_cv_prog_cc_c89 - ac_prog_cc_stdc=c89 + ac_prog_cc_stdc=c89 ;; +esac fi fi @@ -5713,8 +5949,8 @@ if test -z "$CPP"; then if test ${ac_cv_prog_CPP+y} then : printf %s "(cached) " >&6 -else $as_nop - # Double quotes because $CC needs to be expanded +else case e in #( + e) # Double quotes because $CC needs to be expanded for CPP in "$CC -E" "$CC -E -traditional-cpp" cpp /lib/cpp do ac_preproc_ok=false @@ -5732,9 +5968,10 @@ _ACEOF if ac_fn_c_try_cpp "$LINENO" then : -else $as_nop - # Broken: fails on valid input. -continue +else case e in #( + e) # Broken: fails on valid input. +continue ;; +esac fi rm -f conftest.err conftest.i conftest.$ac_ext @@ -5748,15 +5985,16 @@ if ac_fn_c_try_cpp "$LINENO" then : # Broken: success on invalid input. continue -else $as_nop - # Passes both tests. +else case e in #( + e) # Passes both tests. ac_preproc_ok=: -break +break ;; +esac fi rm -f conftest.err conftest.i conftest.$ac_ext done -# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. +# Because of 'break', _AC_PREPROC_IFELSE's cleaning code was skipped. rm -f conftest.i conftest.err conftest.$ac_ext if $ac_preproc_ok then : @@ -5765,7 +6003,8 @@ fi done ac_cv_prog_CPP=$CPP - + ;; +esac fi CPP=$ac_cv_prog_CPP else @@ -5788,9 +6027,10 @@ _ACEOF if ac_fn_c_try_cpp "$LINENO" then : -else $as_nop - # Broken: fails on valid input. -continue +else case e in #( + e) # Broken: fails on valid input. +continue ;; +esac fi rm -f conftest.err conftest.i conftest.$ac_ext @@ -5804,24 +6044,26 @@ if ac_fn_c_try_cpp "$LINENO" then : # Broken: success on invalid input. continue -else $as_nop - # Passes both tests. +else case e in #( + e) # Passes both tests. ac_preproc_ok=: -break +break ;; +esac fi rm -f conftest.err conftest.i conftest.$ac_ext done -# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. +# Because of 'break', _AC_PREPROC_IFELSE's cleaning code was skipped. rm -f conftest.i conftest.err conftest.$ac_ext if $ac_preproc_ok then : -else $as_nop - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error $? "C preprocessor \"$CPP\" fails sanity check -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } ;; +esac fi ac_ext=c @@ -5835,8 +6077,8 @@ printf %s "checking for grep that handles long lines and -e... " >&6; } if test ${ac_cv_path_GREP+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -z "$GREP"; then +else case e in #( + e) if test -z "$GREP"; then ac_path_GREP_found=false # Loop through the user's path and test for each of PROGNAME-LIST as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -5855,9 +6097,10 @@ do as_fn_executable_p "$ac_path_GREP" || continue # Check for GNU ac_path_GREP and select it if it is found. # Check for GNU $ac_path_GREP -case `"$ac_path_GREP" --version 2>&1` in +case `"$ac_path_GREP" --version 2>&1` in #( *GNU*) ac_cv_path_GREP="$ac_path_GREP" ac_path_GREP_found=:;; +#( *) ac_count=0 printf %s 0123456789 >"conftest.in" @@ -5892,7 +6135,8 @@ IFS=$as_save_IFS else ac_cv_path_GREP=$GREP fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_GREP" >&5 printf "%s\n" "$ac_cv_path_GREP" >&6; } @@ -5904,8 +6148,8 @@ printf %s "checking for a sed that does not truncate output... " >&6; } if test ${ac_cv_path_SED+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_script=s/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/ +else case e in #( + e) ac_script=s/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/ for ac_i in 1 2 3 4 5 6 7; do ac_script="$ac_script$as_nl$ac_script" done @@ -5930,9 +6174,10 @@ do as_fn_executable_p "$ac_path_SED" || continue # Check for GNU ac_path_SED and select it if it is found. # Check for GNU $ac_path_SED -case `"$ac_path_SED" --version 2>&1` in +case `"$ac_path_SED" --version 2>&1` in #( *GNU*) ac_cv_path_SED="$ac_path_SED" ac_path_SED_found=:;; +#( *) ac_count=0 printf %s 0123456789 >"conftest.in" @@ -5967,7 +6212,8 @@ IFS=$as_save_IFS else ac_cv_path_SED=$SED fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_SED" >&5 printf "%s\n" "$ac_cv_path_SED" >&6; } @@ -5979,8 +6225,8 @@ printf %s "checking for egrep... " >&6; } if test ${ac_cv_path_EGREP+y} then : printf %s "(cached) " >&6 -else $as_nop - if echo a | $GREP -E '(a|b)' >/dev/null 2>&1 +else case e in #( + e) if echo a | $GREP -E '(a|b)' >/dev/null 2>&1 then ac_cv_path_EGREP="$GREP -E" else if test -z "$EGREP"; then @@ -6002,9 +6248,10 @@ do as_fn_executable_p "$ac_path_EGREP" || continue # Check for GNU ac_path_EGREP and select it if it is found. # Check for GNU $ac_path_EGREP -case `"$ac_path_EGREP" --version 2>&1` in +case `"$ac_path_EGREP" --version 2>&1` in #( *GNU*) ac_cv_path_EGREP="$ac_path_EGREP" ac_path_EGREP_found=:;; +#( *) ac_count=0 printf %s 0123456789 >"conftest.in" @@ -6040,23 +6287,30 @@ else ac_cv_path_EGREP=$EGREP fi - fi + fi ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_EGREP" >&5 printf "%s\n" "$ac_cv_path_EGREP" >&6; } EGREP="$ac_cv_path_EGREP" + EGREP_TRADITIONAL=$EGREP + ac_cv_path_EGREP_TRADITIONAL=$EGREP +CC_BASENAME=$(expr "//$CC" : '.*/\(.*\)') + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for CC compiler name" >&5 printf %s "checking for CC compiler name... " >&6; } if test ${ac_cv_cc_name+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat > conftest.c <conftest.out 2>/dev/null; then ac_cv_cc_name=`grep -v '^#' conftest.out | grep -v '^ *$' | tr -d ' '` -else + if test "x$CC_BASENAME" = xmpicc +then : + ac_cv_cc_name=mpicc +fi +else ac_cv_cc_name="unknown" fi rm -f conftest.c conftest.out - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_cc_name" >&5 printf "%s\n" "$ac_cv_cc_name" >&6; } @@ -6126,8 +6385,8 @@ printf %s "checking whether it is safe to define __EXTENSIONS__... " >&6; } if test ${ac_cv_safe_to_define___extensions__+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ # define __EXTENSIONS__ 1 @@ -6143,10 +6402,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_safe_to_define___extensions__=yes -else $as_nop - ac_cv_safe_to_define___extensions__=no +else case e in #( + e) ac_cv_safe_to_define___extensions__=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_safe_to_define___extensions__" >&5 printf "%s\n" "$ac_cv_safe_to_define___extensions__" >&6; } @@ -6156,8 +6417,8 @@ printf %s "checking whether _XOPEN_SOURCE should be defined... " >&6; } if test ${ac_cv_should_define__xopen_source+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_cv_should_define__xopen_source=no +else case e in #( + e) ac_cv_should_define__xopen_source=no if test $ac_cv_header_wchar_h = yes then : cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -6176,8 +6437,8 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #define _XOPEN_SOURCE 500 @@ -6195,10 +6456,12 @@ if ac_fn_c_try_compile "$LINENO" then : ac_cv_should_define__xopen_source=yes fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext -fi +fi ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_should_define__xopen_source" >&5 printf "%s\n" "$ac_cv_should_define__xopen_source" >&6; } @@ -6223,6 +6486,8 @@ printf "%s\n" "$ac_cv_should_define__xopen_source" >&6; } printf "%s\n" "#define __STDC_WANT_IEC_60559_DFP_EXT__ 1" >>confdefs.h + printf "%s\n" "#define __STDC_WANT_IEC_60559_EXT__ 1" >>confdefs.h + printf "%s\n" "#define __STDC_WANT_IEC_60559_FUNCS_EXT__ 1" >>confdefs.h printf "%s\n" "#define __STDC_WANT_IEC_60559_TYPES_EXT__ 1" >>confdefs.h @@ -6242,8 +6507,9 @@ then : printf "%s\n" "#define _POSIX_1_SOURCE 2" >>confdefs.h -else $as_nop - MINIX= +else case e in #( + e) MINIX= ;; +esac fi if test $ac_cv_safe_to_define___extensions__ = yes then : @@ -6258,11 +6524,41 @@ fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for GCC compatible compiler" >&5 +printf %s "checking for GCC compatible compiler... " >&6; } +if test ${ac_cv_gcc_compat+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + #if !defined(__GNUC__) + #error "not GCC compatible" + #else + /* GCC compatible! */ + #endif + +_ACEOF +if ac_fn_c_try_cpp "$LINENO" +then : + ac_cv_gcc_compat=yes +else case e in #( + e) ac_cv_gcc_compat=no ;; +esac +fi +rm -f conftest.err conftest.i conftest.$ac_ext ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_gcc_compat" >&5 +printf "%s\n" "$ac_cv_gcc_compat" >&6; } + + preset_cxx="$CXX" if test -z "$CXX" then - case "$CC" in + case "$ac_cv_cc_name" in gcc) if test -n "$ac_tool_prefix"; then # Extract the first word of "${ac_tool_prefix}g++", so it can be a program name with args. set dummy ${ac_tool_prefix}g++; ac_word=$2 @@ -6271,8 +6567,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_CXX+y} then : printf %s "(cached) " >&6 -else $as_nop - case $CXX in +else case e in #( + e) case $CXX in [\\/]* | ?:[\\/]*) ac_cv_path_CXX="$CXX" # Let the user override the test with a path. ;; @@ -6297,6 +6593,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi CXX=$ac_cv_path_CXX @@ -6319,8 +6616,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_ac_pt_CXX+y} then : printf %s "(cached) " >&6 -else $as_nop - case $ac_pt_CXX in +else case e in #( + e) case $ac_pt_CXX in [\\/]* | ?:[\\/]*) ac_cv_path_ac_pt_CXX="$ac_pt_CXX" # Let the user override the test with a path. ;; @@ -6345,6 +6642,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi ac_pt_CXX=$ac_cv_path_ac_pt_CXX @@ -6379,8 +6677,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_CXX+y} then : printf %s "(cached) " >&6 -else $as_nop - case $CXX in +else case e in #( + e) case $CXX in [\\/]* | ?:[\\/]*) ac_cv_path_CXX="$CXX" # Let the user override the test with a path. ;; @@ -6405,6 +6703,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi CXX=$ac_cv_path_CXX @@ -6427,8 +6726,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_ac_pt_CXX+y} then : printf %s "(cached) " >&6 -else $as_nop - case $ac_pt_CXX in +else case e in #( + e) case $ac_pt_CXX in [\\/]* | ?:[\\/]*) ac_cv_path_ac_pt_CXX="$ac_pt_CXX" # Let the user override the test with a path. ;; @@ -6453,6 +6752,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi ac_pt_CXX=$ac_cv_path_ac_pt_CXX @@ -6479,7 +6779,7 @@ else CXX="$ac_cv_path_CXX" fi ;; - clang|*/clang) if test -n "$ac_tool_prefix"; then + clang) if test -n "$ac_tool_prefix"; then # Extract the first word of "${ac_tool_prefix}clang++", so it can be a program name with args. set dummy ${ac_tool_prefix}clang++; ac_word=$2 { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 @@ -6487,8 +6787,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_CXX+y} then : printf %s "(cached) " >&6 -else $as_nop - case $CXX in +else case e in #( + e) case $CXX in [\\/]* | ?:[\\/]*) ac_cv_path_CXX="$CXX" # Let the user override the test with a path. ;; @@ -6513,6 +6813,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi CXX=$ac_cv_path_CXX @@ -6535,8 +6836,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_ac_pt_CXX+y} then : printf %s "(cached) " >&6 -else $as_nop - case $ac_pt_CXX in +else case e in #( + e) case $ac_pt_CXX in [\\/]* | ?:[\\/]*) ac_cv_path_ac_pt_CXX="$ac_pt_CXX" # Let the user override the test with a path. ;; @@ -6561,6 +6862,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi ac_pt_CXX=$ac_cv_path_ac_pt_CXX @@ -6587,7 +6889,7 @@ else CXX="$ac_cv_path_CXX" fi ;; - icc|*/icc) if test -n "$ac_tool_prefix"; then + icc) if test -n "$ac_tool_prefix"; then # Extract the first word of "${ac_tool_prefix}icpc", so it can be a program name with args. set dummy ${ac_tool_prefix}icpc; ac_word=$2 { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 @@ -6595,8 +6897,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_CXX+y} then : printf %s "(cached) " >&6 -else $as_nop - case $CXX in +else case e in #( + e) case $CXX in [\\/]* | ?:[\\/]*) ac_cv_path_CXX="$CXX" # Let the user override the test with a path. ;; @@ -6621,6 +6923,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi CXX=$ac_cv_path_CXX @@ -6643,8 +6946,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_ac_pt_CXX+y} then : printf %s "(cached) " >&6 -else $as_nop - case $ac_pt_CXX in +else case e in #( + e) case $ac_pt_CXX in [\\/]* | ?:[\\/]*) ac_cv_path_ac_pt_CXX="$ac_pt_CXX" # Let the user override the test with a path. ;; @@ -6669,6 +6972,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi ac_pt_CXX=$ac_cv_path_ac_pt_CXX @@ -6713,8 +7017,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_CXX+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$CXX"; then +else case e in #( + e) if test -n "$CXX"; then ac_cv_prog_CXX="$CXX" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -6736,7 +7040,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi CXX=$ac_cv_prog_CXX if test -n "$CXX"; then @@ -6762,8 +7067,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_ac_ct_CXX+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$ac_ct_CXX"; then +else case e in #( + e) if test -n "$ac_ct_CXX"; then ac_cv_prog_ac_ct_CXX="$ac_ct_CXX" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -6785,7 +7090,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi ac_ct_CXX=$ac_cv_prog_ac_ct_CXX if test -n "$ac_ct_CXX"; then @@ -6892,20 +7198,20 @@ case $host/$ac_cv_cc_name in #( PY_SUPPORT_TIER=1 ;; #( x86_64-apple-darwin*/clang) : PY_SUPPORT_TIER=1 ;; #( + aarch64-apple-darwin*/clang) : + PY_SUPPORT_TIER=1 ;; #( i686-pc-windows-msvc/msvc) : PY_SUPPORT_TIER=1 ;; #( x86_64-pc-windows-msvc/msvc) : PY_SUPPORT_TIER=1 ;; #( - aarch64-apple-darwin*/clang) : - PY_SUPPORT_TIER=2 ;; #( - aarch64-*-linux-gnu/gcc) : + aarch64-*-linux-gnu/gcc) : PY_SUPPORT_TIER=2 ;; #( aarch64-*-linux-gnu/clang) : PY_SUPPORT_TIER=2 ;; #( powerpc64le-*-linux-gnu/gcc) : PY_SUPPORT_TIER=2 ;; #( - wasm32-unknown-wasi/clang) : + wasm32-unknown-wasip1/clang) : PY_SUPPORT_TIER=2 ;; #( x86_64-*-linux-gnu/clang) : PY_SUPPORT_TIER=2 ;; #( @@ -6924,8 +7230,13 @@ case $host/$ac_cv_cc_name in #( PY_SUPPORT_TIER=3 ;; #( aarch64-apple-ios*/clang) : PY_SUPPORT_TIER=3 ;; #( + aarch64-*-linux-android/clang) : + PY_SUPPORT_TIER=3 ;; #( + x86_64-*-linux-android/clang) : + PY_SUPPORT_TIER=3 ;; #( *) : - PY_SUPPORT_TIER=0 + + PY_SUPPORT_TIER=0 ;; esac @@ -6954,8 +7265,8 @@ printf %s "checking for -Wl,--no-as-needed... " >&6; } if test ${ac_cv_wl_no_as_needed+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) save_LDFLAGS="$LDFLAGS" as_fn_append LDFLAGS " -Wl,--no-as-needed" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -6973,14 +7284,16 @@ if ac_fn_c_try_link "$LINENO" then : NO_AS_NEEDED="-Wl,--no-as-needed" ac_cv_wl_no_as_needed=yes -else $as_nop - NO_AS_NEEDED="" - ac_cv_wl_no_as_needed=no +else case e in #( + e) NO_AS_NEEDED="" + ac_cv_wl_no_as_needed=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext LDFLAGS="$save_LDFLAGS" - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_wl_no_as_needed" >&5 printf "%s\n" "$ac_cv_wl_no_as_needed" >&6; } @@ -7009,6 +7322,9 @@ printf "%s\n" "$ANDROID_API_LEVEL" >&6; } printf "%s\n" "#define ANDROID_API_LEVEL $ANDROID_API_LEVEL" >>confdefs.h + # For __android_log_write() in Python/pylifecycle.c. + LIBS="$LIBS -llog" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for the Android arm ABI" >&5 printf %s "checking for the Android arm ABI... " >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $_arm_arch" >&5 @@ -7034,48 +7350,6 @@ case $ac_sys_system/$ac_sys_release in #( ;; esac -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-emscripten-target" >&5 -printf %s "checking for --with-emscripten-target... " >&6; } - -# Check whether --with-emscripten-target was given. -if test ${with_emscripten_target+y} -then : - withval=$with_emscripten_target; - if test "x$ac_sys_system" = xEmscripten -then : - - case $with_emscripten_target in #( - browser) : - ac_sys_emscripten_target=browser ;; #( - node) : - ac_sys_emscripten_target=node ;; #( - browser-debug) : - ac_sys_emscripten_target=browser-debug ;; #( - node-debug) : - ac_sys_emscripten_target=node-debug ;; #( - *) : - as_fn_error $? "Invalid argument: --with-emscripten-target=browser|node" "$LINENO" 5 - ;; -esac - -else $as_nop - - as_fn_error $? "--with-emscripten-target only applies to Emscripten" "$LINENO" 5 - -fi - -else $as_nop - - if test "x$ac_sys_system" = xEmscripten -then : - ac_sys_emscripten_target=browser -fi - -fi - -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_sys_emscripten_target" >&5 -printf "%s\n" "$ac_sys_emscripten_target" >&6; } - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-wasm-dynamic-linking" >&5 printf %s "checking for --enable-wasm-dynamic-linking... " >&6; } # Check whether --enable-wasm-dynamic-linking was given. @@ -7092,10 +7366,11 @@ then : ;; esac -else $as_nop - +else case e in #( + e) enable_wasm_dynamic_linking=missing - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_wasm_dynamic_linking" >&5 @@ -7117,10 +7392,11 @@ then : ;; esac -else $as_nop - +else case e in #( + e) enable_wasm_pthreads=missing - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_wasm_pthreads" >&5 @@ -7143,25 +7419,27 @@ then : ;; esac -else $as_nop - - case $ac_sys_system/$ac_sys_emscripten_target in #( - Emscripten/browser*) : - EXEEXT=.js ;; #( - Emscripten/node*) : - EXEEXT=.js ;; #( - WASI/*) : +else case e in #( + e) + case $ac_sys_system in #( + Emscripten) : + EXEEXT=.mjs ;; #( + WASI) : EXEEXT=.wasm ;; #( *) : EXEEXT= ;; esac - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $EXEEXT" >&5 printf "%s\n" "$EXEEXT" >&6; } +# Make sure we keep EXEEXT and ac_exeext sync'ed. +ac_exeext=$EXEEXT + # Test whether we're running on a non-case-sensitive system, in which # case we give a warning if no ext is given @@ -7185,7 +7463,7 @@ rmdir CaseSensitiveTestDir case $ac_sys_system in hp*|HP*) - case $CC in + case $ac_cv_cc_name in cc|*/cc) CC="$CC -Ae";; esac;; esac @@ -7278,7 +7556,7 @@ printf "%s\n" "$EXPORTSYMS" >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for GNU ld" >&5 printf %s "checking for GNU ld... " >&6; } ac_prog=ld -if test "$GCC" = yes; then +if test "$ac_cv_cc_name" = "gcc"; then ac_prog=`$CC -print-prog-name=ld` fi case `"$ac_prog" -V 2>&1 < /dev/null` in @@ -7329,9 +7607,10 @@ else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; }; fi -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } ;; +esac fi @@ -7354,8 +7633,9 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : -else $as_nop - enable_profiling=no +else case e in #( + e) enable_profiling=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext @@ -7477,13 +7757,13 @@ if test "$cross_compiling" = yes; then RUNSHARED= fi +# HOSTRUNNER - Program to run CPython for the host platform { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5 printf %s "checking HOSTRUNNER... " >&6; } - if test -z "$HOSTRUNNER" then - case $ac_sys_system/$ac_sys_emscripten_target in #( - Emscripten/node*) : + case $ac_sys_system in #( + Emscripten) : if test -n "$ac_tool_prefix"; then # Extract the first word of "${ac_tool_prefix}node", so it can be a program name with args. @@ -7493,8 +7773,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_NODE+y} then : printf %s "(cached) " >&6 -else $as_nop - case $NODE in +else case e in #( + e) case $NODE in [\\/]* | ?:[\\/]*) ac_cv_path_NODE="$NODE" # Let the user override the test with a path. ;; @@ -7519,6 +7799,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi NODE=$ac_cv_path_NODE @@ -7541,8 +7822,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_ac_pt_NODE+y} then : printf %s "(cached) " >&6 -else $as_nop - case $ac_pt_NODE in +else case e in #( + e) case $ac_pt_NODE in [\\/]* | ?:[\\/]*) ac_cv_path_ac_pt_NODE="$ac_pt_NODE" # Let the user override the test with a path. ;; @@ -7567,6 +7848,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi ac_pt_NODE=$ac_cv_path_ac_pt_NODE @@ -7594,68 +7876,13 @@ else fi HOSTRUNNER="$NODE" - # bigint for ctypes c_longlong, c_longdouble - # no longer available in Node 16 - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for node --experimental-wasm-bigint" >&5 -printf %s "checking for node --experimental-wasm-bigint... " >&6; } -if test ${ac_cv_tool_node_wasm_bigint+y} -then : - printf %s "(cached) " >&6 -else $as_nop - - if $NODE -v --experimental-wasm-bigint > /dev/null 2>&1; then - ac_cv_tool_node_wasm_bigint=yes - else - ac_cv_tool_node_wasm_bigint=no - fi - -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_tool_node_wasm_bigint" >&5 -printf "%s\n" "$ac_cv_tool_node_wasm_bigint" >&6; } - if test "x$ac_cv_tool_node_wasm_bigint" = xyes -then : - - as_fn_append HOSTRUNNER " --experimental-wasm-bigint" - -fi - - if test "x$enable_wasm_pthreads" = xyes -then : - - as_fn_append HOSTRUNNER " --experimental-wasm-threads" - # no longer available in Node 16 - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for node --experimental-wasm-bulk-memory" >&5 -printf %s "checking for node --experimental-wasm-bulk-memory... " >&6; } -if test ${ac_cv_tool_node_wasm_bulk_memory+y} -then : - printf %s "(cached) " >&6 -else $as_nop - - if $NODE -v --experimental-wasm-bulk-memory > /dev/null 2>&1; then - ac_cv_tool_node_wasm_bulk_memory=yes - else - ac_cv_tool_node_wasm_bulk_memory=no - fi - -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_tool_node_wasm_bulk_memory" >&5 -printf "%s\n" "$ac_cv_tool_node_wasm_bulk_memory" >&6; } - if test "x$ac_cv_tool_node_wasm_bulk_memory" = xyes -then : - - as_fn_append HOSTRUNNER " --experimental-wasm-bulk-memory" - -fi - -fi - if test "x$host_cpu" = xwasm64 then : as_fn_append HOSTRUNNER " --experimental-wasm-memory64" fi ;; #( - WASI/*) : - HOSTRUNNER='wasmtime run --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --mapdir /::$(srcdir) --' ;; #( + WASI) : + HOSTRUNNER='wasmtime run --wasm max-wasm-stack=16777216 --wasi preview2=n --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --dir $(srcdir)::/' ;; #( *) : HOSTRUNNER='' ;; @@ -7670,13 +7897,8 @@ if test -n "$HOSTRUNNER"; then fi # LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable -case $ac_sys_system/$ac_sys_emscripten_target in #( - Emscripten/browser*) : - LIBRARY_DEPS='$(PY3LIBRARY) $(WASM_STDLIB) python.html python.worker.js' ;; #( - *) : - LIBRARY_DEPS='$(PY3LIBRARY) $(EXPORTSYMS)' - ;; -esac +LIBRARY_DEPS='$(PY3LIBRARY) $(EXPORTSYMS)' + LINK_PYTHON_DEPS='$(LIBRARY_DEPS)' if test "$PY_ENABLE_SHARED" = 1 || test "$enable_framework" ; then LIBRARY_DEPS="\$(LDLIBRARY) $LIBRARY_DEPS" @@ -7711,8 +7933,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_AR+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$AR"; then +else case e in #( + e) if test -n "$AR"; then ac_cv_prog_AR="$AR" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -7734,7 +7956,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi AR=$ac_cv_prog_AR if test -n "$AR"; then @@ -7760,8 +7983,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_ac_ct_AR+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$ac_ct_AR"; then +else case e in #( + e) if test -n "$ac_ct_AR"; then ac_cv_prog_ac_ct_AR="$ac_ct_AR" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -7783,7 +8006,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi ac_ct_AR=$ac_cv_prog_ac_ct_AR if test -n "$ac_ct_AR"; then @@ -7848,8 +8072,8 @@ if test -z "$INSTALL"; then if test ${ac_cv_path_install+y} then : printf %s "(cached) " >&6 -else $as_nop - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +else case e in #( + e) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS @@ -7903,7 +8127,8 @@ esac IFS=$as_save_IFS rm -rf conftest.one conftest.two conftest.dir - + ;; +esac fi if test ${ac_cv_path_install+y}; then INSTALL=$ac_cv_path_install @@ -7933,8 +8158,8 @@ if test -z "$MKDIR_P"; then if test ${ac_cv_path_mkdir+y} then : printf %s "(cached) " >&6 -else $as_nop - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +else case e in #( + e) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH$PATH_SEPARATOR/opt/sfw/bin do IFS=$as_save_IFS @@ -7948,7 +8173,7 @@ do as_fn_executable_p "$as_dir$ac_prog$ac_exec_ext" || continue case `"$as_dir$ac_prog$ac_exec_ext" --version 2>&1` in #( 'mkdir ('*'coreutils) '* | \ - 'BusyBox '* | \ + *'BusyBox '* | \ 'mkdir (fileutils) '4.1*) ac_cv_path_mkdir=$as_dir$ac_prog$ac_exec_ext break 3;; @@ -7957,18 +8182,17 @@ do done done IFS=$as_save_IFS - + ;; +esac fi test -d ./--version && rmdir ./--version if test ${ac_cv_path_mkdir+y}; then MKDIR_P="$ac_cv_path_mkdir -p" else - # As a last resort, use the slow shell script. Don't cache a - # value for MKDIR_P within a source directory, because that will - # break other packages using the cache if that directory is - # removed, or if the value is a relative name. - MKDIR_P="$ac_install_sh -d" + # As a last resort, use plain mkdir -p, + # in the hope it doesn't have the bugs of ancient mkdir. + MKDIR_P='mkdir -p' fi fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $MKDIR_P" >&5 @@ -7986,7 +8210,9 @@ fi # For calculating the .so ABI tag. + ABIFLAGS="" +ABI_THREAD="" # Check for --disable-gil # --disable-gil @@ -7998,12 +8224,14 @@ then : enableval=$enable_gil; if test "x$enable_gil" = xyes then : disable_gil=no -else $as_nop - disable_gil=yes +else case e in #( + e) disable_gil=yes ;; +esac fi -else $as_nop - disable_gil=no - +else case e in #( + e) disable_gil=no + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $disable_gil" >&5 @@ -8016,6 +8244,7 @@ printf "%s\n" "#define Py_GIL_DISABLED 1" >>confdefs.h # Add "t" for "threaded" ABIFLAGS="${ABIFLAGS}t" + ABI_THREAD="t" fi # Check for --with-pydebug @@ -8038,9 +8267,10 @@ printf "%s\n" "yes" >&6; }; else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; }; Py_DEBUG='false' fi -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac fi @@ -8053,9 +8283,10 @@ printf %s "checking for --with-trace-refs... " >&6; } if test ${with_trace_refs+y} then : withval=$with_trace_refs; -else $as_nop - with_trace_refs=no - +else case e in #( + e) with_trace_refs=no + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_trace_refs" >&5 @@ -8068,6 +8299,10 @@ printf "%s\n" "#define Py_TRACE_REFS 1" >>confdefs.h fi +if test "$disable_gil" = "yes" -a "$with_trace_refs" = "yes"; +then + as_fn_error $? "--disable-gil cannot be used with --with-trace-refs" "$LINENO" 5 +fi # Check for --enable-pystats { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-pystats" >&5 @@ -8076,9 +8311,10 @@ printf %s "checking for --enable-pystats... " >&6; } if test ${enable_pystats+y} then : enableval=$enable_pystats; -else $as_nop - enable_pystats=no - +else case e in #( + e) enable_pystats=no + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_pystats" >&5 @@ -8128,26 +8364,43 @@ printf %s "checking for --enable-experimental-jit... " >&6; } if test ${enable_experimental_jit+y} then : enableval=$enable_experimental_jit; -else $as_nop - enable_experimental_jit=no +else case e in #( + e) enable_experimental_jit=no ;; +esac fi -if test "x$enable_experimental_jit" = xno +case $enable_experimental_jit in + no) jit_flags=""; tier2_flags="" ;; + yes) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=1" ;; + yes-off) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=3" ;; + interpreter) jit_flags=""; tier2_flags="-D_Py_TIER2=4" ;; + interpreter-off) jit_flags=""; tier2_flags="-D_Py_TIER2=6" ;; # Secret option + *) as_fn_error $? "invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter" "$LINENO" 5 ;; +esac +if ${tier2_flags:+false} : +then : + +else case e in #( + e) as_fn_append CFLAGS_NODIST " $tier2_flags" ;; +esac +fi +if ${jit_flags:+false} : then : -else $as_nop - as_fn_append CFLAGS_NODIST " -D_Py_JIT" +else case e in #( + e) as_fn_append CFLAGS_NODIST " $jit_flags" REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host" JIT_STENCILS_H="jit_stencils.h" if test "x$Py_DEBUG" = xtrue then : as_fn_append REGEN_JIT_COMMAND " --debug" -fi +fi ;; +esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_experimental_jit" >&5 -printf "%s\n" "$enable_experimental_jit" >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $tier2_flags $jit_flags" >&5 +printf "%s\n" "$tier2_flags $jit_flags" >&6; } # Enable optimization flags @@ -8169,9 +8422,10 @@ else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; }; fi -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac fi @@ -8183,17 +8437,18 @@ if test "$Py_OPT" = 'true' ; then DEF_MAKE_ALL_RULE="profile-opt" REQUIRE_PGO="yes" DEF_MAKE_RULE="build_all" - case $CC in - *gcc*) + if test "x$ac_cv_gcc_compat" = xyes +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -fno-semantic-interposition" >&5 printf %s "checking whether C compiler accepts -fno-semantic-interposition... " >&6; } -if test ${ax_cv_check_cflags___fno_semantic_interposition+y} +if test ${ax_cv_check_cflags__Werror__fno_semantic_interposition+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS - CFLAGS="$CFLAGS -fno-semantic-interposition" + CFLAGS="$CFLAGS -Werror -fno-semantic-interposition" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -8207,29 +8462,36 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ax_cv_check_cflags___fno_semantic_interposition=yes -else $as_nop - ax_cv_check_cflags___fno_semantic_interposition=no + ax_cv_check_cflags__Werror__fno_semantic_interposition=yes +else case e in #( + e) ax_cv_check_cflags__Werror__fno_semantic_interposition=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___fno_semantic_interposition" >&5 -printf "%s\n" "$ax_cv_check_cflags___fno_semantic_interposition" >&6; } -if test "x$ax_cv_check_cflags___fno_semantic_interposition" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__fno_semantic_interposition" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__fno_semantic_interposition" >&6; } +if test "x$ax_cv_check_cflags__Werror__fno_semantic_interposition" = xyes then : CFLAGS_NODIST="$CFLAGS_NODIST -fno-semantic-interposition" LDFLAGS_NODIST="$LDFLAGS_NODIST -fno-semantic-interposition" -else $as_nop - : +else case e in #( + e) : ;; +esac fi - ;; - esac -elif test "$ac_sys_system" = "Emscripten" -o "$ac_sys_system" = "WASI"; then - DEF_MAKE_ALL_RULE="build_wasm" + +fi +elif test "$ac_sys_system" = "Emscripten"; then + DEF_MAKE_ALL_RULE="build_emscripten" + REQUIRE_PGO="no" + DEF_MAKE_RULE="all" +elif test "$ac_sys_system" = "WASI"; then + DEF_MAKE_ALL_RULE="build_wasm" REQUIRE_PGO="no" DEF_MAKE_RULE="all" else @@ -8254,7 +8516,7 @@ printf "%s\n" "$PROFILE_TASK" >&6; } llvm_bin_dir='' llvm_path="${PATH}" -if test "${CC}" = "clang" +if test "${ac_cv_cc_name}" = "clang" then clang_bin=`which clang` # Some systems install clang elsewhere as a symlink to the real path @@ -8306,22 +8568,23 @@ printf "%s\n" "no" >&6; } ;; esac -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac fi if test "$Py_LTO" = 'true' ; then - case $CC in - *clang*) + case $ac_cv_cc_name in + clang) LDFLAGS_NOLTO="-fno-lto" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -flto=thin" >&5 printf %s "checking whether C compiler accepts -flto=thin... " >&6; } if test ${ax_cv_check_cflags___flto_thin+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS CFLAGS="$CFLAGS -flto=thin" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -8338,19 +8601,22 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ax_cv_check_cflags___flto_thin=yes -else $as_nop - ax_cv_check_cflags___flto_thin=no +else case e in #( + e) ax_cv_check_cflags___flto_thin=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___flto_thin" >&5 printf "%s\n" "$ax_cv_check_cflags___flto_thin" >&6; } if test "x$ax_cv_check_cflags___flto_thin" = xyes then : LDFLAGS_NOLTO="-flto=thin" -else $as_nop - LDFLAGS_NOLTO="-flto" +else case e in #( + e) LDFLAGS_NOLTO="-flto" ;; +esac fi @@ -8362,8 +8628,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_LLVM_AR+y} then : printf %s "(cached) " >&6 -else $as_nop - case $LLVM_AR in +else case e in #( + e) case $LLVM_AR in [\\/]* | ?:[\\/]*) ac_cv_path_LLVM_AR="$LLVM_AR" # Let the user override the test with a path. ;; @@ -8388,6 +8654,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi LLVM_AR=$ac_cv_path_LLVM_AR @@ -8410,8 +8677,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_ac_pt_LLVM_AR+y} then : printf %s "(cached) " >&6 -else $as_nop - case $ac_pt_LLVM_AR in +else case e in #( + e) case $ac_pt_LLVM_AR in [\\/]* | ?:[\\/]*) ac_cv_path_ac_pt_LLVM_AR="$ac_pt_LLVM_AR" # Let the user override the test with a path. ;; @@ -8436,6 +8703,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi ac_pt_LLVM_AR=$ac_cv_path_ac_pt_LLVM_AR @@ -8500,8 +8768,8 @@ printf %s "checking whether C compiler accepts -flto=thin... " >&6; } if test ${ax_cv_check_cflags___flto_thin+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS CFLAGS="$CFLAGS -flto=thin" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -8518,11 +8786,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ax_cv_check_cflags___flto_thin=yes -else $as_nop - ax_cv_check_cflags___flto_thin=no +else case e in #( + e) ax_cv_check_cflags___flto_thin=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___flto_thin" >&5 printf "%s\n" "$ax_cv_check_cflags___flto_thin" >&6; } @@ -8532,12 +8802,13 @@ then : LTOFLAGS="-flto=thin -Wl,-export_dynamic -Wl,-object_path_lto,\"\$@\".lto" LTOCFLAGS="-flto=thin" -else $as_nop - +else case e in #( + e) LTOFLAGS="-flto -Wl,-export_dynamic -Wl,-object_path_lto,\"\$@\".lto" LTOCFLAGS="-flto" - + ;; +esac fi else @@ -8554,8 +8825,8 @@ printf %s "checking whether C compiler accepts -flto=thin... " >&6; } if test ${ax_cv_check_cflags___flto_thin+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS CFLAGS="$CFLAGS -flto=thin" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -8572,19 +8843,22 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ax_cv_check_cflags___flto_thin=yes -else $as_nop - ax_cv_check_cflags___flto_thin=no +else case e in #( + e) ax_cv_check_cflags___flto_thin=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___flto_thin" >&5 printf "%s\n" "$ax_cv_check_cflags___flto_thin" >&6; } if test "x$ax_cv_check_cflags___flto_thin" = xyes then : LTOFLAGS="-flto=thin" -else $as_nop - LTOFLAGS="-flto" +else case e in #( + e) LTOFLAGS="-flto" ;; +esac fi else @@ -8593,14 +8867,14 @@ fi ;; esac ;; - *emcc*) + emcc) if test "$Py_LTO_POLICY" != "default"; then as_fn_error $? "emcc supports only default lto." "$LINENO" 5 fi LTOFLAGS="-flto" LTOCFLAGS="-flto" ;; - *gcc*) + gcc) if test $Py_LTO_POLICY = thin then as_fn_error $? "thin lto is not supported under gcc compiler." "$LINENO" 5 @@ -8644,8 +8918,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_LLVM_PROFDATA+y} then : printf %s "(cached) " >&6 -else $as_nop - case $LLVM_PROFDATA in +else case e in #( + e) case $LLVM_PROFDATA in [\\/]* | ?:[\\/]*) ac_cv_path_LLVM_PROFDATA="$LLVM_PROFDATA" # Let the user override the test with a path. ;; @@ -8670,6 +8944,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi LLVM_PROFDATA=$ac_cv_path_LLVM_PROFDATA @@ -8692,8 +8967,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_ac_pt_LLVM_PROFDATA+y} then : printf %s "(cached) " >&6 -else $as_nop - case $ac_pt_LLVM_PROFDATA in +else case e in #( + e) case $ac_pt_LLVM_PROFDATA in [\\/]* | ?:[\\/]*) ac_cv_path_ac_pt_LLVM_PROFDATA="$ac_pt_LLVM_PROFDATA" # Let the user override the test with a path. ;; @@ -8718,6 +8993,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi ac_pt_LLVM_PROFDATA=$ac_cv_path_ac_pt_LLVM_PROFDATA @@ -8766,15 +9042,13 @@ printf "%s\n" "$as_me: llvm-profdata found via xcrun: ${LLVM_PROFDATA}" >&6;} fi LLVM_PROF_ERR=no -# GNU Autoconf recommends the use of expr instead of basename. -CC_BASENAME=$(expr "//$CC" : '.*/\(.*\)') -case "$CC_BASENAME" in - *clang*) +case "$ac_cv_cc_name" in + clang) # Any changes made here should be reflected in the GCC+Darwin case below PGO_PROF_GEN_FLAG="-fprofile-instr-generate" - PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" - LLVM_PROF_MERGER="${LLVM_PROFDATA} merge -output=code.profclangd *.profclangr" - LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + PGO_PROF_USE_FLAG="-fprofile-instr-use=\"\$(shell pwd)/code.profclangd\"" + LLVM_PROF_MERGER=" ${LLVM_PROFDATA} merge -output=\"\$(shell pwd)/code.profclangd\" \"\$(shell pwd)\"/*.profclangr " + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"\$(shell pwd)/code-%p.profclangr\"" if test $LLVM_PROF_FOUND = not-found then LLVM_PROF_ERR=yes @@ -8784,31 +9058,13 @@ case "$CC_BASENAME" in fi fi ;; - *gcc*) - case $ac_sys_system in - Darwin*) - PGO_PROF_GEN_FLAG="-fprofile-instr-generate" - PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" - LLVM_PROF_MERGER="${LLVM_PROFDATA} merge -output=code.profclangd *.profclangr" - LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" - if test "${LLVM_PROF_FOUND}" = "not-found" - then - LLVM_PROF_ERR=yes - if test "${REQUIRE_PGO}" = "yes" - then - as_fn_error $? "llvm-profdata is required for a --enable-optimizations build but could not be found." "$LINENO" 5 - fi - fi - ;; - *) - PGO_PROF_GEN_FLAG="-fprofile-generate" - PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" - LLVM_PROF_MERGER="true" - LLVM_PROF_FILE="" - ;; - esac + gcc) + PGO_PROF_GEN_FLAG="-fprofile-generate" + PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" + LLVM_PROF_MERGER="true" + LLVM_PROF_FILE="" ;; - *icc*) + icc) PGO_PROF_GEN_FLAG="-prof-gen" PGO_PROF_USE_FLAG="-prof-use" LLVM_PROF_MERGER="true" @@ -8834,9 +9090,10 @@ else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; }; fi -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac fi @@ -8853,8 +9110,8 @@ printf %s "checking whether C compiler accepts -fno-reorder-blocks-and-partition if test ${ax_cv_check_cflags___fno_reorder_blocks_and_partition+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS CFLAGS="$CFLAGS -fno-reorder-blocks-and-partition" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -8871,11 +9128,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ax_cv_check_cflags___fno_reorder_blocks_and_partition=yes -else $as_nop - ax_cv_check_cflags___fno_reorder_blocks_and_partition=no +else case e in #( + e) ax_cv_check_cflags___fno_reorder_blocks_and_partition=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___fno_reorder_blocks_and_partition" >&5 printf "%s\n" "$ax_cv_check_cflags___fno_reorder_blocks_and_partition" >&6; } @@ -8884,8 +9143,9 @@ then : CFLAGS_NODIST="$CFLAGS_NODIST -fno-reorder-blocks-and-partition" -else $as_nop - : +else case e in #( + e) : ;; +esac fi @@ -8905,8 +9165,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_LLVM_BOLT+y} then : printf %s "(cached) " >&6 -else $as_nop - case $LLVM_BOLT in +else case e in #( + e) case $LLVM_BOLT in [\\/]* | ?:[\\/]*) ac_cv_path_LLVM_BOLT="$LLVM_BOLT" # Let the user override the test with a path. ;; @@ -8931,6 +9191,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi LLVM_BOLT=$ac_cv_path_LLVM_BOLT @@ -8953,8 +9214,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_ac_pt_LLVM_BOLT+y} then : printf %s "(cached) " >&6 -else $as_nop - case $ac_pt_LLVM_BOLT in +else case e in #( + e) case $ac_pt_LLVM_BOLT in [\\/]* | ?:[\\/]*) ac_cv_path_ac_pt_LLVM_BOLT="$ac_pt_LLVM_BOLT" # Let the user override the test with a path. ;; @@ -8979,6 +9240,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi ac_pt_LLVM_BOLT=$ac_cv_path_ac_pt_LLVM_BOLT @@ -9022,8 +9284,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_MERGE_FDATA+y} then : printf %s "(cached) " >&6 -else $as_nop - case $MERGE_FDATA in +else case e in #( + e) case $MERGE_FDATA in [\\/]* | ?:[\\/]*) ac_cv_path_MERGE_FDATA="$MERGE_FDATA" # Let the user override the test with a path. ;; @@ -9048,6 +9310,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi MERGE_FDATA=$ac_cv_path_MERGE_FDATA @@ -9070,8 +9333,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_ac_pt_MERGE_FDATA+y} then : printf %s "(cached) " >&6 -else $as_nop - case $ac_pt_MERGE_FDATA in +else case e in #( + e) case $ac_pt_MERGE_FDATA in [\\/]* | ?:[\\/]*) ac_cv_path_ac_pt_MERGE_FDATA="$ac_pt_MERGE_FDATA" # Let the user override the test with a path. ;; @@ -9096,6 +9359,7 @@ done IFS=$as_save_IFS ;; +esac ;; esac fi ac_pt_MERGE_FDATA=$ac_cv_path_ac_pt_MERGE_FDATA @@ -9141,11 +9405,21 @@ then : fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking BOLT_COMMON_FLAGS" >&5 +printf %s "checking BOLT_COMMON_FLAGS... " >&6; } +if test -z "${BOLT_COMMON_FLAGS}" +then + BOLT_COMMON_FLAGS=" -update-debug-sections -skip-funcs=_PyEval_EvalFrameDefault,sre_ucs1_match/1,sre_ucs2_match/1,sre_ucs4_match/1 " + +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking BOLT_INSTRUMENT_FLAGS" >&5 printf %s "checking BOLT_INSTRUMENT_FLAGS... " >&6; } if test -z "${BOLT_INSTRUMENT_FLAGS}" then - BOLT_INSTRUMENT_FLAGS= + BOLT_INSTRUMENT_FLAGS="${BOLT_COMMON_FLAGS}" fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $BOLT_INSTRUMENT_FLAGS" >&5 printf "%s\n" "$BOLT_INSTRUMENT_FLAGS" >&6; } @@ -9155,7 +9429,7 @@ printf "%s\n" "$BOLT_INSTRUMENT_FLAGS" >&6; } printf %s "checking BOLT_APPLY_FLAGS... " >&6; } if test -z "${BOLT_APPLY_FLAGS}" then - BOLT_APPLY_FLAGS=" -update-debug-sections -reorder-blocks=ext-tsp -reorder-functions=hfsort+ -split-functions -icf=1 -inline-all -split-eh -reorder-functions-use-hot-size -peepholes=none -jump-tables=aggressive -inline-ap -indirect-call-promotion=all -dyno-stats -use-gnu-stack -frame-opt=hot " + BOLT_APPLY_FLAGS=" ${BOLT_COMMON_FLAGS} -reorder-blocks=ext-tsp -reorder-functions=cdsort -split-functions -icf=1 -inline-all -split-eh -reorder-functions-use-hot-size -peepholes=none -jump-tables=aggressive -inline-ap -indirect-call-promotion=all -dyno-stats -use-gnu-stack -frame-opt=hot " fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $BOLT_APPLY_FLAGS" >&5 @@ -9174,19 +9448,6 @@ printf "%s\n" "$BOLT_APPLY_FLAGS" >&6; } # compiler and platform. BASECFLAGS tweaks need to be made even if the # user set OPT. -case $CC in - *clang*) - cc_is_clang=1 - ;; - *) - if $CC --version 2>&1 | grep -q clang - then - cc_is_clang=1 - else - cc_is_clang= - fi -esac - save_CFLAGS=$CFLAGS CFLAGS="-fstrict-overflow -fno-strict-overflow" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if $CC supports -fstrict-overflow and -fno-strict-overflow" >&5 @@ -9194,8 +9455,8 @@ printf %s "checking if $CC supports -fstrict-overflow and -fno-strict-overflow.. if test ${ac_cv_cc_supports_fstrict_overflow+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int @@ -9209,12 +9470,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_cc_supports_fstrict_overflow=yes -else $as_nop - ac_cv_cc_supports_fstrict_overflow=no - +else case e in #( + e) ac_cv_cc_supports_fstrict_overflow=no + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_cc_supports_fstrict_overflow" >&5 printf "%s\n" "$ac_cv_cc_supports_fstrict_overflow" >&6; } @@ -9224,9 +9487,10 @@ if test "x$ac_cv_cc_supports_fstrict_overflow" = xyes then : STRICT_OVERFLOW_CFLAGS="-fstrict-overflow" NO_STRICT_OVERFLOW_CFLAGS="-fno-strict-overflow" -else $as_nop - STRICT_OVERFLOW_CFLAGS="" - NO_STRICT_OVERFLOW_CFLAGS="" +else case e in #( + e) STRICT_OVERFLOW_CFLAGS="" + NO_STRICT_OVERFLOW_CFLAGS="" ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-strict-overflow" >&5 @@ -9242,9 +9506,10 @@ then : printf "%s\n" "$as_me: WARNING: --with-strict-overflow=yes requires a compiler that supports -fstrict-overflow" >&2;} fi -else $as_nop - with_strict_overflow=no - +else case e in #( + e) with_strict_overflow=no + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_strict_overflow" >&5 @@ -9258,8 +9523,8 @@ printf %s "checking if $CC supports -Og optimization level... " >&6; } if test ${ac_cv_cc_supports_og+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9277,13 +9542,15 @@ then : ac_cv_cc_supports_og=yes -else $as_nop - +else case e in #( + e) ac_cv_cc_supports_og=no - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_cc_supports_og" >&5 printf "%s\n" "$ac_cv_cc_supports_og" >&6; } @@ -9297,6 +9564,11 @@ then : PYDEBUG_CFLAGS="-Og" fi +# gh-120688: WASI uses -O3 in debug mode to support more recursive calls +if test "$ac_sys_system" = "WASI"; then + PYDEBUG_CFLAGS="-O3" +fi + # tweak OPT based on compiler and platform, only if the user didn't set # it on the command line @@ -9305,7 +9577,7 @@ if test "${OPT-unset}" = "unset" then case $GCC in yes) - if test -n "${cc_is_clang}" + if test "${ac_cv_cc_name}" != "clang" then # bpo-30104: disable strict aliasing to compile correctly dtoa.c, # see Makefile.pre.in for more information @@ -9344,15 +9616,19 @@ case $ac_sys_system in #( if test "x$Py_DEBUG" = xyes then : wasm_debug=yes -else $as_nop - wasm_debug=no +else case e in #( + e) wasm_debug=no ;; +esac fi - as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH -sTOTAL_MEMORY=20971520" + as_fn_append LINKFORSHARED " -sALLOW_MEMORY_GROWTH -sINITIAL_MEMORY=20971520" as_fn_append LDFLAGS_NODIST " -sWASM_BIGINT" - as_fn_append LDFLAGS_NODIST " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" + as_fn_append LINKFORSHARED " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" + as_fn_append LINKFORSHARED " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV" + as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET" + as_fn_append LINKFORSHARED " -sSTACK_SIZE=5MB" if test "x$enable_wasm_dynamic_linking" = xyes then : @@ -9369,44 +9645,20 @@ then : as_fn_append LINKFORSHARED " -sPROXY_TO_PTHREAD" fi + as_fn_append LDFLAGS_NODIST " -sEXIT_RUNTIME" + WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map" - case $ac_sys_emscripten_target in #( - browser*) : - - if test "x$ac_sys_emscripten_target" = xbrowser-debug -then : - wasm_debug=yes -fi - as_fn_append LINKFORSHARED " --preload-file=\$(WASM_ASSETS_DIR)" - WASM_ASSETS_DIR=".\$(prefix)" - WASM_STDLIB="\$(WASM_ASSETS_DIR)/local/lib/python\$(VERSION)/os.py" - WASM_LINKFORSHARED_DEBUG="-gsource-map --emit-symbol-map" - ;; #( - node*) : - - if test "x$ac_sys_emscripten_target" = xnode-debug -then : - wasm_debug=yes -fi - as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH -sNODERAWFS" - as_fn_append LINKFORSHARED " -sEXIT_RUNTIME" - WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map" - - ;; #( - *) : - ;; -esac - - if test "x$wasm_debug" = xyes + if test "x$wasm_debug" = xyes then : as_fn_append LDFLAGS_NODIST " -sASSERTIONS" as_fn_append LINKFORSHARED " $WASM_LINKFORSHARED_DEBUG" -else $as_nop - +else case e in #( + e) as_fn_append LINKFORSHARED " -O2 -g0" - + ;; +esac fi ;; #( WASI) : @@ -9443,7 +9695,7 @@ then : fi - as_fn_append LDFLAGS_NODIST " -z stack-size=524288 -Wl,--stack-first -Wl,--initial-memory=10485760" + as_fn_append LDFLAGS_NODIST " -z stack-size=16777216 -Wl,--stack-first -Wl,--initial-memory=41943040" ;; #( *) : @@ -9479,12 +9731,359 @@ UNIVERSAL_ARCH_FLAGS= if test "x$with_strict_overflow" = xyes then : BASECFLAGS="$BASECFLAGS $STRICT_OVERFLOW_CFLAGS" -else $as_nop - BASECFLAGS="$BASECFLAGS $NO_STRICT_OVERFLOW_CFLAGS" +else case e in #( + e) BASECFLAGS="$BASECFLAGS $NO_STRICT_OVERFLOW_CFLAGS" ;; +esac fi -case $GCC in -yes) +# Enable flags that warn and protect for potential security vulnerabilities. +# These flags should be enabled by default for all builds. + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-safety" >&5 +printf %s "checking for --enable-safety... " >&6; } +# Check whether --enable-safety was given. +if test ${enable_safety+y} +then : + enableval=$enable_safety; if test "x$disable_safety" = xyes +then : + enable_safety=no +else case e in #( + e) enable_safety=yes ;; +esac +fi +else case e in #( + e) enable_safety=no ;; +esac +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_safety" >&5 +printf "%s\n" "$enable_safety" >&6; } + +if test "$enable_safety" = "yes" +then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -fstack-protector-strong" >&5 +printf %s "checking whether C compiler accepts -fstack-protector-strong... " >&6; } +if test ${ax_cv_check_cflags__Werror__fstack_protector_strong+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -fstack-protector-strong" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__fstack_protector_strong=yes +else case e in #( + e) ax_cv_check_cflags__Werror__fstack_protector_strong=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__fstack_protector_strong" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__fstack_protector_strong" >&6; } +if test "x$ax_cv_check_cflags__Werror__fstack_protector_strong" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -fstack-protector-strong" +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -fstack-protector-strong not supported" >&5 +printf "%s\n" "$as_me: WARNING: -fstack-protector-strong not supported" >&2;} ;; +esac +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wtrampolines" >&5 +printf %s "checking whether C compiler accepts -Wtrampolines... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wtrampolines+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wtrampolines" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__Wtrampolines=yes +else case e in #( + e) ax_cv_check_cflags__Werror__Wtrampolines=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wtrampolines" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wtrampolines" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wtrampolines" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Wtrampolines" +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wtrampolines not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wtrampolines not supported" >&2;} ;; +esac +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wimplicit-fallthrough" >&5 +printf %s "checking whether C compiler accepts -Wimplicit-fallthrough... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wimplicit_fallthrough+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wimplicit-fallthrough" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__Wimplicit_fallthrough=yes +else case e in #( + e) ax_cv_check_cflags__Werror__Wimplicit_fallthrough=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wimplicit_fallthrough" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wimplicit_fallthrough" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wimplicit_fallthrough" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Wimplicit-fallthrough" +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wimplicit-fallthrough not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wimplicit-fallthrough not supported" >&2;} ;; +esac +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Werror=format-security" >&5 +printf %s "checking whether C compiler accepts -Werror=format-security... " >&6; } +if test ${ax_cv_check_cflags__Werror__Werror_format_security+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Werror=format-security" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__Werror_format_security=yes +else case e in #( + e) ax_cv_check_cflags__Werror__Werror_format_security=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Werror_format_security" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Werror_format_security" >&6; } +if test "x$ax_cv_check_cflags__Werror__Werror_format_security" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Werror=format-security" +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Werror=format-security not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Werror=format-security not supported" >&2;} ;; +esac +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wbidi-chars=any" >&5 +printf %s "checking whether C compiler accepts -Wbidi-chars=any... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wbidi_chars_any+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wbidi-chars=any" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__Wbidi_chars_any=yes +else case e in #( + e) ax_cv_check_cflags__Werror__Wbidi_chars_any=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wbidi_chars_any" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wbidi_chars_any" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wbidi_chars_any" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Wbidi-chars=any" +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wbidi-chars=any not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wbidi-chars=any not supported" >&2;} ;; +esac +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wall" >&5 +printf %s "checking whether C compiler accepts -Wall... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wall+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wall" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__Wall=yes +else case e in #( + e) ax_cv_check_cflags__Werror__Wall=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wall" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wall" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wall" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Wall" +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wall not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wall not supported" >&2;} ;; +esac +fi + +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-slower-safety" >&5 +printf %s "checking for --enable-slower-safety... " >&6; } +# Check whether --enable-slower-safety was given. +if test ${enable_slower_safety+y} +then : + enableval=$enable_slower_safety; if test "x$disable_slower_safety" = xyes +then : + enable_slower_safety=no +else case e in #( + e) enable_slower_safety=yes ;; +esac +fi +else case e in #( + e) enable_slower_safety=no ;; +esac +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_slower_safety" >&5 +printf "%s\n" "$enable_slower_safety" >&6; } + +if test "$enable_slower_safety" = "yes" +then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -D_FORTIFY_SOURCE=3" >&5 +printf %s "checking whether C compiler accepts -D_FORTIFY_SOURCE=3... " >&6; } +if test ${ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -D_FORTIFY_SOURCE=3" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3=yes +else case e in #( + e) ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3" >&6; } +if test "x$ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3" +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -D_FORTIFY_SOURCE=3 not supported" >&5 +printf "%s\n" "$as_me: WARNING: -D_FORTIFY_SOURCE=3 not supported" >&2;} ;; +esac +fi + +fi + +if test "x$ac_cv_gcc_compat" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -std=c11" @@ -9494,10 +10093,10 @@ printf %s "checking if we can add -Wextra... " >&6; } if test ${ac_cv_enable_extra_warning+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wextra -Werror" + as_fn_append CFLAGS " -Wextra -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9512,12 +10111,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_enable_extra_warning=yes -else $as_nop - ac_cv_enable_extra_warning=no +else case e in #( + e) ac_cv_enable_extra_warning=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS=$py_cflags - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_extra_warning" >&5 printf "%s\n" "$ac_cv_enable_extra_warning" >&6; } @@ -9540,8 +10141,8 @@ printf %s "checking whether $CC accepts and needs -fno-strict-aliasing... " >&6; if test ${ac_cv_no_strict_aliasing+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9577,19 +10178,22 @@ then : ac_cv_no_strict_aliasing=no -else $as_nop - +else case e in #( + e) ac_cv_no_strict_aliasing=yes - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext -else $as_nop - +else case e in #( + e) ac_cv_no_strict_aliasing=no - + ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_no_strict_aliasing" >&5 printf "%s\n" "$ac_cv_no_strict_aliasing" >&6; } @@ -9602,8 +10206,8 @@ fi # ICC doesn't recognize the option, but only emits a warning ## XXX does it emit an unused result warning and can it be disabled? - case "$CC_BASENAME" in #( - *icc*) : + case "$ac_cv_cc_name" in #( + icc) : ac_cv_disable_unused_result_warning=no @@ -9612,10 +10216,10 @@ printf %s "checking if we can disable $CC unused-result warning... " >&6; } if test ${ac_cv_disable_unused_result_warning+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wunused-result -Werror" + as_fn_append CFLAGS " -Wunused-result -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9630,12 +10234,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_disable_unused_result_warning=yes -else $as_nop - ac_cv_disable_unused_result_warning=no +else case e in #( + e) ac_cv_disable_unused_result_warning=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS=$py_cflags - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_disable_unused_result_warning" >&5 printf "%s\n" "$ac_cv_disable_unused_result_warning" >&6; } @@ -9657,10 +10263,10 @@ printf %s "checking if we can disable $CC unused-parameter warning... " >&6; } if test ${ac_cv_disable_unused_parameter_warning+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wunused-parameter -Werror" + as_fn_append CFLAGS " -Wunused-parameter -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9675,12 +10281,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_disable_unused_parameter_warning=yes -else $as_nop - ac_cv_disable_unused_parameter_warning=no +else case e in #( + e) ac_cv_disable_unused_parameter_warning=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS=$py_cflags - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_disable_unused_parameter_warning" >&5 printf "%s\n" "$ac_cv_disable_unused_parameter_warning" >&6; } @@ -9698,10 +10306,10 @@ printf %s "checking if we can disable $CC int-conversion warning... " >&6; } if test ${ac_cv_disable_int_conversion_warning+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wint-conversion -Werror" + as_fn_append CFLAGS " -Wint-conversion -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9716,12 +10324,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_disable_int_conversion_warning=yes -else $as_nop - ac_cv_disable_int_conversion_warning=no +else case e in #( + e) ac_cv_disable_int_conversion_warning=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS=$py_cflags - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_disable_int_conversion_warning" >&5 printf "%s\n" "$ac_cv_disable_int_conversion_warning" >&6; } @@ -9739,10 +10349,10 @@ printf %s "checking if we can disable $CC missing-field-initializers warning... if test ${ac_cv_disable_missing_field_initializers_warning+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wmissing-field-initializers -Werror" + as_fn_append CFLAGS " -Wmissing-field-initializers -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9757,12 +10367,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_disable_missing_field_initializers_warning=yes -else $as_nop - ac_cv_disable_missing_field_initializers_warning=no +else case e in #( + e) ac_cv_disable_missing_field_initializers_warning=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS=$py_cflags - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_disable_missing_field_initializers_warning" >&5 printf "%s\n" "$ac_cv_disable_missing_field_initializers_warning" >&6; } @@ -9780,10 +10392,10 @@ printf %s "checking if we can enable $CC sign-compare warning... " >&6; } if test ${ac_cv_enable_sign_compare_warning+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wsign-compare -Werror" + as_fn_append CFLAGS " -Wsign-compare -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9798,12 +10410,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_enable_sign_compare_warning=yes -else $as_nop - ac_cv_enable_sign_compare_warning=no +else case e in #( + e) ac_cv_enable_sign_compare_warning=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS=$py_cflags - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_sign_compare_warning" >&5 printf "%s\n" "$ac_cv_enable_sign_compare_warning" >&6; } @@ -9821,10 +10435,10 @@ printf %s "checking if we can enable $CC unreachable-code warning... " >&6; } if test ${ac_cv_enable_unreachable_code_warning+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wunreachable-code -Werror" + as_fn_append CFLAGS " -Wunreachable-code -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9839,12 +10453,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_enable_unreachable_code_warning=yes -else $as_nop - ac_cv_enable_unreachable_code_warning=no +else case e in #( + e) ac_cv_enable_unreachable_code_warning=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS=$py_cflags - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_unreachable_code_warning" >&5 printf "%s\n" "$ac_cv_enable_unreachable_code_warning" >&6; } @@ -9873,10 +10489,10 @@ printf %s "checking if we can enable $CC strict-prototypes warning... " >&6; } if test ${ac_cv_enable_strict_prototypes_warning+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) py_cflags=$CFLAGS - as_fn_append CFLAGS "-Wstrict-prototypes -Werror" + as_fn_append CFLAGS " -Wstrict-prototypes -Werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9891,12 +10507,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_enable_strict_prototypes_warning=yes -else $as_nop - ac_cv_enable_strict_prototypes_warning=no +else case e in #( + e) ac_cv_enable_strict_prototypes_warning=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS=$py_cflags - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_strict_prototypes_warning" >&5 printf "%s\n" "$ac_cv_enable_strict_prototypes_warning" >&6; } @@ -9914,8 +10532,8 @@ printf %s "checking if we can make implicit function declaration an error in $CC if test ${ac_cv_enable_implicit_function_declaration_error+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9933,12 +10551,14 @@ then : ac_cv_enable_implicit_function_declaration_error=yes -else $as_nop - +else case e in #( + e) ac_cv_enable_implicit_function_declaration_error=no - + ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_implicit_function_declaration_error" >&5 printf "%s\n" "$ac_cv_enable_implicit_function_declaration_error" >&6; } @@ -9956,8 +10576,8 @@ printf %s "checking if we can use visibility in $CC... " >&6; } if test ${ac_cv_enable_visibility+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9975,12 +10595,14 @@ then : ac_cv_enable_visibility=yes -else $as_nop - +else case e in #( + e) ac_cv_enable_visibility=no - + ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_visibility" >&5 printf "%s\n" "$ac_cv_enable_visibility" >&6; } @@ -10008,22 +10630,63 @@ fi Darwin*) # -Wno-long-double, -no-cpp-precomp, and -mno-fused-madd # used to be here, but non-Apple gcc doesn't accept them. - if test "${CC}" = gcc - then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking which compiler should be used" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking which compiler should be used" >&5 printf %s "checking which compiler should be used... " >&6; } - case "${UNIVERSALSDK}" in - */MacOSX10.4u.sdk) - # Build using 10.4 SDK, force usage of gcc when the - # compiler is gcc, otherwise the user will get very - # confusing error messages when building on OSX 10.6 - CC=gcc-4.0 - CPP=cpp-4.0 - ;; - esac - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 + case "${UNIVERSALSDK}" in + */MacOSX10.4u.sdk) + # Build using 10.4 SDK, force usage of gcc when the + # compiler is gcc, otherwise the user will get very + # confusing error messages when building on OSX 10.6 + CC=gcc-4.0 + CPP=cpp-4.0 + ;; + esac + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 printf "%s\n" "$CC" >&6; } - fi + + # Error on unguarded use of new symbols, which will fail at runtime for + # users on older versions of macOS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wunguarded-availability" >&5 +printf %s "checking whether C compiler accepts -Wunguarded-availability... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wunguarded_availability+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wunguarded-availability" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__Wunguarded_availability=yes +else case e in #( + e) ax_cv_check_cflags__Werror__Wunguarded_availability=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wunguarded_availability" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wunguarded_availability" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wunguarded_availability" = xyes +then : + as_fn_append CFLAGS_NODIST " -Werror=unguarded-availability" +else case e in #( + e) : ;; +esac +fi + LIPO_INTEL64_FLAGS="" if test "${enable_universalsdk}" @@ -10157,11 +10820,12 @@ if ac_fn_c_try_link "$LINENO" then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } as_fn_error $? "check config.log and use the '--with-universal-archs' option" "$LINENO" 5 - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext @@ -10169,9 +10833,9 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ # end of Darwin* tests ;; esac - ;; -*) +else case e in #( + e) case $ac_sys_system in OpenUNIX*|UnixWare*) BASECFLAGS="$BASECFLAGS -K pentium,host,inline,loop_unroll,alloca " @@ -10180,18 +10844,19 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ BASECFLAGS="$BASECFLAGS -belf -Ki486 -DSCO5" ;; esac - ;; + ;; esac +fi -case "$CC_BASENAME" in -*mpicc*) +case "$ac_cv_cc_name" in +mpicc) CFLAGS_NODIST="$CFLAGS_NODIST" ;; -*icc*) +icc) # ICC needs -fp-model strict or floats behave badly CFLAGS_NODIST="$CFLAGS_NODIST -fp-model strict" ;; -*xlc*) +xlc) CFLAGS_NODIST="$CFLAGS_NODIST -qalias=noansi -qmaxmem=-1" ;; esac @@ -10217,12 +10882,12 @@ printf %s "checking whether pthreads are available without options... " >&6; } if test ${ac_cv_pthread_is_default+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_pthread_is_default=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -10246,14 +10911,17 @@ then : ac_cv_kthread=no ac_cv_pthread=no -else $as_nop - ac_cv_pthread_is_default=no +else case e in #( + e) ac_cv_pthread_is_default=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_pthread_is_default" >&5 printf "%s\n" "$ac_cv_pthread_is_default" >&6; } @@ -10273,14 +10941,14 @@ printf %s "checking whether $CC accepts -Kpthread... " >&6; } if test ${ac_cv_kpthread+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_save_cc="$CC" +else case e in #( + e) ac_save_cc="$CC" CC="$CC -Kpthread" if test "$cross_compiling" = yes then : ac_cv_kpthread=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -10300,14 +10968,17 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_kpthread=yes -else $as_nop - ac_cv_kpthread=no +else case e in #( + e) ac_cv_kpthread=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi -CC="$ac_save_cc" +CC="$ac_save_cc" ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_kpthread" >&5 printf "%s\n" "$ac_cv_kpthread" >&6; } @@ -10325,14 +10996,14 @@ printf %s "checking whether $CC accepts -Kthread... " >&6; } if test ${ac_cv_kthread+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_save_cc="$CC" +else case e in #( + e) ac_save_cc="$CC" CC="$CC -Kthread" if test "$cross_compiling" = yes then : ac_cv_kthread=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -10352,14 +11023,17 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_kthread=yes -else $as_nop - ac_cv_kthread=no +else case e in #( + e) ac_cv_kthread=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi -CC="$ac_save_cc" +CC="$ac_save_cc" ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_kthread" >&5 printf "%s\n" "$ac_cv_kthread" >&6; } @@ -10377,14 +11051,14 @@ printf %s "checking whether $CC accepts -pthread... " >&6; } if test ${ac_cv_pthread+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_save_cc="$CC" +else case e in #( + e) ac_save_cc="$CC" CC="$CC -pthread" if test "$cross_compiling" = yes then : ac_cv_pthread=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -10404,14 +11078,17 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_pthread=yes -else $as_nop - ac_cv_pthread=no +else case e in #( + e) ac_cv_pthread=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi -CC="$ac_save_cc" +CC="$ac_save_cc" ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_pthread" >&5 printf "%s\n" "$ac_cv_pthread" >&6; } @@ -10426,8 +11103,8 @@ printf %s "checking whether $CXX also accepts flags for thread support... " >&6; if test ${ac_cv_cxx_thread+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_save_cxx="$CXX" +else case e in #( + e) ac_save_cxx="$CXX" if test "$ac_cv_kpthread" = "yes" then @@ -10458,7 +11135,8 @@ then fi rm -fr conftest* fi -CXX="$ac_save_cxx" +CXX="$ac_save_cxx" ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_cxx_thread" >&5 printf "%s\n" "$ac_cv_cxx_thread" >&6; } @@ -10585,6 +11263,12 @@ if test "x$ac_cv_header_linux_memfd_h" = xyes then : printf "%s\n" "#define HAVE_LINUX_MEMFD_H 1" >>confdefs.h +fi +ac_fn_c_check_header_compile "$LINENO" "linux/netfilter_ipv4.h" "ac_cv_header_linux_netfilter_ipv4_h" "$ac_includes_default" +if test "x$ac_cv_header_linux_netfilter_ipv4_h" = xyes +then : + printf "%s\n" "#define HAVE_LINUX_NETFILTER_IPV4_H 1" >>confdefs.h + fi ac_fn_c_check_header_compile "$LINENO" "linux/random.h" "ac_cv_header_linux_random_h" "$ac_includes_default" if test "x$ac_cv_header_linux_random_h" = xyes @@ -10597,6 +11281,12 @@ if test "x$ac_cv_header_linux_soundcard_h" = xyes then : printf "%s\n" "#define HAVE_LINUX_SOUNDCARD_H 1" >>confdefs.h +fi +ac_fn_c_check_header_compile "$LINENO" "linux/sched.h" "ac_cv_header_linux_sched_h" "$ac_includes_default" +if test "x$ac_cv_header_linux_sched_h" = xyes +then : + printf "%s\n" "#define HAVE_LINUX_SCHED_H 1" >>confdefs.h + fi ac_fn_c_check_header_compile "$LINENO" "linux/tipc.h" "ac_cv_header_linux_tipc_h" "$ac_includes_default" if test "x$ac_cv_header_linux_tipc_h" = xyes @@ -10795,6 +11485,12 @@ if test "x$ac_cv_header_sys_param_h" = xyes then : printf "%s\n" "#define HAVE_SYS_PARAM_H 1" >>confdefs.h +fi +ac_fn_c_check_header_compile "$LINENO" "sys/pidfd.h" "ac_cv_header_sys_pidfd_h" "$ac_includes_default" +if test "x$ac_cv_header_sys_pidfd_h" = xyes +then : + printf "%s\n" "#define HAVE_SYS_PIDFD_H 1" >>confdefs.h + fi ac_fn_c_check_header_compile "$LINENO" "sys/poll.h" "ac_cv_header_sys_poll_h" "$ac_includes_default" if test "x$ac_cv_header_sys_poll_h" = xyes @@ -10967,14 +11663,14 @@ fi ac_header_dirent=no for ac_hdr in dirent.h sys/ndir.h sys/dir.h ndir.h; do - as_ac_Header=`printf "%s\n" "ac_cv_header_dirent_$ac_hdr" | $as_tr_sh` + as_ac_Header=`printf "%s\n" "ac_cv_header_dirent_$ac_hdr" | sed "$as_sed_sh"` { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_hdr that defines DIR" >&5 printf %s "checking for $ac_hdr that defines DIR... " >&6; } if eval test \${$as_ac_Header+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #include <$ac_hdr> @@ -10991,10 +11687,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : eval "$as_ac_Header=yes" -else $as_nop - eval "$as_ac_Header=no" +else case e in #( + e) eval "$as_ac_Header=no" ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi eval ac_res=\$$as_ac_Header { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 @@ -11002,7 +11700,7 @@ printf "%s\n" "$ac_res" >&6; } if eval test \"x\$"$as_ac_Header"\" = x"yes" then : cat >>confdefs.h <<_ACEOF -#define `printf "%s\n" "HAVE_$ac_hdr" | $as_tr_cpp` 1 +#define `printf "%s\n" "HAVE_$ac_hdr" | sed "$as_sed_cpp"` 1 _ACEOF ac_header_dirent=$ac_hdr; break @@ -11016,15 +11714,21 @@ printf %s "checking for library containing opendir... " >&6; } if test ${ac_cv_search_opendir+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_func_search_save_LIBS=$LIBS +else case e in #( + e) ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char opendir (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char opendir (void); int main (void) { @@ -11055,11 +11759,13 @@ done if test ${ac_cv_search_opendir+y} then : -else $as_nop - ac_cv_search_opendir=no +else case e in #( + e) ac_cv_search_opendir=no ;; +esac fi rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS +LIBS=$ac_func_search_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_opendir" >&5 printf "%s\n" "$ac_cv_search_opendir" >&6; } @@ -11076,15 +11782,21 @@ printf %s "checking for library containing opendir... " >&6; } if test ${ac_cv_search_opendir+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_func_search_save_LIBS=$LIBS +else case e in #( + e) ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char opendir (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char opendir (void); int main (void) { @@ -11115,11 +11827,13 @@ done if test ${ac_cv_search_opendir+y} then : -else $as_nop - ac_cv_search_opendir=no +else case e in #( + e) ac_cv_search_opendir=no ;; +esac fi rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS +LIBS=$ac_func_search_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_opendir" >&5 printf "%s\n" "$ac_cv_search_opendir" >&6; } @@ -11312,10 +12026,11 @@ then : printf "%s\n" "#define HAVE_CLOCK_T 1" >>confdefs.h -else $as_nop - +else case e in #( + e) printf "%s\n" "#define clock_t long" >>confdefs.h - + ;; +esac fi @@ -11324,8 +12039,8 @@ printf %s "checking for makedev... " >&6; } if test ${ac_cv_func_makedev+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -11350,12 +12065,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_func_makedev=yes -else $as_nop - ac_cv_func_makedev=no +else case e in #( + e) ac_cv_func_makedev=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_makedev" >&5 printf "%s\n" "$ac_cv_func_makedev" >&6; } @@ -11375,8 +12092,8 @@ printf %s "checking for le64toh... " >&6; } if test ${ac_cv_func_le64toh+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -11399,12 +12116,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_func_le64toh=yes -else $as_nop - ac_cv_func_le64toh=no +else case e in #( + e) ac_cv_func_le64toh=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_le64toh" >&5 printf "%s\n" "$ac_cv_func_le64toh" >&6; } @@ -11454,20 +12173,22 @@ ac_fn_c_check_type "$LINENO" "mode_t" "ac_cv_type_mode_t" "$ac_includes_default" if test "x$ac_cv_type_mode_t" = xyes then : -else $as_nop - +else case e in #( + e) printf "%s\n" "#define mode_t int" >>confdefs.h - + ;; +esac fi ac_fn_c_check_type "$LINENO" "off_t" "ac_cv_type_off_t" "$ac_includes_default" if test "x$ac_cv_type_off_t" = xyes then : -else $as_nop - +else case e in #( + e) printf "%s\n" "#define off_t long int" >>confdefs.h - + ;; +esac fi @@ -11476,8 +12197,8 @@ fi if test "x$ac_cv_type_pid_t" = xyes then : -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #if defined _WIN64 && !defined __CYGWIN__ @@ -11496,14 +12217,16 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_pid_type='int' -else $as_nop - ac_pid_type='__int64' +else case e in #( + e) ac_pid_type='__int64' ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext printf "%s\n" "#define pid_t $ac_pid_type" >>confdefs.h - + ;; +esac fi @@ -11514,43 +12237,33 @@ ac_fn_c_check_type "$LINENO" "size_t" "ac_cv_type_size_t" "$ac_includes_default" if test "x$ac_cv_type_size_t" = xyes then : -else $as_nop - +else case e in #( + e) printf "%s\n" "#define size_t unsigned int" >>confdefs.h - + ;; +esac fi - -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for uid_t in sys/types.h" >&5 -printf %s "checking for uid_t in sys/types.h... " >&6; } -if test ${ac_cv_type_uid_t+y} +ac_fn_c_check_type "$LINENO" "uid_t" "ac_cv_type_uid_t" "$ac_includes_default" +if test "x$ac_cv_type_uid_t" = xyes then : - printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include - -_ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "uid_t" >/dev/null 2>&1 -then : - ac_cv_type_uid_t=yes -else $as_nop - ac_cv_type_uid_t=no -fi -rm -rf conftest* - -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_type_uid_t" >&5 -printf "%s\n" "$ac_cv_type_uid_t" >&6; } -if test $ac_cv_type_uid_t = no; then +else case e in #( + e) printf "%s\n" "#define uid_t int" >>confdefs.h + ;; +esac +fi +ac_fn_c_check_type "$LINENO" "gid_t" "ac_cv_type_gid_t" "$ac_includes_default" +if test "x$ac_cv_type_gid_t" = xyes +then : +else case e in #( + e) printf "%s\n" "#define gid_t int" >>confdefs.h - + ;; +esac fi @@ -11579,28 +12292,30 @@ fi # ANSI C requires sizeof(char) == 1, so no need to check it # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of int" >&5 printf %s "checking size of int... " >&6; } if test ${ac_cv_sizeof_int+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (int))" "ac_cv_sizeof_int" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (int))" "ac_cv_sizeof_int" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_int" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_int" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (int) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_int=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_int" >&5 printf "%s\n" "$ac_cv_sizeof_int" >&6; } @@ -11612,28 +12327,30 @@ printf "%s\n" "#define SIZEOF_INT $ac_cv_sizeof_int" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of long" >&5 printf %s "checking size of long... " >&6; } if test ${ac_cv_sizeof_long+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (long))" "ac_cv_sizeof_long" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (long))" "ac_cv_sizeof_long" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_long" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_long" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (long) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_long=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_long" >&5 printf "%s\n" "$ac_cv_sizeof_long" >&6; } @@ -11650,22 +12367,24 @@ printf %s "checking alignment of long... " >&6; } if test ${ac_cv_alignof_long+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) offsetof (ac__type_alignof_, y)" "ac_cv_alignof_long" "$ac_includes_default +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) offsetof (ac__type_alignof_, y)" "ac_cv_alignof_long" "$ac_includes_default typedef struct { char x; long y; } ac__type_alignof_;" then : -else $as_nop - if test "$ac_cv_type_long" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_long" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute alignment of long -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_alignof_long=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_alignof_long" >&5 printf "%s\n" "$ac_cv_alignof_long" >&6; } @@ -11677,28 +12396,30 @@ printf "%s\n" "#define ALIGNOF_LONG $ac_cv_alignof_long" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of long long" >&5 printf %s "checking size of long long... " >&6; } if test ${ac_cv_sizeof_long_long+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (long long))" "ac_cv_sizeof_long_long" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (long long))" "ac_cv_sizeof_long_long" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_long_long" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_long_long" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (long long) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_long_long=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_long_long" >&5 printf "%s\n" "$ac_cv_sizeof_long_long" >&6; } @@ -11710,28 +12431,30 @@ printf "%s\n" "#define SIZEOF_LONG_LONG $ac_cv_sizeof_long_long" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of void *" >&5 printf %s "checking size of void *... " >&6; } if test ${ac_cv_sizeof_void_p+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (void *))" "ac_cv_sizeof_void_p" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (void *))" "ac_cv_sizeof_void_p" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_void_p" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_void_p" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (void *) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_void_p=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_void_p" >&5 printf "%s\n" "$ac_cv_sizeof_void_p" >&6; } @@ -11743,28 +12466,30 @@ printf "%s\n" "#define SIZEOF_VOID_P $ac_cv_sizeof_void_p" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of short" >&5 printf %s "checking size of short... " >&6; } if test ${ac_cv_sizeof_short+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (short))" "ac_cv_sizeof_short" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (short))" "ac_cv_sizeof_short" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_short" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_short" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (short) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_short=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_short" >&5 printf "%s\n" "$ac_cv_sizeof_short" >&6; } @@ -11776,28 +12501,30 @@ printf "%s\n" "#define SIZEOF_SHORT $ac_cv_sizeof_short" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of float" >&5 printf %s "checking size of float... " >&6; } if test ${ac_cv_sizeof_float+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (float))" "ac_cv_sizeof_float" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (float))" "ac_cv_sizeof_float" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_float" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_float" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (float) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_float=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_float" >&5 printf "%s\n" "$ac_cv_sizeof_float" >&6; } @@ -11809,28 +12536,30 @@ printf "%s\n" "#define SIZEOF_FLOAT $ac_cv_sizeof_float" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of double" >&5 printf %s "checking size of double... " >&6; } if test ${ac_cv_sizeof_double+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (double))" "ac_cv_sizeof_double" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (double))" "ac_cv_sizeof_double" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_double" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_double" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (double) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_double=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_double" >&5 printf "%s\n" "$ac_cv_sizeof_double" >&6; } @@ -11842,28 +12571,30 @@ printf "%s\n" "#define SIZEOF_DOUBLE $ac_cv_sizeof_double" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of fpos_t" >&5 printf %s "checking size of fpos_t... " >&6; } if test ${ac_cv_sizeof_fpos_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (fpos_t))" "ac_cv_sizeof_fpos_t" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (fpos_t))" "ac_cv_sizeof_fpos_t" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_fpos_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_fpos_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (fpos_t) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_fpos_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_fpos_t" >&5 printf "%s\n" "$ac_cv_sizeof_fpos_t" >&6; } @@ -11875,28 +12606,30 @@ printf "%s\n" "#define SIZEOF_FPOS_T $ac_cv_sizeof_fpos_t" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of size_t" >&5 printf %s "checking size of size_t... " >&6; } if test ${ac_cv_sizeof_size_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (size_t))" "ac_cv_sizeof_size_t" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (size_t))" "ac_cv_sizeof_size_t" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_size_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_size_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (size_t) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_size_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_size_t" >&5 printf "%s\n" "$ac_cv_sizeof_size_t" >&6; } @@ -11913,22 +12646,24 @@ printf %s "checking alignment of size_t... " >&6; } if test ${ac_cv_alignof_size_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) offsetof (ac__type_alignof_, y)" "ac_cv_alignof_size_t" "$ac_includes_default +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) offsetof (ac__type_alignof_, y)" "ac_cv_alignof_size_t" "$ac_includes_default typedef struct { char x; size_t y; } ac__type_alignof_;" then : -else $as_nop - if test "$ac_cv_type_size_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_size_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute alignment of size_t -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_alignof_size_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_alignof_size_t" >&5 printf "%s\n" "$ac_cv_alignof_size_t" >&6; } @@ -11940,28 +12675,30 @@ printf "%s\n" "#define ALIGNOF_SIZE_T $ac_cv_alignof_size_t" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of pid_t" >&5 printf %s "checking size of pid_t... " >&6; } if test ${ac_cv_sizeof_pid_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (pid_t))" "ac_cv_sizeof_pid_t" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (pid_t))" "ac_cv_sizeof_pid_t" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_pid_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_pid_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (pid_t) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_pid_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_pid_t" >&5 printf "%s\n" "$ac_cv_sizeof_pid_t" >&6; } @@ -11973,28 +12710,30 @@ printf "%s\n" "#define SIZEOF_PID_T $ac_cv_sizeof_pid_t" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of uintptr_t" >&5 printf %s "checking size of uintptr_t... " >&6; } if test ${ac_cv_sizeof_uintptr_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (uintptr_t))" "ac_cv_sizeof_uintptr_t" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (uintptr_t))" "ac_cv_sizeof_uintptr_t" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_uintptr_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_uintptr_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (uintptr_t) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_uintptr_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_uintptr_t" >&5 printf "%s\n" "$ac_cv_sizeof_uintptr_t" >&6; } @@ -12011,22 +12750,24 @@ printf %s "checking alignment of max_align_t... " >&6; } if test ${ac_cv_alignof_max_align_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) offsetof (ac__type_alignof_, y)" "ac_cv_alignof_max_align_t" "$ac_includes_default +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) offsetof (ac__type_alignof_, y)" "ac_cv_alignof_max_align_t" "$ac_includes_default typedef struct { char x; max_align_t y; } ac__type_alignof_;" then : -else $as_nop - if test "$ac_cv_type_max_align_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_max_align_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute alignment of max_align_t -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_alignof_max_align_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_alignof_max_align_t" >&5 printf "%s\n" "$ac_cv_alignof_max_align_t" >&6; } @@ -12043,8 +12784,8 @@ printf %s "checking for long double... " >&6; } if test ${ac_cv_type_long_double+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$GCC" = yes; then +else case e in #( + e) if test "$GCC" = yes; then ac_cv_type_long_double=yes else cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -12067,11 +12808,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_type_long_double=yes -else $as_nop - ac_cv_type_long_double=no +else case e in #( + e) ac_cv_type_long_double=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - fi + fi ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_type_long_double" >&5 printf "%s\n" "$ac_cv_type_long_double" >&6; } @@ -12083,28 +12826,30 @@ printf "%s\n" "#define HAVE_LONG_DOUBLE 1" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of long double" >&5 printf %s "checking size of long double... " >&6; } if test ${ac_cv_sizeof_long_double+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (long double))" "ac_cv_sizeof_long_double" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (long double))" "ac_cv_sizeof_long_double" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type_long_double" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_long_double" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (long double) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_long_double=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_long_double" >&5 printf "%s\n" "$ac_cv_sizeof_long_double" >&6; } @@ -12117,28 +12862,30 @@ printf "%s\n" "#define SIZEOF_LONG_DOUBLE $ac_cv_sizeof_long_double" >>confdefs. # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of _Bool" >&5 printf %s "checking size of _Bool... " >&6; } if test ${ac_cv_sizeof__Bool+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (_Bool))" "ac_cv_sizeof__Bool" "$ac_includes_default" +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (_Bool))" "ac_cv_sizeof__Bool" "$ac_includes_default" then : -else $as_nop - if test "$ac_cv_type__Bool" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type__Bool" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (_Bool) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof__Bool=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof__Bool" >&5 printf "%s\n" "$ac_cv_sizeof__Bool" >&6; } @@ -12151,15 +12898,15 @@ printf "%s\n" "#define SIZEOF__BOOL $ac_cv_sizeof__Bool" >>confdefs.h # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of off_t" >&5 printf %s "checking size of off_t... " >&6; } if test ${ac_cv_sizeof_off_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (off_t))" "ac_cv_sizeof_off_t" " +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (off_t))" "ac_cv_sizeof_off_t" " #ifdef HAVE_SYS_TYPES_H #include #endif @@ -12167,17 +12914,19 @@ else $as_nop " then : -else $as_nop - if test "$ac_cv_type_off_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_off_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (off_t) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_off_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_off_t" >&5 printf "%s\n" "$ac_cv_sizeof_off_t" >&6; } @@ -12212,24 +12961,25 @@ printf "%s\n" "#define HAVE_LARGEFILE_SUPPORT 1" >>confdefs.h { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - + ;; +esac fi # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of time_t" >&5 printf %s "checking size of time_t... " >&6; } if test ${ac_cv_sizeof_time_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (time_t))" "ac_cv_sizeof_time_t" " +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (time_t))" "ac_cv_sizeof_time_t" " #ifdef HAVE_SYS_TYPES_H #include #endif @@ -12240,17 +12990,19 @@ else $as_nop " then : -else $as_nop - if test "$ac_cv_type_time_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_time_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (time_t) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_time_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_time_t" >&5 printf "%s\n" "$ac_cv_sizeof_time_t" >&6; } @@ -12276,8 +13028,8 @@ printf %s "checking for pthread_t... " >&6; } if test ${ac_cv_have_pthread_t+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -12294,11 +13046,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_have_pthread_t=yes -else $as_nop - ac_cv_have_pthread_t=no +else case e in #( + e) ac_cv_have_pthread_t=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_have_pthread_t" >&5 printf "%s\n" "$ac_cv_have_pthread_t" >&6; } @@ -12307,15 +13061,15 @@ then : # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of pthread_t" >&5 printf %s "checking size of pthread_t... " >&6; } if test ${ac_cv_sizeof_pthread_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (pthread_t))" "ac_cv_sizeof_pthread_t" " +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (pthread_t))" "ac_cv_sizeof_pthread_t" " #ifdef HAVE_PTHREAD_H #include #endif @@ -12323,17 +13077,19 @@ else $as_nop " then : -else $as_nop - if test "$ac_cv_type_pthread_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_pthread_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (pthread_t) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_pthread_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_pthread_t" >&5 printf "%s\n" "$ac_cv_sizeof_pthread_t" >&6; } @@ -12350,29 +13106,31 @@ fi # This checking will be unnecessary after removing deprecated TLS API. # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of pthread_key_t" >&5 printf %s "checking size of pthread_key_t... " >&6; } if test ${ac_cv_sizeof_pthread_key_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (pthread_key_t))" "ac_cv_sizeof_pthread_key_t" "#include +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (pthread_key_t))" "ac_cv_sizeof_pthread_key_t" "#include " then : -else $as_nop - if test "$ac_cv_type_pthread_key_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_pthread_key_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (pthread_key_t) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_pthread_key_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_pthread_key_t" >&5 printf "%s\n" "$ac_cv_sizeof_pthread_key_t" >&6; } @@ -12387,8 +13145,8 @@ printf %s "checking whether pthread_key_t is compatible with int... " >&6; } if test ${ac_cv_pthread_key_t_is_arithmetic_type+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) if test "$ac_cv_sizeof_pthread_key_t" -eq "$ac_cv_sizeof_int" ; then cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -12404,15 +13162,17 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_pthread_key_t_is_arithmetic_type=yes -else $as_nop - ac_cv_pthread_key_t_is_arithmetic_type=no - +else case e in #( + e) ac_cv_pthread_key_t_is_arithmetic_type=no + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext else ac_cv_pthread_key_t_is_arithmetic_type=no fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_pthread_key_t_is_arithmetic_type" >&5 printf "%s\n" "$ac_cv_pthread_key_t_is_arithmetic_type" >&6; } @@ -12471,9 +13231,10 @@ printf "%s\n" "yes" >&6; }; else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; }; DSYMUTIL= fi -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac fi @@ -12485,8 +13246,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_DSYMUTIL_PATH+y} then : printf %s "(cached) " >&6 -else $as_nop - case $DSYMUTIL_PATH in +else case e in #( + e) case $DSYMUTIL_PATH in [\\/]* | ?:[\\/]*) ac_cv_path_DSYMUTIL_PATH="$DSYMUTIL_PATH" # Let the user override the test with a path. ;; @@ -12512,6 +13273,7 @@ IFS=$as_save_IFS test -z "$ac_cv_path_DSYMUTIL_PATH" && ac_cv_path_DSYMUTIL_PATH="not found" ;; +esac ;; esac fi DSYMUTIL_PATH=$ac_cv_path_DSYMUTIL_PATH @@ -12559,9 +13321,10 @@ LDFLAGS="-fsanitize=address $LDFLAGS" # ASan works by controlling memory allocation, our own malloc interferes. with_pymalloc="no" -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac fi @@ -12579,8 +13342,8 @@ printf %s "checking whether C compiler accepts -fsanitize=memory... " >&6; } if test ${ax_cv_check_cflags___fsanitize_memory+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_check_save_flags=$CFLAGS CFLAGS="$CFLAGS -fsanitize=memory" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -12597,11 +13360,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ax_cv_check_cflags___fsanitize_memory=yes -else $as_nop - ax_cv_check_cflags___fsanitize_memory=no +else case e in #( + e) ax_cv_check_cflags___fsanitize_memory=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + CFLAGS=$ax_check_save_flags ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___fsanitize_memory" >&5 printf "%s\n" "$ax_cv_check_cflags___fsanitize_memory" >&6; } @@ -12611,16 +13376,18 @@ then : BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS" LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS" -else $as_nop - as_fn_error $? "The selected compiler doesn't support memory sanitizer" "$LINENO" 5 +else case e in #( + e) as_fn_error $? "The selected compiler doesn't support memory sanitizer" "$LINENO" 5 ;; +esac fi # MSan works by controlling memory allocation, our own malloc interferes. with_pymalloc="no" -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac fi @@ -12637,12 +13404,13 @@ BASECFLAGS="-fsanitize=undefined $BASECFLAGS" LDFLAGS="-fsanitize=undefined $LDFLAGS" with_ubsan="yes" -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } with_ubsan="no" - + ;; +esac fi @@ -12659,12 +13427,13 @@ BASECFLAGS="-fsanitize=thread $BASECFLAGS" LDFLAGS="-fsanitize=thread $LDFLAGS" with_tsan="yes" -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } with_tsan="no" - + ;; +esac fi @@ -12689,7 +13458,6 @@ if test -z "$SHLIB_SUFFIX"; then esac ;; CYGWIN*) SHLIB_SUFFIX=.dll;; - iOS) SHLIB_SUFFIX=.dylib;; *) SHLIB_SUFFIX=.so;; esac fi @@ -12710,7 +13478,7 @@ then LDSHARED="\$(LIBPL)/ld_so_aix \$(CC) -bI:\$(LIBPL)/python.exp" ;; SunOS/5*) - if test "$GCC" = "yes" ; then + if test "$ac_cv_gcc_compat" = "yes" ; then LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared' else @@ -12718,7 +13486,7 @@ then LDCXXSHARED='$(CXX) -G' fi ;; hp*|HP*) - if test "$GCC" = "yes" ; then + if test "$ac_cv_gcc_compat" = "yes" ; then LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared' else @@ -12773,11 +13541,11 @@ then fi ;; iOS/*) - LDSHARED='$(CC) -dynamiclib -F . -framework Python' - LDCXXSHARED='$(CXX) -dynamiclib -F . -framework Python' + LDSHARED='$(CC) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' + LDCXXSHARED='$(CXX) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' BLDSHARED="$LDSHARED" ;; - Emscripten|WASI) + Emscripten*|WASI*) LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared';; Linux*|GNU*|QNX*|VxWorks*|Haiku*) @@ -12811,7 +13579,7 @@ then LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared';; OpenUNIX*|UnixWare*) - if test "$GCC" = "yes" ; then + if test "$ac_cv_gcc_compat" = "yes" ; then LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared' else @@ -12855,13 +13623,13 @@ printf %s "checking CCSHARED... " >&6; } if test -z "$CCSHARED" then case $ac_sys_system/$ac_sys_release in - SunOS*) if test "$GCC" = yes; + SunOS*) if test "$ac_cv_gcc_compat" = "yes"; then CCSHARED="-fPIC"; elif test `uname -p` = sparc; then CCSHARED="-xcode=pic32"; else CCSHARED="-Kpic"; fi;; - hp*|HP*) if test "$GCC" = yes; + hp*|HP*) if test "$ac_cv_gcc_compat" = "yes"; then CCSHARED="-fPIC"; else CCSHARED="+z"; fi;; @@ -12876,12 +13644,12 @@ fi;; FreeBSD*|NetBSD*|OpenBSD*|DragonFly*) CCSHARED="-fPIC";; Haiku*) CCSHARED="-fPIC";; OpenUNIX*|UnixWare*) - if test "$GCC" = "yes" + if test "$ac_cv_gcc_compat" = "yes" then CCSHARED="-fPIC" else CCSHARED="-KPIC" fi;; SCO_SV*) - if test "$GCC" = "yes" + if test "$ac_cv_gcc_compat" = "yes" then CCSHARED="-fPIC" else CCSHARED="-Kpic -belf" fi;; @@ -12941,13 +13709,13 @@ printf "%s\n" "#define THREAD_STACK_SIZE 0x$stack_size" >>confdefs.h then LINKFORSHARED="-Wl,--export-dynamic" fi;; - SunOS/5*) case $CC in - *gcc*) + SunOS/5*) if test "$ac_cv_gcc_compat" = "yes"; then if $CC -Xlinker --help 2>&1 | grep export-dynamic >/dev/null then LINKFORSHARED="-Xlinker --export-dynamic" - fi;; - esac;; + fi + fi + ;; CYGWIN*) if test $enable_shared = "no" then @@ -13049,16 +13817,22 @@ printf %s "checking for sendfile in -lsendfile... " >&6; } if test ${ac_cv_lib_sendfile_sendfile+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsendfile $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sendfile (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sendfile (void); int main (void) { @@ -13070,12 +13844,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sendfile_sendfile=yes -else $as_nop - ac_cv_lib_sendfile_sendfile=no +else case e in #( + e) ac_cv_lib_sendfile_sendfile=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sendfile_sendfile" >&5 printf "%s\n" "$ac_cv_lib_sendfile_sendfile" >&6; } @@ -13092,16 +13868,22 @@ printf %s "checking for dlopen in -ldl... " >&6; } if test ${ac_cv_lib_dl_dlopen+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-ldl $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char dlopen (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char dlopen (void); int main (void) { @@ -13113,12 +13895,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_dl_dlopen=yes -else $as_nop - ac_cv_lib_dl_dlopen=no +else case e in #( + e) ac_cv_lib_dl_dlopen=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dl_dlopen" >&5 printf "%s\n" "$ac_cv_lib_dl_dlopen" >&6; } @@ -13135,16 +13919,22 @@ printf %s "checking for shl_load in -ldld... " >&6; } if test ${ac_cv_lib_dld_shl_load+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-ldld $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char shl_load (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char shl_load (void); int main (void) { @@ -13156,12 +13946,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_dld_shl_load=yes -else $as_nop - ac_cv_lib_dld_shl_load=no +else case e in #( + e) ac_cv_lib_dld_shl_load=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dld_shl_load" >&5 printf "%s\n" "$ac_cv_lib_dld_shl_load" >&6; } @@ -13189,12 +13981,12 @@ then : for ac_func in uuid_create uuid_enc_be do : - as_ac_var=`printf "%s\n" "ac_cv_func_$ac_func" | $as_tr_sh` + as_ac_var=`printf "%s\n" "ac_cv_func_$ac_func" | sed "$as_sed_sh"` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" if eval test \"x\$"$as_ac_var"\" = x"yes" then : cat >>confdefs.h <<_ACEOF -#define `printf "%s\n" "HAVE_$ac_func" | $as_tr_cpp` 1 +#define `printf "%s\n" "HAVE_$ac_func" | sed "$as_sed_cpp"` 1 _ACEOF have_uuid=yes LIBUUID_CFLAGS=${LIBUUID_CFLAGS-""} @@ -13263,12 +14055,12 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - LIBUUID_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "uuid >= 2.20" 2>&1` + LIBUUID_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "uuid >= 2.20" 2>&1` else - LIBUUID_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "uuid >= 2.20" 2>&1` + LIBUUID_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "uuid >= 2.20" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$LIBUUID_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$LIBUUID_PKG_ERRORS" >&5 save_CFLAGS=$CFLAGS @@ -13278,7 +14070,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBUUID_CFLAGS" - LDFLAGS="$LDFLAGS $LIBUUID_LIBS" + LIBS="$LIBS $LIBUUID_LIBS" for ac_header in uuid/uuid.h do : ac_fn_c_check_header_compile "$LINENO" "uuid/uuid.h" "ac_cv_header_uuid_uuid_h" "$ac_includes_default" @@ -13292,16 +14084,22 @@ printf %s "checking for uuid_generate_time in -luuid... " >&6; } if test ${ac_cv_lib_uuid_uuid_generate_time+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-luuid $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char uuid_generate_time (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char uuid_generate_time (void); int main (void) { @@ -13313,12 +14111,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_uuid_uuid_generate_time=yes -else $as_nop - ac_cv_lib_uuid_uuid_generate_time=no +else case e in #( + e) ac_cv_lib_uuid_uuid_generate_time=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_uuid_uuid_generate_time" >&5 printf "%s\n" "$ac_cv_lib_uuid_uuid_generate_time" >&6; } @@ -13335,16 +14135,22 @@ printf %s "checking for uuid_generate_time_safe in -luuid... " >&6; } if test ${ac_cv_lib_uuid_uuid_generate_time_safe+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-luuid $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char uuid_generate_time_safe (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char uuid_generate_time_safe (void); int main (void) { @@ -13356,12 +14162,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_uuid_uuid_generate_time_safe=yes -else $as_nop - ac_cv_lib_uuid_uuid_generate_time_safe=no +else case e in #( + e) ac_cv_lib_uuid_uuid_generate_time_safe=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_uuid_uuid_generate_time_safe" >&5 printf "%s\n" "$ac_cv_lib_uuid_uuid_generate_time_safe" >&6; } @@ -13404,7 +14212,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBUUID_CFLAGS" - LDFLAGS="$LDFLAGS $LIBUUID_LIBS" + LIBS="$LIBS $LIBUUID_LIBS" for ac_header in uuid/uuid.h do : ac_fn_c_check_header_compile "$LINENO" "uuid/uuid.h" "ac_cv_header_uuid_uuid_h" "$ac_includes_default" @@ -13418,16 +14226,22 @@ printf %s "checking for uuid_generate_time in -luuid... " >&6; } if test ${ac_cv_lib_uuid_uuid_generate_time+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-luuid $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char uuid_generate_time (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char uuid_generate_time (void); int main (void) { @@ -13439,12 +14253,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_uuid_uuid_generate_time=yes -else $as_nop - ac_cv_lib_uuid_uuid_generate_time=no +else case e in #( + e) ac_cv_lib_uuid_uuid_generate_time=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_uuid_uuid_generate_time" >&5 printf "%s\n" "$ac_cv_lib_uuid_uuid_generate_time" >&6; } @@ -13461,16 +14277,22 @@ printf %s "checking for uuid_generate_time_safe in -luuid... " >&6; } if test ${ac_cv_lib_uuid_uuid_generate_time_safe+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-luuid $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char uuid_generate_time_safe (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char uuid_generate_time_safe (void); int main (void) { @@ -13482,12 +14304,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_uuid_uuid_generate_time_safe=yes -else $as_nop - ac_cv_lib_uuid_uuid_generate_time_safe=no +else case e in #( + e) ac_cv_lib_uuid_uuid_generate_time_safe=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_uuid_uuid_generate_time_safe" >&5 printf "%s\n" "$ac_cv_lib_uuid_uuid_generate_time_safe" >&6; } @@ -13520,11 +14344,11 @@ LIBS=$save_LIBS else - LIBUUID_CFLAGS=$pkg_cv_LIBUUID_CFLAGS - LIBUUID_LIBS=$pkg_cv_LIBUUID_LIBS + LIBUUID_CFLAGS=$pkg_cv_LIBUUID_CFLAGS + LIBUUID_LIBS=$pkg_cv_LIBUUID_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } - have_uuid=yes + have_uuid=yes printf "%s\n" "#define HAVE_UUID_H 1" >>confdefs.h printf "%s\n" "#define HAVE_UUID_GENERATE_TIME_SAFE 1" >>confdefs.h @@ -13561,6 +14385,14 @@ done fi +# gh-124228: While the libuuid library is available on NetBSD, it supports only UUID version 4. +# This restriction inhibits the proper generation of time-based UUIDs. +if test "$ac_sys_system" = "NetBSD"; then + have_uuid=missing + printf "%s\n" "#define HAVE_UUID_H 0" >>confdefs.h + +fi + if test "x$have_uuid" = xmissing then : have_uuid=no @@ -13574,15 +14406,21 @@ printf %s "checking for library containing sem_init... " >&6; } if test ${ac_cv_search_sem_init+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_func_search_save_LIBS=$LIBS +else case e in #( + e) ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sem_init (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sem_init (void); int main (void) { @@ -13613,11 +14451,13 @@ done if test ${ac_cv_search_sem_init+y} then : -else $as_nop - ac_cv_search_sem_init=no +else case e in #( + e) ac_cv_search_sem_init=no ;; +esac fi rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS +LIBS=$ac_func_search_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_sem_init" >&5 printf "%s\n" "$ac_cv_search_sem_init" >&6; } @@ -13635,16 +14475,22 @@ printf %s "checking for textdomain in -lintl... " >&6; } if test ${ac_cv_lib_intl_textdomain+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lintl $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char textdomain (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char textdomain (void); int main (void) { @@ -13656,12 +14502,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_intl_textdomain=yes -else $as_nop - ac_cv_lib_intl_textdomain=no +else case e in #( + e) ac_cv_lib_intl_textdomain=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_intl_textdomain" >&5 printf "%s\n" "$ac_cv_lib_intl_textdomain" >&6; } @@ -13700,11 +14548,12 @@ printf "%s\n" "#define AIX_GENUINE_CPLUSPLUS 1" >>confdefs.h { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext @@ -13723,14 +14572,61 @@ printf "%s\n" "$AIX_BUILDDATE" >&6; } *) ;; esac +# check for _Complex C type +# +# Note that despite most compilers define __STDC_IEC_559_COMPLEX__ - almost +# none properly support C11+ Annex G (where pure imaginary types +# represented by _Imaginary are mandatory). This is a bug (see e.g. +# llvm/llvm-project#60269), so we don't rely on presence +# of __STDC_IEC_559_COMPLEX__. +if test "$cross_compiling" = yes +then : + ac_cv_c_complex_supported=no +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#include +#define test(type, out) \ +{ \ + type complex z = 1 + 2*I; z = z*z; \ + (out) = (out) || creal(z) != -3 || cimag(z) != 4; \ +} +int main(void) +{ + int res = 0; + test(float, res); + test(double, res); + test(long double, res); + return res; +} +_ACEOF +if ac_fn_c_try_run "$LINENO" +then : + ac_cv_c_complex_supported=yes +else case e in #( + e) ac_cv_c_complex_supported=no ;; +esac +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi + +if test "$ac_cv_c_complex_supported" = "yes"; then + +printf "%s\n" "#define Py_HAVE_C_COMPLEX 1" >>confdefs.h + +fi + # check for systems that require aligned memory access { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking aligned memory access is required" >&5 printf %s "checking aligned memory access is required... " >&6; } if test ${ac_cv_aligned_required+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : # "yes" changes the hash function to FNV, which causes problems with Numba @@ -13740,8 +14636,8 @@ if test "$ac_sys_system" = "Linux-android"; then else ac_cv_aligned_required=yes fi -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main(void) @@ -13760,14 +14656,17 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_aligned_required=no -else $as_nop - ac_cv_aligned_required=yes +else case e in #( + e) ac_cv_aligned_required=yes ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_aligned_required" >&5 printf "%s\n" "$ac_cv_aligned_required" >&6; } @@ -13807,9 +14706,10 @@ case "$withval" in ;; esac -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: default" >&5 -printf "%s\n" "default" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: default" >&5 +printf "%s\n" "default" >&6; } ;; +esac fi @@ -13847,10 +14747,11 @@ printf "%s\n" "\"$withval\"" >&6; } ;; esac -else $as_nop - validate_tzpath "$TZPATH" +else case e in #( + e) validate_tzpath "$TZPATH" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: \"$TZPATH\"" >&5 -printf "%s\n" "\"$TZPATH\"" >&6; } +printf "%s\n" "\"$TZPATH\"" >&6; } ;; +esac fi @@ -13861,16 +14762,22 @@ printf %s "checking for t_open in -lnsl... " >&6; } if test ${ac_cv_lib_nsl_t_open+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lnsl $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char t_open (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char t_open (void); int main (void) { @@ -13882,12 +14789,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_nsl_t_open=yes -else $as_nop - ac_cv_lib_nsl_t_open=no +else case e in #( + e) ac_cv_lib_nsl_t_open=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_nsl_t_open" >&5 printf "%s\n" "$ac_cv_lib_nsl_t_open" >&6; } @@ -13901,16 +14810,22 @@ printf %s "checking for socket in -lsocket... " >&6; } if test ${ac_cv_lib_socket_socket+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsocket $LIBS $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char socket (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char socket (void); int main (void) { @@ -13922,12 +14837,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_socket_socket=yes -else $as_nop - ac_cv_lib_socket_socket=no +else case e in #( + e) ac_cv_lib_socket_socket=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_socket_socket" >&5 printf "%s\n" "$ac_cv_lib_socket_socket" >&6; } @@ -13944,16 +14861,22 @@ printf %s "checking for socket in -lnetwork... " >&6; } if test ${ac_cv_lib_network_socket+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lnetwork $LIBS $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char socket (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char socket (void); int main (void) { @@ -13965,12 +14888,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_network_socket=yes -else $as_nop - ac_cv_lib_network_socket=no +else case e in #( + e) ac_cv_lib_network_socket=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_network_socket" >&5 printf "%s\n" "$ac_cv_lib_network_socket" >&6; } @@ -13993,9 +14918,10 @@ then : printf "%s\n" "$withval" >&6; } LIBS="$withval $LIBS" -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac fi @@ -14007,8 +14933,9 @@ printf %s "checking for --with-system-expat... " >&6; } if test ${with_system_expat+y} then : withval=$with_system_expat; -else $as_nop - with_system_expat="no" +else case e in #( + e) with_system_expat="no" ;; +esac fi @@ -14022,12 +14949,13 @@ then : LIBEXPAT_LDFLAGS=${LIBEXPAT_LDFLAGS-"-lexpat"} LIBEXPAT_INTERNAL= -else $as_nop - +else case e in #( + e) LIBEXPAT_CFLAGS="-I\$(srcdir)/Modules/expat" LIBEXPAT_LDFLAGS="-lm \$(LIBEXPAT_A)" LIBEXPAT_INTERNAL="\$(LIBEXPAT_HEADERS) \$(LIBEXPAT_A)" - + ;; +esac fi @@ -14053,16 +14981,22 @@ printf %s "checking for ffi_call in -lffi... " >&6; } if test ${ac_cv_lib_ffi_ffi_call+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lffi $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char ffi_call (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char ffi_call (void); int main (void) { @@ -14074,12 +15008,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_ffi_ffi_call=yes -else $as_nop - ac_cv_lib_ffi_ffi_call=no +else case e in #( + e) ac_cv_lib_ffi_ffi_call=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ffi_ffi_call" >&5 printf "%s\n" "$ac_cv_lib_ffi_ffi_call" >&6; } @@ -14159,12 +15095,12 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - LIBFFI_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libffi" 2>&1` + LIBFFI_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libffi" 2>&1` else - LIBFFI_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libffi" 2>&1` + LIBFFI_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libffi" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$LIBFFI_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$LIBFFI_PKG_ERRORS" >&5 save_CFLAGS=$CFLAGS @@ -14174,7 +15110,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBFFI_CFLAGS" - LDFLAGS="$LDFLAGS $LIBFFI_LIBS" + LIBS="$LIBS $LIBFFI_LIBS" ac_fn_c_check_header_compile "$LINENO" "ffi.h" "ac_cv_header_ffi_h" "$ac_includes_default" if test "x$ac_cv_header_ffi_h" = xyes then : @@ -14184,16 +15120,22 @@ printf %s "checking for ffi_call in -lffi... " >&6; } if test ${ac_cv_lib_ffi_ffi_call+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lffi $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char ffi_call (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char ffi_call (void); int main (void) { @@ -14205,12 +15147,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_ffi_ffi_call=yes -else $as_nop - ac_cv_lib_ffi_ffi_call=no +else case e in #( + e) ac_cv_lib_ffi_ffi_call=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ffi_ffi_call" >&5 printf "%s\n" "$ac_cv_lib_ffi_ffi_call" >&6; } @@ -14221,8 +15165,9 @@ then : LIBFFI_CFLAGS=${LIBFFI_CFLAGS-""} LIBFFI_LIBS=${LIBFFI_LIBS-"-lffi"} -else $as_nop - have_libffi=no +else case e in #( + e) have_libffi=no ;; +esac fi @@ -14247,7 +15192,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBFFI_CFLAGS" - LDFLAGS="$LDFLAGS $LIBFFI_LIBS" + LIBS="$LIBS $LIBFFI_LIBS" ac_fn_c_check_header_compile "$LINENO" "ffi.h" "ac_cv_header_ffi_h" "$ac_includes_default" if test "x$ac_cv_header_ffi_h" = xyes then : @@ -14257,16 +15202,22 @@ printf %s "checking for ffi_call in -lffi... " >&6; } if test ${ac_cv_lib_ffi_ffi_call+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lffi $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char ffi_call (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char ffi_call (void); int main (void) { @@ -14278,12 +15229,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_ffi_ffi_call=yes -else $as_nop - ac_cv_lib_ffi_ffi_call=no +else case e in #( + e) ac_cv_lib_ffi_ffi_call=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ffi_ffi_call" >&5 printf "%s\n" "$ac_cv_lib_ffi_ffi_call" >&6; } @@ -14294,8 +15247,9 @@ then : LIBFFI_CFLAGS=${LIBFFI_CFLAGS-""} LIBFFI_LIBS=${LIBFFI_LIBS-"-lffi"} -else $as_nop - have_libffi=no +else case e in #( + e) have_libffi=no ;; +esac fi @@ -14310,11 +15264,11 @@ LIBS=$save_LIBS else - LIBFFI_CFLAGS=$pkg_cv_LIBFFI_CFLAGS - LIBFFI_LIBS=$pkg_cv_LIBFFI_LIBS + LIBFFI_CFLAGS=$pkg_cv_LIBFFI_CFLAGS + LIBFFI_LIBS=$pkg_cv_LIBFFI_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } - have_libffi=yes + have_libffi=yes fi fi @@ -14358,8 +15312,8 @@ save_LDFLAGS=$LDFLAGS save_LIBS=$LIBS - CFLAGS="$LIBFFI_CFLAGS $CFLAGS" - LDFLAGS="$LIBFFI_LIBS $LDFLAGS" + CFLAGS="$CFLAGS $LIBFFI_CFLAGS" + LIBS="$LIBS $LIBFFI_LIBS" @@ -14368,8 +15322,8 @@ printf %s "checking for ffi_prep_cif_var... " >&6; } if test ${ac_cv_func_ffi_prep_cif_var+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -14383,11 +15337,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_ffi_prep_cif_var=yes -else $as_nop - ac_cv_func_ffi_prep_cif_var=no +else case e in #( + e) ac_cv_func_ffi_prep_cif_var=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_ffi_prep_cif_var" >&5 printf "%s\n" "$ac_cv_func_ffi_prep_cif_var" >&6; } @@ -14407,8 +15363,8 @@ printf %s "checking for ffi_prep_closure_loc... " >&6; } if test ${ac_cv_func_ffi_prep_closure_loc+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -14422,11 +15378,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_ffi_prep_closure_loc=yes -else $as_nop - ac_cv_func_ffi_prep_closure_loc=no +else case e in #( + e) ac_cv_func_ffi_prep_closure_loc=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_ffi_prep_closure_loc" >&5 printf "%s\n" "$ac_cv_func_ffi_prep_closure_loc" >&6; } @@ -14446,8 +15404,8 @@ printf %s "checking for ffi_closure_alloc... " >&6; } if test ${ac_cv_func_ffi_closure_alloc+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -14461,11 +15419,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_ffi_closure_alloc=yes -else $as_nop - ac_cv_func_ffi_closure_alloc=no +else case e in #( + e) ac_cv_func_ffi_closure_alloc=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_ffi_closure_alloc" >&5 printf "%s\n" "$ac_cv_func_ffi_closure_alloc" >&6; } @@ -14488,46 +15448,256 @@ LIBS=$save_LIBS fi -# Check for use of the system libmpdec library -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-system-libmpdec" >&5 -printf %s "checking for --with-system-libmpdec... " >&6; } - -# Check whether --with-system_libmpdec was given. -if test ${with_system_libmpdec+y} +# Check for libffi with real complex double support. +# This is a workaround, since FFI_TARGET_HAS_COMPLEX_TYPE was defined in libffi v3.2.1, +# but real support was provided only in libffi v3.3.0. +# See https://github.com/python/cpython/issues/125206 for more details. +# +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking libffi has complex type support" >&5 +printf %s "checking libffi has complex type support... " >&6; } +if test ${ac_cv_ffi_complex_double_supported+y} then : - withval=$with_system_libmpdec; -else $as_nop - with_system_libmpdec="no" -fi + printf %s "(cached) " >&6 +else case e in #( + e) save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_system_libmpdec" >&5 -printf "%s\n" "$with_system_libmpdec" >&6; } -if test "x$with_system_libmpdec" = xyes + CPPFLAGS="$CPPFLAGS $LIBFFI_CFLAGS" + LIBS="$LIBS $LIBFFI_LIBS" +if test "$cross_compiling" = yes then : + ac_cv_ffi_complex_double_supported=no +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ - LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} - LIBMPDEC_LDFLAGS=${LIBMPDEC_LDFLAGS-"-lmpdec"} - LIBMPDEC_INTERNAL= - -else $as_nop - - LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" - LIBMPDEC_LDFLAGS="-lm \$(LIBMPDEC_A)" - LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" +#include +#include +int z_is_expected(double complex z) +{ + const double complex expected = CMPLX(1.25, -0.5); + return z == expected; +} +int main(void) +{ + double complex z = 1.25 - 0.5 * I; + ffi_type *args[1] = {&ffi_type_complex_double}; + void *values[1] = {&z}; + ffi_cif cif; + if (ffi_prep_cif(&cif, FFI_DEFAULT_ABI, 1, + &ffi_type_sint, args) != FFI_OK) + { + return 2; + } + ffi_arg rc; + ffi_call(&cif, FFI_FN(z_is_expected), &rc, values); + return !rc; +} - if test "x$with_pydebug" = xyes +_ACEOF +if ac_fn_c_try_run "$LINENO" then : + ac_cv_ffi_complex_double_supported=yes +else case e in #( + e) ac_cv_ffi_complex_double_supported=no ;; +esac +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi - as_fn_append LIBMPDEC_CFLAGS " -DTEST_COVERAGE" -fi +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + ;; +esac fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_ffi_complex_double_supported" >&5 +printf "%s\n" "$ac_cv_ffi_complex_double_supported" >&6; } +if test "$ac_cv_ffi_complex_double_supported" = "yes"; then +printf "%s\n" "#define Py_FFI_SUPPORT_C_COMPLEX 1" >>confdefs.h +fi + +# Check for use of the system libmpdec library +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-system-libmpdec" >&5 +printf %s "checking for --with-system-libmpdec... " >&6; } + +# Check whether --with-system_libmpdec was given. +if test ${with_system_libmpdec+y} +then : + withval=$with_system_libmpdec; +else case e in #( + e) with_system_libmpdec="yes" ;; +esac +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_system_libmpdec" >&5 +printf "%s\n" "$with_system_libmpdec" >&6; } + + + +if test "x$with_system_libmpdec" = xyes +then : + +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libmpdec >= 2.5.0" >&5 +printf %s "checking for libmpdec >= 2.5.0... " >&6; } + +if test -n "$LIBMPDEC_CFLAGS"; then + pkg_cv_LIBMPDEC_CFLAGS="$LIBMPDEC_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libmpdec >= 2.5.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "libmpdec >= 2.5.0") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LIBMPDEC_CFLAGS=`$PKG_CONFIG --cflags "libmpdec >= 2.5.0" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LIBMPDEC_LIBS"; then + pkg_cv_LIBMPDEC_LIBS="$LIBMPDEC_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libmpdec >= 2.5.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "libmpdec >= 2.5.0") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LIBMPDEC_LIBS=`$PKG_CONFIG --libs "libmpdec >= 2.5.0" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LIBMPDEC_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libmpdec >= 2.5.0" 2>&1` + else + LIBMPDEC_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libmpdec >= 2.5.0" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LIBMPDEC_PKG_ERRORS" >&5 + + LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} + LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} + LIBMPDEC_INTERNAL= +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} + LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} + LIBMPDEC_INTERNAL= +else + LIBMPDEC_CFLAGS=$pkg_cv_LIBMPDEC_CFLAGS + LIBMPDEC_LIBS=$pkg_cv_LIBMPDEC_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + +fi +else case e in #( + e) LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" + LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" + LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" + have_mpdec=yes + with_system_libmpdec=no ;; +esac +fi + +if test "x$with_system_libmpdec" = xyes +then : + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + CPPFLAGS="$CPPFLAGS $LIBMPDEC_CFLAGS" + LIBS="$LIBS $LIBMPDEC_LIBS" + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + + #include + #if MPD_VERSION_HEX < 0x02050000 + # error "mpdecimal 2.5.0 or higher required" + #endif + +int +main (void) +{ +const char *x = mpd_version(); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + have_mpdec=yes +else case e in #( + e) have_mpdec=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&5 +printf "%s\n" "$as_me: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&2;} ;; +esac +fi + +if test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no" +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: no system libmpdecimal found; falling back to bundled libmpdecimal (deprecated and scheduled for removal in Python 3.15)" >&5 +printf "%s\n" "$as_me: WARNING: no system libmpdecimal found; falling back to bundled libmpdecimal (deprecated and scheduled for removal in Python 3.15)" >&2;} + LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" + LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" + LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" + have_mpdec=yes + with_system_libmpdec=no +fi + +# Disable forced inlining in debug builds, see GH-94847 +if test "x$with_pydebug" = xyes +then : + as_fn_append LIBMPDEC_CFLAGS " -DTEST_COVERAGE" +fi - # Check whether _decimal should use a coroutine-local or thread-local context { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-decimal-contextvar" >&5 printf %s "checking for --with-decimal-contextvar... " >&6; } @@ -14536,8 +15706,9 @@ printf %s "checking for --with-decimal-contextvar... " >&6; } if test ${with_decimal_contextvar+y} then : withval=$with_decimal_contextvar; -else $as_nop - with_decimal_contextvar="yes" +else case e in #( + e) with_decimal_contextvar="yes" ;; +esac fi @@ -14551,51 +15722,53 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_decimal_contextvar" >&5 printf "%s\n" "$with_decimal_contextvar" >&6; } -# Check for libmpdec machine flavor -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for decimal libmpdec machine" >&5 +if test "x$with_system_libmpdec" = xno +then : + # Check for libmpdec machine flavor + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for decimal libmpdec machine" >&5 printf %s "checking for decimal libmpdec machine... " >&6; } -case $ac_sys_system in #( + case $ac_sys_system in #( Darwin*) : libmpdec_system=Darwin ;; #( SunOS*) : libmpdec_system=sunos ;; #( *) : libmpdec_system=other - ;; + ;; esac -libmpdec_machine=unknown -if test "$libmpdec_system" = Darwin; then - # universal here means: build libmpdec with the same arch options - # the python interpreter was built with - libmpdec_machine=universal -elif test $ac_cv_sizeof_size_t -eq 8; then - if test "$ac_cv_gcc_asm_for_x64" = yes; then - libmpdec_machine=x64 - elif test "$ac_cv_type___uint128_t" = yes; then - libmpdec_machine=uint128 - else - libmpdec_machine=ansi64 - fi -elif test $ac_cv_sizeof_size_t -eq 4; then - if test "$ac_cv_gcc_asm_for_x87" = yes -a "$libmpdec_system" != sunos; then - case $CC in #( + libmpdec_machine=unknown + if test "$libmpdec_system" = Darwin; then + # universal here means: build libmpdec with the same arch options + # the python interpreter was built with + libmpdec_machine=universal + elif test $ac_cv_sizeof_size_t -eq 8; then + if test "$ac_cv_gcc_asm_for_x64" = yes; then + libmpdec_machine=x64 + elif test "$ac_cv_type___uint128_t" = yes; then + libmpdec_machine=uint128 + else + libmpdec_machine=ansi64 + fi + elif test $ac_cv_sizeof_size_t -eq 4; then + if test "$ac_cv_gcc_asm_for_x87" = yes -a "$libmpdec_system" != sunos; then + case $ac_cv_cc_name in #( *gcc*) : libmpdec_machine=ppro ;; #( *clang*) : libmpdec_machine=ppro ;; #( *) : libmpdec_machine=ansi32 - ;; + ;; esac - else - libmpdec_machine=ansi32 - fi -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $libmpdec_machine" >&5 + else + libmpdec_machine=ansi32 + fi + fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $libmpdec_machine" >&5 printf "%s\n" "$libmpdec_machine" >&6; } -case $libmpdec_machine in #( + case $libmpdec_machine in #( x64) : as_fn_append LIBMPDEC_CFLAGS " -DCONFIG_64=1 -DASM=1" ;; #( uint128) : @@ -14612,8 +15785,9 @@ case $libmpdec_machine in #( as_fn_append LIBMPDEC_CFLAGS " -DUNIVERSAL=1" ;; #( *) : as_fn_error $? "_decimal: unsupported architecture" "$LINENO" 5 - ;; + ;; esac +fi if test "$have_ipa_pure_const_bug" = yes; then # Some versions of gcc miscompile inline asm: @@ -14632,6 +15806,9 @@ fi + + + if test "$ac_sys_system" = "Emscripten" -a -z "$LIBSQLITE3_CFLAGS" -a -z "$LIBSQLITE3_LIBS" then : @@ -14695,12 +15872,12 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - LIBSQLITE3_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "sqlite3 >= 3.15.2" 2>&1` + LIBSQLITE3_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "sqlite3 >= 3.15.2" 2>&1` else - LIBSQLITE3_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "sqlite3 >= 3.15.2" 2>&1` + LIBSQLITE3_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "sqlite3 >= 3.15.2" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$LIBSQLITE3_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$LIBSQLITE3_PKG_ERRORS" >&5 LIBSQLITE3_CFLAGS=${LIBSQLITE3_CFLAGS-""} @@ -14716,8 +15893,8 @@ printf "%s\n" "no" >&6; } else - LIBSQLITE3_CFLAGS=$pkg_cv_LIBSQLITE3_CFLAGS - LIBSQLITE3_LIBS=$pkg_cv_LIBSQLITE3_LIBS + LIBSQLITE3_CFLAGS=$pkg_cv_LIBSQLITE3_CFLAGS + LIBSQLITE3_LIBS=$pkg_cv_LIBSQLITE3_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } @@ -14733,7 +15910,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBSQLITE3_CFLAGS" - LDFLAGS="$LIBSQLITE3_LIBS $LDFLAGS" + LIBS="$LIBS $LIBSQLITE3_LIBS" ac_fn_c_check_header_compile "$LINENO" "sqlite3.h" "ac_cv_header_sqlite3_h" "$ac_includes_default" if test "x$ac_cv_header_sqlite3_h" = xyes @@ -14769,16 +15946,22 @@ printf %s "checking for sqlite3_bind_double in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_bind_double+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_bind_double (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_bind_double (void); int main (void) { @@ -14790,12 +15973,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_bind_double=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_bind_double=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_bind_double=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_bind_double" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_bind_double" >&6; } @@ -14805,10 +15990,11 @@ then : LIBS="-lsqlite3 $LIBS" -else $as_nop - +else case e in #( + e) have_supported_sqlite3=no - + ;; +esac fi @@ -14818,16 +16004,22 @@ printf %s "checking for sqlite3_column_decltype in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_column_decltype+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_column_decltype (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_column_decltype (void); int main (void) { @@ -14839,12 +16031,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_column_decltype=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_column_decltype=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_column_decltype=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_column_decltype" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_column_decltype" >&6; } @@ -14854,10 +16048,11 @@ then : LIBS="-lsqlite3 $LIBS" -else $as_nop - +else case e in #( + e) have_supported_sqlite3=no - + ;; +esac fi @@ -14867,16 +16062,22 @@ printf %s "checking for sqlite3_column_double in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_column_double+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_column_double (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_column_double (void); int main (void) { @@ -14888,12 +16089,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_column_double=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_column_double=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_column_double=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_column_double" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_column_double" >&6; } @@ -14903,10 +16106,11 @@ then : LIBS="-lsqlite3 $LIBS" -else $as_nop - +else case e in #( + e) have_supported_sqlite3=no - + ;; +esac fi @@ -14916,16 +16120,22 @@ printf %s "checking for sqlite3_complete in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_complete+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_complete (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_complete (void); int main (void) { @@ -14937,12 +16147,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_complete=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_complete=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_complete=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_complete" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_complete" >&6; } @@ -14952,10 +16164,11 @@ then : LIBS="-lsqlite3 $LIBS" -else $as_nop - +else case e in #( + e) have_supported_sqlite3=no - + ;; +esac fi @@ -14965,16 +16178,22 @@ printf %s "checking for sqlite3_progress_handler in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_progress_handler+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_progress_handler (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_progress_handler (void); int main (void) { @@ -14986,12 +16205,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_progress_handler=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_progress_handler=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_progress_handler=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_progress_handler" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_progress_handler" >&6; } @@ -15001,10 +16222,11 @@ then : LIBS="-lsqlite3 $LIBS" -else $as_nop - +else case e in #( + e) have_supported_sqlite3=no - + ;; +esac fi @@ -15014,16 +16236,22 @@ printf %s "checking for sqlite3_result_double in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_result_double+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_result_double (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_result_double (void); int main (void) { @@ -15035,12 +16263,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_result_double=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_result_double=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_result_double=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_result_double" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_result_double" >&6; } @@ -15050,10 +16280,11 @@ then : LIBS="-lsqlite3 $LIBS" -else $as_nop - +else case e in #( + e) have_supported_sqlite3=no - + ;; +esac fi @@ -15063,16 +16294,22 @@ printf %s "checking for sqlite3_set_authorizer in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_set_authorizer+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_set_authorizer (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_set_authorizer (void); int main (void) { @@ -15084,12 +16321,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_set_authorizer=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_set_authorizer=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_set_authorizer=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_set_authorizer" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_set_authorizer" >&6; } @@ -15099,10 +16338,11 @@ then : LIBS="-lsqlite3 $LIBS" -else $as_nop - +else case e in #( + e) have_supported_sqlite3=no - + ;; +esac fi @@ -15112,16 +16352,22 @@ printf %s "checking for sqlite3_trace_v2 in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_trace_v2+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_trace_v2 (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_trace_v2 (void); int main (void) { @@ -15133,12 +16379,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_trace_v2=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_trace_v2=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_trace_v2=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_trace_v2" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_trace_v2" >&6; } @@ -15148,8 +16396,8 @@ then : LIBS="-lsqlite3 $LIBS" -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for sqlite3_trace in -lsqlite3" >&5 @@ -15157,16 +16405,22 @@ printf %s "checking for sqlite3_trace in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_trace+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_trace (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_trace (void); int main (void) { @@ -15178,12 +16432,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_trace=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_trace=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_trace=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_trace" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_trace" >&6; } @@ -15193,15 +16449,17 @@ then : LIBS="-lsqlite3 $LIBS" -else $as_nop - +else case e in #( + e) have_supported_sqlite3=no - + ;; +esac fi - + ;; +esac fi @@ -15211,16 +16469,22 @@ printf %s "checking for sqlite3_value_double in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_value_double+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_value_double (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_value_double (void); int main (void) { @@ -15232,12 +16496,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_value_double=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_value_double=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_value_double=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_value_double" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_value_double" >&6; } @@ -15247,10 +16513,11 @@ then : LIBS="-lsqlite3 $LIBS" -else $as_nop - +else case e in #( + e) have_supported_sqlite3=no - + ;; +esac fi @@ -15259,16 +16526,22 @@ printf %s "checking for sqlite3_load_extension in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_load_extension+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_load_extension (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_load_extension (void); int main (void) { @@ -15280,21 +16553,24 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_load_extension=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_load_extension=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_load_extension=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_load_extension" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_load_extension" >&6; } if test "x$ac_cv_lib_sqlite3_sqlite3_load_extension" = xyes then : have_sqlite3_load_extension=yes -else $as_nop - have_sqlite3_load_extension=no - +else case e in #( + e) have_sqlite3_load_extension=no + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for sqlite3_serialize in -lsqlite3" >&5 @@ -15302,16 +16578,22 @@ printf %s "checking for sqlite3_serialize in -lsqlite3... " >&6; } if test ${ac_cv_lib_sqlite3_sqlite3_serialize+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lsqlite3 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char sqlite3_serialize (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_serialize (void); int main (void) { @@ -15323,12 +16605,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_sqlite3_sqlite3_serialize=yes -else $as_nop - ac_cv_lib_sqlite3_sqlite3_serialize=no +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_serialize=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_serialize" >&5 printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_serialize" >&6; } @@ -15342,10 +16626,11 @@ printf "%s\n" "#define PY_SQLITE_HAVE_SERIALIZE 1" >>confdefs.h fi -else $as_nop - +else case e in #( + e) have_supported_sqlite3=no - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext @@ -15373,22 +16658,24 @@ printf "%s\n" "n/a" >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: Your version of SQLite does not support loadable extensions" >&5 printf "%s\n" "$as_me: WARNING: Your version of SQLite does not support loadable extensions" >&2;} -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } printf "%s\n" "#define PY_SQLITE_ENABLE_LOAD_EXTENSION 1" >>confdefs.h - + ;; +esac fi -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - + ;; +esac fi @@ -15459,24 +16746,24 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - TCLTK_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "$_QUERY" 2>&1` + TCLTK_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "$_QUERY" 2>&1` else - TCLTK_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "$_QUERY" 2>&1` + TCLTK_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "$_QUERY" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$TCLTK_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$TCLTK_PKG_ERRORS" >&5 - found_tcltk=no + found_tcltk=no elif test $pkg_failed = untried; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - found_tcltk=no + found_tcltk=no else - TCLTK_CFLAGS=$pkg_cv_TCLTK_CFLAGS - TCLTK_LIBS=$pkg_cv_TCLTK_LIBS + TCLTK_CFLAGS=$pkg_cv_TCLTK_CFLAGS + TCLTK_LIBS=$pkg_cv_TCLTK_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } - found_tcltk=yes + found_tcltk=yes fi fi @@ -15556,14 +16843,14 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - X11_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "x11" 2>&1` + X11_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "x11" 2>&1` else - X11_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "x11" 2>&1` + X11_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "x11" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$X11_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$X11_PKG_ERRORS" >&5 - as_fn_error $? "Package requirements (x11) were not met: + as_fn_error $? "Package requirements (x11) were not met: $X11_PKG_ERRORS @@ -15576,8 +16863,8 @@ See the pkg-config man page for more details." "$LINENO" 5 elif test $pkg_failed = untried; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it is in your PATH or set the PKG_CONFIG environment variable to the full path to pkg-config. @@ -15587,10 +16874,10 @@ and X11_LIBS to avoid the need to call pkg-config. See the pkg-config man page for more details. To get pkg-config, see . -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else - X11_CFLAGS=$pkg_cv_X11_CFLAGS - X11_LIBS=$pkg_cv_X11_LIBS + X11_CFLAGS=$pkg_cv_X11_CFLAGS + X11_LIBS=$pkg_cv_X11_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } @@ -15613,7 +16900,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $TCLTK_CFLAGS" - LIBS="$TCLTK_LIBS $LDFLAGS" + LIBS="$LIBS $TCLTK_LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -15655,10 +16942,11 @@ then : have_tcltk=yes as_fn_append TCLTK_CFLAGS " -Wno-strict-prototypes -DWITH_APPINIT=1" -else $as_nop - +else case e in #( + e) have_tcltk=no - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext @@ -15679,7 +16967,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $GDBM_CFLAGS" - LDFLAGS="$GDBM_LIBS $LDFLAGS" + LIBS="$LIBS $GDBM_LIBS" for ac_header in gdbm.h do : ac_fn_c_check_header_compile "$LINENO" "gdbm.h" "ac_cv_header_gdbm_h" "$ac_includes_default" @@ -15692,16 +16980,22 @@ printf %s "checking for gdbm_open in -lgdbm... " >&6; } if test ${ac_cv_lib_gdbm_gdbm_open+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lgdbm $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char gdbm_open (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char gdbm_open (void); int main (void) { @@ -15713,12 +17007,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_gdbm_gdbm_open=yes -else $as_nop - ac_cv_lib_gdbm_gdbm_open=no +else case e in #( + e) ac_cv_lib_gdbm_gdbm_open=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_gdbm_gdbm_open" >&5 printf "%s\n" "$ac_cv_lib_gdbm_gdbm_open" >&6; } @@ -15728,13 +17024,15 @@ then : have_gdbm=yes GDBM_LIBS=${GDBM_LIBS-"-lgdbm"} -else $as_nop - have_gdbm=no +else case e in #( + e) have_gdbm=no ;; +esac fi -else $as_nop - have_gdbm=no +else case e in #( + e) have_gdbm=no ;; +esac fi done @@ -15764,15 +17062,21 @@ printf %s "checking for library containing dbm_open... " >&6; } if test ${ac_cv_search_dbm_open+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_func_search_save_LIBS=$LIBS +else case e in #( + e) ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char dbm_open (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char dbm_open (void); int main (void) { @@ -15803,11 +17107,13 @@ done if test ${ac_cv_search_dbm_open+y} then : -else $as_nop - ac_cv_search_dbm_open=no +else case e in #( + e) ac_cv_search_dbm_open=no ;; +esac fi rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS +LIBS=$ac_func_search_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_dbm_open" >&5 printf "%s\n" "$ac_cv_search_dbm_open" >&6; } @@ -15856,18 +17162,20 @@ printf "%s\n" "$have_ndbm ($dbm_ndbm)" >&6; } if test ${ac_cv_header_gdbm_slash_ndbm_h+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ac_fn_c_check_header_compile "$LINENO" "gdbm/ndbm.h" "ac_cv_header_gdbm_ndbm_h" "$ac_includes_default" if test "x$ac_cv_header_gdbm_ndbm_h" = xyes then : ac_cv_header_gdbm_slash_ndbm_h=yes -else $as_nop - ac_cv_header_gdbm_slash_ndbm_h=no - +else case e in #( + e) ac_cv_header_gdbm_slash_ndbm_h=no + ;; +esac fi - + ;; +esac fi if test "x$ac_cv_header_gdbm_slash_ndbm_h" = xyes @@ -15883,18 +17191,20 @@ fi if test ${ac_cv_header_gdbm_dash_ndbm_h+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ac_fn_c_check_header_compile "$LINENO" "gdbm-ndbm.h" "ac_cv_header_gdbm_ndbm_h" "$ac_includes_default" if test "x$ac_cv_header_gdbm_ndbm_h" = xyes then : ac_cv_header_gdbm_dash_ndbm_h=yes -else $as_nop - ac_cv_header_gdbm_dash_ndbm_h=no - +else case e in #( + e) ac_cv_header_gdbm_dash_ndbm_h=no + ;; +esac fi - + ;; +esac fi if test "x$ac_cv_header_gdbm_dash_ndbm_h" = xyes @@ -15920,15 +17230,21 @@ printf %s "checking for library containing dbm_open... " >&6; } if test ${ac_cv_search_dbm_open+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_func_search_save_LIBS=$LIBS +else case e in #( + e) ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char dbm_open (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char dbm_open (void); int main (void) { @@ -15959,11 +17275,13 @@ done if test ${ac_cv_search_dbm_open+y} then : -else $as_nop - ac_cv_search_dbm_open=no +else case e in #( + e) ac_cv_search_dbm_open=no ;; +esac fi rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS +LIBS=$ac_func_search_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_dbm_open" >&5 printf "%s\n" "$ac_cv_search_dbm_open" >&6; } @@ -15972,8 +17290,9 @@ if test "$ac_res" != no then : test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" have_gdbm_compat=yes -else $as_nop - have_gdbm_compat=no +else case e in #( + e) have_gdbm_compat=no ;; +esac fi @@ -15999,8 +17318,8 @@ printf %s "checking for libdb... " >&6; } if test ${ac_cv_have_libdb+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) save_CFLAGS=$CFLAGS save_CPPFLAGS=$CPPFLAGS save_LDFLAGS=$LDFLAGS @@ -16029,8 +17348,9 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_have_libdb=yes -else $as_nop - ac_cv_have_libdb=no +else case e in #( + e) ac_cv_have_libdb=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext @@ -16041,7 +17361,8 @@ LDFLAGS=$save_LDFLAGS LIBS=$save_LIBS - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_have_libdb" >&5 printf "%s\n" "$ac_cv_have_libdb" >&6; } @@ -16066,8 +17387,9 @@ printf %s "checking for --with-dbmliborder... " >&6; } if test ${with_dbmliborder+y} then : withval=$with_dbmliborder; -else $as_nop - with_dbmliborder=gdbm:ndbm:bdb +else case e in #( + e) with_dbmliborder=gdbm:ndbm:bdb ;; +esac fi @@ -16184,8 +17506,8 @@ printf %s "checking for _POSIX_THREADS defined in unistd.h... " >&6; } if test ${ac_cv_defined__POSIX_THREADS_unistd_h+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -16206,18 +17528,21 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_defined__POSIX_THREADS_unistd_h=yes -else $as_nop - ac_cv_defined__POSIX_THREADS_unistd_h=no +else case e in #( + e) ac_cv_defined__POSIX_THREADS_unistd_h=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined__POSIX_THREADS_unistd_h" >&5 printf "%s\n" "$ac_cv_defined__POSIX_THREADS_unistd_h" >&6; } if test $ac_cv_defined__POSIX_THREADS_unistd_h != "no" then : unistd_defines_pthreads=yes -else $as_nop - unistd_defines_pthreads=no +else case e in #( + e) unistd_defines_pthreads=no ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $unistd_defines_pthreads" >&5 printf "%s\n" "$unistd_defines_pthreads" >&6; } @@ -16255,8 +17580,8 @@ then : printf "%s\n" "yes" >&6; } posix_threads=yes -else $as_nop - +else case e in #( + e) LIBS=$_libs ac_fn_c_check_func "$LINENO" "pthread_detach" "ac_cv_func_pthread_detach" if test "x$ac_cv_func_pthread_detach" = xyes @@ -16264,23 +17589,29 @@ then : posix_threads=yes -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for pthread_create in -lpthreads" >&5 printf %s "checking for pthread_create in -lpthreads... " >&6; } if test ${ac_cv_lib_pthreads_pthread_create+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lpthreads $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char pthread_create (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char pthread_create (void); int main (void) { @@ -16292,12 +17623,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_pthreads_pthread_create=yes -else $as_nop - ac_cv_lib_pthreads_pthread_create=no +else case e in #( + e) ac_cv_lib_pthreads_pthread_create=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_pthreads_pthread_create" >&5 printf "%s\n" "$ac_cv_lib_pthreads_pthread_create" >&6; } @@ -16307,23 +17640,29 @@ then : posix_threads=yes LIBS="$LIBS -lpthreads" -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for pthread_create in -lc_r" >&5 printf %s "checking for pthread_create in -lc_r... " >&6; } if test ${ac_cv_lib_c_r_pthread_create+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lc_r $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char pthread_create (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char pthread_create (void); int main (void) { @@ -16335,12 +17674,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_c_r_pthread_create=yes -else $as_nop - ac_cv_lib_c_r_pthread_create=no +else case e in #( + e) ac_cv_lib_c_r_pthread_create=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_c_r_pthread_create" >&5 printf "%s\n" "$ac_cv_lib_c_r_pthread_create" >&6; } @@ -16350,23 +17691,29 @@ then : posix_threads=yes LIBS="$LIBS -lc_r" -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __pthread_create_system in -lpthread" >&5 printf %s "checking for __pthread_create_system in -lpthread... " >&6; } if test ${ac_cv_lib_pthread___pthread_create_system+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lpthread $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char __pthread_create_system (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char __pthread_create_system (void); int main (void) { @@ -16378,12 +17725,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_pthread___pthread_create_system=yes -else $as_nop - ac_cv_lib_pthread___pthread_create_system=no +else case e in #( + e) ac_cv_lib_pthread___pthread_create_system=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_pthread___pthread_create_system" >&5 printf "%s\n" "$ac_cv_lib_pthread___pthread_create_system" >&6; } @@ -16393,23 +17742,29 @@ then : posix_threads=yes LIBS="$LIBS -lpthread" -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for pthread_create in -lcma" >&5 printf %s "checking for pthread_create in -lcma... " >&6; } if test ${ac_cv_lib_cma_pthread_create+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lcma $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char pthread_create (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char pthread_create (void); int main (void) { @@ -16421,12 +17776,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_cma_pthread_create=yes -else $as_nop - ac_cv_lib_cma_pthread_create=no +else case e in #( + e) ac_cv_lib_cma_pthread_create=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_cma_pthread_create" >&5 printf "%s\n" "$ac_cv_lib_cma_pthread_create" >&6; } @@ -16436,8 +17793,8 @@ then : posix_threads=yes LIBS="$LIBS -lcma" -else $as_nop - +else case e in #( + e) case $ac_sys_system in #( WASI) : posix_threads=stub ;; #( @@ -16445,17 +17802,23 @@ else $as_nop as_fn_error $? "could not find pthreads on your system" "$LINENO" 5 ;; esac - + ;; +esac fi - + ;; +esac fi - + ;; +esac fi - + ;; +esac fi - + ;; +esac fi - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext @@ -16465,16 +17828,22 @@ printf %s "checking for usconfig in -lmpc... " >&6; } if test ${ac_cv_lib_mpc_usconfig+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lmpc $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char usconfig (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char usconfig (void); int main (void) { @@ -16486,12 +17855,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_mpc_usconfig=yes -else $as_nop - ac_cv_lib_mpc_usconfig=no +else case e in #( + e) ac_cv_lib_mpc_usconfig=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mpc_usconfig" >&5 printf "%s\n" "$ac_cv_lib_mpc_usconfig" >&6; } @@ -16537,12 +17908,12 @@ printf %s "checking if PTHREAD_SCOPE_SYSTEM is supported... " >&6; } if test ${ac_cv_pthread_system_supported+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_pthread_system_supported=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -16562,14 +17933,17 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_pthread_system_supported=yes -else $as_nop - ac_cv_pthread_system_supported=no +else case e in #( + e) ac_cv_pthread_system_supported=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_pthread_system_supported" >&5 printf "%s\n" "$ac_cv_pthread_system_supported" >&6; } @@ -16633,8 +18007,8 @@ printf "%s\n" "yes" >&6; } ipv6=yes ;; esac -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* AF_INET6 available check */ @@ -16653,10 +18027,11 @@ then : ipv6=yes -else $as_nop - +else case e in #( + e) ipv6=no - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext @@ -16696,12 +18071,13 @@ then : printf "%s\n" "yes" >&6; } ipv6=yes -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } ipv6=no - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext fi @@ -16710,7 +18086,8 @@ if test "$ipv6" = "yes"; then printf "%s\n" "#define ENABLE_IPV6 1" >>confdefs.h fi - + ;; +esac fi @@ -16729,8 +18106,8 @@ printf %s "checking for IPV6_INRIA_VERSION defined in netinet/in.h... " >&6; } if test ${ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -16751,10 +18128,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h=yes -else $as_nop - ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h=no +else case e in #( + e) ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h" >&5 printf "%s\n" "$ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h" >&6; } @@ -16770,8 +18149,8 @@ printf %s "checking for __KAME__ defined in netinet/in.h... " >&6; } if test ${ac_cv_defined___KAME___netinet_in_h+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -16792,10 +18171,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_defined___KAME___netinet_in_h=yes -else $as_nop - ac_cv_defined___KAME___netinet_in_h=no +else case e in #( + e) ac_cv_defined___KAME___netinet_in_h=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined___KAME___netinet_in_h" >&5 printf "%s\n" "$ac_cv_defined___KAME___netinet_in_h" >&6; } @@ -16814,8 +18195,8 @@ printf %s "checking for __GLIBC__ defined in features.h... " >&6; } if test ${ac_cv_defined___GLIBC___features_h+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -16836,10 +18217,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_defined___GLIBC___features_h=yes -else $as_nop - ac_cv_defined___GLIBC___features_h=no +else case e in #( + e) ac_cv_defined___GLIBC___features_h=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined___GLIBC___features_h" >&5 printf "%s\n" "$ac_cv_defined___GLIBC___features_h" >&6; } @@ -16872,8 +18255,8 @@ printf %s "checking for _TOSHIBA_INET6 defined in sys/param.h... " >&6; } if test ${ac_cv_defined__TOSHIBA_INET6_sys_param_h+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -16894,10 +18277,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_defined__TOSHIBA_INET6_sys_param_h=yes -else $as_nop - ac_cv_defined__TOSHIBA_INET6_sys_param_h=no +else case e in #( + e) ac_cv_defined__TOSHIBA_INET6_sys_param_h=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined__TOSHIBA_INET6_sys_param_h" >&5 printf "%s\n" "$ac_cv_defined__TOSHIBA_INET6_sys_param_h" >&6; } @@ -16915,8 +18300,8 @@ printf %s "checking for __V6D__ defined in /usr/local/v6/include/sys/v6config.h. if test ${ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -16937,10 +18322,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h=yes -else $as_nop - ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h=no +else case e in #( + e) ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h" >&5 printf "%s\n" "$ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h" >&6; } @@ -16959,8 +18346,8 @@ printf %s "checking for _ZETA_MINAMI_INET6 defined in sys/param.h... " >&6; } if test ${ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -16981,10 +18368,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h=yes -else $as_nop - ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h=no +else case e in #( + e) ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h" >&5 printf "%s\n" "$ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h" >&6; } @@ -17020,10 +18409,11 @@ then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: libc" >&5 printf "%s\n" "libc" >&6; } -else $as_nop - +else case e in #( + e) as_fn_error $? "No $ipv6lib library found; cannot continue. You need to fetch lib$ipv6lib.a from appropriate ipv6 kit and compile beforehand." "$LINENO" 5 - + ;; +esac fi fi fi @@ -17034,8 +18424,8 @@ printf %s "checking CAN_RAW_FD_FRAMES... " >&6; } if test ${ac_cv_can_raw_fd_frames+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* CAN_RAW_FD_FRAMES available check */ @@ -17051,11 +18441,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_can_raw_fd_frames=yes -else $as_nop - ac_cv_can_raw_fd_frames=no +else case e in #( + e) ac_cv_can_raw_fd_frames=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_can_raw_fd_frames" >&5 printf "%s\n" "$ac_cv_can_raw_fd_frames" >&6; } @@ -17073,8 +18465,8 @@ printf %s "checking for CAN_RAW_JOIN_FILTERS... " >&6; } if test ${ac_cv_can_raw_join_filters+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -17090,11 +18482,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_can_raw_join_filters=yes -else $as_nop - ac_cv_can_raw_join_filters=no +else case e in #( + e) ac_cv_can_raw_join_filters=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_can_raw_join_filters" >&5 printf "%s\n" "$ac_cv_can_raw_join_filters" >&6; } @@ -17136,8 +18530,8 @@ printf %s "checking for stdatomic.h... " >&6; } if test ${ac_cv_header_stdatomic_h+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -17157,12 +18551,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_header_stdatomic_h=yes -else $as_nop - ac_cv_header_stdatomic_h=no +else case e in #( + e) ac_cv_header_stdatomic_h=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_stdatomic_h" >&5 printf "%s\n" "$ac_cv_header_stdatomic_h" >&6; } @@ -17182,8 +18578,8 @@ printf %s "checking for builtin __atomic_load_n and __atomic_store_n functions.. if test ${ac_cv_builtin_atomic+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -17200,12 +18596,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_builtin_atomic=yes -else $as_nop - ac_cv_builtin_atomic=no +else case e in #( + e) ac_cv_builtin_atomic=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_builtin_atomic" >&5 printf "%s\n" "$ac_cv_builtin_atomic" >&6; } @@ -17227,9 +18625,10 @@ printf %s "checking for --with-mimalloc... " >&6; } if test ${with_mimalloc+y} then : withval=$with_mimalloc; -else $as_nop - with_mimalloc="$ac_cv_header_stdatomic_h" - +else case e in #( + e) with_mimalloc="$ac_cv_header_stdatomic_h" + ;; +esac fi @@ -17250,6 +18649,7 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_mimalloc" >&5 printf "%s\n" "$with_mimalloc" >&6; } +INSTALL_MIMALLOC=$with_mimalloc @@ -17285,31 +18685,6 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_pymalloc" >&5 printf "%s\n" "$with_pymalloc" >&6; } -# Check whether objects such as float, tuple and dict are using -# freelists to optimization memory allocation. -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-freelists" >&5 -printf %s "checking for --with-freelists... " >&6; } - -# Check whether --with-freelists was given. -if test ${with_freelists+y} -then : - withval=$with_freelists; -fi - - -if test -z "$with_freelists" -then - with_freelists="yes" -fi -if test "$with_freelists" != "no" -then - -printf "%s\n" "#define WITH_FREELISTS 1" >>confdefs.h - -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_freelists" >&5 -printf "%s\n" "$with_freelists" >&6; } - # Check for --with-c-locale-coercion { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-c-locale-coercion" >&5 printf %s "checking for --with-c-locale-coercion... " >&6; } @@ -17342,9 +18717,10 @@ printf %s "checking for --with-valgrind... " >&6; } if test ${with_valgrind+y} then : withval=$with_valgrind; -else $as_nop - with_valgrind=no - +else case e in #( + e) with_valgrind=no + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_valgrind" >&5 @@ -17356,9 +18732,10 @@ then : printf "%s\n" "#define WITH_VALGRIND 1" >>confdefs.h -else $as_nop - as_fn_error $? "Valgrind support requested but headers not available" "$LINENO" 5 - +else case e in #( + e) as_fn_error $? "Valgrind support requested but headers not available" "$LINENO" 5 + ;; +esac fi OPT="-DDYNAMIC_ANNOTATIONS_ENABLED=1 $OPT" @@ -17372,8 +18749,9 @@ printf %s "checking for --with-dtrace... " >&6; } if test ${with_dtrace+y} then : withval=$with_dtrace; -else $as_nop - with_dtrace=no +else case e in #( + e) with_dtrace=no ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_dtrace" >&5 @@ -17396,8 +18774,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_path_DTRACE+y} then : printf %s "(cached) " >&6 -else $as_nop - case $DTRACE in +else case e in #( + e) case $DTRACE in [\\/]* | ?:[\\/]*) ac_cv_path_DTRACE="$DTRACE" # Let the user override the test with a path. ;; @@ -17423,6 +18801,7 @@ IFS=$as_save_IFS test -z "$ac_cv_path_DTRACE" && ac_cv_path_DTRACE="not found" ;; +esac ;; esac fi DTRACE=$ac_cv_path_DTRACE @@ -17452,12 +18831,13 @@ printf %s "checking whether DTrace probes require linking... " >&6; } if test ${ac_cv_dtrace_link+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_cv_dtrace_link=no +else case e in #( + e) ac_cv_dtrace_link=no echo 'BEGIN{}' > conftest.d "$DTRACE" $DFLAGS -G -s conftest.d -o conftest.o > /dev/null 2>&1 && \ ac_cv_dtrace_link=yes - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_dtrace_link" >&5 printf "%s\n" "$ac_cv_dtrace_link" >&6; } @@ -17545,14 +18925,22 @@ fi if test "$ac_sys_system" = "Linux-android"; then # When these functions are used in an unprivileged process, they crash rather # than returning an error. - privileged_funcs="chroot initgroups setegid seteuid setgid setregid setresgid - setresuid setreuid setuid" - - # These functions are unimplemented and always return an error. - unimplemented_funcs="sem_open sem_unlink" + blocked_funcs="chroot initgroups setegid seteuid setgid sethostname + setregid setresgid setresuid setreuid setuid" + + # These functions are unimplemented and always return an error + # (https://android.googlesource.com/platform/system/sepolicy/+/refs/heads/android13-release/public/domain.te#1044) + blocked_funcs="$blocked_funcs sem_open sem_unlink" + + # Before API level 23, when fchmodat is called with the unimplemented flag + # AT_SYMLINK_NOFOLLOW, instead of returning ENOTSUP as it should, it actually + # follows the symlink. + if test "$ANDROID_API_LEVEL" -lt 23; then + blocked_funcs="$blocked_funcs fchmodat" + fi - for name in $privileged_funcs $unimplemented_funcs; do - as_func_var=`printf "%s\n" "ac_cv_func_$name" | $as_tr_sh` + for name in $blocked_funcs; do + as_func_var=`printf "%s\n" "ac_cv_func_$name" | sed "$as_sed_sh"` eval "$as_func_var=no" @@ -17625,6 +19013,12 @@ if test "x$ac_cv_func_ctermid" = xyes then : printf "%s\n" "#define HAVE_CTERMID 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "dladdr" "ac_cv_func_dladdr" +if test "x$ac_cv_func_dladdr" = xyes +then : + printf "%s\n" "#define HAVE_DLADDR 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "dup" "ac_cv_func_dup" if test "x$ac_cv_func_dup" = xyes @@ -18207,6 +19601,18 @@ if test "x$ac_cv_func_pthread_kill" = xyes then : printf "%s\n" "#define HAVE_PTHREAD_KILL 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "pthread_getname_np" "ac_cv_func_pthread_getname_np" +if test "x$ac_cv_func_pthread_getname_np" = xyes +then : + printf "%s\n" "#define HAVE_PTHREAD_GETNAME_NP 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "pthread_setname_np" "ac_cv_func_pthread_setname_np" +if test "x$ac_cv_func_pthread_setname_np" = xyes +then : + printf "%s\n" "#define HAVE_PTHREAD_SETNAME_NP 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "ptsname" "ac_cv_func_ptsname" if test "x$ac_cv_func_ptsname" = xyes @@ -18598,10 +20004,10 @@ then : printf "%s\n" "#define HAVE_TRUNCATE 1" >>confdefs.h fi -ac_fn_c_check_func "$LINENO" "ttyname" "ac_cv_func_ttyname" -if test "x$ac_cv_func_ttyname" = xyes +ac_fn_c_check_func "$LINENO" "ttyname_r" "ac_cv_func_ttyname_r" +if test "x$ac_cv_func_ttyname_r" = xyes then : - printf "%s\n" "#define HAVE_TTYNAME 1" >>confdefs.h + printf "%s\n" "#define HAVE_TTYNAME_R 1" >>confdefs.h fi ac_fn_c_check_func "$LINENO" "umask" "ac_cv_func_umask" @@ -18752,8 +20158,8 @@ printf %s "checking for $CC options needed to detect all undeclared functions... if test ${ac_cv_c_undeclared_builtin_options+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_save_CFLAGS=$CFLAGS +else case e in #( + e) ac_save_CFLAGS=$CFLAGS ac_cv_c_undeclared_builtin_options='cannot detect' for ac_arg in '' -fno-builtin; do CFLAGS="$ac_save_CFLAGS $ac_arg" @@ -18772,8 +20178,8 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : -else $as_nop - # This test program should compile successfully. +else case e in #( + e) # This test program should compile successfully. # No library function is consistently available on # freestanding implementations, so test against a dummy # declaration. Include always-available headers on the @@ -18801,26 +20207,29 @@ then : if test x"$ac_arg" = x then : ac_cv_c_undeclared_builtin_options='none needed' -else $as_nop - ac_cv_c_undeclared_builtin_options=$ac_arg +else case e in #( + e) ac_cv_c_undeclared_builtin_options=$ac_arg ;; +esac fi break fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext done CFLAGS=$ac_save_CFLAGS - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_undeclared_builtin_options" >&5 printf "%s\n" "$ac_cv_c_undeclared_builtin_options" >&6; } case $ac_cv_c_undeclared_builtin_options in #( 'cannot detect') : - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error $? "cannot make $CC report undeclared builtins -See \`config.log' for more details" "$LINENO" 5; } ;; #( +See 'config.log' for more details" "$LINENO" 5; } ;; #( 'none needed') : ac_c_undeclared_builtin_options='' ;; #( *) : @@ -18846,8 +20255,8 @@ printf %s "checking for chroot... " >&6; } if test ${ac_cv_func_chroot+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -18861,11 +20270,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_chroot=yes -else $as_nop - ac_cv_func_chroot=no +else case e in #( + e) ac_cv_func_chroot=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_chroot" >&5 printf "%s\n" "$ac_cv_func_chroot" >&6; } @@ -18885,8 +20296,8 @@ printf %s "checking for link... " >&6; } if test ${ac_cv_func_link+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -18900,11 +20311,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_link=yes -else $as_nop - ac_cv_func_link=no +else case e in #( + e) ac_cv_func_link=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_link" >&5 printf "%s\n" "$ac_cv_func_link" >&6; } @@ -18924,8 +20337,8 @@ printf %s "checking for symlink... " >&6; } if test ${ac_cv_func_symlink+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -18939,11 +20352,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_symlink=yes -else $as_nop - ac_cv_func_symlink=no +else case e in #( + e) ac_cv_func_symlink=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_symlink" >&5 printf "%s\n" "$ac_cv_func_symlink" >&6; } @@ -18963,8 +20378,8 @@ printf %s "checking for fchdir... " >&6; } if test ${ac_cv_func_fchdir+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -18978,11 +20393,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_fchdir=yes -else $as_nop - ac_cv_func_fchdir=no +else case e in #( + e) ac_cv_func_fchdir=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_fchdir" >&5 printf "%s\n" "$ac_cv_func_fchdir" >&6; } @@ -19002,8 +20419,8 @@ printf %s "checking for fsync... " >&6; } if test ${ac_cv_func_fsync+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -19017,11 +20434,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_fsync=yes -else $as_nop - ac_cv_func_fsync=no +else case e in #( + e) ac_cv_func_fsync=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_fsync" >&5 printf "%s\n" "$ac_cv_func_fsync" >&6; } @@ -19041,8 +20460,8 @@ printf %s "checking for fdatasync... " >&6; } if test ${ac_cv_func_fdatasync+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -19056,11 +20475,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_fdatasync=yes -else $as_nop - ac_cv_func_fdatasync=no +else case e in #( + e) ac_cv_func_fdatasync=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_fdatasync" >&5 printf "%s\n" "$ac_cv_func_fdatasync" >&6; } @@ -19080,8 +20501,8 @@ printf %s "checking for epoll_create... " >&6; } if test ${ac_cv_func_epoll_create+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -19095,11 +20516,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_epoll_create=yes -else $as_nop - ac_cv_func_epoll_create=no +else case e in #( + e) ac_cv_func_epoll_create=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_epoll_create" >&5 printf "%s\n" "$ac_cv_func_epoll_create" >&6; } @@ -19119,8 +20542,8 @@ printf %s "checking for epoll_create1... " >&6; } if test ${ac_cv_func_epoll_create1+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -19134,11 +20557,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_epoll_create1=yes -else $as_nop - ac_cv_func_epoll_create1=no +else case e in #( + e) ac_cv_func_epoll_create1=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_epoll_create1" >&5 printf "%s\n" "$ac_cv_func_epoll_create1" >&6; } @@ -19158,8 +20583,8 @@ printf %s "checking for kqueue... " >&6; } if test ${ac_cv_func_kqueue+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -19176,11 +20601,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_kqueue=yes -else $as_nop - ac_cv_func_kqueue=no +else case e in #( + e) ac_cv_func_kqueue=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_kqueue" >&5 printf "%s\n" "$ac_cv_func_kqueue" >&6; } @@ -19200,8 +20627,8 @@ printf %s "checking for prlimit... " >&6; } if test ${ac_cv_func_prlimit+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -19218,11 +20645,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_prlimit=yes -else $as_nop - ac_cv_func_prlimit=no +else case e in #( + e) ac_cv_func_prlimit=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_prlimit" >&5 printf "%s\n" "$ac_cv_func_prlimit" >&6; } @@ -19243,8 +20672,8 @@ printf %s "checking for _dyld_shared_cache_contains_path... " >&6; } if test ${ac_cv_func__dyld_shared_cache_contains_path+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -19258,11 +20687,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func__dyld_shared_cache_contains_path=yes -else $as_nop - ac_cv_func__dyld_shared_cache_contains_path=no +else case e in #( + e) ac_cv_func__dyld_shared_cache_contains_path=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func__dyld_shared_cache_contains_path" >&5 printf "%s\n" "$ac_cv_func__dyld_shared_cache_contains_path" >&6; } @@ -19283,8 +20714,8 @@ printf %s "checking for memfd_create... " >&6; } if test ${ac_cv_func_memfd_create+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #ifdef HAVE_SYS_MMAN_H @@ -19305,11 +20736,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_memfd_create=yes -else $as_nop - ac_cv_func_memfd_create=no +else case e in #( + e) ac_cv_func_memfd_create=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_memfd_create" >&5 printf "%s\n" "$ac_cv_func_memfd_create" >&6; } @@ -19330,8 +20763,8 @@ printf %s "checking for eventfd... " >&6; } if test ${ac_cv_func_eventfd+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #ifdef HAVE_SYS_EVENTFD_H @@ -19349,11 +20782,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_eventfd=yes -else $as_nop - ac_cv_func_eventfd=no +else case e in #( + e) ac_cv_func_eventfd=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_eventfd" >&5 printf "%s\n" "$ac_cv_func_eventfd" >&6; } @@ -19374,8 +20809,8 @@ printf %s "checking for timerfd_create... " >&6; } if test ${ac_cv_func_timerfd_create+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #ifdef HAVE_SYS_TIMERFD_H @@ -19393,11 +20828,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_timerfd_create=yes -else $as_nop - ac_cv_func_timerfd_create=no +else case e in #( + e) ac_cv_func_timerfd_create=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_timerfd_create" >&5 printf "%s\n" "$ac_cv_func_timerfd_create" >&6; } @@ -19424,8 +20861,8 @@ printf %s "checking for ctermid_r... " >&6; } if test ${ac_cv_func_ctermid_r+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -19439,11 +20876,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_ctermid_r=yes -else $as_nop - ac_cv_func_ctermid_r=no +else case e in #( + e) ac_cv_func_ctermid_r=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_ctermid_r" >&5 printf "%s\n" "$ac_cv_func_ctermid_r" >&6; } @@ -19462,8 +20901,8 @@ printf %s "checking for flock declaration... " >&6; } if test ${ac_cv_flock_decl+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -19478,12 +20917,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_flock_decl=yes -else $as_nop - ac_cv_flock_decl=no - +else case e in #( + e) ac_cv_flock_decl=no + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_flock_decl" >&5 printf "%s\n" "$ac_cv_flock_decl" >&6; } @@ -19497,22 +20938,28 @@ if test "x$ac_cv_func_flock" = xyes then : printf "%s\n" "#define HAVE_FLOCK 1" >>confdefs.h -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for flock in -lbsd" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for flock in -lbsd" >&5 printf %s "checking for flock in -lbsd... " >&6; } if test ${ac_cv_lib_bsd_flock+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lbsd $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char flock (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char flock (void); int main (void) { @@ -19524,12 +20971,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_bsd_flock=yes -else $as_nop - ac_cv_lib_bsd_flock=no +else case e in #( + e) ac_cv_lib_bsd_flock=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_bsd_flock" >&5 printf "%s\n" "$ac_cv_lib_bsd_flock" >&6; } @@ -19537,7 +20986,8 @@ if test "x$ac_cv_lib_bsd_flock" = xyes then : FCNTL_LIBS="-lbsd" fi - + ;; +esac fi done @@ -19550,8 +21000,8 @@ printf %s "checking for getpagesize... " >&6; } if test ${ac_cv_func_getpagesize+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -19565,11 +21015,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_getpagesize=yes -else $as_nop - ac_cv_func_getpagesize=no +else case e in #( + e) ac_cv_func_getpagesize=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_getpagesize" >&5 printf "%s\n" "$ac_cv_func_getpagesize" >&6; } @@ -19588,8 +21040,8 @@ printf %s "checking for broken unsetenv... " >&6; } if test ${ac_cv_broken_unsetenv+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -19603,12 +21055,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_broken_unsetenv=no -else $as_nop - ac_cv_broken_unsetenv=yes - +else case e in #( + e) ac_cv_broken_unsetenv=yes + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_broken_unsetenv" >&5 printf "%s\n" "$ac_cv_broken_unsetenv" >&6; } @@ -19630,8 +21084,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_TRUE+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$TRUE"; then +else case e in #( + e) if test -n "$TRUE"; then ac_cv_prog_TRUE="$TRUE" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -19653,7 +21107,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi TRUE=$ac_cv_prog_TRUE if test -n "$TRUE"; then @@ -19675,16 +21130,22 @@ printf %s "checking for inet_aton in -lc... " >&6; } if test ${ac_cv_lib_c_inet_aton+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lc $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char inet_aton (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char inet_aton (void); int main (void) { @@ -19696,34 +21157,42 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_c_inet_aton=yes -else $as_nop - ac_cv_lib_c_inet_aton=no +else case e in #( + e) ac_cv_lib_c_inet_aton=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_c_inet_aton" >&5 printf "%s\n" "$ac_cv_lib_c_inet_aton" >&6; } if test "x$ac_cv_lib_c_inet_aton" = xyes then : $ac_cv_prog_TRUE -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for inet_aton in -lresolv" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for inet_aton in -lresolv" >&5 printf %s "checking for inet_aton in -lresolv... " >&6; } if test ${ac_cv_lib_resolv_inet_aton+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lresolv $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char inet_aton (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char inet_aton (void); int main (void) { @@ -19735,24 +21204,123 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_resolv_inet_aton=yes -else $as_nop - ac_cv_lib_resolv_inet_aton=no +else case e in #( + e) ac_cv_lib_resolv_inet_aton=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_resolv_inet_aton" >&5 printf "%s\n" "$ac_cv_lib_resolv_inet_aton" >&6; } if test "x$ac_cv_lib_resolv_inet_aton" = xyes then : - printf "%s\n" "#define HAVE_LIBRESOLV 1" >>confdefs.h + SOCKET_LIBS="-lresolv" +fi + + ;; +esac +fi - LIBS="-lresolv $LIBS" +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for hstrerror in -lc" >&5 +printf %s "checking for hstrerror in -lc... " >&6; } +if test ${ac_cv_lib_c_hstrerror+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS +LIBS="-lc $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char hstrerror (void); +int +main (void) +{ +return hstrerror (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + ac_cv_lib_c_hstrerror=yes +else case e in #( + e) ac_cv_lib_c_hstrerror=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS ;; +esac fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_c_hstrerror" >&5 +printf "%s\n" "$ac_cv_lib_c_hstrerror" >&6; } +if test "x$ac_cv_lib_c_hstrerror" = xyes +then : + $ac_cv_prog_TRUE +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for hstrerror in -lresolv" >&5 +printf %s "checking for hstrerror in -lresolv... " >&6; } +if test ${ac_cv_lib_resolv_hstrerror+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS +LIBS="-lresolv $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char hstrerror (void); +int +main (void) +{ +return hstrerror (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + ac_cv_lib_resolv_hstrerror=yes +else case e in #( + e) ac_cv_lib_resolv_hstrerror=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_resolv_hstrerror" >&5 +printf "%s\n" "$ac_cv_lib_resolv_hstrerror" >&6; } +if test "x$ac_cv_lib_resolv_hstrerror" = xyes +then : + SOCKET_LIBS="-lresolv" +fi + ;; +esac fi @@ -19763,12 +21331,12 @@ printf %s "checking for chflags... " >&6; } if test ${ac_cv_have_chflags+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_have_chflags=cross -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -19784,14 +21352,17 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_have_chflags=yes -else $as_nop - ac_cv_have_chflags=no +else case e in #( + e) ac_cv_have_chflags=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_have_chflags" >&5 printf "%s\n" "$ac_cv_have_chflags" >&6; } @@ -19800,8 +21371,9 @@ if test "$ac_cv_have_chflags" = cross ; then if test "x$ac_cv_func_chflags" = xyes then : ac_cv_have_chflags="yes" -else $as_nop - ac_cv_have_chflags="no" +else case e in #( + e) ac_cv_have_chflags="no" ;; +esac fi fi @@ -19816,12 +21388,12 @@ printf %s "checking for lchflags... " >&6; } if test ${ac_cv_have_lchflags+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_have_lchflags=cross -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -19837,14 +21409,17 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_have_lchflags=yes -else $as_nop - ac_cv_have_lchflags=no +else case e in #( + e) ac_cv_have_lchflags=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_have_lchflags" >&5 printf "%s\n" "$ac_cv_have_lchflags" >&6; } @@ -19853,8 +21428,9 @@ if test "$ac_cv_have_lchflags" = cross ; then if test "x$ac_cv_func_lchflags" = xyes then : ac_cv_have_lchflags="yes" -else $as_nop - ac_cv_have_lchflags="no" +else case e in #( + e) ac_cv_have_lchflags="no" ;; +esac fi fi @@ -19932,12 +21508,12 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - ZLIB_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "zlib >= 1.2.0" 2>&1` + ZLIB_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "zlib >= 1.2.0" 2>&1` else - ZLIB_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "zlib >= 1.2.0" 2>&1` + ZLIB_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "zlib >= 1.2.0" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$ZLIB_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$ZLIB_PKG_ERRORS" >&5 save_CFLAGS=$CFLAGS @@ -19947,7 +21523,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $ZLIB_CFLAGS" - LDFLAGS="$LDFLAGS $ZLIB_LIBS" + LIBS="$LIBS $ZLIB_LIBS" for ac_header in zlib.h do : ac_fn_c_check_header_compile "$LINENO" "zlib.h" "ac_cv_header_zlib_h" "$ac_includes_default" @@ -19961,16 +21537,22 @@ printf %s "checking for gzread in -lz... " >&6; } if test ${ac_cv_lib_z_gzread+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lz $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char gzread (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char gzread (void); int main (void) { @@ -19982,27 +21564,31 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_z_gzread=yes -else $as_nop - ac_cv_lib_z_gzread=no +else case e in #( + e) ac_cv_lib_z_gzread=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_z_gzread" >&5 printf "%s\n" "$ac_cv_lib_z_gzread" >&6; } if test "x$ac_cv_lib_z_gzread" = xyes then : have_zlib=yes -else $as_nop - have_zlib=no +else case e in #( + e) have_zlib=no ;; +esac fi LIBS=$py_check_lib_save_LIBS -else $as_nop - have_zlib=no +else case e in #( + e) have_zlib=no ;; +esac fi done @@ -20017,16 +21603,22 @@ printf %s "checking for inflateCopy in -lz... " >&6; } if test ${ac_cv_lib_z_inflateCopy+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lz $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char inflateCopy (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char inflateCopy (void); int main (void) { @@ -20038,12 +21630,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_z_inflateCopy=yes -else $as_nop - ac_cv_lib_z_inflateCopy=no +else case e in #( + e) ac_cv_lib_z_inflateCopy=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_z_inflateCopy" >&5 printf "%s\n" "$ac_cv_lib_z_inflateCopy" >&6; } @@ -20076,7 +21670,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $ZLIB_CFLAGS" - LDFLAGS="$LDFLAGS $ZLIB_LIBS" + LIBS="$LIBS $ZLIB_LIBS" for ac_header in zlib.h do : ac_fn_c_check_header_compile "$LINENO" "zlib.h" "ac_cv_header_zlib_h" "$ac_includes_default" @@ -20090,16 +21684,22 @@ printf %s "checking for gzread in -lz... " >&6; } if test ${ac_cv_lib_z_gzread+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lz $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char gzread (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char gzread (void); int main (void) { @@ -20111,27 +21711,31 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_z_gzread=yes -else $as_nop - ac_cv_lib_z_gzread=no +else case e in #( + e) ac_cv_lib_z_gzread=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_z_gzread" >&5 printf "%s\n" "$ac_cv_lib_z_gzread" >&6; } if test "x$ac_cv_lib_z_gzread" = xyes then : have_zlib=yes -else $as_nop - have_zlib=no +else case e in #( + e) have_zlib=no ;; +esac fi LIBS=$py_check_lib_save_LIBS -else $as_nop - have_zlib=no +else case e in #( + e) have_zlib=no ;; +esac fi done @@ -20146,16 +21750,22 @@ printf %s "checking for inflateCopy in -lz... " >&6; } if test ${ac_cv_lib_z_inflateCopy+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lz $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char inflateCopy (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char inflateCopy (void); int main (void) { @@ -20167,12 +21777,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_z_inflateCopy=yes -else $as_nop - ac_cv_lib_z_inflateCopy=no +else case e in #( + e) ac_cv_lib_z_inflateCopy=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_z_inflateCopy" >&5 printf "%s\n" "$ac_cv_lib_z_inflateCopy" >&6; } @@ -20195,8 +21807,8 @@ LIBS=$save_LIBS else - ZLIB_CFLAGS=$pkg_cv_ZLIB_CFLAGS - ZLIB_LIBS=$pkg_cv_ZLIB_LIBS + ZLIB_CFLAGS=$pkg_cv_ZLIB_CFLAGS + ZLIB_LIBS=$pkg_cv_ZLIB_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } @@ -20280,12 +21892,12 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - BZIP2_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "bzip2" 2>&1` + BZIP2_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "bzip2" 2>&1` else - BZIP2_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "bzip2" 2>&1` + BZIP2_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "bzip2" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$BZIP2_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$BZIP2_PKG_ERRORS" >&5 save_CFLAGS=$CFLAGS @@ -20295,7 +21907,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $BZIP2_CFLAGS" - LDFLAGS="$LDFLAGS $BZIP2_LIBS" + LIBS="$LIBS $BZIP2_LIBS" for ac_header in bzlib.h do : ac_fn_c_check_header_compile "$LINENO" "bzlib.h" "ac_cv_header_bzlib_h" "$ac_includes_default" @@ -20308,16 +21920,22 @@ printf %s "checking for BZ2_bzCompress in -lbz2... " >&6; } if test ${ac_cv_lib_bz2_BZ2_bzCompress+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lbz2 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char BZ2_bzCompress (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char BZ2_bzCompress (void); int main (void) { @@ -20329,25 +21947,29 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_bz2_BZ2_bzCompress=yes -else $as_nop - ac_cv_lib_bz2_BZ2_bzCompress=no +else case e in #( + e) ac_cv_lib_bz2_BZ2_bzCompress=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_bz2_BZ2_bzCompress" >&5 printf "%s\n" "$ac_cv_lib_bz2_BZ2_bzCompress" >&6; } if test "x$ac_cv_lib_bz2_BZ2_bzCompress" = xyes then : have_bzip2=yes -else $as_nop - have_bzip2=no +else case e in #( + e) have_bzip2=no ;; +esac fi -else $as_nop - have_bzip2=no +else case e in #( + e) have_bzip2=no ;; +esac fi done @@ -20377,7 +21999,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $BZIP2_CFLAGS" - LDFLAGS="$LDFLAGS $BZIP2_LIBS" + LIBS="$LIBS $BZIP2_LIBS" for ac_header in bzlib.h do : ac_fn_c_check_header_compile "$LINENO" "bzlib.h" "ac_cv_header_bzlib_h" "$ac_includes_default" @@ -20390,16 +22012,22 @@ printf %s "checking for BZ2_bzCompress in -lbz2... " >&6; } if test ${ac_cv_lib_bz2_BZ2_bzCompress+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lbz2 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char BZ2_bzCompress (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char BZ2_bzCompress (void); int main (void) { @@ -20411,25 +22039,29 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_bz2_BZ2_bzCompress=yes -else $as_nop - ac_cv_lib_bz2_BZ2_bzCompress=no +else case e in #( + e) ac_cv_lib_bz2_BZ2_bzCompress=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_bz2_BZ2_bzCompress" >&5 printf "%s\n" "$ac_cv_lib_bz2_BZ2_bzCompress" >&6; } if test "x$ac_cv_lib_bz2_BZ2_bzCompress" = xyes then : have_bzip2=yes -else $as_nop - have_bzip2=no +else case e in #( + e) have_bzip2=no ;; +esac fi -else $as_nop - have_bzip2=no +else case e in #( + e) have_bzip2=no ;; +esac fi done @@ -20449,11 +22081,11 @@ LIBS=$save_LIBS else - BZIP2_CFLAGS=$pkg_cv_BZIP2_CFLAGS - BZIP2_LIBS=$pkg_cv_BZIP2_LIBS + BZIP2_CFLAGS=$pkg_cv_BZIP2_CFLAGS + BZIP2_LIBS=$pkg_cv_BZIP2_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } - have_bzip2=yes + have_bzip2=yes fi @@ -20508,12 +22140,12 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - LIBLZMA_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblzma" 2>&1` + LIBLZMA_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblzma" 2>&1` else - LIBLZMA_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblzma" 2>&1` + LIBLZMA_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblzma" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$LIBLZMA_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$LIBLZMA_PKG_ERRORS" >&5 save_CFLAGS=$CFLAGS @@ -20523,7 +22155,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBLZMA_CFLAGS" - LDFLAGS="$LDFLAGS $LIBLZMA_LIBS" + LIBS="$LIBS $LIBLZMA_LIBS" for ac_header in lzma.h do : ac_fn_c_check_header_compile "$LINENO" "lzma.h" "ac_cv_header_lzma_h" "$ac_includes_default" @@ -20536,16 +22168,22 @@ printf %s "checking for lzma_easy_encoder in -llzma... " >&6; } if test ${ac_cv_lib_lzma_lzma_easy_encoder+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-llzma $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char lzma_easy_encoder (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char lzma_easy_encoder (void); int main (void) { @@ -20557,25 +22195,29 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_lzma_lzma_easy_encoder=yes -else $as_nop - ac_cv_lib_lzma_lzma_easy_encoder=no +else case e in #( + e) ac_cv_lib_lzma_lzma_easy_encoder=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_lzma_lzma_easy_encoder" >&5 printf "%s\n" "$ac_cv_lib_lzma_lzma_easy_encoder" >&6; } if test "x$ac_cv_lib_lzma_lzma_easy_encoder" = xyes then : have_liblzma=yes -else $as_nop - have_liblzma=no +else case e in #( + e) have_liblzma=no ;; +esac fi -else $as_nop - have_liblzma=no +else case e in #( + e) have_liblzma=no ;; +esac fi done @@ -20605,7 +22247,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBLZMA_CFLAGS" - LDFLAGS="$LDFLAGS $LIBLZMA_LIBS" + LIBS="$LIBS $LIBLZMA_LIBS" for ac_header in lzma.h do : ac_fn_c_check_header_compile "$LINENO" "lzma.h" "ac_cv_header_lzma_h" "$ac_includes_default" @@ -20618,16 +22260,22 @@ printf %s "checking for lzma_easy_encoder in -llzma... " >&6; } if test ${ac_cv_lib_lzma_lzma_easy_encoder+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-llzma $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char lzma_easy_encoder (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char lzma_easy_encoder (void); int main (void) { @@ -20639,25 +22287,29 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_lzma_lzma_easy_encoder=yes -else $as_nop - ac_cv_lib_lzma_lzma_easy_encoder=no +else case e in #( + e) ac_cv_lib_lzma_lzma_easy_encoder=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_lzma_lzma_easy_encoder" >&5 printf "%s\n" "$ac_cv_lib_lzma_lzma_easy_encoder" >&6; } if test "x$ac_cv_lib_lzma_lzma_easy_encoder" = xyes then : have_liblzma=yes -else $as_nop - have_liblzma=no +else case e in #( + e) have_liblzma=no ;; +esac fi -else $as_nop - have_liblzma=no +else case e in #( + e) have_liblzma=no ;; +esac fi done @@ -20677,11 +22329,11 @@ LIBS=$save_LIBS else - LIBLZMA_CFLAGS=$pkg_cv_LIBLZMA_CFLAGS - LIBLZMA_LIBS=$pkg_cv_LIBLZMA_LIBS + LIBLZMA_CFLAGS=$pkg_cv_LIBLZMA_CFLAGS + LIBLZMA_LIBS=$pkg_cv_LIBLZMA_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } - have_liblzma=yes + have_liblzma=yes fi @@ -20693,8 +22345,8 @@ printf %s "checking for hstrerror... " >&6; } if test ${ac_cv_func_hstrerror+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -20708,11 +22360,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_hstrerror=yes -else $as_nop - ac_cv_func_hstrerror=no +else case e in #( + e) ac_cv_func_hstrerror=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_hstrerror" >&5 printf "%s\n" "$ac_cv_func_hstrerror" >&6; } @@ -20732,8 +22386,8 @@ printf %s "checking for getservbyname... " >&6; } if test ${ac_cv_func_getservbyname+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -20747,11 +22401,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_getservbyname=yes -else $as_nop - ac_cv_func_getservbyname=no +else case e in #( + e) ac_cv_func_getservbyname=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_getservbyname" >&5 printf "%s\n" "$ac_cv_func_getservbyname" >&6; } @@ -20771,8 +22427,8 @@ printf %s "checking for getservbyport... " >&6; } if test ${ac_cv_func_getservbyport+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -20786,11 +22442,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_getservbyport=yes -else $as_nop - ac_cv_func_getservbyport=no +else case e in #( + e) ac_cv_func_getservbyport=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_getservbyport" >&5 printf "%s\n" "$ac_cv_func_getservbyport" >&6; } @@ -20810,8 +22468,8 @@ printf %s "checking for gethostbyname... " >&6; } if test ${ac_cv_func_gethostbyname+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -20825,11 +22483,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_gethostbyname=yes -else $as_nop - ac_cv_func_gethostbyname=no +else case e in #( + e) ac_cv_func_gethostbyname=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_gethostbyname" >&5 printf "%s\n" "$ac_cv_func_gethostbyname" >&6; } @@ -20849,8 +22509,8 @@ printf %s "checking for gethostbyaddr... " >&6; } if test ${ac_cv_func_gethostbyaddr+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -20864,11 +22524,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_gethostbyaddr=yes -else $as_nop - ac_cv_func_gethostbyaddr=no +else case e in #( + e) ac_cv_func_gethostbyaddr=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_gethostbyaddr" >&5 printf "%s\n" "$ac_cv_func_gethostbyaddr" >&6; } @@ -20888,8 +22550,8 @@ printf %s "checking for getprotobyname... " >&6; } if test ${ac_cv_func_getprotobyname+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -20903,11 +22565,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_getprotobyname=yes -else $as_nop - ac_cv_func_getprotobyname=no +else case e in #( + e) ac_cv_func_getprotobyname=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_getprotobyname" >&5 printf "%s\n" "$ac_cv_func_getprotobyname" >&6; } @@ -20930,8 +22594,8 @@ printf %s "checking for inet_aton... " >&6; } if test ${ac_cv_func_inet_aton+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -20950,11 +22614,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_inet_aton=yes -else $as_nop - ac_cv_func_inet_aton=no +else case e in #( + e) ac_cv_func_inet_aton=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_inet_aton" >&5 printf "%s\n" "$ac_cv_func_inet_aton" >&6; } @@ -20974,8 +22640,8 @@ printf %s "checking for inet_ntoa... " >&6; } if test ${ac_cv_func_inet_ntoa+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -20994,11 +22660,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_inet_ntoa=yes -else $as_nop - ac_cv_func_inet_ntoa=no +else case e in #( + e) ac_cv_func_inet_ntoa=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_inet_ntoa" >&5 printf "%s\n" "$ac_cv_func_inet_ntoa" >&6; } @@ -21018,8 +22686,8 @@ printf %s "checking for inet_pton... " >&6; } if test ${ac_cv_func_inet_pton+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -21038,11 +22706,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_inet_pton=yes -else $as_nop - ac_cv_func_inet_pton=no +else case e in #( + e) ac_cv_func_inet_pton=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_inet_pton" >&5 printf "%s\n" "$ac_cv_func_inet_pton" >&6; } @@ -21062,8 +22732,8 @@ printf %s "checking for getpeername... " >&6; } if test ${ac_cv_func_getpeername+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -21082,11 +22752,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_getpeername=yes -else $as_nop - ac_cv_func_getpeername=no +else case e in #( + e) ac_cv_func_getpeername=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_getpeername" >&5 printf "%s\n" "$ac_cv_func_getpeername" >&6; } @@ -21106,8 +22778,8 @@ printf %s "checking for getsockname... " >&6; } if test ${ac_cv_func_getsockname+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -21126,11 +22798,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_getsockname=yes -else $as_nop - ac_cv_func_getsockname=no +else case e in #( + e) ac_cv_func_getsockname=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_getsockname" >&5 printf "%s\n" "$ac_cv_func_getsockname" >&6; } @@ -21150,8 +22824,8 @@ printf %s "checking for accept... " >&6; } if test ${ac_cv_func_accept+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -21170,11 +22844,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_accept=yes -else $as_nop - ac_cv_func_accept=no +else case e in #( + e) ac_cv_func_accept=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_accept" >&5 printf "%s\n" "$ac_cv_func_accept" >&6; } @@ -21194,8 +22870,8 @@ printf %s "checking for bind... " >&6; } if test ${ac_cv_func_bind+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -21214,11 +22890,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_bind=yes -else $as_nop - ac_cv_func_bind=no +else case e in #( + e) ac_cv_func_bind=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_bind" >&5 printf "%s\n" "$ac_cv_func_bind" >&6; } @@ -21238,8 +22916,8 @@ printf %s "checking for connect... " >&6; } if test ${ac_cv_func_connect+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -21258,11 +22936,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_connect=yes -else $as_nop - ac_cv_func_connect=no +else case e in #( + e) ac_cv_func_connect=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_connect" >&5 printf "%s\n" "$ac_cv_func_connect" >&6; } @@ -21282,8 +22962,8 @@ printf %s "checking for listen... " >&6; } if test ${ac_cv_func_listen+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -21302,11 +22982,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_listen=yes -else $as_nop - ac_cv_func_listen=no +else case e in #( + e) ac_cv_func_listen=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_listen" >&5 printf "%s\n" "$ac_cv_func_listen" >&6; } @@ -21326,8 +23008,8 @@ printf %s "checking for recvfrom... " >&6; } if test ${ac_cv_func_recvfrom+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -21346,11 +23028,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_recvfrom=yes -else $as_nop - ac_cv_func_recvfrom=no +else case e in #( + e) ac_cv_func_recvfrom=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_recvfrom" >&5 printf "%s\n" "$ac_cv_func_recvfrom" >&6; } @@ -21370,8 +23054,8 @@ printf %s "checking for sendto... " >&6; } if test ${ac_cv_func_sendto+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -21390,11 +23074,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_sendto=yes -else $as_nop - ac_cv_func_sendto=no +else case e in #( + e) ac_cv_func_sendto=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_sendto" >&5 printf "%s\n" "$ac_cv_func_sendto" >&6; } @@ -21414,8 +23100,8 @@ printf %s "checking for setsockopt... " >&6; } if test ${ac_cv_func_setsockopt+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -21434,11 +23120,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_setsockopt=yes -else $as_nop - ac_cv_func_setsockopt=no +else case e in #( + e) ac_cv_func_setsockopt=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_setsockopt" >&5 printf "%s\n" "$ac_cv_func_setsockopt" >&6; } @@ -21458,8 +23146,8 @@ printf %s "checking for socket... " >&6; } if test ${ac_cv_func_socket+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -21478,11 +23166,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_socket=yes -else $as_nop - ac_cv_func_socket=no +else case e in #( + e) ac_cv_func_socket=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_socket" >&5 printf "%s\n" "$ac_cv_func_socket" >&6; } @@ -21504,8 +23194,8 @@ printf %s "checking for setgroups... " >&6; } if test ${ac_cv_func_setgroups+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -21524,11 +23214,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_setgroups=yes -else $as_nop - ac_cv_func_setgroups=no +else case e in #( + e) ac_cv_func_setgroups=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_setgroups" >&5 printf "%s\n" "$ac_cv_func_setgroups" >&6; } @@ -21552,22 +23244,28 @@ if test "x$ac_cv_func_openpty" = xyes then : printf "%s\n" "#define HAVE_OPENPTY 1" >>confdefs.h -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for openpty in -lutil" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for openpty in -lutil" >&5 printf %s "checking for openpty in -lutil... " >&6; } if test ${ac_cv_lib_util_openpty+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lutil $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char openpty (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char openpty (void); int main (void) { @@ -21579,12 +23277,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_util_openpty=yes -else $as_nop - ac_cv_lib_util_openpty=no +else case e in #( + e) ac_cv_lib_util_openpty=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_util_openpty" >&5 printf "%s\n" "$ac_cv_lib_util_openpty" >&6; } @@ -21592,22 +23292,28 @@ if test "x$ac_cv_lib_util_openpty" = xyes then : printf "%s\n" "#define HAVE_OPENPTY 1" >>confdefs.h LIBS="$LIBS -lutil" -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for openpty in -lbsd" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for openpty in -lbsd" >&5 printf %s "checking for openpty in -lbsd... " >&6; } if test ${ac_cv_lib_bsd_openpty+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lbsd $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char openpty (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char openpty (void); int main (void) { @@ -21619,12 +23325,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_bsd_openpty=yes -else $as_nop - ac_cv_lib_bsd_openpty=no +else case e in #( + e) ac_cv_lib_bsd_openpty=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_bsd_openpty" >&5 printf "%s\n" "$ac_cv_lib_bsd_openpty" >&6; } @@ -21633,9 +23341,11 @@ then : printf "%s\n" "#define HAVE_OPENPTY 1" >>confdefs.h LIBS="$LIBS -lbsd" fi - + ;; +esac fi - + ;; +esac fi done @@ -21644,15 +23354,21 @@ printf %s "checking for library containing login_tty... " >&6; } if test ${ac_cv_search_login_tty+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_func_search_save_LIBS=$LIBS +else case e in #( + e) ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char login_tty (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char login_tty (void); int main (void) { @@ -21683,11 +23399,13 @@ done if test ${ac_cv_search_login_tty+y} then : -else $as_nop - ac_cv_search_login_tty=no +else case e in #( + e) ac_cv_search_login_tty=no ;; +esac fi rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS +LIBS=$ac_func_search_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_login_tty" >&5 printf "%s\n" "$ac_cv_search_login_tty" >&6; } @@ -21709,22 +23427,28 @@ if test "x$ac_cv_func_forkpty" = xyes then : printf "%s\n" "#define HAVE_FORKPTY 1" >>confdefs.h -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for forkpty in -lutil" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for forkpty in -lutil" >&5 printf %s "checking for forkpty in -lutil... " >&6; } if test ${ac_cv_lib_util_forkpty+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lutil $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char forkpty (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char forkpty (void); int main (void) { @@ -21736,12 +23460,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_util_forkpty=yes -else $as_nop - ac_cv_lib_util_forkpty=no +else case e in #( + e) ac_cv_lib_util_forkpty=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_util_forkpty" >&5 printf "%s\n" "$ac_cv_lib_util_forkpty" >&6; } @@ -21749,22 +23475,28 @@ if test "x$ac_cv_lib_util_forkpty" = xyes then : printf "%s\n" "#define HAVE_FORKPTY 1" >>confdefs.h LIBS="$LIBS -lutil" -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for forkpty in -lbsd" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for forkpty in -lbsd" >&5 printf %s "checking for forkpty in -lbsd... " >&6; } if test ${ac_cv_lib_bsd_forkpty+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lbsd $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char forkpty (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char forkpty (void); int main (void) { @@ -21776,12 +23508,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_bsd_forkpty=yes -else $as_nop - ac_cv_lib_bsd_forkpty=no +else case e in #( + e) ac_cv_lib_bsd_forkpty=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_bsd_forkpty" >&5 printf "%s\n" "$ac_cv_lib_bsd_forkpty" >&6; } @@ -21790,9 +23524,11 @@ then : printf "%s\n" "#define HAVE_FORKPTY 1" >>confdefs.h LIBS="$LIBS -lbsd" fi - + ;; +esac fi - + ;; +esac fi done @@ -21841,13 +23577,14 @@ if test "x$ac_cv_func_dup2" = xyes then : printf "%s\n" "#define HAVE_DUP2 1" >>confdefs.h -else $as_nop - case " $LIBOBJS " in +else case e in #( + e) case " $LIBOBJS " in *" dup2.$ac_objext "* ) ;; *) LIBOBJS="$LIBOBJS dup2.$ac_objext" ;; esac - + ;; +esac fi @@ -21930,23 +23667,29 @@ if test "x$ac_cv_func_clock_gettime" = xyes then : printf "%s\n" "#define HAVE_CLOCK_GETTIME 1" >>confdefs.h -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_gettime in -lrt" >&5 printf %s "checking for clock_gettime in -lrt... " >&6; } if test ${ac_cv_lib_rt_clock_gettime+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lrt $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char clock_gettime (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char clock_gettime (void); int main (void) { @@ -21958,12 +23701,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_rt_clock_gettime=yes -else $as_nop - ac_cv_lib_rt_clock_gettime=no +else case e in #( + e) ac_cv_lib_rt_clock_gettime=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_clock_gettime" >&5 printf "%s\n" "$ac_cv_lib_rt_clock_gettime" >&6; } @@ -21979,7 +23724,8 @@ printf "%s\n" "#define TIMEMODULE_LIB rt" >>confdefs.h fi - + ;; +esac fi done @@ -21992,23 +23738,29 @@ if test "x$ac_cv_func_clock_getres" = xyes then : printf "%s\n" "#define HAVE_CLOCK_GETRES 1" >>confdefs.h -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_getres in -lrt" >&5 printf %s "checking for clock_getres in -lrt... " >&6; } if test ${ac_cv_lib_rt_clock_getres+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lrt $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char clock_getres (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char clock_getres (void); int main (void) { @@ -22020,12 +23772,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_rt_clock_getres=yes -else $as_nop - ac_cv_lib_rt_clock_getres=no +else case e in #( + e) ac_cv_lib_rt_clock_getres=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_clock_getres" >&5 printf "%s\n" "$ac_cv_lib_rt_clock_getres" >&6; } @@ -22037,7 +23791,8 @@ then : fi - + ;; +esac fi done @@ -22055,23 +23810,29 @@ if test "x$ac_cv_func_clock_settime" = xyes then : printf "%s\n" "#define HAVE_CLOCK_SETTIME 1" >>confdefs.h -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_settime in -lrt" >&5 printf %s "checking for clock_settime in -lrt... " >&6; } if test ${ac_cv_lib_rt_clock_settime+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lrt $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char clock_settime (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char clock_settime (void); int main (void) { @@ -22083,12 +23844,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_rt_clock_settime=yes -else $as_nop - ac_cv_lib_rt_clock_settime=no +else case e in #( + e) ac_cv_lib_rt_clock_settime=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_clock_settime" >&5 printf "%s\n" "$ac_cv_lib_rt_clock_settime" >&6; } @@ -22100,12 +23863,17 @@ then : fi - + ;; +esac fi done fi +# On Android before API level 23, clock_nanosleep returns the wrong value when +# interrupted by a signal (https://issuetracker.google.com/issues/216495770). +if ! { test "$ac_sys_system" = "Linux-android" && + test "$ANDROID_API_LEVEL" -lt 23; }; then for ac_func in clock_nanosleep do : @@ -22114,23 +23882,29 @@ if test "x$ac_cv_func_clock_nanosleep" = xyes then : printf "%s\n" "#define HAVE_CLOCK_NANOSLEEP 1" >>confdefs.h -else $as_nop - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_nanosleep in -lrt" >&5 +else case e in #( + e) + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_nanosleep in -lrt" >&5 printf %s "checking for clock_nanosleep in -lrt... " >&6; } if test ${ac_cv_lib_rt_clock_nanosleep+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lrt $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char clock_nanosleep (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char clock_nanosleep (void); int main (void) { @@ -22142,27 +23916,31 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_rt_clock_nanosleep=yes -else $as_nop - ac_cv_lib_rt_clock_nanosleep=no +else case e in #( + e) ac_cv_lib_rt_clock_nanosleep=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_clock_nanosleep" >&5 printf "%s\n" "$ac_cv_lib_rt_clock_nanosleep" >&6; } if test "x$ac_cv_lib_rt_clock_nanosleep" = xyes then : - printf "%s\n" "#define HAVE_CLOCK_NANOSLEEP 1" >>confdefs.h + printf "%s\n" "#define HAVE_CLOCK_NANOSLEEP 1" >>confdefs.h fi - + ;; +esac fi done +fi for ac_func in nanosleep @@ -22172,23 +23950,29 @@ if test "x$ac_cv_func_nanosleep" = xyes then : printf "%s\n" "#define HAVE_NANOSLEEP 1" >>confdefs.h -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for nanosleep in -lrt" >&5 printf %s "checking for nanosleep in -lrt... " >&6; } if test ${ac_cv_lib_rt_nanosleep+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lrt $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char nanosleep (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char nanosleep (void); int main (void) { @@ -22200,12 +23984,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_rt_nanosleep=yes -else $as_nop - ac_cv_lib_rt_nanosleep=no +else case e in #( + e) ac_cv_lib_rt_nanosleep=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_nanosleep" >&5 printf "%s\n" "$ac_cv_lib_rt_nanosleep" >&6; } @@ -22217,7 +24003,8 @@ then : fi - + ;; +esac fi done @@ -22227,8 +24014,8 @@ printf %s "checking for major, minor, and makedev... " >&6; } if test ${ac_cv_device_macros+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -22254,12 +24041,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_device_macros=yes -else $as_nop - ac_cv_device_macros=no +else case e in #( + e) ac_cv_device_macros=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_device_macros" >&5 printf "%s\n" "$ac_cv_device_macros" >&6; } @@ -22283,8 +24072,8 @@ printf %s "checking for getaddrinfo... " >&6; } if test ${ac_cv_func_getaddrinfo+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -22304,12 +24093,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_func_getaddrinfo=yes -else $as_nop - ac_cv_func_getaddrinfo=no +else case e in #( + e) ac_cv_func_getaddrinfo=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_getaddrinfo" >&5 printf "%s\n" "$ac_cv_func_getaddrinfo" >&6; } @@ -22322,8 +24113,8 @@ printf %s "checking getaddrinfo bug... " >&6; } if test ${ac_cv_buggy_getaddrinfo+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : if test "$ac_sys_system" = "Linux-android" || test "$ac_sys_system" = "iOS"; then @@ -22333,8 +24124,8 @@ elif test "${enable_ipv6+set}" = set; then else ac_cv_buggy_getaddrinfo=yes fi -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -22430,13 +24221,16 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_buggy_getaddrinfo=no -else $as_nop - ac_cv_buggy_getaddrinfo=yes +else case e in #( + e) ac_cv_buggy_getaddrinfo=yes ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_buggy_getaddrinfo" >&5 printf "%s\n" "$ac_cv_buggy_getaddrinfo" >&6; } @@ -22472,8 +24266,8 @@ printf %s "checking whether struct tm is in sys/time.h or time.h... " >&6; } if test ${ac_cv_struct_tm+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #include @@ -22491,10 +24285,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_struct_tm=time.h -else $as_nop - ac_cv_struct_tm=sys/time.h +else case e in #( + e) ac_cv_struct_tm=sys/time.h ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_struct_tm" >&5 printf "%s\n" "$ac_cv_struct_tm" >&6; } @@ -22526,8 +24322,9 @@ else if test "x$ac_cv_have_decl_tzname" = xyes then : ac_have_decl=1 -else $as_nop - ac_have_decl=0 +else case e in #( + e) ac_have_decl=0 ;; +esac fi printf "%s\n" "#define HAVE_DECL_TZNAME $ac_have_decl" >>confdefs.h @@ -22536,8 +24333,8 @@ printf %s "checking for tzname... " >&6; } if test ${ac_cv_var_tzname+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #if !HAVE_DECL_TZNAME @@ -22555,11 +24352,13 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_var_tzname=yes -else $as_nop - ac_cv_var_tzname=no +else case e in #( + e) ac_cv_var_tzname=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext + conftest$ac_exeext conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_var_tzname" >&5 printf "%s\n" "$ac_cv_var_tzname" >&6; } @@ -22666,8 +24465,8 @@ printf %s "checking for time.h that defines altzone... " >&6; } if test ${ac_cv_header_time_altzone+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -22682,11 +24481,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_header_time_altzone=yes -else $as_nop - ac_cv_header_time_altzone=no +else case e in #( + e) ac_cv_header_time_altzone=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_time_altzone" >&5 printf "%s\n" "$ac_cv_header_time_altzone" >&6; } @@ -22701,8 +24502,8 @@ printf %s "checking for addrinfo... " >&6; } if test ${ac_cv_struct_addrinfo+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -22716,10 +24517,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_struct_addrinfo=yes -else $as_nop - ac_cv_struct_addrinfo=no +else case e in #( + e) ac_cv_struct_addrinfo=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_struct_addrinfo" >&5 printf "%s\n" "$ac_cv_struct_addrinfo" >&6; } @@ -22734,8 +24537,8 @@ printf %s "checking for sockaddr_storage... " >&6; } if test ${ac_cv_struct_sockaddr_storage+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ # include @@ -22751,10 +24554,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_struct_sockaddr_storage=yes -else $as_nop - ac_cv_struct_sockaddr_storage=no +else case e in #( + e) ac_cv_struct_sockaddr_storage=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_struct_sockaddr_storage" >&5 printf "%s\n" "$ac_cv_struct_sockaddr_storage" >&6; } @@ -22769,8 +24574,8 @@ printf %s "checking for sockaddr_alg... " >&6; } if test ${ac_cv_struct_sockaddr_alg+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ # include @@ -22787,10 +24592,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_struct_sockaddr_alg=yes -else $as_nop - ac_cv_struct_sockaddr_alg=no +else case e in #( + e) ac_cv_struct_sockaddr_alg=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_struct_sockaddr_alg" >&5 printf "%s\n" "$ac_cv_struct_sockaddr_alg" >&6; } @@ -22807,8 +24614,8 @@ printf %s "checking for an ANSI C-conforming const... " >&6; } if test ${ac_cv_c_const+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int @@ -22872,10 +24679,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_c_const=yes -else $as_nop - ac_cv_c_const=no +else case e in #( + e) ac_cv_c_const=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_const" >&5 printf "%s\n" "$ac_cv_c_const" >&6; } @@ -22891,8 +24700,8 @@ printf %s "checking for working signed char... " >&6; } if test ${ac_cv_working_signed_char_c+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -22907,11 +24716,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_working_signed_char_c=yes -else $as_nop - ac_cv_working_signed_char_c=no +else case e in #( + e) ac_cv_working_signed_char_c=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_working_signed_char_c" >&5 printf "%s\n" "$ac_cv_working_signed_char_c" >&6; } @@ -22929,8 +24740,8 @@ printf %s "checking for prototypes... " >&6; } if test ${ac_cv_function_prototypes+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int foo(int x) { return 0; } @@ -22945,11 +24756,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_function_prototypes=yes -else $as_nop - ac_cv_function_prototypes=no +else case e in #( + e) ac_cv_function_prototypes=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_function_prototypes" >&5 printf "%s\n" "$ac_cv_function_prototypes" >&6; } @@ -22971,8 +24784,8 @@ printf %s "checking for socketpair... " >&6; } if test ${ac_cv_func_socketpair+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -22989,11 +24802,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_func_socketpair=yes -else $as_nop - ac_cv_func_socketpair=no +else case e in #( + e) ac_cv_func_socketpair=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_socketpair" >&5 printf "%s\n" "$ac_cv_func_socketpair" >&6; } @@ -23013,8 +24828,8 @@ printf %s "checking if sockaddr has sa_len member... " >&6; } if test ${ac_cv_struct_sockaddr_sa_len+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -23031,11 +24846,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_struct_sockaddr_sa_len=yes -else $as_nop - ac_cv_struct_sockaddr_sa_len=no +else case e in #( + e) ac_cv_struct_sockaddr_sa_len=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_struct_sockaddr_sa_len" >&5 printf "%s\n" "$ac_cv_struct_sockaddr_sa_len" >&6; } @@ -23092,8 +24909,8 @@ printf "%s\n" "#define HAVE_GETHOSTBYNAME_R_6_ARG 1" >>confdefs.h { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking gethostbyname_r with 5 args" >&5 @@ -23130,8 +24947,8 @@ printf "%s\n" "#define HAVE_GETHOSTBYNAME_R_5_ARG 1" >>confdefs.h { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking gethostbyname_r with 3 args" >&5 @@ -23166,23 +24983,26 @@ printf "%s\n" "#define HAVE_GETHOSTBYNAME_R_3_ARG 1" >>confdefs.h { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS=$OLD_CFLAGS -else $as_nop - +else case e in #( + e) ac_fn_c_check_func "$LINENO" "gethostbyname" "ac_cv_func_gethostbyname" if test "x$ac_cv_func_gethostbyname" = xyes then : @@ -23190,7 +25010,8 @@ then : fi - + ;; +esac fi @@ -23207,22 +25028,28 @@ ac_fn_c_check_func "$LINENO" "__fpu_control" "ac_cv_func___fpu_control" if test "x$ac_cv_func___fpu_control" = xyes then : -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __fpu_control in -lieee" >&5 +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __fpu_control in -lieee" >&5 printf %s "checking for __fpu_control in -lieee... " >&6; } if test ${ac_cv_lib_ieee___fpu_control+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lieee $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char __fpu_control (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char __fpu_control (void); int main (void) { @@ -23234,12 +25061,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_ieee___fpu_control=yes -else $as_nop - ac_cv_lib_ieee___fpu_control=no +else case e in #( + e) ac_cv_lib_ieee___fpu_control=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ieee___fpu_control" >&5 printf "%s\n" "$ac_cv_lib_ieee___fpu_control" >&6; } @@ -23251,7 +25080,8 @@ then : fi - + ;; +esac fi @@ -23278,9 +25108,10 @@ then LIBM=$withval printf "%s\n" "set LIBM=\"$withval\"" >&6; } else as_fn_error $? "proper usage is --with-libm=STRING" "$LINENO" 5 fi -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: default LIBM=\"$LIBM\"" >&5 -printf "%s\n" "default LIBM=\"$LIBM\"" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: default LIBM=\"$LIBM\"" >&5 +printf "%s\n" "default LIBM=\"$LIBM\"" >&6; } ;; +esac fi @@ -23303,9 +25134,10 @@ then LIBC=$withval printf "%s\n" "set LIBC=\"$withval\"" >&6; } else as_fn_error $? "proper usage is --with-libc=STRING" "$LINENO" 5 fi -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: default LIBC=\"$LIBC\"" >&5 -printf "%s\n" "default LIBC=\"$LIBC\"" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: default LIBC=\"$LIBC\"" >&5 +printf "%s\n" "default LIBC=\"$LIBC\"" >&6; } ;; +esac fi @@ -23319,8 +25151,8 @@ printf %s "checking for x64 gcc inline assembler... " >&6; } if test ${ac_cv_gcc_asm_for_x64+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -23337,12 +25169,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_gcc_asm_for_x64=yes -else $as_nop - ac_cv_gcc_asm_for_x64=no +else case e in #( + e) ac_cv_gcc_asm_for_x64=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_gcc_asm_for_x64" >&5 printf "%s\n" "$ac_cv_gcc_asm_for_x64" >&6; } @@ -23365,26 +25199,34 @@ printf %s "checking whether float word ordering is bigendian... " >&6; } if test ${ax_cv_c_float_words_bigendian+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) ax_cv_c_float_words_bigendian=unknown cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -double d = 90904234967036810337470478905505011476211692735615632014797120844053488865816695273723469097858056257517020191247487429516932130503560650002327564517570778480236724525140520121371739201496540132640109977779420565776568942592.0; +#include + +static double m[] = {9.090423496703681e+223, 0.0}; + +int main (int argc, char *argv[]) +{ + m[atoi (argv[1])] += atof (argv[2]); + return m[atoi (argv[3])] > 0.0; +} _ACEOF -if ac_fn_c_try_compile "$LINENO" +if ac_fn_c_try_link "$LINENO" then : -if grep noonsees conftest.$ac_objext >/dev/null ; then +if grep noonsees conftest* > /dev/null ; then ax_cv_c_float_words_bigendian=yes fi -if grep seesnoon conftest.$ac_objext >/dev/null ; then +if grep seesnoon conftest* >/dev/null ; then if test "$ax_cv_c_float_words_bigendian" = unknown; then ax_cv_c_float_words_bigendian=no else @@ -23394,7 +25236,9 @@ fi fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_c_float_words_bigendian" >&5 printf "%s\n" "$ax_cv_c_float_words_bigendian" >&6; } @@ -23402,41 +25246,30 @@ printf "%s\n" "$ax_cv_c_float_words_bigendian" >&6; } case $ax_cv_c_float_words_bigendian in yes) -printf "%s\n" "#define FLOAT_WORDS_BIGENDIAN 1" >>confdefs.h +printf "%s\n" "#define DOUBLE_IS_BIG_ENDIAN_IEEE754 1" >>confdefs.h ;; no) - ;; - *) - as_fn_error $? " - -Unknown float word ordering. You need to manually preset -ax_cv_c_float_words_bigendian=no (or yes) according to your system. - - " "$LINENO" 5 ;; -esac - - -if test "$ax_cv_c_float_words_bigendian" = "yes" -then - -printf "%s\n" "#define DOUBLE_IS_BIG_ENDIAN_IEEE754 1" >>confdefs.h - -elif test "$ax_cv_c_float_words_bigendian" = "no" -then printf "%s\n" "#define DOUBLE_IS_LITTLE_ENDIAN_IEEE754 1" >>confdefs.h - -else - # Some ARM platforms use a mixed-endian representation for doubles. - # While Python doesn't currently have full support for these platforms - # (see e.g., issue 1762561), we can at least make sure that float <-> string - # conversions work. - # FLOAT_WORDS_BIGENDIAN doesnt actually detect this case, but if it's not big - # or little, then it must be this? + ;; + *) + case $host_cpu in #( + *arm*) : + # Some ARM platforms use a mixed-endian representation for + # doubles. While Python doesn't currently have full support + # for these platforms (see e.g., issue 1762561), we can at + # least make sure that float <-> string conversions work. + # FLOAT_WORDS_BIGENDIAN doesn't actually detect this case, + # but if it's not big or little, then it must be this? printf "%s\n" "#define DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754 1" >>confdefs.h + ;; #( + *) : + as_fn_error $? "Unknown float word ordering. You need to manually preset ax_cv_c_float_words_bigendian=no (or yes) according to your system." "$LINENO" 5 ;; +esac ;; +esac + -fi # The short float repr introduced in Python 3.1 requires the # correctly-rounded string <-> double conversion functions from @@ -23453,8 +25286,8 @@ printf %s "checking whether we can use gcc inline assembler to get and set x87 c if test ${ac_cv_gcc_asm_for_x87+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -23473,12 +25306,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_gcc_asm_for_x87=yes -else $as_nop - ac_cv_gcc_asm_for_x87=no +else case e in #( + e) ac_cv_gcc_asm_for_x87=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_gcc_asm_for_x87" >&5 printf "%s\n" "$ac_cv_gcc_asm_for_x87" >&6; } @@ -23496,8 +25331,8 @@ printf %s "checking whether we can use gcc inline assembler to get and set mc688 if test ${ac_cv_gcc_asm_for_mc68881+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -23516,12 +25351,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_gcc_asm_for_mc68881=yes -else $as_nop - ac_cv_gcc_asm_for_mc68881=no +else case e in #( + e) ac_cv_gcc_asm_for_mc68881=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_gcc_asm_for_mc68881" >&5 printf "%s\n" "$ac_cv_gcc_asm_for_mc68881" >&6; } @@ -23544,16 +25381,16 @@ printf %s "checking for x87-style double rounding... " >&6; } if test ${ac_cv_x87_double_rounding+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) # $BASECFLAGS may affect the result ac_save_cc="$CC" CC="$CC $BASECFLAGS" if test "$cross_compiling" = yes then : ac_cv_x87_double_rounding=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -23579,15 +25416,18 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_x87_double_rounding=no -else $as_nop - ac_cv_x87_double_rounding=yes +else case e in #( + e) ac_cv_x87_double_rounding=yes ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi CC="$ac_save_cc" - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_x87_double_rounding" >&5 printf "%s\n" "$ac_cv_x87_double_rounding" >&6; } @@ -23611,17 +25451,18 @@ LIBS="$LIBS $LIBM" for ac_func in acosh asinh atanh erf erfc expm1 log1p log2 do : - as_ac_var=`printf "%s\n" "ac_cv_func_$ac_func" | $as_tr_sh` + as_ac_var=`printf "%s\n" "ac_cv_func_$ac_func" | sed "$as_sed_sh"` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" if eval test \"x\$"$as_ac_var"\" = x"yes" then : cat >>confdefs.h <<_ACEOF -#define `printf "%s\n" "HAVE_$ac_func" | $as_tr_cpp` 1 +#define `printf "%s\n" "HAVE_$ac_func" | sed "$as_sed_cpp"` 1 _ACEOF -else $as_nop - as_fn_error $? "Python requires C99 compatible libm" "$LINENO" 5 - +else case e in #( + e) as_fn_error $? "Python requires C99 compatible libm" "$LINENO" 5 + ;; +esac fi done @@ -23632,12 +25473,12 @@ printf %s "checking whether POSIX semaphores are enabled... " >&6; } if test ${ac_cv_posix_semaphores_enabled+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_posix_semaphores_enabled=yes -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -23663,14 +25504,17 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_posix_semaphores_enabled=yes -else $as_nop - ac_cv_posix_semaphores_enabled=no +else case e in #( + e) ac_cv_posix_semaphores_enabled=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_posix_semaphores_enabled" >&5 printf "%s\n" "$ac_cv_posix_semaphores_enabled" >&6; } @@ -23688,12 +25532,12 @@ printf %s "checking for broken sem_getvalue... " >&6; } if test ${ac_cv_broken_sem_getvalue+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_broken_sem_getvalue=yes -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -23723,14 +25567,17 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_broken_sem_getvalue=no -else $as_nop - ac_cv_broken_sem_getvalue=yes +else case e in #( + e) ac_cv_broken_sem_getvalue=yes ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_broken_sem_getvalue" >&5 printf "%s\n" "$ac_cv_broken_sem_getvalue" >&6; } @@ -23748,8 +25595,9 @@ ac_fn_check_decl "$LINENO" "RTLD_LAZY" "ac_cv_have_decl_RTLD_LAZY" "#include
    >confdefs.h ac_fn_check_decl "$LINENO" "RTLD_NOW" "ac_cv_have_decl_RTLD_NOW" "#include @@ -23757,8 +25605,9 @@ ac_fn_check_decl "$LINENO" "RTLD_NOW" "ac_cv_have_decl_RTLD_NOW" "#include >confdefs.h ac_fn_check_decl "$LINENO" "RTLD_GLOBAL" "ac_cv_have_decl_RTLD_GLOBAL" "#include @@ -23766,8 +25615,9 @@ ac_fn_check_decl "$LINENO" "RTLD_GLOBAL" "ac_cv_have_decl_RTLD_GLOBAL" "#include if test "x$ac_cv_have_decl_RTLD_GLOBAL" = xyes then : ac_have_decl=1 -else $as_nop - ac_have_decl=0 +else case e in #( + e) ac_have_decl=0 ;; +esac fi printf "%s\n" "#define HAVE_DECL_RTLD_GLOBAL $ac_have_decl" >>confdefs.h ac_fn_check_decl "$LINENO" "RTLD_LOCAL" "ac_cv_have_decl_RTLD_LOCAL" "#include @@ -23775,8 +25625,9 @@ ac_fn_check_decl "$LINENO" "RTLD_LOCAL" "ac_cv_have_decl_RTLD_LOCAL" "#include < if test "x$ac_cv_have_decl_RTLD_LOCAL" = xyes then : ac_have_decl=1 -else $as_nop - ac_have_decl=0 +else case e in #( + e) ac_have_decl=0 ;; +esac fi printf "%s\n" "#define HAVE_DECL_RTLD_LOCAL $ac_have_decl" >>confdefs.h ac_fn_check_decl "$LINENO" "RTLD_NODELETE" "ac_cv_have_decl_RTLD_NODELETE" "#include @@ -23784,8 +25635,9 @@ ac_fn_check_decl "$LINENO" "RTLD_NODELETE" "ac_cv_have_decl_RTLD_NODELETE" "#inc if test "x$ac_cv_have_decl_RTLD_NODELETE" = xyes then : ac_have_decl=1 -else $as_nop - ac_have_decl=0 +else case e in #( + e) ac_have_decl=0 ;; +esac fi printf "%s\n" "#define HAVE_DECL_RTLD_NODELETE $ac_have_decl" >>confdefs.h ac_fn_check_decl "$LINENO" "RTLD_NOLOAD" "ac_cv_have_decl_RTLD_NOLOAD" "#include @@ -23793,8 +25645,9 @@ ac_fn_check_decl "$LINENO" "RTLD_NOLOAD" "ac_cv_have_decl_RTLD_NOLOAD" "#include if test "x$ac_cv_have_decl_RTLD_NOLOAD" = xyes then : ac_have_decl=1 -else $as_nop - ac_have_decl=0 +else case e in #( + e) ac_have_decl=0 ;; +esac fi printf "%s\n" "#define HAVE_DECL_RTLD_NOLOAD $ac_have_decl" >>confdefs.h ac_fn_check_decl "$LINENO" "RTLD_DEEPBIND" "ac_cv_have_decl_RTLD_DEEPBIND" "#include @@ -23802,8 +25655,9 @@ ac_fn_check_decl "$LINENO" "RTLD_DEEPBIND" "ac_cv_have_decl_RTLD_DEEPBIND" "#inc if test "x$ac_cv_have_decl_RTLD_DEEPBIND" = xyes then : ac_have_decl=1 -else $as_nop - ac_have_decl=0 +else case e in #( + e) ac_have_decl=0 ;; +esac fi printf "%s\n" "#define HAVE_DECL_RTLD_DEEPBIND $ac_have_decl" >>confdefs.h ac_fn_check_decl "$LINENO" "RTLD_MEMBER" "ac_cv_have_decl_RTLD_MEMBER" "#include @@ -23811,8 +25665,9 @@ ac_fn_check_decl "$LINENO" "RTLD_MEMBER" "ac_cv_have_decl_RTLD_MEMBER" "#include if test "x$ac_cv_have_decl_RTLD_MEMBER" = xyes then : ac_have_decl=1 -else $as_nop - ac_have_decl=0 +else case e in #( + e) ac_have_decl=0 ;; +esac fi printf "%s\n" "#define HAVE_DECL_RTLD_MEMBER $ac_have_decl" >>confdefs.h @@ -23839,9 +25694,10 @@ printf "%s\n" "$enable_big_digits" >&6; } printf "%s\n" "#define PYLONG_BITS_IN_DIGIT $enable_big_digits" >>confdefs.h -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no value specified" >&5 -printf "%s\n" "no value specified" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no value specified" >&5 +printf "%s\n" "no value specified" >&6; } ;; +esac fi @@ -23855,9 +25711,10 @@ printf "%s\n" "#define HAVE_WCHAR_H 1" >>confdefs.h wchar_h="yes" -else $as_nop - wchar_h="no" - +else case e in #( + e) wchar_h="no" + ;; +esac fi @@ -23866,29 +25723,31 @@ if test "$wchar_h" = yes then # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects -# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# declarations like 'int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking size of wchar_t" >&5 printf %s "checking size of wchar_t... " >&6; } if test ${ac_cv_sizeof_wchar_t+y} then : printf %s "(cached) " >&6 -else $as_nop - if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (wchar_t))" "ac_cv_sizeof_wchar_t" "#include +else case e in #( + e) if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (wchar_t))" "ac_cv_sizeof_wchar_t" "#include " then : -else $as_nop - if test "$ac_cv_type_wchar_t" = yes; then - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +else case e in #( + e) if test "$ac_cv_type_wchar_t" = yes; then + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (wchar_t) -See \`config.log' for more details" "$LINENO" 5; } +See 'config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_wchar_t=0 - fi + fi ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_wchar_t" >&5 printf "%s\n" "$ac_cv_sizeof_wchar_t" >&6; } @@ -23909,13 +25768,13 @@ printf %s "checking whether wchar_t is signed... " >&6; } if test ${ac_cv_wchar_t_signed+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_wchar_t_signed=yes -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -23929,13 +25788,16 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_wchar_t_signed=yes -else $as_nop - ac_cv_wchar_t_signed=no +else case e in #( + e) ac_cv_wchar_t_signed=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_wchar_t_signed" >&5 printf "%s\n" "$ac_cv_wchar_t_signed" >&6; } @@ -23979,8 +25841,8 @@ printf %s "checking whether byte ordering is bigendian... " >&6; } if test ${ac_cv_c_bigendian+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_cv_c_bigendian=unknown +else case e in #( + e) ac_cv_c_bigendian=unknown # See if we're dealing with a universal compiler. cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -24026,8 +25888,8 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext int main (void) { -#if ! (defined BYTE_ORDER && defined BIG_ENDIAN \ - && defined LITTLE_ENDIAN && BYTE_ORDER && BIG_ENDIAN \ +#if ! (defined BYTE_ORDER && defined BIG_ENDIAN \\ + && defined LITTLE_ENDIAN && BYTE_ORDER && BIG_ENDIAN \\ && LITTLE_ENDIAN) bogus endian macros #endif @@ -24058,8 +25920,9 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_c_bigendian=yes -else $as_nop - ac_cv_c_bigendian=no +else case e in #( + e) ac_cv_c_bigendian=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext fi @@ -24103,8 +25966,9 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_c_bigendian=yes -else $as_nop - ac_cv_c_bigendian=no +else case e in #( + e) ac_cv_c_bigendian=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext fi @@ -24131,22 +25995,23 @@ unsigned short int ascii_mm[] = int use_ebcdic (int i) { return ebcdic_mm[i] + ebcdic_ii[i]; } - extern int foo; - -int -main (void) -{ -return use_ascii (foo) == use_ebcdic (foo); - ; - return 0; -} + int + main (int argc, char **argv) + { + /* Intimidate the compiler so that it does not + optimize the arrays away. */ + char *p = argv[0]; + ascii_mm[1] = *p++; ebcdic_mm[1] = *p++; + ascii_ii[1] = *p++; ebcdic_ii[1] = *p++; + return use_ascii (argc) == use_ebcdic (*p); + } _ACEOF -if ac_fn_c_try_compile "$LINENO" +if ac_fn_c_try_link "$LINENO" then : - if grep BIGenDianSyS conftest.$ac_objext >/dev/null; then + if grep BIGenDianSyS conftest$ac_exeext >/dev/null; then ac_cv_c_bigendian=yes fi - if grep LiTTleEnDian conftest.$ac_objext >/dev/null ; then + if grep LiTTleEnDian conftest$ac_exeext >/dev/null ; then if test "$ac_cv_c_bigendian" = unknown; then ac_cv_c_bigendian=no else @@ -24155,9 +26020,10 @@ then : fi fi fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $ac_includes_default int @@ -24180,14 +26046,17 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_c_bigendian=no -else $as_nop - ac_cv_c_bigendian=yes +else case e in #( + e) ac_cv_c_bigendian=yes ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - fi + fi ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_bigendian" >&5 printf "%s\n" "$ac_cv_c_bigendian" >&6; } @@ -24255,30 +26124,33 @@ LDVERSION='$(VERSION)$(ABIFLAGS)' { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDVERSION" >&5 printf "%s\n" "$LDVERSION" >&6; } -# Configure the flags and dependencies used when compiling shared modules +# Configure the flags and dependencies used when compiling shared modules. +# Do not rename LIBPYTHON - it's accessed via sysconfig by package build +# systems (e.g. Meson) to decide whether to link extension modules against +# libpython. MODULE_DEPS_SHARED='$(MODULE_DEPS_STATIC) $(EXPORTSYMS)' -MODULE_LDFLAGS='' +LIBPYTHON='' # On Android and Cygwin the shared libraries must be linked with libpython. if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(LDLIBRARY)" - MODULE_LDFLAGS="\$(BLDLIBRARY)" + LIBPYTHON="\$(BLDLIBRARY)" fi # On iOS the shared libraries must be linked with the Python framework -if test "$ac_sys_system" == "iOS"; then +if test "$ac_sys_system" = "iOS"; then MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(PYTHONFRAMEWORKDIR)/\$(PYTHONFRAMEWORK)" fi -BINLIBDEST='$(LIBDIR)/python$(VERSION)' +BINLIBDEST='$(LIBDIR)/python$(VERSION)$(ABI_THREAD)' # Check for --with-platlibdir -# /usr/$LIDIRNAME/python$VERSION +# /usr/$PLATLIBDIR/python$(VERSION)$(ABI_THREAD) PLATLIBDIR="lib" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-platlibdir" >&5 @@ -24297,23 +26169,24 @@ then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } PLATLIBDIR="$withval" - BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)' + BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)$(ABI_THREAD)' else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } fi -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac fi if test x$PLATFORM_TRIPLET = x; then - LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}" + LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}" else - LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}-${PLATFORM_TRIPLET}" + LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}-${PLATFORM_TRIPLET}" fi @@ -24335,9 +26208,10 @@ else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } fi -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } ;; +esac fi @@ -24348,13 +26222,13 @@ printf %s "checking whether right shift extends the sign bit... " >&6; } if test ${ac_cv_rshift_extends_sign+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_rshift_extends_sign=yes -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main(void) @@ -24366,13 +26240,16 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_rshift_extends_sign=yes -else $as_nop - ac_cv_rshift_extends_sign=no +else case e in #( + e) ac_cv_rshift_extends_sign=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_rshift_extends_sign" >&5 printf "%s\n" "$ac_cv_rshift_extends_sign" >&6; } @@ -24389,8 +26266,8 @@ printf %s "checking for getc_unlocked() and friends... " >&6; } if test ${ac_cv_have_getc_unlocked+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -24410,11 +26287,13 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_have_getc_unlocked=yes -else $as_nop - ac_cv_have_getc_unlocked=no +else case e in #( + e) ac_cv_have_getc_unlocked=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext + conftest$ac_exeext conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_have_getc_unlocked" >&5 printf "%s\n" "$ac_cv_have_getc_unlocked" >&6; } @@ -24445,9 +26324,10 @@ then : ;; esac -else $as_nop - with_readline=readline - +else case e in #( + e) with_readline=readline + ;; +esac fi @@ -24506,12 +26386,12 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - LIBREADLINE_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "readline" 2>&1` + LIBREADLINE_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "readline" 2>&1` else - LIBREADLINE_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "readline" 2>&1` + LIBREADLINE_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "readline" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$LIBREADLINE_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$LIBREADLINE_PKG_ERRORS" >&5 save_CFLAGS=$CFLAGS @@ -24521,7 +26401,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBREADLINE_CFLAGS" - LDFLAGS="$LDFLAGS $LIBREADLINE_LIBS" + LIBS="$LIBS $LIBREADLINE_LIBS" for ac_header in readline/readline.h do : ac_fn_c_check_header_compile "$LINENO" "readline/readline.h" "ac_cv_header_readline_readline_h" "$ac_includes_default" @@ -24534,16 +26414,22 @@ printf %s "checking for readline in -lreadline... " >&6; } if test ${ac_cv_lib_readline_readline+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lreadline $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char readline (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char readline (void); int main (void) { @@ -24555,12 +26441,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_readline_readline=yes -else $as_nop - ac_cv_lib_readline_readline=no +else case e in #( + e) ac_cv_lib_readline_readline=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_readline_readline" >&5 printf "%s\n" "$ac_cv_lib_readline_readline" >&6; } @@ -24571,13 +26459,15 @@ then : READLINE_CFLAGS=${LIBREADLINE_CFLAGS-""} READLINE_LIBS=${LIBREADLINE_LIBS-"-lreadline"} -else $as_nop - with_readline=no +else case e in #( + e) with_readline=no ;; +esac fi -else $as_nop - with_readline=no +else case e in #( + e) with_readline=no ;; +esac fi done @@ -24600,7 +26490,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBREADLINE_CFLAGS" - LDFLAGS="$LDFLAGS $LIBREADLINE_LIBS" + LIBS="$LIBS $LIBREADLINE_LIBS" for ac_header in readline/readline.h do : ac_fn_c_check_header_compile "$LINENO" "readline/readline.h" "ac_cv_header_readline_readline_h" "$ac_includes_default" @@ -24613,16 +26503,22 @@ printf %s "checking for readline in -lreadline... " >&6; } if test ${ac_cv_lib_readline_readline+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-lreadline $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char readline (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char readline (void); int main (void) { @@ -24634,12 +26530,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_readline_readline=yes -else $as_nop - ac_cv_lib_readline_readline=no +else case e in #( + e) ac_cv_lib_readline_readline=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_readline_readline" >&5 printf "%s\n" "$ac_cv_lib_readline_readline" >&6; } @@ -24650,13 +26548,15 @@ then : READLINE_CFLAGS=${LIBREADLINE_CFLAGS-""} READLINE_LIBS=${LIBREADLINE_LIBS-"-lreadline"} -else $as_nop - with_readline=no +else case e in #( + e) with_readline=no ;; +esac fi -else $as_nop - with_readline=no +else case e in #( + e) with_readline=no ;; +esac fi done @@ -24669,8 +26569,8 @@ LIBS=$save_LIBS else - LIBREADLINE_CFLAGS=$pkg_cv_LIBREADLINE_CFLAGS - LIBREADLINE_LIBS=$pkg_cv_LIBREADLINE_LIBS + LIBREADLINE_CFLAGS=$pkg_cv_LIBREADLINE_CFLAGS + LIBREADLINE_LIBS=$pkg_cv_LIBREADLINE_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } @@ -24737,12 +26637,12 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - LIBEDIT_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libedit" 2>&1` + LIBEDIT_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libedit" 2>&1` else - LIBEDIT_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libedit" 2>&1` + LIBEDIT_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libedit" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$LIBEDIT_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$LIBEDIT_PKG_ERRORS" >&5 save_CFLAGS=$CFLAGS @@ -24752,7 +26652,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBEDIT_CFLAGS" - LDFLAGS="$LDFLAGS $LIBEDIT_LIBS" + LIBS="$LIBS $LIBEDIT_LIBS" for ac_header in editline/readline.h do : ac_fn_c_check_header_compile "$LINENO" "editline/readline.h" "ac_cv_header_editline_readline_h" "$ac_includes_default" @@ -24765,16 +26665,22 @@ printf %s "checking for readline in -ledit... " >&6; } if test ${ac_cv_lib_edit_readline+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-ledit $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char readline (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char readline (void); int main (void) { @@ -24786,12 +26692,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_edit_readline=yes -else $as_nop - ac_cv_lib_edit_readline=no +else case e in #( + e) ac_cv_lib_edit_readline=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_edit_readline" >&5 printf "%s\n" "$ac_cv_lib_edit_readline" >&6; } @@ -24804,13 +26712,15 @@ then : READLINE_CFLAGS=${LIBEDIT_CFLAGS-""} READLINE_LIBS=${LIBEDIT_LIBS-"-ledit"} -else $as_nop - with_readline=no +else case e in #( + e) with_readline=no ;; +esac fi -else $as_nop - with_readline=no +else case e in #( + e) with_readline=no ;; +esac fi done @@ -24833,7 +26743,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBEDIT_CFLAGS" - LDFLAGS="$LDFLAGS $LIBEDIT_LIBS" + LIBS="$LIBS $LIBEDIT_LIBS" for ac_header in editline/readline.h do : ac_fn_c_check_header_compile "$LINENO" "editline/readline.h" "ac_cv_header_editline_readline_h" "$ac_includes_default" @@ -24846,16 +26756,22 @@ printf %s "checking for readline in -ledit... " >&6; } if test ${ac_cv_lib_edit_readline+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS LIBS="-ledit $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char readline (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char readline (void); int main (void) { @@ -24867,12 +26783,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_lib_edit_readline=yes -else $as_nop - ac_cv_lib_edit_readline=no +else case e in #( + e) ac_cv_lib_edit_readline=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +LIBS=$ac_check_lib_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_edit_readline" >&5 printf "%s\n" "$ac_cv_lib_edit_readline" >&6; } @@ -24885,13 +26803,15 @@ then : READLINE_CFLAGS=${LIBEDIT_CFLAGS-""} READLINE_LIBS=${LIBEDIT_LIBS-"-ledit"} -else $as_nop - with_readline=no +else case e in #( + e) with_readline=no ;; +esac fi -else $as_nop - with_readline=no +else case e in #( + e) with_readline=no ;; +esac fi done @@ -24904,8 +26824,8 @@ LIBS=$save_LIBS else - LIBEDIT_CFLAGS=$pkg_cv_LIBEDIT_CFLAGS - LIBEDIT_LIBS=$pkg_cv_LIBEDIT_LIBS + LIBEDIT_CFLAGS=$pkg_cv_LIBEDIT_CFLAGS + LIBEDIT_LIBS=$pkg_cv_LIBEDIT_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } @@ -24929,8 +26849,8 @@ then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_readline (CFLAGS: $READLINE_CFLAGS, LIBS: $READLINE_LIBS)" >&5 printf "%s\n" "$with_readline (CFLAGS: $READLINE_CFLAGS, LIBS: $READLINE_LIBS)" >&6; } @@ -24941,7 +26861,7 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $READLINE_CFLAGS" - LIBS="$READLINE_LIBS $LIBS" + LIBS="$LIBS $READLINE_LIBS" LIBS_SAVE=$LIBS @@ -24991,8 +26911,8 @@ printf %s "checking for rl_pre_input_hook in -l$LIBREADLINE... " >&6; } if test ${ac_cv_readline_rl_pre_input_hook+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -25015,13 +26935,15 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_readline_rl_pre_input_hook=yes -else $as_nop - ac_cv_readline_rl_pre_input_hook=no - +else case e in #( + e) ac_cv_readline_rl_pre_input_hook=no + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_readline_rl_pre_input_hook" >&5 printf "%s\n" "$ac_cv_readline_rl_pre_input_hook" >&6; } @@ -25040,8 +26962,8 @@ printf %s "checking for rl_completion_display_matches_hook in -l$LIBREADLINE... if test ${ac_cv_readline_rl_completion_display_matches_hook+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -25064,13 +26986,15 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_readline_rl_completion_display_matches_hook=yes -else $as_nop - ac_cv_readline_rl_completion_display_matches_hook=no - +else case e in #( + e) ac_cv_readline_rl_completion_display_matches_hook=no + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_readline_rl_completion_display_matches_hook" >&5 printf "%s\n" "$ac_cv_readline_rl_completion_display_matches_hook" >&6; } @@ -25089,8 +27013,8 @@ printf %s "checking for rl_resize_terminal in -l$LIBREADLINE... " >&6; } if test ${ac_cv_readline_rl_resize_terminal+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -25113,13 +27037,15 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_readline_rl_resize_terminal=yes -else $as_nop - ac_cv_readline_rl_resize_terminal=no - +else case e in #( + e) ac_cv_readline_rl_resize_terminal=no + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_readline_rl_resize_terminal" >&5 printf "%s\n" "$ac_cv_readline_rl_resize_terminal" >&6; } @@ -25138,8 +27064,8 @@ printf %s "checking for rl_completion_matches in -l$LIBREADLINE... " >&6; } if test ${ac_cv_readline_rl_completion_matches+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -25162,13 +27088,15 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_readline_rl_completion_matches=yes -else $as_nop - ac_cv_readline_rl_completion_matches=no - +else case e in #( + e) ac_cv_readline_rl_completion_matches=no + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_readline_rl_completion_matches" >&5 printf "%s\n" "$ac_cv_readline_rl_completion_matches" >&6; } @@ -25206,8 +27134,8 @@ printf %s "checking for append_history in -l$LIBREADLINE... " >&6; } if test ${ac_cv_readline_append_history+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -25230,13 +27158,15 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_readline_append_history=yes -else $as_nop - ac_cv_readline_append_history=no - +else case e in #( + e) ac_cv_readline_append_history=no + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_readline_append_history" >&5 printf "%s\n" "$ac_cv_readline_append_history" >&6; } @@ -25269,6 +27199,58 @@ printf "%s\n" "#define HAVE_RL_COMPDISP_FUNC_T 1" >>confdefs.h fi + # Some editline versions declare rl_startup_hook as taking no args, others + # declare it as taking 2. + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if rl_startup_hook takes arguments" >&5 +printf %s "checking if rl_startup_hook takes arguments... " >&6; } +if test ${ac_cv_readline_rl_startup_hook_takes_args+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + #include /* Must be first for Gnu Readline */ + #ifdef WITH_EDITLINE + # include + #else + # include + # include + #endif + + extern int test_hook_func(const char *text, int state); +int +main (void) +{ +rl_startup_hook=test_hook_func; + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_readline_rl_startup_hook_takes_args=yes +else case e in #( + e) ac_cv_readline_rl_startup_hook_takes_args=no + ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_readline_rl_startup_hook_takes_args" >&5 +printf "%s\n" "$ac_cv_readline_rl_startup_hook_takes_args" >&6; } + if test "x$ac_cv_readline_rl_startup_hook_takes_args" = xyes +then : + + +printf "%s\n" "#define Py_RL_STARTUP_HOOK_TAKES_ARGS 1" >>confdefs.h + + +fi + CFLAGS=$save_CFLAGS @@ -25276,7 +27258,8 @@ CPPFLAGS=$save_CPPFLAGS LDFLAGS=$save_LDFLAGS LIBS=$save_LIBS - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for broken nice()" >&5 @@ -25284,13 +27267,13 @@ printf %s "checking for broken nice()... " >&6; } if test ${ac_cv_broken_nice+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_broken_nice=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -25307,13 +27290,16 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_broken_nice=yes -else $as_nop - ac_cv_broken_nice=no +else case e in #( + e) ac_cv_broken_nice=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_broken_nice" >&5 printf "%s\n" "$ac_cv_broken_nice" >&6; } @@ -25329,12 +27315,12 @@ printf %s "checking for broken poll()... " >&6; } if test ${ac_cv_broken_poll+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_broken_poll=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -25360,13 +27346,16 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_broken_poll=yes -else $as_nop - ac_cv_broken_poll=no +else case e in #( + e) ac_cv_broken_poll=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_broken_poll" >&5 printf "%s\n" "$ac_cv_broken_poll" >&6; } @@ -25383,13 +27372,13 @@ printf %s "checking for working tzset()... " >&6; } if test ${ac_cv_working_tzset+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_working_tzset=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -25459,13 +27448,16 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_working_tzset=yes -else $as_nop - ac_cv_working_tzset=no +else case e in #( + e) ac_cv_working_tzset=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_working_tzset" >&5 printf "%s\n" "$ac_cv_working_tzset" >&6; } @@ -25482,8 +27474,8 @@ printf %s "checking for tv_nsec in struct stat... " >&6; } if test ${ac_cv_stat_tv_nsec+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -25500,10 +27492,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_stat_tv_nsec=yes -else $as_nop - ac_cv_stat_tv_nsec=no +else case e in #( + e) ac_cv_stat_tv_nsec=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_stat_tv_nsec" >&5 printf "%s\n" "$ac_cv_stat_tv_nsec" >&6; } @@ -25520,8 +27514,8 @@ printf %s "checking for tv_nsec2 in struct stat... " >&6; } if test ${ac_cv_stat_tv_nsec2+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int @@ -25538,10 +27532,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_stat_tv_nsec2=yes -else $as_nop - ac_cv_stat_tv_nsec2=no +else case e in #( + e) ac_cv_stat_tv_nsec2=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_stat_tv_nsec2" >&5 printf "%s\n" "$ac_cv_stat_tv_nsec2" >&6; } @@ -25552,28 +27548,118 @@ printf "%s\n" "#define HAVE_STAT_TV_NSEC2 1" >>confdefs.h fi -have_curses=no -have_panel=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether year with century should be normalized for strftime" >&5 +printf %s "checking whether year with century should be normalized for strftime... " >&6; } +if test ${ac_cv_normalize_century+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) +if test "$cross_compiling" = yes +then : + ac_cv_normalize_century=yes +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +#include -ac_fn_c_check_header_compile "$LINENO" "curses.h" "ac_cv_header_curses_h" "$ac_includes_default" -if test "x$ac_cv_header_curses_h" = xyes +int main(void) +{ + char year[5]; + struct tm date = { + .tm_year = -1801, + .tm_mon = 0, + .tm_mday = 1 + }; + if (strftime(year, sizeof(year), "%Y", &date) && !strcmp(year, "0099")) { + return 1; + } + return 0; +} + +_ACEOF +if ac_fn_c_try_run "$LINENO" then : - printf "%s\n" "#define HAVE_CURSES_H 1" >>confdefs.h + ac_cv_normalize_century=yes +else case e in #( + e) ac_cv_normalize_century=no ;; +esac +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_normalize_century" >&5 +printf "%s\n" "$ac_cv_normalize_century" >&6; } +if test "$ac_cv_normalize_century" = yes +then + +printf "%s\n" "#define Py_NORMALIZE_CENTURY 1" >>confdefs.h fi -ac_fn_c_check_header_compile "$LINENO" "ncurses.h" "ac_cv_header_ncurses_h" "$ac_includes_default" -if test "x$ac_cv_header_ncurses_h" = xyes + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C99-compatible strftime specifiers are supported" >&5 +printf %s "checking whether C99-compatible strftime specifiers are supported... " >&6; } +if test ${ac_cv_strftime_c99_support+y} then : - printf "%s\n" "#define HAVE_NCURSES_H 1" >>confdefs.h + printf %s "(cached) " >&6 +else case e in #( + e) +if test "$cross_compiling" = yes +then : + ac_cv_strftime_c99_support= +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ -fi +#include +#include +int main(void) +{ + char full_date[11]; + struct tm date = { + .tm_year = 0, + .tm_mon = 0, + .tm_mday = 1 + }; + if (strftime(full_date, sizeof(full_date), "%F", &date) && !strcmp(full_date, "1900-01-01")) { + return 0; + } + return 1; +} -if test "x$ac_cv_header_ncurses_h" = xyes +_ACEOF +if ac_fn_c_try_run "$LINENO" then : + ac_cv_strftime_c99_support=yes +else case e in #( + e) as_fn_error $? "Python requires C99-compatible strftime specifiers" "$LINENO" 5 ;; +esac +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_strftime_c99_support" >&5 +printf "%s\n" "$ac_cv_strftime_c99_support" >&6; } + +have_curses=no +have_panel=no + + + +# Check for ncursesw/panelw first. If that fails, try ncurses/panel. + - if test "$ac_sys_system" != "Darwin"; then pkg_failed=no { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ncursesw" >&5 @@ -25626,169 +27712,42 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - CURSES_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "ncursesw" 2>&1` + CURSES_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "ncursesw" 2>&1` else - CURSES_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "ncursesw" 2>&1` + CURSES_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "ncursesw" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$CURSES_PKG_ERRORS" >&5 - - - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for initscr in -lncursesw" >&5 -printf %s "checking for initscr in -lncursesw... " >&6; } -if test ${ac_cv_lib_ncursesw_initscr+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lncursesw $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char initscr (); -int -main (void) -{ -return initscr (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ac_cv_lib_ncursesw_initscr=yes -else $as_nop - ac_cv_lib_ncursesw_initscr=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ncursesw_initscr" >&5 -printf "%s\n" "$ac_cv_lib_ncursesw_initscr" >&6; } -if test "x$ac_cv_lib_ncursesw_initscr" = xyes -then : - - printf "%s\n" "#define HAVE_NCURSESW 1" >>confdefs.h - - have_curses=ncursesw - CURSES_CFLAGS=${CURSES_CFLAGS-""} - CURSES_LIBS=${CURSES_LIBS-"-lncursesw"} - -fi - - -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS - - + # Put the nasty error message in config.log where it belongs + echo "$CURSES_PKG_ERRORS" >&5 + have_curses=no elif test $pkg_failed = untried; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for initscr in -lncursesw" >&5 -printf %s "checking for initscr in -lncursesw... " >&6; } -if test ${ac_cv_lib_ncursesw_initscr+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lncursesw $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char initscr (); -int -main (void) -{ -return initscr (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ac_cv_lib_ncursesw_initscr=yes -else $as_nop - ac_cv_lib_ncursesw_initscr=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ncursesw_initscr" >&5 -printf "%s\n" "$ac_cv_lib_ncursesw_initscr" >&6; } -if test "x$ac_cv_lib_ncursesw_initscr" = xyes -then : - - printf "%s\n" "#define HAVE_NCURSESW 1" >>confdefs.h - - have_curses=ncursesw - CURSES_CFLAGS=${CURSES_CFLAGS-""} - CURSES_LIBS=${CURSES_LIBS-"-lncursesw"} - -fi - - -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS - - - + have_curses=no else - CURSES_CFLAGS=$pkg_cv_CURSES_CFLAGS - CURSES_LIBS=$pkg_cv_CURSES_LIBS + CURSES_CFLAGS=$pkg_cv_CURSES_CFLAGS + CURSES_LIBS=$pkg_cv_CURSES_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } - printf "%s\n" "#define HAVE_NCURSESW 1" >>confdefs.h - - have_curses=ncursesw - -fi - fi - - if test "x$have_curses" = xno -then : +printf "%s\n" "#define HAVE_NCURSESW 1" >>confdefs.h + have_curses=yes pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ncurses" >&5 -printf %s "checking for ncurses... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for panelw" >&5 +printf %s "checking for panelw... " >&6; } -if test -n "$CURSES_CFLAGS"; then - pkg_cv_CURSES_CFLAGS="$CURSES_CFLAGS" +if test -n "$PANEL_CFLAGS"; then + pkg_cv_PANEL_CFLAGS="$PANEL_CFLAGS" elif test -n "$PKG_CONFIG"; then if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncurses\""; } >&5 - ($PKG_CONFIG --exists --print-errors "ncurses") 2>&5 + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"panelw\""; } >&5 + ($PKG_CONFIG --exists --print-errors "panelw") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - pkg_cv_CURSES_CFLAGS=`$PKG_CONFIG --cflags "ncurses" 2>/dev/null` + pkg_cv_PANEL_CFLAGS=`$PKG_CONFIG --cflags "panelw" 2>/dev/null` test "x$?" != "x0" && pkg_failed=yes else pkg_failed=yes @@ -25796,16 +27755,16 @@ fi else pkg_failed=untried fi -if test -n "$CURSES_LIBS"; then - pkg_cv_CURSES_LIBS="$CURSES_LIBS" +if test -n "$PANEL_LIBS"; then + pkg_cv_PANEL_LIBS="$PANEL_LIBS" elif test -n "$PKG_CONFIG"; then if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncurses\""; } >&5 - ($PKG_CONFIG --exists --print-errors "ncurses") 2>&5 + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"panelw\""; } >&5 + ($PKG_CONFIG --exists --print-errors "panelw") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - pkg_cv_CURSES_LIBS=`$PKG_CONFIG --libs "ncurses" 2>/dev/null` + pkg_cv_PANEL_LIBS=`$PKG_CONFIG --libs "panelw" 2>/dev/null` test "x$?" != "x0" && pkg_failed=yes else pkg_failed=yes @@ -25826,204 +27785,50 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - CURSES_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "ncurses" 2>&1` + PANEL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "panelw" 2>&1` else - CURSES_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "ncurses" 2>&1` + PANEL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "panelw" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$CURSES_PKG_ERRORS" >&5 - - - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for initscr in -lncurses" >&5 -printf %s "checking for initscr in -lncurses... " >&6; } -if test ${ac_cv_lib_ncurses_initscr+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lncurses $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char initscr (); -int -main (void) -{ -return initscr (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ac_cv_lib_ncurses_initscr=yes -else $as_nop - ac_cv_lib_ncurses_initscr=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ncurses_initscr" >&5 -printf "%s\n" "$ac_cv_lib_ncurses_initscr" >&6; } -if test "x$ac_cv_lib_ncurses_initscr" = xyes -then : - - have_curses=ncurses - CURSES_CFLAGS=${CURSES_CFLAGS-""} - CURSES_LIBS=${CURSES_LIBS-"-lncurses"} - -fi - - -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS - - + # Put the nasty error message in config.log where it belongs + echo "$PANEL_PKG_ERRORS" >&5 + have_panel=no elif test $pkg_failed = untried; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for initscr in -lncurses" >&5 -printf %s "checking for initscr in -lncurses... " >&6; } -if test ${ac_cv_lib_ncurses_initscr+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lncurses $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char initscr (); -int -main (void) -{ -return initscr (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ac_cv_lib_ncurses_initscr=yes -else $as_nop - ac_cv_lib_ncurses_initscr=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ncurses_initscr" >&5 -printf "%s\n" "$ac_cv_lib_ncurses_initscr" >&6; } -if test "x$ac_cv_lib_ncurses_initscr" = xyes -then : - - have_curses=ncurses - CURSES_CFLAGS=${CURSES_CFLAGS-""} - CURSES_LIBS=${CURSES_LIBS-"-lncurses"} - -fi - - -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS - - - + have_panel=no else - CURSES_CFLAGS=$pkg_cv_CURSES_CFLAGS - CURSES_LIBS=$pkg_cv_CURSES_LIBS + PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS + PANEL_LIBS=$pkg_cv_PANEL_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } - have_curses=ncurses - -fi - -fi - +printf "%s\n" "#define HAVE_PANELW 1" >>confdefs.h + have_panel=yes fi -CURSES_CFLAGS=$(echo $CURSES_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') - -if test "$have_curses" != no -a "$ac_sys_system" = "Darwin"; then - - as_fn_append CURSES_CFLAGS " -D_XOPEN_SOURCE_EXTENDED=1" - printf "%s\n" "#define HAVE_NCURSESW 1" >>confdefs.h - fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking curses module flags" >&5 -printf %s "checking curses module flags... " >&6; } if test "x$have_curses" = xno then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - -else $as_nop - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_curses (CFLAGS: $CURSES_CFLAGS, LIBS: $CURSES_LIBS)" >&5 -printf "%s\n" "$have_curses (CFLAGS: $CURSES_CFLAGS, LIBS: $CURSES_LIBS)" >&6; } - -fi - -ac_fn_c_check_header_compile "$LINENO" "panel.h" "ac_cv_header_panel_h" "$ac_includes_default" -if test "x$ac_cv_header_panel_h" = xyes -then : - printf "%s\n" "#define HAVE_PANEL_H 1" >>confdefs.h - -fi - - -if test "x$ac_cv_header_panel_h" = xyes -then : - - - if test "$ac_sys_system" != "Darwin"; then - if test "x$have_curses" = xncursesw -then : pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for panelw" >&5 -printf %s "checking for panelw... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ncurses" >&5 +printf %s "checking for ncurses... " >&6; } -if test -n "$PANEL_CFLAGS"; then - pkg_cv_PANEL_CFLAGS="$PANEL_CFLAGS" +if test -n "$CURSES_CFLAGS"; then + pkg_cv_CURSES_CFLAGS="$CURSES_CFLAGS" elif test -n "$PKG_CONFIG"; then if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"panelw\""; } >&5 - ($PKG_CONFIG --exists --print-errors "panelw") 2>&5 + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncurses\""; } >&5 + ($PKG_CONFIG --exists --print-errors "ncurses") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - pkg_cv_PANEL_CFLAGS=`$PKG_CONFIG --cflags "panelw" 2>/dev/null` + pkg_cv_CURSES_CFLAGS=`$PKG_CONFIG --cflags "ncurses" 2>/dev/null` test "x$?" != "x0" && pkg_failed=yes else pkg_failed=yes @@ -26031,180 +27836,57 @@ fi else pkg_failed=untried fi -if test -n "$PANEL_LIBS"; then - pkg_cv_PANEL_LIBS="$PANEL_LIBS" +if test -n "$CURSES_LIBS"; then + pkg_cv_CURSES_LIBS="$CURSES_LIBS" elif test -n "$PKG_CONFIG"; then if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"panelw\""; } >&5 - ($PKG_CONFIG --exists --print-errors "panelw") 2>&5 + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncurses\""; } >&5 + ($PKG_CONFIG --exists --print-errors "ncurses") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - pkg_cv_PANEL_LIBS=`$PKG_CONFIG --libs "panelw" 2>/dev/null` + pkg_cv_CURSES_LIBS=`$PKG_CONFIG --libs "ncurses" 2>/dev/null` test "x$?" != "x0" && pkg_failed=yes else - pkg_failed=yes -fi - else - pkg_failed=untried -fi - - - -if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - -if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then - _pkg_short_errors_supported=yes -else - _pkg_short_errors_supported=no -fi - if test $_pkg_short_errors_supported = yes; then - PANEL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "panelw" 2>&1` - else - PANEL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "panelw" 2>&1` - fi - # Put the nasty error message in config.log where it belongs - echo "$PANEL_PKG_ERRORS" >&5 - - - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for update_panels in -lpanelw" >&5 -printf %s "checking for update_panels in -lpanelw... " >&6; } -if test ${ac_cv_lib_panelw_update_panels+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lpanelw $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char update_panels (); -int -main (void) -{ -return update_panels (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ac_cv_lib_panelw_update_panels=yes -else $as_nop - ac_cv_lib_panelw_update_panels=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_panelw_update_panels" >&5 -printf "%s\n" "$ac_cv_lib_panelw_update_panels" >&6; } -if test "x$ac_cv_lib_panelw_update_panels" = xyes -then : - - have_panel=panelw - PANEL_CFLAGS=${PANEL_CFLAGS-""} - PANEL_LIBS=${PANEL_LIBS-"-lpanelw"} - -fi - - -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS - - - -elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for update_panels in -lpanelw" >&5 -printf %s "checking for update_panels in -lpanelw... " >&6; } -if test ${ac_cv_lib_panelw_update_panels+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lpanelw $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char update_panels (); -int -main (void) -{ -return update_panels (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ac_cv_lib_panelw_update_panels=yes -else $as_nop - ac_cv_lib_panelw_update_panels=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_panelw_update_panels" >&5 -printf "%s\n" "$ac_cv_lib_panelw_update_panels" >&6; } -if test "x$ac_cv_lib_panelw_update_panels" = xyes -then : - - have_panel=panelw - PANEL_CFLAGS=${PANEL_CFLAGS-""} - PANEL_LIBS=${PANEL_LIBS-"-lpanelw"} - + pkg_failed=yes +fi + else + pkg_failed=untried fi -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + CURSES_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "ncurses" 2>&1` + else + CURSES_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "ncurses" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$CURSES_PKG_ERRORS" >&5 + have_curses=no +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_curses=no else - PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS - PANEL_LIBS=$pkg_cv_PANEL_LIBS + CURSES_CFLAGS=$pkg_cv_CURSES_CFLAGS + CURSES_LIBS=$pkg_cv_CURSES_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } - have_panel=panelw - -fi - -fi - fi - - if test "x$have_curses" = xncurses -then : +printf "%s\n" "#define HAVE_NCURSES 1" >>confdefs.h + have_curses=yes pkg_failed=no { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for panel" >&5 @@ -26257,97 +27939,194 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - PANEL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "panel" 2>&1` + PANEL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "panel" 2>&1` else - PANEL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "panel" 2>&1` + PANEL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "panel" 2>&1` fi - # Put the nasty error message in config.log where it belongs - echo "$PANEL_PKG_ERRORS" >&5 + # Put the nasty error message in config.log where it belongs + echo "$PANEL_PKG_ERRORS" >&5 + + have_panel=no +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_panel=no +else + PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS + PANEL_LIBS=$pkg_cv_PANEL_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } +printf "%s\n" "#define HAVE_PANEL 1" >>confdefs.h - save_CFLAGS=$CFLAGS + have_panel=yes +fi +fi + + +fi + +save_CFLAGS=$CFLAGS save_CPPFLAGS=$CPPFLAGS save_LDFLAGS=$LDFLAGS save_LIBS=$LIBS - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for update_panels in -lpanel" >&5 -printf %s "checking for update_panels in -lpanel... " >&6; } -if test ${ac_cv_lib_panel_update_panels+y} + # Make sure we've got the header defines. + as_fn_append CPPFLAGS " $CURSES_CFLAGS $PANEL_CFLAGS" + ac_fn_c_check_header_compile "$LINENO" "ncursesw/curses.h" "ac_cv_header_ncursesw_curses_h" "$ac_includes_default" +if test "x$ac_cv_header_ncursesw_curses_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSESW_CURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncursesw/ncurses.h" "ac_cv_header_ncursesw_ncurses_h" "$ac_includes_default" +if test "x$ac_cv_header_ncursesw_ncurses_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSESW_NCURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncursesw/panel.h" "ac_cv_header_ncursesw_panel_h" "$ac_includes_default" +if test "x$ac_cv_header_ncursesw_panel_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSESW_PANEL_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncurses/curses.h" "ac_cv_header_ncurses_curses_h" "$ac_includes_default" +if test "x$ac_cv_header_ncurses_curses_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSES_CURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncurses/ncurses.h" "ac_cv_header_ncurses_ncurses_h" "$ac_includes_default" +if test "x$ac_cv_header_ncurses_ncurses_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSES_NCURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncurses/panel.h" "ac_cv_header_ncurses_panel_h" "$ac_includes_default" +if test "x$ac_cv_header_ncurses_panel_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSES_PANEL_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "curses.h" "ac_cv_header_curses_h" "$ac_includes_default" +if test "x$ac_cv_header_curses_h" = xyes +then : + printf "%s\n" "#define HAVE_CURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncurses.h" "ac_cv_header_ncurses_h" "$ac_includes_default" +if test "x$ac_cv_header_ncurses_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "panel.h" "ac_cv_header_panel_h" "$ac_includes_default" +if test "x$ac_cv_header_panel_h" = xyes +then : + printf "%s\n" "#define HAVE_PANEL_H 1" >>confdefs.h + +fi + + + # Check that we're able to link with crucial curses/panel functions. This + # also serves as a fallback in case pkg-config failed. + as_fn_append LIBS " $CURSES_LIBS $PANEL_LIBS" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing initscr" >&5 +printf %s "checking for library containing initscr... " >&6; } +if test ${ac_cv_search_initscr+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lpanel $LIBS" +else case e in #( + e) ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char update_panels (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char initscr (void); int main (void) { -return update_panels (); +return initscr (); ; return 0; } _ACEOF -if ac_fn_c_try_link "$LINENO" +for ac_lib in '' ncursesw ncurses +do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_panel_update_panels=yes -else $as_nop - ac_cv_lib_panel_update_panels=no + ac_cv_search_initscr=$ac_res fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS + conftest$ac_exeext + if test ${ac_cv_search_initscr+y} +then : + break fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_panel_update_panels" >&5 -printf "%s\n" "$ac_cv_lib_panel_update_panels" >&6; } -if test "x$ac_cv_lib_panel_update_panels" = xyes +done +if test ${ac_cv_search_initscr+y} then : - have_panel=panel - PANEL_CFLAGS=${PANEL_CFLAGS-""} - PANEL_LIBS=${PANEL_LIBS-"-lpanel"} - +else case e in #( + e) ac_cv_search_initscr=no ;; +esac +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_initscr" >&5 +printf "%s\n" "$ac_cv_search_initscr" >&6; } +ac_res=$ac_cv_search_initscr +if test "$ac_res" != no +then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + if test "x$have_curses" = xno +then : + have_curses=yes + CURSES_LIBS=${CURSES_LIBS-"$ac_cv_search_initscr"} +fi +else case e in #( + e) have_curses=no ;; +esac fi - -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS - - - -elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - - save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for update_panels in -lpanel" >&5 -printf %s "checking for update_panels in -lpanel... " >&6; } -if test ${ac_cv_lib_panel_update_panels+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing update_panels" >&5 +printf %s "checking for library containing update_panels... " >&6; } +if test ${ac_cv_search_update_panels+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lpanel $LIBS" +else case e in #( + e) ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char update_panels (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char update_panels (void); int main (void) { @@ -26356,76 +28135,85 @@ return update_panels (); return 0; } _ACEOF -if ac_fn_c_try_link "$LINENO" +for ac_lib in '' panelw panel +do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_panel_update_panels=yes -else $as_nop - ac_cv_lib_panel_update_panels=no + ac_cv_search_update_panels=$ac_res fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS + conftest$ac_exeext + if test ${ac_cv_search_update_panels+y} +then : + break fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_panel_update_panels" >&5 -printf "%s\n" "$ac_cv_lib_panel_update_panels" >&6; } -if test "x$ac_cv_lib_panel_update_panels" = xyes +done +if test ${ac_cv_search_update_panels+y} then : - have_panel=panel - PANEL_CFLAGS=${PANEL_CFLAGS-""} - PANEL_LIBS=${PANEL_LIBS-"-lpanel"} - +else case e in #( + e) ac_cv_search_update_panels=no ;; +esac fi - - -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS - - - -else - PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS - PANEL_LIBS=$pkg_cv_PANEL_LIBS - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } - - have_panel=panel - +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS ;; +esac fi - +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_update_panels" >&5 +printf "%s\n" "$ac_cv_search_update_panels" >&6; } +ac_res=$ac_cv_search_update_panels +if test "$ac_res" != no +then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + if test "x$have_panel" = xno +then : + have_panel=yes + PANEL_LIBS=${PANEL_LIBS-"$ac_cv_search_update_panels"} +fi +else case e in #( + e) have_panel=no ;; +esac fi -fi -PANEL_CFLAGS=$(echo $PANEL_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking panel flags" >&5 -printf %s "checking panel flags... " >&6; } -if test "x$have_panel" = xno + +if test "have_curses" != "no" then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +CURSES_CFLAGS=$(echo $CURSES_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') + +if test "x$ac_sys_system" = xDarwin +then : -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_panel (CFLAGS: $PANEL_CFLAGS, LIBS: $PANEL_LIBS)" >&5 -printf "%s\n" "$have_panel (CFLAGS: $PANEL_CFLAGS, LIBS: $PANEL_LIBS)" >&6; } + as_fn_append CURSES_CFLAGS " -D_XOPEN_SOURCE_EXTENDED=1" fi -# first curses header check -ac_save_cppflags="$CPPFLAGS" -if test "$cross_compiling" = no; then - CPPFLAGS="$CPPFLAGS -I/usr/include/ncursesw" -fi +PANEL_CFLAGS=$(echo $PANEL_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') # On Solaris, term.h requires curses.h ac_fn_c_check_header_compile "$LINENO" "term.h" "ac_cv_header_term_h" " -#ifdef HAVE_CURSES_H -#include +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include #endif " @@ -26442,10 +28230,25 @@ printf %s "checking whether mvwdelch is an expression... " >&6; } if test ${ac_cv_mvwdelch_is_expression+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26460,10 +28263,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_mvwdelch_is_expression=yes -else $as_nop - ac_cv_mvwdelch_is_expression=no +else case e in #( + e) ac_cv_mvwdelch_is_expression=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_mvwdelch_is_expression" >&5 printf "%s\n" "$ac_cv_mvwdelch_is_expression" >&6; } @@ -26475,21 +28280,29 @@ printf "%s\n" "#define MVWDELCH_IS_EXPRESSION 1" >>confdefs.h fi -# Issue #25720: ncurses has introduced the NCURSES_OPAQUE symbol making opaque -# structs since version 5.7. If the macro is defined as zero before including -# [n]curses.h, ncurses will expose fields of the structs regardless of the -# configuration. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether WINDOW has _flags" >&5 printf %s "checking whether WINDOW has _flags... " >&6; } if test ${ac_cv_window_has_flags+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #define NCURSES_OPAQUE 0 - #include +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif int main (void) @@ -26505,10 +28318,12 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_window_has_flags=yes -else $as_nop - ac_cv_window_has_flags=no +else case e in #( + e) ac_cv_window_has_flags=no ;; +esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_window_has_flags" >&5 printf "%s\n" "$ac_cv_window_has_flags" >&6; } @@ -26530,10 +28345,25 @@ printf %s "checking for curses function is_pad... " >&6; } if test ${ac_cv_lib_curses_is_pad+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26549,11 +28379,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_is_pad=yes -else $as_nop - ac_cv_lib_curses_is_pad=no +else case e in #( + e) ac_cv_lib_curses_is_pad=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_is_pad" >&5 printf "%s\n" "$ac_cv_lib_curses_is_pad" >&6; } @@ -26573,10 +28405,25 @@ printf %s "checking for curses function is_term_resized... " >&6; } if test ${ac_cv_lib_curses_is_term_resized+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26592,11 +28439,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_is_term_resized=yes -else $as_nop - ac_cv_lib_curses_is_term_resized=no +else case e in #( + e) ac_cv_lib_curses_is_term_resized=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_is_term_resized" >&5 printf "%s\n" "$ac_cv_lib_curses_is_term_resized" >&6; } @@ -26616,10 +28465,25 @@ printf %s "checking for curses function resize_term... " >&6; } if test ${ac_cv_lib_curses_resize_term+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26635,11 +28499,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_resize_term=yes -else $as_nop - ac_cv_lib_curses_resize_term=no +else case e in #( + e) ac_cv_lib_curses_resize_term=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_resize_term" >&5 printf "%s\n" "$ac_cv_lib_curses_resize_term" >&6; } @@ -26659,10 +28525,25 @@ printf %s "checking for curses function resizeterm... " >&6; } if test ${ac_cv_lib_curses_resizeterm+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26678,11 +28559,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_resizeterm=yes -else $as_nop - ac_cv_lib_curses_resizeterm=no +else case e in #( + e) ac_cv_lib_curses_resizeterm=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_resizeterm" >&5 printf "%s\n" "$ac_cv_lib_curses_resizeterm" >&6; } @@ -26702,10 +28585,25 @@ printf %s "checking for curses function immedok... " >&6; } if test ${ac_cv_lib_curses_immedok+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26721,11 +28619,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_immedok=yes -else $as_nop - ac_cv_lib_curses_immedok=no +else case e in #( + e) ac_cv_lib_curses_immedok=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_immedok" >&5 printf "%s\n" "$ac_cv_lib_curses_immedok" >&6; } @@ -26745,10 +28645,25 @@ printf %s "checking for curses function syncok... " >&6; } if test ${ac_cv_lib_curses_syncok+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26764,11 +28679,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_syncok=yes -else $as_nop - ac_cv_lib_curses_syncok=no +else case e in #( + e) ac_cv_lib_curses_syncok=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_syncok" >&5 printf "%s\n" "$ac_cv_lib_curses_syncok" >&6; } @@ -26788,10 +28705,25 @@ printf %s "checking for curses function wchgat... " >&6; } if test ${ac_cv_lib_curses_wchgat+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26807,11 +28739,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_wchgat=yes -else $as_nop - ac_cv_lib_curses_wchgat=no +else case e in #( + e) ac_cv_lib_curses_wchgat=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_wchgat" >&5 printf "%s\n" "$ac_cv_lib_curses_wchgat" >&6; } @@ -26831,10 +28765,25 @@ printf %s "checking for curses function filter... " >&6; } if test ${ac_cv_lib_curses_filter+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26850,11 +28799,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_filter=yes -else $as_nop - ac_cv_lib_curses_filter=no +else case e in #( + e) ac_cv_lib_curses_filter=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_filter" >&5 printf "%s\n" "$ac_cv_lib_curses_filter" >&6; } @@ -26874,10 +28825,25 @@ printf %s "checking for curses function has_key... " >&6; } if test ${ac_cv_lib_curses_has_key+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26893,11 +28859,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_has_key=yes -else $as_nop - ac_cv_lib_curses_has_key=no +else case e in #( + e) ac_cv_lib_curses_has_key=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_has_key" >&5 printf "%s\n" "$ac_cv_lib_curses_has_key" >&6; } @@ -26917,10 +28885,25 @@ printf %s "checking for curses function typeahead... " >&6; } if test ${ac_cv_lib_curses_typeahead+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26936,11 +28919,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_typeahead=yes -else $as_nop - ac_cv_lib_curses_typeahead=no +else case e in #( + e) ac_cv_lib_curses_typeahead=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_typeahead" >&5 printf "%s\n" "$ac_cv_lib_curses_typeahead" >&6; } @@ -26960,10 +28945,25 @@ printf %s "checking for curses function use_env... " >&6; } if test ${ac_cv_lib_curses_use_env+y} then : printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + int main (void) { @@ -26979,11 +28979,13 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_lib_curses_use_env=yes -else $as_nop - ac_cv_lib_curses_use_env=no +else case e in #( + e) ac_cv_lib_curses_use_env=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_use_env" >&5 printf "%s\n" "$ac_cv_lib_curses_use_env" >&6; } @@ -26998,6 +29000,13 @@ fi CPPFLAGS=$ac_save_cppflags +fi +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for device files" >&5 printf "%s\n" "$as_me: checking for device files" >&6;} @@ -27027,14 +29036,15 @@ printf %s "checking for /dev/ptmx... " >&6; } if test ${ac_cv_file__dev_ptmx+y} then : printf %s "(cached) " >&6 -else $as_nop - test "$cross_compiling" = yes && +else case e in #( + e) test "$cross_compiling" = yes && as_fn_error $? "cannot check for file existence when cross compiling" "$LINENO" 5 if test -r "/dev/ptmx"; then ac_cv_file__dev_ptmx=yes else ac_cv_file__dev_ptmx=no -fi +fi ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_file__dev_ptmx" >&5 printf "%s\n" "$ac_cv_file__dev_ptmx" >&6; } @@ -27053,14 +29063,15 @@ printf %s "checking for /dev/ptc... " >&6; } if test ${ac_cv_file__dev_ptc+y} then : printf %s "(cached) " >&6 -else $as_nop - test "$cross_compiling" = yes && +else case e in #( + e) test "$cross_compiling" = yes && as_fn_error $? "cannot check for file existence when cross compiling" "$LINENO" 5 if test -r "/dev/ptc"; then ac_cv_file__dev_ptc=yes else ac_cv_file__dev_ptc=no -fi +fi ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_file__dev_ptc" >&5 printf "%s\n" "$ac_cv_file__dev_ptc" >&6; } @@ -27096,10 +29107,11 @@ then : printf "%s\n" "#define HAVE_SOCKLEN_T 1" >>confdefs.h -else $as_nop - +else case e in #( + e) printf "%s\n" "#define socklen_t int" >>confdefs.h - + ;; +esac fi @@ -27108,12 +29120,12 @@ printf %s "checking for broken mbstowcs... " >&6; } if test ${ac_cv_broken_mbstowcs+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : ac_cv_broken_mbstowcs=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -27130,13 +29142,16 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_broken_mbstowcs=no -else $as_nop - ac_cv_broken_mbstowcs=yes +else case e in #( + e) ac_cv_broken_mbstowcs=yes ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_broken_mbstowcs" >&5 printf "%s\n" "$ac_cv_broken_mbstowcs" >&6; } @@ -27172,9 +29187,10 @@ printf "%s\n" "#define USE_COMPUTED_GOTOS 0" >>confdefs.h printf "%s\n" "no" >&6; } fi -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no value specified" >&5 -printf "%s\n" "no value specified" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no value specified" >&5 +printf "%s\n" "no value specified" >&6; } ;; +esac fi @@ -27183,16 +29199,16 @@ printf %s "checking whether $CC supports computed gotos... " >&6; } if test ${ac_cv_computed_gotos+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes +else case e in #( + e) if test "$cross_compiling" = yes then : if test "${with_computed_gotos+set}" = set; then ac_cv_computed_gotos="$with_computed_gotos -- configured --with(out)-computed-gotos" else ac_cv_computed_gotos=no fi -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main(int argc, char **argv) @@ -27210,13 +29226,16 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : ac_cv_computed_gotos=yes -else $as_nop - ac_cv_computed_gotos=no +else case e in #( + e) ac_cv_computed_gotos=no ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_computed_gotos" >&5 printf "%s\n" "$ac_cv_computed_gotos" >&6; } @@ -27244,7 +29263,6 @@ done SRCDIRS="\ Modules \ - Modules/_blake2 \ Modules/_ctypes \ Modules/_decimal \ Modules/_decimal/libmpdec \ @@ -27255,6 +29273,7 @@ SRCDIRS="\ Modules/_sre \ Modules/_testcapi \ Modules/_testinternalcapi \ + Modules/_testlimitedcapi \ Modules/_xxtestfuzz \ Modules/cjkcodecs \ Modules/expat \ @@ -27266,8 +29285,7 @@ SRCDIRS="\ Parser/lexer \ Programs \ Python \ - Python/frozen_modules \ - Python/deepfreeze" + Python/frozen_modules" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for build directories" >&5 printf %s "checking for build directories... " >&6; } for dir in $SRCDIRS; do @@ -27284,8 +29302,8 @@ printf %s "checking for -O2... " >&6; } if test ${ac_cv_compile_o2+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) saved_cflags="$CFLAGS" CFLAGS="-O2" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -27302,12 +29320,14 @@ _ACEOF if ac_fn_c_try_compile "$LINENO" then : ac_cv_compile_o2=yes -else $as_nop - ac_cv_compile_o2=no +else case e in #( + e) ac_cv_compile_o2=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext CFLAGS="$saved_cflags" - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_compile_o2" >&5 printf "%s\n" "$ac_cv_compile_o2" >&6; } @@ -27324,8 +29344,8 @@ fi if test "$cross_compiling" = yes then : have_glibc_memmove_bug=undefined -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include @@ -27347,11 +29367,13 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : have_glibc_memmove_bug=no -else $as_nop - have_glibc_memmove_bug=yes +else case e in #( + e) have_glibc_memmove_bug=yes ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi CFLAGS="$saved_cflags" @@ -27367,8 +29389,8 @@ if test "$ac_cv_gcc_asm_for_x87" = yes; then # Some versions of gcc miscompile inline asm: # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=46491 # http://gcc.gnu.org/ml/gcc/2010-11/msg00366.html - case $CC in - *gcc*) + case $ac_cv_cc_name in + gcc) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for gcc ipa-pure-const bug" >&5 printf %s "checking for gcc ipa-pure-const bug... " >&6; } saved_cflags="$CFLAGS" @@ -27376,8 +29398,8 @@ printf %s "checking for gcc ipa-pure-const bug... " >&6; } if test "$cross_compiling" = yes then : have_ipa_pure_const_bug=undefined -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ __attribute__((noinline)) int @@ -27400,11 +29422,13 @@ _ACEOF if ac_fn_c_try_run "$LINENO" then : have_ipa_pure_const_bug=no -else $as_nop - have_ipa_pure_const_bug=yes +else case e in #( + e) have_ipa_pure_const_bug=yes ;; +esac fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac fi CFLAGS="$saved_cflags" @@ -27427,18 +29451,21 @@ printf %s "checking for ensurepip... " >&6; } if test ${with_ensurepip+y} then : withval=$with_ensurepip; -else $as_nop - +else case e in #( + e) case $ac_sys_system in #( Emscripten) : with_ensurepip=no ;; #( WASI) : with_ensurepip=no ;; #( + iOS) : + with_ensurepip=no ;; #( *) : with_ensurepip=upgrade ;; esac - + ;; +esac fi case $with_ensurepip in #( @@ -27461,8 +29488,8 @@ printf %s "checking if the dirent structure of a d_type field... " >&6; } if test ${ac_cv_dirent_d_type+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -27479,12 +29506,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_dirent_d_type=yes -else $as_nop - ac_cv_dirent_d_type=no +else case e in #( + e) ac_cv_dirent_d_type=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_dirent_d_type" >&5 printf "%s\n" "$ac_cv_dirent_d_type" >&6; } @@ -27504,8 +29533,8 @@ printf %s "checking for the Linux getrandom() syscall... " >&6; } if test ${ac_cv_getrandom_syscall+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -27529,12 +29558,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_getrandom_syscall=yes -else $as_nop - ac_cv_getrandom_syscall=no +else case e in #( + e) ac_cv_getrandom_syscall=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_getrandom_syscall" >&5 printf "%s\n" "$ac_cv_getrandom_syscall" >&6; } @@ -27555,8 +29586,8 @@ printf %s "checking for the getrandom() function... " >&6; } if test ${ac_cv_func_getrandom+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -27578,12 +29609,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_func_getrandom=yes -else $as_nop - ac_cv_func_getrandom=no +else case e in #( + e) ac_cv_func_getrandom=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_getrandom" >&5 printf "%s\n" "$ac_cv_func_getrandom" >&6; } @@ -27611,15 +29644,21 @@ printf %s "checking for library containing shm_open... " >&6; } if test ${ac_cv_search_shm_open+y} then : printf %s "(cached) " >&6 -else $as_nop - ac_func_search_save_LIBS=$LIBS +else case e in #( + e) ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char shm_open (); + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char shm_open (void); int main (void) { @@ -27650,11 +29689,13 @@ done if test ${ac_cv_search_shm_open+y} then : -else $as_nop - ac_cv_search_shm_open=no +else case e in #( + e) ac_cv_search_shm_open=no ;; +esac fi rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS +LIBS=$ac_func_search_save_LIBS ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_shm_open" >&5 printf "%s\n" "$ac_cv_search_shm_open" >&6; } @@ -27682,16 +29723,17 @@ fi for ac_func in shm_open shm_unlink do : - as_ac_var=`printf "%s\n" "ac_cv_func_$ac_func" | $as_tr_sh` + as_ac_var=`printf "%s\n" "ac_cv_func_$ac_func" | sed "$as_sed_sh"` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" if eval test \"x\$"$as_ac_var"\" = x"yes" then : cat >>confdefs.h <<_ACEOF -#define `printf "%s\n" "HAVE_$ac_func" | $as_tr_cpp` 1 +#define `printf "%s\n" "HAVE_$ac_func" | sed "$as_sed_cpp"` 1 _ACEOF have_posix_shmem=yes -else $as_nop - have_posix_shmem=no +else case e in #( + e) have_posix_shmem=no ;; +esac fi done @@ -27720,8 +29762,8 @@ then : ;; esac -else $as_nop - +else case e in #( + e) # if pkg-config is installed and openssl has installed a .pc file, # then use that information and don't search ssldirs if test -n "$ac_tool_prefix"; then @@ -27732,8 +29774,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_PKG_CONFIG+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$PKG_CONFIG"; then +else case e in #( + e) if test -n "$PKG_CONFIG"; then ac_cv_prog_PKG_CONFIG="$PKG_CONFIG" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -27755,7 +29797,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi PKG_CONFIG=$ac_cv_prog_PKG_CONFIG if test -n "$PKG_CONFIG"; then @@ -27777,8 +29820,8 @@ printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_ac_ct_PKG_CONFIG+y} then : printf %s "(cached) " >&6 -else $as_nop - if test -n "$ac_ct_PKG_CONFIG"; then +else case e in #( + e) if test -n "$ac_ct_PKG_CONFIG"; then ac_cv_prog_ac_ct_PKG_CONFIG="$ac_ct_PKG_CONFIG" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -27800,7 +29843,8 @@ done done IFS=$as_save_IFS -fi +fi ;; +esac fi ac_ct_PKG_CONFIG=$ac_cv_prog_ac_ct_PKG_CONFIG if test -n "$ac_ct_PKG_CONFIG"; then @@ -27840,7 +29884,8 @@ fi ssldirs="/usr/local/ssl /usr/lib/ssl /usr/ssl /usr/pkg /usr/local /usr" fi - + ;; +esac fi @@ -27903,12 +29948,13 @@ then : printf "%s\n" "yes" >&6; } have_openssl=yes -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } have_openssl=no - + ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext @@ -27927,15 +29973,16 @@ then : rpath_arg="-Wl,--enable-new-dtags,-rpath=" -else $as_nop - +else case e in #( + e) if test "$ac_sys_system" = "Darwin" then rpath_arg="-Wl,-rpath," else rpath_arg="-Wl,-rpath=" fi - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-openssl-rpath" >&5 @@ -27945,9 +29992,10 @@ printf %s "checking for --with-openssl-rpath... " >&6; } if test ${with_openssl_rpath+y} then : withval=$with_openssl_rpath; -else $as_nop - with_openssl_rpath=no - +else case e in #( + e) with_openssl_rpath=no + ;; +esac fi case $with_openssl_rpath in #( @@ -27973,8 +30021,9 @@ then : OPENSSL_RPATH="$with_openssl_rpath" OPENSSL_LDFLAGS_RPATH="${rpath_arg}$with_openssl_rpath" -else $as_nop - as_fn_error $? "--with-openssl-rpath \"$with_openssl_rpath\" is not a directory" "$LINENO" 5 +else case e in #( + e) as_fn_error $? "--with-openssl-rpath \"$with_openssl_rpath\" is not a directory" "$LINENO" 5 ;; +esac fi ;; @@ -28037,8 +30086,8 @@ printf %s "checking whether OpenSSL provides required ssl module APIs... " >&6; if test ${ac_cv_working_openssl_ssl+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -28068,12 +30117,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_working_openssl_ssl=yes -else $as_nop - ac_cv_working_openssl_ssl=no +else case e in #( + e) ac_cv_working_openssl_ssl=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_working_openssl_ssl" >&5 printf "%s\n" "$ac_cv_working_openssl_ssl" >&6; } @@ -28100,8 +30151,8 @@ printf %s "checking whether OpenSSL provides required hashlib module APIs... " > if test ${ac_cv_working_openssl_hashlib+y} then : printf %s "(cached) " >&6 -else $as_nop - +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -28128,12 +30179,14 @@ _ACEOF if ac_fn_c_try_link "$LINENO" then : ac_cv_working_openssl_hashlib=yes -else $as_nop - ac_cv_working_openssl_hashlib=no +else case e in #( + e) ac_cv_working_openssl_hashlib=no ;; +esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext - + ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_working_openssl_hashlib" >&5 printf "%s\n" "$ac_cv_working_openssl_hashlib" >&6; } @@ -28175,21 +30228,19 @@ case "$withval" in ;; esac -else $as_nop - +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: python" >&5 printf "%s\n" "python" >&6; } printf "%s\n" "#define PY_SSL_DEFAULT_CIPHERS 1" >>confdefs.h - + ;; +esac fi # builtin hash modules default_hashlib_hashes="md5,sha1,sha2,sha3,blake2" - -printf "%s\n" "#define PY_BUILTIN_HASHLIB_HASHES /**/" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-builtin-hashlib-hashes" >&5 printf %s "checking for --with-builtin-hashlib-hashes... " >&6; } @@ -28207,13 +30258,15 @@ then : ;; esac -else $as_nop - with_builtin_hashlib_hashes=$default_hashlib_hashes +else case e in #( + e) with_builtin_hashlib_hashes=$default_hashlib_hashes ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_builtin_hashlib_hashes" >&5 printf "%s\n" "$with_builtin_hashlib_hashes" >&6; } + printf "%s\n" "#define PY_BUILTIN_HASHLIB_HASHES \"$with_builtin_hashlib_hashes\"" >>confdefs.h @@ -28238,88 +30291,6 @@ esac done IFS=$as_save_IFS -if test "x$with_builtin_blake2" = xyes -then : - - -pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libb2" >&5 -printf %s "checking for libb2... " >&6; } - -if test -n "$LIBB2_CFLAGS"; then - pkg_cv_LIBB2_CFLAGS="$LIBB2_CFLAGS" - elif test -n "$PKG_CONFIG"; then - if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libb2\""; } >&5 - ($PKG_CONFIG --exists --print-errors "libb2") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then - pkg_cv_LIBB2_CFLAGS=`$PKG_CONFIG --cflags "libb2" 2>/dev/null` - test "x$?" != "x0" && pkg_failed=yes -else - pkg_failed=yes -fi - else - pkg_failed=untried -fi -if test -n "$LIBB2_LIBS"; then - pkg_cv_LIBB2_LIBS="$LIBB2_LIBS" - elif test -n "$PKG_CONFIG"; then - if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libb2\""; } >&5 - ($PKG_CONFIG --exists --print-errors "libb2") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then - pkg_cv_LIBB2_LIBS=`$PKG_CONFIG --libs "libb2" 2>/dev/null` - test "x$?" != "x0" && pkg_failed=yes -else - pkg_failed=yes -fi - else - pkg_failed=untried -fi - - - -if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - -if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then - _pkg_short_errors_supported=yes -else - _pkg_short_errors_supported=no -fi - if test $_pkg_short_errors_supported = yes; then - LIBB2_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libb2" 2>&1` - else - LIBB2_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libb2" 2>&1` - fi - # Put the nasty error message in config.log where it belongs - echo "$LIBB2_PKG_ERRORS" >&5 - - have_libb2=no -elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - have_libb2=no -else - LIBB2_CFLAGS=$pkg_cv_LIBB2_CFLAGS - LIBB2_LIBS=$pkg_cv_LIBB2_LIBS - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } - - have_libb2=yes - -printf "%s\n" "#define HAVE_LIBB2 1" >>confdefs.h - - -fi - -fi - # Check whether to disable test modules. Once set, setup.py will not build # test extension modules and "make install" will not install test suites. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --disable-test-modules" >&5 @@ -28331,20 +30302,14 @@ then : if test "x$enable_test_modules" = xyes then : TEST_MODULES=yes -else $as_nop - TEST_MODULES=no +else case e in #( + e) TEST_MODULES=no ;; +esac fi -else $as_nop - - case $ac_sys_system/$ac_sys_emscripten_target in #( - Emscripten/browser*) : - TEST_MODULES=no ;; #( - *) : - TEST_MODULES=yes - ;; +else case e in #( + e) TEST_MODULES=yes ;; esac - fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $TEST_MODULES" >&5 @@ -28373,12 +30338,8 @@ printf %s "checking whether libatomic is needed by ... " >&6; } if test ${ac_cv_libatomic_needed+y} then : printf %s "(cached) " >&6 -else $as_nop - if test "$cross_compiling" = yes -then : - ac_cv_libatomic_needed=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ // pyatomic.h needs uint64_t and Py_ssize_t types @@ -28418,16 +30379,16 @@ int main() } _ACEOF -if ac_fn_c_try_run "$LINENO" +if ac_fn_c_try_link "$LINENO" then : ac_cv_libatomic_needed=no -else $as_nop - ac_cv_libatomic_needed=yes -fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext +else case e in #( + e) ac_cv_libatomic_needed=yes ;; +esac fi - +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext ;; +esac fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_libatomic_needed" >&5 printf "%s\n" "$ac_cv_libatomic_needed" >&6; } @@ -28440,6 +30401,24 @@ fi CPPFLAGS=$save_CPPFLAGS +# gh-59705: Maximum length in bytes of a thread name +case "$ac_sys_system" in + Linux*) _PYTHREAD_NAME_MAXLEN=15;; # Linux and Android + SunOS*) _PYTHREAD_NAME_MAXLEN=31;; + NetBSD*) _PYTHREAD_NAME_MAXLEN=31;; + Darwin) _PYTHREAD_NAME_MAXLEN=63;; + iOS) _PYTHREAD_NAME_MAXLEN=63;; + FreeBSD*) _PYTHREAD_NAME_MAXLEN=98;; + *) _PYTHREAD_NAME_MAXLEN=;; +esac +if test -n "$_PYTHREAD_NAME_MAXLEN"; then + +printf "%s\n" "#define _PYTHREAD_NAME_MAXLEN $_PYTHREAD_NAME_MAXLEN" >>confdefs.h + +fi + + + # stdlib @@ -28495,7 +30474,7 @@ case $ac_sys_system in #( py_cv_module__scproxy=n/a ;; #( - Emscripten|WASI) : + Emscripten) : @@ -28508,18 +30487,15 @@ case $ac_sys_system in #( py_cv_module__posixsubprocess=n/a py_cv_module__scproxy=n/a py_cv_module__tkinter=n/a - py_cv_module__xxsubinterpreters=n/a - py_cv_module__xxinterpchannels=n/a - py_cv_module__xxinterpqueues=n/a + py_cv_module__interpreters=n/a + py_cv_module__interpchannels=n/a + py_cv_module__interpqueues=n/a py_cv_module_grp=n/a py_cv_module_pwd=n/a py_cv_module_resource=n/a py_cv_module_syslog=n/a py_cv_module_=n/a - case $ac_sys_system/$ac_sys_emscripten_target in #( - Emscripten/browser*) : - py_cv_module_fcntl=n/a @@ -28527,24 +30503,43 @@ case $ac_sys_system in #( py_cv_module_termios=n/a py_cv_module_=n/a - ;; #( - Emscripten/node*) : - ;; #( - WASI/*) : + ;; #( + WASI) : + + + + py_cv_module__curses=n/a + py_cv_module__curses_panel=n/a + py_cv_module__dbm=n/a + py_cv_module__gdbm=n/a + py_cv_module__multiprocessing=n/a + py_cv_module__posixshmem=n/a + py_cv_module__posixsubprocess=n/a + py_cv_module__scproxy=n/a + py_cv_module__tkinter=n/a + py_cv_module__interpreters=n/a + py_cv_module__interpchannels=n/a + py_cv_module__interpqueues=n/a + py_cv_module_grp=n/a + py_cv_module_pwd=n/a + py_cv_module_resource=n/a + py_cv_module_syslog=n/a + py_cv_module_=n/a py_cv_module__ctypes_test=n/a + py_cv_module__testexternalinspection=n/a + py_cv_module__testimportmultiple=n/a + py_cv_module__testmultiphase=n/a + py_cv_module__testsinglephase=n/a py_cv_module_fcntl=n/a py_cv_module_mmap=n/a py_cv_module_termios=n/a + py_cv_module_xxlimited=n/a + py_cv_module_xxlimited_35=n/a py_cv_module_=n/a - - - ;; #( - *) : - ;; -esac + ;; #( *) : @@ -28790,28 +30785,6 @@ then : -fi - - - if test "$py_cv_module__opcode" != "n/a" -then : - py_cv_module__opcode=yes -fi - if test "$py_cv_module__opcode" = yes; then - MODULE__OPCODE_TRUE= - MODULE__OPCODE_FALSE='#' -else - MODULE__OPCODE_TRUE='#' - MODULE__OPCODE_FALSE= -fi - - as_fn_append MODULE_BLOCK "MODULE__OPCODE_STATE=$py_cv_module__opcode$as_nl" - if test "x$py_cv_module__opcode" = xyes -then : - - - - fi @@ -28969,20 +30942,20 @@ then : fi - if test "$py_cv_module__xxsubinterpreters" != "n/a" + if test "$py_cv_module__interpreters" != "n/a" then : - py_cv_module__xxsubinterpreters=yes + py_cv_module__interpreters=yes fi - if test "$py_cv_module__xxsubinterpreters" = yes; then - MODULE__XXSUBINTERPRETERS_TRUE= - MODULE__XXSUBINTERPRETERS_FALSE='#' + if test "$py_cv_module__interpreters" = yes; then + MODULE__INTERPRETERS_TRUE= + MODULE__INTERPRETERS_FALSE='#' else - MODULE__XXSUBINTERPRETERS_TRUE='#' - MODULE__XXSUBINTERPRETERS_FALSE= + MODULE__INTERPRETERS_TRUE='#' + MODULE__INTERPRETERS_FALSE= fi - as_fn_append MODULE_BLOCK "MODULE__XXSUBINTERPRETERS_STATE=$py_cv_module__xxsubinterpreters$as_nl" - if test "x$py_cv_module__xxsubinterpreters" = xyes + as_fn_append MODULE_BLOCK "MODULE__INTERPRETERS_STATE=$py_cv_module__interpreters$as_nl" + if test "x$py_cv_module__interpreters" = xyes then : @@ -28991,20 +30964,20 @@ then : fi - if test "$py_cv_module__xxinterpchannels" != "n/a" + if test "$py_cv_module__interpchannels" != "n/a" then : - py_cv_module__xxinterpchannels=yes + py_cv_module__interpchannels=yes fi - if test "$py_cv_module__xxinterpchannels" = yes; then - MODULE__XXINTERPCHANNELS_TRUE= - MODULE__XXINTERPCHANNELS_FALSE='#' + if test "$py_cv_module__interpchannels" = yes; then + MODULE__INTERPCHANNELS_TRUE= + MODULE__INTERPCHANNELS_FALSE='#' else - MODULE__XXINTERPCHANNELS_TRUE='#' - MODULE__XXINTERPCHANNELS_FALSE= + MODULE__INTERPCHANNELS_TRUE='#' + MODULE__INTERPCHANNELS_FALSE= fi - as_fn_append MODULE_BLOCK "MODULE__XXINTERPCHANNELS_STATE=$py_cv_module__xxinterpchannels$as_nl" - if test "x$py_cv_module__xxinterpchannels" = xyes + as_fn_append MODULE_BLOCK "MODULE__INTERPCHANNELS_STATE=$py_cv_module__interpchannels$as_nl" + if test "x$py_cv_module__interpchannels" = xyes then : @@ -29013,20 +30986,20 @@ then : fi - if test "$py_cv_module__xxinterpqueues" != "n/a" + if test "$py_cv_module__interpqueues" != "n/a" then : - py_cv_module__xxinterpqueues=yes + py_cv_module__interpqueues=yes fi - if test "$py_cv_module__xxinterpqueues" = yes; then - MODULE__XXINTERPQUEUES_TRUE= - MODULE__XXINTERPQUEUES_FALSE='#' + if test "$py_cv_module__interpqueues" = yes; then + MODULE__INTERPQUEUES_TRUE= + MODULE__INTERPQUEUES_FALSE='#' else - MODULE__XXINTERPQUEUES_TRUE='#' - MODULE__XXINTERPQUEUES_FALSE= + MODULE__INTERPQUEUES_TRUE='#' + MODULE__INTERPQUEUES_FALSE= fi - as_fn_append MODULE_BLOCK "MODULE__XXINTERPQUEUES_STATE=$py_cv_module__xxinterpqueues$as_nl" - if test "x$py_cv_module__xxinterpqueues" = xyes + as_fn_append MODULE_BLOCK "MODULE__INTERPQUEUES_STATE=$py_cv_module__interpqueues$as_nl" + if test "x$py_cv_module__interpqueues" = xyes then : @@ -29068,11 +31041,13 @@ then : if test "$ac_cv_func_sem_unlink" = "yes" then : py_cv_module__multiprocessing=yes -else $as_nop - py_cv_module__multiprocessing=missing +else case e in #( + e) py_cv_module__multiprocessing=missing ;; +esac fi -else $as_nop - py_cv_module__multiprocessing=disabled +else case e in #( + e) py_cv_module__multiprocessing=disabled ;; +esac fi fi @@ -29106,11 +31081,13 @@ then : if test "$have_posix_shmem" = "yes" then : py_cv_module__posixshmem=yes -else $as_nop - py_cv_module__posixshmem=missing +else case e in #( + e) py_cv_module__posixshmem=missing ;; +esac fi -else $as_nop - py_cv_module__posixshmem=disabled +else case e in #( + e) py_cv_module__posixshmem=disabled ;; +esac fi fi @@ -29235,11 +31212,13 @@ then : if test "$ac_cv_header_sys_ioctl_h" = "yes" -a "$ac_cv_header_fcntl_h" = "yes" then : py_cv_module_fcntl=yes -else $as_nop - py_cv_module_fcntl=missing +else case e in #( + e) py_cv_module_fcntl=missing ;; +esac fi -else $as_nop - py_cv_module_fcntl=disabled +else case e in #( + e) py_cv_module_fcntl=disabled ;; +esac fi fi @@ -29273,11 +31252,13 @@ then : if test "$ac_cv_header_sys_mman_h" = "yes" -a "$ac_cv_header_sys_stat_h" = "yes" then : py_cv_module_mmap=yes -else $as_nop - py_cv_module_mmap=missing +else case e in #( + e) py_cv_module_mmap=missing ;; +esac fi -else $as_nop - py_cv_module_mmap=disabled +else case e in #( + e) py_cv_module_mmap=disabled ;; +esac fi fi @@ -29311,11 +31292,13 @@ then : if test "$ac_cv_header_sys_socket_h" = "yes" -a "$ac_cv_header_sys_types_h" = "yes" -a "$ac_cv_header_netinet_in_h" = "yes" then : py_cv_module__socket=yes -else $as_nop - py_cv_module__socket=missing +else case e in #( + e) py_cv_module__socket=missing ;; +esac fi -else $as_nop - py_cv_module__socket=disabled +else case e in #( + e) py_cv_module__socket=disabled ;; +esac fi fi @@ -29324,7 +31307,7 @@ fi then : - + as_fn_append MODULE_BLOCK "MODULE__SOCKET_LDFLAGS=$SOCKET_LIBS$as_nl" fi if test "$py_cv_module__socket" = yes; then @@ -29351,11 +31334,13 @@ then : { test "$ac_cv_func_getgrgid" = "yes" || test "$ac_cv_func_getgrgid_r" = "yes"; } then : py_cv_module_grp=yes -else $as_nop - py_cv_module_grp=missing +else case e in #( + e) py_cv_module_grp=missing ;; +esac fi -else $as_nop - py_cv_module_grp=disabled +else case e in #( + e) py_cv_module_grp=disabled ;; +esac fi fi @@ -29389,11 +31374,13 @@ then : if test "$ac_cv_func_getpwuid" = yes -o "$ac_cv_func_getpwuid_r" = yes then : py_cv_module_pwd=yes -else $as_nop - py_cv_module_pwd=missing +else case e in #( + e) py_cv_module_pwd=missing ;; +esac fi -else $as_nop - py_cv_module_pwd=disabled +else case e in #( + e) py_cv_module_pwd=disabled ;; +esac fi fi @@ -29427,11 +31414,13 @@ then : if test "$ac_cv_header_sys_resource_h" = yes then : py_cv_module_resource=yes -else $as_nop - py_cv_module_resource=missing +else case e in #( + e) py_cv_module_resource=missing ;; +esac fi -else $as_nop - py_cv_module_resource=disabled +else case e in #( + e) py_cv_module_resource=disabled ;; +esac fi fi @@ -29465,11 +31454,13 @@ then : if true then : py_cv_module__scproxy=yes -else $as_nop - py_cv_module__scproxy=missing +else case e in #( + e) py_cv_module__scproxy=missing ;; +esac fi -else $as_nop - py_cv_module__scproxy=disabled +else case e in #( + e) py_cv_module__scproxy=disabled ;; +esac fi fi @@ -29503,11 +31494,13 @@ then : if test "$ac_cv_header_syslog_h" = yes then : py_cv_module_syslog=yes -else $as_nop - py_cv_module_syslog=missing +else case e in #( + e) py_cv_module_syslog=missing ;; +esac fi -else $as_nop - py_cv_module_syslog=disabled +else case e in #( + e) py_cv_module_syslog=disabled ;; +esac fi fi @@ -29541,11 +31534,13 @@ then : if test "$ac_cv_header_termios_h" = yes then : py_cv_module_termios=yes -else $as_nop - py_cv_module_termios=missing +else case e in #( + e) py_cv_module_termios=missing ;; +esac fi -else $as_nop - py_cv_module_termios=disabled +else case e in #( + e) py_cv_module_termios=disabled ;; +esac fi fi @@ -29580,11 +31575,13 @@ then : if test "$ac_cv_header_sys_time_h" = "yes" then : py_cv_module_pyexpat=yes -else $as_nop - py_cv_module_pyexpat=missing +else case e in #( + e) py_cv_module_pyexpat=missing ;; +esac fi -else $as_nop - py_cv_module_pyexpat=disabled +else case e in #( + e) py_cv_module_pyexpat=disabled ;; +esac fi fi @@ -29618,11 +31615,13 @@ then : if true then : py_cv_module__elementtree=yes -else $as_nop - py_cv_module__elementtree=missing +else case e in #( + e) py_cv_module__elementtree=missing ;; +esac fi -else $as_nop - py_cv_module__elementtree=disabled +else case e in #( + e) py_cv_module__elementtree=disabled ;; +esac fi fi @@ -29833,11 +31832,13 @@ then : if true then : py_cv_module__md5=yes -else $as_nop - py_cv_module__md5=missing +else case e in #( + e) py_cv_module__md5=missing ;; +esac fi -else $as_nop - py_cv_module__md5=disabled +else case e in #( + e) py_cv_module__md5=disabled ;; +esac fi fi @@ -29845,7 +31846,7 @@ fi if test "x$py_cv_module__md5" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE$as_nl" + fi @@ -29871,11 +31872,13 @@ then : if true then : py_cv_module__sha1=yes -else $as_nop - py_cv_module__sha1=missing +else case e in #( + e) py_cv_module__sha1=missing ;; +esac fi -else $as_nop - py_cv_module__sha1=disabled +else case e in #( + e) py_cv_module__sha1=disabled ;; +esac fi fi @@ -29883,7 +31886,7 @@ fi if test "x$py_cv_module__sha1" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE$as_nl" + fi @@ -29909,11 +31912,13 @@ then : if true then : py_cv_module__sha2=yes -else $as_nop - py_cv_module__sha2=missing +else case e in #( + e) py_cv_module__sha2=missing ;; +esac fi -else $as_nop - py_cv_module__sha2=disabled +else case e in #( + e) py_cv_module__sha2=disabled ;; +esac fi fi @@ -29921,7 +31926,7 @@ fi if test "x$py_cv_module__sha2" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE$as_nl" + fi @@ -29947,11 +31952,13 @@ then : if true then : py_cv_module__sha3=yes -else $as_nop - py_cv_module__sha3=missing +else case e in #( + e) py_cv_module__sha3=missing ;; +esac fi -else $as_nop - py_cv_module__sha3=disabled +else case e in #( + e) py_cv_module__sha3=disabled ;; +esac fi fi @@ -29985,11 +31992,13 @@ then : if true then : py_cv_module__blake2=yes -else $as_nop - py_cv_module__blake2=missing +else case e in #( + e) py_cv_module__blake2=missing ;; +esac fi -else $as_nop - py_cv_module__blake2=disabled +else case e in #( + e) py_cv_module__blake2=disabled ;; +esac fi fi @@ -29997,8 +32006,8 @@ fi if test "x$py_cv_module__blake2" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__BLAKE2_CFLAGS=$LIBB2_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__BLAKE2_LDFLAGS=$LIBB2_LIBS$as_nl" + + fi if test "$py_cv_module__blake2" = yes; then @@ -30013,6 +32022,161 @@ fi printf "%s\n" "$py_cv_module__blake2" >&6; } +LIBHACL_CFLAGS='-I$(srcdir)/Modules/_hacl -I$(srcdir)/Modules/_hacl/include -D_BSD_SOURCE -D_DEFAULT_SOURCE $(PY_STDMODULE_CFLAGS) $(CCSHARED)' +case "$ac_sys_system" in + Linux*) + if test "$ac_cv_func_explicit_bzero" = "no"; then + LIBHACL_CFLAGS="$LIBHACL_CFLAGS -DLINUX_NO_EXPLICIT_BZERO" + fi + ;; +esac + + +# The SIMD files use aligned_alloc, which is not available on older versions of +# Android. +# The *mmintrin.h headers are x86-family-specific, so can't be used on WASI. +if test "$ac_sys_system" != "Linux-android" -a "$ac_sys_system" != "WASI" || test "$ANDROID_API_LEVEL" -ge 28; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -msse -msse2 -msse3 -msse4.1 -msse4.2" >&5 +printf %s "checking whether C compiler accepts -msse -msse2 -msse3 -msse4.1 -msse4.2... " >&6; } +if test ${ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -msse -msse2 -msse3 -msse4.1 -msse4.2" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2=yes +else case e in #( + e) ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2" >&6; } +if test "x$ax_cv_check_cflags__Werror__msse__msse2__msse3__msse4_1__msse4_2" = xyes +then : + + LIBHACL_SIMD128_FLAGS="-msse -msse2 -msse3 -msse4.1 -msse4.2" + + +printf "%s\n" "#define HACL_CAN_COMPILE_SIMD128 1" >>confdefs.h + + + # macOS universal2 builds *support* the -msse etc flags because they're + # available on x86_64. However, performance of the HACL SIMD128 implementation + # isn't great, so it's disabled on ARM64. + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for HACL* SIMD128 implementation" >&5 +printf %s "checking for HACL* SIMD128 implementation... " >&6; } + if test "$UNIVERSAL_ARCHS" == "universal2"; then + LIBHACL_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: universal2" >&5 +printf "%s\n" "universal2" >&6; } + else + LIBHACL_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128.o" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: standard" >&5 +printf "%s\n" "standard" >&6; } + fi + + +else case e in #( + e) : ;; +esac +fi + +fi + + + +# The SIMD files use aligned_alloc, which is not available on older versions of +# Android. +# The *mmintrin.h headers are x86-family-specific, so can't be used on WASI. +# +# Although AVX support is not guaranteed on Android +# (https://developer.android.com/ndk/guides/abis#86-64), this is safe because we do a +# runtime CPUID check. +if test "$ac_sys_system" != "Linux-android" -a "$ac_sys_system" != "WASI" || test "$ANDROID_API_LEVEL" -ge 28; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -mavx2" >&5 +printf %s "checking whether C compiler accepts -mavx2... " >&6; } +if test ${ax_cv_check_cflags__Werror__mavx2+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -mavx2" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__mavx2=yes +else case e in #( + e) ax_cv_check_cflags__Werror__mavx2=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__mavx2" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__mavx2" >&6; } +if test "x$ax_cv_check_cflags__Werror__mavx2" = xyes +then : + + LIBHACL_SIMD256_FLAGS="-mavx2" + +printf "%s\n" "#define HACL_CAN_COMPILE_SIMD256 1" >>confdefs.h + + + # macOS universal2 builds *support* the -mavx2 compiler flag because it's + # available on x86_64; but the HACL SIMD256 build then fails because the + # implementation requires symbols that aren't available on ARM64. Use a + # wrapped implementation if we're building for universal2. + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for HACL* SIMD256 implementation" >&5 +printf %s "checking for HACL* SIMD256 implementation... " >&6; } + if test "$UNIVERSAL_ARCHS" == "universal2"; then + LIBHACL_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: universal2" >&5 +printf "%s\n" "universal2" >&6; } + else + LIBHACL_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256.o" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: standard" >&5 +printf "%s\n" "standard" >&6; } + fi + +else case e in #( + e) : ;; +esac +fi + +fi + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _ctypes" >&5 printf %s "checking for stdlib extension module _ctypes... " >&6; } @@ -30024,11 +32188,13 @@ then : if test "$have_libffi" = yes then : py_cv_module__ctypes=yes -else $as_nop - py_cv_module__ctypes=missing +else case e in #( + e) py_cv_module__ctypes=missing ;; +esac fi -else $as_nop - py_cv_module__ctypes=disabled +else case e in #( + e) py_cv_module__ctypes=disabled ;; +esac fi fi @@ -30059,14 +32225,16 @@ then : if true then : - if test "$have_curses" != "no" + if test "$have_curses" = "yes" then : py_cv_module__curses=yes -else $as_nop - py_cv_module__curses=missing +else case e in #( + e) py_cv_module__curses=missing ;; +esac fi -else $as_nop - py_cv_module__curses=disabled +else case e in #( + e) py_cv_module__curses=disabled ;; +esac fi fi @@ -30098,14 +32266,16 @@ then : if true then : - if test "$have_panel" != "no" + if test "$have_curses" = "yes" && test "$have_panel" = "yes" then : py_cv_module__curses_panel=yes -else $as_nop - py_cv_module__curses_panel=missing +else case e in #( + e) py_cv_module__curses_panel=missing ;; +esac fi -else $as_nop - py_cv_module__curses_panel=disabled +else case e in #( + e) py_cv_module__curses_panel=disabled ;; +esac fi fi @@ -30137,14 +32307,16 @@ then : if true then : - if true + if test "$have_mpdec" = "yes" then : py_cv_module__decimal=yes -else $as_nop - py_cv_module__decimal=missing +else case e in #( + e) py_cv_module__decimal=missing ;; +esac fi -else $as_nop - py_cv_module__decimal=disabled +else case e in #( + e) py_cv_module__decimal=disabled ;; +esac fi fi @@ -30153,7 +32325,7 @@ fi then : as_fn_append MODULE_BLOCK "MODULE__DECIMAL_CFLAGS=$LIBMPDEC_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__DECIMAL_LDFLAGS=$LIBMPDEC_LDFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__DECIMAL_LDFLAGS=$LIBMPDEC_LIBS$as_nl" fi if test "$py_cv_module__decimal" = yes; then @@ -30178,11 +32350,13 @@ then : if test "$have_dbm" != "no" then : py_cv_module__dbm=yes -else $as_nop - py_cv_module__dbm=missing +else case e in #( + e) py_cv_module__dbm=missing ;; +esac fi -else $as_nop - py_cv_module__dbm=disabled +else case e in #( + e) py_cv_module__dbm=disabled ;; +esac fi fi @@ -30216,11 +32390,13 @@ then : if test "$have_gdbm" = yes then : py_cv_module__gdbm=yes -else $as_nop - py_cv_module__gdbm=missing +else case e in #( + e) py_cv_module__gdbm=missing ;; +esac fi -else $as_nop - py_cv_module__gdbm=disabled +else case e in #( + e) py_cv_module__gdbm=disabled ;; +esac fi fi @@ -30254,11 +32430,13 @@ then : if test "$with_readline" != "no" then : py_cv_module_readline=yes -else $as_nop - py_cv_module_readline=missing +else case e in #( + e) py_cv_module_readline=missing ;; +esac fi -else $as_nop - py_cv_module_readline=disabled +else case e in #( + e) py_cv_module_readline=disabled ;; +esac fi fi @@ -30292,11 +32470,13 @@ then : if test "$have_supported_sqlite3" = "yes" then : py_cv_module__sqlite3=yes -else $as_nop - py_cv_module__sqlite3=missing +else case e in #( + e) py_cv_module__sqlite3=missing ;; +esac fi -else $as_nop - py_cv_module__sqlite3=disabled +else case e in #( + e) py_cv_module__sqlite3=disabled ;; +esac fi fi @@ -30330,11 +32510,13 @@ then : if test "$have_tcltk" = "yes" then : py_cv_module__tkinter=yes -else $as_nop - py_cv_module__tkinter=missing +else case e in #( + e) py_cv_module__tkinter=missing ;; +esac fi -else $as_nop - py_cv_module__tkinter=disabled +else case e in #( + e) py_cv_module__tkinter=disabled ;; +esac fi fi @@ -30368,11 +32550,13 @@ then : if test "$have_uuid" = "yes" then : py_cv_module__uuid=yes -else $as_nop - py_cv_module__uuid=missing +else case e in #( + e) py_cv_module__uuid=missing ;; +esac fi -else $as_nop - py_cv_module__uuid=disabled +else case e in #( + e) py_cv_module__uuid=disabled ;; +esac fi fi @@ -30407,11 +32591,13 @@ then : if test "$have_zlib" = yes then : py_cv_module_zlib=yes -else $as_nop - py_cv_module_zlib=missing +else case e in #( + e) py_cv_module_zlib=missing ;; +esac fi -else $as_nop - py_cv_module_zlib=disabled +else case e in #( + e) py_cv_module_zlib=disabled ;; +esac fi fi @@ -30467,11 +32653,13 @@ then : if test "$have_bzip2" = yes then : py_cv_module__bz2=yes -else $as_nop - py_cv_module__bz2=missing +else case e in #( + e) py_cv_module__bz2=missing ;; +esac fi -else $as_nop - py_cv_module__bz2=disabled +else case e in #( + e) py_cv_module__bz2=disabled ;; +esac fi fi @@ -30505,11 +32693,13 @@ then : if test "$have_liblzma" = yes then : py_cv_module__lzma=yes -else $as_nop - py_cv_module__lzma=missing +else case e in #( + e) py_cv_module__lzma=missing ;; +esac fi -else $as_nop - py_cv_module__lzma=disabled +else case e in #( + e) py_cv_module__lzma=disabled ;; +esac fi fi @@ -30544,11 +32734,13 @@ then : if test "$ac_cv_working_openssl_ssl" = yes then : py_cv_module__ssl=yes -else $as_nop - py_cv_module__ssl=missing +else case e in #( + e) py_cv_module__ssl=missing ;; +esac fi -else $as_nop - py_cv_module__ssl=disabled +else case e in #( + e) py_cv_module__ssl=disabled ;; +esac fi fi @@ -30582,11 +32774,13 @@ then : if test "$ac_cv_working_openssl_hashlib" = yes then : py_cv_module__hashlib=yes -else $as_nop - py_cv_module__hashlib=missing +else case e in #( + e) py_cv_module__hashlib=missing ;; +esac fi -else $as_nop - py_cv_module__hashlib=disabled +else case e in #( + e) py_cv_module__hashlib=disabled ;; +esac fi fi @@ -30621,11 +32815,13 @@ then : if true then : py_cv_module__testcapi=yes -else $as_nop - py_cv_module__testcapi=missing +else case e in #( + e) py_cv_module__testcapi=missing ;; +esac fi -else $as_nop - py_cv_module__testcapi=disabled +else case e in #( + e) py_cv_module__testcapi=disabled ;; +esac fi fi @@ -30659,11 +32855,13 @@ then : if true then : py_cv_module__testclinic=yes -else $as_nop - py_cv_module__testclinic=missing +else case e in #( + e) py_cv_module__testclinic=missing ;; +esac fi -else $as_nop - py_cv_module__testclinic=disabled +else case e in #( + e) py_cv_module__testclinic=disabled ;; +esac fi fi @@ -30697,11 +32895,13 @@ then : if true then : py_cv_module__testclinic_limited=yes -else $as_nop - py_cv_module__testclinic_limited=missing +else case e in #( + e) py_cv_module__testclinic_limited=missing ;; +esac fi -else $as_nop - py_cv_module__testclinic_limited=disabled +else case e in #( + e) py_cv_module__testclinic_limited=disabled ;; +esac fi fi @@ -30725,6 +32925,46 @@ fi printf "%s\n" "$py_cv_module__testclinic_limited" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testlimitedcapi" >&5 +printf %s "checking for stdlib extension module _testlimitedcapi... " >&6; } + if test "$py_cv_module__testlimitedcapi" != "n/a" +then : + + if test "$TEST_MODULES" = yes +then : + if true +then : + py_cv_module__testlimitedcapi=yes +else case e in #( + e) py_cv_module__testlimitedcapi=missing ;; +esac +fi +else case e in #( + e) py_cv_module__testlimitedcapi=disabled ;; +esac +fi + +fi + as_fn_append MODULE_BLOCK "MODULE__TESTLIMITEDCAPI_STATE=$py_cv_module__testlimitedcapi$as_nl" + if test "x$py_cv_module__testlimitedcapi" = xyes +then : + + + + +fi + if test "$py_cv_module__testlimitedcapi" = yes; then + MODULE__TESTLIMITEDCAPI_TRUE= + MODULE__TESTLIMITEDCAPI_FALSE='#' +else + MODULE__TESTLIMITEDCAPI_TRUE='#' + MODULE__TESTLIMITEDCAPI_FALSE= +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__testlimitedcapi" >&5 +printf "%s\n" "$py_cv_module__testlimitedcapi" >&6; } + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testinternalcapi" >&5 printf %s "checking for stdlib extension module _testinternalcapi... " >&6; } if test "$py_cv_module__testinternalcapi" != "n/a" @@ -30735,11 +32975,13 @@ then : if true then : py_cv_module__testinternalcapi=yes -else $as_nop - py_cv_module__testinternalcapi=missing +else case e in #( + e) py_cv_module__testinternalcapi=missing ;; +esac fi -else $as_nop - py_cv_module__testinternalcapi=disabled +else case e in #( + e) py_cv_module__testinternalcapi=disabled ;; +esac fi fi @@ -30773,11 +33015,13 @@ then : if true then : py_cv_module__testbuffer=yes -else $as_nop - py_cv_module__testbuffer=missing +else case e in #( + e) py_cv_module__testbuffer=missing ;; +esac fi -else $as_nop - py_cv_module__testbuffer=disabled +else case e in #( + e) py_cv_module__testbuffer=disabled ;; +esac fi fi @@ -30811,11 +33055,13 @@ then : if test "$ac_cv_func_dlopen" = yes then : py_cv_module__testimportmultiple=yes -else $as_nop - py_cv_module__testimportmultiple=missing +else case e in #( + e) py_cv_module__testimportmultiple=missing ;; +esac fi -else $as_nop - py_cv_module__testimportmultiple=disabled +else case e in #( + e) py_cv_module__testimportmultiple=disabled ;; +esac fi fi @@ -30849,11 +33095,13 @@ then : if test "$ac_cv_func_dlopen" = yes then : py_cv_module__testmultiphase=yes -else $as_nop - py_cv_module__testmultiphase=missing +else case e in #( + e) py_cv_module__testmultiphase=missing ;; +esac fi -else $as_nop - py_cv_module__testmultiphase=disabled +else case e in #( + e) py_cv_module__testmultiphase=disabled ;; +esac fi fi @@ -30877,6 +33125,46 @@ fi printf "%s\n" "$py_cv_module__testmultiphase" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testsinglephase" >&5 +printf %s "checking for stdlib extension module _testsinglephase... " >&6; } + if test "$py_cv_module__testsinglephase" != "n/a" +then : + + if test "$TEST_MODULES" = yes +then : + if test "$ac_cv_func_dlopen" = yes +then : + py_cv_module__testsinglephase=yes +else case e in #( + e) py_cv_module__testsinglephase=missing ;; +esac +fi +else case e in #( + e) py_cv_module__testsinglephase=disabled ;; +esac +fi + +fi + as_fn_append MODULE_BLOCK "MODULE__TESTSINGLEPHASE_STATE=$py_cv_module__testsinglephase$as_nl" + if test "x$py_cv_module__testsinglephase" = xyes +then : + + + + +fi + if test "$py_cv_module__testsinglephase" = yes; then + MODULE__TESTSINGLEPHASE_TRUE= + MODULE__TESTSINGLEPHASE_FALSE='#' +else + MODULE__TESTSINGLEPHASE_TRUE='#' + MODULE__TESTSINGLEPHASE_FALSE= +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__testsinglephase" >&5 +printf "%s\n" "$py_cv_module__testsinglephase" >&6; } + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testexternalinspection" >&5 printf %s "checking for stdlib extension module _testexternalinspection... " >&6; } if test "$py_cv_module__testexternalinspection" != "n/a" @@ -30887,11 +33175,13 @@ then : if true then : py_cv_module__testexternalinspection=yes -else $as_nop - py_cv_module__testexternalinspection=missing +else case e in #( + e) py_cv_module__testexternalinspection=missing ;; +esac fi -else $as_nop - py_cv_module__testexternalinspection=disabled +else case e in #( + e) py_cv_module__testexternalinspection=disabled ;; +esac fi fi @@ -30925,11 +33215,13 @@ then : if true then : py_cv_module_xxsubtype=yes -else $as_nop - py_cv_module_xxsubtype=missing +else case e in #( + e) py_cv_module_xxsubtype=missing ;; +esac fi -else $as_nop - py_cv_module_xxsubtype=disabled +else case e in #( + e) py_cv_module_xxsubtype=disabled ;; +esac fi fi @@ -30963,11 +33255,13 @@ then : if true then : py_cv_module__xxtestfuzz=yes -else $as_nop - py_cv_module__xxtestfuzz=missing +else case e in #( + e) py_cv_module__xxtestfuzz=missing ;; +esac fi -else $as_nop - py_cv_module__xxtestfuzz=disabled +else case e in #( + e) py_cv_module__xxtestfuzz=disabled ;; +esac fi fi @@ -31001,11 +33295,13 @@ then : if test "$have_libffi" = yes -a "$ac_cv_func_dlopen" = yes then : py_cv_module__ctypes_test=yes -else $as_nop - py_cv_module__ctypes_test=missing +else case e in #( + e) py_cv_module__ctypes_test=missing ;; +esac fi -else $as_nop - py_cv_module__ctypes_test=disabled +else case e in #( + e) py_cv_module__ctypes_test=disabled ;; +esac fi fi @@ -31013,8 +33309,8 @@ fi if test "x$py_cv_module__ctypes_test" = xyes then : - - as_fn_append MODULE_BLOCK "MODULE__CTYPES_TEST_LDFLAGS=$LIBM$as_nl" + as_fn_append MODULE_BLOCK "MODULE__CTYPES_TEST_CFLAGS=$LIBFFI_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__CTYPES_TEST_LDFLAGS=$LIBFFI_LIBS $LIBM$as_nl" fi if test "$py_cv_module__ctypes_test" = yes; then @@ -31035,16 +33331,18 @@ printf %s "checking for stdlib extension module xxlimited... " >&6; } if test "$py_cv_module_xxlimited" != "n/a" then : - if true + if test "$TEST_MODULES" = yes then : if test "$ac_cv_func_dlopen" = yes then : py_cv_module_xxlimited=yes -else $as_nop - py_cv_module_xxlimited=missing +else case e in #( + e) py_cv_module_xxlimited=missing ;; +esac fi -else $as_nop - py_cv_module_xxlimited=disabled +else case e in #( + e) py_cv_module_xxlimited=disabled ;; +esac fi fi @@ -31073,16 +33371,18 @@ printf %s "checking for stdlib extension module xxlimited_35... " >&6; } if test "$py_cv_module_xxlimited_35" != "n/a" then : - if true + if test "$TEST_MODULES" = yes then : if test "$ac_cv_func_dlopen" = yes then : py_cv_module_xxlimited_35=yes -else $as_nop - py_cv_module_xxlimited_35=missing +else case e in #( + e) py_cv_module_xxlimited_35=missing ;; +esac fi -else $as_nop - py_cv_module_xxlimited_35=disabled +else case e in #( + e) py_cv_module_xxlimited_35=disabled ;; +esac fi fi @@ -31127,8 +33427,8 @@ cat >confcache <<\_ACEOF # config.status only pays attention to the cache file if you give it # the --recheck option to rerun configure. # -# `ac_cv_env_foo' variables (set or unset) will be overridden when -# loading this file, other *unset* `ac_cv_foo' will be assigned the +# 'ac_cv_env_foo' variables (set or unset) will be overridden when +# loading this file, other *unset* 'ac_cv_foo' will be assigned the # following values. _ACEOF @@ -31158,14 +33458,14 @@ printf "%s\n" "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} (set) 2>&1 | case $as_nl`(ac_space=' '; set) 2>&1` in #( *${as_nl}ac_space=\ *) - # `set' does not quote correctly, so add quotes: double-quote + # 'set' does not quote correctly, so add quotes: double-quote # substitution turns \\\\ into \\, and sed turns \\ into \. sed -n \ "s/'/'\\\\''/g; s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p" ;; #( *) - # `set' quotes correctly as required by POSIX, so do not add quotes. + # 'set' quotes correctly as required by POSIX, so do not add quotes. sed -n "/^[_$as_cr_alnum]*_cv_[_$as_cr_alnum]*=/p" ;; esac | @@ -31267,10 +33567,6 @@ if test -z "${MODULE__LSPROF_TRUE}" && test -z "${MODULE__LSPROF_FALSE}"; then as_fn_error $? "conditional \"MODULE__LSPROF\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${MODULE__OPCODE_TRUE}" && test -z "${MODULE__OPCODE_FALSE}"; then - as_fn_error $? "conditional \"MODULE__OPCODE\" was never defined. -Usually this means the macro was only invoked conditionally." "$LINENO" 5 -fi if test -z "${MODULE__PICKLE_TRUE}" && test -z "${MODULE__PICKLE_FALSE}"; then as_fn_error $? "conditional \"MODULE__PICKLE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 @@ -31299,16 +33595,16 @@ if test -z "${MODULE__TYPING_TRUE}" && test -z "${MODULE__TYPING_FALSE}"; then as_fn_error $? "conditional \"MODULE__TYPING\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${MODULE__XXSUBINTERPRETERS_TRUE}" && test -z "${MODULE__XXSUBINTERPRETERS_FALSE}"; then - as_fn_error $? "conditional \"MODULE__XXSUBINTERPRETERS\" was never defined. +if test -z "${MODULE__INTERPRETERS_TRUE}" && test -z "${MODULE__INTERPRETERS_FALSE}"; then + as_fn_error $? "conditional \"MODULE__INTERPRETERS\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${MODULE__XXINTERPCHANNELS_TRUE}" && test -z "${MODULE__XXINTERPCHANNELS_FALSE}"; then - as_fn_error $? "conditional \"MODULE__XXINTERPCHANNELS\" was never defined. +if test -z "${MODULE__INTERPCHANNELS_TRUE}" && test -z "${MODULE__INTERPCHANNELS_FALSE}"; then + as_fn_error $? "conditional \"MODULE__INTERPCHANNELS\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${MODULE__XXINTERPQUEUES_TRUE}" && test -z "${MODULE__XXINTERPQUEUES_FALSE}"; then - as_fn_error $? "conditional \"MODULE__XXINTERPQUEUES\" was never defined. +if test -z "${MODULE__INTERPQUEUES_TRUE}" && test -z "${MODULE__INTERPQUEUES_FALSE}"; then + as_fn_error $? "conditional \"MODULE__INTERPQUEUES\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${MODULE__ZONEINFO_TRUE}" && test -z "${MODULE__ZONEINFO_FALSE}"; then @@ -31511,6 +33807,10 @@ if test -z "${MODULE__TESTCLINIC_LIMITED_TRUE}" && test -z "${MODULE__TESTCLINIC as_fn_error $? "conditional \"MODULE__TESTCLINIC_LIMITED\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE__TESTLIMITEDCAPI_TRUE}" && test -z "${MODULE__TESTLIMITEDCAPI_FALSE}"; then + as_fn_error $? "conditional \"MODULE__TESTLIMITEDCAPI\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${MODULE__TESTINTERNALCAPI_TRUE}" && test -z "${MODULE__TESTINTERNALCAPI_FALSE}"; then as_fn_error $? "conditional \"MODULE__TESTINTERNALCAPI\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 @@ -31527,6 +33827,10 @@ if test -z "${MODULE__TESTMULTIPHASE_TRUE}" && test -z "${MODULE__TESTMULTIPHASE as_fn_error $? "conditional \"MODULE__TESTMULTIPHASE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE__TESTSINGLEPHASE_TRUE}" && test -z "${MODULE__TESTSINGLEPHASE_FALSE}"; then + as_fn_error $? "conditional \"MODULE__TESTSINGLEPHASE\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${MODULE__TESTEXTERNALINSPECTION_TRUE}" && test -z "${MODULE__TESTEXTERNALINSPECTION_FALSE}"; then as_fn_error $? "conditional \"MODULE__TESTEXTERNALINSPECTION\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 @@ -31580,7 +33884,6 @@ cat >>$CONFIG_STATUS <<\_ASEOF || as_write_fail=1 # Be more Bourne compatible DUALCASE=1; export DUALCASE # for MKS sh -as_nop=: if test ${ZSH_VERSION+y} && (emulate sh) >/dev/null 2>&1 then : emulate sh @@ -31589,12 +33892,13 @@ then : # is contrary to our usage. Disable this feature. alias -g '${1+"$@"}'='"$@"' setopt NO_GLOB_SUBST -else $as_nop - case `(set -o) 2>/dev/null` in #( +else case e in #( + e) case `(set -o) 2>/dev/null` in #( *posix*) : set -o posix ;; #( *) : ;; +esac ;; esac fi @@ -31666,7 +33970,7 @@ IFS=$as_save_IFS ;; esac -# We did not find ourselves, most probably we were run as `sh COMMAND' +# We did not find ourselves, most probably we were run as 'sh COMMAND' # in which case we are not to be found in the path. if test "x$as_myself" = x; then as_myself=$0 @@ -31695,7 +33999,6 @@ as_fn_error () } # as_fn_error - # as_fn_set_status STATUS # ----------------------- # Set $? to STATUS, without forking. @@ -31735,11 +34038,12 @@ then : { eval $1+=\$2 }' -else $as_nop - as_fn_append () +else case e in #( + e) as_fn_append () { eval $1=\$$1\$2 - } + } ;; +esac fi # as_fn_append # as_fn_arith ARG... @@ -31753,11 +34057,12 @@ then : { as_val=$(( $* )) }' -else $as_nop - as_fn_arith () +else case e in #( + e) as_fn_arith () { as_val=`expr "$@" || test $? -eq 1` - } + } ;; +esac fi # as_fn_arith @@ -31840,9 +34145,9 @@ if (echo >conf$$.file) 2>/dev/null; then if ln -s conf$$.file conf$$ 2>/dev/null; then as_ln_s='ln -s' # ... but there are two gotchas: - # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail. - # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable. - # In both cases, we have to default to `cp -pR'. + # 1) On MSYS, both 'ln -s file dir' and 'ln file dir' fail. + # 2) DJGPP < 2.04 has no symlinks; 'ln -s' creates a wrapper executable. + # In both cases, we have to default to 'cp -pR'. ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe || as_ln_s='cp -pR' elif ln conf$$.file conf$$ 2>/dev/null; then @@ -31923,10 +34228,12 @@ as_test_x='test -x' as_executable_p=as_fn_executable_p # Sed expression to map a string onto a valid CPP name. -as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" +as_sed_cpp="y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g" +as_tr_cpp="eval sed '$as_sed_cpp'" # deprecated # Sed expression to map a string onto a valid variable name. -as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'" +as_sed_sh="y%*+%pp%;s%[^_$as_cr_alnum]%_%g" +as_tr_sh="eval sed '$as_sed_sh'" # deprecated exec 6>&1 @@ -31941,8 +34248,8 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by python $as_me 3.13, which was -generated by GNU Autoconf 2.71. Invocation command line was +This file was extended by python $as_me 3.14, which was +generated by GNU Autoconf 2.72. Invocation command line was CONFIG_FILES = $CONFIG_FILES CONFIG_HEADERS = $CONFIG_HEADERS @@ -31973,7 +34280,7 @@ _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 ac_cs_usage="\ -\`$as_me' instantiates files and other configuration actions +'$as_me' instantiates files and other configuration actions from templates according to the current configuration. Unless the files and actions are specified as TAGs, all are instantiated by default. @@ -32005,11 +34312,11 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\ -python config.status 3.13 -configured by $0, generated by GNU Autoconf 2.71, +python config.status 3.14 +configured by $0, generated by GNU Autoconf 2.72, with options \\"\$ac_cs_config\\" -Copyright (C) 2021 Free Software Foundation, Inc. +Copyright (C) 2023 Free Software Foundation, Inc. This config.status script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it." @@ -32070,8 +34377,8 @@ do ac_need_defaults=false;; --he | --h) # Conflict between --help and --header - as_fn_error $? "ambiguous option: \`$1' -Try \`$0 --help' for more information.";; + as_fn_error $? "ambiguous option: '$1' +Try '$0 --help' for more information.";; --help | --hel | -h ) printf "%s\n" "$ac_cs_usage"; exit ;; -q | -quiet | --quiet | --quie | --qui | --qu | --q \ @@ -32079,8 +34386,8 @@ Try \`$0 --help' for more information.";; ac_cs_silent=: ;; # This is an error. - -*) as_fn_error $? "unrecognized option: \`$1' -Try \`$0 --help' for more information." ;; + -*) as_fn_error $? "unrecognized option: '$1' +Try '$0 --help' for more information." ;; *) as_fn_append ac_config_targets " $1" ac_need_defaults=false ;; @@ -32142,7 +34449,7 @@ do "Modules/Setup.stdlib") CONFIG_FILES="$CONFIG_FILES Modules/Setup.stdlib" ;; "Modules/ld_so_aix") CONFIG_FILES="$CONFIG_FILES Modules/ld_so_aix" ;; - *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;; + *) as_fn_error $? "invalid argument: '$ac_config_target'" "$LINENO" 5;; esac done @@ -32161,7 +34468,7 @@ fi # creating and moving files from /tmp can sometimes cause problems. # Hook for its removal unless debugging. # Note that there is a small window in which the directory will not be cleaned: -# after its creation but before its name has been assigned to `$tmp'. +# after its creation but before its name has been assigned to '$tmp'. $debug || { tmp= ac_tmp= @@ -32185,7 +34492,7 @@ ac_tmp=$tmp # Set up the scripts for CONFIG_FILES section. # No need to generate them if there are no CONFIG_FILES. -# This happens for instance with `./config.status config.h'. +# This happens for instance with './config.status config.h'. if test -n "$CONFIG_FILES"; then @@ -32343,13 +34650,13 @@ fi # test -n "$CONFIG_FILES" # Set up the scripts for CONFIG_HEADERS section. # No need to generate them if there are no CONFIG_HEADERS. -# This happens for instance with `./config.status Makefile'. +# This happens for instance with './config.status Makefile'. if test -n "$CONFIG_HEADERS"; then cat >"$ac_tmp/defines.awk" <<\_ACAWK || BEGIN { _ACEOF -# Transform confdefs.h into an awk script `defines.awk', embedded as +# Transform confdefs.h into an awk script 'defines.awk', embedded as # here-document in config.status, that substitutes the proper values into # config.h.in to produce config.h. @@ -32459,7 +34766,7 @@ do esac case $ac_mode$ac_tag in :[FHL]*:*);; - :L* | :C*:*) as_fn_error $? "invalid tag \`$ac_tag'" "$LINENO" 5;; + :L* | :C*:*) as_fn_error $? "invalid tag '$ac_tag'" "$LINENO" 5;; :[FH]-) ac_tag=-:-;; :[FH]*) ac_tag=$ac_tag:$ac_tag.in;; esac @@ -32481,19 +34788,19 @@ do -) ac_f="$ac_tmp/stdin";; *) # Look for the file first in the build tree, then in the source tree # (if the path is not absolute). The absolute path cannot be DOS-style, - # because $ac_f cannot contain `:'. + # because $ac_f cannot contain ':'. test -f "$ac_f" || case $ac_f in [\\/$]*) false;; *) test -f "$srcdir/$ac_f" && ac_f="$srcdir/$ac_f";; esac || - as_fn_error 1 "cannot find input file: \`$ac_f'" "$LINENO" 5;; + as_fn_error 1 "cannot find input file: '$ac_f'" "$LINENO" 5;; esac case $ac_f in *\'*) ac_f=`printf "%s\n" "$ac_f" | sed "s/'/'\\\\\\\\''/g"`;; esac as_fn_append ac_file_inputs " '$ac_f'" done - # Let's still pretend it is `configure' which instantiates (i.e., don't + # Let's still pretend it is 'configure' which instantiates (i.e., don't # use $as_me), people would be surprised to read: # /* config.h. Generated by config.status. */ configure_input='Generated from '` @@ -32626,7 +34933,7 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 esac _ACEOF -# Neutralize VPATH when `$srcdir' = `.'. +# Neutralize VPATH when '$srcdir' = '.'. # Shell code in configure.ac might set extrasub. # FIXME: do we really want to maintain this feature? cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 @@ -32657,9 +34964,9 @@ test -z "$ac_datarootdir_hack$ac_datarootdir_seen" && { ac_out=`sed -n '/\${datarootdir}/p' "$ac_tmp/out"`; test -n "$ac_out"; } && { ac_out=`sed -n '/^[ ]*datarootdir[ ]*:*=/p' \ "$ac_tmp/out"`; test -z "$ac_out"; } && - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file contains a reference to the variable \`datarootdir' + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file contains a reference to the variable 'datarootdir' which seems to be undefined. Please make sure it is defined" >&5 -printf "%s\n" "$as_me: WARNING: $ac_file contains a reference to the variable \`datarootdir' +printf "%s\n" "$as_me: WARNING: $ac_file contains a reference to the variable 'datarootdir' which seems to be undefined. Please make sure it is defined" >&2;} rm -f "$ac_tmp/stdin" diff --git a/configure.ac b/configure.ac index 103c24962b7b42..f89a0801948ca5 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ dnl ************************************************************ dnl * Please run autoreconf -ivf -Werror to test your changes! * dnl ************************************************************ dnl -dnl Python's configure script requires autoconf 2.71, autoconf-archive, +dnl Python's configure script requires autoconf 2.72, autoconf-archive, dnl aclocal 1.16, and pkg-config. dnl dnl It is recommended to use the Tools/build/regen-configure.sh shell script @@ -10,9 +10,9 @@ dnl to regenerate the configure script. dnl # Set VERSION so we only need to edit in one place (i.e., here) -m4_define([PYTHON_VERSION], [3.13]) +m4_define([PYTHON_VERSION], [3.14]) -AC_PREREQ([2.71]) +AC_PREREQ([2.72]) AC_INIT([python],[PYTHON_VERSION],[https://github.com/python/cpython/issues/]) @@ -164,7 +164,7 @@ AC_ARG_WITH([build-python], dnl Build Python interpreter is used for regeneration and freezing. ac_cv_prog_PYTHON_FOR_REGEN=$with_build_python PYTHON_FOR_FREEZE="$with_build_python" - PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$with_build_python + PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) _PYTHON_SYSCONFIGDATA_PATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`) '$with_build_python AC_MSG_RESULT([$with_build_python]) ], [ AS_VAR_IF([cross_compiling], [yes], @@ -330,13 +330,16 @@ then *-apple-ios*) ac_sys_system=iOS ;; + *-*-darwin*) + ac_sys_system=Darwin + ;; *-*-vxworks*) ac_sys_system=VxWorks ;; *-*-emscripten) ac_sys_system=Emscripten ;; - *-*-wasi) + *-*-wasi*) ac_sys_system=WASI ;; *) @@ -362,6 +365,7 @@ then case $MACHDEP in aix*) MACHDEP="aix";; + linux-android*) MACHDEP="android";; linux*) MACHDEP="linux";; cygwin*) MACHDEP="cygwin";; darwin*) MACHDEP="darwin";; @@ -378,6 +382,71 @@ then fi AC_MSG_RESULT(["$MACHDEP"]) +dnl For cross compilation, we distinguish between "prefix" (where we install the +dnl files) and "host_prefix" (where we expect to find the files at runtime) + +if test -z "$host_prefix"; then + AS_CASE([$ac_sys_system], + [Emscripten], [host_prefix=/], + [host_prefix='${prefix}'] + ) +fi +AC_SUBST([host_prefix]) + +if test -z "$host_exec_prefix"; then + AS_CASE([$ac_sys_system], + [Emscripten], [host_exec_prefix=$host_prefix], + [host_exec_prefix='${exec_prefix}'] + ) +fi +AC_SUBST([host_exec_prefix]) + +# On cross-compile builds, configure will look for a host-specific compiler by +# prepending the user-provided host triple to the required binary name. +# +# On iOS, this results in binaries like "arm64-apple-ios13.0-simulator-gcc", +# which isn't a binary that exists, and isn't very convenient, as it contains the +# iOS version. As the default cross-compiler name won't exist, configure falls +# back to gcc, which *definitely* won't work. We're providing wrapper scripts for +# these tools; the binary names of these scripts are better defaults than "gcc". +# This only requires that the user put the platform scripts folder (e.g., +# "iOS/Resources/bin") in their path, rather than defining platform-specific +# names/paths for AR, CC, CPP, and CXX explicitly; and if the user forgets to +# either put the platform scripts folder in the path, or specify CC etc, +# configure will fail. +if test -z "$AR"; then + case "$host" in + aarch64-apple-ios*-simulator) AR=arm64-apple-ios-simulator-ar ;; + aarch64-apple-ios*) AR=arm64-apple-ios-ar ;; + x86_64-apple-ios*-simulator) AR=x86_64-apple-ios-simulator-ar ;; + *) + esac +fi +if test -z "$CC"; then + case "$host" in + aarch64-apple-ios*-simulator) CC=arm64-apple-ios-simulator-clang ;; + aarch64-apple-ios*) CC=arm64-apple-ios-clang ;; + x86_64-apple-ios*-simulator) CC=x86_64-apple-ios-simulator-clang ;; + *) + esac +fi +if test -z "$CPP"; then + case "$host" in + aarch64-apple-ios*-simulator) CPP=arm64-apple-ios-simulator-cpp ;; + aarch64-apple-ios*) CPP=arm64-apple-ios-cpp ;; + x86_64-apple-ios*-simulator) CPP=x86_64-apple-ios-simulator-cpp ;; + *) + esac +fi +if test -z "$CXX"; then + case "$host" in + aarch64-apple-ios*-simulator) CXX=arm64-apple-ios-simulator-clang++ ;; + aarch64-apple-ios*) CXX=arm64-apple-ios-clang++ ;; + x86_64-apple-ios*-simulator) CXX=x86_64-apple-ios-simulator-clang++ ;; + *) + esac +fi + AC_MSG_CHECKING([for --enable-universalsdk]) AC_ARG_ENABLE([universalsdk], AS_HELP_STRING([--enable-universalsdk@<:@=SDKDIR@:>@], @@ -487,37 +556,38 @@ AC_ARG_ENABLE([framework], [ case $enableval in yes) - if test "$ac_sys_system" = "iOS"; then - AC_MSG_ERROR([iOS builds must provide an explicit path for --enable-framework]) - fi - - enableval=/Library/Frameworks + case $ac_sys_system in + Darwin) enableval=/Library/Frameworks ;; + iOS) enableval=iOS/Frameworks/\$\(MULTIARCH\) ;; + *) AC_MSG_ERROR([Unknown platform for framework build]) + esac esac + case $enableval in no) - if test "$ac_sys_system" = "iOS"; then - AC_MSG_ERROR([iOS builds must use --enable-framework=]) - fi + case $ac_sys_system in + iOS) AC_MSG_ERROR([iOS builds must use --enable-framework]) ;; + *) + PYTHONFRAMEWORK= + PYTHONFRAMEWORKDIR=no-framework + PYTHONFRAMEWORKPREFIX= + PYTHONFRAMEWORKINSTALLDIR= + PYTHONFRAMEWORKINSTALLNAMEPREFIX= + RESSRCDIR= + FRAMEWORKINSTALLFIRST= + FRAMEWORKINSTALLLAST= + FRAMEWORKALTINSTALLFIRST= + FRAMEWORKALTINSTALLLAST= + FRAMEWORKPYTHONW= + INSTALLTARGETS="commoninstall bininstall maninstall" - PYTHONFRAMEWORK= - PYTHONFRAMEWORKDIR=no-framework - PYTHONFRAMEWORKPREFIX= - PYTHONFRAMEWORKINSTALLDIR= - PYTHONFRAMEWORKINSTALLNAMEPREFIX= - RESSRCDIR= - FRAMEWORKINSTALLFIRST= - FRAMEWORKINSTALLLAST= - FRAMEWORKALTINSTALLFIRST= - FRAMEWORKALTINSTALLLAST= - FRAMEWORKPYTHONW= - INSTALLTARGETS="commoninstall bininstall maninstall" - - if test "x${prefix}" = "xNONE"; then - FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" - else - FRAMEWORKUNIXTOOLSPREFIX="${prefix}" - fi - enable_framework= + if test "x${prefix}" = "xNONE"; then + FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" + else + FRAMEWORKUNIXTOOLSPREFIX="${prefix}" + fi + enable_framework= + esac ;; *) PYTHONFRAMEWORKPREFIX="${enableval}" @@ -605,29 +675,28 @@ AC_ARG_ENABLE([framework], esac esac ],[ - if test "$ac_sys_system" = "iOS"; then - AC_MSG_ERROR([iOS builds must use --enable-framework=]) - fi - - PYTHONFRAMEWORK= - PYTHONFRAMEWORKDIR=no-framework - PYTHONFRAMEWORKPREFIX= - PYTHONFRAMEWORKINSTALLDIR= - PYTHONFRAMEWORKINSTALLNAMEPREFIX= - RESSRCDIR= - FRAMEWORKINSTALLFIRST= - FRAMEWORKINSTALLLAST= - FRAMEWORKALTINSTALLFIRST= - FRAMEWORKALTINSTALLLAST= - FRAMEWORKPYTHONW= - INSTALLTARGETS="commoninstall bininstall maninstall" - if test "x${prefix}" = "xNONE" ; then - FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" - else - FRAMEWORKUNIXTOOLSPREFIX="${prefix}" - fi - enable_framework= - + case $ac_sys_system in + iOS) AC_MSG_ERROR([iOS builds must use --enable-framework]) ;; + *) + PYTHONFRAMEWORK= + PYTHONFRAMEWORKDIR=no-framework + PYTHONFRAMEWORKPREFIX= + PYTHONFRAMEWORKINSTALLDIR= + PYTHONFRAMEWORKINSTALLNAMEPREFIX= + RESSRCDIR= + FRAMEWORKINSTALLFIRST= + FRAMEWORKINSTALLLAST= + FRAMEWORKALTINSTALLFIRST= + FRAMEWORKALTINSTALLLAST= + FRAMEWORKPYTHONW= + INSTALLTARGETS="commoninstall bininstall maninstall" + if test "x${prefix}" = "xNONE" ; then + FRAMEWORKUNIXTOOLSPREFIX="${ac_default_prefix}" + else + FRAMEWORKUNIXTOOLSPREFIX="${prefix}" + fi + enable_framework= + esac ]) AC_SUBST([PYTHONFRAMEWORK]) AC_SUBST([PYTHONFRAMEWORKIDENTIFIER]) @@ -648,6 +717,47 @@ AC_SUBST([INSTALLTARGETS]) AC_DEFINE_UNQUOTED([_PYTHONFRAMEWORK], ["${PYTHONFRAMEWORK}"], [framework name]) +dnl quadrigraphs "@<:@" and "@:>@" produce "[" and "]" in the output +AC_MSG_CHECKING([for --with-app-store-compliance]) +AC_ARG_WITH( + [app_store_compliance], + [AS_HELP_STRING( + [--with-app-store-compliance=@<:@PATCH-FILE@:>@], + [Enable any patches required for compiliance with app stores. + Optional PATCH-FILE specifies the custom patch to apply.] + )],[ + case "$withval" in + yes) + case $ac_sys_system in + Darwin|iOS) + # iOS is able to share the macOS patch + APP_STORE_COMPLIANCE_PATCH="Mac/Resources/app-store-compliance.patch" + ;; + *) AC_MSG_ERROR([no default app store compliance patch available for $ac_sys_system]) ;; + esac + AC_MSG_RESULT([applying default app store compliance patch]) + ;; + *) + APP_STORE_COMPLIANCE_PATCH="${withval}" + AC_MSG_RESULT([applying custom app store compliance patch]) + ;; + esac + ],[ + case $ac_sys_system in + iOS) + # Always apply the compliance patch on iOS; we can use the macOS patch + APP_STORE_COMPLIANCE_PATCH="Mac/Resources/app-store-compliance.patch" + AC_MSG_RESULT([applying default app store compliance patch]) + ;; + *) + # No default app compliance patching on any other platform + APP_STORE_COMPLIANCE_PATCH= + AC_MSG_RESULT([not patching for app store compliance]) + ;; + esac +]) +AC_SUBST([APP_STORE_COMPLIANCE_PATCH]) + AC_SUBST([_PYTHON_HOST_PLATFORM]) if test "$cross_compiling" = yes; then case "$host" in @@ -668,22 +778,36 @@ if test "$cross_compiling" = yes; then _host_device=`echo $host | cut -d '-' -f4` _host_device=${_host_device:=os} - dnl IOS_DEPLOYMENT_TARGET is the minimum supported iOS version - IOS_DEPLOYMENT_TARGET=${_host_os:3} - IOS_DEPLOYMENT_TARGET=${IOS_DEPLOYMENT_TARGET:=12.0} + # IPHONEOS_DEPLOYMENT_TARGET is the minimum supported iOS version + AC_MSG_CHECKING([iOS deployment target]) + IPHONEOS_DEPLOYMENT_TARGET=$(echo ${_host_os} | cut -c4-) + IPHONEOS_DEPLOYMENT_TARGET=${IPHONEOS_DEPLOYMENT_TARGET:=13.0} + AC_MSG_RESULT([$IPHONEOS_DEPLOYMENT_TARGET]) case "$host_cpu" in aarch64) - _host_ident=${IOS_DEPLOYMENT_TARGET}-arm64-iphone${_host_device} + _host_ident=${IPHONEOS_DEPLOYMENT_TARGET}-arm64-iphone${_host_device} ;; *) - _host_ident=${IOS_DEPLOYMENT_TARGET}-$host_cpu-iphone${_host_device} + _host_ident=${IPHONEOS_DEPLOYMENT_TARGET}-$host_cpu-iphone${_host_device} ;; esac ;; + *-*-darwin*) + case "$host_cpu" in + arm*) + _host_ident=arm + ;; + *) + _host_ident=$host_cpu + esac + ;; *-*-vxworks*) _host_ident=$host_cpu ;; + *-*-emscripten) + _host_ident=$(emcc -dumpversion | cut -f1 -d-)-$host_cpu + ;; wasm32-*-* | wasm64-*-*) _host_ident=$host_cpu ;; @@ -819,6 +943,9 @@ AC_SUBST([EXPORT_MACOSX_DEPLOYMENT_TARGET]) CONFIGURE_MACOSX_DEPLOYMENT_TARGET= EXPORT_MACOSX_DEPLOYMENT_TARGET='#' +# Record the value of IPHONEOS_DEPLOYMENT_TARGET enforced by the selected host triple. +AC_SUBST([IPHONEOS_DEPLOYMENT_TARGET]) + # checks for alternative programs # compiler flags are generated in two sets, BASECFLAGS and OPT. OPT is just @@ -854,9 +981,8 @@ AS_CASE([$host], dnl Add the compiler flag for the iOS minimum supported OS version. AS_CASE([$ac_sys_system], [iOS], [ - AS_VAR_APPEND([CFLAGS], [" -mios-version-min=${IOS_DEPLOYMENT_TARGET}"]) - AS_VAR_APPEND([LDFLAGS], [" -mios-version-min=${IOS_DEPLOYMENT_TARGET}"]) - AC_SUBST([IOS_DEPLOYMENT_TARGET]) + AS_VAR_APPEND([CFLAGS], [" -mios-version-min=${IPHONEOS_DEPLOYMENT_TARGET}"]) + AS_VAR_APPEND([LDFLAGS], [" -mios-version-min=${IPHONEOS_DEPLOYMENT_TARGET}"]) ], ) @@ -937,13 +1063,18 @@ AC_PROG_GREP AC_PROG_SED AC_PROG_EGREP +dnl GNU Autoconf recommends the use of expr instead of basename. +AS_VAR_SET([CC_BASENAME], [$(expr "//$CC" : '.*/\(.*\)')]) + dnl detect compiler name dnl check for xlc before clang, newer xlc's can use clang as frontend. dnl check for GCC last, other compilers set __GNUC__, too. dnl msvc is listed for completeness. AC_CACHE_CHECK([for CC compiler name], [ac_cv_cc_name], [ cat > conftest.c <conftest.out 2>/dev/null; then ac_cv_cc_name=`grep -v '^#' conftest.out | grep -v '^ *$' | tr -d ' '` + AS_VAR_IF([CC_BASENAME], [mpicc], [ac_cv_cc_name=mpicc]) else ac_cv_cc_name="unknown" fi @@ -971,16 +1103,28 @@ rm -f conftest.c conftest.out # _POSIX_SOURCE, _POSIX_1_SOURCE, and more AC_USE_SYSTEM_EXTENSIONS +AC_CACHE_CHECK([for GCC compatible compiler], + [ac_cv_gcc_compat], + [AC_PREPROC_IFELSE([AC_LANG_SOURCE([ + #if !defined(__GNUC__) + #error "not GCC compatible" + #else + /* GCC compatible! */ + #endif + ], [])], + [ac_cv_gcc_compat=yes], + [ac_cv_gcc_compat=no])]) + AC_SUBST([CXX]) preset_cxx="$CXX" if test -z "$CXX" then - case "$CC" in + case "$ac_cv_cc_name" in gcc) AC_PATH_TOOL([CXX], [g++], [g++], [notfound]) ;; cc) AC_PATH_TOOL([CXX], [c++], [c++], [notfound]) ;; - clang|*/clang) AC_PATH_TOOL([CXX], [clang++], [clang++], [notfound]) ;; - icc|*/icc) AC_PATH_TOOL([CXX], [icpc], [icpc], [notfound]) ;; + clang) AC_PATH_TOOL([CXX], [clang++], [clang++], [notfound]) ;; + icc) AC_PATH_TOOL([CXX], [icpc], [icpc], [notfound]) ;; esac if test "$CXX" = "notfound" then @@ -1065,14 +1209,14 @@ AC_MSG_CHECKING([for PEP 11 support tier]) AS_CASE([$host/$ac_cv_cc_name], [x86_64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=1], dnl Linux on AMD64, any vendor, glibc, gcc [x86_64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on Intel, any version + [aarch64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on M1, any version [i686-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 32bit Windows on Intel, MSVC [x86_64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 64bit Windows on AMD64, MSVC - [aarch64-apple-darwin*/clang], [PY_SUPPORT_TIER=2], dnl macOS on M1, any version [aarch64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux ARM64, glibc, gcc+clang [aarch64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], [powerpc64le-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux on PPC64 little endian, glibc, gcc - [wasm32-unknown-wasi/clang], [PY_SUPPORT_TIER=2], dnl WebAssembly System Interface, clang + [wasm32-unknown-wasip1/clang], [PY_SUPPORT_TIER=2], dnl WebAssembly System Interface preview1, clang [x86_64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], dnl Linux on AMD64, any vendor, glibc, clang [aarch64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=3], dnl Windows ARM64, MSVC @@ -1082,6 +1226,9 @@ AS_CASE([$host/$ac_cv_cc_name], [x86_64-*-freebsd*/clang], [PY_SUPPORT_TIER=3], dnl FreeBSD on AMD64 [aarch64-apple-ios*-simulator/clang], [PY_SUPPORT_TIER=3], dnl iOS Simulator on arm64 [aarch64-apple-ios*/clang], [PY_SUPPORT_TIER=3], dnl iOS on ARM64 + [aarch64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on ARM64 + [x86_64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on AMD64 + [PY_SUPPORT_TIER=0] ) @@ -1126,6 +1273,9 @@ if $CPP $CPPFLAGS conftest.c >conftest.out 2>/dev/null; then AC_DEFINE_UNQUOTED([ANDROID_API_LEVEL], [$ANDROID_API_LEVEL], [The Android API level.]) + # For __android_log_write() in Python/pylifecycle.c. + LIBS="$LIBS -llog" + AC_MSG_CHECKING([for the Android arm ABI]) AC_MSG_RESULT([$_arm_arch]) if test "$_arm_arch" = 7; then @@ -1147,30 +1297,6 @@ AS_CASE([$ac_sys_system/$ac_sys_release], ] ) -AC_MSG_CHECKING([for --with-emscripten-target]) -AC_ARG_WITH([emscripten-target], - [AS_HELP_STRING([--with-emscripten-target=@<:@browser|node@:>@], [Emscripten platform])], -[ - AS_VAR_IF([ac_sys_system], [Emscripten], [ - AS_CASE([$with_emscripten_target], - [browser], [ac_sys_emscripten_target=browser], - [node], [ac_sys_emscripten_target=node], -dnl Debug builds with source map / dwarf symbols. Py_DEBUG builds easily -dnl run out of stack space. Detached sybmols and map prohibit some -dnl optimizations and increase file size. Options are undocumented so we -dnl are free to remove them in the future. - [browser-debug], [ac_sys_emscripten_target=browser-debug], - [node-debug], [ac_sys_emscripten_target=node-debug], - [AC_MSG_ERROR([Invalid argument: --with-emscripten-target=browser|node])] - ) - ], [ - AC_MSG_ERROR([--with-emscripten-target only applies to Emscripten]) - ]) -], [ - AS_VAR_IF([ac_sys_system], [Emscripten], [ac_sys_emscripten_target=browser]) -]) -AC_MSG_RESULT([$ac_sys_emscripten_target]) - dnl On Emscripten dlopen() requires -s MAIN_MODULE and -fPIC. The flags dnl disables dead code elimination and increases the size of the WASM module dnl by about 1.5 to 2MB. MAIN_MODULE defines __wasm_mutable_globals__. @@ -1215,15 +1341,17 @@ AC_ARG_WITH([suffix], [EXEEXT=$with_suffix] ) ], [ - AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], - [Emscripten/browser*], [EXEEXT=.js], - [Emscripten/node*], [EXEEXT=.js], - [WASI/*], [EXEEXT=.wasm], + AS_CASE([$ac_sys_system], + [Emscripten], [EXEEXT=.mjs], + [WASI], [EXEEXT=.wasm], [EXEEXT=] ) ]) AC_MSG_RESULT([$EXEEXT]) +# Make sure we keep EXEEXT and ac_exeext sync'ed. +AS_VAR_SET([ac_exeext], [$EXEEXT]) + # Test whether we're running on a non-case-sensitive system, in which # case we give a warning if no ext is given AC_SUBST([BUILDEXEEXT]) @@ -1244,7 +1372,7 @@ rmdir CaseSensitiveTestDir case $ac_sys_system in hp*|HP*) - case $CC in + case $ac_cv_cc_name in cc|*/cc) CC="$CC -Ae";; esac;; esac @@ -1330,7 +1458,7 @@ AC_MSG_RESULT([$EXPORTSYMS]) AC_SUBST([GNULD]) AC_MSG_CHECKING([for GNU ld]) ac_prog=ld -if test "$GCC" = yes; then +if test "$ac_cv_cc_name" = "gcc"; then ac_prog=`$CC -print-prog-name=ld` fi case `"$ac_prog" -V 2>&1 < /dev/null` in @@ -1496,48 +1624,20 @@ if test "$cross_compiling" = yes; then RUNSHARED= fi +# HOSTRUNNER - Program to run CPython for the host platform AC_MSG_CHECKING([HOSTRUNNER]) -AC_ARG_VAR([HOSTRUNNER], [Program to run CPython for the host platform]) if test -z "$HOSTRUNNER" then - AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], - [Emscripten/node*], [ + AS_CASE([$ac_sys_system], + [Emscripten], [ AC_PATH_TOOL([NODE], [node], [node]) HOSTRUNNER="$NODE" - # bigint for ctypes c_longlong, c_longdouble - # no longer available in Node 16 - AC_CACHE_CHECK([for node --experimental-wasm-bigint], [ac_cv_tool_node_wasm_bigint], [ - if $NODE -v --experimental-wasm-bigint > /dev/null 2>&1; then - ac_cv_tool_node_wasm_bigint=yes - else - ac_cv_tool_node_wasm_bigint=no - fi - ]) - AS_VAR_IF([ac_cv_tool_node_wasm_bigint], [yes], [ - AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-bigint"]) - ]) - - AS_VAR_IF([enable_wasm_pthreads], [yes], [ - AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-threads"]) - # no longer available in Node 16 - AC_CACHE_CHECK([for node --experimental-wasm-bulk-memory], [ac_cv_tool_node_wasm_bulk_memory], [ - if $NODE -v --experimental-wasm-bulk-memory > /dev/null 2>&1; then - ac_cv_tool_node_wasm_bulk_memory=yes - else - ac_cv_tool_node_wasm_bulk_memory=no - fi - ]) - AS_VAR_IF([ac_cv_tool_node_wasm_bulk_memory], [yes], [ - AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-bulk-memory"]) - ]) - ]) - AS_VAR_IF([host_cpu], [wasm64], [AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-memory64"])]) ], dnl TODO: support other WASI runtimes - dnl wasmtime starts the proces with "/" as CWD. For OOT builds add the + dnl wasmtime starts the process with "/" as CWD. For OOT builds add the dnl directory containing _sysconfigdata to PYTHONPATH. - [WASI/*], [HOSTRUNNER='wasmtime run --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --mapdir /::$(srcdir) --'], + [WASI], [HOSTRUNNER='wasmtime run --wasm max-wasm-stack=16777216 --wasi preview2=n --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --dir $(srcdir)::/'], [HOSTRUNNER=''] ) fi @@ -1550,10 +1650,8 @@ if test -n "$HOSTRUNNER"; then fi # LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable -AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], - [Emscripten/browser*], [LIBRARY_DEPS='$(PY3LIBRARY) $(WASM_STDLIB) python.html python.worker.js'], - [LIBRARY_DEPS='$(PY3LIBRARY) $(EXPORTSYMS)'] -) +LIBRARY_DEPS='$(PY3LIBRARY) $(EXPORTSYMS)' + LINK_PYTHON_DEPS='$(LIBRARY_DEPS)' if test "$PY_ENABLE_SHARED" = 1 || test "$enable_framework" ; then LIBRARY_DEPS="\$(LDLIBRARY) $LIBRARY_DEPS" @@ -1609,7 +1707,9 @@ fi # For calculating the .so ABI tag. AC_SUBST([ABIFLAGS]) +AC_SUBST([ABI_THREAD]) ABIFLAGS="" +ABI_THREAD="" # Check for --disable-gil # --disable-gil @@ -1626,6 +1726,7 @@ then [Define if you want to disable the GIL]) # Add "t" for "threaded" ABIFLAGS="${ABIFLAGS}t" + ABI_THREAD="t" fi # Check for --with-pydebug @@ -1659,6 +1760,10 @@ then [Define if you want to enable tracing references for debugging purpose]) fi +if test "$disable_gil" = "yes" -a "$with_trace_refs" = "yes"; +then + AC_MSG_ERROR([--disable-gil cannot be used with --with-trace-refs]) +fi # Check for --enable-pystats AC_MSG_CHECKING([for --enable-pystats]) @@ -1699,14 +1804,27 @@ fi # Check for --enable-experimental-jit: AC_MSG_CHECKING([for --enable-experimental-jit]) AC_ARG_ENABLE([experimental-jit], - [AS_HELP_STRING([--enable-experimental-jit], + [AS_HELP_STRING([--enable-experimental-jit@<:@=no|yes|yes-off|interpreter@:>@], [build the experimental just-in-time compiler (default is no)])], [], [enable_experimental_jit=no]) -AS_VAR_IF([enable_experimental_jit], - [no], +case $enable_experimental_jit in + no) jit_flags=""; tier2_flags="" ;; + yes) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=1" ;; + yes-off) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=3" ;; + interpreter) jit_flags=""; tier2_flags="-D_Py_TIER2=4" ;; + interpreter-off) jit_flags=""; tier2_flags="-D_Py_TIER2=6" ;; # Secret option + *) AC_MSG_ERROR( + [invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter]) ;; +esac +AS_VAR_IF([tier2_flags], [], - [AS_VAR_APPEND([CFLAGS_NODIST], [" -D_Py_JIT"]) + [], + [AS_VAR_APPEND([CFLAGS_NODIST], [" $tier2_flags"])]) +AS_VAR_IF([jit_flags], + [], + [], + [AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"]) AS_VAR_SET([REGEN_JIT_COMMAND], ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host"]) AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"]) @@ -1716,7 +1834,7 @@ AS_VAR_IF([enable_experimental_jit], [])]) AC_SUBST([REGEN_JIT_COMMAND]) AC_SUBST([JIT_STENCILS_H]) -AC_MSG_RESULT([$enable_experimental_jit]) +AC_MSG_RESULT([$tier2_flags $jit_flags]) # Enable optimization flags AC_SUBST([DEF_MAKE_ALL_RULE]) @@ -1745,17 +1863,19 @@ if test "$Py_OPT" = 'true' ; then DEF_MAKE_ALL_RULE="profile-opt" REQUIRE_PGO="yes" DEF_MAKE_RULE="build_all" - case $CC in - *gcc*) + AS_VAR_IF([ac_cv_gcc_compat], [yes], [ AX_CHECK_COMPILE_FLAG([-fno-semantic-interposition],[ CFLAGS_NODIST="$CFLAGS_NODIST -fno-semantic-interposition" LDFLAGS_NODIST="$LDFLAGS_NODIST -fno-semantic-interposition" - ]) - ;; - esac -elif test "$ac_sys_system" = "Emscripten" -o "$ac_sys_system" = "WASI"; then - dnl Emscripten does not support shared extensions yet. Build - dnl "python.[js,wasm]", "pybuilddir.txt", and "platform" files. + ], [], [-Werror]) + ]) +elif test "$ac_sys_system" = "Emscripten"; then + dnl Build "python.[js,wasm]", "pybuilddir.txt", and "platform" files. + DEF_MAKE_ALL_RULE="build_emscripten" + REQUIRE_PGO="no" + DEF_MAKE_RULE="all" +elif test "$ac_sys_system" = "WASI"; then + dnl Build "python.wasm", "pybuilddir.txt", and "platform" files. DEF_MAKE_ALL_RULE="build_wasm" REQUIRE_PGO="no" DEF_MAKE_RULE="all" @@ -1779,7 +1899,7 @@ AC_MSG_RESULT([$PROFILE_TASK]) llvm_bin_dir='' llvm_path="${PATH}" -if test "${CC}" = "clang" +if test "${ac_cv_cc_name}" = "clang" then clang_bin=`which clang` # Some systems install clang elsewhere as a symlink to the real path @@ -1826,8 +1946,8 @@ esac ], [AC_MSG_RESULT([no])]) if test "$Py_LTO" = 'true' ; then - case $CC in - *clang*) + case $ac_cv_cc_name in + clang) LDFLAGS_NOLTO="-fno-lto" dnl Clang linker requires -flto in order to link objects with LTO information. dnl Thin LTO is faster and works for object files with full LTO information, too. @@ -1890,14 +2010,14 @@ if test "$Py_LTO" = 'true' ; then ;; esac ;; - *emcc*) + emcc) if test "$Py_LTO_POLICY" != "default"; then AC_MSG_ERROR([emcc supports only default lto.]) fi LTOFLAGS="-flto" LTOCFLAGS="-flto" ;; - *gcc*) + gcc) if test $Py_LTO_POLICY = thin then AC_MSG_ERROR([thin lto is not supported under gcc compiler.]) @@ -1956,15 +2076,17 @@ then fi LLVM_PROF_ERR=no -# GNU Autoconf recommends the use of expr instead of basename. -AS_VAR_SET([CC_BASENAME], [$(expr "//$CC" : '.*/\(.*\)')]) -case "$CC_BASENAME" in - *clang*) +case "$ac_cv_cc_name" in + clang) # Any changes made here should be reflected in the GCC+Darwin case below PGO_PROF_GEN_FLAG="-fprofile-instr-generate" - PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" - LLVM_PROF_MERGER="${LLVM_PROFDATA} merge -output=code.profclangd *.profclangr" - LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + PGO_PROF_USE_FLAG="-fprofile-instr-use=\"\$(shell pwd)/code.profclangd\"" + LLVM_PROF_MERGER=m4_normalize(" + ${LLVM_PROFDATA} merge + -output=\"\$(shell pwd)/code.profclangd\" + \"\$(shell pwd)\"/*.profclangr + ") + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"\$(shell pwd)/code-%p.profclangr\"" if test $LLVM_PROF_FOUND = not-found then LLVM_PROF_ERR=yes @@ -1974,31 +2096,13 @@ case "$CC_BASENAME" in fi fi ;; - *gcc*) - case $ac_sys_system in - Darwin*) - PGO_PROF_GEN_FLAG="-fprofile-instr-generate" - PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" - LLVM_PROF_MERGER="${LLVM_PROFDATA} merge -output=code.profclangd *.profclangr" - LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" - if test "${LLVM_PROF_FOUND}" = "not-found" - then - LLVM_PROF_ERR=yes - if test "${REQUIRE_PGO}" = "yes" - then - AC_MSG_ERROR([llvm-profdata is required for a --enable-optimizations build but could not be found.]) - fi - fi - ;; - *) - PGO_PROF_GEN_FLAG="-fprofile-generate" - PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" - LLVM_PROF_MERGER="true" - LLVM_PROF_FILE="" - ;; - esac + gcc) + PGO_PROF_GEN_FLAG="-fprofile-generate" + PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" + LLVM_PROF_MERGER="true" + LLVM_PROF_FILE="" ;; - *icc*) + icc) PGO_PROF_GEN_FLAG="-prof-gen" PGO_PROF_USE_FLAG="-prof-use" LLVM_PROF_MERGER="true" @@ -2068,6 +2172,27 @@ AS_VAR_IF([enable_shared], [yes], [ BOLT_BINARIES="${BOLT_BINARIES} \$(INSTSONAME)" ]) +AC_ARG_VAR( + [BOLT_COMMON_FLAGS], + [Common arguments to llvm-bolt when instrumenting and applying] +) + +AC_MSG_CHECKING([BOLT_COMMON_FLAGS]) +if test -z "${BOLT_COMMON_FLAGS}" +then + AS_VAR_SET( + [BOLT_COMMON_FLAGS], + [m4_normalize(" + [-update-debug-sections] + + dnl At least LLVM 19.x doesn't support computed gotos in PIC compiled code. + dnl Exclude functions containing computed gotos. + dnl TODO this may be fixed in LLVM 20.x via https://github.com/llvm/llvm-project/pull/120267. + [-skip-funcs=_PyEval_EvalFrameDefault,sre_ucs1_match/1,sre_ucs2_match/1,sre_ucs4_match/1] + ")] + ) +fi + AC_ARG_VAR( [BOLT_INSTRUMENT_FLAGS], [Arguments to llvm-bolt when instrumenting binaries] @@ -2075,7 +2200,7 @@ AC_ARG_VAR( AC_MSG_CHECKING([BOLT_INSTRUMENT_FLAGS]) if test -z "${BOLT_INSTRUMENT_FLAGS}" then - BOLT_INSTRUMENT_FLAGS= + BOLT_INSTRUMENT_FLAGS="${BOLT_COMMON_FLAGS}" fi AC_MSG_RESULT([$BOLT_INSTRUMENT_FLAGS]) @@ -2089,9 +2214,9 @@ then AS_VAR_SET( [BOLT_APPLY_FLAGS], [m4_normalize(" - -update-debug-sections + ${BOLT_COMMON_FLAGS} -reorder-blocks=ext-tsp - -reorder-functions=hfsort+ + -reorder-functions=cdsort -split-functions -icf=1 -inline-all @@ -2122,19 +2247,6 @@ AC_MSG_RESULT([$BOLT_APPLY_FLAGS]) # compiler and platform. BASECFLAGS tweaks need to be made even if the # user set OPT. -case $CC in - *clang*) - cc_is_clang=1 - ;; - *) - if $CC --version 2>&1 | grep -q clang - then - cc_is_clang=1 - else - cc_is_clang= - fi -esac - dnl Historically, some of our code assumed that signed integer overflow dnl is defined behaviour via twos-complement. dnl Set STRICT_OVERFLOW_CFLAGS and NO_STRICT_OVERFLOW_CFLAGS depending on compiler support. @@ -2196,6 +2308,11 @@ PYDEBUG_CFLAGS="-O0" AS_VAR_IF([ac_cv_cc_supports_og], [yes], [PYDEBUG_CFLAGS="-Og"]) +# gh-120688: WASI uses -O3 in debug mode to support more recursive calls +if test "$ac_sys_system" = "WASI"; then + PYDEBUG_CFLAGS="-O3" +fi + # tweak OPT based on compiler and platform, only if the user didn't set # it on the command line AC_SUBST([OPT]) @@ -2204,7 +2321,7 @@ if test "${OPT-unset}" = "unset" then case $GCC in yes) - if test -n "${cc_is_clang}" + if test "${ac_cv_cc_name}" != "clang" then # bpo-30104: disable strict aliasing to compile correctly dtoa.c, # see Makefile.pre.in for more information @@ -2244,13 +2361,16 @@ AS_CASE([$ac_sys_system], AS_VAR_IF([Py_DEBUG], [yes], [wasm_debug=yes], [wasm_debug=no]) dnl Start with 20 MB and allow to grow - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH -sTOTAL_MEMORY=20971520"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sALLOW_MEMORY_GROWTH -sINITIAL_MEMORY=20971520"]) dnl map int64_t and uint64_t to JS bigint AS_VAR_APPEND([LDFLAGS_NODIST], [" -sWASM_BIGINT"]) dnl Include file system support - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sSTACK_SIZE=5MB"]) AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE"]) @@ -2261,23 +2381,9 @@ AS_CASE([$ac_sys_system], AS_VAR_APPEND([LDFLAGS_NODIST], [" -sUSE_PTHREADS"]) AS_VAR_APPEND([LINKFORSHARED], [" -sPROXY_TO_PTHREAD"]) ]) - - AS_CASE([$ac_sys_emscripten_target], - [browser*], [ - AS_VAR_IF([ac_sys_emscripten_target], [browser-debug], [wasm_debug=yes]) - AS_VAR_APPEND([LINKFORSHARED], [" --preload-file=\$(WASM_ASSETS_DIR)"]) - WASM_ASSETS_DIR=".\$(prefix)" - WASM_STDLIB="\$(WASM_ASSETS_DIR)/local/lib/python\$(VERSION)/os.py" - dnl separate-dwarf does not seem to work in Chrome DevTools Support. - WASM_LINKFORSHARED_DEBUG="-gsource-map --emit-symbol-map" - ], - [node*], [ - AS_VAR_IF([ac_sys_emscripten_target], [node-debug], [wasm_debug=yes]) - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH -sNODERAWFS"]) - AS_VAR_APPEND([LINKFORSHARED], [" -sEXIT_RUNTIME"]) - WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map" - ] - ) + dnl not completely sure whether or not we want -sEXIT_RUNTIME, keeping it for now. + AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXIT_RUNTIME"]) + WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map" AS_VAR_IF([wasm_debug], [yes], [ AS_VAR_APPEND([LDFLAGS_NODIST], [" -sASSERTIONS"]) @@ -2310,9 +2416,10 @@ AS_CASE([$ac_sys_system], AS_VAR_APPEND([LDFLAGS_NODIST], [" -Wl,--max-memory=10485760"]) ]) - dnl increase initial memory and stack size, move stack first + dnl gh-117645: Set the memory size to 40 MiB, the stack size to 16 MiB, + dnl and move the stack first. dnl https://github.com/WebAssembly/wasi-libc/issues/233 - AS_VAR_APPEND([LDFLAGS_NODIST], [" -z stack-size=524288 -Wl,--stack-first -Wl,--initial-memory=10485760"]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" -z stack-size=16777216 -Wl,--stack-first -Wl,--initial-memory=41943040"]) ] ) @@ -2338,7 +2445,7 @@ AC_DEFUN([PY_CHECK_CC_WARNING], [ AS_VAR_PUSHDEF([py_var], [ac_cv_$1_]m4_normalize($2)[_warning]) AC_CACHE_CHECK([m4_ifblank([$3], [if we can $1 $CC $2 warning], [$3])], [py_var], [ AS_VAR_COPY([py_cflags], [CFLAGS]) - AS_VAR_APPEND([CFLAGS], ["-W$2 -Werror"]) + AS_VAR_APPEND([CFLAGS], [" -W$2 -Werror"]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[]])], [AS_VAR_SET([py_var], [yes])], [AS_VAR_SET([py_var], [no])]) @@ -2352,8 +2459,37 @@ AS_VAR_IF([with_strict_overflow], [yes], [BASECFLAGS="$BASECFLAGS $STRICT_OVERFLOW_CFLAGS"], [BASECFLAGS="$BASECFLAGS $NO_STRICT_OVERFLOW_CFLAGS"]) -case $GCC in -yes) +# Enable flags that warn and protect for potential security vulnerabilities. +# These flags should be enabled by default for all builds. + +AC_MSG_CHECKING([for --enable-safety]) +AC_ARG_ENABLE([safety], + [AS_HELP_STRING([--enable-safety], [enable usage of the security compiler options with no performance overhead])], + [AS_VAR_IF([disable_safety], [yes], [enable_safety=no], [enable_safety=yes])], [enable_safety=no]) +AC_MSG_RESULT([$enable_safety]) + +if test "$enable_safety" = "yes" +then + AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [CFLAGS_NODIST="$CFLAGS_NODIST -fstack-protector-strong"], [AC_MSG_WARN([-fstack-protector-strong not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Wtrampolines], [CFLAGS_NODIST="$CFLAGS_NODIST -Wtrampolines"], [AC_MSG_WARN([-Wtrampolines not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough], [CFLAGS_NODIST="$CFLAGS_NODIST -Wimplicit-fallthrough"], [AC_MSG_WARN([-Wimplicit-fallthrough not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Werror=format-security], [CFLAGS_NODIST="$CFLAGS_NODIST -Werror=format-security"], [AC_MSG_WARN([-Werror=format-security not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Wbidi-chars=any], [CFLAGS_NODIST="$CFLAGS_NODIST -Wbidi-chars=any"], [AC_MSG_WARN([-Wbidi-chars=any not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Wall], [CFLAGS_NODIST="$CFLAGS_NODIST -Wall"], [AC_MSG_WARN([-Wall not supported])], [-Werror]) +fi + +AC_MSG_CHECKING([for --enable-slower-safety]) +AC_ARG_ENABLE([slower-safety], + [AS_HELP_STRING([--enable-slower-safety], [enable usage of the security compiler options with performance overhead])], + [AS_VAR_IF([disable_slower_safety], [yes], [enable_slower_safety=no], [enable_slower_safety=yes])], [enable_slower_safety=no]) +AC_MSG_RESULT([$enable_slower_safety]) + +if test "$enable_slower_safety" = "yes" +then + AX_CHECK_COMPILE_FLAG([-D_FORTIFY_SOURCE=3], [CFLAGS_NODIST="$CFLAGS_NODIST -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3"], [AC_MSG_WARN([-D_FORTIFY_SOURCE=3 not supported])], [-Werror]) +fi + +AS_VAR_IF([ac_cv_gcc_compat], [yes], [ CFLAGS_NODIST="$CFLAGS_NODIST -std=c11" PY_CHECK_CC_WARNING([enable], [extra], [if we can add -Wextra]) @@ -2394,8 +2530,8 @@ yes) # ICC doesn't recognize the option, but only emits a warning ## XXX does it emit an unused result warning and can it be disabled? - AS_CASE(["$CC_BASENAME"], - [*icc*], [ac_cv_disable_unused_result_warning=no] + AS_CASE(["$ac_cv_cc_name"], + [icc], [ac_cv_disable_unused_result_warning=no] [PY_CHECK_CC_WARNING([disable], [unused-result])]) AS_VAR_IF([ac_cv_disable_unused_result_warning], [yes], [BASECFLAGS="$BASECFLAGS -Wno-unused-result" @@ -2488,20 +2624,24 @@ yes) Darwin*) # -Wno-long-double, -no-cpp-precomp, and -mno-fused-madd # used to be here, but non-Apple gcc doesn't accept them. - if test "${CC}" = gcc - then - AC_MSG_CHECKING([which compiler should be used]) - case "${UNIVERSALSDK}" in - */MacOSX10.4u.sdk) - # Build using 10.4 SDK, force usage of gcc when the - # compiler is gcc, otherwise the user will get very - # confusing error messages when building on OSX 10.6 - CC=gcc-4.0 - CPP=cpp-4.0 - ;; - esac - AC_MSG_RESULT([$CC]) - fi + AC_MSG_CHECKING([which compiler should be used]) + case "${UNIVERSALSDK}" in + */MacOSX10.4u.sdk) + # Build using 10.4 SDK, force usage of gcc when the + # compiler is gcc, otherwise the user will get very + # confusing error messages when building on OSX 10.6 + CC=gcc-4.0 + CPP=cpp-4.0 + ;; + esac + AC_MSG_RESULT([$CC]) + + # Error on unguarded use of new symbols, which will fail at runtime for + # users on older versions of macOS + AX_CHECK_COMPILE_FLAG([-Wunguarded-availability], + [AS_VAR_APPEND([CFLAGS_NODIST], [" -Werror=unguarded-availability"])], + [], + [-Werror]) LIPO_INTEL64_FLAGS="" if test "${enable_universalsdk}" @@ -2626,9 +2766,7 @@ yes) # end of Darwin* tests ;; esac - ;; - -*) +], [ case $ac_sys_system in OpenUNIX*|UnixWare*) BASECFLAGS="$BASECFLAGS -K pentium,host,inline,loop_unroll,alloca " @@ -2637,18 +2775,17 @@ yes) BASECFLAGS="$BASECFLAGS -belf -Ki486 -DSCO5" ;; esac - ;; -esac +]) -case "$CC_BASENAME" in -*mpicc*) +case "$ac_cv_cc_name" in +mpicc) CFLAGS_NODIST="$CFLAGS_NODIST" ;; -*icc*) +icc) # ICC needs -fp-model strict or floats behave badly CFLAGS_NODIST="$CFLAGS_NODIST -fp-model strict" ;; -*xlc*) +xlc) CFLAGS_NODIST="$CFLAGS_NODIST -qalias=noansi -qmaxmem=-1" ;; esac @@ -2834,11 +2971,11 @@ AC_DEFINE([STDC_HEADERS], [1], AC_CHECK_HEADERS([ \ alloca.h asm/types.h bluetooth.h conio.h direct.h dlfcn.h endian.h errno.h fcntl.h grp.h \ io.h langinfo.h libintl.h libutil.h linux/auxvec.h sys/auxv.h linux/fs.h linux/limits.h linux/memfd.h \ - linux/random.h linux/soundcard.h \ + linux/netfilter_ipv4.h linux/random.h linux/soundcard.h linux/sched.h \ linux/tipc.h linux/wait.h netdb.h net/ethernet.h netinet/in.h netpacket/packet.h poll.h process.h pthread.h pty.h \ sched.h setjmp.h shadow.h signal.h spawn.h stropts.h sys/audioio.h sys/bsdtty.h sys/devpoll.h \ sys/endian.h sys/epoll.h sys/event.h sys/eventfd.h sys/file.h sys/ioctl.h sys/kern_control.h \ - sys/loadavg.h sys/lock.h sys/memfd.h sys/mkdev.h sys/mman.h sys/modem.h sys/param.h sys/poll.h \ + sys/loadavg.h sys/lock.h sys/memfd.h sys/mkdev.h sys/mman.h sys/modem.h sys/param.h sys/pidfd.h sys/poll.h \ sys/random.h sys/resource.h sys/select.h sys/sendfile.h sys/socket.h sys/soundcard.h sys/stat.h \ sys/statvfs.h sys/sys_domain.h sys/syscall.h sys/sysmacros.h sys/termio.h sys/time.h sys/times.h sys/timerfd.h \ sys/types.h sys/uio.h sys/un.h sys/utsname.h sys/wait.h sys/xattr.h sysexits.h syslog.h \ @@ -3238,7 +3375,6 @@ if test -z "$SHLIB_SUFFIX"; then esac ;; CYGWIN*) SHLIB_SUFFIX=.dll;; - iOS) SHLIB_SUFFIX=.dylib;; *) SHLIB_SUFFIX=.so;; esac fi @@ -3257,7 +3393,7 @@ then LDSHARED="\$(LIBPL)/ld_so_aix \$(CC) -bI:\$(LIBPL)/python.exp" ;; SunOS/5*) - if test "$GCC" = "yes" ; then + if test "$ac_cv_gcc_compat" = "yes" ; then LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared' else @@ -3265,7 +3401,7 @@ then LDCXXSHARED='$(CXX) -G' fi ;; hp*|HP*) - if test "$GCC" = "yes" ; then + if test "$ac_cv_gcc_compat" = "yes" ; then LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared' else @@ -3320,11 +3456,11 @@ then fi ;; iOS/*) - LDSHARED='$(CC) -dynamiclib -F . -framework Python' - LDCXXSHARED='$(CXX) -dynamiclib -F . -framework Python' + LDSHARED='$(CC) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' + LDCXXSHARED='$(CXX) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' BLDSHARED="$LDSHARED" ;; - Emscripten|WASI) + Emscripten*|WASI*) LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared';; Linux*|GNU*|QNX*|VxWorks*|Haiku*) @@ -3358,7 +3494,7 @@ then LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared';; OpenUNIX*|UnixWare*) - if test "$GCC" = "yes" ; then + if test "$ac_cv_gcc_compat" = "yes" ; then LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared' else @@ -3370,7 +3506,7 @@ then LDCXXSHARED='$(CXX) -Wl,-G,-Bexport';; WASI*) AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ - dnl not iplemented yet + dnl not implemented yet ]);; CYGWIN*) LDSHARED="gcc -shared -Wl,--enable-auto-image-base" @@ -3398,13 +3534,13 @@ AC_MSG_CHECKING([CCSHARED]) if test -z "$CCSHARED" then case $ac_sys_system/$ac_sys_release in - SunOS*) if test "$GCC" = yes; + SunOS*) if test "$ac_cv_gcc_compat" = "yes"; then CCSHARED="-fPIC"; elif test `uname -p` = sparc; then CCSHARED="-xcode=pic32"; else CCSHARED="-Kpic"; fi;; - hp*|HP*) if test "$GCC" = yes; + hp*|HP*) if test "$ac_cv_gcc_compat" = "yes"; then CCSHARED="-fPIC"; else CCSHARED="+z"; fi;; @@ -3416,12 +3552,12 @@ then FreeBSD*|NetBSD*|OpenBSD*|DragonFly*) CCSHARED="-fPIC";; Haiku*) CCSHARED="-fPIC";; OpenUNIX*|UnixWare*) - if test "$GCC" = "yes" + if test "$ac_cv_gcc_compat" = "yes" then CCSHARED="-fPIC" else CCSHARED="-KPIC" fi;; SCO_SV*) - if test "$GCC" = "yes" + if test "$ac_cv_gcc_compat" = "yes" then CCSHARED="-fPIC" else CCSHARED="-Kpic -belf" fi;; @@ -3479,13 +3615,13 @@ then then LINKFORSHARED="-Wl,--export-dynamic" fi;; - SunOS/5*) case $CC in - *gcc*) + SunOS/5*) if test "$ac_cv_gcc_compat" = "yes"; then if $CC -Xlinker --help 2>&1 | grep export-dynamic >/dev/null then LINKFORSHARED="-Xlinker --export-dynamic" - fi;; - esac;; + fi + fi + ;; CYGWIN*) if test $enable_shared = "no" then @@ -3543,7 +3679,7 @@ esac AC_MSG_RESULT([$SHLIBS]) dnl perf trampoline is Linux specific and requires an arch-specific -dnl trampoline in asssembly. +dnl trampoline in assembly. AC_MSG_CHECKING([perf trampoline]) AS_CASE([$PLATFORM_TRIPLET], [x86_64-linux-gnu], [perf_trampoline=yes], @@ -3598,7 +3734,7 @@ AS_VAR_IF([have_uuid], [missing], [ ], [ WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $LIBUUID_CFLAGS" - LDFLAGS="$LDFLAGS $LIBUUID_LIBS" + LIBS="$LIBS $LIBUUID_LIBS" AC_CHECK_HEADERS([uuid/uuid.h], [ PY_CHECK_LIB([uuid], [uuid_generate_time], [have_uuid=yes]) PY_CHECK_LIB([uuid], [uuid_generate_time_safe], @@ -3624,6 +3760,13 @@ AS_VAR_IF([have_uuid], [missing], [ ]) ]) +# gh-124228: While the libuuid library is available on NetBSD, it supports only UUID version 4. +# This restriction inhibits the proper generation of time-based UUIDs. +if test "$ac_sys_system" = "NetBSD"; then + have_uuid=missing + AC_DEFINE([HAVE_UUID_H], [0]) +fi + AS_VAR_IF([have_uuid], [missing], [have_uuid=no]) # 'Real Time' functions on Solaris @@ -3664,6 +3807,35 @@ dnl The AIX_BUILDDATE is obtained from the kernel fileset - bos.mp64 *) ;; esac +# check for _Complex C type +# +# Note that despite most compilers define __STDC_IEC_559_COMPLEX__ - almost +# none properly support C11+ Annex G (where pure imaginary types +# represented by _Imaginary are mandatory). This is a bug (see e.g. +# llvm/llvm-project#60269), so we don't rely on presence +# of __STDC_IEC_559_COMPLEX__. +AC_RUN_IFELSE([AC_LANG_SOURCE([[ +#include +#define test(type, out) \ +{ \ + type complex z = 1 + 2*I; z = z*z; \ + (out) = (out) || creal(z) != -3 || cimag(z) != 4; \ +} +int main(void) +{ + int res = 0; + test(float, res); + test(double, res); + test(long double, res); + return res; +}]])], [ac_cv_c_complex_supported=yes], +[ac_cv_c_complex_supported=no], +[ac_cv_c_complex_supported=no]) +if test "$ac_cv_c_complex_supported" = "yes"; then + AC_DEFINE([Py_HAVE_C_COMPLEX], [1], + [Defined if _Complex C type is available.]) +fi + # check for systems that require aligned memory access AC_CACHE_CHECK([aligned memory access is required], [ac_cv_aligned_required], [AC_RUN_IFELSE([AC_LANG_SOURCE([[ @@ -3832,7 +4004,7 @@ AS_VAR_IF([have_libffi], [missing], [ PKG_CHECK_MODULES([LIBFFI], [libffi], [have_libffi=yes], [ WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $LIBFFI_CFLAGS" - LDFLAGS="$LDFLAGS $LIBFFI_LIBS" + LIBS="$LIBS $LIBFFI_LIBS" AC_CHECK_HEADER([ffi.h], [ AC_CHECK_LIB([ffi], [ffi_call], [ have_libffi=yes @@ -3866,8 +4038,8 @@ AS_VAR_IF([have_libffi], [yes], [ AS_VAR_IF([ac_cv_lib_dl_dlopen], [yes], [AS_VAR_APPEND([LIBFFI_LIBS], [" -ldl"])]) WITH_SAVE_ENV([ - CFLAGS="$LIBFFI_CFLAGS $CFLAGS" - LDFLAGS="$LIBFFI_LIBS $LDFLAGS" + CFLAGS="$CFLAGS $LIBFFI_CFLAGS" + LIBS="$LIBS $LIBFFI_LIBS" PY_CHECK_FUNC([ffi_prep_cif_var], [@%:@include ]) PY_CHECK_FUNC([ffi_prep_closure_loc], [@%:@include ]) @@ -3875,35 +4047,104 @@ AS_VAR_IF([have_libffi], [yes], [ ]) ]) +# Check for libffi with real complex double support. +# This is a workaround, since FFI_TARGET_HAS_COMPLEX_TYPE was defined in libffi v3.2.1, +# but real support was provided only in libffi v3.3.0. +# See https://github.com/python/cpython/issues/125206 for more details. +# +AC_CACHE_CHECK([libffi has complex type support], [ac_cv_ffi_complex_double_supported], +[WITH_SAVE_ENV([ + CPPFLAGS="$CPPFLAGS $LIBFFI_CFLAGS" + LIBS="$LIBS $LIBFFI_LIBS" +AC_RUN_IFELSE([AC_LANG_SOURCE([[ +#include +#include +int z_is_expected(double complex z) +{ + const double complex expected = CMPLX(1.25, -0.5); + return z == expected; +} +int main(void) +{ + double complex z = 1.25 - 0.5 * I; + ffi_type *args[1] = {&ffi_type_complex_double}; + void *values[1] = {&z}; + ffi_cif cif; + if (ffi_prep_cif(&cif, FFI_DEFAULT_ABI, 1, + &ffi_type_sint, args) != FFI_OK) + { + return 2; + } + ffi_arg rc; + ffi_call(&cif, FFI_FN(z_is_expected), &rc, values); + return !rc; +} +]])], [ac_cv_ffi_complex_double_supported=yes], +[ac_cv_ffi_complex_double_supported=no], +[ac_cv_ffi_complex_double_supported=no]) +])]) +if test "$ac_cv_ffi_complex_double_supported" = "yes"; then + AC_DEFINE([Py_FFI_SUPPORT_C_COMPLEX], [1], + [Defined if _Complex C type can be used with libffi.]) +fi + # Check for use of the system libmpdec library AC_MSG_CHECKING([for --with-system-libmpdec]) AC_ARG_WITH( [system_libmpdec], [AS_HELP_STRING( [--with-system-libmpdec], - [build _decimal module using an installed libmpdec library, see Doc/library/decimal.rst (default is no)] + [build _decimal module using an installed mpdecimal library, see Doc/library/decimal.rst (default is yes)] )], [], - [with_system_libmpdec="no"]) + [with_system_libmpdec="yes"]) AC_MSG_RESULT([$with_system_libmpdec]) -AS_VAR_IF([with_system_libmpdec], [yes], [ - LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} - LIBMPDEC_LDFLAGS=${LIBMPDEC_LDFLAGS-"-lmpdec"} - LIBMPDEC_INTERNAL= -], [ - LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" - LIBMPDEC_LDFLAGS="-lm \$(LIBMPDEC_A)" - LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" - - dnl Disable forced inlining in debug builds, see GH-94847 - AS_VAR_IF([with_pydebug], [yes], [ - AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DTEST_COVERAGE"]) - ]) -]) - -AC_SUBST([LIBMPDEC_CFLAGS]) -AC_SUBST([LIBMPDEC_INTERNAL]) +AC_DEFUN([USE_BUNDLED_LIBMPDEC], + [LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" + LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" + LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" + have_mpdec=yes + with_system_libmpdec=no]) + +AS_VAR_IF( + [with_system_libmpdec], [yes], + [PKG_CHECK_MODULES( + [LIBMPDEC], [libmpdec >= 2.5.0], [], + [LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} + LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} + LIBMPDEC_INTERNAL=])], + [USE_BUNDLED_LIBMPDEC()]) + +AS_VAR_IF([with_system_libmpdec], [yes], + [WITH_SAVE_ENV([ + CPPFLAGS="$CPPFLAGS $LIBMPDEC_CFLAGS" + LIBS="$LIBS $LIBMPDEC_LIBS" + + AC_LINK_IFELSE([ + AC_LANG_PROGRAM([ + #include + #if MPD_VERSION_HEX < 0x02050000 + # error "mpdecimal 2.5.0 or higher required" + #endif + ], [const char *x = mpd_version();])], + [have_mpdec=yes], + [have_mpdec=no]) + ])], + [AC_MSG_WARN([m4_normalize([ + the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; + consider using a system installed mpdecimal library.])])]) + +AS_IF([test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no"], + [AC_MSG_WARN([m4_normalize([ + no system libmpdecimal found; falling back to bundled libmpdecimal + (deprecated and scheduled for removal in Python 3.15)])]) + USE_BUNDLED_LIBMPDEC()]) + +# Disable forced inlining in debug builds, see GH-94847 +AS_VAR_IF( + [with_pydebug], [yes], + [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DTEST_COVERAGE"])]) # Check whether _decimal should use a coroutine-local or thread-local context AC_MSG_CHECKING([for --with-decimal-contextvar]) @@ -3924,50 +4165,52 @@ fi AC_MSG_RESULT([$with_decimal_contextvar]) -# Check for libmpdec machine flavor -AC_MSG_CHECKING([for decimal libmpdec machine]) -AS_CASE([$ac_sys_system], - [Darwin*], [libmpdec_system=Darwin], - [SunOS*], [libmpdec_system=sunos], - [libmpdec_system=other] -) - -libmpdec_machine=unknown -if test "$libmpdec_system" = Darwin; then - # universal here means: build libmpdec with the same arch options - # the python interpreter was built with - libmpdec_machine=universal -elif test $ac_cv_sizeof_size_t -eq 8; then - if test "$ac_cv_gcc_asm_for_x64" = yes; then - libmpdec_machine=x64 - elif test "$ac_cv_type___uint128_t" = yes; then - libmpdec_machine=uint128 - else - libmpdec_machine=ansi64 - fi -elif test $ac_cv_sizeof_size_t -eq 4; then - if test "$ac_cv_gcc_asm_for_x87" = yes -a "$libmpdec_system" != sunos; then - AS_CASE([$CC], - [*gcc*], [libmpdec_machine=ppro], - [*clang*], [libmpdec_machine=ppro], - [libmpdec_machine=ansi32] - ) - else - libmpdec_machine=ansi32 - fi -fi -AC_MSG_RESULT([$libmpdec_machine]) - -AS_CASE([$libmpdec_machine], - [x64], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DASM=1"])], - [uint128], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DANSI=1 -DHAVE_UINT128_T=1"])], - [ansi64], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DANSI=1"])], - [ppro], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1 -DASM=1 -Wno-unknown-pragmas"])], - [ansi32], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1"])], - [ansi-legacy], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1 -DLEGACY_COMPILER=1"])], - [universal], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DUNIVERSAL=1"])], - [AC_MSG_ERROR([_decimal: unsupported architecture])] -) +AS_VAR_IF( + [with_system_libmpdec], [no], + [# Check for libmpdec machine flavor + AC_MSG_CHECKING([for decimal libmpdec machine]) + AS_CASE([$ac_sys_system], + [Darwin*], [libmpdec_system=Darwin], + [SunOS*], [libmpdec_system=sunos], + [libmpdec_system=other] + ) + + libmpdec_machine=unknown + if test "$libmpdec_system" = Darwin; then + # universal here means: build libmpdec with the same arch options + # the python interpreter was built with + libmpdec_machine=universal + elif test $ac_cv_sizeof_size_t -eq 8; then + if test "$ac_cv_gcc_asm_for_x64" = yes; then + libmpdec_machine=x64 + elif test "$ac_cv_type___uint128_t" = yes; then + libmpdec_machine=uint128 + else + libmpdec_machine=ansi64 + fi + elif test $ac_cv_sizeof_size_t -eq 4; then + if test "$ac_cv_gcc_asm_for_x87" = yes -a "$libmpdec_system" != sunos; then + AS_CASE([$ac_cv_cc_name], + [*gcc*], [libmpdec_machine=ppro], + [*clang*], [libmpdec_machine=ppro], + [libmpdec_machine=ansi32] + ) + else + libmpdec_machine=ansi32 + fi + fi + AC_MSG_RESULT([$libmpdec_machine]) + + AS_CASE([$libmpdec_machine], + [x64], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DASM=1"])], + [uint128], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DANSI=1 -DHAVE_UINT128_T=1"])], + [ansi64], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DANSI=1"])], + [ppro], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1 -DASM=1 -Wno-unknown-pragmas"])], + [ansi32], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1"])], + [ansi-legacy], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1 -DLEGACY_COMPILER=1"])], + [universal], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DUNIVERSAL=1"])], + [AC_MSG_ERROR([_decimal: unsupported architecture])] + )]) if test "$have_ipa_pure_const_bug" = yes; then # Some versions of gcc miscompile inline asm: @@ -3982,6 +4225,9 @@ if test "$have_glibc_memmove_bug" = yes; then AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -U_FORTIFY_SOURCE"]) fi +AC_SUBST([LIBMPDEC_CFLAGS]) +AC_SUBST([LIBMPDEC_INTERNAL]) + dnl detect sqlite3 from Emscripten emport PY_CHECK_EMSCRIPTEN_PORT([LIBSQLITE3], [-sUSE_SQLITE3]) @@ -4006,7 +4252,7 @@ WITH_SAVE_ENV([ dnl bpo-45774/GH-29507: The CPP check in AC_CHECK_HEADER can fail on FreeBSD, dnl hence CPPFLAGS instead of CFLAGS. CPPFLAGS="$CPPFLAGS $LIBSQLITE3_CFLAGS" - LDFLAGS="$LIBSQLITE3_LIBS $LDFLAGS" + LIBS="$LIBS $LIBSQLITE3_LIBS" AC_CHECK_HEADER([sqlite3.h], [ have_sqlite3=yes @@ -4109,7 +4355,7 @@ AS_CASE([$ac_sys_system], WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $TCLTK_CFLAGS" - LIBS="$TCLTK_LIBS $LDFLAGS" + LIBS="$LIBS $TCLTK_LIBS" AC_LINK_IFELSE([ AC_LANG_PROGRAM([ @@ -4151,7 +4397,7 @@ AC_ARG_VAR([GDBM_CFLAGS], [C compiler flags for gdbm]) AC_ARG_VAR([GDBM_LIBS], [additional linker flags for gdbm]) WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $GDBM_CFLAGS" - LDFLAGS="$GDBM_LIBS $LDFLAGS" + LIBS="$LIBS $GDBM_LIBS" AC_CHECK_HEADERS([gdbm.h], [ AC_CHECK_LIB([gdbm], [gdbm_open], [ have_gdbm=yes @@ -4705,7 +4951,7 @@ elif test "$disable_gil" = "yes"; then fi AC_MSG_RESULT([$with_mimalloc]) -AC_SUBST([WITH_MIMALLOC]) +AC_SUBST([INSTALL_MIMALLOC], [$with_mimalloc]) AC_SUBST([MIMALLOC_HEADERS]) # Check for Python-specific malloc support @@ -4730,24 +4976,6 @@ then fi AC_MSG_RESULT([$with_pymalloc]) -# Check whether objects such as float, tuple and dict are using -# freelists to optimization memory allocation. -AC_MSG_CHECKING([for --with-freelists]) -AC_ARG_WITH( - [freelists], - [AS_HELP_STRING([--with-freelists], [enable object freelists (default is yes)])]) - -if test -z "$with_freelists" -then - with_freelists="yes" -fi -if test "$with_freelists" != "no" -then - AC_DEFINE([WITH_FREELISTS], [1], - [Define if you want to compile in object freelists optimization]) -fi -AC_MSG_RESULT([$with_freelists]) - # Check for --with-c-locale-coercion AC_MSG_CHECKING([for --with-c-locale-coercion]) AC_ARG_WITH( @@ -4888,13 +5116,21 @@ fi if test "$ac_sys_system" = "Linux-android"; then # When these functions are used in an unprivileged process, they crash rather # than returning an error. - privileged_funcs="chroot initgroups setegid seteuid setgid setregid setresgid - setresuid setreuid setuid" - - # These functions are unimplemented and always return an error. - unimplemented_funcs="sem_open sem_unlink" + blocked_funcs="chroot initgroups setegid seteuid setgid sethostname + setregid setresgid setresuid setreuid setuid" + + # These functions are unimplemented and always return an error + # (https://android.googlesource.com/platform/system/sepolicy/+/refs/heads/android13-release/public/domain.te#1044) + blocked_funcs="$blocked_funcs sem_open sem_unlink" + + # Before API level 23, when fchmodat is called with the unimplemented flag + # AT_SYMLINK_NOFOLLOW, instead of returning ENOTSUP as it should, it actually + # follows the symlink. + if test "$ANDROID_API_LEVEL" -lt 23; then + blocked_funcs="$blocked_funcs fchmodat" + fi - for name in $privileged_funcs $unimplemented_funcs; do + for name in $blocked_funcs; do AS_VAR_PUSHDEF([func_var], [ac_cv_func_$name]) AS_VAR_SET([func_var], [no]) AS_VAR_POPDEF([func_var]) @@ -4904,7 +5140,7 @@ fi # checks for library functions AC_CHECK_FUNCS([ \ accept4 alarm bind_textdomain_codeset chmod chown clock closefrom close_range confstr \ - copy_file_range ctermid dup dup3 execv explicit_bzero explicit_memset \ + copy_file_range ctermid dladdr dup dup3 execv explicit_bzero explicit_memset \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ gai_strerror getegid geteuid getgid getgrent getgrgid getgrgid_r \ @@ -4916,8 +5152,10 @@ AC_CHECK_FUNCS([ \ mknod mknodat mktime mmap mremap nice openat opendir pathconf pause pipe \ pipe2 plock poll posix_fadvise posix_fallocate posix_openpt posix_spawn posix_spawnp \ posix_spawn_file_actions_addclosefrom_np \ - pread preadv preadv2 process_vm_readv pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ - pthread_kill ptsname ptsname_r pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ + pread preadv preadv2 process_vm_readv \ + pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ + pthread_kill pthread_getname_np pthread_setname_np \ + ptsname ptsname_r pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ rtpSpawn sched_get_priority_max sched_rr_get_interval sched_setaffinity \ sched_setparam sched_setscheduler sem_clockwait sem_getvalue sem_open \ sem_timedwait sem_unlink sendfile setegid seteuid setgid sethostname \ @@ -4926,7 +5164,7 @@ AC_CHECK_FUNCS([ \ sigfillset siginterrupt sigpending sigrelse sigtimedwait sigwait \ sigwaitinfo snprintf splice strftime strlcpy strsignal symlinkat sync \ sysconf tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \ - tmpnam tmpnam_r truncate ttyname umask uname unlinkat unlockpt utimensat utimes vfork \ + tmpnam tmpnam_r truncate ttyname_r umask uname unlinkat unlockpt utimensat utimes vfork \ wait wait3 wait4 waitid waitpid wcscoll wcsftime wcsxfrm wmemcmp writev \ ]) @@ -5037,10 +5275,13 @@ AS_VAR_IF([ac_cv_broken_unsetenv], [yes], [ dnl check for true AC_CHECK_PROGS([TRUE], [true], [/bin/true]) -dnl On some systems (e.g. Solaris 9), hstrerror and inet_aton are in -lresolv +dnl On some systems (e.g. Solaris), hstrerror and inet_aton are in -lresolv dnl On others, they are in the C library, so we to take no action AC_CHECK_LIB([c], [inet_aton], [$ac_cv_prog_TRUE], - AC_CHECK_LIB([resolv], [inet_aton]) + AC_CHECK_LIB([resolv], [inet_aton], [SOCKET_LIBS="-lresolv"]) +) +AC_CHECK_LIB([c], [hstrerror], [$ac_cv_prog_TRUE], + AC_CHECK_LIB([resolv], [hstrerror], [SOCKET_LIBS="-lresolv"]) ) # On Tru64, chflags seems to be present, but calling it will @@ -5101,7 +5342,7 @@ PKG_CHECK_MODULES([ZLIB], [zlib >= 1.2.0], [ ], [ WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $ZLIB_CFLAGS" - LDFLAGS="$LDFLAGS $ZLIB_LIBS" + LIBS="$LIBS $ZLIB_LIBS" AC_CHECK_HEADERS([zlib.h], [ PY_CHECK_LIB([z], [gzread], [have_zlib=yes], [have_zlib=no]) ], [have_zlib=no]) @@ -5125,7 +5366,7 @@ PY_CHECK_EMSCRIPTEN_PORT([BZIP2], [-sUSE_BZIP2]) PKG_CHECK_MODULES([BZIP2], [bzip2], [have_bzip2=yes], [ WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $BZIP2_CFLAGS" - LDFLAGS="$LDFLAGS $BZIP2_LIBS" + LIBS="$LIBS $BZIP2_LIBS" AC_CHECK_HEADERS([bzlib.h], [ AC_CHECK_LIB([bz2], [BZ2_bzCompress], [have_bzip2=yes], [have_bzip2=no]) ], [have_bzip2=no]) @@ -5139,7 +5380,7 @@ PKG_CHECK_MODULES([BZIP2], [bzip2], [have_bzip2=yes], [ PKG_CHECK_MODULES([LIBLZMA], [liblzma], [have_liblzma=yes], [ WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $LIBLZMA_CFLAGS" - LDFLAGS="$LDFLAGS $LIBLZMA_LIBS" + LIBS="$LIBS $LIBLZMA_LIBS" AC_CHECK_HEADERS([lzma.h], [ AC_CHECK_LIB([lzma], [lzma_easy_encoder], [have_liblzma=yes], [have_liblzma=no]) ], [have_liblzma=no]) @@ -5257,11 +5498,16 @@ then ]) fi -AC_CHECK_FUNCS([clock_nanosleep], [], [ - AC_CHECK_LIB([rt], [clock_nanosleep], [ - AC_DEFINE([HAVE_CLOCK_NANOSLEEP], [1]) - ]) -]) +# On Android before API level 23, clock_nanosleep returns the wrong value when +# interrupted by a signal (https://issuetracker.google.com/issues/216495770). +if ! { test "$ac_sys_system" = "Linux-android" && + test "$ANDROID_API_LEVEL" -lt 23; }; then + AC_CHECK_FUNCS([clock_nanosleep], [], [ + AC_CHECK_LIB([rt], [clock_nanosleep], [ + AC_DEFINE([HAVE_CLOCK_NANOSLEEP], [1]) + ]) + ]) +fi AC_CHECK_FUNCS([nanosleep], [], [ AC_CHECK_LIB([rt], [nanosleep], [ @@ -5664,28 +5910,28 @@ AS_VAR_IF([ac_cv_gcc_asm_for_x64], [yes], [ # * Check for various properties of floating point * # ************************************************** -AX_C_FLOAT_WORDS_BIGENDIAN -if test "$ax_cv_c_float_words_bigendian" = "yes" -then - AC_DEFINE([DOUBLE_IS_BIG_ENDIAN_IEEE754], [1], - [Define if C doubles are 64-bit IEEE 754 binary format, stored - with the most significant byte first]) -elif test "$ax_cv_c_float_words_bigendian" = "no" -then - AC_DEFINE([DOUBLE_IS_LITTLE_ENDIAN_IEEE754], [1], - [Define if C doubles are 64-bit IEEE 754 binary format, stored - with the least significant byte first]) -else - # Some ARM platforms use a mixed-endian representation for doubles. - # While Python doesn't currently have full support for these platforms - # (see e.g., issue 1762561), we can at least make sure that float <-> string - # conversions work. - # FLOAT_WORDS_BIGENDIAN doesnt actually detect this case, but if it's not big - # or little, then it must be this? - AC_DEFINE([DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754], [1], - [Define if C doubles are 64-bit IEEE 754 binary format, stored - in ARM mixed-endian order (byte order 45670123)]) -fi +AX_C_FLOAT_WORDS_BIGENDIAN( + [AC_DEFINE([DOUBLE_IS_BIG_ENDIAN_IEEE754], [1], + [Define if C doubles are 64-bit IEEE 754 binary format, + stored with the most significant byte first])], + [AC_DEFINE([DOUBLE_IS_LITTLE_ENDIAN_IEEE754], [1], + [Define if C doubles are 64-bit IEEE 754 binary format, + stored with the least significant byte first])], + [AS_CASE([$host_cpu], + [*arm*], [# Some ARM platforms use a mixed-endian representation for + # doubles. While Python doesn't currently have full support + # for these platforms (see e.g., issue 1762561), we can at + # least make sure that float <-> string conversions work. + # FLOAT_WORDS_BIGENDIAN doesn't actually detect this case, + # but if it's not big or little, then it must be this? + AC_DEFINE([DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754], [1], + [Define if C doubles are 64-bit IEEE 754 binary format, + stored in ARM mixed-endian order (byte order 45670123)])], + [AC_MSG_ERROR([m4_normalize([ + Unknown float word ordering. You need to manually + preset ax_cv_c_float_words_bigendian=no (or yes) + according to your system. + ])])])]) # The short float repr introduced in Python 3.1 requires the # correctly-rounded string <-> double conversion functions from @@ -5979,30 +6225,33 @@ AC_MSG_CHECKING([LDVERSION]) LDVERSION='$(VERSION)$(ABIFLAGS)' AC_MSG_RESULT([$LDVERSION]) -# Configure the flags and dependencies used when compiling shared modules +# Configure the flags and dependencies used when compiling shared modules. +# Do not rename LIBPYTHON - it's accessed via sysconfig by package build +# systems (e.g. Meson) to decide whether to link extension modules against +# libpython. AC_SUBST([MODULE_DEPS_SHARED]) -AC_SUBST([MODULE_LDFLAGS]) +AC_SUBST([LIBPYTHON]) MODULE_DEPS_SHARED='$(MODULE_DEPS_STATIC) $(EXPORTSYMS)' -MODULE_LDFLAGS='' +LIBPYTHON='' # On Android and Cygwin the shared libraries must be linked with libpython. if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(LDLIBRARY)" - MODULE_LDFLAGS="\$(BLDLIBRARY)" + LIBPYTHON="\$(BLDLIBRARY)" fi # On iOS the shared libraries must be linked with the Python framework -if test "$ac_sys_system" == "iOS"; then +if test "$ac_sys_system" = "iOS"; then MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(PYTHONFRAMEWORKDIR)/\$(PYTHONFRAMEWORK)" fi AC_SUBST([BINLIBDEST]) -BINLIBDEST='$(LIBDIR)/python$(VERSION)' +BINLIBDEST='$(LIBDIR)/python$(VERSION)$(ABI_THREAD)' # Check for --with-platlibdir -# /usr/$LIDIRNAME/python$VERSION +# /usr/$PLATLIBDIR/python$(VERSION)$(ABI_THREAD) AC_SUBST([PLATLIBDIR]) PLATLIBDIR="lib" AC_MSG_CHECKING([for --with-platlibdir]) @@ -6021,7 +6270,7 @@ if test -n "$withval" -a "$withval" != yes -a "$withval" != no then AC_MSG_RESULT([yes]) PLATLIBDIR="$withval" - BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)' + BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)$(ABI_THREAD)' else AC_MSG_RESULT([no]) fi], @@ -6031,9 +6280,9 @@ fi], dnl define LIBPL after ABIFLAGS and LDVERSION is defined. AC_SUBST([PY_ENABLE_SHARED]) if test x$PLATFORM_TRIPLET = x; then - LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}" + LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}" else - LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}-${PLATFORM_TRIPLET}" + LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}-${PLATFORM_TRIPLET}" fi AC_SUBST([LIBPL]) @@ -6125,7 +6374,7 @@ AS_VAR_IF([with_readline], [readline], [ ], [ WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $LIBREADLINE_CFLAGS" - LDFLAGS="$LDFLAGS $LIBREADLINE_LIBS" + LIBS="$LIBS $LIBREADLINE_LIBS" AC_CHECK_HEADERS([readline/readline.h], [ AC_CHECK_LIB([readline], [readline], [ LIBREADLINE=readline @@ -6146,7 +6395,7 @@ AS_VAR_IF([with_readline], [edit], [ ], [ WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $LIBEDIT_CFLAGS" - LDFLAGS="$LDFLAGS $LIBEDIT_LIBS" + LIBS="$LIBS $LIBEDIT_LIBS" AC_CHECK_HEADERS([editline/readline.h], [ AC_CHECK_LIB([edit], [readline], [ LIBREADLINE=edit @@ -6170,7 +6419,7 @@ AS_VAR_IF([with_readline], [no], [ WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $READLINE_CFLAGS" - LIBS="$READLINE_LIBS $LIBS" + LIBS="$LIBS $READLINE_LIBS" LIBS_SAVE=$LIBS m4_define([readline_includes], [ @@ -6254,6 +6503,21 @@ AS_VAR_IF([with_readline], [no], [ # in readline as well as newer editline (April 2023) AC_CHECK_TYPES([rl_compdisp_func_t], [], [], [readline_includes]) + # Some editline versions declare rl_startup_hook as taking no args, others + # declare it as taking 2. + AC_CACHE_CHECK([if rl_startup_hook takes arguments], [ac_cv_readline_rl_startup_hook_takes_args], [ + AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([readline_includes] + [extern int test_hook_func(const char *text, int state);], + [rl_startup_hook=test_hook_func;])], + [ac_cv_readline_rl_startup_hook_takes_args=yes], + [ac_cv_readline_rl_startup_hook_takes_args=no] + ) + ]) + AS_VAR_IF([ac_cv_readline_rl_startup_hook_takes_args], [yes], [ + AC_DEFINE([Py_RL_STARTUP_HOOK_TAKES_ARGS], [1], [Define if rl_startup_hook takes arguments]) + ]) + m4_undefine([readline_includes]) ])dnl WITH_SAVE_ENV() ]) @@ -6412,55 +6676,139 @@ then [Define if you have struct stat.st_mtimensec]) fi -dnl check for ncurses/ncursesw and panel/panelw +AC_CACHE_CHECK([whether year with century should be normalized for strftime], [ac_cv_normalize_century], [ +AC_RUN_IFELSE([AC_LANG_SOURCE([[ +#include +#include + +int main(void) +{ + char year[5]; + struct tm date = { + .tm_year = -1801, + .tm_mon = 0, + .tm_mday = 1 + }; + if (strftime(year, sizeof(year), "%Y", &date) && !strcmp(year, "0099")) { + return 1; + } + return 0; +} +]])], +[ac_cv_normalize_century=yes], +[ac_cv_normalize_century=no], +[ac_cv_normalize_century=yes])]) +if test "$ac_cv_normalize_century" = yes +then + AC_DEFINE([Py_NORMALIZE_CENTURY], [1], + [Define if year with century should be normalized for strftime.]) +fi + +AC_CACHE_CHECK([whether C99-compatible strftime specifiers are supported], [ac_cv_strftime_c99_support], [ +AC_RUN_IFELSE([AC_LANG_SOURCE([[ +#include +#include + +int main(void) +{ + char full_date[11]; + struct tm date = { + .tm_year = 0, + .tm_mon = 0, + .tm_mday = 1 + }; + if (strftime(full_date, sizeof(full_date), "%F", &date) && !strcmp(full_date, "1900-01-01")) { + return 0; + } + return 1; +} +]])], +[ac_cv_strftime_c99_support=yes], +[AC_MSG_ERROR([Python requires C99-compatible strftime specifiers])], +[ac_cv_strftime_c99_support=])]) + +dnl check for ncursesw/ncurses and panelw/panel dnl NOTE: old curses is not detected. -dnl have_curses=[no, ncursesw, ncurses] -dnl have_panel=[no, panelw, panel] +dnl have_curses=[no, yes] +dnl have_panel=[no, yes] have_curses=no have_panel=no -AH_TEMPLATE([HAVE_NCURSESW], [Define to 1 if you have the `ncursesw' library.]) -AC_CHECK_HEADERS([curses.h ncurses.h]) +dnl PY_CHECK_CURSES(LIBCURSES, LIBPANEL) +dnl Sets 'have_curses' and 'have_panel'. +dnl For the PKG_CHECK_MODULES() calls, we can safely reuse the first variable +dnl here, since we're only calling the macro a second time if the first call +dnl fails. +AC_DEFUN([PY_CHECK_CURSES], [dnl +AS_VAR_PUSHDEF([curses_var], [m4_toupper([$1])]) +AS_VAR_PUSHDEF([panel_var], [m4_toupper([$2])]) +PKG_CHECK_MODULES([CURSES], [$1], + [AC_DEFINE([HAVE_]curses_var, [1], [Define if you have the '$1' library]) + AS_VAR_SET([have_curses], [yes]) + PKG_CHECK_MODULES([PANEL], [$2], + [AC_DEFINE([HAVE_]panel_var, [1], [Define if you have the '$2' library]) + AS_VAR_SET([have_panel], [yes])], + [AS_VAR_SET([have_panel], [no])])], + [AS_VAR_SET([have_curses], [no])]) +AS_VAR_POPDEF([curses_var]) +AS_VAR_POPDEF([panel_var])]) + +# Check for ncursesw/panelw first. If that fails, try ncurses/panel. +PY_CHECK_CURSES([ncursesw], [panelw]) +AS_VAR_IF([have_curses], [no], + [PY_CHECK_CURSES([ncurses], [panel])]) -AS_VAR_IF([ac_cv_header_ncurses_h], [yes], [ - if test "$ac_sys_system" != "Darwin"; then - dnl On macOS, there is no separate /usr/lib/libncursesw nor libpanelw. - PKG_CHECK_MODULES([CURSES], [ncursesw], [ - AC_DEFINE([HAVE_NCURSESW], [1]) - have_curses=ncursesw - ], [ - WITH_SAVE_ENV([ - AC_CHECK_LIB([ncursesw], [initscr], [ - AC_DEFINE([HAVE_NCURSESW], [1]) - have_curses=ncursesw - CURSES_CFLAGS=${CURSES_CFLAGS-""} - CURSES_LIBS=${CURSES_LIBS-"-lncursesw"} - ]) - ]) - ]) - fi - - AS_VAR_IF([have_curses], [no], [ - PKG_CHECK_MODULES([CURSES], [ncurses], [ - have_curses=ncurses - ], [ - WITH_SAVE_ENV([ - AC_CHECK_LIB([ncurses], [initscr], [ - have_curses=ncurses - CURSES_CFLAGS=${CURSES_CFLAGS-""} - CURSES_LIBS=${CURSES_LIBS-"-lncurses"} - ]) - ]) - ]) - ]) +WITH_SAVE_ENV([ + # Make sure we've got the header defines. + AS_VAR_APPEND([CPPFLAGS], [" $CURSES_CFLAGS $PANEL_CFLAGS"]) + AC_CHECK_HEADERS(m4_normalize([ + ncursesw/curses.h ncursesw/ncurses.h ncursesw/panel.h + ncurses/curses.h ncurses/ncurses.h ncurses/panel.h + curses.h ncurses.h panel.h + ])) -])dnl ac_cv_header_ncurses_h = yes + # Check that we're able to link with crucial curses/panel functions. This + # also serves as a fallback in case pkg-config failed. + AS_VAR_APPEND([LIBS], [" $CURSES_LIBS $PANEL_LIBS"]) + AC_SEARCH_LIBS([initscr], [ncursesw ncurses], + [AS_VAR_IF([have_curses], [no], + [AS_VAR_SET([have_curses], [yes]) + CURSES_LIBS=${CURSES_LIBS-"$ac_cv_search_initscr"}])], + [AS_VAR_SET([have_curses], [no])]) + AC_SEARCH_LIBS([update_panels], [panelw panel], + [AS_VAR_IF([have_panel], [no], + [AS_VAR_SET([have_panel], [yes]) + PANEL_LIBS=${PANEL_LIBS-"$ac_cv_search_update_panels"}])], + [AS_VAR_SET([have_panel], [no])]) + +dnl Issue #25720: ncurses has introduced the NCURSES_OPAQUE symbol making opaque +dnl structs since version 5.7. If the macro is defined as zero before including +dnl [n]curses.h, ncurses will expose fields of the structs regardless of the +dnl configuration. +AC_DEFUN([_CURSES_INCLUDES],dnl +[ +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif +]) +AS_IF([test "have_curses" != "no"], [ dnl remove _XOPEN_SOURCE macro from curses cflags. pyconfig.h sets dnl the macro to 700. CURSES_CFLAGS=$(echo $CURSES_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') -if test "$have_curses" != no -a "$ac_sys_system" = "Darwin"; then +AS_VAR_IF([ac_sys_system], [Darwin], [ dnl On macOS, there is no separate /usr/lib/libncursesw nor libpanelw. dnl System-supplied ncurses combines libncurses/libpanel and supports wide dnl characters, so we can use it like ncursesw. @@ -6470,82 +6818,17 @@ if test "$have_curses" != no -a "$ac_sys_system" = "Darwin"; then dnl _XOPEN_SOURCE_EXTENDED here for ncurses wide char support. AS_VAR_APPEND([CURSES_CFLAGS], [" -D_XOPEN_SOURCE_EXTENDED=1"]) - AC_DEFINE([HAVE_NCURSESW], [1]) -fi - -dnl TODO: detect "curses" and special cases tinfo, terminfo, or termcap - -AC_MSG_CHECKING([curses module flags]) -AS_VAR_IF([have_curses], [no], [ - AC_MSG_RESULT([no]) -], [ - AC_MSG_RESULT([$have_curses (CFLAGS: $CURSES_CFLAGS, LIBS: $CURSES_LIBS)]) ]) -dnl check for ncurses' panel/panelw library -AC_CHECK_HEADERS([panel.h]) - -AS_VAR_IF([ac_cv_header_panel_h], [yes], [ - - if test "$ac_sys_system" != "Darwin"; then - dnl On macOS, there is no separate /usr/lib/libncursesw nor libpanelw. - AS_VAR_IF([have_curses], [ncursesw], [ - PKG_CHECK_MODULES([PANEL], [panelw], [ - have_panel=panelw - ], [ - WITH_SAVE_ENV([ - AC_CHECK_LIB([panelw], [update_panels], [ - have_panel=panelw - PANEL_CFLAGS=${PANEL_CFLAGS-""} - PANEL_LIBS=${PANEL_LIBS-"-lpanelw"} - ]) - ]) - ]) - ]) - fi - - AS_VAR_IF([have_curses], [ncurses], [ - PKG_CHECK_MODULES([PANEL], [panel], [ - have_panel=panel - ], [ - WITH_SAVE_ENV([ - AC_CHECK_LIB([panel], [update_panels], [ - have_panel=panel - PANEL_CFLAGS=${PANEL_CFLAGS-""} - PANEL_LIBS=${PANEL_LIBS-"-lpanel"} - ]) - ]) - ]) - ]) - -])dnl ac_cv_header_panel_h = yes - dnl pyconfig.h defines _XOPEN_SOURCE=700 PANEL_CFLAGS=$(echo $PANEL_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') -AC_MSG_CHECKING([panel flags]) -AS_VAR_IF([have_panel], [no], [ - AC_MSG_RESULT([no]) -], [ - AC_MSG_RESULT([$have_panel (CFLAGS: $PANEL_CFLAGS, LIBS: $PANEL_LIBS)]) -]) - -# first curses header check -ac_save_cppflags="$CPPFLAGS" -if test "$cross_compiling" = no; then - CPPFLAGS="$CPPFLAGS -I/usr/include/ncursesw" -fi - # On Solaris, term.h requires curses.h -AC_CHECK_HEADERS([term.h], [], [], [ -#ifdef HAVE_CURSES_H -#include -#endif -]) +AC_CHECK_HEADERS([term.h], [], [], _CURSES_INCLUDES) # On HP/UX 11.0, mvwdelch is a block with a return statement AC_CACHE_CHECK([whether mvwdelch is an expression], [ac_cv_mvwdelch_is_expression], -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include ]], [[ +AC_COMPILE_IFELSE([AC_LANG_PROGRAM(_CURSES_INCLUDES, [[ int rtn; rtn = mvwdelch(0,0,0); ]])], @@ -6558,15 +6841,8 @@ then [Define if mvwdelch in curses.h is an expression.]) fi -# Issue #25720: ncurses has introduced the NCURSES_OPAQUE symbol making opaque -# structs since version 5.7. If the macro is defined as zero before including -# [n]curses.h, ncurses will expose fields of the structs regardless of the -# configuration. AC_CACHE_CHECK([whether WINDOW has _flags], [ac_cv_window_has_flags], -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #define NCURSES_OPAQUE 0 - #include -]], [[ +AC_COMPILE_IFELSE([AC_LANG_PROGRAM(_CURSES_INCLUDES, [[ WINDOW *w; w->_flags = 0; ]])], @@ -6588,8 +6864,7 @@ AC_DEFUN([PY_CHECK_CURSES_FUNC], [for curses function $1], [py_var], [AC_COMPILE_IFELSE( - [AC_LANG_PROGRAM( - [@%:@include ], [ + [AC_LANG_PROGRAM(_CURSES_INCLUDES, [ #ifndef $1 void *x=$1 #endif @@ -6617,6 +6892,8 @@ PY_CHECK_CURSES_FUNC([has_key]) PY_CHECK_CURSES_FUNC([typeahead]) PY_CHECK_CURSES_FUNC([use_env]) CPPFLAGS=$ac_save_cppflags +])dnl have_curses != no +])dnl save env AC_MSG_NOTICE([checking for device files]) @@ -6756,7 +7033,6 @@ done AC_SUBST([SRCDIRS]) SRCDIRS="\ Modules \ - Modules/_blake2 \ Modules/_ctypes \ Modules/_decimal \ Modules/_decimal/libmpdec \ @@ -6767,6 +7043,7 @@ SRCDIRS="\ Modules/_sre \ Modules/_testcapi \ Modules/_testinternalcapi \ + Modules/_testlimitedcapi \ Modules/_xxtestfuzz \ Modules/cjkcodecs \ Modules/expat \ @@ -6778,8 +7055,7 @@ SRCDIRS="\ Parser/lexer \ Programs \ Python \ - Python/frozen_modules \ - Python/deepfreeze" + Python/frozen_modules" AC_MSG_CHECKING([for build directories]) for dir in $SRCDIRS; do if test ! -d $dir; then @@ -6835,8 +7111,8 @@ if test "$ac_cv_gcc_asm_for_x87" = yes; then # Some versions of gcc miscompile inline asm: # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=46491 # http://gcc.gnu.org/ml/gcc/2010-11/msg00366.html - case $CC in - *gcc*) + case $ac_cv_cc_name in + gcc) AC_MSG_CHECKING([for gcc ipa-pure-const bug]) saved_cflags="$CFLAGS" CFLAGS="-O2" @@ -6880,6 +7156,7 @@ AC_ARG_WITH([ensurepip], AS_CASE([$ac_sys_system], [Emscripten], [with_ensurepip=no], [WASI], [with_ensurepip=no], + [iOS], [with_ensurepip=no], [with_ensurepip=upgrade] ) ]) @@ -7152,8 +7429,6 @@ AC_DEFINE([PY_SSL_DEFAULT_CIPHERS], [1]) # builtin hash modules default_hashlib_hashes="md5,sha1,sha2,sha3,blake2" -AC_DEFINE([PY_BUILTIN_HASHLIB_HASHES], [], [enabled builtin hash modules] -) AC_MSG_CHECKING([for --with-builtin-hashlib-hashes]) AC_ARG_WITH( [builtin-hashlib-hashes], @@ -7170,7 +7445,8 @@ AC_ARG_WITH( AC_MSG_RESULT([$with_builtin_hashlib_hashes]) AC_DEFINE_UNQUOTED([PY_BUILTIN_HASHLIB_HASHES], - ["$with_builtin_hashlib_hashes"]) + ["$with_builtin_hashlib_hashes"], + [enabled builtin hash modules]) as_save_IFS=$IFS IFS=, @@ -7185,27 +7461,13 @@ for builtin_hash in $with_builtin_hashlib_hashes; do done IFS=$as_save_IFS -dnl libb2 for blake2. _blake2 module falls back to vendored copy. -AS_VAR_IF([with_builtin_blake2], [yes], [ - PKG_CHECK_MODULES([LIBB2], [libb2], [ - have_libb2=yes - AC_DEFINE([HAVE_LIBB2], [1], - [Define to 1 if you want to build _blake2 module with libb2]) - ], [have_libb2=no]) -]) - # Check whether to disable test modules. Once set, setup.py will not build # test extension modules and "make install" will not install test suites. AC_MSG_CHECKING([for --disable-test-modules]) AC_ARG_ENABLE([test-modules], [AS_HELP_STRING([--disable-test-modules], [don't build nor install test modules])], [ AS_VAR_IF([enable_test_modules], [yes], [TEST_MODULES=yes], [TEST_MODULES=no]) -], [ - AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], - [Emscripten/browser*], [TEST_MODULES=no], - [TEST_MODULES=yes] - ) -]) +], [TEST_MODULES=yes]) AC_MSG_RESULT([$TEST_MODULES]) AC_SUBST([TEST_MODULES]) @@ -7228,7 +7490,7 @@ CPPFLAGS="${BASECPPFLAGS} -I. -I${srcdir}/Include ${CPPFLAGS}" AC_CACHE_CHECK([whether libatomic is needed by ], [ac_cv_libatomic_needed], -[AC_RUN_IFELSE([AC_LANG_SOURCE([[ +[AC_LINK_IFELSE([AC_LANG_SOURCE([[ // pyatomic.h needs uint64_t and Py_ssize_t types #include // int64_t, intptr_t #ifdef HAVE_SYS_TYPES_H @@ -7265,9 +7527,8 @@ int main() return 0; // all good } ]])], - [ac_cv_libatomic_needed=no], dnl build succeeded - [ac_cv_libatomic_needed=yes], dnl build failed - [ac_cv_libatomic_needed=no]) dnl cross compilation + [ac_cv_libatomic_needed=no], dnl build and link succeeded + [ac_cv_libatomic_needed=yes]) dnl build and link failed ]) AS_VAR_IF([ac_cv_libatomic_needed], [yes], @@ -7276,6 +7537,23 @@ AS_VAR_IF([ac_cv_libatomic_needed], [yes], _RESTORE_VAR([CPPFLAGS]) +# gh-59705: Maximum length in bytes of a thread name +case "$ac_sys_system" in + Linux*) _PYTHREAD_NAME_MAXLEN=15;; # Linux and Android + SunOS*) _PYTHREAD_NAME_MAXLEN=31;; + NetBSD*) _PYTHREAD_NAME_MAXLEN=31;; + Darwin) _PYTHREAD_NAME_MAXLEN=63;; + iOS) _PYTHREAD_NAME_MAXLEN=63;; + FreeBSD*) _PYTHREAD_NAME_MAXLEN=98;; + *) _PYTHREAD_NAME_MAXLEN=;; +esac +if test -n "$_PYTHREAD_NAME_MAXLEN"; then + AC_DEFINE_UNQUOTED([_PYTHREAD_NAME_MAXLEN], [$_PYTHREAD_NAME_MAXLEN], + [Maximum length in bytes of a thread name]) +fi +AC_SUBST([_PYTHREAD_NAME_MAXLEN]) + + # stdlib AC_DEFUN([PY_STDLIB_MOD_SET_NA], [ m4_foreach([mod], [$@], [ @@ -7314,12 +7592,11 @@ AS_CASE([$ac_sys_system], [CYGWIN*], [PY_STDLIB_MOD_SET_NA([_scproxy])], [QNX*], [PY_STDLIB_MOD_SET_NA([_scproxy])], [FreeBSD*], [PY_STDLIB_MOD_SET_NA([_scproxy])], - [Emscripten|WASI], [ + [Emscripten], [ dnl subprocess and multiprocessing are not supported (no fork syscall). dnl curses and tkinter user interface are not available. dnl dbm and gdbm aren't available, too. - dnl Emscripten and WASI provide only stubs for pwd, grp APIs. - dnl resource functions (get/setrusage) are stubs, too. + dnl pwd, grp APIs, and resource functions (get/setrusage) are stubs. PY_STDLIB_MOD_SET_NA( [_curses], [_curses_panel], @@ -7330,33 +7607,58 @@ AS_CASE([$ac_sys_system], [_posixsubprocess], [_scproxy], [_tkinter], - [_xxsubinterpreters], - [_xxinterpchannels], - [_xxinterpqueues], + [_interpreters], + [_interpchannels], + [_interpqueues], [grp], [pwd], [resource], [syslog], ) - AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], - [Emscripten/browser*], [ - dnl These modules are not particularly useful in browsers. - PY_STDLIB_MOD_SET_NA( - [fcntl], - [readline], - [termios], - ) - ], - [Emscripten/node*], [], - [WASI/*], [ - dnl WASI SDK 15.0 does not support file locking, mmap, and more. - PY_STDLIB_MOD_SET_NA( - [_ctypes_test], - [fcntl], - [mmap], - [termios], - ) - ] + dnl fcntl, readline, and termios are not particularly useful in browsers. + PY_STDLIB_MOD_SET_NA( + [fcntl], + [readline], + [termios], + ) + ], + [WASI], [ + dnl subprocess and multiprocessing are not supported (no fork syscall). + dnl curses and tkinter user interface are not available. + dnl dbm and gdbm aren't available, too. + dnl pwd, grp APIs, and resource functions (get/setrusage) are stubs. + PY_STDLIB_MOD_SET_NA( + [_curses], + [_curses_panel], + [_dbm], + [_gdbm], + [_multiprocessing], + [_posixshmem], + [_posixsubprocess], + [_scproxy], + [_tkinter], + [_interpreters], + [_interpchannels], + [_interpqueues], + [grp], + [pwd], + [resource], + [syslog], + ) + dnl WASI SDK 15.0 does not support file locking, mmap, and more. + dnl Test modules that must be compiled as shared libraries are not supported + dnl (see Modules/Setup.stdlib.in). + PY_STDLIB_MOD_SET_NA( + [_ctypes_test], + [_testexternalinspection], + [_testimportmultiple], + [_testmultiphase], + [_testsinglephase], + [fcntl], + [mmap], + [termios], + [xxlimited], + [xxlimited_35], ) ], [PY_STDLIB_MOD_SET_NA([_scproxy])] @@ -7439,7 +7741,6 @@ PY_STDLIB_MOD_SIMPLE([_csv]) PY_STDLIB_MOD_SIMPLE([_heapq]) PY_STDLIB_MOD_SIMPLE([_json]) PY_STDLIB_MOD_SIMPLE([_lsprof]) -PY_STDLIB_MOD_SIMPLE([_opcode]) PY_STDLIB_MOD_SIMPLE([_pickle]) PY_STDLIB_MOD_SIMPLE([_posixsubprocess]) PY_STDLIB_MOD_SIMPLE([_queue]) @@ -7447,9 +7748,9 @@ PY_STDLIB_MOD_SIMPLE([_random]) PY_STDLIB_MOD_SIMPLE([select]) PY_STDLIB_MOD_SIMPLE([_struct]) PY_STDLIB_MOD_SIMPLE([_typing]) -PY_STDLIB_MOD_SIMPLE([_xxsubinterpreters]) -PY_STDLIB_MOD_SIMPLE([_xxinterpchannels]) -PY_STDLIB_MOD_SIMPLE([_xxinterpqueues]) +PY_STDLIB_MOD_SIMPLE([_interpreters]) +PY_STDLIB_MOD_SIMPLE([_interpchannels]) +PY_STDLIB_MOD_SIMPLE([_interpqueues]) PY_STDLIB_MOD_SIMPLE([_zoneinfo]) dnl multiprocessing modules @@ -7477,7 +7778,7 @@ PY_STDLIB_MOD([mmap], PY_STDLIB_MOD([_socket], [], m4_flatten([test "$ac_cv_header_sys_socket_h" = "yes" -a "$ac_cv_header_sys_types_h" = "yes" - -a "$ac_cv_header_netinet_in_h" = "yes"])) + -a "$ac_cv_header_netinet_in_h" = "yes"]), [], [$SOCKET_LIBS]) dnl platform specific extensions PY_STDLIB_MOD([grp], [], @@ -7507,32 +7808,92 @@ PY_STDLIB_MOD_SIMPLE([unicodedata]) dnl By default we always compile these even when OpenSSL is available dnl (issue #14693). The modules are small. -PY_STDLIB_MOD([_md5], - [test "$with_builtin_md5" = yes], [], - [-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE]) -PY_STDLIB_MOD([_sha1], - [test "$with_builtin_sha1" = yes], [], - [-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE]) -PY_STDLIB_MOD([_sha2], - [test "$with_builtin_sha2" = yes], [], - [-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE]) +PY_STDLIB_MOD([_md5], [test "$with_builtin_md5" = yes]) +PY_STDLIB_MOD([_sha1], [test "$with_builtin_sha1" = yes]) +PY_STDLIB_MOD([_sha2], [test "$with_builtin_sha2" = yes]) PY_STDLIB_MOD([_sha3], [test "$with_builtin_sha3" = yes]) -PY_STDLIB_MOD([_blake2], - [test "$with_builtin_blake2" = yes], [], - [$LIBB2_CFLAGS], [$LIBB2_LIBS]) +PY_STDLIB_MOD([_blake2], [test "$with_builtin_blake2" = yes]) + +LIBHACL_CFLAGS='-I$(srcdir)/Modules/_hacl -I$(srcdir)/Modules/_hacl/include -D_BSD_SOURCE -D_DEFAULT_SOURCE $(PY_STDMODULE_CFLAGS) $(CCSHARED)' +case "$ac_sys_system" in + Linux*) + if test "$ac_cv_func_explicit_bzero" = "no"; then + LIBHACL_CFLAGS="$LIBHACL_CFLAGS -DLINUX_NO_EXPLICIT_BZERO" + fi + ;; +esac +AC_SUBST([LIBHACL_CFLAGS]) + +# The SIMD files use aligned_alloc, which is not available on older versions of +# Android. +# The *mmintrin.h headers are x86-family-specific, so can't be used on WASI. +if test "$ac_sys_system" != "Linux-android" -a "$ac_sys_system" != "WASI" || test "$ANDROID_API_LEVEL" -ge 28; then + dnl This can be extended here to detect e.g. Power8, which HACL* should also support. + AX_CHECK_COMPILE_FLAG([-msse -msse2 -msse3 -msse4.1 -msse4.2],[ + [LIBHACL_SIMD128_FLAGS="-msse -msse2 -msse3 -msse4.1 -msse4.2"] + + AC_DEFINE([HACL_CAN_COMPILE_SIMD128], [1], [HACL* library can compile SIMD128 implementations]) + + # macOS universal2 builds *support* the -msse etc flags because they're + # available on x86_64. However, performance of the HACL SIMD128 implementation + # isn't great, so it's disabled on ARM64. + AC_MSG_CHECKING([for HACL* SIMD128 implementation]) + if test "$UNIVERSAL_ARCHS" == "universal2"; then + [LIBHACL_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o"] + AC_MSG_RESULT([universal2]) + else + [LIBHACL_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128.o"] + AC_MSG_RESULT([standard]) + fi + + ], [], [-Werror]) +fi +AC_SUBST([LIBHACL_SIMD128_FLAGS]) +AC_SUBST([LIBHACL_SIMD128_OBJS]) + +# The SIMD files use aligned_alloc, which is not available on older versions of +# Android. +# The *mmintrin.h headers are x86-family-specific, so can't be used on WASI. +# +# Although AVX support is not guaranteed on Android +# (https://developer.android.com/ndk/guides/abis#86-64), this is safe because we do a +# runtime CPUID check. +if test "$ac_sys_system" != "Linux-android" -a "$ac_sys_system" != "WASI" || test "$ANDROID_API_LEVEL" -ge 28; then + AX_CHECK_COMPILE_FLAG([-mavx2],[ + [LIBHACL_SIMD256_FLAGS="-mavx2"] + AC_DEFINE([HACL_CAN_COMPILE_SIMD256], [1], [HACL* library can compile SIMD256 implementations]) + + # macOS universal2 builds *support* the -mavx2 compiler flag because it's + # available on x86_64; but the HACL SIMD256 build then fails because the + # implementation requires symbols that aren't available on ARM64. Use a + # wrapped implementation if we're building for universal2. + AC_MSG_CHECKING([for HACL* SIMD256 implementation]) + if test "$UNIVERSAL_ARCHS" == "universal2"; then + [LIBHACL_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o"] + AC_MSG_RESULT([universal2]) + else + [LIBHACL_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256.o"] + AC_MSG_RESULT([standard]) + fi + ], [], [-Werror]) +fi +AC_SUBST([LIBHACL_SIMD256_FLAGS]) +AC_SUBST([LIBHACL_SIMD256_OBJS]) PY_STDLIB_MOD([_ctypes], [], [test "$have_libffi" = yes], [$NO_STRICT_OVERFLOW_CFLAGS $LIBFFI_CFLAGS], [$LIBFFI_LIBS]) PY_STDLIB_MOD([_curses], - [], [test "$have_curses" != "no"], + [], [test "$have_curses" = "yes"], [$CURSES_CFLAGS], [$CURSES_LIBS] ) PY_STDLIB_MOD([_curses_panel], - [], [test "$have_panel" != "no"], + [], [test "$have_curses" = "yes" && test "$have_panel" = "yes"], [$PANEL_CFLAGS $CURSES_CFLAGS], [$PANEL_LIBS $CURSES_LIBS] ) -PY_STDLIB_MOD([_decimal], [], [], [$LIBMPDEC_CFLAGS], [$LIBMPDEC_LDFLAGS]) +PY_STDLIB_MOD([_decimal], + [], [test "$have_mpdec" = "yes"], + [$LIBMPDEC_CFLAGS], [$LIBMPDEC_LIBS]) PY_STDLIB_MOD([_dbm], [test -n "$with_dbmliborder"], [test "$have_dbm" != "no"], [$DBM_CFLAGS], [$DBM_LIBS]) @@ -7576,21 +7937,23 @@ PY_STDLIB_MOD([_testcapi], [], [], [$LIBATOMIC]) PY_STDLIB_MOD([_testclinic], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testclinic_limited], [test "$TEST_MODULES" = yes]) +PY_STDLIB_MOD([_testlimitedcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testbuffer], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([_testsinglephase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([_testexternalinspection], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([xxsubtype], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_xxtestfuzz], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_ctypes_test], [test "$TEST_MODULES" = yes], [test "$have_libffi" = yes -a "$ac_cv_func_dlopen" = yes], - [], [$LIBM]) + [$LIBFFI_CFLAGS], [$LIBFFI_LIBS $LIBM]) dnl Limited API template modules. dnl Emscripten does not support shared libraries yet. -PY_STDLIB_MOD([xxlimited], [], [test "$ac_cv_func_dlopen" = yes]) -PY_STDLIB_MOD([xxlimited_35], [], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([xxlimited], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([xxlimited_35], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) # substitute multiline block, must come after last PY_STDLIB_MOD() AC_SUBST([MODULE_BLOCK]) diff --git a/iOS/README.rst b/iOS/README.rst index 1043b5ecebbc0c..13b885144932e4 100644 --- a/iOS/README.rst +++ b/iOS/README.rst @@ -33,16 +33,27 @@ Platform by selecting an open the Platforms tab of the Xcode Settings panel. iOS specific arguments to configure =================================== -* ``--enable-framework=DIR`` +* ``--enable-framework[=DIR]`` This argument specifies the location where the Python.framework will be - installed. This argument is required for all iOS builds; a directory *must* - be specified. + installed. If ``DIR`` is not specified, the framework will be installed into + a subdirectory of the ``iOS/Frameworks`` folder. + + This argument *must* be provided when configuring iOS builds. iOS does not + support non-framework builds. * ``--with-framework-name=NAME`` Specify the name for the Python framework; defaults to ``Python``. + .. admonition:: Use this option with care! + + Unless you know what you're doing, changing the name of the Python + framework on iOS is not advised. If you use this option, you won't be able + to run the ``make testios`` target without making significant manual + alterations, and you won't be able to use any binary packages unless you + compile them yourself using your own framework name. + Building Python on iOS ====================== @@ -85,13 +96,9 @@ provide the ``--enable-framework`` flag when configuring the build. The build also requires the use of cross-compilation. The minimal commands for building Python for the ARM64 iOS simulator will look something like:: - $ export PATH="`pwd`/iOS/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" + $ export PATH="$(pwd)/iOS/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" $ ./configure \ - AR=arm64-apple-ios-simulator-ar \ - CC=arm64-apple-ios-simulator-clang \ - CPP=arm64-apple-ios-simulator-cpp \ - CXX=arm64-apple-ios-simulator-clang \ - --enable-framework=/path/to/install \ + --enable-framework \ --host=arm64-apple-ios-simulator \ --build=arm64-apple-darwin \ --with-build-python=/path/to/python.exe @@ -132,9 +139,6 @@ In this invocation: build. Resetting the path to a known "bare bones" value is the easiest way to avoid these problems. -* ``/path/to/install`` is the location where the final ``Python.framework`` will - be output. - * ``--host`` is the architecture and ABI that you want to build, in GNU compiler triple format. This will be one of: @@ -161,16 +165,30 @@ In this invocation: release has been stable, the more likely it is that this constraint can be relaxed - the same micro version will often be sufficient. +* The ``install`` target for iOS builds is slightly different to other + platforms. On most platforms, ``make install`` will install the build into + the final runtime location. This won't be the case for iOS, as the final + runtime location will be on a physical device. + + However, you still need to run the ``install`` target for iOS builds, as it + performs some final framework assembly steps. The location specified with + ``--enable-framework`` will be the location where ``make install`` will + assemble the complete iOS framework. This completed framework can then + be copied and relocated as required. + For a full CPython build, you also need to specify the paths to iOS builds of the binary libraries that CPython depends on (XZ, BZip2, LibFFI and OpenSSL). This can be done by defining the ``LIBLZMA_CFLAGS``, ``LIBLZMA_LIBS``, ``BZIP2_CFLAGS``, ``BZIP2_LIBS``, ``LIBFFI_CFLAGS``, and ``LIBFFI_LIBS`` environment variables, and the ``--with-openssl`` configure option. Versions of these libraries pre-compiled for iOS can be found in `this repository -`__. +`__. LibFFI is +especially important, as many parts of the standard library (including the +``platform``, ``sysconfig`` and ``webbrowser`` modules) require the use of the +``ctypes`` module at runtime. By default, Python will be compiled with an iOS deployment target (i.e., the -minimum supported iOS version) of 12.0. To specify a different deployment +minimum supported iOS version) of 13.0. To specify a different deployment target, provide the version number as part of the ``--host`` argument - for example, ``--host=arm64-apple-ios15.4-simulator`` would compile an ARM64 simulator build with a deployment target of 15.4. @@ -206,7 +224,7 @@ content of the two "thin" ``Python.framework`` directories, plus the ``bin`` and $ lipo -create -output module.dylib path/to/x86_64/module.dylib path/to/arm64/module.dylib -* The header files will be indentical on both architectures, except for +* The header files will be identical on both architectures, except for ``pyconfig.h``. Copy all the headers from one platform (say, arm64), rename ``pyconfig.h`` to ``pyconfig-arm64.h``, and copy the ``pyconfig.h`` for the other architecture into the merged header folder as ``pyconfig-x86_64.h``. @@ -233,16 +251,11 @@ the XCframework:: cp path/to/iphoneos/bin Python.xcframework/ios-arm64 cp path/to/iphoneos/lib Python.xcframework/ios-arm64 - cp path/to/iphonesimulator/bin Python.xcframework/ios-arm64_x86-64-simulator - cp path/to/iphonesimulator/lib Python.xcframework/ios-arm64_x86-64-simulator + cp path/to/iphonesimulator/bin Python.xcframework/ios-arm64_x86_64-simulator + cp path/to/iphonesimulator/lib Python.xcframework/ios-arm64_x86_64-simulator Note that the name of the architecture-specific slice for the simulator will -depend on the CPU architecture that you build. - -Then, add symbolic links to "common" platform names for each slice:: - - ln -si ios-arm64 Python.xcframework/iphoneos - ln -si ios-arm64_x86-64-simulator Python.xcframework/iphonesimulator +depend on the CPU architecture(s) that you build. You now have a Python.xcframework that can be used in a project. @@ -256,44 +269,85 @@ suite passes. To run the test suite, configure a Python build for an iOS simulator (i.e., ``--host=arm64-apple-ios-simulator`` or ``--host=x86_64-apple-ios-simulator`` -), setting the framework location to the testbed project:: +), specifying a framework build (i.e. ``--enable-framework``). Ensure that your +``PATH`` has been configured to include the ``iOS/Resources/bin`` folder and +exclude any non-iOS tools, then run:: - --enable-framework="./iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator" + $ make all + $ make install + $ make testios -Then run ``make all install testiOS``. This will build an iOS framework for your -chosen architecture, install the Python iOS framework into the testbed project, -and run the test suite on an "iPhone SE (3rd generation)" simulator. +This will: -While the test suite is running, Xcode does not display any console output. -After showing some Xcode build commands, the console output will print ``Testing -started``, and then appear to stop. It will remain in this state until the test -suite completes. On a 2022 M1 MacBook Pro, the test suite takes approximately 12 -minutes to run; a couple of extra minutes is required to boot and prepare the -iOS simulator. +* Build an iOS framework for your chosen architecture; +* Finalize the single-platform framework; +* Make a clean copy of the testbed project; +* Install the Python iOS framework into the copy of the testbed project; and +* Run the test suite on an "iPhone SE (3rd generation)" simulator. On success, the test suite will exit and report successful completion of the -test suite. No output of the Python test suite will be displayed. - -On failure, the output of the Python test suite *will* be displayed. This will -show the details of the tests that failed. +test suite. On a 2022 M1 MacBook Pro, the test suite takes approximately 15 +minutes to run; a couple of extra minutes is required to compile the testbed +project, and then boot and prepare the iOS simulator. Debugging test failures ----------------------- -The easiest way to diagnose a single test failure is to open the testbed project -in Xcode and run the tests from there using the "Product > Test" menu item. +Running ``make test`` generates a standalone version of the ``iOS/testbed`` +project, and runs the full test suite. It does this using ``iOS/testbed`` +itself - the folder is an executable module that can be used to create and run +a clone of the testbed project. + +You can generate your own standalone testbed instance by running:: + + $ python iOS/testbed clone --framework iOS/Frameworks/arm64-iphonesimulator my-testbed + +This invocation assumes that ``iOS/Frameworks/arm64-iphonesimulator`` is the +path to the iOS simulator framework for your platform (ARM64 in this case); +``my-testbed`` is the name of the folder for the new testbed clone. + +You can then use the ``my-testbed`` folder to run the Python test suite, +passing in any command line arguments you may require. For example, if you're +trying to diagnose a failure in the ``os`` module, you might run:: + + $ python my-testbed run -- test -W test_os + +This is the equivalent of running ``python -m test -W test_os`` on a desktop +Python build. Any arguments after the ``--`` will be passed to testbed as if +they were arguments to ``python -m`` on a desktop machine. + +You can also open the testbed project in Xcode by running:: + + $ open my-testbed/iOSTestbed.xcodeproj + +This will allow you to use the full Xcode suite of tools for debugging. + +Testing on an iOS device +^^^^^^^^^^^^^^^^^^^^^^^^ + +To test on an iOS device, the app needs to be signed with known developer +credentials. To obtain these credentials, you must have an iOS Developer +account, and your Xcode install will need to be logged into your account (see +the Accounts tab of the Preferences dialog). + +Once the project is open, and you're signed into your Apple Developer account, +select the root node of the project tree (labeled "iOSTestbed"), then the +"Signing & Capabilities" tab in the details page. Select a development team +(this will likely be your own name), and plug in a physical device to your +macOS machine with a USB cable. You should then be able to select your physical +device from the list of targets in the pulldown in the Xcode titlebar. Running specific tests ^^^^^^^^^^^^^^^^^^^^^^ As the test suite is being executed on an iOS simulator, it is not possible to -pass in command line arguments to configure test suite operation. To work around -this limitation, the arguments that would normally be passed as command line -arguments are configured as a static string at the start of the XCTest method -``- (void)testPython`` in ``iOSTestbedTests.m``. To pass an argument to the test -suite, add a a string to the ``argv`` defintion. These arguments will be passed -to the test suite as if they had been passed to ``python -m test`` at the -command line. +pass in command line arguments to configure test suite operation. To work +around this limitation, the arguments that would normally be passed as command +line arguments are configured as part of the ``iOSTestbed-Info.plist`` file +that is used to configure the iOS testbed app. In this file, the ``TestArgs`` +key is an array containing the arguments that would be passed to ``python -m`` +on the command line (including ``test`` in position 0, the name of the test +module to be executed). Disabling automated breakpoints ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/iOS/Resources/Info.plist.in b/iOS/Resources/Info.plist.in index 3ecdc894f0a285..c3e261ecd9eff7 100644 --- a/iOS/Resources/Info.plist.in +++ b/iOS/Resources/Info.plist.in @@ -17,7 +17,7 @@ CFBundlePackageType FMWK CFBundleShortVersionString - %VERSION% + @VERSION@ CFBundleLongVersionString %VERSION%, (c) 2001-2024 Python Software Foundation. CFBundleSignature @@ -29,6 +29,6 @@ iPhoneOS MinimumOSVersion - @IOS_DEPLOYMENT_TARGET@ + @IPHONEOS_DEPLOYMENT_TARGET@ diff --git a/iOS/Resources/bin/arm64-apple-ios-ar b/iOS/Resources/bin/arm64-apple-ios-ar index 8122332b9c1de0..3cf3eb218741fa 100755 --- a/iOS/Resources/bin/arm64-apple-ios-ar +++ b/iOS/Resources/bin/arm64-apple-ios-ar @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphoneos${IOS_SDK_VERSION} ar $@ +xcrun --sdk iphoneos${IOS_SDK_VERSION} ar "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-clang b/iOS/Resources/bin/arm64-apple-ios-clang index 4d525751eba798..c39519cd1f8c94 100755 --- a/iOS/Resources/bin/arm64-apple-ios-clang +++ b/iOS/Resources/bin/arm64-apple-ios-clang @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios $@ +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-clang++ b/iOS/Resources/bin/arm64-apple-ios-clang++ new file mode 100755 index 00000000000000..d9b12925f384b9 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang++ -target arm64-apple-ios "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-cpp b/iOS/Resources/bin/arm64-apple-ios-cpp index 891bb25bb4318c..24da23d3448ae0 100755 --- a/iOS/Resources/bin/arm64-apple-ios-cpp +++ b/iOS/Resources/bin/arm64-apple-ios-cpp @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios -E $@ +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios -E "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-ar b/iOS/Resources/bin/arm64-apple-ios-simulator-ar index 74ed3bc6df1c2b..b836b6db9025bb 100755 --- a/iOS/Resources/bin/arm64-apple-ios-simulator-ar +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-ar @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-clang b/iOS/Resources/bin/arm64-apple-ios-simulator-clang index 32574cad284441..92e8d853d6ebc3 100755 --- a/iOS/Resources/bin/arm64-apple-ios-simulator-clang +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-clang @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ b/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ new file mode 100755 index 00000000000000..076469cc70cf98 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target arm64-apple-ios-simulator "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-cpp b/iOS/Resources/bin/arm64-apple-ios-simulator-cpp index 6aaf6fbe188c32..c57f28cee5bcfe 100755 --- a/iOS/Resources/bin/arm64-apple-ios-simulator-cpp +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-cpp @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator -E $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator -E "$@" diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-ar b/iOS/Resources/bin/x86_64-apple-ios-simulator-ar index 74ed3bc6df1c2b..b836b6db9025bb 100755 --- a/iOS/Resources/bin/x86_64-apple-ios-simulator-ar +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-ar @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar "$@" diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang index bcbe91f6061e16..17cbe0c8a1e213 100755 --- a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator "$@" diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ new file mode 100755 index 00000000000000..565d47b24c214b --- /dev/null +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target x86_64-apple-ios-simulator "$@" diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp b/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp index e6a42d9b85dec7..63fc8e8de2d38d 100755 --- a/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator -E $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator -E "$@" diff --git a/iOS/testbed/Python.xcframework/Info.plist b/iOS/testbed/Python.xcframework/Info.plist new file mode 100644 index 00000000000000..c6418de6e74a4e --- /dev/null +++ b/iOS/testbed/Python.xcframework/Info.plist @@ -0,0 +1,44 @@ + + + + + AvailableLibraries + + + BinaryPath + Python.framework/Python + LibraryIdentifier + ios-arm64 + LibraryPath + Python.framework + SupportedArchitectures + + arm64 + + SupportedPlatform + ios + + + BinaryPath + Python.framework/Python + LibraryIdentifier + ios-arm64_x86_64-simulator + LibraryPath + Python.framework + SupportedArchitectures + + arm64 + x86_64 + + SupportedPlatform + ios + SupportedPlatformVariant + simulator + + + CFBundlePackageType + XFWK + XCFrameworkFormatVersion + 1.0 + + diff --git a/iOS/testbed/Python.xcframework/ios-arm64/README b/iOS/testbed/Python.xcframework/ios-arm64/README new file mode 100644 index 00000000000000..c1b076d12cddb7 --- /dev/null +++ b/iOS/testbed/Python.xcframework/ios-arm64/README @@ -0,0 +1,4 @@ +This directory is intentionally empty. + +It should be used as a target for `--enable-framework` when compiling an iOS on-device +build for testing purposes. diff --git a/iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README b/iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README new file mode 100644 index 00000000000000..ae334e5d769d9d --- /dev/null +++ b/iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README @@ -0,0 +1,4 @@ +This directory is intentionally empty. + +It should be used as a target for `--enable-framework` when compiling an iOS simulator +build for testing purposes (either x86_64 or ARM64). diff --git a/iOS/testbed/__main__.py b/iOS/testbed/__main__.py new file mode 100644 index 00000000000000..b4499f5ac171a8 --- /dev/null +++ b/iOS/testbed/__main__.py @@ -0,0 +1,410 @@ +import argparse +import asyncio +import json +import plistlib +import re +import shutil +import subprocess +import sys +from contextlib import asynccontextmanager +from datetime import datetime +from pathlib import Path + + +DECODE_ARGS = ("UTF-8", "backslashreplace") + +# The system log prefixes each line: +# 2025-01-17 16:14:29.090 Df iOSTestbed[23987:1fd393b4] (Python) ... +# 2025-01-17 16:14:29.090 E iOSTestbed[23987:1fd393b4] (Python) ... + +LOG_PREFIX_REGEX = re.compile( + r"^\d{4}-\d{2}-\d{2}" # YYYY-MM-DD + r"\s+\d+:\d{2}:\d{2}\.\d+" # HH:MM:SS.sss + r"\s+\w+" # Df/E + r"\s+iOSTestbed\[\d+:\w+\]" # Process/thread ID + r"\s+\(Python\)\s" # Logger name +) + + +# Work around a bug involving sys.exit and TaskGroups +# (https://github.com/python/cpython/issues/101515). +def exit(*args): + raise MySystemExit(*args) + + +class MySystemExit(Exception): + pass + + +# All subprocesses are executed through this context manager so that no matter +# what happens, they can always be cancelled from another task, and they will +# always be cleaned up on exit. +@asynccontextmanager +async def async_process(*args, **kwargs): + process = await asyncio.create_subprocess_exec(*args, **kwargs) + try: + yield process + finally: + if process.returncode is None: + # Allow a reasonably long time for Xcode to clean itself up, + # because we don't want stale emulators left behind. + timeout = 10 + process.terminate() + try: + await asyncio.wait_for(process.wait(), timeout) + except TimeoutError: + print( + f"Command {args} did not terminate after {timeout} seconds " + f" - sending SIGKILL" + ) + process.kill() + + # Even after killing the process we must still wait for it, + # otherwise we'll get the warning "Exception ignored in __del__". + await asyncio.wait_for(process.wait(), timeout=1) + + +async def async_check_output(*args, **kwargs): + async with async_process( + *args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs + ) as process: + stdout, stderr = await process.communicate() + if process.returncode == 0: + return stdout.decode(*DECODE_ARGS) + else: + raise subprocess.CalledProcessError( + process.returncode, + args, + stdout.decode(*DECODE_ARGS), + stderr.decode(*DECODE_ARGS), + ) + + +# Return a list of UDIDs associated with booted simulators +async def list_devices(): + # List the testing simulators, in JSON format + raw_json = await async_check_output( + "xcrun", "simctl", "--set", "testing", "list", "-j" + ) + json_data = json.loads(raw_json) + + # Filter out the booted iOS simulators + return [ + simulator["udid"] + for runtime, simulators in json_data["devices"].items() + for simulator in simulators + if runtime.split(".")[-1].startswith("iOS") and simulator["state"] == "Booted" + ] + + +async def find_device(initial_devices): + while True: + new_devices = set(await list_devices()).difference(initial_devices) + if len(new_devices) == 0: + await asyncio.sleep(1) + elif len(new_devices) == 1: + udid = new_devices.pop() + print(f"{datetime.now():%Y-%m-%d %H:%M:%S}: New test simulator detected") + print(f"UDID: {udid}") + return udid + else: + exit(f"Found more than one new device: {new_devices}") + + +async def log_stream_task(initial_devices): + # Wait up to 5 minutes for the build to complete and the simulator to boot. + udid = await asyncio.wait_for(find_device(initial_devices), 5 * 60) + + # Stream the iOS device's logs, filtering out messages that come from the + # XCTest test suite (catching NSLog messages from the test method), or + # Python itself (catching stdout/stderr content routed to the system log + # with config->use_system_logger). + args = [ + "xcrun", + "simctl", + "--set", + "testing", + "spawn", + udid, + "log", + "stream", + "--style", + "compact", + "--predicate", + ( + 'senderImagePath ENDSWITH "/iOSTestbedTests.xctest/iOSTestbedTests"' + ' OR senderImagePath ENDSWITH "/Python.framework/Python"' + ), + ] + + async with async_process( + *args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) as process: + suppress_dupes = False + while line := (await process.stdout.readline()).decode(*DECODE_ARGS): + # Strip the prefix from each log line + line = LOG_PREFIX_REGEX.sub("", line) + # The iOS log streamer can sometimes lag; when it does, it outputs + # a warning about messages being dropped... often multiple times. + # Only print the first of these duplicated warnings. + if line.startswith("=== Messages dropped "): + if not suppress_dupes: + suppress_dupes = True + sys.stdout.write(line) + else: + suppress_dupes = False + sys.stdout.write(line) + sys.stdout.flush() + + +async def xcode_test(location, simulator, verbose): + # Run the test suite on the named simulator + print("Starting xcodebuild...") + args = [ + "xcodebuild", + "test", + "-project", + str(location / "iOSTestbed.xcodeproj"), + "-scheme", + "iOSTestbed", + "-destination", + f"platform=iOS Simulator,name={simulator}", + "-resultBundlePath", + str(location / f"{datetime.now():%Y%m%d-%H%M%S}.xcresult"), + "-derivedDataPath", + str(location / "DerivedData"), + ] + if not verbose: + args += ["-quiet"] + + async with async_process( + *args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) as process: + while line := (await process.stdout.readline()).decode(*DECODE_ARGS): + sys.stdout.write(line) + sys.stdout.flush() + + status = await asyncio.wait_for(process.wait(), timeout=1) + exit(status) + + +def clone_testbed( + source: Path, + target: Path, + framework: Path, + apps: list[Path], +) -> None: + if target.exists(): + print(f"{target} already exists; aborting without creating project.") + sys.exit(10) + + if framework is None: + if not ( + source / "Python.xcframework/ios-arm64_x86_64-simulator/bin" + ).is_dir(): + print( + f"The testbed being cloned ({source}) does not contain " + f"a simulator framework. Re-run with --framework" + ) + sys.exit(11) + else: + if not framework.is_dir(): + print(f"{framework} does not exist.") + sys.exit(12) + elif not ( + framework.suffix == ".xcframework" + or (framework / "Python.framework").is_dir() + ): + print( + f"{framework} is not an XCframework, " + f"or a simulator slice of a framework build." + ) + sys.exit(13) + + print("Cloning testbed project:") + print(f" Cloning {source}...", end="", flush=True) + shutil.copytree(source, target, symlinks=True) + print(" done") + + if framework is not None: + if framework.suffix == ".xcframework": + print(" Installing XCFramework...", end="", flush=True) + xc_framework_path = (target / "Python.xcframework").resolve() + if xc_framework_path.is_dir(): + shutil.rmtree(xc_framework_path) + else: + xc_framework_path.unlink() + xc_framework_path.symlink_to( + framework.relative_to(xc_framework_path.parent, walk_up=True) + ) + print(" done") + else: + print(" Installing simulator framework...", end="", flush=True) + sim_framework_path = ( + target / "Python.xcframework" / "ios-arm64_x86_64-simulator" + ).resolve() + if sim_framework_path.is_dir(): + shutil.rmtree(sim_framework_path) + else: + sim_framework_path.unlink() + sim_framework_path.symlink_to( + framework.relative_to(sim_framework_path.parent, walk_up=True) + ) + print(" done") + else: + print(" Using pre-existing iOS framework.") + + for app_src in apps: + print(f" Installing app {app_src.name!r}...", end="", flush=True) + app_target = target / f"iOSTestbed/app/{app_src.name}" + if app_target.is_dir(): + shutil.rmtree(app_target) + shutil.copytree(app_src, app_target) + print(" done") + + print(f"Successfully cloned testbed: {target.resolve()}") + + +def update_plist(testbed_path, args): + # Add the test runner arguments to the testbed's Info.plist file. + info_plist = testbed_path / "iOSTestbed" / "iOSTestbed-Info.plist" + with info_plist.open("rb") as f: + info = plistlib.load(f) + + info["TestArgs"] = args + + with info_plist.open("wb") as f: + plistlib.dump(info, f) + + +async def run_testbed(simulator: str, args: list[str], verbose: bool=False): + location = Path(__file__).parent + print("Updating plist...", end="", flush=True) + update_plist(location, args) + print(" done.") + + # Get the list of devices that are booted at the start of the test run. + # The simulator started by the test suite will be detected as the new + # entry that appears on the device list. + initial_devices = await list_devices() + + try: + async with asyncio.TaskGroup() as tg: + tg.create_task(log_stream_task(initial_devices)) + tg.create_task(xcode_test(location, simulator=simulator, verbose=verbose)) + except* MySystemExit as e: + raise SystemExit(*e.exceptions[0].args) from None + except* subprocess.CalledProcessError as e: + # Extract it from the ExceptionGroup so it can be handled by `main`. + raise e.exceptions[0] + + +def main(): + parser = argparse.ArgumentParser( + description=( + "Manages the process of testing a Python project in the iOS simulator." + ), + ) + + subcommands = parser.add_subparsers(dest="subcommand") + + clone = subcommands.add_parser( + "clone", + description=( + "Clone the testbed project, copying in an iOS Python framework and" + "any specified application code." + ), + help="Clone a testbed project to a new location.", + ) + clone.add_argument( + "--framework", + help=( + "The location of the XCFramework (or simulator-only slice of an " + "XCFramework) to use when running the testbed" + ), + ) + clone.add_argument( + "--app", + dest="apps", + action="append", + default=[], + help="The location of any code to include in the testbed project", + ) + clone.add_argument( + "location", + help="The path where the testbed will be cloned.", + ) + + run = subcommands.add_parser( + "run", + usage="%(prog)s [-h] [--simulator SIMULATOR] -- [ ...]", + description=( + "Run a testbed project. The arguments provided after `--` will be " + "passed to the running iOS process as if they were arguments to " + "`python -m`." + ), + help="Run a testbed project", + ) + run.add_argument( + "--simulator", + default="iPhone SE (3rd Generation)", + help="The name of the simulator to use (default: 'iPhone SE (3rd Generation)')", + ) + run.add_argument( + "-v", "--verbose", + action="store_true", + help="Enable verbose output", + ) + + try: + pos = sys.argv.index("--") + testbed_args = sys.argv[1:pos] + test_args = sys.argv[pos + 1 :] + except ValueError: + testbed_args = sys.argv[1:] + test_args = [] + + context = parser.parse_args(testbed_args) + + if context.subcommand == "clone": + clone_testbed( + source=Path(__file__).parent, + target=Path(context.location), + framework=Path(context.framework).resolve() if context.framework else None, + apps=[Path(app) for app in context.apps], + ) + elif context.subcommand == "run": + if test_args: + if not ( + Path(__file__).parent / "Python.xcframework/ios-arm64_x86_64-simulator/bin" + ).is_dir(): + print( + f"Testbed does not contain a compiled iOS framework. Use " + f"`python {sys.argv[0]} clone ...` to create a runnable " + f"clone of this testbed." + ) + sys.exit(20) + + asyncio.run( + run_testbed( + simulator=context.simulator, + verbose=context.verbose, + args=test_args, + ) + ) + else: + print(f"Must specify test arguments (e.g., {sys.argv[0]} run -- test)") + print() + parser.print_help(sys.stderr) + sys.exit(21) + else: + parser.print_help(sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj new file mode 100644 index 00000000000000..c7d63909ee2453 --- /dev/null +++ b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj @@ -0,0 +1,580 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 56; + objects = { + +/* Begin PBXBuildFile section */ + 607A66172B0EFA380010BFC8 /* AppDelegate.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66162B0EFA380010BFC8 /* AppDelegate.m */; }; + 607A66222B0EFA390010BFC8 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 607A66212B0EFA390010BFC8 /* Assets.xcassets */; }; + 607A66252B0EFA390010BFC8 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 607A66232B0EFA390010BFC8 /* LaunchScreen.storyboard */; }; + 607A66282B0EFA390010BFC8 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66272B0EFA390010BFC8 /* main.m */; }; + 607A66322B0EFA3A0010BFC8 /* iOSTestbedTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */; }; + 607A664C2B0EFC080010BFC8 /* Python.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; }; + 607A664D2B0EFC080010BFC8 /* Python.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; + 607A66502B0EFFE00010BFC8 /* Python.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; }; + 607A66512B0EFFE00010BFC8 /* Python.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; + 607A66582B0F079F0010BFC8 /* dylib-Info-template.plist in Resources */ = {isa = PBXBuildFile; fileRef = 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */; }; + 608619542CB77BA900F46182 /* app_packages in Resources */ = {isa = PBXBuildFile; fileRef = 608619532CB77BA900F46182 /* app_packages */; }; + 608619562CB7819B00F46182 /* app in Resources */ = {isa = PBXBuildFile; fileRef = 608619552CB7819B00F46182 /* app */; }; +/* End PBXBuildFile section */ + +/* Begin PBXContainerItemProxy section */ + 607A662E2B0EFA3A0010BFC8 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 607A660A2B0EFA380010BFC8 /* Project object */; + proxyType = 1; + remoteGlobalIDString = 607A66112B0EFA380010BFC8; + remoteInfo = iOSTestbed; + }; +/* End PBXContainerItemProxy section */ + +/* Begin PBXCopyFilesBuildPhase section */ + 607A664E2B0EFC080010BFC8 /* Embed Frameworks */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = ""; + dstSubfolderSpec = 10; + files = ( + 607A664D2B0EFC080010BFC8 /* Python.xcframework in Embed Frameworks */, + ); + name = "Embed Frameworks"; + runOnlyForDeploymentPostprocessing = 0; + }; + 607A66522B0EFFE00010BFC8 /* Embed Frameworks */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = ""; + dstSubfolderSpec = 10; + files = ( + 607A66512B0EFFE00010BFC8 /* Python.xcframework in Embed Frameworks */, + ); + name = "Embed Frameworks"; + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXCopyFilesBuildPhase section */ + +/* Begin PBXFileReference section */ + 607A66122B0EFA380010BFC8 /* iOSTestbed.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = iOSTestbed.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 607A66152B0EFA380010BFC8 /* AppDelegate.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = AppDelegate.h; sourceTree = ""; }; + 607A66162B0EFA380010BFC8 /* AppDelegate.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = AppDelegate.m; sourceTree = ""; }; + 607A66212B0EFA390010BFC8 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; + 607A66242B0EFA390010BFC8 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; + 607A66272B0EFA390010BFC8 /* main.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = main.m; sourceTree = ""; }; + 607A662D2B0EFA3A0010BFC8 /* iOSTestbedTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = iOSTestbedTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; + 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = iOSTestbedTests.m; sourceTree = ""; }; + 607A664A2B0EFB310010BFC8 /* Python.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; path = Python.xcframework; sourceTree = ""; }; + 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = "dylib-Info-template.plist"; sourceTree = ""; }; + 607A66592B0F08600010BFC8 /* iOSTestbed-Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = "iOSTestbed-Info.plist"; sourceTree = ""; }; + 608619532CB77BA900F46182 /* app_packages */ = {isa = PBXFileReference; lastKnownFileType = folder; path = app_packages; sourceTree = ""; }; + 608619552CB7819B00F46182 /* app */ = {isa = PBXFileReference; lastKnownFileType = folder; path = app; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + 607A660F2B0EFA380010BFC8 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + 607A664C2B0EFC080010BFC8 /* Python.xcframework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 607A662A2B0EFA3A0010BFC8 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + 607A66502B0EFFE00010BFC8 /* Python.xcframework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 607A66092B0EFA380010BFC8 = { + isa = PBXGroup; + children = ( + 607A664A2B0EFB310010BFC8 /* Python.xcframework */, + 607A66142B0EFA380010BFC8 /* iOSTestbed */, + 607A66302B0EFA3A0010BFC8 /* iOSTestbedTests */, + 607A66132B0EFA380010BFC8 /* Products */, + 607A664F2B0EFFE00010BFC8 /* Frameworks */, + ); + sourceTree = ""; + }; + 607A66132B0EFA380010BFC8 /* Products */ = { + isa = PBXGroup; + children = ( + 607A66122B0EFA380010BFC8 /* iOSTestbed.app */, + 607A662D2B0EFA3A0010BFC8 /* iOSTestbedTests.xctest */, + ); + name = Products; + sourceTree = ""; + }; + 607A66142B0EFA380010BFC8 /* iOSTestbed */ = { + isa = PBXGroup; + children = ( + 608619552CB7819B00F46182 /* app */, + 608619532CB77BA900F46182 /* app_packages */, + 607A66592B0F08600010BFC8 /* iOSTestbed-Info.plist */, + 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */, + 607A66152B0EFA380010BFC8 /* AppDelegate.h */, + 607A66162B0EFA380010BFC8 /* AppDelegate.m */, + 607A66212B0EFA390010BFC8 /* Assets.xcassets */, + 607A66232B0EFA390010BFC8 /* LaunchScreen.storyboard */, + 607A66272B0EFA390010BFC8 /* main.m */, + ); + path = iOSTestbed; + sourceTree = ""; + }; + 607A66302B0EFA3A0010BFC8 /* iOSTestbedTests */ = { + isa = PBXGroup; + children = ( + 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */, + ); + path = iOSTestbedTests; + sourceTree = ""; + }; + 607A664F2B0EFFE00010BFC8 /* Frameworks */ = { + isa = PBXGroup; + children = ( + ); + name = Frameworks; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 607A66112B0EFA380010BFC8 /* iOSTestbed */ = { + isa = PBXNativeTarget; + buildConfigurationList = 607A66412B0EFA3A0010BFC8 /* Build configuration list for PBXNativeTarget "iOSTestbed" */; + buildPhases = ( + 607A660E2B0EFA380010BFC8 /* Sources */, + 607A660F2B0EFA380010BFC8 /* Frameworks */, + 607A66102B0EFA380010BFC8 /* Resources */, + 607A66552B0F061D0010BFC8 /* Install Target Specific Python Standard Library */, + 607A66562B0F06200010BFC8 /* Prepare Python Binary Modules */, + 607A664E2B0EFC080010BFC8 /* Embed Frameworks */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = iOSTestbed; + productName = iOSTestbed; + productReference = 607A66122B0EFA380010BFC8 /* iOSTestbed.app */; + productType = "com.apple.product-type.application"; + }; + 607A662C2B0EFA3A0010BFC8 /* iOSTestbedTests */ = { + isa = PBXNativeTarget; + buildConfigurationList = 607A66442B0EFA3A0010BFC8 /* Build configuration list for PBXNativeTarget "iOSTestbedTests" */; + buildPhases = ( + 607A66292B0EFA3A0010BFC8 /* Sources */, + 607A662A2B0EFA3A0010BFC8 /* Frameworks */, + 607A662B2B0EFA3A0010BFC8 /* Resources */, + 607A66522B0EFFE00010BFC8 /* Embed Frameworks */, + ); + buildRules = ( + ); + dependencies = ( + 607A662F2B0EFA3A0010BFC8 /* PBXTargetDependency */, + ); + name = iOSTestbedTests; + productName = iOSTestbedTests; + productReference = 607A662D2B0EFA3A0010BFC8 /* iOSTestbedTests.xctest */; + productType = "com.apple.product-type.bundle.unit-test"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 607A660A2B0EFA380010BFC8 /* Project object */ = { + isa = PBXProject; + attributes = { + BuildIndependentTargetsInParallel = 1; + LastUpgradeCheck = 1500; + TargetAttributes = { + 607A66112B0EFA380010BFC8 = { + CreatedOnToolsVersion = 15.0.1; + }; + 607A662C2B0EFA3A0010BFC8 = { + CreatedOnToolsVersion = 15.0.1; + TestTargetID = 607A66112B0EFA380010BFC8; + }; + }; + }; + buildConfigurationList = 607A660D2B0EFA380010BFC8 /* Build configuration list for PBXProject "iOSTestbed" */; + compatibilityVersion = "Xcode 14.0"; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = 607A66092B0EFA380010BFC8; + productRefGroup = 607A66132B0EFA380010BFC8 /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 607A66112B0EFA380010BFC8 /* iOSTestbed */, + 607A662C2B0EFA3A0010BFC8 /* iOSTestbedTests */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + 607A66102B0EFA380010BFC8 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 607A66252B0EFA390010BFC8 /* LaunchScreen.storyboard in Resources */, + 607A66582B0F079F0010BFC8 /* dylib-Info-template.plist in Resources */, + 608619562CB7819B00F46182 /* app in Resources */, + 607A66222B0EFA390010BFC8 /* Assets.xcassets in Resources */, + 608619542CB77BA900F46182 /* app_packages in Resources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 607A662B2B0EFA3A0010BFC8 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXShellScriptBuildPhase section */ + 607A66552B0F061D0010BFC8 /* Install Target Specific Python Standard Library */ = { + isa = PBXShellScriptBuildPhase; + alwaysOutOfDate = 1; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + ); + name = "Install Target Specific Python Standard Library"; + outputFileListPaths = ( + ); + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "set -e\n\nmkdir -p \"$CODESIGNING_FOLDER_PATH/python/lib\"\nif [ \"$EFFECTIVE_PLATFORM_NAME\" = \"-iphonesimulator\" ]; then\n echo \"Installing Python modules for iOS Simulator\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64_x86_64-simulator/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nelse\n echo \"Installing Python modules for iOS Device\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nfi\n"; + showEnvVarsInLog = 0; + }; + 607A66562B0F06200010BFC8 /* Prepare Python Binary Modules */ = { + isa = PBXShellScriptBuildPhase; + alwaysOutOfDate = 1; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + ); + name = "Prepare Python Binary Modules"; + outputFileListPaths = ( + ); + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "set -e\n\ninstall_dylib () {\n INSTALL_BASE=$1\n FULL_EXT=$2\n\n # The name of the extension file\n EXT=$(basename \"$FULL_EXT\")\n # The location of the extension file, relative to the bundle\n RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} \n # The path to the extension file, relative to the install base\n PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/}\n # The full dotted name of the extension module, constructed from the file path.\n FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d \".\" -f 1 | tr \"/\" \".\"); \n # A bundle identifier; not actually used, but required by Xcode framework packaging\n FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n # The name of the framework folder.\n FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n # If the framework folder doesn't exist, create it.\n if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n echo \"Creating framework for $RELATIVE_EXT\" \n mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleExecutable -string \"$FULL_MODULE_NAME\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n fi\n \n echo \"Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" \n mv \"$FULL_EXT\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\"\n # Create a placeholder .fwork file where the .so was\n echo \"$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" > ${FULL_EXT%.so}.fwork\n # Create a back reference to the .so file location in the framework\n echo \"${RELATIVE_EXT%.so}.fwork\" > \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin\" \n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib python/lib/$PYTHON_VER/lib-dynload/ \"$FULL_EXT\"\ndone\necho \"Install app package extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/app_packages\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib app_packages/ \"$FULL_EXT\"\ndone\necho \"Install app extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/app\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib app/ \"$FULL_EXT\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n"; + showEnvVarsInLog = 0; + }; +/* End PBXShellScriptBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + 607A660E2B0EFA380010BFC8 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 607A66172B0EFA380010BFC8 /* AppDelegate.m in Sources */, + 607A66282B0EFA390010BFC8 /* main.m in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 607A66292B0EFA3A0010BFC8 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 607A66322B0EFA3A0010BFC8 /* iOSTestbedTests.m in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXTargetDependency section */ + 607A662F2B0EFA3A0010BFC8 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 607A66112B0EFA380010BFC8 /* iOSTestbed */; + targetProxy = 607A662E2B0EFA3A0010BFC8 /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + +/* Begin PBXVariantGroup section */ + 607A66232B0EFA390010BFC8 /* LaunchScreen.storyboard */ = { + isa = PBXVariantGroup; + children = ( + 607A66242B0EFA390010BFC8 /* Base */, + ); + name = LaunchScreen.storyboard; + sourceTree = ""; + }; +/* End PBXVariantGroup section */ + +/* Begin XCBuildConfiguration section */ + 607A663F2B0EFA3A0010BFC8 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + ENABLE_USER_SCRIPT_SANDBOXING = YES; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = iphoneos; + }; + name = Debug; + }; + 607A66402B0EFA3A0010BFC8 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_USER_SCRIPT_SANDBOXING = YES; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + SDKROOT = iphoneos; + VALIDATE_PRODUCT = YES; + }; + name = Release; + }; + 607A66422B0EFA3A0010BFC8 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = NO; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = ""; + ENABLE_USER_SCRIPT_SANDBOXING = NO; + HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\""; + INFOPLIST_FILE = "iOSTestbed/iOSTestbed-Info.plist"; + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; + INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen; + INFOPLIST_KEY_UIMainStoryboardFile = Main; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + MARKETING_VERSION = 3.13.0a1; + PRODUCT_BUNDLE_IDENTIFIER = org.python.iOSTestbed; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = YES; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + 607A66432B0EFA3A0010BFC8 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = NO; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = ""; + ENABLE_TESTABILITY = YES; + ENABLE_USER_SCRIPT_SANDBOXING = NO; + HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\""; + INFOPLIST_FILE = "iOSTestbed/iOSTestbed-Info.plist"; + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; + INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen; + INFOPLIST_KEY_UIMainStoryboardFile = Main; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + MARKETING_VERSION = 3.13.0a1; + PRODUCT_BUNDLE_IDENTIFIER = org.python.iOSTestbed; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = YES; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Release; + }; + 607A66452B0EFA3A0010BFC8 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = NO; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 3HEZE76D99; + GENERATE_INFOPLIST_FILE = YES; + HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\""; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = org.python.iOSTestbedTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = NO; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/iOSTestbed.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/iOSTestbed"; + }; + name = Debug; + }; + 607A66462B0EFA3A0010BFC8 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = NO; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 3HEZE76D99; + GENERATE_INFOPLIST_FILE = YES; + HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\""; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = org.python.iOSTestbedTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = NO; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/iOSTestbed.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/iOSTestbed"; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 607A660D2B0EFA380010BFC8 /* Build configuration list for PBXProject "iOSTestbed" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 607A663F2B0EFA3A0010BFC8 /* Debug */, + 607A66402B0EFA3A0010BFC8 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 607A66412B0EFA3A0010BFC8 /* Build configuration list for PBXNativeTarget "iOSTestbed" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 607A66422B0EFA3A0010BFC8 /* Debug */, + 607A66432B0EFA3A0010BFC8 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 607A66442B0EFA3A0010BFC8 /* Build configuration list for PBXNativeTarget "iOSTestbedTests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 607A66452B0EFA3A0010BFC8 /* Debug */, + 607A66462B0EFA3A0010BFC8 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = 607A660A2B0EFA380010BFC8 /* Project object */; +} diff --git a/iOS/testbed/iOSTestbed/AppDelegate.h b/iOS/testbed/iOSTestbed/AppDelegate.h new file mode 100644 index 00000000000000..f695b3b5efc08b --- /dev/null +++ b/iOS/testbed/iOSTestbed/AppDelegate.h @@ -0,0 +1,11 @@ +// +// AppDelegate.h +// iOSTestbed +// + +#import + +@interface AppDelegate : UIResponder + + +@end diff --git a/iOS/testbed/iOSTestbed/AppDelegate.m b/iOS/testbed/iOSTestbed/AppDelegate.m new file mode 100644 index 00000000000000..e5085399d0ca5f --- /dev/null +++ b/iOS/testbed/iOSTestbed/AppDelegate.m @@ -0,0 +1,19 @@ +// +// AppDelegate.m +// iOSTestbed +// + +#import "AppDelegate.h" + +@interface AppDelegate () + +@end + +@implementation AppDelegate + + +- (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)launchOptions { + return YES; +} + +@end diff --git a/iOS/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json b/iOS/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json new file mode 100644 index 00000000000000..eb878970081645 --- /dev/null +++ b/iOS/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json @@ -0,0 +1,11 @@ +{ + "colors" : [ + { + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/iOS/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json b/iOS/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 00000000000000..13613e3ee1a934 --- /dev/null +++ b/iOS/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,13 @@ +{ + "images" : [ + { + "idiom" : "universal", + "platform" : "ios", + "size" : "1024x1024" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/iOS/testbed/iOSTestbed/Assets.xcassets/Contents.json b/iOS/testbed/iOSTestbed/Assets.xcassets/Contents.json new file mode 100644 index 00000000000000..73c00596a7fca3 --- /dev/null +++ b/iOS/testbed/iOSTestbed/Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/iOS/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard b/iOS/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard new file mode 100644 index 00000000000000..5daafe73a866b7 --- /dev/null +++ b/iOS/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/iOS/testbed/iOSTestbed/app/README b/iOS/testbed/iOSTestbed/app/README new file mode 100644 index 00000000000000..af22c685f87976 --- /dev/null +++ b/iOS/testbed/iOSTestbed/app/README @@ -0,0 +1,7 @@ +This folder can contain any Python application code. + +During the build, any binary modules found in this folder will be processed into +iOS Framework form. + +When the test suite runs, this folder will be on the PYTHONPATH, and will be the +working directory for the test suite. diff --git a/iOS/testbed/iOSTestbed/app_packages/README b/iOS/testbed/iOSTestbed/app_packages/README new file mode 100644 index 00000000000000..42d7fdeb813250 --- /dev/null +++ b/iOS/testbed/iOSTestbed/app_packages/README @@ -0,0 +1,7 @@ +This folder can be a target for installing any Python dependencies needed by the +test suite. + +During the build, any binary modules found in this folder will be processed into +iOS Framework form. + +When the test suite runs, this folder will be on the PYTHONPATH. diff --git a/iOS/testbed/iOSTestbed/dylib-Info-template.plist b/iOS/testbed/iOSTestbed/dylib-Info-template.plist new file mode 100644 index 00000000000000..f652e272f71c88 --- /dev/null +++ b/iOS/testbed/iOSTestbed/dylib-Info-template.plist @@ -0,0 +1,26 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + + CFBundleIdentifier + + CFBundleInfoDictionaryVersion + 6.0 + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleSupportedPlatforms + + iPhoneOS + + MinimumOSVersion + 12.0 + CFBundleVersion + 1 + + diff --git a/iOS/testbed/iOSTestbed/iOSTestbed-Info.plist b/iOS/testbed/iOSTestbed/iOSTestbed-Info.plist new file mode 100644 index 00000000000000..a582f42a212783 --- /dev/null +++ b/iOS/testbed/iOSTestbed/iOSTestbed-Info.plist @@ -0,0 +1,64 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleDisplayName + ${PRODUCT_NAME} + CFBundleExecutable + ${EXECUTABLE_NAME} + CFBundleIdentifier + org.python.iOSTestbed + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + ${PRODUCT_NAME} + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleSignature + ???? + CFBundleVersion + 1 + LSRequiresIPhoneOS + + UIRequiresFullScreen + + UILaunchStoryboardName + Launch Screen + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + UISupportedInterfaceOrientations~ipad + + UIInterfaceOrientationPortrait + UIInterfaceOrientationPortraitUpsideDown + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + TestArgs + + test + -uall + --single-process + --rerun + -W + + + UIApplicationSceneManifest + + UIApplicationSupportsMultipleScenes + + UISceneConfigurations + + + + diff --git a/iOS/testbed/iOSTestbed/main.m b/iOS/testbed/iOSTestbed/main.m new file mode 100644 index 00000000000000..e32bd78c9b42ee --- /dev/null +++ b/iOS/testbed/iOSTestbed/main.m @@ -0,0 +1,16 @@ +// +// main.m +// iOSTestbed +// + +#import +#import "AppDelegate.h" + +int main(int argc, char * argv[]) { + NSString * appDelegateClassName; + @autoreleasepool { + appDelegateClassName = NSStringFromClass([AppDelegate class]); + + return UIApplicationMain(argc, argv, nil, appDelegateClassName); + } +} diff --git a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m new file mode 100644 index 00000000000000..6db38253396c8d --- /dev/null +++ b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m @@ -0,0 +1,162 @@ +#import +#import + +@interface iOSTestbedTests : XCTestCase + +@end + +@implementation iOSTestbedTests + + +- (void)testPython { + const char **argv; + int exit_code; + int failed; + PyStatus status; + PyPreConfig preconfig; + PyConfig config; + PyObject *sys_module; + PyObject *sys_path_attr; + NSArray *test_args; + NSString *python_home; + NSString *path; + wchar_t *wtmp_str; + + NSString *resourcePath = [[NSBundle mainBundle] resourcePath]; + + // Set some other common environment indicators to disable color, as the + // Xcode log can't display color. Stdout will report that it is *not* a + // TTY. + setenv("NO_COLOR", "1", true); + setenv("PY_COLORS", "0", true); + + // Arguments to pass into the test suite runner. + // argv[0] must identify the process; any subsequent arg + // will be handled as if it were an argument to `python -m test` + test_args = [[NSBundle mainBundle] objectForInfoDictionaryKey:@"TestArgs"]; + if (test_args == NULL) { + NSLog(@"Unable to identify test arguments."); + } + argv = malloc(sizeof(char *) * ([test_args count] + 1)); + argv[0] = "iOSTestbed"; + for (int i = 1; i < [test_args count]; i++) { + argv[i] = [[test_args objectAtIndex:i] UTF8String]; + } + NSLog(@"Test command: %@", test_args); + + // Generate an isolated Python configuration. + NSLog(@"Configuring isolated Python..."); + PyPreConfig_InitIsolatedConfig(&preconfig); + PyConfig_InitIsolatedConfig(&config); + + // Configure the Python interpreter: + // Enforce UTF-8 encoding for stderr, stdout, file-system encoding and locale. + // See https://docs.python.org/3/library/os.html#python-utf-8-mode. + preconfig.utf8_mode = 1; + // Use the system logger for stdout/err + config.use_system_logger = 1; + // Don't buffer stdio. We want output to appears in the log immediately + config.buffered_stdio = 0; + // Don't write bytecode; we can't modify the app bundle + // after it has been signed. + config.write_bytecode = 0; + // Ensure that signal handlers are installed + config.install_signal_handlers = 1; + // Run the test module. + config.run_module = Py_DecodeLocale([[test_args objectAtIndex:0] UTF8String], NULL); + // For debugging - enable verbose mode. + // config.verbose = 1; + + NSLog(@"Pre-initializing Python runtime..."); + status = Py_PreInitialize(&preconfig); + if (PyStatus_Exception(status)) { + XCTFail(@"Unable to pre-initialize Python interpreter: %s", status.err_msg); + PyConfig_Clear(&config); + return; + } + + // Set the home for the Python interpreter + python_home = [NSString stringWithFormat:@"%@/python", resourcePath, nil]; + NSLog(@"PythonHome: %@", python_home); + wtmp_str = Py_DecodeLocale([python_home UTF8String], NULL); + status = PyConfig_SetString(&config, &config.home, wtmp_str); + if (PyStatus_Exception(status)) { + XCTFail(@"Unable to set PYTHONHOME: %s", status.err_msg); + PyConfig_Clear(&config); + return; + } + PyMem_RawFree(wtmp_str); + + // Read the site config + status = PyConfig_Read(&config); + if (PyStatus_Exception(status)) { + XCTFail(@"Unable to read site config: %s", status.err_msg); + PyConfig_Clear(&config); + return; + } + + NSLog(@"Configure argc/argv..."); + status = PyConfig_SetBytesArgv(&config, [test_args count], (char**) argv); + if (PyStatus_Exception(status)) { + XCTFail(@"Unable to configure argc/argv: %s", status.err_msg); + PyConfig_Clear(&config); + return; + } + + NSLog(@"Initializing Python runtime..."); + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + XCTFail(@"Unable to initialize Python interpreter: %s", status.err_msg); + PyConfig_Clear(&config); + return; + } + + sys_module = PyImport_ImportModule("sys"); + if (sys_module == NULL) { + XCTFail(@"Could not import sys module"); + return; + } + + sys_path_attr = PyObject_GetAttrString(sys_module, "path"); + if (sys_path_attr == NULL) { + XCTFail(@"Could not access sys.path"); + return; + } + + // Add the app packages path + path = [NSString stringWithFormat:@"%@/app_packages", resourcePath, nil]; + NSLog(@"App packages path: %@", path); + wtmp_str = Py_DecodeLocale([path UTF8String], NULL); + failed = PyList_Insert(sys_path_attr, 0, PyUnicode_FromString([path UTF8String])); + if (failed) { + XCTFail(@"Unable to add app packages to sys.path"); + return; + } + PyMem_RawFree(wtmp_str); + + path = [NSString stringWithFormat:@"%@/app", resourcePath, nil]; + NSLog(@"App path: %@", path); + wtmp_str = Py_DecodeLocale([path UTF8String], NULL); + failed = PyList_Insert(sys_path_attr, 0, PyUnicode_FromString([path UTF8String])); + if (failed) { + XCTFail(@"Unable to add app to sys.path"); + return; + } + PyMem_RawFree(wtmp_str); + + // Ensure the working directory is the app folder. + chdir([path UTF8String]); + + // Start the test suite. Print a separator to differentiate Python startup logs from app logs + NSLog(@"---------------------------------------------------------------------------"); + + exit_code = Py_RunMain(); + XCTAssertEqual(exit_code, 0, @"Test suite did not pass"); + + NSLog(@"---------------------------------------------------------------------------"); + + Py_Finalize(); +} + + +@end diff --git a/install-sh b/install-sh index ec298b53740270..7c56c9c0151036 100755 --- a/install-sh +++ b/install-sh @@ -1,7 +1,7 @@ #!/bin/sh # install - install a program, script, or datafile -scriptversion=2020-11-14.01; # UTC +scriptversion=2023-11-23.18; # UTC # This originates from X11R5 (mit/util/scripts/install.sh), which was # later released in X11R6 (xc/config/util/install.sh) with the @@ -124,9 +124,9 @@ it's up to you to specify -f if you want it. If -S is not specified, no backups are attempted. -Email bug reports to bug-automake@gnu.org. -Automake home page: https://www.gnu.org/software/automake/ -" +Report bugs to . +GNU Automake home page: . +General help using GNU software: ." while test $# -ne 0; do case $1 in diff --git a/pyconfig.h.in b/pyconfig.h.in index e28baef51d5737..30e55158bad4d6 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -16,13 +16,13 @@ support for AIX C++ shared extension modules. */ #undef AIX_GENUINE_CPLUSPLUS -/* The normal alignment of `long', in bytes. */ +/* The normal alignment of 'long', in bytes. */ #undef ALIGNOF_LONG -/* The normal alignment of `max_align_t', in bytes. */ +/* The normal alignment of 'max_align_t', in bytes. */ #undef ALIGNOF_MAX_ALIGN_T -/* The normal alignment of `size_t', in bytes. */ +/* The normal alignment of 'size_t', in bytes. */ #undef ALIGNOF_SIZE_T /* Alternative SOABI used in debug build to load C extensions built in release @@ -47,26 +47,28 @@ /* Define if --enable-ipv6 is specified */ #undef ENABLE_IPV6 -/* Define to 1 if your system stores words within floats with the most - significant word first */ -#undef FLOAT_WORDS_BIGENDIAN - /* Define if getpgrp() must be called as getpgrp(0). */ #undef GETPGRP_HAVE_ARG +/* HACL* library can compile SIMD128 implementations */ +#undef HACL_CAN_COMPILE_SIMD128 + +/* HACL* library can compile SIMD256 implementations */ +#undef HACL_CAN_COMPILE_SIMD256 + /* Define if you have the 'accept' function. */ #undef HAVE_ACCEPT -/* Define to 1 if you have the `accept4' function. */ +/* Define to 1 if you have the 'accept4' function. */ #undef HAVE_ACCEPT4 -/* Define to 1 if you have the `acosh' function. */ +/* Define to 1 if you have the 'acosh' function. */ #undef HAVE_ACOSH /* struct addrinfo (netdb.h) */ #undef HAVE_ADDRINFO -/* Define to 1 if you have the `alarm' function. */ +/* Define to 1 if you have the 'alarm' function. */ #undef HAVE_ALARM /* Define if aligned memory access is required */ @@ -78,19 +80,19 @@ /* Define this if your time.h defines altzone. */ #undef HAVE_ALTZONE -/* Define to 1 if you have the `asinh' function. */ +/* Define to 1 if you have the 'asinh' function. */ #undef HAVE_ASINH /* Define to 1 if you have the header file. */ #undef HAVE_ASM_TYPES_H -/* Define to 1 if you have the `atanh' function. */ +/* Define to 1 if you have the 'atanh' function. */ #undef HAVE_ATANH /* Define if you have the 'bind' function. */ #undef HAVE_BIND -/* Define to 1 if you have the `bind_textdomain_codeset' function. */ +/* Define to 1 if you have the 'bind_textdomain_codeset' function. */ #undef HAVE_BIND_TEXTDOMAIN_CODESET /* Define to 1 if you have the header file. */ @@ -133,43 +135,43 @@ /* Define to 1 if you have the 'chflags' function. */ #undef HAVE_CHFLAGS -/* Define to 1 if you have the `chmod' function. */ +/* Define to 1 if you have the 'chmod' function. */ #undef HAVE_CHMOD -/* Define to 1 if you have the `chown' function. */ +/* Define to 1 if you have the 'chown' function. */ #undef HAVE_CHOWN /* Define if you have the 'chroot' function. */ #undef HAVE_CHROOT -/* Define to 1 if you have the `clock' function. */ +/* Define to 1 if you have the 'clock' function. */ #undef HAVE_CLOCK -/* Define to 1 if you have the `clock_getres' function. */ +/* Define to 1 if you have the 'clock_getres' function. */ #undef HAVE_CLOCK_GETRES -/* Define to 1 if you have the `clock_gettime' function. */ +/* Define to 1 if you have the 'clock_gettime' function. */ #undef HAVE_CLOCK_GETTIME -/* Define to 1 if you have the `clock_nanosleep' function. */ +/* Define to 1 if you have the 'clock_nanosleep' function. */ #undef HAVE_CLOCK_NANOSLEEP -/* Define to 1 if you have the `clock_settime' function. */ +/* Define to 1 if you have the 'clock_settime' function. */ #undef HAVE_CLOCK_SETTIME -/* Define to 1 if the system has the type `clock_t'. */ +/* Define to 1 if the system has the type 'clock_t'. */ #undef HAVE_CLOCK_T -/* Define to 1 if you have the `closefrom' function. */ +/* Define to 1 if you have the 'closefrom' function. */ #undef HAVE_CLOSEFROM -/* Define to 1 if you have the `close_range' function. */ +/* Define to 1 if you have the 'close_range' function. */ #undef HAVE_CLOSE_RANGE /* Define if the C compiler supports computed gotos. */ #undef HAVE_COMPUTED_GOTOS -/* Define to 1 if you have the `confstr' function. */ +/* Define to 1 if you have the 'confstr' function. */ #undef HAVE_CONFSTR /* Define to 1 if you have the header file. */ @@ -178,10 +180,10 @@ /* Define if you have the 'connect' function. */ #undef HAVE_CONNECT -/* Define to 1 if you have the `copy_file_range' function. */ +/* Define to 1 if you have the 'copy_file_range' function. */ #undef HAVE_COPY_FILE_RANGE -/* Define to 1 if you have the `ctermid' function. */ +/* Define to 1 if you have the 'ctermid' function. */ #undef HAVE_CTERMID /* Define if you have the 'ctermid_r' function. */ @@ -226,39 +228,39 @@ /* Define to 1 if you have the header file. */ #undef HAVE_DB_H -/* Define to 1 if you have the declaration of `RTLD_DEEPBIND', and to 0 if you +/* Define to 1 if you have the declaration of 'RTLD_DEEPBIND', and to 0 if you don't. */ #undef HAVE_DECL_RTLD_DEEPBIND -/* Define to 1 if you have the declaration of `RTLD_GLOBAL', and to 0 if you +/* Define to 1 if you have the declaration of 'RTLD_GLOBAL', and to 0 if you don't. */ #undef HAVE_DECL_RTLD_GLOBAL -/* Define to 1 if you have the declaration of `RTLD_LAZY', and to 0 if you +/* Define to 1 if you have the declaration of 'RTLD_LAZY', and to 0 if you don't. */ #undef HAVE_DECL_RTLD_LAZY -/* Define to 1 if you have the declaration of `RTLD_LOCAL', and to 0 if you +/* Define to 1 if you have the declaration of 'RTLD_LOCAL', and to 0 if you don't. */ #undef HAVE_DECL_RTLD_LOCAL -/* Define to 1 if you have the declaration of `RTLD_MEMBER', and to 0 if you +/* Define to 1 if you have the declaration of 'RTLD_MEMBER', and to 0 if you don't. */ #undef HAVE_DECL_RTLD_MEMBER -/* Define to 1 if you have the declaration of `RTLD_NODELETE', and to 0 if you +/* Define to 1 if you have the declaration of 'RTLD_NODELETE', and to 0 if you don't. */ #undef HAVE_DECL_RTLD_NODELETE -/* Define to 1 if you have the declaration of `RTLD_NOLOAD', and to 0 if you +/* Define to 1 if you have the declaration of 'RTLD_NOLOAD', and to 0 if you don't. */ #undef HAVE_DECL_RTLD_NOLOAD -/* Define to 1 if you have the declaration of `RTLD_NOW', and to 0 if you +/* Define to 1 if you have the declaration of 'RTLD_NOW', and to 0 if you don't. */ #undef HAVE_DECL_RTLD_NOW -/* Define to 1 if you have the declaration of `tzname', and to 0 if you don't. +/* Define to 1 if you have the declaration of 'tzname', and to 0 if you don't. */ #undef HAVE_DECL_TZNAME @@ -277,26 +279,29 @@ /* Define to 1 if the dirent structure has a d_type field */ #undef HAVE_DIRENT_D_TYPE -/* Define to 1 if you have the header file, and it defines `DIR'. +/* Define to 1 if you have the header file, and it defines 'DIR'. */ #undef HAVE_DIRENT_H /* Define if you have the 'dirfd' function or macro. */ #undef HAVE_DIRFD +/* Define to 1 if you have the 'dladdr' function. */ +#undef HAVE_DLADDR + /* Define to 1 if you have the header file. */ #undef HAVE_DLFCN_H -/* Define to 1 if you have the `dlopen' function. */ +/* Define to 1 if you have the 'dlopen' function. */ #undef HAVE_DLOPEN -/* Define to 1 if you have the `dup' function. */ +/* Define to 1 if you have the 'dup' function. */ #undef HAVE_DUP -/* Define to 1 if you have the `dup2' function. */ +/* Define to 1 if you have the 'dup2' function. */ #undef HAVE_DUP2 -/* Define to 1 if you have the `dup3' function. */ +/* Define to 1 if you have the 'dup3' function. */ #undef HAVE_DUP3 /* Define if you have the '_dyld_shared_cache_contains_path' function. */ @@ -317,10 +322,10 @@ /* Define if you have the 'epoll_create1' function. */ #undef HAVE_EPOLL_CREATE1 -/* Define to 1 if you have the `erf' function. */ +/* Define to 1 if you have the 'erf' function. */ #undef HAVE_ERF -/* Define to 1 if you have the `erfc' function. */ +/* Define to 1 if you have the 'erfc' function. */ #undef HAVE_ERFC /* Define to 1 if you have the header file. */ @@ -329,34 +334,34 @@ /* Define if you have the 'eventfd' function. */ #undef HAVE_EVENTFD -/* Define to 1 if you have the `execv' function. */ +/* Define to 1 if you have the 'execv' function. */ #undef HAVE_EXECV -/* Define to 1 if you have the `explicit_bzero' function. */ +/* Define to 1 if you have the 'explicit_bzero' function. */ #undef HAVE_EXPLICIT_BZERO -/* Define to 1 if you have the `explicit_memset' function. */ +/* Define to 1 if you have the 'explicit_memset' function. */ #undef HAVE_EXPLICIT_MEMSET -/* Define to 1 if you have the `expm1' function. */ +/* Define to 1 if you have the 'expm1' function. */ #undef HAVE_EXPM1 -/* Define to 1 if you have the `faccessat' function. */ +/* Define to 1 if you have the 'faccessat' function. */ #undef HAVE_FACCESSAT /* Define if you have the 'fchdir' function. */ #undef HAVE_FCHDIR -/* Define to 1 if you have the `fchmod' function. */ +/* Define to 1 if you have the 'fchmod' function. */ #undef HAVE_FCHMOD -/* Define to 1 if you have the `fchmodat' function. */ +/* Define to 1 if you have the 'fchmodat' function. */ #undef HAVE_FCHMODAT -/* Define to 1 if you have the `fchown' function. */ +/* Define to 1 if you have the 'fchown' function. */ #undef HAVE_FCHOWN -/* Define to 1 if you have the `fchownat' function. */ +/* Define to 1 if you have the 'fchownat' function. */ #undef HAVE_FCHOWNAT /* Define to 1 if you have the header file. */ @@ -365,13 +370,13 @@ /* Define if you have the 'fdatasync' function. */ #undef HAVE_FDATASYNC -/* Define to 1 if you have the `fdopendir' function. */ +/* Define to 1 if you have the 'fdopendir' function. */ #undef HAVE_FDOPENDIR -/* Define to 1 if you have the `fdwalk' function. */ +/* Define to 1 if you have the 'fdwalk' function. */ #undef HAVE_FDWALK -/* Define to 1 if you have the `fexecve' function. */ +/* Define to 1 if you have the 'fexecve' function. */ #undef HAVE_FEXECVE /* Define if you have the 'ffi_closure_alloc' function. */ @@ -383,58 +388,58 @@ /* Define if you have the 'ffi_prep_closure_loc' function. */ #undef HAVE_FFI_PREP_CLOSURE_LOC -/* Define to 1 if you have the `flock' function. */ +/* Define to 1 if you have the 'flock' function. */ #undef HAVE_FLOCK -/* Define to 1 if you have the `fork' function. */ +/* Define to 1 if you have the 'fork' function. */ #undef HAVE_FORK -/* Define to 1 if you have the `fork1' function. */ +/* Define to 1 if you have the 'fork1' function. */ #undef HAVE_FORK1 -/* Define to 1 if you have the `forkpty' function. */ +/* Define to 1 if you have the 'forkpty' function. */ #undef HAVE_FORKPTY -/* Define to 1 if you have the `fpathconf' function. */ +/* Define to 1 if you have the 'fpathconf' function. */ #undef HAVE_FPATHCONF -/* Define to 1 if you have the `fseek64' function. */ +/* Define to 1 if you have the 'fseek64' function. */ #undef HAVE_FSEEK64 -/* Define to 1 if you have the `fseeko' function. */ +/* Define to 1 if you have the 'fseeko' function. */ #undef HAVE_FSEEKO -/* Define to 1 if you have the `fstatat' function. */ +/* Define to 1 if you have the 'fstatat' function. */ #undef HAVE_FSTATAT -/* Define to 1 if you have the `fstatvfs' function. */ +/* Define to 1 if you have the 'fstatvfs' function. */ #undef HAVE_FSTATVFS /* Define if you have the 'fsync' function. */ #undef HAVE_FSYNC -/* Define to 1 if you have the `ftell64' function. */ +/* Define to 1 if you have the 'ftell64' function. */ #undef HAVE_FTELL64 -/* Define to 1 if you have the `ftello' function. */ +/* Define to 1 if you have the 'ftello' function. */ #undef HAVE_FTELLO -/* Define to 1 if you have the `ftime' function. */ +/* Define to 1 if you have the 'ftime' function. */ #undef HAVE_FTIME -/* Define to 1 if you have the `ftruncate' function. */ +/* Define to 1 if you have the 'ftruncate' function. */ #undef HAVE_FTRUNCATE -/* Define to 1 if you have the `futimens' function. */ +/* Define to 1 if you have the 'futimens' function. */ #undef HAVE_FUTIMENS -/* Define to 1 if you have the `futimes' function. */ +/* Define to 1 if you have the 'futimes' function. */ #undef HAVE_FUTIMES -/* Define to 1 if you have the `futimesat' function. */ +/* Define to 1 if you have the 'futimesat' function. */ #undef HAVE_FUTIMESAT -/* Define to 1 if you have the `gai_strerror' function. */ +/* Define to 1 if you have the 'gai_strerror' function. */ #undef HAVE_GAI_STRERROR /* Define if we can use gcc inline assembler to get and set mc68881 fpcr */ @@ -465,40 +470,40 @@ /* Define this if you have flockfile(), getc_unlocked(), and funlockfile() */ #undef HAVE_GETC_UNLOCKED -/* Define to 1 if you have the `getegid' function. */ +/* Define to 1 if you have the 'getegid' function. */ #undef HAVE_GETEGID -/* Define to 1 if you have the `getentropy' function. */ +/* Define to 1 if you have the 'getentropy' function. */ #undef HAVE_GETENTROPY -/* Define to 1 if you have the `geteuid' function. */ +/* Define to 1 if you have the 'geteuid' function. */ #undef HAVE_GETEUID -/* Define to 1 if you have the `getgid' function. */ +/* Define to 1 if you have the 'getgid' function. */ #undef HAVE_GETGID -/* Define to 1 if you have the `getgrent' function. */ +/* Define to 1 if you have the 'getgrent' function. */ #undef HAVE_GETGRENT -/* Define to 1 if you have the `getgrgid' function. */ +/* Define to 1 if you have the 'getgrgid' function. */ #undef HAVE_GETGRGID -/* Define to 1 if you have the `getgrgid_r' function. */ +/* Define to 1 if you have the 'getgrgid_r' function. */ #undef HAVE_GETGRGID_R -/* Define to 1 if you have the `getgrnam_r' function. */ +/* Define to 1 if you have the 'getgrnam_r' function. */ #undef HAVE_GETGRNAM_R -/* Define to 1 if you have the `getgrouplist' function. */ +/* Define to 1 if you have the 'getgrouplist' function. */ #undef HAVE_GETGROUPLIST -/* Define to 1 if you have the `getgroups' function. */ +/* Define to 1 if you have the 'getgroups' function. */ #undef HAVE_GETGROUPS /* Define if you have the 'gethostbyaddr' function. */ #undef HAVE_GETHOSTBYADDR -/* Define to 1 if you have the `gethostbyname' function. */ +/* Define to 1 if you have the 'gethostbyname' function. */ #undef HAVE_GETHOSTBYNAME /* Define this if you have some version of gethostbyname_r() */ @@ -513,19 +518,19 @@ /* Define this if you have the 6-arg version of gethostbyname_r(). */ #undef HAVE_GETHOSTBYNAME_R_6_ARG -/* Define to 1 if you have the `gethostname' function. */ +/* Define to 1 if you have the 'gethostname' function. */ #undef HAVE_GETHOSTNAME -/* Define to 1 if you have the `getitimer' function. */ +/* Define to 1 if you have the 'getitimer' function. */ #undef HAVE_GETITIMER -/* Define to 1 if you have the `getloadavg' function. */ +/* Define to 1 if you have the 'getloadavg' function. */ #undef HAVE_GETLOADAVG -/* Define to 1 if you have the `getlogin' function. */ +/* Define to 1 if you have the 'getlogin' function. */ #undef HAVE_GETLOGIN -/* Define to 1 if you have the `getnameinfo' function. */ +/* Define to 1 if you have the 'getnameinfo' function. */ #undef HAVE_GETNAMEINFO /* Define if you have the 'getpagesize' function. */ @@ -534,34 +539,34 @@ /* Define if you have the 'getpeername' function. */ #undef HAVE_GETPEERNAME -/* Define to 1 if you have the `getpgid' function. */ +/* Define to 1 if you have the 'getpgid' function. */ #undef HAVE_GETPGID -/* Define to 1 if you have the `getpgrp' function. */ +/* Define to 1 if you have the 'getpgrp' function. */ #undef HAVE_GETPGRP -/* Define to 1 if you have the `getpid' function. */ +/* Define to 1 if you have the 'getpid' function. */ #undef HAVE_GETPID -/* Define to 1 if you have the `getppid' function. */ +/* Define to 1 if you have the 'getppid' function. */ #undef HAVE_GETPPID -/* Define to 1 if you have the `getpriority' function. */ +/* Define to 1 if you have the 'getpriority' function. */ #undef HAVE_GETPRIORITY /* Define if you have the 'getprotobyname' function. */ #undef HAVE_GETPROTOBYNAME -/* Define to 1 if you have the `getpwent' function. */ +/* Define to 1 if you have the 'getpwent' function. */ #undef HAVE_GETPWENT -/* Define to 1 if you have the `getpwnam_r' function. */ +/* Define to 1 if you have the 'getpwnam_r' function. */ #undef HAVE_GETPWNAM_R -/* Define to 1 if you have the `getpwuid' function. */ +/* Define to 1 if you have the 'getpwuid' function. */ #undef HAVE_GETPWUID -/* Define to 1 if you have the `getpwuid_r' function. */ +/* Define to 1 if you have the 'getpwuid_r' function. */ #undef HAVE_GETPWUID_R /* Define to 1 if the getrandom() function is available */ @@ -570,13 +575,13 @@ /* Define to 1 if the Linux getrandom() syscall is available */ #undef HAVE_GETRANDOM_SYSCALL -/* Define to 1 if you have the `getresgid' function. */ +/* Define to 1 if you have the 'getresgid' function. */ #undef HAVE_GETRESGID -/* Define to 1 if you have the `getresuid' function. */ +/* Define to 1 if you have the 'getresuid' function. */ #undef HAVE_GETRESUID -/* Define to 1 if you have the `getrusage' function. */ +/* Define to 1 if you have the 'getrusage' function. */ #undef HAVE_GETRUSAGE /* Define if you have the 'getservbyname' function. */ @@ -585,29 +590,29 @@ /* Define if you have the 'getservbyport' function. */ #undef HAVE_GETSERVBYPORT -/* Define to 1 if you have the `getsid' function. */ +/* Define to 1 if you have the 'getsid' function. */ #undef HAVE_GETSID /* Define if you have the 'getsockname' function. */ #undef HAVE_GETSOCKNAME -/* Define to 1 if you have the `getspent' function. */ +/* Define to 1 if you have the 'getspent' function. */ #undef HAVE_GETSPENT -/* Define to 1 if you have the `getspnam' function. */ +/* Define to 1 if you have the 'getspnam' function. */ #undef HAVE_GETSPNAM -/* Define to 1 if you have the `getuid' function. */ +/* Define to 1 if you have the 'getuid' function. */ #undef HAVE_GETUID -/* Define to 1 if you have the `getwd' function. */ +/* Define to 1 if you have the 'getwd' function. */ #undef HAVE_GETWD /* Define if glibc has incorrect _FORTIFY_SOURCE wrappers for memmove and bcopy. */ #undef HAVE_GLIBC_MEMMOVE_BUG -/* Define to 1 if you have the `grantpt' function. */ +/* Define to 1 if you have the 'grantpt' function. */ #undef HAVE_GRANTPT /* Define to 1 if you have the header file. */ @@ -619,7 +624,7 @@ /* Define this if you have le64toh() */ #undef HAVE_HTOLE64 -/* Define to 1 if you have the `if_nameindex' function. */ +/* Define to 1 if you have the 'if_nameindex' function. */ #undef HAVE_IF_NAMEINDEX /* Define if you have the 'inet_aton' function. */ @@ -631,7 +636,7 @@ /* Define if you have the 'inet_pton' function. */ #undef HAVE_INET_PTON -/* Define to 1 if you have the `initgroups' function. */ +/* Define to 1 if you have the 'initgroups' function. */ #undef HAVE_INITGROUPS /* Define to 1 if you have the header file. */ @@ -643,10 +648,10 @@ /* Define if gcc has the ipa-pure-const bug. */ #undef HAVE_IPA_PURE_CONST_BUG -/* Define to 1 if you have the `kill' function. */ +/* Define to 1 if you have the 'kill' function. */ #undef HAVE_KILL -/* Define to 1 if you have the `killpg' function. */ +/* Define to 1 if you have the 'killpg' function. */ #undef HAVE_KILLPG /* Define if you have the 'kqueue' function. */ @@ -664,37 +669,31 @@ /* Define to 1 if you have the 'lchflags' function. */ #undef HAVE_LCHFLAGS -/* Define to 1 if you have the `lchmod' function. */ +/* Define to 1 if you have the 'lchmod' function. */ #undef HAVE_LCHMOD -/* Define to 1 if you have the `lchown' function. */ +/* Define to 1 if you have the 'lchown' function. */ #undef HAVE_LCHOWN -/* Define to 1 if you want to build _blake2 module with libb2 */ -#undef HAVE_LIBB2 - /* Define to 1 if you have the `db' library (-ldb). */ #undef HAVE_LIBDB -/* Define to 1 if you have the `dl' library (-ldl). */ +/* Define to 1 if you have the 'dl' library (-ldl). */ #undef HAVE_LIBDL -/* Define to 1 if you have the `dld' library (-ldld). */ +/* Define to 1 if you have the 'dld' library (-ldld). */ #undef HAVE_LIBDLD -/* Define to 1 if you have the `ieee' library (-lieee). */ +/* Define to 1 if you have the 'ieee' library (-lieee). */ #undef HAVE_LIBIEEE /* Define to 1 if you have the header file. */ #undef HAVE_LIBINTL_H -/* Define to 1 if you have the `resolv' library (-lresolv). */ -#undef HAVE_LIBRESOLV - -/* Define to 1 if you have the `sendfile' library (-lsendfile). */ +/* Define to 1 if you have the 'sendfile' library (-lsendfile). */ #undef HAVE_LIBSENDFILE -/* Define to 1 if you have the `sqlite3' library (-lsqlite3). */ +/* Define to 1 if you have the 'sqlite3' library (-lsqlite3). */ #undef HAVE_LIBSQLITE3 /* Define to 1 if you have the header file. */ @@ -703,7 +702,7 @@ /* Define if you have the 'link' function. */ #undef HAVE_LINK -/* Define to 1 if you have the `linkat' function. */ +/* Define to 1 if you have the 'linkat' function. */ #undef HAVE_LINKAT /* Define to 1 if you have the header file. */ @@ -736,6 +735,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_LINUX_MEMFD_H +/* Define to 1 if you have the header file. */ +#undef HAVE_LINUX_NETFILTER_IPV4_H + /* Define to 1 if you have the header file. */ #undef HAVE_LINUX_NETLINK_H @@ -745,6 +747,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_LINUX_RANDOM_H +/* Define to 1 if you have the header file. */ +#undef HAVE_LINUX_SCHED_H + /* Define to 1 if you have the header file. */ #undef HAVE_LINUX_SOUNDCARD_H @@ -760,85 +765,106 @@ /* Define if you have the 'listen' function. */ #undef HAVE_LISTEN -/* Define to 1 if you have the `lockf' function. */ +/* Define to 1 if you have the 'lockf' function. */ #undef HAVE_LOCKF -/* Define to 1 if you have the `log1p' function. */ +/* Define to 1 if you have the 'log1p' function. */ #undef HAVE_LOG1P -/* Define to 1 if you have the `log2' function. */ +/* Define to 1 if you have the 'log2' function. */ #undef HAVE_LOG2 /* Define to 1 if you have the `login_tty' function. */ #undef HAVE_LOGIN_TTY -/* Define to 1 if the system has the type `long double'. */ +/* Define to 1 if the system has the type 'long double'. */ #undef HAVE_LONG_DOUBLE -/* Define to 1 if you have the `lstat' function. */ +/* Define to 1 if you have the 'lstat' function. */ #undef HAVE_LSTAT -/* Define to 1 if you have the `lutimes' function. */ +/* Define to 1 if you have the 'lutimes' function. */ #undef HAVE_LUTIMES /* Define to 1 if you have the header file. */ #undef HAVE_LZMA_H -/* Define to 1 if you have the `madvise' function. */ +/* Define to 1 if you have the 'madvise' function. */ #undef HAVE_MADVISE /* Define this if you have the makedev macro. */ #undef HAVE_MAKEDEV -/* Define to 1 if you have the `mbrtowc' function. */ +/* Define to 1 if you have the 'mbrtowc' function. */ #undef HAVE_MBRTOWC /* Define if you have the 'memfd_create' function. */ #undef HAVE_MEMFD_CREATE -/* Define to 1 if you have the `memrchr' function. */ +/* Define to 1 if you have the 'memrchr' function. */ #undef HAVE_MEMRCHR /* Define to 1 if you have the header file. */ #undef HAVE_MINIX_CONFIG_H -/* Define to 1 if you have the `mkdirat' function. */ +/* Define to 1 if you have the 'mkdirat' function. */ #undef HAVE_MKDIRAT -/* Define to 1 if you have the `mkfifo' function. */ +/* Define to 1 if you have the 'mkfifo' function. */ #undef HAVE_MKFIFO -/* Define to 1 if you have the `mkfifoat' function. */ +/* Define to 1 if you have the 'mkfifoat' function. */ #undef HAVE_MKFIFOAT -/* Define to 1 if you have the `mknod' function. */ +/* Define to 1 if you have the 'mknod' function. */ #undef HAVE_MKNOD -/* Define to 1 if you have the `mknodat' function. */ +/* Define to 1 if you have the 'mknodat' function. */ #undef HAVE_MKNODAT -/* Define to 1 if you have the `mktime' function. */ +/* Define to 1 if you have the 'mktime' function. */ #undef HAVE_MKTIME -/* Define to 1 if you have the `mmap' function. */ +/* Define to 1 if you have the 'mmap' function. */ #undef HAVE_MMAP -/* Define to 1 if you have the `mremap' function. */ +/* Define to 1 if you have the 'mremap' function. */ #undef HAVE_MREMAP -/* Define to 1 if you have the `nanosleep' function. */ +/* Define to 1 if you have the 'nanosleep' function. */ #undef HAVE_NANOSLEEP -/* Define to 1 if you have the `ncursesw' library. */ +/* Define if you have the 'ncurses' library */ +#undef HAVE_NCURSES + +/* Define if you have the 'ncursesw' library */ #undef HAVE_NCURSESW +/* Define to 1 if you have the header file. */ +#undef HAVE_NCURSESW_CURSES_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_NCURSESW_NCURSES_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_NCURSESW_PANEL_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_NCURSES_CURSES_H + /* Define to 1 if you have the header file. */ #undef HAVE_NCURSES_H +/* Define to 1 if you have the header file. */ +#undef HAVE_NCURSES_NCURSES_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_NCURSES_PANEL_H + /* Define to 1 if you have the header file. */ #undef HAVE_NDBM_H -/* Define to 1 if you have the header file, and it defines `DIR'. */ +/* Define to 1 if you have the header file, and it defines 'DIR'. */ #undef HAVE_NDIR_H /* Define to 1 if you have the header file. */ @@ -862,72 +888,78 @@ /* Define to 1 if you have the header file. */ #undef HAVE_NET_IF_H -/* Define to 1 if you have the `nice' function. */ +/* Define to 1 if you have the 'nice' function. */ #undef HAVE_NICE /* Define if the internal form of wchar_t in non-Unicode locales is not Unicode. */ #undef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION -/* Define to 1 if you have the `openat' function. */ +/* Define to 1 if you have the 'openat' function. */ #undef HAVE_OPENAT -/* Define to 1 if you have the `opendir' function. */ +/* Define to 1 if you have the 'opendir' function. */ #undef HAVE_OPENDIR -/* Define to 1 if you have the `openpty' function. */ +/* Define to 1 if you have the 'openpty' function. */ #undef HAVE_OPENPTY +/* Define if you have the 'panel' library */ +#undef HAVE_PANEL + +/* Define if you have the 'panelw' library */ +#undef HAVE_PANELW + /* Define to 1 if you have the header file. */ #undef HAVE_PANEL_H -/* Define to 1 if you have the `pathconf' function. */ +/* Define to 1 if you have the 'pathconf' function. */ #undef HAVE_PATHCONF -/* Define to 1 if you have the `pause' function. */ +/* Define to 1 if you have the 'pause' function. */ #undef HAVE_PAUSE -/* Define to 1 if you have the `pipe' function. */ +/* Define to 1 if you have the 'pipe' function. */ #undef HAVE_PIPE -/* Define to 1 if you have the `pipe2' function. */ +/* Define to 1 if you have the 'pipe2' function. */ #undef HAVE_PIPE2 -/* Define to 1 if you have the `plock' function. */ +/* Define to 1 if you have the 'plock' function. */ #undef HAVE_PLOCK -/* Define to 1 if you have the `poll' function. */ +/* Define to 1 if you have the 'poll' function. */ #undef HAVE_POLL /* Define to 1 if you have the header file. */ #undef HAVE_POLL_H -/* Define to 1 if you have the `posix_fadvise' function. */ +/* Define to 1 if you have the 'posix_fadvise' function. */ #undef HAVE_POSIX_FADVISE -/* Define to 1 if you have the `posix_fallocate' function. */ +/* Define to 1 if you have the 'posix_fallocate' function. */ #undef HAVE_POSIX_FALLOCATE -/* Define to 1 if you have the `posix_openpt' function. */ +/* Define to 1 if you have the 'posix_openpt' function. */ #undef HAVE_POSIX_OPENPT -/* Define to 1 if you have the `posix_spawn' function. */ +/* Define to 1 if you have the 'posix_spawn' function. */ #undef HAVE_POSIX_SPAWN -/* Define to 1 if you have the `posix_spawnp' function. */ +/* Define to 1 if you have the 'posix_spawnp' function. */ #undef HAVE_POSIX_SPAWNP -/* Define to 1 if you have the `posix_spawn_file_actions_addclosefrom_np' +/* Define to 1 if you have the 'posix_spawn_file_actions_addclosefrom_np' function. */ #undef HAVE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSEFROM_NP -/* Define to 1 if you have the `pread' function. */ +/* Define to 1 if you have the 'pread' function. */ #undef HAVE_PREAD -/* Define to 1 if you have the `preadv' function. */ +/* Define to 1 if you have the 'preadv' function. */ #undef HAVE_PREADV -/* Define to 1 if you have the `preadv2' function. */ +/* Define to 1 if you have the 'preadv2' function. */ #undef HAVE_PREADV2 /* Define if you have the 'prlimit' function. */ @@ -936,77 +968,83 @@ /* Define to 1 if you have the header file. */ #undef HAVE_PROCESS_H -/* Define to 1 if you have the `process_vm_readv' function. */ +/* Define to 1 if you have the 'process_vm_readv' function. */ #undef HAVE_PROCESS_VM_READV /* Define if your compiler supports function prototype */ #undef HAVE_PROTOTYPES -/* Define to 1 if you have the `pthread_condattr_setclock' function. */ +/* Define to 1 if you have the 'pthread_condattr_setclock' function. */ #undef HAVE_PTHREAD_CONDATTR_SETCLOCK -/* Define to 1 if you have the `pthread_cond_timedwait_relative_np' function. +/* Define to 1 if you have the 'pthread_cond_timedwait_relative_np' function. */ #undef HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP /* Defined for Solaris 2.6 bug in pthread header. */ #undef HAVE_PTHREAD_DESTRUCTOR -/* Define to 1 if you have the `pthread_getcpuclockid' function. */ +/* Define to 1 if you have the 'pthread_getcpuclockid' function. */ #undef HAVE_PTHREAD_GETCPUCLOCKID +/* Define to 1 if you have the 'pthread_getname_np' function. */ +#undef HAVE_PTHREAD_GETNAME_NP + /* Define to 1 if you have the header file. */ #undef HAVE_PTHREAD_H -/* Define to 1 if you have the `pthread_init' function. */ +/* Define to 1 if you have the 'pthread_init' function. */ #undef HAVE_PTHREAD_INIT -/* Define to 1 if you have the `pthread_kill' function. */ +/* Define to 1 if you have the 'pthread_kill' function. */ #undef HAVE_PTHREAD_KILL -/* Define to 1 if you have the `pthread_sigmask' function. */ +/* Define to 1 if you have the 'pthread_setname_np' function. */ +#undef HAVE_PTHREAD_SETNAME_NP + +/* Define to 1 if you have the 'pthread_sigmask' function. */ #undef HAVE_PTHREAD_SIGMASK /* Define if platform requires stubbed pthreads support */ #undef HAVE_PTHREAD_STUBS -/* Define to 1 if you have the `ptsname' function. */ +/* Define to 1 if you have the 'ptsname' function. */ #undef HAVE_PTSNAME -/* Define to 1 if you have the `ptsname_r' function. */ +/* Define to 1 if you have the 'ptsname_r' function. */ #undef HAVE_PTSNAME_R /* Define to 1 if you have the header file. */ #undef HAVE_PTY_H -/* Define to 1 if you have the `pwrite' function. */ +/* Define to 1 if you have the 'pwrite' function. */ #undef HAVE_PWRITE -/* Define to 1 if you have the `pwritev' function. */ +/* Define to 1 if you have the 'pwritev' function. */ #undef HAVE_PWRITEV -/* Define to 1 if you have the `pwritev2' function. */ +/* Define to 1 if you have the 'pwritev2' function. */ #undef HAVE_PWRITEV2 /* Define to 1 if you have the header file. */ #undef HAVE_READLINE_READLINE_H -/* Define to 1 if you have the `readlink' function. */ +/* Define to 1 if you have the 'readlink' function. */ #undef HAVE_READLINK -/* Define to 1 if you have the `readlinkat' function. */ +/* Define to 1 if you have the 'readlinkat' function. */ #undef HAVE_READLINKAT -/* Define to 1 if you have the `readv' function. */ +/* Define to 1 if you have the 'readv' function. */ #undef HAVE_READV -/* Define to 1 if you have the `realpath' function. */ +/* Define to 1 if you have the 'realpath' function. */ #undef HAVE_REALPATH /* Define if you have the 'recvfrom' function. */ #undef HAVE_RECVFROM -/* Define to 1 if you have the `renameat' function. */ +/* Define to 1 if you have the 'renameat' function. */ #undef HAVE_RENAMEAT /* Define if readline supports append_history */ @@ -1015,7 +1053,7 @@ /* Define if you can turn off readline's signal handling. */ #undef HAVE_RL_CATCH_SIGNAL -/* Define to 1 if the system has the type `rl_compdisp_func_t'. */ +/* Define to 1 if the system has the type 'rl_compdisp_func_t'. */ #undef HAVE_RL_COMPDISP_FUNC_T /* Define if you have readline 2.2 */ @@ -1036,154 +1074,154 @@ /* Define if you have readline 4.0 */ #undef HAVE_RL_RESIZE_TERMINAL -/* Define to 1 if you have the `rtpSpawn' function. */ +/* Define to 1 if you have the 'rtpSpawn' function. */ #undef HAVE_RTPSPAWN -/* Define to 1 if you have the `sched_get_priority_max' function. */ +/* Define to 1 if you have the 'sched_get_priority_max' function. */ #undef HAVE_SCHED_GET_PRIORITY_MAX /* Define to 1 if you have the header file. */ #undef HAVE_SCHED_H -/* Define to 1 if you have the `sched_rr_get_interval' function. */ +/* Define to 1 if you have the 'sched_rr_get_interval' function. */ #undef HAVE_SCHED_RR_GET_INTERVAL -/* Define to 1 if you have the `sched_setaffinity' function. */ +/* Define to 1 if you have the 'sched_setaffinity' function. */ #undef HAVE_SCHED_SETAFFINITY -/* Define to 1 if you have the `sched_setparam' function. */ +/* Define to 1 if you have the 'sched_setparam' function. */ #undef HAVE_SCHED_SETPARAM -/* Define to 1 if you have the `sched_setscheduler' function. */ +/* Define to 1 if you have the 'sched_setscheduler' function. */ #undef HAVE_SCHED_SETSCHEDULER -/* Define to 1 if you have the `sem_clockwait' function. */ +/* Define to 1 if you have the 'sem_clockwait' function. */ #undef HAVE_SEM_CLOCKWAIT -/* Define to 1 if you have the `sem_getvalue' function. */ +/* Define to 1 if you have the 'sem_getvalue' function. */ #undef HAVE_SEM_GETVALUE -/* Define to 1 if you have the `sem_open' function. */ +/* Define to 1 if you have the 'sem_open' function. */ #undef HAVE_SEM_OPEN -/* Define to 1 if you have the `sem_timedwait' function. */ +/* Define to 1 if you have the 'sem_timedwait' function. */ #undef HAVE_SEM_TIMEDWAIT -/* Define to 1 if you have the `sem_unlink' function. */ +/* Define to 1 if you have the 'sem_unlink' function. */ #undef HAVE_SEM_UNLINK -/* Define to 1 if you have the `sendfile' function. */ +/* Define to 1 if you have the 'sendfile' function. */ #undef HAVE_SENDFILE /* Define if you have the 'sendto' function. */ #undef HAVE_SENDTO -/* Define to 1 if you have the `setegid' function. */ +/* Define to 1 if you have the 'setegid' function. */ #undef HAVE_SETEGID -/* Define to 1 if you have the `seteuid' function. */ +/* Define to 1 if you have the 'seteuid' function. */ #undef HAVE_SETEUID -/* Define to 1 if you have the `setgid' function. */ +/* Define to 1 if you have the 'setgid' function. */ #undef HAVE_SETGID /* Define if you have the 'setgroups' function. */ #undef HAVE_SETGROUPS -/* Define to 1 if you have the `sethostname' function. */ +/* Define to 1 if you have the 'sethostname' function. */ #undef HAVE_SETHOSTNAME -/* Define to 1 if you have the `setitimer' function. */ +/* Define to 1 if you have the 'setitimer' function. */ #undef HAVE_SETITIMER /* Define to 1 if you have the header file. */ #undef HAVE_SETJMP_H -/* Define to 1 if you have the `setlocale' function. */ +/* Define to 1 if you have the 'setlocale' function. */ #undef HAVE_SETLOCALE -/* Define to 1 if you have the `setns' function. */ +/* Define to 1 if you have the 'setns' function. */ #undef HAVE_SETNS -/* Define to 1 if you have the `setpgid' function. */ +/* Define to 1 if you have the 'setpgid' function. */ #undef HAVE_SETPGID -/* Define to 1 if you have the `setpgrp' function. */ +/* Define to 1 if you have the 'setpgrp' function. */ #undef HAVE_SETPGRP -/* Define to 1 if you have the `setpriority' function. */ +/* Define to 1 if you have the 'setpriority' function. */ #undef HAVE_SETPRIORITY -/* Define to 1 if you have the `setregid' function. */ +/* Define to 1 if you have the 'setregid' function. */ #undef HAVE_SETREGID -/* Define to 1 if you have the `setresgid' function. */ +/* Define to 1 if you have the 'setresgid' function. */ #undef HAVE_SETRESGID -/* Define to 1 if you have the `setresuid' function. */ +/* Define to 1 if you have the 'setresuid' function. */ #undef HAVE_SETRESUID -/* Define to 1 if you have the `setreuid' function. */ +/* Define to 1 if you have the 'setreuid' function. */ #undef HAVE_SETREUID -/* Define to 1 if you have the `setsid' function. */ +/* Define to 1 if you have the 'setsid' function. */ #undef HAVE_SETSID /* Define if you have the 'setsockopt' function. */ #undef HAVE_SETSOCKOPT -/* Define to 1 if you have the `setuid' function. */ +/* Define to 1 if you have the 'setuid' function. */ #undef HAVE_SETUID -/* Define to 1 if you have the `setvbuf' function. */ +/* Define to 1 if you have the 'setvbuf' function. */ #undef HAVE_SETVBUF /* Define to 1 if you have the header file. */ #undef HAVE_SHADOW_H -/* Define to 1 if you have the `shm_open' function. */ +/* Define to 1 if you have the 'shm_open' function. */ #undef HAVE_SHM_OPEN -/* Define to 1 if you have the `shm_unlink' function. */ +/* Define to 1 if you have the 'shm_unlink' function. */ #undef HAVE_SHM_UNLINK -/* Define to 1 if you have the `shutdown' function. */ +/* Define to 1 if you have the 'shutdown' function. */ #undef HAVE_SHUTDOWN -/* Define to 1 if you have the `sigaction' function. */ +/* Define to 1 if you have the 'sigaction' function. */ #undef HAVE_SIGACTION -/* Define to 1 if you have the `sigaltstack' function. */ +/* Define to 1 if you have the 'sigaltstack' function. */ #undef HAVE_SIGALTSTACK -/* Define to 1 if you have the `sigfillset' function. */ +/* Define to 1 if you have the 'sigfillset' function. */ #undef HAVE_SIGFILLSET -/* Define to 1 if `si_band' is a member of `siginfo_t'. */ +/* Define to 1 if 'si_band' is a member of 'siginfo_t'. */ #undef HAVE_SIGINFO_T_SI_BAND -/* Define to 1 if you have the `siginterrupt' function. */ +/* Define to 1 if you have the 'siginterrupt' function. */ #undef HAVE_SIGINTERRUPT /* Define to 1 if you have the header file. */ #undef HAVE_SIGNAL_H -/* Define to 1 if you have the `sigpending' function. */ +/* Define to 1 if you have the 'sigpending' function. */ #undef HAVE_SIGPENDING -/* Define to 1 if you have the `sigrelse' function. */ +/* Define to 1 if you have the 'sigrelse' function. */ #undef HAVE_SIGRELSE -/* Define to 1 if you have the `sigtimedwait' function. */ +/* Define to 1 if you have the 'sigtimedwait' function. */ #undef HAVE_SIGTIMEDWAIT -/* Define to 1 if you have the `sigwait' function. */ +/* Define to 1 if you have the 'sigwait' function. */ #undef HAVE_SIGWAIT -/* Define to 1 if you have the `sigwaitinfo' function. */ +/* Define to 1 if you have the 'sigwaitinfo' function. */ #undef HAVE_SIGWAITINFO -/* Define to 1 if you have the `snprintf' function. */ +/* Define to 1 if you have the 'snprintf' function. */ #undef HAVE_SNPRINTF /* struct sockaddr_alg (linux/if_alg.h) */ @@ -1201,19 +1239,19 @@ /* Define if you have the 'socketpair' function. */ #undef HAVE_SOCKETPAIR -/* Define to 1 if the system has the type `socklen_t'. */ +/* Define to 1 if the system has the type 'socklen_t'. */ #undef HAVE_SOCKLEN_T /* Define to 1 if you have the header file. */ #undef HAVE_SPAWN_H -/* Define to 1 if you have the `splice' function. */ +/* Define to 1 if you have the 'splice' function. */ #undef HAVE_SPLICE -/* Define to 1 if the system has the type `ssize_t'. */ +/* Define to 1 if the system has the type 'ssize_t'. */ #undef HAVE_SSIZE_T -/* Define to 1 if you have the `statvfs' function. */ +/* Define to 1 if you have the 'statvfs' function. */ #undef HAVE_STATVFS /* Define if you have struct stat.st_mtim.tv_nsec */ @@ -1234,7 +1272,7 @@ /* Has stdatomic.h with atomic_int and atomic_uintptr_t */ #undef HAVE_STD_ATOMIC -/* Define to 1 if you have the `strftime' function. */ +/* Define to 1 if you have the 'strftime' function. */ #undef HAVE_STRFTIME /* Define to 1 if you have the header file. */ @@ -1243,52 +1281,52 @@ /* Define to 1 if you have the header file. */ #undef HAVE_STRING_H -/* Define to 1 if you have the `strlcpy' function. */ +/* Define to 1 if you have the 'strlcpy' function. */ #undef HAVE_STRLCPY /* Define to 1 if you have the header file. */ #undef HAVE_STROPTS_H -/* Define to 1 if you have the `strsignal' function. */ +/* Define to 1 if you have the 'strsignal' function. */ #undef HAVE_STRSIGNAL -/* Define to 1 if `pw_gecos' is a member of `struct passwd'. */ +/* Define to 1 if 'pw_gecos' is a member of 'struct passwd'. */ #undef HAVE_STRUCT_PASSWD_PW_GECOS -/* Define to 1 if `pw_passwd' is a member of `struct passwd'. */ +/* Define to 1 if 'pw_passwd' is a member of 'struct passwd'. */ #undef HAVE_STRUCT_PASSWD_PW_PASSWD -/* Define to 1 if `st_birthtime' is a member of `struct stat'. */ +/* Define to 1 if 'st_birthtime' is a member of 'struct stat'. */ #undef HAVE_STRUCT_STAT_ST_BIRTHTIME -/* Define to 1 if `st_blksize' is a member of `struct stat'. */ +/* Define to 1 if 'st_blksize' is a member of 'struct stat'. */ #undef HAVE_STRUCT_STAT_ST_BLKSIZE -/* Define to 1 if `st_blocks' is a member of `struct stat'. */ +/* Define to 1 if 'st_blocks' is a member of 'struct stat'. */ #undef HAVE_STRUCT_STAT_ST_BLOCKS -/* Define to 1 if `st_flags' is a member of `struct stat'. */ +/* Define to 1 if 'st_flags' is a member of 'struct stat'. */ #undef HAVE_STRUCT_STAT_ST_FLAGS -/* Define to 1 if `st_gen' is a member of `struct stat'. */ +/* Define to 1 if 'st_gen' is a member of 'struct stat'. */ #undef HAVE_STRUCT_STAT_ST_GEN -/* Define to 1 if `st_rdev' is a member of `struct stat'. */ +/* Define to 1 if 'st_rdev' is a member of 'struct stat'. */ #undef HAVE_STRUCT_STAT_ST_RDEV -/* Define to 1 if `tm_zone' is a member of `struct tm'. */ +/* Define to 1 if 'tm_zone' is a member of 'struct tm'. */ #undef HAVE_STRUCT_TM_TM_ZONE /* Define if you have the 'symlink' function. */ #undef HAVE_SYMLINK -/* Define to 1 if you have the `symlinkat' function. */ +/* Define to 1 if you have the 'symlinkat' function. */ #undef HAVE_SYMLINKAT -/* Define to 1 if you have the `sync' function. */ +/* Define to 1 if you have the 'sync' function. */ #undef HAVE_SYNC -/* Define to 1 if you have the `sysconf' function. */ +/* Define to 1 if you have the 'sysconf' function. */ #undef HAVE_SYSCONF /* Define to 1 if you have the header file. */ @@ -1297,7 +1335,7 @@ /* Define to 1 if you have the header file. */ #undef HAVE_SYSLOG_H -/* Define to 1 if you have the `system' function. */ +/* Define to 1 if you have the 'system' function. */ #undef HAVE_SYSTEM /* Define to 1 if you have the header file. */ @@ -1312,7 +1350,7 @@ /* Define to 1 if you have the header file. */ #undef HAVE_SYS_DEVPOLL_H -/* Define to 1 if you have the header file, and it defines `DIR'. +/* Define to 1 if you have the header file, and it defines 'DIR'. */ #undef HAVE_SYS_DIR_H @@ -1355,13 +1393,16 @@ /* Define to 1 if you have the header file. */ #undef HAVE_SYS_MODEM_H -/* Define to 1 if you have the header file, and it defines `DIR'. +/* Define to 1 if you have the header file, and it defines 'DIR'. */ #undef HAVE_SYS_NDIR_H /* Define to 1 if you have the header file. */ #undef HAVE_SYS_PARAM_H +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_PIDFD_H + /* Define to 1 if you have the header file. */ #undef HAVE_SYS_POLL_H @@ -1428,13 +1469,13 @@ /* Define to 1 if you have the header file. */ #undef HAVE_SYS_XATTR_H -/* Define to 1 if you have the `tcgetpgrp' function. */ +/* Define to 1 if you have the 'tcgetpgrp' function. */ #undef HAVE_TCGETPGRP -/* Define to 1 if you have the `tcsetpgrp' function. */ +/* Define to 1 if you have the 'tcsetpgrp' function. */ #undef HAVE_TCSETPGRP -/* Define to 1 if you have the `tempnam' function. */ +/* Define to 1 if you have the 'tempnam' function. */ #undef HAVE_TEMPNAM /* Define to 1 if you have the header file. */ @@ -1443,54 +1484,54 @@ /* Define to 1 if you have the header file. */ #undef HAVE_TERM_H -/* Define to 1 if you have the `timegm' function. */ +/* Define to 1 if you have the 'timegm' function. */ #undef HAVE_TIMEGM /* Define if you have the 'timerfd_create' function. */ #undef HAVE_TIMERFD_CREATE -/* Define to 1 if you have the `times' function. */ +/* Define to 1 if you have the 'times' function. */ #undef HAVE_TIMES -/* Define to 1 if you have the `tmpfile' function. */ +/* Define to 1 if you have the 'tmpfile' function. */ #undef HAVE_TMPFILE -/* Define to 1 if you have the `tmpnam' function. */ +/* Define to 1 if you have the 'tmpnam' function. */ #undef HAVE_TMPNAM -/* Define to 1 if you have the `tmpnam_r' function. */ +/* Define to 1 if you have the 'tmpnam_r' function. */ #undef HAVE_TMPNAM_R -/* Define to 1 if your `struct tm' has `tm_zone'. Deprecated, use - `HAVE_STRUCT_TM_TM_ZONE' instead. */ +/* Define to 1 if your 'struct tm' has 'tm_zone'. Deprecated, use + 'HAVE_STRUCT_TM_TM_ZONE' instead. */ #undef HAVE_TM_ZONE -/* Define to 1 if you have the `truncate' function. */ +/* Define to 1 if you have the 'truncate' function. */ #undef HAVE_TRUNCATE -/* Define to 1 if you have the `ttyname' function. */ -#undef HAVE_TTYNAME +/* Define to 1 if you have the 'ttyname_r' function. */ +#undef HAVE_TTYNAME_R -/* Define to 1 if you don't have `tm_zone' but do have the external array - `tzname'. */ +/* Define to 1 if you don't have 'tm_zone' but do have the external array + 'tzname'. */ #undef HAVE_TZNAME -/* Define to 1 if you have the `umask' function. */ +/* Define to 1 if you have the 'umask' function. */ #undef HAVE_UMASK -/* Define to 1 if you have the `uname' function. */ +/* Define to 1 if you have the 'uname' function. */ #undef HAVE_UNAME /* Define to 1 if you have the header file. */ #undef HAVE_UNISTD_H -/* Define to 1 if you have the `unlinkat' function. */ +/* Define to 1 if you have the 'unlinkat' function. */ #undef HAVE_UNLINKAT -/* Define to 1 if you have the `unlockpt' function. */ +/* Define to 1 if you have the 'unlockpt' function. */ #undef HAVE_UNLOCKPT -/* Define to 1 if you have the `unshare' function. */ +/* Define to 1 if you have the 'unshare' function. */ #undef HAVE_UNSHARE /* Define if you have a useable wchar_t type defined in wchar.h; useable means @@ -1501,10 +1542,10 @@ /* Define to 1 if you have the header file. */ #undef HAVE_UTIL_H -/* Define to 1 if you have the `utimensat' function. */ +/* Define to 1 if you have the 'utimensat' function. */ #undef HAVE_UTIMENSAT -/* Define to 1 if you have the `utimes' function. */ +/* Define to 1 if you have the 'utimes' function. */ #undef HAVE_UTIMES /* Define to 1 if you have the header file. */ @@ -1513,10 +1554,10 @@ /* Define to 1 if you have the header file. */ #undef HAVE_UTMP_H -/* Define to 1 if you have the `uuid_create' function. */ +/* Define to 1 if you have the 'uuid_create' function. */ #undef HAVE_UUID_CREATE -/* Define to 1 if you have the `uuid_enc_be' function. */ +/* Define to 1 if you have the 'uuid_enc_be' function. */ #undef HAVE_UUID_ENC_BE /* Define if uuid_generate_time_safe() exists. */ @@ -1528,44 +1569,44 @@ /* Define to 1 if you have the header file. */ #undef HAVE_UUID_UUID_H -/* Define to 1 if you have the `vfork' function. */ +/* Define to 1 if you have the 'vfork' function. */ #undef HAVE_VFORK -/* Define to 1 if you have the `wait' function. */ +/* Define to 1 if you have the 'wait' function. */ #undef HAVE_WAIT -/* Define to 1 if you have the `wait3' function. */ +/* Define to 1 if you have the 'wait3' function. */ #undef HAVE_WAIT3 -/* Define to 1 if you have the `wait4' function. */ +/* Define to 1 if you have the 'wait4' function. */ #undef HAVE_WAIT4 -/* Define to 1 if you have the `waitid' function. */ +/* Define to 1 if you have the 'waitid' function. */ #undef HAVE_WAITID -/* Define to 1 if you have the `waitpid' function. */ +/* Define to 1 if you have the 'waitpid' function. */ #undef HAVE_WAITPID /* Define if the compiler provides a wchar.h header file. */ #undef HAVE_WCHAR_H -/* Define to 1 if you have the `wcscoll' function. */ +/* Define to 1 if you have the 'wcscoll' function. */ #undef HAVE_WCSCOLL -/* Define to 1 if you have the `wcsftime' function. */ +/* Define to 1 if you have the 'wcsftime' function. */ #undef HAVE_WCSFTIME -/* Define to 1 if you have the `wcsxfrm' function. */ +/* Define to 1 if you have the 'wcsxfrm' function. */ #undef HAVE_WCSXFRM -/* Define to 1 if you have the `wmemcmp' function. */ +/* Define to 1 if you have the 'wmemcmp' function. */ #undef HAVE_WMEMCMP /* Define if tzset() actually switches the local timezone in a meaningful way. */ #undef HAVE_WORKING_TZSET -/* Define to 1 if you have the `writev' function. */ +/* Define to 1 if you have the 'writev' function. */ #undef HAVE_WRITEV /* Define if the zlib library has inflateCopy */ @@ -1574,17 +1615,17 @@ /* Define to 1 if you have the header file. */ #undef HAVE_ZLIB_H -/* Define to 1 if you have the `_getpty' function. */ +/* Define to 1 if you have the '_getpty' function. */ #undef HAVE__GETPTY -/* Define to 1 if the system has the type `__uint128_t'. */ +/* Define to 1 if the system has the type '__uint128_t'. */ #undef HAVE___UINT128_T -/* Define to 1 if `major', `minor', and `makedev' are declared in . +/* Define to 1 if 'major', 'minor', and 'makedev' are declared in . */ #undef MAJOR_IN_MKDEV -/* Define to 1 if `major', `minor', and `makedev' are declared in +/* Define to 1 if 'major', 'minor', and 'makedev' are declared in . */ #undef MAJOR_IN_SYSMACROS @@ -1652,6 +1693,9 @@ /* Defined if Python is built as a shared library. */ #undef Py_ENABLE_SHARED +/* Defined if _Complex C type can be used with libffi. */ +#undef Py_FFI_SUPPORT_C_COMPLEX + /* Define if you want to disable the GIL */ #undef Py_GIL_DISABLED @@ -1659,6 +1703,15 @@ SipHash13: 3, externally defined: 0 */ #undef Py_HASH_ALGORITHM +/* Defined if _Complex C type is available. */ +#undef Py_HAVE_C_COMPLEX + +/* Define if year with century should be normalized for strftime. */ +#undef Py_NORMALIZE_CENTURY + +/* Define if rl_startup_hook takes arguments */ +#undef Py_RL_STARTUP_HOOK_TAKES_ARGS + /* Define if you want to enable internal statistics gathering. */ #undef Py_STATS @@ -1677,58 +1730,58 @@ /* Define if i>>j for signed int i does not extend the sign bit when i < 0 */ #undef SIGNED_RIGHT_SHIFT_ZERO_FILLS -/* The size of `double', as computed by sizeof. */ +/* The size of 'double', as computed by sizeof. */ #undef SIZEOF_DOUBLE -/* The size of `float', as computed by sizeof. */ +/* The size of 'float', as computed by sizeof. */ #undef SIZEOF_FLOAT -/* The size of `fpos_t', as computed by sizeof. */ +/* The size of 'fpos_t', as computed by sizeof. */ #undef SIZEOF_FPOS_T -/* The size of `int', as computed by sizeof. */ +/* The size of 'int', as computed by sizeof. */ #undef SIZEOF_INT -/* The size of `long', as computed by sizeof. */ +/* The size of 'long', as computed by sizeof. */ #undef SIZEOF_LONG -/* The size of `long double', as computed by sizeof. */ +/* The size of 'long double', as computed by sizeof. */ #undef SIZEOF_LONG_DOUBLE -/* The size of `long long', as computed by sizeof. */ +/* The size of 'long long', as computed by sizeof. */ #undef SIZEOF_LONG_LONG -/* The size of `off_t', as computed by sizeof. */ +/* The size of 'off_t', as computed by sizeof. */ #undef SIZEOF_OFF_T -/* The size of `pid_t', as computed by sizeof. */ +/* The size of 'pid_t', as computed by sizeof. */ #undef SIZEOF_PID_T -/* The size of `pthread_key_t', as computed by sizeof. */ +/* The size of 'pthread_key_t', as computed by sizeof. */ #undef SIZEOF_PTHREAD_KEY_T -/* The size of `pthread_t', as computed by sizeof. */ +/* The size of 'pthread_t', as computed by sizeof. */ #undef SIZEOF_PTHREAD_T -/* The size of `short', as computed by sizeof. */ +/* The size of 'short', as computed by sizeof. */ #undef SIZEOF_SHORT -/* The size of `size_t', as computed by sizeof. */ +/* The size of 'size_t', as computed by sizeof. */ #undef SIZEOF_SIZE_T -/* The size of `time_t', as computed by sizeof. */ +/* The size of 'time_t', as computed by sizeof. */ #undef SIZEOF_TIME_T -/* The size of `uintptr_t', as computed by sizeof. */ +/* The size of 'uintptr_t', as computed by sizeof. */ #undef SIZEOF_UINTPTR_T -/* The size of `void *', as computed by sizeof. */ +/* The size of 'void *', as computed by sizeof. */ #undef SIZEOF_VOID_P -/* The size of `wchar_t', as computed by sizeof. */ +/* The size of 'wchar_t', as computed by sizeof. */ #undef SIZEOF_WCHAR_T -/* The size of `_Bool', as computed by sizeof. */ +/* The size of '_Bool', as computed by sizeof. */ #undef SIZEOF__BOOL /* Define to 1 if you have the ANSI C header files. */ @@ -1744,13 +1797,13 @@ /* Library needed by timemodule.c: librt may be needed for clock_gettime() */ #undef TIMEMODULE_LIB -/* Define to 1 if your declares `struct tm'. */ +/* Define to 1 if your declares 'struct tm'. */ #undef TM_IN_SYS_TIME /* Define if you want to use computed gotos in ceval.c. */ #undef USE_COMPUTED_GOTOS -/* Enable extensions on AIX 3, Interix. */ +/* Enable extensions on AIX, Interix, z/OS. */ #ifndef _ALL_SOURCE # undef _ALL_SOURCE #endif @@ -1811,11 +1864,15 @@ #ifndef __STDC_WANT_IEC_60559_DFP_EXT__ # undef __STDC_WANT_IEC_60559_DFP_EXT__ #endif +/* Enable extensions specified by C23 Annex F. */ +#ifndef __STDC_WANT_IEC_60559_EXT__ +# undef __STDC_WANT_IEC_60559_EXT__ +#endif /* Enable extensions specified by ISO/IEC TS 18661-4:2015. */ #ifndef __STDC_WANT_IEC_60559_FUNCS_EXT__ # undef __STDC_WANT_IEC_60559_FUNCS_EXT__ #endif -/* Enable extensions specified by ISO/IEC TS 18661-3:2015. */ +/* Enable extensions specified by C23 Annex H and ISO/IEC TS 18661-3:2015. */ #ifndef __STDC_WANT_IEC_60559_TYPES_EXT__ # undef __STDC_WANT_IEC_60559_TYPES_EXT__ #endif @@ -1859,9 +1916,6 @@ /* Define to build the readline module against libedit. */ #undef WITH_EDITLINE -/* Define if you want to compile in object freelists optimization */ -#undef WITH_FREELISTS - /* Define to 1 if libintl is needed for locale functions. */ #undef WITH_LIBINTL @@ -1923,6 +1977,9 @@ /* framework name */ #undef _PYTHONFRAMEWORK +/* Maximum length in bytes of a thread name */ +#undef _PYTHREAD_NAME_MAXLEN + /* Define to force use of thread-safe errno, h_errno, and other functions */ #undef _REENTRANT @@ -1947,16 +2004,16 @@ /* Define to 'long' if does not define clock_t. */ #undef clock_t -/* Define to empty if `const' does not conform to ANSI C. */ +/* Define to empty if 'const' does not conform to ANSI C. */ #undef const -/* Define to `int' if doesn't define. */ +/* Define as 'int' if doesn't define. */ #undef gid_t -/* Define to `int' if does not define. */ +/* Define to 'int' if does not define. */ #undef mode_t -/* Define to `long int' if does not define. */ +/* Define to 'long int' if does not define. */ #undef off_t /* Define as a signed integer type capable of holding a process identifier. */ @@ -1965,13 +2022,13 @@ /* Define to empty if the keyword does not work. */ #undef signed -/* Define to `unsigned int' if does not define. */ +/* Define as 'unsigned int' if doesn't define. */ #undef size_t /* Define to 'int' if does not define. */ #undef socklen_t -/* Define to `int' if doesn't define. */ +/* Define as 'int' if doesn't define. */ #undef uid_t